6527 files changed, 338112 insertions, 113483 deletions

diff --git a/.gitignore b/.gitignore
index 265959544978..5da004814678 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,6 +37,8 @@
 *.o
 *.o.*
 *.patch
+*.rmeta
+*.rsi
 *.s
 *.so
 *.so.dbg
@@ -97,6 +99,7 @@ modules.order
 !.gitattributes
 !.gitignore
 !.mailmap
+!.rustfmt.toml
 
 #
 # Generated include files
@@ -162,3 +165,6 @@ x509.genkey
 
 # Documentation toolchain
 sphinx_*/
+
+# Rust analyzer configuration
+/rust-project.json
diff --git a/.mailmap b/.mailmap
index d175777af078..191778125ef1 100644
--- a/.mailmap
+++ b/.mailmap
@@ -71,6 +71,9 @@ Ben M Cahill <ben.m.cahill@intel.com>
 Ben Widawsky <bwidawsk@kernel.org> <ben@bwidawsk.net>
 Ben Widawsky <bwidawsk@kernel.org> <ben.widawsky@intel.com>
 Ben Widawsky <bwidawsk@kernel.org> <benjamin.widawsky@intel.com>
+Bjorn Andersson <andersson@kernel.org> <bjorn@kryo.se>
+Bjorn Andersson <andersson@kernel.org> <bjorn.andersson@linaro.org>
+Bjorn Andersson <andersson@kernel.org> <bjorn.andersson@sonymobile.com>
 Björn Steinbrink <B.Steinbrink@gmx.de>
 Björn Töpel <bjorn@kernel.org> <bjorn.topel@gmail.com>
 Björn Töpel <bjorn@kernel.org> <bjorn.topel@intel.com>
diff --git a/.rustfmt.toml b/.rustfmt.toml
new file mode 100644
index 000000000000..3de5cc497465
--- /dev/null
+++ b/.rustfmt.toml
@@ -0,0 +1,12 @@
+edition = "2021"
+newline_style = "Unix"
+
+# Unstable options that help catching some mistakes in formatting and that we may want to enable
+# when they become stable.
+#
+# They are kept here since they are useful to run from time to time.
+#format_code_in_doc_comments = true
+#reorder_impl_items = true
+#comment_width = 100
+#wrap_comments = true
+#normalize_comments = true
diff --git a/Documentation/ABI/testing/debugfs-cros-ec b/Documentation/ABI/testing/debugfs-cros-ec
index 1fe0add99a2a..9a040c6f5e03 100644
--- a/Documentation/ABI/testing/debugfs-cros-ec
+++ b/Documentation/ABI/testing/debugfs-cros-ec
@@ -54,3 +54,25 @@ Description:
 		this feature.
 
 		Output will be in the format: "0x%08x\n".
+
+What:		/sys/kernel/debug/<cros-ec-device>/suspend_timeout_ms
+Date:		August 2022
+KernelVersion:	6.1
+Description:
+		Some ECs have a feature where they will track transitions of
+		a hardware-controlled sleep line, such as Intel's SLP_S0 line,
+		in order to detect cases where a system failed to go into deep
+		sleep states. The suspend_timeout_ms file controls the amount of
+		time in milliseconds the EC will wait before declaring a sleep
+		timeout event and attempting to wake the system.
+
+		Supply 0 to use the default value coded into EC firmware. Supply
+		65535 (EC_HOST_SLEEP_TIMEOUT_INFINITE) to disable the EC sleep
+		failure detection mechanism. Values in between 0 and 65535
+		indicate the number of milliseconds the EC should wait after a
+		sleep transition before declaring a timeout. This includes both
+		the duration after a sleep command was received but before the
+		hardware line changed, as well as the duration between when the
+		hardware line changed and the kernel sent an EC resume command.
+
+		Output will be in the format: "%u\n".
diff --git a/Documentation/ABI/testing/sysfs-amd-pmc b/Documentation/ABI/testing/sysfs-amd-pmc
new file mode 100644
index 000000000000..c421b72844f1
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-amd-pmc
@@ -0,0 +1,13 @@
+What:		/sys/bus/platform/drivers/amd_pmc/*/smu_fw_version
+Date:		October 2022
+Contact:	Mario Limonciello <mario.limonciello@amd.com>
+Description:	Reading this file reports the version of the firmware loaded to
+		System Management Unit (SMU) contained in AMD CPUs and
+		APUs.
+
+What:		/sys/bus/platform/drivers/amd_pmc/*/smu_program
+Date:		October 2022
+Contact:	Mario Limonciello <mario.limonciello@amd.com>
+Description:	Reading this file reports the program corresponding to the SMU
+		firmware version.  The program field is used to disambiguate two
+		APU/CPU models that can share the same firmware binary.
diff --git a/Documentation/ABI/testing/sysfs-amd-pmf b/Documentation/ABI/testing/sysfs-amd-pmf
new file mode 100644
index 000000000000..7fc0e1c2b76b
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-amd-pmf
@@ -0,0 +1,13 @@
+What:		/sys/devices/platform/*/cnqf_enable
+Date:		September 2022
+Contact:	Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+Description:	Reading this file tells if the AMD Platform Management(PMF)
+		Cool n Quiet Framework(CnQF) feature is enabled or not.
+
+		This feature is not enabled by default and gets only turned on
+		if OEM BIOS passes a "flag" to PMF ACPI function (index 11 or 12)
+		or in case the user writes "on".
+
+		To turn off CnQF user can write "off" to the sysfs node.
+		Note: Systems that support auto mode will not have this sysfs file
+		available.
diff --git a/Documentation/ABI/testing/sysfs-bus-bcma b/Documentation/ABI/testing/sysfs-bus-bcma
index 721b4aea3020..e93d3ddca844 100644
--- a/Documentation/ABI/testing/sysfs-bus-bcma
+++ b/Documentation/ABI/testing/sysfs-bus-bcma
@@ -3,7 +3,7 @@ Date:		May 2011
 KernelVersion:	3.0
 Contact:	Rafał Miłecki <zajec5@gmail.com>
 Description:
-		Each BCMA core has it's manufacturer id. See
+		Each BCMA core has its manufacturer id. See
 		include/linux/bcma/bcma.h for possible values.
 
 What:		/sys/bus/bcma/devices/.../id
diff --git a/Documentation/ABI/testing/sysfs-bus-fcoe b/Documentation/ABI/testing/sysfs-bus-fcoe
index 8fe787cc4ab7..5a4f2091ac37 100644
--- a/Documentation/ABI/testing/sysfs-bus-fcoe
+++ b/Documentation/ABI/testing/sysfs-bus-fcoe
@@ -31,7 +31,7 @@ Description:	'FCoE Controller' instances on the fcoe bus.
 		1) Write interface name to ctlr_create 2) Configure the FCoE
 		Controller (ctlr_X) 3) Enable the FCoE Controller to begin
 		discovery and login. The FCoE Controller is destroyed by
-		writing it's name, i.e. ctlr_X to the ctlr_delete file.
+		writing its name, i.e. ctlr_X to the ctlr_delete file.
 
 Attributes:
 
diff --git a/Documentation/ABI/testing/sysfs-bus-iio-proximity b/Documentation/ABI/testing/sysfs-bus-iio-proximity
index 3aac6dab8775..9b9d1cc9b703 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio-proximity
+++ b/Documentation/ABI/testing/sysfs-bus-iio-proximity
@@ -18,7 +18,7 @@ Description:
 		on the signal from which time of flight measurements are
 		taken.
 		The appropriate values to take is dependent on both the
-		sensor and it's operating environment:
+		sensor and its operating environment:
 		* as3935 (0-31 range)
 		18 = indoors (default)
 		14 = outdoors
diff --git a/Documentation/ABI/testing/sysfs-class-power b/Documentation/ABI/testing/sysfs-class-power
index a9ce63cfbe87..e434fc523291 100644
--- a/Documentation/ABI/testing/sysfs-class-power
+++ b/Documentation/ABI/testing/sysfs-class-power
@@ -364,7 +364,10 @@ Date:		April 2019
 Contact:	linux-pm@vger.kernel.org
 Description:
 		Represents a battery percentage level, above which charging will
-		stop.
+		stop. Not all hardware is capable of setting this to an arbitrary
+		percentage. Drivers will round written values to the nearest
+		supported value. Reading back the value will show the actual
+		threshold set by the driver.
 
 		Access: Read, Write
 
diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu
index 760c889b6cd1..f54867cadb0f 100644
--- a/Documentation/ABI/testing/sysfs-devices-system-cpu
+++ b/Documentation/ABI/testing/sysfs-devices-system-cpu
@@ -296,7 +296,7 @@ Description:	Processor frequency boosting control
 
 		This switch controls the boost setting for the whole system.
 		Boosting allows the CPU and the firmware to run at a frequency
-		beyond it's nominal limit.
+		beyond its nominal limit.
 
 		More details can be found in
 		Documentation/admin-guide/pm/cpufreq.rst
diff --git a/Documentation/ABI/testing/sysfs-platform-asus-wmi b/Documentation/ABI/testing/sysfs-platform-asus-wmi
index 04885738cf15..a77a004a1baa 100644
--- a/Documentation/ABI/testing/sysfs-platform-asus-wmi
+++ b/Documentation/ABI/testing/sysfs-platform-asus-wmi
@@ -57,3 +57,44 @@ Description:
 			* 0 - default,
 			* 1 - overboost,
 			* 2 - silent
+
+What:		/sys/devices/platform/<platform>/gpu_mux_mode
+Date:		Aug 2022
+KernelVersion:	6.1
+Contact:	"Luke Jones" <luke@ljones.dev>
+Description:
+		Switch the GPU hardware MUX mode. Laptops with this feature can
+		can be toggled to boot with only the dGPU (discrete mode) or in
+		standard Optimus/Hybrid mode. On switch a reboot is required:
+
+			* 0 - Discrete GPU,
+			* 1 - Optimus/Hybrid,
+
+What:		/sys/devices/platform/<platform>/dgpu_disable
+Date:		Aug 2022
+KernelVersion:	5.17
+Contact:	"Luke Jones" <luke@ljones.dev>
+Description:
+		Disable discrete GPU:
+			* 0 - Enable dGPU,
+			* 1 - Disable dGPU
+
+What:		/sys/devices/platform/<platform>/egpu_enable
+Date:		Aug 2022
+KernelVersion:	5.17
+Contact:	"Luke Jones" <luke@ljones.dev>
+Description:
+		Enable the external GPU paired with ROG X-Flow laptops.
+		Toggling this setting will also trigger ACPI to disable the dGPU:
+
+			* 0 - Disable,
+			* 1 - Enable
+
+What:		/sys/devices/platform/<platform>/panel_od
+Date:		Aug 2022
+KernelVersion:	5.17
+Contact:	"Luke Jones" <luke@ljones.dev>
+Description:
+		Enable an LCD response-time boost to reduce or remove ghosting:
+			* 0 - Disable,
+			* 1 - Enable
diff --git a/Documentation/ABI/testing/sysfs-platform-brcmstb-memc b/Documentation/ABI/testing/sysfs-platform-brcmstb-memc
new file mode 100644
index 000000000000..2f2b750ac2fd
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-platform-brcmstb-memc
@@ -0,0 +1,15 @@
+What:		/sys/bus/platform/devices/*/srpd
+Date:		July 2022
+KernelVersion:	5.21
+Contact:	Florian Fainelli <f.fainelli@gmail.com>
+Description:
+		Self Refresh Power Down (SRPD) inactivity timeout counted in
+		internal DDR controller clock cycles. Possible values range
+		from 0 (disable inactivity timeout) to 65535 (0xffff).
+
+What:		/sys/bus/platform/devices/*/frequency
+Date:		July 2022
+KernelVersion:	5.21
+Contact:	Florian Fainelli <f.fainelli@gmail.com>
+Description:
+		DDR PHY frequency in Hz.
diff --git a/Documentation/ABI/testing/sysfs-platform-chipidea-usb2 b/Documentation/ABI/testing/sysfs-platform-chipidea-usb2
index b0f4684a83fe..b9f7d924f28a 100644
--- a/Documentation/ABI/testing/sysfs-platform-chipidea-usb2
+++ b/Documentation/ABI/testing/sysfs-platform-chipidea-usb2
@@ -2,8 +2,8 @@ What:		/sys/bus/platform/devices/ci_hdrc.0/role
 Date:		Mar 2017
 Contact:	Peter Chen <peter.chen@nxp.com>
 Description:
-		It returns string "gadget" or "host" when read it, it indicates
-		current controller role.
+		When read, it returns string "gadget" or "host", indicating
+		the current controller role.
 
-		It will do role switch when write "gadget" or "host" to it.
+		It will do role switch when "gadget" or "host" is written to it.
 		Only controller at dual-role configuration supports writing.
diff --git a/Documentation/ABI/testing/sysfs-power b/Documentation/ABI/testing/sysfs-power
index 90ec4987074b..f99d433ff311 100644
--- a/Documentation/ABI/testing/sysfs-power
+++ b/Documentation/ABI/testing/sysfs-power
@@ -152,7 +152,7 @@ Description:
 		case further investigation is required to determine which
 		device is causing the problem.  Note that genuine RTC clock
 		values (such as when pm_trace has not been used), can still
-		match a device and output it's name here.
+		match a device and output its name here.
 
 What:		/sys/power/pm_async
 Date:		January 2009
diff --git a/Documentation/RCU/checklist.rst b/Documentation/RCU/checklist.rst
index 42cc5d891bd2..048c5bc1f813 100644
--- a/Documentation/RCU/checklist.rst
+++ b/Documentation/RCU/checklist.rst
@@ -66,8 +66,13 @@ over a rather long period of time, but improvements are always welcome!
 	As a rough rule of thumb, any dereference of an RCU-protected
 	pointer must be covered by rcu_read_lock(), rcu_read_lock_bh(),
 	rcu_read_lock_sched(), or by the appropriate update-side lock.
-	Disabling of preemption can serve as rcu_read_lock_sched(), but
-	is less readable and prevents lockdep from detecting locking issues.
+	Explicit disabling of preemption (preempt_disable(), for example)
+	can serve as rcu_read_lock_sched(), but is less readable and
+	prevents lockdep from detecting locking issues.
+
+	Please not that you *cannot* rely on code known to be built
+	only in non-preemptible kernels.  Such code can and will break,
+	especially in kernels built with CONFIG_PREEMPT_COUNT=y.
 
 	Letting RCU-protected pointers "leak" out of an RCU read-side
 	critical section is every bit as bad as letting them leak out
@@ -185,6 +190,9 @@ over a rather long period of time, but improvements are always welcome!
 
 5.	If call_rcu() or call_srcu() is used, the callback function will
 	be called from softirq context.  In particular, it cannot block.
+	If you need the callback to block, run that code in a workqueue
+	handler scheduled from the callback.  The queue_rcu_work()
+	function does this for you in the case of call_rcu().
 
 6.	Since synchronize_rcu() can block, it cannot be called
 	from any sort of irq context.  The same rule applies
@@ -297,7 +305,8 @@ over a rather long period of time, but improvements are always welcome!
 		the machine.
 
 	d.	Periodically invoke synchronize_rcu(), permitting a limited
-		number of updates per grace period.
+		number of updates per grace period.  Better yet, periodically
+		invoke rcu_barrier() to wait for all outstanding callbacks.
 
 	The same cautions apply to call_srcu() and kfree_rcu().
 
@@ -477,6 +486,6 @@ over a rather long period of time, but improvements are always welcome!
 	So if you need to wait for both an RCU grace period and for
 	all pre-existing call_rcu() callbacks, you will need to execute
 	both rcu_barrier() and synchronize_rcu(), if necessary, using
-	something like workqueues to to execute them concurrently.
+	something like workqueues to execute them concurrently.
 
 	See rcubarrier.rst for more information.
diff --git a/Documentation/RCU/lockdep.rst b/Documentation/RCU/lockdep.rst
index cc860a0c296b..a94f55991a71 100644
--- a/Documentation/RCU/lockdep.rst
+++ b/Documentation/RCU/lockdep.rst
@@ -61,7 +61,7 @@ checking of rcu_dereference() primitives:
 	rcu_access_pointer(p):
 		Return the value of the pointer and omit all barriers,
 		but retain the compiler constraints that prevent duplicating
-		or coalescsing.  This is useful when when testing the
+		or coalescsing.  This is useful when testing the
 		value of the pointer itself, for example, against NULL.
 
 The rcu_dereference_check() check expression can be any boolean
diff --git a/Documentation/RCU/rcu_dereference.rst b/Documentation/RCU/rcu_dereference.rst
index 0b418a5b243c..81e828c8313b 100644
--- a/Documentation/RCU/rcu_dereference.rst
+++ b/Documentation/RCU/rcu_dereference.rst
@@ -128,10 +128,16 @@ Follow these rules to keep your RCU code working properly:
 		This sort of comparison occurs frequently when scanning
 		RCU-protected circular linked lists.
 
-		Note that if checks for being within an RCU read-side
-		critical section are not required and the pointer is never
-		dereferenced, rcu_access_pointer() should be used in place
-		of rcu_dereference().
+		Note that if the pointer comparison is done outside
+		of an RCU read-side critical section, and the pointer
+		is never dereferenced, rcu_access_pointer() should be
+		used in place of rcu_dereference().  In most cases,
+		it is best to avoid accidental dereferences by testing
+		the rcu_access_pointer() return value directly, without
+		assigning it to a variable.
+
+		Within an RCU read-side critical section, there is little
+		reason to use rcu_access_pointer().
 
 	-	The comparison is against a pointer that references memory
 		that was initialized "a long time ago."  The reason
diff --git a/Documentation/RCU/whatisRCU.rst b/Documentation/RCU/whatisRCU.rst
index 77ea260efd12..1c747ac3f2c8 100644
--- a/Documentation/RCU/whatisRCU.rst
+++ b/Documentation/RCU/whatisRCU.rst
@@ -6,13 +6,15 @@ What is RCU?  --  "Read, Copy, Update"
 Please note that the "What is RCU?" LWN series is an excellent place
 to start learning about RCU:
 
-| 1.	What is RCU, Fundamentally?  http://lwn.net/Articles/262464/
-| 2.	What is RCU? Part 2: Usage   http://lwn.net/Articles/263130/
-| 3.	RCU part 3: the RCU API      http://lwn.net/Articles/264090/
-| 4.	The RCU API, 2010 Edition    http://lwn.net/Articles/418853/
-| 	2010 Big API Table           http://lwn.net/Articles/419086/
-| 5.	The RCU API, 2014 Edition    http://lwn.net/Articles/609904/
-|	2014 Big API Table           http://lwn.net/Articles/609973/
+| 1.	What is RCU, Fundamentally?  https://lwn.net/Articles/262464/
+| 2.	What is RCU? Part 2: Usage   https://lwn.net/Articles/263130/
+| 3.	RCU part 3: the RCU API      https://lwn.net/Articles/264090/
+| 4.	The RCU API, 2010 Edition    https://lwn.net/Articles/418853/
+| 	2010 Big API Table           https://lwn.net/Articles/419086/
+| 5.	The RCU API, 2014 Edition    https://lwn.net/Articles/609904/
+|	2014 Big API Table           https://lwn.net/Articles/609973/
+| 6.	The RCU API, 2019 Edition    https://lwn.net/Articles/777036/
+|	2019 Big API Table           https://lwn.net/Articles/777165/
 
 
 What is RCU?
@@ -915,13 +917,18 @@ which an RCU reference is held include:
 The understanding that RCU provides a reference that only prevents a
 change of type is particularly visible with objects allocated from a
 slab cache marked ``SLAB_TYPESAFE_BY_RCU``.  RCU operations may yield a
-reference to an object from such a cache that has been concurrently
-freed and the memory reallocated to a completely different object,
-though of the same type.  In this case RCU doesn't even protect the
-identity of the object from changing, only its type.  So the object
-found may not be the one expected, but it will be one where it is safe
-to take a reference or spinlock and then confirm that the identity
-matches the expectations.
+reference to an object from such a cache that has been concurrently freed
+and the memory reallocated to a completely different object, though of
+the same type.  In this case RCU doesn't even protect the identity of the
+object from changing, only its type.  So the object found may not be the
+one expected, but it will be one where it is safe to take a reference
+(and then potentially acquiring a spinlock), allowing subsequent code
+to check whether the identity matches expectations.  It is tempting
+to simply acquire the spinlock without first taking the reference, but
+unfortunately any spinlock in a ``SLAB_TYPESAFE_BY_RCU`` object must be
+initialized after each and every call to kmem_cache_alloc(), which renders
+reference-free spinlock acquisition completely unsafe.  Therefore, when
+using ``SLAB_TYPESAFE_BY_RCU``, make proper use of a reference counter.
 
 With traditional reference counting -- such as that implemented by the
 kref library in Linux -- there is typically code that runs when the last
@@ -1057,14 +1064,20 @@ SRCU: Initialization/cleanup::
 	init_srcu_struct
 	cleanup_srcu_struct
 
-All: lockdep-checked RCU-protected pointer access::
+All: lockdep-checked RCU utility APIs::
 
-	rcu_access_pointer
-	rcu_dereference_raw
 	RCU_LOCKDEP_WARN
 	rcu_sleep_check
 	RCU_NONIDLE
 
+All: Unchecked RCU-protected pointer access::
+
+	rcu_dereference_raw
+
+All: Unchecked RCU-protected pointer access with dereferencing prohibited::
+
+	rcu_access_pointer
+
 See the comment headers in the source code (or the docbook generated
 from them) for more information.
 
diff --git a/Documentation/admin-guide/README.rst b/Documentation/admin-guide/README.rst
index 9eb6b9042f75..9a969c0157f1 100644
--- a/Documentation/admin-guide/README.rst
+++ b/Documentation/admin-guide/README.rst
@@ -262,8 +262,6 @@ Compiling the kernel
  - Make sure you have at least gcc 5.1 available.
    For more information, refer to :ref:`Documentation/process/changes.rst <changes>`.
 
-   Please note that you can still run a.out user programs with this kernel.
-
  - Do a ``make`` to create a compressed kernel image. It is also
    possible to do ``make install`` if you have lilo installed to suit the
    kernel makefiles, but you may want to check your particular lilo setup first.
@@ -332,85 +330,10 @@ Compiling the kernel
 If something goes wrong
 -----------------------
 
- - If you have problems that seem to be due to kernel bugs, please check
-   the file MAINTAINERS to see if there is a particular person associated
-   with the part of the kernel that you are having trouble with. If there
-   isn't anyone listed there, then the second best thing is to mail
-   them to me (torvalds@linux-foundation.org), and possibly to any other
-   relevant mailing-list or to the newsgroup.
-
- - In all bug-reports, *please* tell what kernel you are talking about,
-   how to duplicate the problem, and what your setup is (use your common
-   sense).  If the problem is new, tell me so, and if the problem is
-   old, please try to tell me when you first noticed it.
-
- - If the bug results in a message like::
-
-     unable to handle kernel paging request at address C0000010
-     Oops: 0002
-     EIP:   0010:XXXXXXXX
-     eax: xxxxxxxx   ebx: xxxxxxxx   ecx: xxxxxxxx   edx: xxxxxxxx
-     esi: xxxxxxxx   edi: xxxxxxxx   ebp: xxxxxxxx
-     ds: xxxx  es: xxxx  fs: xxxx  gs: xxxx
-     Pid: xx, process nr: xx
-     xx xx xx xx xx xx xx xx xx xx
-
-   or similar kernel debugging information on your screen or in your
-   system log, please duplicate it *exactly*.  The dump may look
-   incomprehensible to you, but it does contain information that may
-   help debugging the problem.  The text above the dump is also
-   important: it tells something about why the kernel dumped code (in
-   the above example, it's due to a bad kernel pointer). More information
-   on making sense of the dump is in Documentation/admin-guide/bug-hunting.rst
-
- - If you compiled the kernel with CONFIG_KALLSYMS you can send the dump
-   as is, otherwise you will have to use the ``ksymoops`` program to make
-   sense of the dump (but compiling with CONFIG_KALLSYMS is usually preferred).
-   This utility can be downloaded from
-   https://www.kernel.org/pub/linux/utils/kernel/ksymoops/ .
-   Alternatively, you can do the dump lookup by hand:
-
- - In debugging dumps like the above, it helps enormously if you can
-   look up what the EIP value means.  The hex value as such doesn't help
-   me or anybody else very much: it will depend on your particular
-   kernel setup.  What you should do is take the hex value from the EIP
-   line (ignore the ``0010:``), and look it up in the kernel namelist to
-   see which kernel function contains the offending address.
-
-   To find out the kernel function name, you'll need to find the system
-   binary associated with the kernel that exhibited the symptom.  This is
-   the file 'linux/vmlinux'.  To extract the namelist and match it against
-   the EIP from the kernel crash, do::
-
-     nm vmlinux | sort | less
-
-   This will give you a list of kernel addresses sorted in ascending
-   order, from which it is simple to find the function that contains the
-   offending address.  Note that the address given by the kernel
-   debugging messages will not necessarily match exactly with the
-   function addresses (in fact, that is very unlikely), so you can't
-   just 'grep' the list: the list will, however, give you the starting
-   point of each kernel function, so by looking for the function that
-   has a starting address lower than the one you are searching for but
-   is followed by a function with a higher address you will find the one
-   you want.  In fact, it may be a good idea to include a bit of
-   "context" in your problem report, giving a few lines around the
-   interesting one.
-
-   If you for some reason cannot do the above (you have a pre-compiled
-   kernel image or similar), telling me as much about your setup as
-   possible will help.  Please read
-   'Documentation/admin-guide/reporting-issues.rst' for details.
-
- - Alternatively, you can use gdb on a running kernel. (read-only; i.e. you
-   cannot change values or set break points.) To do this, first compile the
-   kernel with -g; edit arch/x86/Makefile appropriately, then do a ``make
-   clean``. You'll also need to enable CONFIG_PROC_FS (via ``make config``).
-
-   After you've rebooted with the new kernel, do ``gdb vmlinux /proc/kcore``.
-   You can now use all the usual gdb commands. The command to look up the
-   point where your system crashed is ``l *0xXXXXXXXX``. (Replace the XXXes
-   with the EIP value.)
-
-   gdb'ing a non-running kernel currently fails because ``gdb`` (wrongly)
-   disregards the starting offset for which the kernel is compiled.
+If you have problems that seem to be due to kernel bugs, please follow the
+instructions at 'Documentation/admin-guide/reporting-issues.rst'.
+
+Hints on understanding kernel bug reports are in
+'Documentation/admin-guide/bug-hunting.rst'. More on debugging the kernel
+with gdb is in 'Documentation/dev-tools/gdb-kernel-debugging.rst' and
+'Documentation/dev-tools/kgdb.rst'.
diff --git a/Documentation/admin-guide/acpi/dsdt-override.rst b/Documentation/admin-guide/acpi/dsdt-override.rst
deleted file mode 100644
index 50bd7f194bf4..000000000000
--- a/Documentation/admin-guide/acpi/dsdt-override.rst
+++ /dev/null
@@ -1,13 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-===============
-Overriding DSDT
-===============
-
-Linux supports a method of overriding the BIOS DSDT:
-
-CONFIG_ACPI_CUSTOM_DSDT - builds the image into the kernel.
-
-When to use this method is described in detail on the
-Linux/ACPI home page:
-https://01.org/linux-acpi/documentation/overriding-dsdt
diff --git a/Documentation/admin-guide/hw-vuln/spectre.rst b/Documentation/admin-guide/hw-vuln/spectre.rst
index 2ce2a38cdd55..c4dcdb3d0d45 100644
--- a/Documentation/admin-guide/hw-vuln/spectre.rst
+++ b/Documentation/admin-guide/hw-vuln/spectre.rst
@@ -613,6 +613,7 @@ kernel command line.
                 eibrs                   enhanced IBRS
                 eibrs,retpoline         enhanced IBRS + Retpolines
                 eibrs,lfence            enhanced IBRS + LFENCE
+                ibrs                    use IBRS to protect kernel
 
 		Not specifying this option is equivalent to
 		spectre_v2=auto.
diff --git a/Documentation/admin-guide/kdump/vmcoreinfo.rst b/Documentation/admin-guide/kdump/vmcoreinfo.rst
index 8419019b6a88..6726f439958c 100644
--- a/Documentation/admin-guide/kdump/vmcoreinfo.rst
+++ b/Documentation/admin-guide/kdump/vmcoreinfo.rst
@@ -200,7 +200,7 @@ prb
 
 A pointer to the printk ringbuffer (struct printk_ringbuffer). This
 may be pointing to the static boot ringbuffer or the dynamically
-allocated ringbuffer, depending on when the the core dump occurred.
+allocated ringbuffer, depending on when the core dump occurred.
 Used by user-space tools to read the active kernel log buffer.
 
 printk_rb_static
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 426fa892d311..02ee2e2405e7 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -966,10 +966,6 @@
 
 	debugpat	[X86] Enable PAT debugging
 
-	decnet.addr=	[HW,NET]
-			Format: <area>[,<node>]
-			See also Documentation/networking/decnet.rst.
-
 	default_hugepagesz=
 			[HW] The size of the default HugeTLB page. This is
 			the size represented by the legacy /proc/ hugepages
@@ -2436,6 +2432,12 @@
 			0: force disabled
 			1: force enabled
 
+	kunit.enable=	[KUNIT] Enable executing KUnit tests. Requires
+			CONFIG_KUNIT to be set to be fully enabled. The
+			default value can be overridden via
+			KUNIT_DEFAULT_ENABLED.
+			Default is 1 (enabled)
+
 	kvm.ignore_msrs=[KVM] Ignore guest accesses to unhandled MSRs.
 			Default is 0 (don't ignore, but inject #GP)
 
@@ -3207,6 +3209,7 @@
 					       spectre_v2_user=off [X86]
 					       spec_store_bypass_disable=off [X86,PPC]
 					       ssbd=force-off [ARM64]
+					       nospectre_bhb [ARM64]
 					       l1tf=off [X86]
 					       mds=off [X86]
 					       tsx_async_abort=off [X86]
@@ -3613,7 +3616,7 @@
 
 	nohugeiomap	[KNL,X86,PPC,ARM64] Disable kernel huge I/O mappings.
 
-	nohugevmalloc	[PPC] Disable kernel huge vmalloc mappings.
+	nohugevmalloc	[KNL,X86,PPC,ARM64] Disable kernel huge vmalloc mappings.
 
 	nosmt		[KNL,S390] Disable symmetric multithreading (SMT).
 			Equivalent to smt=1.
@@ -3631,6 +3634,10 @@
 			vulnerability. System may allow data leaks with this
 			option.
 
+	nospectre_bhb	[ARM64] Disable all mitigations for Spectre-BHB (branch
+			history injection) vulnerability. System may allow data leaks
+			with this option.
+
 	nospec_store_bypass_disable
 			[HW] Disable all mitigations for the Speculative Store Bypass vulnerability
 
@@ -3805,6 +3812,10 @@
 
 	nox2apic	[X86-64,APIC] Do not enable x2APIC mode.
 
+			NOTE: this parameter will be ignored on systems with the
+			LEGACY_XAPIC_DISABLED bit set in the
+			IA32_XAPIC_DISABLE_STATUS MSR.
+
 	nps_mtm_hs_ctr=	[KNL,ARC]
 			This parameter sets the maximum duration, in
 			cycles, each HW thread of the CTOP can run
diff --git a/Documentation/admin-guide/mm/hugetlbpage.rst b/Documentation/admin-guide/mm/hugetlbpage.rst
index 8e2727dc18d4..19f27c0d92e0 100644
--- a/Documentation/admin-guide/mm/hugetlbpage.rst
+++ b/Documentation/admin-guide/mm/hugetlbpage.rst
@@ -65,7 +65,7 @@ HugePages_Surp
 	may be temporarily larger than the maximum number of surplus huge
 	pages when the system is under memory pressure.
 Hugepagesize
-	is the default hugepage size (in Kb).
+	is the default hugepage size (in kB).
 Hugetlb
         is the total amount of memory (in kB), consumed by huge
         pages of all sizes.
diff --git a/Documentation/admin-guide/perf/alibaba_pmu.rst b/Documentation/admin-guide/perf/alibaba_pmu.rst
new file mode 100644
index 000000000000..11de998bb480
--- /dev/null
+++ b/Documentation/admin-guide/perf/alibaba_pmu.rst
@@ -0,0 +1,100 @@
+=============================================================
+Alibaba's T-Head SoC Uncore Performance Monitoring Unit (PMU)
+=============================================================
+
+The Yitian 710, custom-built by Alibaba Group's chip development business,
+T-Head, implements uncore PMU for performance and functional debugging to
+facilitate system maintenance.
+
+DDR Sub-System Driveway (DRW) PMU Driver
+=========================================
+
+Yitian 710 employs eight DDR5/4 channels, four on each die. Each DDR5 channel
+is independent of others to service system memory requests. And one DDR5
+channel is split into two independent sub-channels. The DDR Sub-System Driveway
+implements separate PMUs for each sub-channel to monitor various performance
+metrics.
+
+The Driveway PMU devices are named as ali_drw_<sys_base_addr> with perf.
+For example, ali_drw_21000 and ali_drw_21080 are two PMU devices for two
+sub-channels of the same channel in die 0. And the PMU device of die 1 is
+prefixed with ali_drw_400XXXXX, e.g. ali_drw_40021000.
+
+Each sub-channel has 36 PMU counters in total, which is classified into
+four groups:
+
+- Group 0: PMU Cycle Counter. This group has one pair of counters
+  pmu_cycle_cnt_low and pmu_cycle_cnt_high, that is used as the cycle count
+  based on DDRC core clock.
+
+- Group 1: PMU Bandwidth Counters. This group has 8 counters that are used
+  to count the total access number of either the eight bank groups in a
+  selected rank, or four ranks separately in the first 4 counters. The base
+  transfer unit is 64B.
+
+- Group 2: PMU Retry Counters. This group has 10 counters, that intend to
+  count the total retry number of each type of uncorrectable error.
+
+- Group 3: PMU Common Counters. This group has 16 counters, that are used
+  to count the common events.
+
+For now, the Driveway PMU driver only uses counters in group 0 and group 3.
+
+The DDR Controller (DDRCTL) and DDR PHY combine to create a complete solution
+for connecting an SoC application bus to DDR memory devices. The DDRCTL
+receives transactions Host Interface (HIF) which is custom-defined by Synopsys.
+These transactions are queued internally and scheduled for access while
+satisfying the SDRAM protocol timing requirements, transaction priorities, and
+dependencies between the transactions. The DDRCTL in turn issues commands on
+the DDR PHY Interface (DFI) to the PHY module, which launches and captures data
+to and from the SDRAM. The driveway PMUs have hardware logic to gather
+statistics and performance logging signals on HIF, DFI, etc.
+
+By counting the READ, WRITE and RMW commands sent to the DDRC through the HIF
+interface, we could calculate the bandwidth. Example usage of counting memory
+data bandwidth::
+
+  perf stat \
+    -e ali_drw_21000/hif_wr/ \
+    -e ali_drw_21000/hif_rd/ \
+    -e ali_drw_21000/hif_rmw/ \
+    -e ali_drw_21000/cycle/ \
+    -e ali_drw_21080/hif_wr/ \
+    -e ali_drw_21080/hif_rd/ \
+    -e ali_drw_21080/hif_rmw/ \
+    -e ali_drw_21080/cycle/ \
+    -e ali_drw_23000/hif_wr/ \
+    -e ali_drw_23000/hif_rd/ \
+    -e ali_drw_23000/hif_rmw/ \
+    -e ali_drw_23000/cycle/ \
+    -e ali_drw_23080/hif_wr/ \
+    -e ali_drw_23080/hif_rd/ \
+    -e ali_drw_23080/hif_rmw/ \
+    -e ali_drw_23080/cycle/ \
+    -e ali_drw_25000/hif_wr/ \
+    -e ali_drw_25000/hif_rd/ \
+    -e ali_drw_25000/hif_rmw/ \
+    -e ali_drw_25000/cycle/ \
+    -e ali_drw_25080/hif_wr/ \
+    -e ali_drw_25080/hif_rd/ \
+    -e ali_drw_25080/hif_rmw/ \
+    -e ali_drw_25080/cycle/ \
+    -e ali_drw_27000/hif_wr/ \
+    -e ali_drw_27000/hif_rd/ \
+    -e ali_drw_27000/hif_rmw/ \
+    -e ali_drw_27000/cycle/ \
+    -e ali_drw_27080/hif_wr/ \
+    -e ali_drw_27080/hif_rd/ \
+    -e ali_drw_27080/hif_rmw/ \
+    -e ali_drw_27080/cycle/ -- sleep 10
+
+The average DRAM bandwidth can be calculated as follows:
+
+- Read Bandwidth =  perf_hif_rd * DDRC_WIDTH * DDRC_Freq / DDRC_Cycle
+- Write Bandwidth = (perf_hif_wr + perf_hif_rmw) * DDRC_WIDTH * DDRC_Freq / DDRC_Cycle
+
+Here, DDRC_WIDTH = 64 bytes.
+
+The current driver does not support sampling. So "perf record" is
+unsupported.  Also attach to a task is unsupported as the events are all
+uncore.
diff --git a/Documentation/admin-guide/perf/index.rst b/Documentation/admin-guide/perf/index.rst
index 9c9ece88ce53..793e1970bc05 100644
--- a/Documentation/admin-guide/perf/index.rst
+++ b/Documentation/admin-guide/perf/index.rst
@@ -18,3 +18,4 @@ Performance monitor support
    xgene-pmu
    arm_dsu_pmu
    thunderx2-pmu
+   alibaba_pmu
diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst
index 83b58eb4ab4d..8f3d30c5a0d8 100644
--- a/Documentation/admin-guide/pm/amd-pstate.rst
+++ b/Documentation/admin-guide/pm/amd-pstate.rst
@@ -182,6 +182,7 @@ to the ``struct sugov_cpu`` that the utilization update belongs to.
 Then, ``amd-pstate`` updates the desired performance according to the CPU
 scheduler assigned.
 
+.. _processor_support:
 
 Processor Support
 =======================
@@ -282,6 +283,8 @@ efficiency frequency management method on AMD processors.
 Kernel Module Options for ``amd-pstate``
 =========================================
 
+.. _shared_mem:
+
 ``shared_mem``
 Use a module param (shared_mem) to enable related processors manually with
 **amd_pstate.shared_mem=1**.
@@ -393,6 +396,76 @@ about part of the output. ::
  CPU_005     712          116384        39        49        166       0.7565  9645075 2214891 38431470  25.1   11.646       469         2.496         kworker/5:0-40
  CPU_006     712          116408        39        49        166       0.6769  8950227 1839034 37192089  24.06  11.272       470         2.496         kworker/6:0-1264
 
+Unit Tests for amd-pstate
+-------------------------
+
+``amd-pstate-ut`` is a test module for testing the ``amd-pstate`` driver.
+
+ * It can help all users to verify their processor support (SBIOS/Firmware or Hardware).
+
+ * Kernel can have a basic function test to avoid the kernel regression during the update.
+
+ * We can introduce more functional or performance tests to align the result together, it will benefit power and performance scale optimization.
+
+1. Test case decriptions
+
+        +---------+--------------------------------+------------------------------------------------------------------------------------+
+        | Index   | Functions                      | Description                                                                        |
+        +=========+================================+====================================================================================+
+        | 0       | amd_pstate_ut_acpi_cpc_valid   || Check whether the _CPC object is present in SBIOS.                                |
+        |         |                                ||                                                                                   |
+        |         |                                || The detail refer to `Processor Support <processor_support_>`_.                    |
+        +---------+--------------------------------+------------------------------------------------------------------------------------+
+        | 1       | amd_pstate_ut_check_enabled    || Check whether AMD P-State is enabled.                                             |
+        |         |                                ||                                                                                   |
+        |         |                                || AMD P-States and ACPI hardware P-States always can be supported in one processor. |
+        |         |                                | But AMD P-States has the higher priority and if it is enabled with                 |
+        |         |                                | :c:macro:`MSR_AMD_CPPC_ENABLE` or ``cppc_set_enable``, it will respond to the      |
+        |         |                                | request from AMD P-States.                                                         |
+        +---------+--------------------------------+------------------------------------------------------------------------------------+
+        | 2       | amd_pstate_ut_check_perf       || Check if the each performance values are reasonable.                              |
+        |         |                                || highest_perf >= nominal_perf > lowest_nonlinear_perf > lowest_perf > 0.           |
+        +---------+--------------------------------+------------------------------------------------------------------------------------+
+        | 3       | amd_pstate_ut_check_freq       || Check if the each frequency values and max freq when set support boost mode       |
+        |         |                                | are reasonable.                                                                    |
+        |         |                                || max_freq >= nominal_freq > lowest_nonlinear_freq > min_freq > 0                   |
+        |         |                                || If boost is not active but supported, this maximum frequency will be larger than  |
+        |         |                                | the one in ``cpuinfo``.                                                            |
+        +---------+--------------------------------+------------------------------------------------------------------------------------+
+
+#. How to execute the tests
+
+   We use test module in the kselftest frameworks to implement it.
+   We create amd-pstate-ut module and tie it into kselftest.(for
+   details refer to Linux Kernel Selftests [4]_).
+
+    1. Build
+
+        + open the :c:macro:`CONFIG_X86_AMD_PSTATE` configuration option.
+        + set the :c:macro:`CONFIG_X86_AMD_PSTATE_UT` configuration option to M.
+        + make project
+        + make selftest ::
+
+            $ cd linux
+            $ make -C tools/testing/selftests
+
+    #. Installation & Steps ::
+
+        $ make -C tools/testing/selftests install INSTALL_PATH=~/kselftest
+        $ sudo ./kselftest/run_kselftest.sh -c amd-pstate
+        TAP version 13
+        1..1
+        # selftests: amd-pstate: amd-pstate-ut.sh
+        # amd-pstate-ut: ok
+        ok 1 selftests: amd-pstate: amd-pstate-ut.sh
+
+    #. Results ::
+
+         $ dmesg | grep "amd_pstate_ut" | tee log.txt
+         [12977.570663] amd_pstate_ut: 1    amd_pstate_ut_acpi_cpc_valid  success!
+         [12977.570673] amd_pstate_ut: 2    amd_pstate_ut_check_enabled   success!
+         [12977.571207] amd_pstate_ut: 3    amd_pstate_ut_check_perf      success!
+         [12977.571212] amd_pstate_ut: 4    amd_pstate_ut_check_freq      success!
 
 Reference
 ===========
@@ -405,3 +478,6 @@ Reference
 
 .. [3] Processor Programming Reference (PPR) for AMD Family 19h Model 51h, Revision A1 Processors
        https://www.amd.com/system/files/TechDocs/56569-A1-PUB.zip
+
+.. [4] Linux Kernel Selftests,
+       https://www.kernel.org/doc/html/latest/dev-tools/kselftest.html
diff --git a/Documentation/admin-guide/sysctl/net.rst b/Documentation/admin-guide/sysctl/net.rst
index 60d44165fba7..6394f5dc2303 100644
--- a/Documentation/admin-guide/sysctl/net.rst
+++ b/Documentation/admin-guide/sysctl/net.rst
@@ -31,17 +31,18 @@ see only some of them, depending on your kernel's configuration.
 
 Table : Subdirectories in /proc/sys/net
 
- ========= =================== = ========== ==================
+ ========= =================== = ========== ===================
  Directory Content               Directory  Content
- ========= =================== = ========== ==================
- core      General parameter     appletalk  Appletalk protocol
- unix      Unix domain sockets   netrom     NET/ROM
- 802       E802 protocol         ax25       AX25
- ethernet  Ethernet protocol     rose       X.25 PLP layer
+ ========= =================== = ========== ===================
+ 802       E802 protocol         mptcp      Multipath TCP
+ appletalk Appletalk protocol    netfilter  Network Filter
+ ax25      AX25                  netrom     NET/ROM
+ bridge    Bridging              rose       X.25 PLP layer
+ core      General parameter     tipc       TIPC
+ ethernet  Ethernet protocol     unix       Unix domain sockets
  ipv4      IP version 4          x25        X.25 protocol
- bridge    Bridging              decnet     DEC net
- ipv6      IP version 6          tipc       TIPC
- ========= =================== = ========== ==================
+ ipv6      IP version 6
+ ========= =================== = ========== ===================
 
 1. /proc/sys/net/core - Network core options
 ============================================
@@ -101,6 +102,9 @@ Values:
 	- 1 - enable JIT hardening for unprivileged users only
 	- 2 - enable JIT hardening for all users
 
+where "privileged user" in this context means a process having
+CAP_BPF or CAP_SYS_ADMIN in the root user name space.
+
 bpf_jit_kallsyms
 ----------------
 
diff --git a/Documentation/admin-guide/tainted-kernels.rst b/Documentation/admin-guide/tainted-kernels.rst
index 7d80e8c307d1..92a8a07f5c43 100644
--- a/Documentation/admin-guide/tainted-kernels.rst
+++ b/Documentation/admin-guide/tainted-kernels.rst
@@ -134,6 +134,12 @@ More detailed explanation for tainting
        scsi/snic on something else than x86_64, scsi/ips on non
        x86/x86_64/itanium, have broken firmware settings for the
        irqchip/irq-gic on arm64 ...).
+     - x86/x86_64: Microcode late loading is dangerous and will result in
+       tainting the kernel. It requires that all CPUs rendezvous to make sure
+       the update happens when the system is as quiescent as possible. However,
+       a higher priority MCE/SMI/NMI can move control flow away from that
+       rendezvous and interrupt the update, which can be detrimental to the
+       machine.
 
  3)  ``R`` if a module was force unloaded by ``rmmod -f``, ``' '`` if all
      modules were unloaded normally.
diff --git a/Documentation/arm64/elf_hwcaps.rst b/Documentation/arm64/elf_hwcaps.rst
index 311021f2e560..bb34287c8e01 100644
--- a/Documentation/arm64/elf_hwcaps.rst
+++ b/Documentation/arm64/elf_hwcaps.rst
@@ -272,6 +272,9 @@ HWCAP2_WFXT
 HWCAP2_EBF16
     Functionality implied by ID_AA64ISAR1_EL1.BF16 == 0b0010.
 
+HWCAP2_SVE_EBF16
+    Functionality implied by ID_AA64ZFR0_EL1.BF16 == 0b0010.
+
 4. Unused AT_HWCAP bits
 -----------------------
 
diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst
index fda97b3fcf01..17d9fc5d14fb 100644
--- a/Documentation/arm64/silicon-errata.rst
+++ b/Documentation/arm64/silicon-errata.rst
@@ -110,6 +110,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A510     | #2441009        | ARM64_ERRATUM_2441009       |
 +----------------+-----------------+-----------------+-----------------------------+
+| ARM            | Cortex-A510     | #2658417        | ARM64_ERRATUM_2658417       |
++----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A710     | #2119858        | ARM64_ERRATUM_2119858       |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A710     | #2054223        | ARM64_ERRATUM_2054223       |
diff --git a/Documentation/arm64/sme.rst b/Documentation/arm64/sme.rst
index 937147f58cc5..16d2db4c2e2e 100644
--- a/Documentation/arm64/sme.rst
+++ b/Documentation/arm64/sme.rst
@@ -331,6 +331,9 @@ The regset data starts with struct user_za_header, containing:
   been read if a PTRACE_GETREGSET of NT_ARM_ZA were executed for each thread
   when the coredump was generated.
 
+* The NT_ARM_TLS note will be extended to two registers, the second register
+  will contain TPIDR2_EL0 on systems that support SME and will be read as
+  zero with writes ignored otherwise.
 
 9.  System runtime configuration
 --------------------------------
diff --git a/Documentation/arm64/sve.rst b/Documentation/arm64/sve.rst
index 93c2c2990584..f338ee2df46d 100644
--- a/Documentation/arm64/sve.rst
+++ b/Documentation/arm64/sve.rst
@@ -111,7 +111,7 @@ the SVE instruction set architecture.
 
 * On syscall, V0..V31 are preserved (as without SVE).  Thus, bits [127:0] of
   Z0..Z31 are preserved.  All other bits of Z0..Z31, and all of P0..P15 and FFR
-  become unspecified on return from a syscall.
+  become zero on return from a syscall.
 
 * The SVE registers are not used to pass arguments to or receive results from
   any syscall.
@@ -452,6 +452,24 @@ The regset data starts with struct user_sve_header, containing:
 * Modifying the system default vector length does not affect the vector length
   of any existing process or thread that does not make an execve() call.
 
+10.  Perf extensions
+--------------------------------
+
+* The arm64 specific DWARF standard [5] added the VG (Vector Granule) register
+  at index 46. This register is used for DWARF unwinding when variable length
+  SVE registers are pushed onto the stack.
+
+* Its value is equivalent to the current SVE vector length (VL) in bits divided
+  by 64.
+
+* The value is included in Perf samples in the regs[46] field if
+  PERF_SAMPLE_REGS_USER is set and the sample_regs_user mask has bit 46 set.
+
+* The value is the current value at the time the sample was taken, and it can
+  change over time.
+
+* If the system doesn't support SVE when perf_event_open is called with these
+  settings, the event will fail to open.
 
 Appendix A.  SVE programmer's model (informative)
 =================================================
@@ -593,3 +611,5 @@ References
     http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055c/IHI0055C_beta_aapcs64.pdf
     http://infocenter.arm.com/help/topic/com.arm.doc.subset.swdev.abi/index.html
     Procedure Call Standard for the ARM 64-bit Architecture (AArch64)
+
+[5] https://github.com/ARM-software/abi-aa/blob/main/aadwarf64/aadwarf64.rst
diff --git a/Documentation/bpf/clang-notes.rst b/Documentation/bpf/clang-notes.rst
new file mode 100644
index 000000000000..528feddf2db9
--- /dev/null
+++ b/Documentation/bpf/clang-notes.rst
@@ -0,0 +1,30 @@
+.. contents::
+.. sectnum::
+
+==========================
+Clang implementation notes
+==========================
+
+This document provides more details specific to the Clang/LLVM implementation of the eBPF instruction set.
+
+Versions
+========
+
+Clang defined "CPU" versions, where a CPU version of 3 corresponds to the current eBPF ISA.
+
+Clang can select the eBPF ISA version using ``-mcpu=v3`` for example to select version 3.
+
+Arithmetic instructions
+=======================
+
+For CPU versions prior to 3, Clang v7.0 and later can enable ``BPF_ALU`` support with
+``-Xclang -target-feature -Xclang +alu32``.  In CPU version 3, support is automatically included.
+
+Atomic operations
+=================
+
+Clang can generate atomic instructions by default when ``-mcpu=v3`` is
+enabled. If a lower version for ``-mcpu`` is set, the only atomic instruction
+Clang can generate is ``BPF_ADD`` *without* ``BPF_FETCH``. If you need to enable
+the atomics features, while keeping a lower ``-mcpu`` version, you can use
+``-Xclang -target-feature -Xclang +alu32``.
diff --git a/Documentation/bpf/index.rst b/Documentation/bpf/index.rst
index 1bc2c5c58bdb..1b50de1983ee 100644
--- a/Documentation/bpf/index.rst
+++ b/Documentation/bpf/index.rst
@@ -26,6 +26,8 @@ that goes into great technical depth about the BPF Architecture.
    classic_vs_extended.rst
    bpf_licensing
    test_debug
+   clang-notes
+   linux-notes
    other
 
 .. only::  subproject and html
diff --git a/Documentation/bpf/instruction-set.rst b/Documentation/bpf/instruction-set.rst
index 1b0e6711dec9..5d798437dad4 100644
--- a/Documentation/bpf/instruction-set.rst
+++ b/Documentation/bpf/instruction-set.rst
@@ -1,7 +1,12 @@
+.. contents::
+.. sectnum::
+
+========================================
+eBPF Instruction Set Specification, v1.0
+========================================
+
+This document specifies version 1.0 of the eBPF instruction set.
 
-====================
-eBPF Instruction Set
-====================
 
 Registers and calling convention
 ================================
@@ -11,10 +16,10 @@ all of which are 64-bits wide.
 
 The eBPF calling convention is defined as:
 
- * R0: return value from function calls, and exit value for eBPF programs
- * R1 - R5: arguments for function calls
- * R6 - R9: callee saved registers that function calls will preserve
- * R10: read-only frame pointer to access stack
+* R0: return value from function calls, and exit value for eBPF programs
+* R1 - R5: arguments for function calls
+* R6 - R9: callee saved registers that function calls will preserve
+* R10: read-only frame pointer to access stack
 
 R0 - R5 are scratch registers and eBPF programs needs to spill/fill them if
 necessary across calls.
@@ -24,17 +29,17 @@ Instruction encoding
 
 eBPF has two instruction encodings:
 
- * the basic instruction encoding, which uses 64 bits to encode an instruction
- * the wide instruction encoding, which appends a second 64-bit immediate value
-   (imm64) after the basic instruction for a total of 128 bits.
+* the basic instruction encoding, which uses 64 bits to encode an instruction
+* the wide instruction encoding, which appends a second 64-bit immediate value
+  (imm64) after the basic instruction for a total of 128 bits.
 
 The basic instruction encoding looks as follows:
 
- =============  =======  ===============  ====================  ============
- 32 bits (MSB)  16 bits  4 bits           4 bits                8 bits (LSB)
- =============  =======  ===============  ====================  ============
- immediate      offset   source register  destination register  opcode
- =============  =======  ===============  ====================  ============
+=============  =======  ===============  ====================  ============
+32 bits (MSB)  16 bits  4 bits           4 bits                8 bits (LSB)
+=============  =======  ===============  ====================  ============
+immediate      offset   source register  destination register  opcode
+=============  =======  ===============  ====================  ============
 
 Note that most instructions do not use all of the fields.
 Unused fields shall be cleared to zero.
@@ -44,30 +49,30 @@ Instruction classes
 
 The three LSB bits of the 'opcode' field store the instruction class:
 
-  =========  =====  ===============================
-  class      value  description
-  =========  =====  ===============================
-  BPF_LD     0x00   non-standard load operations
-  BPF_LDX    0x01   load into register operations
-  BPF_ST     0x02   store from immediate operations
-  BPF_STX    0x03   store from register operations
-  BPF_ALU    0x04   32-bit arithmetic operations
-  BPF_JMP    0x05   64-bit jump operations
-  BPF_JMP32  0x06   32-bit jump operations
-  BPF_ALU64  0x07   64-bit arithmetic operations
-  =========  =====  ===============================
+=========  =====  ===============================  ===================================
+class      value  description                      reference
+=========  =====  ===============================  ===================================
+BPF_LD     0x00   non-standard load operations     `Load and store instructions`_
+BPF_LDX    0x01   load into register operations    `Load and store instructions`_
+BPF_ST     0x02   store from immediate operations  `Load and store instructions`_
+BPF_STX    0x03   store from register operations   `Load and store instructions`_
+BPF_ALU    0x04   32-bit arithmetic operations     `Arithmetic and jump instructions`_
+BPF_JMP    0x05   64-bit jump operations           `Arithmetic and jump instructions`_
+BPF_JMP32  0x06   32-bit jump operations           `Arithmetic and jump instructions`_
+BPF_ALU64  0x07   64-bit arithmetic operations     `Arithmetic and jump instructions`_
+=========  =====  ===============================  ===================================
 
 Arithmetic and jump instructions
 ================================
 
-For arithmetic and jump instructions (BPF_ALU, BPF_ALU64, BPF_JMP and
-BPF_JMP32), the 8-bit 'opcode' field is divided into three parts:
+For arithmetic and jump instructions (``BPF_ALU``, ``BPF_ALU64``, ``BPF_JMP`` and
+``BPF_JMP32``), the 8-bit 'opcode' field is divided into three parts:
 
-  ==============  ======  =================
-  4 bits (MSB)    1 bit   3 bits (LSB)
-  ==============  ======  =================
-  operation code  source  instruction class
-  ==============  ======  =================
+==============  ======  =================
+4 bits (MSB)    1 bit   3 bits (LSB)
+==============  ======  =================
+operation code  source  instruction class
+==============  ======  =================
 
 The 4th bit encodes the source operand:
 
@@ -84,66 +89,66 @@ The four MSB bits store the operation code.
 Arithmetic instructions
 -----------------------
 
-BPF_ALU uses 32-bit wide operands while BPF_ALU64 uses 64-bit wide operands for
+``BPF_ALU`` uses 32-bit wide operands while ``BPF_ALU64`` uses 64-bit wide operands for
 otherwise identical operations.
-The code field encodes the operation as below:
-
-  ========  =====  =================================================
-  code      value  description
-  ========  =====  =================================================
-  BPF_ADD   0x00   dst += src
-  BPF_SUB   0x10   dst -= src
-  BPF_MUL   0x20   dst \*= src
-  BPF_DIV   0x30   dst /= src
-  BPF_OR    0x40   dst \|= src
-  BPF_AND   0x50   dst &= src
-  BPF_LSH   0x60   dst <<= src
-  BPF_RSH   0x70   dst >>= src
-  BPF_NEG   0x80   dst = ~src
-  BPF_MOD   0x90   dst %= src
-  BPF_XOR   0xa0   dst ^= src
-  BPF_MOV   0xb0   dst = src
-  BPF_ARSH  0xc0   sign extending shift right
-  BPF_END   0xd0   byte swap operations (see separate section below)
-  ========  =====  =================================================
-
-BPF_ADD | BPF_X | BPF_ALU means::
+The 'code' field encodes the operation as below:
+
+========  =====  ==========================================================
+code      value  description
+========  =====  ==========================================================
+BPF_ADD   0x00   dst += src
+BPF_SUB   0x10   dst -= src
+BPF_MUL   0x20   dst \*= src
+BPF_DIV   0x30   dst /= src
+BPF_OR    0x40   dst \|= src
+BPF_AND   0x50   dst &= src
+BPF_LSH   0x60   dst <<= src
+BPF_RSH   0x70   dst >>= src
+BPF_NEG   0x80   dst = ~src
+BPF_MOD   0x90   dst %= src
+BPF_XOR   0xa0   dst ^= src
+BPF_MOV   0xb0   dst = src
+BPF_ARSH  0xc0   sign extending shift right
+BPF_END   0xd0   byte swap operations (see `Byte swap instructions`_ below)
+========  =====  ==========================================================
+
+``BPF_ADD | BPF_X | BPF_ALU`` means::
 
   dst_reg = (u32) dst_reg + (u32) src_reg;
 
-BPF_ADD | BPF_X | BPF_ALU64 means::
+``BPF_ADD | BPF_X | BPF_ALU64`` means::
 
   dst_reg = dst_reg + src_reg
 
-BPF_XOR | BPF_K | BPF_ALU means::
+``BPF_XOR | BPF_K | BPF_ALU`` means::
 
   src_reg = (u32) src_reg ^ (u32) imm32
 
-BPF_XOR | BPF_K | BPF_ALU64 means::
+``BPF_XOR | BPF_K | BPF_ALU64`` means::
 
   src_reg = src_reg ^ imm32
 
 
 Byte swap instructions
-----------------------
+~~~~~~~~~~~~~~~~~~~~~~
 
 The byte swap instructions use an instruction class of ``BPF_ALU`` and a 4-bit
-code field of ``BPF_END``.
+'code' field of ``BPF_END``.
 
 The byte swap instructions operate on the destination register
 only and do not use a separate source register or immediate value.
 
-The 1-bit source operand field in the opcode is used to to select what byte
+The 1-bit source operand field in the opcode is used to select what byte
 order the operation convert from or to:
 
-  =========  =====  =================================================
-  source     value  description
-  =========  =====  =================================================
-  BPF_TO_LE  0x00   convert between host byte order and little endian
-  BPF_TO_BE  0x08   convert between host byte order and big endian
-  =========  =====  =================================================
+=========  =====  =================================================
+source     value  description
+=========  =====  =================================================
+BPF_TO_LE  0x00   convert between host byte order and little endian
+BPF_TO_BE  0x08   convert between host byte order and big endian
+=========  =====  =================================================
 
-The imm field encodes the width of the swap operations.  The following widths
+The 'imm' field encodes the width of the swap operations.  The following widths
 are supported: 16, 32 and 64.
 
 Examples:
@@ -156,35 +161,31 @@ Examples:
 
   dst_reg = htobe64(dst_reg)
 
-``BPF_FROM_LE`` and ``BPF_FROM_BE`` exist as aliases for ``BPF_TO_LE`` and
-``BPF_TO_BE`` respectively.
-
-
 Jump instructions
 -----------------
 
-BPF_JMP32 uses 32-bit wide operands while BPF_JMP uses 64-bit wide operands for
+``BPF_JMP32`` uses 32-bit wide operands while ``BPF_JMP`` uses 64-bit wide operands for
 otherwise identical operations.
-The code field encodes the operation as below:
-
-  ========  =====  =========================  ============
-  code      value  description                notes
-  ========  =====  =========================  ============
-  BPF_JA    0x00   PC += off                  BPF_JMP only
-  BPF_JEQ   0x10   PC += off if dst == src
-  BPF_JGT   0x20   PC += off if dst > src     unsigned
-  BPF_JGE   0x30   PC += off if dst >= src    unsigned
-  BPF_JSET  0x40   PC += off if dst & src
-  BPF_JNE   0x50   PC += off if dst != src
-  BPF_JSGT  0x60   PC += off if dst > src     signed
-  BPF_JSGE  0x70   PC += off if dst >= src    signed
-  BPF_CALL  0x80   function call
-  BPF_EXIT  0x90   function / program return  BPF_JMP only
-  BPF_JLT   0xa0   PC += off if dst < src     unsigned
-  BPF_JLE   0xb0   PC += off if dst <= src    unsigned
-  BPF_JSLT  0xc0   PC += off if dst < src     signed
-  BPF_JSLE  0xd0   PC += off if dst <= src    signed
-  ========  =====  =========================  ============
+The 'code' field encodes the operation as below:
+
+========  =====  =========================  ============
+code      value  description                notes
+========  =====  =========================  ============
+BPF_JA    0x00   PC += off                  BPF_JMP only
+BPF_JEQ   0x10   PC += off if dst == src
+BPF_JGT   0x20   PC += off if dst > src     unsigned
+BPF_JGE   0x30   PC += off if dst >= src    unsigned
+BPF_JSET  0x40   PC += off if dst & src
+BPF_JNE   0x50   PC += off if dst != src
+BPF_JSGT  0x60   PC += off if dst > src     signed
+BPF_JSGE  0x70   PC += off if dst >= src    signed
+BPF_CALL  0x80   function call
+BPF_EXIT  0x90   function / program return  BPF_JMP only
+BPF_JLT   0xa0   PC += off if dst < src     unsigned
+BPF_JLE   0xb0   PC += off if dst <= src    unsigned
+BPF_JSLT  0xc0   PC += off if dst < src     signed
+BPF_JSLE  0xd0   PC += off if dst <= src    signed
+========  =====  =========================  ============
 
 The eBPF program needs to store the return value into register R0 before doing a
 BPF_EXIT.
@@ -193,14 +194,26 @@ BPF_EXIT.
 Load and store instructions
 ===========================
 
-For load and store instructions (BPF_LD, BPF_LDX, BPF_ST and BPF_STX), the
+For load and store instructions (``BPF_LD``, ``BPF_LDX``, ``BPF_ST``, and ``BPF_STX``), the
 8-bit 'opcode' field is divided as:
 
-  ============  ======  =================
-  3 bits (MSB)  2 bits  3 bits (LSB)
-  ============  ======  =================
-  mode          size    instruction class
-  ============  ======  =================
+============  ======  =================
+3 bits (MSB)  2 bits  3 bits (LSB)
+============  ======  =================
+mode          size    instruction class
+============  ======  =================
+
+The mode modifier is one of:
+
+  =============  =====  ====================================  =============
+  mode modifier  value  description                           reference
+  =============  =====  ====================================  =============
+  BPF_IMM        0x00   64-bit immediate instructions         `64-bit immediate instructions`_
+  BPF_ABS        0x20   legacy BPF packet access (absolute)   `Legacy BPF Packet access instructions`_
+  BPF_IND        0x40   legacy BPF packet access (indirect)   `Legacy BPF Packet access instructions`_
+  BPF_MEM        0x60   regular load and store operations     `Regular load and store operations`_
+  BPF_ATOMIC     0xc0   atomic operations                     `Atomic operations`_
+  =============  =====  ====================================  =============
 
 The size modifier is one of:
 
@@ -213,19 +226,6 @@ The size modifier is one of:
   BPF_DW         0x18   double word (8 bytes)
   =============  =====  =====================
 
-The mode modifier is one of:
-
-  =============  =====  ====================================
-  mode modifier  value  description
-  =============  =====  ====================================
-  BPF_IMM        0x00   64-bit immediate instructions
-  BPF_ABS        0x20   legacy BPF packet access (absolute)
-  BPF_IND        0x40   legacy BPF packet access (indirect)
-  BPF_MEM        0x60   regular load and store operations
-  BPF_ATOMIC     0xc0   atomic operations
-  =============  =====  ====================================
-
-
 Regular load and store operations
 ---------------------------------
 
@@ -256,44 +256,42 @@ by other eBPF programs or means outside of this specification.
 All atomic operations supported by eBPF are encoded as store operations
 that use the ``BPF_ATOMIC`` mode modifier as follows:
 
-  * ``BPF_ATOMIC | BPF_W | BPF_STX`` for 32-bit operations
-  * ``BPF_ATOMIC | BPF_DW | BPF_STX`` for 64-bit operations
-  * 8-bit and 16-bit wide atomic operations are not supported.
+* ``BPF_ATOMIC | BPF_W | BPF_STX`` for 32-bit operations
+* ``BPF_ATOMIC | BPF_DW | BPF_STX`` for 64-bit operations
+* 8-bit and 16-bit wide atomic operations are not supported.
 
-The imm field is used to encode the actual atomic operation.
+The 'imm' field is used to encode the actual atomic operation.
 Simple atomic operation use a subset of the values defined to encode
-arithmetic operations in the imm field to encode the atomic operation:
+arithmetic operations in the 'imm' field to encode the atomic operation:
 
-  ========  =====  ===========
-  imm       value  description
-  ========  =====  ===========
-  BPF_ADD   0x00   atomic add
-  BPF_OR    0x40   atomic or
-  BPF_AND   0x50   atomic and
-  BPF_XOR   0xa0   atomic xor
-  ========  =====  ===========
+========  =====  ===========
+imm       value  description
+========  =====  ===========
+BPF_ADD   0x00   atomic add
+BPF_OR    0x40   atomic or
+BPF_AND   0x50   atomic and
+BPF_XOR   0xa0   atomic xor
+========  =====  ===========
 
 
-``BPF_ATOMIC | BPF_W  | BPF_STX`` with imm = BPF_ADD means::
+``BPF_ATOMIC | BPF_W  | BPF_STX`` with 'imm' = BPF_ADD means::
 
   *(u32 *)(dst_reg + off16) += src_reg
 
-``BPF_ATOMIC | BPF_DW | BPF_STX`` with imm = BPF ADD means::
+``BPF_ATOMIC | BPF_DW | BPF_STX`` with 'imm' = BPF ADD means::
 
   *(u64 *)(dst_reg + off16) += src_reg
 
-``BPF_XADD`` is a deprecated name for ``BPF_ATOMIC | BPF_ADD``.
-
 In addition to the simple atomic operations, there also is a modifier and
 two complex atomic operations:
 
-  ===========  ================  ===========================
-  imm          value             description
-  ===========  ================  ===========================
-  BPF_FETCH    0x01              modifier: return old value
-  BPF_XCHG     0xe0 | BPF_FETCH  atomic exchange
-  BPF_CMPXCHG  0xf0 | BPF_FETCH  atomic compare and exchange
-  ===========  ================  ===========================
+===========  ================  ===========================
+imm          value             description
+===========  ================  ===========================
+BPF_FETCH    0x01              modifier: return old value
+BPF_XCHG     0xe0 | BPF_FETCH  atomic exchange
+BPF_CMPXCHG  0xf0 | BPF_FETCH  atomic compare and exchange
+===========  ================  ===========================
 
 The ``BPF_FETCH`` modifier is optional for simple atomic operations, and
 always set for the complex atomic operations.  If the ``BPF_FETCH`` flag
@@ -309,16 +307,10 @@ The ``BPF_CMPXCHG`` operation atomically compares the value addressed by
 value that was at ``dst_reg + off`` before the operation is zero-extended
 and loaded back to ``R0``.
 
-Clang can generate atomic instructions by default when ``-mcpu=v3`` is
-enabled. If a lower version for ``-mcpu`` is set, the only atomic instruction
-Clang can generate is ``BPF_ADD`` *without* ``BPF_FETCH``. If you need to enable
-the atomics features, while keeping a lower ``-mcpu`` version, you can use
-``-Xclang -target-feature -Xclang +alu32``.
-
 64-bit immediate instructions
 -----------------------------
 
-Instructions with the ``BPF_IMM`` mode modifier use the wide instruction
+Instructions with the ``BPF_IMM`` 'mode' modifier use the wide instruction
 encoding for an extra imm64 value.
 
 There is currently only one such instruction.
@@ -331,36 +323,6 @@ There is currently only one such instruction.
 Legacy BPF Packet access instructions
 -------------------------------------
 
-eBPF has special instructions for access to packet data that have been
-carried over from classic BPF to retain the performance of legacy socket
-filters running in the eBPF interpreter.
-
-The instructions come in two forms: ``BPF_ABS | <size> | BPF_LD`` and
-``BPF_IND | <size> | BPF_LD``.
-
-These instructions are used to access packet data and can only be used when
-the program context is a pointer to networking packet.  ``BPF_ABS``
-accesses packet data at an absolute offset specified by the immediate data
-and ``BPF_IND`` access packet data at an offset that includes the value of
-a register in addition to the immediate data.
-
-These instructions have seven implicit operands:
-
- * Register R6 is an implicit input that must contain pointer to a
-   struct sk_buff.
- * Register R0 is an implicit output which contains the data fetched from
-   the packet.
- * Registers R1-R5 are scratch registers that are clobbered after a call to
-   ``BPF_ABS | BPF_LD`` or ``BPF_IND | BPF_LD`` instructions.
-
-These instructions have an implicit program exit condition as well. When an
-eBPF program is trying to access the data beyond the packet boundary, the
-program execution will be aborted.
-
-``BPF_ABS | BPF_W | BPF_LD`` means::
-
-  R0 = ntohl(*(u32 *) (((struct sk_buff *) R6)->data + imm32))
-
-``BPF_IND | BPF_W | BPF_LD`` means::
-
-  R0 = ntohl(*(u32 *) (((struct sk_buff *) R6)->data + src_reg + imm32))
+eBPF previously introduced special instructions for access to packet data that were
+carried over from classic BPF. However, these instructions are
+deprecated and should no longer be used.
diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst
index c0b7dae6dbf5..0f858156371d 100644
--- a/Documentation/bpf/kfuncs.rst
+++ b/Documentation/bpf/kfuncs.rst
@@ -137,14 +137,37 @@ KF_ACQUIRE and KF_RET_NULL flags.
 --------------------------
 
 The KF_TRUSTED_ARGS flag is used for kfuncs taking pointer arguments. It
-indicates that the all pointer arguments will always be refcounted, and have
-their offset set to 0. It can be used to enforce that a pointer to a refcounted
-object acquired from a kfunc or BPF helper is passed as an argument to this
-kfunc without any modifications (e.g. pointer arithmetic) such that it is
-trusted and points to the original object. This flag is often used for kfuncs
-that operate (change some property, perform some operation) on an object that
-was obtained using an acquire kfunc. Such kfuncs need an unchanged pointer to
-ensure the integrity of the operation being performed on the expected object.
+indicates that the all pointer arguments will always have a guaranteed lifetime,
+and pointers to kernel objects are always passed to helpers in their unmodified
+form (as obtained from acquire kfuncs).
+
+It can be used to enforce that a pointer to a refcounted object acquired from a
+kfunc or BPF helper is passed as an argument to this kfunc without any
+modifications (e.g. pointer arithmetic) such that it is trusted and points to
+the original object.
+
+Meanwhile, it is also allowed pass pointers to normal memory to such kfuncs,
+but those can have a non-zero offset.
+
+This flag is often used for kfuncs that operate (change some property, perform
+some operation) on an object that was obtained using an acquire kfunc. Such
+kfuncs need an unchanged pointer to ensure the integrity of the operation being
+performed on the expected object.
+
+2.4.6 KF_SLEEPABLE flag
+-----------------------
+
+The KF_SLEEPABLE flag is used for kfuncs that may sleep. Such kfuncs can only
+be called by sleepable BPF programs (BPF_F_SLEEPABLE).
+
+2.4.7 KF_DESTRUCTIVE flag
+--------------------------
+
+The KF_DESTRUCTIVE flag is used to indicate functions calling which is
+destructive to the system. For example such a call can result in system
+rebooting or panicking. Due to this additional restrictions apply to these
+calls. At the moment they only require CAP_SYS_BOOT capability, but more can be
+added later.
 
 2.5 Registering the kfuncs
 --------------------------
diff --git a/Documentation/bpf/linux-notes.rst b/Documentation/bpf/linux-notes.rst
new file mode 100644
index 000000000000..956b0c86699d
--- /dev/null
+++ b/Documentation/bpf/linux-notes.rst
@@ -0,0 +1,53 @@
+.. contents::
+.. sectnum::
+
+==========================
+Linux implementation notes
+==========================
+
+This document provides more details specific to the Linux kernel implementation of the eBPF instruction set.
+
+Byte swap instructions
+======================
+
+``BPF_FROM_LE`` and ``BPF_FROM_BE`` exist as aliases for ``BPF_TO_LE`` and ``BPF_TO_BE`` respectively.
+
+Legacy BPF Packet access instructions
+=====================================
+
+As mentioned in the `ISA standard documentation <instruction-set.rst#legacy-bpf-packet-access-instructions>`_,
+Linux has special eBPF instructions for access to packet data that have been
+carried over from classic BPF to retain the performance of legacy socket
+filters running in the eBPF interpreter.
+
+The instructions come in two forms: ``BPF_ABS | <size> | BPF_LD`` and
+``BPF_IND | <size> | BPF_LD``.
+
+These instructions are used to access packet data and can only be used when
+the program context is a pointer to a networking packet.  ``BPF_ABS``
+accesses packet data at an absolute offset specified by the immediate data
+and ``BPF_IND`` access packet data at an offset that includes the value of
+a register in addition to the immediate data.
+
+These instructions have seven implicit operands:
+
+* Register R6 is an implicit input that must contain a pointer to a
+  struct sk_buff.
+* Register R0 is an implicit output which contains the data fetched from
+  the packet.
+* Registers R1-R5 are scratch registers that are clobbered by the
+  instruction.
+
+These instructions have an implicit program exit condition as well. If an
+eBPF program attempts access data beyond the packet boundary, the
+program execution will be aborted.
+
+``BPF_ABS | BPF_W | BPF_LD`` (0x20) means::
+
+  R0 = ntohl(*(u32 *) ((struct sk_buff *) R6->data + imm))
+
+where ``ntohl()`` converts a 32-bit value from network byte order to host byte order.
+
+``BPF_IND | BPF_W | BPF_LD`` (0x40) means::
+
+  R0 = ntohl(*(u32 *) ((struct sk_buff *) R6->data + src + imm))
diff --git a/Documentation/bpf/map_cgroup_storage.rst b/Documentation/bpf/map_cgroup_storage.rst
index cab9543017bf..8e5fe532c07e 100644
--- a/Documentation/bpf/map_cgroup_storage.rst
+++ b/Documentation/bpf/map_cgroup_storage.rst
@@ -31,7 +31,7 @@ The map uses key of type of either ``__u64 cgroup_inode_id`` or
     };
 
 ``cgroup_inode_id`` is the inode id of the cgroup directory.
-``attach_type`` is the the program's attach type.
+``attach_type`` is the program's attach type.
 
 Linux 5.9 added support for type ``__u64 cgroup_inode_id`` as the key type.
 When this key type is used, then all attach types of the particular cgroup and
@@ -155,7 +155,7 @@ However, the BPF program can still only associate with one map of each type
 ``BPF_MAP_TYPE_CGROUP_STORAGE`` or more than one
 ``BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE``.
 
-In all versions, userspace may use the the attach parameters of cgroup and
+In all versions, userspace may use the attach parameters of cgroup and
 attach type pair in ``struct bpf_cgroup_storage_key`` as the key to the BPF map
 APIs to read or update the storage for a given attachment. For Linux 5.9
 attach type shared storages, only the first value in the struct, cgroup inode
diff --git a/Documentation/conf.py b/Documentation/conf.py
index 255384d094bf..b50c85083149 100644
--- a/Documentation/conf.py
+++ b/Documentation/conf.py
@@ -15,6 +15,18 @@
 import sys
 import os
 import sphinx
+import shutil
+
+# helper
+# ------
+
+def have_command(cmd):
+    """Search ``cmd`` in the ``PATH`` environment.
+
+    If found, return True.
+    If not found, return False.
+    """
+    return shutil.which(cmd) is not None
 
 # Get Sphinx version
 major, minor, patch = sphinx.version_info[:3]
@@ -107,7 +119,32 @@ else:
 autosectionlabel_prefix_document = True
 autosectionlabel_maxdepth = 2
 
-extensions.append("sphinx.ext.imgmath")
+# Load math renderer:
+# For html builder, load imgmath only when its dependencies are met.
+# mathjax is the default math renderer since Sphinx 1.8.
+have_latex =  have_command('latex')
+have_dvipng = have_command('dvipng')
+load_imgmath = have_latex and have_dvipng
+
+# Respect SPHINX_IMGMATH (for html docs only)
+if 'SPHINX_IMGMATH' in os.environ:
+    env_sphinx_imgmath = os.environ['SPHINX_IMGMATH']
+    if 'yes' in env_sphinx_imgmath:
+        load_imgmath = True
+    elif 'no' in env_sphinx_imgmath:
+        load_imgmath = False
+    else:
+        sys.stderr.write("Unknown env SPHINX_IMGMATH=%s ignored.\n" % env_sphinx_imgmath)
+
+# Always load imgmath for Sphinx <1.8 or for epub docs
+load_imgmath = (load_imgmath or (major == 1 and minor < 8)
+                or 'epub' in sys.argv)
+
+if load_imgmath:
+    extensions.append("sphinx.ext.imgmath")
+    math_renderer = 'imgmath'
+else:
+    math_renderer = 'mathjax'
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -333,7 +370,8 @@ html_static_path = ['sphinx-static']
 html_use_smartypants = False
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# Note that the RTD theme ignores this.
+html_sidebars = { '**': ['searchbox.html', 'localtoc.html', 'sourcelink.html']}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
diff --git a/Documentation/asm-annotations.rst b/Documentation/core-api/asm-annotations.rst
index a64f2ca469d4..bc514ed59887 100644
--- a/Documentation/asm-annotations.rst
+++ b/Documentation/core-api/asm-annotations.rst
@@ -43,10 +43,11 @@ annotated objects like this, tools can be run on them to generate more useful
 information. In particular, on properly annotated objects, ``objtool`` can be
 run to check and fix the object if needed. Currently, ``objtool`` can report
 missing frame pointer setup/destruction in functions. It can also
-automatically generate annotations for :doc:`ORC unwinder <x86/orc-unwinder>`
+automatically generate annotations for the ORC unwinder
+(Documentation/x86/orc-unwinder.rst)
 for most code. Both of these are especially important to support reliable
-stack traces which are in turn necessary for :doc:`Kernel live patching
-<livepatch/livepatch>`.
+stack traces which are in turn necessary for kernel live patching
+(Documentation/livepatch/livepatch.rst).
 
 Caveat and Discussion
 ---------------------
diff --git a/Documentation/core-api/cpu_hotplug.rst b/Documentation/core-api/cpu_hotplug.rst
index c6f4ba2fb32d..f75778d37488 100644
--- a/Documentation/core-api/cpu_hotplug.rst
+++ b/Documentation/core-api/cpu_hotplug.rst
@@ -560,7 +560,7 @@ available:
   * cpuhp_state_remove_instance(state, node)
   * cpuhp_state_remove_instance_nocalls(state, node)
 
-The arguments are the same as for the the cpuhp_state_add_instance*()
+The arguments are the same as for the cpuhp_state_add_instance*()
 variants above.
 
 The functions differ in the way how the installed callbacks are treated:
diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst
index dc95df462eea..b0e7b4771fff 100644
--- a/Documentation/core-api/index.rst
+++ b/Documentation/core-api/index.rst
@@ -23,6 +23,7 @@ it.
    printk-formats
    printk-index
    symbol-namespaces
+   asm-annotations
 
 Data structures and low-level utilities
 =======================================
@@ -44,6 +45,8 @@ Library functionality that is used throughout the kernel.
    this_cpu_ops
    timekeeping
    errseq
+   wrappers/atomic_t
+   wrappers/atomic_bitops
 
 Low level entry and exit
 ========================
@@ -67,6 +70,7 @@ Documentation/locking/index.rst for more related documentation.
    local_ops
    padata
    ../RCU/index
+   wrappers/memory-barriers.rst
 
 Low-level hardware management
 =============================
diff --git a/Documentation/core-api/irq/irq-domain.rst b/Documentation/core-api/irq/irq-domain.rst
index d30b4d0a9769..f88a6ee67a35 100644
--- a/Documentation/core-api/irq/irq-domain.rst
+++ b/Documentation/core-api/irq/irq-domain.rst
@@ -71,7 +71,7 @@ variety of methods:
 Note that irq domain lookups must happen in contexts that are
 compatible with a RCU read-side critical section.
 
-The irq_create_mapping() function must be called *atleast once*
+The irq_create_mapping() function must be called *at least once*
 before any call to irq_find_mapping(), lest the descriptor will not
 be allocated.
 
diff --git a/Documentation/core-api/printk-formats.rst b/Documentation/core-api/printk-formats.rst
index 5e89497ba314..dbe1aacc79d0 100644
--- a/Documentation/core-api/printk-formats.rst
+++ b/Documentation/core-api/printk-formats.rst
@@ -625,6 +625,16 @@ Examples::
 	%p4cc	Y10  little-endian (0x20303159)
 	%p4cc	NV12 big-endian (0xb231564e)
 
+Rust
+----
+
+::
+
+	%pA
+
+Only intended to be used from Rust code to format ``core::fmt::Arguments``.
+Do *not* use it from C.
+
 Thanks
 ======
 
diff --git a/Documentation/core-api/wrappers/atomic_bitops.rst b/Documentation/core-api/wrappers/atomic_bitops.rst
new file mode 100644
index 000000000000..bf24e4081a8f
--- /dev/null
+++ b/Documentation/core-api/wrappers/atomic_bitops.rst
@@ -0,0 +1,18 @@
+.. SPDX-License-Identifier: GPL-2.0
+   This is a simple wrapper to bring atomic_bitops.txt into the RST world
+   until such a time as that file can be converted directly.
+
+=============
+Atomic bitops
+=============
+
+.. raw:: latex
+
+    \footnotesize
+
+.. include:: ../../atomic_bitops.txt
+   :literal:
+
+.. raw:: latex
+
+    \normalsize
diff --git a/Documentation/core-api/wrappers/atomic_t.rst b/Documentation/core-api/wrappers/atomic_t.rst
new file mode 100644
index 000000000000..ed109a964c77
--- /dev/null
+++ b/Documentation/core-api/wrappers/atomic_t.rst
@@ -0,0 +1,19 @@
+.. SPDX-License-Identifier: GPL-2.0
+   This is a simple wrapper to bring atomic_t.txt into the RST world
+   until such a time as that file can be converted directly.
+
+============
+Atomic types
+============
+
+.. raw:: latex
+
+    \footnotesize
+
+.. include:: ../../atomic_t.txt
+   :literal:
+
+.. raw:: latex
+
+    \normalsize
+
diff --git a/Documentation/core-api/wrappers/memory-barriers.rst b/Documentation/core-api/wrappers/memory-barriers.rst
new file mode 100644
index 000000000000..532460b5e3eb
--- /dev/null
+++ b/Documentation/core-api/wrappers/memory-barriers.rst
@@ -0,0 +1,18 @@
+.. SPDX-License-Identifier: GPL-2.0
+   This is a simple wrapper to bring memory-barriers.txt into the RST world
+   until such a time as that file can be converted directly.
+
+============================
+Linux kernel memory barriers
+============================
+
+.. raw:: latex
+
+    \footnotesize
+
+.. include:: ../../memory-barriers.txt
+   :literal:
+
+.. raw:: latex
+
+    \normalsize
diff --git a/Documentation/dev-tools/kselftest.rst b/Documentation/dev-tools/kselftest.rst
index e87973763b91..12b575b76b20 100644
--- a/Documentation/dev-tools/kselftest.rst
+++ b/Documentation/dev-tools/kselftest.rst
@@ -320,7 +320,7 @@ A bare bones test module might look like this:
 
    #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
-   #include "../tools/testing/selftests/kselftest/module.h"
+   #include "../tools/testing/selftests/kselftest_module.h"
 
    KSTM_MODULE_GLOBALS();
 
diff --git a/Documentation/dev-tools/kunit/architecture.rst b/Documentation/dev-tools/kunit/architecture.rst
index cf9e6e3eeae4..8efe792bdcb9 100644
--- a/Documentation/dev-tools/kunit/architecture.rst
+++ b/Documentation/dev-tools/kunit/architecture.rst
@@ -6,8 +6,8 @@ KUnit Architecture
 
 The KUnit architecture can be divided into two parts:
 
-- Kernel testing library
-- kunit_tool (Command line test harness)
+- `In-Kernel Testing Framework`_
+- `kunit_tool (Command Line Test Harness)`_
 
 In-Kernel Testing Framework
 ===========================
diff --git a/Documentation/dev-tools/kunit/faq.rst b/Documentation/dev-tools/kunit/faq.rst
index 172e239791a8..fae426f2634a 100644
--- a/Documentation/dev-tools/kunit/faq.rst
+++ b/Documentation/dev-tools/kunit/faq.rst
@@ -31,13 +31,16 @@ For the most part, the KUnit core framework (what we use to write the tests)
 can compile to any architecture. It compiles like just another part of the
 kernel and runs when the kernel boots, or when built as a module, when the
 module is loaded.  However, there is infrastructure, like the KUnit Wrapper
-(``tools/testing/kunit/kunit.py``) that does not support other architectures.
+(``tools/testing/kunit/kunit.py``) that might not support some architectures
+(see :ref:`kunit-on-qemu`).
 
 In short, yes, you can run KUnit on other architectures, but it might require
 more work than using KUnit on UML.
 
 For more information, see :ref:`kunit-on-non-uml`.
 
+.. _kinds-of-tests:
+
 What is the difference between a unit test and other kinds of tests?
 ====================================================================
 Most existing tests for the Linux kernel would be categorized as an integration
@@ -95,8 +98,7 @@ things to try.
    seeing. When tests are built-in, they will execute when the kernel boots, and
    modules will automatically execute associated tests when loaded. Test results
    can be collected from ``/sys/kernel/debug/kunit/<test suite>/results``, and
-   can be parsed with ``kunit.py parse``. For more details, see "KUnit on
-   non-UML architectures" in Documentation/dev-tools/kunit/usage.rst.
+   can be parsed with ``kunit.py parse``. For more details, see :ref:`kunit-on-qemu`.
 
 If none of the above tricks help, you are always welcome to email any issues to
 kunit-dev@googlegroups.com.
diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst
index 595205348d2d..f5d13f1d37be 100644
--- a/Documentation/dev-tools/kunit/index.rst
+++ b/Documentation/dev-tools/kunit/index.rst
@@ -13,7 +13,6 @@ KUnit - Linux Kernel Unit Testing
 	run_wrapper
 	run_manual
 	usage
-	kunit-tool
 	api/index
 	style
 	faq
@@ -29,10 +28,10 @@ KUnit (Kernel unit testing framework) provides a common framework for
 unit tests within the Linux kernel. Using KUnit, you can define groups
 of test cases called test suites. The tests either run on kernel boot
 if built-in, or load as a module. KUnit automatically flags and reports
-failed test cases in the kernel log. The test results appear in `TAP
-(Test Anything Protocol) format <https://testanything.org/>`_. It is inspired by
-JUnit, Python’s unittest.mock, and GoogleTest/GoogleMock (C++ unit testing
-framework).
+failed test cases in the kernel log. The test results appear in
+:doc:`KTAP (Kernel - Test Anything Protocol) format</dev-tools/ktap>`.
+It is inspired by JUnit, Python’s unittest.mock, and GoogleTest/GoogleMock
+(C++ unit testing framework).
 
 KUnit tests are part of the kernel, written in the C (programming)
 language, and test parts of the Kernel implementation (example: a C
@@ -46,8 +45,9 @@ internal system functionality. KUnit runs in kernel space and is not
 restricted to things exposed to user-space.
 
 In addition, KUnit has kunit_tool, a script (``tools/testing/kunit/kunit.py``)
-that configures the Linux kernel, runs KUnit tests under QEMU or UML (`User Mode
-Linux <http://user-mode-linux.sourceforge.net/>`_), parses the test results and
+that configures the Linux kernel, runs KUnit tests under QEMU or UML
+(:doc:`User Mode Linux </virt/uml/user_mode_linux_howto_v2>`),
+parses the test results and
 displays them in a user friendly manner.
 
 Features
@@ -95,6 +95,8 @@ Unit Testing Advantages
 - Improves code quality.
 - Encourages writing testable code.
 
+Read also :ref:`kinds-of-tests`.
+
 How do I use it?
 ================
 
@@ -107,7 +109,5 @@ How do I use it?
     examples.
 *   Documentation/dev-tools/kunit/api/index.rst - KUnit APIs
     used for testing.
-*   Documentation/dev-tools/kunit/kunit-tool.rst - kunit_tool helper
-    script.
 *   Documentation/dev-tools/kunit/faq.rst - KUnit common questions and
     answers.
diff --git a/Documentation/dev-tools/kunit/kunit-tool.rst b/Documentation/dev-tools/kunit/kunit-tool.rst
deleted file mode 100644
index ae52e0f489f9..000000000000
--- a/Documentation/dev-tools/kunit/kunit-tool.rst
+++ /dev/null
@@ -1,232 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-=================
-kunit_tool How-To
-=================
-
-What is kunit_tool?
-===================
-
-kunit_tool is a script (``tools/testing/kunit/kunit.py``) that aids in building
-the Linux kernel as UML (`User Mode Linux
-<http://user-mode-linux.sourceforge.net/>`_), running KUnit tests, parsing
-the test results and displaying them in a user friendly manner.
-
-kunit_tool addresses the problem of being able to run tests without needing a
-virtual machine or actual hardware with User Mode Linux. User Mode Linux is a
-Linux architecture, like ARM or x86; however, unlike other architectures it
-compiles the kernel as a standalone Linux executable that can be run like any
-other program directly inside of a host operating system. To be clear, it does
-not require any virtualization support: it is just a regular program.
-
-What is a .kunitconfig?
-=======================
-
-It's just a defconfig that kunit_tool looks for in the build directory
-(``.kunit`` by default).  kunit_tool uses it to generate a .config as you might
-expect. In addition, it verifies that the generated .config contains the CONFIG
-options in the .kunitconfig; the reason it does this is so that it is easy to
-be sure that a CONFIG that enables a test actually ends up in the .config.
-
-It's also possible to pass a separate .kunitconfig fragment to kunit_tool,
-which is useful if you have several different groups of tests you wish
-to run independently, or if you want to use pre-defined test configs for
-certain subsystems.
-
-Getting Started with kunit_tool
-===============================
-
-If a kunitconfig is present at the root directory, all you have to do is:
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py run
-
-However, you most likely want to use it with the following options:
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all`
-
-- ``--timeout`` sets a maximum amount of time to allow tests to run.
-- ``--jobs`` sets the number of threads to use to build the kernel.
-
-.. note::
-	This command will work even without a .kunitconfig file: if no
-	.kunitconfig is present, a default one will be used instead.
-
-If you wish to use a different .kunitconfig file (such as one provided for
-testing a particular subsystem), you can pass it as an option.
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py run --kunitconfig=fs/ext4/.kunitconfig
-
-For a list of all the flags supported by kunit_tool, you can run:
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py run --help
-
-Configuring, Building, and Running Tests
-========================================
-
-It's also possible to run just parts of the KUnit build process independently,
-which is useful if you want to make manual changes to part of the process.
-
-A .config can be generated from a .kunitconfig by using the ``config`` argument
-when running kunit_tool:
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py config
-
-Similarly, if you just want to build a KUnit kernel from the current .config,
-you can use the ``build`` argument:
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py build
-
-And, if you already have a built UML kernel with built-in KUnit tests, you can
-run the kernel and display the test results with the ``exec`` argument:
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py exec
-
-The ``run`` command which is discussed above is equivalent to running all three
-of these in sequence.
-
-All of these commands accept a number of optional command-line arguments. The
-``--help`` flag will give a complete list of these, or keep reading this page
-for a guide to some of the more useful ones.
-
-Parsing Test Results
-====================
-
-KUnit tests output their results in TAP (Test Anything Protocol) format.
-kunit_tool will, when running tests, parse this output and print a summary
-which is much more pleasant to read. If you wish to look at the raw test
-results in TAP format, you can pass the ``--raw_output`` argument.
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py run --raw_output
-
-The raw output from test runs may contain other, non-KUnit kernel log
-lines. You can see just KUnit output with ``--raw_output=kunit``:
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py run --raw_output=kunit
-
-If you have KUnit results in their raw TAP format, you can parse them and print
-the human-readable summary with the ``parse`` command for kunit_tool. This
-accepts a filename for an argument, or will read from standard input.
-
-.. code-block:: bash
-
-	# Reading from a file
-	./tools/testing/kunit/kunit.py parse /var/log/dmesg
-	# Reading from stdin
-	dmesg | ./tools/testing/kunit/kunit.py parse
-
-This is very useful if you wish to run tests in a configuration not supported
-by kunit_tool (such as on real hardware, or an unsupported architecture).
-
-Filtering Tests
-===============
-
-It's possible to run only a subset of the tests built into a kernel by passing
-a filter to the ``exec`` or ``run`` commands. For example, if you only wanted
-to run KUnit resource tests, you could use:
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py run 'kunit-resource*'
-
-This uses the standard glob format for wildcards.
-
-Running Tests on QEMU
-=====================
-
-kunit_tool supports running tests on QEMU as well as via UML (as mentioned
-elsewhere). The default way of running tests on QEMU requires two flags:
-
-``--arch``
-	Selects a collection of configs (Kconfig as well as QEMU configs
-	options, etc) that allow KUnit tests to be run on the specified
-	architecture in a minimal way; this is usually not much slower than
-	using UML. The architecture argument is the same as the name of the
-	option passed to the ``ARCH`` variable used by Kbuild. Not all
-	architectures are currently supported by this flag, but can be handled
-	by the ``--qemu_config`` discussed later. If ``um`` is passed (or this
-	this flag is ignored) the tests will run via UML. Non-UML architectures,
-	e.g. i386, x86_64, arm, um, etc. Non-UML run on QEMU.
-
-``--cross_compile``
-	Specifies the use of a toolchain by Kbuild. The argument passed here is
-	the same passed to the ``CROSS_COMPILE`` variable used by Kbuild. As a
-	reminder this will be the prefix for the toolchain binaries such as gcc
-	for example ``sparc64-linux-gnu-`` if you have the sparc toolchain
-	installed on your system, or
-	``$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-``
-	if you have downloaded the microblaze toolchain from the 0-day website
-	to a directory in your home directory called ``toolchains``.
-
-In many cases it is likely that you may want to run an architecture which is
-not supported by the ``--arch`` flag, or you may want to just run KUnit tests
-on QEMU using a non-default configuration. For this use case, you can write
-your own QemuConfig. These QemuConfigs are written in Python. They must have an
-import line ``from ..qemu_config import QemuArchParams`` at the top of the file
-and the file must contain a variable called ``QEMU_ARCH`` that has an instance
-of ``QemuArchParams`` assigned to it. An example can be seen in
-``tools/testing/kunit/qemu_configs/x86_64.py``.
-
-Once you have a QemuConfig you can pass it into kunit_tool using the
-``--qemu_config`` flag; when used this flag replaces the ``--arch`` flag. If we
-were to do this with the ``x86_64.py`` example from above, the invocation would
-look something like this:
-
-.. code-block:: bash
-
-	./tools/testing/kunit/kunit.py run \
-		--timeout=60 \
-		--jobs=12 \
-		--qemu_config=./tools/testing/kunit/qemu_configs/x86_64.py
-
-Other Useful Options
-====================
-
-kunit_tool has a number of other command-line arguments which can be useful
-when adapting it to fit your environment or needs.
-
-Some of the more useful ones are:
-
-``--help``
-	Lists all of the available options. Note that different commands
-	(``config``, ``build``, ``run``, etc) will have different supported
-	options. Place ``--help`` before the command to list common options,
-	and after the command for options specific to that command.
-
-``--build_dir``
-	Specifies the build directory that kunit_tool will use. This is where
-	the .kunitconfig file is located, as well as where the .config and
-	compiled kernel will be placed. Defaults to ``.kunit``.
-
-``--make_options``
-	Specifies additional options to pass to ``make`` when compiling a
-	kernel (with the ``build`` or ``run`` commands). For example, to enable
-	compiler warnings, you can pass ``--make_options W=1``.
-
-``--alltests``
-        Builds a UML kernel with all config options enabled using ``make
-        allyesconfig``. This allows you to run as many tests as is possible,
-        but is very slow and prone to breakage as new options are added or
-        modified. In most cases, enabling all tests which have satisfied
-        dependencies by adding ``CONFIG_KUNIT_ALL_TESTS=1`` to your
-        .kunitconfig is preferable.
-
-There are several other options (and new ones are often added), so do check
-``--help`` if you're looking for something not mentioned here.
diff --git a/Documentation/dev-tools/kunit/run_wrapper.rst b/Documentation/dev-tools/kunit/run_wrapper.rst
index cce203138fb7..6b33caf6c8ab 100644
--- a/Documentation/dev-tools/kunit/run_wrapper.rst
+++ b/Documentation/dev-tools/kunit/run_wrapper.rst
@@ -1,8 +1,8 @@
 .. SPDX-License-Identifier: GPL-2.0
 
-=========================
-Run Tests with kunit_tool
-=========================
+=============================
+Running tests with kunit_tool
+=============================
 
 We can either run KUnit tests using kunit_tool or can run tests
 manually, and then use kunit_tool to parse the results. To run tests
@@ -22,7 +22,7 @@ We should see the following:
 
 .. code-block::
 
-	Generating .config...
+	Configuring KUnit Kernel ...
 	Building KUnit kernel...
 	Starting KUnit kernel...
 
@@ -30,7 +30,7 @@ We may want to use the following options:
 
 .. code-block::
 
-	./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all
+	./tools/testing/kunit/kunit.py run --timeout=30 --jobs=`nproc --all`
 
 - ``--timeout`` sets a maximum amount of time for tests to run.
 - ``--jobs`` sets the number of threads to build the kernel.
@@ -58,8 +58,8 @@ To view kunit_tool flags (optional command-line arguments), run:
 
 	./tools/testing/kunit/kunit.py run --help
 
-Create a  ``.kunitconfig`` File
-===============================
+Creating a ``.kunitconfig`` file
+================================
 
 If we want to run a specific set of tests (rather than those listed
 in the KUnit ``defconfig``), we can provide Kconfig options in the
@@ -98,8 +98,8 @@ have not included the options dependencies.
    The build dir needs to be set for ``make menuconfig`` to
    work, therefore  by default use ``make O=.kunit menuconfig``.
 
-Configure, Build, and Run Tests
-===============================
+Configuring, building, and running tests
+========================================
 
 If we want to make manual changes to the KUnit build process, we
 can run part of the KUnit build process independently.
@@ -125,11 +125,11 @@ argument:
 
 	./tools/testing/kunit/kunit.py exec
 
-The ``run`` command discussed in section: **Run Tests with kunit_tool**,
+The ``run`` command discussed in section: **Running tests with kunit_tool**,
 is equivalent to running the above three commands in sequence.
 
-Parse Test Results
-==================
+Parsing test results
+====================
 
 KUnit tests output displays results in TAP (Test Anything Protocol)
 format. When running tests, kunit_tool parses this output and prints
@@ -152,8 +152,8 @@ standard input.
 	# Reading from stdin
 	dmesg | ./tools/testing/kunit/kunit.py parse
 
-Run Selected Test Suites
-========================
+Filtering tests
+===============
 
 By passing a bash style glob filter to the ``exec`` or ``run``
 commands, we can run a subset of the tests built into a kernel . For
@@ -165,8 +165,10 @@ example: if we only want to run KUnit resource tests, use:
 
 This uses the standard glob format with wildcard characters.
 
-Run Tests on qemu
-=================
+.. _kunit-on-qemu:
+
+Running tests on QEMU
+=====================
 
 kunit_tool supports running tests on  qemu as well as
 via UML. To run tests on qemu, by default it requires two flags:
@@ -229,8 +231,8 @@ as
 		--jobs=12 \
 		--qemu_config=./tools/testing/kunit/qemu_configs/x86_64.py
 
-Command-Line Arguments
-======================
+Running command-line arguments
+==============================
 
 kunit_tool has a number of other command-line arguments which can
 be useful for our test environment. Below are the most commonly used
diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst
index 867a4bba6bf6..f4f504f1fb15 100644
--- a/Documentation/dev-tools/kunit/start.rst
+++ b/Documentation/dev-tools/kunit/start.rst
@@ -4,6 +4,10 @@
 Getting Started
 ===============
 
+This page contains an overview of the kunit_tool and KUnit framework,
+teaching how to run existing tests and then how to write a simple test case,
+and covers common problems users face when using KUnit for the first time.
+
 Installing Dependencies
 =======================
 KUnit has the same dependencies as the Linux kernel. As long as you can
@@ -19,30 +23,53 @@ can run kunit_tool:
 
 	./tools/testing/kunit/kunit.py run
 
-For more information on this wrapper, see:
-Documentation/dev-tools/kunit/run_wrapper.rst.
+.. note ::
+	You may see the following error:
+	"The source tree is not clean, please run 'make ARCH=um mrproper'"
 
-Creating a ``.kunitconfig``
----------------------------
+	This happens because internally kunit.py specifies ``.kunit``
+	(default option) as the build directory in the command ``make O=output/dir``
+	through the argument ``--build_dir``.  Hence, before starting an
+	out-of-tree build, the source tree must be clean.
 
-By default, kunit_tool runs a selection of tests. However, you can specify which
-unit tests to run by creating a ``.kunitconfig`` file with kernel config options
-that enable only a specific set of tests and their dependencies.
-The ``.kunitconfig`` file contains a list of kconfig options which are required
-to run the desired targets. The ``.kunitconfig`` also contains any other test
-specific config options, such as test dependencies. For example: the
-``FAT_FS`` tests - ``FAT_KUNIT_TEST``, depends on
-``FAT_FS``. ``FAT_FS`` can be enabled by selecting either ``MSDOS_FS``
-or ``VFAT_FS``. To run ``FAT_KUNIT_TEST``, the ``.kunitconfig`` has:
+	There is also the same caveat mentioned in the "Build directory for
+	the kernel" section of the :doc:`admin-guide </admin-guide/README>`,
+	that is, its use, it must be used for all invocations of ``make``.
+	The good news is that it can indeed be solved by running
+	``make ARCH=um mrproper``, just be aware that this will delete the
+	current configuration and all generated files.
 
-.. code-block:: none
+If everything worked correctly, you should see the following:
 
-	CONFIG_KUNIT=y
-	CONFIG_MSDOS_FS=y
-	CONFIG_FAT_KUNIT_TEST=y
+.. code-block::
 
-1. A good starting point for the ``.kunitconfig`` is the KUnit default config.
-   You can generate it by running:
+	Configuring KUnit Kernel ...
+	Building KUnit Kernel ...
+	Starting KUnit Kernel ...
+
+The tests will pass or fail.
+
+.. note ::
+   Because it is building a lot of sources for the first time,
+   the ``Building KUnit Kernel`` step may take a while.
+
+For detailed information on this wrapper, see:
+Documentation/dev-tools/kunit/run_wrapper.rst.
+
+Selecting which tests to run
+----------------------------
+
+By default, kunit_tool runs all tests reachable with minimal configuration,
+that is, using default values for most of the kconfig options.  However,
+you can select which tests to run by:
+
+- `Customizing Kconfig`_ used to compile the kernel, or
+- `Filtering tests by name`_ to select specifically which compiled tests to run.
+
+Customizing Kconfig
+~~~~~~~~~~~~~~~~~~~
+A good starting point for the ``.kunitconfig`` is the KUnit default config.
+If you didn't run ``kunit.py run`` yet, you can generate it by running:
 
 .. code-block:: bash
 
@@ -54,48 +81,69 @@ or ``VFAT_FS``. To run ``FAT_KUNIT_TEST``, the ``.kunitconfig`` has:
    ``.kunitconfig`` lives in the ``--build_dir`` used by kunit.py, which is
    ``.kunit`` by default.
 
-.. note ::
+Before running the tests, kunit_tool ensures that all config options
+set in ``.kunitconfig`` are set in the kernel ``.config``. It will warn
+you if you have not included dependencies for the options used.
+
+There are many ways to customize the configurations:
+
+a. Edit ``.kunit/.kunitconfig``. The file should contain the list of kconfig
+   options required to run the desired tests, including their dependencies.
    You may want to remove CONFIG_KUNIT_ALL_TESTS from the ``.kunitconfig`` as
    it will enable a number of additional tests that you may not want.
+   If you need to run on an architecture other than UML see :ref:`kunit-on-qemu`.
 
-2. You can then add any other Kconfig options, for example:
+b. Enable additional kconfig options on top of ``.kunit/.kunitconfig``.
+   For example, to include the kernel's linked-list test you can run::
 
-.. code-block:: none
+	./tools/testing/kunit/kunit.py run \
+		--kconfig_add CONFIG_LIST_KUNIT_TEST=y
 
-	CONFIG_LIST_KUNIT_TEST=y
+c. Provide the path of one or more .kunitconfig files from the tree.
+   For example, to run only ``FAT_FS`` and ``EXT4`` tests you can run::
 
-Before running the tests, kunit_tool ensures that all config options
-set in ``.kunitconfig`` are set in the kernel ``.config``. It will warn
-you if you have not included dependencies for the options used.
+	./tools/testing/kunit/kunit.py run \
+		--kunitconfig ./fs/fat/.kunitconfig \
+		--kunitconfig ./fs/ext4/.kunitconfig
 
-.. note ::
-   If you change the ``.kunitconfig``, kunit.py will trigger a rebuild of the
+d. If you change the ``.kunitconfig``, kunit.py will trigger a rebuild of the
    ``.config`` file. But you can edit the ``.config`` file directly or with
    tools like ``make menuconfig O=.kunit``. As long as its a superset of
    ``.kunitconfig``, kunit.py won't overwrite your changes.
 
-Running Tests (KUnit Wrapper)
------------------------------
-1. To make sure that everything is set up correctly, invoke the Python
-   wrapper from your kernel repository:
 
-.. code-block:: bash
+.. note ::
 
-	./tools/testing/kunit/kunit.py run
+	To save a .kunitconfig after finding a satisfactory configuration::
 
-If everything worked correctly, you should see the following:
+		make savedefconfig O=.kunit
+		cp .kunit/defconfig .kunit/.kunitconfig
 
-.. code-block::
+Filtering tests by name
+~~~~~~~~~~~~~~~~~~~~~~~
+If you want to be more specific than Kconfig can provide, it is also possible
+to select which tests to execute at boot-time by passing a glob filter
+(read instructions regarding the pattern in the manpage :manpage:`glob(7)`).
+If there is a ``"."`` (period) in the filter, it will be interpreted as a
+separator between the name of the test suite and the test case,
+otherwise, it will be interpreted as the name of the test suite.
+For example, let's assume we are using the default config:
 
-	Generating .config ...
-	Building KUnit Kernel ...
-	Starting KUnit Kernel ...
+a. inform the name of a test suite, like ``"kunit_executor_test"``,
+   to run every test case it contains::
 
-The tests will pass or fail.
+	./tools/testing/kunit/kunit.py run "kunit_executor_test"
 
-.. note ::
-   Because it is building a lot of sources for the first time, the
-   ``Building KUnit kernel`` may take a while.
+b. inform the name of a test case prefixed by its test suite,
+   like ``"example.example_simple_test"``, to run specifically that test case::
+
+	./tools/testing/kunit/kunit.py run "example.example_simple_test"
+
+c. use wildcard characters (``*?[``) to run any test case that matches the pattern,
+   like ``"*.*64*"`` to run test cases containing ``"64"`` in the name inside
+   any test suite::
+
+	./tools/testing/kunit/kunit.py run "*.*64*"
 
 Running Tests without the KUnit Wrapper
 =======================================
@@ -217,7 +265,7 @@ Now we are ready to write the test cases.
 
 	obj-$(CONFIG_MISC_EXAMPLE_TEST) += example_test.o
 
-4. Add the following lines to ``.kunitconfig``:
+4. Add the following lines to ``.kunit/.kunitconfig``:
 
 .. code-block:: none
 
@@ -254,7 +302,5 @@ Next Steps
     examples.
 *   Documentation/dev-tools/kunit/api/index.rst - KUnit APIs
     used for testing.
-*   Documentation/dev-tools/kunit/kunit-tool.rst - kunit_tool helper
-    script.
 *   Documentation/dev-tools/kunit/faq.rst - KUnit common questions and
     answers.
diff --git a/Documentation/dev-tools/kunit/usage.rst b/Documentation/dev-tools/kunit/usage.rst
index 44158eecb51e..2737863ef365 100644
--- a/Documentation/dev-tools/kunit/usage.rst
+++ b/Documentation/dev-tools/kunit/usage.rst
@@ -165,6 +165,8 @@ built as a module).
 
 For more information, see Documentation/dev-tools/kunit/api/test.rst.
 
+.. _kunit-on-non-uml:
+
 Writing Tests For Other Architectures
 -------------------------------------
 
@@ -544,8 +546,6 @@ By reusing the same ``cases`` array from above, we can write the test as a
 		{}
 	};
 
-.. _kunit-on-non-uml:
-
 Exiting Early on Failed Expectations
 ------------------------------------
 
diff --git a/Documentation/devicetree/bindings/arm/amlogic.yaml b/Documentation/devicetree/bindings/arm/amlogic.yaml
index 61a6cabb375b..f5c1b6cae4c9 100644
--- a/Documentation/devicetree/bindings/arm/amlogic.yaml
+++ b/Documentation/devicetree/bindings/arm/amlogic.yaml
@@ -120,6 +120,7 @@ properties:
           - enum:
               - amlogic,q200
               - amlogic,q201
+              - azw,gt1-ultimate
               - khadas,vim2
               - kingnovel,r-box-pro
               - libretech,aml-s912-pc
@@ -136,6 +137,7 @@ properties:
           - enum:
               - amlogic,s400
               - jethome,jethub-j100
+              - jethome,jethub-j110
           - const: amlogic,a113d
           - const: amlogic,meson-axg
 
diff --git a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
index 1895ce9de461..217a1d674863 100644
--- a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
+++ b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
@@ -29,6 +29,7 @@ properties:
       - description: AST2500 based boards
         items:
           - enum:
+              - amd,daytonax-bmc
               - amd,ethanolx-bmc
               - ampere,mtjade-bmc
               - aspeed,ast2500-evb
@@ -69,6 +70,7 @@ properties:
       - description: AST2600 based boards
         items:
           - enum:
+              - ampere,mtmitchell-bmc
               - aspeed,ast2600-evb
               - aspeed,ast2600-evb-a1
               - facebook,bletchley-bmc
diff --git a/Documentation/devicetree/bindings/arm/atmel-at91.yaml b/Documentation/devicetree/bindings/arm/atmel-at91.yaml
index 2b7848bb7769..9e2e66a7566d 100644
--- a/Documentation/devicetree/bindings/arm/atmel-at91.yaml
+++ b/Documentation/devicetree/bindings/arm/atmel-at91.yaml
@@ -127,6 +127,13 @@ properties:
           - const: atmel,sama5d3
           - const: atmel,sama5
 
+      - description: Microchip SAMA5D3 Ethernet Development System Board
+        items:
+          - const: microchip,sama5d3-eds
+          - const: atmel,sama5d36
+          - const: atmel,sama5d3
+          - const: atmel,sama5
+
       - description: CalAmp LMU5000 board
         items:
           - const: calamp,lmu5000
diff --git a/Documentation/devicetree/bindings/arm/bcm/brcm,bcm4908.yaml b/Documentation/devicetree/bindings/arm/bcm/brcm,bcm4908.yaml
deleted file mode 100644
index 9b745531ff04..000000000000
--- a/Documentation/devicetree/bindings/arm/bcm/brcm,bcm4908.yaml
+++ /dev/null
@@ -1,42 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/arm/bcm/brcm,bcm4908.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Broadcom BCM4908 device tree bindings
-
-description:
-  Broadcom BCM4906 / BCM4908 / BCM49408 Wi-Fi/network SoCs with Brahma CPUs.
-
-maintainers:
-  - Rafał Miłecki <rafal@milecki.pl>
-
-properties:
-  $nodename:
-    const: '/'
-  compatible:
-    oneOf:
-      - description: BCM4906 based boards
-        items:
-          - enum:
-              - netgear,r8000p
-              - tplink,archer-c2300-v1
-          - const: brcm,bcm4906
-          - const: brcm,bcm4908
-
-      - description: BCM4908 based boards
-        items:
-          - enum:
-              - asus,gt-ac5300
-              - netgear,raxe500
-          - const: brcm,bcm4908
-
-      - description: BCM49408 based boards
-        items:
-          - const: brcm,bcm49408
-          - const: brcm,bcm4908
-
-additionalProperties: true
-
-...
diff --git a/Documentation/devicetree/bindings/arm/bcm/brcm,bcmbca.yaml b/Documentation/devicetree/bindings/arm/bcm/brcm,bcmbca.yaml
index 324e59104360..84866e29cab0 100644
--- a/Documentation/devicetree/bindings/arm/bcm/brcm,bcmbca.yaml
+++ b/Documentation/devicetree/bindings/arm/bcm/brcm,bcmbca.yaml
@@ -15,6 +15,7 @@ maintainers:
   - William Zhang <william.zhang@broadcom.com>
   - Anand Gore <anand.gore@broadcom.com>
   - Kursad Oney <kursad.oney@broadcom.com>
+  - Rafał Miłecki <rafal@milecki.pl>
 
 properties:
   $nodename:
@@ -28,6 +29,30 @@ properties:
           - const: brcm,bcm47622
           - const: brcm,bcmbca
 
+      - description: BCM4906 based boards
+        items:
+          - enum:
+              - netgear,r8000p
+              - tplink,archer-c2300-v1
+          - const: brcm,bcm4906
+          - const: brcm,bcm4908
+          - const: brcm,bcmbca
+
+      - description: BCM4908 based boards
+        items:
+          - enum:
+              - asus,gt-ac5300
+              - brcm,bcm94908
+              - netgear,raxe500
+          - const: brcm,bcm4908
+          - const: brcm,bcmbca
+
+      - description: BCM49408 based boards
+        items:
+          - const: brcm,bcm49408
+          - const: brcm,bcm4908
+          - const: brcm,bcmbca
+
       - description: BCM4912 based boards
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt b/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt
index 104cc9b41df4..071421dbc4d0 100644
--- a/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt
+++ b/Documentation/devicetree/bindings/arm/bcm/brcm,brcmstb.txt
@@ -187,15 +187,8 @@ Required properties:
 Sequencer DRAM parameters and control registers. Used for Self-Refresh
 Power-Down (SRPD), among other things.
 
-Required properties:
-- compatible     : should contain one of these
-	"brcm,brcmstb-memc-ddr-rev-b.2.1"
-	"brcm,brcmstb-memc-ddr-rev-b.2.2"
-	"brcm,brcmstb-memc-ddr-rev-b.2.3"
-	"brcm,brcmstb-memc-ddr-rev-b.3.0"
-	"brcm,brcmstb-memc-ddr-rev-b.3.1"
-	"brcm,brcmstb-memc-ddr"
-- reg            : the MEMC DDR register range
+See Documentation/devicetree/bindings/memory-controllers/brcm,brcmstb-memc-ddr.yaml for a
+full list of supported compatible strings and properties.
 
 Example:
 
diff --git a/Documentation/devicetree/bindings/arm/fsl.yaml b/Documentation/devicetree/bindings/arm/fsl.yaml
index 7431579ab0e8..0cea264a9de2 100644
--- a/Documentation/devicetree/bindings/arm/fsl.yaml
+++ b/Documentation/devicetree/bindings/arm/fsl.yaml
@@ -554,8 +554,7 @@ properties:
               - engicam,imx6ul-isiot      # Engicam Is.IoT MX6UL eMMC/NAND Starter kit
               - fsl,imx6ul-14x14-evk      # i.MX6 UltraLite 14x14 EVK Board
               - karo,imx6ul-tx6ul         # Ka-Ro electronics TXUL-0010 Module
-              - kontron,imx6ul-n6310-som  # Kontron N6310 SOM
-              - kontron,imx6ul-n6311-som  # Kontron N6311 SOM
+              - kontron,sl-imx6ul         # Kontron SL i.MX6UL SoM
               - prt,prti6g                # Protonic PRTI6G Board
               - technexion,imx6ul-pico-dwarf   # TechNexion i.MX6UL Pico-Dwarf
               - technexion,imx6ul-pico-hobbit  # TechNexion i.MX6UL Pico-Hobbit
@@ -591,23 +590,17 @@ properties:
           - const: phytec,imx6ul-pcl063   # PHYTEC phyCORE-i.MX 6UL
           - const: fsl,imx6ul
 
-      - description: Kontron N6310 S Board
+      - description: Kontron BL i.MX6UL (N631X S) Board
         items:
-          - const: kontron,imx6ul-n6310-s
-          - const: kontron,imx6ul-n6310-som
+          - const: kontron,bl-imx6ul       # Kontron BL i.MX6UL Carrier Board
+          - const: kontron,sl-imx6ul       # Kontron SL i.MX6UL SoM
           - const: fsl,imx6ul
 
-      - description: Kontron N6311 S Board
+      - description: Kontron BL i.MX6UL 43 (N631X S 43) Board
         items:
-          - const: kontron,imx6ul-n6311-s
-          - const: kontron,imx6ul-n6311-som
-          - const: fsl,imx6ul
-
-      - description: Kontron N6310 S 43 Board
-        items:
-          - const: kontron,imx6ul-n6310-s-43
-          - const: kontron,imx6ul-n6310-s
-          - const: kontron,imx6ul-n6310-som
+          - const: kontron,bl-imx6ul-43    # Kontron BL i.MX6UL Carrier Board with 4.3" Display
+          - const: kontron,bl-imx6ul       # Kontron BL i.MX6UL Carrier Board
+          - const: kontron,sl-imx6ul       # Kontron SL i.MX6UL SoM
           - const: fsl,imx6ul
 
       - description: TQ-Systems TQMa6UL1 SoM on MBa6ULx board
@@ -637,7 +630,7 @@ properties:
           - enum:
               - fsl,imx6ull-14x14-evk     # i.MX6 UltraLiteLite 14x14 EVK Board
               - joz,jozacp                # JOZ Access Point
-              - kontron,imx6ull-n6411-som # Kontron N6411 SOM
+              - kontron,sl-imx6ull        # Kontron SL i.MX6ULL SoM
               - myir,imx6ull-mys-6ulx-eval # MYiR Tech iMX6ULL Evaluation Board
               - toradex,colibri-imx6ull      # Colibri iMX6ULL Modules
               - toradex,colibri-imx6ull-emmc # Colibri iMX6ULL 1GB (eMMC) Module
@@ -698,10 +691,10 @@ properties:
           - const: toradex,colibri-imx6ull-wifi       # Colibri iMX6ULL Wi-Fi / BT Module
           - const: fsl,imx6ull
 
-      - description: Kontron N6411 S Board
+      - description: Kontron BL i.MX6ULL (N6411 S) Board
         items:
-          - const: kontron,imx6ull-n6411-s
-          - const: kontron,imx6ull-n6411-som
+          - const: kontron,bl-imx6ull   # Kontron BL i.MX6ULL Carrier Board
+          - const: kontron,sl-imx6ull   # Kontron SL i.MX6ULL SoM
           - const: fsl,imx6ull
 
       - description: TQ Systems TQMa6ULLx SoM on MBa6ULx board
@@ -825,13 +818,15 @@ properties:
               - emtrion,emcon-mx8mm-avari # emCON-MX8MM SoM on Avari Base
               - fsl,imx8mm-ddr4-evk       # i.MX8MM DDR4 EVK Board
               - fsl,imx8mm-evk            # i.MX8MM EVK Board
+              - gateworks,imx8mm-gw7904
               - gw,imx8mm-gw71xx-0x       # i.MX8MM Gateworks Development Kit
               - gw,imx8mm-gw72xx-0x       # i.MX8MM Gateworks Development Kit
               - gw,imx8mm-gw73xx-0x       # i.MX8MM Gateworks Development Kit
               - gw,imx8mm-gw7901          # i.MX8MM Gateworks Board
               - gw,imx8mm-gw7902          # i.MX8MM Gateworks Board
               - gw,imx8mm-gw7903          # i.MX8MM Gateworks Board
-              - kontron,imx8mm-n801x-som  # i.MX8MM Kontron SL (N801X) SOM
+              - kontron,imx8mm-sl         # i.MX8MM Kontron SL (N801X) SOM
+              - kontron,imx8mm-osm-s      # i.MX8MM Kontron OSM-S (N802X) SOM
               - menlo,mx8menlo            # i.MX8MM Menlo board with Verdin SoM
               - toradex,verdin-imx8mm     # Verdin iMX8M Mini Modules
               - toradex,verdin-imx8mm-nonwifi  # Verdin iMX8M Mini Modules without Wi-Fi / BT
@@ -850,8 +845,14 @@ properties:
 
       - description: Kontron BL i.MX8MM (N801X S) Board
         items:
-          - const: kontron,imx8mm-n801x-s
-          - const: kontron,imx8mm-n801x-som
+          - const: kontron,imx8mm-bl
+          - const: kontron,imx8mm-sl
+          - const: fsl,imx8mm
+
+      - description: Kontron BL i.MX8MM OSM-S (N802X S) Board
+        items:
+          - const: kontron,imx8mm-bl-osm-s
+          - const: kontron,imx8mm-osm-s
           - const: fsl,imx8mm
 
       - description: Toradex Boards with Verdin iMX8M Mini Modules
@@ -936,6 +937,13 @@ properties:
               - toradex,verdin-imx8mp-wifi  # Verdin iMX8M Plus Wi-Fi / BT Modules
           - const: fsl,imx8mp
 
+      - description: Avnet (MSC Branded) Boards with SM2S i.MX8M Plus Modules
+        items:
+          - const: avnet,sm2s-imx8mp-14N0600E-ep1 # SM2S-IMX8PLUS-14N0600E on SM2-MB-EP1 Carrier Board
+          - const: avnet,sm2s-imx8mp-14N0600E     # 14N0600E variant of SM2S-IMX8PLUS SoM
+          - const: avnet,sm2s-imx8mp              # SM2S-IMX8PLUS SoM
+          - const: fsl,imx8mp
+
       - description: Engicam i.Core MX8M Plus SoM based boards
         items:
           - enum:
@@ -1034,6 +1042,12 @@ properties:
               - toradex,colibri-imx8x     # Colibri iMX8X Modules
           - const: fsl,imx8qxp
 
+      - description: i.MX8DXL based Boards
+        items:
+          - enum:
+              - fsl,imx8dxl-evk           # i.MX8DXL EVK Board
+          - const: fsl,imx8dxl
+
       - description: i.MX8QXP Boards with Toradex Coilbri iMX8X Modules
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt b/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt
index f6d6642d81c0..29fa93dad52b 100644
--- a/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt
+++ b/Documentation/devicetree/bindings/arm/marvell/armada-37xx.txt
@@ -1,21 +1,3 @@
-Marvell Armada 37xx Platforms Device Tree Bindings
---------------------------------------------------
-
-Boards using a SoC of the Marvell Armada 37xx family must carry the
-following root node property:
-
- - compatible: must contain "marvell,armada3710"
-
-In addition, boards using the Marvell Armada 3720 SoC shall have the
-following property before the previous one:
-
- - compatible: must contain "marvell,armada3720"
-
-Example:
-
-compatible = "marvell,armada-3720-db", "marvell,armada3720", "marvell,armada3710";
-
-
 Power management
 ----------------
 
@@ -48,11 +30,3 @@ avs: avs@11500 {
 	compatible = "marvell,armada-3700-avs", "syscon";
 	reg = <0x11500 0x40>;
 }
-
-
-CZ.NIC's Turris Mox SOHO router Device Tree Bindings
-----------------------------------------------------
-
-Required root node property:
-
- - compatible: must contain "cznic,turris-mox"
diff --git a/Documentation/devicetree/bindings/arm/marvell/armada-37xx.yaml b/Documentation/devicetree/bindings/arm/marvell/armada-37xx.yaml
new file mode 100644
index 000000000000..6905d29f3108
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/marvell/armada-37xx.yaml
@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/arm/marvell/armada-37xx.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Marvell Armada 37xx Platforms
+
+maintainers:
+  - Robert Marko <robert.marko@sartura.hr>
+
+properties:
+  $nodename:
+    const: '/'
+  compatible:
+    oneOf:
+      - description: Armada 3720 SoC boards
+        items:
+          - enum:
+              - cznic,turris-mox
+              - globalscale,espressobin
+              - marvell,armada-3720-db
+              - methode,edpu
+              - methode,udpu
+          - const: marvell,armada3720
+          - const: marvell,armada3710
+
+      - description: Globalscale Espressobin boards
+        items:
+          - enum:
+              - globalscale,espressobin-emmc
+              - globalscale,espressobin-ultra
+              - globalscale,espressobin-v7
+          - const: globalscale,espressobin
+          - const: marvell,armada3720
+          - const: marvell,armada3710
+
+      - description: Globalscale Espressobin V7 boards
+        items:
+          - enum:
+              - globalscale,espressobin-v7-emmc
+          - const: globalscale,espressobin-v7
+          - const: globalscale,espressobin
+          - const: marvell,armada3720
+          - const: marvell,armada3710
+
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml
index 787d6673f952..84fb0a146b6e 100644
--- a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7622-wed.yaml
@@ -20,6 +20,7 @@ properties:
     items:
       - enum:
           - mediatek,mt7622-wed
+          - mediatek,mt7986-wed
       - const: syscon
 
   reg:
diff --git a/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7986-wed-pcie.yaml b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7986-wed-pcie.yaml
new file mode 100644
index 000000000000..96221f51c1c3
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/mediatek/mediatek,mt7986-wed-pcie.yaml
@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: "http://devicetree.org/schemas/arm/mediatek/mediatek,mt7986-wed-pcie.yaml#"
+$schema: "http://devicetree.org/meta-schemas/core.yaml#"
+
+title: MediaTek PCIE WED Controller for MT7986
+
+maintainers:
+  - Lorenzo Bianconi <lorenzo@kernel.org>
+  - Felix Fietkau <nbd@nbd.name>
+
+description:
+  The mediatek WED PCIE provides a configuration interface for PCIE
+  controller on MT7986 soc.
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - mediatek,mt7986-wed-pcie
+      - const: syscon
+
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    soc {
+      #address-cells = <2>;
+      #size-cells = <2>;
+      wed_pcie: wed-pcie@10003000 {
+        compatible = "mediatek,mt7986-wed-pcie",
+                     "syscon";
+        reg = <0 0x10003000 0 0x10>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/arm/qcom.yaml b/Documentation/devicetree/bindings/arm/qcom.yaml
index fb1d00bcc847..b6257683a700 100644
--- a/Documentation/devicetree/bindings/arm/qcom.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom.yaml
@@ -176,6 +176,9 @@ properties:
               - longcheer,l8910
               - samsung,a3u-eur
               - samsung,a5u-eur
+              - samsung,e5
+              - samsung,e7
+              - samsung,grandmax
               - samsung,j5
               - samsung,serranove
               - wingtech,wt88047
@@ -450,6 +453,7 @@ properties:
 
       - description: Google Pazquel with LTE and Parade (newest rev)
         items:
+          - const: google,pazquel-sku6
           - const: google,pazquel-sku4
           - const: qcom,sc7180
 
@@ -550,6 +554,7 @@ properties:
 
       - description: Qualcomm Technologies, Inc. sc7280 CRD platform (newest rev)
         items:
+          - const: google,zoglin
           - const: google,hoglin
           - const: qcom,sc7280
 
@@ -565,16 +570,31 @@ properties:
           - const: google,piglin
           - const: qcom,sc7280
 
+      - description: Google Evoker (newest rev)
+        items:
+          - const: google,evoker
+          - const: qcom,sc7280
+
       - description: Google Herobrine (newest rev)
         items:
           - const: google,herobrine
           - const: qcom,sc7280
 
+      - description: Google Villager (rev0)
+        items:
+          - const: google,villager-rev0
+          - const: qcom,sc7280
+
       - description: Google Villager (newest rev)
         items:
           - const: google,villager
           - const: qcom,sc7280
 
+      - description: Google Villager with LTE (newest rev)
+        items:
+          - const: google,villager-sku512
+          - const: qcom,sc7280
+
       - items:
           - enum:
               - lenovo,flex-5g
@@ -716,6 +736,7 @@ properties:
           - enum:
               - qcom,sm8450-hdk
               - qcom,sm8450-qrd
+              - sony,pdx223
           - const: qcom,sm8450
 
 additionalProperties: true
diff --git a/Documentation/devicetree/bindings/arm/renesas.yaml b/Documentation/devicetree/bindings/arm/renesas.yaml
index ff80152f092f..2fc81c776a06 100644
--- a/Documentation/devicetree/bindings/arm/renesas.yaml
+++ b/Documentation/devicetree/bindings/arm/renesas.yaml
@@ -264,6 +264,7 @@ properties:
                   - renesas,r8a779m4
                   - renesas,r8a779m5
                   - renesas,r8a779m8
+                  - renesas,r8a779mb
               - enum:
                   - renesas,r8a7795
                   - renesas,r8a77961
@@ -291,6 +292,13 @@ properties:
               - renesas,v3hsk # V3HSK (Y-ASK-RCAR-V3H-WS10)
           - const: renesas,r8a77980
 
+      - description: R-Car V3H2 (R8A77980A)
+        items:
+          - enum:
+              - renesas,condor-i # Condor-I (RTP0RC77980SEBS012SA01)
+          - const: renesas,r8a77980a
+          - const: renesas,r8a77980
+
       - description: R-Car E3 (R8A77990)
         items:
           - enum:
@@ -409,6 +417,14 @@ properties:
           - const: renesas,r8a779m8
           - const: renesas,r8a7795
 
+      - description: R-Car H3Ne-1.7G (R8A779MB)
+        items:
+          - enum:
+              - renesas,h3ulcb      # H3ULCB (R-Car Starter Kit Premier)
+              - renesas,salvator-xs # Salvator-XS (Salvator-X 2nd version)
+          - const: renesas,r8a779mb
+          - const: renesas,r8a7795
+
       - description: RZ/N1D (R9A06G032)
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/rockchip.yaml b/Documentation/devicetree/bindings/arm/rockchip.yaml
index 7811ba64149c..4c64d9ff089c 100644
--- a/Documentation/devicetree/bindings/arm/rockchip.yaml
+++ b/Documentation/devicetree/bindings/arm/rockchip.yaml
@@ -30,6 +30,16 @@ properties:
           - const: amarula,vyasa-rk3288
           - const: rockchip,rk3288
 
+      - description: Anbernic RG353P
+        items:
+          - const: anbernic,rg353p
+          - const: rockchip,rk3566
+
+      - description: Anbernic RG503
+        items:
+          - const: anbernic,rg503
+          - const: rockchip,rk3566
+
       - description: Asus Tinker board
         items:
           - const: asus,rk3288-tinker
@@ -151,6 +161,7 @@ properties:
               - friendlyarm,nanopi-m4b
               - friendlyarm,nanopi-neo4
               - friendlyarm,nanopi-r4s
+              - friendlyarm,nanopi-r4s-enterprise
           - const: rockchip,rk3399
 
       - description: GeekBuying GeekBox
@@ -363,30 +374,55 @@ properties:
           - const: google,gru
           - const: rockchip,rk3399
 
-      - description: Google Scarlet - Innolux display (Acer Chromebook Tab 10)
+      - description: |
+          Google Scarlet - Innolux display (Acer Chromebook Tab 10 and more)
         items:
+          - const: google,scarlet-rev15-sku2
+          - const: google,scarlet-rev15-sku4
           - const: google,scarlet-rev15-sku6
           - const: google,scarlet-rev15
+          - const: google,scarlet-rev14-sku2
+          - const: google,scarlet-rev14-sku4
           - const: google,scarlet-rev14-sku6
           - const: google,scarlet-rev14
+          - const: google,scarlet-rev13-sku2
+          - const: google,scarlet-rev13-sku4
           - const: google,scarlet-rev13-sku6
           - const: google,scarlet-rev13
+          - const: google,scarlet-rev12-sku2
+          - const: google,scarlet-rev12-sku4
           - const: google,scarlet-rev12-sku6
           - const: google,scarlet-rev12
+          - const: google,scarlet-rev11-sku2
+          - const: google,scarlet-rev11-sku4
           - const: google,scarlet-rev11-sku6
           - const: google,scarlet-rev11
+          - const: google,scarlet-rev10-sku2
+          - const: google,scarlet-rev10-sku4
           - const: google,scarlet-rev10-sku6
           - const: google,scarlet-rev10
+          - const: google,scarlet-rev9-sku2
+          - const: google,scarlet-rev9-sku4
           - const: google,scarlet-rev9-sku6
           - const: google,scarlet-rev9
+          - const: google,scarlet-rev8-sku2
+          - const: google,scarlet-rev8-sku4
           - const: google,scarlet-rev8-sku6
           - const: google,scarlet-rev8
+          - const: google,scarlet-rev7-sku2
+          - const: google,scarlet-rev7-sku4
           - const: google,scarlet-rev7-sku6
           - const: google,scarlet-rev7
+          - const: google,scarlet-rev6-sku2
+          - const: google,scarlet-rev6-sku4
           - const: google,scarlet-rev6-sku6
           - const: google,scarlet-rev6
+          - const: google,scarlet-rev5-sku2
+          - const: google,scarlet-rev5-sku4
           - const: google,scarlet-rev5-sku6
           - const: google,scarlet-rev5
+          - const: google,scarlet-rev4-sku2
+          - const: google,scarlet-rev4-sku4
           - const: google,scarlet-rev4-sku6
           - const: google,scarlet-rev4
           - const: google,scarlet
@@ -470,6 +506,11 @@ properties:
           - const: netxeon,r89
           - const: rockchip,rk3288
 
+      - description: OPEN AI LAB EAIDK-610
+        items:
+          - const: openailab,eaidk-610
+          - const: rockchip,rk3399
+
       - description: Orange Pi RK3399 board
         items:
           - const: rockchip,rk3399-orangepi
@@ -494,6 +535,11 @@ properties:
           - const: pine64,pinenote
           - const: rockchip,rk3566
 
+      - description: Pine64 PinePhonePro
+        items:
+          - const: pine64,pinephone-pro
+          - const: rockchip,rk3399
+
       - description: Pine64 Rock64
         items:
           - const: pine64,rock64
@@ -537,6 +583,11 @@ properties:
           - const: radxa,rockpi4
           - const: rockchip,rk3399
 
+      - description: Radxa ROCK 4C+
+        items:
+          - const: radxa,rock-4c-plus
+          - const: rockchip,rk3399
+
       - description: Radxa ROCK Pi E
         items:
           - const: radxa,rockpi-e
diff --git a/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml b/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml
index 5ece38065e54..8c73bc7f4009 100644
--- a/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml
+++ b/Documentation/devicetree/bindings/arm/rockchip/pmu.yaml
@@ -21,10 +21,12 @@ select:
         enum:
           - rockchip,px30-pmu
           - rockchip,rk3066-pmu
+          - rockchip,rk3128-pmu
           - rockchip,rk3288-pmu
           - rockchip,rk3368-pmu
           - rockchip,rk3399-pmu
           - rockchip,rk3568-pmu
+          - rockchip,rk3588-pmu
 
   required:
     - compatible
@@ -35,10 +37,12 @@ properties:
       - enum:
           - rockchip,px30-pmu
           - rockchip,rk3066-pmu
+          - rockchip,rk3128-pmu
           - rockchip,rk3288-pmu
           - rockchip,rk3368-pmu
           - rockchip,rk3399-pmu
           - rockchip,rk3568-pmu
+          - rockchip,rk3588-pmu
       - const: syscon
       - const: simple-mfd
 
diff --git a/Documentation/devicetree/bindings/arm/ti/k3.yaml b/Documentation/devicetree/bindings/arm/ti/k3.yaml
index 61c6ab4f52e2..28b8232e1c5b 100644
--- a/Documentation/devicetree/bindings/arm/ti/k3.yaml
+++ b/Documentation/devicetree/bindings/arm/ti/k3.yaml
@@ -19,32 +19,11 @@ properties:
   compatible:
     oneOf:
 
-      - description: K3 AM654 SoC
+      - description: K3 AM62A7 SoC
         items:
           - enum:
-              - ti,am654-evm
-              - siemens,iot2050-basic
-              - siemens,iot2050-basic-pg2
-              - siemens,iot2050-advanced
-              - siemens,iot2050-advanced-pg2
-          - const: ti,am654
-
-      - description: K3 J721E SoC
-        oneOf:
-          - const: ti,j721e
-          - items:
-              - enum:
-                  - ti,j721e-evm
-                  - ti,j721e-sk
-              - const: ti,j721e
-
-      - description: K3 J7200 SoC
-        oneOf:
-          - const: ti,j7200
-          - items:
-              - enum:
-                  - ti,j7200-evm
-              - const: ti,j7200
+              - ti,am62a7-sk
+          - const: ti,am62a7
 
       - description: K3 AM625 SoC
         items:
@@ -59,6 +38,33 @@ properties:
               - ti,am642-sk
           - const: ti,am642
 
+      - description: K3 AM654 SoC
+        items:
+          - enum:
+              - siemens,iot2050-advanced
+              - siemens,iot2050-advanced-pg2
+              - siemens,iot2050-basic
+              - siemens,iot2050-basic-pg2
+              - ti,am654-evm
+          - const: ti,am654
+
+      - description: K3 J7200 SoC
+        oneOf:
+          - const: ti,j7200
+          - items:
+              - enum:
+                  - ti,j7200-evm
+              - const: ti,j7200
+
+      - description: K3 J721E SoC
+        oneOf:
+          - const: ti,j721e
+          - items:
+              - enum:
+                  - ti,j721e-evm
+                  - ti,j721e-sk
+              - const: ti,j721e
+
       - description: K3 J721s2 SoC
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscc.yaml b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscc.yaml
index 47028d7b98e4..633887dc2f8a 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscc.yaml
@@ -36,13 +36,11 @@ properties:
     items:
       - description: LPASS qdsp6ss register
       - description: LPASS top-cc register
-      - description: LPASS cc register
 
   reg-names:
     items:
       - const: qdsp6ss
       - const: top_cc
-      - const: cc
 
 required:
   - compatible
@@ -59,8 +57,8 @@ examples:
     #include <dt-bindings/clock/qcom,lpass-sc7280.h>
     clock-controller@3000000 {
       compatible = "qcom,sc7280-lpasscc";
-      reg = <0x03000000 0x40>, <0x03c04000 0x4>, <0x03389000 0x24>;
-      reg-names = "qdsp6ss", "top_cc", "cc";
+      reg = <0x03000000 0x40>, <0x03c04000 0x4>;
+      reg-names = "qdsp6ss", "top_cc";
       clocks = <&gcc GCC_CFG_NOC_LPASS_CLK>;
       clock-names = "iface";
       #clock-cells = <1>;
diff --git a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml
index bad9135489de..f50e284e5f46 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sc7280-lpasscorecc.yaml
@@ -22,6 +22,8 @@ properties:
 
   clock-names: true
 
+  reg: true
+
   compatible:
     enum:
       - qcom,sc7280-lpassaoncc
@@ -38,8 +40,14 @@ properties:
   '#power-domain-cells':
     const: 1
 
-  reg:
-    maxItems: 1
+  '#reset-cells':
+    const: 1
+
+  qcom,adsp-pil-mode:
+    description:
+      Indicates if the LPASS would be brought out of reset using
+      peripheral loader.
+    type: boolean
 
 required:
   - compatible
@@ -69,6 +77,11 @@ allOf:
           items:
             - const: bi_tcxo
             - const: lpass_aon_cc_main_rcg_clk_src
+
+        reg:
+          items:
+            - description: lpass core cc register
+            - description: lpass audio csr register
   - if:
       properties:
         compatible:
@@ -90,6 +103,8 @@ allOf:
             - const: bi_tcxo_ao
             - const: iface
 
+        reg:
+          maxItems: 1
   - if:
       properties:
         compatible:
@@ -108,6 +123,8 @@ allOf:
           items:
             - const: bi_tcxo
 
+        reg:
+          maxItems: 1
 examples:
   - |
     #include <dt-bindings/clock/qcom,rpmh.h>
@@ -116,13 +133,15 @@ examples:
     #include <dt-bindings/clock/qcom,lpasscorecc-sc7280.h>
     lpass_audiocc: clock-controller@3300000 {
       compatible = "qcom,sc7280-lpassaudiocc";
-      reg = <0x3300000 0x30000>;
+      reg = <0x3300000 0x30000>,
+            <0x32a9000 0x1000>;
       clocks = <&rpmhcc RPMH_CXO_CLK>,
                <&lpass_aon LPASS_AON_CC_MAIN_RCG_CLK_SRC>;
       clock-names = "bi_tcxo", "lpass_aon_cc_main_rcg_clk_src";
       power-domains = <&lpass_aon LPASS_AON_CC_LPASS_AUDIO_HM_GDSC>;
       #clock-cells = <1>;
       #power-domain-cells = <1>;
+      #reset-cells = <1>;
     };
 
   - |
@@ -165,6 +184,7 @@ examples:
       clocks = <&rpmhcc RPMH_CXO_CLK>, <&rpmhcc RPMH_CXO_CLK_A>,
                <&lpasscore LPASS_CORE_CC_CORE_CLK>;
       clock-names = "bi_tcxo", "bi_tcxo_ao","iface";
+      qcom,adsp-pil-mode;
       #clock-cells = <1>;
       #power-domain-cells = <1>;
     };
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml
index aa11815ad3a3..141cf173f87d 100644
--- a/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,exynos850-clock.yaml
@@ -33,10 +33,13 @@ properties:
     enum:
       - samsung,exynos850-cmu-top
       - samsung,exynos850-cmu-apm
+      - samsung,exynos850-cmu-aud
       - samsung,exynos850-cmu-cmgp
       - samsung,exynos850-cmu-core
       - samsung,exynos850-cmu-dpu
       - samsung,exynos850-cmu-hsi
+      - samsung,exynos850-cmu-is
+      - samsung,exynos850-cmu-mfcmscl
       - samsung,exynos850-cmu-peri
 
   clocks:
@@ -92,6 +95,24 @@ allOf:
       properties:
         compatible:
           contains:
+            const: samsung,exynos850-cmu-aud
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (26 MHz)
+            - description: AUD clock (from CMU_TOP)
+
+        clock-names:
+          items:
+            - const: oscclk
+            - const: dout_aud
+
+  - if:
+      properties:
+        compatible:
+          contains:
             const: samsung,exynos850-cmu-cmgp
 
     then:
@@ -176,6 +197,54 @@ allOf:
       properties:
         compatible:
           contains:
+            const: samsung,exynos850-cmu-is
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (26 MHz)
+            - description: CMU_IS bus clock (from CMU_TOP)
+            - description: Image Texture Processing core clock (from CMU_TOP)
+            - description: Visual Recognition Accelerator clock (from CMU_TOP)
+            - description: Geometric Distortion Correction clock (from CMU_TOP)
+
+        clock-names:
+          items:
+            - const: oscclk
+            - const: dout_is_bus
+            - const: dout_is_itp
+            - const: dout_is_vra
+            - const: dout_is_gdc
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: samsung,exynos850-cmu-mfcmscl
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (26 MHz)
+            - description: Multi-Format Codec clock (from CMU_TOP)
+            - description: Memory to Memory Scaler clock (from CMU_TOP)
+            - description: Multi-Channel Scaler clock (from CMU_TOP)
+            - description: JPEG codec clock (from CMU_TOP)
+
+        clock-names:
+          items:
+            - const: oscclk
+            - const: dout_mfcmscl_mfc
+            - const: dout_mfcmscl_m2m
+            - const: dout_mfcmscl_mcsc
+            - const: dout_mfcmscl_jpeg
+
+  - if:
+      properties:
+        compatible:
+          contains:
             const: samsung,exynos850-cmu-peri
 
     then:
diff --git a/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml b/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml
index eafc715d2d02..2ab4642679c0 100644
--- a/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml
+++ b/Documentation/devicetree/bindings/clock/samsung,exynosautov9-clock.yaml
@@ -35,6 +35,8 @@ properties:
       - samsung,exynosautov9-cmu-top
       - samsung,exynosautov9-cmu-busmc
       - samsung,exynosautov9-cmu-core
+      - samsung,exynosautov9-cmu-fsys0
+      - samsung,exynosautov9-cmu-fsys1
       - samsung,exynosautov9-cmu-fsys2
       - samsung,exynosautov9-cmu-peric0
       - samsung,exynosautov9-cmu-peric1
@@ -111,6 +113,48 @@ allOf:
       properties:
         compatible:
           contains:
+            const: samsung,exynosautov9-cmu-fsys0
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (26 MHz)
+            - description: CMU_FSYS0 bus clock (from CMU_TOP)
+            - description: CMU_FSYS0 pcie clock (from CMU_TOP)
+
+        clock-names:
+          items:
+            - const: oscclk
+            - const: dout_clkcmu_fsys0_bus
+            - const: dout_clkcmu_fsys0_pcie
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: samsung,exynosautov9-cmu-fsys1
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: External reference clock (26 MHz)
+            - description: CMU_FSYS1 bus clock (from CMU_TOP)
+            - description: CMU_FSYS1 mmc card clock (from CMU_TOP)
+            - description: CMU_FSYS1 usb clock (from CMU_TOP)
+
+        clock-names:
+          items:
+            - const: oscclk
+            - const: dout_clkcmu_fsys1_bus
+            - const: dout_clkcmu_fsys1_mmc_card
+            - const: dout_clkcmu_fsys1_usbdrd
+
+  - if:
+      properties:
+        compatible:
+          contains:
             const: samsung,exynosautov9-cmu-fsys2
 
     then:
diff --git a/Documentation/devicetree/bindings/display/bridge/chipone,icn6211.yaml b/Documentation/devicetree/bindings/display/bridge/chipone,icn6211.yaml
index 4f0b7c71313c..5fb54375aeb6 100644
--- a/Documentation/devicetree/bindings/display/bridge/chipone,icn6211.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/chipone,icn6211.yaml
@@ -24,6 +24,15 @@ properties:
     maxItems: 1
     description: virtual channel number of a DSI peripheral
 
+  clock-names:
+    const: refclk
+
+  clocks:
+    maxItems: 1
+    description: |
+        Optional external clock connected to REF_CLK input.
+        The clock rate must be in 10..154 MHz range.
+
   enable-gpios:
     description: Bridge EN pin, chip is reset when EN is low.
 
diff --git a/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.yaml b/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.yaml
index 0c9785c8db51..e3ec697f89e7 100644
--- a/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/renesas,dw-hdmi.yaml
@@ -38,6 +38,9 @@ properties:
   clock-names:
     maxItems: 2
 
+  resets:
+    maxItems: 1
+
   ports:
     $ref: /schemas/graph.yaml#/properties/ports
 
@@ -67,6 +70,7 @@ required:
   - reg
   - clocks
   - clock-names
+  - resets
   - interrupts
   - ports
 
@@ -85,6 +89,7 @@ examples:
         clocks = <&cpg CPG_CORE R8A7795_CLK_S0D4>, <&cpg CPG_MOD 729>;
         clock-names = "iahb", "isfr";
         power-domains = <&sysc R8A7795_PD_ALWAYS_ON>;
+        resets = <&cpg 729>;
 
         ports {
             #address-cells = <1>;
diff --git a/Documentation/devicetree/bindings/display/ingenic,lcd.yaml b/Documentation/devicetree/bindings/display/ingenic,lcd.yaml
index 0049010b37ca..c0bb02fb49f4 100644
--- a/Documentation/devicetree/bindings/display/ingenic,lcd.yaml
+++ b/Documentation/devicetree/bindings/display/ingenic,lcd.yaml
@@ -17,6 +17,8 @@ properties:
     enum:
       - ingenic,jz4740-lcd
       - ingenic,jz4725b-lcd
+      - ingenic,jz4760-lcd
+      - ingenic,jz4760b-lcd
       - ingenic,jz4770-lcd
       - ingenic,jz4780-lcd
 
diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,dp.yaml b/Documentation/devicetree/bindings/display/mediatek/mediatek,dp.yaml
new file mode 100644
index 000000000000..ff781f2174a0
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,dp.yaml
@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/mediatek/mediatek,dp.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek Display Port Controller
+
+maintainers:
+  - Chun-Kuang Hu <chunkuang.hu@kernel.org>
+  - Jitao shi <jitao.shi@mediatek.com>
+
+description: |
+  MediaTek DP and eDP are different hardwares and there are some features
+  which are not supported for eDP. For example, audio is not supported for
+  eDP. Therefore, we need to use two different compatibles to describe them.
+  In addition, We just need to enable the power domain of DP, so the clock
+  of DP is generated by itself and we are not using other PLL to generate
+  clocks.
+
+properties:
+  compatible:
+    enum:
+      - mediatek,mt8195-dp-tx
+      - mediatek,mt8195-edp-tx
+
+  reg:
+    maxItems: 1
+
+  nvmem-cells:
+    maxItems: 1
+    description: efuse data for display port calibration
+
+  nvmem-cell-names:
+    const: dp_calibration_data
+
+  power-domains:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+    properties:
+      port@0:
+        $ref: /schemas/graph.yaml#/properties/port
+        description: Input endpoint of the controller, usually dp_intf
+
+      port@1:
+        $ref: /schemas/graph.yaml#/$defs/port-base
+        unevaluatedProperties: false
+        description: Output endpoint of the controller
+        properties:
+          endpoint:
+            $ref: /schemas/media/video-interfaces.yaml#
+            unevaluatedProperties: false
+            properties:
+              data-lanes:
+                description: |
+                  number of lanes supported by the hardware.
+                  The possible values:
+                  0       - For 1 lane enabled in IP.
+                  0 1     - For 2 lanes enabled in IP.
+                  0 1 2 3 - For 4 lanes enabled in IP.
+                minItems: 1
+                maxItems: 4
+            required:
+              - data-lanes
+
+    required:
+      - port@0
+      - port@1
+
+  max-linkrate-mhz:
+    enum: [ 1620, 2700, 5400, 8100 ]
+    description: maximum link rate supported by the hardware.
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - ports
+  - max-linkrate-mhz
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/power/mt8195-power.h>
+    dptx@1c600000 {
+        compatible = "mediatek,mt8195-dp-tx";
+        reg = <0x1c600000 0x8000>;
+        power-domains = <&spm MT8195_POWER_DOMAIN_DP_TX>;
+        interrupts = <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH 0>;
+        max-linkrate-mhz = <8100>;
+
+        ports {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            port@0 {
+                reg = <0>;
+                dptx_in: endpoint {
+                    remote-endpoint = <&dp_intf0_out>;
+                };
+            };
+            port@1 {
+                reg = <1>;
+                dptx_out: endpoint {
+                    data-lanes = <0 1 2 3>;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml
index 94bc6e1b6451..f2515af8256f 100644
--- a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml
+++ b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml
@@ -24,6 +24,7 @@ properties:
       - qcom,sm8350-dp
 
   reg:
+    minItems: 4
     items:
       - description: ahb register block
       - description: aux register block
@@ -70,14 +71,28 @@ properties:
   operating-points-v2:
     maxItems: 1
 
+  opp-table: true
+
   power-domains:
     maxItems: 1
 
+  aux-bus:
+    $ref: /schemas/display/dp-aux-bus.yaml#
+
+  data-lanes:
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    minItems: 1
+    maxItems: 4
+    items:
+      maximum: 3
+
   "#sound-dai-cells":
     const: 0
 
-  vdda-0p9-supply: true
-  vdda-1p2-supply: true
+  vdda-0p9-supply:
+    deprecated: true
+  vdda-1p2-supply:
+    deprecated: true
 
   ports:
     $ref: /schemas/graph.yaml#/properties/ports
@@ -98,10 +113,33 @@ required:
   - clock-names
   - phys
   - phy-names
-  - "#sound-dai-cells"
   - power-domains
   - ports
 
+allOf:
+  # AUX BUS does not exist on DP controllers
+  # Audio output also is present only on DP output
+  # p1 regions is present on DP, but not on eDP
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,sc7280-edp
+              - qcom,sc8180x-edp
+    then:
+      properties:
+        "#sound-dai-cells": false
+        reg:
+          maxItems: 4
+    else:
+      properties:
+        aux-bus: false
+        reg:
+          minItems: 5
+      required:
+        - "#sound-dai-cells"
+
 additionalProperties: false
 
 examples:
@@ -140,9 +178,6 @@ examples:
 
         power-domains = <&rpmhpd SC7180_CX>;
 
-        vdda-0p9-supply = <&vdda_usb_ss_dp_core>;
-        vdda-1p2-supply = <&vdda_usb_ss_dp_1p2>;
-
         ports {
             #address-cells = <1>;
             #size-cells = <0>;
diff --git a/Documentation/devicetree/bindings/display/msm/dpu-msm8998.yaml b/Documentation/devicetree/bindings/display/msm/dpu-msm8998.yaml
index 2df64afb76e6..253665c693e6 100644
--- a/Documentation/devicetree/bindings/display/msm/dpu-msm8998.yaml
+++ b/Documentation/devicetree/bindings/display/msm/dpu-msm8998.yaml
@@ -62,6 +62,7 @@ patternProperties:
   "^display-controller@[0-9a-f]+$":
     type: object
     description: Node containing the properties of DPU.
+    additionalProperties: false
 
     properties:
       compatible:
@@ -105,6 +106,9 @@ patternProperties:
         maxItems: 1
 
       operating-points-v2: true
+      opp-table:
+        type: object
+
       ports:
         $ref: /schemas/graph.yaml#/properties/ports
         description: |
diff --git a/Documentation/devicetree/bindings/display/msm/dpu-qcm2290.yaml b/Documentation/devicetree/bindings/display/msm/dpu-qcm2290.yaml
index 734d14de966d..c5824e1d2382 100644
--- a/Documentation/devicetree/bindings/display/msm/dpu-qcm2290.yaml
+++ b/Documentation/devicetree/bindings/display/msm/dpu-qcm2290.yaml
@@ -74,6 +74,7 @@ patternProperties:
   "^display-controller@[0-9a-f]+$":
     type: object
     description: Node containing the properties of DPU.
+    additionalProperties: false
 
     properties:
       compatible:
@@ -113,6 +114,8 @@ patternProperties:
         maxItems: 1
 
       operating-points-v2: true
+      opp-table:
+        type: object
 
       ports:
         $ref: /schemas/graph.yaml#/properties/ports
diff --git a/Documentation/devicetree/bindings/display/msm/dpu-sc7180.yaml b/Documentation/devicetree/bindings/display/msm/dpu-sc7180.yaml
index d3c3e4b07897..4890bc25f3fd 100644
--- a/Documentation/devicetree/bindings/display/msm/dpu-sc7180.yaml
+++ b/Documentation/devicetree/bindings/display/msm/dpu-sc7180.yaml
@@ -73,6 +73,7 @@ patternProperties:
   "^display-controller@[0-9a-f]+$":
     type: object
     description: Node containing the properties of DPU.
+    additionalProperties: false
 
     properties:
       compatible:
@@ -114,6 +115,8 @@ patternProperties:
         maxItems: 1
 
       operating-points-v2: true
+      opp-table:
+        type: object
 
       ports:
         $ref: /schemas/graph.yaml#/properties/ports
diff --git a/Documentation/devicetree/bindings/display/msm/dpu-sc7280.yaml b/Documentation/devicetree/bindings/display/msm/dpu-sc7280.yaml
index f427eec3d3a4..584d646021d5 100644
--- a/Documentation/devicetree/bindings/display/msm/dpu-sc7280.yaml
+++ b/Documentation/devicetree/bindings/display/msm/dpu-sc7280.yaml
@@ -72,6 +72,7 @@ patternProperties:
   "^display-controller@[0-9a-f]+$":
     type: object
     description: Node containing the properties of DPU.
+    additionalProperties: false
 
     properties:
       compatible:
@@ -112,6 +113,8 @@ patternProperties:
         maxItems: 1
 
       operating-points-v2: true
+      opp-table:
+        type: object
 
       ports:
         $ref: /schemas/graph.yaml#/properties/ports
diff --git a/Documentation/devicetree/bindings/display/msm/dpu-sdm845.yaml b/Documentation/devicetree/bindings/display/msm/dpu-sdm845.yaml
index 2bb8896beffc..7d1037373175 100644
--- a/Documentation/devicetree/bindings/display/msm/dpu-sdm845.yaml
+++ b/Documentation/devicetree/bindings/display/msm/dpu-sdm845.yaml
@@ -65,6 +65,7 @@ patternProperties:
   "^display-controller@[0-9a-f]+$":
     type: object
     description: Node containing the properties of DPU.
+    additionalProperties: false
 
     properties:
       compatible:
@@ -102,6 +103,9 @@ patternProperties:
         maxItems: 1
 
       operating-points-v2: true
+      opp-table:
+        type: object
+
       ports:
         $ref: /schemas/graph.yaml#/properties/ports
         description: |
diff --git a/Documentation/devicetree/bindings/display/msm/gmu.yaml b/Documentation/devicetree/bindings/display/msm/gmu.yaml
index fe55611d2603..67fdeeabae0c 100644
--- a/Documentation/devicetree/bindings/display/msm/gmu.yaml
+++ b/Documentation/devicetree/bindings/display/msm/gmu.yaml
@@ -20,35 +20,24 @@ description: |
 properties:
   compatible:
     items:
-      - enum:
-          - qcom,adreno-gmu-630.2
+      - pattern: '^qcom,adreno-gmu-6[0-9][0-9]\.[0-9]$'
       - const: qcom,adreno-gmu
 
   reg:
-    items:
-      - description: Core GMU registers
-      - description: GMU PDC registers
-      - description: GMU PDC sequence registers
+    minItems: 3
+    maxItems: 4
 
   reg-names:
-    items:
-      - const: gmu
-      - const: gmu_pdc
-      - const: gmu_pdc_seq
+    minItems: 3
+    maxItems: 4
 
   clocks:
-    items:
-      - description: GMU clock
-      - description: GPU CX clock
-      - description: GPU AXI clock
-      - description: GPU MEMNOC clock
+    minItems: 4
+    maxItems: 7
 
   clock-names:
-    items:
-      - const: gmu
-      - const: cxo
-      - const: axi
-      - const: memnoc
+    minItems: 4
+    maxItems: 7
 
   interrupts:
     items:
@@ -76,6 +65,9 @@ properties:
 
   operating-points-v2: true
 
+  opp-table:
+    type: object
+
 required:
   - compatible
   - reg
@@ -91,6 +83,140 @@ required:
 
 additionalProperties: false
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,adreno-gmu-618.0
+              - qcom,adreno-gmu-630.2
+    then:
+      properties:
+        reg:
+          items:
+            - description: Core GMU registers
+            - description: GMU PDC registers
+            - description: GMU PDC sequence registers
+        reg-names:
+          items:
+            - const: gmu
+            - const: gmu_pdc
+            - const: gmu_pdc_seq
+        clocks:
+          items:
+            - description: GMU clock
+            - description: GPU CX clock
+            - description: GPU AXI clock
+            - description: GPU MEMNOC clock
+        clock-names:
+          items:
+            - const: gmu
+            - const: cxo
+            - const: axi
+            - const: memnoc
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,adreno-gmu-635.0
+    then:
+      properties:
+        reg:
+          items:
+            - description: Core GMU registers
+            - description: Resource controller registers
+            - description: GMU PDC registers
+        reg-names:
+          items:
+            - const: gmu
+            - const: rscc
+            - const: gmu_pdc
+        clocks:
+          items:
+            - description: GMU clock
+            - description: GPU CX clock
+            - description: GPU AXI clock
+            - description: GPU MEMNOC clock
+            - description: GPU AHB clock
+            - description: GPU HUB CX clock
+            - description: GPU SMMU vote clock
+        clock-names:
+          items:
+            - const: gmu
+            - const: cxo
+            - const: axi
+            - const: memnoc
+            - const: ahb
+            - const: hub
+            - const: smmu_vote
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,adreno-gmu-640.1
+    then:
+      properties:
+        reg:
+          items:
+            - description: Core GMU registers
+            - description: GMU PDC registers
+            - description: GMU PDC sequence registers
+        reg-names:
+          items:
+            - const: gmu
+            - const: gmu_pdc
+            - const: gmu_pdc_seq
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,adreno-gmu-650.2
+    then:
+      properties:
+        reg:
+          items:
+            - description: Core GMU registers
+            - description: Resource controller registers
+            - description: GMU PDC registers
+            - description: GMU PDC sequence registers
+        reg-names:
+          items:
+            - const: gmu
+            - const: rscc
+            - const: gmu_pdc
+            - const: gmu_pdc_seq
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,adreno-gmu-640.1
+              - qcom,adreno-gmu-650.2
+    then:
+      properties:
+        clocks:
+          items:
+            - description: GPU AHB clock
+            - description: GMU clock
+            - description: GPU CX clock
+            - description: GPU AXI clock
+            - description: GPU MEMNOC clock
+        clock-names:
+          items:
+            - const: ahb
+            - const: gmu
+            - const: cxo
+            - const: axi
+            - const: memnoc
+
 examples:
   - |
     #include <dt-bindings/clock/qcom,gpucc-sdm845.h>
diff --git a/Documentation/devicetree/bindings/display/msm/gpu.yaml b/Documentation/devicetree/bindings/display/msm/gpu.yaml
index 3397bc31d087..346aabdccf7b 100644
--- a/Documentation/devicetree/bindings/display/msm/gpu.yaml
+++ b/Documentation/devicetree/bindings/display/msm/gpu.yaml
@@ -58,7 +58,8 @@ properties:
       - const: ocmem
 
   iommus:
-    maxItems: 1
+    minItems: 1
+    maxItems: 64
 
   sram:
     $ref: /schemas/types.yaml#/definitions/phandle-array
diff --git a/Documentation/devicetree/bindings/display/msm/mdp4.yaml b/Documentation/devicetree/bindings/display/msm/mdp4.yaml
index f63f60fea27c..58c13f5277b6 100644
--- a/Documentation/devicetree/bindings/display/msm/mdp4.yaml
+++ b/Documentation/devicetree/bindings/display/msm/mdp4.yaml
@@ -36,7 +36,7 @@ properties:
     maxItems: 1
 
   iommus:
-    maxItems: 1
+    maxItems: 4
 
   ports:
     $ref: /schemas/graph.yaml#/properties/ports
diff --git a/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml b/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml
index 2a2756d19681..b4be9bd8ddde 100644
--- a/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml
+++ b/Documentation/devicetree/bindings/display/panel/kingdisplay,kd035g6-54nt.yaml
@@ -23,6 +23,8 @@ properties:
   reg: true
   reset-gpios: true
 
+  spi-3wire: true
+
 required:
   - compatible
   - power-supply
diff --git a/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml b/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml
index 817a9bed7d5a..ebdca5f5a001 100644
--- a/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml
+++ b/Documentation/devicetree/bindings/display/panel/leadtek,ltk035c5444t.yaml
@@ -24,6 +24,8 @@ properties:
   reg: true
   reset-gpios: true
 
+  spi-3wire: true
+
 required:
   - compatible
   - power-supply
diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
index bc8e9c0c1dc3..18241f4051d2 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
@@ -234,6 +234,8 @@ properties:
       - mitsubishi,aa070mc01-ca1
         # Multi-Inno Technology Co.,Ltd MI0700S4T-6 7" 800x480 TFT Resistive Touch Module
       - multi-inno,mi0700s4t-6
+        # Multi-Inno Technology Co.,Ltd MI0800FT-9 8" 800x600 TFT Resistive Touch Module
+      - multi-inno,mi0800ft-9
         # Multi-Inno Technology Co.,Ltd MI1010AIT-1CP 10.1" 1280x800 LVDS IPS Cap Touch Mod.
       - multi-inno,mi1010ait-1cp
         # NEC LCD Technologies, Ltd. 12.1" WXGA (1280x800) LVDS TFT LCD panel
@@ -280,6 +282,8 @@ properties:
       - samsung,atna33xc20
         # Samsung 12.2" (2560x1600 pixels) TFT LCD panel
       - samsung,lsn122dl01-c01
+        # Samsung Electronics 10.1" WXGA (1280x800) TFT LCD panel
+      - samsung,ltl101al01
         # Samsung Electronics 10.1" WSVGA TFT LCD panel
       - samsung,ltn101nt05
         # Samsung Electronics 14" WXGA (1366x768) TFT LCD panel
diff --git a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml
index 940f7f88526f..6f1fc7469f07 100644
--- a/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml
+++ b/Documentation/devicetree/bindings/display/panel/samsung,s6e63m0.yaml
@@ -24,6 +24,10 @@ properties:
   default-brightness: true
   max-brightness: true
 
+  spi-3wire: true
+  spi-cpha: true
+  spi-cpol: true
+
   vdd3-supply:
     description: VDD regulator
 
diff --git a/Documentation/devicetree/bindings/display/panel/sitronix,st7701.yaml b/Documentation/devicetree/bindings/display/panel/sitronix,st7701.yaml
index 6dff59fe4be1..34d5e20c6cb3 100644
--- a/Documentation/devicetree/bindings/display/panel/sitronix,st7701.yaml
+++ b/Documentation/devicetree/bindings/display/panel/sitronix,st7701.yaml
@@ -17,6 +17,9 @@ description: |
   Techstar TS8550B is 480x854, 2-lane MIPI DSI LCD panel which has
   inbuilt ST7701 chip.
 
+  Densitron DMT028VGHMCMI-1A is 480x640, 2-lane MIPI DSI LCD panel
+  which has built-in ST7701 chip.
+
 allOf:
   - $ref: panel-common.yaml#
 
@@ -24,6 +27,7 @@ properties:
   compatible:
     items:
       - enum:
+          - densitron,dmt028vghmcmi-1a
           - techstar,ts8550b
       - const: sitronix,st7701
 
diff --git a/Documentation/devicetree/bindings/display/rockchip/dw_mipi_dsi_rockchip.txt b/Documentation/devicetree/bindings/display/rockchip/dw_mipi_dsi_rockchip.txt
index 39792f051d2d..9a223df8530c 100644
--- a/Documentation/devicetree/bindings/display/rockchip/dw_mipi_dsi_rockchip.txt
+++ b/Documentation/devicetree/bindings/display/rockchip/dw_mipi_dsi_rockchip.txt
@@ -8,6 +8,7 @@ Required properties:
 	"rockchip,px30-mipi-dsi", "snps,dw-mipi-dsi"
 	"rockchip,rk3288-mipi-dsi", "snps,dw-mipi-dsi"
 	"rockchip,rk3399-mipi-dsi", "snps,dw-mipi-dsi"
+	"rockchip,rk3568-mipi-dsi", "snps,dw-mipi-dsi"
 - reg: Represent the physical address range of the controller.
 - interrupts: Represent the controller's interrupt to the CPU(s).
 - clocks, clock-names: Phandles to the controller's pll reference
diff --git a/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml b/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml
index e66ef2da7879..9af40da5688e 100644
--- a/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml
+++ b/Documentation/devicetree/bindings/dsp/fsl,dsp.yaml
@@ -20,6 +20,7 @@ properties:
       - fsl,imx8qxp-dsp
       - fsl,imx8qm-dsp
       - fsl,imx8mp-dsp
+      - fsl,imx8ulp-dsp
       - fsl,imx8qxp-hifi4
       - fsl,imx8qm-hifi4
       - fsl,imx8mp-hifi4
diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.txt b/Documentation/devicetree/bindings/firmware/qcom,scm.txt
deleted file mode 100644
index b3f702cbed87..000000000000
--- a/Documentation/devicetree/bindings/firmware/qcom,scm.txt
+++ /dev/null
@@ -1,61 +0,0 @@
-QCOM Secure Channel Manager (SCM)
-
-Qualcomm processors include an interface to communicate to the secure firmware.
-This interface allows for clients to request different types of actions.  These
-can include CPU power up/down, HDCP requests, loading of firmware, and other
-assorted actions.
-
-Required properties:
-- compatible: must contain one of the following:
- * "qcom,scm-apq8064"
- * "qcom,scm-apq8084"
- * "qcom,scm-ipq4019"
- * "qcom,scm-ipq806x"
- * "qcom,scm-ipq8074"
- * "qcom,scm-mdm9607"
- * "qcom,scm-msm8226"
- * "qcom,scm-msm8660"
- * "qcom,scm-msm8916"
- * "qcom,scm-msm8953"
- * "qcom,scm-msm8960"
- * "qcom,scm-msm8974"
- * "qcom,scm-msm8976"
- * "qcom,scm-msm8994"
- * "qcom,scm-msm8996"
- * "qcom,scm-msm8998"
- * "qcom,scm-qcs404"
- * "qcom,scm-sc7180"
- * "qcom,scm-sc7280"
- * "qcom,scm-sm6125"
- * "qcom,scm-sdm845"
- * "qcom,scm-sdx55"
- * "qcom,scm-sdx65"
- * "qcom,scm-sm6350"
- * "qcom,scm-sm8150"
- * "qcom,scm-sm8250"
- * "qcom,scm-sm8350"
- * "qcom,scm-sm8450"
- and:
- * "qcom,scm"
-- clocks: Specifies clocks needed by the SCM interface, if any:
- * core clock required for "qcom,scm-apq8064", "qcom,scm-msm8660" and
-   "qcom,scm-msm8960"
- * core, iface and bus clocks required for "qcom,scm-apq8084",
-   "qcom,scm-msm8916", "qcom,scm-msm8953", "qcom,scm-msm8974" and "qcom,scm-msm8976"
-- clock-names: Must contain "core" for the core clock, "iface" for the interface
-  clock and "bus" for the bus clock per the requirements of the compatible.
-- qcom,dload-mode: phandle to the TCSR hardware block and offset of the
-		   download mode control register (optional)
-- interconnects: Specifies the bandwidth requirements of the SCM interface (optional)
-
-Example for MSM8916:
-
-	firmware {
-		scm {
-			compatible = "qcom,msm8916", "qcom,scm";
-			clocks = <&gcc GCC_CRYPTO_CLK> ,
-				 <&gcc GCC_CRYPTO_AXI_CLK>,
-				 <&gcc GCC_CRYPTO_AHB_CLK>;
-			clock-names = "core", "bus", "iface";
-		};
-	};
diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
new file mode 100644
index 000000000000..c5b76c9f7ad0
--- /dev/null
+++ b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
@@ -0,0 +1,148 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/firmware/qcom,scm.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: QCOM Secure Channel Manager (SCM)
+
+description: |
+  Qualcomm processors include an interface to communicate to the secure firmware.
+  This interface allows for clients to request different types of actions.
+  These can include CPU power up/down, HDCP requests, loading of firmware,
+  and other assorted actions.
+
+maintainers:
+  - Bjorn Andersson <bjorn.andersson@linaro.org>
+  - Robert Marko <robimarko@gmail.com>
+  - Guru Das Srinagesh <quic_gurus@quicinc.com>
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - qcom,scm-apq8064
+          - qcom,scm-apq8084
+          - qcom,scm-ipq4019
+          - qcom,scm-ipq6018
+          - qcom,scm-ipq806x
+          - qcom,scm-ipq8074
+          - qcom,scm-mdm9607
+          - qcom,scm-msm8226
+          - qcom,scm-msm8660
+          - qcom,scm-msm8916
+          - qcom,scm-msm8953
+          - qcom,scm-msm8960
+          - qcom,scm-msm8974
+          - qcom,scm-msm8976
+          - qcom,scm-msm8994
+          - qcom,scm-msm8996
+          - qcom,scm-msm8998
+          - qcom,scm-sc7180
+          - qcom,scm-sc7280
+          - qcom,scm-sc8280xp
+          - qcom,scm-sdm845
+          - qcom,scm-sdx55
+          - qcom,scm-sdx65
+          - qcom,scm-sm6115
+          - qcom,scm-sm6125
+          - qcom,scm-sm6350
+          - qcom,scm-sm8150
+          - qcom,scm-sm8250
+          - qcom,scm-sm8350
+          - qcom,scm-sm8450
+          - qcom,scm-qcs404
+      - const: qcom,scm
+
+  clocks:
+    minItems: 1
+    maxItems: 3
+
+  clock-names:
+    minItems: 1
+    maxItems: 3
+
+  interconnects:
+    maxItems: 1
+
+  interconnect-names:
+    maxItems: 1
+
+  '#reset-cells':
+    const: 1
+
+  qcom,dload-mode:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    items:
+      - items:
+          - description: phandle to TCSR hardware block
+          - description: offset of the download mode control register
+    description: TCSR hardware block
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,scm-apq8064
+              - qcom,scm-msm8660
+              - qcom,scm-msm8960
+    then:
+      properties:
+        clock-names:
+          items:
+            - const: core
+
+        clocks:
+          maxItems: 1
+
+      required:
+        - clocks
+        - clock-names
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,scm-apq8084
+              - qcom,scm-mdm9607
+              - qcom,scm-msm8916
+              - qcom,scm-msm8953
+              - qcom,scm-msm8974
+              - qcom,scm-msm8976
+    then:
+      properties:
+        clock-names:
+          items:
+            - const: core
+            - const: bus
+            - const: iface
+
+        clocks:
+          minItems: 3
+          maxItems: 3
+
+      required:
+        - clocks
+        - clock-names
+
+required:
+  - compatible
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,gcc-msm8916.h>
+
+    firmware {
+        scm {
+            compatible = "qcom,scm-msm8916", "qcom,scm";
+            clocks = <&gcc GCC_CRYPTO_CLK>,
+                     <&gcc GCC_CRYPTO_AXI_CLK>,
+                     <&gcc GCC_CRYPTO_AHB_CLK>;
+            clock-names = "core", "bus", "iface";
+        };
+    };
diff --git a/Documentation/devicetree/bindings/hwmon/adi,max31760.yaml b/Documentation/devicetree/bindings/hwmon/adi,max31760.yaml
new file mode 100644
index 000000000000..9f2d08d7b978
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/adi,max31760.yaml
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/adi,max31760.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices MAX31760 Fan-Speed Controller
+
+maintainers:
+  - Ibrahim Tilki <Ibrahim.Tilki@analog.com>
+
+description: |
+  Analog Devices MAX31760 Fan-Speed Controller
+  https://datasheets.maximintegrated.com/en/ds/MAX31760.pdf
+
+properties:
+  compatible:
+    enum:
+      - adi,max31760
+
+  reg:
+    description: I2C address of slave device.
+    minimum: 0x50
+    maximum: 0x57
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        fan-controller@50 {
+            reg = <0x50>;
+            compatible = "adi,max31760";
+        };
+    };
diff --git a/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml b/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml
index 76be625d5646..51e8619dbf3c 100644
--- a/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml
+++ b/Documentation/devicetree/bindings/hwmon/microchip,sparx5-temp.yaml
@@ -22,7 +22,7 @@ properties:
 
   clocks:
     items:
-      - description: AHB reference clock
+      - description: System reference clock
 
   '#thermal-sensor-cells':
     const: 0
@@ -40,5 +40,5 @@ examples:
         compatible = "microchip,sparx5-temp";
         reg = <0x10508110 0xc>;
         #thermal-sensor-cells = <0>;
-        clocks = <&ahb_clk>;
+        clocks = <&sys_clk>;
     };
diff --git a/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml b/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml
index 8ea97e774364..d0d549749208 100644
--- a/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml
+++ b/Documentation/devicetree/bindings/hwmon/moortec,mr75203.yaml
@@ -9,6 +9,32 @@ title: Moortec Semiconductor MR75203 PVT Controller bindings
 maintainers:
   - Rahul Tanwar <rtanwar@maxlinear.com>
 
+description: |
+  A Moortec PVT (Process, Voltage, Temperature) monitoring logic design can
+  include many different units.
+  Such a design will usually consists of several Moortec's embedded analog IPs,
+  and a single Moortec controller (mr75203) to configure and control the IPs.
+
+  Some of the Moortec's analog hard IPs that can be used in a design:
+  *) Temperature Sensor (TS) - used to monitor core temperature (e.g. mr74137).
+  *) Voltage Monitor (VM) - used to monitor voltage levels (e.g. mr74138).
+  *) Process Detector (PD) - used to assess silicon speed (e.g. mr74139).
+  *) Delay Chain - ring oscillator connected to the PD, used to measure IO
+     based transistors (e.g. mr76008 ring oscillator at 1.1V, mr76007 ring
+     oscillator at 1.8V).
+  *) Pre Scaler - provides divide-by-X scaling of input voltage, which can then
+     be presented for VM for measurement within its range (e.g. mr76006 -
+     divide by 2 pre-scaler).
+
+  TS, VM & PD also include a digital interface, which consists of configuration
+  inputs and measurement outputs.
+
+  Some of the units have number of series, each series can have slightly
+  different characteristics.
+
+  The mr75203 binding describes configuration for the controller unit, but also
+  for some of the analog IPs.
+
 properties:
   compatible:
     const: moortec,mr75203
@@ -44,12 +70,76 @@ properties:
   "#thermal-sensor-cells":
     const: 1
 
+  moortec,vm-active-channels:
+    description:
+      Defines the number of channels per VM that are actually used and are
+      connected to some input source.
+      Maximum number of items - number of VMs.
+      Maximum value of each item - number of channels.
+      Minimum value of each item - 0 (which means entire VM sensor is not used).
+    $ref: /schemas/types.yaml#/definitions/uint8-array
+
+  moortec,vm-pre-scaler-x2:
+    description:
+      Defines the channels that use a mr76006 pre-scaler to divide the input
+      source by 2.
+      The pre-scaler is used for input sources that exceed the VM input range.
+      The driver uses this information to present to the user with the actual
+      value of the voltage source.
+      For channels that are not listed, no pre-scaler is assumed.
+      Maximum number of items - total number of channels in all VMs.
+      Each channel should not appear more than once.
+    $ref: /schemas/types.yaml#/definitions/uint8-array
+
+  moortec,ts-series:
+    description:
+      Definition of the temperature equation and coefficients that shall be
+      used to convert the digital output to value in milli-Celsius.
+    minimum: 5
+    maximum: 6
+    default: 5
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+  moortec,ts-coeff-g:
+    description:
+      G coefficient for temperature equation.
+      Default for series 5 = 60000
+      Default for series 6 = 57400
+    multipleOf: 1000
+    minimum: 1000
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+  moortec,ts-coeff-h:
+    description:
+      H coefficient for temperature equation.
+      Default for series 5 = 200000
+      Default for series 6 = 249400
+    multipleOf: 1000
+    minimum: 1000
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+  moortec,ts-coeff-cal5:
+    description:
+      cal5 coefficient for temperature equation.
+      Default for series 5 = 4094
+      Default for series 6 = 4096
+    minimum: 1
+    $ref: /schemas/types.yaml#/definitions/uint32
+
+  moortec,ts-coeff-j:
+    description:
+      J coefficient for temperature equation.
+      Default for series 5 = -100
+      Default for series 6 = 0
+    multipleOf: 1000
+    maximum: 0
+    $ref: /schemas/types.yaml#/definitions/int32
+
 required:
   - compatible
   - reg
   - reg-names
   - clocks
-  - resets
   - "#thermal-sensor-cells"
 
 additionalProperties: false
@@ -66,5 +156,9 @@ examples:
         intel,vm-map = [03 01 04 ff ff];
         clocks = <&osc0>;
         resets = <&rcu0 0x40 7>;
+        moortec,vm-active-channels = /bits/ 8 <0x10 0x05>;
+        moortec,vm-pre-scaler-x2 = /bits/ 8 <5 6 20>;
+        moortec,ts-coeff-g = <61400>;
+        moortec,ts-coeff-h = <253700>;
         #thermal-sensor-cells = <1>;
     };
diff --git a/Documentation/devicetree/bindings/hwmon/sensirion,shtc1.yaml b/Documentation/devicetree/bindings/hwmon/sensirion,shtc1.yaml
index 7d49478d9668..159238efa9ed 100644
--- a/Documentation/devicetree/bindings/hwmon/sensirion,shtc1.yaml
+++ b/Documentation/devicetree/bindings/hwmon/sensirion,shtc1.yaml
@@ -10,7 +10,7 @@ maintainers:
   - Christopher Ruehl chris.ruehl@gtsys.com.hk
 
 description: |
-  The SHTC1, SHTW1 and SHTC3 are digital humidity and temperature sensor
+  The SHTC1, SHTW1 and SHTC3 are digital humidity and temperature sensors
   designed especially for battery-driven high-volume consumer electronics
   applications.
   For further information refere to Documentation/hwmon/shtc1.rst
@@ -31,13 +31,13 @@ properties:
   sensirion,blocking-io:
     $ref: /schemas/types.yaml#/definitions/flag
     description:
-      If set, the driver hold the i2c bus until measurement is finished.
+      If set, the driver holds the i2c bus until the measurement is finished.
 
   sensirion,low-precision:
     $ref: /schemas/types.yaml#/definitions/flag
     description:
-      If set, the sensor aquire data with low precision (not recommended).
-      The driver aquire data with high precision by default.
+      If set, the sensor acquires data with low precision (not recommended).
+      The driver acquires data with high precision by default.
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.yaml b/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.yaml
index 529bea56d324..4656f5112b84 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/i2c-imx-lpi2c.yaml
@@ -23,6 +23,7 @@ properties:
               - fsl,imx8dxl-lpi2c
               - fsl,imx8qm-lpi2c
               - fsl,imx8ulp-lpi2c
+              - fsl,imx93-lpi2c
           - const: fsl,imx7ulp-lpi2c
 
   reg:
@@ -37,10 +38,22 @@ properties:
   clock-frequency: true
 
   clock-names:
-    maxItems: 1
+    items:
+      - const: per
+      - const: ipg
 
   clocks:
-    maxItems: 1
+    maxItems: 2
+
+  dmas:
+    items:
+      - description: DMA controller phandle and request line for TX
+      - description: DMA controller phandle and request line for RX
+
+  dma-names:
+    items:
+      - const: tx
+      - const: rx
 
   power-domains:
     maxItems: 1
@@ -63,5 +76,6 @@ examples:
         reg = <0x40A50000 0x10000>;
         interrupt-parent = <&intc>;
         interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
-        clocks = <&clks IMX7ULP_CLK_LPI2C7>;
+        clocks = <&clks IMX7ULP_CLK_LPI2C7>,
+                 <&clks IMX7ULP_CLK_NIC1_BUS_DIV>;
     };
diff --git a/Documentation/devicetree/bindings/i2c/i2c-imx.yaml b/Documentation/devicetree/bindings/i2c/i2c-imx.yaml
index 01720e338b4c..85ee1282d6d2 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-imx.yaml
+++ b/Documentation/devicetree/bindings/i2c/i2c-imx.yaml
@@ -19,6 +19,17 @@ properties:
       - const: fsl,imx21-i2c
       - const: fsl,vf610-i2c
       - items:
+          - enum:
+              - fsl,ls1012a-i2c
+              - fsl,ls1021a-i2c
+              - fsl,ls1028a-i2c
+              - fsl,ls1043a-i2c
+              - fsl,ls1046a-i2c
+              - fsl,ls1088a-i2c
+              - fsl,ls208xa-i2c
+              - fsl,lx2160a-i2c
+          - const: fsl,vf610-i2c
+      - items:
           - const: fsl,imx35-i2c
           - const: fsl,imx1-i2c
       - items:
diff --git a/Documentation/devicetree/bindings/i2c/i2c-rk3x.yaml b/Documentation/devicetree/bindings/i2c/i2c-rk3x.yaml
index ee9f8b91d2e2..82b9d6682297 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-rk3x.yaml
+++ b/Documentation/devicetree/bindings/i2c/i2c-rk3x.yaml
@@ -29,6 +29,7 @@ properties:
       - items:
           - enum:
               - rockchip,rk3036-i2c
+              - rockchip,rk3128-i2c
               - rockchip,rk3368-i2c
           - const: rockchip,rk3288-i2c
       - items:
diff --git a/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml b/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml
index 0ec033e48830..63d665a4f9bb 100644
--- a/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/marvell,mv64xxx-i2c.yaml
@@ -66,6 +66,19 @@ properties:
   resets:
     maxItems: 1
 
+  dmas:
+    items:
+      - description: RX DMA Channel
+      - description: TX DMA Channel
+
+  dma-names:
+    items:
+      - const: rx
+      - const: tx
+
+dependencies:
+  dmas: [ dma-names ]
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/i2c/mellanox,i2c-mlxbf.yaml b/Documentation/devicetree/bindings/i2c/mellanox,i2c-mlxbf.yaml
deleted file mode 100644
index 93198d5d43a6..000000000000
--- a/Documentation/devicetree/bindings/i2c/mellanox,i2c-mlxbf.yaml
+++ /dev/null
@@ -1,77 +0,0 @@
-# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/i2c/mellanox,i2c-mlxbf.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Mellanox I2C SMBus on BlueField SoCs
-
-maintainers:
-  - Khalil Blaiech <kblaiech@nvidia.com>
-
-allOf:
-  - $ref: /schemas/i2c/i2c-controller.yaml#
-
-properties:
-  compatible:
-    enum:
-      - mellanox,i2c-mlxbf1
-      - mellanox,i2c-mlxbf2
-
-  reg:
-    minItems: 3
-    items:
-      - description: Smbus block registers
-      - description: Cause master registers
-      - description: Cause slave registers
-      - description: Cause coalesce registers
-
-  interrupts:
-    maxItems: 1
-
-  clock-frequency:
-    enum: [ 100000, 400000, 1000000 ]
-    description:
-      bus frequency used to configure timing registers;
-      The frequency is expressed in Hz. Default is 100000.
-
-required:
-  - compatible
-  - reg
-  - interrupts
-
-unevaluatedProperties: false
-
-if:
-  properties:
-    compatible:
-      contains:
-        enum:
-          - mellanox,i2c-mlxbf1
-
-then:
-  properties:
-    reg:
-      maxItems: 3
-
-examples:
-  - |
-    i2c@2804000 {
-        compatible = "mellanox,i2c-mlxbf1";
-        reg = <0x02804000 0x800>,
-              <0x02801200 0x020>,
-              <0x02801260 0x020>;
-        interrupts = <57>;
-        clock-frequency = <100000>;
-    };
-
-  - |
-    i2c@2808800 {
-        compatible = "mellanox,i2c-mlxbf2";
-        reg = <0x02808800 0x600>,
-              <0x02808e00 0x020>,
-              <0x02808e20 0x020>,
-              <0x02808e40 0x010>;
-        interrupts = <57>;
-        clock-frequency = <400000>;
-    };
diff --git a/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml b/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml
index 90c9e401229e..e51a85848d6e 100644
--- a/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml
+++ b/Documentation/devicetree/bindings/i2c/qcom,i2c-cci.yaml
@@ -126,6 +126,7 @@ allOf:
           contains:
             enum:
               - qcom,sm8250-cci
+              - qcom,sm8450-cci
     then:
       properties:
         clocks:
diff --git a/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml b/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml
index f9929578c761..c4ace5585e1e 100644
--- a/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/renesas,rcar-i2c.yaml
@@ -52,6 +52,7 @@ properties:
           - enum:
               - renesas,i2c-r8a779a0     # R-Car V3U
               - renesas,i2c-r8a779f0     # R-Car S4-8
+              - renesas,i2c-r8a779g0     # R-Car V4H
           - const: renesas,rcar-gen4-i2c # R-Car Gen4
 
   reg:
diff --git a/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml b/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml
index a41588763786..bf396e9466aa 100644
--- a/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml
@@ -57,6 +57,11 @@ properties:
       - description: interrupt ID for I2C event
       - description: interrupt ID for I2C error
 
+  interrupt-names:
+    items:
+      - const: event
+      - const: error
+
   resets:
     maxItems: 1
 
@@ -92,6 +97,8 @@ properties:
           - description: register offset within syscfg
           - description: register bitmask for FMP bit
 
+  wakeup-source: true
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
index c2e697f6e6cf..0ac5256876a8 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
@@ -24,9 +24,12 @@ properties:
     oneOf:
       - items:
           - enum:
+              - qcom,sc7280-bwmon
               - qcom,sdm845-bwmon
           - const: qcom,msm8998-bwmon
       - const: qcom,msm8998-bwmon       # BWMON v4
+      - const: qcom,sc7280-llcc-bwmon   # BWMON v5
+      - const: qcom,sdm845-llcc-bwmon   # BWMON v5
 
   interconnects:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/leds/backlight/mediatek,mt6370-backlight.yaml b/Documentation/devicetree/bindings/leds/backlight/mediatek,mt6370-backlight.yaml
new file mode 100644
index 000000000000..5533b6562d92
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/backlight/mediatek,mt6370-backlight.yaml
@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/leds/backlight/mediatek,mt6370-backlight.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek MT6370 Backlight
+
+maintainers:
+  - ChiaEn Wu <chiaen_wu@richtek.com>
+
+description: |
+  This module is part of the MT6370 MFD device.
+  The MT6370 Backlight WLED driver supports up to a 29V output voltage for
+  4 channels of 8 series WLEDs. Each channel supports up to 30mA of current
+  capability with 2048 current steps (11 bits, only for MT6370/MT6371) or
+  16384 current steps (14 bits, only for MT6372) in exponential or linear
+  mapping curves.
+
+allOf:
+  - $ref: common.yaml#
+
+properties:
+  compatible:
+    enum:
+      - mediatek,mt6370-backlight
+      - mediatek,mt6372-backlight
+
+  default-brightness:
+    minimum: 0
+
+  max-brightness:
+    minimum: 0
+
+  enable-gpios:
+    description: External backlight 'enable' pin
+    maxItems: 1
+
+  mediatek,bled-pwm-enable:
+    description: |
+      Enable external PWM input for backlight dimming
+    type: boolean
+
+  mediatek,bled-pwm-hys-enable:
+    description: |
+      Enable the backlight input-hysteresis for PWM mode
+    type: boolean
+
+  mediatek,bled-pwm-hys-input-th-steps:
+    $ref: /schemas/types.yaml#/definitions/uint8
+    enum: [1, 4, 16, 64]
+    description: |
+      The selection of the upper and lower bounds threshold of backlight
+      PWM resolution. If we choose selection 64, the variation of PWM
+      resolution needs more than 64 steps.
+
+  mediatek,bled-ovp-shutdown:
+    description: |
+      Enable the backlight shutdown when OVP level triggered
+    type: boolean
+
+  mediatek,bled-ovp-microvolt:
+    enum: [17000000, 21000000, 25000000, 29000000]
+    description: |
+      Backlight OVP level selection.
+
+  mediatek,bled-ocp-shutdown:
+    description: |
+      Enable the backlight shutdown when OCP level triggerred.
+    type: boolean
+
+  mediatek,bled-ocp-microamp:
+    enum: [900000, 1200000, 1500000, 1800000]
+    description: |
+      Backlight OC level selection.
+
+  mediatek,bled-exponential-mode-enable:
+    description: |
+      Enable the exponential mode of backlight brightness. If this property
+      is not enabled, the default is to use linear mode.
+    type: boolean
+
+  mediatek,bled-channel-use:
+    $ref: /schemas/types.yaml#/definitions/uint8
+    description: |
+      Backlight LED channel to be used.
+      Each bit mapping to:
+        - 0: CH4
+        - 1: CH3
+        - 2: CH2
+        - 3: CH1
+    minimum: 1
+    maximum: 15
+
+if:
+  properties:
+    compatible:
+      contains:
+        const: mediatek,mt6372-backlight
+
+then:
+  properties:
+    default-brightness:
+      maximum: 16384
+
+    max-brightness:
+      maximum: 16384
+
+else:
+  properties:
+    default-brightness:
+      maximum: 2048
+
+    max-brightness:
+      maximum: 2048
+
+required:
+  - compatible
+  - mediatek,bled-channel-use
+
+additionalProperties: false
diff --git a/Documentation/devicetree/bindings/media/exynos5-gsc.txt b/Documentation/devicetree/bindings/media/exynos5-gsc.txt
deleted file mode 100644
index 1872688fa408..000000000000
--- a/Documentation/devicetree/bindings/media/exynos5-gsc.txt
+++ /dev/null
@@ -1,38 +0,0 @@
-* Samsung Exynos5 G-Scaler device
-
-G-Scaler is used for scaling and color space conversion on Exynos5 SoCs.
-
-Required properties:
-- compatible: should be one of
-	      "samsung,exynos5250-gsc"
-	      "samsung,exynos5420-gsc"
-	      "samsung,exynos5433-gsc"
-	      "samsung,exynos5-gsc" (deprecated)
-- reg: should contain G-Scaler physical address location and length.
-- interrupts: should contain G-Scaler interrupt number
-
-Optional properties:
-- samsung,sysreg: handle to syscon used to control the system registers to
-  set writeback input and destination
-
-Example:
-
-gsc_0:  gsc@13e00000 {
-	compatible = "samsung,exynos5250-gsc";
-	reg = <0x13e00000 0x1000>;
-	interrupts = <0 85 0>;
-};
-
-Aliases:
-Each G-Scaler node should have a numbered alias in the aliases node,
-in the form of gscN, N = 0...3. G-Scaler driver uses these aliases
-to retrieve the device IDs using "of_alias_get_id()" call.
-
-Example:
-
-aliases {
-	gsc0 =&gsc_0;
-	gsc1 =&gsc_1;
-	gsc2 =&gsc_2;
-	gsc3 =&gsc_3;
-};
diff --git a/Documentation/devicetree/bindings/media/samsung,exynos5250-gsc.yaml b/Documentation/devicetree/bindings/media/samsung,exynos5250-gsc.yaml
new file mode 100644
index 000000000000..878397830a4d
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/samsung,exynos5250-gsc.yaml
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/samsung,exynos5250-gsc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Samsung Exynos SoC G-Scaler
+
+maintainers:
+  - Inki Dae <inki.dae@samsung.com>
+  - Krzysztof Kozlowski <krzk@kernel.org>
+  - Seung-Woo Kim <sw0312.kim@samsung.com
+
+description:
+  G-Scaler is used for scaling and color space conversion on Samsung Exynos
+  SoCs.
+
+  Each G-Scaler node should have a numbered alias in the aliases node, in the
+  form of gscN, N = 0...3.
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - samsung,exynos5250-gsc
+              - samsung,exynos5420-gsc
+          - const: samsung,exynos5-gsc
+      - enum:
+          - samsung,exynos5433-gsc
+      - const: samsung,exynos5-gsc
+        deprecated: True
+
+  clocks:
+    minItems: 1
+    maxItems: 5
+
+  clock-names:
+    minItems: 1
+    maxItems: 5
+
+  interrupts:
+    maxItems: 1
+
+  iommus:
+    maxItems: 1
+
+  power-domains:
+    maxItems: 1
+
+  reg:
+    maxItems: 1
+
+  samsung,sysreg:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description:
+      Syscon used to control the system registers to set writeback input and destination.
+
+required:
+  - compatible
+  - clocks
+  - clock-names
+  - interrupts
+  - reg
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - samsung,exynos5-gsc
+              - samsung,exynos5250-gsc
+              - samsung,exynos5420-gsc
+    then:
+      properties:
+        clocks:
+          maxItems: 1
+        clock-names:
+          items:
+            - const: gscl
+    else:
+      properties:
+        clocks:
+          minItems: 5
+        clock-names:
+          items:
+            - const: pclk
+            - const: aclk
+            - const: aclk_xiu
+            - const: aclk_gsclbend
+            - const: gsd
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/exynos5250.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    video-scaler@13e00000 {
+        compatible = "samsung,exynos5250-gsc", "samsung,exynos5-gsc";
+        reg = <0x13e00000 0x1000>;
+        interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
+        power-domains = <&pd_gsc>;
+        clocks = <&clock CLK_GSCL0>;
+        clock-names = "gscl";
+        iommus = <&sysmmu_gsc0>;
+    };
diff --git a/Documentation/devicetree/bindings/memory-controllers/brcm,brcmstb-memc-ddr.yaml b/Documentation/devicetree/bindings/memory-controllers/brcm,brcmstb-memc-ddr.yaml
new file mode 100644
index 000000000000..4b072c879b02
--- /dev/null
+++ b/Documentation/devicetree/bindings/memory-controllers/brcm,brcmstb-memc-ddr.yaml
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/memory-controllers/brcm,brcmstb-memc-ddr.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Memory controller (MEMC) for Broadcom STB
+
+maintainers:
+  - Florian Fainelli <f.fainelli@gmail.com>
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - brcm,brcmstb-memc-ddr-rev-b.1.x
+          - brcm,brcmstb-memc-ddr-rev-b.2.0
+          - brcm,brcmstb-memc-ddr-rev-b.2.1
+          - brcm,brcmstb-memc-ddr-rev-b.2.2
+          - brcm,brcmstb-memc-ddr-rev-b.2.3
+          - brcm,brcmstb-memc-ddr-rev-b.2.5
+          - brcm,brcmstb-memc-ddr-rev-b.2.6
+          - brcm,brcmstb-memc-ddr-rev-b.2.7
+          - brcm,brcmstb-memc-ddr-rev-b.2.8
+          - brcm,brcmstb-memc-ddr-rev-b.3.0
+          - brcm,brcmstb-memc-ddr-rev-b.3.1
+          - brcm,brcmstb-memc-ddr-rev-c.1.0
+          - brcm,brcmstb-memc-ddr-rev-c.1.1
+          - brcm,brcmstb-memc-ddr-rev-c.1.2
+          - brcm,brcmstb-memc-ddr-rev-c.1.3
+          - brcm,brcmstb-memc-ddr-rev-c.1.4
+      - const: brcm,brcmstb-memc-ddr
+
+  reg:
+    maxItems: 1
+
+  clock-frequency:
+    description: DDR PHY frequency in Hz
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    memory-controller@9902000 {
+        compatible = "brcm,brcmstb-memc-ddr-rev-c.1.1", "brcm,brcmstb-memc-ddr";
+        reg = <0x9902000 0x600>;
+        clock-frequency = <2133000000>;
+    };
diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,mt7621-memc.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,mt7621-memc.yaml
index 85e02854f083..6ccdaf99c778 100644
--- a/Documentation/devicetree/bindings/memory-controllers/mediatek,mt7621-memc.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,mt7621-memc.yaml
@@ -11,7 +11,9 @@ maintainers:
 
 properties:
   compatible:
-    const: mediatek,mt7621-memc
+    items:
+      - const: mediatek,mt7621-memc
+      - const: syscon
 
   reg:
     maxItems: 1
@@ -25,6 +27,6 @@ additionalProperties: false
 examples:
   - |
     memory-controller@5000 {
-        compatible = "mediatek,mt7621-memc";
+        compatible = "mediatek,mt7621-memc", "syscon";
         reg = <0x5000 0x1000>;
     };
diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml
index 71bc5cefb49c..a8fda30cccbb 100644
--- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.yaml
@@ -16,7 +16,7 @@ description: |
   MediaTek SMI have two generations of HW architecture, here is the list
   which generation the SoCs use:
   generation 1: mt2701 and mt7623.
-  generation 2: mt2712, mt6779, mt8167, mt8173, mt8183, mt8186, mt8192 and mt8195.
+  generation 2: mt2712, mt6779, mt8167, mt8173, mt8183, mt8186, mt8188, mt8192 and mt8195.
 
   There's slight differences between the two SMI, for generation 2, the
   register which control the iommu port is at each larb's register base. But
@@ -37,6 +37,8 @@ properties:
           - mediatek,mt8173-smi-common
           - mediatek,mt8183-smi-common
           - mediatek,mt8186-smi-common
+          - mediatek,mt8188-smi-common-vdo
+          - mediatek,mt8188-smi-common-vpp
           - mediatek,mt8192-smi-common
           - mediatek,mt8195-smi-common-vdo
           - mediatek,mt8195-smi-common-vpp
@@ -144,7 +146,16 @@ allOf:
             - const: gals0
             - const: gals1
 
-    else:  # for gen2 HW that don't have gals
+  - if:  # for gen2 HW that don't have gals
+      properties:
+        compatible:
+          enum:
+            - mediatek,mt2712-smi-common
+            - mediatek,mt6795-smi-common
+            - mediatek,mt8167-smi-common
+            - mediatek,mt8173-smi-common
+
+    then:
       properties:
         clocks:
           minItems: 2
diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml
index 59dcd163668f..5f4ac3609887 100644
--- a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.yaml
@@ -25,6 +25,7 @@ properties:
           - mediatek,mt8173-smi-larb
           - mediatek,mt8183-smi-larb
           - mediatek,mt8186-smi-larb
+          - mediatek,mt8188-smi-larb
           - mediatek,mt8192-smi-larb
           - mediatek,mt8195-smi-larb
 
@@ -78,6 +79,7 @@ allOf:
           enum:
             - mediatek,mt8183-smi-larb
             - mediatek,mt8186-smi-larb
+            - mediatek,mt8188-smi-larb
             - mediatek,mt8195-smi-larb
 
     then:
@@ -111,6 +113,7 @@ allOf:
               - mediatek,mt2712-smi-larb
               - mediatek,mt6779-smi-larb
               - mediatek,mt8186-smi-larb
+              - mediatek,mt8188-smi-larb
               - mediatek,mt8192-smi-larb
               - mediatek,mt8195-smi-larb
 
diff --git a/Documentation/devicetree/bindings/memory-controllers/snps,dw-umctl2-ddrc.yaml b/Documentation/devicetree/bindings/memory-controllers/snps,dw-umctl2-ddrc.yaml
new file mode 100644
index 000000000000..e68c4306025a
--- /dev/null
+++ b/Documentation/devicetree/bindings/memory-controllers/snps,dw-umctl2-ddrc.yaml
@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: GPL-2.0-only
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/memory-controllers/snps,dw-umctl2-ddrc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Synopsys DesignWare Universal Multi-Protocol Memory Controller
+
+maintainers:
+  - Krzysztof Kozlowski <krzk@kernel.org>
+  - Manish Narani <manish.narani@xilinx.com>
+  - Michal Simek <michal.simek@xilinx.com>
+
+description: |
+  Synopsys DesignWare Enhanced uMCTL2 DDR Memory Controller is capable of
+  working with the memory devices supporting up to (LP)DDR4 protocol. It can
+  be equipped with SEC/DEC ECC feature if DRAM data bus width is either
+  16-bits or 32-bits or 64-bits wide.
+
+  For instance the ZynqMP DDR controller is based on the DW uMCTL2 v2.40a
+  controller. It has an optional SEC/DEC ECC support in 64- and 32-bits
+  bus width configurations.
+
+properties:
+  compatible:
+    oneOf:
+      - deprecated: true
+        description: Synopsys DW uMCTL2 DDR controller v3.80a
+        const: snps,ddrc-3.80a
+      - description: Synopsys DW uMCTL2 DDR controller
+        const: snps,dw-umctl2-ddrc
+      - description: Xilinx ZynqMP DDR controller v2.40a
+        const: xlnx,zynqmp-ddrc-2.40a
+
+  interrupts:
+    description:
+      DW uMCTL2 DDRC IP-core provides individual IRQ signal for each event":"
+      ECC Corrected Error, ECC Uncorrected Error, ECC Address Protection,
+      Scrubber-Done signal, DFI Parity/CRC Error. Some platforms may have the
+      signals merged before they reach the IRQ controller or have some of them
+      absent in case if the corresponding feature is unavailable/disabled.
+    minItems: 1
+    maxItems: 5
+
+  interrupt-names:
+    minItems: 1
+    maxItems: 5
+    oneOf:
+      - description: Common ECC CE/UE/Scrubber/DFI Errors IRQ
+        items:
+          - const: ecc
+      - description: Individual ECC CE/UE/Scrubber/DFI Errors IRQs
+        items:
+          enum: [ ecc_ce, ecc_ue, ecc_ap, ecc_sbr, dfi_e ]
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    description:
+      A standard set of the clock sources contains CSRs bus clock, AXI-ports
+      reference clock, DDRC core clock, Scrubber standalone clock
+      (synchronous to the DDRC clock).
+    minItems: 1
+    maxItems: 4
+
+  clock-names:
+    minItems: 1
+    maxItems: 4
+    items:
+      enum: [ pclk, aclk, core, sbr ]
+
+  resets:
+    description:
+      Each clock domain can have separate reset signal.
+    minItems: 1
+    maxItems: 4
+
+  reset-names:
+    minItems: 1
+    maxItems: 4
+    items:
+      enum: [ prst, arst, core, sbr ]
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    memory-controller@fd070000 {
+      compatible = "xlnx,zynqmp-ddrc-2.40a";
+      reg = <0xfd070000 0x30000>;
+
+      interrupt-parent = <&gic>;
+      interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
+      interrupt-names = "ecc";
+    };
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    memory-controller@3d400000 {
+      compatible = "snps,dw-umctl2-ddrc";
+      reg = <0x3d400000 0x400000>;
+
+      interrupts = <147 IRQ_TYPE_LEVEL_HIGH>, <148 IRQ_TYPE_LEVEL_HIGH>,
+                   <149 IRQ_TYPE_LEVEL_HIGH>, <150 IRQ_TYPE_LEVEL_HIGH>;
+      interrupt-names = "ecc_ce", "ecc_ue", "ecc_sbr", "dfi_e";
+
+      clocks = <&pclk>, <&aclk>, <&core_clk>, <&sbr_clk>;
+      clock-names = "pclk", "aclk", "core", "sbr";
+    };
+...
diff --git a/Documentation/devicetree/bindings/memory-controllers/synopsys,ddrc-ecc.yaml b/Documentation/devicetree/bindings/memory-controllers/synopsys,ddrc-ecc.yaml
deleted file mode 100644
index f46e95704f53..000000000000
--- a/Documentation/devicetree/bindings/memory-controllers/synopsys,ddrc-ecc.yaml
+++ /dev/null
@@ -1,76 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/memory-controllers/synopsys,ddrc-ecc.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Synopsys IntelliDDR Multi Protocol memory controller
-
-maintainers:
-  - Krzysztof Kozlowski <krzk@kernel.org>
-  - Manish Narani <manish.narani@xilinx.com>
-  - Michal Simek <michal.simek@xilinx.com>
-
-description: |
-  The ZynqMP DDR ECC controller has an optional ECC support in 64-bit and
-  32-bit bus width configurations.
-
-  The Zynq DDR ECC controller has an optional ECC support in half-bus width
-  (16-bit) configuration.
-
-  These both ECC controllers correct single bit ECC errors and detect double bit
-  ECC errors.
-
-properties:
-  compatible:
-    enum:
-      - snps,ddrc-3.80a
-      - xlnx,zynq-ddrc-a05
-      - xlnx,zynqmp-ddrc-2.40a
-
-  interrupts:
-    maxItems: 1
-
-  reg:
-    maxItems: 1
-
-required:
-  - compatible
-  - reg
-
-allOf:
-  - if:
-      properties:
-        compatible:
-          contains:
-            enum:
-              - snps,ddrc-3.80a
-              - xlnx,zynqmp-ddrc-2.40a
-    then:
-      required:
-        - interrupts
-    else:
-      properties:
-        interrupts: false
-
-additionalProperties: false
-
-examples:
-  - |
-    memory-controller@f8006000 {
-        compatible = "xlnx,zynq-ddrc-a05";
-        reg = <0xf8006000 0x1000>;
-    };
-
-  - |
-    axi {
-        #address-cells = <2>;
-        #size-cells = <2>;
-
-        memory-controller@fd070000 {
-            compatible = "xlnx,zynqmp-ddrc-2.40a";
-            reg = <0x0 0xfd070000 0x0 0x30000>;
-            interrupt-parent = <&gic>;
-            interrupts = <0 112 4>;
-        };
-    };
diff --git a/Documentation/devicetree/bindings/memory-controllers/xlnx,zynq-ddrc-a05.yaml b/Documentation/devicetree/bindings/memory-controllers/xlnx,zynq-ddrc-a05.yaml
new file mode 100644
index 000000000000..8f72e2f8588a
--- /dev/null
+++ b/Documentation/devicetree/bindings/memory-controllers/xlnx,zynq-ddrc-a05.yaml
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/memory-controllers/xlnx,zynq-ddrc-a05.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Zynq A05 DDR Memory Controller
+
+maintainers:
+  - Krzysztof Kozlowski <krzk@kernel.org>
+  - Manish Narani <manish.narani@xilinx.com>
+  - Michal Simek <michal.simek@xilinx.com>
+
+description:
+  The Zynq DDR ECC controller has an optional ECC support in half-bus width
+  (16-bit) configuration. It is cappable of correcting single bit ECC errors
+  and detecting double bit ECC errors.
+
+properties:
+  compatible:
+    const: xlnx,zynq-ddrc-a05
+
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    memory-controller@f8006000 {
+      compatible = "xlnx,zynq-ddrc-a05";
+      reg = <0xf8006000 0x1000>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/mfd/mscc,ocelot.yaml b/Documentation/devicetree/bindings/mfd/mscc,ocelot.yaml
new file mode 100644
index 000000000000..8bf45a5673a4
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/mscc,ocelot.yaml
@@ -0,0 +1,160 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mfd/mscc,ocelot.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Ocelot Externally-Controlled Ethernet Switch
+
+maintainers:
+  - Colin Foster <colin.foster@in-advantage.com>
+
+description: |
+  The Ocelot ethernet switch family contains chips that have an internal CPU
+  (VSC7513, VSC7514) and chips that don't (VSC7511, VSC7512). All switches have
+  the option to be controlled externally, which is the purpose of this driver.
+
+  The switch family is a multi-port networking switch that supports many
+  interfaces. Additionally, the device can perform pin control, MDIO buses, and
+  external GPIO expanders.
+
+properties:
+  compatible:
+    enum:
+      - mscc,vsc7512
+
+  reg:
+    maxItems: 1
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 1
+
+  spi-max-frequency:
+    maxItems: 1
+
+patternProperties:
+  "^pinctrl@[0-9a-f]+$":
+    type: object
+    $ref: /schemas/pinctrl/mscc,ocelot-pinctrl.yaml
+
+  "^gpio@[0-9a-f]+$":
+    type: object
+    $ref: /schemas/pinctrl/microchip,sparx5-sgpio.yaml
+    properties:
+      compatible:
+        enum:
+          - mscc,ocelot-sgpio
+
+  "^mdio@[0-9a-f]+$":
+    type: object
+    $ref: /schemas/net/mscc,miim.yaml
+    properties:
+      compatible:
+        enum:
+          - mscc,ocelot-miim
+
+required:
+  - compatible
+  - reg
+  - '#address-cells'
+  - '#size-cells'
+  - spi-max-frequency
+
+additionalProperties: false
+
+examples:
+  - |
+    ocelot_clock: ocelot-clock {
+          compatible = "fixed-clock";
+          #clock-cells = <0>;
+          clock-frequency = <125000000>;
+      };
+
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        soc@0 {
+            compatible = "mscc,vsc7512";
+            spi-max-frequency = <2500000>;
+            reg = <0>;
+            #address-cells = <1>;
+            #size-cells = <1>;
+
+            mdio@7107009c {
+                compatible = "mscc,ocelot-miim";
+                #address-cells = <1>;
+                #size-cells = <0>;
+                reg = <0x7107009c 0x24>;
+
+                sw_phy0: ethernet-phy@0 {
+                    reg = <0x0>;
+                };
+            };
+
+            mdio@710700c0 {
+                compatible = "mscc,ocelot-miim";
+                pinctrl-names = "default";
+                pinctrl-0 = <&miim1_pins>;
+                #address-cells = <1>;
+                #size-cells = <0>;
+                reg = <0x710700c0 0x24>;
+
+                sw_phy4: ethernet-phy@4 {
+                    reg = <0x4>;
+                };
+            };
+
+            gpio: pinctrl@71070034 {
+                compatible = "mscc,ocelot-pinctrl";
+                gpio-controller;
+                #gpio-cells = <2>;
+                gpio-ranges = <&gpio 0 0 22>;
+                reg = <0x71070034 0x6c>;
+
+                sgpio_pins: sgpio-pins {
+                    pins = "GPIO_0", "GPIO_1", "GPIO_2", "GPIO_3";
+                    function = "sg0";
+                };
+
+                miim1_pins: miim1-pins {
+                    pins = "GPIO_14", "GPIO_15";
+                    function = "miim";
+                };
+            };
+
+            gpio@710700f8 {
+                compatible = "mscc,ocelot-sgpio";
+                #address-cells = <1>;
+                #size-cells = <0>;
+                bus-frequency = <12500000>;
+                clocks = <&ocelot_clock>;
+                microchip,sgpio-port-ranges = <0 15>;
+                pinctrl-names = "default";
+                pinctrl-0 = <&sgpio_pins>;
+                reg = <0x710700f8 0x100>;
+
+                sgpio_in0: gpio@0 {
+                    compatible = "microchip,sparx5-sgpio-bank";
+                    reg = <0>;
+                    gpio-controller;
+                    #gpio-cells = <3>;
+                    ngpios = <64>;
+                };
+
+                sgpio_out1: gpio@1 {
+                    compatible = "microchip,sparx5-sgpio-bank";
+                    reg = <1>;
+                    gpio-controller;
+                    #gpio-cells = <3>;
+                    ngpios = <64>;
+                };
+            };
+        };
+    };
+
+...
+
diff --git a/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml b/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml
index 2f816fd0c9ec..d3c25daa995e 100644
--- a/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml
+++ b/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml
@@ -18,6 +18,13 @@ properties:
     oneOf:
       - items:
           - enum:
+              - qcom,msm8998-tcsr
+              - qcom,qcs404-tcsr
+              - qcom,sc7180-tcsr
+              - qcom,sc7280-tcsr
+              - qcom,sdm630-tcsr
+              - qcom,sdm845-tcsr
+              - qcom,sm8150-tcsr
               - qcom,tcsr-apq8064
               - qcom,tcsr-apq8084
               - qcom,tcsr-ipq8064
@@ -27,6 +34,7 @@ properties:
               - qcom,tcsr-msm8953
               - qcom,tcsr-msm8960
               - qcom,tcsr-msm8974
+              - qcom,tcsr-msm8996
           - const: syscon
       - items:
           - const: qcom,tcsr-ipq6018
diff --git a/Documentation/devicetree/bindings/mfd/syscon.yaml b/Documentation/devicetree/bindings/mfd/syscon.yaml
index c10f0b577268..5cbf2c5978b3 100644
--- a/Documentation/devicetree/bindings/mfd/syscon.yaml
+++ b/Documentation/devicetree/bindings/mfd/syscon.yaml
@@ -40,6 +40,8 @@ properties:
               - allwinner,sun50i-a64-system-controller
               - brcm,cru-clkset
               - freecom,fsg-cs2-system-controller
+              - fsl,imx93-aonmix-ns-syscfg
+              - fsl,imx93-wakeupmix-syscfg
               - hisilicon,dsa-subctrl
               - hisilicon,hi6220-sramctrl
               - hisilicon,pcie-sas-subctrl
diff --git a/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml b/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml
index 4207fed62dfe..8b1a0fdcb5e3 100644
--- a/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/cdns,sdhci.yaml
@@ -8,7 +8,6 @@ title: Cadence SD/SDIO/eMMC Host Controller (SD4HC)
 
 maintainers:
   - Masahiro Yamada <yamada.masahiro@socionext.com>
-  - Piotr Sroka <piotrs@cadence.com>
 
 allOf:
   - $ref: mmc-controller.yaml
diff --git a/Documentation/devicetree/bindings/mmc/mmc-controller.yaml b/Documentation/devicetree/bindings/mmc/mmc-controller.yaml
index ff5ce89e5111..802e3ca8be4d 100644
--- a/Documentation/devicetree/bindings/mmc/mmc-controller.yaml
+++ b/Documentation/devicetree/bindings/mmc/mmc-controller.yaml
@@ -88,11 +88,18 @@ properties:
     default: 1
 
   max-frequency:
-    description:
-      Maximum operating frequency of the bus.
+    description: |
+      Maximum operating frequency of the bus:
+        - for eMMC, the maximum supported frequency is 200MHz,
+        - for SD/SDIO cards the SDR104 mode has a max supported
+          frequency of 208MHz,
+        - some mmc host controllers do support a max frequency upto
+          384MHz.
+      So, lets keep the maximum supported value here.
+
     $ref: /schemas/types.yaml#/definitions/uint32
     minimum: 400000
-    maximum: 200000000
+    maximum: 384000000
 
   disable-wp:
     $ref: /schemas/types.yaml#/definitions/flag
diff --git a/Documentation/devicetree/bindings/mmc/mmc-spi-slot.yaml b/Documentation/devicetree/bindings/mmc/mmc-spi-slot.yaml
index c45b91099325..c0662ce9946d 100644
--- a/Documentation/devicetree/bindings/mmc/mmc-spi-slot.yaml
+++ b/Documentation/devicetree/bindings/mmc/mmc-spi-slot.yaml
@@ -23,8 +23,6 @@ properties:
   reg:
     maxItems: 1
 
-  spi-max-frequency: true
-
   interrupts:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/mmc/mtk-sd.yaml b/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
index 083d1ec2f661..d8e1e2e9adf2 100644
--- a/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
+++ b/Documentation/devicetree/bindings/mmc/mtk-sd.yaml
@@ -20,6 +20,7 @@ properties:
           - mediatek,mt2701-mmc
           - mediatek,mt2712-mmc
           - mediatek,mt6779-mmc
+          - mediatek,mt6795-mmc
           - mediatek,mt7620-mmc
           - mediatek,mt7622-mmc
           - mediatek,mt8135-mmc
diff --git a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml
index 14945ebc31d2..0424b06cb655 100644
--- a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml
+++ b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml
@@ -89,6 +89,9 @@ properties:
         - tx
         - rx
 
+  iommus:
+    maxItems: 1
+
   power-domains:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml
index 8d888b435817..ccf0e3e93efa 100644
--- a/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml
+++ b/Documentation/devicetree/bindings/mmc/rockchip-dw-mshc.yaml
@@ -32,6 +32,7 @@ properties:
               - rockchip,px30-dw-mshc
               - rockchip,rk1808-dw-mshc
               - rockchip,rk3036-dw-mshc
+              - rockchip,rk3128-dw-mshc
               - rockchip,rk3228-dw-mshc
               - rockchip,rk3308-dw-mshc
               - rockchip,rk3328-dw-mshc
diff --git a/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml b/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml
index 0ab07759b472..ea9121fb188d 100644
--- a/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml
+++ b/Documentation/devicetree/bindings/mmc/sdhci-am654.yaml
@@ -51,6 +51,9 @@ properties:
 
   sdhci-caps-mask: true
 
+  dma-coherent:
+    type: boolean
+
   # PHY output tap delays:
   # Used to delay the data valid window and align it to the sampling clock.
   # Binding needs to be provided for each supported speed mode otherwise the
diff --git a/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml b/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml
index fc0e81c2066c..a96f143479c7 100644
--- a/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml
+++ b/Documentation/devicetree/bindings/mmc/sdhci-msm.yaml
@@ -38,9 +38,11 @@ properties:
               - qcom,sc7180-sdhci
               - qcom,sc7280-sdhci
               - qcom,sdm630-sdhci
+              - qcom,sdm670-sdhci
               - qcom,sdm845-sdhci
               - qcom,sdx55-sdhci
               - qcom,sdx65-sdhci
+              - qcom,sm6115-sdhci
               - qcom,sm6125-sdhci
               - qcom,sm6350-sdhci
               - qcom,sm8150-sdhci
@@ -96,6 +98,10 @@ properties:
     description:
       Should specify pin control groups used for this controller.
 
+  pinctrl-1:
+    description:
+      Should specify sleep pin control groups used for this controller.
+
   resets:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/net/adi,adin1110.yaml b/Documentation/devicetree/bindings/net/adi,adin1110.yaml
new file mode 100644
index 000000000000..b6bd8ee38a18
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/adi,adin1110.yaml
@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/adi,adin1110.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ADI ADIN1110 MAC-PHY
+
+maintainers:
+  - Alexandru Tachici <alexandru.tachici@analog.com>
+
+description: |
+  The ADIN1110 is a low power single port 10BASE-T1L MAC-
+  PHY designed for industrial Ethernet applications. It integrates
+  an Ethernet PHY core with a MAC and all the associated analog
+  circuitry, input and output clock buffering.
+
+  The ADIN2111 is a low power, low complexity, two-Ethernet ports
+  switch with integrated 10BASE-T1L PHYs and one serial peripheral
+  interface (SPI) port. The device is designed for industrial Ethernet
+  applications using low power constrained nodes and is compliant
+  with the IEEE 802.3cg-2019 Ethernet standard for long reach
+  10 Mbps single pair Ethernet (SPE).
+
+  The device has a 4-wire SPI interface for communication
+  between the MAC and host processor.
+
+allOf:
+  - $ref: ethernet-controller.yaml#
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+properties:
+  compatible:
+    enum:
+      - adi,adin1110
+      - adi,adin2111
+
+  reg:
+    maxItems: 1
+
+  adi,spi-crc:
+    description: |
+      Enable CRC8 checks on SPI read/writes.
+    type: boolean
+
+  interrupts:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    spi {
+
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        ethernet@0 {
+            compatible = "adi,adin2111";
+            reg = <0>;
+            spi-max-frequency = <24500000>;
+
+            adi,spi-crc;
+
+            interrupt-parent = <&gpio>;
+            interrupts = <25 IRQ_TYPE_LEVEL_LOW>;
+
+            local-mac-address = [ 00 11 22 33 44 55 ];
+        };
+    };
diff --git a/Documentation/devicetree/bindings/net/altera_tse.txt b/Documentation/devicetree/bindings/net/altera_tse.txt
deleted file mode 100644
index 1d9148ff5130..000000000000
--- a/Documentation/devicetree/bindings/net/altera_tse.txt
+++ /dev/null
@@ -1,113 +0,0 @@
-* Altera Triple-Speed Ethernet MAC driver (TSE)
-
-Required properties:
-- compatible: Should be "altr,tse-1.0" for legacy SGDMA based TSE, and should
-		be "altr,tse-msgdma-1.0" for the preferred MSGDMA based TSE.
-		ALTR is supported for legacy device trees, but is deprecated.
-		altr should be used for all new designs.
-- reg: Address and length of the register set for the device. It contains
-  the information of registers in the same order as described by reg-names
-- reg-names: Should contain the reg names
-  "control_port": MAC configuration space region
-  "tx_csr":       xDMA Tx dispatcher control and status space region
-  "tx_desc":      MSGDMA Tx dispatcher descriptor space region
-  "rx_csr" :      xDMA Rx dispatcher control and status space region
-  "rx_desc":      MSGDMA Rx dispatcher descriptor space region
-  "rx_resp":      MSGDMA Rx dispatcher response space region
-  "s1":		  SGDMA descriptor memory
-- interrupts: Should contain the TSE interrupts and its mode.
-- interrupt-names: Should contain the interrupt names
-  "rx_irq":       xDMA Rx dispatcher interrupt
-  "tx_irq":       xDMA Tx dispatcher interrupt
-- rx-fifo-depth: MAC receive FIFO buffer depth in bytes
-- tx-fifo-depth: MAC transmit FIFO buffer depth in bytes
-- phy-mode: See ethernet.txt in the same directory.
-- phy-handle: See ethernet.txt in the same directory.
-- phy-addr: See ethernet.txt in the same directory. A configuration should
-		include phy-handle or phy-addr.
-- altr,has-supplementary-unicast:
-		If present, TSE supports additional unicast addresses.
-		Otherwise additional unicast addresses are not supported.
-- altr,has-hash-multicast-filter:
-		If present, TSE supports a hash based multicast filter.
-		Otherwise, hash-based multicast filtering is not supported.
-
-- mdio device tree subnode: When the TSE has a phy connected to its local
-		mdio, there must be device tree subnode with the following
-		required properties:
-
-	- compatible: Must be "altr,tse-mdio".
-	- #address-cells: Must be <1>.
-	- #size-cells: Must be <0>.
-
-	For each phy on the mdio bus, there must be a node with the following
-	fields:
-
-	- reg: phy id used to communicate to phy.
-	- device_type: Must be "ethernet-phy".
-
-The MAC address will be determined using the optional properties defined in
-ethernet.txt.
-
-Example:
-
-	tse_sub_0_eth_tse_0: ethernet@1,00000000 {
-		compatible = "altr,tse-msgdma-1.0";
-		reg =	<0x00000001 0x00000000 0x00000400>,
-			<0x00000001 0x00000460 0x00000020>,
-			<0x00000001 0x00000480 0x00000020>,
-			<0x00000001 0x000004A0 0x00000008>,
-			<0x00000001 0x00000400 0x00000020>,
-			<0x00000001 0x00000420 0x00000020>;
-		reg-names = "control_port", "rx_csr", "rx_desc", "rx_resp", "tx_csr", "tx_desc";
-		interrupt-parent = <&hps_0_arm_gic_0>;
-		interrupts = <0 41 4>, <0 40 4>;
-		interrupt-names = "rx_irq", "tx_irq";
-		rx-fifo-depth = <2048>;
-		tx-fifo-depth = <2048>;
-		address-bits = <48>;
-		max-frame-size = <1500>;
-		local-mac-address = [ 00 00 00 00 00 00 ];
-		phy-mode = "gmii";
-		altr,has-supplementary-unicast;
-		altr,has-hash-multicast-filter;
-		phy-handle = <&phy0>;
-		mdio {
-			compatible = "altr,tse-mdio";
-			#address-cells = <1>;
-			#size-cells = <0>;
-			phy0: ethernet-phy@0 {
-				reg = <0x0>;
-				device_type = "ethernet-phy";
-			};
-
-			phy1: ethernet-phy@1 {
-				reg = <0x1>;
-				device_type = "ethernet-phy";
-			};
-
-		};
-	};
-
-	tse_sub_1_eth_tse_0: ethernet@1,00001000 {
-		compatible = "altr,tse-msgdma-1.0";
-		reg = 	<0x00000001 0x00001000 0x00000400>,
-			<0x00000001 0x00001460 0x00000020>,
-			<0x00000001 0x00001480 0x00000020>,
-			<0x00000001 0x000014A0 0x00000008>,
-			<0x00000001 0x00001400 0x00000020>,
-			<0x00000001 0x00001420 0x00000020>;
-		reg-names = "control_port", "rx_csr", "rx_desc", "rx_resp", "tx_csr", "tx_desc";
-		interrupt-parent = <&hps_0_arm_gic_0>;
-		interrupts = <0 43 4>, <0 42 4>;
-		interrupt-names = "rx_irq", "tx_irq";
-		rx-fifo-depth = <2048>;
-		tx-fifo-depth = <2048>;
-		address-bits = <48>;
-		max-frame-size = <1500>;
-		local-mac-address = [ 00 00 00 00 00 00 ];
-		phy-mode = "gmii";
-		altr,has-supplementary-unicast;
-		altr,has-hash-multicast-filter;
-		phy-handle = <&phy1>;
-	};
diff --git a/Documentation/devicetree/bindings/net/altr,tse.yaml b/Documentation/devicetree/bindings/net/altr,tse.yaml
new file mode 100644
index 000000000000..8d1d94494349
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/altr,tse.yaml
@@ -0,0 +1,168 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/altr,tse.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Altera Triple Speed Ethernet MAC driver (TSE)
+
+maintainers:
+  - Maxime Chevallier <maxime.chevallier@bootlin.com>
+
+properties:
+  compatible:
+    oneOf:
+      - const: altr,tse-1.0
+      - const: ALTR,tse-1.0
+        deprecated: true
+      - const: altr,tse-msgdma-1.0
+
+  interrupts:
+    minItems: 2
+
+  interrupt-names:
+    items:
+      - const: rx_irq
+      - const: tx_irq
+
+  rx-fifo-depth:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Depth in bytes of the RX FIFO
+
+  tx-fifo-depth:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Depth in bytes of the TX FIFO
+
+  altr,has-supplementary-unicast:
+    type: boolean
+    description:
+      If present, TSE supports additional unicast addresses.
+
+  altr,has-hash-multicast-filter:
+    type: boolean
+    description:
+      If present, TSE supports hash based multicast filter.
+
+  mdio:
+    $ref: mdio.yaml#
+    unevaluatedProperties: false
+    description:
+      Creates and registers an MDIO bus.
+
+    properties:
+      compatible:
+        const: altr,tse-mdio
+
+    required:
+      - compatible
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - rx-fifo-depth
+  - tx-fifo-depth
+
+allOf:
+  - $ref: "ethernet-controller.yaml#"
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - const: altr,tse-1.0
+              - const: ALTR,tse-1.0
+    then:
+      properties:
+        reg:
+          minItems: 4
+        reg-names:
+          items:
+            - const: control_port
+            - const: rx_csr
+            - const: tx_csr
+            - const: s1
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - altr,tse-msgdma-1.0
+    then:
+      properties:
+        reg:
+          minItems: 6
+          maxItems: 7
+        reg-names:
+          minItems: 6
+          items:
+            - const: control_port
+            - const: rx_csr
+            - const: rx_desc
+            - const: rx_resp
+            - const: tx_csr
+            - const: tx_desc
+            - const: pcs
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    tse_sub_0: ethernet@c0100000 {
+        compatible = "altr,tse-msgdma-1.0";
+        reg = <0xc0100000 0x00000400>,
+              <0xc0101000 0x00000020>,
+              <0xc0102000 0x00000020>,
+              <0xc0103000 0x00000008>,
+              <0xc0104000 0x00000020>,
+              <0xc0105000 0x00000020>,
+              <0xc0106000 0x00000100>;
+        reg-names = "control_port", "rx_csr", "rx_desc", "rx_resp", "tx_csr", "tx_desc", "pcs";
+        interrupt-parent = <&intc>;
+        interrupts = <0 44 4>,<0 45 4>;
+        interrupt-names = "rx_irq","tx_irq";
+        rx-fifo-depth = <2048>;
+        tx-fifo-depth = <2048>;
+        max-frame-size = <1500>;
+        local-mac-address = [ 00 00 00 00 00 00 ];
+        altr,has-supplementary-unicast;
+        altr,has-hash-multicast-filter;
+        sfp = <&sfp0>;
+        phy-mode = "sgmii";
+        managed = "in-band-status";
+    };
+  - |
+    tse_sub_1_eth_tse_0: ethernet@1,00001000 {
+        compatible = "altr,tse-msgdma-1.0";
+        reg = <0x00001000 0x00000400>,
+              <0x00001460 0x00000020>,
+              <0x00001480 0x00000020>,
+              <0x000014A0 0x00000008>,
+              <0x00001400 0x00000020>,
+              <0x00001420 0x00000020>;
+        reg-names = "control_port", "rx_csr", "rx_desc", "rx_resp", "tx_csr", "tx_desc";
+        interrupt-parent = <&hps_0_arm_gic_0>;
+        interrupts = <0 43 4>, <0 42 4>;
+        interrupt-names = "rx_irq", "tx_irq";
+        rx-fifo-depth = <2048>;
+        tx-fifo-depth = <2048>;
+        max-frame-size = <1500>;
+        local-mac-address = [ 00 00 00 00 00 00 ];
+        phy-mode = "gmii";
+        altr,has-supplementary-unicast;
+        altr,has-hash-multicast-filter;
+        phy-handle = <&phy1>;
+        mdio {
+            compatible = "altr,tse-mdio";
+            #address-cells = <1>;
+            #size-cells = <0>;
+            phy1: ethernet-phy@1 {
+                reg = <0x1>;
+            };
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml b/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml
index b1327c5b86cf..144a3785132c 100644
--- a/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml
+++ b/Documentation/devicetree/bindings/net/can/nxp,sja1000.yaml
@@ -30,8 +30,10 @@ properties:
   clocks:
     maxItems: 1
 
+  power-domains:
+    maxItems: 1
+
   reg-io-width:
-    $ref: /schemas/types.yaml#/definitions/uint32
     description: I/O register width (in bytes) implemented by this device
     default: 1
     enum: [ 1, 2, 4 ]
@@ -105,6 +107,7 @@ allOf:
     then:
       required:
         - clocks
+        - power-domains
 
 unevaluatedProperties: false
 
@@ -129,4 +132,5 @@ examples:
         reg-io-width = <4>;
         interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
         clocks = <&sysctrl R9A06G032_HCLK_CAN0>;
+        power-domains = <&sysctrl>;
     };
diff --git a/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml b/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml
index cc01b9b5752a..253b5d1407ee 100644
--- a/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml
+++ b/Documentation/devicetree/bindings/net/cortina,gemini-ethernet.yaml
@@ -37,6 +37,7 @@ properties:
 patternProperties:
   "^ethernet-port@[0-9]+$":
     type: object
+    unevaluatedProperties: false
     description: contains the resources for ethernet port
     allOf:
       - $ref: ethernet-controller.yaml#
diff --git a/Documentation/devicetree/bindings/net/dsa/ar9331.txt b/Documentation/devicetree/bindings/net/dsa/ar9331.txt
index 320607cbbb17..f824fdae0da2 100644
--- a/Documentation/devicetree/bindings/net/dsa/ar9331.txt
+++ b/Documentation/devicetree/bindings/net/dsa/ar9331.txt
@@ -76,7 +76,6 @@ eth1: ethernet@1a000000 {
 
 				switch_port0: port@0 {
 					reg = <0x0>;
-					label = "cpu";
 					ethernet = <&eth1>;
 
 					phy-mode = "gmii";
diff --git a/Documentation/devicetree/bindings/net/dsa/arrow,xrs700x.yaml b/Documentation/devicetree/bindings/net/dsa/arrow,xrs700x.yaml
index 3f01b65f3b22..259a0c6547f3 100644
--- a/Documentation/devicetree/bindings/net/dsa/arrow,xrs700x.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/arrow,xrs700x.yaml
@@ -61,8 +61,9 @@ examples:
                 };
                 ethernet-port@3 {
                     reg = <3>;
-                    label = "cpu";
                     ethernet = <&fec1>;
+                    phy-mode = "rgmii-id";
+
                     fixed-link {
                         speed = <1000>;
                         full-duplex;
diff --git a/Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml b/Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml
index 23114d691d2a..1219b830b1a4 100644
--- a/Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/brcm,b53.yaml
@@ -169,7 +169,6 @@ examples:
 
                 port@8 {
                     reg = <8>;
-                    label = "cpu";
                     phy-mode = "rgmii-txid";
                     ethernet = <&eth0>;
                     fixed-link {
@@ -252,8 +251,9 @@ examples:
 
                 port@8 {
                     ethernet = <&amac2>;
-                    label = "cpu";
                     reg = <8>;
+                    phy-mode = "internal";
+
                     fixed-link {
                         speed = <1000>;
                         full-duplex;
diff --git a/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml b/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml
index 09317e16cb5d..10ad7e71097b 100644
--- a/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml
@@ -76,6 +76,23 @@ properties:
 required:
   - reg
 
+# CPU and DSA ports must have phylink-compatible link descriptions
+if:
+  oneOf:
+    - required: [ ethernet ]
+    - required: [ link ]
+then:
+  allOf:
+    - required:
+        - phy-mode
+    - oneOf:
+        - required:
+            - fixed-link
+        - required:
+            - phy-handle
+        - required:
+            - managed
+
 additionalProperties: true
 
 ...
diff --git a/Documentation/devicetree/bindings/net/dsa/hirschmann,hellcreek.yaml b/Documentation/devicetree/bindings/net/dsa/hirschmann,hellcreek.yaml
index 228683773151..73b774eadd0b 100644
--- a/Documentation/devicetree/bindings/net/dsa/hirschmann,hellcreek.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/hirschmann,hellcreek.yaml
@@ -91,8 +91,13 @@ examples:
 
                 port@0 {
                     reg = <0>;
-                    label = "cpu";
                     ethernet = <&gmac0>;
+                    phy-mode = "mii";
+
+                    fixed-link {
+                        speed = <100>;
+                        full-duplex;
+                    };
                 };
 
                 port@2 {
diff --git a/Documentation/devicetree/bindings/net/dsa/lan9303.txt b/Documentation/devicetree/bindings/net/dsa/lan9303.txt
index 464d6bf87605..46a732087f5c 100644
--- a/Documentation/devicetree/bindings/net/dsa/lan9303.txt
+++ b/Documentation/devicetree/bindings/net/dsa/lan9303.txt
@@ -46,7 +46,6 @@ I2C managed mode:
 
 			port@0 { /* RMII fixed link to master */
 				reg = <0>;
-				label = "cpu";
 				ethernet = <&master>;
 			};
 
@@ -83,7 +82,6 @@ MDIO managed mode:
 
 					port@0 {
 						reg = <0>;
-						label = "cpu";
 						ethernet = <&master>;
 					};
 
diff --git a/Documentation/devicetree/bindings/net/dsa/lantiq-gswip.txt b/Documentation/devicetree/bindings/net/dsa/lantiq-gswip.txt
index e3829d3e480e..8bb1eff21cb1 100644
--- a/Documentation/devicetree/bindings/net/dsa/lantiq-gswip.txt
+++ b/Documentation/devicetree/bindings/net/dsa/lantiq-gswip.txt
@@ -96,7 +96,6 @@ switch@e108000 {
 
 		port@6 {
 			reg = <0x6>;
-			label = "cpu";
 			ethernet = <&eth0>;
 		};
 	};
diff --git a/Documentation/devicetree/bindings/net/dsa/mediatek,mt7530.yaml b/Documentation/devicetree/bindings/net/dsa/mediatek,mt7530.yaml
index 17ab6c69ecc7..f2e9ff3f580b 100644
--- a/Documentation/devicetree/bindings/net/dsa/mediatek,mt7530.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/mediatek,mt7530.yaml
@@ -4,67 +4,92 @@
 $id: http://devicetree.org/schemas/net/dsa/mediatek,mt7530.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Mediatek MT7530 Ethernet switch
+title: Mediatek MT7530 and MT7531 Ethernet Switches
 
 maintainers:
-  - Sean Wang <sean.wang@mediatek.com>
+  - Arınç ÜNAL <arinc.unal@arinc9.com>
   - Landen Chao <Landen.Chao@mediatek.com>
   - DENG Qingfang <dqfext@gmail.com>
+  - Sean Wang <sean.wang@mediatek.com>
 
 description: |
-  Port 5 of mt7530 and mt7621 switch is muxed between:
-  1. GMAC5: GMAC5 can interface with another external MAC or PHY.
-  2. PHY of port 0 or port 4: PHY interfaces with an external MAC like 2nd GMAC
-     of the SOC. Used in many setups where port 0/4 becomes the WAN port.
-     Note: On a MT7621 SOC with integrated switch: 2nd GMAC can only connected to
-       GMAC5 when the gpios for RGMII2 (GPIO 22-33) are not used and not
-       connected to external component!
-
-  Port 5 modes/configurations:
-  1. Port 5 is disabled and isolated: An external phy can interface to the 2nd
-     GMAC of the SOC.
-     In the case of a build-in MT7530 switch, port 5 shares the RGMII bus with 2nd
-     GMAC and an optional external phy. Mind the GPIO/pinctl settings of the SOC!
-  2. Port 5 is muxed to PHY of port 0/4: Port 0/4 interfaces with 2nd GMAC.
-     It is a simple MAC to PHY interface, port 5 needs to be setup for xMII mode
-     and RGMII delay.
-  3. Port 5 is muxed to GMAC5 and can interface to an external phy.
-     Port 5 becomes an extra switch port.
-     Only works on platform where external phy TX<->RX lines are swapped.
-     Like in the Ubiquiti ER-X-SFP.
-  4. Port 5 is muxed to GMAC5 and interfaces with the 2nd GAMC as 2nd CPU port.
-     Currently a 2nd CPU port is not supported by DSA code.
-
-  Depending on how the external PHY is wired:
-  1. normal: The PHY can only connect to 2nd GMAC but not to the switch
-  2. swapped: RGMII TX, RX are swapped; external phy interface with the switch as
-     a ethernet port. But can't interface to the 2nd GMAC.
-
-    Based on the DT the port 5 mode is configured.
-
-  Driver tries to lookup the phy-handle of the 2nd GMAC of the master device.
-  When phy-handle matches PHY of port 0 or 4 then port 5 set-up as mode 2.
-  phy-mode must be set, see also example 2 below!
-  * mt7621: phy-mode = "rgmii-txid";
-  * mt7623: phy-mode = "rgmii";
-
-  CPU-Ports need a phy-mode property:
-    Allowed values on mt7530 and mt7621:
-      - "rgmii"
-      - "trgmii"
-    On mt7531:
-      - "1000base-x"
-      - "2500base-x"
-      - "rgmii"
-      - "sgmii"
+  There are two versions of MT7530, standalone and in a multi-chip module.
+
+  MT7530 is a part of the multi-chip module in MT7620AN, MT7620DA, MT7620DAN,
+  MT7620NN, MT7621AT, MT7621DAT, MT7621ST and MT7623AI SoCs.
+
+  MT7530 in MT7620AN, MT7620DA, MT7620DAN and MT7620NN SoCs has got 10/100 PHYs
+  and the switch registers are directly mapped into SoC's memory map rather than
+  using MDIO. The DSA driver currently doesn't support this.
+
+  There is only the standalone version of MT7531.
+
+  Port 5 on MT7530 has got various ways of configuration.
+
+  For standalone MT7530:
+
+    - Port 5 can be used as a CPU port.
+
+    - PHY 0 or 4 of the switch can be muxed to connect to the gmac of the SoC
+      which port 5 is wired to. Usually used for connecting the wan port
+      directly to the CPU to achieve 2 Gbps routing in total.
+
+      The driver looks up the reg on the ethernet-phy node which the phy-handle
+      property refers to on the gmac node to mux the specified phy.
+
+      The driver requires the gmac of the SoC to have "mediatek,eth-mac" as the
+      compatible string and the reg must be 1. So, for now, only gmac1 of an
+      MediaTek SoC can benefit this. Banana Pi BPI-R2 suits this.
+      Check out example 5 for a similar configuration.
+
+    - Port 5 can be wired to an external phy. Port 5 becomes a DSA slave.
+      Check out example 7 for a similar configuration.
+
+  For multi-chip module MT7530:
+
+    - Port 5 can be used as a CPU port.
+
+    - PHY 0 or 4 of the switch can be muxed to connect to gmac1 of the SoC.
+      Usually used for connecting the wan port directly to the CPU to achieve 2
+      Gbps routing in total.
+
+      The driver looks up the reg on the ethernet-phy node which the phy-handle
+      property refers to on the gmac node to mux the specified phy.
+
+      For the MT7621 SoCs, rgmii2 group must be claimed with rgmii2 function.
+      Check out example 5.
+
+    - In case of an external phy wired to gmac1 of the SoC, port 5 must not be
+      enabled.
+
+      In case of muxing PHY 0 or 4, the external phy must not be enabled.
+
+      For the MT7621 SoCs, rgmii2 group must be claimed with rgmii2 function.
+      Check out example 6.
 
+    - Port 5 can be muxed to an external phy. Port 5 becomes a DSA slave.
+      The external phy must be wired TX to TX to gmac1 of the SoC for this to
+      work. Ubiquiti EdgeRouter X SFP is wired this way.
+
+      Muxing PHY 0 or 4 won't work when the external phy is connected TX to TX.
+
+      For the MT7621 SoCs, rgmii2 group must be claimed with gpio function.
+      Check out example 7.
 
 properties:
   compatible:
-    enum:
-      - mediatek,mt7530
-      - mediatek,mt7531
-      - mediatek,mt7621
+    oneOf:
+      - description:
+          Standalone MT7530 and multi-chip module MT7530 in MT7623AI SoC
+        const: mediatek,mt7530
+
+      - description:
+          Standalone MT7531
+        const: mediatek,mt7531
+
+      - description:
+          Multi-chip module MT7530 in MT7621AT, MT7621DAT and MT7621ST SoCs
+        const: mediatek,mt7621
 
   reg:
     maxItems: 1
@@ -79,7 +104,14 @@ properties:
   gpio-controller:
     type: boolean
     description:
-      if defined, MT7530's LED controller will run on GPIO mode.
+      If defined, LED controller of the MT7530 switch will run on GPIO mode.
+
+      There are 15 controllable pins.
+      port 0 LED 0..2 as GPIO 0..2
+      port 1 LED 0..2 as GPIO 3..5
+      port 2 LED 0..2 as GPIO 6..8
+      port 3 LED 0..2 as GPIO 9..11
+      port 4 LED 0..2 as GPIO 12..14
 
   "#interrupt-cells":
     const: 1
@@ -92,17 +124,21 @@ properties:
   io-supply:
     description:
       Phandle to the regulator node necessary for the I/O power.
-      See Documentation/devicetree/bindings/regulator/mt6323-regulator.txt
-      for details for the regulator setup on these boards.
+      See Documentation/devicetree/bindings/regulator/mt6323-regulator.txt for
+      details for the regulator setup on these boards.
 
   mediatek,mcm:
     type: boolean
     description:
-      if defined, indicates that either MT7530 is the part on multi-chip
-      module belong to MT7623A has or the remotely standalone chip as the
-      function MT7623N reference board provided for.
+      Used for MT7621AT, MT7621DAT, MT7621ST and MT7623AI SoCs which the MT7530
+      switch is a part of the multi-chip module.
 
   reset-gpios:
+    description:
+      GPIO to reset the switch. Use this if mediatek,mcm is not used.
+      This property is optional because some boards share the reset line with
+      other components which makes it impossible to probe the switch if the
+      reset line is used.
     maxItems: 1
 
   reset-names:
@@ -110,8 +146,8 @@ properties:
 
   resets:
     description:
-      Phandle pointing to the system reset controller with line index for
-      the ethsys.
+      Phandle pointing to the system reset controller with line index for the
+      ethsys.
     maxItems: 1
 
 patternProperties:
@@ -128,31 +164,88 @@ patternProperties:
         properties:
           reg:
             description:
-              Port address described must be 5 or 6 for CPU port and from 0
-              to 5 for user ports.
+              Port address described must be 5 or 6 for CPU port and from 0 to 5
+              for user ports.
 
         allOf:
           - $ref: dsa-port.yaml#
           - if:
-              properties:
-                label:
-                  items:
-                    - const: cpu
+              required: [ ethernet ]
             then:
-              required:
-                - reg
-                - phy-mode
+              properties:
+                reg:
+                  enum:
+                    - 5
+                    - 6
 
 required:
   - compatible
   - reg
 
+$defs:
+  mt7530-dsa-port:
+    patternProperties:
+      "^(ethernet-)?ports$":
+        patternProperties:
+          "^(ethernet-)?port@[0-9]+$":
+            if:
+              required: [ ethernet ]
+            then:
+              if:
+                properties:
+                  reg:
+                    const: 5
+              then:
+                properties:
+                  phy-mode:
+                    enum:
+                      - gmii
+                      - mii
+                      - rgmii
+              else:
+                properties:
+                  phy-mode:
+                    enum:
+                      - rgmii
+                      - trgmii
+
+  mt7531-dsa-port:
+    patternProperties:
+      "^(ethernet-)?ports$":
+        patternProperties:
+          "^(ethernet-)?port@[0-9]+$":
+            if:
+              required: [ ethernet ]
+            then:
+              if:
+                properties:
+                  reg:
+                    const: 5
+              then:
+                properties:
+                  phy-mode:
+                    enum:
+                      - 1000base-x
+                      - 2500base-x
+                      - rgmii
+                      - sgmii
+              else:
+                properties:
+                  phy-mode:
+                    enum:
+                      - 1000base-x
+                      - 2500base-x
+                      - sgmii
+
 allOf:
-  - $ref: "dsa.yaml#"
+  - $ref: dsa.yaml#
   - if:
       required:
         - mediatek,mcm
     then:
+      properties:
+        reset-gpios: false
+
       required:
         - resets
         - reset-names
@@ -163,52 +256,139 @@ allOf:
   - if:
       properties:
         compatible:
-          items:
-            - const: mediatek,mt7530
+          const: mediatek,mt7530
     then:
+      $ref: "#/$defs/mt7530-dsa-port"
       required:
         - core-supply
         - io-supply
 
+  - if:
+      properties:
+        compatible:
+          const: mediatek,mt7531
+    then:
+      $ref: "#/$defs/mt7531-dsa-port"
+      properties:
+        gpio-controller: false
+        mediatek,mcm: false
+
+  - if:
+      properties:
+        compatible:
+          const: mediatek,mt7621
+    then:
+      $ref: "#/$defs/mt7530-dsa-port"
+      required:
+        - mediatek,mcm
+
 unevaluatedProperties: false
 
 examples:
+  # Example 1: Standalone MT7530
   - |
     #include <dt-bindings/gpio/gpio.h>
+
     mdio {
         #address-cells = <1>;
         #size-cells = <0>;
-        switch@0 {
+
+        switch@1f {
             compatible = "mediatek,mt7530";
-            reg = <0>;
+            reg = <0x1f>;
+
+            reset-gpios = <&pio 33 0>;
 
             core-supply = <&mt6323_vpa_reg>;
             io-supply = <&mt6323_vemc3v3_reg>;
-            reset-gpios = <&pio 33 GPIO_ACTIVE_HIGH>;
 
             ethernet-ports {
                 #address-cells = <1>;
                 #size-cells = <0>;
+
                 port@0 {
                     reg = <0>;
-                    label = "lan0";
+                    label = "lan1";
                 };
 
                 port@1 {
                     reg = <1>;
-                    label = "lan1";
+                    label = "lan2";
                 };
 
                 port@2 {
                     reg = <2>;
-                    label = "lan2";
+                    label = "lan3";
                 };
 
                 port@3 {
                     reg = <3>;
+                    label = "lan4";
+                };
+
+                port@4 {
+                    reg = <4>;
+                    label = "wan";
+                };
+
+                port@6 {
+                    reg = <6>;
+                    ethernet = <&gmac0>;
+                    phy-mode = "rgmii";
+
+                    fixed-link {
+                        speed = <1000>;
+                        full-duplex;
+                        pause;
+                    };
+                };
+            };
+        };
+    };
+
+  # Example 2: MT7530 in MT7623AI SoC
+  - |
+    #include <dt-bindings/reset/mt2701-resets.h>
+
+    mdio {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        switch@1f {
+            compatible = "mediatek,mt7530";
+            reg = <0x1f>;
+
+            mediatek,mcm;
+            resets = <&ethsys MT2701_ETHSYS_MCM_RST>;
+            reset-names = "mcm";
+
+            core-supply = <&mt6323_vpa_reg>;
+            io-supply = <&mt6323_vemc3v3_reg>;
+
+            ethernet-ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+                    label = "lan1";
+                };
+
+                port@1 {
+                    reg = <1>;
+                    label = "lan2";
+                };
+
+                port@2 {
+                    reg = <2>;
                     label = "lan3";
                 };
 
+                port@3 {
+                    reg = <3>;
+                    label = "lan4";
+                };
+
                 port@4 {
                     reg = <4>;
                     label = "wan";
@@ -216,96 +396,226 @@ examples:
 
                 port@6 {
                     reg = <6>;
-                    label = "cpu";
                     ethernet = <&gmac0>;
                     phy-mode = "trgmii";
+
                     fixed-link {
                         speed = <1000>;
                         full-duplex;
+                        pause;
                     };
                 };
             };
         };
     };
 
+  # Example 3: Standalone MT7531
   - |
-    //Example 2: MT7621: Port 4 is WAN port: 2nd GMAC -> Port 5 -> PHY port 4.
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
 
-    ethernet {
+    mdio {
         #address-cells = <1>;
         #size-cells = <0>;
-        gmac0: mac@0 {
-            compatible = "mediatek,eth-mac";
+
+        switch@0 {
+            compatible = "mediatek,mt7531";
             reg = <0>;
-            phy-mode = "rgmii";
 
-            fixed-link {
-                speed = <1000>;
-                full-duplex;
-                pause;
+            reset-gpios = <&pio 54 0>;
+
+            interrupt-controller;
+            #interrupt-cells = <1>;
+            interrupt-parent = <&pio>;
+            interrupts = <53 IRQ_TYPE_LEVEL_HIGH>;
+
+            ethernet-ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+                    label = "lan1";
+                };
+
+                port@1 {
+                    reg = <1>;
+                    label = "lan2";
+                };
+
+                port@2 {
+                    reg = <2>;
+                    label = "lan3";
+                };
+
+                port@3 {
+                    reg = <3>;
+                    label = "lan4";
+                };
+
+                port@4 {
+                    reg = <4>;
+                    label = "wan";
+                };
+
+                port@6 {
+                    reg = <6>;
+                    ethernet = <&gmac0>;
+                    phy-mode = "2500base-x";
+
+                    fixed-link {
+                        speed = <2500>;
+                        full-duplex;
+                        pause;
+                    };
+                };
+            };
+        };
+    };
+
+  # Example 4: MT7530 in MT7621AT, MT7621DAT and MT7621ST SoCs
+  - |
+    #include <dt-bindings/interrupt-controller/mips-gic.h>
+    #include <dt-bindings/reset/mt7621-reset.h>
+
+    mdio {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        switch@1f {
+            compatible = "mediatek,mt7621";
+            reg = <0x1f>;
+
+            mediatek,mcm;
+            resets = <&sysc MT7621_RST_MCM>;
+            reset-names = "mcm";
+
+            interrupt-controller;
+            #interrupt-cells = <1>;
+            interrupt-parent = <&gic>;
+            interrupts = <GIC_SHARED 23 IRQ_TYPE_LEVEL_HIGH>;
+
+            ethernet-ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+                    label = "lan1";
+                };
+
+                port@1 {
+                    reg = <1>;
+                    label = "lan2";
+                };
+
+                port@2 {
+                    reg = <2>;
+                    label = "lan3";
+                };
+
+                port@3 {
+                    reg = <3>;
+                    label = "lan4";
+                };
+
+                port@4 {
+                    reg = <4>;
+                    label = "wan";
+                };
+
+                port@6 {
+                    reg = <6>;
+                    ethernet = <&gmac0>;
+                    phy-mode = "trgmii";
+
+                    fixed-link {
+                        speed = <1000>;
+                        full-duplex;
+                        pause;
+                    };
+                };
             };
         };
+    };
 
-        gmac1: mac@1 {
+  # Example 5: MT7621: mux MT7530's phy4 to SoC's gmac1
+  - |
+    #include <dt-bindings/interrupt-controller/mips-gic.h>
+    #include <dt-bindings/reset/mt7621-reset.h>
+
+    ethernet {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        pinctrl-names = "default";
+        pinctrl-0 = <&rgmii2_pins>;
+
+        mac@1 {
             compatible = "mediatek,eth-mac";
             reg = <1>;
-            phy-mode = "rgmii-txid";
-            phy-handle = <&phy4>;
+
+            phy-mode = "rgmii";
+            phy-handle = <&example5_ethphy4>;
         };
 
-        mdio: mdio-bus {
+        mdio {
             #address-cells = <1>;
             #size-cells = <0>;
 
-            /* Internal phy */
-            phy4: ethernet-phy@4 {
+            /* MT7530's phy4 */
+            example5_ethphy4: ethernet-phy@4 {
                 reg = <4>;
             };
 
-            mt7530: switch@1f {
+            switch@1f {
                 compatible = "mediatek,mt7621";
                 reg = <0x1f>;
-                mediatek,mcm;
 
-                resets = <&rstctrl 2>;
+                mediatek,mcm;
+                resets = <&sysc MT7621_RST_MCM>;
                 reset-names = "mcm";
 
+                interrupt-controller;
+                #interrupt-cells = <1>;
+                interrupt-parent = <&gic>;
+                interrupts = <GIC_SHARED 23 IRQ_TYPE_LEVEL_HIGH>;
+
                 ethernet-ports {
                     #address-cells = <1>;
                     #size-cells = <0>;
 
                     port@0 {
                         reg = <0>;
-                        label = "lan0";
+                        label = "lan1";
                     };
 
                     port@1 {
                         reg = <1>;
-                        label = "lan1";
+                        label = "lan2";
                     };
 
                     port@2 {
                         reg = <2>;
-                        label = "lan2";
+                        label = "lan3";
                     };
 
                     port@3 {
                         reg = <3>;
-                        label = "lan3";
+                        label = "lan4";
                     };
 
-                    /* Commented out. Port 4 is handled by 2nd GMAC.
+                    /* Commented out, phy4 is muxed to gmac1.
                     port@4 {
                         reg = <4>;
-                        label = "lan4";
+                        label = "wan";
                     };
                     */
 
                     port@6 {
                         reg = <6>;
-                        label = "cpu";
                         ethernet = <&gmac0>;
-                        phy-mode = "rgmii";
+                        phy-mode = "trgmii";
 
                         fixed-link {
                             speed = <1000>;
@@ -318,82 +628,169 @@ examples:
         };
     };
 
+  # Example 6: MT7621: mux external phy to SoC's gmac1
   - |
-    //Example 3: MT7621: Port 5 is connected to external PHY: Port 5 -> external PHY.
+    #include <dt-bindings/interrupt-controller/mips-gic.h>
+    #include <dt-bindings/reset/mt7621-reset.h>
 
     ethernet {
         #address-cells = <1>;
         #size-cells = <0>;
-        gmac_0: mac@0 {
+
+        pinctrl-names = "default";
+        pinctrl-0 = <&rgmii2_pins>;
+
+        mac@1 {
             compatible = "mediatek,eth-mac";
-            reg = <0>;
+            reg = <1>;
+
             phy-mode = "rgmii";
+            phy-handle = <&example6_ethphy7>;
+        };
+
+        mdio {
+            #address-cells = <1>;
+            #size-cells = <0>;
 
-            fixed-link {
-                speed = <1000>;
-                full-duplex;
-                pause;
+            /* External PHY */
+            example6_ethphy7: ethernet-phy@7 {
+                reg = <7>;
+                phy-mode = "rgmii";
+            };
+
+            switch@1f {
+                compatible = "mediatek,mt7621";
+                reg = <0x1f>;
+
+                mediatek,mcm;
+                resets = <&sysc MT7621_RST_MCM>;
+                reset-names = "mcm";
+
+                interrupt-controller;
+                #interrupt-cells = <1>;
+                interrupt-parent = <&gic>;
+                interrupts = <GIC_SHARED 23 IRQ_TYPE_LEVEL_HIGH>;
+
+                ethernet-ports {
+                    #address-cells = <1>;
+                    #size-cells = <0>;
+
+                    port@0 {
+                        reg = <0>;
+                        label = "lan1";
+                    };
+
+                    port@1 {
+                        reg = <1>;
+                        label = "lan2";
+                    };
+
+                    port@2 {
+                        reg = <2>;
+                        label = "lan3";
+                    };
+
+                    port@3 {
+                        reg = <3>;
+                        label = "lan4";
+                    };
+
+                    port@4 {
+                        reg = <4>;
+                        label = "wan";
+                    };
+
+                    port@6 {
+                        reg = <6>;
+                        ethernet = <&gmac0>;
+                        phy-mode = "trgmii";
+
+                        fixed-link {
+                            speed = <1000>;
+                            full-duplex;
+                            pause;
+                        };
+                    };
+                };
             };
         };
+    };
+
+  # Example 7: MT7621: mux external phy to MT7530's port 5
+  - |
+    #include <dt-bindings/interrupt-controller/mips-gic.h>
+    #include <dt-bindings/reset/mt7621-reset.h>
+
+    ethernet {
+        #address-cells = <1>;
+        #size-cells = <0>;
 
-        mdio0: mdio-bus {
+        pinctrl-names = "default";
+        pinctrl-0 = <&rgmii2_pins>;
+
+        mdio {
             #address-cells = <1>;
             #size-cells = <0>;
 
-            /* External phy */
-            ephy5: ethernet-phy@7 {
+            /* External PHY */
+            example7_ethphy7: ethernet-phy@7 {
                 reg = <7>;
+                phy-mode = "rgmii";
             };
 
             switch@1f {
                 compatible = "mediatek,mt7621";
                 reg = <0x1f>;
-                mediatek,mcm;
 
-                resets = <&rstctrl 2>;
+                mediatek,mcm;
+                resets = <&sysc MT7621_RST_MCM>;
                 reset-names = "mcm";
 
+                interrupt-controller;
+                #interrupt-cells = <1>;
+                interrupt-parent = <&gic>;
+                interrupts = <GIC_SHARED 23 IRQ_TYPE_LEVEL_HIGH>;
+
                 ethernet-ports {
                     #address-cells = <1>;
                     #size-cells = <0>;
 
                     port@0 {
                         reg = <0>;
-                        label = "lan0";
+                        label = "lan1";
                     };
 
                     port@1 {
                         reg = <1>;
-                        label = "lan1";
+                        label = "lan2";
                     };
 
                     port@2 {
                         reg = <2>;
-                        label = "lan2";
+                        label = "lan3";
                     };
 
                     port@3 {
                         reg = <3>;
-                        label = "lan3";
+                        label = "lan4";
                     };
 
                     port@4 {
                         reg = <4>;
-                        label = "lan4";
+                        label = "wan";
                     };
 
                     port@5 {
                         reg = <5>;
-                        label = "lan5";
-                        phy-mode = "rgmii";
-                        phy-handle = <&ephy5>;
+                        label = "extphy";
+                        phy-mode = "rgmii-txid";
+                        phy-handle = <&example7_ethphy7>;
                     };
 
-                    cpu_port0: port@6 {
+                    port@6 {
                         reg = <6>;
-                        label = "cpu";
-                        ethernet = <&gmac_0>;
-                        phy-mode = "rgmii";
+                        ethernet = <&gmac0>;
+                        phy-mode = "trgmii";
 
                         fixed-link {
                             speed = <1000>;
diff --git a/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml b/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml
index 6bbd8145b6c1..4da75b1f9533 100644
--- a/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/microchip,ksz.yaml
@@ -107,8 +107,9 @@ examples:
                 };
                 port@5 {
                     reg = <5>;
-                    label = "cpu";
                     ethernet = <&eth0>;
+                    phy-mode = "rgmii";
+
                     fixed-link {
                         speed = <1000>;
                         full-duplex;
@@ -144,8 +145,9 @@ examples:
                 };
                 port@6 {
                     reg = <6>;
-                    label = "cpu";
                     ethernet = <&eth0>;
+                    phy-mode = "rgmii";
+
                     fixed-link {
                         speed = <1000>;
                         full-duplex;
diff --git a/Documentation/devicetree/bindings/net/dsa/mscc,ocelot.yaml b/Documentation/devicetree/bindings/net/dsa/mscc,ocelot.yaml
new file mode 100644
index 000000000000..8d93ed9c172c
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/dsa/mscc,ocelot.yaml
@@ -0,0 +1,260 @@
+# SPDX-License-Identifier: (GPL-2.0 OR MIT)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/dsa/mscc,ocelot.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Microchip Ocelot Switch Family Device Tree Bindings
+
+maintainers:
+  - Vladimir Oltean <vladimir.oltean@nxp.com>
+  - Claudiu Manoil <claudiu.manoil@nxp.com>
+  - Alexandre Belloni <alexandre.belloni@bootlin.com>
+  - UNGLinuxDriver@microchip.com
+
+description: |
+  There are multiple switches which are either part of the Ocelot-1 family, or
+  derivatives of this architecture. These switches can be found embedded in
+  various SoCs and accessed using MMIO, or as discrete chips and accessed over
+  SPI or PCIe. The present DSA binding shall be used when the host controlling
+  them performs packet I/O primarily through an Ethernet port of the switch
+  (which is attached to an Ethernet port of the host), rather than through
+  Frame DMA or register-based I/O.
+
+  VSC9953 (Seville):
+
+    This is found in the NXP T1040, where it is a memory-mapped platform
+    device.
+
+    The following PHY interface types are supported:
+
+      - phy-mode = "internal": on ports 8 and 9
+      - phy-mode = "sgmii": on ports 0, 1, 2, 3, 4, 5, 6, 7
+      - phy-mode = "qsgmii": on ports 0, 1, 2, 3, 4, 5, 6, 7
+      - phy-mode = "1000base-x": on ports 0, 1, 2, 3, 4, 5, 6, 7
+
+  VSC9959 (Felix):
+
+    This is found in the NXP LS1028A. It is a PCI device, part of the larger
+    enetc root complex. As a result, the ethernet-switch node is a sub-node of
+    the PCIe root complex node and its "reg" property conforms to the parent
+    node bindings, describing it as PF 5 of device 0, bus 0.
+
+    If any external switch port is enabled, the enetc PF2 (enetc_port2) should
+    be enabled as well. This is because the internal MDIO bus (exposed through
+    EA BAR 0) used to access the MAC PCS registers truly belongs to the enetc
+    port 2 and not to Felix.
+
+    The following PHY interface types are supported:
+
+      - phy-mode = "internal": on ports 4 and 5
+      - phy-mode = "sgmii": on ports 0, 1, 2, 3
+      - phy-mode = "qsgmii": on ports 0, 1, 2, 3
+      - phy-mode = "usxgmii": on ports 0, 1, 2, 3
+      - phy-mode = "1000base-x": on ports 0, 1, 2, 3
+      - phy-mode = "2500base-x": on ports 0, 1, 2, 3
+
+properties:
+  compatible:
+    enum:
+      - mscc,vsc9953-switch
+      - pci1957,eef0
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+    description:
+      Used to signal availability of PTP TX timestamps, and state changes of
+      the MAC merge layer of ports that support Frame Preemption.
+
+  little-endian: true
+  big-endian: true
+
+required:
+  - compatible
+  - reg
+
+allOf:
+  - $ref: dsa.yaml#
+  - if:
+      properties:
+        compatible:
+          const: pci1957,eef0
+    then:
+      required:
+        - interrupts
+
+unevaluatedProperties: false
+
+examples:
+  # Felix VSC9959 (NXP LS1028A)
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    pcie { /* Integrated Endpoint Root Complex */
+        #address-cells = <3>;
+        #size-cells = <2>;
+
+        ethernet-switch@0,5 {
+            compatible = "pci1957,eef0";
+            reg = <0x000500 0 0 0 0>;
+            interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
+
+            ethernet-ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy0>;
+                    managed = "in-band-status";
+                };
+
+                port@1 {
+                    reg = <1>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy1>;
+                    managed = "in-band-status";
+                };
+
+                port@2 {
+                    reg = <2>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy2>;
+                    managed = "in-band-status";
+                };
+
+                port@3 {
+                    reg = <3>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy3>;
+                    managed = "in-band-status";
+                };
+
+                port@4 {
+                    reg = <4>;
+                    ethernet = <&enetc_port2>;
+                    phy-mode = "internal";
+
+                    fixed-link {
+                        speed = <2500>;
+                        full-duplex;
+                        pause;
+                    };
+                };
+
+                port@5 {
+                    reg = <5>;
+                    ethernet = <&enetc_port3>;
+                    phy-mode = "internal";
+
+                    fixed-link {
+                        speed = <1000>;
+                        full-duplex;
+                        pause;
+                    };
+                };
+            };
+        };
+    };
+  # Seville VSC9953 (NXP T1040)
+  - |
+    soc {
+        #address-cells = <1>;
+        #size-cells = <1>;
+
+        ethernet-switch@800000 {
+            compatible = "mscc,vsc9953-switch";
+            reg = <0x800000 0x290000>;
+            little-endian;
+
+            ethernet-ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy0>;
+                    managed = "in-band-status";
+                };
+
+                port@1 {
+                    reg = <1>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy1>;
+                    managed = "in-band-status";
+                };
+
+                port@2 {
+                    reg = <2>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy2>;
+                    managed = "in-band-status";
+                };
+
+                port@3 {
+                    reg = <3>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy3>;
+                    managed = "in-band-status";
+                };
+
+                port@4 {
+                    reg = <4>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy4>;
+                    managed = "in-band-status";
+                };
+
+                port@5 {
+                    reg = <5>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy5>;
+                    managed = "in-band-status";
+                };
+
+                port@6 {
+                    reg = <6>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy6>;
+                    managed = "in-band-status";
+                };
+
+                port@7 {
+                    reg = <7>;
+                    phy-mode = "qsgmii";
+                    phy-handle = <&phy7>;
+                    managed = "in-band-status";
+                };
+
+                port@8 {
+                    reg = <8>;
+                    phy-mode = "internal";
+                    ethernet = <&enet0>;
+
+                    fixed-link {
+                        speed = <2500>;
+                        full-duplex;
+                        pause;
+                    };
+                };
+
+                port@9 {
+                    reg = <9>;
+                    phy-mode = "internal";
+                    ethernet = <&enet1>;
+
+                    fixed-link {
+                        speed = <2500>;
+                        full-duplex;
+                        pause;
+                    };
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/net/dsa/ocelot.txt b/Documentation/devicetree/bindings/net/dsa/ocelot.txt
deleted file mode 100644
index 7a271d070b72..000000000000
--- a/Documentation/devicetree/bindings/net/dsa/ocelot.txt
+++ /dev/null
@@ -1,213 +0,0 @@
-Microchip Ocelot switch driver family
-=====================================
-
-Felix
------
-
-Currently the switches supported by the felix driver are:
-
-- VSC9959 (Felix)
-- VSC9953 (Seville)
-
-The VSC9959 switch is found in the NXP LS1028A. It is a PCI device, part of the
-larger ENETC root complex. As a result, the ethernet-switch node is a sub-node
-of the PCIe root complex node and its "reg" property conforms to the parent
-node bindings:
-
-* reg: Specifies PCIe Device Number and Function Number of the endpoint device,
-  in this case for the Ethernet L2Switch it is PF5 (of device 0, bus 0).
-
-It does not require a "compatible" string.
-
-The interrupt line is used to signal availability of PTP TX timestamps and for
-TSN frame preemption.
-
-For the external switch ports, depending on board configuration, "phy-mode" and
-"phy-handle" are populated by board specific device tree instances. Ports 4 and
-5 are fixed as internal ports in the NXP LS1028A instantiation.
-
-The CPU port property ("ethernet") configures the feature called "NPI port" in
-the Ocelot hardware core. The CPU port in Ocelot is a set of queues, which are
-connected, in the Node Processor Interface (NPI) mode, to an Ethernet port.
-By default, in fsl-ls1028a.dtsi, the NPI port is assigned to the internal
-2.5Gbps port@4, but can be moved to the 1Gbps port@5, depending on the specific
-use case.  Moving the NPI port to an external switch port is hardware possible,
-but there is no platform support for the Linux system on the LS1028A chip to
-operate as an entire slave DSA chip.  NPI functionality (and therefore DSA
-tagging) is supported on a single port at a time.
-
-Any port can be disabled (and in fsl-ls1028a.dtsi, they are indeed all disabled
-by default, and should be enabled on a per-board basis). But if any external
-switch port is enabled at all, the ENETC PF2 (enetc_port2) should be enabled as
-well, regardless of whether it is configured as the DSA master or not. This is
-because the Felix PHYLINK implementation accesses the MAC PCS registers, which
-in hardware truly belong to the ENETC port #2 and not to Felix.
-
-Supported PHY interface types (appropriate SerDes protocol setting changes are
-needed in the RCW binary):
-
-* phy_mode = "internal": on ports 4 and 5
-* phy_mode = "sgmii": on ports 0, 1, 2, 3
-* phy_mode = "qsgmii": on ports 0, 1, 2, 3
-* phy_mode = "usxgmii": on ports 0, 1, 2, 3
-* phy_mode = "2500base-x": on ports 0, 1, 2, 3
-
-For the rest of the device tree binding definitions, which are standard DSA and
-PCI, refer to the following documents:
-
-Documentation/devicetree/bindings/net/dsa/dsa.txt
-Documentation/devicetree/bindings/pci/pci.txt
-
-Example:
-
-&soc {
-	pcie@1f0000000 { /* Integrated Endpoint Root Complex */
-		ethernet-switch@0,5 {
-			reg = <0x000500 0 0 0 0>;
-			/* IEP INT_B */
-			interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
-
-			ports {
-				#address-cells = <1>;
-				#size-cells = <0>;
-
-				/* External ports */
-				port@0 {
-					reg = <0>;
-					label = "swp0";
-				};
-
-				port@1 {
-					reg = <1>;
-					label = "swp1";
-				};
-
-				port@2 {
-					reg = <2>;
-					label = "swp2";
-				};
-
-				port@3 {
-					reg = <3>;
-					label = "swp3";
-				};
-
-				/* Tagging CPU port */
-				port@4 {
-					reg = <4>;
-					ethernet = <&enetc_port2>;
-					phy-mode = "internal";
-
-					fixed-link {
-						speed = <2500>;
-						full-duplex;
-					};
-				};
-
-				/* Non-tagging CPU port */
-				port@5 {
-					reg = <5>;
-					phy-mode = "internal";
-					status = "disabled";
-
-					fixed-link {
-						speed = <1000>;
-						full-duplex;
-					};
-				};
-			};
-		};
-	};
-};
-
-The VSC9953 switch is found inside NXP T1040. It is a platform device with the
-following required properties:
-
-- compatible:
-	Must be "mscc,vsc9953-switch".
-
-Supported PHY interface types (appropriate SerDes protocol setting changes are
-needed in the RCW binary):
-
-* phy_mode = "internal": on ports 8 and 9
-* phy_mode = "sgmii": on ports 0, 1, 2, 3, 4, 5, 6, 7
-* phy_mode = "qsgmii": on ports 0, 1, 2, 3, 4, 5, 6, 7
-
-Example:
-
-&soc {
-	ethernet-switch@800000 {
-		#address-cells = <0x1>;
-		#size-cells = <0x0>;
-		compatible = "mscc,vsc9953-switch";
-		little-endian;
-		reg = <0x800000 0x290000>;
-
-		ports {
-			#address-cells = <0x1>;
-			#size-cells = <0x0>;
-
-			port@0 {
-				reg = <0x0>;
-				label = "swp0";
-			};
-
-			port@1 {
-				reg = <0x1>;
-				label = "swp1";
-			};
-
-			port@2 {
-				reg = <0x2>;
-				label = "swp2";
-			};
-
-			port@3 {
-				reg = <0x3>;
-				label = "swp3";
-			};
-
-			port@4 {
-				reg = <0x4>;
-				label = "swp4";
-			};
-
-			port@5 {
-				reg = <0x5>;
-				label = "swp5";
-			};
-
-			port@6 {
-				reg = <0x6>;
-				label = "swp6";
-			};
-
-			port@7 {
-				reg = <0x7>;
-				label = "swp7";
-			};
-
-			port@8 {
-				reg = <0x8>;
-				phy-mode = "internal";
-				ethernet = <&enet0>;
-
-				fixed-link {
-					speed = <2500>;
-					full-duplex;
-				};
-			};
-
-			port@9 {
-				reg = <0x9>;
-				phy-mode = "internal";
-				status = "disabled";
-
-				fixed-link {
-					speed = <2500>;
-					full-duplex;
-				};
-			};
-		};
-	};
-};
diff --git a/Documentation/devicetree/bindings/net/dsa/qca8k.yaml b/Documentation/devicetree/bindings/net/dsa/qca8k.yaml
index f3c88371d76c..978162df51f7 100644
--- a/Documentation/devicetree/bindings/net/dsa/qca8k.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/qca8k.yaml
@@ -159,7 +159,6 @@ examples:
 
                 port@0 {
                     reg = <0>;
-                    label = "cpu";
                     ethernet = <&gmac1>;
                     phy-mode = "rgmii";
 
@@ -221,7 +220,6 @@ examples:
 
                 port@0 {
                     reg = <0>;
-                    label = "cpu";
                     ethernet = <&gmac1>;
                     phy-mode = "rgmii";
 
@@ -268,7 +266,6 @@ examples:
 
                 port@6 {
                     reg = <0>;
-                    label = "cpu";
                     ethernet = <&gmac1>;
                     phy-mode = "sgmii";
 
diff --git a/Documentation/devicetree/bindings/net/dsa/realtek.yaml b/Documentation/devicetree/bindings/net/dsa/realtek.yaml
index 4f99aff029dc..1a7d45a8ad66 100644
--- a/Documentation/devicetree/bindings/net/dsa/realtek.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/realtek.yaml
@@ -189,7 +189,6 @@ examples:
                             };
                             port@5 {
                                     reg = <5>;
-                                    label = "cpu";
                                     ethernet = <&gmac0>;
                                     phy-mode = "rgmii";
                                     fixed-link {
@@ -277,7 +276,6 @@ examples:
                             };
                             port@6 {
                                     reg = <6>;
-                                    label = "cpu";
                                     ethernet = <&fec1>;
                                     phy-mode = "rgmii";
                                     tx-internal-delay-ps = <2000>;
diff --git a/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml b/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml
index 4d428f5ad044..7ca9c19a157c 100644
--- a/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml
+++ b/Documentation/devicetree/bindings/net/dsa/renesas,rzn1-a5psw.yaml
@@ -130,7 +130,8 @@ examples:
             port@4 {
                 reg = <4>;
                 ethernet = <&gmac2>;
-                label = "cpu";
+                phy-mode = "internal";
+
                 fixed-link {
                   speed = <1000>;
                   full-duplex;
diff --git a/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.txt b/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.txt
index bbf4a13f6d75..258bef483673 100644
--- a/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.txt
+++ b/Documentation/devicetree/bindings/net/dsa/vitesse,vsc73xx.txt
@@ -75,7 +75,6 @@ switch@0 {
 		};
 		vsc: port@6 {
 			reg = <6>;
-			label = "cpu";
 			ethernet = <&gmac1>;
 			phy-mode = "rgmii";
 			fixed-link {
@@ -117,7 +116,6 @@ switch@2,0 {
 		};
 		vsc: port@6 {
 			reg = <6>;
-			label = "cpu";
 			ethernet = <&enet0>;
 			phy-mode = "rgmii";
 			fixed-link {
diff --git a/Documentation/devicetree/bindings/net/engleder,tsnep.yaml b/Documentation/devicetree/bindings/net/engleder,tsnep.yaml
index d0e1476e15b5..5bd964a46a9d 100644
--- a/Documentation/devicetree/bindings/net/engleder,tsnep.yaml
+++ b/Documentation/devicetree/bindings/net/engleder,tsnep.yaml
@@ -20,7 +20,26 @@ properties:
     maxItems: 1
 
   interrupts:
-    maxItems: 1
+    minItems: 1
+    maxItems: 8
+
+  interrupt-names:
+    minItems: 1
+    items:
+      - const: mac
+      - const: txrx-1
+      - const: txrx-2
+      - const: txrx-3
+      - const: txrx-4
+      - const: txrx-5
+      - const: txrx-6
+      - const: txrx-7
+    description:
+      The main interrupt for basic MAC features and the first TX/RX queue pair
+      is named "mac". "txrx-[1-7]" are the interrupts for additional TX/RX
+      queue pairs.
+
+  dma-coherent: true
 
   local-mac-address: true
 
@@ -58,7 +77,7 @@ examples:
     axi {
         #address-cells = <2>;
         #size-cells = <2>;
-        tnsep0: ethernet@a0000000 {
+        tsnep0: ethernet@a0000000 {
             compatible = "engleder,tsnep";
             reg = <0x0 0xa0000000 0x0 0x10000>;
             interrupts = <0 89 1>;
@@ -76,4 +95,24 @@ examples:
                 };
             };
         };
+
+        tsnep1: ethernet@a0010000 {
+            compatible = "engleder,tsnep";
+            reg = <0x0 0xa0010000 0x0 0x10000>;
+            interrupts = <0 93 1>, <0 94 1>, <0 95 1>, <0 96 1>;
+            interrupt-names = "mac", "txrx-1", "txrx-2", "txrx-3";
+            interrupt-parent = <&gic>;
+            local-mac-address = [00 00 00 00 00 00];
+            phy-mode = "rgmii";
+            phy-handle = <&phy1>;
+            mdio {
+                #address-cells = <1>;
+                #size-cells = <0>;
+                suppress-preamble;
+                phy1: ethernet-phy@1 {
+                    reg = <1>;
+                    rxc-skew-ps = <1080>;
+                };
+            };
+        };
     };
diff --git a/Documentation/devicetree/bindings/net/ethernet-controller.yaml b/Documentation/devicetree/bindings/net/ethernet-controller.yaml
index c138a1022879..4b3c590fcebf 100644
--- a/Documentation/devicetree/bindings/net/ethernet-controller.yaml
+++ b/Documentation/devicetree/bindings/net/ethernet-controller.yaml
@@ -67,6 +67,7 @@ properties:
       - gmii
       - sgmii
       - qsgmii
+      - qusgmii
       - tbi
       - rev-mii
       - rmii
diff --git a/Documentation/devicetree/bindings/net/ethernet-phy.yaml b/Documentation/devicetree/bindings/net/ethernet-phy.yaml
index ed1415a4381f..ad808e9ce5b9 100644
--- a/Documentation/devicetree/bindings/net/ethernet-phy.yaml
+++ b/Documentation/devicetree/bindings/net/ethernet-phy.yaml
@@ -144,6 +144,12 @@ properties:
       Mark the corresponding energy efficient ethernet mode as
       broken and request the ethernet to stop advertising it.
 
+  pses:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    maxItems: 1
+    description:
+      Specifies a reference to a node representing a Power Sourcing Equipment.
+
   phy-is-integrated:
     $ref: /schemas/types.yaml#/definitions/flag
     description:
diff --git a/Documentation/devicetree/bindings/net/fsl,fec.yaml b/Documentation/devicetree/bindings/net/fsl,fec.yaml
index 5cfb661be124..e0f376f7e274 100644
--- a/Documentation/devicetree/bindings/net/fsl,fec.yaml
+++ b/Documentation/devicetree/bindings/net/fsl,fec.yaml
@@ -21,6 +21,7 @@ properties:
           - fsl,imx28-fec
           - fsl,imx6q-fec
           - fsl,mvf600-fec
+          - fsl,s32v234-fec
       - items:
           - enum:
               - fsl,imx53-fec
diff --git a/Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml b/Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml
new file mode 100644
index 000000000000..3a35ac1c260d
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml
@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/fsl,fman-dtsec.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NXP FMan MAC
+
+maintainers:
+  - Madalin Bucur <madalin.bucur@nxp.com>
+
+description: |
+  Each FMan has several MACs, each implementing an Ethernet interface. Earlier
+  versions of FMan used the Datapath Three Speed Ethernet Controller (dTSEC) for
+  10/100/1000 MBit/s speeds, and the 10-Gigabit Ethernet Media Access Controller
+  (10GEC) for 10 Gbit/s speeds. Later versions of FMan use the Multirate
+  Ethernet Media Access Controller (mEMAC) to handle all speeds.
+
+properties:
+  compatible:
+    enum:
+      - fsl,fman-dtsec
+      - fsl,fman-xgec
+      - fsl,fman-memac
+
+  cell-index:
+    maximum: 64
+    description: |
+      FManV2:
+      register[bit]           MAC             cell-index
+      ============================================================
+      FM_EPI[16]              XGEC            8
+      FM_EPI[16+n]            dTSECn          n-1
+      FM_NPI[11+n]            dTSECn          n-1
+              n = 1,..,5
+
+      FManV3:
+      register[bit]           MAC             cell-index
+      ============================================================
+      FM_EPI[16+n]            mEMACn          n-1
+      FM_EPI[25]              mEMAC10         9
+
+      FM_NPI[11+n]            mEMACn          n-1
+      FM_NPI[10]              mEMAC10         9
+      FM_NPI[11]              mEMAC9          8
+              n = 1,..8
+
+      FM_EPI and FM_NPI are located in the FMan memory map.
+
+      2. SoC registers:
+
+      - P2041, P3041, P4080 P5020, P5040:
+      register[bit]           FMan            MAC             cell
+                              Unit                            index
+      ============================================================
+      DCFG_DEVDISR2[7]        1               XGEC            8
+      DCFG_DEVDISR2[7+n]      1               dTSECn          n-1
+      DCFG_DEVDISR2[15]       2               XGEC            8
+      DCFG_DEVDISR2[15+n]     2               dTSECn          n-1
+              n = 1,..5
+
+      - T1040, T2080, T4240, B4860:
+      register[bit]                   FMan    MAC             cell
+                                      Unit                    index
+      ============================================================
+      DCFG_CCSR_DEVDISR2[n-1]         1       mEMACn          n-1
+      DCFG_CCSR_DEVDISR2[11+n]        2       mEMACn          n-1
+              n = 1,..6,9,10
+
+      EVDISR, DCFG_DEVDISR2 and DCFG_CCSR_DEVDISR2 are located in
+      the specific SoC "Device Configuration/Pin Control" Memory
+      Map.
+
+  reg:
+    maxItems: 1
+
+  fsl,fman-ports:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    maxItems: 2
+    description: |
+      An array of two references: the first is the FMan RX port and the second
+      is the TX port used by this MAC.
+
+  ptp-timer:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: A reference to the IEEE1588 timer
+
+  pcsphy-handle:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: A reference to the PCS (typically found on the SerDes)
+
+  tbi-handle:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: A reference to the (TBI-based) PCS
+
+required:
+  - compatible
+  - cell-index
+  - reg
+  - fsl,fman-ports
+  - ptp-timer
+
+allOf:
+  - $ref: ethernet-controller.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: fsl,fman-dtsec
+    then:
+      required:
+        - tbi-handle
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: fsl,fman-memac
+    then:
+      required:
+        - pcsphy-handle
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    ethernet@e0000 {
+            compatible = "fsl,fman-dtsec";
+            cell-index = <0>;
+            reg = <0xe0000 0x1000>;
+            fsl,fman-ports = <&fman1_rx8 &fman1_tx28>;
+            ptp-timer = <&ptp_timer>;
+            tbi-handle = <&tbi0>;
+    };
+  - |
+    ethernet@e8000 {
+            cell-index = <4>;
+            compatible = "fsl,fman-memac";
+            reg = <0xe8000 0x1000>;
+            fsl,fman-ports = <&fman0_rx_0x0c &fman0_tx_0x2c>;
+            ptp-timer = <&ptp_timer0>;
+            pcsphy-handle = <&pcsphy4>;
+            phy-handle = <&sgmii_phy1>;
+            phy-connection-type = "sgmii";
+    };
+...
diff --git a/Documentation/devicetree/bindings/net/fsl-fman.txt b/Documentation/devicetree/bindings/net/fsl-fman.txt
index 801efc7d6818..b9055335db3b 100644
--- a/Documentation/devicetree/bindings/net/fsl-fman.txt
+++ b/Documentation/devicetree/bindings/net/fsl-fman.txt
@@ -232,133 +232,7 @@ port@81000 {
 =============================================================================
 FMan dTSEC/XGEC/mEMAC Node
 
-DESCRIPTION
-
-mEMAC/dTSEC/XGEC are the Ethernet network interfaces
-
-PROPERTIES
-
-- compatible
-		Usage: required
-		Value type: <stringlist>
-		Definition: A standard property.
-		Must include one of the following:
-		- "fsl,fman-dtsec" for dTSEC MAC
-		- "fsl,fman-xgec" for XGEC MAC
-		- "fsl,fman-memac" for mEMAC MAC
-
-- cell-index
-		Usage: required
-		Value type: <u32>
-		Definition: Specifies the MAC id.
-
-		The cell-index value may be used by the FMan or the SoC, to
-		identify the MAC unit in the FMan (or SoC) memory map.
-		In the tables below there's a description of the cell-index
-		use, there are two tables, one describes the use of cell-index
-		by the FMan, the second describes the use by the SoC:
-
-		1. FMan Registers
-
-		FManV2:
-		register[bit]		MAC		cell-index
-		============================================================
-		FM_EPI[16]		XGEC		8
-		FM_EPI[16+n]		dTSECn		n-1
-		FM_NPI[11+n]		dTSECn		n-1
-			n = 1,..,5
-
-		FManV3:
-		register[bit]		MAC		cell-index
-		============================================================
-		FM_EPI[16+n]		mEMACn		n-1
-		FM_EPI[25]		mEMAC10		9
-
-		FM_NPI[11+n]		mEMACn		n-1
-		FM_NPI[10]		mEMAC10		9
-		FM_NPI[11]		mEMAC9		8
-			n = 1,..8
-
-		FM_EPI and FM_NPI are located in the FMan memory map.
-
-		2. SoC registers:
-
-		- P2041, P3041, P4080 P5020, P5040:
-		register[bit]		FMan		MAC		cell
-					Unit				index
-		============================================================
-		DCFG_DEVDISR2[7]	1		XGEC		8
-		DCFG_DEVDISR2[7+n]	1		dTSECn		n-1
-		DCFG_DEVDISR2[15]	2		XGEC		8
-		DCFG_DEVDISR2[15+n]	2		dTSECn		n-1
-			n = 1,..5
-
-		- T1040, T2080, T4240, B4860:
-		register[bit]			FMan	MAC		cell
-						Unit			index
-		============================================================
-		DCFG_CCSR_DEVDISR2[n-1]		1	mEMACn		n-1
-		DCFG_CCSR_DEVDISR2[11+n]	2	mEMACn		n-1
-			n = 1,..6,9,10
-
-		EVDISR, DCFG_DEVDISR2 and DCFG_CCSR_DEVDISR2 are located in
-		the specific SoC "Device Configuration/Pin Control" Memory
-		Map.
-
-- reg
-		Usage: required
-		Value type: <prop-encoded-array>
-		Definition: A standard property.
-
-- fsl,fman-ports
-		Usage: required
-		Value type: <prop-encoded-array>
-		Definition: An array of two phandles - the first references is
-		the FMan RX port and the second is the TX port used by this
-		MAC.
-
-- ptp-timer
-		Usage required
-		Value type: <phandle>
-		Definition: A phandle for 1EEE1588 timer.
-
-- pcsphy-handle
-		Usage required for "fsl,fman-memac" MACs
-		Value type: <phandle>
-		Definition: A phandle for pcsphy.
-
-- tbi-handle
-		Usage required for "fsl,fman-dtsec" MACs
-		Value type: <phandle>
-		Definition: A phandle for tbiphy.
-
-EXAMPLE
-
-fman1_tx28: port@a8000 {
-	cell-index = <0x28>;
-	compatible = "fsl,fman-v2-port-tx";
-	reg = <0xa8000 0x1000>;
-};
-
-fman1_rx8: port@88000 {
-	cell-index = <0x8>;
-	compatible = "fsl,fman-v2-port-rx";
-	reg = <0x88000 0x1000>;
-};
-
-ptp-timer: ptp_timer@fe000 {
-	compatible = "fsl,fman-ptp-timer";
-	reg = <0xfe000 0x1000>;
-};
-
-ethernet@e0000 {
-	compatible = "fsl,fman-dtsec";
-	cell-index = <0>;
-	reg = <0xe0000 0x1000>;
-	fsl,fman-ports = <&fman1_rx8 &fman1_tx28>;
-	ptp-timer = <&ptp-timer>;
-	tbi-handle = <&tbi0>;
-};
+Refer to Documentation/devicetree/bindings/net/fsl,fman-dtsec.yaml
 
 ============================================================================
 FMan IEEE 1588 Node
diff --git a/Documentation/devicetree/bindings/net/mediatek,mt7620-gsw.txt b/Documentation/devicetree/bindings/net/mediatek,mt7620-gsw.txt
deleted file mode 100644
index 358fed2fab43..000000000000
--- a/Documentation/devicetree/bindings/net/mediatek,mt7620-gsw.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-Mediatek Gigabit Switch
-=======================
-
-The mediatek gigabit switch can be found on Mediatek SoCs (mt7620, mt7621).
-
-Required properties:
-- compatible: Should be "mediatek,mt7620-gsw" or "mediatek,mt7621-gsw"
-- reg: Address and length of the register set for the device
-- interrupts: Should contain the gigabit switches interrupt
-- resets: Should contain the gigabit switches resets
-- reset-names: Should contain the reset names "gsw"
-
-Example:
-
-gsw@10110000 {
-	compatible = "ralink,mt7620-gsw";
-	reg = <0x10110000 8000>;
-
-	resets = <&rstctrl 23>;
-	reset-names = "gsw";
-
-	interrupt-parent = <&intc>;
-	interrupts = <17>;
-};
diff --git a/Documentation/devicetree/bindings/net/mediatek,net.yaml b/Documentation/devicetree/bindings/net/mediatek,net.yaml
index f5564ecddb62..7ef696204c5a 100644
--- a/Documentation/devicetree/bindings/net/mediatek,net.yaml
+++ b/Documentation/devicetree/bindings/net/mediatek,net.yaml
@@ -69,6 +69,15 @@ properties:
       A list of phandle to the syscon node that handles the SGMII setup which is required for
       those SoCs equipped with SGMII.
 
+  mediatek,wed:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    minItems: 2
+    maxItems: 2
+    items:
+      maxItems: 1
+    description:
+      List of phandles to wireless ethernet dispatch nodes.
+
   dma-coherent: true
 
   mdio-bus:
@@ -112,6 +121,8 @@ allOf:
             Phandle to the syscon node that handles the ports slew rate and
             driver current.
 
+        mediatek,wed: false
+
   - if:
       properties:
         compatible:
@@ -144,15 +155,6 @@ allOf:
           minItems: 1
           maxItems: 1
 
-        mediatek,wed:
-          $ref: /schemas/types.yaml#/definitions/phandle-array
-          minItems: 2
-          maxItems: 2
-          items:
-            maxItems: 1
-          description:
-            List of phandles to wireless ethernet dispatch nodes.
-
         mediatek,pcie-mirror:
           $ref: /schemas/types.yaml#/definitions/phandle
           description:
@@ -202,6 +204,8 @@ allOf:
           minItems: 2
           maxItems: 2
 
+        mediatek,wed: false
+
   - if:
       properties:
         compatible:
@@ -238,6 +242,11 @@ allOf:
           minItems: 2
           maxItems: 2
 
+        mediatek,wed-pcie:
+          $ref: /schemas/types.yaml#/definitions/phandle
+          description:
+            Phandle to the mediatek wed-pcie controller.
+
 patternProperties:
   "^mac@[0-1]$":
     type: object
diff --git a/Documentation/devicetree/bindings/net/mediatek-dwmac.yaml b/Documentation/devicetree/bindings/net/mediatek-dwmac.yaml
index 61b2fb9e141b..0fa2132fa4f4 100644
--- a/Documentation/devicetree/bindings/net/mediatek-dwmac.yaml
+++ b/Documentation/devicetree/bindings/net/mediatek-dwmac.yaml
@@ -19,6 +19,7 @@ select:
       contains:
         enum:
           - mediatek,mt2712-gmac
+          - mediatek,mt8188-gmac
           - mediatek,mt8195-gmac
   required:
     - compatible
@@ -37,6 +38,11 @@ properties:
           - enum:
               - mediatek,mt8195-gmac
           - const: snps,dwmac-5.10a
+      - items:
+          - enum:
+              - mediatek,mt8188-gmac
+          - const: mediatek,mt8195-gmac
+          - const: snps,dwmac-5.10a
 
   clocks:
     minItems: 5
@@ -74,7 +80,7 @@ properties:
       or will round down. Range 0~31*170.
       For MT2712 RMII/MII interface, Allowed value need to be a multiple of 550,
       or will round down. Range 0~31*550.
-      For MT8195 RGMII/RMII/MII interface, Allowed value need to be a multiple of 290,
+      For MT8188/MT8195 RGMII/RMII/MII interface, Allowed value need to be a multiple of 290,
       or will round down. Range 0~31*290.
 
   mediatek,rx-delay-ps:
@@ -84,7 +90,7 @@ properties:
       or will round down. Range 0~31*170.
       For MT2712 RMII/MII interface, Allowed value need to be a multiple of 550,
       or will round down. Range 0~31*550.
-      For MT8195 RGMII/RMII/MII interface, Allowed value need to be a multiple
+      For MT8188/MT8195 RGMII/RMII/MII interface, Allowed value need to be a multiple
       of 290, or will round down. Range 0~31*290.
 
   mediatek,rmii-rxc:
diff --git a/Documentation/devicetree/bindings/net/microchip,sparx5-switch.yaml b/Documentation/devicetree/bindings/net/microchip,sparx5-switch.yaml
index 6c86d3d85e99..57ffeb8fc876 100644
--- a/Documentation/devicetree/bindings/net/microchip,sparx5-switch.yaml
+++ b/Documentation/devicetree/bindings/net/microchip,sparx5-switch.yaml
@@ -74,16 +74,20 @@ properties:
 
   ethernet-ports:
     type: object
+    additionalProperties: false
+
+    properties:
+      '#address-cells':
+        const: 1
+      '#size-cells':
+        const: 0
+
     patternProperties:
       "^port@[0-9a-f]+$":
-        type: object
+        $ref: /schemas/net/ethernet-controller.yaml#
+        unevaluatedProperties: false
 
         properties:
-          '#address-cells':
-            const: 1
-          '#size-cells':
-            const: 0
-
           reg:
             description: Switch port number
 
@@ -93,29 +97,11 @@ properties:
               phandle of a Ethernet SerDes PHY.  This defines which SerDes
               instance will handle the Ethernet traffic.
 
-          phy-mode:
-            description:
-              This specifies the interface used by the Ethernet SerDes towards
-              the PHY or SFP.
-
           microchip,bandwidth:
             description: Specifies bandwidth in Mbit/s allocated to the port.
             $ref: "/schemas/types.yaml#/definitions/uint32"
             maximum: 25000
 
-          phy-handle:
-            description:
-              phandle of a Ethernet PHY.  This is optional and if provided it
-              points to the cuPHY used by the Ethernet SerDes.
-
-          sfp:
-            description:
-              phandle of an SFP.  This is optional and used when not specifying
-              a cuPHY.  It points to the SFP node that describes the SFP used by
-              the Ethernet SerDes.
-
-          managed: true
-
           microchip,sd-sgpio:
             description:
               Index of the ports Signal Detect SGPIO in the set of 384 SGPIOs
@@ -144,8 +130,6 @@ required:
   - reg-names
   - interrupts
   - interrupt-names
-  - resets
-  - reset-names
   - ethernet-ports
 
 additionalProperties: false
diff --git a/Documentation/devicetree/bindings/net/nfc/marvell,nci.yaml b/Documentation/devicetree/bindings/net/nfc/marvell,nci.yaml
index a191a04e681c..308485a8ee6c 100644
--- a/Documentation/devicetree/bindings/net/nfc/marvell,nci.yaml
+++ b/Documentation/devicetree/bindings/net/nfc/marvell,nci.yaml
@@ -128,7 +128,7 @@ examples:
 
             i2c-int-rising;
 
-            reset-n-io = <&gpio3 19 GPIO_ACTIVE_HIGH>;
+            reset-n-io = <&gpio3 19 GPIO_ACTIVE_LOW>;
         };
     };
 
@@ -151,7 +151,7 @@ examples:
             interrupt-parent = <&gpio1>;
             interrupts = <17 IRQ_TYPE_EDGE_RISING>;
 
-            reset-n-io = <&gpio3 19 GPIO_ACTIVE_HIGH>;
+            reset-n-io = <&gpio3 19 GPIO_ACTIVE_LOW>;
         };
     };
 
@@ -162,7 +162,7 @@ examples:
         nfc {
             compatible = "marvell,nfc-uart";
 
-            reset-n-io = <&gpio3 16 GPIO_ACTIVE_HIGH>;
+            reset-n-io = <&gpio3 16 GPIO_ACTIVE_LOW>;
 
             hci-muxed;
             flow-control;
diff --git a/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml b/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml
new file mode 100644
index 000000000000..2bd3efff2485
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/nvidia,tegra234-mgbe.yaml
@@ -0,0 +1,162 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/nvidia,tegra234-mgbe.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Tegra234 MGBE Multi-Gigabit Ethernet Controller
+
+maintainers:
+  - Thierry Reding <treding@nvidia.com>
+  - Jon Hunter <jonathanh@nvidia.com>
+
+properties:
+  compatible:
+    const: nvidia,tegra234-mgbe
+
+  reg:
+    maxItems: 3
+
+  reg-names:
+    items:
+      - const: hypervisor
+      - const: mac
+      - const: xpcs
+
+  interrupts:
+    minItems: 1
+    maxItems: 3
+
+  interrupt-names:
+    minItems: 1
+    items:
+      - const: common
+      - const: macsec-ns
+      - const: macsec
+
+  clocks:
+    maxItems: 12
+
+  clock-names:
+    items:
+      - const: mgbe
+      - const: mac
+      - const: mac-divider
+      - const: ptp-ref
+      - const: rx-input-m
+      - const: rx-input
+      - const: tx
+      - const: eee-pcs
+      - const: rx-pcs-input
+      - const: rx-pcs-m
+      - const: rx-pcs
+      - const: tx-pcs
+
+  resets:
+    maxItems: 2
+
+  reset-names:
+    items:
+      - const: mac
+      - const: pcs
+
+  interconnects:
+    items:
+      - description: memory read client
+      - description: memory write client
+
+  interconnect-names:
+    items:
+      - const: dma-mem
+      - const: write
+
+  iommus:
+    maxItems: 1
+
+  power-domains:
+    maxItems: 1
+
+  phy-handle: true
+
+  phy-mode:
+    contains:
+      enum:
+        - usxgmii
+        - 10gbase-kr
+
+  mdio:
+    $ref: mdio.yaml#
+    unevaluatedProperties: false
+    description:
+      Optional node for embedded MDIO controller.
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - interrupt-names
+  - clocks
+  - clock-names
+  - resets
+  - reset-names
+  - power-domains
+  - phy-handle
+  - phy-mode
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/tegra234-clock.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/memory/tegra234-mc.h>
+    #include <dt-bindings/power/tegra234-powergate.h>
+    #include <dt-bindings/reset/tegra234-reset.h>
+
+    ethernet@6800000 {
+        compatible = "nvidia,tegra234-mgbe";
+        reg = <0x06800000 0x10000>,
+              <0x06810000 0x10000>,
+              <0x068a0000 0x10000>;
+        reg-names = "hypervisor", "mac", "xpcs";
+        interrupts = <GIC_SPI 384 IRQ_TYPE_LEVEL_HIGH>;
+        interrupt-names = "common";
+        clocks = <&bpmp TEGRA234_CLK_MGBE0_APP>,
+                 <&bpmp TEGRA234_CLK_MGBE0_MAC>,
+                 <&bpmp TEGRA234_CLK_MGBE0_MAC_DIVIDER>,
+                 <&bpmp TEGRA234_CLK_MGBE0_PTP_REF>,
+                 <&bpmp TEGRA234_CLK_MGBE0_RX_INPUT_M>,
+                 <&bpmp TEGRA234_CLK_MGBE0_RX_INPUT>,
+                 <&bpmp TEGRA234_CLK_MGBE0_TX>,
+                 <&bpmp TEGRA234_CLK_MGBE0_EEE_PCS>,
+                 <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_INPUT>,
+                 <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_M>,
+                 <&bpmp TEGRA234_CLK_MGBE0_RX_PCS>,
+                 <&bpmp TEGRA234_CLK_MGBE0_TX_PCS>;
+        clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m",
+                      "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m",
+                      "rx-pcs", "tx-pcs";
+        resets = <&bpmp TEGRA234_RESET_MGBE0_MAC>,
+                 <&bpmp TEGRA234_RESET_MGBE0_PCS>;
+        reset-names = "mac", "pcs";
+        interconnects = <&mc TEGRA234_MEMORY_CLIENT_MGBEARD &emc>,
+                        <&mc TEGRA234_MEMORY_CLIENT_MGBEAWR &emc>;
+        interconnect-names = "dma-mem", "write";
+        iommus = <&smmu_niso0 TEGRA234_SID_MGBE>;
+        power-domains = <&bpmp TEGRA234_POWER_DOMAIN_MGBEA>;
+
+        phy-handle = <&mgbe0_phy>;
+        phy-mode = "usxgmii";
+
+        mdio {
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            mgbe0_phy: phy@0 {
+                compatible = "ethernet-phy-ieee802.3-c45";
+                reg = <0x0>;
+
+                #phy-cells = <0>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/net/nxp,tja11xx.yaml b/Documentation/devicetree/bindings/net/nxp,tja11xx.yaml
index d51da24f3505..ab8867e6939b 100644
--- a/Documentation/devicetree/bindings/net/nxp,tja11xx.yaml
+++ b/Documentation/devicetree/bindings/net/nxp,tja11xx.yaml
@@ -31,6 +31,22 @@ patternProperties:
         description:
           The ID number for the child PHY. Should be +1 of parent PHY.
 
+      nxp,rmii-refclk-in:
+        type: boolean
+        description: |
+          The REF_CLK is provided for both transmitted and received data
+          in RMII mode. This clock signal is provided by the PHY and is
+          typically derived from an external 25MHz crystal. Alternatively,
+          a 50MHz clock signal generated by an external oscillator can be
+          connected to pin REF_CLK. A third option is to connect a 25MHz
+          clock to pin CLK_IN_OUT. So, the REF_CLK should be configured
+          as input or output according to the actual circuit connection.
+          If present, indicates that the REF_CLK will be configured as
+          interface reference clock input when RMII mode enabled.
+          If not present, the REF_CLK will be configured as interface
+          reference clock output when RMII mode enabled.
+          Only supported on TJA1100 and TJA1101.
+
     required:
       - reg
 
@@ -44,6 +60,7 @@ examples:
 
         tja1101_phy0: ethernet-phy@4 {
             reg = <0x4>;
+            nxp,rmii-refclk-in;
         };
     };
   - |
diff --git a/Documentation/devicetree/bindings/net/pse-pd/podl-pse-regulator.yaml b/Documentation/devicetree/bindings/net/pse-pd/podl-pse-regulator.yaml
new file mode 100644
index 000000000000..c6b1c188abf7
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/pse-pd/podl-pse-regulator.yaml
@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/pse-pd/podl-pse-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Regulator based Power Sourcing Equipment
+
+maintainers:
+  - Oleksij Rempel <o.rempel@pengutronix.de>
+
+description: Regulator based PoDL PSE controller. The device must be referenced
+  by the PHY node to control power injection to the Ethernet cable.
+
+allOf:
+  - $ref: "pse-controller.yaml#"
+
+properties:
+  compatible:
+    const: podl-pse-regulator
+
+  '#pse-cells':
+    const: 0
+
+  pse-supply:
+    description: Power supply for the PSE controller
+
+additionalProperties: false
+
+required:
+  - compatible
+  - pse-supply
+
+examples:
+  - |
+    ethernet-pse {
+      compatible = "podl-pse-regulator";
+      pse-supply = <&reg_t1l1>;
+      #pse-cells = <0>;
+    };
diff --git a/Documentation/devicetree/bindings/net/pse-pd/pse-controller.yaml b/Documentation/devicetree/bindings/net/pse-pd/pse-controller.yaml
new file mode 100644
index 000000000000..b110abb42597
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/pse-pd/pse-controller.yaml
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/pse-pd/pse-controller.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Power Sourcing Equipment (PSE).
+
+description: Binding for the Power Sourcing Equipment (PSE) as defined in the
+  IEEE 802.3 specification. It is designed for hardware which is delivering
+  power over twisted pair/ethernet cable. The ethernet-pse nodes should be
+  used to describe PSE controller and referenced by the ethernet-phy node.
+
+maintainers:
+  - Oleksij Rempel <o.rempel@pengutronix.de>
+
+properties:
+  $nodename:
+    pattern: "^ethernet-pse(@.*)?$"
+
+  "#pse-cells":
+    description:
+      Used to uniquely identify a PSE instance within an IC. Will be
+      0 on PSE nodes with only a single output and at least 1 on nodes
+      controlling several outputs.
+    enum: [0, 1]
+
+required:
+  - "#pse-cells"
+
+additionalProperties: true
+
+...
diff --git a/Documentation/devicetree/bindings/net/qca,ar803x.yaml b/Documentation/devicetree/bindings/net/qca,ar803x.yaml
index b3d4013b7ca6..161d28919316 100644
--- a/Documentation/devicetree/bindings/net/qca,ar803x.yaml
+++ b/Documentation/devicetree/bindings/net/qca,ar803x.yaml
@@ -40,6 +40,14 @@ properties:
       Only supported on the AR8031.
     type: boolean
 
+  qca,disable-hibernation-mode:
+    description: |
+      Disable Atheros AR803X PHYs hibernation mode. If present, indicates
+      that the hardware of PHY will not enter power saving mode when the
+      cable is disconnected. And the RX_CLK always keeps outputting a
+      valid clock.
+    type: boolean
+
   qca,smarteee-tw-us-100m:
     description: EEE Tw parameter for 100M links.
     $ref: /schemas/types.yaml#/definitions/uint32
diff --git a/Documentation/devicetree/bindings/net/ralink,rt2880-net.txt b/Documentation/devicetree/bindings/net/ralink,rt2880-net.txt
deleted file mode 100644
index 9fe1a0a22e44..000000000000
--- a/Documentation/devicetree/bindings/net/ralink,rt2880-net.txt
+++ /dev/null
@@ -1,59 +0,0 @@
-Ralink Frame Engine Ethernet controller
-=======================================
-
-The Ralink frame engine ethernet controller can be found on Ralink and
-Mediatek SoCs (RT288x, RT3x5x, RT366x, RT388x, rt5350, mt7620, mt7621, mt76x8).
-
-Depending on the SoC, there is a number of ports connected to the CPU port
-directly and/or via a (gigabit-)switch.
-
-* Ethernet controller node
-
-Required properties:
-- compatible: Should be one of "ralink,rt2880-eth", "ralink,rt3050-eth",
-  "ralink,rt3050-eth", "ralink,rt3883-eth", "ralink,rt5350-eth",
-  "mediatek,mt7620-eth", "mediatek,mt7621-eth"
-- reg: Address and length of the register set for the device
-- interrupts: Should contain the frame engines interrupt
-- resets: Should contain the frame engines resets
-- reset-names: Should contain the reset names "fe". If a switch is present
-  "esw" is also required.
-
-
-* Ethernet port node
-
-Required properties:
-- compatible: Should be "ralink,eth-port"
-- reg: The number of the physical port
-- phy-handle: reference to the node describing the phy
-
-Example:
-
-mdio-bus {
-	...
-	phy0: ethernet-phy@0 {
-		phy-mode = "mii";
-		reg = <0>;
-	};
-};
-
-ethernet@400000 {
-	compatible = "ralink,rt2880-eth";
-	reg = <0x00400000 10000>;
-
-	#address-cells = <1>;
-	#size-cells = <0>;
-
-	resets = <&rstctrl 18>;
-	reset-names = "fe";
-
-	interrupt-parent = <&cpuintc>;
-	interrupts = <5>;
-
-	port@0 {
-		compatible = "ralink,eth-port";
-		reg = <0>;
-		phy-handle = <&phy0>;
-	};
-
-};
diff --git a/Documentation/devicetree/bindings/net/ralink,rt3050-esw.txt b/Documentation/devicetree/bindings/net/ralink,rt3050-esw.txt
deleted file mode 100644
index 87e315856efa..000000000000
--- a/Documentation/devicetree/bindings/net/ralink,rt3050-esw.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-Ralink Fast Ethernet Embedded Switch
-====================================
-
-The ralink fast ethernet embedded switch can be found on Ralink and Mediatek
-SoCs (RT3x5x, RT5350, MT76x8).
-
-Required properties:
-- compatible: Should be "ralink,rt3050-esw"
-- reg: Address and length of the register set for the device
-- interrupts: Should contain the embedded switches interrupt
-- resets: Should contain the embedded switches resets
-- reset-names: Should contain the reset names "esw"
-
-Optional properties:
-- ralink,portmap: can be used to choose if the default switch setup is
-  llllw or wllll
-- ralink,led_polarity: override the active high/low settings of the leds
-
-Example:
-
-esw@10110000 {
-	compatible = "ralink,rt3050-esw";
-	reg = <0x10110000 8000>;
-
-	resets = <&rstctrl 23>;
-	reset-names = "esw";
-
-	interrupt-parent = <&intc>;
-	interrupts = <17>;
-};
diff --git a/Documentation/devicetree/bindings/net/renesas,etheravb.yaml b/Documentation/devicetree/bindings/net/renesas,etheravb.yaml
index acf347f3cdbe..3f41294f5997 100644
--- a/Documentation/devicetree/bindings/net/renesas,etheravb.yaml
+++ b/Documentation/devicetree/bindings/net/renesas,etheravb.yaml
@@ -40,11 +40,16 @@ properties:
               - renesas,etheravb-r8a77980     # R-Car V3H
               - renesas,etheravb-r8a77990     # R-Car E3
               - renesas,etheravb-r8a77995     # R-Car D3
-              - renesas,etheravb-r8a779a0     # R-Car V3U
           - const: renesas,etheravb-rcar-gen3 # R-Car Gen3 and RZ/G2
 
       - items:
           - enum:
+              - renesas,etheravb-r8a779a0     # R-Car V3U
+              - renesas,etheravb-r8a779g0     # R-Car V4H
+          - const: renesas,etheravb-rcar-gen4 # R-Car Gen4
+
+      - items:
+          - enum:
               - renesas,etheravb-r9a09g011 # RZ/V2M
           - const: renesas,etheravb-rzv2m  # RZ/V2M compatible
 
@@ -207,7 +212,7 @@ allOf:
               - renesas,etheravb-r8a77965
               - renesas,etheravb-r8a77970
               - renesas,etheravb-r8a77980
-              - renesas,etheravb-r8a779a0
+              - renesas,etheravb-rcar-gen4
     then:
       required:
         - tx-internal-delay-ps
diff --git a/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml b/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml
index 083623c8d718..42fb72b6909d 100644
--- a/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml
+++ b/Documentation/devicetree/bindings/net/rockchip-dwmac.yaml
@@ -25,7 +25,9 @@ select:
           - rockchip,rk3368-gmac
           - rockchip,rk3399-gmac
           - rockchip,rk3568-gmac
+          - rockchip,rk3588-gmac
           - rockchip,rv1108-gmac
+          - rockchip,rv1126-gmac
   required:
     - compatible
 
@@ -47,9 +49,11 @@ properties:
               - rockchip,rk3368-gmac
               - rockchip,rk3399-gmac
               - rockchip,rv1108-gmac
+              - rockchip,rv1126-gmac
       - items:
           - enum:
               - rockchip,rk3568-gmac
+              - rockchip,rk3588-gmac
           - const: snps,dwmac-4.20a
 
   clocks:
@@ -81,6 +85,11 @@ properties:
     description: The phandle of the syscon node for the general register file.
     $ref: /schemas/types.yaml#/definitions/phandle
 
+  rockchip,php-grf:
+    description:
+      The phandle of the syscon node for the peripheral general register file.
+    $ref: /schemas/types.yaml#/definitions/phandle
+
   tx_delay:
     description: Delay value for TXD timing. Range value is 0~0x7F, 0x30 as default.
     $ref: /schemas/types.yaml#/definitions/uint32
diff --git a/Documentation/devicetree/bindings/net/snps,dwmac.yaml b/Documentation/devicetree/bindings/net/snps,dwmac.yaml
index 491597c02edf..f94a0a4320f1 100644
--- a/Documentation/devicetree/bindings/net/snps,dwmac.yaml
+++ b/Documentation/devicetree/bindings/net/snps,dwmac.yaml
@@ -74,6 +74,7 @@ properties:
         - rockchip,rk3328-gmac
         - rockchip,rk3366-gmac
         - rockchip,rk3368-gmac
+        - rockchip,rk3588-gmac
         - rockchip,rk3399-gmac
         - rockchip,rv1108-gmac
         - snps,dwmac
@@ -288,6 +289,11 @@ properties:
       is supported. For example, this is used in case of SGMII and
       MAC2MAC connection.
 
+  snps,clk-csr:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Frequency division factor for MDC clock.
+
   mdio:
     $ref: mdio.yaml#
     unevaluatedProperties: false
@@ -301,6 +307,60 @@ properties:
     required:
       - compatible
 
+  stmmac-axi-config:
+    type: object
+    unevaluatedProperties: false
+    description:
+      AXI BUS Mode parameters.
+
+    properties:
+      snps,lpi_en:
+        $ref: /schemas/types.yaml#/definitions/flag
+        description:
+          enable Low Power Interface
+
+      snps,xit_frm:
+        $ref: /schemas/types.yaml#/definitions/flag
+        description:
+          unlock on WoL
+
+      snps,wr_osr_lmt:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description:
+          max write outstanding req. limit
+
+      snps,rd_osr_lmt:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description:
+          max read outstanding req. limit
+
+      snps,kbbe:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        description:
+          do not cross 1KiB boundary.
+
+      snps,blen:
+        $ref: /schemas/types.yaml#/definitions/uint32-array
+        description:
+          this is a vector of supported burst length.
+        minItems: 7
+        maxItems: 7
+
+      snps,fb:
+        $ref: /schemas/types.yaml#/definitions/flag
+        description:
+          fixed-burst
+
+      snps,mb:
+        $ref: /schemas/types.yaml#/definitions/flag
+        description:
+          mixed-burst
+
+      snps,rb:
+        $ref: /schemas/types.yaml#/definitions/flag
+        description:
+          rebuild INCRx Burst
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/net/sunplus,sp7021-emac.yaml b/Documentation/devicetree/bindings/net/sunplus,sp7021-emac.yaml
index 62dffee27c3d..8e51dcdb4796 100644
--- a/Documentation/devicetree/bindings/net/sunplus,sp7021-emac.yaml
+++ b/Documentation/devicetree/bindings/net/sunplus,sp7021-emac.yaml
@@ -32,6 +32,7 @@ properties:
 
   ethernet-ports:
     type: object
+    additionalProperties: false
     description: Ethernet ports to PHY
 
     properties:
@@ -44,6 +45,7 @@ properties:
     patternProperties:
       "^port@[0-1]$":
         type: object
+        additionalProperties: false
         description: Port to PHY
 
         properties:
diff --git a/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml b/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml
index 31bf825c6598..46e330f45768 100644
--- a/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml
+++ b/Documentation/devicetree/bindings/net/ti,cpsw-switch.yaml
@@ -77,6 +77,8 @@ properties:
 
   ethernet-ports:
     type: object
+    additionalProperties: false
+
     properties:
       '#address-cells':
         const: 1
@@ -89,6 +91,7 @@ properties:
         description: CPSW external ports
 
         $ref: ethernet-controller.yaml#
+        unevaluatedProperties: false
 
         properties:
           reg:
@@ -117,6 +120,7 @@ properties:
 
   cpts:
     type: object
+    unevaluatedProperties: false
     description:
       The Common Platform Time Sync (CPTS) module
 
diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml
index b8281d8be940..7d90beaccc60 100644
--- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml
+++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml
@@ -55,6 +55,7 @@ properties:
   compatible:
     enum:
       - ti,am654-cpsw-nuss
+      - ti,j7200-cpswxg-nuss
       - ti,j721e-cpsw-nuss
       - ti,am642-cpsw-nuss
 
@@ -110,16 +111,17 @@ properties:
         const: 0
 
     patternProperties:
-      port@[1-2]:
+      "^port@[1-4]$":
         type: object
         description: CPSWxG NUSS external ports
 
         $ref: ethernet-controller.yaml#
+        unevaluatedProperties: false
 
         properties:
           reg:
             minimum: 1
-            maximum: 2
+            maximum: 4
             description: CPSW port number
 
           phys:
@@ -178,6 +180,19 @@ required:
   - '#address-cells'
   - '#size-cells'
 
+allOf:
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              const: ti,j7200-cpswxg-nuss
+    then:
+      properties:
+        ethernet-ports:
+          patternProperties:
+            "^port@[3-4]$": false
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml
index b783ad0d1f53..e9f78cef6b7f 100644
--- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml
+++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml
@@ -95,6 +95,7 @@ properties:
 
   refclk-mux:
     type: object
+    additionalProperties: false
     description: CPTS reference clock multiplexer clock
     properties:
       '#clock-cells':
diff --git a/Documentation/devicetree/bindings/net/vertexcom-mse102x.yaml b/Documentation/devicetree/bindings/net/vertexcom-mse102x.yaml
index 8156a9aeb589..304757bf9281 100644
--- a/Documentation/devicetree/bindings/net/vertexcom-mse102x.yaml
+++ b/Documentation/devicetree/bindings/net/vertexcom-mse102x.yaml
@@ -7,7 +7,7 @@ $schema: "http://devicetree.org/meta-schemas/core.yaml#"
 title: The Vertexcom MSE102x (SPI) Device Tree Bindings
 
 maintainers:
-  - Stefan Wahren <stefan.wahren@in-tech.com>
+  - Stefan Wahren <stefan.wahren@chargebyte.com>
 
 description:
   Vertexcom's MSE102x are a family of HomePlug GreenPHY chips.
diff --git a/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml b/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml
index 53b4153d9bfc..fec1cc9b9a08 100644
--- a/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/brcm,bcm4329-fmac.yaml
@@ -4,7 +4,7 @@
 $id: http://devicetree.org/schemas/net/wireless/brcm,bcm4329-fmac.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Broadcom BCM4329 family fullmac wireless SDIO devices
+title: Broadcom BCM4329 family fullmac wireless SDIO/PCIE devices
 
 maintainers:
   - Arend van Spriel <arend@broadcom.com>
@@ -41,11 +41,17 @@ properties:
               - cypress,cyw4373-fmac
               - cypress,cyw43012-fmac
           - const: brcm,bcm4329-fmac
-      - const: brcm,bcm4329-fmac
+      - enum:
+          - brcm,bcm4329-fmac
+          - pci14e4,43dc  # BCM4355
+          - pci14e4,4464  # BCM4364
+          - pci14e4,4488  # BCM4377
+          - pci14e4,4425  # BCM4378
+          - pci14e4,4433  # BCM4387
 
   reg:
-    description: SDIO function number for the device, for most cases
-      this will be 1.
+    description: SDIO function number for the device (for most cases
+      this will be 1) or PCI device identifier.
 
   interrupts:
     maxItems: 1
@@ -85,6 +91,31 @@ properties:
       takes precedence.
     type: boolean
 
+  brcm,cal-blob:
+    $ref: /schemas/types.yaml#/definitions/uint8-array
+    description: A per-device calibration blob for the Wi-Fi radio. This
+      should be filled in by the bootloader from platform configuration
+      data, if necessary, and will be uploaded to the device if present.
+
+  brcm,board-type:
+    $ref: /schemas/types.yaml#/definitions/string
+    description: Overrides the board type, which is normally the compatible of
+      the root node. This can be used to decouple the overall system board or
+      device name from the board type for WiFi purposes, which is used to
+      construct firmware and NVRAM configuration filenames, allowing for
+      multiple devices that share the same module or characteristics for the
+      WiFi subsystem to share the same firmware/NVRAM files. On Apple platforms,
+      this should be the Apple module-instance codename prefixed by "apple,",
+      e.g. "apple,honshu".
+
+  apple,antenna-sku:
+    $ref: /schemas/types.yaml#/definitions/string
+    description: Antenna SKU used to identify a specific antenna configuration
+      on Apple platforms. This is use to build firmware filenames, to allow
+      platforms with different antenna configs to have different firmware and/or
+      NVRAM. This would normally be filled in by the bootloader from platform
+      configuration data.
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/net/wireless/microchip,wilc1000.yaml b/Documentation/devicetree/bindings/net/wireless/microchip,wilc1000.yaml
index 60de78f1bc7b..b3405f284580 100644
--- a/Documentation/devicetree/bindings/net/wireless/microchip,wilc1000.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/microchip,wilc1000.yaml
@@ -20,8 +20,6 @@ properties:
 
   reg: true
 
-  spi-max-frequency: true
-
   interrupts:
     maxItems: 1
 
@@ -51,7 +49,10 @@ required:
   - compatible
   - interrupts
 
-additionalProperties: false
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml b/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml
index a677b056f112..f7cf135aa37f 100644
--- a/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml
@@ -66,6 +66,18 @@ properties:
     required:
       - iommus
 
+  qcom,smem-states:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    description: State bits used by the AP to signal the WLAN Q6.
+    items:
+      - description: Signal bits used to enable/disable low power mode
+                     on WCN6750 in the case of WoW (Wake on Wireless).
+
+  qcom,smem-state-names:
+    description: The names of the state bits used for SMP2P output.
+    items:
+      - const: wlan-smp2p-out
+
 required:
   - compatible
   - reg
@@ -448,6 +460,8 @@ examples:
                      <GIC_SPI 799 IRQ_TYPE_EDGE_RISING>;
         qcom,rproc = <&remoteproc_wpss>;
         memory-region = <&wlan_fw_mem>, <&wlan_ce_mem>;
+        qcom,smem-states = <&wlan_smp2p_out 0>;
+        qcom,smem-state-names = "wlan-smp2p-out";
         wifi-firmware {
                 iommus = <&apps_smmu 0x1c02 0x1>;
         };
diff --git a/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml b/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml
index 76199a67d628..b35d2f3ad1ad 100644
--- a/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/silabs,wfx.yaml
@@ -29,12 +29,6 @@ description: >
     Documentation/devicetree/bindings/mmc/mmc-pwrseq-simple.yaml for more
     information.
 
-  For SPI:
-
-    In add of the properties below, please consult
-    Documentation/devicetree/bindings/spi/spi-controller.yaml for optional SPI
-    related properties.
-
 properties:
   compatible:
     items:
@@ -52,8 +46,6 @@ properties:
       bindings.
     maxItems: 1
 
-  spi-max-frequency: true
-
   interrupts:
     description: The interrupt line. Should be IRQ_TYPE_EDGE_RISING. When SPI is
       used, this property is required. When SDIO is used, the "in-band"
@@ -84,12 +76,15 @@ properties:
 
   mac-address: true
 
-additionalProperties: false
-
 required:
   - compatible
   - reg
 
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
+
 examples:
   - |
     #include <dt-bindings/gpio/gpio.h>
diff --git a/Documentation/devicetree/bindings/net/wireless/ti,wlcore.yaml b/Documentation/devicetree/bindings/net/wireless/ti,wlcore.yaml
index d68bb2ec1f7e..e31456730e9f 100644
--- a/Documentation/devicetree/bindings/net/wireless/ti,wlcore.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/ti,wlcore.yaml
@@ -36,8 +36,6 @@ properties:
       This is required when connected via SPI, and optional when connected via
       SDIO.
 
-  spi-max-frequency: true
-
   interrupts:
     minItems: 1
     maxItems: 2
@@ -69,20 +67,22 @@ required:
   - compatible
   - interrupts
 
-if:
-  properties:
-    compatible:
-      contains:
-        enum:
-          - ti,wl1271
-          - ti,wl1273
-          - ti,wl1281
-          - ti,wl1283
-then:
-  required:
-    - ref-clock-frequency
-
-additionalProperties: false
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - ti,wl1271
+              - ti,wl1273
+              - ti,wl1281
+              - ti,wl1283
+    then:
+      required:
+        - ref-clock-frequency
+
+unevaluatedProperties: false
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/pci/qcom,pcie.yaml b/Documentation/devicetree/bindings/pci/qcom,pcie.yaml
index 7d29e2a45183..dd84f1487bed 100644
--- a/Documentation/devicetree/bindings/pci/qcom,pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/qcom,pcie.yaml
@@ -54,11 +54,11 @@ properties:
   # Platform constraints are described later.
   clocks:
     minItems: 3
-    maxItems: 12
+    maxItems: 13
 
   clock-names:
     minItems: 3
-    maxItems: 12
+    maxItems: 13
 
   resets:
     minItems: 1
@@ -424,8 +424,8 @@ allOf:
     then:
       properties:
         clocks:
-          minItems: 11
-          maxItems: 11
+          minItems: 13
+          maxItems: 13
         clock-names:
           items:
             - const: pipe # PIPE clock
@@ -439,6 +439,8 @@ allOf:
             - const: slave_q2a # Slave Q2A clock
             - const: tbu # PCIe TBU clock
             - const: ddrss_sf_tbu # PCIe SF TBU clock
+            - const: aggre0 # Aggre NoC PCIe CENTER SF AXI clock
+            - const: aggre1 # Aggre NoC PCIe1 AXI clock
         resets:
           maxItems: 1
         reset-names:
diff --git a/Documentation/devicetree/bindings/phy/qcom,hdmi-phy-qmp.yaml b/Documentation/devicetree/bindings/phy/qcom,hdmi-phy-qmp.yaml
index eea2e02678ed..83fe4b39b56f 100644
--- a/Documentation/devicetree/bindings/phy/qcom,hdmi-phy-qmp.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,hdmi-phy-qmp.yaml
@@ -28,12 +28,15 @@ properties:
       - const: hdmi_phy
 
   clocks:
-    maxItems: 2
+    minItems: 2
+    maxItems: 3
 
   clock-names:
+    minItems: 2
     items:
       - const: iface
       - const: ref
+      - const: xo
 
   power-domains:
     maxItems: 1
@@ -44,6 +47,9 @@ properties:
   vddio-supply:
     description: phandle to VDD I/O supply regulator
 
+  '#clock-cells':
+    const: 0
+
   '#phy-cells':
     const: 0
 
@@ -75,9 +81,12 @@ examples:
                   "hdmi_phy";
 
       clocks = <&mmcc 116>,
-               <&gcc 214>;
+               <&gcc 214>,
+               <&xo_board>;
       clock-names = "iface",
-                    "ref";
+                    "ref",
+                    "xo";
+      #clock-cells = <0>;
       #phy-cells = <0>;
 
       vddio-supply = <&vreg_l12a_1p8>;
diff --git a/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml b/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml
index b448101fac43..03b7f6aa591d 100644
--- a/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml
+++ b/Documentation/devicetree/bindings/power/mediatek,power-controller.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Mediatek Power Domains Controller
 
 maintainers:
-  - Weiyi Lu <weiyi.lu@mediatek.com>
+  - MandyJH Liu <mandyjh.liu@mediatek.com>
   - Matthias Brugger <mbrugger@suse.com>
 
 description: |
@@ -19,7 +19,7 @@ description: |
 
 properties:
   $nodename:
-    const: power-controller
+    pattern: '^power-controller(@[0-9a-f]+)?$'
 
   compatible:
     enum:
@@ -42,6 +42,23 @@ properties:
 
 patternProperties:
   "^power-domain@[0-9a-f]+$":
+    $ref: "#/$defs/power-domain-node"
+    patternProperties:
+      "^power-domain@[0-9a-f]+$":
+        $ref: "#/$defs/power-domain-node"
+        patternProperties:
+          "^power-domain@[0-9a-f]+$":
+            $ref: "#/$defs/power-domain-node"
+            patternProperties:
+              "^power-domain@[0-9a-f]+$":
+                $ref: "#/$defs/power-domain-node"
+                unevaluatedProperties: false
+            unevaluatedProperties: false
+        unevaluatedProperties: false
+    unevaluatedProperties: false
+
+$defs:
+  power-domain-node:
     type: object
     description: |
       Represents the power domains within the power controller node as documented
@@ -100,123 +117,9 @@ patternProperties:
         $ref: /schemas/types.yaml#/definitions/phandle
         description: phandle to the device containing the SMI register range.
 
-    patternProperties:
-      "^power-domain@[0-9a-f]+$":
-        type: object
-        description: |
-          Represents a power domain child within a power domain parent node.
-
-        properties:
-
-          '#power-domain-cells':
-            description:
-              Must be 0 for nodes representing a single PM domain and 1 for nodes
-              providing multiple PM domains.
-
-          '#address-cells':
-            const: 1
-
-          '#size-cells':
-            const: 0
-
-          reg:
-            maxItems: 1
-
-          clocks:
-            description: |
-              A number of phandles to clocks that need to be enabled during domain
-              power-up sequencing.
-
-          clock-names:
-            description: |
-              List of names of clocks, in order to match the power-up sequencing
-              for each power domain we need to group the clocks by name. BASIC
-              clocks need to be enabled before enabling the corresponding power
-              domain, and should not have a '-' in their name (i.e mm, mfg, venc).
-              SUSBYS clocks need to be enabled before releasing the bus protection,
-              and should contain a '-' in their name (i.e mm-0, isp-0, cam-0).
-
-              In order to follow properly the power-up sequencing, the clocks must
-              be specified by order, adding first the BASIC clocks followed by the
-              SUSBSYS clocks.
-
-          domain-supply:
-            description: domain regulator supply.
-
-          mediatek,infracfg:
-            $ref: /schemas/types.yaml#/definitions/phandle
-            description: phandle to the device containing the INFRACFG register range.
-
-          mediatek,smi:
-            $ref: /schemas/types.yaml#/definitions/phandle
-            description: phandle to the device containing the SMI register range.
-
-        patternProperties:
-          "^power-domain@[0-9a-f]+$":
-            type: object
-            description: |
-              Represents a power domain child within a power domain parent node.
-
-            properties:
-
-              '#power-domain-cells':
-                description:
-                  Must be 0 for nodes representing a single PM domain and 1 for nodes
-                  providing multiple PM domains.
-
-              '#address-cells':
-                const: 1
-
-              '#size-cells':
-                const: 0
-
-              reg:
-                maxItems: 1
-
-              clocks:
-                description: |
-                  A number of phandles to clocks that need to be enabled during domain
-                  power-up sequencing.
-
-              clock-names:
-                description: |
-                  List of names of clocks, in order to match the power-up sequencing
-                  for each power domain we need to group the clocks by name. BASIC
-                  clocks need to be enabled before enabling the corresponding power
-                  domain, and should not have a '-' in their name (i.e mm, mfg, venc).
-                  SUSBYS clocks need to be enabled before releasing the bus protection,
-                  and should contain a '-' in their name (i.e mm-0, isp-0, cam-0).
-
-                  In order to follow properly the power-up sequencing, the clocks must
-                  be specified by order, adding first the BASIC clocks followed by the
-                  SUSBSYS clocks.
-
-              domain-supply:
-                description: domain regulator supply.
-
-              mediatek,infracfg:
-                $ref: /schemas/types.yaml#/definitions/phandle
-                description: phandle to the device containing the INFRACFG register range.
-
-              mediatek,smi:
-                $ref: /schemas/types.yaml#/definitions/phandle
-                description: phandle to the device containing the SMI register range.
-
-            required:
-              - reg
-
-            additionalProperties: false
-
-        required:
-          - reg
-
-        additionalProperties: false
-
     required:
       - reg
 
-    additionalProperties: false
-
 required:
   - compatible
 
diff --git a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
index 3934a2b44894..5b4eda919911 100644
--- a/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
+++ b/Documentation/devicetree/bindings/power/qcom,rpmpd.yaml
@@ -40,6 +40,7 @@ properties:
       - qcom,sm6115-rpmpd
       - qcom,sm6125-rpmpd
       - qcom,sm6350-rpmhpd
+      - qcom,sm6375-rpmpd
       - qcom,sm8150-rpmhpd
       - qcom,sm8250-rpmhpd
       - qcom,sm8350-rpmhpd
diff --git a/Documentation/devicetree/bindings/power/rockchip,power-controller.yaml b/Documentation/devicetree/bindings/power/rockchip,power-controller.yaml
index 3deb0fc8dfd3..0d5e999a58f1 100644
--- a/Documentation/devicetree/bindings/power/rockchip,power-controller.yaml
+++ b/Documentation/devicetree/bindings/power/rockchip,power-controller.yaml
@@ -41,6 +41,8 @@ properties:
       - rockchip,rk3368-power-controller
       - rockchip,rk3399-power-controller
       - rockchip,rk3568-power-controller
+      - rockchip,rk3588-power-controller
+      - rockchip,rv1126-power-controller
 
   "#power-domain-cells":
     const: 1
@@ -119,6 +121,8 @@ $defs:
           "include/dt-bindings/power/rk3368-power.h"
           "include/dt-bindings/power/rk3399-power.h"
           "include/dt-bindings/power/rk3568-power.h"
+          "include/dt-bindings/power/rk3588-power.h"
+          "include/dt-bindings/power/rockchip,rv1126-power.h"
 
       clocks:
         minItems: 1
diff --git a/Documentation/devicetree/bindings/power/rockchip-io-domain.yaml b/Documentation/devicetree/bindings/power/rockchip-io-domain.yaml
index 1727bf108979..d71fc72d4464 100644
--- a/Documentation/devicetree/bindings/power/rockchip-io-domain.yaml
+++ b/Documentation/devicetree/bindings/power/rockchip-io-domain.yaml
@@ -58,6 +58,7 @@ properties:
       - rockchip,rk3568-pmu-io-voltage-domain
       - rockchip,rv1108-io-voltage-domain
       - rockchip,rv1108-pmu-io-voltage-domain
+      - rockchip,rv1126-pmu-io-voltage-domain
 
 required:
   - compatible
@@ -78,6 +79,7 @@ allOf:
   - $ref: "#/$defs/rk3568-pmu"
   - $ref: "#/$defs/rv1108"
   - $ref: "#/$defs/rv1108-pmu"
+  - $ref: "#/$defs/rv1126-pmu"
 
 $defs:
   px30:
@@ -344,6 +346,34 @@ $defs:
         pmu-supply:
           description: The supply connected to PMUIO_VDD.
 
+  rv1126-pmu:
+    if:
+      properties:
+        compatible:
+          contains:
+            const: rockchip,rv1126-pmu-io-voltage-domain
+
+    then:
+      properties:
+        vccio1-supply:
+          description: The supply connected to VCCIO1.
+        vccio2-supply:
+          description: The supply connected to VCCIO2.
+        vccio3-supply:
+          description: The supply connected to VCCIO3.
+        vccio4-supply:
+          description: The supply connected to VCCIO4.
+        vccio5-supply:
+          description: The supply connected to VCCIO5.
+        vccio6-supply:
+          description: The supply connected to VCCIO6.
+        vccio7-supply:
+          description: The supply connected to VCCIO7.
+        pmuio0-supply:
+          description: The supply connected to PMUIO0.
+        pmuio1-supply:
+          description: The supply connected to PMUIO1.
+
 examples:
   - |
     io-domains {
diff --git a/Documentation/devicetree/bindings/regulator/dlg,da9121.yaml b/Documentation/devicetree/bindings/regulator/dlg,da9121.yaml
index 24ace6e1e5ec..63e1161a87de 100644
--- a/Documentation/devicetree/bindings/regulator/dlg,da9121.yaml
+++ b/Documentation/devicetree/bindings/regulator/dlg,da9121.yaml
@@ -83,6 +83,7 @@ properties:
 
   regulators:
     type: object
+    additionalProperties: false
     description: |
       List of regulators provided by the device
 
diff --git a/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml b/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml
index f7e3d8fd3bf3..6c3371d706bb 100644
--- a/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/gpio-regulator.yaml
@@ -85,6 +85,9 @@ properties:
       - current
     default: voltage
 
+  vin-supply:
+    description: Input supply phandle.
+
 required:
   - compatible
   - regulator-name
diff --git a/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml b/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml
index 236348c4710c..71138c611b6c 100644
--- a/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml
+++ b/Documentation/devicetree/bindings/regulator/maxim,max77802.yaml
@@ -79,6 +79,7 @@ patternProperties:
     patternProperties:
       regulator-state-(standby|mem|disk):
         type: object
+        additionalProperties: true
         properties:
           regulator-mode: false
 
diff --git a/Documentation/devicetree/bindings/regulator/maxim,max8997.yaml b/Documentation/devicetree/bindings/regulator/maxim,max8997.yaml
index 4321f061a7f6..2b266ea43716 100644
--- a/Documentation/devicetree/bindings/regulator/maxim,max8997.yaml
+++ b/Documentation/devicetree/bindings/regulator/maxim,max8997.yaml
@@ -111,6 +111,7 @@ properties:
 
   regulators:
     type: object
+    additionalProperties: false
     description:
       List of child nodes that specify the regulators.
 
diff --git a/Documentation/devicetree/bindings/regulator/mediatek,mt6331-regulator.yaml b/Documentation/devicetree/bindings/regulator/mediatek,mt6331-regulator.yaml
new file mode 100644
index 000000000000..771cc134393c
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/mediatek,mt6331-regulator.yaml
@@ -0,0 +1,273 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/mediatek,mt6331-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MT6331 Regulator from MediaTek Integrated
+
+maintainers:
+  - AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
+
+description: |
+  The MT6331 PMIC provides 6 BUCK and 21 LDO (Low Dropout) regulators
+  and nodes are named according to the regulator type:
+  buck-<name> and ldo-<name>.
+  MT6331 regulators node should be sub node of the MT6397 MFD node.
+
+patternProperties:
+  "^buck-v(core2|io18|dvfs11|dvfs12|dvfs13|dvfs14)$":
+    type: object
+    $ref: "regulator.yaml#"
+
+    properties:
+      regulator-name:
+        pattern: "^v(core2|io18|dvfs11|dvfs12|dvfs13|dvfs14)$"
+
+    unevaluatedProperties: false
+
+  "^ldo-v(avdd32aud|auxa32)$":
+    type: object
+    $ref: "regulator.yaml#"
+
+    properties:
+      regulator-name:
+        pattern: "^v(avdd32aud|auxa32)$"
+
+    unevaluatedProperties: false
+
+  "^ldo-v(dig18|emc33|ibr|mc|mch|mipi|rtc|sram|usb10)$":
+    type: object
+    $ref: "regulator.yaml#"
+
+    properties:
+      regulator-name:
+        pattern: "^v(dig18|emc33|ibr|mc|mch|mipi|rtc|sram|usb10)$"
+
+    unevaluatedProperties: false
+
+  "^ldo-vcam(a|af|d|io)$":
+    type: object
+    $ref: "regulator.yaml#"
+
+    properties:
+      regulator-name:
+        pattern: "^vcam(a|af|d|io)$"
+
+    unevaluatedProperties: false
+
+  "^ldo-vtcxo[12]$":
+    type: object
+    $ref: "regulator.yaml#"
+
+    properties:
+      regulator-name:
+        pattern: "^vtcxo[12]$"
+
+    required:
+      - regulator-name
+
+    unevaluatedProperties: false
+
+  "^ldo-vgp[1234]$":
+    type: object
+    $ref: "regulator.yaml#"
+
+    properties:
+      regulator-name:
+        pattern: "^vgp[12]$"
+
+    required:
+      - regulator-name
+
+    unevaluatedProperties: false
+
+additionalProperties: false
+
+examples:
+  - |
+    pmic {
+      regulators {
+        mt6331_vdvfs11_reg: buck-vdvfs11 {
+          regulator-name = "vdvfs11";
+          regulator-min-microvolt = <700000>;
+          regulator-max-microvolt = <1493750>;
+          regulator-ramp-delay = <12500>;
+          regulator-enable-ramp-delay = <1>;
+          regulator-allowed-modes = <0 1>;
+        };
+        mt6331_vdvfs12_reg: buck-vdvfs12 {
+          regulator-name = "vdvfs12";
+          regulator-min-microvolt = <700000>;
+          regulator-max-microvolt = <1493750>;
+          regulator-ramp-delay = <12500>;
+          regulator-enable-ramp-delay = <1>;
+          regulator-allowed-modes = <0 1>;
+        };
+        mt6331_vdvfs13_reg: buck-vdvfs13 {
+          regulator-name = "vdvfs13";
+          regulator-min-microvolt = <700000>;
+          regulator-max-microvolt = <1493750>;
+          regulator-ramp-delay = <12500>;
+          regulator-enable-ramp-delay = <1>;
+          regulator-allowed-modes = <0 1>;
+        };
+        mt6331_vdvfs14_reg: buck-vdvfs14 {
+          regulator-name = "vdvfs14";
+          regulator-min-microvolt = <700000>;
+          regulator-max-microvolt = <1493750>;
+          regulator-ramp-delay = <12500>;
+          regulator-enable-ramp-delay = <1>;
+          regulator-allowed-modes = <0 1>;
+        };
+        mt6331_vcore2_reg: buck-vcore2 {
+          regulator-name = "vcore2";
+          regulator-min-microvolt = <700000>;
+          regulator-max-microvolt = <1493750>;
+          regulator-ramp-delay = <12500>;
+          regulator-enable-ramp-delay = <1>;
+          regulator-allowed-modes = <0 1>;
+        };
+        mt6331_vio18_reg: buck-vio18 {
+          regulator-name = "vio18";
+          regulator-min-microvolt = <1800000>;
+          regulator-max-microvolt = <1800000>;
+          regulator-ramp-delay = <12500>;
+          regulator-enable-ramp-delay = <0>;
+          regulator-allowed-modes = <0 1>;
+        };
+        mt6331_vtcxo1_reg: ldo-vtcxo1 {
+          regulator-name = "vtcxo1";
+          regulator-min-microvolt = <2800000>;
+          regulator-max-microvolt = <2800000>;
+          regulator-always-on;
+          regulator-boot-on;
+        };
+        mt6331_vtcxo2_reg: ldo-vtcxo2 {
+          regulator-name = "vtcxo2";
+          regulator-min-microvolt = <2800000>;
+          regulator-max-microvolt = <2800000>;
+          regulator-always-on;
+          regulator-boot-on;
+        };
+        mt6331_avdd32_aud_reg: ldo-avdd32aud {
+          regulator-name = "avdd32_aud";
+          regulator-min-microvolt = <2800000>;
+          regulator-max-microvolt = <3200000>;
+        };
+        mt6331_vauxa32_reg: ldo-vauxa32 {
+          regulator-name = "vauxa32";
+          regulator-min-microvolt = <2800000>;
+          regulator-max-microvolt = <3200000>;
+        };
+        mt6331_vcama_reg: ldo-vcama {
+          regulator-name = "vcama";
+          regulator-min-microvolt = <1500000>;
+          regulator-max-microvolt = <2800000>;
+          regulator-always-on;
+        };
+        mt6331_vio28_reg: ldo-vio28 {
+          regulator-name = "vio28";
+          regulator-min-microvolt = <2800000>;
+          regulator-max-microvolt = <2800000>;
+          regulator-always-on;
+          regulator-boot-on;
+        };
+        mt6331_vcamaf_reg: ldo-vcamaf {
+          regulator-name = "vcam_af";
+          regulator-min-microvolt = <1200000>;
+          regulator-max-microvolt = <3300000>;
+        };
+        mt6331_vmc_reg: ldo-vmc {
+          regulator-name = "vmc";
+          regulator-min-microvolt = <1800000>;
+          regulator-max-microvolt = <3300000>;
+        };
+        mt6331_vmch_reg: ldo-vmch {
+          regulator-name = "vmch";
+          regulator-min-microvolt = <3000000>;
+          regulator-max-microvolt = <3300000>;
+        };
+        mt6331_vemc33_reg: ldo-vemc33 {
+          regulator-name = "vemc33";
+          regulator-min-microvolt = <3300000>;
+          regulator-max-microvolt = <3300000>;
+        };
+        mt6331_vgp1_reg: ldo-vgp1 {
+          regulator-name = "vgp1";
+          regulator-min-microvolt = <1200000>;
+          regulator-max-microvolt = <3300000>;
+        };
+        mt6331_vsim1_reg: ldo-vsim1 {
+          regulator-name = "vsim1";
+          regulator-min-microvolt = <1700000>;
+          regulator-max-microvolt = <3100000>;
+        };
+        mt6331_vsim2_reg: ldo-vsim2 {
+          regulator-name = "vsim2";
+          regulator-min-microvolt = <1700000>;
+          regulator-max-microvolt = <3100000>;
+        };
+        mt6331_vmipi_reg: ldo-vmipi {
+          regulator-name = "vmipi";
+          regulator-min-microvolt = <1200000>;
+          regulator-max-microvolt = <3300000>;
+        };
+        mt6331_vibr_reg: ldo-vibr {
+          regulator-name = "vibr";
+          regulator-min-microvolt = <1200000>;
+          regulator-max-microvolt = <3300000>;
+        };
+        mt6331_vgp4_reg: ldo-vgp4 {
+          regulator-name = "vgp4";
+          regulator-min-microvolt = <1600000>;
+          regulator-max-microvolt = <2200000>;
+        };
+        mt6331_vcamd_reg: ldo-vcamd {
+          regulator-name = "vcamd";
+          regulator-min-microvolt = <900000>;
+          regulator-max-microvolt = <1500000>;
+        };
+        mt6331_vusb10_reg: ldo-vusb10 {
+          regulator-name = "vusb";
+          regulator-min-microvolt = <1000000>;
+          regulator-max-microvolt = <1300000>;
+          regulator-boot-on;
+        };
+        mt6331_vcamio_reg: ldo-vcamio {
+          regulator-name = "vcam_io";
+          regulator-min-microvolt = <1200000>;
+          regulator-max-microvolt = <1800000>;
+        };
+        mt6331_vsram_reg: ldo-vsram {
+          regulator-name = "vsram";
+          regulator-min-microvolt = <1012500>;
+          regulator-max-microvolt = <1012500>;
+          regulator-always-on;
+          regulator-boot-on;
+        };
+        mt6331_vgp2_reg: ldo-vgp2 {
+          regulator-name = "vgp2";
+          regulator-min-microvolt = <1100000>;
+          regulator-max-microvolt = <1500000>;
+          regulator-boot-on;
+        };
+        mt6331_vgp3_reg: ldo-vgp3 {
+          regulator-name = "vgp3";
+          regulator-min-microvolt = <1200000>;
+          regulator-max-microvolt = <1800000>;
+        };
+        mt6331_vrtc_reg: ldo-vrtc {
+          regulator-name = "vrtc";
+          regulator-min-microvolt = <2800000>;
+          regulator-max-microvolt = <2800000>;
+          regulator-always-on;
+        };
+        mt6331_vdig18_reg: ldo-vdig18 {
+          regulator-name = "dvdd18_dig";
+          regulator-min-microvolt = <1800000>;
+          regulator-max-microvolt = <1800000>;
+        };
+      };
+    };
+...
diff --git a/Documentation/devicetree/bindings/regulator/mediatek,mt6332-regulator.yaml b/Documentation/devicetree/bindings/regulator/mediatek,mt6332-regulator.yaml
new file mode 100644
index 000000000000..3218f43e6957
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/mediatek,mt6332-regulator.yaml
@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/mediatek,mt6332-regulator.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MT6332 Regulator from MediaTek Integrated
+
+maintainers:
+  - AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
+
+description: |
+  The MT6332 Companion PMIC provides 6 BUCK and 4 LDO (Low Dropout)
+  regulators and nodes are named according to the regulator type:
+  buck-<name> and ldo-<name>.
+  MT6332 regulators node should be sub node of the MT6397 MFD node.
+
+patternProperties:
+  "^buck-v(dram|dvfs2|pa|rf18a|rf18b|sbst)$":
+    type: object
+    $ref: "regulator.yaml#"
+
+    properties:
+      regulator-name:
+        pattern: "^v(dram|dvfs2|pa|rf18a|rf18b|sbst)$"
+
+    unevaluatedProperties: false
+
+  "^ldo-v(bif28|dig18|sram|usb33)$":
+    type: object
+    $ref: "regulator.yaml#"
+
+    properties:
+      regulator-name:
+        pattern: "^v(bif28|dig18|sram|usb33)$"
+
+    unevaluatedProperties: false
+
+additionalProperties: false
+
+examples:
+  - |
+    pmic {
+      regulators {
+        mt6332_vdram_reg: buck-vdram {
+          regulator-name = "vdram";
+          regulator-min-microvolt = <700000>;
+          regulator-max-microvolt = <1493750>;
+          regulator-ramp-delay = <12500>;
+          regulator-allowed-modes = <0 1>;
+          regulator-always-on;
+        };
+        mt6332_vdvfs2_reg: buck-vdvfs2 {
+          regulator-name = "vdvfs2";
+          regulator-min-microvolt = <700000>;
+          regulator-max-microvolt = <1312500>;
+          regulator-ramp-delay = <12500>;
+          regulator-enable-ramp-delay = <1>;
+          regulator-allowed-modes = <0 1>;
+        };
+        mt6332_vpa_reg: buck-vpa {
+          regulator-name = "vpa";
+          regulator-min-microvolt = <500000>;
+          regulator-max-microvolt = <3400000>;
+        };
+        mt6332_vrf18a_reg: buck-vrf18a {
+          regulator-name = "vrf18a";
+          regulator-min-microvolt = <1050000>;
+          regulator-max-microvolt = <2240625>;
+          regulator-allowed-modes = <0 1>;
+        };
+        mt6332_vrf18b_reg: buck-vrf18b {
+          regulator-name = "vrf18b";
+          regulator-min-microvolt = <1050000>;
+          regulator-max-microvolt = <2240625>;
+          regulator-allowed-modes = <0 1>;
+        };
+        mt6332_vsbst_reg: buck-vsbst {
+          regulator-name = "vsbst";
+          regulator-min-microvolt = <3500000>;
+          regulator-max-microvolt = <7468750>;
+        };
+        mt6332_vauxb32_reg: ldo-vauxb32 {
+          regulator-name = "vauxb32";
+          regulator-min-microvolt = <2800000>;
+          regulator-max-microvolt = <3200000>;
+        };
+        mt6332_vbif28_reg: ldo-vbif28 {
+          regulator-name = "vbif28";
+          regulator-min-microvolt = <2800000>;
+          regulator-max-microvolt = <2800000>;
+        };
+        mt6332_vdig18_reg: ldo-vdig18 {
+          regulator-name = "vdig18";
+          regulator-min-microvolt = <1200000>;
+          regulator-max-microvolt = <1800000>;
+          regulator-always-on;
+        };
+        mt6332_vsram_reg: ldo-vsram {
+          regulator-name = "vauxa32";
+          regulator-min-microvolt = <700000>;
+          regulator-max-microvolt = <1493750>;
+          regulator-always-on;
+        };
+        mt6332_vusb33_reg: ldo-vusb33 {
+          regulator-name = "vusb33";
+          regulator-min-microvolt = <3300000>;
+          regulator-max-microvolt = <3300000>;
+        };
+      };
+    };
+...
diff --git a/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml b/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml
index 37402c370fbb..364b58730be2 100644
--- a/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/mt6315-regulator.yaml
@@ -29,6 +29,7 @@ properties:
       "^vbuck[1-4]$":
         type: object
         $ref: "regulator.yaml#"
+        unevaluatedProperties: false
 
         properties:
           regulator-compatible:
diff --git a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml
index 9a36bee750af..90c3bda31c23 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml
@@ -37,6 +37,8 @@ description: |
       For PM6150, smps1 - smps5, ldo1 - ldo19
       For PM6150L, smps1 - smps8, ldo1 - ldo11, bob
       For PM6350, smps1 - smps5, ldo1 - ldo22
+      For PM660, smps1 - smps6, ldo1 - ldo3, ldo5 - ldo19
+      For PM660L, smps1 - smps3, smps5, ldo1 - ldo8, bob
       For PM7325, smps1 - smps8, ldo1 - ldo19
       For PM8005, smps1 - smps4
       For PM8009, smps1 - smps2, ldo1 - ldo7
@@ -57,6 +59,8 @@ properties:
       - qcom,pm6150-rpmh-regulators
       - qcom,pm6150l-rpmh-regulators
       - qcom,pm6350-rpmh-regulators
+      - qcom,pm660-rpmh-regulators
+      - qcom,pm660l-rpmh-regulators
       - qcom,pm7325-rpmh-regulators
       - qcom,pm8005-rpmh-regulators
       - qcom,pm8009-rpmh-regulators
@@ -99,12 +103,16 @@ properties:
     type: object
     $ref: "regulator.yaml#"
     description: BOB regulator node.
+    dependencies:
+      regulator-allow-set-load: ["regulator-allowed-modes"]
 
 patternProperties:
   "^(smps|ldo|lvs)[0-9]+$":
     type: object
     $ref: "regulator.yaml#"
     description: smps/ldo regulator nodes(s).
+    dependencies:
+      regulator-allow-set-load: ["regulator-allowed-modes"]
 
 required:
   - compatible
@@ -148,6 +156,38 @@ allOf:
       properties:
         compatible:
           enum:
+            - qcom,pm660-rpmh-regulators
+    then:
+      properties:
+        vdd-l1-l6-l7-supply: true
+        vdd-l2-l3-supply: true
+        vdd-l5-supply: true
+        vdd-l8-l9-l10-l11-l12-l13-l14-supply: true
+        vdd-l15-l16-l17-l18-l19-supply: true
+      patternProperties:
+        "^vdd-s[1-6]-supply$": true
+
+  - if:
+      properties:
+        compatible:
+          enum:
+            - qcom,pm660l-rpmh-regulators
+    then:
+      properties:
+        vdd-bob-supply:
+          description: BOB regulator parent supply phandle.
+        vdd-l1-l9-l10-supply: true
+        vdd-l2-supply: true
+        vdd-l3-l5-l7-l8-supply: true
+        vdd-l4-l6-supply: true
+        vdd-s3-s4-supply: true
+      patternProperties:
+        "^vdd-s[125]-supply$": true
+
+  - if:
+      properties:
+        compatible:
+          enum:
             - qcom,pm7325-rpmh-regulators
     then:
       properties:
diff --git a/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml
index c233461cc980..961eed51912c 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml
@@ -24,6 +24,17 @@ description:
 
   For mp5496, s2
 
+  For pm2250, s1, s2, s3, s4, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11,
+  l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22
+
+  For pm6125 s1, s2, s3, s4, s5, s6, s7, s8, l1, l2, l3, l5, l6, l7, l8, l9,
+  l10, l22, l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24
+
+  For pm660, s1, s2, s3, s4, s5, s6, l1, l2, l3, l5, l6, l7, l8, l9, l10, l22,
+  l12, l13, l14, l15, l16, l17, l18, l19
+
+  For pm660l s1, s2, s3, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, bob
+
   For pm8226, s1, s2, s3, s4, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10,
   l11, l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, l25,
   l26, l27, l28, lvs1
@@ -52,11 +63,6 @@ description:
   l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16, l17, l18, l19,
   l20, l21, l22, l23, l24, l25, l26, l27, l28, lvs1, lvs2
 
-  For pm660, s1, s2, s3, s4, s5, s6, l1, l2, l3, l5, l6, l7, l8, l9, l10, l22,
-  l12, l13, l14, l15, l16, l17, l18, l19
-
-  For pm660l s1, s2, s3, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, bob
-
   For pma8084, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, l1, l2, l3,
   l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14, l15, l16, l17, l18, l19,
   l20, l21, l22, l23, l24, l25, l26, l27, lvs1, lvs2, lvs3, lvs4, 5vs1
@@ -68,9 +74,6 @@ description:
   For pms405, s1, s2, s3, s4, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11,
   l12, l13
 
-  For pm2250, s1, s2, s3, s4, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11,
-  l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22
-
 maintainers:
   - Andy Gross <agross@kernel.org>
   - Bjorn Andersson <bjorn.andersson@linaro.org>
@@ -79,6 +82,10 @@ properties:
   compatible:
     enum:
       - qcom,rpm-mp5496-regulators
+      - qcom,rpm-pm2250-regulators
+      - qcom,rpm-pm6125-regulators
+      - qcom,rpm-pm660-regulators
+      - qcom,rpm-pm660l-regulators
       - qcom,rpm-pm8226-regulators
       - qcom,rpm-pm8841-regulators
       - qcom,rpm-pm8909-regulators
@@ -88,13 +95,10 @@ properties:
       - qcom,rpm-pm8953-regulators
       - qcom,rpm-pm8994-regulators
       - qcom,rpm-pm8998-regulators
-      - qcom,rpm-pm660-regulators
-      - qcom,rpm-pm660l-regulators
       - qcom,rpm-pma8084-regulators
       - qcom,rpm-pmi8994-regulators
       - qcom,rpm-pmi8998-regulators
       - qcom,rpm-pms405-regulators
-      - qcom,rpm-pm2250-regulators
 
 patternProperties:
   ".*-supply$":
diff --git a/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml
index faa4af9fd035..bdf34c2de96b 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom,spmi-regulator.yaml
@@ -12,6 +12,7 @@ maintainers:
 properties:
   compatible:
     enum:
+      - qcom,pm6125-regulators
       - qcom,pm660-regulators
       - qcom,pm660l-regulators
       - qcom,pm8004-regulators
@@ -110,6 +111,25 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,pm6125-regulators
+    then:
+      properties:
+        vdd_l1_l7_l17_l18-supply: true
+        vdd_l2_l3_l4-supply: true
+        vdd_l5_l15_l19_l20_l21_l22-supply: true
+        vdd_l6_l8-supply: true
+        vdd_l9_l11-supply: true
+        vdd_l10_l13_l14-supply: true
+        vdd_l12_l16-supply: true
+        vdd_l23_l24-supply: true
+      patternProperties:
+        "^vdd_s[1-8]-supply$": true
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
               - qcom,pm660-regulators
     then:
       properties:
@@ -120,6 +140,7 @@ allOf:
         vdd_l8_l9_l10_l11_l12_l13_l14-supply: true
       patternProperties:
         "^vdd_s[1-6]-supply$": true
+
   - if:
       properties:
         compatible:
@@ -134,6 +155,7 @@ allOf:
         vdd_l4_l6-supply: true
       patternProperties:
         "^vdd_s[1-5]-supply$": true
+
   - if:
       properties:
         compatible:
@@ -143,6 +165,7 @@ allOf:
     then:
       patternProperties:
         "^vdd_s[25]-supply$": true
+
   - if:
       properties:
         compatible:
@@ -152,6 +175,7 @@ allOf:
     then:
       patternProperties:
         "^vdd_s[1-4]-supply$": true
+
   - if:
       properties:
         compatible:
@@ -171,6 +195,7 @@ allOf:
         vdd_lvs1-supply: true
       patternProperties:
         "^vdd_s[1-5]-supply$": true
+
   - if:
       properties:
         compatible:
@@ -180,6 +205,7 @@ allOf:
     then:
       patternProperties:
         "^vdd_s[1-8]-supply$": true
+
   - if:
       properties:
         compatible:
@@ -195,6 +221,7 @@ allOf:
       patternProperties:
         "^vdd_l[27]-supply$": true
         "^vdd_s[1-4]-supply$": true
+
   - if:
       properties:
         compatible:
@@ -223,6 +250,7 @@ allOf:
         vin_5vs-supply: true
       patternProperties:
         "^vdd_s[1-3]-supply$": true
+
   - if:
       properties:
         compatible:
@@ -241,6 +269,7 @@ allOf:
         vdd_l9_l10_l13_l14_l15_l18-supply: true
       patternProperties:
         "^vdd_s[1-6]-supply$": true
+
   - if:
       properties:
         compatible:
@@ -265,6 +294,7 @@ allOf:
         vdd_lvs_1_2-supply: true
       patternProperties:
         "^vdd_s[1-9][0-2]?-supply$": true
+
   - if:
       properties:
         compatible:
@@ -276,6 +306,7 @@ allOf:
         vdd_l1-supply: true
       patternProperties:
         "^vdd_s[1-3]-supply$": true
+
   - if:
       properties:
         compatible:
@@ -291,6 +322,7 @@ allOf:
       patternProperties:
         "^vdd_l[479]-supply$": true
         "^vdd_s[1-5]-supply$": true
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml
index 1ddc1efd19e2..f97b8083678f 100644
--- a/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom-labibb-regulator.yaml
@@ -20,6 +20,7 @@ properties:
 
   lab:
     type: object
+    additionalProperties: false
 
     properties:
       qcom,soft-start-us:
@@ -33,11 +34,19 @@ properties:
         description:
           Short-circuit and over-current interrupts for lab.
 
+      interrupt-names:
+        minItems: 1
+        items:
+          - const: sc-err
+          - const: ocp
+
     required:
       - interrupts
+      - interrupt-names
 
   ibb:
     type: object
+    additionalProperties: false
 
     properties:
       qcom,discharge-resistor-kohms:
@@ -52,8 +61,15 @@ properties:
         description:
           Short-circuit and over-current interrupts for ibb.
 
+      interrupt-names:
+        minItems: 1
+        items:
+          - const: sc-err
+          - const: ocp
+
     required:
       - interrupts
+      - interrupt-names
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/regulator/richtek,rt4801-regulator.yaml b/Documentation/devicetree/bindings/regulator/richtek,rt4801-regulator.yaml
index 091150c4e579..4a8a221bc902 100644
--- a/Documentation/devicetree/bindings/regulator/richtek,rt4801-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/richtek,rt4801-regulator.yaml
@@ -42,6 +42,7 @@ patternProperties:
   "^DSV(P|N)$":
     type: object
     $ref: regulator.yaml#
+    unevaluatedProperties: false
     description:
       Properties for single display bias regulator.
 
diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml
index 7d0adb74a396..d61e8675f067 100644
--- a/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/rohm,bd71815-regulator.yaml
@@ -27,6 +27,7 @@ properties:
     description:
       properties for wled regulator
     $ref: regulator.yaml#
+    unevaluatedProperties: false
 
     properties:
       regulator-name:
diff --git a/Documentation/devicetree/bindings/regulator/ti,tps65219.yaml b/Documentation/devicetree/bindings/regulator/ti,tps65219.yaml
new file mode 100644
index 000000000000..78be79930fda
--- /dev/null
+++ b/Documentation/devicetree/bindings/regulator/ti,tps65219.yaml
@@ -0,0 +1,173 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/regulator/ti,tps65219.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI tps65219 Power Management Integrated Circuit regulators
+
+maintainers:
+  - Jerome Neanne <jerome.neanne@baylibre.com>
+
+description: |
+  Regulator nodes should be named to buck<number> and ldo<number>.
+
+properties:
+  compatible:
+    enum:
+      - ti,tps65219
+
+  reg:
+    maxItems: 1
+
+  system-power-controller:
+    type: boolean
+    description: Optional property that indicates that this device is
+      controlling system power.
+
+  interrupts:
+    description: Short-circuit, over-current, under-voltage for regulators, PB interrupts.
+    maxItems: 1
+
+  interrupt-controller: true
+
+  '#interrupt-cells':
+    description: Specifies the PIN numbers and Flags, as defined in
+      include/dt-bindings/interrupt-controller/irq.h
+    const: 1
+
+  ti,power-button:
+    type: boolean
+    description: |
+      Optional property that sets the EN/PB/VSENSE pin to be a
+      power-button.
+      TPS65219 has a multipurpose pin called EN/PB/VSENSE that can be either
+      1. EN in which case it functions as an enable pin.
+      2. VSENSE which compares the voltages and triggers an automatic
+      on/off request.
+      3. PB in which case it can be configured to trigger an interrupt
+      to the SoC.
+      ti,power-button reflects the last one of those options
+      where the board has a button wired to the pin and triggers
+      an interrupt on pressing it.
+
+patternProperties:
+  "^buck[1-3]-supply$":
+    description: Input supply phandle of one regulator.
+
+  "^ldo[1-4]-supply$":
+    description: Input supply phandle of one regulator.
+
+  regulators:
+    type: object
+    description: |
+      list of regulators provided by this controller
+
+    patternProperties:
+      "^ldo[1-4]$":
+        type: object
+        $ref: regulator.yaml#
+        description:
+          Properties for single LDO regulator.
+
+        unevaluatedProperties: false
+
+      "^buck[1-3]$":
+        type: object
+        $ref: regulator.yaml#
+        description:
+          Properties for single BUCK regulator.
+
+        unevaluatedProperties: false
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - regulators
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        tps65219: pmic@30 {
+            compatible = "ti,tps65219";
+            reg = <0x30>;
+            buck1-supply = <&vcc_3v3_sys>;
+            buck2-supply = <&vcc_3v3_sys>;
+            buck3-supply = <&vcc_3v3_sys>;
+            ldo1-supply = <&vcc_3v3_sys>;
+            ldo2-supply = <&buck2_reg>;
+            ldo3-supply = <&vcc_3v3_sys>;
+            ldo4-supply = <&vcc_3v3_sys>;
+
+            pinctrl-0 = <&pmic_irq_pins_default>;
+
+            interrupt-parent = <&gic500>;
+            interrupts = <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>;
+            ti,power-button;
+
+            regulators {
+                buck1_reg: buck1 {
+                    regulator-name = "VDD_CORE";
+                    regulator-min-microvolt = <750000>;
+                    regulator-max-microvolt = <750000>;
+                    regulator-boot-on;
+                    regulator-always-on;
+                };
+
+                buck2_reg: buck2 {
+                    regulator-name = "VCC1V8";
+                    regulator-min-microvolt = <1800000>;
+                    regulator-max-microvolt = <1800000>;
+                    regulator-boot-on;
+                    regulator-always-on;
+                };
+
+                buck3_reg: buck3 {
+                    regulator-name = "VDD_LPDDR4";
+                    regulator-min-microvolt = <1100000>;
+                    regulator-max-microvolt = <1100000>;
+                    regulator-boot-on;
+                    regulator-always-on;
+                };
+
+                ldo1_reg: ldo1 {
+                    regulator-name = "VDDSHV_SD_IO_PMIC";
+                    regulator-min-microvolt = <33000000>;
+                    regulator-max-microvolt = <33000000>;
+                };
+
+                ldo2_reg: ldo2 {
+                    regulator-name = "VDDAR_CORE";
+                    regulator-min-microvolt = <850000>;
+                    regulator-max-microvolt = <850000>;
+                    regulator-boot-on;
+                    regulator-always-on;
+                };
+
+                ldo3_reg: ldo3 {
+                    regulator-name = "VDDA_1V8";
+                    regulator-min-microvolt = <18000000>;
+                    regulator-max-microvolt = <18000000>;
+                    regulator-boot-on;
+                    regulator-always-on;
+                };
+
+                ldo4_reg: ldo4 {
+                    regulator-name = "VDD_PHY_2V5";
+                    regulator-min-microvolt = <25000000>;
+                    regulator-max-microvolt = <25000000>;
+                    regulator-boot-on;
+                    regulator-always-on;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/serial/samsung_uart.yaml b/Documentation/devicetree/bindings/serial/samsung_uart.yaml
index 901c1e2cea28..41d3b082eb72 100644
--- a/Documentation/devicetree/bindings/serial/samsung_uart.yaml
+++ b/Documentation/devicetree/bindings/serial/samsung_uart.yaml
@@ -17,7 +17,10 @@ description: |+
 
 properties:
   compatible:
-    items:
+    oneOf:
+      - items:
+          - const: samsung,exynosautov9-uart
+          - const: samsung,exynos850-uart
       - enum:
           - apple,s5l-uart
           - axis,artpec8-uart
diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mm-vpu-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mm-vpu-blk-ctrl.yaml
index 26487daa64d9..d71bb20d4907 100644
--- a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mm-vpu-blk-ctrl.yaml
+++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mm-vpu-blk-ctrl.yaml
@@ -27,25 +27,22 @@ properties:
     const: 1
 
   power-domains:
-    minItems: 4
     maxItems: 4
 
   power-domain-names:
-    items:
-      - const: bus
-      - const: g1
-      - const: g2
-      - const: h1
+    maxItems: 4
 
   clocks:
-    minItems: 3
     maxItems: 3
 
   clock-names:
-    items:
-      - const: g1
-      - const: g2
-      - const: h1
+    maxItems: 3
+
+  interconnects:
+    maxItems: 3
+
+  interconnect-names:
+    maxItems: 3
 
 required:
   - compatible
@@ -55,6 +52,97 @@ required:
   - clocks
   - clock-names
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: fsl,imx8mm-vpu-blk-ctrl
+    then:
+      properties:
+        power-domains:
+          items:
+            - description: bus power domain
+            - description: G1 decoder power domain
+            - description: G2 decoder power domain
+            - description: H1 encoder power domain
+
+        power-domain-names:
+          items:
+            - const: bus
+            - const: g1
+            - const: g2
+            - const: h1
+
+        clocks:
+          items:
+            - description: G1 decoder clk
+            - description: G2 decoder clk
+            - description: H1 encoder clk
+
+        clock-names:
+          items:
+            - const: g1
+            - const: g2
+            - const: h1
+
+        interconnects:
+          items:
+            - description: G1 decoder interconnect
+            - description: G2 decoder interconnect
+            - description: H1 encoder power domain
+
+        interconnect-names:
+          items:
+            - const: g1
+            - const: g2
+            - const: h1
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: fsl,imx8mp-vpu-blk-ctrl
+    then:
+      properties:
+        power-domains:
+          items:
+            - description: bus power domain
+            - description: G1 decoder power domain
+            - description: G2 decoder power domain
+            - description: VC8000E encoder power domain
+
+        power-domain-names:
+          items:
+            - const: bus
+            - const: g1
+            - const: g2
+            - const: vc8000e
+
+        clocks:
+          items:
+            - description: G1 decoder clk
+            - description: G2 decoder clk
+            - description: VC8000E encoder clk
+
+        clock-names:
+          items:
+            - const: g1
+            - const: g2
+            - const: vc8000e
+
+        interconnects:
+          items:
+            - description: G1 decoder interconnect
+            - description: G2 decoder interconnect
+            - description: VC8000E encoder interconnect
+
+        interconnect-names:
+          items:
+            - const: g1
+            - const: g2
+            - const: vc8000e
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml
index 563e1d0e327f..1be4ce2a45e8 100644
--- a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml
+++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hdmi-blk-ctrl.yaml
@@ -52,6 +52,15 @@ properties:
       - const: ref_266m
       - const: ref_24m
 
+  interconnects:
+    maxItems: 3
+
+  interconnect-names:
+    items:
+      - const: hrv
+      - const: lcdif-hdmi
+      - const: hdcp
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hsio-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hsio-blk-ctrl.yaml
index c1e29d94f40e..c29181a9745b 100644
--- a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hsio-blk-ctrl.yaml
+++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-hsio-blk-ctrl.yaml
@@ -48,6 +48,16 @@ properties:
       - const: usb
       - const: pcie
 
+  interconnects:
+    maxItems: 4
+
+  interconnect-names:
+    items:
+      - const: noc-pcie
+      - const: usb1
+      - const: usb2
+      - const: pcie
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-media-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-media-blk-ctrl.yaml
index b246d8386ba4..dadb6108e321 100644
--- a/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-media-blk-ctrl.yaml
+++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx8mp-media-blk-ctrl.yaml
@@ -64,6 +64,20 @@ properties:
       - const: isp
       - const: phy
 
+  interconnects:
+    maxItems: 8
+
+  interconnect-names:
+    items:
+      - const: lcdif-rd
+      - const: lcdif-wr
+      - const: isi0
+      - const: isi1
+      - const: isi2
+      - const: isp0
+      - const: isp1
+      - const: dwe
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml
new file mode 100644
index 000000000000..792ebecec22d
--- /dev/null
+++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-media-blk-ctrl.yaml
@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/soc/imx/fsl,imx93-media-blk-ctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NXP i.MX93 Media blk-ctrl
+
+maintainers:
+  - Peng Fan <peng.fan@nxp.com>
+
+description:
+  The i.MX93 MEDIAMIX domain contains control and status registers known
+  as MEDIAMIX Block Control (MEDIAMIX BLK_CTRL). These registers include
+  clocking, reset, and miscellaneous top-level controls for peripherals
+  within the MEDIAMIX domain
+
+properties:
+  compatible:
+    items:
+      - const: fsl,imx93-media-blk-ctrl
+      - const: syscon
+
+  reg:
+    maxItems: 1
+
+  '#power-domain-cells':
+    const: 1
+
+  power-domains:
+    maxItems: 1
+
+  clocks:
+    maxItems: 10
+
+  clock-names:
+    items:
+      - const: apb
+      - const: axi
+      - const: nic
+      - const: disp
+      - const: cam
+      - const: pxp
+      - const: lcdif
+      - const: isi
+      - const: csi
+      - const: dsi
+
+required:
+  - compatible
+  - reg
+  - power-domains
+  - clocks
+  - clock-names
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/imx93-clock.h>
+    #include <dt-bindings/power/fsl,imx93-power.h>
+
+    media_blk_ctrl: system-controller@4ac10000 {
+      compatible = "fsl,imx93-media-blk-ctrl", "syscon";
+      reg = <0x4ac10000 0x10000>;
+      power-domains = <&mediamix>;
+      clocks = <&clk IMX93_CLK_MEDIA_APB>,
+               <&clk IMX93_CLK_MEDIA_AXI>,
+               <&clk IMX93_CLK_NIC_MEDIA_GATE>,
+               <&clk IMX93_CLK_MEDIA_DISP_PIX>,
+               <&clk IMX93_CLK_CAM_PIX>,
+               <&clk IMX93_CLK_PXP_GATE>,
+               <&clk IMX93_CLK_LCDIF_GATE>,
+               <&clk IMX93_CLK_ISI_GATE>,
+               <&clk IMX93_CLK_MIPI_CSI_GATE>,
+               <&clk IMX93_CLK_MIPI_DSI_GATE>;
+               clock-names = "apb", "axi", "nic", "disp", "cam",
+                             "pxp", "lcdif", "isi", "csi", "dsi";
+      #power-domain-cells = <1>;
+    };
diff --git a/Documentation/devicetree/bindings/soc/imx/fsl,imx93-src.yaml b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-src.yaml
new file mode 100644
index 000000000000..c1cc69b51981
--- /dev/null
+++ b/Documentation/devicetree/bindings/soc/imx/fsl,imx93-src.yaml
@@ -0,0 +1,96 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/soc/imx/fsl,imx93-src.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NXP i.MX93 System Reset Controller
+
+maintainers:
+  - Peng Fan <peng.fan@nxp.com>
+
+description: |
+  The System Reset Controller (SRC) is responsible for the generation of
+  all the system reset signals and boot argument latching.
+
+  Its main functions are as follows,
+  - Deals with all global system reset sources from other modules,
+    and generates global system reset.
+  - Responsible for power gating of MIXs (Slices) and their memory
+    low power control.
+
+properties:
+  compatible:
+    items:
+      - const: fsl,imx93-src
+      - const: syscon
+
+  reg:
+    maxItems: 1
+
+  ranges: true
+
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 1
+
+patternProperties:
+  "power-domain@[0-9a-f]+$":
+
+    type: object
+    properties:
+      compatible:
+        items:
+          - const: fsl,imx93-src-slice
+
+      '#power-domain-cells':
+        const: 0
+
+      reg:
+        items:
+          - description: mix slice register region
+          - description: mem slice register region
+
+      clocks:
+        description: |
+          A number of phandles to clocks that need to be enabled
+          during domain power-up sequencing to ensure reset
+          propagation into devices located inside this power domain.
+        minItems: 1
+        maxItems: 5
+
+    required:
+      - compatible
+      - '#power-domain-cells'
+      - reg
+
+required:
+  - compatible
+  - reg
+  - ranges
+  - '#address-cells'
+  - '#size-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/imx93-clock.h>
+
+    system-controller@44460000 {
+        compatible = "fsl,imx93-src", "syscon";
+        reg = <0x44460000 0x10000>;
+        #address-cells = <1>;
+        #size-cells = <1>;
+        ranges;
+
+        mediamix: power-domain@0 {
+            compatible = "fsl,imx93-src-slice";
+            reg = <0x44462400 0x400>, <0x44465800 0x400>;
+            #power-domain-cells = <0>;
+            clocks = <&clk IMX93_CLK_MEDIA_AXI>,
+                     <&clk IMX93_CLK_MEDIA_APB>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml b/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml
index 627dcc3e8b32..9241e5fc7cff 100644
--- a/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml
+++ b/Documentation/devicetree/bindings/soc/mediatek/mediatek,mutex.yaml
@@ -26,10 +26,12 @@ properties:
     enum:
       - mediatek,mt2701-disp-mutex
       - mediatek,mt2712-disp-mutex
+      - mediatek,mt6795-disp-mutex
       - mediatek,mt8167-disp-mutex
       - mediatek,mt8173-disp-mutex
       - mediatek,mt8183-disp-mutex
       - mediatek,mt8186-disp-mutex
+      - mediatek,mt8186-mdp3-mutex
       - mediatek,mt8192-disp-mutex
       - mediatek,mt8195-disp-mutex
 
diff --git a/Documentation/devicetree/bindings/soc/mediatek/pwrap.txt b/Documentation/devicetree/bindings/soc/mediatek/pwrap.txt
index 0581dbda4828..d24e2bc444be 100644
--- a/Documentation/devicetree/bindings/soc/mediatek/pwrap.txt
+++ b/Documentation/devicetree/bindings/soc/mediatek/pwrap.txt
@@ -28,6 +28,7 @@ Required properties in pwrap device node.
 	"mediatek,mt8173-pwrap" for MT8173 SoCs
 	"mediatek,mt8183-pwrap" for MT8183 SoCs
 	"mediatek,mt8186-pwrap" for MT8186 SoCs
+	"mediatek,mt8188-pwrap", "mediatek,mt8195-pwrap" for MT8188 SoCs
 	"mediatek,mt8195-pwrap" for MT8195 SoCs
 	"mediatek,mt8516-pwrap" for MT8516 SoCs
 - interrupts: IRQ for pwrap in SOC
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml
index 028c5d105adb..f47491aab3b1 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml
@@ -20,6 +20,9 @@ properties:
       - qcom,apr-v2
       - qcom,gpr
 
+  power-domains:
+    maxItems: 1
+
   qcom,apr-domain:
     $ref: /schemas/types.yaml#/definitions/uint32
     enum: [1, 2, 3, 4, 5, 6, 7]
@@ -52,6 +55,26 @@ properties:
         2 = Audio DSP Domain
         3 = Application Processor Domain
 
+  qcom,glink-channels:
+    $ref: /schemas/types.yaml#/definitions/string-array
+    description: Channel name used for the communication
+    items:
+      - const: apr_audio_svc
+
+  qcom,intents:
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    description:
+      List of (size, amount) pairs describing what intents should be
+      preallocated for this virtual channel. This can be used to tweak the
+      default intents available for the channel to meet expectations of the
+      remote.
+
+  qcom,smd-channels:
+    $ref: /schemas/types.yaml#/definitions/string-array
+    description: Channel name used for the communication
+    items:
+      - const: apr_audio_svc
+
   '#address-cells':
     const: 1
 
@@ -97,6 +120,26 @@ patternProperties:
             3 = AMDB Service.
             4 = Voice processing manager.
 
+      clock-controller:
+        $ref: /schemas/sound/qcom,q6dsp-lpass-clocks.yaml#
+        description: Qualcomm DSP LPASS clock controller
+        unevaluatedProperties: false
+
+      dais:
+        type: object
+        oneOf:
+          - $ref: /schemas/sound/qcom,q6apm-dai.yaml#
+          - $ref: /schemas/sound/qcom,q6dsp-lpass-ports.yaml#
+          - $ref: /schemas/sound/qcom,q6asm-dais.yaml#
+        unevaluatedProperties: false
+        description: Qualcomm DSP audio ports
+
+      routing:
+        type: object
+        $ref: /schemas/sound/qcom,q6adm-routing.yaml#
+        unevaluatedProperties: false
+        description: Qualcomm DSP LPASS audio routing
+
       qcom,protection-domain:
         $ref: /schemas/types.yaml#/definitions/string-array
         description: protection domain service name and path for apr service
@@ -107,17 +150,44 @@ patternProperties:
           "tms/servreg", "msm/modem/wlan_pd".
           "tms/servreg", "msm/slpi/sensor_pd".
 
-      '#address-cells':
-        const: 1
+    allOf:
+      - if:
+          properties:
+            compatible:
+              enum:
+                - qcom,q6afe
+        then:
+          properties:
+            dais:
+              properties:
+                compatible:
+                  const: qcom,q6afe-dais
 
-      '#size-cells':
-        const: 0
+      - if:
+          properties:
+            compatible:
+              enum:
+                - qcom,q6apm
+        then:
+          properties:
+            dais:
+              properties:
+                compatible:
+                  enum:
+                    - qcom,q6apm-dais
+                    - qcom,q6apm-lpass-dais
 
-    patternProperties:
-      "^.*@[0-9a-f]+$":
-        type: object
-        description:
-          Service based devices like clock controllers or digital audio interfaces.
+      - if:
+          properties:
+            compatible:
+              enum:
+                - qcom,q6asm
+        then:
+          properties:
+            dais:
+              properties:
+                compatible:
+                  const: qcom,q6asm-dais
 
     additionalProperties: false
 
@@ -125,6 +195,30 @@ required:
   - compatible
   - qcom,domain
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          enum:
+            - qcom,gpr
+    then:
+      properties:
+        power-domains: false
+
+  - if:
+      required:
+        - qcom,glink-channels
+    then:
+      properties:
+        qcom,smd-channels: false
+
+  - if:
+      required:
+        - qcom,smd-channels
+    then:
+      properties:
+        qcom,glink-channels: false
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml
index 473adca4e973..48eda4d0d391 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom-stats.yaml
@@ -20,6 +20,7 @@ properties:
   compatible:
     enum:
       - qcom,rpmh-stats
+      - qcom,sdm845-rpmh-stats
       - qcom,rpm-stats
       # For older RPM firmware versions with fixed offset for the sleep stats
       - qcom,apq8084-rpm-stats
diff --git a/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml b/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml
index ce2875c89329..398663d21ab1 100644
--- a/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml
+++ b/Documentation/devicetree/bindings/soc/renesas/renesas,rzg2l-sysc.yaml
@@ -20,7 +20,7 @@ description:
 properties:
   compatible:
     enum:
-      - renesas,r9a07g043-sysc # RZ/G2UL
+      - renesas,r9a07g043-sysc # RZ/G2UL and RZ/Five
       - renesas,r9a07g044-sysc # RZ/G2{L,LC}
       - renesas,r9a07g054-sysc # RZ/V2L
 
@@ -44,8 +44,6 @@ properties:
 required:
   - compatible
   - reg
-  - interrupts
-  - interrupt-names
 
 additionalProperties: false
 
diff --git a/Documentation/devicetree/bindings/soc/rockchip/grf.yaml b/Documentation/devicetree/bindings/soc/rockchip/grf.yaml
index 75a2b8bb25fb..2ed8cca79b59 100644
--- a/Documentation/devicetree/bindings/soc/rockchip/grf.yaml
+++ b/Documentation/devicetree/bindings/soc/rockchip/grf.yaml
@@ -16,9 +16,12 @@ properties:
           - enum:
               - rockchip,rk3288-sgrf
               - rockchip,rk3566-pipe-grf
+              - rockchip,rk3568-pcie3-phy-grf
               - rockchip,rk3568-pipe-grf
               - rockchip,rk3568-pipe-phy-grf
               - rockchip,rk3568-usb2phy-grf
+              - rockchip,rk3588-pcie3-phy-grf
+              - rockchip,rk3588-pcie3-pipe-grf
               - rockchip,rv1108-usbgrf
           - const: syscon
       - items:
@@ -28,6 +31,7 @@ properties:
               - rockchip,px30-usb2phy-grf
               - rockchip,rk3036-grf
               - rockchip,rk3066-grf
+              - rockchip,rk3128-grf
               - rockchip,rk3188-grf
               - rockchip,rk3228-grf
               - rockchip,rk3288-grf
@@ -45,6 +49,8 @@ properties:
               - rockchip,rk3568-pmugrf
               - rockchip,rv1108-grf
               - rockchip,rv1108-pmugrf
+              - rockchip,rv1126-grf
+              - rockchip,rv1126-pmugrf
           - const: syscon
           - const: simple-mfd
 
@@ -178,6 +184,7 @@ allOf:
           contains:
             enum:
               - rockchip,px30-usb2phy-grf
+              - rockchip,rk3128-grf
               - rockchip,rk3228-grf
               - rockchip,rk3308-usb2phy-grf
               - rockchip,rk3328-usb2phy-grf
diff --git a/Documentation/devicetree/bindings/sound/adi,max98396.yaml b/Documentation/devicetree/bindings/sound/adi,max98396.yaml
index 8d2ef991db40..7985b1410017 100644
--- a/Documentation/devicetree/bindings/sound/adi,max98396.yaml
+++ b/Documentation/devicetree/bindings/sound/adi,max98396.yaml
@@ -78,6 +78,40 @@ properties:
       interleaved on a single output channel.
     type: boolean
 
+  adi,dmon-stuck-enable:
+    description:
+      Enables the "data monitor stuck" feature. Once the data monitor is
+      enabled, it actively monitors the selected input data (from DIN) to the
+      speaker amplifier. Once a data error is detected, the data monitor
+      automatically places the device into software shutdown.
+    type: boolean
+
+  adi,dmon-stuck-threshold-bits:
+    description:
+      Sets the threshold for the "data monitor stuck" feature, in bits.
+    enum: [9, 11, 13, 15]
+    default: 15
+
+  adi,dmon-magnitude-enable:
+    description:
+      Enables the "data monitor magnitude" feature. Once the data monitor is
+      enabled, it actively monitors the selected input data (from DIN) to the
+      speaker amplifier. Once a data error is detected, the data monitor
+      automatically places the device into software shutdown.
+    type: boolean
+
+  adi,dmon-magnitude-threshold-bits:
+    description:
+      Sets the threshold for the "data monitor magnitude" feature, in bits.
+    enum: [2, 3, 4, 5]
+    default: 5
+
+  adi,dmon-duration-ms:
+    description:
+      Sets the duration for the "data monitor" feature, in milliseconds.
+    enum: [64, 256, 1024, 4096]
+    default: 64
+
   reset-gpios:
     maxItems: 1
 
diff --git a/Documentation/devicetree/bindings/sound/allwinner,sun50i-h6-dmic.yaml b/Documentation/devicetree/bindings/sound/allwinner,sun50i-h6-dmic.yaml
new file mode 100644
index 000000000000..2f12cabe4c71
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/allwinner,sun50i-h6-dmic.yaml
@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: (GPL-2.0+ OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/allwinner,sun50i-h6-dmic.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Allwinner H6 DMIC
+
+maintainers:
+  - Ban Tao <fengzheng923@gmail.com>
+
+properties:
+  compatible:
+    const: allwinner,sun50i-h6-dmic
+
+  "#sound-dai-cells":
+    const: 0
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: Bus Clock
+      - description: Module Clock
+
+  clock-names:
+    items:
+      - const: bus
+      - const: mod
+
+  dmas:
+    items:
+      - description: RX DMA Channel
+
+  dma-names:
+    items:
+      - const: rx
+
+  resets:
+    maxItems: 1
+
+required:
+  - "#sound-dai-cells"
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+  - dmas
+  - dma-names
+  - resets
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    #include <dt-bindings/clock/sun50i-h6-ccu.h>
+    #include <dt-bindings/reset/sun50i-h6-ccu.h>
+
+    dmic: dmic@5095000 {
+      #sound-dai-cells = <0>;
+      compatible = "allwinner,sun50i-h6-dmic";
+      reg = <0x05095000 0x400>;
+      interrupts = <GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
+      clocks = <&ccu CLK_BUS_DMIC>, <&ccu CLK_DMIC>;
+      clock-names = "bus", "mod";
+      dmas = <&dma 7>;
+      dma-names = "rx";
+      resets = <&ccu RST_BUS_DMIC>;
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml b/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml
index b4b35edcb493..5b8d59245f82 100644
--- a/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml
+++ b/Documentation/devicetree/bindings/sound/amlogic,gx-sound-card.yaml
@@ -40,6 +40,7 @@ properties:
 patternProperties:
   "^dai-link-[0-9]+$":
     type: object
+    additionalProperties: false
     description: |-
       dai-link child nodes:
         Container for dai-link level properties and the CODEC sub-nodes.
@@ -63,6 +64,7 @@ patternProperties:
     patternProperties:
       "^codec-[0-9]+$":
         type: object
+        additionalProperties: false
         description: |-
           Codecs:
           dai-link representing backend links should have at least one subnode.
diff --git a/Documentation/devicetree/bindings/sound/apple,mca.yaml b/Documentation/devicetree/bindings/sound/apple,mca.yaml
new file mode 100644
index 000000000000..d5dc92b5b654
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/apple,mca.yaml
@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/apple,mca.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Apple MCA I2S transceiver
+
+description: |
+  MCA is an I2S transceiver peripheral found on M1 and other Apple chips. It is
+  composed of a number of identical clusters which can operate independently
+  or in an interlinked fashion. Up to 6 clusters have been seen on an MCA.
+
+maintainers:
+  - Martin Povišer <povik+lin@cutebit.org>
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - apple,t6000-mca
+          - apple,t8103-mca
+      - const: apple,mca
+
+  reg:
+    items:
+      - description: Register region of the MCA clusters proper
+      - description: Register region of the DMA glue and its FIFOs
+
+  interrupts:
+    minItems: 4
+    maxItems: 6
+    description:
+      One interrupt per each cluster
+
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 0
+
+  dmas:
+    minItems: 16
+    maxItems: 24
+    description:
+      DMA channels corresponding to the SERDES units in the peripheral. They are
+      listed in groups of four per cluster, and within the group they are given
+      as associated to the TXA, RXA, TXB, RXB units.
+
+  dma-names:
+    minItems: 16
+    items:
+      - const: tx0a
+      - const: rx0a
+      - const: tx0b
+      - const: rx0b
+      - const: tx1a
+      - const: rx1a
+      - const: tx1b
+      - const: rx1b
+      - const: tx2a
+      - const: rx2a
+      - const: tx2b
+      - const: rx2b
+      - const: tx3a
+      - const: rx3a
+      - const: tx3b
+      - const: rx3b
+      - const: tx4a
+      - const: rx4a
+      - const: tx4b
+      - const: rx4b
+      - const: tx5a
+      - const: rx5a
+      - const: tx5b
+      - const: rx5b
+    description: |
+      Names for the DMA channels: 'tx'/'rx', then cluster number, then 'a'/'b'
+      based on the associated SERDES unit.
+
+  clocks:
+    minItems: 4
+    maxItems: 6
+    description:
+      Clusters' input reference clock.
+
+  resets:
+    maxItems: 1
+
+  power-domains:
+    minItems: 5
+    maxItems: 7
+    description:
+      First a general power domain for register access, then the power
+      domains of individual clusters for their operation.
+
+  '#sound-dai-cells':
+    const: 1
+
+required:
+  - compatible
+  - reg
+  - dmas
+  - dma-names
+  - clocks
+  - power-domains
+  - '#sound-dai-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    mca: i2s@9b600000 {
+      compatible = "apple,t6000-mca", "apple,mca";
+      reg = <0x9b600000 0x10000>,
+            <0x9b200000 0x20000>;
+
+      clocks = <&nco 0>, <&nco 1>, <&nco 2>, <&nco 3>;
+      power-domains = <&ps_audio_p>, <&ps_mca0>, <&ps_mca1>,
+                      <&ps_mca2>, <&ps_mca3>;
+      dmas = <&admac 0>, <&admac 1>, <&admac 2>, <&admac 3>,
+             <&admac 4>, <&admac 5>, <&admac 6>, <&admac 7>,
+             <&admac 8>, <&admac 9>, <&admac 10>, <&admac 11>,
+             <&admac 12>, <&admac 13>, <&admac 14>, <&admac 15>;
+      dma-names = "tx0a", "rx0a", "tx0b", "rx0b",
+                  "tx1a", "rx1a", "tx1b", "rx1b",
+                  "tx2a", "rx2a", "tx2b", "rx2b",
+                  "tx3a", "rx3a", "tx3b", "rx3b";
+
+      #sound-dai-cells = <1>;
+    };
diff --git a/Documentation/devicetree/bindings/sound/audio-graph-port.yaml b/Documentation/devicetree/bindings/sound/audio-graph-port.yaml
index 5c368674d11a..64654ceef208 100644
--- a/Documentation/devicetree/bindings/sound/audio-graph-port.yaml
+++ b/Documentation/devicetree/bindings/sound/audio-graph-port.yaml
@@ -19,14 +19,17 @@ properties:
     description: "device name prefix"
     $ref: /schemas/types.yaml#/definitions/string
   convert-rate:
-    description: CPU to Codec rate convert.
-    $ref: /schemas/types.yaml#/definitions/uint32
+    $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-rate"
   convert-channels:
-    description: CPU to Codec rate channels.
-    $ref: /schemas/types.yaml#/definitions/uint32
+    $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-channels"
+  convert-sample-format:
+    $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-format"
+
 patternProperties:
   "^endpoint(@[0-9a-f]+)?":
     $ref: /schemas/graph.yaml#/$defs/endpoint-base
+    unevaluatedProperties: false
+
     properties:
       mclk-fs:
         description: |
@@ -65,12 +68,18 @@ patternProperties:
             - msb
             - lsb
       convert-rate:
-        description: CPU to Codec rate convert.
-        $ref: /schemas/types.yaml#/definitions/uint32
+        $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-rate"
       convert-channels:
-        description: CPU to Codec rate channels.
-        $ref: /schemas/types.yaml#/definitions/uint32
+        $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-channels"
+      convert-sample-format:
+        $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-format"
 
+      dai-tdm-slot-num:
+        description: Number of slots in use.
+        $ref: /schemas/types.yaml#/definitions/uint32
+      dai-tdm-slot-width:
+        description: Width in bits for each slot.
+        $ref: /schemas/types.yaml#/definitions/uint32
       dai-tdm-slot-width-map:
         description: Mapping of sample widths to slot widths. For hardware
           that cannot support a fixed slot width or a slot width always
diff --git a/Documentation/devicetree/bindings/sound/audio-graph.yaml b/Documentation/devicetree/bindings/sound/audio-graph.yaml
index 4b46794e5153..aaa99c2deda0 100644
--- a/Documentation/devicetree/bindings/sound/audio-graph.yaml
+++ b/Documentation/devicetree/bindings/sound/audio-graph.yaml
@@ -27,11 +27,12 @@ properties:
     description: User specified audio sound widgets.
     $ref: /schemas/types.yaml#/definitions/non-unique-string-array
   convert-rate:
-    description: CPU to Codec rate convert.
-    $ref: /schemas/types.yaml#/definitions/uint32
+    $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-rate"
   convert-channels:
-    description: CPU to Codec rate channels.
-    $ref: /schemas/types.yaml#/definitions/uint32
+    $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-channels"
+  convert-sample-format:
+    $ref: "/schemas/sound/dai-params.yaml#/$defs/dai-sample-format"
+
   pa-gpios:
     maxItems: 1
   hp-det-gpio:
diff --git a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml
index 31800f70e9d9..7356084a2ca2 100644
--- a/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml
+++ b/Documentation/devicetree/bindings/sound/cirrus,cs42l42.yaml
@@ -19,6 +19,7 @@ properties:
   compatible:
     enum:
       - cirrus,cs42l42
+      - cirrus,cs42l83
 
   reg:
     description:
diff --git a/Documentation/devicetree/bindings/sound/dai-params.yaml b/Documentation/devicetree/bindings/sound/dai-params.yaml
new file mode 100644
index 000000000000..f5fb71f9b603
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/dai-params.yaml
@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/dai-params.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Digital Audio Interface (DAI) Stream Parameters
+
+maintainers:
+  - Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
+
+select: false
+
+$defs:
+
+  dai-channels:
+    description: Number of audio channels used by DAI
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 1
+    maximum: 32
+
+  dai-sample-format:
+    description: Audio sample format used by DAI
+    $ref: /schemas/types.yaml#/definitions/string
+    enum:
+      - s8
+      - s16_le
+      - s24_le
+      - s24_3le
+      - s32_le
+
+  dai-sample-rate:
+    description: Audio sample rate used by DAI
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 8000
+    maximum: 192000
+
+properties: {}
+
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/sound/everest,es8326.yaml b/Documentation/devicetree/bindings/sound/everest,es8326.yaml
new file mode 100755
index 000000000000..07781408e788
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/everest,es8326.yaml
@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/everest,es8326.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Everest ES8326 audio CODEC
+
+maintainers:
+  - David Yang <yangxiaohua@everest-semi.com>
+
+properties:
+  compatible:
+    const: everest,es8326
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: clock for master clock (MCLK)
+
+  clock-names:
+    items:
+      - const: mclk
+
+  "#sound-dai-cells":
+    const: 0
+
+  everest,jack-pol:
+    $ref: /schemas/types.yaml#/definitions/uint8
+    description: |
+      just the value of reg 57. Bit(3) decides whether the jack polarity is inverted.
+      Bit(2) decides whether the button on the headset is inverted.
+      Bit(1)/(0) decides the mic properity to be OMTP/CTIA or auto.
+    minimum: 0x00
+    maximum: 0x0f
+    default: 0x0f
+
+  everest,mic1-src:
+    $ref: /schemas/types.yaml#/definitions/uint8
+    description:
+      the value of reg 2A when headset plugged.
+    minimum: 0x00
+    maximum: 0x77
+    default: 0x22
+
+  everest,mic2-src:
+    $ref: /schemas/types.yaml#/definitions/uint8
+    description:
+      the value of reg 2A when headset unplugged.
+    minimum: 0x00
+    maximum: 0x77
+    default: 0x44
+
+  everest,jack-detect-inverted:
+    $ref: /schemas/types.yaml#/definitions/flag
+    description:
+      Defined to invert the jack detection.
+
+  everest,interrupt-src:
+    $ref: /schemas/types.yaml#/definitions/uint8
+    description: |
+      value of reg 0x58, Defines the interrupt source.
+      Bit(2) 1 means button press triggers irq, 0 means not.
+      Bit(3) 1 means PIN9 is the irq source for jack detection. When set to 0,
+      bias change on PIN9 do not triggers irq.
+      Bit(4) 1 means PIN27 is the irq source for jack detection.
+      Bit(5) 1 means PIN9 is the irq source after MIC detect.
+      Bit(6) 1 means PIN27 is the irq source after MIC detect.
+    minimum: 0
+    maximum: 0x3c
+    default: 0x08
+
+  everest,interrupt-clk:
+    $ref: /schemas/types.yaml#/definitions/uint8
+    description: |
+     value of reg 0x59, Defines the interrupt output behavior.
+     Bit(0-3) 0 means irq pulse equals 512*internal clock
+       1 means irq pulse equals 1024*internal clock
+       2 means ...
+       7 means irq pulse equals 65536*internal clock
+       8 means irq mutes PA
+       9 means irq mutes PA and DAC output
+     Bit(4) 1 means we invert the interrupt output.
+     Bit(6) 1 means the chip do not detect jack type after button released.
+       0 means the chip detect jack type again after button released.
+    minimum: 0
+    maximum: 0x7f
+    default: 0x45
+
+required:
+  - compatible
+  - reg
+  - "#sound-dai-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+      #address-cells = <1>;
+      #size-cells = <0>;
+      es8326: codec@19 {
+        compatible = "everest,es8326";
+        reg = <0x19>;
+        clocks = <&clks 10>;
+        clock-names = "mclk";
+        #sound-dai-cells = <0>;
+        everest,mic1-src = [22];
+        everest,mic2-src = [44];
+        everest,jack-pol = [0e];
+        everest,interrupt-src = [08];
+        everest,interrupt-clk = [45];
+      };
+    };
diff --git a/Documentation/devicetree/bindings/sound/fsl,sai.yaml b/Documentation/devicetree/bindings/sound/fsl,sai.yaml
new file mode 100644
index 000000000000..70c4111d59c7
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/fsl,sai.yaml
@@ -0,0 +1,216 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/fsl,sai.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Freescale Synchronous Audio Interface (SAI).
+
+maintainers:
+  - Shengjiu Wang <shengjiu.wang@nxp.com>
+
+description: |
+  The SAI is based on I2S module that used communicating with audio codecs,
+  which provides a synchronous audio interface that supports fullduplex
+  serial interfaces with frame synchronization such as I2S, AC97, TDM, and
+  codec/DSP interfaces.
+
+properties:
+  compatible:
+    oneOf:
+      - enum:
+          - fsl,vf610-sai
+          - fsl,imx6sx-sai
+          - fsl,imx6ul-sai
+          - fsl,imx7ulp-sai
+          - fsl,imx8mq-sai
+          - fsl,imx8qm-sai
+          - fsl,imx8ulp-sai
+      - items:
+          - enum:
+              - fsl,imx8mm-sai
+              - fsl,imx8mn-sai
+              - fsl,imx8mp-sai
+          - const: fsl,imx8mq-sai
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    items:
+      - description: receive and transmit interrupt
+
+  dmas:
+    maxItems: 2
+
+  dma-names:
+    maxItems: 2
+
+  clocks:
+    items:
+      - description: The ipg clock for register access
+      - description: master clock source 0 (obsoleted)
+      - description: master clock source 1
+      - description: master clock source 2
+      - description: master clock source 3
+      - description: PLL clock source for 8kHz series
+      - description: PLL clock source for 11kHz series
+    minItems: 4
+
+  clock-names:
+    oneOf:
+      - items:
+          - const: bus
+          - const: mclk0
+          - const: mclk1
+          - const: mclk2
+          - const: mclk3
+          - const: pll8k
+          - const: pll11k
+        minItems: 4
+      - items:
+          - const: bus
+          - const: mclk1
+          - const: mclk2
+          - const: mclk3
+          - const: pll8k
+          - const: pll11k
+        minItems: 4
+
+  lsb-first:
+    description: |
+      Configures whether the LSB or the MSB is transmitted
+      first for the fifo data. If this property is absent,
+      the MSB is transmitted first as default, or the LSB
+      is transmitted first.
+    type: boolean
+
+  big-endian:
+    description: |
+      required if all the SAI registers are big-endian rather than little-endian.
+    type: boolean
+
+  fsl,sai-synchronous-rx:
+    description: |
+      SAI will work in the synchronous mode (sync Tx with Rx) which means
+      both the transmitter and the receiver will send and receive data by
+      following receiver's bit clocks and frame sync clocks.
+    type: boolean
+
+  fsl,sai-asynchronous:
+    description: |
+      SAI will work in the asynchronous mode, which means both transmitter
+      and receiver will send and receive data by following their own bit clocks
+      and frame sync clocks separately.
+      If both fsl,sai-asynchronous and fsl,sai-synchronous-rx are absent, the
+      default synchronous mode (sync Rx with Tx) will be used, which means both
+      transmitter and receiver will send and receive data by following clocks
+      of transmitter.
+    type: boolean
+
+  fsl,dataline:
+    $ref: /schemas/types.yaml#/definitions/uint32-matrix
+    description: |
+      Configure the dataline. It has 3 value for each configuration
+    maxItems: 16
+    items:
+      items:
+        - description: format Default(0), I2S(1) or PDM(2)
+          enum: [0, 1, 2]
+        - description: dataline mask for 'rx'
+        - description: dataline mask for 'tx'
+
+  fsl,sai-mclk-direction-output:
+    description: SAI will output the SAI MCLK clock.
+    type: boolean
+
+  fsl,shared-interrupt:
+    description: Interrupt is shared with other modules.
+    type: boolean
+
+  "#sound-dai-cells":
+    const: 0
+    description: optional, some dts node didn't add it.
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: fsl,vf610-sai
+    then:
+      properties:
+        dmas:
+          items:
+            - description: DMA controller phandle and request line for TX
+            - description: DMA controller phandle and request line for RX
+        dma-names:
+          items:
+            - const: tx
+            - const: rx
+    else:
+      properties:
+        dmas:
+          items:
+            - description: DMA controller phandle and request line for RX
+            - description: DMA controller phandle and request line for TX
+        dma-names:
+          items:
+            - const: rx
+            - const: tx
+  - if:
+      required:
+        - fsl,sai-asynchronous
+    then:
+      properties:
+        fsl,sai-synchronous-rx: false
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - dmas
+  - dma-names
+  - clocks
+  - clock-names
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/clock/vf610-clock.h>
+    sai2: sai@40031000 {
+        compatible = "fsl,vf610-sai";
+        reg = <0x40031000 0x1000>;
+        interrupts = <86 IRQ_TYPE_LEVEL_HIGH>;
+        pinctrl-names = "default";
+        pinctrl-0 = <&pinctrl_sai2_1>;
+        clocks = <&clks VF610_CLK_PLATFORM_BUS>,
+                 <&clks VF610_CLK_SAI2>,
+                 <&clks 0>, <&clks 0>;
+        clock-names = "bus", "mclk1", "mclk2", "mclk3";
+        dma-names = "tx", "rx";
+        dmas = <&edma0 0 21>,
+               <&edma0 0 20>;
+        big-endian;
+        lsb-first;
+    };
+
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/clock/imx8mm-clock.h>
+    sai1: sai@30010000 {
+        compatible = "fsl,imx8mm-sai", "fsl,imx8mq-sai";
+        reg = <0x30010000 0x10000>;
+        interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
+        clocks = <&clk IMX8MM_CLK_SAI1_IPG>,
+                 <&clk IMX8MM_CLK_DUMMY>,
+                 <&clk IMX8MM_CLK_SAI1_ROOT>,
+                 <&clk IMX8MM_CLK_DUMMY>, <&clk IMX8MM_CLK_DUMMY>;
+        clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3";
+        dmas = <&sdma2 0 2 0>, <&sdma2 1 2 0>;
+        dma-names = "rx", "tx";
+        fsl,dataline = <1 0xff 0xff 2 0xff 0x11>;
+        #sound-dai-cells = <0>;
+    };
diff --git a/Documentation/devicetree/bindings/sound/fsl-sai.txt b/Documentation/devicetree/bindings/sound/fsl-sai.txt
deleted file mode 100644
index fbdefc3fade7..000000000000
--- a/Documentation/devicetree/bindings/sound/fsl-sai.txt
+++ /dev/null
@@ -1,95 +0,0 @@
-Freescale Synchronous Audio Interface (SAI).
-
-The SAI is based on I2S module that used communicating with audio codecs,
-which provides a synchronous audio interface that supports fullduplex
-serial interfaces with frame synchronization such as I2S, AC97, TDM, and
-codec/DSP interfaces.
-
-Required properties:
-
-  - compatible		: Compatible list, contains "fsl,vf610-sai",
-			  "fsl,imx6sx-sai", "fsl,imx6ul-sai",
-			  "fsl,imx7ulp-sai", "fsl,imx8mq-sai",
-			  "fsl,imx8qm-sai", "fsl,imx8mm-sai",
-			  "fsl,imx8mn-sai", "fsl,imx8mp-sai", or
-			  "fsl,imx8ulp-sai".
-
-  - reg			: Offset and length of the register set for the device.
-
-  - clocks		: Must contain an entry for each entry in clock-names.
-
-  - clock-names		: Must include the "bus" for register access and
-			  "mclk1", "mclk2", "mclk3" for bit clock and frame
-			  clock providing.
-                          "pll8k", "pll11k" are optional, they are the clock
-                          source for root clock, one is for 8kHz series rates
-                          another one is for 11kHz series rates.
-  - dmas		: Generic dma devicetree binding as described in
-			  Documentation/devicetree/bindings/dma/dma.txt.
-
-  - dma-names		: Two dmas have to be defined, "tx" and "rx".
-
-  - pinctrl-names	: Must contain a "default" entry.
-
-  - pinctrl-NNN		: One property must exist for each entry in
-			  pinctrl-names. See ../pinctrl/pinctrl-bindings.txt
-			  for details of the property values.
-
-  - lsb-first		: Configures whether the LSB or the MSB is transmitted
-			  first for the fifo data. If this property is absent,
-			  the MSB is transmitted first as default, or the LSB
-			  is transmitted first.
-
-  - fsl,sai-synchronous-rx: This is a boolean property. If present, indicating
-			  that SAI will work in the synchronous mode (sync Tx
-			  with Rx) which means both the transmitter and the
-			  receiver will send and receive data by following
-			  receiver's bit clocks and frame sync clocks.
-
-  - fsl,sai-asynchronous: This is a boolean property. If present, indicating
-			  that SAI will work in the asynchronous mode, which
-			  means both transmitter and receiver will send and
-			  receive data by following their own bit clocks and
-			  frame sync clocks separately.
-
-  - fsl,dataline        : configure the dataline. it has 3 value for each configuration
-                          first one means the type: I2S(1) or PDM(2)
-                          second one is dataline mask for 'rx'
-                          third one is dataline mask for 'tx'.
-                          for example: fsl,dataline = <1 0xff 0xff 2 0xff 0x11>;
-                          it means I2S type rx mask is 0xff, tx mask is 0xff, PDM type
-                          rx mask is 0xff, tx mask is 0x11 (dataline 1 and 4 enabled).
-
-Optional properties:
-
-  - big-endian		: Boolean property, required if all the SAI
-			  registers are big-endian rather than little-endian.
-
-Optional properties (for mx6ul):
-
-  - fsl,sai-mclk-direction-output: This is a boolean property. If present,
-			 indicates that SAI will output the SAI MCLK clock.
-
-Note:
-- If both fsl,sai-asynchronous and fsl,sai-synchronous-rx are absent, the
-  default synchronous mode (sync Rx with Tx) will be used, which means both
-  transmitter and receiver will send and receive data by following clocks
-  of transmitter.
-- fsl,sai-asynchronous and fsl,sai-synchronous-rx are exclusive.
-
-Example:
-sai2: sai@40031000 {
-	      compatible = "fsl,vf610-sai";
-	      reg = <0x40031000 0x1000>;
-	      pinctrl-names = "default";
-	      pinctrl-0 = <&pinctrl_sai2_1>;
-	      clocks = <&clks VF610_CLK_PLATFORM_BUS>,
-		     <&clks VF610_CLK_SAI2>,
-		     <&clks 0>, <&clks 0>;
-	      clock-names = "bus", "mclk1", "mclk2", "mclk3";
-	      dma-names = "tx", "rx";
-	      dmas = <&edma0 0 VF610_EDMA_MUXID0_SAI2_TX>,
-		   <&edma0 0 VF610_EDMA_MUXID0_SAI2_RX>;
-	      big-endian;
-	      lsb-first;
-};
diff --git a/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml b/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml
index 233caa0ade87..67ccddd44489 100644
--- a/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml
+++ b/Documentation/devicetree/bindings/sound/google,sc7180-trogdor.yaml
@@ -61,6 +61,8 @@ patternProperties:
       cpu:
         description: Holds subnode which indicates cpu dai.
         type: object
+        additionalProperties: false
+
         properties:
           sound-dai:
             maxItems: 1
@@ -68,6 +70,8 @@ patternProperties:
       codec:
         description: Holds subnode which indicates codec dai.
         type: object
+        additionalProperties: false
+
         properties:
           sound-dai:
             maxItems: 1
diff --git a/Documentation/devicetree/bindings/sound/imx-audio-card.yaml b/Documentation/devicetree/bindings/sound/imx-audio-card.yaml
index bb3a435722c7..b6f5d486600e 100644
--- a/Documentation/devicetree/bindings/sound/imx-audio-card.yaml
+++ b/Documentation/devicetree/bindings/sound/imx-audio-card.yaml
@@ -58,6 +58,7 @@ patternProperties:
       cpu:
         description: Holds subnode which indicates cpu dai.
         type: object
+        additionalProperties: false
         properties:
           sound-dai:
             maxItems: 1
@@ -65,6 +66,7 @@ patternProperties:
       codec:
         description: Holds subnode which indicates codec dai.
         type: object
+        additionalProperties: false
         properties:
           sound-dai:
             minItems: 1
diff --git a/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml b/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml
index 513cd28b2027..d427f7f623db 100644
--- a/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml
+++ b/Documentation/devicetree/bindings/sound/mt8186-mt6366-da7219-max98357.yaml
@@ -43,6 +43,16 @@ properties:
     required:
       - sound-dai
 
+  mediatek,adsp:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: The phandle of MT8186 ADSP platform.
+
+  mediatek,dai-link:
+    $ref: /schemas/types.yaml#/definitions/string-array
+    description:
+      A list of the desired dai-links in the sound card. Each entry is a
+      name defined in the machine driver.
+
 additionalProperties: false
 
 required:
diff --git a/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml b/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml
index 059a7629b2d3..4fc5b045d3cf 100644
--- a/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml
+++ b/Documentation/devicetree/bindings/sound/mt8186-mt6366-rt1019-rt5682s.yaml
@@ -43,6 +43,16 @@ properties:
     required:
       - sound-dai
 
+  mediatek,adsp:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: The phandle of MT8186 ADSP platform.
+
+  mediatek,dai-link:
+    $ref: /schemas/types.yaml#/definitions/string-array
+    description:
+      A list of the desired dai-links in the sound card. Each entry is a
+      name defined in the machine driver.
+
 additionalProperties: false
 
 required:
diff --git a/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml b/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml
index 4fa179909c62..478be7e3fa29 100644
--- a/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml
+++ b/Documentation/devicetree/bindings/sound/mt8192-mt6359-rt1015-rt5682.yaml
@@ -30,6 +30,8 @@ properties:
 
   headset-codec:
     type: object
+    additionalProperties: false
+
     properties:
       sound-dai:
         $ref: /schemas/types.yaml#/definitions/phandle
@@ -38,6 +40,8 @@ properties:
 
   speaker-codecs:
     type: object
+    additionalProperties: false
+
     properties:
       sound-dai:
         minItems: 1
diff --git a/Documentation/devicetree/bindings/sound/nau8825.txt b/Documentation/devicetree/bindings/sound/nau8825.txt
index 388a7bc60b1f..cb861aca8d40 100644
--- a/Documentation/devicetree/bindings/sound/nau8825.txt
+++ b/Documentation/devicetree/bindings/sound/nau8825.txt
@@ -71,6 +71,9 @@ Optional properties:
 
   - nuvoton,crosstalk-enable: make crosstalk function enable if set.
 
+  - nuvoton,adcout-drive-strong: make the drive strength of ADCOUT IO PIN strong if set.
+      Otherwise, the drive keeps normal strength.
+
   - clocks: list of phandle and clock specifier pairs according to common clock bindings for the
       clocks described in clock-names
   - clock-names: should include "mclk" for the MCLK master clock
diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml
index a6905bcf89d2..1de11e7f33bb 100644
--- a/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,lpass-rx-macro.yaml
@@ -14,6 +14,8 @@ properties:
     enum:
       - qcom,sc7280-lpass-rx-macro
       - qcom,sm8250-lpass-rx-macro
+      - qcom,sm8450-lpass-rx-macro
+      - qcom,sc8280xp-lpass-rx-macro
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml
index 324595a62ae8..de8297b358e8 100644
--- a/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,lpass-tx-macro.yaml
@@ -14,6 +14,8 @@ properties:
     enum:
       - qcom,sc7280-lpass-tx-macro
       - qcom,sm8250-lpass-tx-macro
+      - qcom,sm8450-lpass-tx-macro
+      - qcom,sc8280xp-lpass-tx-macro
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml
index 7b4cc84eda8c..9f473c08cb2e 100644
--- a/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,lpass-va-macro.yaml
@@ -14,6 +14,8 @@ properties:
     enum:
       - qcom,sc7280-lpass-va-macro
       - qcom,sm8250-lpass-va-macro
+      - qcom,sm8450-lpass-va-macro
+      - qcom,sc8280xp-lpass-va-macro
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml b/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml
index 13cdb8a10687..4959ad658eac 100644
--- a/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,lpass-wsa-macro.yaml
@@ -14,6 +14,8 @@ properties:
     enum:
       - qcom,sc7280-lpass-wsa-macro
       - qcom,sm8250-lpass-wsa-macro
+      - qcom,sm8450-lpass-wsa-macro
+      - qcom,sc8280xp-lpass-wsa-macro
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6adm-routing.yaml b/Documentation/devicetree/bindings/sound/qcom,q6adm-routing.yaml
new file mode 100644
index 000000000000..d0f7a79e240a
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/qcom,q6adm-routing.yaml
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/qcom,q6adm-routing.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Audio Device Manager (Q6ADM) routing
+
+maintainers:
+  - Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>
+  - Srinivas Kandagatla <srinivas.kandagatla@linaro.org>
+
+description:
+  Qualcomm Audio Device Manager (Q6ADM) routing node represents routing
+  specific configuration.
+
+properties:
+  compatible:
+    enum:
+      - qcom,q6adm-routing
+
+  "#sound-dai-cells":
+    const: 0
+
+required:
+  - compatible
+  - "#sound-dai-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/soc/qcom,apr.h>
+    #include <dt-bindings/sound/qcom,q6asm.h>
+
+    apr {
+        compatible = "qcom,apr-v2";
+        qcom,domain = <APR_DOMAIN_ADSP>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        service@8 {
+            compatible = "qcom,q6adm";
+            reg = <APR_SVC_ADM>;
+            qcom,protection-domain = "avs/audio", "msm/adsp/audio_pd";
+
+            routing {
+                compatible = "qcom,q6adm-routing";
+                #sound-dai-cells = <0>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6adm.txt b/Documentation/devicetree/bindings/sound/qcom,q6adm.txt
deleted file mode 100644
index 15c353a20de8..000000000000
--- a/Documentation/devicetree/bindings/sound/qcom,q6adm.txt
+++ /dev/null
@@ -1,39 +0,0 @@
-Qualcomm Audio Device Manager (Q6ADM) binding
-
-Q6ADM is one of the APR audio service on Q6DSP.
-Please refer to qcom,apr.txt for details of the coommon apr service bindings
-used by the apr service device.
-
-- but must contain the following property:
-
-- compatible:
-	Usage: required
-	Value type: <stringlist>
-	Definition: must be "qcom,q6adm-v<MAJOR-NUMBER>.<MINOR-NUMBER>".
-		   Or "qcom,q6adm" where the version number can be queried
-		   from DSP.
-		   example "qcom,q6adm-v2.0"
-
-
-= ADM routing
-"routing" subnode of the ADM node represents adm routing specific configuration
-
-- compatible:
-	Usage: required
-	Value type: <stringlist>
-	Definition: must be "qcom,q6adm-routing".
-
-- #sound-dai-cells
-	Usage: required
-	Value type: <u32>
-	Definition: Must be 0
-
-= EXAMPLE
-apr-service@8 {
-	compatible = "qcom,q6adm";
-	reg = <APR_SVC_ADM>;
-	q6routing: routing {
-		compatible = "qcom,q6adm-routing";
-		#sound-dai-cells = <0>;
-	};
-};
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6afe.txt b/Documentation/devicetree/bindings/sound/qcom,q6afe.txt
deleted file mode 100644
index bc6b5f1fe4f1..000000000000
--- a/Documentation/devicetree/bindings/sound/qcom,q6afe.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-Qualcomm Audio Front End (Q6AFE) binding
-
-AFE is one of the APR audio service on Q6DSP
-Please refer to qcom,apr.txt for details of the common apr service bindings
-used by all apr services. Must contain the following properties.
-
-- compatible:
-	Usage: required
-	Value type: <stringlist>
-	Definition: must be "qcom,q6afe-v<MAJOR-NUMBER>.<MINOR-NUMBER>"
-		  Or "qcom,q6afe" where the version number can be queried
-		  from DSP.
-		  example "qcom,q6afe"
-
-= EXAMPLE
-
-apr-service@4 {
-	compatible = "qcom,q6afe";
-	reg = <APR_SVC_AFE>;
-};
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6apm-dai.yaml b/Documentation/devicetree/bindings/sound/qcom,q6apm-dai.yaml
index 5d972784321d..24f7bf2bfd95 100644
--- a/Documentation/devicetree/bindings/sound/qcom,q6apm-dai.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,q6apm-dai.yaml
@@ -16,16 +16,12 @@ properties:
   compatible:
     const: qcom,q6apm-dais
 
-  reg:
-    maxItems: 1
-
   iommus:
     maxItems: 1
 
 required:
   - compatible
   - iommus
-  - reg
 
 additionalProperties: false
 
@@ -37,17 +33,14 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
         qcom,domain = <GPR_DOMAIN_ID_ADSP>;
-        service@1 {
-          compatible = "qcom,q6apm";
-          reg = <1>;
 
-          #address-cells = <1>;
-          #size-cells = <0>;
-
-          apm-dai@1 {
-            compatible = "qcom,q6apm-dais";
-            iommus = <&apps_smmu 0x1801 0x0>;
+        service@1 {
+            compatible = "qcom,q6apm";
             reg = <1>;
-          };
+
+            dais {
+                compatible = "qcom,q6apm-dais";
+                iommus = <&apps_smmu 0x1801 0x0>;
+            };
         };
     };
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml b/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml
new file mode 100644
index 000000000000..8deb8ffb143b
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/qcom,q6asm-dais.yaml
@@ -0,0 +1,112 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/qcom,q6asm-dais.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Audio Stream Manager (Q6ASM)
+
+maintainers:
+  - Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>
+  - Srinivas Kandagatla <srinivas.kandagatla@linaro.org>
+
+description:
+  Q6ASM is one of the APR audio services on Q6DSP. Each of its subnodes
+  represent a dai with board specific configuration.
+
+properties:
+  compatible:
+    enum:
+      - qcom,q6asm-dais
+
+  iommus:
+    maxItems: 1
+
+  "#sound-dai-cells":
+    const: 1
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 0
+
+patternProperties:
+  "^dai@[0-9]+$":
+    type: object
+    description:
+      Q6ASM Digital Audio Interface
+
+    properties:
+      reg:
+        maxItems: 1
+
+      direction:
+        $ref: /schemas/types.yaml#/definitions/uint32
+        enum: [0, 1, 2]
+        description: |
+          The direction of the dai stream::
+           - Q6ASM_DAI_TX_RX (0) for both tx and rx
+           - Q6ASM_DAI_TX (1) for only tx (Capture/Encode)
+           - Q6ASM_DAI_RX (2) for only rx (Playback/Decode)
+
+      is-compress-dai:
+        type: boolean
+        description:
+          Compress offload dai.
+
+    dependencies:
+      is-compress-dai: ["direction"]
+
+    required:
+      - reg
+
+    additionalProperties: false
+
+required:
+  - compatible
+  - "#sound-dai-cells"
+  - "#address-cells"
+  - "#size-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/soc/qcom,apr.h>
+    #include <dt-bindings/sound/qcom,q6asm.h>
+
+    apr {
+        compatible = "qcom,apr-v2";
+        qcom,domain = <APR_DOMAIN_ADSP>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        service@7 {
+            compatible = "qcom,q6asm";
+            reg = <APR_SVC_ASM>;
+            qcom,protection-domain = "avs/audio", "msm/adsp/audio_pd";
+
+            dais {
+                compatible = "qcom,q6asm-dais";
+                iommus = <&apps_smmu 0x1821 0x0>;
+                #address-cells = <1>;
+                #size-cells = <0>;
+                #sound-dai-cells = <1>;
+
+                dai@0 {
+                    reg = <0>;
+                };
+
+                dai@1 {
+                    reg = <1>;
+                };
+
+                dai@2 {
+                    reg = <2>;
+                    is-compress-dai;
+                    direction = <1>;
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6asm.txt b/Documentation/devicetree/bindings/sound/qcom,q6asm.txt
deleted file mode 100644
index 0d0075125243..000000000000
--- a/Documentation/devicetree/bindings/sound/qcom,q6asm.txt
+++ /dev/null
@@ -1,70 +0,0 @@
-Qualcomm Audio Stream Manager (Q6ASM) binding
-
-Q6ASM is one of the APR audio service on Q6DSP.
-Please refer to qcom,apr.txt for details of the common apr service bindings
-used by the apr service device.
-
-- but must contain the following property:
-
-- compatible:
-	Usage: required
-	Value type: <stringlist>
-	Definition: must be "qcom,q6asm-v<MAJOR-NUMBER>.<MINOR-NUMBER>".
-		    Or "qcom,q6asm" where the version number can be queried
-		    from DSP.
-		    example "qcom,q6asm-v2.0"
-
-= ASM DAIs (Digital Audio Interface)
-"dais" subnode of the ASM node represents dai specific configuration
-
-- compatible:
-	Usage: required
-	Value type: <stringlist>
-	Definition: must be "qcom,q6asm-dais".
-
-- #sound-dai-cells
-	Usage: required
-	Value type: <u32>
-	Definition: Must be 1
-
-== ASM DAI is subnode of "dais" and represent a dai, it includes board specific
-configuration of each dai. Must contain the following properties.
-
-- reg
-	Usage: required
-	Value type: <u32>
-	Definition: Must be dai id
-
-- direction:
-	Usage: Required for Compress offload dais
-	Value type: <u32>
-	Definition: Specifies the direction of the dai stream
-			Q6ASM_DAI_TX_RX (0) for both tx and rx
-			Q6ASM_DAI_TX (1) for only tx (Capture/Encode)
-			Q6ASM_DAI_RX (2) for only rx (Playback/Decode)
-
-- is-compress-dai:
-	Usage: Required for Compress offload dais
-	Value type: <boolean>
-	Definition: present for Compress offload dais
-
-
-= EXAMPLE
-#include <dt-bindings/sound/qcom,q6asm.h>
-
-apr-service@7 {
-	compatible = "qcom,q6asm";
-	reg = <APR_SVC_ASM>;
-	q6asmdai: dais {
-		compatible = "qcom,q6asm-dais";
-		#address-cells = <1>;
-		#size-cells = <0>;
-		#sound-dai-cells = <1>;
-
-		dai@0 {
-			reg = <0>;
-			direction = <Q6ASM_DAI_RX>;
-			is-compress-dai;
-		};
-	};
-};
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6core.txt b/Documentation/devicetree/bindings/sound/qcom,q6core.txt
deleted file mode 100644
index 5cd4cc9b1fde..000000000000
--- a/Documentation/devicetree/bindings/sound/qcom,q6core.txt
+++ /dev/null
@@ -1,21 +0,0 @@
-Qualcomm ADSP Core service binding
-
-Q6CORE is one of the APR audio service on Q6DSP.
-Please refer to qcom,apr.txt for details of the common apr service bindings
-used by the apr service device.
-
-- but must contain the following property:
-
-- compatible:
-	Usage: required
-	Value type: <stringlist>
-	Definition: must be "qcom,q6core-v<MAJOR-NUMBER>.<MINOR-NUMBER>".
-		   Or "qcom,q6core" where the version number can be queried
-		   from DSP.
-		   example "qcom,q6core-v2.0"
-
-= EXAMPLE
-apr-service@3 {
-	compatible = "qcom,q6core";
-	reg = <APR_SVC_ADSP_CORE>;
-};
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-clocks.yaml b/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-clocks.yaml
index f83f00737a2f..fd567d20417d 100644
--- a/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-clocks.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-clocks.yaml
@@ -18,9 +18,6 @@ properties:
       - qcom,q6afe-clocks
       - qcom,q6prm-lpass-clocks
 
-  reg:
-    maxItems: 1
-
   '#clock-cells':
     const: 2
     description:
@@ -32,7 +29,6 @@ properties:
 
 required:
   - compatible
-  - reg
   - "#clock-cells"
 
 additionalProperties: false
@@ -42,19 +38,22 @@ examples:
     #include <dt-bindings/soc/qcom,apr.h>
     #include <dt-bindings/sound/qcom,q6afe.h>
     apr {
+        compatible = "qcom,apr-v2";
+        qcom,domain = <APR_DOMAIN_ADSP>;
         #address-cells = <1>;
         #size-cells = <0>;
-        apr-service@4 {
+
+        service@4 {
+            compatible = "qcom,q6afe";
             reg = <APR_SVC_AFE>;
-            #address-cells = <1>;
-            #size-cells = <0>;
-            clock-controller@2 {
-              compatible = "qcom,q6afe-clocks";
-              reg = <2>;
-              #clock-cells = <2>;
+            qcom,protection-domain = "avs/audio", "msm/adsp/audio_pd";
+
+            clock-controller {
+                compatible = "qcom,q6afe-clocks";
+                #clock-cells = <2>;
             };
         };
-      };
+    };
 
   - |
     #include <dt-bindings/soc/qcom,gpr.h>
@@ -63,15 +62,14 @@ examples:
         qcom,domain = <GPR_DOMAIN_ID_ADSP>;
         #address-cells = <1>;
         #size-cells = <0>;
+
         service@2 {
             reg = <GPR_PRM_MODULE_IID>;
             compatible = "qcom,q6prm";
-            #address-cells = <1>;
-            #size-cells = <0>;
-            clock-controller@2 {
-              compatible = "qcom,q6prm-lpass-clocks";
-              reg = <2>;
-              #clock-cells = <2>;
+
+            clock-controller {
+                compatible = "qcom,q6prm-lpass-clocks";
+                #clock-cells = <2>;
             };
         };
-      };
+    };
diff --git a/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml b/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml
index dc7fba7b92d5..e53fc0960a14 100644
--- a/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,q6dsp-lpass-ports.yaml
@@ -18,9 +18,6 @@ properties:
       - qcom,q6afe-dais
       - qcom,q6apm-lpass-dais
 
-  reg:
-    maxItems: 1
-
   '#sound-dai-cells':
     const: 1
 
@@ -145,7 +142,6 @@ patternProperties:
 
 required:
   - compatible
-  - reg
   - "#sound-dai-cells"
   - "#address-cells"
   - "#size-cells"
@@ -157,26 +153,29 @@ examples:
     #include <dt-bindings/soc/qcom,apr.h>
     #include <dt-bindings/sound/qcom,q6afe.h>
     apr {
+        compatible = "qcom,apr-v2";
         #address-cells = <1>;
         #size-cells = <0>;
-        apr-service@4 {
+        qcom,domain = <APR_DOMAIN_ADSP>;
+
+        service@4 {
+            compatible = "qcom,q6afe";
             reg = <APR_SVC_AFE>;
-            #address-cells = <1>;
-            #size-cells = <0>;
-            q6afedai@1 {
-              compatible = "qcom,q6afe-dais";
-              reg = <1>;
-              #address-cells = <1>;
-              #size-cells = <0>;
-              #sound-dai-cells = <1>;
-
-              dai@22 {
-                reg = <QUATERNARY_MI2S_RX>;
-                qcom,sd-lines = <0 1 2 3>;
-              };
+            qcom,protection-domain = "avs/audio", "msm/adsp/audio_pd";
+
+            dais {
+                compatible = "qcom,q6afe-dais";
+                #address-cells = <1>;
+                #size-cells = <0>;
+                #sound-dai-cells = <1>;
+
+                dai@22 {
+                    reg = <QUATERNARY_MI2S_RX>;
+                    qcom,sd-lines = <0 1 2 3>;
+                };
             };
         };
-      };
+    };
   - |
     #include <dt-bindings/soc/qcom,gpr.h>
     gpr {
@@ -184,22 +183,21 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
         qcom,domain = <GPR_DOMAIN_ID_ADSP>;
+
         service@1 {
             compatible = "qcom,q6apm";
             reg = <GPR_APM_MODULE_IID>;
-            #address-cells = <1>;
-            #size-cells = <0>;
-            q6apmdai@1 {
-              compatible = "qcom,q6apm-lpass-dais";
-              reg = <1>;
-              #address-cells = <1>;
-              #size-cells = <0>;
-              #sound-dai-cells = <1>;
-
-              dai@22 {
-                reg = <QUATERNARY_MI2S_RX>;
-                qcom,sd-lines = <0 1 2 3>;
-              };
+
+            dais {
+                compatible = "qcom,q6apm-lpass-dais";
+                #address-cells = <1>;
+                #size-cells = <0>;
+                #sound-dai-cells = <1>;
+
+                dai@22 {
+                    reg = <QUATERNARY_MI2S_RX>;
+                    qcom,sd-lines = <0 1 2 3>;
+                };
             };
         };
-      };
+    };
diff --git a/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml b/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
index e6e27d09783e..70080d04ddc9 100644
--- a/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
+++ b/Documentation/devicetree/bindings/sound/qcom,sm8250.yaml
@@ -20,9 +20,11 @@ properties:
       - qcom,apq8016-sbc-sndcard
       - qcom,db845c-sndcard
       - qcom,msm8916-qdsp6-sndcard
+      - qcom,qrb5165-rb5-sndcard
+      - qcom,sc8280xp-sndcard
       - qcom,sdm845-sndcard
       - qcom,sm8250-sndcard
-      - qcom,qrb5165-rb5-sndcard
+      - qcom,sm8450-sndcard
 
   audio-routing:
     $ref: /schemas/types.yaml#/definitions/non-unique-string-array
@@ -71,6 +73,8 @@ patternProperties:
       cpu:
         description: Holds subnode which indicates cpu dai.
         type: object
+        additionalProperties: false
+
         properties:
           sound-dai:
             maxItems: 1
@@ -78,6 +82,8 @@ patternProperties:
       platform:
         description: Holds subnode which indicates platform dai.
         type: object
+        additionalProperties: false
+
         properties:
           sound-dai:
             maxItems: 1
@@ -85,6 +91,8 @@ patternProperties:
       codec:
         description: Holds subnode which indicates codec dai.
         type: object
+        additionalProperties: false
+
         properties:
           sound-dai:
             minItems: 1
diff --git a/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml b/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml
index e17c0245f77a..268895c90bd5 100644
--- a/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml
+++ b/Documentation/devicetree/bindings/sound/renesas,rsnd.yaml
@@ -129,6 +129,8 @@ properties:
     patternProperties:
       "^dvc-[0-1]$":
         type: object
+        additionalProperties: false
+
         properties:
           dmas:
             maxItems: 1
@@ -145,7 +147,7 @@ properties:
     patternProperties:
       "^mix-[0-1]$":
         type: object
-        # no properties
+        additionalProperties: false
     additionalProperties: false
 
   rcar_sound,ctu:
@@ -154,7 +156,7 @@ properties:
     patternProperties:
       "^ctu-[0-7]$":
         type: object
-        # no properties
+        additionalProperties: false
     additionalProperties: false
 
   rcar_sound,src:
@@ -163,6 +165,8 @@ properties:
     patternProperties:
       "^src-[0-9]$":
         type: object
+        additionalProperties: false
+
         properties:
           interrupts:
             maxItems: 1
@@ -186,6 +190,8 @@ properties:
     patternProperties:
       "^ssiu-[0-9]+$":
         type: object
+        additionalProperties: false
+
         properties:
           dmas:
             maxItems: 2
@@ -206,6 +212,8 @@ properties:
     patternProperties:
       "^ssi-[0-9]$":
         type: object
+        additionalProperties: false
+
         properties:
           interrupts:
             maxItems: 1
@@ -243,6 +251,8 @@ properties:
     patternProperties:
       "^dai([0-9]+)?$":
         type: object
+        additionalProperties: false
+
         properties:
           playback:
             $ref: /schemas/types.yaml#/definitions/phandle-array
diff --git a/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml b/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml
index a01c4ad929b8..447e013f6e17 100644
--- a/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml
+++ b/Documentation/devicetree/bindings/sound/samsung,aries-wm8994.yaml
@@ -23,6 +23,7 @@ properties:
 
   cpu:
     type: object
+    additionalProperties: false
     properties:
       sound-dai:
         minItems: 2
@@ -34,6 +35,7 @@ properties:
       - sound-dai
 
   codec:
+    additionalProperties: false
     type: object
     properties:
       sound-dai:
diff --git a/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml b/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml
index ec50bcb4af5f..31095913e330 100644
--- a/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml
+++ b/Documentation/devicetree/bindings/sound/samsung,midas-audio.yaml
@@ -19,6 +19,7 @@ properties:
 
   cpu:
     type: object
+    additionalProperties: false
     properties:
       sound-dai:
         maxItems: 1
@@ -28,6 +29,7 @@ properties:
 
   codec:
     type: object
+    additionalProperties: false
     properties:
       sound-dai:
         maxItems: 1
diff --git a/Documentation/devicetree/bindings/sound/samsung,snow.yaml b/Documentation/devicetree/bindings/sound/samsung,snow.yaml
index 51a83d3c7274..3d49aa4c9be2 100644
--- a/Documentation/devicetree/bindings/sound/samsung,snow.yaml
+++ b/Documentation/devicetree/bindings/sound/samsung,snow.yaml
@@ -19,6 +19,7 @@ properties:
 
   codec:
     type: object
+    additionalProperties: false
     properties:
       sound-dai:
         description: List of phandles to the CODEC and HDMI IP nodes.
@@ -30,6 +31,7 @@ properties:
 
   cpu:
     type: object
+    additionalProperties: false
     properties:
       sound-dai:
         description: Phandle to the Samsung I2S controller.
diff --git a/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml b/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml
index fe2e15504ebc..56d206f97a96 100644
--- a/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml
+++ b/Documentation/devicetree/bindings/sound/st,stm32-sai.yaml
@@ -60,6 +60,7 @@ required:
 patternProperties:
   "^audio-controller@[0-9a-f]+$":
     type: object
+    additionalProperties: false
     description:
       Two subnodes corresponding to SAI sub-block instances A et B
       can be defined. Subnode can be omitted for unsused sub-block.
@@ -121,6 +122,10 @@ patternProperties:
         description: Configure the SAI device as master clock provider.
         const: 0
 
+      port:
+        $ref: audio-graph-port.yaml#
+        unevaluatedProperties: false
+
     required:
       - compatible
       - "#sound-dai-cells"
diff --git a/Documentation/devicetree/bindings/sound/ti,src4xxx.yaml b/Documentation/devicetree/bindings/sound/ti,src4xxx.yaml
new file mode 100644
index 000000000000..9681b72b4918
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/ti,src4xxx.yaml
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/ti,src4xxx.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments SRC4392 Device Tree Bindings
+
+description: |
+  The SRC4392 is a digital audio codec that can be connected via
+  I2C or SPI. Currently, only I2C bus is supported.
+
+maintainers:
+  - Matt Flax <flatmax@flatmax.com>
+
+allOf:
+  - $ref: name-prefix.yaml#
+
+properties:
+  compatible:
+    const: ti,src4392
+
+  "#sound-dai-cells":
+    const: 0
+
+  reg:
+    maxItems: 1
+
+required:
+  - "#sound-dai-cells"
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        audio-codec@70 {
+            #sound-dai-cells = <0>;
+            compatible = "ti,src4392";
+            reg = <0x70>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/sound/ti,ts3a227e.yaml b/Documentation/devicetree/bindings/sound/ti,ts3a227e.yaml
new file mode 100644
index 000000000000..785930658029
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/ti,ts3a227e.yaml
@@ -0,0 +1,94 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/ti,ts3a227e.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments TS3A227E
+  Autonomous Audio Accessory Detection and Configuration Switch
+
+maintainers:
+  - Dylan Reid <dgreid@chromium.org>
+
+description: |
+  The TS3A227E detect headsets of 3-ring and 4-ring standards and
+  switches automatically to route the microphone correctly. It also
+  handles key press detection in accordance with the Android audio
+  headset specification v1.0.
+
+properties:
+  compatible:
+    enum:
+      - ti,ts3a227e
+
+  reg:
+    const: 0x3b
+
+  interrupts:
+    maxItems: 1
+
+  ti,micbias:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: Intended MICBIAS voltage (datasheet section 9.6.7).
+    enum:
+      - 0 # 2.1 V
+      - 1 # 2.2 V
+      - 2 # 2.3 V
+      - 3 # 2.4 V
+      - 4 # 2.5 V
+      - 5 # 2.6 V
+      - 6 # 2.7 V
+      - 7 # 2.8 V
+    default: 1
+
+  ti,debounce-release-ms:
+    description: key release debounce time in ms (datasheet section 9.6.7).
+    enum:
+      - 0
+      - 20
+    default: 20
+
+  ti,debounce-press-ms:
+    description: key press debounce time in ms (datasheet section 9.6.7).
+    enum:
+      - 2
+      - 40
+      - 80
+      - 120
+    default: 80
+
+  ti,debounce-insertion-ms:
+    description: headset insertion debounce time in ms (datasheet section 9.6.5).
+    enum:
+      - 2
+      - 30
+      - 60
+      - 90
+      - 120
+      - 150
+      - 1000
+      - 2000
+    default: 90
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/irq.h>
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        codec: audio-controller@3b {
+            compatible = "ti,ts3a227e";
+            reg = <0x3b>;
+            interrupt-parent = <&gpio1>;
+            interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/sound/ts3a227e.txt b/Documentation/devicetree/bindings/sound/ts3a227e.txt
deleted file mode 100644
index 21ab45bc7e8f..000000000000
--- a/Documentation/devicetree/bindings/sound/ts3a227e.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-Texas Instruments TS3A227E
-Autonomous Audio Accessory Detection and Configuration Switch
-
-The TS3A227E detect headsets of 3-ring and 4-ring standards and
-switches automatically to route the microphone correctly.  It also
-handles key press detection in accordance with the Android audio
-headset specification v1.0.
-
-Required properties:
-
- - compatible:		Should contain "ti,ts3a227e".
- - reg:			The i2c address. Should contain <0x3b>.
- - interrupts:		Interrupt number for /INT pin from the 227e
-
-Optional properies:
- - ti,micbias:   Intended MICBIAS voltage (datasheet section 9.6.7).
-      Select 0/1/2/3/4/5/6/7 to specify MICBIAS voltage
-      2.1V/2.2V/2.3V/2.4V/2.5V/2.6V/2.7V/2.8V
-      Default value is "1" (2.2V).
-
-Examples:
-
-	i2c {
-		ts3a227e@3b {
-			compatible = "ti,ts3a227e";
-			reg = <0x3b>;
-			interrupt-parent = <&gpio>;
-			interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
-		};
-	};
diff --git a/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml b/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml
index d85d54024b2e..4dd973e341e6 100644
--- a/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml
+++ b/Documentation/devicetree/bindings/spi/atmel,at91rm9200-spi.yaml
@@ -34,6 +34,16 @@ properties:
   clocks:
     maxItems: 1
 
+  dmas:
+    items:
+      - description: TX DMA Channel
+      - description: RX DMA Channel
+
+  dma-names:
+    items:
+      - const: tx
+      - const: rx
+
   atmel,fifo-size:
     $ref: /schemas/types.yaml#/definitions/uint32
     description: |
diff --git a/Documentation/devicetree/bindings/spi/mediatek,spi-mtk-nor.yaml b/Documentation/devicetree/bindings/spi/mediatek,spi-mtk-nor.yaml
index 970b1119898b..a453996c13f2 100644
--- a/Documentation/devicetree/bindings/spi/mediatek,spi-mtk-nor.yaml
+++ b/Documentation/devicetree/bindings/spi/mediatek,spi-mtk-nor.yaml
@@ -85,8 +85,9 @@ examples:
         compatible = "mediatek,mt8173-nor";
         reg = <0 0x1100d000 0 0xe0>;
         interrupts = <1>;
-        clocks = <&pericfg CLK_PERI_SPI>, <&topckgen CLK_TOP_SPINFI_IFR_SEL>;
-        clock-names = "spi", "sf";
+        clocks = <&pericfg CLK_PERI_SPI>, <&topckgen CLK_TOP_SPINFI_IFR_SEL>,
+                 <&pericfg CLK_PERI_NFI>;
+        clock-names = "spi", "sf", "axi";
         #address-cells = <1>;
         #size-cells = <0>;
 
diff --git a/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml b/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml
index 7326c0a28d16..1051690e3753 100644
--- a/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml
+++ b/Documentation/devicetree/bindings/spi/microchip,mpfs-spi.yaml
@@ -4,7 +4,11 @@
 $id: http://devicetree.org/schemas/spi/microchip,mpfs-spi.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Microchip MPFS {Q,}SPI Controller Device Tree Bindings
+title: Microchip FPGA {Q,}SPI Controllers
+
+description:
+  SPI and QSPI controllers on Microchip PolarFire SoC and the "soft"/
+  fabric IP cores they are based on
 
 maintainers:
   - Conor Dooley <conor.dooley@microchip.com>
@@ -14,9 +18,12 @@ allOf:
 
 properties:
   compatible:
-    enum:
-      - microchip,mpfs-spi
-      - microchip,mpfs-qspi
+    oneOf:
+      - items:
+          - const: microchip,mpfs-qspi
+          - const: microchip,coreqspi-rtl-v2
+      - const: microchip,coreqspi-rtl-v2 #FPGA QSPI
+      - const: microchip,mpfs-spi
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/spi/nuvoton,npcm-pspi.txt b/Documentation/devicetree/bindings/spi/nuvoton,npcm-pspi.txt
index b98203ca656d..a4e72e52af59 100644
--- a/Documentation/devicetree/bindings/spi/nuvoton,npcm-pspi.txt
+++ b/Documentation/devicetree/bindings/spi/nuvoton,npcm-pspi.txt
@@ -3,7 +3,8 @@ Nuvoton NPCM Peripheral Serial Peripheral Interface(PSPI) controller driver
 Nuvoton NPCM7xx SOC support two PSPI channels.
 
 Required properties:
- - compatible : "nuvoton,npcm750-pspi" for NPCM7XX BMC
+ - compatible : "nuvoton,npcm750-pspi" for Poleg NPCM7XX.
+				"nuvoton,npcm845-pspi" for Arbel NPCM8XX.
  - #address-cells : should be 1. see spi-bus.txt
  - #size-cells : should be 0. see spi-bus.txt
  - specifies physical base address and size of the register.
diff --git a/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad-peripheral-props.yaml b/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad-peripheral-props.yaml
index 24e0c2181d25..2c3cada75339 100644
--- a/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad-peripheral-props.yaml
+++ b/Documentation/devicetree/bindings/spi/nvidia,tegra210-quad-peripheral-props.yaml
@@ -29,5 +29,4 @@ properties:
     minimum: 0
     maximum: 255
 
-unevaluatedProperties: true
-
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/spi/ralink,mt7621-spi.yaml b/Documentation/devicetree/bindings/spi/ralink,mt7621-spi.yaml
new file mode 100644
index 000000000000..22879f7dcb77
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/ralink,mt7621-spi.yaml
@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/spi/ralink,mt7621-spi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+maintainers:
+  - Sergio Paracuellos <sergio.paracuellos@gmail.com>
+
+title: Mediatek MT7621/MT7628 SPI controller
+
+allOf:
+  - $ref: /schemas/spi/spi-controller.yaml#
+
+properties:
+  compatible:
+    const: ralink,mt7621-spi
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  clock-names:
+    const: spi
+
+  resets:
+    maxItems: 1
+
+  reset-names:
+    const: spi
+
+required:
+  - compatible
+  - reg
+  - resets
+  - "#address-cells"
+  - "#size-cells"
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/mt7621-clk.h>
+    #include <dt-bindings/reset/mt7621-reset.h>
+
+    spi@b00 {
+      compatible = "ralink,mt7621-spi";
+      reg = <0xb00 0x100>;
+      clocks = <&sysc MT7621_CLK_SPI>;
+      clock-names = "spi";
+      resets = <&sysc MT7621_RST_SPI>;
+      reset-names = "spi";
+
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      pinctrl-names = "default";
+      pinctrl-0 = <&spi_pins>;
+    };
diff --git a/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml b/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml
index 5de710adfa63..491a695a2deb 100644
--- a/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml
+++ b/Documentation/devicetree/bindings/spi/renesas,sh-msiof.yaml
@@ -47,10 +47,16 @@ properties:
               - renesas,msiof-r8a77980      # R-Car V3H
               - renesas,msiof-r8a77990      # R-Car E3
               - renesas,msiof-r8a77995      # R-Car D3
-              - renesas,msiof-r8a779a0      # R-Car V3U
           - const: renesas,rcar-gen3-msiof  # generic R-Car Gen3 and RZ/G2
                                             # compatible device
       - items:
+          - enum:
+              - renesas,msiof-r8a779a0      # R-Car V3U
+              - renesas,msiof-r8a779f0      # R-Car S4-8
+              - renesas,msiof-r8a779g0      # R-Car V4H
+          - const: renesas,rcar-gen4-msiof  # generic R-Car Gen4
+                                            # compatible device
+      - items:
           - const: renesas,sh-msiof  # deprecated
 
   reg:
@@ -69,6 +75,12 @@ properties:
   clocks:
     maxItems: 1
 
+  power-domains:
+    maxItems: 1
+
+  resets:
+    maxItems: 1
+
   num-cs:
     description: |
       Total number of chip selects (default is 1).
diff --git a/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml b/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml
index 37c3c272407d..d33b72fabc5d 100644
--- a/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml
+++ b/Documentation/devicetree/bindings/spi/snps,dw-apb-ssi.yaml
@@ -104,7 +104,6 @@ properties:
     const: spi
 
   reg-io-width:
-    $ref: /schemas/types.yaml#/definitions/uint32
     description: I/O register width (in bytes) implemented by this device
     default: 4
     enum: [ 2, 4 ]
diff --git a/Documentation/devicetree/bindings/spi/spi-controller.yaml b/Documentation/devicetree/bindings/spi/spi-controller.yaml
index 655713fba7e2..01042a7f382e 100644
--- a/Documentation/devicetree/bindings/spi/spi-controller.yaml
+++ b/Documentation/devicetree/bindings/spi/spi-controller.yaml
@@ -96,6 +96,11 @@ patternProperties:
     $ref: spi-peripheral-props.yaml
 
     properties:
+      spi-3wire:
+        $ref: /schemas/types.yaml#/definitions/flag
+        description:
+          The device requires 3-wire mode.
+
       spi-cpha:
         $ref: /schemas/types.yaml#/definitions/flag
         description:
diff --git a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml
index 1d46877fe46a..8b44284d30c6 100644
--- a/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml
+++ b/Documentation/devicetree/bindings/spi/spi-fsl-lpspi.yaml
@@ -19,7 +19,9 @@ properties:
           - fsl,imx7ulp-spi
           - fsl,imx8qxp-spi
       - items:
-          - const: fsl,imx8ulp-spi
+          - enum:
+              - fsl,imx8ulp-spi
+              - fsl,imx93-spi
           - const: fsl,imx7ulp-spi
   reg:
     maxItems: 1
@@ -37,6 +39,16 @@ properties:
       - const: per
       - const: ipg
 
+  dmas:
+    items:
+      - description: TX DMA Channel
+      - description: RX DMA Channel
+
+  dma-names:
+    items:
+      - const: tx
+      - const: rx
+
   fsl,spi-only-use-cs1-sel:
     description:
       spi common code does not support use of CS signals discontinuously.
diff --git a/Documentation/devicetree/bindings/spi/spi-mt7621.txt b/Documentation/devicetree/bindings/spi/spi-mt7621.txt
deleted file mode 100644
index d5baec0fa56e..000000000000
--- a/Documentation/devicetree/bindings/spi/spi-mt7621.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-Binding for MTK SPI controller (MT7621 MIPS)
-
-Required properties:
-- compatible: Should be one of the following:
-  - "ralink,mt7621-spi": for mt7621/mt7628/mt7688 platforms
-- #address-cells: should be 1.
-- #size-cells: should be 0.
-- reg: Address and length of the register set for the device
-- resets: phandle to the reset controller asserting this device in
-          reset
-  See ../reset/reset.txt for details.
-
-Optional properties:
-- cs-gpios: see spi-bus.txt.
-
-Example:
-
-- SoC Specific Portion:
-spi0: spi@b00 {
-	compatible = "ralink,mt7621-spi";
-	reg = <0xb00 0x100>;
-	#address-cells = <1>;
-	#size-cells = <0>;
-	resets = <&rstctrl 18>;
-	reset-names = "spi";
-};
diff --git a/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml b/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml
index a4abe1588005..dca677f9e1b9 100644
--- a/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml
+++ b/Documentation/devicetree/bindings/spi/spi-peripheral-props.yaml
@@ -29,11 +29,6 @@ properties:
     description:
       Chip select used by the device.
 
-  spi-3wire:
-    $ref: /schemas/types.yaml#/definitions/flag
-    description:
-      The device requires 3-wire mode.
-
   spi-cs-high:
     $ref: /schemas/types.yaml#/definitions/flag
     description:
diff --git a/Documentation/devicetree/bindings/spi/spi-rockchip.yaml b/Documentation/devicetree/bindings/spi/spi-rockchip.yaml
index 52a78a2e362e..66e49947b703 100644
--- a/Documentation/devicetree/bindings/spi/spi-rockchip.yaml
+++ b/Documentation/devicetree/bindings/spi/spi-rockchip.yaml
@@ -27,6 +27,7 @@ properties:
       - items:
           - enum:
               - rockchip,px30-spi
+              - rockchip,rk3128-spi
               - rockchip,rk3188-spi
               - rockchip,rk3288-spi
               - rockchip,rk3308-spi
@@ -34,6 +35,7 @@ properties:
               - rockchip,rk3368-spi
               - rockchip,rk3399-spi
               - rockchip,rk3568-spi
+              - rockchip,rk3588-spi
               - rockchip,rv1126-spi
           - const: rockchip,rk3066-spi
 
@@ -80,6 +82,9 @@ properties:
       where the "sleep" configuration may describe the state
       the pins should be in during system suspend.
 
+  power-domains:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml b/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml
index 1c426c211e36..d64c1b28fb61 100644
--- a/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml
+++ b/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml
@@ -24,32 +24,31 @@ properties:
 
   compatible:
     oneOf:
-      - const: allwinner,sun4i-a10-sram-controller
+      - enum:
+          - allwinner,sun4i-a10-sram-controller
+          - allwinner,sun50i-a64-sram-controller
         deprecated: true
-      - const: allwinner,sun4i-a10-system-control
-      - const: allwinner,sun5i-a13-system-control
+      - enum:
+          - allwinner,sun4i-a10-system-control
+          - allwinner,sun5i-a13-system-control
+          - allwinner,sun8i-a23-system-control
+          - allwinner,sun8i-h3-system-control
+          - allwinner,sun20i-d1-system-control
+          - allwinner,sun50i-a64-system-control
+          - allwinner,sun50i-h5-system-control
+          - allwinner,sun50i-h616-system-control
       - items:
-          - const: allwinner,sun7i-a20-system-control
+          - enum:
+              - allwinner,suniv-f1c100s-system-control
+              - allwinner,sun7i-a20-system-control
+              - allwinner,sun8i-r40-system-control
           - const: allwinner,sun4i-a10-system-control
-      - const: allwinner,sun8i-a23-system-control
-      - const: allwinner,sun8i-h3-system-control
       - items:
           - const: allwinner,sun8i-v3s-system-control
           - const: allwinner,sun8i-h3-system-control
       - items:
-          - const: allwinner,sun8i-r40-system-control
-          - const: allwinner,sun4i-a10-system-control
-      - const: allwinner,sun50i-a64-sram-controller
-        deprecated: true
-      - const: allwinner,sun50i-a64-system-control
-      - const: allwinner,sun50i-h5-system-control
-      - items:
           - const: allwinner,sun50i-h6-system-control
           - const: allwinner,sun50i-a64-system-control
-      - items:
-          - const: allwinner,suniv-f1c100s-system-control
-          - const: allwinner,sun4i-a10-system-control
-      - const: allwinner,sun50i-h616-system-control
 
   reg:
     maxItems: 1
@@ -76,43 +75,26 @@ patternProperties:
               - const: allwinner,sun4i-a10-sram-d
               - const: allwinner,sun50i-a64-sram-c
               - items:
-                  - const: allwinner,sun5i-a13-sram-a3-a4
-                  - const: allwinner,sun4i-a10-sram-a3-a4
-              - items:
-                  - const: allwinner,sun7i-a20-sram-a3-a4
+                  - enum:
+                      - allwinner,sun5i-a13-sram-a3-a4
+                      - allwinner,sun7i-a20-sram-a3-a4
                   - const: allwinner,sun4i-a10-sram-a3-a4
               - items:
-                  - const: allwinner,sun5i-a13-sram-c1
-                  - const: allwinner,sun4i-a10-sram-c1
-              - items:
-                  - const: allwinner,sun7i-a20-sram-c1
-                  - const: allwinner,sun4i-a10-sram-c1
-              - items:
-                  - const: allwinner,sun8i-a23-sram-c1
-                  - const: allwinner,sun4i-a10-sram-c1
-              - items:
-                  - const: allwinner,sun8i-h3-sram-c1
+                  - enum:
+                      - allwinner,sun5i-a13-sram-c1
+                      - allwinner,sun7i-a20-sram-c1
+                      - allwinner,sun8i-a23-sram-c1
+                      - allwinner,sun8i-h3-sram-c1
+                      - allwinner,sun8i-r40-sram-c1
+                      - allwinner,sun50i-a64-sram-c1
+                      - allwinner,sun50i-h5-sram-c1
+                      - allwinner,sun50i-h6-sram-c1
                   - const: allwinner,sun4i-a10-sram-c1
               - items:
-                  - const: allwinner,sun8i-r40-sram-c1
-                  - const: allwinner,sun4i-a10-sram-c1
-              - items:
-                  - const: allwinner,sun50i-a64-sram-c1
-                  - const: allwinner,sun4i-a10-sram-c1
-              - items:
-                  - const: allwinner,sun50i-h5-sram-c1
-                  - const: allwinner,sun4i-a10-sram-c1
-              - items:
-                  - const: allwinner,sun50i-h6-sram-c1
-                  - const: allwinner,sun4i-a10-sram-c1
-              - items:
-                  - const: allwinner,sun5i-a13-sram-d
-                  - const: allwinner,sun4i-a10-sram-d
-              - items:
-                  - const: allwinner,sun7i-a20-sram-d
-                  - const: allwinner,sun4i-a10-sram-d
-              - items:
-                  - const: allwinner,suniv-f1c100s-sram-d
+                  - enum:
+                      - allwinner,suniv-f1c100s-sram-d
+                      - allwinner,sun5i-a13-sram-d
+                      - allwinner,sun7i-a20-sram-d
                   - const: allwinner,sun4i-a10-sram-d
               - items:
                   - const: allwinner,sun50i-h6-sram-c
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml
index 2f0151e9f6be..1faea7d315c0 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.yaml
+++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml
@@ -105,6 +105,8 @@ patternProperties:
     description: AMS-Taos Inc.
   "^analogix,.*":
     description: Analogix Semiconductor, Inc.
+  "^anbernic,.*":
+    description: Anbernic
   "^andestech,.*":
     description: Andes Technology Corporation
   "^anvo,.*":
@@ -787,6 +789,8 @@ patternProperties:
     description: Cisco Meraki, LLC
   "^merrii,.*":
     description: Merrii Technology Co., Ltd.
+  "^methode,.*":
+    description: Methode Electronics, Inc.
   "^micrel,.*":
     description: Micrel Inc.
   "^microchip,.*":
@@ -819,6 +823,8 @@ patternProperties:
     description: MNT Research GmbH
   "^modtronix,.*":
     description: Modtronix Engineering
+  "^moortec,.*":
+    description: Moortec Semiconductor Ltd.
   "^mosaixtech,.*":
     description: Mosaix Technologies, Inc.
   "^motorola,.*":
@@ -925,6 +931,8 @@ patternProperties:
     description: On Tat Industrial Company
   "^opalkelly,.*":
     description: Opal Kelly Incorporated
+  "^openailab,.*":
+    description: openailab.com
   "^opencores,.*":
     description: OpenCores.org
   "^openembed,.*":
diff --git a/Documentation/doc-guide/kernel-doc.rst b/Documentation/doc-guide/kernel-doc.rst
index 9c779bd7a751..1dcbd7332476 100644
--- a/Documentation/doc-guide/kernel-doc.rst
+++ b/Documentation/doc-guide/kernel-doc.rst
@@ -14,6 +14,9 @@ when it is embedded in source files.
    reasons. The kernel source contains tens of thousands of kernel-doc
    comments. Please stick to the style described here.
 
+.. note:: kernel-doc does not cover Rust code: please see
+   Documentation/rust/general-information.rst instead.
+
 The kernel-doc structure is extracted from the comments, and proper
 `Sphinx C Domain`_ function and type descriptions with anchors are
 generated from them. The descriptions are filtered for special kernel-doc
diff --git a/Documentation/doc-guide/sphinx.rst b/Documentation/doc-guide/sphinx.rst
index 1228b85f6f77..c708cec889af 100644
--- a/Documentation/doc-guide/sphinx.rst
+++ b/Documentation/doc-guide/sphinx.rst
@@ -48,10 +48,6 @@ or ``virtualenv``, depending on how your distribution packaged Python 3.
       on the Sphinx version, it should be installed separately,
       with ``pip install sphinx_rtd_theme``.
 
-   #) Some ReST pages contain math expressions. Due to the way Sphinx works,
-      those expressions are written using LaTeX notation. It needs texlive
-      installed with amsfonts and amsmath in order to evaluate them.
-
 In summary, if you want to install Sphinx version 2.4.4, you should do::
 
        $ virtualenv sphinx_2.4.4
@@ -86,6 +82,27 @@ Depending on the distribution, you may also need to install a series of
 ``texlive`` packages that provide the minimal set of functionalities
 required for ``XeLaTeX`` to work.
 
+Math Expressions in HTML
+------------------------
+
+Some ReST pages contain math expressions. Due to the way Sphinx works,
+those expressions are written using LaTeX notation.
+There are two options for Sphinx to render math expressions in html output.
+One is an extension called `imgmath`_ which converts math expressions into
+images and embeds them in html pages.
+The other is an extension called `mathjax`_ which delegates math rendering
+to JavaScript capable web browsers.
+The former was the only option for pre-6.1 kernel documentation and it
+requires quite a few texlive packages including amsfonts and amsmath among
+others.
+
+Since kernel release 6.1, html pages with math expressions can be built
+without installing any texlive packages. See `Choice of Math Renderer`_ for
+further info.
+
+.. _imgmath: https://www.sphinx-doc.org/en/master/usage/extensions/math.html#module-sphinx.ext.imgmath
+.. _mathjax: https://www.sphinx-doc.org/en/master/usage/extensions/math.html#module-sphinx.ext.mathjax
+
 .. _sphinx-pre-install:
 
 Checking for Sphinx dependencies
@@ -164,6 +181,38 @@ To remove the generated documentation, run ``make cleandocs``.
 	  as well would improve the quality of images embedded in PDF
 	  documents, especially for kernel releases 5.18 and later.
 
+Choice of Math Renderer
+-----------------------
+
+Since kernel release 6.1, mathjax works as a fallback math renderer for
+html output.\ [#sph1_8]_
+
+Math renderer is chosen depending on available commands as shown below:
+
+.. table:: Math Renderer Choices for HTML
+
+    ============= ================= ============
+    Math renderer Required commands Image format
+    ============= ================= ============
+    imgmath       latex, dvipng     PNG (raster)
+    mathjax
+    ============= ================= ============
+
+The choice can be overridden by setting an environment variable
+``SPHINX_IMGMATH`` as shown below:
+
+.. table:: Effect of Setting ``SPHINX_IMGMATH``
+
+    ====================== ========
+    Setting                Renderer
+    ====================== ========
+    ``SPHINX_IMGMATH=yes`` imgmath
+    ``SPHINX_IMGMATH=no``  mathjax
+    ====================== ========
+
+.. [#sph1_8] Fallback of math renderer requires Sphinx >=1.8.
+
+
 Writing Documentation
 =====================
 
diff --git a/Documentation/driver-api/driver-model/devres.rst b/Documentation/driver-api/driver-model/devres.rst
index 55272942e721..5e6de4ab25dc 100644
--- a/Documentation/driver-api/driver-model/devres.rst
+++ b/Documentation/driver-api/driver-model/devres.rst
@@ -301,6 +301,7 @@ IO region
   devm_release_region()
   devm_release_resource()
   devm_request_mem_region()
+  devm_request_free_mem_region()
   devm_request_region()
   devm_request_resource()
 
@@ -334,7 +335,7 @@ IRQ
   devm_irq_alloc_descs_from()
   devm_irq_alloc_generic_chip()
   devm_irq_setup_generic_chip()
-  devm_irq_sim_init()
+  devm_irq_domain_create_sim()
 
 LED
   devm_led_classdev_register()
@@ -392,7 +393,9 @@ PHY
 PINCTRL
   devm_pinctrl_get()
   devm_pinctrl_put()
+  devm_pinctrl_get_select()
   devm_pinctrl_register()
+  devm_pinctrl_register_and_init()
   devm_pinctrl_unregister()
 
 POWER
@@ -401,14 +404,24 @@ POWER
 
 PWM
   devm_pwm_get()
-  devm_of_pwm_get()
   devm_fwnode_pwm_get()
 
 REGULATOR
+  devm_regulator_bulk_register_supply_alias()
   devm_regulator_bulk_get()
+  devm_regulator_bulk_get_enable()
+  devm_regulator_bulk_put()
   devm_regulator_get()
+  devm_regulator_get_enable()
+  devm_regulator_get_enable_optional()
+  devm_regulator_get_exclusive()
+  devm_regulator_get_optional()
+  devm_regulator_irq_helper()
   devm_regulator_put()
   devm_regulator_register()
+  devm_regulator_register_notifier()
+  devm_regulator_register_supply_alias()
+  devm_regulator_unregister_notifier()
 
 RESET
   devm_reset_control_get()
@@ -427,6 +440,8 @@ SLAVE DMA ENGINE
   devm_acpi_dma_controller_register()
 
 SPI
+  devm_spi_alloc_master()
+  devm_spi_alloc_slave()
   devm_spi_register_master()
 
 WATCHDOG
diff --git a/Documentation/driver-api/isa.rst b/Documentation/driver-api/isa.rst
index def4a7b690b5..3df1b1696524 100644
--- a/Documentation/driver-api/isa.rst
+++ b/Documentation/driver-api/isa.rst
@@ -100,7 +100,7 @@ I believe platform_data is available for this, but if rather not, moving
 the isa_driver pointer to the private struct isa_dev is ofcourse fine as
 well.
 
-Then, if the the driver did not provide a .match, it matches. If it did,
+Then, if the driver did not provide a .match, it matches. If it did,
 the driver match() method is called to determine a match.
 
 If it did **not** match, dev->platform_data is reset to indicate this to
diff --git a/Documentation/driver-api/pwm.rst b/Documentation/driver-api/pwm.rst
index fd26c3d895b6..8c71a2055d27 100644
--- a/Documentation/driver-api/pwm.rst
+++ b/Documentation/driver-api/pwm.rst
@@ -40,8 +40,7 @@ after usage with pwm_free().
 
 New users should use the pwm_get() function and pass to it the consumer
 device or a consumer name. pwm_put() is used to free the PWM device. Managed
-variants of the getter, devm_pwm_get(), devm_of_pwm_get(),
-devm_fwnode_pwm_get(), also exist.
+variants of the getter, devm_pwm_get() and devm_fwnode_pwm_get(), also exist.
 
 After being requested, a PWM has to be configured using::
 
diff --git a/Documentation/fb/udlfb.rst b/Documentation/fb/udlfb.rst
index 732b37db3504..99cfbb7a1922 100644
--- a/Documentation/fb/udlfb.rst
+++ b/Documentation/fb/udlfb.rst
@@ -86,17 +86,24 @@ Module Options
 Special configuration for udlfb is usually unnecessary. There are a few
 options, however.
 
-From the command line, pass options to modprobe
-modprobe udlfb fb_defio=0 console=1 shadow=1
+From the command line, pass options to modprobe::
 
-Or modify options on the fly at /sys/module/udlfb/parameters directory via
-sudo nano fb_defio
-change the parameter in place, and save the file.
+  modprobe udlfb fb_defio=0 console=1 shadow=1
 
-Unplug/replug USB device to apply with new settings
+Or change options on the fly by editing
+/sys/module/udlfb/parameters/PARAMETER_NAME ::
 
-Or for permanent option, create file like /etc/modprobe.d/udlfb.conf with text
-options udlfb fb_defio=0 console=1 shadow=1
+  cd /sys/module/udlfb/parameters
+  ls # to see a list of parameter names
+  sudo nano PARAMETER_NAME
+  # change the parameter in place, and save the file.
+
+Unplug/replug USB device to apply with new settings.
+
+Or to apply options permanently, create a modprobe configuration file
+like /etc/modprobe.d/udlfb.conf with text::
+
+  options udlfb fb_defio=0 console=1 shadow=1
 
 Accepted boolean options:
 
diff --git a/Documentation/filesystems/caching/backend-api.rst b/Documentation/filesystems/caching/backend-api.rst
index d7507becf674..3a199fc50828 100644
--- a/Documentation/filesystems/caching/backend-api.rst
+++ b/Documentation/filesystems/caching/backend-api.rst
@@ -122,7 +122,7 @@ volumes, calling::
 to tell fscache that a volume has been withdrawn.  This waits for all
 outstanding accesses on the volume to complete before returning.
 
-When the the cache is completely withdrawn, fscache should be notified by
+When the cache is completely withdrawn, fscache should be notified by
 calling::
 
 	void fscache_relinquish_cache(struct fscache_cache *cache);
diff --git a/Documentation/filesystems/ext4/super.rst b/Documentation/filesystems/ext4/super.rst
index 268888522e35..0152888cac29 100644
--- a/Documentation/filesystems/ext4/super.rst
+++ b/Documentation/filesystems/ext4/super.rst
@@ -456,15 +456,15 @@ The ext4 superblock is laid out as follows in
    * - 0x277
      - __u8
      - s_lastcheck_hi
-     - Upper 8 bits of the s_lastcheck_hi field.
+     - Upper 8 bits of the s_lastcheck field.
    * - 0x278
      - __u8
      - s_first_error_time_hi
-     - Upper 8 bits of the s_first_error_time_hi field.
+     - Upper 8 bits of the s_first_error_time field.
    * - 0x279
      - __u8
      - s_last_error_time_hi
-     - Upper 8 bits of the s_last_error_time_hi field.
+     - Upper 8 bits of the s_last_error_time field.
    * - 0x27A
      - __u8
      - s_pad[2]
diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst
index d0c09663dae8..17df9a02ccff 100644
--- a/Documentation/filesystems/f2fs.rst
+++ b/Documentation/filesystems/f2fs.rst
@@ -286,9 +286,8 @@ compress_algorithm=%s:%d Control compress algorithm and its compress level, now,
 			 algorithm	level range
 			 lz4		3 - 16
 			 zstd		1 - 22
-compress_log_size=%u	 Support configuring compress cluster size, the size will
-			 be 4KB * (1 << %u), 16KB is minimum size, also it's
-			 default size.
+compress_log_size=%u	 Support configuring compress cluster size. The size will
+			 be 4KB * (1 << %u). The default and minimum sizes are 16KB.
 compress_extension=%s	 Support adding specified extension, so that f2fs can enable
 			 compression on those corresponding files, e.g. if all files
 			 with '.ext' has high compression rate, we can set the '.ext'
diff --git a/Documentation/filesystems/idmappings.rst b/Documentation/filesystems/idmappings.rst
index c1db8748389c..b9b31066aef2 100644
--- a/Documentation/filesystems/idmappings.rst
+++ b/Documentation/filesystems/idmappings.rst
@@ -661,7 +661,7 @@ idmappings::
  mount idmapping:      u0:k10000:r10000
 
 Assume a file owned by ``u1000`` is read from disk. The filesystem maps this id
-to ``k21000`` according to it's idmapping. This is what is stored in the
+to ``k21000`` according to its idmapping. This is what is stored in the
 inode's ``i_uid`` and ``i_gid`` fields.
 
 When the caller queries the ownership of this file via ``stat()`` the kernel
diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst
index aee9aaf9f3df..e8f370d9ce9c 100644
--- a/Documentation/filesystems/porting.rst
+++ b/Documentation/filesystems/porting.rst
@@ -922,3 +922,14 @@ is provided - file_open_root_mnt().  In-tree users adjusted.
 no_llseek is gone; don't set .llseek to that - just leave it NULL instead.
 Checks for "does that file have llseek(2), or should it fail with ESPIPE"
 should be done by looking at FMODE_LSEEK in file->f_mode.
+
+---
+
+*mandatory*
+
+filldir_t (readdir callbacks) calling conventions have changed.  Instead of
+returning 0 or -E... it returns bool now.  false means "no more" (as -E... used
+to) and true - "keep going" (as 0 in old calling conventions).  Rationale:
+callers never looked at specific -E... values anyway.  ->iterate() and
+->iterate_shared() instance require no changes at all, all filldir_t ones in
+the tree converted.
diff --git a/Documentation/filesystems/qnx6.rst b/Documentation/filesystems/qnx6.rst
index fd13433d362c..523b798f04e7 100644
--- a/Documentation/filesystems/qnx6.rst
+++ b/Documentation/filesystems/qnx6.rst
@@ -176,7 +176,7 @@ Then userspace.
 The requirement for a static, fixed preallocated system area comes from how
 qnx6fs deals with writes.
 
-Each superblock got it's own half of the system area. So superblock #1
+Each superblock got its own half of the system area. So superblock #1
 always uses blocks from the lower half while superblock #2 just writes to
 blocks represented by the upper half bitmap system area bits.
 
diff --git a/Documentation/filesystems/spufs/spufs.rst b/Documentation/filesystems/spufs/spufs.rst
index 8a42859bb100..ca0441cbe37e 100644
--- a/Documentation/filesystems/spufs/spufs.rst
+++ b/Documentation/filesystems/spufs/spufs.rst
@@ -227,7 +227,7 @@ Files
               from the data buffer, updating the value of the specified signal
               notification register.  The signal  notification  register  will
               either be replaced with the input data or will be updated to the
-              bitwise OR or the old value and the input data, depending on the
+              bitwise OR of the old value and the input data, depending on the
               contents  of  the  signal1_type,  or  signal2_type respectively,
               file.
 
diff --git a/Documentation/filesystems/xfs-delayed-logging-design.rst b/Documentation/filesystems/xfs-delayed-logging-design.rst
index 4ef419f54663..6402ab8e370c 100644
--- a/Documentation/filesystems/xfs-delayed-logging-design.rst
+++ b/Documentation/filesystems/xfs-delayed-logging-design.rst
@@ -100,7 +100,7 @@ transactions together::
 
 	ntp = xfs_trans_dup(tp);
 	xfs_trans_commit(tp);
-	xfs_log_reserve(ntp);
+	xfs_trans_reserve(ntp);
 
 This results in a series of "rolling transactions" where the inode is locked
 across the entire chain of transactions.  Hence while this series of rolling
@@ -191,7 +191,7 @@ transaction rolling mechanism to re-reserve space on every transaction roll. We
 know from the implementation of the permanent transactions how many transaction
 rolls are likely for the common modifications that need to be made.
 
-For example, and inode allocation is typically two transactions - one to
+For example, an inode allocation is typically two transactions - one to
 physically allocate a free inode chunk on disk, and another to allocate an inode
 from an inode chunk that has free inodes in it.  Hence for an inode allocation
 transaction, we might set the reservation log count to a value of 2 to indicate
@@ -200,7 +200,7 @@ chain. Each time a permanent transaction rolls, it consumes an entire unit
 reservation.
 
 Hence when the permanent transaction is first allocated, the log space
-reservation is increases from a single unit reservation to multiple unit
+reservation is increased from a single unit reservation to multiple unit
 reservations. That multiple is defined by the reservation log count, and this
 means we can roll the transaction multiple times before we have to re-reserve
 log space when we roll the transaction. This ensures that the common
@@ -259,7 +259,7 @@ the next transaction in the sequeunce, but we have none remaining. We cannot
 sleep during the transaction commit process waiting for new log space to become
 available, as we may end up on the end of the FIFO queue and the items we have
 locked while we sleep could end up pinning the tail of the log before there is
-enough free space in the log to fulfil all of the pending reservations and
+enough free space in the log to fulfill all of the pending reservations and
 then wake up transaction commit in progress.
 
 To take a new reservation without sleeping requires us to be able to take a
@@ -551,14 +551,14 @@ Essentially, this shows that an item that is in the AIL can still be modified
 and relogged, so any tracking must be separate to the AIL infrastructure. As
 such, we cannot reuse the AIL list pointers for tracking committed items, nor
 can we store state in any field that is protected by the AIL lock. Hence the
-committed item tracking needs it's own locks, lists and state fields in the log
+committed item tracking needs its own locks, lists and state fields in the log
 item.
 
 Similar to the AIL, tracking of committed items is done through a new list
 called the Committed Item List (CIL).  The list tracks log items that have been
 committed and have formatted memory buffers attached to them. It tracks objects
 in transaction commit order, so when an object is relogged it is removed from
-it's place in the list and re-inserted at the tail. This is entirely arbitrary
+its place in the list and re-inserted at the tail. This is entirely arbitrary
 and done to make it easy for debugging - the last items in the list are the
 ones that are most recently modified. Ordering of the CIL is not necessary for
 transactional integrity (as discussed in the next section) so the ordering is
@@ -615,7 +615,7 @@ those changes into the current checkpoint context. We then initialise a new
 context and attach that to the CIL for aggregation of new transactions.
 
 This allows us to unlock the CIL immediately after transfer of all the
-committed items and effectively allow new transactions to be issued while we
+committed items and effectively allows new transactions to be issued while we
 are formatting the checkpoint into the log. It also allows concurrent
 checkpoints to be written into the log buffers in the case of log force heavy
 workloads, just like the existing transaction commit code does. This, however,
@@ -884,9 +884,9 @@ pin the object the first time it is inserted into the CIL - if it is already in
 the CIL during a transaction commit, then we do not pin it again. Because there
 can be multiple outstanding checkpoint contexts, we can still see elevated pin
 counts, but as each checkpoint completes the pin count will retain the correct
-value according to it's context.
+value according to its context.
 
-Just to make matters more slightly more complex, this checkpoint level context
+Just to make matters slightly more complex, this checkpoint level context
 for the pin count means that the pinning of an item must take place under the
 CIL commit/flush lock. If we pin the object outside this lock, we cannot
 guarantee which context the pin count is associated with. This is because of
diff --git a/Documentation/firmware-guide/acpi/enumeration.rst b/Documentation/firmware-guide/acpi/enumeration.rst
index dbb03022b127..b9dc0c603f36 100644
--- a/Documentation/firmware-guide/acpi/enumeration.rst
+++ b/Documentation/firmware-guide/acpi/enumeration.rst
@@ -21,7 +21,7 @@ possible we decided to do following:
   - Devices behind real busses where there is a connector resource
     are represented as struct spi_device or struct i2c_device. Note
     that standard UARTs are not busses so there is no struct uart_device,
-    although some of them may be represented by sturct serdev_device.
+    although some of them may be represented by struct serdev_device.
 
 As both ACPI and Device Tree represent a tree of devices (and their
 resources) this implementation follows the Device Tree way as much as
@@ -205,7 +205,7 @@ Here is what the ACPI namespace for a SPI slave might look like::
 		}
 		...
 
-The SPI device drivers only need to add ACPI IDs in a similar way than with
+The SPI device drivers only need to add ACPI IDs in a similar way to
 the platform device drivers. Below is an example where we add ACPI support
 to at25 SPI eeprom driver (this is meant for the above ACPI snippet)::
 
@@ -362,7 +362,7 @@ These GPIO numbers are controller relative and path "\\_SB.PCI0.GPI0"
 specifies the path to the controller. In order to use these GPIOs in Linux
 we need to translate them to the corresponding Linux GPIO descriptors.
 
-There is a standard GPIO API for that and is documented in
+There is a standard GPIO API for that and it is documented in
 Documentation/admin-guide/gpio/.
 
 In the above example we can get the corresponding two GPIO descriptors with
@@ -538,8 +538,8 @@ information.
 PCI hierarchy representation
 ============================
 
-Sometimes could be useful to enumerate a PCI device, knowing its position on the
-PCI bus.
+Sometimes it could be useful to enumerate a PCI device, knowing its position on
+the PCI bus.
 
 For example, some systems use PCI devices soldered directly on the mother board,
 in a fixed position (ethernet, Wi-Fi, serial ports, etc.). In this conditions it
@@ -550,7 +550,7 @@ To identify a PCI device, a complete hierarchical description is required, from
 the chipset root port to the final device, through all the intermediate
 bridges/switches of the board.
 
-For example, let us assume to have a system with a PCIe serial port, an
+For example, let's assume we have a system with a PCIe serial port, an
 Exar XR17V3521, soldered on the main board. This UART chip also includes
 16 GPIOs and we want to add the property ``gpio-line-names`` [1] to these pins.
 In this case, the ``lspci`` output for this component is::
@@ -593,8 +593,8 @@ of the chipset bridge (also called "root port") with address::
 
 	Bus: 0 - Device: 14 - Function: 1
 
-To find this information is necessary disassemble the BIOS ACPI tables, in
-particular the DSDT (see also [2])::
+To find this information, it is necessary to disassemble the BIOS ACPI tables,
+in particular the DSDT (see also [2])::
 
 	mkdir ~/tables/
 	cd ~/tables/
diff --git a/Documentation/firmware-guide/acpi/osi.rst b/Documentation/firmware-guide/acpi/osi.rst
index 05869c0045d7..784850adfcb6 100644
--- a/Documentation/firmware-guide/acpi/osi.rst
+++ b/Documentation/firmware-guide/acpi/osi.rst
@@ -41,26 +41,23 @@ But it is likely that they will all eventually be added.
 What should an OEM do if they want to support Linux and Windows
 using the same BIOS image?  Often they need to do something different
 for Linux to deal with how Linux is different from Windows.
-Here the BIOS should ask exactly what it wants to know:
 
+In this case, the OEM should create custom ASL to be executed by the
+Linux kernel and changes to Linux kernel drivers to execute this custom
+ASL.  The easiest way to accomplish this is to introduce a device specific
+method (_DSM) that is called from the Linux kernel.
+
+In the past the kernel used to support something like:
 _OSI("Linux-OEM-my_interface_name")
 where 'OEM' is needed if this is an OEM-specific hook,
 and 'my_interface_name' describes the hook, which could be a
 quirk, a bug, or a bug-fix.
 
-In addition, the OEM should send a patch to upstream Linux
-via the linux-acpi@vger.kernel.org mailing list.  When that patch
-is checked into Linux, the OS will answer "YES" when the BIOS
-on the OEM's system uses _OSI to ask if the interface is supported
-by the OS.  Linux distributors can back-port that patch for Linux
-pre-installs, and it will be included by all distributions that
-re-base to upstream.  If the distribution can not update the kernel binary,
-they can also add an acpi_osi=Linux-OEM-my_interface_name
-cmdline parameter to the boot loader, as needed.
-
-If the string refers to a feature where the upstream kernel
-eventually grows support, a patch should be sent to remove
-the string when that support is added to the kernel.
+However this was discovered to be abused by other BIOS vendors to change
+completely unrelated code on completely unrelated systems.  This prompted
+an evaluation of all of it's uses. This uncovered that they aren't needed
+for any of the original reasons. As such, the kernel will not respond to
+any custom Linux-* strings by default.
 
 That was easy.  Read on, to find out how to do it wrong.
 
diff --git a/Documentation/gpu/amdgpu/apu-asic-info-table.csv b/Documentation/gpu/amdgpu/apu-asic-info-table.csv
new file mode 100644
index 000000000000..98c6988e424e
--- /dev/null
+++ b/Documentation/gpu/amdgpu/apu-asic-info-table.csv
@@ -0,0 +1,8 @@
+Product Name, Code Reference, DCN/DCE version, GC version, VCE/UVD/VCN version, SDMA version
+Radeon R* Graphics, CARRIZO/STONEY, DCE 11, 8, VCE 3 / UVD 6, 3
+Ryzen 3000 series / AMD Ryzen Embedded V1*/R1* with Radeon Vega Gfx, RAVEN/PICASSO, DCN 1.0, 9.1.0, VCN 1.0, 4.1.0
+Ryzen 4000 series, RENOIR, DCN 2.1, 9.3, VCN 2.2, 4.1.2
+Ryzen 3000 series / AMD Ryzen Embedded V1*/R1* with Radeon Vega Gfx, RAVEN2, DCN 1.0, 9.2.2, VCN 1.0.1, 4.1.1
+SteamDeck, VANGOGH, DCN 3.0.1, 10.3.1, VCN 3.1.0, 5.2.1
+Ryzen 5000 series, GREEN SARDINE, DCN 2.1, 9.3, VCN 2.2, 4.1.1
+Ryzen 6000 Zen, YELLOW CARP, 3.1.2, 10.3.3, VCN 3.1.1, 5.2.3
diff --git a/Documentation/gpu/amdgpu/dgpu-asic-info-table.csv b/Documentation/gpu/amdgpu/dgpu-asic-info-table.csv
new file mode 100644
index 000000000000..84617aa35dab
--- /dev/null
+++ b/Documentation/gpu/amdgpu/dgpu-asic-info-table.csv
@@ -0,0 +1,24 @@
+Product Name, Code Reference, DCN/DCE version, GC version, VCN version, SDMA version
+AMD Radeon (TM) HD 8500M/ 8600M /M200 /M320 /M330 /M335 Series, HAINAN, --,  6, --, --
+AMD Radeon HD 7800 /7900 /FireGL Series, TAHITI, DCE 6, 6, VCE 1 / UVD 3, --
+AMD Radeon R7 (TM|HD) M265 /M370 /8500M /8600 /8700 /8700M, OLAND, DCE 6, 6, VCE 1 / UVD 3, --
+AMD Radeon (TM) (HD|R7) 7800 /7970 /8800 /8970 /370/ Series, PITCAIRN, DCE 6, 6, VCE 1 / UVD 3, --
+AMD Radeon (TM|R7|R9|HD) E8860 /M360 /7700 /7800 /8800 /9000(M) /W4100 Series, VERDE, DCE 6, 6, VCE 1 / UVD 3, --
+AMD Radeon HD M280X /M380 /7700 /8950 /W5100, BONAIRE, DCE 8, 7, VCE 2 / UVD 4.2, 1
+AMD Radeon (R9|TM) 200 /390 /W8100 /W9100 Series, HAWAII, DCE 8, 7, VCE 2 / UVD 4.2, 1
+AMD Radeon (TM) R(5|7) M315 /M340 /M360, TOPAZ, *, 8, --, 2
+AMD Radeon (TM) R9 200 /380 /W7100 /S7150 /M390 /M395 Series, TONGA, DCE 10, 8, VCE 3 / UVD 5, 3
+AMD Radeon (FirePro) (TM) R9 Fury Series, FIJI, DCE 10, 8, VCE 3 / UVD 6, 3
+Radeon RX 470 /480 /570 /580 /590 Series - AMD Radeon (TM) (Pro WX) 5100 /E9390 /E9560 /E9565 /V7350 /7100 /P30PH, POLARIS10, DCE 11.2, 8, VCE 3.4 / UVD 6.3, 3
+Radeon (TM) (RX|Pro WX) E9260 /460 /V5300X /550 /560(X) Series, POLARIS11, DCE 11.2, 8, VCE 3.4 / UVD 6.3, 3
+Radeon (RX/Pro) 500 /540(X) /550 /640 /WX2100 /WX3100 /WX200 Series, POLARIS12, DCE 11.2, 8, VCE 3.4 / UVD 6.3, 3
+Radeon (RX|TM) (PRO|WX) Vega /MI25 /V320 /V340L /8200 /9100 /SSG MxGPU, VEGA10, DCE 12, 9.0.1, VCE 4.0.0 / UVD 7.0.0, 4.0.0
+AMD Radeon (Pro) VII /MI50 /MI60, VEGA20, DCE 12, 9.4.0, VCE 4.1.0 / UVD 7.2.0, 4.2.0
+MI100, ARCTURUS, *, 9.4.1, VCN 2.5.0, 4.2.2
+MI200, ALDEBARAN, *, 9.4.2, VCN 2.6.0, 4.4.0
+AMD Radeon (RX|Pro) 5600(M|XT) /5700 (M|XT|XTB) /W5700, NAVI10, DCN 2.0.0, 10.1.10, VCN 2.0.0, 5.0.0
+AMD Radeon (Pro) 5300 /5500XTB/5500(XT|M) /W5500M /W5500, NAVI14, DCN 2.0.0, 10.1.1, VCN 2.0.2, 5.0.2
+AMD Radeon RX 6800(XT) /6900(XT) /W6800, SIENNA_CICHLID, DCN 3.0.0, 10.3.0, VCN 3.0.0, 5.2.0
+AMD Radeon RX 6700 XT / 6800M / 6700M, NAVY_FLOUNDER, DCN 3.0.0, 10.3.2, VCN 3.0.0, 5.2.2
+AMD Radeon RX 6600(XT) /6600M /W6600 /W6600M, DIMGREY_CAVEFISH, DCN 3.0.2, 10.3.4, VCN 3.0.16, 5.2.4
+AMD Radeon RX 6500M /6300M /W6500M /W6300M, BEIGE_GOBY, DCN 3.0.3, 10.3.5, VCN 3.0.33, 5.2.5
diff --git a/Documentation/gpu/amdgpu/display/dc-glossary.rst b/Documentation/gpu/amdgpu/display/dc-glossary.rst
index 116f5f0942fd..0b0ffd428dd2 100644
--- a/Documentation/gpu/amdgpu/display/dc-glossary.rst
+++ b/Documentation/gpu/amdgpu/display/dc-glossary.rst
@@ -170,7 +170,7 @@ consider asking in the amdgfx and update this page.
     MC
       Memory Controller
 
-    MPC
+    MPC/MPCC
       Multiple pipes and plane combine
 
     MPO
diff --git a/Documentation/gpu/amdgpu/display/dcn-overview.rst b/Documentation/gpu/amdgpu/display/dcn-overview.rst
index f98624d7828e..9fea6500448b 100644
--- a/Documentation/gpu/amdgpu/display/dcn-overview.rst
+++ b/Documentation/gpu/amdgpu/display/dcn-overview.rst
@@ -124,6 +124,65 @@ depth format), bit-depth reduction/dithering would kick in. In OPP, we would
 also apply a regamma function to introduce the gamma removed earlier back.
 Eventually, we output data in integer format at DIO.
 
+AMD Hardware Pipeline
+---------------------
+
+When discussing graphics on Linux, the **pipeline** term can sometimes be
+overloaded with multiple meanings, so it is important to define what we mean
+when we say **pipeline**. In the DCN driver, we use the term **hardware
+pipeline** or **pipeline** or just **pipe** as an abstraction to indicate a
+sequence of DCN blocks instantiated to address some specific configuration. DC
+core treats DCN blocks as individual resources, meaning we can build a pipeline
+by taking resources for all individual hardware blocks to compose one pipeline.
+In actuality, we can't connect an arbitrary block from one pipe to a block from
+another pipe; they are routed linearly, except for DSC, which can be
+arbitrarily assigned as needed. We have this pipeline concept for trying to
+optimize bandwidth utilization.
+
+.. kernel-figure:: pipeline_4k_no_split.svg
+
+Additionally, let's take a look at parts of the DTN log (see
+'Documentation/gpu/amdgpu/display/dc-debug.rst' for more information) since
+this log can help us to see part of this pipeline behavior in real-time::
+
+ HUBP:  format  addr_hi  width  height ...
+ [ 0]:      8h      81h   3840    2160
+ [ 1]:      0h       0h      0       0
+ [ 2]:      0h       0h      0       0
+ [ 3]:      0h       0h      0       0
+ [ 4]:      0h       0h      0       0
+ ...
+ MPCC:  OPP  DPP ...
+ [ 0]:   0h   0h ...
+
+The first thing to notice from the diagram and DTN log it is the fact that we
+have different clock domains for each part of the DCN blocks. In this example,
+we have just a single **pipeline** where the data flows from DCHUB to DIO, as
+we intuitively expect. Nonetheless, DCN is flexible, as mentioned before, and
+we can split this single pipe differently, as described in the below diagram:
+
+.. kernel-figure:: pipeline_4k_split.svg
+
+Now, if we inspect the DTN log again we can see some interesting changes::
+
+ HUBP:  format  addr_hi  width  height ...
+ [ 0]:      8h      81h   1920    2160 ...
+ ...
+ [ 4]:      0h       0h      0       0 ...
+ [ 5]:      8h      81h   1920    2160 ...
+ ...
+ MPCC:  OPP  DPP ...
+ [ 0]:   0h   0h ...
+ [ 5]:   0h   5h ...
+
+From the above example, we now split the display pipeline into two vertical
+parts of 1920x2160 (i.e., 3440x2160), and as a result, we could reduce the
+clock frequency in the DPP part. This is not only useful for saving power but
+also to better handle the required throughput. The idea to keep in mind here is
+that the pipe configuration can vary a lot according to the display
+configuration, and it is the DML's responsibility to set up all required
+configuration parameters for multiple scenarios supported by our hardware.
+
 Global Sync
 -----------
 
diff --git a/Documentation/gpu/amdgpu/display/dcn2_cm_drm_current.svg b/Documentation/gpu/amdgpu/display/dcn2_cm_drm_current.svg
new file mode 100644
index 000000000000..315ffc5a1a4b
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/dcn2_cm_drm_current.svg
@@ -0,0 +1,1370 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   version="1.1"
+   id="svg2019"
+   width="1702"
+   height="1845"
+   viewBox="0 0 1702 1845"
+   sodipodi:docname="dcn2_cm_drm_current.svg"
+   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs2023" />
+  <sodipodi:namedview
+     id="namedview2021"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     inkscape:zoom="0.56413987"
+     inkscape:cx="1003.2973"
+     inkscape:cy="691.31792"
+     inkscape:window-width="1920"
+     inkscape:window-height="1011"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g2025" />
+  <g
+     inkscape:groupmode="layer"
+     inkscape:label="Image"
+     id="g2025">
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect34"
+       width="208.83351"
+       height="486.09872"
+       x="0.90158081"
+       y="132.77872" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1019"
+       width="126.38867"
+       height="55.320732"
+       x="25.960823"
+       y="188.06937" />
+    <rect
+       style="fill:#d0cee2;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1021"
+       width="126.38867"
+       height="55.320732"
+       x="25.960823"
+       y="346.06937" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1103"
+       width="126.38867"
+       height="55.320732"
+       x="25.960823"
+       y="266.06937" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1105"
+       width="126.38867"
+       height="55.320732"
+       x="25.960823"
+       y="426.06937" />
+    <rect
+       style="fill:#e6e6e6;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1107"
+       width="126.38867"
+       height="55.320732"
+       x="25.960823"
+       y="506.06934" />
+    <path
+       d="m 402.61883,171.88069 c 0.025,10.97895 0.01,1001.03791 0.049,1012.01891 2.5462,-1.1592 5.0637,-2.3779 7.6202,-3.5187 -2.7411,4.8298 -5.4987,9.6453 -8.2624,14.4607 -2.7925,-4.8257 -5.5993,-9.6453 -8.3856,-14.4751 2.6221,1.1736 5.2217,2.3923 7.8398,3.5721 0,-10.8825 -0.035,-1000.84096 -0.031,-1011.72348 0.053,-0.30366 -0.2175,-0.50268 -0.3652,-0.72837 l 1.5347,0.39394 z"
+       id="path381056"
+       sodipodi:nodetypes="cccccccccc"
+       style="stroke-width:0.205176"
+       inkscape:transform-center-x="-325.50819"
+       inkscape:transform-center-y="-121.97185" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect1131"
+       width="239.47592"
+       height="1015.9909"
+       x="298.65631"
+       y="82.399658" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-width:1.27861;stroke-opacity:1"
+       id="rect1133"
+       width="340.44653"
+       height="525.77448"
+       x="298.79562"
+       y="1142.5299" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect1237"
+       width="284.00113"
+       height="945.41412"
+       x="827.75012"
+       y="82.880798"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect1239"
+       width="239.37569"
+       height="179.03308"
+       x="1356.9507"
+       y="98.758331" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-width:1.00744;stroke-opacity:1"
+       id="rect1241"
+       width="239.4184"
+       height="713.64905"
+       x="1356.9543"
+       y="307.7951"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect1345"
+       width="330.32059"
+       height="409.44757"
+       x="1363.8723"
+       y="1147.7572" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect1347"
+       width="369.8941"
+       height="682.32245"
+       x="825.72528"
+       y="1146.6448"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1349"
+       width="163.95746"
+       height="40.144867"
+       x="1389.9988"
+       y="190.80583" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-width:0.964132;stroke-opacity:1"
+       id="rect1619"
+       width="271.99091"
+       height="42.038273"
+       x="1393.981"
+       y="1197.161"
+       inkscape:connector-avoid="true" />
+    <path
+       d="m 1469.3028,1238.8328 c 0.025,10.9789 0.01,175.0378 0.049,186.0188 2.5462,-1.1592 5.0637,-2.3779 7.6202,-3.5187 -2.7411,4.8298 -5.4987,9.6453 -8.2624,14.4607 -2.7925,-4.8257 -5.5993,-9.6453 -8.3856,-14.4751 2.6221,1.1736 5.2217,2.3923 7.8398,3.5721 0,-10.8825 -0.035,-174.8409 -0.031,-185.7234 0.053,-0.3037 -0.2175,-0.5027 -0.3652,-0.7284 l 1.5347,0.394 z"
+       id="path106-1"
+       sodipodi:nodetypes="cccccccccc"
+       style="stroke-width:0.205176"
+       inkscape:transform-center-x="-325.50819"
+       inkscape:transform-center-y="-121.97185" />
+    <path
+       d="m 407.4759,1238.4728 c 0.025,10.9789 0.01,291.0378 0.049,302.0188 2.5462,-1.1592 5.0637,-2.3779 7.6202,-3.5187 -2.7411,4.8298 -5.4987,9.6453 -8.2624,14.4607 -2.7925,-4.8257 -5.5993,-9.6453 -8.3856,-14.4751 2.6221,1.1736 5.2217,2.3923 7.8398,3.5721 0,-10.8825 -0.035,-290.8409 -0.031,-301.7234 0.053,-0.3037 -0.2175,-0.5027 -0.3652,-0.7284 l 1.5347,0.394 z"
+       id="path106-1-7"
+       sodipodi:nodetypes="cccccccccc"
+       style="stroke-width:0.205176"
+       inkscape:transform-center-x="-325.50819"
+       inkscape:transform-center-y="-121.97185" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1623"
+       width="137.32646"
+       height="41.782684"
+       x="1396.3848"
+       y="1268.2837"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1705"
+       width="137.32646"
+       height="41.782684"
+       x="1396.3848"
+       y="1348.2837" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1707"
+       width="137.32646"
+       height="41.782684"
+       x="1396.3848"
+       y="1432.2837" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1731"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="190.42665"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1813"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="254.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1815"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="318.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1817"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="412.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1819"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="560.42664" />
+    <rect
+       style="fill:#e6e6e6;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1821"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="674.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1823"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="744.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1825"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="820.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#d0cee2;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1827"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="890.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1829"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="956.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1831"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1256.4268" />
+    <rect
+       style="fill:#d0cee2;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1833"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1402.4268" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1915"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1330.4268"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1917"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1478.4268" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1919"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1548.4268" />
+    <rect
+       style="fill:#eeeeee;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1921"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1614.4268" />
+    <rect
+       style="fill:#eeeeee;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1923"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1686.4268" />
+    <rect
+       style="fill:#eeeeee;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1925"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1754.4268" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect1925-3"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="134.37756" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect1954"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="276.37756"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.87333333;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:none"
+       id="rect1956"
+       width="168.66573"
+       height="122.14091"
+       x="325.50134"
+       y="358.6615"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#e6e6e6;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect1958"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="672.37756" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect1960"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="744.37756" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.965912;stroke-opacity:1"
+       id="rect1962"
+       width="169.14041"
+       height="54.756817"
+       x="325.26404"
+       y="814.42413"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#d0cee2;fill-opacity:1;stroke:#000000;stroke-width:0.965912;stroke-opacity:1"
+       id="rect1964"
+       width="169.14041"
+       height="54.756817"
+       x="325.26404"
+       y="886.42413"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.965912;stroke-opacity:1"
+       id="rect1966"
+       width="169.14041"
+       height="54.756817"
+       x="325.26404"
+       y="952.42413"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:1.02091;stroke-opacity:1"
+       id="rect2428"
+       width="169.0854"
+       height="61.19017"
+       x="325.29156"
+       y="1468.4518" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-width:0.839627;stroke-opacity:1"
+       id="rect2430"
+       width="169.26669"
+       height="41.344128"
+       x="325.20093"
+       y="1550.3611" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-width:1.13491;stroke-opacity:1"
+       id="rect2432"
+       width="287.86237"
+       height="44.416805"
+       x="325.34854"
+       y="1194.5088" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-width:1.13102;stroke-opacity:1"
+       id="rect2434"
+       width="285.86655"
+       height="44.4207"
+       x="867.34656"
+       y="1194.5068" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2509" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2511" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2513" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2515" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2517" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2519" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2521" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2523" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2525" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2527" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2529" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2531" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 442.80064,179.02883 0.91401,36.01283 423.64827,-2.29193"
+       id="path6876"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1925-3"
+       inkscape:connection-end="#rect1731"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 494.45103,156.71704 205.18886,-0.43622 0.59548,162.1112 167.12755,0.0346"
+       id="path6878"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1925-3"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 447.83425,321.05653 v 19.69319 h 419.52867"
+       id="path7294"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1954"
+       inkscape:connection-end="#rect1815"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 494.45103,298.71704 H 659.10058 V 434.74972 H 867.36292"
+       id="path7296"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-end="#rect1817"
+       inkscape:connection-start="#rect1954"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 867.36292,560.42664 -422.77356,1.33638 0.45008,-79.26417"
+       id="path7720"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 494.45103,696.74972 H 867.36292"
+       id="path7726"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1958"
+       inkscape:connection-end="#rect1821" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 867.36292,766.74972 H 494.45103"
+       id="path7728"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1823"
+       inkscape:connection-end="#rect1960" />
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 494.45105,789.05652 H 657.82418 V 1278.7498 H 867.36292"
+       id="path7730"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-end="#rect1831"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 494.40445,841.80254 h 167.41973 v 0.94718 h 205.53874"
+       id="path8140"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1962"
+       inkscape:connection-end="#rect1825" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 494.40445,913.80254 h 167.41973 v -1.05282 h 205.53874"
+       id="path8142"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1964"
+       inkscape:connection-end="#rect1827" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 494.40445,979.80254 h 167.41973 v -1.05282 h 205.53874"
+       id="path8144"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1966"
+       inkscape:connection-end="#rect1829" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 494.37697,1500.7498 H 867.36292"
+       id="path8150"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect2428"
+       inkscape:connection-end="#rect1917" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 494.46762,1570.7498 h 372.8953"
+       id="path8152"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect2430"
+       inkscape:connection-end="#rect1919" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 1396.3848,1289.175 H 1288.1445 V 582.74972 h -198.4368"
+       id="path8154"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1623"
+       inkscape:connection-end="#rect1819" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 1396.3848,1369.175 -159.2837,-1.364 0.5816,-88.2579 -147.975,-0.8033"
+       id="path8462"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1705"
+       inkscape:connection-end="#rect1831"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 1089.7077,1500.7498 h 375.3403 v -26.6834"
+       id="path8668"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1917"
+       inkscape:connection-end="#rect1707" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 1389.9988,212.74973 H 1089.7077"
+       id="path8672"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1349"
+       inkscape:connection-end="#rect1731" />
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 1389.9988,230.9507 H 1223.331 v 109.79902 h -133.6233"
+       id="path8674"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-end="#rect1815"
+       sodipodi:nodetypes="cccc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="58.63356"
+       y="69.574417"
+       id="text28675"><tspan
+         sodipodi:role="line"
+         id="tspan28673"
+         x="58.63356"
+         y="69.574417" /></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="56.388252"
+       y="221.86568"
+       id="text31913"><tspan
+         sodipodi:role="line"
+         id="tspan31911"
+         x="56.388252"
+         y="221.86568">Matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="53.441296"
+       y="300.866"
+       id="text38843"><tspan
+         sodipodi:role="line"
+         id="tspan38841"
+         x="53.441296"
+         y="300.866">1D LUT</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="53.441296"
+       y="378.866"
+       id="text41049"><tspan
+         sodipodi:role="line"
+         id="tspan41047"
+         x="53.441296"
+         y="378.866">3D LUT</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="40.831493"
+       y="460.87411"
+       id="text45037"><tspan
+         sodipodi:role="line"
+         id="tspan45035"
+         x="40.831493"
+         y="460.87411"
+         style="font-size:18.6667px">Unpacking</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="62.130371"
+       y="540.68872"
+       id="text49945"><tspan
+         sodipodi:role="line"
+         id="tspan49943"
+         x="62.130371"
+         y="540.68872"
+         style="font-size:18.6667px">Other</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1423.2806"
+       y="128.08769"
+       id="text55719"><tspan
+         sodipodi:role="line"
+         id="tspan55717"
+         x="1423.2806"
+         y="128.08769"
+         style="font-size:18.6667px">drm_framebuffer</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1439.9988"
+       y="218.74973"
+       id="text62479"><tspan
+         sodipodi:role="line"
+         id="tspan62477"
+         x="1439.9988"
+         y="218.74973"
+         style="font-size:18.6667px">format</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1475.2806"
+       y="340.08771"
+       id="text64465"><tspan
+         sodipodi:role="line"
+         id="tspan64463"
+         x="1475.2806"
+         y="340.08771"
+         style="font-size:18.6667px">drm_plane</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1589.2806"
+       y="1178.0876"
+       id="text64469"><tspan
+         sodipodi:role="line"
+         id="tspan64467"
+         x="1589.2806"
+         y="1178.0876"
+         style="font-size:18.6667px">drm_crtc</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="872.23993"
+       y="1172.4491"
+       id="text64469-9"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2"
+         x="872.23993"
+         y="1172.4491"
+         style="font-size:18.6667px">Stream</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="328.26071"
+       y="1175.6371"
+       id="text64469-9-2"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-8"
+         x="328.26071"
+         y="1175.6371"
+         style="font-size:18.6667px">MPC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="330.14737"
+       y="114.41869"
+       id="text64469-9-9"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-7"
+         x="330.14737"
+         y="114.41869"
+         style="font-size:18.6667px">DPP</tspan></text>
+    <path
+       d="m 1488.3608,401.47792 c 0.025,10.97895 0.01,773.03788 0.049,784.01888 2.5462,-1.1592 5.0637,-2.3779 7.6202,-3.5187 -2.7411,4.8298 -5.4987,9.6453 -8.2624,14.4607 -2.7925,-4.8257 -5.5993,-9.6453 -8.3856,-14.4751 2.6221,1.1736 5.2217,2.3923 7.8398,3.5721 0,-10.8825 -0.035,-772.84093 -0.031,-783.72345 0.053,-0.30366 -0.2175,-0.50268 -0.3652,-0.72837 l 1.5347,0.39394 z"
+       id="path106"
+       sodipodi:nodetypes="cccccccccc"
+       style="stroke-width:0.205176"
+       inkscape:transform-center-x="-325.50819"
+       inkscape:transform-center-y="-121.97185" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1495.9988"
+       y="1224.7498"
+       id="text78849"><tspan
+         sodipodi:role="line"
+         id="tspan78847"
+         x="1495.9988"
+         y="1224.7498"
+         style="font-size:18.6667px">Blender</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1417.9988"
+       y="1294.7498"
+       id="text78853"><tspan
+         sodipodi:role="line"
+         id="tspan78851"
+         x="1417.9988"
+         y="1294.7498"
+         style="font-size:18.6667px">Degamma</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1451.9988"
+       y="1376.7498"
+       id="text78857"><tspan
+         sodipodi:role="line"
+         id="tspan78855"
+         x="1451.9988"
+         y="1376.7498"
+         style="font-size:18.6667px">CTM</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1427.9988"
+       y="1458.7498"
+       id="text78861"><tspan
+         sodipodi:role="line"
+         id="tspan78859"
+         x="1427.9988"
+         y="1458.7498"
+         style="font-size:18.6667px">Gamma</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="949.99878"
+       y="218.74973"
+       id="text104653"><tspan
+         sodipodi:role="line"
+         id="tspan104651"
+         x="949.99878"
+         y="218.74973"
+         style="font-size:18.6667px">format</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="911.99878"
+       y="284.74973"
+       id="text104657"><tspan
+         sodipodi:role="line"
+         id="tspan104655"
+         x="911.99878"
+         y="284.74973"
+         style="font-size:18.6667px">bias_and_scale</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="887.99878"
+       y="346.74973"
+       id="text104661"><tspan
+         sodipodi:role="line"
+         id="tspan104659"
+         x="887.99878"
+         y="346.74973"
+         style="font-size:18.6667px">color space matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="871.99878"
+       y="442.74973"
+       id="text104665"><tspan
+         sodipodi:role="line"
+         id="tspan104663"
+         x="871.99878"
+         y="442.74973"
+         style="font-size:18.6667px">input_csc_color_matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="905.99878"
+       y="586.74976"
+       id="text125075"><tspan
+         sodipodi:role="line"
+         id="tspan125073"
+         x="905.99878"
+         y="586.74976"
+         style="font-size:18.6667px">in_transfer_func</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="945.0083"
+       y="702.85938"
+       id="text117309"><tspan
+         sodipodi:role="line"
+         id="tspan117307"
+         x="945.0083"
+         y="702.85938"
+         style="font-size:18.6667px">hdr_mult</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="874.5556"
+       y="773.42615"
+       id="text117309-6"><tspan
+         sodipodi:role="line"
+         id="tspan117307-7"
+         x="874.5556"
+         y="773.42615"
+         style="font-size:18.6667px">gamut_remap_matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="913.72528"
+       y="846.55896"
+       id="text117309-5"><tspan
+         sodipodi:role="line"
+         id="tspan117307-3"
+         x="913.72528"
+         y="846.55896"
+         style="font-size:18.6667px">in_shaper_func</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="937.72528"
+       y="918.25787"
+       id="text117309-56"><tspan
+         sodipodi:role="line"
+         id="tspan117307-2"
+         x="937.72528"
+         y="918.25787"
+         style="font-size:18.6667px">lut3d_func</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="945.9895"
+       y="986.10767"
+       id="text117309-9"><tspan
+         sodipodi:role="line"
+         id="tspan117307-1"
+         x="945.9895"
+         y="986.10767"
+         style="font-size:18.6667px">blend_tf</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="975.91803"
+       y="1224.5564"
+       id="text137434"><tspan
+         sodipodi:role="line"
+         id="tspan137432"
+         x="975.91803"
+         y="1224.5564"
+         style="font-size:18.6667px">Blender</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="879.31372"
+       y="1283.4249"
+       id="text137434-2"><tspan
+         sodipodi:role="line"
+         id="tspan137432-7"
+         x="879.31372"
+         y="1283.4249"
+         style="font-size:18.6667px">gamut_remap_matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="925.31372"
+       y="1359.2747"
+       id="text137434-0"><tspan
+         sodipodi:role="line"
+         id="tspan137432-9"
+         x="925.31372"
+         y="1359.2747"
+         style="font-size:18.6667px">func_shaper</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="930.33252"
+       y="1430.2566"
+       id="text137434-3"><tspan
+         sodipodi:role="line"
+         id="tspan137432-6"
+         x="930.33252"
+         y="1430.2566"
+         style="font-size:18.6667px">lut3d_func</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="900.18158"
+       y="1504.6725"
+       id="text137434-06"><tspan
+         sodipodi:role="line"
+         id="tspan137432-2"
+         x="900.18158"
+         y="1504.6725"
+         style="font-size:18.6667px">out_transfer_func</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="904.0307"
+       y="1575.8053"
+       id="text137434-6"><tspan
+         sodipodi:role="line"
+         id="tspan137432-1"
+         x="904.0307"
+         y="1575.8053"
+         style="font-size:18.6667px">csc_color_matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="903.16278"
+       y="1642.1078"
+       id="text137434-8"><tspan
+         sodipodi:role="line"
+         id="tspan137432-79"
+         x="903.16278"
+         y="1642.1078"
+         style="font-size:18.6667px">bit_depth_param</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="939.31372"
+       y="1713.8068"
+       id="text137434-20"><tspan
+         sodipodi:role="line"
+         id="tspan137432-23"
+         x="939.31372"
+         y="1713.8068"
+         style="font-size:18.6667px">clamping</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="894.0307"
+       y="1782.5244"
+       id="text137434-7"><tspan
+         sodipodi:role="line"
+         id="tspan137432-5"
+         x="894.0307"
+         y="1782.5244"
+         style="font-size:18.6667px">output_color_space</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="869.63062"
+       y="117.7968"
+       id="text176429"><tspan
+         sodipodi:role="line"
+         id="tspan176427"
+         x="869.63062"
+         y="117.7968"
+         style="font-size:18.6667px">Plane</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="27.605465"
+       y="162.20587"
+       id="text176429-2"><tspan
+         sodipodi:role="line"
+         id="tspan176427-0"
+         x="27.605465"
+         y="162.20587"
+         style="font-size:18.6667px">Legend</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="298.63959"
+       y="59.293999"
+       id="text176429-9"><tspan
+         sodipodi:role="line"
+         id="tspan176427-2"
+         x="298.63959"
+         y="59.293999"
+         style="font-size:18.6667px">DCN 2.0</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="826.65735"
+       y="59.458061"
+       id="text176429-66"><tspan
+         sodipodi:role="line"
+         id="tspan176427-4"
+         x="826.65735"
+         y="59.458061"
+         style="font-size:18.6667px">DC Interface</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1362.9948"
+       y="59.458061"
+       id="text176429-95"><tspan
+         sodipodi:role="line"
+         id="tspan176427-04"
+         x="1362.9948"
+         y="59.458061"
+         style="font-size:18.6667px">DRM Interface</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="867.34656"
+       y="1194.5068"
+       id="text184728"><tspan
+         sodipodi:role="line"
+         id="tspan184726"
+         x="867.34656"
+         y="1194.5068" /></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="389.01593"
+       y="159.8909"
+       id="text64469-9-3"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-6"
+         x="389.01593"
+         y="159.8909"
+         style="font-size:18.6667px">CNVC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="365.88379"
+       y="303.59048"
+       id="text64469-9-1"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-2"
+         x="365.88379"
+         y="303.59048"
+         style="font-size:18.6667px">Input CSC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="409.06488"
+       y="404.95001"
+       id="text64469-9-93"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-1"
+         x="409.06488"
+         y="404.95001"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle">DeGamma</tspan><tspan
+         sodipodi:role="line"
+         x="409.06488"
+         y="428.28339"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle"
+         id="tspan233797">RAM and ROM</tspan><tspan
+         sodipodi:role="line"
+         x="409.06488"
+         y="451.61676"
+         style="font-size:10.6667px;text-align:center;text-anchor:middle"
+         id="tspan242597">(sRGB, BT2020</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="351.16684"
+       y="700.34918"
+       id="text64469-9-0"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-3"
+         x="351.16684"
+         y="700.34918"
+         style="font-size:18.6667px">HDR Multiply</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="345.16684"
+       y="771.33112"
+       id="text64469-9-6"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-10"
+         x="345.16684"
+         y="771.33112"
+         style="font-size:18.6667px">Gamut Remap</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="412.5184"
+       y="836.04877"
+       id="text64469-9-63"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-20"
+         x="412.5184"
+         y="836.04877"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle">Shaper LUT</tspan><tspan
+         sodipodi:role="line"
+         x="412.5184"
+         y="859.38214"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle"
+         id="tspan222533">RAM</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="413.54086"
+       y="908.61554"
+       id="text64469-9-61"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-55"
+         x="413.54086"
+         y="908.61554"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle">3D LUT</tspan><tspan
+         sodipodi:role="line"
+         x="413.54086"
+         y="931.94891"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle"
+         id="tspan221121">RAM</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="344.0347"
+       y="985.33319"
+       id="text64469-9-47"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-65"
+         x="344.0347"
+         y="985.33319"
+         style="font-size:18.6667px">Blend Gamma</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="433.84625"
+       y="1223.411"
+       id="text64469-9-69"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-37"
+         x="433.84625"
+         y="1223.411"
+         style="font-size:18.6667px">Blender</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="408.64996"
+       y="1492.8102"
+       id="text64469-9-8"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-84"
+         x="408.64996"
+         y="1492.8102"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle">Gamma</tspan><tspan
+         sodipodi:role="line"
+         x="408.64996"
+         y="1516.1436"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle"
+         id="tspan197733">RAM</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="385.46863"
+       y="1575.2261"
+       id="text64469-9-31"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-49"
+         x="385.46863"
+         y="1575.2261"
+         style="font-size:18.6667px">OCSC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1356.5106"
+       y="53.140579"
+       id="text273287"><tspan
+         sodipodi:role="line"
+         id="tspan273285"
+         x="1356.5106"
+         y="53.140579" /></text>
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1411"
+       width="177.28049"
+       height="38.940468"
+       x="1392.2576"
+       y="459.24118" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1411.7203"
+       y="485.93231"
+       id="text78841"><tspan
+         sodipodi:role="line"
+         id="tspan78839"
+         x="1411.7203"
+         y="485.93231"
+         style="font-size:18.6667px">color_encoding</tspan></text>
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1515"
+       width="175.11876"
+       height="43.239407"
+       x="1393.7493"
+       y="525.6983" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:17.3333px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1399.212"
+       y="553.44904"
+       id="text78845"><tspan
+         sodipodi:role="line"
+         id="tspan78843"
+         x="1399.212"
+         y="553.44904">pixel_blend_mode</tspan></text>
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect371322"
+       width="177.28049"
+       height="38.940468"
+       x="1391.6781"
+       y="385.61713" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1427.5771"
+       y="409.29523"
+       id="text78837"><tspan
+         sodipodi:role="line"
+         id="tspan78835"
+         x="1427.5771"
+         y="409.29523"
+         style="font-size:18.6667px">color_range</tspan></text>
+    <path
+       d="m 1469.6815,1334.9145 c 3.2597,-1.4841 6.4827,-3.0444 9.7556,-4.5049 -3.5093,6.1834 -7.0396,12.3483 -10.5778,18.5131 -3.575,-6.178 -7.1683,-12.3481 -10.7354,-18.5314 3.3569,1.5025 6.685,3.0627 10.0367,4.5731 0,0 0,0 1.5209,-0.05 z"
+       id="path371636-7"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 1467.8882,1256.0056 c 3.2597,-1.4841 6.4827,-3.0443 9.7556,-4.5048 -3.5093,6.1833 -7.0396,12.3482 -10.5778,18.5131 -3.575,-6.178 -7.1683,-12.3482 -10.7354,-18.5315 3.3569,1.5025 6.685,3.0628 10.0367,4.5731 0,0 0,0 1.5209,-0.05 z"
+       id="path371636-7-2"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 1487.6154,441.80981 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-7"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 1489.4088,515.3385 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-22"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 400.8254,264.26491 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-6"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.6188,346.76052 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-1"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.6188,658.80913 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-5"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.6188,732.33782 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-9"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.4121,800.48637 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-4"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.2055,875.80845 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-90"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 400.4121,942.16362 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-91"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 407.9989,1458.6579 c 3.2597,-1.4841 6.4827,-3.0444 9.7556,-4.5049 -3.5093,6.1834 -7.0396,12.3483 -10.5778,18.5131 -3.575,-6.178 -7.1683,-12.3481 -10.7354,-18.5314 3.3569,1.5025 6.685,3.0627 10.0367,4.5731 0,0 0,0 1.5209,-0.05 z"
+       id="path371636-7-59"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/display/dcn3_cm_drm_current.svg b/Documentation/gpu/amdgpu/display/dcn3_cm_drm_current.svg
new file mode 100644
index 000000000000..7299ee9b6d64
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/dcn3_cm_drm_current.svg
@@ -0,0 +1,1529 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   version="1.1"
+   id="svg2019"
+   width="1702"
+   height="1845"
+   viewBox="0 0 1702 1845"
+   sodipodi:docname="dcn3_cm_drm_current.svg"
+   inkscape:version="1.1.2 (0a00cf5339, 2022-02-04)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs2023" />
+  <sodipodi:namedview
+     id="namedview2021"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     inkscape:zoom="0.55760543"
+     inkscape:cx="1011.4679"
+     inkscape:cy="249.28021"
+     inkscape:window-width="1920"
+     inkscape:window-height="1011"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g2025" />
+  <g
+     inkscape:groupmode="layer"
+     inkscape:label="Image"
+     id="g2025">
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect34"
+       width="208.83351"
+       height="486.09872"
+       x="0.90158081"
+       y="132.77872" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1019"
+       width="126.38867"
+       height="55.320732"
+       x="25.960823"
+       y="188.06937" />
+    <rect
+       style="fill:#d0cee2;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1021"
+       width="126.38867"
+       height="55.320732"
+       x="25.960823"
+       y="346.06937" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1103"
+       width="126.38867"
+       height="55.320732"
+       x="25.960823"
+       y="266.06937" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1105"
+       width="126.38867"
+       height="55.320732"
+       x="25.960823"
+       y="426.06937" />
+    <rect
+       style="fill:#e6e6e6;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1107"
+       width="126.38867"
+       height="55.320732"
+       x="25.960823"
+       y="506.06934" />
+    <path
+       d="m 402.61883,171.88069 c 0.025,10.97895 0.01,1001.03791 0.049,1012.01891 2.5462,-1.1592 5.0637,-2.3779 7.6202,-3.5187 -2.7411,4.8298 -5.4987,9.6453 -8.2624,14.4607 -2.7925,-4.8257 -5.5993,-9.6453 -8.3856,-14.4751 2.6221,1.1736 5.2217,2.3923 7.8398,3.5721 0,-10.8825 -0.035,-1000.84096 -0.031,-1011.72348 0.053,-0.30366 -0.2175,-0.50268 -0.3652,-0.72837 l 1.5347,0.39394 z"
+       id="path381056"
+       sodipodi:nodetypes="cccccccccc"
+       style="stroke-width:0.205176"
+       inkscape:transform-center-x="-325.50819"
+       inkscape:transform-center-y="-121.97185" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect1131"
+       width="239.47592"
+       height="1015.9909"
+       x="298.65631"
+       y="82.399658" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-width:1.27861;stroke-opacity:1"
+       id="rect1133"
+       width="340.44653"
+       height="525.77448"
+       x="298.79562"
+       y="1142.5299" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect1237"
+       width="284.00113"
+       height="945.41412"
+       x="827.75012"
+       y="82.880798"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect1239"
+       width="239.37569"
+       height="179.03308"
+       x="1356.9507"
+       y="98.758331" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-width:1.00744;stroke-opacity:1"
+       id="rect1241"
+       width="239.4184"
+       height="713.64905"
+       x="1356.9543"
+       y="307.7951"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect1345"
+       width="330.32059"
+       height="409.44757"
+       x="1363.8723"
+       y="1147.7572" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-opacity:1"
+       id="rect1347"
+       width="369.8941"
+       height="682.32245"
+       x="825.72528"
+       y="1146.6448"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1349"
+       width="163.95746"
+       height="40.144867"
+       x="1389.9988"
+       y="190.80583" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-width:0.964132;stroke-opacity:1"
+       id="rect1619"
+       width="271.99091"
+       height="42.038273"
+       x="1393.981"
+       y="1197.161"
+       inkscape:connector-avoid="true" />
+    <path
+       d="m 1469.3028,1238.8328 c 0.025,10.9789 0.01,175.0378 0.049,186.0188 2.5462,-1.1592 5.0637,-2.3779 7.6202,-3.5187 -2.7411,4.8298 -5.4987,9.6453 -8.2624,14.4607 -2.7925,-4.8257 -5.5993,-9.6453 -8.3856,-14.4751 2.6221,1.1736 5.2217,2.3923 7.8398,3.5721 0,-10.8825 -0.035,-174.8409 -0.031,-185.7234 0.053,-0.3037 -0.2175,-0.5027 -0.3652,-0.7284 l 1.5347,0.394 z"
+       id="path106-1"
+       sodipodi:nodetypes="cccccccccc"
+       style="stroke-width:0.205176"
+       inkscape:transform-center-x="-325.50819"
+       inkscape:transform-center-y="-121.97185" />
+    <path
+       d="m 407.4759,1238.4728 c 0.025,10.9789 0.01,291.0378 0.049,302.0188 2.5462,-1.1592 5.0637,-2.3779 7.6202,-3.5187 -2.7411,4.8298 -5.4987,9.6453 -8.2624,14.4607 -2.7925,-4.8257 -5.5993,-9.6453 -8.3856,-14.4751 2.6221,1.1736 5.2217,2.3923 7.8398,3.5721 0,-10.8825 -0.035,-290.8409 -0.031,-301.7234 0.053,-0.3037 -0.2175,-0.5027 -0.3652,-0.7284 l 1.5347,0.394 z"
+       id="path106-1-7"
+       sodipodi:nodetypes="cccccccccc"
+       style="stroke-width:0.205176"
+       inkscape:transform-center-x="-325.50819"
+       inkscape:transform-center-y="-121.97185" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1623"
+       width="137.32646"
+       height="41.782684"
+       x="1396.3848"
+       y="1268.2837"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1705"
+       width="137.32646"
+       height="41.782684"
+       x="1396.3848"
+       y="1348.2837" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1707"
+       width="137.32646"
+       height="41.782684"
+       x="1396.3848"
+       y="1432.2837" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1731"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="190.42665"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1813"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="254.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1815"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="318.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1817"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="412.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1819"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="560.42664" />
+    <rect
+       style="fill:#e6e6e6;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1821"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="674.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1823"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="744.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1825"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="820.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#d0cee2;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1827"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="890.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1829"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="956.42664"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1831"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1256.4268" />
+    <rect
+       style="fill:#d0cee2;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1833"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1402.4268" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1915"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1330.4268"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1917"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1478.4268" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1919"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1548.4268" />
+    <rect
+       style="fill:#eeeeee;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1921"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1614.4268" />
+    <rect
+       style="fill:#eeeeee;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1923"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1686.4268" />
+    <rect
+       style="fill:#eeeeee;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1925"
+       width="222.34483"
+       height="44.64616"
+       x="867.36292"
+       y="1754.4268" />
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect1925-3"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="134.37756" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect1954"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="276.37756"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.87333333;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:none"
+       id="rect1956"
+       width="168.66573"
+       height="122.14091"
+       x="325.50134"
+       y="358.6615"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#e6e6e6;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect1958"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="672.37756" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect1960"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="744.37756" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.965912;stroke-opacity:1"
+       id="rect1962"
+       width="169.14041"
+       height="54.756817"
+       x="325.26404"
+       y="814.42413"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#d0cee2;fill-opacity:1;stroke:#000000;stroke-width:0.965912;stroke-opacity:1"
+       id="rect1964"
+       width="169.14041"
+       height="54.756817"
+       x="325.26404"
+       y="886.42413"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.965912;stroke-opacity:1"
+       id="rect1966"
+       width="169.14041"
+       height="54.756817"
+       x="325.26404"
+       y="952.42413"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect2418"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="510.37756"
+       inkscape:connector-avoid="true" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect2422"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="1262.3777" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.981369;stroke-opacity:1"
+       id="rect2424"
+       width="169.12494"
+       height="56.528603"
+       x="325.27179"
+       y="1322.432" />
+    <rect
+       style="fill:#d0cee2;fill-opacity:1;stroke:#000000;stroke-width:0.981369;stroke-opacity:1"
+       id="rect2426"
+       width="169.12494"
+       height="56.528603"
+       x="325.27179"
+       y="1396.432" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:1.02091;stroke-opacity:1"
+       id="rect2428"
+       width="169.0854"
+       height="61.19017"
+       x="325.29156"
+       y="1468.4518" />
+    <rect
+       style="fill:#fad7ac;fill-opacity:1;stroke:#000000;stroke-width:0.839627;stroke-opacity:1"
+       id="rect2430"
+       width="169.26669"
+       height="41.344128"
+       x="325.20093"
+       y="1550.3611" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-width:1.13491;stroke-opacity:1"
+       id="rect2432"
+       width="287.86237"
+       height="44.416805"
+       x="325.34854"
+       y="1194.5088" />
+    <rect
+       style="fill:#000000;fill-opacity:0;stroke:#000000;stroke-width:1.13102;stroke-opacity:1"
+       id="rect2434"
+       width="285.86655"
+       height="44.4207"
+       x="867.34656"
+       y="1194.5068" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2509" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2511" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2513" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2515" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2517" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2519" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2521" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2523" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2525" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2527" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2529" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:2.66666675;stroke-linecap:butt;stroke-linejoin:miter;stroke-opacity:1;stroke-miterlimit:4;stroke-dasharray:8.00000024, 8.00000024;stroke-dashoffset:0"
+       d="m 867.36292,235.07281 v 0"
+       id="path2531" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 449.83425,179.05653 v 33.6932 h 417.52867"
+       id="path6876"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1925-3"
+       inkscape:connection-end="#rect1731"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 494.45103,156.71704 h 216.64955 v 161.7096 h 156.26234"
+       id="path6878"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1925-3"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 451.83425,321.05653 v 19.69319 h 415.52867"
+       id="path7294"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1954"
+       inkscape:connection-end="#rect1815"
+       sodipodi:nodetypes="ccc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 494.45103,298.71704 H 659.10058 V 434.74972 H 867.36292"
+       id="path7296"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-end="#rect1817"
+       inkscape:connection-start="#rect1954"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 867.36292,560.42664 H 661.10058 V 480.8024 H 494.16705"
+       id="path7720"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 455.83425,594.37756 V 582.74972 H 867.36292"
+       id="path7724"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect2420"
+       inkscape:connection-end="#rect1819"
+       sodipodi:nodetypes="ccc" />
+    <rect
+       style="fill:#afdde9;fill-opacity:1;stroke:#000000;stroke-width:0.872749;stroke-opacity:1"
+       id="rect2420"
+       width="169.23357"
+       height="44.678963"
+       x="325.21747"
+       y="594.37756" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 494.45103,696.74972 H 867.36292"
+       id="path7726"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1958"
+       inkscape:connection-end="#rect1821" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 867.36292,766.74972 H 494.45103"
+       id="path7728"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1823"
+       inkscape:connection-end="#rect1960" />
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 494.45105,789.05652 H 657.82418 V 1278.7498 H 867.36292"
+       id="path7730"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-end="#rect1831"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 494.40445,841.80254 h 167.41973 v 0.94718 h 205.53874"
+       id="path8140"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1962"
+       inkscape:connection-end="#rect1825" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 494.40445,913.80254 h 167.41973 v -1.05282 h 205.53874"
+       id="path8142"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1964"
+       inkscape:connection-end="#rect1827" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 494.40445,979.80254 h 167.41973 v -1.05282 h 205.53874"
+       id="path8144"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1966"
+       inkscape:connection-end="#rect1829" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 494.39673,1352.7498 H 867.36292"
+       id="path8146"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect2424"
+       inkscape:connection-end="#rect1915" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 494.39673,1424.7498 H 867.36292"
+       id="path8148"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect2426"
+       inkscape:connection-end="#rect1833" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 494.37697,1500.7498 H 867.36292"
+       id="path8150"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect2428"
+       inkscape:connection-end="#rect1917" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 494.46762,1570.7498 h 372.8953"
+       id="path8152"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect2430"
+       inkscape:connection-end="#rect1919" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 1396.3848,1289.175 H 1288.1445 V 582.74972 h -198.4368"
+       id="path8154"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1623"
+       inkscape:connection-end="#rect1819" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 1396.3848,1369.175 h -156.2403 v -90.4252 h -150.4368"
+       id="path8462"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1705"
+       inkscape:connection-end="#rect1831"
+       sodipodi:nodetypes="cccc" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="m 1089.7077,1500.7498 h 375.3403 v -26.6834"
+       id="path8668"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1917"
+       inkscape:connection-end="#rect1707" />
+    <path
+       style="fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 1389.9988,212.74973 H 1089.7077"
+       id="path8672"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-start="#rect1349"
+       inkscape:connection-end="#rect1731" />
+    <path
+       style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:2;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:6, 6;stroke-dashoffset:0;stroke-opacity:1"
+       d="M 1389.9988,230.9507 H 1223.331 v 109.79902 h -133.6233"
+       id="path8674"
+       inkscape:connector-type="orthogonal"
+       inkscape:connector-curvature="0"
+       inkscape:connection-end="#rect1815"
+       sodipodi:nodetypes="cccc" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="58.63356"
+       y="69.574417"
+       id="text28675"><tspan
+         sodipodi:role="line"
+         id="tspan28673"
+         x="58.63356"
+         y="69.574417" /></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="56.388252"
+       y="221.86568"
+       id="text31913"><tspan
+         sodipodi:role="line"
+         id="tspan31911"
+         x="56.388252"
+         y="221.86568">Matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="53.441296"
+       y="300.866"
+       id="text38843"><tspan
+         sodipodi:role="line"
+         id="tspan38841"
+         x="53.441296"
+         y="300.866">1D LUT</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="53.441296"
+       y="378.866"
+       id="text41049"><tspan
+         sodipodi:role="line"
+         id="tspan41047"
+         x="53.441296"
+         y="378.866">3D LUT</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="40.831493"
+       y="460.87411"
+       id="text45037"><tspan
+         sodipodi:role="line"
+         id="tspan45035"
+         x="40.831493"
+         y="460.87411"
+         style="font-size:18.6667px">Unpacking</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="62.130371"
+       y="540.68872"
+       id="text49945"><tspan
+         sodipodi:role="line"
+         id="tspan49943"
+         x="62.130371"
+         y="540.68872"
+         style="font-size:18.6667px">Other</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1423.2806"
+       y="128.08769"
+       id="text55719"><tspan
+         sodipodi:role="line"
+         id="tspan55717"
+         x="1423.2806"
+         y="128.08769"
+         style="font-size:18.6667px">drm_framebuffer</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1439.9988"
+       y="218.74973"
+       id="text62479"><tspan
+         sodipodi:role="line"
+         id="tspan62477"
+         x="1439.9988"
+         y="218.74973"
+         style="font-size:18.6667px">format</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1475.2806"
+       y="340.08771"
+       id="text64465"><tspan
+         sodipodi:role="line"
+         id="tspan64463"
+         x="1475.2806"
+         y="340.08771"
+         style="font-size:18.6667px">drm_plane</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1589.2806"
+       y="1178.0876"
+       id="text64469"><tspan
+         sodipodi:role="line"
+         id="tspan64467"
+         x="1589.2806"
+         y="1178.0876"
+         style="font-size:18.6667px">drm_crtc</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="872.23993"
+       y="1172.4491"
+       id="text64469-9"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2"
+         x="872.23993"
+         y="1172.4491"
+         style="font-size:18.6667px">Stream</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="328.26071"
+       y="1175.6371"
+       id="text64469-9-2"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-8"
+         x="328.26071"
+         y="1175.6371"
+         style="font-size:18.6667px">MPC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="330.14737"
+       y="114.41869"
+       id="text64469-9-9"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-7"
+         x="330.14737"
+         y="114.41869"
+         style="font-size:18.6667px">DPP</tspan></text>
+    <path
+       d="m 1488.3608,401.47792 c 0.025,10.97895 0.01,773.03788 0.049,784.01888 2.5462,-1.1592 5.0637,-2.3779 7.6202,-3.5187 -2.7411,4.8298 -5.4987,9.6453 -8.2624,14.4607 -2.7925,-4.8257 -5.5993,-9.6453 -8.3856,-14.4751 2.6221,1.1736 5.2217,2.3923 7.8398,3.5721 0,-10.8825 -0.035,-772.84093 -0.031,-783.72345 0.053,-0.30366 -0.2175,-0.50268 -0.3652,-0.72837 l 1.5347,0.39394 z"
+       id="path106"
+       sodipodi:nodetypes="cccccccccc"
+       style="stroke-width:0.205176"
+       inkscape:transform-center-x="-325.50819"
+       inkscape:transform-center-y="-121.97185" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1495.9988"
+       y="1224.7498"
+       id="text78849"><tspan
+         sodipodi:role="line"
+         id="tspan78847"
+         x="1495.9988"
+         y="1224.7498"
+         style="font-size:18.6667px">Blender</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1417.9988"
+       y="1294.7498"
+       id="text78853"><tspan
+         sodipodi:role="line"
+         id="tspan78851"
+         x="1417.9988"
+         y="1294.7498"
+         style="font-size:18.6667px">Degamma</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1451.9988"
+       y="1376.7498"
+       id="text78857"><tspan
+         sodipodi:role="line"
+         id="tspan78855"
+         x="1451.9988"
+         y="1376.7498"
+         style="font-size:18.6667px">CTM</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1427.9988"
+       y="1458.7498"
+       id="text78861"><tspan
+         sodipodi:role="line"
+         id="tspan78859"
+         x="1427.9988"
+         y="1458.7498"
+         style="font-size:18.6667px">Gamma</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="949.99878"
+       y="218.74973"
+       id="text104653"><tspan
+         sodipodi:role="line"
+         id="tspan104651"
+         x="949.99878"
+         y="218.74973"
+         style="font-size:18.6667px">format</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="911.99878"
+       y="284.74973"
+       id="text104657"><tspan
+         sodipodi:role="line"
+         id="tspan104655"
+         x="911.99878"
+         y="284.74973"
+         style="font-size:18.6667px">bias_and_scale</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="887.99878"
+       y="346.74973"
+       id="text104661"><tspan
+         sodipodi:role="line"
+         id="tspan104659"
+         x="887.99878"
+         y="346.74973"
+         style="font-size:18.6667px">color space matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="871.99878"
+       y="442.74973"
+       id="text104665"><tspan
+         sodipodi:role="line"
+         id="tspan104663"
+         x="871.99878"
+         y="442.74973"
+         style="font-size:18.6667px">input_csc_color_matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="905.99878"
+       y="586.74976"
+       id="text125075"><tspan
+         sodipodi:role="line"
+         id="tspan125073"
+         x="905.99878"
+         y="586.74976"
+         style="font-size:18.6667px">in_transfer_func</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="945.0083"
+       y="702.85938"
+       id="text117309"><tspan
+         sodipodi:role="line"
+         id="tspan117307"
+         x="945.0083"
+         y="702.85938"
+         style="font-size:18.6667px">hdr_mult</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="874.5556"
+       y="773.42615"
+       id="text117309-6"><tspan
+         sodipodi:role="line"
+         id="tspan117307-7"
+         x="874.5556"
+         y="773.42615"
+         style="font-size:18.6667px">gamut_remap_matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="913.72528"
+       y="846.55896"
+       id="text117309-5"><tspan
+         sodipodi:role="line"
+         id="tspan117307-3"
+         x="913.72528"
+         y="846.55896"
+         style="font-size:18.6667px">in_shaper_func</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="937.72528"
+       y="918.25787"
+       id="text117309-56"><tspan
+         sodipodi:role="line"
+         id="tspan117307-2"
+         x="937.72528"
+         y="918.25787"
+         style="font-size:18.6667px">lut3d_func</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="945.9895"
+       y="986.10767"
+       id="text117309-9"><tspan
+         sodipodi:role="line"
+         id="tspan117307-1"
+         x="945.9895"
+         y="986.10767"
+         style="font-size:18.6667px">blend_tf</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="975.91803"
+       y="1224.5564"
+       id="text137434"><tspan
+         sodipodi:role="line"
+         id="tspan137432"
+         x="975.91803"
+         y="1224.5564"
+         style="font-size:18.6667px">Blender</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="879.31372"
+       y="1283.4249"
+       id="text137434-2"><tspan
+         sodipodi:role="line"
+         id="tspan137432-7"
+         x="879.31372"
+         y="1283.4249"
+         style="font-size:18.6667px">gamut_remap_matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="925.31372"
+       y="1359.2747"
+       id="text137434-0"><tspan
+         sodipodi:role="line"
+         id="tspan137432-9"
+         x="925.31372"
+         y="1359.2747"
+         style="font-size:18.6667px">func_shaper</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="930.33252"
+       y="1430.2566"
+       id="text137434-3"><tspan
+         sodipodi:role="line"
+         id="tspan137432-6"
+         x="930.33252"
+         y="1430.2566"
+         style="font-size:18.6667px">lut3d_func</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="900.18158"
+       y="1504.6725"
+       id="text137434-06"><tspan
+         sodipodi:role="line"
+         id="tspan137432-2"
+         x="900.18158"
+         y="1504.6725"
+         style="font-size:18.6667px">out_transfer_func</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="904.0307"
+       y="1575.8053"
+       id="text137434-6"><tspan
+         sodipodi:role="line"
+         id="tspan137432-1"
+         x="904.0307"
+         y="1575.8053"
+         style="font-size:18.6667px">csc_color_matrix</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="903.16278"
+       y="1642.1078"
+       id="text137434-8"><tspan
+         sodipodi:role="line"
+         id="tspan137432-79"
+         x="903.16278"
+         y="1642.1078"
+         style="font-size:18.6667px">bit_depth_param</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="939.31372"
+       y="1713.8068"
+       id="text137434-20"><tspan
+         sodipodi:role="line"
+         id="tspan137432-23"
+         x="939.31372"
+         y="1713.8068"
+         style="font-size:18.6667px">clamping</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="894.0307"
+       y="1782.5244"
+       id="text137434-7"><tspan
+         sodipodi:role="line"
+         id="tspan137432-5"
+         x="894.0307"
+         y="1782.5244"
+         style="font-size:18.6667px">output_color_space</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="869.63062"
+       y="117.7968"
+       id="text176429"><tspan
+         sodipodi:role="line"
+         id="tspan176427"
+         x="869.63062"
+         y="117.7968"
+         style="font-size:18.6667px">Plane</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="27.605465"
+       y="162.20587"
+       id="text176429-2"><tspan
+         sodipodi:role="line"
+         id="tspan176427-0"
+         x="27.605465"
+         y="162.20587"
+         style="font-size:18.6667px">Legend</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="298.63959"
+       y="59.293999"
+       id="text176429-9"><tspan
+         sodipodi:role="line"
+         id="tspan176427-2"
+         x="298.63959"
+         y="59.293999"
+         style="font-size:18.6667px">DCN 3.0</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="826.65735"
+       y="59.458061"
+       id="text176429-66"><tspan
+         sodipodi:role="line"
+         id="tspan176427-4"
+         x="826.65735"
+         y="59.458061"
+         style="font-size:18.6667px">DC Interface</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1362.9948"
+       y="59.458061"
+       id="text176429-95"><tspan
+         sodipodi:role="line"
+         id="tspan176427-04"
+         x="1362.9948"
+         y="59.458061"
+         style="font-size:18.6667px">DRM Interface</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="867.34656"
+       y="1194.5068"
+       id="text184728"><tspan
+         sodipodi:role="line"
+         id="tspan184726"
+         x="867.34656"
+         y="1194.5068" /></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="389.01593"
+       y="159.8909"
+       id="text64469-9-3"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-6"
+         x="389.01593"
+         y="159.8909"
+         style="font-size:18.6667px">CNVC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="365.88379"
+       y="303.59048"
+       id="text64469-9-1"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-2"
+         x="365.88379"
+         y="303.59048"
+         style="font-size:18.6667px">Input CSC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="413.06488"
+       y="390.95001"
+       id="text64469-9-93"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-1"
+         x="413.06488"
+         y="390.95001"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle">DeGamma</tspan><tspan
+         sodipodi:role="line"
+         x="413.06488"
+         y="414.28339"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle"
+         id="tspan233797">ROM</tspan><tspan
+         sodipodi:role="line"
+         x="413.06488"
+         y="437.61676"
+         style="font-size:10.6667px;text-align:center;text-anchor:middle"
+         id="tspan233799">(sRGB, BT2020, Gamma 2.2,</tspan><tspan
+         sodipodi:role="line"
+         x="413.06488"
+         y="460.95013"
+         style="font-size:10.6667px;text-align:center;text-anchor:middle"
+         id="tspan242597">PQ, HLG)</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="372.44986"
+       y="538.83807"
+       id="text64469-9-94"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-78"
+         x="372.44986"
+         y="538.83807"
+         style="font-size:18.6667px">Post CSC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:16px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="335.31775"
+       y="622.23511"
+       id="text64469-9-4"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-5"
+         x="335.31775"
+         y="622.23511"
+         style="font-size:16px">Gamma Correction</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="351.16684"
+       y="700.34918"
+       id="text64469-9-0"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-3"
+         x="351.16684"
+         y="700.34918"
+         style="font-size:18.6667px">HDR Multiply</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="345.16684"
+       y="771.33112"
+       id="text64469-9-6"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-10"
+         x="345.16684"
+         y="771.33112"
+         style="font-size:18.6667px">Gamut Remap</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="412.5184"
+       y="836.04877"
+       id="text64469-9-63"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-20"
+         x="412.5184"
+         y="836.04877"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle">Shaper LUT</tspan><tspan
+         sodipodi:role="line"
+         x="412.5184"
+         y="859.38214"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle"
+         id="tspan222533">RAM</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="413.54086"
+       y="908.61554"
+       id="text64469-9-61"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-55"
+         x="413.54086"
+         y="908.61554"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle">3D LUT</tspan><tspan
+         sodipodi:role="line"
+         x="413.54086"
+         y="931.94891"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle"
+         id="tspan221121">RAM</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="344.0347"
+       y="985.33319"
+       id="text64469-9-47"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-65"
+         x="344.0347"
+         y="985.33319"
+         style="font-size:18.6667px">Blend Gamma</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="433.84625"
+       y="1223.411"
+       id="text64469-9-69"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-37"
+         x="433.84625"
+         y="1223.411"
+         style="font-size:18.6667px">Blender</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="343.73291"
+       y="1290.2421"
+       id="text64469-9-45"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-25"
+         x="343.73291"
+         y="1290.2421"
+         style="font-size:18.6667px">Gamut Remap</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="408.74442"
+       y="1344.2428"
+       id="text64469-9-474"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-4"
+         x="408.74442"
+         y="1344.2428"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle">Shaper LUT</tspan><tspan
+         sodipodi:role="line"
+         x="408.74442"
+         y="1367.5762"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle"
+         id="tspan208303">RAM</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="407.1257"
+       y="1420.2435"
+       id="text64469-9-30"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-786"
+         x="407.1257"
+         y="1420.2435"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle">3D LUT</tspan><tspan
+         sodipodi:role="line"
+         x="407.1257"
+         y="1443.5769"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle"
+         id="tspan204749">RAM</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="408.64996"
+       y="1492.8102"
+       id="text64469-9-8"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-84"
+         x="408.64996"
+         y="1492.8102"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle">Gamma</tspan><tspan
+         sodipodi:role="line"
+         x="408.64996"
+         y="1516.1436"
+         style="font-size:18.6667px;text-align:center;text-anchor:middle"
+         id="tspan197733">RAM</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="385.46863"
+       y="1575.2261"
+       id="text64469-9-31"><tspan
+         sodipodi:role="line"
+         id="tspan64467-2-49"
+         x="385.46863"
+         y="1575.2261"
+         style="font-size:18.6667px">OCSC</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:40px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1356.5106"
+       y="53.140579"
+       id="text273287"><tspan
+         sodipodi:role="line"
+         id="tspan273285"
+         x="1356.5106"
+         y="53.140579" /></text>
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1411"
+       width="177.28049"
+       height="38.940468"
+       x="1392.2576"
+       y="459.24118" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1411.7203"
+       y="485.93231"
+       id="text78841"><tspan
+         sodipodi:role="line"
+         id="tspan78839"
+         x="1411.7203"
+         y="485.93231"
+         style="font-size:18.6667px">color_encoding</tspan></text>
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect1515"
+       width="175.11876"
+       height="43.239407"
+       x="1393.7493"
+       y="525.6983" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:17.3333px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1399.212"
+       y="553.44904"
+       id="text78845"><tspan
+         sodipodi:role="line"
+         id="tspan78843"
+         x="1399.212"
+         y="553.44904">pixel_blend_mode</tspan></text>
+    <rect
+       style="fill:#f9f7ed;fill-opacity:1;stroke:#000000;stroke-opacity:1"
+       id="rect371322"
+       width="177.28049"
+       height="38.940468"
+       x="1391.6781"
+       y="385.61713" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:18.6667px;line-height:1.25;font-family:sans-serif;fill:#000000;fill-opacity:1;stroke:none"
+       x="1427.5771"
+       y="409.29523"
+       id="text78837"><tspan
+         sodipodi:role="line"
+         id="tspan78835"
+         x="1427.5771"
+         y="409.29523"
+         style="font-size:18.6667px">color_range</tspan></text>
+    <path
+       d="m 1469.6815,1334.9145 c 3.2597,-1.4841 6.4827,-3.0444 9.7556,-4.5049 -3.5093,6.1834 -7.0396,12.3483 -10.5778,18.5131 -3.575,-6.178 -7.1683,-12.3481 -10.7354,-18.5314 3.3569,1.5025 6.685,3.0627 10.0367,4.5731 0,0 0,0 1.5209,-0.05 z"
+       id="path371636-7"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 1467.8882,1256.0056 c 3.2597,-1.4841 6.4827,-3.0443 9.7556,-4.5048 -3.5093,6.1833 -7.0396,12.3482 -10.5778,18.5131 -3.575,-6.178 -7.1683,-12.3482 -10.7354,-18.5315 3.3569,1.5025 6.685,3.0628 10.0367,4.5731 0,0 0,0 1.5209,-0.05 z"
+       id="path371636-7-2"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 1487.6154,441.80981 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-7"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 1489.4088,515.3385 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-22"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 400.8254,264.26491 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-6"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.6188,346.76052 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-1"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 404.2055,500.78482 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-0"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.6188,581.69367 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-61"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.6188,658.80913 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-5"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.6188,732.33782 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-9"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.4121,800.48637 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-4"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 402.2055,875.80845 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-90"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 400.4121,942.16362 c 3.2597,-1.48411 6.4827,-3.04435 9.7556,-4.50482 -3.5093,6.18331 -7.0396,12.34823 -10.5778,18.51307 -3.575,-6.17798 -7.1683,-12.34815 -10.7354,-18.53146 3.3569,1.50251 6.685,3.06275 10.0367,4.57314 0,0 0,0 1.5209,-0.0499 z"
+       id="path371636-7-91"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 407.9989,1311.6005 c 3.2597,-1.4841 6.4827,-3.0444 9.7556,-4.5048 -3.5093,6.1833 -7.0396,12.3482 -10.5778,18.513 -3.575,-6.1779 -7.1683,-12.3481 -10.7354,-18.5314 3.3569,1.5025 6.685,3.0627 10.0367,4.5731 0,0 0,0 1.5209,-0.05 z"
+       id="path371636-7-77"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 407.9989,1381.5424 c 3.2597,-1.4841 6.4827,-3.0443 9.7556,-4.5048 -3.5093,6.1833 -7.0396,12.3482 -10.5778,18.5131 -3.575,-6.178 -7.1683,-12.3482 -10.7354,-18.5315 3.3569,1.5025 6.685,3.0627 10.0367,4.5731 0,0 0,0 1.5209,-0.05 z"
+       id="path371636-7-11"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+    <path
+       d="m 407.9989,1458.6579 c 3.2597,-1.4841 6.4827,-3.0444 9.7556,-4.5049 -3.5093,6.1834 -7.0396,12.3483 -10.5778,18.5131 -3.575,-6.178 -7.1683,-12.3481 -10.7354,-18.5314 3.3569,1.5025 6.685,3.0627 10.0367,4.5731 0,0 0,0 1.5209,-0.05 z"
+       id="path371636-7-59"
+       sodipodi:nodetypes="cccccc"
+       style="stroke-width:0.262672"
+       inkscape:transform-center-x="-416.72543"
+       inkscape:transform-center-y="-156.15191" />
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/display/display-manager.rst b/Documentation/gpu/amdgpu/display/display-manager.rst
index 7ce31f89d9a0..b7abb18cfc82 100644
--- a/Documentation/gpu/amdgpu/display/display-manager.rst
+++ b/Documentation/gpu/amdgpu/display/display-manager.rst
@@ -40,3 +40,144 @@ Atomic Implementation
 
 .. kernel-doc:: drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
    :functions: amdgpu_dm_atomic_check amdgpu_dm_atomic_commit_tail
+
+Color Management Properties
+===========================
+
+.. kernel-doc:: drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+   :doc: overview
+
+.. kernel-doc:: drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+   :internal:
+
+
+DC Color Capabilities between DCN generations
+---------------------------------------------
+
+DRM/KMS framework defines three CRTC color correction properties: degamma,
+color transformation matrix (CTM) and gamma, and two properties for degamma and
+gamma LUT sizes. AMD DC programs some of the color correction features
+pre-blending but DRM/KMS has not per-plane color correction properties.
+
+In general, the DRM CRTC color properties are programmed to DC, as follows:
+CRTC gamma after blending, and CRTC degamma pre-blending. Although CTM is
+programmed after blending, it is mapped to DPP hw blocks (pre-blending). Other
+color caps available in the hw is not currently exposed by DRM interface and
+are bypassed.
+
+.. kernel-doc:: drivers/gpu/drm/amd/display/dc/dc.h
+   :doc: color-management-caps
+
+.. kernel-doc:: drivers/gpu/drm/amd/display/dc/dc.h
+   :internal:
+
+The color pipeline has undergone major changes between DCN hardware
+generations. What's possible to do before and after blending depends on
+hardware capabilities, as illustrated below by the DCN 2.0 and DCN 3.0 families
+schemas.
+
+**DCN 2.0 family color caps and mapping**
+
+.. kernel-figure:: dcn2_cm_drm_current.svg
+
+**DCN 3.0 family color caps and mapping**
+
+.. kernel-figure:: dcn3_cm_drm_current.svg
+
+Blend Mode Properties
+=====================
+
+Pixel blend mode is a DRM plane composition property of :c:type:`drm_plane` used to
+describes how pixels from a foreground plane (fg) are composited with the
+background plane (bg). Here, we present main concepts of DRM blend mode to help
+to understand how this property is mapped to AMD DC interface. See more about
+this DRM property and the alpha blending equations in :ref:`DRM Plane
+Composition Properties <plane_composition_properties>`.
+
+Basically, a blend mode sets the alpha blending equation for plane
+composition that fits the mode in which the alpha channel affects the state of
+pixel color values and, therefore, the resulted pixel color. For
+example, consider the following elements of the alpha blending equation:
+
+- *fg.rgb*: Each of the RGB component values from the foreground's pixel.
+- *fg.alpha*: Alpha component value from the foreground's pixel.
+- *bg.rgb*: Each of the RGB component values from the background.
+- *plane_alpha*: Plane alpha value set by the **plane "alpha" property**, see
+  more in :ref:`DRM Plane Composition Properties <plane_composition_properties>`.
+
+in the basic alpha blending equation::
+
+   out.rgb = alpha * fg.rgb + (1 - alpha) * bg.rgb
+
+the alpha channel value of each pixel in a plane is ignored and only the plane
+alpha affects the resulted pixel color values.
+
+DRM has three blend mode to define the blend formula in the plane composition:
+
+* **None**: Blend formula that ignores the pixel alpha.
+
+* **Pre-multiplied**: Blend formula that assumes the pixel color values in a
+  plane was already pre-multiplied by its own alpha channel before storage.
+
+* **Coverage**: Blend formula that assumes the pixel color values were not
+  pre-multiplied with the alpha channel values.
+
+and pre-multiplied is the default pixel blend mode, that means, when no blend
+mode property is created or defined, DRM considers the plane's pixels has
+pre-multiplied color values. On IGT GPU tools, the kms_plane_alpha_blend test
+provides a set of subtests to verify plane alpha and blend mode properties.
+
+The DRM blend mode and its elements are then mapped by AMDGPU display manager
+(DM) to program the blending configuration of the Multiple Pipe/Plane Combined
+(MPC), as follows:
+
+.. kernel-doc:: drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
+   :doc: mpc-overview
+
+.. kernel-doc:: drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
+   :functions: mpcc_blnd_cfg
+
+Therefore, the blending configuration for a single MPCC instance on the MPC
+tree is defined by :c:type:`mpcc_blnd_cfg`, where
+:c:type:`pre_multiplied_alpha` is the alpha pre-multiplied mode flag used to
+set :c:type:`MPCC_ALPHA_MULTIPLIED_MODE`. It controls whether alpha is
+multiplied (true/false), being only true for DRM pre-multiplied blend mode.
+:c:type:`mpcc_alpha_blend_mode` defines the alpha blend mode regarding pixel
+alpha and plane alpha values. It sets one of the three modes for
+:c:type:`MPCC_ALPHA_BLND_MODE`, as described below.
+
+.. kernel-doc:: drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
+   :functions: mpcc_alpha_blend_mode
+
+DM then maps the elements of `enum mpcc_alpha_blend_mode` to those in the DRM
+blend formula, as follows:
+
+* *MPC pixel alpha* matches *DRM fg.alpha* as the alpha component value
+  from the plane's pixel
+* *MPC global alpha* matches *DRM plane_alpha* when the pixel alpha should
+  be ignored and, therefore, pixel values are not pre-multiplied
+* *MPC global gain* assumes *MPC global alpha* value when both *DRM
+  fg.alpha* and *DRM plane_alpha* participate in the blend equation
+
+In short, *fg.alpha* is ignored by selecting
+:c:type:`MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA`. On the other hand, (plane_alpha *
+fg.alpha) component becomes available by selecting
+:c:type:`MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA_COMBINED_GLOBAL_GAIN`. And the
+:c:type:`MPCC_ALPHA_MULTIPLIED_MODE` defines if the pixel color values are
+pre-multiplied by alpha or not.
+
+Blend configuration flow
+------------------------
+
+The alpha blending equation is configured from DRM to DC interface by the
+following path:
+
+1. When updating a :c:type:`drm_plane_state <drm_plane_state>`, DM calls
+   :c:type:`fill_blending_from_plane_state()` that maps
+   :c:type:`drm_plane_state <drm_plane_state>` attributes to
+   :c:type:`dc_plane_info <dc_plane_info>` struct to be handled in the
+   OS-agnostic component (DC).
+
+2. On DC interface, :c:type:`struct mpcc_blnd_cfg <mpcc_blnd_cfg>` programs the
+   MPCC blend configuration considering the :c:type:`dc_plane_info
+   <dc_plane_info>` input from DPP.
diff --git a/Documentation/gpu/amdgpu/display/index.rst b/Documentation/gpu/amdgpu/display/index.rst
index c1fb2fb3c710..f8a4f53d70d8 100644
--- a/Documentation/gpu/amdgpu/display/index.rst
+++ b/Documentation/gpu/amdgpu/display/index.rst
@@ -28,4 +28,5 @@ table of content:
    display-manager.rst
    dc-debug.rst
    dcn-overview.rst
+   mpo-overview.rst
    dc-glossary.rst
diff --git a/Documentation/gpu/amdgpu/display/mpo-cursor.svg b/Documentation/gpu/amdgpu/display/mpo-cursor.svg
new file mode 100644
index 000000000000..9d9de76847c3
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/mpo-cursor.svg
@@ -0,0 +1,435 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="160.5318mm"
+   height="65.443306mm"
+   viewBox="0 0 160.5318 65.443308"
+   version="1.1"
+   id="svg843"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)"
+   sodipodi:docname="mpo-cursor.svg">
+  <defs
+     id="defs837">
+    <marker
+       inkscape:stockid="Arrow1Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Send"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1568"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Mend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1562"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.4,0,0,-0.4,-4,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker1837"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1835"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1580"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Send-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1568-5"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Send-7-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1568-5-2"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Send-9"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1568-7"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Send-9-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1568-7-1"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Send-9-6-9"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1568-7-1-3"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.35"
+     inkscape:cx="895.68984"
+     inkscape:cy="-284.87808"
+     inkscape:document-units="mm"
+     inkscape:current-layer="g1433-6"
+     showgrid="false"
+     inkscape:window-width="2560"
+     inkscape:window-height="1376"
+     inkscape:window-x="0"
+     inkscape:window-y="27"
+     inkscape:window-maximized="1"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0" />
+  <metadata
+     id="metadata840">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(70.035531,-11.72001)">
+    <g
+       id="g1438"
+       transform="matrix(0.33108827,0,0,0.33108827,-46.847588,7.8396545)">
+      <rect
+         y="51.228218"
+         x="-69.09626"
+         height="34.773811"
+         width="66.523811"
+         id="rect1388"
+         style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1.87854159;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <text
+         id="text1392"
+         y="73.238098"
+         x="-59.718166"
+         style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-weight:bold;font-size:12.69999981px;stroke-width:0.26458332"
+           y="73.238098"
+           x="-59.718166"
+           id="tspan1390"
+           sodipodi:role="line">Cursor</tspan></text>
+    </g>
+    <g
+       id="g1433"
+       transform="matrix(0.33108827,0,0,0.33108827,-49.701591,6.5552955)">
+      <rect
+         y="116.32738"
+         x="-60.476192"
+         height="34.773811"
+         width="66.523811"
+         id="rect1388-3"
+         style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1.87854159;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <text
+         id="text1392-6"
+         y="138.44888"
+         x="-53.932037"
+         style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-weight:bold;font-size:12.69999981px;stroke-width:0.26458332"
+           y="138.44888"
+           x="-53.932037"
+           id="tspan1390-7"
+           sodipodi:role="line">Plane 1</tspan></text>
+    </g>
+    <g
+       id="g1428"
+       transform="matrix(0.33108827,0,0,0.33108827,-46.847588,7.8396518)">
+      <rect
+         y="173.66814"
+         x="-69.09626"
+         height="34.773811"
+         width="66.523811"
+         id="rect1388-5"
+         style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1.87854159;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <text
+         id="text1392-3"
+         y="195.78964"
+         x="-62.437382"
+         style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+         xml:space="preserve"><tspan
+           style="font-weight:bold;font-size:12.69999981px;stroke-width:0.26458332"
+           y="195.78964"
+           x="-62.437382"
+           id="tspan1390-5"
+           sodipodi:role="line">Plane 2</tspan></text>
+    </g>
+    <g
+       id="g1433-6"
+       transform="translate(116.41667,-9.0714256)">
+      <g
+         id="g1467"
+         transform="matrix(0.33108827,0,0,0.33108827,-132.72925,15.626721)">
+        <rect
+           style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1.87854159;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect1388-3-2"
+           width="66.523811"
+           height="34.773811"
+           x="-60.476192"
+           y="116.32738" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+           x="-45.380619"
+           y="138.33725"
+           id="text1392-6-9"><tspan
+             sodipodi:role="line"
+             id="tspan1390-7-1"
+             x="-45.380619"
+             y="138.33725"
+             style="font-weight:bold;font-size:12.69999981px;stroke-width:0.26458332">CRTC</tspan></text>
+      </g>
+      <g
+         id="g1438-2"
+         transform="matrix(0.33108827,0,0,0.33108827,-92.282164,27.16881)">
+        <rect
+           y="51.228218"
+           x="-69.09626"
+           height="34.773811"
+           width="66.523811"
+           id="rect1388-7"
+           style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1.87854159;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+        <text
+           id="text1392-0"
+           y="73.238098"
+           x="-59.718166"
+           style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+           xml:space="preserve"><tspan
+             style="font-weight:bold;font-size:12.69999981px;stroke-width:0.26458332"
+             y="73.238098"
+             x="-59.718166"
+             id="tspan1390-9"
+             sodipodi:role="line">Cursor</tspan></text>
+      </g>
+      <g
+         id="g1433-3"
+         transform="matrix(0.33108827,0,0,0.33108827,-61.59731,5.6152398)">
+        <rect
+           y="116.32738"
+           x="-60.476192"
+           height="34.773811"
+           width="66.523811"
+           id="rect1388-3-6"
+           style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1.87854159;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+        <text
+           id="text1392-6-0"
+           y="138.44888"
+           x="-53.932037"
+           style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+           xml:space="preserve"><tspan
+             style="font-weight:bold;font-size:12.69999981px;stroke-width:0.26458332"
+             y="138.44888"
+             x="-53.932037"
+             id="tspan1390-7-6"
+             sodipodi:role="line">Plane 1</tspan></text>
+      </g>
+      <g
+         id="g1428-2"
+         transform="matrix(0.33108827,0,0,0.33108827,-58.743296,6.7994816)">
+        <rect
+           y="173.66814"
+           x="-69.09626"
+           height="34.773811"
+           width="66.523811"
+           id="rect1388-5-6"
+           style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1.87854159;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+        <text
+           id="text1392-3-1"
+           y="195.78964"
+           x="-62.437382"
+           style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+           xml:space="preserve"><tspan
+             style="font-weight:bold;font-size:12.69999981px;stroke-width:0.26458332"
+             y="195.78964"
+             x="-62.437382"
+             id="tspan1390-5-8"
+             sodipodi:role="line">Plane 2</tspan></text>
+      </g>
+      <g
+         id="g1467-7"
+         transform="matrix(0.33108827,0,0,0.33108827,-28.233674,5.6152398)">
+        <rect
+           style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:1.87854159;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect1388-3-2-9"
+           width="66.523811"
+           height="34.773811"
+           x="-60.476192"
+           y="116.32738" />
+        <text
+           xml:space="preserve"
+           style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+           x="-45.380619"
+           y="138.33725"
+           id="text1392-6-9-2"><tspan
+             sodipodi:role="line"
+             id="tspan1390-7-1-0"
+             x="-45.380619"
+             y="138.33725"
+             style="font-weight:bold;font-size:12.69999981px;stroke-width:0.26458332">CRTC</tspan></text>
+      </g>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:1.05120528px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.08760043"
+         x="-164.26541"
+         y="24.302296"
+         id="text1531"><tspan
+           sodipodi:role="line"
+           id="tspan1529"
+           x="-164.26541"
+           y="24.302296"
+           style="font-weight:bold;font-size:4.6720233px;stroke-width:0.08760043">DRM</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-weight:normal;font-size:1.05120528px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.08760043"
+         x="-89.233742"
+         y="24.341078"
+         id="text1531-2"><tspan
+           sodipodi:role="line"
+           id="tspan1529-3"
+           x="-89.233742"
+           y="24.341078"
+           style="font-weight:bold;font-size:4.6720233px;stroke-width:0.08760043">AMD Hardware</tspan></text>
+      <path
+         style="fill:none;stroke:#000000;stroke-width:1.53318286;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send)"
+         d="m -164.26541,39.407343 h 20.27325 v 11.262913"
+         id="path1551"
+         inkscape:connector-curvature="0"
+         sodipodi:nodetypes="ccc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:1.53318286;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-7)"
+         d="m -164.11597,80.385815 h 20.27325 v -11.26291"
+         id="path1551-9"
+         inkscape:connector-curvature="0"
+         sodipodi:nodetypes="ccc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:1.53318286;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-7-2)"
+         d="m -59.24211,70.452123 h 20.273243 V 59.18921"
+         id="path1551-9-8"
+         inkscape:connector-curvature="0"
+         sodipodi:nodetypes="ccc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:1.53318286;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-9)"
+         d="m -163.61178,59.821549 h 9.26062"
+         id="path1551-3"
+         inkscape:connector-curvature="0"
+         sodipodi:nodetypes="cc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:1.53318286;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-9-6)"
+         d="m -93.426241,50.16968 h 9.260617"
+         id="path1551-3-2"
+         inkscape:connector-curvature="0"
+         sodipodi:nodetypes="cc" />
+      <path
+         style="fill:none;stroke:#000000;stroke-width:1.53318286;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow1Send-9-6-9)"
+         d="m -59.637503,50.169684 h 9.260618"
+         id="path1551-3-2-1"
+         inkscape:connector-curvature="0"
+         sodipodi:nodetypes="cc" />
+    </g>
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/display/mpo-overview.rst b/Documentation/gpu/amdgpu/display/mpo-overview.rst
new file mode 100644
index 000000000000..0499aa92d08d
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/mpo-overview.rst
@@ -0,0 +1,242 @@
+========================
+Multiplane Overlay (MPO)
+========================
+
+.. note:: You will get more from this page if you have already read the
+   'Documentation/gpu/amdgpu/display/dcn-overview.rst'.
+
+
+Multiplane Overlay (MPO) allows for multiple framebuffers to be composited via
+fixed-function hardware in the display controller rather than using graphics or
+compute shaders for composition. This can yield some power savings if it means
+the graphics/compute pipelines can be put into low-power states. In summary,
+MPO can bring the following benefits:
+
+* Decreased GPU and CPU workload - no composition shaders needed, no extra
+  buffer copy needed, GPU can remain idle.
+* Plane independent page flips - No need to be tied to global compositor
+  page-flip present rate, reduced latency, independent timing.
+
+.. note:: Keep in mind that MPO is all about power-saving; if you want to learn
+   more about power-save in the display context, check the link:
+   `Power <https://gitlab.freedesktop.org/pq/color-and-hdr/-/blob/main/doc/power.rst>`__.
+
+Multiplane Overlay is only available using the DRM atomic model. The atomic
+model only uses a single userspace IOCTL for configuring the display hardware
+(modesetting, page-flipping, etc) - drmModeAtomicCommit. To query hardware
+resources and limitations userspace also calls into drmModeGetResources which
+reports back the number of planes, CRTCs, and connectors. There are three types
+of DRM planes that the driver can register and work with:
+
+* ``DRM_PLANE_TYPE_PRIMARY``: Primary planes represent a "main" plane for a
+  CRTC, primary planes are the planes operated upon by CRTC modesetting and
+  flipping operations.
+* ``DRM_PLANE_TYPE_CURSOR``: Cursor planes represent a "cursor" plane for a
+  CRTC. Cursor planes are the planes operated upon by the cursor IOCTLs
+* ``DRM_PLANE_TYPE_OVERLAY``: Overlay planes represent all non-primary,
+  non-cursor planes. Some drivers refer to these types of planes as "sprites"
+  internally.
+
+To illustrate how it works, let's take a look at a device that exposes the
+following planes to userspace:
+
+* 4 Primary planes (1 per CRTC).
+* 4 Cursor planes (1 per CRTC).
+* 1 Overlay plane (shared among CRTCs).
+
+.. note:: Keep in mind that different ASICs might expose other numbers of
+   planes.
+
+For this hardware example, we have 4 pipes (if you don't know what AMD pipe
+means, look at 'Documentation/gpu/amdgpu/display/dcn-overview.rst', section
+"AMD Hardware Pipeline"). Typically most AMD devices operate in a pipe-split
+configuration for optimal single display output (e.g., 2 pipes per plane).
+
+A typical MPO configuration from userspace - 1 primary + 1 overlay on a single
+display - will see 4 pipes in use, 2 per plane.
+
+At least 1 pipe must be used per plane (primary and overlay), so for this
+hypothetical hardware that we are using as an example, we have an absolute
+limit of 4 planes across all CRTCs. Atomic commits will be rejected for display
+configurations using more than 4 planes. Again, it is important to stress that
+every DCN has different restrictions; here, we are just trying to provide the
+concept idea.
+
+Plane Restrictions
+==================
+
+AMDGPU imposes restrictions on the use of DRM planes in the driver.
+
+Atomic commits will be rejected for commits which do not follow these
+restrictions:
+
+* Overlay planes must be in ARGB8888 or XRGB8888 format
+* Planes cannot be placed outside of the CRTC destination rectangle
+* Planes cannot be downscaled more than 1/4x of their original size
+* Planes cannot be upscaled more than 16x of their original size
+
+Not every property is available on every plane:
+
+* Only primary planes have color-space and non-RGB format support
+* Only overlay planes have alpha blending support
+
+Cursor Restrictions
+===================
+
+Before we start to describe some restrictions around cursor and MPO, see the
+below image:
+
+.. kernel-figure:: mpo-cursor.svg
+
+The image on the left side represents how DRM expects the cursor and planes to
+be blended. However, AMD hardware handles cursors differently, as you can see
+on the right side; basically, our cursor cannot be drawn outside its associated
+plane as it is being treated as part of the plane. Another consequence of that
+is that cursors inherit the color and scale from the plane.
+
+As a result of the above behavior, do not use legacy API to set up the cursor
+plane when working with MPO; otherwise, you might encounter unexpected
+behavior.
+
+In short, AMD HW has no dedicated cursor planes. A cursor is attached to
+another plane and therefore inherits any scaling or color processing from its
+parent plane.
+
+Use Cases
+=========
+
+Picture-in-Picture (PIP) playback - Underlay strategy
+-----------------------------------------------------
+
+Video playback should be done using the "primary plane as underlay" MPO
+strategy. This is a 2 planes configuration:
+
+* 1 YUV DRM Primary Plane (e.g. NV12 Video)
+* 1 RGBA DRM Overlay Plane (e.g. ARGB8888 desktop). The compositor should
+  prepare the framebuffers for the planes as follows:
+  - The overlay plane contains general desktop UI, video player controls, and video subtitles
+  - Primary plane contains one or more videos
+
+.. note:: Keep in mind that we could extend this configuration to more planes,
+   but that is currently not supported by our driver yet (maybe if we have a
+   userspace request in the future, we can change that).
+
+See below a single-video example:
+
+.. kernel-figure:: single-display-mpo.svg
+
+.. note:: We could extend this behavior to more planes, but that is currently
+   not supported by our driver.
+
+The video buffer should be used directly for the primary plane. The video can
+be scaled and positioned for the desktop using the properties: CRTC_X, CRTC_Y,
+CRTC_W, and CRTC_H. The primary plane should also have the color encoding and
+color range properties set based on the source content:
+
+* ``COLOR_RANGE``, ``COLOR_ENCODING``
+
+The overlay plane should be the native size of the CRTC. The compositor must
+draw a transparent cutout for where the video should be placed on the desktop
+(i.e., set the alpha to zero). The primary plane video will be visible through
+the underlay. The overlay plane's buffer may remain static while the primary
+plane's framebuffer is used for standard double-buffered playback.
+
+The compositor should create a YUV buffer matching the native size of the CRTC.
+Each video buffer should be composited onto this YUV buffer for direct YUV
+scanout. The primary plane should have the color encoding and color range
+properties set based on the source content: ``COLOR_RANGE``,
+``COLOR_ENCODING``. However, be mindful that the source color space and
+encoding match for each video since it affect the entire plane.
+
+The overlay plane should be the native size of the CRTC. The compositor must
+draw a transparent cutout for where each video should be placed on the desktop
+(i.e., set the alpha to zero). The primary plane videos will be visible through
+the underlay. The overlay plane's buffer may remain static while compositing
+operations for video playback will be done on the video buffer.
+
+This kernel interface is validated using IGT GPU Tools. The following tests can
+be run to validate positioning, blending, scaling under a variety of sequences
+and interactions with operations such as DPMS and S3:
+
+- ``kms_plane@plane-panning-bottom-right-pipe-*-planes``
+- ``kms_plane@plane-panning-bottom-right-suspend-pipe-*-``
+- ``kms_plane@plane-panning-top-left-pipe-*-``
+- ``kms_plane@plane-position-covered-pipe-*-``
+- ``kms_plane@plane-position-hole-dpms-pipe-*-``
+- ``kms_plane@plane-position-hole-pipe-*-``
+- ``kms_plane_multiple@atomic-pipe-*-tiling-``
+- ``kms_plane_scaling@pipe-*-plane-scaling``
+- ``kms_plane_alpha_blend@pipe-*-alpha-basic``
+- ``kms_plane_alpha_blend@pipe-*-alpha-transparant-fb``
+- ``kms_plane_alpha_blend@pipe-*-alpha-opaque-fb``
+- ``kms_plane_alpha_blend@pipe-*-constant-alpha-min``
+- ``kms_plane_alpha_blend@pipe-*-constant-alpha-mid``
+- ``kms_plane_alpha_blend@pipe-*-constant-alpha-max``
+
+Multiple Display MPO
+--------------------
+
+AMDGPU supports display MPO when using multiple displays; however, this feature
+behavior heavily relies on the compositor implementation. Keep in mind that
+usespace can define different policies. For example, some OSes can use MPO to
+protect the plane that handles the video playback; notice that we don't have
+many limitations for a single display. Nonetheless, this manipulation can have
+many more restrictions for a multi-display scenario. The below example shows a
+video playback in the middle of two displays, and it is up to the compositor to
+define a policy on how to handle it:
+
+.. kernel-figure:: multi-display-hdcp-mpo.svg
+
+Let's discuss some of the hardware limitations we have when dealing with
+multi-display with MPO.
+
+Limitations
+~~~~~~~~~~~
+
+For simplicity's sake, for discussing the hardware limitation, this
+documentation supposes an example where we have two displays and video playback
+that will be moved around different displays.
+
+* **Hardware limitations**
+
+From the DCN overview page, each display requires at least one pipe and each
+MPO plane needs another pipe. As a result, when the video is in the middle of
+the two displays, we need to use 2 pipes. See the example below where we avoid
+pipe split:
+
+- 1 display (1 pipe) + MPO (1 pipe), we will use two pipes
+- 2 displays (2 pipes) + MPO (1-2 pipes); we will use 4 pipes. MPO in the
+  middle of both displays needs 2 pipes.
+- 3 Displays (3 pipes) + MPO (1-2 pipes), we need 5 pipes.
+
+If we use MPO with multiple displays, the userspace has to decide to enable
+multiple MPO by the price of limiting the number of external displays supported
+or disable it in favor of multiple displays; it is a policy decision. For
+example:
+
+* When ASIC has 3 pipes, AMD hardware can NOT support 2 displays with MPO
+* When ASIC has 4 pipes, AMD hardware can NOT support 3 displays with MPO
+
+Let's briefly explore how userspace can handle these two display configurations
+on an ASIC that only supports three pipes. We can have:
+
+.. kernel-figure:: multi-display-hdcp-mpo-less-pipe-ex.svg
+
+- Total pipes are 3
+- User lights up 2 displays (2 out of 3 pipes are used)
+- User launches video (1 pipe used for MPO)
+- Now, if the user moves the video in the middle of 2 displays, one part of the
+  video won't be MPO since we have used 3/3 pipes.
+
+* **Scaling limitation**
+
+MPO cannot handle scaling less than 0.25 and more than x16. For example:
+
+If 4k video (3840x2160) is playing in windowed mode, the physical size of the
+window cannot be smaller than (960x540).
+
+.. note:: These scaling limitations might vary from ASIC to ASIC.
+
+* **Size Limitation**
+
+The minimum MPO size is 12px.
diff --git a/Documentation/gpu/amdgpu/display/multi-display-hdcp-mpo-less-pipe-ex.svg b/Documentation/gpu/amdgpu/display/multi-display-hdcp-mpo-less-pipe-ex.svg
new file mode 100644
index 000000000000..6d06b39e83fa
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/multi-display-hdcp-mpo-less-pipe-ex.svg
@@ -0,0 +1,220 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="103.08798mm"
+   height="30.335283mm"
+   viewBox="0 0 103.08798 30.335284"
+   version="1.1"
+   id="svg8"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)"
+   sodipodi:docname="multi-display-hdcp-mpo-less-pipe-ex.svg">
+  <defs
+     id="defs2">
+    <marker
+       inkscape:stockid="TriangleOutL"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="TriangleOutL"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1133"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="scale(0.8)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="TriangleOutL"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="TriangleOutL-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1133-0"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="scale(0.8)" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.979899"
+     inkscape:cx="205.69673"
+     inkscape:cy="33.960031"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="1288"
+     inkscape:window-height="1376"
+     inkscape:window-x="1272"
+     inkscape:window-y="27"
+     inkscape:window-maximized="0"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(112.75816,51.721433)">
+    <rect
+       style="fill:#ffe680;fill-opacity:1;stroke:#000000;stroke-width:0.62916809;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836"
+       width="48.988979"
+       height="29.48851"
+       x="-112.44358"
+       y="-51.189243"
+       ry="0" />
+    <rect
+       style="fill:#008000;fill-opacity:1;stroke:none;stroke-width:0.38327959;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect834"
+       width="14.422765"
+       height="15.872559"
+       x="-78.227928"
+       y="-44.469936" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-71.189278"
+       y="-48.114422"
+       id="text844"><tspan
+         sodipodi:role="line"
+         x="-71.189278"
+         y="-48.114422"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846">Protected</tspan><tspan
+         sodipodi:role="line"
+         x="-71.189278"
+         y="-45.584015"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan2288">MPO plane</tspan></text>
+    <rect
+       style="fill:#ffe680;fill-opacity:1;stroke:#000000;stroke-width:0.62916809;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836-3"
+       width="48.988979"
+       height="29.48851"
+       x="-58.973743"
+       y="-51.406849"
+       ry="0" />
+    <rect
+       style="fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.2500253;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect834-6"
+       width="6.1374092"
+       height="15.872559"
+       x="-58.632965"
+       y="-44.469936" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-107.96788"
+       y="-47.033775"
+       id="text844-7"><tspan
+         sodipodi:role="line"
+         x="-107.96788"
+         y="-47.033775"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52777767px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-3">#1</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-15.20074"
+       y="-46.715996"
+       id="text844-7-2"><tspan
+         sodipodi:role="line"
+         x="-15.20074"
+         y="-46.715996"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52777767px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-3-9">#2</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-88.926605"
+       y="-23.173674"
+       id="text844-1"><tspan
+         sodipodi:role="line"
+         x="-88.926605"
+         y="-23.173674"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-7">Desktop</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-33.770744"
+       y="-23.819901"
+       id="text844-1-0"><tspan
+         sodipodi:role="line"
+         x="-33.770744"
+         y="-23.819901"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-7-9">Desktop</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-75.17498"
+       y="-41.219398"
+       id="text844-7-2-3"><tspan
+         sodipodi:role="line"
+         x="-75.17498"
+         y="-41.219398"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52777767px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ffffff;stroke-width:0.06901112"
+         id="tspan846-3-9-6">#3</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-47.142235"
+       y="-48.056866"
+       id="text844-0"><tspan
+         sodipodi:role="line"
+         x="-47.142235"
+         y="-48.056866"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan2288-2">Software</tspan><tspan
+         sodipodi:role="line"
+         x="-47.142235"
+         y="-45.526459"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan2313">Composited Video</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-36.23288"
+       y="-34.592064"
+       id="text844-1-6"><tspan
+         sodipodi:role="line"
+         x="-36.23288"
+         y="-34.592064"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-7-1">Video will not be displayed</tspan></text>
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/display/multi-display-hdcp-mpo.svg b/Documentation/gpu/amdgpu/display/multi-display-hdcp-mpo.svg
new file mode 100644
index 000000000000..84d53a558b05
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/multi-display-hdcp-mpo.svg
@@ -0,0 +1,171 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="103.08798mm"
+   height="30.335283mm"
+   viewBox="0 0 103.08798 30.335284"
+   version="1.1"
+   id="svg8"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)"
+   sodipodi:docname="multi-display-hdcp-mpo.svg">
+  <defs
+     id="defs2">
+    <marker
+       inkscape:stockid="TriangleOutL"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="TriangleOutL"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1133"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="scale(0.8)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="TriangleOutL"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="TriangleOutL-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1133-0"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="scale(0.8)" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.979899"
+     inkscape:cx="196.85789"
+     inkscape:cy="33.960027"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="1288"
+     inkscape:window-height="1376"
+     inkscape:window-x="1272"
+     inkscape:window-y="27"
+     inkscape:window-maximized="0"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(112.75816,51.721433)">
+    <rect
+       style="fill:#ffe680;fill-opacity:1;stroke:#000000;stroke-width:0.62916809;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836"
+       width="48.988979"
+       height="29.48851"
+       x="-112.44358"
+       y="-51.189243"
+       ry="0" />
+    <rect
+       style="fill:#008000;fill-opacity:1;stroke:none;stroke-width:0.38327959;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect834"
+       width="14.422765"
+       height="15.872559"
+       x="-78.227928"
+       y="-44.469936" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-71.189278"
+       y="-48.114422"
+       id="text844"><tspan
+         sodipodi:role="line"
+         x="-71.189278"
+         y="-48.114422"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846">Protected</tspan><tspan
+         sodipodi:role="line"
+         x="-71.189278"
+         y="-45.584015"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan2288">MPO plane</tspan></text>
+    <rect
+       style="fill:#ffe680;fill-opacity:1;stroke:#000000;stroke-width:0.62916809;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836-3"
+       width="48.988979"
+       height="29.48851"
+       x="-58.973743"
+       y="-51.406849"
+       ry="0" />
+    <rect
+       style="fill:#008000;fill-opacity:1;stroke:none;stroke-width:0.2500253;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect834-6"
+       width="6.1374092"
+       height="15.872559"
+       x="-58.632965"
+       y="-44.469936" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-88.926605"
+       y="-23.173674"
+       id="text844-1"><tspan
+         sodipodi:role="line"
+         x="-88.926605"
+         y="-23.173674"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-7">Desktop</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-33.770744"
+       y="-23.819901"
+       id="text844-1-0"><tspan
+         sodipodi:role="line"
+         x="-33.770744"
+         y="-23.819901"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-7-9">Desktop</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-70.364128"
+       y="-35.740372"
+       id="text844-7-2-3"><tspan
+         sodipodi:role="line"
+         x="-70.364128"
+         y="-35.740372"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:3.52777767px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;fill:#ffffff;stroke-width:0.06901112"
+         id="tspan846-3-9-6">Video</tspan></text>
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/display/pipeline_4k_no_split.svg b/Documentation/gpu/amdgpu/display/pipeline_4k_no_split.svg
new file mode 100644
index 000000000000..5fa289d99fcd
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/pipeline_4k_no_split.svg
@@ -0,0 +1,958 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="1055.4928"
+   height="404.60379"
+   viewBox="0 0 279.26579 107.05142"
+   version="1.1"
+   id="svg8"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)"
+   sodipodi:docname="pipeline_4k_no_split.svg">
+  <defs
+     id="defs2">
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker8858"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path8616"
+         style="fill:#aa00d4;fill-opacity:1;fill-rule:evenodd;stroke:#aa00d4;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Send"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path8622"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="matrix(-0.3,0,0,-0.3,0.69,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path8592"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Lend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path8610"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-9"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-2-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-9-9"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-2-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-9-8"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-4"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-5"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-0"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-3"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-1"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-2-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-9-1"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-0-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-3-4"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-6-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-1-0"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-2-8"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-9-6"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-6"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker8858-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path8616-5"
+         style="fill:#00ffcc;fill-opacity:1;fill-rule:evenodd;stroke:#00ffcc;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-56"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-0-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-3-9"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-9"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-3"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-4"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-0"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-7"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-0-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-7-4"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-0-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-7-6"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-0-2-4"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-7-6-9"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-8"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-7"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-0-2-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-7-6-6"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.4"
+     inkscape:cx="213.41584"
+     inkscape:cy="301.17934"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="2560"
+     inkscape:window-height="1376"
+     inkscape:window-x="0"
+     inkscape:window-y="27"
+     inkscape:window-maximized="1"
+     showguides="false"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     units="px"
+     inkscape:snap-global="false" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(349.34521,-0.81564989)">
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:1.00353587;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none"
+       id="rect834"
+       width="30.251244"
+       height="88.477814"
+       x="-317.81958"
+       y="14.782127"
+       ry="2.5590618e-06" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-314.73792"
+       y="21.83947"
+       id="text838"><tspan
+         sodipodi:role="line"
+         id="tspan836"
+         x="-314.73792"
+         y="21.83947"
+         style="stroke-width:0.16370411">DCHUB</tspan></text>
+    <g
+       id="g2025"
+       transform="translate(-2.4916954,-1.2533369)">
+      <text
+         id="text838-5"
+         y="31.969406"
+         x="-254.53119"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="31.969406"
+           x="-254.53119"
+           id="tspan836-3"
+           sodipodi:role="line">DPP</tspan><tspan
+           id="tspan936"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="40.15461"
+           x="-254.53119"
+           sodipodi:role="line">(0)</tspan></text>
+      <rect
+         ry="7.8154301e-07"
+         y="20.596174"
+         x="-269.66983"
+         height="27.021315"
+         width="30.696135"
+         id="rect834-5"
+         style="fill:none;stroke:#000000;stroke-width:0.55864918;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.1637041"
+       x="-297.75696"
+       y="109.44505"
+       id="text1063"><tspan
+         sodipodi:role="line"
+         id="tspan1061"
+         x="-297.75696"
+         y="115.23865"
+         style="stroke-width:0.1637041" /></text>
+    <g
+       id="g1878"
+       transform="translate(-0.62971878)">
+      <text
+         id="text846"
+         y="30.716068"
+         x="-302.27368"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#44aa00;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="30.716068"
+           x="-302.27368"
+           id="tspan844"
+           sodipodi:role="line">HUBP</tspan><tspan
+           id="tspan863"
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="38.901276"
+           x="-302.27368"
+           sodipodi:role="line">(0)</tspan></text>
+      <g
+         id="g1853">
+        <rect
+           style="fill:none;fill-opacity:1;stroke:#55d400;stroke-width:0.94157624;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect1850"
+           width="24.708241"
+           height="17.224693"
+           x="-314.41837"
+           y="24.241148" />
+      </g>
+    </g>
+    <g
+       transform="translate(-0.62971878,18.810271)"
+       id="g1878-1">
+      <text
+         id="text846-2"
+         y="30.716068"
+         x="-302.27368"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#44aa00;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="30.716068"
+           x="-302.27368"
+           id="tspan844-7"
+           sodipodi:role="line">HUBP</tspan><tspan
+           id="tspan863-0"
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="38.901276"
+           x="-302.27368"
+           sodipodi:role="line">(1)</tspan></text>
+      <g
+         id="g1853-9">
+        <rect
+           style="fill:none;fill-opacity:1;stroke:#55d400;stroke-width:0.94157624;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect1850-3"
+           width="24.708241"
+           height="17.224693"
+           x="-314.41837"
+           y="24.241148" />
+      </g>
+    </g>
+    <g
+       transform="translate(-0.62971878,57.029814)"
+       id="g1878-8">
+      <text
+         id="text846-7"
+         y="30.716068"
+         x="-302.27368"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#44aa00;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="30.716068"
+           x="-302.27368"
+           id="tspan844-9"
+           sodipodi:role="line">HUBP</tspan><tspan
+           id="tspan863-20"
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="38.901276"
+           x="-302.27368"
+           sodipodi:role="line">(5)</tspan></text>
+      <g
+         id="g1853-2">
+        <rect
+           style="fill:none;fill-opacity:1;stroke:#55d400;stroke-width:0.94157624;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect1850-37"
+           width="24.708241"
+           height="17.224693"
+           x="-314.41837"
+           y="24.241148" />
+      </g>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="-310.7348"
+       y="72.106789"
+       id="text8862-5"><tspan
+         sodipodi:role="line"
+         id="tspan8860-9"
+         x="-310.7348"
+         y="72.106789"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:14.11111069px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';fill:#44aa00;stroke-width:0.26458332">...</tspan></text>
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.96187615;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94)"
+       d="m -288.90777,32.862737 h 14.20296"
+       id="path1171"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="-264.85431"
+       y="62.072891"
+       id="text8862-5-8"><tspan
+         sodipodi:role="line"
+         id="tspan8860-9-4"
+         x="-264.85431"
+         y="62.072891"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:14.11111069px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';fill:#000000;stroke-width:0.26458332">...</tspan></text>
+    <g
+       id="g2025-5"
+       transform="translate(40.522337,-1.1484419)" />
+    <g
+       id="g7343"
+       transform="translate(40.522337,-1.2533369)">
+      <text
+         id="text838-5-6"
+         y="31.969406"
+         x="-254.53119"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="31.969406"
+           x="-254.53119"
+           id="tspan836-3-3"
+           sodipodi:role="line">MPC</tspan><tspan
+           id="tspan936-2"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="40.15461"
+           x="-254.53119"
+           sodipodi:role="line">(0)</tspan></text>
+      <rect
+         ry="7.8154301e-07"
+         y="20.596174"
+         x="-269.66983"
+         height="27.021315"
+         width="30.696135"
+         id="rect834-5-0"
+         style="fill:none;stroke:#000000;stroke-width:0.55864918;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <g
+       id="g7343-2"
+       transform="translate(84.178583,-1.2533369)">
+      <text
+         id="text838-5-6-54"
+         y="31.969406"
+         x="-254.53119"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="31.969406"
+           x="-254.53119"
+           id="tspan836-3-3-7"
+           sodipodi:role="line">OPP</tspan><tspan
+           id="tspan936-2-44"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="40.15461"
+           x="-254.53119"
+           sodipodi:role="line">(0)</tspan></text>
+      <rect
+         ry="7.8154301e-07"
+         y="20.596174"
+         x="-269.66983"
+         height="27.021315"
+         width="30.696135"
+         id="rect834-5-0-3"
+         style="fill:none;stroke:#000000;stroke-width:0.55864918;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="-221.84029"
+       y="62.072891"
+       id="text8862-5-8-6"><tspan
+         sodipodi:role="line"
+         id="tspan8860-9-4-5"
+         x="-221.84029"
+         y="62.072891"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:14.11111069px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';fill:#000000;stroke-width:0.26458332">...</tspan></text>
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6)"
+       d="m -240.90479,32.86192 h 10.61353"
+       id="path1171-3"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6-0)"
+       d="m -198.19009,32.86192 h 10.61353"
+       id="path1171-3-8"
+       inkscape:connector-curvature="0" />
+    <g
+       id="g7343-2-2"
+       transform="translate(127.15447,-1.1484419)">
+      <text
+         id="text838-5-6-54-0"
+         y="65.122154"
+         x="-254.32814"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           id="tspan936-2-44-8"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="65.122154"
+           x="-254.32814"
+           sodipodi:role="line">OPTC</tspan></text>
+      <rect
+         ry="2.4256708e-06"
+         y="20.805576"
+         x="-269.46042"
+         height="83.865906"
+         width="30.277332"
+         id="rect834-5-0-3-9"
+         style="fill:none;stroke:#000000;stroke-width:0.97745234;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6-0-2)"
+       d="m -154.34485,32.86192 h 10.61353"
+       id="path1171-3-8-6"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="-178.18404"
+       y="62.072891"
+       id="text8862-5-8-6-0"><tspan
+         sodipodi:role="line"
+         id="tspan8860-9-4-5-4"
+         x="-178.18404"
+         y="62.072891"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:14.11111069px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';fill:#000000;stroke-width:0.26458332">...</tspan></text>
+    <path
+       style="fill:#000000;stroke:#000000;stroke-width:0.96187615;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-8)"
+       d="m -335.20986,59.030276 h 14.20296"
+       id="path1171-17"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-349.77686"
+       y="61.404655"
+       id="text838-2"><tspan
+         sodipodi:role="line"
+         id="tspan836-7"
+         x="-349.77686"
+         y="61.404655"
+         style="stroke-width:0.16370411">SDP</tspan></text>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1.32291663;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1.32291667, 1.32291667;stroke-dashoffset:0;stroke-opacity:1"
+       d="M -279.04164,107.86706 V 0.89985833"
+       id="path11907"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1.32291663;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1.3229167, 1.3229167;stroke-dashoffset:0;stroke-opacity:1"
+       d="M -235.57438,107.86707 V 0.89985663"
+       id="path11907-6-1"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1.32291663;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1.32291671, 1.32291671;stroke-dashoffset:0;stroke-opacity:1"
+       d="M -149.96277,107.86707 V 0.89985223"
+       id="path11907-6-1-0"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-257.38275"
+       y="5.003336"
+       id="text838-6"><tspan
+         sodipodi:role="line"
+         id="tspan836-1"
+         x="-257.38275"
+         y="5.003336"
+         style="font-size:4.93888903px;text-align:center;text-anchor:middle;stroke-width:0.16370411">DPPCLK</tspan><tspan
+         sodipodi:role="line"
+         x="-257.38275"
+         y="13.188541"
+         style="font-size:4.93888903px;text-align:center;text-anchor:middle;stroke-width:0.16370411"
+         id="tspan12658">535.916Mhz</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-192.85298"
+       y="5.003336"
+       id="text838-6-5"><tspan
+         sodipodi:role="line"
+         id="tspan836-1-9"
+         x="-192.85298"
+         y="5.003336"
+         style="font-size:4.93888903px;text-align:center;text-anchor:middle;stroke-width:0.16370411">DISPCLK</tspan><tspan
+         sodipodi:role="line"
+         x="-192.85298"
+         y="13.188541"
+         style="font-size:4.93888903px;text-align:center;text-anchor:middle;stroke-width:0.16370411"
+         id="tspan12662">541.275 Mhz</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-302.03766"
+       y="5.003336"
+       id="text838-6-5-4"><tspan
+         sodipodi:role="line"
+         id="tspan836-1-9-9"
+         x="-312.23135"
+         y="5.003336"
+         style="font-size:4.93888903px;stroke-width:0.16370411;text-anchor:middle;text-align:center">DCFCLK</tspan><tspan
+         sodipodi:role="line"
+         x="-312.23135"
+         y="13.188541"
+         style="font-size:4.93888903px;stroke-width:0.16370411;text-anchor:middle;text-align:center"
+         id="tspan12660">506 Mhz</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-96.075348"
+       y="4.568048"
+       id="text838-6-5-0"><tspan
+         sodipodi:role="line"
+         id="tspan836-1-9-91"
+         x="-96.075348"
+         y="4.568048"
+         style="font-size:4.93888903px;stroke-width:0.16370411">SymCLK</tspan></text>
+    <g
+       id="g7343-2-2-7"
+       transform="translate(168.61494,-1.1484419)">
+      <text
+         id="text838-5-6-54-0-7"
+         y="65.122154"
+         x="-254.32814"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           id="tspan936-2-44-8-1"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="65.122154"
+           x="-254.32814"
+           sodipodi:role="line">DIO</tspan></text>
+      <rect
+         ry="2.4256708e-06"
+         y="20.805576"
+         x="-269.46042"
+         height="83.865906"
+         width="30.277332"
+         id="rect834-5-0-3-9-1"
+         style="fill:none;stroke:#000000;stroke-width:0.97745234;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1.32291663;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1.32291672, 1.32291672;stroke-dashoffset:0;stroke-opacity:1"
+       d="M -108.38539,107.86707 V 0.89985092"
+       id="path11907-6-1-0-5"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-143.52292"
+       y="5.0467439"
+       id="text838-6-5-0-9"><tspan
+         sodipodi:role="line"
+         id="tspan836-1-9-91-7"
+         x="-143.52292"
+         y="5.0467439"
+         style="font-size:4.93888903px;stroke-width:0.16370411">VirtualPCLK</tspan></text>
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6-0-2-7)"
+       d="m -112.24183,60.747629 h 10.61353"
+       id="path1171-3-8-6-7"
+       inkscape:connector-curvature="0" />
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/display/pipeline_4k_split.svg b/Documentation/gpu/amdgpu/display/pipeline_4k_split.svg
new file mode 100644
index 000000000000..b43119e7eb8a
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/pipeline_4k_split.svg
@@ -0,0 +1,1062 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="1055.4928"
+   height="404.60379"
+   viewBox="0 0 279.26579 107.05142"
+   version="1.1"
+   id="svg8"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)"
+   sodipodi:docname="pipeline_no_split_4k.svg">
+  <defs
+     id="defs2">
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker8858"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path8616"
+         style="fill:#aa00d4;fill-opacity:1;fill-rule:evenodd;stroke:#aa00d4;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Send"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path8622"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="matrix(-0.3,0,0,-0.3,0.69,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Lend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path8592"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.8,0,0,-0.8,-10,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Lend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Lend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path8610"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="matrix(-1.1,0,0,-1.1,-1.1,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-9"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-2-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-9-9"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-2-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-9-8"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-4"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-5"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-0"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-3"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-1"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-2-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-9-1"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-0-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-3-4"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-6-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-1-0"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-2-8"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-9-6"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-6"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="marker8858-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path8616-5"
+         style="fill:#00ffcc;fill-opacity:1;fill-rule:evenodd;stroke:#00ffcc;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-3-3"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-6-56"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-8-0-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1200-9-3-9"
+         style="fill:#008000;fill-opacity:1;fill-rule:evenodd;stroke:#008000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-9"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-3"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-4"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-0"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-7"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-0-1"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-7-4"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-0-2"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-7-6"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-0-2-4"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-7-6-9"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-8"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-7"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow2Mend"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow2Mend-94-6-0-2-7"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1200-7-9-7-6-6"
+         style="fill:#ff0000;fill-opacity:1;fill-rule:evenodd;stroke:#ff0000;stroke-width:0.625;stroke-linejoin:round;stroke-opacity:1"
+         d="M 8.7185878,4.0337352 -2.2072895,0.01601326 8.7185884,-4.0017078 c -1.7454984,2.3720609 -1.7354408,5.6174519 -6e-7,8.035443 z"
+         transform="scale(-0.6)"
+         inkscape:connector-curvature="0" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.4"
+     inkscape:cx="491.27298"
+     inkscape:cy="306.89362"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="2560"
+     inkscape:window-height="1376"
+     inkscape:window-x="0"
+     inkscape:window-y="27"
+     inkscape:window-maximized="1"
+     showguides="false"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     units="px"
+     inkscape:snap-global="false" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(349.34521,-0.81564989)">
+    <rect
+       style="fill:none;stroke:#000000;stroke-width:1.00353587;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none"
+       id="rect834"
+       width="30.251244"
+       height="88.477814"
+       x="-317.81958"
+       y="14.782127"
+       ry="2.5590618e-06" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-314.73792"
+       y="21.83947"
+       id="text838"><tspan
+         sodipodi:role="line"
+         id="tspan836"
+         x="-314.73792"
+         y="21.83947"
+         style="stroke-width:0.16370411">DCHUB</tspan></text>
+    <g
+       id="g2025"
+       transform="translate(-2.4916954,-1.2533369)">
+      <text
+         id="text838-5"
+         y="31.969406"
+         x="-254.53119"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="31.969406"
+           x="-254.53119"
+           id="tspan836-3"
+           sodipodi:role="line">DPP</tspan><tspan
+           id="tspan936"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="40.15461"
+           x="-254.53119"
+           sodipodi:role="line">(0)</tspan></text>
+      <rect
+         ry="7.8154301e-07"
+         y="20.596174"
+         x="-269.66983"
+         height="27.021315"
+         width="30.696135"
+         id="rect834-5"
+         style="fill:none;stroke:#000000;stroke-width:0.55864918;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.1637041"
+       x="-297.75696"
+       y="109.44505"
+       id="text1063"><tspan
+         sodipodi:role="line"
+         id="tspan1061"
+         x="-297.75696"
+         y="115.23865"
+         style="stroke-width:0.1637041" /></text>
+    <g
+       id="g1878"
+       transform="translate(-0.62971878)">
+      <text
+         id="text846"
+         y="30.716068"
+         x="-302.27368"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#44aa00;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="30.716068"
+           x="-302.27368"
+           id="tspan844"
+           sodipodi:role="line">HUBP</tspan><tspan
+           id="tspan863"
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="38.901276"
+           x="-302.27368"
+           sodipodi:role="line">(0)</tspan></text>
+      <g
+         id="g1853">
+        <rect
+           style="fill:none;fill-opacity:1;stroke:#55d400;stroke-width:0.94157624;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect1850"
+           width="24.708241"
+           height="17.224693"
+           x="-314.41837"
+           y="24.241148" />
+      </g>
+    </g>
+    <g
+       transform="translate(-0.62971878,18.810271)"
+       id="g1878-1">
+      <text
+         id="text846-2"
+         y="30.716068"
+         x="-302.27368"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#44aa00;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="30.716068"
+           x="-302.27368"
+           id="tspan844-7"
+           sodipodi:role="line">HUBP</tspan><tspan
+           id="tspan863-0"
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="38.901276"
+           x="-302.27368"
+           sodipodi:role="line">(1)</tspan></text>
+      <g
+         id="g1853-9">
+        <rect
+           style="fill:none;fill-opacity:1;stroke:#55d400;stroke-width:0.94157624;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect1850-3"
+           width="24.708241"
+           height="17.224693"
+           x="-314.41837"
+           y="24.241148" />
+      </g>
+    </g>
+    <g
+       transform="translate(-0.62971878,57.029814)"
+       id="g1878-8">
+      <text
+         id="text846-7"
+         y="30.716068"
+         x="-302.27368"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#44aa00;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="30.716068"
+           x="-302.27368"
+           id="tspan844-9"
+           sodipodi:role="line">HUBP</tspan><tspan
+           id="tspan863-20"
+           style="text-align:center;text-anchor:middle;fill:#44aa00;stroke-width:0.16370411"
+           y="38.901276"
+           x="-302.27368"
+           sodipodi:role="line">(5)</tspan></text>
+      <g
+         id="g1853-2">
+        <rect
+           style="fill:none;fill-opacity:1;stroke:#55d400;stroke-width:0.94157624;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect1850-37"
+           width="24.708241"
+           height="17.224693"
+           x="-314.41837"
+           y="24.241148" />
+      </g>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="-310.7348"
+       y="72.106789"
+       id="text8862-5"><tspan
+         sodipodi:role="line"
+         id="tspan8860-9"
+         x="-310.7348"
+         y="72.106789"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:14.11111069px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';fill:#44aa00;stroke-width:0.26458332">...</tspan></text>
+    <g
+       id="g2031"
+       transform="translate(-2.4568452,2.9399645)">
+      <text
+         id="text838-5-3"
+         y="84.805916"
+         x="-254.56604"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="84.805916"
+           x="-254.56604"
+           id="tspan836-3-6"
+           sodipodi:role="line">DPP</tspan><tspan
+           id="tspan936-1"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="92.991119"
+           x="-254.56604"
+           sodipodi:role="line">(5)</tspan></text>
+      <rect
+         ry="7.8154301e-07"
+         y="73.432686"
+         x="-269.70468"
+         height="27.021317"
+         width="30.696136"
+         id="rect834-5-2"
+         style="fill:none;stroke:#000000;stroke-width:0.55864918;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.96187615;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-9)"
+       d="m -289.17681,89.892551 h 14.20296"
+       id="path1171-1"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.96187615;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94)"
+       d="m -288.90777,32.862737 h 14.20296"
+       id="path1171"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="-264.85431"
+       y="62.072891"
+       id="text8862-5-8"><tspan
+         sodipodi:role="line"
+         id="tspan8860-9-4"
+         x="-264.85431"
+         y="62.072891"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:14.11111069px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';fill:#000000;stroke-width:0.26458332">...</tspan></text>
+    <g
+       id="g2025-5"
+       transform="translate(40.522337,-1.1484419)" />
+    <g
+       id="g7343"
+       transform="translate(40.522337,-1.2533369)">
+      <text
+         id="text838-5-6"
+         y="31.969406"
+         x="-254.53119"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="31.969406"
+           x="-254.53119"
+           id="tspan836-3-3"
+           sodipodi:role="line">MPC</tspan><tspan
+           id="tspan936-2"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="40.15461"
+           x="-254.53119"
+           sodipodi:role="line">(0)</tspan></text>
+      <rect
+         ry="7.8154301e-07"
+         y="20.596174"
+         x="-269.66983"
+         height="27.021315"
+         width="30.696135"
+         id="rect834-5-0"
+         style="fill:none;stroke:#000000;stroke-width:0.55864918;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <g
+       id="g7343-1"
+       transform="translate(40.522337,56.303939)">
+      <text
+         id="text838-5-6-5"
+         y="31.969406"
+         x="-254.53119"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="31.969406"
+           x="-254.53119"
+           id="tspan836-3-3-5"
+           sodipodi:role="line">MPC</tspan><tspan
+           id="tspan936-2-4"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="40.15461"
+           x="-254.53119"
+           sodipodi:role="line">(5)</tspan></text>
+      <rect
+         ry="7.8154301e-07"
+         y="20.596174"
+         x="-269.66983"
+         height="27.021315"
+         width="30.696135"
+         id="rect834-5-0-7"
+         style="fill:none;stroke:#000000;stroke-width:0.55864918;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <g
+       id="g7343-2"
+       transform="translate(84.178583,-1.2533369)">
+      <text
+         id="text838-5-6-54"
+         y="31.969406"
+         x="-254.53119"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="31.969406"
+           x="-254.53119"
+           id="tspan836-3-3-7"
+           sodipodi:role="line">OPP</tspan><tspan
+           id="tspan936-2-44"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="40.15461"
+           x="-254.53119"
+           sodipodi:role="line">(0)</tspan></text>
+      <rect
+         ry="7.8154301e-07"
+         y="20.596174"
+         x="-269.66983"
+         height="27.021315"
+         width="30.696135"
+         id="rect834-5-0-3"
+         style="fill:none;stroke:#000000;stroke-width:0.55864918;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="-221.84029"
+       y="62.072891"
+       id="text8862-5-8-6"><tspan
+         sodipodi:role="line"
+         id="tspan8860-9-4-5"
+         x="-221.84029"
+         y="62.072891"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:14.11111069px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';fill:#000000;stroke-width:0.26458332">...</tspan></text>
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6)"
+       d="m -240.90479,32.86192 h 10.61353"
+       id="path1171-3"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6-7)"
+       d="m -241.09039,89.891298 h 10.61353"
+       id="path1171-3-5"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6-0)"
+       d="m -198.19009,32.86192 h 10.61353"
+       id="path1171-3-8"
+       inkscape:connector-curvature="0" />
+    <g
+       id="g7343-2-6"
+       transform="translate(84.178583,56.303939)">
+      <text
+         id="text838-5-6-54-8"
+         y="31.969406"
+         x="-254.53119"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="31.969406"
+           x="-254.53119"
+           id="tspan836-3-3-7-8"
+           sodipodi:role="line">OPP</tspan><tspan
+           id="tspan936-2-44-4"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="40.15461"
+           x="-254.53119"
+           sodipodi:role="line">(0)</tspan></text>
+      <rect
+         ry="7.8154301e-07"
+         y="20.596174"
+         x="-269.66983"
+         height="27.021315"
+         width="30.696135"
+         id="rect834-5-0-3-3"
+         style="fill:none;stroke:#000000;stroke-width:0.55864918;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6-0-1)"
+       d="m -198.00111,89.891298 h 10.61353"
+       id="path1171-3-8-9"
+       inkscape:connector-curvature="0" />
+    <g
+       id="g7343-2-2"
+       transform="translate(127.15447,-1.1484419)">
+      <text
+         id="text838-5-6-54-0"
+         y="65.122154"
+         x="-254.32814"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           id="tspan936-2-44-8"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="65.122154"
+           x="-254.32814"
+           sodipodi:role="line">OPTC</tspan></text>
+      <rect
+         ry="2.4256708e-06"
+         y="20.805576"
+         x="-269.46042"
+         height="83.865906"
+         width="30.277332"
+         id="rect834-5-0-3-9"
+         style="fill:none;stroke:#000000;stroke-width:0.97745234;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6-0-2)"
+       d="m -154.34485,32.86192 h 10.61353"
+       id="path1171-3-8-6"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6-0-2-4)"
+       d="m -154.34485,89.891298 h 10.61353"
+       id="path1171-3-8-6-5"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:3.17499995px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.26458332"
+       x="-178.18404"
+       y="62.072891"
+       id="text8862-5-8-6-0"><tspan
+         sodipodi:role="line"
+         id="tspan8860-9-4-5-4"
+         x="-178.18404"
+         y="62.072891"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:14.11111069px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';fill:#000000;stroke-width:0.26458332">...</tspan></text>
+    <path
+       style="fill:#000000;stroke:#000000;stroke-width:0.96187615;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-8)"
+       d="m -335.20986,59.030276 h 14.20296"
+       id="path1171-17"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-349.77686"
+       y="61.404655"
+       id="text838-2"><tspan
+         sodipodi:role="line"
+         id="tspan836-7"
+         x="-349.77686"
+         y="61.404655"
+         style="stroke-width:0.16370411">SDP</tspan></text>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1.32291663;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1.32291667, 1.32291667;stroke-dashoffset:0;stroke-opacity:1"
+       d="M -279.04164,107.86706 V 0.89985833"
+       id="path11907"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1.32291663;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1.3229167, 1.3229167;stroke-dashoffset:0;stroke-opacity:1"
+       d="M -235.57438,107.86707 V 0.89985663"
+       id="path11907-6-1"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1.32291663;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1.32291671, 1.32291671;stroke-dashoffset:0;stroke-opacity:1"
+       d="M -149.96277,107.86707 V 0.89985223"
+       id="path11907-6-1-0"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-257.38275"
+       y="5.003336"
+       id="text838-6"><tspan
+         sodipodi:role="line"
+         id="tspan836-1"
+         x="-257.38275"
+         y="5.003336"
+         style="font-size:4.93888903px;text-align:center;text-anchor:middle;stroke-width:0.16370411">DPPCLK</tspan><tspan
+         sodipodi:role="line"
+         x="-257.38275"
+         y="13.188541"
+         style="font-size:4.93888903px;text-align:center;text-anchor:middle;stroke-width:0.16370411"
+         id="tspan12658">267.958Mhz</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-192.85298"
+       y="5.003336"
+       id="text838-6-5"><tspan
+         sodipodi:role="line"
+         id="tspan836-1-9"
+         x="-192.85298"
+         y="5.003336"
+         style="font-size:4.93888903px;text-align:center;text-anchor:middle;stroke-width:0.16370411">DISPCLK</tspan><tspan
+         sodipodi:role="line"
+         x="-192.85298"
+         y="13.188541"
+         style="font-size:4.93888903px;text-align:center;text-anchor:middle;stroke-width:0.16370411"
+         id="tspan12662">541.275 Mhz</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-302.03766"
+       y="5.003336"
+       id="text838-6-5-4"><tspan
+         sodipodi:role="line"
+         id="tspan836-1-9-9"
+         x="-312.23135"
+         y="5.003336"
+         style="font-size:4.93888903px;stroke-width:0.16370411;text-anchor:middle;text-align:center">DCFCLK</tspan><tspan
+         sodipodi:role="line"
+         x="-312.23135"
+         y="13.188541"
+         style="font-size:4.93888903px;stroke-width:0.16370411;text-anchor:middle;text-align:center"
+         id="tspan12660">506 Mhz</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-96.075348"
+       y="4.568048"
+       id="text838-6-5-0"><tspan
+         sodipodi:role="line"
+         id="tspan836-1-9-91"
+         x="-96.075348"
+         y="4.568048"
+         style="font-size:4.93888903px;stroke-width:0.16370411">SymCLK</tspan></text>
+    <g
+       id="g7343-2-2-7"
+       transform="translate(168.61494,-1.1484419)">
+      <text
+         id="text838-5-6-54-0-7"
+         y="65.122154"
+         x="-254.32814"
+         style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+         xml:space="preserve"><tspan
+           id="tspan936-2-44-8-1"
+           style="text-align:center;text-anchor:middle;stroke-width:0.16370411"
+           y="65.122154"
+           x="-254.32814"
+           sodipodi:role="line">DIO</tspan></text>
+      <rect
+         ry="2.4256708e-06"
+         y="20.805576"
+         x="-269.46042"
+         height="83.865906"
+         width="30.277332"
+         id="rect834-5-0-3-9-1"
+         style="fill:none;stroke:#000000;stroke-width:0.97745234;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none" />
+    </g>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:1.32291663;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:1.32291672, 1.32291672;stroke-dashoffset:0;stroke-opacity:1"
+       d="M -108.38539,107.86707 V 0.89985092"
+       id="path11907-6-1-0-5"
+       inkscape:connector-curvature="0" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:6.54816437px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.16370411"
+       x="-143.52292"
+       y="5.0467439"
+       id="text838-6-5-0-9"><tspan
+         sodipodi:role="line"
+         id="tspan836-1-9-91-7"
+         x="-143.52292"
+         y="5.0467439"
+         style="font-size:4.93888903px;stroke-width:0.16370411">VirtualPCLK</tspan></text>
+    <path
+       style="fill:none;stroke:#ff0000;stroke-width:0.83149505;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#Arrow2Mend-94-6-0-2-7)"
+       d="m -112.24183,60.747629 h 10.61353"
+       id="path1171-3-8-6-7"
+       inkscape:connector-curvature="0" />
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/display/single-display-mpo-multi-video.svg b/Documentation/gpu/amdgpu/display/single-display-mpo-multi-video.svg
new file mode 100644
index 000000000000..fa807115cfe2
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/single-display-mpo-multi-video.svg
@@ -0,0 +1,339 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="112.16296mm"
+   height="93.314507mm"
+   viewBox="0 0 112.16296 93.314508"
+   version="1.1"
+   id="svg8"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)"
+   sodipodi:docname="single-display-mpo-multi-video.svg">
+  <defs
+     id="defs2">
+    <marker
+       inkscape:stockid="TriangleOutL"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="TriangleOutL"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1133"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="scale(0.8)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Send"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1006"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="TriangleOutL"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="TriangleOutL-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1133-0"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="scale(0.8)" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.979899"
+     inkscape:cx="255.95284"
+     inkscape:cy="222.25473"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="1514"
+     inkscape:window-height="1376"
+     inkscape:window-x="1046"
+     inkscape:window-y="27"
+     inkscape:window-maximized="0"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(26.189925,5.2546503)">
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813352px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-2.2053311"
+       y="32.839729"
+       id="text844"><tspan
+         sodipodi:role="line"
+         x="-2.2053313"
+         y="32.839729"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846">Video Buffer (YUV)</tspan></text>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813352px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="30.867918"
+       y="87.638779"
+       id="text844-2"><tspan
+         sodipodi:role="line"
+         x="30.867918"
+         y="87.638779"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-0">CRTC Output</tspan></text>
+    <rect
+       style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:0.53249496;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:1.59748484, 0.53249495;stroke-dashoffset:0"
+       id="rect969"
+       width="111.63046"
+       height="44.238331"
+       x="-25.923677"
+       y="-4.9884028" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813352px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="48.037342"
+       y="45.781586"
+       id="text844-2-9"><tspan
+         sodipodi:role="line"
+         x="48.037342"
+         y="45.781586"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan2868">Hardware Composition</tspan></text>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.48997903;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleOutL)"
+       d="m 32.66992,40.326865 v 9.760144"
+       id="path989"
+       inkscape:connector-curvature="0" />
+    <g
+       id="g4447"
+       style="stroke:#000000">
+      <rect
+         ry="0"
+         y="0.23764905"
+         x="-23.784248"
+         height="29.48851"
+         width="48.988979"
+         id="rect836"
+         style="fill:#000000;fill-opacity:1;stroke:#000000;stroke-width:0.62916809;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="3.8184125"
+         x="-20.589451"
+         height="6.9794898"
+         width="11.594959"
+         id="rect836-2-6"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891468;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="3.8184123"
+         x="-5.6613555"
+         height="6.9794903"
+         width="11.594959"
+         id="rect836-2-6-1"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="3.8184121"
+         x="10.374816"
+         height="6.9794908"
+         width="11.594959"
+         id="rect836-2-6-1-8"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="18.386644"
+         x="-20.341679"
+         height="6.9794903"
+         width="11.594959"
+         id="rect836-2-6-7"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="18.386644"
+         x="-5.4135842"
+         height="6.9794908"
+         width="11.594959"
+         id="rect836-2-6-1-9"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="18.386644"
+         x="10.622585"
+         height="6.9794908"
+         width="11.594959"
+         id="rect836-2-6-1-8-2"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+    </g>
+    <rect
+       style="fill:#808080;fill-opacity:1;stroke:#808080;stroke-width:0.62916809;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836-2"
+       width="48.988979"
+       height="29.48851"
+       x="31.367332"
+       y="0.45039755"
+       ry="0" />
+    <rect
+       style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.14891468;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836-2-6-3"
+       width="11.594959"
+       height="6.9794898"
+       x="34.56213"
+       y="4.0311608"
+       ry="0" />
+    <rect
+       style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836-2-6-1-7"
+       width="11.594959"
+       height="6.9794903"
+       x="49.490223"
+       y="4.0311608"
+       ry="0" />
+    <rect
+       style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836-2-6-1-8-5"
+       width="11.594959"
+       height="6.9794908"
+       x="65.526398"
+       y="4.0311604"
+       ry="0" />
+    <rect
+       style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836-2-6-7-9"
+       width="11.594959"
+       height="6.9794903"
+       x="34.809902"
+       y="18.599392"
+       ry="0" />
+    <rect
+       style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836-2-6-1-9-2"
+       width="11.594959"
+       height="6.9794908"
+       x="49.737995"
+       y="18.599392"
+       ry="0" />
+    <rect
+       style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836-2-6-1-8-2-2"
+       width="11.594959"
+       height="6.9794908"
+       x="65.774162"
+       y="18.599392"
+       ry="0" />
+    <g
+       id="g4562"
+       transform="translate(0,11.22532)">
+      <rect
+         ry="0"
+         y="43.21352"
+         x="5.9767256"
+         height="29.48851"
+         width="48.988979"
+         id="rect836-97"
+         style="fill:#808080;fill-opacity:1;stroke:#808080;stroke-width:0.62916809;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="46.794285"
+         x="9.1715231"
+         height="6.9794898"
+         width="11.594959"
+         id="rect836-2-6-36"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891468;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="46.794285"
+         x="24.099619"
+         height="6.9794903"
+         width="11.594959"
+         id="rect836-2-6-1-1"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="46.794285"
+         x="40.135792"
+         height="6.9794908"
+         width="11.594959"
+         id="rect836-2-6-1-8-29"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="61.362514"
+         x="9.4192953"
+         height="6.9794903"
+         width="11.594959"
+         id="rect836-2-6-7-3"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="61.362514"
+         x="24.347389"
+         height="6.9794908"
+         width="11.594959"
+         id="rect836-2-6-1-9-1"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <rect
+         ry="0"
+         y="61.362514"
+         x="40.38356"
+         height="6.9794908"
+         width="11.594959"
+         id="rect836-2-6-1-8-2-9"
+         style="fill:#008000;fill-opacity:1;stroke:#000000;stroke-width:0.14891469;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813358px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="57.012939"
+       y="33.603645"
+       id="text844-4"><tspan
+         sodipodi:role="line"
+         x="57.012939"
+         y="33.603645"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-7">Desktop Buffer (ARGB)</tspan></text>
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/display/single-display-mpo.svg b/Documentation/gpu/amdgpu/display/single-display-mpo.svg
new file mode 100644
index 000000000000..fb53b0920c87
--- /dev/null
+++ b/Documentation/gpu/amdgpu/display/single-display-mpo.svg
@@ -0,0 +1,266 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="148.71147mm"
+   height="38.356358mm"
+   viewBox="0 0 148.71148 38.356358"
+   version="1.1"
+   id="svg8"
+   inkscape:version="0.92.5 (2060ec1f9f, 2020-04-08)"
+   sodipodi:docname="single-display-mpo.svg">
+  <defs
+     id="defs2">
+    <marker
+       inkscape:stockid="TriangleOutL"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="TriangleOutL"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1133"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="scale(0.8)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="Arrow1Send"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="Arrow1Send"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         id="path1006"
+         d="M 0,0 5,-5 -12.5,0 5,5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="matrix(-0.2,0,0,-0.2,-1.2,0)"
+         inkscape:connector-curvature="0" />
+    </marker>
+    <marker
+       inkscape:stockid="TriangleOutL"
+       orient="auto"
+       refY="0"
+       refX="0"
+       id="TriangleOutL-6"
+       style="overflow:visible"
+       inkscape:isstock="true">
+      <path
+         inkscape:connector-curvature="0"
+         id="path1133-0"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         style="fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:1.00000003pt;stroke-opacity:1"
+         transform="scale(0.8)" />
+    </marker>
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1.979899"
+     inkscape:cx="266.07058"
+     inkscape:cy="81.254796"
+     inkscape:document-units="mm"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="1514"
+     inkscape:window-height="1376"
+     inkscape:window-x="1046"
+     inkscape:window-y="27"
+     inkscape:window-maximized="0"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0" />
+  <metadata
+     id="metadata5">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(26.323559,-14.790568)">
+    <rect
+       style="fill:#999999;fill-opacity:1;stroke:#999999;stroke-width:0.62916809;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect836"
+       width="48.988979"
+       height="29.48851"
+       x="4.9472256"
+       y="17.075632"
+       ry="0" />
+    <g
+       id="g905"
+       transform="matrix(0.26082945,0,0,0.26082945,-26.950235,-0.99631462)">
+      <rect
+         y="67.568451"
+         x="12.473214"
+         height="60.854164"
+         width="102.43154"
+         id="rect834"
+         style="fill:#008000;fill-opacity:1;stroke:none;stroke-width:2;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <path
+         inkscape:transform-center-y="-3.581986e-06"
+         inkscape:transform-center-x="-3.3106064"
+         d="m 73.620805,97.995537 -9.931819,5.734143 -9.931819,5.73413 0,-11.468274 0,-11.468276 9.931819,5.734139 z"
+         inkscape:randomized="0"
+         inkscape:rounded="0"
+         inkscape:flatsided="false"
+         sodipodi:arg2="1.0471976"
+         sodipodi:arg1="0"
+         sodipodi:r2="6.6212125"
+         sodipodi:r1="13.242425"
+         sodipodi:cy="97.995537"
+         sodipodi:cx="60.37838"
+         sodipodi:sides="3"
+         id="path840"
+         style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1.22854495;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+         sodipodi:type="star" />
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813352px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="-10.357052"
+       y="34.176075"
+       id="text844"><tspan
+         sodipodi:role="line"
+         id="tspan842"
+         x="-10.357052"
+         y="34.176075"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112">Video Buffer (YUV)</tspan><tspan
+         sodipodi:role="line"
+         x="-10.357052"
+         y="36.706482"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846">DRM PRIMARY PLANE</tspan></text>
+    <rect
+       style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:0.5216589;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+       id="rect834-3"
+       width="26.717165"
+       height="15.872559"
+       x="16.083126"
+       y="23.883608" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813352px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="28.437061"
+       y="49.158085"
+       id="text844-6"><tspan
+         sodipodi:role="line"
+         x="28.437061"
+         y="49.158085"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-5">Desktop Buffer (ARGB)</tspan><tspan
+         sodipodi:role="line"
+         x="28.437061"
+         y="51.688492"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan886">DRM OVERLAY PLANE</tspan></text>
+    <g
+       id="g2874"
+       transform="matrix(1.0858687,0,0,1.0858687,-4.5597909,-2.06277)">
+      <rect
+         ry="0"
+         y="17.624969"
+         x="71.504173"
+         height="27.156607"
+         width="45.115009"
+         id="rect836-9"
+         style="fill:#999999;fill-opacity:1;stroke:#999999;stroke-width:0.57941455;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0" />
+      <g
+         id="g905-5"
+         transform="matrix(0.24020349,0,0,0.24020349,78.763353,7.664402)">
+        <rect
+           style="fill:#008000;fill-opacity:1;stroke:none;stroke-width:2;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+           id="rect834-6"
+           width="102.43154"
+           height="60.854164"
+           x="12.473214"
+           y="67.568451" />
+        <path
+           sodipodi:type="star"
+           style="fill:#ffffff;fill-opacity:1;stroke:none;stroke-width:1.22854495;stroke-linecap:round;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0"
+           id="path840-2"
+           sodipodi:sides="3"
+           sodipodi:cx="60.37838"
+           sodipodi:cy="97.995537"
+           sodipodi:r1="13.242425"
+           sodipodi:r2="6.6212125"
+           sodipodi:arg1="0"
+           sodipodi:arg2="1.0471976"
+           inkscape:flatsided="false"
+           inkscape:rounded="0"
+           inkscape:randomized="0"
+           d="m 73.620805,97.995537 -9.931819,5.734143 -9.931819,5.73413 0,-11.468274 0,-11.468276 9.931819,5.734139 z"
+           inkscape:transform-center-x="-3.3106064"
+           inkscape:transform-center-y="-3.581986e-06" />
+      </g>
+    </g>
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813352px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="97.551666"
+       y="50.221046"
+       id="text844-2"><tspan
+         sodipodi:role="line"
+         x="97.551666"
+         y="50.221046"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-0">CRTC Output</tspan></text>
+    <rect
+       style="fill:none;fill-opacity:1;stroke:#000000;stroke-width:0.42211887;stroke-linecap:butt;stroke-linejoin:bevel;stroke-miterlimit:4;stroke-dasharray:1.26635657, 0.42211886;stroke-dashoffset:0"
+       id="rect969"
+       width="81.806648"
+       height="37.934238"
+       x="-26.112499"
+       y="15.001627" />
+    <text
+       xml:space="preserve"
+       style="font-style:normal;font-weight:normal;font-size:0.82813352px;line-height:1.25;font-family:sans-serif;letter-spacing:0px;word-spacing:0px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.06901112"
+       x="64.474419"
+       y="31.883568"
+       id="text844-2-9"><tspan
+         sodipodi:role="line"
+         x="64.474419"
+         y="31.883568"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan846-0-3">Hardware</tspan><tspan
+         sodipodi:role="line"
+         x="64.474419"
+         y="34.413975"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:2.02432632px;font-family:sans-serif;-inkscape-font-specification:'sans-serif Bold';text-align:center;text-anchor:middle;stroke-width:0.06901112"
+         id="tspan2868">Composition</tspan></text>
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.48997903;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleOutL)"
+       d="m 57.401149,24.81643 h 9.760144"
+       id="path989"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:none;stroke:#000000;stroke-width:0.48997903;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleOutL-6)"
+       d="m 57.401149,41.794278 h 9.760144"
+       id="path989-6"
+       inkscape:connector-curvature="0" />
+  </g>
+</svg>
diff --git a/Documentation/gpu/amdgpu/driver-misc.rst b/Documentation/gpu/amdgpu/driver-misc.rst
index e3d6b2fa2493..1800543d45f7 100644
--- a/Documentation/gpu/amdgpu/driver-misc.rst
+++ b/Documentation/gpu/amdgpu/driver-misc.rst
@@ -32,6 +32,23 @@ unique_id
 .. kernel-doc:: drivers/gpu/drm/amd/pm/amdgpu_pm.c
    :doc: unique_id
 
+Accelerated Processing Units (APU) Info
+---------------------------------------
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 3, 2, 2, 1, 1, 1
+   :file: ./apu-asic-info-table.csv
+
+Discrete GPU Info
+-----------------
+
+.. csv-table::
+   :header-rows: 1
+   :widths: 3, 2, 2, 1, 1, 1
+   :file: ./dgpu-asic-info-table.csv
+
+
 GPU Memory Usage Information
 ============================
 
diff --git a/Documentation/gpu/amdgpu/thermal.rst b/Documentation/gpu/amdgpu/thermal.rst
index 997231b6adcf..5e27e4eb3959 100644
--- a/Documentation/gpu/amdgpu/thermal.rst
+++ b/Documentation/gpu/amdgpu/thermal.rst
@@ -72,7 +72,8 @@ card's RLC (RunList Controller) firmware powers off the gfx engine
 dynamically when there is no workload on gfx or compute pipes. GFXOFF is on by
 default on supported GPUs.
 
-Userspace can interact with GFXOFF through a debugfs interface:
+Userspace can interact with GFXOFF through a debugfs interface (all values in
+`uint32_t`, unless otherwise noted):
 
 ``amdgpu_gfxoff``
 -----------------
@@ -104,3 +105,18 @@ Read it to check current GFXOFF's status of a GPU::
 If GFXOFF is enabled, the value will be transitioning around [0, 3], always
 getting into 0 when possible. When it's disabled, it's always at 2. Returns
 ``-EINVAL`` if it's not supported.
+
+``amdgpu_gfxoff_count``
+-----------------------
+
+Read it to get the total GFXOFF entry count at the time of query since system
+power-up. The value is an `uint64_t` type, however, due to firmware limitations,
+it can currently overflow as an `uint32_t`. *Only supported in vangogh*
+
+``amdgpu_gfxoff_residency``
+---------------------------
+
+Write 1 to amdgpu_gfxoff_residency to start logging, and 0 to stop. Read it to
+get average GFXOFF residency % multiplied by 100 during the last logging
+interval. E.g. a value of 7854 means 78.54% of the time in the last logging
+interval the GPU was in GFXOFF mode. *Only supported in vangogh*
diff --git a/Documentation/gpu/drm-kms-helpers.rst b/Documentation/gpu/drm-kms-helpers.rst
index 2d473bc64c9f..dbc85fd7a971 100644
--- a/Documentation/gpu/drm-kms-helpers.rst
+++ b/Documentation/gpu/drm-kms-helpers.rst
@@ -122,13 +122,13 @@ format Helper Functions Reference
 .. kernel-doc:: drivers/gpu/drm/drm_format_helper.c
    :export:
 
-Framebuffer CMA Helper Functions Reference
+Framebuffer DMA Helper Functions Reference
 ==========================================
 
-.. kernel-doc:: drivers/gpu/drm/drm_fb_cma_helper.c
-   :doc: framebuffer cma helper functions
+.. kernel-doc:: drivers/gpu/drm/drm_fb_dma_helper.c
+   :doc: framebuffer dma helper functions
 
-.. kernel-doc:: drivers/gpu/drm/drm_fb_cma_helper.c
+.. kernel-doc:: drivers/gpu/drm/drm_fb_dma_helper.c
    :export:
 
 Framebuffer GEM Helper Reference
diff --git a/Documentation/gpu/drm-kms.rst b/Documentation/gpu/drm-kms.rst
index 6f9c064fd323..b4377a545425 100644
--- a/Documentation/gpu/drm-kms.rst
+++ b/Documentation/gpu/drm-kms.rst
@@ -532,6 +532,8 @@ Standard Plane Properties
 .. kernel-doc:: drivers/gpu/drm/drm_plane.c
    :doc: standard plane properties
 
+.. _plane_composition_properties:
+
 Plane Composition Properties
 ----------------------------
 
diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst
index f32ccce5722d..a79fd3549ff8 100644
--- a/Documentation/gpu/drm-mm.rst
+++ b/Documentation/gpu/drm-mm.rst
@@ -300,12 +300,12 @@ Drivers that want to map the GEM object upfront instead of handling page
 faults can implement their own mmap file operation handler.
 
 For platforms without MMU the GEM core provides a helper method
-drm_gem_cma_get_unmapped_area(). The mmap() routines will call this to get a
+drm_gem_dma_get_unmapped_area(). The mmap() routines will call this to get a
 proposed address for the mapping.
 
-To use drm_gem_cma_get_unmapped_area(), drivers must fill the struct
+To use drm_gem_dma_get_unmapped_area(), drivers must fill the struct
 :c:type:`struct file_operations <file_operations>` get_unmapped_area field with
-a pointer on drm_gem_cma_get_unmapped_area().
+a pointer on drm_gem_dma_get_unmapped_area().
 
 More detailed information about get_unmapped_area can be found in
 Documentation/admin-guide/mm/nommu-mmap.rst
@@ -355,16 +355,16 @@ GEM Function Reference
 .. kernel-doc:: drivers/gpu/drm/drm_gem.c
    :export:
 
-GEM CMA Helper Functions Reference
+GEM DMA Helper Functions Reference
 ----------------------------------
 
-.. kernel-doc:: drivers/gpu/drm/drm_gem_cma_helper.c
-   :doc: cma helpers
+.. kernel-doc:: drivers/gpu/drm/drm_gem_dma_helper.c
+   :doc: dma helpers
 
-.. kernel-doc:: include/drm/drm_gem_cma_helper.h
+.. kernel-doc:: include/drm/drm_gem_dma_helper.h
    :internal:
 
-.. kernel-doc:: drivers/gpu/drm/drm_gem_cma_helper.c
+.. kernel-doc:: drivers/gpu/drm/drm_gem_dma_helper.c
    :export:
 
 GEM SHMEM Helper Function Reference
diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
index 513b20ccef1e..393d218e4a0c 100644
--- a/Documentation/gpu/todo.rst
+++ b/Documentation/gpu/todo.rst
@@ -322,18 +322,6 @@ Contact: Daniel Vetter, Noralf Tronnes
 
 Level: Advanced
 
-idr_init_base()
----------------
-
-DRM core&drivers uses a lot of idr (integer lookup directories) for mapping
-userspace IDs to internal objects, and in most places ID=0 means NULL and hence
-is never used. Switching to idr_init_base() for these would make the idr more
-efficient.
-
-Contact: Daniel Vetter
-
-Level: Starter
-
 struct drm_gem_object_funcs
 ---------------------------
 
@@ -343,19 +331,6 @@ converted, except for struct drm_driver.gem_prime_mmap.
 
 Level: Intermediate
 
-Rename CMA helpers to DMA helpers
----------------------------------
-
-CMA (standing for contiguous memory allocator) is really a bit an accident of
-what these were used for first, a much better name would be DMA helpers. In the
-text these should even be called coherent DMA memory helpers (so maybe CDM, but
-no one knows what that means) since underneath they just use dma_alloc_coherent.
-
-Contact: Laurent Pinchart, Daniel Vetter
-
-Level: Intermediate (mostly because it is a huge tasks without good partial
-milestones, not technically itself that challenging)
-
 connector register/unregister fixes
 -----------------------------------
 
@@ -617,17 +592,6 @@ Contact: Javier Martinez Canillas <javierm@redhat.com>
 
 Level: Intermediate
 
-Convert Kernel Selftests (kselftest) to KUnit tests when appropriate
---------------------------------------------------------------------
-
-Many of the `Kselftest <https://www.kernel.org/doc/html/latest/dev-tools/kselftest.html>`_
-tests in DRM could be converted to Kunit tests instead, since that framework
-is more suitable for unit testing.
-
-Contact: Javier Martinez Canillas <javierm@redhat.com>
-
-Level: Starter
-
 Enable trinity for DRM
 ----------------------
 
@@ -715,6 +679,74 @@ Contact: Sam Ravnborg
 
 Level: Advanced
 
+Brightness handling on devices with multiple internal panels
+============================================================
+
+On x86/ACPI devices there can be multiple backlight firmware interfaces:
+(ACPI) video, vendor specific and others. As well as direct/native (PWM)
+register programming by the KMS driver.
+
+To deal with this backlight drivers used on x86/ACPI call
+acpi_video_get_backlight_type() which has heuristics (+quirks) to select
+which backlight interface to use; and backlight drivers which do not match
+the returned type will not register themselves, so that only one backlight
+device gets registered (in a single GPU setup, see below).
+
+At the moment this more or less assumes that there will only
+be 1 (internal) panel on a system.
+
+On systems with 2 panels this may be a problem, depending on
+what interface acpi_video_get_backlight_type() selects:
+
+1. native: in this case the KMS driver is expected to know which backlight
+   device belongs to which output so everything should just work.
+2. video: this does support controlling multiple backlights, but some work
+   will need to be done to get the output <-> backlight device mapping
+
+The above assumes both panels will require the same backlight interface type.
+Things will break on systems with multiple panels where the 2 panels need
+a different type of control. E.g. one panel needs ACPI video backlight control,
+where as the other is using native backlight control. Currently in this case
+only one of the 2 required backlight devices will get registered, based on
+the acpi_video_get_backlight_type() return value.
+
+If this (theoretical) case ever shows up, then supporting this will need some
+work. A possible solution here would be to pass a device and connector-name
+to acpi_video_get_backlight_type() so that it can deal with this.
+
+Note in a way we already have a case where userspace sees 2 panels,
+in dual GPU laptop setups with a mux. On those systems we may see
+either 2 native backlight devices; or 2 native backlight devices.
+
+Userspace already has code to deal with this by detecting if the related
+panel is active (iow which way the mux between the GPU and the panels
+points) and then uses that backlight device. Userspace here very much
+assumes a single panel though. It picks only 1 of the 2 backlight devices
+and then only uses that one.
+
+Note that all userspace code (that I know off) is currently hardcoded
+to assume a single panel.
+
+Before the recent changes to not register multiple (e.g. video + native)
+/sys/class/backlight devices for a single panel (on a single GPU laptop),
+userspace would see multiple backlight devices all controlling the same
+backlight.
+
+To deal with this userspace had to always picks one preferred device under
+/sys/class/backlight and will ignore the others. So to support brightness
+control on multiple panels userspace will need to be updated too.
+
+There are plans to allow brightness control through the KMS API by adding
+a "display brightness" property to drm_connector objects for panels. This
+solves a number of issues with the /sys/class/backlight API, including not
+being able to map a sysfs backlight device to a specific connector. Any
+userspace changes to add support for brightness control on devices with
+multiple panels really should build on top of this new KMS property.
+
+Contact: Hans de Goede
+
+Level: Advanced
+
 Outside DRM
 ===========
 
diff --git a/Documentation/gpu/vkms.rst b/Documentation/gpu/vkms.rst
index 973e2d43108b..49db221c0f52 100644
--- a/Documentation/gpu/vkms.rst
+++ b/Documentation/gpu/vkms.rst
@@ -118,15 +118,10 @@ Add Plane Features
 
 There's lots of plane features we could add support for:
 
-- Clearing primary plane: clear primary plane before plane composition (at the
-  start) for correctness of pixel blend ops. It also guarantees alpha channel
-  is cleared in the target buffer for stable crc. [Good to get started]
-
 - ARGB format on primary plane: blend the primary plane into background with
   translucent alpha.
 
-- Support when the primary plane isn't exactly matching the output size: blend
-  the primary plane into the black background.
+- Add background color KMS property[Good to get started].
 
 - Full alpha blending on all planes.
 
diff --git a/Documentation/hwmon/aquacomputer_d5next.rst b/Documentation/hwmon/aquacomputer_d5next.rst
index 33649a1e3a05..e238533b5fe0 100644
--- a/Documentation/hwmon/aquacomputer_d5next.rst
+++ b/Documentation/hwmon/aquacomputer_d5next.rst
@@ -10,6 +10,7 @@ Supported devices:
 * Aquacomputer Farbwerk 360 RGB controller
 * Aquacomputer Octo fan controller
 * Aquacomputer Quadro fan controller
+* Aquacomputer High Flow Next sensor
 
 Author: Aleksa Savic
 
@@ -20,10 +21,11 @@ This driver exposes hardware sensors of listed Aquacomputer devices, which
 communicate through proprietary USB HID protocols.
 
 For the D5 Next pump, available sensors are pump and fan speed, power, voltage
-and current, as well as coolant temperature. Also available through debugfs are
-the serial number, firmware version and power-on count. Attaching a fan to it is
-optional and allows it to be controlled using temperature curves directly from the
-pump. If it's not connected, the fan-related sensors will report zeroes.
+and current, as well as coolant temperature and eight virtual temp sensors. Also
+available through debugfs are the serial number, firmware version and power-on
+count. Attaching a fan to it is optional and allows it to be controlled using
+temperature curves directly from the pump. If it's not connected, the fan-related
+sensors will report zeroes.
 
 The pump can be configured either through software or via its physical
 interface. Configuring the pump through this driver is not implemented, as it
@@ -31,14 +33,23 @@ seems to require sending it a complete configuration. That includes addressable
 RGB LEDs, for which there is no standard sysfs interface. Thus, that task is
 better suited for userspace tools.
 
-The Octo exposes four temperature sensors and eight PWM controllable fans, along
-with their speed (in RPM), power, voltage and current.
+The Octo exposes four physical and sixteen virtual temperature sensors, as well as
+eight PWM controllable fans, along with their speed (in RPM), power, voltage and
+current.
 
-The Quadro exposes four temperature sensors, a flow sensor and four PWM controllable
-fans, along with their speed (in RPM), power, voltage and current.
+The Quadro exposes four physical and sixteen virtual temperature sensors, a flow
+sensor and four PWM controllable fans, along with their speed (in RPM), power,
+voltage and current.
 
-The Farbwerk and Farbwerk 360 expose four temperature sensors. Depending on the device,
-not all sysfs and debugfs entries will be available.
+The Farbwerk and Farbwerk 360 expose four temperature sensors. Additionally,
+sixteen virtual temperature sensors of the Farbwerk 360 are exposed.
+
+The High Flow Next exposes +5V voltages, water quality, conductivity and flow readings.
+A temperature sensor can be connected to it, in which case it provides its reading
+and an estimation of the dissipated/absorbed power in the liquid cooling loop.
+
+Depending on the device, not all sysfs and debugfs entries will be available.
+Writing to virtual temperature sensors is not currently supported.
 
 Usage notes
 -----------
@@ -49,14 +60,14 @@ the kernel and supports hotswapping.
 Sysfs entries
 -------------
 
-================ ==============================================
-temp[1-4]_input  Temperature sensors (in millidegrees Celsius)
+================ ==============================================================
+temp[1-20]_input Physical/virtual temperature sensors (in millidegrees Celsius)
 fan[1-8]_input   Pump/fan speed (in RPM) / Flow speed (in dL/h)
 power[1-8]_input Pump/fan power (in micro Watts)
 in[0-7]_input    Pump/fan voltage (in milli Volts)
 curr[1-8]_input  Pump/fan current (in milli Amperes)
 pwm[1-8]         Fan PWM (0 - 255)
-================ ==============================================
+================ ==============================================================
 
 Debugfs entries
 ---------------
diff --git a/Documentation/hwmon/asus_wmi_ec_sensors.rst b/Documentation/hwmon/asus_wmi_ec_sensors.rst
deleted file mode 100644
index 1b287f229e86..000000000000
--- a/Documentation/hwmon/asus_wmi_ec_sensors.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0-or-later
-
-Kernel driver asus_wmi_ec_sensors
-=================================
-
-Supported boards:
- * PRIME X570-PRO,
- * Pro WS X570-ACE,
- * ROG CROSSHAIR VIII DARK HERO,
- * ROG CROSSHAIR VIII FORMULA,
- * ROG CROSSHAIR VIII HERO,
- * ROG STRIX B550-E GAMING,
- * ROG STRIX B550-I GAMING,
- * ROG STRIX X570-E GAMING.
-
-Authors:
-    - Eugene Shalygin <eugene.shalygin@gmail.com>
-
-Description:
-------------
-ASUS mainboards publish hardware monitoring information via Super I/O
-chip and the ACPI embedded controller (EC) registers. Some of the sensors
-are only available via the EC.
-
-ASUS WMI interface provides a method (BREC) to read data from EC registers,
-which is utilized by this driver to publish those sensor readings to the
-HWMON system. The driver is aware of and reads the following sensors:
-
-1. Chipset (PCH) temperature
-2. CPU package temperature
-3. Motherboard temperature
-4. Readings from the T_Sensor header
-5. VRM temperature
-6. CPU_Opt fan RPM
-7. Chipset fan RPM
-8. Readings from the "Water flow meter" header (RPM)
-9. Readings from the "Water In" and "Water Out" temperature headers
-10. CPU current
diff --git a/Documentation/hwmon/corsair-psu.rst b/Documentation/hwmon/corsair-psu.rst
index e8378e7a1d8c..3c1b164eb3c0 100644
--- a/Documentation/hwmon/corsair-psu.rst
+++ b/Documentation/hwmon/corsair-psu.rst
@@ -15,7 +15,7 @@ Supported devices:
 
   Corsair HX850i
 
-  Corsair HX1000i
+  Corsair HX1000i (revision 1 and 2)
 
   Corsair HX1200i
 
@@ -86,8 +86,9 @@ Debugfs entries
 ---------------
 
 =======================	========================================================
-uptime			Current uptime of the psu
+ocpmode                 Single or multi rail mode of the PCIe power connectors
+product                 Product name of the psu
+uptime			Session uptime of the psu
 uptime_total		Total uptime of the psu
 vendor			Vendor name of the psu
-product			Product name of the psu
 =======================	========================================================
diff --git a/Documentation/hwmon/emc2305.rst b/Documentation/hwmon/emc2305.rst
new file mode 100644
index 000000000000..2403dbaf2728
--- /dev/null
+++ b/Documentation/hwmon/emc2305.rst
@@ -0,0 +1,37 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver emc2305
+=====================
+
+Supported chips:
+   Microchip EMC2305, EMC2303, EMC2302, EMC2301
+
+   Addresses scanned: I2C 0x27, 0x2c, 0x2d, 0x2e, 0x2f, 0x4c, 0x4d
+   Prefixes: 'emc2305'
+
+   Datasheet: Publicly available at the Microchip website :
+      https://www.microchip.com/en-us/product/EMC2305
+
+Description:
+------------
+This driver implements support for Microchip EMC2301/2/3/5 RPM-based PWM Fan Controller.
+The EMC2305 Fan Controller supports up to 5 independently controlled PWM fan drives.
+Fan rotation speeds are reported in RPM.
+The driver supports the RPM-based PWM control to keep a fan at the desired speed.
+The driver provides the possibility to have one common PWM interface for all FANs
+or up to the maximum available or configured independent PWMs.
+
+The driver provides the following sysfs interfaces in hwmon subsystem:
+
+================= == ===================================================
+fan[1-5]_fault    RO files for tachometers TACH1-TACH5 fault indication
+fan[1-5]_input    RO files for tachometers TACH1-TACH5 input (in RPM)
+pwm[1-5]          RW file for fan[1-5] target duty cycle (0..255)
+================= == ===================================================
+
+sysfs interfaces in thermal subsystem:
+
+================= == ========================================================================
+cur_state         RW file for the current cooling state of the cooling device (0..max_state)
+max_state         RO file for the maximum cooling state of the cooling device
+================= == ========================================================================
diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst
index f7113b0f8b2a..c1d11cf13eef 100644
--- a/Documentation/hwmon/index.rst
+++ b/Documentation/hwmon/index.rst
@@ -44,7 +44,6 @@ Hardware Monitoring Kernel Drivers
    asc7621
    aspeed-pwm-tacho
    asus_ec_sensors
-   asus_wmi_ec_sensors
    asus_wmi_sensors
    bcm54140
    bel-pfe
@@ -63,6 +62,7 @@ Hardware Monitoring Kernel Drivers
    ds620
    emc1403
    emc2103
+   emc2305
    emc6w201
    f71805f
    f71882fg
@@ -133,6 +133,7 @@ Hardware Monitoring Kernel Drivers
    max20751
    max31722
    max31730
+   max31760
    max31785
    max31790
    max34440
@@ -205,6 +206,7 @@ Hardware Monitoring Kernel Drivers
    tps23861
    tps40422
    tps53679
+   tps546d24
    twl4030-madc-hwmon
    ucd9000
    ucd9200
diff --git a/Documentation/hwmon/max31760.rst b/Documentation/hwmon/max31760.rst
new file mode 100644
index 000000000000..b1b55fb843c1
--- /dev/null
+++ b/Documentation/hwmon/max31760.rst
@@ -0,0 +1,77 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+Kernel driver max31760
+======================
+
+Supported chips:
+  * Analog Devices MAX31760
+
+    Prefix: 'max31760'
+
+    Addresses scanned: none
+
+    Datasheet: https://datasheets.maximintegrated.com/en/ds/MAX31760.pdf
+
+
+Author: Ibrahim Tilki <Ibrahim.Tilki@analog.com>
+
+Description
+-----------
+
+The MAX31760 integrates temperature sensing along with precision PWM fan
+control. It accurately measures its local die temperature and the remote
+temperature of a discrete diode-connected transistor, such as a 2N3906,
+or a thermal diode commonly found on CPUs, graphics processor units (GPUs),
+and other ASICs. Multiple temperature thresholds, such as local
+high/overtemperature (OT) and remote high/overtemperature, can be set by an
+I2C-compatible interface. Fan speed is controlled based on the temperature
+reading as an index to a 48-byte lookup table (LUT) containing
+user-programmed PWM values. The flexible LUT-based architecture enables
+the user to program a smooth nonlinear fan speed vs. temperature transfer
+function to minimize acoustic fan noise. Two tachometer inputs allow
+measuring the speeds of two fans independently. When the local or remote
+OT threshold is exceeded, the SHDN pin is asserted low and can be used to
+shut down the system. A dedicated ALERT pin reports that either a local or
+remote high-temperature threshold has been exceeded.
+
+Temperature measurement range: from -55°C to 125°C
+
+Temperature Resolution: 11 Bits, ±0.125°C
+
+Please refer how to instantiate this driver: Documentation/i2c/instantiating-devices.rst
+
+Lookup table for auto fan control
+---------------------------------
+
+========= =================================
+LUT Index Name
+========= =================================
+1         PWM value for T < +18°C
+2         PWM value for +18°C ≤ T < +20°C
+3         PWM value for +20°C ≤ T < +22°C
+...       ...
+47        PWM value for +108°C ≤ T < +110°C
+48        PWM value for T ≥ +110°C
+========= =================================
+
+Sysfs entries
+-------------
+
+=============================== =================================================================================
+fan[1-2]_input                  Fan speed (in RPM)
+fan[1-2]_enable                 Enable fan readings and fan fault alarms
+fan[1-2]_fault                  Fan fault status
+temp[1-2]_label                 "Remote" and "Local" temperature channel labels
+temp[1-2]_input                 Temperature sensor readings (in millidegrees Celsius)
+temp1_fault                     Remote temperature sensor fault status
+temp[1-2]_max                   Temperature max value. Asserts "ALERT" pin when exceeded
+temp[1-2]_max_alarm             Temperature max alarm status
+temp[1-2]_crit                  Temperature critical value. Asserts "SHDN" pin when exceeded
+temp[1-2]_crit_alarm            Temperature critical alarm status
+pwm1                            PWM value for direct fan control
+pwm1_enable                     1: direct fan control, 2: temperature based auto fan control
+pwm1_freq                       PWM frequency in hertz
+pwm1_auto_channels_temp         Temperature source for auto fan control. 1: temp1, 2: temp2, 3: max(temp1, temp2)
+pwm1_auto_point[1-48]_pwm       PWM value for LUT point
+pwm1_auto_point_temp_hyst       Temperature hysteresis for auto fan control. Can be either 2000mC or 4000mC
+=============================== =================================================================================
diff --git a/Documentation/hwmon/max31790.rst b/Documentation/hwmon/max31790.rst
index 7b097c3b9b90..33c5c7330efc 100644
--- a/Documentation/hwmon/max31790.rst
+++ b/Documentation/hwmon/max31790.rst
@@ -38,6 +38,7 @@ Sysfs entries
 fan[1-12]_input    RO  fan tachometer speed in RPM
 fan[1-12]_fault    RO  fan experienced fault
 fan[1-6]_target    RW  desired fan speed in RPM
+fan[1-6]_enable    RW  enable or disable the tachometer input
 pwm[1-6]_enable    RW  regulator mode, 0=disabled (duty cycle=0%), 1=manual mode, 2=rpm mode
 pwm[1-6]           RW  read: current pwm duty cycle,
                        write: target pwm duty cycle (0-255)
diff --git a/Documentation/hwmon/pwm-fan.rst b/Documentation/hwmon/pwm-fan.rst
index 82fe96742fee..f77998b204ef 100644
--- a/Documentation/hwmon/pwm-fan.rst
+++ b/Documentation/hwmon/pwm-fan.rst
@@ -18,3 +18,15 @@ the hwmon's sysfs interface.
 
 The fan rotation speed returned via the optional 'fan1_input' is extrapolated
 from the sampled interrupts from the tachometer signal within 1 second.
+
+The driver provides the following sensor accesses in sysfs:
+
+=============== ======= =======================================================
+fan1_input	ro	fan tachometer speed in RPM
+pwm1_enable	rw	keep enable mode, defines behaviour when pwm1=0
+			0 -> disable pwm and regulator
+			1 -> enable pwm; if pwm==0, disable pwm, keep regulator enabled
+			2 -> enable pwm; if pwm==0, keep pwm and regulator enabled
+			3 -> enable pwm; if pwm==0, disable pwm and regulator
+pwm1		rw	relative speed (0-255), 255=max. speed.
+=============== ======= =======================================================
diff --git a/Documentation/hwmon/tps546d24.rst b/Documentation/hwmon/tps546d24.rst
new file mode 100644
index 000000000000..97adb8a30fc0
--- /dev/null
+++ b/Documentation/hwmon/tps546d24.rst
@@ -0,0 +1,35 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+Kernel driver tps546d24
+=======================
+
+Supported chips:
+
+  * TI TPS546D24
+
+    Prefix: 'tps546d24'
+
+    Addresses scanned: -
+
+    Datasheet: https://www.ti.com/lit/gpn/tps546d24
+
+Author: Duke Du <dukedu83@gmail.com>
+
+
+Description
+-----------
+
+The TPS546D24A is a highly integrated, non-isolated DC/DC converter capable
+of high frequency operation and 40-A current output from a 7-mm x 5-mm
+package.
+
+Two, three, and four TPS546D24A devices can be interconnected
+to provide up to 160 A on a single output. The device has an option to
+overdrive the internal 5-V LDO with an external 5-V supply via the VDD5
+pin to improve efficiency and reduce power dissipation of the converter.
+
+
+Platform data support
+---------------------
+
+The driver supports standard PMBus driver platform data.
diff --git a/Documentation/i2c/slave-interface.rst b/Documentation/i2c/slave-interface.rst
index 58fb143baee4..3f0d320bc80a 100644
--- a/Documentation/i2c/slave-interface.rst
+++ b/Documentation/i2c/slave-interface.rst
@@ -72,12 +72,15 @@ Event types:
 
   'val': unused
 
-  'ret': always 0
+  'ret': 0 if the backend is ready, otherwise some errno
 
 Another I2C master wants to write data to us. This event should be sent once
 our own address and the write bit was detected. The data did not arrive yet, so
-there is nothing to process or return. Wakeup or initialization probably needs
-to be done, though.
+there is nothing to process or return. After returning, the bus driver must
+always ack the address phase. If 'ret' is zero, backend initialization or
+wakeup is done and further data may be received. If 'ret' is an errno, the bus
+driver should nack all incoming bytes until the next stop condition to enforce
+a retry of the transmission.
 
 * I2C_SLAVE_READ_REQUESTED (mandatory)
 
diff --git a/Documentation/i2c/writing-clients.rst b/Documentation/i2c/writing-clients.rst
index 47f7cbf4ed1a..b7d3ae7458f8 100644
--- a/Documentation/i2c/writing-clients.rst
+++ b/Documentation/i2c/writing-clients.rst
@@ -156,7 +156,7 @@ those devices, and a remove() method to unbind.
 ::
 
 	static int foo_probe(struct i2c_client *client);
-	static int foo_remove(struct i2c_client *client);
+	static void foo_remove(struct i2c_client *client);
 
 Remember that the i2c_driver does not create those client handles.  The
 handle may be used during foo_probe().  If foo_probe() reports success
diff --git a/Documentation/index.rst b/Documentation/index.rst
index 4737c18c97ff..bf6aa681c960 100644
--- a/Documentation/index.rst
+++ b/Documentation/index.rst
@@ -1,11 +1,5 @@
 .. SPDX-License-Identifier: GPL-2.0
 
-
-.. The Linux Kernel documentation master file, created by
-   sphinx-quickstart on Fri Feb 12 13:51:46 2016.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
 .. _linux_doc:
 
 The Linux Kernel documentation
@@ -18,133 +12,85 @@ documents into a coherent whole.  Please note that improvements to the
 documentation are welcome; join the linux-doc list at vger.kernel.org if
 you want to help out.
 
-Licensing documentation
------------------------
+Working with the development community
+--------------------------------------
 
-The following describes the license of the Linux kernel source code
-(GPLv2), how to properly mark the license of individual files in the source
-tree, as well as links to the full license text.
-
-* :ref:`kernel_licensing`
-
-User-oriented documentation
----------------------------
-
-The following manuals are written for *users* of the kernel — those who are
-trying to get it to work optimally on a given system.
+The essential guides for interacting with the kernel's development
+community and getting your work upstream.
 
 .. toctree::
-   :maxdepth: 2
-
-   admin-guide/index
-   kbuild/index
-
-Firmware-related documentation
-------------------------------
-The following holds information on the kernel's expectations regarding the
-platform firmwares.
+   :maxdepth: 1
 
-.. toctree::
-   :maxdepth: 2
+   process/development-process
+   process/submitting-patches
+   Code of conduct <process/code-of-conduct>
+   maintainer/index
+   All development-process docs <process/index>
 
-   firmware-guide/index
-   devicetree/index
 
-Application-developer documentation
------------------------------------
+Internal API manuals
+--------------------
 
-The user-space API manual gathers together documents describing aspects of
-the kernel interface as seen by application developers.
+Manuals for use by developers working to interface with the rest of the
+kernel.
 
 .. toctree::
-   :maxdepth: 2
-
-   userspace-api/index
+   :maxdepth: 1
 
+   core-api/index
+   driver-api/index
+   subsystem-apis
+   Locking in the kernel <locking/index>
 
-Introduction to kernel development
-----------------------------------
+Development tools and processes
+-------------------------------
 
-These manuals contain overall information about how to develop the kernel.
-The kernel community is quite large, with thousands of developers
-contributing over the course of a year.  As with any large community,
-knowing how things are done will make the process of getting your changes
-merged much easier.
+Various other manuals with useful information for all kernel developers.
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
-   process/index
-   dev-tools/index
+   process/license-rules
    doc-guide/index
+   dev-tools/index
+   dev-tools/testing-overview
    kernel-hacking/index
    trace/index
-   maintainer/index
    fault-injection/index
    livepatch/index
+   rust/index
 
 
-Kernel API documentation
-------------------------
+User-oriented documentation
+---------------------------
 
-These books get into the details of how specific kernel subsystems work
-from the point of view of a kernel developer.  Much of the information here
-is taken directly from the kernel source, with supplemental material added
-as needed (or at least as we managed to add it — probably *not* all that is
-needed).
+The following manuals are written for *users* of the kernel — those who are
+trying to get it to work optimally on a given system and application
+developers seeking information on the kernel's user-space APIs.
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
-   driver-api/index
-   core-api/index
-   locking/index
-   accounting/index
-   block/index
-   cdrom/index
-   cpu-freq/index
-   fb/index
-   fpga/index
-   hid/index
-   i2c/index
-   iio/index
-   isdn/index
-   infiniband/index
-   leds/index
-   netlabel/index
-   networking/index
-   pcmcia/index
-   power/index
-   target/index
-   timers/index
-   spi/index
-   w1/index
-   watchdog/index
-   virt/index
-   input/index
-   hwmon/index
-   gpu/index
-   security/index
-   sound/index
-   crypto/index
-   filesystems/index
-   mm/index
-   bpf/index
-   usb/index
-   PCI/index
-   scsi/index
-   misc-devices/index
-   scheduler/index
-   mhi/index
-   peci/index
-
-Architecture-agnostic documentation
------------------------------------
+   admin-guide/index
+   The kernel build system <kbuild/index>
+   admin-guide/reporting-issues.rst
+   User-space tools <tools/index>
+   userspace-api/index
+
+See also: the `Linux man pages <https://www.kernel.org/doc/man-pages/>`_,
+which are kept separately from the kernel's own documentation.
+
+Firmware-related documentation
+------------------------------
+The following holds information on the kernel's expectations regarding the
+platform firmwares.
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
+
+   firmware-guide/index
+   devicetree/index
 
-   asm-annotations
 
 Architecture-specific documentation
 -----------------------------------
@@ -163,9 +109,8 @@ of the documentation body, or may require some adjustments and/or conversion
 to ReStructured Text format, or are simply too old.
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
-   tools/index
    staging/index
 
 
diff --git a/Documentation/kbuild/gcc-plugins.rst b/Documentation/kbuild/gcc-plugins.rst
index 0ba76719f1b9..c578c6ba3eb6 100644
--- a/Documentation/kbuild/gcc-plugins.rst
+++ b/Documentation/kbuild/gcc-plugins.rst
@@ -90,7 +90,11 @@ e.g., on Ubuntu for gcc-10::
 
 Or on Fedora::
 
-	dnf install gcc-plugin-devel
+	dnf install gcc-plugin-devel libmpc-devel
+
+Or on Fedora when using cross-compilers that include plugins::
+
+	dnf install libmpc-devel
 
 Enable the GCC plugin infrastructure and some plugin(s) you want to use
 in the kernel config::
@@ -99,6 +103,19 @@ in the kernel config::
 	CONFIG_GCC_PLUGIN_LATENT_ENTROPY=y
 	...
 
+Run gcc (native or cross-compiler) to ensure plugin headers are detected::
+
+	gcc -print-file-name=plugin
+	CROSS_COMPILE=arm-linux-gnu- ${CROSS_COMPILE}gcc -print-file-name=plugin
+
+The word "plugin" means they are not detected::
+
+	plugin
+
+A full path means they are detected::
+
+       /usr/lib/gcc/x86_64-redhat-linux/12/plugin
+
 To compile the minimum tool set including the plugin(s)::
 
 	make scripts
diff --git a/Documentation/kbuild/kbuild.rst b/Documentation/kbuild/kbuild.rst
index ef19b9c13523..08f575e6236c 100644
--- a/Documentation/kbuild/kbuild.rst
+++ b/Documentation/kbuild/kbuild.rst
@@ -48,6 +48,10 @@ KCFLAGS
 -------
 Additional options to the C compiler (for built-in and modules).
 
+KRUSTFLAGS
+----------
+Additional options to the Rust compiler (for built-in and modules).
+
 CFLAGS_KERNEL
 -------------
 Additional options for $(CC) when used to compile
@@ -57,6 +61,15 @@ CFLAGS_MODULE
 -------------
 Additional module specific options to use for $(CC).
 
+RUSTFLAGS_KERNEL
+----------------
+Additional options for $(RUSTC) when used to compile
+code that is compiled as built-in.
+
+RUSTFLAGS_MODULE
+----------------
+Additional module specific options to use for $(RUSTC).
+
 LDFLAGS_MODULE
 --------------
 Additional options used for $(LD) when linking modules.
@@ -69,6 +82,10 @@ HOSTCXXFLAGS
 ------------
 Additional flags to be passed to $(HOSTCXX) when building host programs.
 
+HOSTRUSTFLAGS
+-------------
+Additional flags to be passed to $(HOSTRUSTC) when building host programs.
+
 HOSTLDFLAGS
 -----------
 Additional flags to be passed when linking host programs.
diff --git a/Documentation/kbuild/makefiles.rst b/Documentation/kbuild/makefiles.rst
index 11a296e52d68..5ea1e72d89c8 100644
--- a/Documentation/kbuild/makefiles.rst
+++ b/Documentation/kbuild/makefiles.rst
@@ -29,8 +29,9 @@ This document describes the Linux kernel Makefiles.
 	   --- 4.1 Simple Host Program
 	   --- 4.2 Composite Host Programs
 	   --- 4.3 Using C++ for host programs
-	   --- 4.4 Controlling compiler options for host programs
-	   --- 4.5 When host programs are actually built
+	   --- 4.4 Using Rust for host programs
+	   --- 4.5 Controlling compiler options for host programs
+	   --- 4.6 When host programs are actually built
 
 	=== 5 Userspace Program support
 	   --- 5.1 Simple Userspace Program
@@ -835,7 +836,24 @@ Both possibilities are described in the following.
 		qconf-cxxobjs := qconf.o
 		qconf-objs    := check.o
 
-4.4 Controlling compiler options for host programs
+4.4 Using Rust for host programs
+--------------------------------
+
+	Kbuild offers support for host programs written in Rust. However,
+	since a Rust toolchain is not mandatory for kernel compilation,
+	it may only be used in scenarios where Rust is required to be
+	available (e.g. when  ``CONFIG_RUST`` is enabled).
+
+	Example::
+
+		hostprogs     := target
+		target-rust   := y
+
+	Kbuild will compile ``target`` using ``target.rs`` as the crate root,
+	located in the same directory as the ``Makefile``. The crate may
+	consist of several source files (see ``samples/rust/hostprogs``).
+
+4.5 Controlling compiler options for host programs
 --------------------------------------------------
 
 	When compiling host programs, it is possible to set specific flags.
@@ -867,7 +885,7 @@ Both possibilities are described in the following.
 	When linking qconf, it will be passed the extra option
 	"-L$(QTDIR)/lib".
 
-4.5 When host programs are actually built
+4.6 When host programs are actually built
 -----------------------------------------
 
 	Kbuild will only build host-programs when they are referenced
@@ -1181,6 +1199,17 @@ When kbuild executes, the following steps are followed (roughly):
 	The first example utilises the trick that a config option expands
 	to 'y' when selected.
 
+    KBUILD_RUSTFLAGS
+	$(RUSTC) compiler flags
+
+	Default value - see top level Makefile
+	Append or modify as required per architecture.
+
+	Often, the KBUILD_RUSTFLAGS variable depends on the configuration.
+
+	Note that target specification file generation (for ``--target``)
+	is handled in ``scripts/generate_rust_target.rs``.
+
     KBUILD_AFLAGS_KERNEL
 	Assembler options specific for built-in
 
@@ -1208,6 +1237,19 @@ When kbuild executes, the following steps are followed (roughly):
 	are used for $(CC).
 	From commandline CFLAGS_MODULE shall be used (see kbuild.rst).
 
+    KBUILD_RUSTFLAGS_KERNEL
+	$(RUSTC) options specific for built-in
+
+	$(KBUILD_RUSTFLAGS_KERNEL) contains extra Rust compiler flags used to
+	compile resident kernel code.
+
+    KBUILD_RUSTFLAGS_MODULE
+	Options for $(RUSTC) when building modules
+
+	$(KBUILD_RUSTFLAGS_MODULE) is used to add arch-specific options that
+	are used for $(RUSTC).
+	From commandline RUSTFLAGS_MODULE shall be used (see kbuild.rst).
+
     KBUILD_LDFLAGS_MODULE
 	Options for $(LD) when linking modules
 
diff --git a/Documentation/locking/seqlock.rst b/Documentation/locking/seqlock.rst
index 64405e5da63e..bfda1a5fecad 100644
--- a/Documentation/locking/seqlock.rst
+++ b/Documentation/locking/seqlock.rst
@@ -39,7 +39,7 @@ as the writer can invalidate a pointer that the reader is following.
 Sequence counters (``seqcount_t``)
 ==================================
 
-This is the the raw counting mechanism, which does not protect against
+This is the raw counting mechanism, which does not protect against
 multiple writers.  Write side critical sections must thus be serialized
 by an external lock.
 
diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt
index 832b5d36e279..06f80e3785c5 100644
--- a/Documentation/memory-barriers.txt
+++ b/Documentation/memory-barriers.txt
@@ -52,7 +52,7 @@ CONTENTS
 
      - Varieties of memory barrier.
      - What may not be assumed about memory barriers?
-     - Data dependency barriers (historical).
+     - Address-dependency barriers (historical).
      - Control dependencies.
      - SMP barrier pairing.
      - Examples of memory barrier sequences.
@@ -187,9 +187,9 @@ As a further example, consider this sequence of events:
 	B = 4;		Q = P;
 	P = &B;		D = *Q;
 
-There is an obvious data dependency here, as the value loaded into D depends on
-the address retrieved from P by CPU 2.  At the end of the sequence, any of the
-following results are possible:
+There is an obvious address dependency here, as the value loaded into D depends
+on the address retrieved from P by CPU 2.  At the end of the sequence, any of
+the following results are possible:
 
 	(Q == &A) and (D == 1)
 	(Q == &B) and (D == 2)
@@ -391,58 +391,62 @@ Memory barriers come in four basic varieties:
      memory system as time progresses.  All stores _before_ a write barrier
      will occur _before_ all the stores after the write barrier.
 
-     [!] Note that write barriers should normally be paired with read or data
-     dependency barriers; see the "SMP barrier pairing" subsection.
+     [!] Note that write barriers should normally be paired with read or
+     address-dependency barriers; see the "SMP barrier pairing" subsection.
 
 
- (2) Data dependency barriers.
+ (2) Address-dependency barriers (historical).
 
-     A data dependency barrier is a weaker form of read barrier.  In the case
-     where two loads are performed such that the second depends on the result
-     of the first (eg: the first load retrieves the address to which the second
-     load will be directed), a data dependency barrier would be required to
-     make sure that the target of the second load is updated after the address
-     obtained by the first load is accessed.
+     An address-dependency barrier is a weaker form of read barrier.  In the
+     case where two loads are performed such that the second depends on the
+     result of the first (eg: the first load retrieves the address to which
+     the second load will be directed), an address-dependency barrier would
+     be required to make sure that the target of the second load is updated
+     after the address obtained by the first load is accessed.
 
-     A data dependency barrier is a partial ordering on interdependent loads
-     only; it is not required to have any effect on stores, independent loads
-     or overlapping loads.
+     An address-dependency barrier is a partial ordering on interdependent
+     loads only; it is not required to have any effect on stores, independent
+     loads or overlapping loads.
 
      As mentioned in (1), the other CPUs in the system can be viewed as
      committing sequences of stores to the memory system that the CPU being
-     considered can then perceive.  A data dependency barrier issued by the CPU
-     under consideration guarantees that for any load preceding it, if that
-     load touches one of a sequence of stores from another CPU, then by the
-     time the barrier completes, the effects of all the stores prior to that
-     touched by the load will be perceptible to any loads issued after the data
-     dependency barrier.
+     considered can then perceive.  An address-dependency barrier issued by
+     the CPU under consideration guarantees that for any load preceding it,
+     if that load touches one of a sequence of stores from another CPU, then
+     by the time the barrier completes, the effects of all the stores prior to
+     that touched by the load will be perceptible to any loads issued after
+     the address-dependency barrier.
 
      See the "Examples of memory barrier sequences" subsection for diagrams
      showing the ordering constraints.
 
-     [!] Note that the first load really has to have a _data_ dependency and
+     [!] Note that the first load really has to have an _address_ dependency and
      not a control dependency.  If the address for the second load is dependent
      on the first load, but the dependency is through a conditional rather than
      actually loading the address itself, then it's a _control_ dependency and
      a full read barrier or better is required.  See the "Control dependencies"
      subsection for more information.
 
-     [!] Note that data dependency barriers should normally be paired with
+     [!] Note that address-dependency barriers should normally be paired with
      write barriers; see the "SMP barrier pairing" subsection.
 
+     [!] Kernel release v5.9 removed kernel APIs for explicit address-
+     dependency barriers.  Nowadays, APIs for marking loads from shared
+     variables such as READ_ONCE() and rcu_dereference() provide implicit
+     address-dependency barriers.
 
  (3) Read (or load) memory barriers.
 
-     A read barrier is a data dependency barrier plus a guarantee that all the
-     LOAD operations specified before the barrier will appear to happen before
-     all the LOAD operations specified after the barrier with respect to the
-     other components of the system.
+     A read barrier is an address-dependency barrier plus a guarantee that all
+     the LOAD operations specified before the barrier will appear to happen
+     before all the LOAD operations specified after the barrier with respect to
+     the other components of the system.
 
      A read barrier is a partial ordering on loads only; it is not required to
      have any effect on stores.
 
-     Read memory barriers imply data dependency barriers, and so can substitute
-     for them.
+     Read memory barriers imply address-dependency barriers, and so can
+     substitute for them.
 
      [!] Note that read barriers should normally be paired with write barriers;
      see the "SMP barrier pairing" subsection.
@@ -550,17 +554,21 @@ There are certain things that the Linux kernel memory barriers do not guarantee:
 	    Documentation/core-api/dma-api.rst
 
 
-DATA DEPENDENCY BARRIERS (HISTORICAL)
--------------------------------------
+ADDRESS-DEPENDENCY BARRIERS (HISTORICAL)
+----------------------------------------
 
 As of v4.15 of the Linux kernel, an smp_mb() was added to READ_ONCE() for
 DEC Alpha, which means that about the only people who need to pay attention
 to this section are those working on DEC Alpha architecture-specific code
 and those working on READ_ONCE() itself.  For those who need it, and for
 those who are interested in the history, here is the story of
-data-dependency barriers.
+address-dependency barriers.
+
+[!] While address dependencies are observed in both load-to-load and
+load-to-store relations, address-dependency barriers are not necessary
+for load-to-store situations.
 
-The usage requirements of data dependency barriers are a little subtle, and
+The requirement of address-dependency barriers is a little subtle, and
 it's not always obvious that they're needed.  To illustrate, consider the
 following sequence of events:
 
@@ -570,11 +578,14 @@ following sequence of events:
 	B = 4;
 	<write barrier>
 	WRITE_ONCE(P, &B);
-			      Q = READ_ONCE(P);
+			      Q = READ_ONCE_OLD(P);
 			      D = *Q;
 
-There's a clear data dependency here, and it would seem that by the end of the
-sequence, Q must be either &A or &B, and that:
+[!] READ_ONCE_OLD() corresponds to READ_ONCE() of pre-4.15 kernel, which
+doesn't imply an address-dependency barrier.
+
+There's a clear address dependency here, and it would seem that by the end of
+the sequence, Q must be either &A or &B, and that:
 
 	(Q == &A) implies (D == 1)
 	(Q == &B) implies (D == 4)
@@ -588,8 +599,8 @@ While this may seem like a failure of coherency or causality maintenance, it
 isn't, and this behaviour can be observed on certain real CPUs (such as the DEC
 Alpha).
 
-To deal with this, a data dependency barrier or better must be inserted
-between the address load and the data load:
+To deal with this, READ_ONCE() provides an implicit address-dependency barrier
+since kernel release v4.15:
 
 	CPU 1		      CPU 2
 	===============	      ===============
@@ -598,7 +609,7 @@ between the address load and the data load:
 	<write barrier>
 	WRITE_ONCE(P, &B);
 			      Q = READ_ONCE(P);
-			      <data dependency barrier>
+			      <implicit address-dependency barrier>
 			      D = *Q;
 
 This enforces the occurrence of one of the two implications, and prevents the
@@ -615,13 +626,13 @@ odd-numbered bank is idle, one can see the new value of the pointer P (&B),
 but the old value of the variable B (2).
 
 
-A data-dependency barrier is not required to order dependent writes
-because the CPUs that the Linux kernel supports don't do writes
-until they are certain (1) that the write will actually happen, (2)
-of the location of the write, and (3) of the value to be written.
+An address-dependency barrier is not required to order dependent writes
+because the CPUs that the Linux kernel supports don't do writes until they
+are certain (1) that the write will actually happen, (2) of the location of
+the write, and (3) of the value to be written.
 But please carefully read the "CONTROL DEPENDENCIES" section and the
-Documentation/RCU/rcu_dereference.rst file:  The compiler can and does
-break dependencies in a great many highly creative ways.
+Documentation/RCU/rcu_dereference.rst file:  The compiler can and does break
+dependencies in a great many highly creative ways.
 
 	CPU 1		      CPU 2
 	===============	      ===============
@@ -629,12 +640,12 @@ break dependencies in a great many highly creative ways.
 	B = 4;
 	<write barrier>
 	WRITE_ONCE(P, &B);
-			      Q = READ_ONCE(P);
+			      Q = READ_ONCE_OLD(P);
 			      WRITE_ONCE(*Q, 5);
 
-Therefore, no data-dependency barrier is required to order the read into
+Therefore, no address-dependency barrier is required to order the read into
 Q with the store into *Q.  In other words, this outcome is prohibited,
-even without a data-dependency barrier:
+even without an implicit address-dependency barrier of modern READ_ONCE():
 
 	(Q == &B) && (B == 4)
 
@@ -645,12 +656,12 @@ can be used to record rare error conditions and the like, and the CPUs'
 naturally occurring ordering prevents such records from being lost.
 
 
-Note well that the ordering provided by a data dependency is local to
+Note well that the ordering provided by an address dependency is local to
 the CPU containing it.  See the section on "Multicopy atomicity" for
 more information.
 
 
-The data dependency barrier is very important to the RCU system,
+The address-dependency barrier is very important to the RCU system,
 for example.  See rcu_assign_pointer() and rcu_dereference() in
 include/linux/rcupdate.h.  This permits the current target of an RCU'd
 pointer to be replaced with a new modified target, without the replacement
@@ -667,20 +678,21 @@ not understand them.  The purpose of this section is to help you prevent
 the compiler's ignorance from breaking your code.
 
 A load-load control dependency requires a full read memory barrier, not
-simply a data dependency barrier to make it work correctly.  Consider the
-following bit of code:
+simply an (implicit) address-dependency barrier to make it work correctly.
+Consider the following bit of code:
 
 	q = READ_ONCE(a);
+	<implicit address-dependency barrier>
 	if (q) {
-		<data dependency barrier>  /* BUG: No data dependency!!! */
+		/* BUG: No address dependency!!! */
 		p = READ_ONCE(b);
 	}
 
-This will not have the desired effect because there is no actual data
+This will not have the desired effect because there is no actual address
 dependency, but rather a control dependency that the CPU may short-circuit
 by attempting to predict the outcome in advance, so that other CPUs see
-the load from b as having happened before the load from a.  In such a
-case what's actually required is:
+the load from b as having happened before the load from a.  In such a case
+what's actually required is:
 
 	q = READ_ONCE(a);
 	if (q) {
@@ -927,9 +939,9 @@ General barriers pair with each other, though they also pair with most
 other types of barriers, albeit without multicopy atomicity.  An acquire
 barrier pairs with a release barrier, but both may also pair with other
 barriers, including of course general barriers.  A write barrier pairs
-with a data dependency barrier, a control dependency, an acquire barrier,
+with an address-dependency barrier, a control dependency, an acquire barrier,
 a release barrier, a read barrier, or a general barrier.  Similarly a
-read barrier, control dependency, or a data dependency barrier pairs
+read barrier, control dependency, or an address-dependency barrier pairs
 with a write barrier, an acquire barrier, a release barrier, or a
 general barrier:
 
@@ -948,7 +960,7 @@ Or:
 	a = 1;
 	<write barrier>
 	WRITE_ONCE(b, &a);    x = READ_ONCE(b);
-			      <data dependency barrier>
+			      <implicit address-dependency barrier>
 			      y = *x;
 
 Or even:
@@ -968,8 +980,8 @@ Basically, the read barrier always has to be there, even though it can be of
 the "weaker" type.
 
 [!] Note that the stores before the write barrier would normally be expected to
-match the loads after the read barrier or the data dependency barrier, and vice
-versa:
+match the loads after the read barrier or the address-dependency barrier, and
+vice versa:
 
 	CPU 1                               CPU 2
 	===================                 ===================
@@ -1021,8 +1033,8 @@ STORE B, STORE C } all occurring before the unordered set of { STORE D, STORE E
 	                   V
 
 
-Secondly, data dependency barriers act as partial orderings on data-dependent
-loads.  Consider the following sequence of events:
+Secondly, address-dependency barriers act as partial orderings on address-
+dependent loads.  Consider the following sequence of events:
 
 	CPU 1			CPU 2
 	=======================	=======================
@@ -1067,8 +1079,8 @@ effectively random order, despite the write barrier issued by CPU 1:
 In the above example, CPU 2 perceives that B is 7, despite the load of *C
 (which would be B) coming after the LOAD of C.
 
-If, however, a data dependency barrier were to be placed between the load of C
-and the load of *C (ie: B) on CPU 2:
+If, however, an address-dependency barrier were to be placed between the load
+of C and the load of *C (ie: B) on CPU 2:
 
 	CPU 1			CPU 2
 	=======================	=======================
@@ -1078,7 +1090,7 @@ and the load of *C (ie: B) on CPU 2:
 	<write barrier>
 	STORE C = &B		LOAD X
 	STORE D = 4		LOAD C (gets &B)
-				<data dependency barrier>
+				<address-dependency barrier>
 				LOAD *C (reads B)
 
 then the following will occur:
@@ -1101,7 +1113,7 @@ then the following will occur:
 	                               |        +-------+       |       |
 	                               |        | X->9  |------>|       |
 	                               |        +-------+       |       |
-	  Makes sure all effects --->   \   ddddddddddddddddd   |       |
+	  Makes sure all effects --->   \   aaaaaaaaaaaaaaaaa   |       |
 	  prior to the store of C        \      +-------+       |       |
 	  are perceptible to              ----->| B->2  |------>|       |
 	  subsequent loads                      +-------+       |       |
@@ -1292,7 +1304,7 @@ Which might appear as this:
 	LOAD with immediate effect              :       :       +-------+
 
 
-Placing a read barrier or a data dependency barrier just before the second
+Placing a read barrier or an address-dependency barrier just before the second
 load:
 
 	CPU 1			CPU 2
@@ -1816,20 +1828,20 @@ which may then reorder things however it wishes.
 CPU MEMORY BARRIERS
 -------------------
 
-The Linux kernel has eight basic CPU memory barriers:
+The Linux kernel has seven basic CPU memory barriers:
 
-	TYPE		MANDATORY		SMP CONDITIONAL
-	===============	=======================	===========================
-	GENERAL		mb()			smp_mb()
-	WRITE		wmb()			smp_wmb()
-	READ		rmb()			smp_rmb()
-	DATA DEPENDENCY				READ_ONCE()
+	TYPE			MANDATORY	SMP CONDITIONAL
+	=======================	===============	===============
+	GENERAL			mb()		smp_mb()
+	WRITE			wmb()		smp_wmb()
+	READ			rmb()		smp_rmb()
+	ADDRESS DEPENDENCY			READ_ONCE()
 
 
-All memory barriers except the data dependency barriers imply a compiler
-barrier.  Data dependencies do not impose any additional compiler ordering.
+All memory barriers except the address-dependency barriers imply a compiler
+barrier.  Address dependencies do not impose any additional compiler ordering.
 
-Aside: In the case of data dependencies, the compiler would be expected
+Aside: In the case of address dependencies, the compiler would be expected
 to issue the loads in the correct order (eg. `a[b]` would have to load
 the value of b before loading a[b]), however there is no guarantee in
 the C specification that the compiler may not speculate the value of b
@@ -2749,7 +2761,8 @@ is discarded from the CPU's cache and reloaded.  To deal with this, the
 appropriate part of the kernel must invalidate the overlapping bits of the
 cache on each CPU.
 
-See Documentation/core-api/cachetlb.rst for more information on cache management.
+See Documentation/core-api/cachetlb.rst for more information on cache
+management.
 
 
 CACHE COHERENCY VS MMIO
@@ -2889,8 +2902,8 @@ AND THEN THERE'S THE ALPHA
 The DEC Alpha CPU is one of the most relaxed CPUs there is.  Not only that,
 some versions of the Alpha CPU have a split data cache, permitting them to have
 two semantically-related cache lines updated at separate times.  This is where
-the data dependency barrier really becomes necessary as this synchronises both
-caches with the memory coherence system, thus making it seem like pointer
+the address-dependency barrier really becomes necessary as this synchronises
+both caches with the memory coherence system, thus making it seem like pointer
 changes vs new data occur in the right order.
 
 The Alpha defines the Linux kernel's memory model, although as of v4.15
diff --git a/Documentation/mm/unevictable-lru.rst b/Documentation/mm/unevictable-lru.rst
index b280367d6a44..4a0e158aa9ce 100644
--- a/Documentation/mm/unevictable-lru.rst
+++ b/Documentation/mm/unevictable-lru.rst
@@ -197,7 +197,7 @@ unevictable list for the memory cgroup and node being scanned.
 There may be situations where a page is mapped into a VM_LOCKED VMA, but the
 page is not marked as PG_mlocked.  Such pages will make it all the way to
 shrink_active_list() or shrink_page_list() where they will be detected when
-vmscan walks the reverse map in page_referenced() or try_to_unmap().  The page
+vmscan walks the reverse map in folio_referenced() or try_to_unmap().  The page
 is culled to the unevictable list when it is released by the shrinker.
 
 To "cull" an unevictable page, vmscan simply puts the page back on the LRU list
@@ -267,7 +267,7 @@ the LRU.  Such pages can be "noticed" by memory management in several places:
  (4) in the fault path and when a VM_LOCKED stack segment is expanded; or
 
  (5) as mentioned above, in vmscan:shrink_page_list() when attempting to
-     reclaim a page in a VM_LOCKED VMA by page_referenced() or try_to_unmap().
+     reclaim a page in a VM_LOCKED VMA by folio_referenced() or try_to_unmap().
 
 mlocked pages become unlocked and rescued from the unevictable list when:
 
@@ -547,7 +547,7 @@ vmscan's shrink_inactive_list() and shrink_page_list() also divert obviously
 unevictable pages found on the inactive lists to the appropriate memory cgroup
 and node unevictable list.
 
-rmap's page_referenced_one(), called via vmscan's shrink_active_list() or
+rmap's folio_referenced_one(), called via vmscan's shrink_active_list() or
 shrink_page_list(), and rmap's try_to_unmap_one() called via shrink_page_list(),
 check for (3) pages still mapped into VM_LOCKED VMAs, and call mlock_vma_page()
 to correct them.  Such pages are culled to the unevictable list when released
diff --git a/Documentation/networking/bonding.rst b/Documentation/networking/bonding.rst
index 7823a069a903..96cd7a26f3d9 100644
--- a/Documentation/networking/bonding.rst
+++ b/Documentation/networking/bonding.rst
@@ -846,7 +846,7 @@ primary_reselect
 tlb_dynamic_lb
 
 	Specifies if dynamic shuffling of flows is enabled in tlb
-	mode. The value has no effect on any other modes.
+	or alb mode. The value has no effect on any other modes.
 
 	The default behavior of tlb mode is to shuffle active flows across
 	slaves based on the load in that interval. This gives nice lb
diff --git a/Documentation/networking/decnet.rst b/Documentation/networking/decnet.rst
deleted file mode 100644
index b8bc11ff8370..000000000000
--- a/Documentation/networking/decnet.rst
+++ /dev/null
@@ -1,243 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-=========================================
-Linux DECnet Networking Layer Information
-=========================================
-
-1. Other documentation....
-==========================
-
-   - Project Home Pages
-     - http://www.chygwyn.com/				   - Kernel info
-     - http://linux-decnet.sourceforge.net/                - Userland tools
-     - http://www.sourceforge.net/projects/linux-decnet/   - Status page
-
-2. Configuring the kernel
-=========================
-
-Be sure to turn on the following options:
-
-    - CONFIG_DECNET (obviously)
-    - CONFIG_PROC_FS (to see what's going on)
-    - CONFIG_SYSCTL (for easy configuration)
-
-if you want to try out router support (not properly debugged yet)
-you'll need the following options as well...
-
-    - CONFIG_DECNET_ROUTER (to be able to add/delete routes)
-    - CONFIG_NETFILTER (will be required for the DECnet routing daemon)
-
-Don't turn on SIOCGIFCONF support for DECnet unless you are really sure
-that you need it, in general you won't and it can cause ifconfig to
-malfunction.
-
-Run time configuration has changed slightly from the 2.4 system. If you
-want to configure an endnode, then the simplified procedure is as follows:
-
- - Set the MAC address on your ethernet card before starting _any_ other
-   network protocols.
-
-As soon as your network card is brought into the UP state, DECnet should
-start working. If you need something more complicated or are unsure how
-to set the MAC address, see the next section. Also all configurations which
-worked with 2.4 will work under 2.5 with no change.
-
-3. Command line options
-=======================
-
-You can set a DECnet address on the kernel command line for compatibility
-with the 2.4 configuration procedure, but in general it's not needed any more.
-If you do st a DECnet address on the command line, it has only one purpose
-which is that its added to the addresses on the loopback device.
-
-With 2.4 kernels, DECnet would only recognise addresses as local if they
-were added to the loopback device. In 2.5, any local interface address
-can be used to loop back to the local machine. Of course this does not
-prevent you adding further addresses to the loopback device if you
-want to.
-
-N.B. Since the address list of an interface determines the addresses for
-which "hello" messages are sent, if you don't set an address on the loopback
-interface then you won't see any entries in /proc/net/neigh for the local
-host until such time as you start a connection. This doesn't affect the
-operation of the local communications in any other way though.
-
-The kernel command line takes options looking like the following::
-
-    decnet.addr=1,2
-
-the two numbers are the node address 1,2 = 1.2 For 2.2.xx kernels
-and early 2.3.xx kernels, you must use a comma when specifying the
-DECnet address like this. For more recent 2.3.xx kernels, you may
-use almost any character except space, although a `.` would be the most
-obvious choice :-)
-
-There used to be a third number specifying the node type. This option
-has gone away in favour of a per interface node type. This is now set
-using /proc/sys/net/decnet/conf/<dev>/forwarding. This file can be
-set with a single digit, 0=EndNode, 1=L1 Router and  2=L2 Router.
-
-There are also equivalent options for modules. The node address can
-also be set through the /proc/sys/net/decnet/ files, as can other system
-parameters.
-
-Currently the only supported devices are ethernet and ip_gre. The
-ethernet address of your ethernet card has to be set according to the DECnet
-address of the node in order for it to be autoconfigured (and then appear in
-/proc/net/decnet_dev). There is a utility available at the above
-FTP sites called dn2ethaddr which can compute the correct ethernet
-address to use. The address can be set by ifconfig either before or
-at the time the device is brought up. If you are using RedHat you can
-add the line::
-
-    MACADDR=AA:00:04:00:03:04
-
-or something similar, to /etc/sysconfig/network-scripts/ifcfg-eth0 or
-wherever your network card's configuration lives. Setting the MAC address
-of your ethernet card to an address starting with "hi-ord" will cause a
-DECnet address which matches to be added to the interface (which you can
-verify with iproute2).
-
-The default device for routing can be set through the /proc filesystem
-by setting /proc/sys/net/decnet/default_device to the
-device you want DECnet to route packets out of when no specific route
-is available. Usually this will be eth0, for example::
-
-    echo -n "eth0" >/proc/sys/net/decnet/default_device
-
-If you don't set the default device, then it will default to the first
-ethernet card which has been autoconfigured as described above. You can
-confirm that by looking in the default_device file of course.
-
-There is a list of what the other files under /proc/sys/net/decnet/ do
-on the kernel patch web site (shown above).
-
-4. Run time kernel configuration
-================================
-
-
-This is either done through the sysctl/proc interface (see the kernel web
-pages for details on what the various options do) or through the iproute2
-package in the same way as IPv4/6 configuration is performed.
-
-Documentation for iproute2 is included with the package, although there is
-as yet no specific section on DECnet, most of the features apply to both
-IP and DECnet, albeit with DECnet addresses instead of IP addresses and
-a reduced functionality.
-
-If you want to configure a DECnet router you'll need the iproute2 package
-since its the _only_ way to add and delete routes currently. Eventually
-there will be a routing daemon to send and receive routing messages for
-each interface and update the kernel routing tables accordingly. The
-routing daemon will use netfilter to listen to routing packets, and
-rtnetlink to update the kernels routing tables.
-
-The DECnet raw socket layer has been removed since it was there purely
-for use by the routing daemon which will now use netfilter (a much cleaner
-and more generic solution) instead.
-
-5. How can I tell if its working?
-=================================
-
-Here is a quick guide of what to look for in order to know if your DECnet
-kernel subsystem is working.
-
-   - Is the node address set (see /proc/sys/net/decnet/node_address)
-   - Is the node of the correct type
-     (see /proc/sys/net/decnet/conf/<dev>/forwarding)
-   - Is the Ethernet MAC address of each Ethernet card set to match
-     the DECnet address. If in doubt use the dn2ethaddr utility available
-     at the ftp archive.
-   - If the previous two steps are satisfied, and the Ethernet card is up,
-     you should find that it is listed in /proc/net/decnet_dev and also
-     that it appears as a directory in /proc/sys/net/decnet/conf/. The
-     loopback device (lo) should also appear and is required to communicate
-     within a node.
-   - If you have any DECnet routers on your network, they should appear
-     in /proc/net/decnet_neigh, otherwise this file will only contain the
-     entry for the node itself (if it doesn't check to see if lo is up).
-   - If you want to send to any node which is not listed in the
-     /proc/net/decnet_neigh file, you'll need to set the default device
-     to point to an Ethernet card with connection to a router. This is
-     again done with the /proc/sys/net/decnet/default_device file.
-   - Try starting a simple server and client, like the dnping/dnmirror
-     over the loopback interface. With luck they should communicate.
-     For this step and those after, you'll need the DECnet library
-     which can be obtained from the above ftp sites as well as the
-     actual utilities themselves.
-   - If this seems to work, then try talking to a node on your local
-     network, and see if you can obtain the same results.
-   - At this point you are on your own... :-)
-
-6. How to send a bug report
-===========================
-
-If you've found a bug and want to report it, then there are several things
-you can do to help me work out exactly what it is that is wrong. Useful
-information (_most_ of which _is_ _essential_) includes:
-
- - What kernel version are you running ?
- - What version of the patch are you running ?
- - How far though the above set of tests can you get ?
- - What is in the /proc/decnet* files and /proc/sys/net/decnet/* files ?
- - Which services are you running ?
- - Which client caused the problem ?
- - How much data was being transferred ?
- - Was the network congested ?
- - How can the problem be reproduced ?
- - Can you use tcpdump to get a trace ? (N.B. Most (all?) versions of
-   tcpdump don't understand how to dump DECnet properly, so including
-   the hex listing of the packet contents is _essential_, usually the -x flag.
-   You may also need to increase the length grabbed with the -s flag. The
-   -e flag also provides very useful information (ethernet MAC addresses))
-
-7. MAC FAQ
-==========
-
-A quick FAQ on ethernet MAC addresses to explain how Linux and DECnet
-interact and how to get the best performance from your hardware.
-
-Ethernet cards are designed to normally only pass received network frames
-to a host computer when they are addressed to it, or to the broadcast address.
-
-Linux has an interface which allows the setting of extra addresses for
-an ethernet card to listen to. If the ethernet card supports it, the
-filtering operation will be done in hardware, if not the extra unwanted packets
-received will be discarded by the host computer. In the latter case,
-significant processor time and bus bandwidth can be used up on a busy
-network (see the NAPI documentation for a longer explanation of these
-effects).
-
-DECnet makes use of this interface to allow running DECnet on an ethernet
-card which has already been configured using TCP/IP (presumably using the
-built in MAC address of the card, as usual) and/or to allow multiple DECnet
-addresses on each physical interface. If you do this, be aware that if your
-ethernet card doesn't support perfect hashing in its MAC address filter
-then your computer will be doing more work than required. Some cards
-will simply set themselves into promiscuous mode in order to receive
-packets from the DECnet specified addresses. So if you have one of these
-cards its better to set the MAC address of the card as described above
-to gain the best efficiency. Better still is to use a card which supports
-NAPI as well.
-
-
-8. Mailing list
-===============
-
-If you are keen to get involved in development, or want to ask questions
-about configuration, or even just report bugs, then there is a mailing
-list that you can join, details are at:
-
-http://sourceforge.net/mail/?group_id=4993
-
-9. Legal Info
-=============
-
-The Linux DECnet project team have placed their code under the GPL. The
-software is provided "as is" and without warranty express or implied.
-DECnet is a trademark of Compaq. This software is not a product of
-Compaq. We acknowledge the help of people at Compaq in providing extra
-documentation above and beyond what was previously publicly available.
-
-Steve Whitehouse <SteveW@ACM.org>
-
diff --git a/Documentation/networking/device_drivers/can/freescale/flexcan.rst b/Documentation/networking/device_drivers/can/freescale/flexcan.rst
index 4e3eec6cecd2..106cd2890135 100644
--- a/Documentation/networking/device_drivers/can/freescale/flexcan.rst
+++ b/Documentation/networking/device_drivers/can/freescale/flexcan.rst
@@ -5,7 +5,7 @@ Flexcan CAN Controller driver
 =============================
 
 Authors: Marc Kleine-Budde <mkl@pengutronix.de>,
-Dario Binacchi <dario.binacchi@amarula.solutions.com>
+Dario Binacchi <dario.binacchi@amarulasolutions.com>
 
 On/off RTR frames reception
 ===========================
diff --git a/Documentation/networking/device_drivers/ethernet/index.rst b/Documentation/networking/device_drivers/ethernet/index.rst
index 7f1777173abb..5196905582c5 100644
--- a/Documentation/networking/device_drivers/ethernet/index.rst
+++ b/Documentation/networking/device_drivers/ethernet/index.rst
@@ -52,6 +52,7 @@ Contents:
    ti/tlan
    toshiba/spider_net
    wangxun/txgbe
+   wangxun/ngbe
 
 .. only::  subproject and html
 
diff --git a/Documentation/networking/device_drivers/ethernet/wangxun/ngbe.rst b/Documentation/networking/device_drivers/ethernet/wangxun/ngbe.rst
new file mode 100644
index 000000000000..43a02f9943e1
--- /dev/null
+++ b/Documentation/networking/device_drivers/ethernet/wangxun/ngbe.rst
@@ -0,0 +1,14 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=============================================================
+Linux Base Driver for WangXun(R) Gigabit PCI Express Adapters
+=============================================================
+
+WangXun Gigabit Linux driver.
+Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd.
+
+Support
+=======
+ If you have problems with the software or hardware, please contact our
+ customer support team via email at nic-support@net-swift.com or check our website
+ at https://www.net-swift.com
diff --git a/Documentation/networking/devlink/ice.rst b/Documentation/networking/devlink/ice.rst
index 8c082b139bbd..0c89ceb8986d 100644
--- a/Documentation/networking/devlink/ice.rst
+++ b/Documentation/networking/devlink/ice.rst
@@ -139,6 +139,42 @@ EMP firmware image.
 The driver does not currently support reloading the driver via
 ``DEVLINK_RELOAD_ACTION_DRIVER_REINIT``.
 
+Port split
+==========
+
+The ``ice`` driver supports port splitting only for port 0, as the FW has
+a predefined set of available port split options for the whole device.
+
+A system reboot is required for port split to be applied.
+
+The following command will select the port split option with 4 ports:
+
+.. code:: shell
+
+    $ devlink port split pci/0000:16:00.0/0 count 4
+
+The list of all available port options will be printed to dynamic debug after
+each ``split`` and ``unsplit`` command. The first option is the default.
+
+.. code:: shell
+
+    ice 0000:16:00.0: Available port split options and max port speeds (Gbps):
+    ice 0000:16:00.0: Status  Split      Quad 0          Quad 1
+    ice 0000:16:00.0:         count  L0  L1  L2  L3  L4  L5  L6  L7
+    ice 0000:16:00.0: Active  2     100   -   -   - 100   -   -   -
+    ice 0000:16:00.0:         2      50   -  50   -   -   -   -   -
+    ice 0000:16:00.0: Pending 4      25  25  25  25   -   -   -   -
+    ice 0000:16:00.0:         4      25  25   -   -  25  25   -   -
+    ice 0000:16:00.0:         8      10  10  10  10  10  10  10  10
+    ice 0000:16:00.0:         1     100   -   -   -   -   -   -   -
+
+There could be multiple FW port options with the same port split count. When
+the same port split count request is issued again, the next FW port option with
+the same port split count will be selected.
+
+``devlink port unsplit`` will select the option with a split count of 1. If
+there is no FW option available with split count 1, you will receive an error.
+
 Regions
 =======
 
diff --git a/Documentation/networking/devlink/index.rst b/Documentation/networking/devlink/index.rst
index e3a5f985673e..4b653d040627 100644
--- a/Documentation/networking/devlink/index.rst
+++ b/Documentation/networking/devlink/index.rst
@@ -13,10 +13,8 @@ new APIs prefixed by ``devl_*``. The older APIs handle all the locking
 in devlink core, but don't allow registration of most sub-objects once
 the main devlink object is itself registered. The newer ``devl_*`` APIs assume
 the devlink instance lock is already held. Drivers can take the instance
-lock by calling ``devl_lock()``. It is also held in most of the callbacks.
-Eventually all callbacks will be invoked under the devlink instance lock,
-refer to the use of the ``DEVLINK_NL_FLAG_NO_LOCK`` flag in devlink core
-to find out which callbacks are not converted, yet.
+lock by calling ``devl_lock()``. It is also held all callbacks of devlink
+netlink commands.
 
 Drivers are encouraged to use the devlink instance lock for their own needs.
 
diff --git a/Documentation/networking/dsa/configuration.rst b/Documentation/networking/dsa/configuration.rst
index 2b08f1a772d3..827701f8cbfe 100644
--- a/Documentation/networking/dsa/configuration.rst
+++ b/Documentation/networking/dsa/configuration.rst
@@ -49,6 +49,9 @@ In this documentation the following Ethernet interfaces are used:
 *eth0*
   the master interface
 
+*eth1*
+  another master interface
+
 *lan1*
   a slave interface
 
@@ -360,3 +363,96 @@ the ``self`` flag) has been removed. This results in the following changes:
 
 Script writers are therefore encouraged to use the ``master static`` set of
 flags when working with bridge FDB entries on DSA switch interfaces.
+
+Affinity of user ports to CPU ports
+-----------------------------------
+
+Typically, DSA switches are attached to the host via a single Ethernet
+interface, but in cases where the switch chip is discrete, the hardware design
+may permit the use of 2 or more ports connected to the host, for an increase in
+termination throughput.
+
+DSA can make use of multiple CPU ports in two ways. First, it is possible to
+statically assign the termination traffic associated with a certain user port
+to be processed by a certain CPU port. This way, user space can implement
+custom policies of static load balancing between user ports, by spreading the
+affinities according to the available CPU ports.
+
+Secondly, it is possible to perform load balancing between CPU ports on a per
+packet basis, rather than statically assigning user ports to CPU ports.
+This can be achieved by placing the DSA masters under a LAG interface (bonding
+or team). DSA monitors this operation and creates a mirror of this software LAG
+on the CPU ports facing the physical DSA masters that constitute the LAG slave
+devices.
+
+To make use of multiple CPU ports, the firmware (device tree) description of
+the switch must mark all the links between CPU ports and their DSA masters
+using the ``ethernet`` reference/phandle. At startup, only a single CPU port
+and DSA master will be used - the numerically first port from the firmware
+description which has an ``ethernet`` property. It is up to the user to
+configure the system for the switch to use other masters.
+
+DSA uses the ``rtnl_link_ops`` mechanism (with a "dsa" ``kind``) to allow
+changing the DSA master of a user port. The ``IFLA_DSA_MASTER`` u32 netlink
+attribute contains the ifindex of the master device that handles each slave
+device. The DSA master must be a valid candidate based on firmware node
+information, or a LAG interface which contains only slaves which are valid
+candidates.
+
+Using iproute2, the following manipulations are possible:
+
+  .. code-block:: sh
+
+    # See the DSA master in current use
+    ip -d link show dev swp0
+        (...)
+        dsa master eth0
+
+    # Static CPU port distribution
+    ip link set swp0 type dsa master eth1
+    ip link set swp1 type dsa master eth0
+    ip link set swp2 type dsa master eth1
+    ip link set swp3 type dsa master eth0
+
+    # CPU ports in LAG, using explicit assignment of the DSA master
+    ip link add bond0 type bond mode balance-xor && ip link set bond0 up
+    ip link set eth1 down && ip link set eth1 master bond0
+    ip link set swp0 type dsa master bond0
+    ip link set swp1 type dsa master bond0
+    ip link set swp2 type dsa master bond0
+    ip link set swp3 type dsa master bond0
+    ip link set eth0 down && ip link set eth0 master bond0
+    ip -d link show dev swp0
+        (...)
+        dsa master bond0
+
+    # CPU ports in LAG, relying on implicit migration of the DSA master
+    ip link add bond0 type bond mode balance-xor && ip link set bond0 up
+    ip link set eth0 down && ip link set eth0 master bond0
+    ip link set eth1 down && ip link set eth1 master bond0
+    ip -d link show dev swp0
+        (...)
+        dsa master bond0
+
+Notice that in the case of CPU ports under a LAG, the use of the
+``IFLA_DSA_MASTER`` netlink attribute is not strictly needed, but rather, DSA
+reacts to the ``IFLA_MASTER`` attribute change of its present master (``eth0``)
+and migrates all user ports to the new upper of ``eth0``, ``bond0``. Similarly,
+when ``bond0`` is destroyed using ``RTM_DELLINK``, DSA migrates the user ports
+that were assigned to this interface to the first physical DSA master which is
+eligible, based on the firmware description (it effectively reverts to the
+startup configuration).
+
+In a setup with more than 2 physical CPU ports, it is therefore possible to mix
+static user to CPU port assignment with LAG between DSA masters. It is not
+possible to statically assign a user port towards a DSA master that has any
+upper interfaces (this includes LAG devices - the master must always be the LAG
+in this case).
+
+Live changing of the DSA master (and thus CPU port) affinity of a user port is
+permitted, in order to allow dynamic redistribution in response to traffic.
+
+Physical DSA masters are allowed to join and leave at any time a LAG interface
+used as a DSA master; however, DSA will reject a LAG interface as a valid
+candidate for being a DSA master unless it has at least one physical DSA master
+as a slave device.
diff --git a/Documentation/networking/dsa/dsa.rst b/Documentation/networking/dsa/dsa.rst
index d742ba6bd211..a94ddf83348a 100644
--- a/Documentation/networking/dsa/dsa.rst
+++ b/Documentation/networking/dsa/dsa.rst
@@ -303,6 +303,20 @@ These frames are then queued for transmission using the master network device
 Ethernet switch will be able to process these incoming frames from the
 management interface and deliver them to the physical switch port.
 
+When using multiple CPU ports, it is possible to stack a LAG (bonding/team)
+device between the DSA slave devices and the physical DSA masters. The LAG
+device is thus also a DSA master, but the LAG slave devices continue to be DSA
+masters as well (just with no user port assigned to them; this is needed for
+recovery in case the LAG DSA master disappears). Thus, the data path of the LAG
+DSA master is used asymmetrically. On RX, the ``ETH_P_XDSA`` handler, which
+calls ``dsa_switch_rcv()``, is invoked early (on the physical DSA master;
+LAG slave). Therefore, the RX data path of the LAG DSA master is not used.
+On the other hand, TX takes place linearly: ``dsa_slave_xmit`` calls
+``dsa_enqueue_skb``, which calls ``dev_queue_xmit`` towards the LAG DSA master.
+The latter calls ``dev_queue_xmit`` towards one physical DSA master or the
+other, and in both cases, the packet exits the system through a hardware path
+towards the switch.
+
 Graphical representation
 ------------------------
 
@@ -629,6 +643,24 @@ Switch configuration
   PHY cannot be found. In this case, probing of the DSA switch continues
   without that particular port.
 
+- ``port_change_master``: method through which the affinity (association used
+  for traffic termination purposes) between a user port and a CPU port can be
+  changed. By default all user ports from a tree are assigned to the first
+  available CPU port that makes sense for them (most of the times this means
+  the user ports of a tree are all assigned to the same CPU port, except for H
+  topologies as described in commit 2c0b03258b8b). The ``port`` argument
+  represents the index of the user port, and the ``master`` argument represents
+  the new DSA master ``net_device``. The CPU port associated with the new
+  master can be retrieved by looking at ``struct dsa_port *cpu_dp =
+  master->dsa_ptr``. Additionally, the master can also be a LAG device where
+  all the slave devices are physical DSA masters. LAG DSA masters also have a
+  valid ``master->dsa_ptr`` pointer, however this is not unique, but rather a
+  duplicate of the first physical DSA master's (LAG slave) ``dsa_ptr``. In case
+  of a LAG DSA master, a further call to ``port_lag_join`` will be emitted
+  separately for the physical CPU ports associated with the physical DSA
+  masters, requesting them to create a hardware LAG associated with the LAG
+  interface.
+
 PHY devices and link management
 -------------------------------
 
@@ -1095,9 +1127,3 @@ capable hardware, but does not enforce a strict switch device driver model. On
 the other DSA enforces a fairly strict device driver model, and deals with most
 of the switch specific. At some point we should envision a merger between these
 two subsystems and get the best of both worlds.
-
-Other hanging fruits
---------------------
-
-- allowing more than one CPU/management interface:
-  http://comments.gmane.org/gmane.linux.network/365657
diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst
index dbca3e9ec782..d578b8bcd8a4 100644
--- a/Documentation/networking/ethtool-netlink.rst
+++ b/Documentation/networking/ethtool-netlink.rst
@@ -220,6 +220,8 @@ Userspace to kernel:
   ``ETHTOOL_MSG_PHC_VCLOCKS_GET``       get PHC virtual clocks info
   ``ETHTOOL_MSG_MODULE_SET``            set transceiver module parameters
   ``ETHTOOL_MSG_MODULE_GET``            get transceiver module parameters
+  ``ETHTOOL_MSG_PSE_SET``               set PSE parameters
+  ``ETHTOOL_MSG_PSE_GET``               get PSE parameters
   ===================================== =================================
 
 Kernel to userspace:
@@ -260,6 +262,7 @@ Kernel to userspace:
   ``ETHTOOL_MSG_STATS_GET_REPLY``          standard statistics
   ``ETHTOOL_MSG_PHC_VCLOCKS_GET_REPLY``    PHC virtual clocks info
   ``ETHTOOL_MSG_MODULE_GET_REPLY``         transceiver module parameters
+  ``ETHTOOL_MSG_PSE_GET_REPLY``            PSE parameters
   ======================================== =================================
 
 ``GET`` requests are sent by userspace applications to retrieve device
@@ -426,6 +429,7 @@ Kernel response contents:
   ``ETHTOOL_A_LINKMODES_DUPLEX``              u8      duplex mode
   ``ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG``    u8      Master/slave port mode
   ``ETHTOOL_A_LINKMODES_MASTER_SLAVE_STATE``  u8      Master/slave port state
+  ``ETHTOOL_A_LINKMODES_RATE_MATCHING``       u8      PHY rate matching
   ==========================================  ======  ==========================
 
 For ``ETHTOOL_A_LINKMODES_OURS``, value represents advertised modes and mask
@@ -449,6 +453,7 @@ Request contents:
   ``ETHTOOL_A_LINKMODES_SPEED``               u32     link speed (Mb/s)
   ``ETHTOOL_A_LINKMODES_DUPLEX``              u8      duplex mode
   ``ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG``    u8      Master/slave port mode
+  ``ETHTOOL_A_LINKMODES_RATE_MATCHING``       u8      PHY rate matching
   ``ETHTOOL_A_LINKMODES_LANES``               u32     lanes
   ==========================================  ======  ==========================
 
@@ -1625,6 +1630,62 @@ For SFF-8636 modules, low power mode is forced by the host according to table
 For CMIS modules, low power mode is forced by the host according to table 6-12
 in revision 5.0 of the specification.
 
+PSE_GET
+=======
+
+Gets PSE attributes.
+
+Request contents:
+
+  =====================================  ======  ==========================
+  ``ETHTOOL_A_PSE_HEADER``               nested  request header
+  =====================================  ======  ==========================
+
+Kernel response contents:
+
+  ======================================  ======  =============================
+  ``ETHTOOL_A_PSE_HEADER``                nested  reply header
+  ``ETHTOOL_A_PODL_PSE_ADMIN_STATE``         u32  Operational state of the PoDL
+                                                  PSE functions
+  ``ETHTOOL_A_PODL_PSE_PW_D_STATUS``         u32  power detection status of the
+                                                  PoDL PSE.
+  ======================================  ======  =============================
+
+When set, the optional ``ETHTOOL_A_PODL_PSE_ADMIN_STATE`` attribute identifies
+the operational state of the PoDL PSE functions.  The operational state of the
+PSE function can be changed using the ``ETHTOOL_A_PODL_PSE_ADMIN_CONTROL``
+action. This option is corresponding to ``IEEE 802.3-2018`` 30.15.1.1.2
+aPoDLPSEAdminState. Possible values are:
+
+.. kernel-doc:: include/uapi/linux/ethtool.h
+    :identifiers: ethtool_podl_pse_admin_state
+
+When set, the optional ``ETHTOOL_A_PODL_PSE_PW_D_STATUS`` attribute identifies
+the power detection status of the PoDL PSE.  The status depend on internal PSE
+state machine and automatic PD classification support. This option is
+corresponding to ``IEEE 802.3-2018`` 30.15.1.1.3 aPoDLPSEPowerDetectionStatus.
+Possible values are:
+
+.. kernel-doc:: include/uapi/linux/ethtool.h
+    :identifiers: ethtool_podl_pse_pw_d_status
+
+PSE_SET
+=======
+
+Sets PSE parameters.
+
+Request contents:
+
+  ======================================  ======  =============================
+  ``ETHTOOL_A_PSE_HEADER``                nested  request header
+  ``ETHTOOL_A_PODL_PSE_ADMIN_CONTROL``       u32  Control PoDL PSE Admin state
+  ======================================  ======  =============================
+
+When set, the optional ``ETHTOOL_A_PODL_PSE_ADMIN_CONTROL`` attribute is used
+to control PoDL PSE Admin functions. This option is implementing
+``IEEE 802.3-2018`` 30.15.1.2.1 acPoDLPSEAdminControl. See
+``ETHTOOL_A_PODL_PSE_ADMIN_STATE`` for supported values.
+
 Request translation
 ===================
 
diff --git a/Documentation/networking/index.rst b/Documentation/networking/index.rst
index 03b215bddde8..16a153bcc5fe 100644
--- a/Documentation/networking/index.rst
+++ b/Documentation/networking/index.rst
@@ -47,7 +47,6 @@ Contents:
    cdc_mbim
    dccp
    dctcp
-   decnet
    dns_resolver
    driver
    eql
@@ -93,6 +92,7 @@ Contents:
    radiotap-headers
    rds
    regulatory
+   representors
    rxrpc
    sctp
    secid
diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst
index a759872a2883..e7b3fa7bb3f7 100644
--- a/Documentation/networking/ip-sysctl.rst
+++ b/Documentation/networking/ip-sysctl.rst
@@ -1040,6 +1040,35 @@ tcp_challenge_ack_limit - INTEGER
 	TCP stack implements per TCP socket limits anyway.
 	Default: INT_MAX (unlimited)
 
+tcp_ehash_entries - INTEGER
+	Show the number of hash buckets for TCP sockets in the current
+	networking namespace.
+
+	A negative value means the networking namespace does not own its
+	hash buckets and shares the initial networking namespace's one.
+
+tcp_child_ehash_entries - INTEGER
+	Control the number of hash buckets for TCP sockets in the child
+	networking namespace, which must be set before clone() or unshare().
+
+	If the value is not 0, the kernel uses a value rounded up to 2^n
+	as the actual hash bucket size.  0 is a special value, meaning
+	the child networking namespace will share the initial networking
+	namespace's hash buckets.
+
+	Note that the child will use the global one in case the kernel
+	fails to allocate enough memory.  In addition, the global hash
+	buckets are spread over available NUMA nodes, but the allocation
+	of the child hash table depends on the current process's NUMA
+	policy, which could result in performance differences.
+
+	Note also that the default value of tcp_max_tw_buckets and
+	tcp_max_syn_backlog depend on the hash bucket size.
+
+	Possible values: 0, 2^n (n: 0 - 24 (16Mi))
+
+	Default: 0
+
 UDP variables
 =============
 
diff --git a/Documentation/networking/phy.rst b/Documentation/networking/phy.rst
index 704f31da5167..06f4fcdb58b6 100644
--- a/Documentation/networking/phy.rst
+++ b/Documentation/networking/phy.rst
@@ -308,6 +308,21 @@ Some of the interface modes are described below:
     rate of 125Mpbs using a 4B/5B encoding scheme, resulting in an underlying
     data rate of 100Mpbs.
 
+``PHY_INTERFACE_MODE_QUSGMII``
+    This defines the Cisco the Quad USGMII mode, which is the Quad variant of
+    the USGMII (Universal SGMII) link. It's very similar to QSGMII, but uses
+    a Packet Control Header (PCH) instead of the 7 bytes preamble to carry not
+    only the port id, but also so-called "extensions". The only documented
+    extension so-far in the specification is the inclusion of timestamps, for
+    PTP-enabled PHYs. This mode isn't compatible with QSGMII, but offers the
+    same capabilities in terms of link speed and negociation.
+
+``PHY_INTERFACE_MODE_1000BASEKX``
+    This is 1000BASE-X as defined by IEEE 802.3 Clause 36 with Clause 73
+    autonegotiation. Generally, it will be used with a Clause 70 PMD. To
+    contrast with the 1000BASE-X phy mode used for Clause 38 and 39 PMDs, this
+    interface mode has different autonegotiation and only supports full duplex.
+
 Pause frames / flow control
 ===========================
 
diff --git a/Documentation/networking/representors.rst b/Documentation/networking/representors.rst
new file mode 100644
index 000000000000..ee1f5cd54496
--- /dev/null
+++ b/Documentation/networking/representors.rst
@@ -0,0 +1,259 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+=============================
+Network Function Representors
+=============================
+
+This document describes the semantics and usage of representor netdevices, as
+used to control internal switching on SmartNICs.  For the closely-related port
+representors on physical (multi-port) switches, see
+:ref:`Documentation/networking/switchdev.rst <switchdev>`.
+
+Motivation
+----------
+
+Since the mid-2010s, network cards have started offering more complex
+virtualisation capabilities than the legacy SR-IOV approach (with its simple
+MAC/VLAN-based switching model) can support.  This led to a desire to offload
+software-defined networks (such as OpenVSwitch) to these NICs to specify the
+network connectivity of each function.  The resulting designs are variously
+called SmartNICs or DPUs.
+
+Network function representors bring the standard Linux networking stack to
+virtual switches and IOV devices.  Just as each physical port of a Linux-
+controlled switch has a separate netdev, so does each virtual port of a virtual
+switch.
+When the system boots, and before any offload is configured, all packets from
+the virtual functions appear in the networking stack of the PF via the
+representors.  The PF can thus always communicate freely with the virtual
+functions.
+The PF can configure standard Linux forwarding between representors, the uplink
+or any other netdev (routing, bridging, TC classifiers).
+
+Thus, a representor is both a control plane object (representing the function in
+administrative commands) and a data plane object (one end of a virtual pipe).
+As a virtual link endpoint, the representor can be configured like any other
+netdevice; in some cases (e.g. link state) the representee will follow the
+representor's configuration, while in others there are separate APIs to
+configure the representee.
+
+Definitions
+-----------
+
+This document uses the term "switchdev function" to refer to the PCIe function
+which has administrative control over the virtual switch on the device.
+Typically, this will be a PF, but conceivably a NIC could be configured to grant
+these administrative privileges instead to a VF or SF (subfunction).
+Depending on NIC design, a multi-port NIC might have a single switchdev function
+for the whole device or might have a separate virtual switch, and hence
+switchdev function, for each physical network port.
+If the NIC supports nested switching, there might be separate switchdev
+functions for each nested switch, in which case each switchdev function should
+only create representors for the ports on the (sub-)switch it directly
+administers.
+
+A "representee" is the object that a representor represents.  So for example in
+the case of a VF representor, the representee is the corresponding VF.
+
+What does a representor do?
+---------------------------
+
+A representor has three main roles.
+
+1. It is used to configure the network connection the representee sees, e.g.
+   link up/down, MTU, etc.  For instance, bringing the representor
+   administratively UP should cause the representee to see a link up / carrier
+   on event.
+2. It provides the slow path for traffic which does not hit any offloaded
+   fast-path rules in the virtual switch.  Packets transmitted on the
+   representor netdevice should be delivered to the representee; packets
+   transmitted by the representee which fail to match any switching rule should
+   be received on the representor netdevice.  (That is, there is a virtual pipe
+   connecting the representor to the representee, similar in concept to a veth
+   pair.)
+   This allows software switch implementations (such as OpenVSwitch or a Linux
+   bridge) to forward packets between representees and the rest of the network.
+3. It acts as a handle by which switching rules (such as TC filters) can refer
+   to the representee, allowing these rules to be offloaded.
+
+The combination of 2) and 3) means that the behaviour (apart from performance)
+should be the same whether a TC filter is offloaded or not.  E.g. a TC rule
+on a VF representor applies in software to packets received on that representor
+netdevice, while in hardware offload it would apply to packets transmitted by
+the representee VF.  Conversely, a mirred egress redirect to a VF representor
+corresponds in hardware to delivery directly to the representee VF.
+
+What functions should have a representor?
+-----------------------------------------
+
+Essentially, for each virtual port on the device's internal switch, there
+should be a representor.
+Some vendors have chosen to omit representors for the uplink and the physical
+network port, which can simplify usage (the uplink netdev becomes in effect the
+physical port's representor) but does not generalise to devices with multiple
+ports or uplinks.
+
+Thus, the following should all have representors:
+
+ - VFs belonging to the switchdev function.
+ - Other PFs on the local PCIe controller, and any VFs belonging to them.
+ - PFs and VFs on external PCIe controllers on the device (e.g. for any embedded
+   System-on-Chip within the SmartNIC).
+ - PFs and VFs with other personalities, including network block devices (such
+   as a vDPA virtio-blk PF backed by remote/distributed storage), if (and only
+   if) their network access is implemented through a virtual switch port. [#]_
+   Note that such functions can require a representor despite the representee
+   not having a netdev.
+ - Subfunctions (SFs) belonging to any of the above PFs or VFs, if they have
+   their own port on the switch (as opposed to using their parent PF's port).
+ - Any accelerators or plugins on the device whose interface to the network is
+   through a virtual switch port, even if they do not have a corresponding PCIe
+   PF or VF.
+
+This allows the entire switching behaviour of the NIC to be controlled through
+representor TC rules.
+
+It is a common misunderstanding to conflate virtual ports with PCIe virtual
+functions or their netdevs.  While in simple cases there will be a 1:1
+correspondence between VF netdevices and VF representors, more advanced device
+configurations may not follow this.
+A PCIe function which does not have network access through the internal switch
+(not even indirectly through the hardware implementation of whatever services
+the function provides) should *not* have a representor (even if it has a
+netdev).
+Such a function has no switch virtual port for the representor to configure or
+to be the other end of the virtual pipe.
+The representor represents the virtual port, not the PCIe function nor the 'end
+user' netdevice.
+
+.. [#] The concept here is that a hardware IP stack in the device performs the
+   translation between block DMA requests and network packets, so that only
+   network packets pass through the virtual port onto the switch.  The network
+   access that the IP stack "sees" would then be configurable through tc rules;
+   e.g. its traffic might all be wrapped in a specific VLAN or VxLAN.  However,
+   any needed configuration of the block device *qua* block device, not being a
+   networking entity, would not be appropriate for the representor and would
+   thus use some other channel such as devlink.
+   Contrast this with the case of a virtio-blk implementation which forwards the
+   DMA requests unchanged to another PF whose driver then initiates and
+   terminates IP traffic in software; in that case the DMA traffic would *not*
+   run over the virtual switch and the virtio-blk PF should thus *not* have a
+   representor.
+
+How are representors created?
+-----------------------------
+
+The driver instance attached to the switchdev function should, for each virtual
+port on the switch, create a pure-software netdevice which has some form of
+in-kernel reference to the switchdev function's own netdevice or driver private
+data (``netdev_priv()``).
+This may be by enumerating ports at probe time, reacting dynamically to the
+creation and destruction of ports at run time, or a combination of the two.
+
+The operations of the representor netdevice will generally involve acting
+through the switchdev function.  For example, ``ndo_start_xmit()`` might send
+the packet through a hardware TX queue attached to the switchdev function, with
+either packet metadata or queue configuration marking it for delivery to the
+representee.
+
+How are representors identified?
+--------------------------------
+
+The representor netdevice should *not* directly refer to a PCIe device (e.g.
+through ``net_dev->dev.parent`` / ``SET_NETDEV_DEV()``), either of the
+representee or of the switchdev function.
+Instead, it should implement the ``ndo_get_devlink_port()`` netdevice op, which
+the kernel uses to provide the ``phys_switch_id`` and ``phys_port_name`` sysfs
+nodes.  (Some legacy drivers implement ``ndo_get_port_parent_id()`` and
+``ndo_get_phys_port_name()`` directly, but this is deprecated.)  See
+:ref:`Documentation/networking/devlink/devlink-port.rst <devlink_port>` for the
+details of this API.
+
+It is expected that userland will use this information (e.g. through udev rules)
+to construct an appropriately informative name or alias for the netdevice.  For
+instance if the switchdev function is ``eth4`` then a representor with a
+``phys_port_name`` of ``p0pf1vf2`` might be renamed ``eth4pf1vf2rep``.
+
+There are as yet no established conventions for naming representors which do not
+correspond to PCIe functions (e.g. accelerators and plugins).
+
+How do representors interact with TC rules?
+-------------------------------------------
+
+Any TC rule on a representor applies (in software TC) to packets received by
+that representor netdevice.  Thus, if the delivery part of the rule corresponds
+to another port on the virtual switch, the driver may choose to offload it to
+hardware, applying it to packets transmitted by the representee.
+
+Similarly, since a TC mirred egress action targeting the representor would (in
+software) send the packet through the representor (and thus indirectly deliver
+it to the representee), hardware offload should interpret this as delivery to
+the representee.
+
+As a simple example, if ``PORT_DEV`` is the physical port representor and
+``REP_DEV`` is a VF representor, the following rules::
+
+    tc filter add dev $REP_DEV parent ffff: protocol ipv4 flower \
+        action mirred egress redirect dev $PORT_DEV
+    tc filter add dev $PORT_DEV parent ffff: protocol ipv4 flower skip_sw \
+        action mirred egress mirror dev $REP_DEV
+
+would mean that all IPv4 packets from the VF are sent out the physical port, and
+all IPv4 packets received on the physical port are delivered to the VF in
+addition to ``PORT_DEV``.  (Note that without ``skip_sw`` on the second rule,
+the VF would get two copies, as the packet reception on ``PORT_DEV`` would
+trigger the TC rule again and mirror the packet to ``REP_DEV``.)
+
+On devices without separate port and uplink representors, ``PORT_DEV`` would
+instead be the switchdev function's own uplink netdevice.
+
+Of course the rules can (if supported by the NIC) include packet-modifying
+actions (e.g. VLAN push/pop), which should be performed by the virtual switch.
+
+Tunnel encapsulation and decapsulation are rather more complicated, as they
+involve a third netdevice (a tunnel netdev operating in metadata mode, such as
+a VxLAN device created with ``ip link add vxlan0 type vxlan external``) and
+require an IP address to be bound to the underlay device (e.g. switchdev
+function uplink netdev or port representor).  TC rules such as::
+
+    tc filter add dev $REP_DEV parent ffff: flower \
+        action tunnel_key set id $VNI src_ip $LOCAL_IP dst_ip $REMOTE_IP \
+                              dst_port 4789 \
+        action mirred egress redirect dev vxlan0
+    tc filter add dev vxlan0 parent ffff: flower enc_src_ip $REMOTE_IP \
+        enc_dst_ip $LOCAL_IP enc_key_id $VNI enc_dst_port 4789 \
+        action tunnel_key unset action mirred egress redirect dev $REP_DEV
+
+where ``LOCAL_IP`` is an IP address bound to ``PORT_DEV``, and ``REMOTE_IP`` is
+another IP address on the same subnet, mean that packets sent by the VF should
+be VxLAN encapsulated and sent out the physical port (the driver has to deduce
+this by a route lookup of ``LOCAL_IP`` leading to ``PORT_DEV``, and also
+perform an ARP/neighbour table lookup to find the MAC addresses to use in the
+outer Ethernet frame), while UDP packets received on the physical port with UDP
+port 4789 should be parsed as VxLAN and, if their VSID matches ``$VNI``,
+decapsulated and forwarded to the VF.
+
+If this all seems complicated, just remember the 'golden rule' of TC offload:
+the hardware should ensure the same final results as if the packets were
+processed through the slow path, traversed software TC (except ignoring any
+``skip_hw`` rules and applying any ``skip_sw`` rules) and were transmitted or
+received through the representor netdevices.
+
+Configuring the representee's MAC
+---------------------------------
+
+The representee's link state is controlled through the representor.  Setting the
+representor administratively UP or DOWN should cause carrier ON or OFF at the
+representee.
+
+Setting an MTU on the representor should cause that same MTU to be reported to
+the representee.
+(On hardware that allows configuring separate and distinct MTU and MRU values,
+the representor MTU should correspond to the representee's MRU and vice-versa.)
+
+Currently there is no way to use the representor to set the station permanent
+MAC address of the representee; other methods available to do this include:
+
+ - legacy SR-IOV (``ip link set DEVICE vf NUM mac LLADDR``)
+ - devlink port function (see **devlink-port(8)** and
+   :ref:`Documentation/networking/devlink/devlink-port.rst <devlink_port>`)
diff --git a/Documentation/networking/smc-sysctl.rst b/Documentation/networking/smc-sysctl.rst
index 742e90e6d822..6d8acdbe9be1 100644
--- a/Documentation/networking/smc-sysctl.rst
+++ b/Documentation/networking/smc-sysctl.rst
@@ -34,3 +34,28 @@ smcr_buf_type - INTEGER
         - 1 - Use virtually contiguous buffers
         - 2 - Mixed use of the two types. Try physically contiguous buffers first.
           If not available, use virtually contiguous buffers then.
+
+smcr_testlink_time - INTEGER
+	How frequently SMC-R link sends out TEST_LINK LLC messages to confirm
+	viability, after the last activity of connections on it. Value 0 means
+	disabling TEST_LINK.
+
+	Default: 30 seconds.
+
+wmem - INTEGER
+	Initial size of send buffer used by SMC sockets.
+	The default value inherits from net.ipv4.tcp_wmem[1].
+
+	The minimum value is 16KiB and there is no hard limit for max value, but
+	only allowed 512KiB for SMC-R and 1MiB for SMC-D.
+
+	Default: 16K
+
+rmem - INTEGER
+	Initial size of receive buffer (RMB) used by SMC sockets.
+	The default value inherits from net.ipv4.tcp_rmem[1].
+
+	The minimum value is 16KiB and there is no hard limit for max value, but
+	only allowed 512KiB for SMC-R and 1MiB for SMC-D.
+
+	Default: 128K
diff --git a/Documentation/networking/switchdev.rst b/Documentation/networking/switchdev.rst
index bbf272e9d607..758f1dae3fce 100644
--- a/Documentation/networking/switchdev.rst
+++ b/Documentation/networking/switchdev.rst
@@ -1,5 +1,6 @@
 .. SPDX-License-Identifier: GPL-2.0
 .. include:: <isonum.txt>
+.. _switchdev:
 
 ===============================================
 Ethernet switch device driver model (switchdev)
diff --git a/Documentation/process/5.Posting.rst b/Documentation/process/5.Posting.rst
index 906235c11c24..d87f1fee4cbc 100644
--- a/Documentation/process/5.Posting.rst
+++ b/Documentation/process/5.Posting.rst
@@ -256,8 +256,10 @@ The tags in common use are:
  - Cc: the named person received a copy of the patch and had the
    opportunity to comment on it.
 
-Be careful in the addition of tags to your patches: only Cc: is appropriate
-for addition without the explicit permission of the person named.
+Be careful in the addition of tags to your patches, as only Cc: is appropriate
+for addition without the explicit permission of the person named; using
+Reported-by: is fine most of the time as well, but ask for permission if
+the bug was reported in private.
 
 
 Sending the patch
diff --git a/Documentation/process/changes.rst b/Documentation/process/changes.rst
index 19c286c23786..9a90197989dd 100644
--- a/Documentation/process/changes.rst
+++ b/Documentation/process/changes.rst
@@ -31,6 +31,8 @@ you probably needn't concern yourself with pcmciautils.
 ====================== ===============  ========================================
 GNU C                  5.1              gcc --version
 Clang/LLVM (optional)  11.0.0           clang --version
+Rust (optional)        1.62.0           rustc --version
+bindgen (optional)     0.56.0           bindgen --version
 GNU make               3.81             make --version
 bash                   4.2              bash --version
 binutils               2.23             ld -v
@@ -80,6 +82,29 @@ kernels. Older releases aren't guaranteed to work, and we may drop workarounds
 from the kernel that were used to support older versions. Please see additional
 docs on :ref:`Building Linux with Clang/LLVM <kbuild_llvm>`.
 
+Rust (optional)
+---------------
+
+A particular version of the Rust toolchain is required. Newer versions may or
+may not work because the kernel depends on some unstable Rust features, for
+the moment.
+
+Each Rust toolchain comes with several "components", some of which are required
+(like ``rustc``) and some that are optional. The ``rust-src`` component (which
+is optional) needs to be installed to build the kernel. Other components are
+useful for developing.
+
+Please see Documentation/rust/quick-start.rst for instructions on how to
+satisfy the build requirements of Rust support. In particular, the ``Makefile``
+target ``rustavailable`` is useful to check why the Rust toolchain may not
+be detected.
+
+bindgen (optional)
+------------------
+
+``bindgen`` is used to generate the Rust bindings to the C side of the kernel.
+It depends on ``libclang``.
+
 Make
 ----
 
@@ -348,6 +373,12 @@ Sphinx
 Please see :ref:`sphinx_install` in :ref:`Documentation/doc-guide/sphinx.rst <sphinxdoc>`
 for details about Sphinx requirements.
 
+rustdoc
+-------
+
+``rustdoc`` is used to generate the documentation for Rust code. Please see
+Documentation/rust/general-information.rst for more information.
+
 Getting updated software
 ========================
 
@@ -364,6 +395,16 @@ Clang/LLVM
 
 - :ref:`Getting LLVM <getting_llvm>`.
 
+Rust
+----
+
+- Documentation/rust/quick-start.rst.
+
+bindgen
+-------
+
+- Documentation/rust/quick-start.rst.
+
 Make
 ----
 
diff --git a/Documentation/process/code-of-conduct-interpretation.rst b/Documentation/process/code-of-conduct-interpretation.rst
index e899f14a4ba2..922e0b547bc3 100644
--- a/Documentation/process/code-of-conduct-interpretation.rst
+++ b/Documentation/process/code-of-conduct-interpretation.rst
@@ -51,7 +51,7 @@ the Technical Advisory Board (TAB) or other maintainers if you're
 uncertain how to handle situations that come up.  It will not be
 considered a violation report unless you want it to be.  If you are
 uncertain about approaching the TAB or any other maintainers, please
-reach out to our conflict mediator, Mishi Choudhary <mishi@linux.com>.
+reach out to our conflict mediator, Joanna Lee <joanna.lee@gesmer.com>.
 
 In the end, "be kind to each other" is really what the end goal is for
 everybody.  We know everyone is human and we all fail at times, but the
@@ -127,10 +127,12 @@ are listed at https://kernel.org/code-of-conduct.html.  Members can not
 access reports made before they joined or after they have left the
 committee.
 
-The initial Code of Conduct Committee consists of volunteer members of
-the TAB, as well as a professional mediator acting as a neutral third
-party.  The first task of the committee is to establish documented
-processes, which will be made public.
+The Code of Conduct Committee consists of volunteer community members
+appointed by the TAB, as well as a professional mediator acting as a
+neutral third party.  The processes the Code of Conduct committee will
+use to address reports is varied and will depend on the individual
+circumstance, however, this file serves as documentation for the
+general process used.
 
 Any member of the committee, including the mediator, can be contacted
 directly if a reporter does not wish to include the full committee in a
@@ -141,16 +143,16 @@ processes (see above) and consults with the TAB as needed and
 appropriate, for instance to request and receive information about the
 kernel community.
 
-Any decisions by the committee will be brought to the TAB, for
-implementation of enforcement with the relevant maintainers if needed.
-A decision by the Code of Conduct Committee can be overturned by the TAB
-by a two-thirds vote.
+Any decisions regarding enforcement recommendations will be brought to
+the TAB for implementation of enforcement with the relevant maintainers
+if needed.  A decision by the Code of Conduct Committee can be overturned
+by the TAB by a two-thirds vote.
 
 At quarterly intervals, the Code of Conduct Committee and TAB will
 provide a report summarizing the anonymised reports that the Code of
 Conduct committee has received and their status, as well details of any
 overridden decisions including complete and identifiable voting details.
 
-We expect to establish a different process for Code of Conduct Committee
-staffing beyond the bootstrap period.  This document will be updated
-with that information when this occurs.
+Because how we interpret and enforce the Code of Conduct will evolve over
+time, this document will be updated when necessary to reflect any
+changes.
diff --git a/Documentation/process/coding-style.rst b/Documentation/process/coding-style.rst
index 03eb53fd029a..007e49ef6cec 100644
--- a/Documentation/process/coding-style.rst
+++ b/Documentation/process/coding-style.rst
@@ -1186,6 +1186,68 @@ expression used.  For instance:
 	#endif /* CONFIG_SOMETHING */
 
 
+22) Do not crash the kernel
+---------------------------
+
+In general, the decision to crash the kernel belongs to the user, rather
+than to the kernel developer.
+
+Avoid panic()
+*************
+
+panic() should be used with care and primarily only during system boot.
+panic() is, for example, acceptable when running out of memory during boot and
+not being able to continue.
+
+Use WARN() rather than BUG()
+****************************
+
+Do not add new code that uses any of the BUG() variants, such as BUG(),
+BUG_ON(), or VM_BUG_ON(). Instead, use a WARN*() variant, preferably
+WARN_ON_ONCE(), and possibly with recovery code. Recovery code is not
+required if there is no reasonable way to at least partially recover.
+
+"I'm too lazy to do error handling" is not an excuse for using BUG(). Major
+internal corruptions with no way of continuing may still use BUG(), but need
+good justification.
+
+Use WARN_ON_ONCE() rather than WARN() or WARN_ON()
+**************************************************
+
+WARN_ON_ONCE() is generally preferred over WARN() or WARN_ON(), because it
+is common for a given warning condition, if it occurs at all, to occur
+multiple times. This can fill up and wrap the kernel log, and can even slow
+the system enough that the excessive logging turns into its own, additional
+problem.
+
+Do not WARN lightly
+*******************
+
+WARN*() is intended for unexpected, this-should-never-happen situations.
+WARN*() macros are not to be used for anything that is expected to happen
+during normal operation. These are not pre- or post-condition asserts, for
+example. Again: WARN*() must not be used for a condition that is expected
+to trigger easily, for example, by user space actions. pr_warn_once() is a
+possible alternative, if you need to notify the user of a problem.
+
+Do not worry about panic_on_warn users
+**************************************
+
+A few more words about panic_on_warn: Remember that ``panic_on_warn`` is an
+available kernel option, and that many users set this option. This is why
+there is a "Do not WARN lightly" writeup, above. However, the existence of
+panic_on_warn users is not a valid reason to avoid the judicious use
+WARN*(). That is because, whoever enables panic_on_warn has explicitly
+asked the kernel to crash if a WARN*() fires, and such users must be
+prepared to deal with the consequences of a system that is somewhat more
+likely to crash.
+
+Use BUILD_BUG_ON() for compile-time assertions
+**********************************************
+
+The use of BUILD_BUG_ON() is acceptable and encouraged, because it is a
+compile-time assertion that has no effect at runtime.
+
 Appendix I) References
 ----------------------
 
diff --git a/Documentation/process/deprecated.rst b/Documentation/process/deprecated.rst
index a6e36d9c3d14..c8fd53a11a20 100644
--- a/Documentation/process/deprecated.rst
+++ b/Documentation/process/deprecated.rst
@@ -138,17 +138,20 @@ be NUL terminated. This can lead to various linear read overflows and
 other misbehavior due to the missing termination. It also NUL-pads
 the destination buffer if the source contents are shorter than the
 destination buffer size, which may be a needless performance penalty
-for callers using only NUL-terminated strings. The safe replacement is
+for callers using only NUL-terminated strings.
+
+When the destination is required to be NUL-terminated, the replacement is
 strscpy(), though care must be given to any cases where the return value
 of strncpy() was used, since strscpy() does not return a pointer to the
 destination, but rather a count of non-NUL bytes copied (or negative
 errno when it truncates). Any cases still needing NUL-padding should
 instead use strscpy_pad().
 
-If a caller is using non-NUL-terminated strings, strncpy() can
-still be used, but destinations should be marked with the `__nonstring
+If a caller is using non-NUL-terminated strings, strtomem() should be
+used, and the destinations should be marked with the `__nonstring
 <https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html>`_
-attribute to avoid future compiler warnings.
+attribute to avoid future compiler warnings. For cases still needing
+NUL-padding, strtomem_pad() can be used.
 
 strlcpy()
 ---------
diff --git a/Documentation/process/index.rst b/Documentation/process/index.rst
index 2ba2a1582bbe..d4b6217472b0 100644
--- a/Documentation/process/index.rst
+++ b/Documentation/process/index.rst
@@ -5,6 +5,7 @@
 
 .. _process_index:
 
+=============================================
 Working with the kernel development community
 =============================================
 
diff --git a/Documentation/process/maintainer-pgp-guide.rst b/Documentation/process/maintainer-pgp-guide.rst
index 29e7d7b1cd44..40bfbd3b7648 100644
--- a/Documentation/process/maintainer-pgp-guide.rst
+++ b/Documentation/process/maintainer-pgp-guide.rst
@@ -121,57 +121,56 @@ edit your ``~/.gnupg/gpg-agent.conf`` file to set your own values::
     to remove anything you had in place for older versions of GnuPG, as
     it may not be doing the right thing any more.
 
-Set up a refresh cronjob
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-You will need to regularly refresh your keyring in order to get the
-latest changes on other people's public keys, which is best done with a
-daily cronjob::
-
-    @daily /usr/bin/gpg2 --refresh >/dev/null 2>&1
-
-Check the full path to your ``gpg`` or ``gpg2`` command and use the
-``gpg2`` command if regular ``gpg`` for you is the legacy GnuPG v.1.
-
-.. _master_key:
+.. _protect_your_key:
 
-Protect your master PGP key
-===========================
+Protect your PGP key
+====================
 
 This guide assumes that you already have a PGP key that you use for Linux
 kernel development purposes. If you do not yet have one, please see the
 "`Protecting Code Integrity`_" document mentioned earlier for guidance
 on how to create a new one.
 
-You should also make a new key if your current one is weaker than 2048 bits
-(RSA).
-
-Master key vs. Subkeys
-----------------------
-
-Subkeys are fully independent PGP keypairs that are tied to the "master"
-key using certifying key signatures (certificates). It is important to
-understand the following:
-
-1. There are no technical differences between the "master key" and "subkeys."
-2. At creation time, we assign functional limitations to each key by
-   giving it specific capabilities.
-3. A PGP key can have 4 capabilities:
-
-   - **[S]** key can be used for signing
-   - **[E]** key can be used for encryption
-   - **[A]** key can be used for authentication
-   - **[C]** key can be used for certifying other keys
-
-4. A single key may have multiple capabilities.
-5. A subkey is fully independent from the master key. A message
-   encrypted to a subkey cannot be decrypted with the master key. If you
-   lose your private subkey, it cannot be recreated from the master key
-   in any way.
-
-The key carrying the **[C]** (certify) capability is considered the
-"master" key because it is the only key that can be used to indicate
-relationship with other keys. Only the **[C]** key can be used to:
+You should also make a new key if your current one is weaker than 2048
+bits (RSA).
+
+Understanding PGP Subkeys
+-------------------------
+
+A PGP key rarely consists of a single keypair -- usually it is a
+collection of independent subkeys that can be used for different
+purposes based on their capabilities, assigned at their creation time.
+PGP defines four capabilities that a key can have:
+
+- **[S]** keys can be used for signing
+- **[E]** keys can be used for encryption
+- **[A]** keys can be used for authentication
+- **[C]** keys can be used for certifying other keys
+
+The key with the **[C]** capability is often called the "master" key,
+but this terminology is misleading because it implies that the Certify
+key can be used in place of any of other subkey on the same chain (like
+a physical "master key" can be used to open the locks made for other
+keys). Since this is not the case, this guide will refer to it as "the
+Certify key" to avoid any ambiguity.
+
+It is critical to fully understand the following:
+
+1. All subkeys are fully independent from each other. If you lose a
+   private subkey, it cannot be restored or recreated from any other
+   private key on your chain.
+2. With the exception of the Certify key, there can be multiple subkeys
+   with identical capabilities (e.g. you can have 2 valid encryption
+   subkeys, 3 valid signing subkeys, but only one valid certification
+   subkey). All subkeys are fully independent -- a message encrypted to
+   one **[E]** subkey cannot be decrypted with any other **[E]** subkey
+   you may also have.
+3. A single subkey may have multiple capabilities (e.g. your **[C]** key
+   can also be your **[S]** key).
+
+The key carrying the **[C]** (certify) capability is the only key that
+can be used to indicate relationship with other keys. Only the **[C]**
+key can be used to:
 
 - add or revoke other keys (subkeys) with S/E/A capabilities
 - add, change or revoke identities (uids) associated with the key
@@ -180,7 +179,7 @@ relationship with other keys. Only the **[C]** key can be used to:
 
 By default, GnuPG creates the following when generating new keys:
 
-- A master key carrying both Certify and Sign capabilities (**[SC]**)
+- One subkey carrying both Certify and Sign capabilities (**[SC]**)
 - A separate subkey with the Encryption capability (**[E]**)
 
 If you used the default parameters when generating your key, then that
@@ -192,9 +191,6 @@ for example::
     uid           [ultimate] Alice Dev <adev@kernel.org>
     ssb   rsa2048 2018-01-23 [E] [expires: 2020-01-23]
 
-Any key carrying the **[C]** capability is your master key, regardless
-of any other capabilities it may have assigned to it.
-
 The long line under the ``sec`` entry is your key fingerprint --
 whenever you see ``[fpr]`` in the examples below, that 40-character
 string is what it refers to.
@@ -215,37 +211,30 @@ strong passphrase. To set it or change it, use::
 Create a separate Signing subkey
 --------------------------------
 
-Our goal is to protect your master key by moving it to offline media, so
-if you only have a combined **[SC]** key, then you should create a separate
-signing subkey::
+Our goal is to protect your Certify key by moving it to offline media,
+so if you only have a combined **[SC]** key, then you should create a
+separate signing subkey::
 
     $ gpg --quick-addkey [fpr] ed25519 sign
 
-Remember to tell the keyservers about this change, so others can pull down
-your new subkey::
-
-    $ gpg --send-key [fpr]
-
 .. note:: ECC support in GnuPG
 
     GnuPG 2.1 and later has full support for Elliptic Curve
     Cryptography, with ability to combine ECC subkeys with traditional
-    RSA master keys. The main upside of ECC cryptography is that it is
-    much faster computationally and creates much smaller signatures when
+    RSA keys. The main upside of ECC cryptography is that it is much
+    faster computationally and creates much smaller signatures when
     compared byte for byte with 2048+ bit RSA keys. Unless you plan on
     using a smartcard device that does not support ECC operations, we
     recommend that you create an ECC signing subkey for your kernel
     work.
 
-    If for some reason you prefer to stay with RSA subkeys, just replace
-    "ed25519" with "rsa2048" in the above command. Additionally, if you
-    plan to use a hardware device that does not support ED25519 ECC
-    keys, like Nitrokey Pro or a Yubikey, then you should use
-    "nistp256" instead or "ed25519."
+    Note, that if you plan to use a hardware device that does not
+    support ED25519 ECC keys, you should choose "nistp256" instead or
+    "ed25519."
 
 
-Back up your master key for disaster recovery
----------------------------------------------
+Back up your Certify key for disaster recovery
+----------------------------------------------
 
 The more signatures you have on your PGP key from other developers, the
 more reasons you have to create a backup version that lives on something
@@ -277,9 +266,7 @@ home, such as your bank vault.
     Your printer is probably no longer a simple dumb device connected to
     your parallel port, but since the output is still encrypted with
     your passphrase, printing out even to "cloud-integrated" modern
-    printers should remain a relatively safe operation. One option is to
-    change the passphrase on your master key immediately after you are
-    done with paperkey.
+    printers should remain a relatively safe operation.
 
 Back up your whole GnuPG directory
 ----------------------------------
@@ -300,7 +287,7 @@ will use for backup purposes. You will need to encrypt them using LUKS
 -- refer to your distro's documentation on how to accomplish this.
 
 For the encryption passphrase, you can use the same one as on your
-master key.
+PGP key.
 
 Once the encryption process is over, re-insert the USB drive and make
 sure it gets properly mounted. Copy your entire ``.gnupg`` directory
@@ -319,7 +306,7 @@ far away, because you'll need to use it every now and again for things
 like editing identities, adding or revoking subkeys, or signing other
 people's keys.
 
-Remove the master key from  your homedir
+Remove the Certify key from your homedir
 ----------------------------------------
 
 The files in our home directory are not as well protected as we like to
@@ -334,7 +321,7 @@ think.  They can be leaked or stolen via many different means:
 Protecting your key with a good passphrase greatly helps reduce the risk
 of any of the above, but passphrases can be discovered via keyloggers,
 shoulder-surfing, or any number of other means. For this reason, the
-recommended setup is to remove your master key from your home directory
+recommended setup is to remove your Certify key from your home directory
 and store it on offline storage.
 
 .. warning::
@@ -343,7 +330,7 @@ and store it on offline storage.
     your GnuPG directory in its entirety. What we are about to do will
     render your key useless if you do not have a usable backup!
 
-First, identify the keygrip of your master key::
+First, identify the keygrip of your Certify key::
 
     $ gpg --with-keygrip --list-key [fpr]
 
@@ -359,7 +346,7 @@ The output will be something like this::
           Keygrip = 3333000000000000000000000000000000000000
 
 Find the keygrip entry that is beneath the ``pub`` line (right under the
-master key fingerprint). This will correspond directly to a file in your
+Certify key fingerprint). This will correspond directly to a file in your
 ``~/.gnupg`` directory::
 
     $ cd ~/.gnupg/private-keys-v1.d
@@ -369,13 +356,13 @@ master key fingerprint). This will correspond directly to a file in your
     3333000000000000000000000000000000000000.key
 
 All you have to do is simply remove the .key file that corresponds to
-the master keygrip::
+the Certify key keygrip::
 
     $ cd ~/.gnupg/private-keys-v1.d
     $ rm 1111000000000000000000000000000000000000.key
 
 Now, if you issue the ``--list-secret-keys`` command, it will show that
-the master key is missing (the ``#`` indicates it is not available)::
+the Certify key is missing (the ``#`` indicates it is not available)::
 
     $ gpg --list-secret-keys
     sec#  rsa2048 2018-01-24 [SC] [expires: 2020-01-24]
@@ -404,7 +391,7 @@ file, which still contains your private keys.
 Move the subkeys to a dedicated crypto device
 =============================================
 
-Even though the master key is now safe from being leaked or stolen, the
+Even though the Certify key is now safe from being leaked or stolen, the
 subkeys are still in your home directory. Anyone who manages to get
 their hands on those will be able to decrypt your communication or fake
 your signatures (if they know the passphrase). Furthermore, each time a
@@ -447,7 +434,8 @@ functionality. There are several options available:
 - `Yubikey 5`_: proprietary hardware and software, but cheaper than
   Nitrokey Pro and comes available in the USB-C form that is more useful
   with newer laptops. Offers additional security features such as FIDO
-  U2F, among others, and now finally supports ECC keys (NISTP).
+  U2F, among others, and now finally supports NISTP and ED25519 ECC
+  keys.
 
 `LWN has a good review`_ of some of the above models, as well as several
 others. Your choice will depend on cost, shipping availability in your
@@ -460,7 +448,7 @@ geographical region, and open/proprietary hardware considerations.
     Foundation.
 
 .. _`Nitrokey Start`: https://shop.nitrokey.com/shop/product/nitrokey-start-6
-.. _`Nitrokey Pro 2`: https://shop.nitrokey.com/shop/product/nitrokey-pro-2-3
+.. _`Nitrokey Pro 2`: https://shop.nitrokey.com/shop/product/nkpr2-nitrokey-pro-2-3
 .. _`Yubikey 5`: https://www.yubico.com/products/yubikey-5-overview/
 .. _Gnuk: https://www.fsij.org/doc-gnuk/
 .. _`LWN has a good review`: https://lwn.net/Articles/736231/
@@ -627,10 +615,10 @@ Other common GnuPG operations
 Here is a quick reference for some common operations you'll need to do
 with your PGP key.
 
-Mounting your master key offline storage
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Mounting your safe offline storage
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You will need your master key for any of the operations below, so you
+You will need your Certify key for any of the operations below, so you
 will first need to mount your backup offline storage and tell GnuPG to
 use it::
 
@@ -644,7 +632,7 @@ your regular home directory location).
 Extending key expiration date
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The master key has the default expiration date of 2 years from the date
+The Certify key has the default expiration date of 2 years from the date
 of creation. This is done both for security reasons and to make obsolete
 keys eventually disappear from keyservers.
 
@@ -685,6 +673,7 @@ remote end.
 
 .. _`Agent Forwarding over SSH`: https://wiki.gnupg.org/AgentForwarding
 
+.. _pgp_with_git:
 
 Using PGP with Git
 ==================
@@ -828,6 +817,63 @@ You can tell git to always sign commits::
 
 .. _verify_identities:
 
+
+How to work with signed patches
+-------------------------------
+
+It is possible to use your PGP key to sign patches sent to kernel
+developer mailing lists. Since existing email signature mechanisms
+(PGP-Mime or PGP-inline) tend to cause problems with regular code
+review tasks, you should use the tool kernel.org created for this
+purpose that puts cryptographic attestation signatures into message
+headers (a-la DKIM):
+
+- `Patatt Patch Attestation`_
+
+.. _`Patatt Patch Attestation`: https://pypi.org/project/patatt/
+
+Installing and configuring patatt
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Patatt is packaged for many distributions already, so please check there
+first. You can also install it from pypi using "``pip install patatt``".
+
+If you already have your PGP key configured with git (via the
+``user.signingKey`` configuration parameter), then patatt requires no
+further configuration. You can start signing your patches by installing
+the git-send-email hook in the repository you want::
+
+    patatt install-hook
+
+Now any patches you send with ``git send-email`` will be automatically
+signed with your cryptographic signature.
+
+Checking patatt signatures
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you are using ``b4`` to retrieve and apply patches, then it will
+automatically attempt to verify all DKIM and patatt signatures it
+encounters, for example::
+
+    $ b4 am 20220720205013.890942-1-broonie@kernel.org
+    [...]
+    Checking attestation on all messages, may take a moment...
+    ---
+      ✓ [PATCH v1 1/3] kselftest/arm64: Correct buffer allocation for SVE Z registers
+      ✓ [PATCH v1 2/3] arm64/sve: Document our actual ABI for clearing registers on syscall
+      ✓ [PATCH v1 3/3] kselftest/arm64: Enforce actual ABI for SVE syscalls
+      ---
+      ✓ Signed: openpgp/broonie@kernel.org
+      ✓ Signed: DKIM/kernel.org
+
+.. note::
+
+    Patatt and b4 are still in active development and you should check
+    the latest documentation for these projects for any new or updated
+    features.
+
+.. _kernel_identities:
+
 How to verify kernel developer identities
 =========================================
 
@@ -899,65 +945,17 @@ the new default in GnuPG v2). To set it, add (or modify) the
 
     trust-model tofu+pgp
 
-How to use keyservers (more) safely
------------------------------------
-
-If you get a "No public key" error when trying to validate someone's
-tag, then you should attempt to lookup that key using a keyserver. It is
-important to keep in mind that there is absolutely no guarantee that the
-key you retrieve from PGP keyservers belongs to the actual person --
-that much is by design. You are supposed to use the Web of Trust to
-establish key validity.
-
-How to properly maintain the Web of Trust is beyond the scope of this
-document, simply because doing it properly requires both effort and
-dedication that tends to be beyond the caring threshold of most human
-beings. Here are some shortcuts that will help you reduce the risk of
-importing a malicious key.
-
-First, let's say you've tried to run ``git verify-tag`` but it returned
-an error saying the key is not found::
-
-    $ git verify-tag sunxi-fixes-for-4.15-2
-    gpg: Signature made Sun 07 Jan 2018 10:51:55 PM EST
-    gpg:                using RSA key DA73759BF8619E484E5A3B47389A54219C0F2430
-    gpg:                issuer "wens@...org"
-    gpg: Can't check signature: No public key
-
-Let's query the keyserver for more info about that key fingerprint (the
-fingerprint probably belongs to a subkey, so we can't use it directly
-without finding out the ID of the master key it is associated with)::
-
-    $ gpg --search DA73759BF8619E484E5A3B47389A54219C0F2430
-    gpg: data source: hkp://keys.gnupg.net
-    (1) Chen-Yu Tsai <wens@...org>
-          4096 bit RSA key C94035C21B4F2AEB, created: 2017-03-14, expires: 2019-03-15
-    Keys 1-1 of 1 for "DA73759BF8619E484E5A3B47389A54219C0F2430".  Enter number(s), N)ext, or Q)uit > q
-
-Locate the ID of the master key in the output, in our example
-``C94035C21B4F2AEB``. Now display the key of Linus Torvalds that you
-have on your keyring::
-
-    $ gpg --list-key torvalds@kernel.org
-    pub   rsa2048 2011-09-20 [SC]
-          ABAF11C65A2970B130ABE3C479BE3E4300411886
-    uid           [ unknown] Linus Torvalds <torvalds@kernel.org>
-    sub   rsa2048 2011-09-20 [E]
-
-Next, find a trust path from Linus Torvalds to the key-id you found via ``gpg
---search`` of the unknown key.  For this, you can use several tools including
-https://github.com/mricon/wotmate,
-https://git.kernel.org/pub/scm/docs/kernel/pgpkeys.git/tree/graphs, and
-https://the.earth.li/~noodles/pathfind.html.
-
-If you get a few decent trust paths, then it's a pretty good indication
-that it is a valid key. You can add it to your keyring from the
-keyserver now::
-
-    $ gpg --recv-key C94035C21B4F2AEB
-
-This process is not perfect, and you are obviously trusting the
-administrators of the PGP Pathfinder service to not be malicious (in
-fact, this goes against :ref:`devs_not_infra`). However, if you
-do not carefully maintain your own web of trust, then it is a marked
-improvement over blindly trusting keyservers.
+Using the kernel.org web of trust repository
+--------------------------------------------
+
+Kernel.org maintains a git repository with developers' public keys as a
+replacement for replicating keyserver networks that have gone mostly
+dark in the past few years. The full documentation for how to set up
+that repository as your source of public keys can be found here:
+
+- `Kernel developer PGP Keyring`_
+
+If you are a kernel developer, please consider submitting your key for
+inclusion into that keyring.
+
+.. _`Kernel developer PGP Keyring`: https://korg.docs.kernel.org/pgpkeys.html
diff --git a/Documentation/process/stable-kernel-rules.rst b/Documentation/process/stable-kernel-rules.rst
index c61865e91f52..2fd8aa593a28 100644
--- a/Documentation/process/stable-kernel-rules.rst
+++ b/Documentation/process/stable-kernel-rules.rst
@@ -97,6 +97,12 @@ text, like this:
 
     commit <sha1> upstream.
 
+or alternatively:
+
+.. code-block:: none
+
+    [ Upstream commit <sha1> ]
+
 Additionally, some patches submitted via :ref:`option_1` may have additional
 patch prerequisites which can be cherry-picked. This can be specified in the
 following format in the sign-off area:
diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst
index be49d8f2601b..7dc94555417d 100644
--- a/Documentation/process/submitting-patches.rst
+++ b/Documentation/process/submitting-patches.rst
@@ -715,8 +715,8 @@ references.
 
 .. _backtraces:
 
-Backtraces in commit mesages
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Backtraces in commit messages
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Backtraces help document the call chain leading to a problem. However,
 not all backtraces are helpful. For example, early boot call chains are
diff --git a/Documentation/rust/arch-support.rst b/Documentation/rust/arch-support.rst
new file mode 100644
index 000000000000..6982b63775da
--- /dev/null
+++ b/Documentation/rust/arch-support.rst
@@ -0,0 +1,19 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Arch Support
+============
+
+Currently, the Rust compiler (``rustc``) uses LLVM for code generation,
+which limits the supported architectures that can be targeted. In addition,
+support for building the kernel with LLVM/Clang varies (please see
+Documentation/kbuild/llvm.rst). This support is needed for ``bindgen``
+which uses ``libclang``.
+
+Below is a general summary of architectures that currently work. Level of
+support corresponds to ``S`` values in the ``MAINTAINERS`` file.
+
+============  ================  ==============================================
+Architecture  Level of support  Constraints
+============  ================  ==============================================
+``x86``       Maintained        ``x86_64`` only.
+============  ================  ==============================================
diff --git a/Documentation/rust/coding-guidelines.rst b/Documentation/rust/coding-guidelines.rst
new file mode 100644
index 000000000000..aa8ed082613e
--- /dev/null
+++ b/Documentation/rust/coding-guidelines.rst
@@ -0,0 +1,216 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Coding Guidelines
+=================
+
+This document describes how to write Rust code in the kernel.
+
+
+Style & formatting
+------------------
+
+The code should be formatted using ``rustfmt``. In this way, a person
+contributing from time to time to the kernel does not need to learn and
+remember one more style guide. More importantly, reviewers and maintainers
+do not need to spend time pointing out style issues anymore, and thus
+less patch roundtrips may be needed to land a change.
+
+.. note:: Conventions on comments and documentation are not checked by
+  ``rustfmt``. Thus those are still needed to be taken care of.
+
+The default settings of ``rustfmt`` are used. This means the idiomatic Rust
+style is followed. For instance, 4 spaces are used for indentation rather
+than tabs.
+
+It is convenient to instruct editors/IDEs to format while typing,
+when saving or at commit time. However, if for some reason reformatting
+the entire kernel Rust sources is needed at some point, the following can be
+run::
+
+	make LLVM=1 rustfmt
+
+It is also possible to check if everything is formatted (printing a diff
+otherwise), for instance for a CI, with::
+
+	make LLVM=1 rustfmtcheck
+
+Like ``clang-format`` for the rest of the kernel, ``rustfmt`` works on
+individual files, and does not require a kernel configuration. Sometimes it may
+even work with broken code.
+
+
+Comments
+--------
+
+"Normal" comments (i.e. ``//``, rather than code documentation which starts
+with ``///`` or ``//!``) are written in Markdown the same way as documentation
+comments are, even though they will not be rendered. This improves consistency,
+simplifies the rules and allows to move content between the two kinds of
+comments more easily. For instance:
+
+.. code-block:: rust
+
+	// `object` is ready to be handled now.
+	f(object);
+
+Furthermore, just like documentation, comments are capitalized at the beginning
+of a sentence and ended with a period (even if it is a single sentence). This
+includes ``// SAFETY:``, ``// TODO:`` and other "tagged" comments, e.g.:
+
+.. code-block:: rust
+
+	// FIXME: The error should be handled properly.
+
+Comments should not be used for documentation purposes: comments are intended
+for implementation details, not users. This distinction is useful even if the
+reader of the source file is both an implementor and a user of an API. In fact,
+sometimes it is useful to use both comments and documentation at the same time.
+For instance, for a ``TODO`` list or to comment on the documentation itself.
+For the latter case, comments can be inserted in the middle; that is, closer to
+the line of documentation to be commented. For any other case, comments are
+written after the documentation, e.g.:
+
+.. code-block:: rust
+
+	/// Returns a new [`Foo`].
+	///
+	/// # Examples
+	///
+	// TODO: Find a better example.
+	/// ```
+	/// let foo = f(42);
+	/// ```
+	// FIXME: Use fallible approach.
+	pub fn f(x: i32) -> Foo {
+	    // ...
+	}
+
+One special kind of comments are the ``// SAFETY:`` comments. These must appear
+before every ``unsafe`` block, and they explain why the code inside the block is
+correct/sound, i.e. why it cannot trigger undefined behavior in any case, e.g.:
+
+.. code-block:: rust
+
+	// SAFETY: `p` is valid by the safety requirements.
+	unsafe { *p = 0; }
+
+``// SAFETY:`` comments are not to be confused with the ``# Safety`` sections
+in code documentation. ``# Safety`` sections specify the contract that callers
+(for functions) or implementors (for traits) need to abide by. ``// SAFETY:``
+comments show why a call (for functions) or implementation (for traits) actually
+respects the preconditions stated in a ``# Safety`` section or the language
+reference.
+
+
+Code documentation
+------------------
+
+Rust kernel code is not documented like C kernel code (i.e. via kernel-doc).
+Instead, the usual system for documenting Rust code is used: the ``rustdoc``
+tool, which uses Markdown (a lightweight markup language).
+
+To learn Markdown, there are many guides available out there. For instance,
+the one at:
+
+	https://commonmark.org/help/
+
+This is how a well-documented Rust function may look like:
+
+.. code-block:: rust
+
+	/// Returns the contained [`Some`] value, consuming the `self` value,
+	/// without checking that the value is not [`None`].
+	///
+	/// # Safety
+	///
+	/// Calling this method on [`None`] is *[undefined behavior]*.
+	///
+	/// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html
+	///
+	/// # Examples
+	///
+	/// ```
+	/// let x = Some("air");
+	/// assert_eq!(unsafe { x.unwrap_unchecked() }, "air");
+	/// ```
+	pub unsafe fn unwrap_unchecked(self) -> T {
+	    match self {
+	        Some(val) => val,
+
+	        // SAFETY: The safety contract must be upheld by the caller.
+	        None => unsafe { hint::unreachable_unchecked() },
+	    }
+	}
+
+This example showcases a few ``rustdoc`` features and some conventions followed
+in the kernel:
+
+  - The first paragraph must be a single sentence briefly describing what
+    the documented item does. Further explanations must go in extra paragraphs.
+
+  - Unsafe functions must document their safety preconditions under
+    a ``# Safety`` section.
+
+  - While not shown here, if a function may panic, the conditions under which
+    that happens must be described under a ``# Panics`` section.
+
+    Please note that panicking should be very rare and used only with a good
+    reason. In almost all cases, a fallible approach should be used, typically
+    returning a ``Result``.
+
+  - If providing examples of usage would help readers, they must be written in
+    a section called ``# Examples``.
+
+  - Rust items (functions, types, constants...) must be linked appropriately
+    (``rustdoc`` will create a link automatically).
+
+  - Any ``unsafe`` block must be preceded by a ``// SAFETY:`` comment
+    describing why the code inside is sound.
+
+    While sometimes the reason might look trivial and therefore unneeded,
+    writing these comments is not just a good way of documenting what has been
+    taken into account, but most importantly, it provides a way to know that
+    there are no *extra* implicit constraints.
+
+To learn more about how to write documentation for Rust and extra features,
+please take a look at the ``rustdoc`` book at:
+
+	https://doc.rust-lang.org/rustdoc/how-to-write-documentation.html
+
+
+Naming
+------
+
+Rust kernel code follows the usual Rust naming conventions:
+
+	https://rust-lang.github.io/api-guidelines/naming.html
+
+When existing C concepts (e.g. macros, functions, objects...) are wrapped into
+a Rust abstraction, a name as close as reasonably possible to the C side should
+be used in order to avoid confusion and to improve readability when switching
+back and forth between the C and Rust sides. For instance, macros such as
+``pr_info`` from C are named the same in the Rust side.
+
+Having said that, casing should be adjusted to follow the Rust naming
+conventions, and namespacing introduced by modules and types should not be
+repeated in the item names. For instance, when wrapping constants like:
+
+.. code-block:: c
+
+	#define GPIO_LINE_DIRECTION_IN	0
+	#define GPIO_LINE_DIRECTION_OUT	1
+
+The equivalent in Rust may look like (ignoring documentation):
+
+.. code-block:: rust
+
+	pub mod gpio {
+	    pub enum LineDirection {
+	        In = bindings::GPIO_LINE_DIRECTION_IN as _,
+	        Out = bindings::GPIO_LINE_DIRECTION_OUT as _,
+	    }
+	}
+
+That is, the equivalent of ``GPIO_LINE_DIRECTION_IN`` would be referred to as
+``gpio::LineDirection::In``. In particular, it should not be named
+``gpio::gpio_line_direction::GPIO_LINE_DIRECTION_IN``.
diff --git a/Documentation/rust/general-information.rst b/Documentation/rust/general-information.rst
new file mode 100644
index 000000000000..49029ee82e55
--- /dev/null
+++ b/Documentation/rust/general-information.rst
@@ -0,0 +1,79 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+General Information
+===================
+
+This document contains useful information to know when working with
+the Rust support in the kernel.
+
+
+Code documentation
+------------------
+
+Rust kernel code is documented using ``rustdoc``, its built-in documentation
+generator.
+
+The generated HTML docs include integrated search, linked items (e.g. types,
+functions, constants), source code, etc. They may be read at (TODO: link when
+in mainline and generated alongside the rest of the documentation):
+
+	http://kernel.org/
+
+The docs can also be easily generated and read locally. This is quite fast
+(same order as compiling the code itself) and no special tools or environment
+are needed. This has the added advantage that they will be tailored to
+the particular kernel configuration used. To generate them, use the ``rustdoc``
+target with the same invocation used for compilation, e.g.::
+
+	make LLVM=1 rustdoc
+
+To read the docs locally in your web browser, run e.g.::
+
+	xdg-open rust/doc/kernel/index.html
+
+To learn about how to write the documentation, please see coding-guidelines.rst.
+
+
+Extra lints
+-----------
+
+While ``rustc`` is a very helpful compiler, some extra lints and analyses are
+available via ``clippy``, a Rust linter. To enable it, pass ``CLIPPY=1`` to
+the same invocation used for compilation, e.g.::
+
+	make LLVM=1 CLIPPY=1
+
+Please note that Clippy may change code generation, thus it should not be
+enabled while building a production kernel.
+
+
+Abstractions vs. bindings
+-------------------------
+
+Abstractions are Rust code wrapping kernel functionality from the C side.
+
+In order to use functions and types from the C side, bindings are created.
+Bindings are the declarations for Rust of those functions and types from
+the C side.
+
+For instance, one may write a ``Mutex`` abstraction in Rust which wraps
+a ``struct mutex`` from the C side and calls its functions through the bindings.
+
+Abstractions are not available for all the kernel internal APIs and concepts,
+but it is intended that coverage is expanded as time goes on. "Leaf" modules
+(e.g. drivers) should not use the C bindings directly. Instead, subsystems
+should provide as-safe-as-possible abstractions as needed.
+
+
+Conditional compilation
+-----------------------
+
+Rust code has access to conditional compilation based on the kernel
+configuration:
+
+.. code-block:: rust
+
+	#[cfg(CONFIG_X)]       // Enabled               (`y` or `m`)
+	#[cfg(CONFIG_X="y")]   // Enabled as a built-in (`y`)
+	#[cfg(CONFIG_X="m")]   // Enabled as a module   (`m`)
+	#[cfg(not(CONFIG_X))]  // Disabled
diff --git a/Documentation/rust/index.rst b/Documentation/rust/index.rst
new file mode 100644
index 000000000000..4ae8c66b94fa
--- /dev/null
+++ b/Documentation/rust/index.rst
@@ -0,0 +1,22 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Rust
+====
+
+Documentation related to Rust within the kernel. To start using Rust
+in the kernel, please read the quick-start.rst guide.
+
+.. toctree::
+    :maxdepth: 1
+
+    quick-start
+    general-information
+    coding-guidelines
+    arch-support
+
+.. only::  subproject and html
+
+   Indices
+   =======
+
+   * :ref:`genindex`
diff --git a/Documentation/rust/quick-start.rst b/Documentation/rust/quick-start.rst
new file mode 100644
index 000000000000..13b7744b1e27
--- /dev/null
+++ b/Documentation/rust/quick-start.rst
@@ -0,0 +1,232 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Quick Start
+===========
+
+This document describes how to get started with kernel development in Rust.
+
+
+Requirements: Building
+----------------------
+
+This section explains how to fetch the tools needed for building.
+
+Some of these requirements might be available from Linux distributions
+under names like ``rustc``, ``rust-src``, ``rust-bindgen``, etc. However,
+at the time of writing, they are likely not to be recent enough unless
+the distribution tracks the latest releases.
+
+To easily check whether the requirements are met, the following target
+can be used::
+
+	make LLVM=1 rustavailable
+
+This triggers the same logic used by Kconfig to determine whether
+``RUST_IS_AVAILABLE`` should be enabled; but it also explains why not
+if that is the case.
+
+
+rustc
+*****
+
+A particular version of the Rust compiler is required. Newer versions may or
+may not work because, for the moment, the kernel depends on some unstable
+Rust features.
+
+If ``rustup`` is being used, enter the checked out source code directory
+and run::
+
+	rustup override set $(scripts/min-tool-version.sh rustc)
+
+Otherwise, fetch a standalone installer or install ``rustup`` from:
+
+	https://www.rust-lang.org
+
+
+Rust standard library source
+****************************
+
+The Rust standard library source is required because the build system will
+cross-compile ``core`` and ``alloc``.
+
+If ``rustup`` is being used, run::
+
+	rustup component add rust-src
+
+The components are installed per toolchain, thus upgrading the Rust compiler
+version later on requires re-adding the component.
+
+Otherwise, if a standalone installer is used, the Rust repository may be cloned
+into the installation folder of the toolchain::
+
+	git clone --recurse-submodules \
+		--branch $(scripts/min-tool-version.sh rustc) \
+		https://github.com/rust-lang/rust \
+		$(rustc --print sysroot)/lib/rustlib/src/rust
+
+In this case, upgrading the Rust compiler version later on requires manually
+updating this clone.
+
+
+libclang
+********
+
+``libclang`` (part of LLVM) is used by ``bindgen`` to understand the C code
+in the kernel, which means LLVM needs to be installed; like when the kernel
+is compiled with ``CC=clang`` or ``LLVM=1``.
+
+Linux distributions are likely to have a suitable one available, so it is
+best to check that first.
+
+There are also some binaries for several systems and architectures uploaded at:
+
+	https://releases.llvm.org/download.html
+
+Otherwise, building LLVM takes quite a while, but it is not a complex process:
+
+	https://llvm.org/docs/GettingStarted.html#getting-the-source-code-and-building-llvm
+
+Please see Documentation/kbuild/llvm.rst for more information and further ways
+to fetch pre-built releases and distribution packages.
+
+
+bindgen
+*******
+
+The bindings to the C side of the kernel are generated at build time using
+the ``bindgen`` tool. A particular version is required.
+
+Install it via (note that this will download and build the tool from source)::
+
+	cargo install --locked --version $(scripts/min-tool-version.sh bindgen) bindgen
+
+
+Requirements: Developing
+------------------------
+
+This section explains how to fetch the tools needed for developing. That is,
+they are not needed when just building the kernel.
+
+
+rustfmt
+*******
+
+The ``rustfmt`` tool is used to automatically format all the Rust kernel code,
+including the generated C bindings (for details, please see
+coding-guidelines.rst).
+
+If ``rustup`` is being used, its ``default`` profile already installs the tool,
+thus nothing needs to be done. If another profile is being used, the component
+can be installed manually::
+
+	rustup component add rustfmt
+
+The standalone installers also come with ``rustfmt``.
+
+
+clippy
+******
+
+``clippy`` is a Rust linter. Running it provides extra warnings for Rust code.
+It can be run by passing ``CLIPPY=1`` to ``make`` (for details, please see
+general-information.rst).
+
+If ``rustup`` is being used, its ``default`` profile already installs the tool,
+thus nothing needs to be done. If another profile is being used, the component
+can be installed manually::
+
+	rustup component add clippy
+
+The standalone installers also come with ``clippy``.
+
+
+cargo
+*****
+
+``cargo`` is the Rust native build system. It is currently required to run
+the tests since it is used to build a custom standard library that contains
+the facilities provided by the custom ``alloc`` in the kernel. The tests can
+be run using the ``rusttest`` Make target.
+
+If ``rustup`` is being used, all the profiles already install the tool,
+thus nothing needs to be done.
+
+The standalone installers also come with ``cargo``.
+
+
+rustdoc
+*******
+
+``rustdoc`` is the documentation tool for Rust. It generates pretty HTML
+documentation for Rust code (for details, please see
+general-information.rst).
+
+``rustdoc`` is also used to test the examples provided in documented Rust code
+(called doctests or documentation tests). The ``rusttest`` Make target uses
+this feature.
+
+If ``rustup`` is being used, all the profiles already install the tool,
+thus nothing needs to be done.
+
+The standalone installers also come with ``rustdoc``.
+
+
+rust-analyzer
+*************
+
+The `rust-analyzer <https://rust-analyzer.github.io/>`_ language server can
+be used with many editors to enable syntax highlighting, completion, go to
+definition, and other features.
+
+``rust-analyzer`` needs a configuration file, ``rust-project.json``, which
+can be generated by the ``rust-analyzer`` Make target.
+
+
+Configuration
+-------------
+
+``Rust support`` (``CONFIG_RUST``) needs to be enabled in the ``General setup``
+menu. The option is only shown if a suitable Rust toolchain is found (see
+above), as long as the other requirements are met. In turn, this will make
+visible the rest of options that depend on Rust.
+
+Afterwards, go to::
+
+	Kernel hacking
+	    -> Sample kernel code
+	        -> Rust samples
+
+And enable some sample modules either as built-in or as loadable.
+
+
+Building
+--------
+
+Building a kernel with a complete LLVM toolchain is the best supported setup
+at the moment. That is::
+
+	make LLVM=1
+
+For architectures that do not support a full LLVM toolchain, use::
+
+	make CC=clang
+
+Using GCC also works for some configurations, but it is very experimental at
+the moment.
+
+
+Hacking
+-------
+
+To dive deeper, take a look at the source code of the samples
+at ``samples/rust/``, the Rust support code under ``rust/`` and
+the ``Rust hacking`` menu under ``Kernel hacking``.
+
+If GDB/Binutils is used and Rust symbols are not getting demangled, the reason
+is the toolchain does not support Rust's new v0 mangling scheme yet.
+There are a few ways out:
+
+  - Install a newer release (GDB >= 10.2, Binutils >= 2.36).
+
+  - Some versions of GDB (e.g. vanilla GDB 10.1) are able to use
+    the pre-demangled names embedded in the debug info (``CONFIG_DEBUG_INFO``).
diff --git a/Documentation/scheduler/sched-design-CFS.rst b/Documentation/scheduler/sched-design-CFS.rst
index 59b2d1fb4dc4..03db55504515 100644
--- a/Documentation/scheduler/sched-design-CFS.rst
+++ b/Documentation/scheduler/sched-design-CFS.rst
@@ -94,7 +94,7 @@ other HZ detail.  Thus the CFS scheduler has no notion of "timeslices" in the
 way the previous scheduler had, and has no heuristics whatsoever.  There is
 only one central tunable (you have to switch on CONFIG_SCHED_DEBUG):
 
-   /proc/sys/kernel/sched_min_granularity_ns
+   /sys/kernel/debug/sched/min_granularity_ns
 
 which can be used to tune the scheduler from "desktop" (i.e., low latencies) to
 "server" (i.e., good batching) workloads.  It defaults to a setting suitable
diff --git a/Documentation/security/landlock.rst b/Documentation/security/landlock.rst
index 5c77730b4479..c0029d5d02eb 100644
--- a/Documentation/security/landlock.rst
+++ b/Documentation/security/landlock.rst
@@ -7,7 +7,7 @@ Landlock LSM: kernel documentation
 ==================================
 
 :Author: Mickaël Salaün
-:Date: May 2022
+:Date: September 2022
 
 Landlock's goal is to create scoped access-control (i.e. sandboxing).  To
 harden a whole system, this feature should be available to any process,
@@ -49,13 +49,13 @@ Filesystem access rights
 ------------------------
 
 All access rights are tied to an inode and what can be accessed through it.
-Reading the content of a directory doesn't imply to be allowed to read the
+Reading the content of a directory does not imply to be allowed to read the
 content of a listed inode.  Indeed, a file name is local to its parent
 directory, and an inode can be referenced by multiple file names thanks to
 (hard) links.  Being able to unlink a file only has a direct impact on the
 directory, not the unlinked inode.  This is the reason why
-`LANDLOCK_ACCESS_FS_REMOVE_FILE` or `LANDLOCK_ACCESS_FS_REFER` are not allowed
-to be tied to files but only to directories.
+``LANDLOCK_ACCESS_FS_REMOVE_FILE`` or ``LANDLOCK_ACCESS_FS_REFER`` are not
+allowed to be tied to files but only to directories.
 
 Tests
 =====
diff --git a/Documentation/sound/kernel-api/writing-an-alsa-driver.rst b/Documentation/sound/kernel-api/writing-an-alsa-driver.rst
index 176b73583b7a..07a620c5ca74 100644
--- a/Documentation/sound/kernel-api/writing-an-alsa-driver.rst
+++ b/Documentation/sound/kernel-api/writing-an-alsa-driver.rst
@@ -3565,13 +3565,17 @@ given size.
 The second argument (type) and the third argument (device pointer) are
 dependent on the bus. For normal devices, pass the device pointer
 (typically identical as ``card->dev``) to the third argument with
-``SNDRV_DMA_TYPE_DEV`` type. For the continuous buffer unrelated to the
+``SNDRV_DMA_TYPE_DEV`` type.
+
+For the continuous buffer unrelated to the
 bus can be pre-allocated with ``SNDRV_DMA_TYPE_CONTINUOUS`` type.
 You can pass NULL to the device pointer in that case, which is the
 default mode implying to allocate with ``GFP_KERNEL`` flag.
-If you need a different GFP flag, you can pass it by encoding the flag
-into the device pointer via a special macro
-:c:func:`snd_dma_continuous_data()`.
+If you need a restricted (lower) address, set up the coherent DMA mask
+bits for the device, and pass the device pointer, like the normal
+device memory allocations.  For this type, it's still allowed to pass
+NULL to the device pointer, too, if no address restriction is needed.
+
 For the scatter-gather buffers, use ``SNDRV_DMA_TYPE_DEV_SG`` with the
 device pointer (see the `Non-Contiguous Buffers`_ section).
 
@@ -3811,15 +3815,6 @@ arguments here.  Since each vmalloc call should succeed at any time,
 we don't need to pre-allocate the buffers like other continuous
 pages.
 
-If you need the 32bit DMA allocation, pass the device pointer encoded
-by :c:func:`snd_dma_continuous_data()` with ``GFP_KERNEL|__GFP_DMA32``
-argument.
-
-::
-
-  snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC,
-          snd_dma_continuous_data(GFP_KERNEL | __GFP_DMA32), 0, 0);
-
 Proc Interface
 ==============
 
diff --git a/Documentation/staging/index.rst b/Documentation/staging/index.rst
index abd0d18254d2..ded8254bc0d7 100644
--- a/Documentation/staging/index.rst
+++ b/Documentation/staging/index.rst
@@ -14,45 +14,3 @@ Unsorted Documentation
    static-keys
    tee
    xz
-
-Atomic Types
-============
-
-.. raw:: latex
-
-    \footnotesize
-
-.. include:: ../atomic_t.txt
-   :literal:
-
-.. raw:: latex
-
-    \normalsize
-
-Atomic bitops
-=============
-
-.. raw:: latex
-
-    \footnotesize
-
-.. include:: ../atomic_bitops.txt
-   :literal:
-
-.. raw:: latex
-
-    \normalsize
-
-Memory Barriers
-===============
-
-.. raw:: latex
-
-    \footnotesize
-
-.. include:: ../memory-barriers.txt
-   :literal:
-
-.. raw:: latex
-
-    \normalsize
diff --git a/Documentation/subsystem-apis.rst b/Documentation/subsystem-apis.rst
new file mode 100644
index 000000000000..af65004a80aa
--- /dev/null
+++ b/Documentation/subsystem-apis.rst
@@ -0,0 +1,58 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+==============================
+Kernel subsystem documentation
+==============================
+
+These books get into the details of how specific kernel subsystems work
+from the point of view of a kernel developer.  Much of the information here
+is taken directly from the kernel source, with supplemental material added
+as needed (or at least as we managed to add it — probably *not* all that is
+needed).
+
+**Fixme**: much more organizational work is needed here.
+
+.. toctree::
+   :maxdepth: 1
+
+   driver-api/index
+   core-api/index
+   locking/index
+   accounting/index
+   block/index
+   cdrom/index
+   cpu-freq/index
+   fb/index
+   fpga/index
+   hid/index
+   i2c/index
+   iio/index
+   isdn/index
+   infiniband/index
+   leds/index
+   netlabel/index
+   networking/index
+   pcmcia/index
+   power/index
+   target/index
+   timers/index
+   spi/index
+   w1/index
+   watchdog/index
+   virt/index
+   input/index
+   hwmon/index
+   gpu/index
+   security/index
+   sound/index
+   crypto/index
+   filesystems/index
+   mm/index
+   bpf/index
+   usb/index
+   PCI/index
+   scsi/index
+   misc-devices/index
+   scheduler/index
+   mhi/index
+   peci/index
diff --git a/Documentation/trace/histogram.rst b/Documentation/trace/histogram.rst
index 859fd1b76c63..c1b685a38f6b 100644
--- a/Documentation/trace/histogram.rst
+++ b/Documentation/trace/histogram.rst
@@ -412,7 +412,7 @@ Extended error information
   Because the default sort key above is 'hitcount', the above shows a
   the list of call_sites by increasing hitcount, so that at the bottom
   we see the functions that made the most kmalloc calls during the
-  run.  If instead we we wanted to see the top kmalloc callers in
+  run.  If instead we wanted to see the top kmalloc callers in
   terms of the number of bytes requested rather than the number of
   calls, and we wanted the top caller to appear at the top, we can use
   the 'sort' parameter, along with the 'descending' modifier::
diff --git a/Documentation/trace/kprobes.rst b/Documentation/trace/kprobes.rst
index f318bceda1e6..48cf778a2468 100644
--- a/Documentation/trace/kprobes.rst
+++ b/Documentation/trace/kprobes.rst
@@ -328,8 +328,8 @@ Configuring Kprobes
 ===================
 
 When configuring the kernel using make menuconfig/xconfig/oldconfig,
-ensure that CONFIG_KPROBES is set to "y". Under "General setup", look
-for "Kprobes".
+ensure that CONFIG_KPROBES is set to "y", look for "Kprobes" under
+"General architecture-dependent options".
 
 So that you can load and unload Kprobes-based instrumentation modules,
 make sure "Loadable module support" (CONFIG_MODULES) and "Module
diff --git a/Documentation/trace/timerlat-tracer.rst b/Documentation/trace/timerlat-tracer.rst
index d643c95c01eb..db17df312bc8 100644
--- a/Documentation/trace/timerlat-tracer.rst
+++ b/Documentation/trace/timerlat-tracer.rst
@@ -20,7 +20,7 @@ For example::
         [root@f32 ~]# cd /sys/kernel/tracing/
         [root@f32 tracing]# echo timerlat > current_tracer
 
-It is possible to follow the trace by reading the trace trace file::
+It is possible to follow the trace by reading the trace file::
 
   [root@f32 tracing]# cat trace
   # tracer: timerlat
diff --git a/Documentation/translations/zh_CN/IRQ.txt b/Documentation/translations/zh_CN/IRQ.txt
deleted file mode 100644
index 9aec8dca4fcf..000000000000
--- a/Documentation/translations/zh_CN/IRQ.txt
+++ /dev/null
@@ -1,39 +0,0 @@
-Chinese translated version of Documentation/core-api/irq/index.rst
-
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
-
-Maintainer: Eric W. Biederman <ebiederman@xmission.com>
-Chinese maintainer: Fu Wei <tekkamanninja@gmail.com>
----------------------------------------------------------------------
-Documentation/core-api/irq/index.rst 的中文翻译
-
-如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
-交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
-英文版维护者： Eric W. Biederman <ebiederman@xmission.com>
-中文版维护者： 傅炜 Fu Wei <tekkamanninja@gmail.com>
-中文版翻译者： 傅炜 Fu Wei <tekkamanninja@gmail.com>
-中文版校译者： 傅炜 Fu Wei <tekkamanninja@gmail.com>
-
-
-以下为正文
----------------------------------------------------------------------
-何为 IRQ?
-
-一个 IRQ 是�自�个设备的一个中断请求。目�，它们�以�自一个硬件引脚，
-或�自一个数�包。多个设备�能连接到�个硬件引脚，从而共享一个 IRQ。
-
-一个 IRQ 编�是用于告知硬件中断�的内核标识。通常情况下，这是一个
-全局 irq_desc 数组的索引，但是除了在 linux/interrupt.h 中的实现，
-具体的细节是体系结构特定的。
-
-一个 IRQ 编�是设备上�个�能的中断�的枚举。通常情况下，枚举的编�是
-该引脚在系统内中断控制器的所有输入引脚中的编�。对于 ISA 总线中的情况，
-枚举的是在两个 i8259 中断控制器中 16 个输入引脚。
-
-架构�以对 IRQ 编�指定�外的�义，在硬件涉�任何手工�置的情况下，
-是被�倡的。ISA 的 IRQ 是一个分�这类�外�义的典型例�。
diff --git a/Documentation/translations/zh_CN/PCI/acpi-info.rst b/Documentation/translations/zh_CN/PCI/acpi-info.rst
new file mode 100644
index 000000000000..a35f39dcd858
--- /dev/null
+++ b/Documentation/translations/zh_CN/PCI/acpi-info.rst
@@ -0,0 +1,139 @@
+.. SPDX-License-Identifier: GPL-2.0
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/PCI/acpi-info.rst
+
+:翻译:
+
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+
+:校译:
+
+
+=====================
+PCI主桥的ACPI注�事项
+=====================
+
+一般的规则是，ACPI命�空间应该�述�作系统�能使用的所有东西，除�有其他方法让�作系
+统找到它[1, 2]。
+
+例如，没有标准的硬件机制�枚举PCI主桥，所以ACPI命�空间必须�述�个主桥�访问它
+下�的PCI�置空间的方法�主桥转�到PCI的地�空间窗�（使用_CRS）以�传统的INTx
+中断的路由（使用_PRT）。
+
+在主桥下�的PCI设备，通常�需�通过ACPI�述。�作系统�以通过标准的PCI枚举机制�
+�现它们，使用�置访问��现和识别设备，并读�和测�它们的BAR。然而，如果ACPI为它们
+�供电�管�或热�拔功能，或者如果设备有由平�中断控制器连接的INTx中断，需�一个_PRT
+��述这些连接，这�情况下ACPI�以�述PCI设备。
+
+ACPI资��述是通过ACPI命�空间中设备的_CRS对象完�的[2]。_CRS就�一个通用的PCI BAR：
+�作系统�以读�_CRS并找出正在消耗的资�，�使它没有该设备的驱动程�[3]。这一点很��，
+因为它�味�一个旧的�作系统�以正确地工作，�使是在�作系统�知�的新设备的系统上。新设
+备�能什么都��，但�作系统至少�以确�没有资�与它们冲�。
+
+�MCFG�HPET�ECDT等��表，�是�留地�空间的机制。��表是在�作系统在�动�期且在它
+能够解�ACPI命�空间之�需�知�的东西。如果定义了一个新的表，�使旧的�作系统忽略了这
+个表，它也需�正常�行。_CRS�许这样�，因为它是通用的，�以被旧的�作系统解�；而��表
+则��许。
+
+如果�作系统�管�一个通过ACPI�述的���现的设备，该设备将有一个特定的_HID/_CID，以
+告诉�作系统与之绑定的驱动程�，并且_CRS告诉�作系统和驱动程�该设备的寄存器在哪里。
+
+PCI主桥是PNP0A03或PNP0A08设备。它们的_CRS应该�述它们所消耗的所有地�空间。这包括它
+们转�到PCI总线上的所有窗�，以��转�到PCI的主桥本身的寄存器。主桥的寄存器包括次�/下
+级总线寄存器，决定了桥下�的总线范围，窗�寄存器�述了桥洞，等等。这些都是设备相关的，�
+架构相关的东西，所以PNP0A03/PNP0A08驱动�以管�它们的唯一方法是通过_PRS/_CRS/_SRS，
+它包�了特定于设备的细节。主桥寄存器也包括ECAM空间，因为它是由主桥消耗的。
+
+ACPI定义了一个Consumer/Producer��区分桥寄存器（“Consumer�下文译作消费者）和
+桥洞（“Producer�下文译作生产者）[4, 5]，但是早期的BIOS没有正确使用这个�。其结果
+是，目�的ACPI规范�为扩展地�空间�述符定义了消费者/生产者；在旧的QWord/Word/Word地
+�空间�述符中，该�应该被忽略。因此，�作系统必须�定所有的QWord/Word/Word�述符都是
+窗�。
+
+在增加扩展地�空间�述符之�，消费者/生产者的失败�味�没有办法�述PNP0A03/PNP0A08设
+备本身的桥寄存器。解决办法是在PNP0C02��器中�述桥寄存器（包括ECAM空间）[6]。
+除了ECAM之外，桥寄存器空间�正是特定于设备的，所以通用的PNP0A03/PNP0A08驱动程
+�(pci_root.c)没有必�了解它。
+
+新的架构应该能够在PNP0A03设备中使用“消费者�扩展地�空间�述符，用于桥寄存器，包括
+ECAM，尽管对[6]的严格解释�能�止这样�。旧的x86和ia64内核�定所有的地�空间�述
+符，包括“消费者�扩展地�空间的�述符，都是窗�，所以在这些架构上以这�方��述桥寄
+存器是�安全的。
+
+PNP0C02“主��设备基本上是万能的。除了“��将这些资�用于其他用途�之外，没有其他的编
+程模型。因此，PNP0C02 _CRS应该声明ACPI命�空间中(1)没有被_CRS声明的任何其他设备对
+象的地�空间，(2)�应该被OS分�给其他东西。
+
+除�有一个标准的固件接�用于�置访问，例如ia64 SAL接�[7]，�则PCIe规范�求使用增强
+型�置访问方法（ECAM）。主桥消耗ECAM内存地�空间并将内存访问转�为PCI�置访问。该规范
+定义了ECAM地�空间的布局和功能；�有地�空间的基础是特定于设备的。ACPI�作系统从��
+MCFG表或PNP0A03设备中的_CBA方法中了解基础地�。
+
+MCFG表必须�述�热�拔主桥的ECAM空间[8]。由于MCFG是一个��表，�能通过热�拔更新，
+PNP0A03设备中的_CBA方法�述了�热�拔主桥的ECAM空间[9]。请注�，对于MCFG和_CBA，
+基�总是对应于总线0，�使桥器下�的总线范围（通过_CRS报告）�从0开始。
+
+
+[1] ACPI 6.2, sec 6.1:
+    对于任何在�枚举类型的总线上的设备（例如，ISA总线），OSPM会枚举设备的标识符，ACPI
+    系统固件必须为�个设备�供一个_HID对象...以使OSPM能够�到这一点。
+
+[2] ACPI 6.2, sec 3.7:
+    �作系统枚举主�设备时，�需通过读�ACPI命�空间�寻找具有硬件ID的设备。
+
+    ACPI枚举的�个设备都包括ACPI命�空间中ACPI定义的对象，该对象报告设备�能�用的硬
+    件资�[_PRS]，报告设备当�使用的资�[_CRS]的对象，以��置这些资�的对象[_SRS]。
+    这些信�被���用�作系统（OSPM）用��置设备。
+
+[3] ACPI 6.2, sec 6.2:
+    OSPM使用设备�置对象��置通过ACPI列举的设备的硬件资�。设备�置对象�供了关于当�
+    和�能的资�需求的信�，共享资�之间的关系，以��置硬件资�的方法。
+
+    当OSPM枚举一个设备时，它调用_PRS�确定该设备的资�需求。它也�以调用_CRS�找到该设
+    备的当�资�设置。利用这些信�，���用系统决定设备应该消耗什么资�，并通过调用设备
+    的_SRS控制方法�设置这些资�。
+
+    在ACPI中，设备�以消耗资�（例如，传统的键盘），�供资�（例如，一个专有的PCI桥），
+    或者两者都�。除��有规定，设备的资�被�定为�自设备层次结构中设备上方最近的匹�资
+    �。
+
+[4] ACPI 6.2, sec 6.4.3.5.1, 2, 3, 4:
+    QWord/DWord/Word 地�空间�述符 (.1, .2, .3)
+      常规标志: Bit [0] 被忽略。
+
+    扩展地�空间�述符 (.4)
+      常规标志: Bit [0] 消费者/生产者:
+
+        * 1 – 这个设备消费这个资�
+        * 0 – 该设备生产和消费该资�
+
+[5] ACPI 6.2, sec 19.6.43:
+    ResourceUsage指定内存范围是由这个设备（ResourceConsumer）消费还是传递给�设备
+    （ResourceProducer）。如果没有指定，那么就�定是ResourceConsumer。
+
+[6] PCI Firmware 3.2, sec 4.1.2:
+    如果�作系统�能原生的懂得�留MMCFG区域，MMCFG区域必须由固件�留。在MCFG表中或通
+    过_CBA方法（�第4.1.3节）报告的地�范围必须通过声明主�资���留。对于大多数系统，
+    主�资�将出现在ACPI命�空间的根部（在_SB下），在一个节点的_HID为EISAID（PNP0C0
+    2），在这�情况下的资��应该�求在根PCI总线的_CRS。这些资��以选择在Int15 E820
+    或EFIGetMemoryMap中作为�留内存返回，但必须始终通过ACPI作为主�资�报告。
+
+[7] PCI Express 4.0, sec 7.2.2:
+    对于PC兼容的系统，或者没有实现�许访问�置空间的处�器架构特定固件接�标准的系统，需
+    �使用本节中定义的ECAM。
+
+[8] PCI Firmware 3.2, sec 4.1.2:
+    MCFG表是一个ACPI表，用于沟通的基础地�对应的�热的�移动的PCI段组范围内的PCI段组在
+    �动时�供给�作系统。这对PC兼容系统�说是必需的。
+
+    MCFG表仅用于沟通在�动时系统�用的PCI段组对应的基�。
+
+[9] PCI Firmware 3.2, sec 4.1.3:
+    _CBA (Memory mapped Configuration Base Address) 控制方法是一个�选的ACPI对
+    象，用于返回热�拔主桥的64�内存映射的�置基�。_CBA 返回的基�是与处�器相关的地�。
+    _CBA 控制方法被评估为一个整数。
+
+    这个控制方法出现在主桥对象下。当_CBA方法出现在一个活动的主桥对象下时，�作系统会评
+    估这个结构，以确定内存映射的�置基�，对应于_CRS方法中指定的总线编�范围的PCI段组。
+    一个包�_CBA方法的ACPI命�空间对象也必须包�一个相应的_SEG方法。
diff --git a/Documentation/translations/zh_CN/PCI/index.rst b/Documentation/translations/zh_CN/PCI/index.rst
index 16acb2bd9b58..cbeb33c34a98 100644
--- a/Documentation/translations/zh_CN/PCI/index.rst
+++ b/Documentation/translations/zh_CN/PCI/index.rst
@@ -10,9 +10,6 @@
 :校译:
 
 
-
-.. _cn_PCI_index.rst:
-
 ===================
 Linux PCI总线�系统
 ===================
@@ -26,12 +23,12 @@ Linux PCI总线�系统
    pci-iov-howto
    msi-howto
    sysfs-pci
+   acpi-info
 
 
 Todolist:
 
-   acpi-info
-   pci-error-recovery
-   pcieaer-howto
-   endpoint/index
-   boot-interrupts
+* pci-error-recovery
+* pcieaer-howto
+* endpoint/index
+* boot-interrupts
diff --git a/Documentation/translations/zh_CN/admin-guide/README.rst b/Documentation/translations/zh_CN/admin-guide/README.rst
index d20949e8bf6f..e679cbc3c89d 100644
--- a/Documentation/translations/zh_CN/admin-guide/README.rst
+++ b/Documentation/translations/zh_CN/admin-guide/README.rst
@@ -6,10 +6,10 @@
 
  �想� Wu XiangCheng <bobwxc@email.cn>
 
-Linux内核5.x版本 <http://kernel.org/>
+Linux内核6.x版本 <http://kernel.org/>
 =========================================
 
-以下是Linux版本5的�行注记。仔细阅读它们，
+以下是Linux版本6的�行注记。仔细阅读它们，
 它们会告诉你这些都是什么，解释如何安装内核，以��到问题时该如何�。
 
 什么是Linux？
@@ -61,27 +61,27 @@ Linux内核5.x版本 <http://kernel.org/>
  - 如果您�安装完整的�代�，请把内核tar档案包放在您有��的目录中（例如您
    的主目录）并将其解包::
 
-     xz -cd linux-5.x.tar.xz | tar xvf -
+     xz -cd linux-6.x.tar.xz | tar xvf -
 
    将“X�替��最新内核的版本�。
 
    ���】使用 /usr/src/linux 目录�这里有一组库头文件使用的内核头文件
    （通常是�完整的）。它们应该与库匹�，而�是被内核的�化�得一团糟。
 
- - 您还�以通过打补�在5.x版本之间�级。补�以xz格�分�。�通过打补�进行
-   安装，请获�所有较新的补�文件，进入内核�代�（linux-5.x）的目录并
+ - 您还�以通过打补�在6.x版本之间�级。补�以xz格�分�。�通过打补�进行
+   安装，请获�所有较新的补�文件，进入内核�代�（linux-6.x）的目录并
    执行::
 
-     xz -cd ../patch-5.x.xz | patch -p1
+     xz -cd ../patch-6.x.xz | patch -p1
 
    请�按顺�】替�所有大于当��代�树版本的“x�，这样就�以了。您�能想�
    删除备份文件（文件�类似xxx~ 或 xxx.orig)，并确�没有失败的补�（文件�
    类似xxx# 或 xxx.rej）。如果有，�是你就是我犯了错误。
 
-   与5.x内核的补���，5.x.y内核（也称为稳定版内核）的补��是增�的，而是
-   直接应用于基本的5.x内核。例如，如果您的基本内核是5.0，并且希望应用5.0.3
-   补�，则�应先应用5.0.1和5.0.2的补�。类似地，如果您�行的是5.0.2内核，
-   并且希望跳转到5.0.3，那么在应用5.0.3补�之�，必须首先撤销5.0.2补�
+   与6.x内核的补���，6.x.y内核（也称为稳定版内核）的补��是增�的，而是
+   直接应用于基本的6.x内核。例如，如果您的基本内核是6.0，并且希望应用6.0.3
+   补�，则�应先应用6.0.1和6.0.2的补�。类似地，如果您�行的是6.0.2内核，
+   并且希望跳转到6.0.3，那么在应用6.0.3补�之�，必须首先撤销6.0.2补�
    （�patch -R）。更多关于这方�的内容，请阅读
    :ref:`Documentation/process/applying-patches.rst <applying_patches>` 。
 
@@ -103,7 +103,7 @@ Linux内核5.x版本 <http://kernel.org/>
 软件�求
 ---------
 
-   编译和�行5.x内核需���软件包的最新版本。请�考
+   编译和�行6.x内核需���软件包的最新版本。请�考
    :ref:`Documentation/process/changes.rst <changes>`
    �了解最低版本�求以�如何�级软件包。请注�，使用过旧版本的这些包�能会
    导致很难追踪的间接错误，因此��以为在生�或�作过程中出现明显问题时�以
@@ -116,12 +116,12 @@ Linux内核5.x版本 <http://kernel.org/>
    ``make O=output/dir`` 选项�以为输出文件（包括 .config）指定备用�置。
    例如::
 
-     kernel source code: /usr/src/linux-5.x
+     kernel source code: /usr/src/linux-6.x
      build directory:    /home/name/build/kernel
 
    ��置和构建内核，请使用::
 
-     cd /usr/src/linux-5.x
+     cd /usr/src/linux-6.x
      make O=/home/name/build/kernel menuconfig
      make O=/home/name/build/kernel
      sudo make O=/home/name/build/kernel modules_install install
@@ -227,8 +227,6 @@ Linux内核5.x版本 <http://kernel.org/>
  - 确�您至少有gcc 5.1�用。
    有关更多信�，请�阅 :ref:`Documentation/process/changes.rst <changes>` 。
 
-   请注�，您�然�以使用此内核�行a.out用户程�。
-
  - 执行 ``make`` �创建压缩内核映�。如果您安装了lilo以适�内核makefile，
    那么也�以进行 ``make install`` ，但是您�能需�先检查特定的lilo设置。
 
@@ -282,67 +280,12 @@ Linux内核5.x版本 <http://kernel.org/>
 若�到问题
 -----------
 
- - 如果您�现了一些�能由于内核缺陷所导致的问题，请检查MAINTAINERS（维护者）
-   文件看看是�有人与令您�到麻烦的内核部分相关。如果无人在此列出，那么第二
-   个最好的方案就是把它们�给我（torvalds@linux-foundation.org），也�能��
-   到任何其他相关的邮件列表或新闻组。
-
- - 在所有的缺陷报告中，�请】告诉我们您在说什么内核，如何�现问题，以�您的
-   设置是什么的（使用您的常识）。如果问题是新的，请告诉我；如果问题是旧的，
-   请�试告诉我您什么时候首次注�到它。
-
- - 如果缺陷导致如下消�::
-
-     unable to handle kernel paging request at address C0000010
-     Oops: 0002
-     EIP:   0010:XXXXXXXX
-     eax: xxxxxxxx   ebx: xxxxxxxx   ecx: xxxxxxxx   edx: xxxxxxxx
-     esi: xxxxxxxx   edi: xxxxxxxx   ebp: xxxxxxxx
-     ds: xxxx  es: xxxx  fs: xxxx  gs: xxxx
-     Pid: xx, process nr: xx
-     xx xx xx xx xx xx xx xx xx xx
-
-   或者类似的内核调试信�显示在�幕上或在系统日志里，请�如实】�制它。
-   �能对你�说转储（dump）看起����解，但它确实包��能有助于调试问题的
-   信�。转储上方的文本也很��：它说明了内核转储代�的原因（在上�的示例中，
-   是由于内核指针错误）。更多关于如何�解转储的信�，请��
-   Documentation/admin-guide/bug-hunting.rst。
-
- - 如果使用 CONFIG_KALLSYMS 编译内核，则�以按原样��转储，�则必须使用
-   ``ksymoops`` 程���解转储（但通常首选使用CONFIG_KALLSYMS编译）。
-   此实用程��从
-   https://www.kernel.org/pub/linux/utils/kernel/ksymoops/ 下载。
-   或者，您�以手动执行转储查找：
-
- - 在调试�上�这样的转储时，如果您�以查找EIP值的�义，这将�常有帮助。
-   �六进制值本身对我或其他任何人都没有太大帮助：它会�决于特定的内核设置。
-   您应该�的是从EIP行获��六进制值（忽略 ``0010:`` ），然�在内核�字列表
-   中查找它，以查看哪个内核函数包�有问题的地�。
-
-   �找到内核函数�，您需�找到与显示症状的内核相关�的系统二进制文件。就是
-   文件“linux/vmlinux�。����字列表并将其与内核崩溃中的EIP进行匹�，
-   请执行::
-
-     nm vmlinux | sort | less
-
-   这将为您�供一个按��排�的内核地�列表，从中很容易找到包�有问题的地�
-   的函数。请注�，内核调试消��供的地��一定与函数地�完全匹�（事实上，
-   这是��能的），因此您�能�“grep�列表：�过列表将为您�供�个内核函数
-   的起点，因此通过查找起始地�低于你正在�索的地�，但�一个函数的高于的
-   函数，你会找到您想�的。实际上，在您的问题报告中加入一些“上下文��能是
-   一个好主�，给出相关的上下几行。
-
-   如果您由于�些原因无法完�上述�作（如您使用预编译的内核映�或类似的映�），
-   请尽�能多地告诉我您的相关设置信�，这会有所帮助。有关详细信�请阅读
-   ‘Documentation/admin-guide/reporting-issues.rst’。
-
- - 或者，您�以在正在�行的内核上使用gdb（�读的；��能更改值或设置断点）。
-   为此，请首先使用-g编译内核；适当地编辑arch/x86/Makefile，然�执行 ``make
-   clean`` 。您还需��用CONFIG_PROC_FS（通过 ``make config`` ）。
-
-   使用新内核�新�动�，执行 ``gdb vmlinux /proc/kcore`` 。现在�以使用所有
-   普通的gdb命令。查找系统崩溃点的命令是 ``l *0xXXXXXXXX`` （将xxx替�为EIP
-   值）。
-
-   用gdb无法调试一个当�未�行的内核是由于gdb（错误地）忽略了编译内核的起始
-   �移�。
+如果您�现了一些�能由于内核缺陷所导致的问题，请�阅：
+Documentation/translations/zh_CN/admin-guide/reporting-issues.rst 。
+
+想��解内核错误报告，请�阅：
+Documentation/translations/zh_CN/admin-guide/bug-hunting.rst 。
+
+更多用GDB调试内核的信�，请�阅：
+Documentation/translations/zh_CN/dev-tools/gdb-kernel-debugging.rst
+和 Documentation/dev-tools/kgdb.rst 。
diff --git a/Documentation/translations/zh_CN/admin-guide/bootconfig.rst b/Documentation/translations/zh_CN/admin-guide/bootconfig.rst
new file mode 100644
index 000000000000..072d17f5f199
--- /dev/null
+++ b/Documentation/translations/zh_CN/admin-guide/bootconfig.rst
@@ -0,0 +1,293 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/admin-guide/bootconfig.rst
+
+:译者: �想� Wu XiangCheng <bobwxc@email.cn>
+
+========
+引导�置
+========
+
+:作者: Masami Hiramatsu <mhiramat@kernel.org>
+
+概述
+====
+
+引导�置扩展了现有的内核命令行，以一�更有效率的方�在引导内核时进一步支�
+键值数�。这�许管�员传递一份结构化关键字的�置文件。
+
+�置文件语法
+============
+
+引导�置文件的语法采用�常简�的键值结构。�个关键字由点连接的��组�，键
+和值由 ``=`` 连接。值以分�（ ``;`` ）或�行符（ ``\n`` ）结尾。数组值中�
+个元素由逗�（ ``,`` ）分隔。::
+
+  KEY[.WORD[...]] = VALUE[, VALUE2[...]][;]
+
+与内核命令行语法��，逗�和 ``=`` 周围�许有空格。
+
+关键字��许包�字��数字�连字符（ ``-`` ）和下划线（ ``_`` ）。值�包�
+�打�字符和空格，但分�（ ``;`` ）��行符（ ``\n`` ）�逗�（ ``,`` ）�
+井�（ ``#`` ）和�大括�（ ``}`` ）等分隔符除外。
+
+如果你需�在值中使用这些分隔符，�以用�引�（ ``"VALUE"`` ）或�引�
+（ ``'VALUE'`` ）括起�。注�，引�无法转义。
+
+键的值�以为空或�存在。这些键用于检查该键是�存在（类似布尔值）。
+
+键值语法
+--------
+
+引导�置文件语法�许用户通过大括��并键�部分相�的关键字。例如::
+
+ foo.bar.baz = value1
+ foo.bar.qux.quux = value2
+
+也�以写�::
+
+ foo.bar {
+    baz = value1
+    qux.quux = value2
+ }
+
+或者更紧凑一些，写�::
+
+ foo.bar { baz = value1; qux.quux = value2 }
+
+在这两�样�中，引导解�时相�的关键字都会自动�并。因此�以追加类似的树或
+键值。
+
+相�关键字的值
+--------------
+
+�止两个或多个值或数组共享�一个关键字。例如::
+
+ foo = bar, baz
+ foo = qux  # !错误! 我们��以�定义相�的关键字
+
+如果你想�更新值，必须显�使用覆盖�作符 ``:=`` 。例如::
+
+ foo = bar, baz
+ foo := qux
+
+这样 ``foo`` 关键字的值就��了 ``qux`` 。这对于通过添加（部分）自定义引导
+�置�覆盖默认值�常有用，�于解�默认引导�置。
+
+如果你想对现有关键字追加值作为数组�员，�以使用 ``+=`` �作符。例如::
+
+ foo = bar, baz
+ foo += qux
+
+这样， ``foo`` 关键字就�时拥有了 ``bar`` ， ``baz`` 和 ``qux`` 。
+
+此外，父关键字下��时存在值和�关键字。
+例如，下列�置是�行的。::
+
+ foo = value1
+ foo.bar = value2
+ foo := value3 # 这会更新foo的值。
+
+注�，裸值�能直接放进结构化关键字中，必须在大括�外定义它。例如::
+
+ foo {
+     bar = value1
+     bar {
+         baz = value2
+         qux = value3
+     }
+ }
+
+�时，关键字下值节点的顺�是固定的。如果值和�关键字�时存在，值永远是该关
+键字的第一个�节点。因此如果用户先指定�关键字，如::
+
+ foo.bar = value1
+ foo = value2
+
+则在程�（和/proc/bootconfig）中，它会按如下显示::
+
+ foo = value2
+ foo.bar = value1
+
+注释
+----
+
+�置语法接�shell脚本风格的注释。注释以井�（ ``#`` ）开始，到�行符
+（ ``\n`` ）结�。
+
+::
+
+ # comment line
+ foo = value # value is set to foo.
+ bar = 1, # 1st element
+       2, # 2nd element
+       3  # 3rd element
+
+会被解�为::
+
+ foo = value
+ bar = 1, 2, 3
+
+注�你�能把注释放在值和分隔符（ ``,`` 或 ``;`` ）之间。如下�置语法是错误的::
+
+ key = 1 # comment
+       ,2
+
+
+/proc/bootconfig
+================
+
+/proc/bootconfig是引导�置的用户空间接�。与/proc/cmdline��，此文件内容以
+键值列表样�显示。
+�个键值对一行，样�如下::
+
+ KEY[.WORDS...] = "[VALUE]"[,"VALUE2"...]
+
+
+用引导�置引导内核
+==================
+
+用引导�置引导内核有两�方法：将引导�置附加到initrd镜�或直接嵌入内核中。
+
+*initrd: initial RAM disk，�始内存�盘*
+
+将引导�置附加到initrd
+----------------------
+
+由于默认情况下引导�置文件是用initrd加载的，因此它将被添加到initrd（initramfs）
+镜�文件的末尾，其中包�填充�大��校验值和12字节幻数，如下所示::
+
+ [initrd][bootconfig][padding][size(le32)][checksum(le32)][#BOOTCONFIG\n]
+
+大�和校验值为�端�存放的32�无符�值。
+
+当引导�置被加到initrd镜�时，整个文件大�会对�到4字节。空字符（ ``\0`` ）
+会填补对�空隙。因此 ``size`` 就是引导�置文件的长度+填充的字节。
+
+Linux内核在内存中解�initrd镜�的最�部分以获�引导�置数�。由于这�“背负��
+的方法，��引导加载器传递了正确的initrd文件大�，就无需更改或更新引导加载器
+和内核镜�本身。如果引导加载器�外传递了更长的大�，内核将无法找到引导�置数
+�。
+
+Linux内核在tools/bootconfig下�供了 ``bootconfig`` 命令�完�此�作，管�员
+�以用它从initrd镜�中删除或追加�置文件。你�以用以下命令�构建它::
+
+ # make -C tools/bootconfig
+
+��initrd镜�添加你的引导�置文件，请按如下命令�作（旧数�会自动移除）::
+
+ # tools/bootconfig/bootconfig -a your-config /boot/initrd.img-X.Y.Z
+
+�从镜�中移除�置，�以使用-d选项::
+
+ # tools/bootconfig/bootconfig -d /boot/initrd.img-X.Y.Z
+
+然�在内核命令行上添加 ``bootconfig`` 告诉内核去initrd文件末尾寻找内核�置。
+
+将引导�置嵌入内核
+------------------
+
+如果你�能使用initrd，也�以通过Kconfig选项将引导�置文件嵌入内核中。在此情
+况下，你需�用以下选项�新编译内核::
+
+ CONFIG_BOOT_CONFIG_EMBED=y
+ CONFIG_BOOT_CONFIG_EMBED_FILE="/引导�置/文件/的/路径"
+
+``CONFIG_BOOT_CONFIG_EMBED_FILE`` 需�从��树或对象树开始的引导�置文件的
+�对/相对路径。内核会将其嵌入作为默认引导�置。
+
+与将引导�置附加到initrd一样，你也需�在内核命令行上添加 ``bootconfig`` 告诉
+内核去�用内嵌的引导�置。
+
+注�，�使你已�设置了此选项，��用附加到initrd的其他引导�置覆盖内嵌的引导
+�置。
+
+通过引导�置传递内核�数
+========================
+
+除了内核命令行，引导�置也�以用于传递内核�数。所有 ``kernel`` 关键字下的键
+值对都将直接传递给内核命令行。此外， ``init`` 下的键值对将通过命令行传递给
+init进程。�数按以下顺�与用户给定的内核命令行字符串相连，因此命令行�数�以
+覆盖引导�置�数（这�决于�系统如何处��数，但通常��的�数将被��的�数
+覆盖）::
+
+ [bootconfig params][cmdline params] -- [bootconfig init params][cmdline init params]
+
+如果引导�置文件给出的kernel/init�数是::
+
+ kernel {
+   root = 01234567-89ab-cdef-0123-456789abcd
+ }
+ init {
+  splash
+ }
+
+这将被�制到内核命令行字符串中，如下所示::
+
+ root="01234567-89ab-cdef-0123-456789abcd" -- splash
+
+如果用户给出的其他命令行是::
+
+ ro bootconfig -- quiet
+
+则最�的内核命令行如下::
+
+ root="01234567-89ab-cdef-0123-456789abcd" ro bootconfig -- splash quiet
+
+
+�置文件的�制
+==============
+
+当�最大的�置大�是32KB，关键字总数（�是键值�目）必须少于1024个节点。
+注�：这�是�目数而是节点数，�目必须消耗超过2个节点（一个关键字和一个值）。
+所以从�论上讲最多512个键值对。如果关键字平�包�3个��，则�有256个键值对。
+在大多数情况下，�置项的数�将少于100个�目，�于8KB，因此这应该足够了。如果
+节点数超过1024，解�器将返回错误，�使文件大��于32KB。（请注�，此最大尺寸
+�包括填充的空字符。）
+无论如何，因为 ``bootconfig`` 命令在附加�动�置到initrd映�时会验�它，用户
+�以在引导之�注�到它。
+
+
+引导�置API
+===========
+
+用户�以查询或�历键值对，也�以查找（�缀）根关键字节点，并在查找该节点下的
+键值。
+
+如果您有一个关键字字符串，则�以直接使用 xbc_find_value() 查询该键的值。如果
+你想知�引导�置里有哪些关键字，�以使用 xbc_for_each_key_value() 迭代键值对。
+请注�，您需�使用 xbc_array_for_each_value() 访问数组的值，例如::
+
+ vnode = NULL;
+ xbc_find_value("key.word", &vnode);
+ if (vnode && xbc_node_is_array(vnode))
+    xbc_array_for_each_value(vnode, value) {
+      printk("%s ", value);
+    }
+
+如果您想查找具有�缀字符串的键，�以使用 xbc_find_node() 通过�缀字符串查找
+节点，然�用 xbc_node_for_each_key_value() 迭代�缀节点下的键。
+
+但最典型的用法是获��缀下的命�值或�缀下的命�数组，例如::
+
+ root = xbc_find_node("key.prefix");
+ value = xbc_node_find_value(root, "option", &vnode);
+ ...
+ xbc_node_for_each_array_value(root, "array-option", value, anode) {
+    ...
+ }
+
+这将访问值“key.prefix.option�的值和“key.prefix.array-option�的数组。
+
+�是�需�的，因为在�始化之��置�读。如果需�修改，必须�制所有数�和关键字。
+
+
+函数与结构体
+============
+
+相关定义的kernel-doc��：
+
+ - include/linux/bootconfig.h
+ - lib/bootconfig.c
diff --git a/Documentation/translations/zh_CN/admin-guide/index.rst b/Documentation/translations/zh_CN/admin-guide/index.rst
index 2f6970d0a032..ac2960da33e6 100644
--- a/Documentation/translations/zh_CN/admin-guide/index.rst
+++ b/Documentation/translations/zh_CN/admin-guide/index.rst
@@ -63,6 +63,7 @@ Todolist:
 .. toctree::
    :maxdepth: 1
 
+   bootconfig
    clearing-warn-once
    cpu-load
    cputopology
@@ -80,7 +81,6 @@ Todolist:
 *   binderfs
 *   binfmt-misc
 *   blockdev/index
-*   bootconfig
 *   braille-console
 *   btmrvl
 *   cgroup-v1/index
diff --git a/Documentation/translations/zh_CN/core-api/circular-buffers.rst b/Documentation/translations/zh_CN/core-api/circular-buffers.rst
new file mode 100644
index 000000000000..694ad8e61070
--- /dev/null
+++ b/Documentation/translations/zh_CN/core-api/circular-buffers.rst
@@ -0,0 +1,210 @@
+.. SPDX-License-Identifier: GPL-2.0+
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/core-api/circular-buffers.rst
+
+:翻译:
+
+ 周彬彬 Binbin Zhou <zhoubinbin@loongson.cn>
+
+:校译:
+
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+ �想� Wu Xiangcheng <bobwxc@email.cn>
+ 时奎亮 Alex Shi <alexs@kernel.org>
+
+==========
+环形缓冲区
+==========
+
+:作者: David Howells <dhowells@redhat.com>
+:作者: Paul E. McKenney <paulmck@linux.ibm.com>
+
+
+Linux �供了许多�用于实现循环缓冲的特性。有两组这样的特性：
+
+ (1) 用于确定2次方大�的缓冲区信�的便利函数。
+
+ (2) �以代替缓冲区中对象的生产者和消费者共享�的内存�障。
+
+如下所述，�使用这些设施，�需�一个生产者和一个消费者。�以通过�列化�处�多个
+生产者，并通过�列化�处�多个消费者。
+
+.. Contents:
+
+ (*) 什么是环形缓冲区？
+
+ (*) 测�2次幂缓冲区
+
+ (*) 内存�障与环形缓冲区的结�使用
+     - 生产者
+     - 消费者
+
+ (*) 延伸阅读
+
+
+
+什么是环形缓冲区？
+==================
+
+首先，什么是环形缓冲区？环形缓冲区是具有固定的有�大�的缓冲区，它有两个索引：
+
+ (1) 'head'索引 - 生产者将元素�入缓冲区的�置。
+
+ (2) 'tail'索引 - 消费者在缓冲区中找到下一个元素的�置。
+
+通常，当tail指针等于head指针时，表明缓冲区是空的；而当head指针比tail指针少一个时，
+表明缓冲区是满的。
+
+添加元素时，递增head索引；删除元素时，递增tail索引。tail索引�应该跳过head索引，
+两个索引在到达缓冲区末端时都应该被赋值为0，从而�许海�的数��过缓冲区。
+
+通常情况下，元素都有相�的�元大�，但这并�是使用以下技术的严格�求。如果�在缓
+冲区中包�多个元素或��大�的元素，则索引�以增加超过1，��是两个索引都没有超过
+�一个。然而，实现者必须�心，因为超过一个��大�的区域�能会覆盖缓冲区的末端并
+且缓冲区会被分�两段。
+
+测�2次幂缓冲区
+===============
+
+计算任�大�的环形缓冲区的�用或剩余容�通常是一个费时的�作，需�使用模（除法）
+指令。但是如果缓冲区的大�为2次幂，则�以使用更快的按�与指令代替。
+
+Linux�供了一组用于处�2次幂环形缓冲区的�。�以通过以下方�使用::
+
+	#include <linux/circ_buf.h>
+
+这些�包括：
+
+ (#) 测�缓冲区的剩余容�::
+
+	CIRC_SPACE(head_index, tail_index, buffer_size);
+
+     返回缓冲区[1]中��入元素的剩余空间大�。
+
+
+ (#) 测�缓冲区中的最大连续立��用空间::
+
+	CIRC_SPACE_TO_END(head_index, tail_index, buffer_size);
+
+     返回缓冲区[1]中剩余的连续空间的大�，元素�以立��入其中，而�必绕回到缓冲
+     区的开头。
+
+
+ (#) 测�缓冲区的使用数::
+
+	CIRC_CNT(head_index, tail_index, buffer_size);
+
+     返回当��用缓冲区[2]的元素数�。
+
+
+ (#) 测�缓冲区的连续使用数::
+
+	CIRC_CNT_TO_END(head_index, tail_index, buffer_size);
+
+     返回�以从缓冲区中��的连续元素[2]的数�，而�必绕回到缓冲区的开头。
+
+这里的�一个��义上都会返回一个介于0和buffer_size-1之间的值，但是：
+
+ (1) CIRC_SPACE*()是为了在生产者中使用。对生产者�说，它们将返回一个下�，因为生
+     产者控制�head索引，但消费者�能�然在�一个CPU上耗尽缓冲区并移动tail索引。
+
+     对消费者�说，它将显示一个上�，因为生产者�能正忙于耗尽空间。
+
+ (2) CIRC_CNT*()是为了在消费者中使用。对消费者�说，它们将返回一个下�，因为消费
+     者控制�tail索引，但生产者�能�然在�一个CPU上填充缓冲区并移动head索引。
+
+     对于生产者，它将显示一个上�，因为消费者�能正忙于清空缓冲区。
+
+ (3) 对于第三方�说，生产者和消费者对索引的写入顺�是无法��的，因为它们是独立的，
+     而且�能是在��的CPU上进行的，所以在这�情况下的结果�是一�猜测，甚至�能
+     是错误的。
+
+内存�障与环形缓冲区的结�使用
+==============================
+
+通过将内存�障与环形缓冲区结�使用，�以��以下需求：
+
+ (1) 使用�个��控制对缓冲区两端的访问，从而�许�时填充和清空缓冲区；以�
+
+ (2) 使用原�计数器�作。
+
+这有两个方�：填充缓冲区的生产者和清空缓冲区的消费者。在任何时候，�应有一个生产
+者在填充缓冲区，�样的也�应有一个消费者在清空缓冲区，但�方�以�时�作。
+
+
+生产者
+------
+
+生产者看起��这样::
+
+	spin_lock(&producer_lock);
+
+	unsigned long head = buffer->head;
+	/* spin_unlock()和下一个spin_lock()�供必�的排�。 */
+	unsigned long tail = READ_ONCE(buffer->tail);
+
+	if (CIRC_SPACE(head, tail, buffer->size) >= 1) {
+		/* 添加一个元素到缓冲区 */
+		struct item *item = buffer[head];
+
+		produce_item(item);
+
+		smp_store_release(buffer->head,
+				  (head + 1) & (buffer->size - 1));
+
+		/* wake_up()将确�在唤醒任何人之��交head */
+		wake_up(consumer);
+	}
+
+	spin_unlock(&producer_lock);
+
+这将表明CPU必须在head索引使其对消费者�用之�写入新项目的内容，�时CPU必须在唤醒
+消费者之�写入修改�的head索引。
+
+请注�，wake_up()并���任何形�的�障，除�确实唤醒了�些东西。因此我们�能��
+它�进行排�。但是数组中始终有一个元素留空，因此生产者必须产生两个元素，然���
+能破�消费者当�正在读�的元素。�时，消费者连续调用之间�对的解�-加��供了索引
+读�（指示消费者已清空给定元素）和生产者对该相�元素的写入之间的必�顺�。
+
+
+消费者
+------
+
+消费者看起��这样::
+
+	spin_lock(&consumer_lock);
+
+	/* 读�该索引处的内容之�，先读�索引 */
+	unsigned long head = smp_load_acquire(buffer->head);
+	unsigned long tail = buffer->tail;
+
+	if (CIRC_CNT(head, tail, buffer->size) >= 1) {
+
+		/* 从缓冲区中��一个元素 */
+		struct item *item = buffer[tail];
+
+		consume_item(item);
+
+		/* 在递增tail之�完�对�述符的读�。 */
+		smp_store_release(buffer->tail,
+				  (tail + 1) & (buffer->size - 1));
+	}
+
+	spin_unlock(&consumer_lock);
+
+这表明CPU在读�新元素之�确�索引是最新的，然�在写入新的尾指针之�应确�CPU已完
+�读�该元素，这将擦除该元素。
+
+请注�，使用READ_ONCE()和smp_load_acquire()�读���（head）索引。这�以防止编译
+器丢弃并�新加载其缓存值。如果您能确定��（head）索引将仅使用一次，则这�是必须
+的。smp_load_acquire()还�以强制CPU对�续的内存引用进行排�。类似地，两�算法都使
+用smp_store_release()�写入线程的索引。这记录了我们正在写入�以并�读�的内容的事
+实，以防止编译器破�存储，并强制对以�的访问进行排�。
+
+
+延伸阅读
+========
+
+关于Linux的内存�障设施的�述，请查看Documentation/memory-barriers.txt。
diff --git a/Documentation/translations/zh_CN/core-api/generic-radix-tree.rst b/Documentation/translations/zh_CN/core-api/generic-radix-tree.rst
new file mode 100644
index 000000000000..eacd1d2ebddc
--- /dev/null
+++ b/Documentation/translations/zh_CN/core-api/generic-radix-tree.rst
@@ -0,0 +1,23 @@
+.. SPDX-License-Identifier: GPL-2.0+
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/core-api/generic-radix-tree.rst
+
+:翻译:
+
+ 周彬彬 Binbin Zhou <zhoubinbin@loongson.cn>
+
+===================
+通用基数树/稀�数组
+===================
+
+通用基数树/稀�数组的相关内容请�include/linux/generic-radix-tree.h文件中的
+“DOC: Generic radix trees/sparse arrays�。
+
+通用基数树函数
+--------------
+
+该API在以下内核代�中:
+
+include/linux/generic-radix-tree.h
diff --git a/Documentation/translations/zh_CN/core-api/idr.rst b/Documentation/translations/zh_CN/core-api/idr.rst
new file mode 100644
index 000000000000..97a16e76b81b
--- /dev/null
+++ b/Documentation/translations/zh_CN/core-api/idr.rst
@@ -0,0 +1,80 @@
+.. SPDX-License-Identifier: GPL-2.0+
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/core-api/idr.rst
+
+:翻译:
+
+ 周彬彬 Binbin Zhou <zhoubinbin@loongson.cn>
+
+:校译:
+
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+ �想� Wu Xiangcheng <bobwxc@email.cn>
+ 时奎亮 Alex Shi <alexs@kernel.org>
+
+======
+ID分�
+======
+
+:作者: Matthew Wilcox
+
+概述
+====
+
+�解决的一个常�问题是分�标识符（IDs）；它通常是标识事物的数字。比如包括文件�述
+符�进程ID�网络�议中的数�包标识符�SCSI标记和设备实例编�。IDR和IDA为这个问题
+�供了一个��的解决方案，以���个人都自创。IDR�供将ID映射到指针的能力，而IDA
+仅�供ID分�，因此内存效率更高。
+
+IDR接�已�被废弃，请使用 ``XArray`` 。
+
+IDR的用法
+=========
+
+首先�始化一个IDR，对于��分�的IDR使用DEFINE_IDR()，或者对于动�分�的IDR使用
+idr_init()。
+
+您�以调用idr_alloc()�分�一个未使用的ID。通过调用idr_find()查询与该ID相关的指针，
+并通过调用idr_remove()释放该ID。
+
+如果需�更改与一个ID相关�的指针，�以调用idr_replace()。这样�的一个常�原因是通
+过将 ``NULL`` 指针传递给分�函数��留ID；用�留的ID�始化对象，最�将�始化的对
+象�入IDR。
+
+一些用户需�分�大于 ``INT_MAX`` 的ID。到目�为止，所有这些用户都满足 ``UINT_MAX``
+的�制，他们使用idr_alloc_u32()。如果您需�超出u32的ID，我们将与您�作以满足您的
+需求。
+
+如果需�按顺�分�ID，�以使用idr_alloc_cyclic()。处�较大数�的ID时，IDR的效率会
+�低，所以使用这个函数会有一点代价。
+
+�对IDR使用的所有指针进行�作，您�以使用基于回调的idr_for_each()或迭代器样�的
+idr_for_each_entry()。您�能需�使用idr_for_each_entry_continue()�继续迭代。如果
+迭代器�符�您的需求，您也�以使用idr_get_next()。
+
+当使用完IDR�，您�以调用idr_destroy()�释放IDR�用的内存。这并�会释放IDR指�的
+对象；如果您想这样�，请使用其中一个迭代器�执行此�作。
+
+您�以使用idr_is_empty()�查看当�是�分�了任何ID。
+
+如果在从IDR分�一个新ID时需�带�，您�能需�传递一组�制性的GFP标志，但这�能导
+致IDR无法分�内存。为了解决该问题，您�以在获��之�调用idr_preload()，然�在分
+�之�调用idr_preload_end()。
+
+IDR�步的相关内容请�include/linux/idr.h文件中的“DOC: idr sync�。
+
+IDA的用法
+=========
+
+IDA的用法的相关内容请�lib/idr.c文件中的“DOC: IDA description�。
+
+函数和数�结构
+==============
+
+该API在以下内核代�中:
+
+include/linux/idr.h
+
+lib/idr.c
diff --git a/Documentation/translations/zh_CN/core-api/index.rst b/Documentation/translations/zh_CN/core-api/index.rst
index 8a94ad87465d..37756d240b5e 100644
--- a/Documentation/translations/zh_CN/core-api/index.rst
+++ b/Documentation/translations/zh_CN/core-api/index.rst
@@ -44,15 +44,15 @@
    assoc_array
    xarray
    rbtree
+   idr
+   circular-buffers
+   generic-radix-tree
+   packing
 
 Todolist:
 
 
 
-   idr
-   circular-buffers
-   generic-radix-tree
-   packing
    this_cpu_ops
    timekeeping
    errseq
diff --git a/Documentation/translations/zh_CN/core-api/packing.rst b/Documentation/translations/zh_CN/core-api/packing.rst
new file mode 100644
index 000000000000..c0aab3a349d0
--- /dev/null
+++ b/Documentation/translations/zh_CN/core-api/packing.rst
@@ -0,0 +1,160 @@
+.. SPDX-License-Identifier: GPL-2.0+
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/core-api/packing.rst
+
+:翻译:
+
+ 周彬彬 Binbin Zhou <zhoubinbin@loongson.cn>
+
+:校译:
+
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+ �想� Wu Xiangcheng <bobwxc@email.cn>
+ 时奎亮 Alex Shi <alexs@kernel.org>
+
+========================
+通用的�域打包和解包函数
+========================
+
+问题陈述
+--------
+
+使用硬件时，必须在几�与其交互的方法之间进行选择。
+
+�以将指针映射到在硬件设备的内存区上精心设计的结构体，并将其字段作为结构�员（�
+能声明为�域）访问。但是由于CPU和硬件设备之间潜在的字节顺��匹�，以这�方�编写
+代�会�低其�移�性。
+
+此外，必须密切注�将硬件文档中的寄存器定义转�为结构的�域索引。此外，一些硬件
+（通常是网络设备）倾�于以��任何��字边界（有时甚至是64�）的方�对其寄存器字
+段进行分组。这就造�了�得�在结构中定义寄存器字段的“高�和“低�部分的�便。
+
+结构域定义的更��的替代方法是通过移动适当数�的����所需的字段。但这�然�能
+防止字节顺��匹�，除�所有内存访问都是�字节执行的。此外，代�很容易�得�乱无
+章，�时�能会在所需的许多�移�作中丢失一些高层次的想法。
+
+许多驱动程�采用了�移的方法，然�试图用定制的���少�乱无章的东西，但更多的时
+候，这些�所采用的�径�旧妨�了代�真正的�移�性。
+
+解决方案
+--------
+
+该API涉�2个基本�作：
+
+  - 将一个CPU�使用的数字打包到内存缓冲区中（具有硬件约�/特殊性）。
+  - 将内存缓冲区（具有硬件约�/特殊性）解压缩为一个CPU�使用的数字。
+
+该API�供了对所述硬件约�和特殊性以�CPU字节�的抽象，因此这两者之间�能�匹�。
+
+这些API函数的基本�元是u64。从CPU的角度�看，�63总是�味�字节7的��移�7，尽管
+�是逻辑上的。问题是：我们将这个比特放在内存的什么�置？
+
+以下示例介�了打包u64字段的内存布局。打包缓冲区中的字节�移�始终默认为0，1...7。
+示例显示的是逻辑字节和�所在的�置。
+
+1. 通常情况下（无特殊性），我们会这样�：
+
+::
+
+  63 62 61 60 59 58 57 56 55 54 53 52 51 50 49 48 47 46 45 44 43 42 41 40 39 38 37 36 35 34 33 32
+  7                       6                       5                        4
+  31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10  9  8  7  6  5  4  3  2  1  0
+  3                       2                       1                        0
+
+也就是说，CPU�使用的u64的MSByte(7)�于内存�移�0处，而u64的LSByte(0)�于内存�移�7处。
+
+这对应于大多数人认为的“大端�，其中�i对应于数字2^i。这在代�注释中也称为“逻辑�符�。
+
+
+2. 如果设置了QUIRK_MSB_ON_THE_RIGHT，我们按如下方��作：
+
+::
+
+  56 57 58 59 60 61 62 63 48 49 50 51 52 53 54 55 40 41 42 43 44 45 46 47 32 33 34 35 36 37 38 39
+  7                       6                        5                       4
+  24 25 26 27 28 29 30 31 16 17 18 19 20 21 22 23  8  9 10 11 12 13 14 15  0  1  2  3  4  5  6  7
+  3                       2                        1                       0
+
+也就是说，QUIRK_MSB_ON_THE_RIGHT�会影�字节定�，但会�转字节内的��移�。
+
+
+3. 如果设置了QUIRK_LITTLE_ENDIAN，我们按如下方��作：
+
+::
+
+  39 38 37 36 35 34 33 32 47 46 45 44 43 42 41 40 55 54 53 52 51 50 49 48 63 62 61 60 59 58 57 56
+  4                       5                       6                       7
+  7  6  5  4  3  2  1  0  15 14 13 12 11 10  9  8 23 22 21 20 19 18 17 16 31 30 29 28 27 26 25 24
+  0                       1                       2                       3
+
+因此，QUIRK_LITTLE_ENDIAN�味�在内存区域内，�个4字节的字的�个字节都被放置在与
+该字的边界相比的镜��置。
+
+
+4. 如果设置了QUIRK_MSB_ON_THE_RIGHT和QUIRK_LITTLE_ENDIAN，我们这样�：
+
+::
+
+  32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
+  4                       5                       6                       7
+  0  1  2  3  4  5  6  7  8   9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
+  0                       1                       2                       3
+
+
+5. 如果�设置了QUIRK_LSW32_IS_FIRST，我们这样�：
+
+::
+
+  31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16 15 14 13 12 11 10  9  8  7  6  5  4  3  2  1  0
+  3                       2                       1                        0
+  63 62 61 60 59 58 57 56 55 54 53 52 51 50 49 48 47 46 45 44 43 42 41 40 39 38 37 36 35 34 33 32
+  7                       6                       5                        4
+
+在这�情况下，8字节内存区域解释如下：�4字节对应最���的4字节的字，�4字节对应
+更��的4字节的字。
+
+6. 如果设置了QUIRK_LSW32_IS_FIRST和QUIRK_MSB_ON_THE_RIGHT，我们这样�：
+
+::
+
+  24 25 26 27 28 29 30 31 16 17 18 19 20 21 22 23  8  9 10 11 12 13 14 15  0  1  2  3  4  5  6  7
+  3                       2                        1                       0
+  56 57 58 59 60 61 62 63 48 49 50 51 52 53 54 55 40 41 42 43 44 45 46 47 32 33 34 35 36 37 38 39
+  7                       6                        5                       4
+
+
+7. 如果设置了QUIRK_LSW32_IS_FIRST和QUIRK_LITTLE_ENDIAN，则如下所示：
+
+::
+
+  7  6  5  4  3  2  1  0  15 14 13 12 11 10  9  8 23 22 21 20 19 18 17 16 31 30 29 28 27 26 25 24
+  0                       1                       2                       3
+  39 38 37 36 35 34 33 32 47 46 45 44 43 42 41 40 55 54 53 52 51 50 49 48 63 62 61 60 59 58 57 56
+  4                       5                       6                       7
+
+
+8. 如果设置了QUIRK_LSW32_IS_FIRST，QUIRK_LITTLE_ENDIAN和QUIRK_MSB_ON_THE_RIGHT，
+   则如下所示：
+
+::
+
+  0  1  2  3  4  5  6  7  8   9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
+  0                       1                       2                       3
+  32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
+  4                       5                       6                       7
+
+
+我们总是认为我们的�移�好�没有特殊性，然�在访问内存区域之�翻译它们。
+
+预期用途
+--------
+
+选择使用该API的驱动程�首先需�确定上述3�quirk组�（共8�）中的哪一�与硬件文档
+中�述的相匹�。然�，他们应该�装packing()函数，创建一个新的xxx_packing()，使用
+适当的QUIRK_* one-hot �集��调用它。
+
+packing()函数返回一个int类型的错误�，以防止程�员使用�正确的API。这些错误预计�
+会在�行时�生，因此xxx_packing()返回void并简�地接�这些错误是��的。它�以选择
+转储栈或打�错误�述。
diff --git a/Documentation/translations/zh_CN/devicetree/changesets.rst b/Documentation/translations/zh_CN/devicetree/changesets.rst
new file mode 100644
index 000000000000..2ace05f3c377
--- /dev/null
+++ b/Documentation/translations/zh_CN/devicetree/changesets.rst
@@ -0,0 +1,37 @@
+.. SPDX-License-Identifier: GPL-2.0
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/Devicetree/changesets.rst
+
+:翻译:
+
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+
+:校译:
+
+
+============
+设备树�更集
+============
+
+设备树�更集是一�方法，它�许人们以这样一�方�在实时树中使用�化，��么使用全部的
+�化，�么�使用。如果在使用�更集的过程中�生错误，那么树将被回滚到之�的状�。一个
+�更集也�以在使用�被删除。
+
+当一个�更集被使用时，所有的改�在�出OF_RECONFIG通知器之�被一次性使用到树上。这是
+为了让接收者在收到通知时看到一个完整的�一致的树的状�。
+
+一个�化集的顺�如下。
+
+1. of_changeset_init() - �始化一个�更集。
+
+2. 一些DT树�化的调用，of_changeset_attach_node(), of_changeset_detach_node(),
+   of_changeset_add_property(), of_changeset_remove_property,
+   of_changeset_update_property()�准备一组�更。此时�会对活动树�任何�更。所有
+   的�更�作都记录在of_changeset的 `entries` 列表中。
+
+3. of_changeset_apply() - 将�更使用到树上。�么整个�更集被使用，�么如果有错误，
+   树会被��到之�的状�。核心通过�确�正确的顺�。如果需�的�，�以使用一个解�的
+   __of_changeset_apply版本。
+
+如果一个�功使用的�更集需�被删除，�以用of_changeset_revert()�完�。
diff --git a/Documentation/translations/zh_CN/devicetree/dynamic-resolution-notes.rst b/Documentation/translations/zh_CN/devicetree/dynamic-resolution-notes.rst
new file mode 100644
index 000000000000..115190341305
--- /dev/null
+++ b/Documentation/translations/zh_CN/devicetree/dynamic-resolution-notes.rst
@@ -0,0 +1,31 @@
+.. SPDX-License-Identifier: GPL-2.0
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/Devicetree/dynamic-resolution-notes.rst
+
+:翻译:
+
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+
+:校译:
+
+========================
+Devicetree动�解�器说明
+========================
+
+本文�述了内核内DeviceTree解�器的实现，它�于drivers/of/resolver.c中。
+
+解�器如何工作？
+----------------
+
+解�器被赋予一个任�的树作为输入，该树用适当的dtc选项编译，并有一个/plugin/标签。这就产
+生了适当的__fixups__和__local_fixups__节点。
+
+解�器�次通过以下步骤工作:
+
+1. 从实时树中获�最大的设备树phandle值 + 1.
+2. 调整树的所有本地 phandles，以解决这个�。
+3. 使用 __local__fixups__ 节点信�以相�的�调整所有本地引用。
+4. 对于__fixups__节点中的�个属性，找到它在实时树中引用的节点。这是用�标记该节点的标签。
+5. 检索fixup的目标的phandle。
+6. 对于属性中的�个fixup，找到节点:属性:�移的�置，并用phandle值替�它。
diff --git a/Documentation/translations/zh_CN/devicetree/index.rst b/Documentation/translations/zh_CN/devicetree/index.rst
index 3fc355fe0037..7451dbfdd3e5 100644
--- a/Documentation/translations/zh_CN/devicetree/index.rst
+++ b/Documentation/translations/zh_CN/devicetree/index.rst
@@ -24,21 +24,16 @@ Open Firmware 和 Devicetree
 
    usage-model
    of_unittest
-
-Todolist:
-
-*   kernel-api
+   kernel-api
 
 Devicetree Overlays
 ===================
 .. toctree::
    :maxdepth: 1
 
-Todolist:
-
-*   changesets
-*   dynamic-resolution-notes
-*   overlay-notes
+   changesets
+   dynamic-resolution-notes
+   overlay-notes
 
 Devicetree Bindings
 ===================
diff --git a/Documentation/translations/zh_CN/devicetree/kernel-api.rst b/Documentation/translations/zh_CN/devicetree/kernel-api.rst
new file mode 100644
index 000000000000..6aa3b685494e
--- /dev/null
+++ b/Documentation/translations/zh_CN/devicetree/kernel-api.rst
@@ -0,0 +1,58 @@
+.. SPDX-License-Identifier: GPL-2.0
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/Devicetree/kernel-api.rst
+
+:翻译:
+
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+
+:校译:
+
+
+=================
+内核中的设备树API
+=================
+
+核心函数
+--------
+
+该API在以下内核代�中:
+
+drivers/of/base.c
+
+include/linux/of.h
+
+drivers/of/property.c
+
+include/linux/of_graph.h
+
+drivers/of/address.c
+
+drivers/of/irq.c
+
+drivers/of/fdt.c
+
+驱动模型函数
+------------
+
+该API在以下内核代�中:
+
+include/linux/of_device.h
+
+drivers/of/device.c
+
+include/linux/of_platform.h
+
+drivers/of/platform.c
+
+覆盖和动�DT函数
+----------------
+
+该API在以下内核代�中:
+
+drivers/of/resolver.c
+
+drivers/of/dynamic.c
+
+drivers/of/overlay.c
diff --git a/Documentation/translations/zh_CN/devicetree/overlay-notes.rst b/Documentation/translations/zh_CN/devicetree/overlay-notes.rst
new file mode 100644
index 000000000000..1bd482cb0a1b
--- /dev/null
+++ b/Documentation/translations/zh_CN/devicetree/overlay-notes.rst
@@ -0,0 +1,140 @@
+.. SPDX-License-Identifier: GPL-2.0
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/Devicetree/overlay-notes.rst
+
+:翻译:
+
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+
+:校译:
+
+==============
+设备树覆盖说明
+==============
+
+本文档�述了drivers/of/overlay.c中的内核内设备树覆盖功能的实现，是
+Documentation/devicetree/dynamic-resolution-notes.rst[1]的�套文档。
+
+覆盖如何工作
+------------
+
+设备树覆盖的目的是修改内核的实时树，并使修改以�映�化的方�影�内核的状�。
+由于内核主�处�的是设备，任何新的设备节点如果导致一个活动的设备，就应该创建它，
+而如果设备节点被�用或被全部删除，�影�的设备应该被�消注册。
+
+让我们举个例�，我们有一个foo�，它的基本树形图如下::
+
+    ---- foo.dts ---------------------------------------------------------------
+	/* FOO平� */
+	/dts-v1/;
+	/ {
+		compatible = "corp,foo";
+
+		/* 共享的资� */
+		res: res {
+		};
+
+		/* 芯片上的外围设备 */
+		ocp: ocp {
+			/* 总是被实例化的外围设备 */
+			peripheral1 { ... };
+		};
+	};
+    ---- foo.dts ---------------------------------------------------------------
+
+覆盖bar.dts,
+::
+
+    ---- bar.dts - 按标签覆盖目标�置 ----------------------------
+	/dts-v1/;
+	/�件/;
+	&ocp {
+		/* bar外围 */
+		bar {
+			compatible = "corp,bar";
+			... /* ��属性和�节点 */
+		};
+	};
+    ---- bar.dts ---------------------------------------------------------------
+
+当加载（并按照[1]中�述的方�解决）时，应该产生foo+bar.dts::
+
+    ---- foo+bar.dts -----------------------------------------------------------
+	/* FOO平� + bar外围 */
+	/ {
+		compatible = "corp,foo";
+
+		/* 共享资� */
+		res: res {
+		};
+
+		/* 芯片上的外围设备 */
+		ocp: ocp {
+			/* 总是被实例化的外围设备 */
+			peripheral1 { ... };
+
+			/* bar外围 */
+			bar {
+				compatible = "corp,bar";
+				... /* ��属性和�节点 */
+			};
+		};
+	};
+    ---- foo+bar.dts -----------------------------------------------------------
+
+作为覆盖的结果，已�创建了一个新的设备节点（bar），因此将注册一个bar平�设备，
+如果加载了匹�的设备驱动程�，将按预期创建设备。
+
+如果基础DT�是用-@选项编译的，那么“&ocp�标签将�能用于将覆盖节点解�到基础
+DT中的适当�置。在这�情况下，�以�供目标路径。通过标签的目标�置的语法是比
+较好的，因为�管标签在DT中出现在哪里，覆盖都�以被应用到任何包�标签的基础DT上。
+
+上�的bar.dts例�被修改为使用目标路径语法，�为::
+
+    ---- bar.dts - 通过明确的路径覆盖目标�置 --------------------
+	/dts-v1/;
+	/�件/;
+	&{/ocp} {
+		/* bar外围 */
+		bar {
+			compatible = "corp,bar";
+			... /* ��外围设备和�节点 */
+		}
+	};
+    ---- bar.dts ---------------------------------------------------------------
+
+
+内核中关于覆盖的API
+-------------------
+
+该API相当容易使用。
+
+1) 调用of_overlay_fdt_apply()�创建和应用一个覆盖的�更集。返回值是一个
+   错误或一个识别这个覆盖的cookie。
+
+2) 调用of_overlay_remove()�删除和清�先�通过调用of_overlay_fdt_apply()
+   而创建的覆盖�更集。��许删除一个被�一个覆盖的覆盖�化集。
+
+最�，如果你需�一次性删除所有的覆盖，�需调用of_overlay_remove_all()，
+它将以正确的顺�删除�一个覆盖。
+
+你�以选择注册在覆盖�作中被调用的通知器。详�
+of_overlay_notifier_register/unregister和enum of_overlay_notify_action。
+
+OF_OVERLAY_PRE_APPLY�OF_OVERLAY_POST_APPLY或OF_OVERLAY_PRE_REMOVE
+的通知器回调�以存储指�覆盖层中的设备树节点或其内容的指针，但这些指针�能�
+续到OF_OVERLAY_POST_REMOVE的通知器回调。在OF_OVERLAY_POST_REMOVE通
+知器被调用�，包�覆盖层的内存将被kfree()ed。请注�，�使OF_OVERLAY_POST_REMOVE
+的通知器返回错误，内存也会被kfree()ed。
+
+drivers/of/dynamic.c中的�更集通知器是第二�类型的通知器，�以通过应用或移除
+覆盖层�触�。这些通知器��许在覆盖层或其内容中存储指�设备树节点的指针。当包�
+覆盖层的内存因移除覆盖层而被释放时，覆盖层代�并�能防止这类指针�然有效。
+
+任何其他�留指�覆盖层节点或数�的指针的代�都被认为是一个错误，因为在移除覆盖层
+�，该指针将指�已释放的内存。
+
+覆盖层的用户必须特别注�系统上�生的整体�作，以确�其他内核代���留任何指�覆
+盖层节点或数�的指针。任何无�中使用这�指针的例�是，如果一个驱动或�系统模�在
+应用了覆盖�被加载，并且该驱动或�系统扫�了整个设备树或其大部分，包括覆盖节点。
diff --git a/Documentation/translations/zh_CN/driver-api/gpio/index.rst b/Documentation/translations/zh_CN/driver-api/gpio/index.rst
new file mode 100644
index 000000000000..9ab64e94aced
--- /dev/null
+++ b/Documentation/translations/zh_CN/driver-api/gpio/index.rst
@@ -0,0 +1,69 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../../disclaimer-zh_CN.rst
+
+:Original: Documentation/driver-api/gpio/index.rst
+
+:翻译:
+
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+
+:校译:
+
+=======================
+通用型输入/输出（GPIO）
+=======================
+
+目录:
+
+.. toctree::
+   :maxdepth: 2
+
+   legacy
+
+Todolist:
+
+*   intro
+*   using-gpio
+*   driver
+*   consumer
+*   board
+*   drivers-on-gpio
+*   bt8xxgpio
+
+核心
+====
+
+该API在以下内核代�中:
+
+include/linux/gpio/driver.h
+
+drivers/gpio/gpiolib.c
+
+ACPI支�
+========
+
+该API在以下内核代�中:
+
+drivers/gpio/gpiolib-acpi.c
+
+设备树支�
+==========
+
+该API在以下内核代�中:
+
+drivers/gpio/gpiolib-of.c
+
+设备管�支�
+============
+
+该API在以下内核代�中:
+
+drivers/gpio/gpiolib-devres.c
+
+sysfs帮助（函数）
+=================
+
+该API在以下内核代�中:
+
+drivers/gpio/gpiolib-sysfs.c
diff --git a/Documentation/translations/zh_CN/gpio.txt b/Documentation/translations/zh_CN/driver-api/gpio/legacy.rst
index a23ee14fc927..6399521d0548 100644
--- a/Documentation/translations/zh_CN/gpio.txt
+++ b/Documentation/translations/zh_CN/driver-api/gpio/legacy.rst
@@ -1,39 +1,28 @@
-Chinese translated version of Documentation/admin-guide/gpio
+.. SPDX-License-Identifier: GPL-2.0
 
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
+.. include:: ../../disclaimer-zh_CN.rst
 
-Maintainer: Grant Likely <grant.likely@secretlab.ca>
-		Linus Walleij <linus.walleij@linaro.org>
-Chinese maintainer: Fu Wei <tekkamanninja@gmail.com>
----------------------------------------------------------------------
-Documentation/admin-guide/gpio 的中文翻译
+:Original: Documentation/driver-api/gpio/legacy.rst
 
-如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
-交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
-英文版维护者： Grant Likely <grant.likely@secretlab.ca>
-		Linus Walleij <linus.walleij@linaro.org>
-中文版维护者： 傅炜 Fu Wei <tekkamanninja@gmail.com>
-中文版翻译者： 傅炜 Fu Wei <tekkamanninja@gmail.com>
-中文版校译者： 傅炜 Fu Wei <tekkamanninja@gmail.com>
+:翻译:
 
+ 傅炜 Fu Wei <tekkamanninja@gmail.com>
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
 
-以下为正文
----------------------------------------------------------------------
-GPIO 接�
+:校译:
 
-本文档�供了一个在Linux下访问GPIO的公约概述。
+
+传统GPIO接�
+============
+
+本文档概述了Linux下的GPIO访问公约。
 
 这些函数以 gpio_* 作为�缀。其他的函数��许使用这样的�缀或相关的
 __gpio_* �缀。
 
 
-什么是GPIO?
-==========
+什么是GPIO？
+============
 "通用输入/输出�"(GPIO)是一个�活的由软件控制的数字信�。他们�
 由多�芯片�供,且对于从事嵌入�和定制硬件的 Linux 开�者�说是
 比较熟悉。�个GPIO 都代表一个连接到特定引脚或�栅阵列(BGA)�装中
@@ -99,6 +88,7 @@ GPIO 公约
 
 标识 GPIO
 ---------
+
 GPIO 是通过无符�整型�标识的,范围是 0 到 MAX_INT。�留“负�数
 用于其他目的,例如标识信�“在这个��上��用�或指示错误。未接触底层
 硬件的代�会忽略这些整数。
@@ -115,7 +105,7 @@ FPGA 的特定æ�¿å­�上使用 80-95。编å�·ä¸�一定è¦�è¿žç»­,那些平å�°ä¸­ï¼
 
 如果你��始化一个带有无效 GPIO 编�的结构体,�以使用一些负编�
 (如"-EINVAL")，那将使其永远�会是有效。�测试这样一个结构体中的编�
-是�关�一个 GPIO，你�使用以下断言:
+是�关�一个 GPIO，你�使用以下断言::
 
 	int gpio_is_valid(int number);
 
@@ -128,11 +118,12 @@ FPGA 的特定æ�¿å­�上使用 80-95。编å�·ä¸�一定è¦�è¿žç»­,那些平å�°ä¸­ï¼
 
 使用 GPIO
 ---------
+
 对于一个 GPIO，系统应该�的第一件事情就是通过 gpio_request()
 函数分�它，�下文。
 
 接下�是设置I/O方�，这通常是在�级�动代�中为所使用的 GPIO 设置
-platform_device 时完�。
+platform_device 时完�::
 
 	/* 设置为输入或输出, 返回 0 或负的错误代� */
 	int gpio_direction_input(unsigned gpio);
@@ -157,12 +148,13 @@ get/set(获�/设置)函数调用没法返回错误,且有�能是�置错误
 
 
 访问自旋�安全的 GPIO
--------------------
+---------------------
+
 大多数 GPIO 控制器�以通过内存读/写指令�访问。这些指令�会休眠,�以
 安全地在硬(�线程)中断例程和类似的上下文中完�。
 
 对于那些用 gpio_cansleep()测试总是返回失败的 GPIO(�下文)，使用
-以下的函数访问:
+以下的函数访问::
 
 	/* GPIO 输入:返回零或�零 */
 	int gpio_get_value(unsigned gpio);
@@ -188,17 +180,18 @@ GPIO值是布尔值，零表示低电平，é�žé›¶è¡¨ç¤ºé«˜ç”µå¹³ã€‚当读å�–一ä
 
 
 访问�能休眠的 GPIO
------------------
+-------------------
+
 �些 GPIO 控制器必须通过基于总线(如 I2C 或 SPI)的消�访问。读或写这些
 GPIO 值的命令需�等待其信�排到队首���命令，�获得其�馈。期间需�
 休眠，这�能在 IRQ 例程(中断上下文)中执行。
 
 支�此类 GPIO 的平�通过以下函数返回�零值�区分出这� GPIO。(此函数需�
-一个之�通过 gpio_request 分�到的有效 GPIO 编�):
+一个之�通过 gpio_request 分�到的有效 GPIO 编�)::
 
 	int gpio_cansleep(unsigned gpio);
 
-为了访问这� GPIO,内核定义了一套��的函数:
+为了访问这� GPIO,内核定义了一套��的函数::
 
 	/* GPIO 输入:返回零或�零 ,�能会休眠 */
 	int gpio_get_value_cansleep(unsigned gpio);
@@ -214,25 +207,26 @@ GPIO 值的命令需�等待其信�排到队首���命令，�获得其
 事实，这些处�例程实际上和自旋�安全的函数是一样的。
 
 ** 除此之外 ** 调用设置和�置此类 GPIO 的函数也必须在�许休眠的上下文中，
-因为它们�能也需�访问 GPIO 控制器芯片: (这些设置函数通常在�级�动代�或者
-驱动探测/断开代�中，所以这是一个容易满足的约��件。)
+因为它们�能也需�访问 GPIO 控制器芯片 (这些设置函数通常在�级�动代�或者
+驱动探测/断开代�中，所以这是一个容易满足的约��件。) ::
 
-	gpio_direction_input()
-	gpio_direction_output()
-	gpio_request()
+                gpio_direction_input()
+                gpio_direction_output()
+                gpio_request()
 
-## 	gpio_request_one()
-##	gpio_request_array()
-## 	gpio_free_array()
+        ## 	gpio_request_one()
+        ##	gpio_request_array()
+        ## 	gpio_free_array()
 
-	gpio_free()
-	gpio_set_debounce()
+                gpio_free()
+                gpio_set_debounce()
 
 
 
 声明和释放 GPIO
-----------------------------
-为了有助于�获系统�置错误,定义了两个函数。
+----------------
+
+为了有助于�获系统�置错误,定义了两个函数::
 
 	/* 申请 GPIO, 返回 0 或负的错误代�.
 	 * �空标签�能有助于诊断.
@@ -256,9 +250,9 @@ GPIO 值的命令需�等待其信�排到队首���命令，�获得其
 �些平��能也使用 GPIO 作为电�管�激活信�(例如通过关闭未使用芯片区和
 简�地关闭未使用时钟)。
 
-对于 GPIO 使用 pinctrl �系统已知的引脚，�系统应该被告知其使用情况；
+对于 GPIO 使用引脚控制�系统已知的引脚，�系统应该被告知其使用情况；
 一个 gpiolib 驱动的 .request()�作应调用 pinctrl_gpio_request()，
-而 gpiolib 驱动的 .free()�作应调用 pinctrl_gpio_free()。pinctrl
+而 gpiolib 驱动的 .free()�作应调用 pinctrl_gpio_free()。引脚控制
 �系统�许 pinctrl_gpio_request()在�个引脚或引脚组以�用形�“属于�
 一个设备时都�功返回。
 
@@ -270,7 +264,7 @@ GPIO 值的命令需�等待其信�排到队首���命令，�获得其
 
 �些平��许部分或所有 GPIO 信�使用��的引脚。类似的，GPIO 或引脚的
 其他方�也需��置，如上拉/下拉。平�软件应该在对这些 GPIO 调用
-gpio_request()�将这类细节�置好，例如使用 pinctrl �系统的映射表，
+gpio_request()�将这类细节�置好，例如使用引脚控制�系统的映射表，
 使得 GPIO 的用户无须关注这些细节。
 
 还有一个值得注�的是在释放 GPIO �，你必须�止使用它。
@@ -278,7 +272,7 @@ gpio_request()�将这类细节�置好，例如使用 pinctrl �系统的映
 
 注�:申请一个 GPIO 并没有以任何方��置它，��过标识那个 GPIO 处于使用
 状�。必须有�外的代��处�引脚�置(如控制 GPIO 使用的引脚�上拉/下拉)。
-考虑到大多数情况下声明 GPIO 之�就会立��置它们,所以定义了以下三个辅助函数:
+考虑到大多数情况下声明 GPIO 之�就会立��置它们,所以定义了以下三个辅助函数::
 
 	/* 申请一个 GPIO 信�, �时通过特定的'flags'�始化�置,
 	 * 其他和 gpio_request()的�数和返回值相�
@@ -326,7 +320,7 @@ gpio_request()�将这类细节�置好，例如使用 pinctrl �系统的映
 将�这些标志�能扩展到支�更多的属性。
 
 更进一步,为了更简�地声明/释放多个 GPIO,'struct gpio'被引进��装所有
-这三个领域:
+这三个领域::
 
 	struct gpio {
 		unsigned	gpio;
@@ -334,7 +328,7 @@ gpio_request()�将这类细节�置好，例如使用 pinctrl �系统的映
 		const char	*label;
 	};
 
-一个典型的用例:
+一个典型的用例::
 
 	static struct gpio leds_gpios[] = {
 		{ 32, GPIOF_OUT_INIT_HIGH, "Power LED" }, /* 默认开� */
@@ -356,9 +350,10 @@ gpio_request()�将这类细节�置好，例如使用 pinctrl �系统的映
 
 
 GPIO 映射到 IRQ
---------------------
+----------------
+
 GPIO 编�是无符�整数;IRQ 编�也是。这些构�了两个逻辑上��的命�空间
-(GPIO 0 �一定使用 IRQ 0)。你�以通过以下函数在它们之间实现映射:
+(GPIO 0 �一定使用 IRQ 0)。你�以通过以下函数在它们之间实现映射::
 
 	/* 映射 GPIO 编�到 IRQ 编� */
 	int gpio_to_irq(unsigned gpio);
@@ -384,7 +379,8 @@ irq_to_gpio()返回的é�žé”™è¯¯å€¼å¤§å¤šæ•°é€šå¸¸å�¯ä»¥è¢« gpio_get_value()所ä½
 
 
 模拟开�信�
-----------------------------
+------------
+
 有时在�有低电平信�作为实际驱动结果(译者注:多个输出连接于一点，逻辑电平
 结果为所有输出的逻辑与)的时候,共享的信�线需�使用“开��信�。(该术语
 适用于 CMOS 管；而 TTL 用“集电�开路�。)一个上拉电阻使信�为高电平。这
@@ -408,9 +404,44 @@ irq_to_gpio()返回的é�žé”™è¯¯å€¼å¤§å¤šæ•°é€šå¸¸å�¯ä»¥è¢« gpio_get_value()所ä½
 这�一定是错误的。一个常�的例�就是 I2C 时钟的延长：一个需�较慢时钟的
 从设备延迟 SCK 的上�沿，而 I2C 主设备相应地调整其信�传输速率。
 
+GPIO控制器和引脚控制�系统
+--------------------------
+
+SOC上的GPIO控制器�能与引脚控制�系统紧密结�，�引脚�以与�选的gpio功
+能一起被其他功能使用。我们已�涵盖了这样的情况，例如一个GPIO控制器需��
+留一个引脚或通过调用以下任何一个引脚�设置其方�::
+
+  pinctrl_gpio_request()
+  pinctrl_gpio_free()
+  pinctrl_gpio_direction_input()
+  pinctrl_gpio_direction_output()
+
+但是，引脚控制�系统是如何将GPIO��（这是一个全局事项）与�个引脚控制器
+上的�个引脚交�关�的？
+
+这是通过注册引脚的“范围��实现的，这基本上是交��考表。这些�述是在
+Documentation/driver-api/pin-control.rst
+
+虽然引脚分�完全由引脚控制�系统管�，但gpio（在gpiolib下）�由gpio驱动
+维护。�能�生的情况是，SoC中的��引脚范围由��的gpio驱动器管�。
+
+这使得在调用 "pinctrl_gpio_request" 之�，让gpio驱动�pin ctrl�系
+统宣布它们的引脚范围是��的，以便在使用任何gpio之��求引脚控制�系统准
+备相应的引脚。
+
+为此，gpio控制器�以用引脚控制�系统注册其引脚范围。目�有两�方法：有或
+无DT。
+
+关于对DT的支�，请�考 Documentation/devicetree/bindings/gpio/gpio.txt.
+
+对于�DT支�，用户�以用适当的�数调用gpiochip_add_pin_range()，将一
+系列的gpio引脚注册到引脚控制驱动上。为此，必须将引脚控制设备的�称字符串
+作为�数之一传给这个程�。
+
+
+这些公约忽略了什么？
+====================
 
-这些公约忽略了什么?
-================
 这些公约忽略的最大一件事就是引脚�用，因为这属于高度芯片特定的属性且
 没有�移�性。�个平��能�需�明确的�用信�；有的对于任�给定的引脚
 �能�有两个功能选项；有的�能�个引脚有八个功能选项；有的�能�以将
@@ -433,8 +464,9 @@ Linux 的系统。)
 当�，动�定义 GPIO 并�是标准的，例如作为�置一个带有�些 GPIO 扩展器的
 附加电路�的副作用。
 
-GPIO 实现者的框架 (�选)
-=====================
+GPIO 实现者的框架（�选）
+=========================
+
 ���到了，有一个�选的实现框架，让平�使用相�的编程接�，更加简�地支�
 ���类的 GPIO 控制器。这个框架称为"gpiolib"。
 
@@ -444,15 +476,16 @@ GPIO 实现者的框架 (�选)
 
 
 控制器驱动: gpio_chip
--------------------
+---------------------
+
 在框架中�个 GPIO 控制器都包装为一个 "struct gpio_chip"，他包�了
 该类型的�个控制器的常用信�:
 
- - 设置 GPIO 方�的方法
- - 用于访问 GPIO 值的方法
- - 告知调用其方法是��能休眠的标志
- - �选的 debugfs 信�导出方法 (显示类似上拉�置一样的�外状�)
- - 诊断标签
+	- 设置 GPIO 方�的方法
+	- 用于访问 GPIO 值的方法
+	- 告知调用其方法是��能休眠的标志
+	- �选的 debugfs 信�导出方法 (显示类似上拉�置一样的�外状�)
+	- 诊断标签
 
 也包�了�自 device.platform_data 的�个实例的数�：它第一个 GPIO 的
 编�和它�用的 GPIO 的数�。
@@ -471,7 +504,8 @@ GPIO 实现者的框架 (�选)
 
 
 平�支�
--------
+--------
+
 为了支�这个框架，一个平�的 Kconfig 文件将会 "select"(选择)
 ARCH_REQUIRE_GPIOLIB 或 ARCH_WANT_OPTIONAL_GPIOLIB，并让它的
 <asm/gpio.h> 包� <asm-generic/gpio.h>，�时定义三个方法:
@@ -489,7 +523,7 @@ ARCH_WANT_OPTIONAL_GPIOLIB �味� gpiolib 核心默认关闭,且用户�以
 如果这些选项都没被选择,该平�就�通过 GPIO-lib 支� GPIO,且代���以
 被用户使能。
 
-以下这些方法的实现�以直接使用框架代�,并总是通过 gpio_chip 调度:
+以下这些方法的实现�以直接使用框架代�,并总是通过 gpio_chip 调度::
 
   #define gpio_get_value	__gpio_get_value
   #define gpio_set_value	__gpio_set_value
@@ -508,7 +542,8 @@ arch_initcall()或者更早的地方集�进平��始化代�，使这些 G
 且他们通常�以作为 IRQ 使用。
 
 �级支�
--------
+--------
+
 对于外部 GPIO 控制器(例如 I2C 或 SPI 扩展器�专用芯片�多功能器件�FPGA
 或 CPLD)，大多数常用�级特定代�都�以注册控制器设备，并��他们的驱动知�
 gpiochip_add()所使用的 GPIO 编�。他们的起始编�通常跟在平�特定的 GPIO
@@ -526,8 +561,9 @@ GPIO �以工作之���被注册。解决这类�赖的的一�方法是
 设备��无效时移除它们。
 
 
-用户空间的 Sysfs 接�(�选)
-========================
+用户空间的 Sysfs 接�（�选）
+=============================
+
 使用“gpiolib�实现框架的平��以选择�置一个 GPIO 的 sysfs 用户接�。
 这��于 debugfs 接�，因为它�供的是对 GPIO方�和值的控制，而��显示
 一个GPIO 的状�摘�。此外,它�以出现在没有调试支�的产�级系统中。
@@ -548,6 +584,7 @@ GPIO �以工作之���被注册。解决这类�赖的的一�方法是
 
 Sysfs 中的路径
 --------------
+
 在/sys/class/gpio 中有 3 类入�:
 
    -	用于在用户空间控制 GPIO 的控制接�;
@@ -625,8 +662,9 @@ GPIO 控制器的路径类似 /sys/class/gpio/gpiochip42/ (对于从#42 GPIO
 
 
 从内核代�中导出
--------------
-内核代��以明确地管�那些已通过 gpio_request()申请的 GPIO 的导出:
+----------------
+
+内核代��以明确地管�那些已通过 gpio_request()申请的 GPIO 的导出::
 
 	/* 导出 GPIO 到用户空间 */
 	int gpio_export(unsigned gpio, bool direction_may_change);
@@ -648,3 +686,9 @@ GPIO 控制器的路径类似 /sys/class/gpio/gpiochip42/ (对于从#42 GPIO
 在 GPIO 被导出之�，gpio_export_link()�许在 sysfs 文件系统的任何地方
 创建一个到这个 GPIO sysfs 节点的符�链接。这样驱动就�以通过一个�述性的
 �字，在 sysfs 中他们所拥有的设备下�供一个(到这个 GPIO sysfs 节点的)接�。
+
+
+API�考
+=======
+
+本节中列出的函数已被废弃。在新的代�中应该使用基于GPIO�述符的API。
diff --git a/Documentation/translations/zh_CN/driver-api/index.rst b/Documentation/translations/zh_CN/driver-api/index.rst
new file mode 100644
index 000000000000..ba354e1f4e6d
--- /dev/null
+++ b/Documentation/translations/zh_CN/driver-api/index.rst
@@ -0,0 +1,132 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/driver-api/index.rst
+
+:翻译:
+
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+
+:校译:
+
+========================
+Linux驱动实现者的API指�
+========================
+
+内核�供了���样的接��支�设备驱动的开�。这份文档�是对其中一些接�进行了
+一定程度的整�——希望��时间的推移，它能�得更好��用的�节�以在下�看到。
+
+.. class:: toc-title
+
+	   目录列表：
+
+.. toctree::
+   :maxdepth: 2
+
+   gpio/index
+   io_ordering
+
+Todolist:
+
+*   driver-model/index
+*   basics
+*   infrastructure
+*   ioctl
+*   early-userspace/index
+*   pm/index
+*   clk
+*   device-io
+*   dma-buf
+*   device_link
+*   component
+*   message-based
+*   infiniband
+*   aperture
+*   frame-buffer
+*   regulator
+*   reset
+*   iio/index
+*   input
+*   usb/index
+*   firewire
+*   pci/index
+*   cxl/index
+*   spi
+*   i2c
+*   ipmb
+*   ipmi
+*   i3c/index
+*   interconnect
+*   devfreq
+*   hsi
+*   edac
+*   scsi
+*   libata
+*   target
+*   mailbox
+*   mtdnand
+*   miscellaneous
+*   mei/index
+*   mtd/index
+*   mmc/index
+*   nvdimm/index
+*   w1
+*   rapidio/index
+*   s390-drivers
+*   vme
+*   80211/index
+*   uio-howto
+*   firmware/index
+*   pin-control
+*   md/index
+*   media/index
+*   misc_devices
+*   nfc/index
+*   dmaengine/index
+*   slimbus
+*   soundwire/index
+*   thermal/index
+*   fpga/index
+*   acpi/index
+*   auxiliary_bus
+*   backlight/lp855x-driver.rst
+*   connector
+*   console
+*   dcdbas
+*   eisa
+*   isa
+*   isapnp
+*   io-mapping
+*   generic-counter
+*   memory-devices/index
+*   men-chameleon-bus
+*   ntb
+*   nvmem
+*   parport-lowlevel
+*   pps
+*   ptp
+*   phy/index
+*   pwm
+*   pldmfw/index
+*   rfkill
+*   serial/index
+*   sm501
+*   surface_aggregator/index
+*   switchtec
+*   sync_file
+*   tty/index
+*   vfio-mediated-device
+*   vfio
+*   vfio-pci-device-specific-driver-acceptance
+*   xilinx/index
+*   xillybus
+*   zorro
+*   hte/index
+
+.. only::  subproject and html
+
+   Indices
+   =======
+
+   * :ref:`genindex`
diff --git a/Documentation/translations/zh_CN/driver-api/io_ordering.rst b/Documentation/translations/zh_CN/driver-api/io_ordering.rst
new file mode 100644
index 000000000000..4dbfa4ce92a0
--- /dev/null
+++ b/Documentation/translations/zh_CN/driver-api/io_ordering.rst
@@ -0,0 +1,60 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: ../disclaimer-zh_CN.rst
+
+:Original: Documentation/driver-api/io_ordering.rst
+
+:翻译:
+
+ 林永� Lin Yongting <linyongting@gmail.com>
+ �延腾 Yanteng Si <siyanteng@loongson.cn>
+
+:校译:
+
+===========================
+对内存映射地�的I/O写入排�
+===========================
+
+在�些平�上，所谓的内存映射I/O是弱顺�。在这些平�上，驱动开�者有责任
+��I/O内存映射地�的写�作按程�图�的顺�达到设备。通常读�一个“安全�
+设备寄存器或桥寄存器，触�IO芯片清刷未处�的写�作到达设备��处�读�作，
+而达到��目的。驱动程�通常在spinlock�护的临界区退出之�使用这�技术。
+这也�以����的写�作�在��的写�作之�到达设备（这�常类似于内存
+�障�作，mb()，�过仅适用于I/O）。
+
+�设一个设备驱动程的具体例�::
+
+                ...
+        CPU A:  spin_lock_irqsave(&dev_lock, flags)
+        CPU A:  val = readl(my_status);
+        CPU A:  ...
+        CPU A:  writel(newval, ring_ptr);
+        CPU A:  spin_unlock_irqrestore(&dev_lock, flags)
+                ...
+        CPU B:  spin_lock_irqsave(&dev_lock, flags)
+        CPU B:  val = readl(my_status);
+        CPU B:  ...
+        CPU B:  writel(newval2, ring_ptr);
+        CPU B:  spin_unlock_irqrestore(&dev_lock, flags)
+                ...
+
+上述例�中，设备�能会先接收到newval2的值，然�接收到newval的值，问题就
+�生了。�过很容易通过下�方法�修�::
+
+                ...
+        CPU A:  spin_lock_irqsave(&dev_lock, flags)
+        CPU A:  val = readl(my_status);
+        CPU A:  ...
+        CPU A:  writel(newval, ring_ptr);
+        CPU A:  (void)readl(safe_register); /* �置寄存器？*/
+        CPU A:  spin_unlock_irqrestore(&dev_lock, flags)
+                ...
+        CPU B:  spin_lock_irqsave(&dev_lock, flags)
+        CPU B:  val = readl(my_status);
+        CPU B:  ...
+        CPU B:  writel(newval2, ring_ptr);
+        CPU B:  (void)readl(safe_register); /* �置寄存器？*/
+        CPU B:  spin_unlock_irqrestore(&dev_lock, flags)
+
+在解决方案中，读�safe_register寄存器，触�IO芯片清刷未处�的写�作，
+�处���的读�作，防止引�数��一致问题。
diff --git a/Documentation/translations/zh_CN/index.rst b/Documentation/translations/zh_CN/index.rst
index bf85baca8b3e..2fc60e60feb4 100644
--- a/Documentation/translations/zh_CN/index.rst
+++ b/Documentation/translations/zh_CN/index.rst
@@ -108,6 +108,7 @@ TODOList:
    :maxdepth: 2
 
    core-api/index
+   driver-api/index
    locking/index
    accounting/index
    cpu-freq/index
@@ -120,10 +121,10 @@ TODOList:
    scheduler/index
    mm/index
    peci/index
+   PCI/index
 
 TODOList:
 
-* driver-api/index
 * block/index
 * cdrom/index
 * ide/index
@@ -148,7 +149,6 @@ TODOList:
 * crypto/index
 * bpf/index
 * usb/index
-* PCI/index
 * scsi/index
 * misc-devices/index
 * mhi/index
diff --git a/Documentation/translations/zh_CN/io_ordering.txt b/Documentation/translations/zh_CN/io_ordering.txt
deleted file mode 100644
index 7bb3086227ae..000000000000
--- a/Documentation/translations/zh_CN/io_ordering.txt
+++ /dev/null
@@ -1,67 +0,0 @@
-Chinese translated version of Documentation/driver-api/io_ordering.rst
-
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
-
-Chinese maintainer: Lin Yongting <linyongting@gmail.com>
----------------------------------------------------------------------
-Documentation/driver-api/io_ordering.rst 的中文翻译
-
-如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
-交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
-
-中文版维护者： 林永� Lin Yongting <linyongting@gmail.com>
-中文版翻译者： 林永� Lin Yongting <linyongting@gmail.com>
-中文版校译者： 林永� Lin Yongting <linyongting@gmail.com>
-
-
-以下为正文
----------------------------------------------------------------------
-
-在�些平�上，所谓的内存映射I/O是弱顺�。在这些平�上，驱动开�者有责任
-��I/O内存映射地�的写�作按程�图�的顺�达到设备。通常读�一个“安全�
-设备寄存器或桥寄存器，触�IO芯片清刷未处�的写�作到达设备��处�读�作，
-而达到��目的。驱动程�通常在spinlock�护的临界区退出之�使用这�技术。
-这也�以����的写�作�在��的写�作之�到达设备（这�常类似于内存
-�障�作，mb()，�过仅适用于I/O）。
-
-�设一个设备驱动程的具体例�：
-
-        ...
-CPU A:  spin_lock_irqsave(&dev_lock, flags)
-CPU A:  val = readl(my_status);
-CPU A:  ...
-CPU A:  writel(newval, ring_ptr);
-CPU A:  spin_unlock_irqrestore(&dev_lock, flags)
-        ...
-CPU B:  spin_lock_irqsave(&dev_lock, flags)
-CPU B:  val = readl(my_status);
-CPU B:  ...
-CPU B:  writel(newval2, ring_ptr);
-CPU B:  spin_unlock_irqrestore(&dev_lock, flags)
-        ...
-
-上述例�中，设备�能会先接收到newval2的值，然�接收到newval的值，问题就
-�生了。�过很容易通过下�方法�修�：
-
-        ...
-CPU A:  spin_lock_irqsave(&dev_lock, flags)
-CPU A:  val = readl(my_status);
-CPU A:  ...
-CPU A:  writel(newval, ring_ptr);
-CPU A:  (void)readl(safe_register); /* �置寄存器？*/
-CPU A:  spin_unlock_irqrestore(&dev_lock, flags)
-        ...
-CPU B:  spin_lock_irqsave(&dev_lock, flags)
-CPU B:  val = readl(my_status);
-CPU B:  ...
-CPU B:  writel(newval2, ring_ptr);
-CPU B:  (void)readl(safe_register); /* �置寄存器？*/
-CPU B:  spin_unlock_irqrestore(&dev_lock, flags)
-
-在解决方案中，读�safe_register寄存器，触�IO芯片清刷未处�的写�作，
-�处���的读�作，防止引�数��一致问题。
diff --git a/Documentation/translations/zh_CN/oops-tracing.txt b/Documentation/translations/zh_CN/oops-tracing.txt
deleted file mode 100644
index c5f3bda7abcb..000000000000
--- a/Documentation/translations/zh_CN/oops-tracing.txt
+++ /dev/null
@@ -1,212 +0,0 @@
-Chinese translated version of Documentation/admin-guide/bug-hunting.rst
-
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
-
-Chinese maintainer: Dave Young <hidave.darkstar@gmail.com>
----------------------------------------------------------------------
-Documentation/admin-guide/bug-hunting.rst 的中文翻译
-
-如果想评论或更新本文的内容，请直接�系原文档的维护者。如果你使用英文
-交�有困难的�，也�以�中文版维护者求助。如果本翻译更新��时或者翻
-译存在问题，请�系中文版维护者。
-
-中文版维护者： �瑞 Dave Young <hidave.darkstar@gmail.com>
-中文版翻译者： �瑞 Dave Young <hidave.darkstar@gmail.com>
-中文版校译者： �阳 Li Yang <leoyang.li@nxp.com>
-               王� Wang Cong <xiyou.wangcong@gmail.com>
-
-以下为正文
----------------------------------------------------------------------
-
-注�： ksymoops 在2.6中是没有用的。 请以原有格�使用Oops(�自dmesg，等等)。
-忽略任何这样那样关于“解�Oops�或者“通过ksymoops�行�的文档。 如果你贴出�行过
-ksymoops的�自2.6的Oops，人们�会让你�贴一次。
-
-快速总结
--------------
-
-�现Oops并��给看似相关的内核领域的维护者。别太担心对�上�。如果你�确定就�给
-和你所�的事情相关的代�的负责人。 如果��现试��述怎样�构。 那甚至比oops更有
-价值。
-
-如果你对于��给�一无所知， �给linux-kernel@vger.kernel.org。感谢你帮助Linux
-尽�能地稳定。
-
-Oops在哪里?
-----------------------
-
-通常Oops文本由klogd从内核缓冲区里读�并传给syslogd，由syslogd写到syslog文件中，
-典型地是/var/log/messages(�赖于/etc/syslog.conf)。有时klogd崩溃了,这�情况下你
-能够�行dmesg > file�从内核缓冲区中读�数�并�存下�。 �则你�以
-cat /proc/kmsg > file， 然而你必须介入中止传输， kmsg是一个“永�结�的文件�。如
-果机器崩溃�到你�能输入命令或者�盘��用那么你有三�选择:-
-
-（1） 手抄�幕上的文本待机器����输入计算机。 麻烦但如果没有针对崩溃的准备，
-这是仅有的选择。 �外，你�以用数�相机把�幕�下�-�太好，但比没有强。 如果信
-�滚动到了终端的上�，你会�现以高分辩率�动（比如，vga=791）会让你读到更多的文
-本。（注�：这需�vesafb，所以对‘早期’的oops没有帮助）
-
-（2）用串�终端�动（请�看Documentation/admin-guide/serial-console.rst），�行一个null
-modem到�一�机器并用你喜欢的通讯工具获�输出。Minicom工作地很好。
-
-（3）使用Kdump（请�看Documentation/admin-guide/kdump/kdump.rst），
-使用在Documentation/admin-guide/kdump/gdbmacros.txt中定义的dmesg gdb�，从旧的内存中��内核
-环形缓冲区。
-
-完整信�
-----------------
-
-注�：以下�自于Linus的邮件适用于2.4内核。 我因为历�原因�留了它，并且因为其中
-一些信��然适用。 特别注�的是，请忽略任何ksymoops的引用。
-
-From: Linus Torvalds <torvalds@osdl.org>
-
-怎样跟踪Oops.. [原�到linux-kernel的一�邮件]
-
-主�的�门是有五年和这些烦人的oops消�打交�的�验;-)
-
-实际上，你有办法使它更简�。我有两个��的方法：
-
-	gdb /usr/src/linux/vmlinux
-	gdb> disassemble <offending_function>
-
-那是�现问题的简�办法，至少如果bug报告�的好的情况下（象这个一样-�行ksymoops
-得到oops�生的函数�函数内的�移）。
-
-哦，如果报告�生的内核以相�的编译器和相似的�置编译它会有帮助的。
-
-�一件��的事是�汇编bug报告的“Code�部分：ksymoops也会用正确的工具��这件事，
-但如果没有那些工具你�以写一个傻程�：
-
-	char str[] = "\xXX\xXX\xXX...";
-	main(){}
-
-并用gcc -g编译它然�执行“disassemble str�（XX部分是由Oops报告的值-你�以仅剪切
-粘贴并用“\x�替�空格-我就是这么�的，因为我懒得写程�自动�这一切）。
-
-�外，你�以用scripts/decodecode这个shell脚本。它的使用方法是：
-decodecode < oops.txt
-
-“Code�之�的�六进制字节�能（在�些架构上）有一些当�指令之�的指令字节以�
-当�和之�的指令字节
-
-Code: f9 0f 8d f9 00 00 00 8d 42 0c e8 dd 26 11 c7 a1 60 ea 2b f9 8b 50 08 a1
-64 ea 2b f9 8d 34 82 8b 1e 85 db 74 6d 8b 15 60 ea 2b f9 <8b> 43 04 39 42 54
-7e 04 40 89 42 54 8b 43 04 3b 05 00 f6 52 c0
-
-最�，如果你想知�代��自哪里，你�以：
-
-	cd /usr/src/linux
-	make fs/buffer.s 	# 或任何产生BUG的文件
-
-然�你会比gdb�汇编更清楚的知��生了什么。
-
-现在，问题是把你所拥有的所有数�结�起�：C��（关于它应该怎样的一般知识），
-汇编代��其�汇编得到的代�（�外还有从“oops�消�得到的寄存器状�-对了解��的
-指针有用，而且当你有了汇编代�你也能拿其它的寄存器和任何它们对应的C表达��匹�
-）。
-
-实际上，你仅需看看哪里�匹�（这个例�是“Code��汇编和编译器生�的代��匹�）。
-然�你须�找出为什么�匹�。通常很简�-你看到代�使用了空指针然�你看代�想知�
-空指针是怎么出现的，还有检查它是��法..
-
-现在，如果明白这是一项耗时的工作而且需�一�点儿的专心，没错。这就是我为什么大多
-�是忽略那些没有符�表信�的崩溃报告的原因：简�的说太难查找了（我有一些
-程�用于在内核代�段中�索特定的模�，而且有时我也已�能找出那些崩溃的地方，但是
-仅仅是找出正确的�列也确实需�相当扎实的内核知识）
-
-_有时_会�生这�情况，我仅看到崩溃中的�汇编代��列， 然�我马上就明白问题出在
-哪里。这时我��识到自己干这个工作已�太长时间了;-)
-
-		Linus
-
-
----------------------------------------------------------------------------
-关于Oops跟踪的注解：
-
-为了帮助Linus和其它内核开�者，klogd纳入了大�的支��处��护错误。为了拥有对
-地�解�的完整支�至少应该使用1.3-pl3的sysklogd包。
-
-当�护错误�生时，klogd守护进程自动把内核日志信�中的��地�翻译�它们相应的符
-�。
-
-klogd执行两�类型的地�解�。首先是��翻译其次是动�翻译。��翻译和ksymoops
-一样使用System.map文件。为了���翻译klogd守护进程必须在�始化时能找到system
-map文件。关于klogd怎样�索map文件请�看klogd手册页。
-
-动�地�翻译在使用内核�装载模�时很��。 因为内核模�的内存是从内核动�内存池
-里分�的，所以�管是模�开始�置还是模�中函数和符�的�置都�是固定的。
-
-内核支��许程�决定装载哪些模�和它们在内存中�置的系统调用。使用这些系统调用
-klogd守护进程生�一张符�表用于调试�生在�装载模�中的�护错误。
-
-至少klogd会�供产生�护错误的模��。还�有�外的符�信�供�装载模�开�者选择
-以从模�中输出符�信�。
-
-因为内核模�环境�能是动�的，所以必须有一�机制当模�环境�生改�时�通知klogd
-守护进程。 有一些�用的命令行选项�许klogd�当�执行中的守护进程��信�，告知符
-�信�应该被刷新了。 更多信�请�看klogd手册页。
-
-sysklogd�布时包�一个补�修改了modules-2.0.0包，无论何时一个模�装载或者�载都
-会自动�klogd��信�。打上这个补��供了必�的对调试�生于内核�装载模�的�护
-错误的无�支�。
-
-以下是被klogd处�过的�生在�装载模�中的一个�护错误例�：
----------------------------------------------------------------------------
-Aug 29 09:51:01 blizard kernel: Unable to handle kernel paging request at virtual address f15e97cc
-Aug 29 09:51:01 blizard kernel: current->tss.cr3 = 0062d000, %cr3 = 0062d000
-Aug 29 09:51:01 blizard kernel: *pde = 00000000
-Aug 29 09:51:01 blizard kernel: Oops: 0002
-Aug 29 09:51:01 blizard kernel: CPU:    0
-Aug 29 09:51:01 blizard kernel: EIP:    0010:[oops:_oops+16/3868]
-Aug 29 09:51:01 blizard kernel: EFLAGS: 00010212
-Aug 29 09:51:01 blizard kernel: eax: 315e97cc   ebx: 003a6f80   ecx: 001be77b   edx: 00237c0c
-Aug 29 09:51:01 blizard kernel: esi: 00000000   edi: bffffdb3   ebp: 00589f90   esp: 00589f8c
-Aug 29 09:51:01 blizard kernel: ds: 0018   es: 0018   fs: 002b   gs: 002b   ss: 0018
-Aug 29 09:51:01 blizard kernel: Process oops_test (pid: 3374, process nr: 21, stackpage=00589000)
-Aug 29 09:51:01 blizard kernel: Stack: 315e97cc 00589f98 0100b0b4 bffffed4 0012e38e 00240c64 003a6f80 00000001
-Aug 29 09:51:01 blizard kernel:        00000000 00237810 bfffff00 0010a7fa 00000003 00000001 00000000 bfffff00
-Aug 29 09:51:01 blizard kernel:        bffffdb3 bffffed4 ffffffda 0000002b 0007002b 0000002b 0000002b 00000036
-Aug 29 09:51:01 blizard kernel: Call Trace: [oops:_oops_ioctl+48/80] [_sys_ioctl+254/272] [_system_call+82/128]
-Aug 29 09:51:01 blizard kernel: Code: c7 00 05 00 00 00 eb 08 90 90 90 90 90 90 90 90 89 ec 5d c3
----------------------------------------------------------------------------
-
-Dr. G.W. Wettstein           Oncology Research Div. Computing Facility
-Roger Maris Cancer Center    INTERNET: greg@wind.rmcc.com
-820 4th St. N.
-Fargo, ND  58122
-Phone: 701-234-7556
-
-
----------------------------------------------------------------------------
-�污染的内核
-
-一些oops报告在程�记数器之�包�字符串'Tainted: '。这表明内核已�被一些东西给污
-染了。 该字符串之�紧跟�一系列的�置�感的字符，�个代表一个特定的污染值。
-
-  1：'G'如果所有装载的模�都有GPL或相容的许��，'P'如果装载了任何的专有模�。
-没有模�MODULE_LICENSE或者带有insmod认为是与GPL�相容的的MODULE_LICENSE的模�被
-认定是专有的。
-
-  2：'F'如果有任何通过“insmod -f�被强制装载的模�，' '如果所有模�都被正常装载。
-
-  3：'S'如果oops�生在SMP内核中，�行于没有�明安全�行多处�器的硬件。 当�这�
-情况仅�于几��支�SMP的速龙处�器。
-
-  4：'R'如果模�通过“insmod -f�被强制装载，' '如果所有模�都被正常装载。
-
-  5：'M'如果任何处�器报告了机器检查异常，' '如果没有�生机器检查异常。
-
-  6：'B'如果页释放函数�现了一个错误的页引用或者一些�预期的页标志。
-
-  7：'U'如果用户或者用户应用程�特别请求设置污染标志，�则' '。
-
-  8：'D'如果内核刚刚死掉，比如有OOPS或者BUG。
-
-使用'Tainted: '字符串的主�原因是�告诉内核调试者，这是�是一个干净的内核亦或�
-生了任何的�正常的事。污染是永久的：�使出错的模�已�被�载了，污染值�然存在，
-以表明内核��值得信任。
diff --git a/Documentation/translations/zh_CN/process/coding-style.rst b/Documentation/translations/zh_CN/process/coding-style.rst
index 638d714bec83..fa28ef0a7fee 100644
--- a/Documentation/translations/zh_CN/process/coding-style.rst
+++ b/Documentation/translations/zh_CN/process/coding-style.rst
@@ -1,21 +1,23 @@
 .. include:: ../disclaimer-zh_CN.rst
 
-:Original: :ref:`Documentation/process/coding-style.rst <codingstyle>`
+:Original: Documentation/process/coding-style.rst
 
 .. _cn_codingstyle:
 
-译者::
+:译者:
+ - 张� Zhang Le <r0bertz@gentoo.org>
+ - Andy Deng <theandy.deng@gmail.com>
+ - �想� <bobwxc@email.cn>
 
-  中文版维护者： 张� Zhang Le <r0bertz@gentoo.org>
-  中文版翻译者： 张� Zhang Le <r0bertz@gentoo.org>
-  中文版校译者： 王� Wang Cong <xiyou.wangcong@gmail.com>
-                 wheelz <kernel.zeng@gmail.com>
-                 管旭东 Xudong Guan <xudong.guan@gmail.com>
-                 Li Zefan <lizf@cn.fujitsu.com>
-                 Wang Chen <wangchen@cn.fujitsu.com>
+:校译:
+ - 王� Wang Cong <xiyou.wangcong@gmail.com>
+ - wheelz <kernel.zeng@gmail.com>
+ - 管旭东 Xudong Guan <xudong.guan@gmail.com>
+ - Li Zefan <lizf@cn.fujitsu.com>
+ - Wang Chen <wangchen@cn.fujitsu.com>
 
 Linux 内核代�风格
-=========================
+==================
 
 这是一个简短的文档，�述了 linux 内核的首选代�风格。代�风格是因人而异的，
 而且我�愿�把自己的观点强加给任何人，但这就�我去�任何事情都必须�循的原则
@@ -29,7 +31,7 @@ Linux 内核代�风格
 
 
 1) 缩进
---------------
+-------
 
 制表符是 8 个字符，所以缩进也是 8 个字符。有些异端�动试图将缩进�为 4 (甚至
 2�) 字符深，这几乎相当于�试将圆周率的值定义为 3。
@@ -73,6 +75,22 @@ Linux 内核代�风格
 	if (condition) do_this;
 	  do_something_everytime;
 
+��使用逗����使用大括�：
+
+.. code-block:: c
+
+	if (condition)
+		do_this(), do_that();
+
+使用大括�包裹多语�：
+
+.. code-block:: c
+
+	if (condition) {
+		do_this();
+		do_that();
+	}
+
 也��在一行里放多个赋值语�。内核代�风格超级简�。就是���能导致别人误读
 的表达�。
 
@@ -83,20 +101,25 @@ Linux 内核代�风格
 
 
 2) 把长的行和字符串打散
-------------------------------
+-----------------------
 
 代�风格的�义就在于使用平常使用的工具�维�代�的�读性和�维护性。
 
 �一行的长度的�制是 80 列，我们强烈建议您�守这个惯例。
 
 长于 80 列的语��打散�有�义的片段。除�超过 80 列能显著增加�读性，并且�
-会��信�。�片段�明显短于�片段，并明显��。这�样适用于有�很长�数列表
-的函数头。然而，�对��打散对用户��的字符串，例如 printk 信�，因为这样就
+会��信�。
+
+�片段�明显短于�片段，并明显��。一��常常用的样�是将�体与函数左括�对�。
+
+这�样适用于有�很长�数列表的函数头。
+
+然而，�对��打散对用户��的字符串，例如 printk 信�，因为这样就
 很难对它们 grep。
 
 
 3) 大括�和空格的放置
-------------------------------
+---------------------
 
 C 语言风格中�外一个常�问题是大括�的放置。和缩进大���，选择或弃用��放
 置策略并没有多少技术上的原因，�过首选的方�，就� Kernighan 和 Ritchie 展示
@@ -132,12 +155,12 @@ C 语言风格中�外一个常�问题是大括�的放置。和缩进大�
 		body of function
 	}
 
-全世界的异端�能会抱怨这个�一致性是... 呃... �一致的，�过所有�维�全的人
+全世界的异端�能会抱怨这个�一致性是……呃……�一致，�过所有�维�全的人
 都知� (a) K&R 是 **正确的** 并且 (b) K&R 是正确的。此外，�管怎样函数都是特
 殊的 (C 函数是�能嵌套的)。
 
 注�结�大括�独自��一行，除�它��跟��一个语�的剩余部分，也就是 do 语
-�中的 "while" 或者 if 语�中的 "else"，�这样：
+�中的 ``while`` 或者 if 语�中的 ``else`` ，�这样：
 
 .. code-block:: c
 
@@ -191,7 +214,7 @@ C 语言风格中�外一个常�问题是大括�的放置。和缩进大�
 	}
 
 3.1) 空格
-********************
+*********
 
 Linux 内核的空格使用方� (主�) �决于它是用于函数还是关键字。(大多数) 关键字
 ��加一个空格。值得注�的例外是 sizeof, typeof, alignof 和 __attribute__，这
@@ -254,7 +277,7 @@ Linux 内核的空格使用方å¼� (主è¦�) å�–决于它是用于函数还是关é
 
 
 4) 命�
-------------------------------
+-------
 
 C 是一个简朴的语言，你的命�也应该这样。和 Modula-2 和 Pascal 程�员��，
 C 程�员�使用类似 ThisVariableIsATemporaryCounter 这样�丽的�字。C 程�员会
@@ -275,11 +298,31 @@ C 程�员�使用类似 ThisVariableIsATemporaryCounter 这样�丽的�字
 �能的�。类似的， ``tmp`` �以用�称呼任�类型的临时��。
 
 如果你怕混淆了你的本地���，你就�到�一个问题了，��函数增长�尔蒙失衡综
-�症。请看第六章 (函数)。
+��。请看第六章 (函数)。
 
+对于符��称和文档，��引入新的“master/slave�（或独立于“master�的“slave�）
+和“blacklist/whitelist�。
+
+“master/slave�推�替�为：
+    '{primary,main} / {secondary,replica,subordinate}'
+    '{initiator,requester} / {target,responder}'
+    '{controller,host} / {device,worker,proxy}'
+    'leader/follower'
+    'director/performer'
+
+“blacklist/whitelist�推�替�为：
+    'denylist/allowlist'
+    'blocklist/passlist'
+
+引入新用法的例外情况是：维护用户空间ABI/API，或更新现有（截至2020年）硬件或
+�议规范的代�时�求这些术语。对于新规范，尽�能将术语的规范用法转�为内核
+编�标准。
+
+.. warning::
+	以上主从�黑白��规则�适用于中文文档，请勿更改中文术语�
 
 5) Typedef
------------
+----------
 
 ��使用类似 ``vps_t`` 之类的东西。
 
@@ -308,7 +351,7 @@ C 程�员�使用类似 ThisVariableIsATemporaryCounter 这样�丽的�字
 
      .. note::
 
-       ��明性和 "访问函数" 本身是�好的。我们使用 pte_t 等类型的原因在于真
+       ��明性和“访问函数�本身是�好的。我们使用 pte_t 等类型的原因在于真
        的是完全没有任何共用的�访问信�。
 
  (b) 清楚的整数类型，如此，这层抽象就�以 **帮助** 消除到底是 ``int`` 还是
@@ -353,7 +396,7 @@ C 程�员�使用类似 ThisVariableIsATemporaryCounter 这样�丽的�字
 
 
 6) 函数
-------------------------------
+-------
 
 函数应该简短而漂亮，并且�完�一件事情。函数应该�以一�或者两�显示完 (我们
 都知� ISO/ANSI �幕大�是 80x24)，��一件事情，而且把它�好。
@@ -383,12 +426,46 @@ C 程�员�使用类似 ThisVariableIsATemporaryCounter 这样�丽的�字
 	}
 	EXPORT_SYMBOL(system_is_up);
 
-在函数原型中，包�函数�和它们的数�类型。虽然 C 语言里没有这样的�求，在
+6.1) 函数原型
+*************
+
+在函数原型中包��数�和它们的数�类型。虽然 C 语言里没有这样的�求，但在
 Linux 里这是�倡的�法，因为这样�以很简�的给读者�供更多的有价值的信�。
 
+��在函数声明里使用 ``extern`` 关键字，因为这会导致代�行�长，并且�是严格
+必需的。
+
+写函数原型时，请�� `元素顺�规则 <https://lore.kernel.org/mm-commits/CAHk-=wiOCLRny5aifWNhr621kYrJwhfURsa0vFPeUEm8mF0ufg@mail.gmail.com/>`_ 。
+例如下列函数声明::
+
+ __init void * __must_check action(enum magic value, size_t size, u8 count,
+				   char *fmt, ...) __printf(4, 5) __malloc;
+
+推�的函数原型元素顺�是：
+
+- 储存类型（下方的 ``static __always_inline`` ，注� ``__always_inline``
+  技术上�讲是个属性但被当� ``inline`` ）
+- 储存类型属性（上方的 ``__init`` ——�节声明，但也� ``__cold`` ）
+- 返回类型（上方的 ``void *`` ）
+- 返回类型属性（上方的 ``__must_check`` ）
+- 函数�（上方的 ``action`` ）
+- 函数�数（上方的 ``(enum magic value, size_t size, u8 count, char *fmt, ...)`` ，
+  注�必须写上�数�）
+- 函数�数属性（上方的 ``__printf(4, 5)`` ）
+- 函数行为属性（上方的 ``__malloc`` ）
+
+请注�，对于函数 **定义** （�实际函数体），编译器��许在函数�数之�添加函
+数�数属性。在这�情况下，它们应该跟�存储类型属性（例如，与上�的 **声明**
+示例相比，请注�下�的 ``__printf(4, 5)`` 的�置�生了�化）::
+
+ static __always_inline __init __printf(4, 5) void * __must_check action(enum magic value,
+		size_t size, u8 count, char *fmt, ...) __malloc
+ {
+	...
+ }
 
 7) 集中的函数退出途径
-------------------------------
+---------------------
 
 虽然被�些人声称已�过时，但是 goto 语�的等价物还是�常被编译器所使用，具体
 形�是无�件跳转指令。
@@ -432,7 +509,7 @@ Linux 里这是æ��倡的å�šæ³•ï¼Œå› ä¸ºè¿™æ ·å�¯ä»¥å¾ˆç®€å�•çš„给读者æ��ä¾›æ›
 		return result;
 	}
 
-一个需�注�的常�错误是 ``一个 err 错误`` ，就�这样：
+一个需�注�的常�错误是 ``� err 错误`` ，就�这样：
 
 .. code-block:: c
 
@@ -456,19 +533,19 @@ Linux 里这是æ��倡的å�šæ³•ï¼Œå› ä¸ºè¿™æ ·å�¯ä»¥å¾ˆç®€å�•çš„给读者æ��ä¾›æ›
 
 
 8) 注释
-------------------------------
+-------
 
 注释是好的，�过有过度注释的�险。永远��在注释里解释你的代�是如何�作的：
 更好的�法是让别人一看你的代�就�以明白，解释写的很差的代�是浪费时间。
 
-一般的，你想�你的注释告诉别人你的代��了什么，而�是怎么�的。也请你��把
+一般�说你用注释告诉别人你的代��了什么，而�是怎么�的。也请你��把
 注释放在一个函数体内部：如果函数��到你需�独立的注释其中的一部分，你很�能
 需�回到第六章看一看。你�以�一些�注释�注明或警告�些很�明 (或者槽糕) 的
 �法，但��加太多。你应该�的，是把注释放在函数的头部，告诉人们它�了什么，
 也�以加上它�这些事情的原因。
 
-当注释内核 API 函数时，请使用 kernel-doc 格�。请看
-Documentation/doc-guide/ 和 scripts/kernel-doc 以获得详细信�。
+当注释内核 API 函数时，请使用 kernel-doc 格�。详�
+Documentation/translations/zh_CN/doc-guide/index.rst 和 scripts/kernel-doc 。
 
 长 (多行) 注释的首选风格是：
 
@@ -500,17 +577,18 @@ Documentation/doc-guide/ 和 scripts/kernel-doc 以获得详细信�。
 
 
 9) 你已�把事情弄糟了
-------------------------------
+---------------------
 
-这没什么，我们都是这样。�能你的使用了很长时间 Unix 的朋�已�告诉你
+这没什么，我们都是这样。�能你长期使用 Unix 的朋�已�告诉你
 ``GNU emacs`` 能自动帮你格�化 C �代�，而且你也注�到了，确实是这样，�过它
 所使用的默认值和我们想�的相去甚远 (实际上，甚至比�机打的还�差——无数个猴�
-在 GNU emacs 里打字永远�会创造出一个好程�) (译注：Infinite Monkey Theorem)
+在 GNU emacs 里打字永远�会创造出一个好程�)
+*(译注：Infinite Monkey Theorem)*
 
 所以你�么放弃 GNU emacs，�么改�它让它使用更��的设定。�采用�一个方案，
 你�以把下�这段粘贴到你的 .emacs 文件里。
 
-.. code-block:: none
+.. code-block:: elisp
 
   (defun c-lineup-arglist-tabs-only (ignored)
     "Line up argument lists by tabs, not spaces"
@@ -529,7 +607,7 @@ Documentation/doc-guide/ 和 scripts/kernel-doc 以获得详细信�。
           (c-offsets-alist . (
                   (arglist-close         . c-lineup-arglist-tabs-only)
                   (arglist-cont-nonempty .
-		      (c-lineup-gcc-asm-reg c-lineup-arglist-tabs-only))
+                      (c-lineup-gcc-asm-reg c-lineup-arglist-tabs-only))
                   (arglist-intro         . +)
                   (brace-list-intro      . +)
                   (c                     . c-lineup-C-comments)
@@ -573,9 +651,14 @@ Documentation/doc-guide/ 和 scripts/kernel-doc 以获得详细信�。
 ``indent`` 有很多选项，特别是�新格�化注释的时候，你�能需�看一下它的手册。
 �过记�： ``indent`` �能修正�的编程习惯。
 
+请注�，您还�以使用 ``clang-format`` 工具帮助您处�这些规则，快速自动�新格
+�化部分代�，并审阅整个文件以�现代�风格错误�打字错误和�能的改进。它还�
+以方便地排� ``#include`` ，对���/�，�排文本和其他类似任务。
+详� Documentation/process/clang-format.rst 。
+
 
 10) Kconfig �置文件
-------------------------------
+--------------------
 
 对于�布��树的所有 Kconfig* �置文件�说，它们缩进方�有所��。紧挨�
 ``config`` 定义的行，用一个制表符缩进，然而 help 信�的缩进则�外增加 2 个空
@@ -598,11 +681,11 @@ Documentation/doc-guide/ 和 scripts/kernel-doc 以获得详细信�。
 	depends on ADFS_FS
 	...
 
-�查看�置文件的完整文档，请看 Documentation/kbuild/kconfig-language.rst。
+�查看�置文件的完整文档，请看 Documentation/kbuild/kconfig-language.rst 。
 
 
 11) 数�结构
-------------------------------
+------------
 
 如果一个数�结构，在创建和销�它的�线执行环境之外��，那么它必须�有一个引
 用计数器。内核里没有垃圾收集 (并且内核之外的垃圾收集慢且效率低下)，这�味�你
@@ -626,7 +709,7 @@ mm_count)，和文件系统 (``struct super_block``: s_count å’Œ s_active) 中æ‰
 
 
 12) �，枚举和RTL
-------------------------------
+-----------------
 
 用于定义常�的�的�字�枚举里的标签需�大写。
 
@@ -638,7 +721,7 @@ mm_count)，和文件系统 (``struct super_block``: s_count å’Œ s_active) 中æ‰
 
 �的�字请用大写字�，�过形如函数的�的�字�以用�写字�。
 
-一般的，如果能写�内�函数就��写��函数的�。
+通常如果能写�内�函数就��写��函数的�。
 
 �有多个语�的�应该被包�在一个 do-while 代��里：
 
@@ -696,18 +779,18 @@ mm_count)，和文件系统 (``struct super_block``: s_count å’Œ s_active) 中æ‰
 		(ret);				\
 	})
 
-ret 是本地��的通用�字 - __foo_ret 更�容易与一个已存在的��冲�。
+ret 是本地��的通用�字—— __foo_ret 更�容易与一个已存在的��冲�。
 
 cpp 手册对�的讲解很详细。gcc internals 手册也详细讲解了 RTL，内核里的汇编语
 言�常用到它。
 
 
 13) 打�内核消�
-------------------------------
+----------------
 
-内核开�者应该是�过良好教育的。请一定注�内核信�的拼写，以给人以好的�象。
+内核开�者应该看起�有文化。请一定注�内核信�的拼写，以给人良好的�象。
 ��用�规范的��比如 ``dont``，而�用 ``do not`` 或者 ``don't`` 。��这些信
-�简�明了,无歧义。
+�简�明了�无歧义。
 
 内核信��必以英文��结�。
 
@@ -724,17 +807,18 @@ dev_info() 等等。对于那些�和�个特定设备相关连的信�，<li
 或设定了 CONFIG_DYNAMIC_DEBUG。实际这�样是为了 dev_dbg()，一个相关约定是在一
 个已�开�了 DEBUG 时，使用 VERBOSE_DEBUG �添加 dev_vdbg()。
 
-许多�系统拥有 Kconfig 调试选项�开� -DDEBUG 在对应的 Makefile 里�；在其他
+许多�系统拥有 Kconfig 调试选项�开�对应 Makefile 里�的 -DDEBUG；在其他
 情况下，特殊文件使用 #define DEBUG。当一�调试信�需�被无�件打�时，例如，
 如果已�包�一个调试相关的 #ifdef �件，printk(KERN_DEBUG ...) 就�被使用。
 
 
 14) 分�内存
-------------------------------
+------------
 
 内核�供了下�的一般用途的内存分�函数：
 kmalloc(), kzalloc(), kmalloc_array(), kcalloc(), vmalloc() 和 vzalloc()。
-请�考 API 文档以获�有关它们的详细信�。
+请�考 API 文档以获�有关它们的详细信�：
+Documentation/translations/zh_CN/core-api/memory-allocation.rst 。
 
 传递结构体大�的首选形�是这样的：
 
@@ -761,11 +845,13 @@ kmalloc(), kzalloc(), kmalloc_array(), kcalloc(), vmalloc() 和 vzalloc()。
 
 	p = kcalloc(n, sizeof(...), ...);
 
-两�形�检查分�大� n * sizeof(...) 的溢出，如果溢出返回 NULL。
+两�形�都会检查分� n * sizeof(...) 大�时内存的溢出，如果溢出返回 NULL。
 
+在没有 __GFP_NOWARN 的情况下使用时，这些通用分�函数都会在失败时�起堆栈转储，
+因此当返回NULL时，没有必��出�外的失败消�。
 
 15) 内�弊病
-------------------------------
+------------
 
 有一个常�的误解是 ``内�`` 是 gcc �供的�以让代��行更快的一个选项。虽然使
 用内�函数有时候是�当的 (比如作为一�替代�的方�，请看第�二章)，�过很多情
@@ -786,7 +872,7 @@ inline gcc 也�以自动使其内�。而且其他用户�能会�求移除
 
 
 16) 函数返回值�命�
-------------------------------
+--------------------
 
 函数�以返回多���类型的值，最常�的一�是表明函数执行�功或者失败的值。这样
 的一个值�以表示为一个错误代�整数 (-Exxx�失败，0��功) 或者一个 ``�功``
@@ -797,7 +883,7 @@ inline gcc 也�以自动使其内�。而且其他用户�能会�求移除
 产生这� bug，请�循下�的惯例::
 
 	如果函数的�字是一个动作或者强制性的命令，那么这个函数应该返回错误代
-	�整数。如果是一个判断，那么函数应该返回一个 "�功" 布尔值。
+	�整数。如果是一个判断，那么函数应该返回一个“�功�布尔值。
 
 比如， ``add work`` 是一个命令，所以 add_work() 在�功时返回 0，在失败时返回
 -EBUSY。类似的，因为 ``PCI device present`` 是一个判断，所以 pci_dev_present()
@@ -806,13 +892,35 @@ inline gcc 也�以自动使其内�。而且其他用户�能会�求移除
 所有 EXPORTed 函数都必须�守这个惯例，所有的公共函数也都应该如此。�有
 (static) 函数�需�如此，但是我们也推�这样�。
 
-返回值是实际计算结果而�是计算是��功的标志的函数��此惯例的�制。一般的，
+返回值是实际计算结果而�是计算是��功的标志的函数��此惯例的�制。通常
 他们通过返回一些正常值范围之外的结果�表示出错。典型的例�是返回指针的函数，
 他们使用 NULL 或者 ERR_PTR 机制�报告错误。
 
+17) 使用布尔类型
+----------------
+
+Linux内核布尔（bool）类型是C99 _Bool类型的别�。布尔值�能为0或1，而对布尔的
+��或显�转�将自动将值转�为true或false。在使用布尔类型时 **�需�** 构造，
+它会消除一类错误。
+
+使用布尔值时，应使用true和false定义，而�是1和0。
 
-17) ���新�明内核�
-------------------------------
+布尔函数返回类型和堆栈��总是�以在适当的时候使用。鼓励使用布尔��高�读性，
+并且布尔值在存储时通常比“int�更好。
+
+如果缓存行布局或值的大�很��，请��使用布尔，因为其大�和对�方�根�编译
+的体系结构而��。针对对�和大�进行优化的结构体�应使用布尔。
+
+如果一个结构体有多个true/false值，请考虑将它们�并为具有1比特�员的�域，或使
+用适当的固定宽度类型，如u8。
+
+类似地，对于函数�数，多个true/false值�以�并为�个按�的“标志��数，如果调
+用点具有裸true/false常�，“标志��数通常是更具�读性的替代方法。
+
+总之，在结构体和�数中有�地使用布尔�以�高�读性。
+
+18) ���新�明内核�
+----------------------
 
 头文件 include/linux/kernel.h 包�了一些�，你应该使用它们，而��自己写一些
 它们的��。比如，如果你需�计算一个数组的长度，使用这个�
@@ -832,11 +940,11 @@ inline gcc 也�以自动使其内�。而且其他用户�能会�求移除
 在你的代�里自己�新定义。
 
 
-18) 编辑器模�行和其他需�罗嗦的事情
---------------------------------------------------
+19) 编辑器模�行和其他需�罗嗦的事情
+------------------------------------
 
 有一些编辑器�以解释嵌入在�文件里的由一些特殊标记标明的�置信�。比如，emacs
-能够解释被标记�这样的行：
+能够解�被标记�这样的行：
 
 .. code-block:: c
 
@@ -852,7 +960,7 @@ inline gcc 也�以自动使其内�。而且其他用户�能会�求移除
 	End:
 	*/
 
-Vim 能够解释这样的标记：
+Vim 能够解�这样的标记：
 
 .. code-block:: c
 
@@ -863,8 +971,8 @@ Vim 能够解释这样的标记：
 的模�，或者使用其他�以产生正确的缩进的巧妙方法。
 
 
-19) 内�汇编
-------------------------------
+20) 内�汇编
+------------
 
 在特定架构的代�中，你�能需�内�汇编与 CPU 和平�相关功能连接。需�这么�时
 就��犹豫。然而，当 C �以完�工作时，��平白无故地使用内�汇编。在�能的情
@@ -880,8 +988,8 @@ Vim 能够解释这样的标记：
 移除了。你�必总是这样�，尽管，这�必�的举动会�制优化。
 
 在写一个包�多�指令的�个内�汇编语�时，把��指令用引�分割而且��一行，
-除了最�一�指令外，在�个指令结尾加上 \n\t，让汇编输出时�以正确地缩进下一�
-指令：
+除了最�一�指令外，在�个指令结尾加上 ``\n\t`` ，让汇编输出时�以正确地缩进
+下一�指令：
 
 .. code-block:: c
 
@@ -890,10 +998,10 @@ Vim 能够解释这样的标记：
 	     : /* outputs */ : /* inputs */ : /* clobbers */);
 
 
-20) �件编译
-------------------------------
+21) �件编译
+------------
 
-���能，就��在 .c 文件里�使用预处��件 (#if, #ifdef)；这样�让代�更难
+���能，就��在 .c 文件里�使用预处��件 (#if, #ifdef)；这样�会让代�更难
 阅读并且更难去跟踪逻辑。替代方案是，在头文件中用预处��件�供给那些 .c 文件
 使用，�给 #else �供一个空桩 (no-op stub) 版本，然�在 .c 文件内无�件地调用
 那些 (定义在头文件内的) 函数。这样�，编译器会��为桩函数 (stub) 的调用生�
@@ -904,8 +1012,8 @@ Vim 能够解释这样的标记：
 �件到这个辅助函数内。
 
 如果你有一个在特定�置中，�能��未使用的函数或��，编译器会警告它定义了但
-未使用，把它标记为 __maybe_unused 而�是将它包�在一个预处��件中。(然而，如
-果一个函数或��总是未使用，就直接删除它。)
+未使用，请把它标记为 __maybe_unused 而�是将它包�在一个预处��件中。(然而，
+如果一个函数或��总是未使用，就直接删除它。)
 
 在代�中，尽�能地使用 IS_ENABLED ��转化�个 Kconfig 标记为 C 的布尔
 表达�，并在一般的 C �件中使用它：
@@ -931,23 +1039,45 @@ Vim 能够解释这样的标记：
 	#endif /* CONFIG_SOMETHING */
 
 
-附录 I) �考
--------------------
+附录 I) �考资料
+----------------
 
-The C Programming Language, 第二版
+The C Programming Language, 2nd Edition
 作者：Brian W. Kernighan 和 Denni M. Ritchie.
 Prentice Hall, Inc., 1988.
-ISBN 0-13-110362-8 (软皮), 0-13-110370-9 (硬皮).
+ISBN 0-13-110362-8 (平装), 0-13-110370-9 (精装).
+
+.. note::
+
+    《C程�设计语言（第2版）》
+    作者：[美] Brian W. Kernighan / [美] Dennis M. Ritchie
+    译者：��文 / �志 / 尤晋元（审校）
+    出版社：机械工业出版社，2019
+    ISBN：9787111617945
 
 The Practice of Programming
 作者：Brian W. Kernighan 和 Rob Pike.
 Addison-Wesley, Inc., 1999.
 ISBN 0-201-61586-X.
 
+.. note::
+
+    《程�设计实践》
+    作者：[美] Brian W. Kernighan / [美] Rob Pike
+    出版社：机械工业出版社，2005
+    ISBN：9787111091578
+
+    《程�设计实践》
+    作者：[美] Brian W. Kernighan / Rob Pike
+    译者：裘宗燕
+    出版社：机械工业出版社，2000
+    ISBN：9787111075738
+
 GNU 手册 - �循 K&R 标准和此文本 - cpp, gcc, gcc internals and indent,
 都�以从 https://www.gnu.org/manual/ 找到
 
 WG14 是 C 语言的国际标准化工作组，URL: http://www.open-std.org/JTC1/SC22/WG14/
 
-Kernel process/coding-style.rst，作者 greg@kroah.com �表于 OLS 2002：
+内核文档 Documentation/process/coding-style.rst，
+作者 greg@kroah.com �表于 OLS 2002：
 http://www.kroah.com/linux/talks/ols_2002_kernel_codingstyle_talk/html/
diff --git a/Documentation/translations/zh_CN/process/email-clients.rst b/Documentation/translations/zh_CN/process/email-clients.rst
index 102023651118..34d51cdadc7b 100644
--- a/Documentation/translations/zh_CN/process/email-clients.rst
+++ b/Documentation/translations/zh_CN/process/email-clients.rst
@@ -1,17 +1,20 @@
-.. _cn_email_clients:
+.. SPDX-License-Identifier: GPL-2.0-or-later
 
 .. include:: ../disclaimer-zh_CN.rst
 
-:Original: :ref:`Documentation/process/email-clients.rst <email_clients>`
+.. _cn_email_clients:
 
-译者::
+:Original: Documentation/process/email-clients.rst
 
-        中文版维护者： 贾��  Harry Wei <harryxiyou@gmail.com>
-        中文版翻译者： 贾��  Harry Wei <harryxiyou@gmail.com>
-                       时奎亮  Alex Shi <alex.shi@linux.alibaba.com>
-        中文版校译者： Yinglin Luan <synmyth@gmail.com>
-        	       Xiaochen Wang <wangxiaochen0@gmail.com>
-                       yaxinsn <yaxinsn@163.com>
+:译者:
+ - 贾��  Harry Wei <harryxiyou@gmail.com>
+ - 时奎亮  Alex Shi <alexs@kernel.org>
+ - �想�  Wu XiangCheng <bobwxc@email.cn>
+
+:校译:
+ - Yinglin Luan <synmyth@gmail.com>
+ - Xiaochen Wang <wangxiaochen0@gmail.com>
+ - yaxinsn <yaxinsn@163.com>
 
 Linux邮件客户端�置信�
 =======================
@@ -27,12 +30,17 @@ Git
 改日志。如果工作正常，�将补���到相应的邮件列表。
 
 
-普通�置
+通用�置
 --------
+
 Linux内核补�是通过邮件被�交的，最好把补�作为邮件体的内嵌文本。有些维护者
 接收附件，但是附件的内容格�应该是"text/plain"。然而，附件一般是�赞�的，
 因为这会使补�的引用部分在评论过程中�的很困难。
 
+�时也强烈建议在补�或其他邮件的正文中使用纯文本格�。https://useplaintext.email
+有助于了解如何�置你喜欢的邮件客户端，并在您还没有首选的情况下列出一些推�的
+客户端。
+
 用���Linux内核补�的邮件客户端在��补�时应该处于文本的原始状�。例如，
 他们�能改�或者删除制表符或者空格，甚至是在�一行的开头或者结尾。
 
@@ -40,17 +48,17 @@ Linux内核补�是通过邮件被�交的，最好把补�作为邮件体的
 
 ��让你的邮件客户端进行自动�行。这样也会破�你的补�。
 
-邮件客户端�能改�文本的字符集编�方�。���的补��能是ASCII或者UTF-8编�方�，
-如果你使用UTF-8编�方���邮件，那么你将会��一些�能�生的字符集问题。
+邮件客户端�能改�文本的字符集编�方�。���的补��能是ASCII或者UTF-8编�
+方�，如果你使用UTF-8编�方���邮件，那么你将会��一些�能�生的字符集问题。
 
-邮件客户端应该形�并且�� References: 或者 In-Reply-To: 标题，那么
-邮件�题就�会中断。
+邮件客户端应该生�并且��“References:�或者“In-Reply-To:�邮件头，这样邮件会�
+就�会中断。
 
-�制粘帖(或者剪贴粘帖)通常�能用于补�，因为制表符会转�为空格。使用xclipboard, xclip
-或者xcutsel也许�以，但是最好测试一下或者��使用�制粘帖。
+�制粘帖(或者剪贴粘帖)通常�能用于补�，因为制表符会转�为空格。使用xclipboard,
+xclip或者xcutsel也许�以，但是最好测试一下或者��使用�制粘帖。
 
-��在使用PGP/GPG署�的邮件中包�补�。这样会使得很多脚本�能读�和适用于你的补�。
-（这个问题应该是�以修�的）
+��在使用PGP/GPG签�的邮件中包�补�。这样会使得很多脚本�能读�和适用于你的
+补�。（这个问题应该是�以修�的）
 
 在给内核邮件列表��补�之�，给自己��一个补�是个�错的主�，�存接收到的
 邮件，将补�用'patch'命令打上，如果�功了，�给内核邮件列表��。
@@ -58,98 +66,133 @@ Linux内核补�是通过邮件被�交的，最好把补�作为邮件体的
 
 一些邮件客户端�示
 ------------------
+
 这里给出一些详细的MUA�置�示，�以用于给Linux内核��补�。这些并��味是
 所有的软件包�置总结。
 
 说明：
-TUI = 以文本为基础的用户接�
-GUI = 图形界�用户接�
+
+- TUI = 以文本为基础的用户接�
+- GUI = 图形界�用户接�
 
 Alpine (TUI)
-~~~~~~~~~~~~
+************
 
 �置选项：
-在"Sending Preferences"部分：
 
-- "Do Not Send Flowed Text"必须开�
-- "Strip Whitespace Before Sending"必须关闭
+在 :menuselection:`Sending Preferences` ��：
+
+- :menuselection:`Do Not Send Flowed Text` 必须开�
+- :menuselection:`Strip Whitespace Before Sending` 必须关闭
+
+当写邮件时，光标应该放在补�会出现的地方，然�按下 :kbd:`CTRL-R` 组�键，使指
+定的补�文件嵌入到邮件中。
+
+Claws Mail (GUI)
+****************
+
+�以用，有人用它�功地�过补�。
 
-当写邮件时，光标应该放在补�会出现的地方，然�按下CTRL-R组�键，使指定的
-补�文件嵌入到邮件中。
+用 :menuselection:`Message-->Insert File` (:kbd:`CTRL-I`) 或外置编辑器�入补�。
+
+若�在Claws编辑窗��修改�入的补�，需关闭
+:menuselection:`Configuration-->Preferences-->Compose-->Wrapping`
+的 `Auto wrapping` 。
 
 Evolution (GUI)
-~~~~~~~~~~~~~~~
+***************
 
-一些开�者�功的使用它��补�
+一些开�者�功的使用它��补�。
 
-当选择邮件选项：Preformat
-  从Format->Heading->Preformatted (Ctrl-7)或者工具�
+撰写邮件时：
+从 :menuselection:`格�-->段�样�-->预格�化` (:kbd:`CTRL-7`)
+或工具�选择 :menuselection:`预格�化` ；
 
 然�使用：
-  Insert->Text File... (Alt-n x)�入补�文件。
+:menuselection:`�入-->文本文件...` (:kbd:`ALT-N x`) �入补�文件。
 
-你还�以"diff -Nru old.c new.c | xclip"，选择Preformat，然�使用中间键进行粘帖。
+你还�以 ``diff -Nru old.c new.c | xclip`` ，选择 :menuselection:`预格�化` ，
+然�使用鼠标中键进行粘帖。
 
 Kmail (GUI)
-~~~~~~~~~~~
+***********
 
 一些开�者�功的使用它��补�。
 
-默认设置�为HTML格�是�适的；���用它。
+默认撰写设置�用HTML格�是�适的；���用它。
 
-当书写一�邮件的时候，在选项下���选择自动�行。唯一的缺点就是你在邮件中输入的任何文本
-都�会被自动�行，因此你必须在��补�之�手动�行。最简�的方法就是�用自动�行�书写邮件，
-然�把它�存为�稿。一旦你在�稿中�次打开它，它已�全部自动�行了，那么你的邮件虽然没有
-选择自动�行，但是还�会失去已有的自动�行。
+当书写一�邮件的时候，在选项下���选择自动�行。唯一的缺点就是你在邮件中输
+入的任何文本都�会被自动�行，因此你必须在��补�之�手动�行。最简�的方法
+就是�用自动�行�书写邮件，然�把它�存为�稿。一旦你在�稿中�次打开它，它
+已�全部自动�行了，那么你的邮件虽然没有选择自动�行，但是还�会失去已有的自
+动�行。
 
-在邮件的底部，�入补�之�，放上常用的补�定界符：三个连字�(---)。
+在邮件的底部，�入补�之�，放上常用的补�定界符：三个连字符(``---``)。
 
-然�在"Message"���目，选择�入文件，接�选�你的补�文件。还有一个�外的选项，你�以
-通过它�置你的邮件建立工具���，还�以带上"insert file"图标。
+然�在 :menuselection:`信件` ��，选择 :menuselection:`�入文本文件` ，接
+�选�你的补�文件。还有一个�外的选项，你�以通过它�置你的创建新邮件工具�，
+加上 :menuselection:`�入文本文件` 图标。
 
-你�以安全地通过GPG标记附件，但是内嵌补�最好��使用GPG标记它们。作为内嵌文本的签�补�，
-当从GPG中��7�编�时会使他们�的更加��。
+将编辑器窗�拉到足够宽��折行。对于KMail 1.13.5 (KDE 4.5.4)，它会在��邮件
+时对编辑器窗�中显示折行的地方自动�行。在选项��中�消自动�行��能解决。
+因此，如果你的补�中有�常长的行，必须在��之�把编辑器窗�拉得�常宽。
+��：https://bugs.kde.org/show_bug.cgi?id=174034
 
-如果你��以附件的形���补�，那么就�键点击附件，然�选中属性，�出"Suggest automatic
-display"，这样内嵌附件更容易让读者看到。
+你�以安全地用GPG签�附件，但是内嵌补�最好��使用GPG签�它们。作为内嵌文本
+�入的签�补�将使其难以从7-bit编�中��。
 
-当你��存将���的内嵌文本补�，你�以从消�列表窗格选择包�补�的邮件，然��击选择
-"save as"。你�以使用一个没有更改的包�补�的邮件，如果它是以正确的形�组�。当你正真在它
-自己的窗�之下察看，那时没有选项�以�存邮件--已�有一个这样的bug被汇报到了kmail的bugzilla
-并且希望这将会被处�。邮件是以�针对�个用户�读写的��被�存的，所以如果你想把邮件�制到其他地方，
-你�得�把他们的��改为组或者整体�读。
+如果你��以附件的形���补�，那么就�键点击附件，然�选择
+:menuselection:`属性` ，打开 :menuselection:`建议自动显示` ，使附件内�更容
+易让读者看到。
+
+当你��存将���的内嵌文本补�，你�以从消�列表窗格选择包�补�的邮件，然
+��键选择 :menuselection:`�存为` 。如果整个电�邮件的组�正确，您�直接将
+其作为补�使用。电�邮件以当�用户�读写���存，因此您必须 ``chmod`` ，以
+使其在�制到别处时用户组和其他人�读。
 
 Lotus Notes (GUI)
-~~~~~~~~~~~~~~~~~
+*****************
 
 ��使用它。
 
+IBM Verse (Web GUI)
+*******************
+
+�上�。
+
 Mutt (TUI)
-~~~~~~~~~~
+**********
+
+很多Linux开�人员使用mutt客户端，这�明它肯定工作得�常漂亮。
 
-很多Linux开�人员使用mutt客户端，所以�明它肯定工作的�常漂亮。
+Mutt�自带编辑器，所以�管你使用什么编辑器，�自动断行就行。大多数编辑器都有
+:menuselection:`�入文件` 选项，它�以在�改�文件内容的情况下�入文件。
 
-Mutt�自带编辑器，所以�管你使用什么编辑器都�应该带有自动断行。大多数编辑器都带有
-一个"insert file"选项，它�以通过�改�文件内容的方��入文件。
+用 ``vim`` 作为mutt的编辑器::
 
-'vim'作为mutt的编辑器：
   set editor="vi"
 
-  如果使用xclip，敲入以下命令
+如果使用xclip，敲入以下命令::
+
   :set paste
-  按中键之�或者shift-insert或者使用
+
+然��按中键或者shift-insert或者使用::
+
   :r filename
 
-如果想�把补�作为内嵌文本。
-(a)ttach工作的很好，�带有"set paste"。
+把补��入为内嵌文本。
+在未设置  ``set paste`` 时(a)ttach工作的很好。
 
 你�以通过 ``git format-patch`` 生�补�，然�用 Mutt��它们::
 
-        $ mutt -H 0001-some-bug-fix.patch
+    $ mutt -H 0001-some-bug-fix.patch
 
 �置选项：
+
 它应该以默认设置的形�工作。
-然而，把"send_charset"设置为"us-ascii::utf-8"也是一个�错的主�。
+然而，把 ``send_charset`` 设置一下也是一个�错的主�::
+
+  set send_charset="us-ascii:utf-8"
 
 Mutt 是高度��置的。 这里是个使用mutt通过 Gmail ��的补�的最��置::
 
@@ -178,71 +221,107 @@ Mutt 是高度��置的。 这里是个使用mutt通过 Gmail ��的补�
   set from = "username@gmail.com"
   set use_from = yes
 
-Mutt文档�有更多信�:
+Mutt文档�有更多信�：
 
-    http://dev.mutt.org/trac/wiki/UseCases/Gmail
+    https://gitlab.com/muttmua/mutt/-/wikis/UseCases/Gmail
 
-    http://dev.mutt.org/doc/manual.html
+    http://www.mutt.org/doc/manual/
 
 Pine (TUI)
-~~~~~~~~~~
+**********
 
 Pine过去有一些空格删�问题，但是这些现在应该都被修�了。
 
-如果�以，请使用alpine(pine的继承者)
+如果�以，请使用alpine（pine的继承者）。
 
 �置选项：
-- 最近的版本需�消除�程文本
-- "no-strip-whitespace-before-send"选项也是需�的。
+
+- 最近的版本需� ``quell-flowed-text``
+- ``no-strip-whitespace-before-send`` 选项也是需�的。
 
 
 Sylpheed (GUI)
-~~~~~~~~~~~~~~
+**************
 
 - 内嵌文本�以很好的工作（或者使用附件）。
 - �许使用外部的编辑器。
-- 对于目录较多时�常慢。
+- 收件箱较多时�常慢。
 - 如果通过non-SSL连接，无法使用TLS SMTP授�。
-- 在组�窗�中有一个很有用的ruler bar。
-- 给地�本中添加地�就�会正确的了解显示�。
+- 撰写窗�的标尺很有用。
+- 将地�添加到通讯簿时无法正确�解显示的�称。
 
 Thunderbird (GUI)
-~~~~~~~~~~~~~~~~~
+*****************
+
+Thunderbird是Outlook的克隆版本，它很容易��文本，但也有一些方法强制修正。
+
+在完�修改�（包括安装扩展），您需��新�动Thunderbird。
+
+- �许使用外部编辑器：
+
+  使用Thunderbird�补�最简�的方法是使用扩展�打开您最喜欢的外部编辑器。
+
+  下�是一些能够�到这一点的扩展样例。
+
+  - “External Editor Revived�
+
+    https://github.com/Frederick888/external-editor-revived
+
+    https://addons.thunderbird.net/en-GB/thunderbird/addon/external-editor-revived/
+
+    它需�安装“本地消�主机（native messaging host）�。
+    ��以下文档:
+    https://github.com/Frederick888/external-editor-revived/wiki
+
+  - “External Editor�
+
+    https://github.com/exteditor/exteditor
+
+    下载并安装此扩展，然�打开 :menuselection:`新建消�` 窗�, 用
+    :menuselection:`查看-->工具�-->自定义...` 给它增加一个按钮，直接点击此
+    按钮��使用外置编辑器。
+
+    请注�，“External Editor��求你的编辑器�能fork，���说，编辑器必须在
+    关闭��返回。你�能需�传递�外的�数或修改编辑器设置。最值得注�的是，
+    如果您使用的是gvim，那么您必须将 :menuselection:`external editor` 设置的
+    编辑器字段设置为 ``/usr/bin/gvim --nofork"`` （�设�执行文件在
+    ``/usr/bin`` ），以传递 ``-f`` �数。如果您正在使用其他编辑器，请阅读其
+    手册了解如何处�。
 
-默认情况下，thunderbird很容易��文本，但是还有一些方法�以强制它�得更好。
+若�修正内部编辑器，请执行以下�作：
 
-- 在用户��设置里，组�和寻�，��选择"Compose messages in HTML format"。
+- 修改你的Thunderbird设置，��使用 ``format=flowed`` �
+  回到主窗�，按照
+  :menuselection:`主��-->首选项-->常规-->�置编辑器...`
+  打开Thunderbird的�置编辑器。
 
-- 编辑你的Thunderbird�置设置�使它��拆行使用：user_pref("mailnews.wraplength", 0);
+  - 将 ``mailnews.send_plaintext_flowed`` 设为 ``false``
 
-- 编辑你的Thunderbird�置设置，使它��使用"format=flowed"格�：user_pref("mailnews.
-  send_plaintext_flowed", false);
+  - 将 ``mailnews.wraplength`` 从 ``72`` 改为 ``0``
 
-- 你需�使Thunderbird�为预先格�方�：
-  如果默认情况下你书写的是HTML格�，那�是很难。仅仅从标题�的下拉框中选择"Preformat"格�。
-  如果默认情况下你书写的是文本格�，你�得把它改为HTML格�（仅仅作为一次性的）�书写新的消�，
-  然�强制使它回到文本格�，�则它就会拆行。�实现它，在写信的图标上使用shift键�使它�为HTML
-  格�，然�标题�的下拉框中选择"Preformat"格�。
+- ��写HTML邮件�
+  回到主窗�，打开
+  :menuselection:`主��-->账户设置-->你的@邮件.地�-->通讯录/编写&地�簿` ，
+  关掉 ``以HTML格�编写消�`` 。
 
-- �许使用外部的编辑器：
-  针对Thunderbird打补�最简�的方法就是使用一个"external editor"扩展，然�使用你最喜欢的
-  $EDITOR�读�或者�并补�到文本中。�实现它，�以下载并且安装这个扩展，然�添加一个使用它的
-  按键View->Toolbars->Customize...最�当你书写信�的时候仅仅点击它就�以了。
+- �用纯文本格�查看邮件�
+  回到主窗�， :menuselection:`主��-->查看-->消�体为-->纯文本` �
 
 TkRat (GUI)
-~~~~~~~~~~~
+***********
 
 �以使用它。使用"Insert file..."或者外部的编辑器。
 
 Gmail (Web GUI)
-~~~~~~~~~~~~~~~
+***************
 
 ��使用它��补�。
 
 Gmail网页客户端自动地把制表符转�为空格。
 
-虽然制表符转�为空格问题�以被外部编辑器解决，�时它还会使用回车�行把�行拆分为78个字符。
+虽然制表符转�为空格问题�以被外部编辑器解决，但它�时还会使用回车�行把�行
+拆分为78个字符。
 
-�一个问题是Gmail还会把任何�是ASCII的字符的信�改为base64编�。它把东西�的�欧洲人的�字。
+�一个问题是Gmail还会把任何�有�ASCII的字符的消�改用base64编�，如欧洲人的
+�字。
 
-                                ###
diff --git a/Documentation/translations/zh_CN/process/submit-checklist.rst b/Documentation/translations/zh_CN/process/submit-checklist.rst
index a64858d321fc..3d6ee21c74ae 100644
--- a/Documentation/translations/zh_CN/process/submit-checklist.rst
+++ b/Documentation/translations/zh_CN/process/submit-checklist.rst
@@ -1,105 +1,111 @@
 .. include:: ../disclaimer-zh_CN.rst
 
-:Original: :ref:`Documentation/process/submit-checklist.rst <submitchecklist>`
-:Translator: Alex Shi <alex.shi@linux.alibaba.com>
+:Original: Documentation/process/submit-checklist.rst
+:Translator:
+ - Alex Shi <alexs@kernel.org>
+ - Wu XiangCheng <bobwxc@email.cn>
 
 .. _cn_submitchecklist:
 
-Linux内核补��交清�
-~~~~~~~~~~~~~~~~~~~~~
+Linux内核补��交检查�
+~~~~~~~~~~~~~~~~~~~~~~~
 
 如果开�人员希望看到他们的内核补��交更快地被接�，那么他们应该�一些基本
 的事情。
 
-这些都是在
-:ref:`Documentation/translations/zh_CN/process/submitting-patches.rst <cn_submittingpatches>`
+这些都是在 Documentation/translations/zh_CN/process/submitting-patches.rst
 和其他有关�交Linux内核补�的文档中�供的。
 
-1) 如果使用工具，则包括定义/声明该工具的文件。���赖于其他头文件拉入您使用
+1) 如果使用工具，则包括定义/声明该工具的文件。���赖其他头文件�引入您使用
    的头文件。
 
 2) 干净的编译：
 
-   a) 使用适用或修改的 ``CONFIG`` 选项 ``=y``�``=m`` 和 ``=n`` 。没有GCC
+   a) 使用�适的 ``CONFIG`` 选项 ``=y``�``=m`` 和 ``=n`` 。没有 ``gcc``
       警告/错误，没有链接器警告/错误。
 
-   b) 通过allnoconfig�allmodconfig
+   b) 通过 ``allnoconfig`` � ``allmodconfig``
 
    c) 使用 ``O=builddir`` 时�以�功编译
 
-3) 通过使用本地交�编译工具或其他一些构建场在多个CPU体系结构上构建。
+   d) 任何 Doucmentation/ 下的�更都能�功构建且�引入新警告/错误。
+      用 ``make htmldocs`` 或 ``make pdfdocs`` 检验构建情况并修�问题。
+
+3) 通过使用本地交�编译工具或其他一些构建设施在多个CPU体系结构上构建。
 
 4) PPC64是一�很好的交�编译检查体系结构，因为它倾�于对64�的数使用无符�
    长整型。
 
-5) 如下所述 :ref:`Documentation/translations/zh_CN/process/coding-style.rst <cn_codingstyle>`.
-   检查您的补�是�为常规样�。在�交（ ``scripts/check patch.pl`` ）之�，
-   使用补�样�检查器检查是�有轻微的冲�。您应该能够处�您的补�中存在的所有
+5) 按 Documentation/translations/zh_CN/process/coding-style.rst 所述检查您的
+   补�是�为常规样�。在�交之�使用补�样�检查器 ``scripts/checkpatch.pl``
+   检查是�有轻微的冲�。您应该能够处�您的补�中存在的所有
    �规行为。
 
-6) 任何新的或修改过的 ``CONFIG`` 选项都�会弄��置��，并默认为关闭，除�
-   它们符� ``Documentation/kbuild/kconfig-language.rst`` 中记录的异常�件,
-   ��属性：默认值.
+6) 任何新的或修改过的 ``CONFIG`` 选项都�应�乱�置��，并默认为关闭，除�
+   它们符� ``Documentation/kbuild/kconfig-language.rst`` ��属性：默认值中
+   记录的例外�件。
 
 7) 所有新的 ``kconfig`` 选项都有帮助文本。
 
 8) 已仔细审查了相关的 ``Kconfig`` 组�。这很难用测试�纠正——脑力在这里是有
    回报的。
 
-9) 用 sparse 检查干净。
+9) 通过 sparse 清查。
+   （�� Documentation/translations/zh_CN/dev-tools/sparse.rst ）
 
 10) 使用 ``make checkstack`` 和 ``make namespacecheck`` 并修�他们�现的任何
     问题。
 
     .. note::
 
-        ``checkstack`` 并没有明确指出问题，但是任何一个在堆栈上使用超过512
+        ``checkstack`` 并�会明确指出问题，但是任何一个在堆栈上使用超过512
         字节的函数都�以进行更改。
 
-11) 包括 :ref:`kernel-doc <kernel_doc>` 内核文档以记录全局内核API。（��函数
-    �需�，但也�以。）使用 ``make htmldocs`` 或 ``make pdfdocs`` 检查
-    :ref:`kernel-doc <kernel_doc>` 并修�任何问题。
+11) 包括 :ref:`kernel-doc <kernel_doc_zh>` 内核文档以记录全局内核API。（��
+    函数�需�，但也�以。）使用 ``make htmldocs`` 或 ``make pdfdocs`` 检查
+    :ref:`kernel-doc <kernel_doc_zh>` 并修�任何问题。
 
-12) 通过以下选项�时�用的测试 ``CONFIG_PREEMPT``, ``CONFIG_DEBUG_PREEMPT``,
+12) 通过以下选项�时�用的测试： ``CONFIG_PREEMPT``, ``CONFIG_DEBUG_PREEMPT``,
     ``CONFIG_DEBUG_SLAB``, ``CONFIG_DEBUG_PAGEALLOC``, ``CONFIG_DEBUG_MUTEXES``,
     ``CONFIG_DEBUG_SPINLOCK``, ``CONFIG_DEBUG_ATOMIC_SLEEP``,
-    ``CONFIG_PROVE_RCU`` and ``CONFIG_DEBUG_OBJECTS_RCU_HEAD``
-
-13) 已�过构建和�行时测试，包括有或没有 ``CONFIG_SMP``, ``CONFIG_PREEMPT``.
+    ``CONFIG_PROVE_RCU`` 和 ``CONFIG_DEBUG_OBJECTS_RCU_HEAD`` 。
 
-14) 如果补�程�影�IO/�盘等：使用或�使用 ``CONFIG_LBDAF`` 进行测试。
+13) 在 ``CONFIG_SMP``, ``CONFIG_PREEMPT`` 开�和关闭的情况下都进行构建和�行
+    时测试。
 
-15) 所有代�路径都已在�用所有lockdep功能的情况下�行。
+14) 所有代�路径都已在�用所有死�检测（lockdep）功能的情况下�行。
 
-16) 所有新的/proc�目都记录在 ``Documentation/``
+15) 所有新的 ``/proc`` �目都记录在 ``Documentation/``
 
-17) 所有新的内核引导�数都记录在
+16) 所有新的内核引导�数都记录在
     Documentation/admin-guide/kernel-parameters.rst 中。
 
-18) 所有新的模��数都记录在 ``MODULE_PARM_DESC()``
+17) 所有新的模��数都记录在 ``MODULE_PARM_DESC()``
 
-19) 所有新的用户空间接�都记录在 ``Documentation/ABI/`` 中。有关详细信�，
+18) 所有新的用户空间接�都记录在 ``Documentation/ABI/`` 中。有关详细信�，
     请�阅 ``Documentation/ABI/README`` 。更改用户空间接�的补�应该抄�
     linux-api@vger.kernel.org。
 
-20) 已通过至少注入slab和page分�失败进行检查。请�阅 ``Documentation/fault-injection/``
+19) 已通过至少注入slab和page分�失败进行检查。请�阅 ``Documentation/fault-injection/`` 。
     如果新代�是实质性的，那么添加�系统特定的故障注入�能是�适的。
 
-21) 新添加的代�已�用 ``gcc -W`` 编译（使用 ``make EXTRA-CFLAGS=-W`` ）。这
+20) 新添加的代�已�用 ``gcc -W`` 编译（使用 ``make EXTRA-CFLAGS=-W`` ）。这
     将产生大�噪声，但对于查找诸如“警告：有符�和无符�之间的比较�之类的错误
     很有用。
 
-22) 在它被�并到-mm补�集中之�进行测试，以确�它�然与所有其他排队的补�以
+21) 在它被�并到-mm补�集中之�进行测试，以确�它�然与所有其他排队的补�以
     �VM�VFS和其他�系统中的��更改一起工作。
 
-23) 所有内存�障例如 ``barrier()``, ``rmb()``, ``wmb()`` 都需��代�中的注
+22) 所有内存�障（例如 ``barrier()``, ``rmb()``, ``wmb()`` ）都需��代�注
     释�解释它们正在执行的�作�其原因的逻辑。
 
-24) 如果补�添加了任何ioctl，那么也�更新 ``Documentation/userspace-api/ioctl/ioctl-number.rst``
+23) 如果补�添加了任何ioctl，那么也�更新
+    ``Documentation/userspace-api/ioctl/ioctl-number.rst`` 。
 
-25) 如果修改�的�代��赖或使用与以下 ``Kconfig`` 符�相关的任何内核API或
+24) 如果修改�的�代��赖或使用与以下 ``Kconfig`` 符�相关的任何内核API或
     功能，则在�用相关 ``Kconfig`` 符�和/或 ``=m`` （如果该选项�用）的情况
     下测试以下多个构建[并�所有这些都�时存在，�是它们的��/�机组�]：
 
-    ``CONFIG_SMP``, ``CONFIG_SYSFS``, ``CONFIG_PROC_FS``, ``CONFIG_INPUT``, ``CONFIG_PCI``, ``CONFIG_BLOCK``, ``CONFIG_PM``, ``CONFIG_MAGIC_SYSRQ``,
-    ``CONFIG_NET``, ``CONFIG_INET=n`` (但是�者伴� ``CONFIG_NET=y``).
+    ``CONFIG_SMP``, ``CONFIG_SYSFS``, ``CONFIG_PROC_FS``, ``CONFIG_INPUT``,
+    ``CONFIG_PCI``, ``CONFIG_BLOCK``, ``CONFIG_PM``, ``CONFIG_MAGIC_SYSRQ``,
+    ``CONFIG_NET``, ``CONFIG_INET=n`` （但是最�一个需� ``CONFIG_NET=y`` ）。
diff --git a/Documentation/translations/zh_CN/process/submitting-patches.rst b/Documentation/translations/zh_CN/process/submitting-patches.rst
index ebb7f37575c1..f8978f02057c 100644
--- a/Documentation/translations/zh_CN/process/submitting-patches.rst
+++ b/Documentation/translations/zh_CN/process/submitting-patches.rst
@@ -1,142 +1,92 @@
-.. _cn_submittingpatches:
+.. SPDX-License-Identifier: GPL-2.0-or-later
 
 .. include:: ../disclaimer-zh_CN.rst
 
-:Original: :ref:`Documentation/process/submitting-patches.rst <submittingpatches>`
+.. _cn_submittingpatches:
+
+:Original: Documentation/process/submitting-patches.rst
 
-译者::
+:译者:
+ - 钟宇 TripleX Chung <xxx.phy@gmail.com>
+ - 时奎亮 Alex Shi <alexs@kernel.org>
+ - �想� Wu XiangCheng <bobwxc@email.cn>
 
-        中文版维护者： 钟宇 TripleX Chung <xxx.phy@gmail.com>
-        中文版翻译者： 钟宇 TripleX Chung <xxx.phy@gmail.com>
-                       时奎亮 Alex Shi <alex.shi@linux.alibaba.com>
-        中文版校译者： �阳 Li Yang <leoyang.li@nxp.com>
-                       王� Wang Cong <xiyou.wangcong@gmail.com>
+:校译:
+ - �阳 Li Yang <leoyang.li@nxp.com>
+ - 王� Wang Cong <xiyou.wangcong@gmail.com>
 
 
-如何让你的改动进入内核
-======================
+�交补�：如何让你的改动进入内核
+================================
 
 对于想�将改动�交到 Linux 内核的个人或者公��说，如果�熟悉“规矩�，
-�交的�程会让人�惧。本文档收集了一系列建议，这些建议�以大大的�高你
+�交的�程会让人�惧。本文档包�了一系列建议，�以大大�高你
 的改动被接�的机会.
 
-以下文档�有大�简�的建议， 具体请�：
-:ref:`Documentation/process <development_process_main>`
-�样，:ref:`Documentation/translations/zh_CN/process/submit-checklist.rst <cn_submitchecklist>`
-给出在�交代��需�检查的项目的列表。
+本文档以较为简�的行文给出了大�建议。关于内核开��程如何进行的详细信�，
+��： Documentation/translations/zh_CN/process/development-process.rst 。
+Documentation/translations/zh_CN/process/submit-checklist.rst 给出了一系列
+�交补�之��检查的事项。设备树相关的补�，请�阅
+Documentation/devicetree/bindings/submitting-patches.rst 。
 
-其中许多步骤�述了Git版本控制系统的默认行为；如果您使用Git�准备补�，
-您将�现它为您完�的大部分机械工作，尽管您�然需�准备和记录一组��的
-补�。一般�说，使用git将使您作为内核开�人员的生活更轻�。
+本文档�设您正在使用 ``git`` 准备你的补�。如果您�熟悉 ``git`` ，最好学习
+如何使用它，这将使您作为内核开�人员的生活�得更加轻�。
 
+部分�系统和维护人员的树有一些关于其工作�程和�求的�外信�，请�阅
+Documentation/process/maintainer-handbooks.rst 。
 
-0) 获�当���树
------------------
+获�当���树
+--------------
 
-如果您没有一个�以使用当�内核�代�的存储库，请使用git获�一个。您将�
-从主线存储库开始，它�以通过以下方�获�::
+如果您手头没有当�内核�代�的存储库，请使用 ``git`` 获�一份。您需�先获�
+主线存储库，它�以通过以下命令拉�::
 
-        git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
+    git clone git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
 
-但是，请注�，您�能�希望直接针对主线树进行开�。大多数�系统维护人员�
+但是，请注�，您�能�想直接针对主线树进行开�。大多数�系统维护人员�
 行自己的树，并希望看到针对这些树准备的补�。请��MAINTAINERS文件中�系
-统的 **T:** 项以查找该树，或者简�地询问维护者该树是�未在其中列出。
-
-�然�以通过tarballs下载内核版本（如下一节所述），但这是进行内核开�的
-一�困难的方�。
-
-1) "diff -up"
--------------
-
-使用 "diff -up" 或者 "diff -uprN" �创建补�。
-
-所有内核的改动，都是以补�的形�呈现的，补�由 diff(1) 生�。创建补�的
-时候，�确认它是以 "unified diff" 格�创建的，这�格�由 diff(1) 的 '-u'
-�数生�。而且，请使用 '-p' �数，那样会显示�个改动所在的C函数，使得
-产生的补�容易读得多。补�应该基于内核�代�树的根目录，而�是里边的任
-何�目录。
-
-为一个�独的文件创建补�，一般�说这样�就够了::
-
-        SRCTREE=linux
-        MYFILE=drivers/net/mydriver.c
+统的 **T:** 项以查找该树，或者直接询问维护者该树是�未在其中列出。
 
-        cd $SRCTREE
-        cp $MYFILE $MYFILE.orig
-        vi $MYFILE      # make your change
-        cd ..
-        diff -up $SRCTREE/$MYFILE{.orig,} > /tmp/patch
+.. _zh_describe_changes:
 
-为多个文件创建补�，你�以解开一个没有修改过的内核�代�树，然�和你自
-己的代�树之间� diff 。例如::
-
-        MYSRC=/devel/linux
-
-        tar xvfz linux-3.19.tar.gz
-        mv linux-3.19 linux-3.19-vanilla
-        diff -uprN -X linux-3.19-vanilla/Documentation/dontdiff \
-                linux-3.19-vanilla $MYSRC > /tmp/patch
-
-"dontdiff" 是内核在编译的时候产生的文件的列表，列表中的文件在 diff(1)
-产生的补�里会被跳过。
-
-确定你的补�里没有包�任何�属于这次补��交的�外文件。记得在用diff(1)
-生�补�之�，审阅一次补�，以确�准确。
-
-如果你的改动很散乱，你应该研究一下如何将补�分割�独立的部分，将改动分
-割�一系列�乎逻辑的步骤。这样更容易让其他内核开�者审核，如果你想你的
-补�被接�，这是很��的。请�阅：
-:ref:`cn_split_changes`
-
-如果你用 ``git`` , ``git rebase -i`` �以帮助你这一点。如果你�用 ``git``,
-``quilt`` <https://savannah.nongnu.org/projects/quilt> �外一个�行的选择。
-
-.. _cn_describe_changes:
-
-2) �述你的改动
----------------
+�述你的改动
+------------
 
 �述你的问题。无论您的补�是一行错误修�还是5000行新功能，都必须有一个潜在
-的问题激励您完�这项工作。让审稿人相信有一个问题值得解决，让他们读完第一段
-是有�义的。
+的问题激励您完�这项工作。说�审阅者相信有一个问题值得解决，让他们读完第一段
+�就能明白这一点。
 
 �述用户��的影�。直接崩溃和�定是相当有说�力的，但并�是所有的错误都那么
-明目张胆。�使在代�审查期间�现了这个问题，也��述一下您认为它�能对用户产
+明目张胆。�使在代�审阅期间�现了这个问题，也��述一下您认为它�能对用户产
 生的影�。请记�，大多数Linux安装�行的内核�自二级稳定树或特定于供应商/产�
 的树，�从上游精选特定的补�，因此请包�任何�以帮助您将更改定�到下游的内容：
 触�的场景�DMESG的摘录�崩溃�述�性能回归�延迟尖峰��定等。
 
-�化优化和�衡。如果您声称在性能�内存消耗�堆栈�用空间或二进制大�方�有所
-改进，请包括支�它们的数字。但也��述�明显的�本。优化通常�是�费的，而是
-在CPU�内存和�读性之间进行�衡；或者，探索性的工作，在��的工作负载之间进
+质�优化和�衡。如果您声称在性能�内存消耗�堆栈�用空间或二进制大�方�有所
+改进，请包括支�它们的数�。但也��述�明显的�本。优化通常�是零�本的，而是
+在CPU�内存和�读性之间进行�衡；或者，�探索性的工作，在��的工作负载之间进
 行�衡。请�述优化的预期缺点，以便审阅者�以�衡�本和收益。
 
-一旦问题建立起�，就�详细地�述一下您实际在�什么。对于审阅者�说，用简�的
-英语�述代�的�化是很��的，以验�代�的行为是�符�您的�愿。
+�出问题之�，就�详细地�述一下您实际在�的技术细节。对于审阅者�说，用简练的
+英语�述代�的�化是很��的，以验�代�的行为是�符�您的�图。
 
-如果您将补��述写在一个表�中，这个表��以很容易地作为“�交日志�放入Linux
-的�代�管�系统git中，那么维护人员将�常感谢您。� :ref:`cn_explicit_in_reply_to`.
+如果您将补��述写�“标准格��，�以很容易地作为“�交日志�放入Linux的�代
+�管�系统 ``git`` 中，那么维护人员将�常感谢您。
+�� :ref:`zh_the_canonical_patch_format` 。
 
 �个补��解决一个问题。如果你的�述开始�长，这就表明你�能需�拆分你的补�。
-请� :ref:`cn_split_changes`
+请� :ref:`zh_split_changes` 。
 
-�交或�新�交修补程�或修补程�系列时，请包括完整的修补程�说明和�由。��
+�交或�新�交补�或补�系列时，请包括完整的补�说明和�由。��
 �说这是补�（系列）的第几版。��期望�系统维护人员引用更早的补�版本或引用
 URL�查找补��述并将其放入补�中。也就是说，补�（系列）�其�述应该是独立的。
-这对维护人员和审查人员都有好处。一些评审者�能甚至没有收到补�的早期版本。
+这对维护人员和审阅者都有好处。一些审阅者�能甚至没有收到补�的早期版本。
 
-�述你在命令语气中的�化，例如“make xyzzy do frotz�而�是“[This patch]make
+用祈使��述你的�更，例如“make xyzzy do frotz�而�是“[This patch]make
 xyzzy do frotz�或“[I]changed xyzzy to do frotz�，就好�你在命令代�库改�
 它的行为一样。
 
-如果修补程�修�了一个记录的bug�目，请按编�和URL引用该bug�目。如果补��
-自邮件列表讨论，请给出邮件列表存档的URL；使用带有 ``Message-ID`` 的
-https://lore.kernel.org/ �定�，以确�链接�会过时。
-
-但是，在没有外部资�的情况下，尽�让你的解释��解。除了�供邮件列表存档或
-bug的URL之外，还�总结需��交补�的相关讨论�点。
-
-如果您想�引用一个特定的�交，���引用�交的 SHA-1 ID。还请包括�交的一行
+如果您想�引用一个特定的�交，���引用�交的SHA-1 ID。还请包括�交的一行
 摘�，以便于审阅者了解它是关于什么的。例如::
 
         Commit e21d2170f36602ae2708 ("video: remove unnecessary
@@ -144,82 +94,104 @@ bug的URL之外，还�总结需��交补�的相关讨论�点。
         platform_set_drvdata(), but left the variable "dev" unused,
         delete it.
 
-您还应该确�至少使用�12� SHA-1 ID. 内核存储库包�*许多*对象，使与较短的ID
+您还应该确�至少使用�12�SHA-1 ID。内核存储库包� *许多* 对象，使较短的ID
 �生冲�的�能性很大。记�，�使现在�会与您的六个字符ID�生冲�，这�情况
-�能五年�改�。
+也�能在五年�改�。
+
+如果该�更的相关讨论或背景信��以在网上查阅，请加上“Link:�标签指�它。例如
+你的补�修�了一个缺陷，需�添加一个带有URL的标签指�邮件列表存档或缺陷跟踪器
+的相关报告；如果该补�是由一些早先邮件列表讨论或网络上的记录引起的，请指�它。
+
+当链接到邮件列表存档时，请首选lore.kernel.org邮件存档�务。用邮件中的
+``Message-ID`` 头（去掉尖括�）�以创建链接URL。例如::
+
+    Link: https://lore.kernel.org/r/30th.anniversary.repost@klaava.Helsinki.FI/
+
+请检查该链接以确��用且指�正确的邮件。
 
-如果修补程�修�了特定�交中的错误，例如，使用 ``git bisct`` ，请使用带有�
-12个字符SHA-1 ID 的"Fixes:"标记和�行摘�。为了简化��将标记拆分为多个，
-行�标记��分�脚本“75列�行�规则的�制。例如::
+�过，在没有外部资�的情况下，也�尽�让你的解释��解。除了�供邮件列表存档或
+缺陷的URL之外，还�需�总结该补�的相关讨论�点。
 
-        Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed")
+如果补�修�了特定�交中的错误，例如使用 ``git bisct`` �现了一个问题，请使用
+带有�12个字符SHA-1 ID的“Fixes:�标签和�行摘�。为了简化解�脚本，��将该
+标签拆分为多行，标签��“75列�行�规则的�制。例如::
 
-下列 ``git config`` 设置�以添加让 ``git log``, ``git show`` 漂亮的显示格�::
+  Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed")
+
+下列 ``git config`` 设置�以让 ``git log``, ``git show`` 增加上述风格的显示格�::
 
 	[core]
 		abbrev = 12
 	[pretty]
 		fixes = Fixes: %h (\"%s\")
 
-.. _cn_split_changes:
+使用示例::
+
+	$ git log -1 --pretty=fixes 54a4f0239f2e
+	Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed")
 
-3) 拆分你的改动
----------------
+.. _zh_split_changes:
 
-将�个逻辑更改分隔�一个�独的补�。
+拆分你的改动
+------------
+
+将�个 **逻辑更改** 拆分�一个�独的补�。
 
 例如，如果你的改动里�时有bug修正和性能优化，那么把这些改动拆分到两个或
-者更多的补�文件中。如果你的改动包�对API的修改，并且修改了驱动程��适
-应这些新的API，那么把这些修改分�两个补�。
+者更多的补�文件中。如果你的改动包�对API的修改，并且增加了一个使用该新API
+的驱动，那么把这些修改分�两个补�。
 
 �一方�，如果你将一个�独的改动��多个补�文件，那么将它们�并�一个
 �独的补�文件。这样一个逻辑上�独的改动�被包�在一个补�文件里。
 
-如果有一个补��赖�外一个补��完�它的改动，那没问题。简�的在你的补
-��述里指出“这个补��赖�补��就好了。
+需�记�的一点是，�个补�的更改都应易于�解，以便审阅者验�。�个补�都应该
+对其价值进行�述。
+
+如果有一个补��赖�外一个补��完�它的改动，那没问题。直接在你的补
+��述里指出 **“这个补��赖�补��** 就好了。
 
-在将您的更改划分为一系列补�时，�特别注�确�内核在系列中的�个补�之�
-都能正常构建和�行。使用 ``git bisect`` �追踪问题的开�者�能会在任何时
-候分割你的补�系列；如果你在中间引入错误，他们�会感谢你。
+在将您的更改划分为一系列补�时，�特别注�确�内核在应用系列中的�个补�之�
+都能正常构建和�行。使用 ``git bisect`` �追踪问题的开�者�能会在任何地方分
+割你的补�系列；如果你在中间引入错误，他们�会感谢你。
 
-如果你�能将补�浓缩�更少的文件，那么�次大约��出15个，然�等待审查
+如果你�能将补�系列浓缩得更�，那么�次大约��出15个补�，然�等待审阅
 和集�。
 
-4) 检查你的更改风格
--------------------
+检查你的更改风格
+----------------
 
-检查您的补�是�存在基本样�冲�，详细信��在
-:ref:`Documentation/translations/zh_CN/process/coding-style.rst <cn_codingstyle>`
-中找到。如果�这样�，�会浪费审稿人的时间，并且会导致你的补�被拒�，甚至
+检查您的补�是���了基本样�规定，详细信���
+Documentation/translations/zh_CN/process/coding-style.rst
+中找到。如果�这样�，�会浪费审阅者的时间，并且会导致你的补�被拒�，甚至
 �能没有被阅读。
 
 一个��的例外是在将代�从一个文件移动到�一个文件时——在这�情况下，您�应
 该在移动代�的�一个补�中修改移动的代�。这清楚地�述了移动代�和您的更改
-的行为。这大大有助于审查实际差异，并�许工具更好地跟踪代�本身的历�。
+的行为。这大大有助于审阅实际差异，并�许工具更好地跟踪代�本身的历�。
 
 在�交之�，使用补�样�检查程�检查补�（scripts/check patch.pl）。�过，
 请注�，样�检查程�应该被视为一个指�，而�是作为人类判断的替代�。如果您
-的代�看起�更好，但有�规行为，那么最好��使用它。
+的代�看起�更好，但有�规行为，那么最好别管它。
 
 检查者报告三个级别：
 
  - ERROR：很�能出错的事情
- - WARNING：需�仔细审查的事项
+ - WARNING：需�仔细审阅的事项
  - CHECK：需��考的事情
 
 您应该能够判断您的补�中存在的所有�规行为。
 
-5) 选择补�收件人
------------------
+选择补�收件人
+--------------
 
-您应该总是在任何补�上�制相应的�系统维护人员，以获得他们维护的代�；查看
+您应该总是知会任何补�相应代�的�系统维护人员；查看
 维护人员文件和�代�修订历�记录，以了解这些维护人员是�。脚本
-scripts/get_Maintainer.pl在这个步骤中�常有用。如果您找�到正在工作的�系统
+scripts/get_maintainer.pl在这个步骤中�常有用。如果您找�到正在工作的�系统
 的维护人员，那么Andrew Morton（akpm@linux-foundation.org）将充当最�的维护
 人员。
 
-您通常还应该选择至少一个邮件列表�接收补�集的。linux-kernel@vger.kernel.org
-作为最�一个解决办法的列表，但是这个列表上的体积已�引起了许多开�人员的拒�。
+您通常还应该选择至少一个邮件列表�接收补�集的副本。linux-kernel@vger.kernel.org
+是所有补�的默认列表，但是这个列表的��已�导致了许多开�人员��看它。
 在MAINTAINERS文件中查找�系统特定的列表；您的补��能会在那里得到更多的关注。
 �过，请����垃圾邮件到无关的列表。
 
@@ -229,189 +201,170 @@ http://vger.kernel.org/vger-lists.html 上找到它们的列表。ä¸�过，也æœ
 
 ��一次��超过15个补�到vger邮件列表����
 
-Linus Torvalds 是决定改动能�进入 Linux 内核的最终�决者。他的 e-mail
-地�是 <torvalds@linux-foundation.org> 。他收到的 e-mail 很多，所以一般
-的说，最好别给他� e-mail。
-
-如果您有修��利用安全�洞的补�，请将该补���到 security@kernel.org。对于
-严�的bug，�以考虑短期暂�以�许分销商�用户�布补�；在这�情况下，显然�应
-将补���到任何公共列表。
+Linus Torvalds是决定改动能�进入 Linux 内核的最终�决者。他的邮件地�是
+torvalds@linux-foundation.org 。他收到的邮件很多，所以一般�说最好 **别**
+给他�邮件。
 
-修�已�布内核中严�错误的补�程�应该指�稳定版维护人员，方法是放这样的一行::
+如果您有修��利用安全�洞的补�，请将该补���到 security@kernel.org 。对于
+严�的bug，�以考虑短期�令以�许分销商（有时间）�用户�布补�；在这�情况下，
+显然�应将补���到任何公共列表。
+�� Documentation/translations/zh_CN/admin-guide/security-bugs.rst 。
 
-        Cc: stable@vger.kernel.org
+修�已�布内核中严�错误的补�程�应该抄�给稳定版维护人员，方法是把以下列行
+放进补�的签准区（注�，�是电�邮件收件人）::
 
-进入补�的签准区（注�，�是电�邮件收件人）。除了这个文件之外，您还应该阅读
-:ref:`Documentation/process/stable-kernel-rules.rst <stable_kernel_rules>`
+  Cc: stable@vger.kernel.org
 
-但是，请注�，一些�系统维护人员希望得出他们自己的结论，�哪些补�应该被放到
-稳定的树上。尤其是网络维护人员，�希望看到�个开�人员在补�中添加�上�这样
-的行。
+除了本文件之外，您还应该阅读
+Documentation/translations/zh_CN/process/stable-kernel-rules.rst 。
 
-如果更改影�到用户和内核接�，请�手册页维护人员（如维护人员文件中所列）��
+如果更改影�到用户侧内核接�，请�手册页维护人员（如维护人员文件中所列）��
 手册页补�，或至少��更改通知，以便一些信�进入手册页。还应将用户空间API
-更改�制到 linux-api@vger.kernel.org。
+更改抄�到 linux-api@vger.kernel.org 。
 
 
-6) 没有 MIME 编�，没有链接，没有压缩，没有附件，�有纯文本
------------------------------------------------------------
+��MIME编�，��链接，��压缩，��附件，��纯文本
+------------------------------------------------------
 
 Linus 和其他的内核开�者需�阅读和评论你�交的改动。对于内核开�者�说
-，�以“引用�你的改动很��，使用一般的 e-mail 工具，他们就�以在你的
+，�以“引用�你的改动很��，使用一般的邮件工具，他们就�以在你的
 代�的任何�置添加评论。
 
-因为这个原因，所有的�交的补�都是 e-mail 中“内嵌�的。
+因为这个原因，所有的�交的补�都是邮件中“内嵌�的。最简�（和推�）的方法就
+是使用 ``git send-email`` 。https://git-send-email.io 有 ``git send-email``
+的交互�教程。
+
+如果你选择�用 ``git send-email`` ：
 
 .. warning::
-   如果你使用剪切-粘贴你的补�，�心你的编辑器的自动�行功能破�你的补�
 
-��将补�作为 MIME 编�的附件，�管是�压缩。很多�行的 e-mail 软件�
-是任何时候都将 MIME 编�的附件当作纯文本��的，这会使得别人无法在你的
-代�中加评论。�外，MIME 编�的附件会让 Linus 多花一点时间�处�，这就
-�低了你的改动被接�的�能性。
+  如果你使用剪切-粘贴你的补�，�心你的编辑器的自动�行功能破�你的补�
 
-例外：如果你的邮递员弄�了补�，那么有人�能会�求你使用mime�新��补�
+��将补�作为MIME编�的附件，�管是�压缩。很多�行的邮件软件�
+是任何时候都将MIME编�的附件当作纯文本��的，这会使得别人无法在你的
+代�中加评论。�外，MIME编�的附件会让Linus多花一点时间�处�，这就
+�低了你的改动被接�的�能性。
 
-请�阅 :ref:`Documentation/translations/zh_CN/process/email-clients.rst <cn_email_clients>`
-以获�有关�置电�邮件客户端以使其��影�地��修补程�的�示。
+例外：如果你的邮路��了补�，那么有人�能会�求你使用MIME�新��补�。
 
-7) e-mail 的大�
-----------------
+请�阅 Documentation/translations/zh_CN/process/email-clients.rst
+以获�有关�置电�邮件客户端以使其��影�地��补�的�示。
 
-大的改动对邮件列表��适，对�些维护者也��适。如果你的补�，在�压缩
-的情况下，超过了300kB，那么你最好将补�放在一个能通过 internet 访问的�
-务器上，然�用指�你的补�的 URL 替代。但是请注�，如果您的补�超过了
-300kb，那么它几乎肯定需�被破�。
+回�审阅��
+------------
 
-8）回�评审��
----------------
+你的补�几乎肯定会得到审阅者对补�改进方法的评论（以回�邮件的形�）。您必须
+对这些评论作出回应；让补�被忽略的一个好办法就是忽略审阅者的��。直接回�邮
+件�回应����。�会导致代�更改的��或问题几乎肯定会带�注释或�更日志的
+改�，以便下一个审阅者更好地了解正在�生的事情。
 
-你的补�几乎肯定会得到评审者对补�改进方法的评论。您必须对这些评论作出
-回应；让补�被忽略的一个好办法就是忽略审阅者的��。�会导致代�更改的
-��或问题几乎肯定会带�注释或�更日志的改�，以便下一个评审者更好地了解
-正在�生的事情。
+一定�告诉审阅者你在�什么改�，并感谢他们的时间。代�审阅是一个累人且耗时的
+过程，审阅者有时会�得暴�。�使在这�情况下，也�礼貌地回应并解决他们指出的
+问题。当��下一版时，在��邮件或独立补�里加上 ``patch changelog`` 说明与
+�一版本的��之处（�� :ref:`zh_the_canonical_patch_format` ）。
 
-一定�告诉审稿人你在�什么改�，并感谢他们的时间。代�审查是一个累人且
-耗时的过程，审查人员有时会�得暴�。�使在这�情况下，也�礼貌地回应并
-解决他们指出的问题。
+.. _zh_resend_reminders:
 
-9）��泄气或��烦
--------------------
+��泄气或��烦
+----------------
 
-�交更改�，请�心等待。审阅者是忙碌的人，�能无法立�访问您的修补程�。
+�交更改�，请�心等待。审阅者是大忙人，�能无法立�审阅您的补�。
 
-曾几何时，补�曾在没有评论的情况下消失在空白中，但开�过程比现在更加顺利。
+曾几何时，补�曾在没收到评论的情况下消失在虚空中，但现在开�过程应该更加顺利了。
 您应该在一周左�的时间内收到评论；如果没有收到评论，请确�您已将补���
-到正确的�置。在�新�交或�系审阅者之�至少等待一周-在诸如�并窗�之类的
+到正确的�置。在�新�交或�系审阅者之�至少等待一周——在诸如�并窗�之类的
 �忙时间�能更长。
 
-10）主题中包� PATCH
---------------------
+在等了几个星期�，用带RESEND的主题��补�也是�以的::
 
-由于到linus和linux内核的电�邮件��很高，通常会在主题行��加上[PATCH]
-�缀. 这使Linus和其他内核开�人员更容易将补�与其他电�邮件讨论区分开。
+   [PATCH Vx RESEND] sub/sys: Condensed patch summary
 
-11）签署你的作�-开�者原始认�
--------------------------------
+当你�布补�（系列）修改版的时候，��加上“RESEND�——“RESEND��适用于�
+新�交之�未�修改的补�（系列）。
 
-为了加强对��了何事的追踪，尤其是对那些�过好几层的维护者的补�，我们
-建议在��出去的补�上加一个 “sign-off� 的过程。
+主题中包� PATCH
+----------------
 
-"sign-off" 是在补�的注释的最�的简�的一行文字，认�你编写了它或者其他
+由于到Linus和linux-kernel的电�邮件��很高，通常会在主题行��加上[PATCH]
+�缀。这使Linus和其他内核开�人员更容易将补�与其他电�邮件讨论区分开。
+
+``git send-email`` 会自动为你加上。
+
+签署你的作�——开�者��认�
+------------------------------
+
+为了加强对��了何事的追踪，尤其是对那些�过好几层维护者�最终到达的补�，我
+们在通过邮件��的补�上引入了“签署（sign-off）��程。
+
+“签署�是在补�注释最�的一行简�文字，认�你编写了它或者其他
 人有�力将它作为开放�代�的补�传递。规则很简�：如果你能认�如下信�:
 
-开�者���书 1.1
+开�者��认� 1.1
 ^^^^^^^^^^^^^^^^^^
 
 对于本项目的贡献，我认�如下信�：
 
-      （a）这些贡献是完全或者部分的由我创建，我有�利以文件中指出
+       (a) 这些贡献是完全或者部分的由我创建，我有�利以文件中指出
            的开放�代�许���交它；或者
-      （b）这些贡献基于以�的工作，�我所知，这些以�的工作��当的开放
-           �代�许���护，而且，根�许��，我有��交修改�的贡献，
+
+       (b) 这些贡献基于以�的工作，�我所知，这些以�的工作��当的开放
+           �代�许���护，而且，根�文件中指出的许��，我有��交修改�的贡献，
            无论是完全还是部分由我创造，这些贡献都使用�一个开放�代�许��
-           （除�我被�许用其它的许��），正如文件中指出的；或者
-      （c）这些贡献由认�（a），（b）或者（c）的人直接�供给我，而
+           （除�我被�许用其它的许��）；或者
+
+       (c) 这些贡献由认�（a），（b）或者（c）的人直接�供给我，而
            且我没有修改它。
-      （d）我�解并��这个项目和贡献是公开的，贡献的记录（包括我
-           一起�交的个人记录，包括 sign-off ）被永久维护并且�以和这个项目
+
+       (d) 我�解并��这个项目和贡献是公开的，贡献的记录（包括我
+           一起�交的个人记录，包括sign-off）被永久维护并且�以和这个项目
            或者开放�代�的许���步地��行。
 
 那么加入这样一行::
 
-       Signed-off-by: Random J Developer <random@developer.example.org>
-
-使用你的真�（抱歉，�能使用��或者匿�。）
-
-有人在最�加上标签。现在这些东西会被忽略，但是你�以这样�，�标记公�
-内部的过程，或者�是指出关于 sign-off 的一些特殊细节。
-
-如果您是�系统或分支维护人员，有时需��微修改收到的补�，以便�并它们，
-因为树和�交者中的代��完全相�。如果你严格�守规则（c），你应该�求�交者
-�新�布，但这完全是在浪费时间和精力。规则（b）�许您调整代�，但是更改一个
-�交者的代�并让他认�您的错误是�常�礼貌的。�解决此问题，建议在最�一个
-由签�行和您的行之间添加一行，指示更改的性质。虽然这并�是强制性的，但似乎
-在�述�加上您的邮件和/或姓�（全部用方括�括起�），这足以让人注�到您对最
-�一分钟的更改负有责任。例如::
-
-	Signed-off-by: Random J Developer <random@developer.example.org>
-	[lucky@maintainer.example.org: struct foo moved from foo.c to foo.h]
-	Signed-off-by: Lucky K Maintainer <lucky@maintainer.example.org>
-
-如果您维护一个稳定的分支机构，�时希望对作者进行致谢�跟踪更改��并修�并
-�护�交者��投诉，那么这��法尤其有用。请注�，在任何情况下都�能更改作者
-的ID（From 头），因为它是出现在更改日志中的标识。
-
-对回�（back-porters）的特别说明：在�交消�的顶部（主题行之�）�入一个补�
-的起�指示似乎是一�常�且有用的实践，以便于跟踪。例如，下�是我们在3.x稳定
-版本中看到的内容::
-
-  Date:   Tue Oct 7 07:26:38 2014 -0400
-
-    libata: Un-break ATA blacklist
-
-    commit 1c40279960bcd7d52dbdf1d466b20d24b99176c8 upstream.
-
-还有， 这里是一个旧版内核中的一个回�补�::
+  Signed-off-by: Random J Developer <random@developer.example.org>
 
-    Date:   Tue May 13 22:12:27 2008 +0200
+使用你的真�（抱歉，�能使用��或者匿�。）如果使用 ``git commit -s`` 的�
+将会自动完�。撤销也应当包�“Signed-off-by�， ``git revert -s`` 会帮你�定。
 
-        wireless, airo: waitbusy() won't delay
+有些人会在最�加上�外的标签。现在这些东西会被忽略，但是你�以这样�，�标记
+公�内部的过程，或者�是指出关于签署的一些特殊细节。
 
-        [backport of 2.6 commit b7acbdfbd1f277c1eb23f344f899cfa4cd0bf36a]
+作者签署之�的任何其他签署（Signed-off-by:'s）��自处�和传递补�的人员，但
+未�与其开�。签署链应当�映补�传播到维护者并最终传播到Linus所�过的 **真实**
+路径，首个签署指明�个作者的主�作者身份。
 
-12）何时使用Acked-by:，CC:，和Co-Developed by:
-----------------------------------------------
+何时使用Acked-by:，CC:，和Co-Developed by:
+------------------------------------------
 
-Singed-off-by: 标记表示签�者�与了补�的开�，或者他/她在补�的传递路径中。
+Singed-off-by: 标签表示签�者�与了补�的开�，或者他/她在补�的传递路径中。
 
-如果一个人没有直接�与补�的准备或处�，但希望表示并记录他们对补�的批准，
-那么他们�以�求在补�的�更日志中添加一个 Acked-by:
+如果一个人没有直接�与补�的准备或处�，但希望表示并记录他们对补�的批准/赞�，
+那么他们�以�求在补�的�更日志中添加一个Acked-by:。
 
-Acked-by：通常由�影�代�的维护者使用，当该维护者既没有贡献也没有转�补�时。
+Acked-by: 通常由�影�代�的维护者使用，当该维护者既没有贡献也没有转�补�时。
 
-Acked-by: ��签字人那样正�。这是一个记录，确认人至少审查了补�，并表示接�。
-因此，补��并有时会手动将Acker的“Yep，looks good to me�转�为 Acked-By：（但
+Acked-by: ��签署那样正�。这是一个记录，确认人至少审阅了补�，并表示接�。
+因此，补��并有时会手动将Acker的“Yep，looks good to me�转�为 Acked-By:（但
 请注�，通常最好�求一个明确的Ack）。
 
 Acked-by：�一定表示对整个补�的确认。例如，如果一个补�影�多个�系统，并且
-有一个：�自一个�系统维护者，那么这通常表示�确认影�维护者代�的部分。这里
-应该仔细判断。如有疑问，应�考邮件列表档案中的原始讨论。
+有一个�自�个�系统维护者的Acked-By:，那么这通常表示�确认影�维护者代�的部
+分。这里应该仔细判断。如有疑问，应�考邮件列表存档中的原始讨论。
 
-如果�人有机会对补�进行评论，但没有�供此类评论，您�以选择在补�中添加 ``Cc:``
-这是唯一一个标签，它�以在没有被它命�的人显��作的情况下添加，但它应该表明
-这个人是在补�上抄�的。讨论中包�了潜在利益相关方。
+如果�人本应有机会对补�进行评论，但没有�供此类评论，您�以选择在补�中添加
+``Cc:`` 这是唯一�以在没有被该人明确��的情况下添加的标签——但它应该表明
+这个人是在补�上抄�的。此标签记录了讨论中包�的潜在利益相关方。
 
 Co-developed-by: 声明补�是由多个开�人员共�创建的；当几个人在一个补�上工
-作时，它用于将属性赋予共�作者（除了 From: 所赋予的作者之外）。因为
-Co-developed-by: 表示作者身份，所以�个共�开�人：必须紧跟在相关�作作者的
-签�之�。标准的签核程��求：标记的签核顺�应尽�能�映补�的时间历�，而�
-管作者是通过 From ：还是由 Co-developed-by: 共�开�的。值得注�的是，最�一
-个签字人：必须始终是�交补�的开�人员。
+作时，它用于给出共�作者（除了From:所给出的作者之外）。因为Co-developed-by:
+表示作者身份，所以�个Co-developed-by:必须紧跟在相关�作作者的签署之�。标准
+签署程��求Singed-off-by:标签的顺�应尽�能�映补�的时间历�，无论作者是通
+过From:还是Co-developed-by:表明。值得注�的是，最�一个Singed-off-by:必须是
+�交补�的开�人员。
 
-注�，当作者也是电�邮件标题“�件人：�行中列出的人时，“From: � 标记是�选的。
+注�，如果From:作者也是电�邮件标题的From:行中列出的人，则From:标签是�选的。
 
-作者�交的补�程�示例::
+被From:作者�交的补�示例::
 
 	<changelog>
 
@@ -421,7 +374,7 @@ Co-developed-by: 表示作者身份，所以�个共�开�人：必须紧跟
 	Signed-off-by: Second Co-Author <second@coauthor.example.org>
 	Signed-off-by: From Author <from@author.example.org>
 
-�作开�者�交的补�示例::
+被�作开�者�交的补�示例::
 
 	From: From Author <from@author.example.org>
 
@@ -434,76 +387,85 @@ Co-developed-by: 表示作者身份，所以�个共�开�人：必须紧跟
 	Signed-off-by: Submitting Co-Author <sub@coauthor.example.org>
 
 
-13）使用报告人：�测试人：�审核人：�建议人：�修�人：
---------------------------------------------------------
+使用Reported-by:�Tested-by:�Reviewed-by:�Suggested-by:和Fixes:
+-----------------------------------------------------------------
 
 Reported-by: 给那些�现错误并报告错误的人致谢，它希望激励他们在将��次帮助
-我们。请注�，如果bug是以�有方�报告的，那么在使用Reported-by标记之�，请
-先请求��。
+我们。请注�，如果bug是以�有方�报告的，那么在使用Reported-by标签之�，请
+先请求许�。此标签是为Bug设计的；请��将其用于感谢功能请求。
 
-Tested-by: 标记表示补�已由指定的人（在�些环境中）�功测试。这个标签通知
-维护人员已�执行了一些测试，为将�的补��供了一�定�测试人员的方法，并确
-�测试人员的信誉。
+Tested-by: 标签表示补�已由指定的人（在�些环境中）�功测试。这个标签通知
+维护人员已�执行了一些测试，为将�的补��供了一�定�测试人员的方法，并彰显测试人员的功劳。
 
-Reviewed-by：相�，根�审查人的声明，表明该补�已被审查并被认为是�接�的：
+Reviewed-by：根�审阅者的监�声明，表明该补�已被审阅并被认为是�接�的：
 
 
-审查人的监�声明
+审阅者的监�声明
 ^^^^^^^^^^^^^^^^
 
-通过�供我的 Reviewed-by，我声明：
+通过�供我的Reviewed-by:标签，我声明：
 
-        (a) 我已�对这个补�进行了一次技术审查，以评估它是�适�被包�到
+        (a) 我已�对这个补�进行了一次技术审阅，以评估它是�适�被包�到
             主线内核中。
 
         (b) 与补�相关的任何问题�顾虑或问题都已�馈给�交者。我对�交者对
             我的评论的回应感到满�。
 
-        (c) 虽然这一�交�能会改进一些东西，但我相信，此时，（1）对内核
+        (c) 虽然这一�交�能��被改进，但我相信，此时，（1）对内核
             进行了有价值的修改，（2）没有包�争论中涉�的已知问题。
 
-        (d) 虽然我已�审查了补�并认为它是�全的，但我�会（除��有明确
-            说明）作出任何��或��它将在任何给定情况下实现其规定的目的
+        (d) 虽然我已�审阅了补�并认为它是�全的，但我�会（除��有明确
+            说明）作出任何��或担�它会在任何给定情况下实现其规定的目的
             或正常�行。
 
-Reviewed-by 是一�观点声明，�补�是对内核的适当修改，没有任何�留的严�技术
-问题。任何感兴趣的审阅者（完�工作的人）都�以为一个补��供一个 Review-by
-标签。此标签用于�审阅者�供致谢，并通知维护者已在修补程�上完�的审阅程度。
-Reviewed-by: 当由已知了解主题区域并执行彻底检查的审阅者�供时，通常会增加
+Reviewed-by是一�观点声明，�补�是对内核的适当修改，没有任何�留的严�技术
+问题。任何感兴趣的审阅者（完�工作的人）都�以为一个补��供一个Reviewed-by
+标签。此标签用于�审阅者�供致谢，并通知维护者补�的审阅进度。
+当Reviewed-by:标签由已知了解主题区域并执行彻底检查的审阅者�供时，通常会增加
 补�进入内核的�能性。
 
+一旦从测试人员或审阅者的“Tested-by�和“Reviewed-by�标签出现在邮件列表中，
+作者应在��下一个版本时将其添加到适用的补�中。但是，如果补�在以下版本中�
+生了实质性更改，这些标签�能��适用，因此应该删除。通常，在补�更改日志中
+（在 ``---`` 分隔符之�）应该�到删除�人的测试者或审阅者标签。
+
 Suggested-by: 表示补�的想法是由指定的人�出的，并确�将此想法归功于指定的
 人。请注�，未�许�，�得添加此标签，特别是如果该想法未在公共论�上�布。
-这就是说，如果我们勤快地致谢我们的创�者，他们很有希望在未�得到鼓舞，�次
+也就是说，如果我们勤快地致谢创��供者，他们将�到鼓舞，很有希望在未��次
 帮助我们。
 
-Fixes: 指示补�在以�的�交中修�了一个问题。它�以很容易地确定错误的��，
-这有助于检查错误修�。这个标记还帮助稳定内核团队确定应该接收修�的稳定内核
-版本。这是指示补�修�的错误的首选方法。请�阅 :ref:`cn_describe_changes`
-�述您的更改以了解更多详细信�。
+Fixes: 指示补�修�了之��交的一个问题。它�以便于确定错误的��，这有助于
+检查错误修�。这个标签还帮助稳定内核团队确定应该接收修�的稳定内核版本。这是
+指示补�修�的错误的首选方法。请�阅 :ref:`zh_describe_changes` 了解更多信�。
 
-.. _cn_the_canonical_patch_format:
+.. note::
 
-12）标准补�格�
-----------------
+  附加Fixes:标签�会改�稳定内核规则�程，也�改�所有稳定版补�抄�
+  stable@vger.kernel.org的�求。有关更多信�，请阅读
+  Documentation/translations/zh_CN/process/stable-kernel-rules.rst 。
+
+.. _zh_the_canonical_patch_format:
+
+标准补�格�
+------------
 
 本节�述如何格�化补�本身。请注�，如果您的补�存储在 ``Git`` 存储库中，则
-�以使用 ``git format-patch`` 进行正确的补�格�设置。但是，这些工具无法创建
+�以使用 ``git format-patch`` 进行正确的补�格�化。但是，这些工具无法创建
 必�的文本，因此请务必阅读下�的说明。
 
-标准的补�，标题行是::
+标准的补�标题行是::
 
     Subject: [PATCH 001/123] �系统:一��概述
 
-标准补�的信体存在如下部分：
+标准补�的信体包�如下部分：
 
-  - 一个 "from" 行指出补�作者。�跟空行（仅当��修补程�的人�是作者时�需�）。
+  - 一个 ``from`` 行指出补�作者。�跟空行（仅当��补�的人�是作者时�需�）。
 
-  - 解释的正文，行以75列包装，这将被�制到永久�更日志��述这个补�。
+  - 说明文字，�行最长75列，这将被�制到永久�更日志��述这个补�。
 
   - 一个空行
 
-  - 上��述的“Signed-off-by� 行，也将出现在更改日志中。
+  - 上述的 ``Signed-off-by:`` 行，也将出现在更改日志中。
 
   - �包� ``---`` 的标记线。
 
@@ -511,29 +473,29 @@ Fixes: 指示补ä¸�在以å‰�çš„æ��交中修å¤�了一个问题。它å�¯ä»¥å¾ˆå®¹æ
 
   - 实际补�（ ``diff`` 输出）。
 
-标题行的格�，使得对标题行按字��排��常的容易 - 很多 e-mail 客户端都
-�以支� - 因为�列�是用零填充的，所以按数字排�和按字�排�是一样的。
+标题行的格�，使得对标题行按字��排��常的容易——很多邮件客户端都
+�以支�——因为�列�是用零填充的，所以按数字排�和按字�排�是一样的。
 
-e-mail 标题中的“�系统�标识哪个内核�系统将被打补�。
+邮件标题中的“�系统�标识哪个内核�系统将被打补�。
 
-e-mail 标题中的“一��概述�扼�的�述 e-mail 中的补�。“一��概述�
+邮件标题中的“一��概述�扼�的�述邮件中的补�。“一��概述�
 �应该是一个文件�。对于一个补�系列（“补�系列�指一系列的多个相关补
 �），��对�个补�都使用�样的“一��概述�。
 
-记� e-mail 的“一��概述�会�为该补�的全局唯一标识。它会蔓延到 git
+记�邮件的“一��概述�会�为该补�的全局唯一标识。它会进入 ``git``
 的改动记录里。然�“一��概述�会被用在开�者的讨论里，用�指代这个补
-�。用户将希望通过 google ��索"一��概述"�找到那些讨论这个补�的文
+�。用户将希望通过�索引擎�索“一��概述��找到那些讨论这个补�的文
 章。当人们在两三个月�使用诸如 ``gitk`` 或 ``git log --oneline`` 之类
 的工具查看数�个补�时，也会很快看到它。
 
 出于这些原因，概述必须�超过70-75个字符，并且必须�述补�的更改以�为
-什么需�补�。既�简����述性很有挑战性，但写得好的概述应该这样�。
+什么需�补�。既�简����述性很有挑战性，但写得好的概述应该这样。
 
 概述的�缀�以用方括�括起�：“Subject: [PATCH <tag>...] <概述>�。标记
 �被视为概述的一部分，而是�述应该如何处�补�。如果补�的多个版本已�
-�出�以�应评审（�“v1，v2，v3�）或“rfc�，以指示评审请求，那么通用标记
-�能包括版本�述符。如果一个补�系列中有四个补�，那么�个补��以这样
-编�：1/4�2/4�3/4�4/4。这�以确�开�人员了解补�应用的顺�，并且他们
+�出�以�应评审（�“v1，v2，v3�）则必须包�版本�，或包�“RFC�以指示
+评审请求。如果一个补�系列中有四个补�，那么�个补��以这样编�：1/4�2/4�
+3/4�4/4。这�以确�开�人员了解补�应用的顺�，且
 已�查看或应用了补�系列中的所有补�。
 
 一些标题的例�::
@@ -541,95 +503,134 @@ e-mail 标题中的“一å�¥è¯�概述â€�扼è¦�çš„æ��è¿° e-mail 中的补ä¸�。â€
     Subject: [patch 2/5] ext2: improve scalability of bitmap searching
     Subject: [PATCHv2 001/207] x86: fix eflags tracking
 
-"From" 行是信体里的最上�一行，具有如下格�：
+``From`` 行是信体里的最上�一行，具有如下格�::
+
         From: Patch Author <author@example.com>
 
-"From" 行指明在永久改动日志里，�会被确认为作者。如果没有 "From" 行，那
-么邮件头里的 "From: " 行会被用�决定改动日志中的作者。
+``From`` 行指明在永久改动日志里，�会被确认为作者。如果没有 ``From`` 行，那
+么邮件头里的 ``From:`` 行会被用�决定改动日志中的作者。
 
-说明的主题将会被�交到永久的�代�改动日志里，因此对那些早已��记得和
-这个补�相关的讨论细节的有能力的读者�说，是有�义的。包括补�程�定�
-错误的（内核日志消��OOPS消�等）症状，对于�索�交日志以寻找适用补�的人
-尤其有用。如果一个补�修�了一个编译失败，那么�能�需�包�所有编译失败；
+说明文字将会被�交到永久的�代�改动日志里，因此应针对那些早已��记得和这
+个补�相关的讨论细节的读者。包括补�处�的故障症状（内核日志消��oops消�
+等），这对于�能正在�索�交日志以查找适用补�的人特别有用。文本应该写得如
+此详细，以便在数周�数月甚至数年�阅读时，能够为读者�供所需的细节信�，以
+掌�创建补�的 **原因** 。
+
+如果一个补�修�了一个编译失败，那么�能�需�包� *所有* 编译失败；
 ��足够让�索补�的人能够找到它就行了。与概述一样，既�简����述性。
 
-"---" 标记行对于补�处�工具�找到哪里是改动日志信�的结�，是��缺少
+``---`` 标记行对于补�处�工具�找到哪里是改动日志信�的结�，是��缺少
 的。
 
-对于 "---" 标记之�的�外注解，一个好的用途就是用�写 diffstat，用�显
-示修改了什么文件和�个文件都增加和删除了多少行。diffstat 对于比较大的补
-�特别有用。其余那些�是和时刻或者开�者相关的注解，��适放到永久的改
-动日志里的，也应该放这里。
-使用 diffstat的选项 "-p 1 -w 70" 这样文件�就会从内核�代�树的目录开始
+对于 ``---`` 标记之�的�外注解，一个好的用途就是用�写 ``diffstat`` ，用�显
+示修改了什么文件和�个文件都增加和删除了多少行。 ``diffstat`` 对于比较大的补
+�特别有用。
+使用 ``diffstat`` 的选项 ``-p 1 -w 70`` 这样文件�就会从内核�代�树的目录开始
 ，�会�用太宽的空间（很容易适�80列的宽度，也许会有一些缩进。）
+（ ``git`` 默认会生��适的diffstat。）
+
+其余那些�适用于当时或者与维护者相关的注解，��适放到永久的改动日志里的，也
+应该放这里。较好的例�就是 ``补�更改记录`` ，记录了v1和v2版本补�之间的差异。
+
+请将此信�放在将�更日志与补�的其余部分分隔开的 ``---`` 行 **之�** 。版本
+信��是�交到git树的�更日志的一部分。�是供审阅人员使用的附加信�。如果将
+其放置在�交标记上方，则需�手动交互�能将其删除。如果它�于分隔线以下，则在
+应用补�时会自动剥离::
+
+  <commit message>
+  ...
+  Signed-off-by: Author <author@mail>
+  ---
+  V2 -> V3: Removed redundant helper function
+  V1 -> V2: Cleaned up coding style and addressed review comments
 
-在��的�考资料中能看到适当的补�格�的更多细节。
+  path/to/file | 5+++--
+  ...
 
-.. _cn_explicit_in_reply_to:
+在��的�考资料中能看到正确补�格�的更多细节。
 
-15) 明确回�邮件头(In-Reply-To)
--------------------------------
+.. _zh_backtraces:
 
-手动添加回�补�的的标题头(In-Reply_To:) 是有帮助的（例如，使用 ``git send-email`` ）
-将补�与以�的相关讨论关�起�，例如，将bug修�程�链接到电�邮件和bug报告。
+�交消�中的回溯（Backtraces）
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+回溯有助于记录导致问题的调用链。然而，并�所有回溯都有帮助。例如，早期引导调
+用链是独特而明显的。而�字�制完整的dmesg输出则会增加时间戳�模�列表�寄存
+器和堆栈转储等分散注�力的信�。
+
+因此，最有用的回溯应该从转储中��相关信�，以更容易集中在真实问题上。下�是
+一个剪�良好的回溯示例::
+
+  unchecked MSR access error: WRMSR to 0xd51 (tried to write 0x0000000000000064)
+  at rIP: 0xffffffffae059994 (native_write_msr+0x4/0x20)
+  Call Trace:
+  mba_wrmsr
+  update_domains
+  rdtgroup_mkdir
+
+.. _zh_explicit_in_reply_to:
+
+明确回�邮件头（In-Reply-To）
+-----------------------------
+
+手动添加回�补�的的邮件头（In-Reply_To:）是有用的（例如，使用 ``git send-email`` ），
+�以将补�与以�的相关讨论关�起�，例如，将bug补�链接到电�邮件和bug报告。
 但是，对于多补�系列，最好��在回�时使用链接到该系列的旧版本。这样，
-补�的多个版本就�会�为电�邮件客户端中无法管�的引用�列。如果链接有用，
+补�的多个版本就�会�为电�邮件客户端中无法管�的引用树。如果链接有用，
 �以使用 https://lore.kernel.org/ �定�器（例如，在��电�邮件文本中）
 链接到补�系列的早期版本。
 
-16) ��git pull请求
---------------------
-
-如果您有一系列补�，那么让维护人员通过git pull�作将它们直接拉入�系统存储
-库�能是最方便的。但是，请注�，从开�人员那里获�补�比从邮件列表中获�补
-�需�更高的信任度。因此，许多�系统维护人员�愿�接�请求，特别是�自新的
-未知开�人员的请求。如果有疑问，您�以在��邮件中使用pull 请求作为补�系列
-正常�布的一个选项，让维护人员�以选择使用其中之一。
-
-pull 请求的主题行中应该有[Git Pull]。请求本身应该在一行中包�存储库�称和
-感兴趣的分支；它应该看起��::
+给出基础树信�
+--------------
 
-  Please pull from
+当其他开�人员收到您的补�并开始审阅时，知�应该将您的工作放到代�树历�记录
+中的什么�置通常很有用。这对于自动化�续集��水（CI）特别有用，这些�水线试
+图�行一系列测试，以便在维护人员开始审阅之�确定�交的质�。
 
-      git://jdelvare.pck.nerim.net/jdelvare-2.6 i2c-for-linus
+如果您使用 ``git format-patch`` 生�补�，则�以通过 ``--base`` 标志在�交中
+自动包�基础树信�。使用此选项最简��最方便的方法是��主题分支::
 
-  to get these changes:
+    $ git checkout -t -b my-topical-branch master
+    Branch 'my-topical-branch' set up to track local branch 'master'.
+    Switched to a new branch 'my-topical-branch'
 
+    [perform your edits and commits]
 
-pull 请求还应该包�一�整体消�，说明请求中将包�什么，一个补�本身的 ``Git shortlog``
-以�一个显示补�系列整体效果的 ``diffstat`` 。当然，将所有这些信�收集在一起
-的最简�方法是让 ``git`` 使用 ``git request-pull`` 命令为您完�这些工作。
+    $ git format-patch --base=auto --cover-letter -o outgoing/ master
+    outgoing/0000-cover-letter.patch
+    outgoing/0001-First-Commit.patch
+    outgoing/...
 
-一些维护人员（包括Linus）希望看到�自已签��交的请求；这增加了他们对你的
-请求信心。特别是，在没有签�标签的情况下，Linus �会从� Github 这样的公共
-托管站点拉请求。
+当你编辑 ``outgoing/0000-cover-letter.patch`` 时，您会注�到在它的最底部有一
+行 ``base-commit:`` 尾注，它为审阅者和CI工具�供了足够的信�以正确执行
+``git am`` 而�必担心冲�::
 
-创建此类签�的第一步是生�一个 GNRPG 密钥，并由一个或多个核心内核开�人员对
-其进行签�。这一步对新开�人员�说�能很困难，但没有办法绕过它。�加会议是
-找到�以签署您的密钥的开�人员的好方法。
+    $ git checkout -b patch-review [base-commit-id]
+    Switched to a new branch 'patch-review'
+    $ git am patches.mbox
+    Applying: First Commit
+    Applying: ...
 
-一旦您在Git 中准备了一个您希望有人拉的补�系列，就用 ``git tag -s`` 创建一
-个签�标记。这将创建一个新标记，标识该系列中的最�一次�交，并包�用您的�
-钥创建的签�。您还�以将changelog样�的消�添加到标记中；这是一个�述拉请求
-整体效果的�想�置。
+有关此选项的更多信�，请�阅 ``man git-format-patch`` 。
 
-如果维护人员将�从中��的树�是您正在使用的存储库，请��忘记将已签�的标记
-显�推�到公共树。
+.. note::
 
-生�拉请求时，请使用已签�的标记作为目标。这样的命令�以实现::
+    ``--base`` 功能是在2.9.0版git中引入的。
 
-  git request-pull master git://my.public.tree/linux.git my-signed-tag
+如果您�使用git格�化补�，�然�以包�相�的 ``base-commit`` 尾注，以指示您
+的工作所基于的树的�交哈希。你应该在��邮件或系列的第一个补�中添加它，它应
+该放在 ``---`` 行的下�或所有其他内容之�，��在你的电�邮件签�之�。
 
 �考文献
 --------
 
-Andrew Morton, "The perfect patch" (tpp).
+Andrew Morton，“完美的补��（tpp）
   <https://www.ozlabs.org/~akpm/stuff/tpp.txt>
 
-Jeff Garzik, "Linux kernel patch submission format".
+Jeff Garzik，“Linux内核补��交格��
   <https://web.archive.org/web/20180829112450/http://linux.yyz.us/patch-format.html>
 
-Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer".
+Greg Kroah-Hartman，“如何惹�内核�系统维护人员�
   <http://www.kroah.com/log/linux/maintainer.html>
 
   <http://www.kroah.com/log/linux/maintainer-02.html>
@@ -642,16 +643,15 @@ Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer".
 
   <http://www.kroah.com/log/linux/maintainer-06.html>
 
-NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people!
+����别��巨型补�炸弹给linux-kernel@vger.kernel.org的人们了�
   <https://lore.kernel.org/r/20050711.125305.08322243.davem@davemloft.net>
 
-Kernel Documentation/process/coding-style.rst:
-  :ref:`Documentation/translations/zh_CN/process/coding-style.rst <cn_codingstyle>`
+内核 Documentation/translations/zh_CN/process/coding-style.rst
 
-Linus Torvalds's mail on the canonical patch format:
+Linus Torvalds关于标准补�格�的邮件
   <https://lore.kernel.org/r/Pine.LNX.4.58.0504071023190.28951@ppc970.osdl.org>
 
-Andi Kleen, "On submitting kernel patches"
-  Some strategies to get difficult or controversial changes in.
+Andi Kleen，“�交补�之路�
+  一些帮助�入困难或有争议的�更的策略。
 
   http://halobates.de/on-submitting-patches.pdf
diff --git a/Documentation/translations/zh_CN/scheduler/sched-design-CFS.rst b/Documentation/translations/zh_CN/scheduler/sched-design-CFS.rst
index 26b0f36f793d..3076402406c4 100644
--- a/Documentation/translations/zh_CN/scheduler/sched-design-CFS.rst
+++ b/Documentation/translations/zh_CN/scheduler/sched-design-CFS.rst
@@ -80,7 +80,7 @@ p->se.vruntime。一旦p->se.vruntimeå�˜å¾—足够大，其它的任务将æˆ�为æ
 CFS使用纳秒粒度的计时，��赖于任何jiffies或HZ的细节。因此CFS并��之�的调度器那样
 有“时间片�的概念，也没有任何���的设计。唯一�调的�数（你需�打开CONFIG_SCHED_DEBUG）是：
 
-   /proc/sys/kernel/sched_min_granularity_ns
+   /sys/kernel/debug/sched/min_granularity_ns
 
 它�以用�将调度器从“桌��模�（也就是低时延）调节为“�务器�（也就是高批处�）模�。
 它的默认设置是适�桌�的工作负载。SCHED_BATCH也被CFS调度器模�处�。
diff --git a/Documentation/translations/zh_TW/oops-tracing.txt b/Documentation/translations/zh_TW/oops-tracing.txt
deleted file mode 100644
index be8e59f2abaf..000000000000
--- a/Documentation/translations/zh_TW/oops-tracing.txt
+++ /dev/null
@@ -1,212 +0,0 @@
-Chinese translated version of Documentation/admin-guide/bug-hunting.rst
-
-If you have any comment or update to the content, please contact the
-original document maintainer directly.  However, if you have a problem
-communicating in English you can also ask the Chinese maintainer for
-help.  Contact the Chinese maintainer if this translation is outdated
-or if there is a problem with the translation.
-
-Traditional Chinese maintainer:  Hu Haowen <src.res@email.cn>
----------------------------------------------------------------------
-Documentation/admin-guide/bug-hunting.rst 的�體中文版翻譯
-
-如果想評論或更新本文的內容，請直接�繫原文檔的維護者。如果你使用英文
-交�有困難的話，也�以��體中文版維護者求助。如果本翻譯更新��時或
-者翻譯存在�題，請�繫�體中文版維護者。
-
-�體中文版維護者： 胡皓文 Hu Haowen <src.res@email.cn>
-�體中文版翻譯者： 胡皓文 Hu Haowen <src.res@email.cn>
-�體中文版校譯者： 胡皓文 Hu Haowen <src.res@email.cn>
-
-以下爲正文
----------------------------------------------------------------------
-
-注�： ksymoops 在2.6中是沒有用的。 請以原有格�使用Oops(來自dmesg，等等)。
-忽略任何這樣那樣關於「解碼Oops�或者「通�ksymoops�行�的文檔。 如果你貼出�行�
-ksymoops的來自2.6的Oops，人們�會讓你�貼一次。
-
-快速總�
--------------
-
-發�Oops並發�給看似相關的內核領域的維護者。別太擔心��上號。如果你�確定就發給
-和你所�的事情相關的代碼的負責人。 如果���試著�述怎樣�構。 那甚至比oops更有
-價值。
-
-如果你�於發�給誰一無所知， 發給linux-kernel@vger.kernel.org。感�你幫助Linux
-儘�能地穩定。
-
-Oops在哪裡?
-----------------------
-
-通常Oops文本由klogd從內核緩��里讀�並傳給syslogd，由syslogd寫到syslog文件中，
-典型地是/var/log/messages(�賴於/etc/syslog.conf)。有時klogd崩潰了,這種情�下你
-能夠�行dmesg > file來從內核緩��中讀�數據並�存下來。 �則你�以
-cat /proc/kmsg > file， 然而你必須介入中止傳輸， kmsg是一個「永���的文件�。如
-果機器崩潰壞到你�能輸入命令或者�碟��用那麼你有三種�擇:-
-
-（1） 手抄�幕上的文本待機器�啓後�輸入計算機。 麻煩但如果沒有��崩潰的準備，
-這是僅有的�擇。 �外，你�以用數�相機把�幕�下來-�太好，但比沒有強。 如果信
-�滾動到了終端的上�，你會發�以高分辯率啓動（比如，vga=791）會讓你讀到更多的文
-本。（注�：這需�vesafb，所以�『早期�的oops沒有幫助）
-
-（2）用串�終端啓動（請�看Documentation/admin-guide/serial-console.rst），�行一個null
-modem到�一�機器並用你喜歡的通訊工具��輸出。Minicom工作地很好。
-
-（3）使用Kdump（請�看Documentation/admin-guide/kdump/kdump.rst），
-使用在Documentation/admin-guide/kdump/gdbmacros.txt中定義的dmesg gdb�，從舊的內存中��內核
-環形緩��。
-
-完整信�
-----------------
-
-注�：以下來自於Linus的郵件�用於2.4內核。 我因爲歷�原因�留了它，並且因爲其中
-一些信��然�用。 特別注�的是，請忽略任何ksymoops的引用。
-
-From: Linus Torvalds <torvalds@osdl.org>
-
-怎樣跟蹤Oops.. [原發到linux-kernel的一�郵件]
-
-主�的竅門是有五年和這些煩人的oops消�打交�的經驗;-)
-
-實際上，你有辦法使它更簡單。我有兩個��的方法：
-
-	gdb /usr/src/linux/vmlinux
-	gdb> disassemble <offending_function>
-
-那是發��題的簡單辦法，至少如果bug報告�的好的情�下（象這個一樣-�行ksymoops
-得到oops發生的函數�函數內的�移）。
-
-哦，如果報告發生的內核以相�的編譯器和相似的�置編譯它會有幫助的。
-
-�一件��的事是�彙編bug報告的「Code�部分：ksymoops也會用正確的工具來�這件事，
-但如果沒有那些工具你�以寫一個傻程�：
-
-	char str[] = "\xXX\xXX\xXX...";
-	main(){}
-
-並用gcc -g編譯它然後執行「disassemble str�（XX部分是由Oops報告的值-你�以僅剪切
-粘貼並用「\x�替�空格-我就是這麼�的，因爲我懶得寫程�自動�這一切）。
-
-�外，你�以用scripts/decodecode這個shell腳本。它的使用方法是：
-decodecode < oops.txt
-
-「Code�之後的�六進�字節�能（在�些架構上）有一些當�指令之�的指令字節以�
-當�和之後的指令字節
-
-Code: f9 0f 8d f9 00 00 00 8d 42 0c e8 dd 26 11 c7 a1 60 ea 2b f9 8b 50 08 a1
-64 ea 2b f9 8d 34 82 8b 1e 85 db 74 6d 8b 15 60 ea 2b f9 <8b> 43 04 39 42 54
-7e 04 40 89 42 54 8b 43 04 3b 05 00 f6 52 c0
-
-最後，如果你想知�代碼來自哪裡，你�以：
-
-	cd /usr/src/linux
-	make fs/buffer.s 	# 或任何產生BUG的文件
-
-然後你會比gdb�彙編更清楚的知�發生了什麼。
-
-�在，�題是把你所�有的所有數據��起來：C�碼（關於它應該怎樣的一般知識），
-彙編代碼�其�彙編得到的代碼（�外還有從「oops�消�得到的寄存器狀態-�了解毀壞的
-指�有用，而且當你有了彙編代碼你也能拿其它的寄存器和任何它們�應的C表���匹�
-）。
-
-實際上，你僅需看看哪裡�匹�（這個例�是「Code��彙編和編譯器生�的代碼�匹�）。
-然後你須�找出爲什麼�匹�。通常很簡單-你看到代碼使用了空指�然後你看代碼想知�
-空指�是怎麼出�的，還有檢查它是��法..
-
-�在，如果明白這是一項耗時的工作而且需�一�點兒的專心，沒錯。這就是我爲什麼大多
-�是忽略那些沒有符號表信�的崩潰報告的原因：簡單的說太難查找了（我有一些
-程�用於在內核代碼段中�索特定的模�，而且有時我也已經能找出那些崩潰的地方，但是
-僅僅是找出正確的�列也確實需�相當紮實的內核知識）
-
-_有時_會發生這種情�，我僅看到崩潰中的�彙編代碼�列， 然後我馬上就明白�題出在
-哪裡。這時我��識到自己幹這個工作已經太長時間了;-)
-
-		Linus
-
-
----------------------------------------------------------------------------
-關於Oops跟蹤的註解：
-
-爲了幫助Linus和其它內核開發者，klogd�入了大�的支�來處��護錯誤。爲了�有�
-地�解�的完整支�至少應該使用1.3-pl3的sysklogd包。
-
-當�護錯誤發生時，klogd守護進程自動把內核日誌信�中的��地�翻譯�它們相應的符
-號。
-
-klogd執行兩種類型的地�解�。首先是�態翻譯其次是動態翻譯。�態翻譯和ksymoops
-一樣使用System.map文件。爲了��態翻譯klogd守護進程必須在�始化時能找到system
-map文件。關於klogd怎樣�索map文件請�看klogd手冊�。
-
-動態地�翻譯在使用內核��載模塊時很��。 因爲內核模塊的內存是從內核動態內存池
-里分�的，所以�管是模塊開始�置還是模塊中函數和符號的�置都�是固定的。
-
-內核支��許程�決定�載哪些模塊和它們在內存中�置的系統調用。使用這些系統調用
-klogd守護進程生�一張符號表用於調試發生在��載模塊中的�護錯誤。
-
-至少klogd會�供產生�護錯誤的模塊�。還�有�外的符號信�供��載模塊開發者�擇
-以從模塊中輸出符號信�。
-
-因爲內核模塊環境�能是動態的，所以必須有一種機制當模塊環境發生改變時來通知klogd
-守護進程。 有一些�用的命令行�項�許klogd�當�執行中的守護進程發�信號，告知符
-號信�應該被刷新了。 更多信�請�看klogd手冊�。
-
-sysklogd發布時包�一個補�修改了modules-2.0.0包，無論何時一個模塊�載或者�載都
-會自動�klogd發�信號。打上這個補��供了必�的�調試發生於內核��載模塊的�護
-錯誤的無縫支�。
-
-以下是被klogd處��的發生在��載模塊中的一個�護錯誤例�：
----------------------------------------------------------------------------
-Aug 29 09:51:01 blizard kernel: Unable to handle kernel paging request at virtual address f15e97cc
-Aug 29 09:51:01 blizard kernel: current->tss.cr3 = 0062d000, %cr3 = 0062d000
-Aug 29 09:51:01 blizard kernel: *pde = 00000000
-Aug 29 09:51:01 blizard kernel: Oops: 0002
-Aug 29 09:51:01 blizard kernel: CPU:    0
-Aug 29 09:51:01 blizard kernel: EIP:    0010:[oops:_oops+16/3868]
-Aug 29 09:51:01 blizard kernel: EFLAGS: 00010212
-Aug 29 09:51:01 blizard kernel: eax: 315e97cc   ebx: 003a6f80   ecx: 001be77b   edx: 00237c0c
-Aug 29 09:51:01 blizard kernel: esi: 00000000   edi: bffffdb3   ebp: 00589f90   esp: 00589f8c
-Aug 29 09:51:01 blizard kernel: ds: 0018   es: 0018   fs: 002b   gs: 002b   ss: 0018
-Aug 29 09:51:01 blizard kernel: Process oops_test (pid: 3374, process nr: 21, stackpage=00589000)
-Aug 29 09:51:01 blizard kernel: Stack: 315e97cc 00589f98 0100b0b4 bffffed4 0012e38e 00240c64 003a6f80 00000001
-Aug 29 09:51:01 blizard kernel:        00000000 00237810 bfffff00 0010a7fa 00000003 00000001 00000000 bfffff00
-Aug 29 09:51:01 blizard kernel:        bffffdb3 bffffed4 ffffffda 0000002b 0007002b 0000002b 0000002b 00000036
-Aug 29 09:51:01 blizard kernel: Call Trace: [oops:_oops_ioctl+48/80] [_sys_ioctl+254/272] [_system_call+82/128]
-Aug 29 09:51:01 blizard kernel: Code: c7 00 05 00 00 00 eb 08 90 90 90 90 90 90 90 90 89 ec 5d c3
----------------------------------------------------------------------------
-
-Dr. G.W. Wettstein           Oncology Research Div. Computing Facility
-Roger Maris Cancer Center    INTERNET: greg@wind.rmcc.com
-820 4th St. N.
-Fargo, ND  58122
-Phone: 701-234-7556
-
-
----------------------------------------------------------------------------
-�汙染的內核
-
-一些oops報告在程�記數器之後包�字符串'Tainted: '。這表明內核已經被一些�西給汙
-染了。 該字符串之後緊跟著一系列的�置�感的字符，�個代表一個特定的汙染值。
-
-  1：'G'如果所有�載的模塊都有GPL或相容的許�證，'P'如果�載了任何的專有模塊。
-沒有模塊MODULE_LICENSE或者帶有insmod�爲是與GPL�相容的的MODULE_LICENSE的模塊被
-�定是專有的。
-
-  2：'F'如果有任何通�「insmod -f�被強制�載的模塊，' '如果所有模塊都被正常�載。
-
-  3：'S'如果oops發生在SMP內核中，�行於沒有證明安全�行多處�器的硬體。 當�這種
-情�僅�於幾種�支�SMP的速�處�器。
-
-  4：'R'如果模塊通�「insmod -f�被強制�載，' '如果所有模塊都被正常�載。
-
-  5：'M'如果任何處�器報告了機器檢查異常，' '如果沒有發生機器檢查異常。
-
-  6：'B'如果�釋放函數發�了一個錯誤的�引用或者一些��期的�標誌。
-
-  7：'U'如果用戶或者用戶應用程�特別請求設置汙染標誌，�則' '。
-
-  8：'D'如果內核剛剛死掉，比如有OOPS或者BUG。
-
-使用'Tainted: '字符串的主�原因是�告訴內核調試者，這是�是一個乾淨的內核亦或發
-生了任何的�正常的事。汙染是永久的：�使出錯的模塊已經被�載了，汙染值�然存在，
-以表明內核��值得信任。
-
diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst
index a61eac0c73f8..c78da9ce0ec4 100644
--- a/Documentation/userspace-api/index.rst
+++ b/Documentation/userspace-api/index.rst
@@ -26,6 +26,7 @@ place where this information is gathered.
    ioctl/index
    iommu
    media/index
+   netlink/index
    sysfs-platform_profile
    vduse
    futex2
diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst
index 3b985b19f39d..5f81e2a24a5c 100644
--- a/Documentation/userspace-api/ioctl/ioctl-number.rst
+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst
@@ -308,7 +308,6 @@ Code  Seq#    Include File                                           Comments
 0x89  00-06  arch/x86/include/asm/sockios.h
 0x89  0B-DF  linux/sockios.h
 0x89  E0-EF  linux/sockios.h                                         SIOCPROTOPRIVATE range
-0x89  E0-EF  linux/dn.h                                              PROTOPRIVATE range
 0x89  F0-FF  linux/sockios.h                                         SIOCDEVPRIVATE range
 0x8B  all    linux/wireless.h
 0x8C  00-3F                                                          WiNRADiO driver
diff --git a/Documentation/userspace-api/landlock.rst b/Documentation/userspace-api/landlock.rst
index b8ea59493964..cec780c2f497 100644
--- a/Documentation/userspace-api/landlock.rst
+++ b/Documentation/userspace-api/landlock.rst
@@ -8,7 +8,7 @@ Landlock: unprivileged access control
 =====================================
 
 :Author: Mickaël Salaün
-:Date: May 2022
+:Date: September 2022
 
 The goal of Landlock is to enable to restrict ambient rights (e.g. global
 filesystem access) for a set of processes.  Because Landlock is a stackable
@@ -69,7 +69,7 @@ should try to protect users as much as possible whatever the kernel they are
 using.  To avoid binary enforcement (i.e. either all security features or
 none), we can leverage a dedicated Landlock command to get the current version
 of the Landlock ABI and adapt the handled accesses.  Let's check if we should
-remove the `LANDLOCK_ACCESS_FS_REFER` access right which is only supported
+remove the ``LANDLOCK_ACCESS_FS_REFER`` access right which is only supported
 starting with the second version of the ABI.
 
 .. code-block:: c
@@ -128,7 +128,7 @@ descriptor.
 It may also be required to create rules following the same logic as explained
 for the ruleset creation, by filtering access rights according to the Landlock
 ABI version.  In this example, this is not required because
-`LANDLOCK_ACCESS_FS_REFER` is not allowed by any rule.
+``LANDLOCK_ACCESS_FS_REFER`` is not allowed by any rule.
 
 We now have a ruleset with one rule allowing read access to ``/usr`` while
 denying all other handled accesses for the filesystem.  The next step is to
@@ -154,8 +154,8 @@ The current thread is now ready to sandbox itself with the ruleset.
     }
     close(ruleset_fd);
 
-If the `landlock_restrict_self` system call succeeds, the current thread is now
-restricted and this policy will be enforced on all its subsequently created
+If the ``landlock_restrict_self`` system call succeeds, the current thread is
+now restricted and this policy will be enforced on all its subsequently created
 children as well.  Once a thread is landlocked, there is no way to remove its
 security policy; only adding more restrictions is allowed.  These threads are
 now in a new Landlock domain, merge of their parent one (if any) with the new
@@ -170,12 +170,13 @@ It is recommended setting access rights to file hierarchy leaves as much as
 possible.  For instance, it is better to be able to have ``~/doc/`` as a
 read-only hierarchy and ``~/tmp/`` as a read-write hierarchy, compared to
 ``~/`` as a read-only hierarchy and ``~/tmp/`` as a read-write hierarchy.
-Following this good practice leads to self-sufficient hierarchies that don't
+Following this good practice leads to self-sufficient hierarchies that do not
 depend on their location (i.e. parent directories).  This is particularly
 relevant when we want to allow linking or renaming.  Indeed, having consistent
 access rights per directory enables to change the location of such directory
 without relying on the destination directory access rights (except those that
-are required for this operation, see `LANDLOCK_ACCESS_FS_REFER` documentation).
+are required for this operation, see ``LANDLOCK_ACCESS_FS_REFER``
+documentation).
 Having self-sufficient hierarchies also helps to tighten the required access
 rights to the minimal set of data.  This also helps avoid sinkhole directories,
 i.e.  directories where data can be linked to but not linked from.  However,
@@ -259,7 +260,7 @@ Backward and forward compatibility
 
 Landlock is designed to be compatible with past and future versions of the
 kernel.  This is achieved thanks to the system call attributes and the
-associated bitflags, particularly the ruleset's `handled_access_fs`.  Making
+associated bitflags, particularly the ruleset's ``handled_access_fs``.  Making
 handled access right explicit enables the kernel and user space to have a clear
 contract with each other.  This is required to make sure sandboxing will not
 get stricter with a system update, which could break applications.
@@ -380,8 +381,8 @@ by the Documentation/admin-guide/cgroup-v1/memory.rst.
 Previous limitations
 ====================
 
-File renaming and linking (ABI 1)
----------------------------------
+File renaming and linking (ABI < 2)
+-----------------------------------
 
 Because Landlock targets unprivileged access controls, it needs to properly
 handle composition of rules.  Such property also implies rules nesting.
@@ -394,7 +395,7 @@ according to the potentially lost constraints.  To protect against privilege
 escalations through renaming or linking, and for the sake of simplicity,
 Landlock previously limited linking and renaming to the same directory.
 Starting with the Landlock ABI version 2, it is now possible to securely
-control renaming and linking thanks to the new `LANDLOCK_ACCESS_FS_REFER`
+control renaming and linking thanks to the new ``LANDLOCK_ACCESS_FS_REFER``
 access right.
 
 .. _kernel_support:
@@ -403,14 +404,14 @@ Kernel support
 ==============
 
 Landlock was first introduced in Linux 5.13 but it must be configured at build
-time with `CONFIG_SECURITY_LANDLOCK=y`.  Landlock must also be enabled at boot
+time with ``CONFIG_SECURITY_LANDLOCK=y``.  Landlock must also be enabled at boot
 time as the other security modules.  The list of security modules enabled by
-default is set with `CONFIG_LSM`.  The kernel configuration should then
-contains `CONFIG_LSM=landlock,[...]` with `[...]`  as the list of other
+default is set with ``CONFIG_LSM``.  The kernel configuration should then
+contains ``CONFIG_LSM=landlock,[...]`` with ``[...]``  as the list of other
 potentially useful security modules for the running system (see the
-`CONFIG_LSM` help).
+``CONFIG_LSM`` help).
 
-If the running kernel doesn't have `landlock` in `CONFIG_LSM`, then we can
+If the running kernel does not have ``landlock`` in ``CONFIG_LSM``, then we can
 still enable it by adding ``lsm=landlock,[...]`` to
 Documentation/admin-guide/kernel-parameters.rst thanks to the bootloader
 configuration.
diff --git a/Documentation/userspace-api/netlink/index.rst b/Documentation/userspace-api/netlink/index.rst
new file mode 100644
index 000000000000..b0c21538d97d
--- /dev/null
+++ b/Documentation/userspace-api/netlink/index.rst
@@ -0,0 +1,12 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+
+================
+Netlink Handbook
+================
+
+Netlink documentation for users.
+
+.. toctree::
+   :maxdepth: 2
+
+   intro
diff --git a/Documentation/userspace-api/netlink/intro.rst b/Documentation/userspace-api/netlink/intro.rst
new file mode 100644
index 000000000000..0955e9f203d3
--- /dev/null
+++ b/Documentation/userspace-api/netlink/intro.rst
@@ -0,0 +1,681 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+
+=======================
+Introduction to Netlink
+=======================
+
+Netlink is often described as an ioctl() replacement.
+It aims to replace fixed-format C structures as supplied
+to ioctl() with a format which allows an easy way to add
+or extended the arguments.
+
+To achieve this Netlink uses a minimal fixed-format metadata header
+followed by multiple attributes in the TLV (type, length, value) format.
+
+Unfortunately the protocol has evolved over the years, in an organic
+and undocumented fashion, making it hard to coherently explain.
+To make the most practical sense this document starts by describing
+netlink as it is used today and dives into more "historical" uses
+in later sections.
+
+Opening a socket
+================
+
+Netlink communication happens over sockets, a socket needs to be
+opened first:
+
+.. code-block:: c
+
+  fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
+
+The use of sockets allows for a natural way of exchanging information
+in both directions (to and from the kernel). The operations are still
+performed synchronously when applications send() the request but
+a separate recv() system call is needed to read the reply.
+
+A very simplified flow of a Netlink "call" will therefore look
+something like:
+
+.. code-block:: c
+
+  fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
+
+  /* format the request */
+  send(fd, &request, sizeof(request));
+  n = recv(fd, &response, RSP_BUFFER_SIZE);
+  /* interpret the response */
+
+Netlink also provides natural support for "dumping", i.e. communicating
+to user space all objects of a certain type (e.g. dumping all network
+interfaces).
+
+.. code-block:: c
+
+  fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC);
+
+  /* format the dump request */
+  send(fd, &request, sizeof(request));
+  while (1) {
+    n = recv(fd, &buffer, RSP_BUFFER_SIZE);
+    /* one recv() call can read multiple messages, hence the loop below */
+    for (nl_msg in buffer) {
+      if (nl_msg.nlmsg_type == NLMSG_DONE)
+        goto dump_finished;
+      /* process the object */
+    }
+  }
+  dump_finished:
+
+The first two arguments of the socket() call require little explanation -
+it is opening a Netlink socket, with all headers provided by the user
+(hence NETLINK, RAW). The last argument is the protocol within Netlink.
+This field used to identify the subsystem with which the socket will
+communicate.
+
+Classic vs Generic Netlink
+--------------------------
+
+Initial implementation of Netlink depended on a static allocation
+of IDs to subsystems and provided little supporting infrastructure.
+Let us refer to those protocols collectively as **Classic Netlink**.
+The list of them is defined on top of the ``include/uapi/linux/netlink.h``
+file, they include among others - general networking (NETLINK_ROUTE),
+iSCSI (NETLINK_ISCSI), and audit (NETLINK_AUDIT).
+
+**Generic Netlink** (introduced in 2005) allows for dynamic registration of
+subsystems (and subsystem ID allocation), introspection and simplifies
+implementing the kernel side of the interface.
+
+The following section describes how to use Generic Netlink, as the
+number of subsystems using Generic Netlink outnumbers the older
+protocols by an order of magnitude. There are also no plans for adding
+more Classic Netlink protocols to the kernel.
+Basic information on how communicating with core networking parts of
+the Linux kernel (or another of the 20 subsystems using Classic
+Netlink) differs from Generic Netlink is provided later in this document.
+
+Generic Netlink
+===============
+
+In addition to the Netlink fixed metadata header each Netlink protocol
+defines its own fixed metadata header. (Similarly to how network
+headers stack - Ethernet > IP > TCP we have Netlink > Generic N. > Family.)
+
+A Netlink message always starts with struct nlmsghdr, which is followed
+by a protocol-specific header. In case of Generic Netlink the protocol
+header is struct genlmsghdr.
+
+The practical meaning of the fields in case of Generic Netlink is as follows:
+
+.. code-block:: c
+
+  struct nlmsghdr {
+	__u32	nlmsg_len;	/* Length of message including headers */
+	__u16	nlmsg_type;	/* Generic Netlink Family (subsystem) ID */
+	__u16	nlmsg_flags;	/* Flags - request or dump */
+	__u32	nlmsg_seq;	/* Sequence number */
+	__u32	nlmsg_pid;	/* Port ID, set to 0 */
+  };
+  struct genlmsghdr {
+	__u8	cmd;		/* Command, as defined by the Family */
+	__u8	version;	/* Irrelevant, set to 1 */
+	__u16	reserved;	/* Reserved, set to 0 */
+  };
+  /* TLV attributes follow... */
+
+In Classic Netlink :c:member:`nlmsghdr.nlmsg_type` used to identify
+which operation within the subsystem the message was referring to
+(e.g. get information about a netdev). Generic Netlink needs to mux
+multiple subsystems in a single protocol so it uses this field to
+identify the subsystem, and :c:member:`genlmsghdr.cmd` identifies
+the operation instead. (See :ref:`res_fam` for
+information on how to find the Family ID of the subsystem of interest.)
+Note that the first 16 values (0 - 15) of this field are reserved for
+control messages both in Classic Netlink and Generic Netlink.
+See :ref:`nl_msg_type` for more details.
+
+There are 3 usual types of message exchanges on a Netlink socket:
+
+ - performing a single action (``do``);
+ - dumping information (``dump``);
+ - getting asynchronous notifications (``multicast``).
+
+Classic Netlink is very flexible and presumably allows other types
+of exchanges to happen, but in practice those are the three that get
+used.
+
+Asynchronous notifications are sent by the kernel and received by
+the user sockets which subscribed to them. ``do`` and ``dump`` requests
+are initiated by the user. :c:member:`nlmsghdr.nlmsg_flags` should
+be set as follows:
+
+ - for ``do``: ``NLM_F_REQUEST | NLM_F_ACK``
+ - for ``dump``: ``NLM_F_REQUEST | NLM_F_ACK | NLM_F_DUMP``
+
+:c:member:`nlmsghdr.nlmsg_seq` should be a set to a monotonically
+increasing value. The value gets echoed back in responses and doesn't
+matter in practice, but setting it to an increasing value for each
+message sent is considered good hygiene. The purpose of the field is
+matching responses to requests. Asynchronous notifications will have
+:c:member:`nlmsghdr.nlmsg_seq` of ``0``.
+
+:c:member:`nlmsghdr.nlmsg_pid` is the Netlink equivalent of an address.
+This field can be set to ``0`` when talking to the kernel.
+See :ref:`nlmsg_pid` for the (uncommon) uses of the field.
+
+The expected use for :c:member:`genlmsghdr.version` was to allow
+versioning of the APIs provided by the subsystems. No subsystem to
+date made significant use of this field, so setting it to ``1`` seems
+like a safe bet.
+
+.. _nl_msg_type:
+
+Netlink message types
+---------------------
+
+As previously mentioned :c:member:`nlmsghdr.nlmsg_type` carries
+protocol specific values but the first 16 identifiers are reserved
+(first subsystem specific message type should be equal to
+``NLMSG_MIN_TYPE`` which is ``0x10``).
+
+There are only 4 Netlink control messages defined:
+
+ - ``NLMSG_NOOP`` - ignore the message, not used in practice;
+ - ``NLMSG_ERROR`` - carries the return code of an operation;
+ - ``NLMSG_DONE`` - marks the end of a dump;
+ - ``NLMSG_OVERRUN`` - socket buffer has overflown, not used to date.
+
+``NLMSG_ERROR`` and ``NLMSG_DONE`` are of practical importance.
+They carry return codes for operations. Note that unless
+the ``NLM_F_ACK`` flag is set on the request Netlink will not respond
+with ``NLMSG_ERROR`` if there is no error. To avoid having to special-case
+this quirk it is recommended to always set ``NLM_F_ACK``.
+
+The format of ``NLMSG_ERROR`` is described by struct nlmsgerr::
+
+  ----------------------------------------------
+  | struct nlmsghdr - response header          |
+  ----------------------------------------------
+  |    int error                               |
+  ----------------------------------------------
+  | struct nlmsghdr - original request header |
+  ----------------------------------------------
+  | ** optionally (1) payload of the request   |
+  ----------------------------------------------
+  | ** optionally (2) extended ACK             |
+  ----------------------------------------------
+
+There are two instances of struct nlmsghdr here, first of the response
+and second of the request. ``NLMSG_ERROR`` carries the information about
+the request which led to the error. This could be useful when trying
+to match requests to responses or re-parse the request to dump it into
+logs.
+
+The payload of the request is not echoed in messages reporting success
+(``error == 0``) or if ``NETLINK_CAP_ACK`` setsockopt() was set.
+The latter is common
+and perhaps recommended as having to read a copy of every request back
+from the kernel is rather wasteful. The absence of request payload
+is indicated by ``NLM_F_CAPPED`` in :c:member:`nlmsghdr.nlmsg_flags`.
+
+The second optional element of ``NLMSG_ERROR`` are the extended ACK
+attributes. See :ref:`ext_ack` for more details. The presence
+of extended ACK is indicated by ``NLM_F_ACK_TLVS`` in
+:c:member:`nlmsghdr.nlmsg_flags`.
+
+``NLMSG_DONE`` is simpler, the request is never echoed but the extended
+ACK attributes may be present::
+
+  ----------------------------------------------
+  | struct nlmsghdr - response header          |
+  ----------------------------------------------
+  |    int error                               |
+  ----------------------------------------------
+  | ** optionally extended ACK                 |
+  ----------------------------------------------
+
+.. _res_fam:
+
+Resolving the Family ID
+-----------------------
+
+This section explains how to find the Family ID of a subsystem.
+It also serves as an example of Generic Netlink communication.
+
+Generic Netlink is itself a subsystem exposed via the Generic Netlink API.
+To avoid a circular dependency Generic Netlink has a statically allocated
+Family ID (``GENL_ID_CTRL`` which is equal to ``NLMSG_MIN_TYPE``).
+The Generic Netlink family implements a command used to find out information
+about other families (``CTRL_CMD_GETFAMILY``).
+
+To get information about the Generic Netlink family named for example
+``"test1"`` we need to send a message on the previously opened Generic Netlink
+socket. The message should target the Generic Netlink Family (1), be a
+``do`` (2) call to ``CTRL_CMD_GETFAMILY`` (3). A ``dump`` version of this
+call would make the kernel respond with information about *all* the families
+it knows about. Last but not least the name of the family in question has
+to be specified (4) as an attribute with the appropriate type::
+
+  struct nlmsghdr:
+    __u32 nlmsg_len:	32
+    __u16 nlmsg_type:	GENL_ID_CTRL               // (1)
+    __u16 nlmsg_flags:	NLM_F_REQUEST | NLM_F_ACK  // (2)
+    __u32 nlmsg_seq:	1
+    __u32 nlmsg_pid:	0
+
+  struct genlmsghdr:
+    __u8 cmd:		CTRL_CMD_GETFAMILY         // (3)
+    __u8 version:	2 /* or 1, doesn't matter */
+    __u16 reserved:	0
+
+  struct nlattr:                                   // (4)
+    __u16 nla_len:	10
+    __u16 nla_type:	CTRL_ATTR_FAMILY_NAME
+    char data: 		test1\0
+
+  (padding:)
+    char data:		\0\0
+
+The length fields in Netlink (:c:member:`nlmsghdr.nlmsg_len`
+and :c:member:`nlattr.nla_len`) always *include* the header.
+Attribute headers in netlink must be aligned to 4 bytes from the start
+of the message, hence the extra ``\0\0`` after ``CTRL_ATTR_FAMILY_NAME``.
+The attribute lengths *exclude* the padding.
+
+If the family is found kernel will reply with two messages, the response
+with all the information about the family::
+
+  /* Message #1 - reply */
+  struct nlmsghdr:
+    __u32 nlmsg_len:	136
+    __u16 nlmsg_type:	GENL_ID_CTRL
+    __u16 nlmsg_flags:	0
+    __u32 nlmsg_seq:	1    /* echoed from our request */
+    __u32 nlmsg_pid:	5831 /* The PID of our user space process */
+
+  struct genlmsghdr:
+    __u8 cmd:		CTRL_CMD_GETFAMILY
+    __u8 version:	2
+    __u16 reserved:	0
+
+  struct nlattr:
+    __u16 nla_len:	10
+    __u16 nla_type:	CTRL_ATTR_FAMILY_NAME
+    char data: 		test1\0
+
+  (padding:)
+    data:		\0\0
+
+  struct nlattr:
+    __u16 nla_len:	6
+    __u16 nla_type:	CTRL_ATTR_FAMILY_ID
+    __u16: 		123  /* The Family ID we are after */
+
+  (padding:)
+    char data:		\0\0
+
+  struct nlattr:
+    __u16 nla_len:	9
+    __u16 nla_type:	CTRL_ATTR_FAMILY_VERSION
+    __u16: 		1
+
+  /* ... etc, more attributes will follow. */
+
+And the error code (success) since ``NLM_F_ACK`` had been set on the request::
+
+  /* Message #2 - the ACK */
+  struct nlmsghdr:
+    __u32 nlmsg_len:	36
+    __u16 nlmsg_type:	NLMSG_ERROR
+    __u16 nlmsg_flags:	NLM_F_CAPPED /* There won't be a payload */
+    __u32 nlmsg_seq:	1    /* echoed from our request */
+    __u32 nlmsg_pid:	5831 /* The PID of our user space process */
+
+  int error:		0
+
+  struct nlmsghdr: /* Copy of the request header as we sent it */
+    __u32 nlmsg_len:	32
+    __u16 nlmsg_type:	GENL_ID_CTRL
+    __u16 nlmsg_flags:	NLM_F_REQUEST | NLM_F_ACK
+    __u32 nlmsg_seq:	1
+    __u32 nlmsg_pid:	0
+
+The order of attributes (struct nlattr) is not guaranteed so the user
+has to walk the attributes and parse them.
+
+Note that Generic Netlink sockets are not associated or bound to a single
+family. A socket can be used to exchange messages with many different
+families, selecting the recipient family on message-by-message basis using
+the :c:member:`nlmsghdr.nlmsg_type` field.
+
+.. _ext_ack:
+
+Extended ACK
+------------
+
+Extended ACK controls reporting of additional error/warning TLVs
+in ``NLMSG_ERROR`` and ``NLMSG_DONE`` messages. To maintain backward
+compatibility this feature has to be explicitly enabled by setting
+the ``NETLINK_EXT_ACK`` setsockopt() to ``1``.
+
+Types of extended ack attributes are defined in enum nlmsgerr_attrs.
+The most commonly used attributes are ``NLMSGERR_ATTR_MSG``,
+``NLMSGERR_ATTR_OFFS`` and ``NLMSGERR_ATTR_MISS_*``.
+
+``NLMSGERR_ATTR_MSG`` carries a message in English describing
+the encountered problem. These messages are far more detailed
+than what can be expressed thru standard UNIX error codes.
+
+``NLMSGERR_ATTR_OFFS`` points to the attribute which caused the problem.
+
+``NLMSGERR_ATTR_MISS_TYPE`` and ``NLMSGERR_ATTR_MISS_NEST``
+inform about a missing attribute.
+
+Extended ACKs can be reported on errors as well as in case of success.
+The latter should be treated as a warning.
+
+Extended ACKs greatly improve the usability of Netlink and should
+always be enabled, appropriately parsed and reported to the user.
+
+Advanced topics
+===============
+
+Dump consistency
+----------------
+
+Some of the data structures kernel uses for storing objects make
+it hard to provide an atomic snapshot of all the objects in a dump
+(without impacting the fast-paths updating them).
+
+Kernel may set the ``NLM_F_DUMP_INTR`` flag on any message in a dump
+(including the ``NLMSG_DONE`` message) if the dump was interrupted and
+may be inconsistent (e.g. missing objects). User space should retry
+the dump if it sees the flag set.
+
+Introspection
+-------------
+
+The basic introspection abilities are enabled by access to the Family
+object as reported in :ref:`res_fam`. User can query information about
+the Generic Netlink family, including which operations are supported
+by the kernel and what attributes the kernel understands.
+Family information includes the highest ID of an attribute kernel can parse,
+a separate command (``CTRL_CMD_GETPOLICY``) provides detailed information
+about supported attributes, including ranges of values the kernel accepts.
+
+Querying family information is useful in cases when user space needs
+to make sure that the kernel has support for a feature before issuing
+a request.
+
+.. _nlmsg_pid:
+
+nlmsg_pid
+---------
+
+:c:member:`nlmsghdr.nlmsg_pid` is the Netlink equivalent of an address.
+It is referred to as Port ID, sometimes Process ID because for historical
+reasons if the application does not select (bind() to) an explicit Port ID
+kernel will automatically assign it the ID equal to its Process ID
+(as reported by the getpid() system call).
+
+Similarly to the bind() semantics of the TCP/IP network protocols the value
+of zero means "assign automatically", hence it is common for applications
+to leave the :c:member:`nlmsghdr.nlmsg_pid` field initialized to ``0``.
+
+The field is still used today in rare cases when kernel needs to send
+a unicast notification. User space application can use bind() to associate
+its socket with a specific PID, it then communicates its PID to the kernel.
+This way the kernel can reach the specific user space process.
+
+This sort of communication is utilized in UMH (User Mode Helper)-like
+scenarios when kernel needs to trigger user space processing or ask user
+space for a policy decision.
+
+Multicast notifications
+-----------------------
+
+One of the strengths of Netlink is the ability to send event notifications
+to user space. This is a unidirectional form of communication (kernel ->
+user) and does not involve any control messages like ``NLMSG_ERROR`` or
+``NLMSG_DONE``.
+
+For example the Generic Netlink family itself defines a set of multicast
+notifications about registered families. When a new family is added the
+sockets subscribed to the notifications will get the following message::
+
+  struct nlmsghdr:
+    __u32 nlmsg_len:	136
+    __u16 nlmsg_type:	GENL_ID_CTRL
+    __u16 nlmsg_flags:	0
+    __u32 nlmsg_seq:	0
+    __u32 nlmsg_pid:	0
+
+  struct genlmsghdr:
+    __u8 cmd:		CTRL_CMD_NEWFAMILY
+    __u8 version:	2
+    __u16 reserved:	0
+
+  struct nlattr:
+    __u16 nla_len:	10
+    __u16 nla_type:	CTRL_ATTR_FAMILY_NAME
+    char data: 		test1\0
+
+  (padding:)
+    data:		\0\0
+
+  struct nlattr:
+    __u16 nla_len:	6
+    __u16 nla_type:	CTRL_ATTR_FAMILY_ID
+    __u16: 		123  /* The Family ID we are after */
+
+  (padding:)
+    char data:		\0\0
+
+  struct nlattr:
+    __u16 nla_len:	9
+    __u16 nla_type:	CTRL_ATTR_FAMILY_VERSION
+    __u16: 		1
+
+  /* ... etc, more attributes will follow. */
+
+The notification contains the same information as the response
+to the ``CTRL_CMD_GETFAMILY`` request.
+
+The Netlink headers of the notification are mostly 0 and irrelevant.
+The :c:member:`nlmsghdr.nlmsg_seq` may be either zero or a monotonically
+increasing notification sequence number maintained by the family.
+
+To receive notifications the user socket must subscribe to the relevant
+notification group. Much like the Family ID, the Group ID for a given
+multicast group is dynamic and can be found inside the Family information.
+The ``CTRL_ATTR_MCAST_GROUPS`` attribute contains nests with names
+(``CTRL_ATTR_MCAST_GRP_NAME``) and IDs (``CTRL_ATTR_MCAST_GRP_ID``) of
+the groups family.
+
+Once the Group ID is known a setsockopt() call adds the socket to the group:
+
+.. code-block:: c
+
+  unsigned int group_id;
+
+  /* .. find the group ID... */
+
+  setsockopt(fd, SOL_NETLINK, NETLINK_ADD_MEMBERSHIP,
+             &group_id, sizeof(group_id));
+
+The socket will now receive notifications.
+
+It is recommended to use separate sockets for receiving notifications
+and sending requests to the kernel. The asynchronous nature of notifications
+means that they may get mixed in with the responses making the message
+handling much harder.
+
+Buffer sizing
+-------------
+
+Netlink sockets are datagram sockets rather than stream sockets,
+meaning that each message must be received in its entirety by a single
+recv()/recvmsg() system call. If the buffer provided by the user is too
+short, the message will be truncated and the ``MSG_TRUNC`` flag set
+in struct msghdr (struct msghdr is the second argument
+of the recvmsg() system call, *not* a Netlink header).
+
+Upon truncation the remaining part of the message is discarded.
+
+Netlink expects that the user buffer will be at least 8kB or a page
+size of the CPU architecture, whichever is bigger. Particular Netlink
+families may, however, require a larger buffer. 32kB buffer is recommended
+for most efficient handling of dumps (larger buffer fits more dumped
+objects and therefore fewer recvmsg() calls are needed).
+
+Classic Netlink
+===============
+
+The main differences between Classic and Generic Netlink are the dynamic
+allocation of subsystem identifiers and availability of introspection.
+In theory the protocol does not differ significantly, however, in practice
+Classic Netlink experimented with concepts which were abandoned in Generic
+Netlink (really, they usually only found use in a small corner of a single
+subsystem). This section is meant as an explainer of a few of such concepts,
+with the explicit goal of giving the Generic Netlink
+users the confidence to ignore them when reading the uAPI headers.
+
+Most of the concepts and examples here refer to the ``NETLINK_ROUTE`` family,
+which covers much of the configuration of the Linux networking stack.
+Real documentation of that family, deserves a chapter (or a book) of its own.
+
+Families
+--------
+
+Netlink refers to subsystems as families. This is a remnant of using
+sockets and the concept of protocol families, which are part of message
+demultiplexing in ``NETLINK_ROUTE``.
+
+Sadly every layer of encapsulation likes to refer to whatever it's carrying
+as "families" making the term very confusing:
+
+ 1. AF_NETLINK is a bona fide socket protocol family
+ 2. AF_NETLINK's documentation refers to what comes after its own
+    header (struct nlmsghdr) in a message as a "Family Header"
+ 3. Generic Netlink is a family for AF_NETLINK (struct genlmsghdr follows
+    struct nlmsghdr), yet it also calls its users "Families".
+
+Note that the Generic Netlink Family IDs are in a different "ID space"
+and overlap with Classic Netlink protocol numbers (e.g. ``NETLINK_CRYPTO``
+has the Classic Netlink protocol ID of 21 which Generic Netlink will
+happily allocate to one of its families as well).
+
+Strict checking
+---------------
+
+The ``NETLINK_GET_STRICT_CHK`` socket option enables strict input checking
+in ``NETLINK_ROUTE``. It was needed because historically kernel did not
+validate the fields of structures it didn't process. This made it impossible
+to start using those fields later without risking regressions in applications
+which initialized them incorrectly or not at all.
+
+``NETLINK_GET_STRICT_CHK`` declares that the application is initializing
+all fields correctly. It also opts into validating that message does not
+contain trailing data and requests that kernel rejects attributes with
+type higher than largest attribute type known to the kernel.
+
+``NETLINK_GET_STRICT_CHK`` is not used outside of ``NETLINK_ROUTE``.
+
+Unknown attributes
+------------------
+
+Historically Netlink ignored all unknown attributes. The thinking was that
+it would free the application from having to probe what kernel supports.
+The application could make a request to change the state and check which
+parts of the request "stuck".
+
+This is no longer the case for new Generic Netlink families and those opting
+in to strict checking. See enum netlink_validation for validation types
+performed.
+
+Fixed metadata and structures
+-----------------------------
+
+Classic Netlink made liberal use of fixed-format structures within
+the messages. Messages would commonly have a structure with
+a considerable number of fields after struct nlmsghdr. It was also
+common to put structures with multiple members inside attributes,
+without breaking each member into an attribute of its own.
+
+This has caused problems with validation and extensibility and
+therefore using binary structures is actively discouraged for new
+attributes.
+
+Request types
+-------------
+
+``NETLINK_ROUTE`` categorized requests into 4 types ``NEW``, ``DEL``, ``GET``,
+and ``SET``. Each object can handle all or some of those requests
+(objects being netdevs, routes, addresses, qdiscs etc.) Request type
+is defined by the 2 lowest bits of the message type, so commands for
+new objects would always be allocated with a stride of 4.
+
+Each object would also have it's own fixed metadata shared by all request
+types (e.g. struct ifinfomsg for netdev requests, struct ifaddrmsg for address
+requests, struct tcmsg for qdisc requests).
+
+Even though other protocols and Generic Netlink commands often use
+the same verbs in their message names (``GET``, ``SET``) the concept
+of request types did not find wider adoption.
+
+Notification echo
+-----------------
+
+``NLM_F_ECHO`` requests for notifications resulting from the request
+to be queued onto the requesting socket. This is useful to discover
+the impact of the request.
+
+Note that this feature is not universally implemented.
+
+Other request-type-specific flags
+---------------------------------
+
+Classic Netlink defined various flags for its ``GET``, ``NEW``
+and ``DEL`` requests in the upper byte of nlmsg_flags in struct nlmsghdr.
+Since request types have not been generalized the request type specific
+flags are rarely used (and considered deprecated for new families).
+
+For ``GET`` - ``NLM_F_ROOT`` and ``NLM_F_MATCH`` are combined into
+``NLM_F_DUMP``, and not used separately. ``NLM_F_ATOMIC`` is never used.
+
+For ``DEL`` - ``NLM_F_NONREC`` is only used by nftables and ``NLM_F_BULK``
+only by FDB some operations.
+
+The flags for ``NEW`` are used most commonly in classic Netlink. Unfortunately,
+the meaning is not crystal clear. The following description is based on the
+best guess of the intention of the authors, and in practice all families
+stray from it in one way or another. ``NLM_F_REPLACE`` asks to replace
+an existing object, if no matching object exists the operation should fail.
+``NLM_F_EXCL`` has the opposite semantics and only succeeds if object already
+existed.
+``NLM_F_CREATE`` asks for the object to be created if it does not
+exist, it can be combined with ``NLM_F_REPLACE`` and ``NLM_F_EXCL``.
+
+A comment in the main Netlink uAPI header states::
+
+   4.4BSD ADD		NLM_F_CREATE|NLM_F_EXCL
+   4.4BSD CHANGE	NLM_F_REPLACE
+
+   True CHANGE		NLM_F_CREATE|NLM_F_REPLACE
+   Append		NLM_F_CREATE
+   Check		NLM_F_EXCL
+
+which seems to indicate that those flags predate request types.
+``NLM_F_REPLACE`` without ``NLM_F_CREATE`` was initially used instead
+of ``SET`` commands.
+``NLM_F_EXCL`` without ``NLM_F_CREATE`` was used to check if object exists
+without creating it, presumably predating ``GET`` commands.
+
+``NLM_F_APPEND`` indicates that if one key can have multiple objects associated
+with it (e.g. multiple next-hop objects for a route) the new object should be
+added to the list rather than replacing the entire list.
+
+uAPI reference
+==============
+
+.. kernel-doc:: include/uapi/linux/netlink.h
diff --git a/Documentation/virt/kvm/x86/mmu.rst b/Documentation/virt/kvm/x86/mmu.rst
index 8739120f4300..8364afa228ec 100644
--- a/Documentation/virt/kvm/x86/mmu.rst
+++ b/Documentation/virt/kvm/x86/mmu.rst
@@ -377,7 +377,7 @@ Emulating cr0.wp
 ================
 
 If tdp is not enabled, the host must keep cr0.wp=1 so page write protection
-works for the guest kernel, not guest guest userspace.  When the guest
+works for the guest kernel, not guest userspace.  When the guest
 cr0.wp=1, this does not present a problem.  However when the guest cr0.wp=0,
 we cannot map the permissions for gpte.u=1, gpte.w=0 to any spte (the
 semantics require allowing any guest kernel access plus user read access).
diff --git a/Documentation/w1/masters/ds2490.rst b/Documentation/w1/masters/ds2490.rst
index 7e5b50f9c0f5..842e7ae80424 100644
--- a/Documentation/w1/masters/ds2490.rst
+++ b/Documentation/w1/masters/ds2490.rst
@@ -52,7 +52,7 @@ Notes and limitations.
   clear the entire bulk in buffer.  It would be possible to read the
   maximum buffer size to not run into this error condition, only extra
   bytes in the buffer is a logic error in the driver.  The code should
-  should match reads and writes as well as data sizes.  Reads and
+  match reads and writes as well as data sizes.  Reads and
   writes are serialized and the status verifies that the chip is idle
   (and data is available) before the read is executed, so it should
   not happen.
diff --git a/Documentation/w1/w1-generic.rst b/Documentation/w1/w1-generic.rst
index da4e8b4e9b01..99255b6d0e53 100644
--- a/Documentation/w1/w1-generic.rst
+++ b/Documentation/w1/w1-generic.rst
@@ -113,7 +113,7 @@ generally only make sense when searching is disabled, as a search will
 redetect manually removed devices that are present and timeout manually
 added devices that aren't on the bus.
 
-Bus searches occur at an interval, specified as a summ of timeout and
+Bus searches occur at an interval, specified as a sum of timeout and
 timeout_us module parameters (either of which may be 0) for as long as
 w1_master_search remains greater than 0 or is -1.  Each search attempt
 decrements w1_master_search by 1 (down to 0) and increments
diff --git a/Documentation/x86/entry_64.rst b/Documentation/x86/entry_64.rst
index e433e08f7018..0afdce3c06f4 100644
--- a/Documentation/x86/entry_64.rst
+++ b/Documentation/x86/entry_64.rst
@@ -33,8 +33,8 @@ Some of these entries are:
  - interrupt: An array of entries.  Every IDT vector that doesn't
    explicitly point somewhere else gets set to the corresponding
    value in interrupts.  These point to a whole array of
-   magically-generated functions that make their way to do_IRQ with
-   the interrupt number as a parameter.
+   magically-generated functions that make their way to common_interrupt()
+   with the interrupt number as a parameter.
 
  - APIC interrupts: Various special-purpose interrupts for things
    like TLB shootdown.
diff --git a/Documentation/x86/microcode.rst b/Documentation/x86/microcode.rst
index a320d37982ed..b627c6f36bcf 100644
--- a/Documentation/x86/microcode.rst
+++ b/Documentation/x86/microcode.rst
@@ -6,6 +6,7 @@ The Linux Microcode Loader
 
 :Authors: - Fenghua Yu <fenghua.yu@intel.com>
           - Borislav Petkov <bp@suse.de>
+	  - Ashok Raj <ashok.raj@intel.com>
 
 The kernel has a x86 microcode loading facility which is supposed to
 provide microcode loading methods in the OS. Potential use cases are
@@ -92,15 +93,8 @@ vendor's site.
 Late loading
 ============
 
-There are two legacy user space interfaces to load microcode, either through
-/dev/cpu/microcode or through /sys/devices/system/cpu/microcode/reload file
-in sysfs.
-
-The /dev/cpu/microcode method is deprecated because it needs a special
-userspace tool for that.
-
-The easier method is simply installing the microcode packages your distro
-supplies and running::
+You simply install the microcode packages your distro supplies and
+run::
 
   # echo 1 > /sys/devices/system/cpu/microcode/reload
 
@@ -110,6 +104,110 @@ The loading mechanism looks for microcode blobs in
 /lib/firmware/{intel-ucode,amd-ucode}. The default distro installation
 packages already put them there.
 
+Since kernel 5.19, late loading is not enabled by default.
+
+The /dev/cpu/microcode method has been removed in 5.19.
+
+Why is late loading dangerous?
+==============================
+
+Synchronizing all CPUs
+----------------------
+
+The microcode engine which receives the microcode update is shared
+between the two logical threads in a SMT system. Therefore, when
+the update is executed on one SMT thread of the core, the sibling
+"automatically" gets the update.
+
+Since the microcode can "simulate" MSRs too, while the microcode update
+is in progress, those simulated MSRs transiently cease to exist. This
+can result in unpredictable results if the SMT sibling thread happens to
+be in the middle of an access to such an MSR. The usual observation is
+that such MSR accesses cause #GPs to be raised to signal that former are
+not present.
+
+The disappearing MSRs are just one common issue which is being observed.
+Any other instruction that's being patched and gets concurrently
+executed by the other SMT sibling, can also result in similar,
+unpredictable behavior.
+
+To eliminate this case, a stop_machine()-based CPU synchronization was
+introduced as a way to guarantee that all logical CPUs will not execute
+any code but just wait in a spin loop, polling an atomic variable.
+
+While this took care of device or external interrupts, IPIs including
+LVT ones, such as CMCI etc, it cannot address other special interrupts
+that can't be shut off. Those are Machine Check (#MC), System Management
+(#SMI) and Non-Maskable interrupts (#NMI).
+
+Machine Checks
+--------------
+
+Machine Checks (#MC) are non-maskable. There are two kinds of MCEs.
+Fatal un-recoverable MCEs and recoverable MCEs. While un-recoverable
+errors are fatal, recoverable errors can also happen in kernel context
+are also treated as fatal by the kernel.
+
+On certain Intel machines, MCEs are also broadcast to all threads in a
+system. If one thread is in the middle of executing WRMSR, a MCE will be
+taken at the end of the flow. Either way, they will wait for the thread
+performing the wrmsr(0x79) to rendezvous in the MCE handler and shutdown
+eventually if any of the threads in the system fail to check in to the
+MCE rendezvous.
+
+To be paranoid and get predictable behavior, the OS can choose to set
+MCG_STATUS.MCIP. Since MCEs can be at most one in a system, if an
+MCE was signaled, the above condition will promote to a system reset
+automatically. OS can turn off MCIP at the end of the update for that
+core.
+
+System Management Interrupt
+---------------------------
+
+SMIs are also broadcast to all CPUs in the platform. Microcode update
+requests exclusive access to the core before writing to MSR 0x79. So if
+it does happen such that, one thread is in WRMSR flow, and the 2nd got
+an SMI, that thread will be stopped in the first instruction in the SMI
+handler.
+
+Since the secondary thread is stopped in the first instruction in SMI,
+there is very little chance that it would be in the middle of executing
+an instruction being patched. Plus OS has no way to stop SMIs from
+happening.
+
+Non-Maskable Interrupts
+-----------------------
+
+When thread0 of a core is doing the microcode update, if thread1 is
+pulled into NMI, that can cause unpredictable behavior due to the
+reasons above.
+
+OS can choose a variety of methods to avoid running into this situation.
+
+
+Is the microcode suitable for late loading?
+-------------------------------------------
+
+Late loading is done when the system is fully operational and running
+real workloads. Late loading behavior depends on what the base patch on
+the CPU is before upgrading to the new patch.
+
+This is true for Intel CPUs.
+
+Consider, for example, a CPU has patch level 1 and the update is to
+patch level 3.
+
+Between patch1 and patch3, patch2 might have deprecated a software-visible
+feature.
+
+This is unacceptable if software is even potentially using that feature.
+For instance, say MSR_X is no longer available after an update,
+accessing that MSR will cause a #GP fault.
+
+Basically there is no way to declare a new microcode update suitable
+for late-loading. This is another one of the problems that caused late
+loading to be not enabled by default.
+
 Builtin microcode
 =================
 
diff --git a/MAINTAINERS b/MAINTAINERS
index f5ca4aefd184..f5899cc20be2 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -348,7 +348,6 @@ M:	"Rafael J. Wysocki" <rafael@kernel.org>
 R:	Len Brown <lenb@kernel.org>
 L:	linux-acpi@vger.kernel.org
 S:	Supported
-W:	https://01.org/linux-acpi
 Q:	https://patchwork.kernel.org/project/linux-acpi/list/
 B:	https://bugzilla.kernel.org
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm
@@ -427,7 +426,6 @@ M:	Rafael J. Wysocki <rafael@kernel.org>
 R:	Zhang Rui <rui.zhang@intel.com>
 L:	linux-acpi@vger.kernel.org
 S:	Supported
-W:	https://01.org/linux-acpi
 B:	https://bugzilla.kernel.org
 F:	drivers/acpi/*thermal*
 
@@ -750,6 +748,12 @@ S:	Supported
 F:	drivers/infiniband/hw/erdma
 F:	include/uapi/rdma/erdma-abi.h
 
+ALIBABA PMU DRIVER
+M:	Shuai Xue <xueshuai@linux.alibaba.com>
+S:	Supported
+F:	Documentation/admin-guide/perf/alibaba_pmu.rst
+F:	drivers/perf/alibaba_uncore_dwr_pmu.c
+
 ALIENWARE WMI DRIVER
 L:	Dell.Client.Kernel@dell.com
 S:	Maintained
@@ -821,6 +825,13 @@ L:	linux-media@vger.kernel.org
 S:	Maintained
 F:	drivers/staging/media/sunxi/cedrus/
 
+ALLWINNER DMIC DRIVERS
+M:	Ban Tao <fengzheng923@gmail.com>
+L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
+S:	Maintained
+F:	Documentation/devicetree/bindings/sound/allwinner,sun50i-h6-dmic.yaml
+F:	sound/soc/sunxi/sun50i-dmic.c
+
 ALPHA PORT
 M:	Richard Henderson <richard.henderson@linaro.org>
 M:	Ivan Kokshaysky <ink@jurassic.park.msu.ru>
@@ -879,6 +890,13 @@ L:	netdev@vger.kernel.org
 S:	Maintained
 F:	drivers/net/ethernet/altera/
 
+ALTERA TSE PCS
+M:	Maxime Chevallier <maxime.chevallier@bootlin.com>
+L:	netdev@vger.kernel.org
+S:	Supported
+F:	drivers/net/pcs/pcs-altera-tse.c
+F:	include/linux/pcs-altera-tse.h
+
 ALTERA UART/JTAG UART SERIAL DRIVERS
 M:	Tobias Klauser <tklauser@distanz.ch>
 L:	linux-serial@vger.kernel.org
@@ -1022,6 +1040,13 @@ L:	platform-driver-x86@vger.kernel.org
 S:	Maintained
 F:	drivers/platform/x86/amd/pmc.c
 
+AMD PMF DRIVER
+M:	Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+L:	platform-driver-x86@vger.kernel.org
+S:	Maintained
+F:	Documentation/ABI/testing/sysfs-amd-pmf
+F:	drivers/platform/x86/amd/pmf/
+
 AMD HSMP DRIVER
 M:	Naveen Krishna Chatradhi <naveenkrishna.chatradhi@amd.com>
 R:	Carlos Bilbao <carlos.bilbao@amd.com>
@@ -1045,6 +1070,7 @@ L:	linux-pm@vger.kernel.org
 S:	Supported
 F:	Documentation/admin-guide/pm/amd-pstate.rst
 F:	drivers/cpufreq/amd-pstate*
+F:	include/linux/amd-pstate.h
 F:	tools/power/x86/amd_pstate_tracer/amd_pstate_trace.py
 
 AMD PTDMA DRIVER
@@ -1334,6 +1360,15 @@ F:	drivers/iio/amplifiers/hmc425a.c
 F:	drivers/staging/iio/*/ad*
 X:	drivers/iio/*/adjd*
 
+ANALOG DEVICES INC MAX31760 DRIVER
+M:	Ibrahim Tilki <Ibrahim.Tilki@analog.com>
+S:	Maintained
+W:	http://wiki.analog.com/
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/hwmon/adi,max31760.yaml
+F:	Documentation/hwmon/max31760.rst
+F:	drivers/hwmon/max31760.c
+
 ANALOGBITS PLL LIBRARIES
 M:	Paul Walmsley <paul.walmsley@sifive.com>
 S:	Supported
@@ -1899,6 +1934,15 @@ F:	include/dt-bindings/pinctrl/apple.h
 F:	include/linux/apple-mailbox.h
 F:	include/linux/soc/apple/*
 
+ARM/APPLE MACHINE SOUND DRIVERS
+M:	Martin Povišer <povik+lin@cutebit.org>
+L:	asahi@lists.linux.dev
+L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
+S:	Maintained
+F:	Documentation/devicetree/bindings/sound/apple,*
+F:	sound/soc/apple/*
+F:	sound/soc/codecs/cs42l83-i2c.c
+
 ARM/ARTPEC MACHINE SUPPORT
 M:	Jesper Nilsson <jesper.nilsson@axis.com>
 M:	Lars Persson <lars.persson@axis.com>
@@ -2394,6 +2438,7 @@ N:	atmel
 ARM/Microchip Sparx5 SoC support
 M:	Lars Povlsen <lars.povlsen@microchip.com>
 M:	Steen Hegelund <Steen.Hegelund@microchip.com>
+M:	Daniel Machon <daniel.machon@microchip.com>
 M:	UNGLinuxDriver@microchip.com
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Supported
@@ -2579,7 +2624,7 @@ W:	http://www.armlinux.org.uk/
 
 ARM/QUALCOMM SUPPORT
 M:	Andy Gross <agross@kernel.org>
-M:	Bjorn Andersson <bjorn.andersson@linaro.org>
+M:	Bjorn Andersson <andersson@kernel.org>
 R:	Konrad Dybcio <konrad.dybcio@somainline.org>
 L:	linux-arm-msm@vger.kernel.org
 S:	Maintained
@@ -2588,6 +2633,7 @@ F:	Documentation/devicetree/bindings/*/qcom*
 F:	Documentation/devicetree/bindings/soc/qcom/
 F:	arch/arm/boot/dts/qcom-*.dts
 F:	arch/arm/boot/dts/qcom-*.dtsi
+F:	arch/arm/configs/qcom_defconfig
 F:	arch/arm/mach-qcom/
 F:	arch/arm64/boot/dts/qcom/
 F:	drivers/*/*/qcom*
@@ -2650,7 +2696,7 @@ F:	arch/arm/boot/dts/rtd*
 F:	arch/arm/mach-realtek/
 F:	arch/arm64/boot/dts/realtek/
 
-ARM/RENESAS ARM64 ARCHITECTURE
+ARM/RENESAS ARCHITECTURE
 M:	Geert Uytterhoeven <geert+renesas@glider.be>
 M:	Magnus Damm <magnus.damm@gmail.com>
 L:	linux-renesas-soc@vger.kernel.org
@@ -2661,6 +2707,16 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git nex
 F:	Documentation/devicetree/bindings/arm/renesas.yaml
 F:	Documentation/devicetree/bindings/hwinfo/renesas,prr.yaml
 F:	Documentation/devicetree/bindings/soc/renesas/
+F:	arch/arm/boot/dts/emev2*
+F:	arch/arm/boot/dts/gr-peach*
+F:	arch/arm/boot/dts/iwg20d-q7*
+F:	arch/arm/boot/dts/r7s*
+F:	arch/arm/boot/dts/r8a*
+F:	arch/arm/boot/dts/r9a*
+F:	arch/arm/boot/dts/sh*
+F:	arch/arm/configs/shmobile_defconfig
+F:	arch/arm/include/debug/renesas-scif.S
+F:	arch/arm/mach-shmobile/
 F:	arch/arm64/boot/dts/renesas/
 F:	drivers/soc/renesas/
 F:	include/linux/soc/renesas/
@@ -2670,7 +2726,6 @@ M:	Russell King <linux@armlinux.org.uk>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 W:	http://www.armlinux.org.uk/
-F:	arch/arm/include/asm/hardware/entry-macro-iomd.S
 F:	arch/arm/include/asm/hardware/ioc.h
 F:	arch/arm/include/asm/hardware/iomd.h
 F:	arch/arm/include/asm/hardware/memc.h
@@ -2772,29 +2827,6 @@ L:	linux-media@vger.kernel.org
 S:	Maintained
 F:	drivers/media/platform/samsung/s5p-mfc/
 
-ARM/SHMOBILE ARM ARCHITECTURE
-M:	Geert Uytterhoeven <geert+renesas@glider.be>
-M:	Magnus Damm <magnus.damm@gmail.com>
-L:	linux-renesas-soc@vger.kernel.org
-S:	Supported
-Q:	http://patchwork.kernel.org/project/linux-renesas-soc/list/
-C:	irc://irc.libera.chat/renesas-soc
-T:	git git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git next
-F:	Documentation/devicetree/bindings/arm/renesas.yaml
-F:	Documentation/devicetree/bindings/soc/renesas/
-F:	arch/arm/boot/dts/emev2*
-F:	arch/arm/boot/dts/gr-peach*
-F:	arch/arm/boot/dts/iwg20d-q7*
-F:	arch/arm/boot/dts/r7s*
-F:	arch/arm/boot/dts/r8a*
-F:	arch/arm/boot/dts/r9a*
-F:	arch/arm/boot/dts/sh*
-F:	arch/arm/configs/shmobile_defconfig
-F:	arch/arm/include/debug/renesas-scif.S
-F:	arch/arm/mach-shmobile/
-F:	drivers/soc/renesas/
-F:	include/linux/soc/renesas/
-
 ARM/SOCFPGA ARCHITECTURE
 M:	Dinh Nguyen <dinguyen@kernel.org>
 S:	Maintained
@@ -3087,6 +3119,8 @@ W:	http://wiki.xilinx.com
 T:	git https://github.com/Xilinx/linux-xlnx.git
 F:	Documentation/devicetree/bindings/i2c/cdns,i2c-r1p10.yaml
 F:	Documentation/devicetree/bindings/i2c/xlnx,xps-iic-2.00.a.yaml
+F:	Documentation/devicetree/bindings/memory-controllers/snps,dw-umctl2-ddrc.yaml
+F:	Documentation/devicetree/bindings/memory-controllers/xlnx,zynq-ddrc-a05.yaml
 F:	Documentation/devicetree/bindings/spi/xlnx,zynq-qspi.yaml
 F:	arch/arm/mach-zynq/
 F:	drivers/clocksource/timer-cadence-ttc.c
@@ -3233,13 +3267,6 @@ L:	linux-hwmon@vger.kernel.org
 S:	Maintained
 F:	drivers/hwmon/asus_wmi_sensors.c
 
-ASUS WMI EC HARDWARE MONITOR DRIVER
-M:	Eugene Shalygin <eugene.shalygin@gmail.com>
-M:	Denis Pauk <pauk.denis@gmail.com>
-L:	linux-hwmon@vger.kernel.org
-S:	Maintained
-F:	drivers/hwmon/asus_wmi_ec_sensors.c
-
 ASUS EC HARDWARE MONITOR DRIVER
 M:	Eugene Shalygin <eugene.shalygin@gmail.com>
 L:	linux-hwmon@vger.kernel.org
@@ -3825,6 +3852,7 @@ F:	kernel/bpf/dispatcher.c
 F:	kernel/bpf/trampoline.c
 F:	include/linux/bpf*
 F:	include/linux/filter.h
+F:	include/linux/tnum.h
 
 BPF [BTF]
 M:	Martin KaFai Lau <martin.lau@linux.dev>
@@ -3950,6 +3978,7 @@ M:	William Zhang <william.zhang@broadcom.com>
 M:	Anand Gore <anand.gore@broadcom.com>
 M:	Kursad Oney <kursad.oney@broadcom.com>
 M:	Florian Fainelli <f.fainelli@gmail.com>
+M:	Rafał Miłecki <rafal@milecki.pl>
 R:	Broadcom internal kernel review list <bcm-kernel-feedback-list@broadcom.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
@@ -4896,6 +4925,7 @@ M:	Prashant Malani <pmalani@chromium.org>
 L:	chrome-platform@lists.linux.dev
 S:	Maintained
 F:	drivers/platform/chrome/cros_ec_typec.c
+F:	drivers/platform/chrome/cros_typec_switch.c
 
 CHROMEOS EC USB PD NOTIFY DRIVER
 M:	Prashant Malani <pmalani@chromium.org>
@@ -5374,8 +5404,8 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git
 F:	drivers/cpuidle/cpuidle-big_little.c
 
 CPUIDLE DRIVER - ARM EXYNOS
-M:	Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
 M:	Daniel Lezcano <daniel.lezcano@linaro.org>
+R:	Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>
 M:	Kukjin Kim <kgene@kernel.org>
 L:	linux-pm@vger.kernel.org
 L:	linux-samsung-soc@vger.kernel.org
@@ -5723,13 +5753,6 @@ F:	include/linux/tfrc.h
 F:	include/uapi/linux/dccp.h
 F:	net/dccp/
 
-DECnet NETWORK LAYER
-L:	linux-decnet-user@lists.sourceforge.net
-S:	Orphan
-W:	http://linux-decnet.sourceforge.net
-F:	Documentation/networking/decnet.rst
-F:	net/decnet/
-
 DECSTATION PLATFORM SUPPORT
 M:	"Maciej W. Rozycki" <macro@orcam.me.uk>
 L:	linux-mips@vger.kernel.org
@@ -6182,7 +6205,7 @@ F:	Documentation/devicetree/bindings/memory-controllers/samsung,exynos5422-dmc.y
 F:	drivers/memory/samsung/exynos5422-dmc.c
 
 DME1737 HARDWARE MONITOR DRIVER
-M:	Juerg Haefliger <juergh@gmail.com>
+M:	Juerg Haefliger <juergh@proton.me>
 L:	linux-hwmon@vger.kernel.org
 S:	Maintained
 F:	Documentation/hwmon/dme1737.rst
@@ -6422,6 +6445,11 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/display/panel/feiyang,fy07024di26a30d.yaml
 F:	drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
 
+DRM DRIVER FOR GENERIC EDP PANELS
+R:	Douglas Anderson <dianders@chromium.org>
+F:	Documentation/devicetree/bindings/display/panel/panel-edp.yaml
+F:	drivers/gpu/drm/panel/panel-edp.c
+
 DRM DRIVER FOR GENERIC USB DISPLAY
 M:	Noralf Trønnes <noralf@tronnes.org>
 S:	Maintained
@@ -7535,7 +7563,7 @@ M:	Adrian Hunter <adrian.hunter@intel.com>
 M:	Ritesh Harjani <riteshh@codeaurora.org>
 M:	Asutosh Das <asutoshd@codeaurora.org>
 L:	linux-mmc@vger.kernel.org
-S:	Maintained
+S:	Supported
 F:	drivers/mmc/host/cqhci*
 
 EMULEX 10Gbps iSCSI - OneConnect DRIVER
@@ -7688,7 +7716,6 @@ R:	Kees Cook <keescook@chromium.org>
 L:	linux-mm@kvack.org
 S:	Supported
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git for-next/execve
-F:	arch/alpha/kernel/binfmt_loader.c
 F:	fs/*binfmt_*.c
 F:	fs/exec.c
 F:	include/linux/binfmts.h
@@ -8005,6 +8032,7 @@ L:	linux-hardening@vger.kernel.org
 S:	Supported
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git for-next/hardening
 F:	include/linux/fortify-string.h
+F:	lib/fortify_kunit.c
 F:	lib/test_fortify/*
 F:	scripts/test_fortify.sh
 K:	\b__NO_FORTIFY\b
@@ -8944,7 +8972,7 @@ F:	include/linux/hw_random.h
 
 HARDWARE SPINLOCK CORE
 M:	Ohad Ben-Cohen <ohad@wizery.com>
-M:	Bjorn Andersson <bjorn.andersson@linaro.org>
+M:	Bjorn Andersson <andersson@kernel.org>
 R:	Baolin Wang <baolin.wang7@gmail.com>
 L:	linux-remoteproc@vger.kernel.org
 S:	Maintained
@@ -10379,7 +10407,6 @@ INTEL MENLOW THERMAL DRIVER
 M:	Sujith Thomas <sujith.thomas@intel.com>
 L:	linux-pm@vger.kernel.org
 S:	Supported
-W:	https://01.org/linux-acpi
 F:	drivers/thermal/intel/intel_menlow.c
 
 INTEL P-Unit IPC DRIVER
@@ -12411,7 +12438,6 @@ F:	drivers/power/supply/max77976_charger.c
 
 MAXIM MUIC CHARGER DRIVERS FOR EXYNOS BASED BOARDS
 M:	Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>
-M:	Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
 L:	linux-pm@vger.kernel.org
 S:	Supported
 B:	mailto:linux-samsung-soc@vger.kernel.org
@@ -12423,7 +12449,6 @@ F:	drivers/power/supply/max77693_charger.c
 MAXIM PMIC AND MUIC DRIVERS FOR EXYNOS BASED BOARDS
 M:	Chanwoo Choi <cw00.choi@samsung.com>
 M:	Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>
-M:	Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
 L:	linux-kernel@vger.kernel.org
 S:	Supported
 B:	mailto:linux-samsung-soc@vger.kernel.org
@@ -12984,9 +13009,9 @@ F:	drivers/input/touchscreen/melfas_mip4.c
 
 MELLANOX BLUEFIELD I2C DRIVER
 M:	Khalil Blaiech <kblaiech@nvidia.com>
+M:	Asmaa Mnebhi <asmaa@nvidia.com>
 L:	linux-i2c@vger.kernel.org
 S:	Supported
-F:	Documentation/devicetree/bindings/i2c/mellanox,i2c-mlxbf.yaml
 F:	drivers/i2c/busses/i2c-mlxbf.c
 
 MELLANOX ETHERNET DRIVER (mlx4_en)
@@ -13435,6 +13460,14 @@ F:	Documentation/devicetree/bindings/nvmem/microchip,sama7g5-otpc.yaml
 F:	drivers/nvmem/microchip-otpc.c
 F:	include/dt-bindings/nvmem/microchip,sama7g5-otpc.h
 
+MICROCHIP PCI1XXXX I2C DRIVER
+M:	Tharun Kumar P <tharunkumar.pasumarthi@microchip.com>
+M:	Kumaravel Thiagarajan <kumaravel.thiagarajan@microchip.com>
+M:	Microchip Linux Driver Support <UNGLinuxDriver@microchip.com>
+L:	linux-i2c@vger.kernel.org
+S:	Maintained
+F:	drivers/i2c/busses/i2c-mchp-pci1xxxx.c
+
 MICROCHIP PWM DRIVER
 M:	Claudiu Beznea <claudiu.beznea@microchip.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
@@ -14450,6 +14483,7 @@ M:	Willy Tarreau <w@1wt.eu>
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wtarreau/nolibc.git
 F:	tools/include/nolibc/
+F:	tools/testing/selftests/nolibc/
 
 NSDEPS
 M:	Matthias Maennich <maennich@google.com>
@@ -14532,6 +14566,7 @@ M:	Daniel Dadap <ddadap@nvidia.com>
 L:	platform-driver-x86@vger.kernel.org
 S:	Supported
 F:	drivers/platform/x86/nvidia-wmi-ec-backlight.c
+F:	include/linux/platform_data/x86/nvidia-wmi-ec-backlight.h
 
 NVM EXPRESS DRIVER
 M:	Keith Busch <kbusch@kernel.org>
@@ -14747,6 +14782,13 @@ F:	net/dsa/tag_ocelot.c
 F:	net/dsa/tag_ocelot_8021q.c
 F:	tools/testing/selftests/drivers/net/ocelot/*
 
+OCELOT EXTERNAL SWITCH CONTROL
+M:	Colin Foster <colin.foster@in-advantage.com>
+S:	Supported
+F:	Documentation/devicetree/bindings/mfd/mscc,ocelot.yaml
+F:	drivers/mfd/ocelot*
+F:	include/linux/mfd/ocelot.h
+
 OCXL (Open Coherent Accelerator Processor Interface OpenCAPI) DRIVER
 M:	Frederic Barrat <fbarrat@linux.ibm.com>
 M:	Andrew Donnellan <ajd@linux.ibm.com>
@@ -14941,6 +14983,7 @@ F:	drivers/regulator/palmas-regulator*.c
 F:	drivers/regulator/pbias-regulator.c
 F:	drivers/regulator/tps65217-regulator.c
 F:	drivers/regulator/tps65218-regulator.c
+F:	drivers/regulator/tps65219-regulator.c
 F:	drivers/regulator/tps65910-regulator.c
 F:	drivers/regulator/twl-regulator.c
 F:	drivers/regulator/twl6030-regulator.c
@@ -16126,7 +16169,7 @@ F:	drivers/gpio/gpio-sama5d2-piobu.c
 F:	drivers/pinctrl/pinctrl-at91*
 
 PIN CONTROLLER - QUALCOMM
-M:	Bjorn Andersson <bjorn.andersson@linaro.org>
+M:	Bjorn Andersson <andersson@kernel.org>
 L:	linux-arm-msm@vger.kernel.org
 S:	Maintained
 F:	Documentation/devicetree/bindings/pinctrl/qcom,*.txt
@@ -16539,14 +16582,6 @@ T:	git git://linuxtv.org/media_tree.git
 F:	drivers/media/usb/pwc/*
 F:	include/trace/events/pwc.h
 
-PWM FAN DRIVER
-M:	Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
-L:	linux-hwmon@vger.kernel.org
-S:	Supported
-F:	Documentation/devicetree/bindings/hwmon/pwm-fan.txt
-F:	Documentation/hwmon/pwm-fan.rst
-F:	drivers/hwmon/pwm-fan.c
-
 PWM IR Transmitter
 M:	Sean Young <sean@mess.org>
 L:	linux-media@vger.kernel.org
@@ -16615,6 +16650,9 @@ M:	Srinivas Kandagatla <srinivas.kandagatla@linaro.org>
 M:	Banajit Goswami <bgoswami@quicinc.com>
 L:	alsa-devel@alsa-project.org (moderated for non-subscribers)
 S:	Supported
+F:	Documentation/devicetree/bindings/soc/qcom/qcom,apr.yaml
+F:	Documentation/devicetree/bindings/sound/qcom,*
+F:	drivers/soc/qcom/apr.c
 F:	include/dt-bindings/sound/qcom,wcd9335.h
 F:	sound/soc/codecs/lpass-rx-macro.*
 F:	sound/soc/codecs/lpass-tx-macro.*
@@ -16819,7 +16857,7 @@ F:	Documentation/devicetree/bindings/media/*camss*
 F:	drivers/media/platform/qcom/camss/
 
 QUALCOMM CLOCK DRIVERS
-M:	Bjorn Andersson <bjorn.andersson@linaro.org>
+M:	Bjorn Andersson <andersson@kernel.org>
 L:	linux-arm-msm@vger.kernel.org
 S:	Supported
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git
@@ -17309,7 +17347,7 @@ S:	Supported
 F:	fs/reiserfs/
 
 REMOTE PROCESSOR (REMOTEPROC) SUBSYSTEM
-M:	Bjorn Andersson <bjorn.andersson@linaro.org>
+M:	Bjorn Andersson <andersson@kernel.org>
 M:	Mathieu Poirier <mathieu.poirier@linaro.org>
 L:	linux-remoteproc@vger.kernel.org
 S:	Maintained
@@ -17322,7 +17360,7 @@ F:	include/linux/remoteproc.h
 F:	include/linux/remoteproc/
 
 REMOTE PROCESSOR MESSAGING (RPMSG) SUBSYSTEM
-M:	Bjorn Andersson <bjorn.andersson@linaro.org>
+M:	Bjorn Andersson <andersson@kernel.org>
 M:	Mathieu Poirier <mathieu.poirier@linaro.org>
 L:	linux-remoteproc@vger.kernel.org
 S:	Maintained
@@ -17551,6 +17589,7 @@ F:	drivers/mailbox/mailbox-mpfs.c
 F:	drivers/pci/controller/pcie-microchip-host.c
 F:	drivers/rtc/rtc-mpfs.c
 F:	drivers/soc/microchip/
+F:	drivers/spi/spi-microchip-core-qspi.c
 F:	drivers/spi/spi-microchip-core.c
 F:	drivers/usb/musb/mpfs.c
 F:	include/soc/microchip/mpfs.h
@@ -17758,6 +17797,24 @@ F:	include/rv/
 F:	kernel/trace/rv/
 F:	tools/verification/
 
+RUST
+M:	Miguel Ojeda <ojeda@kernel.org>
+M:	Alex Gaynor <alex.gaynor@gmail.com>
+M:	Wedson Almeida Filho <wedsonaf@gmail.com>
+R:	Boqun Feng <boqun.feng@gmail.com>
+R:	Gary Guo <gary@garyguo.net>
+R:	Björn Roy Baron <bjorn3_gh@protonmail.com>
+L:	rust-for-linux@vger.kernel.org
+S:	Supported
+W:	https://github.com/Rust-for-Linux/linux
+B:	https://github.com/Rust-for-Linux/linux/issues
+T:	git https://github.com/Rust-for-Linux/linux.git rust-next
+F:	Documentation/rust/
+F:	rust/
+F:	samples/rust/
+F:	scripts/*rust*
+K:	\b(?i:rust)\b
+
 RXRPC SOCKETS (AF_RXRPC)
 M:	David Howells <dhowells@redhat.com>
 M:	Marc Dionne <marc.dionne@auristor.com>
@@ -17985,7 +18042,6 @@ F:	drivers/platform/x86/samsung-laptop.c
 
 SAMSUNG MULTIFUNCTION PMIC DEVICE DRIVERS
 M:	Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>
-M:	Bartlomiej Zolnierkiewicz <b.zolnierkie@samsung.com>
 L:	linux-kernel@vger.kernel.org
 L:	linux-samsung-soc@vger.kernel.org
 S:	Supported
@@ -18280,7 +18336,7 @@ F:	drivers/mmc/host/sdhci-brcmstb*
 SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) DRIVER
 M:	Adrian Hunter <adrian.hunter@intel.com>
 L:	linux-mmc@vger.kernel.org
-S:	Maintained
+S:	Supported
 F:	drivers/mmc/host/sdhci*
 
 SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) MICROCHIP DRIVER
@@ -18303,7 +18359,7 @@ S:	Maintained
 F:	drivers/mmc/host/sdhci-spear.c
 
 SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) TI OMAP DRIVER
-M:	Kishon Vijay Abraham I <kishon@ti.com>
+M:	Vignesh Raghavendra <vigneshr@ti.com>
 L:	linux-mmc@vger.kernel.org
 S:	Maintained
 F:	drivers/mmc/host/sdhci-omap.c
@@ -19504,6 +19560,11 @@ L:	netdev@vger.kernel.org
 S:	Maintained
 F:	drivers/net/ethernet/dlink/sundance.c
 
+SUN HAPPY MEAL ETHERNET DRIVER
+M:	Sean Anderson <seanga2@gmail.com>
+S:	Maintained
+F:	drivers/net/ethernet/sun/sunhme.*
+
 SUNPLUS ETHERNET DRIVER
 M:	Wells Lu <wellslutw@gmail.com>
 L:	netdev@vger.kernel.org
@@ -19961,7 +20022,7 @@ S:	Supported
 F:	drivers/net/team/
 F:	include/linux/if_team.h
 F:	include/uapi/linux/if_team.h
-F:	tools/testing/selftests/net/team/
+F:	tools/testing/selftests/drivers/net/team/
 
 TECHNOLOGIC SYSTEMS TS-5500 PLATFORM SUPPORT
 M:	"Savoir-faire Linux Inc." <kernel@savoirfairelinux.com>
@@ -20612,6 +20673,13 @@ Q:	https://patchwork.kernel.org/project/linux-integrity/list/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git
 F:	drivers/char/tpm/
 
+TPS546D24 DRIVER
+M:	Duke Du <dukedu83@gmail.com>
+L:	linux-hwmon@vger.kernel.org
+S:	Maintained
+F:	Documentation/hwmon/tps546d24.rst
+F:	drivers/hwmon/pmbus/tps546d24.c
+
 TRACING
 M:	Steven Rostedt <rostedt@goodmis.org>
 M:	Ingo Molnar <mingo@redhat.com>
@@ -21815,7 +21883,7 @@ F:	lib/test_scanf.c
 F:	lib/vsprintf.c
 
 VT1211 HARDWARE MONITOR DRIVER
-M:	Juerg Haefliger <juergh@gmail.com>
+M:	Juerg Haefliger <juergh@proton.me>
 L:	linux-hwmon@vger.kernel.org
 S:	Maintained
 F:	Documentation/hwmon/vt1211.rst
@@ -21874,9 +21942,11 @@ F:	drivers/input/tablet/wacom_serial4.c
 
 WANGXUN ETHERNET DRIVER
 M:	Jiawen Wu <jiawenwu@trustnetic.com>
+M:	Mengyuan Lou <mengyuanlou@net-swift.com>
+W:	https://www.net-swift.com
 L:	netdev@vger.kernel.org
 S:	Maintained
-F:	Documentation/networking/device_drivers/ethernet/wangxun/txgbe.rst
+F:	Documentation/networking/device_drivers/ethernet/wangxun/*
 F:	drivers/net/ethernet/wangxun/
 
 WATCHDOG DEVICE DRIVERS
diff --git a/Makefile b/Makefile
index 647a42a1f800..f659d3085121 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@
 VERSION = 6
 PATCHLEVEL = 0
 SUBLEVEL = 0
-EXTRAVERSION = -rc7
+EXTRAVERSION =
 NAME = Hurr durr I'ma ninja sloth
 
 # *DOCUMENTATION*
@@ -120,6 +120,15 @@ endif
 
 export KBUILD_CHECKSRC
 
+# Enable "clippy" (a linter) as part of the Rust compilation.
+#
+# Use 'make CLIPPY=1' to enable it.
+ifeq ("$(origin CLIPPY)", "command line")
+  KBUILD_CLIPPY := $(CLIPPY)
+endif
+
+export KBUILD_CLIPPY
+
 # Use make M=dir or set the environment variable KBUILD_EXTMOD to specify the
 # directory of external module to build. Setting M= takes precedence.
 ifeq ("$(origin M)", "command line")
@@ -270,14 +279,14 @@ no-dot-config-targets := $(clean-targets) \
 			 cscope gtags TAGS tags help% %docs check% coccicheck \
 			 $(version_h) headers headers_% archheaders archscripts \
 			 %asm-generic kernelversion %src-pkg dt_binding_check \
-			 outputmakefile
+			 outputmakefile rustavailable rustfmt rustfmtcheck
 # Installation targets should not require compiler. Unfortunately, vdso_install
 # is an exception where build artifacts may be updated. This must be fixed.
 no-compiler-targets := $(no-dot-config-targets) install dtbs_install \
 			headers_install modules_install kernelrelease image_name
 no-sync-config-targets := $(no-dot-config-targets) %install kernelrelease \
 			  image_name
-single-targets := %.a %.i %.ko %.lds %.ll %.lst %.mod %.o %.s %.symtypes %/
+single-targets := %.a %.i %.rsi %.ko %.lds %.ll %.lst %.mod %.o %.s %.symtypes %/
 
 config-build	:=
 mixed-build	:=
@@ -439,6 +448,7 @@ else
 HOSTCC	= gcc
 HOSTCXX	= g++
 endif
+HOSTRUSTC = rustc
 HOSTPKG_CONFIG	= pkg-config
 
 KBUILD_USERHOSTCFLAGS := -Wall -Wmissing-prototypes -Wstrict-prototypes \
@@ -447,8 +457,26 @@ KBUILD_USERHOSTCFLAGS := -Wall -Wmissing-prototypes -Wstrict-prototypes \
 KBUILD_USERCFLAGS  := $(KBUILD_USERHOSTCFLAGS) $(USERCFLAGS)
 KBUILD_USERLDFLAGS := $(USERLDFLAGS)
 
+# These flags apply to all Rust code in the tree, including the kernel and
+# host programs.
+export rust_common_flags := --edition=2021 \
+			    -Zbinary_dep_depinfo=y \
+			    -Dunsafe_op_in_unsafe_fn -Drust_2018_idioms \
+			    -Dunreachable_pub -Dnon_ascii_idents \
+			    -Wmissing_docs \
+			    -Drustdoc::missing_crate_level_docs \
+			    -Dclippy::correctness -Dclippy::style \
+			    -Dclippy::suspicious -Dclippy::complexity \
+			    -Dclippy::perf \
+			    -Dclippy::let_unit_value -Dclippy::mut_mut \
+			    -Dclippy::needless_bitwise_bool \
+			    -Dclippy::needless_continue \
+			    -Wclippy::dbg_macro
+
 KBUILD_HOSTCFLAGS   := $(KBUILD_USERHOSTCFLAGS) $(HOST_LFS_CFLAGS) $(HOSTCFLAGS)
 KBUILD_HOSTCXXFLAGS := -Wall -O2 $(HOST_LFS_CFLAGS) $(HOSTCXXFLAGS)
+KBUILD_HOSTRUSTFLAGS := $(rust_common_flags) -O -Cstrip=debuginfo \
+			-Zallow-features= $(HOSTRUSTFLAGS)
 KBUILD_HOSTLDFLAGS  := $(HOST_LFS_LDFLAGS) $(HOSTLDFLAGS)
 KBUILD_HOSTLDLIBS   := $(HOST_LFS_LIBS) $(HOSTLDLIBS)
 
@@ -473,6 +501,12 @@ OBJDUMP		= $(CROSS_COMPILE)objdump
 READELF		= $(CROSS_COMPILE)readelf
 STRIP		= $(CROSS_COMPILE)strip
 endif
+RUSTC		= rustc
+RUSTDOC		= rustdoc
+RUSTFMT		= rustfmt
+CLIPPY_DRIVER	= clippy-driver
+BINDGEN		= bindgen
+CARGO		= cargo
 PAHOLE		= pahole
 RESOLVE_BTFIDS	= $(objtree)/tools/bpf/resolve_btfids/resolve_btfids
 LEX		= flex
@@ -498,9 +532,11 @@ CHECKFLAGS     := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
 		  -Wbitwise -Wno-return-void -Wno-unknown-attribute $(CF)
 NOSTDINC_FLAGS :=
 CFLAGS_MODULE   =
+RUSTFLAGS_MODULE =
 AFLAGS_MODULE   =
 LDFLAGS_MODULE  =
 CFLAGS_KERNEL	=
+RUSTFLAGS_KERNEL =
 AFLAGS_KERNEL	=
 LDFLAGS_vmlinux =
 
@@ -529,15 +565,43 @@ KBUILD_CFLAGS   := -Wall -Wundef -Werror=strict-prototypes -Wno-trigraphs \
 		   -Werror=return-type -Wno-format-security \
 		   -std=gnu11
 KBUILD_CPPFLAGS := -D__KERNEL__
+KBUILD_RUSTFLAGS := $(rust_common_flags) \
+		    --target=$(objtree)/rust/target.json \
+		    -Cpanic=abort -Cembed-bitcode=n -Clto=n \
+		    -Cforce-unwind-tables=n -Ccodegen-units=1 \
+		    -Csymbol-mangling-version=v0 \
+		    -Crelocation-model=static \
+		    -Zfunction-sections=n \
+		    -Dclippy::float_arithmetic
+
 KBUILD_AFLAGS_KERNEL :=
 KBUILD_CFLAGS_KERNEL :=
+KBUILD_RUSTFLAGS_KERNEL :=
 KBUILD_AFLAGS_MODULE  := -DMODULE
 KBUILD_CFLAGS_MODULE  := -DMODULE
+KBUILD_RUSTFLAGS_MODULE := --cfg MODULE
 KBUILD_LDFLAGS_MODULE :=
 KBUILD_LDFLAGS :=
 CLANG_FLAGS :=
 
+ifeq ($(KBUILD_CLIPPY),1)
+	RUSTC_OR_CLIPPY_QUIET := CLIPPY
+	RUSTC_OR_CLIPPY = $(CLIPPY_DRIVER)
+else
+	RUSTC_OR_CLIPPY_QUIET := RUSTC
+	RUSTC_OR_CLIPPY = $(RUSTC)
+endif
+
+ifdef RUST_LIB_SRC
+	export RUST_LIB_SRC
+endif
+
+# Allows the usage of unstable features in stable compilers.
+export RUSTC_BOOTSTRAP := 1
+
 export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC HOSTPKG_CONFIG
+export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN CARGO
+export HOSTRUSTC KBUILD_HOSTRUSTFLAGS
 export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL
 export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
 export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD
@@ -546,9 +610,10 @@ export KBUILD_USERCFLAGS KBUILD_USERLDFLAGS
 
 export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
 export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
+export KBUILD_RUSTFLAGS RUSTFLAGS_KERNEL RUSTFLAGS_MODULE
 export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE
-export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE
-export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL
+export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_RUSTFLAGS_MODULE KBUILD_LDFLAGS_MODULE
+export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL KBUILD_RUSTFLAGS_KERNEL
 export PAHOLE_FLAGS
 
 # Files to ignore in find ... statements
@@ -729,7 +794,7 @@ $(KCONFIG_CONFIG):
 #
 # Do not use $(call cmd,...) here. That would suppress prompts from syncconfig,
 # so you cannot notice that Kconfig is waiting for the user input.
-%/config/auto.conf %/config/auto.conf.cmd %/generated/autoconf.h: $(KCONFIG_CONFIG)
+%/config/auto.conf %/config/auto.conf.cmd %/generated/autoconf.h %/generated/rustc_cfg: $(KCONFIG_CONFIG)
 	$(Q)$(kecho) "  SYNC    $@"
 	$(Q)$(MAKE) -f $(srctree)/Makefile syncconfig
 else # !may-sync-config
@@ -758,10 +823,17 @@ KBUILD_CFLAGS	+= $(call cc-disable-warning, address-of-packed-member)
 
 ifdef CONFIG_CC_OPTIMIZE_FOR_PERFORMANCE
 KBUILD_CFLAGS += -O2
+KBUILD_RUSTFLAGS += -Copt-level=2
 else ifdef CONFIG_CC_OPTIMIZE_FOR_SIZE
 KBUILD_CFLAGS += -Os
+KBUILD_RUSTFLAGS += -Copt-level=s
 endif
 
+# Always set `debug-assertions` and `overflow-checks` because their default
+# depends on `opt-level` and `debug-assertions`, respectively.
+KBUILD_RUSTFLAGS += -Cdebug-assertions=$(if $(CONFIG_RUST_DEBUG_ASSERTIONS),y,n)
+KBUILD_RUSTFLAGS += -Coverflow-checks=$(if $(CONFIG_RUST_OVERFLOW_CHECKS),y,n)
+
 # Tell gcc to never replace conditional load with a non-conditional one
 ifdef CONFIG_CC_IS_GCC
 # gcc-10 renamed --param=allow-store-data-races=0 to
@@ -792,6 +864,9 @@ KBUILD_CFLAGS-$(CONFIG_WERROR) += -Werror
 KBUILD_CFLAGS-$(CONFIG_CC_NO_ARRAY_BOUNDS) += -Wno-array-bounds
 KBUILD_CFLAGS += $(KBUILD_CFLAGS-y) $(CONFIG_CC_IMPLICIT_FALLTHROUGH)
 
+KBUILD_RUSTFLAGS-$(CONFIG_WERROR) += -Dwarnings
+KBUILD_RUSTFLAGS += $(KBUILD_RUSTFLAGS-y)
+
 ifdef CONFIG_CC_IS_CLANG
 KBUILD_CPPFLAGS += -Qunused-arguments
 # The kernel builds with '-std=gnu11' so use of GNU extensions is acceptable.
@@ -812,12 +887,15 @@ KBUILD_CFLAGS += $(call cc-disable-warning, dangling-pointer)
 
 ifdef CONFIG_FRAME_POINTER
 KBUILD_CFLAGS	+= -fno-omit-frame-pointer -fno-optimize-sibling-calls
+KBUILD_RUSTFLAGS += -Cforce-frame-pointers=y
 else
 # Some targets (ARM with Thumb2, for example), can't be built with frame
 # pointers.  For those, we don't have FUNCTION_TRACER automatically
 # select FRAME_POINTER.  However, FUNCTION_TRACER adds -pg, and this is
 # incompatible with -fomit-frame-pointer with current GCC, so we don't use
 # -fomit-frame-pointer with FUNCTION_TRACER.
+# In the Rust target specification, "frame-pointer" is set explicitly
+# to "may-omit".
 ifndef CONFIG_FUNCTION_TRACER
 KBUILD_CFLAGS	+= -fomit-frame-pointer
 endif
@@ -831,8 +909,8 @@ endif
 # Initialize all stack variables with a zero value.
 ifdef CONFIG_INIT_STACK_ALL_ZERO
 KBUILD_CFLAGS	+= -ftrivial-auto-var-init=zero
-ifdef CONFIG_CC_IS_CLANG
-# https://bugs.llvm.org/show_bug.cgi?id=45497
+ifdef CONFIG_CC_HAS_AUTO_VAR_INIT_ZERO_ENABLER
+# https://github.com/llvm/llvm-project/issues/44842
 KBUILD_CFLAGS	+= -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang
 endif
 endif
@@ -882,8 +960,10 @@ ifdef CONFIG_DEBUG_SECTION_MISMATCH
 KBUILD_CFLAGS += -fno-inline-functions-called-once
 endif
 
+# `rustc`'s `-Zfunction-sections` applies to data too (as of 1.59.0).
 ifdef CONFIG_LD_DEAD_CODE_DATA_ELIMINATION
 KBUILD_CFLAGS_KERNEL += -ffunction-sections -fdata-sections
+KBUILD_RUSTFLAGS_KERNEL += -Zfunction-sections=y
 LDFLAGS_vmlinux += --gc-sections
 endif
 
@@ -921,18 +1001,7 @@ export CC_FLAGS_LTO
 endif
 
 ifdef CONFIG_CFI_CLANG
-CC_FLAGS_CFI	:= -fsanitize=cfi \
-		   -fsanitize-cfi-cross-dso \
-		   -fno-sanitize-cfi-canonical-jump-tables \
-		   -fno-sanitize-trap=cfi \
-		   -fno-sanitize-blacklist
-
-ifdef CONFIG_CFI_PERMISSIVE
-CC_FLAGS_CFI	+= -fsanitize-recover=cfi
-endif
-
-# If LTO flags are filtered out, we must also filter out CFI.
-CC_FLAGS_LTO	+= $(CC_FLAGS_CFI)
+CC_FLAGS_CFI	:= -fsanitize=kcfi
 KBUILD_CFLAGS	+= $(CC_FLAGS_CFI)
 export CC_FLAGS_CFI
 endif
@@ -1026,10 +1095,11 @@ include $(addprefix $(srctree)/, $(include-y))
 # Do not add $(call cc-option,...) below this line. When you build the kernel
 # from the clean source tree, the GCC plugins do not exist at this point.
 
-# Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments
+# Add user supplied CPPFLAGS, AFLAGS, CFLAGS and RUSTFLAGS as the last assignments
 KBUILD_CPPFLAGS += $(KCPPFLAGS)
 KBUILD_AFLAGS   += $(KAFLAGS)
 KBUILD_CFLAGS   += $(KCFLAGS)
+KBUILD_RUSTFLAGS += $(KRUSTFLAGS)
 
 KBUILD_LDFLAGS_MODULE += --build-id=sha1
 LDFLAGS_vmlinux += --build-id=sha1
@@ -1104,6 +1174,7 @@ ifeq ($(KBUILD_EXTMOD),)
 core-y			+= kernel/ certs/ mm/ fs/ ipc/ security/ crypto/
 core-$(CONFIG_BLOCK)	+= block/
 core-$(CONFIG_IO_URING)	+= io_uring/
+core-$(CONFIG_RUST)	+= rust/
 
 vmlinux-dirs	:= $(patsubst %/,%,$(filter %/, \
 		     $(core-y) $(core-m) $(drivers-y) $(drivers-m) \
@@ -1206,6 +1277,10 @@ prepare0: archprepare
 
 # All the preparing..
 prepare: prepare0
+ifdef CONFIG_RUST
+	$(Q)$(CONFIG_SHELL) $(srctree)/scripts/rust_is_available.sh -v
+	$(Q)$(MAKE) $(build)=rust
+endif
 
 PHONY += remove-stale-files
 remove-stale-files:
@@ -1499,7 +1574,7 @@ endif # CONFIG_MODULES
 # Directories & files removed with 'make clean'
 CLEAN_FILES += include/ksym vmlinux.symvers modules-only.symvers \
 	       modules.builtin modules.builtin.modinfo modules.nsdeps \
-	       compile_commands.json .thinlto-cache
+	       compile_commands.json .thinlto-cache rust/test rust/doc
 
 # Directories & files removed with 'make mrproper'
 MRPROPER_FILES += include/config include/generated          \
@@ -1510,7 +1585,8 @@ MRPROPER_FILES += include/config include/generated          \
 		  certs/signing_key.pem \
 		  certs/x509.genkey \
 		  vmlinux-gdb.py \
-		  *.spec
+		  *.spec \
+		  rust/target.json rust/libmacros.so
 
 # clean - Delete most, but leave enough to build external modules
 #
@@ -1535,6 +1611,9 @@ $(mrproper-dirs):
 
 mrproper: clean $(mrproper-dirs)
 	$(call cmd,rmfiles)
+	@find . $(RCS_FIND_IGNORE) \
+		\( -name '*.rmeta' \) \
+		-type f -print | xargs rm -f
 
 # distclean
 #
@@ -1622,6 +1701,24 @@ help:
 	@echo  '  kselftest-merge   - Merge all the config dependencies of'
 	@echo  '		      kselftest to existing .config.'
 	@echo  ''
+	@echo  'Rust targets:'
+	@echo  '  rustavailable   - Checks whether the Rust toolchain is'
+	@echo  '		    available and, if not, explains why.'
+	@echo  '  rustfmt	  - Reformat all the Rust code in the kernel'
+	@echo  '  rustfmtcheck	  - Checks if all the Rust code in the kernel'
+	@echo  '		    is formatted, printing a diff otherwise.'
+	@echo  '  rustdoc	  - Generate Rust documentation'
+	@echo  '		    (requires kernel .config)'
+	@echo  '  rusttest        - Runs the Rust tests'
+	@echo  '                    (requires kernel .config; downloads external repos)'
+	@echo  '  rust-analyzer	  - Generate rust-project.json rust-analyzer support file'
+	@echo  '		    (requires kernel .config)'
+	@echo  '  dir/file.[os]   - Build specified target only'
+	@echo  '  dir/file.rsi    - Build macro expanded source, similar to C preprocessing.'
+	@echo  '                    Run with RUSTFMT=n to skip reformatting if needed.'
+	@echo  '                    The output is not intended to be compilable.'
+	@echo  '  dir/file.ll     - Build the LLVM assembly file'
+	@echo  ''
 	@$(if $(dtstree), \
 		echo 'Devicetree:'; \
 		echo '* dtbs             - Build device tree blobs for enabled boards'; \
@@ -1694,6 +1791,52 @@ PHONY += $(DOC_TARGETS)
 $(DOC_TARGETS):
 	$(Q)$(MAKE) $(build)=Documentation $@
 
+
+# Rust targets
+# ---------------------------------------------------------------------------
+
+# "Is Rust available?" target
+PHONY += rustavailable
+rustavailable:
+	$(Q)$(CONFIG_SHELL) $(srctree)/scripts/rust_is_available.sh -v && echo "Rust is available!"
+
+# Documentation target
+#
+# Using the singular to avoid running afoul of `no-dot-config-targets`.
+PHONY += rustdoc
+rustdoc: prepare
+	$(Q)$(MAKE) $(build)=rust $@
+
+# Testing target
+PHONY += rusttest
+rusttest: prepare
+	$(Q)$(MAKE) $(build)=rust $@
+
+# Formatting targets
+PHONY += rustfmt rustfmtcheck
+
+# We skip `rust/alloc` since we want to minimize the diff w.r.t. upstream.
+#
+# We match using absolute paths since `find` does not resolve them
+# when matching, which is a problem when e.g. `srctree` is `..`.
+# We `grep` afterwards in order to remove the directory entry itself.
+rustfmt:
+	$(Q)find $(abs_srctree) -type f -name '*.rs' \
+		-o -path $(abs_srctree)/rust/alloc -prune \
+		-o -path $(abs_objtree)/rust/test -prune \
+		| grep -Fv $(abs_srctree)/rust/alloc \
+		| grep -Fv $(abs_objtree)/rust/test \
+		| grep -Fv generated \
+		| xargs $(RUSTFMT) $(rustfmt_flags)
+
+rustfmtcheck: rustfmt_flags = --check
+rustfmtcheck: rustfmt
+
+# IDE support targets
+PHONY += rust-analyzer
+rust-analyzer:
+	$(Q)$(MAKE) $(build)=rust $@
+
 # Misc
 # ---------------------------------------------------------------------------
 
@@ -1861,7 +2004,7 @@ $(clean-dirs):
 clean: $(clean-dirs)
 	$(call cmd,rmfiles)
 	@find $(or $(KBUILD_EXTMOD), .) $(RCS_FIND_IGNORE) \
-		\( -name '*.[aios]' -o -name '*.ko' -o -name '.*.cmd' \
+		\( -name '*.[aios]' -o -name '*.rsi' -o -name '*.ko' -o -name '.*.cmd' \
 		-o -name '*.ko.*' \
 		-o -name '*.dtb' -o -name '*.dtbo' -o -name '*.dtb.S' -o -name '*.dt.yaml' \
 		-o -name '*.dwo' -o -name '*.lst' \
diff --git a/arch/Kconfig b/arch/Kconfig
index 8b311e400ec1..266862428a84 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -355,6 +355,12 @@ config HAVE_RSEQ
 	  This symbol should be selected by an architecture if it
 	  supports an implementation of restartable sequences.
 
+config HAVE_RUST
+	bool
+	help
+	  This symbol should be selected by an architecture if it
+	  supports Rust.
+
 config HAVE_FUNCTION_ARG_ACCESS_API
 	bool
 	help
@@ -738,11 +744,13 @@ config ARCH_SUPPORTS_CFI_CLANG
 	  An architecture should select this option if it can support Clang's
 	  Control-Flow Integrity (CFI) checking.
 
+config ARCH_USES_CFI_TRAPS
+	bool
+
 config CFI_CLANG
 	bool "Use Clang's Control Flow Integrity (CFI)"
-	depends on LTO_CLANG && ARCH_SUPPORTS_CFI_CLANG
-	depends on CLANG_VERSION >= 140000
-	select KALLSYMS
+	depends on ARCH_SUPPORTS_CFI_CLANG
+	depends on $(cc-option,-fsanitize=kcfi)
 	help
 	  This option enables Clang’s forward-edge Control Flow Integrity
 	  (CFI) checking, where the compiler injects a runtime check to each
@@ -754,16 +762,6 @@ config CFI_CLANG
 
 	    https://clang.llvm.org/docs/ControlFlowIntegrity.html
 
-config CFI_CLANG_SHADOW
-	bool "Use CFI shadow to speed up cross-module checks"
-	default y
-	depends on CFI_CLANG && MODULES
-	help
-	  If you select this option, the kernel builds a fast look-up table of
-	  CFI check functions in loaded modules to reduce performance overhead.
-
-	  If unsure, say Y.
-
 config CFI_PERMISSIVE
 	bool "Use CFI in permissive mode"
 	depends on CFI_CLANG
diff --git a/arch/alpha/include/asm/a.out.h b/arch/alpha/include/asm/a.out.h
deleted file mode 100644
index d2346b7caff1..000000000000
--- a/arch/alpha/include/asm/a.out.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __ALPHA_A_OUT_H__
-#define __ALPHA_A_OUT_H__
-
-#include <uapi/asm/a.out.h>
-
-
-/* Assume that start addresses below 4G belong to a TASO application.
-   Unfortunately, there is no proper bit in the exec header to check.
-   Worse, we have to notice the start address before swapping to use
-   /sbin/loader, which of course is _not_ a TASO application.  */
-#define SET_AOUT_PERSONALITY(BFPM, EX) \
-	set_personality (((BFPM->taso || EX.ah.entry < 0x100000000L \
-			   ? ADDR_LIMIT_32BIT : 0) | PER_OSF4))
-
-#endif /* __A_OUT_GNU_H__ */
diff --git a/arch/alpha/include/asm/core_apecs.h b/arch/alpha/include/asm/core_apecs.h
index 2d9726fc02ef..69a2fc62c9c3 100644
--- a/arch/alpha/include/asm/core_apecs.h
+++ b/arch/alpha/include/asm/core_apecs.h
@@ -384,7 +384,7 @@ struct el_apecs_procdata
 		}						\
 	} while (0)
 
-__EXTERN_INLINE unsigned int apecs_ioread8(const void __iomem *xaddr)
+__EXTERN_INLINE u8 apecs_ioread8(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long) xaddr;
 	unsigned long result, base_and_type;
@@ -420,7 +420,7 @@ __EXTERN_INLINE void apecs_iowrite8(u8 b, void __iomem *xaddr)
 	*(vuip) ((addr << 5) + base_and_type) = w;
 }
 
-__EXTERN_INLINE unsigned int apecs_ioread16(const void __iomem *xaddr)
+__EXTERN_INLINE u16 apecs_ioread16(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long) xaddr;
 	unsigned long result, base_and_type;
@@ -456,7 +456,7 @@ __EXTERN_INLINE void apecs_iowrite16(u16 b, void __iomem *xaddr)
 	*(vuip) ((addr << 5) + base_and_type) = w;
 }
 
-__EXTERN_INLINE unsigned int apecs_ioread32(const void __iomem *xaddr)
+__EXTERN_INLINE u32 apecs_ioread32(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long) xaddr;
 	if (addr < APECS_DENSE_MEM)
@@ -472,6 +472,22 @@ __EXTERN_INLINE void apecs_iowrite32(u32 b, void __iomem *xaddr)
 	*(vuip)addr = b;
 }
 
+__EXTERN_INLINE u64 apecs_ioread64(const void __iomem *xaddr)
+{
+	unsigned long addr = (unsigned long) xaddr;
+	if (addr < APECS_DENSE_MEM)
+		addr = ((addr - APECS_IO) << 5) + APECS_IO + 0x18;
+	return *(vulp)addr;
+}
+
+__EXTERN_INLINE void apecs_iowrite64(u64 b, void __iomem *xaddr)
+{
+	unsigned long addr = (unsigned long) xaddr;
+	if (addr < APECS_DENSE_MEM)
+		addr = ((addr - APECS_IO) << 5) + APECS_IO + 0x18;
+	*(vulp)addr = b;
+}
+
 __EXTERN_INLINE void __iomem *apecs_ioportmap(unsigned long addr)
 {
 	return (void __iomem *)(addr + APECS_IO);
diff --git a/arch/alpha/include/asm/core_cia.h b/arch/alpha/include/asm/core_cia.h
index cb22991f6761..d26bdfb7ca3b 100644
--- a/arch/alpha/include/asm/core_cia.h
+++ b/arch/alpha/include/asm/core_cia.h
@@ -342,7 +342,7 @@ struct el_CIA_sysdata_mcheck {
 #define vuip	volatile unsigned int __force *
 #define vulp	volatile unsigned long __force *
 
-__EXTERN_INLINE unsigned int cia_ioread8(const void __iomem *xaddr)
+__EXTERN_INLINE u8 cia_ioread8(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long) xaddr;
 	unsigned long result, base_and_type;
@@ -374,7 +374,7 @@ __EXTERN_INLINE void cia_iowrite8(u8 b, void __iomem *xaddr)
 	*(vuip) ((addr << 5) + base_and_type) = w;
 }
 
-__EXTERN_INLINE unsigned int cia_ioread16(const void __iomem *xaddr)
+__EXTERN_INLINE u16 cia_ioread16(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long) xaddr;
 	unsigned long result, base_and_type;
@@ -404,7 +404,7 @@ __EXTERN_INLINE void cia_iowrite16(u16 b, void __iomem *xaddr)
 	*(vuip) ((addr << 5) + base_and_type) = w;
 }
 
-__EXTERN_INLINE unsigned int cia_ioread32(const void __iomem *xaddr)
+__EXTERN_INLINE u32 cia_ioread32(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long) xaddr;
 	if (addr < CIA_DENSE_MEM)
@@ -420,6 +420,22 @@ __EXTERN_INLINE void cia_iowrite32(u32 b, void __iomem *xaddr)
 	*(vuip)addr = b;
 }
 
+__EXTERN_INLINE u64 cia_ioread64(const void __iomem *xaddr)
+{
+	unsigned long addr = (unsigned long) xaddr;
+	if (addr < CIA_DENSE_MEM)
+		addr = ((addr - CIA_IO) << 5) + CIA_IO + 0x18;
+	return *(vulp)addr;
+}
+
+__EXTERN_INLINE void cia_iowrite64(u64 b, void __iomem *xaddr)
+{
+	unsigned long addr = (unsigned long) xaddr;
+	if (addr < CIA_DENSE_MEM)
+		addr = ((addr - CIA_IO) << 5) + CIA_IO + 0x18;
+	*(vulp)addr = b;
+}
+
 __EXTERN_INLINE void __iomem *cia_ioportmap(unsigned long addr)
 {
 	return (void __iomem *)(addr + CIA_IO);
diff --git a/arch/alpha/include/asm/core_lca.h b/arch/alpha/include/asm/core_lca.h
index ec86314418cb..d8c3e72ef8f6 100644
--- a/arch/alpha/include/asm/core_lca.h
+++ b/arch/alpha/include/asm/core_lca.h
@@ -230,7 +230,7 @@ union el_lca {
 	} while (0)
 
 
-__EXTERN_INLINE unsigned int lca_ioread8(const void __iomem *xaddr)
+__EXTERN_INLINE u8 lca_ioread8(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long) xaddr;
 	unsigned long result, base_and_type;
@@ -266,7 +266,7 @@ __EXTERN_INLINE void lca_iowrite8(u8 b, void __iomem *xaddr)
 	*(vuip) ((addr << 5) + base_and_type) = w;
 }
 
-__EXTERN_INLINE unsigned int lca_ioread16(const void __iomem *xaddr)
+__EXTERN_INLINE u16 lca_ioread16(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long) xaddr;
 	unsigned long result, base_and_type;
@@ -302,7 +302,7 @@ __EXTERN_INLINE void lca_iowrite16(u16 b, void __iomem *xaddr)
 	*(vuip) ((addr << 5) + base_and_type) = w;
 }
 
-__EXTERN_INLINE unsigned int lca_ioread32(const void __iomem *xaddr)
+__EXTERN_INLINE u32 lca_ioread32(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long) xaddr;
 	if (addr < LCA_DENSE_MEM)
@@ -318,6 +318,22 @@ __EXTERN_INLINE void lca_iowrite32(u32 b, void __iomem *xaddr)
 	*(vuip)addr = b;
 }
 
+__EXTERN_INLINE u64 lca_ioread64(const void __iomem *xaddr)
+{
+	unsigned long addr = (unsigned long) xaddr;
+	if (addr < LCA_DENSE_MEM)
+		addr = ((addr - LCA_IO) << 5) + LCA_IO + 0x18;
+	return *(vulp)addr;
+}
+
+__EXTERN_INLINE void lca_iowrite64(u64 b, void __iomem *xaddr)
+{
+	unsigned long addr = (unsigned long) xaddr;
+	if (addr < LCA_DENSE_MEM)
+		addr = ((addr - LCA_IO) << 5) + LCA_IO + 0x18;
+	*(vulp)addr = b;
+}
+
 __EXTERN_INLINE void __iomem *lca_ioportmap(unsigned long addr)
 {
 	return (void __iomem *)(addr + LCA_IO);
diff --git a/arch/alpha/include/asm/core_marvel.h b/arch/alpha/include/asm/core_marvel.h
index b266e02e284b..d99f3a82e0e5 100644
--- a/arch/alpha/include/asm/core_marvel.h
+++ b/arch/alpha/include/asm/core_marvel.h
@@ -332,10 +332,10 @@ struct io7 {
 #define vucp	volatile unsigned char __force *
 #define vusp	volatile unsigned short __force *
 
-extern unsigned int marvel_ioread8(const void __iomem *);
+extern u8 marvel_ioread8(const void __iomem *);
 extern void marvel_iowrite8(u8 b, void __iomem *);
 
-__EXTERN_INLINE unsigned int marvel_ioread16(const void __iomem *addr)
+__EXTERN_INLINE u16 marvel_ioread16(const void __iomem *addr)
 {
 	return __kernel_ldwu(*(vusp)addr);
 }
diff --git a/arch/alpha/include/asm/core_mcpcia.h b/arch/alpha/include/asm/core_mcpcia.h
index cb24d1bd6141..ed2bf8ad40ed 100644
--- a/arch/alpha/include/asm/core_mcpcia.h
+++ b/arch/alpha/include/asm/core_mcpcia.h
@@ -248,6 +248,7 @@ struct el_MCPCIA_uncorrected_frame_mcheck {
 
 #define vip	volatile int __force *
 #define vuip	volatile unsigned int __force *
+#define vulp	volatile unsigned long __force *
 
 #ifndef MCPCIA_ONE_HAE_WINDOW
 #define MCPCIA_FROB_MMIO						\
@@ -267,7 +268,7 @@ extern inline int __mcpcia_is_mmio(unsigned long addr)
 	return (addr & 0x80000000UL) == 0;
 }
 
-__EXTERN_INLINE unsigned int mcpcia_ioread8(const void __iomem *xaddr)
+__EXTERN_INLINE u8 mcpcia_ioread8(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long)xaddr & MCPCIA_MEM_MASK;
 	unsigned long hose = (unsigned long)xaddr & ~MCPCIA_MEM_MASK;
@@ -291,7 +292,7 @@ __EXTERN_INLINE void mcpcia_iowrite8(u8 b, void __iomem *xaddr)
 	*(vuip) ((addr << 5) + hose + 0x00) = w;
 }
 
-__EXTERN_INLINE unsigned int mcpcia_ioread16(const void __iomem *xaddr)
+__EXTERN_INLINE u16 mcpcia_ioread16(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long)xaddr & MCPCIA_MEM_MASK;
 	unsigned long hose = (unsigned long)xaddr & ~MCPCIA_MEM_MASK;
@@ -315,7 +316,7 @@ __EXTERN_INLINE void mcpcia_iowrite16(u16 b, void __iomem *xaddr)
 	*(vuip) ((addr << 5) + hose + 0x08) = w;
 }
 
-__EXTERN_INLINE unsigned int mcpcia_ioread32(const void __iomem *xaddr)
+__EXTERN_INLINE u32 mcpcia_ioread32(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long)xaddr;
 
@@ -335,6 +336,26 @@ __EXTERN_INLINE void mcpcia_iowrite32(u32 b, void __iomem *xaddr)
 	*(vuip)addr = b;
 }
 
+__EXTERN_INLINE u64 mcpcia_ioread64(const void __iomem *xaddr)
+{
+	unsigned long addr = (unsigned long)xaddr;
+
+	if (!__mcpcia_is_mmio(addr))
+		addr = ((addr & 0xffff) << 5) + (addr & ~0xfffful) + 0x18;
+
+	return *(vulp)addr;
+}
+
+__EXTERN_INLINE void mcpcia_iowrite64(u64 b, void __iomem *xaddr)
+{
+	unsigned long addr = (unsigned long)xaddr;
+
+	if (!__mcpcia_is_mmio(addr))
+		addr = ((addr & 0xffff) << 5) + (addr & ~0xfffful) + 0x18;
+
+	*(vulp)addr = b;
+}
+
 
 __EXTERN_INLINE void __iomem *mcpcia_ioportmap(unsigned long addr)
 {
@@ -362,6 +383,7 @@ __EXTERN_INLINE int mcpcia_is_mmio(const volatile void __iomem *xaddr)
 
 #undef vip
 #undef vuip
+#undef vulp
 
 #undef __IO_PREFIX
 #define __IO_PREFIX		mcpcia
diff --git a/arch/alpha/include/asm/core_t2.h b/arch/alpha/include/asm/core_t2.h
index 12bb7addc789..ab956b1625b5 100644
--- a/arch/alpha/include/asm/core_t2.h
+++ b/arch/alpha/include/asm/core_t2.h
@@ -360,6 +360,7 @@ struct el_t2_frame_corrected {
 
 #define vip	volatile int *
 #define vuip	volatile unsigned int *
+#define vulp	volatile unsigned long *
 
 extern inline u8 t2_inb(unsigned long addr)
 {
@@ -402,6 +403,17 @@ extern inline void t2_outl(u32 b, unsigned long addr)
 	mb();
 }
 
+extern inline u64 t2_inq(unsigned long addr)
+{
+	return *(vulp) ((addr << 5) + T2_IO + 0x18);
+}
+
+extern inline void t2_outq(u64 b, unsigned long addr)
+{
+	*(vulp) ((addr << 5) + T2_IO + 0x18) = b;
+	mb();
+}
+
 
 /*
  * Memory functions.
@@ -572,7 +584,7 @@ __EXTERN_INLINE int t2_is_mmio(const volatile void __iomem *addr)
    it doesn't make sense to merge the pio and mmio routines.  */
 
 #define IOPORT(OS, NS)							\
-__EXTERN_INLINE unsigned int t2_ioread##NS(const void __iomem *xaddr)		\
+__EXTERN_INLINE u##NS t2_ioread##NS(const void __iomem *xaddr)		\
 {									\
 	if (t2_is_mmio(xaddr))						\
 		return t2_read##OS(xaddr);				\
@@ -590,11 +602,13 @@ __EXTERN_INLINE void t2_iowrite##NS(u##NS b, void __iomem *xaddr)	\
 IOPORT(b, 8)
 IOPORT(w, 16)
 IOPORT(l, 32)
+IOPORT(q, 64)
 
 #undef IOPORT
 
 #undef vip
 #undef vuip
+#undef vulp
 
 #undef __IO_PREFIX
 #define __IO_PREFIX		t2
diff --git a/arch/alpha/include/asm/io.h b/arch/alpha/include/asm/io.h
index d277189b2677..1c3605d874e9 100644
--- a/arch/alpha/include/asm/io.h
+++ b/arch/alpha/include/asm/io.h
@@ -90,6 +90,8 @@ static inline void * phys_to_virt(unsigned long address)
 }
 #endif
 
+#define virt_to_phys		virt_to_phys
+#define phys_to_virt		phys_to_virt
 #define page_to_phys(page)	page_to_pa(page)
 
 /* Maximum PIO space address supported?  */
@@ -153,6 +155,7 @@ static inline void generic_##NAME(TYPE b, QUAL void __iomem *addr)	\
 REMAP1(unsigned int, ioread8, const)
 REMAP1(unsigned int, ioread16, const)
 REMAP1(unsigned int, ioread32, const)
+REMAP1(u64, ioread64, const)
 REMAP1(u8, readb, const volatile)
 REMAP1(u16, readw, const volatile)
 REMAP1(u32, readl, const volatile)
@@ -161,6 +164,7 @@ REMAP1(u64, readq, const volatile)
 REMAP2(u8, iowrite8, /**/)
 REMAP2(u16, iowrite16, /**/)
 REMAP2(u32, iowrite32, /**/)
+REMAP2(u64, iowrite64, /**/)
 REMAP2(u8, writeb, volatile)
 REMAP2(u16, writew, volatile)
 REMAP2(u32, writel, volatile)
@@ -242,6 +246,12 @@ extern u32		inl(unsigned long port);
 extern void		outb(u8 b, unsigned long port);
 extern void		outw(u16 b, unsigned long port);
 extern void		outl(u32 b, unsigned long port);
+#define inb inb
+#define inw inw
+#define inl inl
+#define outb outb
+#define outw outw
+#define outl outl
 
 extern u8		readb(const volatile void __iomem *addr);
 extern u16		readw(const volatile void __iomem *addr);
@@ -251,6 +261,14 @@ extern void		writeb(u8 b, volatile void __iomem *addr);
 extern void		writew(u16 b, volatile void __iomem *addr);
 extern void		writel(u32 b, volatile void __iomem *addr);
 extern void		writeq(u64 b, volatile void __iomem *addr);
+#define readb readb
+#define readw readw
+#define readl readl
+#define readq readq
+#define writeb writeb
+#define writew writew
+#define writel writel
+#define writeq writeq
 
 extern u8		__raw_readb(const volatile void __iomem *addr);
 extern u16		__raw_readw(const volatile void __iomem *addr);
@@ -260,6 +278,14 @@ extern void		__raw_writeb(u8 b, volatile void __iomem *addr);
 extern void		__raw_writew(u16 b, volatile void __iomem *addr);
 extern void		__raw_writel(u32 b, volatile void __iomem *addr);
 extern void		__raw_writeq(u64 b, volatile void __iomem *addr);
+#define __raw_readb __raw_readb
+#define __raw_readw __raw_readw
+#define __raw_readl __raw_readl
+#define __raw_readq __raw_readq
+#define __raw_writeb __raw_writeb
+#define __raw_writew __raw_writew
+#define __raw_writel __raw_writel
+#define __raw_writeq __raw_writeq
 
 /*
  * Mapping from port numbers to __iomem space is pretty easy.
@@ -277,6 +303,9 @@ extern inline void ioport_unmap(void __iomem *addr)
 {
 }
 
+#define ioport_map ioport_map
+#define ioport_unmap ioport_unmap
+
 static inline void __iomem *ioremap(unsigned long port, unsigned long size)
 {
 	return IO_CONCAT(__IO_PREFIX,ioremap) (port, size);
@@ -358,6 +387,11 @@ extern inline void outw(u16 b, unsigned long port)
 }
 #endif
 
+#define ioread8 ioread8
+#define ioread16 ioread16
+#define iowrite8 iowrite8
+#define iowrite16 iowrite16
+
 #if IO_CONCAT(__IO_PREFIX,trivial_io_lq)
 extern inline unsigned int ioread32(const void __iomem *addr)
 {
@@ -368,12 +402,27 @@ extern inline unsigned int ioread32(const void __iomem *addr)
 	return ret;
 }
 
+extern inline u64 ioread64(const void __iomem *addr)
+{
+	unsigned int ret;
+	mb();
+	ret = IO_CONCAT(__IO_PREFIX,ioread64)(addr);
+	mb();
+	return ret;
+}
+
 extern inline void iowrite32(u32 b, void __iomem *addr)
 {
 	mb();
 	IO_CONCAT(__IO_PREFIX, iowrite32)(b, addr);
 }
 
+extern inline void iowrite64(u64 b, void __iomem *addr)
+{
+	mb();
+	IO_CONCAT(__IO_PREFIX, iowrite64)(b, addr);
+}
+
 extern inline u32 inl(unsigned long port)
 {
 	return ioread32(ioport_map(port, 4));
@@ -385,6 +434,11 @@ extern inline void outl(u32 b, unsigned long port)
 }
 #endif
 
+#define ioread32 ioread32
+#define ioread64 ioread64
+#define iowrite32 iowrite32
+#define iowrite64 iowrite64
+
 #if IO_CONCAT(__IO_PREFIX,trivial_rw_bw) == 1
 extern inline u8 __raw_readb(const volatile void __iomem *addr)
 {
@@ -505,6 +559,10 @@ extern u8 readb_relaxed(const volatile void __iomem *addr);
 extern u16 readw_relaxed(const volatile void __iomem *addr);
 extern u32 readl_relaxed(const volatile void __iomem *addr);
 extern u64 readq_relaxed(const volatile void __iomem *addr);
+#define readb_relaxed readb_relaxed
+#define readw_relaxed readw_relaxed
+#define readl_relaxed readl_relaxed
+#define readq_relaxed readq_relaxed
 
 #if IO_CONCAT(__IO_PREFIX,trivial_io_bw)
 extern inline u8 readb_relaxed(const volatile void __iomem *addr)
@@ -557,6 +615,10 @@ static inline void memsetw_io(volatile void __iomem *addr, u16 c, long len)
 	_memset_c_io(addr, 0x0001000100010001UL * c, len);
 }
 
+#define memset_io memset_io
+#define memcpy_fromio memcpy_fromio
+#define memcpy_toio memcpy_toio
+
 /*
  * String versions of in/out ops:
  */
@@ -567,6 +629,13 @@ extern void outsb (unsigned long port, const void *src, unsigned long count);
 extern void outsw (unsigned long port, const void *src, unsigned long count);
 extern void outsl (unsigned long port, const void *src, unsigned long count);
 
+#define insb insb
+#define insw insw
+#define insl insl
+#define outsb outsb
+#define outsw outsw
+#define outsl outsl
+
 /*
  * The Alpha Jensen hardware for some rather strange reason puts
  * the RTC clock at 0x170 instead of 0x70. Probably due to some
@@ -587,21 +656,29 @@ extern void outsl (unsigned long port, const void *src, unsigned long count);
 #define RTC_ALWAYS_BCD	0
 
 /*
- * Some mucking forons use if[n]def writeq to check if platform has it.
- * It's a bloody bad idea and we probably want ARCH_HAS_WRITEQ for them
- * to play with; for now just use cpp anti-recursion logics and make sure
- * that damn thing is defined and expands to itself.
- */
-
-#define writeq writeq
-#define readq readq
-
-/*
  * Convert a physical pointer to a virtual kernel pointer for /dev/mem
  * access
  */
 #define xlate_dev_mem_ptr(p)	__va(p)
 
+/*
+ * These get provided from <asm-generic/iomap.h> since alpha does not
+ * select GENERIC_IOMAP.
+ */
+#define ioread64 ioread64
+#define iowrite64 iowrite64
+#define ioread64be ioread64be
+#define iowrite64be iowrite64be
+#define ioread8_rep ioread8_rep
+#define ioread16_rep ioread16_rep
+#define ioread32_rep ioread32_rep
+#define iowrite8_rep iowrite8_rep
+#define iowrite16_rep iowrite16_rep
+#define iowrite32_rep iowrite32_rep
+#define pci_iounmap pci_iounmap
+
+#include <asm-generic/io.h>
+
 #endif /* __KERNEL__ */
 
 #endif /* __ALPHA_IO_H */
diff --git a/arch/alpha/include/asm/io_trivial.h b/arch/alpha/include/asm/io_trivial.h
index a1a29cbe02fa..00032093bcfc 100644
--- a/arch/alpha/include/asm/io_trivial.h
+++ b/arch/alpha/include/asm/io_trivial.h
@@ -6,13 +6,13 @@
 /* This file may be included multiple times.  */
 
 #if IO_CONCAT(__IO_PREFIX,trivial_io_bw)
-__EXTERN_INLINE unsigned int
+__EXTERN_INLINE u8
 IO_CONCAT(__IO_PREFIX,ioread8)(const void __iomem *a)
 {
 	return __kernel_ldbu(*(const volatile u8 __force *)a);
 }
 
-__EXTERN_INLINE unsigned int
+__EXTERN_INLINE u16
 IO_CONCAT(__IO_PREFIX,ioread16)(const void __iomem *a)
 {
 	return __kernel_ldwu(*(const volatile u16 __force *)a);
@@ -32,7 +32,7 @@ IO_CONCAT(__IO_PREFIX,iowrite16)(u16 b, void __iomem *a)
 #endif
 
 #if IO_CONCAT(__IO_PREFIX,trivial_io_lq)
-__EXTERN_INLINE unsigned int
+__EXTERN_INLINE u32
 IO_CONCAT(__IO_PREFIX,ioread32)(const void __iomem *a)
 {
 	return *(const volatile u32 __force *)a;
@@ -43,6 +43,18 @@ IO_CONCAT(__IO_PREFIX,iowrite32)(u32 b, void __iomem *a)
 {
 	*(volatile u32 __force *)a = b;
 }
+
+__EXTERN_INLINE u64
+IO_CONCAT(__IO_PREFIX,ioread64)(const void __iomem *a)
+{
+	return *(const volatile u64 __force *)a;
+}
+
+__EXTERN_INLINE void
+IO_CONCAT(__IO_PREFIX,iowrite64)(u64 b, void __iomem *a)
+{
+	*(volatile u64 __force *)a = b;
+}
 #endif
 
 #if IO_CONCAT(__IO_PREFIX,trivial_rw_bw) == 1
diff --git a/arch/alpha/include/asm/jensen.h b/arch/alpha/include/asm/jensen.h
index 1c4131453db2..66eb049eb421 100644
--- a/arch/alpha/include/asm/jensen.h
+++ b/arch/alpha/include/asm/jensen.h
@@ -98,6 +98,7 @@ __EXTERN_INLINE void jensen_set_hae(unsigned long addr)
 }
 
 #define vuip	volatile unsigned int *
+#define vulp	volatile unsigned long *
 
 /*
  * IO functions
@@ -183,6 +184,12 @@ __EXTERN_INLINE u32 jensen_inl(unsigned long addr)
 	return *(vuip) ((addr << 7) + EISA_IO + 0x60);
 }
 
+__EXTERN_INLINE u64 jensen_inq(unsigned long addr)
+{
+	jensen_set_hae(0);
+	return *(vulp) ((addr << 7) + EISA_IO + 0x60);
+}
+
 __EXTERN_INLINE void jensen_outw(u16 b, unsigned long addr)
 {
 	jensen_set_hae(0);
@@ -197,6 +204,13 @@ __EXTERN_INLINE void jensen_outl(u32 b, unsigned long addr)
 	mb();
 }
 
+__EXTERN_INLINE void jensen_outq(u64 b, unsigned long addr)
+{
+	jensen_set_hae(0);
+	*(vulp) ((addr << 7) + EISA_IO + 0x60) = b;
+	mb();
+}
+
 /*
  * Memory functions.
  */
@@ -305,7 +319,7 @@ __EXTERN_INLINE int jensen_is_mmio(const volatile void __iomem *addr)
    that it doesn't make sense to merge them.  */
 
 #define IOPORT(OS, NS)							\
-__EXTERN_INLINE unsigned int jensen_ioread##NS(const void __iomem *xaddr)	\
+__EXTERN_INLINE u##NS jensen_ioread##NS(const void __iomem *xaddr)	\
 {									\
 	if (jensen_is_mmio(xaddr))					\
 		return jensen_read##OS(xaddr - 0x100000000ul);		\
@@ -323,10 +337,12 @@ __EXTERN_INLINE void jensen_iowrite##NS(u##NS b, void __iomem *xaddr)	\
 IOPORT(b, 8)
 IOPORT(w, 16)
 IOPORT(l, 32)
+IOPORT(q, 64)
 
 #undef IOPORT
 
 #undef vuip
+#undef vulp
 
 #undef __IO_PREFIX
 #define __IO_PREFIX		jensen
diff --git a/arch/alpha/include/asm/machvec.h b/arch/alpha/include/asm/machvec.h
index e49fabce7b33..8623f995d34c 100644
--- a/arch/alpha/include/asm/machvec.h
+++ b/arch/alpha/include/asm/machvec.h
@@ -46,13 +46,15 @@ struct alpha_machine_vector
 	void (*mv_pci_tbi)(struct pci_controller *hose,
 			   dma_addr_t start, dma_addr_t end);
 
-	unsigned int (*mv_ioread8)(const void __iomem *);
-	unsigned int (*mv_ioread16)(const void __iomem *);
-	unsigned int (*mv_ioread32)(const void __iomem *);
+	u8 (*mv_ioread8)(const void __iomem *);
+	u16 (*mv_ioread16)(const void __iomem *);
+	u32 (*mv_ioread32)(const void __iomem *);
+	u64 (*mv_ioread64)(const void __iomem *);
 
 	void (*mv_iowrite8)(u8, void __iomem *);
 	void (*mv_iowrite16)(u16, void __iomem *);
 	void (*mv_iowrite32)(u32, void __iomem *);
+	void (*mv_iowrite64)(u64, void __iomem *);
 
 	u8 (*mv_readb)(const volatile void __iomem *);
 	u16 (*mv_readw)(const volatile void __iomem *);
diff --git a/arch/alpha/kernel/Makefile b/arch/alpha/kernel/Makefile
index 5a74581bf0ee..6a274c0d53a2 100644
--- a/arch/alpha/kernel/Makefile
+++ b/arch/alpha/kernel/Makefile
@@ -47,10 +47,6 @@ else
 # Misc support
 obj-$(CONFIG_ALPHA_SRM)		+= srmcons.o
 
-ifdef CONFIG_BINFMT_AOUT
-obj-y	+= binfmt_loader.o
-endif
-
 # Core logic support
 obj-$(CONFIG_ALPHA_APECS)	+= core_apecs.o
 obj-$(CONFIG_ALPHA_CIA)		+= core_cia.o
diff --git a/arch/alpha/kernel/binfmt_loader.c b/arch/alpha/kernel/binfmt_loader.c
deleted file mode 100644
index e4be7a543ecf..000000000000
--- a/arch/alpha/kernel/binfmt_loader.c
+++ /dev/null
@@ -1,46 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/init.h>
-#include <linux/fs.h>
-#include <linux/file.h>
-#include <linux/mm_types.h>
-#include <linux/binfmts.h>
-#include <linux/a.out.h>
-
-static int load_binary(struct linux_binprm *bprm)
-{
-	struct exec *eh = (struct exec *)bprm->buf;
-	unsigned long loader;
-	struct file *file;
-	int retval;
-
-	if (eh->fh.f_magic != 0x183 || (eh->fh.f_flags & 0x3000) != 0x3000)
-		return -ENOEXEC;
-
-	if (bprm->loader)
-		return -ENOEXEC;
-
-	loader = bprm->vma->vm_end - sizeof(void *);
-
-	file = open_exec("/sbin/loader");
-	retval = PTR_ERR(file);
-	if (IS_ERR(file))
-		return retval;
-
-	/* Remember if the application is TASO.  */
-	bprm->taso = eh->ah.entry < 0x100000000UL;
-
-	bprm->interpreter = file;
-	bprm->loader = loader;
-	return 0;
-}
-
-static struct linux_binfmt loader_format = {
-	.load_binary	= load_binary,
-};
-
-static int __init init_loader_binfmt(void)
-{
-	insert_binfmt(&loader_format);
-	return 0;
-}
-arch_initcall(init_loader_binfmt);
diff --git a/arch/alpha/kernel/core_marvel.c b/arch/alpha/kernel/core_marvel.c
index 1efca79ac83c..6d0b3baf97ff 100644
--- a/arch/alpha/kernel/core_marvel.c
+++ b/arch/alpha/kernel/core_marvel.c
@@ -803,7 +803,7 @@ void __iomem *marvel_ioportmap (unsigned long addr)
 	return (void __iomem *)addr;
 }
 
-unsigned int
+unsigned u8
 marvel_ioread8(const void __iomem *xaddr)
 {
 	unsigned long addr = (unsigned long) xaddr;
diff --git a/arch/alpha/kernel/io.c b/arch/alpha/kernel/io.c
index 838586abb1e0..eda09778268f 100644
--- a/arch/alpha/kernel/io.c
+++ b/arch/alpha/kernel/io.c
@@ -41,6 +41,15 @@ unsigned int ioread32(const void __iomem *addr)
 	return ret;
 }
 
+u64 ioread64(const void __iomem *addr)
+{
+	unsigned int ret;
+	mb();
+	ret = IO_CONCAT(__IO_PREFIX,ioread64)(addr);
+	mb();
+	return ret;
+}
+
 void iowrite8(u8 b, void __iomem *addr)
 {
 	mb();
@@ -59,12 +68,20 @@ void iowrite32(u32 b, void __iomem *addr)
 	IO_CONCAT(__IO_PREFIX,iowrite32)(b, addr);
 }
 
+void iowrite64(u64 b, void __iomem *addr)
+{
+	mb();
+	IO_CONCAT(__IO_PREFIX,iowrite64)(b, addr);
+}
+
 EXPORT_SYMBOL(ioread8);
 EXPORT_SYMBOL(ioread16);
 EXPORT_SYMBOL(ioread32);
+EXPORT_SYMBOL(ioread64);
 EXPORT_SYMBOL(iowrite8);
 EXPORT_SYMBOL(iowrite16);
 EXPORT_SYMBOL(iowrite32);
+EXPORT_SYMBOL(iowrite64);
 
 u8 inb(unsigned long port)
 {
diff --git a/arch/alpha/kernel/machvec_impl.h b/arch/alpha/kernel/machvec_impl.h
index 393d5d6ca5d2..c2ebcb39e589 100644
--- a/arch/alpha/kernel/machvec_impl.h
+++ b/arch/alpha/kernel/machvec_impl.h
@@ -78,9 +78,11 @@
 	.mv_ioread8 =		CAT(low,_ioread8),			\
 	.mv_ioread16 =		CAT(low,_ioread16),			\
 	.mv_ioread32 =		CAT(low,_ioread32),			\
+	.mv_ioread64 =		CAT(low,_ioread64),			\
 	.mv_iowrite8 =		CAT(low,_iowrite8),			\
 	.mv_iowrite16 =		CAT(low,_iowrite16),			\
 	.mv_iowrite32 =		CAT(low,_iowrite32),			\
+	.mv_iowrite64 =		CAT(low,_iowrite64),			\
 	.mv_readb =		CAT(low,_readb),			\
 	.mv_readw =		CAT(low,_readw),			\
 	.mv_readl =		CAT(low,_readl),			\
diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c
index d257293401e2..68ec314d3fac 100644
--- a/arch/alpha/kernel/osf_sys.c
+++ b/arch/alpha/kernel/osf_sys.c
@@ -108,7 +108,7 @@ struct osf_dirent_callback {
 	int error;
 };
 
-static int
+static bool
 osf_filldir(struct dir_context *ctx, const char *name, int namlen,
 	    loff_t offset, u64 ino, unsigned int d_type)
 {
@@ -120,11 +120,11 @@ osf_filldir(struct dir_context *ctx, const char *name, int namlen,
 
 	buf->error = -EINVAL;	/* only used if we fail */
 	if (reclen > buf->count)
-		return -EINVAL;
+		return false;
 	d_ino = ino;
 	if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) {
 		buf->error = -EOVERFLOW;
-		return -EOVERFLOW;
+		return false;
 	}
 	if (buf->basep) {
 		if (put_user(offset, buf->basep))
@@ -141,10 +141,10 @@ osf_filldir(struct dir_context *ctx, const char *name, int namlen,
 	dirent = (void __user *)dirent + reclen;
 	buf->dirent = dirent;
 	buf->count -= reclen;
-	return 0;
+	return true;
 Efault:
 	buf->error = -EFAULT;
-	return -EFAULT;
+	return false;
 }
 
 SYSCALL_DEFINE4(osf_getdirentries, unsigned int, fd,
@@ -1278,45 +1278,15 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr,
 	return addr;
 }
 
-#ifdef CONFIG_OSF4_COMPAT
-/* Clear top 32 bits of iov_len in the user's buffer for
-   compatibility with old versions of OSF/1 where iov_len
-   was defined as int. */
-static int
-osf_fix_iov_len(const struct iovec __user *iov, unsigned long count)
-{
-	unsigned long i;
-
-	for (i = 0 ; i < count ; i++) {
-		int __user *iov_len_high = (int __user *)&iov[i].iov_len + 1;
-
-		if (put_user(0, iov_len_high))
-			return -EFAULT;
-	}
-	return 0;
-}
-#endif
-
 SYSCALL_DEFINE3(osf_readv, unsigned long, fd,
 		const struct iovec __user *, vector, unsigned long, count)
 {
-#ifdef CONFIG_OSF4_COMPAT
-	if (unlikely(personality(current->personality) == PER_OSF4))
-		if (osf_fix_iov_len(vector, count))
-			return -EFAULT;
-#endif
-
 	return sys_readv(fd, vector, count);
 }
 
 SYSCALL_DEFINE3(osf_writev, unsigned long, fd,
 		const struct iovec __user *, vector, unsigned long, count)
 {
-#ifdef CONFIG_OSF4_COMPAT
-	if (unlikely(personality(current->personality) == PER_OSF4))
-		if (osf_fix_iov_len(vector, count))
-			return -EFAULT;
-#endif
 	return sys_writev(fd, vector, count);
 }
 
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 87badeae3181..22dc1d6936bc 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -28,7 +28,6 @@ config ARM
 	select ARCH_HAS_GCOV_PROFILE_ALL
 	select ARCH_KEEP_MEMBLOCK
 	select ARCH_MIGHT_HAVE_PC_PARPORT
-	select ARCH_NO_SG_CHAIN if !ARM_HAS_SG_CHAIN
 	select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX
 	select ARCH_OPTIONAL_KERNEL_RWX_DEFAULT if CPU_V7
 	select ARCH_SUPPORTS_ATOMIC_RMW
@@ -42,6 +41,7 @@ config ARM
 	select ARCH_WANT_LD_ORPHAN_WARN
 	select BINFMT_FLAT_ARGVP_ENVP_ON_STACK
 	select BUILDTIME_TABLE_SORT if MMU
+	select COMMON_CLK if !(ARCH_RPC || ARCH_FOOTBRIDGE)
 	select CLONE_BACKWARDS
 	select CPU_PM if SUSPEND || CPU_IDLE
 	select DCACHE_WORD_ACCESS if HAVE_EFFICIENT_UNALIGNED_ACCESS
@@ -110,6 +110,7 @@ config ARM
 	select HAVE_MOD_ARCH_SPECIFIC
 	select HAVE_NMI
 	select HAVE_OPTPROBES if !THUMB2_KERNEL
+	select HAVE_PCI if MMU
 	select HAVE_PERF_EVENTS
 	select HAVE_PERF_REGS
 	select HAVE_PERF_USER_STACK_DUMP
@@ -126,13 +127,17 @@ config ARM
 	select OF_EARLY_FLATTREE if OF
 	select OLD_SIGACTION
 	select OLD_SIGSUSPEND3
+	select PCI_DOMAINS_GENERIC if PCI
 	select PCI_SYSCALL if PCI
 	select PERF_USE_VMALLOC
 	select RTC_LIB
+	select SPARSE_IRQ if !(ARCH_FOOTBRIDGE || ARCH_RPC)
 	select SYS_SUPPORTS_APM_EMULATION
 	select THREAD_INFO_IN_TASK
+	select TIMER_OF if OF
 	select HAVE_ARCH_VMAP_STACK if MMU && ARM_HAS_GROUP_RELOCS
 	select TRACE_IRQFLAGS_SUPPORT if !CPU_V7M
+	select USE_OF if !(ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100)
 	# Above selects are sorted alphabetically; please add new ones
 	# according to that.  Thanks.
 	help
@@ -154,12 +159,8 @@ config ARM_HAS_GROUP_RELOCS
 	  which is usually sufficient, but not for allyesconfig, so we disable
 	  this feature when doing compile testing.
 
-config ARM_HAS_SG_CHAIN
-	bool
-
 config ARM_DMA_USE_IOMMU
 	bool
-	select ARM_HAS_SG_CHAIN
 	select NEED_SG_DMA_LENGTH
 
 if ARM_DMA_USE_IOMMU
@@ -245,7 +246,7 @@ config ARCH_MTD_XIP
 config ARM_PATCH_PHYS_VIRT
 	bool "Patch physical to virtual translations at runtime" if EMBEDDED
 	default y
-	depends on !XIP_KERNEL && MMU
+	depends on MMU
 	help
 	  Patch phys-to-virt and virt-to-phys translation functions at
 	  boot and module load time according to the position of the
@@ -274,7 +275,7 @@ config NEED_MACH_MEMORY_H
 
 config PHYS_OFFSET
 	hex "Physical address of main memory" if MMU
-	depends on !ARM_PATCH_PHYS_VIRT
+	depends on !ARM_PATCH_PHYS_VIRT || !AUTO_ZRELADDR
 	default DRAM_BASE if !MMU
 	default 0x00000000 if ARCH_FOOTBRIDGE
 	default 0x10000000 if ARCH_OMAP1 || ARCH_RPC
@@ -307,13 +308,8 @@ config MMU
 config ARM_SINGLE_ARMV7M
 	def_bool !MMU
 	select ARM_NVIC
-	select AUTO_ZRELADDR
-	select TIMER_OF
-	select COMMON_CLK
 	select CPU_V7M
 	select NO_IOPORT_MAP
-	select SPARSE_IRQ
-	select USE_OF
 
 config ARCH_MMAP_RND_BITS_MIN
 	default 8
@@ -323,94 +319,31 @@ config ARCH_MMAP_RND_BITS_MAX
 	default 15 if PAGE_OFFSET=0x80000000
 	default 16
 
-#
-# The "ARM system type" choice list is ordered alphabetically by option
-# text.  Please add new entries in the option alphabetic order.
-#
-choice
-	prompt "ARM system type"
-	depends on MMU
-	default ARCH_MULTIPLATFORM
-
 config ARCH_MULTIPLATFORM
-	bool "Allow multiple platforms to be selected"
-	select ARCH_FLATMEM_ENABLE
-	select ARCH_SPARSEMEM_ENABLE
-	select ARCH_SELECT_MEMORY_MODEL
-	select ARM_HAS_SG_CHAIN
-	select ARM_PATCH_PHYS_VIRT
-	select AUTO_ZRELADDR
-	select TIMER_OF
-	select COMMON_CLK
-	select HAVE_PCI
-	select PCI_DOMAINS_GENERIC if PCI
-	select SPARSE_IRQ
-	select USE_OF
-
-config ARCH_FOOTBRIDGE
-	bool "FootBridge"
-	depends on CPU_LITTLE_ENDIAN
-	depends on ATAGS
-	select CPU_SA110
-	select FOOTBRIDGE
-	select NEED_MACH_MEMORY_H
-	help
-	  Support for systems based on the DC21285 companion chip
-	  ("FootBridge"), such as the Simtec CATS and the Rebel NetWinder.
-
-config ARCH_RPC
-	bool "RiscPC"
-	depends on !CC_IS_CLANG && GCC_VERSION < 90100 && GCC_VERSION >= 60000
-	depends on CPU_LITTLE_ENDIAN
-	depends on ATAGS
-	select ARCH_ACORN
-	select ARCH_MAY_HAVE_PC_FDC
-	select ARCH_SPARSEMEM_ENABLE
-	select ARM_HAS_SG_CHAIN
-	select CPU_SA110
-	select FIQ
-	select HAVE_PATA_PLATFORM
-	select ISA_DMA_API
-	select LEGACY_TIMER_TICK
-	select NEED_MACH_IO_H
-	select NEED_MACH_MEMORY_H
-	select NO_IOPORT_MAP
+	bool "Require kernel to be portable to multiple machines" if EXPERT
+	depends on MMU && !(ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100)
+	default y
 	help
-	  On the Acorn Risc-PC, Linux can support the internal IDE disk and
-	  CD-ROM interface, serial and parallel port, and the floppy drive.
+	  In general, all Arm machines can be supported in a single
+	  kernel image, covering either Armv4/v5 or Armv6/v7.
 
-config ARCH_SA1100
-	bool "SA1100-based"
-	depends on CPU_LITTLE_ENDIAN
-	depends on ATAGS
-	select ARCH_MTD_XIP
-	select ARCH_SPARSEMEM_ENABLE
-	select CLKSRC_MMIO
-	select CLKSRC_PXA
-	select TIMER_OF if OF
-	select COMMON_CLK
-	select CPU_FREQ
-	select CPU_SA1100
-	select GPIOLIB
-	select IRQ_DOMAIN
-	select ISA
-	select NEED_MACH_MEMORY_H
-	select SPARSE_IRQ
-	help
-	  Support for StrongARM 11x0 based boards.
+	  However, some configuration options require hardcoding machine
+	  specific physical addresses or enable errata workarounds that may
+	  break other machines.
 
-endchoice
+	  Selecting N here allows using those options, including
+	  DEBUG_UNCOMPRESS, XIP_KERNEL and ZBOOT_ROM. If unsure, say Y.
 
-menu "Multiple platform selection"
-	depends on ARCH_MULTIPLATFORM
+menu "Platform selection"
+	depends on MMU
 
 comment "CPU Core family selection"
 
 config ARCH_MULTI_V4
-	bool "ARMv4 based platforms (FA526)"
+	bool "ARMv4 based platforms (FA526, StrongARM)"
 	depends on !ARCH_MULTI_V6_V7
 	select ARCH_MULTI_V4_V5
-	select CPU_FA526
+	select CPU_FA526 if !(CPU_SA110 || CPU_SA1100)
 
 config ARCH_MULTI_V4T
 	bool "ARMv4T based platforms (ARM720T, ARM920T, ...)"
@@ -472,7 +405,6 @@ config ARCH_AIROHA
 	select ARM_GIC_V3
 	select ARM_PSCI
 	select HAVE_ARM_ARCH_TIMER
-	select COMMON_CLK
 	help
 	  Support for Airoha EN7523 SoCs
 
@@ -573,6 +505,8 @@ source "arch/arm/mach-rda/Kconfig"
 
 source "arch/arm/mach-realtek/Kconfig"
 
+source "arch/arm/mach-rpc/Kconfig"
+
 source "arch/arm/mach-rockchip/Kconfig"
 
 source "arch/arm/mach-s3c/Kconfig"
@@ -638,7 +572,6 @@ config ARCH_ACORN
 config PLAT_ORION
 	bool
 	select CLKSRC_MMIO
-	select COMMON_CLK
 	select GENERIC_IRQ_CHIP
 	select IRQ_DOMAIN
 
@@ -989,11 +922,6 @@ config ISA
 	  (MCA) or VESA.  ISA is an older system, now being displaced by PCI;
 	  newer boards don't support it.  If you have ISA, say Y, otherwise N.
 
-# Select ISA DMA controller support
-config ISA_DMA
-	bool
-	select ISA_DMA_API
-
 # Select ISA DMA interface
 config ISA_DMA_API
 	bool
@@ -1054,7 +982,7 @@ config SMP
 
 config SMP_ON_UP
 	bool "Allow booting SMP kernel on uniprocessor systems"
-	depends on SMP && !XIP_KERNEL && MMU
+	depends on SMP && MMU
 	default y
 	help
 	  SMP kernels contain instructions which fail on non-SMP processors.
@@ -1303,7 +1231,7 @@ config THUMB2_KERNEL
 
 config ARM_PATCH_IDIV
 	bool "Runtime patch udiv/sdiv instructions into __aeabi_{u}idiv()"
-	depends on CPU_32v7 && !XIP_KERNEL
+	depends on CPU_32v7
 	default y
 	help
 	  The ARM compiler inserts calls to __aeabi_idiv() and
@@ -1358,13 +1286,13 @@ config OABI_COMPAT
 	  at all). If in doubt say N.
 
 config ARCH_SELECT_MEMORY_MODEL
-	bool
+	def_bool y
 
 config ARCH_FLATMEM_ENABLE
-	bool
+	def_bool !(ARCH_RPC || ARCH_SA1100)
 
 config ARCH_SPARSEMEM_ENABLE
-	bool
+	def_bool !ARCH_FOOTBRIDGE
 	select SPARSEMEM_STATIC if SPARSEMEM
 
 config HIGHMEM
@@ -1671,7 +1599,6 @@ config CMDLINE
 choice
 	prompt "Kernel command line type" if CMDLINE != ""
 	default CMDLINE_FROM_BOOTLOADER
-	depends on ATAGS
 
 config CMDLINE_FROM_BOOTLOADER
 	bool "Use bootloader kernel arguments if available"
@@ -1698,6 +1625,7 @@ endchoice
 config XIP_KERNEL
 	bool "Kernel Execute-In-Place from ROM"
 	depends on !ARM_LPAE && !ARCH_MULTIPLATFORM
+	depends on !ARM_PATCH_IDIV && !ARM_PATCH_PHYS_VIRT && !SMP_ON_UP
 	help
 	  Execute-In-Place allows the kernel to run from non-volatile storage
 	  directly addressable by the CPU, such as NOR flash. This saves RAM
@@ -1772,7 +1700,8 @@ config CRASH_DUMP
 	  For more details see Documentation/admin-guide/kdump/kdump.rst
 
 config AUTO_ZRELADDR
-	bool "Auto calculation of the decompressed kernel image address"
+	bool "Auto calculation of the decompressed kernel image address" if !ARCH_MULTIPLATFORM
+	default !(ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100)
 	help
 	  ZRELADDR is the physical address where the decompressed kernel
 	  image will be placed. If AUTO_ZRELADDR is selected, the address
diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug
index 792796a348c3..c345775f035b 100644
--- a/arch/arm/Kconfig.debug
+++ b/arch/arm/Kconfig.debug
@@ -1904,7 +1904,8 @@ config DEBUG_UART_8250_PALMCHIP
 
 config DEBUG_UNCOMPRESS
 	bool "Enable decompressor debugging via DEBUG_LL output"
-	depends on ARCH_MULTIPLATFORM || PLAT_SAMSUNG || ARM_SINGLE_ARMV7M
+	depends on !ARCH_MULTIPLATFORM
+	depends on !(ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100)
 	depends on DEBUG_LL && !DEBUG_OMAP2PLUS_UART && \
 		     (!DEBUG_TEGRA_UART || !ZBOOT_ROM) && \
 		     !DEBUG_BRCMSTB_UART && !DEBUG_SEMIHOSTING
@@ -1921,9 +1922,8 @@ config DEBUG_UNCOMPRESS
 
 config UNCOMPRESS_INCLUDE
 	string
-	default "debug/uncompress.h" if ARCH_MULTIPLATFORM || ARCH_MSM || \
-					PLAT_SAMSUNG || ARM_SINGLE_ARMV7M
-	default "mach/uncompress.h"
+	default "mach/uncompress.h" if ARCH_FOOTBRIDGE || ARCH_RPC || ARCH_SA1100
+	default "debug/uncompress.h"
 
 config EARLY_PRINTK
 	bool "Early printk"
diff --git a/arch/arm/Makefile b/arch/arm/Makefile
index 56f655deebb1..0e04bc6b2ad3 100644
--- a/arch/arm/Makefile
+++ b/arch/arm/Makefile
@@ -22,6 +22,9 @@ GZFLAGS		:=-9
 # Never generate .eh_frame
 KBUILD_CFLAGS	+= $(call cc-option,-fno-dwarf2-cfi-asm)
 
+# Disable FDPIC ABI
+KBUILD_CFLAGS	+= $(call cc-option,-mno-fdpic)
+
 # This should work on most of the modern platforms
 KBUILD_DEFCONFIG := multi_v7_defconfig
 
@@ -224,40 +227,24 @@ machine-$(CONFIG_ARCH_ZYNQ)		+= zynq
 machine-$(CONFIG_PLAT_VERSATILE)	+= versatile
 machine-$(CONFIG_PLAT_SPEAR)		+= spear
 
-# Platform directory name.  This list is sorted alphanumerically
-# by CONFIG_* macro name.
-plat-$(CONFIG_PLAT_ORION)	+= orion
+# legacy platforms provide their own mach/*.h headers globally,
+# these three are mutually exclusive
+machdirs-$(CONFIG_ARCH_FOOTBRIDGE)	+= arch/arm/mach-footbridge
+machdirs-$(CONFIG_ARCH_RPC)		+= arch/arm/mach-rpc
+machdirs-$(CONFIG_ARCH_SA1100)		+= arch/arm/mach-sa1100
+KBUILD_CPPFLAGS += $(patsubst %,-I$(srctree)/%/include,$(machdirs-y))
 
 # The byte offset of the kernel image in RAM from the start of RAM.
 TEXT_OFFSET := $(textofs-y)
 
-# The first directory contains additional information for the boot setup code
-ifneq ($(machine-y),)
-MACHINE  := arch/arm/mach-$(word 1,$(machine-y))/
-else
-MACHINE  :=
-endif
-ifeq ($(CONFIG_ARCH_MULTIPLATFORM),y)
-MACHINE  :=
-endif
-
-machdirs := $(patsubst %,arch/arm/mach-%/,$(machine-y))
-platdirs := $(patsubst %,arch/arm/plat-%/,$(sort $(plat-y)))
-
-ifneq ($(CONFIG_ARCH_MULTIPLATFORM),y)
-ifneq ($(CONFIG_ARM_SINGLE_ARMV7M),y)
-KBUILD_CPPFLAGS += $(patsubst %,-I$(srctree)/%include,$(machdirs) $(platdirs))
-endif
-endif
-
 export	TEXT_OFFSET GZFLAGS MMUEXT
 
 # If we have a machine-specific directory, then include it in the build.
-core-y				+= $(machdirs) $(platdirs)
-
+core-y				+= $(patsubst %,arch/arm/mach-%/,$(machine-y))
 # For cleaning
-core-				+= $(patsubst %,arch/arm/mach-%/, $(machine-))
-core-				+= $(patsubst %,arch/arm/plat-%/, $(plat-))
+core-				+= $(patsubst %,arch/arm/mach-%/,$(machine-))
+
+core-$(CONFIG_PLAT_ORION)	+= arch/arm/plat-orion/
 
 libs-y				:= arch/arm/lib/ $(libs-y)
 
@@ -310,7 +297,7 @@ bootpImage uImage: zImage
 zImage: Image
 
 $(BOOT_TARGETS): vmlinux
-	$(Q)$(MAKE) $(build)=$(boot) MACHINE=$(MACHINE) $(boot)/$@
+	$(Q)$(MAKE) $(build)=$(boot) $(boot)/$@
 	@$(kecho) '  Kernel: $(boot)/$@ is ready'
 
 $(INSTALL_TARGETS): KBUILD_IMAGE = $(boot)/$(patsubst %install,%Image,$@)
@@ -324,7 +311,7 @@ ifeq ($(CONFIG_VDSO),y)
 endif
 
 # My testing targets (bypasses dependencies)
-bp:;	$(Q)$(MAKE) $(build)=$(boot) MACHINE=$(MACHINE) $(boot)/bootpImage
+bp:;	$(Q)$(MAKE) $(build)=$(boot) $(boot)/bootpImage
 
 
 define archhelp
diff --git a/arch/arm/boot/Makefile b/arch/arm/boot/Makefile
index 54a09f9464fb..abd6a2889fd0 100644
--- a/arch/arm/boot/Makefile
+++ b/arch/arm/boot/Makefile
@@ -10,22 +10,16 @@
 #
 # Copyright (C) 1995-2002 Russell King
 #
-
 OBJCOPYFLAGS	:=-O binary -R .comment -S
 
-ifneq ($(MACHINE),)
-include $(MACHINE)/Makefile.boot
-endif
-
-# Note: the following conditions must always be true:
 #   ZRELADDR == virt_to_phys(PAGE_OFFSET + TEXT_OFFSET)
-#   PARAMS_PHYS must be within 4MB of ZRELADDR
-#   INITRD_PHYS must be in RAM
-ZRELADDR    := $(zreladdr-y)
-PARAMS_PHYS := $(params_phys-y)
-INITRD_PHYS := $(initrd_phys-y)
+ifdef CONFIG_PHYS_OFFSET
+add_hex = $(shell printf 0x%x $$(( $(1) + $(2) )) )
+ZRELADDR    := $(call add_hex, $(CONFIG_PHYS_OFFSET), $(TEXT_OFFSET))
+endif
 
-export ZRELADDR INITRD_PHYS PARAMS_PHYS
+PHYS_OFFSET := $(CONFIG_PHYS_OFFSET)
+export ZRELADDR PARAMS_PHYS PHYS_OFFSET
 
 targets := Image zImage xipImage bootpImage uImage
 
@@ -90,17 +84,10 @@ $(obj)/uImage:	$(obj)/zImage FORCE
 	@$(check_for_multiple_loadaddr)
 	$(call if_changed,uimage)
 
-$(obj)/bootp/bootp: $(obj)/zImage initrd FORCE
+$(obj)/bootp/bootp: $(obj)/zImage FORCE
 	$(Q)$(MAKE) $(build)=$(obj)/bootp $@
 
 $(obj)/bootpImage: $(obj)/bootp/bootp FORCE
 	$(call if_changed,objcopy)
 
-PHONY += initrd
-initrd:
-	@test "$(INITRD_PHYS)" != "" || \
-	(echo This machine does not support INITRD; exit -1)
-	@test "$(INITRD)" != "" || \
-	(echo You must specify INITRD; exit -1)
-
 subdir-	    := bootp compressed dts
diff --git a/arch/arm/boot/bootp/Makefile b/arch/arm/boot/bootp/Makefile
index 981a8d03f064..a2934e6fd89a 100644
--- a/arch/arm/boot/bootp/Makefile
+++ b/arch/arm/boot/bootp/Makefile
@@ -5,9 +5,40 @@
 # This file is included by the global makefile so that you can add your own
 # architecture-specific flags and dependencies.
 #
-
 GCOV_PROFILE	:= n
 
+ifdef PHYS_OFFSET
+add_hex = $(shell printf 0x%x $$(( $(1) + $(2) )) )
+
+# If PHYS_OFFSET is set, INITRD_PHYS and PARAMS_PHYS can be derived,
+# otherwise they must be passed on the command line.
+#
+# Note: the following conditions must always be true:
+#   PARAMS_PHYS must be within 4MB of ZRELADDR
+#   INITRD_PHYS must be in RAM
+
+PARAMS_PHYS := $(call add_hex, $(PHYS_OFFSET), 0x100)
+
+# guess an initrd location if possible
+initrd_offset-$(CONFIG_ARCH_FOOTBRIDGE)	+= 0x00800000
+initrd_offset-$(CONFIG_ARCH_SA1100)	+= 0x00800000
+initrd_offset-$(CONFIG_ARCH_RPC)	+= 0x08000000
+INITRD_OFFSET := $(initrd_offset-y)
+ifdef INITRD_OFFSET
+INITRD_PHYS := $(call add_hex, $(PHYS_OFFSET), $(INITRD_OFFSET))
+endif
+
+endif
+
+PHONY += initrd
+initrd:
+	@test "$(PARAMS_PHYS)" != "" || \
+	(echo bootpImage: You must specify PHYS_OFFSET of PARAMS_PHYS ; exit -1)
+	@test "$(INITRD_PHYS)" != "" || \
+	(echo bootpImage: You must specify INITRD_OFFSET or INITRD_PHYS ; exit -1)
+	@test "$(INITRD)" != "" || \
+	(echo bootpImage: You must specify INITRD; exit -1)
+
 LDFLAGS_bootp	:= --no-undefined -X \
 		 --defsym initrd_phys=$(INITRD_PHYS) \
 		 --defsym params_phys=$(PARAMS_PHYS) -T
@@ -24,6 +55,6 @@ $(obj)/bootp:	$(src)/bootp.lds $(addprefix $(obj)/,init.o kernel.o initrd.o) FOR
 
 $(obj)/kernel.o: arch/arm/boot/zImage FORCE
 
-$(obj)/initrd.o: $(INITRD) FORCE
+$(obj)/initrd.o: initrd $(INITRD) FORCE
 
 PHONY += $(INITRD)
diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S
index bf79f2f78d23..9f406e9c0ea6 100644
--- a/arch/arm/boot/compressed/head.S
+++ b/arch/arm/boot/compressed/head.S
@@ -67,11 +67,7 @@
 #if defined(CONFIG_ARCH_SA1100)
 		.macro	loadsp, rb, tmp1, tmp2
 		mov	\rb, #0x80000000	@ physical base address
-#ifdef CONFIG_DEBUG_LL_SER3
-		add	\rb, \rb, #0x00050000	@ Ser3
-#else
 		add	\rb, \rb, #0x00010000	@ Ser1
-#endif
 		.endm
 #else
 		.macro	loadsp,	rb, tmp1, tmp2
diff --git a/arch/arm/boot/compressed/misc.c b/arch/arm/boot/compressed/misc.c
index cb2e069dc73f..abfed1aa2baa 100644
--- a/arch/arm/boot/compressed/misc.c
+++ b/arch/arm/boot/compressed/misc.c
@@ -23,7 +23,9 @@ unsigned int __machine_arch_type;
 #include <linux/types.h>
 #include <linux/linkage.h>
 #include "misc.h"
+#ifdef CONFIG_ARCH_EP93XX
 #include "misc-ep93xx.h"
+#endif
 
 static void putstr(const char *ptr);
 
diff --git a/arch/arm/boot/compressed/vmlinux.lds.S b/arch/arm/boot/compressed/vmlinux.lds.S
index 1bcb68ac4b01..3fcb3e62dc56 100644
--- a/arch/arm/boot/compressed/vmlinux.lds.S
+++ b/arch/arm/boot/compressed/vmlinux.lds.S
@@ -23,6 +23,7 @@ SECTIONS
     *(.ARM.extab*)
     *(.note.*)
     *(.rel.*)
+    *(.printk_index)
     /*
      * Discard any r/w data - this produces a link error if we have any,
      * which is required for PIC decompression.  Local data generates
@@ -57,6 +58,7 @@ SECTIONS
     *(.rodata)
     *(.rodata.*)
     *(.data.rel.ro)
+    *(.data.rel.ro.*)
   }
   .piggydata : {
     *(.piggydata)
diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile
index 05d8aef6e5d2..6aa7dc4db2fc 100644
--- a/arch/arm/boot/dts/Makefile
+++ b/arch/arm/boot/dts/Makefile
@@ -61,6 +61,7 @@ dtb-$(CONFIG_SOC_SAM_V7) += \
 	at91-sama5d2_icp.dtb \
 	at91-sama5d2_ptc_ek.dtb \
 	at91-sama5d2_xplained.dtb \
+	at91-sama5d3_eds.dtb \
 	at91-sama5d3_ksz9477_evb.dtb \
 	at91-sama5d3_xplained.dtb \
 	at91-dvk_som60.dtb \
@@ -706,8 +707,8 @@ dtb-$(CONFIG_SOC_IMX6UL) += \
 	imx6ul-geam.dtb \
 	imx6ul-isiot-emmc.dtb \
 	imx6ul-isiot-nand.dtb \
-	imx6ul-kontron-n6310-s.dtb \
-	imx6ul-kontron-n6310-s-43.dtb \
+	imx6ul-kontron-bl.dtb \
+	imx6ul-kontron-bl-43.dtb \
 	imx6ul-liteboard.dtb \
 	imx6ul-tqma6ul1-mba6ulx.dtb \
 	imx6ul-tqma6ul2-mba6ulx.dtb \
@@ -736,6 +737,7 @@ dtb-$(CONFIG_SOC_IMX6UL) += \
 	imx6ull-colibri-wifi-iris.dtb \
 	imx6ull-colibri-wifi-iris-v2.dtb \
 	imx6ull-jozacp.dtb \
+	imx6ull-kontron-bl.dtb \
 	imx6ull-myir-mys-6ulx-eval.dtb \
 	imx6ull-opos6uldev.dtb \
 	imx6ull-phytec-segin-ff-rdk-nand.dtb \
@@ -788,6 +790,7 @@ dtb-$(CONFIG_SOC_IMXRT) += \
 dtb-$(CONFIG_SOC_LAN966) += \
 	lan966x-kontron-kswitch-d10-mmt-6g-2gs.dtb \
 	lan966x-kontron-kswitch-d10-mmt-8g.dtb \
+	lan966x-pcb8290.dtb \
 	lan966x-pcb8291.dtb \
 	lan966x-pcb8309.dtb
 dtb-$(CONFIG_SOC_LS1021A) += \
@@ -1047,6 +1050,9 @@ dtb-$(CONFIG_ARCH_QCOM) += \
 	qcom-ipq8064-rb3011.dtb \
 	qcom-msm8226-samsung-s3ve3g.dtb \
 	qcom-msm8660-surf.dtb \
+	qcom-msm8916-samsung-e5.dtb \
+	qcom-msm8916-samsung-e7.dtb \
+	qcom-msm8916-samsung-grandmax.dtb \
 	qcom-msm8916-samsung-serranove.dtb \
 	qcom-msm8960-cdp.dtb \
 	qcom-msm8974-lge-nexus5-hammerhead.dtb \
@@ -1574,8 +1580,10 @@ dtb-$(CONFIG_ARCH_ASPEED) += \
 	aspeed-ast2500-evb.dtb \
 	aspeed-ast2600-evb-a1.dtb \
 	aspeed-ast2600-evb.dtb \
+	aspeed-bmc-amd-daytonax.dtb \
 	aspeed-bmc-amd-ethanolx.dtb \
 	aspeed-bmc-ampere-mtjade.dtb \
+	aspeed-bmc-ampere-mtmitchell.dtb \
 	aspeed-bmc-arm-stardragon4800-rep2.dtb \
 	aspeed-bmc-asrock-e3c246d4i.dtb \
 	aspeed-bmc-asrock-romed8hm3.dtb \
diff --git a/arch/arm/boot/dts/am335x-baltos-ir2110.dts b/arch/arm/boot/dts/am335x-baltos-ir2110.dts
index daf4cb398070..75992eec830f 100644
--- a/arch/arm/boot/dts/am335x-baltos-ir2110.dts
+++ b/arch/arm/boot/dts/am335x-baltos-ir2110.dts
@@ -81,3 +81,147 @@
 	pinctrl-0 = <&mmc1_pins>;
 	cd-gpios = <&gpio1 15 GPIO_ACTIVE_LOW>;
 };
+
+&gpio0 {
+	gpio-line-names =
+		"MDIO",
+		"MDC",
+		"NC",
+		"NC",
+		"I2C1_SDA",
+		"I2C1_SCL",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"UART1_CTSN",
+		"UART1_RTSN",
+		"UART1_RX",
+		"UART1_TX",
+		"onrisc:blue:wlan",
+		"onrisc:green:app",
+		"USB0_DRVVBUS",
+		"ETH2_INT",
+		"NC",
+		"RMII1_TXD1",
+		"MMC1_DAT0",
+		"MMC1_DAT1",
+		"NC",
+		"NC",
+		"MMC1_DAT2",
+		"MMC1_DAT3",
+		"RMII1_TXD0",
+		"NC",
+		"GPMC_WAIT0",
+		"GPMC_WP_N";
+};
+
+&gpio1 {
+	gpio-line-names =
+		"GPMC_AD0",
+		"GPMC_AD1",
+		"GPMC_AD2",
+		"GPMC_AD3",
+		"GPMC_AD4",
+		"GPMC_AD5",
+		"GPMC_AD6",
+		"GPMC_AD7",
+		"NC",
+		"NC",
+		"CONSOLE_RX",
+		"CONSOLE_TX",
+		"NC",
+		"NC",
+		"NC",
+		"SD_CD",
+		"RGMII2_TCTL",
+		"RGMII2_RCTL",
+		"RGMII2_TD3",
+		"RGMII2_TD2",
+		"RGMII2_TD1",
+		"RGMII2_TD0",
+		"RGMII2_TCLK",
+		"RGMII2_RCLK",
+		"RGMII2_RD3",
+		"RGMII2_RD2",
+		"RGMII2_RD1",
+		"RGMII2_RD0",
+		"PMIC_INT1",
+		"GPMC_CSN0_Flash",
+		"MMC1_CLK",
+		"MMC1_CMD";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"GPMC_CSN3_BUS",
+		"GPMC_CLK",
+		"GPMC_ADVN_ALE",
+		"GPMC_OEN_RE_N",
+		"GPMC_WE_N",
+		"GPMC_BEN0_CLE",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"SW2_0",
+		"SW2_1",
+		"RMII1_RXD1",
+		"RMII1_RXD0",
+		"UART1_DTR",
+		"UART1_DSR",
+		"UART1_DCD",
+		"UART1_RI",
+		"MMC0_DAT3",
+		"MMC0_DAT2",
+		"MMC0_DAT1",
+		"MMC0_DAT0",
+		"MMC0_CLK",
+		"MMC0_CMD";
+};
+
+&gpio3 {
+	gpio-line-names =
+		"onrisc:red:power",
+		"RMII1_CRS_DV",
+		"RMII1_RXER",
+		"RMII1_TXEN",
+		"NC",
+		"NC",
+		"NC",
+		"WLAN_IRQ",
+		"WLAN_EN",
+		"SW2_2",
+		"SW2_3",
+		"NC",
+		"NC",
+		"NC",
+		"ModeA0",
+		"ModeA1",
+		"ModeA2",
+		"ModeA3",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC";
+};
diff --git a/arch/arm/boot/dts/am335x-baltos-ir3220.dts b/arch/arm/boot/dts/am335x-baltos-ir3220.dts
index 2123bd589484..087e084506d2 100644
--- a/arch/arm/boot/dts/am335x-baltos-ir3220.dts
+++ b/arch/arm/boot/dts/am335x-baltos-ir3220.dts
@@ -91,6 +91,10 @@
 		interrupts = <20 IRQ_TYPE_EDGE_RISING>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&tca6416_pins>;
+		gpio-line-names = "GP_IN0", "GP_IN1", "GP_IN2", "GP_IN3",
+				  "GP_OUT0", "GP_OUT1", "GP_OUT2", "GP_OUT3",
+				  "ModeA0", "ModeA1", "ModeA2", "ModeA3",
+				  "ModeB0", "ModeB1", "ModeB2", "ModeB3";
 	};
 };
 
@@ -123,3 +127,147 @@
 	pinctrl-0 = <&mmc1_pins>;
 	cd-gpios = <&gpio2 18 GPIO_ACTIVE_LOW>;
 };
+
+&gpio0 {
+	gpio-line-names =
+		"MDIO",
+		"MDC",
+		"UART2_RX",
+		"UART2_TX",
+		"I2C1_SDA",
+		"I2C1_SCL",
+		"WLAN_BTN",
+		"W_DISABLE",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"UART1_CTSN",
+		"UART1_RTSN",
+		"UART1_RX",
+		"UART1_TX",
+		"onrisc:blue:wlan",
+		"onrisc:green:app",
+		"USB0_DRVVBUS",
+		"ETH2_INT",
+		"TCA6416_INT",
+		"RMII1_TXD1",
+		"MMC1_DAT0",
+		"MMC1_DAT1",
+		"NC",
+		"NC",
+		"MMC1_DAT2",
+		"MMC1_DAT3",
+		"RMII1_TXD0",
+		"NC",
+		"GPMC_WAIT0",
+		"GPMC_WP_N";
+};
+
+&gpio1 {
+	gpio-line-names =
+		"GPMC_AD0",
+		"GPMC_AD1",
+		"GPMC_AD2",
+		"GPMC_AD3",
+		"GPMC_AD4",
+		"GPMC_AD5",
+		"GPMC_AD6",
+		"GPMC_AD7",
+		"NC",
+		"NC",
+		"CONSOLE_RX",
+		"CONSOLE_TX",
+		"UART2_DTR",
+		"UART2_DSR",
+		"UART2_DCD",
+		"UART2_RI",
+		"RGMII2_TCTL",
+		"RGMII2_RCTL",
+		"RGMII2_TD3",
+		"RGMII2_TD2",
+		"RGMII2_TD1",
+		"RGMII2_TD0",
+		"RGMII2_TCLK",
+		"RGMII2_RCLK",
+		"RGMII2_RD3",
+		"RGMII2_RD2",
+		"RGMII2_RD1",
+		"RGMII2_RD0",
+		"PMIC_INT1",
+		"GPMC_CSN0_Flash",
+		"MMC1_CLK",
+		"MMC1_CMD";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"GPMC_CSN3_BUS",
+		"GPMC_CLK",
+		"GPMC_ADVN_ALE",
+		"GPMC_OEN_RE_N",
+		"GPMC_WE_N",
+		"GPMC_BEN0_CLE",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"SD_CD",
+		"SD_WP",
+		"RMII1_RXD1",
+		"RMII1_RXD0",
+		"UART1_DTR",
+		"UART1_DSR",
+		"UART1_DCD",
+		"UART1_RI",
+		"MMC0_DAT3",
+		"MMC0_DAT2",
+		"MMC0_DAT1",
+		"MMC0_DAT0",
+		"MMC0_CLK",
+		"MMC0_CMD";
+};
+
+&gpio3 {
+	gpio-line-names =
+		"onrisc:red:power",
+		"RMII1_CRS_DV",
+		"RMII1_RXER",
+		"RMII1_TXEN",
+		"3G_PWR_EN",
+		"UART2_CTSN",
+		"UART2_RTSN",
+		"WLAN_IRQ",
+		"WLAN_EN",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"USB1_DRVVBUS",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC";
+};
diff --git a/arch/arm/boot/dts/am335x-baltos-ir5221.dts b/arch/arm/boot/dts/am335x-baltos-ir5221.dts
index 2f3872dbf4f4..faeb39aab60a 100644
--- a/arch/arm/boot/dts/am335x-baltos-ir5221.dts
+++ b/arch/arm/boot/dts/am335x-baltos-ir5221.dts
@@ -99,6 +99,10 @@
 		interrupts = <20 IRQ_TYPE_EDGE_RISING>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&tca6416_pins>;
+		gpio-line-names = "GP_IN0", "GP_IN1", "GP_IN2", "GP_IN3",
+				  "GP_OUT0", "GP_OUT1", "GP_OUT2", "GP_OUT3",
+				  "ModeA0", "ModeA1", "ModeA2", "ModeA3",
+				  "ModeB0", "ModeB1", "ModeB2", "ModeB3";
 	};
 };
 
@@ -147,3 +151,147 @@
 	pinctrl-0 = <&mmc1_pins>;
 	cd-gpios = <&gpio2 18 GPIO_ACTIVE_LOW>;
 };
+
+&gpio0 {
+	gpio-line-names =
+		"MDIO",
+		"MDC",
+		"UART2_RX",
+		"UART2_TX",
+		"I2C1_SDA",
+		"I2C1_SCL",
+		"WLAN_BTN",
+		"W_DISABLE",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"UART1_CTSN",
+		"UART1_RTSN",
+		"UART1_RX",
+		"UART1_TX",
+		"onrisc:blue:wlan",
+		"onrisc:green:app",
+		"USB0_DRVVBUS",
+		"ETH2_INT",
+		"TCA6416_INT",
+		"RMII1_TXD1",
+		"MMC1_DAT0",
+		"MMC1_DAT1",
+		"NC",
+		"NC",
+		"MMC1_DAT2",
+		"MMC1_DAT3",
+		"RMII1_TXD0",
+		"NC",
+		"GPMC_WAIT0",
+		"GPMC_WP_N";
+};
+
+&gpio1 {
+	gpio-line-names =
+		"GPMC_AD0",
+		"GPMC_AD1",
+		"GPMC_AD2",
+		"GPMC_AD3",
+		"GPMC_AD4",
+		"GPMC_AD5",
+		"GPMC_AD6",
+		"GPMC_AD7",
+		"DCAN1_TX",
+		"DCAN1_RX",
+		"CONSOLE_RX",
+		"CONSOLE_TX",
+		"UART2_DTR",
+		"UART2_DSR",
+		"UART2_DCD",
+		"UART2_RI",
+		"RGMII2_TCTL",
+		"RGMII2_RCTL",
+		"RGMII2_TD3",
+		"RGMII2_TD2",
+		"RGMII2_TD1",
+		"RGMII2_TD0",
+		"RGMII2_TCLK",
+		"RGMII2_RCLK",
+		"RGMII2_RD3",
+		"RGMII2_RD2",
+		"RGMII2_RD1",
+		"RGMII2_RD0",
+		"PMIC_INT1",
+		"GPMC_CSN0_Flash",
+		"MMC1_CLK",
+		"MMC1_CMD";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"GPMC_CSN3_BUS",
+		"GPMC_CLK",
+		"GPMC_ADVN_ALE",
+		"GPMC_OEN_RE_N",
+		"GPMC_WE_N",
+		"GPMC_BEN0_CLE",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"SD_CD",
+		"SD_WP",
+		"RMII1_RXD1",
+		"RMII1_RXD0",
+		"UART1_DTR",
+		"UART1_DSR",
+		"UART1_DCD",
+		"UART1_RI",
+		"MMC0_DAT3",
+		"MMC0_DAT2",
+		"MMC0_DAT1",
+		"MMC0_DAT0",
+		"MMC0_CLK",
+		"MMC0_CMD";
+};
+
+&gpio3 {
+	gpio-line-names =
+		"onrisc:red:power",
+		"RMII1_CRS_DV",
+		"RMII1_RXER",
+		"RMII1_TXEN",
+		"3G_PWR_EN",
+		"UART2_CTSN",
+		"UART2_RTSN",
+		"WLAN_IRQ",
+		"WLAN_EN",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"USB1_DRVVBUS",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC";
+};
diff --git a/arch/arm/boot/dts/am335x-baltos.dtsi b/arch/arm/boot/dts/am335x-baltos.dtsi
index d3eafee79a23..6161c8929a78 100644
--- a/arch/arm/boot/dts/am335x-baltos.dtsi
+++ b/arch/arm/boot/dts/am335x-baltos.dtsi
@@ -197,7 +197,7 @@
 		rb-gpios = <&gpmc 0 GPIO_ACTIVE_HIGH>; /* gpmc_wait0 */
 		nand-bus-width = <8>;
 		ti,nand-ecc-opt = "bch8";
-		ti,nand-xfer-type = "polled";
+		ti,nand-xfer-type = "prefetch-dma";
 
 		gpmc,device-nand = "true";
 		gpmc,device-width = <1>;
diff --git a/arch/arm/boot/dts/am335x-netcan-plus-1xx.dts b/arch/arm/boot/dts/am335x-netcan-plus-1xx.dts
index 57e756b0f192..2e049489ac06 100644
--- a/arch/arm/boot/dts/am335x-netcan-plus-1xx.dts
+++ b/arch/arm/boot/dts/am335x-netcan-plus-1xx.dts
@@ -85,3 +85,147 @@
 
 	status = "okay";
 };
+
+&gpio0 {
+	gpio-line-names =
+		"MDIO",
+		"MDC",
+		"NC",
+		"NC",
+		"I2C1_SDA",
+		"I2C1_SCL",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"can_data",
+		"can_error",
+		"onrisc:blue:wlan",
+		"onrisc:green:app",
+		"USB0_DRVVBUS",
+		"ETH2_INT",
+		"NC",
+		"NC",
+		"MMC1_DAT0",
+		"MMC1_DAT1",
+		"NC",
+		"NC",
+		"MMC1_DAT2",
+		"MMC1_DAT3",
+		"NC",
+		"NC",
+		"GPMC_WAIT0",
+		"GPMC_WP_N";
+};
+
+&gpio1 {
+	gpio-line-names =
+		"GPMC_AD0",
+		"GPMC_AD1",
+		"GPMC_AD2",
+		"GPMC_AD3",
+		"GPMC_AD4",
+		"GPMC_AD5",
+		"GPMC_AD6",
+		"GPMC_AD7",
+		"DCAN1_TX",
+		"DCAN1_RX",
+		"CONSOLE_RX",
+		"CONSOLE_TX",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"RGMII2_TCTL",
+		"RGMII2_RCTL",
+		"RGMII2_TD3",
+		"RGMII2_TD2",
+		"RGMII2_TD1",
+		"RGMII2_TD0",
+		"RGMII2_TCLK",
+		"RGMII2_RCLK",
+		"RGMII2_RD3",
+		"RGMII2_RD2",
+		"RGMII2_RD1",
+		"RGMII2_RD0",
+		"PMIC_INT1",
+		"GPMC_CSN0_Flash",
+		"MMC1_CLK",
+		"MMC1_CMD";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"GPMC_CSN3_BUS",
+		"GPMC_CLK",
+		"GPMC_ADVN_ALE",
+		"GPMC_OEN_RE_N",
+		"GPMC_WE_N",
+		"GPMC_BEN0_CLE",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"SW2_0",
+		"SW2_1",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"MMC0_DAT3",
+		"MMC0_DAT2",
+		"MMC0_DAT1",
+		"MMC0_DAT0",
+		"MMC0_CLK",
+		"MMC0_CMD";
+};
+
+&gpio3 {
+	gpio-line-names =
+		"onrisc:red:power",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"WLAN_IRQ",
+		"WLAN_EN",
+		"SW2_2",
+		"SW2_3",
+		"NC",
+		"NC",
+		"NC",
+		"ModeA0",
+		"ModeA1",
+		"ModeA2",
+		"ModeA3",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC";
+};
diff --git a/arch/arm/boot/dts/am335x-netcom-plus-2xx.dts b/arch/arm/boot/dts/am335x-netcom-plus-2xx.dts
index c6cc1c6218a9..6ed886c3306b 100644
--- a/arch/arm/boot/dts/am335x-netcom-plus-2xx.dts
+++ b/arch/arm/boot/dts/am335x-netcom-plus-2xx.dts
@@ -93,3 +93,147 @@
 	ti,dual-emac-pvid = <2>;
 	phy-handle = <&phy1>;
 };
+
+&gpio0 {
+	gpio-line-names =
+		"MDIO",
+		"MDC",
+		"UART2_RX",
+		"UART2_TX",
+		"I2C1_SDA",
+		"I2C1_SCL",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"UART1_CTSN",
+		"UART1_RTSN",
+		"UART1_RX",
+		"UART1_TX",
+		"onrisc:blue:wlan",
+		"onrisc:green:app",
+		"USB0_DRVVBUS",
+		"ETH2_INT",
+		"NC",
+		"NC",
+		"MMC1_DAT0",
+		"MMC1_DAT1",
+		"NC",
+		"NC",
+		"MMC1_DAT2",
+		"MMC1_DAT3",
+		"NC",
+		"NC",
+		"GPMC_WAIT0",
+		"GPMC_WP_N";
+};
+
+&gpio1 {
+	gpio-line-names =
+		"GPMC_AD0",
+		"GPMC_AD1",
+		"GPMC_AD2",
+		"GPMC_AD3",
+		"GPMC_AD4",
+		"GPMC_AD5",
+		"GPMC_AD6",
+		"GPMC_AD7",
+		"NC",
+		"NC",
+		"CONSOLE_RX",
+		"CONSOLE_TX",
+		"UART2_DTR",
+		"UART2_DSR",
+		"UART2_DCD",
+		"UART2_RI",
+		"RGMII2_TCTL",
+		"RGMII2_RCTL",
+		"RGMII2_TD3",
+		"RGMII2_TD2",
+		"RGMII2_TD1",
+		"RGMII2_TD0",
+		"RGMII2_TCLK",
+		"RGMII2_RCLK",
+		"RGMII2_RD3",
+		"RGMII2_RD2",
+		"RGMII2_RD1",
+		"RGMII2_RD0",
+		"PMIC_INT1",
+		"GPMC_CSN0_Flash",
+		"MMC1_CLK",
+		"MMC1_CMD";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"GPMC_CSN3_BUS",
+		"GPMC_CLK",
+		"GPMC_ADVN_ALE",
+		"GPMC_OEN_RE_N",
+		"GPMC_WE_N",
+		"GPMC_BEN0_CLE",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"SW2_0",
+		"SW2_1",
+		"NC",
+		"NC",
+		"UART1_DTR",
+		"UART1_DSR",
+		"UART1_DCD",
+		"UART1_RI",
+		"MMC0_DAT3",
+		"MMC0_DAT2",
+		"MMC0_DAT1",
+		"MMC0_DAT0",
+		"MMC0_CLK",
+		"MMC0_CMD";
+};
+
+&gpio3 {
+	gpio-line-names =
+		"onrisc:red:power",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"UART2_CTSN",
+		"UART2_RTSN",
+		"WLAN_IRQ",
+		"WLAN_EN",
+		"SW2_2",
+		"SW2_3",
+		"NC",
+		"NC",
+		"NC",
+		"ModeA0",
+		"ModeA1",
+		"ModeA2",
+		"ModeA3",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC";
+};
diff --git a/arch/arm/boot/dts/am335x-netcom-plus-8xx.dts b/arch/arm/boot/dts/am335x-netcom-plus-8xx.dts
index 96dffd3ffd85..ad3adc7679f9 100644
--- a/arch/arm/boot/dts/am335x-netcom-plus-8xx.dts
+++ b/arch/arm/boot/dts/am335x-netcom-plus-8xx.dts
@@ -71,6 +71,10 @@
 		interrupts = <20 IRQ_TYPE_EDGE_RISING>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&tca6416_pins>;
+		gpio-line-names = "GP_IN0", "GP_IN1", "GP_IN2", "GP_IN3",
+				  "GP_IN4", "GP_IN5", "GP_IN6", "GP_IN7",
+				  "GP_OUT0", "GP_OUT1", "GP_OUT2", "GP_OUT3",
+				  "GP_OUT4", "GP_OUT5", "GP_OUT6", "GP_OUT7";
 	};
 };
 
@@ -86,6 +90,10 @@
 		reg = <0x20>;
 		gpio-controller;
 		#gpio-cells = <2>;
+		gpio-line-names = "CH1_M0", "CH1_M1", "CH1_M2", "CH1_M3",
+				  "CH2_M0", "CH2_M1", "CH2_M2", "CH2_M3",
+				  "CH3_M0", "CH3_M1", "CH3_M2", "CH3_M3",
+				  "CH4_M0", "CH4_M1", "CH4_M2", "CH4_M3";
 	};
 
 	tca6416c: gpio@21 {
@@ -93,6 +101,10 @@
 		reg = <0x21>;
 		gpio-controller;
 		#gpio-cells = <2>;
+		gpio-line-names = "CH5_M0", "CH5_M1", "CH5_M2", "CH5_M3",
+				  "CH6_M0", "CH6_M1", "CH6_M2", "CH6_M3",
+				  "CH7_M0", "CH7_M1", "CH7_M2", "CH7_M3",
+				  "CH8_M0", "CH8_M1", "CH8_M2", "CH8_M3";
 	};
 };
 
@@ -113,3 +125,147 @@
 	ti,dual-emac-pvid = <2>;
 	phy-handle = <&phy1>;
 };
+
+&gpio0 {
+	gpio-line-names =
+		"MDIO",
+		"MDC",
+		"NC",
+		"NC",
+		"I2C1_SDA",
+		"I2C1_SCL",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"I2C2_SDA",
+		"I2C2_SCL",
+		"NC",
+		"NC",
+		"onrisc:blue:wlan",
+		"onrisc:green:app",
+		"USB0_DRVVBUS",
+		"ETH2_INT",
+		"NC",
+		"NC",
+		"MMC1_DAT0",
+		"MMC1_DAT1",
+		"NC",
+		"NC",
+		"MMC1_DAT2",
+		"MMC1_DAT3",
+		"NC",
+		"NC",
+		"GPMC_WAIT0",
+		"GPMC_WP_N";
+};
+
+&gpio1 {
+	gpio-line-names =
+		"GPMC_AD0",
+		"GPMC_AD1",
+		"GPMC_AD2",
+		"GPMC_AD3",
+		"GPMC_AD4",
+		"GPMC_AD5",
+		"GPMC_AD6",
+		"GPMC_AD7",
+		"NC",
+		"NC",
+		"CONSOLE_RX",
+		"CONSOLE_TX",
+		"SW2_0_alt",
+		"SW2_1_alt",
+		"SW2_2_alt",
+		"SW2_3_alt",
+		"RGMII2_TCTL",
+		"RGMII2_RCTL",
+		"RGMII2_TD3",
+		"RGMII2_TD2",
+		"RGMII2_TD1",
+		"RGMII2_TD0",
+		"RGMII2_TCLK",
+		"RGMII2_RCLK",
+		"RGMII2_RD3",
+		"RGMII2_RD2",
+		"RGMII2_RD1",
+		"RGMII2_RD0",
+		"PMIC_INT1",
+		"GPMC_CSN0_Flash",
+		"MMC1_CLK",
+		"MMC1_CMD";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"GPMC_CSN3_BUS",
+		"GPMC_CLK",
+		"GPMC_ADVN_ALE",
+		"GPMC_OEN_RE_N",
+		"GPMC_WE_N",
+		"GPMC_BEN0_CLE",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"SW2_0",
+		"SW2_1",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"MMC0_DAT3",
+		"MMC0_DAT2",
+		"MMC0_DAT1",
+		"MMC0_DAT0",
+		"MMC0_CLK",
+		"MMC0_CMD";
+};
+
+&gpio3 {
+	gpio-line-names =
+		"onrisc:red:power",
+		"NC",
+		"NC",
+		"NC",
+		"3G_PWR_EN",
+		"NC",
+		"NC",
+		"WLAN_IRQ",
+		"WLAN_EN",
+		"SW2_2",
+		"SW2_3",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC",
+		"NC";
+};
diff --git a/arch/arm/boot/dts/am33xx-l4.dtsi b/arch/arm/boot/dts/am33xx-l4.dtsi
index 7da42a5b959c..7e50fe633d8a 100644
--- a/arch/arm/boot/dts/am33xx-l4.dtsi
+++ b/arch/arm/boot/dts/am33xx-l4.dtsi
@@ -1502,8 +1502,7 @@
 			mmc1: mmc@0 {
 				compatible = "ti,am335-sdhci";
 				ti,needs-special-reset;
-				dmas = <&edma_xbar 24 0 0
-					&edma_xbar 25 0 0>;
+				dmas = <&edma 24 0>, <&edma 25 0>;
 				dma-names = "tx", "rx";
 				interrupts = <64>;
 				reg = <0x0 0x1000>;
diff --git a/arch/arm/boot/dts/am5748.dtsi b/arch/arm/boot/dts/am5748.dtsi
index c260aa1a85bd..a1f029e9d1f3 100644
--- a/arch/arm/boot/dts/am5748.dtsi
+++ b/arch/arm/boot/dts/am5748.dtsi
@@ -25,6 +25,10 @@
 	status = "disabled";
 };
 
+&usb4_tm {
+	status = "disabled";
+};
+
 &atl_tm {
 	status = "disabled";
 };
diff --git a/arch/arm/boot/dts/armada-370.dtsi b/arch/arm/boot/dts/armada-370.dtsi
index 46e6d3ed8f35..9dc928859ad3 100644
--- a/arch/arm/boot/dts/armada-370.dtsi
+++ b/arch/arm/boot/dts/armada-370.dtsi
@@ -60,16 +60,26 @@
 				reg = <0x0800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 58>;
 				#interrupt-cells = <1>;
                                 ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
                                           0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 58>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie0_intc 0>,
+						<0 0 0 2 &pcie0_intc 1>,
+						<0 0 0 3 &pcie0_intc 2>,
+						<0 0 0 4 &pcie0_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 5>;
 				status = "disabled";
+
+				pcie0_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie2: pcie@2,0 {
@@ -78,16 +88,26 @@
 				reg = <0x1000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 62>;
 				#interrupt-cells = <1>;
                                 ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0
                                           0x81000000 0 0 0x81000000 0x2 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 62>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie2_intc 0>,
+						<0 0 0 2 &pcie2_intc 1>,
+						<0 0 0 3 &pcie2_intc 2>,
+						<0 0 0 4 &pcie2_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 9>;
 				status = "disabled";
+
+				pcie2_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 
diff --git a/arch/arm/boot/dts/armada-375.dtsi b/arch/arm/boot/dts/armada-375.dtsi
index 7f2f24a29e6c..929deaf312a5 100644
--- a/arch/arm/boot/dts/armada-375.dtsi
+++ b/arch/arm/boot/dts/armada-375.dtsi
@@ -568,16 +568,26 @@
 				reg = <0x0800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie0_intc 0>,
+						<0 0 0 2 &pcie0_intc 1>,
+						<0 0 0 3 &pcie0_intc 2>,
+						<0 0 0 4 &pcie0_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 5>;
 				status = "disabled";
+
+				pcie0_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie1: pcie@2,0 {
@@ -586,16 +596,26 @@
 				reg = <0x1000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0
 					  0x81000000 0 0 0x81000000 0x2 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie1_intc 0>,
+						<0 0 0 2 &pcie1_intc 1>,
+						<0 0 0 3 &pcie1_intc 2>,
+						<0 0 0 4 &pcie1_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <1>;
 				clocks = <&gateclk 6>;
 				status = "disabled";
+
+				pcie1_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 		};
diff --git a/arch/arm/boot/dts/armada-380.dtsi b/arch/arm/boot/dts/armada-380.dtsi
index cff1269f3fbf..ce1dddb2269b 100644
--- a/arch/arm/boot/dts/armada-380.dtsi
+++ b/arch/arm/boot/dts/armada-380.dtsi
@@ -64,16 +64,26 @@
 				reg = <0x0800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie1_intc 0>,
+						<0 0 0 2 &pcie1_intc 1>,
+						<0 0 0 3 &pcie1_intc 2>,
+						<0 0 0 4 &pcie1_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 8>;
 				status = "disabled";
+
+				pcie1_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			/* x1 port */
@@ -83,16 +93,26 @@
 				reg = <0x1000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0
 					  0x81000000 0 0 0x81000000 0x2 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie2_intc 0>,
+						<0 0 0 2 &pcie2_intc 1>,
+						<0 0 0 3 &pcie2_intc 2>,
+						<0 0 0 4 &pcie2_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 5>;
 				status = "disabled";
+
+				pcie2_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			/* x1 port */
@@ -102,16 +122,26 @@
 				reg = <0x1800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0
 					  0x81000000 0 0 0x81000000 0x3 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie3_intc 0>,
+						<0 0 0 2 &pcie3_intc 1>,
+						<0 0 0 3 &pcie3_intc 2>,
+						<0 0 0 4 &pcie3_intc 3>;
 				marvell,pcie-port = <2>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 6>;
 				status = "disabled";
+
+				pcie3_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/armada-385-turris-omnia.dts b/arch/arm/boot/dts/armada-385-turris-omnia.dts
index d1e0db6e5730..72ac807cae25 100644
--- a/arch/arm/boot/dts/armada-385-turris-omnia.dts
+++ b/arch/arm/boot/dts/armada-385-turris-omnia.dts
@@ -105,6 +105,33 @@
 		 */
 		status = "disabled";
 	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "SPDIF";
+		simple-audio-card,format = "i2s";
+
+		simple-audio-card,cpu {
+			sound-dai = <&audio_controller 1>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&spdif_out>;
+		};
+	};
+
+	spdif_out: spdif-out {
+		#sound-dai-cells = <0>;
+		compatible = "linux,spdif-dit";
+	};
+};
+
+&audio_controller {
+	/* Pin header U16, GPIO51 in SPDIFO mode */
+	pinctrl-0 = <&spdif_pins>;
+	pinctrl-names = "default";
+	spdif-mode;
+	status = "okay";
 };
 
 &bm {
@@ -166,6 +193,7 @@
 	buffer-manager = <&bm>;
 	bm,pool-long = <2>;
 	bm,pool-short = <3>;
+	label = "wan";
 };
 
 &i2c0 {
@@ -476,7 +504,7 @@
 		marvell,function = "spi0";
 	};
 
-	spi0cs1_pins: spi0cs1-pins {
+	spi0cs2_pins: spi0cs2-pins {
 		marvell,pins = "mpp26";
 		marvell,function = "spi0";
 	};
@@ -511,7 +539,7 @@
 		};
 	};
 
-	/* MISO, MOSI, SCLK and CS1 are routed to pin header CN11 */
+	/* MISO, MOSI, SCLK and CS2 are routed to pin header CN11 */
 };
 
 &uart0 {
diff --git a/arch/arm/boot/dts/armada-388-db.dts b/arch/arm/boot/dts/armada-388-db.dts
index 5130eccc32af..2bcec5419b66 100644
--- a/arch/arm/boot/dts/armada-388-db.dts
+++ b/arch/arm/boot/dts/armada-388-db.dts
@@ -36,6 +36,11 @@
 			i2c@11000 {
 				status = "okay";
 				clock-frequency = <100000>;
+				audio_codec: audio-codec@4a {
+					#sound-dai-cells = <0>;
+					compatible = "cirrus,cs42l51";
+					reg = <0x4a>;
+				};
 			};
 
 			i2c@11100 {
@@ -99,6 +104,12 @@
 				no-1-8-v;
 			};
 
+			audio-controller@e8000 {
+				pinctrl-0 = <&i2s_pins>;
+				pinctrl-names = "default";
+				status = "disabled";
+			};
+
 			usb3@f0000 {
 				status = "okay";
 			};
@@ -128,6 +139,64 @@
 			};
 		};
 	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "Armada 385 DB Audio";
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,widgets =
+			"Headphone", "Out Jack",
+			"Line", "In Jack";
+		simple-audio-card,routing =
+			"Out Jack", "HPL",
+			"Out Jack", "HPR",
+			"AIN1L", "In Jack",
+			"AIN1R", "In Jack";
+		status = "disabled";
+
+		simple-audio-card,dai-link@0 {
+			format = "i2s";
+			cpu {
+				sound-dai = <&audio_controller 0>;
+			};
+
+			codec {
+				sound-dai = <&audio_codec>;
+			};
+		};
+
+		simple-audio-card,dai-link@1 {
+			format = "i2s";
+			cpu {
+				sound-dai = <&audio_controller 1>;
+			};
+
+			codec {
+				sound-dai = <&spdif_out>;
+			};
+		};
+
+		simple-audio-card,dai-link@2 {
+			format = "i2s";
+			cpu {
+				sound-dai = <&audio_controller 1>;
+			};
+
+			codec {
+				sound-dai = <&spdif_in>;
+			};
+		};
+	};
+
+	spdif_out: spdif-out {
+		#sound-dai-cells = <0>;
+		compatible = "linux,spdif-dit";
+	};
+
+	spdif_in: spdif-in {
+		#sound-dai-cells = <0>;
+		compatible = "linux,spdif-dir";
+	};
 };
 
 &spi0 {
diff --git a/arch/arm/boot/dts/armada-38x.dtsi b/arch/arm/boot/dts/armada-38x.dtsi
index df3c8d1d8f64..446861b6b17b 100644
--- a/arch/arm/boot/dts/armada-38x.dtsi
+++ b/arch/arm/boot/dts/armada-38x.dtsi
@@ -289,6 +289,18 @@
 					marvell,pins = "mpp44";
 					marvell,function = "sata3";
 				};
+
+				i2s_pins: i2s-pins {
+					marvell,pins = "mpp48", "mpp49",
+						       "mpp50", "mpp51",
+						       "mpp52", "mpp53";
+					marvell,function = "audio";
+				};
+
+				spdif_pins: spdif-pins {
+					marvell,pins = "mpp51";
+					marvell,function = "audio";
+				};
 			};
 
 			gpio0: gpio@18100 {
@@ -298,6 +310,7 @@
 				reg-names = "gpio", "pwm";
 				ngpios = <32>;
 				gpio-controller;
+				gpio-ranges = <&pinctrl 0 0 32>;
 				#gpio-cells = <2>;
 				#pwm-cells = <2>;
 				interrupt-controller;
@@ -316,6 +329,7 @@
 				reg-names = "gpio", "pwm";
 				ngpios = <28>;
 				gpio-controller;
+				gpio-ranges = <&pinctrl 0 32 28>;
 				#gpio-cells = <2>;
 				#pwm-cells = <2>;
 				interrupt-controller;
@@ -618,6 +632,18 @@
 				status = "disabled";
 			};
 
+			audio_controller: audio-controller@e8000 {
+				#sound-dai-cells = <1>;
+				compatible = "marvell,armada-380-audio";
+				reg = <0xe8000 0x4000>, <0x18410 0xc>,
+				      <0x18204 0x4>;
+				reg-names = "i2s_regs", "pll_regs", "soc_ctrl";
+				interrupts = <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gateclk 0>;
+				clock-names = "internal";
+				status = "disabled";
+			};
+
 			usb3_0: usb3@f0000 {
 				compatible = "marvell,armada-380-xhci";
 				reg = <0xf0000 0x4000>,<0xf4000 0x4000>;
diff --git a/arch/arm/boot/dts/armada-39x.dtsi b/arch/arm/boot/dts/armada-39x.dtsi
index e0b7c2099831..923b035a3ab3 100644
--- a/arch/arm/boot/dts/armada-39x.dtsi
+++ b/arch/arm/boot/dts/armada-39x.dtsi
@@ -438,16 +438,26 @@
 				reg = <0x0800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &gic GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie1_intc 0>,
+						<0 0 0 2 &pcie1_intc 1>,
+						<0 0 0 3 &pcie1_intc 2>,
+						<0 0 0 4 &pcie1_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 8>;
 				status = "disabled";
+
+				pcie1_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			/* x1 port */
@@ -457,16 +467,26 @@
 				reg = <0x1000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0
 					  0x81000000 0 0 0x81000000 0x2 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &gic GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie2_intc 0>,
+						<0 0 0 2 &pcie2_intc 1>,
+						<0 0 0 3 &pcie2_intc 2>,
+						<0 0 0 4 &pcie2_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 5>;
 				status = "disabled";
+
+				pcie2_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			/* x1 port */
@@ -476,16 +496,26 @@
 				reg = <0x1800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0
 					  0x81000000 0 0 0x81000000 0x3 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &gic GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie3_intc 0>,
+						<0 0 0 2 &pcie3_intc 1>,
+						<0 0 0 3 &pcie3_intc 2>,
+						<0 0 0 4 &pcie3_intc 3>;
 				marvell,pcie-port = <2>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 6>;
 				status = "disabled";
+
+				pcie3_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			/*
@@ -498,16 +528,26 @@
 				reg = <0x2000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&gic GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x4 0 1 0
 					  0x81000000 0 0 0x81000000 0x4 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &gic GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie4_intc 0>,
+						<0 0 0 2 &pcie4_intc 1>,
+						<0 0 0 3 &pcie4_intc 2>,
+						<0 0 0 4 &pcie4_intc 3>;
 				marvell,pcie-port = <3>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 7>;
 				status = "disabled";
+
+				pcie4_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 
diff --git a/arch/arm/boot/dts/armada-xp-98dx3236.dtsi b/arch/arm/boot/dts/armada-xp-98dx3236.dtsi
index 38a052a0312d..b21ffb819b1d 100644
--- a/arch/arm/boot/dts/armada-xp-98dx3236.dtsi
+++ b/arch/arm/boot/dts/armada-xp-98dx3236.dtsi
@@ -76,16 +76,26 @@
 				reg = <0x0800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 58>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 58>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie1_intc 0>,
+						<0 0 0 2 &pcie1_intc 1>,
+						<0 0 0 3 &pcie1_intc 2>,
+						<0 0 0 4 &pcie1_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 5>;
 				status = "disabled";
+
+				pcie1_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 
diff --git a/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts b/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts
index 87dcb502f72d..0dad95ea26c2 100644
--- a/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts
+++ b/arch/arm/boot/dts/armada-xp-lenovo-ix4-300d.dts
@@ -164,7 +164,7 @@
 		};
 	};
 
-	spi3 {
+	spi-3 {
 		compatible = "spi-gpio";
 		status = "okay";
 		gpio-sck = <&gpio0 25 GPIO_ACTIVE_LOW>;
diff --git a/arch/arm/boot/dts/armada-xp-mv78230.dtsi b/arch/arm/boot/dts/armada-xp-mv78230.dtsi
index 8558bf6bb54c..bf9360f41e0a 100644
--- a/arch/arm/boot/dts/armada-xp-mv78230.dtsi
+++ b/arch/arm/boot/dts/armada-xp-mv78230.dtsi
@@ -83,16 +83,26 @@
 				reg = <0x0800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 58>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 58>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie1_intc 0>,
+						<0 0 0 2 &pcie1_intc 1>,
+						<0 0 0 3 &pcie1_intc 2>,
+						<0 0 0 4 &pcie1_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 5>;
 				status = "disabled";
+
+				pcie1_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie2: pcie@2,0 {
@@ -101,16 +111,26 @@
 				reg = <0x1000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 59>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0
 					  0x81000000 0 0 0x81000000 0x2 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 59>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie2_intc 0>,
+						<0 0 0 2 &pcie2_intc 1>,
+						<0 0 0 3 &pcie2_intc 2>,
+						<0 0 0 4 &pcie2_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <1>;
 				clocks = <&gateclk 6>;
 				status = "disabled";
+
+				pcie2_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie3: pcie@3,0 {
@@ -119,16 +139,26 @@
 				reg = <0x1800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 60>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0
 					  0x81000000 0 0 0x81000000 0x3 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 60>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie3_intc 0>,
+						<0 0 0 2 &pcie3_intc 1>,
+						<0 0 0 3 &pcie3_intc 2>,
+						<0 0 0 4 &pcie3_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <2>;
 				clocks = <&gateclk 7>;
 				status = "disabled";
+
+				pcie3_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie4: pcie@4,0 {
@@ -137,16 +167,26 @@
 				reg = <0x2000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 61>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x4 0 1 0
 					  0x81000000 0 0 0x81000000 0x4 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 61>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie4_intc 0>,
+						<0 0 0 2 &pcie4_intc 1>,
+						<0 0 0 3 &pcie4_intc 2>,
+						<0 0 0 4 &pcie4_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <3>;
 				clocks = <&gateclk 8>;
 				status = "disabled";
+
+				pcie4_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie5: pcie@5,0 {
@@ -155,16 +195,26 @@
 				reg = <0x2800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 62>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x5 0 1 0
 					  0x81000000 0 0 0x81000000 0x5 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 62>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie5_intc 0>,
+						<0 0 0 2 &pcie5_intc 1>,
+						<0 0 0 3 &pcie5_intc 2>,
+						<0 0 0 4 &pcie5_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 9>;
 				status = "disabled";
+
+				pcie5_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 
diff --git a/arch/arm/boot/dts/armada-xp-mv78260.dtsi b/arch/arm/boot/dts/armada-xp-mv78260.dtsi
index 2d85fe8ac327..0714af52e607 100644
--- a/arch/arm/boot/dts/armada-xp-mv78260.dtsi
+++ b/arch/arm/boot/dts/armada-xp-mv78260.dtsi
@@ -98,16 +98,26 @@
 				reg = <0x0800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 58>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 58>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie1_intc 0>,
+						<0 0 0 2 &pcie1_intc 1>,
+						<0 0 0 3 &pcie1_intc 2>,
+						<0 0 0 4 &pcie1_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 5>;
 				status = "disabled";
+
+				pcie1_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie2: pcie@2,0 {
@@ -116,16 +126,26 @@
 				reg = <0x1000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 59>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0
 					  0x81000000 0 0 0x81000000 0x2 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 59>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie2_intc 0>,
+						<0 0 0 2 &pcie2_intc 1>,
+						<0 0 0 3 &pcie2_intc 2>,
+						<0 0 0 4 &pcie2_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <1>;
 				clocks = <&gateclk 6>;
 				status = "disabled";
+
+				pcie2_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie3: pcie@3,0 {
@@ -134,16 +154,26 @@
 				reg = <0x1800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 60>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0
 					  0x81000000 0 0 0x81000000 0x3 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 60>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie3_intc 0>,
+						<0 0 0 2 &pcie3_intc 1>,
+						<0 0 0 3 &pcie3_intc 2>,
+						<0 0 0 4 &pcie3_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <2>;
 				clocks = <&gateclk 7>;
 				status = "disabled";
+
+				pcie3_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie4: pcie@4,0 {
@@ -152,16 +182,26 @@
 				reg = <0x2000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 61>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x4 0 1 0
 					  0x81000000 0 0 0x81000000 0x4 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 61>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie4_intc 0>,
+						<0 0 0 2 &pcie4_intc 1>,
+						<0 0 0 3 &pcie4_intc 2>,
+						<0 0 0 4 &pcie4_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <3>;
 				clocks = <&gateclk 8>;
 				status = "disabled";
+
+				pcie4_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie5: pcie@5,0 {
@@ -170,16 +210,26 @@
 				reg = <0x2800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 62>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x5 0 1 0
 					  0x81000000 0 0 0x81000000 0x5 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 62>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie5_intc 0>,
+						<0 0 0 2 &pcie5_intc 1>,
+						<0 0 0 3 &pcie5_intc 2>,
+						<0 0 0 4 &pcie5_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 9>;
 				status = "disabled";
+
+				pcie5_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie6: pcie@6,0 {
@@ -188,16 +238,26 @@
 				reg = <0x3000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 63>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x6 0 1 0
 					  0x81000000 0 0 0x81000000 0x6 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 63>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie6_intc 0>,
+						<0 0 0 2 &pcie6_intc 1>,
+						<0 0 0 3 &pcie6_intc 2>,
+						<0 0 0 4 &pcie6_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <1>;
 				clocks = <&gateclk 10>;
 				status = "disabled";
+
+				pcie6_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie7: pcie@7,0 {
@@ -206,16 +266,26 @@
 				reg = <0x3800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 64>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x7 0 1 0
 					  0x81000000 0 0 0x81000000 0x7 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 64>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie7_intc 0>,
+						<0 0 0 2 &pcie7_intc 1>,
+						<0 0 0 3 &pcie7_intc 2>,
+						<0 0 0 4 &pcie7_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <2>;
 				clocks = <&gateclk 11>;
 				status = "disabled";
+
+				pcie7_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie8: pcie@8,0 {
@@ -224,16 +294,26 @@
 				reg = <0x4000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 65>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x8 0 1 0
 					  0x81000000 0 0 0x81000000 0x8 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 65>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie8_intc 0>,
+						<0 0 0 2 &pcie8_intc 1>,
+						<0 0 0 3 &pcie8_intc 2>,
+						<0 0 0 4 &pcie8_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <3>;
 				clocks = <&gateclk 12>;
 				status = "disabled";
+
+				pcie8_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie9: pcie@9,0 {
@@ -242,16 +322,26 @@
 				reg = <0x4800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 99>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x9 0 1 0
 					  0x81000000 0 0 0x81000000 0x9 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 99>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie9_intc 0>,
+						<0 0 0 2 &pcie9_intc 1>,
+						<0 0 0 3 &pcie9_intc 2>,
+						<0 0 0 4 &pcie9_intc 3>;
 				marvell,pcie-port = <2>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 26>;
 				status = "disabled";
+
+				pcie9_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 
diff --git a/arch/arm/boot/dts/armada-xp-mv78460.dtsi b/arch/arm/boot/dts/armada-xp-mv78460.dtsi
index 230a3fd36b30..16185edf9aa5 100644
--- a/arch/arm/boot/dts/armada-xp-mv78460.dtsi
+++ b/arch/arm/boot/dts/armada-xp-mv78460.dtsi
@@ -119,16 +119,26 @@
 				reg = <0x0800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 58>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 58>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie1_intc 0>,
+						<0 0 0 2 &pcie1_intc 1>,
+						<0 0 0 3 &pcie1_intc 2>,
+						<0 0 0 4 &pcie1_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 5>;
 				status = "disabled";
+
+				pcie1_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie2: pcie@2,0 {
@@ -137,16 +147,26 @@
 				reg = <0x1000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 59>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0
 					  0x81000000 0 0 0x81000000 0x2 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 59>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie2_intc 0>,
+						<0 0 0 2 &pcie2_intc 1>,
+						<0 0 0 3 &pcie2_intc 2>,
+						<0 0 0 4 &pcie2_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <1>;
 				clocks = <&gateclk 6>;
 				status = "disabled";
+
+				pcie2_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie3: pcie@3,0 {
@@ -155,16 +175,26 @@
 				reg = <0x1800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 60>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x3 0 1 0
 					  0x81000000 0 0 0x81000000 0x3 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 60>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie3_intc 0>,
+						<0 0 0 2 &pcie3_intc 1>,
+						<0 0 0 3 &pcie3_intc 2>,
+						<0 0 0 4 &pcie3_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <2>;
 				clocks = <&gateclk 7>;
 				status = "disabled";
+
+				pcie3_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie4: pcie@4,0 {
@@ -173,16 +203,26 @@
 				reg = <0x2000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 61>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x4 0 1 0
 					  0x81000000 0 0 0x81000000 0x4 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 61>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie4_intc 0>,
+						<0 0 0 2 &pcie4_intc 1>,
+						<0 0 0 3 &pcie4_intc 2>,
+						<0 0 0 4 &pcie4_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <3>;
 				clocks = <&gateclk 8>;
 				status = "disabled";
+
+				pcie4_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie5: pcie@5,0 {
@@ -191,16 +231,26 @@
 				reg = <0x2800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 62>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x5 0 1 0
 					  0x81000000 0 0 0x81000000 0x5 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 62>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie5_intc 0>,
+						<0 0 0 2 &pcie5_intc 1>,
+						<0 0 0 3 &pcie5_intc 2>,
+						<0 0 0 4 &pcie5_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 9>;
 				status = "disabled";
+
+				pcie5_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie6: pcie@6,0 {
@@ -209,16 +259,26 @@
 				reg = <0x3000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 63>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x6 0 1 0
 					  0x81000000 0 0 0x81000000 0x6 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 63>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie6_intc 0>,
+						<0 0 0 2 &pcie6_intc 1>,
+						<0 0 0 3 &pcie6_intc 2>,
+						<0 0 0 4 &pcie6_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <1>;
 				clocks = <&gateclk 10>;
 				status = "disabled";
+
+				pcie6_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie7: pcie@7,0 {
@@ -227,16 +287,26 @@
 				reg = <0x3800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 64>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x7 0 1 0
 					  0x81000000 0 0 0x81000000 0x7 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 64>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie7_intc 0>,
+						<0 0 0 2 &pcie7_intc 1>,
+						<0 0 0 3 &pcie7_intc 2>,
+						<0 0 0 4 &pcie7_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <2>;
 				clocks = <&gateclk 11>;
 				status = "disabled";
+
+				pcie7_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie8: pcie@8,0 {
@@ -245,16 +315,26 @@
 				reg = <0x4000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 65>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x8 0 1 0
 					  0x81000000 0 0 0x81000000 0x8 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 65>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie8_intc 0>,
+						<0 0 0 2 &pcie8_intc 1>,
+						<0 0 0 3 &pcie8_intc 2>,
+						<0 0 0 4 &pcie8_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <3>;
 				clocks = <&gateclk 12>;
 				status = "disabled";
+
+				pcie8_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie9: pcie@9,0 {
@@ -263,16 +343,26 @@
 				reg = <0x4800 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 99>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0x9 0 1 0
 					  0x81000000 0 0 0x81000000 0x9 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 99>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie9_intc 0>,
+						<0 0 0 2 &pcie9_intc 1>,
+						<0 0 0 3 &pcie9_intc 2>,
+						<0 0 0 4 &pcie9_intc 3>;
 				marvell,pcie-port = <2>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 26>;
 				status = "disabled";
+
+				pcie9_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie10: pcie@a,0 {
@@ -281,16 +371,26 @@
 				reg = <0x5000 0 0 0 0>;
 				#address-cells = <3>;
 				#size-cells = <2>;
+				interrupt-names = "intx";
+				interrupts-extended = <&mpic 103>;
 				#interrupt-cells = <1>;
 				ranges = <0x82000000 0 0 0x82000000 0xa 0 1 0
 					  0x81000000 0 0 0x81000000 0xa 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &mpic 103>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie10_intc 0>,
+						<0 0 0 2 &pcie10_intc 1>,
+						<0 0 0 3 &pcie10_intc 2>,
+						<0 0 0 4 &pcie10_intc 3>;
 				marvell,pcie-port = <3>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gateclk 27>;
 				status = "disabled";
+
+				pcie10_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 
diff --git a/arch/arm/boot/dts/aspeed-ast2600-evb-a1.dts b/arch/arm/boot/dts/aspeed-ast2600-evb-a1.dts
index d0a5c2ff0fec..f34a2b1ec2f0 100644
--- a/arch/arm/boot/dts/aspeed-ast2600-evb-a1.dts
+++ b/arch/arm/boot/dts/aspeed-ast2600-evb-a1.dts
@@ -5,7 +5,7 @@
 
 / {
 	model = "AST2600 A1 EVB";
-	compatible = "aspeed,ast2600-evb-a1", "aspeed,ast2600";
+	compatible = "aspeed,ast2600-evb-a1", "aspeed,ast2600-evb", "aspeed,ast2600";
 
 	/delete-node/regulator-vcc-sdhci0;
 	/delete-node/regulator-vcc-sdhci1;
diff --git a/arch/arm/boot/dts/aspeed-ast2600-evb.dts b/arch/arm/boot/dts/aspeed-ast2600-evb.dts
index c698e6538269..de83c0eb1d6e 100644
--- a/arch/arm/boot/dts/aspeed-ast2600-evb.dts
+++ b/arch/arm/boot/dts/aspeed-ast2600-evb.dts
@@ -8,7 +8,7 @@
 
 / {
 	model = "AST2600 EVB";
-	compatible = "aspeed,ast2600-evb-a1", "aspeed,ast2600";
+	compatible = "aspeed,ast2600-evb", "aspeed,ast2600";
 
 	aliases {
 		serial4 = &uart5;
@@ -182,6 +182,7 @@
 		status = "okay";
 		m25p,fast-read;
 		label = "bmc";
+		spi-rx-bus-width = <4>;
 		spi-max-frequency = <50000000>;
 #include "openbmc-flash-layout-64.dtsi"
 	};
@@ -196,6 +197,7 @@
 		status = "okay";
 		m25p,fast-read;
 		label = "pnor";
+		spi-rx-bus-width = <4>;
 		spi-max-frequency = <100000000>;
 	};
 };
@@ -207,11 +209,6 @@
 
 &i2c0 {
 	status = "okay";
-
-	temp@2e {
-		compatible = "adi,adt7490";
-		reg = <0x2e>;
-	};
 };
 
 &i2c1 {
@@ -240,10 +237,26 @@
 
 &i2c7 {
 	status = "okay";
+
+	temp@2e {
+		compatible = "adi,adt7490";
+		reg = <0x2e>;
+	};
+
+	eeprom@50 {
+		compatible = "atmel,24c08";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
 };
 
 &i2c8 {
 	status = "okay";
+
+	lm75@4d {
+		compatible = "national,lm75";
+		reg = <0x4d>;
+	};
 };
 
 &i2c9 {
diff --git a/arch/arm/boot/dts/aspeed-bmc-amd-daytonax.dts b/arch/arm/boot/dts/aspeed-bmc-amd-daytonax.dts
new file mode 100644
index 000000000000..64bb9bf92de2
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed-bmc-amd-daytonax.dts
@@ -0,0 +1,319 @@
+// SPDX-License-Identifier: GPL-2.0
+/dts-v1/;
+
+#include "aspeed-g5.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+
+/ {
+	model = "AMD DaytonaX BMC";
+	compatible = "amd,daytonax-bmc", "aspeed,ast2500";
+
+	memory@80000000 {
+		reg = <0x80000000 0x20000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		video_engine_memory: jpegbuffer {
+			size = <0x02000000>;	/* 32M */
+			alignment = <0x01000000>;
+			compatible = "shared-dma-pool";
+			reusable;
+		};
+	};
+
+	aliases {
+		serial0 = &uart1;
+		serial4 = &uart5;
+	};
+
+	chosen {
+		stdout-path = &uart5;
+		bootargs = "console=ttyS4,115200";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-fault {
+			gpios = <&gpio ASPEED_GPIO(A, 2) GPIO_ACTIVE_LOW>;
+		};
+
+		led-identify {
+			gpios = <&gpio ASPEED_GPIO(A, 3) GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	iio-hwmon {
+		compatible = "iio-hwmon";
+		io-channels = <&adc 0>, <&adc 1>, <&adc 2>, <&adc 3>, <&adc 4>,
+			<&adc 5>, <&adc 6>, <&adc 7>, <&adc 8>, <&adc 9>,
+			<&adc 10>, <&adc 11>, <&adc 12>, <&adc 13>, <&adc 14>,
+			<&adc 15>;
+	};
+};
+
+&fmc {
+	status = "okay";
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "bmc";
+		#include "openbmc-flash-layout.dtsi"
+	};
+};
+
+&mac0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rgmii1_default &pinctrl_mdio1_default>;
+};
+
+&uart1 {
+	//Host Console
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_txd1_default
+		&pinctrl_rxd1_default
+		&pinctrl_nrts1_default
+		&pinctrl_ndtr1_default
+		&pinctrl_ndsr1_default
+		&pinctrl_ncts1_default
+		&pinctrl_ndcd1_default
+		&pinctrl_nri1_default>;
+};
+
+&uart5 {
+	//BMC Console
+	status = "okay";
+};
+
+&vuart {
+	status = "okay";
+	aspeed,lpc-io-reg = <0x3f8>;
+	aspeed,lpc-interrupts = <4 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&adc {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_adc0_default
+		&pinctrl_adc1_default
+		&pinctrl_adc2_default
+		&pinctrl_adc3_default
+		&pinctrl_adc4_default
+		&pinctrl_adc5_default
+		&pinctrl_adc6_default
+		&pinctrl_adc7_default
+		&pinctrl_adc8_default
+		&pinctrl_adc9_default
+		&pinctrl_adc10_default
+		&pinctrl_adc11_default
+		&pinctrl_adc12_default
+		&pinctrl_adc13_default
+		&pinctrl_adc14_default
+		&pinctrl_adc15_default>;
+};
+
+&gpio {
+	status = "okay";
+	gpio-line-names =
+	/*A0-A7*/	"","","led-fault","led-identify","","","","",
+	/*B0-B7*/	"","","","","","","","",
+	/*C0-C7*/	"id-button","","","","","","","",
+	/*D0-D7*/	"","","ASSERT_BMC_READY","","","","","",
+	/*E0-E7*/	"reset-button","reset-control","power-button","power-control","",
+			"power-good","power-ok","",
+	/*F0-F7*/	"","","","","","","BATTERY_DETECT","",
+	/*G0-G7*/	"","","","","","","","",
+	/*H0-H7*/	"","","","","","","","",
+	/*I0-I7*/	"","","","","","","","",
+	/*J0-J7*/	"","","","","","","","",
+	/*K0-K7*/	"","","","","","","","",
+	/*L0-L7*/	"","","","","","","","",
+	/*M0-M7*/	"","","","","","","","",
+	/*N0-N7*/	"","","","","","","","",
+	/*O0-O7*/	"","","","","","","","",
+	/*P0-P7*/	"","","","","","","","",
+	/*Q0-Q7*/	"","","","","","","","",
+	/*R0-R7*/	"","","","","","","","",
+	/*S0-S7*/	"","","","","","","","",
+	/*T0-T7*/	"","","","","","","","",
+	/*U0-U7*/	"","","","","","","","",
+	/*V0-V7*/	"","","","","","","","",
+	/*W0-W7*/	"","","","","","","","",
+	/*X0-X7*/	"","","","","","","","",
+	/*Y0-Y7*/	"","","","","","","","",
+	/*Z0-Z7*/	"","","","","","","","",
+	/*AA0-AA7*/	"","","","","","","","",
+	/*AB0-AB7*/	"FM_BMC_READ_SPD_TEMP","","","","","","","",
+	/*AC0-AC7*/	"","","","","","","","";
+};
+
+&i2c0 {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&i2c2 {
+	status = "okay";
+};
+
+&i2c3 {
+	status = "okay";
+};
+
+&i2c4 {
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+};
+
+&i2c6 {
+	status = "okay";
+};
+
+&i2c7 {
+	status = "okay";
+};
+
+&i2c8 {
+	status = "okay";
+};
+
+&i2c10 {
+	status = "okay";
+};
+
+&i2c11 {
+	status = "okay";
+};
+
+&i2c12 {
+	status = "okay";
+};
+
+&kcs3 {
+	status = "okay";
+	aspeed,lpc-io-reg = <0xca2>;
+};
+
+&lpc_snoop {
+	status = "okay";
+	snoop-ports = <0x80>, <0x81>;
+};
+
+&lpc_ctrl {
+	status = "okay";
+};
+
+&pwm_tacho {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm0_default
+		&pinctrl_pwm1_default
+		&pinctrl_pwm2_default
+		&pinctrl_pwm3_default
+		&pinctrl_pwm4_default
+		&pinctrl_pwm5_default
+		&pinctrl_pwm6_default
+		&pinctrl_pwm7_default>;
+
+	fan@0 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x00>;
+	};
+
+	fan@1 {
+		reg = <0x00>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x01>;
+	};
+
+	fan@2 {
+		reg = <0x01>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x02>;
+	};
+
+	fan@3 {
+		reg = <0x01>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x03>;
+	};
+
+	fan@4 {
+		reg = <0x02>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x04>;
+	};
+
+	fan@5 {
+		reg = <0x02>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x05>;
+	};
+
+	fan@6 {
+		reg = <0x03>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x06>;
+	};
+
+	fan@7 {
+		reg = <0x03>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x07>;
+	};
+
+	fan@8 {
+		reg = <0x04>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x08>;
+	};
+
+	fan@9 {
+		reg = <0x04>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x09>;
+	};
+
+	fan@10 {
+		reg = <0x05>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0a>;
+	};
+
+	fan@11 {
+		reg = <0x05>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0b>;
+	};
+
+	fan@12 {
+		reg = <0x06>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0c>;
+	};
+
+	fan@13 {
+		reg = <0x06>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0d>;
+	};
+
+	fan@14 {
+		reg = <0x07>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0e>;
+	};
+
+	fan@15 {
+		reg = <0x07>;
+		aspeed,fan-tach-ch = /bits/ 8 <0x0f>;
+	};
+};
+
+&video {
+	status = "okay";
+	memory-region = <&video_engine_memory>;
+};
+
+&vhub {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/aspeed-bmc-ampere-mtjade.dts b/arch/arm/boot/dts/aspeed-bmc-ampere-mtjade.dts
index 82a6f14a45f0..d127cbcc7998 100644
--- a/arch/arm/boot/dts/aspeed-bmc-ampere-mtjade.dts
+++ b/arch/arm/boot/dts/aspeed-bmc-ampere-mtjade.dts
@@ -97,101 +97,6 @@
 		};
 	};
 
-	gpio-keys {
-		compatible = "gpio-keys";
-
-		event-shutdown-ack {
-			label = "SHUTDOWN_ACK";
-			gpios = <&gpio ASPEED_GPIO(G, 2) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(G, 2)>;
-		};
-
-		event-reboot-ack {
-			label = "REBOOT_ACK";
-			gpios = <&gpio ASPEED_GPIO(J, 3) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(J, 3)>;
-		};
-
-		event-s0-overtemp {
-			label = "S0_OVERTEMP";
-			gpios = <&gpio ASPEED_GPIO(G, 3) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(G, 3)>;
-		};
-
-		event-s0-hightemp {
-			label = "S0_HIGHTEMP";
-			gpios = <&gpio ASPEED_GPIO(J, 0) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(J, 0)>;
-		};
-
-		event-s0-cpu-fault {
-			label = "S0_CPU_FAULT";
-			gpios = <&gpio ASPEED_GPIO(J, 1) GPIO_ACTIVE_HIGH>;
-			linux,code = <ASPEED_GPIO(J, 1)>;
-		};
-
-		event-s0-scp-auth-fail {
-			label = "S0_SCP_AUTH_FAIL";
-			gpios = <&gpio ASPEED_GPIO(J, 2) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(J, 2)>;
-		};
-
-		event-s1-scp-auth-fail {
-			label = "S1_SCP_AUTH_FAIL";
-			gpios = <&gpio ASPEED_GPIO(Z, 5) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(Z, 5)>;
-		};
-
-		event-s1-overtemp {
-			label = "S1_OVERTEMP";
-			gpios = <&gpio ASPEED_GPIO(Z, 6) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(Z, 6)>;
-		};
-
-		event-s1-hightemp {
-			label = "S1_HIGHTEMP";
-			gpios = <&gpio ASPEED_GPIO(AB, 0) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(AB, 0)>;
-		};
-
-		event-s1-cpu-fault {
-			label = "S1_CPU_FAULT";
-			gpios = <&gpio ASPEED_GPIO(Z, 1) GPIO_ACTIVE_HIGH>;
-			linux,code = <ASPEED_GPIO(Z, 1)>;
-		};
-
-		event-id {
-			label = "ID_BUTTON";
-			gpios = <&gpio ASPEED_GPIO(Q, 5) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(Q, 5)>;
-		};
-
-		event-psu1-vin-good {
-			label = "PSU1_VIN_GOOD";
-			gpios = <&gpio ASPEED_GPIO(H, 4) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(H, 4)>;
-		};
-
-		event-psu2-vin-good {
-			label = "PSU2_VIN_GOOD";
-			gpios = <&gpio ASPEED_GPIO(H, 5) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(H, 5)>;
-		};
-
-		event-psu1-present {
-			label = "PSU1_PRESENT";
-			gpios = <&gpio ASPEED_GPIO(I, 0) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(I, 0)>;
-		};
-
-		event-psu2-present {
-			label = "PSU2_PRESENT";
-			gpios = <&gpio ASPEED_GPIO(I, 1) GPIO_ACTIVE_LOW>;
-			linux,code = <ASPEED_GPIO(I, 1)>;
-		};
-
-	};
-
 	gpioA0mux: mux-controller {
 		compatible = "gpio-mux";
 		#mux-control-cells = <0>;
diff --git a/arch/arm/boot/dts/aspeed-bmc-ampere-mtmitchell.dts b/arch/arm/boot/dts/aspeed-bmc-ampere-mtmitchell.dts
new file mode 100644
index 000000000000..606cd4be245a
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed-bmc-ampere-mtmitchell.dts
@@ -0,0 +1,546 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// Copyright (c) 2022, Ampere Computing LLC
+
+/dts-v1/;
+
+#include "aspeed-g6.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+
+/ {
+	model = "Ampere Mt.Mitchell BMC";
+	compatible = "ampere,mtmitchell-bmc", "aspeed,ast2600";
+
+	chosen {
+		stdout-path = &uart5;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x80000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		gfx_memory: framebuffer {
+			size = <0x01000000>;
+			alignment = <0x01000000>;
+			compatible = "shared-dma-pool";
+			reusable;
+		};
+
+		video_engine_memory: video {
+			size = <0x04000000>;
+			alignment = <0x01000000>;
+			compatible = "shared-dma-pool";
+			reusable;
+		};
+
+		vga_memory: region@bf000000 {
+			no-map;
+			compatible = "shared-dma-pool";
+			reg = <0xbf000000 0x01000000>;  /* 16M */
+		};
+	};
+
+	voltage_mon_reg: voltage-mon-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "ltc2497_reg";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	gpioI5mux: mux-controller {
+		compatible = "gpio-mux";
+		#mux-control-cells = <0>;
+		mux-gpios = <&gpio0 ASPEED_GPIO(I, 5) GPIO_ACTIVE_HIGH>;
+	};
+
+	adc0mux: adc0mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc0 0>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc1mux: adc1mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc0 1>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc2mux: adc2mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc0 2>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc3mux: adc3mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc0 3>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc4mux: adc4mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc0 4>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc5mux: adc5mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc0 5>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc6mux: adc6mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc0 6>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc7mux: adc7mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc0 7>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc8mux: adc8mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc1 0>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc9mux: adc9mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc1 1>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc10mux: adc10mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc1 2>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc11mux: adc11mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc1 3>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc12mux: adc12mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc1 4>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc13mux: adc13mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc1 5>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc14mux: adc14mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc1 6>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	adc15mux: adc15mux {
+		compatible = "io-channel-mux";
+		io-channels = <&adc1 7>;
+		#io-channel-cells = <1>;
+		io-channel-names = "parent";
+		mux-controls = <&gpioI5mux>;
+		channels = "s0", "s1";
+	};
+
+	iio-hwmon {
+		compatible = "iio-hwmon";
+		io-channels = <&adc0mux 0>, <&adc0mux 1>,
+			<&adc1mux 0>, <&adc1mux 1>,
+			<&adc2mux 0>, <&adc2mux 1>,
+			<&adc3mux 0>, <&adc3mux 1>,
+			<&adc4mux 0>, <&adc4mux 1>,
+			<&adc5mux 0>, <&adc5mux 1>,
+			<&adc6mux 0>, <&adc6mux 1>,
+			<&adc7mux 0>, <&adc7mux 1>,
+			<&adc8mux 0>, <&adc8mux 1>,
+			<&adc9mux 0>, <&adc9mux 1>,
+			<&adc10mux 0>, <&adc10mux 1>,
+			<&adc11mux 0>, <&adc11mux 1>,
+			<&adc12mux 0>, <&adc12mux 1>,
+			<&adc13mux 0>, <&adc13mux 1>,
+			<&adc14mux 0>, <&adc14mux 1>,
+			<&adc15mux 0>, <&adc15mux 1>,
+			<&adc_i2c 0>, <&adc_i2c 1>,
+			<&adc_i2c 2>, <&adc_i2c 3>,
+			<&adc_i2c 4>, <&adc_i2c 5>,
+			<&adc_i2c 6>, <&adc_i2c 7>,
+			<&adc_i2c 8>, <&adc_i2c 9>,
+			<&adc_i2c 10>, <&adc_i2c 11>,
+			<&adc_i2c 12>, <&adc_i2c 13>,
+			<&adc_i2c 14>, <&adc_i2c 15>;
+	};
+};
+
+&mdio0 {
+	status = "okay";
+
+	ethphy0: ethernet-phy@0 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0>;
+	};
+};
+
+&mac0 {
+	status = "okay";
+
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy0>;
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rgmii1_default>;
+};
+
+&fmc {
+	status = "okay";
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "bmc";
+		spi-max-frequency = <50000000>;
+#include "openbmc-flash-layout-64.dtsi"
+	};
+
+	flash@1 {
+		status = "okay";
+		m25p,fast-read;
+		label = "alt-bmc";
+		spi-max-frequency = <50000000>;
+#include "openbmc-flash-layout-64-alt.dtsi"
+	};
+};
+
+&spi1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_spi1_default>;
+
+	flash@0 {
+		status = "okay";
+		m25p,fast-read;
+		label = "pnor";
+		spi-max-frequency = <20000000>;
+	};
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&uart3 {
+	status = "okay";
+};
+
+&uart4 {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	temperature-sensor@2e {
+		compatible = "adi,adt7490";
+		reg = <0x2e>;
+	};
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&i2c2 {
+	status = "okay";
+
+	psu@58 {
+		compatible = "pmbus";
+		reg = <0x58>;
+	};
+
+	psu@59 {
+		compatible = "pmbus";
+		reg = <0x59>;
+	};
+};
+
+&i2c3 {
+	status = "okay";
+};
+
+&i2c4 {
+	status = "okay";
+
+	adc_i2c: adc@16 {
+		compatible = "lltc,ltc2497";
+		reg = <0x16>;
+		vref-supply = <&voltage_mon_reg>;
+		#io-channel-cells = <1>;
+	 };
+
+	eeprom@50 {
+		compatible = "atmel,24c64";
+		reg = <0x50>;
+		pagesize = <32>;
+	};
+
+	i2c-mux@70 {
+		compatible = "nxp,pca9545";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x70>;
+		i2c-mux-idle-disconnect;
+
+		i2c4_bus70_chn0: i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0>;
+
+			outlet_temp1: temperature-sensor@48 {
+				compatible = "ti,tmp75";
+				reg = <0x48>;
+			};
+			psu1_inlet_temp2: temperature-sensor@49 {
+				compatible = "ti,tmp75";
+				reg = <0x49>;
+			};
+		};
+
+		i2c4_bus70_chn1: i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x1>;
+
+			pcie_zone_temp1: temperature-sensor@48 {
+				compatible = "ti,tmp75";
+				reg = <0x48>;
+			};
+			psu0_inlet_temp2: temperature-sensor@49 {
+				compatible = "ti,tmp75";
+				reg = <0x49>;
+			};
+		};
+
+		i2c4_bus70_chn2: i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x2>;
+
+			pcie_zone_temp2: temperature-sensor@48 {
+				compatible = "ti,tmp75";
+				reg = <0x48>;
+			};
+			outlet_temp2: temperature-sensor@49 {
+				compatible = "ti,tmp75";
+				reg = <0x49>;
+			};
+		};
+
+		i2c4_bus70_chn3: i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x3>;
+
+			mb_inlet_temp1: temperature-sensor@7c {
+				compatible = "microchip,emc1413";
+				reg = <0x7c>;
+			};
+			mb_inlet_temp2: temperature-sensor@4c {
+				compatible = "microchip,emc1413";
+				reg = <0x4c>;
+			};
+		};
+	};
+};
+
+&i2c5 {
+	status = "okay";
+
+	i2c-mux@70 {
+		compatible = "nxp,pca9548";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x70>;
+		i2c-mux-idle-disconnect;
+	};
+};
+
+&i2c6 {
+	status = "okay";
+	rtc@51 {
+		compatible = "nxp,pcf85063a";
+		reg = <0x51>;
+	};
+};
+
+&i2c7 {
+	status = "okay";
+};
+
+&i2c9 {
+	status = "okay";
+};
+
+&i2c11 {
+	status = "okay";
+};
+
+&i2c14 {
+	status = "okay";
+	eeprom@50 {
+		compatible = "atmel,24c64";
+		reg = <0x50>;
+		pagesize = <32>;
+	};
+
+	bmc_ast2600_cpu: temperature-sensor@35 {
+		compatible = "ti,tmp175";
+		reg = <0x35>;
+	};
+};
+
+&adc0 {
+	ref_voltage = <2500>;
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_adc0_default &pinctrl_adc1_default
+		&pinctrl_adc2_default &pinctrl_adc3_default
+		&pinctrl_adc4_default &pinctrl_adc5_default
+		&pinctrl_adc6_default &pinctrl_adc7_default>;
+};
+
+&adc1 {
+	ref_voltage = <2500>;
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_adc8_default &pinctrl_adc9_default
+		&pinctrl_adc10_default &pinctrl_adc11_default
+		&pinctrl_adc12_default &pinctrl_adc13_default
+		&pinctrl_adc14_default &pinctrl_adc15_default>;
+};
+
+&vhub {
+	status = "okay";
+};
+
+&video {
+	status = "okay";
+	memory-region = <&video_engine_memory>;
+};
+
+&gpio0 {
+	gpio-line-names =
+	/*A0-A7*/	"","","","","","i2c2-reset-n","i2c6-reset-n","i2c4-reset-n",
+	/*B0-B7*/	"","","","","host0-sysreset-n","host0-pmin-n","","",
+	/*C0-C7*/	"s0-vrd-fault-n","s1-vrd-fault-n","","",
+			"irq-n","","vrd-sel","spd-sel",
+	/*D0-D7*/	"presence-ps0","presence-ps1","hsc-12vmain-alt2-n","ext-high-temp-n",
+			"","bmc-ncsi-txen","","",
+	/*E0-E7*/	"","","clk50m-bmc-ncsi","","","","","",
+	/*F0-F7*/	"s0-pcp-oc-warn-n","s1-pcp-oc-warn-n","power-chassis-control",
+			"cpu-bios-recover","s0-heartbeat","hs-csout-prochot",
+			"s0-vr-hot-n","s1-vr-hot-n",
+	/*G0-G7*/	"","","hsc-12vmain-alt1-n","","","","","",
+	/*H0-H7*/	"","","wd-disable-n","power-chassis-good","","","","",
+	/*I0-I7*/	"","","","","","adc-sw","power-button","rtc-battery-voltage-read-enable",
+	/*J0-J7*/	"","","","","","","","",
+	/*K0-K7*/	"","","","","","","","",
+	/*L0-L7*/	"","","","","","","","",
+	/*M0-M7*/	"","s0-ddr-save","soc-spi-nor-access","presence-cpu0",
+			"s0-rtc-lock","","","",
+	/*N0-N7*/	"hpm-fw-recovery","hpm-stby-rst-n","jtag-sel-s0","led-sw-hb",
+			"jtag-dbgr-prsnt-n","s1-heartbeat","","",
+	/*O0-O7*/	"","","","","","","","",
+	/*P0-P7*/	"ps0-ac-loss-n","ps1-ac-loss-n","","",
+			"led-fault","cpld-user-mode","jtag-srst-n","led-bmc-hb",
+	/*Q0-Q7*/	"","","","","","","","",
+	/*R0-R7*/	"","","","","","","","",
+	/*S0-S7*/	"","","identify-button","led-identify",
+			"s1-ddr-save","spi-nor-access","sys-pgood","presence-cpu1",
+	/*T0-T7*/	"","","","","","","","",
+	/*U0-U7*/	"","","","","","","","",
+	/*V0-V7*/	"s0-hightemp-n","s0-fault-alert","s0-sys-auth-failure-n",
+			"host0-reboot-ack-n","host0-ready","host0-shd-req-n",
+			"host0-shd-ack-n","s0-overtemp-n",
+	/*W0-W7*/	"ocp-aux-pwren","ocp-main-pwren","ocp-pgood","",
+			"bmc-ok","bmc-ready","spi0-program-sel","spi0-backup-sel",
+	/*X0-X7*/	"i2c-backup-sel","s1-fault-alert","s1-fw-boot-ok",
+			"s1-hightemp-n","s0-spi-auth-fail-n","s1-sys-auth-failure-n",
+			"s1-overtemp-n","s1-spi-auth-fail-n",
+	/*Y0-Y7*/	"","","","","","","","host0-special-boot",
+	/*Z0-Z7*/	"reset-button","ps0-pgood","ps1-pgood","","","","","";
+};
+
+&gpio1 {
+	gpio-line-names =
+	/*18A0-18A7*/	"","","","","","","","",
+	/*18B0-18B7*/	"","","","","","","s0-soc-pgood","",
+	/*18C0-18C7*/	"uart1-mode0","uart1-mode1","uart2-mode0","uart2-mode1",
+			"uart3-mode0","uart3-mode1","uart4-mode0","uart4-mode1",
+	/*18D0-18D7*/	"","","","","","","","",
+	/*18E0-18E3*/	"","","","";
+};
diff --git a/arch/arm/boot/dts/aspeed-bmc-facebook-bletchley.dts b/arch/arm/boot/dts/aspeed-bmc-facebook-bletchley.dts
index 41d2b1535d9a..1fc3e7cbf0d1 100644
--- a/arch/arm/boot/dts/aspeed-bmc-facebook-bletchley.dts
+++ b/arch/arm/boot/dts/aspeed-bmc-facebook-bletchley.dts
@@ -7,6 +7,7 @@
 #include <dt-bindings/usb/pd.h>
 #include <dt-bindings/leds/leds-pca955x.h>
 #include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/i2c/i2c.h>
 
 / {
 	model = "Facebook Bletchley BMC";
@@ -792,11 +793,6 @@
 		reg = <0x4f>;
 	};
 
-	hdc1080@40 {
-		compatible = "ti,hdc1080";
-		reg = <0x40>;
-	};
-
 	front_leds: pca9552@67 {
 		compatible = "nxp,pca9552";
 		reg = <0x67>;
@@ -857,6 +853,13 @@
 	multi-master;
 	aspeed,hw-timeout-ms = <1000>;
 	status = "okay";
+
+	//USB Debug Connector
+	ipmb13@10 {
+		compatible = "ipmb-dev";
+		reg = <(0x10 | I2C_OWN_SLAVE_ADDRESS)>;
+		i2c-protocol;
+	};
 };
 
 &gpio0 {
diff --git a/arch/arm/boot/dts/aspeed-bmc-facebook-elbert.dts b/arch/arm/boot/dts/aspeed-bmc-facebook-elbert.dts
index 27b43fe099f1..8e1a1d1b282d 100644
--- a/arch/arm/boot/dts/aspeed-bmc-facebook-elbert.dts
+++ b/arch/arm/boot/dts/aspeed-bmc-facebook-elbert.dts
@@ -183,3 +183,21 @@
 &i2c11 {
 	status = "okay";
 };
+
+/*
+ * BMC's "mac3" controller is connected to BCM53134P's IMP_RGMII port
+ * directly (fixed link, no PHY in between).
+ * Note: BMC's "mdio0" controller is connected to BCM53134P's MDIO
+ * interface, and the MDIO channel will be enabled in dts later, when
+ * BCM53134 is added to "bcm53xx" DSA driver.
+ */
+&mac3 {
+	status = "okay";
+	phy-mode = "rgmii";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_rgmii4_default>;
+	fixed-link {
+		speed = <1000>;
+		full-duplex;
+	};
+};
diff --git a/arch/arm/boot/dts/aspeed-bmc-facebook-yosemitev2.dts b/arch/arm/boot/dts/aspeed-bmc-facebook-yosemitev2.dts
index 8864e9c312a8..6bf2ff85a40e 100644
--- a/arch/arm/boot/dts/aspeed-bmc-facebook-yosemitev2.dts
+++ b/arch/arm/boot/dts/aspeed-bmc-facebook-yosemitev2.dts
@@ -207,11 +207,16 @@
 
 &i2c12 {
 	status = "okay";
-	//MEZZ_FRU
-	eeprom@51 {
-		compatible = "atmel,24c64";
-		reg = <0x51>;
-		pagesize = <32>;
+};
+
+&i2c13 {
+	status = "okay";
+	// Debug Card
+	multi-master;
+	ipmb13@10 {
+		compatible = "ipmb-dev";
+		reg = <(0x10 | I2C_OWN_SLAVE_ADDRESS)>;
+		i2c-protocol;
 	};
 };
 
diff --git a/arch/arm/boot/dts/aspeed-g6.dtsi b/arch/arm/boot/dts/aspeed-g6.dtsi
index 6660564855ff..1387a763a6a5 100644
--- a/arch/arm/boot/dts/aspeed-g6.dtsi
+++ b/arch/arm/boot/dts/aspeed-g6.dtsi
@@ -756,6 +756,62 @@
 				status = "disabled";
 			};
 
+			uart6: serial@1e790000 {
+				compatible = "ns16550a";
+				reg = <0x1e790000 0x20>;
+				reg-shift = <2>;
+				reg-io-width = <4>;
+				interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART6CLK>;
+				no-loopback-test;
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_uart6_default>;
+
+				status = "disabled";
+			};
+
+			uart7: serial@1e790100 {
+				compatible = "ns16550a";
+				reg = <0x1e790100 0x20>;
+				reg-shift = <2>;
+				reg-io-width = <4>;
+				interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART7CLK>;
+				no-loopback-test;
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_uart7_default>;
+
+				status = "disabled";
+			};
+
+			uart8: serial@1e790200 {
+				compatible = "ns16550a";
+				reg = <0x1e790200 0x20>;
+				reg-shift = <2>;
+				reg-io-width = <4>;
+				interrupts = <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART8CLK>;
+				no-loopback-test;
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_uart8_default>;
+
+				status = "disabled";
+			};
+
+			uart9: serial@1e790300 {
+				compatible = "ns16550a";
+				reg = <0x1e790300 0x20>;
+				reg-shift = <2>;
+				reg-io-width = <4>;
+				interrupts = <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&syscon ASPEED_CLK_GATE_UART9CLK>;
+				no-loopback-test;
+				pinctrl-names = "default";
+				pinctrl-0 = <&pinctrl_uart9_default>;
+
+				status = "disabled";
+			};
+
 			i2c: bus@1e78a000 {
 				compatible = "simple-bus";
 				#address-cells = <1>;
diff --git a/arch/arm/boot/dts/at91-sam9x60ek.dts b/arch/arm/boot/dts/at91-sam9x60ek.dts
index 81c38e101f58..4ba52ba11dc6 100644
--- a/arch/arm/boot/dts/at91-sam9x60ek.dts
+++ b/arch/arm/boot/dts/at91-sam9x60ek.dts
@@ -34,48 +34,6 @@
 		};
 	};
 
-	regulators: regulators {
-		compatible = "simple-bus";
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		vdd_1v8: fixed-regulator-vdd_1v8@0 {
-			compatible = "regulator-fixed";
-			regulator-name = "VDD_1V8";
-			regulator-min-microvolt = <1800000>;
-			regulator-max-microvolt = <1800000>;
-			regulator-always-on;
-			status = "okay";
-		};
-
-		vdd_1v15: fixed-regulator-vdd_1v15@1 {
-			compatible = "regulator-fixed";
-			regulator-name = "VDD_1V15";
-			regulator-min-microvolt = <1150000>;
-			regulator-max-microvolt = <1150000>;
-			regulator-always-on;
-			status = "okay";
-		};
-
-		vdd1_3v3: fixed-regulator-vdd1_3v3@2 {
-			compatible = "regulator-fixed";
-			regulator-name = "VDD1_3V3";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-			status = "okay";
-		};
-
-		vdd2_3v3: regulator-fixed-vdd2_3v3@3 {
-			compatible = "regulator-fixed";
-			regulator-name = "VDD2_3V3";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-			status = "okay";
-		};
-	};
-
 	gpio-keys {
 		compatible = "gpio-keys";
 		pinctrl-names = "default";
@@ -111,6 +69,42 @@
 			linux,default-trigger = "heartbeat";
 		};
 	};
+
+	vdd_1v8: fixed-regulator-vdd_1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		status = "okay";
+	};
+
+	vdd_1v15: fixed-regulator-vdd_1v15 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_1V15";
+		regulator-min-microvolt = <1150000>;
+		regulator-max-microvolt = <1150000>;
+		regulator-always-on;
+		status = "okay";
+	};
+
+	vdd1_3v3: fixed-regulator-vdd1_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD1_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		status = "okay";
+	};
+
+	vdd2_3v3: regulator-fixed-vdd2_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD2_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		status = "okay";
+	};
 };
 
 &adc {
@@ -264,8 +258,9 @@
 	status = "okay";
 
 	uart1: serial@200 {
-		compatible = "microchip,sam9x60-usart", "atmel,at91sam9260-usart";
+		compatible = "microchip,sam9x60-dbgu", "microchip,sam9x60-usart", "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
 		reg = <0x200 0x200>;
+		atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 		interrupts = <14 IRQ_TYPE_LEVEL_HIGH 7>;
 		dmas = <&dma0
 			(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
diff --git a/arch/arm/boot/dts/at91-sama5d3_eds.dts b/arch/arm/boot/dts/at91-sama5d3_eds.dts
new file mode 100644
index 000000000000..c287b03d768b
--- /dev/null
+++ b/arch/arm/boot/dts/at91-sama5d3_eds.dts
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * at91-sama5d3_eds.dts - Device Tree file for the SAMA5D3 Ethernet
+ *    Development System board.
+ *
+ * Copyright (C) 2022 Microchip Technology Inc. and its subsidiaries
+ *
+ * Author: Jerry Ray <jerry.ray@microchip.com>
+ */
+/dts-v1/;
+#include "sama5d36.dtsi"
+
+/ {
+	model = "SAMA5D3 Ethernet Development System";
+	compatible = "microchip,sama5d3-eds", "atmel,sama5d36",
+		     "atmel,sama5d3", "atmel,sama5";
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_key_gpio>;
+
+		button-3 {
+			label = "PB_USER";
+			gpios = <&pioE 29 GPIO_ACTIVE_LOW>;
+			linux,code = <0x104>;
+			wakeup-source;
+		};
+	};
+
+	memory@20000000 {
+		reg = <0x20000000 0x10000000>;
+	};
+
+	vcc_3v3_reg: regulator-1 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	vcc_2v5_reg: regulator-2 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_2V5";
+		regulator-min-microvolt = <2500000>;
+		regulator-max-microvolt = <2500000>;
+		regulator-always-on;
+		vin-supply = <&vcc_3v3_reg>;
+	};
+
+	vcc_1v8_reg: regulator-3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		vin-supply = <&vcc_3v3_reg>;
+	};
+
+	vcc_1v2_reg: regulator-4 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_1V2";
+		regulator-min-microvolt = <1200000>;
+		regulator-max-microvolt = <1200000>;
+		regulator-always-on;
+	};
+
+	vcc_mmc0_reg: regulator-5 {
+		compatible = "regulator-fixed";
+		regulator-name = "mmc0-card-supply";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_vcc_mmc0_reg_gpio>;
+		gpio = <&pioE 2 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&can0 {
+	status = "okay";
+};
+
+&dbgu {
+	status = "okay";
+};
+
+&ebi {
+	pinctrl-0 = <&pinctrl_ebi_nand_addr>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	nand_controller: nand-controller {
+		status = "okay";
+
+		nand@3 {
+			reg = <0x3 0x0 0x2>;
+			atmel,rb = <0>;
+			nand-bus-width = <8>;
+			nand-ecc-mode = "hw";
+			nand-ecc-strength = <4>;
+			nand-ecc-step-size = <512>;
+			nand-on-flash-bbt;
+			label = "atmel_nand";
+
+			partitions {
+				compatible = "fixed-partitions";
+				#address-cells = <1>;
+				#size-cells = <1>;
+
+				at91bootstrap@0 {
+					label = "at91bootstrap";
+					reg = <0x0 0x40000>;
+				};
+
+				bootloader@40000 {
+					label = "bootloader";
+					reg = <0x40000 0xc0000>;
+				};
+
+				bootloaderenvred@100000 {
+					label = "bootloader env redundant";
+					reg = <0x100000 0x40000>;
+				};
+
+				bootloaderenv@140000 {
+					label = "bootloader env";
+					reg = <0x140000 0x40000>;
+				};
+
+				dtb@180000 {
+					label = "device tree";
+					reg = <0x180000 0x80000>;
+				};
+
+				kernel@200000 {
+					label = "kernel";
+					reg = <0x200000 0x600000>;
+				};
+
+				rootfs@800000 {
+					label = "rootfs";
+					reg = <0x800000 0x0f800000>;
+				};
+			};
+		};
+	};
+};
+
+&i2c0 {
+	pinctrl-0 = <&pinctrl_i2c0_pu>;
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&i2c2 {
+	pinctrl-0 = <&pinctrl_i2c2_pu>;
+	status = "okay";
+};
+
+&main_xtal {
+	clock-frequency = <12000000>;
+};
+
+&mmc0 {
+	pinctrl-0 = <&pinctrl_mmc0_clk_cmd_dat0 &pinctrl_mmc0_dat1_3
+		     &pinctrl_mmc0_dat4_7 &pinctrl_mmc0_cd>;
+	vmmc-supply = <&vcc_mmc0_reg>;
+	vqmmc-supply = <&vcc_3v3_reg>;
+	status = "okay";
+	slot@0 {
+		reg = <0>;
+		bus-width = <8>;
+		cd-gpios = <&pioE 0 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&pinctrl {
+	board {
+		pinctrl_i2c0_pu: i2c0-pu {
+			atmel,pins =
+				<AT91_PIOA 30 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>,
+				<AT91_PIOA 31 AT91_PERIPH_A AT91_PINCTRL_PULL_UP>;
+		};
+
+		pinctrl_i2c2_pu: i2c2-pu {
+			atmel,pins =
+				<AT91_PIOA 18 AT91_PERIPH_B AT91_PINCTRL_PULL_UP>,
+				<AT91_PIOA 19 AT91_PERIPH_B AT91_PINCTRL_PULL_UP>;
+		};
+
+		pinctrl_key_gpio: key-gpio-0 {
+			atmel,pins =
+				<AT91_PIOE 29 AT91_PERIPH_GPIO AT91_PINCTRL_PULL_UP_DEGLITCH>;
+		};
+
+		pinctrl_mmc0_cd: mmc0-cd {
+			atmel,pins =
+				<AT91_PIOE 0 AT91_PERIPH_GPIO AT91_PINCTRL_PULL_UP_DEGLITCH>;
+		};
+
+		/* Reserved for reset signal to the RGMII connector. */
+		pinctrl_rgmii_rstn: rgmii-rstn {
+			atmel,pins =
+				<AT91_PIOD 18 AT91_PERIPH_GPIO AT91_PINCTRL_PULL_UP_DEGLITCH>;
+		};
+
+		/* Reserved for an interrupt line from the RMII and RGMII connectors. */
+		pinctrl_spi_irqn: spi-irqn {
+			atmel,pins =
+				<AT91_PIOB 28 AT91_PERIPH_GPIO AT91_PINCTRL_DEGLITCH>;
+		};
+
+		pinctrl_spi0_cs: spi0-cs-default {
+			atmel,pins =
+				<AT91_PIOD 13 AT91_PERIPH_GPIO AT91_PINCTRL_NONE
+				 AT91_PIOD 16 AT91_PERIPH_GPIO AT91_PINCTRL_NONE>;
+		};
+
+		pinctrl_spi1_cs: spi1-cs-default {
+			atmel,pins = <AT91_PIOC 25 AT91_PERIPH_GPIO AT91_PINCTRL_NONE
+				      AT91_PIOC 28 AT91_PERIPH_GPIO AT91_PINCTRL_NONE>;
+		};
+
+		pinctrl_usba_vbus: usba-vbus {
+			atmel,pins =
+				<AT91_PIOE 9 AT91_PERIPH_GPIO AT91_PINCTRL_DEGLITCH>;
+		};
+
+		pinctrl_usb_default: usb-default {
+			atmel,pins =
+				<AT91_PIOE 3 AT91_PERIPH_GPIO AT91_PINCTRL_NONE
+				 AT91_PIOE 4 AT91_PERIPH_GPIO AT91_PINCTRL_NONE>;
+		};
+
+		/* Reserved for VBUS fault interrupt. */
+		pinctrl_vbusfault_irqn: vbusfault-irqn {
+			atmel,pins =
+				<AT91_PIOE 5 AT91_PERIPH_GPIO AT91_PINCTRL_DEGLITCH>;
+		};
+
+		pinctrl_vcc_mmc0_reg_gpio: vcc-mmc0-reg-gpio-default {
+			atmel,pins = <AT91_PIOE 2 AT91_PERIPH_GPIO AT91_PINCTRL_PULL_UP>;
+		};
+	};
+};
+
+&slow_xtal {
+	clock-frequency = <32768>;
+};
+
+&spi0 {
+	pinctrl-names = "default", "cs";
+	pinctrl-1 = <&pinctrl_spi0_cs>;
+	cs-gpios = <&pioD 13 0>, <0>, <0>, <&pioD 16 0>;
+	status = "okay";
+};
+
+&spi1 {
+	pinctrl-names = "default", "cs";
+	pinctrl-1 = <&pinctrl_spi1_cs>;
+	cs-gpios = <&pioC 25 0>, <0>, <0>, <&pioC 28 0>;
+	status = "okay";
+};
+
+&tcb0 {
+	timer0: timer@0 {
+		compatible = "atmel,tcb-timer";
+		reg = <0>;
+	};
+
+	timer1: timer@1 {
+		compatible = "atmel,tcb-timer";
+		reg = <1>;
+	};
+};
+
+&usb0 {	/* USB Device port with VBUS detection. */
+	atmel,vbus-gpio = <&pioE 9 GPIO_ACTIVE_HIGH>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usba_vbus>;
+	status = "okay";
+};
+
+&usb1 {	/* 3-port Host. First port is unused. */
+	atmel,vbus-gpio = <0
+			   &pioE 3 GPIO_ACTIVE_HIGH
+			   &pioE 4 GPIO_ACTIVE_HIGH
+			  >;
+	num-ports = <3>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usb_default>;
+	status = "okay";
+};
+
+&usb2 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/at91rm9200.dtsi b/arch/arm/boot/dts/at91rm9200.dtsi
index d1181ead18e5..7a113325abb9 100644
--- a/arch/arm/boot/dts/at91rm9200.dtsi
+++ b/arch/arm/boot/dts/at91rm9200.dtsi
@@ -13,6 +13,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	#address-cells = <1>;
@@ -596,6 +597,7 @@
 			dbgu: serial@fffff200 {
 				compatible = "atmel,at91rm9200-dbgu", "atmel,at91rm9200-usart";
 				reg = <0xfffff200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_dbgu>;
@@ -607,6 +609,7 @@
 			usart0: serial@fffc0000 {
 				compatible = "atmel,at91rm9200-usart";
 				reg = <0xfffc0000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -620,6 +623,7 @@
 			usart1: serial@fffc4000 {
 				compatible = "atmel,at91rm9200-usart";
 				reg = <0xfffc4000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -633,6 +637,7 @@
 			usart2: serial@fffc8000 {
 				compatible = "atmel,at91rm9200-usart";
 				reg = <0xfffc8000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -646,6 +651,7 @@
 			usart3: serial@fffcc000 {
 				compatible = "atmel,at91rm9200-usart";
 				reg = <0xfffcc000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <23 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
diff --git a/arch/arm/boot/dts/at91sam9260.dtsi b/arch/arm/boot/dts/at91sam9260.dtsi
index 9d9820db9482..789fe356dbf6 100644
--- a/arch/arm/boot/dts/at91sam9260.dtsi
+++ b/arch/arm/boot/dts/at91sam9260.dtsi
@@ -11,6 +11,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	#address-cells = <1>;
@@ -532,6 +533,7 @@
 			dbgu: serial@fffff200 {
 				compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
 				reg = <0xfffff200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_dbgu>;
@@ -543,6 +545,7 @@
 			usart0: serial@fffb0000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffb0000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -556,6 +559,7 @@
 			usart1: serial@fffb4000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffb4000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -569,6 +573,7 @@
 			usart2: serial@fffb8000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffb8000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -582,6 +587,7 @@
 			usart3: serial@fffd0000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffd0000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <23 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -595,6 +601,7 @@
 			uart0: serial@fffd4000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffd4000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <24 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -608,6 +615,7 @@
 			uart1: serial@fffd8000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffd8000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <25 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
diff --git a/arch/arm/boot/dts/at91sam9261.dtsi b/arch/arm/boot/dts/at91sam9261.dtsi
index 259aca565305..ee0bd1aceb3f 100644
--- a/arch/arm/boot/dts/at91sam9261.dtsi
+++ b/arch/arm/boot/dts/at91sam9261.dtsi
@@ -9,6 +9,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	#address-cells = <1>;
@@ -179,6 +180,7 @@
 			usart0: serial@fffb0000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffb0000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -192,6 +194,7 @@
 			usart1: serial@fffb4000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffb4000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -205,6 +208,7 @@
 			usart2: serial@fffb8000{
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffb8000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -301,6 +305,7 @@
 			dbgu: serial@fffff200 {
 				compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
 				reg = <0xfffff200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_dbgu>;
diff --git a/arch/arm/boot/dts/at91sam9263.dtsi b/arch/arm/boot/dts/at91sam9263.dtsi
index c080df8c2312..3ce9ea987312 100644
--- a/arch/arm/boot/dts/at91sam9263.dtsi
+++ b/arch/arm/boot/dts/at91sam9263.dtsi
@@ -9,6 +9,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	#address-cells = <1>;
@@ -540,6 +541,7 @@
 			dbgu: serial@ffffee00 {
 				compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
 				reg = <0xffffee00 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_dbgu>;
@@ -551,6 +553,7 @@
 			usart0: serial@fff8c000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfff8c000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -564,6 +567,7 @@
 			usart1: serial@fff90000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfff90000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -577,6 +581,7 @@
 			usart2: serial@fff94000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfff94000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <9 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
diff --git a/arch/arm/boot/dts/at91sam9g45.dtsi b/arch/arm/boot/dts/at91sam9g45.dtsi
index 09794561c7ce..95f5d76234db 100644
--- a/arch/arm/boot/dts/at91sam9g45.dtsi
+++ b/arch/arm/boot/dts/at91sam9g45.dtsi
@@ -13,6 +13,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	#address-cells = <1>;
@@ -675,6 +676,7 @@
 
 			dbgu: serial@ffffee00 {
 				compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				reg = <0xffffee00 0x200>;
 				interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
 				pinctrl-names = "default";
@@ -687,6 +689,7 @@
 			usart0: serial@fff8c000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfff8c000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -700,6 +703,7 @@
 			usart1: serial@fff90000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfff90000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -713,6 +717,7 @@
 			usart2: serial@fff94000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfff94000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <9 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -726,6 +731,7 @@
 			usart3: serial@fff98000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfff98000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <10 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
diff --git a/arch/arm/boot/dts/at91sam9n12.dtsi b/arch/arm/boot/dts/at91sam9n12.dtsi
index 556f35ce49e3..83114d26f10d 100644
--- a/arch/arm/boot/dts/at91sam9n12.dtsi
+++ b/arch/arm/boot/dts/at91sam9n12.dtsi
@@ -11,6 +11,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	#address-cells = <1>;
@@ -593,6 +594,7 @@
 			dbgu: serial@fffff200 {
 				compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
 				reg = <0xfffff200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_dbgu>;
@@ -618,6 +620,7 @@
 			usart0: serial@f801c000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf801c000 0x4000>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <5 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_usart0>;
@@ -629,6 +632,7 @@
 			usart1: serial@f8020000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8020000 0x4000>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_usart1>;
@@ -640,6 +644,7 @@
 			usart2: serial@f8024000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8024000 0x4000>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_usart2>;
@@ -651,6 +656,7 @@
 			usart3: serial@f8028000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8028000 0x4000>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_usart3>;
diff --git a/arch/arm/boot/dts/at91sam9rl.dtsi b/arch/arm/boot/dts/at91sam9rl.dtsi
index 12c634811820..364a2ff0a763 100644
--- a/arch/arm/boot/dts/at91sam9rl.dtsi
+++ b/arch/arm/boot/dts/at91sam9rl.dtsi
@@ -11,6 +11,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/pwm/pwm.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	#address-cells = <1>;
@@ -175,6 +176,7 @@
 			usart0: serial@fffb0000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffb0000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -188,6 +190,7 @@
 			usart1: serial@fffb4000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffb4000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -201,6 +204,7 @@
 			usart2: serial@fffb8000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffb8000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -214,6 +218,7 @@
 			usart3: serial@fffbc000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfffbc000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <9 IRQ_TYPE_LEVEL_HIGH 5>;
 				atmel,use-dma-rx;
 				atmel,use-dma-tx;
@@ -322,6 +327,7 @@
 			dbgu: serial@fffff200 {
 				compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
 				reg = <0xfffff200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_dbgu>;
diff --git a/arch/arm/boot/dts/at91sam9x5.dtsi b/arch/arm/boot/dts/at91sam9x5.dtsi
index ea3b11336c79..0c26c925761b 100644
--- a/arch/arm/boot/dts/at91sam9x5.dtsi
+++ b/arch/arm/boot/dts/at91sam9x5.dtsi
@@ -13,6 +13,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	#address-cells = <1>;
@@ -674,6 +675,7 @@
 			dbgu: serial@fffff200 {
 				compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
 				reg = <0xfffff200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <1 IRQ_TYPE_LEVEL_HIGH 7>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_dbgu>;
@@ -688,6 +690,7 @@
 			usart0: serial@f801c000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf801c000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <5 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_usart0>;
@@ -702,6 +705,7 @@
 			usart1: serial@f8020000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8020000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_usart1>;
@@ -716,6 +720,7 @@
 			usart2: serial@f8024000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8024000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_usart2>;
@@ -775,6 +780,7 @@
 			uart0: serial@f8040000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8040000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <15 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_uart0>;
@@ -786,6 +792,7 @@
 			uart1: serial@f8044000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8044000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <16 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_uart1>;
diff --git a/arch/arm/boot/dts/at91sam9x5_usart3.dtsi b/arch/arm/boot/dts/at91sam9x5_usart3.dtsi
index 098d3fef5c37..a47c765e1b20 100644
--- a/arch/arm/boot/dts/at91sam9x5_usart3.dtsi
+++ b/arch/arm/boot/dts/at91sam9x5_usart3.dtsi
@@ -8,6 +8,7 @@
 
 #include <dt-bindings/pinctrl/at91.h>
 #include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	aliases {
@@ -44,6 +45,7 @@
 			usart3: serial@f8028000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8028000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <8 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_usart3>;
diff --git a/arch/arm/boot/dts/dove.dtsi b/arch/arm/boot/dts/dove.dtsi
index 89e0bdaf3a85..00a36fba2fd2 100644
--- a/arch/arm/boot/dts/dove.dtsi
+++ b/arch/arm/boot/dts/dove.dtsi
@@ -122,8 +122,18 @@
 				bus-range = <0x00 0xff>;
 
 				#interrupt-cells = <1>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &intc 16>;
+				interrupt-names = "intx", "error";
+				interrupts = <16>, <15>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie0_intc 0>,
+						<0 0 0 2 &pcie0_intc 1>,
+						<0 0 0 3 &pcie0_intc 2>,
+						<0 0 0 4 &pcie0_intc 3>;
+
+				pcie0_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie1: pcie@2 {
@@ -141,8 +151,18 @@
 				bus-range = <0x00 0xff>;
 
 				#interrupt-cells = <1>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &intc 18>;
+				interrupt-names = "intx", "error";
+				interrupts = <18>, <17>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie1_intc 0>,
+						<0 0 0 2 &pcie1_intc 1>,
+						<0 0 0 3 &pcie1_intc 2>,
+						<0 0 0 4 &pcie1_intc 3>;
+
+				pcie1_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 
diff --git a/arch/arm/boot/dts/exynos4412-midas.dtsi b/arch/arm/boot/dts/exynos4412-midas.dtsi
index b967397a46c5..8e1c19a8ad06 100644
--- a/arch/arm/boot/dts/exynos4412-midas.dtsi
+++ b/arch/arm/boot/dts/exynos4412-midas.dtsi
@@ -586,7 +586,7 @@
 		clocks = <&camera 1>;
 		clock-names = "extclk";
 		samsung,camclk-out = <1>;
-		gpios = <&gpm1 6 GPIO_ACTIVE_HIGH>;
+		gpios = <&gpm1 6 GPIO_ACTIVE_LOW>;
 
 		port {
 			is_s5k6a3_ep: endpoint {
diff --git a/arch/arm/boot/dts/exynos4412-origen.dts b/arch/arm/boot/dts/exynos4412-origen.dts
index 6db09dba07ff..a3905e27b9cd 100644
--- a/arch/arm/boot/dts/exynos4412-origen.dts
+++ b/arch/arm/boot/dts/exynos4412-origen.dts
@@ -95,7 +95,7 @@
 };
 
 &ehci {
-	samsung,vbus-gpio = <&gpx3 5 1>;
+	samsung,vbus-gpio = <&gpx3 5 GPIO_ACTIVE_HIGH>;
 	status = "okay";
 	phys = <&exynos_usbphy 2>, <&exynos_usbphy 3>;
 	phy-names = "hsic0", "hsic1";
diff --git a/arch/arm/boot/dts/gemini-ns2502.dts b/arch/arm/boot/dts/gemini-ns2502.dts
index 704abd212df5..e6eeb35e8819 100644
--- a/arch/arm/boot/dts/gemini-ns2502.dts
+++ b/arch/arm/boot/dts/gemini-ns2502.dts
@@ -39,10 +39,6 @@
 		phy0: ethernet-phy@1 {
 			reg = <1>;
 			device_type = "ethernet-phy";
-			/* We lack the knowledge of necessary GPIO to achieve
-			 * Gigabit
-			 */
-			max-speed = <100>;
 		};
 	};
 };
@@ -50,7 +46,7 @@
 &ethernet {
 	status = "okay";
 	ethernet-port@0 {
-		phy-mode = "rgmii";
+		phy-mode = "rgmii-id";
 		phy-handle = <&phy0>;
 	};
 };
@@ -65,30 +61,9 @@
 	pinctrl-1 = <&pflash_disabled_pins>;
 
 	partitions {
-		compatible = "fixed-partitions";
-		#address-cells = <1>;
-		#size-cells = <1>;
-
-		partition@0 {
-			label = "RedBoot";
-			reg = <0x00000000 0x00020000>;
-		};
-		partition@20000 {
-			label = "kernel";
-			reg = <0x00020000 0x00700000>;
-		};
-		partition@720000 {
-			label = "VCTL";
-			reg = <0x00720000 0x00020000>;
-		};
-		partition@740000 {
-			label = "CurConf";
-			reg = <0x00740000 0x000a0000>;
-		};
-		partition@7e0000 {
-			label = "FIS";
-			reg = <0x007e0000 0x00010000>;
-		};
+		compatible = "redboot-fis";
+		/* Eraseblock at 0x7e0000 */
+		fis-index-block = <0x3f>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/gemini-ssi1328.dts b/arch/arm/boot/dts/gemini-ssi1328.dts
index 2b3e7db84fed..42e85f07cf76 100644
--- a/arch/arm/boot/dts/gemini-ssi1328.dts
+++ b/arch/arm/boot/dts/gemini-ssi1328.dts
@@ -40,10 +40,6 @@
 		phy0: ethernet-phy@1 {
 			reg = <1>;
 			device_type = "ethernet-phy";
-			/* We lack the knowledge of necessary GPIO to achieve
-			 * Gigabit
-			 */
-			max-speed = <100>;
 		};
 		/* WAN ICPlus IP101A */
 		phy1: ethernet-phy@2 {
diff --git a/arch/arm/boot/dts/imx23-xfi3.dts b/arch/arm/boot/dts/imx23-xfi3.dts
index a6213c590f94..b1d8210f3ecc 100644
--- a/arch/arm/boot/dts/imx23-xfi3.dts
+++ b/arch/arm/boot/dts/imx23-xfi3.dts
@@ -158,19 +158,19 @@
 		default-brightness-level = <6>;
 	};
 
-	gpio_keys {
+	gpio-keys {
 		compatible = "gpio-keys";
 		pinctrl-names = "default";
 		pinctrl-0 = <&key_pins_a>;
 
-		voldown {
+		key-voldown {
 			label = "volume-down";
 			linux,code = <114>;
 			gpios = <&gpio2 7 0>;
 			debounce-interval = <20>;
 		};
 
-		volup {
+		key-volup {
 			label = "volume-up";
 			linux,code = <115>;
 			gpios = <&gpio2 8 0>;
diff --git a/arch/arm/boot/dts/imx25-eukrea-mbimxsd25-baseboard.dts b/arch/arm/boot/dts/imx25-eukrea-mbimxsd25-baseboard.dts
index 3f38c2e60a74..c7207ea437c4 100644
--- a/arch/arm/boot/dts/imx25-eukrea-mbimxsd25-baseboard.dts
+++ b/arch/arm/boot/dts/imx25-eukrea-mbimxsd25-baseboard.dts
@@ -13,12 +13,12 @@
 	model = "Eukrea MBIMXSD25";
 	compatible = "eukrea,mbimxsd25-baseboard", "eukrea,cpuimx25", "fsl,imx25";
 
-	gpio_keys {
+	gpio-keys {
 		compatible = "gpio-keys";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_gpiokeys>;
 
-		bp1 {
+		button {
 			label = "BP1";
 			gpios = <&gpio3 18 GPIO_ACTIVE_LOW>;
 			linux,code = <BTN_MISC>;
diff --git a/arch/arm/boot/dts/imx25.dtsi b/arch/arm/boot/dts/imx25.dtsi
index bc4de0c05511..5f90d72b840b 100644
--- a/arch/arm/boot/dts/imx25.dtsi
+++ b/arch/arm/boot/dts/imx25.dtsi
@@ -515,7 +515,7 @@
 				#interrupt-cells = <2>;
 			};
 
-			sdma: sdma@53fd4000 {
+			sdma: dma-controller@53fd4000 {
 				compatible = "fsl,imx25-sdma";
 				reg = <0x53fd4000 0x4000>;
 				clocks = <&clks 112>, <&clks 68>;
diff --git a/arch/arm/boot/dts/imx28-cfa10049.dts b/arch/arm/boot/dts/imx28-cfa10049.dts
index a92b05ef390f..9ef0d567ea48 100644
--- a/arch/arm/boot/dts/imx28-cfa10049.dts
+++ b/arch/arm/boot/dts/imx28-cfa10049.dts
@@ -327,7 +327,7 @@
 		};
 	};
 
-	spi2 {
+	spi-2 {
 		compatible = "spi-gpio";
 		pinctrl-names = "default";
 		pinctrl-0 = <&spi2_pins_cfa10049>;
@@ -351,7 +351,7 @@
 		};
 	};
 
-	spi3 {
+	spi-3 {
 		compatible = "spi-gpio";
 		pinctrl-names = "default";
 		pinctrl-0 = <&spi3_pins_cfa10049>;
@@ -388,12 +388,12 @@
 		};
 	};
 
-	gpio_keys {
+	gpio-keys {
 		compatible = "gpio-keys";
 		pinctrl-names = "default";
 		pinctrl-0 = <&rotary_btn_pins_cfa10049>;
 
-		rotary_button {
+		rotary-button {
 			label = "rotary_button";
 			gpios = <&gpio3 26 1>;
 			debounce-interval = <10>;
diff --git a/arch/arm/boot/dts/imx28-cfa10055.dts b/arch/arm/boot/dts/imx28-cfa10055.dts
index d05c370dfc17..fac5bbda7a93 100644
--- a/arch/arm/boot/dts/imx28-cfa10055.dts
+++ b/arch/arm/boot/dts/imx28-cfa10055.dts
@@ -129,7 +129,7 @@
 		};
 	};
 
-	spi2 {
+	spi-2 {
 		compatible = "spi-gpio";
 		pinctrl-names = "default";
 		pinctrl-0 = <&spi2_pins_cfa10055>;
diff --git a/arch/arm/boot/dts/imx28-cfa10056.dts b/arch/arm/boot/dts/imx28-cfa10056.dts
index c1060bd5f17f..c5f3337e8b39 100644
--- a/arch/arm/boot/dts/imx28-cfa10056.dts
+++ b/arch/arm/boot/dts/imx28-cfa10056.dts
@@ -88,7 +88,7 @@
 		};
 	};
 
-	spi2 {
+	spi-2 {
 		compatible = "spi-gpio";
 		pinctrl-names = "default";
 		pinctrl-0 = <&spi2_pins_cfa10056>;
diff --git a/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts b/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts
index bacb846f99e3..73f521c46c1e 100644
--- a/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts
+++ b/arch/arm/boot/dts/imx28-duckbill-2-enocean.dts
@@ -204,7 +204,7 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&enocean_button>;
 
-		enocean {
+		key-enocean {
 			label = "EnOcean";
 			linux,code = <KEY_NEW>;
 			gpios = <&gpio3 3 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm/boot/dts/imx28-eukrea-mbmx28lc.dtsi b/arch/arm/boot/dts/imx28-eukrea-mbmx28lc.dtsi
index 3280fddaaf0d..b285a946e2c2 100644
--- a/arch/arm/boot/dts/imx28-eukrea-mbmx28lc.dtsi
+++ b/arch/arm/boot/dts/imx28-eukrea-mbmx28lc.dtsi
@@ -19,12 +19,12 @@
 		default-brightness-level = <10>;
 	};
 
-	button-sw3 {
+	gpio-keys-0 {
 		compatible = "gpio-keys";
 		pinctrl-names = "default";
 		pinctrl-0 = <&gpio_button_sw3_pins_mbmx28lc>;
 
-		sw3 {
+		switch-sw3 {
 			label = "SW3";
 			gpios = <&gpio1 21 GPIO_ACTIVE_LOW>;
 			linux,code = <BTN_MISC>;
@@ -32,12 +32,12 @@
 		};
 	};
 
-	button-sw4 {
+	gpio-keys-1 {
 		compatible = "gpio-keys";
 		pinctrl-names = "default";
 		pinctrl-0 = <&gpio_button_sw4_pins_mbmx28lc>;
 
-		sw4 {
+		switch-sw4 {
 			label = "SW4";
 			gpios = <&gpio1 20 GPIO_ACTIVE_LOW>;
 			linux,code = <BTN_MISC>;
diff --git a/arch/arm/boot/dts/imx28-tx28.dts b/arch/arm/boot/dts/imx28-tx28.dts
index 6d7b044fec5b..096f246032c6 100644
--- a/arch/arm/boot/dts/imx28-tx28.dts
+++ b/arch/arm/boot/dts/imx28-tx28.dts
@@ -221,7 +221,7 @@
 		linux,no-autorepeat;
 	};
 
-	spi_gpio: spi-gpio {
+	spi_gpio: spi {
 		compatible = "spi-gpio";
 		#address-cells = <1>;
 		#size-cells = <0>;
diff --git a/arch/arm/boot/dts/imx31.dtsi b/arch/arm/boot/dts/imx31.dtsi
index 5c4938b0d5a1..95c05f17a6d5 100644
--- a/arch/arm/boot/dts/imx31.dtsi
+++ b/arch/arm/boot/dts/imx31.dtsi
@@ -297,7 +297,7 @@
 				#interrupt-cells = <2>;
 			};
 
-			sdma: sdma@53fd4000 {
+			sdma: dma-controller@53fd4000 {
 				compatible = "fsl,imx31-sdma";
 				reg = <0x53fd4000 0x4000>;
 				interrupts = <34>;
diff --git a/arch/arm/boot/dts/imx35-eukrea-mbimxsd35-baseboard.dts b/arch/arm/boot/dts/imx35-eukrea-mbimxsd35-baseboard.dts
index b1c11170ac25..7f4f812b0811 100644
--- a/arch/arm/boot/dts/imx35-eukrea-mbimxsd35-baseboard.dts
+++ b/arch/arm/boot/dts/imx35-eukrea-mbimxsd35-baseboard.dts
@@ -13,12 +13,12 @@
 	model = "Eukrea CPUIMX35";
 	compatible = "eukrea,mbimxsd35-baseboard", "eukrea,cpuimx35", "fsl,imx35";
 
-	gpio_keys {
+	gpio-keys {
 		compatible = "gpio-keys";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_bp1>;
 
-		bp1 {
+		button {
 			label = "BP1";
 			gpios = <&gpio3 25 GPIO_ACTIVE_LOW>;
 			linux,code = <BTN_MISC>;
diff --git a/arch/arm/boot/dts/imx35.dtsi b/arch/arm/boot/dts/imx35.dtsi
index 8e41c8b7bd70..d650f54c3fc6 100644
--- a/arch/arm/boot/dts/imx35.dtsi
+++ b/arch/arm/boot/dts/imx35.dtsi
@@ -284,7 +284,7 @@
 				#interrupt-cells = <2>;
 			};
 
-			sdma: sdma@53fd4000 {
+			sdma: dma-controller@53fd4000 {
 				compatible = "fsl,imx35-sdma";
 				reg = <0x53fd4000 0x4000>;
 				clocks = <&clks 9>, <&clks 65>;
diff --git a/arch/arm/boot/dts/imx50-kobo-aura.dts b/arch/arm/boot/dts/imx50-kobo-aura.dts
index 82ce8c43be86..51bf6117fb12 100644
--- a/arch/arm/boot/dts/imx50-kobo-aura.dts
+++ b/arch/arm/boot/dts/imx50-kobo-aura.dts
@@ -38,20 +38,20 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_gpiokeys>;
 
-		power {
+		key-power {
 			label = "Power Button";
 			gpios = <&gpio4 10 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_POWER>;
 		};
 
-		hallsensor {
+		event-hallsensor {
 			label = "Hallsensor";
 			gpios = <&gpio5 15 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_RESERVED>;
 			linux,input-type = <EV_SW>;
 		};
 
-		frontlight {
+		event-frontlight {
 			label = "Frontlight";
 			gpios = <&gpio4 1 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_DISPLAYTOGGLE>;
diff --git a/arch/arm/boot/dts/imx50.dtsi b/arch/arm/boot/dts/imx50.dtsi
index c0c7575fbecf..3d9a9f37f672 100644
--- a/arch/arm/boot/dts/imx50.dtsi
+++ b/arch/arm/boot/dts/imx50.dtsi
@@ -421,7 +421,7 @@
 				status = "disabled";
 			};
 
-			sdma: sdma@63fb0000 {
+			sdma: dma-controller@63fb0000 {
 				compatible = "fsl,imx50-sdma", "fsl,imx35-sdma";
 				reg = <0x63fb0000 0x4000>;
 				interrupts = <6>;
diff --git a/arch/arm/boot/dts/imx51-apf51dev.dts b/arch/arm/boot/dts/imx51-apf51dev.dts
index c66f274ba4e9..b61d55ca1467 100644
--- a/arch/arm/boot/dts/imx51-apf51dev.dts
+++ b/arch/arm/boot/dts/imx51-apf51dev.dts
@@ -63,7 +63,7 @@
 	leds {
 		compatible = "gpio-leds";
 
-		user {
+		led-user {
 			label = "Heartbeat";
 			gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "heartbeat";
diff --git a/arch/arm/boot/dts/imx51-babbage.dts b/arch/arm/boot/dts/imx51-babbage.dts
index 552196d8a60a..a1f9c6a72275 100644
--- a/arch/arm/boot/dts/imx51-babbage.dts
+++ b/arch/arm/boot/dts/imx51-babbage.dts
@@ -154,7 +154,7 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_gpio_keys>;
 
-		power {
+		key-power {
 			label = "Power Button";
 			gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>;
 			linux,code = <KEY_POWER>;
diff --git a/arch/arm/boot/dts/imx51-zii-rdu1.dts b/arch/arm/boot/dts/imx51-zii-rdu1.dts
index ec8ca3ac2c1c..3140f038aa98 100644
--- a/arch/arm/boot/dts/imx51-zii-rdu1.dts
+++ b/arch/arm/boot/dts/imx51-zii-rdu1.dts
@@ -137,7 +137,7 @@
 		};
 	};
 
-	spi_gpio: spi-gpio {
+	spi_gpio: spi {
 		compatible = "spi-gpio";
 		#address-cells = <1>;
 		#size-cells = <0>;
diff --git a/arch/arm/boot/dts/imx51.dtsi b/arch/arm/boot/dts/imx51.dtsi
index 592d9c23a447..853707574d2e 100644
--- a/arch/arm/boot/dts/imx51.dtsi
+++ b/arch/arm/boot/dts/imx51.dtsi
@@ -504,7 +504,7 @@
 				status = "disabled";
 			};
 
-			sdma: sdma@83fb0000 {
+			sdma: dma-controller@83fb0000 {
 				compatible = "fsl,imx51-sdma", "fsl,imx35-sdma";
 				reg = <0x83fb0000 0x4000>;
 				interrupts = <6>;
diff --git a/arch/arm/boot/dts/imx53-ard.dts b/arch/arm/boot/dts/imx53-ard.dts
index 6208fbb2e741..23a7492e2929 100644
--- a/arch/arm/boot/dts/imx53-ard.dts
+++ b/arch/arm/boot/dts/imx53-ard.dts
@@ -61,34 +61,34 @@
 	gpio-keys {
 		compatible = "gpio-keys";
 
-		home {
+		key-home {
 			label = "Home";
 			gpios = <&gpio5 10 0>;
 			linux,code = <KEY_HOME>;
 			wakeup-source;
 		};
 
-		back {
+		key-back {
 			label = "Back";
 			gpios = <&gpio5 11 0>;
 			linux,code = <KEY_BACK>;
 			wakeup-source;
 		};
 
-		program {
+		key-program {
 			label = "Program";
 			gpios = <&gpio5 12 0>;
 			linux,code = <KEY_PROGRAM >;
 			wakeup-source;
 		};
 
-		volume-up {
+		key-volume-up {
 			label = "Volume Up";
 			gpios = <&gpio5 13 0>;
 			linux,code = <KEY_VOLUMEUP>;
 		};
 
-		volume-down {
+		key-volume-down {
 			label = "Volume Down";
 			gpios = <&gpio4 0 0>;
 			linux,code = <KEY_VOLUMEDOWN>;
diff --git a/arch/arm/boot/dts/imx53-qsb-common.dtsi b/arch/arm/boot/dts/imx53-qsb-common.dtsi
index fe4244044a0f..50fef8dd3675 100644
--- a/arch/arm/boot/dts/imx53-qsb-common.dtsi
+++ b/arch/arm/boot/dts/imx53-qsb-common.dtsi
@@ -45,20 +45,20 @@
 	gpio-keys {
 		compatible = "gpio-keys";
 
-		power {
+		key-power {
 			label = "Power Button";
 			gpios = <&gpio1 8 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_POWER>;
 		};
 
-		volume-up {
+		key-volume-up {
 			label = "Volume Up";
 			gpios = <&gpio2 14 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_VOLUMEUP>;
 			wakeup-source;
 		};
 
-		volume-down {
+		key-volume-down {
 			label = "Volume Down";
 			gpios = <&gpio2 15 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_VOLUMEDOWN>;
@@ -71,7 +71,7 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&led_pin_gpio7_7>;
 
-		user {
+		led-user {
 			label = "Heartbeat";
 			gpios = <&gpio7 7 0>;
 			linux,default-trigger = "heartbeat";
diff --git a/arch/arm/boot/dts/imx53-smd.dts b/arch/arm/boot/dts/imx53-smd.dts
index 9be44e807188..f8d17967a67e 100644
--- a/arch/arm/boot/dts/imx53-smd.dts
+++ b/arch/arm/boot/dts/imx53-smd.dts
@@ -19,13 +19,13 @@
 	gpio-keys {
 		compatible = "gpio-keys";
 
-		volume-up {
+		key-volume-up {
 			label = "Volume Up";
 			gpios = <&gpio2 14 0>;
 			linux,code = <KEY_VOLUMEUP>;
 		};
 
-		volume-down {
+		key-volume-down {
 			label = "Volume Down";
 			gpios = <&gpio2 15 0>;
 			linux,code = <KEY_VOLUMEDOWN>;
diff --git a/arch/arm/boot/dts/imx53-tx53.dtsi b/arch/arm/boot/dts/imx53-tx53.dtsi
index 8712e9851465..892dd1a4bac3 100644
--- a/arch/arm/boot/dts/imx53-tx53.dtsi
+++ b/arch/arm/boot/dts/imx53-tx53.dtsi
@@ -81,7 +81,7 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_gpio_key>;
 
-		power {
+		key-power {
 			label = "Power Button";
 			gpios = <&gpio5 2 GPIO_ACTIVE_HIGH>;
 			linux,code = <116>; /* KEY_POWER */
diff --git a/arch/arm/boot/dts/imx53.dtsi b/arch/arm/boot/dts/imx53.dtsi
index b7a6469d3472..56b3c13f4eb7 100644
--- a/arch/arm/boot/dts/imx53.dtsi
+++ b/arch/arm/boot/dts/imx53.dtsi
@@ -710,7 +710,7 @@
 				status = "disabled";
 			};
 
-			sdma: sdma@63fb0000 {
+			sdma: dma-controller@63fb0000 {
 				compatible = "fsl,imx53-sdma", "fsl,imx35-sdma";
 				reg = <0x63fb0000 0x4000>;
 				interrupts = <6>;
diff --git a/arch/arm/boot/dts/imx6-logicpd-baseboard.dtsi b/arch/arm/boot/dts/imx6-logicpd-baseboard.dtsi
index d9de9b4f0c52..d477a937b47a 100644
--- a/arch/arm/boot/dts/imx6-logicpd-baseboard.dtsi
+++ b/arch/arm/boot/dts/imx6-logicpd-baseboard.dtsi
@@ -6,7 +6,7 @@
 	keyboard {
 		compatible = "gpio-keys";
 
-		btn0 {
+		button-0 {
 			gpios = <&pcf8575 0 GPIO_ACTIVE_LOW>;
 			label = "btn0";
 			linux,code = <KEY_WAKEUP>;
@@ -14,7 +14,7 @@
 			wakeup-source;
 		};
 
-		btn1 {
+		button-1 {
 			gpios = <&pcf8575 1 GPIO_ACTIVE_LOW>;
 			label = "btn1";
 			linux,code = <KEY_WAKEUP>;
@@ -22,7 +22,7 @@
 			wakeup-source;
 		};
 
-		btn2 {
+		button-2 {
 			gpios = <&pcf8575 2 GPIO_ACTIVE_LOW>;
 			label = "btn2";
 			linux,code = <KEY_WAKEUP>;
@@ -30,7 +30,7 @@
 			wakeup-source;
 		};
 
-		btn3 {
+		button-3 {
 			gpios = <&pcf8575 3 GPIO_ACTIVE_LOW>;
 			label = "btn3";
 			linux,code = <KEY_WAKEUP>;
diff --git a/arch/arm/boot/dts/imx6dl-b1x5pv2.dtsi b/arch/arm/boot/dts/imx6dl-b1x5pv2.dtsi
index ec5b66453156..337db29b0010 100644
--- a/arch/arm/boot/dts/imx6dl-b1x5pv2.dtsi
+++ b/arch/arm/boot/dts/imx6dl-b1x5pv2.dtsi
@@ -188,7 +188,7 @@
 	rotary-encoder-key {
 		compatible = "gpio-keys";
 
-		rotary-encoder-press {
+		rotary-encoder-event {
 			label = "rotary-encoder press";
 			gpios = <&tca6424a 0 GPIO_ACTIVE_HIGH>;
 			linux,code = <KEY_ENTER>;
diff --git a/arch/arm/boot/dts/imx6dl-prtmvt.dts b/arch/arm/boot/dts/imx6dl-prtmvt.dts
index a35a1c66e770..1f8cddd83ccb 100644
--- a/arch/arm/boot/dts/imx6dl-prtmvt.dts
+++ b/arch/arm/boot/dts/imx6dl-prtmvt.dts
@@ -51,98 +51,98 @@
 		pinctrl-0 = <&pinctrl_gpiokeys>;
 		autorepeat;
 
-		power {
+		key-power {
 			label = "Power Button";
 			gpios = <&gpio2 23 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_POWER>;
 			wakeup-source;
 		};
 
-		f1 {
+		key-f1 {
 			label = "GPIO Key F1";
 			linux,code = <KEY_F1>;
 			gpios = <&gpio_pca 0 GPIO_ACTIVE_LOW>;
 		};
 
-		f2 {
+		key-f2 {
 			label = "GPIO Key F2";
 			linux,code = <KEY_F2>;
 			gpios = <&gpio_pca 1 GPIO_ACTIVE_LOW>;
 		};
 
-		f3 {
+		key-f3 {
 			label = "GPIO Key F3";
 			linux,code = <KEY_F3>;
 			gpios = <&gpio_pca 2 GPIO_ACTIVE_LOW>;
 		};
 
-		f4 {
+		key-f4 {
 			label = "GPIO Key F4";
 			linux,code = <KEY_F4>;
 			gpios = <&gpio_pca 3 GPIO_ACTIVE_LOW>;
 		};
 
-		f5 {
+		key-f5 {
 			label = "GPIO Key F5";
 			linux,code = <KEY_F5>;
 			gpios = <&gpio_pca 4 GPIO_ACTIVE_LOW>;
 		};
 
-		cycle {
+		key-cycle {
 			label = "GPIO Key CYCLE";
 			linux,code = <KEY_CYCLEWINDOWS>;
 			gpios = <&gpio_pca 5 GPIO_ACTIVE_LOW>;
 		};
 
-		esc {
+		key-esc {
 			label = "GPIO Key ESC";
 			linux,code = <KEY_ESC>;
 			gpios = <&gpio_pca 6 GPIO_ACTIVE_LOW>;
 		};
 
-		up {
+		key-up {
 			label = "GPIO Key UP";
 			linux,code = <KEY_UP>;
 			gpios = <&gpio_pca 7 GPIO_ACTIVE_LOW>;
 		};
 
-		down {
+		key-down {
 			label = "GPIO Key DOWN";
 			linux,code = <KEY_DOWN>;
 			gpios = <&gpio_pca 8 GPIO_ACTIVE_LOW>;
 		};
 
-		ok {
+		key-ok {
 			label = "GPIO Key OK";
 			linux,code = <KEY_OK>;
 			gpios = <&gpio_pca 9 GPIO_ACTIVE_LOW>;
 		};
 
-		f6 {
+		key-f6 {
 			label = "GPIO Key F6";
 			linux,code = <KEY_F6>;
 			gpios = <&gpio_pca 10 GPIO_ACTIVE_LOW>;
 		};
 
-		f7 {
+		key-f7 {
 			label = "GPIO Key F7";
 			linux,code = <KEY_F7>;
 			gpios = <&gpio_pca 11 GPIO_ACTIVE_LOW>;
 		};
 
-		f8 {
+		key-f8 {
 			label = "GPIO Key F8";
 			linux,code = <KEY_F8>;
 			gpios = <&gpio_pca 12 GPIO_ACTIVE_LOW>;
 		};
 
-		f9 {
+		key-f9 {
 			label = "GPIO Key F9";
 			linux,code = <KEY_F9>;
 			gpios = <&gpio_pca 13 GPIO_ACTIVE_LOW>;
 		};
 
-		f10 {
+		key-f10 {
 			label = "GPIO Key F10";
 			linux,code = <KEY_F10>;
 			gpios = <&gpio_pca 14 GPIO_ACTIVE_LOW>;
diff --git a/arch/arm/boot/dts/imx6dl-prtvt7.dts b/arch/arm/boot/dts/imx6dl-prtvt7.dts
index 0a0b7acddfb2..a1eb53851794 100644
--- a/arch/arm/boot/dts/imx6dl-prtvt7.dts
+++ b/arch/arm/boot/dts/imx6dl-prtvt7.dts
@@ -62,91 +62,91 @@
 		compatible = "gpio-keys";
 		autorepeat;
 
-		esc {
+		key-esc {
 			label = "GPIO Key ESC";
 			linux,code = <KEY_ESC>;
 			gpios = <&gpio_pca 0 GPIO_ACTIVE_LOW>;
 		};
 
-		up {
+		key-up {
 			label = "GPIO Key UP";
 			linux,code = <KEY_UP>;
 			gpios = <&gpio_pca 1 GPIO_ACTIVE_LOW>;
 		};
 
-		down {
+		key-down {
 			label = "GPIO Key DOWN";
 			linux,code = <KEY_DOWN>;
 			gpios = <&gpio_pca 4 GPIO_ACTIVE_LOW>;
 		};
 
-		enter {
+		key-enter {
 			label = "GPIO Key Enter";
 			linux,code = <KEY_ENTER>;
 			gpios = <&gpio_pca 3 GPIO_ACTIVE_LOW>;
 		};
 
-		cycle {
+		key-cycle {
 			label = "GPIO Key CYCLE";
 			linux,code = <KEY_CYCLEWINDOWS>;
 			gpios = <&gpio_pca 2 GPIO_ACTIVE_LOW>;
 		};
 
-		f1 {
+		key-f1 {
 			label = "GPIO Key F1";
 			linux,code = <KEY_F1>;
 			gpios = <&gpio_pca 14 GPIO_ACTIVE_LOW>;
 		};
 
-		f2 {
+		key-f2 {
 			label = "GPIO Key F2";
 			linux,code = <KEY_F2>;
 			gpios = <&gpio_pca 13 GPIO_ACTIVE_LOW>;
 		};
 
-		f3 {
+		key-f3 {
 			label = "GPIO Key F3";
 			linux,code = <KEY_F3>;
 			gpios = <&gpio_pca 12 GPIO_ACTIVE_LOW>;
 		};
 
-		f4 {
+		key-f4 {
 			label = "GPIO Key F4";
 			linux,code = <KEY_F4>;
 			gpios = <&gpio_pca 11 GPIO_ACTIVE_LOW>;
 		};
 
-		f5 {
+		key-f5 {
 			label = "GPIO Key F5";
 			linux,code = <KEY_F5>;
 			gpios = <&gpio_pca 10 GPIO_ACTIVE_LOW>;
 		};
 
-		f6 {
+		key-f6 {
 			label = "GPIO Key F6";
 			linux,code = <KEY_F6>;
 			gpios = <&gpio_pca 5 GPIO_ACTIVE_LOW>;
 		};
 
-		f7 {
+		key-f7 {
 			label = "GPIO Key F7";
 			linux,code = <KEY_F7>;
 			gpios = <&gpio_pca 6 GPIO_ACTIVE_LOW>;
 		};
 
-		f8 {
+		key-f8 {
 			label = "GPIO Key F8";
 			linux,code = <KEY_F8>;
 			gpios = <&gpio_pca 7 GPIO_ACTIVE_LOW>;
 		};
 
-		f9 {
+		key-f9 {
 			label = "GPIO Key F9";
 			linux,code = <KEY_F9>;
 			gpios = <&gpio_pca 8 GPIO_ACTIVE_LOW>;
 		};
 
-		f10 {
+		key-f10 {
 			label = "GPIO Key F10";
 			linux,code = <KEY_F10>;
 			gpios = <&gpio_pca 9 GPIO_ACTIVE_LOW>;
diff --git a/arch/arm/boot/dts/imx6dl-riotboard.dts b/arch/arm/boot/dts/imx6dl-riotboard.dts
index e7d9bfbfd0e4..e7be05f205d3 100644
--- a/arch/arm/boot/dts/imx6dl-riotboard.dts
+++ b/arch/arm/boot/dts/imx6dl-riotboard.dts
@@ -90,6 +90,7 @@
 	pinctrl-0 = <&pinctrl_enet>;
 	phy-mode = "rgmii-id";
 	phy-handle = <&rgmii_phy>;
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
 	fsl,err006687-workaround-present;
diff --git a/arch/arm/boot/dts/imx6dl-victgo.dts b/arch/arm/boot/dts/imx6dl-victgo.dts
index 779b52858a25..72df1dba83be 100644
--- a/arch/arm/boot/dts/imx6dl-victgo.dts
+++ b/arch/arm/boot/dts/imx6dl-victgo.dts
@@ -18,14 +18,14 @@
 		pinctrl-0 = <&pinctrl_gpiokeys>;
 		autorepeat;
 
-		power {
+		key-power {
 			label = "Power Button";
 			gpios = <&gpio2 23 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_POWER>;
 			wakeup-source;
 		};
 
-		enter {
+		key-enter {
 			label = "Rotary Key";
 			gpios = <&gpio2 05 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_ENTER>;
diff --git a/arch/arm/boot/dts/imx6dl-yapp4-common.dtsi b/arch/arm/boot/dts/imx6dl-yapp4-common.dtsi
index 674af39c884a..52162e8c7274 100644
--- a/arch/arm/boot/dts/imx6dl-yapp4-common.dtsi
+++ b/arch/arm/boot/dts/imx6dl-yapp4-common.dtsi
@@ -55,6 +55,7 @@
 	panel: panel {
 		compatible = "dataimage,scf0700c48ggu18";
 		power-supply = <&sw2_reg>;
+		backlight = <&backlight>;
 		status = "disabled";
 
 		port {
diff --git a/arch/arm/boot/dts/imx6dl.dtsi b/arch/arm/boot/dts/imx6dl.dtsi
index 8e0ed209ede0..dc919e09a505 100644
--- a/arch/arm/boot/dts/imx6dl.dtsi
+++ b/arch/arm/boot/dts/imx6dl.dtsi
@@ -84,6 +84,9 @@
 		ocram: sram@900000 {
 			compatible = "mmio-sram";
 			reg = <0x00900000 0x20000>;
+			ranges = <0 0x00900000 0x20000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
 			clocks = <&clks IMX6QDL_CLK_OCRAM>;
 		};
 
diff --git a/arch/arm/boot/dts/imx6q-arm2.dts b/arch/arm/boot/dts/imx6q-arm2.dts
index 0b40f52268b3..75586299d9ca 100644
--- a/arch/arm/boot/dts/imx6q-arm2.dts
+++ b/arch/arm/boot/dts/imx6q-arm2.dts
@@ -178,6 +178,7 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_enet>;
 	phy-mode = "rgmii";
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
 	fsl,err006687-workaround-present;
diff --git a/arch/arm/boot/dts/imx6q-evi.dts b/arch/arm/boot/dts/imx6q-evi.dts
index c63f371ede8b..78d941fef5df 100644
--- a/arch/arm/boot/dts/imx6q-evi.dts
+++ b/arch/arm/boot/dts/imx6q-evi.dts
@@ -146,6 +146,7 @@
 	pinctrl-0 = <&pinctrl_enet>;
 	phy-mode = "rgmii";
 	phy-reset-gpios = <&gpio1 25 GPIO_ACTIVE_LOW>;
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
 	fsl,err006687-workaround-present;
diff --git a/arch/arm/boot/dts/imx6q-mccmon6.dts b/arch/arm/boot/dts/imx6q-mccmon6.dts
index 55692c73943d..f08b37010291 100644
--- a/arch/arm/boot/dts/imx6q-mccmon6.dts
+++ b/arch/arm/boot/dts/imx6q-mccmon6.dts
@@ -100,8 +100,10 @@
 	pinctrl-0 = <&pinctrl_enet>;
 	phy-mode = "rgmii";
 	phy-reset-gpios = <&gpio1 27 GPIO_ACTIVE_LOW>;
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
+	fsl,err006687-workaround-present;
 	status = "okay";
 };
 
diff --git a/arch/arm/boot/dts/imx6q-novena.dts b/arch/arm/boot/dts/imx6q-novena.dts
index 225cf6b7a7a4..ee8c0bd3ecfd 100644
--- a/arch/arm/boot/dts/imx6q-novena.dts
+++ b/arch/arm/boot/dts/imx6q-novena.dts
@@ -86,7 +86,7 @@
 			linux,code = <KEY_POWER>;
 		};
 
-		lid {
+		lid-event {
 			label = "Lid";
 			gpios = <&gpio4 12 GPIO_ACTIVE_LOW>;
 			linux,input-type = <5>;	/* EV_SW */
@@ -99,7 +99,7 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_leds_novena>;
 
-		heartbeat {
+		led-heartbeat {
 			label = "novena:white:panel";
 			gpios = <&gpio1 21 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "default-on";
diff --git a/arch/arm/boot/dts/imx6q-pistachio.dts b/arch/arm/boot/dts/imx6q-pistachio.dts
index 7a33e54cc0f1..bad8d831e64e 100644
--- a/arch/arm/boot/dts/imx6q-pistachio.dts
+++ b/arch/arm/boot/dts/imx6q-pistachio.dts
@@ -100,7 +100,7 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_gpio_keys>;
 
-		power {
+		key-power {
 			label = "Power Button";
 			gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
 			wakeup-source;
diff --git a/arch/arm/boot/dts/imx6q-sabrelite.dts b/arch/arm/boot/dts/imx6q-sabrelite.dts
index dc51262e7b2f..7c6a2f234ccb 100644
--- a/arch/arm/boot/dts/imx6q-sabrelite.dts
+++ b/arch/arm/boot/dts/imx6q-sabrelite.dts
@@ -1,43 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0 OR X11
 /*
  * Copyright 2011 Freescale Semiconductor, Inc.
  * Copyright 2011 Linaro Ltd.
  *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License
- *     version 2 as published by the Free Software Foundation.
- *
- *     This file is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/imx6q-utilite-pro.dts b/arch/arm/boot/dts/imx6q-utilite-pro.dts
index d16ff2083d62..ad59b23ef27a 100644
--- a/arch/arm/boot/dts/imx6q-utilite-pro.dts
+++ b/arch/arm/boot/dts/imx6q-utilite-pro.dts
@@ -89,7 +89,7 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_gpio_keys>;
 
-		power {
+		key-power {
 			label = "Power Button";
 			gpios = <&gpio1 29 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_POWER>;
diff --git a/arch/arm/boot/dts/imx6q-var-dt6customboard.dts b/arch/arm/boot/dts/imx6q-var-dt6customboard.dts
index 63550351340d..2290c1237634 100644
--- a/arch/arm/boot/dts/imx6q-var-dt6customboard.dts
+++ b/arch/arm/boot/dts/imx6q-var-dt6customboard.dts
@@ -28,7 +28,7 @@
 		compatible = "gpio-keys";
 		autorepeat;
 
-		back {
+		key-back {
 			gpios = <&gpio4 26 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_BACK>;
 			label = "Key Back";
@@ -37,7 +37,7 @@
 			wakeup-source;
 		};
 
-		home {
+		key-home {
 			gpios = <&gpio5 11 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_HOME>;
 			label = "Key Home";
@@ -46,7 +46,7 @@
 			wakeup-source;
 		};
 
-		menu {
+		key-menu {
 			gpios = <&gpio4 25 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_MENU>;
 			label = "Key Menu";
diff --git a/arch/arm/boot/dts/imx6q.dtsi b/arch/arm/boot/dts/imx6q.dtsi
index 3b77eae40e39..df86049a695b 100644
--- a/arch/arm/boot/dts/imx6q.dtsi
+++ b/arch/arm/boot/dts/imx6q.dtsi
@@ -163,6 +163,9 @@
 		ocram: sram@900000 {
 			compatible = "mmio-sram";
 			reg = <0x00900000 0x40000>;
+			ranges = <0 0x00900000 0x40000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
 			clocks = <&clks IMX6QDL_CLK_OCRAM>;
 		};
 
diff --git a/arch/arm/boot/dts/imx6qdl-dhcom-pdk2.dtsi b/arch/arm/boot/dts/imx6qdl-dhcom-pdk2.dtsi
index fe72650295a5..6248b126b557 100644
--- a/arch/arm/boot/dts/imx6qdl-dhcom-pdk2.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-dhcom-pdk2.dtsi
@@ -332,37 +332,4 @@
 			MX6QDL_PAD_GPIO_0__GPIO1_IO00		0xb1 /* Int */
 		>;
 	};
-
-	pinctrl_ipu1_lcdif: ipu1-lcdif-grp {
-		fsl,pins = <
-			MX6QDL_PAD_DI0_DISP_CLK__IPU1_DI0_DISP_CLK	0x38
-			MX6QDL_PAD_DI0_PIN2__IPU1_DI0_PIN02		0x38
-			MX6QDL_PAD_DI0_PIN3__IPU1_DI0_PIN03		0x38
-			MX6QDL_PAD_DI0_PIN15__IPU1_DI0_PIN15		0x38
-			MX6QDL_PAD_DISP0_DAT0__IPU1_DISP0_DATA00	0x38
-			MX6QDL_PAD_DISP0_DAT1__IPU1_DISP0_DATA01	0x38
-			MX6QDL_PAD_DISP0_DAT2__IPU1_DISP0_DATA02	0x38
-			MX6QDL_PAD_DISP0_DAT3__IPU1_DISP0_DATA03	0x38
-			MX6QDL_PAD_DISP0_DAT4__IPU1_DISP0_DATA04	0x38
-			MX6QDL_PAD_DISP0_DAT5__IPU1_DISP0_DATA05	0x38
-			MX6QDL_PAD_DISP0_DAT6__IPU1_DISP0_DATA06	0x38
-			MX6QDL_PAD_DISP0_DAT7__IPU1_DISP0_DATA07	0x38
-			MX6QDL_PAD_DISP0_DAT8__IPU1_DISP0_DATA08	0x38
-			MX6QDL_PAD_DISP0_DAT9__IPU1_DISP0_DATA09	0x38
-			MX6QDL_PAD_DISP0_DAT10__IPU1_DISP0_DATA10	0x38
-			MX6QDL_PAD_DISP0_DAT11__IPU1_DISP0_DATA11	0x38
-			MX6QDL_PAD_DISP0_DAT12__IPU1_DISP0_DATA12	0x38
-			MX6QDL_PAD_DISP0_DAT13__IPU1_DISP0_DATA13	0x38
-			MX6QDL_PAD_DISP0_DAT14__IPU1_DISP0_DATA14	0x38
-			MX6QDL_PAD_DISP0_DAT15__IPU1_DISP0_DATA15	0x38
-			MX6QDL_PAD_DISP0_DAT16__IPU1_DISP0_DATA16	0x38
-			MX6QDL_PAD_DISP0_DAT17__IPU1_DISP0_DATA17	0x38
-			MX6QDL_PAD_DISP0_DAT18__IPU1_DISP0_DATA18	0x38
-			MX6QDL_PAD_DISP0_DAT19__IPU1_DISP0_DATA19	0x38
-			MX6QDL_PAD_DISP0_DAT20__IPU1_DISP0_DATA20	0x38
-			MX6QDL_PAD_DISP0_DAT21__IPU1_DISP0_DATA21	0x38
-			MX6QDL_PAD_DISP0_DAT22__IPU1_DISP0_DATA22	0x38
-			MX6QDL_PAD_DISP0_DAT23__IPU1_DISP0_DATA23	0x38
-		>;
-	};
 };
diff --git a/arch/arm/boot/dts/imx6qdl-dhcom-som.dtsi b/arch/arm/boot/dts/imx6qdl-dhcom-som.dtsi
index 5befbe13d1a3..eaa87b333164 100644
--- a/arch/arm/boot/dts/imx6qdl-dhcom-som.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-dhcom-som.dtsi
@@ -667,6 +667,39 @@
 		>;
 	};
 
+	pinctrl_ipu1_lcdif: ipu1-lcdif-grp {
+		fsl,pins = <
+			MX6QDL_PAD_DI0_DISP_CLK__IPU1_DI0_DISP_CLK	0x38
+			MX6QDL_PAD_DI0_PIN2__IPU1_DI0_PIN02		0x38
+			MX6QDL_PAD_DI0_PIN3__IPU1_DI0_PIN03		0x38
+			MX6QDL_PAD_DI0_PIN15__IPU1_DI0_PIN15		0x38
+			MX6QDL_PAD_DISP0_DAT0__IPU1_DISP0_DATA00	0x38
+			MX6QDL_PAD_DISP0_DAT1__IPU1_DISP0_DATA01	0x38
+			MX6QDL_PAD_DISP0_DAT2__IPU1_DISP0_DATA02	0x38
+			MX6QDL_PAD_DISP0_DAT3__IPU1_DISP0_DATA03	0x38
+			MX6QDL_PAD_DISP0_DAT4__IPU1_DISP0_DATA04	0x38
+			MX6QDL_PAD_DISP0_DAT5__IPU1_DISP0_DATA05	0x38
+			MX6QDL_PAD_DISP0_DAT6__IPU1_DISP0_DATA06	0x38
+			MX6QDL_PAD_DISP0_DAT7__IPU1_DISP0_DATA07	0x38
+			MX6QDL_PAD_DISP0_DAT8__IPU1_DISP0_DATA08	0x38
+			MX6QDL_PAD_DISP0_DAT9__IPU1_DISP0_DATA09	0x38
+			MX6QDL_PAD_DISP0_DAT10__IPU1_DISP0_DATA10	0x38
+			MX6QDL_PAD_DISP0_DAT11__IPU1_DISP0_DATA11	0x38
+			MX6QDL_PAD_DISP0_DAT12__IPU1_DISP0_DATA12	0x38
+			MX6QDL_PAD_DISP0_DAT13__IPU1_DISP0_DATA13	0x38
+			MX6QDL_PAD_DISP0_DAT14__IPU1_DISP0_DATA14	0x38
+			MX6QDL_PAD_DISP0_DAT15__IPU1_DISP0_DATA15	0x38
+			MX6QDL_PAD_DISP0_DAT16__IPU1_DISP0_DATA16	0x38
+			MX6QDL_PAD_DISP0_DAT17__IPU1_DISP0_DATA17	0x38
+			MX6QDL_PAD_DISP0_DAT18__IPU1_DISP0_DATA18	0x38
+			MX6QDL_PAD_DISP0_DAT19__IPU1_DISP0_DATA19	0x38
+			MX6QDL_PAD_DISP0_DAT20__IPU1_DISP0_DATA20	0x38
+			MX6QDL_PAD_DISP0_DAT21__IPU1_DISP0_DATA21	0x38
+			MX6QDL_PAD_DISP0_DAT22__IPU1_DISP0_DATA22	0x38
+			MX6QDL_PAD_DISP0_DAT23__IPU1_DISP0_DATA23	0x38
+		>;
+	};
+
 	pinctrl_pcie: pcie-grp {
 		fsl,pins = <
 			MX6QDL_PAD_CSI0_DATA_EN__GPIO5_IO20	0x1b0b1 /* Wake */
diff --git a/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi
index b1df2beb2832..728810b9d677 100644
--- a/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-gw52xx.dtsi
@@ -137,6 +137,16 @@
 		regulator-always-on;
 	};
 
+	reg_can1_stby: regulator-can1-stby {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_can1>;
+		regulator-name = "can1_stby";
+		gpio = <&gpio1 9 GPIO_ACTIVE_LOW>;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
 	reg_usb_otg_vbus: regulator-usb-otg-vbus {
 		compatible = "regulator-fixed";
 		regulator-name = "usb_otg_vbus";
@@ -170,6 +180,7 @@
 &can1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_flexcan1>;
+	xceiver-supply = <&reg_can1_stby>;
 	status = "okay";
 };
 
@@ -612,7 +623,6 @@
 		fsl,pins = <
 			MX6QDL_PAD_KEY_ROW2__FLEXCAN1_RX	0x1b0b1
 			MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX	0x1b0b1
-			MX6QDL_PAD_GPIO_9__GPIO1_IO09		0x4001b0b0 /* CAN_STBY */
 		>;
 	};
 
@@ -702,6 +712,12 @@
 		>;
 	};
 
+	pinctrl_reg_can1: regcan1grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_9__GPIO1_IO09		0x4001b0b0 /* CAN_STBY */
+		>;
+	};
+
 	pinctrl_uart1: uart1grp {
 		fsl,pins = <
 			MX6QDL_PAD_SD3_DAT7__UART1_TX_DATA	0x1b0b1
diff --git a/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi
index a0710d562766..6c0c109046d8 100644
--- a/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-gw53xx.dtsi
@@ -129,6 +129,16 @@
 		regulator-always-on;
 	};
 
+	reg_can1_stby: regulator-can1-stby {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_can1>;
+		regulator-name = "can1_stby";
+		gpio = <&gpio1 2 GPIO_ACTIVE_LOW>;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
 	reg_usb_h1_vbus: regulator-usb-h1-vbus {
 		compatible = "regulator-fixed";
 		regulator-name = "usb_h1_vbus";
@@ -170,6 +180,7 @@
 &can1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_flexcan1>;
+	xceiver-supply = <&reg_can1_stby>;
 	status = "okay";
 };
 
@@ -600,7 +611,6 @@
 		fsl,pins = <
 			MX6QDL_PAD_KEY_ROW2__FLEXCAN1_RX	0x1b0b1
 			MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX	0x1b0b1
-			MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x4001b0b0 /* CAN_STBY */
 		>;
 	};
 
@@ -691,6 +701,12 @@
 		>;
 	};
 
+	pinctrl_reg_can1: regcan1grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x4001b0b0 /* CAN_STBY */
+		>;
+	};
+
 	pinctrl_uart1: uart1grp {
 		fsl,pins = <
 			MX6QDL_PAD_SD3_DAT7__UART1_TX_DATA	0x1b0b1
diff --git a/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi b/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi
index cda48bf2f168..a9b04f9f1c2b 100644
--- a/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-gw54xx.dtsi
@@ -137,6 +137,16 @@
 			regulator-always-on;
 		};
 
+		reg_can1_stby: regulator-can1-stby {
+			compatible = "regulator-fixed";
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_reg_can1>;
+			regulator-name = "can1_stby";
+			gpio = <&gpio1 2 GPIO_ACTIVE_LOW>;
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3300000>;
+		};
+
 		reg_usb_h1_vbus: regulator@2 {
 			compatible = "regulator-fixed";
 			reg = <2>;
@@ -200,6 +210,7 @@
 &can1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_flexcan1>;
+	xceiver-supply = <&reg_can1_stby>;
 	status = "okay";
 };
 
@@ -687,7 +698,6 @@
 		fsl,pins = <
 			MX6QDL_PAD_KEY_ROW2__FLEXCAN1_RX	0x1b0b1
 			MX6QDL_PAD_KEY_COL2__FLEXCAN1_TX	0x1b0b1
-			MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x4001b0b0 /* CAN_STBY */
 		>;
 	};
 
@@ -786,6 +796,12 @@
 		>;
 	};
 
+	pinctrl_reg_can1: regcan1grp {
+		fsl,pins = <
+			MX6QDL_PAD_GPIO_2__GPIO1_IO02		0x4001b0b0 /* CAN_STBY */
+		>;
+	};
+
 	pinctrl_uart1: uart1grp {
 		fsl,pins = <
 			MX6QDL_PAD_SD3_DAT7__UART1_TX_DATA	0x1b0b1
diff --git a/arch/arm/boot/dts/imx6qdl-kontron-samx6i.dtsi b/arch/arm/boot/dts/imx6qdl-kontron-samx6i.dtsi
index 6b791d515e29..85aeebc9485d 100644
--- a/arch/arm/boot/dts/imx6qdl-kontron-samx6i.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-kontron-samx6i.dtsi
@@ -260,7 +260,23 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_enet>;
 	phy-mode = "rgmii";
-	phy-reset-gpios = <&gpio1 25 GPIO_ACTIVE_LOW>;
+	phy-handle = <&ethphy>;
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			reset-gpios = <&gpio1 25 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <1000>;
+		};
+	};
+};
+
+&hdmi {
+	ddc-i2c-bus = <&i2c2>;
 };
 
 &i2c_intern {
@@ -387,7 +403,7 @@
 
 /* HDMI_CTRL */
 &i2c2 {
-	clock-frequency = <375000>;
+	clock-frequency = <100000>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_i2c2>;
 };
diff --git a/arch/arm/boot/dts/imx6qdl-mba6.dtsi b/arch/arm/boot/dts/imx6qdl-mba6.dtsi
index f4dca20669d6..78555a618851 100644
--- a/arch/arm/boot/dts/imx6qdl-mba6.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-mba6.dtsi
@@ -244,7 +244,6 @@
 	status = "okay";
 };
 
-
 &uart3 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart3>;
diff --git a/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi b/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi
index 0ad4cb4f1e82..a53a5d0766a5 100644
--- a/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nit6xlite.dtsi
@@ -192,6 +192,7 @@
 	phy-mode = "rgmii";
 	phy-handle = <&ethphy>;
 	phy-reset-gpios = <&gpio1 27 GPIO_ACTIVE_LOW>;
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
 	fsl,err006687-workaround-present;
diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
index beaa2dcd436c..57c21a01f126 100644
--- a/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nitrogen6_max.dtsi
@@ -334,6 +334,7 @@
 	phy-mode = "rgmii";
 	phy-handle = <&ethphy>;
 	phy-reset-gpios = <&gpio1 27 GPIO_ACTIVE_LOW>;
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
 	fsl,err006687-workaround-present;
diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi
index ee7e2371f94b..000e9dc97b1a 100644
--- a/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nitrogen6_som2.dtsi
@@ -263,6 +263,7 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_enet>;
 	phy-mode = "rgmii";
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
 	fsl,err006687-workaround-present;
diff --git a/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi b/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi
index 904d5d051d63..731759bdd7f5 100644
--- a/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-nitrogen6x.dtsi
@@ -267,6 +267,7 @@
 	phy-mode = "rgmii";
 	phy-handle = <&ethphy>;
 	phy-reset-gpios = <&gpio1 27 GPIO_ACTIVE_LOW>;
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
 	fsl,err006687-workaround-present;
diff --git a/arch/arm/boot/dts/imx6qdl-phytec-mira-peb-av-02.dtsi b/arch/arm/boot/dts/imx6qdl-phytec-mira-peb-av-02.dtsi
index 393475cb0f4d..0020dbb1722c 100644
--- a/arch/arm/boot/dts/imx6qdl-phytec-mira-peb-av-02.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-phytec-mira-peb-av-02.dtsi
@@ -64,7 +64,7 @@
 		interrupt-parent = <&gpio3>;
 		interrupts = <2 IRQ_TYPE_NONE>;
 		status = "disabled";
-        };
+	};
 };
 
 &ipu1_di0_disp0 {
diff --git a/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi b/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
index 1368a4762037..3dbb460ef102 100644
--- a/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sabreauto.dtsi
@@ -295,6 +295,7 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_enet>;
 	phy-mode = "rgmii-id";
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
 	fsl,err006687-workaround-present;
diff --git a/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi b/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi
index 901b9a761b66..22f8e2783cdf 100644
--- a/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-sabrelite.dtsi
@@ -1,43 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0 OR X11
 /*
  * Copyright 2011 Freescale Semiconductor, Inc.
  * Copyright 2011 Linaro Ltd.
  *
- * This file is dual-licensed: you can use it either under the terms
- * of the GPL or the X11 license, at your option. Note that this dual
- * licensing only applies to this file, and not this project as a
- * whole.
- *
- *  a) This file is free software; you can redistribute it and/or
- *     modify it under the terms of the GNU General Public License
- *     version 2 as published by the Free Software Foundation.
- *
- *     This file is distributed in the hope that it will be useful,
- *     but WITHOUT ANY WARRANTY; without even the implied warranty of
- *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *     GNU General Public License for more details.
- *
- * Or, alternatively,
- *
- *  b) Permission is hereby granted, free of charge, to any person
- *     obtaining a copy of this software and associated documentation
- *     files (the "Software"), to deal in the Software without
- *     restriction, including without limitation the rights to use,
- *     copy, modify, merge, publish, distribute, sublicense, and/or
- *     sell copies of the Software, and to permit persons to whom the
- *     Software is furnished to do so, subject to the following
- *     conditions:
- *
- *     The above copyright notice and this permission notice shall be
- *     included in all copies or substantial portions of the Software.
- *
- *     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- *     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- *     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- *     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- *     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- *     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- *     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- *     OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #include <dt-bindings/clock/imx6qdl-clock.h>
diff --git a/arch/arm/boot/dts/imx6qdl-skov-cpu-revc.dtsi b/arch/arm/boot/dts/imx6qdl-skov-cpu-revc.dtsi
index 8254bce1b8a2..b81799d7076a 100644
--- a/arch/arm/boot/dts/imx6qdl-skov-cpu-revc.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-skov-cpu-revc.dtsi
@@ -2,35 +2,60 @@
 //
 // Copyright (C) 2020 Pengutronix, Ulrich Oelmann <kernel@pengutronix.de>
 
+/ {
+	touchscreen {
+		compatible = "resistive-adc-touch";
+		io-channels = <&adc_ts 1>, <&adc_ts 3>, <&adc_ts 4>, <&adc_ts 5>;
+		io-channel-names = "y", "z1", "z2", "x";
+		touchscreen-min-pressure = <65000>;
+		touchscreen-inverted-y;
+		touchscreen-swapped-x-y;
+		touchscreen-x-plate-ohms = <300>;
+		touchscreen-y-plate-ohms = <800>;
+	};
+};
+
 &ecspi4 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_ecspi4>;
 	cs-gpios = <&gpio3 20 GPIO_ACTIVE_LOW>;
 	status = "okay";
 
-	touchscreen@0 {
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_touch>;
-		compatible = "ti,tsc2046";
+	adc_ts: adc@0 {
+		compatible = "ti,tsc2046e-adc";
 		reg = <0>;
+		pinctrl-0 = <&pinctrl_touch>;
+		pinctrl-names ="default";
 		spi-max-frequency = <1000000>;
 		interrupts-extended = <&gpio3 19 IRQ_TYPE_LEVEL_LOW>;
-		vcc-supply = <&reg_3v3>;
-		pendown-gpio = <&gpio3 19 GPIO_ACTIVE_LOW>;
-		ti,x-plate-ohms = /bits/ 16 <850>;
-		ti,y-plate-ohms = /bits/ 16 <295>;
-		ti,pressure-min = /bits/ 16 <2>;
-		ti,pressure-max = /bits/ 16 <1500>;
-		ti,vref-mv = /bits/ 16 <3300>;
-		ti,settle-delay-usec = /bits/ 16 <15>;
-		ti,vref-delay-usecs = /bits/ 16 <0>;
-		ti,penirq-recheck-delay-usecs = /bits/ 16 <100>;
-		ti,debounce-max = /bits/ 16 <100>;
-		ti,debounce-tol = /bits/ 16 <(~0)>;
-		ti,debounce-rep = /bits/ 16 <4>;
-		touchscreen-swapped-x-y;
-		touchscreen-inverted-y;
-		wakeup-source;
+		#io-channel-cells = <1>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		channel@1 {
+			reg = <1>;
+			settling-time-us = <700>;
+			oversampling-ratio = <5>;
+		};
+
+		channel@3 {
+			reg = <3>;
+			settling-time-us = <700>;
+			oversampling-ratio = <5>;
+		};
+
+		channel@4 {
+			reg = <4>;
+			settling-time-us = <700>;
+			oversampling-ratio = <5>;
+		};
+
+		channel@5 {
+			reg = <5>;
+			settling-time-us = <700>;
+			oversampling-ratio = <5>;
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/imx6qdl-tqma6a.dtsi b/arch/arm/boot/dts/imx6qdl-tqma6a.dtsi
index 7dc3f0005b0f..aff46f3040c1 100644
--- a/arch/arm/boot/dts/imx6qdl-tqma6a.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-tqma6a.dtsi
@@ -7,6 +7,7 @@
 #include <dt-bindings/gpio/gpio.h>
 
 &fec {
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
 	fsl,err006687-workaround-present;
@@ -35,6 +36,7 @@
 		compatible = "st,24c64", "atmel,24c64";
 		reg = <0x50>;
 		pagesize = <32>;
+		vcc-supply = <&reg_3p3v>;
 	};
 };
 
diff --git a/arch/arm/boot/dts/imx6qdl-tqma6b.dtsi b/arch/arm/boot/dts/imx6qdl-tqma6b.dtsi
index dd0925766440..a3f6543c3aaa 100644
--- a/arch/arm/boot/dts/imx6qdl-tqma6b.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-tqma6b.dtsi
@@ -29,5 +29,6 @@
 		compatible = "st,24c64", "atmel,24c64";
 		reg = <0x50>;
 		pagesize = <32>;
+		vcc-supply = <&reg_3p3v>;
 	};
 };
diff --git a/arch/arm/boot/dts/imx6qdl-ts7970.dtsi b/arch/arm/boot/dts/imx6qdl-ts7970.dtsi
index d6ba4b2a60f6..c096d25a6f5b 100644
--- a/arch/arm/boot/dts/imx6qdl-ts7970.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-ts7970.dtsi
@@ -192,6 +192,7 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_enet>;
 	phy-mode = "rgmii";
+	/delete-property/ interrupts;
 	interrupts-extended = <&gpio1 6 IRQ_TYPE_LEVEL_HIGH>,
 			      <&intc 0 119 IRQ_TYPE_LEVEL_HIGH>;
 	fsl,err006687-workaround-present;
diff --git a/arch/arm/boot/dts/imx6qdl-vicut1.dtsi b/arch/arm/boot/dts/imx6qdl-vicut1.dtsi
index c5a98b0110dd..c4e6cf0527ba 100644
--- a/arch/arm/boot/dts/imx6qdl-vicut1.dtsi
+++ b/arch/arm/boot/dts/imx6qdl-vicut1.dtsi
@@ -37,6 +37,16 @@
 		power-supply = <&reg_3v3>;
 	};
 
+	/* only for backwards compatibility with old HW */
+	backlight_isb: backlight-isb {
+		compatible = "pwm-backlight";
+		pwms = <&pwm2 0 5000000 0>;
+		brightness-levels = <0 8 48 255>;
+		num-interpolated-steps = <5>;
+		default-brightness-level = <0>;
+		power-supply = <&reg_3v3>;
+	};
+
 	connector {
 		compatible = "composite-video-connector";
 		label = "Composite0";
@@ -370,6 +380,12 @@
 	status = "okay";
 };
 
+&pwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+	status = "okay";
+};
+
 &pwm3 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_pwm3>;
@@ -601,6 +617,12 @@
 		>;
 	};
 
+	pinctrl_pwm2: pwm2grp {
+		fsl,pins = <
+			MX6QDL_PAD_DISP0_DAT9__PWM2_OUT			0x1b0b0
+		>;
+	};
+
 	pinctrl_pwm3: pwm3grp {
 		fsl,pins = <
 			MX6QDL_PAD_SD4_DAT1__PWM3_OUT			0x1b0b0
diff --git a/arch/arm/boot/dts/imx6qdl.dtsi b/arch/arm/boot/dts/imx6qdl.dtsi
index 4f7fefc14d0a..ff1e0173b39b 100644
--- a/arch/arm/boot/dts/imx6qdl.dtsi
+++ b/arch/arm/boot/dts/imx6qdl.dtsi
@@ -929,7 +929,7 @@
 				interrupts = <0 125 IRQ_TYPE_LEVEL_HIGH>;
 			};
 
-			sdma: sdma@20ec000 {
+			sdma: dma-controller@20ec000 {
 				compatible = "fsl,imx6q-sdma", "fsl,imx35-sdma";
 				reg = <0x020ec000 0x4000>;
 				interrupts = <0 2 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/imx6qp.dtsi b/arch/arm/boot/dts/imx6qp.dtsi
index 050365513836..fc164991d2ae 100644
--- a/arch/arm/boot/dts/imx6qp.dtsi
+++ b/arch/arm/boot/dts/imx6qp.dtsi
@@ -9,12 +9,18 @@
 		ocram2: sram@940000 {
 			compatible = "mmio-sram";
 			reg = <0x00940000 0x20000>;
+			ranges = <0 0x00940000 0x20000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
 			clocks = <&clks IMX6QDL_CLK_OCRAM>;
 		};
 
 		ocram3: sram@960000 {
 			compatible = "mmio-sram";
 			reg = <0x00960000 0x20000>;
+			ranges = <0 0x00960000 0x20000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
 			clocks = <&clks IMX6QDL_CLK_OCRAM>;
 		};
 
diff --git a/arch/arm/boot/dts/imx6sl.dtsi b/arch/arm/boot/dts/imx6sl.dtsi
index 06a515121dfc..28111efb19a6 100644
--- a/arch/arm/boot/dts/imx6sl.dtsi
+++ b/arch/arm/boot/dts/imx6sl.dtsi
@@ -61,10 +61,10 @@
 				<792000  1175000>,
 				<396000  975000>;
 			fsl,soc-operating-points =
-				/* ARM kHz      SOC-PU uV */
-				<996000         1225000>,
-				<792000         1175000>,
-				<396000         1175000>;
+				/* ARM kHz	SOC-PU uV */
+				<996000		1225000>,
+				<792000		1175000>,
+				<396000		1175000>;
 			clock-latency = <61036>; /* two CLK32 periods */
 			#cooling-cells = <2>;
 			clocks = <&clks IMX6SL_CLK_ARM>, <&clks IMX6SL_CLK_PLL2_PFD2>,
@@ -115,6 +115,9 @@
 		ocram: sram@900000 {
 			compatible = "mmio-sram";
 			reg = <0x00900000 0x20000>;
+			ranges = <0 0x00900000 0x20000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
 			clocks = <&clks IMX6SL_CLK_OCRAM>;
 		};
 
@@ -222,7 +225,7 @@
 
 				uart5: serial@2018000 {
 					compatible = "fsl,imx6sl-uart",
-						   "fsl,imx6q-uart", "fsl,imx21-uart";
+						     "fsl,imx6q-uart", "fsl,imx21-uart";
 					reg = <0x02018000 0x4000>;
 					interrupts = <0 30 IRQ_TYPE_LEVEL_HIGH>;
 					clocks = <&clks IMX6SL_CLK_UART>,
@@ -235,7 +238,7 @@
 
 				uart1: serial@2020000 {
 					compatible = "fsl,imx6sl-uart",
-						   "fsl,imx6q-uart", "fsl,imx21-uart";
+						     "fsl,imx6q-uart", "fsl,imx21-uart";
 					reg = <0x02020000 0x4000>;
 					interrupts = <0 26 IRQ_TYPE_LEVEL_HIGH>;
 					clocks = <&clks IMX6SL_CLK_UART>,
@@ -248,7 +251,7 @@
 
 				uart2: serial@2024000 {
 					compatible = "fsl,imx6sl-uart",
-						   "fsl,imx6q-uart", "fsl,imx21-uart";
+						     "fsl,imx6q-uart", "fsl,imx21-uart";
 					reg = <0x02024000 0x4000>;
 					interrupts = <0 27 IRQ_TYPE_LEVEL_HIGH>;
 					clocks = <&clks IMX6SL_CLK_UART>,
@@ -309,7 +312,7 @@
 
 				uart3: serial@2034000 {
 					compatible = "fsl,imx6sl-uart",
-						   "fsl,imx6q-uart", "fsl,imx21-uart";
+						     "fsl,imx6q-uart", "fsl,imx21-uart";
 					reg = <0x02034000 0x4000>;
 					interrupts = <0 28 IRQ_TYPE_LEVEL_HIGH>;
 					clocks = <&clks IMX6SL_CLK_UART>,
@@ -322,7 +325,7 @@
 
 				uart4: serial@2038000 {
 					compatible = "fsl,imx6sl-uart",
-						   "fsl,imx6q-uart", "fsl,imx21-uart";
+						     "fsl,imx6q-uart", "fsl,imx21-uart";
 					reg = <0x02038000 0x4000>;
 					interrupts = <0 29 IRQ_TYPE_LEVEL_HIGH>;
 					clocks = <&clks IMX6SL_CLK_UART>,
@@ -711,7 +714,7 @@
 						#power-domain-cells = <0>;
 						power-supply = <&reg_pu>;
 						clocks = <&clks IMX6SL_CLK_GPU2D_OVG>,
-						         <&clks IMX6SL_CLK_GPU2D_PODF>;
+							 <&clks IMX6SL_CLK_GPU2D_PODF>;
 					};
 
 					pd_disp: power-domain@2 {
@@ -747,7 +750,7 @@
 				interrupts = <0 6 IRQ_TYPE_LEVEL_HIGH>;
 			};
 
-			sdma: sdma@20ec000 {
+			sdma: dma-controller@20ec000 {
 				compatible = "fsl,imx6sl-sdma", "fsl,imx6q-sdma";
 				reg = <0x020ec000 0x4000>;
 				interrupts = <0 2 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/imx6sll.dtsi b/arch/arm/boot/dts/imx6sll.dtsi
index d4a000c3dde7..2873369a57c0 100644
--- a/arch/arm/boot/dts/imx6sll.dtsi
+++ b/arch/arm/boot/dts/imx6sll.dtsi
@@ -115,6 +115,9 @@
 		ocram: sram@900000 {
 			compatible = "mmio-sram";
 			reg = <0x00900000 0x20000>;
+			ranges = <0 0x00900000 0x20000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
 		};
 
 		intc: interrupt-controller@a01000 {
diff --git a/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi b/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi
index 35861bbea94e..c84ea1fac5e9 100644
--- a/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi
+++ b/arch/arm/boot/dts/imx6sx-udoo-neo.dtsi
@@ -226,7 +226,7 @@
 &iomuxc {
 	pinctrl_bt_reg: btreggrp {
 		fsl,pins =
-			<MX6SX_PAD_KEY_ROW2__GPIO2_IO_17        0x15059>;
+			<MX6SX_PAD_KEY_ROW2__GPIO2_IO_17	0x15059>;
 	};
 
 	pinctrl_enet1: enet1grp {
@@ -306,7 +306,6 @@
 		>;
 	};
 
-
 	pinctrl_uart1: uart1grp {
 		fsl,pins =
 			<MX6SX_PAD_GPIO1_IO04__UART1_DCE_TX	0x1b0b1>,
@@ -347,24 +346,23 @@
 
 	pinctrl_otg1_reg: otg1grp {
 		fsl,pins =
-			<MX6SX_PAD_GPIO1_IO09__GPIO1_IO_9        0x10b0>;
+			<MX6SX_PAD_GPIO1_IO09__GPIO1_IO_9	0x10b0>;
 	};
 
-
 	pinctrl_otg2_reg: otg2grp {
 		fsl,pins =
-			<MX6SX_PAD_NAND_RE_B__GPIO4_IO_12        0x10b0>;
+			<MX6SX_PAD_NAND_RE_B__GPIO4_IO_12	0x10b0>;
 	};
 
 	pinctrl_usb_otg1: usbotg1grp {
 		fsl,pins =
-			<MX6SX_PAD_GPIO1_IO10__ANATOP_OTG1_ID    0x17059>,
-			<MX6SX_PAD_GPIO1_IO08__USB_OTG1_OC       0x10b0>;
+			<MX6SX_PAD_GPIO1_IO10__ANATOP_OTG1_ID	0x17059>,
+			<MX6SX_PAD_GPIO1_IO08__USB_OTG1_OC	0x10b0>;
 	};
 
 	pinctrl_usb_otg2: usbot2ggrp {
 		fsl,pins =
-			<MX6SX_PAD_QSPI1A_DATA0__USB_OTG2_OC     0x10b0>;
+			<MX6SX_PAD_QSPI1A_DATA0__USB_OTG2_OC	0x10b0>;
 	};
 
 	pinctrl_usdhc2: usdhc2grp {
diff --git a/arch/arm/boot/dts/imx6sx.dtsi b/arch/arm/boot/dts/imx6sx.dtsi
index 4d075e2bf749..abc3572d699e 100644
--- a/arch/arm/boot/dts/imx6sx.dtsi
+++ b/arch/arm/boot/dts/imx6sx.dtsi
@@ -164,12 +164,18 @@
 		ocram_s: sram@8f8000 {
 			compatible = "mmio-sram";
 			reg = <0x008f8000 0x4000>;
+			ranges = <0 0x008f8000 0x4000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
 			clocks = <&clks IMX6SX_CLK_OCRAM_S>;
 		};
 
 		ocram: sram@900000 {
 			compatible = "mmio-sram";
 			reg = <0x00900000 0x20000>;
+			ranges = <0 0x00900000 0x20000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
 			clocks = <&clks IMX6SX_CLK_OCRAM>;
 		};
 
@@ -842,7 +848,7 @@
 				reg = <0x020e4000 0x4000>;
 			};
 
-			sdma: sdma@20ec000 {
+			sdma: dma-controller@20ec000 {
 				compatible = "fsl,imx6sx-sdma", "fsl,imx6q-sdma";
 				reg = <0x020ec000 0x4000>;
 				interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/imx6ul-14x14-evk.dtsi b/arch/arm/boot/dts/imx6ul-14x14-evk.dtsi
index 1a18c41ce385..c83e64a62d8a 100644
--- a/arch/arm/boot/dts/imx6ul-14x14-evk.dtsi
+++ b/arch/arm/boot/dts/imx6ul-14x14-evk.dtsi
@@ -82,7 +82,7 @@
 			"AMIC", "MICB";
 	};
 
-	spi4 {
+	spi-4 {
 		compatible = "spi-gpio";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_spi4>;
diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6310-s-43.dts b/arch/arm/boot/dts/imx6ul-kontron-bl-43.dts
index 5bfad4655b22..0c643706a158 100644
--- a/arch/arm/boot/dts/imx6ul-kontron-n6310-s-43.dts
+++ b/arch/arm/boot/dts/imx6ul-kontron-bl-43.dts
@@ -5,12 +5,12 @@
  * Copyright (c) 2019 Krzysztof Kozlowski <krzk@kernel.org>
  */
 
-#include "imx6ul-kontron-n6310-s.dts"
+#include "imx6ul-kontron-bl.dts"
 
 / {
-	model = "Kontron N6310 S 43";
-	compatible = "kontron,imx6ul-n6310-s-43", "kontron,imx6ul-n6310-s",
-		     "kontron,imx6ul-n6310-som", "fsl,imx6ul";
+	model = "Kontron BL i.MX6UL 43 (N631X S 43)";
+	compatible = "kontron,bl-imx6ul-43", "kontron,bl-imx6ul",
+		     "kontron,sl-imx6ul", "fsl,imx6ul";
 
 	backlight {
 		compatible = "pwm-backlight";
diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6x1x-s.dtsi b/arch/arm/boot/dts/imx6ul-kontron-bl-common.dtsi
index 43868311f48a..43868311f48a 100644
--- a/arch/arm/boot/dts/imx6ul-kontron-n6x1x-s.dtsi
+++ b/arch/arm/boot/dts/imx6ul-kontron-bl-common.dtsi
diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6310-s.dts b/arch/arm/boot/dts/imx6ul-kontron-bl.dts
index 5a3e06d6219b..dadf6d3d5f52 100644
--- a/arch/arm/boot/dts/imx6ul-kontron-n6310-s.dts
+++ b/arch/arm/boot/dts/imx6ul-kontron-bl.dts
@@ -7,11 +7,10 @@
 
 /dts-v1/;
 
-#include "imx6ul-kontron-n6310-som.dtsi"
-#include "imx6ul-kontron-n6x1x-s.dtsi"
+#include "imx6ul-kontron-sl.dtsi"
+#include "imx6ul-kontron-bl-common.dtsi"
 
 / {
-	model = "Kontron N6310 S";
-	compatible = "kontron,imx6ul-n6310-s", "kontron,imx6ul-n6310-som",
-		     "fsl,imx6ul";
+	model = "Kontron BL i.MX6UL (N631X S)";
+	compatible = "kontron,bl-imx6ul", "kontron,sl-imx6ul", "fsl,imx6ul";
 };
diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6310-som.dtsi b/arch/arm/boot/dts/imx6ul-kontron-n6310-som.dtsi
deleted file mode 100644
index acd936540d89..000000000000
--- a/arch/arm/boot/dts/imx6ul-kontron-n6310-som.dtsi
+++ /dev/null
@@ -1,41 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2017 exceet electronics GmbH
- * Copyright (C) 2018 Kontron Electronics GmbH
- * Copyright (c) 2019 Krzysztof Kozlowski <krzk@kernel.org>
- */
-
-#include "imx6ul.dtsi"
-#include "imx6ul-kontron-n6x1x-som-common.dtsi"
-
-/ {
-	model = "Kontron N6310 SOM";
-	compatible = "kontron,imx6ul-n6310-som", "fsl,imx6ul";
-
-	memory@80000000 {
-		reg = <0x80000000 0x10000000>;
-		device_type = "memory";
-	};
-};
-
-&qspi {
-	flash@0 {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		compatible = "spi-nand";
-		spi-max-frequency = <108000000>;
-		spi-tx-bus-width = <4>;
-		spi-rx-bus-width = <4>;
-		reg = <0>;
-
-		partition@0 {
-			label = "ubi1";
-			reg = <0x00000000 0x08000000>;
-		};
-
-		partition@8000000 {
-			label = "ubi2";
-			reg = <0x08000000 0x08000000>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6311-s.dts b/arch/arm/boot/dts/imx6ul-kontron-n6311-s.dts
deleted file mode 100644
index 239a1af3aeaa..000000000000
--- a/arch/arm/boot/dts/imx6ul-kontron-n6311-s.dts
+++ /dev/null
@@ -1,16 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2017 exceet electronics GmbH
- * Copyright (C) 2018 Kontron Electronics GmbH
- */
-
-/dts-v1/;
-
-#include "imx6ul-kontron-n6311-som.dtsi"
-#include "imx6ul-kontron-n6x1x-s.dtsi"
-
-/ {
-	model = "Kontron N6311 S";
-	compatible = "kontron,imx6ul-n6311-s", "kontron,imx6ul-n6311-som",
-		     "fsl,imx6ul";
-};
diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6311-som.dtsi b/arch/arm/boot/dts/imx6ul-kontron-n6311-som.dtsi
deleted file mode 100644
index 29ed38dce580..000000000000
--- a/arch/arm/boot/dts/imx6ul-kontron-n6311-som.dtsi
+++ /dev/null
@@ -1,40 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2017 exceet electronics GmbH
- * Copyright (C) 2018 Kontron Electronics GmbH
- */
-
-#include "imx6ul.dtsi"
-#include "imx6ul-kontron-n6x1x-som-common.dtsi"
-
-/ {
-	model = "Kontron N6311 SOM";
-	compatible = "kontron,imx6ul-n6311-som", "fsl,imx6ul";
-
-	memory@80000000 {
-		reg = <0x80000000 0x20000000>;
-		device_type = "memory";
-	};
-};
-
-&qspi {
-	flash@0 {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		compatible = "spi-nand";
-		spi-max-frequency = <104000000>;
-		spi-tx-bus-width = <4>;
-		spi-rx-bus-width = <4>;
-		reg = <0>;
-
-		partition@0 {
-			label = "ubi1";
-			reg = <0x00000000 0x08000000>;
-		};
-
-		partition@8000000 {
-			label = "ubi2";
-			reg = <0x08000000 0x18000000>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/imx6ul-kontron-n6x1x-som-common.dtsi b/arch/arm/boot/dts/imx6ul-kontron-sl-common.dtsi
index 09a83dbdf651..dcf88f610346 100644
--- a/arch/arm/boot/dts/imx6ul-kontron-n6x1x-som-common.dtsi
+++ b/arch/arm/boot/dts/imx6ul-kontron-sl-common.dtsi
@@ -11,6 +11,11 @@
 	chosen {
 		stdout-path = &uart4;
 	};
+
+	memory@80000000 {
+		reg = <0x80000000 0x10000000>;
+		device_type = "memory";
+	};
 };
 
 &ecspi2 {
@@ -55,6 +60,16 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_qspi>;
 	status = "okay";
+
+	spi-flash@0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "spi-nand";
+		spi-max-frequency = <104000000>;
+		spi-tx-bus-width = <4>;
+		spi-rx-bus-width = <4>;
+		reg = <0>;
+	};
 };
 
 &wdog1 {
diff --git a/arch/arm/boot/dts/imx6ul-kontron-sl.dtsi b/arch/arm/boot/dts/imx6ul-kontron-sl.dtsi
new file mode 100644
index 000000000000..0580d043e5ae
--- /dev/null
+++ b/arch/arm/boot/dts/imx6ul-kontron-sl.dtsi
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017 exceet electronics GmbH
+ * Copyright (C) 2018 Kontron Electronics GmbH
+ * Copyright (c) 2019 Krzysztof Kozlowski <krzk@kernel.org>
+ */
+
+#include "imx6ul.dtsi"
+#include "imx6ul-kontron-sl-common.dtsi"
+
+/ {
+	model = "Kontron SL i.MX6UL (N631X SOM)";
+	compatible = "kontron,sl-imx6ul", "fsl,imx6ul";
+};
diff --git a/arch/arm/boot/dts/imx6ul-tx6ul.dtsi b/arch/arm/boot/dts/imx6ul-tx6ul.dtsi
index c485d058e079..15ee0275feaf 100644
--- a/arch/arm/boot/dts/imx6ul-tx6ul.dtsi
+++ b/arch/arm/boot/dts/imx6ul-tx6ul.dtsi
@@ -212,7 +212,7 @@
 		enable-active-high;
 	};
 
-	spi_gpio: spi-gpio {
+	spi_gpio: spi {
 		#address-cells = <1>;
 		#size-cells = <0>;
 		compatible = "spi-gpio";
diff --git a/arch/arm/boot/dts/imx6ul.dtsi b/arch/arm/boot/dts/imx6ul.dtsi
index c95efd1d8c2d..2b5996395701 100644
--- a/arch/arm/boot/dts/imx6ul.dtsi
+++ b/arch/arm/boot/dts/imx6ul.dtsi
@@ -744,7 +744,7 @@
 				status = "disabled";
 			};
 
-			sdma: sdma@20ec000 {
+			sdma: dma-controller@20ec000 {
 				compatible = "fsl,imx6ul-sdma", "fsl,imx6q-sdma",
 					     "fsl,imx35-sdma";
 				reg = <0x020ec000 0x4000>;
diff --git a/arch/arm/boot/dts/imx6ull-kontron-bl.dts b/arch/arm/boot/dts/imx6ull-kontron-bl.dts
new file mode 100644
index 000000000000..fa016465cdbc
--- /dev/null
+++ b/arch/arm/boot/dts/imx6ull-kontron-bl.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017 exceet electronics GmbH
+ * Copyright (C) 2019 Kontron Electronics GmbH
+ */
+
+/dts-v1/;
+
+#include "imx6ull-kontron-sl.dtsi"
+#include "imx6ul-kontron-bl-common.dtsi"
+
+/ {
+	model = "Kontron BL i.MX6ULL (N641X S)";
+	compatible = "kontron,bl-imx6ull", "kontron,sl-imx6ull", "fsl,imx6ull";
+};
diff --git a/arch/arm/boot/dts/imx6ull-kontron-n6411-s.dts b/arch/arm/boot/dts/imx6ull-kontron-n6411-s.dts
deleted file mode 100644
index 57588a5e1e34..000000000000
--- a/arch/arm/boot/dts/imx6ull-kontron-n6411-s.dts
+++ /dev/null
@@ -1,16 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2017 exceet electronics GmbH
- * Copyright (C) 2019 Kontron Electronics GmbH
- */
-
-/dts-v1/;
-
-#include "imx6ull-kontron-n6411-som.dtsi"
-#include "imx6ul-kontron-n6x1x-s.dtsi"
-
-/ {
-	model = "Kontron N6411 S";
-	compatible = "kontron,imx6ull-n6411-s", "kontron,imx6ull-n6411-som",
-		     "fsl,imx6ull";
-};
diff --git a/arch/arm/boot/dts/imx6ull-kontron-n6411-som.dtsi b/arch/arm/boot/dts/imx6ull-kontron-n6411-som.dtsi
deleted file mode 100644
index d000606c0704..000000000000
--- a/arch/arm/boot/dts/imx6ull-kontron-n6411-som.dtsi
+++ /dev/null
@@ -1,40 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2017 exceet electronics GmbH
- * Copyright (C) 2018 Kontron Electronics GmbH
- */
-
-#include "imx6ull.dtsi"
-#include "imx6ul-kontron-n6x1x-som-common.dtsi"
-
-/ {
-	model = "Kontron N6411 SOM";
-	compatible = "kontron,imx6ull-n6311-som", "fsl,imx6ull";
-
-	memory@80000000 {
-		reg = <0x80000000 0x20000000>;
-		device_type = "memory";
-	};
-};
-
-&qspi {
-	flash@0 {
-		#address-cells = <1>;
-		#size-cells = <1>;
-		compatible = "spi-nand";
-		spi-max-frequency = <104000000>;
-		spi-tx-bus-width = <4>;
-		spi-rx-bus-width = <4>;
-		reg = <0>;
-
-		partition@0 {
-			label = "ubi1";
-			reg = <0x00000000 0x08000000>;
-		};
-
-		partition@8000000 {
-			label = "ubi2";
-			reg = <0x08000000 0x18000000>;
-		};
-	};
-};
diff --git a/arch/arm/boot/dts/imx6ull-kontron-sl.dtsi b/arch/arm/boot/dts/imx6ull-kontron-sl.dtsi
new file mode 100644
index 000000000000..93f10eb3494f
--- /dev/null
+++ b/arch/arm/boot/dts/imx6ull-kontron-sl.dtsi
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2017 exceet electronics GmbH
+ * Copyright (C) 2018 Kontron Electronics GmbH
+ */
+
+#include "imx6ull.dtsi"
+#include "imx6ul-kontron-sl-common.dtsi"
+
+/ {
+	model = "Kontron SL i.MX6ULL (N641X SOM)";
+	compatible = "kontron,sl-imx6ull", "fsl,imx6ull";
+};
diff --git a/arch/arm/boot/dts/imx7d-pico.dtsi b/arch/arm/boot/dts/imx7d-pico.dtsi
index e519897fae08..e0bff39e8d3e 100644
--- a/arch/arm/boot/dts/imx7d-pico.dtsi
+++ b/arch/arm/boot/dts/imx7d-pico.dtsi
@@ -41,7 +41,7 @@
 		regulator-max-microvolt = <3300000>;
 		gpio = <&gpio1 6 GPIO_ACTIVE_HIGH>;
 		enable-active-high;
-        };
+	};
 
 	reg_wlreg_on: regulator-wlreg_on {
 		compatible = "regulator-fixed";
@@ -432,7 +432,7 @@
 			MX7D_PAD_ENET1_RGMII_RD2__ENET1_RGMII_RD2	0x1
 			MX7D_PAD_ENET1_RGMII_RD3__ENET1_RGMII_RD3	0x1
 			MX7D_PAD_ENET1_RGMII_RX_CTL__ENET1_RGMII_RX_CTL	0x1
-			MX7D_PAD_SD3_RESET_B__GPIO6_IO11                0x1  /* Ethernet reset */
+			MX7D_PAD_SD3_RESET_B__GPIO6_IO11		0x1  /* Ethernet reset */
 		>;
 	};
 
@@ -493,19 +493,19 @@
 
 	pinctrl_pwm1: pwm1 {
 		fsl,pins = <
-			MX7D_PAD_GPIO1_IO08__PWM1_OUT   0x7f
+			MX7D_PAD_GPIO1_IO08__PWM1_OUT	0x7f
 		>;
 	};
 
 	pinctrl_pwm2: pwm2 {
 		fsl,pins = <
-			MX7D_PAD_GPIO1_IO09__PWM2_OUT   0x7f
+			MX7D_PAD_GPIO1_IO09__PWM2_OUT	0x7f
 		>;
 	};
 
 	pinctrl_pwm3: pwm3 {
 		fsl,pins = <
-			MX7D_PAD_GPIO1_IO10__PWM3_OUT   0x7f
+			MX7D_PAD_GPIO1_IO10__PWM3_OUT	0x7f
 		>;
 	};
 
diff --git a/arch/arm/boot/dts/imx7d-sdb.dts b/arch/arm/boot/dts/imx7d-sdb.dts
index 78f4224a9bf4..f483bc0afe5e 100644
--- a/arch/arm/boot/dts/imx7d-sdb.dts
+++ b/arch/arm/boot/dts/imx7d-sdb.dts
@@ -24,14 +24,14 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_gpio_keys>;
 
-		volume-up {
+		key-volume-up {
 			label = "Volume Up";
 			gpios = <&gpio5 11 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_VOLUMEUP>;
 			wakeup-source;
 		};
 
-		volume-down {
+		key-volume-down {
 			label = "Volume Down";
 			gpios = <&gpio5 10 GPIO_ACTIVE_LOW>;
 			linux,code = <KEY_VOLUMEDOWN>;
@@ -39,7 +39,7 @@
 		};
 	};
 
-	spi4 {
+	spi-4 {
 		compatible = "spi-gpio";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_spi4>;
@@ -206,12 +206,7 @@
 		interrupt-parent = <&gpio2>;
 		interrupts = <29 0>;
 		pendown-gpio = <&gpio2 29 GPIO_ACTIVE_HIGH>;
-		ti,x-min = /bits/ 16 <0>;
-		ti,x-max = /bits/ 16 <0>;
-		ti,y-min = /bits/ 16 <0>;
-		ti,y-max = /bits/ 16 <0>;
-		ti,pressure-max = /bits/ 16 <0>;
-		ti,x-plate-ohms = /bits/ 16 <400>;
+		touchscreen-max-pressure = <255>;
 		wakeup-source;
 	};
 };
diff --git a/arch/arm/boot/dts/imx7d-zii-rmu2.dts b/arch/arm/boot/dts/imx7d-zii-rmu2.dts
index 1065941807e8..1c9f25848bf7 100644
--- a/arch/arm/boot/dts/imx7d-zii-rmu2.dts
+++ b/arch/arm/boot/dts/imx7d-zii-rmu2.dts
@@ -24,7 +24,7 @@
 		pinctrl-0 = <&pinctrl_leds_debug>;
 		pinctrl-names = "default";
 
-		debug {
+		led-debug {
 			label = "zii:green:debug1";
 			gpios = <&gpio2 8 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "heartbeat";
diff --git a/arch/arm/boot/dts/imx7d-zii-rpu2.dts b/arch/arm/boot/dts/imx7d-zii-rpu2.dts
index 893bd30aa2a3..9d29490ab4c9 100644
--- a/arch/arm/boot/dts/imx7d-zii-rpu2.dts
+++ b/arch/arm/boot/dts/imx7d-zii-rpu2.dts
@@ -36,7 +36,7 @@
 		pinctrl-0 = <&pinctrl_leds_debug>;
 		pinctrl-names = "default";
 
-		debug {
+		led-debug {
 			label = "zii:green:debug1";
 			gpios = <&gpio2 8 GPIO_ACTIVE_HIGH>;
 			linux,default-trigger = "heartbeat";
diff --git a/arch/arm/boot/dts/imx7s.dtsi b/arch/arm/boot/dts/imx7s.dtsi
index 29148285f9fc..0fc9e6b8b05d 100644
--- a/arch/arm/boot/dts/imx7s.dtsi
+++ b/arch/arm/boot/dts/imx7s.dtsi
@@ -1224,7 +1224,7 @@
 				status = "disabled";
 			};
 
-			sdma: sdma@30bd0000 {
+			sdma: dma-controller@30bd0000 {
 				compatible = "fsl,imx7d-sdma", "fsl,imx35-sdma";
 				reg = <0x30bd0000 0x10000>;
 				interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm/boot/dts/imx7ulp.dtsi b/arch/arm/boot/dts/imx7ulp.dtsi
index bcec98b96411..7f7d2d5122fb 100644
--- a/arch/arm/boot/dts/imx7ulp.dtsi
+++ b/arch/arm/boot/dts/imx7ulp.dtsi
@@ -328,8 +328,9 @@
 			compatible = "fsl,imx7ulp-lpi2c";
 			reg = <0x40a40000 0x10000>;
 			interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&pcc3 IMX7ULP_CLK_LPI2C6>;
-			clock-names = "ipg";
+			clocks = <&pcc3 IMX7ULP_CLK_LPI2C6>,
+				 <&scg1 IMX7ULP_CLK_NIC1_BUS_DIV>;
+			clock-names = "per", "ipg";
 			assigned-clocks = <&pcc3 IMX7ULP_CLK_LPI2C6>;
 			assigned-clock-parents = <&scg1 IMX7ULP_CLK_FIRC>;
 			assigned-clock-rates = <48000000>;
@@ -340,8 +341,9 @@
 			compatible = "fsl,imx7ulp-lpi2c";
 			reg = <0x40a50000 0x10000>;
 			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&pcc3 IMX7ULP_CLK_LPI2C7>;
-			clock-names = "ipg";
+			clocks = <&pcc3 IMX7ULP_CLK_LPI2C7>,
+				 <&scg1 IMX7ULP_CLK_NIC1_BUS_DIV>;
+			clock-names = "per", "ipg";
 			assigned-clocks = <&pcc3 IMX7ULP_CLK_LPI2C7>;
 			assigned-clock-parents = <&scg1 IMX7ULP_CLK_FIRC>;
 			assigned-clock-rates = <48000000>;
diff --git a/arch/arm/boot/dts/integratorap-im-pd1.dts b/arch/arm/boot/dts/integratorap-im-pd1.dts
index 4c22e4436271..cc514cf07bff 100644
--- a/arch/arm/boot/dts/integratorap-im-pd1.dts
+++ b/arch/arm/boot/dts/integratorap-im-pd1.dts
@@ -249,6 +249,7 @@
 		/* 640x480 16bpp @ 25.175MHz is 36827428 bytes/s */
 		max-memory-bandwidth = <40000000>;
 		memory-region = <&impd1_ram>;
+		dma-ranges;
 
 		port@0 {
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/integratorap.dts b/arch/arm/boot/dts/integratorap.dts
index 9b652cc27b14..9148287fa0a9 100644
--- a/arch/arm/boot/dts/integratorap.dts
+++ b/arch/arm/boot/dts/integratorap.dts
@@ -160,6 +160,7 @@
 
 	pci: pciv3@62000000 {
 		compatible = "arm,integrator-ap-pci", "v3,v360epc-pci";
+		device_type = "pci";
 		#interrupt-cells = <1>;
 		#size-cells = <2>;
 		#address-cells = <3>;
@@ -261,7 +262,7 @@
 		lm0: bus@c0000000 {
 			compatible = "simple-bus";
 			ranges = <0x00000000 0xc0000000 0x10000000>;
-			dma-ranges = <0x00000000 0x80000000 0x10000000>;
+			dma-ranges = <0x00000000 0xc0000000 0x10000000>;
 			reg = <0xc0000000 0x10000000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
@@ -269,7 +270,7 @@
 		lm1: bus@d0000000 {
 			compatible = "simple-bus";
 			ranges = <0x00000000 0xd0000000 0x10000000>;
-			dma-ranges = <0x00000000 0x80000000 0x10000000>;
+			dma-ranges = <0x00000000 0xd0000000 0x10000000>;
 			reg = <0xd0000000 0x10000000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
@@ -277,7 +278,7 @@
 		lm2: bus@e0000000 {
 			compatible = "simple-bus";
 			ranges = <0x00000000 0xe0000000 0x10000000>;
-			dma-ranges = <0x00000000 0x80000000 0x10000000>;
+			dma-ranges = <0x00000000 0xe0000000 0x10000000>;
 			reg = <0xe0000000 0x10000000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
@@ -285,7 +286,7 @@
 		lm3: bus@f0000000 {
 			compatible = "simple-bus";
 			ranges = <0x00000000 0xf0000000 0x10000000>;
-			dma-ranges = <0x00000000 0x80000000 0x10000000>;
+			dma-ranges = <0x00000000 0xf0000000 0x10000000>;
 			reg = <0xf0000000 0x10000000>;
 			#address-cells = <1>;
 			#size-cells = <1>;
diff --git a/arch/arm/boot/dts/kirkwood-6192.dtsi b/arch/arm/boot/dts/kirkwood-6192.dtsi
index 396bcba08adb..705c0d7effed 100644
--- a/arch/arm/boot/dts/kirkwood-6192.dtsi
+++ b/arch/arm/boot/dts/kirkwood-6192.dtsi
@@ -26,12 +26,22 @@
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &intc 9>;
+				interrupt-names = "intx", "error";
+				interrupts = <9>, <44>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie_intc 0>,
+						<0 0 0 2 &pcie_intc 1>,
+						<0 0 0 3 &pcie_intc 2>,
+						<0 0 0 4 &pcie_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gate_clk 2>;
 				status = "disabled";
+
+				pcie_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/kirkwood-6281.dtsi b/arch/arm/boot/dts/kirkwood-6281.dtsi
index faa05849a40d..8e311165fd13 100644
--- a/arch/arm/boot/dts/kirkwood-6281.dtsi
+++ b/arch/arm/boot/dts/kirkwood-6281.dtsi
@@ -26,12 +26,22 @@
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &intc 9>;
+				interrupt-names = "intx", "error";
+				interrupts = <9>, <44>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie_intc 0>,
+						<0 0 0 2 &pcie_intc 1>,
+						<0 0 0 3 &pcie_intc 2>,
+						<0 0 0 4 &pcie_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gate_clk 2>;
 				status = "disabled";
+
+				pcie_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/kirkwood-6282.dtsi b/arch/arm/boot/dts/kirkwood-6282.dtsi
index e84c54b77dea..e33723160ce7 100644
--- a/arch/arm/boot/dts/kirkwood-6282.dtsi
+++ b/arch/arm/boot/dts/kirkwood-6282.dtsi
@@ -30,12 +30,22 @@
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &intc 9>;
+				interrupt-names = "intx", "error";
+				interrupts = <9>, <44>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie0_intc 0>,
+						<0 0 0 2 &pcie0_intc 1>,
+						<0 0 0 3 &pcie0_intc 2>,
+						<0 0 0 4 &pcie0_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gate_clk 2>;
 				status = "disabled";
+
+				pcie0_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 
 			pcie1: pcie@2,0 {
@@ -48,12 +58,22 @@
 				ranges = <0x82000000 0 0 0x82000000 0x2 0 1 0
 					  0x81000000 0 0 0x81000000 0x2 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &intc 10>;
+				interrupt-names = "intx", "error";
+				interrupts = <10>, <45>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie1_intc 0>,
+						<0 0 0 2 &pcie1_intc 1>,
+						<0 0 0 3 &pcie1_intc 2>,
+						<0 0 0 4 &pcie1_intc 3>;
 				marvell,pcie-port = <1>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gate_clk 18>;
 				status = "disabled";
+
+				pcie1_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/kirkwood-98dx4122.dtsi b/arch/arm/boot/dts/kirkwood-98dx4122.dtsi
index 299c147298c3..c3469a2fc58a 100644
--- a/arch/arm/boot/dts/kirkwood-98dx4122.dtsi
+++ b/arch/arm/boot/dts/kirkwood-98dx4122.dtsi
@@ -26,12 +26,22 @@
 				ranges = <0x82000000 0 0 0x82000000 0x1 0 1 0
 					  0x81000000 0 0 0x81000000 0x1 0 1 0>;
 				bus-range = <0x00 0xff>;
-				interrupt-map-mask = <0 0 0 0>;
-				interrupt-map = <0 0 0 0 &intc 9>;
+				interrupt-names = "intx", "error";
+				interrupts = <9>, <44>;
+				interrupt-map-mask = <0 0 0 7>;
+				interrupt-map = <0 0 0 1 &pcie_intc 0>,
+						<0 0 0 2 &pcie_intc 1>,
+						<0 0 0 3 &pcie_intc 2>,
+						<0 0 0 4 &pcie_intc 3>;
 				marvell,pcie-port = <0>;
 				marvell,pcie-lane = <0>;
 				clocks = <&gate_clk 2>;
 				status = "disabled";
+
+				pcie_intc: interrupt-controller {
+					interrupt-controller;
+					#interrupt-cells = <1>;
+				};
 			};
 		};
 	};
diff --git a/arch/arm/boot/dts/kirkwood-lsxl.dtsi b/arch/arm/boot/dts/kirkwood-lsxl.dtsi
index 7b151acb9984..88b70ba1c8fe 100644
--- a/arch/arm/boot/dts/kirkwood-lsxl.dtsi
+++ b/arch/arm/boot/dts/kirkwood-lsxl.dtsi
@@ -10,6 +10,11 @@
 
 	ocp@f1000000 {
 		pinctrl: pin-controller@10000 {
+			/* Non-default UART pins */
+			pmx_uart0: pmx-uart0 {
+				marvell,pins = "mpp4", "mpp5";
+			};
+
 			pmx_power_hdd: pmx-power-hdd {
 				marvell,pins = "mpp10";
 				marvell,function = "gpo";
@@ -213,22 +218,11 @@
 &mdio {
 	status = "okay";
 
-	ethphy0: ethernet-phy@0 {
-		reg = <0>;
-	};
-
 	ethphy1: ethernet-phy@8 {
 		reg = <8>;
 	};
 };
 
-&eth0 {
-	status = "okay";
-	ethernet0-port@0 {
-		phy-handle = <&ethphy0>;
-	};
-};
-
 &eth1 {
 	status = "okay";
 	ethernet1-port@0 {
diff --git a/arch/arm/boot/dts/lan966x-pcb8290.dts b/arch/arm/boot/dts/lan966x-pcb8290.dts
new file mode 100644
index 000000000000..77187f59f04d
--- /dev/null
+++ b/arch/arm/boot/dts/lan966x-pcb8290.dts
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * lan966x-pcb8290.dts - Device Tree file for LAN966X-PCB8290 board
+ *
+ * Copyright (C) 2022 Microchip Technology Inc. and its subsidiaries
+ *
+ * Author: Horatiu Vultur <horatiu.vultur@microchip.com>
+ */
+/dts-v1/;
+#include "lan966x.dtsi"
+#include "dt-bindings/phy/phy-lan966x-serdes.h"
+
+/ {
+	model = "Microchip EVB LAN9668";
+	compatible = "microchip,lan9668-pcb8290", "microchip,lan9668", "microchip,lan966";
+
+	gpio-restart {
+		compatible = "gpio-restart";
+		gpios = <&gpio 56 GPIO_ACTIVE_LOW>;
+		priority = <200>;
+	};
+};
+
+&aes {
+	status = "disabled"; /* Reserved by secure OS */
+};
+
+&gpio {
+	miim_a_pins: mdio-pins {
+		/* MDC, MDIO */
+		pins =  "GPIO_28", "GPIO_29";
+		function = "miim_a";
+	};
+
+	pps_out_pins: pps-out-pins {
+		/* 1pps output */
+		pins = "GPIO_38";
+		function = "ptpsync_3";
+	};
+
+	ptp_ext_pins: ptp-ext-pins {
+		/* 1pps input */
+		pins = "GPIO_35";
+		function = "ptpsync_0";
+	};
+
+	udc_pins: ucd-pins {
+		/* VBUS_DET B */
+		pins = "GPIO_8";
+		function = "usb_slave_b";
+	};
+};
+
+&mdio0 {
+	pinctrl-0 = <&miim_a_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	ext_phy0: ethernet-phy@7 {
+		reg = <7>;
+		coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>;
+	};
+
+	ext_phy1: ethernet-phy@8 {
+		reg = <8>;
+		coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>;
+	};
+
+	ext_phy2: ethernet-phy@9 {
+		reg = <9>;
+		coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>;
+	};
+
+	ext_phy3: ethernet-phy@10 {
+		reg = <10>;
+		coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>;
+	};
+
+	ext_phy4: ethernet-phy@15 {
+		reg = <15>;
+		coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>;
+	};
+
+	ext_phy5: ethernet-phy@16 {
+		reg = <16>;
+		coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>;
+	};
+
+	ext_phy6: ethernet-phy@17 {
+		reg = <17>;
+		coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>;
+	};
+
+	ext_phy7: ethernet-phy@18 {
+		reg = <18>;
+		coma-mode-gpios = <&gpio 60 GPIO_OPEN_DRAIN>;
+	};
+};
+
+&port0 {
+	reg = <2>;
+	phy-handle = <&ext_phy2>;
+	phy-mode = "qsgmii";
+	phys = <&serdes 0 SERDES6G(1)>;
+	status = "okay";
+};
+
+&port1 {
+	reg = <3>;
+	phy-handle = <&ext_phy3>;
+	phy-mode = "qsgmii";
+	phys = <&serdes 1 SERDES6G(1)>;
+	status = "okay";
+};
+
+&port2 {
+	reg = <0>;
+	phy-handle = <&ext_phy0>;
+	phy-mode = "qsgmii";
+	phys = <&serdes 2 SERDES6G(1)>;
+	status = "okay";
+};
+
+&port3 {
+	reg = <1>;
+	phy-handle = <&ext_phy1>;
+	phy-mode = "qsgmii";
+	phys = <&serdes 3 SERDES6G(1)>;
+	status = "okay";
+};
+
+&port4 {
+	reg = <6>;
+	phy-handle = <&ext_phy6>;
+	phy-mode = "qsgmii";
+	phys = <&serdes 4 SERDES6G(2)>;
+	status = "okay";
+};
+
+&port5 {
+	reg = <7>;
+	phy-handle = <&ext_phy7>;
+	phy-mode = "qsgmii";
+	phys = <&serdes 5 SERDES6G(2)>;
+	status = "okay";
+};
+
+&port6 {
+	reg = <4>;
+	phy-handle = <&ext_phy4>;
+	phy-mode = "qsgmii";
+	phys = <&serdes 6 SERDES6G(2)>;
+	status = "okay";
+};
+
+&port7 {
+	reg = <5>;
+	phy-handle = <&ext_phy5>;
+	phy-mode = "qsgmii";
+	phys = <&serdes 7 SERDES6G(2)>;
+	status = "okay";
+};
+
+&serdes {
+	status = "okay";
+};
+
+&switch {
+	pinctrl-0 = <&pps_out_pins>, <&ptp_ext_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&udc {
+	pinctrl-0 = <&udc_pins>;
+	pinctrl-names = "default";
+	atmel,vbus-gpio = <&gpio 8 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/lan966x-pcb8291.dts b/arch/arm/boot/dts/lan966x-pcb8291.dts
index 24d9055c4a08..f4f054cdf2a8 100644
--- a/arch/arm/boot/dts/lan966x-pcb8291.dts
+++ b/arch/arm/boot/dts/lan966x-pcb8291.dts
@@ -23,6 +23,38 @@
 		gpios = <&gpio 56 GPIO_ACTIVE_LOW>;
 		priority = <200>;
 	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-s0-blue {
+			label = "s0:blue";
+			gpios = <&sgpio_out 2 0 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-s0-green {
+			label = "s0:green";
+			gpios = <&sgpio_out 2 1 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-s1-blue {
+			label = "s1:blue";
+			gpios = <&sgpio_out 3 0 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-s1-green {
+			label = "s1:green";
+			gpios = <&sgpio_out 3 1 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+	};
+};
+
+&aes {
+	status = "disabled"; /* Reserved by secure OS */
 };
 
 &gpio {
diff --git a/arch/arm/boot/dts/lan966x-pcb8309.dts b/arch/arm/boot/dts/lan966x-pcb8309.dts
index 05ce27ed5648..c436cd20d4b4 100644
--- a/arch/arm/boot/dts/lan966x-pcb8309.dts
+++ b/arch/arm/boot/dts/lan966x-pcb8309.dts
@@ -42,6 +42,34 @@
 		};
 	};
 
+	leds {
+		compatible = "gpio-leds";
+
+		led-s0-green {
+			label = "s0:green";
+			gpios = <&sgpio_out 2 0 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-s0-red {
+			label = "s0:red";
+			gpios = <&sgpio_out 2 1 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-s1-green {
+			label = "s1:green";
+			gpios = <&sgpio_out 3 0 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-s1-red {
+			label = "s1:red";
+			gpios = <&sgpio_out 3 1 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+	};
+
 	mux: mux-controller {
 		compatible = "gpio-mux";
 		#mux-control-cells = <0>;
@@ -69,6 +97,10 @@
 	};
 };
 
+&aes {
+	status = "disabled"; /* Reserved by secure OS */
+};
+
 &flx3 {
 	atmel,flexcom-mode = <ATMEL_FLEXCOM_MODE_USART>;
 	status = "okay";
diff --git a/arch/arm/boot/dts/ls1021a.dtsi b/arch/arm/boot/dts/ls1021a.dtsi
index fa761620f073..7c5510e34494 100644
--- a/arch/arm/boot/dts/ls1021a.dtsi
+++ b/arch/arm/boot/dts/ls1021a.dtsi
@@ -647,7 +647,7 @@
 			status = "disabled";
 		};
 
-		edma0: edma@2c00000 {
+		edma0: dma-controller@2c00000 {
 			#dma-cells = <2>;
 			compatible = "fsl,vf610-edma";
 			reg = <0x0 0x2c00000 0x0 0x10000>,
diff --git a/arch/arm/boot/dts/qcom-apq8026-asus-sparrow.dts b/arch/arm/boot/dts/qcom-apq8026-asus-sparrow.dts
index ace8cea27949..215613c65250 100644
--- a/arch/arm/boot/dts/qcom-apq8026-asus-sparrow.dts
+++ b/arch/arm/boot/dts/qcom-apq8026-asus-sparrow.dts
@@ -8,6 +8,8 @@
 #include "qcom-msm8226.dtsi"
 #include "qcom-pm8226.dtsi"
 
+/delete-node/ &adsp_region;
+
 / {
 	model = "ASUS ZenWatch 2";
 	compatible = "asus,sparrow", "qcom,apq8026";
@@ -57,6 +59,10 @@
 	};
 };
 
+&adsp {
+	status = "okay";
+};
+
 &blsp1_uart1 {
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/qcom-apq8026-lg-lenok.dts b/arch/arm/boot/dts/qcom-apq8026-lg-lenok.dts
index 2b7e52fda6a7..193569f0ca5f 100644
--- a/arch/arm/boot/dts/qcom-apq8026-lg-lenok.dts
+++ b/arch/arm/boot/dts/qcom-apq8026-lg-lenok.dts
@@ -8,6 +8,8 @@
 #include "qcom-msm8226.dtsi"
 #include "qcom-pm8226.dtsi"
 
+/delete-node/ &adsp_region;
+
 / {
 	model = "LG G Watch R";
 	compatible = "lg,lenok", "qcom,apq8026";
@@ -23,6 +25,13 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	reserved-memory {
+		adsp_region: adsp@3300000 {
+			reg = <0x03300000 0x1400000>;
+			no-map;
+		};
+	};
+
 	vreg_wlan: wlan-regulator {
 		compatible = "regulator-fixed";
 
@@ -38,6 +47,10 @@
 	};
 };
 
+&adsp {
+	status = "okay";
+};
+
 &blsp1_i2c1 {
 	status = "okay";
 
diff --git a/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts b/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts
index 70a1dd629c7a..573e4dc66bb0 100644
--- a/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts
+++ b/arch/arm/boot/dts/qcom-apq8060-dragonboard.dts
@@ -159,6 +159,19 @@
 				};
 			};
 
+			dragon_gsbi3_i2c_pins: gsbi3_i2c {
+				mux {
+					pins = "gpio43", "gpio44";
+					function = "gsbi3";
+				};
+				pinconf {
+					pins = "gpio43", "gpio44";
+					drive-strength = <8>;
+					/* These have external pull-up 2.2kOhm to 1.8V */
+					bias-disable;
+				};
+			};
+
 			dragon_gsbi8_i2c_pins: gsbi8_i2c {
 				mux {
 					pins = "gpio64", "gpio65";
@@ -240,6 +253,22 @@
 					bias-pull-up;
 				};
 			};
+
+			dragon_tma340_gpios: tma340 {
+				reset {
+					/* RESET line, TS_ATTN, WAKE_CTP */
+					pins = "gpio58";
+					function = "gpio";
+					drive-strength = <6>;
+					bias-disable;
+				};
+				irq {
+					pins = "gpio61"; /* IRQ line */
+					function = "gpio";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
 		};
 
 		qcom,ssbi@500000 {
@@ -444,6 +473,45 @@
 			};
 		};
 
+		gsbi@16200000 {
+			qcom,mode = <GSBI_PROT_I2C>;
+			status = "okay";
+
+			gsbi3_i2c: i2c@16280000 {
+				pinctrl-names = "default";
+				pinctrl-0 = <&dragon_gsbi3_i2c_pins>;
+				status = "okay";
+
+				touchscreen@24 {
+					compatible = "cypress,cy8ctma340";
+					reg = <0x24>;
+					/* Certainly we can do at least 400 kHz */
+					clock-frequency = <400000>;
+					/* IRQ on GPIO61 called /CTP_INT */
+					interrupt-parent = <&tlmm>;
+					interrupts = <61 IRQ_TYPE_EDGE_FALLING>;
+					/*
+					 * The I2C bus is using a PCA9306 level translator from L16A
+					 * to L2B so these two voltages are needed and L16A is
+					 * kind of the IO voltage, however L16Aisn't really fed to
+					 * the TMA340, which relies entirely on L2B (PM8901 L2).
+					 */
+					vcpin-supply = <&pm8058_l16>;
+					vdd-supply = <&pm8901_l2>;
+					/* GPIO58, called WAKE_CTP */
+					reset-gpios = <&tlmm 58 GPIO_ACTIVE_LOW>;
+					touchscreen-size-x = <480>;
+					touchscreen-size-y = <800>;
+					active-interval-ms = <0>;
+					touch-timeout-ms = <255>;
+					lowpower-interval-ms = <10>;
+					bootloader-key = /bits/ 8 <0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&dragon_tma340_gpios>;
+				};
+			};
+		};
+
 		gsbi@19800000 {
 			status = "okay";
 			qcom,mode = <GSBI_PROT_I2C>;
@@ -634,7 +702,8 @@
 					bias-pull-down;
 				};
 				l2 {
-					regulator-min-microvolt = <2850000>;
+					/* TMA340 requires strictly 3.3V */
+					regulator-min-microvolt = <3300000>;
 					regulator-max-microvolt = <3300000>;
 					bias-pull-down;
 				};
diff --git a/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts b/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts
index e3bf57cd7423..529629a0a9dc 100644
--- a/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts
+++ b/arch/arm/boot/dts/qcom-apq8064-cm-qs600.dts
@@ -215,7 +215,7 @@
 			vdda_refclk-supply = <&v3p3_fixed>;
 			pinctrl-0 = <&pcie_pins>;
 			pinctrl-names = "default";
-			perst-gpio = <&tlmm_pinmux 27 GPIO_ACTIVE_LOW>;
+			perst-gpios = <&tlmm_pinmux 27 GPIO_ACTIVE_LOW>;
 		};
 
 		amba {
diff --git a/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts b/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts
index 0322cb88d448..a7f90217661b 100644
--- a/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts
+++ b/arch/arm/boot/dts/qcom-apq8064-ifc6410.dts
@@ -287,7 +287,7 @@
 			vdda_refclk-supply = <&ext_3p3v>;
 			pinctrl-0 = <&pcie_pins>;
 			pinctrl-names = "default";
-			perst-gpio = <&tlmm_pinmux 27 GPIO_ACTIVE_LOW>;
+			perst-gpios = <&tlmm_pinmux 27 GPIO_ACTIVE_LOW>;
 		};
 
 		qcom,ssbi@500000 {
diff --git a/arch/arm/boot/dts/qcom-apq8064.dtsi b/arch/arm/boot/dts/qcom-apq8064.dtsi
index ada4c828bf2f..942aa2278355 100644
--- a/arch/arm/boot/dts/qcom-apq8064.dtsi
+++ b/arch/arm/boot/dts/qcom-apq8064.dtsi
@@ -2,6 +2,7 @@
 /dts-v1/;
 
 #include <dt-bindings/clock/qcom,gcc-msm8960.h>
+#include <dt-bindings/clock/qcom,lcc-msm8960.h>
 #include <dt-bindings/reset/qcom,gcc-msm8960.h>
 #include <dt-bindings/clock/qcom,mmcc-msm8960.h>
 #include <dt-bindings/clock/qcom,rpmcc.h>
@@ -815,6 +816,10 @@
 			#clock-cells = <1>;
 			#power-domain-cells = <1>;
 			#reset-cells = <1>;
+			clocks = <&cxo_board>,
+				 <&pxo_board>,
+				 <&lcc PLL4>;
+			clock-names = "cxo", "pxo", "pll4";
 
 			tsens: thermal-sensor {
 				compatible = "qcom,msm8960-tsens";
@@ -834,6 +839,20 @@
 			reg = <0x28000000 0x1000>;
 			#clock-cells = <1>;
 			#reset-cells = <1>;
+			clocks = <&pxo_board>,
+				 <&gcc PLL4_VOTE>,
+				 <0>,
+				 <0>, <0>,
+				 <0>, <0>,
+				 <0>;
+			clock-names = "pxo",
+				      "pll4_vote",
+				      "mi2s_codec_clk",
+				      "codec_i2s_mic_codec_clk",
+				      "spare_i2s_mic_codec_clk",
+				      "codec_i2s_spkr_codec_clk",
+				      "spare_i2s_spkr_codec_clk",
+				      "pcm_codec_clk";
 		};
 
 		mmcc: clock-controller@4000000 {
@@ -842,6 +861,22 @@
 			#clock-cells = <1>;
 			#power-domain-cells = <1>;
 			#reset-cells = <1>;
+			clocks = <&pxo_board>,
+				 <&gcc PLL3>,
+				 <&gcc PLL8_VOTE>,
+				 <&dsi0_phy 1>,
+				 <&dsi0_phy 0>,
+				 <0>,
+				 <0>,
+				 <0>;
+			clock-names = "pxo",
+				      "pll3",
+				      "pll8_vote",
+				      "dsi1pll",
+				      "dsi1pllbyte",
+				      "dsi2pll",
+				      "dsi2pllbyte",
+				      "hdmipll";
 		};
 
 		l2cc: clock-controller@2011000 {
@@ -1384,7 +1419,7 @@
 		};
 
 		pcie: pci@1b500000 {
-			compatible = "qcom,pcie-apq8064", "snps,dw-pcie";
+			compatible = "qcom,pcie-apq8064";
 			reg = <0x1b500000 0x1000>,
 			      <0x1b502000 0x80>,
 			      <0x1b600000 0x100>,
diff --git a/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts b/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts
index 3051a861ff0c..91716298ec5e 100644
--- a/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts
+++ b/arch/arm/boot/dts/qcom-apq8074-dragonboard.dts
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
+#include <dt-bindings/gpio/gpio.h>
 #include "qcom-msm8974.dtsi"
 #include "qcom-pm8841.dtsi"
 #include "qcom-pm8941.dtsi"
@@ -261,7 +262,7 @@
 &sdhc_2 {
 	status = "okay";
 
-	cd-gpios = <&tlmm 62 0x1>;
+	cd-gpios = <&tlmm 62 GPIO_ACTIVE_LOW>;
 	vmmc-supply = <&pm8941_l21>;
 	vqmmc-supply = <&pm8941_l13>;
 
diff --git a/arch/arm/boot/dts/qcom-apq8084.dtsi b/arch/arm/boot/dts/qcom-apq8084.dtsi
index 72f9255855a1..f2fb7c975af8 100644
--- a/arch/arm/boot/dts/qcom-apq8084.dtsi
+++ b/arch/arm/boot/dts/qcom-apq8084.dtsi
@@ -239,6 +239,11 @@
 			reg = <0xf9011000 0x1000>;
 		};
 
+		sram@fc190000 {
+			compatible = "qcom,apq8084-rpm-stats";
+			reg = <0xfc190000 0x10000>;
+		};
+
 		qfprom: qfprom@fc4bc000 {
 			compatible = "qcom,apq8084-qfprom", "qcom,qfprom";
 			reg = <0xfc4bc000 0x1000>;
@@ -383,14 +388,9 @@
 			reg = <0xfc400000 0x4000>;
 		};
 
-		tcsr_mutex_regs: syscon@fd484000 {
-			compatible = "syscon";
-			reg = <0xfd484000 0x2000>;
-		};
-
-		tcsr_mutex: hwlock {
-			compatible = "qcom,tcsr-mutex";
-			syscon = <&tcsr_mutex_regs 0 0x80>;
+		tcsr_mutex: hwlock@fd484000 {
+			compatible = "qcom,apq8084-tcsr-mutex", "qcom,tcsr-mutex";
+			reg = <0xfd484000 0x1000>;
 			#hwlock-cells = <1>;
 		};
 
@@ -422,26 +422,26 @@
 		mmc@f9824900 {
 			compatible = "qcom,apq8084-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf9824900 0x11c>, <0xf9824000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC1_APPS_CLK>,
-				 <&gcc GCC_SDCC1_AHB_CLK>,
+			clocks = <&gcc GCC_SDCC1_AHB_CLK>,
+				 <&gcc GCC_SDCC1_APPS_CLK>,
 				 <&xo_board>;
-			clock-names = "core", "iface", "xo";
+			clock-names = "iface", "core", "xo";
 			status = "disabled";
 		};
 
 		mmc@f98a4900 {
 			compatible = "qcom,apq8084-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf98a4900 0x11c>, <0xf98a4000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 			interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC2_APPS_CLK>,
-				 <&gcc GCC_SDCC2_AHB_CLK>,
+			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&gcc GCC_SDCC2_APPS_CLK>,
 				 <&xo_board>;
-			clock-names = "core", "iface", "xo";
+			clock-names = "iface", "core", "xo";
 			status = "disabled";
 		};
 
diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi b/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi
index 03bb9e1768c4..0505270cf508 100644
--- a/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi
+++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk01.1.dtsi
@@ -14,6 +14,7 @@
  *
  */
 
+#include <dt-bindings/gpio/gpio.h>
 #include "qcom-ipq4019.dtsi"
 
 / {
@@ -72,7 +73,7 @@
 			pinctrl-0 = <&spi_0_pins>;
 			pinctrl-names = "default";
 			status = "okay";
-			cs-gpios = <&tlmm 54 0>;
+			cs-gpios = <&tlmm 54 GPIO_ACTIVE_HIGH>;
 
 			mx25l25635e@0 {
 				#address-cells = <1>;
diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi b/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi
index 44a9597d8bfd..a63b3778636d 100644
--- a/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi
+++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk04.1.dtsi
@@ -87,7 +87,7 @@
 			pinctrl-0 = <&spi_0_pins>;
 			pinctrl-names = "default";
 			status = "okay";
-			cs-gpios = <&tlmm 12 0>;
+			cs-gpios = <&tlmm 12 GPIO_ACTIVE_HIGH>;
 
 			flash@0 {
 				#address-cells = <1>;
@@ -100,7 +100,7 @@
 
 		pci@40000000 {
 			status = "okay";
-			perst-gpio = <&tlmm 38 0x1>;
+			perst-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
 		};
 
 		qpic-nand@79b0000 {
diff --git a/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts b/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts
index c7a6e77da272..ea2987fcbff8 100644
--- a/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts
+++ b/arch/arm/boot/dts/qcom-ipq4019-ap.dk07.1-c1.dts
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 // Copyright (c) 2018, The Linux Foundation. All rights reserved.
 
+#include <dt-bindings/gpio/gpio.h>
 #include "qcom-ipq4019-ap.dk07.1.dtsi"
 
 / {
@@ -10,7 +11,7 @@
 	soc {
 		pci@40000000 {
 			status = "okay";
-			perst-gpio = <&tlmm 38 0x1>;
+			perst-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
 		};
 
 		spi@78b6000 {
@@ -50,7 +51,7 @@
 			pinctrl-0 = <&spi_0_pins>;
 			pinctrl-names = "default";
 			status = "okay";
-			cs-gpios = <&tlmm 12 0>;
+			cs-gpios = <&tlmm 12 GPIO_ACTIVE_HIGH>;
 
 			flash@0 {
 				#address-cells = <1>;
diff --git a/arch/arm/boot/dts/qcom-ipq4019.dtsi b/arch/arm/boot/dts/qcom-ipq4019.dtsi
index bb307b8f678c..b23591110bd2 100644
--- a/arch/arm/boot/dts/qcom-ipq4019.dtsi
+++ b/arch/arm/boot/dts/qcom-ipq4019.dtsi
@@ -224,12 +224,13 @@
 		sdhci: mmc@7824900 {
 			compatible = "qcom,sdhci-msm-v4";
 			reg = <0x7824900 0x11c>, <0x7824000 0x800>;
+			reg-names = "hc", "core";
 			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
 			bus-width = <8>;
-			clocks = <&gcc GCC_SDCC1_APPS_CLK>, <&gcc GCC_SDCC1_AHB_CLK>,
+			clocks = <&gcc GCC_SDCC1_AHB_CLK>, <&gcc GCC_SDCC1_APPS_CLK>,
 				 <&gcc GCC_DCD_XO_CLK>;
-			clock-names = "core", "iface", "xo";
+			clock-names = "iface", "core", "xo";
 			status = "disabled";
 		};
 
@@ -412,7 +413,7 @@
 		};
 
 		pcie0: pci@40000000 {
-			compatible = "qcom,pcie-ipq4019", "snps,dw-pcie";
+			compatible = "qcom,pcie-ipq4019";
 			reg =  <0x40000000 0xf1d
 				0x40000f20 0xa8
 				0x80000 0x2000
diff --git a/arch/arm/boot/dts/qcom-ipq8062-smb208.dtsi b/arch/arm/boot/dts/qcom-ipq8062-smb208.dtsi
new file mode 100644
index 000000000000..9d06255104c7
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq8062-smb208.dtsi
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "qcom-ipq8062.dtsi"
+
+&rpm {
+	smb208_regulators: regulators {
+		compatible = "qcom,rpm-smb208-regulators";
+
+		smb208_s1a: s1a {
+			regulator-min-microvolt = <1050000>;
+			regulator-max-microvolt = <1150000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+
+		smb208_s1b: s1b {
+			regulator-min-microvolt = <1050000>;
+			regulator-max-microvolt = <1150000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+
+		smb208_s2a: s2a {
+			regulator-min-microvolt = < 800000>;
+			regulator-max-microvolt = <1150000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+
+		smb208_s2b: s2b {
+			regulator-min-microvolt = < 800000>;
+			regulator-max-microvolt = <1150000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom-ipq8062.dtsi b/arch/arm/boot/dts/qcom-ipq8062.dtsi
new file mode 100644
index 000000000000..5d3ebd3e2e51
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq8062.dtsi
@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "qcom-ipq8064-v2.0.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ8062";
+	compatible = "qcom,ipq8062", "qcom,ipq8064";
+};
diff --git a/arch/arm/boot/dts/qcom-ipq8064-v2.0-smb208.dtsi b/arch/arm/boot/dts/qcom-ipq8064-v2.0-smb208.dtsi
new file mode 100644
index 000000000000..0442580b22de
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq8064-v2.0-smb208.dtsi
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "qcom-ipq8064-v2.0.dtsi"
+
+&rpm {
+	smb208_regulators: regulators {
+		compatible = "qcom,rpm-smb208-regulators";
+
+		smb208_s1a: s1a {
+			regulator-min-microvolt = <1050000>;
+			regulator-max-microvolt = <1150000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+
+		smb208_s1b: s1b {
+			regulator-min-microvolt = <1050000>;
+			regulator-max-microvolt = <1150000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+
+		smb208_s2a: s2a {
+			regulator-min-microvolt = < 800000>;
+			regulator-max-microvolt = <1250000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+
+		smb208_s2b: s2b {
+			regulator-min-microvolt = < 800000>;
+			regulator-max-microvolt = <1250000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom-ipq8064-v2.0.dtsi b/arch/arm/boot/dts/qcom-ipq8064-v2.0.dtsi
new file mode 100644
index 000000000000..2f117d576daf
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq8064-v2.0.dtsi
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "qcom-ipq8064.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ8064-v2.0";
+
+	aliases {
+		serial0 = &gsbi4_serial;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	reserved-memory {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges;
+
+		rsvd@41200000 {
+			reg = <0x41200000 0x300000>;
+			no-map;
+		};
+	};
+};
+
+&gsbi4 {
+	qcom,mode = <GSBI_PROT_I2C_UART>;
+	status = "okay";
+
+	serial@16340000 {
+		status = "okay";
+	};
+	/*
+	 * The i2c device on gsbi4 should not be enabled.
+	 * On ipq806x designs gsbi4 i2c is meant for exclusive
+	 * RPM usage. Turning this on in kernel manifests as
+	 * i2c failure for the RPM.
+	 */
+};
+
+&pcie0 {
+	compatible = "qcom,pcie-ipq8064-v2";
+};
+
+&pcie1 {
+	compatible = "qcom,pcie-ipq8064-v2";
+};
+
+&pcie2 {
+	compatible = "qcom,pcie-ipq8064-v2";
+};
+
+&sata {
+	ports-implemented = <0x1>;
+};
+
+&ss_phy_0 {
+	qcom,rx-eq = <2>;
+	qcom,tx-deamp_3_5db = <32>;
+	qcom,mpll = <5>;
+};
+
+&ss_phy_1 {
+	qcom,rx-eq = <2>;
+	qcom,tx-deamp_3_5db = <32>;
+	qcom,mpll = <5>;
+};
diff --git a/arch/arm/boot/dts/qcom-ipq8064.dtsi b/arch/arm/boot/dts/qcom-ipq8064.dtsi
index c8337c870bdb..90c08b51680a 100644
--- a/arch/arm/boot/dts/qcom-ipq8064.dtsi
+++ b/arch/arm/boot/dts/qcom-ipq8064.dtsi
@@ -332,24 +332,64 @@
 		ranges;
 		compatible = "simple-bus";
 
-		lpass@28100000 {
-			compatible = "qcom,lpass-cpu";
-			status = "disabled";
-			clocks = <&lcc AHBIX_CLK>,
-					<&lcc MI2S_OSR_CLK>,
-					<&lcc MI2S_BIT_CLK>;
-			clock-names = "ahbix-clk",
-					"mi2s-osr-clk",
-					"mi2s-bit-clk";
-			interrupts = <GIC_SPI 85 IRQ_TYPE_EDGE_RISING>;
-			interrupt-names = "lpass-irq-lpaif";
-			reg = <0x28100000 0x10000>;
-			reg-names = "lpass-lpaif";
+		stmmac_axi_setup: stmmac-axi-config {
+			snps,wr_osr_lmt = <7>;
+			snps,rd_osr_lmt = <7>;
+			snps,blen = <16 0 0 0 0 0 0>;
+		};
+
+		vsdcc_fixed: vsdcc-regulator {
+			compatible = "regulator-fixed";
+			regulator-name = "SDCC Power";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-always-on;
+		};
+
+		rpm: rpm@108000 {
+			compatible = "qcom,rpm-ipq8064";
+			reg = <0x00108000 0x1000>;
+			qcom,ipc = <&l2cc 0x8 2>;
+
+			interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>,
+					<GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>,
+					<GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ack", "err", "wakeup";
+
+			clocks = <&gcc RPM_MSG_RAM_H_CLK>;
+			clock-names = "ram";
+
+			rpmcc: clock-controller {
+				compatible = "qcom,rpmcc-ipq806x", "qcom,rpmcc";
+				#clock-cells = <1>;
+			};
+		};
+
+		qcom,ssbi@500000 {
+			compatible = "qcom,ssbi";
+			reg = <0x00500000 0x1000>;
+			qcom,controller-type = "pmic-arbiter";
+		};
+
+		qfprom: qfprom@700000 {
+			compatible = "qcom,ipq8064-qfprom", "qcom,qfprom";
+			reg = <0x00700000 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			speedbin_efuse: speedbin@c0 {
+				reg = <0xc0 0x4>;
+			};
+			tsens_calib: calib@400 {
+				reg = <0x400 0xb>;
+			};
+			tsens_calib_backup: calib_backup@410 {
+				reg = <0x410 0xb>;
+			};
 		};
 
 		qcom_pinmux: pinmux@800000 {
 			compatible = "qcom,ipq8064-pinctrl";
-			reg = <0x800000 0x4000>;
+			reg = <0x00800000 0x4000>;
 
 			gpio-controller;
 			gpio-ranges = <&qcom_pinmux 0 0 69>;
@@ -471,6 +511,35 @@
 			};
 		};
 
+		gcc: clock-controller@900000 {
+			compatible = "qcom,gcc-ipq8064", "syscon";
+			clocks = <&pxo_board>, <&cxo_board>;
+			clock-names = "pxo", "cxo";
+			reg = <0x00900000 0x4000>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+
+			tsens: thermal-sensor@900000 {
+				compatible = "qcom,ipq8064-tsens";
+
+				nvmem-cells = <&tsens_calib>, <&tsens_calib_backup>;
+				nvmem-cell-names = "calib", "calib_backup";
+				interrupts = <GIC_SPI 178 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "uplow";
+
+				#qcom,sensors = <11>;
+				#thermal-sensor-cells = <1>;
+			};
+		};
+
+		sfpb_mutex: hwlock@1200600 {
+			compatible = "qcom,sfpb-mutex";
+			reg = <0x01200600 0x100>;
+
+			#hwlock-cells = <1>;
+		};
+
 		intc: interrupt-controller@2000000 {
 			compatible = "qcom,msm-qgic2";
 			interrupt-controller;
@@ -500,46 +569,198 @@
 			cpu-offset = <0x80000>;
 		};
 
+		l2cc: clock-controller@2011000 {
+			compatible = "qcom,kpss-gcc", "syscon";
+			reg = <0x02011000 0x1000>;
+			clocks = <&gcc PLL8_VOTE>, <&pxo_board>;
+			clock-names = "pll8_vote", "pxo";
+			clock-output-names = "acpu_l2_aux";
+		};
+
 		acc0: clock-controller@2088000 {
 			compatible = "qcom,kpss-acc-v1";
 			reg = <0x02088000 0x1000>, <0x02008000 0x1000>;
 		};
 
+		saw0: regulator@2089000 {
+			compatible = "qcom,saw2";
+			reg = <0x02089000 0x1000>, <0x02009000 0x1000>;
+			regulator;
+		};
+
 		acc1: clock-controller@2098000 {
 			compatible = "qcom,kpss-acc-v1";
 			reg = <0x02098000 0x1000>, <0x02008000 0x1000>;
 		};
 
-		adm_dma: dma-controller@18300000 {
-			compatible = "qcom,adm";
-			reg = <0x18300000 0x100000>;
-			interrupts = <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>;
-			#dma-cells = <1>;
+		saw1: regulator@2099000 {
+			compatible = "qcom,saw2";
+			reg = <0x02099000 0x1000>, <0x02009000 0x1000>;
+			regulator;
+		};
 
-			clocks = <&gcc ADM0_CLK>, <&gcc ADM0_PBUS_CLK>;
-			clock-names = "core", "iface";
+		nss_common: syscon@03000000 {
+			compatible = "syscon";
+			reg = <0x03000000 0x0000FFFF>;
+		};
 
-			resets = <&gcc ADM0_RESET>,
-				 <&gcc ADM0_PBUS_RESET>,
-				 <&gcc ADM0_C0_RESET>,
-				 <&gcc ADM0_C1_RESET>,
-				 <&gcc ADM0_C2_RESET>;
-			reset-names = "clk", "pbus", "c0", "c1", "c2";
-			qcom,ee = <0>;
+		usb3_0: usb3@100f8800 {
+			compatible = "qcom,ipq8064-dwc3", "qcom,dwc3";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			reg = <0x100f8800 0x8000>;
+			clocks = <&gcc USB30_0_MASTER_CLK>;
+			clock-names = "core";
+
+			ranges;
+
+			resets = <&gcc USB30_0_MASTER_RESET>;
+			reset-names = "master";
 
 			status = "disabled";
+
+			dwc3_0: dwc3@10000000 {
+				compatible = "snps,dwc3";
+				reg = <0x10000000 0xcd00>;
+				interrupts = <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH>;
+				phys = <&hs_phy_0>, <&ss_phy_0>;
+				phy-names = "usb2-phy", "usb3-phy";
+				dr_mode = "host";
+				snps,dis_u3_susphy_quirk;
+			};
 		};
 
-		saw0: regulator@2089000 {
-			compatible = "qcom,saw2";
-			reg = <0x02089000 0x1000>, <0x02009000 0x1000>;
-			regulator;
+		hs_phy_0: phy@100f8800 {
+			compatible = "qcom,ipq806x-usb-phy-hs";
+			reg = <0x100f8800 0x30>;
+			clocks = <&gcc USB30_0_UTMI_CLK>;
+			clock-names = "ref";
+			#phy-cells = <0>;
+
+			status = "disabled";
 		};
 
-		saw1: regulator@2099000 {
-			compatible = "qcom,saw2";
-			reg = <0x02099000 0x1000>, <0x02009000 0x1000>;
-			regulator;
+		ss_phy_0: phy@100f8830 {
+			compatible = "qcom,ipq806x-usb-phy-ss";
+			reg = <0x100f8830 0x30>;
+			clocks = <&gcc USB30_0_MASTER_CLK>;
+			clock-names = "ref";
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		usb3_1: usb3@110f8800 {
+			compatible = "qcom,ipq8064-dwc3", "qcom,dwc3";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			reg = <0x110f8800 0x8000>;
+			clocks = <&gcc USB30_1_MASTER_CLK>;
+			clock-names = "core";
+
+			ranges;
+
+			resets = <&gcc USB30_1_MASTER_RESET>;
+			reset-names = "master";
+
+			status = "disabled";
+
+			dwc3_1: dwc3@11000000 {
+				compatible = "snps,dwc3";
+				reg = <0x11000000 0xcd00>;
+				interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
+				phys = <&hs_phy_1>, <&ss_phy_1>;
+				phy-names = "usb2-phy", "usb3-phy";
+				dr_mode = "host";
+				snps,dis_u3_susphy_quirk;
+			};
+		};
+
+		hs_phy_1: phy@110f8800 {
+			compatible = "qcom,ipq806x-usb-phy-hs";
+			reg = <0x110f8800 0x30>;
+			clocks = <&gcc USB30_1_UTMI_CLK>;
+			clock-names = "ref";
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		ss_phy_1: phy@110f8830 {
+			compatible = "qcom,ipq806x-usb-phy-ss";
+			reg = <0x110f8830 0x30>;
+			clocks = <&gcc USB30_1_MASTER_CLK>;
+			clock-names = "ref";
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		sdcc3bam: dma-controller@12182000 {
+			compatible = "qcom,bam-v1.3.0";
+			reg = <0x12182000 0x8000>;
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc SDC3_H_CLK>;
+			clock-names = "bam_clk";
+			#dma-cells = <1>;
+			qcom,ee = <0>;
+		};
+
+		sdcc1bam: dma-controller@12402000 {
+			compatible = "qcom,bam-v1.3.0";
+			reg = <0x12402000 0x8000>;
+			interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&gcc SDC1_H_CLK>;
+			clock-names = "bam_clk";
+			#dma-cells = <1>;
+			qcom,ee = <0>;
+		};
+
+		amba: amba {
+			compatible = "simple-bus";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			sdcc3: mmc@12180000 {
+				compatible = "arm,pl18x", "arm,primecell";
+				arm,primecell-periphid = <0x00051180>;
+				status = "disabled";
+				reg = <0x12180000 0x2000>;
+				interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "cmd_irq";
+				clocks = <&gcc SDC3_CLK>, <&gcc SDC3_H_CLK>;
+				clock-names = "mclk", "apb_pclk";
+				bus-width = <8>;
+				cap-sd-highspeed;
+				cap-mmc-highspeed;
+				max-frequency = <192000000>;
+				sd-uhs-sdr104;
+				sd-uhs-ddr50;
+				vqmmc-supply = <&vsdcc_fixed>;
+				dmas = <&sdcc3bam 2>, <&sdcc3bam 1>;
+				dma-names = "tx", "rx";
+			};
+
+			sdcc1: mmc@12400000 {
+				status = "disabled";
+				compatible = "arm,pl18x", "arm,primecell";
+				arm,primecell-periphid = <0x00051180>;
+				reg = <0x12400000 0x2000>;
+				interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "cmd_irq";
+				clocks = <&gcc SDC1_CLK>, <&gcc SDC1_H_CLK>;
+				clock-names = "mclk", "apb_pclk";
+				bus-width = <8>;
+				max-frequency = <96000000>;
+				non-removable;
+				cap-sd-highspeed;
+				cap-mmc-highspeed;
+				mmc-ddr-1_8v;
+				vmmc-supply = <&vsdcc_fixed>;
+				dmas = <&sdcc1bam 2>, <&sdcc1bam 1>;
+				dma-names = "tx", "rx";
+			};
 		};
 
 		gsbi1: gsbi@12440000 {
@@ -654,56 +875,6 @@
 			};
 		};
 
-		gsbi5: gsbi@1a200000 {
-			compatible = "qcom,gsbi-v1.0.0";
-			cell-index = <5>;
-			reg = <0x1a200000 0x100>;
-			clocks = <&gcc GSBI5_H_CLK>;
-			clock-names = "iface";
-			#address-cells = <1>;
-			#size-cells = <1>;
-			ranges;
-			status = "disabled";
-
-			syscon-tcsr = <&tcsr>;
-
-			gsbi5_serial: serial@1a240000 {
-				compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm";
-				reg = <0x1a240000 0x1000>,
-				      <0x1a200000 0x1000>;
-				interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&gcc GSBI5_UART_CLK>, <&gcc GSBI5_H_CLK>;
-				clock-names = "core", "iface";
-				status = "disabled";
-			};
-
-			i2c@1a280000 {
-				compatible = "qcom,i2c-qup-v1.1.1";
-				reg = <0x1a280000 0x1000>;
-				interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
-
-				clocks = <&gcc GSBI5_QUP_CLK>, <&gcc GSBI5_H_CLK>;
-				clock-names = "core", "iface";
-				status = "disabled";
-
-				#address-cells = <1>;
-				#size-cells = <0>;
-			};
-
-			spi@1a280000 {
-				compatible = "qcom,spi-qup-v1.1.1";
-				reg = <0x1a280000 0x1000>;
-				interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
-
-				clocks = <&gcc GSBI5_QUP_CLK>, <&gcc GSBI5_H_CLK>;
-				clock-names = "core", "iface";
-				status = "disabled";
-
-				#address-cells = <1>;
-				#size-cells = <0>;
-			};
-		};
-
 		gsbi6: gsbi@16500000 {
 			compatible = "qcom,gsbi-v1.0.0";
 			reg = <0x16500000 0x100>;
@@ -784,6 +955,82 @@
 			};
 		};
 
+		adm_dma: dma-controller@18300000 {
+			compatible = "qcom,adm";
+			reg = <0x18300000 0x100000>;
+			interrupts = <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+
+			clocks = <&gcc ADM0_CLK>, <&gcc ADM0_PBUS_CLK>;
+			clock-names = "core", "iface";
+
+			resets = <&gcc ADM0_RESET>,
+				 <&gcc ADM0_PBUS_RESET>,
+				 <&gcc ADM0_C0_RESET>,
+				 <&gcc ADM0_C1_RESET>,
+				 <&gcc ADM0_C2_RESET>;
+			reset-names = "clk", "pbus", "c0", "c1", "c2";
+			qcom,ee = <0>;
+
+			status = "disabled";
+		};
+
+		gsbi5: gsbi@1a200000 {
+			compatible = "qcom,gsbi-v1.0.0";
+			cell-index = <5>;
+			reg = <0x1a200000 0x100>;
+			clocks = <&gcc GSBI5_H_CLK>;
+			clock-names = "iface";
+			#address-cells = <1>;
+
+			#size-cells = <1>;
+			ranges;
+			status = "disabled";
+
+			syscon-tcsr = <&tcsr>;
+
+			gsbi5_serial: serial@1a240000 {
+				compatible = "qcom,msm-uartdm-v1.3", "qcom,msm-uartdm";
+				reg = <0x1a240000 0x1000>,
+				      <0x1a200000 0x1000>;
+				interrupts = <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GSBI5_UART_CLK>, <&gcc GSBI5_H_CLK>;
+				clock-names = "core", "iface";
+				status = "disabled";
+			};
+
+			i2c@1a280000 {
+				compatible = "qcom,i2c-qup-v1.1.1";
+				reg = <0x1a280000 0x1000>;
+				interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GSBI5_QUP_CLK>, <&gcc GSBI5_H_CLK>;
+				clock-names = "core", "iface";
+				status = "disabled";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+
+			spi@1a280000 {
+				compatible = "qcom,spi-qup-v1.1.1";
+				reg = <0x1a280000 0x1000>;
+				interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
+
+				clocks = <&gcc GSBI5_QUP_CLK>, <&gcc GSBI5_H_CLK>;
+				clock-names = "core", "iface";
+				status = "disabled";
+
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
+		tcsr: syscon@1a400000 {
+			compatible = "qcom,tcsr-ipq8064", "syscon";
+			reg = <0x1a400000 0x100>;
+		};
+
 		rng@1a500000 {
 			compatible = "qcom,prng";
 			reg = <0x1a500000 0x200>;
@@ -791,17 +1038,6 @@
 			clock-names = "core";
 		};
 
-		sata_phy: sata-phy@1b400000 {
-			compatible = "qcom,ipq806x-sata-phy";
-			reg = <0x1b400000 0x200>;
-
-			clocks = <&gcc SATA_PHY_CFG_CLK>;
-			clock-names = "cfg";
-
-			#phy-cells = <0>;
-			status = "disabled";
-		};
-
 		nand: nand-controller@1ac00000 {
 			compatible = "qcom,ipq806x-nand";
 			reg = <0x1ac00000 0x800>;
@@ -824,113 +1060,17 @@
 			status = "disabled";
 		};
 
-		sata: sata@29000000 {
-			compatible = "qcom,ipq806x-ahci", "generic-ahci";
-			reg = <0x29000000 0x180>;
-
-			interrupts = <GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH>;
-
-			clocks = <&gcc SFAB_SATA_S_H_CLK>,
-				 <&gcc SATA_H_CLK>,
-				 <&gcc SATA_A_CLK>,
-				 <&gcc SATA_RXOOB_CLK>,
-				 <&gcc SATA_PMALIVE_CLK>;
-			clock-names = "slave_face", "iface", "core",
-					"rxoob", "pmalive";
+		sata_phy: sata-phy@1b400000 {
+			compatible = "qcom,ipq806x-sata-phy";
+			reg = <0x1b400000 0x200>;
 
-			assigned-clocks = <&gcc SATA_RXOOB_CLK>, <&gcc SATA_PMALIVE_CLK>;
-			assigned-clock-rates = <100000000>, <100000000>;
+			clocks = <&gcc SATA_PHY_CFG_CLK>;
+			clock-names = "cfg";
 
-			phys = <&sata_phy>;
-			phy-names = "sata-phy";
+			#phy-cells = <0>;
 			status = "disabled";
 		};
 
-		qcom,ssbi@500000 {
-			compatible = "qcom,ssbi";
-			reg = <0x00500000 0x1000>;
-			qcom,controller-type = "pmic-arbiter";
-		};
-
-		qfprom: qfprom@700000 {
-			compatible = "qcom,ipq8064-qfprom", "qcom,qfprom";
-			reg = <0x00700000 0x1000>;
-			#address-cells = <1>;
-			#size-cells = <1>;
-			speedbin_efuse: speedbin@c0 {
-				reg = <0xc0 0x4>;
-			};
-			tsens_calib: calib@400 {
-				reg = <0x400 0xb>;
-			};
-			tsens_calib_backup: calib_backup@410 {
-				reg = <0x410 0xb>;
-			};
-		};
-
-		gcc: clock-controller@900000 {
-			compatible = "qcom,gcc-ipq8064", "syscon";
-			clocks = <&pxo_board>, <&cxo_board>;
-			clock-names = "pxo", "cxo";
-			reg = <0x00900000 0x4000>;
-			#clock-cells = <1>;
-			#reset-cells = <1>;
-			#power-domain-cells = <1>;
-
-			tsens: thermal-sensor@900000 {
-				compatible = "qcom,ipq8064-tsens";
-
-				nvmem-cells = <&tsens_calib>, <&tsens_calib_backup>;
-				nvmem-cell-names = "calib", "calib_backup";
-				interrupts = <GIC_SPI 178 IRQ_TYPE_LEVEL_HIGH>;
-				interrupt-names = "uplow";
-
-				#qcom,sensors = <11>;
-				#thermal-sensor-cells = <1>;
-			};
-		};
-
-		rpm: rpm@108000 {
-			compatible = "qcom,rpm-ipq8064";
-			reg = <0x108000 0x1000>;
-			qcom,ipc = <&l2cc 0x8 2>;
-
-			interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>,
-					<GIC_SPI 21 IRQ_TYPE_LEVEL_HIGH>,
-					<GIC_SPI 22 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "ack", "err", "wakeup";
-
-			clocks = <&gcc RPM_MSG_RAM_H_CLK>;
-			clock-names = "ram";
-
-			rpmcc: clock-controller {
-				compatible = "qcom,rpmcc-ipq806x", "qcom,rpmcc";
-				#clock-cells = <1>;
-				clocks = <&pxo_board>;
-				clock-names = "pxo";
-			};
-		};
-
-		tcsr: syscon@1a400000 {
-			compatible = "qcom,tcsr-ipq8064", "syscon";
-			reg = <0x1a400000 0x100>;
-		};
-
-		l2cc: clock-controller@2011000 {
-			compatible = "qcom,kpss-gcc", "syscon";
-			reg = <0x2011000 0x1000>;
-			clocks = <&gcc PLL8_VOTE>, <&pxo_board>;
-			clock-names = "pll8_vote", "pxo";
-			clock-output-names = "acpu_l2_aux";
-		};
-
-		lcc: clock-controller@28000000 {
-			compatible = "qcom,lcc-ipq8064";
-			reg = <0x28000000 0x1000>;
-			#clock-cells = <1>;
-			#reset-cells = <1>;
-		};
-
 		pcie0: pci@1b500000 {
 			compatible = "qcom,pcie-ipq8064";
 			reg = <0x1b500000 0x1000
@@ -979,7 +1119,7 @@
 			pinctrl-names = "default";
 
 			status = "disabled";
-			perst-gpio = <&qcom_pinmux 3 GPIO_ACTIVE_LOW>;
+			perst-gpios = <&qcom_pinmux 3 GPIO_ACTIVE_LOW>;
 		};
 
 		pcie1: pci@1b700000 {
@@ -1030,7 +1170,7 @@
 			pinctrl-names = "default";
 
 			status = "disabled";
-			perst-gpio = <&qcom_pinmux 48 GPIO_ACTIVE_LOW>;
+			perst-gpios = <&qcom_pinmux 48 GPIO_ACTIVE_LOW>;
 		};
 
 		pcie2: pci@1b900000 {
@@ -1081,12 +1221,7 @@
 			pinctrl-names = "default";
 
 			status = "disabled";
-			perst-gpio = <&qcom_pinmux 63 GPIO_ACTIVE_LOW>;
-		};
-
-		nss_common: syscon@03000000 {
-			compatible = "syscon";
-			reg = <0x03000000 0x0000FFFF>;
+			perst-gpios = <&qcom_pinmux 63 GPIO_ACTIVE_LOW>;
 		};
 
 		qsgmii_csr: syscon@1bb00000 {
@@ -1094,10 +1229,48 @@
 			reg = <0x1bb00000 0x000001FF>;
 		};
 
-		stmmac_axi_setup: stmmac-axi-config {
-			snps,wr_osr_lmt = <7>;
-			snps,rd_osr_lmt = <7>;
-			snps,blen = <16 0 0 0 0 0 0>;
+		lcc: clock-controller@28000000 {
+			compatible = "qcom,lcc-ipq8064";
+			reg = <0x28000000 0x1000>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		lpass@28100000 {
+			compatible = "qcom,lpass-cpu";
+			status = "disabled";
+			clocks = <&lcc AHBIX_CLK>,
+					<&lcc MI2S_OSR_CLK>,
+					<&lcc MI2S_BIT_CLK>;
+			clock-names = "ahbix-clk",
+					"mi2s-osr-clk",
+					"mi2s-bit-clk";
+			interrupts = <GIC_SPI 85 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "lpass-irq-lpaif";
+			reg = <0x28100000 0x10000>;
+			reg-names = "lpass-lpaif";
+		};
+
+		sata: sata@29000000 {
+			compatible = "qcom,ipq806x-ahci", "generic-ahci";
+			reg = <0x29000000 0x180>;
+
+			interrupts = <GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc SFAB_SATA_S_H_CLK>,
+				 <&gcc SATA_H_CLK>,
+				 <&gcc SATA_A_CLK>,
+				 <&gcc SATA_RXOOB_CLK>,
+				 <&gcc SATA_PMALIVE_CLK>;
+			clock-names = "slave_face", "iface", "core",
+					"rxoob", "pmalive";
+
+			assigned-clocks = <&gcc SATA_RXOOB_CLK>, <&gcc SATA_PMALIVE_CLK>;
+			assigned-clock-rates = <100000000>, <100000000>;
+
+			phys = <&sata_phy>;
+			phy-names = "sata-phy";
+			status = "disabled";
 		};
 
 		gmac0: ethernet@37000000 {
@@ -1195,179 +1368,5 @@
 
 			status = "disabled";
 		};
-
-		hs_phy_0: phy@100f8800 {
-			compatible = "qcom,ipq806x-usb-phy-hs";
-			reg = <0x100f8800 0x30>;
-			clocks = <&gcc USB30_0_UTMI_CLK>;
-			clock-names = "ref";
-			#phy-cells = <0>;
-
-			status = "disabled";
-		};
-
-		ss_phy_0: phy@100f8830 {
-			compatible = "qcom,ipq806x-usb-phy-ss";
-			reg = <0x100f8830 0x30>;
-			clocks = <&gcc USB30_0_MASTER_CLK>;
-			clock-names = "ref";
-			#phy-cells = <0>;
-
-			status = "disabled";
-		};
-
-		usb3_0: usb3@100f8800 {
-			compatible = "qcom,ipq8064-dwc3", "qcom,dwc3";
-			#address-cells = <1>;
-			#size-cells = <1>;
-			reg = <0x100f8800 0x8000>;
-			clocks = <&gcc USB30_0_MASTER_CLK>;
-			clock-names = "core";
-
-			ranges;
-
-			resets = <&gcc USB30_0_MASTER_RESET>;
-			reset-names = "master";
-
-			status = "disabled";
-
-			dwc3_0: dwc3@10000000 {
-				compatible = "snps,dwc3";
-				reg = <0x10000000 0xcd00>;
-				interrupts = <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH>;
-				phys = <&hs_phy_0>, <&ss_phy_0>;
-				phy-names = "usb2-phy", "usb3-phy";
-				dr_mode = "host";
-				snps,dis_u3_susphy_quirk;
-			};
-		};
-
-		hs_phy_1: phy@110f8800 {
-			compatible = "qcom,ipq806x-usb-phy-hs";
-			reg = <0x110f8800 0x30>;
-			clocks = <&gcc USB30_1_UTMI_CLK>;
-			clock-names = "ref";
-			#phy-cells = <0>;
-
-			status = "disabled";
-		};
-
-		ss_phy_1: phy@110f8830 {
-			compatible = "qcom,ipq806x-usb-phy-ss";
-			reg = <0x110f8830 0x30>;
-			clocks = <&gcc USB30_1_MASTER_CLK>;
-			clock-names = "ref";
-			#phy-cells = <0>;
-
-			status = "disabled";
-		};
-
-		usb3_1: usb3@110f8800 {
-			compatible = "qcom,ipq8064-dwc3", "qcom,dwc3";
-			#address-cells = <1>;
-			#size-cells = <1>;
-			reg = <0x110f8800 0x8000>;
-			clocks = <&gcc USB30_1_MASTER_CLK>;
-			clock-names = "core";
-
-			ranges;
-
-			resets = <&gcc USB30_1_MASTER_RESET>;
-			reset-names = "master";
-
-			status = "disabled";
-
-			dwc3_1: dwc3@11000000 {
-				compatible = "snps,dwc3";
-				reg = <0x11000000 0xcd00>;
-				interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
-				phys = <&hs_phy_1>, <&ss_phy_1>;
-				phy-names = "usb2-phy", "usb3-phy";
-				dr_mode = "host";
-				snps,dis_u3_susphy_quirk;
-			};
-		};
-
-		vsdcc_fixed: vsdcc-regulator {
-			compatible = "regulator-fixed";
-			regulator-name = "SDCC Power";
-			regulator-min-microvolt = <3300000>;
-			regulator-max-microvolt = <3300000>;
-			regulator-always-on;
-		};
-
-		sdcc1bam: dma-controller@12402000 {
-			compatible = "qcom,bam-v1.3.0";
-			reg = <0x12402000 0x8000>;
-			interrupts = <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&gcc SDC1_H_CLK>;
-			clock-names = "bam_clk";
-			#dma-cells = <1>;
-			qcom,ee = <0>;
-		};
-
-		sdcc3bam: dma-controller@12182000 {
-			compatible = "qcom,bam-v1.3.0";
-			reg = <0x12182000 0x8000>;
-			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&gcc SDC3_H_CLK>;
-			clock-names = "bam_clk";
-			#dma-cells = <1>;
-			qcom,ee = <0>;
-		};
-
-		amba: amba {
-			compatible = "simple-bus";
-			#address-cells = <1>;
-			#size-cells = <1>;
-			ranges;
-
-			sdcc1: mmc@12400000 {
-				status = "disabled";
-				compatible = "arm,pl18x", "arm,primecell";
-				arm,primecell-periphid = <0x00051180>;
-				reg = <0x12400000 0x2000>;
-				interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
-				interrupt-names = "cmd_irq";
-				clocks = <&gcc SDC1_CLK>, <&gcc SDC1_H_CLK>;
-				clock-names = "mclk", "apb_pclk";
-				bus-width = <8>;
-				max-frequency = <96000000>;
-				non-removable;
-				cap-sd-highspeed;
-				cap-mmc-highspeed;
-				mmc-ddr-1_8v;
-				vmmc-supply = <&vsdcc_fixed>;
-				dmas = <&sdcc1bam 2>, <&sdcc1bam 1>;
-				dma-names = "tx", "rx";
-			};
-
-			sdcc3: mmc@12180000 {
-				compatible = "arm,pl18x", "arm,primecell";
-				arm,primecell-periphid = <0x00051180>;
-				status = "disabled";
-				reg = <0x12180000 0x2000>;
-				interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
-				interrupt-names = "cmd_irq";
-				clocks = <&gcc SDC3_CLK>, <&gcc SDC3_H_CLK>;
-				clock-names = "mclk", "apb_pclk";
-				bus-width = <8>;
-				cap-sd-highspeed;
-				cap-mmc-highspeed;
-				max-frequency = <192000000>;
-				sd-uhs-sdr104;
-				sd-uhs-ddr50;
-				vqmmc-supply = <&vsdcc_fixed>;
-				dmas = <&sdcc3bam 2>, <&sdcc3bam 1>;
-				dma-names = "tx", "rx";
-			};
-		};
-
-		sfpb_mutex: hwlock@1200600 {
-			compatible = "qcom,sfpb-mutex";
-			reg = <0x01200600 0x100>;
-
-			#hwlock-cells = <1>;
-		};
 	};
 };
diff --git a/arch/arm/boot/dts/qcom-ipq8065-smb208.dtsi b/arch/arm/boot/dts/qcom-ipq8065-smb208.dtsi
new file mode 100644
index 000000000000..803e6ff99ef8
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq8065-smb208.dtsi
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "qcom-ipq8065.dtsi"
+
+&rpm {
+	smb208_regulators: regulators {
+		compatible = "qcom,rpm-smb208-regulators";
+
+		smb208_s1a: s1a {
+			regulator-min-microvolt = <1050000>;
+			regulator-max-microvolt = <1150000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+
+		smb208_s1b: s1b {
+			regulator-min-microvolt = <1050000>;
+			regulator-max-microvolt = <1150000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+
+		smb208_s2a: s2a {
+			regulator-min-microvolt = <775000>;
+			regulator-max-microvolt = <1275000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+
+		smb208_s2b: s2b {
+			regulator-min-microvolt = <775000>;
+			regulator-max-microvolt = <1275000>;
+
+			qcom,switch-mode-frequency = <1200000>;
+		};
+	};
+};
diff --git a/arch/arm/boot/dts/qcom-ipq8065.dtsi b/arch/arm/boot/dts/qcom-ipq8065.dtsi
new file mode 100644
index 000000000000..ea49f6cc416d
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-ipq8065.dtsi
@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "qcom-ipq8064-v2.0.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ8065";
+	compatible = "qcom,ipq8065", "qcom,ipq8064";
+};
diff --git a/arch/arm/boot/dts/qcom-msm8226-samsung-s3ve3g.dts b/arch/arm/boot/dts/qcom-msm8226-samsung-s3ve3g.dts
index d159188c8b95..290e1df631f0 100644
--- a/arch/arm/boot/dts/qcom-msm8226-samsung-s3ve3g.dts
+++ b/arch/arm/boot/dts/qcom-msm8226-samsung-s3ve3g.dts
@@ -18,8 +18,6 @@
 	};
 };
 
-&soc {
-	serial@f991f000 {
-		status = "ok";
-	};
+&blsp1_uart3 {
+	status = "ok";
 };
diff --git a/arch/arm/boot/dts/qcom-msm8226.dtsi b/arch/arm/boot/dts/qcom-msm8226.dtsi
index 0b5effdb269a..cf2d56929428 100644
--- a/arch/arm/boot/dts/qcom-msm8226.dtsi
+++ b/arch/arm/boot/dts/qcom-msm8226.dtsi
@@ -8,6 +8,7 @@
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/clock/qcom,gcc-msm8974.h>
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/reset/qcom,gcc-msm8974.h>
 
 / {
@@ -44,13 +45,6 @@
 		};
 	};
 
-	tcsr_mutex: hwlock {
-		compatible = "qcom,tcsr-mutex";
-		syscon = <&tcsr_mutex_block 0 0x80>;
-
-		#hwlock-cells = <1>;
-	};
-
 	reserved-memory {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -60,6 +54,11 @@
 			reg = <0x3000000 0x100000>;
 			no-map;
 		};
+
+		adsp_region: adsp@dc00000 {
+			reg = <0x0dc00000 0x1900000>;
+			no-map;
+		};
 	};
 
 	smd {
@@ -115,6 +114,31 @@
 		hwlocks = <&tcsr_mutex 3>;
 	};
 
+	smp2p-adsp {
+		compatible = "qcom,smp2p";
+		qcom,smem = <443>, <429>;
+
+		interrupt-parent = <&intc>;
+		interrupts = <GIC_SPI 158 IRQ_TYPE_EDGE_RISING>;
+
+		qcom,ipc = <&apcs 8 10>;
+
+		qcom,local-pid = <0>;
+		qcom,remote-pid = <2>;
+
+		adsp_smp2p_out: master-kernel {
+			qcom,entry-name = "master-kernel";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		adsp_smp2p_in: slave-kernel {
+			qcom,entry-name = "slave-kernel";
+
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+	};
+
 	soc: soc {
 		compatible = "simple-bus";
 		#address-cells = <1>;
@@ -137,14 +161,14 @@
 		sdhc_1: mmc@f9824900 {
 			compatible = "qcom,msm8226-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf9824900 0x11c>, <0xf9824000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC1_APPS_CLK>,
-				 <&gcc GCC_SDCC1_AHB_CLK>,
+			clocks = <&gcc GCC_SDCC1_AHB_CLK>,
+				 <&gcc GCC_SDCC1_APPS_CLK>,
 				 <&xo_board>;
-			clock-names = "core", "iface", "xo";
+			clock-names = "iface", "core", "xo";
 			pinctrl-names = "default";
 			pinctrl-0 = <&sdhc1_default_state>;
 			status = "disabled";
@@ -153,14 +177,14 @@
 		sdhc_2: mmc@f98a4900 {
 			compatible = "qcom,msm8226-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf98a4900 0x11c>, <0xf98a4000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 			interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC2_APPS_CLK>,
-				 <&gcc GCC_SDCC2_AHB_CLK>,
+			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&gcc GCC_SDCC2_APPS_CLK>,
 				 <&xo_board>;
-			clock-names = "core", "iface", "xo";
+			clock-names = "iface", "core", "xo";
 			pinctrl-names = "default";
 			pinctrl-0 = <&sdhc2_default_state>;
 			status = "disabled";
@@ -169,14 +193,14 @@
 		sdhc_3: mmc@f9864900 {
 			compatible = "qcom,msm8226-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf9864900 0x11c>, <0xf9864000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 			interrupts = <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC3_APPS_CLK>,
-				 <&gcc GCC_SDCC3_AHB_CLK>,
+			clocks = <&gcc GCC_SDCC3_AHB_CLK>,
+				 <&gcc GCC_SDCC3_APPS_CLK>,
 				 <&xo_board>;
-			clock-names = "core", "iface", "xo";
+			clock-names = "iface", "core", "xo";
 			pinctrl-names = "default";
 			pinctrl-0 = <&sdhc3_default_state>;
 			status = "disabled";
@@ -508,9 +532,44 @@
 			reg = <0xfc428000 0x4000>;
 		};
 
-		tcsr_mutex_block: syscon@fd484000 {
-			compatible = "syscon";
-			reg = <0xfd484000 0x2000>;
+		tcsr_mutex: hwlock@fd484000 {
+			compatible = "qcom,msm8226-tcsr-mutex", "qcom,tcsr-mutex";
+			reg = <0xfd484000 0x1000>;
+			#hwlock-cells = <1>;
+		};
+
+		adsp: remoteproc@fe200000 {
+			compatible = "qcom,msm8226-adsp-pil";
+			reg = <0xfe200000 0x100>;
+
+			interrupts-extended = <&intc GIC_SPI 162 IRQ_TYPE_EDGE_RISING>,
+					      <&adsp_smp2p_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&adsp_smp2p_in 1 IRQ_TYPE_EDGE_RISING>,
+					      <&adsp_smp2p_in 2 IRQ_TYPE_EDGE_RISING>,
+					      <&adsp_smp2p_in 3 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "wdog", "fatal", "ready", "handover", "stop-ack";
+
+			power-domains = <&rpmpd MSM8226_VDDCX>;
+			power-domain-names = "cx";
+
+			clocks = <&xo_board>;
+			clock-names = "xo";
+
+			memory-region = <&adsp_region>;
+
+			qcom,smem-states = <&adsp_smp2p_out 0>;
+			qcom,smem-state-names = "stop";
+
+			status = "disabled";
+
+			smd-edge {
+				interrupts = <GIC_SPI 156 IRQ_TYPE_EDGE_RISING>;
+
+				qcom,ipc = <&apcs 8 8>;
+				qcom,smd-edge = <1>;
+
+				label = "lpass";
+			};
 		};
 	};
 
diff --git a/arch/arm/boot/dts/qcom-msm8660-surf.dts b/arch/arm/boot/dts/qcom-msm8660-surf.dts
index 414280d9bdba..be18f1be29a1 100644
--- a/arch/arm/boot/dts/qcom-msm8660-surf.dts
+++ b/arch/arm/boot/dts/qcom-msm8660-surf.dts
@@ -15,38 +15,23 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	soc {
-		gsbi@19c00000 {
-			status = "okay";
-			qcom,mode = <GSBI_PROT_I2C_UART>;
-			serial@19c40000 {
-				status = "okay";
-			};
-		};
-
-		/* Temporary fixed regulator */
-		vsdcc_fixed: vsdcc-regulator {
-			compatible = "regulator-fixed";
-			regulator-name = "SDCC Power";
-			regulator-min-microvolt = <2700000>;
-			regulator-max-microvolt = <2700000>;
-			regulator-always-on;
-		};
+	/* Temporary fixed regulator */
+	vsdcc_fixed: vsdcc-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "SDCC Power";
+		regulator-min-microvolt = <2700000>;
+		regulator-max-microvolt = <2700000>;
+		regulator-always-on;
+	};
+};
 
-		amba {
-			/* eMMC */
-			sdcc1: mmc@12400000 {
-				status = "okay";
-				vmmc-supply = <&vsdcc_fixed>;
-			};
+&gsbi12 {
+	qcom,mode = <GSBI_PROT_I2C_UART>;
+	status = "okay";
+};
 
-			/* External micro SD card */
-			sdcc3: mmc@12180000 {
-				status = "okay";
-				vmmc-supply = <&vsdcc_fixed>;
-			};
-		};
-	};
+&gsbi12_serial {
+	status = "okay";
 };
 
 &pm8058 {
@@ -76,3 +61,15 @@
 		keypad,num-columns = <5>;
 	};
 };
+
+/* eMMC */
+&sdcc1 {
+	vmmc-supply = <&vsdcc_fixed>;
+	status = "okay";
+};
+
+/* External micro SD card */
+&sdcc3 {
+	vmmc-supply = <&vsdcc_fixed>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/qcom-msm8660.dtsi b/arch/arm/boot/dts/qcom-msm8660.dtsi
index 63a501c63cf8..ddce7d64ba99 100644
--- a/arch/arm/boot/dts/qcom-msm8660.dtsi
+++ b/arch/arm/boot/dts/qcom-msm8660.dtsi
@@ -50,22 +50,25 @@
 	};
 
 	clocks {
-		cxo_board {
+		cxo_board: cxo-board-clk {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <19200000>;
+			clock-output-names = "cxo_board";
 		};
 
-		pxo_board: pxo_board {
+		pxo_board: pxo-board-clk {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <27000000>;
+			clock-output-names = "pxo_board";
 		};
 
-		sleep_clk {
+		sleep-clk {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <32768>;
+			clock-output-names = "sleep_clk";
 		};
 	};
 
@@ -129,6 +132,59 @@
 			#power-domain-cells = <1>;
 			#reset-cells = <1>;
 			reg = <0x900000 0x4000>;
+			clocks = <&pxo_board>, <&cxo_board>;
+			clock-names = "pxo", "cxo";
+		};
+
+		gsbi1: gsbi@16000000 {
+			compatible = "qcom,gsbi-v1.0.0";
+			cell-index = <12>;
+			reg = <0x16000000 0x100>;
+			clocks = <&gcc GSBI1_H_CLK>;
+			clock-names = "iface";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			syscon-tcsr = <&tcsr>;
+
+			status = "disabled";
+
+			gsbi1_spi: spi@16080000 {
+				compatible = "qcom,spi-qup-v1.1.1";
+				reg = <0x16080000 0x1000>;
+				interrupts = <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GSBI1_QUP_CLK>, <&gcc GSBI1_H_CLK>;
+				clock-names = "core", "iface";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
+		};
+
+		gsbi3: gsbi@16200000 {
+			compatible = "qcom,gsbi-v1.0.0";
+			cell-index = <12>;
+			reg = <0x16200000 0x100>;
+			clocks = <&gcc GSBI3_H_CLK>;
+			clock-names = "iface";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges;
+
+			syscon-tcsr = <&tcsr>;
+			status = "disabled";
+
+			gsbi3_i2c: i2c@16280000 {
+				compatible = "qcom,i2c-qup-v1.1.1";
+				reg = <0x16280000 0x1000>;
+				interrupts = <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&gcc GSBI3_QUP_CLK>, <&gcc GSBI3_H_CLK>;
+				clock-names = "core", "iface";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				status = "disabled";
+			};
 		};
 
 		gsbi6: gsbi@16500000 {
diff --git a/arch/arm/boot/dts/qcom-msm8916-samsung-e5.dts b/arch/arm/boot/dts/qcom-msm8916-samsung-e5.dts
new file mode 100644
index 000000000000..c8d34de8a71e
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-msm8916-samsung-e5.dts
@@ -0,0 +1,3 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include "arm64/qcom/msm8916-samsung-e5.dts"
+#include "qcom-msm8916-smp.dtsi"
diff --git a/arch/arm/boot/dts/qcom-msm8916-samsung-e7.dts b/arch/arm/boot/dts/qcom-msm8916-samsung-e7.dts
new file mode 100644
index 000000000000..85be286c8608
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-msm8916-samsung-e7.dts
@@ -0,0 +1,3 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include "arm64/qcom/msm8916-samsung-e7.dts"
+#include "qcom-msm8916-smp.dtsi"
diff --git a/arch/arm/boot/dts/qcom-msm8916-samsung-grandmax.dts b/arch/arm/boot/dts/qcom-msm8916-samsung-grandmax.dts
new file mode 100644
index 000000000000..d3abe0536238
--- /dev/null
+++ b/arch/arm/boot/dts/qcom-msm8916-samsung-grandmax.dts
@@ -0,0 +1,3 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include "arm64/qcom/msm8916-samsung-grandmax.dts"
+#include "qcom-msm8916-smp.dtsi"
diff --git a/arch/arm/boot/dts/qcom-msm8960-cdp.dts b/arch/arm/boot/dts/qcom-msm8960-cdp.dts
index d1fd0fe12ffe..9157e3c4f48f 100644
--- a/arch/arm/boot/dts/qcom-msm8960-cdp.dts
+++ b/arch/arm/boot/dts/qcom-msm8960-cdp.dts
@@ -15,318 +15,6 @@
 		stdout-path = "serial0:115200n8";
 	};
 
-	soc {
-		gsbi@16400000 {
-			status = "okay";
-			qcom,mode = <GSBI_PROT_I2C_UART>;
-			serial@16440000 {
-				status = "okay";
-			};
-		};
-
-		amba {
-			/* eMMC */
-			sdcc1: mmc@12400000 {
-				status = "okay";
-			};
-
-			/* External micro SD card */
-			sdcc3: mmc@12180000 {
-				status = "okay";
-			};
-		};
-
-		rpm@108000 {
-			regulators {
-				compatible = "qcom,rpm-pm8921-regulators";
-				vin_lvs1_3_6-supply = <&pm8921_s4>;
-				vin_lvs2-supply = <&pm8921_s4>;
-				vin_lvs4_5_7-supply = <&pm8921_s4>;
-				vdd_ncp-supply = <&pm8921_l6>;
-				vdd_l1_l2_l12_l18-supply = <&pm8921_s4>;
-				vdd_l21_l23_l29-supply = <&pm8921_s8>;
-				vdd_l24-supply = <&pm8921_s1>;
-				vdd_l25-supply = <&pm8921_s1>;
-				vdd_l27-supply = <&pm8921_s7>;
-				vdd_l28-supply = <&pm8921_s7>;
-
-				/* Buck SMPS */
-				pm8921_s1: s1 {
-					regulator-always-on;
-					regulator-min-microvolt = <1225000>;
-					regulator-max-microvolt = <1225000>;
-					qcom,switch-mode-frequency = <3200000>;
-					bias-pull-down;
-				};
-
-				pm8921_s2: s2 {
-					regulator-min-microvolt = <1300000>;
-					regulator-max-microvolt = <1300000>;
-					qcom,switch-mode-frequency = <1600000>;
-					bias-pull-down;
-				};
-
-				pm8921_s3: s3 {
-					regulator-min-microvolt = <500000>;
-					regulator-max-microvolt = <1150000>;
-					qcom,switch-mode-frequency = <4800000>;
-					bias-pull-down;
-				};
-
-				pm8921_s4: s4 {
-					regulator-always-on;
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <1800000>;
-					qcom,switch-mode-frequency = <1600000>;
-					bias-pull-down;
-					qcom,force-mode = <QCOM_RPM_FORCE_MODE_AUTO>;
-				};
-
-				pm8921_s7: s7 {
-					regulator-min-microvolt = <1150000>;
-					regulator-max-microvolt = <1150000>;
-					qcom,switch-mode-frequency = <3200000>;
-					bias-pull-down;
-				};
-
-				pm8921_s8: s8 {
-					regulator-always-on;
-					regulator-min-microvolt = <2050000>;
-					regulator-max-microvolt = <2050000>;
-					qcom,switch-mode-frequency = <1600000>;
-					bias-pull-down;
-				};
-
-				/* PMOS LDO */
-				pm8921_l1: l1 {
-					regulator-always-on;
-					regulator-min-microvolt = <1050000>;
-					regulator-max-microvolt = <1050000>;
-					bias-pull-down;
-				};
-
-				pm8921_l2: l2 {
-					regulator-min-microvolt = <1200000>;
-					regulator-max-microvolt = <1200000>;
-					bias-pull-down;
-				};
-
-				pm8921_l3: l3 {
-					regulator-min-microvolt = <3075000>;
-					regulator-max-microvolt = <3075000>;
-					bias-pull-down;
-				};
-
-				pm8921_l4: l4 {
-					regulator-always-on;
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <1800000>;
-					bias-pull-down;
-				};
-
-				pm8921_l5: l5 {
-					regulator-min-microvolt = <2950000>;
-					regulator-max-microvolt = <2950000>;
-					bias-pull-down;
-				};
-
-				pm8921_l6: l6 {
-					regulator-min-microvolt = <2950000>;
-					regulator-max-microvolt = <2950000>;
-					bias-pull-down;
-				};
-
-				pm8921_l7: l7 {
-					regulator-always-on;
-					regulator-min-microvolt = <1850000>;
-					regulator-max-microvolt = <2950000>;
-					bias-pull-down;
-				};
-
-				pm8921_l8: l8 {
-					regulator-min-microvolt = <2800000>;
-					regulator-max-microvolt = <3000000>;
-					bias-pull-down;
-				};
-
-				pm8921_l9: l9 {
-					regulator-min-microvolt = <3000000>;
-					regulator-max-microvolt = <3000000>;
-					bias-pull-down;
-				};
-
-				pm8921_l10: l10 {
-					regulator-min-microvolt = <3000000>;
-					regulator-max-microvolt = <3000000>;
-					bias-pull-down;
-				};
-
-				pm8921_l11: l11 {
-					regulator-min-microvolt = <2850000>;
-					regulator-max-microvolt = <2850000>;
-					bias-pull-down;
-				};
-
-				pm8921_l12: l12 {
-					regulator-min-microvolt = <1200000>;
-					regulator-max-microvolt = <1200000>;
-					bias-pull-down;
-				};
-
-				pm8921_l14: l14 {
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <1800000>;
-					bias-pull-down;
-				};
-
-				pm8921_l15: l15 {
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <2950000>;
-					bias-pull-down;
-				};
-
-				pm8921_l16: l16 {
-					regulator-min-microvolt = <2800000>;
-					regulator-max-microvolt = <2800000>;
-					bias-pull-down;
-				};
-
-				pm8921_l17: l17 {
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <2950000>;
-					bias-pull-down;
-				};
-
-				pm8921_l18: l18 {
-					regulator-min-microvolt = <1300000>;
-					regulator-max-microvolt = <1300000>;
-					bias-pull-down;
-				};
-
-				pm8921_l21: l21 {
-					regulator-min-microvolt = <1900000>;
-					regulator-max-microvolt = <1900000>;
-					bias-pull-down;
-				};
-
-				pm8921_l22: l22 {
-					regulator-min-microvolt = <2750000>;
-					regulator-max-microvolt = <2750000>;
-					bias-pull-down;
-				};
-
-				pm8921_l23: l23 {
-					regulator-always-on;
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <1800000>;
-					bias-pull-down;
-				};
-
-				pm8921_l24: l24 {
-					regulator-min-microvolt = <750000>;
-					regulator-max-microvolt = <1150000>;
-					bias-pull-down;
-				};
-
-				pm8921_l25: l25 {
-					regulator-always-on;
-					regulator-min-microvolt = <1250000>;
-					regulator-max-microvolt = <1250000>;
-					bias-pull-down;
-				};
-
-				/* Low Voltage Switch */
-				pm8921_lvs1: lvs1 {
-					bias-pull-down;
-				};
-
-				pm8921_lvs2: lvs2 {
-					bias-pull-down;
-				};
-
-				pm8921_lvs3: lvs3 {
-					bias-pull-down;
-				};
-
-				pm8921_lvs4: lvs4 {
-					bias-pull-down;
-				};
-
-				pm8921_lvs5: lvs5 {
-					bias-pull-down;
-				};
-
-				pm8921_lvs6: lvs6 {
-					bias-pull-down;
-				};
-
-				pm8921_lvs7: lvs7 {
-					bias-pull-down;
-				};
-
-				pm8921_ncp: ncp {
-					regulator-min-microvolt = <1800000>;
-					regulator-max-microvolt = <1800000>;
-					qcom,switch-mode-frequency = <1600000>;
-				};
-			};
-		};
-
-		gsbi@16000000 {
-			status = "okay";
-			qcom,mode = <GSBI_PROT_SPI>;
-			pinctrl-names = "default";
-			pinctrl-0 = <&spi1_default>;
-			spi@16080000 {
-				status = "okay";
-				ethernet@0 {
-					compatible = "micrel,ks8851";
-					reg = <0>;
-					interrupt-parent = <&msmgpio>;
-					interrupts = <90 8>;
-					spi-max-frequency = <5400000>;
-					vdd-supply = <&ext_l2>;
-					vdd-io-supply = <&pm8921_lvs6>;
-					reset-gpios = <&msmgpio 89 0>;
-				};
-			};
-		};
-
-		pinctrl@800000 {
-			spi1_default: spi1_default {
-				mux {
-					pins = "gpio6", "gpio7", "gpio9";
-					function = "gsbi1";
-				};
-
-				mosi {
-					pins = "gpio6";
-					drive-strength = <12>;
-					bias-disable;
-				};
-
-				miso {
-					pins = "gpio7";
-					drive-strength = <12>;
-					bias-disable;
-				};
-
-				cs {
-					pins = "gpio8";
-					drive-strength = <12>;
-					bias-disable;
-					output-low;
-				};
-
-				clk {
-					pins = "gpio9";
-					drive-strength = <12>;
-					bias-disable;
-				};
-			};
-		};
-	};
-
 	regulators {
 		compatible = "simple-bus";
 
@@ -340,6 +28,71 @@
 	};
 };
 
+&gsbi1 {
+	qcom,mode = <GSBI_PROT_SPI>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi1_default>;
+	status = "okay";
+};
+
+&gsbi1_spi {
+	status = "okay";
+
+	ethernet@0 {
+		compatible = "micrel,ks8851";
+		reg = <0>;
+		interrupt-parent = <&msmgpio>;
+		interrupts = <90 8>;
+		spi-max-frequency = <5400000>;
+		vdd-supply = <&ext_l2>;
+		vdd-io-supply = <&pm8921_lvs6>;
+		reset-gpios = <&msmgpio 89 0>;
+	};
+};
+
+&gsbi5 {
+	qcom,mode = <GSBI_PROT_I2C_UART>;
+	status = "okay";
+};
+
+&gsbi5_serial {
+	status = "okay";
+};
+
+&msmgpio {
+	spi1_default: spi1_default {
+		 mux {
+			pins = "gpio6", "gpio7", "gpio9";
+			function = "gsbi1";
+		 };
+
+		 mosi {
+			pins = "gpio6";
+			drive-strength = <12>;
+			bias-disable;
+		 };
+
+		 miso {
+			pins = "gpio7";
+			drive-strength = <12>;
+			bias-disable;
+		 };
+
+		 cs {
+			pins = "gpio8";
+			drive-strength = <12>;
+			bias-disable;
+			output-low;
+		 };
+
+		 clk {
+			pins = "gpio9";
+			drive-strength = <12>;
+			bias-disable;
+		 };
+	};
+};
+
 &pmicintc {
 	keypad@148 {
 		linux,keymap = <
@@ -352,3 +105,249 @@
 		keypad,num-columns = <5>;
 	};
 };
+
+&rpm {
+	regulators {
+		compatible = "qcom,rpm-pm8921-regulators";
+		vin_lvs1_3_6-supply = <&pm8921_s4>;
+		vin_lvs2-supply = <&pm8921_s4>;
+		vin_lvs4_5_7-supply = <&pm8921_s4>;
+		vdd_ncp-supply = <&pm8921_l6>;
+		vdd_l1_l2_l12_l18-supply = <&pm8921_s4>;
+		vdd_l21_l23_l29-supply = <&pm8921_s8>;
+		vdd_l24-supply = <&pm8921_s1>;
+		vdd_l25-supply = <&pm8921_s1>;
+		vdd_l27-supply = <&pm8921_s7>;
+		vdd_l28-supply = <&pm8921_s7>;
+
+		/* Buck SMPS */
+		pm8921_s1: s1 {
+			regulator-always-on;
+			regulator-min-microvolt = <1225000>;
+			regulator-max-microvolt = <1225000>;
+			qcom,switch-mode-frequency = <3200000>;
+			bias-pull-down;
+		};
+
+		pm8921_s2: s2 {
+			regulator-min-microvolt = <1300000>;
+			regulator-max-microvolt = <1300000>;
+			qcom,switch-mode-frequency = <1600000>;
+			bias-pull-down;
+		};
+
+		pm8921_s3: s3 {
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <1150000>;
+			qcom,switch-mode-frequency = <4800000>;
+			bias-pull-down;
+		};
+
+		pm8921_s4: s4 {
+			regulator-always-on;
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			qcom,switch-mode-frequency = <1600000>;
+			bias-pull-down;
+			qcom,force-mode = <QCOM_RPM_FORCE_MODE_AUTO>;
+		};
+
+		pm8921_s7: s7 {
+			regulator-min-microvolt = <1150000>;
+			regulator-max-microvolt = <1150000>;
+			qcom,switch-mode-frequency = <3200000>;
+			bias-pull-down;
+		};
+
+		pm8921_s8: s8 {
+			regulator-always-on;
+			regulator-min-microvolt = <2050000>;
+			regulator-max-microvolt = <2050000>;
+			qcom,switch-mode-frequency = <1600000>;
+			bias-pull-down;
+		};
+
+		/* PMOS LDO */
+		pm8921_l1: l1 {
+			regulator-always-on;
+			regulator-min-microvolt = <1050000>;
+			regulator-max-microvolt = <1050000>;
+			bias-pull-down;
+		};
+
+		pm8921_l2: l2 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			bias-pull-down;
+		};
+
+		pm8921_l3: l3 {
+			regulator-min-microvolt = <3075000>;
+			regulator-max-microvolt = <3075000>;
+			bias-pull-down;
+		};
+
+		pm8921_l4: l4 {
+			regulator-always-on;
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			bias-pull-down;
+		};
+
+		pm8921_l5: l5 {
+			regulator-min-microvolt = <2950000>;
+			regulator-max-microvolt = <2950000>;
+			bias-pull-down;
+		};
+
+		pm8921_l6: l6 {
+			regulator-min-microvolt = <2950000>;
+			regulator-max-microvolt = <2950000>;
+			bias-pull-down;
+		};
+
+		pm8921_l7: l7 {
+			regulator-always-on;
+			regulator-min-microvolt = <1850000>;
+			regulator-max-microvolt = <2950000>;
+			bias-pull-down;
+		};
+
+		pm8921_l8: l8 {
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3000000>;
+			bias-pull-down;
+		};
+
+		pm8921_l9: l9 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			bias-pull-down;
+		};
+
+		pm8921_l10: l10 {
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			bias-pull-down;
+		};
+
+		pm8921_l11: l11 {
+			regulator-min-microvolt = <2850000>;
+			regulator-max-microvolt = <2850000>;
+			bias-pull-down;
+		};
+
+		pm8921_l12: l12 {
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			bias-pull-down;
+		};
+
+		pm8921_l14: l14 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			bias-pull-down;
+		};
+
+		pm8921_l15: l15 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+			bias-pull-down;
+		};
+
+		pm8921_l16: l16 {
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			bias-pull-down;
+		};
+
+		pm8921_l17: l17 {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2950000>;
+			bias-pull-down;
+		};
+
+		pm8921_l18: l18 {
+			regulator-min-microvolt = <1300000>;
+			regulator-max-microvolt = <1300000>;
+			bias-pull-down;
+		};
+
+		pm8921_l21: l21 {
+			regulator-min-microvolt = <1900000>;
+			regulator-max-microvolt = <1900000>;
+			bias-pull-down;
+		};
+
+		pm8921_l22: l22 {
+			regulator-min-microvolt = <2750000>;
+			regulator-max-microvolt = <2750000>;
+			bias-pull-down;
+		};
+
+		pm8921_l23: l23 {
+			regulator-always-on;
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			bias-pull-down;
+		};
+
+		pm8921_l24: l24 {
+			regulator-min-microvolt = <750000>;
+			regulator-max-microvolt = <1150000>;
+			bias-pull-down;
+		};
+
+		pm8921_l25: l25 {
+			regulator-always-on;
+			regulator-min-microvolt = <1250000>;
+			regulator-max-microvolt = <1250000>;
+			bias-pull-down;
+		};
+
+		/* Low Voltage Switch */
+		pm8921_lvs1: lvs1 {
+			bias-pull-down;
+		};
+
+		pm8921_lvs2: lvs2 {
+			bias-pull-down;
+		};
+
+		pm8921_lvs3: lvs3 {
+			bias-pull-down;
+		};
+
+		pm8921_lvs4: lvs4 {
+			bias-pull-down;
+		};
+
+		pm8921_lvs5: lvs5 {
+			bias-pull-down;
+		};
+
+		pm8921_lvs6: lvs6 {
+			bias-pull-down;
+		};
+
+		pm8921_lvs7: lvs7 {
+			bias-pull-down;
+		};
+
+		pm8921_ncp: ncp {
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			qcom,switch-mode-frequency = <1600000>;
+		};
+	};
+};
+
+/* eMMC */
+&sdcc1 {
+	status = "okay";
+};
+
+/* External micro SD card */
+&sdcc3 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/qcom-msm8960.dtsi b/arch/arm/boot/dts/qcom-msm8960.dtsi
index 19554f3b5196..c5740da3754c 100644
--- a/arch/arm/boot/dts/qcom-msm8960.dtsi
+++ b/arch/arm/boot/dts/qcom-msm8960.dtsi
@@ -3,6 +3,7 @@
 
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/clock/qcom,gcc-msm8960.h>
+#include <dt-bindings/clock/qcom,lcc-msm8960.h>
 #include <dt-bindings/mfd/qcom-rpm.h>
 #include <dt-bindings/soc/qcom,gsbi.h>
 
@@ -56,14 +57,14 @@
 	};
 
 	clocks {
-		cxo_board {
+		cxo_board: cxo_board {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <19200000>;
 			clock-output-names = "cxo_board";
 		};
 
-		pxo_board {
+		pxo_board: pxo_board {
 			compatible = "fixed-clock";
 			#clock-cells = <0>;
 			clock-frequency = <27000000>;
@@ -130,6 +131,10 @@
 			#power-domain-cells = <1>;
 			#reset-cells = <1>;
 			reg = <0x900000 0x4000>;
+			clocks = <&cxo_board>,
+				 <&pxo_board>,
+				 <&lcc PLL4>;
+			clock-names = "cxo", "pxo", "pll4";
 		};
 
 		lcc: clock-controller@28000000 {
@@ -137,6 +142,20 @@
 			reg = <0x28000000 0x1000>;
 			#clock-cells = <1>;
 			#reset-cells = <1>;
+			clocks = <&pxo_board>,
+				 <&gcc PLL4_VOTE>,
+				 <0>,
+				 <0>, <0>,
+				 <0>, <0>,
+				 <0>;
+			clock-names = "pxo",
+				      "pll4_vote",
+				      "mi2s_codec_clk",
+				      "codec_i2s_mic_codec_clk",
+				      "spare_i2s_mic_codec_clk",
+				      "codec_i2s_spkr_codec_clk",
+				      "spare_i2s_spkr_codec_clk",
+				      "pcm_codec_clk";
 		};
 
 		clock-controller@4000000 {
@@ -145,6 +164,22 @@
 			#clock-cells = <1>;
 			#power-domain-cells = <1>;
 			#reset-cells = <1>;
+			clocks = <&pxo_board>,
+				 <&gcc PLL3>,
+				 <&gcc PLL8_VOTE>,
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>;
+			clock-names = "pxo",
+				      "pll3",
+				      "pll8_vote",
+				      "dsi1pll",
+				      "dsi1pllbyte",
+				      "dsi2pll",
+				      "dsi2pllbyte",
+				      "hdmipll";
 		};
 
 		l2cc: clock-controller@2011000 {
@@ -152,7 +187,7 @@
 			reg = <0x2011000 0x1000>;
 		};
 
-		rpm@108000 {
+		rpm: rpm@108000 {
 			compatible = "qcom,rpm-msm8960";
 			reg = <0x108000 0x1000>;
 			qcom,ipc = <&l2cc 0x8 2>;
@@ -307,7 +342,7 @@
 			reg = <0x1a400000 0x100>;
 		};
 
-		gsbi@16000000 {
+		gsbi1: gsbi@16000000 {
 			compatible = "qcom,gsbi-v1.0.0";
 			cell-index = <1>;
 			reg = <0x16000000 0x100>;
@@ -317,7 +352,7 @@
 			#size-cells = <1>;
 			ranges;
 
-			spi@16080000 {
+			gsbi1_spi: spi@16080000 {
 				compatible = "qcom,spi-qup-v1.1.1";
 				#address-cells = <1>;
 				#size-cells = <0>;
diff --git a/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts b/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts
index ec5d340562b6..6daceaa87802 100644
--- a/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts
+++ b/arch/arm/boot/dts/qcom-msm8974-lge-nexus5-hammerhead.dts
@@ -175,7 +175,7 @@
 			ak8963@f {
 				compatible = "asahi-kasei,ak8963";
 				reg = <0x0f>;
-				gpios = <&tlmm 67 0>;
+				gpios = <&tlmm 67 GPIO_ACTIVE_HIGH>;
 				vid-supply = <&pm8941_lvs1>;
 				vdd-supply = <&pm8941_l17>;
 			};
diff --git a/arch/arm/boot/dts/qcom-msm8974.dtsi b/arch/arm/boot/dts/qcom-msm8974.dtsi
index 8baca2a77717..7a9be0acf3f5 100644
--- a/arch/arm/boot/dts/qcom-msm8974.dtsi
+++ b/arch/arm/boot/dts/qcom-msm8974.dtsi
@@ -439,14 +439,14 @@
 		sdhc_1: mmc@f9824900 {
 			compatible = "qcom,msm8974-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf9824900 0x11c>, <0xf9824000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC1_APPS_CLK>,
-				 <&gcc GCC_SDCC1_AHB_CLK>,
+			clocks = <&gcc GCC_SDCC1_AHB_CLK>,
+				 <&gcc GCC_SDCC1_APPS_CLK>,
 				 <&xo_board>;
-			clock-names = "core", "iface", "xo";
+			clock-names = "iface", "core", "xo";
 			bus-width = <8>;
 			non-removable;
 
@@ -456,14 +456,14 @@
 		sdhc_3: mmc@f9864900 {
 			compatible = "qcom,msm8974-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf9864900 0x11c>, <0xf9864000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 			interrupts = <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC3_APPS_CLK>,
-				 <&gcc GCC_SDCC3_AHB_CLK>,
+			clocks = <&gcc GCC_SDCC3_AHB_CLK>,
+				 <&gcc GCC_SDCC3_APPS_CLK>,
 				 <&xo_board>;
-			clock-names = "core", "iface", "xo";
+			clock-names = "iface", "core", "xo";
 			bus-width = <4>;
 
 			#address-cells = <1>;
@@ -475,14 +475,14 @@
 		sdhc_2: mmc@f98a4900 {
 			compatible = "qcom,msm8974-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf98a4900 0x11c>, <0xf98a4000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 			interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
-			clocks = <&gcc GCC_SDCC2_APPS_CLK>,
-				 <&gcc GCC_SDCC2_AHB_CLK>,
+			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&gcc GCC_SDCC2_APPS_CLK>,
 				 <&xo_board>;
-			clock-names = "core", "iface", "xo";
+			clock-names = "iface", "core", "xo";
 			bus-width = <4>;
 
 			#address-cells = <1>;
@@ -762,6 +762,11 @@
 			};
 		};
 
+		sram@fc190000 {
+			compatible = "qcom,msm8974-rpm-stats";
+			reg = <0xfc190000 0x10000>;
+		};
+
 		etf@fc307000 {
 			compatible = "arm,coresight-tmc", "arm,primecell";
 			reg = <0xfc307000 0x1000>;
diff --git a/arch/arm/boot/dts/qcom-msm8974pro.dtsi b/arch/arm/boot/dts/qcom-msm8974pro.dtsi
index 1e882e16a221..58df6e75ab6d 100644
--- a/arch/arm/boot/dts/qcom-msm8974pro.dtsi
+++ b/arch/arm/boot/dts/qcom-msm8974pro.dtsi
@@ -10,10 +10,10 @@
 };
 
 &sdhc_1 {
-	clocks = <&gcc GCC_SDCC1_APPS_CLK>,
-		 <&gcc GCC_SDCC1_AHB_CLK>,
+	clocks = <&gcc GCC_SDCC1_AHB_CLK>,
+		 <&gcc GCC_SDCC1_APPS_CLK>,
 		 <&xo_board>,
 		 <&gcc GCC_SDCC1_CDCCAL_FF_CLK>,
 		 <&gcc GCC_SDCC1_CDCCAL_SLEEP_CLK>;
-	clock-names = "core", "iface", "xo", "cal", "sleep";
+	clock-names = "iface", "core", "xo", "cal", "sleep";
 };
diff --git a/arch/arm/boot/dts/qcom-pm8941.dtsi b/arch/arm/boot/dts/qcom-pm8941.dtsi
index 59d0cde63251..9cd49deb9fa7 100644
--- a/arch/arm/boot/dts/qcom-pm8941.dtsi
+++ b/arch/arm/boot/dts/qcom-pm8941.dtsi
@@ -93,7 +93,7 @@
 			#thermal-sensor-cells = <0>;
 		};
 
-		pm8941_vadc: vadc@3100 {
+		pm8941_vadc: adc@3100 {
 			compatible = "qcom,spmi-vadc";
 			reg = <0x3100>;
 			interrupts = <0x0 0x31 0x0 IRQ_TYPE_EDGE_RISING>;
@@ -144,7 +144,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pm8941_lpg: lpg {
+		pm8941_lpg: pwm {
 			compatible = "qcom,pm8941-lpg";
 
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/qcom-pma8084.dtsi b/arch/arm/boot/dts/qcom-pma8084.dtsi
index 7b8a8d9695da..e77602e9f95c 100644
--- a/arch/arm/boot/dts/qcom-pma8084.dtsi
+++ b/arch/arm/boot/dts/qcom-pma8084.dtsi
@@ -56,7 +56,7 @@
 			io-channel-names = "thermal";
 		};
 
-		pma8084_vadc: vadc@3100 {
+		pma8084_vadc: adc@3100 {
 			compatible = "qcom,spmi-vadc";
 			reg = <0x3100>;
 			interrupts = <0x0 0x31 0x0 IRQ_TYPE_EDGE_RISING>;
diff --git a/arch/arm/boot/dts/qcom-pmx55.dtsi b/arch/arm/boot/dts/qcom-pmx55.dtsi
index 9de7578a4c5f..e1b869480bbd 100644
--- a/arch/arm/boot/dts/qcom-pmx55.dtsi
+++ b/arch/arm/boot/dts/qcom-pmx55.dtsi
@@ -16,7 +16,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		power-on@800 {
+		pon@800 {
 			compatible = "qcom,pm8916-pon";
 			reg = <0x0800>;
 
diff --git a/arch/arm/boot/dts/qcom-sdx65.dtsi b/arch/arm/boot/dts/qcom-sdx65.dtsi
index 8daefd50217a..4cd405db5500 100644
--- a/arch/arm/boot/dts/qcom-sdx65.dtsi
+++ b/arch/arm/boot/dts/qcom-sdx65.dtsi
@@ -334,7 +334,7 @@
 		sdhc_1: mmc@8804000 {
 			compatible = "qcom,sdx65-sdhci", "qcom,sdhci-msm-v5";
 			reg = <0x08804000 0x1000>;
-			reg-names = "hc_mem";
+			reg-names = "hc";
 			interrupts = <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "hc_irq", "pwr_irq";
diff --git a/arch/arm/boot/dts/r8a7742.dtsi b/arch/arm/boot/dts/r8a7742.dtsi
index 420e0b3259d4..758a1bf02fae 100644
--- a/arch/arm/boot/dts/r8a7742.dtsi
+++ b/arch/arm/boot/dts/r8a7742.dtsi
@@ -633,7 +633,7 @@
 			status = "disabled";
 		};
 
-		usbphy: usb-phy@e6590100 {
+		usbphy: usb-phy-controller@e6590100 {
 			compatible = "renesas,usb-phy-r8a7742",
 				     "renesas,rcar-gen2-usb-phy";
 			reg = <0 0xe6590100 0 0x100>;
@@ -645,11 +645,11 @@
 			resets = <&cpg 704>;
 			status = "disabled";
 
-			usb0: usb-channel@0 {
+			usb0: usb-phy@0 {
 				reg = <0>;
 				#phy-cells = <1>;
 			};
-			usb2: usb-channel@2 {
+			usb2: usb-phy@2 {
 				reg = <2>;
 				#phy-cells = <1>;
 			};
diff --git a/arch/arm/boot/dts/r8a7743.dtsi b/arch/arm/boot/dts/r8a7743.dtsi
index c0c145a5fe8d..20f1d98a048d 100644
--- a/arch/arm/boot/dts/r8a7743.dtsi
+++ b/arch/arm/boot/dts/r8a7743.dtsi
@@ -584,7 +584,7 @@
 			status = "disabled";
 		};
 
-		usbphy: usb-phy@e6590100 {
+		usbphy: usb-phy-controller@e6590100 {
 			compatible = "renesas,usb-phy-r8a7743",
 				     "renesas,rcar-gen2-usb-phy";
 			reg = <0 0xe6590100 0 0x100>;
@@ -596,11 +596,11 @@
 			resets = <&cpg 704>;
 			status = "disabled";
 
-			usb0: usb-channel@0 {
+			usb0: usb-phy@0 {
 				reg = <0>;
 				#phy-cells = <1>;
 			};
-			usb2: usb-channel@2 {
+			usb2: usb-phy@2 {
 				reg = <2>;
 				#phy-cells = <1>;
 			};
diff --git a/arch/arm/boot/dts/r8a7744.dtsi b/arch/arm/boot/dts/r8a7744.dtsi
index 3f4fb53dd6df..96b2d5a4e8f6 100644
--- a/arch/arm/boot/dts/r8a7744.dtsi
+++ b/arch/arm/boot/dts/r8a7744.dtsi
@@ -584,7 +584,7 @@
 			status = "disabled";
 		};
 
-		usbphy: usb-phy@e6590100 {
+		usbphy: usb-phy-controller@e6590100 {
 			compatible = "renesas,usb-phy-r8a7744",
 				     "renesas,rcar-gen2-usb-phy";
 			reg = <0 0xe6590100 0 0x100>;
@@ -596,11 +596,11 @@
 			resets = <&cpg 704>;
 			status = "disabled";
 
-			usb0: usb-channel@0 {
+			usb0: usb-phy@0 {
 				reg = <0>;
 				#phy-cells = <1>;
 			};
-			usb2: usb-channel@2 {
+			usb2: usb-phy@2 {
 				reg = <2>;
 				#phy-cells = <1>;
 			};
diff --git a/arch/arm/boot/dts/r8a7745.dtsi b/arch/arm/boot/dts/r8a7745.dtsi
index fe8e98a66d93..afc902e532d8 100644
--- a/arch/arm/boot/dts/r8a7745.dtsi
+++ b/arch/arm/boot/dts/r8a7745.dtsi
@@ -525,7 +525,7 @@
 			status = "disabled";
 		};
 
-		usbphy: usb-phy@e6590100 {
+		usbphy: usb-phy-controller@e6590100 {
 			compatible = "renesas,usb-phy-r8a7745",
 				     "renesas,rcar-gen2-usb-phy";
 			reg = <0 0xe6590100 0 0x100>;
@@ -537,11 +537,11 @@
 			resets = <&cpg 704>;
 			status = "disabled";
 
-			usb0: usb-channel@0 {
+			usb0: usb-phy@0 {
 				reg = <0>;
 				#phy-cells = <1>;
 			};
-			usb2: usb-channel@2 {
+			usb2: usb-phy@2 {
 				reg = <2>;
 				#phy-cells = <1>;
 			};
diff --git a/arch/arm/boot/dts/r8a77470.dtsi b/arch/arm/boot/dts/r8a77470.dtsi
index c90f2a270214..a5cf663a0118 100644
--- a/arch/arm/boot/dts/r8a77470.dtsi
+++ b/arch/arm/boot/dts/r8a77470.dtsi
@@ -357,7 +357,7 @@
 			status = "disabled";
 		};
 
-		usbphy0: usb-phy@e6590100 {
+		usbphy0: usb-phy-controller@e6590100 {
 			compatible = "renesas,usb-phy-r8a77470",
 				     "renesas,rcar-gen2-usb-phy";
 			reg = <0 0xe6590100 0 0x100>;
@@ -369,7 +369,7 @@
 			resets = <&cpg 704>;
 			status = "disabled";
 
-			usb0: usb-channel@0 {
+			usb0: usb-phy@0 {
 				reg = <0>;
 				#phy-cells = <1>;
 			};
@@ -393,7 +393,7 @@
 			status = "disabled";
 		};
 
-		usbphy1: usb-phy@e6598100 {
+		usbphy1: usb-phy-controller@e6598100 {
 			compatible = "renesas,usb-phy-r8a77470",
 				     "renesas,rcar-gen2-usb-phy";
 			reg = <0 0xe6598100 0 0x100>;
@@ -405,7 +405,7 @@
 			resets = <&cpg 706>;
 			status = "disabled";
 
-			usb1: usb-channel@0 {
+			usb1: usb-phy@0 {
 				reg = <0>;
 				#phy-cells = <1>;
 			};
diff --git a/arch/arm/boot/dts/r8a7790.dtsi b/arch/arm/boot/dts/r8a7790.dtsi
index a640488d513b..db171e3c62f2 100644
--- a/arch/arm/boot/dts/r8a7790.dtsi
+++ b/arch/arm/boot/dts/r8a7790.dtsi
@@ -654,7 +654,7 @@
 			status = "disabled";
 		};
 
-		usbphy: usb-phy@e6590100 {
+		usbphy: usb-phy-controller@e6590100 {
 			compatible = "renesas,usb-phy-r8a7790",
 				     "renesas,rcar-gen2-usb-phy";
 			reg = <0 0xe6590100 0 0x100>;
@@ -666,11 +666,11 @@
 			resets = <&cpg 704>;
 			status = "disabled";
 
-			usb0: usb-channel@0 {
+			usb0: usb-phy@0 {
 				reg = <0>;
 				#phy-cells = <1>;
 			};
-			usb2: usb-channel@2 {
+			usb2: usb-phy@2 {
 				reg = <2>;
 				#phy-cells = <1>;
 			};
diff --git a/arch/arm/boot/dts/r8a7791.dtsi b/arch/arm/boot/dts/r8a7791.dtsi
index 542ed0a71872..d8f91d9f42ae 100644
--- a/arch/arm/boot/dts/r8a7791.dtsi
+++ b/arch/arm/boot/dts/r8a7791.dtsi
@@ -608,7 +608,7 @@
 			status = "disabled";
 		};
 
-		usbphy: usb-phy@e6590100 {
+		usbphy: usb-phy-controller@e6590100 {
 			compatible = "renesas,usb-phy-r8a7791",
 				     "renesas,rcar-gen2-usb-phy";
 			reg = <0 0xe6590100 0 0x100>;
@@ -620,11 +620,11 @@
 			resets = <&cpg 704>;
 			status = "disabled";
 
-			usb0: usb-channel@0 {
+			usb0: usb-phy@0 {
 				reg = <0>;
 				#phy-cells = <1>;
 			};
-			usb2: usb-channel@2 {
+			usb2: usb-phy@2 {
 				reg = <2>;
 				#phy-cells = <1>;
 			};
diff --git a/arch/arm/boot/dts/r8a7794.dtsi b/arch/arm/boot/dts/r8a7794.dtsi
index b601ee6f7580..7aa781ff3bff 100644
--- a/arch/arm/boot/dts/r8a7794.dtsi
+++ b/arch/arm/boot/dts/r8a7794.dtsi
@@ -506,7 +506,7 @@
 			status = "disabled";
 		};
 
-		usbphy: usb-phy@e6590100 {
+		usbphy: usb-phy-controller@e6590100 {
 			compatible = "renesas,usb-phy-r8a7794",
 				     "renesas,rcar-gen2-usb-phy";
 			reg = <0 0xe6590100 0 0x100>;
@@ -518,11 +518,11 @@
 			resets = <&cpg 704>;
 			status = "disabled";
 
-			usb0: usb-channel@0 {
+			usb0: usb-phy@0 {
 				reg = <0>;
 				#phy-cells = <1>;
 			};
-			usb2: usb-channel@2 {
+			usb2: usb-phy@2 {
 				reg = <2>;
 				#phy-cells = <1>;
 			};
diff --git a/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts b/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts
index 4bf813335e21..c18bbd7141c4 100644
--- a/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts
+++ b/arch/arm/boot/dts/r9a06g032-rzn1d400-db.dts
@@ -26,6 +26,22 @@
 	};
 };
 
+&can0 {
+	pinctrl-0 = <&pins_can0>;
+	pinctrl-names = "default";
+
+	/* Assuming CN10/CN11 are wired for CAN1 */
+	status = "okay";
+};
+
+&can1 {
+	pinctrl-0 = <&pins_can1>;
+	pinctrl-names = "default";
+
+	/* Please only enable can0 or can1, depending on CN10/CN11 */
+	/* status = "okay"; */
+};
+
 &eth_miic {
 	status = "okay";
 	renesas,miic-switch-portin = <MIIC_GMAC2_PORT>;
@@ -52,6 +68,18 @@
 };
 
 &pinctrl{
+	pins_can0: pins_can0 {
+		pinmux = <RZN1_PINMUX(162, RZN1_FUNC_CAN)>,	/* CAN0_TXD */
+			 <RZN1_PINMUX(163, RZN1_FUNC_CAN)>;	/* CAN0_RXD */
+		drive-strength = <6>;
+	};
+
+	pins_can1: pins_can1 {
+		pinmux = <RZN1_PINMUX(109, RZN1_FUNC_CAN)>,	/* CAN1_TXD */
+			 <RZN1_PINMUX(110, RZN1_FUNC_CAN)>;	/* CAN1_RXD */
+		drive-strength = <6>;
+	};
+
 	pins_eth3: pins_eth3 {
 		pinmux = <RZN1_PINMUX(36, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII)>,
 			 <RZN1_PINMUX(37, RZN1_FUNC_CLK_ETH_MII_RGMII_RMII)>,
diff --git a/arch/arm/boot/dts/r9a06g032.dtsi b/arch/arm/boot/dts/r9a06g032.dtsi
index 5b97fa85474f..563024c9a4ae 100644
--- a/arch/arm/boot/dts/r9a06g032.dtsi
+++ b/arch/arm/boot/dts/r9a06g032.dtsi
@@ -423,6 +423,26 @@
 			interrupts =
 				<GIC_PPI 9 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>;
 		};
+
+		can0: can@52104000 {
+			compatible = "renesas,r9a06g032-sja1000","renesas,rzn1-sja1000";
+			reg = <0x52104000 0x800>;
+			reg-io-width = <4>;
+			interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sysctrl R9A06G032_HCLK_CAN0>;
+			power-domains = <&sysctrl>;
+			status = "disabled";
+		};
+
+		can1: can@52105000 {
+			compatible = "renesas,r9a06g032-sja1000", "renesas,rzn1-sja1000";
+			reg = <0x52105000 0x800>;
+			reg-io-width = <4>;
+			interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sysctrl R9A06G032_HCLK_CAN1>;
+			power-domains = <&sysctrl>;
+			status = "disabled";
+		};
 	};
 
 	timer {
diff --git a/arch/arm/boot/dts/rk3036-evb.dts b/arch/arm/boot/dts/rk3036-evb.dts
index 2a7e6624efb9..9fd4d9db9f8f 100644
--- a/arch/arm/boot/dts/rk3036-evb.dts
+++ b/arch/arm/boot/dts/rk3036-evb.dts
@@ -15,16 +15,20 @@
 };
 
 &emac {
-	pinctrl-names = "default";
-	pinctrl-0 = <&emac_xfer>, <&emac_mdio>;
 	phy = <&phy0>;
-	phy-reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; /* PHY_RST */
 	phy-reset-duration = <10>; /* millisecond */
-
+	phy-reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; /* PHY_RST */
+	pinctrl-names = "default";
+	pinctrl-0 = <&emac_xfer>, <&emac_mdio>;
 	status = "okay";
 
-	phy0: ethernet-phy@0 {
-		reg = <0>;
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		phy0: ethernet-phy@0 {
+			reg = <0>;
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/rk3036-kylin.dts b/arch/arm/boot/dts/rk3036-kylin.dts
index e817eba8c622..67e1e04139e7 100644
--- a/arch/arm/boot/dts/rk3036-kylin.dts
+++ b/arch/arm/boot/dts/rk3036-kylin.dts
@@ -80,16 +80,20 @@
 };
 
 &emac {
-	pinctrl-names = "default";
-	pinctrl-0 = <&emac_xfer>, <&emac_mdio>;
 	phy = <&phy0>;
-	phy-reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; /* PHY_RST */
 	phy-reset-duration = <10>; /* millisecond */
-
+	phy-reset-gpios = <&gpio2 RK_PC6 GPIO_ACTIVE_LOW>; /* PHY_RST */
+	pinctrl-names = "default";
+	pinctrl-0 = <&emac_xfer>, <&emac_mdio>;
 	status = "okay";
 
-	phy0: ethernet-phy@0 {
-		reg = <0>;
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		phy0: ethernet-phy@0 {
+			reg = <0>;
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/rk3036.dtsi b/arch/arm/boot/dts/rk3036.dtsi
index 9b0f0497567d..78686fc72ce6 100644
--- a/arch/arm/boot/dts/rk3036.dtsi
+++ b/arch/arm/boot/dts/rk3036.dtsi
@@ -225,11 +225,9 @@
 	};
 
 	emac: ethernet@10200000 {
-		compatible = "rockchip,rk3036-emac", "snps,arc-emac";
+		compatible = "rockchip,rk3036-emac";
 		reg = <0x10200000 0x4000>;
 		interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 		rockchip,grf = <&grf>;
 		clocks = <&cru HCLK_MAC>, <&cru SCLK_MACREF>, <&cru SCLK_MAC>;
 		clock-names = "hclk", "macref", "macclk";
diff --git a/arch/arm/boot/dts/rk3066a-marsboard.dts b/arch/arm/boot/dts/rk3066a-marsboard.dts
index a66d915aa0f6..8beecd628282 100644
--- a/arch/arm/boot/dts/rk3066a-marsboard.dts
+++ b/arch/arm/boot/dts/rk3066a-marsboard.dts
@@ -150,18 +150,21 @@
 #include "tps65910.dtsi"
 
 &emac {
-	status = "okay";
-
 	phy = <&phy0>;
 	phy-supply = <&vcc_rmii>;
-
 	pinctrl-names = "default";
 	pinctrl-0 = <&emac_xfer>, <&emac_mdio>, <&phy_int>;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
 
-	phy0: ethernet-phy@0 {
-		reg = <0>;
-		interrupt-parent = <&gpio1>;
-		interrupts = <RK_PD2 IRQ_TYPE_LEVEL_LOW>;
+		phy0: ethernet-phy@0 {
+			reg = <0>;
+			interrupt-parent = <&gpio1>;
+			interrupts = <RK_PD2 IRQ_TYPE_LEVEL_LOW>;
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/rk3066a-rayeager.dts b/arch/arm/boot/dts/rk3066a-rayeager.dts
index dbbc5170094e..3eee42137b6d 100644
--- a/arch/arm/boot/dts/rk3066a-rayeager.dts
+++ b/arch/arm/boot/dts/rk3066a-rayeager.dts
@@ -142,15 +142,20 @@
 };
 
 &emac {
-	pinctrl-names = "default";
-	pinctrl-0 = <&emac_xfer>, <&emac_mdio>, <&rmii_rst>;
 	phy = <&phy0>;
 	phy-supply = <&vcc_rmii>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&emac_xfer>, <&emac_mdio>, <&rmii_rst>;
 	status = "okay";
 
-	phy0: ethernet-phy@0 {
-		reg = <0>;
-		reset-gpios = <&gpio1 RK_PD6 GPIO_ACTIVE_LOW>;
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		phy0: ethernet-phy@0 {
+			reg = <0>;
+			reset-gpios = <&gpio1 RK_PD6 GPIO_ACTIVE_LOW>;
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/rk3188-radxarock.dts b/arch/arm/boot/dts/rk3188-radxarock.dts
index a9ed3cd2c2da..e7cf18823558 100644
--- a/arch/arm/boot/dts/rk3188-radxarock.dts
+++ b/arch/arm/boot/dts/rk3188-radxarock.dts
@@ -126,18 +126,21 @@
 };
 
 &emac {
-	status = "okay";
-
+	phy = <&phy0>;
+	phy-supply = <&vcc_rmii>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&emac_xfer>, <&emac_mdio>, <&phy_int>;
+	status = "okay";
 
-	phy = <&phy0>;
-	phy-supply = <&vcc_rmii>;
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
 
-	phy0: ethernet-phy@0 {
-		reg = <0>;
-		interrupt-parent = <&gpio3>;
-		interrupts = <RK_PD2 IRQ_TYPE_LEVEL_LOW>;
+		phy0: ethernet-phy@0 {
+			reg = <0>;
+			interrupt-parent = <&gpio3>;
+			interrupts = <RK_PD2 IRQ_TYPE_LEVEL_LOW>;
+		};
 	};
 };
 
diff --git a/arch/arm/boot/dts/rk3xxx.dtsi b/arch/arm/boot/dts/rk3xxx.dtsi
index 616a828e0c6e..bf285091a9eb 100644
--- a/arch/arm/boot/dts/rk3xxx.dtsi
+++ b/arch/arm/boot/dts/rk3xxx.dtsi
@@ -186,8 +186,6 @@
 		compatible = "snps,arc-emac";
 		reg = <0x10204000 0x3c>;
 		interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>;
-		#address-cells = <1>;
-		#size-cells = <0>;
 
 		rockchip,grf = <&grf>;
 
diff --git a/arch/arm/boot/dts/s5pv210-aries.dtsi b/arch/arm/boot/dts/s5pv210-aries.dtsi
index 5541df4df628..964c5fe51755 100644
--- a/arch/arm/boot/dts/s5pv210-aries.dtsi
+++ b/arch/arm/boot/dts/s5pv210-aries.dtsi
@@ -738,7 +738,7 @@
 		samsung,pin-pud = <S5PV210_PIN_PULL_NONE>;
 	};
 
-	magnetometer_i2c_pins: yas529-i2c-pins-pins {
+	magnetometer_i2c_pins: yas529-i2c-pins {
 		samsung,pins = "gpj0-0", "gpj0-1";
 		samsung,pin-pud = <S5PV210_PIN_PULL_NONE>;
 		samsung,pin-drv = <S5PV210_PIN_DRV_LV1>;
@@ -788,7 +788,7 @@
 		samsung,pin-drv = <S5PV210_PIN_DRV_LV1>;
 	};
 
-	pmic_i2c_pins: pmic-i2c-pins-pins {
+	pmic_i2c_pins: pmic-i2c-pins {
 		samsung,pins = "gpj4-0", "gpj4-3";
 		samsung,pin-pud = <S5PV210_PIN_PULL_NONE>;
 		samsung,pin-drv = <S5PV210_PIN_DRV_LV1>;
diff --git a/arch/arm/boot/dts/s5pv210-galaxys.dts b/arch/arm/boot/dts/s5pv210-galaxys.dts
index cdd3653d487f..532d3f5bceb1 100644
--- a/arch/arm/boot/dts/s5pv210-galaxys.dts
+++ b/arch/arm/boot/dts/s5pv210-galaxys.dts
@@ -150,7 +150,7 @@
 	pinctrl-names = "default";
 	pinctrl-0 = <&sleep_cfg>;
 
-	fm_i2c_pins: fm-i2c-pins-pins {
+	fm_i2c_pins: fm-i2c-pins {
 		samsung,pins = "gpd1-2", "gpd1-3";
 		samsung,pin-pud = <S5PV210_PIN_PULL_NONE>;
 		samsung,pin-drv = <S5PV210_PIN_DRV_LV1>;
diff --git a/arch/arm/boot/dts/sam9x60.dtsi b/arch/arm/boot/dts/sam9x60.dtsi
index d3f60f6a456d..8f5477e307dd 100644
--- a/arch/arm/boot/dts/sam9x60.dtsi
+++ b/arch/arm/boot/dts/sam9x60.dtsi
@@ -12,6 +12,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 #include <dt-bindings/mfd/atmel-flexcom.h>
 
 / {
@@ -583,6 +584,7 @@
 			dbgu: serial@fffff200 {
 				compatible = "microchip,sam9x60-dbgu", "microchip,sam9x60-usart", "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
 				reg = <0xfffff200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <47 IRQ_TYPE_LEVEL_HIGH 7>;
 				dmas = <&dma0
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
diff --git a/arch/arm/boot/dts/sama5d2.dtsi b/arch/arm/boot/dts/sama5d2.dtsi
index 2c50a021aa76..14c35c12a115 100644
--- a/arch/arm/boot/dts/sama5d2.dtsi
+++ b/arch/arm/boot/dts/sama5d2.dtsi
@@ -9,6 +9,7 @@
 #include <dt-bindings/dma/at91.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 #include <dt-bindings/iio/adc/at91-sama5d2_adc.h>
 
 / {
@@ -441,6 +442,7 @@
 			uart0: serial@f801c000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf801c000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <24 IRQ_TYPE_LEVEL_HIGH 7>;
 				dmas = <&dma0
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
@@ -457,6 +459,7 @@
 			uart1: serial@f8020000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8020000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <25 IRQ_TYPE_LEVEL_HIGH 7>;
 				dmas = <&dma0
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
@@ -473,6 +476,7 @@
 			uart2: serial@f8024000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8024000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <26 IRQ_TYPE_LEVEL_HIGH 7>;
 				dmas = <&dma0
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
@@ -530,6 +534,7 @@
 				uart5: serial@200 {
 					compatible = "atmel,at91sam9260-usart";
 					reg = <0x200 0x200>;
+					atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 					interrupts = <19 IRQ_TYPE_LEVEL_HIGH 7>;
 					clocks = <&pmc PMC_TYPE_PERIPHERAL 19>;
 					clock-names = "usart";
@@ -600,6 +605,7 @@
 				uart6: serial@200 {
 					compatible = "atmel,at91sam9260-usart";
 					reg = <0x200 0x200>;
+					atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 					interrupts = <20 IRQ_TYPE_LEVEL_HIGH 7>;
 					clocks = <&pmc PMC_TYPE_PERIPHERAL 20>;
 					clock-names = "usart";
@@ -769,6 +775,7 @@
 			uart3: serial@fc008000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfc008000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <27 IRQ_TYPE_LEVEL_HIGH 7>;
 				dmas = <&dma1
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
@@ -785,6 +792,7 @@
 			uart4: serial@fc00c000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfc00c000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				dmas = <&dma0
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1) |
 					 AT91_XDMAC_DT_PERID(43))>,
@@ -810,6 +818,7 @@
 				uart7: serial@200 {
 					compatible = "atmel,at91sam9260-usart";
 					reg = <0x200 0x200>;
+					atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 					interrupts = <21 IRQ_TYPE_LEVEL_HIGH 7>;
 					clocks = <&pmc PMC_TYPE_PERIPHERAL 21>;
 					clock-names = "usart";
@@ -880,6 +889,7 @@
 				uart8: serial@200 {
 					compatible = "atmel,at91sam9260-usart";
 					reg = <0x200 0x200>;
+					atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 					interrupts = <22 IRQ_TYPE_LEVEL_HIGH 7>;
 					clocks = <&pmc PMC_TYPE_PERIPHERAL 22>;
 					clock-names = "usart";
@@ -951,6 +961,7 @@
 				uart9: serial@200 {
 					compatible = "atmel,at91sam9260-usart";
 					reg = <0x200 0x200>;
+					atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 					interrupts = <23 IRQ_TYPE_LEVEL_HIGH 7>;
 					clocks = <&pmc PMC_TYPE_PERIPHERAL 23>;
 					clock-names = "usart";
diff --git a/arch/arm/boot/dts/sama5d3.dtsi b/arch/arm/boot/dts/sama5d3.dtsi
index 2d0935ad2225..bde8e92d60bb 100644
--- a/arch/arm/boot/dts/sama5d3.dtsi
+++ b/arch/arm/boot/dts/sama5d3.dtsi
@@ -12,6 +12,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	#address-cells = <1>;
@@ -194,6 +195,7 @@
 			usart0: serial@f001c000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf001c000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <12 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma0 2 AT91_DMA_CFG_PER_ID(3)>,
 				       <&dma0 2 (AT91_DMA_CFG_PER_ID(4) | AT91_DMA_CFG_FIFOCFG_ASAP)>;
@@ -208,6 +210,7 @@
 			usart1: serial@f0020000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf0020000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <13 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma0 2 AT91_DMA_CFG_PER_ID(5)>,
 				       <&dma0 2 (AT91_DMA_CFG_PER_ID(6) | AT91_DMA_CFG_FIFOCFG_ASAP)>;
@@ -222,6 +225,7 @@
 			uart0: serial@f0024000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf0024000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <16 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_uart0>;
@@ -356,6 +360,7 @@
 			usart2: serial@f8020000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8020000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <14 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma1 2 AT91_DMA_CFG_PER_ID(7)>,
 				       <&dma1 2 (AT91_DMA_CFG_PER_ID(8) | AT91_DMA_CFG_FIFOCFG_ASAP)>;
@@ -370,6 +375,7 @@
 			usart3: serial@f8024000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8024000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <15 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma1 2 AT91_DMA_CFG_PER_ID(9)>,
 				       <&dma1 2 (AT91_DMA_CFG_PER_ID(10) | AT91_DMA_CFG_FIFOCFG_ASAP)>;
@@ -464,6 +470,7 @@
 			dbgu: serial@ffffee00 {
 				compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
 				reg = <0xffffee00 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <2 IRQ_TYPE_LEVEL_HIGH 7>;
 				dmas = <&dma1 2 AT91_DMA_CFG_PER_ID(13)>,
 				       <&dma1 2 (AT91_DMA_CFG_PER_ID(14) | AT91_DMA_CFG_FIFOCFG_ASAP)>;
diff --git a/arch/arm/boot/dts/sama5d3_uart.dtsi b/arch/arm/boot/dts/sama5d3_uart.dtsi
index a3eaba995cf4..44d1173f2ffb 100644
--- a/arch/arm/boot/dts/sama5d3_uart.dtsi
+++ b/arch/arm/boot/dts/sama5d3_uart.dtsi
@@ -9,6 +9,7 @@
 #include <dt-bindings/pinctrl/at91.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/clock/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	aliases {
@@ -39,6 +40,7 @@
 			uart0: serial@f0024000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf0024000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <16 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_uart0>;
@@ -50,6 +52,7 @@
 			uart1: serial@f8028000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8028000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <17 IRQ_TYPE_LEVEL_HIGH 5>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_uart1>;
diff --git a/arch/arm/boot/dts/sama5d4.dtsi b/arch/arm/boot/dts/sama5d4.dtsi
index 1e5c01898ccf..af62157ae214 100644
--- a/arch/arm/boot/dts/sama5d4.dtsi
+++ b/arch/arm/boot/dts/sama5d4.dtsi
@@ -8,6 +8,7 @@
 
 #include <dt-bindings/clock/at91.h>
 #include <dt-bindings/dma/at91.h>
+#include <dt-bindings/mfd/at91-usart.h>
 #include <dt-bindings/pinctrl/at91.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/gpio/gpio.h>
@@ -278,6 +279,7 @@
 			uart0: serial@f8004000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8004000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <27 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma0
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1)
@@ -437,6 +439,7 @@
 			usart0: serial@f802c000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf802c000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <6 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma0
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1)
@@ -455,6 +458,7 @@
 			usart1: serial@f8030000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xf8030000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <7 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma0
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1)
@@ -490,6 +494,7 @@
 			uart1: serial@fc004000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfc004000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <28 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma0
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1)
@@ -508,6 +513,7 @@
 			usart2: serial@fc008000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfc008000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <29 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma1
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1)
@@ -526,6 +532,7 @@
 			usart3: serial@fc00c000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfc00c000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <30 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma1
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1)
@@ -544,6 +551,7 @@
 			usart4: serial@fc010000 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0xfc010000 0x100>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <31 IRQ_TYPE_LEVEL_HIGH 5>;
 				dmas = <&dma1
 					(AT91_XDMAC_DT_MEM_IF(0) | AT91_XDMAC_DT_PER_IF(1)
@@ -770,6 +778,7 @@
 			dbgu: serial@fc069000 {
 				compatible = "atmel,at91sam9260-dbgu", "atmel,at91sam9260-usart";
 				reg = <0xfc069000 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <45 IRQ_TYPE_LEVEL_HIGH 7>;
 				pinctrl-names = "default";
 				pinctrl-0 = <&pinctrl_dbgu>;
diff --git a/arch/arm/boot/dts/sama7g5.dtsi b/arch/arm/boot/dts/sama7g5.dtsi
index bb6d71e6dfeb..7bd8ae8e8d38 100644
--- a/arch/arm/boot/dts/sama7g5.dtsi
+++ b/arch/arm/boot/dts/sama7g5.dtsi
@@ -14,6 +14,7 @@
 #include <dt-bindings/clock/at91.h>
 #include <dt-bindings/dma/at91.h>
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/mfd/at91-usart.h>
 
 / {
 	model = "Microchip SAMA7G5 family SoC";
@@ -603,6 +604,7 @@
 			uart0: serial@200 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0x200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&pmc PMC_TYPE_PERIPHERAL 38>;
 				clock-names = "usart";
@@ -651,6 +653,7 @@
 			uart3: serial@200 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0x200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&pmc PMC_TYPE_PERIPHERAL 41>;
 				clock-names = "usart";
@@ -694,6 +697,7 @@
 			uart4: serial@200 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0x200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&pmc PMC_TYPE_PERIPHERAL 42>;
 				clock-names = "usart";
@@ -719,6 +723,7 @@
 			uart7: serial@200 {
 				compatible = "atmel,at91sam9260-usart";
 				reg = <0x200 0x200>;
+				atmel,usart-mode = <AT91_USART_MODE_SERIAL>;
 				interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&pmc PMC_TYPE_PERIPHERAL 45>;
 				clock-names = "usart";
@@ -866,9 +871,9 @@
 				#address-cells = <1>;
 				#size-cells = <0>;
 				atmel,fifo-size = <32>;
-				dmas = <&dma0 AT91_XDMAC_DT_PERID(27)>,
-					    <&dma0 AT91_XDMAC_DT_PERID(28)>;
-				dma-names = "rx", "tx";
+				dmas = <&dma0 AT91_XDMAC_DT_PERID(28)>,
+					    <&dma0 AT91_XDMAC_DT_PERID(27)>;
+				dma-names = "tx", "rx";
 				status = "disabled";
 			};
 		};
diff --git a/arch/arm/boot/dts/stm32mp13-pinctrl.dtsi b/arch/arm/boot/dts/stm32mp13-pinctrl.dtsi
index d2472cd8f1d0..efdd163eba30 100644
--- a/arch/arm/boot/dts/stm32mp13-pinctrl.dtsi
+++ b/arch/arm/boot/dts/stm32mp13-pinctrl.dtsi
@@ -6,6 +6,40 @@
 #include <dt-bindings/pinctrl/stm32-pinfunc.h>
 
 &pinctrl {
+	i2c1_pins_a: i2c1-0 {
+		pins {
+			pinmux = <STM32_PINMUX('D', 12, AF5)>, /* I2C1_SCL */
+				 <STM32_PINMUX('E', 8, AF5)>; /* I2C1_SDA */
+			bias-disable;
+			drive-open-drain;
+			slew-rate = <0>;
+		};
+	};
+
+	i2c1_sleep_pins_a: i2c1-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('D', 12, ANALOG)>, /* I2C1_SCL */
+				 <STM32_PINMUX('E', 8, ANALOG)>; /* I2C1_SDA */
+		};
+	};
+
+	i2c5_pins_a: i2c5-0 {
+		pins {
+			pinmux = <STM32_PINMUX('D', 1, AF4)>, /* I2C5_SCL */
+				 <STM32_PINMUX('H', 6, AF4)>; /* I2C5_SDA */
+			bias-disable;
+			drive-open-drain;
+			slew-rate = <0>;
+		};
+	};
+
+	i2c5_sleep_pins_a: i2c5-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('D', 1, ANALOG)>, /* I2C5_SCL */
+				 <STM32_PINMUX('H', 6, ANALOG)>; /* I2C5_SDA */
+		};
+	};
+
 	sdmmc1_b4_pins_a: sdmmc1-b4-0 {
 		pins {
 			pinmux = <STM32_PINMUX('C', 8, AF12)>, /* SDMMC1_D0 */
@@ -108,6 +142,29 @@
 		};
 	};
 
+	spi5_pins_a: spi5-0 {
+		pins1 {
+			pinmux = <STM32_PINMUX('H', 7, AF6)>, /* SPI5_SCK */
+				 <STM32_PINMUX('H', 3, AF5)>; /* SPI5_MOSI */
+			bias-disable;
+			drive-push-pull;
+			slew-rate = <1>;
+		};
+
+		pins2 {
+			pinmux = <STM32_PINMUX('A', 8, AF5)>; /* SPI5_MISO */
+			bias-disable;
+		};
+	};
+
+	spi5_sleep_pins_a: spi5-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('H', 7, ANALOG)>, /* SPI5_SCK */
+				 <STM32_PINMUX('A', 8, ANALOG)>, /* SPI5_MISO */
+				 <STM32_PINMUX('H', 3, ANALOG)>; /* SPI5_MOSI */
+		};
+	};
+
 	uart4_pins_a: uart4-0 {
 		pins1 {
 			pinmux = <STM32_PINMUX('D', 6, AF8)>; /* UART4_TX */
diff --git a/arch/arm/boot/dts/stm32mp131.dtsi b/arch/arm/boot/dts/stm32mp131.dtsi
index 3a921db23e9f..dd35a607073d 100644
--- a/arch/arm/boot/dts/stm32mp131.dtsi
+++ b/arch/arm/boot/dts/stm32mp131.dtsi
@@ -97,6 +97,34 @@
 			};
 		};
 
+		spi2: spi@4000b000 {
+			compatible = "st,stm32h7-spi";
+			reg = <0x4000b000 0x400>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc SPI2_K>;
+			resets = <&rcc SPI2_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&dmamux1 39 0x400 0x01>,
+			       <&dmamux1 40 0x400 0x01>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		spi3: spi@4000c000 {
+			compatible = "st,stm32h7-spi";
+			reg = <0x4000c000 0x400>;
+			interrupts = <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc SPI3_K>;
+			resets = <&rcc SPI3_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&dmamux1 61 0x400 0x01>,
+			       <&dmamux1 62 0x400 0x01>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
 		uart4: serial@40010000 {
 			compatible = "st,stm32h7-uart";
 			reg = <0x40010000 0x400>;
@@ -106,6 +134,56 @@
 			status = "disabled";
 		};
 
+		i2c1: i2c@40012000 {
+			compatible = "st,stm32mp13-i2c";
+			reg = <0x40012000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C1_K>;
+			resets = <&rcc I2C1_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&dmamux1 33 0x400 0x1>,
+			       <&dmamux1 34 0x400 0x1>;
+			dma-names = "rx", "tx";
+			st,syscfg-fmp = <&syscfg 0x4 0x1>;
+			i2c-analog-filter;
+			status = "disabled";
+		};
+
+		i2c2: i2c@40013000 {
+			compatible = "st,stm32mp13-i2c";
+			reg = <0x40013000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C2_K>;
+			resets = <&rcc I2C2_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&dmamux1 35 0x400 0x1>,
+			       <&dmamux1 36 0x400 0x1>;
+			dma-names = "rx", "tx";
+			st,syscfg-fmp = <&syscfg 0x4 0x2>;
+			i2c-analog-filter;
+			status = "disabled";
+		};
+
+		spi1: spi@44004000 {
+			compatible = "st,stm32h7-spi";
+			reg = <0x44004000 0x400>;
+			interrupts = <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc SPI1_K>;
+			resets = <&rcc SPI1_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&dmamux1 37 0x400 0x01>,
+			       <&dmamux1 38 0x400 0x01>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
 		dma1: dma-controller@48000000 {
 			compatible = "st,stm32-dma";
 			reg = <0x48000000 0x400>;
@@ -153,6 +231,88 @@
 			dma-channels = <16>;
 		};
 
+		spi4: spi@4c002000 {
+			compatible = "st,stm32h7-spi";
+			reg = <0x4c002000 0x400>;
+			interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc SPI4_K>;
+			resets = <&rcc SPI4_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&dmamux1 83 0x400 0x01>,
+			       <&dmamux1 84 0x400 0x01>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		spi5: spi@4c003000 {
+			compatible = "st,stm32h7-spi";
+			reg = <0x4c003000 0x400>;
+			interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc SPI5_K>;
+			resets = <&rcc SPI5_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&dmamux1 85 0x400 0x01>,
+			       <&dmamux1 86 0x400 0x01>;
+			dma-names = "rx", "tx";
+			status = "disabled";
+		};
+
+		i2c3: i2c@4c004000 {
+			compatible = "st,stm32mp13-i2c";
+			reg = <0x4c004000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C3_K>;
+			resets = <&rcc I2C3_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&dmamux1 73 0x400 0x1>,
+			       <&dmamux1 74 0x400 0x1>;
+			dma-names = "rx", "tx";
+			st,syscfg-fmp = <&syscfg 0x4 0x4>;
+			i2c-analog-filter;
+			status = "disabled";
+		};
+
+		i2c4: i2c@4c005000 {
+			compatible = "st,stm32mp13-i2c";
+			reg = <0x4c005000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C4_K>;
+			resets = <&rcc I2C4_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&dmamux1 75 0x400 0x1>,
+			       <&dmamux1 76 0x400 0x1>;
+			dma-names = "rx", "tx";
+			st,syscfg-fmp = <&syscfg 0x4 0x8>;
+			i2c-analog-filter;
+			status = "disabled";
+		};
+
+		i2c5: i2c@4c006000 {
+			compatible = "st,stm32mp13-i2c";
+			reg = <0x4c006000 0x400>;
+			interrupt-names = "event", "error";
+			interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&rcc I2C5_K>;
+			resets = <&rcc I2C5_R>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			dmas = <&dmamux1 115 0x400 0x1>,
+			       <&dmamux1 116 0x400 0x1>;
+			dma-names = "rx", "tx";
+			st,syscfg-fmp = <&syscfg 0x4 0x10>;
+			i2c-analog-filter;
+			status = "disabled";
+		};
+
 		rcc: rcc@50000000 {
 			compatible = "st,stm32mp13-rcc", "syscon";
 			reg = <0x50000000 0x1000>;
diff --git a/arch/arm/boot/dts/stm32mp135f-dk.dts b/arch/arm/boot/dts/stm32mp135f-dk.dts
index e6b8ffd332c7..de341d17e87d 100644
--- a/arch/arm/boot/dts/stm32mp135f-dk.dts
+++ b/arch/arm/boot/dts/stm32mp135f-dk.dts
@@ -68,6 +68,32 @@
 	};
 };
 
+&i2c1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c1_pins_a>;
+	pinctrl-1 = <&i2c1_sleep_pins_a>;
+	i2c-scl-rising-time-ns = <96>;
+	i2c-scl-falling-time-ns = <3>;
+	clock-frequency = <1000000>;
+	status = "okay";
+	/* spare dmas for other usage */
+	/delete-property/dmas;
+	/delete-property/dma-names;
+};
+
+&i2c5 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&i2c5_pins_a>;
+	pinctrl-1 = <&i2c5_sleep_pins_a>;
+	i2c-scl-rising-time-ns = <170>;
+	i2c-scl-falling-time-ns = <5>;
+	clock-frequency = <400000>;
+	status = "okay";
+	/* spare dmas for other usage */
+	/delete-property/dmas;
+	/delete-property/dma-names;
+};
+
 &iwdg2 {
 	timeout-sec = <32>;
 	status = "okay";
@@ -90,6 +116,13 @@
 	status = "okay";
 };
 
+&spi5 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&spi5_pins_a>;
+	pinctrl-1 = <&spi5_sleep_pins_a>;
+	status = "disabled";
+};
+
 &uart4 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart4_pins_a>;
diff --git a/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi b/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi
index 2cc9341d43d2..a9d2bec99014 100644
--- a/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi
+++ b/arch/arm/boot/dts/stm32mp15-pinctrl.dtsi
@@ -1261,7 +1261,7 @@
 	};
 
 	qspi_bk1_pins_a: qspi-bk1-0 {
-		pins1 {
+		pins {
 			pinmux = <STM32_PINMUX('F', 8, AF10)>, /* QSPI_BK1_IO0 */
 				 <STM32_PINMUX('F', 9, AF10)>, /* QSPI_BK1_IO1 */
 				 <STM32_PINMUX('F', 7, AF9)>, /* QSPI_BK1_IO2 */
@@ -1270,12 +1270,6 @@
 			drive-push-pull;
 			slew-rate = <1>;
 		};
-		pins2 {
-			pinmux = <STM32_PINMUX('B', 6, AF10)>; /* QSPI_BK1_NCS */
-			bias-pull-up;
-			drive-push-pull;
-			slew-rate = <1>;
-		};
 	};
 
 	qspi_bk1_sleep_pins_a: qspi-bk1-sleep-0 {
@@ -1283,13 +1277,12 @@
 			pinmux = <STM32_PINMUX('F', 8, ANALOG)>, /* QSPI_BK1_IO0 */
 				 <STM32_PINMUX('F', 9, ANALOG)>, /* QSPI_BK1_IO1 */
 				 <STM32_PINMUX('F', 7, ANALOG)>, /* QSPI_BK1_IO2 */
-				 <STM32_PINMUX('F', 6, ANALOG)>, /* QSPI_BK1_IO3 */
-				 <STM32_PINMUX('B', 6, ANALOG)>; /* QSPI_BK1_NCS */
+				 <STM32_PINMUX('F', 6, ANALOG)>; /* QSPI_BK1_IO3 */
 		};
 	};
 
 	qspi_bk2_pins_a: qspi-bk2-0 {
-		pins1 {
+		pins {
 			pinmux = <STM32_PINMUX('H', 2, AF9)>, /* QSPI_BK2_IO0 */
 				 <STM32_PINMUX('H', 3, AF9)>, /* QSPI_BK2_IO1 */
 				 <STM32_PINMUX('G', 10, AF11)>, /* QSPI_BK2_IO2 */
@@ -1298,7 +1291,34 @@
 			drive-push-pull;
 			slew-rate = <1>;
 		};
-		pins2 {
+	};
+
+	qspi_bk2_sleep_pins_a: qspi-bk2-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('H', 2, ANALOG)>, /* QSPI_BK2_IO0 */
+				 <STM32_PINMUX('H', 3, ANALOG)>, /* QSPI_BK2_IO1 */
+				 <STM32_PINMUX('G', 10, ANALOG)>, /* QSPI_BK2_IO2 */
+				 <STM32_PINMUX('G', 7, ANALOG)>; /* QSPI_BK2_IO3 */
+		};
+	};
+
+	qspi_cs1_pins_a: qspi-cs1-0 {
+		pins {
+			pinmux = <STM32_PINMUX('B', 6, AF10)>; /* QSPI_BK1_NCS */
+			bias-pull-up;
+			drive-push-pull;
+			slew-rate = <1>;
+		};
+	};
+
+	qspi_cs1_sleep_pins_a: qspi-cs1-sleep-0 {
+		pins {
+			pinmux = <STM32_PINMUX('B', 6, ANALOG)>; /* QSPI_BK1_NCS */
+		};
+	};
+
+	qspi_cs2_pins_a: qspi-cs2-0 {
+		pins {
 			pinmux = <STM32_PINMUX('C', 0, AF10)>; /* QSPI_BK2_NCS */
 			bias-pull-up;
 			drive-push-pull;
@@ -1306,13 +1326,9 @@
 		};
 	};
 
-	qspi_bk2_sleep_pins_a: qspi-bk2-sleep-0 {
+	qspi_cs2_sleep_pins_a: qspi-cs2-sleep-0 {
 		pins {
-			pinmux = <STM32_PINMUX('H', 2, ANALOG)>, /* QSPI_BK2_IO0 */
-				 <STM32_PINMUX('H', 3, ANALOG)>, /* QSPI_BK2_IO1 */
-				 <STM32_PINMUX('G', 10, ANALOG)>, /* QSPI_BK2_IO2 */
-				 <STM32_PINMUX('G', 7, ANALOG)>, /* QSPI_BK2_IO3 */
-				 <STM32_PINMUX('C', 0, ANALOG)>; /* QSPI_BK2_NCS */
+			pinmux = <STM32_PINMUX('C', 0, ANALOG)>; /* QSPI_BK2_NCS */
 		};
 	};
 
diff --git a/arch/arm/boot/dts/stm32mp151.dtsi b/arch/arm/boot/dts/stm32mp151.dtsi
index 742fdeeff4b6..e02b3f5d44cb 100644
--- a/arch/arm/boot/dts/stm32mp151.dtsi
+++ b/arch/arm/boot/dts/stm32mp151.dtsi
@@ -127,6 +127,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x40000000 0x400>;
+			interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM2_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 18 0x400 0x1>,
@@ -160,6 +162,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x40001000 0x400>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM3_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 23 0x400 0x1>,
@@ -194,6 +198,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x40002000 0x400>;
+			interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM4_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 29 0x400 0x1>,
@@ -226,6 +232,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x40003000 0x400>;
+			interrupts = <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM5_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 55 0x400 0x1>,
@@ -260,6 +268,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x40004000 0x400>;
+			interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM6_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 69 0x400 0x1>;
@@ -278,6 +288,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x40005000 0x400>;
+			interrupts = <GIC_SPI 55 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM7_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 70 0x400 0x1>;
@@ -296,6 +308,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x40006000 0x400>;
+			interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM12_K>;
 			clock-names = "int";
 			status = "disabled";
@@ -318,6 +332,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x40007000 0x400>;
+			interrupts = <GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM13_K>;
 			clock-names = "int";
 			status = "disabled";
@@ -340,6 +356,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x40008000 0x400>;
+			interrupts = <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM14_K>;
 			clock-names = "int";
 			status = "disabled";
@@ -623,6 +641,11 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x44000000 0x400>;
+			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 27 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "brk", "up", "trg-com", "cc";
 			clocks = <&rcc TIM1_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 11 0x400 0x1>,
@@ -659,6 +682,11 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x44001000 0x400>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "brk", "up", "trg-com", "cc";
 			clocks = <&rcc TIM8_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 47 0x400 0x1>,
@@ -746,6 +774,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x44006000 0x400>;
+			interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM15_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 105 0x400 0x1>,
@@ -773,6 +803,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x44007000 0x400>;
+			interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM16_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 109 0x400 0x1>,
@@ -797,6 +829,8 @@
 			#size-cells = <0>;
 			compatible = "st,stm32-timers";
 			reg = <0x44008000 0x400>;
+			interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "global";
 			clocks = <&rcc TIM17_K>;
 			clock-names = "int";
 			dmas = <&dmamux1 111 0x400 0x1>,
diff --git a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2-of10.dts b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2-of10.dts
index 2a2829283456..9a2a4bc7d079 100644
--- a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2-of10.dts
+++ b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2-of10.dts
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) STMicroelectronics 2019 - All Rights Reserved
  * Copyright (c) 2020 Engicam srl
- * Copyright (c) 2020 Amarula Solutons(India)
+ * Copyright (c) 2020 Amarula Solutions(India)
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2.dts b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2.dts
index 1f75f1d45181..60ce4425a7fd 100644
--- a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2.dts
+++ b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-ctouch2.dts
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) STMicroelectronics 2019 - All Rights Reserved
  * Copyright (c) 2020 Engicam srl
- * Copyright (c) 2020 Amarula Solutons(India)
+ * Copyright (c) 2020 Amarula Solutions(India)
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-edimm2.2.dts b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-edimm2.2.dts
index ba92d7d8ed00..390ee8c05754 100644
--- a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-edimm2.2.dts
+++ b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1-edimm2.2.dts
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) STMicroelectronics 2019 - All Rights Reserved
  * Copyright (c) 2020 Engicam srl
- * Copyright (c) 2020 Amarula Solutons(India)
+ * Copyright (c) 2020 Amarula Solutions(India)
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1.dtsi b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1.dtsi
index 01166ccacf2b..9de893101b40 100644
--- a/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1.dtsi
+++ b/arch/arm/boot/dts/stm32mp157a-icore-stm32mp1.dtsi
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) STMicroelectronics 2019 - All Rights Reserved
  * Copyright (c) 2020 Engicam srl
- * Copyright (c) 2020 Amarula Solutons(India)
+ * Copyright (c) 2020 Amarula Solutions(India)
  */
 
 / {
diff --git a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0-of7.dts b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0-of7.dts
index fae656edd820..0d7560ba2950 100644
--- a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0-of7.dts
+++ b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0-of7.dts
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) STMicroelectronics 2019 - All Rights Reserved
  * Copyright (c) 2020 Engicam srl
- * Copyright (c) 2020 Amarula Solutons(India)
+ * Copyright (c) 2020 Amarula Solutions(India)
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0.dts b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0.dts
index b9d0d3d6ad15..d949559be020 100644
--- a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0.dts
+++ b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1-microdev2.0.dts
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) STMicroelectronics 2019 - All Rights Reserved
  * Copyright (c) 2020 Engicam srl
- * Copyright (c) 2020 Amarula Solutons(India)
+ * Copyright (c) 2020 Amarula Solutions(India)
  */
 
 /dts-v1/;
diff --git a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1.dtsi b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1.dtsi
index 0b85175f151e..fb4600a59869 100644
--- a/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1.dtsi
+++ b/arch/arm/boot/dts/stm32mp157a-microgea-stm32mp1.dtsi
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) STMicroelectronics 2019 - All Rights Reserved
  * Copyright (c) 2020 Engicam srl
- * Copyright (c) 2020 Amarula Solutons(India)
+ * Copyright (c) 2020 Amarula Solutions(India)
  */
 
 / {
diff --git a/arch/arm/boot/dts/stm32mp157c-emstamp-argon.dtsi b/arch/arm/boot/dts/stm32mp157c-emstamp-argon.dtsi
index ac53ee3c496b..30156b7546ed 100644
--- a/arch/arm/boot/dts/stm32mp157c-emstamp-argon.dtsi
+++ b/arch/arm/boot/dts/stm32mp157c-emstamp-argon.dtsi
@@ -435,12 +435,6 @@
 	pinctrl-0 = <&spi1_pins_a>;
 	cs-gpios = <&gpioz 3 0>;
 	status = "disabled";
-
-	spidev@0  {
-		compatible = "spidev";
-		reg = <0>;
-		spi-max-frequency = <100000>;
-	};
 };
 
 &timers1 {
diff --git a/arch/arm/boot/dts/stm32mp157c-ev1.dts b/arch/arm/boot/dts/stm32mp157c-ev1.dts
index d142dd30e16b..050c3c27a420 100644
--- a/arch/arm/boot/dts/stm32mp157c-ev1.dts
+++ b/arch/arm/boot/dts/stm32mp157c-ev1.dts
@@ -255,8 +255,16 @@
 
 &qspi {
 	pinctrl-names = "default", "sleep";
-	pinctrl-0 = <&qspi_clk_pins_a &qspi_bk1_pins_a &qspi_bk2_pins_a>;
-	pinctrl-1 = <&qspi_clk_sleep_pins_a &qspi_bk1_sleep_pins_a &qspi_bk2_sleep_pins_a>;
+	pinctrl-0 = <&qspi_clk_pins_a
+		     &qspi_bk1_pins_a
+		     &qspi_cs1_pins_a
+		     &qspi_bk2_pins_a
+		     &qspi_cs2_pins_a>;
+	pinctrl-1 = <&qspi_clk_sleep_pins_a
+		     &qspi_bk1_sleep_pins_a
+		     &qspi_cs1_sleep_pins_a
+		     &qspi_bk2_sleep_pins_a
+		     &qspi_cs2_sleep_pins_a>;
 	reg = <0x58003000 0x1000>, <0x70000000 0x4000000>;
 	#address-cells = <1>;
 	#size-cells = <0>;
diff --git a/arch/arm/boot/dts/uniphier-ld4-ref.dts b/arch/arm/boot/dts/uniphier-ld4-ref.dts
index c46c2e8a10a7..e007db084787 100644
--- a/arch/arm/boot/dts/uniphier-ld4-ref.dts
+++ b/arch/arm/boot/dts/uniphier-ld4-ref.dts
@@ -36,11 +36,11 @@
 };
 
 &ethsc {
-	interrupts = <1 8>;
+	interrupts = <1 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serialsc {
-	interrupts = <1 8>;
+	interrupts = <1 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serial0 {
@@ -56,7 +56,7 @@
 };
 
 &gpio {
-	xirq1 {
+	xirq1-hog {
 		gpio-hog;
 		gpios = <UNIPHIER_GPIO_IRQ(1) 0>;
 		input;
diff --git a/arch/arm/boot/dts/uniphier-ld4.dtsi b/arch/arm/boot/dts/uniphier-ld4.dtsi
index b52957ccda0d..9dceff12a633 100644
--- a/arch/arm/boot/dts/uniphier-ld4.dtsi
+++ b/arch/arm/boot/dts/uniphier-ld4.dtsi
@@ -6,6 +6,7 @@
 //   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
 
 #include <dt-bindings/gpio/uniphier-gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
 
 / {
 	compatible = "socionext,uniphier-ld4";
@@ -55,7 +56,8 @@
 			compatible = "socionext,uniphier-system-cache";
 			reg = <0x500c0000 0x2000>, <0x503c0100 0x4>,
 			      <0x506c0000 0x400>;
-			interrupts = <0 174 4>, <0 175 4>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>;
 			cache-unified;
 			cache-size = <(512 * 1024)>;
 			cache-sets = <256>;
@@ -69,7 +71,7 @@
 			reg = <0x54006000 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 39 4>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi0>;
 			clocks = <&peri_clk 11>;
@@ -80,7 +82,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006800 0x40>;
-			interrupts = <0 33 4>;
+			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart0>;
 			clocks = <&peri_clk 0>;
@@ -91,7 +93,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006900 0x40>;
-			interrupts = <0 35 4>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart1>;
 			clocks = <&peri_clk 1>;
@@ -102,7 +104,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006a00 0x40>;
-			interrupts = <0 37 4>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart2>;
 			clocks = <&peri_clk 2>;
@@ -113,7 +115,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006b00 0x40>;
-			interrupts = <0 29 4>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart3>;
 			clocks = <&peri_clk 3>;
@@ -140,7 +142,7 @@
 			reg = <0x58400000 0x40>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 41 1>;
+			interrupts = <GIC_SPI 41 IRQ_TYPE_EDGE_RISING>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c0>;
 			clocks = <&peri_clk 4>;
@@ -154,7 +156,7 @@
 			reg = <0x58480000 0x40>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 42 1>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_EDGE_RISING>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c1>;
 			clocks = <&peri_clk 5>;
@@ -168,7 +170,7 @@
 			reg = <0x58500000 0x40>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 43 1>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_EDGE_RISING>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c2>;
 			clocks = <&peri_clk 6>;
@@ -182,7 +184,7 @@
 			reg = <0x58580000 0x40>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 44 1>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_EDGE_RISING>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c3>;
 			clocks = <&peri_clk 7>;
@@ -240,8 +242,13 @@
 		dmac: dma-controller@5a000000 {
 			compatible = "socionext,uniphier-mio-dmac";
 			reg = <0x5a000000 0x1000>;
-			interrupts = <0 68 4>, <0 68 4>, <0 69 4>, <0 70 4>,
-				     <0 71 4>, <0 72 4>, <0 73 4>;
+			interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&mio_clk 7>;
 			resets = <&mio_rst 7>;
 			#dma-cells = <1>;
@@ -251,7 +258,7 @@
 			compatible = "socionext,uniphier-sd-v2.91";
 			status = "disabled";
 			reg = <0x5a400000 0x200>;
-			interrupts = <0 76 4>;
+			interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default", "uhs";
 			pinctrl-0 = <&pinctrl_sd>;
 			pinctrl-1 = <&pinctrl_sd_uhs>;
@@ -271,7 +278,7 @@
 			compatible = "socionext,uniphier-sd-v2.91";
 			status = "disabled";
 			reg = <0x5a500000 0x200>;
-			interrupts = <0 78 4>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_emmc>;
 			clocks = <&mio_clk 1>;
@@ -289,7 +296,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a800100 0x100>;
-			interrupts = <0 80 4>;
+			interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb0>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 8>,
@@ -303,7 +310,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a810100 0x100>;
-			interrupts = <0 81 4>;
+			interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb1>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 9>,
@@ -317,7 +324,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a820100 0x100>;
-			interrupts = <0 82 4>;
+			interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb2>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 10>,
@@ -358,14 +365,16 @@
 		timer@60000200 {
 			compatible = "arm,cortex-a9-global-timer";
 			reg = <0x60000200 0x20>;
-			interrupts = <1 11 0x104>;
+			interrupts = <GIC_PPI 11
+				(GIC_CPU_MASK_RAW(1) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&arm_timer_clk>;
 		};
 
 		timer@60000600 {
 			compatible = "arm,cortex-a9-twd-timer";
 			reg = <0x60000600 0x20>;
-			interrupts = <1 13 0x104>;
+			interrupts = <GIC_PPI 13
+				(GIC_CPU_MASK_RAW(1) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&arm_timer_clk>;
 		};
 
@@ -407,7 +416,7 @@
 			reg = <0x68000000 0x20>, <0x68100000 0x1000>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 65 4>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_nand>;
 			clock-names = "nand", "nand_x", "ecc";
diff --git a/arch/arm/boot/dts/uniphier-ld6b-ref.dts b/arch/arm/boot/dts/uniphier-ld6b-ref.dts
index 5bc7fe11b517..223a78b4a761 100644
--- a/arch/arm/boot/dts/uniphier-ld6b-ref.dts
+++ b/arch/arm/boot/dts/uniphier-ld6b-ref.dts
@@ -40,11 +40,11 @@
 };
 
 &ethsc {
-	interrupts = <4 8>;
+	interrupts = <4 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serialsc {
-	interrupts = <4 8>;
+	interrupts = <4 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serial0 {
@@ -60,7 +60,7 @@
 };
 
 &gpio {
-	xirq4 {
+	xirq4-hog {
 		gpio-hog;
 		gpios = <UNIPHIER_GPIO_IRQ(4) 0>;
 		input;
diff --git a/arch/arm/boot/dts/uniphier-pinctrl.dtsi b/arch/arm/boot/dts/uniphier-pinctrl.dtsi
index c0fd029b37e5..f909ec2e5333 100644
--- a/arch/arm/boot/dts/uniphier-pinctrl.dtsi
+++ b/arch/arm/boot/dts/uniphier-pinctrl.dtsi
@@ -196,11 +196,21 @@
 		function = "usb0";
 	};
 
+	pinctrl_usb0_device: usb0-device {
+		groups = "usb0_device";
+		function = "usb0";
+	};
+
 	pinctrl_usb1: usb1 {
 		groups = "usb1";
 		function = "usb1";
 	};
 
+	pinctrl_usb1_device: usb1-device {
+		groups = "usb1_device";
+		function = "usb1";
+	};
+
 	pinctrl_usb2: usb2 {
 		groups = "usb2";
 		function = "usb2";
diff --git a/arch/arm/boot/dts/uniphier-pro4-ace.dts b/arch/arm/boot/dts/uniphier-pro4-ace.dts
index 27ff2b7b9d0e..6baee4410d9c 100644
--- a/arch/arm/boot/dts/uniphier-pro4-ace.dts
+++ b/arch/arm/boot/dts/uniphier-pro4-ace.dts
@@ -99,3 +99,11 @@
 &usb1 {
 	status = "okay";
 };
+
+&ahci0 {
+	status = "okay";
+};
+
+&ahci1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/uniphier-pro4-ref.dts b/arch/arm/boot/dts/uniphier-pro4-ref.dts
index 3b9b61314d01..d2ce5c039865 100644
--- a/arch/arm/boot/dts/uniphier-pro4-ref.dts
+++ b/arch/arm/boot/dts/uniphier-pro4-ref.dts
@@ -39,11 +39,11 @@
 };
 
 &ethsc {
-	interrupts = <2 8>;
+	interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serialsc {
-	interrupts = <2 8>;
+	interrupts = <2 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serial0 {
@@ -59,7 +59,7 @@
 };
 
 &gpio {
-	xirq2 {
+	xirq2-hog {
 		gpio-hog;
 		gpios = <UNIPHIER_GPIO_IRQ(2) 0>;
 		input;
@@ -108,3 +108,11 @@
 		reg = <0>;
 	};
 };
+
+&ahci0 {
+	status = "okay";
+};
+
+&ahci1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/uniphier-pro4.dtsi b/arch/arm/boot/dts/uniphier-pro4.dtsi
index a53b73ee93e9..a309e64c57c8 100644
--- a/arch/arm/boot/dts/uniphier-pro4.dtsi
+++ b/arch/arm/boot/dts/uniphier-pro4.dtsi
@@ -6,6 +6,7 @@
 //   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
 
 #include <dt-bindings/gpio/uniphier-gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
 
 / {
 	compatible = "socionext,uniphier-pro4";
@@ -63,7 +64,8 @@
 			compatible = "socionext,uniphier-system-cache";
 			reg = <0x500c0000 0x2000>, <0x503c0100 0x4>,
 			      <0x506c0000 0x400>;
-			interrupts = <0 174 4>, <0 175 4>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>;
 			cache-unified;
 			cache-size = <(768 * 1024)>;
 			cache-sets = <256>;
@@ -77,7 +79,7 @@
 			reg = <0x54006000 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 39 4>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi0>;
 			clocks = <&peri_clk 11>;
@@ -88,7 +90,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006800 0x40>;
-			interrupts = <0 33 4>;
+			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart0>;
 			clocks = <&peri_clk 0>;
@@ -99,7 +101,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006900 0x40>;
-			interrupts = <0 35 4>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart1>;
 			clocks = <&peri_clk 1>;
@@ -110,7 +112,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006a00 0x40>;
-			interrupts = <0 37 4>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart2>;
 			clocks = <&peri_clk 2>;
@@ -121,7 +123,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006b00 0x40>;
-			interrupts = <0 177 4>;
+			interrupts = <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart3>;
 			clocks = <&peri_clk 3>;
@@ -148,7 +150,7 @@
 			reg = <0x58780000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 41 4>;
+			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c0>;
 			clocks = <&peri_clk 4>;
@@ -162,7 +164,7 @@
 			reg = <0x58781000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 42 4>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c1>;
 			clocks = <&peri_clk 5>;
@@ -176,7 +178,7 @@
 			reg = <0x58782000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 43 4>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c2>;
 			clocks = <&peri_clk 6>;
@@ -190,7 +192,7 @@
 			reg = <0x58783000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 44 4>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c3>;
 			clocks = <&peri_clk 7>;
@@ -206,7 +208,7 @@
 			reg = <0x58785000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 25 4>;
+			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 9>;
 			resets = <&peri_rst 9>;
 			clock-frequency = <400000>;
@@ -218,7 +220,7 @@
 			reg = <0x58786000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 26 4>;
+			interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 10>;
 			resets = <&peri_rst 10>;
 			clock-frequency = <400000>;
@@ -274,8 +276,14 @@
 		dmac: dma-controller@5a000000 {
 			compatible = "socionext,uniphier-mio-dmac";
 			reg = <0x5a000000 0x1000>;
-			interrupts = <0 68 4>, <0 68 4>, <0 69 4>, <0 70 4>,
-				     <0 71 4>, <0 72 4>, <0 73 4>, <0 74 4>;
+			interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&mio_clk 7>;
 			resets = <&mio_rst 7>;
 			#dma-cells = <1>;
@@ -285,7 +293,7 @@
 			compatible = "socionext,uniphier-sd-v2.91";
 			status = "disabled";
 			reg = <0x5a400000 0x200>;
-			interrupts = <0 76 4>;
+			interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default", "uhs";
 			pinctrl-0 = <&pinctrl_sd>;
 			pinctrl-1 = <&pinctrl_sd_uhs>;
@@ -305,7 +313,7 @@
 			compatible = "socionext,uniphier-sd-v2.91";
 			status = "disabled";
 			reg = <0x5a500000 0x200>;
-			interrupts = <0 78 4>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_emmc>;
 			clocks = <&mio_clk 1>;
@@ -323,7 +331,7 @@
 			compatible = "socionext,uniphier-sd-v2.91";
 			status = "disabled";
 			reg = <0x5a600000 0x200>;
-			interrupts = <0 85 4>;
+			interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_sd1>;
 			clocks = <&mio_clk 2>;
@@ -339,7 +347,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a800100 0x100>;
-			interrupts = <0 80 4>;
+			interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb2>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 8>,
@@ -355,7 +363,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a810100 0x100>;
-			interrupts = <0 81 4>;
+			interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb3>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 9>,
@@ -376,7 +384,7 @@
 				compatible = "socionext,uniphier-pro4-pinctrl";
 			};
 
-			usb-phy {
+			usb-controller {
 				compatible = "socionext,uniphier-pro4-usb2-phy";
 				#address-cells = <1>;
 				#size-cells = <0>;
@@ -403,6 +411,11 @@
 					vbus-supply = <&usb1_vbus>;
 				};
 			};
+
+			sg_clk: clock {
+				compatible = "socionext,uniphier-pro4-sg-clock";
+				#clock-cells = <1>;
+			};
 		};
 
 		soc-glue@5f900000 {
@@ -431,7 +444,7 @@
 		xdmac: dma-controller@5fc10000 {
 			compatible = "socionext,uniphier-xdmac";
 			reg = <0x5fc10000 0x5300>;
-			interrupts = <0 188 4>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
 			dma-channels = <16>;
 			#dma-cells = <2>;
 		};
@@ -446,14 +459,16 @@
 		timer@60000200 {
 			compatible = "arm,cortex-a9-global-timer";
 			reg = <0x60000200 0x20>;
-			interrupts = <1 11 0x304>;
+			interrupts = <GIC_PPI 11
+				(GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&arm_timer_clk>;
 		};
 
 		timer@60000600 {
 			compatible = "arm,cortex-a9-twd-timer";
 			reg = <0x60000600 0x20>;
-			interrupts = <1 13 0x304>;
+			interrupts = <GIC_PPI 13
+				(GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&arm_timer_clk>;
 		};
 
@@ -485,7 +500,7 @@
 			compatible = "socionext,uniphier-pro4-ave4";
 			status = "disabled";
 			reg = <0x65000000 0x8500>;
-			interrupts = <0 66 4>;
+			interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ether_rgmii>;
 			clock-names = "gio", "ether", "ether-gb", "ether-phy";
@@ -503,12 +518,105 @@
 			};
 		};
 
+		ahci0: sata@65600000 {
+			compatible = "socionext,uniphier-pro4-ahci",
+				     "generic-ahci";
+			status = "disabled";
+			reg = <0x65600000 0x10000>;
+			interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sys_clk 12>, <&sys_clk 28>;
+			resets = <&sys_rst 12>, <&sys_rst 28>, <&ahci0_rst 3>;
+			ports-implemented = <1>;
+			phys = <&ahci0_phy>;
+			assigned-clocks = <&sg_clk 0>;
+			assigned-clock-rates = <25000000>;
+		};
+
+		sata-controller@65700000 {
+			compatible = "socionext,uniphier-pxs2-ahci-glue",
+				     "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x65700000 0x100>;
+
+			ahci0_rst: reset-controller@0 {
+				compatible = "socionext,uniphier-pro4-ahci-reset";
+				reg = <0x0 0x4>;
+				clock-names = "gio", "link";
+				clocks = <&sys_clk 12>, <&sys_clk 28>;
+				reset-names = "gio", "link";
+				resets = <&sys_rst 12>, <&sys_rst 28>;
+				#reset-cells = <1>;
+			};
+
+			ahci0_phy: sata-phy@10 {
+				compatible = "socionext,uniphier-pro4-ahci-phy";
+				reg = <0x10 0x40>;
+				clock-names = "link", "gio";
+				clocks = <&sys_clk 28>, <&sys_clk 12>;
+				reset-names = "link", "gio", "phy",
+					      "pm", "tx", "rx";
+				resets = <&sys_rst 28>, <&sys_rst 12>,
+					 <&sys_rst 30>,
+					 <&ahci0_rst 0>, <&ahci0_rst 1>,
+					 <&ahci0_rst 2>;
+				#phy-cells = <0>;
+			};
+		};
+
+		ahci1: sata@65800000 {
+			compatible = "socionext,uniphier-pro4-ahci",
+				     "generic-ahci";
+			status = "disabled";
+			reg = <0x65800000 0x10000>;
+			interrupts = <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sys_clk 12>, <&sys_clk 29>;
+			resets = <&sys_rst 12>, <&sys_rst 29>, <&ahci1_rst 3>;
+			ports-implemented = <1>;
+			phys = <&ahci1_phy>;
+			assigned-clocks = <&sg_clk 0>;
+			assigned-clock-rates = <25000000>;
+		};
+
+		sata-controller@65900000 {
+			compatible = "socionext,uniphier-pro4-ahci-glue",
+				     "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x65900000 0x100>;
+
+			ahci1_rst: reset-controller@0 {
+				compatible = "socionext,uniphier-pro4-ahci-reset";
+				reg = <0x0 0x4>;
+				clock-names = "gio", "link";
+				clocks = <&sys_clk 12>, <&sys_clk 29>;
+				reset-names = "gio", "link";
+				resets = <&sys_rst 12>, <&sys_rst 29>;
+				#reset-cells = <1>;
+			};
+
+			ahci1_phy: sata-phy@10 {
+				compatible = "socionext,uniphier-pro4-ahci-phy";
+				reg = <0x10 0x40>;
+				clock-names = "link", "gio";
+				clocks = <&sys_clk 29>, <&sys_clk 12>;
+				reset-names = "link", "gio", "phy",
+					      "pm", "tx", "rx";
+				resets = <&sys_rst 29>, <&sys_rst 12>,
+					 <&sys_rst 30>,
+					 <&ahci1_rst 0>, <&ahci1_rst 1>,
+					 <&ahci1_rst 2>;
+				#phy-cells = <0>;
+			};
+		};
+
 		usb0: usb@65a00000 {
 			compatible = "socionext,uniphier-dwc3", "snps,dwc3";
 			status = "disabled";
 			reg = <0x65a00000 0xcd00>;
 			interrupt-names = "host", "peripheral";
-			interrupts = <0 134 4>, <0 135 4>;
+			interrupts = <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb0>;
 			clock-names = "ref", "bus_early", "suspend";
@@ -518,7 +626,7 @@
 			dr_mode = "host";
 		};
 
-		usb-glue@65b00000 {
+		usb-controller@65b00000 {
 			compatible = "socionext,uniphier-pro4-dwc3-glue",
 				     "simple-mfd";
 			#address-cells = <1>;
@@ -561,7 +669,8 @@
 			status = "disabled";
 			reg = <0x65c00000 0xcd00>;
 			interrupt-names = "host", "peripheral";
-			interrupts = <0 137 4>, <0 138 4>;
+			interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb1>;
 			clock-names = "ref", "bus_early", "suspend";
@@ -571,7 +680,7 @@
 			dr_mode = "host";
 		};
 
-		usb-glue@65d00000 {
+		usb-controller@65d00000 {
 			compatible = "socionext,uniphier-pro4-dwc3-glue",
 				     "simple-mfd";
 			#address-cells = <1>;
@@ -605,7 +714,7 @@
 			reg = <0x68000000 0x20>, <0x68100000 0x1000>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 65 4>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_nand>;
 			clock-names = "nand", "nand_x", "ecc";
diff --git a/arch/arm/boot/dts/uniphier-pro5.dtsi b/arch/arm/boot/dts/uniphier-pro5.dtsi
index 3525125832dd..100edd7438d8 100644
--- a/arch/arm/boot/dts/uniphier-pro5.dtsi
+++ b/arch/arm/boot/dts/uniphier-pro5.dtsi
@@ -5,6 +5,8 @@
 // Copyright (C) 2015-2016 Socionext Inc.
 //   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
 
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
 / {
 	compatible = "socionext,uniphier-pro5";
 	#address-cells = <1>;
@@ -135,7 +137,8 @@
 			compatible = "socionext,uniphier-system-cache";
 			reg = <0x500c0000 0x2000>, <0x503c0100 0x8>,
 			      <0x506c0000 0x400>;
-			interrupts = <0 190 4>, <0 191 4>;
+			interrupts = <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
 			cache-unified;
 			cache-size = <(2 * 1024 * 1024)>;
 			cache-sets = <512>;
@@ -148,7 +151,8 @@
 			compatible = "socionext,uniphier-system-cache";
 			reg = <0x500c8000 0x2000>, <0x503c8100 0x8>,
 			      <0x506c8000 0x400>;
-			interrupts = <0 174 4>, <0 175 4>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>;
 			cache-unified;
 			cache-size = <(2 * 1024 * 1024)>;
 			cache-sets = <512>;
@@ -162,7 +166,7 @@
 			reg = <0x54006000 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 39 4>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi0>;
 			clocks = <&peri_clk 11>;
@@ -175,7 +179,7 @@
 			reg = <0x54006100 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 216 4>;
+			interrupts = <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi1>;
 			clocks = <&peri_clk 11>;	/* common with spi0 */
@@ -186,7 +190,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006800 0x40>;
-			interrupts = <0 33 4>;
+			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart0>;
 			clocks = <&peri_clk 0>;
@@ -197,7 +201,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006900 0x40>;
-			interrupts = <0 35 4>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart1>;
 			clocks = <&peri_clk 1>;
@@ -208,7 +212,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006a00 0x40>;
-			interrupts = <0 37 4>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart2>;
 			clocks = <&peri_clk 2>;
@@ -219,7 +223,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006b00 0x40>;
-			interrupts = <0 177 4>;
+			interrupts = <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart3>;
 			clocks = <&peri_clk 3>;
@@ -246,7 +250,7 @@
 			reg = <0x58780000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 41 4>;
+			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c0>;
 			clocks = <&peri_clk 4>;
@@ -260,7 +264,7 @@
 			reg = <0x58781000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 42 4>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c1>;
 			clocks = <&peri_clk 5>;
@@ -274,7 +278,7 @@
 			reg = <0x58782000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 43 4>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c2>;
 			clocks = <&peri_clk 6>;
@@ -288,7 +292,7 @@
 			reg = <0x58783000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 44 4>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c3>;
 			clocks = <&peri_clk 7>;
@@ -304,7 +308,7 @@
 			reg = <0x58785000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 25 4>;
+			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 9>;
 			resets = <&peri_rst 9>;
 			clock-frequency = <400000>;
@@ -316,7 +320,7 @@
 			reg = <0x58786000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 26 4>;
+			interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 10>;
 			resets = <&peri_rst 10>;
 			clock-frequency = <400000>;
@@ -415,7 +419,7 @@
 		xdmac: dma-controller@5fc10000 {
 			compatible = "socionext,uniphier-xdmac";
 			reg = <0x5fc10000 0x5300>;
-			interrupts = <0 188 4>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
 			dma-channels = <16>;
 			#dma-cells = <2>;
 		};
@@ -430,14 +434,16 @@
 		timer@60000200 {
 			compatible = "arm,cortex-a9-global-timer";
 			reg = <0x60000200 0x20>;
-			interrupts = <1 11 0x304>;
+			interrupts = <GIC_PPI 11
+				(GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&arm_timer_clk>;
 		};
 
 		timer@60000600 {
 			compatible = "arm,cortex-a9-twd-timer";
 			reg = <0x60000600 0x20>;
-			interrupts = <1 13 0x304>;
+			interrupts = <GIC_PPI 13
+				(GIC_CPU_MASK_RAW(3) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&arm_timer_clk>;
 		};
 
@@ -470,7 +476,7 @@
 			status = "disabled";
 			reg = <0x65a00000 0xcd00>;
 			interrupt-names = "host";
-			interrupts = <0 134 4>;
+			interrupts = <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb0>;
 			clock-names = "ref", "bus_early", "suspend";
@@ -480,7 +486,7 @@
 			dr_mode = "host";
 		};
 
-		usb-glue@65b00000 {
+		usb-controller@65b00000 {
 			compatible = "socionext,uniphier-pro5-dwc3-glue",
 				     "simple-mfd";
 			#address-cells = <1>;
@@ -534,7 +540,7 @@
 			status = "disabled";
 			reg = <0x65c00000 0xcd00>;
 			interrupt-names = "host";
-			interrupts = <0 137 4>;
+			interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb1>, <&pinctrl_usb2>;
 			clock-names = "ref", "bus_early", "suspend";
@@ -544,7 +550,7 @@
 			dr_mode = "host";
 		};
 
-		usb-glue@65d00000 {
+		usb-controller@65d00000 {
 			compatible = "socionext,uniphier-pro5-dwc3-glue",
 				     "simple-mfd";
 			#address-cells = <1>;
@@ -614,8 +620,7 @@
 		};
 
 		pcie_ep: pcie-ep@66000000 {
-			compatible = "socionext,uniphier-pro5-pcie-ep",
-				     "snps,dw-pcie-ep";
+			compatible = "socionext,uniphier-pro5-pcie-ep";
 			status = "disabled";
 			reg-names = "dbi", "dbi2", "link", "addr_space";
 			reg = <0x66000000 0x1000>, <0x66001000 0x1000>,
@@ -650,7 +655,7 @@
 			reg = <0x68000000 0x20>, <0x68100000 0x1000>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 65 4>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_nand>;
 			clock-names = "nand", "nand_x", "ecc";
@@ -663,7 +668,7 @@
 			compatible = "socionext,uniphier-sd-v3.1";
 			status = "disabled";
 			reg = <0x68400000 0x800>;
-			interrupts = <0 78 4>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_emmc>;
 			clocks = <&sd_clk 1>;
@@ -679,7 +684,7 @@
 			compatible = "socionext,uniphier-sd-v3.1";
 			status = "disabled";
 			reg = <0x68800000 0x800>;
-			interrupts = <0 76 4>;
+			interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default", "uhs";
 			pinctrl-0 = <&pinctrl_sd>;
 			pinctrl-1 = <&pinctrl_sd_uhs>;
diff --git a/arch/arm/boot/dts/uniphier-pxs2-gentil.dts b/arch/arm/boot/dts/uniphier-pxs2-gentil.dts
index 759384b60663..5f18b926c50a 100644
--- a/arch/arm/boot/dts/uniphier-pxs2-gentil.dts
+++ b/arch/arm/boot/dts/uniphier-pxs2-gentil.dts
@@ -99,3 +99,7 @@
 &usb1 {
 	status = "okay";
 };
+
+&ahci {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/uniphier-pxs2.dtsi b/arch/arm/boot/dts/uniphier-pxs2.dtsi
index 03301ddb3403..ca4dccf56a67 100644
--- a/arch/arm/boot/dts/uniphier-pxs2.dtsi
+++ b/arch/arm/boot/dts/uniphier-pxs2.dtsi
@@ -6,6 +6,7 @@
 //   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
 
 #include <dt-bindings/gpio/uniphier-gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/thermal/thermal.h>
 
 / {
@@ -161,7 +162,10 @@
 			compatible = "socionext,uniphier-system-cache";
 			reg = <0x500c0000 0x2000>, <0x503c0100 0x8>,
 			      <0x506c0000 0x400>;
-			interrupts = <0 174 4>, <0 175 4>, <0 190 4>, <0 191 4>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>;
 			cache-unified;
 			cache-size = <(1280 * 1024)>;
 			cache-sets = <512>;
@@ -175,7 +179,7 @@
 			reg = <0x54006000 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 39 4>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi0>;
 			clocks = <&peri_clk 11>;
@@ -188,7 +192,7 @@
 			reg = <0x54006100 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 216 4>;
+			interrupts = <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi1>;
 			clocks = <&peri_clk 12>;
@@ -199,7 +203,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006800 0x40>;
-			interrupts = <0 33 4>;
+			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart0>;
 			clocks = <&peri_clk 0>;
@@ -210,7 +214,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006900 0x40>;
-			interrupts = <0 35 4>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart1>;
 			clocks = <&peri_clk 1>;
@@ -221,7 +225,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006a00 0x40>;
-			interrupts = <0 37 4>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart2>;
 			clocks = <&peri_clk 2>;
@@ -232,7 +236,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006b00 0x40>;
-			interrupts = <0 177 4>;
+			interrupts = <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart3>;
 			clocks = <&peri_clk 3>;
@@ -259,7 +263,7 @@
 		audio@56000000 {
 			compatible = "socionext,uniphier-pxs2-aio";
 			reg = <0x56000000 0x80000>;
-			interrupts = <0 144 4>;
+			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ain1>,
 				    <&pinctrl_ain2>,
@@ -317,7 +321,7 @@
 			reg = <0x58780000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 41 4>;
+			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c0>;
 			clocks = <&peri_clk 4>;
@@ -331,7 +335,7 @@
 			reg = <0x58781000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 42 4>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c1>;
 			clocks = <&peri_clk 5>;
@@ -345,7 +349,7 @@
 			reg = <0x58782000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 43 4>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c2>;
 			clocks = <&peri_clk 6>;
@@ -359,7 +363,7 @@
 			reg = <0x58783000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 44 4>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c3>;
 			clocks = <&peri_clk 7>;
@@ -373,7 +377,7 @@
 			reg = <0x58784000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 45 4>;
+			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 8>;
 			resets = <&peri_rst 8>;
 			clock-frequency = <400000>;
@@ -385,7 +389,7 @@
 			reg = <0x58785000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 25 4>;
+			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 9>;
 			resets = <&peri_rst 9>;
 			clock-frequency = <400000>;
@@ -397,7 +401,7 @@
 			reg = <0x58786000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 26 4>;
+			interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 10>;
 			resets = <&peri_rst 10>;
 			clock-frequency = <400000>;
@@ -454,7 +458,7 @@
 			compatible = "socionext,uniphier-sd-v3.1.1";
 			status = "disabled";
 			reg = <0x5a000000 0x800>;
-			interrupts = <0 78 4>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_emmc>;
 			clocks = <&sd_clk 1>;
@@ -470,7 +474,7 @@
 			compatible = "socionext,uniphier-sd-v3.1.1";
 			status = "disabled";
 			reg = <0x5a400000 0x800>;
-			interrupts = <0 76 4>;
+			interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default", "uhs";
 			pinctrl-0 = <&pinctrl_sd>;
 			pinctrl-1 = <&pinctrl_sd_uhs>;
@@ -515,7 +519,7 @@
 		xdmac: dma-controller@5fc10000 {
 			compatible = "socionext,uniphier-xdmac";
 			reg = <0x5fc10000 0x5300>;
-			interrupts = <0 188 4>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
 			dma-channels = <16>;
 			#dma-cells = <2>;
 		};
@@ -530,14 +534,16 @@
 		timer@60000200 {
 			compatible = "arm,cortex-a9-global-timer";
 			reg = <0x60000200 0x20>;
-			interrupts = <1 11 0xf04>;
+			interrupts = <GIC_PPI 11
+				(GIC_CPU_MASK_RAW(0xf) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&arm_timer_clk>;
 		};
 
 		timer@60000600 {
 			compatible = "arm,cortex-a9-twd-timer";
 			reg = <0x60000600 0x20>;
-			interrupts = <1 13 0xf04>;
+			interrupts = <GIC_PPI 13
+				(GIC_CPU_MASK_RAW(0xf) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&arm_timer_clk>;
 		};
 
@@ -564,9 +570,9 @@
 				#reset-cells = <1>;
 			};
 
-			pvtctl: pvtctl {
+			pvtctl: thermal-sensor {
 				compatible = "socionext,uniphier-pxs2-thermal";
-				interrupts = <0 3 4>;
+				interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
 				#thermal-sensor-cells = <0>;
 				socionext,tmod-calibration = <0x0f86 0x6844>;
 			};
@@ -576,7 +582,7 @@
 			compatible = "socionext,uniphier-pxs2-ave4";
 			status = "disabled";
 			reg = <0x65000000 0x8500>;
-			interrupts = <0 66 4>;
+			interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ether_rgmii>;
 			clock-names = "ether";
@@ -593,12 +599,52 @@
 			};
 		};
 
+		ahci: sata@65600000 {
+			compatible = "socionext,uniphier-pxs2-ahci",
+				     "generic-ahci";
+			status = "disabled";
+			reg = <0x65600000 0x10000>;
+			interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sys_clk 28>;
+			resets = <&sys_rst 28>, <&ahci_rst 0>;
+			ports-implemented = <1>;
+			phys = <&ahci_phy>;
+		};
+
+		sata-controller@65700000 {
+			compatible = "socionext,uniphier-pxs2-ahci-glue",
+				     "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x65700000 0x100>;
+
+			ahci_rst: reset-controller@0 {
+				compatible = "socionext,uniphier-pxs2-ahci-reset";
+				reg = <0x0 0x4>;
+				clock-names = "link";
+				clocks = <&sys_clk 28>;
+				reset-names = "link";
+				resets = <&sys_rst 28>;
+				#reset-cells = <1>;
+			};
+
+			ahci_phy: sata-phy@10 {
+				compatible = "socionext,uniphier-pxs2-ahci-phy";
+				reg = <0x10 0x10>;
+				clock-names = "link";
+				clocks = <&sys_clk 28>;
+				reset-names = "link", "phy";
+				resets = <&sys_rst 28>, <&sys_rst 30>;
+				#phy-cells = <0>;
+			};
+		};
+
 		usb0: usb@65a00000 {
 			compatible = "socionext,uniphier-dwc3", "snps,dwc3";
 			status = "disabled";
 			reg = <0x65a00000 0xcd00>;
 			interrupt-names = "dwc_usb3";
-			interrupts = <0 134 4>;
+			interrupts = <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb0>, <&pinctrl_usb2>;
 			clock-names = "ref", "bus_early", "suspend";
@@ -609,7 +655,7 @@
 			dr_mode = "host";
 		};
 
-		usb-glue@65b00000 {
+		usb-controller@65b00000 {
 			compatible = "socionext,uniphier-pxs2-dwc3-glue",
 				     "simple-mfd";
 			#address-cells = <1>;
@@ -694,7 +740,7 @@
 			status = "disabled";
 			reg = <0x65c00000 0xcd00>;
 			interrupt-names = "dwc_usb3";
-			interrupts = <0 137 4>;
+			interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb1>, <&pinctrl_usb3>;
 			clock-names = "ref", "bus_early", "suspend";
@@ -704,7 +750,7 @@
 			dr_mode = "host";
 		};
 
-		usb-glue@65d00000 {
+		usb-controller@65d00000 {
 			compatible = "socionext,uniphier-pxs2-dwc3-glue",
 				     "simple-mfd";
 			#address-cells = <1>;
@@ -780,7 +826,7 @@
 			reg = <0x68000000 0x20>, <0x68100000 0x1000>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 65 4>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_nand>;
 			clock-names = "nand", "nand_x", "ecc";
diff --git a/arch/arm/boot/dts/uniphier-sld8-ref.dts b/arch/arm/boot/dts/uniphier-sld8-ref.dts
index 6db949ec7411..2446f9e15360 100644
--- a/arch/arm/boot/dts/uniphier-sld8-ref.dts
+++ b/arch/arm/boot/dts/uniphier-sld8-ref.dts
@@ -36,11 +36,11 @@
 };
 
 &ethsc {
-	interrupts = <0 8>;
+	interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serialsc {
-	interrupts = <0 8>;
+	interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serial0 {
@@ -56,7 +56,7 @@
 };
 
 &gpio {
-	xirq0 {
+	xirq0-hog {
 		gpio-hog;
 		gpios = <UNIPHIER_GPIO_IRQ(0) 0>;
 		input;
diff --git a/arch/arm/boot/dts/uniphier-sld8.dtsi b/arch/arm/boot/dts/uniphier-sld8.dtsi
index 96a766deb8d1..67b12dfe513b 100644
--- a/arch/arm/boot/dts/uniphier-sld8.dtsi
+++ b/arch/arm/boot/dts/uniphier-sld8.dtsi
@@ -6,6 +6,7 @@
 //   Author: Masahiro Yamada <yamada.masahiro@socionext.com>
 
 #include <dt-bindings/gpio/uniphier-gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
 
 / {
 	compatible = "socionext,uniphier-sld8";
@@ -55,7 +56,8 @@
 			compatible = "socionext,uniphier-system-cache";
 			reg = <0x500c0000 0x2000>, <0x503c0100 0x4>,
 			      <0x506c0000 0x400>;
-			interrupts = <0 174 4>, <0 175 4>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>;
 			cache-unified;
 			cache-size = <(256 * 1024)>;
 			cache-sets = <256>;
@@ -69,7 +71,7 @@
 			reg = <0x54006000 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 39 4>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi0>;
 			clocks = <&peri_clk 11>;
@@ -80,7 +82,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006800 0x40>;
-			interrupts = <0 33 4>;
+			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart0>;
 			clocks = <&peri_clk 0>;
@@ -91,7 +93,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006900 0x40>;
-			interrupts = <0 35 4>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart1>;
 			clocks = <&peri_clk 1>;
@@ -102,7 +104,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006a00 0x40>;
-			interrupts = <0 37 4>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart2>;
 			clocks = <&peri_clk 2>;
@@ -113,7 +115,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006b00 0x40>;
-			interrupts = <0 29 4>;
+			interrupts = <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart3>;
 			clocks = <&peri_clk 3>;
@@ -144,7 +146,7 @@
 			reg = <0x58400000 0x40>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 41 1>;
+			interrupts = <GIC_SPI 41 IRQ_TYPE_EDGE_RISING>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c0>;
 			clocks = <&peri_clk 4>;
@@ -158,7 +160,7 @@
 			reg = <0x58480000 0x40>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 42 1>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_EDGE_RISING>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c1>;
 			clocks = <&peri_clk 5>;
@@ -172,7 +174,7 @@
 			reg = <0x58500000 0x40>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 43 1>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_EDGE_RISING>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c2>;
 			clocks = <&peri_clk 6>;
@@ -186,7 +188,7 @@
 			reg = <0x58580000 0x40>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 44 1>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_EDGE_RISING>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c3>;
 			clocks = <&peri_clk 7>;
@@ -244,8 +246,13 @@
 		dmac: dma-controller@5a000000 {
 			compatible = "socionext,uniphier-mio-dmac";
 			reg = <0x5a000000 0x1000>;
-			interrupts = <0 68 4>, <0 68 4>, <0 69 4>, <0 70 4>,
-				     <0 71 4>, <0 72 4>, <0 73 4>;
+			interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&mio_clk 7>;
 			resets = <&mio_rst 7>;
 			#dma-cells = <1>;
@@ -255,7 +262,7 @@
 			compatible = "socionext,uniphier-sd-v2.91";
 			status = "disabled";
 			reg = <0x5a400000 0x200>;
-			interrupts = <0 76 4>;
+			interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default", "uhs";
 			pinctrl-0 = <&pinctrl_sd>;
 			pinctrl-1 = <&pinctrl_sd_uhs>;
@@ -275,7 +282,7 @@
 			compatible = "socionext,uniphier-sd-v2.91";
 			status = "disabled";
 			reg = <0x5a500000 0x200>;
-			interrupts = <0 78 4>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_emmc>;
 			clocks = <&mio_clk 1>;
@@ -293,7 +300,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a800100 0x100>;
-			interrupts = <0 80 4>;
+			interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb0>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 8>,
@@ -307,7 +314,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a810100 0x100>;
-			interrupts = <0 81 4>;
+			interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb1>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 9>,
@@ -321,7 +328,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a820100 0x100>;
-			interrupts = <0 82 4>;
+			interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb2>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 10>,
@@ -362,14 +369,16 @@
 		timer@60000200 {
 			compatible = "arm,cortex-a9-global-timer";
 			reg = <0x60000200 0x20>;
-			interrupts = <1 11 0x104>;
+			interrupts = <GIC_PPI 11
+				(GIC_CPU_MASK_RAW(1) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&arm_timer_clk>;
 		};
 
 		timer@60000600 {
 			compatible = "arm,cortex-a9-twd-timer";
 			reg = <0x60000600 0x20>;
-			interrupts = <1 13 0x104>;
+			interrupts = <GIC_PPI 13
+				(GIC_CPU_MASK_RAW(1) | IRQ_TYPE_LEVEL_HIGH)>;
 			clocks = <&arm_timer_clk>;
 		};
 
@@ -411,7 +420,7 @@
 			reg = <0x68000000 0x20>, <0x68100000 0x1000>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 65 4>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_nand>;
 			clock-names = "nand", "nand_x", "ecc";
diff --git a/arch/arm/boot/dts/uniphier-support-card.dtsi b/arch/arm/boot/dts/uniphier-support-card.dtsi
index 444802fee9fb..97e7d5db8eb8 100644
--- a/arch/arm/boot/dts/uniphier-support-card.dtsi
+++ b/arch/arm/boot/dts/uniphier-support-card.dtsi
@@ -8,13 +8,13 @@
 &system_bus {
 	status = "okay";
 	ranges = <1 0x00000000 0x42000000 0x02000000>;
-	interrupt-parent = <&gpio>;
 
 	ethsc: ethernet@1,1f00000 {
 		compatible = "smsc,lan9118", "smsc,lan9115";
 		reg = <1 0x01f00000 0x1000>;
 		phy-mode = "mii";
 		reg-io-width = <4>;
+		interrupt-parent = <&gpio>;
 	};
 
 	serialsc: serial@1,1fb0000 {
@@ -22,5 +22,6 @@
 		reg = <1 0x01fb0000 0x20>;
 		clock-frequency = <12288000>;
 		reg-shift = <1>;
+		interrupt-parent = <&gpio>;
 	};
 };
diff --git a/arch/arm/boot/dts/vf610-bk4.dts b/arch/arm/boot/dts/vf610-bk4.dts
index 830c85476b3d..551a4c3ff4fa 100644
--- a/arch/arm/boot/dts/vf610-bk4.dts
+++ b/arch/arm/boot/dts/vf610-bk4.dts
@@ -61,7 +61,7 @@
 		regulator-max-microvolt = <3300000>;
 	};
 
-	spi-gpio {
+	spi {
 		compatible = "spi-gpio";
 		pinctrl-0 = <&pinctrl_gpio_spi>;
 		pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/vf610-pinfunc.h b/arch/arm/boot/dts/vf610-pinfunc.h
index f1e5a7cf58a9..b7b7322a2d1b 100644
--- a/arch/arm/boot/dts/vf610-pinfunc.h
+++ b/arch/arm/boot/dts/vf610-pinfunc.h
@@ -420,7 +420,7 @@
 #define VF610_PAD_PTD29__FTM3_CH2		0x104 0x000 ALT4 0x0
 #define VF610_PAD_PTD29__DSPI2_SIN		0x104 0x000 ALT5 0x0
 #define VF610_PAD_PTD29__DEBUG_OUT11		0x104 0x000 ALT7 0x0
-#define VF610_PAD_PTD28__GPIO_66	 	0x108 0x000 ALT0 0x0
+#define VF610_PAD_PTD28__GPIO_66		0x108 0x000 ALT0 0x0
 #define VF610_PAD_PTD28__FB_AD28		0x108 0x000 ALT1 0x0
 #define VF610_PAD_PTD28__NF_IO12		0x108 0x000 ALT2 0x0
 #define VF610_PAD_PTD28__I2C2_SCL		0x108 0x34C ALT3 0x1
@@ -802,5 +802,55 @@
 #define VF610_PAD_PTE28__EWM_OUT		0x214 0x000 ALT7 0x0
 #define VF610_PAD_PTA7__GPIO_134		0x218 0x000 ALT0 0x0
 #define VF610_PAD_PTA7__VIU_PIX_CLK		0x218 0x3AC ALT1 0x1
+#define VF610_PAD_DDR_RESETB			0x21c 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A15__DDR_A_15		0x220 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A14__DDR_A_14		0x224 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A13__DDR_A_13		0x228 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A12__DDR_A_12		0x22c 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A11__DDR_A_11		0x230 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A10__DDR_A_10		0x234 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A9__DDR_A_9		0x238 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A8__DDR_A_8		0x23c 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A7__DDR_A_7		0x240 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A6__DDR_A_6		0x244 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A5__DDR_A_5		0x248 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A4__DDR_A_4		0x24c 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A3__DDR_A_3		0x250 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A2__DDR_A_2		0x254 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A1__DDR_A_1		0x258 0x000 ALT0 0x0
+#define VF610_PAD_DDR_A0__DDR_A_0		0x25c 0x000 ALT0 0x0
+#define VF610_PAD_DDR_BA2__DDR_BA_2		0x260 0x000 ALT0 0x0
+#define VF610_PAD_DDR_BA1__DDR_BA_1		0x264 0x000 ALT0 0x0
+#define VF610_PAD_DDR_BA0__DDR_BA_0		0x268 0x000 ALT0 0x0
+#define VF610_PAD_DDR_CAS__DDR_CAS_B		0x26c 0x000 ALT0 0x0
+#define VF610_PAD_DDR_CKE__DDR_CKE_0		0x270 0x000 ALT0 0x0
+#define VF610_PAD_DDR_CLK__DDR_CLK_0		0x274 0x000 ALT0 0x0
+#define VF610_PAD_DDR_CS__DDR_CS_B_0		0x278 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D15__DDR_D_15		0x27c 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D14__DDR_D_14		0x280 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D13__DDR_D_13		0x284 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D12__DDR_D_12		0x288 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D11__DDR_D_11		0x28c 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D10__DDR_D_10		0x290 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D9__DDR_D_9		0x294 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D8__DDR_D_8		0x298 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D7__DDR_D_7		0x29c 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D6__DDR_D_6		0x2a0 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D5__DDR_D_5		0x2a4 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D4__DDR_D_4		0x2a8 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D3__DDR_D_3		0x2ac 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D2__DDR_D_2		0x2b0 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D1__DDR_D_1		0x2b4 0x000 ALT0 0x0
+#define VF610_PAD_DDR_D0__DDR_D_0		0x2b8 0x000 ALT0 0x0
+#define VF610_PAD_DDR_DQM1__DDR_DQM_1		0x2bc 0x000 ALT0 0x0
+#define VF610_PAD_DDR_DQM0__DDR_DQM_0		0x2c0 0x000 ALT0 0x0
+#define VF610_PAD_DDR_DQS1__DDR_DQS_1		0x2c4 0x000 ALT0 0x0
+#define VF610_PAD_DDR_DQS0__DDR_DQS_0		0x2c8 0x000 ALT0 0x0
+#define VF610_PAD_DDR_RAS__DDR_RAS_B		0x2cc 0x000 ALT0 0x0
+#define VF610_PAD_DDR_WE__DDR_WE_B		0x2d0 0x000 ALT0 0x0
+#define VF610_PAD_DDR_ODT1__DDR_ODT_0		0x2d4 0x000 ALT0 0x0
+#define VF610_PAD_DDR_ODT0__DDR_ODT_1		0x2d8 0x000 ALT0 0x0
+#define VF610_PAD_DDR_DDRBYTE1__DDR_DDRBYTE1	0x2dc 0x000 ALT0 0x0
+#define VF610_PAD_DDR_DDRBYTE2__DDR_DDRBYTE2	0x2e0 0x000 ALT0 0x0
 
 #endif
diff --git a/arch/arm/boot/dts/vf610-twr.dts b/arch/arm/boot/dts/vf610-twr.dts
index dbb5ffcdcec4..6c246d5aa032 100644
--- a/arch/arm/boot/dts/vf610-twr.dts
+++ b/arch/arm/boot/dts/vf610-twr.dts
@@ -169,7 +169,7 @@
 	       VDDA-supply = <&reg_3p3v>;
 	       VDDIO-supply = <&reg_3p3v>;
 	       clocks = <&clks VF610_CLK_SAI2>;
-       };
+	};
 };
 
 &iomuxc {
diff --git a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
index 1f9686c33a84..42ed4a04a12e 100644
--- a/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
+++ b/arch/arm/boot/dts/vf610-zii-dev-rev-b.dts
@@ -288,7 +288,7 @@
 		};
 	};
 
-	spi0 {
+	spi-0 {
 		compatible = "spi-gpio";
 		pinctrl-0 = <&pinctrl_gpio_spi0>;
 		pinctrl-names = "default";
diff --git a/arch/arm/boot/dts/vf610.dtsi b/arch/arm/boot/dts/vf610.dtsi
index 956182d08e74..2fba923821d0 100644
--- a/arch/arm/boot/dts/vf610.dtsi
+++ b/arch/arm/boot/dts/vf610.dtsi
@@ -2,7 +2,6 @@
 //
 // Copyright 2013 Freescale Semiconductor, Inc.
 
-
 #include "vf500.dtsi"
 
 &a5_cpu {
diff --git a/arch/arm/configs/aspeed_g4_defconfig b/arch/arm/configs/aspeed_g4_defconfig
index 67feb060bb19..a5c65d28ca63 100644
--- a/arch/arm/configs/aspeed_g4_defconfig
+++ b/arch/arm/configs/aspeed_g4_defconfig
@@ -7,6 +7,7 @@ CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
 CONFIG_LOG_BUF_SHIFT=16
 CONFIG_CGROUPS=y
+CONFIG_NAMESPACES=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_RD_BZIP2 is not set
 # CONFIG_RD_LZO is not set
@@ -23,12 +24,9 @@ CONFIG_VMSPLIT_2G=y
 CONFIG_AEABI=y
 CONFIG_UACCESS_WITH_MEMCPY=y
 # CONFIG_ATAGS is not set
-CONFIG_ARM_APPENDED_DTB=y
-CONFIG_ARM_ATAG_DTB_COMPAT=y
 CONFIG_KEXEC=y
 CONFIG_JUMP_LABEL=y
 CONFIG_STRICT_KERNEL_RWX=y
-# CONFIG_BLK_DEV_BSG is not set
 # CONFIG_BLK_DEBUG_FS is not set
 # CONFIG_MQ_IOSCHED_DEADLINE is not set
 # CONFIG_MQ_IOSCHED_KYBER is not set
@@ -47,7 +45,11 @@ CONFIG_INET=y
 CONFIG_IP_MULTICAST=y
 CONFIG_SYN_COOKIES=y
 # CONFIG_INET_DIAG is not set
+CONFIG_IPV6_ROUTER_PREF=y
+CONFIG_IPV6_ROUTE_INFO=y
+CONFIG_IPV6_OPTIMISTIC_DAD=y
 # CONFIG_IPV6_SIT is not set
+CONFIG_IPV6_MULTIPLE_TABLES=y
 CONFIG_NETFILTER=y
 # CONFIG_NETFILTER_ADVANCED is not set
 CONFIG_VLAN_8021Q=y
@@ -66,6 +68,7 @@ CONFIG_MTD_UBI=y
 CONFIG_MTD_UBI_FASTMAP=y
 CONFIG_MTD_UBI_BLOCK=y
 CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_NBD=y
 CONFIG_EEPROM_AT24=y
 CONFIG_NETDEVICES=y
 CONFIG_NETCONSOLE=y
@@ -145,6 +148,7 @@ CONFIG_SENSORS_ASPEED=y
 CONFIG_SENSORS_IIO_HWMON=y
 CONFIG_SENSORS_LM75=y
 CONFIG_SENSORS_NCT7904=y
+CONFIG_SENSORS_OCC_P8_I2C=y
 CONFIG_PMBUS=y
 CONFIG_SENSORS_ADM1275=y
 CONFIG_SENSORS_IBM_CFFPS=y
diff --git a/arch/arm/configs/aspeed_g5_defconfig b/arch/arm/configs/aspeed_g5_defconfig
index 247ab72b2590..c7c11cbaa39d 100644
--- a/arch/arm/configs/aspeed_g5_defconfig
+++ b/arch/arm/configs/aspeed_g5_defconfig
@@ -7,6 +7,7 @@ CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
 CONFIG_LOG_BUF_SHIFT=16
 CONFIG_CGROUPS=y
+CONFIG_NAMESPACES=y
 CONFIG_BLK_DEV_INITRD=y
 # CONFIG_RD_BZIP2 is not set
 # CONFIG_RD_LZO is not set
@@ -33,7 +34,6 @@ CONFIG_VFP=y
 CONFIG_NEON=y
 CONFIG_KERNEL_MODE_NEON=y
 CONFIG_JUMP_LABEL=y
-# CONFIG_BLK_DEV_BSG is not set
 # CONFIG_BLK_DEBUG_FS is not set
 # CONFIG_MQ_IOSCHED_DEADLINE is not set
 # CONFIG_MQ_IOSCHED_KYBER is not set
@@ -65,6 +65,7 @@ CONFIG_NETFILTER=y
 # CONFIG_NETFILTER_ADVANCED is not set
 CONFIG_VLAN_8021Q=y
 CONFIG_NET_NCSI=y
+CONFIG_MCTP=y
 # CONFIG_WIRELESS is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
@@ -82,6 +83,8 @@ CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_NBD=y
 CONFIG_EEPROM_AT24=y
 CONFIG_EEPROM_AT25=y
+CONFIG_SCSI=y
+CONFIG_BLK_DEV_SD=y
 CONFIG_NETDEVICES=y
 CONFIG_NETCONSOLE=y
 # CONFIG_NET_VENDOR_ALACRITECH is not set
@@ -119,6 +122,8 @@ CONFIG_FTGMAC100=y
 # CONFIG_NET_VENDOR_WIZNET is not set
 CONFIG_BROADCOM_PHY=y
 CONFIG_REALTEK_PHY=y
+CONFIG_MCTP_SERIAL=y
+CONFIG_MCTP_TRANSPORT_I2C=y
 # CONFIG_USB_NET_DRIVERS is not set
 # CONFIG_WLAN is not set
 CONFIG_INPUT_EVDEV=y
@@ -146,6 +151,8 @@ CONFIG_IPMI_KCS_BMC_CDEV_IPMI=y
 CONFIG_IPMI_KCS_BMC_SERIO=y
 CONFIG_ASPEED_BT_IPMI_BMC=y
 CONFIG_HW_RANDOM_TIMERIOMEM=y
+CONFIG_TCG_TPM=y
+CONFIG_TCG_TIS_I2C=y
 # CONFIG_I2C_COMPAT is not set
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_MUX_GPIO=y
@@ -180,6 +187,7 @@ CONFIG_SENSORS_IR38064=y
 CONFIG_SENSORS_ISL68137=y
 CONFIG_SENSORS_LM25066=y
 CONFIG_SENSORS_MAX31785=y
+CONFIG_SENSORS_MP5023=y
 CONFIG_SENSORS_UCD9000=y
 CONFIG_SENSORS_UCD9200=y
 CONFIG_SENSORS_SBTSI=y
@@ -200,6 +208,9 @@ CONFIG_USB_DYNAMIC_MINORS=y
 CONFIG_USB_EHCI_HCD=y
 CONFIG_USB_EHCI_ROOT_HUB_TT=y
 CONFIG_USB_EHCI_HCD_PLATFORM=y
+CONFIG_USB_STORAGE=y
+CONFIG_USB_SERIAL=y
+CONFIG_USB_SERIAL_PL2303=y
 CONFIG_USB_GADGET=y
 CONFIG_USB_ASPEED_VHUB=y
 CONFIG_USB_CONFIGFS=y
@@ -243,9 +254,13 @@ CONFIG_FSI_MASTER_ASPEED=y
 CONFIG_FSI_SCOM=y
 CONFIG_FSI_SBEFIFO=y
 CONFIG_FSI_OCC=y
+CONFIG_PECI=y
+CONFIG_PECI_CPU=y
+CONFIG_PECI_ASPEED=y
 CONFIG_EXT4_FS=y
 CONFIG_FANOTIFY=y
 CONFIG_OVERLAY_FS=y
+CONFIG_VFAT_FS=y
 CONFIG_TMPFS=y
 CONFIG_JFFS2_FS=y
 # CONFIG_JFFS2_FS_WRITEBUFFER is not set
@@ -261,6 +276,9 @@ CONFIG_PSTORE_PMSG=y
 CONFIG_PSTORE_FTRACE=y
 CONFIG_PSTORE_RAM=y
 # CONFIG_NETWORK_FILESYSTEMS is not set
+CONFIG_NLS_CODEPAGE_437=y
+CONFIG_NLS_ISO8859_1=y
+# CONFIG_SECURITYFS is not set
 CONFIG_HARDENED_USERCOPY=y
 CONFIG_FORTIFY_SOURCE=y
 CONFIG_CRYPTO_HMAC=y
diff --git a/arch/arm/configs/assabet_defconfig b/arch/arm/configs/assabet_defconfig
index 801383e4135d..8ba8eb7a4adf 100644
--- a/arch/arm/configs/assabet_defconfig
+++ b/arch/arm/configs/assabet_defconfig
@@ -1,6 +1,8 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_ASSABET=y
 CONFIG_CMDLINE="mem=32M console=ttySA0,38400n8 initrd=0xc0800000,3M root=/dev/ram"
diff --git a/arch/arm/configs/at91_dt_defconfig b/arch/arm/configs/at91_dt_defconfig
index e185eee18856..9ea08337b174 100644
--- a/arch/arm/configs/at91_dt_defconfig
+++ b/arch/arm/configs/at91_dt_defconfig
@@ -40,7 +40,6 @@ CONFIG_IP_PNP_RARP=y
 # CONFIG_INET_DIAG is not set
 CONFIG_IPV6_SIT_6RD=y
 CONFIG_CAN=y
-CONFIG_CAN_AT91=y
 CONFIG_CFG80211=y
 CONFIG_MAC80211=y
 CONFIG_DEVTMPFS=y
@@ -81,6 +80,7 @@ CONFIG_DM9000=y
 # CONFIG_NET_VENDOR_STMICRO is not set
 CONFIG_DAVICOM_PHY=y
 CONFIG_MICREL_PHY=y
+CONFIG_CAN_AT91=y
 CONFIG_LIBERTAS=m
 CONFIG_LIBERTAS_SDIO=m
 CONFIG_LIBERTAS_SPI=m
diff --git a/arch/arm/configs/badge4_defconfig b/arch/arm/configs/badge4_defconfig
index 506f3378da07..337e5c9718ae 100644
--- a/arch/arm/configs/badge4_defconfig
+++ b/arch/arm/configs/badge4_defconfig
@@ -1,12 +1,13 @@
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_EXPERT=y
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_BADGE4=y
 CONFIG_UNUSED_BOARD_FILES=y
 CONFIG_CMDLINE="init=/linuxrc root=/dev/mtdblock3"
 CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=m
 CONFIG_MODULES=y
 CONFIG_MODVERSIONS=y
 CONFIG_PARTITION_ADVANCED=y
diff --git a/arch/arm/configs/bcm2835_defconfig b/arch/arm/configs/bcm2835_defconfig
index c4d2e2334b6e..a51babd178c2 100644
--- a/arch/arm/configs/bcm2835_defconfig
+++ b/arch/arm/configs/bcm2835_defconfig
@@ -1,6 +1,6 @@
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT_VOLUNTARY=y
 CONFIG_BSD_PROCESS_ACCT=y
diff --git a/arch/arm/configs/cerfcube_defconfig b/arch/arm/configs/cerfcube_defconfig
index 7e6f7dfa3023..9ada868e2648 100644
--- a/arch/arm/configs/cerfcube_defconfig
+++ b/arch/arm/configs/cerfcube_defconfig
@@ -1,6 +1,8 @@
 CONFIG_SYSVIPC=y
 CONFIG_BSD_PROCESS_ACCT=y
 CONFIG_LOG_BUF_SHIFT=14
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_CERF=y
 CONFIG_SA1100_CERF_FLASH_16MB=y
diff --git a/arch/arm/configs/cm_x300_defconfig b/arch/arm/configs/cm_x300_defconfig
index 31f41159bef2..95144e380b4b 100644
--- a/arch/arm/configs/cm_x300_defconfig
+++ b/arch/arm/configs/cm_x300_defconfig
@@ -1,7 +1,7 @@
 CONFIG_LOCALVERSION="-cm-x300"
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
 CONFIG_LOG_BUF_SHIFT=18
diff --git a/arch/arm/configs/collie_defconfig b/arch/arm/configs/collie_defconfig
index d35cc59ce847..2a2d2cb3ce2e 100644
--- a/arch/arm/configs/collie_defconfig
+++ b/arch/arm/configs/collie_defconfig
@@ -5,6 +5,8 @@ CONFIG_BLK_DEV_INITRD=y
 CONFIG_EXPERT=y
 # CONFIG_BASE_FULL is not set
 # CONFIG_EPOLL is not set
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_COLLIE=y
 CONFIG_CMDLINE="noinitrd root=/dev/mtdblock2 rootfstype=jffs2 fbcon=rotate:1"
diff --git a/arch/arm/configs/corgi_defconfig b/arch/arm/configs/corgi_defconfig
index 1f137f74050f..df84640f4f57 100644
--- a/arch/arm/configs/corgi_defconfig
+++ b/arch/arm/configs/corgi_defconfig
@@ -16,7 +16,6 @@ CONFIG_MACH_HUSKY=y
 CONFIG_UNUSED_BOARD_FILES=y
 CONFIG_CMDLINE="console=ttyS0,115200n8 console=tty1 noinitrd root=/dev/mtdblock2 rootfstype=jffs2   debug"
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=m
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_MODULE_FORCE_UNLOAD=y
diff --git a/arch/arm/configs/davinci_all_defconfig b/arch/arm/configs/davinci_all_defconfig
index fc71a03a9c8c..821d966c95a5 100644
--- a/arch/arm/configs/davinci_all_defconfig
+++ b/arch/arm/configs/davinci_all_defconfig
@@ -1,6 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_IKCONFIG=y
diff --git a/arch/arm/configs/dove_defconfig b/arch/arm/configs/dove_defconfig
index 16ed5c110e8d..ff37f46c82fb 100644
--- a/arch/arm/configs/dove_defconfig
+++ b/arch/arm/configs/dove_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_EXPERT=y
@@ -116,7 +116,6 @@ CONFIG_NLS_ISO8859_1=y
 CONFIG_NLS_ISO8859_2=y
 CONFIG_NLS_UTF8=y
 CONFIG_TIMER_STATS=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_CRYPTO_NULL=y
 CONFIG_CRYPTO_ECB=m
 CONFIG_CRYPTO_PCBC=m
@@ -136,6 +135,7 @@ CONFIG_CRC_CCITT=y
 CONFIG_LIBCRC32C=y
 CONFIG_PRINTK_TIME=y
 # CONFIG_DEBUG_BUGVERBOSE is not set
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig
index 1ce74f46e114..deb24a4bd011 100644
--- a/arch/arm/configs/exynos_defconfig
+++ b/arch/arm/configs/exynos_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_CGROUPS=y
@@ -374,8 +374,8 @@ CONFIG_FONTS=y
 CONFIG_FONT_7x14=y
 CONFIG_PRINTK_TIME=y
 CONFIG_DYNAMIC_DEBUG=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
 CONFIG_SOFTLOCKUP_DETECTOR=y
diff --git a/arch/arm/configs/ezx_defconfig b/arch/arm/configs/ezx_defconfig
index 1a41391d7367..ef7b0a0aee3a 100644
--- a/arch/arm/configs/ezx_defconfig
+++ b/arch/arm/configs/ezx_defconfig
@@ -1,7 +1,7 @@
 CONFIG_LOCALVERSION="-ezx200910312315"
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_LOG_BUF_SHIFT=14
@@ -25,7 +25,6 @@ CONFIG_CPU_FREQ_GOV_ONDEMAND=m
 CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m
 CONFIG_CPU_IDLE=y
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=m
 CONFIG_PM=y
 CONFIG_APM_EMULATION=y
 CONFIG_MODULES=y
diff --git a/arch/arm/configs/footbridge_defconfig b/arch/arm/configs/footbridge_defconfig
index 504070812ad0..87c489337d0e 100644
--- a/arch/arm/configs/footbridge_defconfig
+++ b/arch/arm/configs/footbridge_defconfig
@@ -4,12 +4,13 @@ CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
 CONFIG_EXPERT=y
 # CONFIG_HOTPLUG is not set
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_FOOTBRIDGE=y
 CONFIG_ARCH_EBSA285_HOST=y
 CONFIG_ARCH_NETWINDER=y
 CONFIG_FPE_NWFPE=y
 CONFIG_FPE_NWFPE_XP=y
-CONFIG_BINFMT_AOUT=y
 CONFIG_MODULES=y
 CONFIG_PARTITION_ADVANCED=y
 CONFIG_ACORN_PARTITION=y
diff --git a/arch/arm/configs/h3600_defconfig b/arch/arm/configs/h3600_defconfig
index 5bd1ec539610..4e272875c797 100644
--- a/arch/arm/configs/h3600_defconfig
+++ b/arch/arm/configs/h3600_defconfig
@@ -4,6 +4,8 @@ CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_H3600=y
 # CONFIG_CPU_FREQ_STAT is not set
diff --git a/arch/arm/configs/hackkit_defconfig b/arch/arm/configs/hackkit_defconfig
index b9327b2eacd3..3c91a851fd08 100644
--- a/arch/arm/configs/hackkit_defconfig
+++ b/arch/arm/configs/hackkit_defconfig
@@ -1,13 +1,14 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_HACKKIT=y
 CONFIG_UNUSED_BOARD_FILES=y
 CONFIG_CMDLINE="console=ttySA0,115200 root=/dev/ram0 initrd=0xc0400000,8M init=/rootshell"
 CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=y
 CONFIG_MODULES=y
 CONFIG_NET=y
 CONFIG_PACKET=y
diff --git a/arch/arm/configs/hisi_defconfig b/arch/arm/configs/hisi_defconfig
index 1db5356b1ccd..0376a65e8bc1 100644
--- a/arch/arm/configs/hisi_defconfig
+++ b/arch/arm/configs/hisi_defconfig
@@ -1,4 +1,4 @@
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_BLK_DEV_INITRD=y
diff --git a/arch/arm/configs/imx_v4_v5_defconfig b/arch/arm/configs/imx_v4_v5_defconfig
index bfa2a95638af..711a79e9be00 100644
--- a/arch/arm/configs/imx_v4_v5_defconfig
+++ b/arch/arm/configs/imx_v4_v5_defconfig
@@ -1,6 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_LOG_BUF_SHIFT=14
diff --git a/arch/arm/configs/imx_v6_v7_defconfig b/arch/arm/configs/imx_v6_v7_defconfig
index 01012537a9b9..6429c4106ab5 100644
--- a/arch/arm/configs/imx_v6_v7_defconfig
+++ b/arch/arm/configs/imx_v6_v7_defconfig
@@ -1,6 +1,6 @@
 CONFIG_KERNEL_LZO=y
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BPF_SYSCALL=y
 CONFIG_PREEMPT_VOLUNTARY=y
@@ -65,7 +65,6 @@ CONFIG_IP_PNP=y
 CONFIG_IP_PNP_DHCP=y
 CONFIG_NETFILTER=y
 CONFIG_CAN=y
-CONFIG_CAN_FLEXCAN=y
 CONFIG_BT=y
 CONFIG_BT_BNEP=m
 CONFIG_BT_HCIUART=y
@@ -136,6 +135,7 @@ CONFIG_SMSC911X=y
 # CONFIG_NET_VENDOR_STMICRO is not set
 CONFIG_MICREL_PHY=y
 CONFIG_AT803X_PHY=y
+CONFIG_CAN_FLEXCAN=y
 CONFIG_USB_PEGASUS=m
 CONFIG_USB_RTL8150=m
 CONFIG_USB_RTL8152=y
diff --git a/arch/arm/configs/integrator_defconfig b/arch/arm/configs/integrator_defconfig
index 9ca43c84b452..61711d4bbf74 100644
--- a/arch/arm/configs/integrator_defconfig
+++ b/arch/arm/configs/integrator_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_IKCONFIG=y
diff --git a/arch/arm/configs/iop32x_defconfig b/arch/arm/configs/iop32x_defconfig
index c16e92cdfd00..19e30e790d35 100644
--- a/arch/arm/configs/iop32x_defconfig
+++ b/arch/arm/configs/iop32x_defconfig
@@ -12,7 +12,6 @@ CONFIG_MACH_N2100=y
 CONFIG_UNUSED_BOARD_FILES=y
 CONFIG_CMDLINE="console=ttyS0,115200 root=/dev/nfs ip=bootp cachepolicy=writealloc"
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_PARTITION_ADVANCED=y
diff --git a/arch/arm/configs/ixp4xx_defconfig b/arch/arm/configs/ixp4xx_defconfig
index 6b65ac2a72e7..3cb995b9616a 100644
--- a/arch/arm/configs/ixp4xx_defconfig
+++ b/arch/arm/configs/ixp4xx_defconfig
@@ -99,7 +99,7 @@ CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=8192
 CONFIG_EEPROM_AT24=y
-CONFIG_EEPROM_LEGACY=y
+# CONFIG_EEPROM_LEGACY is not set
 # CONFIG_SCSI_PROC_FS is not set
 CONFIG_BLK_DEV_SD=y
 # CONFIG_BLK_DEV_BSG is not set
diff --git a/arch/arm/configs/jornada720_defconfig b/arch/arm/configs/jornada720_defconfig
index 3dcf89d3e1f1..ae1d68da4f2a 100644
--- a/arch/arm/configs/jornada720_defconfig
+++ b/arch/arm/configs/jornada720_defconfig
@@ -1,12 +1,13 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_SYSFS_DEPRECATED_V2=y
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_JORNADA720=y
 CONFIG_SA1100_JORNADA720_SSP=y
 CONFIG_UNUSED_BOARD_FILES=y
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=y
 CONFIG_PM=y
 CONFIG_MODULES=y
 CONFIG_NET=y
diff --git a/arch/arm/configs/keystone_defconfig b/arch/arm/configs/keystone_defconfig
index 68b89b90ca29..4a5b9adbf2a1 100644
--- a/arch/arm/configs/keystone_defconfig
+++ b/arch/arm/configs/keystone_defconfig
@@ -109,8 +109,6 @@ CONFIG_IP6_NF_IPTABLES=m
 CONFIG_IP_SCTP=y
 CONFIG_VLAN_8021Q=y
 CONFIG_CAN=m
-CONFIG_CAN_C_CAN=m
-CONFIG_CAN_C_CAN_PLATFORM=m
 CONFIG_PCI=y
 CONFIG_PCI_MSI=y
 CONFIG_DEVTMPFS=y
@@ -137,6 +135,8 @@ CONFIG_TI_KEYSTONE_NETCP_ETHSS=y
 CONFIG_MARVELL_PHY=y
 CONFIG_MICREL_PHY=y
 CONFIG_DP83867_PHY=y
+CONFIG_CAN_C_CAN=m
+CONFIG_CAN_C_CAN_PLATFORM=m
 CONFIG_INPUT_EVDEV=m
 CONFIG_INPUT_MISC=y
 CONFIG_INPUT_GPIO_DECODER=m
@@ -152,7 +152,6 @@ CONFIG_SPI=y
 CONFIG_SPI_CADENCE_QUADSPI=y
 CONFIG_SPI_DAVINCI=y
 CONFIG_SPI_SPIDEV=y
-CONFIG_PTP_1588_CLOCK=y
 CONFIG_PINCTRL_SINGLE=y
 CONFIG_GPIOLIB=y
 CONFIG_GPIO_SYSFS=y
@@ -224,7 +223,6 @@ CONFIG_NFSD=y
 CONFIG_NFSD_V3_ACL=y
 CONFIG_NLS_CODEPAGE_437=y
 CONFIG_NLS_ISO8859_1=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_CRYPTO_USER=y
 CONFIG_CRYPTO_AUTHENC=y
 CONFIG_CRYPTO_CBC=y
@@ -236,5 +234,6 @@ CONFIG_CRYPTO_USER_API_HASH=y
 CONFIG_CRYPTO_USER_API_SKCIPHER=y
 CONFIG_DMA_CMA=y
 CONFIG_PRINTK_TIME=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_DEBUG_SHIRQ=y
 CONFIG_DEBUG_USER=y
diff --git a/arch/arm/configs/lart_defconfig b/arch/arm/configs/lart_defconfig
index 0c2f19d756c0..916177d07a39 100644
--- a/arch/arm/configs/lart_defconfig
+++ b/arch/arm/configs/lart_defconfig
@@ -1,6 +1,8 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_LART=y
 CONFIG_UNUSED_BOARD_FILES=y
@@ -8,7 +10,6 @@ CONFIG_CMDLINE="console=ttySA0,9600 root=/dev/ram"
 CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
 CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=y
 CONFIG_PM=y
 CONFIG_MODULES=y
 CONFIG_NET=y
diff --git a/arch/arm/configs/lpc18xx_defconfig b/arch/arm/configs/lpc18xx_defconfig
index 142c1700f450..56eae6a0a311 100644
--- a/arch/arm/configs/lpc18xx_defconfig
+++ b/arch/arm/configs/lpc18xx_defconfig
@@ -150,9 +150,9 @@ CONFIG_JFFS2_FS=y
 CONFIG_CRC_ITU_T=y
 CONFIG_CRC7=y
 CONFIG_PRINTK_TIME=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 # CONFIG_ENABLE_MUST_CHECK is not set
 # CONFIG_DEBUG_BUGVERBOSE is not set
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
 # CONFIG_SLUB_DEBUG is not set
diff --git a/arch/arm/configs/lpc32xx_defconfig b/arch/arm/configs/lpc32xx_defconfig
index 8a41fe4e62f1..fabb66a53350 100644
--- a/arch/arm/configs/lpc32xx_defconfig
+++ b/arch/arm/configs/lpc32xx_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_IKCONFIG=y
diff --git a/arch/arm/configs/magician_defconfig b/arch/arm/configs/magician_defconfig
index 9cbb63c69436..5a8776f6aba3 100644
--- a/arch/arm/configs/magician_defconfig
+++ b/arch/arm/configs/magician_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_PREEMPT=y
 CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig
index 58810e98de3d..6f6b5d0918f7 100644
--- a/arch/arm/configs/milbeaut_m10v_defconfig
+++ b/arch/arm/configs/milbeaut_m10v_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_CGROUPS=y
 CONFIG_BLK_DEV_INITRD=y
diff --git a/arch/arm/configs/mmp2_defconfig b/arch/arm/configs/mmp2_defconfig
index 4d39c615117b..7984640e994e 100644
--- a/arch/arm/configs/mmp2_defconfig
+++ b/arch/arm/configs/mmp2_defconfig
@@ -73,10 +73,10 @@ CONFIG_ROOT_NFS=y
 CONFIG_CRC_CCITT=y
 CONFIG_PRINTK_TIME=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
 # CONFIG_DEBUG_PREEMPT is not set
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 # CONFIG_DYNAMIC_DEBUG is not set
 CONFIG_DEBUG_USER=y
 CONFIG_DEBUG_LL=y
diff --git a/arch/arm/configs/moxart_defconfig b/arch/arm/configs/moxart_defconfig
index 082a38a14c12..ea31f116d577 100644
--- a/arch/arm/configs/moxart_defconfig
+++ b/arch/arm/configs/moxart_defconfig
@@ -1,6 +1,6 @@
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_PREEMPT=y
 CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
diff --git a/arch/arm/configs/mps2_defconfig b/arch/arm/configs/mps2_defconfig
index 700568474549..3ed73f184d83 100644
--- a/arch/arm/configs/mps2_defconfig
+++ b/arch/arm/configs/mps2_defconfig
@@ -94,9 +94,9 @@ CONFIG_NFS_V4_2=y
 CONFIG_ROOT_NFS=y
 CONFIG_NLS=y
 CONFIG_PRINTK_TIME=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 # CONFIG_ENABLE_MUST_CHECK is not set
 # CONFIG_DEBUG_BUGVERBOSE is not set
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_DEBUG_FS=y
 # CONFIG_SLUB_DEBUG is not set
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/arm/configs/multi_v4t_defconfig b/arch/arm/configs/multi_v4t_defconfig
index 6c3e45b71ab5..e2fd822f741a 100644
--- a/arch/arm/configs/multi_v4t_defconfig
+++ b/arch/arm/configs/multi_v4t_defconfig
@@ -71,8 +71,6 @@ CONFIG_POWER_RESET_SYSCON_POWEROFF=y
 CONFIG_WATCHDOG=y
 CONFIG_GPIO_WATCHDOG=y
 CONFIG_AT91RM9200_WATCHDOG=y
-CONFIG_REGULATOR=y
-CONFIG_REGULATOR_FIXED_VOLTAGE=y
 CONFIG_REGULATOR_GPIO=y
 CONFIG_FB=y
 CONFIG_FB_CLPS711X=y
diff --git a/arch/arm/configs/multi_v5_defconfig b/arch/arm/configs/multi_v5_defconfig
index e0be0e0023f3..60fc52b95690 100644
--- a/arch/arm/configs/multi_v5_defconfig
+++ b/arch/arm/configs/multi_v5_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_LOG_BUF_SHIFT=19
@@ -149,7 +149,6 @@ CONFIG_SPI_SUN6I=y
 CONFIG_GPIO_ASPEED=m
 CONFIG_GPIO_ASPEED_SGPIO=y
 CONFIG_GPIO_MXC=y
-CONFIG_POWER_RESET=y
 CONFIG_POWER_RESET_GPIO=y
 CONFIG_POWER_RESET_QNAP=y
 CONFIG_SENSORS_ADT7475=y
@@ -290,8 +289,8 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_DEV_MARVELL_CESA=y
 CONFIG_CRC_CCITT=y
 CONFIG_LIBCRC32C=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index 12b35008571f..23003a43ffe9 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_CGROUPS=y
 CONFIG_BLK_DEV_INITRD=y
@@ -167,12 +167,6 @@ CONFIG_NET_DSA=m
 CONFIG_QRTR=m
 CONFIG_QRTR_SMD=m
 CONFIG_CAN=y
-CONFIG_CAN_AT91=m
-CONFIG_CAN_FLEXCAN=m
-CONFIG_CAN_SUN4I=y
-CONFIG_CAN_XILINXCAN=y
-CONFIG_CAN_RCAR=m
-CONFIG_CAN_MCP251X=y
 CONFIG_BT=m
 CONFIG_BT_HCIUART=m
 CONFIG_BT_HCIUART_BCM=y
@@ -260,7 +254,6 @@ CONFIG_SATA_HIGHBANK=y
 CONFIG_SATA_MV=y
 CONFIG_SATA_RCAR=y
 CONFIG_NETDEVICES=y
-CONFIG_NET_VENDOR_ASIX=y
 CONFIG_VIRTIO_NET=y
 CONFIG_B53_SPI_DRIVER=m
 CONFIG_B53_MDIO_DRIVER=m
@@ -300,6 +293,12 @@ CONFIG_MARVELL_PHY=y
 CONFIG_AT803X_PHY=y
 CONFIG_ROCKCHIP_PHY=y
 CONFIG_DP83867_PHY=y
+CONFIG_CAN_AT91=m
+CONFIG_CAN_FLEXCAN=m
+CONFIG_CAN_SUN4I=y
+CONFIG_CAN_XILINXCAN=y
+CONFIG_CAN_RCAR=m
+CONFIG_CAN_MCP251X=y
 CONFIG_MDIO_MSCC_MIIM=m
 CONFIG_USB_PEGASUS=y
 CONFIG_USB_RTL8152=m
@@ -378,8 +377,6 @@ CONFIG_SERIAL_MSM=y
 CONFIG_SERIAL_MSM_CONSOLE=y
 CONFIG_SERIAL_VT8500=y
 CONFIG_SERIAL_VT8500_CONSOLE=y
-CONFIG_SERIAL_OMAP=y
-CONFIG_SERIAL_OMAP_CONSOLE=y
 CONFIG_SERIAL_BCM63XX=y
 CONFIG_SERIAL_BCM63XX_CONSOLE=y
 CONFIG_SERIAL_XILINX_PS_UART=y
@@ -465,7 +462,6 @@ CONFIG_SPI_TEGRA20_SLINK=y
 CONFIG_SPI_XILINX=y
 CONFIG_SPI_SPIDEV=y
 CONFIG_SPMI=y
-CONFIG_PTP_1588_CLOCK=y
 CONFIG_PINCTRL_AS3722=y
 CONFIG_PINCTRL_MICROCHIP_SGPIO=y
 CONFIG_PINCTRL_OCELOT=y
@@ -716,7 +712,6 @@ CONFIG_ROCKCHIP_DW_MIPI_DSI=y
 CONFIG_ROCKCHIP_INNO_HDMI=y
 CONFIG_DRM_ATMEL_HLCDC=m
 CONFIG_DRM_RCAR_DU=m
-CONFIG_DRM_RCAR_LVDS=y
 CONFIG_DRM_SUN4I=m
 CONFIG_DRM_MSM=m
 CONFIG_DRM_FSL_DCU=m
@@ -783,7 +778,6 @@ CONFIG_SND_ATMEL_SOC_I2S=m
 CONFIG_SND_BCM2835_SOC_I2S=m
 CONFIG_SND_IMX_SOC=m
 CONFIG_SND_SOC_FSL_ASOC_CARD=m
-CONFIG_SND_SOC_FSL_SAI=m
 CONFIG_SND_PXA_SOC_SSP=m
 CONFIG_SND_MMP_SOC_SSPA=m
 CONFIG_SND_PXA910_SOC=m
@@ -820,7 +814,7 @@ CONFIG_SND_SOC_TEGRA_TRIMSLICE=m
 CONFIG_SND_SOC_TEGRA_ALC5632=m
 CONFIG_SND_SOC_TEGRA_MAX98090=m
 CONFIG_SND_SOC_DAVINCI_MCASP=m
-CONFIG_SND_SOC_AC97=m
+CONFIG_SND_SOC_AC97_CODEC=m
 CONFIG_SND_SOC_AK4642=m
 CONFIG_SND_SOC_CPCAP=m
 CONFIG_SND_SOC_CS42L51_I2C=m
@@ -875,7 +869,6 @@ CONFIG_USB_GPIO_VBUS=y
 CONFIG_USB_ISP1301=y
 CONFIG_USB_MXS_PHY=y
 CONFIG_USB_GADGET=y
-CONFIG_USB_FSL_USB2=y
 CONFIG_USB_RENESAS_USBHS_UDC=m
 CONFIG_USB_ASPEED_VHUB=m
 CONFIG_USB_CONFIGFS=m
@@ -1051,7 +1044,6 @@ CONFIG_MSM_MMCC_8960=y
 CONFIG_MSM_MMCC_8974=y
 CONFIG_HWSPINLOCK=y
 CONFIG_HWSPINLOCK_QCOM=y
-CONFIG_MICROCHIP_PIT64B=y
 CONFIG_BCM2835_MBOX=y
 CONFIG_QCOM_APCS_IPC=y
 CONFIG_QCOM_IPCC=y
@@ -1071,9 +1063,13 @@ CONFIG_ASPEED_LPC_CTRL=m
 CONFIG_ASPEED_LPC_SNOOP=m
 CONFIG_ASPEED_P2A_CTRL=m
 CONFIG_RASPBERRYPI_POWER=y
+CONFIG_QCOM_COMMAND_DB=m
 CONFIG_QCOM_CPR=y
 CONFIG_QCOM_GSBI=y
+CONFIG_QCOM_OCMEM=m
 CONFIG_QCOM_RMTFS_MEM=m
+CONFIG_QCOM_RPMH=y
+CONFIG_QCOM_RPMHPD=y
 CONFIG_QCOM_RPMPD=y
 CONFIG_QCOM_SMEM=y
 CONFIG_QCOM_SMD_RPM=y
diff --git a/arch/arm/configs/mv78xx0_defconfig b/arch/arm/configs/mv78xx0_defconfig
index a53ccd49f8ff..877c5150a987 100644
--- a/arch/arm/configs/mv78xx0_defconfig
+++ b/arch/arm/configs/mv78xx0_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_LOG_BUF_SHIFT=14
diff --git a/arch/arm/configs/mvebu_v5_defconfig b/arch/arm/configs/mvebu_v5_defconfig
index ef3a33ebc29a..2467afd32146 100644
--- a/arch/arm/configs/mvebu_v5_defconfig
+++ b/arch/arm/configs/mvebu_v5_defconfig
@@ -1,6 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_FHANDLE=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_LOG_BUF_SHIFT=19
@@ -184,13 +184,13 @@ CONFIG_NLS_CODEPAGE_850=y
 CONFIG_NLS_ISO8859_1=y
 CONFIG_NLS_ISO8859_2=y
 CONFIG_NLS_UTF8=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_CRYPTO_CBC=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_DEV_MARVELL_CESA=y
 CONFIG_CRC_CCITT=y
 CONFIG_LIBCRC32C=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/arm/configs/mxs_defconfig b/arch/arm/configs/mxs_defconfig
index 155553ee06f4..feb38a94c1a7 100644
--- a/arch/arm/configs/mxs_defconfig
+++ b/arch/arm/configs/mxs_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT_VOLUNTARY=y
 CONFIG_TASKSTATS=y
@@ -38,7 +38,6 @@ CONFIG_SYN_COOKIES=y
 # CONFIG_INET_DIAG is not set
 # CONFIG_IPV6 is not set
 CONFIG_CAN=m
-CONFIG_CAN_FLEXCAN=m
 # CONFIG_WIRELESS is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
@@ -62,6 +61,7 @@ CONFIG_ICPLUS_PHY=y
 CONFIG_MICREL_PHY=y
 CONFIG_REALTEK_PHY=y
 CONFIG_SMSC_PHY=y
+CONFIG_CAN_FLEXCAN=m
 CONFIG_USB_USBNET=y
 CONFIG_USB_NET_SMSC95XX=y
 # CONFIG_WLAN is not set
@@ -163,8 +163,8 @@ CONFIG_CRC_ITU_T=m
 CONFIG_CRC7=m
 CONFIG_FONTS=y
 CONFIG_PRINTK_TIME=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_FRAME_WARN=2048
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_SOFTLOCKUP_DETECTOR=y
diff --git a/arch/arm/configs/neponset_defconfig b/arch/arm/configs/neponset_defconfig
index 907403529e30..c333406ce5e3 100644
--- a/arch/arm/configs/neponset_defconfig
+++ b/arch/arm/configs/neponset_defconfig
@@ -1,6 +1,8 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_ASSABET=y
 CONFIG_ASSABET_NEPONSET=y
@@ -9,7 +11,6 @@ CONFIG_ZBOOT_ROM_BSS=0xc1000000
 CONFIG_ZBOOT_ROM=y
 CONFIG_CMDLINE="console=ttySA0,38400n8 cpufreq=221200 rw root=/dev/mtdblock2 mtdparts=sa1100:512K(boot),1M(kernel),2560K(initrd),4M(root) load_ramdisk=1 prompt_ramdisk=0 mem=32M noinitrd initrd=0xc0800000,3M"
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=y
 CONFIG_PM=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
diff --git a/arch/arm/configs/netwinder_defconfig b/arch/arm/configs/netwinder_defconfig
index cf7bbcf9d98a..30ff6fbce5a3 100644
--- a/arch/arm/configs/netwinder_defconfig
+++ b/arch/arm/configs/netwinder_defconfig
@@ -1,11 +1,12 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_FOOTBRIDGE=y
 CONFIG_ARCH_NETWINDER=y
 CONFIG_DEPRECATED_PARAM_STRUCT=y
 CONFIG_CMDLINE="root=0x801"
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=y
 CONFIG_PARTITION_ADVANCED=y
 CONFIG_NET=y
 CONFIG_PACKET=y
diff --git a/arch/arm/configs/omap1_defconfig b/arch/arm/configs/omap1_defconfig
index 54a9f50122af..70511fe4b3ec 100644
--- a/arch/arm/configs/omap1_defconfig
+++ b/arch/arm/configs/omap1_defconfig
@@ -1,6 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_BSD_PROCESS_ACCT=y
@@ -218,7 +218,6 @@ CONFIG_NLS_KOI8_R=y
 CONFIG_NLS_UTF8=y
 # CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_DEBUG_KERNEL=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_SECURITY=y
 CONFIG_CRYPTO_ECB=y
 CONFIG_CRYPTO_PCBC=y
@@ -232,6 +231,7 @@ CONFIG_FONT_8x16=y
 CONFIG_FONT_6x11=y
 CONFIG_FONT_MINI_4x6=y
 # CONFIG_DEBUG_BUGVERBOSE is not set
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_SPINLOCK=y
 CONFIG_DEBUG_MUTEXES=y
diff --git a/arch/arm/configs/omap2plus_defconfig b/arch/arm/configs/omap2plus_defconfig
index 99d015cf8919..965853c1c530 100644
--- a/arch/arm/configs/omap2plus_defconfig
+++ b/arch/arm/configs/omap2plus_defconfig
@@ -2,7 +2,7 @@ CONFIG_KERNEL_LZMA=y
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_AUDIT=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BSD_PROCESS_ACCT=y
 CONFIG_IKCONFIG=y
@@ -249,8 +249,6 @@ CONFIG_NET_ACT_POLICE=m
 CONFIG_NET_ACT_GACT=m
 CONFIG_NET_SWITCHDEV=y
 CONFIG_CAN=m
-CONFIG_CAN_C_CAN=m
-CONFIG_CAN_C_CAN_PLATFORM=m
 CONFIG_BT=m
 CONFIG_BT_RFCOMM=m
 CONFIG_BT_RFCOMM_TTY=y
@@ -350,6 +348,8 @@ CONFIG_AT803X_PHY=y
 CONFIG_SMSC_PHY=y
 CONFIG_DP83848_PHY=y
 CONFIG_DP83867_PHY=y
+CONFIG_CAN_C_CAN=m
+CONFIG_CAN_C_CAN_PLATFORM=m
 CONFIG_PPP=m
 CONFIG_PPP_BSDCOMP=m
 CONFIG_PPP_DEFLATE=m
@@ -419,8 +419,6 @@ CONFIG_SERIAL_8250_DETECT_IRQ=y
 CONFIG_SERIAL_8250_RSA=y
 CONFIG_SERIAL_8250_OMAP=y
 CONFIG_SERIAL_OF_PLATFORM=y
-CONFIG_SERIAL_OMAP=y
-CONFIG_SERIAL_OMAP_CONSOLE=y
 CONFIG_SERIAL_DEV_BUS=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_SPI=y
@@ -430,7 +428,6 @@ CONFIG_SPI_TI_QSPI=m
 CONFIG_HSI=m
 CONFIG_OMAP_SSI=m
 CONFIG_SSI_PROTOCOL=m
-CONFIG_PTP_1588_CLOCK=y
 CONFIG_PINCTRL_SINGLE=y
 CONFIG_DEBUG_GPIO=y
 CONFIG_GPIO_SYSFS=y
diff --git a/arch/arm/configs/orion5x_defconfig b/arch/arm/configs/orion5x_defconfig
index 1311d9583fcc..0629b088a584 100644
--- a/arch/arm/configs/orion5x_defconfig
+++ b/arch/arm/configs/orion5x_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_LOG_BUF_SHIFT=14
@@ -138,9 +138,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_DEV_MARVELL_CESA=y
 CONFIG_CRC_T10DIF=y
 # CONFIG_DEBUG_BUGVERBOSE is not set
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 # CONFIG_SLUB_DEBUG is not set
 CONFIG_LATENCYTOP=y
 # CONFIG_FTRACE is not set
diff --git a/arch/arm/configs/oxnas_v6_defconfig b/arch/arm/configs/oxnas_v6_defconfig
index 600f78b363dd..d206c4f04490 100644
--- a/arch/arm/configs/oxnas_v6_defconfig
+++ b/arch/arm/configs/oxnas_v6_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_CGROUPS=y
 CONFIG_BLK_DEV_INITRD=y
diff --git a/arch/arm/configs/pcm027_defconfig b/arch/arm/configs/pcm027_defconfig
index 06bc9a8fef90..a392312a13ce 100644
--- a/arch/arm/configs/pcm027_defconfig
+++ b/arch/arm/configs/pcm027_defconfig
@@ -1,6 +1,6 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_BSD_PROCESS_ACCT=y
diff --git a/arch/arm/configs/pleb_defconfig b/arch/arm/configs/pleb_defconfig
index d87263336cb2..fd2667873273 100644
--- a/arch/arm/configs/pleb_defconfig
+++ b/arch/arm/configs/pleb_defconfig
@@ -4,6 +4,8 @@ CONFIG_BLK_DEV_INITRD=y
 CONFIG_EXPERT=y
 # CONFIG_HOTPLUG is not set
 # CONFIG_SHMEM is not set
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_PLEB=y
 CONFIG_UNUSED_BOARD_FILES=y
diff --git a/arch/arm/configs/pxa168_defconfig b/arch/arm/configs/pxa168_defconfig
index 70d327895ccf..826ebbef2e3c 100644
--- a/arch/arm/configs/pxa168_defconfig
+++ b/arch/arm/configs/pxa168_defconfig
@@ -4,7 +4,7 @@ CONFIG_SYSFS_DEPRECATED_V2=y
 CONFIG_MACH_ASPENITE=y
 CONFIG_MACH_ZYLONITE2=y
 CONFIG_MACH_AVENGERS_LITE=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_AEABI=y
@@ -50,9 +50,9 @@ CONFIG_ROOT_NFS=y
 CONFIG_CRC_CCITT=y
 CONFIG_PRINTK_TIME=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 # CONFIG_DEBUG_PREEMPT is not set
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_DEBUG_USER=y
 CONFIG_DEBUG_LL=y
 # CONFIG_CRYPTO_ANSI_CPRNG is not set
diff --git a/arch/arm/configs/pxa910_defconfig b/arch/arm/configs/pxa910_defconfig
index 5072bde71508..353008de5678 100644
--- a/arch/arm/configs/pxa910_defconfig
+++ b/arch/arm/configs/pxa910_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_LOG_BUF_SHIFT=14
@@ -58,9 +58,9 @@ CONFIG_ROOT_NFS=y
 CONFIG_CRC_CCITT=y
 CONFIG_PRINTK_TIME=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 # CONFIG_DEBUG_PREEMPT is not set
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_DEBUG_USER=y
 CONFIG_DEBUG_LL=y
 CONFIG_DEBUG_MMP_UART2=y
diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig
index 104a45722799..ca6d0049362b 100644
--- a/arch/arm/configs/pxa_defconfig
+++ b/arch/arm/configs/pxa_defconfig
@@ -1,7 +1,7 @@
 CONFIG_SYSVIPC=y
 CONFIG_POSIX_MQUEUE=y
 CONFIG_FHANDLE=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_BSD_PROCESS_ACCT=y
@@ -646,7 +646,6 @@ CONFIG_NLS_ASCII=m
 CONFIG_NLS_ISO8859_1=m
 CONFIG_NLS_ISO8859_15=m
 CONFIG_NLS_UTF8=m
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_TIMER_STATS=y
 CONFIG_SECURITY=y
 CONFIG_CRYPTO_MANAGER=y
@@ -682,6 +681,7 @@ CONFIG_FONT_6x11=y
 CONFIG_FONT_MINI_4x6=y
 CONFIG_PRINTK_TIME=y
 CONFIG_DYNAMIC_DEBUG=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_FRAME_WARN=0
 CONFIG_STRIP_ASM_SYMS=y
 CONFIG_MAGIC_SYSRQ=y
diff --git a/arch/arm/configs/qcom_defconfig b/arch/arm/configs/qcom_defconfig
index 8a59441701a8..0afa62b92de7 100644
--- a/arch/arm/configs/qcom_defconfig
+++ b/arch/arm/configs/qcom_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_IKCONFIG=y
@@ -245,6 +245,7 @@ CONFIG_QCOM_Q6V5_PAS=y
 CONFIG_QCOM_Q6V5_PIL=y
 CONFIG_QCOM_WCNSS_PIL=y
 CONFIG_RPMSG_CHAR=y
+CONFIG_RPMSG_CTRL=y
 CONFIG_RPMSG_QCOM_GLINK_SMEM=y
 CONFIG_RPMSG_QCOM_SMD=y
 CONFIG_QCOM_COMMAND_DB=y
@@ -253,10 +254,14 @@ CONFIG_QCOM_OCMEM=y
 CONFIG_QCOM_PM=y
 CONFIG_QCOM_RPMH=y
 CONFIG_QCOM_RPMHPD=y
+CONFIG_QCOM_RMTFS_MEM=y
+CONFIG_QCOM_RPMPD=y
 CONFIG_QCOM_SMEM=y
 CONFIG_QCOM_SMD_RPM=y
 CONFIG_QCOM_SMP2P=y
 CONFIG_QCOM_SMSM=y
+CONFIG_QCOM_SOCINFO=y
+CONFIG_QCOM_STATS=y
 CONFIG_QCOM_WCNSS_CTRL=y
 CONFIG_EXTCON_QCOM_SPMI_MISC=y
 CONFIG_IIO=y
diff --git a/arch/arm/configs/rpc_defconfig b/arch/arm/configs/rpc_defconfig
index 16d74a1f027a..210974364d61 100644
--- a/arch/arm/configs/rpc_defconfig
+++ b/arch/arm/configs/rpc_defconfig
@@ -4,10 +4,11 @@ CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_RPC=y
 CONFIG_CPU_SA110=y
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=y
 CONFIG_PARTITION_ADVANCED=y
 CONFIG_BSD_DISKLABEL=y
 CONFIG_SLAB=y
diff --git a/arch/arm/configs/s3c6400_defconfig b/arch/arm/configs/s3c6400_defconfig
index 4f04f583c738..93258d5b57ff 100644
--- a/arch/arm/configs/s3c6400_defconfig
+++ b/arch/arm/configs/s3c6400_defconfig
@@ -62,8 +62,8 @@ CONFIG_TMPFS=y
 CONFIG_TMPFS_POSIX_ACL=y
 CONFIG_CRAMFS=y
 CONFIG_ROMFS_FS=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_RT_MUTEXES=y
 CONFIG_DEBUG_SPINLOCK=y
diff --git a/arch/arm/configs/s5pv210_defconfig b/arch/arm/configs/s5pv210_defconfig
index 789e900a8a08..4c1e480b5bbd 100644
--- a/arch/arm/configs/s5pv210_defconfig
+++ b/arch/arm/configs/s5pv210_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_CGROUPS=y
@@ -115,8 +115,8 @@ CONFIG_NLS_ASCII=y
 CONFIG_NLS_ISO8859_1=y
 CONFIG_NLS_UTF8=y
 CONFIG_CRC_CCITT=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 # CONFIG_DEBUG_PREEMPT is not set
 CONFIG_DEBUG_RT_MUTEXES=y
diff --git a/arch/arm/configs/sama5_defconfig b/arch/arm/configs/sama5_defconfig
index 3a6a4851ef26..877bbe7b777e 100644
--- a/arch/arm/configs/sama5_defconfig
+++ b/arch/arm/configs/sama5_defconfig
@@ -43,9 +43,6 @@ CONFIG_BRIDGE_VLAN_FILTERING=y
 CONFIG_NET_DSA=m
 CONFIG_VLAN_8021Q=m
 CONFIG_CAN=y
-CONFIG_CAN_AT91=y
-CONFIG_CAN_M_CAN=y
-CONFIG_CAN_M_CAN_PLATFORM=y
 CONFIG_CFG80211=y
 CONFIG_MAC80211=y
 CONFIG_MAC80211_LEDS=y
@@ -90,6 +87,9 @@ CONFIG_MACB=y
 # CONFIG_NET_VENDOR_STMICRO is not set
 # CONFIG_NET_VENDOR_WIZNET is not set
 CONFIG_MICREL_PHY=y
+CONFIG_CAN_AT91=y
+CONFIG_CAN_M_CAN=y
+CONFIG_CAN_M_CAN_PLATFORM=y
 CONFIG_USB_LAN78XX=m
 CONFIG_LIBERTAS_THINFIRM=m
 CONFIG_LIBERTAS_THINFIRM_USB=m
diff --git a/arch/arm/configs/sama7_defconfig b/arch/arm/configs/sama7_defconfig
index b2311f004618..72af50d9e48a 100644
--- a/arch/arm/configs/sama7_defconfig
+++ b/arch/arm/configs/sama7_defconfig
@@ -59,8 +59,6 @@ CONFIG_BRIDGE_VLAN_FILTERING=y
 CONFIG_NET_DSA=m
 CONFIG_VLAN_8021Q=m
 CONFIG_CAN=y
-CONFIG_CAN_M_CAN=y
-CONFIG_CAN_M_CAN_PLATFORM=y
 CONFIG_BT=y
 CONFIG_BT_RFCOMM=y
 CONFIG_BT_RFCOMM_TTY=y
@@ -107,6 +105,8 @@ CONFIG_BLK_DEV_SD=y
 CONFIG_NETDEVICES=y
 CONFIG_MACB=y
 CONFIG_MICREL_PHY=y
+CONFIG_CAN_M_CAN=y
+CONFIG_CAN_M_CAN_PLATFORM=y
 CONFIG_INPUT_EVDEV=y
 CONFIG_KEYBOARD_GPIO=y
 # CONFIG_INPUT_MOUSE is not set
diff --git a/arch/arm/configs/shannon_defconfig b/arch/arm/configs/shannon_defconfig
index 42252e85ee49..dfcea70b8034 100644
--- a/arch/arm/configs/shannon_defconfig
+++ b/arch/arm/configs/shannon_defconfig
@@ -1,6 +1,8 @@
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_BLK_DEV_INITRD=y
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_SHANNON=y
 CONFIG_UNUSED_BOARD_FILES=y
diff --git a/arch/arm/configs/shmobile_defconfig b/arch/arm/configs/shmobile_defconfig
index a29bebb3742e..452aef74cc5c 100644
--- a/arch/arm/configs/shmobile_defconfig
+++ b/arch/arm/configs/shmobile_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_IKCONFIG=y
 CONFIG_IKCONFIG_PROC=y
 CONFIG_CGROUPS=y
@@ -32,8 +32,8 @@ CONFIG_UNIX=y
 CONFIG_INET=y
 CONFIG_IP_PNP=y
 CONFIG_IP_PNP_DHCP=y
+CONFIG_NET_DSA=y
 CONFIG_CAN=y
-CONFIG_CAN_RCAR=y
 CONFIG_PCI=y
 CONFIG_PCI_MSI=y
 CONFIG_PCI_RCAR_GEN2=y
@@ -52,11 +52,14 @@ CONFIG_BLK_DEV_SD=y
 CONFIG_ATA=y
 CONFIG_SATA_RCAR=y
 CONFIG_NETDEVICES=y
+CONFIG_NET_DSA_RZN1_A5PSW=y
 CONFIG_SH_ETH=y
 CONFIG_RAVB=y
 CONFIG_SMSC911X=y
+CONFIG_STMMAC_ETH=y
 CONFIG_MICREL_PHY=y
 CONFIG_SMSC_PHY=y
+CONFIG_CAN_RCAR=y
 CONFIG_INPUT_EVDEV=y
 CONFIG_KEYBOARD_GPIO=y
 # CONFIG_INPUT_MOUSE is not set
@@ -91,6 +94,7 @@ CONFIG_SPI_SH_MSIOF=y
 CONFIG_SPI_SH_HSPI=y
 CONFIG_PINCTRL_RZA1=y
 CONFIG_PINCTRL_RZA2=y
+CONFIG_PINCTRL_RZN1=y
 CONFIG_GPIO_EM=y
 CONFIG_GPIO_RCAR=y
 CONFIG_GPIO_PCA953X=y
@@ -180,6 +184,8 @@ CONFIG_RTC_DRV_DA9063=y
 CONFIG_RTC_DRV_SH=y
 CONFIG_RTC_DRV_RZN1=y
 CONFIG_DMADEVICES=y
+CONFIG_DW_DMAC=y
+CONFIG_RZN1_DMAMUX=y
 CONFIG_RCAR_DMAC=y
 CONFIG_RENESAS_USB_DMAC=y
 CONFIG_STAGING=y
diff --git a/arch/arm/configs/simpad_defconfig b/arch/arm/configs/simpad_defconfig
index cc451728f6d9..4e00a4c2c287 100644
--- a/arch/arm/configs/simpad_defconfig
+++ b/arch/arm/configs/simpad_defconfig
@@ -5,6 +5,8 @@ CONFIG_LOG_BUF_SHIFT=14
 CONFIG_EXPERT=y
 CONFIG_KALLSYMS_ALL=y
 CONFIG_KALLSYMS_EXTRA_PASS=y
+CONFIG_ARCH_MULTI_V4=y
+# CONFIG_ARCH_MULTI_V7 is not set
 CONFIG_ARCH_SA1100=y
 CONFIG_SA1100_SIMPAD=y
 CONFIG_UNUSED_BOARD_FILES=y
diff --git a/arch/arm/configs/socfpga_defconfig b/arch/arm/configs/socfpga_defconfig
index d91ae3f0d698..70739e09d0f4 100644
--- a/arch/arm/configs/socfpga_defconfig
+++ b/arch/arm/configs/socfpga_defconfig
@@ -33,9 +33,6 @@ CONFIG_NETWORK_PHY_TIMESTAMPING=y
 CONFIG_VLAN_8021Q=y
 CONFIG_VLAN_8021Q_GVRP=y
 CONFIG_CAN=y
-CONFIG_CAN_C_CAN=y
-CONFIG_CAN_C_CAN_PLATFORM=y
-CONFIG_CAN_DEBUG_DEVICES=y
 CONFIG_PCI=y
 CONFIG_PCI_MSI=y
 CONFIG_PCIE_ALTERA=y
@@ -70,6 +67,9 @@ CONFIG_IXGBE=m
 CONFIG_STMMAC_ETH=y
 CONFIG_MARVELL_PHY=y
 CONFIG_MICREL_PHY=y
+CONFIG_CAN_C_CAN=y
+CONFIG_CAN_C_CAN_PLATFORM=y
+CONFIG_CAN_DEBUG_DEVICES=y
 CONFIG_INPUT_EVDEV=y
 CONFIG_INPUT_TOUCHSCREEN=y
 CONFIG_TOUCHSCREEN_STMPE=y
diff --git a/arch/arm/configs/sp7021_defconfig b/arch/arm/configs/sp7021_defconfig
index 703b9aaa40f0..aa7dfd670db5 100644
--- a/arch/arm/configs/sp7021_defconfig
+++ b/arch/arm/configs/sp7021_defconfig
@@ -13,7 +13,6 @@ CONFIG_LOG_BUF_SHIFT=14
 # CONFIG_RD_LZ4 is not set
 CONFIG_CC_OPTIMIZE_FOR_SIZE=y
 CONFIG_PERF_EVENTS=y
-CONFIG_SLAB=y
 CONFIG_ARCH_SUNPLUS=y
 # CONFIG_VDSO is not set
 CONFIG_SMP=y
@@ -25,6 +24,7 @@ CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_MODVERSIONS=y
 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
+CONFIG_SLAB=y
 CONFIG_UEVENT_HELPER=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
 CONFIG_DEVTMPFS=y
diff --git a/arch/arm/configs/spear13xx_defconfig b/arch/arm/configs/spear13xx_defconfig
index 0227dd566c28..bfde0c86cdc5 100644
--- a/arch/arm/configs/spear13xx_defconfig
+++ b/arch/arm/configs/spear13xx_defconfig
@@ -98,7 +98,7 @@ CONFIG_NLS_DEFAULT="utf8"
 CONFIG_NLS_CODEPAGE_437=y
 CONFIG_NLS_ASCII=m
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_SPINLOCK=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
diff --git a/arch/arm/configs/spear3xx_defconfig b/arch/arm/configs/spear3xx_defconfig
index 254d970a4011..a96ed5cf778e 100644
--- a/arch/arm/configs/spear3xx_defconfig
+++ b/arch/arm/configs/spear3xx_defconfig
@@ -78,7 +78,7 @@ CONFIG_NLS_DEFAULT="utf8"
 CONFIG_NLS_CODEPAGE_437=y
 CONFIG_NLS_ASCII=m
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_SPINLOCK=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
diff --git a/arch/arm/configs/spear6xx_defconfig b/arch/arm/configs/spear6xx_defconfig
index 2809c4eb77e7..3d631b1f3cfa 100644
--- a/arch/arm/configs/spear6xx_defconfig
+++ b/arch/arm/configs/spear6xx_defconfig
@@ -67,7 +67,7 @@ CONFIG_NLS_DEFAULT="utf8"
 CONFIG_NLS_CODEPAGE_437=y
 CONFIG_NLS_ASCII=m
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DEBUG_FS=y
 CONFIG_DEBUG_SPINLOCK=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
diff --git a/arch/arm/configs/spitz_defconfig b/arch/arm/configs/spitz_defconfig
index 1284a1d92ca3..66d74653f3fb 100644
--- a/arch/arm/configs/spitz_defconfig
+++ b/arch/arm/configs/spitz_defconfig
@@ -13,7 +13,6 @@ CONFIG_MACH_AKITA=y
 CONFIG_MACH_BORZOI=y
 CONFIG_CMDLINE="console=ttyS0,115200n8 console=tty1 noinitrd root=/dev/mtdblock2 rootfstype=jffs2   debug"
 CONFIG_FPE_NWFPE=y
-CONFIG_BINFMT_AOUT=m
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_MODULE_FORCE_UNLOAD=y
diff --git a/arch/arm/configs/stm32_defconfig b/arch/arm/configs/stm32_defconfig
index 1f5446cda8b6..dc1a32f50b7e 100644
--- a/arch/arm/configs/stm32_defconfig
+++ b/arch/arm/configs/stm32_defconfig
@@ -74,9 +74,9 @@ CONFIG_NLS=y
 CONFIG_CRC_ITU_T=y
 CONFIG_CRC7=y
 CONFIG_PRINTK_TIME=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 # CONFIG_ENABLE_MUST_CHECK is not set
 # CONFIG_DEBUG_BUGVERBOSE is not set
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 # CONFIG_SLUB_DEBUG is not set
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/arm/configs/sunxi_defconfig b/arch/arm/configs/sunxi_defconfig
index 3d14827e0a31..bddc82f78942 100644
--- a/arch/arm/configs/sunxi_defconfig
+++ b/arch/arm/configs/sunxi_defconfig
@@ -1,4 +1,4 @@
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_CGROUPS=y
 CONFIG_BLK_DEV_INITRD=y
@@ -26,7 +26,6 @@ CONFIG_IP_PNP_BOOTP=y
 # CONFIG_INET_DIAG is not set
 # CONFIG_IPV6 is not set
 CONFIG_CAN=y
-CONFIG_CAN_SUN4I=y
 # CONFIG_WIRELESS is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
@@ -52,6 +51,7 @@ CONFIG_STMMAC_ETH=y
 # CONFIG_NET_VENDOR_WIZNET is not set
 CONFIG_MICREL_PHY=y
 CONFIG_REALTEK_PHY=y
+CONFIG_CAN_SUN4I=y
 # CONFIG_WLAN is not set
 CONFIG_INPUT_EVDEV=y
 CONFIG_KEYBOARD_SUN4I_LRADC=y
diff --git a/arch/arm/configs/tegra_defconfig b/arch/arm/configs/tegra_defconfig
index 71400af6cef4..f32047e24b63 100644
--- a/arch/arm/configs/tegra_defconfig
+++ b/arch/arm/configs/tegra_defconfig
@@ -1,5 +1,5 @@
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_PREEMPT=y
 CONFIG_IKCONFIG=y
@@ -54,7 +54,6 @@ CONFIG_IPV6_MIP6=y
 CONFIG_IPV6_TUNNEL=y
 CONFIG_IPV6_MULTIPLE_TABLES=y
 CONFIG_CAN=y
-CONFIG_CAN_MCP251X=y
 CONFIG_BT=y
 CONFIG_BT_RFCOMM=y
 CONFIG_BT_BNEP=y
@@ -99,6 +98,7 @@ CONFIG_NETDEVICES=y
 CONFIG_DUMMY=y
 CONFIG_IGB=y
 CONFIG_R8169=y
+CONFIG_CAN_MCP251X=y
 CONFIG_USB_PEGASUS=y
 CONFIG_USB_USBNET=y
 CONFIG_USB_NET_SMSC75XX=y
diff --git a/arch/arm/configs/vexpress_defconfig b/arch/arm/configs/vexpress_defconfig
index 4e3a0133e4d3..ac3fd7523698 100644
--- a/arch/arm/configs/vexpress_defconfig
+++ b/arch/arm/configs/vexpress_defconfig
@@ -133,8 +133,8 @@ CONFIG_NLS_CODEPAGE_437=y
 CONFIG_NLS_ISO8859_1=y
 # CONFIG_CRYPTO_HW is not set
 CONFIG_DMA_CMA=y
-CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_DEBUG_KERNEL=y
+CONFIG_DEBUG_INFO_DWARF_TOOLCHAIN_DEFAULT=y
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_DETECT_HUNG_TASK=y
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/arm/configs/vt8500_v6_v7_defconfig b/arch/arm/configs/vt8500_v6_v7_defconfig
index cb8d38e9562a..41607a84abc8 100644
--- a/arch/arm/configs/vt8500_v6_v7_defconfig
+++ b/arch/arm/configs/vt8500_v6_v7_defconfig
@@ -1,4 +1,4 @@
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BLK_DEV_INITRD=y
 CONFIG_ARCH_MULTI_V6=y
diff --git a/arch/arm/configs/xcep_defconfig b/arch/arm/configs/xcep_defconfig
index 0453948d52ef..ea59e4b6bfc5 100644
--- a/arch/arm/configs/xcep_defconfig
+++ b/arch/arm/configs/xcep_defconfig
@@ -1,7 +1,7 @@
 CONFIG_LOCALVERSION=".xcep-itech"
 # CONFIG_LOCALVERSION_AUTO is not set
 CONFIG_SYSVIPC=y
-CONFIG_NO_HZ=y
+CONFIG_NO_HZ_IDLE=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BSD_PROCESS_ACCT=y
 CONFIG_IKCONFIG=y
diff --git a/arch/arm/include/asm/dma-direct.h b/arch/arm/include/asm/dma-direct.h
deleted file mode 100644
index 4f7bcde03abb..000000000000
--- a/arch/arm/include/asm/dma-direct.h
+++ /dev/null
@@ -1 +0,0 @@
-#include <mach/dma-direct.h>
diff --git a/arch/arm/include/asm/dma.h b/arch/arm/include/asm/dma.h
index 05f29a72150b..c6aded1b069c 100644
--- a/arch/arm/include/asm/dma.h
+++ b/arch/arm/include/asm/dma.h
@@ -106,7 +106,7 @@ extern void set_dma_sg(unsigned int chan, struct scatterlist *sg, int nr_sg);
  */
 extern void __set_dma_addr(unsigned int chan, void *addr);
 #define set_dma_addr(chan, addr)				\
-	__set_dma_addr(chan, (void *)__bus_to_virt(addr))
+	__set_dma_addr(chan, (void *)isa_bus_to_virt(addr))
 
 /* Set the DMA byte count for this channel
  *
diff --git a/arch/arm/include/asm/fpstate.h b/arch/arm/include/asm/fpstate.h
index ca42fd9ae0b3..e29d9c7a5238 100644
--- a/arch/arm/include/asm/fpstate.h
+++ b/arch/arm/include/asm/fpstate.h
@@ -46,9 +46,6 @@ union vfp_state {
 	struct vfp_hard_struct	hard;
 };
 
-extern void vfp_flush_thread(union vfp_state *);
-extern void vfp_release_thread(union vfp_state *);
-
 #define FP_HARD_SIZE 35
 
 struct fp_hard_struct {
diff --git a/arch/arm/include/asm/hardware/dec21285.h b/arch/arm/include/asm/hardware/dec21285.h
index 3f18a56a025d..894f2a635cbb 100644
--- a/arch/arm/include/asm/hardware/dec21285.h
+++ b/arch/arm/include/asm/hardware/dec21285.h
@@ -22,6 +22,13 @@
 #define DC21285_IO(x)		(x)
 #endif
 
+/*
+ * The footbridge is programmed to expose the system RAM at 0xe0000000.
+ * The requirement is that the RAM isn't placed at bus address 0, which
+ * would clash with VGA cards.
+ */
+#define BUS_OFFSET 0xe0000000
+
 #define CSR_PCICMD		DC21285_IO(0x0004)
 #define CSR_CLASSREV		DC21285_IO(0x0008)
 #define CSR_PCICACHELINESIZE	DC21285_IO(0x000c)
@@ -81,19 +88,6 @@
 #define SA110_CNTL_XCSDIR(x)		((x)<<28)
 #define SA110_CNTL_PCICFN		(1 << 31)
 
-/*
- * footbridge_cfn_mode() is used when we want
- * to check whether we are the central function
- */
-#define __footbridge_cfn_mode() (*CSR_SA110_CNTL & SA110_CNTL_PCICFN)
-#if defined(CONFIG_FOOTBRIDGE_HOST) && defined(CONFIG_FOOTBRIDGE_ADDIN)
-#define footbridge_cfn_mode() __footbridge_cfn_mode()
-#elif defined(CONFIG_FOOTBRIDGE_HOST)
-#define footbridge_cfn_mode() (1)
-#else
-#define footbridge_cfn_mode() (0)
-#endif
-
 #define CSR_PCIADDR_EXTN	DC21285_IO(0x0140)
 #define CSR_PREFETCHMEMRANGE	DC21285_IO(0x0144)
 #define CSR_XBUS_CYCLE		DC21285_IO(0x0148)
diff --git a/arch/arm/include/asm/mach/dma.h b/arch/arm/include/asm/mach/dma.h
index 1506422af383..5ec11d7f0d04 100644
--- a/arch/arm/include/asm/mach/dma.h
+++ b/arch/arm/include/asm/mach/dma.h
@@ -44,8 +44,3 @@ struct dma_struct {
  * isa_dma_add - add an ISA-style DMA channel
  */
 extern int isa_dma_add(unsigned int, dma_t *dma);
-
-/*
- * Add the ISA DMA controller.  Always takes channels 0-7.
- */
-extern void isa_init_dma(void);
diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h
index a55a9038abc8..d8eef4bd8c71 100644
--- a/arch/arm/include/asm/memory.h
+++ b/arch/arm/include/asm/memory.h
@@ -370,17 +370,6 @@ static inline unsigned long __virt_to_idmap(unsigned long x)
 #define virt_to_idmap(x)	__virt_to_idmap((unsigned long)(x))
 
 /*
- * Virtual <-> DMA view memory address translations
- * Again, these are *only* valid on the kernel direct mapped RAM
- * memory.  Use of these is *deprecated* (and that doesn't mean
- * use the __ prefixed forms instead.)  See dma-mapping.h.
- */
-#ifndef __virt_to_bus
-#define __virt_to_bus	__virt_to_phys
-#define __bus_to_virt	__phys_to_virt
-#endif
-
-/*
  * Conversion between a struct page and a physical address.
  *
  *  page_to_pfn(page)	convert a struct page * to a PFN number
diff --git a/arch/arm/include/asm/stacktrace.h b/arch/arm/include/asm/stacktrace.h
index 3e78f921b8b2..36b2ff44fcbb 100644
--- a/arch/arm/include/asm/stacktrace.h
+++ b/arch/arm/include/asm/stacktrace.h
@@ -21,6 +21,9 @@ struct stackframe {
 	struct llist_node *kr_cur;
 	struct task_struct *tsk;
 #endif
+#ifdef CONFIG_UNWINDER_FRAME_POINTER
+	bool ex_frame;
+#endif
 };
 
 static __always_inline
@@ -34,6 +37,9 @@ void arm_get_current_stackframe(struct pt_regs *regs, struct stackframe *frame)
 		frame->kr_cur = NULL;
 		frame->tsk = current;
 #endif
+#ifdef CONFIG_UNWINDER_FRAME_POINTER
+		frame->ex_frame = in_entry_text(frame->pc);
+#endif
 }
 
 extern int unwind_frame(struct stackframe *frame);
@@ -41,5 +47,7 @@ extern void walk_stackframe(struct stackframe *frame,
 			    int (*fn)(struct stackframe *, void *), void *data);
 extern void dump_mem(const char *lvl, const char *str, unsigned long bottom,
 		     unsigned long top);
+extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
+			   const char *loglvl);
 
 #endif	/* __ASM_STACKTRACE_H */
diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile
index 553866751e1a..95034d32213c 100644
--- a/arch/arm/kernel/Makefile
+++ b/arch/arm/kernel/Makefile
@@ -45,7 +45,6 @@ obj-$(CONFIG_ISA_DMA_API)	+= dma.o
 obj-$(CONFIG_FIQ)		+= fiq.o fiqasm.o
 obj-$(CONFIG_MODULES)		+= armksyms.o module.o
 obj-$(CONFIG_ARM_MODULE_PLTS)	+= module-plts.o
-obj-$(CONFIG_ISA_DMA)		+= dma-isa.o
 obj-$(CONFIG_PCI)		+= bios32.o isa.o
 obj-$(CONFIG_ARM_CPU_SUSPEND)	+= sleep.o suspend.o
 obj-$(CONFIG_HIBERNATION)	+= hibernate.o
diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c
index 02839d8b6202..264827281113 100644
--- a/arch/arm/kernel/devtree.c
+++ b/arch/arm/kernel/devtree.c
@@ -194,14 +194,12 @@ const struct machine_desc * __init setup_machine_fdt(void *dt_virt)
 {
 	const struct machine_desc *mdesc, *mdesc_best = NULL;
 
-#if defined(CONFIG_ARCH_MULTIPLATFORM) || defined(CONFIG_ARM_SINGLE_ARMV7M)
 	DT_MACHINE_START(GENERIC_DT, "Generic DT based system")
 		.l2c_aux_val = 0x0,
 		.l2c_aux_mask = ~0x0,
 	MACHINE_END
 
 	mdesc_best = &__mach_desc_GENERIC_DT;
-#endif
 
 	if (!dt_virt || !early_init_dt_verify(dt_virt))
 		return NULL;
diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index 3d9cace63884..96f3fbd51764 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -201,7 +201,7 @@ void __show_regs(struct pt_regs *regs)
 void show_regs(struct pt_regs * regs)
 {
 	__show_regs(regs);
-	dump_stack();
+	dump_backtrace(regs, NULL, KERN_DEFAULT);
 }
 
 ATOMIC_NOTIFIER_HEAD(thread_notify_head);
diff --git a/arch/arm/kernel/return_address.c b/arch/arm/kernel/return_address.c
index 8aac1e10b117..38f1ea9c724d 100644
--- a/arch/arm/kernel/return_address.c
+++ b/arch/arm/kernel/return_address.c
@@ -47,6 +47,7 @@ here:
 	frame.kr_cur = NULL;
 	frame.tsk = current;
 #endif
+	frame.ex_frame = false;
 
 	walk_stackframe(&frame, save_return_addr, &data);
 
diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c
index d0fa2037460a..85443b5d1922 100644
--- a/arch/arm/kernel/stacktrace.c
+++ b/arch/arm/kernel/stacktrace.c
@@ -9,6 +9,8 @@
 #include <asm/stacktrace.h>
 #include <asm/traps.h>
 
+#include "reboot.h"
+
 #if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND)
 /*
  * Unwind the current stack frame and store the new register values in the
@@ -39,29 +41,74 @@
  * Note that with framepointer enabled, even the leaf functions have the same
  * prologue and epilogue, therefore we can ignore the LR value in this case.
  */
-int notrace unwind_frame(struct stackframe *frame)
+
+extern unsigned long call_with_stack_end;
+
+static int frame_pointer_check(struct stackframe *frame)
 {
 	unsigned long high, low;
 	unsigned long fp = frame->fp;
+	unsigned long pc = frame->pc;
+
+	/*
+	 * call_with_stack() is the only place we allow SP to jump from one
+	 * stack to another, with FP and SP pointing to different stacks,
+	 * skipping the FP boundary check at this point.
+	 */
+	if (pc >= (unsigned long)&call_with_stack &&
+			pc < (unsigned long)&call_with_stack_end)
+		return 0;
 
 	/* only go to a higher address on the stack */
 	low = frame->sp;
 	high = ALIGN(low, THREAD_SIZE);
 
-#ifdef CONFIG_CC_IS_CLANG
 	/* check current frame pointer is within bounds */
+#ifdef CONFIG_CC_IS_CLANG
 	if (fp < low + 4 || fp > high - 4)
 		return -EINVAL;
-
-	frame->sp = frame->fp;
-	frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp));
-	frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 4));
 #else
-	/* check current frame pointer is within bounds */
 	if (fp < low + 12 || fp > high - 4)
 		return -EINVAL;
+#endif
+
+	return 0;
+}
+
+int notrace unwind_frame(struct stackframe *frame)
+{
+	unsigned long fp = frame->fp;
+
+	if (frame_pointer_check(frame))
+		return -EINVAL;
+
+	/*
+	 * When we unwind through an exception stack, include the saved PC
+	 * value into the stack trace.
+	 */
+	if (frame->ex_frame) {
+		struct pt_regs *regs = (struct pt_regs *)frame->sp;
+
+		/*
+		 * We check that 'regs + sizeof(struct pt_regs)' (that is,
+		 * &regs[1]) does not exceed the bottom of the stack to avoid
+		 * accessing data outside the task's stack. This may happen
+		 * when frame->ex_frame is a false positive.
+		 */
+		if ((unsigned long)&regs[1] > ALIGN(frame->sp, THREAD_SIZE))
+			return -EINVAL;
+
+		frame->pc = regs->ARM_pc;
+		frame->ex_frame = false;
+		return 0;
+	}
 
 	/* restore the registers from the stack frame */
+#ifdef CONFIG_CC_IS_CLANG
+	frame->sp = frame->fp;
+	frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp));
+	frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp + 4));
+#else
 	frame->fp = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 12));
 	frame->sp = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 8));
 	frame->pc = READ_ONCE_NOCHECK(*(unsigned long *)(fp - 4));
@@ -72,6 +119,9 @@ int notrace unwind_frame(struct stackframe *frame)
 					(void *)frame->fp, &frame->kr_cur);
 #endif
 
+	if (in_entry_text(frame->pc))
+		frame->ex_frame = true;
+
 	return 0;
 }
 #endif
@@ -102,7 +152,6 @@ static int save_trace(struct stackframe *frame, void *d)
 {
 	struct stack_trace_data *data = d;
 	struct stack_trace *trace = data->trace;
-	struct pt_regs *regs;
 	unsigned long addr = frame->pc;
 
 	if (data->no_sched_functions && in_sched_functions(addr))
@@ -113,19 +162,6 @@ static int save_trace(struct stackframe *frame, void *d)
 	}
 
 	trace->entries[trace->nr_entries++] = addr;
-
-	if (trace->nr_entries >= trace->max_entries)
-		return 1;
-
-	if (!in_entry_text(frame->pc))
-		return 0;
-
-	regs = (struct pt_regs *)frame->sp;
-	if ((unsigned long)&regs[1] > ALIGN(frame->sp, THREAD_SIZE))
-		return 0;
-
-	trace->entries[trace->nr_entries++] = regs->ARM_pc;
-
 	return trace->nr_entries >= trace->max_entries;
 }
 
@@ -167,6 +203,9 @@ here:
 	frame.kr_cur = NULL;
 	frame.tsk = tsk;
 #endif
+#ifdef CONFIG_UNWINDER_FRAME_POINTER
+	frame.ex_frame = false;
+#endif
 
 	walk_stackframe(&frame, save_trace, &data);
 }
@@ -188,6 +227,9 @@ void save_stack_trace_regs(struct pt_regs *regs, struct stack_trace *trace)
 	frame.kr_cur = NULL;
 	frame.tsk = current;
 #endif
+#ifdef CONFIG_UNWINDER_FRAME_POINTER
+	frame.ex_frame = in_entry_text(frame.pc);
+#endif
 
 	walk_stackframe(&frame, save_trace, &data);
 }
diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c
index 9283dc65be31..20b2db6dcd1c 100644
--- a/arch/arm/kernel/traps.c
+++ b/arch/arm/kernel/traps.c
@@ -205,14 +205,14 @@ static void dump_instr(const char *lvl, struct pt_regs *regs)
 }
 
 #ifdef CONFIG_ARM_UNWIND
-static inline void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
-				  const char *loglvl)
+void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
+		    const char *loglvl)
 {
 	unwind_backtrace(regs, tsk, loglvl);
 }
 #else
-static void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
-			   const char *loglvl)
+void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
+		    const char *loglvl)
 {
 	unsigned int fp, mode;
 	int ok = 1;
@@ -487,7 +487,7 @@ asmlinkage void do_undefinstr(struct pt_regs *regs)
 die_sig:
 #ifdef CONFIG_DEBUG_USER
 	if (user_debug & UDBG_UNDEFINED) {
-		pr_info("%s (%d): undefined instruction: pc=%p\n",
+		pr_info("%s (%d): undefined instruction: pc=%px\n",
 			current->comm, task_pid_nr(current), pc);
 		__show_regs(regs);
 		dump_instr(KERN_INFO, regs);
@@ -920,9 +920,9 @@ asmlinkage void handle_bad_stack(struct pt_regs *regs)
 {
 	unsigned long tsk_stk = (unsigned long)current->stack;
 #ifdef CONFIG_IRQSTACKS
-	unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
+	unsigned long irq_stk = (unsigned long)raw_cpu_read(irq_stack_ptr);
 #endif
-	unsigned long ovf_stk = (unsigned long)this_cpu_read(overflow_stack_ptr);
+	unsigned long ovf_stk = (unsigned long)raw_cpu_read(overflow_stack_ptr);
 
 	console_verbose();
 	pr_emerg("Insufficient stack space to handle exception!");
diff --git a/arch/arm/lib/call_with_stack.S b/arch/arm/lib/call_with_stack.S
index 0a268a6c513c..5030d4e8d126 100644
--- a/arch/arm/lib/call_with_stack.S
+++ b/arch/arm/lib/call_with_stack.S
@@ -46,4 +46,6 @@ UNWIND( .setfp	fpreg, sp	)
 	pop	{fpreg, pc}
 UNWIND( .fnend			)
 #endif
+	.globl call_with_stack_end
+call_with_stack_end:
 ENDPROC(call_with_stack)
diff --git a/arch/arm/mach-aspeed/Kconfig b/arch/arm/mach-aspeed/Kconfig
index cd8a15be0724..080019aa6fcd 100644
--- a/arch/arm/mach-aspeed/Kconfig
+++ b/arch/arm/mach-aspeed/Kconfig
@@ -19,9 +19,9 @@ config MACH_ASPEED_G4
 	select PINCTRL_ASPEED_G4
 	select FTTMR010_TIMER
 	help
-	 Say yes if you intend to run on an Aspeed ast2400 or similar
-	 fourth generation BMCs, such as those used by OpenPower Power8
-	 systems.
+	  Say yes if you intend to run on an Aspeed ast2400 or similar
+	  fourth generation BMCs, such as those used by OpenPower Power8
+	  systems.
 
 config MACH_ASPEED_G5
 	bool "Aspeed SoC 5th Generation"
@@ -29,8 +29,8 @@ config MACH_ASPEED_G5
 	select PINCTRL_ASPEED_G5
 	select FTTMR010_TIMER
 	help
-	 Say yes if you intend to run on an Aspeed ast2500 or similar
-	 fifth generation Aspeed BMCs.
+	  Say yes if you intend to run on an Aspeed ast2500 or similar
+	  fifth generation Aspeed BMCs.
 
 config MACH_ASPEED_G6
 	bool "Aspeed SoC 6th Generation"
@@ -40,7 +40,7 @@ config MACH_ASPEED_G6
 	select ARM_GIC
 	select HAVE_ARM_ARCH_TIMER
 	help
-	 Say yes if you intend to run on an Aspeed ast2600 or similar
-	 sixth generation Aspeed BMCs.
+	  Say yes if you intend to run on an Aspeed ast2600 or similar
+	  sixth generation Aspeed BMCs.
 
 endif
diff --git a/arch/arm/mach-at91/Makefile.boot b/arch/arm/mach-at91/Makefile.boot
deleted file mode 100644
index 5dde7328a7a9..000000000000
--- a/arch/arm/mach-at91/Makefile.boot
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-# Empty file waiting for deletion once Makefile.boot isn't needed any more.
-# Patch waits for application at
-# https://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=7889/1 .
diff --git a/arch/arm/mach-davinci/Makefile.boot b/arch/arm/mach-davinci/Makefile.boot
deleted file mode 100644
index d36b251f325b..000000000000
--- a/arch/arm/mach-davinci/Makefile.boot
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-zreladdr-$(CONFIG_ARCH_DAVINCI_DA8XX)		+= 0xc0008000
-params_phys-$(CONFIG_ARCH_DAVINCI_DA8XX)	:= 0xc0000100
-initrd_phys-$(CONFIG_ARCH_DAVINCI_DA8XX)	:= 0xc0800000
-
-zreladdr-$(CONFIG_ARCH_DAVINCI_DMx)		+= 0x80008000
-params_phys-$(CONFIG_ARCH_DAVINCI_DMx)		:= 0x80000100
-initrd_phys-$(CONFIG_ARCH_DAVINCI_DMx)		:= 0x80800000
diff --git a/arch/arm/mach-davinci/board-da850-evm.c b/arch/arm/mach-davinci/board-da850-evm.c
index 92d74bc71967..d752ee2b30ff 100644
--- a/arch/arm/mach-davinci/board-da850-evm.c
+++ b/arch/arm/mach-davinci/board-da850-evm.c
@@ -516,8 +516,8 @@ exp_setup_sela_fail:
 	return ret;
 }
 
-static int da850_evm_ui_expander_teardown(struct i2c_client *client,
-					unsigned gpio, unsigned ngpio, void *c)
+static void da850_evm_ui_expander_teardown(struct i2c_client *client,
+					   unsigned gpio, unsigned ngpio, void *c)
 {
 	platform_device_unregister(&da850_evm_ui_keys_device);
 
@@ -529,8 +529,6 @@ static int da850_evm_ui_expander_teardown(struct i2c_client *client,
 	gpio_free(gpio + DA850_EVM_UI_EXP_SEL_C);
 	gpio_free(gpio + DA850_EVM_UI_EXP_SEL_B);
 	gpio_free(gpio + DA850_EVM_UI_EXP_SEL_A);
-
-	return 0;
 }
 
 /* assign the baseboard expander's GPIOs after the UI board's */
@@ -697,13 +695,11 @@ io_exp_setup_sw_fail:
 	return ret;
 }
 
-static int da850_evm_bb_expander_teardown(struct i2c_client *client,
-					unsigned gpio, unsigned ngpio, void *c)
+static void da850_evm_bb_expander_teardown(struct i2c_client *client,
+					   unsigned gpio, unsigned ngpio, void *c)
 {
 	platform_device_unregister(&da850_evm_bb_leds_device);
 	platform_device_unregister(&da850_evm_bb_keys_device);
-
-	return 0;
 }
 
 static struct pca953x_platform_data da850_evm_ui_expander_info = {
diff --git a/arch/arm/mach-dove/Makefile b/arch/arm/mach-dove/Makefile
index e83f6492834d..da373a5768ba 100644
--- a/arch/arm/mach-dove/Makefile
+++ b/arch/arm/mach-dove/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/arch/arm/plat-orion/include
+ccflags-y := -I$(srctree)/arch/arm/plat-orion/include
 
 obj-y				+= common.o
 obj-$(CONFIG_DOVE_LEGACY)	+= irq.o mpp.o
diff --git a/arch/arm/mach-dove/Makefile.boot b/arch/arm/mach-dove/Makefile.boot
deleted file mode 100644
index e4dd1d26038f..000000000000
--- a/arch/arm/mach-dove/Makefile.boot
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-   zreladdr-y	+= 0x00008000
-params_phys-y	:= 0x00000100
-initrd_phys-y	:= 0x00800000
diff --git a/arch/arm/mach-ep93xx/Makefile.boot b/arch/arm/mach-ep93xx/Makefile.boot
deleted file mode 100644
index 4c0a039a5027..000000000000
--- a/arch/arm/mach-ep93xx/Makefile.boot
+++ /dev/null
@@ -1,2 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-# Empty file waiting for deletion once Makefile.boot isn't needed any more.
diff --git a/arch/arm/mach-footbridge/Kconfig b/arch/arm/mach-footbridge/Kconfig
index acc10b1caa69..b5e7cbfed119 100644
--- a/arch/arm/mach-footbridge/Kconfig
+++ b/arch/arm/mach-footbridge/Kconfig
@@ -1,38 +1,37 @@
 # SPDX-License-Identifier: GPL-2.0-only
-if ARCH_FOOTBRIDGE
+menuconfig ARCH_FOOTBRIDGE
+	bool "FootBridge Implementations"
+	depends on ARCH_MULTI_V4 && !(ARCH_MULTI_V4T || ARCH_MULTI_V5)
+	depends on !(ARCH_MOXART || ARCH_GEMINI || ARCH_SA1100)
+	depends on ATAGS
+	depends on CPU_LITTLE_ENDIAN
+	depends on MMU
+	select ARCH_NO_SG_CHAIN
+	select CPU_SA110
+	select FOOTBRIDGE
+	select NEED_MACH_MEMORY_H
+	help
+	  Support for systems based on the DC21285 companion chip
+	  ("FootBridge"), such as the Simtec CATS and the Rebel NetWinder.
 
-menu "Footbridge Implementations"
+if ARCH_FOOTBRIDGE
 
 config ARCH_CATS
 	bool "CATS"
 	depends on UNUSED_BOARD_FILES
 	select CLKEVT_I8253
 	select CLKSRC_I8253
-	select FOOTBRIDGE_HOST
 	select ISA
-	select ISA_DMA
 	select FORCE_PCI
 	help
 	  Say Y here if you intend to run this kernel on the CATS.
 
 	  Saying N will reduce the size of the Footbridge kernel.
 
-config ARCH_EBSA285_ADDIN
-	bool "EBSA285 (addin mode)"
-	select ARCH_EBSA285
-	select FOOTBRIDGE_ADDIN
-	help
-	  Say Y here if you intend to run this kernel on the EBSA285 card
-	  in addin mode.
-
-	  Saying N will reduce the size of the Footbridge kernel.
-
 config ARCH_EBSA285_HOST
 	bool "EBSA285 (host mode)"
 	select ARCH_EBSA285
-	select FOOTBRIDGE_HOST
 	select ISA
-	select ISA_DMA
 	select ARCH_MAY_HAVE_PC_FDC
 	select FORCE_PCI
 	help
@@ -45,9 +44,7 @@ config ARCH_NETWINDER
 	bool "NetWinder"
 	select CLKEVT_I8253
 	select CLKSRC_I8253
-	select FOOTBRIDGE_HOST
 	select ISA
-	select ISA_DMA
 	select FORCE_PCI
 	help
 	  Say Y here if you intend to run this kernel on the Rebel.COM
@@ -57,23 +54,12 @@ config ARCH_NETWINDER
 
 	  Saying N will reduce the size of the Footbridge kernel.
 
-endmenu
-
 # Footbridge support
 config FOOTBRIDGE
-	select ARCH_HAS_PHYS_TO_DMA
-	bool
-
-# Footbridge in host mode
-config FOOTBRIDGE_HOST
-	bool
+	def_bool y
 	select ARCH_MIGHT_HAVE_PC_SERIO
+	select ISA_DMA_API
 
-# Footbridge in addin mode
-config FOOTBRIDGE_ADDIN
-	bool
-
-# EBSA285 board in either host or addin mode
 config ARCH_EBSA285
 	bool
 
diff --git a/arch/arm/mach-footbridge/Makefile b/arch/arm/mach-footbridge/Makefile
index 6262993c0555..55d570739f19 100644
--- a/arch/arm/mach-footbridge/Makefile
+++ b/arch/arm/mach-footbridge/Makefile
@@ -5,11 +5,11 @@
 
 # Object file lists.
 
-obj-y			:= common.o dma.o isa-irq.o
+obj-y			:= common.o isa-irq.o isa.o isa-rtc.o dma-isa.o
 
 pci-y			+= dc21285.o
 pci-$(CONFIG_ARCH_CATS) += cats-pci.o
-pci-$(CONFIG_ARCH_EBSA285_HOST) += ebsa285-pci.o
+pci-$(CONFIG_ARCH_EBSA285) += ebsa285-pci.o
 pci-$(CONFIG_ARCH_NETWINDER) += netwinder-pci.o
 
 obj-$(CONFIG_ARCH_CATS) += cats-hw.o isa-timer.o
@@ -18,4 +18,3 @@ obj-$(CONFIG_ARCH_NETWINDER) += netwinder-hw.o isa-timer.o
 
 obj-$(CONFIG_PCI)	+=$(pci-y)
 
-obj-$(CONFIG_ISA)	+= isa.o isa-rtc.o
diff --git a/arch/arm/mach-footbridge/Makefile.boot b/arch/arm/mach-footbridge/Makefile.boot
deleted file mode 100644
index e4313e912cac..000000000000
--- a/arch/arm/mach-footbridge/Makefile.boot
+++ /dev/null
@@ -1,5 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-   zreladdr-y	+= 0x00008000
-params_phys-y	:= 0x00000100
-initrd_phys-y	:= 0x00800000
-
diff --git a/arch/arm/mach-footbridge/common.c b/arch/arm/mach-footbridge/common.c
index 5020eb96b025..629e4676ed77 100644
--- a/arch/arm/mach-footbridge/common.c
+++ b/arch/arm/mach-footbridge/common.c
@@ -198,9 +198,6 @@ void __init footbridge_init_irq(void)
 
 	__fb_init_irq();
 
-	if (!footbridge_cfn_mode())
-		return;
-
 	if (machine_is_ebsa285())
 		/* The following is dependent on which slot
 		 * you plug the Southbridge card into.  We
@@ -221,21 +218,13 @@ void __init footbridge_init_irq(void)
  * commented out since there is a "No Fix" problem with it.  Not mapping
  * it means that we have extra bullet protection on our feet.
  */
-static struct map_desc fb_common_io_desc[] __initdata = {
+static struct map_desc ebsa285_host_io_desc[] __initdata = {
 	{
 		.virtual	= ARMCSR_BASE,
 		.pfn		= __phys_to_pfn(DC21285_ARMCSR_BASE),
 		.length		= ARMCSR_SIZE,
 		.type		= MT_DEVICE,
-	}
-};
-
-/*
- * The mapping when the footbridge is in host mode.  We don't map any of
- * this when we are in add-in mode.
- */
-static struct map_desc ebsa285_host_io_desc[] __initdata = {
-#if defined(CONFIG_ARCH_FOOTBRIDGE) && defined(CONFIG_FOOTBRIDGE_HOST)
+	},
 	{
 		.virtual	= PCIMEM_BASE,
 		.pfn		= __phys_to_pfn(DC21285_PCI_MEM),
@@ -257,26 +246,12 @@ static struct map_desc ebsa285_host_io_desc[] __initdata = {
 		.length		= PCIIACK_SIZE,
 		.type		= MT_DEVICE,
 	},
-#endif
 };
 
 void __init footbridge_map_io(void)
 {
-	/*
-	 * Set up the common mapping first; we need this to
-	 * determine whether we're in host mode or not.
-	 */
-	iotable_init(fb_common_io_desc, ARRAY_SIZE(fb_common_io_desc));
-
-	/*
-	 * Now, work out what we've got to map in addition on this
-	 * platform.
-	 */
-	if (footbridge_cfn_mode()) {
-		iotable_init(ebsa285_host_io_desc, ARRAY_SIZE(ebsa285_host_io_desc));
-		pci_map_io_early(__phys_to_pfn(DC21285_PCI_IO));
-	}
-
+	iotable_init(ebsa285_host_io_desc, ARRAY_SIZE(ebsa285_host_io_desc));
+	pci_map_io_early(__phys_to_pfn(DC21285_PCI_IO));
 	vga_base = PCIMEM_BASE;
 }
 
@@ -306,49 +281,3 @@ void footbridge_restart(enum reboot_mode mode, const char *cmd)
 		*CSR_SA110_CNTL |= (1 << 13);
 	}
 }
-
-#ifdef CONFIG_FOOTBRIDGE_ADDIN
-
-static inline unsigned long fb_bus_sdram_offset(void)
-{
-	return *CSR_PCISDRAMBASE & 0xfffffff0;
-}
-
-/*
- * These two functions convert virtual addresses to PCI addresses and PCI
- * addresses to virtual addresses.  Note that it is only legal to use these
- * on memory obtained via get_zeroed_page or kmalloc.
- */
-unsigned long __virt_to_bus(unsigned long res)
-{
-	WARN_ON(res < PAGE_OFFSET || res >= (unsigned long)high_memory);
-
-	return res + (fb_bus_sdram_offset() - PAGE_OFFSET);
-}
-EXPORT_SYMBOL(__virt_to_bus);
-
-unsigned long __bus_to_virt(unsigned long res)
-{
-	res = res - (fb_bus_sdram_offset() - PAGE_OFFSET);
-
-	WARN_ON(res < PAGE_OFFSET || res >= (unsigned long)high_memory);
-
-	return res;
-}
-EXPORT_SYMBOL(__bus_to_virt);
-#else
-static inline unsigned long fb_bus_sdram_offset(void)
-{
-	return BUS_OFFSET;
-}
-#endif /* CONFIG_FOOTBRIDGE_ADDIN */
-
-dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	return paddr + (fb_bus_sdram_offset() - PHYS_OFFSET);
-}
-
-phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr)
-{
-	return dev_addr - (fb_bus_sdram_offset() - PHYS_OFFSET);
-}
diff --git a/arch/arm/mach-footbridge/dc21285.c b/arch/arm/mach-footbridge/dc21285.c
index f9713dc561cf..f8920d0010de 100644
--- a/arch/arm/mach-footbridge/dc21285.c
+++ b/arch/arm/mach-footbridge/dc21285.c
@@ -5,6 +5,7 @@
  *  Copyright (C) 1998-2001 Russell King
  *  Copyright (C) 1998-2000 Phil Blundell
  */
+#include <linux/dma-map-ops.h>
 #include <linux/kernel.h>
 #include <linux/pci.h>
 #include <linux/interrupt.h>
@@ -241,13 +242,26 @@ static irqreturn_t dc21285_parity_irq(int irq, void *dev_id)
 	return IRQ_HANDLED;
 }
 
+static int dc21285_pci_bus_notifier(struct notifier_block *nb,
+				    unsigned long action,
+				    void *data)
+{
+	if (action != BUS_NOTIFY_ADD_DEVICE)
+		return NOTIFY_DONE;
+
+	dma_direct_set_offset(data, PHYS_OFFSET, BUS_OFFSET, SZ_256M);
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block dc21285_pci_bus_nb = {
+	.notifier_call = dc21285_pci_bus_notifier,
+};
+
 int __init dc21285_setup(int nr, struct pci_sys_data *sys)
 {
 	struct resource *res;
 
-	if (nr || !footbridge_cfn_mode())
-		return 0;
-
 	res = kcalloc(2, sizeof(struct resource), GFP_KERNEL);
 	if (!res) {
 		printk("out of memory for root bus resources");
@@ -269,6 +283,8 @@ int __init dc21285_setup(int nr, struct pci_sys_data *sys)
 	pci_add_resource_offset(&sys->resources, &res[0], sys->mem_offset);
 	pci_add_resource_offset(&sys->resources, &res[1], sys->mem_offset);
 
+	bus_register_notifier(&pci_bus_type, &dc21285_pci_bus_nb);
+
 	return 1;
 }
 
@@ -278,7 +294,6 @@ int __init dc21285_setup(int nr, struct pci_sys_data *sys)
 void __init dc21285_preinit(void)
 {
 	unsigned int mem_size, mem_mask;
-	int cfn_mode;
 
 	pcibios_min_mem = 0x81000000;
 
@@ -298,21 +313,15 @@ void __init dc21285_preinit(void)
 	*CSR_CSRBASEOFFSET    = 0;
 	*CSR_PCIADDR_EXTN     = 0;
 
-	cfn_mode = __footbridge_cfn_mode();
-
 	printk(KERN_INFO "PCI: DC21285 footbridge, revision %02lX, in "
-		"%s mode\n", *CSR_CLASSREV & 0xff, cfn_mode ?
-		"central function" : "addin");
-
-	if (footbridge_cfn_mode()) {
-		/*
-		 * Clear any existing errors - we aren't
-		 * interested in historical data...
-		 */
-		*CSR_SA110_CNTL	= (*CSR_SA110_CNTL & 0xffffde07) |
-				  SA110_CNTL_RXSERR;
-		*CSR_PCICMD = (*CSR_PCICMD & 0xffff) | PCICMD_ERROR_BITS;
-	}
+		"central function mode\n", *CSR_CLASSREV & 0xff);
+
+	/*
+	 * Clear any existing errors - we aren't
+	 * interested in historical data...
+	 */
+	*CSR_SA110_CNTL	= (*CSR_SA110_CNTL & 0xffffde07) | SA110_CNTL_RXSERR;
+	*CSR_PCICMD = (*CSR_PCICMD & 0xffff) | PCICMD_ERROR_BITS;
 
 	timer_setup(&serr_timer, dc21285_enable_error, 0);
 	timer_setup(&perr_timer, dc21285_enable_error, 0);
@@ -331,29 +340,18 @@ void __init dc21285_preinit(void)
 	dc21285_request_irq(IRQ_PCI_DPERR, dc21285_dparity_irq, 0,
 			    "PCI data parity", NULL);
 
-	if (cfn_mode) {
-		/*
-		 * Map our SDRAM at a known address in PCI space, just in case
-		 * the firmware had other ideas.  Using a nonzero base is
-		 * necessary, since some VGA cards forcefully use PCI addresses
-		 * in the range 0x000a0000 to 0x000c0000. (eg, S3 cards).
-		 */
-		*CSR_PCICSRBASE       = 0xf4000000;
-		*CSR_PCICSRIOBASE     = 0;
-		*CSR_PCISDRAMBASE     = __virt_to_bus(PAGE_OFFSET);
-		*CSR_PCIROMBASE       = 0;
-		*CSR_PCICMD = PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
-			      PCI_COMMAND_INVALIDATE | PCICMD_ERROR_BITS;
-	} else if (footbridge_cfn_mode() != 0) {
-		/*
-		 * If we are not compiled to accept "add-in" mode, then
-		 * we are using a constant virt_to_bus translation which
-		 * can not hope to cater for the way the host BIOS  has
-		 * set up the machine.
-		 */
-		panic("PCI: this kernel is compiled for central "
-			"function mode only");
-	}
+	/*
+	 * Map our SDRAM at a known address in PCI space, just in case
+	 * the firmware had other ideas.  Using a nonzero base is
+	 * necessary, since some VGA cards forcefully use PCI addresses
+	 * in the range 0x000a0000 to 0x000c0000. (eg, S3 cards).
+	 */
+	*CSR_PCICSRBASE       = 0xf4000000;
+	*CSR_PCICSRIOBASE     = 0;
+	*CSR_PCISDRAMBASE     = BUS_OFFSET;
+	*CSR_PCIROMBASE       = 0;
+	*CSR_PCICMD = PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
+		      PCI_COMMAND_INVALIDATE | PCICMD_ERROR_BITS;
 }
 
 void __init dc21285_postinit(void)
diff --git a/arch/arm/kernel/dma-isa.c b/arch/arm/mach-footbridge/dma-isa.c
index 2d90ecce5a11..937f5376d5e7 100644
--- a/arch/arm/kernel/dma-isa.c
+++ b/arch/arm/mach-footbridge/dma-isa.c
@@ -1,7 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- *  linux/arch/arm/kernel/dma-isa.c
- *
  *  Copyright (C) 1999-2000 Russell King
  *
  *  ISA DMA primitives
@@ -13,6 +11,7 @@
  *   arch/arm/kernel/dma-ebsa285.c
  *   Copyright (C) 1998 Phil Blundell
  */
+#include <linux/dma-map-ops.h>
 #include <linux/ioport.h>
 #include <linux/init.h>
 #include <linux/dma-mapping.h>
@@ -20,6 +19,7 @@
 
 #include <asm/dma.h>
 #include <asm/mach/dma.h>
+#include <asm/hardware/dec21285.h>
 
 #define ISA_DMA_MASK		0
 #define ISA_DMA_MODE		1
@@ -157,7 +157,7 @@ static dma_t isa_dma[8];
 /*
  * ISA DMA always starts at channel 0
  */
-void __init isa_init_dma(void)
+static int __init isa_dma_init(void)
 {
 	/*
 	 * Try to autodetect presence of an ISA DMA controller.
@@ -222,4 +222,9 @@ void __init isa_init_dma(void)
 
 		request_dma(DMA_ISA_CASCADE, "cascade");
 	}
+
+	dma_direct_set_offset(&isa_dma_dev, PHYS_OFFSET, BUS_OFFSET, SZ_256M);
+
+	return 0;
 }
+core_initcall(isa_dma_init);
diff --git a/arch/arm/mach-footbridge/dma.c b/arch/arm/mach-footbridge/dma.c
deleted file mode 100644
index 86618074a7a5..000000000000
--- a/arch/arm/mach-footbridge/dma.c
+++ /dev/null
@@ -1,58 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- *  linux/arch/arm/kernel/dma-ebsa285.c
- *
- *  Copyright (C) 1998 Phil Blundell
- *
- * DMA functions specific to EBSA-285/CATS architectures
- *
- *  Changelog:
- *   09-Nov-1998 RMK	Split out ISA DMA functions to dma-isa.c
- *   17-Mar-1999 RMK	Allow any EBSA285-like architecture to have
- *			ISA DMA controllers.
- */
-#include <linux/init.h>
-#include <linux/io.h>
-#include <linux/spinlock.h>
-#include <linux/scatterlist.h>
-
-#include <asm/dma.h>
-
-#include <asm/mach/dma.h>
-#include <asm/hardware/dec21285.h>
-
-#if 0
-static int fb_dma_request(unsigned int chan, dma_t *dma)
-{
-	return -EINVAL;
-}
-
-static void fb_dma_enable(unsigned int chan, dma_t *dma)
-{
-}
-
-static void fb_dma_disable(unsigned int chan, dma_t *dma)
-{
-}
-
-static struct dma_ops fb_dma_ops = {
-	.type		= "fb",
-	.request	= fb_dma_request,
-	.enable		= fb_dma_enable,
-	.disable	= fb_dma_disable,
-};
-#endif
-
-static int __init fb_dma_init(void)
-{
-#if 0
-	dma[_DC21285_DMA(0)].d_ops = &fb_dma_ops;
-	dma[_DC21285_DMA(1)].d_ops = &fb_dma_ops;
-#endif
-#ifdef CONFIG_ISA_DMA
-	if (footbridge_cfn_mode())
-		isa_init_dma();
-#endif
-	return 0;
-}
-core_initcall(fb_dma_init);
diff --git a/arch/arm/mach-footbridge/include/mach/dma-direct.h b/arch/arm/mach-footbridge/include/mach/dma-direct.h
deleted file mode 100644
index 01f9e8367c00..000000000000
--- a/arch/arm/mach-footbridge/include/mach/dma-direct.h
+++ /dev/null
@@ -1,8 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef MACH_FOOTBRIDGE_DMA_DIRECT_H
-#define MACH_FOOTBRIDGE_DMA_DIRECT_H 1
-
-dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr);
-phys_addr_t dma_to_phys(struct device *dev, dma_addr_t dev_addr);
-
-#endif /* MACH_FOOTBRIDGE_DMA_DIRECT_H */
diff --git a/arch/arm/mach-footbridge/include/mach/isa-dma.h b/arch/arm/mach-footbridge/include/mach/isa-dma.h
index 8a1b991076e1..b10731a1f66a 100644
--- a/arch/arm/mach-footbridge/include/mach/isa-dma.h
+++ b/arch/arm/mach-footbridge/include/mach/isa-dma.h
@@ -10,17 +10,9 @@
 #ifndef __ASM_ARCH_DMA_H
 #define __ASM_ARCH_DMA_H
 
-/*
- * The 21285 has two internal DMA channels; we call these 8 and 9.
- * On CATS hardware we have an additional eight ISA dma channels
- * numbered 0..7.
- */
-#define _ISA_DMA(x)		(0+(x))
-#define _DC21285_DMA(x)		(8+(x))
-
-#define MAX_DMA_CHANNELS	10
+#define MAX_DMA_CHANNELS	8
 
-#define DMA_FLOPPY		_ISA_DMA(2)
-#define DMA_ISA_CASCADE		_ISA_DMA(4)
+#define DMA_FLOPPY		(2)
+#define DMA_ISA_CASCADE		(4)
 
 #endif /* _ASM_ARCH_DMA_H */
diff --git a/arch/arm/mach-footbridge/include/mach/memory.h b/arch/arm/mach-footbridge/include/mach/memory.h
index 3a5d2638c18f..9516877667d7 100644
--- a/arch/arm/mach-footbridge/include/mach/memory.h
+++ b/arch/arm/mach-footbridge/include/mach/memory.h
@@ -16,37 +16,6 @@
 #ifndef __ASM_ARCH_MEMORY_H
 #define __ASM_ARCH_MEMORY_H
 
-
-#if defined(CONFIG_FOOTBRIDGE_ADDIN)
-/*
- * If we may be using add-in footbridge mode, then we must
- * use the out-of-line translation that makes use of the
- * PCI BAR
- */
-#ifndef __ASSEMBLY__
-extern unsigned long __virt_to_bus(unsigned long);
-extern unsigned long __bus_to_virt(unsigned long);
-#endif
-#define __virt_to_bus	__virt_to_bus
-#define __bus_to_virt	__bus_to_virt
-
-#elif defined(CONFIG_FOOTBRIDGE_HOST)
-
-/*
- * The footbridge is programmed to expose the system RAM at 0xe0000000.
- * The requirement is that the RAM isn't placed at bus address 0, which
- * would clash with VGA cards.
- */
-#define BUS_OFFSET		0xe0000000
-#define __virt_to_bus(x)	((x) + (BUS_OFFSET - PAGE_OFFSET))
-#define __bus_to_virt(x)	((x) - (BUS_OFFSET - PAGE_OFFSET))
-
-#else
-
-#error "Undefined footbridge mode"
-
-#endif
-
 /*
  * Cache flushing area.
  */
diff --git a/arch/arm/mach-footbridge/isa.c b/arch/arm/mach-footbridge/isa.c
index ec5af521cf95..84caccddce44 100644
--- a/arch/arm/mach-footbridge/isa.c
+++ b/arch/arm/mach-footbridge/isa.c
@@ -79,16 +79,12 @@ static int __init footbridge_isa_init(void)
 {
 	int err = 0;
 
-	if (!footbridge_cfn_mode())
-		return 0;
-
 	/* Personal server doesn't have RTC */
-	if (!machine_is_personal_server()) {
-		isa_rtc_init();
-		err = platform_device_register(&rtc_device);
-		if (err)
-			printk(KERN_ERR "Unable to register RTC device: %d\n", err);
-	}
+	isa_rtc_init();
+	err = platform_device_register(&rtc_device);
+	if (err)
+		printk(KERN_ERR "Unable to register RTC device: %d\n", err);
+
 	err = platform_device_register(&serial_device);
 	if (err)
 		printk(KERN_ERR "Unable to register serial device: %d\n", err);
diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig
index 77e435df8dfe..ab767f059929 100644
--- a/arch/arm/mach-imx/Kconfig
+++ b/arch/arm/mach-imx/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 menuconfig ARCH_MXC
 	bool "Freescale i.MX family"
-	depends on (ARCH_MULTI_V4_V5 && CPU_LITTLE_ENDIAN) || \
+	depends on ((ARCH_MULTI_V4T || ARCH_MULTI_V5) && CPU_LITTLE_ENDIAN) || \
 		ARCH_MULTI_V6_V7 || ARM_SINGLE_ARMV7M
 	select CLKSRC_IMX_GPT
 	select GENERIC_IRQ_CHIP
diff --git a/arch/arm/mach-imx/Makefile.boot b/arch/arm/mach-imx/Makefile.boot
deleted file mode 100644
index e69de29bb2d1..000000000000
--- a/arch/arm/mach-imx/Makefile.boot
+++ /dev/null
diff --git a/arch/arm/mach-iop32x/Makefile.boot b/arch/arm/mach-iop32x/Makefile.boot
deleted file mode 100644
index 5c3af01c4000..000000000000
--- a/arch/arm/mach-iop32x/Makefile.boot
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-   zreladdr-y	+= 0xa0008000
-params_phys-y	:= 0xa0000100
-initrd_phys-y	:= 0xa0800000
diff --git a/arch/arm/mach-lpc18xx/Makefile.boot b/arch/arm/mach-lpc18xx/Makefile.boot
deleted file mode 100644
index cec195d4fcba..000000000000
--- a/arch/arm/mach-lpc18xx/Makefile.boot
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-# Empty file waiting for deletion once Makefile.boot isn't needed any more.
-# Patch waits for application at
-# http://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=7889/1 .
diff --git a/arch/arm/mach-lpc32xx/Makefile.boot b/arch/arm/mach-lpc32xx/Makefile.boot
deleted file mode 100644
index 37d09ddb27f8..000000000000
--- a/arch/arm/mach-lpc32xx/Makefile.boot
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-   zreladdr-y	+= 0x80008000
-params_phys-y	:= 0x80000100
-initrd_phys-y	:= 0x82000000
diff --git a/arch/arm/mach-mv78xx0/Makefile b/arch/arm/mach-mv78xx0/Makefile
index a839e960b8c6..50aff70065f2 100644
--- a/arch/arm/mach-mv78xx0/Makefile
+++ b/arch/arm/mach-mv78xx0/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/arch/arm/plat-orion/include
+ccflags-y := -I$(srctree)/arch/arm/plat-orion/include
 
 obj-y				+= common.o mpp.o irq.o pcie.o
 obj-$(CONFIG_MACH_DB78X00_BP)	+= db78x00-bp-setup.o
diff --git a/arch/arm/mach-mvebu/Makefile b/arch/arm/mach-mvebu/Makefile
index cb106899dd7c..c21733cbb4fa 100644
--- a/arch/arm/mach-mvebu/Makefile
+++ b/arch/arm/mach-mvebu/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/arch/arm/plat-orion/include
+ccflags-y := -I$(srctree)/arch/arm/plat-orion/include
 
 AFLAGS_coherency_ll.o		:= -Wa,-march=armv7-a
 CFLAGS_pmsu.o			:= -march=armv7-a
diff --git a/arch/arm/mach-nspire/Kconfig b/arch/arm/mach-nspire/Kconfig
index eb9916233dea..b7a3871876d7 100644
--- a/arch/arm/mach-nspire/Kconfig
+++ b/arch/arm/mach-nspire/Kconfig
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config ARCH_NSPIRE
 	bool "TI-NSPIRE based"
-	depends on ARCH_MULTI_V4_V5
+	depends on ARCH_MULTI_V4T
 	depends on CPU_LITTLE_ENDIAN
 	select CPU_ARM926T
 	select GENERIC_IRQ_CHIP
diff --git a/arch/arm/mach-omap1/Makefile.boot b/arch/arm/mach-omap1/Makefile.boot
deleted file mode 100644
index 2c771515a606..000000000000
--- a/arch/arm/mach-omap1/Makefile.boot
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-   zreladdr-y		+= 0x10008000
-params_phys-y		:= 0x10000100
-initrd_phys-y		:= 0x10800000
diff --git a/arch/arm/mach-orion5x/Makefile b/arch/arm/mach-orion5x/Makefile
index 1a585a62d5e6..572c3520f7fe 100644
--- a/arch/arm/mach-orion5x/Makefile
+++ b/arch/arm/mach-orion5x/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
-ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/arch/arm/plat-orion/include
+ccflags-y := -I$(srctree)/arch/arm/plat-orion/include
 
 obj-y				+= common.o pci.o irq.o mpp.o
 obj-$(CONFIG_MACH_DB88F5281)	+= db88f5281-setup.o
diff --git a/arch/arm/mach-rpc/Kconfig b/arch/arm/mach-rpc/Kconfig
new file mode 100644
index 000000000000..55f6d829b677
--- /dev/null
+++ b/arch/arm/mach-rpc/Kconfig
@@ -0,0 +1,21 @@
+config ARCH_RPC
+	bool "RiscPC"
+	depends on ARCH_MULTI_V4 && !(ARCH_MULTI_V4T || ARCH_MULTI_V5)
+	depends on !(ARCH_FOOTBRIDGE || ARCH_SA1100 || ARCH_MOXART || ARCH_GEMINI)
+	depends on !CC_IS_CLANG && GCC_VERSION < 90100 && GCC_VERSION >= 60000
+	depends on CPU_LITTLE_ENDIAN
+	depends on ATAGS
+	depends on MMU
+	select ARCH_ACORN
+	select ARCH_MAY_HAVE_PC_FDC
+	select CPU_SA110
+	select FIQ
+	select HAVE_PATA_PLATFORM
+	select ISA_DMA_API
+	select LEGACY_TIMER_TICK
+	select NEED_MACH_IO_H
+	select NEED_MACH_MEMORY_H
+	select NO_IOPORT_MAP
+	help
+	  On the Acorn Risc-PC, Linux can support the internal IDE disk and
+	  CD-ROM interface, serial and parallel port, and the floppy drive.
diff --git a/arch/arm/mach-rpc/Makefile.boot b/arch/arm/mach-rpc/Makefile.boot
deleted file mode 100644
index 0ed8e8fbde99..000000000000
--- a/arch/arm/mach-rpc/Makefile.boot
+++ /dev/null
@@ -1,5 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-   zreladdr-y	+= 0x10008000
-params_phys-y	:= 0x10000100
-initrd_phys-y	:= 0x18000000
-
diff --git a/arch/arm/mach-s3c/Makefile.boot b/arch/arm/mach-s3c/Makefile.boot
deleted file mode 100644
index 7f19e226035e..000000000000
--- a/arch/arm/mach-s3c/Makefile.boot
+++ /dev/null
@@ -1,9 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-
-ifeq ($(CONFIG_PM_H1940),y)
-	zreladdr-y	+= 0x30108000
-	params_phys-y	:= 0x30100100
-else
-	zreladdr-y	+= 0x30008000
-	params_phys-y	:= 0x30000100
-endif
diff --git a/arch/arm/mach-s3c/devs.h b/arch/arm/mach-s3c/devs.h
index 02b0c5750572..991b9b2006a1 100644
--- a/arch/arm/mach-s3c/devs.h
+++ b/arch/arm/mach-s3c/devs.h
@@ -33,8 +33,6 @@ extern struct platform_device s3c64xx_device_onenand1;
 extern struct platform_device s3c64xx_device_pcm0;
 extern struct platform_device s3c64xx_device_pcm1;
 extern struct platform_device s3c64xx_device_spi0;
-extern struct platform_device s3c64xx_device_spi1;
-extern struct platform_device s3c64xx_device_spi2;
 
 extern struct platform_device s3c_device_adc;
 extern struct platform_device s3c_device_cfcon;
diff --git a/arch/arm/mach-sa1100/Kconfig b/arch/arm/mach-sa1100/Kconfig
index 7e0161cb1c1f..fb9cd10705de 100644
--- a/arch/arm/mach-sa1100/Kconfig
+++ b/arch/arm/mach-sa1100/Kconfig
@@ -1,7 +1,25 @@
 # SPDX-License-Identifier: GPL-2.0-only
-if ARCH_SA1100
+menuconfig ARCH_SA1100
+	bool "SA11x0 Implementations"
+	depends on ARCH_MULTI_V4 && !(ARCH_MULTI_V4T || ARCH_MULTI_V5)
+	depends on !(ARCH_MOXART || ARCH_GEMINI)
+	depends on ATAGS
+	depends on CPU_LITTLE_ENDIAN
+	depends on MMU
+	select ARCH_NO_SG_CHAIN
+	select ARCH_MTD_XIP
+	select CLKSRC_MMIO
+	select CLKSRC_PXA
+	select CPU_FREQ
+	select CPU_SA1100
+	select GPIOLIB
+	select IRQ_DOMAIN
+	select ISA
+	select NEED_MACH_MEMORY_H
+	help
+	  Support for StrongARM 11x0 based boards.
 
-menu "SA11x0 Implementations"
+if ARCH_SA1100
 
 config SA1100_ASSABET
 	bool "Assabet"
@@ -179,7 +197,4 @@ config SA1100_SSP
 	  This isn't for audio support, but for attached sensors and
 	  other devices, eg for BadgePAD 4 sensor support.
 
-endmenu
-
 endif
-
diff --git a/arch/arm/mach-sa1100/Makefile.boot b/arch/arm/mach-sa1100/Makefile.boot
deleted file mode 100644
index 9d8246f2cab4..000000000000
--- a/arch/arm/mach-sa1100/Makefile.boot
+++ /dev/null
@@ -1,9 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-ifeq ($(CONFIG_SA1111),y)
-   zreladdr-y	+= 0xc0208000
-else
-   zreladdr-y	+= 0xc0008000
-endif
-params_phys-y	:= 0xc0000100
-initrd_phys-y	:= 0xc0800000
-
diff --git a/arch/arm/mach-shmobile/Kconfig b/arch/arm/mach-shmobile/Kconfig
index 50909c4b95b2..37f862f13c8d 100644
--- a/arch/arm/mach-shmobile/Kconfig
+++ b/arch/arm/mach-shmobile/Kconfig
@@ -6,5 +6,4 @@ menuconfig ARCH_RENESAS
 	select GPIOLIB
 	select NO_IOPORT_MAP
 	select PINCTRL
-	select SOC_BUS
 	select ZONE_DMA if ARM_LPAE
diff --git a/arch/arm/mach-stm32/Makefile.boot b/arch/arm/mach-stm32/Makefile.boot
deleted file mode 100644
index 5dde7328a7a9..000000000000
--- a/arch/arm/mach-stm32/Makefile.boot
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-# Empty file waiting for deletion once Makefile.boot isn't needed any more.
-# Patch waits for application at
-# https://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=7889/1 .
diff --git a/arch/arm/mach-sunplus/Kconfig b/arch/arm/mach-sunplus/Kconfig
index 926cde5e3cd9..d0c2416e6f24 100644
--- a/arch/arm/mach-sunplus/Kconfig
+++ b/arch/arm/mach-sunplus/Kconfig
@@ -18,8 +18,8 @@ config SOC_SP7021
 	select ARM_PSCI
 	select PINCTRL
 	select PINCTRL_SPPCTL
-	select SERIAL_SUNPLUS
-	select SERIAL_SUNPLUS_CONSOLE
+	select SERIAL_SUNPLUS if TTY
+	select SERIAL_SUNPLUS_CONSOLE if TTY
 	help
 	  Support for Sunplus SP7021 SoC. It is based on ARM 4-core
 	  Cortex-A7 with various peripherals (e.g.: I2C, SPI, SDIO,
diff --git a/arch/arm/mach-versatile/Makefile.boot b/arch/arm/mach-versatile/Makefile.boot
deleted file mode 100644
index cec195d4fcba..000000000000
--- a/arch/arm/mach-versatile/Makefile.boot
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-# Empty file waiting for deletion once Makefile.boot isn't needed any more.
-# Patch waits for application at
-# http://www.arm.linux.org.uk/developer/patches/viewpatch.php?id=7889/1 .
diff --git a/arch/arm/mach-vt8500/Makefile.boot b/arch/arm/mach-vt8500/Makefile.boot
deleted file mode 100644
index 883985f4b6c1..000000000000
--- a/arch/arm/mach-vt8500/Makefile.boot
+++ /dev/null
@@ -1,4 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-   zreladdr-y	+= 0x00008000
-params_phys-y	:= 0x00000100
-initrd_phys-y	:= 0x01000000
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index 089c9c644cce..ef691a5720d2 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -307,7 +307,7 @@ void __init dma_contiguous_remap(void)
 
 static int __dma_update_pte(pte_t *pte, unsigned long addr, void *data)
 {
-	struct page *page = virt_to_page(addr);
+	struct page *page = virt_to_page((void *)addr);
 	pgprot_t prot = *(pgprot_t *)data;
 
 	set_pte_ext(pte, mk_pte(page, prot), 0);
diff --git a/arch/arm/mm/dump.c b/arch/arm/mm/dump.c
index fb688003d156..059eb4cdc9c2 100644
--- a/arch/arm/mm/dump.c
+++ b/arch/arm/mm/dump.c
@@ -26,7 +26,7 @@ static struct addr_marker address_markers[] = {
 	{ MODULES_VADDR,	"Modules" },
 	{ PAGE_OFFSET,		"Kernel Mapping" },
 	{ 0,			"vmalloc() Area" },
-	{ VMALLOC_END,		"vmalloc() End" },
+	{ FDT_FIXED_BASE,	"FDT Area" },
 	{ FIXADDR_START,	"Fixmap Area" },
 	{ VECTORS_BASE,	"Vectors" },
 	{ VECTORS_BASE + PAGE_SIZE * 2, "Vectors End" },
@@ -200,6 +200,7 @@ static const struct prot_bits section_bits[] = {
 };
 
 struct pg_level {
+	const char *name;
 	const struct prot_bits *bits;
 	size_t num;
 	u64 mask;
@@ -213,9 +214,11 @@ static struct pg_level pg_level[] = {
 	}, { /* p4d */
 	}, { /* pud */
 	}, { /* pmd */
+		.name	= (CONFIG_PGTABLE_LEVELS > 2) ? "PMD" : "PGD",
 		.bits	= section_bits,
 		.num	= ARRAY_SIZE(section_bits),
 	}, { /* pte */
+		.name	= "PTE",
 		.bits	= pte_bits,
 		.num	= ARRAY_SIZE(pte_bits),
 	},
@@ -282,7 +285,8 @@ static void note_page(struct pg_state *st, unsigned long addr,
 				delta >>= 10;
 				unit++;
 			}
-			pt_dump_seq_printf(st->seq, "%9lu%c", delta, *unit);
+			pt_dump_seq_printf(st->seq, "%9lu%c %s", delta, *unit,
+					   pg_level[st->level].name);
 			if (st->current_domain)
 				pt_dump_seq_printf(st->seq, " %s",
 							st->current_domain);
@@ -346,7 +350,7 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
 		addr = start + i * PMD_SIZE;
 		domain = get_domain_name(pmd);
 		if (pmd_none(*pmd) || pmd_large(*pmd) || !pmd_present(*pmd))
-			note_page(st, addr, 3, pmd_val(*pmd), domain);
+			note_page(st, addr, 4, pmd_val(*pmd), domain);
 		else
 			walk_pte(st, pmd, addr, domain);
 
diff --git a/arch/arm/mm/kasan_init.c b/arch/arm/mm/kasan_init.c
index 29caee9c79ce..46d9f4a622cb 100644
--- a/arch/arm/mm/kasan_init.c
+++ b/arch/arm/mm/kasan_init.c
@@ -268,12 +268,17 @@ void __init kasan_init(void)
 
 	/*
 	 * 1. The module global variables are in MODULES_VADDR ~ MODULES_END,
-	 *    so we need to map this area.
+	 *    so we need to map this area if CONFIG_KASAN_VMALLOC=n. With
+	 *    VMALLOC support KASAN will manage this region dynamically,
+	 *    refer to kasan_populate_vmalloc() and ARM's implementation of
+	 *    module_alloc().
 	 * 2. PKMAP_BASE ~ PKMAP_BASE+PMD_SIZE's shadow and MODULES_VADDR
 	 *    ~ MODULES_END's shadow is in the same PMD_SIZE, so we can't
 	 *    use kasan_populate_zero_shadow.
 	 */
-	create_mapping((void *)MODULES_VADDR, (void *)(PKMAP_BASE + PMD_SIZE));
+	if (!IS_ENABLED(CONFIG_KASAN_VMALLOC) && IS_ENABLED(CONFIG_MODULES))
+		create_mapping((void *)MODULES_VADDR, (void *)(MODULES_END));
+	create_mapping((void *)PKMAP_BASE, (void *)(PKMAP_BASE + PMD_SIZE));
 
 	/*
 	 * KAsan may reuse the contents of kasan_early_shadow_pte directly, so
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index a49f0b9c0f75..463fc2a8448f 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -300,7 +300,11 @@ static struct mem_type mem_types[] __ro_after_init = {
 		.prot_pte  = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
 			     L_PTE_XN | L_PTE_RDONLY,
 		.prot_l1   = PMD_TYPE_TABLE,
+#ifdef CONFIG_ARM_LPAE
+		.prot_sect = PMD_TYPE_SECT | L_PMD_SECT_RDONLY | PMD_SECT_AP2,
+#else
 		.prot_sect = PMD_TYPE_SECT,
+#endif
 		.domain    = DOMAIN_KERNEL,
 	},
 	[MT_ROM] = {
diff --git a/arch/arm/plat-orion/Makefile b/arch/arm/plat-orion/Makefile
index 4e3f25de13c1..830b0be038c6 100644
--- a/arch/arm/plat-orion/Makefile
+++ b/arch/arm/plat-orion/Makefile
@@ -2,7 +2,7 @@
 #
 # Makefile for the linux kernel.
 #
-ccflags-$(CONFIG_ARCH_MULTIPLATFORM) := -I$(srctree)/$(src)/include
+ccflags-y := -I$(srctree)/$(src)/include
 
 orion-gpio-$(CONFIG_GPIOLIB)      += gpio.o
 obj-$(CONFIG_PLAT_ORION_LEGACY)   += irq.o pcie.o time.o common.o mpp.o
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 1ce7685ad5de..0e48d1e4c578 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -149,6 +149,7 @@ config ARM64
 	select HAVE_ARCH_AUDITSYSCALL
 	select HAVE_ARCH_BITREVERSE
 	select HAVE_ARCH_COMPILER_H
+	select HAVE_ARCH_HUGE_VMALLOC
 	select HAVE_ARCH_HUGE_VMAP
 	select HAVE_ARCH_JUMP_LABEL
 	select HAVE_ARCH_JUMP_LABEL_RELATIVE
@@ -230,6 +231,7 @@ config ARM64
 	select HAVE_ARCH_USERFAULTFD_MINOR if USERFAULTFD
 	select TRACE_IRQFLAGS_SUPPORT
 	select TRACE_IRQFLAGS_NMI_SUPPORT
+	select HAVE_SOFTIRQ_ON_OWN_STACK
 	help
 	  ARM 64-bit (AArch64) Linux support.
 
@@ -733,6 +735,19 @@ config ARM64_ERRATUM_2077057
 
 	  If unsure, say Y.
 
+config ARM64_ERRATUM_2658417
+	bool "Cortex-A510: 2658417: remove BF16 support due to incorrect result"
+	default y
+	help
+	  This option adds the workaround for ARM Cortex-A510 erratum 2658417.
+	  Affected Cortex-A510 (r0p0 to r1p1) may produce the wrong result for
+	  BFMMLA or VMMLA instructions in rare circumstances when a pair of
+	  A510 CPUs are using shared neon hardware. As the sharing is not
+	  discoverable by the kernel, hide the BF16 HWCAP to indicate that
+	  user-space should not be using these instructions.
+
+	  If unsure, say Y.
+
 config ARM64_ERRATUM_2119858
 	bool "Cortex-A710/X2: 2119858: workaround TRBE overwriting trace data in FILL mode"
 	default y
@@ -1562,6 +1577,9 @@ config THUMB2_COMPAT_VDSO
 	  Compile the compat vDSO with '-mthumb -fomit-frame-pointer' if y,
 	  otherwise with '-marm'.
 
+config COMPAT_ALIGNMENT_FIXUPS
+	bool "Fix up misaligned multi-word loads and stores in user space"
+
 menuconfig ARMV8_DEPRECATED
 	bool "Emulate deprecated/obsolete ARMv8 instructions"
 	depends on SYSCTL
diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms
index 74e9e9de3759..76580b932e44 100644
--- a/arch/arm64/Kconfig.platforms
+++ b/arch/arm64/Kconfig.platforms
@@ -33,6 +33,11 @@ config ARCH_APPLE
 	  This enables support for Apple's in-house ARM SoC family, starting
 	  with the Apple M1.
 
+menuconfig ARCH_BCM
+	bool "Broadcom SoC Support"
+
+if ARCH_BCM
+
 config ARCH_BCM2835
 	bool "Broadcom BCM2835 family"
 	select TIMER_OF
@@ -47,15 +52,6 @@ config ARCH_BCM2835
 	  This enables support for the Broadcom BCM2837 and BCM2711 SoC.
 	  These SoCs are used in the Raspberry Pi 3 and 4 devices.
 
-config ARCH_BCM4908
-	bool "Broadcom BCM4908 family"
-	select ARCH_BCMBCA
-	select GPIOLIB
-	help
-	  This enables support for the Broadcom BCM4906, BCM4908 and
-	  BCM49408 SoCs. These SoCs use Brahma-B53 cores and can be
-	  found in home routers.
-
 config ARCH_BCM_IPROC
 	bool "Broadcom iProc SoC Family"
 	select COMMON_CLK_IPROC
@@ -66,6 +62,7 @@ config ARCH_BCM_IPROC
 
 config ARCH_BCMBCA
 	bool "Broadcom Broadband Carrier Access (BCA) origin SoC"
+	select GPIOLIB
 	help
 	  Say Y if you intend to run the kernel on a Broadcom Broadband ARM-based
 	  BCA chipset.
@@ -73,6 +70,16 @@ config ARCH_BCMBCA
 	  This enables support for Broadcom BCA ARM-based broadband chipsets,
 	  including the DSL, PON and Wireless family of chips.
 
+config ARCH_BRCMSTB
+	bool "Broadcom Set-Top-Box SoCs"
+	select ARCH_HAS_RESET_CONTROLLER
+	select GENERIC_IRQ_CHIP
+	select PINCTRL
+	help
+	  This enables support for Broadcom's ARMv8 Set Top Box SoCs
+
+endif
+
 config ARCH_BERLIN
 	bool "Marvell Berlin SoC Family"
 	select DW_APB_ICTL
@@ -87,14 +94,6 @@ config ARCH_BITMAIN
 	help
 	  This enables support for the Bitmain SoC Family.
 
-config ARCH_BRCMSTB
-	bool "Broadcom Set-Top-Box SoCs"
-	select ARCH_HAS_RESET_CONTROLLER
-	select GENERIC_IRQ_CHIP
-	select PINCTRL
-	help
-	  This enables support for Broadcom's ARMv8 Set Top Box SoCs
-
 config ARCH_EXYNOS
 	bool "ARMv8 based Samsung Exynos SoC family"
 	select COMMON_CLK_SAMSUNG
@@ -136,12 +135,6 @@ config ARCH_K3
 	  This enables support for Texas Instruments' K3 multicore SoC
 	  architecture.
 
-config ARCH_LAYERSCAPE
-	bool "ARMv8 based Freescale Layerscape SoC family"
-	select EDAC_SUPPORT
-	help
-	  This enables support for the Freescale Layerscape SoC family.
-
 config ARCH_LG1K
 	bool "LG Electronics LG1K SoC Family"
 	help
@@ -200,6 +193,17 @@ config ARCH_MVEBU
 	   - Armada 8K SoC Family
 	   - 98DX2530 SoC Family
 
+menuconfig ARCH_NXP
+	bool "NXP SoC support"
+
+if ARCH_NXP
+
+config ARCH_LAYERSCAPE
+	bool "ARMv8 based Freescale Layerscape SoC family"
+	select EDAC_SUPPORT
+	help
+	  This enables support for the Freescale Layerscape SoC family.
+
 config ARCH_MXC
 	bool "ARMv8 based NXP i.MX SoC family"
 	select ARM64_ERRATUM_843419
@@ -214,6 +218,13 @@ config ARCH_MXC
 	  This enables support for the ARMv8 based SoCs in the
 	  NXP i.MX family.
 
+config ARCH_S32
+	bool "NXP S32 SoC Family"
+	help
+	  This enables support for the NXP S32 family of processors.
+
+endif
+
 config ARCH_NPCM
 	bool "Nuvoton NPCM Architecture"
 	select PINCTRL
@@ -243,7 +254,6 @@ config ARCH_RENESAS
 	bool "Renesas SoC Platforms"
 	select GPIOLIB
 	select PINCTRL
-	select SOC_BUS
 	help
 	  This enables support for the ARMv8 based Renesas SoCs.
 
@@ -257,11 +267,6 @@ config ARCH_ROCKCHIP
 	  This enables support for the ARMv8 based Rockchip chipsets,
 	  like the RK3368.
 
-config ARCH_S32
-	bool "NXP S32 SoC Family"
-	help
-	  This enables support for the NXP S32 family of processors.
-
 config ARCH_SEATTLE
 	bool "AMD Seattle SoC Family"
 	help
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi
index 548539c93ab0..97e3e6907acd 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi
+++ b/arch/arm64/boot/dts/allwinner/sun50i-a100.dtsi
@@ -101,6 +101,18 @@
 			#reset-cells = <1>;
 		};
 
+		dma: dma-controller@3002000 {
+			compatible = "allwinner,sun50i-a100-dma";
+			reg = <0x03002000 0x1000>;
+			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_DMA>, <&ccu CLK_MBUS_DMA>;
+			clock-names = "bus", "mbus";
+			resets = <&ccu RST_BUS_DMA>;
+			dma-channels = <8>;
+			dma-requests = <52>;
+			#dma-cells = <1>;
+		};
+
 		gic: interrupt-controller@3021000 {
 			compatible = "arm,gic-400";
 			reg = <0x03021000 0x1000>, <0x03022000 0x2000>,
@@ -209,6 +221,8 @@
 			interrupts = <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&ccu CLK_BUS_I2C0>;
 			resets = <&ccu RST_BUS_I2C0>;
+			dmas = <&dma 43>, <&dma 43>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -222,6 +236,8 @@
 			interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&ccu CLK_BUS_I2C1>;
 			resets = <&ccu RST_BUS_I2C1>;
+			dmas = <&dma 44>, <&dma 44>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -235,6 +251,8 @@
 			interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&ccu CLK_BUS_I2C2>;
 			resets = <&ccu RST_BUS_I2C2>;
+			dmas = <&dma 45>, <&dma 45>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -248,6 +266,8 @@
 			interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&ccu CLK_BUS_I2C3>;
 			resets = <&ccu RST_BUS_I2C3>;
+			dmas = <&dma 46>, <&dma 46>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 			#address-cells = <1>;
 			#size-cells = <0>;
@@ -325,6 +345,8 @@
 			interrupts = <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&r_ccu CLK_R_APB2_I2C0>;
 			resets = <&r_ccu RST_R_APB2_I2C0>;
+			dmas = <&dma 50>, <&dma 50>;
+			dma-names = "rx", "tx";
 			pinctrl-names = "default";
 			pinctrl-0 = <&r_i2c0_pins>;
 			status = "disabled";
@@ -340,6 +362,8 @@
 			interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&r_ccu CLK_R_APB2_I2C1>;
 			resets = <&r_ccu RST_R_APB2_I2C1>;
+			dmas = <&dma 51>, <&dma 51>;
+			dma-names = "rx", "tx";
 			pinctrl-names = "default";
 			pinctrl-0 = <&r_i2c1_pins>;
 			status = "disabled";
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts b/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts
index 6249e9e02928..9ec49ac2f6fd 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-beelink-gs1.dts
@@ -5,6 +5,7 @@
 
 #include "sun50i-h6.dtsi"
 #include "sun50i-h6-cpu-opp.dtsi"
+#include "sun50i-h6-gpu-opp.dtsi"
 
 #include <dt-bindings/gpio/gpio.h>
 
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6-gpu-opp.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h6-gpu-opp.dtsi
new file mode 100644
index 000000000000..b48049c4fc85
--- /dev/null
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h6-gpu-opp.dtsi
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+// Copyright (C) 2022 Clément Péron <peron.clem@gmail.com>
+
+/ {
+	gpu_opp_table: opp-table-gpu {
+		compatible = "operating-points-v2";
+
+		opp-216000000 {
+			opp-hz = /bits/ 64 <216000000>;
+			opp-microvolt = <810000 810000 1200000>;
+		};
+
+		opp-264000000 {
+			opp-hz = /bits/ 64 <264000000>;
+			opp-microvolt = <810000 810000 1200000>;
+		};
+
+		opp-312000000 {
+			opp-hz = /bits/ 64 <312000000>;
+			opp-microvolt = <810000 810000 1200000>;
+		};
+
+		opp-336000000 {
+			opp-hz = /bits/ 64 <336000000>;
+			opp-microvolt = <810000 810000 1200000>;
+		};
+
+		opp-360000000 {
+			opp-hz = /bits/ 64 <360000000>;
+			opp-microvolt = <820000 820000 1200000>;
+		};
+
+		opp-384000000 {
+			opp-hz = /bits/ 64 <384000000>;
+			opp-microvolt = <830000 830000 1200000>;
+		};
+
+		opp-408000000 {
+			opp-hz = /bits/ 64 <408000000>;
+			opp-microvolt = <840000 840000 1200000>;
+		};
+
+		opp-420000000 {
+			opp-hz = /bits/ 64 <420000000>;
+			opp-microvolt = <850000 850000 1200000>;
+		};
+
+		opp-432000000 {
+			opp-hz = /bits/ 64 <432000000>;
+			opp-microvolt = <860000 860000 1200000>;
+		};
+
+		opp-456000000 {
+			opp-hz = /bits/ 64 <456000000>;
+			opp-microvolt = <870000 870000 1200000>;
+		};
+
+		opp-504000000 {
+			opp-hz = /bits/ 64 <504000000>;
+			opp-microvolt = <890000 890000 1200000>;
+		};
+
+		opp-540000000 {
+			opp-hz = /bits/ 64 <540000000>;
+			opp-microvolt = <910000 910000 1200000>;
+		};
+
+		opp-576000000 {
+			opp-hz = /bits/ 64 <576000000>;
+			opp-microvolt = <930000 930000 1200000>;
+		};
+
+		opp-624000000 {
+			opp-hz = /bits/ 64 <624000000>;
+			opp-microvolt = <950000 950000 1200000>;
+		};
+
+		opp-756000000 {
+			opp-hz = /bits/ 64 <756000000>;
+			opp-microvolt = <1040000 1040000 1200000>;
+		};
+	};
+};
+
+&gpu {
+	operating-points-v2 = <&gpu_opp_table>;
+};
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi
index 5a28303d3d4c..53f6660656ac 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h6.dtsi
@@ -186,6 +186,7 @@
 			clocks = <&ccu CLK_GPU>, <&ccu CLK_BUS_GPU>;
 			clock-names = "core", "bus";
 			resets = <&ccu RST_BUS_GPU>;
+			#cooling-cells = <2>;
 			status = "disabled";
 		};
 
@@ -1072,9 +1073,55 @@
 		};
 
 		gpu-thermal {
-			polling-delay-passive = <0>;
-			polling-delay = <0>;
+			polling-delay-passive = <1000>;
+			polling-delay = <2000>;
 			thermal-sensors = <&ths 1>;
+
+			trips {
+				gpu_alert0: gpu-alert-0 {
+					temperature = <95000>;
+					hysteresis = <2000>;
+					type = "passive";
+				};
+
+				gpu_alert1: gpu-alert-1 {
+					temperature = <100000>;
+					hysteresis = <2000>;
+					type = "passive";
+				};
+
+				gpu_alert2: gpu-alert-2 {
+					temperature = <105000>;
+					hysteresis = <2000>;
+					type = "passive";
+				};
+
+				gpu-crit {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+
+			cooling-maps {
+				// Forbid the GPU to go over 756MHz
+				map0 {
+					trip = <&gpu_alert0>;
+					cooling-device = <&gpu 1 THERMAL_NO_LIMIT>;
+				};
+
+				// Forbid the GPU to go over 624MHz
+				map1 {
+					trip = <&gpu_alert1>;
+					cooling-device = <&gpu 2 THERMAL_NO_LIMIT>;
+				};
+
+				// Forbid the GPU to go over 576MHz
+				map2 {
+					trip = <&gpu_alert2>;
+					cooling-device = <&gpu 3 THERMAL_NO_LIMIT>;
+				};
+			};
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/amlogic/Makefile b/arch/arm64/boot/dts/amlogic/Makefile
index 8773211df50e..e213aeebb774 100644
--- a/arch/arm64/boot/dts/amlogic/Makefile
+++ b/arch/arm64/boot/dts/amlogic/Makefile
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 dtb-$(CONFIG_ARCH_MESON) += meson-a1-ad401.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-axg-jethome-jethub-j100.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-axg-jethome-jethub-j110-rev-2.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-axg-jethome-jethub-j110-rev-3.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-axg-s400.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-g12a-radxa-zero.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-g12a-sei510.dtb
@@ -43,6 +45,7 @@ dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-libretech-cc-v2.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-libretech-cc.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-nexbox-a95x.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxl-s905x-p212.dtb
+dtb-$(CONFIG_ARCH_MESON) += meson-gxm-gt1-ultimate.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxm-khadas-vim2.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxm-mecool-kiii-pro.dtb
 dtb-$(CONFIG_ARCH_MESON) += meson-gxm-minix-neo-u9h.dtb
diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j100.dts b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j100.dts
index 8b0d586aa84e..b2d6ba660914 100644
--- a/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j100.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j100.dts
@@ -1,270 +1,29 @@
 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
 /*
- * Copyright (c) 2021 Vyacheslav Bocharov <adeep@lexina.in>
- * Copyright (c) 2020 JetHome
- * Author: Aleksandr Kazantsev <ak@tvip.ru>
- * Author: Alexey Shevelkin <ash@tvip.ru>
+ * Copyright (c) 2022 Vyacheslav Bocharov <adeep@lexina.in>
+ * Copyright (c) 2022 JetHome
  * Author: Vyacheslav Bocharov <adeep@lexina.in>
  */
 
 /dts-v1/;
 
-#include "meson-axg.dtsi"
-#include <dt-bindings/input/input.h>
-#include <dt-bindings/thermal/thermal.h>
+#include "meson-axg-jethome-jethub-j1xx.dtsi"
 
 / {
 	compatible = "jethome,jethub-j100", "amlogic,a113d", "amlogic,meson-axg";
-	model = "JetHome JetHub J100";
-	aliases {
-		serial0 = &uart_AO;   /* Console */
-		serial2 = &uart_AO_B; /* External UART (Wireless Module) */
-		ethernet0 = &ethmac;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
+	model = "JetHome JetHub D1 (J100)";
 
 	/* 1024MB RAM */
 	memory@0 {
 		device_type = "memory";
 		reg = <0x0 0x0 0x0 0x40000000>;
 	};
-
-	reserved-memory {
-		linux,cma {
-			size = <0x0 0x400000>;
-		};
-	};
-
-	emmc_pwrseq: emmc-pwrseq {
-		compatible = "mmc-pwrseq-emmc";
-		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
-	};
-
-	vcc_3v3: regulator-vcc_3v3 {
-		compatible = "regulator-fixed";
-		regulator-name = "VCC_3V3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&vddao_3v3>;
-		regulator-always-on;
-	};
-
-	vcc_5v: regulator-vcc_5v {
-		compatible = "regulator-fixed";
-		regulator-name = "VCC5V";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		regulator-always-on;
-	};
-
-	vddao_3v3: regulator-vddao_3v3 {
-		compatible = "regulator-fixed";
-		regulator-name = "VDDAO_3V3";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&vcc_5v>;
-		regulator-always-on;
-	};
-
-	vddio_ao18: regulator-vddio_ao18 {
-		compatible = "regulator-fixed";
-		regulator-name = "VDDIO_AO18";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-		vin-supply = <&vddao_3v3>;
-		regulator-always-on;
-	};
-
-	vddio_boot: regulator-vddio_boot {
-		compatible = "regulator-fixed";
-		regulator-name = "VDDIO_BOOT";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		vin-supply = <&vddao_3v3>;
-		regulator-always-on;
-	};
-
-	vccq_1v8: regulator-vccq_1v8 {
-		compatible = "regulator-fixed";
-		regulator-name = "VCCQ_1V8";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-		vin-supply = <&vddao_3v3>;
-		regulator-always-on;
-	};
-
-	usb_pwr: regulator-usb_pwr {
-		compatible = "regulator-fixed";
-		regulator-name = "USB_PWR";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		vin-supply = <&vcc_5v>;
-		regulator-always-on;
-	};
-
-	sdio_pwrseq: sdio-pwrseq {
-		compatible = "mmc-pwrseq-simple";
-		reset-gpios = <&gpio GPIOX_7 GPIO_ACTIVE_LOW>;
-		clocks = <&wifi32k>;
-		clock-names = "ext_clock";
-	};
-
-	wifi32k: wifi32k {
-		compatible = "pwm-clock";
-		#clock-cells = <0>;
-		clock-frequency = <32768>;
-		pwms = <&pwm_ab 0 30518 0>; /* PWM_A at 32.768KHz */
-	};
-
-	thermal-zones {
-		cpu_thermal: cpu-thermal {
-			polling-delay-passive = <250>;
-			polling-delay = <1000>;
-			thermal-sensors = <&scpi_sensors 0>;
-			trips {
-				cpu_passive: cpu-passive {
-					temperature = <70000>; /* millicelsius */
-					hysteresis = <2000>; /* millicelsius */
-					type = "passive";
-				};
-
-				cpu_hot: cpu-hot {
-					temperature = <80000>; /* millicelsius */
-					hysteresis = <2000>; /* millicelsius */
-					type = "hot";
-				};
-
-				cpu_critical: cpu-critical {
-					temperature = <100000>; /* millicelsius */
-					hysteresis = <2000>; /* millicelsius */
-					type = "critical";
-				};
-			};
-
-			cpu_cooling_maps: cooling-maps {
-				map0 {
-					trip = <&cpu_passive>;
-					cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
-							<&cpu1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
-							<&cpu2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
-							<&cpu3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
-				};
-
-				map1 {
-					trip = <&cpu_hot>;
-					cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
-							<&cpu1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
-							<&cpu2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
-							<&cpu3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
-				};
-			};
-		};
-	};
-
-	onewire {
-		compatible = "w1-gpio";
-		gpios = <&gpio GPIOA_14 GPIO_ACTIVE_HIGH>;
-		#gpio-cells = <1>;
-	};
-};
-
-&efuse {
-	sn: sn@32 {
-		reg = <0x32 0x20>;
-	};
-
-	eth_mac: eth_mac@0 {
-		reg = <0x0 0x6>;
-	};
-
-	bt_mac: bt_mac@6 {
-		reg = <0x6 0x6>;
-	};
-
-	wifi_mac: wifi_mac@c {
-		reg = <0xc 0x6>;
-	};
-
-	bid: bid@12 {
-		reg = <0x12 0x20>;
-	};
-};
-
-&ethmac {
-	status = "okay";
-	pinctrl-0 = <&eth_rmii_x_pins>;
-	pinctrl-names = "default";
-	phy-handle = <&eth_phy0>;
-	phy-mode = "rmii";
-
-	mdio {
-		compatible = "snps,dwmac-mdio";
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		/* ICPlus IP101A/G Ethernet PHY (vendor_id=0x0243, model_id=0x0c54) */
-		eth_phy0: ethernet-phy@0 {
-			/* compatible = "ethernet-phy-id0243.0c54";*/
-			max-speed = <100>;
-			reg = <0>;
-
-			reset-assert-us = <10000>;
-			reset-deassert-us = <10000>;
-			reset-gpios = <&gpio GPIOZ_5 GPIO_ACTIVE_LOW>;
-		};
-	};
-};
-
-/* Internal I2C bus (on CPU module) */
-&i2c1 {
-	status = "okay";
-	pinctrl-0 = <&i2c1_z_pins>;
-	pinctrl-names = "default";
-
-	/* RTC */
-	pcf8563: pcf8563@51 {
-		compatible = "nxp,pcf8563";
-		reg = <0x51>;
-		status = "okay";
-	};
 };
 
-/* Peripheral I2C bus (on motherboard) */
-&i2c_AO {
-	status = "okay";
-	pinctrl-0 = <&i2c_ao_sck_10_pins>, <&i2c_ao_sda_11_pins>;
-	pinctrl-names = "default";
-};
-
-&pwm_ab {
-	status = "okay";
-	pinctrl-0 = <&pwm_a_x20_pins>;
-	pinctrl-names = "default";
-};
 
 /* wifi module */
 &sd_emmc_b {
-	status = "okay";
-	#address-cells = <1>;
-	#size-cells = <0>;
-
-	pinctrl-0 = <&sdio_pins>;
-	pinctrl-1 = <&sdio_clk_gate_pins>;
-	pinctrl-names = "default", "clk-gate";
-
-	bus-width = <4>;
-	cap-sd-highspeed;
-	max-frequency = <50000000>;
 	non-removable;
-	disable-wp;
-
-	mmc-pwrseq = <&sdio_pwrseq>;
-
-	vmmc-supply = <&vddao_3v3>;
-	vqmmc-supply = <&vddio_boot>;
 
 	brcmf: wifi@1 {
 		reg = <1>;
@@ -272,99 +31,10 @@
 	};
 };
 
-/* emmc storage */
-&sd_emmc_c {
-	status = "okay";
-	pinctrl-0 = <&emmc_pins>, <&emmc_ds_pins>;
-	pinctrl-1 = <&emmc_clk_gate_pins>;
-	pinctrl-names = "default", "clk-gate";
-
-	bus-width = <8>;
-	cap-mmc-highspeed;
-	max-frequency = <200000000>;
-	non-removable;
-	disable-wp;
-	mmc-ddr-1_8v;
-	mmc-hs200-1_8v;
-
-	mmc-pwrseq = <&emmc_pwrseq>;
-
-	vmmc-supply = <&vcc_3v3>;
-	vqmmc-supply = <&vccq_1v8>;
-};
-
 /* UART Bluetooth */
 &uart_B {
-	status = "okay";
-	pinctrl-0 = <&uart_b_z_pins>, <&uart_b_z_cts_rts_pins>;
-	pinctrl-names = "default";
-	uart-has-rtscts;
-
 	bluetooth {
 		compatible = "brcm,bcm43438-bt";
 		shutdown-gpios = <&gpio GPIOZ_7 GPIO_ACTIVE_HIGH>;
 	};
 };
-
-/* UART Console */
-&uart_AO {
-	status = "okay";
-	pinctrl-0 = <&uart_ao_a_pins>;
-	pinctrl-names = "default";
-};
-
-/* UART Wireless module */
-&uart_AO_B {
-	status = "okay";
-	pinctrl-0 = <&uart_ao_b_pins>;
-	pinctrl-names = "default";
-};
-
-&usb {
-	status = "okay";
-	phy-supply = <&usb_pwr>;
-};
-
-&spicc1 {
-	status = "okay";
-	pinctrl-0 = <&spi1_x_pins>, <&spi1_ss0_x_pins>;
-	pinctrl-names = "default";
-};
-
-&gpio {
-	gpio-line-names =
-		"", "", "", "", "", // 0 - 4
-		"", "", "", "", "", // 5 - 9
-		"UserButton", "", "", "", "", // 10 - 14
-		"", "", "", "", "", // 15 - 19
-		"", "", "", "", "", // 20 - 24
-		"", "LedRed", "LedGreen", "Output3", "Output2", // 25 - 29
-		"Output1", "", "", "", "", // 30 - 34
-		"", "ZigBeeBOOT", "", "", "", // 35 - 39
-		"1Wire", "ZigBeeRESET", "", "Input4", "Input3", // 40 - 44
-		"Input2", "Input1", "", "", "", // 45 - 49
-		"", "", "", "", "", // 50 - 54
-		"", "", "", "", "", // 55 - 59
-		"", "", "", "", "", // 60 - 64
-		"", "", "", "", "", // 65 - 69
-		"", "", "", "", "", // 70 - 74
-		"", "", "", "", "", // 75 - 79
-		"", "", "", "", "", // 80 - 84
-		"", ""; // 85-86
-};
-
-&cpu0 {
-	#cooling-cells = <2>;
-};
-
-&cpu1 {
-	#cooling-cells = <2>;
-};
-
-&cpu2 {
-	#cooling-cells = <2>;
-};
-
-&cpu3 {
-	#cooling-cells = <2>;
-};
diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-2.dts b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-2.dts
new file mode 100644
index 000000000000..0062667c4f65
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-2.dts
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2022 Vyacheslav Bocharov <adeep@lexina.in>
+ * Copyright (c) 2022 JetHome
+ * Author: Vyacheslav Bocharov <adeep@lexina.in>
+ */
+
+/dts-v1/;
+
+#include "meson-axg-jethome-jethub-j1xx.dtsi"
+
+/ {
+	compatible = "jethome,jethub-j110", "amlogic,a113d", "amlogic,meson-axg";
+	model = "JetHome JetHub D1p (J110) HW rev.2";
+
+	/* 2GiB or 4GiB RAM */
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x80000000>;
+	};
+};
+
+
+/* wifi module */
+&sd_emmc_b {
+	broken-cd;/* cd-gpios = <&gpio_ao GPIOAO_3 GPIO_ACTIVE_LOW>;*/
+};
+
+/* UART Bluetooth */
+&uart_B {
+	bluetooth {
+		compatible = "realtek,rtl8822cs-bt";
+		enable-gpios  = <&gpio GPIOZ_7 GPIO_ACTIVE_HIGH>;
+		host-wake-gpios = <&gpio GPIOZ_8 GPIO_ACTIVE_HIGH>;
+		device-wake-gpios = <&gpio GPIOZ_6 GPIO_ACTIVE_HIGH>;
+	};
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-3.dts b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-3.dts
new file mode 100644
index 000000000000..c2d22b00c1cd
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j110-rev-3.dts
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2022 Vyacheslav Bocharov <adeep@lexina.in>
+ * Copyright (c) 2022 JetHome
+ * Author: Vyacheslav Bocharov <adeep@lexina.in>
+ */
+
+/dts-v1/;
+
+#include "meson-axg-jethome-jethub-j1xx.dtsi"
+
+/ {
+	compatible = "jethome,jethub-j110", "amlogic,a113d", "amlogic,meson-axg";
+	model = "JetHome JetHub D1p (J110) Hw rev.3";
+
+	/* 2GiB or 4GiB RAM */
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x80000000>;
+	};
+};
+
+
+/* wifi module */
+&sd_emmc_b {
+	broken-cd;/* cd-gpios = <&gpio_ao GPIOAO_3 GPIO_ACTIVE_LOW>;*/
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j1xx.dtsi b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j1xx.dtsi
new file mode 100644
index 000000000000..5836b0030931
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-axg-jethome-jethub-j1xx.dtsi
@@ -0,0 +1,351 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2022 Vyacheslav Bocharov <adeep@lexina.in>
+ * Copyright (c) 2022 JetHome
+ * Author: Vyacheslav Bocharov <adeep@lexina.in>
+ * Author: Aleksandr Kazantsev <ak@tvip.ru>
+ * Author: Alexey Shevelkin <ash@tvip.ru>
+ */
+
+/dts-v1/;
+
+#include "meson-axg.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/thermal/thermal.h>
+
+/ {
+	aliases {
+		serial0 = &uart_AO;   /* Console */
+		serial2 = &uart_AO_B; /* External UART (Wireless Module) */
+		ethernet0 = &ethmac;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	reserved-memory {
+		linux,cma {
+			size = <0x0 0x400000>;
+		};
+	};
+
+	emmc_pwrseq: emmc-pwrseq {
+		compatible = "mmc-pwrseq-emmc";
+		reset-gpios = <&gpio BOOT_9 GPIO_ACTIVE_LOW>;
+	};
+
+	vcc_3v3: regulator-vcc_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vddao_3v3>;
+		regulator-always-on;
+	};
+
+	vcc_5v: regulator-vcc_5v {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+	};
+
+	vddao_3v3: regulator-vddao_3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDAO_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_5v>;
+		regulator-always-on;
+	};
+
+	vddio_ao18: regulator-vddio_ao18 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_AO18";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vddao_3v3>;
+		regulator-always-on;
+	};
+
+	vddio_boot: regulator-vddio_boot {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_BOOT";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vddao_3v3>;
+		regulator-always-on;
+	};
+
+	vccq_1v8: regulator-vccq_1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCCQ_1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vddao_3v3>;
+		regulator-always-on;
+	};
+
+	usb_pwr: regulator-usb_pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "USB_PWR";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc_5v>;
+		regulator-always-on;
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpio GPIOX_7 GPIO_ACTIVE_LOW>;
+		clocks = <&wifi32k>;
+		clock-names = "ext_clock";
+	};
+
+	wifi32k: wifi32k {
+		compatible = "pwm-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		pwms = <&pwm_ab 0 30518 0>; /* PWM_A at 32.768KHz */
+	};
+
+	thermal-zones {
+		cpu_thermal: cpu-thermal {
+			polling-delay-passive = <250>;
+			polling-delay = <1000>;
+			thermal-sensors = <&scpi_sensors 0>;
+			trips {
+				cpu_passive: cpu-passive {
+					temperature = <70000>; /* millicelsius */
+					hysteresis = <2000>; /* millicelsius */
+					type = "passive";
+				};
+
+				cpu_hot: cpu-hot {
+					temperature = <80000>; /* millicelsius */
+					hysteresis = <2000>; /* millicelsius */
+					type = "hot";
+				};
+
+				cpu_critical: cpu-critical {
+					temperature = <100000>; /* millicelsius */
+					hysteresis = <2000>; /* millicelsius */
+					type = "critical";
+				};
+			};
+
+			cpu_cooling_maps: cooling-maps {
+				map0 {
+					trip = <&cpu_passive>;
+					cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							<&cpu1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							<&cpu2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							<&cpu3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+
+				map1 {
+					trip = <&cpu_hot>;
+					cooling-device = <&cpu0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							<&cpu1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							<&cpu2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+							<&cpu3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+		};
+	};
+
+	onewire {
+		compatible = "w1-gpio";
+		gpios = <&gpio GPIOA_14 GPIO_ACTIVE_HIGH>;
+		#gpio-cells = <1>;
+	};
+};
+
+&efuse {
+	sn: sn@32 {
+		reg = <0x32 0x20>;
+	};
+
+	eth_mac: eth_mac@0 {
+		reg = <0x0 0x6>;
+	};
+
+	bt_mac: bt_mac@6 {
+		reg = <0x6 0x6>;
+	};
+
+	wifi_mac: wifi_mac@c {
+		reg = <0xc 0x6>;
+	};
+
+	bid: bid@12 {
+		reg = <0x12 0x20>;
+	};
+};
+
+&ethmac {
+	status = "okay";
+	pinctrl-0 = <&eth_rmii_x_pins>;
+	pinctrl-names = "default";
+	phy-handle = <&eth_phy0>;
+	phy-mode = "rmii";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		/* ICPlus IP101A/G Ethernet PHY (vendor_id=0x0243, model_id=0x0c54) */
+		eth_phy0: ethernet-phy@0 {
+			/* compatible = "ethernet-phy-id0243.0c54";*/
+			max-speed = <100>;
+			reg = <0>;
+
+			reset-assert-us = <10000>;
+			reset-deassert-us = <10000>;
+			reset-gpios = <&gpio GPIOZ_5 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+/* Internal I2C bus (on CPU module) */
+&i2c1 {
+	status = "okay";
+	pinctrl-0 = <&i2c1_z_pins>;
+	pinctrl-names = "default";
+
+	/* RTC */
+	pcf8563: pcf8563@51 {
+		compatible = "nxp,pcf8563";
+		reg = <0x51>;
+		status = "okay";
+	};
+};
+
+/* Peripheral I2C bus (on motherboard) */
+&i2c_AO {
+	status = "okay";
+	pinctrl-0 = <&i2c_ao_sck_10_pins>, <&i2c_ao_sda_11_pins>;
+	pinctrl-names = "default";
+};
+
+&pwm_ab {
+	status = "okay";
+	pinctrl-0 = <&pwm_a_x20_pins>;
+	pinctrl-names = "default";
+};
+
+/* wifi module */
+&sd_emmc_b {
+	status = "okay";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	pinctrl-0 = <&sdio_pins>;
+	pinctrl-1 = <&sdio_clk_gate_pins>;
+	pinctrl-names = "default", "clk-gate";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	max-frequency = <50000000>;
+	disable-wp;
+
+	mmc-pwrseq = <&sdio_pwrseq>;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddio_boot>;
+};
+
+/* emmc storage */
+&sd_emmc_c {
+	status = "okay";
+	pinctrl-0 = <&emmc_pins>, <&emmc_ds_pins>;
+	pinctrl-1 = <&emmc_clk_gate_pins>;
+	pinctrl-names = "default", "clk-gate";
+
+	bus-width = <8>;
+	cap-mmc-highspeed;
+	max-frequency = <200000000>;
+	non-removable;
+	disable-wp;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+
+	mmc-pwrseq = <&emmc_pwrseq>;
+
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vccq_1v8>;
+};
+
+/* UART Bluetooth */
+&uart_B {
+	status = "okay";
+	pinctrl-0 = <&uart_b_z_pins>, <&uart_b_z_cts_rts_pins>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+};
+
+/* UART Console */
+&uart_AO {
+	status = "okay";
+	pinctrl-0 = <&uart_ao_a_pins>;
+	pinctrl-names = "default";
+};
+
+/* UART Wireless module */
+&uart_AO_B {
+	status = "okay";
+	pinctrl-0 = <&uart_ao_b_pins>;
+	pinctrl-names = "default";
+};
+
+&usb {
+	status = "okay";
+	phy-supply = <&usb_pwr>;
+};
+
+&spicc1 {
+	status = "okay";
+	pinctrl-0 = <&spi1_x_pins>, <&spi1_ss0_x_pins>;
+	pinctrl-names = "default";
+};
+
+&gpio {
+	gpio-line-names =
+		"", "", "", "", "", // 0 - 4
+		"", "", "", "", "", // 5 - 9
+		"UserButton", "", "", "", "", // 10 - 14
+		"", "", "", "", "", // 15 - 19
+		"", "", "", "", "", // 20 - 24
+		"", "LedRed", "LedGreen", "Output3", "Output2", // 25 - 29
+		"Output1", "", "", "", "", // 30 - 34
+		"", "ZigBeeBOOT", "", "", "", // 35 - 39
+		"1Wire", "ZigBeeRESET", "", "Input4", "Input3", // 40 - 44
+		"Input2", "Input1", "", "", "", // 45 - 49
+		"", "", "", "", "", // 50 - 54
+		"", "", "", "", "", // 55 - 59
+		"", "", "", "", "", // 60 - 64
+		"", "", "", "", "", // 65 - 69
+		"", "", "", "", "", // 70 - 74
+		"", "", "", "", "", // 75 - 79
+		"", "", "", "", "", // 80 - 84
+		"", ""; // 85-86
+};
+
+&cpu0 {
+	#cooling-cells = <2>;
+};
+
+&cpu1 {
+	#cooling-cells = <2>;
+};
+
+&cpu2 {
+	#cooling-cells = <2>;
+};
+
+&cpu3 {
+	#cooling-cells = <2>;
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts b/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts
index b4e86196e346..b2bb94981838 100644
--- a/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-g12a-x96-max.dts
@@ -111,7 +111,6 @@
 		vin-supply = <&dc_in>;
 
 		gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>;
-		enable-active-low;
 	};
 
 	vddao_1v8: regulator-vddao_1v8 {
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxm-gt1-ultimate.dts b/arch/arm64/boot/dts/amlogic/meson-gxm-gt1-ultimate.dts
new file mode 100644
index 000000000000..2c267884cc16
--- /dev/null
+++ b/arch/arm64/boot/dts/amlogic/meson-gxm-gt1-ultimate.dts
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) Christian Hewitt <christianshewitt@gmail.com>
+ */
+
+/dts-v1/;
+
+#include "meson-gxm.dtsi"
+#include "meson-gx-p23x-q20x.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+
+/ {
+	compatible = "azw,gt1-ultimate", "amlogic,s912", "amlogic,meson-gxm";
+	model = "Beelink GT1 Ultimate";
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-white {
+			color = <LED_COLOR_ID_WHITE>;
+			function = LED_FUNCTION_POWER;
+			gpios = <&gpio_ao GPIOAO_9 GPIO_ACTIVE_HIGH>;
+			default-state = "on";
+			panic-indicator;
+		};
+	};
+
+	adc-keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 0>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1710000>;
+
+		button-function {
+			label = "update";
+			linux,code = <KEY_VENDOR>;
+			press-threshold-microvolt = <10000>;
+		};
+	};
+};
+
+&ethmac {
+	pinctrl-0 = <&eth_pins>;
+	pinctrl-names = "default";
+	phy-handle = <&external_phy>;
+	amlogic,tx-delay-ns = <2>;
+	phy-mode = "rgmii";
+};
+
+&external_mdio {
+	external_phy: ethernet-phy@0 {
+		/* Realtek RTL8211F (0x001cc916) */
+		reg = <0>;
+		max-speed = <1000>;
+
+		reset-assert-us = <10000>;
+		reset-deassert-us = <80000>;
+		reset-gpios = <&gpio GPIOZ_14 GPIO_ACTIVE_LOW>;
+
+		interrupt-parent = <&gpio_intc>;
+		/* MAC_INTR on GPIOZ_15 */
+		interrupts = <25 IRQ_TYPE_LEVEL_LOW>;
+	};
+};
+
+&ir {
+	linux,rc-map-name = "rc-beelink-gs1";
+};
+
+&sd_emmc_a {
+	brcmf: wifi@1 {
+		reg = <1>;
+		compatible = "brcm,bcm4329-fmac";
+	};
+};
+
+&uart_A {
+	status = "okay";
+	pinctrl-0 = <&uart_a_pins>, <&uart_a_cts_rts_pins>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+
+	bluetooth {
+		compatible = "brcm,bcm43438-bt";
+		shutdown-gpios = <&gpio GPIOX_17 GPIO_ACTIVE_HIGH>;
+		max-speed = <2000000>;
+		clocks = <&wifi32k>;
+		clock-names = "lpo";
+	};
+};
diff --git a/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts b/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts
index 603337ca5608..9068a334ea57 100644
--- a/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-sm1-sei610.dts
@@ -171,7 +171,6 @@
 		regulator-max-microvolt = <3300000>;
 		vin-supply = <&vddao_3v3>;
 		gpio = <&gpio GPIOH_8 GPIO_OPEN_DRAIN>;
-		enable-active-low;
 		regulator-always-on;
 	};
 
diff --git a/arch/arm64/boot/dts/apple/t8103-j274.dts b/arch/arm64/boot/dts/apple/t8103-j274.dts
index 2cd429efba5b..c1f3ba9c39f6 100644
--- a/arch/arm64/boot/dts/apple/t8103-j274.dts
+++ b/arch/arm64/boot/dts/apple/t8103-j274.dts
@@ -21,6 +21,10 @@
 	};
 };
 
+&wifi0 {
+	brcm,board-type = "apple,atlantisb";
+};
+
 /*
  * Force the bus number assignments so that we can declare some of the
  * on-board devices and properties that are populated by the bootloader
diff --git a/arch/arm64/boot/dts/apple/t8103-j293.dts b/arch/arm64/boot/dts/apple/t8103-j293.dts
index 49cdf4b560a3..ecb10d237a05 100644
--- a/arch/arm64/boot/dts/apple/t8103-j293.dts
+++ b/arch/arm64/boot/dts/apple/t8103-j293.dts
@@ -17,6 +17,10 @@
 	model = "Apple MacBook Pro (13-inch, M1, 2020)";
 };
 
+&wifi0 {
+	brcm,board-type = "apple,honshu";
+};
+
 /*
  * Remove unused PCIe ports and disable the associated DARTs.
  */
diff --git a/arch/arm64/boot/dts/apple/t8103-j313.dts b/arch/arm64/boot/dts/apple/t8103-j313.dts
index b0ebb45bdb6f..df741737b8e6 100644
--- a/arch/arm64/boot/dts/apple/t8103-j313.dts
+++ b/arch/arm64/boot/dts/apple/t8103-j313.dts
@@ -17,6 +17,10 @@
 	model = "Apple MacBook Air (M1, 2020)";
 };
 
+&wifi0 {
+	brcm,board-type = "apple,shikoku";
+};
+
 /*
  * Remove unused PCIe ports and disable the associated DARTs.
  */
diff --git a/arch/arm64/boot/dts/apple/t8103-j456.dts b/arch/arm64/boot/dts/apple/t8103-j456.dts
index 884fddf7d363..8c6bf9592510 100644
--- a/arch/arm64/boot/dts/apple/t8103-j456.dts
+++ b/arch/arm64/boot/dts/apple/t8103-j456.dts
@@ -21,6 +21,10 @@
 	};
 };
 
+&wifi0 {
+	brcm,board-type = "apple,capri";
+};
+
 &i2c0 {
 	hpm2: usb-pd@3b {
 		compatible = "apple,cd321x";
diff --git a/arch/arm64/boot/dts/apple/t8103-j457.dts b/arch/arm64/boot/dts/apple/t8103-j457.dts
index d7c622931627..fe7c0aaf7d62 100644
--- a/arch/arm64/boot/dts/apple/t8103-j457.dts
+++ b/arch/arm64/boot/dts/apple/t8103-j457.dts
@@ -21,6 +21,10 @@
 	};
 };
 
+&wifi0 {
+	brcm,board-type = "apple,santorini";
+};
+
 /*
  * Force the bus number assignments so that we can declare some of the
  * on-board devices and properties that are populated by the bootloader
diff --git a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi
index fe2ae40fa9dd..3d15b8e2a6c1 100644
--- a/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi
+++ b/arch/arm64/boot/dts/apple/t8103-jxxx.dtsi
@@ -71,8 +71,10 @@
 &port00 {
 	bus-range = <1 1>;
 	wifi0: network@0,0 {
+		compatible = "pci14e4,4425";
 		reg = <0x10000 0x0 0x0 0x0 0x0>;
 		/* To be filled by the loader */
 		local-mac-address = [00 00 00 00 00 00];
+		apple,antenna-sku = "XX";
 	};
 };
diff --git a/arch/arm64/boot/dts/broadcom/Makefile b/arch/arm64/boot/dts/broadcom/Makefile
index e8584d3b698f..05d8c5ecf3b0 100644
--- a/arch/arm64/boot/dts/broadcom/Makefile
+++ b/arch/arm64/boot/dts/broadcom/Makefile
@@ -8,7 +8,6 @@ dtb-$(CONFIG_ARCH_BCM2835) += bcm2711-rpi-400.dtb \
 			      bcm2837-rpi-cm3-io3.dtb \
 			      bcm2837-rpi-zero-2-w.dtb
 
-subdir-y	+= bcm4908
 subdir-y	+= bcmbca
 subdir-y	+= northstar2
 subdir-y	+= stingray
diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/Makefile b/arch/arm64/boot/dts/broadcom/bcm4908/Makefile
deleted file mode 100644
index 6e364e304d4f..000000000000
--- a/arch/arm64/boot/dts/broadcom/bcm4908/Makefile
+++ /dev/null
@@ -1,5 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-dtb-$(CONFIG_ARCH_BCM4908) += bcm4906-netgear-r8000p.dtb
-dtb-$(CONFIG_ARCH_BCM4908) += bcm4906-tplink-archer-c2300-v1.dtb
-dtb-$(CONFIG_ARCH_BCM4908) += bcm4908-asus-gt-ac5300.dtb
-dtb-$(CONFIG_ARCH_BCM4908) += bcm4908-netgear-raxe500.dtb
diff --git a/arch/arm64/boot/dts/broadcom/bcmbca/Makefile b/arch/arm64/boot/dts/broadcom/bcmbca/Makefile
index 38f14307184b..27741b71ba9e 100644
--- a/arch/arm64/boot/dts/broadcom/bcmbca/Makefile
+++ b/arch/arm64/boot/dts/broadcom/bcmbca/Makefile
@@ -1,5 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0
 dtb-$(CONFIG_ARCH_BCMBCA) += \
+				bcm4906-netgear-r8000p.dtb \
+				bcm4906-tplink-archer-c2300-v1.dtb \
+				bcm4908-asus-gt-ac5300.dtb \
+				bcm4908-netgear-raxe500.dtb \
+				bcm94908.dtb \
 				bcm4912-asus-gt-ax6000.dtb \
 				bcm94912.dtb \
 				bcm963158.dtb \
diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906-netgear-r8000p.dts b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906-netgear-r8000p.dts
index 2dd028438c22..d8b60575eb4f 100644
--- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906-netgear-r8000p.dts
+++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906-netgear-r8000p.dts
@@ -7,7 +7,7 @@
 #include "bcm4906.dtsi"
 
 / {
-	compatible = "netgear,r8000p", "brcm,bcm4906", "brcm,bcm4908";
+	compatible = "netgear,r8000p", "brcm,bcm4906", "brcm,bcm4908", "brcm,bcmbca";
 	model = "Netgear R8000P";
 
 	memory@0 {
diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906-tplink-archer-c2300-v1.dts b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906-tplink-archer-c2300-v1.dts
index 064f7f549665..296393d4aaab 100644
--- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906-tplink-archer-c2300-v1.dts
+++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906-tplink-archer-c2300-v1.dts
@@ -7,7 +7,7 @@
 #include "bcm4906.dtsi"
 
 / {
-	compatible = "tplink,archer-c2300-v1", "brcm,bcm4906", "brcm,bcm4908";
+	compatible = "tplink,archer-c2300-v1", "brcm,bcm4906", "brcm,bcm4908", "brcm,bcmbca";
 	model = "TP-Link Archer C2300 V1";
 
 	memory@0 {
diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906.dtsi b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906.dtsi
index d084c33d5ca8..d084c33d5ca8 100644
--- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4906.dtsi
+++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4906.dtsi
diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-asus-gt-ac5300.dts b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908-asus-gt-ac5300.dts
index 04f8524b5335..839ca33178b0 100644
--- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-asus-gt-ac5300.dts
+++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908-asus-gt-ac5300.dts
@@ -2,11 +2,12 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
 
 #include "bcm4908.dtsi"
 
 / {
-	compatible = "asus,gt-ac5300", "brcm,bcm4908";
+	compatible = "asus,gt-ac5300", "brcm,bcm4908", "brcm,bcmbca";
 	model = "Asus GT-AC5300";
 
 	memory@0 {
@@ -118,6 +119,53 @@
 	};
 };
 
+&leds {
+	led-power@11 {
+		reg = <0x11>;
+		function = LED_FUNCTION_POWER;
+		color = <LED_COLOR_ID_WHITE>;
+		default-state = "on";
+		active-low;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pins_led_17_a>;
+	};
+
+	led-wan-red@12 {
+		reg = <0x12>;
+		function = LED_FUNCTION_WAN;
+		color = <LED_COLOR_ID_RED>;
+		active-low;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pins_led_18_a>;
+	};
+
+	led-wps@14 {
+		reg = <0x14>;
+		function = LED_FUNCTION_WPS;
+		color = <LED_COLOR_ID_WHITE>;
+		active-low;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pins_led_20_a>;
+	};
+
+	led-wan-white@15 {
+		reg = <0x15>;
+		function = LED_FUNCTION_WAN;
+		color = <LED_COLOR_ID_WHITE>;
+		active-low;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pins_led_21_a>;
+	};
+
+	led-lan@19 {
+		reg = <0x19>;
+		function = LED_FUNCTION_LAN;
+		color = <LED_COLOR_ID_WHITE>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pins_led_25_a>;
+	};
+};
+
 &nandcs {
 	nand-ecc-strength = <4>;
 	nand-ecc-step-size = <512>;
diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-netgear-raxe500.dts b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908-netgear-raxe500.dts
index 3c2cf2d238b6..23b96c663239 100644
--- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908-netgear-raxe500.dts
+++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908-netgear-raxe500.dts
@@ -3,7 +3,7 @@
 #include "bcm4908.dtsi"
 
 / {
-	compatible = "netgear,raxe500", "brcm,bcm4908";
+	compatible = "netgear,raxe500", "brcm,bcm4908", "brcm,bcmbca";
 	model = "Netgear RAXE500";
 
 	memory@0 {
diff --git a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908.dtsi
index 967d2cd3c3ce..dac9d3b4e91d 100644
--- a/arch/arm64/boot/dts/broadcom/bcm4908/bcm4908.dtsi
+++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm4908.dtsi
@@ -352,6 +352,61 @@
 				groups = "led_9_grp_a";
 			};
 
+			pins_led_10_a: led_10-a-pins {
+				function = "led_10";
+				groups = "led_10_grp_a";
+			};
+
+			pins_led_11_a: led_11-a-pins {
+				function = "led_11";
+				groups = "led_11_grp_a";
+			};
+
+			pins_led_12_a: led_12-a-pins {
+				function = "led_12";
+				groups = "led_12_grp_a";
+			};
+
+			pins_led_13_a: led_13-a-pins {
+				function = "led_13";
+				groups = "led_13_grp_a";
+			};
+
+			pins_led_14_a: led_14-a-pins {
+				function = "led_14";
+				groups = "led_14_grp_a";
+			};
+
+			pins_led_15_a: led_15-a-pins {
+				function = "led_15";
+				groups = "led_15_grp_a";
+			};
+
+			pins_led_16_a: led_16-a-pins {
+				function = "led_16";
+				groups = "led_16_grp_a";
+			};
+
+			pins_led_17_a: led_17-a-pins {
+				function = "led_17";
+				groups = "led_17_grp_a";
+			};
+
+			pins_led_18_a: led_18-a-pins {
+				function = "led_18";
+				groups = "led_18_grp_a";
+			};
+
+			pins_led_19_a: led_19-a-pins {
+				function = "led_19";
+				groups = "led_19_grp_a";
+			};
+
+			pins_led_20_a: led_20-a-pins {
+				function = "led_20";
+				groups = "led_20_grp_a";
+			};
+
 			pins_led_21_a: led_21-a-pins {
 				function = "led_21";
 				groups = "led_21_grp_a";
@@ -362,6 +417,21 @@
 				groups = "led_22_grp_a";
 			};
 
+			pins_led_23_a: led_23-a-pins {
+				function = "led_23";
+				groups = "led_23_grp_a";
+			};
+
+			pins_led_24_a: led_24-a-pins {
+				function = "led_24";
+				groups = "led_24_grp_a";
+			};
+
+			pins_led_25_a: led_25-a-pins {
+				function = "led_25";
+				groups = "led_25_grp_a";
+			};
+
 			pins_led_26_a: led_26-a-pins {
 				function = "led_26";
 				groups = "led_26_grp_a";
@@ -387,6 +457,11 @@
 				groups = "led_30_grp_a";
 			};
 
+			pins_led_31_a: led_31-a-pins {
+				function = "led_31";
+				groups = "led_31_grp_a";
+			};
+
 			pins_hs_uart: hs_uart-pins {
 				function = "hs_uart";
 				groups = "hs_uart_grp";
@@ -442,6 +517,14 @@
 			status = "okay";
 		};
 
+		leds: leds@800 {
+			compatible = "brcm,bcm4908-leds", "brcm,bcm63138-leds";
+			reg = <0x800 0xdc>;
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
 		nand-controller@1800 {
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm64/boot/dts/broadcom/bcmbca/bcm94908.dts b/arch/arm64/boot/dts/broadcom/bcmbca/bcm94908.dts
new file mode 100644
index 000000000000..fcbd3c430ace
--- /dev/null
+++ b/arch/arm64/boot/dts/broadcom/bcmbca/bcm94908.dts
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022 Broadcom Ltd.
+ */
+
+/dts-v1/;
+
+#include "bcm4908.dtsi"
+
+/ {
+	model = "Broadcom BCM94908 Reference Board";
+	compatible = "brcm,bcm94908", "brcm,bcm4908", "brcm,bcmbca";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x08000000>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi b/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi
index 91c9bd1b47dd..bde6a6bb8dfc 100644
--- a/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos5433-tm2-common.dtsi
@@ -795,7 +795,7 @@
 		reg = <0x27>;
 		interrupt-parent = <&gpa1>;
 		interrupts = <3 IRQ_TYPE_EDGE_RISING>;
-		en-gpios = <&gpf1 4 GPIO_ACTIVE_HIGH>;
+		en-gpios = <&gpf1 4 GPIO_ACTIVE_LOW>;
 		wake-gpios = <&gpj0 2 GPIO_ACTIVE_HIGH>;
 	};
 };
diff --git a/arch/arm64/boot/dts/exynos/exynos850.dtsi b/arch/arm64/boot/dts/exynos/exynos850.dtsi
index 9076afd4bb3e..c61441f3a89a 100644
--- a/arch/arm64/boot/dts/exynos/exynos850.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos850.dtsi
@@ -286,6 +286,21 @@
 			clock-names = "oscclk";
 		};
 
+		cmu_mfcmscl: clock-controller@12c00000 {
+			compatible = "samsung,exynos850-cmu-mfcmscl";
+			reg = <0x12c00000 0x8000>;
+			#clock-cells = <1>;
+
+			clocks = <&oscclk>,
+				 <&cmu_top CLK_DOUT_MFCMSCL_MFC>,
+				 <&cmu_top CLK_DOUT_MFCMSCL_M2M>,
+				 <&cmu_top CLK_DOUT_MFCMSCL_MCSC>,
+				 <&cmu_top CLK_DOUT_MFCMSCL_JPEG>;
+			clock-names = "oscclk", "dout_mfcmscl_mfc",
+				      "dout_mfcmscl_m2m", "dout_mfcmscl_mcsc",
+				      "dout_mfcmscl_jpeg";
+		};
+
 		cmu_dpu: clock-controller@13000000 {
 			compatible = "samsung,exynos850-cmu-dpu";
 			reg = <0x13000000 0x8000>;
@@ -308,6 +323,29 @@
 				      "dout_hsi_mmc_card", "dout_hsi_usb20drd";
 		};
 
+		cmu_is: clock-controller@14500000 {
+			compatible = "samsung,exynos850-cmu-is";
+			reg = <0x14500000 0x8000>;
+			#clock-cells = <1>;
+
+			clocks = <&oscclk>,
+				 <&cmu_top CLK_DOUT_IS_BUS>,
+				 <&cmu_top CLK_DOUT_IS_ITP>,
+				 <&cmu_top CLK_DOUT_IS_VRA>,
+				 <&cmu_top CLK_DOUT_IS_GDC>;
+			clock-names = "oscclk", "dout_is_bus", "dout_is_itp",
+				      "dout_is_vra", "dout_is_gdc";
+		};
+
+		cmu_aud: clock-controller@14a00000 {
+			compatible = "samsung,exynos850-cmu-aud";
+			reg = <0x14a00000 0x8000>;
+			#clock-cells = <1>;
+
+			clocks = <&oscclk>, <&cmu_top CLK_DOUT_AUD>;
+			clock-names = "oscclk", "dout_aud";
+		};
+
 		pinctrl_alive: pinctrl@11850000 {
 			compatible = "samsung,exynos850-pinctrl";
 			reg = <0x11850000 0x1000>;
@@ -465,6 +503,51 @@
 			status = "disabled";
 		};
 
+		sysmmu_mfcmscl: sysmmu@12c50000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x12c50000 0x9000>;
+			interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "sysmmu";
+			clocks = <&cmu_mfcmscl CLK_GOUT_MFCMSCL_SYSMMU_CLK>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_dpu: sysmmu@130c0000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x130c0000 0x9000>;
+			interrupts = <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "sysmmu";
+			clocks = <&cmu_dpu CLK_GOUT_DPU_SMMU_CLK>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_is0: sysmmu@14550000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x14550000 0x9000>;
+			interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "sysmmu";
+			clocks = <&cmu_is CLK_GOUT_IS_SYSMMU_IS0_CLK>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_is1: sysmmu@14570000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x14570000 0x9000>;
+			interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "sysmmu";
+			clocks = <&cmu_is CLK_GOUT_IS_SYSMMU_IS1_CLK>;
+			#iommu-cells = <0>;
+		};
+
+		sysmmu_aud: sysmmu@14850000 {
+			compatible = "samsung,exynos-sysmmu";
+			reg = <0x14850000 0x9000>;
+			interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
+			clock-names = "sysmmu";
+			clocks = <&cmu_aud CLK_GOUT_AUD_SYSMMU_CLK>;
+			#iommu-cells = <0>;
+		};
+
 		sysreg_peri: syscon@10020000 {
 			compatible = "samsung,exynos850-sysreg", "syscon";
 			reg = <0x10020000 0x10000>;
diff --git a/arch/arm64/boot/dts/exynos/exynosautov9.dtsi b/arch/arm64/boot/dts/exynos/exynosautov9.dtsi
index 2013718532f3..5dc361734cfe 100644
--- a/arch/arm64/boot/dts/exynos/exynosautov9.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynosautov9.dtsi
@@ -207,6 +207,34 @@
 				      "dout_clkcmu_peric1_ip";
 		};
 
+		cmu_fsys1: clock-controller@17040000 {
+			compatible = "samsung,exynosautov9-cmu-fsys1";
+			reg = <0x17040000 0x8000>;
+			#clock-cells = <1>;
+
+			clocks = <&xtcxo>,
+				 <&cmu_top DOUT_CLKCMU_FSYS1_BUS>,
+				 <&cmu_top GOUT_CLKCMU_FSYS1_MMC_CARD>,
+				 <&cmu_top DOUT_CLKCMU_FSYS1_USBDRD>;
+			clock-names = "oscclk",
+				      "dout_clkcmu_fsys1_bus",
+				      "gout_clkcmu_fsys1_mmc_card",
+				      "dout_clkcmu_fsys1_usbdrd";
+		};
+
+		cmu_fsys0: clock-controller@17700000 {
+			compatible = "samsung,exynosautov9-cmu-fsys0";
+			reg = <0x17700000 0x8000>;
+			#clock-cells = <1>;
+
+			clocks = <&xtcxo>,
+				 <&cmu_top DOUT_CLKCMU_FSYS0_BUS>,
+				 <&cmu_top DOUT_CLKCMU_FSYS0_PCIE>;
+			clock-names = "oscclk",
+				      "dout_clkcmu_fsys0_bus",
+				      "dout_clkcmu_fsys0_pcie";
+		};
+
 		cmu_fsys2: clock-controller@17c00000 {
 			compatible = "samsung,exynosautov9-cmu-fsys2";
 			reg = <0x17c00000 0x8000>;
diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile
index 8bf7f7ecebaa..3ea9edc87909 100644
--- a/arch/arm64/boot/dts/freescale/Makefile
+++ b/arch/arm64/boot/dts/freescale/Makefile
@@ -23,6 +23,7 @@ dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1088a-rdb.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1088a-ten64.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-qds.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-rdb.dtb
+dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2081a-rdb.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2080a-simu.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2088a-qds.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls2088a-rdb.dtb
@@ -48,6 +49,7 @@ dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-85bb.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-899b.dtb
 dtb-$(CONFIG_ARCH_LAYERSCAPE) += fsl-ls1028a-qds-9999.dtb
 
+dtb-$(CONFIG_ARCH_MXC) += imx8dxl-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-beacon-kit.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-data-modul-edm-sbc.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-ddr4-evk.dtb
@@ -55,7 +57,8 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mm-emcon-avari.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-icore-mx8mm-ctouch2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-icore-mx8mm-edimm2.2.dtb
-dtb-$(CONFIG_ARCH_MXC) += imx8mm-kontron-n801x-s.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mm-kontron-bl.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mm-kontron-bl-osm-s.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-mx8menlo.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-nitrogen-r2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-phyboard-polis-rdk.dtb
@@ -67,6 +70,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw73xx-0x.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw7901.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw7902.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw7903.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mm-venice-gw7904.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-nonwifi-dahlia.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-nonwifi-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-wifi-dahlia.dtb
@@ -83,6 +87,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mn-venice-gw7902.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-dhcom-pdk2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-icore-mx8mp-edimm2.2.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-msc-sm2s-ep1.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-phyboard-pollux-rdk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-tqma8mpql-mba8mpxl.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-venice-gw74xx.dtb
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-kbox-a-230-ls.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-kbox-a-230-ls.dts
index 6b575efd84a7..73eb6061c73e 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-kbox-a-230-ls.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-kbox-a-230-ls.dts
@@ -59,6 +59,10 @@
 	status = "okay";
 };
 
+&enetc_port3 {
+	status = "okay";
+};
+
 &i2c3 {
 	eeprom@57 {
 		compatible = "atmel,24c32";
@@ -104,7 +108,10 @@
 };
 
 &mscc_felix_port4 {
-	ethernet = <&enetc_port2>;
+	status = "okay";
+};
+
+&mscc_felix_port5 {
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts
index 330e34f933a3..113b1df74bf8 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-kontron-sl28-var2.dts
@@ -39,6 +39,10 @@
 	status = "okay";
 };
 
+&enetc_port3 {
+	status = "okay";
+};
+
 &mscc_felix {
 	status = "okay";
 };
@@ -60,6 +64,9 @@
 };
 
 &mscc_felix_port4 {
-	ethernet = <&enetc_port2>;
+	status = "okay";
+};
+
+&mscc_felix_port5 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts
index e0cd1516d05b..ecd2c1ea177f 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a-rdb.dts
@@ -29,6 +29,9 @@
 		ethernet3 = &mscc_felix_port1;
 		ethernet4 = &mscc_felix_port2;
 		ethernet5 = &mscc_felix_port3;
+		ethernet6 = &mscc_felix_port4;
+		ethernet7 = &mscc_felix_port5;
+		ethernet8 = &enetc_port3;
 	};
 
 	chosen {
@@ -151,6 +154,10 @@
 	status = "okay";
 };
 
+&enetc_port3 {
+	status = "okay";
+};
+
 &esdhc {
 	sd-uhs-sdr104;
 	sd-uhs-sdr50;
@@ -278,7 +285,10 @@
 };
 
 &mscc_felix_port4 {
-	ethernet = <&enetc_port2>;
+	status = "okay";
+};
+
+&mscc_felix_port5 {
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi
index 5627dd7734f3..ac1c3a7e5f7a 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1028a.dtsi
@@ -1156,6 +1156,7 @@
 					mscc_felix_port4: port@4 {
 						reg = <4>;
 						phy-mode = "internal";
+						ethernet = <&enetc_port2>;
 						status = "disabled";
 
 						fixed-link {
@@ -1168,6 +1169,7 @@
 					mscc_felix_port5: port@5 {
 						reg = <5>;
 						phy-mode = "internal";
+						ethernet = <&enetc_port3>;
 						status = "disabled";
 
 						fixed-link {
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts
index fea167d222cf..9b726c2a4842 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a-qds.dts
@@ -3,7 +3,7 @@
  * Device Tree Include file for Freescale Layerscape-1043A family SoC.
  *
  * Copyright 2014-2015 Freescale Semiconductor, Inc.
- * Copyright 2018 NXP
+ * Copyright 2018-2021 NXP
  *
  * Mingkai Hu <Mingkai.hu@freescale.com>
  */
@@ -24,6 +24,22 @@
 		serial1 = &duart1;
 		serial2 = &duart2;
 		serial3 = &duart3;
+		sgmii-riser-s1-p1 = &sgmii_phy_s1_p1;
+		sgmii-riser-s2-p1 = &sgmii_phy_s2_p1;
+		sgmii-riser-s3-p1 = &sgmii_phy_s3_p1;
+		sgmii-riser-s4-p1 = &sgmii_phy_s4_p1;
+		qsgmii-s1-p1 = &qsgmii_phy_s1_p1;
+		qsgmii-s1-p2 = &qsgmii_phy_s1_p2;
+		qsgmii-s1-p3 = &qsgmii_phy_s1_p3;
+		qsgmii-s1-p4 = &qsgmii_phy_s1_p4;
+		qsgmii-s2-p1 = &qsgmii_phy_s2_p1;
+		qsgmii-s2-p2 = &qsgmii_phy_s2_p2;
+		qsgmii-s2-p3 = &qsgmii_phy_s2_p3;
+		qsgmii-s2-p4 = &qsgmii_phy_s2_p4;
+		emi1-slot1 = &ls1043mdio_s1;
+		emi1-slot2 = &ls1043mdio_s2;
+		emi1-slot3 = &ls1043mdio_s3;
+		emi1-slot4 = &ls1043mdio_s4;
 	};
 
 	chosen {
@@ -62,8 +78,11 @@
 	};
 
 	fpga: board-control@2,0 {
-		compatible = "fsl,ls1043aqds-fpga", "fsl,fpga-qixis";
+		compatible = "fsl,ls1043aqds-fpga", "fsl,fpga-qixis", "simple-mfd";
 		reg = <0x2 0x0 0x0000100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 2 0 0x100>;
 	};
 };
 
@@ -153,3 +172,153 @@
 };
 
 #include "fsl-ls1043-post.dtsi"
+
+&fman0 {
+	ethernet@e0000 {
+		phy-handle = <&qsgmii_phy_s2_p1>;
+		phy-connection-type = "sgmii";
+	};
+
+	ethernet@e2000 {
+		phy-handle = <&qsgmii_phy_s2_p2>;
+		phy-connection-type = "sgmii";
+	};
+
+	ethernet@e4000 {
+		phy-handle = <&rgmii_phy1>;
+		phy-connection-type = "rgmii";
+	};
+
+	ethernet@e6000 {
+		phy-handle = <&rgmii_phy2>;
+		phy-connection-type = "rgmii";
+	};
+
+	ethernet@e8000 {
+		phy-handle = <&qsgmii_phy_s2_p3>;
+		phy-connection-type = "sgmii";
+	};
+
+	ethernet@ea000 {
+		phy-handle = <&qsgmii_phy_s2_p4>;
+		phy-connection-type = "sgmii";
+	};
+
+	ethernet@f0000 { /* DTSEC9/10GEC1 */
+		fixed-link = <1 1 10000 0 0>;
+		phy-connection-type = "xgmii";
+	};
+};
+
+&fpga {
+	mdio-mux-emi1@54 {
+		compatible = "mdio-mux-mmioreg", "mdio-mux";
+		mdio-parent-bus = <&mdio0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x54 1>;    /* BRDCFG4 */
+		mux-mask = <0xe0>; /* EMI1 */
+
+		/* On-board RGMII1 PHY */
+		ls1043mdio0: mdio@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			rgmii_phy1: ethernet-phy@1 { /* MAC3 */
+				reg = <0x1>;
+			};
+		};
+
+		/* On-board RGMII2 PHY */
+		ls1043mdio1: mdio@20 {
+			reg = <0x20>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			rgmii_phy2: ethernet-phy@2 { /* MAC4 */
+				reg = <0x2>;
+			};
+		};
+
+		/* Slot 1 */
+		ls1043mdio_s1: mdio@40 {
+			reg = <0x40>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			qsgmii_phy_s1_p1: ethernet-phy@4 {
+				reg = <0x4>;
+			};
+
+			qsgmii_phy_s1_p2: ethernet-phy@5 {
+				reg = <0x5>;
+			};
+
+			qsgmii_phy_s1_p3: ethernet-phy@6 {
+				reg = <0x6>;
+			};
+
+			qsgmii_phy_s1_p4: ethernet-phy@7 {
+				reg = <0x7>;
+			};
+
+			sgmii_phy_s1_p1: ethernet-phy@1c {
+				reg = <0x1c>;
+			};
+		};
+
+		/* Slot 2 */
+		ls1043mdio_s2: mdio@60 {
+			reg = <0x60>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			qsgmii_phy_s2_p1: ethernet-phy@8 {
+				reg = <0x8>;
+			};
+
+			qsgmii_phy_s2_p2: ethernet-phy@9 {
+				reg = <0x9>;
+			};
+
+			qsgmii_phy_s2_p3: ethernet-phy@a {
+				reg = <0xa>;
+			};
+
+			qsgmii_phy_s2_p4: ethernet-phy@b {
+				reg = <0xb>;
+			};
+
+			sgmii_phy_s2_p1: ethernet-phy@1c {
+				reg = <0x1c>;
+			};
+		};
+
+		/* Slot 3 */
+		ls1043mdio_s3: mdio@80 {
+			reg = <0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			sgmii_phy_s3_p1: ethernet-phy@1c {
+				reg = <0x1c>;
+			};
+		};
+
+		/* Slot 4 */
+		ls1043mdio_s4: mdio@a0 {
+			reg = <0xa0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			sgmii_phy_s4_p1: ethernet-phy@1c {
+				reg = <0x1c>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts
index b290605e92cf..26f8540cb101 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a-rdb.dts
@@ -29,23 +29,33 @@
 
 &i2c0 {
 	status = "okay";
+
 	ina220@40 {
 		compatible = "ti,ina220";
 		reg = <0x40>;
 		shunt-resistor = <1000>;
 	};
+
 	adt7461a@4c {
 		compatible = "adi,adt7461";
 		reg = <0x4c>;
 	};
+
+	rtc@51 {
+		compatible = "nxp,pcf85263";
+		reg = <0x51>;
+	};
+
 	eeprom@52 {
 		compatible = "atmel,24c512";
 		reg = <0x52>;
 	};
+
 	eeprom@53 {
 		compatible = "atmel,24c512";
 		reg = <0x53>;
 	};
+
 	rtc@68 {
 		compatible = "pericom,pt7c4338";
 		reg = <0x68>;
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi
index ca3d5a90d6d4..704f72caddd3 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1043a.dtsi
@@ -11,6 +11,7 @@
 #include <dt-bindings/clock/fsl,qoriq-clockgen.h>
 #include <dt-bindings/thermal/thermal.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/gpio/gpio.h>
 
 / {
 	compatible = "fsl,ls1043a";
@@ -300,6 +301,8 @@
 		#address-cells = <2>;
 		#size-cells = <2>;
 		ranges;
+		dma-ranges = <0x0 0x0 0x0 0x0 0x10000 0x00000000>;
+		dma-coherent;
 
 		clockgen: clocking@1ee1000 {
 			compatible = "fsl,ls1043a-clockgen";
@@ -393,7 +396,7 @@
 
 		dcfg: dcfg@1ee0000 {
 			compatible = "fsl,ls1043a-dcfg", "syscon";
-			reg = <0x0 0x1ee0000 0x0 0x10000>;
+			reg = <0x0 0x1ee0000 0x0 0x1000>;
 			big-endian;
 		};
 
@@ -536,7 +539,7 @@
 		};
 
 		i2c0: i2c@2180000 {
-			compatible = "fsl,vf610-i2c";
+			compatible = "fsl,ls1043a-i2c", "fsl,vf610-i2c";
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x0 0x2180000 0x0 0x10000>;
@@ -551,7 +554,7 @@
 		};
 
 		i2c1: i2c@2190000 {
-			compatible = "fsl,vf610-i2c";
+			compatible = "fsl,ls1043a-i2c", "fsl,vf610-i2c";
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x0 0x2190000 0x0 0x10000>;
@@ -559,11 +562,12 @@
 			clock-names = "i2c";
 			clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
 					    QORIQ_CLK_PLL_DIV(1)>;
+			scl-gpios = <&gpio4 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 			status = "disabled";
 		};
 
 		i2c2: i2c@21a0000 {
-			compatible = "fsl,vf610-i2c";
+			compatible = "fsl,ls1043a-i2c", "fsl,vf610-i2c";
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x0 0x21a0000 0x0 0x10000>;
@@ -571,11 +575,12 @@
 			clock-names = "i2c";
 			clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
 					    QORIQ_CLK_PLL_DIV(1)>;
+			scl-gpios = <&gpio4 10 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 			status = "disabled";
 		};
 
 		i2c3: i2c@21b0000 {
-			compatible = "fsl,vf610-i2c";
+			compatible = "fsl,ls1043a-i2c", "fsl,vf610-i2c";
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x0 0x21b0000 0x0 0x10000>;
@@ -583,6 +588,7 @@
 			clock-names = "i2c";
 			clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
 					    QORIQ_CLK_PLL_DIV(1)>;
+			scl-gpios = <&gpio4 12 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 			status = "disabled";
 		};
 
@@ -810,48 +816,59 @@
 					    QORIQ_CLK_PLL_DIV(1)>;
 		};
 
-		usb0: usb@2f00000 {
-			compatible = "snps,dwc3";
-			reg = <0x0 0x2f00000 0x0 0x10000>;
-			interrupts = <0 60 0x4>;
-			dr_mode = "host";
-			snps,quirk-frame-length-adjustment = <0x20>;
-			snps,dis_rxdet_inp3_quirk;
-			snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
-			status = "disabled";
-		};
+		aux_bus: aux_bus {
+			#address-cells = <2>;
+			#size-cells = <2>;
+			compatible = "simple-bus";
+			ranges;
+			dma-ranges = <0x0 0x0 0x0 0x0 0x100 0x00000000>;
+
+			usb0: usb@2f00000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0x2f00000 0x0 0x10000>;
+				interrupts = <0 60 IRQ_TYPE_LEVEL_HIGH>;
+				dr_mode = "host";
+				snps,quirk-frame-length-adjustment = <0x20>;
+				snps,dis_rxdet_inp3_quirk;
+				usb3-lpm-capable;
+				snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
+				status = "disabled";
+			};
 
-		usb1: usb@3000000 {
-			compatible = "snps,dwc3";
-			reg = <0x0 0x3000000 0x0 0x10000>;
-			interrupts = <0 61 0x4>;
-			dr_mode = "host";
-			snps,quirk-frame-length-adjustment = <0x20>;
-			snps,dis_rxdet_inp3_quirk;
-			snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
-			status = "disabled";
-		};
+			usb1: usb@3000000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0x3000000 0x0 0x10000>;
+				interrupts = <0 61 IRQ_TYPE_LEVEL_HIGH>;
+				dr_mode = "host";
+				snps,quirk-frame-length-adjustment = <0x20>;
+				snps,dis_rxdet_inp3_quirk;
+				usb3-lpm-capable;
+				snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
+				status = "disabled";
+			};
 
-		usb2: usb@3100000 {
-			compatible = "snps,dwc3";
-			reg = <0x0 0x3100000 0x0 0x10000>;
-			interrupts = <0 63 0x4>;
-			dr_mode = "host";
-			snps,quirk-frame-length-adjustment = <0x20>;
-			snps,dis_rxdet_inp3_quirk;
-			snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
-			status = "disabled";
-		};
+			usb2: usb@3100000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0x3100000 0x0 0x10000>;
+				interrupts = <0 63 IRQ_TYPE_LEVEL_HIGH>;
+				dr_mode = "host";
+				snps,quirk-frame-length-adjustment = <0x20>;
+				snps,dis_rxdet_inp3_quirk;
+				usb3-lpm-capable;
+				snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
+				status = "disabled";
+			};
 
-		sata: sata@3200000 {
-			compatible = "fsl,ls1043a-ahci";
-			reg = <0x0 0x3200000 0x0 0x10000>,
-				<0x0 0x20140520 0x0 0x4>;
-			reg-names = "ahci", "sata-ecc";
-			interrupts = <0 69 0x4>;
-			clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
-					    QORIQ_CLK_PLL_DIV(1)>;
-			dma-coherent;
+			sata: sata@3200000 {
+				compatible = "fsl,ls1043a-ahci";
+				reg = <0x0 0x3200000 0x0 0x10000>,
+					<0x0 0x20140520 0x0 0x4>;
+				reg-names = "ahci", "sata-ecc";
+				interrupts = <0 69 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+						    QORIQ_CLK_PLL_DIV(1)>;
+				dma-coherent;
+			};
 		};
 
 		msi1: msi-controller1@1571000 {
@@ -880,13 +897,12 @@
 			reg = <0x00 0x03400000 0x0 0x00100000>, /* controller registers */
 			      <0x40 0x00000000 0x0 0x00002000>; /* configuration space */
 			reg-names = "regs", "config";
-			interrupts = <0 118 0x4>, /* controller interrupt */
-				     <0 117 0x4>; /* PME interrupt */
-			interrupt-names = "intr", "pme";
+			interrupts = <0 117 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 118 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pme", "aer";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
-			dma-coherent;
 			num-viewport = <6>;
 			bus-range = <0x0 0xff>;
 			ranges = <0x81000000 0x0 0x00000000 0x40 0x00010000 0x0 0x00010000   /* downstream I/O */
@@ -898,6 +914,8 @@
 					<0000 0 0 2 &gic 0 111 0x4>,
 					<0000 0 0 3 &gic 0 112 0x4>,
 					<0000 0 0 4 &gic 0 113 0x4>;
+			fsl,pcie-scfg = <&scfg 0>;
+			big-endian;
 			status = "disabled";
 		};
 
@@ -906,13 +924,12 @@
 			reg = <0x00 0x03500000 0x0 0x00100000>, /* controller registers */
 			      <0x48 0x00000000 0x0 0x00002000>; /* configuration space */
 			reg-names = "regs", "config";
-			interrupts = <0 128 0x4>,
-				     <0 127 0x4>;
-			interrupt-names = "intr", "pme";
+			interrupts = <0 127 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 128 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pme", "aer";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
-			dma-coherent;
 			num-viewport = <6>;
 			bus-range = <0x0 0xff>;
 			ranges = <0x81000000 0x0 0x00000000 0x48 0x00010000 0x0 0x00010000   /* downstream I/O */
@@ -924,6 +941,8 @@
 					<0000 0 0 2 &gic 0 121 0x4>,
 					<0000 0 0 3 &gic 0 122 0x4>,
 					<0000 0 0 4 &gic 0 123 0x4>;
+			fsl,pcie-scfg = <&scfg 1>;
+			big-endian;
 			status = "disabled";
 		};
 
@@ -932,13 +951,12 @@
 			reg = <0x00 0x03600000 0x0 0x00100000>, /* controller registers */
 			      <0x50 0x00000000 0x0 0x00002000>; /* configuration space */
 			reg-names = "regs", "config";
-			interrupts = <0 162 0x4>,
-				     <0 161 0x4>;
-			interrupt-names = "intr", "pme";
+			interrupts = <0 161 IRQ_TYPE_LEVEL_HIGH>,
+				     <0 162 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pme", "aer";
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
-			dma-coherent;
 			num-viewport = <6>;
 			bus-range = <0x0 0xff>;
 			ranges = <0x81000000 0x0 0x00000000 0x50 0x00010000 0x0 0x00010000   /* downstream I/O */
@@ -950,6 +968,8 @@
 					<0000 0 0 2 &gic 0 155 0x4>,
 					<0000 0 0 3 &gic 0 156 0x4>,
 					<0000 0 0 4 &gic 0 157 0x4>;
+			fsl,pcie-scfg = <&scfg 2>;
+			big-endian;
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts b/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts
index eec62c63dafe..b2fcbba60d3a 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a-qds.dts
@@ -3,7 +3,7 @@
  * Device Tree Include file for Freescale Layerscape-1046A family SoC.
  *
  * Copyright 2016 Freescale Semiconductor, Inc.
- * Copyright 2018 NXP
+ * Copyright 2018-2019 NXP
  *
  * Shaohui Xie <Shaohui.Xie@nxp.com>
  */
@@ -17,14 +17,26 @@
 	compatible = "fsl,ls1046a-qds", "fsl,ls1046a";
 
 	aliases {
+		emi1-slot1 = &ls1046mdio_s1;
+		emi1-slot2 = &ls1046mdio_s2;
+		emi1-slot4 = &ls1046mdio_s4;
 		gpio0 = &gpio0;
 		gpio1 = &gpio1;
 		gpio2 = &gpio2;
 		gpio3 = &gpio3;
+		qsgmii-s2-p1 = &qsgmii_phy_s2_p1;
+		qsgmii-s2-p2 = &qsgmii_phy_s2_p2;
+		qsgmii-s2-p3 = &qsgmii_phy_s2_p3;
+		qsgmii-s2-p4 = &qsgmii_phy_s2_p4;
 		serial0 = &duart0;
 		serial1 = &duart1;
 		serial2 = &duart2;
 		serial3 = &duart3;
+		sgmii-s1-p1 = &sgmii_phy_s1_p1;
+		sgmii-s1-p2 = &sgmii_phy_s1_p2;
+		sgmii-s1-p3 = &sgmii_phy_s1_p3;
+		sgmii-s1-p4 = &sgmii_phy_s1_p4;
+		sgmii-s4-p1 = &sgmii_phy_s4_p1;
 	};
 
 	chosen {
@@ -153,8 +165,9 @@
 	};
 
 	fpga: board-control@2,0 {
-		compatible = "fsl,ls1046aqds-fpga", "fsl,fpga-qixis";
+		compatible = "fsl,ls1046aqds-fpga", "fsl,fpga-qixis", "simple-mfd";
 		reg = <0x2 0x0 0x0000100>;
+		ranges = <0 2 0 0x100>;
 	};
 };
 
@@ -169,7 +182,7 @@
 		compatible = "spansion,m25p80";
 		#address-cells = <1>;
 		#size-cells = <1>;
-		spi-max-frequency = <20000000>;
+		spi-max-frequency = <50000000>;
 		spi-rx-bus-width = <4>;
 		spi-tx-bus-width = <4>;
 		reg = <0>;
@@ -177,3 +190,141 @@
 };
 
 #include "fsl-ls1046-post.dtsi"
+
+&fman0 {
+	ethernet@e0000 {
+		phy-handle = <&qsgmii_phy_s2_p1>;
+		phy-connection-type = "sgmii";
+	};
+
+	ethernet@e2000 {
+		phy-handle = <&sgmii_phy_s4_p1>;
+		phy-connection-type = "sgmii";
+	};
+
+	ethernet@e4000 {
+		phy-handle = <&rgmii_phy1>;
+		phy-connection-type = "rgmii";
+	};
+
+	ethernet@e6000 {
+		phy-handle = <&rgmii_phy2>;
+		phy-connection-type = "rgmii";
+	};
+
+	ethernet@e8000 {
+		phy-handle = <&sgmii_phy_s1_p3>;
+		phy-connection-type = "sgmii";
+	};
+
+	ethernet@ea000 {
+		phy-handle = <&sgmii_phy_s1_p4>;
+		phy-connection-type = "sgmii";
+	};
+
+	ethernet@f0000 { /* DTSEC9/10GEC1 */
+		phy-handle = <&sgmii_phy_s1_p1>;
+		phy-connection-type = "xgmii";
+	};
+
+	ethernet@f2000 { /* DTSEC10/10GEC2 */
+		phy-handle = <&sgmii_phy_s1_p2>;
+		phy-connection-type = "xgmii";
+	};
+};
+
+&fpga {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	mdio-mux-emi1 {
+		compatible = "mdio-mux-mmioreg", "mdio-mux";
+		mdio-parent-bus = <&mdio0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x54 1>;    /* BRDCFG4 */
+		mux-mask = <0xe0>; /* EMI1 */
+
+		/* On-board RGMII1 PHY */
+		ls1046mdio0: mdio@0 {
+			reg = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			rgmii_phy1: ethernet-phy@1 { /* MAC3 */
+				reg = <0x1>;
+			};
+		};
+
+		/* On-board RGMII2 PHY */
+		ls1046mdio1: mdio@1 {
+			reg = <0x20>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			rgmii_phy2: ethernet-phy@2 { /* MAC4 */
+				reg = <0x2>;
+			};
+		};
+
+		/* Slot 1 */
+		ls1046mdio_s1: mdio@2 {
+			reg = <0x40>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			sgmii_phy_s1_p1: ethernet-phy@1c {
+				reg = <0x1c>;
+			};
+
+			sgmii_phy_s1_p2: ethernet-phy@1d {
+				reg = <0x1d>;
+			};
+
+			sgmii_phy_s1_p3: ethernet-phy@1e {
+				reg = <0x1e>;
+			};
+
+			sgmii_phy_s1_p4: ethernet-phy@1f {
+				reg = <0x1f>;
+			};
+		};
+
+		/* Slot 2 */
+		ls1046mdio_s2: mdio@3 {
+			reg = <0x60>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			qsgmii_phy_s2_p1: ethernet-phy@8 {
+				reg = <0x8>;
+			};
+
+			qsgmii_phy_s2_p2: ethernet-phy@9 {
+				reg = <0x9>;
+			};
+
+			qsgmii_phy_s2_p3: ethernet-phy@a {
+				reg = <0xa>;
+			};
+
+			qsgmii_phy_s2_p4: ethernet-phy@b {
+				reg = <0xb>;
+			};
+		};
+
+		/* Slot 4 */
+		ls1046mdio_s4: mdio@5 {
+			reg = <0x80>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+
+			sgmii_phy_s4_p1: ethernet-phy@1c {
+				reg = <0x1c>;
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi
index feab604322cf..3d9e29824bb2 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls1046a.dtsi
@@ -11,6 +11,7 @@
 #include <dt-bindings/clock/fsl,qoriq-clockgen.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/thermal/thermal.h>
+#include <dt-bindings/gpio/gpio.h>
 
 / {
 	compatible = "fsl,ls1046a";
@@ -272,6 +273,8 @@
 		#address-cells = <2>;
 		#size-cells = <2>;
 		ranges;
+		dma-ranges = <0x0 0x0 0x0 0x0 0x10000 0x00000000>;
+		dma-coherent;
 
 		ddr: memory-controller@1080000 {
 			compatible = "fsl,qoriq-memory-controller";
@@ -354,7 +357,6 @@
 			ranges = <0x0 0x00 0x1700000 0x100000>;
 			reg = <0x00 0x1700000 0x0 0x100000>;
 			interrupts = <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
-			dma-coherent;
 
 			sec_jr0: jr@10000 {
 				compatible = "fsl,sec-v5.4-job-ring",
@@ -500,7 +502,7 @@
 		};
 
 		i2c0: i2c@2180000 {
-			compatible = "fsl,vf610-i2c";
+			compatible = "fsl,ls1046a-i2c", "fsl,vf610-i2c";
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x0 0x2180000 0x0 0x10000>;
@@ -514,35 +516,38 @@
 		};
 
 		i2c1: i2c@2190000 {
-			compatible = "fsl,vf610-i2c";
+			compatible = "fsl,ls1046a-i2c", "fsl,vf610-i2c";
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x0 0x2190000 0x0 0x10000>;
 			interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
 					    QORIQ_CLK_PLL_DIV(2)>;
+			scl-gpios = <&gpio3 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 			status = "disabled";
 		};
 
 		i2c2: i2c@21a0000 {
-			compatible = "fsl,vf610-i2c";
+			compatible = "fsl,ls1046a-i2c", "fsl,vf610-i2c";
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x0 0x21a0000 0x0 0x10000>;
 			interrupts = <GIC_SPI 58 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
 					    QORIQ_CLK_PLL_DIV(2)>;
+			scl-gpios = <&gpio3 10 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 			status = "disabled";
 		};
 
 		i2c3: i2c@21b0000 {
-			compatible = "fsl,vf610-i2c";
+			compatible = "fsl,ls1046a-i2c", "fsl,vf610-i2c";
 			#address-cells = <1>;
 			#size-cells = <0>;
 			reg = <0x0 0x21b0000 0x0 0x10000>;
 			interrupts = <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
 					    QORIQ_CLK_PLL_DIV(2)>;
+			scl-gpios = <&gpio3 12 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 			status = "disabled";
 		};
 
@@ -709,44 +714,55 @@
 					    QORIQ_CLK_PLL_DIV(2)>;
 		};
 
-		usb0: usb@2f00000 {
-			compatible = "snps,dwc3";
-			reg = <0x0 0x2f00000 0x0 0x10000>;
-			interrupts = <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>;
-			dr_mode = "host";
-			snps,quirk-frame-length-adjustment = <0x20>;
-			snps,dis_rxdet_inp3_quirk;
-			snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
-		};
-
-		usb1: usb@3000000 {
-			compatible = "snps,dwc3";
-			reg = <0x0 0x3000000 0x0 0x10000>;
-			interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
-			dr_mode = "host";
-			snps,quirk-frame-length-adjustment = <0x20>;
-			snps,dis_rxdet_inp3_quirk;
-			snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
-		};
-
-		usb2: usb@3100000 {
-			compatible = "snps,dwc3";
-			reg = <0x0 0x3100000 0x0 0x10000>;
-			interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
-			dr_mode = "host";
-			snps,quirk-frame-length-adjustment = <0x20>;
-			snps,dis_rxdet_inp3_quirk;
-			snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
-		};
-
-		sata: sata@3200000 {
-			compatible = "fsl,ls1046a-ahci";
-			reg = <0x0 0x3200000 0x0 0x10000>,
-				<0x0 0x20140520 0x0 0x4>;
-			reg-names = "ahci", "sata-ecc";
-			interrupts = <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
-					    QORIQ_CLK_PLL_DIV(2)>;
+		aux_bus: aux_bus {
+			#address-cells = <2>;
+			#size-cells = <2>;
+			compatible = "simple-bus";
+			ranges;
+			dma-ranges = <0x0 0x0 0x0 0x0 0x100 0x00000000>;
+
+			usb0: usb@2f00000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0x2f00000 0x0 0x10000>;
+				interrupts = <GIC_SPI 60 IRQ_TYPE_LEVEL_HIGH>;
+				dr_mode = "host";
+				snps,quirk-frame-length-adjustment = <0x20>;
+				snps,dis_rxdet_inp3_quirk;
+				snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
+				usb3-lpm-capable;
+			};
+
+			usb1: usb@3000000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0x3000000 0x0 0x10000>;
+				interrupts = <GIC_SPI 61 IRQ_TYPE_LEVEL_HIGH>;
+				dr_mode = "host";
+				snps,quirk-frame-length-adjustment = <0x20>;
+				snps,dis_rxdet_inp3_quirk;
+				snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
+				usb3-lpm-capable;
+			};
+
+			usb2: usb@3100000 {
+				compatible = "snps,dwc3";
+				reg = <0x0 0x3100000 0x0 0x10000>;
+				interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
+				dr_mode = "host";
+				snps,quirk-frame-length-adjustment = <0x20>;
+				snps,dis_rxdet_inp3_quirk;
+				snps,incr-burst-type-adjustment = <1>, <4>, <8>, <16>;
+				usb3-lpm-capable;
+			};
+
+			sata: sata@3200000 {
+				compatible = "fsl,ls1046a-ahci";
+				reg = <0x0 0x3200000 0x0 0x10000>,
+					<0x0 0x20140520 0x0 0x4>;
+				reg-names = "ahci", "sata-ecc";
+				interrupts = <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clockgen QORIQ_CLK_PLATFORM_PLL
+						    QORIQ_CLK_PLL_DIV(2)>;
+			};
 		};
 
 		msi1: msi-controller@1580000 {
@@ -790,7 +806,6 @@
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
-			dma-coherent;
 			num-viewport = <8>;
 			bus-range = <0x0 0xff>;
 			ranges = <0x81000000 0x0 0x00000000 0x40 0x00010000 0x0 0x00010000   /* downstream I/O */
@@ -802,6 +817,7 @@
 					<0000 0 0 2 &gic GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>,
 					<0000 0 0 3 &gic GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>,
 					<0000 0 0 4 &gic GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
+			big-endian;
 			status = "disabled";
 		};
 
@@ -810,8 +826,11 @@
 			reg = <0x00 0x03400000 0x0 0x00100000>,
 			      <0x40 0x00000000 0x8 0x00000000>;
 			reg-names = "regs", "addr_space";
+			interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pme";
 			num-ib-windows = <6>;
 			num-ob-windows = <8>;
+			big-endian;
 			status = "disabled";
 		};
 
@@ -826,7 +845,6 @@
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
-			dma-coherent;
 			num-viewport = <8>;
 			bus-range = <0x0 0xff>;
 			ranges = <0x81000000 0x0 0x00000000 0x48 0x00010000 0x0 0x00010000   /* downstream I/O */
@@ -838,6 +856,7 @@
 					<0000 0 0 2 &gic GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
 					<0000 0 0 3 &gic GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
 					<0000 0 0 4 &gic GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+			big-endian;
 			status = "disabled";
 		};
 
@@ -846,8 +865,11 @@
 			reg = <0x00 0x03500000 0x0 0x00100000>,
 			      <0x48 0x00000000 0x8 0x00000000>;
 			reg-names = "regs", "addr_space";
+			interrupts = <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pme";
 			num-ib-windows = <6>;
 			num-ob-windows = <8>;
+			big-endian;
 			status = "disabled";
 		};
 
@@ -862,7 +884,6 @@
 			#address-cells = <3>;
 			#size-cells = <2>;
 			device_type = "pci";
-			dma-coherent;
 			num-viewport = <8>;
 			bus-range = <0x0 0xff>;
 			ranges = <0x81000000 0x0 0x00000000 0x50 0x00010000 0x0 0x00010000   /* downstream I/O */
@@ -874,6 +895,7 @@
 					<0000 0 0 2 &gic GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>,
 					<0000 0 0 3 &gic GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>,
 					<0000 0 0 4 &gic GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>;
+			big-endian;
 			status = "disabled";
 		};
 
@@ -882,8 +904,11 @@
 			reg = <0x00 0x03600000 0x0 0x00100000>,
 			      <0x50 0x00000000 0x8 0x00000000>;
 			reg-names = "regs", "addr_space";
+			interrupts = <GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "pme";
 			num-ib-windows = <6>;
 			num-ob-windows = <8>;
+			big-endian;
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts
index 44894356059c..8b6915136997 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a-rdb.dts
@@ -14,6 +14,7 @@
 
 #include "fsl-ls2080a.dtsi"
 #include "fsl-ls208xa-rdb.dtsi"
+#include <dt-bindings/interrupt-controller/arm-gic.h>
 
 / {
 	model = "Freescale Layerscape 2080a RDB Board";
@@ -23,3 +24,71 @@
 		stdout-path = "serial1:115200n8";
 	};
 };
+
+&dpmac5 {
+	phy-handle = <&mdio2_phy1>;
+	phy-connection-type = "10gbase-r";
+};
+
+&dpmac6 {
+	phy-handle = <&mdio2_phy2>;
+	phy-connection-type = "10gbase-r";
+};
+
+&dpmac7 {
+	phy-handle = <&mdio2_phy3>;
+	phy-connection-type = "10gbase-r";
+};
+
+&dpmac8 {
+	phy-handle = <&mdio2_phy4>;
+	phy-connection-type = "10gbase-r";
+};
+
+&emdio1 {
+	status = "disabled";
+
+	/* CS4340 PHYs */
+	mdio1_phy1: emdio1-phy@10 {
+		reg = <0x10>;
+	};
+
+	mdio1_phy2: emdio1-phy@11 {
+		reg = <0x11>;
+	};
+
+	mdio1_phy3: emdio1-phy@12 {
+		reg = <0x12>;
+	};
+
+	mdio1_phy4: emdio1-phy@13 {
+		reg = <0x13>;
+	};
+};
+
+&emdio2 {
+	/* AQR405 PHYs */
+	mdio2_phy1: emdio2-phy@0 {
+		compatible = "ethernet-phy-ieee802.3-c45";
+		interrupts = <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>;
+		reg = <0x0>;
+	};
+
+	mdio2_phy2: emdio2-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c45";
+		interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
+		reg = <0x1>;
+	};
+
+	mdio2_phy3: emdio2-phy@2 {
+		compatible = "ethernet-phy-ieee802.3-c45";
+		interrupts = <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>;
+		reg = <0x2>;
+	};
+
+	mdio2_phy4: emdio2-phy@3 {
+		compatible = "ethernet-phy-ieee802.3-c45";
+		interrupts = <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>;
+		reg = <0x3>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi
index 6f6667b70028..a2cadf757148 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2080a.dtsi
@@ -150,3 +150,7 @@
 	ranges = <0x81000000 0x0 0x00000000 0x16 0x00010000 0x0 0x00010000   /* downstream I/O */
 		  0x82000000 0x0 0x40000000 0x16 0x40000000 0x0 0x40000000>; /* non-prefetchable memory */
 };
+
+&timer {
+	fsl,erratum-a008585;
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls2081a-rdb.dts b/arch/arm64/boot/dts/freescale/fsl-ls2081a-rdb.dts
new file mode 100644
index 000000000000..4461e16fd53a
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/fsl-ls2081a-rdb.dts
@@ -0,0 +1,132 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Device Tree file for NXP LS2081A RDB Board.
+ *
+ * Copyright 2017 NXP
+ *
+ * Priyanka Jain <priyanka.jain@nxp.com>
+ *
+ */
+
+/dts-v1/;
+
+#include "fsl-ls2088a.dtsi"
+
+/ {
+	model = "NXP Layerscape 2081A RDB Board";
+	compatible = "fsl,ls2081a-rdb", "fsl,ls2081a";
+
+	aliases {
+		serial0 = &serial0;
+		serial1 = &serial1;
+	};
+
+	chosen {
+		stdout-path = "serial1:115200n8";
+	};
+};
+
+&dspi {
+	status = "okay";
+
+	n25q512a: flash@0 {
+		compatible = "jedec,spi-nor";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <3000000>;
+		reg = <0>;
+	};
+};
+
+&esdhc {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	pca9547: mux@75 {
+		compatible = "nxp,pca9547";
+		reg = <0x75>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x1>;
+
+			rtc@51 {
+				compatible = "nxp,pcf2129";
+				reg = <0x51>;
+			};
+		};
+
+		i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x2>;
+
+			ina220@40 {
+				compatible = "ti,ina220";
+				reg = <0x40>;
+				shunt-resistor = <500>;
+			};
+		};
+
+		i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x3>;
+
+			adt7481@4c {
+				compatible = "adi,adt7461";
+				reg = <0x4c>;
+			};
+		};
+	};
+};
+
+&ifc {
+	status = "disabled";
+};
+
+&qspi {
+	status = "okay";
+
+	s25fs512s0: flash@0 {
+		compatible = "jedec,spi-nor";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
+		spi-max-frequency = <20000000>;
+		reg = <0>;
+	};
+
+	s25fs512s1: flash@1 {
+		compatible = "jedec,spi-nor";
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
+		spi-max-frequency = <20000000>;
+		reg = <1>;
+	};
+};
+
+&sata0 {
+	status = "okay";
+};
+
+&sata1 {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+};
+
+&usb1 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa-qds.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa-qds.dtsi
index 6fab73d484b6..f598669e742f 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls208xa-qds.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa-qds.dtsi
@@ -9,6 +9,27 @@
  *
  */
 
+/* Update DPMAC connections to external PHYs, under SerDes 0x2a_0x49. */
+&dpmac9 {
+	phy-handle = <&mdio0_phy12>;
+	phy-connection-type = "sgmii";
+};
+
+&dpmac10 {
+	phy-handle = <&mdio0_phy13>;
+	phy-connection-type = "sgmii";
+};
+
+&dpmac11 {
+	phy-handle = <&mdio0_phy14>;
+	phy-connection-type = "sgmii";
+};
+
+&dpmac12 {
+	phy-handle = <&mdio0_phy15>;
+	phy-connection-type = "sgmii";
+};
+
 &esdhc {
 	mmc-hs200-1_8v;
 	status = "okay";
@@ -36,9 +57,47 @@
 	     reg = <0x2 0x0 0x10000>;
 	};
 
-	cpld@3,0 {
-	     reg = <0x3 0x0 0x10000>;
-	     compatible = "fsl,ls2080aqds-fpga", "fsl,fpga-qixis";
+	boardctrl: board-control@3,0 {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "fsl,ls208xaqds-fpga", "fsl,fpga-qixis", "simple-mfd";
+		reg = <3 0 0x1000>;
+		ranges = <0 3 0 0x1000>;
+
+		mdio-mux-emi1@54 {
+			compatible = "mdio-mux-mmioreg", "mdio-mux";
+			mdio-parent-bus = <&emdio1>;
+			reg = <0x54 1>;		/* BRDCFG4 */
+			mux-mask = <0xe0>;	/* EMI1_MDIO */
+			#address-cells=<1>;
+			#size-cells = <0>;
+
+			/* Child MDIO buses, one for each riser card:
+			 * reg = 0x0, 0x20, 0x40, 0x60, 0x80, 0xa0.
+			 * VSC8234 PHYs on the riser cards.
+			 */
+			mdio_mux3: mdio@60 {
+				reg = <0x60>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				mdio0_phy12: mdio-phy0@1c {
+					reg = <0x1c>;
+				};
+
+				mdio0_phy13: mdio-phy1@1d {
+					reg = <0x1d>;
+				};
+
+				mdio0_phy14: mdio-phy2@1e {
+					reg = <0x1e>;
+				};
+
+				mdio0_phy15: mdio-phy3@1f {
+					reg = <0x1f>;
+				};
+			};
+		};
 	};
 };
 
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi
index f8135c5c252d..3d9647b3da14 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa-rdb.dtsi
@@ -49,6 +49,8 @@
 		reg = <0x75>;
 		#address-cells = <1>;
 		#size-cells = <0>;
+		idle-state = <0>;
+
 		i2c@1 {
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi
index d76f1c42f3fa..f1b9cc8714dc 100644
--- a/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi
+++ b/arch/arm64/boot/dts/freescale/fsl-ls208xa.dtsi
@@ -239,13 +239,12 @@
 		};
 	};
 
-	timer {
+	timer: timer {
 		compatible = "arm,armv8-timer";
 		interrupts = <1 13 4>, /* Physical Secure PPI, active-low */
 			     <1 14 4>, /* Physical Non-Secure PPI, active-low */
 			     <1 11 4>, /* Virtual PPI, active-low */
 			     <1 10 4>; /* Hypervisor PPI, active-low */
-		fsl,erratum-a008585;
 	};
 
 	pmu {
diff --git a/arch/arm64/boot/dts/freescale/imx8-ss-ddr.dtsi b/arch/arm64/boot/dts/freescale/imx8-ss-ddr.dtsi
index 8b5cad4e2700..7d5183c6c5be 100644
--- a/arch/arm64/boot/dts/freescale/imx8-ss-ddr.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-ss-ddr.dtsi
@@ -10,7 +10,7 @@ ddr_subsys: bus@5c000000 {
 	#size-cells = <1>;
 	ranges = <0x5c000000 0x0 0x5c000000 0x1000000>;
 
-	ddr-pmu@5c020000 {
+	ddr_pmu0: ddr-pmu@5c020000 {
 		compatible = "fsl,imx8-ddr-pmu";
 		reg = <0x5c020000 0x10000>;
 		interrupts = <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/boot/dts/freescale/imx8-ss-dma.dtsi b/arch/arm64/boot/dts/freescale/imx8-ss-dma.dtsi
index 960a802b8b6e..d7b4229bb4a2 100644
--- a/arch/arm64/boot/dts/freescale/imx8-ss-dma.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8-ss-dma.dtsi
@@ -111,8 +111,9 @@ dma_subsys: bus@5a000000 {
 	i2c0: i2c@5a800000 {
 		reg = <0x5a800000 0x4000>;
 		interrupts = <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&i2c0_lpcg IMX_LPCG_CLK_0>;
-		clock-names = "per";
+		clocks = <&i2c0_lpcg IMX_LPCG_CLK_0>,
+			 <&i2c0_lpcg IMX_LPCG_CLK_4>;
+		clock-names = "per", "ipg";
 		assigned-clocks = <&clk IMX_SC_R_I2C_0 IMX_SC_PM_CLK_PER>;
 		assigned-clock-rates = <24000000>;
 		power-domains = <&pd IMX_SC_R_I2C_0>;
@@ -122,8 +123,9 @@ dma_subsys: bus@5a000000 {
 	i2c1: i2c@5a810000 {
 		reg = <0x5a810000 0x4000>;
 		interrupts = <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&i2c1_lpcg IMX_LPCG_CLK_0>;
-		clock-names = "per";
+		clocks = <&i2c1_lpcg IMX_LPCG_CLK_0>,
+			 <&i2c1_lpcg IMX_LPCG_CLK_4>;
+		clock-names = "per", "ipg";
 		assigned-clocks = <&clk IMX_SC_R_I2C_1 IMX_SC_PM_CLK_PER>;
 		assigned-clock-rates = <24000000>;
 		power-domains = <&pd IMX_SC_R_I2C_1>;
@@ -133,8 +135,9 @@ dma_subsys: bus@5a000000 {
 	i2c2: i2c@5a820000 {
 		reg = <0x5a820000 0x4000>;
 		interrupts = <GIC_SPI 222 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&i2c2_lpcg IMX_LPCG_CLK_0>;
-		clock-names = "per";
+		clocks = <&i2c2_lpcg IMX_LPCG_CLK_0>,
+			 <&i2c2_lpcg IMX_LPCG_CLK_4>;
+		clock-names = "per", "ipg";
 		assigned-clocks = <&clk IMX_SC_R_I2C_2 IMX_SC_PM_CLK_PER>;
 		assigned-clock-rates = <24000000>;
 		power-domains = <&pd IMX_SC_R_I2C_2>;
@@ -144,8 +147,9 @@ dma_subsys: bus@5a000000 {
 	i2c3: i2c@5a830000 {
 		reg = <0x5a830000 0x4000>;
 		interrupts = <GIC_SPI 223 IRQ_TYPE_LEVEL_HIGH>;
-		clocks = <&i2c3_lpcg IMX_LPCG_CLK_0>;
-		clock-names = "per";
+		clocks = <&i2c3_lpcg IMX_LPCG_CLK_0>,
+			 <&i2c3_lpcg IMX_LPCG_CLK_4>;
+		clock-names = "per", "ipg";
 		assigned-clocks = <&clk IMX_SC_R_I2C_3 IMX_SC_PM_CLK_PER>;
 		assigned-clock-rates = <24000000>;
 		power-domains = <&pd IMX_SC_R_I2C_3>;
diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
new file mode 100644
index 000000000000..ca2a43e0cbf6
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
@@ -0,0 +1,426 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2019~2020, 2022 NXP
+ */
+
+/dts-v1/;
+
+#include "imx8dxl.dtsi"
+
+/ {
+	model = "Freescale i.MX8DXL EVK";
+	compatible = "fsl,imx8dxl-evk", "fsl,imx8dxl";
+
+	aliases {
+		i2c2 = &i2c2;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		serial0 = &lpuart0;
+	};
+
+	chosen {
+		stdout-path = &lpuart0;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x00000000 0x80000000 0 0x40000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		/*
+		 * Memory reserved for optee usage. Please do not use.
+		 * This will be automatically added to dtb if OP-TEE is installed.
+		 * optee@96000000 {
+		 *     reg = <0 0x96000000 0 0x2000000>;
+		 *     no-map;
+		 * };
+		 */
+
+		/* global autoconfigured region for contiguous allocations */
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0 0x14000000>;
+			alloc-ranges = <0 0x98000000 0 0x14000000>;
+			linux,cma-default;
+		};
+	};
+
+	mux3_en: regulator-0 {
+		compatible = "regulator-fixed";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-name = "mux3_en";
+		gpio = <&pca6416_2 8 GPIO_ACTIVE_LOW>;
+		regulator-always-on;
+	};
+
+	reg_fec1_sel: regulator-1 {
+		compatible = "regulator-fixed";
+		regulator-name = "fec1_supply";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&pca6416_1 11 GPIO_ACTIVE_LOW>;
+		regulator-always-on;
+		status = "disabled";
+	};
+
+	reg_fec1_io: regulator-2 {
+		compatible = "regulator-fixed";
+		regulator-name = "fec1_io_supply";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		gpio = <&max7322 0 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+		status = "disabled";
+	};
+
+	reg_usdhc2_vmmc: regulator-3 {
+		compatible = "regulator-fixed";
+		regulator-name = "SD1_SPWR";
+		regulator-min-microvolt = <3000000>;
+		regulator-max-microvolt = <3000000>;
+		gpio = <&lsio_gpio4 30 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		off-on-delay-us = <3480>;
+	};
+};
+
+&eqos {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_eqos>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy0>;
+	nvmem-cells = <&fec_mac1>;
+	nvmem-cell-names = "mac-address";
+	snps,reset-gpios = <&pca6416_1 2 GPIO_ACTIVE_LOW>;
+	snps,reset-delays-us = <10 20 200000>;
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+			eee-broken-1000t;
+			qca,disable-smarteee;
+			vddio-supply = <&vddio0>;
+
+			vddio0: vddio-regulator {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+		};
+	};
+};
+
+/*
+ * fec1 shares the some PINs with usdhc2.
+ * by default usdhc2 is enabled in this dts.
+ * Please disable usdhc2 to enable fec1
+ */
+&fec1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_fec1>;
+	phy-mode = "rgmii-txid";
+	phy-handle = <&ethphy1>;
+	fsl,magic-packet;
+	rx-internal-delay-ps = <2000>;
+	nvmem-cells = <&fec_mac0>;
+	nvmem-cell-names = "mac-address";
+	status = "disabled";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy1: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			reset-gpios = <&pca6416_1 0 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
+			qca,disable-smarteee;
+			vddio-supply = <&vddio1>;
+
+			vddio1: vddio-regulator {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+		};
+	};
+};
+
+&i2c2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	status = "okay";
+
+	pca6416_1: gpio@20 {
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	pca6416_2: gpio@21 {
+		compatible = "ti,tca6416";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	pca9548_1: i2c-mux@70 {
+		compatible = "nxp,pca9548";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x70>;
+
+		i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x0>;
+
+			max7322: gpio@68 {
+				compatible = "maxim,max7322";
+				reg = <0x68>;
+				gpio-controller;
+				#gpio-cells = <2>;
+				status = "disabled";
+			};
+		};
+
+		i2c@4 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x4>;
+		};
+
+		i2c@5 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x5>;
+		};
+
+		i2c@6 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0x6>;
+		};
+	};
+};
+
+&lpuart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpuart0>;
+	status = "okay";
+};
+
+&lsio_gpio4 {
+	status = "okay";
+};
+
+&lsio_gpio5 {
+	status = "okay";
+};
+
+&thermal_zones {
+	pmic-thermal0 {
+		polling-delay-passive = <250>;
+		polling-delay = <2000>;
+		thermal-sensors = <&tsens IMX_SC_R_PMIC_0>;
+
+		trips {
+			pmic_alert0: trip0 {
+				temperature = <110000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+
+			pmic_crit0: trip1 {
+				temperature = <125000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&pmic_alert0>;
+				cooling-device =
+					<&A35_0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+					<&A35_1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+			};
+		};
+	};
+};
+
+&usdhc1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	bus-width = <8>;
+	no-sd;
+	no-sdio;
+	non-removable;
+	status = "okay";
+};
+
+&usdhc2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	bus-width = <4>;
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	cd-gpios = <&lsio_gpio5 1 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&lsio_gpio5 0 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hog>;
+
+	pinctrl_hog: hoggrp {
+		fsl,pins = <
+			IMX8DXL_COMP_CTL_GPIO_1V8_3V3_GPIORHB_PAD	0x000514a0
+			IMX8DXL_COMP_CTL_GPIO_1V8_3V3_GPIORHK_PAD	0x000014a0
+			IMX8DXL_SPI3_CS0_ADMA_ACM_MCLK_OUT1		0x0600004c
+			IMX8DXL_SNVS_TAMPER_OUT1_LSIO_GPIO2_IO05_IN	0x0600004c
+		>;
+	};
+
+	pinctrl_usbotg1: usbotg1grp {
+		fsl,pins = <
+			IMX8DXL_USB_SS3_TC0_CONN_USB_OTG1_PWR		0x00000021
+		>;
+	};
+
+	pinctrl_usbotg2: usbotg2grp {
+		fsl,pins = <
+			IMX8DXL_USB_SS3_TC1_CONN_USB_OTG2_PWR		0x00000021
+		>;
+	};
+
+	pinctrl_eqos: eqosgrp {
+		fsl,pins = <
+			IMX8DXL_ENET0_MDC_CONN_EQOS_MDC				0x06000020
+			IMX8DXL_ENET0_MDIO_CONN_EQOS_MDIO			0x06000020
+			IMX8DXL_ENET1_RGMII_RXC_CONN_EQOS_RGMII_RXC		0x06000020
+			IMX8DXL_ENET1_RGMII_RXD0_CONN_EQOS_RGMII_RXD0		0x06000020
+			IMX8DXL_ENET1_RGMII_RXD1_CONN_EQOS_RGMII_RXD1		0x06000020
+			IMX8DXL_ENET1_RGMII_RXD2_CONN_EQOS_RGMII_RXD2		0x06000020
+			IMX8DXL_ENET1_RGMII_RXD3_CONN_EQOS_RGMII_RXD3		0x06000020
+			IMX8DXL_ENET1_RGMII_RX_CTL_CONN_EQOS_RGMII_RX_CTL	0x06000020
+			IMX8DXL_ENET1_RGMII_TXC_CONN_EQOS_RGMII_TXC		0x06000020
+			IMX8DXL_ENET1_RGMII_TXD0_CONN_EQOS_RGMII_TXD0		0x06000020
+			IMX8DXL_ENET1_RGMII_TXD1_CONN_EQOS_RGMII_TXD1		0x06000020
+			IMX8DXL_ENET1_RGMII_TXD2_CONN_EQOS_RGMII_TXD2		0x06000020
+			IMX8DXL_ENET1_RGMII_TXD3_CONN_EQOS_RGMII_TXD3		0x06000020
+			IMX8DXL_ENET1_RGMII_TX_CTL_CONN_EQOS_RGMII_TX_CTL	0x06000020
+		>;
+	};
+
+	pinctrl_fec1: fec1grp {
+		fsl,pins = <
+			IMX8DXL_COMP_CTL_GPIO_1V8_3V3_ENET_ENETB0_PAD		0x000014a0
+			IMX8DXL_COMP_CTL_GPIO_1V8_3V3_ENET_ENETB1_PAD		0x000014a0
+			IMX8DXL_ENET0_MDC_CONN_ENET0_MDC			0x06000020
+			IMX8DXL_ENET0_MDIO_CONN_ENET0_MDIO			0x06000020
+			IMX8DXL_ENET0_RGMII_RXC_CONN_ENET0_RGMII_RXC		0x00000060
+			IMX8DXL_ENET0_RGMII_RXD0_CONN_ENET0_RGMII_RXD0		0x00000060
+			IMX8DXL_ENET0_RGMII_RXD1_CONN_ENET0_RGMII_RXD1		0x00000060
+			IMX8DXL_ENET0_RGMII_RXD2_CONN_ENET0_RGMII_RXD2		0x00000060
+			IMX8DXL_ENET0_RGMII_RXD3_CONN_ENET0_RGMII_RXD3		0x00000060
+			IMX8DXL_ENET0_RGMII_RX_CTL_CONN_ENET0_RGMII_RX_CTL	0x00000060
+			IMX8DXL_ENET0_RGMII_TXC_CONN_ENET0_RGMII_TXC		0x00000060
+			IMX8DXL_ENET0_RGMII_TXD0_CONN_ENET0_RGMII_TXD0		0x00000060
+			IMX8DXL_ENET0_RGMII_TXD1_CONN_ENET0_RGMII_TXD1		0x00000060
+			IMX8DXL_ENET0_RGMII_TXD2_CONN_ENET0_RGMII_TXD2		0x00000060
+			IMX8DXL_ENET0_RGMII_TXD3_CONN_ENET0_RGMII_TXD3		0x00000060
+			IMX8DXL_ENET0_RGMII_TX_CTL_CONN_ENET0_RGMII_TX_CTL	0x00000060
+		>;
+	};
+
+	pinctrl_lpspi3: lpspi3grp {
+		fsl,pins = <
+			IMX8DXL_SPI3_SCK_ADMA_SPI3_SCK		0x6000040
+			IMX8DXL_SPI3_SDO_ADMA_SPI3_SDO		0x6000040
+			IMX8DXL_SPI3_SDI_ADMA_SPI3_SDI		0x6000040
+			IMX8DXL_SPI3_CS1_ADMA_SPI3_CS1		0x6000040
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			IMX8DXL_SPI1_SCK_ADMA_I2C2_SDA		0x06000021
+			IMX8DXL_SPI1_SDO_ADMA_I2C2_SCL		0x06000021
+		>;
+	};
+
+	pinctrl_cm40_lpuart: cm40lpuartgrp {
+		fsl,pins = <
+			IMX8DXL_ADC_IN2_M40_UART0_RX		0x06000020
+			IMX8DXL_ADC_IN3_M40_UART0_TX		0x06000020
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			IMX8DXL_SPI1_CS0_ADMA_I2C3_SDA		0x06000021
+			IMX8DXL_SPI1_SDI_ADMA_I2C3_SCL		0x06000021
+		>;
+	};
+
+	pinctrl_lpuart0: lpuart0grp {
+		fsl,pins = <
+			IMX8DXL_UART0_RX_ADMA_UART0_RX		0x06000020
+			IMX8DXL_UART0_TX_ADMA_UART0_TX		0x06000020
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			IMX8DXL_EMMC0_CLK_CONN_EMMC0_CLK	0x06000041
+			IMX8DXL_EMMC0_CMD_CONN_EMMC0_CMD	0x00000021
+			IMX8DXL_EMMC0_DATA0_CONN_EMMC0_DATA0	0x00000021
+			IMX8DXL_EMMC0_DATA1_CONN_EMMC0_DATA1	0x00000021
+			IMX8DXL_EMMC0_DATA2_CONN_EMMC0_DATA2	0x00000021
+			IMX8DXL_EMMC0_DATA3_CONN_EMMC0_DATA3	0x00000021
+			IMX8DXL_EMMC0_DATA4_CONN_EMMC0_DATA4	0x00000021
+			IMX8DXL_EMMC0_DATA5_CONN_EMMC0_DATA5	0x00000021
+			IMX8DXL_EMMC0_DATA6_CONN_EMMC0_DATA6	0x00000021
+			IMX8DXL_EMMC0_DATA7_CONN_EMMC0_DATA7	0x00000021
+			IMX8DXL_EMMC0_STROBE_CONN_EMMC0_STROBE	0x00000041
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
+		fsl,pins = <
+			IMX8DXL_ENET0_RGMII_TX_CTL_LSIO_GPIO4_IO30	0x00000040 /* RESET_B */
+			IMX8DXL_ENET0_RGMII_TXD1_LSIO_GPIO5_IO00	0x00000021 /* WP */
+			IMX8DXL_ENET0_RGMII_TXD2_LSIO_GPIO5_IO01	0x00000021 /* CD */
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			IMX8DXL_ENET0_RGMII_RXC_CONN_USDHC1_CLK		0x06000041
+			IMX8DXL_ENET0_RGMII_RX_CTL_CONN_USDHC1_CMD	0x00000021
+			IMX8DXL_ENET0_RGMII_RXD0_CONN_USDHC1_DATA0	0x00000021
+			IMX8DXL_ENET0_RGMII_RXD1_CONN_USDHC1_DATA1	0x00000021
+			IMX8DXL_ENET0_RGMII_RXD2_CONN_USDHC1_DATA2	0x00000021
+			IMX8DXL_ENET0_RGMII_RXD3_CONN_USDHC1_DATA3	0x00000021
+			IMX8DXL_ENET0_RGMII_TXD0_CONN_USDHC1_VSELECT	0x00000021
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-ss-adma.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl-ss-adma.dtsi
new file mode 100644
index 000000000000..795d1d472fae
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8dxl-ss-adma.dtsi
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2019~2020, 2022 NXP
+ */
+
+&audio_ipg_clk {
+	clock-frequency = <160000000>;
+};
+
+&dma_ipg_clk {
+	clock-frequency = <160000000>;
+};
+
+&i2c0 {
+	compatible = "fsl,imx8dxl-lpi2c", "fsl,imx8qxp-lpi2c", "fsl,imx7ulp-lpi2c";
+	interrupts = <GIC_SPI 222 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&i2c1 {
+	compatible = "fsl,imx8dxl-lpi2c", "fsl,imx8qxp-lpi2c", "fsl,imx7ulp-lpi2c";
+	interrupts = <GIC_SPI 223 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&i2c2 {
+	compatible = "fsl,imx8qxp-lpi2c", "fsl,imx7ulp-lpi2c";
+	interrupts = <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&i2c3 {
+	compatible = "fsl,imx8qxp-lpi2c", "fsl,imx7ulp-lpi2c";
+	interrupts = <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lpuart0 {
+	compatible = "fsl,imx8qxp-lpuart", "fsl,imx7ulp-lpuart";
+	interrupts = <GIC_SPI 228 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lpuart1 {
+	compatible = "fsl,imx8qxp-lpuart", "fsl,imx7ulp-lpuart";
+	interrupts = <GIC_SPI 229 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lpuart2 {
+	compatible = "fsl,imx8qxp-lpuart", "fsl,imx7ulp-lpuart";
+	interrupts = <GIC_SPI 230 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lpuart3 {
+	compatible = "fsl,imx8qxp-lpuart", "fsl,imx7ulp-lpuart";
+	interrupts = <GIC_SPI 231 IRQ_TYPE_LEVEL_HIGH>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-ss-conn.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl-ss-conn.dtsi
new file mode 100644
index 000000000000..69c4849f2132
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8dxl-ss-conn.dtsi
@@ -0,0 +1,142 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2019~2020, 2022 NXP
+ */
+
+/delete-node/ &enet1_lpcg;
+/delete-node/ &fec2;
+
+&conn_subsys {
+	conn_enet0_root_clk: clock-conn-enet0-root {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <250000000>;
+		clock-output-names = "conn_enet0_root_clk";
+	};
+
+	eqos: ethernet@5b050000 {
+		compatible = "nxp,imx8dxl-dwmac-eqos", "snps,dwmac-5.10a";
+		reg = <0x5b050000 0x10000>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "eth_wake_irq", "macirq";
+		clocks = <&eqos_lpcg IMX_LPCG_CLK_4>,
+			 <&eqos_lpcg IMX_LPCG_CLK_6>,
+			 <&eqos_lpcg IMX_LPCG_CLK_0>,
+			 <&eqos_lpcg IMX_LPCG_CLK_5>,
+			 <&eqos_lpcg IMX_LPCG_CLK_2>;
+		clock-names = "stmmaceth", "pclk", "ptp_ref", "tx", "mem";
+		assigned-clocks = <&clk IMX_SC_R_ENET_1 IMX_SC_PM_CLK_PER>;
+		assigned-clock-rates = <125000000>;
+		power-domains = <&pd IMX_SC_R_ENET_1>;
+		status = "disabled";
+	};
+
+	usbotg2: usb@5b0e0000 {
+		compatible = "fsl,imx8dxl-usb", "fsl,imx7ulp-usb";
+		reg = <0x5b0e0000 0x200>;
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>;
+		fsl,usbphy = <&usbphy2>;
+		fsl,usbmisc = <&usbmisc2 0>;
+		/*
+		 * usbotg1 and usbotg2 share one clcok.
+		 * scu firmware disables the access to the clock and keeps
+		 * it always on in case other core (M4) uses one of these.
+		 */
+		clocks = <&clk_dummy>;
+		ahb-burst-config = <0x0>;
+		tx-burst-size-dword = <0x10>;
+		rx-burst-size-dword = <0x10>;
+		#stream-id-cells = <1>;
+		power-domains = <&pd IMX_SC_R_USB_1>;
+		status = "disabled";
+
+		clk_dummy: clock-dummy {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <0>;
+			clock-output-names = "clk_dummy";
+		};
+	};
+
+	usbmisc2: usbmisc@5b0e0200 {
+		#index-cells = <1>;
+		compatible = "fsl,imx7ulp-usbmisc";
+		reg = <0x5b0e0200 0x200>;
+	};
+
+	usbphy2: usbphy@0x5b110000 {
+		compatible = "fsl,imx8dxl-usbphy", "fsl,imx7ulp-usbphy";
+		reg = <0x5b110000 0x1000>;
+		clocks = <&usb2_2_lpcg IMX_LPCG_CLK_7>;
+		power-domains = <&pd IMX_SC_R_USB_1_PHY>;
+		status = "disabled";
+	};
+
+	eqos_lpcg: clock-controller@5b240000 {
+		compatible = "fsl,imx8qxp-lpcg";
+		reg = <0x5b240000 0x10000>;
+		#clock-cells = <1>;
+		clocks = <&conn_enet0_root_clk>,
+			 <&conn_axi_clk>,
+			 <&conn_axi_clk>,
+			 <&clk IMX_SC_R_ENET_1 IMX_SC_PM_CLK_PER>,
+			 <&conn_ipg_clk>;
+		clock-indices = <IMX_LPCG_CLK_0>, <IMX_LPCG_CLK_2>,
+				<IMX_LPCG_CLK_4>, <IMX_LPCG_CLK_5>,
+				<IMX_LPCG_CLK_6>;
+		clock-output-names = "eqos_ptp",
+				     "eqos_mem_clk",
+				     "eqos_aclk",
+				     "eqos_clk",
+				     "eqos_csr_clk";
+		power-domains = <&pd IMX_SC_R_ENET_1>;
+	};
+
+	usb2_2_lpcg: clock-controller@5b280000 {
+		compatible = "fsl,imx8qxp-lpcg";
+		reg = <0x5b280000 0x10000>;
+		#clock-cells = <1>;
+		clock-indices = <IMX_LPCG_CLK_7>;
+		clocks = <&conn_ipg_clk>;
+		clock-output-names = "usboh3_2_phy_ipg_clk";
+		power-domains = <&pd IMX_SC_R_USB_1_PHY>;
+	};
+
+};
+
+&enet0_lpcg {
+	clocks = <&conn_enet0_root_clk>,
+		 <&conn_enet0_root_clk>,
+		 <&conn_axi_clk>,
+		 <&clk IMX_SC_R_ENET_0 IMX_SC_C_TXCLK>,
+		 <&conn_ipg_clk>,
+		 <&conn_ipg_clk>;
+};
+
+&fec1 {
+	compatible = "fsl,imx8qm-fec";
+	interrupts = <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>,
+		     <GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>;
+	assigned-clocks = <&clk IMX_SC_R_ENET_0 IMX_SC_C_CLKDIV>;
+	assigned-clock-rates = <125000000>;
+};
+
+&usdhc1 {
+	compatible = "fsl,imx8dxl-usdhc", "fsl,imx8qxp-usdhc";
+	interrupts = <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&usdhc2 {
+	compatible = "fsl,imx8dxl-usdhc", "fsl,imx8qxp-usdhc";
+	interrupts = <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&usdhc3 {
+	compatible = "fsl,imx8dxl-usdhc", "fsl,imx8qxp-usdhc";
+	interrupts = <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-ss-ddr.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl-ss-ddr.dtsi
new file mode 100644
index 000000000000..550f513708d8
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8dxl-ss-ddr.dtsi
@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2020 NXP
+ */
+
+&ddr_pmu0 {
+	compatible = "fsl,imx8-ddr-pmu";
+	interrupts = <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-ss-lsio.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl-ss-lsio.dtsi
new file mode 100644
index 000000000000..815bd987b09b
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8dxl-ss-lsio.dtsi
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2019~2020, 2022 NXP
+ */
+
+&lsio_gpio0 {
+	compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio";
+	interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_gpio1 {
+	compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio";
+	interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_gpio2 {
+	compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio";
+	interrupts = <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_gpio3 {
+	compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio";
+	interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_gpio4 {
+	compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio";
+	interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_gpio5 {
+	compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio";
+	interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_gpio6 {
+	compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio";
+	interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_gpio7 {
+	compatible = "fsl,imx8dxl-gpio", "fsl,imx35-gpio";
+	interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_mu0 {
+	compatible = "fsl,imx8qxp-mu", "fsl,imx6sx-mu";
+	interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_mu1 {
+	compatible = "fsl,imx8-mu-scu", "fsl,imx8qxp-mu", "fsl,imx6sx-mu";
+	interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_mu2 {
+	compatible = "fsl,imx8-mu-scu", "fsl,imx8qxp-mu", "fsl,imx6sx-mu";
+	interrupts = <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_mu3 {
+	compatible = "fsl,imx8-mu-scu", "fsl,imx8qxp-mu", "fsl,imx6sx-mu";
+	interrupts = <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_mu4 {
+	compatible = "fsl,imx8-mu-scu", "fsl,imx8qxp-mu", "fsl,imx6sx-mu";
+	interrupts = <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>;
+};
+
+&lsio_mu5 {
+	compatible = "fsl,imx8qxp-mu", "fsl,imx6sx-mu";
+	interrupts = <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>;
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8dxl.dtsi b/arch/arm64/boot/dts/freescale/imx8dxl.dtsi
new file mode 100644
index 000000000000..5ddbda0b4def
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8dxl.dtsi
@@ -0,0 +1,238 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2019~2020, 2022 NXP
+ */
+
+#include <dt-bindings/clock/imx8-clock.h>
+#include <dt-bindings/firmware/imx/rsrc.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/pinctrl/pads-imx8dxl.h>
+#include <dt-bindings/thermal/thermal.h>
+
+/ {
+	interrupt-parent = <&gic>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	aliases {
+		ethernet0 = &fec1;
+		ethernet1 = &eqos;
+		gpio0 = &lsio_gpio0;
+		gpio1 = &lsio_gpio1;
+		gpio2 = &lsio_gpio2;
+		gpio3 = &lsio_gpio3;
+		gpio4 = &lsio_gpio4;
+		gpio5 = &lsio_gpio5;
+		gpio6 = &lsio_gpio6;
+		gpio7 = &lsio_gpio7;
+		mu1 = &lsio_mu1;
+	};
+
+	cpus: cpus {
+		#address-cells = <2>;
+		#size-cells = <0>;
+
+		/* We have 1 clusters with 2 Cortex-A35 cores */
+		A35_0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a35";
+			reg = <0x0 0x0>;
+			enable-method = "psci";
+			next-level-cache = <&A35_L2>;
+			clocks = <&clk IMX_SC_R_A35 IMX_SC_PM_CLK_CPU>;
+			#cooling-cells = <2>;
+			operating-points-v2 = <&a35_opp_table>;
+		};
+
+		A35_1: cpu@1 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a35";
+			reg = <0x0 0x1>;
+			enable-method = "psci";
+			next-level-cache = <&A35_L2>;
+			clocks = <&clk IMX_SC_R_A35 IMX_SC_PM_CLK_CPU>;
+			#cooling-cells = <2>;
+			operating-points-v2 = <&a35_opp_table>;
+		};
+
+		A35_L2: l2-cache0 {
+			compatible = "cache";
+		};
+	};
+
+	a35_opp_table: opp-table {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-900000000 {
+			opp-hz = /bits/ 64 <900000000>;
+			opp-microvolt = <1000000>;
+			clock-latency-ns = <150000>;
+		};
+
+		opp-1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <1100000>;
+			clock-latency-ns = <150000>;
+			opp-suspend;
+		};
+	};
+
+	gic: interrupt-controller@51a00000 {
+		compatible = "arm,gic-v3";
+		reg = <0x0 0x51a00000 0 0x10000>, /* GIC Dist */
+		      <0x0 0x51b00000 0 0xc0000>; /* GICR (RD_base + SGI_base) */
+		#interrupt-cells = <3>;
+		interrupt-controller;
+		interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		dsp_reserved: dsp@92400000 {
+			reg = <0 0x92400000 0 0x2000000>;
+			no-map;
+		};
+	};
+
+	pmu {
+		compatible = "arm,armv8-pmuv3";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+	};
+
+	system-controller {
+		compatible = "fsl,imx-scu";
+		mbox-names = "tx0",
+			     "rx0",
+			     "gip3";
+		mboxes = <&lsio_mu1 0 0
+			  &lsio_mu1 1 0
+			  &lsio_mu1 3 3>;
+
+		pd: power-controller {
+			compatible = "fsl,scu-pd";
+			#power-domain-cells = <1>;
+			wakeup-irq = <160 163 235 236 237 228 229 230 231 238
+				     239 240 166 169>;
+		};
+
+		clk: clock-controller {
+			compatible = "fsl,imx8dxl-clk", "fsl,scu-clk";
+			#clock-cells = <2>;
+			clocks = <&xtal32k &xtal24m>;
+			clock-names = "xtal_32KHz", "xtal_24Mhz";
+		};
+
+		iomuxc: pinctrl {
+			compatible = "fsl,imx8dxl-iomuxc";
+		};
+
+		ocotp: ocotp {
+			compatible = "fsl,imx8qxp-scu-ocotp";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			fec_mac0: mac@2c4 {
+				reg = <0x2c4 6>;
+			};
+
+			fec_mac1: mac@2c6 {
+				reg = <0x2c6 6>;
+			};
+		};
+
+		rtc: rtc {
+			compatible = "fsl,imx8qxp-sc-rtc";
+		};
+
+		sc_pwrkey: keys {
+			compatible = "fsl,imx8qxp-sc-key", "fsl,imx-sc-key";
+			linux,keycode = <KEY_POWER>;
+			wakeup-source;
+		};
+
+		watchdog {
+			compatible = "fsl,imx-sc-wdt";
+			timeout-sec = <60>;
+		};
+
+		tsens: thermal-sensor {
+			compatible = "fsl,imx-sc-thermal";
+			#thermal-sensor-cells = <1>;
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>, /* Physical Secure */
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>, /* Physical Non-Secure */
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>, /* Virtual */
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>; /* Hypervisor */
+	};
+
+	thermal_zones: thermal-zones {
+		cpu-thermal0 {
+			polling-delay-passive = <250>;
+			polling-delay = <2000>;
+			thermal-sensors = <&tsens IMX_SC_R_SYSTEM>;
+
+			trips {
+				cpu_alert0: trip0 {
+					temperature = <107000>;
+					hysteresis = <2000>;
+					type = "passive";
+				};
+				cpu_crit0: trip1 {
+					temperature = <127000>;
+					hysteresis = <2000>;
+					type = "critical";
+				};
+			};
+
+			cooling-maps {
+				map0 {
+					trip = <&cpu_alert0>;
+					cooling-device =
+					<&A35_0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+					<&A35_1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+				};
+			};
+		};
+	};
+
+	/* The two values below cannot be changed by the board */
+	xtal32k: clock-xtal32k {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		clock-output-names = "xtal_32KHz";
+	};
+
+	xtal24m: clock-xtal24m {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <24000000>;
+		clock-output-names = "xtal_24MHz";
+	};
+
+	/* sorted in register address */
+	#include "imx8-ss-adma.dtsi"
+	#include "imx8-ss-conn.dtsi"
+	#include "imx8-ss-ddr.dtsi"
+	#include "imx8-ss-lsio.dtsi"
+};
+
+#include "imx8dxl-ss-adma.dtsi"
+#include "imx8dxl-ss-conn.dtsi"
+#include "imx8dxl-ss-lsio.dtsi"
+#include "imx8dxl-ss-ddr.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm.dtsi
index 9e6170d9394e..def7bb5d37cf 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm-icore-mx8mm.dtsi
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2018 NXP
  * Copyright (c) 2019 Engicam srl
- * Copyright (c) 2020 Amarula Solutons(India)
+ * Copyright (c) 2020 Amarula Solutions(India)
  */
 
 / {
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl-osm-s.dts b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl-osm-s.dts
new file mode 100644
index 000000000000..8b16bd68576c
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl-osm-s.dts
@@ -0,0 +1,376 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Copyright (C) 2022 Kontron Electronics GmbH
+ */
+
+/dts-v1/;
+
+#include "imx8mm-kontron-osm-s.dtsi"
+
+/ {
+	model = "Kontron BL i.MX8MM OSM-S (N802X S)";
+	compatible = "kontron,imx8mm-bl-osm-s", "kontron,imx8mm-osm-s", "fsl,imx8mm";
+
+	aliases {
+		ethernet1 = &usbnet;
+	};
+
+	/* fixed crystal dedicated to mcp2542fd */
+	osc_can: clock-osc-can {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <40000000>;
+		clock-output-names = "osc-can";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_led>;
+
+		led1 {
+			label = "led1";
+			gpios = <&gpio1 12 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "heartbeat";
+		};
+
+		led2 {
+			label = "led2";
+			gpios = <&gpio1 13 GPIO_ACTIVE_LOW>;
+		};
+
+		led3 {
+			label = "led3";
+			gpios = <&gpio1 14 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	pwm-beeper {
+		compatible = "pwm-beeper";
+		pwms = <&pwm2 0 5000 0>;
+	};
+
+	reg_rst_eth2: regulator-rst-eth2 {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usb_eth2>;
+		gpio = <&gpio3 2 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+		regulator-name = "rst-usb-eth2";
+	};
+
+	reg_usb1_vbus: regulator-usb1-vbus {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_usb1_vbus>;
+		gpio = <&gpio3 25 GPIO_ACTIVE_LOW>;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-name = "usb1-vbus";
+	};
+
+	reg_vdd_5v: regulator-5v {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-name = "vdd-5v";
+	};
+};
+
+&ecspi2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi2>;
+	cs-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>;
+	status = "okay";
+
+	can@0 {
+		compatible = "microchip,mcp251xfd";
+		reg = <0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_can>;
+		clocks = <&osc_can>;
+		interrupts-extended = <&gpio4 28 IRQ_TYPE_LEVEL_LOW>;
+		/*
+		 * Limit the SPI clock to 15 MHz to prevent issues
+		 * with corrupted data due to chip errata.
+		 */
+		spi-max-frequency = <15000000>;
+		vdd-supply = <&reg_vdd_3v3>;
+		xceiver-supply = <&reg_vdd_5v>;
+	};
+};
+
+&ecspi3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi3>;
+	cs-gpios = <&gpio5 25 GPIO_ACTIVE_LOW>;
+	status = "okay";
+
+	eeram@0 {
+		compatible = "microchip,48l640";
+		reg = <0>;
+		spi-max-frequency = <20000000>;
+	};
+};
+
+&fec1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet>;
+	phy-connection-type = "rgmii-rxid";
+	phy-handle = <&ethphy>;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy: ethernet-phy@0 {
+			reg = <0>;
+			reset-assert-us = <1>;
+			reset-deassert-us = <15000>;
+			reset-gpios = <&gpio1 1 GPIO_ACTIVE_LOW>;
+		};
+	};
+};
+
+&gpio1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio1>;
+	gpio-line-names = "", "", "", "dio1-out", "", "", "dio1-in", "dio2-out",
+			  "dio2-in", "dio3-out", "dio3-in", "dio4-out", "", "", "", "",
+			  "", "", "", "", "", "", "", "",
+			  "", "", "", "", "", "", "", "";
+};
+
+&gpio5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5>;
+	gpio-line-names = "", "", "dio4-in", "", "", "", "", "",
+			  "", "", "", "", "", "", "", "",
+			  "", "", "", "", "", "", "", "",
+			  "", "", "", "", "", "", "", "";
+};
+
+&i2c4 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c4>;
+	status = "okay";
+};
+
+&pwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	linux,rs485-enabled-at-boot-time;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&usbotg1 {
+	dr_mode = "otg";
+	disable-over-current;
+	vbus-supply = <&reg_usb1_vbus>;
+	status = "okay";
+};
+
+&usbotg2 {
+	dr_mode = "host";
+	disable-over-current;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	usb1@1 {
+		compatible = "usb424,9514";
+		reg = <1>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		usbnet: ethernet@1 {
+			compatible = "usb424,ec00";
+			reg = <1>;
+			local-mac-address = [ 00 00 00 00 00 00 ];
+		};
+	};
+};
+
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>;
+	vmmc-supply = <&reg_vdd_3v3>;
+	vqmmc-supply = <&reg_nvcc_sd>;
+	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_can: cangrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI3_RXFS_GPIO4_IO28		0x19
+		>;
+	};
+
+	pinctrl_ecspi2: ecspi2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ECSPI2_MISO_ECSPI2_MISO		0x82
+			MX8MM_IOMUXC_ECSPI2_MOSI_ECSPI2_MOSI		0x82
+			MX8MM_IOMUXC_ECSPI2_SCLK_ECSPI2_SCLK		0x82
+			MX8MM_IOMUXC_ECSPI2_SS0_GPIO5_IO13		0x19
+		>;
+	};
+
+	pinctrl_ecspi3: ecspi3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART2_RXD_ECSPI3_MISO		0x82
+			MX8MM_IOMUXC_UART1_TXD_ECSPI3_MOSI		0x82
+			MX8MM_IOMUXC_UART1_RXD_ECSPI3_SCLK		0x82
+			MX8MM_IOMUXC_UART2_TXD_GPIO5_IO25		0x19
+		>;
+	};
+
+	pinctrl_enet: enetgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ENET_MDC_ENET1_MDC			0x3
+			MX8MM_IOMUXC_ENET_MDIO_ENET1_MDIO		0x3
+			MX8MM_IOMUXC_ENET_TD3_ENET1_RGMII_TD3		0x1f
+			MX8MM_IOMUXC_ENET_TD2_ENET1_RGMII_TD2		0x1f
+			MX8MM_IOMUXC_ENET_TD1_ENET1_RGMII_TD1		0x1f
+			MX8MM_IOMUXC_ENET_TD0_ENET1_RGMII_TD0		0x1f
+			MX8MM_IOMUXC_ENET_RD3_ENET1_RGMII_RD3		0x91
+			MX8MM_IOMUXC_ENET_RD2_ENET1_RGMII_RD2		0x91
+			MX8MM_IOMUXC_ENET_RD1_ENET1_RGMII_RD1		0x91
+			MX8MM_IOMUXC_ENET_RD0_ENET1_RGMII_RD0		0x91
+			MX8MM_IOMUXC_ENET_TXC_ENET1_RGMII_TXC		0x1f
+			MX8MM_IOMUXC_ENET_RXC_ENET1_RGMII_RXC		0x91
+			MX8MM_IOMUXC_ENET_RX_CTL_ENET1_RGMII_RX_CTL	0x91
+			MX8MM_IOMUXC_ENET_TX_CTL_ENET1_RGMII_TX_CTL	0x1f
+			MX8MM_IOMUXC_GPIO1_IO01_GPIO1_IO1		0x19 /* PHY RST */
+			MX8MM_IOMUXC_GPIO1_IO05_GPIO1_IO5		0x19 /* ETH IRQ */
+		>;
+	};
+
+	pinctrl_gpio_led: gpioledgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO12_GPIO1_IO12		0x19
+			MX8MM_IOMUXC_GPIO1_IO13_GPIO1_IO13		0x19
+			MX8MM_IOMUXC_GPIO1_IO14_GPIO1_IO14		0x19
+		>;
+	};
+
+	pinctrl_gpio1: gpio1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO03_GPIO1_IO3		0x19
+			MX8MM_IOMUXC_GPIO1_IO07_GPIO1_IO7		0x19
+			MX8MM_IOMUXC_GPIO1_IO09_GPIO1_IO9		0x19
+			MX8MM_IOMUXC_GPIO1_IO11_GPIO1_IO11		0x19
+			MX8MM_IOMUXC_GPIO1_IO06_GPIO1_IO6		0x19
+			MX8MM_IOMUXC_GPIO1_IO08_GPIO1_IO8		0x19
+			MX8MM_IOMUXC_GPIO1_IO10_GPIO1_IO10		0x19
+		>;
+	};
+
+	pinctrl_gpio5: gpio5grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI3_MCLK_GPIO5_IO2		0x19
+		>;
+	};
+
+	pinctrl_i2c4: i2c4grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C4_SCL_I2C4_SCL			0x400001c3
+			MX8MM_IOMUXC_I2C4_SDA_I2C4_SDA			0x400001c3
+		>;
+	};
+
+	pinctrl_pwm2: pwm2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SPDIF_RX_PWM2_OUT			0x19
+		>;
+	};
+
+	pinctrl_reg_usb1_vbus: regusb1vbusgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI5_MCLK_GPIO3_IO25		0x19
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI2_RXC_UART1_DCE_RX		0x140
+			MX8MM_IOMUXC_SAI2_RXFS_UART1_DCE_TX		0x140
+			MX8MM_IOMUXC_SAI2_RXD0_UART1_DCE_RTS_B		0x140
+			MX8MM_IOMUXC_SAI2_TXFS_UART1_DCE_CTS_B		0x140
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI3_TXFS_UART2_DCE_RX		0x140
+			MX8MM_IOMUXC_SAI3_TXC_UART2_DCE_TX		0x140
+			MX8MM_IOMUXC_SAI3_RXD_UART2_DCE_RTS_B		0x140
+			MX8MM_IOMUXC_SAI3_RXC_UART2_DCE_CTS_B		0x140
+		>;
+	};
+
+	pinctrl_usb_eth2: usbeth2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_CE1_B_GPIO3_IO2		0x19
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK			0x190
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD			0x1d0
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0		0x1d0
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d0
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d0
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d0
+			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x019
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x1d0
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK			0x194
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD			0x1d4
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0		0x1d4
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d4
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d4
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d4
+			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x019
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x1d0
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK			0x196
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD			0x1d6
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0		0x1d6
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d6
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d6
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d6
+			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x019
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x1d0
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts
index 23be1ec538ba..a079322a3793 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-s.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts
@@ -5,11 +5,11 @@
 
 /dts-v1/;
 
-#include "imx8mm-kontron-n801x-som.dtsi"
+#include "imx8mm-kontron-sl.dtsi"
 
 / {
-	model = "Kontron i.MX8MM N801X S";
-	compatible = "kontron,imx8mm-n801x-s", "kontron,imx8mm-n801x-som", "fsl,imx8mm";
+	model = "Kontron BL i.MX8MM (N801X S)";
+	compatible = "kontron,imx8mm-bl", "kontron,imx8mm-sl", "fsl,imx8mm";
 
 	aliases {
 		ethernet1 = &usbnet;
@@ -321,6 +321,7 @@
 			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d0
 			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d0
 			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x019
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x1d0
 		>;
 	};
 
@@ -333,6 +334,7 @@
 			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d4
 			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d4
 			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x019
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x1d0
 		>;
 	};
 
@@ -345,6 +347,7 @@
 			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d6
 			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d6
 			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x019
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x1d0
 		>;
 	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi
new file mode 100644
index 000000000000..8d10f5b41297
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi
@@ -0,0 +1,330 @@
+// SPDX-License-Identifier: GPL-2.0+ OR MIT
+/*
+ * Copyright (C) 2022 Kontron Electronics GmbH
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include "imx8mm.dtsi"
+
+/ {
+	model = "Kontron OSM-S i.MX8MM (N802X SOM)";
+	compatible = "kontron,imx8mm-osm-s", "fsl,imx8mm";
+
+	memory@40000000 {
+		device_type = "memory";
+		/*
+		 * There are multiple SoM flavors with different DDR sizes.
+		 * The smallest is 1GB. For larger sizes the bootloader will
+		 * update the reg property.
+		 */
+		reg = <0x0 0x40000000 0 0x80000000>;
+	};
+
+	chosen {
+		stdout-path = &uart3;
+	};
+};
+
+&A53_0 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&A53_1 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&A53_2 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&A53_3 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&ddrc {
+	operating-points-v2 = <&ddrc_opp_table>;
+
+	ddrc_opp_table: opp-table {
+		compatible = "operating-points-v2";
+
+		opp-100M {
+			opp-hz = /bits/ 64 <100000000>;
+		};
+
+		opp-750M {
+			opp-hz = /bits/ 64 <750000000>;
+		};
+	};
+};
+
+&ecspi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	cs-gpios = <&gpio5 9 GPIO_ACTIVE_LOW>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "mxicy,mx25r1635f", "jedec,spi-nor";
+		spi-max-frequency = <80000000>;
+		reg = <0>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "u-boot";
+				reg = <0x0 0x1e0000>;
+			};
+
+			partition@1e0000 {
+				label = "env";
+				reg = <0x1e0000 0x10000>;
+			};
+
+			partition@1f0000 {
+				label = "env_redundant";
+				reg = <0x1f0000 0x10000>;
+			};
+		};
+	};
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	status = "okay";
+
+	pca9450: pmic@25 {
+		compatible = "nxp,pca9450a";
+		reg = <0x25>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pmic>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
+
+		regulators {
+			reg_vdd_soc: BUCK1 {
+				regulator-name = "+0V8_VDD_SOC (BUCK1)";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <850000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+				nxp,dvs-run-voltage = <850000>;
+				nxp,dvs-standby-voltage = <800000>;
+			};
+
+			reg_vdd_arm: BUCK2 {
+				regulator-name = "+0V9_VDD_ARM (BUCK2)";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <950000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+				nxp,dvs-run-voltage = <950000>;
+				nxp,dvs-standby-voltage = <850000>;
+			};
+
+			reg_vdd_dram: BUCK3 {
+				regulator-name = "+0V9_VDD_DRAM&PU (BUCK3)";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <950000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_vdd_3v3: BUCK4 {
+				regulator-name = "+3V3 (BUCK4)";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_vdd_1v8: BUCK5 {
+				regulator-name = "+1V8 (BUCK5)";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_nvcc_dram: BUCK6 {
+				regulator-name = "+1V1_NVCC_DRAM (BUCK6)";
+				regulator-min-microvolt = <1100000>;
+				regulator-max-microvolt = <1100000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_nvcc_snvs: LDO1 {
+				regulator-name = "+1V8_NVCC_SNVS (LDO1)";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_vdd_snvs: LDO2 {
+				regulator-name = "+0V8_VDD_SNVS (LDO2)";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <900000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_vdda: LDO3 {
+				regulator-name = "+1V8_VDDA (LDO3)";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_vdd_phy: LDO4 {
+				regulator-name = "+0V9_VDD_PHY (LDO4)";
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_nvcc_sd: LDO5 {
+				regulator-name = "NVCC_SD (LDO5)";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+			};
+		};
+	};
+
+	rtc@52 {
+		compatible = "microcrystal,rv3028";
+		reg = <0x52>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_rtc>;
+		interrupts-extended = <&gpio4 1 IRQ_TYPE_LEVEL_HIGH>;
+		trickle-diode-disable;
+	};
+};
+
+&uart3 { /* console */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	status = "okay";
+};
+
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	vmmc-supply = <&reg_vdd_3v3>;
+	vqmmc-supply = <&reg_vdd_1v8>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&wdog1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ECSPI1_MISO_ECSPI1_MISO		0x82
+			MX8MM_IOMUXC_ECSPI1_MOSI_ECSPI1_MOSI		0x82
+			MX8MM_IOMUXC_ECSPI1_SCLK_ECSPI1_SCLK		0x82
+			MX8MM_IOMUXC_ECSPI1_SS0_GPIO5_IO9		0x19
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C1_SCL_I2C1_SCL			0x400001c3
+			MX8MM_IOMUXC_I2C1_SDA_I2C1_SDA			0x400001c3
+		>;
+	};
+
+	pinctrl_pmic: pmicgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO00_GPIO1_IO0		0x141
+		>;
+	};
+
+	pinctrl_rtc: rtcgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI1_RXC_GPIO4_IO1			0x19
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART3_RXD_UART3_DCE_RX		0x140
+			MX8MM_IOMUXC_UART3_TXD_UART3_DCE_TX		0x140
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK			0x190
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD			0x1d0
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0		0x1d0
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1		0x1d0
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2		0x1d0
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3		0x1d0
+			MX8MM_IOMUXC_SD1_DATA4_USDHC1_DATA4		0x1d0
+			MX8MM_IOMUXC_SD1_DATA5_USDHC1_DATA5		0x1d0
+			MX8MM_IOMUXC_SD1_DATA6_USDHC1_DATA6		0x1d0
+			MX8MM_IOMUXC_SD1_DATA7_USDHC1_DATA7		0x1d0
+			MX8MM_IOMUXC_SD1_RESET_B_USDHC1_RESET_B		0x019
+			MX8MM_IOMUXC_SD1_STROBE_USDHC1_STROBE		0x190
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK			0x194
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD			0x1d4
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0		0x1d4
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1		0x1d4
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2		0x1d4
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3		0x1d4
+			MX8MM_IOMUXC_SD1_DATA4_USDHC1_DATA4		0x1d4
+			MX8MM_IOMUXC_SD1_DATA5_USDHC1_DATA5		0x1d4
+			MX8MM_IOMUXC_SD1_DATA6_USDHC1_DATA6		0x1d4
+			MX8MM_IOMUXC_SD1_DATA7_USDHC1_DATA7		0x1d4
+			MX8MM_IOMUXC_SD1_RESET_B_USDHC1_RESET_B		0x019
+			MX8MM_IOMUXC_SD1_STROBE_USDHC1_STROBE		0x194
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD1_CLK_USDHC1_CLK			0x196
+			MX8MM_IOMUXC_SD1_CMD_USDHC1_CMD			0x1d6
+			MX8MM_IOMUXC_SD1_DATA0_USDHC1_DATA0		0x1d6
+			MX8MM_IOMUXC_SD1_DATA1_USDHC1_DATA1		0x1d6
+			MX8MM_IOMUXC_SD1_DATA2_USDHC1_DATA2		0x1d6
+			MX8MM_IOMUXC_SD1_DATA3_USDHC1_DATA3		0x1d6
+			MX8MM_IOMUXC_SD1_DATA4_USDHC1_DATA4		0x1d6
+			MX8MM_IOMUXC_SD1_DATA5_USDHC1_DATA5		0x1d6
+			MX8MM_IOMUXC_SD1_DATA6_USDHC1_DATA6		0x1d6
+			MX8MM_IOMUXC_SD1_DATA7_USDHC1_DATA7		0x1d6
+			MX8MM_IOMUXC_SD1_RESET_B_USDHC1_RESET_B		0x019
+			MX8MM_IOMUXC_SD1_STROBE_USDHC1_STROBE		0x196
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO02_WDOG1_WDOG_B		0xc6
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-som.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-kontron-sl.dtsi
index 8f90eb02550d..0679728d2489 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-n801x-som.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-sl.dtsi
@@ -6,8 +6,8 @@
 #include "imx8mm.dtsi"
 
 / {
-	model = "Kontron i.MX8MM N801X SoM";
-	compatible = "kontron,imx8mm-n801x-som", "fsl,imx8mm";
+	model = "Kontron SL i.MX8MM (N801X SOM)";
+	compatible = "kontron,imx8mm-sl", "fsl,imx8mm";
 
 	memory@40000000 {
 		device_type = "memory";
@@ -46,10 +46,6 @@
 	ddrc_opp_table: opp-table {
 		compatible = "operating-points-v2";
 
-		opp-25M {
-			opp-hz = /bits/ 64 <25000000>;
-		};
-
 		opp-100M {
 			opp-hz = /bits/ 64 <100000000>;
 		};
@@ -70,6 +66,27 @@
 		compatible = "mxicy,mx25r1635f", "jedec,spi-nor";
 		spi-max-frequency = <80000000>;
 		reg = <0>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "u-boot";
+				reg = <0x0 0x1e0000>;
+			};
+
+			partition@1e0000 {
+				label = "env";
+				reg = <0x1e0000 0x10000>;
+			};
+
+			partition@1f0000 {
+				label = "env_redundant";
+				reg = <0x1f0000 0x10000>;
+			};
+		};
 	};
 };
 
@@ -86,11 +103,10 @@
 		pinctrl-0 = <&pinctrl_pmic>;
 		interrupt-parent = <&gpio1>;
 		interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
-		sd-vsel-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>;
 
 		regulators {
 			reg_vdd_soc: BUCK1 {
-				regulator-name = "buck1";
+				regulator-name = "+0V8_VDD_SOC (BUCK1)";
 				regulator-min-microvolt = <800000>;
 				regulator-max-microvolt = <850000>;
 				regulator-boot-on;
@@ -101,7 +117,7 @@
 			};
 
 			reg_vdd_arm: BUCK2 {
-				regulator-name = "buck2";
+				regulator-name = "+0V9_VDD_ARM (BUCK2)";
 				regulator-min-microvolt = <850000>;
 				regulator-max-microvolt = <950000>;
 				regulator-boot-on;
@@ -112,7 +128,7 @@
 			};
 
 			reg_vdd_dram: BUCK3 {
-				regulator-name = "buck3";
+				regulator-name = "+0V9_VDD_DRAM&PU (BUCK3)";
 				regulator-min-microvolt = <850000>;
 				regulator-max-microvolt = <950000>;
 				regulator-boot-on;
@@ -120,7 +136,7 @@
 			};
 
 			reg_vdd_3v3: BUCK4 {
-				regulator-name = "buck4";
+				regulator-name = "+3V3 (BUCK4)";
 				regulator-min-microvolt = <3300000>;
 				regulator-max-microvolt = <3300000>;
 				regulator-boot-on;
@@ -128,7 +144,7 @@
 			};
 
 			reg_vdd_1v8: BUCK5 {
-				regulator-name = "buck5";
+				regulator-name = "+1V8 (BUCK5)";
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <1800000>;
 				regulator-boot-on;
@@ -136,7 +152,7 @@
 			};
 
 			reg_nvcc_dram: BUCK6 {
-				regulator-name = "buck6";
+				regulator-name = "+1V1_NVCC_DRAM (BUCK6)";
 				regulator-min-microvolt = <1100000>;
 				regulator-max-microvolt = <1100000>;
 				regulator-boot-on;
@@ -144,7 +160,7 @@
 			};
 
 			reg_nvcc_snvs: LDO1 {
-				regulator-name = "ldo1";
+				regulator-name = "+1V8_NVCC_SNVS (LDO1)";
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <1800000>;
 				regulator-boot-on;
@@ -152,7 +168,7 @@
 			};
 
 			reg_vdd_snvs: LDO2 {
-				regulator-name = "ldo2";
+				regulator-name = "+0V8_VDD_SNVS (LDO2)";
 				regulator-min-microvolt = <800000>;
 				regulator-max-microvolt = <900000>;
 				regulator-boot-on;
@@ -160,7 +176,7 @@
 			};
 
 			reg_vdda: LDO3 {
-				regulator-name = "ldo3";
+				regulator-name = "+1V8_VDDA (LDO3)";
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <1800000>;
 				regulator-boot-on;
@@ -168,7 +184,7 @@
 			};
 
 			reg_vdd_phy: LDO4 {
-				regulator-name = "ldo4";
+				regulator-name = "+0V9_VDD_PHY (LDO4)";
 				regulator-min-microvolt = <900000>;
 				regulator-max-microvolt = <900000>;
 				regulator-boot-on;
@@ -176,7 +192,7 @@
 			};
 
 			reg_nvcc_sd: LDO5 {
-				regulator-name = "ldo5";
+				regulator-name = "NVCC_SD (LDO5)";
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <3300000>;
 			};
@@ -229,7 +245,6 @@
 	pinctrl_pmic: pmicgrp {
 		fsl,pins = <
 			MX8MM_IOMUXC_GPIO1_IO00_GPIO1_IO0		0x141
-			MX8MM_IOMUXC_GPIO1_IO04_GPIO1_IO4		0x141
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx-0x.dts b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx-0x.dts
index b1e7540f0281..641be3af989d 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx-0x.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw72xx-0x.dts
@@ -17,4 +17,3 @@
 		stdout-path = &uart2;
 	};
 };
-
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7902.dts b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7902.dts
index 6dc5eda2d256..31f4c735fe4f 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7902.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7902.dts
@@ -222,7 +222,6 @@
 		compatible = "microchip,mcp2515";
 		reg = <0>;
 		clocks = <&can20m>;
-		oscillator-frequency = <20000000>;
 		interrupt-parent = <&gpio2>;
 		interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
 		spi-max-frequency = <10000000>;
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7903.dts b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7903.dts
index a65761a53f23..19f6d2943d26 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7903.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7903.dts
@@ -250,7 +250,7 @@
 };
 
 &gpio2 {
-	gpio-line-names = "dig2_in", "dig2_out#", "", "", "", "", "", "",
+	gpio-line-names = "dig2_in", "dig2_out#", "dig2_ctl", "", "", "", "dig1_ctl", "",
 		"dig1_out#", "dig1_in", "", "", "", "", "", "",
 		"", "", "", "", "", "", "", "",
 		"", "", "", "", "", "", "", "";
@@ -630,6 +630,8 @@
 			MX8MM_IOMUXC_GPIO1_IO12_GPIO1_IO12	0x40000041 /* RS232# */
 			MX8MM_IOMUXC_SD1_DATA7_GPIO2_IO9	0x40000041 /* DIG1_IN */
 			MX8MM_IOMUXC_SD1_DATA6_GPIO2_IO8	0x40000041 /* DIG1_OUT */
+			MX8MM_IOMUXC_SD1_DATA4_GPIO2_IO6	0x40000041 /* DIG1_CTL */
+			MX8MM_IOMUXC_SD1_DATA0_GPIO2_IO2	0x40000041 /* DIG2_CTL */
 			MX8MM_IOMUXC_SD1_CLK_GPIO2_IO0		0x40000041 /* DIG2_IN */
 			MX8MM_IOMUXC_SD1_CMD_GPIO2_IO1		0x40000041 /* DIG2_OUT */
 			MX8MM_IOMUXC_ECSPI1_MOSI_GPIO5_IO7	0x40000041 /* SIM1DET# */
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7904.dts b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7904.dts
new file mode 100644
index 000000000000..a67771d02146
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-venice-gw7904.dts
@@ -0,0 +1,888 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2022 Gateworks Corporation
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/phy/phy-imx8-pcie.h>
+
+#include "imx8mm.dtsi"
+
+/ {
+	model = "Gateworks Venice GW7904 i.MX8MM board";
+	compatible = "gateworks,imx8mm-gw7904", "fsl,imx8mm";
+
+	chosen {
+		stdout-path = &uart2;
+	};
+
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0x0 0x40000000 0 0x80000000>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		key-0 {
+			label = "user_pb";
+			gpios = <&gpio 2 GPIO_ACTIVE_LOW>;
+			linux,code = <BTN_0>;
+		};
+
+		key-1 {
+			label = "user_pb1x";
+			linux,code = <BTN_1>;
+			interrupt-parent = <&gsc>;
+			interrupts = <0>;
+		};
+
+		key-2 {
+			label = "key_erased";
+			linux,code = <BTN_2>;
+			interrupt-parent = <&gsc>;
+			interrupts = <1>;
+		};
+
+		key-3 {
+			label = "eeprom_wp";
+			linux,code = <BTN_3>;
+			interrupt-parent = <&gsc>;
+			interrupts = <2>;
+		};
+
+		key-4 {
+			label = "switch_hold";
+			linux,code = <BTN_5>;
+			interrupt-parent = <&gsc>;
+			interrupts = <7>;
+		};
+	};
+
+	led-controller {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_leds>;
+
+		led-0 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "led01_grn";
+			gpios = <&gpioled 0 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-1 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_YELLOW>;
+			label = "led01_yel";
+			gpios = <&gpioled 1 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-2 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "led02_grn";
+			gpios = <&gpioled 2 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-3 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_YELLOW>;
+			label = "led02_yel";
+			gpios = <&gpioled 3 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-4 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "led03_grn";
+			gpios = <&gpioled 4 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-5 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_YELLOW>;
+			label = "led03_yel";
+			gpios = <&gpioled 5 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-6 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "led04_grn";
+			gpios = <&gpioled 6 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-7 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_YELLOW>;
+			label = "led04_yel";
+			gpios = <&gpioled 7 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-8 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "led05_grn";
+			gpios = <&gpioled 8 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-9 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_YELLOW>;
+			label = "led05_yel";
+			gpios = <&gpioled 9 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-10 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "led06_grn";
+			gpios = <&gpio1 8 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-11 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_RED>;
+			label = "led06_red";
+			gpios = <&gpio1 9 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-12 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "led07_grn";
+			gpios = <&gpio1 10 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-13 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_RED>;
+			label = "led07_red";
+			gpios = <&gpio1 11 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-14 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "led08_grn";
+			gpios = <&gpioled 10 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-15 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_YELLOW>;
+			label = "led08_yel";
+			gpios = <&gpioled 11 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-16 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "led09_grn";
+			gpios = <&gpioled 12 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-17 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_YELLOW>;
+			label = "led09_yel";
+			gpios = <&gpioled 13 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-18 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "led10_grn";
+			gpios = <&gpioled 14 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+
+		led-19 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_YELLOW>;
+			label = "led10_yel";
+			gpios = <&gpioled 15 GPIO_ACTIVE_LOW>;
+			default-state = "off";
+		};
+	};
+
+	pcie0_refclk: pcie0-refclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+};
+
+&A53_0 {
+	cpu-supply = <&buck2>;
+};
+
+&A53_1 {
+	cpu-supply = <&buck2>;
+};
+
+&A53_2 {
+	cpu-supply = <&buck2>;
+};
+
+&A53_3 {
+	cpu-supply = <&buck2>;
+};
+
+&ddrc {
+	operating-points-v2 = <&ddrc_opp_table>;
+
+	ddrc_opp_table: opp-table {
+		compatible = "operating-points-v2";
+
+		opp-25M {
+			opp-hz = /bits/ 64 <25000000>;
+		};
+
+		opp-100M {
+			opp-hz = /bits/ 64 <100000000>;
+		};
+
+		opp-750M {
+			opp-hz = /bits/ 64 <750000000>;
+		};
+	};
+};
+
+&fec1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_fec1>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy0>;
+	local-mac-address = [00 00 00 00 00 00];
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+		};
+	};
+};
+
+&gpio1 {
+	gpio-line-names = "", "", "", "", "", "", "", "",
+		"", "", "", "", "rs232_en#", "", "", "",
+		"", "", "", "", "", "", "", "",
+		"", "", "", "", "", "", "", "";
+};
+
+&gpio5 {
+	gpio-line-names = "", "", "", "", "", "", "", "",
+		"", "", "", "", "pci_wdis#", "", "", "",
+		"", "", "", "", "", "", "", "",
+		"", "", "", "", "", "", "", "";
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	status = "okay";
+
+	gsc: gsc@20 {
+		compatible = "gw,gsc";
+		reg = <0x20>;
+		pinctrl-0 = <&pinctrl_gsc>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <26 IRQ_TYPE_EDGE_FALLING>;
+		interrupt-controller;
+		#interrupt-cells = <1>;
+
+		adc {
+			compatible = "gw,gsc-adc";
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			channel@6 {
+				gw,mode = <0>;
+				reg = <0x06>;
+				label = "temp";
+			};
+
+			channel@82 {
+				gw,mode = <2>;
+				reg = <0x82>;
+				label = "vin";
+				gw,voltage-divider-ohms = <22100 1000>;
+				gw,voltage-offset-microvolt = <700000>;
+			};
+
+			channel@84 {
+				gw,mode = <2>;
+				reg = <0x84>;
+				label = "vdd_5p0";
+				gw,voltage-divider-ohms = <10000 10000>;
+			};
+
+			channel@86 {
+				gw,mode = <2>;
+				reg = <0x86>;
+				label = "vdd_3p3";
+				gw,voltage-divider-ohms = <10000 10000>;
+			};
+
+			channel@88 {
+				gw,mode = <2>;
+				reg = <0x88>;
+				label = "vdd_0p9";
+			};
+
+			channel@8c {
+				gw,mode = <2>;
+				reg = <0x8c>;
+				label = "vdd_soc";
+			};
+
+			channel@8e {
+				gw,mode = <2>;
+				reg = <0x8e>;
+				label = "vdd_arm";
+			};
+
+			channel@90 {
+				gw,mode = <2>;
+				reg = <0x90>;
+				label = "vdd_1p8";
+			};
+
+			channel@92 {
+				gw,mode = <2>;
+				reg = <0x92>;
+				label = "vdd_dram";
+			};
+
+			channel@a2 {
+				gw,mode = <2>;
+				reg = <0xa2>;
+				label = "vdd_gsc";
+				gw,voltage-divider-ohms = <10000 10000>;
+			};
+		};
+	};
+
+	gpio: gpio@23 {
+		compatible = "nxp,pca9555";
+		reg = <0x23>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gsc>;
+		interrupts = <4>;
+	};
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		pagesize = <16>;
+	};
+
+	eeprom@51 {
+		compatible = "atmel,24c02";
+		reg = <0x51>;
+		pagesize = <16>;
+	};
+
+	eeprom@52 {
+		compatible = "atmel,24c02";
+		reg = <0x52>;
+		pagesize = <16>;
+	};
+
+	eeprom@53 {
+		compatible = "atmel,24c02";
+		reg = <0x53>;
+		pagesize = <16>;
+	};
+
+	rtc@68 {
+		compatible = "dallas,ds1672";
+		reg = <0x68>;
+	};
+};
+
+&i2c2 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	status = "okay";
+
+	pmic@4b {
+		compatible = "rohm,bd71847";
+		reg = <0x4b>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pmic>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <8 IRQ_TYPE_LEVEL_LOW>;
+		rohm,reset-snvs-powered;
+		#clock-cells = <0>;
+		clocks = <&osc_32k 0>;
+		clock-output-names = "clk-32k-out";
+
+		regulators {
+			/* vdd_soc: 0.805-0.900V (typ=0.8V) */
+			BUCK1 {
+				regulator-name = "buck1";
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <1250>;
+			};
+
+			/* vdd_arm: 0.805-1.0V (typ=0.9V) */
+			buck2: BUCK2 {
+				regulator-name = "buck2";
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1300000>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <1250>;
+				rohm,dvs-run-voltage = <1000000>;
+				rohm,dvs-idle-voltage = <900000>;
+			};
+
+			/* vdd_0p9: 0.805-1.0V (typ=0.9V) */
+			BUCK3 {
+				regulator-name = "buck3";
+				regulator-min-microvolt = <700000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* vdd_3p3 */
+			BUCK4 {
+				regulator-name = "buck4";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* vdd_1p8 */
+			BUCK5 {
+				regulator-name = "buck5";
+				regulator-min-microvolt = <1605000>;
+				regulator-max-microvolt = <1995000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* vdd_dram */
+			BUCK6 {
+				regulator-name = "buck6";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* nvcc_snvs_1p8 */
+			LDO1 {
+				regulator-name = "ldo1";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <1900000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* vdd_snvs_0p8 */
+			LDO2 {
+				regulator-name = "ldo2";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <900000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			/* vdda_1p8 */
+			LDO3 {
+				regulator-name = "ldo3";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			LDO4 {
+				regulator-name = "ldo4";
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			LDO6 {
+				regulator-name = "ldo6";
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+&i2c3 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	status = "okay";
+
+	accelerometer@19 {
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_accel>;
+		compatible = "st,lis2de12";
+		reg = <0x19>;
+		st,drdy-int-pin = <1>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
+		interrupt-names = "INT1";
+	};
+};
+
+&i2c4 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c4>;
+	status = "okay";
+
+	gpioled: gpio@27 {
+		compatible = "nxp,pca9555";
+		reg = <0x27>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+};
+
+&pcie_phy {
+	fsl,refclk-pad-mode = <IMX8_PCIE_REFCLK_PAD_INPUT>;
+	fsl,clkreq-unsupported;
+	clocks = <&pcie0_refclk>;
+	clock-names = "ref";
+	status = "okay";
+};
+
+&pcie0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pcie0>;
+	reset-gpio = <&gpio5 11 GPIO_ACTIVE_LOW>;
+	clocks = <&clk IMX8MM_CLK_PCIE1_ROOT>, <&clk IMX8MM_CLK_PCIE1_AUX>,
+		 <&pcie0_refclk>;
+	clock-names = "pcie", "pcie_aux", "pcie_bus";
+	assigned-clocks = <&clk IMX8MM_CLK_PCIE1_AUX>,
+			  <&clk IMX8MM_CLK_PCIE1_CTRL>;
+	assigned-clock-rates = <10000000>, <250000000>;
+	assigned-clock-parents = <&clk IMX8MM_SYS_PLL2_50M>,
+				 <&clk IMX8MM_SYS_PLL2_250M>;
+	status = "okay";
+};
+
+&pgc_mipi {
+	status = "disabled";
+};
+
+/* off-board RS232 */
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+/* console */
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	status = "okay";
+};
+
+/* off-board RS232 */
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	status = "okay";
+};
+
+&usbotg1 {
+	dr_mode = "host";
+	disable-over-current;
+	status = "okay";
+};
+
+/* microSD */
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+	bus-width = <4>;
+	vmmc-supply = <&reg_3p3v>;
+	status = "okay";
+};
+
+/* eMMC */
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&wdog1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hog>;
+
+	pinctrl_hog: hoggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO12_GPIO1_IO12	0x40000041 /* RS232# */
+			MX8MM_IOMUXC_ECSPI2_MISO_GPIO5_IO12	0x40000041 /* PCI_WDIS# */
+		>;
+	};
+
+	pinctrl_accel: accelgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO15_GPIO1_IO15	0x159
+		>;
+	};
+
+	pinctrl_fec1: fec1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ENET_MDC_ENET1_MDC			0x3
+			MX8MM_IOMUXC_ENET_MDIO_ENET1_MDIO		0x3
+			MX8MM_IOMUXC_ENET_TD3_ENET1_RGMII_TD3		0x1f
+			MX8MM_IOMUXC_ENET_TD2_ENET1_RGMII_TD2		0x1f
+			MX8MM_IOMUXC_ENET_TD1_ENET1_RGMII_TD1		0x1f
+			MX8MM_IOMUXC_ENET_TD0_ENET1_RGMII_TD0		0x1f
+			MX8MM_IOMUXC_ENET_RD3_ENET1_RGMII_RD3		0x91
+			MX8MM_IOMUXC_ENET_RD2_ENET1_RGMII_RD2		0x91
+			MX8MM_IOMUXC_ENET_RD1_ENET1_RGMII_RD1		0x91
+			MX8MM_IOMUXC_ENET_RD0_ENET1_RGMII_RD0		0x91
+			MX8MM_IOMUXC_ENET_TXC_ENET1_RGMII_TXC		0x1f
+			MX8MM_IOMUXC_ENET_RXC_ENET1_RGMII_RXC		0x91
+			MX8MM_IOMUXC_ENET_RX_CTL_ENET1_RGMII_RX_CTL	0x91
+			MX8MM_IOMUXC_ENET_TX_CTL_ENET1_RGMII_TX_CTL	0x1f
+			MX8MM_IOMUXC_SAI2_TXFS_GPIO4_IO24		0x19 /* IRQ# */
+			MX8MM_IOMUXC_SAI2_TXC_GPIO4_IO25		0x19 /* RST# */
+		>;
+	};
+
+	pinctrl_gpio_leds: gpioledsgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO08_GPIO1_IO8	0x40000019
+			MX8MM_IOMUXC_GPIO1_IO09_GPIO1_IO9	0x40000019
+			MX8MM_IOMUXC_GPIO1_IO10_GPIO1_IO10	0x40000019
+			MX8MM_IOMUXC_GPIO1_IO11_GPIO1_IO11	0x40000019
+		>;
+	};
+
+	pinctrl_gsc: gscgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SAI2_TXD0_GPIO4_IO26	0x159
+		>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C1_SCL_I2C1_SCL		0x400001c3
+			MX8MM_IOMUXC_I2C1_SDA_I2C1_SDA		0x400001c3
+		>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C2_SCL_I2C2_SCL		0x400001c3
+			MX8MM_IOMUXC_I2C2_SDA_I2C2_SDA		0x400001c3
+		>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C3_SCL_I2C3_SCL		0x400001c3
+			MX8MM_IOMUXC_I2C3_SDA_I2C3_SDA		0x400001c3
+		>;
+	};
+
+	pinctrl_i2c4: i2c4grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_I2C4_SCL_I2C4_SCL		0x400001c3
+			MX8MM_IOMUXC_I2C4_SDA_I2C4_SDA		0x400001c3
+		>;
+	};
+
+	pinctrl_pcie0: pciegrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_ECSPI2_MOSI_GPIO5_IO11	0x41
+		>;
+	};
+
+	pinctrl_pmic: pmicgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_DATA02_GPIO3_IO8	0x41
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART1_RXD_UART1_DCE_RX	0x140
+			MX8MM_IOMUXC_UART1_TXD_UART1_DCE_TX	0x140
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART2_RXD_UART2_DCE_RX	0x140
+			MX8MM_IOMUXC_UART2_TXD_UART2_DCE_TX	0x140
+		>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_UART3_RXD_UART3_DCE_RX	0x140
+			MX8MM_IOMUXC_UART3_TXD_UART3_DCE_TX	0x140
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK		0x190
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD		0x1d0
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0	0x1d0
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1	0x1d0
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2	0x1d0
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3	0x1d0
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK		0x194
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD		0x1d4
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0	0x1d4
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1	0x1d4
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2	0x1d4
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3	0x1d4
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CLK_USDHC2_CLK		0x196
+			MX8MM_IOMUXC_SD2_CMD_USDHC2_CMD		0x1d6
+			MX8MM_IOMUXC_SD2_DATA0_USDHC2_DATA0	0x1d6
+			MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1	0x1d6
+			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2	0x1d6
+			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3	0x1d6
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2-gpiogrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12	0x1c4
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT  0x1d0
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_WE_B_USDHC3_CLK	0x190
+			MX8MM_IOMUXC_NAND_WP_B_USDHC3_CMD	0x1d0
+			MX8MM_IOMUXC_NAND_DATA04_USDHC3_DATA0	0x1d0
+			MX8MM_IOMUXC_NAND_DATA05_USDHC3_DATA1	0x1d0
+			MX8MM_IOMUXC_NAND_DATA06_USDHC3_DATA2	0x1d0
+			MX8MM_IOMUXC_NAND_DATA07_USDHC3_DATA3	0x1d0
+			MX8MM_IOMUXC_NAND_RE_B_USDHC3_DATA4	0x1d0
+			MX8MM_IOMUXC_NAND_CE2_B_USDHC3_DATA5	0x1d0
+			MX8MM_IOMUXC_NAND_CE3_B_USDHC3_DATA6	0x1d0
+			MX8MM_IOMUXC_NAND_CLE_USDHC3_DATA7	0x1d0
+			MX8MM_IOMUXC_NAND_CE1_B_USDHC3_STROBE	0x190
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_WE_B_USDHC3_CLK	0x194
+			MX8MM_IOMUXC_NAND_WP_B_USDHC3_CMD	0x1d4
+			MX8MM_IOMUXC_NAND_DATA04_USDHC3_DATA0	0x1d4
+			MX8MM_IOMUXC_NAND_DATA05_USDHC3_DATA1	0x1d4
+			MX8MM_IOMUXC_NAND_DATA06_USDHC3_DATA2	0x1d4
+			MX8MM_IOMUXC_NAND_DATA07_USDHC3_DATA3	0x1d4
+			MX8MM_IOMUXC_NAND_RE_B_USDHC3_DATA4	0x1d4
+			MX8MM_IOMUXC_NAND_CE2_B_USDHC3_DATA5	0x1d4
+			MX8MM_IOMUXC_NAND_CE3_B_USDHC3_DATA6	0x1d4
+			MX8MM_IOMUXC_NAND_CLE_USDHC3_DATA7	0x1d4
+			MX8MM_IOMUXC_NAND_CE1_B_USDHC3_STROBE	0x194
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_NAND_WE_B_USDHC3_CLK	0x196
+			MX8MM_IOMUXC_NAND_WP_B_USDHC3_CMD	0x1d6
+			MX8MM_IOMUXC_NAND_DATA04_USDHC3_DATA0	0x1d6
+			MX8MM_IOMUXC_NAND_DATA05_USDHC3_DATA1	0x1d6
+			MX8MM_IOMUXC_NAND_DATA06_USDHC3_DATA2	0x1d6
+			MX8MM_IOMUXC_NAND_DATA07_USDHC3_DATA3	0x1d6
+			MX8MM_IOMUXC_NAND_RE_B_USDHC3_DATA4	0x1d6
+			MX8MM_IOMUXC_NAND_CE2_B_USDHC3_DATA5	0x1d6
+			MX8MM_IOMUXC_NAND_CE3_B_USDHC3_DATA6	0x1d6
+			MX8MM_IOMUXC_NAND_CLE_USDHC3_DATA7	0x1d6
+			MX8MM_IOMUXC_NAND_CE1_B_USDHC3_STROBE	0x196
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX8MM_IOMUXC_GPIO1_IO02_WDOG1_WDOG_B	0xc6
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi
index 3ec0c9ac3170..bcab830c6e95 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi
@@ -53,6 +53,21 @@
 		};
 	};
 
+	hdmi_connector: hdmi-connector {
+		compatible = "hdmi-connector";
+		ddc-i2c-bus = <&i2c2>;
+		label = "hdmi";
+		type = "a";
+		status = "disabled";
+	};
+
+	panel_lvds: panel-lvds {
+		compatible = "panel-lvds";
+		backlight = <&backlight>;
+		data-mapping = "vesa-24";
+		status = "disabled";
+	};
+
 	/* Carrier Board Supplies */
 	reg_1p8v: regulator-1p8v {
 		compatible = "regulator-fixed";
@@ -561,8 +576,8 @@
 		status = "disabled";
 	};
 
-	lvds_ti_sn65dsi83: bridge@2c {
-		compatible = "ti,sn65dsi83";
+	lvds_ti_sn65dsi84: bridge@2c {
+		compatible = "ti,sn65dsi84";
 		/* Verdin GPIO_9_DSI (SN65DSI84 IRQ, SODIMM 17, unused) */
 		/* Verdin GPIO_10_DSI (SODIMM 21) */
 		enable-gpios = <&gpio3 3 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-venice-gw7902.dts b/arch/arm64/boot/dts/freescale/imx8mn-venice-gw7902.dts
index 636f8602b979..dd4302ac1de4 100644
--- a/arch/arm64/boot/dts/freescale/imx8mn-venice-gw7902.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mn-venice-gw7902.dts
@@ -213,7 +213,6 @@
 		compatible = "microchip,mcp2515";
 		reg = <0>;
 		clocks = <&can20m>;
-		oscillator-frequency = <20000000>;
 		interrupt-parent = <&gpio2>;
 		interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
 		spi-max-frequency = <10000000>;
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk2.dts b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk2.dts
index 2ca2ede2e94e..382fbedaf6ba 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk2.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-dhcom-pdk2.dts
@@ -1,18 +1,23 @@
 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
 /*
  * Copyright (C) 2022 Marek Vasut <marex@denx.de>
+ *
+ * DHCOM iMX8MP variant:
+ * DHCM-iMX8ML8-C160-R409-F1638-SPI16-GE-CAN2-SD-RTC-WBTA-ADC-T-RGB-CSI2-HS-I-01D2
+ * DHCOM PCB number: 660-100 or newer
+ * PDK2 PCB number: 516-400 or newer
  */
 
 /dts-v1/;
 
 #include <dt-bindings/leds/common.h>
-#include <dt-bindings/net/qca-ar803x.h>
 #include <dt-bindings/phy/phy-imx8-pcie.h>
 #include "imx8mp-dhcom-som.dtsi"
 
 / {
 	model = "DH electronics i.MX8M Plus DHCOM Premium Developer Kit (2)";
-	compatible = "dh,imx8mp-dhcom-pdk2", "fsl,imx8mp";
+	compatible = "dh,imx8mp-dhcom-pdk2", "dh,imx8mp-dhcom-som",
+		     "fsl,imx8mp";
 
 	chosen {
 		stdout-path = &uart1;
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
index f6b017ab5f53..9f1469db554d 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
@@ -5,6 +5,7 @@
 
 /dts-v1/;
 
+#include <dt-bindings/phy/phy-imx8-pcie.h>
 #include "imx8mp.dtsi"
 
 / {
@@ -33,6 +34,12 @@
 		      <0x1 0x00000000 0 0xc0000000>;
 	};
 
+	pcie0_refclk: pcie0-refclk {
+		compatible = "fixed-clock";
+			#clock-cells = <0>;
+			clock-frequency = <100000000>;
+	};
+
 	reg_can1_stby: regulator-can1-stby {
 		compatible = "regulator-fixed";
 		regulator-name = "can1-stby";
@@ -55,6 +62,17 @@
 		enable-active-high;
 	};
 
+	reg_pcie0: regulator-pcie {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pcie0_reg>;
+		regulator-name = "MPCIE_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio2 6 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
 	reg_usdhc2_vmmc: regulator-usdhc2 {
 		compatible = "regulator-fixed";
 		pinctrl-names = "default";
@@ -350,6 +368,28 @@
 	 */
 };
 
+&pcie_phy {
+	fsl,refclk-pad-mode = <IMX8_PCIE_REFCLK_PAD_INPUT>;
+	clocks = <&pcie0_refclk>;
+	clock-names = "ref";
+	status = "okay";
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pcie0>;
+	reset-gpio = <&gpio2 7 GPIO_ACTIVE_LOW>;
+	clocks = <&clk IMX8MP_CLK_HSIO_ROOT>,
+		 <&clk IMX8MP_CLK_PCIE_ROOT>,
+		 <&clk IMX8MP_CLK_HSIO_AXI>;
+	clock-names = "pcie", "pcie_aux", "pcie_bus";
+	assigned-clocks = <&clk IMX8MP_CLK_PCIE_AUX>;
+	assigned-clock-rates = <10000000>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL2_50M>;
+	vpcie-supply = <&reg_pcie0>;
+	status = "okay";
+};
+
 &snvs_pwrkey {
 	status = "okay";
 };
@@ -502,6 +542,19 @@
 		>;
 	};
 
+	pinctrl_pcie0: pcie0grp {
+		fsl,pins = <
+			MX8MP_IOMUXC_I2C4_SCL__PCIE_CLKREQ_B	0x61 /* open drain, pull up */
+			MX8MP_IOMUXC_SD1_DATA5__GPIO2_IO07	0x41
+		>;
+	};
+
+	pinctrl_pcie0_reg: pcie0reggrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_DATA4__GPIO2_IO06	0x41
+		>;
+	};
+
 	pinctrl_pmic: pmicgrp {
 		fsl,pins = <
 			MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03	0x000001c0
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts
index dd703b6a5e17..a02b31c42db4 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp-edimm2.2.dts
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2018 NXP
  * Copyright (c) 2019 Engicam srl
- * Copyright (c) 2020 Amarula Solutons(India)
+ * Copyright (c) 2020 Amarula Solutions(India)
  */
 
 /dts-v1/;
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi
index 5116079cce22..a6319824ea2e 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-icore-mx8mp.dtsi
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2018 NXP
  * Copyright (c) 2019 Engicam srl
- * Copyright (c) 2020 Amarula Solutons(India)
+ * Copyright (c) 2020 Amarula Solutions(India)
  */
 
 / {
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-14N0600E.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-14N0600E.dtsi
new file mode 100644
index 000000000000..2f5cc013e8d6
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-14N0600E.dtsi
@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022 Avnet Embedded GmbH
+ */
+/dts-v1/;
+
+#include "imx8mp-msc-sm2s.dtsi"
+
+/ {
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0x0 0x40000000 0 0x80000000>; /* bank0, 2GiB */
+	};
+};
+
+&cpu_alert0 {
+	temperature = <95000>;
+};
+
+&cpu_crit0 {
+	temperature = <105000>;
+};
+
+&soc_alert0 {
+	temperature = <95000>;
+};
+
+&soc_crit0 {
+	temperature = <105000>;
+};
+
+&tca6424 {
+	gbe0-int-hog {
+		gpio-hog;
+		input;
+		gpios = <3 GPIO_ACTIVE_LOW>;
+	};
+
+	gbe1-int-hog {
+		gpio-hog;
+		input;
+		gpios = <4 GPIO_ACTIVE_LOW>;
+	};
+
+	cam2-rst-hog {
+		gpio-hog;
+		output-high;
+		gpios = <9 GPIO_ACTIVE_LOW>;
+	};
+
+	cam2-pwr-hog {
+		gpio-hog;
+		output-high;
+		gpios = <10 GPIO_ACTIVE_LOW>;
+	};
+
+	tpm-int-hog {
+		gpio-hog;
+		input;
+		gpios = <13 GPIO_ACTIVE_LOW>;
+	};
+
+	wifi-int-hog {
+		gpio-hog;
+		input;
+		gpios = <14 GPIO_ACTIVE_LOW>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-ep1.dts b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-ep1.dts
new file mode 100644
index 000000000000..470ff8e31e32
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s-ep1.dts
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022 Avnet Embedded GmbH
+ */
+
+/dts-v1/;
+
+#include "imx8mp-msc-sm2s-14N0600E.dtsi"
+#include <dt-bindings/clock/imx8mp-clock.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "MSC SM2-MB-EP1 Carrier Board with SM2S-IMX8PLUS-QC6-14N0600E SoM";
+	compatible = "avnet,sm2s-imx8mp-14N0600E-ep1",
+		     "avnet,sm2s-imx8mp-14N0600E", "avnet,sm2s-imx8mp",
+		     "fsl,imx8mp";
+};
+
+&flexcan1 {
+	status = "okay";
+};
+
+&flexcan2 {
+	status = "okay";
+};
+
+&usdhc2 {
+	no-1-8-v;
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_smarc_gpio>;
+
+	pinctrl_smarc_gpio: smarcgpiosgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO11__GPIO1_IO11	0x19>, /* GPIO0 */
+			<MX8MP_IOMUXC_SAI1_RXC__GPIO4_IO01	0x19>, /* GPIO1 */
+			<MX8MP_IOMUXC_SAI1_RXD0__GPIO4_IO02	0x19>, /* GPIO2 */
+			<MX8MP_IOMUXC_SAI1_RXD1__GPIO4_IO03	0x19>, /* GPIO3 */
+			<MX8MP_IOMUXC_SAI3_RXC__GPIO4_IO29	0x19>, /* GPIO4 */
+			<MX8MP_IOMUXC_SAI3_MCLK__GPIO5_IO02	0x19>, /* GPIO5 */
+			<MX8MP_IOMUXC_SAI1_TXD6__GPIO4_IO18	0x19>, /* GPIO6 */
+			<MX8MP_IOMUXC_GPIO1_IO10__GPIO1_IO10	0x19>, /* GPIO7 */
+			<MX8MP_IOMUXC_SAI1_MCLK__GPIO4_IO20	0x19>, /* GPIO8 */
+			<MX8MP_IOMUXC_SAI2_RXFS__GPIO4_IO21	0x19>, /* GPIO9 */
+			<MX8MP_IOMUXC_SAI2_RXC__GPIO4_IO22	0x19>, /* GPIO10 */
+			<MX8MP_IOMUXC_SAI3_RXFS__GPIO4_IO28	0x19>, /* GPIO11 */
+			<MX8MP_IOMUXC_SAI1_TXD7__GPIO4_IO19	0x19>, /* GPIO12 */
+			<MX8MP_IOMUXC_SAI1_RXFS__GPIO4_IO00	0x19>; /* GPIO13 */
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s.dtsi
new file mode 100644
index 000000000000..5dbec71747c3
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-msc-sm2s.dtsi
@@ -0,0 +1,820 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022 Avnet Embedded GmbH
+ */
+
+/dts-v1/;
+
+#include "imx8mp.dtsi"
+#include <dt-bindings/net/ti-dp83867.h>
+
+/ {
+	aliases {
+		rtc0 = &sys_rtc;
+		rtc1 = &snvs_rtc;
+	};
+
+	chosen {
+		stdout-path = &uart2;
+	};
+
+	reg_usb0_host_vbus: regulator-usb0-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb0_host_vbus";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usb0_vbus>;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio1 12 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usb1_host_vbus: regulator-usb1-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb1_host_vbus";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usb1_vbus>;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		gpio = <&gpio1 14 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usdhc2_vmmc: regulator-usdhc2 {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usdhc2_vmmc>;
+		regulator-name = "VSD_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio2 19 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		startup-delay-us = <100>;
+		off-on-delay-us = <12000>;
+	};
+
+	reg_flexcan1_xceiver: regulator-flexcan1 {
+		compatible = "regulator-fixed";
+		regulator-name = "flexcan1-xceiver";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	reg_flexcan2_xceiver: regulator-flexcan2 {
+		compatible = "regulator-fixed";
+		regulator-name = "flexcan2-xceiver";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
+	lcd0_backlight: backlight-0 {
+		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_lcd0_backlight>;
+		pwms = <&pwm1 0 100000 0>;
+		brightness-levels = <0 255>;
+		num-interpolated-steps = <255>;
+		default-brightness-level = <255>;
+		enable-gpios = <&gpio1 5 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+
+	lcd1_backlight: backlight-1 {
+		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_lcd1_backlight>;
+		pwms = <&pwm2 0 100000 0>;
+		brightness-levels = <0 255>;
+		num-interpolated-steps = <255>;
+		default-brightness-level = <255>;
+		enable-gpios = <&gpio1 6 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_leds>;
+		status = "okay";
+
+		led-sw {
+			label = "sw-led";
+			gpios = <&gpio1 8 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	extcon_usb0: extcon-usb0 {
+		compatible = "linux,extcon-usb-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usb0_extcon>;
+		id-gpio = <&gpio1 3 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&A53_0 {
+	cpu-supply = <&vcc_arm>;
+};
+
+&A53_1 {
+	cpu-supply = <&vcc_arm>;
+};
+
+&A53_2 {
+	cpu-supply = <&vcc_arm>;
+};
+
+&A53_3 {
+	cpu-supply = <&vcc_arm>;
+};
+
+&ecspi1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi1>;
+	cs-gpios = <0>, <&gpio2 8 GPIO_ACTIVE_LOW>;
+};
+
+&ecspi2 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_ecspi2>;
+	cs-gpios = <0>, <&gpio2 9 GPIO_ACTIVE_LOW>;
+};
+
+&eqos {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_eqos>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy0>;
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			eee-broken-1000t;
+			reset-gpios = <&tca6424 16 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <1000>;
+			reset-deassert-us = <1000>;
+			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+			ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
+		};
+	};
+};
+
+&fec {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_fec>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy1>;
+	fsl,magic-packet;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy1: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			eee-broken-1000t;
+			reset-gpios = <&tca6424 17 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <1000>;
+			reset-deassert-us = <1000>;
+			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+			ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
+		};
+	};
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	clock-frequency = <400000>;
+	status = "okay";
+
+	id_eeprom: eeprom@50 {
+		compatible = "atmel,24c64";
+		reg = <0x50>;
+		pagesize = <32>;
+	};
+};
+
+&i2c2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c2>;
+	clock-frequency = <400000>;
+	status = "disabled";
+};
+
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c3>;
+	clock-frequency = <400000>;
+	status = "disabled";
+};
+
+&i2c4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c4>;
+	clock-frequency = <400000>;
+	status = "disabled";
+};
+
+&i2c5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c5>;
+	clock-frequency = <400000>;
+	status = "disabled";
+};
+
+&i2c6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c6>;
+	clock-frequency = <400000>;
+	status = "okay";
+
+	tca6424: gpio@22 {
+		compatible = "ti,tca6424";
+		reg = <0x22>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_tca6424>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-line-names = "BOOT_SEL0#", "BOOT_SEL1#", "BOOT_SEL2#",
+			"gbe0_int", "gbe1_int", "pmic_int", "rtc_int", "lvds_int",
+			"PCIE_WAKE#", "cam2_rst", "cam2_pwr", "SLEEP#",
+			"wifi_pd", "tpm_int", "wifi_int", "PCIE_A_RST#",
+			"gbe0_rst", "gbe1_rst", "LID#", "BATLOW#", "CHARGING#",
+			"CHARGER_PRSNT#";
+		interrupt-parent = <&gpio1>;
+		interrupts = <9 IRQ_TYPE_EDGE_RISING>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+	};
+
+	dsi_lvds_bridge: bridge@2d {
+		compatible = "ti,sn65dsi83";
+		reg = <0x2d>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_lvds_bridge>;
+		enable-gpios = <&gpio1 7 GPIO_ACTIVE_HIGH>;
+		status = "disabled";
+	};
+
+	pmic: pmic@30 {
+		compatible = "ricoh,rn5t567";
+		reg = <0x30>;
+		interrupt-parent = <&tca6424>;
+		interrupts = <5 IRQ_TYPE_EDGE_FALLING>;
+
+		regulators {
+			DCDC1 {
+				regulator-name = "VCC_SOC";
+				regulator-always-on;
+				regulator-min-microvolt = <950000>;
+				regulator-max-microvolt = <950000>;
+			};
+
+			DCDC2 {
+				regulator-name = "VCC_DRAM";
+				regulator-always-on;
+				regulator-min-microvolt = <1100000>;
+				regulator-max-microvolt = <1100000>;
+			};
+
+			vcc_arm: DCDC3 {
+				regulator-name = "VCC_ARM";
+				regulator-always-on;
+				regulator-min-microvolt = <950000>;
+				regulator-max-microvolt = <950000>;
+			};
+
+			DCDC4 {
+				regulator-name = "VCC_1V8";
+				regulator-always-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			LDO1 {
+				regulator-name = "VCC_LDO1_2V5";
+				regulator-always-on;
+				regulator-min-microvolt = <2500000>;
+				regulator-max-microvolt = <2500000>;
+			};
+
+			LDO2 {
+				regulator-name = "VCC_LDO2_1V8";
+				regulator-always-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			LDO3 {
+				regulator-name = "VCC_ETH_2V5";
+				regulator-always-on;
+				regulator-min-microvolt = <2500000>;
+				regulator-max-microvolt = <2500000>;
+			};
+
+			LDO4 {
+				regulator-name = "VCC_DDR4_2V5";
+				regulator-always-on;
+				regulator-min-microvolt = <2500000>;
+				regulator-max-microvolt = <2500000>;
+			};
+
+			LDO5 {
+				regulator-name = "VCC_LDO5_1V8";
+				regulator-always-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			LDORTC1 {
+				regulator-name = "VCC_SNVS_1V8";
+				regulator-always-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			LDORTC2 {
+				regulator-name = "VCC_SNVS_3V3";
+				regulator-always-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+			};
+		};
+	};
+
+	sys_rtc: rtc@32 {
+		compatible = "ricoh,r2221tl";
+		reg = <0x32>;
+		interrupt-parent = <&tca6424>;
+		interrupts = <6 IRQ_TYPE_EDGE_FALLING>;
+	};
+
+	tmp_sensor: temperature-sensor@71 {
+		compatible = "ti,tmp103";
+		reg = <0x71>;
+	};
+};
+
+&flexcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	xceiver-supply = <&reg_flexcan1_xceiver>;
+	status = "disabled";
+};
+
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	xceiver-supply = <&reg_flexcan2_xceiver>;
+	status = "disabled";
+};
+
+&flexspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexspi0>;
+	status = "okay";
+
+	qspi_flash: flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		spi-max-frequency = <80000000>;
+		spi-tx-bus-width = <4>;
+		spi-rx-bus-width = <4>;
+	};
+};
+
+&pwm1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm1>;
+	status = "disabled";
+};
+
+&pwm2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm2>;
+	status = "disabled";
+};
+
+&pwm3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm3>;
+	status = "disabled";
+};
+
+&pwm4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm4>;
+	status = "disabled";
+};
+
+&snvs_pwrkey {
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&uart4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart4>;
+	status = "disabled";
+};
+
+&usb3_phy0 {
+	vbus-supply = <&reg_usb0_host_vbus>;
+	status = "okay";
+};
+
+&usb3_phy1 {
+	vbus-supply = <&reg_usb1_host_vbus>;
+	status = "okay";
+};
+
+&usb3_0 {
+	status = "okay";
+};
+
+&usb3_1 {
+	status = "okay";
+};
+
+&usb_dwc3_0 {
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	extcon = <&extcon_usb0>;
+	status = "okay";
+};
+
+&usb_dwc3_1 {
+	dr_mode = "host";
+	status = "okay";
+};
+
+&usdhc2 {
+	assigned-clocks = <&clk IMX8MP_CLK_USDHC2>;
+	assigned-clock-rates = <400000000>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	cd-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&gpio2 20 GPIO_ACTIVE_HIGH>;
+	bus-width = <4>;
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	status = "okay";
+};
+
+&usdhc3 {
+	assigned-clocks = <&clk IMX8MP_CLK_USDHC3>;
+	assigned-clock-rates = <400000000>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc3>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&wdog1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	fsl,ext-reset-output;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_ecspi1: ecspi1grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_ECSPI1_MISO__ECSPI1_MISO		0x82>,
+			<MX8MP_IOMUXC_ECSPI1_MOSI__ECSPI1_MOSI		0x82>,
+			<MX8MP_IOMUXC_ECSPI1_SCLK__ECSPI1_SCLK		0x82>,
+			<MX8MP_IOMUXC_ECSPI1_SS0__ECSPI1_SS0		0x40000>,
+			<MX8MP_IOMUXC_SD1_DATA6__GPIO2_IO08		0x40000>;
+	};
+
+	pinctrl_ecspi2: ecspi2grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_ECSPI2_MISO__ECSPI2_MISO		0x82>,
+			<MX8MP_IOMUXC_ECSPI2_MOSI__ECSPI2_MOSI		0x82>,
+			<MX8MP_IOMUXC_ECSPI2_SCLK__ECSPI2_SCLK		0x82>,
+			<MX8MP_IOMUXC_ECSPI2_SS0__ECSPI2_SS0		0x40000>,
+			<MX8MP_IOMUXC_SD1_DATA7__GPIO2_IO09		0x40000>;
+	};
+
+	pinctrl_eqos: eqosgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_ENET_MDC__ENET_QOS_MDC		0x3>,
+			<MX8MP_IOMUXC_ENET_MDIO__ENET_QOS_MDIO		0x3>,
+			<MX8MP_IOMUXC_ENET_RD0__ENET_QOS_RGMII_RD0	0x91>,
+			<MX8MP_IOMUXC_ENET_RD1__ENET_QOS_RGMII_RD1	0x91>,
+			<MX8MP_IOMUXC_ENET_RD2__ENET_QOS_RGMII_RD2	0x91>,
+			<MX8MP_IOMUXC_ENET_RD3__ENET_QOS_RGMII_RD3	0x91>,
+			<MX8MP_IOMUXC_ENET_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x91>,
+			<MX8MP_IOMUXC_ENET_RX_CTL__ENET_QOS_RGMII_RX_CTL	0x91>,
+			<MX8MP_IOMUXC_ENET_TD0__ENET_QOS_RGMII_TD0	0x1f>,
+			<MX8MP_IOMUXC_ENET_TD1__ENET_QOS_RGMII_TD1	0x1f>,
+			<MX8MP_IOMUXC_ENET_TD2__ENET_QOS_RGMII_TD2	0x1f>,
+			<MX8MP_IOMUXC_ENET_TD3__ENET_QOS_RGMII_TD3	0x1f>,
+			<MX8MP_IOMUXC_ENET_TX_CTL__ENET_QOS_RGMII_TX_CTL	0x1f>,
+			<MX8MP_IOMUXC_ENET_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x1f>;
+	};
+
+	pinctrl_fec: fecgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SAI1_RXD2__ENET1_MDC		0x3>,
+			<MX8MP_IOMUXC_SAI1_RXD3__ENET1_MDIO		0x3>,
+			<MX8MP_IOMUXC_SAI1_RXD4__ENET1_RGMII_RD0	0x91>,
+			<MX8MP_IOMUXC_SAI1_RXD5__ENET1_RGMII_RD1	0x91>,
+			<MX8MP_IOMUXC_SAI1_RXD6__ENET1_RGMII_RD2	0x91>,
+			<MX8MP_IOMUXC_SAI1_RXD7__ENET1_RGMII_RD3	0x91>,
+			<MX8MP_IOMUXC_SAI1_TXC__ENET1_RGMII_RXC		0x91>,
+			<MX8MP_IOMUXC_SAI1_TXFS__ENET1_RGMII_RX_CTL	0x91>,
+			<MX8MP_IOMUXC_SAI1_TXD0__ENET1_RGMII_TD0	0x1f>,
+			<MX8MP_IOMUXC_SAI1_TXD1__ENET1_RGMII_TD1	0x1f>,
+			<MX8MP_IOMUXC_SAI1_TXD2__ENET1_RGMII_TD2	0x1f>,
+			<MX8MP_IOMUXC_SAI1_TXD3__ENET1_RGMII_TD3	0x1f>,
+			<MX8MP_IOMUXC_SAI1_TXD4__ENET1_RGMII_TX_CTL	0x1f>,
+			<MX8MP_IOMUXC_SAI1_TXD5__ENET1_RGMII_TXC	0x1f>;
+	};
+
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SAI5_RXD1__CAN1_TX		0x154>,
+			<MX8MP_IOMUXC_SAI5_RXD2__CAN1_RX		0x154>;
+	};
+
+	pinctrl_flexcan2: flexcan2grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SAI5_MCLK__CAN2_RX		0x154>,
+			<MX8MP_IOMUXC_SAI5_RXD3__CAN2_TX		0x154>;
+	};
+
+	pinctrl_flexspi0: flexspi0grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_NAND_ALE__FLEXSPI_A_SCLK		0x1c2>,
+			<MX8MP_IOMUXC_NAND_CE0_B__FLEXSPI_A_SS0_B	0x82>,
+			<MX8MP_IOMUXC_NAND_DATA00__FLEXSPI_A_DATA00	0x82>,
+			<MX8MP_IOMUXC_NAND_DATA01__FLEXSPI_A_DATA01	0x82>,
+			<MX8MP_IOMUXC_NAND_DATA02__FLEXSPI_A_DATA02	0x82>,
+			<MX8MP_IOMUXC_NAND_DATA03__FLEXSPI_A_DATA03	0x82>,
+			<MX8MP_IOMUXC_NAND_DQS__GPIO3_IO14		0x19>;
+	};
+
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_I2C1_SCL__I2C1_SCL		0x400001c3>,
+			<MX8MP_IOMUXC_I2C1_SDA__I2C1_SDA		0x400001c3>;
+	};
+
+	pinctrl_i2c2: i2c2grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_I2C2_SCL__I2C2_SCL		0x400001c3>,
+			<MX8MP_IOMUXC_I2C2_SDA__I2C2_SDA		0x400001c3>;
+	};
+
+	pinctrl_i2c3: i2c3grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_I2C3_SCL__I2C3_SCL		0x400001c3>,
+			<MX8MP_IOMUXC_I2C3_SDA__I2C3_SDA		0x400001c3>;
+	};
+
+	pinctrl_i2c4: i2c4grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_I2C4_SCL__I2C4_SCL		0x400001c3>,
+			<MX8MP_IOMUXC_I2C4_SDA__I2C4_SDA		0x400001c3>;
+	};
+
+	pinctrl_i2c5: i2c5grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SPDIF_TX__I2C5_SCL		0x400001c3>,
+			<MX8MP_IOMUXC_SPDIF_RX__I2C5_SDA		0x400001c3>;
+	};
+
+	pinctrl_i2c6: i2c6grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SAI5_RXFS__I2C6_SCL		0x400001c3>,
+			<MX8MP_IOMUXC_SAI5_RXC__I2C6_SDA		0x400001c3>;
+	};
+
+	pinctrl_lcd0_backlight: lcd0-backlightgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO05__GPIO1_IO05		0x41>;
+	};
+
+	pinctrl_lcd1_backlight: lcd1-backlightgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO06__GPIO1_IO06		0x41>;
+	};
+
+	pinctrl_leds: ledsgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO08__GPIO1_IO08		0x19>;
+	};
+
+	pinctrl_lvds_bridge: lvds-bridgegrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO07__GPIO1_IO07		0x41>;
+	};
+
+	pinctrl_pwm1: pwm1grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SPDIF_EXT_CLK__PWM1_OUT		0x116>;
+	};
+
+	pinctrl_pwm2: pwm2grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SAI5_RXD0__PWM2_OUT		0x116>;
+	};
+
+	pinctrl_pwm3: pwm3grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO10__PWM3_OUT		0x116>;
+	};
+
+	pinctrl_pwm4: pwm4grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SAI3_MCLK__PWM4_OUT		0x116>;
+	};
+
+	pinctrl_tca6424: tca6424grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO09__GPIO1_IO09		0x41>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_UART1_RXD__UART1_DCE_RX		0x49>,
+			<MX8MP_IOMUXC_UART1_TXD__UART1_DCE_TX		0x49>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SD1_DATA4__GPIO2_IO06		0x1c4>,
+			<MX8MP_IOMUXC_SD1_DATA5__GPIO2_IO07		0x1c4>,
+			<MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX		0x49>,
+			<MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX		0x49>;
+	};
+
+	pinctrl_uart3: uart3grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SD1_RESET_B__GPIO2_IO10		0x1c4>,
+			<MX8MP_IOMUXC_SD1_STROBE__GPIO2_IO11		0x1c4>,
+			<MX8MP_IOMUXC_UART3_RXD__UART3_DCE_RX		0x49>,
+			<MX8MP_IOMUXC_UART3_TXD__UART3_DCE_TX		0x49>;
+	};
+
+	pinctrl_uart4: uart4grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_UART4_RXD__UART4_DCE_RX		0x49>,
+			<MX8MP_IOMUXC_UART4_TXD__UART4_DCE_TX		0x49>;
+	};
+
+	pinctrl_usb0_extcon: usb0-extcongrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO03__GPIO1_IO03		0x19>;
+	};
+
+	pinctrl_usb0_vbus: usb0-vbusgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO12__GPIO1_IO12		0x19>;
+	};
+
+	pinctrl_usb1_vbus: usb1-vbusgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO14__GPIO1_IO14		0x19>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2-gpiogrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SD2_CD_B__GPIO2_IO12		0x1c4>,
+			<MX8MP_IOMUXC_SD2_WP__GPIO2_IO20		0x1c4>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK		0x190>,
+			<MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD		0x1d0>,
+			<MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0		0x1d0>,
+			<MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1		0x1d0>,
+			<MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2		0x1d0>,
+			<MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3		0x1d0>,
+			<MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT	0xc1>;
+	};
+
+	pinctrl_usdhc2_vmmc: usdhc2-vmmcgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SD2_RESET_B__GPIO2_IO19		0x41>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK		0x194>,
+			<MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD		0x1d4>,
+			<MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0		0x1d4>,
+			<MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1		0x1d4>,
+			<MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2		0x1d4>,
+			<MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3		0x1d4>,
+			<MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT	0xc1>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK		0x196>,
+			<MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD		0x1d6>,
+			<MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0		0x1d6>,
+			<MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1		0x1d6>,
+			<MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2		0x1d6>,
+			<MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3		0x1d6>,
+			<MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT	0xc1>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins =
+			<MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK		0x190>,
+			<MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD		0x1d0>,
+			<MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0		0x1d0>,
+			<MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1		0x1d0>,
+			<MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2		0x1d0>,
+			<MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3		0x1d0>,
+			<MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4		0x1d0>,
+			<MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5		0x1d0>,
+			<MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6		0x1d0>,
+			<MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7		0x1d0>,
+			<MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE		0x190>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK		0x194>,
+			<MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD		0x1d4>,
+			<MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0		0x1d4>,
+			<MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1		0x1d4>,
+			<MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2		0x1d4>,
+			<MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3		0x1d4>,
+			<MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4		0x1d4>,
+			<MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5		0x1d4>,
+			<MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6		0x1d4>,
+			<MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7		0x1d4>,
+			<MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE		0x194>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK		0x196>,
+			<MX8MP_IOMUXC_NAND_WP_B__USDHC3_CMD		0x1d6>,
+			<MX8MP_IOMUXC_NAND_DATA04__USDHC3_DATA0		0x1d6>,
+			<MX8MP_IOMUXC_NAND_DATA05__USDHC3_DATA1		0x1d6>,
+			<MX8MP_IOMUXC_NAND_DATA06__USDHC3_DATA2		0x1d6>,
+			<MX8MP_IOMUXC_NAND_DATA07__USDHC3_DATA3		0x1d6>,
+			<MX8MP_IOMUXC_NAND_RE_B__USDHC3_DATA4		0x1d6>,
+			<MX8MP_IOMUXC_NAND_CE2_B__USDHC3_DATA5		0x1d6>,
+			<MX8MP_IOMUXC_NAND_CE3_B__USDHC3_DATA6		0x1d6>,
+			<MX8MP_IOMUXC_NAND_CLE__USDHC3_DATA7		0x1d6>,
+			<MX8MP_IOMUXC_NAND_CE1_B__USDHC3_STROBE		0x196>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_GPIO1_IO02__WDOG1_WDOG_B		0xc6>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts
index 0e237b2f9541..7bf6f81e87b4 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tqma8mpql-mba8mpxl.dts
@@ -139,6 +139,13 @@
 		regulator-max-microvolt = <3300000>;
 	};
 
+	reg_vcc_5v0: regulator-5v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC_5V0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
 	reserved-memory {
 		#address-cells = <2>;
 		#size-cells = <2>;
@@ -447,6 +454,38 @@
 	status = "okay";
 };
 
+&usb3_0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usb0>;
+	fsl,over-current-active-low;
+	status = "okay";
+};
+
+&usb3_phy0 {
+	vbus-supply = <&reg_vcc_5v0>;
+	status = "okay";
+};
+
+&usb_dwc3_0 {
+	/* dual role is implemented, but not a full featured OTG */
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
+	status = "okay";
+
+	connector {
+		compatible = "gpio-usb-b-connector", "usb-b-connector";
+		type = "micro";
+		label = "X29";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usbcon0>;
+		id-gpios = <&gpio1 10 GPIO_ACTIVE_HIGH>;
+	};
+};
+
 &usdhc2 {
 	pinctrl-names = "default", "state_100mhz", "state_200mhz";
 	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
@@ -672,6 +711,15 @@
 			   <MX8MP_IOMUXC_UART4_TXD__UART4_DCE_TX	0x140>;
 	};
 
+	pinctrl_usb0: usb0grp {
+		fsl,pins = <MX8MP_IOMUXC_GPIO1_IO13__USB1_OTG_OC	0x1c0>,
+			   <MX8MP_IOMUXC_GPIO1_IO12__USB1_OTG_PWR	0x1c0>;
+	};
+
+	pinctrl_usbcon0: usb0congrp {
+		fsl,pins = <MX8MP_IOMUXC_GPIO1_IO10__GPIO1_IO10		0x1c0>;
+	};
+
 	pinctrl_usdhc2: usdhc2grp {
 		fsl,pins = <MX8MP_IOMUXC_SD2_CLK__USDHC2_CLK		0x192>,
 			   <MX8MP_IOMUXC_SD2_CMD__USDHC2_CMD		0x1d2>,
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts b/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts
index 211e6a1b296e..06b4c93c5876 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-venice-gw74xx.dts
@@ -8,6 +8,7 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/linux-event-codes.h>
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/phy/phy-imx8-pcie.h>
 
 #include "imx8mp.dtsi"
 
@@ -100,6 +101,12 @@
 		};
 	};
 
+	pcie0_refclk: pcie0-refclk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+	};
+
 	pps {
 		compatible = "pps-gpio";
 		pinctrl-names = "default";
@@ -134,13 +141,29 @@
 		compatible = "regulator-fixed";
 		regulator-name = "wl";
 		gpio = <&gpio3 9 GPIO_ACTIVE_HIGH>;
-		startup-delay-us = <100>;
+		startup-delay-us = <70000>;
 		enable-active-high;
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
 	};
 };
 
+&A53_0 {
+	cpu-supply = <&reg_arm>;
+};
+
+&A53_1 {
+	cpu-supply = <&reg_arm>;
+};
+
+&A53_2 {
+	cpu-supply = <&reg_arm>;
+};
+
+&A53_3 {
+	cpu-supply = <&reg_arm>;
+};
+
 /* off-board header */
 &ecspi2 {
 	pinctrl-names = "default";
@@ -199,8 +222,8 @@
 &gpio2 {
 	gpio-line-names =
 		"", "", "", "", "", "", "", "",
-		"", "", "", "", "", "", "", "",
-		"pcie3_wdis#", "", "", "pcie1_wdis@", "pcie2_wdis#", "", "", "",
+		"", "", "", "", "", "", "pcie3_wdis#", "",
+		"", "", "pcie2_wdis#", "", "", "", "", "",
 		"", "", "", "", "", "", "", "";
 };
 
@@ -361,7 +384,7 @@
 				regulator-ramp-delay = <3125>;
 			};
 
-			BUCK2 {
+			reg_arm: BUCK2 {
 				regulator-name = "BUCK2";
 				regulator-min-microvolt = <720000>;
 				regulator-max-microvolt = <1025000>;
@@ -546,6 +569,28 @@
 	status = "okay";
 };
 
+&pcie_phy {
+	fsl,refclk-pad-mode = <IMX8_PCIE_REFCLK_PAD_INPUT>;
+	fsl,clkreq-unsupported;
+	clocks = <&pcie0_refclk>;
+	clock-names = "ref";
+	status = "okay";
+};
+
+&pcie {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pcie0>;
+	reset-gpio = <&gpio2 17 GPIO_ACTIVE_LOW>;
+	clocks = <&clk IMX8MP_CLK_HSIO_ROOT>,
+		 <&clk IMX8MP_CLK_PCIE_ROOT>,
+		 <&clk IMX8MP_CLK_HSIO_AXI>;
+	clock-names = "pcie", "pcie_aux", "pcie_bus";
+	assigned-clocks = <&clk IMX8MP_CLK_PCIE_AUX>;
+	assigned-clock-rates = <10000000>;
+	assigned-clock-parents = <&clk IMX8MP_SYS_PLL2_50M>;
+	status = "okay";
+};
+
 /* GPS / off-board header */
 &uart1 {
 	pinctrl-names = "default";
@@ -560,6 +605,21 @@
 	status = "okay";
 };
 
+/* bluetooth HCI */
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>, <&pinctrl_uart3_gpio>;
+	cts-gpios = <&gpio3 21 GPIO_ACTIVE_LOW>;
+	rts-gpios = <&gpio3 22 GPIO_ACTIVE_LOW>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm4330-bt";
+		shutdown-gpios = <&gpio3 8 GPIO_ACTIVE_HIGH>;
+	};
+};
+
 &uart4 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart4>;
@@ -567,20 +627,35 @@
 };
 
 /* USB1 - Type C front panel */
-&usb3_phy0 {
+&usb3_0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_usb1>;
+	fsl,over-current-active-low;
 	status = "okay";
 };
 
-&usb3_0 {
-	fsl,over-current-active-low;
+&usb3_phy0 {
 	status = "okay";
 };
 
 &usb_dwc3_0 {
-	dr_mode = "host";
+	/* dual role is implemented but not a full featured OTG */
+	adp-disable;
+	hnp-disable;
+	srp-disable;
+	dr_mode = "otg";
+	usb-role-switch;
+	role-switch-default-mode = "peripheral";
 	status = "okay";
+
+	connector {
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_usbcon1>;
+		compatible = "gpio-usb-b-connector", "usb-b-connector";
+		type = "micro";
+		label = "Type-C";
+		id-gpios = <&gpio1 10 GPIO_ACTIVE_HIGH>;
+	};
 };
 
 /* USB2 - USB3.0 Hub */
@@ -600,6 +675,25 @@
 	status = "okay";
 };
 
+/* SDIO WiFi */
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	bus-width = <4>;
+	non-removable;
+	vmmc-supply = <&reg_wifi_en>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	wifi@0 {
+		compatible = "cypress,cyw4373-fmac";
+		reg = <0>;
+	};
+};
+
 /* eMMC */
 &usdhc3 {
 	assigned-clocks = <&clk IMX8MP_CLK_USDHC3>;
@@ -629,7 +723,6 @@
 			MX8MP_IOMUXC_GPIO1_IO09__GPIO1_IO09	0x40000040 /* DIO0 */
 			MX8MP_IOMUXC_GPIO1_IO11__GPIO1_IO11	0x40000040 /* DIO1 */
 			MX8MP_IOMUXC_NAND_DQS__GPIO3_IO14	0x40000040 /* M2SKT_OFF# */
-			MX8MP_IOMUXC_SD2_DATA2__GPIO2_IO17	0x40000150 /* PCIE1_WDIS# */
 			MX8MP_IOMUXC_SD2_DATA3__GPIO2_IO18	0x40000150 /* PCIE2_WDIS# */
 			MX8MP_IOMUXC_SD2_CMD__GPIO2_IO14	0x40000150 /* PCIE3_WDIS# */
 			MX8MP_IOMUXC_NAND_DATA00__GPIO3_IO06	0x40000040 /* M2SKT_RST# */
@@ -742,6 +835,12 @@
 		>;
 	};
 
+	pinctrl_pcie0: pciegrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD2_DATA2__GPIO2_IO17	0x110
+		>;
+	};
+
 	pinctrl_pmic: pmicgrp {
 		fsl,pins = <
 			MX8MP_IOMUXC_NAND_DATA01__GPIO3_IO07	0x140
@@ -829,7 +928,12 @@
 	pinctrl_usb1: usb1grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_GPIO1_IO13__USB1_OTG_OC	0x140
-			MX8MP_IOMUXC_GPIO1_IO10__USB1_OTG_ID	0x140
+		>;
+	};
+
+	pinctrl_usbcon1: usb1congrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO10__GPIO1_IO10	0x140
 		>;
 	};
 
@@ -844,6 +948,28 @@
 		>;
 	};
 
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK	0x194
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD	0x1d4
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0	0x1d4
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1	0x1d4
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2	0x1d4
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3	0x1d4
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_SD1_CLK__USDHC1_CLK	0x196
+			MX8MP_IOMUXC_SD1_CMD__USDHC1_CMD	0x1d6
+			MX8MP_IOMUXC_SD1_DATA0__USDHC1_DATA0	0x1d6
+			MX8MP_IOMUXC_SD1_DATA1__USDHC1_DATA1	0x1d6
+			MX8MP_IOMUXC_SD1_DATA2__USDHC1_DATA2	0x1d6
+			MX8MP_IOMUXC_SD1_DATA3__USDHC1_DATA3	0x1d6
+		>;
+	};
+
 	pinctrl_usdhc3: usdhc3grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_NAND_WE_B__USDHC3_CLK	0x190
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi
index 1c74c6a19449..7b712d1888ea 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi
@@ -146,6 +146,22 @@
 	};
 };
 
+&A53_0 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&A53_1 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&A53_2 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
+&A53_3 {
+	cpu-supply = <&reg_vdd_arm>;
+};
+
 &cpu_alert0 {
 	temperature = <95000>;
 };
@@ -286,7 +302,6 @@
 	status = "disabled";
 };
 
-
 /* Verdin CAN_2 */
 &flexcan2 {
 	pinctrl-names = "default";
@@ -454,7 +469,7 @@
 				regulator-ramp-delay = <3125>;
 			};
 
-			BUCK2 {
+			reg_vdd_arm: BUCK2 {
 				nxp,dvs-run-voltage = <950000>;
 				nxp,dvs-standby-voltage = <850000>;
 				regulator-always-on;
diff --git a/arch/arm64/boot/dts/freescale/imx8mp.dtsi b/arch/arm64/boot/dts/freescale/imx8mp.dtsi
index fe178b7d063c..53493dc7d976 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp.dtsi
@@ -5,8 +5,10 @@
 
 #include <dt-bindings/clock/imx8mp-clock.h>
 #include <dt-bindings/power/imx8mp-power.h>
+#include <dt-bindings/reset/imx8mp-reset.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/interconnect/fsl,imx8mp.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/thermal/thermal.h>
 
@@ -469,6 +471,11 @@
 					wakeup-source;
 					status = "disabled";
 				};
+
+				snvs_lpgpr: snvs-lpgpr {
+					compatible = "fsl,imx8mp-snvs-lpgpr",
+						     "fsl,imx7d-snvs-lpgpr";
+				};
 			};
 
 			clk: clock-controller@30380000 {
@@ -597,6 +604,33 @@
 						reg = <IMX8MP_POWER_DOMAIN_MEDIAMIX_ISPDWP>;
 						clocks = <&clk IMX8MP_CLK_MEDIA_ISP_ROOT>;
 					};
+
+					pgc_vpumix: power-domain@19 {
+						#power-domain-cells = <0>;
+						reg = <IMX8MP_POWER_DOMAIN_VPUMIX>;
+						clocks =<&clk IMX8MP_CLK_VPU_ROOT>;
+					};
+
+					pgc_vpu_g1: power-domain@20 {
+						#power-domain-cells = <0>;
+						power-domains = <&pgc_vpumix>;
+						reg = <IMX8MP_POWER_DOMAIN_VPU_G1>;
+						clocks = <&clk IMX8MP_CLK_VPU_G1_ROOT>;
+					};
+
+					pgc_vpu_g2: power-domain@21 {
+						#power-domain-cells = <0>;
+						power-domains = <&pgc_vpumix>;
+						reg = <IMX8MP_POWER_DOMAIN_VPU_G2>;
+						clocks = <&clk IMX8MP_CLK_VPU_G2_ROOT>;
+					};
+
+					pgc_vpu_vc8000e: power-domain@22 {
+						#power-domain-cells = <0>;
+						power-domains = <&pgc_vpumix>;
+						reg = <IMX8MP_POWER_DOMAIN_VPU_VC8000E>;
+						clocks = <&clk IMX8MP_CLK_VPU_VC8KE_ROOT>;
+					};
 				};
 			};
 		};
@@ -1064,6 +1098,18 @@
 						     "lcdif1", "isi", "mipi-csi2",
 						     "lcdif2", "isp", "dwe",
 						     "mipi-dsi2";
+				interconnects =
+					<&noc IMX8MP_ICM_LCDIF_RD &noc IMX8MP_ICN_MEDIA>,
+					<&noc IMX8MP_ICM_LCDIF_WR &noc IMX8MP_ICN_MEDIA>,
+					<&noc IMX8MP_ICM_ISI0 &noc IMX8MP_ICN_MEDIA>,
+					<&noc IMX8MP_ICM_ISI1 &noc IMX8MP_ICN_MEDIA>,
+					<&noc IMX8MP_ICM_ISI2 &noc IMX8MP_ICN_MEDIA>,
+					<&noc IMX8MP_ICM_ISP0 &noc IMX8MP_ICN_MEDIA>,
+					<&noc IMX8MP_ICM_ISP1 &noc IMX8MP_ICN_MEDIA>,
+					<&noc IMX8MP_ICM_DWE &noc IMX8MP_ICN_MEDIA>;
+				interconnect-names = "lcdif-rd", "lcdif-wr", "isi0",
+						     "isi1", "isi2", "isp0", "isp1",
+						     "dwe";
 				clocks = <&clk IMX8MP_CLK_MEDIA_APB_ROOT>,
 					 <&clk IMX8MP_CLK_MEDIA_AXI_ROOT>,
 					 <&clk IMX8MP_CLK_MEDIA_CAM1_PIX_ROOT>,
@@ -1084,6 +1130,17 @@
 				#power-domain-cells = <1>;
 			};
 
+			pcie_phy: pcie-phy@32f00000 {
+				compatible = "fsl,imx8mp-pcie-phy";
+				reg = <0x32f00000 0x10000>;
+				resets = <&src IMX8MP_RESET_PCIEPHY>,
+					 <&src IMX8MP_RESET_PCIEPHY_PERST>;
+				reset-names = "pciephy", "perst";
+				power-domains = <&hsio_blk_ctrl IMX8MP_HSIOBLK_PD_PCIE_PHY>;
+				#phy-cells = <0>;
+				status = "disabled";
+			};
+
 			hsio_blk_ctrl: blk-ctrl@32f10000 {
 				compatible = "fsl,imx8mp-hsio-blk-ctrl", "syscon";
 				reg = <0x32f10000 0x24>;
@@ -1095,10 +1152,46 @@
 						<&pgc_hsiomix>, <&pgc_pcie_phy>;
 				power-domain-names = "bus", "usb", "usb-phy1",
 						     "usb-phy2", "pcie", "pcie-phy";
+				interconnects = <&noc IMX8MP_ICM_NOC_PCIE &noc IMX8MP_ICN_HSIO>,
+						<&noc IMX8MP_ICM_USB1 &noc IMX8MP_ICN_HSIO>,
+						<&noc IMX8MP_ICM_USB2 &noc IMX8MP_ICN_HSIO>,
+						<&noc IMX8MP_ICM_PCIE &noc IMX8MP_ICN_HSIO>;
+				interconnect-names = "noc-pcie", "usb1", "usb2", "pcie";
 				#power-domain-cells = <1>;
 			};
 		};
 
+		pcie: pcie@33800000 {
+			compatible = "fsl,imx8mp-pcie";
+			reg = <0x33800000 0x400000>, <0x1ff00000 0x80000>;
+			reg-names = "dbi", "config";
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			bus-range = <0x00 0xff>;
+			ranges =  <0x81000000 0 0x00000000 0x1ff80000 0 0x00010000>, /* downstream I/O 64KB */
+				  <0x82000000 0 0x18000000 0x18000000 0 0x07f00000>; /* non-prefetchable memory */
+			num-lanes = <1>;
+			num-viewport = <4>;
+			interrupts = <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &gic GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &gic GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gic GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gic GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>;
+			fsl,max-link-speed = <3>;
+			linux,pci-domain = <0>;
+			power-domains = <&hsio_blk_ctrl IMX8MP_HSIOBLK_PD_PCIE>;
+			resets = <&src IMX8MP_RESET_PCIE_CTRL_APPS_EN>,
+				 <&src IMX8MP_RESET_PCIE_CTRL_APPS_TURNOFF>;
+			reset-names = "apps", "turnoff";
+			phys = <&pcie_phy>;
+			phy-names = "pcie-phy";
+			status = "disabled";
+		};
+
 		gpu3d: gpu@38000000 {
 			compatible = "vivante,gc";
 			reg = <0x38000000 0x8000>;
@@ -1130,6 +1223,23 @@
 			power-domains = <&pgc_gpu2d>;
 		};
 
+		vpumix_blk_ctrl: blk-ctrl@38330000 {
+			compatible = "fsl,imx8mp-vpu-blk-ctrl", "syscon";
+			reg = <0x38330000 0x100>;
+			#power-domain-cells = <1>;
+			power-domains = <&pgc_vpumix>, <&pgc_vpu_g1>,
+					<&pgc_vpu_g2>, <&pgc_vpu_vc8000e>;
+			power-domain-names = "bus", "g1", "g2", "vc8000e";
+			clocks = <&clk IMX8MP_CLK_VPU_G1_ROOT>,
+				 <&clk IMX8MP_CLK_VPU_G2_ROOT>,
+				 <&clk IMX8MP_CLK_VPU_VC8KE_ROOT>;
+			clock-names = "g1", "g2", "vc8000e";
+			interconnects = <&noc IMX8MP_ICM_VPU_G1 &noc IMX8MP_ICN_VIDEO>,
+					<&noc IMX8MP_ICM_VPU_G2 &noc IMX8MP_ICN_VIDEO>,
+					<&noc IMX8MP_ICM_VPU_H1 &noc IMX8MP_ICN_VIDEO>;
+			interconnect-names = "g1", "g2", "vc8000e";
+		};
+
 		gic: interrupt-controller@38800000 {
 			compatible = "arm,gic-v3";
 			reg = <0x38800000 0x10000>,
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi
index 9eec8a7eecfc..ae08556b2ef2 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mq-librem5.dtsi
@@ -7,6 +7,7 @@
 
 #include "dt-bindings/input/input.h"
 #include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/leds/common.h>
 #include "dt-bindings/pwm/pwm.h"
 #include "dt-bindings/usb/pd.h"
 #include "imx8mq.dtsi"
@@ -54,6 +55,31 @@
 		};
 	};
 
+	led-controller {
+		compatible = "pwm-leds";
+
+		led-0 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_BLUE>;
+			max-brightness = <248>;
+			pwms = <&pwm2 0 50000 0>;
+		};
+
+		led-1 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			max-brightness = <248>;
+			pwms = <&pwm4 0 50000 0>;
+		};
+
+		led-2 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_RED>;
+			max-brightness = <248>;
+			pwms = <&pwm3 0 50000 0>;
+		};
+	};
+
 	reg_aud_1v8: regulator-audio-1v8 {
 		compatible = "regulator-fixed";
 		pinctrl-names = "default";
@@ -747,6 +773,10 @@
 		interrupt-names = "irq";
 
 		connector {
+			compatible = "usb-c-connector";
+			label = "USB-C";
+			data-role = "dual";
+
 			ports {
 				#address-cells = <1>;
 				#size-cells = <0>;
@@ -1070,6 +1100,12 @@
 	pinctrl-0 = <&pinctrl_i2c4>;
 	status = "okay";
 
+	vcm@c {
+		compatible = "dongwoon,dw9714";
+		reg = <0x0c>;
+		vcc-supply = <&reg_csi_1v8>;
+	};
+
 	bat: fuel-gauge@36 {
 		compatible = "maxim,max17055";
 		reg = <0x36>;
@@ -1077,6 +1113,7 @@
 		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_gauge>;
+		power-supplies = <&bq25895>;
 		maxim,over-heat-temp = <700>;
 		maxim,over-volt = <4500>;
 		maxim,rsns-microohm = <5000>;
@@ -1106,8 +1143,6 @@
 };
 
 &mipi_csi1 {
-	#address-cells = <1>;
-	#size-cells = <0>;
 	status = "okay";
 
 	ports {
@@ -1265,6 +1300,7 @@
 	#size-cells = <0>;
 	dr_mode = "otg";
 	snps,dis_u3_susphy_quirk;
+	usb-role-switch;
 	status = "okay";
 
 	port@0 {
diff --git a/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts b/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts
index 8956a46788fa..055031bba8c4 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mq-mnt-reform2.dts
@@ -210,7 +210,6 @@
 	status = "okay";
 };
 
-
 &reg_1p8v {
 	vin-supply = <&reg_main_5v>;
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mq.dtsi b/arch/arm64/boot/dts/freescale/imx8mq.dtsi
index e9f0cdd10ab6..19eaa523564d 100644
--- a/arch/arm64/boot/dts/freescale/imx8mq.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mq.dtsi
@@ -534,7 +534,7 @@
 				status = "disabled";
 			};
 
-			sdma2: sdma@302c0000 {
+			sdma2: dma-controller@302c0000 {
 				compatible = "fsl,imx8mq-sdma","fsl,imx7d-sdma";
 				reg = <0x302c0000 0x10000>;
 				interrupts = <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>;
@@ -1302,7 +1302,7 @@
 				status = "disabled";
 			};
 
-			sdma1: sdma@30bd0000 {
+			sdma1: dma-controller@30bd0000 {
 				compatible = "fsl,imx8mq-sdma","fsl,imx7d-sdma";
 				reg = <0x30bd0000 0x10000>;
 				interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts b/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts
index 33e84c4e9ed8..f1c6d933a17c 100644
--- a/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts
@@ -19,6 +19,21 @@
 		device_type = "memory";
 		reg = <0x0 0x80000000 0 0x80000000>;
 	};
+
+	clock_ext_rmii: clock-ext-rmii {
+		compatible = "fixed-clock";
+		clock-frequency = <50000000>;
+		clock-output-names = "ext_rmii_clk";
+		#clock-cells = <0>;
+	};
+
+	clock_ext_ts: clock-ext-ts {
+		compatible = "fixed-clock";
+		/* External ts clock is 50MHZ from PHY on EVK board. */
+		clock-frequency = <50000000>;
+		clock-output-names = "ext_ts_clk";
+		#clock-cells = <0>;
+	};
 };
 
 &lpuart5 {
@@ -38,7 +53,49 @@
 	status = "okay";
 };
 
+&fec {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_enet>;
+	pinctrl-1 = <&pinctrl_enet>;
+	clocks = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>,
+		 <&pcc4 IMX8ULP_CLK_ENET>,
+		 <&cgc1 IMX8ULP_CLK_ENET_TS_SEL>,
+		 <&clock_ext_rmii>;
+	clock-names = "ipg", "ahb", "ptp", "enet_clk_ref";
+	assigned-clocks = <&cgc1 IMX8ULP_CLK_ENET_TS_SEL>;
+	assigned-clock-parents = <&clock_ext_ts>;
+	phy-mode = "rmii";
+	phy-handle = <&ethphy>;
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy: ethernet-phy@1 {
+			reg = <1>;
+			micrel,led-mode = <1>;
+		};
+	};
+};
+
 &iomuxc1 {
+	pinctrl_enet: enetgrp {
+		fsl,pins = <
+			MX8ULP_PAD_PTE15__ENET0_MDC     0x43
+			MX8ULP_PAD_PTE14__ENET0_MDIO    0x43
+			MX8ULP_PAD_PTE17__ENET0_RXER    0x43
+			MX8ULP_PAD_PTE18__ENET0_CRS_DV  0x43
+			MX8ULP_PAD_PTF1__ENET0_RXD0     0x43
+			MX8ULP_PAD_PTE20__ENET0_RXD1    0x43
+			MX8ULP_PAD_PTE16__ENET0_TXEN    0x43
+			MX8ULP_PAD_PTE23__ENET0_TXD0    0x43
+			MX8ULP_PAD_PTE22__ENET0_TXD1    0x43
+			MX8ULP_PAD_PTE19__ENET0_REFCLK  0x43
+			MX8ULP_PAD_PTF10__ENET0_1588_CLKIN 0x43
+		>;
+	};
+
 	pinctrl_lpuart5: lpuart5grp {
 		fsl,pins = <
 			MX8ULP_PAD_PTF14__LPUART5_TX	0x3
diff --git a/arch/arm64/boot/dts/freescale/imx8ulp-pinfunc.h b/arch/arm64/boot/dts/freescale/imx8ulp-pinfunc.h
index b204ac79b449..b204ac79b449 100755..100644
--- a/arch/arm64/boot/dts/freescale/imx8ulp-pinfunc.h
+++ b/arch/arm64/boot/dts/freescale/imx8ulp-pinfunc.h
diff --git a/arch/arm64/boot/dts/freescale/imx8ulp.dtsi b/arch/arm64/boot/dts/freescale/imx8ulp.dtsi
index bb56390b8f54..06ce5f19aa8a 100644
--- a/arch/arm64/boot/dts/freescale/imx8ulp.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8ulp.dtsi
@@ -16,6 +16,7 @@
 	#size-cells = <2>;
 
 	aliases {
+		ethernet0 = &fec;
 		gpio0 = &gpiod;
 		gpio1 = &gpioe;
 		gpio2 = &gpiof;
@@ -62,6 +63,14 @@
 		interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
+	pmu {
+		compatible = "arm,cortex-a35-pmu";
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_PPI 7
+			     (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>;
+		interrupt-affinity = <&A35_0>, <&A35_1>;
+	};
+
 	psci {
 		compatible = "arm,psci-1.0";
 		method = "smc";
@@ -111,7 +120,7 @@
 		#size-cells = <1>;
 		ranges = <0 0x0 0x2201f000 0x1000>;
 
-		scmi_buf: scmi-buf@0 {
+		scmi_buf: scmi-sram-section@0 {
 			compatible = "arm,scmi-shmem";
 			reg = <0x0 0x400>;
 		};
@@ -143,6 +152,13 @@
 		#size-cells = <1>;
 		ranges = <0x0 0x0 0x0 0x40000000>;
 
+		s4muap: mailbox@27020000 {
+			compatible = "fsl,imx8ulp-mu-s4";
+			reg = <0x27020000 0x10000>;
+			interrupts = <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>;
+			#mbox-cells = <2>;
+		};
+
 		per_bridge3: bus@29000000 {
 			compatible = "simple-bus";
 			reg = <0x29000000 0x800000>;
@@ -150,6 +166,23 @@
 			#size-cells = <1>;
 			ranges;
 
+			mu: mailbox@29220000 {
+				compatible = "fsl,imx8ulp-mu";
+				reg = <0x29220000 0x10000>;
+				interrupts = <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>;
+				#mbox-cells = <2>;
+				status = "disabled";
+			};
+
+			mu3: mailbox@29230000 {
+				compatible = "fsl,imx8ulp-mu";
+				reg = <0x29230000 0x10000>;
+				interrupts = <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&pcc3 IMX8ULP_CLK_MU3_A>;
+				#mbox-cells = <2>;
+				status = "disabled";
+			};
+
 			wdog3: watchdog@292a0000 {
 				compatible = "fsl,imx8ulp-wdt", "fsl,imx7ulp-wdt";
 				reg = <0x292a0000 0x10000>;
@@ -163,8 +196,6 @@
 			cgc1: clock-controller@292c0000 {
 				compatible = "fsl,imx8ulp-cgc1";
 				reg = <0x292c0000 0x10000>;
-				clocks = <&rosc>, <&sosc>, <&frosc>, <&lposc>;
-				clock-names = "rosc", "sosc", "frosc", "lposc";
 				#clock-cells = <1>;
 			};
 
@@ -193,7 +224,7 @@
 					 <&pcc3 IMX8ULP_CLK_LPI2C4>;
 				clock-names = "per", "ipg";
 				assigned-clocks = <&pcc3 IMX8ULP_CLK_LPI2C4>;
-				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>;
+				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>;
 				assigned-clock-rates = <48000000>;
 				status = "disabled";
 			};
@@ -206,7 +237,7 @@
 					 <&pcc3 IMX8ULP_CLK_LPI2C5>;
 				clock-names = "per", "ipg";
 				assigned-clocks = <&pcc3 IMX8ULP_CLK_LPI2C5>;
-				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>;
+				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>;
 				assigned-clock-rates = <48000000>;
 				status = "disabled";
 			};
@@ -239,8 +270,8 @@
 					 <&pcc3 IMX8ULP_CLK_LPSPI4>;
 				clock-names = "per", "ipg";
 				assigned-clocks = <&pcc3 IMX8ULP_CLK_LPSPI4>;
-				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>;
-				assigned-clock-rates = <16000000>;
+				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>;
+				assigned-clock-rates = <48000000>;
 				status = "disabled";
 			};
 
@@ -254,8 +285,8 @@
 					 <&pcc3 IMX8ULP_CLK_LPSPI5>;
 				clock-names = "per", "ipg";
 				assigned-clocks = <&pcc3 IMX8ULP_CLK_LPSPI5>;
-				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>;
-				assigned-clock-rates = <16000000>;
+				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>;
+				assigned-clock-rates = <48000000>;
 				status = "disabled";
 			};
 		};
@@ -282,7 +313,7 @@
 					 <&pcc4 IMX8ULP_CLK_LPI2C6>;
 				clock-names = "per", "ipg";
 				assigned-clocks = <&pcc4 IMX8ULP_CLK_LPI2C6>;
-				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>;
+				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>;
 				assigned-clock-rates = <48000000>;
 				status = "disabled";
 			};
@@ -295,7 +326,7 @@
 					 <&pcc4 IMX8ULP_CLK_LPI2C7>;
 				clock-names = "per", "ipg";
 				assigned-clocks = <&pcc4 IMX8ULP_CLK_LPI2C7>;
-				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_XBAR_DIVBUS>;
+				assigned-clock-parents = <&cgc1 IMX8ULP_CLK_FROSC_DIV2>;
 				assigned-clock-rates = <48000000>;
 				status = "disabled";
 			};
@@ -367,6 +398,16 @@
 				bus-width = <4>;
 				status = "disabled";
 			};
+
+			fec: ethernet@29950000 {
+				compatible = "fsl,imx8ulp-fec", "fsl,imx6ul-fec", "fsl,imx6q-fec";
+				reg = <0x29950000 0x10000>;
+				interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
+				interrupt-names = "int0";
+				fsl,num-tx-queues = <1>;
+				fsl,num-rx-queues = <1>;
+				status = "disabled";
+			};
 		};
 
 		gpioe: gpio@2d000080 {
@@ -407,8 +448,6 @@
 			cgc2: clock-controller@2da60000 {
 				compatible = "fsl,imx8ulp-cgc2";
 				reg = <0x2da60000 0x10000>;
-				clocks = <&sosc>, <&frosc>;
-				clock-names = "sosc", "frosc";
 				#clock-cells = <1>;
 			};
 
diff --git a/arch/arm64/boot/dts/freescale/imx93.dtsi b/arch/arm64/boot/dts/freescale/imx93.dtsi
index f83a07c7c9b1..3a5713bb4880 100644
--- a/arch/arm64/boot/dts/freescale/imx93.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx93.dtsi
@@ -7,6 +7,7 @@
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/power/fsl,imx93-power.h>
 
 #include "imx93-pinfunc.h"
 
@@ -16,6 +17,14 @@
 	#size-cells = <2>;
 
 	aliases {
+		i2c0 = &lpi2c1;
+		i2c1 = &lpi2c2;
+		i2c2 = &lpi2c3;
+		i2c3 = &lpi2c4;
+		i2c4 = &lpi2c5;
+		i2c5 = &lpi2c6;
+		i2c6 = &lpi2c7;
+		i2c7 = &lpi2c8;
 		mmc0 = &usdhc1;
 		mmc1 = &usdhc2;
 		mmc2 = &usdhc3;
@@ -72,6 +81,11 @@
 		clock-output-names = "clk_ext1";
 	};
 
+	pmu {
+		compatible = "arm,cortex-a55-pmu";
+		interrupts = <GIC_PPI 7 (GIC_CPU_MASK_SIMPLE(2) | IRQ_TYPE_LEVEL_HIGH)>;
+	};
+
 	psci {
 		compatible = "arm,psci-1.0";
 		method = "smc";
@@ -112,6 +126,11 @@
 			#size-cells = <1>;
 			ranges;
 
+			anomix_ns_gpr: syscon@44210000 {
+				compatible = "fsl,imx93-aonmix-ns-syscfg", "syscon";
+				reg = <0x44210000 0x1000>;
+			};
+
 			mu1: mailbox@44230000 {
 				compatible = "fsl,imx93-mu", "fsl,imx8ulp-mu";
 				reg = <0x44230000 0x10000>;
@@ -128,6 +147,50 @@
 				clock-names = "per";
 			};
 
+			lpi2c1: i2c@44340000 {
+				compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c";
+				reg = <0x44340000 0x10000>;
+				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clk IMX93_CLK_LPI2C1_GATE>,
+					 <&clk IMX93_CLK_BUS_AON>;
+				clock-names = "per", "ipg";
+				status = "disabled";
+			};
+
+			lpi2c2: i2c@44350000 {
+				compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c";
+				reg = <0x44350000 0x10000>;
+				interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clk IMX93_CLK_LPI2C2_GATE>,
+					 <&clk IMX93_CLK_BUS_AON>;
+				clock-names = "per", "ipg";
+				status = "disabled";
+			};
+
+			lpspi1: spi@44360000 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				compatible = "fsl,imx93-spi", "fsl,imx7ulp-spi";
+				reg = <0x44360000 0x10000>;
+				interrupts = <GIC_SPI 16 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clk IMX93_CLK_LPSPI1_GATE>,
+					 <&clk IMX93_CLK_BUS_AON>;
+				clock-names = "per", "ipg";
+				status = "disabled";
+			};
+
+			lpspi2: spi@44370000 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				compatible = "fsl,imx93-spi", "fsl,imx7ulp-spi";
+				reg = <0x44370000 0x10000>;
+				interrupts = <GIC_SPI 17 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clk IMX93_CLK_LPSPI2_GATE>,
+					 <&clk IMX93_CLK_BUS_AON>;
+				clock-names = "per", "ipg";
+				status = "disabled";
+			};
+
 			lpuart1: serial@44380000 {
 				compatible = "fsl,imx93-lpuart", "fsl,imx7ulp-lpuart";
 				reg = <0x44380000 0x1000>;
@@ -161,6 +224,30 @@
 				status = "okay";
 			};
 
+			src: system-controller@44460000 {
+				compatible = "fsl,imx93-src", "syscon";
+				reg = <0x44460000 0x10000>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges;
+
+				mediamix: power-domain@44462400 {
+					compatible = "fsl,imx93-src-slice";
+					reg = <0x44462400 0x400>, <0x44465800 0x400>;
+					#power-domain-cells = <0>;
+					clocks = <&clk IMX93_CLK_MEDIA_AXI>,
+						 <&clk IMX93_CLK_MEDIA_APB>;
+				};
+
+				mlmix: power-domain@44461800 {
+					compatible = "fsl,imx93-src-slice";
+					reg = <0x44461800 0x400>, <0x44464800 0x400>;
+					#power-domain-cells = <0>;
+					clocks = <&clk IMX93_CLK_ML_APB>,
+						 <&clk IMX93_CLK_ML>;
+				};
+			};
+
 			anatop: anatop@44480000 {
 				compatible = "fsl,imx93-anatop", "syscon";
 				reg = <0x44480000 0x10000>;
@@ -174,6 +261,11 @@
 			#size-cells = <1>;
 			ranges;
 
+			wakeupmix_gpr: syscon@42420000 {
+				compatible = "fsl,imx93-wakeupmix-syscfg", "syscon";
+				reg = <0x42420000 0x1000>;
+			};
+
 			mu2: mailbox@42440000 {
 				compatible = "fsl,imx93-mu", "fsl,imx8ulp-mu";
 				reg = <0x42440000 0x10000>;
@@ -182,6 +274,26 @@
 				status = "disabled";
 			};
 
+			lpi2c3: i2c@42530000 {
+				compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c";
+				reg = <0x42530000 0x10000>;
+				interrupts = <GIC_SPI 62 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clk IMX93_CLK_LPI2C3_GATE>,
+					 <&clk IMX93_CLK_BUS_WAKEUP>;
+				clock-names = "per", "ipg";
+				status = "disabled";
+			};
+
+			lpi2c4: i2c@42540000 {
+				compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c";
+				reg = <0x42540000 0x10000>;
+				interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clk IMX93_CLK_LPI2C4_GATE>,
+					 <&clk IMX93_CLK_BUS_WAKEUP>;
+				clock-names = "per", "ipg";
+				status = "disabled";
+			};
+
 			lpuart3: serial@42570000 {
 				compatible = "fsl,imx93-lpuart", "fsl,imx7ulp-lpuart";
 				reg = <0x42570000 0x1000>;
@@ -235,6 +347,47 @@
 				clock-names = "ipg";
 				status = "disabled";
 			};
+
+			lpi2c5: i2c@426b0000 {
+				compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c";
+				reg = <0x426b0000 0x10000>;
+				interrupts = <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clk IMX93_CLK_LPI2C5_GATE>,
+					 <&clk IMX93_CLK_BUS_WAKEUP>;
+				clock-names = "per", "ipg";
+				status = "disabled";
+			};
+
+			lpi2c6: i2c@426c0000 {
+				compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c";
+				reg = <0x426c0000 0x10000>;
+				interrupts = <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clk IMX93_CLK_LPI2C6_GATE>,
+					 <&clk IMX93_CLK_BUS_WAKEUP>;
+				clock-names = "per", "ipg";
+				status = "disabled";
+			};
+
+			lpi2c7: i2c@426d0000 {
+				compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c";
+				reg = <0x426d0000 0x10000>;
+				interrupts = <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clk IMX93_CLK_LPI2C7_GATE>,
+					 <&clk IMX93_CLK_BUS_WAKEUP>;
+				clock-names = "per", "ipg";
+				status = "disabled";
+			};
+
+			lpi2c8: i2c@426e0000 {
+				compatible = "fsl,imx93-lpi2c", "fsl,imx7ulp-lpi2c";
+				reg = <0x426e0000 0x10000>;
+				interrupts = <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clk IMX93_CLK_LPI2C8_GATE>,
+					 <&clk IMX93_CLK_BUS_WAKEUP>;
+				clock-names = "per", "ipg";
+				status = "disabled";
+			};
+
 		};
 
 		aips3: bus@42800000 {
@@ -248,8 +401,8 @@
 				compatible = "fsl,imx93-usdhc", "fsl,imx8mm-usdhc";
 				reg = <0x42850000 0x10000>;
 				interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&clk IMX93_CLK_DUMMY>,
-					 <&clk IMX93_CLK_DUMMY>,
+				clocks = <&clk IMX93_CLK_BUS_WAKEUP>,
+					 <&clk IMX93_CLK_WAKEUP_AXI>,
 					 <&clk IMX93_CLK_USDHC1_GATE>;
 				clock-names = "ipg", "ahb", "per";
 				bus-width = <8>;
@@ -262,8 +415,8 @@
 				compatible = "fsl,imx93-usdhc", "fsl,imx8mm-usdhc";
 				reg = <0x42860000 0x10000>;
 				interrupts = <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&clk IMX93_CLK_DUMMY>,
-					 <&clk IMX93_CLK_DUMMY>,
+				clocks = <&clk IMX93_CLK_BUS_WAKEUP>,
+					 <&clk IMX93_CLK_WAKEUP_AXI>,
 					 <&clk IMX93_CLK_USDHC2_GATE>;
 				clock-names = "ipg", "ahb", "per";
 				bus-width = <4>;
@@ -276,8 +429,8 @@
 				compatible = "fsl,imx93-usdhc", "fsl,imx8mm-usdhc";
 				reg = <0x428b0000 0x10000>;
 				interrupts = <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH>;
-				clocks = <&clk IMX93_CLK_DUMMY>,
-					 <&clk IMX93_CLK_DUMMY>,
+				clocks = <&clk IMX93_CLK_BUS_WAKEUP>,
+					 <&clk IMX93_CLK_WAKEUP_AXI>,
 					 <&clk IMX93_CLK_USDHC3_GATE>;
 				clock-names = "ipg", "ahb", "per";
 				bus-width = <4>;
@@ -295,6 +448,9 @@
 			interrupts = <GIC_SPI 57 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			clocks = <&clk IMX93_CLK_GPIO2_GATE>,
+				 <&clk IMX93_CLK_GPIO2_GATE>;
+			clock-names = "gpio", "port";
 			gpio-ranges = <&iomuxc 0 32 32>;
 		};
 
@@ -306,6 +462,9 @@
 			interrupts = <GIC_SPI 59 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			clocks = <&clk IMX93_CLK_GPIO3_GATE>,
+				 <&clk IMX93_CLK_GPIO3_GATE>;
+			clock-names = "gpio", "port";
 			gpio-ranges = <&iomuxc 0 64 32>;
 		};
 
@@ -317,6 +476,9 @@
 			interrupts = <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			clocks = <&clk IMX93_CLK_GPIO4_GATE>,
+				 <&clk IMX93_CLK_GPIO4_GATE>;
+			clock-names = "gpio", "port";
 			gpio-ranges = <&iomuxc 0 96 32>;
 		};
 
@@ -328,7 +490,39 @@
 			interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
+			clocks = <&clk IMX93_CLK_GPIO1_GATE>,
+				 <&clk IMX93_CLK_GPIO1_GATE>;
+			clock-names = "gpio", "port";
 			gpio-ranges = <&iomuxc 0 0 32>;
 		};
+
+		s4muap: mailbox@47520000 {
+			compatible = "fsl,imx93-mu-s4";
+			reg = <0x47520000 0x10000>;
+			interrupts = <GIC_SPI 31 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "txirq", "rxirq";
+			#mbox-cells = <2>;
+		};
+
+		media_blk_ctrl: system-controller@4ac10000 {
+			compatible = "fsl,imx93-media-blk-ctrl", "syscon";
+			reg = <0x4ac10000 0x10000>;
+			power-domains = <&mediamix>;
+			clocks = <&clk IMX93_CLK_MEDIA_APB>,
+				 <&clk IMX93_CLK_MEDIA_AXI>,
+				 <&clk IMX93_CLK_NIC_MEDIA_GATE>,
+				 <&clk IMX93_CLK_MEDIA_DISP_PIX>,
+				 <&clk IMX93_CLK_CAM_PIX>,
+				 <&clk IMX93_CLK_PXP_GATE>,
+				 <&clk IMX93_CLK_LCDIF_GATE>,
+				 <&clk IMX93_CLK_ISI_GATE>,
+				 <&clk IMX93_CLK_MIPI_CSI_GATE>,
+				 <&clk IMX93_CLK_MIPI_DSI_GATE>;
+			clock-names = "apb", "axi", "nic", "disp", "cam",
+				      "pxp", "lcdif", "isi", "csi", "dsi";
+			#power-domain-cells = <1>;
+			status = "disabled";
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/marvell/Makefile b/arch/arm64/boot/dts/marvell/Makefile
index b6d493e34dc5..058237681fe5 100644
--- a/arch/arm64/boot/dts/marvell/Makefile
+++ b/arch/arm64/boot/dts/marvell/Makefile
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 # Mvebu SoC Family
 dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-db.dtb
+dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-eDPU.dtb
 dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-espressobin.dtb
 dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-espressobin-emmc.dtb
 dtb-$(CONFIG_ARCH_MVEBU) += armada-3720-espressobin-ultra.dtb
diff --git a/arch/arm64/boot/dts/marvell/ac5-98dx25xx.dtsi b/arch/arm64/boot/dts/marvell/ac5-98dx25xx.dtsi
index 80b44c7df56a..44ed6f963b75 100644
--- a/arch/arm64/boot/dts/marvell/ac5-98dx25xx.dtsi
+++ b/arch/arm64/boot/dts/marvell/ac5-98dx25xx.dtsi
@@ -95,6 +95,36 @@
 				status = "okay";
 			};
 
+			uart1: serial@12100 {
+				compatible = "snps,dw-apb-uart";
+				reg = <0x11000 0x100>;
+				reg-shift = <2>;
+				interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
+				reg-io-width = <1>;
+				clocks = <&cnm_clock>;
+				status = "disabled";
+			};
+
+			uart2: serial@12200 {
+				compatible = "snps,dw-apb-uart";
+				reg = <0x12200 0x100>;
+				reg-shift = <2>;
+				interrupts = <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>;
+				reg-io-width = <1>;
+				clocks = <&cnm_clock>;
+				status = "disabled";
+			};
+
+			uart3: serial@12300 {
+				compatible = "snps,dw-apb-uart";
+				reg = <0x12300 0x100>;
+				reg-shift = <2>;
+				interrupts = <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>;
+				reg-io-width = <1>;
+				clocks = <&cnm_clock>;
+				status = "disabled";
+			};
+
 			mdio: mdio@22004 {
 				#address-cells = <1>;
 				#size-cells = <0>;
@@ -117,8 +147,8 @@
 				pinctrl-names = "default", "gpio";
 				pinctrl-0 = <&i2c0_pins>;
 				pinctrl-1 = <&i2c0_gpio>;
-				scl_gpio = <&gpio0 26 GPIO_ACTIVE_HIGH>;
-				sda_gpio = <&gpio0 27 GPIO_ACTIVE_HIGH>;
+				scl-gpios = <&gpio0 26 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+				sda-gpios = <&gpio0 27 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 				status = "disabled";
 			};
 
@@ -136,8 +166,8 @@
 				pinctrl-names = "default", "gpio";
 				pinctrl-0 = <&i2c1_pins>;
 				pinctrl-1 = <&i2c1_gpio>;
-				scl_gpio = <&gpio0 20 GPIO_ACTIVE_HIGH>;
-				sda_gpio = <&gpio0 21 GPIO_ACTIVE_HIGH>;
+				scl-gpios = <&gpio0 20 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+				sda-gpios = <&gpio0 21 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 				status = "disabled";
 			};
 
diff --git a/arch/arm64/boot/dts/marvell/armada-3720-eDPU.dts b/arch/arm64/boot/dts/marvell/armada-3720-eDPU.dts
new file mode 100644
index 000000000000..57fc698e55d0
--- /dev/null
+++ b/arch/arm64/boot/dts/marvell/armada-3720-eDPU.dts
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+
+#include "armada-3720-uDPU.dtsi"
+
+/ {
+	model = "Methode eDPU Board";
+	compatible = "methode,edpu", "marvell,armada3720", "marvell,armada3710";
+};
+
+&eth0 {
+	phy-mode = "2500base-x";
+};
diff --git a/arch/arm64/boot/dts/marvell/armada-3720-espressobin-ultra.dts b/arch/arm64/boot/dts/marvell/armada-3720-espressobin-ultra.dts
index 070725b81be5..1b2ed63ae6a2 100644
--- a/arch/arm64/boot/dts/marvell/armada-3720-espressobin-ultra.dts
+++ b/arch/arm64/boot/dts/marvell/armada-3720-espressobin-ultra.dts
@@ -12,8 +12,8 @@
 
 / {
 	model = "Globalscale Marvell ESPRESSOBin Ultra Board";
-	compatible = "globalscale,espressobin-ultra", "marvell,armada3720",
-		     "marvell,armada3710";
+	compatible = "globalscale,espressobin-ultra", "globalscale,espressobin",
+		     "marvell,armada3720", "marvell,armada3710";
 
 	aliases {
 		/* ethernet1 is WAN port */
diff --git a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts
index b20c8e7d923b..c76eceabd33e 100644
--- a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts
+++ b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dts
@@ -1,66 +1,12 @@
 // SPDX-License-Identifier: (GPL-2.0+ OR MIT)
-/*
- * Device tree for the uDPU board.
- * Based on Marvell Armada 3720 development board (DB-88F3720-DDR3)
- * Copyright (C) 2016 Marvell
- * Copyright (C) 2019 Methode Electronics
- * Copyright (C) 2019 Telus
- *
- * Vladimir Vid <vladimir.vid@sartura.hr>
- */
 
 /dts-v1/;
 
-#include <dt-bindings/gpio/gpio.h>
-#include "armada-372x.dtsi"
+#include "armada-3720-uDPU.dtsi"
 
 / {
 	model = "Methode uDPU Board";
-	compatible = "methode,udpu", "marvell,armada3720";
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-
-	memory@0 {
-		device_type = "memory";
-		reg = <0x00000000 0x00000000 0x00000000 0x20000000>;
-	};
-
-	leds {
-		pinctrl-names = "default";
-		compatible = "gpio-leds";
-
-		power1 {
-			label = "udpu:green:power";
-			gpios = <&gpionb 11 GPIO_ACTIVE_LOW>;
-		};
-
-		power2 {
-			label = "udpu:red:power";
-			gpios = <&gpionb 12 GPIO_ACTIVE_LOW>;
-		};
-
-		network1 {
-			label = "udpu:green:network";
-			gpios = <&gpionb 13 GPIO_ACTIVE_LOW>;
-		};
-
-		network2 {
-			label = "udpu:red:network";
-			gpios = <&gpionb 14 GPIO_ACTIVE_LOW>;
-		};
-
-		alarm1 {
-			label = "udpu:green:alarm";
-			gpios = <&gpionb 15 GPIO_ACTIVE_LOW>;
-		};
-
-		alarm2 {
-			label = "udpu:red:alarm";
-			gpios = <&gpionb 16 GPIO_ACTIVE_LOW>;
-		};
-	};
+	compatible = "methode,udpu", "marvell,armada3720", "marvell,armada3710";
 
 	sfp_eth0: sfp-eth0 {
 		compatible = "sff,sfp";
@@ -71,55 +17,6 @@
 		tx-fault-gpios = <&gpiosb 5 GPIO_ACTIVE_HIGH>;
 		maximum-power-milliwatt = <3000>;
 	};
-
-	sfp_eth1: sfp-eth1 {
-		compatible = "sff,sfp";
-		i2c-bus = <&i2c1>;
-		los-gpios = <&gpiosb 7 GPIO_ACTIVE_HIGH>;
-		mod-def0-gpios = <&gpiosb 8 GPIO_ACTIVE_LOW>;
-		tx-disable-gpios = <&gpiosb 9 GPIO_ACTIVE_HIGH>;
-		tx-fault-gpios = <&gpiosb 10 GPIO_ACTIVE_HIGH>;
-		maximum-power-milliwatt = <3000>;
-	};
-};
-
-&sdhci0 {
-	status = "okay";
-	bus-width = <8>;
-	mmc-ddr-1_8v;
-	mmc-hs400-1_8v;
-	marvell,pad-type = "fixed-1-8v";
-	non-removable;
-	no-sd;
-	no-sdio;
-};
-
-&spi0 {
-	status = "okay";
-	pinctrl-names = "default";
-	pinctrl-0 = <&spi_quad_pins>;
-
-	flash@0 {
-		compatible = "jedec,spi-nor";
-		reg = <0>;
-		spi-max-frequency = <54000000>;
-
-		partitions {
-			compatible = "fixed-partitions";
-			#address-cells = <1>;
-			#size-cells = <1>;
-
-			partition@0 {
-				label = "firmware";
-				reg = <0x0 0x180000>;
-			};
-
-			partition@180000 {
-				label = "u-boot-env";
-				reg = <0x180000 0x10000>;
-			};
-		};
-	};
 };
 
 &pinctrl_nb {
@@ -127,11 +24,6 @@
 		groups = "i2c1";
 		function = "gpio";
 	};
-
-	i2c2_recovery_pins: i2c2-recovery-pins {
-		groups = "i2c2";
-		function = "gpio";
-	};
 };
 
 &i2c0 {
@@ -144,50 +36,7 @@
 	sda-gpios = <&gpionb 1 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 };
 
-&i2c1 {
-	status = "okay";
-	pinctrl-names = "default", "recovery";
-	pinctrl-0 = <&i2c2_pins>;
-	pinctrl-1 = <&i2c2_recovery_pins>;
-	/delete-property/mrvl,i2c-fast-mode;
-	scl-gpios = <&gpionb 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
-	sda-gpios = <&gpionb 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
-
-	nct375@48 {
-		status = "okay";
-		compatible = "ti,tmp75c";
-		reg = <0x48>;
-	};
-
-	nct375@49 {
-		status = "okay";
-		compatible = "ti,tmp75c";
-		reg = <0x49>;
-	};
-};
-
 &eth0 {
 	phy-mode = "sgmii";
-	status = "okay";
-	managed = "in-band-status";
-	phys = <&comphy1 0>;
 	sfp = <&sfp_eth0>;
 };
-
-&eth1 {
-	phy-mode = "sgmii";
-	status = "okay";
-	managed = "in-band-status";
-	phys = <&comphy0 1>;
-	sfp = <&sfp_eth1>;
-};
-
-&usb3 {
-	status = "okay";
-	phys = <&usb2_utmi_otg_phy>;
-	phy-names = "usb2-utmi-otg-phy";
-};
-
-&uart0 {
-	status = "okay";
-};
diff --git a/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi
new file mode 100644
index 000000000000..3f79923376fb
--- /dev/null
+++ b/arch/arm64/boot/dts/marvell/armada-3720-uDPU.dtsi
@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Device tree for the uDPU board.
+ * Based on Marvell Armada 3720 development board (DB-88F3720-DDR3)
+ * Copyright (C) 2016 Marvell
+ * Copyright (C) 2019 Methode Electronics
+ * Copyright (C) 2019 Telus
+ *
+ * Vladimir Vid <vladimir.vid@sartura.hr>
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include "armada-372x.dtsi"
+
+/ {
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	memory@0 {
+		device_type = "memory";
+		reg = <0x00000000 0x00000000 0x00000000 0x20000000>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-power1 {
+			label = "udpu:green:power";
+			gpios = <&gpionb 11 GPIO_ACTIVE_LOW>;
+		};
+
+		led-power2 {
+			label = "udpu:red:power";
+			gpios = <&gpionb 12 GPIO_ACTIVE_LOW>;
+		};
+
+		led-network1 {
+			label = "udpu:green:network";
+			gpios = <&gpionb 13 GPIO_ACTIVE_LOW>;
+		};
+
+		led-network2 {
+			label = "udpu:red:network";
+			gpios = <&gpionb 14 GPIO_ACTIVE_LOW>;
+		};
+
+		led-alarm1 {
+			label = "udpu:green:alarm";
+			gpios = <&gpionb 15 GPIO_ACTIVE_LOW>;
+		};
+
+		led-alarm2 {
+			label = "udpu:red:alarm";
+			gpios = <&gpionb 16 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	sfp_eth1: sfp-eth1 {
+		compatible = "sff,sfp";
+		i2c-bus = <&i2c1>;
+		los-gpio = <&gpiosb 7 GPIO_ACTIVE_HIGH>;
+		mod-def0-gpio = <&gpiosb 8 GPIO_ACTIVE_LOW>;
+		tx-disable-gpio = <&gpiosb 9 GPIO_ACTIVE_HIGH>;
+		tx-fault-gpio = <&gpiosb 10 GPIO_ACTIVE_HIGH>;
+		maximum-power-milliwatt = <3000>;
+	};
+};
+
+&sdhci0 {
+	status = "okay";
+	bus-width = <8>;
+	mmc-ddr-1_8v;
+	mmc-hs400-1_8v;
+	marvell,pad-type = "fixed-1-8v";
+	non-removable;
+	no-sd;
+	no-sdio;
+};
+
+&spi0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&spi_quad_pins>;
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <54000000>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			partition@0 {
+				label = "firmware";
+				reg = <0x0 0x180000>;
+			};
+
+			partition@180000 {
+				label = "u-boot-env";
+				reg = <0x180000 0x10000>;
+			};
+		};
+	};
+};
+
+&pinctrl_nb {
+	i2c2_recovery_pins: i2c2-recovery-pins {
+		groups = "i2c2";
+		function = "gpio";
+	};
+};
+
+&i2c1 {
+	status = "okay";
+	pinctrl-names = "default", "recovery";
+	pinctrl-0 = <&i2c2_pins>;
+	pinctrl-1 = <&i2c2_recovery_pins>;
+	/delete-property/mrvl,i2c-fast-mode;
+	scl-gpios = <&gpionb 2 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpionb 3 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+
+	temp-sensor@48 {
+		compatible = "ti,tmp75c";
+		reg = <0x48>;
+	};
+
+	temp-sensor@49 {
+		compatible = "ti,tmp75c";
+		reg = <0x49>;
+	};
+};
+
+&eth0 {
+	status = "okay";
+	managed = "in-band-status";
+	phys = <&comphy1 0>;
+};
+
+&eth1 {
+	phy-mode = "sgmii";
+	status = "okay";
+	managed = "in-band-status";
+	phys = <&comphy0 1>;
+	sfp = <&sfp_eth1>;
+};
+
+&usb3 {
+	status = "okay";
+	phys = <&usb2_utmi_otg_phy>;
+	phy-names = "usb2-utmi-otg-phy";
+};
+
+&uart0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/mediatek/Makefile b/arch/arm64/boot/dts/mediatek/Makefile
index af362a085a02..0ec90cb3ef28 100644
--- a/arch/arm64/boot/dts/mediatek/Makefile
+++ b/arch/arm64/boot/dts/mediatek/Makefile
@@ -37,6 +37,7 @@ dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-kodama-sku32.dtb
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-krane-sku0.dtb
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-kukui-krane-sku176.dtb
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt8183-pumpkin.dtb
+dtb-$(CONFIG_ARCH_MEDIATEK) += mt8186-evb.dtb
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt8192-asurada-hayato-r1.dtb
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt8192-asurada-spherion-r0.dtb
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt8192-evb.dtb
diff --git a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi
index 4797537cb368..e6d7453e56e0 100644
--- a/arch/arm64/boot/dts/mediatek/mt2712e.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt2712e.dtsi
@@ -756,7 +756,7 @@
 		snps,mtl-tx-config = <&mtl_tx_setup>;
 		snps,txpbl = <1>;
 		snps,rxpbl = <1>;
-		clk_csr = <0>;
+		snps,clk-csr = <0>;
 		status = "disabled";
 	};
 
diff --git a/arch/arm64/boot/dts/mediatek/mt6795.dtsi b/arch/arm64/boot/dts/mediatek/mt6795.dtsi
index d4842b4a4eb7..46f0e54be766 100644
--- a/arch/arm64/boot/dts/mediatek/mt6795.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt6795.dtsi
@@ -230,6 +230,14 @@
 			reg = <0 0x10200620 0 0x20>;
 		};
 
+		systimer: timer@10200670 {
+			compatible = "mediatek,mt6795-systimer";
+			reg = <0 0x10200670 0 0x10>;
+			interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&system_clk>;
+			clock-names = "clk13m";
+		};
+
 		gic: interrupt-controller@10221000 {
 			compatible = "arm,gic-400";
 			#interrupt-cells = <3>;
diff --git a/arch/arm64/boot/dts/mediatek/mt7986a-rfb.dts b/arch/arm64/boot/dts/mediatek/mt7986a-rfb.dts
index 882277a52b69..afe37b702eef 100644
--- a/arch/arm64/boot/dts/mediatek/mt7986a-rfb.dts
+++ b/arch/arm64/boot/dts/mediatek/mt7986a-rfb.dts
@@ -115,6 +115,13 @@
 	status = "okay";
 };
 
+&wifi {
+	status = "okay";
+	pinctrl-names = "default", "dbdc";
+	pinctrl-0 = <&wf_2g_5g_pins>;
+	pinctrl-1 = <&wf_dbdc_pins>;
+};
+
 &pio {
 	uart1_pins: uart1-pins {
 		mux {
@@ -129,4 +136,35 @@
 			groups = "uart2";
 		};
 	};
+
+	wf_2g_5g_pins: wf-2g-5g-pins {
+		mux {
+			function = "wifi";
+			groups = "wf_2g", "wf_5g";
+		};
+		conf {
+			pins = "WF0_HB1", "WF0_HB2", "WF0_HB3", "WF0_HB4",
+			       "WF0_HB0", "WF0_HB0_B", "WF0_HB5", "WF0_HB6",
+			       "WF0_HB7", "WF0_HB8", "WF0_HB9", "WF0_HB10",
+			       "WF0_TOP_CLK", "WF0_TOP_DATA", "WF1_HB1",
+			       "WF1_HB2", "WF1_HB3", "WF1_HB4", "WF1_HB0",
+			       "WF1_HB5", "WF1_HB6", "WF1_HB7", "WF1_HB8",
+			       "WF1_TOP_CLK", "WF1_TOP_DATA";
+			drive-strength = <4>;
+		};
+	};
+
+	wf_dbdc_pins: wf-dbdc-pins {
+		mux {
+			function = "wifi";
+			groups = "wf_dbdc";
+		};
+		conf {
+			pins = "WF0_HB1", "WF0_HB2", "WF0_HB3", "WF0_HB4",
+			       "WF0_HB0", "WF0_HB0_B", "WF0_HB5", "WF0_HB6",
+			       "WF0_HB7", "WF0_HB8", "WF0_HB9", "WF0_HB10",
+			       "WF0_TOP_CLK", "WF0_TOP_DATA";
+			drive-strength = <4>;
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt7986a.dtsi b/arch/arm64/boot/dts/mediatek/mt7986a.dtsi
index e3a407d03551..72e0d9722e07 100644
--- a/arch/arm64/boot/dts/mediatek/mt7986a.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt7986a.dtsi
@@ -7,6 +7,7 @@
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/clock/mt7986-clk.h>
+#include <dt-bindings/reset/mt7986-resets.h>
 
 / {
 	interrupt-parent = <&gic>;
@@ -70,6 +71,11 @@
 			reg = <0 0x43000000 0 0x30000>;
 			no-map;
 		};
+
+		wmcpu_emi: wmcpu-reserved@4fc00000 {
+			no-map;
+			reg = <0 0x4fc00000 0 0x00100000>;
+		};
 	};
 
 	timer {
@@ -222,6 +228,28 @@
 			 #reset-cells = <1>;
 		};
 
+		wed_pcie: wed-pcie@10003000 {
+			compatible = "mediatek,mt7986-wed-pcie",
+				     "syscon";
+			reg = <0 0x10003000 0 0x10>;
+		};
+
+		wed0: wed@15010000 {
+			compatible = "mediatek,mt7986-wed",
+				     "syscon";
+			reg = <0 0x15010000 0 0x1000>;
+			interrupt-parent = <&gic>;
+			interrupts = <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
+		wed1: wed@15011000 {
+			compatible = "mediatek,mt7986-wed",
+				     "syscon";
+			reg = <0 0x15011000 0 0x1000>;
+			interrupt-parent = <&gic>;
+			interrupts = <GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
 		eth: ethernet@15100000 {
 			compatible = "mediatek,mt7986-eth";
 			reg = <0 0x15100000 0 0x80000>;
@@ -256,11 +284,30 @@
 						 <&apmixedsys CLK_APMIXED_SGMPLL>;
 			mediatek,ethsys = <&ethsys>;
 			mediatek,sgmiisys = <&sgmiisys0>, <&sgmiisys1>;
+			mediatek,wed-pcie = <&wed_pcie>;
+			mediatek,wed = <&wed0>, <&wed1>;
 			#reset-cells = <1>;
 			#address-cells = <1>;
 			#size-cells = <0>;
 			status = "disabled";
 		};
+
+		wifi: wifi@18000000 {
+			compatible = "mediatek,mt7986-wmac";
+			resets = <&watchdog MT7986_TOPRGU_CONSYS_SW_RST>;
+			reset-names = "consys";
+			clocks = <&topckgen CLK_TOP_CONN_MCUSYS_SEL>,
+				 <&topckgen CLK_TOP_AP2CNN_HOST_SEL>;
+			clock-names = "mcu", "ap2conn";
+			reg = <0 0x18000000 0 0x1000000>,
+			      <0 0x10003000 0 0x1000>,
+			      <0 0x11d10000 0 0x1000>;
+			interrupts = <GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>;
+			memory-region = <&wmcpu_emi>;
+		};
 	};
 
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt7986b-rfb.dts b/arch/arm64/boot/dts/mediatek/mt7986b-rfb.dts
index 0f49d5764ff3..3443013b5971 100644
--- a/arch/arm64/boot/dts/mediatek/mt7986b-rfb.dts
+++ b/arch/arm64/boot/dts/mediatek/mt7986b-rfb.dts
@@ -98,3 +98,43 @@
 		};
 	};
 };
+
+&wifi {
+	status = "okay";
+	pinctrl-names = "default", "dbdc";
+	pinctrl-0 = <&wf_2g_5g_pins>;
+	pinctrl-1 = <&wf_dbdc_pins>;
+};
+
+&pio {
+	wf_2g_5g_pins: wf-2g-5g-pins {
+		mux {
+			function = "wifi";
+			groups = "wf_2g", "wf_5g";
+		};
+		conf {
+			pins = "WF0_HB1", "WF0_HB2", "WF0_HB3", "WF0_HB4",
+			       "WF0_HB0", "WF0_HB0_B", "WF0_HB5", "WF0_HB6",
+			       "WF0_HB7", "WF0_HB8", "WF0_HB9", "WF0_HB10",
+			       "WF0_TOP_CLK", "WF0_TOP_DATA", "WF1_HB1",
+			       "WF1_HB2", "WF1_HB3", "WF1_HB4", "WF1_HB0",
+			       "WF1_HB5", "WF1_HB6", "WF1_HB7", "WF1_HB8",
+			       "WF1_TOP_CLK", "WF1_TOP_DATA";
+			drive-strength = <4>;
+		};
+	};
+
+	wf_dbdc_pins: wf-dbdc-pins {
+		mux {
+			function = "wifi";
+			groups = "wf_dbdc";
+		};
+		conf {
+			pins = "WF0_HB1", "WF0_HB2", "WF0_HB3", "WF0_HB4",
+			       "WF0_HB0", "WF0_HB0_B", "WF0_HB5", "WF0_HB6",
+			       "WF0_HB7", "WF0_HB8", "WF0_HB9", "WF0_HB10",
+			       "WF0_TOP_CLK", "WF0_TOP_DATA";
+			drive-strength = <4>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/mediatek/mt8167.dtsi b/arch/arm64/boot/dts/mediatek/mt8167.dtsi
index 54655f2feb04..fbe1a1128cc6 100644
--- a/arch/arm64/boot/dts/mediatek/mt8167.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8167.dtsi
@@ -36,9 +36,8 @@
 		};
 
 		scpsys: syscon@10006000 {
-			compatible = "syscon", "simple-mfd";
+			compatible = "mediatek,mt8167-scpsys", "syscon", "simple-mfd";
 			reg = <0 0x10006000 0 0x1000>;
-			#power-domain-cells = <1>;
 
 			spm: power-controller {
 				compatible = "mediatek,mt8167-power-controller";
diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
index 6d9513c1f5bf..7640b5158ff9 100644
--- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi
@@ -444,9 +444,8 @@
 		};
 
 		scpsys: syscon@10006000 {
-			compatible = "syscon", "simple-mfd";
+			compatible = "mediatek,mt8173-scpsys", "syscon", "simple-mfd";
 			reg = <0 0x10006000 0 0x1000>;
-			#power-domain-cells = <1>;
 
 			/* System Power Manager */
 			spm: power-controller {
@@ -1468,7 +1467,7 @@
 			clock-names = "venc_sel";
 			assigned-clocks = <&topckgen CLK_TOP_VENC_SEL>;
 			assigned-clock-parents = <&topckgen CLK_TOP_VCODECPLL>;
-			power-domains = <&scpsys MT8173_POWER_DOMAIN_VENC>;
+			power-domains = <&spm MT8173_POWER_DOMAIN_VENC>;
 		};
 
 		jpegdec: jpegdec@18004000 {
@@ -1519,7 +1518,7 @@
 			assigned-clocks = <&topckgen CLK_TOP_VENC_LT_SEL>;
 			assigned-clock-parents =
 				 <&topckgen CLK_TOP_VCODECPLL_370P5>;
-			power-domains = <&scpsys MT8173_POWER_DOMAIN_VENC_LT>;
+			power-domains = <&spm MT8173_POWER_DOMAIN_VENC_LT>;
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt8183-pumpkin.dts b/arch/arm64/boot/dts/mediatek/mt8183-pumpkin.dts
index 530e0c9ce0c9..a1d01639df30 100644
--- a/arch/arm64/boot/dts/mediatek/mt8183-pumpkin.dts
+++ b/arch/arm64/boot/dts/mediatek/mt8183-pumpkin.dts
@@ -7,6 +7,7 @@
 /dts-v1/;
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
 #include "mt8183.dtsi"
 #include "mt6358.dtsi"
 
@@ -122,6 +123,18 @@
 	clock-frequency = <100000>;
 };
 
+&keyboard {
+	pinctrl-names = "default";
+	pinctrl-0 = <&keyboard_pins>;
+	status = "okay";
+	linux,keymap = <MATRIX_KEY(0x00, 0x00, KEY_VOLUMEDOWN)
+			MATRIX_KEY(0x01, 0x00, KEY_VOLUMEUP)>;
+	keypad,num-rows = <2>;
+	keypad,num-columns = <1>;
+	debounce-delay-ms = <32>;
+	mediatek,keys-per-group = <2>;
+};
+
 &mmc0 {
 	status = "okay";
 	pinctrl-names = "default", "state_uhs";
@@ -226,6 +239,14 @@
 		};
 	};
 
+	keyboard_pins: keyboard {
+		pins_keyboard {
+			pinmux = <PINMUX_GPIO91__FUNC_KPROW1>,
+				 <PINMUX_GPIO92__FUNC_KPROW0>,
+				 <PINMUX_GPIO93__FUNC_KPCOL0>;
+		};
+	};
+
 	mmc0_pins_default: mmc0-pins-default {
 		pins_cmd_dat {
 			pinmux = <PINMUX_GPIO123__FUNC_MSDC0_DAT0>,
diff --git a/arch/arm64/boot/dts/mediatek/mt8183.dtsi b/arch/arm64/boot/dts/mediatek/mt8183.dtsi
index 9d32871973a2..a70b669c49ba 100644
--- a/arch/arm64/boot/dts/mediatek/mt8183.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8183.dtsi
@@ -761,9 +761,8 @@
 		};
 
 		scpsys: syscon@10006000 {
-			compatible = "syscon", "simple-mfd";
+			compatible = "mediatek,mt8183-scpsys", "syscon", "simple-mfd";
 			reg = <0 0x10006000 0 0x1000>;
-			#power-domain-cells = <1>;
 
 			/* System Power Manager */
 			spm: power-controller {
@@ -943,6 +942,15 @@
 			clock-names = "spi", "wrap";
 		};
 
+		keyboard: keyboard@10010000 {
+			compatible = "mediatek,mt6779-keypad";
+			reg = <0 0x10010000 0 0x1000>;
+			interrupts = <GIC_SPI 186 IRQ_TYPE_EDGE_FALLING>;
+			clocks = <&clk26m>;
+			clock-names = "kpd";
+			status = "disabled";
+		};
+
 		scp: scp@10500000 {
 			compatible = "mediatek,mt8183-scp";
 			reg = <0 0x10500000 0 0x80000>,
@@ -1691,6 +1699,60 @@
 			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0 0x1000>;
 		};
 
+		mdp3-rdma0@14001000 {
+			compatible = "mediatek,mt8183-mdp3-rdma";
+			reg = <0 0x14001000 0 0x1000>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x1000 0x1000>;
+			mediatek,gce-events = <CMDQ_EVENT_MDP_RDMA0_SOF>,
+					      <CMDQ_EVENT_MDP_RDMA0_EOF>;
+			power-domains = <&spm MT8183_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_MDP_RDMA0>,
+				 <&mmsys CLK_MM_MDP_RSZ1>;
+			iommus = <&iommu M4U_PORT_MDP_RDMA0>;
+			mboxes = <&gce 20 CMDQ_THR_PRIO_LOWEST 0>,
+				 <&gce 21 CMDQ_THR_PRIO_LOWEST 0>;
+		};
+
+		mdp3-rsz0@14003000 {
+			compatible = "mediatek,mt8183-mdp3-rsz";
+			reg = <0 0x14003000 0 0x1000>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x3000 0x1000>;
+			mediatek,gce-events = <CMDQ_EVENT_MDP_RSZ0_SOF>,
+					      <CMDQ_EVENT_MDP_RSZ0_EOF>;
+			clocks = <&mmsys CLK_MM_MDP_RSZ0>;
+		};
+
+		mdp3-rsz1@14004000 {
+			compatible = "mediatek,mt8183-mdp3-rsz";
+			reg = <0 0x14004000 0 0x1000>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x4000 0x1000>;
+			mediatek,gce-events = <CMDQ_EVENT_MDP_RSZ1_SOF>,
+					      <CMDQ_EVENT_MDP_RSZ1_EOF>;
+			clocks = <&mmsys CLK_MM_MDP_RSZ1>;
+		};
+
+		mdp3-wrot0@14005000 {
+			compatible = "mediatek,mt8183-mdp3-wrot";
+			reg = <0 0x14005000 0 0x1000>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x5000 0x1000>;
+			mediatek,gce-events = <CMDQ_EVENT_MDP_WROT0_SOF>,
+					      <CMDQ_EVENT_MDP_WROT0_EOF>;
+			power-domains = <&spm MT8183_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_MDP_WROT0>;
+			iommus = <&iommu M4U_PORT_MDP_WROT0>;
+		};
+
+		mdp3-wdma@14006000 {
+			compatible = "mediatek,mt8183-mdp3-wdma";
+			reg = <0 0x14006000 0 0x1000>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x6000 0x1000>;
+			mediatek,gce-events = <CMDQ_EVENT_MDP_WDMA0_SOF>,
+					      <CMDQ_EVENT_MDP_WDMA0_EOF>;
+			power-domains = <&spm MT8183_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_MDP_WDMA0>;
+			iommus = <&iommu M4U_PORT_MDP_WDMA0>;
+		};
+
 		ovl0: ovl@14008000 {
 			compatible = "mediatek,mt8183-disp-ovl";
 			reg = <0 0x14008000 0 0x1000>;
@@ -1834,6 +1896,15 @@
 			power-domains = <&spm MT8183_POWER_DOMAIN_DISP>;
 		};
 
+		mdp3-ccorr@1401c000 {
+			compatible = "mediatek,mt8183-mdp3-ccorr";
+			reg = <0 0x1401c000 0 0x1000>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0xc000 0x1000>;
+			mediatek,gce-events = <CMDQ_EVENT_MDP_CCORR_SOF>,
+					      <CMDQ_EVENT_MDP_CCORR_EOF>;
+			clocks = <&mmsys CLK_MM_MDP_CCORR>;
+		};
+
 		imgsys: syscon@15020000 {
 			compatible = "mediatek,mt8183-imgsys", "syscon";
 			reg = <0 0x15020000 0 0x1000>;
diff --git a/arch/arm64/boot/dts/mediatek/mt8186-evb.dts b/arch/arm64/boot/dts/mediatek/mt8186-evb.dts
new file mode 100644
index 000000000000..ed74a3617c13
--- /dev/null
+++ b/arch/arm64/boot/dts/mediatek/mt8186-evb.dts
@@ -0,0 +1,220 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright (C) 2022 MediaTek Inc.
+ */
+/dts-v1/;
+#include "mt8186.dtsi"
+
+/ {
+	model = "MediaTek MT8186 evaluation board";
+	compatible = "mediatek,mt8186-evb", "mediatek,mt8186";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:921600n8";
+	};
+
+	memory@40000000 {
+		device_type = "memory";
+		reg = <0 0x40000000 0 0x80000000>;
+	};
+};
+
+&i2c0 {
+	status = "okay";
+
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_pins>;
+};
+
+&i2c1 {
+	status = "okay";
+
+	clock-frequency = <400000>;
+	i2c-scl-internal-delay-ns = <8000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins>;
+};
+
+&i2c2 {
+	status = "okay";
+
+	clock-frequency = <400000>;
+	i2c-scl-internal-delay-ns = <10000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c2_pins>;
+};
+
+&i2c3 {
+	status = "okay";
+
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3_pins>;
+};
+
+&i2c4 {
+	status = "okay";
+
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c4_pins>;
+};
+
+&i2c5 {
+	status = "okay";
+
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c5_pins>;
+};
+
+&i2c6 {
+	status = "okay";
+
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c6_pins>;
+};
+
+&i2c7 {
+	status = "okay";
+
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c7_pins>;
+};
+
+&i2c8 {
+	status = "okay";
+
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_pins>;
+};
+
+&i2c9 {
+	status = "okay";
+
+	clock-frequency = <400000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c9_pins>;
+};
+
+&pio {
+	i2c0_pins: i2c0-default-pins {
+		pins-bus {
+			pinmux = <PINMUX_GPIO128__FUNC_SDA0>,
+				 <PINMUX_GPIO127__FUNC_SCL0>;
+			bias-disable;
+			drive-strength-microamp = <1000>;
+			input-enable;
+		};
+	};
+
+	i2c1_pins: i2c1-default-pins {
+		pins-bus {
+			pinmux = <PINMUX_GPIO130__FUNC_SDA1>,
+				 <PINMUX_GPIO129__FUNC_SCL1>;
+			bias-disable;
+			drive-strength-microamp = <1000>;
+			input-enable;
+		};
+	};
+
+	i2c2_pins: i2c2-default-pins {
+		pins-bus {
+			pinmux = <PINMUX_GPIO132__FUNC_SDA2>,
+				 <PINMUX_GPIO131__FUNC_SCL2>;
+			bias-disable;
+			drive-strength-microamp = <1000>;
+			input-enable;
+		};
+	};
+
+	i2c3_pins: i2c3-default-pins {
+		pins-bus {
+			pinmux = <PINMUX_GPIO134__FUNC_SDA3>,
+				 <PINMUX_GPIO133__FUNC_SCL3>;
+			bias-disable;
+			drive-strength-microamp = <1000>;
+			input-enable;
+		};
+	};
+
+	i2c4_pins: i2c4-default-pins {
+		pins-bus {
+			pinmux = <PINMUX_GPIO136__FUNC_SDA4>,
+				 <PINMUX_GPIO135__FUNC_SCL4>;
+			bias-disable;
+			drive-strength-microamp = <1000>;
+			input-enable;
+		};
+	};
+
+	i2c5_pins: i2c5-default-pins {
+		pins-bus {
+			pinmux = <PINMUX_GPIO138__FUNC_SDA5>,
+				 <PINMUX_GPIO137__FUNC_SCL5>;
+			bias-disable;
+			drive-strength-microamp = <1000>;
+			input-enable;
+		};
+	};
+
+	i2c6_pins: i2c6-default-pins {
+		pins-bus {
+			pinmux = <PINMUX_GPIO140__FUNC_SDA6>,
+				 <PINMUX_GPIO139__FUNC_SCL6>;
+			bias-pull-up = <MTK_PULL_SET_RSEL_001>;
+			drive-strength-microamp = <1000>;
+			input-enable;
+		};
+	};
+
+	i2c7_pins: i2c7-default-pins {
+		pins-bus {
+			pinmux = <PINMUX_GPIO142__FUNC_SDA7>,
+				 <PINMUX_GPIO141__FUNC_SCL7>;
+			bias-disable;
+			drive-strength-microamp = <1000>;
+			input-enable;
+		};
+	};
+
+	i2c8_pins: i2c8-default-pins {
+		pins-bus {
+			pinmux = <PINMUX_GPIO144__FUNC_SDA8>,
+				 <PINMUX_GPIO143__FUNC_SCL8>;
+			bias-disable;
+			drive-strength-microamp = <1000>;
+			input-enable;
+		};
+	};
+
+	i2c9_pins: i2c9-default-pins {
+		pins-bus {
+			pinmux = <PINMUX_GPIO146__FUNC_SDA9>,
+				 <PINMUX_GPIO145__FUNC_SCL9>;
+			bias-pull-up = <MTK_PULL_SET_RSEL_001>;
+			drive-strength-microamp = <1000>;
+			input-enable;
+		};
+	};
+};
+
+&u3phy0 {
+	status = "okay";
+};
+
+&u3phy1 {
+	status = "okay";
+};
+
+&uart0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/mediatek/mt8186.dtsi b/arch/arm64/boot/dts/mediatek/mt8186.dtsi
new file mode 100644
index 000000000000..64693c17af9e
--- /dev/null
+++ b/arch/arm64/boot/dts/mediatek/mt8186.dtsi
@@ -0,0 +1,819 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Copyright (C) 2022 MediaTek Inc.
+ * Author: Allen-KH Cheng <allen-kh.cheng@mediatek.com>
+ */
+/dts-v1/;
+#include <dt-bindings/clock/mt8186-clk.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/pinctrl/mt8186-pinfunc.h>
+#include <dt-bindings/power/mt8186-power.h>
+#include <dt-bindings/phy/phy.h>
+#include <dt-bindings/reset/mt8186-resets.h>
+
+/ {
+	compatible = "mediatek,mt8186";
+	interrupt-parent = <&gic>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+
+				core2 {
+					cpu = <&cpu2>;
+				};
+
+				core3 {
+					cpu = <&cpu3>;
+				};
+
+				core4 {
+					cpu = <&cpu4>;
+				};
+
+				core5 {
+					cpu = <&cpu5>;
+				};
+			};
+
+			cluster1 {
+				core0 {
+					cpu = <&cpu6>;
+				};
+
+				core1 {
+					cpu = <&cpu7>;
+				};
+			};
+		};
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x000>;
+			enable-method = "psci";
+			clock-frequency = <2000000000>;
+			capacity-dmips-mhz = <382>;
+			cpu-idle-states = <&cpu_off_l &cluster_off_l>;
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu1: cpu@100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x100>;
+			enable-method = "psci";
+			clock-frequency = <2000000000>;
+			capacity-dmips-mhz = <382>;
+			cpu-idle-states = <&cpu_off_l &cluster_off_l>;
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu2: cpu@200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x200>;
+			enable-method = "psci";
+			clock-frequency = <2000000000>;
+			capacity-dmips-mhz = <382>;
+			cpu-idle-states = <&cpu_off_l &cluster_off_l>;
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu3: cpu@300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x300>;
+			enable-method = "psci";
+			clock-frequency = <2000000000>;
+			capacity-dmips-mhz = <382>;
+			cpu-idle-states = <&cpu_off_l &cluster_off_l>;
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu4: cpu@400 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x400>;
+			enable-method = "psci";
+			clock-frequency = <2000000000>;
+			capacity-dmips-mhz = <382>;
+			cpu-idle-states = <&cpu_off_l &cluster_off_l>;
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu5: cpu@500 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x500>;
+			enable-method = "psci";
+			clock-frequency = <2000000000>;
+			capacity-dmips-mhz = <382>;
+			cpu-idle-states = <&cpu_off_l &cluster_off_l>;
+			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
+		};
+
+		cpu6: cpu@600 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a76";
+			reg = <0x600>;
+			enable-method = "psci";
+			clock-frequency = <2050000000>;
+			capacity-dmips-mhz = <1024>;
+			cpu-idle-states = <&cpu_off_b &cluster_off_b>;
+			next-level-cache = <&l2_1>;
+			#cooling-cells = <2>;
+		};
+
+		cpu7: cpu@700 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a76";
+			reg = <0x700>;
+			enable-method = "psci";
+			clock-frequency = <2050000000>;
+			capacity-dmips-mhz = <1024>;
+			cpu-idle-states = <&cpu_off_b &cluster_off_b>;
+			next-level-cache = <&l2_1>;
+			#cooling-cells = <2>;
+		};
+
+		idle-states {
+			entry-method = "psci";
+
+			cpu_off_l: cpu-off-l {
+				compatible = "arm,idle-state";
+				arm,psci-suspend-param = <0x00010001>;
+				local-timer-stop;
+				entry-latency-us = <50>;
+				exit-latency-us = <100>;
+				min-residency-us = <1600>;
+			};
+
+			cpu_off_b: cpu-off-b {
+				compatible = "arm,idle-state";
+				arm,psci-suspend-param = <0x00010001>;
+				local-timer-stop;
+				entry-latency-us = <50>;
+				exit-latency-us = <100>;
+				min-residency-us = <1400>;
+			};
+
+			cluster_off_l: cluster-off-l {
+				compatible = "arm,idle-state";
+				arm,psci-suspend-param = <0x01010001>;
+				local-timer-stop;
+				entry-latency-us = <100>;
+				exit-latency-us = <250>;
+				min-residency-us = <2100>;
+			};
+
+			cluster_off_b: cluster-off-b {
+				compatible = "arm,idle-state";
+				arm,psci-suspend-param = <0x01010001>;
+				local-timer-stop;
+				entry-latency-us = <100>;
+				exit-latency-us = <250>;
+				min-residency-us = <1900>;
+			};
+		};
+
+		l2_0: l2-cache0 {
+			compatible = "cache";
+			next-level-cache = <&l3_0>;
+		};
+
+		l2_1: l2-cache1 {
+			compatible = "cache";
+			next-level-cache = <&l3_0>;
+		};
+
+		l3_0: l3-cache {
+			compatible = "cache";
+		};
+	};
+
+	clk13m: oscillator-13m {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <13000000>;
+		clock-output-names = "clk13m";
+	};
+
+	clk26m: oscillator-26m {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <26000000>;
+		clock-output-names = "clk26m";
+	};
+
+	clk32k: oscillator-32k {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		clock-output-names = "clk32k";
+	};
+
+	pmu-a55 {
+		compatible = "arm,cortex-a55-pmu";
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH &ppi_cluster0>;
+	};
+
+	pmu-a76 {
+		compatible = "arm,cortex-a76-pmu";
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH &ppi_cluster1>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW 0>;
+	};
+
+	soc {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		compatible = "simple-bus";
+		ranges;
+
+		gic: interrupt-controller@c000000 {
+			compatible = "arm,gic-v3";
+			#interrupt-cells = <4>;
+			#redistributor-regions = <1>;
+			interrupt-parent = <&gic>;
+			interrupt-controller;
+			reg = <0 0x0c000000 0 0x40000>,
+			      <0 0x0c040000 0 0x200000>;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			ppi-partitions {
+				ppi_cluster0: interrupt-partition-0 {
+					affinity = <&cpu0 &cpu1 &cpu2 &cpu3 &cpu4 &cpu5>;
+				};
+
+				ppi_cluster1: interrupt-partition-1 {
+					affinity = <&cpu6 &cpu7>;
+				};
+			};
+		};
+
+		mcusys: syscon@c53a000 {
+			compatible = "mediatek,mt8186-mcusys", "syscon";
+			reg = <0 0xc53a000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		topckgen: syscon@10000000 {
+			compatible = "mediatek,mt8186-topckgen", "syscon";
+			reg = <0 0x10000000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		infracfg_ao: syscon@10001000 {
+			compatible = "mediatek,mt8186-infracfg_ao", "syscon";
+			reg = <0 0x10001000 0 0x1000>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		pericfg: syscon@10003000 {
+			compatible = "mediatek,mt8186-pericfg", "syscon";
+			reg = <0 0x10003000 0 0x1000>;
+		};
+
+		pio: pinctrl@10005000 {
+			compatible = "mediatek,mt8186-pinctrl";
+			reg = <0 0x10005000 0 0x1000>,
+			      <0 0x10002000 0 0x0200>,
+			      <0 0x10002200 0 0x0200>,
+			      <0 0x10002400 0 0x0200>,
+			      <0 0x10002600 0 0x0200>,
+			      <0 0x10002a00 0 0x0200>,
+			      <0 0x10002c00 0 0x0200>,
+			      <0 0x1000b000 0 0x1000>;
+			reg-names = "iocfg0", "iocfg_lt", "iocfg_lm", "iocfg_lb",
+				    "iocfg_bl", "iocfg_rb", "iocfg_rt", "eint";
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pio 0 0 185>;
+			interrupt-controller;
+			interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH 0>;
+			#interrupt-cells = <2>;
+		};
+
+		watchdog: watchdog@10007000 {
+			compatible = "mediatek,mt8186-wdt",
+				     "mediatek,mt6589-wdt";
+			mediatek,disable-extrst;
+			reg = <0 0x10007000 0 0x1000>;
+			#reset-cells = <1>;
+		};
+
+		apmixedsys: syscon@1000c000 {
+			compatible = "mediatek,mt8186-apmixedsys", "syscon";
+			reg = <0 0x1000c000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		pwrap: pwrap@1000d000 {
+			compatible = "mediatek,mt8186-pwrap", "syscon";
+			reg = <0 0x1000d000 0 0x1000>;
+			reg-names = "pwrap";
+			interrupts = <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&infracfg_ao CLK_INFRA_AO_PMIC_AP>,
+				 <&infracfg_ao CLK_INFRA_AO_PMIC_TMR>;
+			clock-names = "spi", "wrap";
+		};
+
+		systimer: timer@10017000 {
+			compatible = "mediatek,mt8186-timer",
+				     "mediatek,mt6765-timer";
+			reg = <0 0x10017000 0 0x1000>;
+			interrupts = <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&clk13m>;
+		};
+
+		scp: scp@10500000 {
+			compatible = "mediatek,mt8186-scp";
+			reg = <0 0x10500000 0 0x40000>,
+			      <0 0x105c0000 0 0x19080>;
+			reg-names = "sram", "cfg";
+			interrupts = <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH 0>;
+		};
+
+		nor_flash: spi@11000000 {
+			compatible = "mediatek,mt8186-nor";
+			reg = <0 0x11000000 0 0x1000>;
+			clocks = <&topckgen CLK_TOP_SPINOR>,
+				 <&infracfg_ao CLK_INFRA_AO_SPINOR>,
+				 <&infracfg_ao CLK_INFRA_AO_FLASHIF_133M>,
+				 <&infracfg_ao CLK_INFRA_AO_FLASHIF_66M>;
+			clock-names = "spi", "sf", "axi", "axi_s";
+			assigned-clocks = <&topckgen CLK_TOP_SPINOR>;
+			assigned-clock-parents = <&topckgen CLK_TOP_UNIVPLL_D3_D8>;
+			interrupts = <GIC_SPI 293 IRQ_TYPE_LEVEL_HIGH 0>;
+			status = "disabled";
+		};
+
+		auxadc: adc@11001000 {
+			compatible = "mediatek,mt8186-auxadc", "mediatek,mt8173-auxadc";
+			reg = <0 0x11001000 0 0x1000>;
+			#io-channel-cells = <1>;
+			clocks = <&infracfg_ao CLK_INFRA_AO_AUXADC>;
+			clock-names = "main";
+		};
+
+		uart0: serial@11002000 {
+			compatible = "mediatek,mt8186-uart",
+				     "mediatek,mt6577-uart";
+			reg = <0 0x11002000 0 0x1000>;
+			interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART0>;
+			clock-names = "baud", "bus";
+			status = "disabled";
+		};
+
+		uart1: serial@11003000 {
+			compatible = "mediatek,mt8186-uart",
+				     "mediatek,mt6577-uart";
+			reg = <0 0x11003000 0 0x1000>;
+			interrupts = <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART1>;
+			clock-names = "baud", "bus";
+			status = "disabled";
+		};
+
+		i2c0: i2c@11007000 {
+			compatible = "mediatek,mt8186-i2c";
+			reg = <0 0x11007000 0 0x1000>,
+			      <0 0x10200100 0 0x100>;
+			interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C0>,
+				 <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			clock-names = "main", "dma";
+			clock-div = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c1: i2c@11008000 {
+			compatible = "mediatek,mt8186-i2c";
+			reg = <0 0x11008000 0 0x1000>,
+			      <0 0x10200200 0 0x100>;
+			interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C1>,
+				 <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			clock-names = "main", "dma";
+			clock-div = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c2: i2c@11009000 {
+			compatible = "mediatek,mt8186-i2c";
+			reg = <0 0x11009000 0 0x1000>,
+			      <0 0x10200300 0 0x180>;
+			interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C2>,
+				 <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			clock-names = "main", "dma";
+			clock-div = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@1100f000 {
+			compatible = "mediatek,mt8186-i2c";
+			reg = <0 0x1100f000 0 0x1000>,
+			      <0 0x10200480 0 0x100>;
+			interrupts = <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C3>,
+				 <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			clock-names = "main", "dma";
+			clock-div = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c4: i2c@11011000 {
+			compatible = "mediatek,mt8186-i2c";
+			reg = <0 0x11011000 0 0x1000>,
+			      <0 0x10200580 0 0x180>;
+			interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C4>,
+				 <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			clock-names = "main", "dma";
+			clock-div = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c5: i2c@11016000 {
+			compatible = "mediatek,mt8186-i2c";
+			reg = <0 0x11016000 0 0x1000>,
+			      <0 0x10200700 0 0x100>;
+			interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C5>,
+				 <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			clock-names = "main", "dma";
+			clock-div = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c6: i2c@1100d000 {
+			compatible = "mediatek,mt8186-i2c";
+			reg = <0 0x1100d000 0 0x1000>,
+			      <0 0x10200800 0 0x100>;
+			interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C6>,
+				 <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			clock-names = "main", "dma";
+			clock-div = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c7: i2c@11004000 {
+			compatible = "mediatek,mt8186-i2c";
+			reg = <0 0x11004000 0 0x1000>,
+			      <0 0x10200900 0 0x180>;
+			interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C7>,
+				 <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			clock-names = "main", "dma";
+			clock-div = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c8: i2c@11005000 {
+			compatible = "mediatek,mt8186-i2c";
+			reg = <0 0x11005000 0 0x1000>,
+			      <0 0x10200A80 0 0x180>;
+			interrupts = <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C8>,
+				 <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			clock-names = "main", "dma";
+			clock-div = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		spi0: spi@1100a000 {
+			compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0 0x1100a000 0 0x1000>;
+			interrupts = <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&topckgen CLK_TOP_MAINPLL_D5>,
+				 <&topckgen CLK_TOP_SPI>,
+				 <&infracfg_ao CLK_INFRA_AO_SPI0>;
+			clock-names = "parent-clk", "sel-clk", "spi-clk";
+			status = "disabled";
+		};
+
+		pwm0: pwm@1100e000 {
+			compatible = "mediatek,mt8186-disp-pwm", "mediatek,mt8183-disp-pwm";
+			reg = <0 0x1100e000 0 0x1000>;
+			interrupts = <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH 0>;
+			#pwm-cells = <2>;
+			clocks = <&topckgen CLK_TOP_DISP_PWM>,
+				 <&infracfg_ao CLK_INFRA_AO_DISP_PWM>;
+			clock-names = "main", "mm";
+			status = "disabled";
+		};
+
+		spi1: spi@11010000 {
+			compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0 0x11010000 0 0x1000>;
+			interrupts = <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&topckgen CLK_TOP_MAINPLL_D5>,
+				 <&topckgen CLK_TOP_SPI>,
+				 <&infracfg_ao CLK_INFRA_AO_SPI1>;
+			clock-names = "parent-clk", "sel-clk", "spi-clk";
+			status = "disabled";
+		};
+
+		spi2: spi@11012000 {
+			compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0 0x11012000 0 0x1000>;
+			interrupts = <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&topckgen CLK_TOP_MAINPLL_D5>,
+				 <&topckgen CLK_TOP_SPI>,
+				 <&infracfg_ao CLK_INFRA_AO_SPI2>;
+			clock-names = "parent-clk", "sel-clk", "spi-clk";
+			status = "disabled";
+		};
+
+		spi3: spi@11013000 {
+			compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0 0x11013000 0 0x1000>;
+			interrupts = <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&topckgen CLK_TOP_MAINPLL_D5>,
+				 <&topckgen CLK_TOP_SPI>,
+				 <&infracfg_ao CLK_INFRA_AO_SPI3>;
+			clock-names = "parent-clk", "sel-clk", "spi-clk";
+			status = "disabled";
+		};
+
+		spi4: spi@11014000 {
+			compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0 0x11014000 0 0x1000>;
+			interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&topckgen CLK_TOP_MAINPLL_D5>,
+				 <&topckgen CLK_TOP_SPI>,
+				 <&infracfg_ao CLK_INFRA_AO_SPI4>;
+			clock-names = "parent-clk", "sel-clk", "spi-clk";
+			status = "disabled";
+		};
+
+		spi5: spi@11015000 {
+			compatible = "mediatek,mt8186-spi", "mediatek,mt6765-spi";
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0 0x11015000 0 0x1000>;
+			interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&topckgen CLK_TOP_MAINPLL_D5>,
+				 <&topckgen CLK_TOP_SPI>,
+				 <&infracfg_ao CLK_INFRA_AO_SPI5>;
+			clock-names = "parent-clk", "sel-clk", "spi-clk";
+			status = "disabled";
+		};
+
+		imp_iic_wrap: clock-controller@11017000 {
+			compatible = "mediatek,mt8186-imp_iic_wrap";
+			reg = <0 0x11017000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		uart2: serial@11018000 {
+			compatible = "mediatek,mt8186-uart",
+				     "mediatek,mt6577-uart";
+			reg = <0 0x11018000 0 0x1000>;
+			interrupts = <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&clk26m>, <&infracfg_ao CLK_INFRA_AO_UART2>;
+			clock-names = "baud", "bus";
+			status = "disabled";
+		};
+
+		i2c9: i2c@11019000 {
+			compatible = "mediatek,mt8186-i2c";
+			reg = <0 0x11019000 0 0x1000>,
+			      <0 0x10200c00 0 0x180>;
+			interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&imp_iic_wrap CLK_IMP_IIC_WRAP_AP_CLOCK_I2C9>,
+				 <&infracfg_ao CLK_INFRA_AO_AP_DMA>;
+			clock-names = "main", "dma";
+			clock-div = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		mmc0: mmc@11230000 {
+			compatible = "mediatek,mt8186-mmc",
+				     "mediatek,mt8183-mmc";
+			reg = <0 0x11230000 0 0x1000>,
+			      <0 0x11cd0000 0 0x1000>;
+			clocks = <&topckgen CLK_TOP_MSDC50_0>,
+				 <&infracfg_ao CLK_INFRA_AO_MSDC0>,
+				 <&infracfg_ao CLK_INFRA_AO_MSDC0_SRC>;
+			clock-names = "source", "hclk", "source_cg";
+			interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH 0>;
+			assigned-clocks = <&topckgen CLK_TOP_MSDC50_0>;
+			assigned-clock-parents = <&apmixedsys CLK_APMIXED_MSDCPLL>;
+			status = "disabled";
+		};
+
+		mmc1: mmc@11240000 {
+			compatible = "mediatek,mt8186-mmc",
+				     "mediatek,mt8183-mmc";
+			reg = <0 0x11240000 0 0x1000>,
+			      <0 0x11c90000 0 0x1000>;
+			clocks = <&topckgen CLK_TOP_MSDC30_1>,
+				 <&infracfg_ao CLK_INFRA_AO_MSDC1>,
+				 <&infracfg_ao CLK_INFRA_AO_MSDC1_SRC>;
+			clock-names = "source", "hclk", "source_cg";
+			interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH 0>;
+			assigned-clocks = <&topckgen CLK_TOP_MSDC30_1>;
+			assigned-clock-parents = <&topckgen CLK_TOP_MSDCPLL_D2>;
+			status = "disabled";
+		};
+
+		u3phy0: t-phy@11c80000 {
+			compatible = "mediatek,mt8186-tphy",
+				     "mediatek,generic-tphy-v2";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x11c80000 0x1000>;
+			status = "disabled";
+
+			u2port1: usb-phy@0 {
+				reg = <0x0 0x700>;
+				clocks = <&clk26m>;
+				clock-names = "ref";
+				#phy-cells = <1>;
+			};
+
+			u3port1: usb-phy@700 {
+				reg = <0x700 0x900>;
+				clocks = <&clk26m>;
+				clock-names = "ref";
+				#phy-cells = <1>;
+			};
+		};
+
+		u3phy1: t-phy@11ca0000 {
+			compatible = "mediatek,mt8186-tphy",
+				     "mediatek,generic-tphy-v2";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0x0 0x0 0x11ca0000 0x1000>;
+			status = "disabled";
+
+			u2port0: usb-phy@0 {
+				reg = <0x0 0x700>;
+				clocks = <&clk26m>;
+				clock-names = "ref";
+				#phy-cells = <1>;
+				mediatek,discth = <0x8>;
+			};
+		};
+
+		efuse: efuse@11cb0000 {
+			compatible = "mediatek,mt8186-efuse", "mediatek,efuse";
+			reg = <0 0x11cb0000 0 0x1000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+		};
+
+		mipi_tx0: dsi-phy@11cc0000 {
+			compatible = "mediatek,mt8183-mipi-tx";
+			reg = <0 0x11cc0000 0 0x1000>;
+			clocks = <&clk26m>;
+			#clock-cells = <0>;
+			#phy-cells = <0>;
+			clock-output-names = "mipi_tx0_pll";
+			status = "disabled";
+		};
+
+		mfgsys: clock-controller@13000000 {
+			compatible = "mediatek,mt8186-mfgsys";
+			reg = <0 0x13000000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		mmsys: syscon@14000000 {
+			compatible = "mediatek,mt8186-mmsys", "syscon";
+			reg = <0 0x14000000 0 0x1000>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		wpesys: clock-controller@14020000 {
+			compatible = "mediatek,mt8186-wpesys";
+			reg = <0 0x14020000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		imgsys1: clock-controller@15020000 {
+			compatible = "mediatek,mt8186-imgsys1";
+			reg = <0 0x15020000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		imgsys2: clock-controller@15820000 {
+			compatible = "mediatek,mt8186-imgsys2";
+			reg = <0 0x15820000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		vdecsys: clock-controller@1602f000 {
+			compatible = "mediatek,mt8186-vdecsys";
+			reg = <0 0x1602f000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		vencsys: clock-controller@17000000 {
+			compatible = "mediatek,mt8186-vencsys";
+			reg = <0 0x17000000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		camsys: clock-controller@1a000000 {
+			compatible = "mediatek,mt8186-camsys";
+			reg = <0 0x1a000000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		camsys_rawa: clock-controller@1a04f000 {
+			compatible = "mediatek,mt8186-camsys_rawa";
+			reg = <0 0x1a04f000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		camsys_rawb: clock-controller@1a06f000 {
+			compatible = "mediatek,mt8186-camsys_rawb";
+			reg = <0 0x1a06f000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		mdpsys: clock-controller@1b000000 {
+			compatible = "mediatek,mt8186-mdpsys";
+			reg = <0 0x1b000000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		ipesys: clock-controller@1c000000 {
+			compatible = "mediatek,mt8186-ipesys";
+			reg = <0 0x1c000000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/mediatek/mt8192.dtsi b/arch/arm64/boot/dts/mediatek/mt8192.dtsi
index cbae5a5ee4a0..6b20376191a7 100644
--- a/arch/arm64/boot/dts/mediatek/mt8192.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8192.dtsi
@@ -6,12 +6,14 @@
 
 /dts-v1/;
 #include <dt-bindings/clock/mt8192-clk.h>
+#include <dt-bindings/gce/mt8192-gce.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
 #include <dt-bindings/memory/mt8192-larb-port.h>
 #include <dt-bindings/pinctrl/mt8192-pinfunc.h>
 #include <dt-bindings/phy/phy.h>
 #include <dt-bindings/power/mt8192-power.h>
+#include <dt-bindings/reset/mt8192-resets.h>
 
 / {
 	compatible = "mediatek,mt8192";
@@ -19,6 +21,14 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	aliases {
+		ovl0 = &ovl0;
+		ovl-2l0 = &ovl_2l0;
+		ovl-2l2 = &ovl_2l2;
+		rdma0 = &rdma0;
+		rdma4 = &rdma4;
+	};
+
 	clk26m: oscillator0 {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
@@ -306,9 +316,8 @@
 		};
 
 		scpsys: syscon@10006000 {
-			compatible = "syscon", "simple-mfd";
+			compatible = "mediatek,mt8192-scpsys", "syscon", "simple-mfd";
 			reg = <0 0x10006000 0 0x1000>;
-			#power-domain-cells = <1>;
 
 			/* System Power Manager */
 			spm: power-controller {
@@ -553,6 +562,15 @@
 			assigned-clock-parents = <&topckgen CLK_TOP_OSC_D10>;
 		};
 
+		gce: mailbox@10228000 {
+			compatible = "mediatek,mt8192-gce";
+			reg = <0 0x10228000 0 0x4000>;
+			interrupts = <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH 0>;
+			#mbox-cells = <2>;
+			clocks = <&infracfg CLK_INFRA_GCE>;
+			clock-names = "gce";
+		};
+
 		scp_adsp: clock-controller@10720000 {
 			compatible = "mediatek,mt8192-scp_adsp";
 			reg = <0 0x10720000 0 0x1000>;
@@ -599,6 +617,17 @@
 			status = "disabled";
 		};
 
+		pwm0: pwm@1100e000 {
+			compatible = "mediatek,mt8183-disp-pwm";
+			reg = <0 0x1100e000 0 0x1000>;
+			interrupts = <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH 0>;
+			#pwm-cells = <2>;
+			clocks = <&topckgen CLK_TOP_DISP_PWM_SEL>,
+				 <&infracfg CLK_INFRA_DISP_PWM>;
+			clock-names = "main", "mm";
+			status = "disabled";
+		};
+
 		spi1: spi@11010000 {
 			compatible = "mediatek,mt8192-spi",
 				     "mediatek,mt6765-spi";
@@ -724,9 +753,12 @@
 			assigned-clock-parents = <&topckgen CLK_TOP_UNIVPLL_D5_D4>,
 						 <&topckgen CLK_TOP_UNIVPLL_D5_D4>;
 			clocks = <&infracfg CLK_INFRA_SSUSB>,
-				 <&infracfg CLK_INFRA_SSUSB_XHCI>,
-				 <&apmixedsys CLK_APMIXED_USBPLL>;
-			clock-names = "sys_ck", "xhci_ck", "ref_ck";
+				 <&apmixedsys CLK_APMIXED_USBPLL>,
+				 <&clk26m>,
+				 <&clk26m>,
+				 <&infracfg CLK_INFRA_SSUSB_XHCI>;
+			clock-names = "sys_ck", "ref_ck", "mcu_ck", "dma_ck",
+				      "xhci_ck";
 			wakeup-source;
 			mediatek,syscon-wakeup = <&pericfg 0x420 102>;
 			status = "disabled";
@@ -1084,6 +1116,16 @@
 			};
 		};
 
+		mipi_tx0: dsi-phy@11e50000 {
+			compatible = "mediatek,mt8183-mipi-tx";
+			reg = <0 0x11e50000 0 0x1000>;
+			clocks = <&apmixedsys CLK_APMIXED_MIPID26M>;
+			#clock-cells = <0>;
+			#phy-cells = <0>;
+			clock-output-names = "mipi_tx0_pll";
+			status = "disabled";
+		};
+
 		i2c0: i2c@11f00000 {
 			compatible = "mediatek,mt8192-i2c";
 			reg = <0 0x11f00000 0 0x1000>,
@@ -1166,6 +1208,20 @@
 			compatible = "mediatek,mt8192-mmsys", "syscon";
 			reg = <0 0x14000000 0 0x1000>;
 			#clock-cells = <1>;
+			#reset-cells = <1>;
+			mboxes = <&gce 0 CMDQ_THR_PRIO_HIGHEST>,
+				 <&gce 1 CMDQ_THR_PRIO_HIGHEST>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0 0x1000>;
+		};
+
+		mutex: mutex@14001000 {
+			compatible = "mediatek,mt8192-disp-mutex";
+			reg = <0 0x14001000 0 0x1000>;
+			interrupts = <GIC_SPI 252 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&mmsys CLK_MM_DISP_MUTEX0>;
+			mediatek,gce-events = <CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_0>,
+					      <CMDQ_EVENT_DISP_STREAM_DONE_ENG_EVENT_1>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
 		};
 
 		smi_common: smi@14002000 {
@@ -1199,6 +1255,140 @@
 			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
 		};
 
+		ovl0: ovl@14005000 {
+			compatible = "mediatek,mt8192-disp-ovl";
+			reg = <0 0x14005000 0 0x1000>;
+			interrupts = <GIC_SPI 254 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&mmsys CLK_MM_DISP_OVL0>;
+			iommus = <&iommu0 M4U_PORT_L0_OVL_RDMA0>,
+				 <&iommu0 M4U_PORT_L0_OVL_RDMA0_HDR>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x5000 0x1000>;
+		};
+
+		ovl_2l0: ovl@14006000 {
+			compatible = "mediatek,mt8192-disp-ovl-2l";
+			reg = <0 0x14006000 0 0x1000>;
+			interrupts = <GIC_SPI 255 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_DISP_OVL0_2L>;
+			iommus = <&iommu0 M4U_PORT_L1_OVL_2L_RDMA0>,
+				 <&iommu0 M4U_PORT_L1_OVL_2L_RDMA0_HDR>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x6000 0x1000>;
+		};
+
+		rdma0: rdma@14007000 {
+			compatible = "mediatek,mt8192-disp-rdma",
+				     "mediatek,mt8183-disp-rdma";
+			reg = <0 0x14007000 0 0x1000>;
+			interrupts = <GIC_SPI 256 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&mmsys CLK_MM_DISP_RDMA0>;
+			iommus = <&iommu0 M4U_PORT_L0_DISP_RDMA0>;
+			mediatek,rdma-fifo-size = <5120>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x7000 0x1000>;
+		};
+
+		color0: color@14009000 {
+			compatible = "mediatek,mt8192-disp-color",
+				     "mediatek,mt8173-disp-color";
+			reg = <0 0x14009000 0 0x1000>;
+			interrupts = <GIC_SPI 258 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_DISP_COLOR0>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0x9000 0x1000>;
+		};
+
+		ccorr0: ccorr@1400a000 {
+			compatible = "mediatek,mt8192-disp-ccorr";
+			reg = <0 0x1400a000 0 0x1000>;
+			interrupts = <GIC_SPI 259 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_DISP_CCORR0>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xa000 0x1000>;
+		};
+
+		aal0: aal@1400b000 {
+			compatible = "mediatek,mt8192-disp-aal",
+				     "mediatek,mt8183-disp-aal";
+			reg = <0 0x1400b000 0 0x1000>;
+			interrupts = <GIC_SPI 260 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_DISP_AAL0>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xb000 0x1000>;
+		};
+
+		gamma0: gamma@1400c000 {
+			compatible = "mediatek,mt8192-disp-gamma",
+				     "mediatek,mt8183-disp-gamma";
+			reg = <0 0x1400c000 0 0x1000>;
+			interrupts = <GIC_SPI 261 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_DISP_GAMMA0>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xc000 0x1000>;
+		};
+
+		postmask0: postmask@1400d000 {
+			compatible = "mediatek,mt8192-disp-postmask";
+			reg = <0 0x1400d000 0 0x1000>;
+			interrupts = <GIC_SPI 262 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_DISP_POSTMASK0>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xd000 0x1000>;
+		};
+
+		dither0: dither@1400e000 {
+			compatible = "mediatek,mt8192-disp-dither",
+				     "mediatek,mt8183-disp-dither";
+			reg = <0 0x1400e000 0 0x1000>;
+			interrupts = <GIC_SPI 263 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_DISP_DITHER0>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1400XXXX 0xe000 0x1000>;
+		};
+
+		dsi0: dsi@14010000 {
+			compatible = "mediatek,mt8183-dsi";
+			reg = <0 0x14010000 0 0x1000>;
+			interrupts = <GIC_SPI 265 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&mmsys CLK_MM_DSI0>,
+				 <&mmsys CLK_MM_DSI_DSI0>,
+				 <&mipi_tx0>;
+			clock-names = "engine", "digital", "hs";
+			phys = <&mipi_tx0>;
+			phy-names = "dphy";
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			resets = <&mmsys MT8192_MMSYS_SW0_RST_B_DISP_DSI0>;
+			status = "disabled";
+
+			port {
+				dsi_out: endpoint { };
+			};
+		};
+
+		ovl_2l2: ovl@14014000 {
+			compatible = "mediatek,mt8192-disp-ovl-2l";
+			reg = <0 0x14014000 0 0x1000>;
+			interrupts = <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_DISP_OVL2_2L>;
+			iommus = <&iommu0 M4U_PORT_L1_OVL_2L_RDMA2>,
+				 <&iommu0 M4U_PORT_L1_OVL_2L_RDMA2_HDR>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0x4000 0x1000>;
+		};
+
+		rdma4: rdma@14015000 {
+			compatible = "mediatek,mt8192-disp-rdma",
+				     "mediatek,mt8183-disp-rdma";
+			reg = <0 0x14015000 0 0x1000>;
+			interrupts = <GIC_SPI 269 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8192_POWER_DOMAIN_DISP>;
+			clocks = <&mmsys CLK_MM_DISP_RDMA4>;
+			iommus = <&iommu0 M4U_PORT_L1_DISP_RDMA4>;
+			mediatek,rdma-fifo-size = <2048>;
+			mediatek,gce-client-reg = <&gce SUBSYS_1401XXXX 0x5000 0x1000>;
+		};
+
 		dpi0: dpi@14016000 {
 			compatible = "mediatek,mt8192-dpi";
 			reg = <0 0x14016000 0 0x1000>;
diff --git a/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi b/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
index fcc600674339..9b62e161db26 100644
--- a/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
@@ -4,6 +4,7 @@
  */
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/spmi/spmi.h>
 #include "mt8195.dtsi"
 #include "mt6359.dtsi"
 
@@ -17,6 +18,7 @@
 		i2c5 = &i2c5;
 		i2c7 = &i2c7;
 		mmc0 = &mmc0;
+		mmc1 = &mmc1;
 		serial0 = &uart0;
 	};
 
@@ -104,6 +106,18 @@
 		enable-active-high;
 		regulator-always-on;
 	};
+
+	reserved_memory: reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		scp_mem: memory@50000000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x50000000 0 0x2900000>;
+			no-map;
+		};
+	};
 };
 
 &i2c0 {
@@ -121,6 +135,16 @@
 	i2c-scl-internal-delay-ns = <12500>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c1_pins>;
+
+	trackpad@15 {
+		compatible = "elan,ekth3000";
+		reg = <0x15>;
+		interrupts-extended = <&pio 6 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&trackpad_pins>;
+		vcc-supply = <&pp3300_s3>;
+		wakeup-source;
+	};
 };
 
 &i2c2 {
@@ -137,6 +161,14 @@
 	clock-frequency = <400000>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c3_pins>;
+
+	tpm@50 {
+		compatible = "google,cr50";
+		reg = <0x50>;
+		interrupts-extended = <&pio 88 IRQ_TYPE_EDGE_FALLING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cr50_int>;
+	};
 };
 
 &i2c4 {
@@ -207,6 +239,24 @@
 	vqmmc-supply = <&mt6359_vufs_ldo_reg>;
 };
 
+&mmc1 {
+	status = "okay";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cd-gpios = <&pio 54 GPIO_ACTIVE_LOW>;
+	max-frequency = <200000000>;
+	no-mmc;
+	no-sdio;
+	pinctrl-names = "default", "state_uhs";
+	pinctrl-0 = <&mmc1_pins_default>, <&mmc1_pins_detect>;
+	pinctrl-1 = <&mmc1_pins_default>;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
+	vmmc-supply = <&mt_pmic_vmch_ldo_reg>;
+	vqmmc-supply = <&mt_pmic_vmc_ldo_reg>;
+};
+
 /* for CPU-L */
 &mt6359_vcore_buck_reg {
 	regulator-always-on;
@@ -414,6 +464,21 @@
 		"AP_SPI_FLASH_MOSI",
 		"AP_SPI_FLASH_MISO";
 
+	cr50_int: cr50-irq-default-pins {
+		pins-gsc-ap-int-odl {
+			pinmux = <PINMUX_GPIO88__FUNC_GPIO88>;
+			input-enable;
+		};
+	};
+
+	cros_ec_int: cros-ec-irq-default-pins {
+		pins-ec-ap-int-odl {
+			pinmux = <PINMUX_GPIO4__FUNC_GPIO4>;
+			bias-pull-up = <MTK_PUPD_SET_R1R0_01>;
+			input-enable;
+		};
+	};
+
 	i2c0_pins: i2c0-default-pins {
 		pins-bus {
 			pinmux = <PINMUX_GPIO8__FUNC_SDA0>,
@@ -540,6 +605,32 @@
 		};
 	};
 
+	mmc1_pins_detect: mmc1-detect-pins {
+		pins-insert {
+			pinmux = <PINMUX_GPIO54__FUNC_GPIO54>;
+			bias-pull-up;
+		};
+	};
+
+	mmc1_pins_default: mmc1-default-pins {
+		pins-cmd-dat {
+			pinmux = <PINMUX_GPIO110__FUNC_MSDC1_CMD>,
+				 <PINMUX_GPIO112__FUNC_MSDC1_DAT0>,
+				 <PINMUX_GPIO113__FUNC_MSDC1_DAT1>,
+				 <PINMUX_GPIO114__FUNC_MSDC1_DAT2>,
+				 <PINMUX_GPIO115__FUNC_MSDC1_DAT3>;
+			input-enable;
+			drive-strength = <8>;
+			bias-pull-up = <MTK_PUPD_SET_R1R0_01>;
+		};
+
+		pins-clk {
+			pinmux = <PINMUX_GPIO111__FUNC_MSDC1_CLK>;
+			drive-strength = <8>;
+			bias-pull-down = <MTK_PUPD_SET_R1R0_10>;
+		};
+	};
+
 	nor_pins_default: nor-default-pins {
 		pins-ck-io {
 			pinmux = <PINMUX_GPIO142__FUNC_SPINOR_IO0>,
@@ -600,6 +691,14 @@
 		};
 	};
 
+	scp_pins: scp-default-pins {
+		pins-vreq {
+			pinmux = <PINMUX_GPIO76__FUNC_SCP_VREQ_VAO>;
+			bias-disable;
+			input-enable;
+		};
+	};
+
 	spi0_pins: spi0-default-pins {
 		pins-cs-mosi-clk {
 			pinmux = <PINMUX_GPIO132__FUNC_SPIM0_CSB>,
@@ -622,6 +721,14 @@
 		};
 	};
 
+	trackpad_pins: trackpad-default-pins {
+		pins-int-n {
+			pinmux = <PINMUX_GPIO6__FUNC_GPIO6>;
+			input-enable;
+			bias-pull-up;
+		};
+	};
+
 	touchscreen_pins: touchscreen-default-pins {
 		pins-int-n {
 			pinmux = <PINMUX_GPIO92__FUNC_GPIO92>;
@@ -643,12 +750,128 @@
 	interrupts-extended = <&pio 222 IRQ_TYPE_LEVEL_HIGH>;
 };
 
+&scp {
+	status = "okay";
+
+	firmware-name = "mediatek/mt8195/scp.img";
+	memory-region = <&scp_mem>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&scp_pins>;
+
+	cros-ec-rpmsg {
+		compatible = "google,cros-ec-rpmsg";
+		mediatek,rpmsg-name = "cros-ec-rpmsg";
+	};
+};
+
 &spi0 {
 	status = "okay";
 
 	pinctrl-names = "default";
 	pinctrl-0 = <&spi0_pins>;
 	mediatek,pad-select = <0>;
+
+	cros_ec: ec@0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		compatible = "google,cros-ec-spi";
+		reg = <0>;
+		interrupts-extended = <&pio 4 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&cros_ec_int>;
+		spi-max-frequency = <3000000>;
+
+		keyboard-backlight {
+			compatible = "google,cros-kbd-led-backlight";
+		};
+
+		i2c_tunnel: i2c-tunnel {
+			compatible = "google,cros-ec-i2c-tunnel";
+			google,remote-bus = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+		};
+
+		mt_pmic_vmc_ldo_reg: regulator@0 {
+			compatible = "google,cros-ec-regulator";
+			reg = <0>;
+			regulator-name = "mt_pmic_vmc_ldo";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <3600000>;
+		};
+
+		mt_pmic_vmch_ldo_reg: regulator@1 {
+			compatible = "google,cros-ec-regulator";
+			reg = <1>;
+			regulator-name = "mt_pmic_vmch_ldo";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3600000>;
+		};
+
+		typec {
+			compatible = "google,cros-ec-typec";
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			usb_c0: connector@0 {
+				compatible = "usb-c-connector";
+				reg = <0>;
+				power-role = "dual";
+				data-role = "host";
+				try-power-role = "source";
+			};
+
+			usb_c1: connector@1 {
+				compatible = "usb-c-connector";
+				reg = <1>;
+				power-role = "dual";
+				data-role = "host";
+				try-power-role = "source";
+			};
+		};
+	};
+};
+
+&spmi {
+	#address-cells = <2>;
+	#size-cells = <0>;
+
+	mt6315@6 {
+		compatible = "mediatek,mt6315-regulator";
+		reg = <0x6 SPMI_USID>;
+
+		regulators {
+			mt6315_6_vbuck1: vbuck1 {
+				regulator-compatible = "vbuck1";
+				regulator-name = "Vbcpu";
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1193750>;
+				regulator-enable-ramp-delay = <256>;
+				regulator-ramp-delay = <6250>;
+				regulator-allowed-modes = <0 1 2>;
+				regulator-always-on;
+			};
+		};
+	};
+
+	mt6315@7 {
+		compatible = "mediatek,mt6315-regulator";
+		reg = <0x7 SPMI_USID>;
+
+		regulators {
+			mt6315_7_vbuck1: vbuck1 {
+				regulator-compatible = "vbuck1";
+				regulator-name = "Vgpu";
+				regulator-min-microvolt = <625000>;
+				regulator-max-microvolt = <1193750>;
+				regulator-enable-ramp-delay = <256>;
+				regulator-ramp-delay = <6250>;
+				regulator-allowed-modes = <0 1 2>;
+				regulator-always-on;
+			};
+		};
+	};
 };
 
 &u3phy0 {
@@ -700,3 +923,36 @@
 	vusb33-supply = <&mt6359_vusb_ldo_reg>;
 	vbus-supply = <&usb_vbus>;
 };
+
+#include <arm/cros-ec-keyboard.dtsi>
+#include <arm/cros-ec-sbs.dtsi>
+
+&keyboard_controller {
+	function-row-physmap = <
+		MATRIX_KEY(0x00, 0x02, 0)	/* T1 */
+		MATRIX_KEY(0x03, 0x02, 0)	/* T2 */
+		MATRIX_KEY(0x02, 0x02, 0)	/* T3 */
+		MATRIX_KEY(0x01, 0x02, 0)	/* T4 */
+		MATRIX_KEY(0x03, 0x04, 0)	/* T5 */
+		MATRIX_KEY(0x02, 0x04, 0)	/* T6 */
+		MATRIX_KEY(0x01, 0x04, 0)	/* T7 */
+		MATRIX_KEY(0x02, 0x09, 0)	/* T8 */
+		MATRIX_KEY(0x01, 0x09, 0)	/* T9 */
+		MATRIX_KEY(0x00, 0x04, 0)	/* T10 */
+	>;
+
+	linux,keymap = <
+		MATRIX_KEY(0x00, 0x02, KEY_BACK)
+		MATRIX_KEY(0x03, 0x02, KEY_REFRESH)
+		MATRIX_KEY(0x02, 0x02, KEY_ZOOM)
+		MATRIX_KEY(0x01, 0x02, KEY_SCALE)
+		MATRIX_KEY(0x03, 0x04, KEY_SYSRQ)
+		MATRIX_KEY(0x02, 0x04, KEY_BRIGHTNESSDOWN)
+		MATRIX_KEY(0x01, 0x04, KEY_BRIGHTNESSUP)
+		MATRIX_KEY(0x02, 0x09, KEY_MUTE)
+		MATRIX_KEY(0x01, 0x09, KEY_VOLUMEDOWN)
+		MATRIX_KEY(0x00, 0x04, KEY_VOLUMEUP)
+
+		CROS_STD_MAIN_KEYMAP
+	>;
+};
diff --git a/arch/arm64/boot/dts/mediatek/mt8195.dtsi b/arch/arm64/boot/dts/mediatek/mt8195.dtsi
index 066c14989708..905d1a90b406 100644
--- a/arch/arm64/boot/dts/mediatek/mt8195.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8195.dtsi
@@ -6,10 +6,13 @@
 
 /dts-v1/;
 #include <dt-bindings/clock/mt8195-clk.h>
+#include <dt-bindings/gce/mt8195-gce.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/memory/mt8195-memory-port.h>
 #include <dt-bindings/phy/phy.h>
 #include <dt-bindings/pinctrl/mt8195-pinfunc.h>
+#include <dt-bindings/power/mt8195-power.h>
 
 / {
 	compatible = "mediatek,mt8195";
@@ -17,6 +20,11 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
+	aliases {
+		gce0 = &gce0;
+		gce1 = &gce1;
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -26,6 +34,7 @@
 			compatible = "arm,cortex-a55";
 			reg = <0x000>;
 			enable-method = "psci";
+			performance-domains = <&performance 0>;
 			clock-frequency = <1701000000>;
 			capacity-dmips-mhz = <578>;
 			cpu-idle-states = <&cpu_off_l &cluster_off_l>;
@@ -38,6 +47,7 @@
 			compatible = "arm,cortex-a55";
 			reg = <0x100>;
 			enable-method = "psci";
+			performance-domains = <&performance 0>;
 			clock-frequency = <1701000000>;
 			capacity-dmips-mhz = <578>;
 			cpu-idle-states = <&cpu_off_l &cluster_off_l>;
@@ -50,6 +60,7 @@
 			compatible = "arm,cortex-a55";
 			reg = <0x200>;
 			enable-method = "psci";
+			performance-domains = <&performance 0>;
 			clock-frequency = <1701000000>;
 			capacity-dmips-mhz = <578>;
 			cpu-idle-states = <&cpu_off_l &cluster_off_l>;
@@ -62,6 +73,7 @@
 			compatible = "arm,cortex-a55";
 			reg = <0x300>;
 			enable-method = "psci";
+			performance-domains = <&performance 0>;
 			clock-frequency = <1701000000>;
 			capacity-dmips-mhz = <578>;
 			cpu-idle-states = <&cpu_off_l &cluster_off_l>;
@@ -74,6 +86,7 @@
 			compatible = "arm,cortex-a78";
 			reg = <0x400>;
 			enable-method = "psci";
+			performance-domains = <&performance 1>;
 			clock-frequency = <2171000000>;
 			capacity-dmips-mhz = <1024>;
 			cpu-idle-states = <&cpu_off_b &cluster_off_b>;
@@ -86,6 +99,7 @@
 			compatible = "arm,cortex-a78";
 			reg = <0x500>;
 			enable-method = "psci";
+			performance-domains = <&performance 1>;
 			clock-frequency = <2171000000>;
 			capacity-dmips-mhz = <1024>;
 			cpu-idle-states = <&cpu_off_b &cluster_off_b>;
@@ -98,6 +112,7 @@
 			compatible = "arm,cortex-a78";
 			reg = <0x600>;
 			enable-method = "psci";
+			performance-domains = <&performance 1>;
 			clock-frequency = <2171000000>;
 			capacity-dmips-mhz = <1024>;
 			cpu-idle-states = <&cpu_off_b &cluster_off_b>;
@@ -110,6 +125,7 @@
 			compatible = "arm,cortex-a78";
 			reg = <0x700>;
 			enable-method = "psci";
+			performance-domains = <&performance 1>;
 			clock-frequency = <2171000000>;
 			capacity-dmips-mhz = <1024>;
 			cpu-idle-states = <&cpu_off_b &cluster_off_b>;
@@ -217,6 +233,17 @@
 		       <&cpu4>, <&cpu5>, <&cpu6>, <&cpu7>;
 	};
 
+	dmic_codec: dmic-codec {
+		compatible = "dmic-codec";
+		num-channels = <2>;
+		wakeup-delay-ms = <50>;
+	};
+
+	sound: mt8195-sound {
+		mediatek,platform = <&afe>;
+		status = "disabled";
+	};
+
 	clk26m: oscillator-26m {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
@@ -231,6 +258,12 @@
 		clock-output-names = "clk32k";
 	};
 
+	performance: performance-controller@11bc10 {
+		compatible = "mediatek,cpufreq-hw";
+		reg = <0 0x0011bc10 0 0x120>, <0 0x0011bd30 0 0x120>;
+		#performance-domain-cells = <1>;
+	};
+
 	pmu-a55 {
 		compatible = "arm,cortex-a55-pmu";
 		interrupt-parent = <&gic>;
@@ -324,10 +357,337 @@
 			#interrupt-cells = <2>;
 		};
 
+		scpsys: syscon@10006000 {
+			compatible = "mediatek,mt8195-scpsys", "syscon", "simple-mfd";
+			reg = <0 0x10006000 0 0x1000>;
+
+			/* System Power Manager */
+			spm: power-controller {
+				compatible = "mediatek,mt8195-power-controller";
+				#address-cells = <1>;
+				#size-cells = <0>;
+				#power-domain-cells = <1>;
+
+				/* power domain of the SoC */
+				mfg0: power-domain@MT8195_POWER_DOMAIN_MFG0 {
+					reg = <MT8195_POWER_DOMAIN_MFG0>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+					#power-domain-cells = <1>;
+
+					power-domain@MT8195_POWER_DOMAIN_MFG1 {
+						reg = <MT8195_POWER_DOMAIN_MFG1>;
+						clocks = <&apmixedsys CLK_APMIXED_MFGPLL>;
+						clock-names = "mfg";
+						mediatek,infracfg = <&infracfg_ao>;
+						#address-cells = <1>;
+						#size-cells = <0>;
+						#power-domain-cells = <1>;
+
+						power-domain@MT8195_POWER_DOMAIN_MFG2 {
+							reg = <MT8195_POWER_DOMAIN_MFG2>;
+							#power-domain-cells = <0>;
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_MFG3 {
+							reg = <MT8195_POWER_DOMAIN_MFG3>;
+							#power-domain-cells = <0>;
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_MFG4 {
+							reg = <MT8195_POWER_DOMAIN_MFG4>;
+							#power-domain-cells = <0>;
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_MFG5 {
+							reg = <MT8195_POWER_DOMAIN_MFG5>;
+							#power-domain-cells = <0>;
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_MFG6 {
+							reg = <MT8195_POWER_DOMAIN_MFG6>;
+							#power-domain-cells = <0>;
+						};
+					};
+				};
+
+				power-domain@MT8195_POWER_DOMAIN_VPPSYS0 {
+					reg = <MT8195_POWER_DOMAIN_VPPSYS0>;
+					clocks = <&topckgen CLK_TOP_VPP>,
+						 <&topckgen CLK_TOP_CAM>,
+						 <&topckgen CLK_TOP_CCU>,
+						 <&topckgen CLK_TOP_IMG>,
+						 <&topckgen CLK_TOP_VENC>,
+						 <&topckgen CLK_TOP_VDEC>,
+						 <&topckgen CLK_TOP_WPE_VPP>,
+						 <&topckgen CLK_TOP_CFG_VPP0>,
+						 <&vppsys0 CLK_VPP0_SMI_COMMON>,
+						 <&vppsys0 CLK_VPP0_GALS_VDO0_LARB0>,
+						 <&vppsys0 CLK_VPP0_GALS_VDO0_LARB1>,
+						 <&vppsys0 CLK_VPP0_GALS_VENCSYS>,
+						 <&vppsys0 CLK_VPP0_GALS_VENCSYS_CORE1>,
+						 <&vppsys0 CLK_VPP0_GALS_INFRA>,
+						 <&vppsys0 CLK_VPP0_GALS_CAMSYS>,
+						 <&vppsys0 CLK_VPP0_GALS_VPP1_LARB5>,
+						 <&vppsys0 CLK_VPP0_GALS_VPP1_LARB6>,
+						 <&vppsys0 CLK_VPP0_SMI_REORDER>,
+						 <&vppsys0 CLK_VPP0_SMI_IOMMU>,
+						 <&vppsys0 CLK_VPP0_GALS_IMGSYS_CAMSYS>,
+						 <&vppsys0 CLK_VPP0_GALS_EMI0_EMI1>,
+						 <&vppsys0 CLK_VPP0_SMI_SUB_COMMON_REORDER>,
+						 <&vppsys0 CLK_VPP0_SMI_RSI>,
+						 <&vppsys0 CLK_VPP0_SMI_COMMON_LARB4>,
+						 <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>,
+						 <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>,
+						 <&vppsys0 CLK_VPP0_GALS_VDO0_VDO1_VENCSYS_CORE1>;
+					clock-names = "vppsys", "vppsys1", "vppsys2", "vppsys3",
+						      "vppsys4", "vppsys5", "vppsys6", "vppsys7",
+						      "vppsys0-0", "vppsys0-1", "vppsys0-2", "vppsys0-3",
+						      "vppsys0-4", "vppsys0-5", "vppsys0-6", "vppsys0-7",
+						      "vppsys0-8", "vppsys0-9", "vppsys0-10", "vppsys0-11",
+						      "vppsys0-12", "vppsys0-13", "vppsys0-14",
+						      "vppsys0-15", "vppsys0-16", "vppsys0-17",
+						      "vppsys0-18";
+					mediatek,infracfg = <&infracfg_ao>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+					#power-domain-cells = <1>;
+
+					power-domain@MT8195_POWER_DOMAIN_VDEC1 {
+						reg = <MT8195_POWER_DOMAIN_VDEC1>;
+						clocks = <&vdecsys CLK_VDEC_LARB1>;
+						clock-names = "vdec1-0";
+						mediatek,infracfg = <&infracfg_ao>;
+						#power-domain-cells = <0>;
+					};
+
+					power-domain@MT8195_POWER_DOMAIN_VENC_CORE1 {
+						reg = <MT8195_POWER_DOMAIN_VENC_CORE1>;
+						mediatek,infracfg = <&infracfg_ao>;
+						#power-domain-cells = <0>;
+					};
+
+					power-domain@MT8195_POWER_DOMAIN_VDOSYS0 {
+						reg = <MT8195_POWER_DOMAIN_VDOSYS0>;
+						clocks = <&topckgen CLK_TOP_CFG_VDO0>,
+							 <&vdosys0 CLK_VDO0_SMI_GALS>,
+							 <&vdosys0 CLK_VDO0_SMI_COMMON>,
+							 <&vdosys0 CLK_VDO0_SMI_EMI>,
+							 <&vdosys0 CLK_VDO0_SMI_IOMMU>,
+							 <&vdosys0 CLK_VDO0_SMI_LARB>,
+							 <&vdosys0 CLK_VDO0_SMI_RSI>;
+						clock-names = "vdosys0", "vdosys0-0", "vdosys0-1",
+							      "vdosys0-2", "vdosys0-3",
+							      "vdosys0-4", "vdosys0-5";
+						mediatek,infracfg = <&infracfg_ao>;
+						#address-cells = <1>;
+						#size-cells = <0>;
+						#power-domain-cells = <1>;
+
+						power-domain@MT8195_POWER_DOMAIN_VPPSYS1 {
+							reg = <MT8195_POWER_DOMAIN_VPPSYS1>;
+							clocks = <&topckgen CLK_TOP_CFG_VPP1>,
+								 <&vppsys1 CLK_VPP1_VPPSYS1_GALS>,
+								 <&vppsys1 CLK_VPP1_VPPSYS1_LARB>;
+							clock-names = "vppsys1", "vppsys1-0",
+								      "vppsys1-1";
+							mediatek,infracfg = <&infracfg_ao>;
+							#power-domain-cells = <0>;
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_WPESYS {
+							reg = <MT8195_POWER_DOMAIN_WPESYS>;
+							clocks = <&wpesys CLK_WPE_SMI_LARB7>,
+								 <&wpesys CLK_WPE_SMI_LARB8>,
+								 <&wpesys CLK_WPE_SMI_LARB7_P>,
+								 <&wpesys CLK_WPE_SMI_LARB8_P>;
+							clock-names = "wepsys-0", "wepsys-1", "wepsys-2",
+								      "wepsys-3";
+							mediatek,infracfg = <&infracfg_ao>;
+							#power-domain-cells = <0>;
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_VDEC0 {
+							reg = <MT8195_POWER_DOMAIN_VDEC0>;
+							clocks = <&vdecsys_soc CLK_VDEC_SOC_LARB1>;
+							clock-names = "vdec0-0";
+							mediatek,infracfg = <&infracfg_ao>;
+							#power-domain-cells = <0>;
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_VDEC2 {
+							reg = <MT8195_POWER_DOMAIN_VDEC2>;
+							clocks = <&vdecsys_core1 CLK_VDEC_CORE1_LARB1>;
+							clock-names = "vdec2-0";
+							mediatek,infracfg = <&infracfg_ao>;
+							#power-domain-cells = <0>;
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_VENC {
+							reg = <MT8195_POWER_DOMAIN_VENC>;
+							mediatek,infracfg = <&infracfg_ao>;
+							#power-domain-cells = <0>;
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_VDOSYS1 {
+							reg = <MT8195_POWER_DOMAIN_VDOSYS1>;
+							clocks = <&topckgen CLK_TOP_CFG_VDO1>,
+								 <&vdosys1 CLK_VDO1_SMI_LARB2>,
+								 <&vdosys1 CLK_VDO1_SMI_LARB3>,
+								 <&vdosys1 CLK_VDO1_GALS>;
+							clock-names = "vdosys1", "vdosys1-0",
+								      "vdosys1-1", "vdosys1-2";
+							mediatek,infracfg = <&infracfg_ao>;
+							#address-cells = <1>;
+							#size-cells = <0>;
+							#power-domain-cells = <1>;
+
+							power-domain@MT8195_POWER_DOMAIN_DP_TX {
+								reg = <MT8195_POWER_DOMAIN_DP_TX>;
+								mediatek,infracfg = <&infracfg_ao>;
+								#power-domain-cells = <0>;
+							};
+
+							power-domain@MT8195_POWER_DOMAIN_EPD_TX {
+								reg = <MT8195_POWER_DOMAIN_EPD_TX>;
+								mediatek,infracfg = <&infracfg_ao>;
+								#power-domain-cells = <0>;
+							};
+
+							power-domain@MT8195_POWER_DOMAIN_HDMI_TX {
+								reg = <MT8195_POWER_DOMAIN_HDMI_TX>;
+								clocks = <&topckgen CLK_TOP_HDMI_APB>;
+								clock-names = "hdmi_tx";
+								#power-domain-cells = <0>;
+							};
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_IMG {
+							reg = <MT8195_POWER_DOMAIN_IMG>;
+							clocks = <&imgsys CLK_IMG_LARB9>,
+								 <&imgsys CLK_IMG_GALS>;
+							clock-names = "img-0", "img-1";
+							mediatek,infracfg = <&infracfg_ao>;
+							#address-cells = <1>;
+							#size-cells = <0>;
+							#power-domain-cells = <1>;
+
+							power-domain@MT8195_POWER_DOMAIN_DIP {
+								reg = <MT8195_POWER_DOMAIN_DIP>;
+								#power-domain-cells = <0>;
+							};
+
+							power-domain@MT8195_POWER_DOMAIN_IPE {
+								reg = <MT8195_POWER_DOMAIN_IPE>;
+								clocks = <&topckgen CLK_TOP_IPE>,
+									 <&imgsys CLK_IMG_IPE>,
+									 <&ipesys CLK_IPE_SMI_LARB12>;
+								clock-names = "ipe", "ipe-0", "ipe-1";
+								mediatek,infracfg = <&infracfg_ao>;
+								#power-domain-cells = <0>;
+							};
+						};
+
+						power-domain@MT8195_POWER_DOMAIN_CAM {
+							reg = <MT8195_POWER_DOMAIN_CAM>;
+							clocks = <&camsys CLK_CAM_LARB13>,
+								 <&camsys CLK_CAM_LARB14>,
+								 <&camsys CLK_CAM_CAM2MM0_GALS>,
+								 <&camsys CLK_CAM_CAM2MM1_GALS>,
+								 <&camsys CLK_CAM_CAM2SYS_GALS>;
+							clock-names = "cam-0", "cam-1", "cam-2", "cam-3",
+								      "cam-4";
+							mediatek,infracfg = <&infracfg_ao>;
+							#address-cells = <1>;
+							#size-cells = <0>;
+							#power-domain-cells = <1>;
+
+							power-domain@MT8195_POWER_DOMAIN_CAM_RAWA {
+								reg = <MT8195_POWER_DOMAIN_CAM_RAWA>;
+								#power-domain-cells = <0>;
+							};
+
+							power-domain@MT8195_POWER_DOMAIN_CAM_RAWB {
+								reg = <MT8195_POWER_DOMAIN_CAM_RAWB>;
+								#power-domain-cells = <0>;
+							};
+
+							power-domain@MT8195_POWER_DOMAIN_CAM_MRAW {
+								reg = <MT8195_POWER_DOMAIN_CAM_MRAW>;
+								#power-domain-cells = <0>;
+							};
+						};
+					};
+				};
+
+				power-domain@MT8195_POWER_DOMAIN_PCIE_MAC_P0 {
+					reg = <MT8195_POWER_DOMAIN_PCIE_MAC_P0>;
+					mediatek,infracfg = <&infracfg_ao>;
+					#power-domain-cells = <0>;
+				};
+
+				power-domain@MT8195_POWER_DOMAIN_PCIE_MAC_P1 {
+					reg = <MT8195_POWER_DOMAIN_PCIE_MAC_P1>;
+					mediatek,infracfg = <&infracfg_ao>;
+					#power-domain-cells = <0>;
+				};
+
+				power-domain@MT8195_POWER_DOMAIN_PCIE_PHY {
+					reg = <MT8195_POWER_DOMAIN_PCIE_PHY>;
+					#power-domain-cells = <0>;
+				};
+
+				power-domain@MT8195_POWER_DOMAIN_SSUSB_PCIE_PHY {
+					reg = <MT8195_POWER_DOMAIN_SSUSB_PCIE_PHY>;
+					#power-domain-cells = <0>;
+				};
+
+				power-domain@MT8195_POWER_DOMAIN_CSI_RX_TOP {
+					reg = <MT8195_POWER_DOMAIN_CSI_RX_TOP>;
+					clocks = <&topckgen CLK_TOP_SENINF>,
+						 <&topckgen CLK_TOP_SENINF2>;
+					clock-names = "csi_rx_top", "csi_rx_top1";
+					#power-domain-cells = <0>;
+				};
+
+				power-domain@MT8195_POWER_DOMAIN_ETHER {
+					reg = <MT8195_POWER_DOMAIN_ETHER>;
+					clocks = <&pericfg_ao CLK_PERI_AO_ETHERNET_MAC>;
+					clock-names = "ether";
+					#power-domain-cells = <0>;
+				};
+
+				power-domain@MT8195_POWER_DOMAIN_ADSP {
+					reg = <MT8195_POWER_DOMAIN_ADSP>;
+					clocks = <&topckgen CLK_TOP_ADSP>,
+						 <&topckgen CLK_TOP_AUDIO_LOCAL_BUS>;
+					clock-names = "adsp", "adsp1";
+					#address-cells = <1>;
+					#size-cells = <0>;
+					mediatek,infracfg = <&infracfg_ao>;
+					#power-domain-cells = <1>;
+
+					power-domain@MT8195_POWER_DOMAIN_AUDIO {
+						reg = <MT8195_POWER_DOMAIN_AUDIO>;
+						clocks = <&topckgen CLK_TOP_A1SYS_HP>,
+							 <&topckgen CLK_TOP_AUD_INTBUS>,
+							 <&topckgen CLK_TOP_AUDIO_LOCAL_BUS>,
+							 <&infracfg_ao CLK_INFRA_AO_AUDIO_26M_B>;
+						clock-names = "audio", "audio1", "audio2",
+							      "audio3";
+						mediatek,infracfg = <&infracfg_ao>;
+						#power-domain-cells = <0>;
+					};
+				};
+			};
+		};
+
 		watchdog: watchdog@10007000 {
 			compatible = "mediatek,mt8195-wdt",
 				     "mediatek,mt6589-wdt";
+			mediatek,disable-extrst;
 			reg = <0 0x10007000 0 0x100>;
+			#reset-cells = <1>;
 		};
 
 		apmixedsys: syscon@1000c000 {
@@ -356,12 +716,150 @@
 			assigned-clock-parents = <&topckgen CLK_TOP_ULPOSC1_D10>;
 		};
 
+		spmi: spmi@10027000 {
+			compatible = "mediatek,mt8195-spmi";
+			reg = <0 0x10027000 0 0x000e00>,
+			      <0 0x10029000 0 0x000100>;
+			reg-names = "pmif", "spmimst";
+			clocks = <&infracfg_ao CLK_INFRA_AO_PMIC_AP>,
+				 <&infracfg_ao CLK_INFRA_AO_PMIC_TMR>,
+				 <&topckgen CLK_TOP_SPMI_M_MST>;
+			clock-names = "pmif_sys_ck",
+				      "pmif_tmr_ck",
+				      "spmimst_clk_mux";
+			assigned-clocks = <&topckgen CLK_TOP_PWRAP_ULPOSC>;
+			assigned-clock-parents = <&topckgen CLK_TOP_ULPOSC1_D10>;
+		};
+
+		iommu_infra: infra-iommu@10315000 {
+			compatible = "mediatek,mt8195-iommu-infra";
+			reg = <0 0x10315000 0 0x5000>;
+			interrupts = <GIC_SPI 795 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 796 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 797 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 798 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 799 IRQ_TYPE_LEVEL_HIGH 0>;
+			#iommu-cells = <1>;
+		};
+
+		gce0: mailbox@10320000 {
+			compatible = "mediatek,mt8195-gce";
+			reg = <0 0x10320000 0 0x4000>;
+			interrupts = <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH 0>;
+			#mbox-cells = <2>;
+			clocks = <&infracfg_ao CLK_INFRA_AO_GCE>;
+		};
+
+		gce1: mailbox@10330000 {
+			compatible = "mediatek,mt8195-gce";
+			reg = <0 0x10330000 0 0x4000>;
+			interrupts = <GIC_SPI 228 IRQ_TYPE_LEVEL_HIGH 0>;
+			#mbox-cells = <2>;
+			clocks = <&infracfg_ao CLK_INFRA_AO_GCE2>;
+		};
+
+		scp: scp@10500000 {
+			compatible = "mediatek,mt8195-scp";
+			reg = <0 0x10500000 0 0x100000>,
+			      <0 0x10720000 0 0xe0000>,
+			      <0 0x10700000 0 0x8000>;
+			reg-names = "sram", "cfg", "l1tcm";
+			interrupts = <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH 0>;
+			status = "disabled";
+		};
+
 		scp_adsp: clock-controller@10720000 {
 			compatible = "mediatek,mt8195-scp_adsp";
 			reg = <0 0x10720000 0 0x1000>;
 			#clock-cells = <1>;
 		};
 
+		adsp: dsp@10803000 {
+			compatible = "mediatek,mt8195-dsp";
+			reg = <0 0x10803000 0 0x1000>,
+			      <0 0x10840000 0 0x40000>;
+			reg-names = "cfg", "sram";
+			clocks = <&topckgen CLK_TOP_ADSP>,
+				 <&clk26m>,
+				 <&topckgen CLK_TOP_AUDIO_LOCAL_BUS>,
+				 <&topckgen CLK_TOP_MAINPLL_D7_D2>,
+				 <&scp_adsp CLK_SCP_ADSP_AUDIODSP>,
+				 <&topckgen CLK_TOP_AUDIO_H>;
+			clock-names = "adsp_sel",
+				 "clk26m_ck",
+				 "audio_local_bus",
+				 "mainpll_d7_d2",
+				 "scp_adsp_audiodsp",
+				 "audio_h";
+			power-domains = <&spm MT8195_POWER_DOMAIN_ADSP>;
+			mbox-names = "rx", "tx";
+			mboxes = <&adsp_mailbox0>, <&adsp_mailbox1>;
+			status = "disabled";
+		};
+
+		adsp_mailbox0: mailbox@10816000 {
+			compatible = "mediatek,mt8195-adsp-mbox";
+			#mbox-cells = <0>;
+			reg = <0 0x10816000 0 0x1000>;
+			interrupts = <GIC_SPI 702 IRQ_TYPE_LEVEL_HIGH 0>;
+		};
+
+		adsp_mailbox1: mailbox@10817000 {
+			compatible = "mediatek,mt8195-adsp-mbox";
+			#mbox-cells = <0>;
+			reg = <0 0x10817000 0 0x1000>;
+			interrupts = <GIC_SPI 703 IRQ_TYPE_LEVEL_HIGH 0>;
+		};
+
+		afe: mt8195-afe-pcm@10890000 {
+			compatible = "mediatek,mt8195-audio";
+			reg = <0 0x10890000 0 0x10000>;
+			mediatek,topckgen = <&topckgen>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_AUDIO>;
+			interrupts = <GIC_SPI 822 IRQ_TYPE_LEVEL_HIGH 0>;
+			resets = <&watchdog 14>;
+			reset-names = "audiosys";
+			clocks = <&clk26m>,
+				<&apmixedsys CLK_APMIXED_APLL1>,
+				<&apmixedsys CLK_APMIXED_APLL2>,
+				<&topckgen CLK_TOP_APLL12_DIV0>,
+				<&topckgen CLK_TOP_APLL12_DIV1>,
+				<&topckgen CLK_TOP_APLL12_DIV2>,
+				<&topckgen CLK_TOP_APLL12_DIV3>,
+				<&topckgen CLK_TOP_APLL12_DIV9>,
+				<&topckgen CLK_TOP_A1SYS_HP>,
+				<&topckgen CLK_TOP_AUD_INTBUS>,
+				<&topckgen CLK_TOP_AUDIO_H>,
+				<&topckgen CLK_TOP_AUDIO_LOCAL_BUS>,
+				<&topckgen CLK_TOP_DPTX_MCK>,
+				<&topckgen CLK_TOP_I2SO1_MCK>,
+				<&topckgen CLK_TOP_I2SO2_MCK>,
+				<&topckgen CLK_TOP_I2SI1_MCK>,
+				<&topckgen CLK_TOP_I2SI2_MCK>,
+				<&infracfg_ao CLK_INFRA_AO_AUDIO_26M_B>,
+				<&scp_adsp CLK_SCP_ADSP_AUDIODSP>;
+			clock-names = "clk26m",
+				"apll1_ck",
+				"apll2_ck",
+				"apll12_div0",
+				"apll12_div1",
+				"apll12_div2",
+				"apll12_div3",
+				"apll12_div9",
+				"a1sys_hp_sel",
+				"aud_intbus_sel",
+				"audio_h_sel",
+				"audio_local_bus_sel",
+				"dptx_m_sel",
+				"i2so1_m_sel",
+				"i2so2_m_sel",
+				"i2si1_m_sel",
+				"i2si2_m_sel",
+				"infra_ao_audio_26m_b",
+				"scp_adsp_audiodsp";
+			status = "disabled";
+		};
+
 		uart0: serial@11001100 {
 			compatible = "mediatek,mt8195-uart",
 				     "mediatek,mt6577-uart";
@@ -560,8 +1058,10 @@
 			clocks = <&infracfg_ao CLK_INFRA_AO_SSUSB>,
 				 <&topckgen CLK_TOP_SSUSB_REF>,
 				 <&apmixedsys CLK_APMIXED_USB1PLL>,
+				 <&clk26m>,
 				 <&infracfg_ao CLK_INFRA_AO_SSUSB_XHCI>;
-			clock-names = "sys_ck", "ref_ck", "mcu_ck", "xhci_ck";
+			clock-names = "sys_ck", "ref_ck", "mcu_ck", "dma_ck",
+				      "xhci_ck";
 			mediatek,syscon-wakeup = <&pericfg 0x400 103>;
 			wakeup-source;
 			status = "disabled";
@@ -625,8 +1125,10 @@
 			clocks = <&pericfg_ao CLK_PERI_AO_SSUSB_1P_BUS>,
 				 <&topckgen CLK_TOP_SSUSB_P1_REF>,
 				 <&apmixedsys CLK_APMIXED_USB1PLL>,
+				 <&clk26m>,
 				 <&pericfg_ao CLK_PERI_AO_SSUSB_1P_XHCI>;
-			clock-names = "sys_ck", "ref_ck", "mcu_ck","xhci_ck";
+			clock-names = "sys_ck", "ref_ck", "mcu_ck", "dma_ck",
+				      "xhci_ck";
 			mediatek,syscon-wakeup = <&pericfg 0x400 104>;
 			wakeup-source;
 			status = "disabled";
@@ -646,8 +1148,11 @@
 						 <&topckgen CLK_TOP_UNIVPLL_D5_D4>;
 			clocks = <&pericfg_ao CLK_PERI_AO_SSUSB_2P_BUS>,
 				 <&topckgen CLK_TOP_SSUSB_P2_REF>,
+				 <&clk26m>,
+				 <&clk26m>,
 				 <&pericfg_ao CLK_PERI_AO_SSUSB_2P_XHCI>;
-			clock-names = "sys_ck", "ref_ck", "xhci_ck";
+			clock-names = "sys_ck", "ref_ck", "mcu_ck", "dma_ck",
+				      "xhci_ck";
 			mediatek,syscon-wakeup = <&pericfg 0x400 105>;
 			wakeup-source;
 			status = "disabled";
@@ -667,8 +1172,11 @@
 						 <&topckgen CLK_TOP_UNIVPLL_D5_D4>;
 			clocks = <&pericfg_ao CLK_PERI_AO_SSUSB_3P_BUS>,
 				 <&topckgen CLK_TOP_SSUSB_P3_REF>,
+				 <&clk26m>,
+				 <&clk26m>,
 				 <&pericfg_ao CLK_PERI_AO_SSUSB_3P_XHCI>;
-			clock-names = "sys_ck", "ref_ck", "xhci_ck";
+			clock-names = "sys_ck", "ref_ck", "mcu_ck", "dma_ck",
+				      "xhci_ck";
 			mediatek,syscon-wakeup = <&pericfg 0x400 106>;
 			wakeup-source;
 			status = "disabled";
@@ -828,7 +1336,7 @@
 			clock-names = "main", "dma";
 			#address-cells = <1>;
 			#size-cells = <0>;
-			status = "okay";
+			status = "disabled";
 		};
 
 		i2c1: i2c@11e01000 {
@@ -968,6 +1476,70 @@
 			#clock-cells = <1>;
 		};
 
+		vppsys0: clock-controller@14000000 {
+			compatible = "mediatek,mt8195-vppsys0";
+			reg = <0 0x14000000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		smi_sub_common_vpp0_vpp1_2x1: smi@14010000 {
+			compatible = "mediatek,mt8195-smi-sub-common";
+			reg = <0 0x14010000 0 0x1000>;
+			clocks = <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>,
+			       <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>,
+			       <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>;
+			clock-names = "apb", "smi", "gals0";
+			mediatek,smi = <&smi_common_vpp>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS0>;
+		};
+
+		smi_sub_common_vdec_vpp0_2x1: smi@14011000 {
+			compatible = "mediatek,mt8195-smi-sub-common";
+			reg = <0 0x14011000 0 0x1000>;
+			clocks = <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>,
+				 <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>,
+				 <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>;
+			clock-names = "apb", "smi", "gals0";
+			mediatek,smi = <&smi_common_vpp>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS0>;
+		};
+
+		smi_common_vpp: smi@14012000 {
+			compatible = "mediatek,mt8195-smi-common-vpp";
+			reg = <0 0x14012000 0 0x1000>;
+			clocks = <&vppsys0 CLK_VPP0_SMI_COMMON_LARB4>,
+			       <&vppsys0 CLK_VPP0_SMI_COMMON_LARB4>,
+			       <&vppsys0 CLK_VPP0_SMI_RSI>,
+			       <&vppsys0 CLK_VPP0_SMI_RSI>;
+			clock-names = "apb", "smi", "gals0", "gals1";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS0>;
+		};
+
+		larb4: larb@14013000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x14013000 0 0x1000>;
+			mediatek,larb-id = <4>;
+			mediatek,smi = <&smi_sub_common_vpp0_vpp1_2x1>;
+			clocks = <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>,
+			       <&vppsys0 CLK_VPP0_SMI_COMMON_LARB4>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS0>;
+		};
+
+		iommu_vpp: iommu@14018000 {
+			compatible = "mediatek,mt8195-iommu-vpp";
+			reg = <0 0x14018000 0 0x1000>;
+			mediatek,larbs = <&larb1 &larb3 &larb4 &larb6 &larb8
+					  &larb12 &larb14 &larb16 &larb18
+					  &larb20 &larb22 &larb23 &larb26
+					  &larb27>;
+			interrupts = <GIC_SPI 594 IRQ_TYPE_LEVEL_HIGH 0>;
+			clocks = <&vppsys0 CLK_VPP0_SMI_IOMMU>;
+			clock-names = "bclk";
+			#iommu-cells = <1>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS0>;
+		};
+
 		wpesys: clock-controller@14e00000 {
 			compatible = "mediatek,mt8195-wpesys";
 			reg = <0 0x14e00000 0 0x1000>;
@@ -986,18 +1558,116 @@
 			#clock-cells = <1>;
 		};
 
+		larb7: larb@14e04000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x14e04000 0 0x1000>;
+			mediatek,larb-id = <7>;
+			mediatek,smi = <&smi_common_vdo>;
+			clocks = <&wpesys CLK_WPE_SMI_LARB7>,
+				 <&wpesys CLK_WPE_SMI_LARB7>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_WPESYS>;
+		};
+
+		larb8: larb@14e05000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x14e05000 0 0x1000>;
+			mediatek,larb-id = <8>;
+			mediatek,smi = <&smi_common_vpp>;
+			clocks = <&wpesys CLK_WPE_SMI_LARB8>,
+			       <&wpesys CLK_WPE_SMI_LARB8>,
+			       <&vppsys0 CLK_VPP0_GALS_VPP1_WPE>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_WPESYS>;
+		};
+
+		vppsys1: clock-controller@14f00000 {
+			compatible = "mediatek,mt8195-vppsys1";
+			reg = <0 0x14f00000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		larb5: larb@14f02000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x14f02000 0 0x1000>;
+			mediatek,larb-id = <5>;
+			mediatek,smi = <&smi_common_vdo>;
+			clocks = <&vppsys1 CLK_VPP1_VPPSYS1_LARB>,
+			       <&vppsys1 CLK_VPP1_VPPSYS1_GALS>,
+			       <&vppsys0 CLK_VPP0_GALS_VPP1_LARB5>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS1>;
+		};
+
+		larb6: larb@14f03000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x14f03000 0 0x1000>;
+			mediatek,larb-id = <6>;
+			mediatek,smi = <&smi_sub_common_vpp0_vpp1_2x1>;
+			clocks = <&vppsys1 CLK_VPP1_VPPSYS1_LARB>,
+			       <&vppsys1 CLK_VPP1_VPPSYS1_GALS>,
+			       <&vppsys0 CLK_VPP0_GALS_VPP1_LARB6>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VPPSYS1>;
+		};
+
 		imgsys: clock-controller@15000000 {
 			compatible = "mediatek,mt8195-imgsys";
 			reg = <0 0x15000000 0 0x1000>;
 			#clock-cells = <1>;
 		};
 
+		larb9: larb@15001000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x15001000 0 0x1000>;
+			mediatek,larb-id = <9>;
+			mediatek,smi = <&smi_sub_common_img1_3x1>;
+			clocks = <&imgsys CLK_IMG_LARB9>,
+				 <&imgsys CLK_IMG_LARB9>,
+				 <&imgsys CLK_IMG_GALS>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_IMG>;
+		};
+
+		smi_sub_common_img0_3x1: smi@15002000 {
+			compatible = "mediatek,mt8195-smi-sub-common";
+			reg = <0 0x15002000 0 0x1000>;
+			clocks = <&imgsys CLK_IMG_IPE>,
+				 <&imgsys CLK_IMG_IPE>,
+				 <&vppsys0 CLK_VPP0_GALS_IMGSYS_CAMSYS>;
+			clock-names = "apb", "smi", "gals0";
+			mediatek,smi = <&smi_common_vpp>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_IMG>;
+		};
+
+		smi_sub_common_img1_3x1: smi@15003000 {
+			compatible = "mediatek,mt8195-smi-sub-common";
+			reg = <0 0x15003000 0 0x1000>;
+			clocks = <&imgsys CLK_IMG_LARB9>,
+				 <&imgsys CLK_IMG_LARB9>,
+				 <&imgsys CLK_IMG_GALS>;
+			clock-names = "apb", "smi", "gals0";
+			mediatek,smi = <&smi_common_vdo>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_IMG>;
+		};
+
 		imgsys1_dip_top: clock-controller@15110000 {
 			compatible = "mediatek,mt8195-imgsys1_dip_top";
 			reg = <0 0x15110000 0 0x1000>;
 			#clock-cells = <1>;
 		};
 
+		larb10: larb@15120000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x15120000 0 0x1000>;
+			mediatek,larb-id = <10>;
+			mediatek,smi = <&smi_sub_common_img1_3x1>;
+			clocks = <&imgsys CLK_IMG_DIP0>,
+			       <&imgsys1_dip_top CLK_IMG1_DIP_TOP_LARB10>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_DIP>;
+		};
+
 		imgsys1_dip_nr: clock-controller@15130000 {
 			compatible = "mediatek,mt8195-imgsys1_dip_nr";
 			reg = <0 0x15130000 0 0x1000>;
@@ -1010,18 +1680,129 @@
 			#clock-cells = <1>;
 		};
 
+		larb11: larb@15230000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x15230000 0 0x1000>;
+			mediatek,larb-id = <11>;
+			mediatek,smi = <&smi_sub_common_img1_3x1>;
+			clocks = <&imgsys CLK_IMG_WPE0>,
+			       <&imgsys1_wpe CLK_IMG1_WPE_LARB11>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_DIP>;
+		};
+
 		ipesys: clock-controller@15330000 {
 			compatible = "mediatek,mt8195-ipesys";
 			reg = <0 0x15330000 0 0x1000>;
 			#clock-cells = <1>;
 		};
 
+		larb12: larb@15340000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x15340000 0 0x1000>;
+			mediatek,larb-id = <12>;
+			mediatek,smi = <&smi_sub_common_img0_3x1>;
+			clocks = <&ipesys CLK_IPE_SMI_LARB12>,
+				 <&ipesys CLK_IPE_SMI_LARB12>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_IPE>;
+		};
+
 		camsys: clock-controller@16000000 {
 			compatible = "mediatek,mt8195-camsys";
 			reg = <0 0x16000000 0 0x1000>;
 			#clock-cells = <1>;
 		};
 
+		larb13: larb@16001000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x16001000 0 0x1000>;
+			mediatek,larb-id = <13>;
+			mediatek,smi = <&smi_sub_common_cam_4x1>;
+			clocks = <&camsys CLK_CAM_LARB13>,
+			       <&camsys CLK_CAM_LARB13>,
+			       <&camsys CLK_CAM_CAM2MM0_GALS>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM>;
+		};
+
+		larb14: larb@16002000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x16002000 0 0x1000>;
+			mediatek,larb-id = <14>;
+			mediatek,smi = <&smi_sub_common_cam_7x1>;
+			clocks = <&camsys CLK_CAM_LARB14>,
+				 <&camsys CLK_CAM_LARB14>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM>;
+		};
+
+		smi_sub_common_cam_4x1: smi@16004000 {
+			compatible = "mediatek,mt8195-smi-sub-common";
+			reg = <0 0x16004000 0 0x1000>;
+			clocks = <&camsys CLK_CAM_LARB13>,
+				 <&camsys CLK_CAM_LARB13>,
+				 <&camsys CLK_CAM_CAM2MM0_GALS>;
+			clock-names = "apb", "smi", "gals0";
+			mediatek,smi = <&smi_common_vdo>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM>;
+		};
+
+		smi_sub_common_cam_7x1: smi@16005000 {
+			compatible = "mediatek,mt8195-smi-sub-common";
+			reg = <0 0x16005000 0 0x1000>;
+			clocks = <&camsys CLK_CAM_LARB14>,
+				 <&camsys CLK_CAM_CAM2MM1_GALS>,
+				 <&vppsys0 CLK_VPP0_GALS_IMGSYS_CAMSYS>;
+			clock-names = "apb", "smi", "gals0";
+			mediatek,smi = <&smi_common_vpp>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM>;
+		};
+
+		larb16: larb@16012000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x16012000 0 0x1000>;
+			mediatek,larb-id = <16>;
+			mediatek,smi = <&smi_sub_common_cam_7x1>;
+			clocks = <&camsys_rawa CLK_CAM_RAWA_LARBX>,
+				 <&camsys_rawa CLK_CAM_RAWA_LARBX>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM_RAWA>;
+		};
+
+		larb17: larb@16013000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x16013000 0 0x1000>;
+			mediatek,larb-id = <17>;
+			mediatek,smi = <&smi_sub_common_cam_4x1>;
+			clocks = <&camsys_yuva CLK_CAM_YUVA_LARBX>,
+				 <&camsys_yuva CLK_CAM_YUVA_LARBX>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM_RAWA>;
+		};
+
+		larb27: larb@16014000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x16014000 0 0x1000>;
+			mediatek,larb-id = <27>;
+			mediatek,smi = <&smi_sub_common_cam_7x1>;
+			clocks = <&camsys_rawb CLK_CAM_RAWB_LARBX>,
+				 <&camsys_rawb CLK_CAM_RAWB_LARBX>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM_RAWB>;
+		};
+
+		larb28: larb@16015000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x16015000 0 0x1000>;
+			mediatek,larb-id = <28>;
+			mediatek,smi = <&smi_sub_common_cam_4x1>;
+			clocks = <&camsys_yuvb CLK_CAM_YUVB_LARBX>,
+				 <&camsys_yuvb CLK_CAM_YUVB_LARBX>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM_RAWB>;
+		};
+
 		camsys_rawa: clock-controller@1604f000 {
 			compatible = "mediatek,mt8195-camsys_rawa";
 			reg = <0 0x1604f000 0 0x1000>;
@@ -1052,24 +1833,103 @@
 			#clock-cells = <1>;
 		};
 
+		larb25: larb@16141000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x16141000 0 0x1000>;
+			mediatek,larb-id = <25>;
+			mediatek,smi = <&smi_sub_common_cam_4x1>;
+			clocks = <&camsys CLK_CAM_LARB13>,
+				 <&camsys_mraw CLK_CAM_MRAW_LARBX>,
+				 <&camsys CLK_CAM_CAM2MM0_GALS>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM_MRAW>;
+		};
+
+		larb26: larb@16142000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x16142000 0 0x1000>;
+			mediatek,larb-id = <26>;
+			mediatek,smi = <&smi_sub_common_cam_7x1>;
+			clocks = <&camsys_mraw CLK_CAM_MRAW_LARBX>,
+				 <&camsys_mraw CLK_CAM_MRAW_LARBX>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM_MRAW>;
+
+		};
+
 		ccusys: clock-controller@17200000 {
 			compatible = "mediatek,mt8195-ccusys";
 			reg = <0 0x17200000 0 0x1000>;
 			#clock-cells = <1>;
 		};
 
+		larb18: larb@17201000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x17201000 0 0x1000>;
+			mediatek,larb-id = <18>;
+			mediatek,smi = <&smi_sub_common_cam_7x1>;
+			clocks = <&ccusys CLK_CCU_LARB18>,
+				 <&ccusys CLK_CCU_LARB18>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_CAM>;
+		};
+
+		larb24: larb@1800d000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x1800d000 0 0x1000>;
+			mediatek,larb-id = <24>;
+			mediatek,smi = <&smi_common_vdo>;
+			clocks = <&vdecsys_soc CLK_VDEC_SOC_LARB1>,
+				 <&vdecsys_soc CLK_VDEC_SOC_LARB1>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDEC0>;
+		};
+
+		larb23: larb@1800e000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x1800e000 0 0x1000>;
+			mediatek,larb-id = <23>;
+			mediatek,smi = <&smi_sub_common_vdec_vpp0_2x1>;
+			clocks = <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>,
+				 <&vdecsys_soc CLK_VDEC_SOC_LARB1>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDEC0>;
+		};
+
 		vdecsys_soc: clock-controller@1800f000 {
 			compatible = "mediatek,mt8195-vdecsys_soc";
 			reg = <0 0x1800f000 0 0x1000>;
 			#clock-cells = <1>;
 		};
 
+		larb21: larb@1802e000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x1802e000 0 0x1000>;
+			mediatek,larb-id = <21>;
+			mediatek,smi = <&smi_common_vdo>;
+			clocks = <&vdecsys CLK_VDEC_LARB1>,
+				 <&vdecsys CLK_VDEC_LARB1>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDEC1>;
+		};
+
 		vdecsys: clock-controller@1802f000 {
 			compatible = "mediatek,mt8195-vdecsys";
 			reg = <0 0x1802f000 0 0x1000>;
 			#clock-cells = <1>;
 		};
 
+		larb22: larb@1803e000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x1803e000 0 0x1000>;
+			mediatek,larb-id = <22>;
+			mediatek,smi = <&smi_sub_common_vdec_vpp0_2x1>;
+			clocks = <&vppsys0 CLK_VPP0_GALS_VDEC_VDEC_CORE1>,
+				 <&vdecsys_core1 CLK_VDEC_CORE1_LARB1>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDEC2>;
+		};
+
 		vdecsys_core1: clock-controller@1803f000 {
 			compatible = "mediatek,mt8195-vdecsys_core1";
 			reg = <0 0x1803f000 0 0x1000>;
@@ -1088,10 +1948,212 @@
 			#clock-cells = <1>;
 		};
 
+		larb19: larb@1a010000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x1a010000 0 0x1000>;
+			mediatek,larb-id = <19>;
+			mediatek,smi = <&smi_common_vdo>;
+			clocks = <&vencsys CLK_VENC_VENC>,
+				 <&vencsys CLK_VENC_GALS>;
+			clock-names = "apb", "smi";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VENC>;
+		};
+
 		vencsys_core1: clock-controller@1b000000 {
 			compatible = "mediatek,mt8195-vencsys_core1";
 			reg = <0 0x1b000000 0 0x1000>;
 			#clock-cells = <1>;
 		};
+
+		vdosys0: syscon@1c01a000 {
+			compatible = "mediatek,mt8195-mmsys", "syscon";
+			reg = <0 0x1c01a000 0 0x1000>;
+			mboxes = <&gce0 0 CMDQ_THR_PRIO_4>;
+			#clock-cells = <1>;
+		};
+
+		larb20: larb@1b010000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x1b010000 0 0x1000>;
+			mediatek,larb-id = <20>;
+			mediatek,smi = <&smi_common_vpp>;
+			clocks = <&vencsys_core1 CLK_VENC_CORE1_LARB>,
+				 <&vencsys_core1 CLK_VENC_CORE1_GALS>,
+				 <&vppsys0 CLK_VPP0_GALS_VDO0_VDO1_VENCSYS_CORE1>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VENC_CORE1>;
+		};
+
+		ovl0: ovl@1c000000 {
+			compatible = "mediatek,mt8195-disp-ovl", "mediatek,mt8183-disp-ovl";
+			reg = <0 0x1c000000 0 0x1000>;
+			interrupts = <GIC_SPI 636 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+			clocks = <&vdosys0 CLK_VDO0_DISP_OVL0>;
+			iommus = <&iommu_vdo M4U_PORT_L0_DISP_OVL0_RDMA0>;
+			mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x0000 0x1000>;
+		};
+
+		rdma0: rdma@1c002000 {
+			compatible = "mediatek,mt8195-disp-rdma";
+			reg = <0 0x1c002000 0 0x1000>;
+			interrupts = <GIC_SPI 638 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+			clocks = <&vdosys0 CLK_VDO0_DISP_RDMA0>;
+			iommus = <&iommu_vdo M4U_PORT_L0_DISP_RDMA0>;
+			mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x2000 0x1000>;
+		};
+
+		color0: color@1c003000 {
+			compatible = "mediatek,mt8195-disp-color", "mediatek,mt8173-disp-color";
+			reg = <0 0x1c003000 0 0x1000>;
+			interrupts = <GIC_SPI 639 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+			clocks = <&vdosys0 CLK_VDO0_DISP_COLOR0>;
+			mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x3000 0x1000>;
+		};
+
+		ccorr0: ccorr@1c004000 {
+			compatible = "mediatek,mt8195-disp-ccorr", "mediatek,mt8192-disp-ccorr";
+			reg = <0 0x1c004000 0 0x1000>;
+			interrupts = <GIC_SPI 640 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+			clocks = <&vdosys0 CLK_VDO0_DISP_CCORR0>;
+			mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x4000 0x1000>;
+		};
+
+		aal0: aal@1c005000 {
+			compatible = "mediatek,mt8195-disp-aal", "mediatek,mt8183-disp-aal";
+			reg = <0 0x1c005000 0 0x1000>;
+			interrupts = <GIC_SPI 641 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+			clocks = <&vdosys0 CLK_VDO0_DISP_AAL0>;
+			mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x5000 0x1000>;
+		};
+
+		gamma0: gamma@1c006000 {
+			compatible = "mediatek,mt8195-disp-gamma", "mediatek,mt8183-disp-gamma";
+			reg = <0 0x1c006000 0 0x1000>;
+			interrupts = <GIC_SPI 642 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+			clocks = <&vdosys0 CLK_VDO0_DISP_GAMMA0>;
+			mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x6000 0x1000>;
+		};
+
+		dither0: dither@1c007000 {
+			compatible = "mediatek,mt8195-disp-dither", "mediatek,mt8183-disp-dither";
+			reg = <0 0x1c007000 0 0x1000>;
+			interrupts = <GIC_SPI 643 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+			clocks = <&vdosys0 CLK_VDO0_DISP_DITHER0>;
+			mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x7000 0x1000>;
+		};
+
+		dsc0: dsc@1c009000 {
+			compatible = "mediatek,mt8195-disp-dsc";
+			reg = <0 0x1c009000 0 0x1000>;
+			interrupts = <GIC_SPI 645 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+			clocks = <&vdosys0 CLK_VDO0_DSC_WRAP0>;
+			mediatek,gce-client-reg = <&gce0 SUBSYS_1c00XXXX 0x9000 0x1000>;
+		};
+
+		merge0: merge@1c014000 {
+			compatible = "mediatek,mt8195-disp-merge";
+			reg = <0 0x1c014000 0 0x1000>;
+			interrupts = <GIC_SPI 656 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+			clocks = <&vdosys0 CLK_VDO0_VPP_MERGE0>;
+			mediatek,gce-client-reg = <&gce0 SUBSYS_1c01XXXX 0x4000 0x1000>;
+		};
+
+		mutex: mutex@1c016000 {
+			compatible = "mediatek,mt8195-disp-mutex";
+			reg = <0 0x1c016000 0 0x1000>;
+			interrupts = <GIC_SPI 658 IRQ_TYPE_LEVEL_HIGH 0>;
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+			clocks = <&vdosys0 CLK_VDO0_DISP_MUTEX0>;
+			mediatek,gce-events = <CMDQ_EVENT_VDO0_DISP_STREAM_DONE_0>;
+		};
+
+		larb0: larb@1c018000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x1c018000 0 0x1000>;
+			mediatek,larb-id = <0>;
+			mediatek,smi = <&smi_common_vdo>;
+			clocks = <&vdosys0 CLK_VDO0_SMI_LARB>,
+				 <&vdosys0 CLK_VDO0_SMI_LARB>,
+				 <&vppsys0 CLK_VPP0_GALS_VDO0_LARB0>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+		};
+
+		larb1: larb@1c019000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x1c019000 0 0x1000>;
+			mediatek,larb-id = <1>;
+			mediatek,smi = <&smi_common_vpp>;
+			clocks = <&vdosys0 CLK_VDO0_SMI_LARB>,
+				 <&vppsys0 CLK_VPP0_GALS_VDO0_VDO1_VENCSYS_CORE1>,
+				 <&vppsys0 CLK_VPP0_GALS_VDO0_LARB1>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+		};
+
+		vdosys1: syscon@1c100000 {
+			compatible = "mediatek,mt8195-mmsys", "syscon";
+			reg = <0 0x1c100000 0 0x1000>;
+			#clock-cells = <1>;
+		};
+
+		smi_common_vdo: smi@1c01b000 {
+			compatible = "mediatek,mt8195-smi-common-vdo";
+			reg = <0 0x1c01b000 0 0x1000>;
+			clocks = <&vdosys0 CLK_VDO0_SMI_COMMON>,
+				 <&vdosys0 CLK_VDO0_SMI_EMI>,
+				 <&vdosys0 CLK_VDO0_SMI_RSI>,
+				 <&vdosys0 CLK_VDO0_SMI_GALS>;
+			clock-names = "apb", "smi", "gals0", "gals1";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+
+		};
+
+		iommu_vdo: iommu@1c01f000 {
+			compatible = "mediatek,mt8195-iommu-vdo";
+			reg = <0 0x1c01f000 0 0x1000>;
+			mediatek,larbs = <&larb0 &larb2 &larb5 &larb7 &larb9
+					  &larb10 &larb11 &larb13 &larb17
+					  &larb19 &larb21 &larb24 &larb25
+					  &larb28>;
+			interrupts = <GIC_SPI 669 IRQ_TYPE_LEVEL_HIGH 0>;
+			#iommu-cells = <1>;
+			clocks = <&vdosys0 CLK_VDO0_SMI_IOMMU>;
+			clock-names = "bclk";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS0>;
+		};
+
+		larb2: larb@1c102000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x1c102000 0 0x1000>;
+			mediatek,larb-id = <2>;
+			mediatek,smi = <&smi_common_vdo>;
+			clocks = <&vdosys1 CLK_VDO1_SMI_LARB2>,
+				 <&vdosys1 CLK_VDO1_SMI_LARB2>,
+				 <&vdosys1 CLK_VDO1_GALS>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS1>;
+		};
+
+		larb3: larb@1c103000 {
+			compatible = "mediatek,mt8195-smi-larb";
+			reg = <0 0x1c103000 0 0x1000>;
+			mediatek,larb-id = <3>;
+			mediatek,smi = <&smi_common_vpp>;
+			clocks = <&vdosys1 CLK_VDO1_SMI_LARB3>,
+				 <&vdosys1 CLK_VDO1_GALS>,
+				 <&vppsys0 CLK_VPP0_GALS_VDO0_VDO1_VENCSYS_CORE1>;
+			clock-names = "apb", "smi", "gals";
+			power-domains = <&spm MT8195_POWER_DOMAIN_VDOSYS1>;
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/nvidia/tegra186.dtsi b/arch/arm64/boot/dts/nvidia/tegra186.dtsi
index 59a10fb184f8..6602fe421ee8 100644
--- a/arch/arm64/boot/dts/nvidia/tegra186.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra186.dtsi
@@ -672,6 +672,10 @@
 		clock-names = "div-clk";
 		resets = <&bpmp TEGRA186_RESET_I2C1>;
 		reset-names = "i2c";
+		iommus = <&smmu TEGRA186_SID_GPCDMA_0>;
+		dma-coherent;
+		dmas = <&gpcdma 21>, <&gpcdma 21>;
+		dma-names = "rx", "tx";
 		status = "disabled";
 	};
 
@@ -685,6 +689,10 @@
 		clock-names = "div-clk";
 		resets = <&bpmp TEGRA186_RESET_I2C3>;
 		reset-names = "i2c";
+		iommus = <&smmu TEGRA186_SID_GPCDMA_0>;
+		dma-coherent;
+		dmas = <&gpcdma 23>, <&gpcdma 23>;
+		dma-names = "rx", "tx";
 		status = "disabled";
 	};
 
@@ -702,6 +710,10 @@
 		pinctrl-names = "default", "idle";
 		pinctrl-0 = <&state_dpaux1_i2c>;
 		pinctrl-1 = <&state_dpaux1_off>;
+		iommus = <&smmu TEGRA186_SID_GPCDMA_0>;
+		dma-coherent;
+		dmas = <&gpcdma 26>, <&gpcdma 26>;
+		dma-names = "rx", "tx";
 		status = "disabled";
 	};
 
@@ -733,6 +745,10 @@
 		pinctrl-names = "default", "idle";
 		pinctrl-0 = <&state_dpaux_i2c>;
 		pinctrl-1 = <&state_dpaux_off>;
+		iommus = <&smmu TEGRA186_SID_GPCDMA_0>;
+		dma-coherent;
+		dmas = <&gpcdma 30>, <&gpcdma 30>;
+		dma-names = "rx", "tx";
 		status = "disabled";
 	};
 
@@ -746,6 +762,10 @@
 		clock-names = "div-clk";
 		resets = <&bpmp TEGRA186_RESET_I2C7>;
 		reset-names = "i2c";
+		iommus = <&smmu TEGRA186_SID_GPCDMA_0>;
+		dma-coherent;
+		dmas = <&gpcdma 27>, <&gpcdma 27>;
+		dma-names = "rx", "tx";
 		status = "disabled";
 	};
 
@@ -759,6 +779,10 @@
 		clock-names = "div-clk";
 		resets = <&bpmp TEGRA186_RESET_I2C9>;
 		reset-names = "i2c";
+		iommus = <&smmu TEGRA186_SID_GPCDMA_0>;
+		dma-coherent;
+		dmas = <&gpcdma 31>, <&gpcdma 31>;
+		dma-names = "rx", "tx";
 		status = "disabled";
 	};
 
@@ -1176,6 +1200,10 @@
 		clock-names = "div-clk";
 		resets = <&bpmp TEGRA186_RESET_I2C2>;
 		reset-names = "i2c";
+		iommus = <&smmu TEGRA186_SID_GPCDMA_0>;
+		dma-coherent;
+		dmas = <&gpcdma 22>, <&gpcdma 22>;
+		dma-names = "rx", "tx";
 		status = "disabled";
 	};
 
@@ -1189,6 +1217,10 @@
 		clock-names = "div-clk";
 		resets = <&bpmp TEGRA186_RESET_I2C8>;
 		reset-names = "i2c";
+		iommus = <&smmu TEGRA186_SID_GPCDMA_0>;
+		dma-coherent;
+		dmas = <&gpcdma 0>, <&gpcdma 0>;
+		dma-names = "rx", "tx";
 		status = "disabled";
 	};
 
@@ -1485,15 +1517,14 @@
 		iommus = <&smmu TEGRA186_SID_HOST1X>;
 
 		/* Context isolation domains */
-		iommu-map = <
-			0 &smmu TEGRA186_SID_HOST1X_CTX0 1
-			1 &smmu TEGRA186_SID_HOST1X_CTX1 1
-			2 &smmu TEGRA186_SID_HOST1X_CTX2 1
-			3 &smmu TEGRA186_SID_HOST1X_CTX3 1
-			4 &smmu TEGRA186_SID_HOST1X_CTX4 1
-			5 &smmu TEGRA186_SID_HOST1X_CTX5 1
-			6 &smmu TEGRA186_SID_HOST1X_CTX6 1
-			7 &smmu TEGRA186_SID_HOST1X_CTX7 1>;
+		iommu-map = <0 &smmu TEGRA186_SID_HOST1X_CTX0 1>,
+			    <1 &smmu TEGRA186_SID_HOST1X_CTX1 1>,
+			    <2 &smmu TEGRA186_SID_HOST1X_CTX2 1>,
+			    <3 &smmu TEGRA186_SID_HOST1X_CTX3 1>,
+			    <4 &smmu TEGRA186_SID_HOST1X_CTX4 1>,
+			    <5 &smmu TEGRA186_SID_HOST1X_CTX5 1>,
+			    <6 &smmu TEGRA186_SID_HOST1X_CTX6 1>,
+			    <7 &smmu TEGRA186_SID_HOST1X_CTX7 1>;
 
 		dpaux1: dpaux@15040000 {
 			compatible = "nvidia,tegra186-dpaux";
diff --git a/arch/arm64/boot/dts/nvidia/tegra194.dtsi b/arch/arm64/boot/dts/nvidia/tegra194.dtsi
index d0ed55e5c860..41f3a7e188d0 100644
--- a/arch/arm64/boot/dts/nvidia/tegra194.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra194.dtsi
@@ -805,6 +805,10 @@
 			clock-names = "div-clk";
 			resets = <&bpmp TEGRA194_RESET_I2C1>;
 			reset-names = "i2c";
+			iommus = <&smmu TEGRA194_SID_GPCDMA_0>;
+			dma-coherent;
+			dmas = <&gpcdma 21>, <&gpcdma 21>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
@@ -830,6 +834,10 @@
 			clock-names = "div-clk";
 			resets = <&bpmp TEGRA194_RESET_I2C3>;
 			reset-names = "i2c";
+			iommus = <&smmu TEGRA194_SID_GPCDMA_0>;
+			dma-coherent;
+			dmas = <&gpcdma 23>, <&gpcdma 23>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
@@ -847,6 +855,10 @@
 			pinctrl-0 = <&state_dpaux1_i2c>;
 			pinctrl-1 = <&state_dpaux1_off>;
 			pinctrl-names = "default", "idle";
+			iommus = <&smmu TEGRA194_SID_GPCDMA_0>;
+			dma-coherent;
+			dmas = <&gpcdma 26>, <&gpcdma 26>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
@@ -864,6 +876,10 @@
 			pinctrl-0 = <&state_dpaux0_i2c>;
 			pinctrl-1 = <&state_dpaux0_off>;
 			pinctrl-names = "default", "idle";
+			iommus = <&smmu TEGRA194_SID_GPCDMA_0>;
+			dma-coherent;
+			dmas = <&gpcdma 30>, <&gpcdma 30>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
@@ -881,6 +897,10 @@
 			pinctrl-0 = <&state_dpaux2_i2c>;
 			pinctrl-1 = <&state_dpaux2_off>;
 			pinctrl-names = "default", "idle";
+			iommus = <&smmu TEGRA194_SID_GPCDMA_0>;
+			dma-coherent;
+			dmas = <&gpcdma 27>, <&gpcdma 27>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
@@ -898,6 +918,10 @@
 			pinctrl-0 = <&state_dpaux3_i2c>;
 			pinctrl-1 = <&state_dpaux3_off>;
 			pinctrl-names = "default", "idle";
+			iommus = <&smmu TEGRA194_SID_GPCDMA_0>;
+			dma-coherent;
+			dmas = <&gpcdma 31>, <&gpcdma 31>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
@@ -1565,6 +1589,10 @@
 			clock-names = "div-clk";
 			resets = <&bpmp TEGRA194_RESET_I2C2>;
 			reset-names = "i2c";
+			iommus = <&smmu TEGRA194_SID_GPCDMA_0>;
+			dma-coherent;
+			dmas = <&gpcdma 22>, <&gpcdma 22>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
@@ -1578,6 +1606,10 @@
 			clock-names = "div-clk";
 			resets = <&bpmp TEGRA194_RESET_I2C8>;
 			reset-names = "i2c";
+			iommus = <&smmu TEGRA194_SID_GPCDMA_0>;
+			dma-coherent;
+			dmas = <&gpcdma 0>, <&gpcdma 0>;
+			dma-names = "rx", "tx";
 			status = "disabled";
 		};
 
@@ -1869,15 +1901,14 @@
 			iommus = <&smmu TEGRA194_SID_HOST1X>;
 
 			/* Context isolation domains */
-			iommu-map = <
-				0 &smmu TEGRA194_SID_HOST1X_CTX0 1
-				1 &smmu TEGRA194_SID_HOST1X_CTX1 1
-				2 &smmu TEGRA194_SID_HOST1X_CTX2 1
-				3 &smmu TEGRA194_SID_HOST1X_CTX3 1
-				4 &smmu TEGRA194_SID_HOST1X_CTX4 1
-				5 &smmu TEGRA194_SID_HOST1X_CTX5 1
-				6 &smmu TEGRA194_SID_HOST1X_CTX6 1
-				7 &smmu TEGRA194_SID_HOST1X_CTX7 1>;
+			iommu-map = <0 &smmu TEGRA194_SID_HOST1X_CTX0 1>,
+				    <1 &smmu TEGRA194_SID_HOST1X_CTX1 1>,
+				    <2 &smmu TEGRA194_SID_HOST1X_CTX2 1>,
+				    <3 &smmu TEGRA194_SID_HOST1X_CTX3 1>,
+				    <4 &smmu TEGRA194_SID_HOST1X_CTX4 1>,
+				    <5 &smmu TEGRA194_SID_HOST1X_CTX5 1>,
+				    <6 &smmu TEGRA194_SID_HOST1X_CTX6 1>,
+				    <7 &smmu TEGRA194_SID_HOST1X_CTX7 1>;
 
 			nvdec@15140000 {
 				compatible = "nvidia,tegra194-nvdec";
diff --git a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts
index 5f3a1c56b2eb..7c569695b705 100644
--- a/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra210-smaug.dts
@@ -17,6 +17,7 @@
 
 	aliases {
 		serial0 = &uarta;
+		serial3 = &uartd;
 	};
 
 	chosen {
@@ -1309,6 +1310,22 @@
 		status = "okay";
 	};
 
+	uartd: serial@70006300 {
+		compatible = "nvidia,tegra30-hsuart";
+		status = "okay";
+
+		bluetooth {
+			compatible = "brcm,bcm43540-bt";
+			max-speed = <4000000>;
+			brcm,bt-pcm-int-params = [01 02 00 01 01];
+			device-wakeup-gpios = <&gpio TEGRA_GPIO(H, 3) GPIO_ACTIVE_HIGH>;
+			shutdown-gpios = <&gpio TEGRA_GPIO(H, 4) GPIO_ACTIVE_HIGH>;
+			interrupt-parent = <&gpio>;
+			interrupts = <TEGRA_GPIO(H, 5) IRQ_TYPE_LEVEL_LOW>;
+			interrupt-names = "host-wakeup";
+		};
+	};
+
 	i2c@7000c400 {
 		status = "okay";
 		clock-frequency = <1000000>;
@@ -1692,6 +1709,25 @@
 		};
 	};
 
+	mmc@700b0200 {
+		power-gpios = <&gpio TEGRA_GPIO(H, 1) GPIO_ACTIVE_HIGH>;
+		bus-width = <4>;
+		non-removable;
+		vqmmc-supply = <&pp1800>;
+		vmmc-supply = <&pp3300>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		status = "okay";
+
+		wifi@1 {
+			compatible = "brcm,bcm4354-fmac", "brcm,bcm4329-fmac";
+			reg = <1>;
+			interrupt-parent = <&gpio>;
+			interrupts = <TEGRA_GPIO(H, 2) IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "host-wake";
+		};
+	};
+
 	mmc@700b0600 {
 		bus-width = <8>;
 		non-removable;
diff --git a/arch/arm64/boot/dts/nvidia/tegra234-p3701-0000.dtsi b/arch/arm64/boot/dts/nvidia/tegra234-p3701-0000.dtsi
index 798de9226ba5..9e4d72cfa69f 100644
--- a/arch/arm64/boot/dts/nvidia/tegra234-p3701-0000.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra234-p3701-0000.dtsi
@@ -6,6 +6,42 @@
 	model = "NVIDIA Jetson AGX Orin";
 	compatible = "nvidia,p3701-0000", "nvidia,tegra234";
 
+	vdd_1v8_ls: regulator-vdd-1v8-ls {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_1V8_LS";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+	};
+
+	vdd_1v8_ao: regulator-vdd-1v8-ao {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_1V8_AO";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+	};
+
+	vdd_3v3_pcie: regulator-vdd-3v3-pcie {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_3V3_PCIE";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio TEGRA234_MAIN_GPIO(Z, 2) GPIO_ACTIVE_HIGH>;
+		regulator-boot-on;
+		enable-active-high;
+	};
+
+	vdd_12v_pcie: regulator-vdd-12v-pcie {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_12V_PCIE";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		gpio = <&gpio TEGRA234_MAIN_GPIO(A, 1) GPIO_ACTIVE_LOW>;
+		regulator-boot-on;
+		enable-active-low;
+	};
+
 	bus@0 {
 		spi@3270000 {
 			status = "okay";
diff --git a/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701-0000.dts b/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701-0000.dts
index 02a10bb38562..57ab75328814 100644
--- a/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701-0000.dts
+++ b/arch/arm64/boot/dts/nvidia/tegra234-p3737-0000+p3701-0000.dts
@@ -2009,6 +2009,7 @@
 
 		hda@3510000 {
 			nvidia,model = "NVIDIA Jetson AGX Orin HDA";
+			status = "okay";
 		};
 	};
 
@@ -2017,6 +2018,27 @@
 		stdout-path = "serial0:115200n8";
 	};
 
+	bus@0 {
+		ethernet@6800000 {
+			status = "okay";
+
+			phy-handle = <&mgbe0_phy>;
+			phy-mode = "usxgmii";
+
+			mdio {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				mgbe0_phy: phy@0 {
+					compatible = "ethernet-phy-ieee802.3-c45";
+					reg = <0x0>;
+
+					#phy-cells = <0>;
+				};
+			};
+		};
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 		status = "okay";
@@ -2111,4 +2133,55 @@
 
 		label = "NVIDIA Jetson AGX Orin APE";
 	};
+
+	pcie@14100000 {
+		status = "okay";
+
+		vddio-pex-ctl-supply = <&vdd_1v8_ao>;
+
+		phys = <&p2u_hsio_3>;
+		phy-names = "p2u-0";
+	};
+
+	pcie@14160000 {
+		status = "okay";
+
+		vddio-pex-ctl-supply = <&vdd_1v8_ao>;
+
+		phys = <&p2u_hsio_4>, <&p2u_hsio_5>, <&p2u_hsio_6>,
+		       <&p2u_hsio_7>;
+		phy-names = "p2u-0", "p2u-1", "p2u-2", "p2u-3";
+	};
+
+	pcie@141a0000 {
+		status = "okay";
+
+		vddio-pex-ctl-supply = <&vdd_1v8_ls>;
+		vpcie3v3-supply = <&vdd_3v3_pcie>;
+		vpcie12v-supply = <&vdd_12v_pcie>;
+
+		phys = <&p2u_nvhs_0>, <&p2u_nvhs_1>, <&p2u_nvhs_2>,
+		       <&p2u_nvhs_3>, <&p2u_nvhs_4>, <&p2u_nvhs_5>,
+		       <&p2u_nvhs_6>, <&p2u_nvhs_7>;
+		phy-names = "p2u-0", "p2u-1", "p2u-2", "p2u-3", "p2u-4",
+			    "p2u-5", "p2u-6", "p2u-7";
+	};
+
+	pcie-ep@141a0000 {
+		status = "disabled";
+
+		vddio-pex-ctl-supply = <&vdd_1v8_ls>;
+
+		reset-gpios = <&gpio TEGRA234_MAIN_GPIO(AF, 1) GPIO_ACTIVE_LOW>;
+
+		nvidia,refclk-select-gpios = <&gpio_aon
+					      TEGRA234_AON_GPIO(AA, 4)
+					      GPIO_ACTIVE_HIGH>;
+
+		phys = <&p2u_nvhs_0>, <&p2u_nvhs_1>, <&p2u_nvhs_2>,
+		       <&p2u_nvhs_3>, <&p2u_nvhs_4>, <&p2u_nvhs_5>,
+		       <&p2u_nvhs_6>, <&p2u_nvhs_7>;
+		phy-names = "p2u-0", "p2u-1", "p2u-2", "p2u-3", "p2u-4",
+			    "p2u-5", "p2u-6", "p2u-7";
+	};
 };
diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi
index 81a0f599685f..0170bfa8a467 100644
--- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi
@@ -23,7 +23,6 @@
 
 		gpcdma: dma-controller@2600000 {
 			compatible = "nvidia,tegra234-gpcdma",
-				     "nvidia,tegra194-gpcdma",
 				     "nvidia,tegra186-gpcdma";
 			reg = <0x2600000 0x210000>;
 			resets = <&bpmp TEGRA234_RESET_GPCDMA>;
@@ -570,6 +569,24 @@
 			interconnect-names = "dma-mem";
 			iommus = <&smmu_niso1 TEGRA234_SID_HOST1X>;
 
+			/* Context isolation domains */
+			iommu-map = <0 &smmu_niso0 TEGRA234_SID_HOST1X_CTX0 1>,
+				    <1 &smmu_niso0 TEGRA234_SID_HOST1X_CTX1 1>,
+				    <2 &smmu_niso0 TEGRA234_SID_HOST1X_CTX2 1>,
+				    <3 &smmu_niso0 TEGRA234_SID_HOST1X_CTX3 1>,
+				    <4 &smmu_niso0 TEGRA234_SID_HOST1X_CTX4 1>,
+				    <5 &smmu_niso0 TEGRA234_SID_HOST1X_CTX5 1>,
+				    <6 &smmu_niso0 TEGRA234_SID_HOST1X_CTX6 1>,
+				    <7 &smmu_niso0 TEGRA234_SID_HOST1X_CTX7 1>,
+				    <8 &smmu_niso1 TEGRA234_SID_HOST1X_CTX0 1>,
+				    <9 &smmu_niso1 TEGRA234_SID_HOST1X_CTX1 1>,
+				    <10 &smmu_niso1 TEGRA234_SID_HOST1X_CTX2 1>,
+				    <11 &smmu_niso1 TEGRA234_SID_HOST1X_CTX3 1>,
+				    <12 &smmu_niso1 TEGRA234_SID_HOST1X_CTX4 1>,
+				    <13 &smmu_niso1 TEGRA234_SID_HOST1X_CTX5 1>,
+				    <14 &smmu_niso1 TEGRA234_SID_HOST1X_CTX6 1>,
+				    <15 &smmu_niso1 TEGRA234_SID_HOST1X_CTX7 1>;
+
 			vic@15340000 {
 				compatible = "nvidia,tegra234-vic";
 				reg = <0x15340000 0x00040000>;
@@ -737,6 +754,10 @@
 			clock-names = "div-clk", "parent";
 			resets = <&bpmp TEGRA234_RESET_I2C1>;
 			reset-names = "i2c";
+			iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>;
+			dma-coherent;
+			dmas = <&gpcdma 21>, <&gpcdma 21>;
+			dma-names = "rx", "tx";
 		};
 
 		cam_i2c: i2c@3180000 {
@@ -752,6 +773,10 @@
 			clock-names = "div-clk", "parent";
 			resets = <&bpmp TEGRA234_RESET_I2C3>;
 			reset-names = "i2c";
+			iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>;
+			dma-coherent;
+			dmas = <&gpcdma 23>, <&gpcdma 23>;
+			dma-names = "rx", "tx";
 		};
 
 		dp_aux_ch1_i2c: i2c@3190000 {
@@ -767,6 +792,10 @@
 			clock-names = "div-clk", "parent";
 			resets = <&bpmp TEGRA234_RESET_I2C4>;
 			reset-names = "i2c";
+			iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>;
+			dma-coherent;
+			dmas = <&gpcdma 26>, <&gpcdma 26>;
+			dma-names = "rx", "tx";
 		};
 
 		dp_aux_ch0_i2c: i2c@31b0000 {
@@ -782,6 +811,10 @@
 			clock-names = "div-clk", "parent";
 			resets = <&bpmp TEGRA234_RESET_I2C6>;
 			reset-names = "i2c";
+			iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>;
+			dma-coherent;
+			dmas = <&gpcdma 30>, <&gpcdma 30>;
+			dma-names = "rx", "tx";
 		};
 
 		dp_aux_ch2_i2c: i2c@31c0000 {
@@ -797,6 +830,10 @@
 			clock-names = "div-clk", "parent";
 			resets = <&bpmp TEGRA234_RESET_I2C7>;
 			reset-names = "i2c";
+			iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>;
+			dma-coherent;
+			dmas = <&gpcdma 27>, <&gpcdma 27>;
+			dma-names = "rx", "tx";
 		};
 
 		dp_aux_ch3_i2c: i2c@31e0000 {
@@ -812,6 +849,10 @@
 			clock-names = "div-clk", "parent";
 			resets = <&bpmp TEGRA234_RESET_I2C9>;
 			reset-names = "i2c";
+			iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>;
+			dma-coherent;
+			dmas = <&gpcdma 31>, <&gpcdma 31>;
+			dma-names = "rx", "tx";
 		};
 
 		spi@3270000 {
@@ -897,6 +938,7 @@
 			interconnects = <&mc TEGRA234_MEMORY_CLIENT_HDAR &emc>,
 					<&mc TEGRA234_MEMORY_CLIENT_HDAW &emc>;
 			interconnect-names = "dma-mem", "write";
+			iommus = <&smmu_niso0 TEGRA234_SID_HDA>;
 			status = "disabled";
 		};
 
@@ -925,6 +967,142 @@
 			#mbox-cells = <2>;
 		};
 
+		ethernet@6800000 {
+			compatible = "nvidia,tegra234-mgbe";
+			reg = <0x06800000 0x10000>,
+			      <0x06810000 0x10000>,
+			      <0x068a0000 0x10000>;
+			reg-names = "hypervisor", "mac", "xpcs";
+			interrupts = <GIC_SPI 384 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "common";
+			clocks = <&bpmp TEGRA234_CLK_MGBE0_APP>,
+				 <&bpmp TEGRA234_CLK_MGBE0_MAC>,
+				 <&bpmp TEGRA234_CLK_MGBE0_MAC_DIVIDER>,
+				 <&bpmp TEGRA234_CLK_MGBE0_PTP_REF>,
+				 <&bpmp TEGRA234_CLK_MGBE0_RX_INPUT_M>,
+				 <&bpmp TEGRA234_CLK_MGBE0_RX_INPUT>,
+				 <&bpmp TEGRA234_CLK_MGBE0_TX>,
+				 <&bpmp TEGRA234_CLK_MGBE0_EEE_PCS>,
+				 <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_INPUT>,
+				 <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_M>,
+				 <&bpmp TEGRA234_CLK_MGBE0_RX_PCS>,
+				 <&bpmp TEGRA234_CLK_MGBE0_TX_PCS>;
+			clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m",
+				      "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m",
+				      "rx-pcs", "tx-pcs";
+			resets = <&bpmp TEGRA234_RESET_MGBE0_MAC>,
+				 <&bpmp TEGRA234_RESET_MGBE0_PCS>;
+			reset-names = "mac", "pcs";
+			interconnects = <&mc TEGRA234_MEMORY_CLIENT_MGBEARD &emc>,
+					<&mc TEGRA234_MEMORY_CLIENT_MGBEAWR &emc>;
+			interconnect-names = "dma-mem", "write";
+			iommus = <&smmu_niso0 TEGRA234_SID_MGBE>;
+			power-domains = <&bpmp TEGRA234_POWER_DOMAIN_MGBEA>;
+			status = "disabled";
+		};
+
+		ethernet@6900000 {
+			compatible = "nvidia,tegra234-mgbe";
+			reg = <0x06900000 0x10000>,
+			      <0x06910000 0x10000>,
+			      <0x069a0000 0x10000>;
+			reg-names = "hypervisor", "mac", "xpcs";
+			interrupts = <GIC_SPI 392 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "common";
+			clocks = <&bpmp TEGRA234_CLK_MGBE1_APP>,
+				 <&bpmp TEGRA234_CLK_MGBE1_MAC>,
+				 <&bpmp TEGRA234_CLK_MGBE1_MAC_DIVIDER>,
+				 <&bpmp TEGRA234_CLK_MGBE1_PTP_REF>,
+				 <&bpmp TEGRA234_CLK_MGBE1_RX_INPUT_M>,
+				 <&bpmp TEGRA234_CLK_MGBE1_RX_INPUT>,
+				 <&bpmp TEGRA234_CLK_MGBE1_TX>,
+				 <&bpmp TEGRA234_CLK_MGBE1_EEE_PCS>,
+				 <&bpmp TEGRA234_CLK_MGBE1_RX_PCS_INPUT>,
+				 <&bpmp TEGRA234_CLK_MGBE1_RX_PCS_M>,
+				 <&bpmp TEGRA234_CLK_MGBE1_RX_PCS>,
+				 <&bpmp TEGRA234_CLK_MGBE1_TX_PCS>;
+			clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m",
+				      "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m",
+				      "rx-pcs", "tx-pcs";
+			resets = <&bpmp TEGRA234_RESET_MGBE1_MAC>,
+				 <&bpmp TEGRA234_RESET_MGBE1_PCS>;
+			reset-names = "mac", "pcs";
+			interconnects = <&mc TEGRA234_MEMORY_CLIENT_MGBEBRD &emc>,
+					<&mc TEGRA234_MEMORY_CLIENT_MGBEBWR &emc>;
+			interconnect-names = "dma-mem", "write";
+			iommus = <&smmu_niso0 TEGRA234_SID_MGBE_VF1>;
+			power-domains = <&bpmp TEGRA234_POWER_DOMAIN_MGBEB>;
+			status = "disabled";
+		};
+
+		ethernet@6a00000 {
+			compatible = "nvidia,tegra234-mgbe";
+			reg = <0x06a00000 0x10000>,
+			      <0x06a10000 0x10000>,
+			      <0x06aa0000 0x10000>;
+			reg-names = "hypervisor", "mac", "xpcs";
+			interrupts = <GIC_SPI 400 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "common";
+			clocks = <&bpmp TEGRA234_CLK_MGBE2_APP>,
+				 <&bpmp TEGRA234_CLK_MGBE2_MAC>,
+				 <&bpmp TEGRA234_CLK_MGBE2_MAC_DIVIDER>,
+				 <&bpmp TEGRA234_CLK_MGBE2_PTP_REF>,
+				 <&bpmp TEGRA234_CLK_MGBE2_RX_INPUT_M>,
+				 <&bpmp TEGRA234_CLK_MGBE2_RX_INPUT>,
+				 <&bpmp TEGRA234_CLK_MGBE2_TX>,
+				 <&bpmp TEGRA234_CLK_MGBE2_EEE_PCS>,
+				 <&bpmp TEGRA234_CLK_MGBE2_RX_PCS_INPUT>,
+				 <&bpmp TEGRA234_CLK_MGBE2_RX_PCS_M>,
+				 <&bpmp TEGRA234_CLK_MGBE2_RX_PCS>,
+				 <&bpmp TEGRA234_CLK_MGBE2_TX_PCS>;
+			clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m",
+				      "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m",
+				      "rx-pcs", "tx-pcs";
+			resets = <&bpmp TEGRA234_RESET_MGBE2_MAC>,
+				 <&bpmp TEGRA234_RESET_MGBE2_PCS>;
+			reset-names = "mac", "pcs";
+			interconnects = <&mc TEGRA234_MEMORY_CLIENT_MGBECRD &emc>,
+					<&mc TEGRA234_MEMORY_CLIENT_MGBECWR &emc>;
+			interconnect-names = "dma-mem", "write";
+			iommus = <&smmu_niso0 TEGRA234_SID_MGBE_VF2>;
+			power-domains = <&bpmp TEGRA234_POWER_DOMAIN_MGBEC>;
+			status = "disabled";
+		};
+
+		ethernet@6b00000 {
+			compatible = "nvidia,tegra234-mgbe";
+			reg = <0x06b00000 0x10000>,
+			      <0x06b10000 0x10000>,
+			      <0x06ba0000 0x10000>;
+			reg-names = "hypervisor", "mac", "xpcs";
+			interrupts = <GIC_SPI 408 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "common";
+			clocks = <&bpmp TEGRA234_CLK_MGBE3_APP>,
+				 <&bpmp TEGRA234_CLK_MGBE3_MAC>,
+				 <&bpmp TEGRA234_CLK_MGBE3_MAC_DIVIDER>,
+				 <&bpmp TEGRA234_CLK_MGBE3_PTP_REF>,
+				 <&bpmp TEGRA234_CLK_MGBE3_RX_INPUT_M>,
+				 <&bpmp TEGRA234_CLK_MGBE3_RX_INPUT>,
+				 <&bpmp TEGRA234_CLK_MGBE3_TX>,
+				 <&bpmp TEGRA234_CLK_MGBE3_EEE_PCS>,
+				 <&bpmp TEGRA234_CLK_MGBE3_RX_PCS_INPUT>,
+				 <&bpmp TEGRA234_CLK_MGBE3_RX_PCS_M>,
+				 <&bpmp TEGRA234_CLK_MGBE3_RX_PCS>,
+				 <&bpmp TEGRA234_CLK_MGBE3_TX_PCS>;
+			clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m",
+				      "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m",
+				      "rx-pcs", "tx-pcs";
+			resets = <&bpmp TEGRA234_RESET_MGBE3_MAC>,
+				 <&bpmp TEGRA234_RESET_MGBE3_PCS>;
+			reset-names = "mac", "pcs";
+			interconnects = <&mc TEGRA234_MEMORY_CLIENT_MGBEDRD &emc>,
+					<&mc TEGRA234_MEMORY_CLIENT_MGBEDWR &emc>;
+			interconnect-names = "dma-mem", "write";
+			iommus = <&smmu_niso0 TEGRA234_SID_MGBE_VF3>;
+			power-domains = <&bpmp TEGRA234_POWER_DOMAIN_MGBED>;
+			status = "disabled";
+		};
+
 		smmu_niso1: iommu@8000000 {
 			compatible = "nvidia,tegra234-smmu", "nvidia,smmu-500";
 			reg = <0x8000000 0x1000000>,
@@ -1081,6 +1259,198 @@
 			status = "okay";
 		};
 
+		p2u_hsio_0: phy@3e00000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03e00000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_hsio_1: phy@3e10000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03e10000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_hsio_2: phy@3e20000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03e20000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_hsio_3: phy@3e30000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03e30000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_hsio_4: phy@3e40000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03e40000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_hsio_5: phy@3e50000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03e50000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_hsio_6: phy@3e60000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03e60000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_hsio_7: phy@3e70000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03e70000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_nvhs_0: phy@3e90000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03e90000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_nvhs_1: phy@3ea0000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03ea0000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_nvhs_2: phy@3eb0000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03eb0000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_nvhs_3: phy@3ec0000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03ec0000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_nvhs_4: phy@3ed0000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03ed0000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_nvhs_5: phy@3ee0000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03ee0000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_nvhs_6: phy@3ef0000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03ef0000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_nvhs_7: phy@3f00000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03f00000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_gbe_0: phy@3f20000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03f20000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_gbe_1: phy@3f30000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03f30000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_gbe_2: phy@3f40000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03f40000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_gbe_3: phy@3f50000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03f50000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_gbe_4: phy@3f60000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03f60000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_gbe_5: phy@3f70000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03f70000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_gbe_6: phy@3f80000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03f80000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
+		p2u_gbe_7: phy@3f90000 {
+			compatible = "nvidia,tegra234-p2u";
+			reg = <0x03f90000 0x10000>;
+			reg-names = "ctl";
+
+			#phy-cells = <0>;
+		};
+
 		hsp_aon: hsp@c150000 {
 			compatible = "nvidia,tegra234-hsp", "nvidia,tegra194-hsp";
 			reg = <0x0c150000 0x90000>;
@@ -1109,6 +1479,10 @@
 			assigned-clock-parents = <&bpmp TEGRA234_CLK_PLLP_OUT0>;
 			resets = <&bpmp TEGRA234_RESET_I2C2>;
 			reset-names = "i2c";
+			iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>;
+			dma-coherent;
+			dmas = <&gpcdma 22>, <&gpcdma 22>;
+			dma-names = "rx", "tx";
 		};
 
 		gen8_i2c: i2c@c250000 {
@@ -1125,6 +1499,10 @@
 			assigned-clock-parents = <&bpmp TEGRA234_CLK_PLLP_OUT0>;
 			resets = <&bpmp TEGRA234_RESET_I2C8>;
 			reset-names = "i2c";
+			iommus = <&smmu_niso0 TEGRA234_SID_GPCDMA>;
+			dma-coherent;
+			dmas = <&gpcdma 0>, <&gpcdma 0>;
+			dma-names = "rx", "tx";
 		};
 
 		rtc@c2a0000 {
@@ -1495,6 +1873,741 @@
 		status = "okay";
 	};
 
+	pcie@140a0000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4CA>;
+		reg = <0x00 0x140a0000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x2a000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x2a040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x2a080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <4>;
+		num-viewport = <8>;
+		linux,pci-domain = <8>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX2_C8_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX2_CORE_8_APB>,
+			 <&bpmp TEGRA234_RESET_PEX2_CORE_8>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 8>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x32 0x40000000 0x32 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */
+			 <0x02000000 0x0  0x40000000 0x35 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x2a100000 0x00 0x2a100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE8AR &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE8AW &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE8 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie@140c0000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4CB>;
+		reg = <0x00 0x140c0000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x2c000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x2c040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x2c080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <4>;
+		num-viewport = <8>;
+		linux,pci-domain = <9>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX2_C9_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX2_CORE_9_APB>,
+			 <&bpmp TEGRA234_RESET_PEX2_CORE_9>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 359 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 9>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x35 0x40000000 0x35 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */
+			 <0x02000000 0x0  0x40000000 0x38 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x2c100000 0x00 0x2c100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE9AR &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE9AW &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE9 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie@140e0000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4CC>;
+		reg = <0x00 0x140e0000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x2e000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x2e040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x2e080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <4>;
+		num-viewport = <8>;
+		linux,pci-domain = <10>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX2_C10_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX2_CORE_10_APB>,
+			 <&bpmp TEGRA234_RESET_PEX2_CORE_10>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 360 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 361 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 360 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 10>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x38 0x40000000 0x38 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */
+			 <0x02000000 0x0  0x40000000 0x3b 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x2e100000 0x00 0x2e100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE10AR &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE10AW &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE10 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie@14100000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX1A>;
+		reg = <0x00 0x14100000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x30000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x30040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x30080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <1>;
+		num-viewport = <8>;
+		linux,pci-domain = <1>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX0_C1_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX0_CORE_1_APB>,
+			 <&bpmp TEGRA234_RESET_PEX0_CORE_1>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 1>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x20 0x80000000 0x20 0x80000000 0x0 0x28000000>, /* prefetchable memory (640 MB) */
+			 <0x02000000 0x0  0x40000000 0x20 0xa8000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x30100000 0x00 0x30100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE1R &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE1W &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE1 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie@14120000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX1A>;
+		reg = <0x00 0x14120000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x32000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x32040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x32080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <1>;
+		num-viewport = <8>;
+		linux,pci-domain = <2>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX0_C2_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX0_CORE_2_APB>,
+			 <&bpmp TEGRA234_RESET_PEX0_CORE_2>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 48 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 47 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 2>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x20 0xc0000000 0x20 0xc0000000 0x0 0x28000000>, /* prefetchable memory (640 MB) */
+			 <0x02000000 0x0  0x40000000 0x20 0xe8000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x32100000 0x00 0x32100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE2AR &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE2AW &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE2 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie@14140000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX1A>;
+		reg = <0x00 0x14140000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x34000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x34040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x34080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <1>;
+		num-viewport = <8>;
+		linux,pci-domain = <3>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX0_C3_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX0_CORE_3_APB>,
+			 <&bpmp TEGRA234_RESET_PEX0_CORE_3>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 3>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x21 0x00000000 0x21 0x00000000 0x0 0x28000000>, /* prefetchable memory (640 MB) */
+			 <0x02000000 0x0  0x40000000 0x21 0xe8000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x34100000 0x00 0x34100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE3R &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE3W &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE3 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie@14160000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4BB>;
+		reg = <0x00 0x14160000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x36000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x36040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x36080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <4>;
+		num-viewport = <8>;
+		linux,pci-domain = <4>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX0_C4_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX0_CORE_4_APB>,
+			 <&bpmp TEGRA234_RESET_PEX0_CORE_4>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 51 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 4>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x21 0x40000000 0x21 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */
+			 <0x02000000 0x0  0x40000000 0x24 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x36100000 0x00 0x36100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE4R &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE4W &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE4 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie@14180000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4BA>;
+		reg = <0x00 0x14180000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x38000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x38040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x38080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <4>;
+		num-viewport = <8>;
+		linux,pci-domain = <0>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX0_C0_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX0_CORE_0_APB>,
+			 <&bpmp TEGRA234_RESET_PEX0_CORE_0>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 0>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x24 0x40000000 0x24 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */
+			 <0x02000000 0x0  0x40000000 0x27 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x38100000 0x00 0x38100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE0R &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE0W &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE0 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie@141a0000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX8A>;
+		reg = <0x00 0x141a0000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x3a000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x3a040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x3a080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <8>;
+		num-viewport = <8>;
+		linux,pci-domain = <5>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX1_C5_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX1_CORE_5_APB>,
+			 <&bpmp TEGRA234_RESET_PEX1_CORE_5>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 5>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x27 0x40000000 0x27 0x40000000 0x3 0xe8000000>, /* prefetchable memory (16000 MB) */
+			 <0x02000000 0x0  0x40000000 0x2b 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x3a100000 0x00 0x3a100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE5R &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE5W &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE5 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie@141c0000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4A>;
+		reg = <0x00 0x141c0000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x3c000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x3c040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x3c080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <4>;
+		num-viewport = <8>;
+		linux,pci-domain = <6>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX1_C6_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX1_CORE_6_APB>,
+			 <&bpmp TEGRA234_RESET_PEX1_CORE_6>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 352 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 352 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 6>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x2b 0x40000000 0x2b 0x40000000 0x2 0xe8000000>, /* prefetchable memory (11904 MB) */
+			 <0x02000000 0x0  0x40000000 0x2e 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x3c100000 0x00 0x3c100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE6AR &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE6AW &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE6 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie@141e0000 {
+		compatible = "nvidia,tegra234-pcie";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX8B>;
+		reg = <0x00 0x141e0000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x3e000000 0x0 0x00040000>, /* configuration space (256K) */
+		      <0x00 0x3e040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x3e080000 0x0 0x00040000>; /* DBI reg space (256K)       */
+		reg-names = "appl", "config", "atu_dma", "dbi";
+
+		#address-cells = <3>;
+		#size-cells = <2>;
+		device_type = "pci";
+		num-lanes = <8>;
+		num-viewport = <8>;
+		linux,pci-domain = <7>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX2_C7_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX2_CORE_7_APB>,
+			 <&bpmp TEGRA234_RESET_PEX2_CORE_7>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>, /* controller interrupt */
+			     <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>; /* MSI interrupt */
+		interrupt-names = "intr", "msi";
+
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 0>;
+		interrupt-map = <0 0 0 0 &gic GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+
+		nvidia,bpmp = <&bpmp 7>;
+
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		bus-range = <0x0 0xff>;
+
+		ranges = <0x43000000 0x2e 0x40000000 0x2e 0x40000000 0x3 0xe8000000>, /* prefetchable memory (16000 MB) */
+			 <0x02000000 0x0  0x40000000 0x32 0x28000000 0x0 0x08000000>, /* non-prefetchable memory (128 MB) */
+			 <0x01000000 0x0  0x3e100000 0x00 0x3e100000 0x0 0x00100000>; /* downstream I/O (1 MB) */
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE7AR &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE7AW &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE7 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie-ep@141a0000 {
+		compatible = "nvidia,tegra234-pcie-ep";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX8A>;
+		reg = <0x00 0x141a0000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x3a040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x3a080000 0x0 0x00040000>, /* DBI reg space (256K)       */
+		      <0x27 0x40000000 0x4 0x00000000>; /* Address Space (16G)        */
+		reg-names = "appl", "atu_dma", "dbi", "addr_space";
+
+		num-lanes = <8>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX1_C5_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX1_CORE_5_APB>,
+			 <&bpmp TEGRA234_RESET_PEX1_CORE_5>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>;	/* controller interrupt */
+		interrupt-names = "intr";
+
+		nvidia,bpmp = <&bpmp 5>;
+
+		nvidia,enable-ext-refclk;
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE5R &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE5W &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE5 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie-ep@141c0000{
+		compatible = "nvidia,tegra234-pcie-ep";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4A>;
+		reg = <0x00 0x141c0000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x3c040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x3c080000 0x0 0x00040000>, /* DBI space (256K)           */
+		      <0x2b 0x40000000 0x3 0x00000000>; /* Address Space (12G)        */
+		reg-names = "appl", "atu_dma", "dbi", "addr_space";
+
+		num-lanes = <4>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX1_C6_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX1_CORE_6_APB>,
+			 <&bpmp TEGRA234_RESET_PEX1_CORE_6>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 352 IRQ_TYPE_LEVEL_HIGH>;	/* controller interrupt */
+		interrupt-names = "intr";
+
+		nvidia,bpmp = <&bpmp 6>;
+
+		nvidia,enable-ext-refclk;
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE6AR &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE6AW &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso0 TEGRA234_SID_PCIE6 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie-ep@141e0000{
+		compatible = "nvidia,tegra234-pcie-ep";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX8B>;
+		reg = <0x00 0x141e0000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x3e040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x3e080000 0x0 0x00040000>, /* DBI space (256K)           */
+		      <0x2e 0x40000000 0x4 0x00000000>; /* Address Space (16G)        */
+		reg-names = "appl", "atu_dma", "dbi", "addr_space";
+
+		num-lanes = <8>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX2_C7_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX2_CORE_7_APB>,
+			 <&bpmp TEGRA234_RESET_PEX2_CORE_7>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;	/* controller interrupt */
+		interrupt-names = "intr";
+
+		nvidia,bpmp = <&bpmp 7>;
+
+		nvidia,enable-ext-refclk;
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE7AR &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE7AW &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE7 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
+	pcie-ep@140e0000{
+		compatible = "nvidia,tegra234-pcie-ep";
+		power-domains = <&bpmp TEGRA234_POWER_DOMAIN_PCIEX4CC>;
+		reg = <0x00 0x140e0000 0x0 0x00020000>, /* appl registers (128K)      */
+		      <0x00 0x2e040000 0x0 0x00040000>, /* iATU_DMA reg space (256K)  */
+		      <0x00 0x2e080000 0x0 0x00040000>, /* DBI space (256K)           */
+		      <0x38 0x40000000 0x3 0x00000000>; /* Address Space (12G)        */
+		reg-names = "appl", "atu_dma", "dbi", "addr_space";
+
+		num-lanes = <4>;
+
+		clocks = <&bpmp TEGRA234_CLK_PEX2_C10_CORE>;
+		clock-names = "core";
+
+		resets = <&bpmp TEGRA234_RESET_PEX2_CORE_10_APB>,
+			 <&bpmp TEGRA234_RESET_PEX2_CORE_10>;
+		reset-names = "apb", "core";
+
+		interrupts = <GIC_SPI 360 IRQ_TYPE_LEVEL_HIGH>;	/* controller interrupt */
+		interrupt-names = "intr";
+
+		nvidia,bpmp = <&bpmp 10>;
+
+		nvidia,enable-ext-refclk;
+		nvidia,aspm-cmrt-us = <60>;
+		nvidia,aspm-pwr-on-t-us = <20>;
+		nvidia,aspm-l0s-entrance-latency-us = <3>;
+
+		interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE10AR &emc>,
+				<&mc TEGRA234_MEMORY_CLIENT_PCIE10AW &emc>;
+		interconnect-names = "dma-mem", "write";
+		iommu-map = <0x0 &smmu_niso1 TEGRA234_SID_PCIE10 0x1000>;
+		iommu-map-mask = <0x0>;
+		dma-coherent;
+
+		status = "disabled";
+	};
+
 	sram@40000000 {
 		compatible = "nvidia,tegra234-sysram", "mmio-sram";
 		reg = <0x0 0x40000000 0x0 0x80000>;
diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile
index 1d86a33de528..d7669a7cee9f 100644
--- a/arch/arm64/boot/dts/qcom/Makefile
+++ b/arch/arm64/boot/dts/qcom/Makefile
@@ -15,6 +15,9 @@ dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-longcheer-l8910.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-mtp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-samsung-a3u-eur.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-samsung-a5u-eur.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-samsung-e5.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-samsung-e7.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-samsung-grandmax.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-samsung-j5.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-samsung-serranove.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8916-wingtech-wt88047.dtb
@@ -101,8 +104,11 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sc7180-trogdor-wormdingler-rev1-boe-rt5682s.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7180-trogdor-r1.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7180-trogdor-r1-lte.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7280-herobrine-crd.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sc7280-herobrine-evoker-r0.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7280-herobrine-herobrine-r1.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7280-herobrine-villager-r0.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sc7280-herobrine-villager-r1.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sc7280-herobrine-villager-r1-lte.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7280-idp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7280-idp2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sc7280-crd-r3.dtb
@@ -152,3 +158,4 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sm8350-sony-xperia-sagami-pdx214.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8350-sony-xperia-sagami-pdx215.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8450-hdk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8450-qrd.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sm8450-sony-xperia-nagara-pdx223.dtb
diff --git a/arch/arm64/boot/dts/qcom/apq8096-db820c.dts b/arch/arm64/boot/dts/qcom/apq8096-db820c.dts
index c1cb1ba5173c..5cdc7ac1a9c0 100644
--- a/arch/arm64/boot/dts/qcom/apq8096-db820c.dts
+++ b/arch/arm64/boot/dts/qcom/apq8096-db820c.dts
@@ -14,6 +14,7 @@
 #include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
 #include <dt-bindings/sound/qcom,q6afe.h>
 #include <dt-bindings/sound/qcom,q6asm.h>
+#include <dt-bindings/sound/qcom,wcd9335.h>
 
 /*
  * GPIO name legend: proper name = the GPIO line is used as GPIO
@@ -502,20 +503,20 @@
 
 &pcie0 {
 	status = "okay";
-	perst-gpio = <&tlmm 35 GPIO_ACTIVE_LOW>;
+	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
 	vddpe-3v3-supply = <&wlan_en>;
 	vdda-supply = <&vreg_l28a_0p925>;
 };
 
 &pcie1 {
 	status = "okay";
-	perst-gpio = <&tlmm 130 GPIO_ACTIVE_LOW>;
+	perst-gpios = <&tlmm 130 GPIO_ACTIVE_LOW>;
 	vdda-supply = <&vreg_l28a_0p925>;
 };
 
 &pcie2 {
 	status = "okay";
-	perst-gpio = <&tlmm 114 GPIO_ACTIVE_LOW>;
+	perst-gpios = <&tlmm 114 GPIO_ACTIVE_LOW>;
 	vdda-supply = <&vreg_l28a_0p925>;
 };
 
@@ -1064,7 +1065,7 @@
 	};
 
 		codec {
-			sound-dai = <&wcd9335 6>;
+			sound-dai = <&wcd9335 AIF4_PB>;
 		};
 	};
 
@@ -1079,7 +1080,7 @@
 		};
 
 		codec {
-			sound-dai = <&wcd9335 1>;
+			sound-dai = <&wcd9335 AIF1_CAP>;
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts b/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts
index 567b33106556..92f264891d84 100644
--- a/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts
+++ b/arch/arm64/boot/dts/qcom/apq8096-ifc6640.dts
@@ -368,7 +368,7 @@
 
 	bus-width = <4>;
 
-	cd-gpios = <&tlmm 38 0x1>;
+	cd-gpios = <&tlmm 38 GPIO_ACTIVE_LOW>;
 
 	vmmc-supply = <&vreg_l21a_2p95>;
 	vqmmc-supply = <&vreg_l13a_2p95>;
diff --git a/arch/arm64/boot/dts/qcom/ipq6018.dtsi b/arch/arm64/boot/dts/qcom/ipq6018.dtsi
index aaad7d9059f6..a7c7ca980a71 100644
--- a/arch/arm64/boot/dts/qcom/ipq6018.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq6018.dtsi
@@ -129,12 +129,6 @@
 		};
 	};
 
-	tcsr_mutex: hwlock {
-		compatible = "qcom,tcsr-mutex";
-		syscon = <&tcsr_mutex_regs 0 0x80>;
-		#hwlock-cells = <1>;
-	};
-
 	pmuv8: pmu {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_PPI 7 (GIC_CPU_MASK_SIMPLE(4) |
@@ -252,13 +246,14 @@
 			#reset-cells = <1>;
 		};
 
-		tcsr_mutex_regs: syscon@1905000 {
-			compatible = "syscon";
-			reg = <0x0 0x01905000 0x0 0x8000>;
+		tcsr_mutex: hwlock@1905000 {
+			compatible = "qcom,ipq6018-tcsr-mutex", "qcom,tcsr-mutex";
+			reg = <0x0 0x01905000 0x0 0x1000>;
+			#hwlock-cells = <1>;
 		};
 
 		tcsr: syscon@1937000 {
-			compatible = "syscon";
+			compatible = "qcom,tcsr-ipq6018", "syscon";
 			reg = <0x0 0x01937000 0x0 0x21000>;
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts b/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts
index 81dc3a0bcd7d..7143c936de61 100644
--- a/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts
+++ b/arch/arm64/boot/dts/qcom/ipq8074-hk01.dts
@@ -51,12 +51,12 @@
 
 &pcie0 {
 	status = "okay";
-	perst-gpio = <&tlmm 61 0x1>;
+	perst-gpios = <&tlmm 61 0x1>;
 };
 
 &pcie1 {
 	status = "okay";
-	perst-gpio = <&tlmm 58 0x1>;
+	perst-gpios = <&tlmm 58 0x1>;
 };
 
 &pcie_phy0 {
diff --git a/arch/arm64/boot/dts/qcom/ipq8074-hk10.dtsi b/arch/arm64/boot/dts/qcom/ipq8074-hk10.dtsi
index 40415d988e4a..db4b87944cdf 100644
--- a/arch/arm64/boot/dts/qcom/ipq8074-hk10.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq8074-hk10.dtsi
@@ -39,12 +39,12 @@
 
 &pcie0 {
 	status = "ok";
-	perst-gpio = <&tlmm 58 0x1>;
+	perst-gpios = <&tlmm 58 0x1>;
 };
 
 &pcie1 {
 	status = "ok";
-	perst-gpio = <&tlmm 61 0x1>;
+	perst-gpios = <&tlmm 61 0x1>;
 };
 
 &pcie_phy0 {
diff --git a/arch/arm64/boot/dts/qcom/ipq8074.dtsi b/arch/arm64/boot/dts/qcom/ipq8074.dtsi
index d53675fc1595..a47acf9bdf24 100644
--- a/arch/arm64/boot/dts/qcom/ipq8074.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq8074.dtsi
@@ -199,7 +199,7 @@
 
 		pcie_qmp0: phy@86000 {
 			compatible = "qcom,ipq8074-qmp-pcie-phy";
-			reg = <0x00086000 0x1000>;
+			reg = <0x00086000 0x1c4>;
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges;
@@ -227,7 +227,7 @@
 
 		pcie_qmp1: phy@8e000 {
 			compatible = "qcom,ipq8074-qmp-pcie-phy";
-			reg = <0x0008e000 0x1000>;
+			reg = <0x0008e000 0x1c4>;
 			#address-cells = <1>;
 			#size-cells = <1>;
 			ranges;
@@ -383,7 +383,7 @@
 		sdhc_1: mmc@7824900 {
 			compatible = "qcom,sdhci-msm-v4";
 			reg = <0x7824900 0x500>, <0x7824000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 
 			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi b/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi
index 10f6509a8709..3255bd3fcb55 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-a2015-common.dtsi
@@ -61,9 +61,9 @@
 		};
 	};
 
-	reg_vdd_tsp: regulator-vdd-tsp {
+	reg_vdd_tsp_a: regulator-vdd-tsp-a {
 		compatible = "regulator-fixed";
-		regulator-name = "vdd_tsp";
+		regulator-name = "vdd_tsp_a";
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
 
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts
index bc198a2eea25..6db5f78ca286 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-a3u-eur.dts
@@ -73,8 +73,8 @@
 		touchscreen-size-x = <540>;
 		touchscreen-size-y = <960>;
 
-		vdd-supply = <&reg_vdd_tsp>;
-		vddo-supply = <&pm8916_l6>;
+		vcca-supply = <&reg_vdd_tsp_a>;
+		vdd-supply = <&pm8916_l6>;
 
 		pinctrl-names = "default";
 		pinctrl-0 = <&ts_int_default>;
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts
index 7f2ab1891d91..5fb8ecd0c9ca 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-a5u-eur.dts
@@ -42,7 +42,7 @@
 		touchscreen-size-x = <720>;
 		touchscreen-size-y = <1280>;
 
-		avdd-supply = <&reg_vdd_tsp>;
+		avdd-supply = <&reg_vdd_tsp_a>;
 		vdd-supply = <&pm8916_l6>;
 
 		pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-e2015-common.dtsi b/arch/arm64/boot/dts/qcom/msm8916-samsung-e2015-common.dtsi
new file mode 100644
index 000000000000..542010fdfb8a
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-e2015-common.dtsi
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "msm8916-samsung-a2015-common.dtsi"
+
+/ {
+	haptic {
+		compatible = "regulator-haptic";
+		haptic-supply = <&reg_motor_vdd>;
+		min-microvolt = <3300000>;
+		max-microvolt = <3300000>;
+	};
+
+	i2c-muic {
+		/* SM5504 MUIC instead of SM5502 */
+		/delete-node/ extcon@25;
+
+		muic: extcon@14 {
+			compatible = "siliconmitus,sm5504-muic";
+			reg = <0x14>;
+
+			interrupt-parent = <&msmgpio>;
+			interrupts = <12 IRQ_TYPE_EDGE_FALLING>;
+
+			pinctrl-names = "default";
+			pinctrl-0 = <&muic_int_default>;
+		};
+	};
+
+	reg_motor_vdd: regulator-motor-vdd {
+		compatible = "regulator-fixed";
+		regulator-name = "motor_vdd";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&msmgpio 76 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&motor_en_default>;
+	};
+
+	reg_touch_key: regulator-touch-key {
+		compatible = "regulator-fixed";
+		regulator-name = "touch_key";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+
+		gpio = <&msmgpio 97 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&tkey_en_default>;
+	};
+};
+
+&blsp_i2c2 {
+	/* lis2hh12 accelerometer instead of BMC150 */
+	status = "disabled";
+
+	/delete-node/ accelerometer@10;
+	/delete-node/ magnetometer@12;
+};
+
+&touchkey {
+	vcc-supply = <&reg_touch_key>;
+	vdd-supply = <&reg_touch_key>;
+};
+
+&msmgpio {
+	motor_en_default: motor-en-default {
+		pins = "gpio76";
+		function = "gpio";
+
+		drive-strength = <2>;
+		bias-disable;
+	};
+
+	tkey_en_default: tkey-en-default {
+		pins = "gpio97";
+		function = "gpio";
+
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-e5.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-e5.dts
new file mode 100644
index 000000000000..777eb934eb4b
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-e5.dts
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/dts-v1/;
+
+#include "msm8916-samsung-e2015-common.dtsi"
+
+/*
+ * NOTE: The original firmware from Samsung can only boot ARM32 kernels on some
+ * variants.
+ * Unfortunately, the firmware is signed and cannot be replaced easily.
+ * There seems to be no way to boot ARM64 kernels on 32-bit devices at the
+ * moment, even though the hardware would support it.
+ *
+ * However, it is possible to use this device tree by compiling an ARM32 kernel
+ * instead. For clarity and build testing this device tree is maintained next
+ * to the other MSM8916 device trees. However, it is actually used through
+ * arch/arm/boot/dts/qcom-msm8916-samsung-e5.dts
+ */
+
+/ {
+	model = "Samsung Galaxy E5";
+	compatible = "samsung,e5", "qcom,msm8916";
+	chassis-type = "handset";
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-e7.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-e7.dts
new file mode 100644
index 000000000000..b412b61ca258
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-e7.dts
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/dts-v1/;
+
+#include "msm8916-samsung-e2015-common.dtsi"
+
+/*
+ * NOTE: The original firmware from Samsung can only boot ARM32 kernels on some
+ * variants.
+ * Unfortunately, the firmware is signed and cannot be replaced easily.
+ * There seems to be no way to boot ARM64 kernels on 32-bit devices at the
+ * moment, even though the hardware would support it.
+ *
+ * However, it is possible to use this device tree by compiling an ARM32 kernel
+ * instead. For clarity and build testing this device tree is maintained next
+ * to the other MSM8916 device trees. However, it is actually used through
+ * arch/arm/boot/dts/qcom-msm8916-samsung-e7.dts
+ */
+
+/ {
+	model = "Samsung Galaxy E7";
+	compatible = "samsung,e7", "qcom,msm8916";
+	chassis-type = "handset";
+};
+
+&pm8916_l17 {
+	regulator-min-microvolt = <3000000>;
+	regulator-max-microvolt = <3000000>;
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-grandmax.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-grandmax.dts
new file mode 100644
index 000000000000..bc7134698978
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-grandmax.dts
@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/dts-v1/;
+
+#include "msm8916-samsung-e2015-common.dtsi"
+#include <dt-bindings/leds/common.h>
+
+/*
+ * NOTE: The original firmware from Samsung can only boot ARM32 kernels on some
+ * variants.
+ * Unfortunately, the firmware is signed and cannot be replaced easily.
+ * There seems to be no way to boot ARM64 kernels on 32-bit devices at the
+ * moment, even though the hardware would support it.
+ *
+ * However, it is possible to use this device tree by compiling an ARM32 kernel
+ * instead. For clarity and build testing this device tree is maintained next
+ * to the other MSM8916 device trees. However, it is actually used through
+ * arch/arm/boot/dts/qcom-msm8916-samsung-grandmax.dts
+ */
+
+/ {
+	model = "Samsung Galaxy Grand Max";
+	compatible = "samsung,grandmax", "qcom,msm8916";
+	chassis-type = "handset";
+
+	/delete-node/ gpio-hall-sensor;
+	/delete-node/ i2c-nfc;
+	/delete-node/ i2c-tkey;
+
+	gpio-leds {
+		compatible = "gpio-leds";
+		keyled {
+			gpios = <&msmgpio 60 GPIO_ACTIVE_HIGH>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&gpio_leds_default>;
+		};
+	};
+};
+
+&reg_motor_vdd {
+	gpio = <&msmgpio 72 GPIO_ACTIVE_HIGH>;
+};
+
+&reg_touch_key {
+	status = "disabled";
+};
+
+&msmgpio {
+	gpio_leds_default: gpio-led-default {
+		pins = "gpio60";
+		function = "gpio";
+
+		drive-strength = <2>;
+		bias-disable;
+	};
+};
+
+&motor_en_default {
+	pins = "gpio72";
+};
diff --git a/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts b/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts
index 439e89cf7878..bbd6bb3f4fd7 100644
--- a/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts
+++ b/arch/arm64/boot/dts/qcom/msm8916-samsung-serranove.dts
@@ -210,6 +210,15 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&imu_irq_default>;
 	};
+
+	magnetometer@2e {
+		compatible = "yamaha,yas537";
+		reg = <0x2e>;
+
+		mount-matrix =  "0",  "1",  "0",
+				"1",  "0",  "0",
+				"0",  "0", "-1";
+	};
 };
 
 &blsp_i2c4 {
diff --git a/arch/arm64/boot/dts/qcom/msm8916.dtsi b/arch/arm64/boot/dts/qcom/msm8916.dtsi
index 48bc2e09128d..a831064700ee 100644
--- a/arch/arm64/boot/dts/qcom/msm8916.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8916.dtsi
@@ -936,6 +936,20 @@
 			#reset-cells = <1>;
 			#power-domain-cells = <1>;
 			reg = <0x01800000 0x80000>;
+			clocks = <&xo_board>,
+				 <&sleep_clk>,
+				 <&dsi_phy0 1>,
+				 <&dsi_phy0 0>,
+				 <0>,
+				 <0>,
+				 <0>;
+			clock-names = "xo",
+				      "sleep_clk",
+				      "dsi0pll",
+				      "dsi0pllbyte",
+				      "ext_mclk",
+				      "ext_pri_i2s",
+				      "ext_sec_i2s";
 		};
 
 		tcsr_mutex: hwlock@1905000 {
@@ -1469,7 +1483,7 @@
 		sdhc_1: mmc@7824000 {
 			compatible = "qcom,msm8916-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0x07824900 0x11c>, <0x07824000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 
 			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
@@ -1487,7 +1501,7 @@
 		sdhc_2: mmc@7864000 {
 			compatible = "qcom,msm8916-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0x07864900 0x11c>, <0x07864000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 
 			interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/boot/dts/qcom/msm8953.dtsi b/arch/arm64/boot/dts/qcom/msm8953.dtsi
index 8416a45ca4fd..6b992a6d56c1 100644
--- a/arch/arm64/boot/dts/qcom/msm8953.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8953.dtsi
@@ -722,7 +722,7 @@
 		};
 
 		tcsr_phy_clk_scheme_sel: syscon@193f044 {
-			compatible = "syscon";
+			compatible = "qcom,tcsr-msm8953", "syscon";
 			reg = <0x193f044 0x4>;
 		};
 
@@ -799,7 +799,7 @@
 			compatible = "qcom,msm8953-sdhci", "qcom,sdhci-msm-v4";
 
 			reg = <0x7824900 0x500>, <0x7824000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 
 			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
@@ -859,7 +859,7 @@
 			compatible = "qcom,msm8953-sdhci", "qcom,sdhci-msm-v4";
 
 			reg = <0x7864900 0x500>, <0x7864000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 
 			interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/boot/dts/qcom/msm8992-xiaomi-libra.dts b/arch/arm64/boot/dts/qcom/msm8992-xiaomi-libra.dts
index cbe11c060df9..c4e87d0aec42 100644
--- a/arch/arm64/boot/dts/qcom/msm8992-xiaomi-libra.dts
+++ b/arch/arm64/boot/dts/qcom/msm8992-xiaomi-libra.dts
@@ -17,7 +17,7 @@
 	chassis-type = "handset";
 
 	/* required for bootloader to select correct board */
-	qcom,msm-id = <251 0 252 0>;
+	qcom,msm-id = <251 0>, <252 0>;
 	qcom,pmic-id = <65545 65546 0 0>;
 	qcom,board-id = <12 0>;
 
diff --git a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi
index 61ec905025b0..f9d8bd09e074 100644
--- a/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8994-msft-lumia-octagon.dtsi
@@ -499,7 +499,7 @@
 		linux,code = <KEY_POWER>;
 	};
 
-	volwnkey {
+	resin {
 		compatible = "qcom,pm8941-resin";
 		interrupts = <0 8 1 IRQ_TYPE_EDGE_BOTH>;
 		debounce = <15625>;
diff --git a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi
index f430d797196f..ff60b7004d26 100644
--- a/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8994-sony-xperia-kitakami.dtsi
@@ -471,7 +471,7 @@
 &sdhc2 {
 	status = "okay";
 
-	cd-gpios = <&tlmm 100 0>;
+	cd-gpios = <&tlmm 100 GPIO_ACTIVE_HIGH>;
 	vmmc-supply = <&pm8994_l21>;
 	vqmmc-supply = <&pm8994_l13>;
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8994.dtsi b/arch/arm64/boot/dts/qcom/msm8994.dtsi
index 8bc6c070e306..ded5b7ceeaf9 100644
--- a/arch/arm64/boot/dts/qcom/msm8994.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8994.dtsi
@@ -6,6 +6,7 @@
 #include <dt-bindings/clock/qcom,gcc-msm8994.h>
 #include <dt-bindings/clock/qcom,mmcc-msm8994.h>
 #include <dt-bindings/clock/qcom,rpmcc.h>
+#include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 
 / {
@@ -164,12 +165,6 @@
 		reg = <0 0x80000000 0 0>;
 	};
 
-	tcsr_mutex: hwlock {
-		compatible = "qcom,tcsr-mutex";
-		syscon = <&tcsr_mutex_regs 0 0x80>;
-		#hwlock-cells = <1>;
-	};
-
 	pmu {
 		compatible = "arm,cortex-a53-pmu";
 		interrupts = <GIC_PPI 7 (GIC_CPU_MASK_SIMPLE(4)| IRQ_TYPE_LEVEL_HIGH)>;
@@ -464,7 +459,7 @@
 		sdhc1: mmc@f9824900 {
 			compatible = "qcom,msm8994-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf9824900 0x1a0>, <0xf9824000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 
 			interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
@@ -487,7 +482,7 @@
 		sdhc2: mmc@f98a4900 {
 			compatible = "qcom,msm8994-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0xf98a4900 0x11c>, <0xf98a4000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 
 			interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
 				<GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
@@ -502,7 +497,7 @@
 			pinctrl-0 = <&sdc2_clk_on &sdc2_cmd_on &sdc2_data_on>;
 			pinctrl-1 = <&sdc2_clk_off &sdc2_cmd_off &sdc2_data_off>;
 
-			cd-gpios = <&tlmm 100 0>;
+			cd-gpios = <&tlmm 100 GPIO_ACTIVE_HIGH>;
 			bus-width = <4>;
 			status = "disabled";
 		};
@@ -762,9 +757,10 @@
 			#interrupt-cells = <4>;
 		};
 
-		tcsr_mutex_regs: syscon@fd484000 {
-			compatible = "syscon";
-			reg = <0xfd484000 0x2000>;
+		tcsr_mutex: hwlock@fd484000 {
+			compatible = "qcom,msm8994-tcsr-mutex", "qcom,tcsr-mutex";
+			reg = <0xfd484000 0x1000>;
+			#hwlock-cells = <1>;
 		};
 
 		tlmm: pinctrl@fd510000 {
diff --git a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi
index e165b5e890a0..ca7c8d2e1d3d 100644
--- a/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996-sony-xperia-tone.dtsi
@@ -196,8 +196,8 @@
 
 &pcie0 {
 	status = "okay";
-	perst-gpio = <&tlmm 35 GPIO_ACTIVE_LOW>;
-	wake-gpio = <&tlmm 37 GPIO_ACTIVE_HIGH>;
+	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
+	wake-gpios = <&tlmm 37 GPIO_ACTIVE_HIGH>;
 	vddpe-3v3-supply = <&wlan_en>;
 	vdda-supply = <&pm8994_l28>;
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi
index 627649979891..77819186086a 100644
--- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-common.dtsi
@@ -58,6 +58,14 @@
 		};
 	};
 
+	irled {
+		compatible = "pwm-ir-tx";
+		pwms = <&pm8994_lpg 1 1000000>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&irled_default>;
+	};
+
 	reserved-memory {
 		memory@88800000 {
 			reg = <0x0 0x88800000 0x0 0x1400000>;
@@ -297,6 +305,41 @@
 	linux,code = <KEY_VOLUMEDOWN>;
 };
 
+&pm8994_lpg {
+	status = "okay";
+
+	qcom,power-source = <1>;
+};
+
+&pmi8994_lpg {
+	status = "okay";
+
+	qcom,power-source = <1>;
+
+	multi-led {
+		color = <LED_COLOR_ID_RGB>;
+		function = LED_FUNCTION_STATUS;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_BLUE>;
+		};
+
+		led@2 {
+			reg = <2>;
+			color = <LED_COLOR_ID_GREEN>;
+		};
+
+		led@3 {
+			reg = <3>;
+			color = <LED_COLOR_ID_RED>;
+		};
+	};
+};
+
 &slpi_pil {
 	status = "okay";
 
@@ -608,6 +651,15 @@
 };
 
 &pm8994_gpios {
+	irled_default: irled-default-state {
+		pins = "gpio5";
+		function = PMIC_GPIO_FUNC_FUNC1;
+		output-low;
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_HIGH>;
+		power-source = <PM8994_GPIO_S4>;
+		bias-disable;
+	};
+
 	wlan_en_default: wlan-en-state {
 		pins = "gpio8";
 		function = PMIC_GPIO_FUNC_NORMAL;
diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts
index 25f30ec277c1..4e5264f4116a 100644
--- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts
+++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-gemini.dts
@@ -8,6 +8,7 @@
 #include "msm8996-xiaomi-common.dtsi"
 #include <dt-bindings/sound/qcom,q6afe.h>
 #include <dt-bindings/sound/qcom,q6asm.h>
+#include <dt-bindings/sound/qcom,wcd9335.h>
 #include <dt-bindings/input/ti-drv260x.h>
 
 / {
@@ -193,7 +194,7 @@
 		};
 
 		codec {
-			sound-dai = <&wcd9335 6>;
+			sound-dai = <&wcd9335 AIF4_PB>;
 		};
 	};
 
@@ -208,7 +209,7 @@
 		};
 
 		codec {
-			sound-dai = <&wcd9335 1>;
+			sound-dai = <&wcd9335 AIF1_CAP>;
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-scorpio.dts b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-scorpio.dts
index 30a9e4bed4af..79be5fb1295b 100644
--- a/arch/arm64/boot/dts/qcom/msm8996-xiaomi-scorpio.dts
+++ b/arch/arm64/boot/dts/qcom/msm8996-xiaomi-scorpio.dts
@@ -9,6 +9,7 @@
 #include "pmi8996.dtsi"
 #include <dt-bindings/sound/qcom,q6afe.h>
 #include <dt-bindings/sound/qcom,q6asm.h>
+#include <dt-bindings/sound/qcom,wcd9335.h>
 
 / {
 	model = "Xiaomi Mi Note 2";
@@ -116,6 +117,25 @@
 			"qcom/msm8996/scorpio/modem.mbn";
 };
 
+&pm8994_lpg {
+	pinctrl-names = "default";
+	pinctrl-0 = <&keypad_default>;
+
+	led@3 {
+		reg = <3>;
+		color = <LED_COLOR_ID_WHITE>;
+		function = LED_FUNCTION_KBD_BACKLIGHT;
+		function-enumerator = <1>;
+	};
+
+	led@6 {
+		reg = <6>;
+		color = <LED_COLOR_ID_WHITE>;
+		function = LED_FUNCTION_KBD_BACKLIGHT;
+		function-enumerator = <0>;
+	};
+};
+
 &q6asmdai {
 	dai@0 {
 		reg = <0>;
@@ -171,7 +191,7 @@
 		};
 
 		codec {
-			sound-dai = <&wcd9335 6>;
+			sound-dai = <&wcd9335 AIF4_PB>;
 		};
 	};
 
@@ -186,7 +206,7 @@
 		};
 
 		codec {
-			sound-dai = <&wcd9335 1>;
+			sound-dai = <&wcd9335 AIF1_CAP>;
 		};
 	};
 };
@@ -258,6 +278,15 @@
 		"PMIC_SLB",		/* GPIO_20 */
 		"UIM_BATT_ALARM",	/* GPIO_21 */
 		"NC";			/* GPIO_22 */
+
+	keypad_default: keypad-default-state {
+		pins = "gpio7", "gpio10";
+		function = PMIC_GPIO_FUNC_FUNC1;
+		output-low;
+		qcom,drive-strength = <PMIC_GPIO_STRENGTH_LOW>;
+		power-source = <PM8994_GPIO_S4>;
+		bias-disable;
+	};
 };
 
 &pm8994_mpps {
diff --git a/arch/arm64/boot/dts/qcom/msm8996.dtsi b/arch/arm64/boot/dts/qcom/msm8996.dtsi
index 742eac4ce9b3..c0a2baffa49d 100644
--- a/arch/arm64/boot/dts/qcom/msm8996.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8996.dtsi
@@ -7,6 +7,7 @@
 #include <dt-bindings/clock/qcom,mmcc-msm8996.h>
 #include <dt-bindings/clock/qcom,rpmcc.h>
 #include <dt-bindings/interconnect/qcom,msm8996.h>
+#include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/soc/qcom,apr.h>
 #include <dt-bindings/thermal/thermal.h>
@@ -359,16 +360,10 @@
 	firmware {
 		scm {
 			compatible = "qcom,scm-msm8996", "qcom,scm";
-			qcom,dload-mode = <&tcsr 0x13000>;
+			qcom,dload-mode = <&tcsr_2 0x13000>;
 		};
 	};
 
-	tcsr_mutex: hwlock {
-		compatible = "qcom,tcsr-mutex";
-		syscon = <&tcsr_mutex_regs 0 0x1000>;
-		#hwlock-cells = <1>;
-	};
-
 	memory@80000000 {
 		device_type = "memory";
 		/* We expect the bootloader to fill in the reg */
@@ -838,12 +833,18 @@
 				 <&rpmcc RPM_SMD_PCNOC_A_CLK>;
 		};
 
-		tcsr_mutex_regs: syscon@740000 {
-			compatible = "syscon";
-			reg = <0x00740000 0x40000>;
+		tcsr_mutex: hwlock@740000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x00740000 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsr_1: syscon@760000 {
+			compatible = "qcom,tcsr-msm8996", "syscon";
+			reg = <0x00760000 0x20000>;
 		};
 
-		tcsr: syscon@7a0000 {
+		tcsr_2: syscon@7a0000 {
 			compatible = "qcom,tcsr-msm8996", "syscon";
 			reg = <0x007a0000 0x18000>;
 		};
@@ -1161,9 +1162,13 @@
 					    "hdmi_phy";
 
 				clocks = <&mmcc MDSS_AHB_CLK>,
-					 <&gcc GCC_HDMI_CLKREF_CLK>;
+					 <&gcc GCC_HDMI_CLKREF_CLK>,
+					 <&xo_board>;
 				clock-names = "iface",
-					      "ref";
+					      "ref",
+					      "xo";
+
+				#clock-cells = <0>;
 
 				status = "disabled";
 			};
@@ -2413,7 +2418,7 @@
 			qcom,smem-states = <&mpss_smp2p_out 0>;
 			qcom,smem-state-names = "stop";
 
-			qcom,halt-regs = <&tcsr_mutex_regs 0x23000 0x25000 0x24000>;
+			qcom,halt-regs = <&tcsr_1 0x3000 0x5000 0x4000>;
 
 			status = "disabled";
 
@@ -3018,7 +3023,7 @@
 		sdhc1: mmc@7464900 {
 			compatible = "qcom,msm8996-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0x07464900 0x11c>, <0x07464000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 
 			interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>,
 					<GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>;
@@ -3042,7 +3047,7 @@
 		sdhc2: mmc@74a4900 {
 			compatible = "qcom,msm8996-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0x074a4900 0x314>, <0x074a4000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 
 			interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
 				      <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
@@ -3337,7 +3342,7 @@
 					interrupt-names = "intr1", "intr2";
 					interrupt-controller;
 					#interrupt-cells = <1>;
-					reset-gpios = <&tlmm 64 0>;
+					reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>;
 
 					slim-ifc-dev = <&tasha_ifd>;
 
@@ -3499,7 +3504,7 @@
 		};
 
 		saw3: syscon@9a10000 {
-			compatible = "syscon";
+			compatible = "qcom,tcsr-msm8996", "syscon";
 			reg = <0x09a10000 0x1000>;
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi
index 02d21bff2198..f05f16ac5cc1 100644
--- a/arch/arm64/boot/dts/qcom/msm8998.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi
@@ -308,12 +308,6 @@
 		};
 	};
 
-	tcsr_mutex: hwlock {
-		compatible = "qcom,tcsr-mutex";
-		syscon = <&tcsr_mutex_regs 0 0x1000>;
-		#hwlock-cells = <1>;
-	};
-
 	psci {
 		compatible = "arm,psci-1.0";
 		method = "smc";
@@ -1047,9 +1041,15 @@
 			};
 		};
 
-		tcsr_mutex_regs: syscon@1f40000 {
-			compatible = "syscon";
-			reg = <0x01f40000 0x40000>;
+		tcsr_mutex: hwlock@1f40000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x01f40000 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsr_regs_1: syscon@1f60000 {
+			compatible = "qcom,msm8998-tcsr", "syscon";
+			reg = <0x01f60000 0x20000>;
 		};
 
 		tlmm: pinctrl@3400000 {
@@ -1340,7 +1340,7 @@
 			resets = <&gcc GCC_MSS_RESTART>;
 			reset-names = "mss_restart";
 
-			qcom,halt-regs = <&tcsr_mutex_regs 0x23000 0x25000 0x24000>;
+			qcom,halt-regs = <&tcsr_regs_1 0x3000 0x5000 0x4000>;
 
 			power-domains = <&rpmpd MSM8998_VDDCX>,
 					<&rpmpd MSM8998_VDDMX>;
@@ -2076,9 +2076,9 @@
 		};
 
 		sdhc2: mmc@c0a4900 {
-			compatible = "qcom,sdhci-msm-v4";
+			compatible = "qcom,msm8998-sdhci", "qcom,sdhci-msm-v4";
 			reg = <0x0c0a4900 0x314>, <0x0c0a4000 0x800>;
-			reg-names = "hc_mem", "core_mem";
+			reg-names = "hc", "core";
 
 			interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>;
diff --git a/arch/arm64/boot/dts/qcom/pm6150l.dtsi b/arch/arm64/boot/dts/qcom/pm6150l.dtsi
index 7aa2ef90cb6a..f02c223ef448 100644
--- a/arch/arm64/boot/dts/qcom/pm6150l.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm6150l.dtsi
@@ -20,10 +20,29 @@
 			#size-cells = <0>;
 			#io-channel-cells = <1>;
 
+			adc-chan@0 {
+				reg = <ADC5_REF_GND>;
+				qcom,pre-scaling = <1 1>;
+				label = "ref_gnd";
+			};
+
+			adc-chan@1 {
+				reg = <ADC5_1P25VREF>;
+				qcom,pre-scaling = <1 1>;
+				label = "vref_1p25";
+			};
+
 			adc-chan@6 {
 				reg = <ADC5_DIE_TEMP>;
+				qcom,pre-scaling = <1 1>;
 				label = "die_temp";
 			};
+
+			adc-chan@83 {
+				reg = <ADC5_VPH_PWR>;
+				qcom,pre-scaling = <1 3>;
+				label = "vph_pwr";
+			};
 		};
 
 		pm6150l_adc_tm: adc-tm@3500 {
diff --git a/arch/arm64/boot/dts/qcom/pm660.dtsi b/arch/arm64/boot/dts/qcom/pm660.dtsi
index d0eefbb51663..e1622b16c08b 100644
--- a/arch/arm64/boot/dts/qcom/pm660.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm660.dtsi
@@ -187,7 +187,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pm660_spmi_regulators: pm660-regulators {
+		pm660_spmi_regulators: regulators {
 			compatible = "qcom,pm660-regulators";
 		};
 	};
diff --git a/arch/arm64/boot/dts/qcom/pm660l.dtsi b/arch/arm64/boot/dts/qcom/pm660l.dtsi
index c7945470ffee..8aa0a5078772 100644
--- a/arch/arm64/boot/dts/qcom/pm660l.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm660l.dtsi
@@ -65,7 +65,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pm660l_lpg: lpg@b100 {
+		pm660l_lpg: pwm {
 			compatible = "qcom,pm660l-lpg";
 
 			status = "disabled";
@@ -81,7 +81,7 @@
 			status = "disabled";
 		};
 
-		pm660l_spmi_regulators: pm660l-regulators {
+		pm660l_spmi_regulators: regulators {
 			compatible = "qcom,pm660l-regulators";
 		};
 	};
diff --git a/arch/arm64/boot/dts/qcom/pm7250b.dtsi b/arch/arm64/boot/dts/qcom/pm7250b.dtsi
new file mode 100644
index 000000000000..61f7a6345150
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/pm7250b.dtsi
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (C) 2022 Luca Weiss <luca.weiss@fairphone.com>
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/spmi/spmi.h>
+
+/ {
+	thermal-zones {
+		pm7250b-thermal {
+			polling-delay-passive = <100>;
+			polling-delay = <0>;
+
+			thermal-sensors = <&pm7250b_temp>;
+
+			trips {
+				trip0 {
+					temperature = <95000>;
+					hysteresis = <0>;
+					type = "passive";
+				};
+
+				trip1 {
+					temperature = <115000>;
+					hysteresis = <0>;
+					type = "hot";
+				};
+
+				trip2 {
+					temperature = <145000>;
+					hysteresis = <0>;
+					type = "critical";
+				};
+			};
+		};
+	};
+};
+
+&spmi_bus {
+	pmic@2 {
+		compatible = "qcom,pm7250b", "qcom,spmi-pmic";
+		reg = <0x2 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		pm7250b_temp: temp-alarm@2400 {
+			compatible = "qcom,spmi-temp-alarm";
+			reg = <0x2400>;
+			interrupts = <0x2 0x24 0x0 IRQ_TYPE_EDGE_BOTH>;
+			io-channels = <&pm7250b_adc ADC5_DIE_TEMP>;
+			io-channel-names = "thermal";
+			#thermal-sensor-cells = <0>;
+		};
+
+		pm7250b_adc: adc@3100 {
+			compatible = "qcom,spmi-adc5";
+			reg = <0x3100>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			#io-channel-cells = <1>;
+			interrupts = <0x2 0x31 0x0 IRQ_TYPE_EDGE_RISING>;
+
+			adc-chan@0 {
+				reg = <ADC5_REF_GND>;
+				qcom,pre-scaling = <1 1>;
+				label = "ref_gnd";
+			};
+
+			adc-chan@1 {
+				reg = <ADC5_1P25VREF>;
+				qcom,pre-scaling = <1 1>;
+				label = "vref_1p25";
+			};
+
+			adc-chan@2 {
+				reg = <ADC5_DIE_TEMP>;
+				qcom,pre-scaling = <1 1>;
+				label = "die_temp";
+			};
+
+			adc-chan@7 {
+				reg = <ADC5_USB_IN_I>;
+				qcom,pre-scaling = <1 1>;
+				label = "usb_in_i_uv";
+			};
+
+			adc-chan@8 {
+				reg = <ADC5_USB_IN_V_16>;
+				qcom,pre-scaling = <1 16>;
+				label = "usb_in_v_div_16";
+			};
+
+			adc-chan@9 {
+				reg = <ADC5_CHG_TEMP>;
+				qcom,pre-scaling = <1 1>;
+				label = "chg_temp";
+			};
+
+			adc-chan@e {
+				reg = <ADC5_AMUX_THM2>;
+				qcom,hw-settle-time = <200>;
+				qcom,pre-scaling = <1 1>;
+				label = "smb1390_therm";
+			};
+
+			adc-chan@1e {
+				reg = <ADC5_MID_CHG_DIV6>;
+				qcom,pre-scaling = <1 6>;
+				label = "chg_mid";
+			};
+
+			adc-chan@83 {
+				reg = <ADC5_VPH_PWR>;
+				qcom,pre-scaling = <1 3>;
+				label = "vph_pwr";
+			};
+
+			adc-chan@84 {
+				reg = <ADC5_VBAT_SNS>;
+				qcom,pre-scaling = <1 3>;
+				label = "vbat_sns";
+			};
+
+			adc-chan@99 {
+				reg = <ADC5_SBUx>;
+				qcom,pre-scaling = <1 3>;
+				label = "chg_sbux";
+			};
+		};
+
+		pm7250b_adc_tm: adc-tm@3500 {
+			compatible = "qcom,spmi-adc-tm5";
+			reg = <0x3500>;
+			interrupts = <0x2 0x35 0x0 IRQ_TYPE_EDGE_RISING>;
+			#thermal-sensor-cells = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+	};
+
+	pmic@3 {
+		compatible = "qcom,pm7250b", "qcom,spmi-pmic";
+		reg = <0x3 SPMI_USID>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/pm8150.dtsi b/arch/arm64/boot/dts/qcom/pm8150.dtsi
index fd8434215924..574fa95a2871 100644
--- a/arch/arm64/boot/dts/qcom/pm8150.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8150.dtsi
@@ -47,7 +47,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pon: power-on@800 {
+		pon: pon@800 {
 			compatible = "qcom,pm8998-pon";
 			reg = <0x0800>;
 			mode-bootloader = <0x2>;
diff --git a/arch/arm64/boot/dts/qcom/pm8150b.dtsi b/arch/arm64/boot/dts/qcom/pm8150b.dtsi
index 5d1ec3a6cc3c..cdded791d96e 100644
--- a/arch/arm64/boot/dts/qcom/pm8150b.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8150b.dtsi
@@ -46,7 +46,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		power-on@800 {
+		pon@800 {
 			compatible = "qcom,pm8916-pon";
 			reg = <0x0800>;
 
@@ -128,7 +128,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pm8150b_lpg: lpg {
+		pm8150b_lpg: pwm {
 			compatible = "qcom,pm8150b-lpg";
 
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/pm8150l.dtsi b/arch/arm64/boot/dts/qcom/pm8150l.dtsi
index c62d023b39a2..135bfb8d629b 100644
--- a/arch/arm64/boot/dts/qcom/pm8150l.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8150l.dtsi
@@ -46,7 +46,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		power-on@800 {
+		pon@800 {
 			compatible = "qcom,pm8916-pon";
 			reg = <0x0800>;
 
@@ -116,7 +116,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pm8150l_lpg: lpg {
+		pm8150l_lpg: pwm {
 			compatible = "qcom,pm8150l-lpg";
 
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/pm8350c.dtsi b/arch/arm64/boot/dts/qcom/pm8350c.dtsi
index e0bbb67717fe..f28e71487d5c 100644
--- a/arch/arm64/boot/dts/qcom/pm8350c.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8350c.dtsi
@@ -30,9 +30,8 @@
 			#interrupt-cells = <2>;
 		};
 
-		pm8350c_pwm: pwm@e800 {
+		pm8350c_pwm: pwm {
 			compatible = "qcom,pm8350c-pwm";
-			reg = <0xe800>;
 			#pwm-cells = <2>;
 			status = "disabled";
 		};
diff --git a/arch/arm64/boot/dts/qcom/pm8953.dtsi b/arch/arm64/boot/dts/qcom/pm8953.dtsi
index 741c538a9cee..a1d36f9ebbd2 100644
--- a/arch/arm64/boot/dts/qcom/pm8953.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8953.dtsi
@@ -45,7 +45,7 @@
 			#thermal-sensor-cells = <0>;
 		};
 
-		pm8953_vadc: vadc@3100 {
+		pm8953_vadc: adc@3100 {
 			compatible = "qcom,spmi-vadc";
 			reg = <0x3100>;
 			interrupts = <0x00 0x31 0x00 0x01>;
diff --git a/arch/arm64/boot/dts/qcom/pm8994.dtsi b/arch/arm64/boot/dts/qcom/pm8994.dtsi
index ab342397fcd8..e92e5ac414d3 100644
--- a/arch/arm64/boot/dts/qcom/pm8994.dtsi
+++ b/arch/arm64/boot/dts/qcom/pm8994.dtsi
@@ -135,7 +135,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pm8994_lpg: lpg {
+		pm8994_lpg: pwm {
 			compatible = "qcom,pm8994-lpg";
 
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/pmi8994.dtsi b/arch/arm64/boot/dts/qcom/pmi8994.dtsi
index 84c44912ec93..542c215dde10 100644
--- a/arch/arm64/boot/dts/qcom/pmi8994.dtsi
+++ b/arch/arm64/boot/dts/qcom/pmi8994.dtsi
@@ -21,7 +21,7 @@
 		};
 
 		pmi8994_mpps: mpps@a000 {
-			compatible = "qcom,pmi8994-mpp";
+			compatible = "qcom,pmi8994-mpp", "qcom,spmi-mpp";
 			reg = <0xa000>;
 			gpio-controller;
 			gpio-ranges = <&pmi8994_mpps 0 0 4>;
@@ -37,7 +37,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pmi8994_lpg: lpg {
+		pmi8994_lpg: pwm {
 			compatible = "qcom,pmi8994-lpg";
 
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/pmi8998.dtsi b/arch/arm64/boot/dts/qcom/pmi8998.dtsi
index 6d3d212560c1..3852a012bb0f 100644
--- a/arch/arm64/boot/dts/qcom/pmi8998.dtsi
+++ b/arch/arm64/boot/dts/qcom/pmi8998.dtsi
@@ -42,7 +42,7 @@
 			};
 		};
 
-		pmi8998_lpg: lpg {
+		pmi8998_lpg: pwm {
 			compatible = "qcom,pmi8998-lpg";
 
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/pmk8350.dtsi b/arch/arm64/boot/dts/qcom/pmk8350.dtsi
index 0f94c46a1444..a7ec9d11946d 100644
--- a/arch/arm64/boot/dts/qcom/pmk8350.dtsi
+++ b/arch/arm64/boot/dts/qcom/pmk8350.dtsi
@@ -39,16 +39,13 @@
 			#address-cells = <1>;
 			#size-cells = <0>;
 			interrupts = <0x0 0x31 0x0 IRQ_TYPE_EDGE_RISING>;
-			interrupt-names = "eoc-int-en-set";
 			#io-channel-cells = <1>;
-			io-channel-ranges;
 		};
 
 		pmk8350_adc_tm: adc-tm@3400 {
 			compatible = "qcom,adc-tm7";
 			reg = <0x3400>;
 			interrupts = <0x0 0x34 0x0 IRQ_TYPE_EDGE_RISING>;
-			interrupt-names = "threshold";
 			#address-cells = <1>;
 			#size-cells = <0>;
 			#thermal-sensor-cells = <1>;
diff --git a/arch/arm64/boot/dts/qcom/pmm8155au_1.dtsi b/arch/arm64/boot/dts/qcom/pmm8155au_1.dtsi
index 68e9122363ae..20c5d60c8c2c 100644
--- a/arch/arm64/boot/dts/qcom/pmm8155au_1.dtsi
+++ b/arch/arm64/boot/dts/qcom/pmm8155au_1.dtsi
@@ -46,7 +46,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		pon: power-on@800 {
+		pon: pon@800 {
 			compatible = "qcom,pm8916-pon";
 			reg = <0x0800>;
 			pwrkey {
diff --git a/arch/arm64/boot/dts/qcom/pmm8155au_2.dtsi b/arch/arm64/boot/dts/qcom/pmm8155au_2.dtsi
index c307fc662511..1da4606e8ee6 100644
--- a/arch/arm64/boot/dts/qcom/pmm8155au_2.dtsi
+++ b/arch/arm64/boot/dts/qcom/pmm8155au_2.dtsi
@@ -45,7 +45,7 @@
 		#address-cells = <1>;
 		#size-cells = <0>;
 
-		power-on@800 {
+		pon@800 {
 			compatible = "qcom,pm8916-pon";
 			reg = <0x0800>;
 
diff --git a/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi b/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi
index 1721ebe5759b..1678ef0f8684 100644
--- a/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi
+++ b/arch/arm64/boot/dts/qcom/qcs404-evb.dtsi
@@ -99,7 +99,7 @@
 &pcie {
 	status = "okay";
 
-	perst-gpio = <&tlmm 43 GPIO_ACTIVE_LOW>;
+	perst-gpios = <&tlmm 43 GPIO_ACTIVE_LOW>;
 
 	pinctrl-names = "default";
 	pinctrl-0 = <&perst_state>;
diff --git a/arch/arm64/boot/dts/qcom/qcs404.dtsi b/arch/arm64/boot/dts/qcom/qcs404.dtsi
index 9ab990061522..80f2d05595fa 100644
--- a/arch/arm64/boot/dts/qcom/qcs404.dtsi
+++ b/arch/arm64/boot/dts/qcom/qcs404.dtsi
@@ -295,12 +295,6 @@
 		hwlocks = <&tcsr_mutex 3>;
 	};
 
-	tcsr_mutex: hwlock {
-		compatible = "qcom,tcsr-mutex";
-		syscon = <&tcsr_mutex_regs 0 0x1000>;
-		#hwlock-cells = <1>;
-	};
-
 	soc: soc@0 {
 		#address-cells = <1>;
 		#size-cells = <1>;
@@ -726,13 +720,14 @@
 			assigned-clock-rates = <19200000>;
 		};
 
-		tcsr_mutex_regs: syscon@1905000 {
-			compatible = "syscon";
+		tcsr_mutex: hwlock@1905000 {
+			compatible = "qcom,tcsr-mutex";
 			reg = <0x01905000 0x20000>;
+			#hwlock-cells = <1>;
 		};
 
 		tcsr: syscon@1937000 {
-			compatible = "syscon";
+			compatible = "qcom,qcs404-tcsr", "syscon";
 			reg = <0x01937000 0x25000>;
 		};
 
@@ -1297,7 +1292,7 @@
 		};
 
 		pcie: pci@10000000 {
-			compatible = "qcom,pcie-qcs404", "snps,dw-pcie";
+			compatible = "qcom,pcie-qcs404";
 			reg =  <0x10000000 0xf1d>,
 			       <0x10000f20 0xa8>,
 			       <0x07780000 0x2000>,
diff --git a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts
index ba547ca9fc6b..87ab0e1ecd16 100644
--- a/arch/arm64/boot/dts/qcom/sa8155p-adp.dts
+++ b/arch/arm64/boot/dts/qcom/sa8155p-adp.dts
@@ -333,9 +333,6 @@
 	snps,reset-active-low;
 	snps,reset-delays-us = <0 11000 70000>;
 
-	snps,ptp-ref-clk-rate = <250000000>;
-	snps,ptp-req-clk-rate = <96000000>;
-
 	snps,mtl-rx-config = <&mtl_rx_setup>;
 	snps,mtl-tx-config = <&mtl_tx_setup>;
 
diff --git a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts
index 9398f0349944..b608b82dff03 100644
--- a/arch/arm64/boot/dts/qcom/sa8295p-adp.dts
+++ b/arch/arm64/boot/dts/qcom/sa8295p-adp.dts
@@ -35,7 +35,6 @@
 			regulator-min-microvolt = <1200000>;
 			regulator-max-microvolt = <1208000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l5a: ldo5 {
@@ -43,7 +42,6 @@
 			regulator-min-microvolt = <912000>;
 			regulator-max-microvolt = <912000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l7a: ldo7 {
@@ -51,7 +49,6 @@
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <1800000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l13a: ldo13 {
@@ -59,7 +56,6 @@
 			regulator-min-microvolt = <3072000>;
 			regulator-max-microvolt = <3072000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 	};
 
@@ -72,7 +68,6 @@
 			regulator-min-microvolt = <912000>;
 			regulator-max-microvolt = <912000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l2c: ldo2 {
@@ -80,7 +75,6 @@
 			regulator-min-microvolt = <3072000>;
 			regulator-max-microvolt = <3072000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l3c: ldo3 {
@@ -96,7 +90,6 @@
 			regulator-min-microvolt = <1200000>;
 			regulator-max-microvolt = <1208000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l6c: ldo6 {
@@ -112,7 +105,6 @@
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <1800000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l10c: ldo10 {
@@ -141,7 +133,6 @@
 			regulator-min-microvolt = <1200000>;
 			regulator-max-microvolt = <1200000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l7g: ldo7 {
@@ -149,7 +140,6 @@
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <1800000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l8g: ldo8 {
@@ -157,7 +147,6 @@
 			regulator-min-microvolt = <880000>;
 			regulator-max-microvolt = <880000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 	};
 };
@@ -194,9 +183,10 @@
 		#size-cells = <0>;
 
 		pm8450a_gpios: gpio@c000 {
-			compatible = "qcom,pm8150-gpio";
+			compatible = "qcom,pm8150-gpio", "qcom,spmi-gpio";
 			reg = <0xc000>;
 			gpio-controller;
+			gpio-ranges = <&pm8450a_gpios 0 0 10>;
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
@@ -210,9 +200,10 @@
 		#size-cells = <0>;
 
 		pm8450c_gpios: gpio@c000 {
-			compatible = "qcom,pm8150-gpio";
+			compatible = "qcom,pm8150-gpio", "qcom,spmi-gpio";
 			reg = <0xc000>;
 			gpio-controller;
+			gpio-ranges = <&pm8450c_gpios 0 0 10>;
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
@@ -226,9 +217,10 @@
 		#size-cells = <0>;
 
 		pm8450e_gpios: gpio@c000 {
-			compatible = "qcom,pm8150-gpio";
+			compatible = "qcom,pm8150-gpio", "qcom,spmi-gpio";
 			reg = <0xc000>;
 			gpio-controller;
+			gpio-ranges = <&pm8450e_gpios 0 0 10>;
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
@@ -242,9 +234,10 @@
 		#size-cells = <0>;
 
 		pm8450g_gpios: gpio@c000 {
-			compatible = "qcom,pm8150-gpio";
+			compatible = "qcom,pm8150-gpio", "qcom,spmi-gpio";
 			reg = <0xc000>;
 			gpio-controller;
+			gpio-ranges = <&pm8450g_gpios 0 0 10>;
 			#gpio-cells = <2>;
 			interrupt-controller;
 			#interrupt-cells = <2>;
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts
index 8290d036044a..edfcd47e1a00 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-coachz-r1.dts
@@ -24,8 +24,6 @@
 };
 
 &pm6150_adc {
-	status = "disabled";
-
 	/delete-node/ skin-temp-thermistor@4e;
 	/delete-node/ charger-thermistor@4f;
 };
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts
index bfbf26fd2cd4..d49de65aa960 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r0.dts
@@ -16,17 +16,6 @@
 	compatible = "google,lazor-rev0", "qcom,sc7180";
 };
 
-&pp3300_hub {
-	/* pp3300_l7c is used to power the USB hub */
-	/delete-property/regulator-always-on;
-	/delete-property/regulator-boot-on;
-};
-
-&pp3300_l7c {
-	regulator-always-on;
-	regulator-boot-on;
-};
-
 &sn65dsi86_out {
 	/*
 	 * Lane 0 was incorrectly mapped on the cable, but we've now decided
@@ -35,3 +24,11 @@
 	 */
 	lane-polarities = <1 0>;
 };
+
+&usb_hub_2_x {
+	 vdd-supply = <&pp3300_l7c>;
+};
+
+&usb_hub_3_x {
+	 vdd-supply = <&pp3300_l7c>;
+};
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts
index d45a59afd7fc..80c7108bc51b 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor-r1.dts
@@ -16,13 +16,11 @@
 	compatible = "google,lazor-rev1", "google,lazor-rev2", "qcom,sc7180";
 };
 
-&pp3300_hub {
-	/* pp3300_l7c is used to power the USB hub */
-	/delete-property/regulator-always-on;
-	/delete-property/regulator-boot-on;
+
+&usb_hub_2_x {
+	 vdd-supply = <&pp3300_l7c>;
 };
 
-&pp3300_l7c {
-	regulator-always-on;
-	regulator-boot-on;
+&usb_hub_3_x {
+	 vdd-supply = <&pp3300_l7c>;
 };
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi
index 2cf7d5212c61..002663d752da 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-lazor.dtsi
@@ -55,8 +55,6 @@ ap_ts_pen_1v8: &i2c4 {
 };
 
 &pm6150_adc {
-	status = "disabled";
-
 	/delete-node/ charger-thermistor@4f;
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pazquel-lte-parade.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pazquel-lte-parade.dts
index 764c451c1a85..767cb7450c0d 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pazquel-lte-parade.dts
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pazquel-lte-parade.dts
@@ -14,7 +14,7 @@
 
 / {
 	model = "Google Pazquel (Parade,LTE)";
-	compatible = "google,pazquel-sku4", "qcom,sc7180";
+	compatible = "google,pazquel-sku6", "google,pazquel-sku4", "qcom,sc7180";
 };
 
 &ap_sar_sensor_i2c {
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts
index 76a130bad60a..8467ff41e6d5 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-pompom-r1.dts
@@ -34,13 +34,10 @@
 	/delete-node/ charger-thermistor@0;
 };
 
-&pp3300_hub {
-	/* pp3300_l7c is used to power the USB hub */
-	/delete-property/regulator-always-on;
-	/delete-property/regulator-boot-on;
+&usb_hub_2_x {
+	 vdd-supply = <&pp3300_l7c>;
 };
 
-&pp3300_l7c {
-	regulator-always-on;
-	regulator-boot-on;
+&usb_hub_3_x {
+	 vdd-supply = <&pp3300_l7c>;
 };
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts b/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts
index 59a23d0e9651..bc097d1b1b23 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor-r1.dts
@@ -44,17 +44,6 @@ ap_ts_pen_1v8: &i2c4 {
 	compatible = "auo,b116xa01";
 };
 
-&pp3300_hub {
-	/* pp3300_l7c is used to power the USB hub */
-	/delete-property/regulator-always-on;
-	/delete-property/regulator-boot-on;
-};
-
-&pp3300_l7c {
-	regulator-always-on;
-	regulator-boot-on;
-};
-
 &sdhc_2 {
 	status = "okay";
 };
@@ -63,6 +52,14 @@ ap_ts_pen_1v8: &i2c4 {
 	interrupts = <58 IRQ_TYPE_EDGE_FALLING>;
 };
 
+&usb_hub_2_x {
+	 vdd-supply = <&pp3300_l7c>;
+};
+
+&usb_hub_3_x {
+	 vdd-supply = <&pp3300_l7c>;
+};
+
 /* PINCTRL - modifications to sc7180-trogdor.dtsi */
 
 &trackpad_int_1v8_odl {
diff --git a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi
index b5f534db135a..eae22e6e97c1 100644
--- a/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7180-trogdor.dtsi
@@ -299,7 +299,7 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&en_pp3300_hub>;
 
-		regulator-always-on;
+		/* The BIOS leaves this regulator on */
 		regulator-boot-on;
 
 		vin-supply = <&pp3300_a>;
@@ -936,6 +936,24 @@ ap_spi_fp: &spi10 {
 
 &usb_1_dwc3 {
 	dr_mode = "host";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	/* 2.x hub on port 1 */
+	usb_hub_2_x: hub@1 {
+		compatible = "usbbda,5411";
+		reg = <1>;
+		vdd-supply = <&pp3300_hub>;
+		peer-hub = <&usb_hub_3_x>;
+	};
+
+	/* 3.x hub on port 2 */
+	usb_hub_3_x: hub@2 {
+		compatible = "usbbda,411";
+		reg = <2>;
+		vdd-supply = <&pp3300_hub>;
+		peer-hub = <&usb_hub_2_x>;
+	};
 };
 
 &usb_1_hsphy {
diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi
index b82c335c25af..58976a1ba06b 100644
--- a/arch/arm64/boot/dts/qcom/sc7180.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi
@@ -555,12 +555,6 @@
 		};
 	};
 
-	tcsr_mutex: hwlock {
-		compatible = "qcom,tcsr-mutex";
-		syscon = <&tcsr_mutex_regs 0 0x1000>;
-		#hwlock-cells = <1>;
-	};
-
 	smem {
 		compatible = "qcom,smem";
 		memory-region = <&smem_mem>;
@@ -1462,13 +1456,19 @@
 			status = "disabled";
 		};
 
-		tcsr_mutex_regs: syscon@1f40000 {
-			compatible = "syscon";
-			reg = <0 0x01f40000 0 0x40000>;
+		tcsr_mutex: hwlock@1f40000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0 0x01f40000 0 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsr_regs_1: syscon@1f60000 {
+			compatible = "qcom,sc7180-tcsr", "syscon";
+			reg = <0 0x01f60000 0 0x20000>;
 		};
 
-		tcsr_regs: syscon@1fc0000 {
-			compatible = "syscon";
+		tcsr_regs_2: syscon@1fc0000 {
+			compatible = "qcom,sc7180-tcsr", "syscon";
 			reg = <0 0x01fc0000 0 0x40000>;
 		};
 
@@ -1932,8 +1932,8 @@
 				 <&pdc_reset PDC_MODEM_SYNC_RESET>;
 			reset-names = "mss_restart", "pdc_reset";
 
-			qcom,halt-regs = <&tcsr_mutex_regs 0x23000 0x25000 0x24000>;
-			qcom,spare-regs = <&tcsr_regs 0xb3e4>;
+			qcom,halt-regs = <&tcsr_regs_1 0x3000 0x5000 0x4000>;
+			qcom,spare-regs = <&tcsr_regs_2 0xb3e4>;
 
 			status = "disabled";
 
diff --git a/arch/arm64/boot/dts/qcom/sc7280-chrome-common.dtsi b/arch/arm64/boot/dts/qcom/sc7280-chrome-common.dtsi
index cfe2741456a1..25f31c81b2b7 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-chrome-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280-chrome-common.dtsi
@@ -83,17 +83,6 @@
 	};
 };
 
-/* Modem setup is different on Chrome setups than typical Qualcomm setup */
-&remoteproc_mpss {
-	status = "okay";
-	compatible = "qcom,sc7280-mss-pil";
-	iommus = <&apps_smmu 0x124 0x0>, <&apps_smmu 0x488 0x7>;
-	interconnects = <&mc_virt MASTER_LLCC 0 &mc_virt SLAVE_EBI1 0>;
-	memory-region = <&mba_mem>, <&mpss_mem>;
-	firmware-name = "qcom/sc7280-herobrine/modem/mba.mbn",
-			"qcom/sc7280-herobrine/modem/qdsp6sw.mbn";
-};
-
 &remoteproc_wpss {
 	status = "okay";
 	firmware-name = "ath11k/WCN6750/hw1.0/wpss.mdt";
diff --git a/arch/arm64/boot/dts/qcom/sc7280-crd-r3.dts b/arch/arm64/boot/dts/qcom/sc7280-crd-r3.dts
index 344338ad8a01..dddb505e220b 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-crd-r3.dts
+++ b/arch/arm64/boot/dts/qcom/sc7280-crd-r3.dts
@@ -87,6 +87,36 @@ ap_ts_pen_1v8: &i2c13 {
 	pins = "gpio51";
 };
 
+&sound {
+	audio-routing =
+		"IN1_HPHL", "HPHL_OUT",
+		"IN2_HPHR", "HPHR_OUT",
+		"AMIC1", "MIC BIAS1",
+		"AMIC2", "MIC BIAS2",
+		"VA DMIC0", "MIC BIAS1",
+		"VA DMIC1", "MIC BIAS1",
+		"VA DMIC2", "MIC BIAS3",
+		"VA DMIC3", "MIC BIAS3",
+		"TX SWR_ADC0", "ADC1_OUTPUT",
+		"TX SWR_ADC1", "ADC2_OUTPUT",
+		"TX SWR_ADC2", "ADC3_OUTPUT",
+		"TX SWR_DMIC0", "DMIC1_OUTPUT",
+		"TX SWR_DMIC1", "DMIC2_OUTPUT",
+		"TX SWR_DMIC2", "DMIC3_OUTPUT",
+		"TX SWR_DMIC3", "DMIC4_OUTPUT",
+		"TX SWR_DMIC4", "DMIC5_OUTPUT",
+		"TX SWR_DMIC5", "DMIC6_OUTPUT",
+		"TX SWR_DMIC6", "DMIC7_OUTPUT",
+		"TX SWR_DMIC7", "DMIC8_OUTPUT";
+};
+
+&wcd9385 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&wcd_reset_n>, <&us_euro_hs_sel>;
+	pinctrl-1 = <&wcd_reset_n_sleep>, <&us_euro_hs_sel>;
+	us-euro-gpios = <&tlmm 81 GPIO_ACTIVE_HIGH>;
+};
+
 &tlmm {
 	tp_int_odl: tp-int-odl {
 		pins = "gpio7";
@@ -105,4 +135,11 @@ ap_ts_pen_1v8: &i2c13 {
 		function = "gpio";
 		bias-disable;
 	};
+
+	us_euro_hs_sel: us-euro-hs-sel {
+		pins = "gpio81";
+		function = "gpio";
+		bias-pull-down;
+		drive-strength = <2>;
+	};
 };
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-audio-wcd9385.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine-audio-wcd9385.dtsi
index 859faaa8b7e0..c72e53aaf997 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-herobrine-audio-wcd9385.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-audio-wcd9385.dtsi
@@ -5,6 +5,161 @@
  * Copyright (c) 2022, The Linux Foundation. All rights reserved.
  */
 
+/ {
+	/* BOARD-SPECIFIC TOP LEVEL NODES */
+	sound: sound {
+		compatible = "google,sc7280-herobrine";
+		model = "sc7280-wcd938x-max98360a-1mic";
+
+		audio-routing =
+			"IN1_HPHL", "HPHL_OUT",
+			"IN2_HPHR", "HPHR_OUT",
+			"AMIC1", "MIC BIAS1",
+			"AMIC2", "MIC BIAS2",
+			"VA DMIC0", "MIC BIAS1",
+			"VA DMIC1", "MIC BIAS1",
+			"VA DMIC2", "MIC BIAS3",
+			"VA DMIC3", "MIC BIAS3",
+			"TX SWR_ADC0", "ADC1_OUTPUT",
+			"TX SWR_ADC1", "ADC2_OUTPUT",
+			"TX SWR_ADC2", "ADC3_OUTPUT",
+			"TX SWR_DMIC0", "DMIC1_OUTPUT",
+			"TX SWR_DMIC1", "DMIC2_OUTPUT",
+			"TX SWR_DMIC2", "DMIC3_OUTPUT",
+			"TX SWR_DMIC3", "DMIC4_OUTPUT",
+			"TX SWR_DMIC4", "DMIC5_OUTPUT",
+			"TX SWR_DMIC5", "DMIC6_OUTPUT",
+			"TX SWR_DMIC6", "DMIC7_OUTPUT",
+			"TX SWR_DMIC7", "DMIC8_OUTPUT";
+
+		qcom,msm-mbhc-hphl-swh = <1>;
+		qcom,msm-mbhc-gnd-swh = <1>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+		#sound-dai-cells = <0>;
+
+		dai-link@0 {
+			link-name = "MAX98360A";
+			reg = <0>;
+
+			cpu {
+				sound-dai = <&lpass_cpu MI2S_SECONDARY>;
+			};
+
+			codec {
+				sound-dai = <&max98360a>;
+			};
+		};
+
+		dai-link@1 {
+			link-name = "DisplayPort";
+			reg = <1>;
+
+			cpu {
+				sound-dai = <&lpass_cpu LPASS_DP_RX>;
+			};
+
+			codec {
+				sound-dai = <&mdss_dp>;
+			};
+		};
+
+		dai-link@2 {
+			link-name = "WCD9385 Playback";
+			reg = <2>;
+
+			cpu {
+				sound-dai = <&lpass_cpu LPASS_CDC_DMA_RX0>;
+			};
+
+			codec {
+				sound-dai = <&wcd9385 0>, <&swr0 0>, <&lpass_rx_macro 0>;
+			};
+		};
+
+		dai-link@3 {
+			link-name = "WCD9385 Capture";
+			reg = <3>;
+
+			cpu {
+				sound-dai = <&lpass_cpu LPASS_CDC_DMA_TX3>;
+			};
+
+			codec {
+				sound-dai = <&wcd9385 1>, <&swr1 0>, <&lpass_tx_macro 0>;
+			};
+		};
+
+		dai-link@4 {
+			link-name = "DMIC";
+			reg = <4>;
+
+			cpu {
+				sound-dai = <&lpass_cpu LPASS_CDC_DMA_VA_TX0>;
+			};
+
+			codec {
+				sound-dai = <&lpass_va_macro 0>;
+			};
+		};
+	};
+};
+
+/* ADDITIONS TO NODES DEFINED IN PARENT DEVICE TREE FILES */
+
+&lpass_cpu {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&mi2s1_data0>, <&mi2s1_sclk>, <&mi2s1_ws>;
+
+	dai-link@1 {
+		reg = <MI2S_SECONDARY>;
+		qcom,playback-sd-lines = <0>;
+	};
+
+	dai-link@5 {
+		reg = <LPASS_DP_RX>;
+	};
+
+	dai-link@6 {
+		reg = <LPASS_CDC_DMA_RX0>;
+	};
+
+	dai-link@19 {
+		reg = <LPASS_CDC_DMA_TX3>;
+	};
+
+	dai-link@25 {
+		reg = <LPASS_CDC_DMA_VA_TX0>;
+	};
+};
+
+&lpass_rx_macro {
+	status = "okay";
+};
+
+&lpass_tx_macro {
+	status = "okay";
+};
+
+&lpass_va_macro {
+	status = "okay";
+};
+
+&swr0 {
+	status = "okay";
+};
+
+&swr1 {
+	status = "okay";
+};
+
+&wcd9385 {
+	status = "okay";
+};
+
 /* PINCTRL */
 
 &lpass_dmic01_clk {
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-crd.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-crd.dts
index 7881bbc641a0..f0f26af1e421 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-herobrine-crd.dts
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-crd.dts
@@ -9,10 +9,11 @@
 
 #include "sc7280-herobrine.dtsi"
 #include "sc7280-herobrine-audio-wcd9385.dtsi"
+#include "sc7280-herobrine-lte-sku.dtsi"
 
 / {
 	model = "Qualcomm Technologies, Inc. sc7280 CRD platform (rev5+)";
-	compatible = "google,hoglin", "qcom,sc7280";
+	compatible = "google,zoglin", "google,hoglin", "qcom,sc7280";
 
 	/* FIXED REGULATORS */
 
@@ -167,7 +168,7 @@ ap_ts_pen_1v8: &i2c13 {
 			  "PMIC_EDP_BL_PWM",
 			  "";
 
-	edp_bl_reg_en: edp-bl-reg-en {
+	edp_bl_reg_en: edp-bl-reg-en-state {
 		pins = "gpio6";
 		function = "normal";
 		bias-disable;
@@ -371,7 +372,5 @@ ap_ts_pen_1v8: &i2c13 {
 			  "",				/* 170 */
 			  "MOS_BLE_UART_TX",
 			  "MOS_BLE_UART_RX",
-			  "",
-			  "",
 			  "";
 };
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-evoker-r0.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-evoker-r0.dts
new file mode 100644
index 000000000000..ccbe50b6249a
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-evoker-r0.dts
@@ -0,0 +1,333 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Google Evoker board device tree source
+ *
+ * Copyright 2022 Google LLC.
+ */
+
+/dts-v1/;
+
+#include "sc7280-herobrine.dtsi"
+
+/ {
+	model = "Google Evoker";
+	compatible = "google,evoker", "qcom,sc7280";
+};
+
+/*
+ * ADDITIONS TO FIXED REGULATORS DEFINED IN PARENT DEVICE TREE FILES
+ *
+ * Sort order matches the order in the parent files (parents before children).
+ */
+
+&pp3300_codec {
+	status = "okay";
+};
+
+/* ADDITIONS TO NODES DEFINED IN PARENT DEVICE TREE FILES */
+
+ap_tp_i2c: &i2c0 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	trackpad: trackpad@2c {
+		compatible = "hid-over-i2c";
+		reg = <0x2c>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&tp_int_odl>;
+
+		interrupt-parent = <&tlmm>;
+		interrupts = <7 IRQ_TYPE_EDGE_FALLING>;
+
+		hid-descr-addr = <0x20>;
+		vcc-supply = <&pp3300_z1>;
+
+		wakeup-source;
+	};
+};
+
+ts_i2c: &i2c13 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	ap_ts: touchscreen@10 {
+		compatible = "elan,ekth6915";
+		reg = <0x10>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&ts_int_conn>, <&ts_rst_conn>;
+
+		interrupt-parent = <&tlmm>;
+		interrupts = <55 IRQ_TYPE_LEVEL_LOW>;
+
+		reset-gpios = <&tlmm 54 GPIO_ACTIVE_LOW>;
+
+		vcc33-supply = <&ts_avdd>;
+	};
+};
+
+&ap_sar_sensor_i2c {
+	status = "okay";
+};
+
+&ap_sar_sensor0 {
+	status = "okay";
+};
+
+&ap_sar_sensor1 {
+	status = "okay";
+};
+
+&mdss_edp {
+	status = "okay";
+};
+
+&mdss_edp_phy {
+	status = "okay";
+};
+
+/* For nvme */
+&pcie1 {
+	status = "okay";
+};
+
+/* For nvme */
+&pcie1_phy {
+	status = "okay";
+};
+
+&pwmleds {
+	status = "okay";
+};
+
+/* For eMMC */
+&sdhc_1 {
+	status = "okay";
+};
+
+/* PINCTRL - ADDITIONS TO NODES IN PARENT DEVICE TREE FILES */
+
+&ts_rst_conn {
+	bias-disable;
+};
+
+/* PINCTRL - BOARD-SPECIFIC */
+
+/*
+ * Methodology for gpio-line-names:
+ * - If a pin goes to herobrine board and is named it gets that name.
+ * - If a pin goes to herobrine board and is not named, it gets no name.
+ * - If a pin is totally internal to Qcard then it gets Qcard name.
+ * - If a pin is not hooked up on Qcard, it gets no name.
+ */
+
+&pm8350c_gpios {
+	gpio-line-names = "FLASH_STROBE_1",		/* 1 */
+			  "AP_SUSPEND",
+			  "PM8008_1_RST_N",
+			  "",
+			  "",
+			  "",
+			  "PMIC_EDP_BL_EN",
+			  "PMIC_EDP_BL_PWM",
+			  "";
+};
+
+&tlmm {
+	gpio-line-names = "AP_TP_I2C_SDA",		/* 0 */
+			  "AP_TP_I2C_SCL",
+			  "SSD_RST_L",
+			  "PE_WAKE_ODL",
+			  "AP_SAR_SDA",
+			  "AP_SAR_SCL",
+			  "PRB_SC_GPIO_6",
+			  "TP_INT_ODL",
+			  "HP_I2C_SDA",
+			  "HP_I2C_SCL",
+
+			  "GNSS_L1_EN",			/* 10 */
+			  "GNSS_L5_EN",
+			  "SPI_AP_MOSI",
+			  "SPI_AP_MISO",
+			  "SPI_AP_CLK",
+			  "SPI_AP_CS0_L",
+			  /*
+			   * AP_FLASH_WP is crossystem ABI. Schematics
+			   * call it BIOS_FLASH_WP_OD.
+			   */
+			  "AP_FLASH_WP",
+			  "",
+			  "AP_EC_INT_L",
+			  "",
+
+			  "UF_CAM_RST_L",		/* 20 */
+			  "WF_CAM_RST_L",
+			  "UART_AP_TX_DBG_RX",
+			  "UART_DBG_TX_AP_RX",
+			  "",
+			  "PM8008_IRQ_1",
+			  "HOST2WLAN_SOL",
+			  "WLAN2HOST_SOL",
+			  "MOS_BT_UART_CTS",
+			  "MOS_BT_UART_RFR",
+
+			  "MOS_BT_UART_TX",		/* 30 */
+			  "MOS_BT_UART_RX",
+			  "PRB_SC_GPIO_32",
+			  "HUB_RST_L",
+			  "",
+			  "",
+			  "AP_SPI_FP_MISO",
+			  "AP_SPI_FP_MOSI",
+			  "AP_SPI_FP_CLK",
+			  "AP_SPI_FP_CS_L",
+
+			  "AP_EC_SPI_MISO",		/* 40 */
+			  "AP_EC_SPI_MOSI",
+			  "AP_EC_SPI_CLK",
+			  "AP_EC_SPI_CS_L",
+			  "LCM_RST_L",
+			  "EARLY_EUD_N",
+			  "",
+			  "DP_HOT_PLUG_DET",
+			  "IO_BRD_MLB_ID0",
+			  "IO_BRD_MLB_ID1",
+
+			  "IO_BRD_MLB_ID2",		/* 50 */
+			  "SSD_EN",
+			  "TS_I2C_SDA_CONN",
+			  "TS_I2C_CLK_CONN",
+			  "TS_RST_CONN",
+			  "TS_INT_CONN",
+			  "AP_I2C_TPM_SDA",
+			  "AP_I2C_TPM_SCL",
+			  "PRB_SC_GPIO_58",
+			  "PRB_SC_GPIO_59",
+
+			  "EDP_HOT_PLUG_DET_N",		/* 60 */
+			  "FP_TO_AP_IRQ_L",
+			  "",
+			  "AMP_EN",
+			  "CAM0_MCLK_GPIO_64",
+			  "CAM1_MCLK_GPIO_65",
+			  "WF_CAM_MCLK",
+			  "PRB_SC_GPIO_67",
+			  "FPMCU_BOOT0",
+			  "UF_CAM_SDA",
+
+			  "UF_CAM_SCL",			/* 70 */
+			  "",
+			  "",
+			  "WF_CAM_SDA",
+			  "WF_CAM_SCL",
+			  "",
+			  "",
+			  "EN_FP_RAILS",
+			  "FP_RST_L",
+			  "PCIE1_CLKREQ_ODL",
+
+			  "EN_PP3300_DX_EDP",		/* 80 */
+			  "SC_GPIO_81",
+			  "FORCED_USB_BOOT",
+			  "WCD_RESET_N",
+			  "MOS_WLAN_EN",
+			  "MOS_BT_EN",
+			  "MOS_SW_CTRL",
+			  "MOS_PCIE0_RST",
+			  "MOS_PCIE0_CLKREQ_N",
+			  "MOS_PCIE0_WAKE_N",
+
+			  "MOS_LAA_AS_EN",		/* 90 */
+			  "SD_CD_ODL",
+			  "",
+			  "",
+			  "MOS_BT_WLAN_SLIMBUS_CLK",
+			  "MOS_BT_WLAN_SLIMBUS_DAT0",
+			  "HP_MCLK",
+			  "HP_BCLK",
+			  "HP_DOUT",
+			  "HP_DIN",
+
+			  "HP_LRCLK",			/* 100 */
+			  "HP_IRQ",
+			  "",
+			  "",
+			  "GSC_AP_INT_ODL",
+			  "EN_PP3300_CODEC",
+			  "AMP_BCLK",
+			  "AMP_DIN",
+			  "AMP_LRCLK",
+			  "UIM1_DATA_GPIO_109",
+
+			  "UIM1_CLK_GPIO_110",		/* 110 */
+			  "UIM1_RESET_GPIO_111",
+			  "PRB_SC_GPIO_112",
+			  "UIM0_DATA",
+			  "UIM0_CLK",
+			  "UIM0_RST",
+			  "UIM0_PRESENT_ODL",
+			  "SDM_RFFE0_CLK",
+			  "SDM_RFFE0_DATA",
+			  "WF_CAM_EN",
+
+			  "FASTBOOT_SEL_0",		/* 120 */
+			  "SC_GPIO_121",
+			  "FASTBOOT_SEL_1",
+			  "SC_GPIO_123",
+			  "FASTBOOT_SEL_2",
+			  "SM_RFFE4_CLK_GRFC_8",
+			  "SM_RFFE4_DATA_GRFC_9",
+			  "WLAN_COEX_UART1_RX",
+			  "WLAN_COEX_UART1_TX",
+			  "PRB_SC_GPIO_129",
+
+			  "LCM_ID0",			/* 130 */
+			  "LCM_ID1",
+			  "",
+			  "SDR_QLINK_REQ",
+			  "SDR_QLINK_EN",
+			  "QLINK0_WMSS_RESET_N",
+			  "SMR526_QLINK1_REQ",
+			  "SMR526_QLINK1_EN",
+			  "SMR526_QLINK1_WMSS_RESET_N",
+			  "PRB_SC_GPIO_139",
+
+			  "SAR1_IRQ_ODL",		/* 140 */
+			  "SAR0_IRQ_ODL",
+			  "PRB_SC_GPIO_142",
+			  "",
+			  "WCD_SWR_TX_CLK",
+			  "WCD_SWR_TX_DATA0",
+			  "WCD_SWR_TX_DATA1",
+			  "WCD_SWR_RX_CLK",
+			  "WCD_SWR_RX_DATA0",
+			  "WCD_SWR_RX_DATA1",
+
+			  "DMIC01_CLK",			/* 150 */
+			  "DMIC01_DATA",
+			  "DMIC23_CLK",
+			  "DMIC23_DATA",
+			  "",
+			  "",
+			  "EC_IN_RW_ODL",
+			  "HUB_EN",
+			  "WCD_SWR_TX_DATA2",
+			  "",
+
+			  "",				/* 160 */
+			  "",
+			  "",
+			  "",
+			  "",
+			  "",
+			  "",
+			  "",
+			  "",
+			  "",
+
+			  "",				/* 170 */
+			  "MOS_BLE_UART_TX",
+			  "MOS_BLE_UART_RX",
+			  "",
+			  "",
+			  "";
+};
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-herobrine-r1.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-herobrine-r1.dts
index c1647a85a371..c1a671968725 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-herobrine-herobrine-r1.dts
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-herobrine-r1.dts
@@ -8,6 +8,7 @@
 /dts-v1/;
 
 #include "sc7280-herobrine.dtsi"
+#include "sc7280-herobrine-lte-sku.dtsi"
 
 / {
 	model = "Google Herobrine (rev1+)";
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi
new file mode 100644
index 000000000000..a92eeccd2b2a
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-lte-sku.dtsi
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Google Herobrine dts fragment for LTE SKUs
+ *
+ * Copyright 2022 Google LLC.
+ */
+/* Modem setup is different on Chrome setups than typical Qualcomm setup */
+
+&remoteproc_mpss {
+	compatible = "qcom,sc7280-mss-pil";
+	iommus = <&apps_smmu 0x124 0x0>, <&apps_smmu 0x488 0x7>;
+	interconnects = <&mc_virt MASTER_LLCC 0 &mc_virt SLAVE_EBI1 0>;
+	memory-region = <&mba_mem>, <&mpss_mem>;
+	firmware-name = "qcom/sc7280-herobrine/modem/mba.mbn",
+			"qcom/sc7280-herobrine/modem/qdsp6sw.mbn";
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r0.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r0.dts
index 2cacafd8faa8..73e24cc55a09 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r0.dts
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r0.dts
@@ -7,327 +7,10 @@
 
 /dts-v1/;
 
-#include "sc7280-herobrine.dtsi"
+#include "sc7280-herobrine-villager.dtsi"
+#include "sc7280-herobrine-lte-sku.dtsi"
 
 / {
-	model = "Google Villager (rev0+)";
-	compatible = "google,villager", "qcom,sc7280";
-};
-
-/*
- * ADDITIONS TO FIXED REGULATORS DEFINED IN PARENT DEVICE TREE FILES
- *
- * Sort order matches the order in the parent files (parents before children).
- */
-
-&pp3300_codec {
-	status = "okay";
-};
-
-/* ADDITIONS TO NODES DEFINED IN PARENT DEVICE TREE FILES */
-
-ap_tp_i2c: &i2c0 {
-	status = "okay";
-	clock-frequency = <400000>;
-
-	trackpad: trackpad@2c {
-		compatible = "hid-over-i2c";
-		reg = <0x2c>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&tp_int_odl>;
-
-		interrupt-parent = <&tlmm>;
-		interrupts = <7 IRQ_TYPE_EDGE_FALLING>;
-
-		hid-descr-addr = <0x20>;
-		vcc-supply = <&pp3300_z1>;
-
-		wakeup-source;
-	};
-};
-
-ts_i2c: &i2c13 {
-	status = "okay";
-	clock-frequency = <400000>;
-
-	ap_ts: touchscreen@10 {
-		compatible = "elan,ekth6915";
-		reg = <0x10>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&ts_int_conn>, <&ts_rst_conn>;
-
-		interrupt-parent = <&tlmm>;
-		interrupts = <55 IRQ_TYPE_LEVEL_LOW>;
-
-		reset-gpios = <&tlmm 54 GPIO_ACTIVE_LOW>;
-
-		vcc33-supply = <&ts_avdd>;
-	};
-};
-
-&ap_sar_sensor_i2c {
-	status = "okay";
-};
-
-&ap_sar_sensor0 {
-	status = "okay";
-};
-
-&ap_sar_sensor1 {
-	status = "okay";
-};
-
-&mdss_edp {
-	status = "okay";
-};
-
-&mdss_edp_phy {
-	status = "okay";
-};
-
-/* For nvme */
-&pcie1 {
-	status = "okay";
-};
-
-/* For nvme */
-&pcie1_phy {
-	status = "okay";
-};
-
-&pwmleds {
-	status = "okay";
-};
-
-/* For eMMC */
-&sdhc_1 {
-	status = "okay";
-};
-
-/* PINCTRL - ADDITIONS TO NODES IN PARENT DEVICE TREE FILES */
-
-&ts_rst_conn {
-	bias-disable;
-};
-
-/* PINCTRL - BOARD-SPECIFIC */
-
-/*
- * Methodology for gpio-line-names:
- * - If a pin goes to herobrine board and is named it gets that name.
- * - If a pin goes to herobrine board and is not named, it gets no name.
- * - If a pin is totally internal to Qcard then it gets Qcard name.
- * - If a pin is not hooked up on Qcard, it gets no name.
- */
-
-&pm8350c_gpios {
-	gpio-line-names = "FLASH_STROBE_1",		/* 1 */
-			  "AP_SUSPEND",
-			  "PM8008_1_RST_N",
-			  "",
-			  "",
-			  "",
-			  "PMIC_EDP_BL_EN",
-			  "PMIC_EDP_BL_PWM",
-			  "";
-};
-
-&tlmm {
-	gpio-line-names = "AP_TP_I2C_SDA",		/* 0 */
-			  "AP_TP_I2C_SCL",
-			  "SSD_RST_L",
-			  "PE_WAKE_ODL",
-			  "AP_SAR_SDA",
-			  "AP_SAR_SCL",
-			  "PRB_SC_GPIO_6",
-			  "TP_INT_ODL",
-			  "HP_I2C_SDA",
-			  "HP_I2C_SCL",
-
-			  "GNSS_L1_EN",			/* 10 */
-			  "GNSS_L5_EN",
-			  "SPI_AP_MOSI",
-			  "SPI_AP_MISO",
-			  "SPI_AP_CLK",
-			  "SPI_AP_CS0_L",
-			  /*
-			   * AP_FLASH_WP is crossystem ABI. Schematics
-			   * call it BIOS_FLASH_WP_OD.
-			   */
-			  "AP_FLASH_WP",
-			  "",
-			  "AP_EC_INT_L",
-			  "",
-
-			  "UF_CAM_RST_L",		/* 20 */
-			  "WF_CAM_RST_L",
-			  "UART_AP_TX_DBG_RX",
-			  "UART_DBG_TX_AP_RX",
-			  "",
-			  "PM8008_IRQ_1",
-			  "HOST2WLAN_SOL",
-			  "WLAN2HOST_SOL",
-			  "MOS_BT_UART_CTS",
-			  "MOS_BT_UART_RFR",
-
-			  "MOS_BT_UART_TX",		/* 30 */
-			  "MOS_BT_UART_RX",
-			  "PRB_SC_GPIO_32",
-			  "HUB_RST_L",
-			  "",
-			  "",
-			  "AP_SPI_FP_MISO",
-			  "AP_SPI_FP_MOSI",
-			  "AP_SPI_FP_CLK",
-			  "AP_SPI_FP_CS_L",
-
-			  "AP_EC_SPI_MISO",		/* 40 */
-			  "AP_EC_SPI_MOSI",
-			  "AP_EC_SPI_CLK",
-			  "AP_EC_SPI_CS_L",
-			  "LCM_RST_L",
-			  "EARLY_EUD_N",
-			  "",
-			  "DP_HOT_PLUG_DET",
-			  "IO_BRD_MLB_ID0",
-			  "IO_BRD_MLB_ID1",
-
-			  "IO_BRD_MLB_ID2",		/* 50 */
-			  "SSD_EN",
-			  "TS_I2C_SDA_CONN",
-			  "TS_I2C_CLK_CONN",
-			  "TS_RST_CONN",
-			  "TS_INT_CONN",
-			  "AP_I2C_TPM_SDA",
-			  "AP_I2C_TPM_SCL",
-			  "PRB_SC_GPIO_58",
-			  "PRB_SC_GPIO_59",
-
-			  "EDP_HOT_PLUG_DET_N",		/* 60 */
-			  "FP_TO_AP_IRQ_L",
-			  "",
-			  "AMP_EN",
-			  "CAM0_MCLK_GPIO_64",
-			  "CAM1_MCLK_GPIO_65",
-			  "WF_CAM_MCLK",
-			  "PRB_SC_GPIO_67",
-			  "FPMCU_BOOT0",
-			  "UF_CAM_SDA",
-
-			  "UF_CAM_SCL",			/* 70 */
-			  "",
-			  "",
-			  "WF_CAM_SDA",
-			  "WF_CAM_SCL",
-			  "",
-			  "",
-			  "EN_FP_RAILS",
-			  "FP_RST_L",
-			  "PCIE1_CLKREQ_ODL",
-
-			  "EN_PP3300_DX_EDP",		/* 80 */
-			  "SC_GPIO_81",
-			  "FORCED_USB_BOOT",
-			  "WCD_RESET_N",
-			  "MOS_WLAN_EN",
-			  "MOS_BT_EN",
-			  "MOS_SW_CTRL",
-			  "MOS_PCIE0_RST",
-			  "MOS_PCIE0_CLKREQ_N",
-			  "MOS_PCIE0_WAKE_N",
-
-			  "MOS_LAA_AS_EN",		/* 90 */
-			  "SD_CD_ODL",
-			  "",
-			  "",
-			  "MOS_BT_WLAN_SLIMBUS_CLK",
-			  "MOS_BT_WLAN_SLIMBUS_DAT0",
-			  "HP_MCLK",
-			  "HP_BCLK",
-			  "HP_DOUT",
-			  "HP_DIN",
-
-			  "HP_LRCLK",			/* 100 */
-			  "HP_IRQ",
-			  "",
-			  "",
-			  "GSC_AP_INT_ODL",
-			  "EN_PP3300_CODEC",
-			  "AMP_BCLK",
-			  "AMP_DIN",
-			  "AMP_LRCLK",
-			  "UIM1_DATA_GPIO_109",
-
-			  "UIM1_CLK_GPIO_110",		/* 110 */
-			  "UIM1_RESET_GPIO_111",
-			  "PRB_SC_GPIO_112",
-			  "UIM0_DATA",
-			  "UIM0_CLK",
-			  "UIM0_RST",
-			  "UIM0_PRESENT_ODL",
-			  "SDM_RFFE0_CLK",
-			  "SDM_RFFE0_DATA",
-			  "WF_CAM_EN",
-
-			  "FASTBOOT_SEL_0",		/* 120 */
-			  "SC_GPIO_121",
-			  "FASTBOOT_SEL_1",
-			  "SC_GPIO_123",
-			  "FASTBOOT_SEL_2",
-			  "SM_RFFE4_CLK_GRFC_8",
-			  "SM_RFFE4_DATA_GRFC_9",
-			  "WLAN_COEX_UART1_RX",
-			  "WLAN_COEX_UART1_TX",
-			  "PRB_SC_GPIO_129",
-
-			  "LCM_ID0",			/* 130 */
-			  "LCM_ID1",
-			  "",
-			  "SDR_QLINK_REQ",
-			  "SDR_QLINK_EN",
-			  "QLINK0_WMSS_RESET_N",
-			  "SMR526_QLINK1_REQ",
-			  "SMR526_QLINK1_EN",
-			  "SMR526_QLINK1_WMSS_RESET_N",
-			  "PRB_SC_GPIO_139",
-
-			  "SAR1_IRQ_ODL",		/* 140 */
-			  "SAR0_IRQ_ODL",
-			  "PRB_SC_GPIO_142",
-			  "",
-			  "WCD_SWR_TX_CLK",
-			  "WCD_SWR_TX_DATA0",
-			  "WCD_SWR_TX_DATA1",
-			  "WCD_SWR_RX_CLK",
-			  "WCD_SWR_RX_DATA0",
-			  "WCD_SWR_RX_DATA1",
-
-			  "DMIC01_CLK",			/* 150 */
-			  "DMIC01_DATA",
-			  "DMIC23_CLK",
-			  "DMIC23_DATA",
-			  "",
-			  "",
-			  "EC_IN_RW_ODL",
-			  "HUB_EN",
-			  "WCD_SWR_TX_DATA2",
-			  "",
-
-			  "",				/* 160 */
-			  "",
-			  "",
-			  "",
-			  "",
-			  "",
-			  "",
-			  "",
-			  "",
-			  "",
-
-			  "",				/* 170 */
-			  "MOS_BLE_UART_TX",
-			  "MOS_BLE_UART_RX",
-			  "",
-			  "",
-			  "";
+	model = "Google Villager (rev0)";
+	compatible = "google,villager-rev0", "qcom,sc7280";
 };
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1-lte.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1-lte.dts
new file mode 100644
index 000000000000..f1017809e5da
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1-lte.dts
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Google Villager board device tree source
+ *
+ * Copyright 2022 Google LLC.
+ */
+
+#include "sc7280-herobrine-villager-r1.dts"
+#include "sc7280-herobrine-lte-sku.dtsi"
+
+/ {
+	model = "Google Villager (rev1+) with LTE";
+	compatible = "google,villager-sku512", "qcom,sc7280";
+};
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1.dts b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1.dts
new file mode 100644
index 000000000000..cfc648726930
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager-r1.dts
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Google Villager board device tree source
+ *
+ * Copyright 2022 Google LLC.
+ */
+
+/dts-v1/;
+
+#include "sc7280-herobrine-villager.dtsi"
+#include "sc7280-herobrine-audio-wcd9385.dtsi"
+
+/ {
+	model = "Google Villager (rev1+)";
+	compatible = "google,villager", "qcom,sc7280";
+};
+
+&lpass_va_macro {
+	vdd-micb-supply = <&pp1800_l2c>;
+};
+
+&sound {
+	audio-routing =
+			"IN1_HPHL", "HPHL_OUT",
+			"IN2_HPHR", "HPHR_OUT",
+			"AMIC1", "MIC BIAS1",
+			"AMIC2", "MIC BIAS2",
+			"VA DMIC0", "vdd-micb",
+			"VA DMIC1", "vdd-micb",
+			"VA DMIC2", "vdd-micb",
+			"VA DMIC3", "vdd-micb",
+			"TX SWR_ADC0", "ADC1_OUTPUT",
+			"TX SWR_ADC1", "ADC2_OUTPUT",
+			"TX SWR_ADC2", "ADC3_OUTPUT",
+			"TX SWR_DMIC0", "DMIC1_OUTPUT",
+			"TX SWR_DMIC1", "DMIC2_OUTPUT",
+			"TX SWR_DMIC2", "DMIC3_OUTPUT",
+			"TX SWR_DMIC3", "DMIC4_OUTPUT",
+			"TX SWR_DMIC4", "DMIC5_OUTPUT",
+			"TX SWR_DMIC5", "DMIC6_OUTPUT",
+			"TX SWR_DMIC6", "DMIC7_OUTPUT",
+			"TX SWR_DMIC7", "DMIC8_OUTPUT";
+};
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager.dtsi
new file mode 100644
index 000000000000..4566722bf4dd
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine-villager.dtsi
@@ -0,0 +1,326 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Google Villager board device tree source
+ *
+ * Copyright 2022 Google LLC.
+ */
+
+#include "sc7280-herobrine.dtsi"
+
+/*
+ * ADDITIONS TO FIXED REGULATORS DEFINED IN PARENT DEVICE TREE FILES
+ *
+ * Sort order matches the order in the parent files (parents before children).
+ */
+
+&pp3300_codec {
+	status = "okay";
+};
+
+/* ADDITIONS TO NODES DEFINED IN PARENT DEVICE TREE FILES */
+
+ap_tp_i2c: &i2c0 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	trackpad: trackpad@2c {
+		compatible = "hid-over-i2c";
+		reg = <0x2c>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&tp_int_odl>;
+
+		interrupt-parent = <&tlmm>;
+		interrupts = <7 IRQ_TYPE_EDGE_FALLING>;
+
+		hid-descr-addr = <0x20>;
+		vcc-supply = <&pp3300_z1>;
+
+		wakeup-source;
+	};
+};
+
+ts_i2c: &i2c13 {
+	status = "okay";
+	clock-frequency = <400000>;
+
+	ap_ts: touchscreen@10 {
+		compatible = "elan,ekth6915";
+		reg = <0x10>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&ts_int_conn>, <&ts_rst_conn>;
+
+		interrupt-parent = <&tlmm>;
+		interrupts = <55 IRQ_TYPE_LEVEL_LOW>;
+
+		reset-gpios = <&tlmm 54 GPIO_ACTIVE_LOW>;
+
+		vcc33-supply = <&ts_avdd>;
+	};
+};
+
+&ap_sar_sensor_i2c {
+	status = "okay";
+};
+
+&ap_sar_sensor0 {
+	status = "okay";
+};
+
+&ap_sar_sensor1 {
+	status = "okay";
+};
+
+&mdss_edp {
+	status = "okay";
+};
+
+&mdss_edp_phy {
+	status = "okay";
+};
+
+/* For nvme */
+&pcie1 {
+	status = "okay";
+};
+
+/* For nvme */
+&pcie1_phy {
+	status = "okay";
+};
+
+&pwmleds {
+	status = "okay";
+};
+
+/* For eMMC */
+&sdhc_1 {
+	status = "okay";
+};
+
+/* PINCTRL - ADDITIONS TO NODES IN PARENT DEVICE TREE FILES */
+
+&ts_rst_conn {
+	bias-disable;
+};
+
+/* PINCTRL - BOARD-SPECIFIC */
+
+/*
+ * Methodology for gpio-line-names:
+ * - If a pin goes to herobrine board and is named it gets that name.
+ * - If a pin goes to herobrine board and is not named, it gets no name.
+ * - If a pin is totally internal to Qcard then it gets Qcard name.
+ * - If a pin is not hooked up on Qcard, it gets no name.
+ */
+
+&pm8350c_gpios {
+	gpio-line-names = "FLASH_STROBE_1",		/* 1 */
+			  "AP_SUSPEND",
+			  "PM8008_1_RST_N",
+			  "",
+			  "",
+			  "",
+			  "PMIC_EDP_BL_EN",
+			  "PMIC_EDP_BL_PWM",
+			  "";
+};
+
+&tlmm {
+	gpio-line-names = "AP_TP_I2C_SDA",		/* 0 */
+			  "AP_TP_I2C_SCL",
+			  "SSD_RST_L",
+			  "PE_WAKE_ODL",
+			  "AP_SAR_SDA",
+			  "AP_SAR_SCL",
+			  "PRB_SC_GPIO_6",
+			  "TP_INT_ODL",
+			  "HP_I2C_SDA",
+			  "HP_I2C_SCL",
+
+			  "GNSS_L1_EN",			/* 10 */
+			  "GNSS_L5_EN",
+			  "SPI_AP_MOSI",
+			  "SPI_AP_MISO",
+			  "SPI_AP_CLK",
+			  "SPI_AP_CS0_L",
+			  /*
+			   * AP_FLASH_WP is crossystem ABI. Schematics
+			   * call it BIOS_FLASH_WP_OD.
+			   */
+			  "AP_FLASH_WP",
+			  "",
+			  "AP_EC_INT_L",
+			  "",
+
+			  "UF_CAM_RST_L",		/* 20 */
+			  "WF_CAM_RST_L",
+			  "UART_AP_TX_DBG_RX",
+			  "UART_DBG_TX_AP_RX",
+			  "",
+			  "PM8008_IRQ_1",
+			  "HOST2WLAN_SOL",
+			  "WLAN2HOST_SOL",
+			  "MOS_BT_UART_CTS",
+			  "MOS_BT_UART_RFR",
+
+			  "MOS_BT_UART_TX",		/* 30 */
+			  "MOS_BT_UART_RX",
+			  "PRB_SC_GPIO_32",
+			  "HUB_RST_L",
+			  "",
+			  "",
+			  "AP_SPI_FP_MISO",
+			  "AP_SPI_FP_MOSI",
+			  "AP_SPI_FP_CLK",
+			  "AP_SPI_FP_CS_L",
+
+			  "AP_EC_SPI_MISO",		/* 40 */
+			  "AP_EC_SPI_MOSI",
+			  "AP_EC_SPI_CLK",
+			  "AP_EC_SPI_CS_L",
+			  "LCM_RST_L",
+			  "EARLY_EUD_N",
+			  "",
+			  "DP_HOT_PLUG_DET",
+			  "IO_BRD_MLB_ID0",
+			  "IO_BRD_MLB_ID1",
+
+			  "IO_BRD_MLB_ID2",		/* 50 */
+			  "SSD_EN",
+			  "TS_I2C_SDA_CONN",
+			  "TS_I2C_CLK_CONN",
+			  "TS_RST_CONN",
+			  "TS_INT_CONN",
+			  "AP_I2C_TPM_SDA",
+			  "AP_I2C_TPM_SCL",
+			  "PRB_SC_GPIO_58",
+			  "PRB_SC_GPIO_59",
+
+			  "EDP_HOT_PLUG_DET_N",		/* 60 */
+			  "FP_TO_AP_IRQ_L",
+			  "",
+			  "AMP_EN",
+			  "CAM0_MCLK_GPIO_64",
+			  "CAM1_MCLK_GPIO_65",
+			  "WF_CAM_MCLK",
+			  "PRB_SC_GPIO_67",
+			  "FPMCU_BOOT0",
+			  "UF_CAM_SDA",
+
+			  "UF_CAM_SCL",			/* 70 */
+			  "",
+			  "",
+			  "WF_CAM_SDA",
+			  "WF_CAM_SCL",
+			  "",
+			  "",
+			  "EN_FP_RAILS",
+			  "FP_RST_L",
+			  "PCIE1_CLKREQ_ODL",
+
+			  "EN_PP3300_DX_EDP",		/* 80 */
+			  "SC_GPIO_81",
+			  "FORCED_USB_BOOT",
+			  "WCD_RESET_N",
+			  "MOS_WLAN_EN",
+			  "MOS_BT_EN",
+			  "MOS_SW_CTRL",
+			  "MOS_PCIE0_RST",
+			  "MOS_PCIE0_CLKREQ_N",
+			  "MOS_PCIE0_WAKE_N",
+
+			  "MOS_LAA_AS_EN",		/* 90 */
+			  "SD_CD_ODL",
+			  "",
+			  "",
+			  "MOS_BT_WLAN_SLIMBUS_CLK",
+			  "MOS_BT_WLAN_SLIMBUS_DAT0",
+			  "HP_MCLK",
+			  "HP_BCLK",
+			  "HP_DOUT",
+			  "HP_DIN",
+
+			  "HP_LRCLK",			/* 100 */
+			  "HP_IRQ",
+			  "",
+			  "",
+			  "GSC_AP_INT_ODL",
+			  "EN_PP3300_CODEC",
+			  "AMP_BCLK",
+			  "AMP_DIN",
+			  "AMP_LRCLK",
+			  "UIM1_DATA_GPIO_109",
+
+			  "UIM1_CLK_GPIO_110",		/* 110 */
+			  "UIM1_RESET_GPIO_111",
+			  "PRB_SC_GPIO_112",
+			  "UIM0_DATA",
+			  "UIM0_CLK",
+			  "UIM0_RST",
+			  "UIM0_PRESENT_ODL",
+			  "SDM_RFFE0_CLK",
+			  "SDM_RFFE0_DATA",
+			  "WF_CAM_EN",
+
+			  "FASTBOOT_SEL_0",		/* 120 */
+			  "SC_GPIO_121",
+			  "FASTBOOT_SEL_1",
+			  "SC_GPIO_123",
+			  "FASTBOOT_SEL_2",
+			  "SM_RFFE4_CLK_GRFC_8",
+			  "SM_RFFE4_DATA_GRFC_9",
+			  "WLAN_COEX_UART1_RX",
+			  "WLAN_COEX_UART1_TX",
+			  "PRB_SC_GPIO_129",
+
+			  "LCM_ID0",			/* 130 */
+			  "LCM_ID1",
+			  "",
+			  "SDR_QLINK_REQ",
+			  "SDR_QLINK_EN",
+			  "QLINK0_WMSS_RESET_N",
+			  "SMR526_QLINK1_REQ",
+			  "SMR526_QLINK1_EN",
+			  "SMR526_QLINK1_WMSS_RESET_N",
+			  "PRB_SC_GPIO_139",
+
+			  "SAR1_IRQ_ODL",		/* 140 */
+			  "SAR0_IRQ_ODL",
+			  "PRB_SC_GPIO_142",
+			  "",
+			  "WCD_SWR_TX_CLK",
+			  "WCD_SWR_TX_DATA0",
+			  "WCD_SWR_TX_DATA1",
+			  "WCD_SWR_RX_CLK",
+			  "WCD_SWR_RX_DATA0",
+			  "WCD_SWR_RX_DATA1",
+
+			  "DMIC01_CLK",			/* 150 */
+			  "DMIC01_DATA",
+			  "DMIC23_CLK",
+			  "DMIC23_DATA",
+			  "",
+			  "",
+			  "EC_IN_RW_ODL",
+			  "HUB_EN",
+			  "WCD_SWR_TX_DATA2",
+			  "",
+
+			  "",				/* 160 */
+			  "",
+			  "",
+			  "",
+			  "",
+			  "",
+			  "",
+			  "",
+			  "",
+			  "",
+
+			  "",				/* 170 */
+			  "MOS_BLE_UART_TX",
+			  "MOS_BLE_UART_RX",
+			  "",
+			  "",
+			  "";
+};
diff --git a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi
index 3f8996c00b05..c11e37160f34 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280-herobrine.dtsi
@@ -144,8 +144,8 @@
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
 
+		/* The BIOS leaves this regulator on */
 		regulator-boot-on;
-		regulator-always-on;
 
 		gpio = <&tlmm 157 GPIO_ACTIVE_HIGH>;
 		enable-active-high;
@@ -296,6 +296,14 @@
 
 	/* BOARD-SPECIFIC TOP LEVEL NODES */
 
+	max98360a: audio-codec-0 {
+		compatible = "maxim,max98360a";
+		pinctrl-names = "default";
+		pinctrl-0 = <&amp_en>;
+		sdmode-gpios = <&tlmm 63 GPIO_ACTIVE_HIGH>;
+		#sound-dai-cells = <0>;
+	};
+
 	pwmleds: pwmleds {
 		compatible = "pwm-leds";
 		status = "disabled";
@@ -446,7 +454,7 @@ ap_i2c_tpm: &i2c14 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie1_clkreq_n>, <&ssd_rst_l>, <&pe_wake_odl>;
 
-	perst-gpio = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
 	vddpe-3v3-supply = <&pp3300_ssd>;
 };
 
@@ -596,25 +604,32 @@ ap_ec_spi: &spi10 {
 
 &usb_1_dwc3 {
 	dr_mode = "host";
-};
 
-&usb_1_hsphy {
-	status = "okay";
-};
+	#address-cells = <1>;
+	#size-cells = <0>;
 
-&usb_1_qmpphy {
-	status = "okay";
-};
+	/* 2.x hub on port 1 */
+	usb_hub_2_x: hub@1 {
+		compatible = "usbbda,5411";
+		reg = <1>;
+		vdd-supply = <&pp3300_hub>;
+		peer-hub = <&usb_hub_3_x>;
+	};
 
-&usb_2 {
-	status = "okay";
+	/* 3.x hub on port 2 */
+	usb_hub_3_x: hub@2 {
+		compatible = "usbbda,411";
+		reg = <2>;
+		vdd-supply = <&pp3300_hub>;
+		peer-hub = <&usb_hub_2_x>;
+	};
 };
 
-&usb_2_dwc3 {
-	dr_mode = "host";
+&usb_1_hsphy {
+	status = "okay";
 };
 
-&usb_2_hsphy {
+&usb_1_qmpphy {
 	status = "okay";
 };
 
@@ -729,27 +744,27 @@ ap_ec_spi: &spi10 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&bios_flash_wp_od>;
 
-	amp_en: amp-en {
+	amp_en: amp-en-pins {
 		pins = "gpio63";
 		function = "gpio";
 		bias-disable;
 		drive-strength = <2>;
 	};
 
-	ap_ec_int_l: ap-ec-int-l {
+	ap_ec_int_l: ap-ec-int-l-pins {
 		pins = "gpio18";
 		function = "gpio";
 		bias-pull-up;
 	};
 
-	bios_flash_wp_od: bios-flash-wp-od {
+	bios_flash_wp_od: bios-flash-wp-od-pins {
 		pins = "gpio16";
 		function = "gpio";
 		/* Has external pull */
 		bias-disable;
 	};
 
-	en_fp_rails: en-fp-rails {
+	en_fp_rails: en-fp-rails-pins {
 		pins = "gpio77";
 		function = "gpio";
 		bias-disable;
@@ -757,60 +772,60 @@ ap_ec_spi: &spi10 {
 		output-high;
 	};
 
-	en_pp3300_codec: en-pp3300-codec {
+	en_pp3300_codec: en-pp3300-codec-pins {
 		pins = "gpio105";
 		function = "gpio";
 		bias-disable;
 		drive-strength = <2>;
 	};
 
-	en_pp3300_dx_edp: en-pp3300-dx-edp {
+	en_pp3300_dx_edp: en-pp3300-dx-edp-pins {
 		pins = "gpio80";
 		function = "gpio";
 		bias-disable;
 		drive-strength = <2>;
 	};
 
-	fp_rst_l: fp-rst-l {
+	fp_rst_l: fp-rst-l-pins {
 		pins = "gpio78";
 		function = "gpio";
 		bias-disable;
 		drive-strength = <2>;
 	};
 
-	fp_to_ap_irq_l: fp-to-ap-irq-l {
+	fp_to_ap_irq_l: fp-to-ap-irq-l-pins {
 		pins = "gpio61";
 		function = "gpio";
 		/* Has external pullup */
 		bias-disable;
 	};
 
-	fpmcu_boot0: fpmcu-boot0 {
+	fpmcu_boot0: fpmcu-boot0-pins {
 		pins = "gpio68";
 		function = "gpio";
 		bias-disable;
 	};
 
-	gsc_ap_int_odl: gsc-ap-int-odl {
+	gsc_ap_int_odl: gsc-ap-int-odl-pins {
 		pins = "gpio104";
 		function = "gpio";
 		bias-pull-up;
 	};
 
-	hp_irq: hp-irq {
+	hp_irq: hp-irq-pins {
 		pins = "gpio101";
 		function = "gpio";
 		bias-pull-up;
 	};
 
-	hub_en: hub-en {
+	hub_en: hub-en-pins {
 		pins = "gpio157";
 		function = "gpio";
 		bias-disable;
 		drive-strength = <2>;
 	};
 
-	pe_wake_odl: pe-wake-odl {
+	pe_wake_odl: pe-wake-odl-pins {
 		pins = "gpio3";
 		function = "gpio";
 		/* Has external pull */
@@ -819,45 +834,45 @@ ap_ec_spi: &spi10 {
 	};
 
 	/* For ap_spi_fp */
-	qup_spi9_cs_gpio_init_high: qup-spi9-cs-gpio-init-high {
+	qup_spi9_cs_gpio_init_high: qup-spi9-cs-gpio-init-high-pins {
 		pins = "gpio39";
 		function = "gpio";
 		output-high;
 	};
 
 	/* For ap_ec_spi */
-	qup_spi10_cs_gpio_init_high: qup-spi10-cs-gpio-init-high {
+	qup_spi10_cs_gpio_init_high: qup-spi10-cs-gpio-init-high-pins {
 		pins = "gpio43";
 		function = "gpio";
 		output-high;
 	};
 
-	sar0_irq_odl: sar0-irq-odl {
+	sar0_irq_odl: sar0-irq-odl-pins {
 		pins = "gpio141";
 		function = "gpio";
 		bias-pull-up;
 	};
 
-	sar1_irq_odl: sar1-irq-odl {
+	sar1_irq_odl: sar1-irq-odl-pins {
 		pins = "gpio140";
 		function = "gpio";
 		bias-pull-up;
 	};
 
-	sd_cd_odl: sd-cd-odl {
+	sd_cd_odl: sd-cd-odl-pins {
 		pins = "gpio91";
 		function = "gpio";
 		bias-pull-up;
 	};
 
-	ssd_en: ssd-en {
+	ssd_en: ssd-en-pins {
 		pins = "gpio51";
 		function = "gpio";
 		bias-disable;
 		drive-strength = <2>;
 	};
 
-	ssd_rst_l: ssd-rst-l {
+	ssd_rst_l: ssd-rst-l-pins {
 		pins = "gpio2";
 		function = "gpio";
 		bias-disable;
@@ -865,14 +880,14 @@ ap_ec_spi: &spi10 {
 		output-low;
 	};
 
-	tp_int_odl: tp-int-odl {
+	tp_int_odl: tp-int-odl-pins {
 		pins = "gpio7";
 		function = "gpio";
 		/* Has external pullup */
 		bias-disable;
 	};
 
-	wf_cam_en: wf-cam-en {
+	wf_cam_en: wf-cam-en-pins {
 		pins = "gpio119";
 		function = "gpio";
 		/* Has external pulldown */
diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp-ec-h1.dtsi b/arch/arm64/boot/dts/qcom/sc7280-idp-ec-h1.dtsi
index a7c346aa3b02..7f5143e9bb80 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-idp-ec-h1.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280-idp-ec-h1.dtsi
@@ -79,26 +79,26 @@ ap_h1_spi: &spi14 {
 };
 
 &tlmm {
-	ap_ec_int_l: ap-ec-int-l {
+	ap_ec_int_l: ap-ec-int-l-pins {
 		pins = "gpio18";
 		function = "gpio";
 		input-enable;
 		bias-pull-up;
 	};
 
-	h1_ap_int_odl: h1-ap-int-odl {
+	h1_ap_int_odl: h1-ap-int-odl-pins {
 		pins = "gpio104";
 		function = "gpio";
 		input-enable;
 		bias-pull-up;
 	};
 
-	qup_spi10_cs_gpio_init_high: qup-spi10-cs-gpio-init-high {
+	qup_spi10_cs_gpio_init_high: qup-spi10-cs-gpio-init-high-pins {
 		pins = "gpio43";
 		output-high;
 	};
 
-	qup_spi14_cs_gpio_init_high: qup-spi14-cs-gpio-init-high {
+	qup_spi14_cs_gpio_init_high: qup-spi14-cs-gpio-init-high-pins {
 		pins = "gpio59";
 		output-high;
 	};
diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp.dts b/arch/arm64/boot/dts/qcom/sc7280-idp.dts
index 6d3ff80582ae..7559164cdda0 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-idp.dts
+++ b/arch/arm64/boot/dts/qcom/sc7280-idp.dts
@@ -10,6 +10,7 @@
 #include <dt-bindings/iio/qcom,spmi-adc7-pmr735a.h>
 #include "sc7280-idp.dtsi"
 #include "pmr735a.dtsi"
+#include "sc7280-herobrine-lte-sku.dtsi"
 
 / {
 	model = "Qualcomm Technologies, Inc. sc7280 IDP SKU1 platform";
@@ -78,7 +79,7 @@
 };
 
 &pmk8350_vadc {
-	pmr735a_die_temp {
+	pmr735a-die-temp@403 {
 		reg = <PMR735A_ADC7_DIE_TEMP>;
 		label = "pmr735a_die_temp";
 		qcom,pre-scaling = <1 1>;
diff --git a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi
index a74e0b730db6..cd432a2856a7 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280-idp.dtsi
@@ -20,6 +20,42 @@
 		serial1 = &uart7;
 	};
 
+	max98360a: audio-codec-0 {
+		compatible = "maxim,max98360a";
+		pinctrl-names = "default";
+		pinctrl-0 = <&amp_en>;
+		sdmode-gpios = <&tlmm 63 GPIO_ACTIVE_HIGH>;
+		#sound-dai-cells = <0>;
+	};
+
+	wcd9385: audio-codec-1 {
+		compatible = "qcom,wcd9385-codec";
+		pinctrl-names = "default", "sleep";
+		pinctrl-0 = <&wcd_reset_n>;
+		pinctrl-1 = <&wcd_reset_n_sleep>;
+
+		reset-gpios = <&tlmm 83 GPIO_ACTIVE_HIGH>;
+
+		qcom,rx-device = <&wcd_rx>;
+		qcom,tx-device = <&wcd_tx>;
+
+		vdd-rxtx-supply = <&vreg_l18b_1p8>;
+		vdd-io-supply = <&vreg_l18b_1p8>;
+		vdd-buck-supply = <&vreg_l17b_1p8>;
+		vdd-mic-bias-supply = <&vreg_bob>;
+
+		qcom,micbias1-microvolt = <1800000>;
+		qcom,micbias2-microvolt = <1800000>;
+		qcom,micbias3-microvolt = <1800000>;
+		qcom,micbias4-microvolt = <1800000>;
+
+		qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000 500000
+							  500000 500000 500000>;
+		qcom,mbhc-headset-vthreshold-microvolt = <1700000>;
+		qcom,mbhc-headphone-vthreshold-microvolt = <50000>;
+		#sound-dai-cells = <1>;
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 		label = "gpio-keys";
@@ -49,6 +85,104 @@
 		pinctrl-names = "default";
 		pinctrl-0 = <&nvme_pwren>;
 	};
+
+	sound: sound {
+		compatible = "google,sc7280-herobrine";
+		model = "sc7280-wcd938x-max98360a-1mic";
+
+		audio-routing =
+			"IN1_HPHL", "HPHL_OUT",
+			"IN2_HPHR", "HPHR_OUT",
+			"AMIC1", "MIC BIAS1",
+			"AMIC2", "MIC BIAS2",
+			"VA DMIC0", "MIC BIAS3",
+			"VA DMIC1", "MIC BIAS3",
+			"VA DMIC2", "MIC BIAS1",
+			"VA DMIC3", "MIC BIAS1",
+			"TX SWR_ADC0", "ADC1_OUTPUT",
+			"TX SWR_ADC1", "ADC2_OUTPUT",
+			"TX SWR_ADC2", "ADC3_OUTPUT",
+			"TX SWR_DMIC0", "DMIC1_OUTPUT",
+			"TX SWR_DMIC1", "DMIC2_OUTPUT",
+			"TX SWR_DMIC2", "DMIC3_OUTPUT",
+			"TX SWR_DMIC3", "DMIC4_OUTPUT",
+			"TX SWR_DMIC4", "DMIC5_OUTPUT",
+			"TX SWR_DMIC5", "DMIC6_OUTPUT",
+			"TX SWR_DMIC6", "DMIC7_OUTPUT",
+			"TX SWR_DMIC7", "DMIC8_OUTPUT";
+
+		qcom,msm-mbhc-hphl-swh = <1>;
+		qcom,msm-mbhc-gnd-swh = <1>;
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+		#sound-dai-cells = <0>;
+
+		dai-link@0 {
+			link-name = "MAX98360A";
+			reg = <0>;
+
+			cpu {
+				sound-dai = <&lpass_cpu MI2S_SECONDARY>;
+			};
+
+			codec {
+				sound-dai = <&max98360a>;
+			};
+		};
+
+		dai-link@1 {
+			link-name = "DisplayPort";
+			reg = <1>;
+
+			cpu {
+				sound-dai = <&lpass_cpu LPASS_DP_RX>;
+			};
+
+			codec {
+				sound-dai = <&mdss_dp>;
+			};
+		};
+
+		dai-link@2 {
+			link-name = "WCD9385 Playback";
+			reg = <2>;
+
+			cpu {
+				sound-dai = <&lpass_cpu LPASS_CDC_DMA_RX0>;
+			};
+
+			codec {
+				sound-dai = <&wcd9385 0>, <&swr0 0>, <&lpass_rx_macro 0>;
+			};
+		};
+
+		dai-link@3 {
+			link-name = "WCD9385 Capture";
+			reg = <3>;
+
+			cpu {
+				sound-dai = <&lpass_cpu LPASS_CDC_DMA_TX3>;
+			};
+
+			codec {
+				sound-dai = <&wcd9385 1>, <&swr1 0>, <&lpass_tx_macro 0>;
+			};
+		};
+
+		dai-link@4 {
+			link-name = "DMIC";
+			reg = <4>;
+
+			cpu {
+				sound-dai = <&lpass_cpu LPASS_CDC_DMA_VA_TX0>;
+			};
+
+			codec {
+				sound-dai = <&lpass_va_macro 0>;
+			};
+		};
+	};
 };
 
 &apps_rsc {
@@ -246,9 +380,50 @@
 	modem-init;
 };
 
+&lpass_cpu {
+	status = "okay";
+
+	pinctrl-names = "default";
+	pinctrl-0 = <&mi2s1_data0>, <&mi2s1_sclk>, <&mi2s1_ws>;
+
+	dai-link@1 {
+		reg = <MI2S_SECONDARY>;
+		qcom,playback-sd-lines = <0>;
+	};
+
+	dai-link@5 {
+		reg = <LPASS_DP_RX>;
+	};
+
+	dai-link@6 {
+		reg = <LPASS_CDC_DMA_RX0>;
+	};
+
+	dai-link@19 {
+		reg = <LPASS_CDC_DMA_TX3>;
+	};
+
+	dai-link@25 {
+		reg = <LPASS_CDC_DMA_VA_TX0>;
+	};
+};
+
+&lpass_rx_macro {
+	status = "okay";
+};
+
+&lpass_tx_macro {
+	status = "okay";
+};
+
+&lpass_va_macro {
+	status = "okay";
+	vdd-micb-supply = <&vreg_bob>;
+};
+
 &pcie1 {
 	status = "okay";
-	perst-gpio = <&tlmm 2 GPIO_ACTIVE_LOW>;
+	perst-gpios = <&tlmm 2 GPIO_ACTIVE_LOW>;
 
 	vddpe-3v3-supply = <&nvme_3v3_regulator>;
 
@@ -264,7 +439,7 @@
 };
 
 &pmk8350_vadc {
-	pmk8350_die_temp {
+	pmk8350-die-temp@3 {
 		reg = <PMK8350_ADC7_DIE_TEMP>;
 		label = "pmk8350_die_temp";
 		qcom,pre-scaling = <1 1>;
@@ -306,6 +481,28 @@
 	cd-gpios = <&tlmm 91 GPIO_ACTIVE_LOW>;
 };
 
+&swr0 {
+	status = "okay";
+
+	wcd_rx: codec@0,4 {
+		compatible = "sdw20217010d00";
+		reg = <0 4>;
+		#sound-dai-cells = <1>;
+		qcom,rx-port-mapping = <1 2 3 4 5>;
+	};
+};
+
+&swr1 {
+	status = "okay";
+
+	wcd_tx: codec@0,3 {
+		compatible = "sdw20217010d00";
+		reg = <0 3>;
+		#sound-dai-cells = <1>;
+		qcom,tx-port-mapping = <1 2 3 4>;
+	};
+};
+
 &uart5 {
 	compatible = "qcom,geni-debug-uart";
 	status = "okay";
@@ -550,18 +747,24 @@
 };
 
 &tlmm {
-	bt_en: bt-en {
+	amp_en: amp-en {
+		pins = "gpio63";
+		bias-pull-down;
+		drive-strength = <2>;
+	};
+
+	bt_en: bt-en-pins {
 		pins = "gpio85";
 		function = "gpio";
 		output-low;
 		bias-disable;
 	};
 
-	nvme_pwren: nvme-pwren {
+	nvme_pwren: nvme-pwren-pins {
 		function = "gpio";
 	};
 
-	pcie1_reset_n: pcie1-reset-n {
+	pcie1_reset_n: pcie1-reset-n-pins {
 		pins = "gpio2";
 		function = "gpio";
 
@@ -570,7 +773,7 @@
 		bias-disable;
 	};
 
-	pcie1_wake_n: pcie1-wake-n {
+	pcie1_wake_n: pcie1-wake-n-pins {
 		pins = "gpio3";
 		function = "gpio";
 
@@ -578,7 +781,7 @@
 		bias-pull-up;
 	};
 
-	qup_uart7_sleep_cts: qup-uart7-sleep-cts {
+	qup_uart7_sleep_cts: qup-uart7-sleep-cts-pins {
 		pins = "gpio28";
 		function = "gpio";
 		/*
@@ -591,7 +794,7 @@
 		bias-bus-hold;
 	};
 
-	qup_uart7_sleep_rts: qup-uart7-sleep-rts {
+	qup_uart7_sleep_rts: qup-uart7-sleep-rts-pins {
 		pins = "gpio29";
 		function = "gpio";
 		/*
@@ -603,7 +806,7 @@
 		bias-pull-down;
 	};
 
-	qup_uart7_sleep_tx: qup-uart7-sleep-tx {
+	qup_uart7_sleep_tx: qup-uart7-sleep-tx-pins {
 		pins = "gpio30";
 		function = "gpio";
 		/*
@@ -613,7 +816,7 @@
 		bias-pull-up;
 	};
 
-	qup_uart7_sleep_rx: qup-uart7-sleep-rx {
+	qup_uart7_sleep_rx: qup-uart7-sleep-rx-pins {
 		pins = "gpio31";
 		function = "gpio";
 		/*
@@ -624,15 +827,28 @@
 		bias-pull-up;
 	};
 
-	sd_cd: sd-cd {
+	sd_cd: sd-cd-pins {
 		pins = "gpio91";
 		function = "gpio";
 		bias-pull-up;
 	};
 
-	sw_ctrl: sw-ctrl {
+	sw_ctrl: sw-ctrl-pins {
 		pins = "gpio86";
 		function = "gpio";
 		bias-pull-down;
 	};
+
+	wcd_reset_n: wcd-reset-n {
+		pins = "gpio83";
+		function = "gpio";
+		drive-strength = <8>;
+	};
+
+	wcd_reset_n_sleep: wcd-reset-n-sleep {
+		pins = "gpio83";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
 };
diff --git a/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi b/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi
index 7adf31bb9827..4b8c676b0bb1 100644
--- a/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280-qcard.dtsi
@@ -28,6 +28,38 @@
 		bluetooth0 = &bluetooth;
 		serial0 = &uart5;
 		serial1 = &uart7;
+		wifi0 = &wifi;
+	};
+
+	wcd9385: audio-codec-1 {
+		compatible = "qcom,wcd9385-codec";
+		pinctrl-names = "default", "sleep";
+		pinctrl-0 = <&wcd_reset_n>, <&us_euro_hs_sel>;
+		pinctrl-1 = <&wcd_reset_n_sleep>, <&us_euro_hs_sel>;
+
+		reset-gpios = <&tlmm 83 GPIO_ACTIVE_HIGH>;
+		us-euro-gpios = <&tlmm 81 GPIO_ACTIVE_HIGH>;
+
+		qcom,rx-device = <&wcd_rx>;
+		qcom,tx-device = <&wcd_tx>;
+
+		vdd-rxtx-supply = <&vreg_l18b_1p8>;
+		vdd-io-supply = <&vreg_l18b_1p8>;
+		vdd-buck-supply = <&vreg_l17b_1p8>;
+		vdd-mic-bias-supply = <&vreg_bob>;
+
+		qcom,micbias1-microvolt = <1800000>;
+		qcom,micbias2-microvolt = <1800000>;
+		qcom,micbias3-microvolt = <1800000>;
+		qcom,micbias4-microvolt = <1800000>;
+
+		qcom,mbhc-buttons-vthreshold-microvolt = <75000 150000 237000 500000 500000
+							  500000 500000 500000>;
+		qcom,mbhc-headset-vthreshold-microvolt = <1700000>;
+		qcom,mbhc-headphone-vthreshold-microvolt = <50000>;
+		#sound-dai-cells = <1>;
+
+		status = "disabled";
 	};
 
 	pm8350c_pwm_backlight: backlight {
@@ -309,6 +341,10 @@
 	modem-init;
 };
 
+&lpass_va_macro {
+	vdd-micb-supply = <&vreg_bob>;
+};
+
 /* NOTE: Not all Qcards have eDP connector stuffed */
 &mdss_edp {
 	aux-bus {
@@ -378,6 +414,24 @@
 	no-sdio;
 };
 
+&swr0 {
+	wcd_rx: codec@0,4 {
+		compatible = "sdw20217010d00";
+		reg = <0 4>;
+		#sound-dai-cells = <1>;
+		qcom,rx-port-mapping = <1 2 3 4 5>;
+	};
+};
+
+&swr1 {
+	wcd_tx: codec@0,3 {
+		compatible = "sdw20217010d00";
+		reg = <0 3>;
+		#sound-dai-cells = <1>;
+		qcom,tx-port-mapping = <1 2 3 4>;
+	};
+};
+
 uart_dbg: &uart5 {
 	compatible = "qcom,geni-debug-uart";
 	status = "okay";
@@ -541,7 +595,7 @@ mos_bt_uart: &uart7 {
 };
 
 &tlmm {
-	mos_bt_en: mos-bt-en {
+	mos_bt_en: mos-bt-en-pins {
 		pins = "gpio85";
 		function = "gpio";
 		drive-strength = <2>;
@@ -549,7 +603,7 @@ mos_bt_uart: &uart7 {
 	};
 
 	/* For mos_bt_uart */
-	qup_uart7_sleep_cts: qup-uart7-sleep-cts {
+	qup_uart7_sleep_cts: qup-uart7-sleep-cts-pins {
 		pins = "gpio28";
 		function = "gpio";
 		/*
@@ -563,7 +617,7 @@ mos_bt_uart: &uart7 {
 	};
 
 	/* For mos_bt_uart */
-	qup_uart7_sleep_rts: qup-uart7-sleep-rts {
+	qup_uart7_sleep_rts: qup-uart7-sleep-rts-pins {
 		pins = "gpio29";
 		function = "gpio";
 		/*
@@ -576,7 +630,7 @@ mos_bt_uart: &uart7 {
 	};
 
 	/* For mos_bt_uart */
-	qup_uart7_sleep_rx: qup-uart7-sleep-rx {
+	qup_uart7_sleep_rx: qup-uart7-sleep-rx-pins {
 		pins = "gpio31";
 		function = "gpio";
 		/*
@@ -588,7 +642,7 @@ mos_bt_uart: &uart7 {
 	};
 
 	/* For mos_bt_uart */
-	qup_uart7_sleep_tx: qup-uart7-sleep-tx {
+	qup_uart7_sleep_tx: qup-uart7-sleep-tx-pins {
 		pins = "gpio30";
 		function = "gpio";
 		/*
@@ -598,15 +652,35 @@ mos_bt_uart: &uart7 {
 		bias-pull-up;
 	};
 
-	ts_int_conn: ts-int-conn {
+	ts_int_conn: ts-int-conn-pins {
 		pins = "gpio55";
 		function = "gpio";
 		bias-pull-up;
 	};
 
-	ts_rst_conn: ts-rst-conn {
+	ts_rst_conn: ts-rst-conn-pins {
 		pins = "gpio54";
 		function = "gpio";
 		drive-strength = <2>;
 	};
+
+	us_euro_hs_sel: us-euro-hs-sel {
+		pins = "gpio81";
+		function = "gpio";
+		bias-pull-down;
+		drive-strength = <2>;
+	};
+
+	wcd_reset_n: wcd-reset-n {
+		pins = "gpio83";
+		function = "gpio";
+		drive-strength = <8>;
+	};
+
+	wcd_reset_n_sleep: wcd-reset-n-sleep {
+		pins = "gpio83";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
 };
diff --git a/arch/arm64/boot/dts/qcom/sc7280.dtsi b/arch/arm64/boot/dts/qcom/sc7280.dtsi
index 13d7f267b289..212580316d3e 100644
--- a/arch/arm64/boot/dts/qcom/sc7280.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7280.dtsi
@@ -22,6 +22,7 @@
 #include <dt-bindings/reset/qcom,sdm845-aoss.h>
 #include <dt-bindings/reset/qcom,sdm845-pdc.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
+#include <dt-bindings/sound/qcom,lpass.h>
 #include <dt-bindings/thermal/thermal.h>
 
 / {
@@ -828,6 +829,7 @@
 			#clock-cells = <1>;
 			#reset-cells = <1>;
 			#power-domain-cells = <1>;
+			power-domains = <&rpmhpd SC7280_CX>;
 		};
 
 		ipcc: mailbox@408000 {
@@ -2043,7 +2045,9 @@
 				 <&gcc GCC_PCIE_1_SLV_AXI_CLK>,
 				 <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>,
 				 <&gcc GCC_AGGRE_NOC_PCIE_TBU_CLK>,
-				 <&gcc GCC_DDRSS_PCIE_SF_CLK>;
+				 <&gcc GCC_DDRSS_PCIE_SF_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_CENTER_SF_AXI_CLK>,
+				 <&gcc GCC_AGGRE_NOC_PCIE_1_AXI_CLK>;
 
 			clock-names = "pipe",
 				      "pipe_mux",
@@ -2055,7 +2059,9 @@
 				      "bus_slave",
 				      "slave_q2a",
 				      "tbu",
-				      "ddrss_sf_tbu";
+				      "ddrss_sf_tbu",
+				      "aggre0",
+				      "aggre1";
 
 			assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>;
 			assigned-clock-rates = <19200000>;
@@ -2155,12 +2161,17 @@
 		};
 
 		tcsr_mutex: hwlock@1f40000 {
-			compatible = "qcom,tcsr-mutex", "syscon";
-			reg = <0 0x01f40000 0 0x40000>;
+			compatible = "qcom,tcsr-mutex";
+			reg = <0 0x01f40000 0 0x20000>;
 			#hwlock-cells = <1>;
 		};
 
-		tcsr: syscon@1fc0000 {
+		tcsr_1: syscon@1f60000 {
+			compatible = "qcom,sc7280-tcsr", "syscon";
+			reg = <0 0x01f60000 0 0x20000>;
+		};
+
+		tcsr_2: syscon@1fc0000 {
 			compatible = "qcom,sc7280-tcsr", "syscon";
 			reg = <0 0x01fc0000 0 0x30000>;
 		};
@@ -2168,14 +2179,121 @@
 		lpasscc: lpasscc@3000000 {
 			compatible = "qcom,sc7280-lpasscc";
 			reg = <0 0x03000000 0 0x40>,
-			      <0 0x03c04000 0 0x4>,
-			      <0 0x03389000 0 0x24>;
-			reg-names = "qdsp6ss", "top_cc", "cc";
+			      <0 0x03c04000 0 0x4>;
+			reg-names = "qdsp6ss", "top_cc";
 			clocks = <&gcc GCC_CFG_NOC_LPASS_CLK>;
 			clock-names = "iface";
 			#clock-cells = <1>;
 		};
 
+		lpass_rx_macro: codec@3200000 {
+			compatible = "qcom,sc7280-lpass-rx-macro";
+			reg = <0 0x03200000 0 0x1000>;
+
+			pinctrl-names = "default";
+			pinctrl-0 = <&lpass_rx_swr_clk>, <&lpass_rx_swr_data>;
+
+			clocks = <&lpass_aon LPASS_AON_CC_TX_MCLK_CLK>,
+				 <&lpass_aon LPASS_AON_CC_TX_MCLK_2X_CLK>,
+				 <&lpass_va_macro>;
+			clock-names = "mclk", "npl", "fsgen";
+
+			power-domains = <&lpass_hm LPASS_CORE_CC_LPASS_CORE_HM_GDSC>,
+					<&lpass_aon LPASS_AON_CC_LPASS_AUDIO_HM_GDSC>;
+			power-domain-names = "macro", "dcodec";
+
+			#clock-cells = <0>;
+			#sound-dai-cells = <1>;
+
+			status = "disabled";
+		};
+
+		swr0: soundwire@3210000 {
+			compatible = "qcom,soundwire-v1.6.0";
+			reg = <0 0x03210000 0 0x2000>;
+
+			interrupts = <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&lpass_rx_macro>;
+			clock-names = "iface";
+
+			qcom,din-ports = <0>;
+			qcom,dout-ports = <5>;
+
+			resets = <&lpass_audiocc LPASS_AUDIO_SWR_RX_CGCR>;
+			reset-names = "swr_audio_cgcr";
+
+			qcom,ports-word-length =	/bits/ 8 <0x01 0x07 0x04 0xff 0xff>;
+			qcom,ports-sinterval-low =	/bits/ 8 <0x03 0x3f 0x1f 0x03 0x03>;
+			qcom,ports-offset1 =		/bits/ 8 <0x00 0x00 0x0b 0x01 0x01>;
+			qcom,ports-offset2 =		/bits/ 8 <0x00 0x00 0x0b 0x00 0x00>;
+			qcom,ports-lane-control =	/bits/ 8 <0x01 0x00 0x00 0x00 0x00>;
+			qcom,ports-block-pack-mode =	/bits/ 8 <0xff 0x00 0x01 0xff 0xff>;
+			qcom,ports-hstart =		/bits/ 8 <0xff 0x03 0xff 0xff 0xff>;
+			qcom,ports-hstop =		/bits/ 8 <0xff 0x06 0xff 0xff 0xff>;
+			qcom,ports-block-group-count =	/bits/ 8 <0xff 0xff 0xff 0xff 0x00>;
+
+			#sound-dai-cells = <1>;
+			#address-cells = <2>;
+			#size-cells = <0>;
+
+			status = "disabled";
+		};
+
+		lpass_tx_macro: codec@3220000 {
+			compatible = "qcom,sc7280-lpass-tx-macro";
+			reg = <0 0x03220000 0 0x1000>;
+
+			pinctrl-names = "default";
+			pinctrl-0 = <&lpass_tx_swr_clk>, <&lpass_tx_swr_data>;
+
+			clocks = <&lpass_aon LPASS_AON_CC_TX_MCLK_CLK>,
+				 <&lpass_aon LPASS_AON_CC_TX_MCLK_2X_CLK>,
+				 <&lpass_va_macro>;
+			clock-names = "mclk", "npl", "fsgen";
+
+			power-domains = <&lpass_hm LPASS_CORE_CC_LPASS_CORE_HM_GDSC>,
+					<&lpass_aon LPASS_AON_CC_LPASS_AUDIO_HM_GDSC>;
+			power-domain-names = "macro", "dcodec";
+
+			#clock-cells = <0>;
+			#sound-dai-cells = <1>;
+
+			status = "disabled";
+		};
+
+		swr1: soundwire@3230000 {
+			compatible = "qcom,soundwire-v1.6.0";
+			reg = <0 0x03230000 0 0x2000>;
+
+			interrupts-extended = <&intc GIC_SPI 496 IRQ_TYPE_LEVEL_HIGH>,
+					      <&pdc 130 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&lpass_tx_macro>;
+			clock-names = "iface";
+
+			qcom,din-ports = <3>;
+			qcom,dout-ports = <0>;
+
+			resets = <&lpass_audiocc LPASS_AUDIO_SWR_TX_CGCR>;
+			reset-names = "swr_audio_cgcr";
+
+			qcom,ports-sinterval-low =	/bits/ 8 <0x01 0x03 0x03>;
+			qcom,ports-offset1 =		/bits/ 8 <0x01 0x00 0x02>;
+			qcom,ports-offset2 =		/bits/ 8 <0x00 0x00 0x00>;
+			qcom,ports-hstart =		/bits/ 8 <0xff 0xff 0xff>;
+			qcom,ports-hstop =		/bits/ 8 <0xff 0xff 0xff>;
+			qcom,ports-word-length =	/bits/ 8 <0xff 0x00 0xff>;
+			qcom,ports-block-pack-mode =	/bits/ 8 <0xff 0xff 0xff>;
+			qcom,ports-block-group-count =	/bits/ 8 <0xff 0xff 0xff>;
+			qcom,ports-lane-control =	/bits/ 8 <0x00 0x01 0x00>;
+			qcom,port-offset = <1>;
+
+			#sound-dai-cells = <1>;
+			#address-cells = <2>;
+			#size-cells = <0>;
+
+			status = "disabled";
+		};
+
 		lpass_audiocc: clock-controller@3300000 {
 			compatible = "qcom,sc7280-lpassaudiocc";
 			reg = <0 0x03300000 0 0x30000>;
@@ -2185,6 +2303,27 @@
 			power-domains = <&lpass_aon LPASS_AON_CC_LPASS_AUDIO_HM_GDSC>;
 			#clock-cells = <1>;
 			#power-domain-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		lpass_va_macro: codec@3370000 {
+			compatible = "qcom,sc7280-lpass-va-macro";
+			reg = <0 0x03370000 0 0x1000>;
+
+			pinctrl-names = "default";
+			pinctrl-0 = <&lpass_dmic01_clk>, <&lpass_dmic01_data>;
+
+			clocks = <&lpass_aon LPASS_AON_CC_TX_MCLK_CLK>;
+			clock-names = "mclk";
+
+			power-domains = <&lpass_hm LPASS_CORE_CC_LPASS_CORE_HM_GDSC>,
+					<&lpass_aon LPASS_AON_CC_LPASS_AUDIO_HM_GDSC>;
+			power-domain-names = "macro", "dcodec";
+
+			#clock-cells = <0>;
+			#sound-dai-cells = <1>;
+
+			status = "disabled";
 		};
 
 		lpass_aon: clock-controller@3380000 {
@@ -2192,13 +2331,13 @@
 			reg = <0 0x03380000 0 0x30000>;
 			clocks = <&rpmhcc RPMH_CXO_CLK>,
 			       <&rpmhcc RPMH_CXO_CLK_A>,
-			       <&lpasscore LPASS_CORE_CC_CORE_CLK>;
+			       <&lpass_core LPASS_CORE_CC_CORE_CLK>;
 			clock-names = "bi_tcxo", "bi_tcxo_ao", "iface";
 			#clock-cells = <1>;
 			#power-domain-cells = <1>;
 		};
 
-		lpasscore: clock-controller@3900000 {
+		lpass_core: clock-controller@3900000 {
 			compatible = "qcom,sc7280-lpasscorecc";
 			reg = <0 0x03900000 0 0x50000>;
 			clocks = <&rpmhcc RPMH_CXO_CLK>;
@@ -2208,6 +2347,67 @@
 			#power-domain-cells = <1>;
 		};
 
+		lpass_cpu: audio@3987000 {
+			compatible = "qcom,sc7280-lpass-cpu";
+
+			reg = <0 0x03987000 0 0x68000>,
+			      <0 0x03b00000 0 0x29000>,
+			      <0 0x03260000 0 0xc000>,
+			      <0 0x03280000 0 0x29000>,
+			      <0 0x03340000 0 0x29000>,
+			      <0 0x0336c000 0 0x3000>;
+			reg-names = "lpass-hdmiif",
+				    "lpass-lpaif",
+				    "lpass-rxtx-cdc-dma-lpm",
+				    "lpass-rxtx-lpaif",
+				    "lpass-va-lpaif",
+				    "lpass-va-cdc-dma-lpm";
+
+			iommus = <&apps_smmu 0x1820 0>,
+				 <&apps_smmu 0x1821 0>,
+				 <&apps_smmu 0x1832 0>;
+
+			power-domains =	<&rpmhpd SC7280_LCX>;
+			power-domain-names = "lcx";
+			required-opps = <&rpmhpd_opp_nom>;
+
+			clocks = <&lpass_aon LPASS_AON_CC_AUDIO_HM_H_CLK>,
+				 <&lpass_core LPASS_CORE_CC_EXT_MCLK0_CLK>,
+				 <&lpass_core LPASS_CORE_CC_SYSNOC_MPORT_CORE_CLK>,
+				 <&lpass_core LPASS_CORE_CC_EXT_IF0_IBIT_CLK>,
+				 <&lpass_core LPASS_CORE_CC_EXT_IF1_IBIT_CLK>,
+				 <&lpass_audiocc LPASS_AUDIO_CC_CODEC_MEM_CLK>,
+				 <&lpass_audiocc LPASS_AUDIO_CC_CODEC_MEM0_CLK>,
+				 <&lpass_audiocc LPASS_AUDIO_CC_CODEC_MEM1_CLK>,
+				 <&lpass_audiocc LPASS_AUDIO_CC_CODEC_MEM2_CLK>,
+				 <&lpass_aon LPASS_AON_CC_VA_MEM0_CLK>;
+			clock-names = "aon_cc_audio_hm_h",
+				      "audio_cc_ext_mclk0",
+				      "core_cc_sysnoc_mport_core",
+				      "core_cc_ext_if0_ibit",
+				      "core_cc_ext_if1_ibit",
+				      "audio_cc_codec_mem",
+				      "audio_cc_codec_mem0",
+				      "audio_cc_codec_mem1",
+				      "audio_cc_codec_mem2",
+				      "aon_cc_va_mem0";
+
+			#sound-dai-cells = <1>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			interrupts = <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 268 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "lpass-irq-lpaif",
+					  "lpass-irq-hdmi",
+					  "lpass-irq-vaif",
+					  "lpass-irq-rxtxif";
+
+			status = "disabled";
+		};
+
 		lpass_hm: clock-controller@3c00000 {
 			compatible = "qcom,sc7280-lpasshm";
 			reg = <0 0x3c00000 0 0x28>;
@@ -2352,11 +2552,19 @@
 					opp-supported-hw = <0x03>;
 				};
 
-				opp-550000000 {
+				/* Only applicable for SKUs which has 550Mhz as Fmax */
+				opp-550000000-0 {
+					opp-hz = /bits/ 64 <550000000>;
+					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
+					opp-peak-kBps = <8368000>;
+					opp-supported-hw = <0x01>;
+				};
+
+				opp-550000000-1 {
 					opp-hz = /bits/ 64 <550000000>;
 					opp-level = <RPMH_REGULATOR_LEVEL_SVS_L1>;
 					opp-peak-kBps = <6832000>;
-					opp-supported-hw = <0x03>;
+					opp-supported-hw = <0x02>;
 				};
 
 				opp-608000000 {
@@ -2522,9 +2730,9 @@
 				 <&pdc_reset PDC_MODEM_SYNC_RESET>;
 			reset-names = "mss_restart", "pdc_reset";
 
-			qcom,halt-regs = <&tcsr_mutex 0x23000 0x25000 0x28000 0x33000>;
-			qcom,ext-regs = <&tcsr 0x10000 0x10004 &tcsr_mutex 0x26004 0x26008>;
-			qcom,qaccept-regs = <&tcsr_mutex 0x23030 0x23040 0x23020>;
+			qcom,halt-regs = <&tcsr_1 0x3000 0x5000 0x8000 0x13000>;
+			qcom,ext-regs = <&tcsr_2 0x10000 0x10004 &tcsr_1 0x6004 0x6008>;
+			qcom,qaccept-regs = <&tcsr_1 0x3030 0x3040 0x3020>;
 
 			status = "disabled";
 
@@ -3181,6 +3389,7 @@
 					  "dm_hs_phy_irq";
 
 			power-domains = <&gcc GCC_USB30_SEC_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
 
 			resets = <&gcc GCC_USB30_SEC_BCR>;
 
@@ -3259,7 +3468,7 @@
 				 <&pdc_reset PDC_WPSS_SYNC_RESET>;
 			reset-names = "restart", "pdc_sync";
 
-			qcom,halt-regs = <&tcsr_mutex 0x37000>;
+			qcom,halt-regs = <&tcsr_1 0x17000>;
 
 			status = "disabled";
 
@@ -3275,6 +3484,82 @@
 			};
 		};
 
+		pmu@9091000 {
+			compatible = "qcom,sc7280-llcc-bwmon";
+			reg = <0 0x9091000 0 0x1000>;
+
+			interrupts = <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>;
+
+			interconnects = <&mc_virt MASTER_LLCC 3 &mc_virt SLAVE_EBI1 3>;
+
+			operating-points-v2 = <&llcc_bwmon_opp_table>;
+
+			llcc_bwmon_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-0 {
+					opp-peak-kBps = <800000>;
+				};
+				opp-1 {
+					opp-peak-kBps = <1804000>;
+				};
+				opp-2 {
+					opp-peak-kBps = <2188000>;
+				};
+				opp-3 {
+					opp-peak-kBps = <3072000>;
+				};
+				opp-4 {
+					opp-peak-kBps = <4068000>;
+				};
+				opp-5 {
+					opp-peak-kBps = <6220000>;
+				};
+				opp-6 {
+					opp-peak-kBps = <6832000>;
+				};
+				opp-7 {
+					opp-peak-kBps = <8532000>;
+				};
+			};
+		};
+
+		pmu@90b6400 {
+			compatible = "qcom,sc7280-cpu-bwmon", "qcom,msm8998-bwmon";
+			reg = <0 0x090b6400 0 0x600>;
+
+			interrupts = <GIC_SPI 581 IRQ_TYPE_LEVEL_HIGH>;
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC 3 &gem_noc SLAVE_LLCC 3>;
+			operating-points-v2 = <&cpu_bwmon_opp_table>;
+
+			cpu_bwmon_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-0 {
+					opp-peak-kBps = <2400000>;
+				};
+				opp-1 {
+					opp-peak-kBps = <4800000>;
+				};
+				opp-2 {
+					opp-peak-kBps = <7456000>;
+				};
+				opp-3 {
+					opp-peak-kBps = <9600000>;
+				};
+				opp-4 {
+					opp-peak-kBps = <12896000>;
+				};
+				opp-5 {
+					opp-peak-kBps = <14928000>;
+				};
+				opp-6 {
+					opp-peak-kBps = <17056000>;
+				};
+			};
+		};
+
 		dc_noc: interconnect@90e0000 {
 			reg = <0 0x090e0000 0 0x5080>;
 			compatible = "qcom,sc7280-dc-noc";
@@ -3367,6 +3652,7 @@
 					  "ss_phy_irq";
 
 			power-domains = <&gcc GCC_USB30_PRIM_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
 
 			resets = <&gcc GCC_USB30_PRIM_BCR>;
 
@@ -3374,6 +3660,8 @@
 					<&gem_noc MASTER_APPSS_PROC 0 &cnoc2 SLAVE_USB3_0 0>;
 			interconnect-names = "usb-ddr", "apps-usb";
 
+			wakeup-source;
+
 			usb_1_dwc3: usb@a600000 {
 				compatible = "snps,dwc3";
 				reg = <0 0x0a600000 0 0xe000>;
@@ -3384,7 +3672,6 @@
 				phys = <&usb_1_hsphy>, <&usb_1_ssphy>;
 				phy-names = "usb2-phy", "usb3-phy";
 				maximum-speed = "super-speed";
-				wakeup-source;
 			};
 		};
 
@@ -3971,791 +4258,791 @@
 			gpio-ranges = <&tlmm 0 0 175>;
 			wakeup-parent = <&pdc>;
 
-			dp_hot_plug_det: dp-hot-plug-det {
+			dp_hot_plug_det: dp-hot-plug-det-pins {
 				pins = "gpio47";
 				function = "dp_hot";
 			};
 
-			edp_hot_plug_det: edp-hot-plug-det {
+			edp_hot_plug_det: edp-hot-plug-det-pins {
 				pins = "gpio60";
 				function = "edp_hot";
 			};
 
-			mi2s0_data0: mi2s0-data0 {
+			mi2s0_data0: mi2s0-data0-pins {
 				pins = "gpio98";
 				function = "mi2s0_data0";
 			};
 
-			mi2s0_data1: mi2s0-data1 {
+			mi2s0_data1: mi2s0-data1-pins {
 				pins = "gpio99";
 				function = "mi2s0_data1";
 			};
 
-			mi2s0_mclk: mi2s0-mclk {
+			mi2s0_mclk: mi2s0-mclk-pins {
 				pins = "gpio96";
 				function = "pri_mi2s";
 			};
 
-			mi2s0_sclk: mi2s0-sclk {
+			mi2s0_sclk: mi2s0-sclk-pins {
 				pins = "gpio97";
 				function = "mi2s0_sck";
 			};
 
-			mi2s0_ws: mi2s0-ws {
+			mi2s0_ws: mi2s0-ws-pins {
 				pins = "gpio100";
 				function = "mi2s0_ws";
 			};
 
-			mi2s1_data0: mi2s1-data0 {
+			mi2s1_data0: mi2s1-data0-pins {
 				pins = "gpio107";
 				function = "mi2s1_data0";
 			};
 
-			mi2s1_sclk: mi2s1-sclk {
+			mi2s1_sclk: mi2s1-sclk-pins {
 				pins = "gpio106";
 				function = "mi2s1_sck";
 			};
 
-			mi2s1_ws: mi2s1-ws {
+			mi2s1_ws: mi2s1-ws-pins {
 				pins = "gpio108";
 				function = "mi2s1_ws";
 			};
 
-			pcie1_clkreq_n: pcie1-clkreq-n {
+			pcie1_clkreq_n: pcie1-clkreq-n-pins {
 				pins = "gpio79";
 				function = "pcie1_clkreqn";
 			};
 
-			qspi_clk: qspi-clk {
+			qspi_clk: qspi-clk-pins {
 				pins = "gpio14";
 				function = "qspi_clk";
 			};
 
-			qspi_cs0: qspi-cs0 {
+			qspi_cs0: qspi-cs0-pins {
 				pins = "gpio15";
 				function = "qspi_cs";
 			};
 
-			qspi_cs1: qspi-cs1 {
+			qspi_cs1: qspi-cs1-pins {
 				pins = "gpio19";
 				function = "qspi_cs";
 			};
 
-			qspi_data01: qspi-data01 {
+			qspi_data01: qspi-data01-pins {
 				pins = "gpio12", "gpio13";
 				function = "qspi_data";
 			};
 
-			qspi_data12: qspi-data12 {
+			qspi_data12: qspi-data12-pins {
 				pins = "gpio16", "gpio17";
 				function = "qspi_data";
 			};
 
-			qup_i2c0_data_clk: qup-i2c0-data-clk {
+			qup_i2c0_data_clk: qup-i2c0-data-clk-pins {
 				pins = "gpio0", "gpio1";
 				function = "qup00";
 			};
 
-			qup_i2c1_data_clk: qup-i2c1-data-clk {
+			qup_i2c1_data_clk: qup-i2c1-data-clk-pins {
 				pins = "gpio4", "gpio5";
 				function = "qup01";
 			};
 
-			qup_i2c2_data_clk: qup-i2c2-data-clk {
+			qup_i2c2_data_clk: qup-i2c2-data-clk-pins {
 				pins = "gpio8", "gpio9";
 				function = "qup02";
 			};
 
-			qup_i2c3_data_clk: qup-i2c3-data-clk {
+			qup_i2c3_data_clk: qup-i2c3-data-clk-pins {
 				pins = "gpio12", "gpio13";
 				function = "qup03";
 			};
 
-			qup_i2c4_data_clk: qup-i2c4-data-clk {
+			qup_i2c4_data_clk: qup-i2c4-data-clk-pins {
 				pins = "gpio16", "gpio17";
 				function = "qup04";
 			};
 
-			qup_i2c5_data_clk: qup-i2c5-data-clk {
+			qup_i2c5_data_clk: qup-i2c5-data-clk-pins {
 				pins = "gpio20", "gpio21";
 				function = "qup05";
 			};
 
-			qup_i2c6_data_clk: qup-i2c6-data-clk {
+			qup_i2c6_data_clk: qup-i2c6-data-clk-pins {
 				pins = "gpio24", "gpio25";
 				function = "qup06";
 			};
 
-			qup_i2c7_data_clk: qup-i2c7-data-clk {
+			qup_i2c7_data_clk: qup-i2c7-data-clk-pins {
 				pins = "gpio28", "gpio29";
 				function = "qup07";
 			};
 
-			qup_i2c8_data_clk: qup-i2c8-data-clk {
+			qup_i2c8_data_clk: qup-i2c8-data-clk-pins {
 				pins = "gpio32", "gpio33";
 				function = "qup10";
 			};
 
-			qup_i2c9_data_clk: qup-i2c9-data-clk {
+			qup_i2c9_data_clk: qup-i2c9-data-clk-pins {
 				pins = "gpio36", "gpio37";
 				function = "qup11";
 			};
 
-			qup_i2c10_data_clk: qup-i2c10-data-clk {
+			qup_i2c10_data_clk: qup-i2c10-data-clk-pins {
 				pins = "gpio40", "gpio41";
 				function = "qup12";
 			};
 
-			qup_i2c11_data_clk: qup-i2c11-data-clk {
+			qup_i2c11_data_clk: qup-i2c11-data-clk-pins {
 				pins = "gpio44", "gpio45";
 				function = "qup13";
 			};
 
-			qup_i2c12_data_clk: qup-i2c12-data-clk {
+			qup_i2c12_data_clk: qup-i2c12-data-clk-pins {
 				pins = "gpio48", "gpio49";
 				function = "qup14";
 			};
 
-			qup_i2c13_data_clk: qup-i2c13-data-clk {
+			qup_i2c13_data_clk: qup-i2c13-data-clk-pins {
 				pins = "gpio52", "gpio53";
 				function = "qup15";
 			};
 
-			qup_i2c14_data_clk: qup-i2c14-data-clk {
+			qup_i2c14_data_clk: qup-i2c14-data-clk-pins {
 				pins = "gpio56", "gpio57";
 				function = "qup16";
 			};
 
-			qup_i2c15_data_clk: qup-i2c15-data-clk {
+			qup_i2c15_data_clk: qup-i2c15-data-clk-pins {
 				pins = "gpio60", "gpio61";
 				function = "qup17";
 			};
 
-			qup_spi0_data_clk: qup-spi0-data-clk {
+			qup_spi0_data_clk: qup-spi0-data-clk-pins {
 				pins = "gpio0", "gpio1", "gpio2";
 				function = "qup00";
 			};
 
-			qup_spi0_cs: qup-spi0-cs {
+			qup_spi0_cs: qup-spi0-cs-pins {
 				pins = "gpio3";
 				function = "qup00";
 			};
 
-			qup_spi0_cs_gpio: qup-spi0-cs-gpio {
+			qup_spi0_cs_gpio: qup-spi0-cs-gpio-pins {
 				pins = "gpio3";
 				function = "gpio";
 			};
 
-			qup_spi1_data_clk: qup-spi1-data-clk {
+			qup_spi1_data_clk: qup-spi1-data-clk-pins {
 				pins = "gpio4", "gpio5", "gpio6";
 				function = "qup01";
 			};
 
-			qup_spi1_cs: qup-spi1-cs {
+			qup_spi1_cs: qup-spi1-cs-pins {
 				pins = "gpio7";
 				function = "qup01";
 			};
 
-			qup_spi1_cs_gpio: qup-spi1-cs-gpio {
+			qup_spi1_cs_gpio: qup-spi1-cs-gpio-pins {
 				pins = "gpio7";
 				function = "gpio";
 			};
 
-			qup_spi2_data_clk: qup-spi2-data-clk {
+			qup_spi2_data_clk: qup-spi2-data-clk-pins {
 				pins = "gpio8", "gpio9", "gpio10";
 				function = "qup02";
 			};
 
-			qup_spi2_cs: qup-spi2-cs {
+			qup_spi2_cs: qup-spi2-cs-pins {
 				pins = "gpio11";
 				function = "qup02";
 			};
 
-			qup_spi2_cs_gpio: qup-spi2-cs-gpio {
+			qup_spi2_cs_gpio: qup-spi2-cs-gpio-pins {
 				pins = "gpio11";
 				function = "gpio";
 			};
 
-			qup_spi3_data_clk: qup-spi3-data-clk {
+			qup_spi3_data_clk: qup-spi3-data-clk-pins {
 				pins = "gpio12", "gpio13", "gpio14";
 				function = "qup03";
 			};
 
-			qup_spi3_cs: qup-spi3-cs {
+			qup_spi3_cs: qup-spi3-cs-pins {
 				pins = "gpio15";
 				function = "qup03";
 			};
 
-			qup_spi3_cs_gpio: qup-spi3-cs-gpio {
+			qup_spi3_cs_gpio: qup-spi3-cs-gpio-pins {
 				pins = "gpio15";
 				function = "gpio";
 			};
 
-			qup_spi4_data_clk: qup-spi4-data-clk {
+			qup_spi4_data_clk: qup-spi4-data-clk-pins {
 				pins = "gpio16", "gpio17", "gpio18";
 				function = "qup04";
 			};
 
-			qup_spi4_cs: qup-spi4-cs {
+			qup_spi4_cs: qup-spi4-cs-pins {
 				pins = "gpio19";
 				function = "qup04";
 			};
 
-			qup_spi4_cs_gpio: qup-spi4-cs-gpio {
+			qup_spi4_cs_gpio: qup-spi4-cs-gpio-pins {
 				pins = "gpio19";
 				function = "gpio";
 			};
 
-			qup_spi5_data_clk: qup-spi5-data-clk {
+			qup_spi5_data_clk: qup-spi5-data-clk-pins {
 				pins = "gpio20", "gpio21", "gpio22";
 				function = "qup05";
 			};
 
-			qup_spi5_cs: qup-spi5-cs {
+			qup_spi5_cs: qup-spi5-cs-pins {
 				pins = "gpio23";
 				function = "qup05";
 			};
 
-			qup_spi5_cs_gpio: qup-spi5-cs-gpio {
+			qup_spi5_cs_gpio: qup-spi5-cs-gpio-pins {
 				pins = "gpio23";
 				function = "gpio";
 			};
 
-			qup_spi6_data_clk: qup-spi6-data-clk {
+			qup_spi6_data_clk: qup-spi6-data-clk-pins {
 				pins = "gpio24", "gpio25", "gpio26";
 				function = "qup06";
 			};
 
-			qup_spi6_cs: qup-spi6-cs {
+			qup_spi6_cs: qup-spi6-cs-pins {
 				pins = "gpio27";
 				function = "qup06";
 			};
 
-			qup_spi6_cs_gpio: qup-spi6-cs-gpio {
+			qup_spi6_cs_gpio: qup-spi6-cs-gpio-pins {
 				pins = "gpio27";
 				function = "gpio";
 			};
 
-			qup_spi7_data_clk: qup-spi7-data-clk {
+			qup_spi7_data_clk: qup-spi7-data-clk-pins {
 				pins = "gpio28", "gpio29", "gpio30";
 				function = "qup07";
 			};
 
-			qup_spi7_cs: qup-spi7-cs {
+			qup_spi7_cs: qup-spi7-cs-pins {
 				pins = "gpio31";
 				function = "qup07";
 			};
 
-			qup_spi7_cs_gpio: qup-spi7-cs-gpio {
+			qup_spi7_cs_gpio: qup-spi7-cs-gpio-pins {
 				pins = "gpio31";
 				function = "gpio";
 			};
 
-			qup_spi8_data_clk: qup-spi8-data-clk {
+			qup_spi8_data_clk: qup-spi8-data-clk-pins {
 				pins = "gpio32", "gpio33", "gpio34";
 				function = "qup10";
 			};
 
-			qup_spi8_cs: qup-spi8-cs {
+			qup_spi8_cs: qup-spi8-cs-pins {
 				pins = "gpio35";
 				function = "qup10";
 			};
 
-			qup_spi8_cs_gpio: qup-spi8-cs-gpio {
+			qup_spi8_cs_gpio: qup-spi8-cs-gpio-pins {
 				pins = "gpio35";
 				function = "gpio";
 			};
 
-			qup_spi9_data_clk: qup-spi9-data-clk {
+			qup_spi9_data_clk: qup-spi9-data-clk-pins {
 				pins = "gpio36", "gpio37", "gpio38";
 				function = "qup11";
 			};
 
-			qup_spi9_cs: qup-spi9-cs {
+			qup_spi9_cs: qup-spi9-cs-pins {
 				pins = "gpio39";
 				function = "qup11";
 			};
 
-			qup_spi9_cs_gpio: qup-spi9-cs-gpio {
+			qup_spi9_cs_gpio: qup-spi9-cs-gpio-pins {
 				pins = "gpio39";
 				function = "gpio";
 			};
 
-			qup_spi10_data_clk: qup-spi10-data-clk {
+			qup_spi10_data_clk: qup-spi10-data-clk-pins {
 				pins = "gpio40", "gpio41", "gpio42";
 				function = "qup12";
 			};
 
-			qup_spi10_cs: qup-spi10-cs {
+			qup_spi10_cs: qup-spi10-cs-pins {
 				pins = "gpio43";
 				function = "qup12";
 			};
 
-			qup_spi10_cs_gpio: qup-spi10-cs-gpio {
+			qup_spi10_cs_gpio: qup-spi10-cs-gpio-pins {
 				pins = "gpio43";
 				function = "gpio";
 			};
 
-			qup_spi11_data_clk: qup-spi11-data-clk {
+			qup_spi11_data_clk: qup-spi11-data-clk-pins {
 				pins = "gpio44", "gpio45", "gpio46";
 				function = "qup13";
 			};
 
-			qup_spi11_cs: qup-spi11-cs {
+			qup_spi11_cs: qup-spi11-cs-pins {
 				pins = "gpio47";
 				function = "qup13";
 			};
 
-			qup_spi11_cs_gpio: qup-spi11-cs-gpio {
+			qup_spi11_cs_gpio: qup-spi11-cs-gpio-pins {
 				pins = "gpio47";
 				function = "gpio";
 			};
 
-			qup_spi12_data_clk: qup-spi12-data-clk {
+			qup_spi12_data_clk: qup-spi12-data-clk-pins {
 				pins = "gpio48", "gpio49", "gpio50";
 				function = "qup14";
 			};
 
-			qup_spi12_cs: qup-spi12-cs {
+			qup_spi12_cs: qup-spi12-cs-pins {
 				pins = "gpio51";
 				function = "qup14";
 			};
 
-			qup_spi12_cs_gpio: qup-spi12-cs-gpio {
+			qup_spi12_cs_gpio: qup-spi12-cs-gpio-pins {
 				pins = "gpio51";
 				function = "gpio";
 			};
 
-			qup_spi13_data_clk: qup-spi13-data-clk {
+			qup_spi13_data_clk: qup-spi13-data-clk-pins {
 				pins = "gpio52", "gpio53", "gpio54";
 				function = "qup15";
 			};
 
-			qup_spi13_cs: qup-spi13-cs {
+			qup_spi13_cs: qup-spi13-cs-pins {
 				pins = "gpio55";
 				function = "qup15";
 			};
 
-			qup_spi13_cs_gpio: qup-spi13-cs-gpio {
+			qup_spi13_cs_gpio: qup-spi13-cs-gpio-pins {
 				pins = "gpio55";
 				function = "gpio";
 			};
 
-			qup_spi14_data_clk: qup-spi14-data-clk {
+			qup_spi14_data_clk: qup-spi14-data-clk-pins {
 				pins = "gpio56", "gpio57", "gpio58";
 				function = "qup16";
 			};
 
-			qup_spi14_cs: qup-spi14-cs {
+			qup_spi14_cs: qup-spi14-cs-pins {
 				pins = "gpio59";
 				function = "qup16";
 			};
 
-			qup_spi14_cs_gpio: qup-spi14-cs-gpio {
+			qup_spi14_cs_gpio: qup-spi14-cs-gpio-pins {
 				pins = "gpio59";
 				function = "gpio";
 			};
 
-			qup_spi15_data_clk: qup-spi15-data-clk {
+			qup_spi15_data_clk: qup-spi15-data-clk-pins {
 				pins = "gpio60", "gpio61", "gpio62";
 				function = "qup17";
 			};
 
-			qup_spi15_cs: qup-spi15-cs {
+			qup_spi15_cs: qup-spi15-cs-pins {
 				pins = "gpio63";
 				function = "qup17";
 			};
 
-			qup_spi15_cs_gpio: qup-spi15-cs-gpio {
+			qup_spi15_cs_gpio: qup-spi15-cs-gpio-pins {
 				pins = "gpio63";
 				function = "gpio";
 			};
 
-			qup_uart0_cts: qup-uart0-cts {
+			qup_uart0_cts: qup-uart0-cts-pins {
 				pins = "gpio0";
 				function = "qup00";
 			};
 
-			qup_uart0_rts: qup-uart0-rts {
+			qup_uart0_rts: qup-uart0-rts-pins {
 				pins = "gpio1";
 				function = "qup00";
 			};
 
-			qup_uart0_tx: qup-uart0-tx {
+			qup_uart0_tx: qup-uart0-tx-pins {
 				pins = "gpio2";
 				function = "qup00";
 			};
 
-			qup_uart0_rx: qup-uart0-rx {
+			qup_uart0_rx: qup-uart0-rx-pins {
 				pins = "gpio3";
 				function = "qup00";
 			};
 
-			qup_uart1_cts: qup-uart1-cts {
+			qup_uart1_cts: qup-uart1-cts-pins {
 				pins = "gpio4";
 				function = "qup01";
 			};
 
-			qup_uart1_rts: qup-uart1-rts {
+			qup_uart1_rts: qup-uart1-rts-pins {
 				pins = "gpio5";
 				function = "qup01";
 			};
 
-			qup_uart1_tx: qup-uart1-tx {
+			qup_uart1_tx: qup-uart1-tx-pins {
 				pins = "gpio6";
 				function = "qup01";
 			};
 
-			qup_uart1_rx: qup-uart1-rx {
+			qup_uart1_rx: qup-uart1-rx-pins {
 				pins = "gpio7";
 				function = "qup01";
 			};
 
-			qup_uart2_cts: qup-uart2-cts {
+			qup_uart2_cts: qup-uart2-cts-pins {
 				pins = "gpio8";
 				function = "qup02";
 			};
 
-			qup_uart2_rts: qup-uart2-rts {
+			qup_uart2_rts: qup-uart2-rts-pins {
 				pins = "gpio9";
 				function = "qup02";
 			};
 
-			qup_uart2_tx: qup-uart2-tx {
+			qup_uart2_tx: qup-uart2-tx-pins {
 				pins = "gpio10";
 				function = "qup02";
 			};
 
-			qup_uart2_rx: qup-uart2-rx {
+			qup_uart2_rx: qup-uart2-rx-pins {
 				pins = "gpio11";
 				function = "qup02";
 			};
 
-			qup_uart3_cts: qup-uart3-cts {
+			qup_uart3_cts: qup-uart3-cts-pins {
 				pins = "gpio12";
 				function = "qup03";
 			};
 
-			qup_uart3_rts: qup-uart3-rts {
+			qup_uart3_rts: qup-uart3-rts-pins {
 				pins = "gpio13";
 				function = "qup03";
 			};
 
-			qup_uart3_tx: qup-uart3-tx {
+			qup_uart3_tx: qup-uart3-tx-pins {
 				pins = "gpio14";
 				function = "qup03";
 			};
 
-			qup_uart3_rx: qup-uart3-rx {
+			qup_uart3_rx: qup-uart3-rx-pins {
 				pins = "gpio15";
 				function = "qup03";
 			};
 
-			qup_uart4_cts: qup-uart4-cts {
+			qup_uart4_cts: qup-uart4-cts-pins {
 				pins = "gpio16";
 				function = "qup04";
 			};
 
-			qup_uart4_rts: qup-uart4-rts {
+			qup_uart4_rts: qup-uart4-rts-pins {
 				pins = "gpio17";
 				function = "qup04";
 			};
 
-			qup_uart4_tx: qup-uart4-tx {
+			qup_uart4_tx: qup-uart4-tx-pins {
 				pins = "gpio18";
 				function = "qup04";
 			};
 
-			qup_uart4_rx: qup-uart4-rx {
+			qup_uart4_rx: qup-uart4-rx-pins {
 				pins = "gpio19";
 				function = "qup04";
 			};
 
-			qup_uart5_cts: qup-uart5-cts {
+			qup_uart5_cts: qup-uart5-cts-pins {
 				pins = "gpio20";
 				function = "qup05";
 			};
 
-			qup_uart5_rts: qup-uart5-rts {
+			qup_uart5_rts: qup-uart5-rts-pins {
 				pins = "gpio21";
 				function = "qup05";
 			};
 
-			qup_uart5_tx: qup-uart5-tx {
+			qup_uart5_tx: qup-uart5-tx-pins {
 				pins = "gpio22";
 				function = "qup05";
 			};
 
-			qup_uart5_rx: qup-uart5-rx {
+			qup_uart5_rx: qup-uart5-rx-pins {
 				pins = "gpio23";
 				function = "qup05";
 			};
 
-			qup_uart6_cts: qup-uart6-cts {
+			qup_uart6_cts: qup-uart6-cts-pins {
 				pins = "gpio24";
 				function = "qup06";
 			};
 
-			qup_uart6_rts: qup-uart6-rts {
+			qup_uart6_rts: qup-uart6-rts-pins {
 				pins = "gpio25";
 				function = "qup06";
 			};
 
-			qup_uart6_tx: qup-uart6-tx {
+			qup_uart6_tx: qup-uart6-tx-pins {
 				pins = "gpio26";
 				function = "qup06";
 			};
 
-			qup_uart6_rx: qup-uart6-rx {
+			qup_uart6_rx: qup-uart6-rx-pins {
 				pins = "gpio27";
 				function = "qup06";
 			};
 
-			qup_uart7_cts: qup-uart7-cts {
+			qup_uart7_cts: qup-uart7-cts-pins {
 				pins = "gpio28";
 				function = "qup07";
 			};
 
-			qup_uart7_rts: qup-uart7-rts {
+			qup_uart7_rts: qup-uart7-rts-pins {
 				pins = "gpio29";
 				function = "qup07";
 			};
 
-			qup_uart7_tx: qup-uart7-tx {
+			qup_uart7_tx: qup-uart7-tx-pins {
 				pins = "gpio30";
 				function = "qup07";
 			};
 
-			qup_uart7_rx: qup-uart7-rx {
+			qup_uart7_rx: qup-uart7-rx-pins {
 				pins = "gpio31";
 				function = "qup07";
 			};
 
-			qup_uart8_cts: qup-uart8-cts {
+			qup_uart8_cts: qup-uart8-cts-pins {
 				pins = "gpio32";
 				function = "qup10";
 			};
 
-			qup_uart8_rts: qup-uart8-rts {
+			qup_uart8_rts: qup-uart8-rts-pins {
 				pins = "gpio33";
 				function = "qup10";
 			};
 
-			qup_uart8_tx: qup-uart8-tx {
+			qup_uart8_tx: qup-uart8-tx-pins {
 				pins = "gpio34";
 				function = "qup10";
 			};
 
-			qup_uart8_rx: qup-uart8-rx {
+			qup_uart8_rx: qup-uart8-rx-pins {
 				pins = "gpio35";
 				function = "qup10";
 			};
 
-			qup_uart9_cts: qup-uart9-cts {
+			qup_uart9_cts: qup-uart9-cts-pins {
 				pins = "gpio36";
 				function = "qup11";
 			};
 
-			qup_uart9_rts: qup-uart9-rts {
+			qup_uart9_rts: qup-uart9-rts-pins {
 				pins = "gpio37";
 				function = "qup11";
 			};
 
-			qup_uart9_tx: qup-uart9-tx {
+			qup_uart9_tx: qup-uart9-tx-pins {
 				pins = "gpio38";
 				function = "qup11";
 			};
 
-			qup_uart9_rx: qup-uart9-rx {
+			qup_uart9_rx: qup-uart9-rx-pins {
 				pins = "gpio39";
 				function = "qup11";
 			};
 
-			qup_uart10_cts: qup-uart10-cts {
+			qup_uart10_cts: qup-uart10-cts-pins {
 				pins = "gpio40";
 				function = "qup12";
 			};
 
-			qup_uart10_rts: qup-uart10-rts {
+			qup_uart10_rts: qup-uart10-rts-pins {
 				pins = "gpio41";
 				function = "qup12";
 			};
 
-			qup_uart10_tx: qup-uart10-tx {
+			qup_uart10_tx: qup-uart10-tx-pins {
 				pins = "gpio42";
 				function = "qup12";
 			};
 
-			qup_uart10_rx: qup-uart10-rx {
+			qup_uart10_rx: qup-uart10-rx-pins {
 				pins = "gpio43";
 				function = "qup12";
 			};
 
-			qup_uart11_cts: qup-uart11-cts {
+			qup_uart11_cts: qup-uart11-cts-pins {
 				pins = "gpio44";
 				function = "qup13";
 			};
 
-			qup_uart11_rts: qup-uart11-rts {
+			qup_uart11_rts: qup-uart11-rts-pins {
 				pins = "gpio45";
 				function = "qup13";
 			};
 
-			qup_uart11_tx: qup-uart11-tx {
+			qup_uart11_tx: qup-uart11-tx-pins {
 				pins = "gpio46";
 				function = "qup13";
 			};
 
-			qup_uart11_rx: qup-uart11-rx {
+			qup_uart11_rx: qup-uart11-rx-pins {
 				pins = "gpio47";
 				function = "qup13";
 			};
 
-			qup_uart12_cts: qup-uart12-cts {
+			qup_uart12_cts: qup-uart12-cts-pins {
 				pins = "gpio48";
 				function = "qup14";
 			};
 
-			qup_uart12_rts: qup-uart12-rts {
+			qup_uart12_rts: qup-uart12-rts-pins {
 				pins = "gpio49";
 				function = "qup14";
 			};
 
-			qup_uart12_tx: qup-uart12-tx {
+			qup_uart12_tx: qup-uart12-tx-pins {
 				pins = "gpio50";
 				function = "qup14";
 			};
 
-			qup_uart12_rx: qup-uart12-rx {
+			qup_uart12_rx: qup-uart12-rx-pins {
 				pins = "gpio51";
 				function = "qup14";
 			};
 
-			qup_uart13_cts: qup-uart13-cts {
+			qup_uart13_cts: qup-uart13-cts-pins {
 				pins = "gpio52";
 				function = "qup15";
 			};
 
-			qup_uart13_rts: qup-uart13-rts {
+			qup_uart13_rts: qup-uart13-rts-pins {
 				pins = "gpio53";
 				function = "qup15";
 			};
 
-			qup_uart13_tx: qup-uart13-tx {
+			qup_uart13_tx: qup-uart13-tx-pins {
 				pins = "gpio54";
 				function = "qup15";
 			};
 
-			qup_uart13_rx: qup-uart13-rx {
+			qup_uart13_rx: qup-uart13-rx-pins {
 				pins = "gpio55";
 				function = "qup15";
 			};
 
-			qup_uart14_cts: qup-uart14-cts {
+			qup_uart14_cts: qup-uart14-cts-pins {
 				pins = "gpio56";
 				function = "qup16";
 			};
 
-			qup_uart14_rts: qup-uart14-rts {
+			qup_uart14_rts: qup-uart14-rts-pins {
 				pins = "gpio57";
 				function = "qup16";
 			};
 
-			qup_uart14_tx: qup-uart14-tx {
+			qup_uart14_tx: qup-uart14-tx-pins {
 				pins = "gpio58";
 				function = "qup16";
 			};
 
-			qup_uart14_rx: qup-uart14-rx {
+			qup_uart14_rx: qup-uart14-rx-pins {
 				pins = "gpio59";
 				function = "qup16";
 			};
 
-			qup_uart15_cts: qup-uart15-cts {
+			qup_uart15_cts: qup-uart15-cts-pins {
 				pins = "gpio60";
 				function = "qup17";
 			};
 
-			qup_uart15_rts: qup-uart15-rts {
+			qup_uart15_rts: qup-uart15-rts-pins {
 				pins = "gpio61";
 				function = "qup17";
 			};
 
-			qup_uart15_tx: qup-uart15-tx {
+			qup_uart15_tx: qup-uart15-tx-pins {
 				pins = "gpio62";
 				function = "qup17";
 			};
 
-			qup_uart15_rx: qup-uart15-rx {
+			qup_uart15_rx: qup-uart15-rx-pins {
 				pins = "gpio63";
 				function = "qup17";
 			};
 
-			sdc1_clk: sdc1-clk {
+			sdc1_clk: sdc1-clk-pins {
 				pins = "sdc1_clk";
 			};
 
-			sdc1_cmd: sdc1-cmd {
+			sdc1_cmd: sdc1-cmd-pins {
 				pins = "sdc1_cmd";
 			};
 
-			sdc1_data: sdc1-data {
+			sdc1_data: sdc1-data-pins {
 				pins = "sdc1_data";
 			};
 
-			sdc1_rclk: sdc1-rclk {
+			sdc1_rclk: sdc1-rclk-pins {
 				pins = "sdc1_rclk";
 			};
 
-			sdc1_clk_sleep: sdc1-clk-sleep {
+			sdc1_clk_sleep: sdc1-clk-sleep-pins {
 				pins = "sdc1_clk";
 				drive-strength = <2>;
 				bias-bus-hold;
 			};
 
-			sdc1_cmd_sleep: sdc1-cmd-sleep {
+			sdc1_cmd_sleep: sdc1-cmd-sleep-pins {
 				pins = "sdc1_cmd";
 				drive-strength = <2>;
 				bias-bus-hold;
 			};
 
-			sdc1_data_sleep: sdc1-data-sleep {
+			sdc1_data_sleep: sdc1-data-sleep-pins {
 				pins = "sdc1_data";
 				drive-strength = <2>;
 				bias-bus-hold;
 			};
 
-			sdc1_rclk_sleep: sdc1-rclk-sleep {
+			sdc1_rclk_sleep: sdc1-rclk-sleep-pins {
 				pins = "sdc1_rclk";
 				drive-strength = <2>;
 				bias-bus-hold;
 			};
 
-			sdc2_clk: sdc2-clk {
+			sdc2_clk: sdc2-clk-pins {
 				pins = "sdc2_clk";
 			};
 
-			sdc2_cmd: sdc2-cmd {
+			sdc2_cmd: sdc2-cmd-pins {
 				pins = "sdc2_cmd";
 			};
 
-			sdc2_data: sdc2-data {
+			sdc2_data: sdc2-data-pins {
 				pins = "sdc2_data";
 			};
 
-			sdc2_clk_sleep: sdc2-clk-sleep {
+			sdc2_clk_sleep: sdc2-clk-sleep-pins {
 				pins = "sdc2_clk";
 				drive-strength = <2>;
 				bias-bus-hold;
 			};
 
-			sdc2_cmd_sleep: sdc2-cmd-sleep {
+			sdc2_cmd_sleep: sdc2-cmd-sleep-pins {
 				pins = "sdc2_cmd";
 				drive-strength = <2>;
 				bias-bus-hold;
 			};
 
-			sdc2_data_sleep: sdc2-data-sleep {
+			sdc2_data_sleep: sdc2-data-sleep-pins {
 				pins = "sdc2_data";
 				drive-strength = <2>;
 				bias-bus-hold;
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
index 45058ad0a1c8..fea7d8273ccd 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
@@ -87,7 +87,6 @@
 			regulator-min-microvolt = <1200000>;
 			regulator-max-microvolt = <1200000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 			regulator-boot-on;
 			regulator-always-on;
 		};
@@ -97,7 +96,6 @@
 			regulator-min-microvolt = <912000>;
 			regulator-max-microvolt = <912000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l6b: ldo6 {
@@ -105,7 +103,6 @@
 			regulator-min-microvolt = <880000>;
 			regulator-max-microvolt = <880000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 			regulator-boot-on;
 		};
 	};
@@ -119,7 +116,6 @@
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <1800000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l7c: ldo7 {
@@ -135,7 +131,6 @@
 			regulator-min-microvolt = <3072000>;
 			regulator-max-microvolt = <3072000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 	};
 
@@ -158,7 +153,6 @@
 			regulator-min-microvolt = <1200000>;
 			regulator-max-microvolt = <1200000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l6d: ldo6 {
@@ -166,7 +160,6 @@
 			regulator-min-microvolt = <880000>;
 			regulator-max-microvolt = <880000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l7d: ldo7 {
@@ -174,7 +167,6 @@
 			regulator-min-microvolt = <3072000>;
 			regulator-max-microvolt = <3072000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l9d: ldo9 {
@@ -182,7 +174,6 @@
 			regulator-min-microvolt = <912000>;
 			regulator-max-microvolt = <912000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 	};
 };
@@ -203,16 +194,20 @@
 	clock-frequency = <400000>;
 
 	pinctrl-names = "default";
-	pinctrl-0 = <&qup0_i2c4_default>, <&ts0_default>;
+	pinctrl-0 = <&qup0_i2c4_default>;
 
 	status = "okay";
 
 	touchscreen@10 {
 		compatible = "hid-over-i2c";
 		reg = <0x10>;
+
 		hid-descr-addr = <0x1>;
 		interrupts-extended = <&tlmm 175 IRQ_TYPE_LEVEL_LOW>;
 		vdd-supply = <&vreg_misc_3p3>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&ts0_default>;
 	};
 };
 
@@ -228,24 +223,36 @@
 	clock-frequency = <400000>;
 
 	pinctrl-names = "default";
-	pinctrl-0 = <&qup2_i2c5_default>, <&kybd_default>, <&tpad_default>;
+	pinctrl-0 = <&qup2_i2c5_default>;
 
 	status = "okay";
 
 	touchpad@15 {
 		compatible = "hid-over-i2c";
 		reg = <0x15>;
+
 		hid-descr-addr = <0x1>;
 		interrupts-extended = <&tlmm 182 IRQ_TYPE_LEVEL_LOW>;
 		vdd-supply = <&vreg_misc_3p3>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&tpad_default>;
+
+		wakeup-source;
 	};
 
 	keyboard@68 {
 		compatible = "hid-over-i2c";
 		reg = <0x68>;
+
 		hid-descr-addr = <0x1>;
 		interrupts-extended = <&tlmm 104 IRQ_TYPE_LEVEL_LOW>;
 		vdd-supply = <&vreg_misc_3p3>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&kybd_default>;
+
+		wakeup-source;
 	};
 };
 
@@ -414,7 +421,7 @@
 		int-n {
 			pins = "gpio175";
 			function = "gpio";
-			bias-pull-up;
+			bias-disable;
 		};
 
 		reset-n {
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
index 84dc92dda0b8..b2b744bb8a53 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
@@ -79,7 +79,6 @@
 			regulator-min-microvolt = <1200000>;
 			regulator-max-microvolt = <1200000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 			regulator-boot-on;
 		};
 
@@ -88,7 +87,6 @@
 			regulator-min-microvolt = <912000>;
 			regulator-max-microvolt = <912000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l6b: ldo6 {
@@ -96,7 +94,6 @@
 			regulator-min-microvolt = <880000>;
 			regulator-max-microvolt = <880000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 			regulator-boot-on;
 			regulator-always-on;	// FIXME: VDD_A_EDP_0_0P9
 		};
@@ -111,7 +108,6 @@
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <1800000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l12c: ldo12 {
@@ -119,7 +115,6 @@
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <1800000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l13c: ldo13 {
@@ -127,7 +122,6 @@
 			regulator-min-microvolt = <3072000>;
 			regulator-max-microvolt = <3072000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 	};
 
@@ -142,7 +136,6 @@
 			regulator-min-microvolt = <1200000>;
 			regulator-max-microvolt = <1200000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l4d: ldo4 {
@@ -150,7 +143,6 @@
 			regulator-min-microvolt = <1200000>;
 			regulator-max-microvolt = <1200000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l7d: ldo7 {
@@ -158,7 +150,6 @@
 			regulator-min-microvolt = <3072000>;
 			regulator-max-microvolt = <3072000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 
 		vreg_l9d: ldo9 {
@@ -166,7 +157,6 @@
 			regulator-min-microvolt = <912000>;
 			regulator-max-microvolt = <912000>;
 			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
-			regulator-allow-set-load;
 		};
 	};
 };
@@ -187,7 +177,7 @@
 	clock-frequency = <400000>;
 
 	pinctrl-names = "default";
-	pinctrl-0 = <&qup0_i2c4_default>, <&ts0_default>;
+	pinctrl-0 = <&qup0_i2c4_default>;
 
 	status = "okay";
 
@@ -195,9 +185,13 @@
 	touchscreen@10 {
 		compatible = "hid-over-i2c";
 		reg = <0x10>;
+
 		hid-descr-addr = <0x1>;
 		interrupts-extended = <&tlmm 175 IRQ_TYPE_LEVEL_LOW>;
 		vdd-supply = <&vreg_misc_3p3>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&ts0_default>;
 	};
 };
 
@@ -213,35 +207,63 @@
 	clock-frequency = <400000>;
 
 	pinctrl-names = "default";
-	pinctrl-0 = <&qup2_i2c5_default>, <&kybd_default>, <&tpad_default>;
+	pinctrl-0 = <&qup2_i2c5_default>;
 
 	status = "okay";
 
+	touchpad@15 {
+		compatible = "hid-over-i2c";
+		reg = <0x15>;
+
+		hid-descr-addr = <0x1>;
+		interrupts-extended = <&tlmm 182 IRQ_TYPE_LEVEL_LOW>;
+		vdd-supply = <&vreg_misc_3p3>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&tpad_default>;
+
+		wakeup-source;
+
+		status = "disabled";
+	};
+
 	touchpad@2c {
 		compatible = "hid-over-i2c";
 		reg = <0x2c>;
+
 		hid-descr-addr = <0x20>;
 		interrupts-extended = <&tlmm 182 IRQ_TYPE_LEVEL_LOW>;
 		vdd-supply = <&vreg_misc_3p3>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&tpad_default>;
+
+		wakeup-source;
 	};
 
 	keyboard@68 {
 		compatible = "hid-over-i2c";
 		reg = <0x68>;
+
 		hid-descr-addr = <0x1>;
 		interrupts-extended = <&tlmm 104 IRQ_TYPE_LEVEL_LOW>;
 		vdd-supply = <&vreg_misc_3p3>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&kybd_default>;
+
+		wakeup-source;
 	};
 };
 
 &remoteproc_adsp {
-	firmware-name = "qcom/sc8280xp/qcadsp8280.mbn";
+	firmware-name = "qcom/sc8280xp/LENOVO/21BX/qcadsp8280.mbn";
 
 	status = "okay";
 };
 
 &remoteproc_nsp0 {
-	firmware-name = "qcom/sc8280xp/qccdsp8280.mbn";
+	firmware-name = "qcom/sc8280xp/LENOVO/21BX/qccdsp8280.mbn";
 
 	status = "okay";
 };
@@ -373,7 +395,7 @@
 		int-n {
 			pins = "gpio175";
 			function = "gpio";
-			bias-pull-up;
+			bias-disable;
 		};
 
 		reset-n {
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-pmics.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp-pmics.dtsi
index ae90b97aecb8..24836b6b9bbc 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-pmics.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-pmics.dtsi
@@ -60,9 +60,8 @@
 			#interrupt-cells = <2>;
 		};
 
-		pmc8280c_lpg: lpg@e800 {
+		pmc8280c_lpg: pwm {
 			compatible = "qcom,pm8350c-pwm";
-			reg = <0xe800>;
 
 			#address-cells = <1>;
 			#size-cells = <0>;
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
index 49ea8b5612fc..c32bcded2aef 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
@@ -1312,6 +1312,8 @@
 					<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_USB3_0 0>;
 			interconnect-names = "usb-ddr", "apps-usb";
 
+			wakeup-source;
+
 			status = "disabled";
 
 			usb_0_dwc3: usb@a600000 {
@@ -1364,6 +1366,8 @@
 					<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_USB3_1 0>;
 			interconnect-names = "usb-ddr", "apps-usb";
 
+			wakeup-source;
+
 			status = "disabled";
 
 			usb_1_dwc3: usb@a800000 {
diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi
index 1bc9091cad2a..b51b85f583e5 100644
--- a/arch/arm64/boot/dts/qcom/sdm630.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi
@@ -697,9 +697,15 @@
 			#thermal-sensor-cells = <1>;
 		};
 
-		tcsr_mutex_regs: syscon@1f40000 {
-			compatible = "syscon";
-			reg = <0x01f40000 0x40000>;
+		tcsr_mutex: hwlock@1f40000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x01f40000 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsr_regs_1: syscon@1f60000 {
+			compatible = "qcom,sdm630-tcsr", "syscon";
+			reg = <0x01f60000 0x20000>;
 		};
 
 		tlmm: pinctrl@3100000 {
@@ -2351,12 +2357,6 @@
 		};
 	};
 
-	tcsr_mutex: hwlock {
-		compatible = "qcom,tcsr-mutex";
-		syscon = <&tcsr_mutex_regs 0 0x1000>;
-		#hwlock-cells = <1>;
-	};
-
 	sound: sound {
 	};
 
diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts
index c6e2c571b452..132417e2d11e 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-db845c.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-db845c.dts
@@ -522,7 +522,7 @@
 
 &pcie0 {
 	status = "okay";
-	perst-gpio = <&tlmm 35 GPIO_ACTIVE_LOW>;
+	perst-gpios = <&tlmm 35 GPIO_ACTIVE_LOW>;
 	enable-gpio = <&tlmm 134 GPIO_ACTIVE_HIGH>;
 
 	vddpe-3v3-supply = <&pcie0_3p3v_dual>;
@@ -540,7 +540,7 @@
 
 &pcie1 {
 	status = "okay";
-	perst-gpio = <&tlmm 102 GPIO_ACTIVE_LOW>;
+	perst-gpios = <&tlmm 102 GPIO_ACTIVE_LOW>;
 
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie1_default_state>;
@@ -1081,7 +1081,7 @@
 	pinctrl-names = "default";
 	clock-names = "extclk";
 	clocks = <&rpmhcc RPMH_LN_BB_CLK2>;
-	reset-gpios = <&tlmm 64 0>;
+	reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>;
 	vdd-buck-supply = <&vreg_s4a_1p8>;
 	vdd-buck-sido-supply = <&vreg_s4a_1p8>;
 	vdd-tx-supply = <&vreg_s4a_1p8>;
@@ -1214,8 +1214,6 @@
 		reset-gpios = <&tlmm 9 GPIO_ACTIVE_LOW>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&cam0_default>;
-		gpios = <&tlmm 13 0>,
-			<&tlmm 9 GPIO_ACTIVE_LOW>;
 
 		clocks = <&clock_camcc CAM_CC_MCLK0_CLK>;
 		clock-names = "xvclk";
@@ -1228,8 +1226,6 @@
 		 * both have to be enabled through the power management
 		 * gpios.
 		 */
-		power-domains = <&clock_camcc TITAN_TOP_GDSC>;
-
 		dovdd-supply = <&vreg_lvs1a_1p8>;
 		avdd-supply = <&cam0_avdd_2v8>;
 		dvdd-supply = <&cam0_dvdd_1v2>;
@@ -1255,11 +1251,9 @@
 		reg = <0x60>;
 
 		// CAM3_RST_N
-		enable-gpios = <&tlmm 21 0>;
+		enable-gpios = <&tlmm 21 GPIO_ACTIVE_HIGH>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&cam3_default>;
-		gpios = <&tlmm 16 0>,
-			<&tlmm 21 0>;
 
 		clocks = <&clock_camcc CAM_CC_MCLK3_CLK>;
 		clock-names = "xclk";
@@ -1273,8 +1267,6 @@
 		 *
 		 * No 1.2V vddd-supply regulator is used.
 		 */
-		power-domains = <&clock_camcc TITAN_TOP_GDSC>;
-
 		vdddo-supply = <&vreg_lvs1a_1p8>;
 		vdda-supply = <&cam3_avdd_2v8>;
 
diff --git a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
index 7713e8060c5b..de2d10e0315a 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-mtp.dts
@@ -536,42 +536,42 @@
 		reg = <ADC5_XO_THERM_100K_PU>;
 		label = "xo_therm";
 		qcom,ratiometric;
-		qcom,hw-settle-time-us = <200>;
+		qcom,hw-settle-time = <200>;
 	};
 
 	adc-chan@4d {
 		reg = <ADC5_AMUX_THM1_100K_PU>;
 		label = "msm_therm";
 		qcom,ratiometric;
-		qcom,hw-settle-time-us = <200>;
+		qcom,hw-settle-time = <200>;
 	};
 
 	adc-chan@4f {
 		reg = <ADC5_AMUX_THM3_100K_PU>;
 		label = "pa_therm1";
 		qcom,ratiometric;
-		qcom,hw-settle-time-us = <200>;
+		qcom,hw-settle-time = <200>;
 	};
 
 	adc-chan@51 {
 		reg = <ADC5_AMUX_THM5_100K_PU>;
 		label = "quiet_therm";
 		qcom,ratiometric;
-		qcom,hw-settle-time-us = <200>;
+		qcom,hw-settle-time = <200>;
 	};
 
 	adc-chan@83 {
 		reg = <ADC5_VPH_PWR>;
 		label = "vph_pwr";
 		qcom,ratiometric;
-		qcom,hw-settle-time-us = <200>;
+		qcom,hw-settle-time = <200>;
 	};
 
 	adc-chan@85 {
 		reg = <ADC5_VCOIN>;
 		label = "vcoin";
 		qcom,ratiometric;
-		qcom,hw-settle-time-us = <200>;
+		qcom,hw-settle-time = <200>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts
index 82c27f90d300..0f470cf1ed1c 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium.dts
@@ -546,7 +546,7 @@
 	pinctrl-names = "default";
 	clock-names = "extclk";
 	clocks = <&rpmhcc RPMH_LN_BB_CLK2>;
-	reset-gpios = <&tlmm 64 0>;
+	reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>;
 	vdd-buck-supply = <&vreg_s4a_1p8>;
 	vdd-buck-sido-supply = <&vreg_s4a_1p8>;
 	vdd-tx-supply = <&vreg_s4a_1p8>;
diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts
index 7747081b9887..afc17e4d403f 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-polaris.dts
@@ -126,7 +126,7 @@
 		regulator-min-microvolt = <1800000>;
 		regulator-max-microvolt = <1800000>;
 
-		gpio = <&tlmm 23 0>;
+		gpio = <&tlmm 23 GPIO_ACTIVE_HIGH>;
 		regulator-always-on;
 		regulator-boot-on;
 		enable-active-high;
@@ -522,7 +522,7 @@
 };
 
 &pm8998_gpio {
-	volume_up_gpio: pm8998_gpio6 {
+	volume_up_gpio: pm8998-gpio6-state {
 		pinconf {
 			qcom,drive-strength = <PMIC_GPIO_STRENGTH_NO>;
 			function = "normal";
@@ -617,7 +617,7 @@
 		pins = "gpio6", "gpio10";
 		function = "gpio";
 		drive-strength = <8>;
-		bias-disable = <0>;
+		bias-disable;
 	};
 
 	sde_dsi_suspend: sde-dsi-suspend {
@@ -712,7 +712,7 @@
 	pinctrl-names = "default";
 	clock-names = "extclk";
 	clocks = <&rpmhcc RPMH_LN_BB_CLK2>;
-	reset-gpios = <&tlmm 64 0>;
+	reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>;
 	vdd-buck-sido-supply = <&vreg_s4a_1p8>;
 	vdd-buck-supply = <&vreg_s4a_1p8>;
 	vdd-tx-supply = <&vreg_s4a_1p8>;
diff --git a/arch/arm64/boot/dts/qcom/sdm845.dtsi b/arch/arm64/boot/dts/qcom/sdm845.dtsi
index f0e286715d1b..d761da47220d 100644
--- a/arch/arm64/boot/dts/qcom/sdm845.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845.dtsi
@@ -919,12 +919,6 @@
 		};
 	};
 
-	tcsr_mutex: hwlock {
-		compatible = "qcom,tcsr-mutex";
-		syscon = <&tcsr_mutex_regs 0 0x1000>;
-		#hwlock-cells = <1>;
-	};
-
 	smp2p-cdsp {
 		compatible = "qcom,smp2p";
 		qcom,smem = <94>, <432>;
@@ -2138,11 +2132,48 @@
 
 		llcc: system-cache-controller@1100000 {
 			compatible = "qcom,sdm845-llcc";
-			reg = <0 0x01100000 0 0x200000>, <0 0x01300000 0 0x50000>;
+			reg = <0 0x01100000 0 0x31000>, <0 0x01300000 0 0x50000>;
 			reg-names = "llcc_base", "llcc_broadcast_base";
 			interrupts = <GIC_SPI 582 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
+		pmu@114a000 {
+			compatible = "qcom,sdm845-llcc-bwmon";
+			reg = <0 0x0114a000 0 0x1000>;
+			interrupts = <GIC_SPI 580 IRQ_TYPE_LEVEL_HIGH>;
+			interconnects = <&mem_noc MASTER_LLCC 3 &mem_noc SLAVE_EBI1 3>;
+
+			operating-points-v2 = <&llcc_bwmon_opp_table>;
+
+			llcc_bwmon_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				/*
+				 * The interconnect path bandwidth taken from
+				 * cpu4_opp_table bandwidth for gladiator_noc-mem_noc
+				 * interconnect.  This also matches the
+				 * bandwidth table of qcom,llccbw (qcom,bw-tbl,
+				 * bus width: 4 bytes) from msm-4.9 downstream
+				 * kernel.
+				 */
+				opp-0 {
+					opp-peak-kBps = <800000>;
+				};
+				opp-1 {
+					opp-peak-kBps = <1804000>;
+				};
+				opp-2 {
+					opp-peak-kBps = <3072000>;
+				};
+				opp-3 {
+					opp-peak-kBps = <5412000>;
+				};
+				opp-4 {
+					opp-peak-kBps = <7216000>;
+				};
+			};
+		};
+
 		pmu@1436400 {
 			compatible = "qcom,sdm845-bwmon", "qcom,msm8998-bwmon";
 			reg = <0 0x01436400 0 0x600>;
@@ -2588,9 +2619,15 @@
 			status = "disabled";
 		};
 
-		tcsr_mutex_regs: syscon@1f40000 {
-			compatible = "syscon";
-			reg = <0 0x01f40000 0 0x40000>;
+		tcsr_mutex: hwlock@1f40000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0 0x01f40000 0 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsr_regs_1: syscon@1f60000 {
+			compatible = "qcom,sdm845-tcsr", "syscon";
+			reg = <0 0x01f60000 0 0x20000>;
 		};
 
 		tlmm: pinctrl@3400000 {
@@ -3207,7 +3244,7 @@
 				 <&pdc_reset PDC_MODEM_SYNC_RESET>;
 			reset-names = "mss_restart", "pdc_reset";
 
-			qcom,halt-regs = <&tcsr_mutex_regs 0x23000 0x25000 0x24000>;
+			qcom,halt-regs = <&tcsr_regs_1 0x3000 0x5000 0x4000>;
 
 			power-domains = <&rpmhpd SDM845_CX>,
 					<&rpmhpd SDM845_MX>,
@@ -4836,7 +4873,7 @@
 
 		aoss_qmp: power-controller@c300000 {
 			compatible = "qcom,sdm845-aoss-qmp", "qcom,aoss-qmp";
-			reg = <0 0x0c300000 0 0x100000>;
+			reg = <0 0x0c300000 0 0x400>;
 			interrupts = <GIC_SPI 389 IRQ_TYPE_EDGE_RISING>;
 			mboxes = <&apss_shared 0>;
 
@@ -4851,6 +4888,11 @@
 			};
 		};
 
+		sram@c3f0000 {
+			compatible = "qcom,sdm845-rpmh-stats";
+			reg = <0 0x0c3f0000 0 0x400>;
+		};
+
 		spmi_bus: spmi@c440000 {
 			compatible = "qcom,spmi-pmic-arb";
 			reg = <0 0x0c440000 0 0x1100>,
diff --git a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts
index a7af1bed4312..be59a8ba9c1f 100644
--- a/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts
+++ b/arch/arm64/boot/dts/qcom/sdm850-lenovo-yoga-c630.dts
@@ -772,7 +772,7 @@
 	pinctrl-names = "default";
 	clock-names = "extclk";
 	clocks = <&rpmhcc RPMH_LN_BB_CLK2>;
-	reset-gpios = <&tlmm 64 0>;
+	reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>;
 	vdd-buck-supply = <&vreg_s4a_1p8>;
 	vdd-buck-sido-supply = <&vreg_s4a_1p8>;
 	vdd-tx-supply = <&vreg_s4a_1p8>;
diff --git a/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts b/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts
index b0315eeb1320..f954fe5cb61a 100644
--- a/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts
+++ b/arch/arm64/boot/dts/qcom/sdm850-samsung-w737.dts
@@ -704,7 +704,7 @@
 	pinctrl-names = "default";
 	clock-names = "extclk";
 	clocks = <&rpmhcc RPMH_LN_BB_CLK2>;
-	reset-gpios = <&tlmm 64 0>;
+	reset-gpios = <&tlmm 64 GPIO_ACTIVE_HIGH>;
 	vdd-buck-supply = <&vreg_s4a_1p8>;
 	vdd-buck-sido-supply = <&vreg_s4a_1p8>;
 	vdd-tx-supply = <&vreg_s4a_1p8>;
diff --git a/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts b/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts
index 0aad2e94e757..6a8b88cc4385 100644
--- a/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts
+++ b/arch/arm64/boot/dts/qcom/sm6125-sony-xperia-seine-pdx201.dts
@@ -87,7 +87,7 @@
 };
 
 &sdc2_off_state {
-	sd-cd {
+	sd-cd-pins {
 		pins = "gpio98";
 		drive-strength = <2>;
 		bias-disable;
@@ -95,7 +95,7 @@
 };
 
 &sdc2_on_state {
-	sd-cd {
+	sd-cd-pins {
 		pins = "gpio98";
 		drive-strength = <2>;
 		bias-pull-up;
diff --git a/arch/arm64/boot/dts/qcom/sm6125.dtsi b/arch/arm64/boot/dts/qcom/sm6125.dtsi
index 8c582a9e4ada..1fe3fa3ad877 100644
--- a/arch/arm64/boot/dts/qcom/sm6125.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6125.dtsi
@@ -387,19 +387,19 @@
 			#interrupt-cells = <2>;
 
 			sdc2_off_state: sdc2-off-state {
-				clk {
+				clk-pins {
 					pins = "sdc2_clk";
 					drive-strength = <2>;
 					bias-disable;
 				};
 
-				cmd {
+				cmd-pins {
 					pins = "sdc2_cmd";
 					drive-strength = <2>;
 					bias-pull-up;
 				};
 
-				data {
+				data-pins {
 					pins = "sdc2_data";
 					drive-strength = <2>;
 					bias-pull-up;
@@ -413,13 +413,13 @@
 					bias-disable;
 				};
 
-				cmd {
+				cmd-pins-pins {
 					pins = "sdc2_cmd";
 					drive-strength = <10>;
 					bias-pull-up;
 				};
 
-				data {
+				data-pins {
 					pins = "sdc2_data";
 					drive-strength = <10>;
 					bias-pull-up;
diff --git a/arch/arm64/boot/dts/qcom/sm6350.dtsi b/arch/arm64/boot/dts/qcom/sm6350.dtsi
index d06aefdf3d9e..c39de7d3ace0 100644
--- a/arch/arm64/boot/dts/qcom/sm6350.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6350.dtsi
@@ -1,11 +1,14 @@
 // SPDX-License-Identifier: BSD-3-Clause
 /*
  * Copyright (c) 2021, Konrad Dybcio <konrad.dybcio@somainline.org>
+ * Copyright (c) 2022, Luca Weiss <luca.weiss@fairphone.com>
  */
 
 #include <dt-bindings/clock/qcom,gcc-sm6350.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
+#include <dt-bindings/dma/qcom-gpi.h>
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interconnect/qcom,sm6350.h>
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
@@ -517,6 +520,26 @@
 			};
 		};
 
+		gpi_dma0: dma-controller@800000 {
+			compatible = "qcom,sm6350-gpi-dma";
+			reg = <0 0x00800000 0 0x60000>;
+			interrupts = <GIC_SPI 244 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 245 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 246 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 247 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 248 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 249 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 250 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 251 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 252 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 253 IRQ_TYPE_LEVEL_HIGH>;
+			dma-channels = <10>;
+			dma-channel-mask = <0x1f>;
+			iommus = <&apps_smmu 0x56 0x0>;
+			#dma-cells = <3>;
+			status = "disabled";
+		};
+
 		qupv3_id_0: geniqup@8c0000 {
 			compatible = "qcom,geni-se-qup";
 			reg = <0x0 0x8c0000 0x0 0x2000>;
@@ -537,8 +560,15 @@
 				pinctrl-names = "default";
 				pinctrl-0 = <&qup_i2c0_default>;
 				interrupts = <GIC_SPI 601 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&gpi_dma0 0 0 QCOM_GPI_I2C>,
+				       <&gpi_dma0 1 0 QCOM_GPI_I2C>;
+				dma-names = "tx", "rx";
 				#address-cells = <1>;
 				#size-cells = <0>;
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>,
+						<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_0 0>,
+						<&aggre1_noc MASTER_QUP_0 0 &clk_virt SLAVE_EBI_CH0 0>;
+				interconnect-names = "qup-core", "qup-config", "qup-memory";
 				status = "disabled";
 			};
 
@@ -550,12 +580,39 @@
 				pinctrl-names = "default";
 				pinctrl-0 = <&qup_i2c2_default>;
 				interrupts = <GIC_SPI 603 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&gpi_dma0 0 2 QCOM_GPI_I2C>,
+				       <&gpi_dma0 1 2 QCOM_GPI_I2C>;
+				dma-names = "tx", "rx";
 				#address-cells = <1>;
 				#size-cells = <0>;
+				interconnects = <&clk_virt MASTER_QUP_CORE_0 0 &clk_virt SLAVE_QUP_CORE_0 0>,
+						<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_0 0>,
+						<&aggre1_noc MASTER_QUP_0 0 &clk_virt SLAVE_EBI_CH0 0>;
+				interconnect-names = "qup-core", "qup-config", "qup-memory";
 				status = "disabled";
 			};
 		};
 
+		gpi_dma1: dma-controller@900000 {
+			compatible = "qcom,sm6350-gpi-dma";
+			reg = <0 0x00900000 0 0x60000>;
+			interrupts = <GIC_SPI 645 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 646 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 647 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 648 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 649 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 650 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 651 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 652 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 653 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 654 IRQ_TYPE_LEVEL_HIGH>;
+			dma-channels = <10>;
+			dma-channel-mask = <0x3f>;
+			iommus = <&apps_smmu 0x4d6 0x0>;
+			#dma-cells = <3>;
+			status = "disabled";
+		};
+
 		qupv3_id_1: geniqup@9c0000 {
 			compatible = "qcom,geni-se-qup";
 			reg = <0x0 0x9c0000 0x0 0x2000>;
@@ -576,8 +633,15 @@
 				pinctrl-names = "default";
 				pinctrl-0 = <&qup_i2c6_default>;
 				interrupts = <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&gpi_dma1 0 0 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 0 QCOM_GPI_I2C>;
+				dma-names = "tx", "rx";
 				#address-cells = <1>;
 				#size-cells = <0>;
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
+						<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre2_noc MASTER_QUP_1 0 &clk_virt SLAVE_EBI_CH0 0>;
+				interconnect-names = "qup-core", "qup-config", "qup-memory";
 				status = "disabled";
 			};
 
@@ -589,8 +653,15 @@
 				pinctrl-names = "default";
 				pinctrl-0 = <&qup_i2c7_default>;
 				interrupts = <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&gpi_dma1 0 1 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 1 QCOM_GPI_I2C>;
+				dma-names = "tx", "rx";
 				#address-cells = <1>;
 				#size-cells = <0>;
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
+						<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre2_noc MASTER_QUP_1 0 &clk_virt SLAVE_EBI_CH0 0>;
+				interconnect-names = "qup-core", "qup-config", "qup-memory";
 				status = "disabled";
 			};
 
@@ -602,8 +673,15 @@
 				pinctrl-names = "default";
 				pinctrl-0 = <&qup_i2c8_default>;
 				interrupts = <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&gpi_dma1 0 2 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 2 QCOM_GPI_I2C>;
+				dma-names = "tx", "rx";
 				#address-cells = <1>;
 				#size-cells = <0>;
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
+						<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre2_noc MASTER_QUP_1 0 &clk_virt SLAVE_EBI_CH0 0>;
+				interconnect-names = "qup-core", "qup-config", "qup-memory";
 				status = "disabled";
 			};
 
@@ -615,6 +693,9 @@
 				pinctrl-names = "default";
 				pinctrl-0 = <&qup_uart9_default>;
 				interrupts = <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>;
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
+						<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_1 0>;
+				interconnect-names = "qup-core", "qup-config";
 				status = "disabled";
 			};
 
@@ -626,13 +707,67 @@
 				pinctrl-names = "default";
 				pinctrl-0 = <&qup_i2c10_default>;
 				interrupts = <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>;
+				dmas = <&gpi_dma1 0 4 QCOM_GPI_I2C>,
+				       <&gpi_dma1 1 4 QCOM_GPI_I2C>;
+				dma-names = "tx", "rx";
 				#address-cells = <1>;
 				#size-cells = <0>;
+				interconnects = <&clk_virt MASTER_QUP_CORE_1 0 &clk_virt SLAVE_QUP_CORE_1 0>,
+						<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_QUP_1 0>,
+						<&aggre2_noc MASTER_QUP_1 0 &clk_virt SLAVE_EBI_CH0 0>;
+				interconnect-names = "qup-core", "qup-config", "qup-memory";
 				status = "disabled";
 			};
 
 		};
 
+		config_noc: interconnect@1500000 {
+			compatible = "qcom,sm6350-config-noc";
+			reg = <0 0x01500000 0 0x28000>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		system_noc: interconnect@1620000 {
+			compatible = "qcom,sm6350-system-noc";
+			reg = <0 0x01620000 0 0x17080>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+
+			clk_virt: interconnect-clk-virt {
+				compatible = "qcom,sm6350-clk-virt";
+				#interconnect-cells = <2>;
+				qcom,bcm-voters = <&apps_bcm_voter>;
+			};
+		};
+
+		aggre1_noc: interconnect@16e0000 {
+			compatible = "qcom,sm6350-aggre1-noc";
+			reg = <0 0x016e0000 0 0x15080>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		aggre2_noc: interconnect@1700000 {
+			compatible = "qcom,sm6350-aggre2-noc";
+			reg = <0 0x01700000 0 0x1f880>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+
+			compute_noc: interconnect-compute-noc {
+				compatible = "qcom,sm6350-compute-noc";
+				#interconnect-cells = <2>;
+				qcom,bcm-voters = <&apps_bcm_voter>;
+			};
+		};
+
+		mmss_noc: interconnect@1740000 {
+			compatible = "qcom,sm6350-mmss-noc";
+			reg = <0 0x01740000 0 0x1c100>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
 		ufs_mem_hc: ufs@1d84000 {
 			compatible = "qcom,sm6350-ufshc", "qcom,ufshc",
 				     "jedec,ufs-2.0";
@@ -933,6 +1068,10 @@
 				 <&gcc GCC_SDCC2_APPS_CLK>,
 				 <&rpmhcc RPMH_CXO_CLK>;
 			clock-names = "iface", "core", "xo";
+			interconnects = <&aggre2_noc MASTER_SDCC_2 0 &clk_virt SLAVE_EBI_CH0 0>,
+					<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_SDCC_2 0>;
+			interconnect-names = "sdhc-ddr", "cpu-sdhc";
+
 			qcom,dll-config = <0x0007642c>;
 			qcom,ddr-config = <0x80040868>;
 			power-domains = <&rpmhpd SM6350_CX>;
@@ -947,11 +1086,15 @@
 				opp-100000000 {
 					opp-hz = /bits/ 64 <100000000>;
 					required-opps = <&rpmhpd_opp_svs_l1>;
+					opp-peak-kBps = <790000 131000>;
+					opp-avg-kBps = <50000 50000>;
 				};
 
 				opp-202000000 {
 					opp-hz = /bits/ 64 <202000000>;
 					required-opps = <&rpmhpd_opp_nom>;
+					opp-peak-kBps = <3190000 294000>;
+					opp-avg-kBps = <261438 300000>;
 				};
 			};
 		};
@@ -1017,12 +1160,33 @@
 			};
 		};
 
+		dc_noc: interconnect@9160000 {
+			compatible = "qcom,sm6350-dc-noc";
+			reg = <0 0x09160000 0 0x3200>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
 		system-cache-controller@9200000 {
 			compatible = "qcom,sm6350-llcc";
 			reg = <0 0x09200000 0 0x50000>, <0 0x09600000 0 0x50000>;
 			reg-names = "llcc_base", "llcc_broadcast_base";
 		};
 
+		gem_noc: interconnect@9680000 {
+			compatible = "qcom,sm6350-gem-noc";
+			reg = <0 0x09680000 0 0x3e200>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
+		npu_noc: interconnect@9990000 {
+			compatible = "qcom,sm6350-npu-noc";
+			reg = <0 0x09990000 0 0x1600>;
+			#interconnect-cells = <2>;
+			qcom,bcm-voters = <&apps_bcm_voter>;
+		};
+
 		usb_1: usb@a6f8800 {
 			compatible = "qcom,sm6350-dwc3", "qcom,dwc3";
 			reg = <0 0x0a6f8800 0 0x400>;
@@ -1054,6 +1218,10 @@
 
 			resets = <&gcc GCC_USB30_PRIM_BCR>;
 
+			interconnects = <&aggre2_noc MASTER_USB3 0 &clk_virt SLAVE_EBI_CH0 0>,
+					<&gem_noc MASTER_AMPSS_M0 0 &config_noc SLAVE_USB3 0>;
+			interconnect-names = "usb-ddr", "apps-usb";
+
 			usb_1_dwc3: usb@a600000 {
 				compatible = "snps,dwc3";
 				reg = <0 0x0a600000 0 0xcd00>;
@@ -1146,49 +1314,49 @@
 			#interrupt-cells = <2>;
 			gpio-ranges = <&tlmm 0 0 157>;
 
-			qup_uart9_default: qup-uart9-default {
+			qup_uart9_default: qup-uart9-default-state {
 				pins = "gpio25", "gpio26";
 				function = "qup13_f2";
 				drive-strength = <2>;
 				bias-disable;
 			};
 
-			qup_i2c0_default: qup-i2c0-default {
+			qup_i2c0_default: qup-i2c0-default-state {
 				pins = "gpio0", "gpio1";
 				function = "qup00";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c2_default: qup-i2c2-default {
+			qup_i2c2_default: qup-i2c2-default-state {
 				pins = "gpio45", "gpio46";
 				function = "qup02";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c6_default: qup-i2c6-default {
+			qup_i2c6_default: qup-i2c6-default-state {
 				pins = "gpio13", "gpio14";
 				function = "qup10";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c7_default: qup-i2c7-default {
+			qup_i2c7_default: qup-i2c7-default-state {
 				pins = "gpio27", "gpio28";
 				function = "qup11";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c8_default: qup-i2c8-default {
+			qup_i2c8_default: qup-i2c8-default-state {
 				pins = "gpio19", "gpio20";
 				function = "qup12";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c10_default: qup-i2c10-default {
+			qup_i2c10_default: qup-i2c10-default-state {
 				pins = "gpio4", "gpio5";
 				function = "qup14";
 				drive-strength = <2>;
diff --git a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts
index c76abe7587b4..30c94fd4fe61 100644
--- a/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts
+++ b/arch/arm64/boot/dts/qcom/sm7225-fairphone-fp4.dts
@@ -12,6 +12,7 @@
 #include "sm7225.dtsi"
 #include "pm6150l.dtsi"
 #include "pm6350.dtsi"
+#include "pm7250b.dtsi"
 
 / {
 	model = "Fairphone 4";
@@ -70,6 +71,36 @@
 			qcom,vmid = <15>;
 		};
 	};
+
+	thermal-zones {
+		chg-skin-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&pm7250b_adc_tm 0>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		conn-thermal {
+			polling-delay-passive = <0>;
+			polling-delay = <0>;
+			thermal-sensors = <&pm7250b_adc_tm 1>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+	};
 };
 
 &adsp {
@@ -353,6 +384,42 @@
 	linux,code = <KEY_VOLUMEDOWN>;
 };
 
+&pm7250b_adc {
+	adc-chan@4d {
+		reg = <ADC5_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "charger_skin_therm";
+	};
+
+	adc-chan@4f {
+		reg = <ADC5_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "conn_therm";
+	};
+};
+
+&pm7250b_adc_tm {
+	status = "okay";
+
+	charger-skin-therm@0 {
+		reg = <0>;
+		io-channels = <&pm7250b_adc ADC5_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	conn-therm@1 {
+		reg = <1>;
+		io-channels = <&pm7250b_adc ADC5_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+};
+
 &qupv3_id_1 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sm8150.dtsi b/arch/arm64/boot/dts/qcom/sm8150.dtsi
index 7d509ecd44da..cef8c4f4f0ff 100644
--- a/arch/arm64/boot/dts/qcom/sm8150.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8150.dtsi
@@ -585,12 +585,6 @@
 		};
 	};
 
-	tcsr_mutex: hwlock {
-		compatible = "qcom,tcsr-mutex";
-		syscon = <&tcsr_mutex_regs 0 0x1000>;
-		#hwlock-cells = <1>;
-	};
-
 	memory@80000000 {
 		device_type = "memory";
 		/* We expect the bootloader to fill in the size */
@@ -2054,9 +2048,15 @@
 			qcom,bcm-voters = <&apps_bcm_voter>;
 		};
 
-		tcsr_mutex_regs: syscon@1f40000 {
-			compatible = "syscon";
-			reg = <0x0 0x01f40000 0x0 0x40000>;
+		tcsr_mutex: hwlock@1f40000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x0 0x01f40000 0x0 0x20000>;
+			#hwlock-cells = <1>;
+		};
+
+		tcsr_regs_1: syscon@1f60000 {
+			compatible = "qcom,sm8150-tcsr", "syscon";
+			reg = <0x0 0x01f60000 0x0 0x20000>;
 		};
 
 		remoteproc_slpi: remoteproc@2400000 {
@@ -3394,57 +3394,49 @@
 					compute-cb@1 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <1>;
-						iommus = <&apps_smmu 0x1401 0x2040>,
-							 <&apps_smmu 0x1421 0x0>,
-							 <&apps_smmu 0x2001 0x420>,
-							 <&apps_smmu 0x2041 0x0>;
+						iommus = <&apps_smmu 0x1001 0x0460>;
 					};
 
 					compute-cb@2 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <2>;
-						iommus = <&apps_smmu 0x2 0x3440>,
-							 <&apps_smmu 0x22 0x3400>;
+						iommus = <&apps_smmu 0x1002 0x0460>;
 					};
 
 					compute-cb@3 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <3>;
-						iommus = <&apps_smmu 0x3 0x3440>,
-							 <&apps_smmu 0x1423 0x0>,
-							 <&apps_smmu 0x2023 0x0>;
+						iommus = <&apps_smmu 0x1003 0x0460>;
 					};
 
 					compute-cb@4 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <4>;
-						iommus = <&apps_smmu 0x4 0x3440>,
-							 <&apps_smmu 0x24 0x3400>;
+						iommus = <&apps_smmu 0x1004 0x0460>;
 					};
 
 					compute-cb@5 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <5>;
-						iommus = <&apps_smmu 0x5 0x3440>,
-							 <&apps_smmu 0x25 0x3400>;
+						iommus = <&apps_smmu 0x1005 0x0460>;
 					};
 
 					compute-cb@6 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <6>;
-						iommus = <&apps_smmu 0x6 0x3460>;
+						iommus = <&apps_smmu 0x1006 0x0460>;
 					};
 
 					compute-cb@7 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <7>;
-						iommus = <&apps_smmu 0x7 0x3460>;
+						iommus = <&apps_smmu 0x1007 0x0460>;
 					};
 
 					compute-cb@8 {
 						compatible = "qcom,fastrpc-compute-cb";
 						reg = <8>;
-						iommus = <&apps_smmu 0x8 0x3460>;
+						iommus = <&apps_smmu 0x1008 0x0460>;
 					};
 
 					/* note: secure cb9 in downstream */
diff --git a/arch/arm64/boot/dts/qcom/sm8250-mtp.dts b/arch/arm64/boot/dts/qcom/sm8250-mtp.dts
index 7ab3627cc347..a102aa5efa32 100644
--- a/arch/arm64/boot/dts/qcom/sm8250-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/sm8250-mtp.dts
@@ -635,7 +635,7 @@
 	wcd938x: codec {
 		compatible = "qcom,wcd9380-codec";
 		#sound-dai-cells = <1>;
-		reset-gpios = <&tlmm 32 0>;
+		reset-gpios = <&tlmm 32 GPIO_ACTIVE_HIGH>;
 		vdd-buck-supply = <&vreg_s4a_1p8>;
 		vdd-rxtx-supply = <&vreg_s4a_1p8>;
 		vdd-io-supply = <&vreg_s4a_1p8>;
diff --git a/arch/arm64/boot/dts/qcom/sm8250.dtsi b/arch/arm64/boot/dts/qcom/sm8250.dtsi
index bc773e210023..a5b62cadb129 100644
--- a/arch/arm64/boot/dts/qcom/sm8250.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8250.dtsi
@@ -1792,7 +1792,7 @@
 		};
 
 		pcie0: pci@1c00000 {
-			compatible = "qcom,pcie-sm8250", "snps,dw-pcie";
+			compatible = "qcom,pcie-sm8250";
 			reg = <0 0x01c00000 0 0x3000>,
 			      <0 0x60000000 0 0xf1d>,
 			      <0 0x60000f20 0 0xa8>,
@@ -1810,8 +1810,16 @@
 			ranges = <0x01000000 0x0 0x60200000 0 0x60200000 0x0 0x100000>,
 				 <0x02000000 0x0 0x60300000 0 0x60300000 0x0 0x3d00000>;
 
-			interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>;
-			interrupt-names = "msi";
+			interrupts = <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0", "msi1", "msi2", "msi3",
+					  "msi4", "msi5", "msi6", "msi7";
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
 			interrupt-map = <0 0 0 1 &intc 0 149 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
@@ -1893,7 +1901,7 @@
 		};
 
 		pcie1: pci@1c08000 {
-			compatible = "qcom,pcie-sm8250", "snps,dw-pcie";
+			compatible = "qcom,pcie-sm8250";
 			reg = <0 0x01c08000 0 0x3000>,
 			      <0 0x40000000 0 0xf1d>,
 			      <0 0x40000f20 0 0xa8>,
@@ -2001,7 +2009,7 @@
 		};
 
 		pcie2: pci@1c10000 {
-			compatible = "qcom,pcie-sm8250", "snps,dw-pcie";
+			compatible = "qcom,pcie-sm8250";
 			reg = <0 0x01c10000 0 0x3000>,
 			      <0 0x64000000 0 0xf1d>,
 			      <0 0x64000f20 0 0xa8>,
@@ -3571,6 +3579,25 @@
 						};
 					};
 				};
+
+				dsi_opp_table: opp-table {
+					compatible = "operating-points-v2";
+
+					opp-187500000 {
+						opp-hz = /bits/ 64 <187500000>;
+						required-opps = <&rpmhpd_opp_low_svs>;
+					};
+
+					opp-300000000 {
+						opp-hz = /bits/ 64 <300000000>;
+						required-opps = <&rpmhpd_opp_svs>;
+					};
+
+					opp-358000000 {
+						opp-hz = /bits/ 64 <358000000>;
+						required-opps = <&rpmhpd_opp_svs_l1>;
+					};
+				};
 			};
 
 			dsi0_phy: dsi-phy@ae94400 {
@@ -3663,25 +3690,6 @@
 				clock-names = "iface", "ref";
 
 				status = "disabled";
-
-				dsi_opp_table: opp-table {
-					compatible = "operating-points-v2";
-
-					opp-187500000 {
-						opp-hz = /bits/ 64 <187500000>;
-						required-opps = <&rpmhpd_opp_low_svs>;
-					};
-
-					opp-300000000 {
-						opp-hz = /bits/ 64 <300000000>;
-						required-opps = <&rpmhpd_opp_svs>;
-					};
-
-					opp-358000000 {
-						opp-hz = /bits/ 64 <358000000>;
-						required-opps = <&rpmhpd_opp_svs_l1>;
-					};
-				};
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi
index cb9bbd234b7b..b3c9952ac173 100644
--- a/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8350-sony-xperia-sagami.dtsi
@@ -222,8 +222,8 @@
 &tlmm {
 	gpio-reserved-ranges = <44 4>;
 
-	ts_int_default: ts-int-default {
-		pin = "gpio23";
+	ts_int_default: ts-int-default-state {
+		pins = "gpio23";
 		function = "gpio";
 		drive-strength = <2>;
 		bias-disable;
diff --git a/arch/arm64/boot/dts/qcom/sm8350.dtsi b/arch/arm64/boot/dts/qcom/sm8350.dtsi
index e72a04411888..a86d9ea93b9d 100644
--- a/arch/arm64/boot/dts/qcom/sm8350.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8350.dtsi
@@ -1762,150 +1762,150 @@
 			wakeup-parent = <&pdc>;
 
 			qup_uart3_default_state: qup-uart3-default-state {
-				rx {
+				rx-pins {
 					pins = "gpio18";
 					function = "qup3";
 				};
-				tx {
+				tx-pins {
 					pins = "gpio19";
 					function = "qup3";
 				};
 			};
 
-			qup_uart6_default: qup-uart6-default {
+			qup_uart6_default: qup-uart6-default-state {
 				pins = "gpio30", "gpio31";
 				function = "qup6";
 				drive-strength = <2>;
 				bias-disable;
 			};
 
-			qup_uart18_default: qup-uart18-default {
+			qup_uart18_default: qup-uart18-default-state {
 				pins = "gpio58", "gpio59";
 				function = "qup18";
 				drive-strength = <2>;
 				bias-disable;
 			};
 
-			qup_i2c0_default: qup-i2c0-default {
+			qup_i2c0_default: qup-i2c0-default-state {
 				pins = "gpio4", "gpio5";
 				function = "qup0";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c1_default: qup-i2c1-default {
+			qup_i2c1_default: qup-i2c1-default-state {
 				pins = "gpio8", "gpio9";
 				function = "qup1";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c2_default: qup-i2c2-default {
+			qup_i2c2_default: qup-i2c2-default-state {
 				pins = "gpio12", "gpio13";
 				function = "qup2";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c4_default: qup-i2c4-default {
+			qup_i2c4_default: qup-i2c4-default-state {
 				pins = "gpio20", "gpio21";
 				function = "qup4";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c5_default: qup-i2c5-default {
+			qup_i2c5_default: qup-i2c5-default-state {
 				pins = "gpio24", "gpio25";
 				function = "qup5";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c6_default: qup-i2c6-default {
+			qup_i2c6_default: qup-i2c6-default-state {
 				pins = "gpio28", "gpio29";
 				function = "qup6";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c7_default: qup-i2c7-default {
+			qup_i2c7_default: qup-i2c7-default-state {
 				pins = "gpio32", "gpio33";
 				function = "qup7";
 				drive-strength = <2>;
 				bias-disable;
 			};
 
-			qup_i2c8_default: qup-i2c8-default {
+			qup_i2c8_default: qup-i2c8-default-state {
 				pins = "gpio36", "gpio37";
 				function = "qup8";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c9_default: qup-i2c9-default {
+			qup_i2c9_default: qup-i2c9-default-state {
 				pins = "gpio40", "gpio41";
 				function = "qup9";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c10_default: qup-i2c10-default {
+			qup_i2c10_default: qup-i2c10-default-state {
 				pins = "gpio44", "gpio45";
 				function = "qup10";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c11_default: qup-i2c11-default {
+			qup_i2c11_default: qup-i2c11-default-state {
 				pins = "gpio48", "gpio49";
 				function = "qup11";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c12_default: qup-i2c12-default {
+			qup_i2c12_default: qup-i2c12-default-state {
 				pins = "gpio52", "gpio53";
 				function = "qup12";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c13_default: qup-i2c13-default {
+			qup_i2c13_default: qup-i2c13-default-state {
 				pins = "gpio0", "gpio1";
 				function = "qup13";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c14_default: qup-i2c14-default {
+			qup_i2c14_default: qup-i2c14-default-state {
 				pins = "gpio56", "gpio57";
 				function = "qup14";
 				drive-strength = <2>;
 				bias-disable;
 			};
 
-			qup_i2c15_default: qup-i2c15-default {
+			qup_i2c15_default: qup-i2c15-default-state {
 				pins = "gpio60", "gpio61";
 				function = "qup15";
 				drive-strength = <2>;
 				bias-disable;
 			};
 
-			qup_i2c16_default: qup-i2c16-default {
+			qup_i2c16_default: qup-i2c16-default-state {
 				pins = "gpio64", "gpio65";
 				function = "qup16";
 				drive-strength = <2>;
 				bias-disable;
 			};
 
-			qup_i2c17_default: qup-i2c17-default {
+			qup_i2c17_default: qup-i2c17-default-state {
 				pins = "gpio72", "gpio73";
 				function = "qup17";
 				drive-strength = <2>;
 				bias-disable;
 			};
 
-			qup_i2c19_default: qup-i2c19-default {
+			qup_i2c19_default: qup-i2c19-default-state {
 				pins = "gpio76", "gpio77";
 				function = "qup19";
 				drive-strength = <2>;
@@ -2128,7 +2128,7 @@
 
 		ufs_mem_phy: phy@1d87000 {
 			compatible = "qcom,sm8350-qmp-ufs-phy";
-			reg = <0 0x01d87000 0 0xe10>;
+			reg = <0 0x01d87000 0 0x1c4>;
 			#address-cells = <2>;
 			#size-cells = <2>;
 			ranges;
diff --git a/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara-pdx223.dts b/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara-pdx223.dts
new file mode 100644
index 000000000000..d68765eb6d4f
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sm8450-sony-xperia-nagara-pdx223.dts
@@ -0,0 +1,634 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) 2022, Konrad Dybcio <konrad.dybcio@somainline.org>
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+#include "sm8450.dtsi"
+
+/delete-node/ &adsp_mem;
+/delete-node/ &rmtfs_mem;
+/delete-node/ &video_mem;
+
+/ {
+	model = "Sony Xperia 1 IV";
+	compatible = "sony,pdx223", "qcom,sm8450";
+	chassis-type = "handset";
+
+	aliases {
+		serial0 = &uart7;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	reserved-memory {
+		adsp_mem: memory@85700000 {
+			reg = <0x0 0x85700000 0x0 0x2800000>;
+			no-map;
+		};
+
+		video_mem: memory@9fd00000 {
+			reg = <0x0 0x9fd00000 0x0 0x700000>;
+			no-map;
+		};
+
+		rmtfs_mem: memory@f3300000 {
+			compatible = "qcom,rmtfs-mem";
+			reg = <0x0 0xf3300000 0x0 0x280000>;
+			no-map;
+
+			qcom,client-id = <1>;
+			qcom,vmid = <15>;
+		};
+
+		ramoops@ffc00000 {
+			compatible = "ramoops";
+			reg = <0 0xffc00000 0 0x200000>;
+			console-size = <0x40000>;
+			record-size = <0x1000>;
+			ecc-size = <16>;
+			no-map;
+		};
+	};
+
+	/* Sadly, the voltages for these GPIO regulators are unknown. */
+	imx650_vana_vreg: imx650-vana-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "imx650_vana_vreg";
+		gpio = <&tlmm 23 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vph_pwr: vph-pwr-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&apps_rsc {
+	pm8350-rpmh-regulators {
+		compatible = "qcom,pm8350-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+		vdd-s9-supply = <&vph_pwr>;
+		vdd-s10-supply = <&vph_pwr>;
+		vdd-s11-supply = <&vph_pwr>;
+		vdd-s12-supply = <&vph_pwr>;
+
+		vdd-l1-l4-supply = <&pm8350_s11>;
+		vdd-l2-l7-supply = <&vreg_bob>;
+		vdd-l3-l5-supply = <&vreg_bob>;
+		vdd-l6-l9-l10-supply = <&pm8350_s12>;
+
+		/*
+		 * ARC regulators:
+		 * s5 - gfx.lvl
+		 * l8 - lcx.lvl
+		 */
+
+		pm8350_s10: smps10 {
+			regulator-name = "pm8350_s10";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+		};
+
+		pm8350_s11: smps11 {
+			regulator-name = "pm8350_s11";
+			regulator-min-microvolt = <848000>;
+			regulator-max-microvolt = <1104000>;
+		};
+
+		pm8350_s12: smps12 {
+			regulator-name = "pm8350_s12";
+			regulator-min-microvolt = <1224000>;
+			regulator-max-microvolt = <1400000>;
+		};
+
+		pm8350_l1: ldo1 {
+			regulator-name = "pm8350_l1";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <920000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350_l2: ldo2 {
+			regulator-name = "pm8350_l2";
+			regulator-min-microvolt = <3072000>;
+			regulator-max-microvolt = <3072000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350_l3: ldo3 {
+			regulator-name = "pm8350_l3";
+			regulator-min-microvolt = <904000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350_l5: ldo5 {
+			regulator-name = "pm8350_l5";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350_l6: ldo6 {
+			regulator-name = "pm8350_l6";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350_l7: ldo7 {
+			regulator-name = "pm8350_l7";
+			regulator-min-microvolt = <2504000>;
+			regulator-max-microvolt = <2504000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350_l9: ldo9 {
+			regulator-name = "pm8350_l9";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	pm8350c-rpmh-regulators {
+		compatible = "qcom,pm8350c-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+		vdd-s9-supply = <&vph_pwr>;
+		vdd-s10-supply = <&vph_pwr>;
+
+		vdd-l1-l12-supply = <&vreg_bob>;
+		vdd-l2-l8-supply = <&vreg_bob>;
+		vdd-l3-l4-l5-l7-l13-supply = <&vreg_bob>;
+		vdd-l6-l9-l11-supply = <&vreg_bob>;
+		vdd-l10-supply = <&pm8350_s12>;
+
+		vdd-bob-supply = <&vph_pwr>;
+
+		/*
+		 * ARC regulators:
+		 * s2 - mxc.lvl
+		 * s4 - mss.lvl
+		 * s6 - cx.lvl
+		 */
+
+		pm8350c_s1: smps1 {
+			regulator-name = "pm8350c_s1";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2024000>;
+		};
+
+		pm8350c_s10: smps10 {
+			regulator-name = "pm8350c_s10";
+			regulator-min-microvolt = <1000000>;
+			regulator-max-microvolt = <1100000>;
+		};
+
+		vreg_bob: bob {
+			regulator-name = "vreg_bob";
+			regulator-min-microvolt = <3400000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+		};
+
+		pm8350c_l1: ldo1 {
+			regulator-name = "pm8350c_l1";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l2: ldo2 {
+			regulator-name = "pm8350c_l2";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l3: ldo3 {
+			regulator-name = "pm8350c_l3";
+			regulator-min-microvolt = <3296000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l4: ldo4 {
+			regulator-name = "pm8350c_l4";
+			regulator-min-microvolt = <1704000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l5: ldo5 {
+			regulator-name = "pm8350c_l5";
+			regulator-min-microvolt = <1704000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l6: ldo6 {
+			regulator-name = "pm8350c_l6";
+			regulator-min-microvolt = <2960000>;
+			/* Originally max = 3008000 but SDHCI expects 2960000 */
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l7: ldo7 {
+			regulator-name = "pm8350c_l7";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l8: ldo8 {
+			regulator-name = "pm8350c_l8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l9: ldo9 {
+			regulator-name = "pm8350c_l9";
+			regulator-min-microvolt = <2960000>;
+			/* Originally max = 3008000 but SDHCI expects 2960000 */
+			regulator-max-microvolt = <2960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l10: ldo10 {
+			regulator-name = "pm8350c_l10";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l12: ldo12 {
+			regulator-name = "pm8350c_l12";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1968000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8350c_l13: ldo13 {
+			regulator-name = "pm8350c_l13";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	pm8450-rpmh-regulators {
+		compatible = "qcom,pm8450-rpmh-regulators";
+		qcom,pmic-id = "h";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+
+		vdd-l2-supply = <&vreg_bob>;
+		vdd-l3-supply = <&vreg_bob>;
+		vdd-l4-supply = <&vreg_bob>;
+
+		/*
+		 * ARC regulators:
+		 * S2 - ebi.lvl
+		 * S4 - mmcx.lvl
+		 * S6 - mx.lvl
+		 * L1 - lmx.lvl
+		 */
+
+		pm8450_s3: smps3 {
+			regulator-name = "pm8450_s3";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <600000>;
+		};
+
+		pm8450_l2: ldo2 {
+			regulator-name = "pm8450_l2";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		pm8450_l3: ldo3 {
+			regulator-name = "pm8450_l3";
+			regulator-min-microvolt = <912000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	pmr735a-rpmh-regulators {
+		compatible = "qcom,pmr735a-rpmh-regulators";
+		qcom,pmic-id = "e";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+
+		vdd-l1-l2-supply = <&pmr735a_s2>;
+		vdd-l3-supply = <&pmr735a_s1>;
+		vdd-l4-supply = <&pm8350c_s1>;
+		vdd-l5-l6-supply = <&pm8350c_s1>;
+		vdd-l7-bob-supply = <&vreg_bob>;
+
+		pmr735a_s1: smps1 {
+			regulator-name = "pmr735a_s1";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1296000>;
+		};
+
+		pmr735a_s2: smps2 {
+			regulator-name = "pmr735a_s2";
+			regulator-min-microvolt = <500000>;
+			regulator-max-microvolt = <1040000>;
+		};
+
+		pmr735a_s3: smps3 {
+			regulator-name = "pmr735a_s3";
+			regulator-min-microvolt = <435000>;
+			regulator-max-microvolt = <2352000>;
+		};
+
+		pmr735a_l1: ldo1 {
+			regulator-name = "pmr735a_l1";
+			regulator-min-microvolt = <800000>;
+			regulator-max-microvolt = <800000>;
+		};
+
+		pmr735a_l2: ldo2 {
+			regulator-name = "pmr735a_l2";
+			regulator-min-microvolt = <480000>;
+			regulator-max-microvolt = <912000>;
+		};
+
+		pmr735a_l3: ldo3 {
+			regulator-name = "pmr735a_l3";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pmr735a_l4: ldo4 {
+			regulator-name = "pmr735a_l4";
+			regulator-min-microvolt = <1776000>;
+			regulator-max-microvolt = <1776000>;
+		};
+
+		pmr735a_l5: ldo5 {
+			regulator-name = "pmr735a_l5";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <880000>;
+		};
+
+		pmr735a_l6: ldo6 {
+			regulator-name = "pmr735a_l6";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+		};
+
+		pmr735a_l7: ldo7 {
+			regulator-name = "pmr735a_l7";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+		};
+	};
+};
+
+&gpi_dma0 {
+	status = "okay";
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpi_dma2 {
+	status = "okay";
+};
+
+/* I2C4 is used, it hosts a Samsung touchscreen, but GPI DMA is broken.. */
+
+&i2c5 {
+	clock-frequency = <400000>;
+	status = "okay";
+
+	/* Dialog SLG51000 CMIC @ 75 */
+};
+
+&i2c9 {
+	clock-frequency = <400000>;
+	status = "okay";
+
+	/* NXP SN1X0 NFC @ 28 */
+};
+
+&i2c13 {
+	clock-frequency = <400000>;
+	status = "okay";
+
+	/* Richwave RTC6226 FM Radio Receiver @ 64 */
+};
+
+&i2c14 {
+	clock-frequency = <1000000>;
+	status = "okay";
+
+	cs35l41_l: speaker-amp@40 {
+		compatible = "cirrus,cs35l41";
+		reg = <0x40>;
+		interrupt-parent = <&tlmm>;
+		interrupts = <182 IRQ_TYPE_LEVEL_LOW>;
+		reset-gpios = <&tlmm 183 GPIO_ACTIVE_HIGH>;
+		cirrus,boost-peak-milliamp = <4000>;
+		cirrus,boost-ind-nanohenry = <1000>;
+		cirrus,boost-cap-microfarad = <15>;
+		cirrus,gpio2-src-select = <2>;
+		cirrus,gpio2-output-enable;
+		cirrus,asp-sdout-hiz = <3>;
+		#sound-dai-cells = <1>;
+	};
+
+	cs35l41_r: speaker-amp@41 {
+		compatible = "cirrus,cs35l41";
+		reg = <0x41>;
+		interrupt-parent = <&tlmm>;
+		interrupts = <182 IRQ_TYPE_LEVEL_LOW>;
+		reset-gpios = <&tlmm 183 GPIO_ACTIVE_HIGH>;
+		cirrus,boost-peak-milliamp = <4000>;
+		cirrus,boost-ind-nanohenry = <1000>;
+		cirrus,boost-cap-microfarad = <15>;
+		cirrus,gpio2-src-select = <2>;
+		cirrus,gpio2-output-enable;
+		cirrus,asp-sdout-hiz = <3>;
+		#sound-dai-cells = <1>;
+	};
+};
+
+&i2c15 {
+	clock-frequency = <400000>;
+	status = "okay";
+
+	/* AMS TCS3490 RGB+IR color sensor @ 72 */
+};
+
+&i2c19 {
+	clock-frequency = <1000000>;
+	status = "okay";
+
+	/* Cirrus Logic CS40L25A boosted haptics driver @ 40 */
+};
+
+&pcie0 {
+	max-link-speed = <2>;
+	status = "okay";
+};
+
+&pcie0_phy {
+	vdda-phy-supply = <&pm8350_l5>;
+	vdda-pll-supply = <&pm8350_l6>;
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/adsp.mbn";
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/cdsp.mbn";
+	status = "okay";
+};
+
+&remoteproc_slpi {
+	firmware-name = "qcom/slpi.mbn";
+	status = "okay";
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	status = "okay";
+};
+
+&qupv3_id_2 {
+	status = "okay";
+};
+
+&sdhc_2 {
+	cd-gpios = <&tlmm 92 GPIO_ACTIVE_HIGH>;
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&sdc2_default_state &sdc2_card_det_n>;
+	pinctrl-1 = <&sdc2_sleep_state &sdc2_card_det_n>;
+	vmmc-supply = <&pm8350c_l9>;
+	vqmmc-supply = <&pm8350c_l6>;
+	/* Forbid SDR104/SDR50 - broken hw! */
+	sdhci-caps-mask = <0x3 0x0>;
+	no-sdio;
+	no-mmc;
+	status = "okay";
+};
+
+&spi10 {
+	status = "okay";
+
+	/* NXP SN1X0 NFC Secure Element @ 0 */
+};
+
+&tlmm {
+	gpio-reserved-ranges = <28 4>;
+
+	sdc2_default_state: sdc2-default-state {
+		clk-pins {
+			pins = "sdc2_clk";
+			drive-strength = <16>;
+			bias-disable;
+		};
+
+		cmd-pins {
+			pins = "sdc2_cmd";
+			drive-strength = <16>;
+			bias-pull-up;
+		};
+
+		data-pins {
+			pins = "sdc2_data";
+			drive-strength = <16>;
+			bias-pull-up;
+		};
+	};
+
+	ts_int_default: ts-int-default-state {
+		pins = "gpio23";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-disable;
+		input-enable;
+	};
+
+	sdc2_card_det_n: sd-card-det-n-state {
+		pins = "gpio92";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+};
+
+&uart7 {
+	status = "okay";
+};
+
+&usb_1 {
+	status = "okay";
+};
+
+&usb_1_dwc3 {
+	dr_mode = "peripheral";
+};
+
+&usb_1_hsphy {
+	vdda-pll-supply = <&pm8350_l5>;
+	vdda18-supply = <&pm8350c_l1>;
+	vdda33-supply = <&pm8350_l2>;
+	status = "okay";
+};
+
+&usb_1_qmpphy {
+	vdda-phy-supply = <&pm8350_l6>;
+	vdda-pll-supply = <&pm8350_l1>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi
index 4978c5ba5dd0..d32f08df743d 100644
--- a/arch/arm64/boot/dts/qcom/sm8450.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi
@@ -438,11 +438,6 @@
 			no-map;
 		};
 
-		camera_mem: memory@85200000 {
-			reg = <0x0 0x85200000 0x0 0x500000>;
-			no-map;
-		};
-
 		video_mem: memory@85700000 {
 			reg = <0x0 0x85700000 0x0 0x700000>;
 			no-map;
@@ -505,6 +500,11 @@
 			no-map;
 		};
 
+		camera_mem: memory@9f500000 {
+			reg = <0x0 0x9f500000 0x0 0x800000>;
+			no-map;
+		};
+
 		rmtfs_mem: memory@9fd00000 {
 			compatible = "qcom,rmtfs-mem";
 			reg = <0x0 0x9fd00000 0x0 0x280000>;
@@ -514,6 +514,11 @@
 			qcom,vmid = <15>;
 		};
 
+		xbl_sc_mem2: memory@a6e00000 {
+			reg = <0x0 0xa6e00000 0x0 0x40000>;
+			no-map;
+		};
+
 		global_sync_mem: memory@a6f00000 {
 			reg = <0x0 0xa6f00000 0x0 0x100000>;
 			no-map;
@@ -2379,22 +2384,42 @@
 			gpio-ranges = <&tlmm 0 0 211>;
 			wakeup-parent = <&pdc>;
 
+			sdc2_sleep_state: sdc2-sleep-state {
+				clk-pins {
+					pins = "sdc2_clk";
+					drive-strength = <2>;
+					bias-disable;
+				};
+
+				cmd-pins {
+					pins = "sdc2_cmd";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+
+				data-pins {
+					pins = "sdc2_data";
+					drive-strength = <2>;
+					bias-pull-up;
+				};
+			};
+
 			pcie0_default_state: pcie0-default-state {
-				perst {
+				perst-pins {
 					pins = "gpio94";
 					function = "gpio";
 					drive-strength = <2>;
 					bias-pull-down;
 				};
 
-				clkreq {
+				clkreq-pins {
 					pins = "gpio95";
 					function = "pcie0_clkreqn";
 					drive-strength = <2>;
 					bias-pull-up;
 				};
 
-				wake {
+				wake-pins {
 					pins = "gpio96";
 					function = "gpio";
 					drive-strength = <2>;
@@ -2403,21 +2428,21 @@
 			};
 
 			pcie1_default_state: pcie1-default-state {
-				perst {
+				perst-pins {
 					pins = "gpio97";
 					function = "gpio";
 					drive-strength = <2>;
 					bias-pull-down;
 				};
 
-				clkreq {
+				clkreq-pins {
 					pins = "gpio98";
 					function = "pcie1_clkreqn";
 					drive-strength = <2>;
 					bias-pull-up;
 				};
 
-				wake {
+				wake-pins {
 					pins = "gpio99";
 					function = "gpio";
 					drive-strength = <2>;
@@ -2425,350 +2450,350 @@
 				};
 			};
 
-			qup_i2c0_data_clk: qup-i2c0-data-clk {
+			qup_i2c0_data_clk: qup-i2c0-data-clk-state {
 				pins = "gpio0", "gpio1";
 				function = "qup0";
 			};
 
-			qup_i2c1_data_clk: qup-i2c1-data-clk {
+			qup_i2c1_data_clk: qup-i2c1-data-clk-state {
 				pins = "gpio4", "gpio5";
 				function = "qup1";
 			};
 
-			qup_i2c2_data_clk: qup-i2c2-data-clk {
+			qup_i2c2_data_clk: qup-i2c2-data-clk-state {
 				pins = "gpio8", "gpio9";
 				function = "qup2";
 			};
 
-			qup_i2c3_data_clk: qup-i2c3-data-clk {
+			qup_i2c3_data_clk: qup-i2c3-data-clk-state {
 				pins = "gpio12", "gpio13";
 				function = "qup3";
 			};
 
-			qup_i2c4_data_clk: qup-i2c4-data-clk {
+			qup_i2c4_data_clk: qup-i2c4-data-clk-state {
 				pins = "gpio16", "gpio17";
 				function = "qup4";
 			};
 
-			qup_i2c5_data_clk: qup-i2c5-data-clk {
+			qup_i2c5_data_clk: qup-i2c5-data-clk-state {
 				pins = "gpio206", "gpio207";
 				function = "qup5";
 			};
 
-			qup_i2c6_data_clk: qup-i2c6-data-clk {
+			qup_i2c6_data_clk: qup-i2c6-data-clk-state {
 				pins = "gpio20", "gpio21";
 				function = "qup6";
 			};
 
-			qup_i2c8_data_clk: qup-i2c8-data-clk {
+			qup_i2c8_data_clk: qup-i2c8-data-clk-state {
 				pins = "gpio28", "gpio29";
 				function = "qup8";
 			};
 
-			qup_i2c9_data_clk: qup-i2c9-data-clk {
+			qup_i2c9_data_clk: qup-i2c9-data-clk-state {
 				pins = "gpio32", "gpio33";
 				function = "qup9";
 			};
 
-			qup_i2c10_data_clk: qup-i2c10-data-clk {
+			qup_i2c10_data_clk: qup-i2c10-data-clk-state {
 				pins = "gpio36", "gpio37";
 				function = "qup10";
 			};
 
-			qup_i2c11_data_clk: qup-i2c11-data-clk {
+			qup_i2c11_data_clk: qup-i2c11-data-clk-state {
 				pins = "gpio40", "gpio41";
 				function = "qup11";
 			};
 
-			qup_i2c12_data_clk: qup-i2c12-data-clk {
+			qup_i2c12_data_clk: qup-i2c12-data-clk-state {
 				pins = "gpio44", "gpio45";
 				function = "qup12";
 			};
 
-			qup_i2c13_data_clk: qup-i2c13-data-clk {
+			qup_i2c13_data_clk: qup-i2c13-data-clk-state {
 				pins = "gpio48", "gpio49";
 				function = "qup13";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c14_data_clk: qup-i2c14-data-clk {
+			qup_i2c14_data_clk: qup-i2c14-data-clk-state {
 				pins = "gpio52", "gpio53";
 				function = "qup14";
 				drive-strength = <2>;
 				bias-pull-up;
 			};
 
-			qup_i2c15_data_clk: qup-i2c15-data-clk {
+			qup_i2c15_data_clk: qup-i2c15-data-clk-state {
 				pins = "gpio56", "gpio57";
 				function = "qup15";
 			};
 
-			qup_i2c16_data_clk: qup-i2c16-data-clk {
+			qup_i2c16_data_clk: qup-i2c16-data-clk-state {
 				pins = "gpio60", "gpio61";
 				function = "qup16";
 			};
 
-			qup_i2c17_data_clk: qup-i2c17-data-clk {
+			qup_i2c17_data_clk: qup-i2c17-data-clk-state {
 				pins = "gpio64", "gpio65";
 				function = "qup17";
 			};
 
-			qup_i2c18_data_clk: qup-i2c18-data-clk {
+			qup_i2c18_data_clk: qup-i2c18-data-clk-state {
 				pins = "gpio68", "gpio69";
 				function = "qup18";
 			};
 
-			qup_i2c19_data_clk: qup-i2c19-data-clk {
+			qup_i2c19_data_clk: qup-i2c19-data-clk-state {
 				pins = "gpio72", "gpio73";
 				function = "qup19";
 			};
 
-			qup_i2c20_data_clk: qup-i2c20-data-clk {
+			qup_i2c20_data_clk: qup-i2c20-data-clk-state {
 				pins = "gpio76", "gpio77";
 				function = "qup20";
 			};
 
-			qup_i2c21_data_clk: qup-i2c21-data-clk {
+			qup_i2c21_data_clk: qup-i2c21-data-clk-state {
 				pins = "gpio80", "gpio81";
 				function = "qup21";
 			};
 
-			qup_spi0_cs: qup-spi0-cs {
+			qup_spi0_cs: qup-spi0-cs-state {
 				pins = "gpio3";
 				function = "qup0";
 			};
 
-			qup_spi0_data_clk: qup-spi0-data-clk {
+			qup_spi0_data_clk: qup-spi0-data-clk-state {
 				pins = "gpio0", "gpio1", "gpio2";
 				function = "qup0";
 			};
 
-			qup_spi1_cs: qup-spi1-cs {
+			qup_spi1_cs: qup-spi1-cs-state {
 				pins = "gpio7";
 				function = "qup1";
 			};
 
-			qup_spi1_data_clk: qup-spi1-data-clk {
+			qup_spi1_data_clk: qup-spi1-data-clk-state {
 				pins = "gpio4", "gpio5", "gpio6";
 				function = "qup1";
 			};
 
-			qup_spi2_cs: qup-spi2-cs {
+			qup_spi2_cs: qup-spi2-cs-state {
 				pins = "gpio11";
 				function = "qup2";
 			};
 
-			qup_spi2_data_clk: qup-spi2-data-clk {
+			qup_spi2_data_clk: qup-spi2-data-clk-state {
 				pins = "gpio8", "gpio9", "gpio10";
 				function = "qup2";
 			};
 
-			qup_spi3_cs: qup-spi3-cs {
+			qup_spi3_cs: qup-spi3-cs-state {
 				pins = "gpio15";
 				function = "qup3";
 			};
 
-			qup_spi3_data_clk: qup-spi3-data-clk {
+			qup_spi3_data_clk: qup-spi3-data-clk-state {
 				pins = "gpio12", "gpio13", "gpio14";
 				function = "qup3";
 			};
 
-			qup_spi4_cs: qup-spi4-cs {
+			qup_spi4_cs: qup-spi4-cs-state {
 				pins = "gpio19";
 				function = "qup4";
 				drive-strength = <6>;
 				bias-disable;
 			};
 
-			qup_spi4_data_clk: qup-spi4-data-clk {
+			qup_spi4_data_clk: qup-spi4-data-clk-state {
 				pins = "gpio16", "gpio17", "gpio18";
 				function = "qup4";
 			};
 
-			qup_spi5_cs: qup-spi5-cs {
+			qup_spi5_cs: qup-spi5-cs-state {
 				pins = "gpio85";
 				function = "qup5";
 			};
 
-			qup_spi5_data_clk: qup-spi5-data-clk {
+			qup_spi5_data_clk: qup-spi5-data-clk-state {
 				pins = "gpio206", "gpio207", "gpio84";
 				function = "qup5";
 			};
 
-			qup_spi6_cs: qup-spi6-cs {
+			qup_spi6_cs: qup-spi6-cs-state {
 				pins = "gpio23";
 				function = "qup6";
 			};
 
-			qup_spi6_data_clk: qup-spi6-data-clk {
+			qup_spi6_data_clk: qup-spi6-data-clk-state {
 				pins = "gpio20", "gpio21", "gpio22";
 				function = "qup6";
 			};
 
-			qup_spi8_cs: qup-spi8-cs {
+			qup_spi8_cs: qup-spi8-cs-state {
 				pins = "gpio31";
 				function = "qup8";
 			};
 
-			qup_spi8_data_clk: qup-spi8-data-clk {
+			qup_spi8_data_clk: qup-spi8-data-clk-state {
 				pins = "gpio28", "gpio29", "gpio30";
 				function = "qup8";
 			};
 
-			qup_spi9_cs: qup-spi9-cs {
+			qup_spi9_cs: qup-spi9-cs-state {
 				pins = "gpio35";
 				function = "qup9";
 			};
 
-			qup_spi9_data_clk: qup-spi9-data-clk {
+			qup_spi9_data_clk: qup-spi9-data-clk-state {
 				pins = "gpio32", "gpio33", "gpio34";
 				function = "qup9";
 			};
 
-			qup_spi10_cs: qup-spi10-cs {
+			qup_spi10_cs: qup-spi10-cs-state {
 				pins = "gpio39";
 				function = "qup10";
 			};
 
-			qup_spi10_data_clk: qup-spi10-data-clk {
+			qup_spi10_data_clk: qup-spi10-data-clk-state {
 				pins = "gpio36", "gpio37", "gpio38";
 				function = "qup10";
 			};
 
-			qup_spi11_cs: qup-spi11-cs {
+			qup_spi11_cs: qup-spi11-cs-state {
 				pins = "gpio43";
 				function = "qup11";
 			};
 
-			qup_spi11_data_clk: qup-spi11-data-clk {
+			qup_spi11_data_clk: qup-spi11-data-clk-state {
 				pins = "gpio40", "gpio41", "gpio42";
 				function = "qup11";
 			};
 
-			qup_spi12_cs: qup-spi12-cs {
+			qup_spi12_cs: qup-spi12-cs-state {
 				pins = "gpio47";
 				function = "qup12";
 			};
 
-			qup_spi12_data_clk: qup-spi12-data-clk {
+			qup_spi12_data_clk: qup-spi12-data-clk-state {
 				pins = "gpio44", "gpio45", "gpio46";
 				function = "qup12";
 			};
 
-			qup_spi13_cs: qup-spi13-cs {
+			qup_spi13_cs: qup-spi13-cs-state {
 				pins = "gpio51";
 				function = "qup13";
 			};
 
-			qup_spi13_data_clk: qup-spi13-data-clk {
+			qup_spi13_data_clk: qup-spi13-data-clk-state {
 				pins = "gpio48", "gpio49", "gpio50";
 				function = "qup13";
 			};
 
-			qup_spi14_cs: qup-spi14-cs {
+			qup_spi14_cs: qup-spi14-cs-state {
 				pins = "gpio55";
 				function = "qup14";
 			};
 
-			qup_spi14_data_clk: qup-spi14-data-clk {
+			qup_spi14_data_clk: qup-spi14-data-clk-state {
 				pins = "gpio52", "gpio53", "gpio54";
 				function = "qup14";
 			};
 
-			qup_spi15_cs: qup-spi15-cs {
+			qup_spi15_cs: qup-spi15-cs-state {
 				pins = "gpio59";
 				function = "qup15";
 			};
 
-			qup_spi15_data_clk: qup-spi15-data-clk {
+			qup_spi15_data_clk: qup-spi15-data-clk-state {
 				pins = "gpio56", "gpio57", "gpio58";
 				function = "qup15";
 			};
 
-			qup_spi16_cs: qup-spi16-cs {
+			qup_spi16_cs: qup-spi16-cs-state {
 				pins = "gpio63";
 				function = "qup16";
 			};
 
-			qup_spi16_data_clk: qup-spi16-data-clk {
+			qup_spi16_data_clk: qup-spi16-data-clk-state {
 				pins = "gpio60", "gpio61", "gpio62";
 				function = "qup16";
 			};
 
-			qup_spi17_cs: qup-spi17-cs {
+			qup_spi17_cs: qup-spi17-cs-state {
 				pins = "gpio67";
 				function = "qup17";
 			};
 
-			qup_spi17_data_clk: qup-spi17-data-clk {
+			qup_spi17_data_clk: qup-spi17-data-clk-state {
 				pins = "gpio64", "gpio65", "gpio66";
 				function = "qup17";
 			};
 
-			qup_spi18_cs: qup-spi18-cs {
+			qup_spi18_cs: qup-spi18-cs-state {
 				pins = "gpio71";
 				function = "qup18";
 				drive-strength = <6>;
 				bias-disable;
 			};
 
-			qup_spi18_data_clk: qup-spi18-data-clk {
+			qup_spi18_data_clk: qup-spi18-data-clk-state {
 				pins = "gpio68", "gpio69", "gpio70";
 				function = "qup18";
 				drive-strength = <6>;
 				bias-disable;
 			};
 
-			qup_spi19_cs: qup-spi19-cs {
+			qup_spi19_cs: qup-spi19-cs-state {
 				pins = "gpio75";
 				function = "qup19";
 				drive-strength = <6>;
 				bias-disable;
 			};
 
-			qup_spi19_data_clk: qup-spi19-data-clk {
+			qup_spi19_data_clk: qup-spi19-data-clk-state {
 				pins = "gpio72", "gpio73", "gpio74";
 				function = "qup19";
 				drive-strength = <6>;
 				bias-disable;
 			};
 
-			qup_spi20_cs: qup-spi20-cs {
+			qup_spi20_cs: qup-spi20-cs-state {
 				pins = "gpio79";
 				function = "qup20";
 			};
 
-			qup_spi20_data_clk: qup-spi20-data-clk {
+			qup_spi20_data_clk: qup-spi20-data-clk-state {
 				pins = "gpio76", "gpio77", "gpio78";
 				function = "qup20";
 			};
 
-			qup_spi21_cs: qup-spi21-cs {
+			qup_spi21_cs: qup-spi21-cs-state {
 				pins = "gpio83";
 				function = "qup21";
 			};
 
-			qup_spi21_data_clk: qup-spi21-data-clk {
+			qup_spi21_data_clk: qup-spi21-data-clk-state {
 				pins = "gpio80", "gpio81", "gpio82";
 				function = "qup21";
 			};
 
-			qup_uart7_rx: qup-uart7-rx {
+			qup_uart7_rx: qup-uart7-rx-state {
 				pins = "gpio26";
 				function = "qup7";
 				drive-strength = <2>;
 				bias-disable;
 			};
 
-			qup_uart7_tx: qup-uart7-tx {
+			qup_uart7_tx: qup-uart7-tx-state {
 				pins = "gpio27";
 				function = "qup7";
 				drive-strength = <2>;
 				bias-disable;
 			};
 
-			qup_uart20_default: qup-uart20-default {
+			qup_uart20_default: qup-uart20-default-state {
 				pins = "gpio76", "gpio77", "gpio78", "gpio79";
 				function = "qup20";
 			};
@@ -3069,7 +3094,9 @@
 		ufs_mem_hc: ufshc@1d84000 {
 			compatible = "qcom,sm8450-ufshc", "qcom,ufshc",
 				     "jedec,ufs-2.0";
-			reg = <0 0x01d84000 0 0x3000>;
+			reg = <0 0x01d84000 0 0x3000>,
+			      <0 0x01d88000 0 0x8000>;
+			reg-names = "std", "ice";
 			interrupts = <GIC_SPI 265 IRQ_TYPE_LEVEL_HIGH>;
 			phys = <&ufs_mem_phy_lanes>;
 			phy-names = "ufsphy";
@@ -3093,7 +3120,8 @@
 				"ref_clk",
 				"tx_lane0_sync_clk",
 				"rx_lane0_sync_clk",
-				"rx_lane1_sync_clk";
+				"rx_lane1_sync_clk",
+				"ice_core_clk";
 			clocks =
 				<&gcc GCC_UFS_PHY_AXI_CLK>,
 				<&gcc GCC_AGGRE_UFS_PHY_AXI_CLK>,
@@ -3102,7 +3130,8 @@
 				<&rpmhcc RPMH_CXO_CLK>,
 				<&gcc GCC_UFS_PHY_TX_SYMBOL_0_CLK>,
 				<&gcc GCC_UFS_PHY_RX_SYMBOL_0_CLK>,
-				<&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>;
+				<&gcc GCC_UFS_PHY_RX_SYMBOL_1_CLK>,
+				<&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
 			freq-table-hz =
 				<75000000 300000000>,
 				<0 0>,
@@ -3111,13 +3140,14 @@
 				<75000000 300000000>,
 				<0 0>,
 				<0 0>,
-				<0 0>;
+				<0 0>,
+				<75000000 300000000>;
 			status = "disabled";
 		};
 
 		ufs_mem_phy: phy@1d87000 {
 			compatible = "qcom,sm8450-qmp-ufs-phy";
-			reg = <0 0x01d87000 0 0xe10>;
+			reg = <0 0x01d87000 0 0x1c4>;
 			#address-cells = <2>;
 			#size-cells = <2>;
 			ranges;
@@ -3140,6 +3170,45 @@
 			};
 		};
 
+		sdhc_2: sdhci@8804000 {
+			compatible = "qcom,sm8450-sdhci", "qcom,sdhci-msm-v5";
+			reg = <0 0x08804000 0 0x1000>;
+
+			interrupts = <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 223 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "hc_irq", "pwr_irq";
+
+			clocks = <&gcc GCC_SDCC2_AHB_CLK>,
+				 <&gcc GCC_SDCC2_APPS_CLK>,
+				 <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "iface", "core", "xo";
+			resets = <&gcc GCC_SDCC2_BCR>;
+			interconnects = <&aggre2_noc MASTER_SDCC_2 0 &mc_virt SLAVE_EBI1 0>,
+					<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_SDCC_2 0>;
+			interconnect-names = "sdhc-ddr","cpu-sdhc";
+			iommus = <&apps_smmu 0x4a0 0x0>;
+			power-domains = <&rpmhpd SM8450_CX>;
+			operating-points-v2 = <&sdhc2_opp_table>;
+			bus-width = <4>;
+			dma-coherent;
+
+			status = "disabled";
+
+			sdhc2_opp_table: opp-table {
+				compatible = "operating-points-v2";
+
+				opp-100000000 {
+					opp-hz = /bits/ 64 <100000000>;
+					required-opps = <&rpmhpd_opp_low_svs>;
+				};
+
+				opp-202000000 {
+					opp-hz = /bits/ 64 <202000000>;
+					required-opps = <&rpmhpd_opp_svs_l1>;
+				};
+			};
+		};
+
 		usb_1: usb@a6f8800 {
 			compatible = "qcom,sm8450-dwc3", "qcom,dwc3";
 			reg = <0 0x0a6f8800 0 0x400>;
diff --git a/arch/arm64/boot/dts/renesas/Makefile b/arch/arm64/boot/dts/renesas/Makefile
index 7a647860ef35..0699b51c1247 100644
--- a/arch/arm64/boot/dts/renesas/Makefile
+++ b/arch/arm64/boot/dts/renesas/Makefile
@@ -56,6 +56,7 @@ dtb-$(CONFIG_ARCH_R8A77970) += r8a77970-v3msk.dtb
 
 dtb-$(CONFIG_ARCH_R8A77980) += r8a77980-condor.dtb
 dtb-$(CONFIG_ARCH_R8A77980) += r8a77980-v3hsk.dtb
+dtb-$(CONFIG_ARCH_R8A77980) += r8a77980a-condor-i.dtb
 
 dtb-$(CONFIG_ARCH_R8A77990) += r8a77990-ebisu.dtb
 
diff --git a/arch/arm64/boot/dts/renesas/condor-common.dtsi b/arch/arm64/boot/dts/renesas/condor-common.dtsi
new file mode 100644
index 000000000000..dfbe35bf46e0
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/condor-common.dtsi
@@ -0,0 +1,548 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for the Condor board with R-Car V3H
+ *
+ * Copyright (C) 2018 Renesas Electronics Corp.
+ * Copyright (C) 2018 Cogent Embedded, Inc.
+ */
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
+		serial0 = &scif0;
+		ethernet0 = &gether;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	d1_8v: regulator-2 {
+		compatible = "regulator-fixed";
+		regulator-name = "D1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	d3_3v: regulator-0 {
+		compatible = "regulator-fixed";
+		regulator-name = "D3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	hdmi-out {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con: endpoint {
+				remote-endpoint = <&adv7511_out>;
+			};
+		};
+	};
+
+	lvds-decoder {
+		compatible = "thine,thc63lvd1024";
+		vcc-supply = <&d3_3v>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				thc63lvd1024_in: endpoint {
+					remote-endpoint = <&lvds0_out>;
+				};
+			};
+
+			port@2 {
+				reg = <2>;
+				thc63lvd1024_out: endpoint {
+					remote-endpoint = <&adv7511_in>;
+				};
+			};
+		};
+	};
+
+	memory@48000000 {
+		device_type = "memory";
+		/* first 128MB is reserved for secure area. */
+		reg = <0 0x48000000 0 0x78000000>;
+	};
+
+	vddq_vin01: regulator-1 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDQ_VIN01";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	x1_clk: x1-clock {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <148500000>;
+	};
+};
+
+&canfd {
+	pinctrl-0 = <&canfd0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	channel0 {
+		status = "okay";
+	};
+};
+
+&csi40 {
+	status = "okay";
+
+	ports {
+		port@0 {
+			csi40_in: endpoint {
+				clock-lanes = <0>;
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&max9286_out0>;
+			};
+		};
+	};
+};
+
+&csi41 {
+	status = "okay";
+
+	ports {
+		port@0 {
+			csi41_in: endpoint {
+				clock-lanes = <0>;
+				data-lanes = <1 2 3 4>;
+				remote-endpoint = <&max9286_out1>;
+			};
+		};
+	};
+};
+
+&du {
+	clocks = <&cpg CPG_MOD 724>,
+		 <&x1_clk>;
+	clock-names = "du.0", "dclkin.0";
+	status = "okay";
+};
+
+&extal_clk {
+	clock-frequency = <16666666>;
+};
+
+&extalr_clk {
+	clock-frequency = <32768>;
+};
+
+&gether {
+	pinctrl-0 = <&gether_pins>;
+	pinctrl-names = "default";
+
+	phy-mode = "rgmii-id";
+	phy-handle = <&phy0>;
+	renesas,no-ether-link;
+	status = "okay";
+
+	phy0: ethernet-phy@0 {
+		compatible = "ethernet-phy-id0022.1622",
+			     "ethernet-phy-ieee802.3-c22";
+		rxc-skew-ps = <1500>;
+		reg = <0>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <23 IRQ_TYPE_LEVEL_LOW>;
+		reset-gpios = <&gpio4 22 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&i2c0 {
+	pinctrl-0 = <&i2c0_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	io_expander0: gpio@20 {
+		compatible = "onnn,pca9654";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	io_expander1: gpio@21 {
+		compatible = "onnn,pca9654";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	hdmi@39 {
+		compatible = "adi,adv7511w";
+		reg = <0x39>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
+		avdd-supply = <&d1_8v>;
+		dvdd-supply = <&d1_8v>;
+		pvdd-supply = <&d1_8v>;
+		bgvdd-supply = <&d1_8v>;
+		dvdd-3v-supply = <&d3_3v>;
+
+		adi,input-depth = <8>;
+		adi,input-colorspace = "rgb";
+		adi,input-clock = "1x";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				adv7511_in: endpoint {
+					remote-endpoint = <&thc63lvd1024_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				adv7511_out: endpoint {
+					remote-endpoint = <&hdmi_con>;
+				};
+			};
+		};
+	};
+};
+
+&i2c1 {
+	pinctrl-0 = <&i2c1_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	gmsl0: gmsl-deserializer@48 {
+		compatible = "maxim,max9286";
+		reg = <0x48>;
+
+		maxim,gpio-poc = <0 GPIO_ACTIVE_LOW>;
+		enable-gpios = <&io_expander0 0 GPIO_ACTIVE_HIGH>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+			};
+
+			port@1 {
+				reg = <1>;
+			};
+
+			port@2 {
+				reg = <2>;
+			};
+
+			port@3 {
+				reg = <3>;
+			};
+
+			port@4 {
+				reg = <4>;
+				max9286_out0: endpoint {
+					clock-lanes = <0>;
+					data-lanes = <1 2 3 4>;
+					remote-endpoint = <&csi40_in>;
+				};
+			};
+		};
+
+		i2c-mux {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			i2c@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+
+				status = "disabled";
+			};
+
+			i2c@1 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <1>;
+
+				status = "disabled";
+			};
+
+			i2c@2 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <2>;
+
+				status = "disabled";
+			};
+
+			i2c@3 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <3>;
+
+				status = "disabled";
+			};
+		};
+	};
+
+	gmsl1: gmsl-deserializer@4a {
+		compatible = "maxim,max9286";
+		reg = <0x4a>;
+
+		maxim,gpio-poc = <0 GPIO_ACTIVE_LOW>;
+		enable-gpios = <&io_expander1 0 GPIO_ACTIVE_HIGH>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+			};
+
+			port@1 {
+				reg = <1>;
+			};
+
+			port@2 {
+				reg = <2>;
+			};
+
+			port@3 {
+				reg = <3>;
+			};
+
+			port@4 {
+				reg = <4>;
+				max9286_out1: endpoint {
+					clock-lanes = <0>;
+					data-lanes = <1 2 3 4>;
+					remote-endpoint = <&csi41_in>;
+				};
+			};
+		};
+
+		i2c-mux {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			i2c@0 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <0>;
+
+				status = "disabled";
+			};
+
+			i2c@1 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <1>;
+
+				status = "disabled";
+			};
+
+			i2c@2 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <2>;
+
+				status = "disabled";
+			};
+
+			i2c@3 {
+				#address-cells = <1>;
+				#size-cells = <0>;
+				reg = <3>;
+
+				status = "disabled";
+			};
+		};
+	};
+};
+
+&lvds0 {
+	status = "okay";
+
+	ports {
+		port@1 {
+			lvds0_out: endpoint {
+				remote-endpoint = <&thc63lvd1024_in>;
+			};
+		};
+	};
+};
+
+&mmc0 {
+	pinctrl-0 = <&mmc_pins>;
+	pinctrl-1 = <&mmc_pins>;
+	pinctrl-names = "default", "state_uhs";
+
+	vmmc-supply = <&d3_3v>;
+	vqmmc-supply = <&vddq_vin01>;
+	mmc-hs200-1_8v;
+	bus-width = <8>;
+	no-sd;
+	no-sdio;
+	non-removable;
+	status = "okay";
+};
+
+&pciec {
+	status = "okay";
+};
+
+&pcie_bus_clk {
+	clock-frequency = <100000000>;
+};
+
+&pcie_phy {
+	status = "okay";
+};
+
+&pfc {
+	canfd0_pins: canfd0 {
+		groups = "canfd0_data_a";
+		function = "canfd0";
+	};
+
+	gether_pins: gether {
+		groups = "gether_mdio_a", "gether_rgmii",
+			 "gether_txcrefclk", "gether_txcrefclk_mega";
+		function = "gether";
+	};
+
+	i2c0_pins: i2c0 {
+		groups = "i2c0";
+		function = "i2c0";
+	};
+
+	i2c1_pins: i2c1 {
+		groups = "i2c1";
+		function = "i2c1";
+	};
+
+	mmc_pins: mmc {
+		groups = "mmc_data8", "mmc_ctrl", "mmc_ds";
+		function = "mmc";
+		power-source = <1800>;
+	};
+
+	qspi0_pins: qspi0 {
+		groups = "qspi0_ctrl", "qspi0_data4";
+		function = "qspi0";
+	};
+
+	scif0_pins: scif0 {
+		groups = "scif0_data";
+		function = "scif0";
+	};
+
+	scif_clk_pins: scif_clk {
+		groups = "scif_clk_b";
+		function = "scif_clk";
+	};
+};
+
+&rpc {
+	pinctrl-0 = <&qspi0_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+
+	flash@0 {
+		compatible = "spansion,s25fs512s", "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <50000000>;
+		spi-rx-bus-width = <4>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			bootparam@0 {
+				reg = <0x00000000 0x040000>;
+				read-only;
+			};
+			cr7@40000 {
+				reg = <0x00040000 0x080000>;
+				read-only;
+			};
+			cert_header_sa3@c0000 {
+				reg = <0x000c0000 0x080000>;
+				read-only;
+			};
+			bl2@140000 {
+				reg = <0x00140000 0x040000>;
+				read-only;
+			};
+			cert_header_sa6@180000 {
+				reg = <0x00180000 0x040000>;
+				read-only;
+			};
+			bl31@1c0000 {
+				reg = <0x001c0000 0x460000>;
+				read-only;
+			};
+			uboot@640000 {
+				reg = <0x00640000 0x0c0000>;
+				read-only;
+			};
+			uboot-env@700000 {
+				reg = <0x00700000 0x040000>;
+				read-only;
+			};
+			dtb@740000 {
+				reg = <0x00740000 0x080000>;
+			};
+			kernel@7c0000 {
+				reg = <0x007c0000 0x1400000>;
+			};
+			user@1bc0000 {
+				reg = <0x01bc0000 0x2440000>;
+			};
+		};
+	};
+};
+
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
+
+&scif0 {
+	pinctrl-0 = <&scif0_pins>, <&scif_clk_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&scif_clk {
+	clock-frequency = <14745600>;
+};
diff --git a/arch/arm64/boot/dts/renesas/ebisu.dtsi b/arch/arm64/boot/dts/renesas/ebisu.dtsi
index 8fc03491a11c..bbc29452d1be 100644
--- a/arch/arm64/boot/dts/renesas/ebisu.dtsi
+++ b/arch/arm64/boot/dts/renesas/ebisu.dtsi
@@ -13,6 +13,14 @@
 	compatible = "renesas,ebisu";
 
 	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
+		i2c6 = &i2c6;
+		i2c7 = &i2c7;
 		serial0 = &scif2;
 		ethernet0 = &avb;
 		mmc0 = &sdhi3;
diff --git a/arch/arm64/boot/dts/renesas/hihope-common.dtsi b/arch/arm64/boot/dts/renesas/hihope-common.dtsi
index b062f41ee270..83104af2813e 100644
--- a/arch/arm64/boot/dts/renesas/hihope-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/hihope-common.dtsi
@@ -10,6 +10,14 @@
 
 / {
 	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
+		i2c6 = &i2c6;
+		i2c7 = &iic_pmic;
 		serial0 = &scif2;
 		serial1 = &hscif0;
 		mmc0 = &sdhi3;
diff --git a/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts b/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts
index 3cf2e076940f..9ae67263c0df 100644
--- a/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts
+++ b/arch/arm64/boot/dts/renesas/r8a774a1-beacon-rzg2m-kit.dts
@@ -14,6 +14,14 @@
 	compatible = "beacon,beacon-rzg2m", "renesas,r8a774a1";
 
 	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
+		i2c6 = &i2c6;
+		i2c7 = &iic_pmic;
 		serial0 = &scif2;
 		serial1 = &hscif0;
 		serial2 = &hscif1;
diff --git a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
index e7d17776624d..7e643243c3be 100644
--- a/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774a1.dtsi
@@ -17,17 +17,6 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	aliases {
-		i2c0 = &i2c0;
-		i2c1 = &i2c1;
-		i2c2 = &i2c2;
-		i2c3 = &i2c3;
-		i2c4 = &i2c4;
-		i2c5 = &i2c5;
-		i2c6 = &i2c6;
-		i2c7 = &iic_pmic;
-	};
-
 	/*
 	 * The external audio clocks are configured as 0 Hz fixed frequency
 	 * clocks by default.
@@ -2334,7 +2323,6 @@
 			reg-names = "regs", "dirmap", "wbuf";
 			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 917>;
-			clock-names = "rpc";
 			power-domains = <&sysc R8A774A1_PD_ALWAYS_ON>;
 			resets = <&cpg 917>;
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
index f62d95760e82..d541b48c7e38 100644
--- a/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774b1.dtsi
@@ -2191,7 +2191,6 @@
 			reg-names = "regs", "dirmap", "wbuf";
 			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 917>;
-			clock-names = "rpc";
 			power-domains = <&sysc R8A774B1_PD_ALWAYS_ON>;
 			resets = <&cpg 917>;
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/renesas/r8a774c0.dtsi b/arch/arm64/boot/dts/renesas/r8a774c0.dtsi
index c563d26a7a71..151e32ac0368 100644
--- a/arch/arm64/boot/dts/renesas/r8a774c0.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774c0.dtsi
@@ -1671,7 +1671,6 @@
 			reg-names = "regs", "dirmap", "wbuf";
 			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 917>;
-			clock-names = "rpc";
 			power-domains = <&sysc R8A774C0_PD_ALWAYS_ON>;
 			resets = <&cpg 917>;
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
index 8ec59094882b..c5a0e7866b2f 100644
--- a/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a774e1.dtsi
@@ -769,7 +769,7 @@
 			status = "disabled";
 		};
 
-		i2c_dvfs: i2c@e60b0000 {
+		iic_pmic: i2c@e60b0000 {
 			#address-cells = <1>;
 			#size-cells = <0>;
 			compatible = "renesas,iic-r8a774e1",
@@ -2423,7 +2423,6 @@
 			reg-names = "regs", "dirmap", "wbuf";
 			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 917>;
-			clock-names = "rpc";
 			power-domains = <&sysc R8A774E1_PD_ALWAYS_ON>;
 			resets = <&cpg 917>;
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77951.dtsi b/arch/arm64/boot/dts/renesas/r8a77951.dtsi
index a297af22a195..07c8763c1e77 100644
--- a/arch/arm64/boot/dts/renesas/r8a77951.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77951.dtsi
@@ -21,17 +21,6 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	aliases {
-		i2c0 = &i2c0;
-		i2c1 = &i2c1;
-		i2c2 = &i2c2;
-		i2c3 = &i2c3;
-		i2c4 = &i2c4;
-		i2c5 = &i2c5;
-		i2c6 = &i2c6;
-		i2c7 = &i2c_dvfs;
-	};
-
 	/*
 	 * The external audio clocks are configured as 0 Hz fixed frequency
 	 * clocks by default.
diff --git a/arch/arm64/boot/dts/renesas/r8a77960.dtsi b/arch/arm64/boot/dts/renesas/r8a77960.dtsi
index 4159c23d3874..1424d4ad941f 100644
--- a/arch/arm64/boot/dts/renesas/r8a77960.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77960.dtsi
@@ -16,17 +16,6 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	aliases {
-		i2c0 = &i2c0;
-		i2c1 = &i2c1;
-		i2c2 = &i2c2;
-		i2c3 = &i2c3;
-		i2c4 = &i2c4;
-		i2c5 = &i2c5;
-		i2c6 = &i2c6;
-		i2c7 = &i2c_dvfs;
-	};
-
 	/*
 	 * The external audio clocks are configured as 0 Hz fixed frequency
 	 * clocks by default.
diff --git a/arch/arm64/boot/dts/renesas/r8a77965.dtsi b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
index 21a5e1cdd9f1..997f29521f66 100644
--- a/arch/arm64/boot/dts/renesas/r8a77965.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77965.dtsi
@@ -21,17 +21,6 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	aliases {
-		i2c0 = &i2c0;
-		i2c1 = &i2c1;
-		i2c2 = &i2c2;
-		i2c3 = &i2c3;
-		i2c4 = &i2c4;
-		i2c5 = &i2c5;
-		i2c6 = &i2c6;
-		i2c7 = &i2c_dvfs;
-	};
-
 	/*
 	 * The external audio clocks are configured as 0 Hz fixed frequency
 	 * clocks by default.
diff --git a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
index 49d1a929aef7..004a5eacd460 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77970-eagle.dts
@@ -15,6 +15,11 @@
 	compatible = "renesas,eagle", "renesas,r8a77970";
 
 	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
 		serial0 = &scif0;
 		ethernet0 = &avb;
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
index 39f3e6cbba3d..c2b65f8de547 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77970-v3msk.dts
@@ -15,6 +15,11 @@
 	compatible = "renesas,v3msk", "renesas,r8a77970";
 
 	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
 		serial0 = &scif0;
 	};
 
diff --git a/arch/arm64/boot/dts/renesas/r8a77970.dtsi b/arch/arm64/boot/dts/renesas/r8a77970.dtsi
index 2703ef3a38c2..ed6e2e47c604 100644
--- a/arch/arm64/boot/dts/renesas/r8a77970.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77970.dtsi
@@ -16,14 +16,6 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	aliases {
-		i2c0 = &i2c0;
-		i2c1 = &i2c1;
-		i2c2 = &i2c2;
-		i2c3 = &i2c3;
-		i2c4 = &i2c4;
-	};
-
 	/* External CAN clock - to be overridden by boards that provide it */
 	can_clk: can {
 		compatible = "fixed-clock";
@@ -1053,7 +1045,6 @@
 			reg-names = "regs", "dirmap", "wbuf";
 			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 917>;
-			clock-names = "rpc";
 			power-domains = <&sysc R8A77970_PD_ALWAYS_ON>;
 			resets = <&cpg 917>;
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77980-condor.dts b/arch/arm64/boot/dts/renesas/r8a77980-condor.dts
index 43ed033eb512..1d326552e2fa 100644
--- a/arch/arm64/boot/dts/renesas/r8a77980-condor.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77980-condor.dts
@@ -8,541 +8,9 @@
 
 /dts-v1/;
 #include "r8a77980.dtsi"
-#include <dt-bindings/gpio/gpio.h>
+#include "condor-common.dtsi"
 
 / {
 	model = "Renesas Condor board based on r8a77980";
 	compatible = "renesas,condor", "renesas,r8a77980";
-
-	aliases {
-		serial0 = &scif0;
-		ethernet0 = &gether;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
-
-	d1_8v: regulator-2 {
-		compatible = "regulator-fixed";
-		regulator-name = "D1.8V";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-		regulator-boot-on;
-		regulator-always-on;
-	};
-
-	d3_3v: regulator-0 {
-		compatible = "regulator-fixed";
-		regulator-name = "D3.3V";
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
-		regulator-boot-on;
-		regulator-always-on;
-	};
-
-	hdmi-out {
-		compatible = "hdmi-connector";
-		type = "a";
-
-		port {
-			hdmi_con: endpoint {
-				remote-endpoint = <&adv7511_out>;
-			};
-		};
-	};
-
-	lvds-decoder {
-		compatible = "thine,thc63lvd1024";
-		vcc-supply = <&d3_3v>;
-
-		ports {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			port@0 {
-				reg = <0>;
-				thc63lvd1024_in: endpoint {
-					remote-endpoint = <&lvds0_out>;
-				};
-			};
-
-			port@2 {
-				reg = <2>;
-				thc63lvd1024_out: endpoint {
-					remote-endpoint = <&adv7511_in>;
-				};
-			};
-		};
-	};
-
-	memory@48000000 {
-		device_type = "memory";
-		/* first 128MB is reserved for secure area. */
-		reg = <0 0x48000000 0 0x78000000>;
-	};
-
-	vddq_vin01: regulator-1 {
-		compatible = "regulator-fixed";
-		regulator-name = "VDDQ_VIN01";
-		regulator-min-microvolt = <1800000>;
-		regulator-max-microvolt = <1800000>;
-		regulator-boot-on;
-		regulator-always-on;
-	};
-
-	x1_clk: x1-clock {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <148500000>;
-	};
-};
-
-&canfd {
-	pinctrl-0 = <&canfd0_pins>;
-	pinctrl-names = "default";
-	status = "okay";
-
-	channel0 {
-		status = "okay";
-	};
-};
-
-&csi40 {
-	status = "okay";
-
-	ports {
-		port@0 {
-			csi40_in: endpoint {
-				clock-lanes = <0>;
-				data-lanes = <1 2 3 4>;
-				remote-endpoint = <&max9286_out0>;
-			};
-		};
-	};
-};
-
-&csi41 {
-	status = "okay";
-
-	ports {
-		port@0 {
-			csi41_in: endpoint {
-				clock-lanes = <0>;
-				data-lanes = <1 2 3 4>;
-				remote-endpoint = <&max9286_out1>;
-			};
-		};
-	};
-};
-
-&du {
-	clocks = <&cpg CPG_MOD 724>,
-		 <&x1_clk>;
-	clock-names = "du.0", "dclkin.0";
-	status = "okay";
-};
-
-&extal_clk {
-	clock-frequency = <16666666>;
-};
-
-&extalr_clk {
-	clock-frequency = <32768>;
-};
-
-&gether {
-	pinctrl-0 = <&gether_pins>;
-	pinctrl-names = "default";
-
-	phy-mode = "rgmii-id";
-	phy-handle = <&phy0>;
-	renesas,no-ether-link;
-	status = "okay";
-
-	phy0: ethernet-phy@0 {
-		compatible = "ethernet-phy-id0022.1622",
-			     "ethernet-phy-ieee802.3-c22";
-		rxc-skew-ps = <1500>;
-		reg = <0>;
-		interrupt-parent = <&gpio4>;
-		interrupts = <23 IRQ_TYPE_LEVEL_LOW>;
-		reset-gpios = <&gpio4 22 GPIO_ACTIVE_LOW>;
-	};
-};
-
-&i2c0 {
-	pinctrl-0 = <&i2c0_pins>;
-	pinctrl-names = "default";
-
-	status = "okay";
-	clock-frequency = <400000>;
-
-	io_expander0: gpio@20 {
-		compatible = "onnn,pca9654";
-		reg = <0x20>;
-		gpio-controller;
-		#gpio-cells = <2>;
-	};
-
-	io_expander1: gpio@21 {
-		compatible = "onnn,pca9654";
-		reg = <0x21>;
-		gpio-controller;
-		#gpio-cells = <2>;
-	};
-
-	hdmi@39 {
-		compatible = "adi,adv7511w";
-		reg = <0x39>;
-		interrupt-parent = <&gpio1>;
-		interrupts = <20 IRQ_TYPE_LEVEL_LOW>;
-		avdd-supply = <&d1_8v>;
-		dvdd-supply = <&d1_8v>;
-		pvdd-supply = <&d1_8v>;
-		bgvdd-supply = <&d1_8v>;
-		dvdd-3v-supply = <&d3_3v>;
-
-		adi,input-depth = <8>;
-		adi,input-colorspace = "rgb";
-		adi,input-clock = "1x";
-
-		ports {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			port@0 {
-				reg = <0>;
-				adv7511_in: endpoint {
-					remote-endpoint = <&thc63lvd1024_out>;
-				};
-			};
-
-			port@1 {
-				reg = <1>;
-				adv7511_out: endpoint {
-					remote-endpoint = <&hdmi_con>;
-				};
-			};
-		};
-	};
-};
-
-&i2c1 {
-	pinctrl-0 = <&i2c1_pins>;
-	pinctrl-names = "default";
-
-	status = "okay";
-	clock-frequency = <400000>;
-
-	gmsl0: gmsl-deserializer@48 {
-		compatible = "maxim,max9286";
-		reg = <0x48>;
-
-		maxim,gpio-poc = <0 GPIO_ACTIVE_LOW>;
-		enable-gpios = <&io_expander0 0 GPIO_ACTIVE_HIGH>;
-
-		ports {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			port@0 {
-				reg = <0>;
-			};
-
-			port@1 {
-				reg = <1>;
-			};
-
-			port@2 {
-				reg = <2>;
-			};
-
-			port@3 {
-				reg = <3>;
-			};
-
-			port@4 {
-				reg = <4>;
-				max9286_out0: endpoint {
-					clock-lanes = <0>;
-					data-lanes = <1 2 3 4>;
-					remote-endpoint = <&csi40_in>;
-				};
-			};
-		};
-
-		i2c-mux {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			i2c@0 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				reg = <0>;
-
-				status = "disabled";
-			};
-
-			i2c@1 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				reg = <1>;
-
-				status = "disabled";
-			};
-
-			i2c@2 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				reg = <2>;
-
-				status = "disabled";
-			};
-
-			i2c@3 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				reg = <3>;
-
-				status = "disabled";
-			};
-		};
-	};
-
-	gmsl1: gmsl-deserializer@4a {
-		compatible = "maxim,max9286";
-		reg = <0x4a>;
-
-		maxim,gpio-poc = <0 GPIO_ACTIVE_LOW>;
-		enable-gpios = <&io_expander1 0 GPIO_ACTIVE_HIGH>;
-
-		ports {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			port@0 {
-				reg = <0>;
-			};
-
-			port@1 {
-				reg = <1>;
-			};
-
-			port@2 {
-				reg = <2>;
-			};
-
-			port@3 {
-				reg = <3>;
-			};
-
-			port@4 {
-				reg = <4>;
-				max9286_out1: endpoint {
-					clock-lanes = <0>;
-					data-lanes = <1 2 3 4>;
-					remote-endpoint = <&csi41_in>;
-				};
-			};
-		};
-
-		i2c-mux {
-			#address-cells = <1>;
-			#size-cells = <0>;
-
-			i2c@0 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				reg = <0>;
-
-				status = "disabled";
-			};
-
-			i2c@1 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				reg = <1>;
-
-				status = "disabled";
-			};
-
-			i2c@2 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				reg = <2>;
-
-				status = "disabled";
-			};
-
-			i2c@3 {
-				#address-cells = <1>;
-				#size-cells = <0>;
-				reg = <3>;
-
-				status = "disabled";
-			};
-		};
-	};
-};
-
-&lvds0 {
-	status = "okay";
-
-	ports {
-		port@1 {
-			lvds0_out: endpoint {
-				remote-endpoint = <&thc63lvd1024_in>;
-			};
-		};
-	};
-};
-
-&mmc0 {
-	pinctrl-0 = <&mmc_pins>;
-	pinctrl-1 = <&mmc_pins>;
-	pinctrl-names = "default", "state_uhs";
-
-	vmmc-supply = <&d3_3v>;
-	vqmmc-supply = <&vddq_vin01>;
-	mmc-hs200-1_8v;
-	bus-width = <8>;
-	no-sd;
-	no-sdio;
-	non-removable;
-	status = "okay";
-};
-
-&pciec {
-	status = "okay";
-};
-
-&pcie_bus_clk {
-	clock-frequency = <100000000>;
-};
-
-&pcie_phy {
-	status = "okay";
-};
-
-&pfc {
-	canfd0_pins: canfd0 {
-		groups = "canfd0_data_a";
-		function = "canfd0";
-	};
-
-	gether_pins: gether {
-		groups = "gether_mdio_a", "gether_rgmii",
-			 "gether_txcrefclk", "gether_txcrefclk_mega";
-		function = "gether";
-	};
-
-	i2c0_pins: i2c0 {
-		groups = "i2c0";
-		function = "i2c0";
-	};
-
-	i2c1_pins: i2c1 {
-		groups = "i2c1";
-		function = "i2c1";
-	};
-
-	mmc_pins: mmc {
-		groups = "mmc_data8", "mmc_ctrl", "mmc_ds";
-		function = "mmc";
-		power-source = <1800>;
-	};
-
-	qspi0_pins: qspi0 {
-		groups = "qspi0_ctrl", "qspi0_data4";
-		function = "qspi0";
-	};
-
-	scif0_pins: scif0 {
-		groups = "scif0_data";
-		function = "scif0";
-	};
-
-	scif_clk_pins: scif_clk {
-		groups = "scif_clk_b";
-		function = "scif_clk";
-	};
-};
-
-&rpc {
-	pinctrl-0 = <&qspi0_pins>;
-	pinctrl-names = "default";
-
-	status = "okay";
-
-	flash@0 {
-		compatible = "spansion,s25fs512s", "jedec,spi-nor";
-		reg = <0>;
-		spi-max-frequency = <50000000>;
-		spi-rx-bus-width = <4>;
-
-		partitions {
-			compatible = "fixed-partitions";
-			#address-cells = <1>;
-			#size-cells = <1>;
-
-			bootparam@0 {
-				reg = <0x00000000 0x040000>;
-				read-only;
-			};
-			cr7@40000 {
-				reg = <0x00040000 0x080000>;
-				read-only;
-			};
-			cert_header_sa3@c0000 {
-				reg = <0x000c0000 0x080000>;
-				read-only;
-			};
-			bl2@140000 {
-				reg = <0x00140000 0x040000>;
-				read-only;
-			};
-			cert_header_sa6@180000 {
-				reg = <0x00180000 0x040000>;
-				read-only;
-			};
-			bl31@1c0000 {
-				reg = <0x001c0000 0x460000>;
-				read-only;
-			};
-			uboot@640000 {
-				reg = <0x00640000 0x0c0000>;
-				read-only;
-			};
-			uboot-env@700000 {
-				reg = <0x00700000 0x040000>;
-				read-only;
-			};
-			dtb@740000 {
-				reg = <0x00740000 0x080000>;
-			};
-			kernel@7c0000 {
-				reg = <0x007c0000 0x1400000>;
-			};
-			user@1bc0000 {
-				reg = <0x01bc0000 0x2440000>;
-			};
-		};
-	};
-};
-
-&rwdt {
-	timeout-sec = <60>;
-	status = "okay";
-};
-
-&scif0 {
-	pinctrl-0 = <&scif0_pins>, <&scif_clk_pins>;
-	pinctrl-names = "default";
-
-	status = "okay";
-};
-
-&scif_clk {
-	clock-frequency = <14745600>;
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts b/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts
index 1d09d8867651..d168b0e7747d 100644
--- a/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77980-v3hsk.dts
@@ -15,6 +15,12 @@
 	compatible = "renesas,v3hsk", "renesas,r8a77980";
 
 	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
 		serial0 = &scif0;
 		ethernet0 = &gether;
 	};
diff --git a/arch/arm64/boot/dts/renesas/r8a77980.dtsi b/arch/arm64/boot/dts/renesas/r8a77980.dtsi
index 8594be72f221..c4ac28a0f716 100644
--- a/arch/arm64/boot/dts/renesas/r8a77980.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77980.dtsi
@@ -16,15 +16,6 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	aliases {
-		i2c0 = &i2c0;
-		i2c1 = &i2c1;
-		i2c2 = &i2c2;
-		i2c3 = &i2c3;
-		i2c4 = &i2c4;
-		i2c5 = &i2c5;
-	};
-
 	/* External CAN clock - to be overridden by boards that provide it */
 	can_clk: can {
 		compatible = "fixed-clock";
@@ -1359,7 +1350,6 @@
 			reg-names = "regs", "dirmap", "wbuf";
 			interrupts = <GIC_SPI 38 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 917>;
-			clock-names = "rpc";
 			power-domains = <&sysc R8A77980_PD_ALWAYS_ON>;
 			resets = <&cpg 917>;
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/renesas/r8a77980a-condor-i.dts b/arch/arm64/boot/dts/renesas/r8a77980a-condor-i.dts
new file mode 100644
index 000000000000..9f488dea0f34
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a77980a-condor-i.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for the Condor-I board on r8a77980A (ES2.0)
+ *
+ * Copyright (C) 2022 Renesas Electronics Corp.
+ */
+
+/dts-v1/;
+#include "r8a77980a.dtsi"
+#include "condor-common.dtsi"
+
+/ {
+	model = "Renesas Condor-I board based on r8a77980A (ES2.0)";
+	compatible = "renesas,condor-i", "renesas,r8a77980a", "renesas,r8a77980";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77980a.dtsi b/arch/arm64/boot/dts/renesas/r8a77980a.dtsi
new file mode 100644
index 000000000000..25b2d27b6167
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a77980a.dtsi
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: (GPL-2.0 or MIT)
+/*
+ * Device Tree Source for the R-Car V3H2 (R8A77980A) SoC
+ *
+ * Copyright (C) 2022 Renesas Electronics Corp.
+ */
+#include "r8a77980.dtsi"
+
+/ {
+	compatible = "renesas,r8a77980a", "renesas,r8a77980";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77990.dtsi b/arch/arm64/boot/dts/renesas/r8a77990.dtsi
index 565e9d85946e..3053b4b21497 100644
--- a/arch/arm64/boot/dts/renesas/r8a77990.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a77990.dtsi
@@ -14,17 +14,6 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	aliases {
-		i2c0 = &i2c0;
-		i2c1 = &i2c1;
-		i2c2 = &i2c2;
-		i2c3 = &i2c3;
-		i2c4 = &i2c4;
-		i2c5 = &i2c5;
-		i2c6 = &i2c6;
-		i2c7 = &i2c7;
-	};
-
 	/*
 	 * The external audio clocks are configured as 0 Hz fixed frequency
 	 * clocks by default.
diff --git a/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi
index 53c4a26198e3..99b73e21c82c 100644
--- a/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779a0-falcon-cpu.dtsi
@@ -16,6 +16,13 @@
 	compatible = "renesas,falcon-cpu", "renesas,r8a779a0";
 
 	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
+		i2c6 = &i2c6;
 		serial0 = &scif0;
 	};
 
diff --git a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi
index 3d668709d8a8..ed9400f903c9 100644
--- a/arch/arm64/boot/dts/renesas/r8a779a0.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779a0.dtsi
@@ -14,16 +14,6 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	aliases {
-		i2c0 = &i2c0;
-		i2c1 = &i2c1;
-		i2c2 = &i2c2;
-		i2c3 = &i2c3;
-		i2c4 = &i2c4;
-		i2c5 = &i2c5;
-		i2c6 = &i2c6;
-	};
-
 	/* External CAN clock - to be overridden by boards that provide it */
 	can_clk: can {
 		compatible = "fixed-clock";
@@ -257,7 +247,7 @@
 
 		cmt0: timer@e60f0000 {
 			compatible = "renesas,r8a779a0-cmt0",
-				     "renesas,rcar-gen3-cmt0";
+				     "renesas,rcar-gen4-cmt0";
 			reg = <0 0xe60f0000 0 0x1004>;
 			interrupts = <GIC_SPI 500 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 501 IRQ_TYPE_LEVEL_HIGH>;
@@ -270,7 +260,7 @@
 
 		cmt1: timer@e6130000 {
 			compatible = "renesas,r8a779a0-cmt1",
-				     "renesas,rcar-gen3-cmt1";
+				     "renesas,rcar-gen4-cmt1";
 			reg = <0 0xe6130000 0 0x1004>;
 			interrupts = <GIC_SPI 448 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
@@ -289,7 +279,7 @@
 
 		cmt2: timer@e6140000 {
 			compatible = "renesas,r8a779a0-cmt1",
-				     "renesas,rcar-gen3-cmt1";
+				     "renesas,rcar-gen4-cmt1";
 			reg = <0 0xe6140000 0 0x1004>;
 			interrupts = <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH>,
@@ -308,7 +298,7 @@
 
 		cmt3: timer@e6148000 {
 			compatible = "renesas,r8a779a0-cmt1",
-				     "renesas,rcar-gen3-cmt1";
+				     "renesas,rcar-gen4-cmt1";
 			reg = <0 0xe6148000 0 0x1004>;
 			interrupts = <GIC_SPI 464 IRQ_TYPE_LEVEL_HIGH>,
 				     <GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH>,
@@ -2075,7 +2065,7 @@
 
 		mmc0: mmc@ee140000 {
 			compatible = "renesas,sdhi-r8a779a0",
-				     "renesas,rcar-gen3-sdhi";
+				     "renesas,rcar-gen4-sdhi";
 			reg = <0 0xee140000 0 0x2000>;
 			interrupts = <GIC_SPI 236 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 706>, <&cpg CPG_CORE R8A779A0_CLK_SD0H>;
@@ -2096,7 +2086,6 @@
 			reg-names = "regs", "dirmap", "wbuf";
 			interrupts = <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cpg CPG_MOD 629>;
-			clock-names = "rpc";
 			power-domains = <&sysc R8A779A0_PD_ALWAYS_ON>;
 			resets = <&cpg 629>;
 			#address-cells = <1>;
diff --git a/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi b/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi
index 28fbf7bc1eb4..a45df1041705 100644
--- a/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779f0-spider-cpu.dtsi
@@ -11,6 +11,16 @@
 	model = "Renesas Spider CPU board";
 	compatible = "renesas,spider-cpu", "renesas,r8a779f0";
 
+	aliases {
+		serial0 = &scif3;
+		serial1 = &scif0;
+	};
+
+	chosen {
+		bootargs = "ignore_loglevel rw root=/dev/nfs ip=on";
+		stdout-path = "serial0:115200n8";
+	};
+
 	memory@48000000 {
 		device_type = "memory";
 		/* first 128MB is reserved for secure area. */
@@ -21,6 +31,24 @@
 		device_type = "memory";
 		reg = <0x4 0x80000000 0x0 0x80000000>;
 	};
+
+	reg_1p8v: regulator-1p8v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-1.8V";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "fixed-3.3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
 };
 
 &extal_clk {
@@ -46,6 +74,27 @@
 	};
 };
 
+/*
+ * This board also has a microSD slot which we will not support upstream
+ * because we cannot directly switch voltages in software.
+ */
+&mmc0 {
+	pinctrl-0 = <&mmc_pins>;
+	pinctrl-1 = <&mmc_pins>;
+	pinctrl-names = "default", "state_uhs";
+
+	vmmc-supply = <&reg_3p3v>;
+	vqmmc-supply = <&reg_1p8v>;
+	mmc-hs200-1_8v;
+	mmc-hs400-1_8v;
+	bus-width = <8>;
+	no-sd;
+	no-sdio;
+	non-removable;
+	full-pwr-cycle-in-suspend;
+	status = "okay";
+};
+
 &pfc {
 	pinctrl-0 = <&scif_clk_pins>;
 	pinctrl-names = "default";
@@ -55,6 +104,12 @@
 		function = "i2c4";
 	};
 
+	mmc_pins: mmc {
+		groups = "mmc_data8", "mmc_ctrl", "mmc_ds";
+		function = "mmc";
+		power-source = <1800>;
+	};
+
 	scif0_pins: scif0 {
 		groups = "scif0_data", "scif0_ctrl";
 		function = "scif0";
diff --git a/arch/arm64/boot/dts/renesas/r8a779f0-spider.dts b/arch/arm64/boot/dts/renesas/r8a779f0-spider.dts
index 7a7c8ffba711..7aac3f4d319c 100644
--- a/arch/arm64/boot/dts/renesas/r8a779f0-spider.dts
+++ b/arch/arm64/boot/dts/renesas/r8a779f0-spider.dts
@@ -12,15 +12,6 @@
 / {
 	model = "Renesas Spider CPU and Breakout boards based on r8a779f0";
 	compatible = "renesas,spider-breakout", "renesas,spider-cpu", "renesas,r8a779f0";
-
-	aliases {
-		serial0 = &scif3;
-		serial1 = &scif0;
-	};
-
-	chosen {
-		stdout-path = "serial0:115200n8";
-	};
 };
 
 &i2c4 {
diff --git a/arch/arm64/boot/dts/renesas/r8a779f0.dtsi b/arch/arm64/boot/dts/renesas/r8a779f0.dtsi
index 384817ffa4de..c2f152bcf10e 100644
--- a/arch/arm64/boot/dts/renesas/r8a779f0.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779f0.dtsi
@@ -301,6 +301,76 @@
 			#interrupt-cells = <2>;
 		};
 
+		cmt0: timer@e60f0000 {
+			compatible = "renesas,r8a779f0-cmt0",
+				     "renesas,rcar-gen4-cmt0";
+			reg = <0 0xe60f0000 0 0x1004>;
+			interrupts = <GIC_SPI 448 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 910>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 910>;
+			status = "disabled";
+		};
+
+		cmt1: timer@e6130000 {
+			compatible = "renesas,r8a779f0-cmt1",
+				     "renesas,rcar-gen4-cmt1";
+			reg = <0 0xe6130000 0 0x1004>;
+			interrupts = <GIC_SPI 450 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 451 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 452 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 453 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 454 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 455 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 911>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 911>;
+			status = "disabled";
+		};
+
+		cmt2: timer@e6140000 {
+			compatible = "renesas,r8a779f0-cmt1",
+				     "renesas,rcar-gen4-cmt1";
+			reg = <0 0xe6140000 0 0x1004>;
+			interrupts = <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 459 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 460 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 463 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 464 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 912>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 912>;
+			status = "disabled";
+		};
+
+		cmt3: timer@e6148000 {
+			compatible = "renesas,r8a779f0-cmt1",
+				     "renesas,rcar-gen4-cmt1";
+			reg = <0 0xe6148000 0 0x1004>;
+			interrupts = <GIC_SPI 466 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 467 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 468 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 469 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 470 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 471 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 472 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 473 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 913>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 913>;
+			status = "disabled";
+		};
+
 		cpg: clock-controller@e6150000 {
 			compatible = "renesas,r8a779f0-cpg-mssr";
 			reg = <0 0xe6150000 0 0x4000>;
@@ -334,6 +404,71 @@
 			#thermal-sensor-cells = <1>;
 		};
 
+		tmu0: timer@e61e0000 {
+			compatible = "renesas,tmu-r8a779f0", "renesas,tmu";
+			reg = <0 0xe61e0000 0 0x30>;
+			interrupts = <GIC_SPI 474 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 475 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 476 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 713>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 713>;
+			status = "disabled";
+		};
+
+		tmu1: timer@e6fc0000 {
+			compatible = "renesas,tmu-r8a779f0", "renesas,tmu";
+			reg = <0 0xe6fc0000 0 0x30>;
+			interrupts = <GIC_SPI 477 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 478 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 479 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 714>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 714>;
+			status = "disabled";
+		};
+
+		tmu2: timer@e6fd0000 {
+			compatible = "renesas,tmu-r8a779f0", "renesas,tmu";
+			reg = <0 0xe6fd0000 0 0x30>;
+			interrupts = <GIC_SPI 481 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 482 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 483 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 715>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 715>;
+			status = "disabled";
+		};
+
+		tmu3: timer@e6fe0000 {
+			compatible = "renesas,tmu-r8a779f0", "renesas,tmu";
+			reg = <0 0xe6fe0000 0 0x30>;
+			interrupts = <GIC_SPI 485 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 486 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 487 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 716>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 716>;
+			status = "disabled";
+		};
+
+		tmu4: timer@ffc00000 {
+			compatible = "renesas,tmu-r8a779f0", "renesas,tmu";
+			reg = <0 0xffc00000 0 0x30>;
+			interrupts = <GIC_SPI 489 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 490 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 491 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 717>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 717>;
+			status = "disabled";
+		};
+
 		i2c0: i2c@e6500000 {
 			compatible = "renesas,i2c-r8a779f0",
 				     "renesas,rcar-gen4-i2c";
@@ -584,6 +719,70 @@
 			status = "disabled";
 		};
 
+		msiof0: spi@e6e90000 {
+			compatible = "renesas,msiof-r8a779f0",
+				     "renesas,rcar-gen4-msiof";
+			reg = <0 0xe6e90000 0 0x0064>;
+			interrupts = <GIC_SPI 230 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 618>;
+			dmas = <&dmac0 0x41>, <&dmac0 0x40>,
+			       <&dmac1 0x41>, <&dmac1 0x40>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 618>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof1: spi@e6ea0000 {
+			compatible = "renesas,msiof-r8a779f0",
+				     "renesas,rcar-gen4-msiof";
+			reg = <0 0xe6ea0000 0 0x0064>;
+			interrupts = <GIC_SPI 231 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 619>;
+			dmas = <&dmac0 0x43>, <&dmac0 0x42>,
+			       <&dmac1 0x43>, <&dmac1 0x42>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 619>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof2: spi@e6c00000 {
+			compatible = "renesas,msiof-r8a779f0",
+				     "renesas,rcar-gen4-msiof";
+			reg = <0 0xe6c00000 0 0x0064>;
+			interrupts = <GIC_SPI 232 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 620>;
+			dmas = <&dmac0 0x45>, <&dmac0 0x44>,
+			       <&dmac1 0x45>, <&dmac1 0x44>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 620>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		msiof3: spi@e6c10000 {
+			compatible = "renesas,msiof-r8a779f0",
+				     "renesas,rcar-gen4-msiof";
+			reg = <0 0xe6c10000 0 0x0064>;
+			interrupts = <GIC_SPI 233 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 621>;
+			dmas = <&dmac0 0x47>, <&dmac0 0x46>,
+			       <&dmac1 0x47>, <&dmac1 0x46>;
+			dma-names = "tx", "rx", "tx", "rx";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 621>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		dmac0: dma-controller@e7350000 {
 			compatible = "renesas,dmac-r8a779f0",
 				     "renesas,rcar-gen4-dmac";
@@ -670,6 +869,19 @@
 				 <&ipmmu_ds0 30>, <&ipmmu_ds0 31>;
 		};
 
+		mmc0: mmc@ee140000 {
+			compatible = "renesas,sdhi-r8a779f0",
+				     "renesas,rcar-gen4-sdhi";
+			reg = <0 0xee140000 0 0x2000>;
+			interrupts = <GIC_SPI 236 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 706>, <&cpg CPG_CORE R8A779F0_CLK_SD0H>;
+			clock-names = "core", "clkh";
+			power-domains = <&sysc R8A779F0_PD_ALWAYS_ON>;
+			resets = <&cpg 706>;
+			max-frequency = <200000000>;
+			status = "disabled";
+		};
+
 		ipmmu_rt0: iommu@ee480000 {
 			compatible = "renesas,ipmmu-r8a779f0",
 				     "renesas,rcar-gen4-ipmmu-vmsa";
diff --git a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-cpu.dtsi b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-cpu.dtsi
index ea4ae4b893ab..895f0bd9f754 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-cpu.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-cpu.dtsi
@@ -7,10 +7,80 @@
 
 #include "r8a779g0.dtsi"
 
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+
 / {
 	model = "Renesas White Hawk CPU board";
 	compatible = "renesas,white-hawk-cpu", "renesas,r8a779g0";
 
+	aliases {
+		ethernet0 = &avb0;
+		serial0 = &hscif0;
+	};
+
+	chosen {
+		bootargs = "ignore_loglevel rw root=/dev/nfs ip=on";
+		stdout-path = "serial0:921600n8";
+	};
+
+	keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&keys_pins>;
+		pinctrl-names = "default";
+
+		key-1 {
+			gpios = <&gpio5 0 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_1>;
+			label = "SW47";
+			wakeup-source;
+			debounce-interval = <20>;
+		};
+
+		key-2 {
+			gpios = <&gpio5 1 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_2>;
+			label = "SW48";
+			wakeup-source;
+			debounce-interval = <20>;
+		};
+
+		key-3 {
+			gpios = <&gpio5 2 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_3>;
+			label = "SW49";
+			wakeup-source;
+			debounce-interval = <20>;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+
+		led-1 {
+			gpios = <&gpio7 0 GPIO_ACTIVE_HIGH>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <1>;
+		};
+
+		led-2 {
+			gpios = <&gpio7 1 GPIO_ACTIVE_HIGH>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <2>;
+		};
+
+		led-3 {
+			gpios = <&gpio7 2 GPIO_ACTIVE_HIGH>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <3>;
+		};
+	};
+
 	memory@48000000 {
 		device_type = "memory";
 		/* first 128MB is reserved for secure area. */
@@ -28,6 +98,24 @@
 	};
 };
 
+&avb0 {
+	pinctrl-0 = <&avb0_pins>;
+	pinctrl-names = "default";
+	phy-handle = <&phy0>;
+	tx-internal-delay-ps = <2000>;
+	status = "okay";
+
+	phy0: ethernet-phy@0 {
+		compatible = "ethernet-phy-id0022.1622",
+			     "ethernet-phy-ieee802.3-c22";
+		rxc-skew-ps = <1500>;
+		reg = <0>;
+		interrupt-parent = <&gpio7>;
+		interrupts = <5 IRQ_TYPE_LEVEL_LOW>;
+		reset-gpios = <&gpio7 10 GPIO_ACTIVE_LOW>;
+	};
+};
+
 &extal_clk {
 	clock-frequency = <16666666>;
 };
@@ -40,6 +128,69 @@
 	status = "okay";
 };
 
+&i2c0 {
+	pinctrl-0 = <&i2c0_pins>;
+	pinctrl-names = "default";
+
+	status = "okay";
+	clock-frequency = <400000>;
+
+	eeprom@50 {
+		compatible = "rohm,br24g01", "atmel,24c01";
+		label = "cpu-board";
+		reg = <0x50>;
+		pagesize = <8>;
+	};
+};
+
+&pfc {
+	pinctrl-0 = <&scif_clk_pins>;
+	pinctrl-names = "default";
+
+	avb0_pins: avb0 {
+		mux {
+			groups = "avb0_link", "avb0_mdio", "avb0_rgmii",
+				 "avb0_txcrefclk";
+			function = "avb0";
+		};
+
+		pins_mdio {
+			groups = "avb0_mdio";
+			drive-strength = <21>;
+		};
+
+		pins_mii {
+			groups = "avb0_rgmii";
+			drive-strength = <21>;
+		};
+
+	};
+	hscif0_pins: hscif0 {
+		groups = "hscif0_data";
+		function = "hscif0";
+	};
+
+	i2c0_pins: i2c0 {
+		groups = "i2c0";
+		function = "i2c0";
+	};
+
+	keys_pins: keys {
+		pins = "GP_5_0", "GP_5_1", "GP_5_2";
+		bias-pull-up;
+	};
+
+	scif_clk_pins: scif_clk {
+		groups = "scif_clk";
+		function = "scif_clk";
+	};
+};
+
 &scif_clk {
 	clock-frequency = <24000000>;
 };
+
+&rwdt {
+	timeout-sec = <60>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-csi-dsi.dtsi b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-csi-dsi.dtsi
new file mode 100644
index 000000000000..ae7522b60e5d
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-csi-dsi.dtsi
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Device Tree Source for the R-Car V4H White Hawk CSI/DSI sub-board
+ *
+ * Copyright (C) 2022 Glider bv
+ */
+
+&i2c0 {
+	eeprom@52 {
+		compatible = "rohm,br24g01", "atmel,24c01";
+		label = "csi-dsi-sub-board-id";
+		reg = <0x52>;
+		pagesize = <8>;
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-ethernet.dtsi b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-ethernet.dtsi
new file mode 100644
index 000000000000..4f411f95c674
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk-ethernet.dtsi
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/*
+ * Device Tree Source for the R-Car V4H White Hawk RAVB/Ethernet(1000Base-T1)
+ * sub-board
+ *
+ * Copyright (C) 2022 Glider bv
+ */
+
+&i2c0 {
+	eeprom@53 {
+		compatible = "rohm,br24g01", "atmel,24c01";
+		label = "ethernet-sub-board-id";
+		reg = <0x53>;
+		pagesize = <8>;
+	};
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts
index bc0ac109b17c..04a2b6b83e74 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts
+++ b/arch/arm64/boot/dts/renesas/r8a779g0-white-hawk.dts
@@ -7,16 +7,19 @@
 
 /dts-v1/;
 #include "r8a779g0-white-hawk-cpu.dtsi"
+#include "r8a779g0-white-hawk-csi-dsi.dtsi"
+#include "r8a779g0-white-hawk-ethernet.dtsi"
 
 / {
 	model = "Renesas White Hawk CPU and Breakout boards based on r8a779g0";
 	compatible = "renesas,white-hawk-breakout", "renesas,white-hawk-cpu", "renesas,r8a779g0";
+};
 
-	aliases {
-		serial0 = &hscif0;
-	};
-
-	chosen {
-		stdout-path = "serial0:921600n8";
+&i2c0 {
+	eeprom@51 {
+		compatible = "rohm,br24g01", "atmel,24c01";
+		label = "breakout-board";
+		reg = <0x51>;
+		pagesize = <8>;
 	};
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a779g0.dtsi b/arch/arm64/boot/dts/renesas/r8a779g0.dtsi
index 1c15726cff8b..d70f0600ae5a 100644
--- a/arch/arm64/boot/dts/renesas/r8a779g0.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a779g0.dtsi
@@ -59,6 +59,161 @@
 		#size-cells = <2>;
 		ranges;
 
+		rwdt: watchdog@e6020000 {
+			compatible = "renesas,r8a779g0-wdt",
+				     "renesas,rcar-gen4-wdt";
+			reg = <0 0xe6020000 0 0x0c>;
+			interrupts = <GIC_SPI 330 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 907>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 907>;
+			status = "disabled";
+		};
+
+		pfc: pinctrl@e6050000 {
+			compatible = "renesas,pfc-r8a779g0";
+			reg = <0 0xe6050000 0 0x16c>, <0 0xe6050800 0 0x16c>,
+			      <0 0xe6058000 0 0x16c>, <0 0xe6058800 0 0x16c>,
+			      <0 0xe6060000 0 0x16c>, <0 0xe6060800 0 0x16c>,
+			      <0 0xe6061000 0 0x16c>, <0 0xe6061800 0 0x16c>,
+			      <0 0xe6068000 0 0x16c>;
+		};
+
+		gpio0: gpio@e6050180 {
+			compatible = "renesas,gpio-r8a779g0",
+				     "renesas,rcar-gen4-gpio";
+			reg = <0 0xe6050180 0 0x54>;
+			interrupts = <GIC_SPI 619 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 915>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 915>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pfc 0 0 19>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio1: gpio@e6050980 {
+			compatible = "renesas,gpio-r8a779g0",
+				     "renesas,rcar-gen4-gpio";
+			reg = <0 0xe6050980 0 0x54>;
+			interrupts = <GIC_SPI 623 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 915>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 915>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pfc 0 32 29>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio2: gpio@e6058180 {
+			compatible = "renesas,gpio-r8a779g0",
+				     "renesas,rcar-gen4-gpio";
+			reg = <0 0xe6058180 0 0x54>;
+			interrupts = <GIC_SPI 627 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 916>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 916>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pfc 0 64 20>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio3: gpio@e6058980 {
+			compatible = "renesas,gpio-r8a779g0",
+				     "renesas,rcar-gen4-gpio";
+			reg = <0 0xe6058980 0 0x54>;
+			interrupts = <GIC_SPI 631 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 916>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 916>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pfc 0 96 30>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio4: gpio@e6060180 {
+			compatible = "renesas,gpio-r8a779g0",
+				     "renesas,rcar-gen4-gpio";
+			reg = <0 0xe6060180 0 0x54>;
+			interrupts = <GIC_SPI 635 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 917>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 917>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pfc 0 128 25>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio5: gpio@e6060980 {
+			compatible = "renesas,gpio-r8a779g0",
+				     "renesas,rcar-gen4-gpio";
+			reg = <0 0xe6060980 0 0x54>;
+			interrupts = <GIC_SPI 639 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 917>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 917>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pfc 0 160 21>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio6: gpio@e6061180 {
+			compatible = "renesas,gpio-r8a779g0",
+				     "renesas,rcar-gen4-gpio";
+			reg = <0 0xe6061180 0 0x54>;
+			interrupts = <GIC_SPI 643 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 917>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 917>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pfc 0 192 21>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio7: gpio@e6061980 {
+			compatible = "renesas,gpio-r8a779g0",
+				     "renesas,rcar-gen4-gpio";
+			reg = <0 0xe6061980 0 0x54>;
+			interrupts = <GIC_SPI 647 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 917>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 917>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pfc 0 224 21>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gpio8: gpio@e6068180 {
+			compatible = "renesas,gpio-r8a779g0",
+				     "renesas,rcar-gen4-gpio";
+			reg = <0 0xe6068180 0 0x54>;
+			interrupts = <GIC_SPI 651 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 918>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 918>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pfc 0 256 14>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
 		cpg: clock-controller@e6150000 {
 			compatible = "renesas,r8a779g0-cpg-mssr";
 			reg = <0 0xe6150000 0 0x4000>;
@@ -80,6 +235,90 @@
 			#power-domain-cells = <1>;
 		};
 
+		i2c0: i2c@e6500000 {
+			compatible = "renesas,i2c-r8a779g0",
+				     "renesas,rcar-gen4-i2c";
+			reg = <0 0xe6500000 0 0x40>;
+			interrupts = <GIC_SPI 610 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 518>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 518>;
+			i2c-scl-internal-delay-ns = <110>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c1: i2c@e6508000 {
+			compatible = "renesas,i2c-r8a779g0",
+				     "renesas,rcar-gen4-i2c";
+			reg = <0 0xe6508000 0 0x40>;
+			interrupts = <GIC_SPI 611 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 519>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 519>;
+			i2c-scl-internal-delay-ns = <110>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c2: i2c@e6510000 {
+			compatible = "renesas,i2c-r8a779g0",
+				     "renesas,rcar-gen4-i2c";
+			reg = <0 0xe6510000 0 0x40>;
+			interrupts = <GIC_SPI 612 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 520>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 520>;
+			i2c-scl-internal-delay-ns = <110>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c3: i2c@e66d0000 {
+			compatible = "renesas,i2c-r8a779g0",
+				     "renesas,rcar-gen4-i2c";
+			reg = <0 0xe66d0000 0 0x40>;
+			interrupts = <GIC_SPI 613 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 521>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 521>;
+			i2c-scl-internal-delay-ns = <110>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c4: i2c@e66d8000 {
+			compatible = "renesas,i2c-r8a779g0",
+				     "renesas,rcar-gen4-i2c";
+			reg = <0 0xe66d8000 0 0x40>;
+			interrupts = <GIC_SPI 614 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 522>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 522>;
+			i2c-scl-internal-delay-ns = <110>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		i2c5: i2c@e66e0000 {
+			compatible = "renesas,i2c-r8a779g0",
+				     "renesas,rcar-gen4-i2c";
+			reg = <0 0xe66e0000 0 0x40>;
+			interrupts = <GIC_SPI 615 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD 523>;
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 523>;
+			i2c-scl-internal-delay-ns = <110>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		hscif0: serial@e6540000 {
 			compatible = "renesas,hscif-r8a779g0",
 				     "renesas,rcar-gen4-hscif",
@@ -95,6 +334,147 @@
 			status = "disabled";
 		};
 
+		avb0: ethernet@e6800000 {
+			compatible = "renesas,etheravb-r8a779g0",
+				     "renesas,etheravb-rcar-gen4";
+			reg = <0 0xe6800000 0 0x800>;
+			interrupts = <GIC_SPI 335 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 336 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 337 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 338 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 339 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 340 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 341 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 342 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 343 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 344 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 345 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 346 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 347 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 348 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 349 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 350 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 351 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 352 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 353 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 354 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 355 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 356 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 357 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 358 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 359 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1", "ch2", "ch3", "ch4",
+					  "ch5", "ch6", "ch7", "ch8", "ch9",
+					  "ch10", "ch11", "ch12", "ch13",
+					  "ch14", "ch15", "ch16", "ch17",
+					  "ch18", "ch19", "ch20", "ch21",
+					  "ch22", "ch23", "ch24";
+			clocks = <&cpg CPG_MOD 211>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 211>;
+			phy-mode = "rgmii";
+			rx-internal-delay-ps = <0>;
+			tx-internal-delay-ps = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		avb1: ethernet@e6810000 {
+			compatible = "renesas,etheravb-r8a779g0",
+				     "renesas,etheravb-rcar-gen4";
+			reg = <0 0xe6810000 0 0x800>;
+			interrupts = <GIC_SPI 360 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 361 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 362 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 363 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 364 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 365 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 366 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 367 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 368 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 369 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 370 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 371 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 372 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 373 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 374 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 375 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 376 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 377 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 378 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 379 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 380 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 381 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 382 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 383 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 384 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1", "ch2", "ch3", "ch4",
+					  "ch5", "ch6", "ch7", "ch8", "ch9",
+					  "ch10", "ch11", "ch12", "ch13",
+					  "ch14", "ch15", "ch16", "ch17",
+					  "ch18", "ch19", "ch20", "ch21",
+					  "ch22", "ch23", "ch24";
+			clocks = <&cpg CPG_MOD 212>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 212>;
+			phy-mode = "rgmii";
+			rx-internal-delay-ps = <0>;
+			tx-internal-delay-ps = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
+		avb2: ethernet@e6820000 {
+			compatible = "renesas,etheravb-r8a779g0",
+				     "renesas,etheravb-rcar-gen4";
+			reg = <0 0xe6820000 0 0x1000>;
+			interrupts = <GIC_SPI 385 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 386 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 387 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 388 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 389 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 390 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 391 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 392 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 393 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 394 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 395 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 396 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 397 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 398 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 399 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 400 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 401 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 402 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 403 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 404 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 406 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 407 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 408 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 409 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ch0", "ch1", "ch2", "ch3", "ch4",
+					  "ch5", "ch6", "ch7", "ch8", "ch9",
+					  "ch10", "ch11", "ch12", "ch13",
+					  "ch14", "ch15", "ch16", "ch17",
+					  "ch18", "ch19", "ch20", "ch21",
+					  "ch22", "ch23", "ch24";
+			clocks = <&cpg CPG_MOD 213>;
+			clock-names = "fck";
+			power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>;
+			resets = <&cpg 213>;
+			phy-mode = "rgmii";
+			rx-internal-delay-ps = <0>;
+			tx-internal-delay-ps = <0>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+			status = "disabled";
+		};
+
 		gic: interrupt-controller@f1000000 {
 			compatible = "arm,gic-v3";
 			#interrupt-cells = <3>;
diff --git a/arch/arm64/boot/dts/renesas/r8a779mb.dtsi b/arch/arm64/boot/dts/renesas/r8a779mb.dtsi
new file mode 100644
index 000000000000..40d1dce2f350
--- /dev/null
+++ b/arch/arm64/boot/dts/renesas/r8a779mb.dtsi
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: (GPL-2.0 or MIT)
+/*
+ * Device Tree Source for the R-Car H3Ne-1.7G (R8A779MB) SoC
+ *
+ * Copyright (C) 2022 Glider bv
+ */
+
+#include "r8a77951.dtsi"
+
+/ {
+	compatible = "renesas,r8a779mb", "renesas,r8a7795";
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a07g043.dtsi b/arch/arm64/boot/dts/renesas/r9a07g043.dtsi
index 40201a16d653..689aa4ba416b 100644
--- a/arch/arm64/boot/dts/renesas/r9a07g043.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a07g043.dtsi
@@ -13,14 +13,14 @@
 	#address-cells = <2>;
 	#size-cells = <2>;
 
-	audio_clk1: audio-clk1 {
+	audio_clk1: audio1-clk {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		/* This value must be overridden by boards that provide it */
 		clock-frequency = <0>;
 	};
 
-	audio_clk2: audio-clk2 {
+	audio_clk2: audio2-clk {
 		compatible = "fixed-clock";
 		#clock-cells = <0>;
 		/* This value must be overridden by boards that provide it */
@@ -196,6 +196,8 @@
 			interrupt-names = "error", "rx", "tx";
 			clocks = <&cpg CPG_MOD R9A07G043_RSPI0_CLKB>;
 			resets = <&cpg R9A07G043_RSPI0_RST>;
+			dmas = <&dmac 0x2e95>, <&dmac 0x2e96>;
+			dma-names = "tx", "rx";
 			power-domains = <&cpg>;
 			num-cs = <1>;
 			#address-cells = <1>;
@@ -212,6 +214,8 @@
 			interrupt-names = "error", "rx", "tx";
 			clocks = <&cpg CPG_MOD R9A07G043_RSPI1_CLKB>;
 			resets = <&cpg R9A07G043_RSPI1_RST>;
+			dmas = <&dmac 0x2e99>, <&dmac 0x2e9a>;
+			dma-names = "tx", "rx";
 			power-domains = <&cpg>;
 			num-cs = <1>;
 			#address-cells = <1>;
@@ -228,6 +232,8 @@
 			interrupt-names = "error", "rx", "tx";
 			clocks = <&cpg CPG_MOD R9A07G043_RSPI2_CLKB>;
 			resets = <&cpg R9A07G043_RSPI2_RST>;
+			dmas = <&dmac 0x2e9d>, <&dmac 0x2e9e>;
+			dma-names = "tx", "rx";
 			power-domains = <&cpg>;
 			num-cs = <1>;
 			#address-cells = <1>;
@@ -334,8 +340,8 @@
 			compatible = "renesas,r9a07g043-sci", "renesas,sci";
 			reg = <0 0x1004d000 0 0x400>;
 			interrupts = <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 406 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 407 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 406 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 407 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 408 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "eri", "rxi", "txi", "tei";
 			clocks = <&cpg CPG_MOD R9A07G043_SCI0_CLKP>;
@@ -349,8 +355,8 @@
 			compatible = "renesas,r9a07g043-sci", "renesas,sci";
 			reg = <0 0x1004d400 0 0x400>;
 			interrupts = <GIC_SPI 409 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 410 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 411 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 410 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 411 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 412 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "eri", "rxi", "txi", "tei";
 			clocks = <&cpg CPG_MOD R9A07G043_SCI1_CLKP>;
@@ -613,7 +619,7 @@
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_LOW>;
 		};
 
-		sdhi0: mmc@11c00000  {
+		sdhi0: mmc@11c00000 {
 			compatible = "renesas,sdhi-r9a07g043",
 				     "renesas,rcar-gen3-sdhi";
 			reg = <0x0 0x11c00000 0 0x10000>;
diff --git a/arch/arm64/boot/dts/renesas/r9a07g043u11-smarc.dts b/arch/arm64/boot/dts/renesas/r9a07g043u11-smarc.dts
index 121e55282d18..059885a01ede 100644
--- a/arch/arm64/boot/dts/renesas/r9a07g043u11-smarc.dts
+++ b/arch/arm64/boot/dts/renesas/r9a07g043u11-smarc.dts
@@ -6,7 +6,19 @@
  */
 
 /dts-v1/;
+
+/*
+ * DIP-Switch SW1 setting
+ * 1 : High; 0: Low
+ * SW1-2 : SW_SD0_DEV_SEL	(0: uSD; 1: eMMC)
+ * SW1-3 : SW_ET0_EN_N		(0: ETHER0; 1: CAN0, CAN1, SSI1, RSPI1)
+ * Please change below macros according to SW1 setting on the SoM
+ */
+#define SW_SW0_DEV_SEL	1
+#define SW_ET0_EN_N	1
+
 #include "r9a07g043.dtsi"
+#include "rzg2ul-smarc-som.dtsi"
 #include "rzg2ul-smarc.dtsi"
 
 / {
diff --git a/arch/arm64/boot/dts/renesas/r9a07g044.dtsi b/arch/arm64/boot/dts/renesas/r9a07g044.dtsi
index 3652e511160f..2283d4fb8736 100644
--- a/arch/arm64/boot/dts/renesas/r9a07g044.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a07g044.dtsi
@@ -261,6 +261,8 @@
 			interrupt-names = "error", "rx", "tx";
 			clocks = <&cpg CPG_MOD R9A07G044_RSPI0_CLKB>;
 			resets = <&cpg R9A07G044_RSPI0_RST>;
+			dmas = <&dmac 0x2e95>, <&dmac 0x2e96>;
+			dma-names = "tx", "rx";
 			power-domains = <&cpg>;
 			num-cs = <1>;
 			#address-cells = <1>;
@@ -277,6 +279,8 @@
 			interrupt-names = "error", "rx", "tx";
 			clocks = <&cpg CPG_MOD R9A07G044_RSPI1_CLKB>;
 			resets = <&cpg R9A07G044_RSPI1_RST>;
+			dmas = <&dmac 0x2e99>, <&dmac 0x2e9a>;
+			dma-names = "tx", "rx";
 			power-domains = <&cpg>;
 			num-cs = <1>;
 			#address-cells = <1>;
@@ -293,6 +297,8 @@
 			interrupt-names = "error", "rx", "tx";
 			clocks = <&cpg CPG_MOD R9A07G044_RSPI2_CLKB>;
 			resets = <&cpg R9A07G044_RSPI2_RST>;
+			dmas = <&dmac 0x2e9d>, <&dmac 0x2e9e>;
+			dma-names = "tx", "rx";
 			power-domains = <&cpg>;
 			num-cs = <1>;
 			#address-cells = <1>;
@@ -394,8 +400,8 @@
 			compatible = "renesas,r9a07g044-sci", "renesas,sci";
 			reg = <0 0x1004d000 0 0x400>;
 			interrupts = <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 406 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 407 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 406 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 407 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 408 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "eri", "rxi", "txi", "tei";
 			clocks = <&cpg CPG_MOD R9A07G044_SCI0_CLKP>;
@@ -409,8 +415,8 @@
 			compatible = "renesas,r9a07g044-sci", "renesas,sci";
 			reg = <0 0x1004d400 0 0x400>;
 			interrupts = <GIC_SPI 409 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 410 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 411 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 410 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 411 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 412 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "eri", "rxi", "txi", "tei";
 			clocks = <&cpg CPG_MOD R9A07G044_SCI1_CLKP>;
@@ -638,6 +644,10 @@
 			reg = <0 0x11030000 0 0x10000>;
 			gpio-controller;
 			#gpio-cells = <2>;
+			#address-cells = <2>;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&irqc>;
+			interrupt-controller;
 			gpio-ranges = <&pinctrl 0 0 392>;
 			clocks = <&cpg CPG_MOD R9A07G044_GPIO_HCLK>;
 			power-domains = <&cpg>;
@@ -646,6 +656,61 @@
 				 <&cpg R9A07G044_GPIO_SPARE_RESETN>;
 		};
 
+		irqc: interrupt-controller@110a0000 {
+			compatible = "renesas,r9a07g044-irqc",
+				     "renesas,rzg2l-irqc";
+			#interrupt-cells = <2>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0 0x110a0000 0 0x10000>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 446 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 447 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 448 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 450 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 451 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 452 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 453 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 454 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 455 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 459 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 460 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 463 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 464 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 466 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 467 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 468 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 469 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 470 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 471 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 472 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 473 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 474 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 475 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD R9A07G044_IA55_CLK>,
+				 <&cpg CPG_MOD R9A07G044_IA55_PCLK>;
+			clock-names = "clk", "pclk";
+			power-domains = <&cpg>;
+			resets = <&cpg R9A07G044_IA55_RESETN>;
+		};
+
 		dmac: dma-controller@11820000 {
 			compatible = "renesas,r9a07g044-dmac",
 				     "renesas,rz-dmac";
@@ -713,7 +778,7 @@
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_LOW>;
 		};
 
-		sdhi0: mmc@11c00000  {
+		sdhi0: mmc@11c00000 {
 			compatible = "renesas,sdhi-r9a07g044",
 				     "renesas,rcar-gen3-sdhi";
 			reg = <0x0 0x11c00000 0 0x10000>;
diff --git a/arch/arm64/boot/dts/renesas/r9a07g054.dtsi b/arch/arm64/boot/dts/renesas/r9a07g054.dtsi
index 4d6b9d7684c9..358d4c34465f 100644
--- a/arch/arm64/boot/dts/renesas/r9a07g054.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a07g054.dtsi
@@ -261,6 +261,8 @@
 			interrupt-names = "error", "rx", "tx";
 			clocks = <&cpg CPG_MOD R9A07G054_RSPI0_CLKB>;
 			resets = <&cpg R9A07G054_RSPI0_RST>;
+			dmas = <&dmac 0x2e95>, <&dmac 0x2e96>;
+			dma-names = "tx", "rx";
 			power-domains = <&cpg>;
 			num-cs = <1>;
 			#address-cells = <1>;
@@ -277,6 +279,8 @@
 			interrupt-names = "error", "rx", "tx";
 			clocks = <&cpg CPG_MOD R9A07G054_RSPI1_CLKB>;
 			resets = <&cpg R9A07G054_RSPI1_RST>;
+			dmas = <&dmac 0x2e99>, <&dmac 0x2e9a>;
+			dma-names = "tx", "rx";
 			power-domains = <&cpg>;
 			num-cs = <1>;
 			#address-cells = <1>;
@@ -293,6 +297,8 @@
 			interrupt-names = "error", "rx", "tx";
 			clocks = <&cpg CPG_MOD R9A07G054_RSPI2_CLKB>;
 			resets = <&cpg R9A07G054_RSPI2_RST>;
+			dmas = <&dmac 0x2e9d>, <&dmac 0x2e9e>;
+			dma-names = "tx", "rx";
 			power-domains = <&cpg>;
 			num-cs = <1>;
 			#address-cells = <1>;
@@ -399,8 +405,8 @@
 			compatible = "renesas,r9a07g054-sci", "renesas,sci";
 			reg = <0 0x1004d000 0 0x400>;
 			interrupts = <GIC_SPI 405 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 406 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 407 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 406 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 407 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 408 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "eri", "rxi", "txi", "tei";
 			clocks = <&cpg CPG_MOD R9A07G054_SCI0_CLKP>;
@@ -414,8 +420,8 @@
 			compatible = "renesas,r9a07g054-sci", "renesas,sci";
 			reg = <0 0x1004d400 0 0x400>;
 			interrupts = <GIC_SPI 409 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 410 IRQ_TYPE_LEVEL_HIGH>,
-				     <GIC_SPI 411 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 410 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 411 IRQ_TYPE_EDGE_RISING>,
 				     <GIC_SPI 412 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-names = "eri", "rxi", "txi", "tei";
 			clocks = <&cpg CPG_MOD R9A07G054_SCI1_CLKP>;
@@ -644,6 +650,10 @@
 			reg = <0 0x11030000 0 0x10000>;
 			gpio-controller;
 			#gpio-cells = <2>;
+			#address-cells = <2>;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&irqc>;
+			interrupt-controller;
 			gpio-ranges = <&pinctrl 0 0 392>;
 			clocks = <&cpg CPG_MOD R9A07G054_GPIO_HCLK>;
 			power-domains = <&cpg>;
@@ -652,6 +662,61 @@
 				 <&cpg R9A07G054_GPIO_SPARE_RESETN>;
 		};
 
+		irqc: interrupt-controller@110a0000 {
+			compatible = "renesas,r9a07g054-irqc",
+				     "renesas,rzg2l-irqc";
+			#interrupt-cells = <2>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0 0x110a0000 0 0x10000>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 444 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 445 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 446 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 447 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 448 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 449 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 450 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 451 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 452 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 453 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 454 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 455 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 456 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 457 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 458 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 459 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 460 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 461 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 462 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 463 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 464 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 465 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 466 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 467 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 468 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 469 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 470 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 471 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 472 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 473 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 474 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 475 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD R9A07G054_IA55_CLK>,
+				 <&cpg CPG_MOD R9A07G054_IA55_PCLK>;
+			clock-names = "clk", "pclk";
+			power-domains = <&cpg>;
+			resets = <&cpg R9A07G054_IA55_RESETN>;
+		};
+
 		dmac: dma-controller@11820000 {
 			compatible = "renesas,r9a07g054-dmac",
 				     "renesas,rz-dmac";
@@ -719,7 +784,7 @@
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_LOW>;
 		};
 
-		sdhi0: mmc@11c00000  {
+		sdhi0: mmc@11c00000 {
 			compatible = "renesas,sdhi-r9a07g054",
 				     "renesas,rcar-gen3-sdhi";
 			reg = <0x0 0x11c00000 0 0x10000>;
diff --git a/arch/arm64/boot/dts/renesas/r9a09g011-v2mevk2.dts b/arch/arm64/boot/dts/renesas/r9a09g011-v2mevk2.dts
index c3a52fa0b16e..5c15d73d059f 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g011-v2mevk2.dts
+++ b/arch/arm64/boot/dts/renesas/r9a09g011-v2mevk2.dts
@@ -7,6 +7,7 @@
 
 /dts-v1/;
 #include "r9a09g011.dtsi"
+#include <dt-bindings/pinctrl/rzv2m-pinctrl.h>
 
 / {
 	model = "RZ/V2M Evaluation Kit 2.0";
@@ -53,6 +54,32 @@
 	clock-frequency = <48000000>;
 };
 
+&i2c0 {
+	pinctrl-0 = <&i2c0_pins>;
+	pinctrl-names = "default";
+	clock-frequency = <400000>;
+	status = "okay";
+};
+
+&i2c2 {
+	pinctrl-0 = <&i2c2_pins>;
+	pinctrl-names = "default";
+	clock-frequency = <100000>;
+	status = "okay";
+};
+
+&pinctrl {
+	i2c0_pins: i2c0 {
+		pinmux = <RZV2M_PORT_PINMUX(5, 0, 2)>, /* SDA */
+			 <RZV2M_PORT_PINMUX(5, 1, 2)>; /* SCL */
+	};
+
+	i2c2_pins: i2c2 {
+		pinmux = <RZV2M_PORT_PINMUX(3, 8, 2)>, /* SDA */
+			 <RZV2M_PORT_PINMUX(3, 9, 2)>; /* SCL */
+	};
+};
+
 &uart0 {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/renesas/r9a09g011.dtsi b/arch/arm64/boot/dts/renesas/r9a09g011.dtsi
index d4cc5459fbb7..fb1a97202c38 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g011.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g011.dtsi
@@ -123,6 +123,34 @@
 			#power-domain-cells = <0>;
 		};
 
+		i2c0: i2c@a4030000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r9a09g011", "renesas,rzv2m-i2c";
+			reg = <0 0xa4030000 0 0x80>;
+			interrupts = <GIC_SPI 232 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 236 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "tia", "tis";
+			clocks = <&cpg CPG_MOD R9A09G011_IIC_PCLK0>;
+			resets = <&cpg R9A09G011_IIC_GPA_PRESETN>;
+			power-domains = <&cpg>;
+			status = "disabled";
+		};
+
+		i2c2: i2c@a4030100 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			compatible = "renesas,i2c-r9a09g011", "renesas,rzv2m-i2c";
+			reg = <0 0xa4030100 0 0x80>;
+			interrupts = <GIC_SPI 234 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 238 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "tia", "tis";
+			clocks = <&cpg CPG_MOD R9A09G011_IIC_PCLK1>;
+			resets = <&cpg R9A09G011_IIC_GPB_PRESETN>;
+			power-domains = <&cpg>;
+			status = "disabled";
+		};
+
 		uart0: serial@a4040000 {
 			compatible = "renesas,r9a09g011-uart", "renesas,em-uart";
 			reg = <0 0xa4040000 0 0x80>;
@@ -132,6 +160,56 @@
 			clock-names = "sclk", "pclk";
 			status = "disabled";
 		};
+
+		pinctrl: pinctrl@b6250000 {
+			compatible = "renesas,r9a09g011-pinctrl";
+			reg = <0 0xb6250000 0 0x800>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&pinctrl 0 0 352>;
+			interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 70 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 71 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 72 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 73 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 74 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 75 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 77 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 81 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 85 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 86 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 87 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 88 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 89 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 90 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 91 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 92 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 98 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cpg CPG_MOD R9A09G011_PFC_PCLK>;
+			power-domains = <&cpg>;
+			resets = <&cpg R9A09G011_PFC_PRESETN>;
+		};
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi
index 9410796c8ad6..c4faff092380 100644
--- a/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2l-smarc-som.dtsi
@@ -6,6 +6,7 @@
  */
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irqc-rzg2l.h>
 #include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
 
 /* SW1[2] should be at position 2/OFF to enable 64 GB eMMC */
@@ -94,6 +95,8 @@
 		compatible = "ethernet-phy-id0022.1640",
 			     "ethernet-phy-ieee802.3-c22";
 		reg = <7>;
+		interrupt-parent = <&irqc>;
+		interrupts = <RZG2L_IRQ2 IRQ_TYPE_LEVEL_LOW>;
 		rxc-skew-psec = <2400>;
 		txc-skew-psec = <2400>;
 		rxdv-skew-psec = <0>;
@@ -120,6 +123,8 @@
 		compatible = "ethernet-phy-id0022.1640",
 			     "ethernet-phy-ieee802.3-c22";
 		reg = <7>;
+		interrupt-parent = <&irqc>;
+		interrupts = <RZG2L_IRQ3 IRQ_TYPE_LEVEL_LOW>;
 		rxc-skew-psec = <2400>;
 		txc-skew-psec = <2400>;
 		rxdv-skew-psec = <0>;
@@ -171,7 +176,8 @@
 			 <RZG2L_PORT_PINMUX(25, 0, 1)>, /* ET0_RXD0 */
 			 <RZG2L_PORT_PINMUX(25, 1, 1)>, /* ET0_RXD1 */
 			 <RZG2L_PORT_PINMUX(26, 0, 1)>, /* ET0_RXD2 */
-			 <RZG2L_PORT_PINMUX(26, 1, 1)>; /* ET0_RXD3 */
+			 <RZG2L_PORT_PINMUX(26, 1, 1)>, /* ET0_RXD3 */
+			 <RZG2L_PORT_PINMUX(1, 0, 1)>;  /* IRQ2 */
 	};
 
 	eth1_pins: eth1 {
@@ -189,7 +195,8 @@
 			 <RZG2L_PORT_PINMUX(34, 1, 1)>, /* ET1_RXD0 */
 			 <RZG2L_PORT_PINMUX(35, 0, 1)>, /* ET1_RXD1 */
 			 <RZG2L_PORT_PINMUX(35, 1, 1)>, /* ET1_RXD2 */
-			 <RZG2L_PORT_PINMUX(36, 0, 1)>; /* ET1_RXD3 */
+			 <RZG2L_PORT_PINMUX(36, 0, 1)>, /* ET1_RXD3 */
+			 <RZG2L_PORT_PINMUX(1, 1, 1)>;  /* IRQ3 */
 	};
 
 	gpio-sd0-pwr-en-hog {
diff --git a/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi
index cf3b3d118ef1..2a0feb53f0dc 100644
--- a/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2ul-smarc-som.dtsi
@@ -263,8 +263,3 @@
 	status = "okay";
 	timeout-sec = <60>;
 };
-
-&wdt2 {
-	status = "okay";
-	timeout-sec = <60>;
-};
diff --git a/arch/arm64/boot/dts/renesas/rzg2ul-smarc.dtsi b/arch/arm64/boot/dts/renesas/rzg2ul-smarc.dtsi
index f9835c12023e..2a1331ed1a5c 100644
--- a/arch/arm64/boot/dts/renesas/rzg2ul-smarc.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg2ul-smarc.dtsi
@@ -5,17 +5,6 @@
  * Copyright (C) 2022 Renesas Electronics Corp.
  */
 
-/*
- * DIP-Switch SW1 setting
- * 1 : High; 0: Low
- * SW1-2 : SW_SD0_DEV_SEL	(0: uSD; 1: eMMC)
- * SW1-3 : SW_ET0_EN_N		(0: ETHER0; 1: CAN0, CAN1, SSI1, RSPI1)
- * Please change below macros according to SW1 setting
- */
-#define SW_SW0_DEV_SEL	1
-#define SW_ET0_EN_N	1
-
-#include "rzg2ul-smarc-som.dtsi"
 #include "rzg2ul-smarc-pinfunction.dtsi"
 #include "rz-smarc-common.dtsi"
 
diff --git a/arch/arm64/boot/dts/renesas/salvator-common.dtsi b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
index b7c7911858b2..d97473401819 100644
--- a/arch/arm64/boot/dts/renesas/salvator-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
@@ -33,6 +33,14 @@
 
 / {
 	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
+		i2c6 = &i2c6;
+		i2c7 = &i2c_dvfs;
 		serial0 = &scif2;
 		serial1 = &hscif1;
 		ethernet0 = &avb;
diff --git a/arch/arm64/boot/dts/renesas/ulcb.dtsi b/arch/arm64/boot/dts/renesas/ulcb.dtsi
index 0772dfe4adff..29cedf4dc1a9 100644
--- a/arch/arm64/boot/dts/renesas/ulcb.dtsi
+++ b/arch/arm64/boot/dts/renesas/ulcb.dtsi
@@ -21,6 +21,14 @@
 	model = "Renesas R-Car Gen3 ULCB board";
 
 	aliases {
+		i2c0 = &i2c0;
+		i2c1 = &i2c1;
+		i2c2 = &i2c2;
+		i2c3 = &i2c3;
+		i2c4 = &i2c4;
+		i2c5 = &i2c5;
+		i2c6 = &i2c6;
+		i2c7 = &i2c_dvfs;
 		serial0 = &scif2;
 		ethernet0 = &avb;
 		mmc0 = &sdhi2;
diff --git a/arch/arm64/boot/dts/rockchip/Makefile b/arch/arm64/boot/dts/rockchip/Makefile
index ef79a672804a..8c15593c0ca4 100644
--- a/arch/arm64/boot/dts/rockchip/Makefile
+++ b/arch/arm64/boot/dts/rockchip/Makefile
@@ -21,6 +21,7 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-lion-haikou.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-orion-r68-meta.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-px5-evb.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3368-r88.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-eaidk-610.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-evb.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-ficus.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-firefly.dtb
@@ -40,12 +41,15 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-m4.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-m4b.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-neo4.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-r4s.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-nanopi-r4s-enterprise.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-orangepi.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-pinebook-pro.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-pinephone-pro.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-puma-haikou.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-roc-pc.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-roc-pc-mezzanine.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-roc-pc-plus.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-4c-plus.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4a.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4a-plus.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rock-pi-4b.dtb
@@ -57,6 +61,8 @@ dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-rockpro64.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399-sapphire-excavator.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3399pro-rock-pi-n10.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-anbernic-rg353p.dtb
+dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-anbernic-rg503.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-pinenote-v1.1.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-pinenote-v1.2.dtb
 dtb-$(CONFIG_ARCH_ROCKCHIP) += rk3566-quartz64-a.dtb
diff --git a/arch/arm64/boot/dts/rockchip/px30-evb.dts b/arch/arm64/boot/dts/rockchip/px30-evb.dts
index 848bc39cf86a..07008d84434c 100644
--- a/arch/arm64/boot/dts/rockchip/px30-evb.dts
+++ b/arch/arm64/boot/dts/rockchip/px30-evb.dts
@@ -450,8 +450,8 @@
 		dvdd-supply = <&vcc1v5_dvp>;
 		dovdd-supply = <&vcc1v8_dvp>;
 		pinctrl-names = "default";
-		pinctrl-0 = <&cif_clkout_m0>;
-		reset-gpios = <&gpio2 14 GPIO_ACTIVE_LOW>;
+		pinctrl-0 = <&cif_clkout_m0 &mipi_pdn>;
+		reset-gpios = <&gpio2 RK_PB6 GPIO_ACTIVE_LOW>;
 
 		port {
 			ucam_out: endpoint {
@@ -537,6 +537,19 @@
 				<0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
+
+	cif-m0 {
+		cif_clkout_m0: cif-clkout-m0 {
+			rockchip,pins =
+				<2 RK_PB3 1 &pcfg_pull_none_12ma>;
+		};
+	};
+
+	mipi {
+		mipi_pdn: mipi-pdn {
+			rockchip,pins = <2 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
 };
 
 &pmu_io_domains {
diff --git a/arch/arm64/boot/dts/rockchip/px30.dtsi b/arch/arm64/boot/dts/rockchip/px30.dtsi
index 214f94fea3dc..bfa3580429d1 100644
--- a/arch/arm64/boot/dts/rockchip/px30.dtsi
+++ b/arch/arm64/boot/dts/rockchip/px30.dtsi
@@ -365,6 +365,28 @@
 		status = "disabled";
 	};
 
+	i2s0_8ch: i2s@ff060000 {
+		compatible = "rockchip,px30-i2s-tdm";
+		reg = <0x0 0xff060000 0x0 0x1000>;
+		interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cru SCLK_I2S0_TX>, <&cru SCLK_I2S0_RX>, <&cru HCLK_I2S0>;
+		clock-names = "mclk_tx", "mclk_rx", "hclk";
+		dmas = <&dmac 16>, <&dmac 17>;
+		dma-names = "tx", "rx";
+		rockchip,grf = <&grf>;
+		resets = <&cru SRST_I2S0_TX>, <&cru SRST_I2S0_RX>;
+		reset-names = "tx-m", "rx-m";
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2s0_8ch_sclktx &i2s0_8ch_sclkrx
+			     &i2s0_8ch_lrcktx &i2s0_8ch_lrckrx
+			     &i2s0_8ch_sdo0 &i2s0_8ch_sdi0
+			     &i2s0_8ch_sdo1 &i2s0_8ch_sdi1
+			     &i2s0_8ch_sdo2 &i2s0_8ch_sdi2
+			     &i2s0_8ch_sdo3 &i2s0_8ch_sdi3>;
+		#sound-dai-cells = <0>;
+		status = "disabled";
+	};
+
 	i2s1_2ch: i2s@ff070000 {
 		compatible = "rockchip,px30-i2s", "rockchip,rk3066-i2s";
 		reg = <0x0 0xff070000 0x0 0x1000>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts b/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts
index 415aa9ff8bd4..72899a714310 100644
--- a/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3326-odroid-go2.dts
@@ -52,6 +52,25 @@
 		pwms = <&pwm1 0 25000 0>;
 	};
 
+	battery: battery {
+		compatible = "simple-battery";
+		charge-full-design-microamp-hours = <3000000>;
+		charge-term-current-microamp = <300000>;
+		constant-charge-current-max-microamp = <2000000>;
+		constant-charge-voltage-max-microvolt = <4200000>;
+		factory-internal-resistance-micro-ohms = <180000>;
+		voltage-max-design-microvolt = <4100000>;
+		voltage-min-design-microvolt = <3500000>;
+
+		ocv-capacity-celsius = <20>;
+		ocv-capacity-table-0 =	<4046950 100>, <4001920 95>, <3967900 90>, <3919950 85>,
+					<3888450 80>, <3861850 75>, <3831540 70>, <3799130 65>,
+					<3768190 60>, <3745650 55>, <3726610 50>, <3711630 45>,
+					<3696720 40>, <3685660 35>, <3674950 30>, <3663050 25>,
+					<3649470 20>, <3635260 15>, <3616920 10>, <3592440 5>,
+					<3574170 0>;
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 		pinctrl-names = "default";
@@ -472,6 +491,13 @@
 			};
 		};
 
+		rk817_charger: charger {
+			monitored-battery = <&battery>;
+			rockchip,resistor-sense-micro-ohms = <10000>;
+			rockchip,sleep-enter-current-microamp = <300000>;
+			rockchip,sleep-filter-current-microamp = <100000>;
+		};
+
 		rk817_codec: codec {
 			rockchip,mic-in-differential;
 		};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-eaidk-610.dts b/arch/arm64/boot/dts/rockchip/rk3399-eaidk-610.dts
new file mode 100644
index 000000000000..d1f343345f67
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3399-eaidk-610.dts
@@ -0,0 +1,939 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2022 Fuzhou Rockchip Electronics Co., Ltd.
+ */
+
+/dts-v1/;
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/pwm/pwm.h>
+#include <dt-bindings/usb/pd.h>
+#include "rk3399.dtsi"
+#include "rk3399-opp.dtsi"
+
+/ {
+	model = "OPEN AI LAB EAIDK-610";
+	compatible = "openailab,eaidk-610", "rockchip,rk3399";
+
+	aliases {
+		mmc0 = &sdio0;
+		mmc1 = &sdmmc;
+		mmc2 = &sdhci;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pwm0 0 25000 0>;
+		brightness-levels = <
+			  0   1   2   3   4   5   6   7
+			  8   9  10  11  12  13  14  15
+			 16  17  18  19  20  21  22  23
+			 24  25  26  27  28  29  30  31
+			 32  33  34  35  36  37  38  39
+			 40  41  42  43  44  45  46  47
+			 48  49  50  51  52  53  54  55
+			 56  57  58  59  60  61  62  63
+			 64  65  66  67  68  69  70  71
+			 72  73  74  75  76  77  78  79
+			 80  81  82  83  84  85  86  87
+			 88  89  90  91  92  93  94  95
+			 96  97  98  99 100 101 102 103
+			104 105 106 107 108 109 110 111
+			112 113 114 115 116 117 118 119
+			120 121 122 123 124 125 126 127
+			128 129 130 131 132 133 134 135
+			136 137 138 139 140 141 142 143
+			144 145 146 147 148 149 150 151
+			152 153 154 155 156 157 158 159
+			160 161 162 163 164 165 166 167
+			168 169 170 171 172 173 174 175
+			176 177 178 179 180 181 182 183
+			184 185 186 187 188 189 190 191
+			192 193 194 195 196 197 198 199
+			200 201 202 203 204 205 206 207
+			208 209 210 211 212 213 214 215
+			216 217 218 219 220 221 222 223
+			224 225 226 227 228 229 230 231
+			232 233 234 235 236 237 238 239
+			240 241 242 243 244 245 246 247
+			248 249 250 251 252 253 254 255>;
+		default-brightness-level = <200>;
+	};
+
+	clkin_gmac: external-gmac-clock {
+		compatible = "fixed-clock";
+		clock-frequency = <125000000>;
+		clock-output-names = "clkin_gmac";
+		#clock-cells = <0>;
+	};
+
+	dc_12v: dc-12v {
+		compatible = "regulator-fixed";
+		regulator-name = "dc_12v";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		autorepeat;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pwrbtn>;
+
+		key-power {
+			debounce-interval = <100>;
+			gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>;
+			label = "GPIO Key Power";
+			linux,code = <KEY_POWER>;
+			wakeup-source;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&work_led_pin>, <&user_led_pin>,
+			    <&heartbeat_led_pin>, <&wlan_active_led_pin>,
+			    <&bt_active_led_pin>;
+
+		work_led: led-0 {
+			label = "blue:work";
+			default-state = "on";
+			gpios = <&gpio0 RK_PA2 GPIO_ACTIVE_HIGH>;
+		};
+
+		user_led: led-1 {
+			label = "read:user";
+			default-state = "off";
+			gpios = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>;
+		};
+
+		heartbeat_led: led-2 {
+			label = "green:heartbeat";
+			linux,default-trigger = "heartbeat";
+			gpios = <&gpio0 RK_PB5 GPIO_ACTIVE_HIGH>;
+		};
+
+		wlan_active_led: led-3 {
+			label = "yellow:wlan";
+			gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "phy0tx";
+			default-state = "off";
+		};
+
+		bt_active_led: led-4 {
+			label = "blue:bt";
+			gpios = <&gpio2 RK_PD4 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "hci0-power";
+			default-state = "off";
+		};
+	};
+
+	rt5651-sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "realtek,rt5651-codec";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,widgets =
+			"Microphone", "Mic Jack",
+			"Headphone", "Headphone Jack";
+		simple-audio-card,routing =
+			"Mic Jack", "MICBIAS1",
+			"IN1P", "Mic Jack",
+			"Headphone Jack", "HPOL",
+			"Headphone Jack", "HPOR";
+		simple-audio-card,cpu {
+			sound-dai = <&i2s1>;
+		};
+		simple-audio-card,codec {
+			sound-dai = <&rt5651>;
+		};
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clocks = <&rk808 1>;
+		clock-names = "ext_clock";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_enable_h>;
+
+		/*
+		 * On the module itself this is one of these (depending
+		 * on the actual card populated):
+		 * - SDIO_RESET_L_WL_REG_ON
+		 * - PDN (power down when low)
+		 */
+		reset-gpios = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>;
+	};
+
+	/* switched by pmic_sleep */
+	vcc1v8_s3: vcca1v8_s3: vcc1v8-s3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc1v8_s3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc_1v8>;
+	};
+
+	vcc3v3_sys: vcc3v3-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&dc_12v>;
+	};
+
+	vcc5v0_sys: vcc5v0-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&dc_12v>;
+	};
+
+	/* For USB3.0 Port1/2 */
+	vcc5v0_host1: vcc5v0-host1-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc5v0_host1_en>;
+		regulator-name = "vcc5v0_host1";
+		regulator-always-on;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	/* For USB2.0 Port1/2 */
+	vcc5v0_host3: vcc5v0-host3-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio0 RK_PB0 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc5v0_host3_en>;
+		regulator-name = "vcc5v0_host3";
+		regulator-always-on;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vcc5v0_typec: vcc5v0-typec-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio4 RK_PC5 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc5v0_typec_en>;
+		regulator-name = "vcc5v0_typec";
+		regulator-always-on;
+		vin-supply = <&vcc3v3_sys>;
+	};
+
+	vdd_log: vdd-log {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_log";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+	};
+};
+
+&cpu_l0 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_l1 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_l2 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_l3 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_b0 {
+	cpu-supply = <&vdd_cpu_b>;
+};
+
+&cpu_b1 {
+	cpu-supply = <&vdd_cpu_b>;
+};
+
+&emmc_phy {
+	status = "okay";
+};
+
+&gmac {
+	assigned-clocks = <&cru SCLK_RMII_SRC>;
+	assigned-clock-parents = <&clkin_gmac>;
+	clock_in_out = "input";
+	phy-supply = <&vcc_lan>;
+	phy-mode = "rgmii";
+	pinctrl-names = "default";
+	pinctrl-0 = <&rgmii_pins>;
+	snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>;
+	snps,reset-active-low;
+	snps,reset-delays-us = <0 10000 50000>;
+	tx_delay = <0x28>;
+	rx_delay = <0x11>;
+	status = "okay";
+};
+
+&gpu {
+	mali-supply = <&vdd_gpu>;
+	status = "okay";
+};
+
+&hdmi {
+	ddc-i2c-bus = <&i2c3>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&hdmi_cec>;
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	rk808: pmic@1b {
+		compatible = "rockchip,rk808";
+		reg = <0x1b>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <21 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_int_l>;
+		rockchip,system-power-controller;
+		wakeup-source;
+		#clock-cells = <1>;
+		clock-output-names = "xin32k", "rk808-clkout2";
+
+		vcc1-supply = <&vcc3v3_sys>;
+		vcc2-supply = <&vcc3v3_sys>;
+		vcc3-supply = <&vcc3v3_sys>;
+		vcc4-supply = <&vcc3v3_sys>;
+		vcc6-supply = <&vcc3v3_sys>;
+		vcc7-supply = <&vcc3v3_sys>;
+		vcc8-supply = <&vcc3v3_sys>;
+		vcc9-supply = <&vcc3v3_sys>;
+		vcc10-supply = <&vcc3v3_sys>;
+		vcc11-supply = <&vcc3v3_sys>;
+		vcc12-supply = <&vcc3v3_sys>;
+		vddio-supply = <&vcc_3v0>;
+
+		regulators {
+			vdd_center: DCDC_REG1 {
+				regulator-name = "vdd_center";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <6001>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_cpu_l: DCDC_REG2 {
+				regulator-name = "vdd_cpu_l";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <6001>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_ddr: DCDC_REG3 {
+				regulator-name = "vcc_ddr";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc_1v8: DCDC_REG4 {
+				regulator-name = "vcc_1v8";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vcc1v8_dvp: LDO_REG1 {
+				regulator-name = "vcc1v8_dvp";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc2v8_dvp: LDO_REG2 {
+				regulator-name = "vcc2v8_dvp";
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc1v8_pmu: LDO_REG3 {
+				regulator-name = "vcc1v8_pmu";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vcc_sdio: LDO_REG4 {
+				regulator-name = "vcc_sdio";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3000000>;
+				};
+			};
+
+			vcca3v0_codec: LDO_REG5 {
+				regulator-name = "vcca3v0_codec";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_1v5: LDO_REG6 {
+				regulator-name = "vcc_1v5";
+				regulator-min-microvolt = <1500000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1500000>;
+				};
+			};
+
+			vcca1v8_codec: LDO_REG7 {
+				regulator-name = "vcca1v8_codec";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_3v0: LDO_REG8 {
+				regulator-name = "vcc_3v0";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-always-on;
+				regulator-boot-on;
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3000000>;
+				};
+			};
+
+			vcc3v3_s3: vcc_lan: SWITCH_REG1 {
+				regulator-name = "vcc3v3_s3";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc3v3_s0: SWITCH_REG2 {
+				regulator-name = "vcc3v3_s0";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+		};
+	};
+
+	vdd_cpu_b: regulator@40 {
+		compatible = "silergy,syr827";
+		reg = <0x40>;
+		fcs,suspend-voltage-selector = <1>;
+		regulator-name = "vdd_cpu_b";
+		pinctrl-names = "default";
+		pinctrl-0 = <&vsel1_pin>;
+		regulator-min-microvolt = <712500>;
+		regulator-max-microvolt = <1500000>;
+		regulator-ramp-delay = <1000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&vcc3v3_sys>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+
+	vdd_gpu: regulator@41 {
+		compatible = "silergy,syr828";
+		reg = <0x41>;
+		fcs,suspend-voltage-selector = <1>;
+		regulator-name = "vdd_gpu";
+		pinctrl-names = "default";
+		pinctrl-0 = <&vsel2_pin>;
+		regulator-min-microvolt = <712500>;
+		regulator-max-microvolt = <1500000>;
+		regulator-ramp-delay = <1000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&vcc3v3_sys>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+};
+
+&i2c1 {
+	i2c-scl-rising-time-ns = <300>;
+	i2c-scl-falling-time-ns = <15>;
+	status = "okay";
+
+	rt5651: audio-codec@1a {
+		compatible = "rockchip,rt5651";
+		reg = <0x1a>;
+		clocks = <&cru SCLK_I2S_8CH_OUT>;
+		clock-names = "mclk";
+		hp-det-gpio = <&gpio4 RK_PD4 GPIO_ACTIVE_LOW>;
+		spk-con-gpio = <&gpio0 RK_PB3 GPIO_ACTIVE_HIGH>;
+		#sound-dai-cells = <0>;
+	};
+
+};
+
+&i2c3 {
+	i2c-scl-rising-time-ns = <450>;
+	i2c-scl-falling-time-ns = <15>;
+	status = "okay";
+};
+
+&i2c4 {
+	i2c-scl-rising-time-ns = <600>;
+	i2c-scl-falling-time-ns = <20>;
+	status = "okay";
+
+	fusb0: typec-portc@22 {
+		compatible = "fcs,fusb302";
+		reg = <0x22>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <RK_PA2 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&fusb0_int>;
+		vbus-supply = <&vcc5v0_typec>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				usbc0_role_sw: endpoint@0 {
+					remote-endpoint = <&dwc3_0_role_switch>;
+				};
+			};
+		};
+
+		connector {
+			compatible = "usb-c-connector";
+			data-role = "dual";
+			label = "USB-C";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usbc_hs: endpoint {
+						remote-endpoint = <&u2phy0_typec_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usbc_ss: endpoint {
+						remote-endpoint = <&tcphy0_typec_ss>;
+					};
+				};
+			};
+		};
+	};
+};
+
+&i2s1 {
+	rockchip,playback-channels = <2>;
+	rockchip,capture-channels = <2>;
+	status = "okay";
+};
+
+&i2s2 {
+	status = "okay";
+};
+
+&io_domains {
+	status = "okay";
+
+	audio-supply = <&vcca1v8_codec>;
+	bt656-supply = <&vcc_3v0>;
+	gpio1830-supply = <&vcc_3v0>;
+	sdmmc-supply = <&vcc_sdio>;
+};
+
+&pmu_io_domains {
+	status = "okay";
+
+	pmu1830-supply = <&vcc_3v0>;
+};
+
+&pinctrl {
+	buttons {
+		pwrbtn: pwrbtn {
+			rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	bt {
+		bt_enable_h: bt-enable-h {
+			rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		bt_host_wake_l: bt-host-wake-l {
+			rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		bt_wake_l: bt-wake-l {
+			rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	fusb302x {
+		fusb0_int: fusb0-int {
+			rockchip,pins = <1 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	lcd-panel {
+		lcd_panel_reset: lcd-panel-reset {
+			rockchip,pins = <4 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	leds {
+		work_led_pin: work-led-pin {
+			rockchip,pins = <0 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		user_led_pin: user-led-pin {
+			rockchip,pins = <0 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		heartbeat_led_pin: heartbeat-led-pin {
+			rockchip,pins = <0 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		wlan_active_led_pin: wlan-led-pin {
+			rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		bt_active_led_pin: bt-led-pin {
+			rockchip,pins = <2 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+
+	};
+
+	pmic {
+		pmic_int_l: pmic-int-l {
+			rockchip,pins = <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		vsel1_pin: vsel1-pin {
+			rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		vsel2_pin: vsel2-pin {
+			rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+
+	rt5651 {
+		rt5651_hpcon: rt5640-hpcon {
+			rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	sdio-pwrseq {
+		wifi_enable_h: wifi-enable-h {
+			rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	usb-typec {
+		vcc5v0_typec_en: vcc5v0_typec_en {
+			rockchip,pins = <4 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	usb2 {
+		vcc5v0_host3_en: vcc5v0-host3-en {
+			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		vcc5v0_host1_en: vcc5v0-host1-en {
+			rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	wifi {
+		wifi_host_wake_l: wifi-host-wake-l {
+			rockchip,pins = <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&pwm0 {
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&vcca1v8_s3>;
+	status = "okay";
+};
+
+&sdio0 {
+	/* WiFi & BT combo module AMPAK AP6255 */
+	#address-cells = <1>;
+	#size-cells = <0>;
+	bus-width = <4>;
+	clock-frequency = <50000000>;
+	cap-sdio-irq;
+	cap-sd-highspeed;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	non-removable;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>;
+	sd-uhs-sdr104;
+	status = "okay";
+
+	brcmf: wifi@1 {
+		compatible = "brcm,bcm4329-fmac";
+		reg = <1>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PA3 GPIO_ACTIVE_HIGH>;
+		interrupt-names = "host-wake";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_host_wake_l>;
+	};
+};
+
+&sdmmc {
+	bus-width = <4>;
+	cap-mmc-highspeed;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	max-frequency = <150000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_bus4>;
+	status = "okay";
+};
+
+&sdhci {
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+&tcphy0 {
+	status = "okay";
+};
+
+&tcphy0_usb3 {
+	orientation-switch;
+	port {
+		tcphy0_typec_ss: endpoint {
+			remote-endpoint = <&usbc_ss>;
+		};
+	};
+};
+
+&tcphy1 {
+	status = "okay";
+};
+
+&tsadc {
+	/* tshut mode 0:CRU 1:GPIO */
+	rockchip,hw-tshut-mode = <1>;
+	/* tshut polarity 0:LOW 1:HIGH */
+	rockchip,hw-tshut-polarity = <1>;
+	status = "okay";
+};
+
+&u2phy0 {
+	status = "okay";
+
+	u2phy0_otg: otg-port {
+		status = "okay";
+	};
+
+	u2phy0_host: host-port {
+		phy-supply = <&vcc5v0_host3>;
+		status = "okay";
+	};
+
+	port {
+		u2phy0_typec_hs: endpoint {
+			remote-endpoint = <&usbc_hs>;
+		};
+	};
+};
+
+&u2phy1 {
+	status = "okay";
+
+	u2phy1_otg: otg-port {
+		status = "okay";
+	};
+
+	u2phy1_host: host-port {
+		phy-supply = <&vcc5v0_host3>;
+		status = "okay";
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_xfer &uart0_cts &uart0_rts>;
+	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm4345c5";
+		clocks = <&rk808 1>;
+		clock-names = "lpo";
+		device-wakeup-gpios = <&gpio2 RK_PD2 GPIO_ACTIVE_HIGH>;
+		host-wakeup-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>;
+		shutdown-gpios = <&gpio0 RK_PB1 GPIO_ACTIVE_HIGH>;
+		max-speed = <1500000>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_host_wake_l &bt_wake_l &bt_enable_h>;
+		vbat-supply = <&vcc3v3_sys>;
+		vddio-supply = <&vcc_1v8>;
+	};
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&usb_host0_ehci {
+	status = "okay";
+};
+
+&usb_host0_ohci {
+	status = "okay";
+};
+
+&usb_host1_ehci {
+	status = "okay";
+};
+
+&usb_host1_ohci {
+	status = "okay";
+};
+
+&usbdrd3_0 {
+	status = "okay";
+};
+
+&usbdrd_dwc3_0 {
+	status = "okay";
+	usb-role-switch;
+
+	port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		dwc3_0_role_switch: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&usbc0_role_sw>;
+		};
+	};
+};
+
+&usbdrd3_1 {
+	status = "okay";
+};
+
+&usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+};
+
+&vopb {
+	status = "okay";
+};
+
+&vopb_mmu {
+	status = "okay";
+};
+
+&vopl {
+	status = "okay";
+};
+
+&vopl_mmu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet-inx.dts b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet-inx.dts
index 2d721a974790..5d1879033e7c 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet-inx.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet-inx.dts
@@ -11,17 +11,29 @@
 
 / {
 	model = "Google Scarlet";
-	compatible = "google,scarlet-rev15-sku6", "google,scarlet-rev15",
+	compatible = "google,scarlet-rev15-sku2", "google,scarlet-rev15-sku4",
+		     "google,scarlet-rev15-sku6", "google,scarlet-rev15",
+		     "google,scarlet-rev14-sku2", "google,scarlet-rev14-sku4",
 		     "google,scarlet-rev14-sku6", "google,scarlet-rev14",
+		     "google,scarlet-rev13-sku2", "google,scarlet-rev13-sku4",
 		     "google,scarlet-rev13-sku6", "google,scarlet-rev13",
+		     "google,scarlet-rev12-sku2", "google,scarlet-rev12-sku4",
 		     "google,scarlet-rev12-sku6", "google,scarlet-rev12",
+		     "google,scarlet-rev11-sku2", "google,scarlet-rev11-sku4",
 		     "google,scarlet-rev11-sku6", "google,scarlet-rev11",
+		     "google,scarlet-rev10-sku2", "google,scarlet-rev10-sku4",
 		     "google,scarlet-rev10-sku6", "google,scarlet-rev10",
+		     "google,scarlet-rev9-sku2",  "google,scarlet-rev9-sku4",
 		     "google,scarlet-rev9-sku6",  "google,scarlet-rev9",
+		     "google,scarlet-rev8-sku2",  "google,scarlet-rev8-sku4",
 		     "google,scarlet-rev8-sku6",  "google,scarlet-rev8",
+		     "google,scarlet-rev7-sku2",  "google,scarlet-rev7-sku4",
 		     "google,scarlet-rev7-sku6",  "google,scarlet-rev7",
+		     "google,scarlet-rev6-sku2",  "google,scarlet-rev6-sku4",
 		     "google,scarlet-rev6-sku6",  "google,scarlet-rev6",
+		     "google,scarlet-rev5-sku2",  "google,scarlet-rev5-sku4",
 		     "google,scarlet-rev5-sku6",  "google,scarlet-rev5",
+		     "google,scarlet-rev4-sku2",  "google,scarlet-rev4-sku4",
 		     "google,scarlet-rev4-sku6",  "google,scarlet-rev4",
 		     "google,scarlet", "google,gru", "rockchip,rk3399";
 };
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi
index 40d4053fba80..ed3348b558f8 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-gru-scarlet.dtsi
@@ -768,6 +768,16 @@ camera: &i2c7 {
 		<4 RK_PA0 1 &pcfg_pull_none_6ma>;
 };
 
+&i2s0_8ch_bus_bclk_off {
+	rockchip,pins =
+		<3 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none_6ma>,
+		<3 RK_PD1 1 &pcfg_pull_none_6ma>,
+		<3 RK_PD2 1 &pcfg_pull_none_6ma>,
+		<3 RK_PD3 1 &pcfg_pull_none_6ma>,
+		<3 RK_PD7 1 &pcfg_pull_none_6ma>,
+		<4 RK_PA0 1 &pcfg_pull_none_6ma>;
+};
+
 /* there is no external pull up, so need to set this pin pull up */
 &sdmmc_cd_pin {
 	rockchip,pins = <1 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-nanopi-r4s-enterprise.dts b/arch/arm64/boot/dts/rockchip/rk3399-nanopi-r4s-enterprise.dts
new file mode 100644
index 000000000000..a23d11ca0eb6
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3399-nanopi-r4s-enterprise.dts
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+#include "rk3399-nanopi-r4s.dts"
+
+/ {
+	model = "FriendlyElec NanoPi R4S Enterprise Edition";
+	compatible = "friendlyarm,nanopi-r4s-enterprise", "rockchip,rk3399";
+};
+
+&gmac {
+	nvmem-cells = <&mac_address>;
+	nvmem-cell-names = "mac-address";
+};
+
+&i2c2 {
+	eeprom@51 {
+		compatible = "microchip,24c02", "atmel,24c02";
+		reg = <0x51>;
+		pagesize = <16>;
+		size = <256>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+
+		mac_address: mac-address@fa {
+			reg = <0xfa 0x06>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
new file mode 100644
index 000000000000..2e058c315025
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3399-pinephone-pro.dts
@@ -0,0 +1,398 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2020 Martijn Braam <martijn@brixit.nl>
+ * Copyright (c) 2021 Kamil Trzciński <ayufan@ayufan.eu>
+ */
+
+/*
+ * PinePhone Pro datasheet:
+ * https://files.pine64.org/doc/PinePhonePro/PinephonePro-Schematic-V1.0-20211127.pdf
+ */
+
+/dts-v1/;
+#include <dt-bindings/input/linux-event-codes.h>
+#include "rk3399.dtsi"
+#include "rk3399-opp.dtsi"
+
+/ {
+	model = "Pine64 PinePhonePro";
+	compatible = "pine64,pinephone-pro", "rockchip,rk3399";
+	chassis-type = "handset";
+
+	aliases {
+		mmc0 = &sdio0;
+		mmc1 = &sdmmc;
+		mmc2 = &sdhci;
+	};
+
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pwrbtn_pin>;
+
+		key-power {
+			debounce-interval = <20>;
+			gpios = <&gpio0 RK_PA5 GPIO_ACTIVE_LOW>;
+			label = "Power";
+			linux,code = <KEY_POWER>;
+			wakeup-source;
+		};
+	};
+
+	vcc_sys: vcc-sys-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_sys";
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc3v3_sys: vcc3v3-sys-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_sys>;
+	};
+
+	vcca1v8_s3: vcc1v8-s3-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vcca1v8_s3";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc3v3_sys>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc1v8_codec: vcc1v8-codec-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio3 RK_PA4 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc1v8_codec_en>;
+		regulator-name = "vcc1v8_codec";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc3v3_sys>;
+	};
+};
+
+&cpu_l0 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_l1 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_l2 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_l3 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_b0 {
+	cpu-supply = <&vdd_cpu_b>;
+};
+
+&cpu_b1 {
+	cpu-supply = <&vdd_cpu_b>;
+};
+
+&emmc_phy {
+	status = "okay";
+};
+
+&i2c0 {
+	clock-frequency = <400000>;
+	i2c-scl-rising-time-ns = <168>;
+	i2c-scl-falling-time-ns = <4>;
+	status = "okay";
+
+	rk818: pmic@1c {
+		compatible = "rockchip,rk818";
+		reg = <0x1c>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <RK_PC5 IRQ_TYPE_LEVEL_LOW>;
+		#clock-cells = <1>;
+		clock-output-names = "xin32k", "rk808-clkout2";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_int_l>;
+		rockchip,system-power-controller;
+		wakeup-source;
+
+		vcc1-supply = <&vcc_sys>;
+		vcc2-supply = <&vcc_sys>;
+		vcc3-supply = <&vcc_sys>;
+		vcc4-supply = <&vcc_sys>;
+		vcc6-supply = <&vcc_sys>;
+		vcc7-supply = <&vcc3v3_sys>;
+		vcc8-supply = <&vcc_sys>;
+		vcc9-supply = <&vcc3v3_sys>;
+
+		regulators {
+			vdd_cpu_l: DCDC_REG1 {
+				regulator-name = "vdd_cpu_l";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <875000>;
+				regulator-max-microvolt = <975000>;
+				regulator-ramp-delay = <6001>;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_center: DCDC_REG2 {
+				regulator-name = "vdd_center";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <1000000>;
+				regulator-ramp-delay = <6001>;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_ddr: DCDC_REG3 {
+				regulator-name = "vcc_ddr";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc_1v8: DCDC_REG4 {
+				regulator-name = "vcc_1v8";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcca3v0_codec: LDO_REG1 {
+				regulator-name = "vcca3v0_codec";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+			};
+
+			vcc3v0_touch: LDO_REG2 {
+				regulator-name = "vcc3v0_touch";
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+			};
+
+			vcca1v8_codec: LDO_REG3 {
+				regulator-name = "vcca1v8_codec";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			rk818_pwr_on: LDO_REG4 {
+				regulator-name = "rk818_pwr_on";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc_3v0: LDO_REG5 {
+				regulator-name = "vcc_3v0";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc_1v5: LDO_REG6 {
+				regulator-name = "vcc_1v5";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1500000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc1v8_dvp: LDO_REG7 {
+				regulator-name = "vcc1v8_dvp";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+
+			vcc3v3_s3: LDO_REG8 {
+				regulator-name = "vcc3v3_s3";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vccio_sd: LDO_REG9 {
+				regulator-name = "vccio_sd";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+			};
+
+			vcc3v3_s0: SWITCH_REG {
+				regulator-name = "vcc3v3_s0";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+		};
+	};
+
+	vdd_cpu_b: regulator@40 {
+		compatible = "silergy,syr827";
+		reg = <0x40>;
+		fcs,suspend-voltage-selector = <1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vsel1_pin>;
+		regulator-name = "vdd_cpu_b";
+		regulator-min-microvolt = <875000>;
+		regulator-max-microvolt = <1150000>;
+		regulator-ramp-delay = <1000>;
+		regulator-always-on;
+		regulator-boot-on;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+
+	vdd_gpu: regulator@41 {
+		compatible = "silergy,syr828";
+		reg = <0x41>;
+		fcs,suspend-voltage-selector = <1>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vsel2_pin>;
+		regulator-name = "vdd_gpu";
+		regulator-min-microvolt = <875000>;
+		regulator-max-microvolt = <975000>;
+		regulator-ramp-delay = <1000>;
+		regulator-always-on;
+		regulator-boot-on;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+};
+
+&cluster0_opp {
+	opp04 {
+		status = "disabled";
+	};
+
+	opp05 {
+		status = "disabled";
+	};
+};
+
+&cluster1_opp {
+	opp06 {
+		opp-hz = /bits/ 64 <1500000000>;
+		opp-microvolt = <1100000 1100000 1150000>;
+	};
+
+	opp07 {
+		status = "disabled";
+	};
+};
+
+&io_domains {
+	bt656-supply = <&vcc1v8_dvp>;
+	audio-supply = <&vcca1v8_codec>;
+	sdmmc-supply = <&vccio_sd>;
+	gpio1830-supply = <&vcc_3v0>;
+	status = "okay";
+};
+
+&pmu_io_domains {
+	pmu1830-supply = <&vcc_1v8>;
+	status = "okay";
+};
+
+&pinctrl {
+	buttons {
+		pwrbtn_pin: pwrbtn-pin {
+			rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	pmic {
+		pmic_int_l: pmic-int-l {
+			rockchip,pins = <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		vsel1_pin: vsel1-pin {
+			rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		vsel2_pin: vsel2-pin {
+			rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+
+	sound {
+		vcc1v8_codec_en: vcc1v8-codec-en {
+			rockchip,pins = <3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+};
+
+&sdmmc {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	max-frequency = <150000000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_cd &sdmmc_bus4>;
+	vmmc-supply = <&vcc3v3_sys>;
+	vqmmc-supply = <&vccio_sd>;
+	status = "okay";
+};
+
+&sdhci {
+	bus-width = <8>;
+	mmc-hs200-1_8v;
+	non-removable;
+	status = "okay";
+};
+
+&tsadc {
+	rockchip,hw-tshut-mode = <1>;
+	rockchip,hw-tshut-polarity = <1>;
+	status = "okay";
+};
+
+&uart2 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi
index acb174d3a8c5..2f4b1b2e3ac7 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-roc-pc.dtsi
@@ -271,6 +271,8 @@
 };
 
 &hdmi {
+	avdd-0v9-supply = <&vcca0v9_hdmi>;
+	avdd-1v8-supply = <&vcca1v8_hdmi>;
 	ddc-i2c-bus = <&i2c3>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&hdmi_cec>;
@@ -369,8 +371,8 @@
 				};
 			};
 
-			vcc1v8_hdmi: LDO_REG2 {
-				regulator-name = "vcc1v8_hdmi";
+			vcca1v8_hdmi: LDO_REG2 {
+				regulator-name = "vcca1v8_hdmi";
 				regulator-always-on;
 				regulator-boot-on;
 				regulator-min-microvolt = <1800000>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock-4c-plus.dts b/arch/arm64/boot/dts/rockchip/rk3399-rock-4c-plus.dts
new file mode 100644
index 000000000000..f9884902f874
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3399-rock-4c-plus.dts
@@ -0,0 +1,703 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2019 Fuzhou Rockchip Electronics Co., Ltd
+ * Copyright (c) 2019 Radxa Limited
+ * Copyright (c) 2022 Amarula Solutions(India)
+ */
+
+/dts-v1/;
+#include <dt-bindings/leds/common.h>
+#include "rk3399.dtsi"
+#include "rk3399-t-opp.dtsi"
+
+/ {
+	model = "Radxa ROCK 4C+";
+	compatible = "radxa,rock-4c-plus", "rockchip,rk3399";
+
+	aliases {
+		mmc0 = &sdmmc;
+		mmc1 = &sdhci;
+	};
+
+	chosen {
+		stdout-path = "serial2:1500000n8";
+	};
+
+	clkin_gmac: external-gmac-clock {
+		compatible = "fixed-clock";
+		clock-frequency = <125000000>;
+		clock-output-names = "clkin_gmac";
+		#clock-cells = <0>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&user_led1 &user_led2>;
+
+		/* USER_LED1 */
+		led-0 {
+			function = LED_FUNCTION_POWER;
+			color = <LED_COLOR_ID_GREEN>;
+			gpios = <&gpio3 RK_PD4 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-on";
+		};
+
+		/* USER_LED2 */
+		led-1 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_BLUE>;
+			gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clocks = <&rk809 1>;
+		clock-names = "ext_clock";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_enable_h>;
+		reset-gpios = <&gpio0 RK_PB2 GPIO_ACTIVE_LOW>;
+	};
+
+	vcc_3v3: vcc-3v3-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_3v3";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc3v3_sys>;
+	};
+
+	vcc3v3_phy1: vcc3v3-phy1-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_phy1";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_3v3>;
+	};
+
+	vcc5v0_host1: vcc5v0-host-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio3 RK_PD6 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc5v0_host_en>;
+		regulator-name = "vcc5v0_host1";
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&vcc5v0_host0_s0>;
+	};
+
+	vcc5v0_sys: vcc5v0-sys-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc5v0_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	vcc5v0_typec: vcc5v0-typec-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc5v0_typec0_en>;
+		regulator-name = "vcc5v0_typec";
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	vdd_log: vdd-log-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_log";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+};
+
+&cpu_l0 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_l1 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_l2 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_l3 {
+	cpu-supply = <&vdd_cpu_l>;
+};
+
+&cpu_b0 {
+	cpu-supply = <&vdd_cpu_b>;
+};
+
+&cpu_b1 {
+	cpu-supply = <&vdd_cpu_b>;
+};
+
+&emmc_phy {
+	status = "okay";
+};
+
+&gmac {
+	assigned-clocks = <&cru SCLK_RMII_SRC>;
+	assigned-clock-parents = <&clkin_gmac>;
+	clock_in_out = "input";
+	phy-supply = <&vcc3v3_phy1>;
+	phy-mode = "rgmii";
+	pinctrl-names = "default";
+	pinctrl-0 = <&rgmii_pins>;
+	snps,reset-gpio = <&gpio3 RK_PB7 GPIO_ACTIVE_LOW>;
+	snps,reset-active-low;
+	snps,reset-delays-us = <0 10000 50000>;
+	tx_delay = <0x2a>;
+	rx_delay = <0x21>;
+	status = "okay";
+};
+
+&gpu {
+	mali-supply = <&vdd_gpu>;
+	status = "okay";
+};
+
+&hdmi {
+	avdd-0v9-supply = <&vcc_0v9_s0>;
+	avdd-1v8-supply = <&vcc_1v8_s0>;
+	ddc-i2c-bus = <&i2c3>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&hdmi_cec>;
+	status = "okay";
+};
+
+&hdmi_sound {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+	i2c-scl-falling-time-ns = <30>;
+	i2c-scl-rising-time-ns = <180>;
+	clock-frequency = <400000>;
+
+	rk809: pmic@20 {
+		compatible = "rockchip,rk809";
+		reg = <0x20>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <RK_PC5 IRQ_TYPE_LEVEL_LOW>;
+		#clock-cells = <1>;
+		clock-output-names = "rk808-clkout1", "rk808-clkout2";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_int_l>;
+		rockchip,system-power-controller;
+		wakeup-source;
+
+		vcc1-supply = <&vcc5v0_sys>;
+		vcc2-supply = <&vcc5v0_sys>;
+		vcc3-supply = <&vcc5v0_sys>;
+		vcc4-supply = <&vcc5v0_sys>;
+		vcc5-supply = <&vcc_buck5_s3>;
+		vcc6-supply = <&vcc_buck5_s3>;
+		vcc7-supply = <&vcc5v0_sys>;
+		vcc8-supply = <&vcc3v3_sys>;
+		vcc9-supply = <&vcc5v0_sys>;
+
+		regulators {
+			vdd_center: DCDC_REG1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vdd_center";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <900000>;
+				};
+			};
+
+			vdd_cpu_l: DCDC_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-ramp-delay = <6001>;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vdd_cpu_l";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_ddr: DCDC_REG3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vcc_ddr";
+				regulator-initial-mode = <0x2>;
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc3v3_sys: DCDC_REG4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vcc3v3_sys";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vcc_buck5_s3: DCDC_REG5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_buck5_s3";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vcc_0v9_s3: LDO_REG1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-name = "vcc_0v9_s3";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_1v8_s3: LDO_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc_1v8_s3";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vcc_0v9_s0: LDO_REG3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-name = "vcc_0v9_s0";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <900000>;
+				};
+			};
+
+			vcc_1v8_s0: LDO_REG4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc_1v8_s0";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_mipi: LDO_REG5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-name = "vcc_mipi";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_1v5_s0: LDO_REG6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1500000>;
+				regulator-max-microvolt = <1500000>;
+				regulator-name = "vcc_1v5_s0";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_3v0_s0: LDO_REG7 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3000000>;
+				regulator-max-microvolt = <3000000>;
+				regulator-name = "vcc_3v0_s0";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_sdio_s0: LDO_REG8 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_sdio_s0";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_cam: LDO_REG9 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_cam";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc5v0_host0_s0: SWITCH_REG1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vcc5v0_host0_s0";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			lcd_3v3: SWITCH_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "lcd_3v3";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+		};
+	};
+
+	vdd_cpu_b: regulator@40 {
+		compatible = "silergy,syr827";
+		reg = <0x40>;
+		fcs,suspend-voltage-selector = <1>;
+		regulator-compatible = "fan53555-reg";
+		pinctrl-0 = <&vsel1_gpio>;
+		vsel-gpios = <&gpio1 RK_PC1 GPIO_ACTIVE_HIGH>;
+		regulator-name = "vdd_cpu_b";
+		regulator-min-microvolt = <712500>;
+		regulator-max-microvolt = <1500000>;
+		regulator-ramp-delay = <1000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&vcc5v0_sys>;
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+
+	vdd_gpu: regulator@41 {
+		compatible = "silergy,syr828";
+		reg = <0x41>;
+		fcs,suspend-voltage-selector = <1>;
+		regulator-compatible = "fan53555-reg";
+		pinctrl-0 = <&vsel2_gpio>;
+		vsel-gpios = <&gpio1 RK_PB6 GPIO_ACTIVE_HIGH>;
+		regulator-name = "vdd_gpu";
+		regulator-min-microvolt = <712500>;
+		regulator-max-microvolt = <1500000>;
+		regulator-ramp-delay = <1000>;
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&vcc5v0_sys>;
+		regulator-initial-mode = <1>; /* 1:force PWM 2:auto */
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+};
+
+&i2c3 {
+	i2c-scl-rising-time-ns = <450>;
+	i2c-scl-falling-time-ns = <15>;
+	status = "okay";
+};
+
+&i2s2 {
+	status = "okay";
+};
+
+&io_domains {
+	audio-supply = <&vcc_1v8_s0>;
+	bt656-supply = <&vcc_3v0_s0>;
+	gpio1830-supply = <&vcc_3v0_s0>;
+	sdmmc-supply = <&vcc_sdio_s0>;
+	status = "okay";
+};
+
+&pinctrl {
+	bt {
+		bt_enable_h: bt-enable-h {
+			rockchip,pins = <0 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		bt_host_wake_l: bt-host-wake-l {
+			rockchip,pins = <0 RK_PA4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		bt_wake_l: bt-wake-l {
+			rockchip,pins = <2 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	leds {
+		user_led1: user-led1 {
+			rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		user_led2: user-led2 {
+			rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	pmic {
+		pmic_int_l: pmic-int-l {
+			rockchip,pins = <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		vsel1_gpio: vsel1-gpio {
+			rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		vsel2_gpio: vsel2-gpio {
+			rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+
+	sdmmc {
+		sdmmc_bus4: sdmmc-bus4 {
+			rockchip,pins = <4 8 1 &pcfg_pull_up_8ma>,
+					<4 9 1 &pcfg_pull_up_8ma>,
+					<4 10 1 &pcfg_pull_up_8ma>,
+					<4 11 1 &pcfg_pull_up_8ma>;
+		};
+
+		sdmmc_clk: sdmmc-clk {
+			rockchip,pins = <4 12 1 &pcfg_pull_none_18ma>;
+		};
+
+		sdmmc_cmd: sdmmc-cmd {
+			rockchip,pins = <4 13 1 &pcfg_pull_up_8ma>;
+		};
+	};
+
+	usb-typec {
+		vcc5v0_typec0_en: vcc5v0-typec-en {
+			rockchip,pins = <1 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	usb2 {
+		vcc5v0_host_en: vcc5v0-host-en {
+			rockchip,pins = <3 RK_PD6 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	wifi {
+		wifi_enable_h: wifi-enable-h {
+			rockchip,pins = <0 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		wifi_host_wake_l: wifi-host-wake-l {
+			rockchip,pins = <0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&pmu_io_domains {
+	pmu1830-supply = <&vcc_3v0_s0>;
+	status = "okay";
+};
+
+&saradc {
+	status = "okay";
+	vref-supply = <&vcc_1v8_s3>;
+};
+
+&sdhci {
+	max-frequency = <150000000>;
+	bus-width = <8>;
+	mmc-hs400-1_8v;
+	non-removable;
+	mmc-hs400-enhanced-strobe;
+	status = "okay";
+};
+
+&sdio0 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	bus-width = <4>;
+	clock-frequency = <50000000>;
+	cap-sdio-irq;
+	cap-sd-highspeed;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	non-removable;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdio0_bus4 &sdio0_cmd &sdio0_clk>;
+	sd-uhs-sdr104;
+	status = "okay";
+
+	brcmf: wifi@1 {
+		compatible = "brcm,bcm4329-fmac";
+		reg = <1>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PA3 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "host-wake";
+		pinctrl-names = "default";
+		pinctrl-0 = <&wifi_host_wake_l>;
+	};
+};
+
+&sdmmc {
+	bus-width = <4>;
+	cap-mmc-highspeed;
+	cap-sd-highspeed;
+	card-detect-delay = <800>;
+	disable-wp;
+	pinctrl-names = "default";
+	pinctrl-0 = <&sdmmc_clk &sdmmc_cmd &sdmmc_bus4>;
+	cd-gpios = <&gpio0 RK_PA7 GPIO_ACTIVE_LOW>;
+	vqmmc-supply = <&vcc_sdio_s0>;
+	status = "okay";
+};
+
+&tcphy0 {
+	status = "okay";
+};
+
+&tcphy1 {
+	status = "okay";
+};
+
+&u2phy0 {
+	status = "okay";
+
+	u2phy0_otg: otg-port {
+		status = "okay";
+	};
+
+	u2phy0_host: host-port {
+		phy-supply = <&vcc5v0_host1>;
+		status = "okay";
+	};
+};
+
+&u2phy1 {
+	status = "okay";
+
+	u2phy1_otg: otg-port {
+		status = "okay";
+	};
+
+	u2phy1_host: host-port {
+		phy-supply = <&vcc5v0_host1>;
+		status = "okay";
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_xfer &uart0_cts &uart0_rts>;
+	status = "okay";
+
+	bluetooth {
+		compatible = "brcm,bcm4345c5";
+		clocks = <&rk809 1>;
+		clock-names = "lpo";
+		device-wakeup-gpios = <&gpio2 RK_PD3 GPIO_ACTIVE_HIGH>;
+		host-wakeup-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_HIGH>;
+		shutdown-gpios = <&gpio0 RK_PB1 GPIO_ACTIVE_HIGH>;
+		max-speed = <1500000>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&bt_host_wake_l &bt_wake_l &bt_enable_h>;
+		vbat-supply = <&vcc3v3_sys>;
+		vddio-supply = <&vcc_1v8_s3>;
+	};
+};
+
+&uart2 {
+	status = "okay";
+};
+
+&usb_host0_ehci {
+	status = "okay";
+};
+
+&usb_host0_ohci {
+	status = "okay";
+};
+
+&usb_host1_ehci {
+	status = "okay";
+};
+
+&usb_host1_ohci {
+	status = "okay";
+};
+
+&usbdrd3_0 {
+	extcon = <&u2phy0>;
+	status = "okay";
+};
+
+&usbdrd_dwc3_0 {
+	status = "okay";
+	dr_mode = "host";
+};
+
+&usbdrd3_1 {
+	status = "okay";
+};
+
+&usbdrd_dwc3_1 {
+	status = "okay";
+	dr_mode = "host";
+};
+
+&vopb {
+	status = "okay";
+};
+
+&vopb_mmu {
+	status = "okay";
+};
+
+&vopl {
+	status = "okay";
+};
+
+&vopl_mmu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi
index 401e1ae9d944..645ced6617a6 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-rock-pi-4.dtsi
@@ -6,6 +6,7 @@
 
 /dts-v1/;
 #include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/leds/common.h>
 #include <dt-bindings/pwm/pwm.h>
 #include "rk3399.dtsi"
 #include "rk3399-opp.dtsi"
@@ -27,6 +28,20 @@
 		#clock-cells = <0>;
 	};
 
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&user_led2>;
+
+		/* USER_LED2 */
+		led-0 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_BLUE>;
+			gpios = <&gpio3 RK_PD5 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
 	sdio_pwrseq: sdio-pwrseq {
 		compatible = "mmc-pwrseq-simple";
 		clocks = <&rk808 1>;
@@ -59,32 +74,33 @@
 		};
 	};
 
-	vcc12v_dcin: dc-12v {
+	vbus_typec: vbus-typec-regulator {
 		compatible = "regulator-fixed";
-		regulator-name = "vcc12v_dcin";
+		enable-active-high;
+		gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc5v0_typec_en>;
+		regulator-name = "vbus_typec";
 		regulator-always-on;
-		regulator-boot-on;
-		regulator-min-microvolt = <12000000>;
-		regulator-max-microvolt = <12000000>;
+		vin-supply = <&vcc5v0_sys>;
 	};
 
-	vcc5v0_sys: vcc-sys {
+	vcc12v_dcin: dc-12v {
 		compatible = "regulator-fixed";
-		regulator-name = "vcc5v0_sys";
+		regulator-name = "vcc12v_dcin";
 		regulator-always-on;
 		regulator-boot-on;
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		vin-supply = <&vcc12v_dcin>;
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
 	};
 
-	vcc_0v9: vcc-0v9 {
+	vcc3v3_lan: vcc3v3-lan-regulator {
 		compatible = "regulator-fixed";
-		regulator-name = "vcc_0v9";
+		regulator-name = "vcc3v3_lan";
 		regulator-always-on;
 		regulator-boot-on;
-		regulator-min-microvolt = <900000>;
-		regulator-max-microvolt = <900000>;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
 		vin-supply = <&vcc3v3_sys>;
 	};
 
@@ -121,24 +137,24 @@
 		vin-supply = <&vcc5v0_sys>;
 	};
 
-	vcc5v0_typec: vcc5v0-typec-regulator {
+	vcc5v0_sys: vcc-sys {
 		compatible = "regulator-fixed";
-		enable-active-high;
-		gpio = <&gpio1 RK_PA3 GPIO_ACTIVE_HIGH>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&vcc5v0_typec_en>;
-		regulator-name = "vcc5v0_typec";
+		regulator-name = "vcc5v0_sys";
 		regulator-always-on;
-		vin-supply = <&vcc5v0_sys>;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc12v_dcin>;
 	};
 
-	vcc_lan: vcc3v3-phy-regulator {
+	vcc_0v9: vcc-0v9 {
 		compatible = "regulator-fixed";
-		regulator-name = "vcc_lan";
+		regulator-name = "vcc_0v9";
 		regulator-always-on;
 		regulator-boot-on;
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		vin-supply = <&vcc3v3_sys>;
 	};
 
 	vdd_log: vdd-log {
@@ -185,7 +201,7 @@
 	assigned-clocks = <&cru SCLK_RMII_SRC>;
 	assigned-clock-parents = <&clkin_gmac>;
 	clock_in_out = "input";
-	phy-supply = <&vcc_lan>;
+	phy-supply = <&vcc3v3_lan>;
 	phy-mode = "rgmii";
 	pinctrl-names = "default";
 	pinctrl-0 = <&rgmii_pins>;
@@ -203,6 +219,8 @@
 };
 
 &hdmi {
+	avdd-0v9-supply = <&vcca0v9_hdmi>;
+	avdd-1v8-supply = <&vcca1v8_hdmi>;
 	ddc-i2c-bus = <&i2c3>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&hdmi_cec>;
@@ -290,8 +308,8 @@
 				};
 			};
 
-			vcc1v8_codec: LDO_REG1 {
-				regulator-name = "vcc1v8_codec";
+			vcca1v8_codec: LDO_REG1 {
+				regulator-name = "vcca1v8_codec";
 				regulator-always-on;
 				regulator-boot-on;
 				regulator-min-microvolt = <1800000>;
@@ -301,8 +319,8 @@
 				};
 			};
 
-			vcc1v8_hdmi: LDO_REG2 {
-				regulator-name = "vcc1v8_hdmi";
+			vcca1v8_hdmi: LDO_REG2 {
+				regulator-name = "vcca1v8_hdmi";
 				regulator-always-on;
 				regulator-boot-on;
 				regulator-min-microvolt = <1800000>;
@@ -359,8 +377,8 @@
 				};
 			};
 
-			vcc0v9_hdmi: LDO_REG7 {
-				regulator-name = "vcc0v9_hdmi";
+			vcca0v9_hdmi: LDO_REG7 {
+				regulator-name = "vcca0v9_hdmi";
 				regulator-always-on;
 				regulator-boot-on;
 				regulator-min-microvolt = <900000>;
@@ -499,21 +517,10 @@
 };
 
 &io_domains {
-	status = "okay";
-
+	audio-supply = <&vcca1v8_codec>;
 	bt656-supply = <&vcc_3v0>;
-	audio-supply = <&vcc1v8_codec>;
-	sdmmc-supply = <&vcc_sdio>;
 	gpio1830-supply = <&vcc_3v0>;
-};
-
-&pmu_io_domains {
-	status = "okay";
-
-	pmu1830-supply = <&vcc_3v0>;
-};
-
-&pcie_phy {
+	sdmmc-supply = <&vcc_sdio>;
 	status = "okay";
 };
 
@@ -528,6 +535,10 @@
 	status = "okay";
 };
 
+&pcie_phy {
+	status = "okay";
+};
+
 &pinctrl {
 	bt {
 		bt_enable_h: bt-enable-h {
@@ -553,12 +564,32 @@
 		};
 	};
 
+	leds {
+		user_led2: user-led2 {
+			rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	pcie {
 		pcie_pwr_en: pcie-pwr-en {
 			rockchip,pins = <2 RK_PD2 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
 
+	pmic {
+		pmic_int_l: pmic-int-l {
+			rockchip,pins = <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		vsel1_pin: vsel1-pin {
+			rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+
+		vsel2_pin: vsel2-pin {
+			rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
+
 	sdio0 {
 		sdio0_bus4: sdio0-bus4 {
 			rockchip,pins = <2 RK_PC4 1 &pcfg_pull_up_20ma>,
@@ -576,20 +607,6 @@
 		};
 	};
 
-	pmic {
-		pmic_int_l: pmic-int-l {
-			rockchip,pins = <1 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>;
-		};
-
-		vsel1_pin: vsel1-pin {
-			rockchip,pins = <1 RK_PC1 RK_FUNC_GPIO &pcfg_pull_down>;
-		};
-
-		vsel2_pin: vsel2-pin {
-			rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_down>;
-		};
-	};
-
 	usb-typec {
 		vcc5v0_typec_en: vcc5v0-typec-en {
 			rockchip,pins = <1 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>;
@@ -613,6 +630,11 @@
 	};
 };
 
+&pmu_io_domains {
+	pmu1830-supply = <&vcc_3v0>;
+	status = "okay";
+};
+
 &pwm2 {
 	status = "okay";
 };
@@ -623,6 +645,14 @@
 	vref-supply = <&vcc_1v8>;
 };
 
+&sdhci {
+	bus-width = <8>;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	non-removable;
+	status = "okay";
+};
+
 &sdio0 {
 	#address-cells = <1>;
 	#size-cells = <0>;
@@ -650,14 +680,6 @@
 	status = "okay";
 };
 
-&sdhci {
-	bus-width = <8>;
-	mmc-hs400-1_8v;
-	mmc-hs400-enhanced-strobe;
-	non-removable;
-	status = "okay";
-};
-
 &spdif {
 
 	spdif_p0: port {
@@ -739,13 +761,13 @@
 	status = "okay";
 };
 
-&usbdrd_dwc3_0 {
+&usbdrd3_1 {
 	status = "okay";
-	dr_mode = "host";
 };
 
-&usbdrd3_1 {
+&usbdrd_dwc3_0 {
 	status = "okay";
+	dr_mode = "host";
 };
 
 &usbdrd_dwc3_1 {
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-t-opp.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-t-opp.dtsi
new file mode 100644
index 000000000000..1ababadda9df
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3399-t-opp.dtsi
@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright (c) 2016-2017 Fuzhou Rockchip Electronics Co., Ltd
+ * Copyright (c) 2022 Radxa Limited
+ */
+
+/ {
+	cluster0_opp: opp-table-0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp00 {
+			opp-hz = /bits/ 64 <408000000>;
+			opp-microvolt = <875000 875000 1250000>;
+			clock-latency-ns = <40000>;
+		};
+		opp01 {
+			opp-hz = /bits/ 64 <600000000>;
+			opp-microvolt = <875000 875000 1250000>;
+		};
+		opp02 {
+			opp-hz = /bits/ 64 <816000000>;
+			opp-microvolt = <900000 900000 1250000>;
+		};
+		opp03 {
+			opp-hz = /bits/ 64 <1008000000>;
+			opp-microvolt = <975000 975000 1250000>;
+		};
+	};
+
+	cluster1_opp: opp-table-1 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp00 {
+			opp-hz = /bits/ 64 <408000000>;
+			opp-microvolt = <875000 875000 1250000>;
+			clock-latency-ns = <40000>;
+		};
+		opp01 {
+			opp-hz = /bits/ 64 <600000000>;
+			opp-microvolt = <875000 875000 1250000>;
+		};
+		opp02 {
+			opp-hz = /bits/ 64 <816000000>;
+			opp-microvolt = <875000 875000 1250000>;
+		};
+		opp03 {
+			opp-hz = /bits/ 64 <1008000000>;
+			opp-microvolt = <925000 925000 1250000>;
+		};
+		opp04 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <1000000 1000000 1250000>;
+		};
+		opp05 {
+			opp-hz = /bits/ 64 <1416000000>;
+			opp-microvolt = <1075000 1075000 1250000>;
+		};
+		opp06 {
+			opp-hz = /bits/ 64 <1512000000>;
+			opp-microvolt = <1150000 1150000 1250000>;
+		};
+	};
+
+	gpu_opp_table: opp-table-2 {
+		compatible = "operating-points-v2";
+
+		opp00 {
+			opp-hz = /bits/ 64 <200000000>;
+			opp-microvolt = <875000 875000 1150000>;
+		};
+		opp01 {
+			opp-hz = /bits/ 64 <300000000>;
+			opp-microvolt = <875000 875000 1150000>;
+		};
+		opp02 {
+			opp-hz = /bits/ 64 <400000000>;
+			opp-microvolt = <875000 875000 1150000>;
+		};
+		opp03 {
+			opp-hz = /bits/ 64 <600000000>;
+			opp-microvolt = <975000 975000 1150000>;
+		};
+	};
+};
+
+&cpu_l0 {
+	operating-points-v2 = <&cluster0_opp>;
+};
+
+&cpu_l1 {
+	operating-points-v2 = <&cluster0_opp>;
+};
+
+&cpu_l2 {
+	operating-points-v2 = <&cluster0_opp>;
+};
+
+&cpu_l3 {
+	operating-points-v2 = <&cluster0_opp>;
+};
+
+&cpu_b0 {
+	operating-points-v2 = <&cluster1_opp>;
+};
+
+&cpu_b1 {
+	operating-points-v2 = <&cluster1_opp>;
+};
+
+&gpu {
+	operating-points-v2 = <&gpu_opp_table>;
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
index 9d5b0e8c9cca..92c2207e686c 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
@@ -1664,8 +1664,9 @@
 		dma-names = "tx", "rx";
 		clock-names = "i2s_clk", "i2s_hclk";
 		clocks = <&cru SCLK_I2S0_8CH>, <&cru HCLK_I2S0_8CH>;
-		pinctrl-names = "default";
+		pinctrl-names = "bclk_on", "bclk_off";
 		pinctrl-0 = <&i2s0_8ch_bus>;
+		pinctrl-1 = <&i2s0_8ch_bus_bclk_off>;
 		power-domains = <&power RK3399_PD_SDIOAUDIO>;
 		#sound-dai-cells = <0>;
 		status = "disabled";
@@ -1701,7 +1702,7 @@
 
 	vopl: vop@ff8f0000 {
 		compatible = "rockchip,rk3399-vop-lit";
-		reg = <0x0 0xff8f0000 0x0 0x3efc>;
+		reg = <0x0 0xff8f0000 0x0 0x2000>, <0x0 0xff8f2000 0x0 0x400>;
 		interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH 0>;
 		assigned-clocks = <&cru ACLK_VOP1>, <&cru HCLK_VOP1>;
 		assigned-clock-rates = <400000000>, <100000000>;
@@ -1757,7 +1758,7 @@
 
 	vopb: vop@ff900000 {
 		compatible = "rockchip,rk3399-vop-big";
-		reg = <0x0 0xff900000 0x0 0x3efc>;
+		reg = <0x0 0xff900000 0x0 0x2000>, <0x0 0xff902000 0x0 0x1000>;
 		interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH 0>;
 		assigned-clocks = <&cru ACLK_VOP0>, <&cru HCLK_VOP0>;
 		assigned-clock-rates = <400000000>, <100000000>;
@@ -2409,6 +2410,19 @@
 					<3 RK_PD7 1 &pcfg_pull_none>,
 					<4 RK_PA0 1 &pcfg_pull_none>;
 			};
+
+			i2s0_8ch_bus_bclk_off: i2s0-8ch-bus-bclk-off {
+				rockchip,pins =
+					<3 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>,
+					<3 RK_PD1 1 &pcfg_pull_none>,
+					<3 RK_PD2 1 &pcfg_pull_none>,
+					<3 RK_PD3 1 &pcfg_pull_none>,
+					<3 RK_PD4 1 &pcfg_pull_none>,
+					<3 RK_PD5 1 &pcfg_pull_none>,
+					<3 RK_PD6 1 &pcfg_pull_none>,
+					<3 RK_PD7 1 &pcfg_pull_none>,
+					<4 RK_PA0 1 &pcfg_pull_none>;
+			};
 		};
 
 		i2s1 {
@@ -2420,6 +2434,15 @@
 					<4 RK_PA6 1 &pcfg_pull_none>,
 					<4 RK_PA7 1 &pcfg_pull_none>;
 			};
+
+			i2s1_2ch_bus_bclk_off: i2s1-2ch-bus-bclk-off {
+				rockchip,pins =
+					<4 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>,
+					<4 RK_PA4 1 &pcfg_pull_none>,
+					<4 RK_PA5 1 &pcfg_pull_none>,
+					<4 RK_PA6 1 &pcfg_pull_none>,
+					<4 RK_PA7 1 &pcfg_pull_none>;
+			};
 		};
 
 		sdio0 {
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg353p.dts b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg353p.dts
new file mode 100644
index 000000000000..7a20e2d6876a
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg353p.dts
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+#include "rk3566-anbernic-rgxx3.dtsi"
+
+/ {
+	model = "RG353P";
+	compatible = "anbernic,rg353p", "rockchip,rk3566";
+
+	aliases {
+		mmc0 = &sdhci;
+		mmc1 = &sdmmc0;
+		mmc2 = &sdmmc1;
+		mmc3 = &sdmmc2;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		power-supply = <&vcc_sys>;
+		pwms = <&pwm4 0 25000 0>;
+	};
+};
+
+&gpio_keys_control {
+	button-a {
+		gpios = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>;
+		label = "EAST";
+		linux,code = <BTN_EAST>;
+	};
+
+	button-left {
+		gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>;
+		label = "DPAD-LEFT";
+		linux,code = <BTN_DPAD_LEFT>;
+	};
+
+	button-r1 {
+		gpios = <&gpio3 RK_PB4 GPIO_ACTIVE_LOW>;
+		label = "TR";
+		linux,code = <BTN_TR>;
+	};
+
+	button-r2 {
+		gpios = <&gpio3 RK_PB3 GPIO_ACTIVE_LOW>;
+		label = "TR2";
+		linux,code = <BTN_TR2>;
+	};
+
+	button-right {
+		gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_LOW>;
+		label = "DPAD-RIGHT";
+		linux,code = <BTN_DPAD_RIGHT>;
+	};
+
+	button-y {
+		gpios = <&gpio3 RK_PC1 GPIO_ACTIVE_LOW>;
+		label = "WEST";
+		linux,code = <BTN_WEST>;
+	};
+};
+
+&i2c0 {
+	/* This hardware is physically present but unused. */
+	power-monitor@62 {
+		compatible = "cellwise,cw2015";
+		reg = <0x62>;
+		status = "disabled";
+	};
+};
+
+&i2c2 {
+	pintctrl-names = "default";
+	pinctrl-0 = <&i2c2m1_xfer>;
+	status = "okay";
+};
+
+&pwm4 {
+	status = "okay";
+};
+
+&sdhci {
+	pinctrl-0 = <&emmc_bus8>, <&emmc_clk>, <&emmc_cmd>, <&emmc_datastrobe>, <&emmc_rstnout>;
+	pinctrl-names = "default";
+	bus-width = <8>;
+	mmc-hs200-1_8v;
+	non-removable;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vcc_1v8>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg503.dts b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg503.dts
new file mode 100644
index 000000000000..3dc01549a5b4
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rg503.dts
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+#include "rk3566-anbernic-rgxx3.dtsi"
+
+/ {
+	model = "RG503";
+	compatible = "anbernic,rg503", "rockchip,rk3566";
+
+	aliases {
+		mmc0 = &sdmmc0;
+		mmc1 = &sdmmc1;
+		mmc2 = &sdmmc2;
+	};
+
+	gpio_spi: spi {
+		compatible = "spi-gpio";
+		pinctrl-names = "default";
+		pinctrl-0 = <&spi_pins>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		sck-gpios = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>;
+		mosi-gpios = <&gpio4 RK_PB0 GPIO_ACTIVE_HIGH>;
+		cs-gpios = <&gpio4 RK_PA7 GPIO_ACTIVE_HIGH>;
+		num-chipselects = <0>;
+	};
+};
+
+&gpio_keys_control {
+	button-a {
+		gpios = <&gpio3 RK_PC1 GPIO_ACTIVE_LOW>;
+		label = "EAST";
+		linux,code = <BTN_EAST>;
+	};
+
+	button-left {
+		gpios = <&gpio3 RK_PA5 GPIO_ACTIVE_LOW>;
+		label = "DPAD-LEFT";
+		linux,code = <BTN_DPAD_LEFT>;
+	};
+
+	button-right {
+		gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>;
+		label = "DPAD-RIGHT";
+		linux,code = <BTN_DPAD_RIGHT>;
+	};
+
+	button-r1 {
+		gpios = <&gpio3 RK_PB3 GPIO_ACTIVE_LOW>;
+		label = "TR";
+		linux,code = <BTN_TR>;
+	};
+
+	button-r2 {
+		gpios = <&gpio3 RK_PB4 GPIO_ACTIVE_LOW>;
+		label = "TR2";
+		linux,code = <BTN_TR2>;
+	};
+
+	button-right {
+		gpios = <&gpio3 RK_PA6 GPIO_ACTIVE_LOW>;
+		label = "DPAD-RIGHT";
+		linux,code = <BTN_DPAD_RIGHT>;
+	};
+
+	button-y {
+		gpios = <&gpio3 RK_PC2 GPIO_ACTIVE_LOW>;
+		label = "WEST";
+		linux,code = <BTN_WEST>;
+	};
+};
+
+&pinctrl {
+	gpio-spi {
+		spi_pins: spi-pins {
+			rockchip,pins =
+				<4 RK_PA7 RK_FUNC_GPIO &pcfg_pull_none>,
+				<4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>,
+				<4 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rgxx3.dtsi b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rgxx3.dtsi
new file mode 100644
index 000000000000..2b455143b86d
--- /dev/null
+++ b/arch/arm64/boot/dts/rockchip/rk3566-anbernic-rgxx3.dtsi
@@ -0,0 +1,831 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/rockchip.h>
+#include <dt-bindings/soc/rockchip,vop2.h>
+#include "rk3566.dtsi"
+
+/ {
+	chosen: chosen {
+		stdout-path = "serial2:1500000n8";
+	};
+
+	adc-joystick {
+		compatible = "adc-joystick";
+		io-channels = <&adc_mux 0>,
+			      <&adc_mux 1>,
+			      <&adc_mux 2>,
+			      <&adc_mux 3>;
+		pinctrl-0 = <&joy_mux_en>;
+		pinctrl-names = "default";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		axis@0 {
+			reg = <0>;
+			abs-flat = <32>;
+			abs-fuzz = <32>;
+			abs-range = <1023 15>;
+			linux,code = <ABS_X>;
+		};
+
+		axis@1 {
+			reg = <1>;
+			abs-flat = <32>;
+			abs-fuzz = <32>;
+			abs-range = <15 1023>;
+			linux,code = <ABS_RX>;
+		};
+
+		axis@2 {
+			reg = <2>;
+			abs-flat = <32>;
+			abs-fuzz = <32>;
+			abs-range = <15 1023>;
+			linux,code = <ABS_Y>;
+		};
+
+		axis@3 {
+			reg = <3>;
+			abs-flat = <32>;
+			abs-fuzz = <32>;
+			abs-range = <1023 15>;
+			linux,code = <ABS_RY>;
+		};
+	};
+
+	adc_keys: adc-keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 0>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1800000>;
+		poll-interval = <60>;
+
+		/*
+		 * Button is mapped to F key in BSP kernel, but
+		 * according to input guidelines it should be mode.
+		 */
+		button-mode {
+			label = "MODE";
+			linux,code = <BTN_MODE>;
+			press-threshold-microvolt = <1750>;
+		};
+	};
+
+	adc_mux: adc-mux {
+		compatible = "io-channel-mux";
+		channels = "left_x", "right_x", "left_y", "right_y";
+		#io-channel-cells = <1>;
+		io-channels = <&saradc 3>;
+		io-channel-names = "parent";
+		mux-controls = <&gpio_mux>;
+		settle-time-us = <100>;
+	};
+
+	gpio_keys_control: gpio-keys-control {
+		compatible = "gpio-keys";
+		pinctrl-0 = <&btn_pins_ctrl>;
+		pinctrl-names = "default";
+
+		button-b {
+			gpios = <&gpio3 RK_PC3 GPIO_ACTIVE_LOW>;
+			label = "SOUTH";
+			linux,code = <BTN_SOUTH>;
+		};
+
+		button-down {
+			gpios = <&gpio3 RK_PA4 GPIO_ACTIVE_LOW>;
+			label = "DPAD-DOWN";
+			linux,code = <BTN_DPAD_DOWN>;
+		};
+
+		button-l1 {
+			gpios = <&gpio3 RK_PB1 GPIO_ACTIVE_LOW>;
+			label = "TL";
+			linux,code = <BTN_TL>;
+		};
+
+		button-l2 {
+			gpios = <&gpio3 RK_PB2 GPIO_ACTIVE_LOW>;
+			label = "TL2";
+			linux,code = <BTN_TL2>;
+		};
+
+		button-select {
+			gpios = <&gpio3 RK_PB6 GPIO_ACTIVE_LOW>;
+			label = "SELECT";
+			linux,code = <BTN_SELECT>;
+		};
+
+		button-start {
+			gpios = <&gpio3 RK_PB5 GPIO_ACTIVE_LOW>;
+			label = "START";
+			linux,code = <BTN_START>;
+		};
+
+		button-thumbl {
+			gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_LOW>;
+			label = "THUMBL";
+			linux,code = <BTN_THUMBL>;
+		};
+
+		button-thumbr {
+			gpios = <&gpio3 RK_PA2 GPIO_ACTIVE_LOW>;
+			label = "THUMBR";
+			linux,code = <BTN_THUMBR>;
+		};
+
+		button-up {
+			gpios = <&gpio3 RK_PA3 GPIO_ACTIVE_LOW>;
+			label = "DPAD-UP";
+			linux,code = <BTN_DPAD_UP>;
+		};
+
+		button-x {
+			gpios = <&gpio3 RK_PC0 GPIO_ACTIVE_LOW>;
+			label = "NORTH";
+			linux,code = <BTN_NORTH>;
+		};
+	};
+
+	gpio_keys_vol: gpio-keys-vol {
+		compatible = "gpio-keys";
+		autorepeat;
+		pinctrl-0 = <&btn_pins_vol>;
+		pinctrl-names = "default";
+
+		button-vol-down {
+			gpios = <&gpio3 RK_PB0 GPIO_ACTIVE_LOW>;
+			label = "VOLUMEDOWN";
+			linux,code = <KEY_VOLUMEDOWN>;
+		};
+
+		button-vol-up {
+			gpios = <&gpio3 RK_PA7 GPIO_ACTIVE_LOW>;
+			label = "VOLUMEUP";
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	gpio_mux: mux-controller {
+		compatible = "gpio-mux";
+		mux-gpios = <&gpio0 RK_PB6 GPIO_ACTIVE_LOW>,
+			    <&gpio0 RK_PB7 GPIO_ACTIVE_LOW>;
+		#mux-control-cells = <0>;
+	};
+
+	hdmi-con {
+		compatible = "hdmi-connector";
+		ddc-i2c-bus = <&i2c5>;
+		type = "c";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
+	leds: gpio-leds {
+		compatible = "gpio-leds";
+		pinctrl-0 = <&led_pins>;
+		pinctrl-names = "default";
+
+		green_led: led-0 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "on";
+			function = LED_FUNCTION_POWER;
+			gpios = <&gpio0 RK_PC5 GPIO_ACTIVE_HIGH>;
+		};
+
+		amber_led: led-1 {
+			color = <LED_COLOR_ID_AMBER>;
+			function = LED_FUNCTION_CHARGING;
+			gpios = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>;
+			retain-state-suspended;
+		};
+
+		red_led: led-2 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			gpios = <&gpio0 RK_PC7 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	/* Channels reversed for both headphones and speakers. */
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "anbernic_rk817";
+		simple-audio-card,aux-devs = <&spk_amp>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,hp-det-gpio = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,widgets =
+			"Microphone", "Mic Jack",
+			"Headphone", "Headphones",
+			"Speaker", "Internal Speakers";
+		simple-audio-card,routing =
+			"MICL", "Mic Jack",
+			"Headphones", "HPOL",
+			"Headphones", "HPOR",
+			"Internal Speakers", "Speaker Amp OUTL",
+			"Internal Speakers", "Speaker Amp OUTR",
+			"Speaker Amp INL", "HPOL",
+			"Speaker Amp INR", "HPOR";
+		simple-audio-card,pin-switches = "Internal Speakers";
+
+		simple-audio-card,codec {
+			sound-dai = <&rk817>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s1_8ch>;
+		};
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		clocks = <&rk817 1>;
+		clock-names = "ext_clock";
+		pinctrl-0 = <&wifi_enable_h>;
+		pinctrl-names = "default";
+		post-power-on-delay-ms = <200>;
+		reset-gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_LOW>;
+	};
+
+	spk_amp: audio-amplifier {
+		compatible = "simple-audio-amplifier";
+		enable-gpios = <&gpio4 RK_PC2 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&spk_amp_enable_h>;
+		pinctrl-names = "default";
+		sound-name-prefix = "Speaker Amp";
+	};
+
+	vcc3v3_lcd0_n: regulator-vcc3v3-lcd0 {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 RK_PC2 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		pinctrl-0 = <&vcc_lcd_h>;
+		pinctrl-names = "default";
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-name = "vcc3v3_lcd0_n";
+		vin-supply = <&vcc_3v3>;
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+
+	vcc_sys: regulator-vcc-sys {
+		compatible = "regulator-fixed";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3800000>;
+		regulator-max-microvolt = <3800000>;
+		regulator-name = "vcc_sys";
+	};
+
+	vcc_wifi: regulator-vcc-wifi {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio0 RK_PA0 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&vcc_wifi_h>;
+		pinctrl-names = "default";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-name = "vcc_wifi";
+	};
+
+	vibrator: pwm-vibrator {
+		compatible = "pwm-vibrator";
+		pwm-names = "enable";
+		pwms = <&pwm5 0 1000000000 0>;
+	};
+};
+
+&combphy1 {
+	status = "okay";
+};
+
+&cpu0 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu1 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu2 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&cpu3 {
+	cpu-supply = <&vdd_cpu>;
+};
+
+&gpu {
+	mali-supply = <&vdd_gpu>;
+	status = "okay";
+};
+
+&hdmi {
+	ddc-i2c-bus = <&i2c5>;
+	pinctrl-0 = <&hdmitxm0_cec>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&hdmi_in {
+	hdmi_in_vp0: endpoint {
+		remote-endpoint = <&vp0_out_hdmi>;
+	};
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
+&hdmi_sound {
+	status = "okay";
+};
+
+&i2c0 {
+	status = "okay";
+
+	rk817: pmic@20 {
+		compatible = "rockchip,rk817";
+		reg = <0x20>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PA3 IRQ_TYPE_LEVEL_LOW>;
+		clock-output-names = "rk808-clkout1", "rk808-clkout2";
+		clock-names = "mclk";
+		clocks = <&cru I2S1_MCLKOUT_TX>;
+		assigned-clocks = <&cru I2S1_MCLKOUT_TX>;
+		assigned-clock-parents = <&cru CLK_I2S1_8CH_TX>;
+		#clock-cells = <1>;
+		#sound-dai-cells = <0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&i2s1m0_mclk>, <&pmic_int_l>;
+		wakeup-source;
+
+		vcc1-supply = <&vcc_sys>;
+		vcc2-supply = <&vcc_sys>;
+		vcc3-supply = <&vcc_sys>;
+		vcc4-supply = <&vcc_sys>;
+		vcc5-supply = <&vcc_sys>;
+		vcc6-supply = <&vcc_sys>;
+		vcc7-supply = <&vcc_sys>;
+		vcc8-supply = <&vcc_sys>;
+		vcc9-supply = <&dcdc_boost>;
+
+		regulators {
+			vdd_logic: DCDC_REG1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-init-microvolt = <900000>;
+				regulator-ramp-delay = <6001>;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vdd_logic";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <900000>;
+				};
+			};
+
+			vdd_gpu: DCDC_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1350000>;
+				regulator-init-microvolt = <900000>;
+				regulator-ramp-delay = <6001>;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vdd_gpu";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_ddr: DCDC_REG3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vcc_ddr";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vcc_3v3: DCDC_REG4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-initial-mode = <0x2>;
+				regulator-name = "vcc_3v3";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vcca1v8_pmu: LDO_REG1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca1v8_pmu";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdda_0v9: LDO_REG2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-name = "vdda_0v9";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda0v9_pmu: LDO_REG3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <900000>;
+				regulator-max-microvolt = <900000>;
+				regulator-name = "vdda0v9_pmu";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <900000>;
+				};
+			};
+
+			vccio_acodec: LDO_REG4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vccio_acodec";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vccio_sd: LDO_REG5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vccio_sd";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc3v3_pmu: LDO_REG6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc3v3_pmu";
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vcc_1v8: LDO_REG7 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc_1v8";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc1v8_dvp: LDO_REG8 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc1v8_dvp";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc2v8_dvp: LDO_REG9 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <2800000>;
+				regulator-max-microvolt = <2800000>;
+				regulator-name = "vcc2v8_dvp";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			dcdc_boost: BOOST {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <4700000>;
+				regulator-max-microvolt = <5400000>;
+				regulator-name = "boost";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			otg_switch: OTG_SWITCH {
+				regulator-name = "otg_switch";
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+		};
+	};
+
+	vdd_cpu: regulator@40 {
+		compatible = "fcs,fan53555";
+		reg = <0x40>;
+		fcs,suspend-voltage-selector = <1>;
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <712500>;
+		regulator-max-microvolt = <1390000>;
+		regulator-init-microvolt = <900000>;
+		regulator-name = "vdd_cpu";
+		regulator-ramp-delay = <2300>;
+		vin-supply = <&vcc_sys>;
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+};
+
+&i2c1 {
+	/* Unknown/unused device at 0x3c */
+	status = "disabled";
+};
+
+&i2c5 {
+	pinctrl-0 = <&i2c5m1_xfer>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&i2s0_8ch {
+	status = "okay";
+};
+
+&i2s1_8ch {
+	pinctrl-0 = <&i2s1m0_sclktx
+		     &i2s1m0_lrcktx
+		     &i2s1m0_sdi0
+		     &i2s1m0_sdo0>;
+	pinctrl-names = "default";
+	rockchip,trcm-sync-tx-only;
+	status = "okay";
+};
+
+&pinctrl {
+	audio-amplifier {
+		spk_amp_enable_h: spk-amp-enable-h {
+			rockchip,pins =
+				<4 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	gpio-btns {
+		btn_pins_ctrl: btn-pins-ctrl {
+			rockchip,pins =
+				<3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PA4 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PA6 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB1 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB2 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB3 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB4 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PC2 RK_FUNC_GPIO &pcfg_pull_up>,
+				<3 RK_PC3 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		btn_pins_vol: btn-pins-vol {
+			rockchip,pins =
+			<3 RK_PA7 RK_FUNC_GPIO &pcfg_pull_up>,
+			<3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	gpio-led {
+		led_pins: led-pins {
+			rockchip,pins =
+				<0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>,
+				<0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>,
+				<0 RK_PC7 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	joy-mux {
+		joy_mux_en: joy-mux-en {
+			rockchip,pins =
+				<0 RK_PB5 RK_FUNC_GPIO &pcfg_output_low>;
+		};
+	};
+
+	pmic {
+		pmic_int_l: pmic-int-l {
+			rockchip,pins =
+				<0 RK_PA3 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
+	sdio-pwrseq {
+		wifi_enable_h: wifi-enable-h {
+			rockchip,pins =
+				<4 RK_PA2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	vcc3v3-lcd {
+		vcc_lcd_h: vcc-lcd-h {
+			rockchip,pins =
+				<0 RK_PC2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	vcc-wifi {
+		vcc_wifi_h: vcc-wifi-h {
+			rockchip,pins =
+				<0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+};
+
+&pmu_io_domains {
+	status = "okay";
+	pmuio1-supply = <&vcc3v3_pmu>;
+	pmuio2-supply = <&vcc3v3_pmu>;
+	vccio1-supply = <&vccio_acodec>;
+	vccio3-supply = <&vccio_sd>;
+	vccio4-supply = <&vcc_1v8>;
+	vccio5-supply = <&vcc_3v3>;
+	vccio6-supply = <&vcc1v8_dvp>;
+	vccio7-supply = <&vcc_3v3>;
+};
+
+&pwm5 {
+	status = "okay";
+};
+
+&saradc {
+	vref-supply = <&vcc_1v8>;
+	status = "okay";
+};
+
+&sdmmc0 {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	pinctrl-0 = <&sdmmc0_bus4 &sdmmc0_clk &sdmmc0_cmd &sdmmc0_det>;
+	pinctrl-names = "default";
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vccio_sd>;
+	status = "okay";
+};
+
+&sdmmc1 {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cd-gpios = <&gpio2 RK_PB2 GPIO_ACTIVE_LOW>;
+	disable-wp;
+	pinctrl-0 = <&sdmmc1_bus4 &sdmmc1_cmd &sdmmc1_clk &sdmmc1_det>;
+	pinctrl-names = "default";
+	sd-uhs-sdr104;
+	vmmc-supply = <&vcc_3v3>;
+	vqmmc-supply = <&vcc1v8_dvp>;
+	status = "okay";
+};
+
+&sdmmc2 {
+	bus-width = <4>;
+	cap-sd-highspeed;
+	cap-sdio-irq;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&sdio_pwrseq>;
+	non-removable;
+	pinctrl-0 = <&sdmmc2m0_bus4 &sdmmc2m0_cmd &sdmmc2m0_clk>;
+	pinctrl-names = "default";
+	vmmc-supply = <&vcc_wifi>;
+	vqmmc-supply = <&vcca1v8_pmu>;
+	status = "okay";
+};
+
+&tsadc {
+	rockchip,hw-tshut-mode = <1>;
+	rockchip,hw-tshut-polarity = <0>;
+	status = "okay";
+};
+
+&uart1 {
+	pinctrl-0 = <&uart1m1_xfer &uart1m1_ctsn &uart1m1_rtsn>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "realtek,rtl8821cs-bt";
+		device-wake-gpios = <&gpio4 4 GPIO_ACTIVE_HIGH>;
+		enable-gpios = <&gpio4 3 GPIO_ACTIVE_HIGH>;
+		host-wake-gpios = <&gpio4 5 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&uart2 {
+	status = "okay";
+};
+
+/*
+ * Lack the schematics to verify, but port works as a peripheral
+ * (and not a host or OTG port).
+ */
+&usb_host0_xhci {
+	dr_mode = "peripheral";
+	phys = <&usb2phy0_otg>;
+	phy-names = "usb2-phy";
+	status = "okay";
+};
+
+&usb_host1_ehci {
+	status = "okay";
+};
+
+&usb_host1_ohci {
+	status = "okay";
+};
+
+&usb_host1_xhci {
+	phy-names = "usb2-phy", "usb3-phy";
+	phys = <&usb2phy1_host>, <&combphy1 PHY_TYPE_USB3>;
+	status = "okay";
+};
+
+&usb2phy0 {
+	status = "okay";
+};
+
+&usb2phy0_otg {
+	status = "okay";
+};
+
+&usb2phy1 {
+	status = "okay";
+};
+
+&usb2phy1_host {
+	status = "okay";
+};
+
+&vop {
+	assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>;
+	assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>;
+	status = "okay";
+};
+
+&vop_mmu {
+	status = "okay";
+};
+
+&vp0 {
+	vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
+		remote-endpoint = <&hdmi_in_vp0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts
index 528bb4e8ac77..77b179cd20e7 100644
--- a/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3566-quartz64-b.dts
@@ -4,6 +4,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/pinctrl/rockchip.h>
+#include <dt-bindings/soc/rockchip,vop2.h>
 #include "rk3566.dtsi"
 
 / {
@@ -28,6 +29,17 @@
 		#clock-cells = <0>;
 	};
 
+	hdmi-con {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
 	leds {
 		compatible = "gpio-leds";
 
@@ -42,6 +54,21 @@
 		};
 	};
 
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,name = "Analog RK809";
+		simple-audio-card,mclk-fs = <256>;
+
+		simple-audio-card,cpu {
+			sound-dai = <&i2s1_8ch>;
+		};
+
+		simple-audio-card,codec {
+			sound-dai = <&rk809>;
+		};
+	};
+
 	sdio_pwrseq: sdio-pwrseq {
 		status = "okay";
 		compatible = "mmc-pwrseq-simple";
@@ -54,6 +81,18 @@
 		power-off-delay-us = <5000000>;
 	};
 
+	vcc3v3_pcie_p: vcc3v3-pcie-p-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pcie_enable_h>;
+		regulator-name = "vcc3v3_pcie_p";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_3v3>;
+	};
+
 	vcc5v0_in: vcc5v0-in-regulator {
 		compatible = "regulator-fixed";
 		regulator-name = "vcc5v0_in";
@@ -113,6 +152,10 @@
 	status = "okay";
 };
 
+&combphy2 {
+	status = "okay";
+};
+
 &cpu0 {
 	cpu-supply = <&vdd_cpu>;
 };
@@ -152,6 +195,33 @@
 	status = "okay";
 };
 
+&gpu {
+	mali-supply = <&vdd_gpu>;
+	status = "okay";
+};
+
+&hdmi {
+	avdd-0v9-supply = <&vdda0v9_image>;
+	avdd-1v8-supply = <&vcca1v8_image>;
+	status = "okay";
+};
+
+&hdmi_in {
+	hdmi_in_vp0: endpoint {
+		remote-endpoint = <&vp0_out_hdmi>;
+	};
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
+&hdmi_sound {
+	status = "okay";
+};
+
 &i2c0 {
 	status = "okay";
 
@@ -177,11 +247,16 @@
 		reg = <0x20>;
 		interrupt-parent = <&gpio0>;
 		interrupts = <RK_PA7 IRQ_TYPE_LEVEL_LOW>;
+		assigned-clocks = <&cru I2S1_MCLKOUT_TX>;
+		assigned-clock-parents = <&cru CLK_I2S1_8CH_TX>;
+		clock-names = "mclk";
+		clocks = <&cru I2S1_MCLKOUT_TX>;
 		clock-output-names = "rk808-clkout1", "rk808-clkout2";
 
 		pinctrl-names = "default";
-		pinctrl-0 = <&pmic_int>;
+		pinctrl-0 = <&pmic_int>, <&i2s1m0_mclk>;
 		rockchip,system-power-controller;
+		#sound-dai-cells = <0>;
 		wakeup-source;
 		#clock-cells = <1>;
 
@@ -420,6 +495,20 @@
 	status = "disabled";
 };
 
+&i2s0_8ch {
+	status = "okay";
+};
+
+&i2s1_8ch {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2s1m0_sclktx
+		     &i2s1m0_lrcktx
+		     &i2s1m0_sdi0
+		     &i2s1m0_sdo0>;
+	rockchip,trcm-sync-tx-only;
+	status = "okay";
+};
+
 &mdio1 {
 	rgmii_phy1: ethernet-phy@1 {
 		compatible = "ethernet-phy-ieee802.3-c22";
@@ -427,6 +516,14 @@
 	};
 };
 
+&pcie2x1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie_reset_h>;
+	reset-gpios = <&gpio1 RK_PB2 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_pcie_p>;
+	status = "okay";
+};
+
 &pinctrl {
 	bt {
 		bt_enable_h: bt-enable-h {
@@ -448,6 +545,16 @@
 		};
 	};
 
+	pcie {
+		pcie_enable_h: pcie-enable-h {
+			rockchip,pins = <0 RK_PA6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		pcie_reset_h: pcie-reset-h {
+			rockchip,pins = <1 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	pmic {
 		pmic_int: pmic_int {
 			rockchip,pins =
@@ -613,3 +720,20 @@
 &usb_host0_ohci {
 	status = "okay";
 };
+
+&vop {
+	assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>;
+	assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>;
+	status = "okay";
+};
+
+&vop_mmu {
+	status = "okay";
+};
+
+&vp0 {
+	vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
+		remote-endpoint = <&hdmi_in_vp0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts b/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts
index 57759b66d44d..dba648c2f57e 100644
--- a/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts
@@ -4,6 +4,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/pinctrl/rockchip.h>
+#include <dt-bindings/soc/rockchip,vop2.h>
 #include "rk3566.dtsi"
 
 / {
@@ -27,6 +28,17 @@
 		#clock-cells = <0>;
 	};
 
+	hdmi-con {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
 	leds {
 		compatible = "gpio-leds";
 
@@ -149,6 +161,29 @@
 	status = "okay";
 };
 
+&gpu {
+	mali-supply = <&vdd_gpu>;
+	status = "okay";
+};
+
+&hdmi {
+	avdd-0v9-supply = <&vdda0v9_image>;
+	avdd-1v8-supply = <&vcca1v8_image>;
+	status = "okay";
+};
+
+&hdmi_in {
+	hdmi_in_vp0: endpoint {
+		remote-endpoint = <&vp0_out_hdmi>;
+	};
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
 &i2c0 {
 	status = "okay";
 
@@ -577,3 +612,20 @@
 &usb_host0_ohci {
 	status = "okay";
 };
+
+&vop {
+	assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>;
+	assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>;
+	status = "okay";
+};
+
+&vop_mmu {
+	status = "okay";
+};
+
+&vp0 {
+	vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
+		remote-endpoint = <&hdmi_in_vp0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts b/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts
index 93d383b8be87..c282f6e79960 100644
--- a/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3568-bpi-r2-pro.dts
@@ -46,7 +46,7 @@
 		};
 	};
 
-	dc_12v: dc-12v {
+	dc_12v: dc-12v-regulator {
 		compatible = "regulator-fixed";
 		regulator-name = "dc_12v";
 		regulator-always-on;
@@ -66,7 +66,7 @@
 		};
 	};
 
-	vcc3v3_sys: vcc3v3-sys {
+	vcc3v3_sys: vcc3v3-sys-regulator {
 		compatible = "regulator-fixed";
 		regulator-name = "vcc3v3_sys";
 		regulator-always-on;
@@ -76,7 +76,7 @@
 		vin-supply = <&dc_12v>;
 	};
 
-	vcc5v0_sys: vcc5v0-sys {
+	vcc5v0_sys: vcc5v0-sys-regulator {
 		compatible = "regulator-fixed";
 		regulator-name = "vcc5v0_sys";
 		regulator-always-on;
@@ -86,7 +86,67 @@
 		vin-supply = <&dc_12v>;
 	};
 
-	vcc5v0_usb: vcc5v0_usb {
+	pcie30_avdd0v9: pcie30-avdd0v9-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "pcie30_avdd0v9";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <900000>;
+		regulator-max-microvolt = <900000>;
+		vin-supply = <&vcc3v3_sys>;
+	};
+
+	pcie30_avdd1v8: pcie30-avdd1v8-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "pcie30_avdd1v8";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc3v3_sys>;
+	};
+
+	/* pi6c pcie clock generator feeds both ports */
+	vcc3v3_pi6c_05: vcc3v3-pi6c-05-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_pcie";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>;
+		startup-delay-us = <200000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
+	/* actually fed by vcc3v3_sys, dependent on pi6c clock generator */
+	vcc3v3_minipcie: vcc3v3-minipcie-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_minipcie";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		gpio = <&gpio0 RK_PC6 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&minipcie_enable_h>;
+		startup-delay-us = <50000>;
+		vin-supply = <&vcc3v3_pi6c_05>;
+	};
+
+	/* actually fed by vcc3v3_sys, dependent on pi6c clock generator */
+	vcc3v3_ngff: vcc3v3-ngff-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc3v3_ngff";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		gpio = <&gpio0 RK_PB7 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&ngffpcie_enable_h>;
+		startup-delay-us = <50000>;
+		vin-supply = <&vcc3v3_pi6c_05>;
+	};
+
+	vcc5v0_usb: vcc5v0-usb-regulator {
 		compatible = "regulator-fixed";
 		regulator-name = "vcc5v0_usb";
 		regulator-always-on;
@@ -96,7 +156,7 @@
 		vin-supply = <&dc_12v>;
 	};
 
-	vcc5v0_usb_host: vcc5v0-usb-host {
+	vcc5v0_usb_host: vcc5v0-usb-host-regulator {
 		compatible = "regulator-fixed";
 		enable-active-high;
 		gpio = <&gpio0 RK_PA6 GPIO_ACTIVE_HIGH>;
@@ -108,7 +168,7 @@
 		vin-supply = <&vcc5v0_usb>;
 	};
 
-	vcc5v0_usb_otg: vcc5v0-usb-otg {
+	vcc5v0_usb_otg: vcc5v0-usb-otg-regulator {
 		compatible = "regulator-fixed";
 		enable-active-high;
 		gpio = <&gpio0 RK_PA5 GPIO_ACTIVE_HIGH>;
@@ -513,6 +573,32 @@
 	};
 };
 
+&pcie30phy {
+	data-lanes = <1 2>;
+	phy-supply = <&vcc3v3_pi6c_05>;
+	status = "okay";
+};
+
+&pcie3x1 {
+	/* M.2 slot */
+	num-lanes = <1>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&ngffpcie_reset_h>;
+	reset-gpios = <&gpio3 RK_PA1 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_ngff>;
+	status = "okay";
+};
+
+&pcie3x2 {
+	/* mPCIe slot */
+	num-lanes = <1>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&minipcie_reset_h>;
+	reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_minipcie>;
+	status = "okay";
+};
+
 &pinctrl {
 	leds {
 		blue_led_pin: blue-led-pin {
@@ -529,6 +615,24 @@
 		};
 	};
 
+	pcie {
+		minipcie_enable_h: minipcie-enable-h {
+			rockchip,pins = <0 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none_drv_level_5>;
+		};
+
+		ngffpcie_enable_h: ngffpcie-enable-h {
+			rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none_drv_level_5>;
+		};
+
+		minipcie_reset_h: minipcie-reset-h {
+			rockchip,pins = <2 RK_PD6 RK_FUNC_GPIO &pcfg_pull_none_drv_level_5>;
+		};
+
+		ngffpcie_reset_h: ngffpcie-reset-h {
+			rockchip,pins = <3 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none_drv_level_5>;
+		};
+	};
+
 	pmic {
 		pmic_int: pmic_int {
 			rockchip,pins =
@@ -708,6 +812,19 @@
 	status = "okay";
 };
 
+&usb2phy1 {
+	/* USB for PCIe/M2 */
+	status = "okay";
+};
+
+&usb2phy1_host {
+	status = "okay";
+};
+
+&usb2phy1_otg {
+	status = "okay";
+};
+
 &vop {
 	assigned-clocks = <&cru DCLK_VOP0>, <&cru DCLK_VOP1>;
 	assigned-clock-parents = <&pmucru PLL_HPLL>, <&cru PLL_VPLL>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3568-rock-3a.dts b/arch/arm64/boot/dts/rockchip/rk3568-rock-3a.dts
index b2e040dffb59..fb87a168fe96 100644
--- a/arch/arm64/boot/dts/rockchip/rk3568-rock-3a.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3568-rock-3a.dts
@@ -67,6 +67,18 @@
 		regulator-boot-on;
 	};
 
+	vcc3v3_pcie: vcc3v3-pcie-regulator {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio0 RK_PD4 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pcie_enable_h>;
+		regulator-name = "vcc3v3_pcie";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc5v0_sys>;
+	};
+
 	vcc3v3_sys: vcc3v3-sys {
 		compatible = "regulator-fixed";
 		regulator-name = "vcc3v3_sys";
@@ -131,6 +143,38 @@
 		regulator-max-microvolt = <5000000>;
 		vin-supply = <&vcc5v0_usb>;
 	};
+
+	vcc_cam: vcc-cam {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio1 RK_PB1 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc_cam_en>;
+		regulator-name = "vcc_cam";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc3v3_sys>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
+
+	vcc_mipi: vcc-mipi {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpio = <&gpio3 RK_PC0 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&vcc_mipi_en>;
+		regulator-name = "vcc_mipi";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc3v3_sys>;
+
+		regulator-state-mem {
+			regulator-off-in-suspend;
+		};
+	};
 };
 
 &combphy0 {
@@ -141,6 +185,10 @@
 	status = "okay";
 };
 
+&combphy2 {
+	status = "okay";
+};
+
 &cpu0 {
 	cpu-supply = <&vdd_cpu>;
 };
@@ -440,6 +488,35 @@
 	};
 };
 
+&i2c3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c3m1_xfer>;
+	status = "disabled";
+};
+
+&i2c4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c4m1_xfer>;
+	status = "disabled";
+};
+
+&i2c5 {
+	status = "okay";
+
+	hym8563: rtc@51 {
+		compatible = "haoyu,hym8563";
+		reg = <0x51>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <RK_PD3 IRQ_TYPE_LEVEL_LOW>;
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+		clock-output-names = "rtcic_32kout";
+		pinctrl-names = "default";
+		pinctrl-0 = <&hym8563_int>;
+		wakeup-source;
+	};
+};
+
 &i2s0_8ch {
 	status = "okay";
 };
@@ -461,19 +538,55 @@
 	};
 };
 
+&pcie2x1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie_reset_h>;
+	reset-gpios = <&gpio3 RK_PC1 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc3v3_pcie>;
+	status = "okay";
+};
+
 &pinctrl {
+	cam {
+		vcc_cam_en: vcc_cam_en {
+			rockchip,pins = <1 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	display {
+		vcc_mipi_en: vcc_mipi_en {
+			rockchip,pins = <3 RK_PC0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	ethernet {
 		eth_phy_rst: eth_phy_rst {
 			rockchip,pins = <3 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
 
+	hym8563 {
+		hym8563_int: hym8563-int {
+			rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
 	leds {
 		led_user_en: led_user_en {
 			rockchip,pins = <0 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
 
+	pcie {
+		pcie_enable_h: pcie-enable-h {
+			rockchip,pins = <0 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		pcie_reset_h: pcie-reset-h {
+			rockchip,pins = <3 RK_PC1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	pmic {
 		pmic_int: pmic_int {
 			rockchip,pins =
diff --git a/arch/arm64/boot/dts/rockchip/rk3568.dtsi b/arch/arm64/boot/dts/rockchip/rk3568.dtsi
index 2bdf8c7e9765..ba67b58f05b7 100644
--- a/arch/arm64/boot/dts/rockchip/rk3568.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3568.dtsi
@@ -42,6 +42,128 @@
 		reg = <0x0 0xfe190200 0x0 0x20>;
 	};
 
+	pcie30_phy_grf: syscon@fdcb8000 {
+		compatible = "rockchip,rk3568-pcie3-phy-grf", "syscon";
+		reg = <0x0 0xfdcb8000 0x0 0x10000>;
+	};
+
+	pcie30phy: phy@fe8c0000 {
+		compatible = "rockchip,rk3568-pcie3-phy";
+		reg = <0x0 0xfe8c0000 0x0 0x20000>;
+		#phy-cells = <0>;
+		clocks = <&pmucru CLK_PCIE30PHY_REF_M>, <&pmucru CLK_PCIE30PHY_REF_N>,
+			 <&cru PCLK_PCIE30PHY>;
+		clock-names = "refclk_m", "refclk_n", "pclk";
+		resets = <&cru SRST_PCIE30PHY>;
+		reset-names = "phy";
+		rockchip,phy-grf = <&pcie30_phy_grf>;
+		status = "disabled";
+	};
+
+	pcie3x1: pcie@fe270000 {
+		compatible = "rockchip,rk3568-pcie";
+		#address-cells = <3>;
+		#size-cells = <2>;
+		bus-range = <0x0 0xf>;
+		clocks = <&cru ACLK_PCIE30X1_MST>, <&cru ACLK_PCIE30X1_SLV>,
+			 <&cru ACLK_PCIE30X1_DBI>, <&cru PCLK_PCIE30X1>,
+			 <&cru CLK_PCIE30X1_AUX_NDFT>;
+		clock-names = "aclk_mst", "aclk_slv",
+			      "aclk_dbi", "pclk", "aux";
+		device_type = "pci";
+		interrupts = <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "sys", "pmc", "msg", "legacy", "err";
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 7>;
+		interrupt-map = <0 0 0 1 &pcie3x1_intc 0>,
+				<0 0 0 2 &pcie3x1_intc 1>,
+				<0 0 0 3 &pcie3x1_intc 2>,
+				<0 0 0 4 &pcie3x1_intc 3>;
+		linux,pci-domain = <1>;
+		num-ib-windows = <6>;
+		num-ob-windows = <2>;
+		max-link-speed = <3>;
+		msi-map = <0x0 &gic 0x1000 0x1000>;
+		num-lanes = <1>;
+		phys = <&pcie30phy>;
+		phy-names = "pcie-phy";
+		power-domains = <&power RK3568_PD_PIPE>;
+		reg = <0x3 0xc0400000 0x0 0x00400000>,
+		      <0x0 0xfe270000 0x0 0x00010000>,
+		      <0x3 0x7f000000 0x0 0x01000000>;
+		ranges = <0x01000000 0x0 0x3ef00000 0x3 0x7ef00000 0x0 0x00100000>,
+			 <0x02000000 0x0 0x00000000 0x3 0x40000000 0x0 0x3ef00000>;
+		reg-names = "dbi", "apb", "config";
+		resets = <&cru SRST_PCIE30X1_POWERUP>;
+		reset-names = "pipe";
+		/* bifurcation; lane1 when using 1+1 */
+		status = "disabled";
+
+		pcie3x1_intc: legacy-interrupt-controller {
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <1>;
+			interrupt-parent = <&gic>;
+			interrupts = <GIC_SPI 157 IRQ_TYPE_EDGE_RISING>;
+		};
+	};
+
+	pcie3x2: pcie@fe280000 {
+		compatible = "rockchip,rk3568-pcie";
+		#address-cells = <3>;
+		#size-cells = <2>;
+		bus-range = <0x0 0xf>;
+		clocks = <&cru ACLK_PCIE30X2_MST>, <&cru ACLK_PCIE30X2_SLV>,
+			 <&cru ACLK_PCIE30X2_DBI>, <&cru PCLK_PCIE30X2>,
+			 <&cru CLK_PCIE30X2_AUX_NDFT>;
+		clock-names = "aclk_mst", "aclk_slv",
+			      "aclk_dbi", "pclk", "aux";
+		device_type = "pci";
+		interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-names = "sys", "pmc", "msg", "legacy", "err";
+		#interrupt-cells = <1>;
+		interrupt-map-mask = <0 0 0 7>;
+		interrupt-map = <0 0 0 1 &pcie3x2_intc 0>,
+				<0 0 0 2 &pcie3x2_intc 1>,
+				<0 0 0 3 &pcie3x2_intc 2>,
+				<0 0 0 4 &pcie3x2_intc 3>;
+		linux,pci-domain = <2>;
+		num-ib-windows = <6>;
+		num-ob-windows = <2>;
+		max-link-speed = <3>;
+		msi-map = <0x0 &gic 0x2000 0x1000>;
+		num-lanes = <2>;
+		phys = <&pcie30phy>;
+		phy-names = "pcie-phy";
+		power-domains = <&power RK3568_PD_PIPE>;
+		reg = <0x3 0xc0800000 0x0 0x00400000>,
+		      <0x0 0xfe280000 0x0 0x00010000>,
+		      <0x3 0xbf000000 0x0 0x01000000>;
+		ranges = <0x01000000 0x0 0x3ef00000 0x3 0xbef00000 0x0 0x00100000>,
+			 <0x02000000 0x0 0x00000000 0x3 0x80000000 0x0 0x3ef00000>;
+		reg-names = "dbi", "apb", "config";
+		resets = <&cru SRST_PCIE30X2_POWERUP>;
+		reset-names = "pipe";
+		/* bifurcation; lane0 when using 1+1 */
+		status = "disabled";
+
+		pcie3x2_intc: legacy-interrupt-controller {
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <1>;
+			interrupt-parent = <&gic>;
+			interrupts = <GIC_SPI 162 IRQ_TYPE_EDGE_RISING>;
+		};
+	};
+
 	gmac0: ethernet@fe2a0000 {
 		compatible = "rockchip,rk3568-gmac", "snps,dwmac-4.20a";
 		reg = <0x0 0xfe2a0000 0x0 0x10000>;
diff --git a/arch/arm64/boot/dts/rockchip/rk356x.dtsi b/arch/arm64/boot/dts/rockchip/rk356x.dtsi
index 319981c3e9f7..164708f1eb67 100644
--- a/arch/arm64/boot/dts/rockchip/rk356x.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk356x.dtsi
@@ -592,6 +592,46 @@
 		status = "disabled";
 	};
 
+	vpu: video-codec@fdea0400 {
+		compatible = "rockchip,rk3568-vpu";
+		reg = <0x0 0xfdea0000 0x0 0x800>;
+		interrupts = <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>;
+		clock-names = "aclk", "hclk";
+		iommus = <&vdpu_mmu>;
+		power-domains = <&power RK3568_PD_VPU>;
+	};
+
+	vdpu_mmu: iommu@fdea0800 {
+		compatible = "rockchip,rk3568-iommu";
+		reg = <0x0 0xfdea0800 0x0 0x40>;
+		interrupts = <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>;
+		clock-names = "aclk", "iface";
+		clocks = <&cru ACLK_VPU>, <&cru HCLK_VPU>;
+		power-domains = <&power RK3568_PD_VPU>;
+		#iommu-cells = <0>;
+	};
+
+	vepu: video-codec@fdee0000 {
+		compatible = "rockchip,rk3568-vepu";
+		reg = <0x0 0xfdee0000 0x0 0x800>;
+		interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cru ACLK_JENC>, <&cru HCLK_JENC>;
+		clock-names = "aclk", "hclk";
+		iommus = <&vepu_mmu>;
+		power-domains = <&power RK3568_PD_RGA>;
+	};
+
+	vepu_mmu: iommu@fdee0800 {
+		compatible = "rockchip,rk3568-iommu";
+		reg = <0x0 0xfdee0800 0x0 0x40>;
+		interrupts = <GIC_SPI 63 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&cru ACLK_JENC>, <&cru HCLK_JENC>;
+		clock-names = "aclk", "iface";
+		power-domains = <&power RK3568_PD_RGA>;
+		#iommu-cells = <0>;
+	};
+
 	sdmmc2: mmc@fe000000 {
 		compatible = "rockchip,rk3568-dw-mshc", "rockchip,rk3288-dw-mshc";
 		reg = <0x0 0xfe000000 0x0 0x4000>;
@@ -699,6 +739,62 @@
 		status = "disabled";
 	};
 
+	dsi0: dsi@fe060000 {
+		compatible = "rockchip,rk3568-mipi-dsi", "snps,dw-mipi-dsi";
+		reg = <0x00 0xfe060000 0x00 0x10000>;
+		interrupts = <GIC_SPI 68 IRQ_TYPE_LEVEL_HIGH>;
+		clock-names = "pclk", "hclk";
+		clocks = <&cru PCLK_DSITX_0>, <&cru HCLK_VO>;
+		phy-names = "dphy";
+		phys = <&dsi_dphy0>;
+		power-domains = <&power RK3568_PD_VO>;
+		reset-names = "apb";
+		resets = <&cru SRST_P_DSITX_0>;
+		rockchip,grf = <&grf>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dsi0_in: port@0 {
+				reg = <0>;
+			};
+
+			dsi0_out: port@1 {
+				reg = <1>;
+			};
+		};
+	};
+
+	dsi1: dsi@fe070000 {
+		compatible = "rockchip,rk3568-mipi-dsi", "snps,dw-mipi-dsi";
+		reg = <0x0 0xfe070000 0x0 0x10000>;
+		interrupts = <GIC_SPI 69 IRQ_TYPE_LEVEL_HIGH>;
+		clock-names = "pclk", "hclk";
+		clocks = <&cru PCLK_DSITX_1>, <&cru HCLK_VO>;
+		phy-names = "dphy";
+		phys = <&dsi_dphy1>;
+		power-domains = <&power RK3568_PD_VO>;
+		reset-names = "apb";
+		resets = <&cru SRST_P_DSITX_1>;
+		rockchip,grf = <&grf>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			dsi1_in: port@0 {
+				reg = <0>;
+			};
+
+			dsi1_out: port@1 {
+				reg = <1>;
+			};
+		};
+	};
+
 	hdmi: hdmi@fe0a0000 {
 		compatible = "rockchip,rk3568-dw-hdmi";
 		reg = <0x0 0xfe0a0000 0x0 0x20000>;
@@ -1594,6 +1690,42 @@
 		status = "disabled";
 	};
 
+	csi_dphy: phy@fe870000 {
+		compatible = "rockchip,rk3568-csi-dphy";
+		reg = <0x0 0xfe870000 0x0 0x10000>;
+		clocks = <&cru PCLK_MIPICSIPHY>;
+		clock-names = "pclk";
+		#phy-cells = <0>;
+		resets = <&cru SRST_P_MIPICSIPHY>;
+		reset-names = "apb";
+		rockchip,grf = <&grf>;
+		status = "disabled";
+	};
+
+	dsi_dphy0: mipi-dphy@fe850000 {
+		compatible = "rockchip,rk3568-dsi-dphy";
+		reg = <0x0 0xfe850000 0x0 0x10000>;
+		clock-names = "ref", "pclk";
+		clocks = <&pmucru CLK_MIPIDSIPHY0_REF>, <&cru PCLK_MIPIDSIPHY0>;
+		#phy-cells = <0>;
+		power-domains = <&power RK3568_PD_VO>;
+		reset-names = "apb";
+		resets = <&cru SRST_P_MIPIDSIPHY0>;
+		status = "disabled";
+	};
+
+	dsi_dphy1: mipi-dphy@fe860000 {
+		compatible = "rockchip,rk3568-dsi-dphy";
+		reg = <0x0 0xfe860000 0x0 0x10000>;
+		clock-names = "ref", "pclk";
+		clocks = <&pmucru CLK_MIPIDSIPHY1_REF>, <&cru PCLK_MIPIDSIPHY1>;
+		#phy-cells = <0>;
+		power-domains = <&power RK3568_PD_VO>;
+		reset-names = "apb";
+		resets = <&cru SRST_P_MIPIDSIPHY1>;
+		status = "disabled";
+	};
+
 	usb2phy0: usb2phy@fe8a0000 {
 		compatible = "rockchip,rk3568-usb2phy";
 		reg = <0x0 0xfe8a0000 0x0 0x10000>;
diff --git a/arch/arm64/boot/dts/socionext/Makefile b/arch/arm64/boot/dts/socionext/Makefile
index dda3da33614b..33989a9643ac 100644
--- a/arch/arm64/boot/dts/socionext/Makefile
+++ b/arch/arm64/boot/dts/socionext/Makefile
@@ -5,4 +5,6 @@ dtb-$(CONFIG_ARCH_UNIPHIER) += \
 	uniphier-ld20-akebi96.dtb \
 	uniphier-ld20-global.dtb \
 	uniphier-ld20-ref.dtb \
-	uniphier-pxs3-ref.dtb
+	uniphier-pxs3-ref.dtb \
+	uniphier-pxs3-ref-gadget0.dtb \
+	uniphier-pxs3-ref-gadget1.dtb
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts b/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts
index 617d2b1e9b1e..414aeb99e68f 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld11-ref.dts
@@ -39,11 +39,11 @@
 };
 
 &ethsc {
-	interrupts = <0 8>;
+	interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serialsc {
-	interrupts = <0 8>;
+	interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serial0 {
@@ -51,7 +51,7 @@
 };
 
 &gpio {
-	xirq0 {
+	xirq0-hog {
 		gpio-hog;
 		gpios = <UNIPHIER_GPIO_IRQ(0) 0>;
 		input;
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
index 15dcfc259854..1c76b4375b2e 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld11.dtsi
@@ -7,6 +7,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/gpio/uniphier-gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
 
 / {
 	compatible = "socionext,uniphier-ld11";
@@ -35,6 +36,7 @@
 			reg = <0 0x000>;
 			clocks = <&sys_clk 33>;
 			enable-method = "psci";
+			next-level-cache = <&l2>;
 			operating-points-v2 = <&cluster0_opp>;
 		};
 
@@ -44,8 +46,13 @@
 			reg = <0 0x001>;
 			clocks = <&sys_clk 33>;
 			enable-method = "psci";
+			next-level-cache = <&l2>;
 			operating-points-v2 = <&cluster0_opp>;
 		};
+
+		l2: l2-cache {
+			compatible = "cache";
+		};
 	};
 
 	cluster0_opp: opp-table {
@@ -102,10 +109,10 @@
 
 	timer {
 		compatible = "arm,armv8-timer";
-		interrupts = <1 13 4>,
-			     <1 14 4>,
-			     <1 11 4>,
-			     <1 10 4>;
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
 	reserved-memory {
@@ -131,7 +138,7 @@
 			reg = <0x54006000 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 39 4>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi0>;
 			clocks = <&peri_clk 11>;
@@ -144,7 +151,7 @@
 			reg = <0x54006100 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 216 4>;
+			interrupts = <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi1>;
 			clocks = <&peri_clk 12>;
@@ -155,7 +162,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006800 0x40>;
-			interrupts = <0 33 4>;
+			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart0>;
 			clocks = <&peri_clk 0>;
@@ -166,7 +173,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006900 0x40>;
-			interrupts = <0 35 4>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart1>;
 			clocks = <&peri_clk 1>;
@@ -177,7 +184,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006a00 0x40>;
-			interrupts = <0 37 4>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart2>;
 			clocks = <&peri_clk 2>;
@@ -188,7 +195,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006b00 0x40>;
-			interrupts = <0 177 4>;
+			interrupts = <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart3>;
 			clocks = <&peri_clk 3>;
@@ -223,7 +230,7 @@
 		audio@56000000 {
 			compatible = "socionext,uniphier-ld11-aio";
 			reg = <0x56000000 0x80000>;
-			interrupts = <0 144 4>;
+			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_aout1>,
 				    <&pinctrl_aoutiec1>;
@@ -323,7 +330,7 @@
 			reg = <0x58780000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 41 4>;
+			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c0>;
 			clocks = <&peri_clk 4>;
@@ -337,7 +344,7 @@
 			reg = <0x58781000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 42 4>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c1>;
 			clocks = <&peri_clk 5>;
@@ -350,7 +357,7 @@
 			reg = <0x58782000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 43 4>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 6>;
 			resets = <&peri_rst 6>;
 			clock-frequency = <400000>;
@@ -362,7 +369,7 @@
 			reg = <0x58783000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 44 4>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c3>;
 			clocks = <&peri_clk 7>;
@@ -376,7 +383,7 @@
 			reg = <0x58784000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 45 4>;
+			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c4>;
 			clocks = <&peri_clk 8>;
@@ -389,7 +396,7 @@
 			reg = <0x58785000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 25 4>;
+			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 9>;
 			resets = <&peri_rst 9>;
 			clock-frequency = <400000>;
@@ -440,7 +447,7 @@
 		emmc: mmc@5a000000 {
 			compatible = "socionext,uniphier-sd4hc", "cdns,sd4hc";
 			reg = <0x5a000000 0x400>;
-			interrupts = <0 78 4>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_emmc>;
 			clocks = <&sys_clk 4>;
@@ -460,7 +467,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a800100 0x100>;
-			interrupts = <0 243 4>;
+			interrupts = <GIC_SPI 243 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb0>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 8>,
@@ -476,7 +483,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a810100 0x100>;
-			interrupts = <0 244 4>;
+			interrupts = <GIC_SPI 244 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb1>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 9>,
@@ -492,7 +499,7 @@
 			compatible = "socionext,uniphier-ehci", "generic-ehci";
 			status = "disabled";
 			reg = <0x5a820100 0x100>;
-			interrupts = <0 245 4>;
+			interrupts = <GIC_SPI 245 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb2>;
 			clocks = <&sys_clk 8>, <&mio_clk 7>, <&mio_clk 10>,
@@ -530,7 +537,7 @@
 				compatible = "socionext,uniphier-ld11-pinctrl";
 			};
 
-			usb-phy {
+			usb-controller {
 				compatible = "socionext,uniphier-ld11-usb2-phy";
 				#address-cells = <1>;
 				#size-cells = <0>;
@@ -573,7 +580,7 @@
 		xdmac: dma-controller@5fc10000 {
 			compatible = "socionext,uniphier-xdmac";
 			reg = <0x5fc10000 0x5300>;
-			interrupts = <0 188 4>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
 			dma-channels = <16>;
 			#dma-cells = <2>;
 		};
@@ -591,7 +598,7 @@
 			      <0x5fe40000 0x80000>;	/* GICR */
 			interrupt-controller;
 			#interrupt-cells = <3>;
-			interrupts = <1 9 4>;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		sysctrl@61840000 {
@@ -618,7 +625,7 @@
 			compatible = "socionext,uniphier-ld11-ave4";
 			status = "disabled";
 			reg = <0x65000000 0x8500>;
-			interrupts = <0 66 4>;
+			interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
 			clock-names = "ether";
 			clocks = <&sys_clk 6>;
 			reset-names = "ether";
@@ -640,7 +647,7 @@
 			reg = <0x68000000 0x20>, <0x68100000 0x1000>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 65 4>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_nand>;
 			clock-names = "nand", "nand_x", "ecc";
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20-akebi96.dts b/arch/arm64/boot/dts/socionext/uniphier-ld20-akebi96.dts
index aa159a11292c..fba454adae7d 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld20-akebi96.dts
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld20-akebi96.dts
@@ -110,7 +110,7 @@
 		spi-max-frequency = <12500000>;
 		interrupt-parent = <&gpio>;
 		interrupt-names = "udc";
-		interrupts = <0 2>;
+		interrupts = <0 IRQ_TYPE_EDGE_FALLING>;
 	};
 };
 
@@ -168,12 +168,12 @@
 
 &gpio {
 	/* IRQs for Max3421 */
-	xirq0 {
+	xirq0-hog {
 		gpio-hog;
 		gpios = <UNIPHIER_GPIO_IRQ(0) 1>;
 		input;
 	};
-	xirq10 {
+	xirq10-hog {
 		gpio-hog;
 		gpios = <UNIPHIER_GPIO_IRQ(10) 1>;
 		input;
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts b/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts
index 39ee279a1eb9..a5f2083f8b75 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld20-ref.dts
@@ -39,11 +39,11 @@
 };
 
 &ethsc {
-	interrupts = <0 8>;
+	interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serialsc {
-	interrupts = <0 8>;
+	interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serial0 {
@@ -51,7 +51,7 @@
 };
 
 &gpio {
-	xirq0 {
+	xirq0-hog {
 		gpio-hog;
 		gpios = <UNIPHIER_GPIO_IRQ(0) 0>;
 		input;
diff --git a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
index 8f2c1c1e2c64..9308458f9611 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-ld20.dtsi
@@ -7,6 +7,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/gpio/uniphier-gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/thermal/thermal.h>
 
 / {
@@ -45,6 +46,7 @@
 			reg = <0 0x000>;
 			clocks = <&sys_clk 32>;
 			enable-method = "psci";
+			next-level-cache = <&a72_l2>;
 			operating-points-v2 = <&cluster0_opp>;
 			#cooling-cells = <2>;
 		};
@@ -55,6 +57,7 @@
 			reg = <0 0x001>;
 			clocks = <&sys_clk 32>;
 			enable-method = "psci";
+			next-level-cache = <&a72_l2>;
 			operating-points-v2 = <&cluster0_opp>;
 			#cooling-cells = <2>;
 		};
@@ -65,6 +68,7 @@
 			reg = <0 0x100>;
 			clocks = <&sys_clk 33>;
 			enable-method = "psci";
+			next-level-cache = <&a53_l2>;
 			operating-points-v2 = <&cluster1_opp>;
 			#cooling-cells = <2>;
 		};
@@ -75,12 +79,21 @@
 			reg = <0 0x101>;
 			clocks = <&sys_clk 33>;
 			enable-method = "psci";
+			next-level-cache = <&a53_l2>;
 			operating-points-v2 = <&cluster1_opp>;
 			#cooling-cells = <2>;
 		};
+
+		a72_l2: l2-cache0 {
+			compatible = "cache";
+		};
+
+		a53_l2: l2-cache1 {
+			compatible = "cache";
+		};
 	};
 
-	cluster0_opp: opp-table0 {
+	cluster0_opp: opp-table-0 {
 		compatible = "operating-points-v2";
 		opp-shared;
 
@@ -118,7 +131,7 @@
 		};
 	};
 
-	cluster1_opp: opp-table1 {
+	cluster1_opp: opp-table-1 {
 		compatible = "operating-points-v2";
 		opp-shared;
 
@@ -176,10 +189,10 @@
 
 	timer {
 		compatible = "arm,armv8-timer";
-		interrupts = <1 13 4>,
-			     <1 14 4>,
-			     <1 11 4>,
-			     <1 10 4>;
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
 	thermal-zones {
@@ -236,7 +249,7 @@
 			reg = <0x54006000 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 39 4>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi0>;
 			clocks = <&peri_clk 11>;
@@ -249,7 +262,7 @@
 			reg = <0x54006100 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 216 4>;
+			interrupts = <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi1>;
 			clocks = <&peri_clk 12>;
@@ -262,7 +275,7 @@
 			reg = <0x54006200 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 229 4>;
+			interrupts = <GIC_SPI 229 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi2>;
 			clocks = <&peri_clk 13>;
@@ -275,7 +288,7 @@
 			reg = <0x54006300 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 230 4>;
+			interrupts = <GIC_SPI 230 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi3>;
 			clocks = <&peri_clk 14>;
@@ -286,7 +299,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006800 0x40>;
-			interrupts = <0 33 4>;
+			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart0>;
 			clocks = <&peri_clk 0>;
@@ -297,7 +310,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006900 0x40>;
-			interrupts = <0 35 4>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart1>;
 			clocks = <&peri_clk 1>;
@@ -308,7 +321,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006a00 0x40>;
-			interrupts = <0 37 4>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart2>;
 			clocks = <&peri_clk 2>;
@@ -319,7 +332,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006b00 0x40>;
-			interrupts = <0 177 4>;
+			interrupts = <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart3>;
 			clocks = <&peri_clk 3>;
@@ -348,7 +361,7 @@
 		audio@56000000 {
 			compatible = "socionext,uniphier-ld20-aio";
 			reg = <0x56000000 0x80000>;
-			interrupts = <0 144 4>;
+			interrupts = <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_aout1>,
 				    <&pinctrl_aoutiec1>;
@@ -448,7 +461,7 @@
 			reg = <0x58780000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 41 4>;
+			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c0>;
 			clocks = <&peri_clk 4>;
@@ -462,7 +475,7 @@
 			reg = <0x58781000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 42 4>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c1>;
 			clocks = <&peri_clk 5>;
@@ -475,7 +488,7 @@
 			reg = <0x58782000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 43 4>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 6>;
 			resets = <&peri_rst 6>;
 			clock-frequency = <400000>;
@@ -487,7 +500,7 @@
 			reg = <0x58783000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 44 4>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c3>;
 			clocks = <&peri_clk 7>;
@@ -501,7 +514,7 @@
 			reg = <0x58784000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 45 4>;
+			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c4>;
 			clocks = <&peri_clk 8>;
@@ -514,7 +527,7 @@
 			reg = <0x58785000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 25 4>;
+			interrupts = <GIC_SPI 25 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 9>;
 			resets = <&peri_rst 9>;
 			clock-frequency = <400000>;
@@ -570,7 +583,7 @@
 		emmc: mmc@5a000000 {
 			compatible = "socionext,uniphier-sd4hc", "cdns,sd4hc";
 			reg = <0x5a000000 0x400>;
-			interrupts = <0 78 4>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_emmc>;
 			clocks = <&sys_clk 4>;
@@ -590,7 +603,7 @@
 			compatible = "socionext,uniphier-sd-v3.1.1";
 			status = "disabled";
 			reg = <0x5a400000 0x800>;
-			interrupts = <0 76 4>;
+			interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_sd>;
 			clocks = <&sd_clk 0>;
@@ -675,7 +688,7 @@
 		xdmac: dma-controller@5fc10000 {
 			compatible = "socionext,uniphier-xdmac";
 			reg = <0x5fc10000 0x5300>;
-			interrupts = <0 188 4>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
 			dma-channels = <16>;
 			#dma-cells = <2>;
 		};
@@ -693,7 +706,7 @@
 			      <0x5fe80000 0x80000>;	/* GICR */
 			interrupt-controller;
 			#interrupt-cells = <3>;
-			interrupts = <1 9 4>;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		sysctrl@61840000 {
@@ -715,9 +728,9 @@
 				compatible = "socionext,uniphier-wdt";
 			};
 
-			pvtctl: pvtctl {
+			pvtctl: thermal-sensor {
 				compatible = "socionext,uniphier-ld20-thermal";
-				interrupts = <0 3 4>;
+				interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
 				#thermal-sensor-cells = <0>;
 				socionext,tmod-calibration = <0x0f22 0x68ee>;
 			};
@@ -727,7 +740,7 @@
 			compatible = "socionext,uniphier-ld20-ave4";
 			status = "disabled";
 			reg = <0x65000000 0x8500>;
-			interrupts = <0 66 4>;
+			interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ether_rgmii>;
 			clock-names = "ether";
@@ -749,7 +762,7 @@
 			status = "disabled";
 			reg = <0x65a00000 0xcd00>;
 			interrupt-names = "host";
-			interrupts = <0 134 4>;
+			interrupts = <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb0>, <&pinctrl_usb1>,
 				    <&pinctrl_usb2>, <&pinctrl_usb3>;
@@ -762,7 +775,7 @@
 			dr_mode = "host";
 		};
 
-		usb-glue@65b00000 {
+		usb-controller@65b00000 {
 			compatible = "socionext,uniphier-ld20-dwc3-glue",
 				     "simple-mfd";
 			#address-cells = <1>;
@@ -895,7 +908,7 @@
 		};
 
 		pcie: pcie@66000000 {
-			compatible = "socionext,uniphier-pcie", "snps,dw-pcie";
+			compatible = "socionext,uniphier-pcie";
 			status = "disabled";
 			reg-names = "dbi", "link", "config";
 			reg = <0x66000000 0x1000>, <0x66010000 0x10000>,
@@ -915,7 +928,8 @@
 				<0x82000000 0 0x20000000 0x20000000 0 0x0ffe0000>;
 			#interrupt-cells = <1>;
 			interrupt-names = "dma", "msi";
-			interrupts = <0 224 4>, <0 225 4>;
+			interrupts = <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-map-mask = <0 0 0 7>;
 			interrupt-map = <0 0 0 1 &pcie_intc 0>,	/* INTA */
 					<0 0 0 2 &pcie_intc 1>,	/* INTB */
@@ -928,7 +942,7 @@
 				interrupt-controller;
 				#interrupt-cells = <1>;
 				interrupt-parent = <&gic>;
-				interrupts = <0 226 4>;
+				interrupts = <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
@@ -950,7 +964,7 @@
 			reg = <0x68000000 0x20>, <0x68100000 0x1000>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 65 4>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_nand>;
 			clock-names = "nand", "nand_x", "ecc";
diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget0.dts b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget0.dts
new file mode 100644
index 000000000000..7069f51bc120
--- /dev/null
+++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget0.dts
@@ -0,0 +1,41 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+//
+// Device Tree Source for UniPhier PXs3 Reference Board (for USB-Device #0)
+//
+// Copyright (C) 2021 Socionext Inc.
+//   Author: Kunihiko Hayashi <hayashi.kunihiko@socionext.com>
+
+/dts-v1/;
+#include "uniphier-pxs3-ref.dts"
+
+/ {
+	model = "UniPhier PXs3 Reference Board (USB-Device #0)";
+};
+
+/* I2C3 pinctrl is shared with USB*VBUSIN */
+&i2c3 {
+	status = "disabled";
+};
+
+&usb0 {
+	status = "okay";
+	dr_mode = "peripheral";
+	pinctrl-0 = <&pinctrl_usb0_device>;
+	snps,dis_enblslpm_quirk;
+	snps,dis_u2_susphy_quirk;
+	snps,dis_u3_susphy_quirk;
+	snps,usb2_gadget_lpm_disable;
+	phy-names = "usb2-phy", "usb3-phy";
+	phys = <&usb0_hsphy0>, <&usb0_ssphy0>;
+};
+
+&usb0_hsphy0 {
+	/delete-property/ vbus-supply;
+};
+
+&usb0_ssphy0 {
+	/delete-property/ vbus-supply;
+};
+
+/delete-node/ &usb0_hsphy1;
+/delete-node/ &usb0_ssphy1;
diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget1.dts b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget1.dts
new file mode 100644
index 000000000000..a3cfa8113ffb
--- /dev/null
+++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref-gadget1.dts
@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+//
+// Device Tree Source for UniPhier PXs3 Reference Board (for USB-Device #1)
+//
+// Copyright (C) 2021 Socionext Inc.
+//   Author: Kunihiko Hayashi <hayashi.kunihiko@socionext.com>
+
+/dts-v1/;
+#include "uniphier-pxs3-ref.dts"
+
+/ {
+	model = "UniPhier PXs3 Reference Board (USB-Device #1)";
+};
+
+/* I2C3 pinctrl is shared with USB*VBUSIN */
+&i2c3 {
+	status = "disabled";
+};
+
+&usb1 {
+	status = "okay";
+	dr_mode = "peripheral";
+	pinctrl-0 = <&pinctrl_usb1_device>;
+	snps,dis_enblslpm_quirk;
+	snps,dis_u2_susphy_quirk;
+	snps,dis_u3_susphy_quirk;
+	snps,usb2_gadget_lpm_disable;
+	phy-names = "usb2-phy", "usb3-phy";
+	phys = <&usb1_hsphy0>, <&usb1_ssphy0>;
+};
+
+&usb1_hsphy0 {
+	/delete-property/ vbus-supply;
+};
+
+&usb1_ssphy0 {
+	/delete-property/ vbus-supply;
+};
+
+/delete-node/ &usb1_hsphy1;
diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts
index 086040306fb3..1ced6190ab2b 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts
+++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3-ref.dts
@@ -40,11 +40,11 @@
 };
 
 &ethsc {
-	interrupts = <4 8>;
+	interrupts = <4 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &serialsc {
-	interrupts = <4 8>;
+	interrupts = <4 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &spi0 {
@@ -68,7 +68,7 @@
 };
 
 &gpio {
-	xirq4 {
+	xirq4-hog {
 		gpio-hog;
 		gpios = <UNIPHIER_GPIO_IRQ(4) 0>;
 		input;
@@ -137,6 +137,14 @@
 	};
 };
 
+&ahci0 {
+	status = "okay";
+};
+
+&ahci1 {
+	status = "okay";
+};
+
 &pinctrl_ether_rgmii {
 	tx {
 		pins = "RGMII0_TXCLK", "RGMII0_TXD0", "RGMII0_TXD1",
diff --git a/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi b/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi
index ba75adedbf79..b0c29510a7da 100644
--- a/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi
+++ b/arch/arm64/boot/dts/socionext/uniphier-pxs3.dtsi
@@ -7,6 +7,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/gpio/uniphier-gpio.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/thermal/thermal.h>
 
 / {
@@ -42,6 +43,7 @@
 			reg = <0 0x000>;
 			clocks = <&sys_clk 33>;
 			enable-method = "psci";
+			next-level-cache = <&l2>;
 			operating-points-v2 = <&cluster0_opp>;
 			#cooling-cells = <2>;
 		};
@@ -52,6 +54,7 @@
 			reg = <0 0x001>;
 			clocks = <&sys_clk 33>;
 			enable-method = "psci";
+			next-level-cache = <&l2>;
 			operating-points-v2 = <&cluster0_opp>;
 			#cooling-cells = <2>;
 		};
@@ -62,6 +65,7 @@
 			reg = <0 0x002>;
 			clocks = <&sys_clk 33>;
 			enable-method = "psci";
+			next-level-cache = <&l2>;
 			operating-points-v2 = <&cluster0_opp>;
 			#cooling-cells = <2>;
 		};
@@ -72,9 +76,14 @@
 			reg = <0 0x003>;
 			clocks = <&sys_clk 33>;
 			enable-method = "psci";
+			next-level-cache = <&l2>;
 			operating-points-v2 = <&cluster0_opp>;
 			#cooling-cells = <2>;
 		};
+
+		l2: l2-cache {
+			compatible = "cache";
+		};
 	};
 
 	cluster0_opp: opp-table {
@@ -135,10 +144,10 @@
 
 	timer {
 		compatible = "arm,armv8-timer";
-		interrupts = <1 13 4>,
-			     <1 14 4>,
-			     <1 11 4>,
-			     <1 10 4>;
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
 	thermal-zones {
@@ -195,7 +204,7 @@
 			reg = <0x54006000 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 39 4>;
+			interrupts = <GIC_SPI 39 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi0>;
 			clocks = <&peri_clk 11>;
@@ -208,7 +217,7 @@
 			reg = <0x54006100 0x100>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 216 4>;
+			interrupts = <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_spi1>;
 			clocks = <&peri_clk 12>;
@@ -219,7 +228,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006800 0x40>;
-			interrupts = <0 33 4>;
+			interrupts = <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart0>;
 			clocks = <&peri_clk 0>;
@@ -230,7 +239,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006900 0x40>;
-			interrupts = <0 35 4>;
+			interrupts = <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart1>;
 			clocks = <&peri_clk 1>;
@@ -241,7 +250,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006a00 0x40>;
-			interrupts = <0 37 4>;
+			interrupts = <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart2>;
 			clocks = <&peri_clk 2>;
@@ -252,7 +261,7 @@
 			compatible = "socionext,uniphier-uart";
 			status = "disabled";
 			reg = <0x54006b00 0x40>;
-			interrupts = <0 177 4>;
+			interrupts = <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_uart3>;
 			clocks = <&peri_clk 3>;
@@ -284,7 +293,7 @@
 			reg = <0x58780000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 41 4>;
+			interrupts = <GIC_SPI 41 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c0>;
 			clocks = <&peri_clk 4>;
@@ -298,7 +307,7 @@
 			reg = <0x58781000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 42 4>;
+			interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c1>;
 			clocks = <&peri_clk 5>;
@@ -312,7 +321,7 @@
 			reg = <0x58782000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 43 4>;
+			interrupts = <GIC_SPI 43 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c2>;
 			clocks = <&peri_clk 6>;
@@ -326,7 +335,7 @@
 			reg = <0x58783000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 44 4>;
+			interrupts = <GIC_SPI 44 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_i2c3>;
 			clocks = <&peri_clk 7>;
@@ -340,7 +349,7 @@
 			reg = <0x58786000 0x80>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 26 4>;
+			interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&peri_clk 10>;
 			resets = <&peri_rst 10>;
 			clock-frequency = <400000>;
@@ -396,7 +405,7 @@
 		emmc: mmc@5a000000 {
 			compatible = "socionext,uniphier-sd4hc", "cdns,sd4hc";
 			reg = <0x5a000000 0x400>;
-			interrupts = <0 78 4>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_emmc>;
 			clocks = <&sys_clk 4>;
@@ -416,7 +425,7 @@
 			compatible = "socionext,uniphier-sd-v3.1.1";
 			status = "disabled";
 			reg = <0x5a400000 0x800>;
-			interrupts = <0 76 4>;
+			interrupts = <GIC_SPI 76 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default", "uhs";
 			pinctrl-0 = <&pinctrl_sd>;
 			pinctrl-1 = <&pinctrl_sd_uhs>;
@@ -505,7 +514,7 @@
 		xdmac: dma-controller@5fc10000 {
 			compatible = "socionext,uniphier-xdmac";
 			reg = <0x5fc10000 0x5300>;
-			interrupts = <0 188 4>;
+			interrupts = <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>;
 			dma-channels = <16>;
 			#dma-cells = <2>;
 		};
@@ -523,7 +532,7 @@
 			      <0x5fe80000 0x80000>;	/* GICR */
 			interrupt-controller;
 			#interrupt-cells = <3>;
-			interrupts = <1 9 4>;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
 		sysctrl@61840000 {
@@ -545,9 +554,9 @@
 				compatible = "socionext,uniphier-wdt";
 			};
 
-			pvtctl: pvtctl {
+			pvtctl: thermal-sensor {
 				compatible = "socionext,uniphier-pxs3-thermal";
-				interrupts = <0 3 4>;
+				interrupts = <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>;
 				#thermal-sensor-cells = <0>;
 				socionext,tmod-calibration = <0x0f22 0x68ee>;
 			};
@@ -557,7 +566,7 @@
 			compatible = "socionext,uniphier-pxs3-ave4";
 			status = "disabled";
 			reg = <0x65000000 0x8500>;
-			interrupts = <0 66 4>;
+			interrupts = <GIC_SPI 66 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ether_rgmii>;
 			clock-names = "ether";
@@ -578,7 +587,7 @@
 			compatible = "socionext,uniphier-pxs3-ave4";
 			status = "disabled";
 			reg = <0x65200000 0x8500>;
-			interrupts = <0 67 4>;
+			interrupts = <GIC_SPI 67 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_ether1_rgmii>;
 			clock-names = "ether";
@@ -595,12 +604,92 @@
 			};
 		};
 
+		ahci0: sata@65600000 {
+			compatible = "socionext,uniphier-pxs3-ahci",
+				     "generic-ahci";
+			status = "disabled";
+			reg = <0x65600000 0x10000>;
+			interrupts = <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sys_clk 28>;
+			resets = <&sys_rst 28>, <&ahci0_rst 0>;
+			ports-implemented = <1>;
+			phys = <&ahci0_phy>;
+		};
+
+		sata-controller@65700000 {
+			compatible = "socionext,uniphier-pxs3-ahci-glue",
+				     "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x65700000 0x100>;
+
+			ahci0_rst: reset-controller@0 {
+				compatible = "socionext,uniphier-pxs3-ahci-reset";
+				reg = <0x0 0x4>;
+				clock-names = "link";
+				clocks = <&sys_clk 28>;
+				reset-names = "link";
+				resets = <&sys_rst 28>;
+				#reset-cells = <1>;
+			};
+
+			ahci0_phy: sata-phy@10 {
+				compatible = "socionext,uniphier-pxs3-ahci-phy";
+				reg = <0x10 0x10>;
+				clock-names = "link", "phy";
+				clocks = <&sys_clk 28>, <&sys_clk 30>;
+				reset-names = "link", "phy";
+				resets = <&sys_rst 28>, <&sys_rst 30>;
+				#phy-cells = <0>;
+			};
+		};
+
+		ahci1: sata@65800000 {
+			compatible = "socionext,uniphier-pxs3-ahci",
+				     "generic-ahci";
+			status = "disabled";
+			reg = <0x65800000 0x10000>;
+			interrupts = <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&sys_clk 29>;
+			resets = <&sys_rst 29>, <&ahci1_rst 0>;
+			ports-implemented = <1>;
+			phys = <&ahci1_phy>;
+		};
+
+		sata-controller@65900000 {
+			compatible = "socionext,uniphier-pxs3-ahci-glue",
+				     "simple-mfd";
+			#address-cells = <1>;
+			#size-cells = <1>;
+			ranges = <0 0x65900000 0x100>;
+
+			ahci1_rst: reset-controller@0 {
+				compatible = "socionext,uniphier-pxs3-ahci-reset";
+				reg = <0x0 0x4>;
+				clock-names = "link";
+				clocks = <&sys_clk 29>;
+				reset-names = "link";
+				resets = <&sys_rst 29>;
+				#reset-cells = <1>;
+			};
+
+			ahci1_phy: sata-phy@10 {
+				compatible = "socionext,uniphier-pxs3-ahci-phy";
+				reg = <0x10 0x10>;
+				clock-names = "link", "phy";
+				clocks = <&sys_clk 29>, <&sys_clk 30>;
+				reset-names = "link", "phy";
+				resets = <&sys_rst 29>, <&sys_rst 30>;
+				#phy-cells = <0>;
+			};
+		};
+
 		usb0: usb@65a00000 {
 			compatible = "socionext,uniphier-dwc3", "snps,dwc3";
 			status = "disabled";
 			reg = <0x65a00000 0xcd00>;
 			interrupt-names = "dwc_usb3";
-			interrupts = <0 134 4>;
+			interrupts = <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb0>, <&pinctrl_usb2>;
 			clock-names = "ref", "bus_early", "suspend";
@@ -611,7 +700,7 @@
 			dr_mode = "host";
 		};
 
-		usb-glue@65b00000 {
+		usb-controller@65b00000 {
 			compatible = "socionext,uniphier-pxs3-dwc3-glue",
 				     "simple-mfd";
 			#address-cells = <1>;
@@ -702,7 +791,7 @@
 			status = "disabled";
 			reg = <0x65c00000 0xcd00>;
 			interrupt-names = "dwc_usb3";
-			interrupts = <0 137 4>;
+			interrupts = <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_usb1>, <&pinctrl_usb3>;
 			clock-names = "ref", "bus_early", "suspend";
@@ -713,7 +802,7 @@
 			dr_mode = "host";
 		};
 
-		usb-glue@65d00000 {
+		usb-controller@65d00000 {
 			compatible = "socionext,uniphier-pxs3-dwc3-glue",
 				     "simple-mfd";
 			#address-cells = <1>;
@@ -792,7 +881,7 @@
 		};
 
 		pcie: pcie@66000000 {
-			compatible = "socionext,uniphier-pcie", "snps,dw-pcie";
+			compatible = "socionext,uniphier-pcie";
 			status = "disabled";
 			reg-names = "dbi", "link", "config";
 			reg = <0x66000000 0x1000>, <0x66010000 0x10000>,
@@ -812,7 +901,8 @@
 				<0x82000000 0 0x20000000 0x20000000 0 0x0ffe0000>;
 			#interrupt-cells = <1>;
 			interrupt-names = "dma", "msi";
-			interrupts = <0 224 4>, <0 225 4>;
+			interrupts = <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-map-mask = <0 0 0 7>;
 			interrupt-map = <0 0 0 1 &pcie_intc 0>,	/* INTA */
 					<0 0 0 2 &pcie_intc 1>,	/* INTB */
@@ -825,7 +915,7 @@
 				interrupt-controller;
 				#interrupt-cells = <1>;
 				interrupt-parent = <&gic>;
-				interrupts = <0 226 4>;
+				interrupts = <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>;
 			};
 		};
 
@@ -847,7 +937,7 @@
 			reg = <0x68000000 0x20>, <0x68100000 0x1000>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			interrupts = <0 65 4>;
+			interrupts = <GIC_SPI 65 IRQ_TYPE_LEVEL_HIGH>;
 			pinctrl-names = "default";
 			pinctrl-0 = <&pinctrl_nand>;
 			clock-names = "nand", "nand_x", "ecc";
diff --git a/arch/arm64/boot/dts/ti/Makefile b/arch/arm64/boot/dts/ti/Makefile
index 02e5d80344d0..4555a5be2257 100644
--- a/arch/arm64/boot/dts/ti/Makefile
+++ b/arch/arm64/boot/dts/ti/Makefile
@@ -23,3 +23,5 @@ dtb-$(CONFIG_ARCH_K3) += k3-am642-evm.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am642-sk.dtb
 
 dtb-$(CONFIG_ARCH_K3) += k3-am625-sk.dtb
+
+dtb-$(CONFIG_ARCH_K3) += k3-am62a7-sk.dtb
diff --git a/arch/arm64/boot/dts/ti/k3-am62-main.dtsi b/arch/arm64/boot/dts/ti/k3-am62-main.dtsi
index 12ab7548dc77..03660476364f 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-main.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am62-main.dtsi
@@ -54,6 +54,12 @@
 			reg = <0x4044 0x8>;
 			#phy-cells = <1>;
 		};
+
+		epwm_tbclk: clock@4130 {
+			compatible = "ti,am62-epwm-tbclk", "syscon";
+			reg = <0x4130 0x4>;
+			#clock-cells = <1>;
+		};
 	};
 
 	dmss: bus@48000000 {
@@ -584,4 +590,31 @@
 		interrupt-names = "int0", "int1";
 		bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>;
 	};
+
+	epwm0: pwm@23000000 {
+		compatible = "ti,am64-epwm", "ti,am3352-ehrpwm";
+		#pwm-cells = <3>;
+		reg = <0x00 0x23000000 0x00 0x100>;
+		power-domains = <&k3_pds 86 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&epwm_tbclk 0>, <&k3_clks 86 0>;
+		clock-names = "tbclk", "fck";
+	};
+
+	epwm1: pwm@23010000 {
+		compatible = "ti,am64-epwm", "ti,am3352-ehrpwm";
+		#pwm-cells = <3>;
+		reg = <0x00 0x23010000 0x00 0x100>;
+		power-domains = <&k3_pds 87 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&epwm_tbclk 1>, <&k3_clks 87 0>;
+		clock-names = "tbclk", "fck";
+	};
+
+	epwm2: pwm@23020000 {
+		compatible = "ti,am64-epwm", "ti,am3352-ehrpwm";
+		#pwm-cells = <3>;
+		reg = <0x00 0x23020000 0x00 0x100>;
+		power-domains = <&k3_pds 88 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&epwm_tbclk 2>, <&k3_clks 88 0>;
+		clock-names = "tbclk", "fck";
+	};
 };
diff --git a/arch/arm64/boot/dts/ti/k3-am625-sk.dts b/arch/arm64/boot/dts/ti/k3-am625-sk.dts
index 9b4dbae9d4aa..93a5f0817efc 100644
--- a/arch/arm64/boot/dts/ti/k3-am625-sk.dts
+++ b/arch/arm64/boot/dts/ti/k3-am625-sk.dts
@@ -502,3 +502,15 @@
 &main_mcan0 {
 	status = "disabled";
 };
+
+&epwm0 {
+	status = "disabled";
+};
+
+&epwm1 {
+	status = "disabled";
+};
+
+&epwm2 {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi
new file mode 100644
index 000000000000..bc4b50bcd177
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am62a-main.dtsi
@@ -0,0 +1,298 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for AM62A SoC Family Main Domain peripherals
+ *
+ * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/
+ */
+
+&cbass_main {
+	oc_sram: sram@70000000 {
+		compatible = "mmio-sram";
+		reg = <0x00 0x70000000 0x00 0x10000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x0 0x00 0x70000000 0x10000>;
+	};
+
+	gic500: interrupt-controller@1800000 {
+		compatible = "arm,gic-v3";
+		reg = <0x00 0x01800000 0x00 0x10000>,	/* GICD */
+		      <0x00 0x01880000 0x00 0xc0000>,	/* GICR */
+		      <0x00 0x01880000 0x00 0xc0000>,   /* GICR */
+		      <0x01 0x00000000 0x00 0x2000>,    /* GICC */
+		      <0x01 0x00010000 0x00 0x1000>,    /* GICH */
+		      <0x01 0x00020000 0x00 0x2000>;    /* GICV */
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+		#interrupt-cells = <3>;
+		interrupt-controller;
+		/*
+		 * vcpumntirq:
+		 * virtual CPU interface maintenance interrupt
+		 */
+		interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH>;
+
+		gic_its: msi-controller@1820000 {
+			compatible = "arm,gic-v3-its";
+			reg = <0x00 0x01820000 0x00 0x10000>;
+			socionext,synquacer-pre-its = <0x1000000 0x400000>;
+			msi-controller;
+			#msi-cells = <1>;
+		};
+	};
+
+	main_conf: syscon@100000 {
+		compatible = "ti,j721e-system-controller", "syscon", "simple-mfd";
+		reg = <0x00 0x00100000 0x00 0x20000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x00 0x00 0x00100000 0x20000>;
+	};
+
+	dmss: bus@48000000 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-ranges;
+		ranges = <0x00 0x48000000 0x00 0x48000000 0x00 0x06000000>;
+
+		ti,sci-dev-id = <25>;
+
+		secure_proxy_main: mailbox@4d000000 {
+			compatible = "ti,am654-secure-proxy";
+			reg = <0x00 0x4d000000 0x00 0x80000>,
+			      <0x00 0x4a600000 0x00 0x80000>,
+			      <0x00 0x4a400000 0x00 0x80000>;
+			reg-names = "target_data", "rt", "scfg";
+			#mbox-cells = <1>;
+			interrupt-names = "rx_012";
+			interrupts = <GIC_SPI 34 IRQ_TYPE_LEVEL_HIGH>;
+		};
+	};
+
+	dmsc: system-controller@44043000 {
+		compatible = "ti,k2g-sci";
+		reg = <0x00 0x44043000 0x00 0xfe0>;
+		reg-names = "debug_messages";
+		ti,host-id = <12>;
+		mbox-names = "rx", "tx";
+		mboxes= <&secure_proxy_main 12>,
+			<&secure_proxy_main 13>;
+
+		k3_pds: power-controller {
+			compatible = "ti,sci-pm-domain";
+			#power-domain-cells = <2>;
+		};
+
+		k3_clks: clock-controller {
+			compatible = "ti,k2g-sci-clk";
+			#clock-cells = <2>;
+		};
+
+		k3_reset: reset-controller {
+			compatible = "ti,sci-reset";
+			#reset-cells = <2>;
+		};
+	};
+
+	main_pmx0: pinctrl@f4000 {
+		compatible = "pinctrl-single";
+		reg = <0x00 0xf4000 0x00 0x2ac>;
+		#pinctrl-cells = <1>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0xffffffff>;
+	};
+
+	main_uart0: serial@2800000 {
+		compatible = "ti,am64-uart", "ti,am654-uart";
+		reg = <0x00 0x02800000 0x00 0x100>;
+		interrupts = <GIC_SPI 178 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 146 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 146 0>;
+		clock-names = "fclk";
+		status = "disabled";
+	};
+
+	main_uart1: serial@2810000 {
+		compatible = "ti,am64-uart", "ti,am654-uart";
+		reg = <0x00 0x02810000 0x00 0x100>;
+		interrupts = <GIC_SPI 179 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 152 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 152 0>;
+		clock-names = "fclk";
+		status = "disabled";
+	};
+
+	main_uart2: serial@2820000 {
+		compatible = "ti,am64-uart", "ti,am654-uart";
+		reg = <0x00 0x02820000 0x00 0x100>;
+		interrupts = <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 153 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 153 0>;
+		clock-names = "fclk";
+		status = "disabled";
+	};
+
+	main_uart3: serial@2830000 {
+		compatible = "ti,am64-uart", "ti,am654-uart";
+		reg = <0x00 0x02830000 0x00 0x100>;
+		interrupts = <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 154 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 154 0>;
+		clock-names = "fclk";
+		status = "disabled";
+	};
+
+	main_uart4: serial@2840000 {
+		compatible = "ti,am64-uart", "ti,am654-uart";
+		reg = <0x00 0x02840000 0x00 0x100>;
+		interrupts = <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 155 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 155 0>;
+		clock-names = "fclk";
+		status = "disabled";
+	};
+
+	main_uart5: serial@2850000 {
+		compatible = "ti,am64-uart", "ti,am654-uart";
+		reg = <0x00 0x02850000 0x00 0x100>;
+		interrupts = <GIC_SPI 183 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 156 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 156 0>;
+		clock-names = "fclk";
+		status = "disabled";
+	};
+
+	main_uart6: serial@2860000 {
+		compatible = "ti,am64-uart", "ti,am654-uart";
+		reg = <0x00 0x02860000 0x00 0x100>;
+		interrupts = <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 158 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 158 0>;
+		clock-names = "fclk";
+		status = "disabled";
+	};
+
+	main_i2c0: i2c@20000000 {
+		compatible = "ti,am64-i2c", "ti,omap4-i2c";
+		reg = <0x00 0x20000000 0x00 0x100>;
+		interrupts = <GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		power-domains = <&k3_pds 102 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 102 2>;
+		clock-names = "fck";
+		status = "disabled";
+	};
+
+	main_i2c1: i2c@20010000 {
+		compatible = "ti,am64-i2c", "ti,omap4-i2c";
+		reg = <0x00 0x20010000 0x00 0x100>;
+		interrupts = <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		power-domains = <&k3_pds 103 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 103 2>;
+		clock-names = "fck";
+		status = "disabled";
+	};
+
+	main_i2c2: i2c@20020000 {
+		compatible = "ti,am64-i2c", "ti,omap4-i2c";
+		reg = <0x00 0x20020000 0x00 0x100>;
+		interrupts = <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		power-domains = <&k3_pds 104 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 104 2>;
+		clock-names = "fck";
+		status = "disabled";
+	};
+
+	main_i2c3: i2c@20030000 {
+		compatible = "ti,am64-i2c", "ti,omap4-i2c";
+		reg = <0x00 0x20030000 0x00 0x100>;
+		interrupts = <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		power-domains = <&k3_pds 105 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 105 2>;
+		clock-names = "fck";
+		status = "disabled";
+	};
+
+	main_gpio_intr: interrupt-controller@a00000 {
+		compatible = "ti,sci-intr";
+		reg = <0x00 0x00a00000 0x00 0x800>;
+		ti,intr-trigger-type = <1>;
+		interrupt-controller;
+		interrupt-parent = <&gic500>;
+		#interrupt-cells = <1>;
+		ti,sci = <&dmsc>;
+		ti,sci-dev-id = <3>;
+		ti,interrupt-ranges = <0 32 16>;
+		status = "disabled";
+	};
+
+	main_gpio0: gpio@600000 {
+		compatible = "ti,am64-gpio", "ti,keystone-gpio";
+		reg = <0x00 0x00600000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&main_gpio_intr>;
+		interrupts = <190>, <191>, <192>,
+			     <193>, <194>, <195>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <87>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 77 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 77 0>;
+		clock-names = "gpio";
+		status = "disabled";
+	};
+
+	main_gpio1: gpio@601000 {
+		compatible = "ti,am64-gpio", "ti,keystone-gpio";
+		reg = <0x00 0x00601000 0x0 0x100>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&main_gpio_intr>;
+		interrupts = <180>, <181>, <182>,
+			     <183>, <184>, <185>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		ti,ngpio = <88>;
+		ti,davinci-gpio-unbanked = <0>;
+		power-domains = <&k3_pds 78 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 78 0>;
+		clock-names = "gpio";
+		status = "disabled";
+	};
+
+	sdhci1: mmc@fa00000 {
+		compatible = "ti,am62-sdhci";
+		reg = <0x00 0xfa00000 0x00 0x260>, <0x00 0xfa08000 0x00 0x134>;
+		interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 58 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 58 5>, <&k3_clks 58 6>;
+		clock-names = "clk_ahb", "clk_xin";
+		ti,trm-icp = <0x2>;
+		ti,otap-del-sel-legacy = <0x0>;
+		ti,otap-del-sel-sd-hs = <0x0>;
+		ti,otap-del-sel-sdr12 = <0xf>;
+		ti,otap-del-sel-sdr25 = <0xf>;
+		ti,otap-del-sel-sdr50 = <0xc>;
+		ti,otap-del-sel-sdr104 = <0x6>;
+		ti,otap-del-sel-ddr50 = <0x9>;
+		ti,itap-del-sel-legacy = <0x0>;
+		ti,itap-del-sel-sd-hs = <0x0>;
+		ti,itap-del-sel-sdr12 = <0x0>;
+		ti,itap-del-sel-sdr25 = <0x0>;
+		ti,clkbuf-sel = <0x7>;
+		bus-width = <4>;
+		no-1-8-v;
+		status = "disabled";
+	};
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi
new file mode 100644
index 000000000000..6d1e501b94ab
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am62a-mcu.dtsi
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for AM625 SoC Family MCU Domain peripherals
+ *
+ * Copyright (C) 2020-2022 Texas Instruments Incorporated - https://www.ti.com/
+ */
+
+&cbass_mcu {
+	mcu_pmx0: pinctrl@4084000 {
+		compatible = "pinctrl-single";
+		reg = <0x00 0x04084000 0x00 0x88>;
+		#pinctrl-cells = <1>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0xffffffff>;
+		status = "disabled";
+	};
+
+	mcu_uart0: serial@4a00000 {
+		compatible = "ti,am64-uart", "ti,am654-uart";
+		reg = <0x00 0x04a00000 0x00 0x100>;
+		interrupts = <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 149 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 149 0>;
+		clock-names = "fclk";
+		status = "disabled";
+	};
+
+	mcu_i2c0: i2c@4900000 {
+		compatible = "ti,am64-i2c", "ti,omap4-i2c";
+		reg = <0x00 0x04900000 0x00 0x100>;
+		interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		power-domains = <&k3_pds 106 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 106 2>;
+		clock-names = "fck";
+		status = "disabled";
+	};
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am62a-wakeup.dtsi b/arch/arm64/boot/dts/ti/k3-am62a-wakeup.dtsi
new file mode 100644
index 000000000000..99afac40e8d4
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am62a-wakeup.dtsi
@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for AM62A SoC Family Wakeup Domain peripherals
+ *
+ * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/
+ */
+
+&cbass_wakeup {
+	wkup_conf: syscon@43000000 {
+		compatible = "ti,j721e-system-controller", "syscon", "simple-mfd";
+		reg = <0x00 0x43000000 0x00 0x20000>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0x00 0x00 0x43000000 0x20000>;
+
+		chipid: chipid@14 {
+			compatible = "ti,am654-chipid";
+			reg = <0x14 0x4>;
+		};
+	};
+
+	wkup_uart0: serial@2b300000 {
+		compatible = "ti,am64-uart", "ti,am654-uart";
+		reg = <0x00 0x2b300000 0x00 0x100>;
+		interrupts = <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 114 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 114 0>;
+		clock-names = "fclk";
+		status = "disabled";
+	};
+
+	wkup_i2c0: i2c@2b200000 {
+		compatible = "ti,am64-i2c", "ti,omap4-i2c";
+		reg = <0x00 0x02b200000 0x00 0x100>;
+		interrupts = <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+		power-domains = <&k3_pds 107 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 107 4>;
+		clock-names = "fck";
+		status = "disabled";
+	};
+
+	wkup_rtc0: rtc@2b1f0000 {
+		compatible = "ti,am62-rtc";
+		reg = <0x00 0x2b1f0000 0x00 0x100>;
+		interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>;
+		clocks = <&k3_clks 117 6> , <&k3_clks 117 0>;
+		clock-names = "vbus", "osc32k";
+		power-domains = <&k3_pds 117 TI_SCI_PD_EXCLUSIVE>;
+		wakeup-source;
+		status = "disabled";
+	};
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am62a.dtsi b/arch/arm64/boot/dts/ti/k3-am62a.dtsi
new file mode 100644
index 000000000000..6eb87c3f9f3c
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am62a.dtsi
@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for AM62A SoC Family
+ *
+ * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/pinctrl/k3.h>
+#include <dt-bindings/soc/ti,sci_pm_domain.h>
+
+/ {
+	model = "Texas Instruments K3 AM62A SoC";
+	compatible = "ti,am62a7";
+	interrupt-parent = <&gic500>;
+	#address-cells = <2>;
+	#size-cells = <2>;
+
+	chosen { };
+
+	firmware {
+		optee {
+			compatible = "linaro,optee-tz";
+			method = "smc";
+		};
+
+		psci: psci {
+			compatible = "arm,psci-1.0";
+			method = "smc";
+		};
+	};
+
+	a53_timer0: timer-cl0-cpu0 {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>, /* cntpsirq */
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>, /* cntpnsirq */
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>, /* cntvirq */
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>; /* cnthpirq */
+	};
+
+	pmu: pmu {
+		compatible = "arm,cortex-a53-pmu";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_HIGH>;
+	};
+
+	cbass_main: bus@f0000 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		ranges = <0x00 0x000f0000 0x00 0x000f0000 0x00 0x00030000>, /* Main MMRs */
+			 <0x00 0x00420000 0x00 0x00420000 0x00 0x00001000>, /* ESM0 */
+			 <0x00 0x00600000 0x00 0x00600000 0x00 0x00001100>, /* GPIO */
+			 <0x00 0x00703000 0x00 0x00703000 0x00 0x00000200>, /* USB0 debug trace */
+			 <0x00 0x0070c000 0x00 0x0070c000 0x00 0x00000200>, /* USB1 debug trace */
+			 <0x00 0x00a40000 0x00 0x00a40000 0x00 0x00000800>, /* Timesync router */
+			 <0x00 0x01000000 0x00 0x01000000 0x00 0x01b28400>, /* First peripheral window */
+			 <0x00 0x08000000 0x00 0x08000000 0x00 0x00200000>, /* Main CPSW */
+			 <0x00 0x0e000000 0x00 0x0e000000 0x00 0x01d20000>, /* Second peripheral window */
+			 <0x00 0x0fd00000 0x00 0x0fd00000 0x00 0x00020000>, /* GPU */
+			 <0x00 0x20000000 0x00 0x20000000 0x00 0x0a008000>, /* Third peripheral window */
+			 <0x00 0x30040000 0x00 0x30040000 0x00 0x00080000>, /* PRUSS-M */
+			 <0x00 0x30101000 0x00 0x30101000 0x00 0x00010100>, /* CSI window */
+			 <0x00 0x30200000 0x00 0x30200000 0x00 0x00010000>, /* DSS */
+			 <0x00 0x30210000 0x00 0x30210000 0x00 0x00010000>, /* VPU */
+			 <0x00 0x31000000 0x00 0x31000000 0x00 0x00050000>, /* USB0 DWC3 Core window */
+			 <0x00 0x31100000 0x00 0x31100000 0x00 0x00050000>, /* USB1 DWC3 Core window */
+			 <0x00 0x40900000 0x00 0x40900000 0x00 0x00030000>, /* SA3UL */
+			 <0x00 0x43600000 0x00 0x43600000 0x00 0x00010000>, /* SA3 sproxy data */
+			 <0x00 0x44043000 0x00 0x44043000 0x00 0x00000fe0>, /* TI SCI DEBUG */
+			 <0x00 0x44860000 0x00 0x44860000 0x00 0x00040000>, /* SA3 sproxy config */
+			 <0x00 0x48000000 0x00 0x48000000 0x00 0x06400000>, /* DMSS */
+			 <0x00 0x60000000 0x00 0x60000000 0x00 0x08000000>, /* FSS0 DAT1 */
+			 <0x00 0x70000000 0x00 0x70000000 0x00 0x00010000>, /* OCSRAM */
+			 <0x00 0x7e000000 0x00 0x7e000000 0x00 0x00100000>, /* C7x_0 */
+			 <0x01 0x00000000 0x01 0x00000000 0x00 0x00310000>, /* A53 PERIPHBASE */
+			 <0x05 0x00000000 0x05 0x00000000 0x01 0x00000000>, /* FSS0 DAT3 */
+
+			 /* MCU Domain Range */
+			 <0x00 0x04000000 0x00 0x04000000 0x00 0x01ff1400>,
+			 <0x00 0x79000000 0x00 0x79000000 0x00 0x00008000>, /* MCU R5 ATCM */
+			 <0x00 0x79020000 0x00 0x79020000 0x00 0x00008000>, /* MCU R5 BTCM */
+			 <0x00 0x79100000 0x00 0x79100000 0x00 0x00040000>, /* MCU R5 IRAM0 */
+			 <0x00 0x79140000 0x00 0x79140000 0x00 0x00040000>, /* MCU R5 IRAM1 */
+
+			 /* Wakeup Domain Range */
+			 <0x00 0x00b00000 0x00 0x00b00000 0x00 0x00002400>,
+			 <0x00 0x2b000000 0x00 0x2b000000 0x00 0x00300400>,
+			 <0x00 0x43000000 0x00 0x43000000 0x00 0x00020000>,
+			 <0x00 0x78000000 0x00 0x78000000 0x00 0x00008000>, /* DM R5 ATCM */
+			 <0x00 0x78100000 0x00 0x78100000 0x00 0x00008000>; /* DM R5 BTCM */
+
+		cbass_mcu: bus@4000000 {
+			compatible = "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x00 0x04000000 0x00 0x04000000 0x00 0x01ff1400>, /* Peripheral window */
+				 <0x00 0x79000000 0x00 0x79000000 0x00 0x00008000>, /* MCU R5 ATCM */
+				 <0x00 0x79020000 0x00 0x79020000 0x00 0x00008000>, /* MCU R5 BTCM */
+				 <0x00 0x79100000 0x00 0x79100000 0x00 0x00040000>, /* MCU IRAM0 */
+				 <0x00 0x79140000 0x00 0x79140000 0x00 0x00040000>; /* MCU IRAM1 */
+		};
+
+		cbass_wakeup: bus@b00000 {
+			compatible = "simple-bus";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges = <0x00 0x00b00000 0x00 0x00b00000 0x00 0x00002400>, /* VTM */
+				 <0x00 0x2b000000 0x00 0x2b000000 0x00 0x00300400>, /* Peripheral Window */
+				 <0x00 0x43000000 0x00 0x43000000 0x00 0x00020000>, /* WKUP CTRL MMR */
+				 <0x00 0x78000000 0x00 0x78000000 0x00 0x00008000>, /* DM R5 ATCM*/
+				 <0x00 0x78100000 0x00 0x78100000 0x00 0x00008000>; /* DM R5 BTCM*/
+		};
+	};
+};
+
+/* Now include the peripherals for each bus segments */
+#include "k3-am62a-main.dtsi"
+#include "k3-am62a-mcu.dtsi"
+#include "k3-am62a-wakeup.dtsi"
diff --git a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
new file mode 100644
index 000000000000..576dbce80ad8
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
@@ -0,0 +1,223 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AM62A SK: https://www.ti.com/lit/zip/sprr459
+ *
+ * Copyright (C) 2022 Texas Instruments Incorporated - https://www.ti.com/
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/gpio/gpio.h>
+#include "k3-am62a7.dtsi"
+
+/ {
+	compatible =  "ti,am62a7-sk", "ti,am62a7";
+	model = "Texas Instruments AM62A7 SK";
+
+	aliases {
+		serial2 = &main_uart0;
+		mmc1 = &sdhci1;
+	};
+
+	chosen {
+		stdout-path = "serial2:115200n8";
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		/* 2G RAM */
+		reg = <0x00000000 0x80000000 0x00000000 0x80000000>;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		secure_tfa_ddr: tfa@9e780000 {
+			reg = <0x00 0x9e780000 0x00 0x80000>;
+			alignment = <0x1000>;
+			no-map;
+		};
+
+		secure_ddr: optee@9e800000 {
+			reg = <0x00 0x9e800000 0x00 0x01800000>; /* for OP-TEE */
+			alignment = <0x1000>;
+			no-map;
+		};
+
+		wkup_r5fss0_core0_memory_region: r5f-dma-memory@9c900000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0x9c900000 0x00 0x01e00000>;
+			no-map;
+		};
+	};
+
+	vmain_pd: regulator-0 {
+		/* TPS25750 PD CONTROLLER OUTPUT */
+		compatible = "regulator-fixed";
+		regulator-name = "vmain_pd";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc_5v0: regulator-1 {
+		/* Output of TPS63070 */
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vmain_pd>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vcc_3v3_sys: regulator-2 {
+		/* output of LM5141-Q1 */
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_3v3_sys";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vmain_pd>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	vdd_mmc1: regulator-3 {
+		/* TPS22918DBVR */
+		compatible = "regulator-fixed";
+		regulator-name = "vdd_mmc1";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		enable-active-high;
+		gpio = <&exp1 3 GPIO_ACTIVE_HIGH>;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&usr_led_pins_default>;
+
+		led-0 {
+			label = "am62a-sk:green:heartbeat";
+			gpios = <&main_gpio1 49 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+			function = LED_FUNCTION_HEARTBEAT;
+			default-state = "off";
+		};
+	};
+};
+
+&main_pmx0 {
+	main_uart0_pins_default: main-uart0-pins-default {
+		pinctrl-single,pins = <
+			AM62AX_IOPAD(0x1c8, PIN_INPUT, 0) /* (D14) UART0_RXD */
+			AM62AX_IOPAD(0x1cc, PIN_OUTPUT, 0) /* (E14) UART0_TXD */
+		>;
+	};
+
+	main_i2c0_pins_default: main-i2c0-pins-default {
+		pinctrl-single,pins = <
+			AM62AX_IOPAD(0x1e0, PIN_INPUT_PULLUP, 0) /* (B16) I2C0_SCL */
+			AM62AX_IOPAD(0x1e4, PIN_INPUT_PULLUP, 0) /* (A16) I2C0_SDA */
+		>;
+	};
+
+	main_i2c1_pins_default: main-i2c1-pins-default {
+		pinctrl-single,pins = <
+			AM62AX_IOPAD(0x1e8, PIN_INPUT_PULLUP, 0) /* (B17) I2C1_SCL */
+			AM62AX_IOPAD(0x1ec, PIN_INPUT_PULLUP, 0) /* (A17) I2C1_SDA */
+		>;
+	};
+
+	main_i2c2_pins_default: main-i2c2-pins-default {
+		pinctrl-single,pins = <
+			AM62AX_IOPAD(0x0b0, PIN_INPUT_PULLUP, 1) /* (K22) GPMC0_CSn2.I2C2_SCL */
+			AM62AX_IOPAD(0x0b4, PIN_INPUT_PULLUP, 1) /* (K24) GPMC0_CSn3.I2C2_SDA */
+		>;
+	};
+
+	main_mmc1_pins_default: main-mmc1-pins-default {
+		pinctrl-single,pins = <
+			AM62AX_IOPAD(0x23c, PIN_INPUT, 0) /* (A21) MMC1_CMD */
+			AM62AX_IOPAD(0x234, PIN_INPUT, 0) /* (B22) MMC1_CLK */
+			AM62AX_IOPAD(0x230, PIN_INPUT, 0) /* (A22) MMC1_DAT0 */
+			AM62AX_IOPAD(0x22c, PIN_INPUT, 0) /* (B21) MMC1_DAT1 */
+			AM62AX_IOPAD(0x228, PIN_INPUT, 0) /* (C21) MMC1_DAT2 */
+			AM62AX_IOPAD(0x224, PIN_INPUT, 0) /* (D22) MMC1_DAT3 */
+			AM62AX_IOPAD(0x240, PIN_INPUT, 0) /* (D17) MMC1_SDCD */
+		>;
+	};
+
+	usr_led_pins_default: usr-led-pins-default {
+		pinctrl-single,pins = <
+			AM62AX_IOPAD(0x244, PIN_OUTPUT, 7) /* (D18) MMC1_SDWP.GPIO1_49 */
+		>;
+	};
+};
+
+&main_i2c0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c0_pins_default>;
+	clock-frequency = <400000>;
+};
+
+&main_i2c1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c1_pins_default>;
+	clock-frequency = <400000>;
+
+	exp1: gpio@22 {
+		compatible = "ti,tca6424";
+		reg = <0x22>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		gpio-line-names = "GPIO_CPSW2_RST", "GPIO_CPSW1_RST",
+				   "BT_EN_SOC", "MMC1_SD_EN",
+				   "VPP_EN", "EXP_PS_3V3_En",
+				   "EXP_PS_5V0_En", "EXP_HAT_DETECT",
+				   "GPIO_AUD_RSTn", "GPIO_eMMC_RSTn",
+				   "UART1_FET_BUF_EN", "BT_UART_WAKE_SOC",
+				   "GPIO_HDMI_RSTn", "CSI_GPIO0",
+				   "CSI_GPIO1", "WLAN_ALERTn",
+				   "HDMI_INTn", "TEST_GPIO2",
+				   "MCASP1_FET_EN", "MCASP1_BUF_BT_EN",
+				   "MCASP1_FET_SEL", "UART1_FET_SEL",
+				   "PD_I2C_IRQ", "IO_EXP_TEST_LED";
+	};
+};
+
+&sdhci1 {
+	/* SD/MMC */
+	status = "okay";
+	vmmc-supply = <&vdd_mmc1>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mmc1_pins_default>;
+	ti,driver-strength-ohm = <50>;
+	disable-wp;
+};
+
+&main_gpio0 {
+	status = "okay";
+};
+
+&main_gpio1 {
+	status = "okay";
+};
+
+&main_gpio_intr {
+	status = "okay";
+};
+
+&main_uart0 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_uart0_pins_default>;
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am62a7.dtsi b/arch/arm64/boot/dts/ti/k3-am62a7.dtsi
new file mode 100644
index 000000000000..331d89fda29d
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am62a7.dtsi
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Device Tree Source for AM62A7 SoC family in Quad core configuration
+ *
+ * TRM: https://www.ti.com/lit/zip/spruj16
+ *
+ * Copyright (C) 2020-2022 Texas Instruments Incorporated - https://www.ti.com/
+ */
+
+/dts-v1/;
+
+#include "k3-am62a.dtsi"
+
+/ {
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu-map {
+			cluster0: cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+
+				core2 {
+					cpu = <&cpu2>;
+				};
+
+				core3 {
+					cpu = <&cpu3>;
+				};
+			};
+		};
+
+		cpu0: cpu@0 {
+			compatible = "arm,cortex-a53";
+			reg = <0x000>;
+			device_type = "cpu";
+			enable-method = "psci";
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <128>;
+			next-level-cache = <&L2_0>;
+		};
+
+		cpu1: cpu@1 {
+			compatible = "arm,cortex-a53";
+			reg = <0x001>;
+			device_type = "cpu";
+			enable-method = "psci";
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <128>;
+			next-level-cache = <&L2_0>;
+		};
+
+		cpu2: cpu@2 {
+			compatible = "arm,cortex-a53";
+			reg = <0x002>;
+			device_type = "cpu";
+			enable-method = "psci";
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <128>;
+			next-level-cache = <&L2_0>;
+		};
+
+		cpu3: cpu@3 {
+			compatible = "arm,cortex-a53";
+			reg = <0x003>;
+			device_type = "cpu";
+			enable-method = "psci";
+			i-cache-size = <0x8000>;
+			i-cache-line-size = <64>;
+			i-cache-sets = <256>;
+			d-cache-size = <0x8000>;
+			d-cache-line-size = <64>;
+			d-cache-sets = <128>;
+			next-level-cache = <&L2_0>;
+		};
+	};
+
+	L2_0: l2-cache0 {
+		compatible = "cache";
+		cache-level = <2>;
+		cache-size = <0x40000>;
+		cache-line-size = <64>;
+		cache-sets = <512>;
+	};
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi
index ada00575f0f2..d6aa23681bbe 100644
--- a/arch/arm64/boot/dts/ti/k3-am64-main.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am64-main.dtsi
@@ -550,7 +550,7 @@
 		};
 	};
 
-	cpts@39000000 {
+	main_cpts0: cpts@39000000 {
 		compatible = "ti,j721e-cpts";
 		reg = <0x0 0x39000000 0x0 0x400>;
 		reg-names = "cpts";
@@ -1308,4 +1308,52 @@
 		interrupt-names = "int0", "int1";
 		bosch,mram-cfg = <0x0 128 64 64 64 64 32 32>;
 	};
+
+	crypto: crypto@40900000 {
+		compatible = "ti,am64-sa2ul";
+		reg = <0x00 0x40900000 0x00 0x1200>;
+		power-domains = <&k3_pds 133 TI_SCI_PD_SHARED>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges = <0x00 0x40900000 0x00 0x40900000 0x00 0x30000>;
+		dmas = <&main_pktdma 0xc001 0>, <&main_pktdma 0x4002 0>,
+		       <&main_pktdma 0x4003 0>;
+		dma-names = "tx", "rx1", "rx2";
+
+		rng: rng@40910000 {
+			compatible = "inside-secure,safexcel-eip76";
+			reg = <0x00 0x40910000 0x00 0x7d>;
+			interrupts = <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&k3_clks 133 1>;
+			status = "disabled"; /* Used by OP-TEE */
+		};
+	};
+
+	gpmc0: memory-controller@3b000000 {
+		compatible = "ti,am64-gpmc";
+		power-domains = <&k3_pds 80 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 80 0>;
+		clock-names = "fck";
+		reg = <0x00 0x03b000000 0x00 0x400>,
+		      <0x00 0x050000000 0x00 0x8000000>;
+		reg-names = "cfg", "data";
+		interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
+		gpmc,num-cs = <3>;
+		gpmc,num-waitpins = <2>;
+		#address-cells = <2>;
+		#size-cells = <1>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		gpio-controller;
+		#gpio-cells = <2>;
+	};
+
+	elm0: ecc@25010000 {
+		compatible = "ti,am64-elm";
+		reg = <0x00 0x25010000 0x00 0x2000>;
+		interrupts = <GIC_SPI 132 IRQ_TYPE_LEVEL_HIGH>;
+		power-domains = <&k3_pds 54 TI_SCI_PD_EXCLUSIVE>;
+		clocks = <&k3_clks 54 0>;
+		clock-names = "fck";
+	};
 };
diff --git a/arch/arm64/boot/dts/ti/k3-am64.dtsi b/arch/arm64/boot/dts/ti/k3-am64.dtsi
index 016dd8511ca6..c858725133af 100644
--- a/arch/arm64/boot/dts/ti/k3-am64.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am64.dtsi
@@ -82,6 +82,7 @@
 			 <0x00 0x3b000000 0x00 0x3b000000 0x00 0x00000400>, /* GPMC0_CFG */
 			 <0x00 0x3cd00000 0x00 0x3cd00000 0x00 0x00000200>, /* TIMERMGR0_CONFIG */
 			 <0x00 0x3f004000 0x00 0x3f004000 0x00 0x00000400>, /* GICSS0_REGS */
+			 <0x00 0x40900000 0x00 0x40900000 0x00 0x00030000>, /* SA2_UL0 */
 			 <0x00 0x43000000 0x00 0x43000000 0x00 0x00020000>, /* CTRL_MMR0 */
 			 <0x00 0x44043000 0x00 0x44043000 0x00 0x00000fe0>, /* TI SCI DEBUG */
 			 <0x00 0x48000000 0x00 0x48000000 0x00 0x06400000>, /* DMASS */
diff --git a/arch/arm64/boot/dts/ti/k3-am642-evm.dts b/arch/arm64/boot/dts/ti/k3-am642-evm.dts
index ad150c704623..5cf913860f80 100644
--- a/arch/arm64/boot/dts/ti/k3-am642-evm.dts
+++ b/arch/arm64/boot/dts/ti/k3-am642-evm.dts
@@ -676,3 +676,11 @@
 	pinctrl-0 = <&main_mcan1_pins_default>;
 	phys = <&transceiver2>;
 };
+
+&gpmc0 {
+	status = "disabled";
+};
+
+&elm0 {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am642-sk.dts b/arch/arm64/boot/dts/ti/k3-am642-sk.dts
index 2620469a7517..738d0cf6c40a 100644
--- a/arch/arm64/boot/dts/ti/k3-am642-sk.dts
+++ b/arch/arm64/boot/dts/ti/k3-am642-sk.dts
@@ -9,6 +9,7 @@
 #include <dt-bindings/phy/phy.h>
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/net/ti-dp83867.h>
+#include <dt-bindings/leds/common.h>
 #include "k3-am642.dtsi"
 
 / {
@@ -150,6 +151,74 @@
 		vin-supply = <&com8_ls_en>;
 		gpio = <&main_gpio0 48 GPIO_ACTIVE_HIGH>;
 	};
+
+	led-controller {
+		compatible = "gpio-leds";
+
+		led-0 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <1>;
+			gpios = <&exp2 0 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-1 {
+			color = <LED_COLOR_ID_RED>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <2>;
+			gpios = <&exp2 1 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-2 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <3>;
+			gpios = <&exp2 2 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-3 {
+			color = <LED_COLOR_ID_AMBER>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <4>;
+			gpios = <&exp2 3 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-4 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <5>;
+			gpios = <&exp2 4 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-5 {
+			color = <LED_COLOR_ID_RED>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <6>;
+			gpios = <&exp2 5 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-6 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_INDICATOR;
+			function-enumerator = <7>;
+			gpios = <&exp2 6 GPIO_ACTIVE_HIGH>;
+			default-state = "off";
+		};
+
+		led-7 {
+			color = <LED_COLOR_ID_AMBER>;
+			function = LED_FUNCTION_HEARTBEAT;
+			function-enumerator = <8>;
+			linux,default-trigger = "heartbeat";
+			gpios = <&exp2 7 GPIO_ACTIVE_HIGH>;
+		};
+	};
 };
 
 &main_pmx0 {
@@ -330,6 +399,14 @@
 				  "VPP_LDO_EN", "RPI_PS_3V3_En",
 				  "RPI_PS_5V0_En", "RPI_HAT_DETECT";
 	};
+
+	exp2: gpio@60 {
+		compatible = "ti,tpic2810";
+		reg = <0x60>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		gpio-line-names = "LED1","LED2","LED3","LED4","LED5","LED6","LED7","LED8";
+	};
 };
 
 &main_i2c3 {
@@ -607,3 +684,11 @@
 &main_mcan1 {
 	status = "disabled";
 };
+
+&gpmc0 {
+	status = "disabled";
+};
+
+&elm0 {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi
index 8919fede3cd7..4005a73cfea9 100644
--- a/arch/arm64/boot/dts/ti/k3-am65-main.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-am65-main.dtsi
@@ -112,13 +112,13 @@
 	crypto: crypto@4e00000 {
 		compatible = "ti,am654-sa2ul";
 		reg = <0x0 0x4e00000 0x0 0x1200>;
-		power-domains = <&k3_pds 136 TI_SCI_PD_EXCLUSIVE>;
+		power-domains = <&k3_pds 136 TI_SCI_PD_SHARED>;
 		#address-cells = <2>;
 		#size-cells = <2>;
 		ranges = <0x0 0x04e00000 0x00 0x04e00000 0x0 0x30000>;
 
-		dmas = <&main_udmap 0xc000>, <&main_udmap 0x4000>,
-				<&main_udmap 0x4001>;
+		dmas = <&main_udmap 0xc001>, <&main_udmap 0x4002>,
+				<&main_udmap 0x4003>;
 		dma-names = "tx", "rx1", "rx2";
 		dma-coherent;
 
@@ -127,6 +127,7 @@
 			reg = <0x0 0x4e10000 0x0 0x7d>;
 			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&k3_clks 136 1>;
+			status = "disabled"; /* Used by OP-TEE */
 		};
 	};
 
diff --git a/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts b/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts
index 121975dc8239..7e8552fd2b6a 100644
--- a/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts
+++ b/arch/arm64/boot/dts/ti/k3-j7200-common-proc-board.dts
@@ -134,15 +134,17 @@
 		>;
 	};
 
-	main_usbss0_pins_default: main-usbss0-pins-default {
+	vdd_sd_dv_pins_default: vdd-sd-dv-pins-default {
 		pinctrl-single,pins = <
-			J721E_IOPAD(0x120, PIN_OUTPUT, 0) /* (T4) USB0_DRVVBUS */
+			J721E_IOPAD(0xd0, PIN_OUTPUT, 7) /* (T5) SPI0_D1.GPIO0_55 */
 		>;
 	};
+};
 
-	vdd_sd_dv_pins_default: vdd-sd-dv-pins-default {
+&main_pmx1 {
+	main_usbss0_pins_default: main-usbss0-pins-default {
 		pinctrl-single,pins = <
-			J721E_IOPAD(0xd0, PIN_OUTPUT, 7) /* (T5) SPI0_D1.GPIO0_55 */
+			J721E_IOPAD(0x04, PIN_OUTPUT, 0) /* (T4) USB0_DRVVBUS */
 		>;
 	};
 };
diff --git a/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi b/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi
index 16684a2f054d..80a57916bcb3 100644
--- a/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-j7200-main.dtsi
@@ -295,7 +295,16 @@
 	main_pmx0: pinctrl@11c000 {
 		compatible = "pinctrl-single";
 		/* Proxy 0 addressing */
-		reg = <0x00 0x11c000 0x00 0x2b4>;
+		reg = <0x00 0x11c000 0x00 0x10c>;
+		#pinctrl-cells = <1>;
+		pinctrl-single,register-width = <32>;
+		pinctrl-single,function-mask = <0xffffffff>;
+	};
+
+	main_pmx1: pinctrl@11c11c {
+		compatible = "pinctrl-single";
+		/* Proxy 0 addressing */
+		reg = <0x00 0x11c11c 0x00 0xc>;
 		#pinctrl-cells = <1>;
 		pinctrl-single,register-width = <32>;
 		pinctrl-single,function-mask = <0xffffffff>;
@@ -739,6 +748,24 @@
 		clock-names = "gpio";
 	};
 
+	watchdog0: watchdog@2200000 {
+		compatible = "ti,j7-rti-wdt";
+		reg = <0x0 0x2200000 0x0 0x100>;
+		clocks = <&k3_clks 252 1>;
+		power-domains = <&k3_pds 252 TI_SCI_PD_EXCLUSIVE>;
+		assigned-clocks = <&k3_clks 252 1>;
+		assigned-clock-parents = <&k3_clks 252 5>;
+	};
+
+	watchdog1: watchdog@2210000 {
+		compatible = "ti,j7-rti-wdt";
+		reg = <0x0 0x2210000 0x0 0x100>;
+		clocks = <&k3_clks 253 1>;
+		power-domains = <&k3_pds 253 TI_SCI_PD_EXCLUSIVE>;
+		assigned-clocks = <&k3_clks 253 1>;
+		assigned-clock-parents = <&k3_clks 253 5>;
+	};
+
 	main_r5fss0: r5fss@5c00000 {
 		compatible = "ti,j7200-r5fss";
 		ti,cluster-mode = <1>;
diff --git a/arch/arm64/boot/dts/ti/k3-j7200-mcu-wakeup.dtsi b/arch/arm64/boot/dts/ti/k3-j7200-mcu-wakeup.dtsi
index ff13bbeed30c..e5be78a58682 100644
--- a/arch/arm64/boot/dts/ti/k3-j7200-mcu-wakeup.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-j7200-mcu-wakeup.dtsi
@@ -375,4 +375,24 @@
 			ti,loczrama = <1>;
 		};
 	};
+
+	mcu_crypto: crypto@40900000 {
+		compatible = "ti,j721e-sa2ul";
+		reg = <0x00 0x40900000 0x00 0x1200>;
+		power-domains = <&k3_pds 265 TI_SCI_PD_SHARED>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges = <0x00 0x40900000 0x00 0x40900000 0x00 0x30000>;
+		dmas = <&mcu_udmap 0xf501>, <&mcu_udmap 0x7502>,
+		       <&mcu_udmap 0x7503>;
+		dma-names = "tx", "rx1", "rx2";
+		dma-coherent;
+
+		rng: rng@40910000 {
+			compatible = "inside-secure,safexcel-eip76";
+			reg = <0x00 0x40910000 0x00 0x7d>;
+			interrupts = <GIC_SPI 945 IRQ_TYPE_LEVEL_HIGH>;
+			status = "disabled"; /* Used by OP-TEE */
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi
index 43b6cf5791ee..917c9dc99efa 100644
--- a/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-j721e-main.dtsi
@@ -343,7 +343,7 @@
 			compatible = "inside-secure,safexcel-eip76";
 			reg = <0x0 0x4e10000 0x0 0x7d>;
 			interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&k3_clks 264 1>;
+			clocks = <&k3_clks 264 2>;
 		};
 	};
 
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 5b167649097e..7b49c80286be 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -18,6 +18,7 @@ CONFIG_NUMA_BALANCING=y
 CONFIG_MEMCG=y
 CONFIG_BLK_CGROUP=y
 CONFIG_CGROUP_PIDS=y
+CONFIG_CGROUP_FREEZER=y
 CONFIG_CGROUP_HUGETLB=y
 CONFIG_CPUSETS=y
 CONFIG_CGROUP_DEVICE=y
@@ -33,8 +34,8 @@ CONFIG_ARCH_ACTIONS=y
 CONFIG_ARCH_SUNXI=y
 CONFIG_ARCH_ALPINE=y
 CONFIG_ARCH_APPLE=y
+CONFIG_ARCH_BCM=y
 CONFIG_ARCH_BCM2835=y
-CONFIG_ARCH_BCM4908=y
 CONFIG_ARCH_BCMBCA=y
 CONFIG_ARCH_BCM_IPROC=y
 CONFIG_ARCH_BERLIN=y
@@ -81,7 +82,6 @@ CONFIG_RANDOMIZE_BASE=y
 CONFIG_HIBERNATION=y
 CONFIG_WQ_POWER_EFFICIENT_DEFAULT=y
 CONFIG_ENERGY_MODEL=y
-CONFIG_ARM_CPUIDLE=y
 CONFIG_ARM_PSCI_CPUIDLE=y
 CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_STAT=y
@@ -103,6 +103,8 @@ CONFIG_ARM_SCMI_CPUFREQ=y
 CONFIG_ARM_TEGRA186_CPUFREQ=y
 CONFIG_QORIQ_CPUFREQ=y
 CONFIG_ACPI=y
+CONFIG_ACPI_HOTPLUG_MEMORY=y
+CONFIG_ACPI_HMAT=y
 CONFIG_ACPI_APEI=y
 CONFIG_ACPI_APEI_GHES=y
 CONFIG_ACPI_APEI_PCIEAER=y
@@ -127,6 +129,8 @@ CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
 # CONFIG_COMPAT_BRK is not set
+CONFIG_MEMORY_HOTPLUG=y
+CONFIG_MEMORY_HOTREMOVE=y
 CONFIG_KSM=y
 CONFIG_MEMORY_FAILURE=y
 CONFIG_TRANSPARENT_HUGEPAGE=y
@@ -140,12 +144,16 @@ CONFIG_IP_PNP_DHCP=y
 CONFIG_IP_PNP_BOOTP=y
 CONFIG_IPV6=m
 CONFIG_NETFILTER=y
+CONFIG_BRIDGE_NETFILTER=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_EVENTS=y
+CONFIG_NETFILTER_XT_MARK=m
 CONFIG_NETFILTER_XT_TARGET_CHECKSUM=m
 CONFIG_NETFILTER_XT_TARGET_LOG=m
 CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=m
 CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
+CONFIG_NETFILTER_XT_MATCH_IPVS=m
+CONFIG_IP_VS=m
 CONFIG_IP_NF_IPTABLES=m
 CONFIG_IP_NF_FILTER=m
 CONFIG_IP_NF_TARGET_REJECT=m
@@ -355,6 +363,7 @@ CONFIG_SMSC911X=y
 CONFIG_SNI_AVE=y
 CONFIG_SNI_NETSEC=y
 CONFIG_STMMAC_ETH=m
+CONFIG_DWMAC_TEGRA=m
 CONFIG_TI_K3_AM65_CPSW_NUSS=y
 CONFIG_QCOM_IPA=m
 CONFIG_MESON_GXL_PHY=m
@@ -475,6 +484,7 @@ CONFIG_I2C_QCOM_GENI=m
 CONFIG_I2C_QUP=y
 CONFIG_I2C_RIIC=y
 CONFIG_I2C_RK3X=y
+CONFIG_I2C_RZV2M=m
 CONFIG_I2C_S3C2410=y
 CONFIG_I2C_SH_MOBILE=y
 CONFIG_I2C_TEGRA=y
@@ -507,6 +517,8 @@ CONFIG_SPI_QCOM_GENI=m
 CONFIG_SPI_S3C64XX=y
 CONFIG_SPI_SH_MSIOF=m
 CONFIG_SPI_SUN6I=y
+CONFIG_SPI_TEGRA210_QUAD=m
+CONFIG_SPI_TEGRA114=m
 CONFIG_SPI_SPIDEV=m
 CONFIG_SPMI=y
 CONFIG_PINCTRL_MAX77620=y
@@ -535,6 +547,7 @@ CONFIG_PINCTRL_QDF2XXX=y
 CONFIG_PINCTRL_QCOM_SPMI_PMIC=y
 CONFIG_PINCTRL_SC7180=y
 CONFIG_PINCTRL_SC7280=y
+CONFIG_PINCTRL_SC8180X=y
 CONFIG_PINCTRL_SC8280XP=y
 CONFIG_PINCTRL_SDM845=y
 CONFIG_PINCTRL_SM8150=y
@@ -585,6 +598,7 @@ CONFIG_SENSORS_INA2XX=m
 CONFIG_SENSORS_INA3221=m
 CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
 CONFIG_CPU_THERMAL=y
+CONFIG_DEVFREQ_THERMAL=y
 CONFIG_THERMAL_EMULATION=y
 CONFIG_IMX_SC_THERMAL=m
 CONFIG_IMX8MM_THERMAL=m
@@ -795,6 +809,8 @@ CONFIG_SND_SOC_APQ8016_SBC=m
 CONFIG_SND_SOC_MSM8996=m
 CONFIG_SND_SOC_SDM845=m
 CONFIG_SND_SOC_SM8250=m
+CONFIG_SND_SOC_SC7180=m
+CONFIG_SND_SOC_SC7280=m
 CONFIG_SND_SOC_ROCKCHIP=m
 CONFIG_SND_SOC_ROCKCHIP_SPDIF=m
 CONFIG_SND_SOC_ROCKCHIP_RT5645=m
@@ -915,6 +931,7 @@ CONFIG_MMC_SDHCI=y
 CONFIG_MMC_SDHCI_ACPI=y
 CONFIG_MMC_SDHCI_PLTFM=y
 CONFIG_MMC_SDHCI_OF_ARASAN=y
+CONFIG_MMC_SDHCI_OF_DWCMSHC=y
 CONFIG_MMC_SDHCI_OF_ESDHC=y
 CONFIG_MMC_SDHCI_CADENCE=y
 CONFIG_MMC_SDHCI_ESDHC_IMX=y
@@ -940,6 +957,7 @@ CONFIG_SCSI_UFSHCD=y
 CONFIG_SCSI_UFSHCD_PLATFORM=y
 CONFIG_SCSI_UFS_QCOM=m
 CONFIG_SCSI_UFS_HISI=y
+CONFIG_SCSI_UFS_RENESAS=m
 CONFIG_SCSI_UFS_EXYNOS=y
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
@@ -996,7 +1014,7 @@ CONFIG_MV_XOR=y
 CONFIG_MV_XOR_V2=y
 CONFIG_OWL_DMA=y
 CONFIG_PL330_DMA=y
-CONFIG_TEGRA186_GPC_DMA=m
+CONFIG_TEGRA186_GPC_DMA=y
 CONFIG_TEGRA20_APB_DMA=y
 CONFIG_TEGRA210_ADMA=m
 CONFIG_QCOM_BAM_DMA=y
@@ -1059,6 +1077,7 @@ CONFIG_MSM_GCC_8998=y
 CONFIG_QCS_GCC_404=y
 CONFIG_SC_GCC_7180=y
 CONFIG_SC_GCC_7280=y
+CONFIG_SC_GCC_8180X=y
 CONFIG_SC_GCC_8280XP=y
 CONFIG_SDM_CAMCC_845=m
 CONFIG_SDM_GPUCC_845=y
@@ -1089,11 +1108,13 @@ CONFIG_ARM_SMMU_V3=y
 CONFIG_MTK_IOMMU=y
 CONFIG_QCOM_IOMMU=y
 CONFIG_REMOTEPROC=y
+CONFIG_QCOM_Q6V5_ADSP=m
 CONFIG_QCOM_Q6V5_MSS=m
 CONFIG_QCOM_Q6V5_PAS=m
 CONFIG_QCOM_SYSMON=m
 CONFIG_QCOM_WCNSS_PIL=m
 CONFIG_RPMSG_CHAR=m
+CONFIG_RPMSG_CTRL=m
 CONFIG_RPMSG_QCOM_GLINK_RPM=y
 CONFIG_RPMSG_QCOM_GLINK_SMEM=m
 CONFIG_RPMSG_QCOM_SMD=y
@@ -1110,6 +1131,8 @@ CONFIG_QCOM_AOSS_QMP=y
 CONFIG_QCOM_COMMAND_DB=y
 CONFIG_QCOM_CPR=y
 CONFIG_QCOM_GENI_SE=y
+CONFIG_QCOM_LLCC=m
+CONFIG_QCOM_OCMEM=m
 CONFIG_QCOM_RMTFS_MEM=m
 CONFIG_QCOM_RPMH=y
 CONFIG_QCOM_RPMHPD=y
@@ -1119,6 +1142,7 @@ CONFIG_QCOM_SMD_RPM=y
 CONFIG_QCOM_SMP2P=y
 CONFIG_QCOM_SMSM=y
 CONFIG_QCOM_SOCINFO=m
+CONFIG_QCOM_SPM=m
 CONFIG_QCOM_STATS=m
 CONFIG_QCOM_WCNSS_CTRL=m
 CONFIG_QCOM_APR=m
@@ -1208,6 +1232,7 @@ CONFIG_PHY_HISTB_COMBPHY=y
 CONFIG_PHY_HISI_INNO_USB2=y
 CONFIG_PHY_MVEBU_CP110_COMPHY=y
 CONFIG_PHY_MTK_TPHY=y
+CONFIG_PHY_QCOM_EDP=m
 CONFIG_PHY_QCOM_PCIE2=m
 CONFIG_PHY_QCOM_QMP=m
 CONFIG_PHY_QCOM_QUSB2=m
@@ -1230,8 +1255,14 @@ CONFIG_PHY_UNIPHIER_USB3=y
 CONFIG_PHY_TEGRA_XUSB=y
 CONFIG_PHY_AM654_SERDES=m
 CONFIG_PHY_J721E_WIZ=m
+CONFIG_ARM_CCI_PMU=m
+CONFIG_ARM_CCN=m
+CONFIG_ARM_CMN=m
 CONFIG_ARM_SMMU_V3_PMU=m
+CONFIG_ARM_DSU_PMU=m
 CONFIG_FSL_IMX8_DDR_PMU=m
+CONFIG_ARM_SPE_PMU=m
+CONFIG_ARM_DMC620_PMU=m
 CONFIG_QCOM_L2_PMU=y
 CONFIG_QCOM_L3_PMU=y
 CONFIG_HISI_PMU=y
@@ -1270,6 +1301,7 @@ CONFIG_INTERCONNECT_QCOM_OSM_L3=m
 CONFIG_INTERCONNECT_QCOM_QCS404=m
 CONFIG_INTERCONNECT_QCOM_SC7180=m
 CONFIG_INTERCONNECT_QCOM_SC7280=y
+CONFIG_INTERCONNECT_QCOM_SC8180X=y
 CONFIG_INTERCONNECT_QCOM_SC8280XP=y
 CONFIG_INTERCONNECT_QCOM_SDM845=y
 CONFIG_INTERCONNECT_QCOM_SM8150=m
@@ -1326,4 +1358,12 @@ CONFIG_DEBUG_FS=y
 # CONFIG_SCHED_DEBUG is not set
 # CONFIG_DEBUG_PREEMPT is not set
 # CONFIG_FTRACE is not set
+CONFIG_CORESIGHT=m
+CONFIG_CORESIGHT_LINK_AND_SINK_TMC=m
+CONFIG_CORESIGHT_CATU=m
+CONFIG_CORESIGHT_SINK_TPIU=m
+CONFIG_CORESIGHT_SINK_ETBV10=m
+CONFIG_CORESIGHT_STM=m
+CONFIG_CORESIGHT_CPU_DEBUG=m
+CONFIG_CORESIGHT_CTI=m
 CONFIG_MEMTEST=y
diff --git a/arch/arm64/crypto/ghash-ce-core.S b/arch/arm64/crypto/ghash-ce-core.S
index 7868330dd54e..ebe5558929b7 100644
--- a/arch/arm64/crypto/ghash-ce-core.S
+++ b/arch/arm64/crypto/ghash-ce-core.S
@@ -6,6 +6,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/assembler.h>
 
 	SHASH		.req	v0
@@ -350,11 +351,11 @@ CPU_LE(	rev64		T1.16b, T1.16b	)
 	 * void pmull_ghash_update(int blocks, u64 dg[], const char *src,
 	 *			   struct ghash_key const *k, const char *head)
 	 */
-SYM_FUNC_START(pmull_ghash_update_p64)
+SYM_TYPED_FUNC_START(pmull_ghash_update_p64)
 	__pmull_ghash	p64
 SYM_FUNC_END(pmull_ghash_update_p64)
 
-SYM_FUNC_START(pmull_ghash_update_p8)
+SYM_TYPED_FUNC_START(pmull_ghash_update_p8)
 	__pmull_ghash	p8
 SYM_FUNC_END(pmull_ghash_update_p8)
 
diff --git a/arch/arm64/crypto/sm3-ce-core.S b/arch/arm64/crypto/sm3-ce-core.S
index ef97d3187cb7..ca70cfacd0d0 100644
--- a/arch/arm64/crypto/sm3-ce-core.S
+++ b/arch/arm64/crypto/sm3-ce-core.S
@@ -6,6 +6,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/assembler.h>
 
 	.irp		b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
@@ -73,7 +74,7 @@
 	 *                       int blocks)
 	 */
 	.text
-SYM_FUNC_START(sm3_ce_transform)
+SYM_TYPED_FUNC_START(sm3_ce_transform)
 	/* load state */
 	ld1		{v8.4s-v9.4s}, [x0]
 	rev64		v8.4s, v8.4s
diff --git a/arch/arm64/include/asm/alternative-macros.h b/arch/arm64/include/asm/alternative-macros.h
index 7e157ab6cd50..3622e9f4fb44 100644
--- a/arch/arm64/include/asm/alternative-macros.h
+++ b/arch/arm64/include/asm/alternative-macros.h
@@ -2,10 +2,22 @@
 #ifndef __ASM_ALTERNATIVE_MACROS_H
 #define __ASM_ALTERNATIVE_MACROS_H
 
+#include <linux/const.h>
+#include <vdso/bits.h>
+
 #include <asm/cpucaps.h>
 #include <asm/insn-def.h>
 
-#define ARM64_CB_PATCH ARM64_NCAPS
+/*
+ * Binutils 2.27.0 can't handle a 'UL' suffix on constants, so for the assembly
+ * macros below we must use we must use `(1 << ARM64_CB_SHIFT)`.
+ */
+#define ARM64_CB_SHIFT	15
+#define ARM64_CB_BIT	BIT(ARM64_CB_SHIFT)
+
+#if ARM64_NCAPS >= ARM64_CB_BIT
+#error "cpucaps have overflown ARM64_CB_BIT"
+#endif
 
 #ifndef __ASSEMBLY__
 
@@ -73,8 +85,8 @@
 #define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...)	\
 	__ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
 
-#define ALTERNATIVE_CB(oldinstr, cb) \
-	__ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb)
+#define ALTERNATIVE_CB(oldinstr, feature, cb) \
+	__ALTERNATIVE_CFG_CB(oldinstr, (1 << ARM64_CB_SHIFT) | (feature), 1, cb)
 #else
 
 #include <asm/assembler.h>
@@ -82,7 +94,7 @@
 .macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
 	.word \orig_offset - .
 	.word \alt_offset - .
-	.hword \feature
+	.hword (\feature)
 	.byte \orig_len
 	.byte \alt_len
 .endm
@@ -141,10 +153,10 @@
 661:
 .endm
 
-.macro alternative_cb cb
+.macro alternative_cb cap, cb
 	.set .Lasm_alt_mode, 0
 	.pushsection .altinstructions, "a"
-	altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
+	altinstruction_entry 661f, \cb, (1 << ARM64_CB_SHIFT) | \cap, 662f-661f, 0
 	.popsection
 661:
 .endm
@@ -207,4 +219,46 @@ alternative_endif
 #define ALTERNATIVE(oldinstr, newinstr, ...)   \
 	_ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
 
+#ifndef __ASSEMBLY__
+
+#include <linux/types.h>
+
+static __always_inline bool
+alternative_has_feature_likely(unsigned long feature)
+{
+	compiletime_assert(feature < ARM64_NCAPS,
+			   "feature must be < ARM64_NCAPS");
+
+	asm_volatile_goto(
+	ALTERNATIVE_CB("b	%l[l_no]", %[feature], alt_cb_patch_nops)
+	:
+	: [feature] "i" (feature)
+	:
+	: l_no);
+
+	return true;
+l_no:
+	return false;
+}
+
+static __always_inline bool
+alternative_has_feature_unlikely(unsigned long feature)
+{
+	compiletime_assert(feature < ARM64_NCAPS,
+			   "feature must be < ARM64_NCAPS");
+
+	asm_volatile_goto(
+	ALTERNATIVE("nop", "b	%l[l_yes]", %[feature])
+	:
+	: [feature] "i" (feature)
+	:
+	: l_yes);
+
+	return false;
+l_yes:
+	return true;
+}
+
+#endif /* __ASSEMBLY__ */
+
 #endif /* __ASM_ALTERNATIVE_MACROS_H */
diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 5846145be523..e5957a53be39 100644
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -293,7 +293,7 @@ alternative_endif
 alternative_if_not ARM64_KVM_PROTECTED_MODE
 	ASM_BUG()
 alternative_else_nop_endif
-alternative_cb kvm_compute_final_ctr_el0
+alternative_cb ARM64_ALWAYS_SYSTEM, kvm_compute_final_ctr_el0
 	movz	\reg, #0
 	movk	\reg, #0, lsl #16
 	movk	\reg, #0, lsl #32
@@ -384,8 +384,8 @@ alternative_cb_end
 	.macro	tcr_compute_pa_size, tcr, pos, tmp0, tmp1
 	mrs	\tmp0, ID_AA64MMFR0_EL1
 	// Narrow PARange to fit the PS field in TCR_ELx
-	ubfx	\tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3
-	mov	\tmp1, #ID_AA64MMFR0_PARANGE_MAX
+	ubfx	\tmp0, \tmp0, #ID_AA64MMFR0_EL1_PARANGE_SHIFT, #3
+	mov	\tmp1, #ID_AA64MMFR0_EL1_PARANGE_MAX
 	cmp	\tmp0, \tmp1
 	csel	\tmp0, \tmp1, \tmp0, hi
 	bfi	\tcr, \tmp0, \pos, #3
@@ -512,7 +512,7 @@ alternative_endif
  */
 	.macro	reset_pmuserenr_el0, tmpreg
 	mrs	\tmpreg, id_aa64dfr0_el1
-	sbfx	\tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4
+	sbfx	\tmpreg, \tmpreg, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4
 	cmp	\tmpreg, #1			// Skip if no PMU present
 	b.lt	9000f
 	msr	pmuserenr_el0, xzr		// Disable PMU access from EL0
@@ -524,7 +524,7 @@ alternative_endif
  */
 	.macro	reset_amuserenr_el0, tmpreg
 	mrs	\tmpreg, id_aa64pfr0_el1	// Check ID_AA64PFR0_EL1
-	ubfx	\tmpreg, \tmpreg, #ID_AA64PFR0_AMU_SHIFT, #4
+	ubfx	\tmpreg, \tmpreg, #ID_AA64PFR0_EL1_AMU_SHIFT, #4
 	cbz	\tmpreg, .Lskip_\@		// Skip if no AMU present
 	msr_s	SYS_AMUSERENR_EL0, xzr		// Disable AMU access from EL0
 .Lskip_\@:
@@ -612,7 +612,7 @@ alternative_endif
 	.macro	offset_ttbr1, ttbr, tmp
 #ifdef CONFIG_ARM64_VA_BITS_52
 	mrs_s	\tmp, SYS_ID_AA64MMFR2_EL1
-	and	\tmp, \tmp, #(0xf << ID_AA64MMFR2_LVA_SHIFT)
+	and	\tmp, \tmp, #(0xf << ID_AA64MMFR2_EL1_VARange_SHIFT)
 	cbnz	\tmp, .Lskipoffs_\@
 	orr	\ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
 .Lskipoffs_\@ :
@@ -877,7 +877,7 @@ alternative_endif
 
 	.macro __mitigate_spectre_bhb_loop      tmp
 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
-alternative_cb  spectre_bhb_patch_loop_iter
+alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_iter
 	mov	\tmp, #32		// Patched to correct the immediate
 alternative_cb_end
 .Lspectre_bhb_loop\@:
@@ -890,7 +890,7 @@ alternative_cb_end
 
 	.macro mitigate_spectre_bhb_loop	tmp
 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
-alternative_cb	spectre_bhb_patch_loop_mitigation_enable
+alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_mitigation_enable
 	b	.L_spectre_bhb_loop_done\@	// Patched to NOP
 alternative_cb_end
 	__mitigate_spectre_bhb_loop	\tmp
@@ -904,7 +904,7 @@ alternative_cb_end
 	stp	x0, x1, [sp, #-16]!
 	stp	x2, x3, [sp, #-16]!
 	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_3
-alternative_cb	smccc_patch_fw_mitigation_conduit
+alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit
 	nop					// Patched to SMC/HVC #0
 alternative_cb_end
 	ldp	x2, x3, [sp], #16
@@ -914,7 +914,7 @@ alternative_cb_end
 
 	.macro mitigate_spectre_bhb_clear_insn
 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
-alternative_cb	spectre_bhb_patch_clearbhb
+alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_clearbhb
 	/* Patched to NOP when not supported */
 	clearbhb
 	isb
diff --git a/arch/arm64/include/asm/atomic_ll_sc.h b/arch/arm64/include/asm/atomic_ll_sc.h
index fe0db8d416fb..0890e4f568fb 100644
--- a/arch/arm64/include/asm/atomic_ll_sc.h
+++ b/arch/arm64/include/asm/atomic_ll_sc.h
@@ -12,19 +12,6 @@
 
 #include <linux/stringify.h>
 
-#ifdef CONFIG_ARM64_LSE_ATOMICS
-#define __LL_SC_FALLBACK(asm_ops)					\
-"	b	3f\n"							\
-"	.subsection	1\n"						\
-"3:\n"									\
-asm_ops "\n"								\
-"	b	4f\n"							\
-"	.previous\n"							\
-"4:\n"
-#else
-#define __LL_SC_FALLBACK(asm_ops) asm_ops
-#endif
-
 #ifndef CONFIG_CC_HAS_K_CONSTRAINT
 #define K
 #endif
@@ -36,38 +23,36 @@ asm_ops "\n"								\
  */
 
 #define ATOMIC_OP(op, asm_op, constraint)				\
-static inline void							\
+static __always_inline void						\
 __ll_sc_atomic_##op(int i, atomic_t *v)					\
 {									\
 	unsigned long tmp;						\
 	int result;							\
 									\
 	asm volatile("// atomic_" #op "\n"				\
-	__LL_SC_FALLBACK(						\
 	"	prfm	pstl1strm, %2\n"				\
 	"1:	ldxr	%w0, %2\n"					\
 	"	" #asm_op "	%w0, %w0, %w3\n"			\
 	"	stxr	%w1, %w0, %2\n"					\
-	"	cbnz	%w1, 1b\n")					\
+	"	cbnz	%w1, 1b\n"					\
 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
 	: __stringify(constraint) "r" (i));				\
 }
 
 #define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
-static inline int							\
+static __always_inline int						\
 __ll_sc_atomic_##op##_return##name(int i, atomic_t *v)			\
 {									\
 	unsigned long tmp;						\
 	int result;							\
 									\
 	asm volatile("// atomic_" #op "_return" #name "\n"		\
-	__LL_SC_FALLBACK(						\
 	"	prfm	pstl1strm, %2\n"				\
 	"1:	ld" #acq "xr	%w0, %2\n"				\
 	"	" #asm_op "	%w0, %w0, %w3\n"			\
 	"	st" #rel "xr	%w1, %w0, %2\n"				\
 	"	cbnz	%w1, 1b\n"					\
-	"	" #mb )							\
+	"	" #mb							\
 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
 	: __stringify(constraint) "r" (i)				\
 	: cl);								\
@@ -76,20 +61,19 @@ __ll_sc_atomic_##op##_return##name(int i, atomic_t *v)			\
 }
 
 #define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint) \
-static inline int							\
+static __always_inline int						\
 __ll_sc_atomic_fetch_##op##name(int i, atomic_t *v)			\
 {									\
 	unsigned long tmp;						\
 	int val, result;						\
 									\
 	asm volatile("// atomic_fetch_" #op #name "\n"			\
-	__LL_SC_FALLBACK(						\
 	"	prfm	pstl1strm, %3\n"				\
 	"1:	ld" #acq "xr	%w0, %3\n"				\
 	"	" #asm_op "	%w1, %w0, %w4\n"			\
 	"	st" #rel "xr	%w2, %w1, %3\n"				\
 	"	cbnz	%w2, 1b\n"					\
-	"	" #mb )							\
+	"	" #mb							\
 	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\
 	: __stringify(constraint) "r" (i)				\
 	: cl);								\
@@ -135,38 +119,36 @@ ATOMIC_OPS(andnot, bic, )
 #undef ATOMIC_OP
 
 #define ATOMIC64_OP(op, asm_op, constraint)				\
-static inline void							\
+static __always_inline void						\
 __ll_sc_atomic64_##op(s64 i, atomic64_t *v)				\
 {									\
 	s64 result;							\
 	unsigned long tmp;						\
 									\
 	asm volatile("// atomic64_" #op "\n"				\
-	__LL_SC_FALLBACK(						\
 	"	prfm	pstl1strm, %2\n"				\
 	"1:	ldxr	%0, %2\n"					\
 	"	" #asm_op "	%0, %0, %3\n"				\
 	"	stxr	%w1, %0, %2\n"					\
-	"	cbnz	%w1, 1b")					\
+	"	cbnz	%w1, 1b"					\
 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
 	: __stringify(constraint) "r" (i));				\
 }
 
 #define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op, constraint)\
-static inline long							\
+static __always_inline long						\
 __ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v)		\
 {									\
 	s64 result;							\
 	unsigned long tmp;						\
 									\
 	asm volatile("// atomic64_" #op "_return" #name "\n"		\
-	__LL_SC_FALLBACK(						\
 	"	prfm	pstl1strm, %2\n"				\
 	"1:	ld" #acq "xr	%0, %2\n"				\
 	"	" #asm_op "	%0, %0, %3\n"				\
 	"	st" #rel "xr	%w1, %0, %2\n"				\
 	"	cbnz	%w1, 1b\n"					\
-	"	" #mb )							\
+	"	" #mb							\
 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)		\
 	: __stringify(constraint) "r" (i)				\
 	: cl);								\
@@ -175,20 +157,19 @@ __ll_sc_atomic64_##op##_return##name(s64 i, atomic64_t *v)		\
 }
 
 #define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op, constraint)\
-static inline long							\
+static __always_inline long						\
 __ll_sc_atomic64_fetch_##op##name(s64 i, atomic64_t *v)			\
 {									\
 	s64 result, val;						\
 	unsigned long tmp;						\
 									\
 	asm volatile("// atomic64_fetch_" #op #name "\n"		\
-	__LL_SC_FALLBACK(						\
 	"	prfm	pstl1strm, %3\n"				\
 	"1:	ld" #acq "xr	%0, %3\n"				\
 	"	" #asm_op "	%1, %0, %4\n"				\
 	"	st" #rel "xr	%w2, %1, %3\n"				\
 	"	cbnz	%w2, 1b\n"					\
-	"	" #mb )							\
+	"	" #mb							\
 	: "=&r" (result), "=&r" (val), "=&r" (tmp), "+Q" (v->counter)	\
 	: __stringify(constraint) "r" (i)				\
 	: cl);								\
@@ -233,14 +214,13 @@ ATOMIC64_OPS(andnot, bic, )
 #undef ATOMIC64_OP_RETURN
 #undef ATOMIC64_OP
 
-static inline s64
+static __always_inline s64
 __ll_sc_atomic64_dec_if_positive(atomic64_t *v)
 {
 	s64 result;
 	unsigned long tmp;
 
 	asm volatile("// atomic64_dec_if_positive\n"
-	__LL_SC_FALLBACK(
 	"	prfm	pstl1strm, %2\n"
 	"1:	ldxr	%0, %2\n"
 	"	subs	%0, %0, #1\n"
@@ -248,7 +228,7 @@ __ll_sc_atomic64_dec_if_positive(atomic64_t *v)
 	"	stlxr	%w1, %0, %2\n"
 	"	cbnz	%w1, 1b\n"
 	"	dmb	ish\n"
-	"2:")
+	"2:"
 	: "=&r" (result), "=&r" (tmp), "+Q" (v->counter)
 	:
 	: "cc", "memory");
@@ -257,7 +237,7 @@ __ll_sc_atomic64_dec_if_positive(atomic64_t *v)
 }
 
 #define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl, constraint)	\
-static inline u##sz							\
+static __always_inline u##sz						\
 __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr,			\
 					 unsigned long old,		\
 					 u##sz new)			\
@@ -274,7 +254,6 @@ __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr,			\
 		old = (u##sz)old;					\
 									\
 	asm volatile(							\
-	__LL_SC_FALLBACK(						\
 	"	prfm	pstl1strm, %[v]\n"				\
 	"1:	ld" #acq "xr" #sfx "\t%" #w "[oldval], %[v]\n"		\
 	"	eor	%" #w "[tmp], %" #w "[oldval], %" #w "[old]\n"	\
@@ -282,7 +261,7 @@ __ll_sc__cmpxchg_case_##name##sz(volatile void *ptr,			\
 	"	st" #rel "xr" #sfx "\t%w[tmp], %" #w "[new], %[v]\n"	\
 	"	cbnz	%w[tmp], 1b\n"					\
 	"	" #mb "\n"						\
-	"2:")								\
+	"2:"								\
 	: [tmp] "=&r" (tmp), [oldval] "=&r" (oldval),			\
 	  [v] "+Q" (*(u##sz *)ptr)					\
 	: [old] __stringify(constraint) "r" (old), [new] "r" (new)	\
@@ -316,7 +295,7 @@ __CMPXCHG_CASE( ,  ,  mb_, 64, dmb ish,  , l, "memory", L)
 #undef __CMPXCHG_CASE
 
 #define __CMPXCHG_DBL(name, mb, rel, cl)				\
-static inline long							\
+static __always_inline long						\
 __ll_sc__cmpxchg_double##name(unsigned long old1,			\
 				      unsigned long old2,		\
 				      unsigned long new1,		\
@@ -326,7 +305,6 @@ __ll_sc__cmpxchg_double##name(unsigned long old1,			\
 	unsigned long tmp, ret;						\
 									\
 	asm volatile("// __cmpxchg_double" #name "\n"			\
-	__LL_SC_FALLBACK(						\
 	"	prfm	pstl1strm, %2\n"				\
 	"1:	ldxp	%0, %1, %2\n"					\
 	"	eor	%0, %0, %3\n"					\
@@ -336,7 +314,7 @@ __ll_sc__cmpxchg_double##name(unsigned long old1,			\
 	"	st" #rel "xp	%w0, %5, %6, %2\n"			\
 	"	cbnz	%w0, 1b\n"					\
 	"	" #mb "\n"						\
-	"2:")								\
+	"2:"								\
 	: "=&r" (tmp), "=&r" (ret), "+Q" (*(unsigned long *)ptr)	\
 	: "r" (old1), "r" (old2), "r" (new1), "r" (new2)		\
 	: cl);								\
diff --git a/arch/arm64/include/asm/atomic_lse.h b/arch/arm64/include/asm/atomic_lse.h
index 5d460f6b7675..52075e93de6c 100644
--- a/arch/arm64/include/asm/atomic_lse.h
+++ b/arch/arm64/include/asm/atomic_lse.h
@@ -11,7 +11,8 @@
 #define __ASM_ATOMIC_LSE_H
 
 #define ATOMIC_OP(op, asm_op)						\
-static inline void __lse_atomic_##op(int i, atomic_t *v)		\
+static __always_inline void						\
+__lse_atomic_##op(int i, atomic_t *v)					\
 {									\
 	asm volatile(							\
 	__LSE_PREAMBLE							\
@@ -25,7 +26,7 @@ ATOMIC_OP(or, stset)
 ATOMIC_OP(xor, steor)
 ATOMIC_OP(add, stadd)
 
-static inline void __lse_atomic_sub(int i, atomic_t *v)
+static __always_inline void __lse_atomic_sub(int i, atomic_t *v)
 {
 	__lse_atomic_add(-i, v);
 }
@@ -33,7 +34,8 @@ static inline void __lse_atomic_sub(int i, atomic_t *v)
 #undef ATOMIC_OP
 
 #define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...)			\
-static inline int __lse_atomic_fetch_##op##name(int i, atomic_t *v)	\
+static __always_inline int						\
+__lse_atomic_fetch_##op##name(int i, atomic_t *v)			\
 {									\
 	int old;							\
 									\
@@ -63,7 +65,8 @@ ATOMIC_FETCH_OPS(add, ldadd)
 #undef ATOMIC_FETCH_OPS
 
 #define ATOMIC_FETCH_OP_SUB(name)					\
-static inline int __lse_atomic_fetch_sub##name(int i, atomic_t *v)	\
+static __always_inline int						\
+__lse_atomic_fetch_sub##name(int i, atomic_t *v)			\
 {									\
 	return __lse_atomic_fetch_add##name(-i, v);			\
 }
@@ -76,12 +79,14 @@ ATOMIC_FETCH_OP_SUB(        )
 #undef ATOMIC_FETCH_OP_SUB
 
 #define ATOMIC_OP_ADD_SUB_RETURN(name)					\
-static inline int __lse_atomic_add_return##name(int i, atomic_t *v)	\
+static __always_inline int						\
+__lse_atomic_add_return##name(int i, atomic_t *v)			\
 {									\
 	return __lse_atomic_fetch_add##name(i, v) + i;			\
 }									\
 									\
-static inline int __lse_atomic_sub_return##name(int i, atomic_t *v)	\
+static __always_inline int						\
+__lse_atomic_sub_return##name(int i, atomic_t *v)			\
 {									\
 	return __lse_atomic_fetch_sub(i, v) - i;			\
 }
@@ -93,13 +98,14 @@ ATOMIC_OP_ADD_SUB_RETURN(        )
 
 #undef ATOMIC_OP_ADD_SUB_RETURN
 
-static inline void __lse_atomic_and(int i, atomic_t *v)
+static __always_inline void __lse_atomic_and(int i, atomic_t *v)
 {
 	return __lse_atomic_andnot(~i, v);
 }
 
 #define ATOMIC_FETCH_OP_AND(name, mb, cl...)				\
-static inline int __lse_atomic_fetch_and##name(int i, atomic_t *v)	\
+static __always_inline int						\
+__lse_atomic_fetch_and##name(int i, atomic_t *v)			\
 {									\
 	return __lse_atomic_fetch_andnot##name(~i, v);			\
 }
@@ -112,7 +118,8 @@ ATOMIC_FETCH_OP_AND(        , al, "memory")
 #undef ATOMIC_FETCH_OP_AND
 
 #define ATOMIC64_OP(op, asm_op)						\
-static inline void __lse_atomic64_##op(s64 i, atomic64_t *v)		\
+static __always_inline void						\
+__lse_atomic64_##op(s64 i, atomic64_t *v)				\
 {									\
 	asm volatile(							\
 	__LSE_PREAMBLE							\
@@ -126,7 +133,7 @@ ATOMIC64_OP(or, stset)
 ATOMIC64_OP(xor, steor)
 ATOMIC64_OP(add, stadd)
 
-static inline void __lse_atomic64_sub(s64 i, atomic64_t *v)
+static __always_inline void __lse_atomic64_sub(s64 i, atomic64_t *v)
 {
 	__lse_atomic64_add(-i, v);
 }
@@ -134,7 +141,8 @@ static inline void __lse_atomic64_sub(s64 i, atomic64_t *v)
 #undef ATOMIC64_OP
 
 #define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...)			\
-static inline long __lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v)\
+static __always_inline long						\
+__lse_atomic64_fetch_##op##name(s64 i, atomic64_t *v)			\
 {									\
 	s64 old;							\
 									\
@@ -164,7 +172,8 @@ ATOMIC64_FETCH_OPS(add, ldadd)
 #undef ATOMIC64_FETCH_OPS
 
 #define ATOMIC64_FETCH_OP_SUB(name)					\
-static inline long __lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v)	\
+static __always_inline long						\
+__lse_atomic64_fetch_sub##name(s64 i, atomic64_t *v)			\
 {									\
 	return __lse_atomic64_fetch_add##name(-i, v);			\
 }
@@ -177,12 +186,14 @@ ATOMIC64_FETCH_OP_SUB(        )
 #undef ATOMIC64_FETCH_OP_SUB
 
 #define ATOMIC64_OP_ADD_SUB_RETURN(name)				\
-static inline long __lse_atomic64_add_return##name(s64 i, atomic64_t *v)\
+static __always_inline long						\
+__lse_atomic64_add_return##name(s64 i, atomic64_t *v)			\
 {									\
 	return __lse_atomic64_fetch_add##name(i, v) + i;		\
 }									\
 									\
-static inline long __lse_atomic64_sub_return##name(s64 i, atomic64_t *v)\
+static __always_inline long						\
+__lse_atomic64_sub_return##name(s64 i, atomic64_t *v)			\
 {									\
 	return __lse_atomic64_fetch_sub##name(i, v) - i;		\
 }
@@ -194,13 +205,14 @@ ATOMIC64_OP_ADD_SUB_RETURN(        )
 
 #undef ATOMIC64_OP_ADD_SUB_RETURN
 
-static inline void __lse_atomic64_and(s64 i, atomic64_t *v)
+static __always_inline void __lse_atomic64_and(s64 i, atomic64_t *v)
 {
 	return __lse_atomic64_andnot(~i, v);
 }
 
 #define ATOMIC64_FETCH_OP_AND(name, mb, cl...)				\
-static inline long __lse_atomic64_fetch_and##name(s64 i, atomic64_t *v)	\
+static __always_inline long						\
+__lse_atomic64_fetch_and##name(s64 i, atomic64_t *v)			\
 {									\
 	return __lse_atomic64_fetch_andnot##name(~i, v);		\
 }
@@ -212,7 +224,7 @@ ATOMIC64_FETCH_OP_AND(        , al, "memory")
 
 #undef ATOMIC64_FETCH_OP_AND
 
-static inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v)
+static __always_inline s64 __lse_atomic64_dec_if_positive(atomic64_t *v)
 {
 	unsigned long tmp;
 
diff --git a/arch/arm64/include/asm/brk-imm.h b/arch/arm64/include/asm/brk-imm.h
index ec7720dbe2c8..6e000113e508 100644
--- a/arch/arm64/include/asm/brk-imm.h
+++ b/arch/arm64/include/asm/brk-imm.h
@@ -17,6 +17,7 @@
  * 0x401: for compile time BRK instruction
  * 0x800: kernel-mode BUG() and WARN() traps
  * 0x9xx: tag-based KASAN trap (allowed values 0x900 - 0x9ff)
+ * 0x8xxx: Control-Flow Integrity traps
  */
 #define KPROBES_BRK_IMM			0x004
 #define UPROBES_BRK_IMM			0x005
@@ -28,4 +29,9 @@
 #define KASAN_BRK_IMM			0x900
 #define KASAN_BRK_MASK			0x0ff
 
+#define CFI_BRK_IMM_TARGET		GENMASK(4, 0)
+#define CFI_BRK_IMM_TYPE		GENMASK(9, 5)
+#define CFI_BRK_IMM_BASE		0x8000
+#define CFI_BRK_IMM_MASK		(CFI_BRK_IMM_TARGET | CFI_BRK_IMM_TYPE)
+
 #endif
diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h
index 34256bda0da9..c0b178d1bb4f 100644
--- a/arch/arm64/include/asm/cache.h
+++ b/arch/arm64/include/asm/cache.h
@@ -45,10 +45,6 @@ static inline unsigned int arch_slab_minalign(void)
 #define arch_slab_minalign() arch_slab_minalign()
 #endif
 
-#define CTR_CACHE_MINLINE_MASK	\
-	(0xf << CTR_EL0_DMINLINE_SHIFT | \
-	 CTR_EL0_IMINLINE_MASK << CTR_EL0_IMINLINE_SHIFT)
-
 #define CTR_L1IP(ctr)		SYS_FIELD_GET(CTR_EL0, L1Ip, ctr)
 
 #define ICACHEF_ALIASING	0
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index fd7d75a275f6..f73f11b55042 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -6,6 +6,7 @@
 #ifndef __ASM_CPUFEATURE_H
 #define __ASM_CPUFEATURE_H
 
+#include <asm/alternative-macros.h>
 #include <asm/cpucaps.h>
 #include <asm/cputype.h>
 #include <asm/hwcap.h>
@@ -419,12 +420,8 @@ static __always_inline bool is_hyp_code(void)
 }
 
 extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
-extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
-extern struct static_key_false arm64_const_caps_ready;
 
-/* ARM64 CAPS + alternative_cb */
-#define ARM64_NPATCHABLE (ARM64_NCAPS + 1)
-extern DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);
+extern DECLARE_BITMAP(boot_capabilities, ARM64_NCAPS);
 
 #define for_each_available_cap(cap)		\
 	for_each_set_bit(cap, cpu_hwcaps, ARM64_NCAPS)
@@ -440,7 +437,7 @@ unsigned long cpu_get_elf_hwcap2(void);
 
 static __always_inline bool system_capabilities_finalized(void)
 {
-	return static_branch_likely(&arm64_const_caps_ready);
+	return alternative_has_feature_likely(ARM64_ALWAYS_SYSTEM);
 }
 
 /*
@@ -448,11 +445,11 @@ static __always_inline bool system_capabilities_finalized(void)
  *
  * Before the capability is detected, this returns false.
  */
-static inline bool cpus_have_cap(unsigned int num)
+static __always_inline bool cpus_have_cap(unsigned int num)
 {
 	if (num >= ARM64_NCAPS)
 		return false;
-	return test_bit(num, cpu_hwcaps);
+	return arch_test_bit(num, cpu_hwcaps);
 }
 
 /*
@@ -467,7 +464,7 @@ static __always_inline bool __cpus_have_const_cap(int num)
 {
 	if (num >= ARM64_NCAPS)
 		return false;
-	return static_branch_unlikely(&cpu_hwcap_keys[num]);
+	return alternative_has_feature_unlikely(num);
 }
 
 /*
@@ -553,7 +550,7 @@ cpuid_feature_cap_perfmon_field(u64 features, int field, u64 cap)
 	u64 mask = GENMASK_ULL(field + 3, field);
 
 	/* Treat IMPLEMENTATION DEFINED functionality as unimplemented */
-	if (val == ID_AA64DFR0_PMUVER_IMP_DEF)
+	if (val == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
 		val = 0;
 
 	if (val > cap) {
@@ -597,43 +594,43 @@ static inline s64 arm64_ftr_value(const struct arm64_ftr_bits *ftrp, u64 val)
 
 static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
 {
-	return cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL_SHIFT) == 0x1 ||
-		cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL0_SHIFT) == 0x1;
+	return cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_EL1_BIGEND_SHIFT) == 0x1 ||
+		cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_EL1_BIGENDEL0_SHIFT) == 0x1;
 }
 
 static inline bool id_aa64pfr0_32bit_el1(u64 pfr0)
 {
-	u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_SHIFT);
+	u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_EL1_SHIFT);
 
-	return val == ID_AA64PFR0_ELx_32BIT_64BIT;
+	return val == ID_AA64PFR0_EL1_ELx_32BIT_64BIT;
 }
 
 static inline bool id_aa64pfr0_32bit_el0(u64 pfr0)
 {
-	u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL0_SHIFT);
+	u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_EL0_SHIFT);
 
-	return val == ID_AA64PFR0_ELx_32BIT_64BIT;
+	return val == ID_AA64PFR0_EL1_ELx_32BIT_64BIT;
 }
 
 static inline bool id_aa64pfr0_sve(u64 pfr0)
 {
-	u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_SVE_SHIFT);
+	u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_SVE_SHIFT);
 
 	return val > 0;
 }
 
 static inline bool id_aa64pfr1_sme(u64 pfr1)
 {
-	u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_SME_SHIFT);
+	u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_SME_SHIFT);
 
 	return val > 0;
 }
 
 static inline bool id_aa64pfr1_mte(u64 pfr1)
 {
-	u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_MTE_SHIFT);
+	u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MTE_SHIFT);
 
-	return val >= ID_AA64PFR1_MTE;
+	return val >= ID_AA64PFR1_EL1_MTE_MTE2;
 }
 
 void __init setup_cpu_features(void);
@@ -659,7 +656,7 @@ static inline bool supports_csv2p3(int scope)
 		pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
 
 	csv2_val = cpuid_feature_extract_unsigned_field(pfr0,
-							ID_AA64PFR0_CSV2_SHIFT);
+							ID_AA64PFR0_EL1_CSV2_SHIFT);
 	return csv2_val == 3;
 }
 
@@ -694,10 +691,10 @@ static inline bool system_supports_4kb_granule(void)
 
 	mmfr0 =	read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
 	val = cpuid_feature_extract_unsigned_field(mmfr0,
-						ID_AA64MMFR0_TGRAN4_SHIFT);
+						ID_AA64MMFR0_EL1_TGRAN4_SHIFT);
 
-	return (val >= ID_AA64MMFR0_TGRAN4_SUPPORTED_MIN) &&
-	       (val <= ID_AA64MMFR0_TGRAN4_SUPPORTED_MAX);
+	return (val >= ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN) &&
+	       (val <= ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX);
 }
 
 static inline bool system_supports_64kb_granule(void)
@@ -707,10 +704,10 @@ static inline bool system_supports_64kb_granule(void)
 
 	mmfr0 =	read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
 	val = cpuid_feature_extract_unsigned_field(mmfr0,
-						ID_AA64MMFR0_TGRAN64_SHIFT);
+						ID_AA64MMFR0_EL1_TGRAN64_SHIFT);
 
-	return (val >= ID_AA64MMFR0_TGRAN64_SUPPORTED_MIN) &&
-	       (val <= ID_AA64MMFR0_TGRAN64_SUPPORTED_MAX);
+	return (val >= ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MIN) &&
+	       (val <= ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MAX);
 }
 
 static inline bool system_supports_16kb_granule(void)
@@ -720,10 +717,10 @@ static inline bool system_supports_16kb_granule(void)
 
 	mmfr0 =	read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
 	val = cpuid_feature_extract_unsigned_field(mmfr0,
-						ID_AA64MMFR0_TGRAN16_SHIFT);
+						ID_AA64MMFR0_EL1_TGRAN16_SHIFT);
 
-	return (val >= ID_AA64MMFR0_TGRAN16_SUPPORTED_MIN) &&
-	       (val <= ID_AA64MMFR0_TGRAN16_SUPPORTED_MAX);
+	return (val >= ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN) &&
+	       (val <= ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX);
 }
 
 static inline bool system_supports_mixed_endian_el0(void)
@@ -738,7 +735,7 @@ static inline bool system_supports_mixed_endian(void)
 
 	mmfr0 =	read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
 	val = cpuid_feature_extract_unsigned_field(mmfr0,
-						ID_AA64MMFR0_BIGENDEL_SHIFT);
+						ID_AA64MMFR0_EL1_BIGEND_SHIFT);
 
 	return val == 0x1;
 }
@@ -840,13 +837,13 @@ extern int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
 static inline u32 id_aa64mmfr0_parange_to_phys_shift(int parange)
 {
 	switch (parange) {
-	case ID_AA64MMFR0_PARANGE_32: return 32;
-	case ID_AA64MMFR0_PARANGE_36: return 36;
-	case ID_AA64MMFR0_PARANGE_40: return 40;
-	case ID_AA64MMFR0_PARANGE_42: return 42;
-	case ID_AA64MMFR0_PARANGE_44: return 44;
-	case ID_AA64MMFR0_PARANGE_48: return 48;
-	case ID_AA64MMFR0_PARANGE_52: return 52;
+	case ID_AA64MMFR0_EL1_PARANGE_32: return 32;
+	case ID_AA64MMFR0_EL1_PARANGE_36: return 36;
+	case ID_AA64MMFR0_EL1_PARANGE_40: return 40;
+	case ID_AA64MMFR0_EL1_PARANGE_42: return 42;
+	case ID_AA64MMFR0_EL1_PARANGE_44: return 44;
+	case ID_AA64MMFR0_EL1_PARANGE_48: return 48;
+	case ID_AA64MMFR0_EL1_PARANGE_52: return 52;
 	/*
 	 * A future PE could use a value unknown to the kernel.
 	 * However, by the "D10.1.4 Principles of the ID scheme
@@ -868,14 +865,14 @@ static inline bool cpu_has_hw_af(void)
 
 	mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
 	return cpuid_feature_extract_unsigned_field(mmfr1,
-						ID_AA64MMFR1_HADBS_SHIFT);
+						ID_AA64MMFR1_EL1_HAFDBS_SHIFT);
 }
 
 static inline bool cpu_has_pan(void)
 {
 	u64 mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
 	return cpuid_feature_extract_unsigned_field(mmfr1,
-						    ID_AA64MMFR1_PAN_SHIFT);
+						    ID_AA64MMFR1_EL1_PAN_SHIFT);
 }
 
 #ifdef CONFIG_ARM64_AMU_EXTN
@@ -896,8 +893,8 @@ static inline unsigned int get_vmid_bits(u64 mmfr1)
 	int vmid_bits;
 
 	vmid_bits = cpuid_feature_extract_unsigned_field(mmfr1,
-						ID_AA64MMFR1_VMIDBITS_SHIFT);
-	if (vmid_bits == ID_AA64MMFR1_VMIDBITS_16)
+						ID_AA64MMFR1_EL1_VMIDBits_SHIFT);
+	if (vmid_bits == ID_AA64MMFR1_EL1_VMIDBits_16)
 		return 16;
 
 	/*
@@ -907,6 +904,8 @@ static inline unsigned int get_vmid_bits(u64 mmfr1)
 	return 8;
 }
 
+struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id);
+
 extern struct arm64_ftr_override id_aa64mmfr1_override;
 extern struct arm64_ftr_override id_aa64pfr0_override;
 extern struct arm64_ftr_override id_aa64pfr1_override;
diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h
index 2630faa5bc08..668569adf4d3 100644
--- a/arch/arm64/include/asm/el2_setup.h
+++ b/arch/arm64/include/asm/el2_setup.h
@@ -40,7 +40,7 @@
 
 .macro __init_el2_debug
 	mrs	x1, id_aa64dfr0_el1
-	sbfx	x0, x1, #ID_AA64DFR0_PMUVER_SHIFT, #4
+	sbfx	x0, x1, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4
 	cmp	x0, #1
 	b.lt	.Lskip_pmu_\@			// Skip if no PMU present
 	mrs	x0, pmcr_el0			// Disable debug access traps
@@ -49,7 +49,7 @@
 	csel	x2, xzr, x0, lt			// all PMU counters from EL1
 
 	/* Statistical profiling */
-	ubfx	x0, x1, #ID_AA64DFR0_PMSVER_SHIFT, #4
+	ubfx	x0, x1, #ID_AA64DFR0_EL1_PMSVer_SHIFT, #4
 	cbz	x0, .Lskip_spe_\@		// Skip if SPE not present
 
 	mrs_s	x0, SYS_PMBIDR_EL1              // If SPE available at EL2,
@@ -65,7 +65,7 @@
 
 .Lskip_spe_\@:
 	/* Trace buffer */
-	ubfx	x0, x1, #ID_AA64DFR0_TRBE_SHIFT, #4
+	ubfx	x0, x1, #ID_AA64DFR0_EL1_TraceBuffer_SHIFT, #4
 	cbz	x0, .Lskip_trace_\@		// Skip if TraceBuffer is not present
 
 	mrs_s	x0, SYS_TRBIDR_EL1
@@ -83,7 +83,7 @@
 /* LORegions */
 .macro __init_el2_lor
 	mrs	x1, id_aa64mmfr1_el1
-	ubfx	x0, x1, #ID_AA64MMFR1_LOR_SHIFT, 4
+	ubfx	x0, x1, #ID_AA64MMFR1_EL1_LO_SHIFT, 4
 	cbz	x0, .Lskip_lor_\@
 	msr_s	SYS_LORC_EL1, xzr
 .Lskip_lor_\@:
@@ -97,7 +97,7 @@
 /* GICv3 system register access */
 .macro __init_el2_gicv3
 	mrs	x0, id_aa64pfr0_el1
-	ubfx	x0, x0, #ID_AA64PFR0_GIC_SHIFT, #4
+	ubfx	x0, x0, #ID_AA64PFR0_EL1_GIC_SHIFT, #4
 	cbz	x0, .Lskip_gicv3_\@
 
 	mrs_s	x0, SYS_ICC_SRE_EL2
@@ -132,12 +132,12 @@
 /* Disable any fine grained traps */
 .macro __init_el2_fgt
 	mrs	x1, id_aa64mmfr0_el1
-	ubfx	x1, x1, #ID_AA64MMFR0_FGT_SHIFT, #4
+	ubfx	x1, x1, #ID_AA64MMFR0_EL1_FGT_SHIFT, #4
 	cbz	x1, .Lskip_fgt_\@
 
 	mov	x0, xzr
 	mrs	x1, id_aa64dfr0_el1
-	ubfx	x1, x1, #ID_AA64DFR0_PMSVER_SHIFT, #4
+	ubfx	x1, x1, #ID_AA64DFR0_EL1_PMSVer_SHIFT, #4
 	cmp	x1, #3
 	b.lt	.Lset_debug_fgt_\@
 	/* Disable PMSNEVFR_EL1 read and write traps */
@@ -149,7 +149,7 @@
 
 	mov	x0, xzr
 	mrs	x1, id_aa64pfr1_el1
-	ubfx	x1, x1, #ID_AA64PFR1_SME_SHIFT, #4
+	ubfx	x1, x1, #ID_AA64PFR1_EL1_SME_SHIFT, #4
 	cbz	x1, .Lset_fgt_\@
 
 	/* Disable nVHE traps of TPIDR2 and SMPRI */
@@ -162,7 +162,7 @@
 	msr_s	SYS_HFGITR_EL2, xzr
 
 	mrs	x1, id_aa64pfr0_el1		// AMU traps UNDEF without AMU
-	ubfx	x1, x1, #ID_AA64PFR0_AMU_SHIFT, #4
+	ubfx	x1, x1, #ID_AA64PFR0_EL1_AMU_SHIFT, #4
 	cbz	x1, .Lskip_fgt_\@
 
 	msr_s	SYS_HAFGRTR_EL2, xzr
diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h
index d94aecff9690..19713d0f013b 100644
--- a/arch/arm64/include/asm/exception.h
+++ b/arch/arm64/include/asm/exception.h
@@ -58,8 +58,9 @@ asmlinkage void call_on_irq_stack(struct pt_regs *regs,
 asmlinkage void asm_exit_to_user_mode(struct pt_regs *regs);
 
 void do_mem_abort(unsigned long far, unsigned long esr, struct pt_regs *regs);
-void do_undefinstr(struct pt_regs *regs);
-void do_bti(struct pt_regs *regs);
+void do_undefinstr(struct pt_regs *regs, unsigned long esr);
+void do_el0_bti(struct pt_regs *regs);
+void do_el1_bti(struct pt_regs *regs, unsigned long esr);
 void do_debug_exception(unsigned long addr_if_watchpoint, unsigned long esr,
 			struct pt_regs *regs);
 void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs);
@@ -70,9 +71,11 @@ void do_sysinstr(unsigned long esr, struct pt_regs *regs);
 void do_sp_pc_abort(unsigned long addr, unsigned long esr, struct pt_regs *regs);
 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned long esr);
 void do_cp15instr(unsigned long esr, struct pt_regs *regs);
+int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs);
 void do_el0_svc(struct pt_regs *regs);
 void do_el0_svc_compat(struct pt_regs *regs);
-void do_ptrauth_fault(struct pt_regs *regs, unsigned long esr);
+void do_el0_fpac(struct pt_regs *regs, unsigned long esr);
+void do_el1_fpac(struct pt_regs *regs, unsigned long esr);
 void do_serror(struct pt_regs *regs, unsigned long esr);
 void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags);
 
diff --git a/arch/arm64/include/asm/ftrace.h b/arch/arm64/include/asm/ftrace.h
index dbc45a4157fa..329dbbd4d50b 100644
--- a/arch/arm64/include/asm/ftrace.h
+++ b/arch/arm64/include/asm/ftrace.h
@@ -26,7 +26,7 @@
 #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
 #define ARCH_SUPPORTS_FTRACE_OPS 1
 #else
-#define MCOUNT_ADDR		((unsigned long)function_nocfi(_mcount))
+#define MCOUNT_ADDR		((unsigned long)_mcount)
 #endif
 
 /* The BL at the callsite's adjusted rec->ip */
diff --git a/arch/arm64/include/asm/hw_breakpoint.h b/arch/arm64/include/asm/hw_breakpoint.h
index bc7aaed4b34e..fa4c6ff3aa9b 100644
--- a/arch/arm64/include/asm/hw_breakpoint.h
+++ b/arch/arm64/include/asm/hw_breakpoint.h
@@ -142,7 +142,7 @@ static inline int get_num_brps(void)
 	u64 dfr0 = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
 	return 1 +
 		cpuid_feature_extract_unsigned_field(dfr0,
-						ID_AA64DFR0_BRPS_SHIFT);
+						ID_AA64DFR0_EL1_BRPs_SHIFT);
 }
 
 /* Determine number of WRP registers available. */
@@ -151,7 +151,7 @@ static inline int get_num_wrps(void)
 	u64 dfr0 = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
 	return 1 +
 		cpuid_feature_extract_unsigned_field(dfr0,
-						ID_AA64DFR0_WRPS_SHIFT);
+						ID_AA64DFR0_EL1_WRPs_SHIFT);
 }
 
 #endif	/* __ASM_BREAKPOINT_H */
diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h
index cef4ae7a3d8b..298b386d3ebe 100644
--- a/arch/arm64/include/asm/hwcap.h
+++ b/arch/arm64/include/asm/hwcap.h
@@ -119,6 +119,7 @@
 #define KERNEL_HWCAP_SME_FA64		__khwcap2_feature(SME_FA64)
 #define KERNEL_HWCAP_WFXT		__khwcap2_feature(WFXT)
 #define KERNEL_HWCAP_EBF16		__khwcap2_feature(EBF16)
+#define KERNEL_HWCAP_SVE_EBF16		__khwcap2_feature(SVE_EBF16)
 
 /*
  * This yields a mask that user programs can use to figure out what
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index b208da3bebec..7784081088e7 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -63,7 +63,7 @@
  * specific registers encoded in the instructions).
  */
 .macro kern_hyp_va	reg
-alternative_cb kvm_update_va_mask
+alternative_cb ARM64_ALWAYS_SYSTEM, kvm_update_va_mask
 	and     \reg, \reg, #1		/* mask with va_mask */
 	ror	\reg, \reg, #1		/* rotate to the first tag bit */
 	add	\reg, \reg, #0		/* insert the low 12 bits of the tag */
@@ -97,7 +97,7 @@ alternative_cb_end
 	hyp_pa	\reg, \tmp
 
 	/* Load kimage_voffset. */
-alternative_cb kvm_get_kimage_voffset
+alternative_cb ARM64_ALWAYS_SYSTEM, kvm_get_kimage_voffset
 	movz	\tmp, #0
 	movk	\tmp, #0, lsl #16
 	movk	\tmp, #0, lsl #32
@@ -131,6 +131,7 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v)
 				    "add %0, %0, #0\n"
 				    "add %0, %0, #0, lsl 12\n"
 				    "ror %0, %0, #63\n",
+				    ARM64_ALWAYS_SYSTEM,
 				    kvm_update_va_mask)
 		     : "+r" (v));
 	return v;
diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h
index 9f339dffbc1a..1b098bd4cd37 100644
--- a/arch/arm64/include/asm/kvm_pgtable.h
+++ b/arch/arm64/include/asm/kvm_pgtable.h
@@ -16,9 +16,9 @@
 static inline u64 kvm_get_parange(u64 mmfr0)
 {
 	u64 parange = cpuid_feature_extract_unsigned_field(mmfr0,
-				ID_AA64MMFR0_PARANGE_SHIFT);
-	if (parange > ID_AA64MMFR0_PARANGE_MAX)
-		parange = ID_AA64MMFR0_PARANGE_MAX;
+				ID_AA64MMFR0_EL1_PARANGE_SHIFT);
+	if (parange > ID_AA64MMFR0_EL1_PARANGE_MAX)
+		parange = ID_AA64MMFR0_EL1_PARANGE_MAX;
 
 	return parange;
 }
diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h
index 43f8c25b3fda..1436fa1cde24 100644
--- a/arch/arm64/include/asm/linkage.h
+++ b/arch/arm64/include/asm/linkage.h
@@ -39,4 +39,8 @@
 	SYM_START(name, SYM_L_WEAK, SYM_A_NONE)		\
 	bti c ;
 
+#define SYM_TYPED_FUNC_START(name)				\
+	SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)	\
+	bti c ;
+
 #endif
diff --git a/arch/arm64/include/asm/lse.h b/arch/arm64/include/asm/lse.h
index 29c85810ae69..c503db8e73b0 100644
--- a/arch/arm64/include/asm/lse.h
+++ b/arch/arm64/include/asm/lse.h
@@ -13,14 +13,13 @@
 #include <linux/jump_label.h>
 #include <linux/stringify.h>
 #include <asm/alternative.h>
+#include <asm/alternative-macros.h>
 #include <asm/atomic_lse.h>
 #include <asm/cpucaps.h>
 
-extern struct static_key_false cpu_hwcap_keys[ARM64_NCAPS];
-
 static __always_inline bool system_uses_lse_atomics(void)
 {
-	return static_branch_likely(&cpu_hwcap_keys[ARM64_HAS_LSE_ATOMICS]);
+	return alternative_has_feature_likely(ARM64_HAS_LSE_ATOMICS);
 }
 
 #define __lse_ll_sc_body(op, ...)					\
diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h
index c7ccd82db1d2..d3f8b5df0c1f 100644
--- a/arch/arm64/include/asm/mmu_context.h
+++ b/arch/arm64/include/asm/mmu_context.h
@@ -147,7 +147,7 @@ static inline void cpu_install_ttbr0(phys_addr_t ttbr0, unsigned long t0sz)
  * Atomically replaces the active TTBR1_EL1 PGD with a new VA-compatible PGD,
  * avoiding the possibility of conflicting TLB entries being allocated.
  */
-static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
+static inline void cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
 {
 	typedef void (ttbr_replace_func)(phys_addr_t);
 	extern ttbr_replace_func idmap_cpu_replace_ttbr1;
@@ -168,7 +168,7 @@ static inline void __nocfi cpu_replace_ttbr1(pgd_t *pgdp, pgd_t *idmap)
 		ttbr1 |= TTBR_CNP_BIT;
 	}
 
-	replace_phys = (void *)__pa_symbol(function_nocfi(idmap_cpu_replace_ttbr1));
+	replace_phys = (void *)__pa_symbol(idmap_cpu_replace_ttbr1);
 
 	__cpu_install_idmap(idmap);
 	replace_phys(ttbr1);
diff --git a/arch/arm64/include/asm/module.h b/arch/arm64/include/asm/module.h
index 4e7fa2623896..18734fed3bdd 100644
--- a/arch/arm64/include/asm/module.h
+++ b/arch/arm64/include/asm/module.h
@@ -58,11 +58,20 @@ static inline bool is_forbidden_offset_for_adrp(void *place)
 }
 
 struct plt_entry get_plt_entry(u64 dst, void *pc);
-bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b);
 
-static inline bool plt_entry_is_initialized(const struct plt_entry *e)
+static inline const Elf_Shdr *find_section(const Elf_Ehdr *hdr,
+				    const Elf_Shdr *sechdrs,
+				    const char *name)
 {
-	return e->adrp || e->add || e->br;
+	const Elf_Shdr *s, *se;
+	const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
+
+	for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
+		if (strcmp(name, secstrs + s->sh_name) == 0)
+			return s;
+	}
+
+	return NULL;
 }
 
 #endif /* __ASM_MODULE_H */
diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h
index aa523591a44e..760c62f8e22f 100644
--- a/arch/arm64/include/asm/mte.h
+++ b/arch/arm64/include/asm/mte.h
@@ -42,7 +42,9 @@ void mte_sync_tags(pte_t old_pte, pte_t pte);
 void mte_copy_page_tags(void *kto, const void *kfrom);
 void mte_thread_init_user(void);
 void mte_thread_switch(struct task_struct *next);
+void mte_cpu_setup(void);
 void mte_suspend_enter(void);
+void mte_suspend_exit(void);
 long set_mte_ctrl(struct task_struct *task, unsigned long arg);
 long get_mte_ctrl(struct task_struct *task);
 int mte_ptrace_copy_tags(struct task_struct *child, long request,
@@ -72,6 +74,9 @@ static inline void mte_thread_switch(struct task_struct *next)
 static inline void mte_suspend_enter(void)
 {
 }
+static inline void mte_suspend_exit(void)
+{
+}
 static inline long set_mte_ctrl(struct task_struct *task, unsigned long arg)
 {
 	return 0;
diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 86eb0bfe3b38..61883518fc50 100644
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -410,7 +410,7 @@ long get_tagged_addr_ctrl(struct task_struct *task);
  * The top of the current task's task stack
  */
 #define current_top_of_stack()	((unsigned long)current->stack + THREAD_SIZE)
-#define on_thread_stack()	(on_task_stack(current, current_stack_pointer, 1, NULL))
+#define on_thread_stack()	(on_task_stack(current, current_stack_pointer, 1))
 
 #endif /* __ASSEMBLY__ */
 #endif /* __ASM_PROCESSOR_H */
diff --git a/arch/arm64/include/asm/sdei.h b/arch/arm64/include/asm/sdei.h
index 7bea1d705dd6..4292d9bafb9d 100644
--- a/arch/arm64/include/asm/sdei.h
+++ b/arch/arm64/include/asm/sdei.h
@@ -43,22 +43,5 @@ unsigned long do_sdei_event(struct pt_regs *regs,
 unsigned long sdei_arch_get_entry_point(int conduit);
 #define sdei_arch_get_entry_point(x)	sdei_arch_get_entry_point(x)
 
-struct stack_info;
-
-bool _on_sdei_stack(unsigned long sp, unsigned long size,
-		    struct stack_info *info);
-static inline bool on_sdei_stack(unsigned long sp, unsigned long size,
-				struct stack_info *info)
-{
-	if (!IS_ENABLED(CONFIG_VMAP_STACK))
-		return false;
-	if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))
-		return false;
-	if (in_nmi())
-		return _on_sdei_stack(sp, size, info);
-
-	return false;
-}
-
 #endif /* __ASSEMBLY__ */
 #endif	/* __ASM_SDEI_H */
diff --git a/arch/arm64/include/asm/stacktrace.h b/arch/arm64/include/asm/stacktrace.h
index 6ebdcdff77f5..5a0edb064ea4 100644
--- a/arch/arm64/include/asm/stacktrace.h
+++ b/arch/arm64/include/asm/stacktrace.h
@@ -22,39 +22,86 @@ extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk,
 
 DECLARE_PER_CPU(unsigned long *, irq_stack_ptr);
 
-static inline bool on_irq_stack(unsigned long sp, unsigned long size,
-				struct stack_info *info)
+static inline struct stack_info stackinfo_get_irq(void)
 {
 	unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr);
 	unsigned long high = low + IRQ_STACK_SIZE;
 
-	return on_stack(sp, size, low, high, STACK_TYPE_IRQ, info);
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
 }
 
-static inline bool on_task_stack(const struct task_struct *tsk,
-				 unsigned long sp, unsigned long size,
-				 struct stack_info *info)
+static inline bool on_irq_stack(unsigned long sp, unsigned long size)
+{
+	struct stack_info info = stackinfo_get_irq();
+	return stackinfo_on_stack(&info, sp, size);
+}
+
+static inline struct stack_info stackinfo_get_task(const struct task_struct *tsk)
 {
 	unsigned long low = (unsigned long)task_stack_page(tsk);
 	unsigned long high = low + THREAD_SIZE;
 
-	return on_stack(sp, size, low, high, STACK_TYPE_TASK, info);
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
+}
+
+static inline bool on_task_stack(const struct task_struct *tsk,
+				 unsigned long sp, unsigned long size)
+{
+	struct stack_info info = stackinfo_get_task(tsk);
+	return stackinfo_on_stack(&info, sp, size);
 }
 
 #ifdef CONFIG_VMAP_STACK
 DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
 
-static inline bool on_overflow_stack(unsigned long sp, unsigned long size,
-				struct stack_info *info)
+static inline struct stack_info stackinfo_get_overflow(void)
 {
 	unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack);
 	unsigned long high = low + OVERFLOW_STACK_SIZE;
 
-	return on_stack(sp, size, low, high, STACK_TYPE_OVERFLOW, info);
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
+}
+#else
+#define stackinfo_get_overflow()	stackinfo_get_unknown()
+#endif
+
+#if defined(CONFIG_ARM_SDE_INTERFACE) && defined(CONFIG_VMAP_STACK)
+DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
+DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
+
+static inline struct stack_info stackinfo_get_sdei_normal(void)
+{
+	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
+	unsigned long high = low + SDEI_STACK_SIZE;
+
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
+}
+
+static inline struct stack_info stackinfo_get_sdei_critical(void)
+{
+	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
+	unsigned long high = low + SDEI_STACK_SIZE;
+
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
 }
 #else
-static inline bool on_overflow_stack(unsigned long sp, unsigned long size,
-			struct stack_info *info) { return false; }
+#define stackinfo_get_sdei_normal()	stackinfo_get_unknown()
+#define stackinfo_get_sdei_critical()	stackinfo_get_unknown()
 #endif
 
 #endif	/* __ASM_STACKTRACE_H */
diff --git a/arch/arm64/include/asm/stacktrace/common.h b/arch/arm64/include/asm/stacktrace/common.h
index f58eb944c46f..508f734de46e 100644
--- a/arch/arm64/include/asm/stacktrace/common.h
+++ b/arch/arm64/include/asm/stacktrace/common.h
@@ -2,13 +2,6 @@
 /*
  * Common arm64 stack unwinder code.
  *
- * To implement a new arm64 stack unwinder:
- *     1) Include this header
- *
- *     2) Call into unwind_next_common() from your top level unwind
- *        function, passing it the validation and translation callbacks
- *        (though the later can be NULL if no translation is required).
- *
  * See: arch/arm64/kernel/stacktrace.c for the reference implementation.
  *
  * Copyright (C) 2012 ARM Ltd.
@@ -16,78 +9,60 @@
 #ifndef __ASM_STACKTRACE_COMMON_H
 #define __ASM_STACKTRACE_COMMON_H
 
-#include <linux/bitmap.h>
-#include <linux/bitops.h>
 #include <linux/kprobes.h>
 #include <linux/types.h>
 
-enum stack_type {
-	STACK_TYPE_UNKNOWN,
-	STACK_TYPE_TASK,
-	STACK_TYPE_IRQ,
-	STACK_TYPE_OVERFLOW,
-	STACK_TYPE_SDEI_NORMAL,
-	STACK_TYPE_SDEI_CRITICAL,
-	STACK_TYPE_HYP,
-	__NR_STACK_TYPES
-};
-
 struct stack_info {
 	unsigned long low;
 	unsigned long high;
-	enum stack_type type;
 };
 
-/*
- * A snapshot of a frame record or fp/lr register values, along with some
- * accounting information necessary for robust unwinding.
+/**
+ * struct unwind_state - state used for robust unwinding.
  *
  * @fp:          The fp value in the frame record (or the real fp)
  * @pc:          The lr value in the frame record (or the real lr)
  *
- * @stacks_done: Stacks which have been entirely unwound, for which it is no
- *               longer valid to unwind to.
- *
- * @prev_fp:     The fp that pointed to this frame record, or a synthetic value
- *               of 0. This is used to ensure that within a stack, each
- *               subsequent frame record is at an increasing address.
- * @prev_type:   The type of stack this frame record was on, or a synthetic
- *               value of STACK_TYPE_UNKNOWN. This is used to detect a
- *               transition from one stack to another.
- *
  * @kr_cur:      When KRETPROBES is selected, holds the kretprobe instance
  *               associated with the most recently encountered replacement lr
  *               value.
  *
  * @task:        The task being unwound.
+ *
+ * @stack:       The stack currently being unwound.
+ * @stacks:      An array of stacks which can be unwound.
+ * @nr_stacks:   The number of stacks in @stacks.
  */
 struct unwind_state {
 	unsigned long fp;
 	unsigned long pc;
-	DECLARE_BITMAP(stacks_done, __NR_STACK_TYPES);
-	unsigned long prev_fp;
-	enum stack_type prev_type;
 #ifdef CONFIG_KRETPROBES
 	struct llist_node *kr_cur;
 #endif
 	struct task_struct *task;
+
+	struct stack_info stack;
+	struct stack_info *stacks;
+	int nr_stacks;
 };
 
-static inline bool on_stack(unsigned long sp, unsigned long size,
-			    unsigned long low, unsigned long high,
-			    enum stack_type type, struct stack_info *info)
+static inline struct stack_info stackinfo_get_unknown(void)
+{
+	return (struct stack_info) {
+		.low = 0,
+		.high = 0,
+	};
+}
+
+static inline bool stackinfo_on_stack(const struct stack_info *info,
+				      unsigned long sp, unsigned long size)
 {
-	if (!low)
+	if (!info->low)
 		return false;
 
-	if (sp < low || sp + size < sp || sp + size > high)
+	if (sp < info->low || sp + size < sp || sp + size > info->high)
 		return false;
 
-	if (info) {
-		info->low = low;
-		info->high = high;
-		info->type = type;
-	}
 	return true;
 }
 
@@ -99,99 +74,101 @@ static inline void unwind_init_common(struct unwind_state *state,
 	state->kr_cur = NULL;
 #endif
 
-	/*
-	 * Prime the first unwind.
-	 *
-	 * In unwind_next() we'll check that the FP points to a valid stack,
-	 * which can't be STACK_TYPE_UNKNOWN, and the first unwind will be
-	 * treated as a transition to whichever stack that happens to be. The
-	 * prev_fp value won't be used, but we set it to 0 such that it is
-	 * definitely not an accessible stack address.
-	 */
-	bitmap_zero(state->stacks_done, __NR_STACK_TYPES);
-	state->prev_fp = 0;
-	state->prev_type = STACK_TYPE_UNKNOWN;
+	state->stack = stackinfo_get_unknown();
 }
 
-/*
- * stack_trace_translate_fp_fn() - Translates a non-kernel frame pointer to
- * a kernel address.
- *
- * @fp:   the frame pointer to be updated to its kernel address.
- * @type: the stack type associated with frame pointer @fp
- *
- * Returns true and success and @fp is updated to the corresponding
- * kernel virtual address; otherwise returns false.
- */
-typedef bool (*stack_trace_translate_fp_fn)(unsigned long *fp,
-					    enum stack_type type);
+static struct stack_info *unwind_find_next_stack(const struct unwind_state *state,
+						 unsigned long sp,
+						 unsigned long size)
+{
+	for (int i = 0; i < state->nr_stacks; i++) {
+		struct stack_info *info = &state->stacks[i];
 
-/*
- * on_accessible_stack_fn() - Check whether a stack range is on any
- * of the possible stacks.
+		if (stackinfo_on_stack(info, sp, size))
+			return info;
+	}
+
+	return NULL;
+}
+
+/**
+ * unwind_consume_stack() - Check if an object is on an accessible stack,
+ * updating stack boundaries so that future unwind steps cannot consume this
+ * object again.
  *
- * @tsk:  task whose stack is being unwound
- * @sp:   stack address being checked
- * @size: size of the stack range being checked
- * @info: stack unwinding context
+ * @state: the current unwind state.
+ * @sp:    the base address of the object.
+ * @size:  the size of the object.
+ *
+ * Return: 0 upon success, an error code otherwise.
  */
-typedef bool (*on_accessible_stack_fn)(const struct task_struct *tsk,
-				       unsigned long sp, unsigned long size,
-				       struct stack_info *info);
-
-static inline int unwind_next_common(struct unwind_state *state,
-				     struct stack_info *info,
-				     on_accessible_stack_fn accessible,
-				     stack_trace_translate_fp_fn translate_fp)
+static inline int unwind_consume_stack(struct unwind_state *state,
+				       unsigned long sp,
+				       unsigned long size)
 {
-	unsigned long fp = state->fp, kern_fp = fp;
-	struct task_struct *tsk = state->task;
+	struct stack_info *next;
 
-	if (fp & 0x7)
-		return -EINVAL;
+	if (stackinfo_on_stack(&state->stack, sp, size))
+		goto found;
 
-	if (!accessible(tsk, fp, 16, info))
-		return -EINVAL;
-
-	if (test_bit(info->type, state->stacks_done))
+	next = unwind_find_next_stack(state, sp, size);
+	if (!next)
 		return -EINVAL;
 
 	/*
-	 * If fp is not from the current address space perform the necessary
-	 * translation before dereferencing it to get the next fp.
-	 */
-	if (translate_fp && !translate_fp(&kern_fp, info->type))
-		return -EINVAL;
-
-	/*
-	 * As stacks grow downward, any valid record on the same stack must be
-	 * at a strictly higher address than the prior record.
+	 * Stack transitions are strictly one-way, and once we've
+	 * transitioned from one stack to another, it's never valid to
+	 * unwind back to the old stack.
+	 *
+	 * Remove the current stack from the list of stacks so that it cannot
+	 * be found on a subsequent transition.
 	 *
-	 * Stacks can nest in several valid orders, e.g.
+	 * Note that stacks can nest in several valid orders, e.g.
 	 *
-	 * TASK -> IRQ -> OVERFLOW -> SDEI_NORMAL
-	 * TASK -> SDEI_NORMAL -> SDEI_CRITICAL -> OVERFLOW
-	 * HYP -> OVERFLOW
+	 *   TASK -> IRQ -> OVERFLOW -> SDEI_NORMAL
+	 *   TASK -> SDEI_NORMAL -> SDEI_CRITICAL -> OVERFLOW
+	 *   HYP -> OVERFLOW
 	 *
-	 * ... but the nesting itself is strict. Once we transition from one
-	 * stack to another, it's never valid to unwind back to that first
-	 * stack.
+	 * ... so we do not check the specific order of stack
+	 * transitions.
 	 */
-	if (info->type == state->prev_type) {
-		if (fp <= state->prev_fp)
-			return -EINVAL;
-	} else {
-		__set_bit(state->prev_type, state->stacks_done);
-	}
+	state->stack = *next;
+	*next = stackinfo_get_unknown();
+
+found:
+	/*
+	 * Future unwind steps can only consume stack above this frame record.
+	 * Update the current stack to start immediately above it.
+	 */
+	state->stack.low = sp + size;
+	return 0;
+}
+
+/**
+ * unwind_next_frame_record() - Unwind to the next frame record.
+ *
+ * @state:        the current unwind state.
+ *
+ * Return: 0 upon success, an error code otherwise.
+ */
+static inline int
+unwind_next_frame_record(struct unwind_state *state)
+{
+	unsigned long fp = state->fp;
+	int err;
+
+	if (fp & 0x7)
+		return -EINVAL;
+
+	err = unwind_consume_stack(state, fp, 16);
+	if (err)
+		return err;
 
 	/*
-	 * Record this frame record's values and location. The prev_fp and
-	 * prev_type are only meaningful to the next unwind_next() invocation.
+	 * Record this frame record's values.
 	 */
-	state->fp = READ_ONCE(*(unsigned long *)(kern_fp));
-	state->pc = READ_ONCE(*(unsigned long *)(kern_fp + 8));
-	state->prev_fp = fp;
-	state->prev_type = info->type;
+	state->fp = READ_ONCE(*(unsigned long *)(fp));
+	state->pc = READ_ONCE(*(unsigned long *)(fp + 8));
 
 	return 0;
 }
diff --git a/arch/arm64/include/asm/stacktrace/nvhe.h b/arch/arm64/include/asm/stacktrace/nvhe.h
index d5527b600390..25ab83a315a7 100644
--- a/arch/arm64/include/asm/stacktrace/nvhe.h
+++ b/arch/arm64/include/asm/stacktrace/nvhe.h
@@ -20,8 +20,8 @@
 
 #include <asm/stacktrace/common.h>
 
-/*
- * kvm_nvhe_unwind_init - Start an unwind from the given nVHE HYP fp and pc
+/**
+ * kvm_nvhe_unwind_init() - Start an unwind from the given nVHE HYP fp and pc
  *
  * @state : unwind_state to initialize
  * @fp    : frame pointer at which to start the unwinding.
diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 818df938a7ad..debc1c0b2b7f 100644
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -190,19 +190,6 @@
 #define SYS_MVFR1_EL1			sys_reg(3, 0, 0, 3, 1)
 #define SYS_MVFR2_EL1			sys_reg(3, 0, 0, 3, 2)
 
-#define SYS_ID_AA64PFR0_EL1		sys_reg(3, 0, 0, 4, 0)
-#define SYS_ID_AA64PFR1_EL1		sys_reg(3, 0, 0, 4, 1)
-
-#define SYS_ID_AA64DFR0_EL1		sys_reg(3, 0, 0, 5, 0)
-#define SYS_ID_AA64DFR1_EL1		sys_reg(3, 0, 0, 5, 1)
-
-#define SYS_ID_AA64AFR0_EL1		sys_reg(3, 0, 0, 5, 4)
-#define SYS_ID_AA64AFR1_EL1		sys_reg(3, 0, 0, 5, 5)
-
-#define SYS_ID_AA64MMFR0_EL1		sys_reg(3, 0, 0, 7, 0)
-#define SYS_ID_AA64MMFR1_EL1		sys_reg(3, 0, 0, 7, 1)
-#define SYS_ID_AA64MMFR2_EL1		sys_reg(3, 0, 0, 7, 2)
-
 #define SYS_ACTLR_EL1			sys_reg(3, 0, 1, 0, 1)
 #define SYS_RGSR_EL1			sys_reg(3, 0, 1, 0, 5)
 #define SYS_GCR_EL1			sys_reg(3, 0, 1, 0, 6)
@@ -436,19 +423,11 @@
 #define SYS_ICC_IGRPEN0_EL1		sys_reg(3, 0, 12, 12, 6)
 #define SYS_ICC_IGRPEN1_EL1		sys_reg(3, 0, 12, 12, 7)
 
-#define SYS_TPIDR_EL1			sys_reg(3, 0, 13, 0, 4)
-
-#define SYS_SCXTNUM_EL1			sys_reg(3, 0, 13, 0, 7)
-
 #define SYS_CNTKCTL_EL1			sys_reg(3, 0, 14, 1, 0)
 
 #define SYS_CCSIDR_EL1			sys_reg(3, 1, 0, 0, 0)
 #define SYS_AIDR_EL1			sys_reg(3, 1, 0, 0, 7)
 
-#define SMIDR_EL1_IMPLEMENTER_SHIFT	24
-#define SMIDR_EL1_SMPS_SHIFT	15
-#define SMIDR_EL1_AFFINITY_SHIFT	0
-
 #define SYS_RNDR_EL0			sys_reg(3, 3, 2, 4, 0)
 #define SYS_RNDRRS_EL0			sys_reg(3, 3, 2, 4, 1)
 
@@ -537,7 +516,6 @@
 #define SYS_HFGWTR_EL2			sys_reg(3, 4, 1, 1, 5)
 #define SYS_HFGITR_EL2			sys_reg(3, 4, 1, 1, 6)
 #define SYS_TRFCR_EL2			sys_reg(3, 4, 1, 2, 1)
-#define SYS_HCRX_EL2			sys_reg(3, 4, 1, 2, 2)
 #define SYS_HDFGRTR_EL2			sys_reg(3, 4, 3, 1, 4)
 #define SYS_HDFGWTR_EL2			sys_reg(3, 4, 3, 1, 5)
 #define SYS_HAFGRTR_EL2			sys_reg(3, 4, 3, 1, 6)
@@ -690,164 +668,30 @@
 #define MAIR_ATTRIDX(attr, idx)		((attr) << ((idx) * 8))
 
 /* id_aa64pfr0 */
-#define ID_AA64PFR0_CSV3_SHIFT		60
-#define ID_AA64PFR0_CSV2_SHIFT		56
-#define ID_AA64PFR0_DIT_SHIFT		48
-#define ID_AA64PFR0_AMU_SHIFT		44
-#define ID_AA64PFR0_MPAM_SHIFT		40
-#define ID_AA64PFR0_SEL2_SHIFT		36
-#define ID_AA64PFR0_SVE_SHIFT		32
-#define ID_AA64PFR0_RAS_SHIFT		28
-#define ID_AA64PFR0_GIC_SHIFT		24
-#define ID_AA64PFR0_ASIMD_SHIFT		20
-#define ID_AA64PFR0_FP_SHIFT		16
-#define ID_AA64PFR0_EL3_SHIFT		12
-#define ID_AA64PFR0_EL2_SHIFT		8
-#define ID_AA64PFR0_EL1_SHIFT		4
-#define ID_AA64PFR0_EL0_SHIFT		0
-
-#define ID_AA64PFR0_AMU			0x1
-#define ID_AA64PFR0_SVE			0x1
-#define ID_AA64PFR0_RAS_V1		0x1
-#define ID_AA64PFR0_RAS_V1P1		0x2
-#define ID_AA64PFR0_FP_NI		0xf
-#define ID_AA64PFR0_FP_SUPPORTED	0x0
-#define ID_AA64PFR0_ASIMD_NI		0xf
-#define ID_AA64PFR0_ASIMD_SUPPORTED	0x0
-#define ID_AA64PFR0_ELx_64BIT_ONLY	0x1
-#define ID_AA64PFR0_ELx_32BIT_64BIT	0x2
-
-/* id_aa64pfr1 */
-#define ID_AA64PFR1_SME_SHIFT		24
-#define ID_AA64PFR1_MPAMFRAC_SHIFT	16
-#define ID_AA64PFR1_RASFRAC_SHIFT	12
-#define ID_AA64PFR1_MTE_SHIFT		8
-#define ID_AA64PFR1_SSBS_SHIFT		4
-#define ID_AA64PFR1_BT_SHIFT		0
-
-#define ID_AA64PFR1_SSBS_PSTATE_NI	0
-#define ID_AA64PFR1_SSBS_PSTATE_ONLY	1
-#define ID_AA64PFR1_SSBS_PSTATE_INSNS	2
-#define ID_AA64PFR1_BT_BTI		0x1
-#define ID_AA64PFR1_SME			1
-
-#define ID_AA64PFR1_MTE_NI		0x0
-#define ID_AA64PFR1_MTE_EL0		0x1
-#define ID_AA64PFR1_MTE			0x2
-#define ID_AA64PFR1_MTE_ASYMM		0x3
+#define ID_AA64PFR0_EL1_ELx_64BIT_ONLY		0x1
+#define ID_AA64PFR0_EL1_ELx_32BIT_64BIT		0x2
 
 /* id_aa64mmfr0 */
-#define ID_AA64MMFR0_ECV_SHIFT		60
-#define ID_AA64MMFR0_FGT_SHIFT		56
-#define ID_AA64MMFR0_EXS_SHIFT		44
-#define ID_AA64MMFR0_TGRAN4_2_SHIFT	40
-#define ID_AA64MMFR0_TGRAN64_2_SHIFT	36
-#define ID_AA64MMFR0_TGRAN16_2_SHIFT	32
-#define ID_AA64MMFR0_TGRAN4_SHIFT	28
-#define ID_AA64MMFR0_TGRAN64_SHIFT	24
-#define ID_AA64MMFR0_TGRAN16_SHIFT	20
-#define ID_AA64MMFR0_BIGENDEL0_SHIFT	16
-#define ID_AA64MMFR0_SNSMEM_SHIFT	12
-#define ID_AA64MMFR0_BIGENDEL_SHIFT	8
-#define ID_AA64MMFR0_ASID_SHIFT		4
-#define ID_AA64MMFR0_PARANGE_SHIFT	0
-
-#define ID_AA64MMFR0_ASID_8		0x0
-#define ID_AA64MMFR0_ASID_16		0x2
-
-#define ID_AA64MMFR0_TGRAN4_NI			0xf
-#define ID_AA64MMFR0_TGRAN4_SUPPORTED_MIN	0x0
-#define ID_AA64MMFR0_TGRAN4_SUPPORTED_MAX	0x7
-#define ID_AA64MMFR0_TGRAN64_NI			0xf
-#define ID_AA64MMFR0_TGRAN64_SUPPORTED_MIN	0x0
-#define ID_AA64MMFR0_TGRAN64_SUPPORTED_MAX	0x7
-#define ID_AA64MMFR0_TGRAN16_NI			0x0
-#define ID_AA64MMFR0_TGRAN16_SUPPORTED_MIN	0x1
-#define ID_AA64MMFR0_TGRAN16_SUPPORTED_MAX	0xf
-
-#define ID_AA64MMFR0_PARANGE_32		0x0
-#define ID_AA64MMFR0_PARANGE_36		0x1
-#define ID_AA64MMFR0_PARANGE_40		0x2
-#define ID_AA64MMFR0_PARANGE_42		0x3
-#define ID_AA64MMFR0_PARANGE_44		0x4
-#define ID_AA64MMFR0_PARANGE_48		0x5
-#define ID_AA64MMFR0_PARANGE_52		0x6
+#define ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN	0x0
+#define ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX	0x7
+#define ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MIN	0x0
+#define ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MAX	0x7
+#define ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN	0x1
+#define ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX	0xf
 
 #define ARM64_MIN_PARANGE_BITS		32
 
-#define ID_AA64MMFR0_TGRAN_2_SUPPORTED_DEFAULT	0x0
-#define ID_AA64MMFR0_TGRAN_2_SUPPORTED_NONE	0x1
-#define ID_AA64MMFR0_TGRAN_2_SUPPORTED_MIN	0x2
-#define ID_AA64MMFR0_TGRAN_2_SUPPORTED_MAX	0x7
+#define ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_DEFAULT	0x0
+#define ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_NONE		0x1
+#define ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MIN		0x2
+#define ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MAX		0x7
 
 #ifdef CONFIG_ARM64_PA_BITS_52
-#define ID_AA64MMFR0_PARANGE_MAX	ID_AA64MMFR0_PARANGE_52
+#define ID_AA64MMFR0_EL1_PARANGE_MAX	ID_AA64MMFR0_EL1_PARANGE_52
 #else
-#define ID_AA64MMFR0_PARANGE_MAX	ID_AA64MMFR0_PARANGE_48
+#define ID_AA64MMFR0_EL1_PARANGE_MAX	ID_AA64MMFR0_EL1_PARANGE_48
 #endif
 
-/* id_aa64mmfr1 */
-#define ID_AA64MMFR1_ECBHB_SHIFT	60
-#define ID_AA64MMFR1_TIDCP1_SHIFT	52
-#define ID_AA64MMFR1_HCX_SHIFT		40
-#define ID_AA64MMFR1_AFP_SHIFT		44
-#define ID_AA64MMFR1_ETS_SHIFT		36
-#define ID_AA64MMFR1_TWED_SHIFT		32
-#define ID_AA64MMFR1_XNX_SHIFT		28
-#define ID_AA64MMFR1_SPECSEI_SHIFT	24
-#define ID_AA64MMFR1_PAN_SHIFT		20
-#define ID_AA64MMFR1_LOR_SHIFT		16
-#define ID_AA64MMFR1_HPD_SHIFT		12
-#define ID_AA64MMFR1_VHE_SHIFT		8
-#define ID_AA64MMFR1_VMIDBITS_SHIFT	4
-#define ID_AA64MMFR1_HADBS_SHIFT	0
-
-#define ID_AA64MMFR1_VMIDBITS_8		0
-#define ID_AA64MMFR1_VMIDBITS_16	2
-
-#define ID_AA64MMFR1_TIDCP1_NI		0
-#define ID_AA64MMFR1_TIDCP1_IMP		1
-
-/* id_aa64mmfr2 */
-#define ID_AA64MMFR2_E0PD_SHIFT		60
-#define ID_AA64MMFR2_EVT_SHIFT		56
-#define ID_AA64MMFR2_BBM_SHIFT		52
-#define ID_AA64MMFR2_TTL_SHIFT		48
-#define ID_AA64MMFR2_FWB_SHIFT		40
-#define ID_AA64MMFR2_IDS_SHIFT		36
-#define ID_AA64MMFR2_AT_SHIFT		32
-#define ID_AA64MMFR2_ST_SHIFT		28
-#define ID_AA64MMFR2_NV_SHIFT		24
-#define ID_AA64MMFR2_CCIDX_SHIFT	20
-#define ID_AA64MMFR2_LVA_SHIFT		16
-#define ID_AA64MMFR2_IESB_SHIFT		12
-#define ID_AA64MMFR2_LSM_SHIFT		8
-#define ID_AA64MMFR2_UAO_SHIFT		4
-#define ID_AA64MMFR2_CNP_SHIFT		0
-
-/* id_aa64dfr0 */
-#define ID_AA64DFR0_MTPMU_SHIFT		48
-#define ID_AA64DFR0_TRBE_SHIFT		44
-#define ID_AA64DFR0_TRACE_FILT_SHIFT	40
-#define ID_AA64DFR0_DOUBLELOCK_SHIFT	36
-#define ID_AA64DFR0_PMSVER_SHIFT	32
-#define ID_AA64DFR0_CTX_CMPS_SHIFT	28
-#define ID_AA64DFR0_WRPS_SHIFT		20
-#define ID_AA64DFR0_BRPS_SHIFT		12
-#define ID_AA64DFR0_PMUVER_SHIFT	8
-#define ID_AA64DFR0_TRACEVER_SHIFT	4
-#define ID_AA64DFR0_DEBUGVER_SHIFT	0
-
-#define ID_AA64DFR0_PMUVER_8_0		0x1
-#define ID_AA64DFR0_PMUVER_8_1		0x4
-#define ID_AA64DFR0_PMUVER_8_4		0x5
-#define ID_AA64DFR0_PMUVER_8_5		0x6
-#define ID_AA64DFR0_PMUVER_8_7		0x7
-#define ID_AA64DFR0_PMUVER_IMP_DEF	0xf
-
-#define ID_AA64DFR0_PMSVER_8_2		0x1
-#define ID_AA64DFR0_PMSVER_8_3		0x2
-
 #define ID_DFR0_PERFMON_SHIFT		24
 
 #define ID_DFR0_PERFMON_8_0		0x3
@@ -955,20 +799,20 @@
 #define ID_PFR1_PROGMOD_SHIFT		0
 
 #if defined(CONFIG_ARM64_4K_PAGES)
-#define ID_AA64MMFR0_TGRAN_SHIFT		ID_AA64MMFR0_TGRAN4_SHIFT
-#define ID_AA64MMFR0_TGRAN_SUPPORTED_MIN	ID_AA64MMFR0_TGRAN4_SUPPORTED_MIN
-#define ID_AA64MMFR0_TGRAN_SUPPORTED_MAX	ID_AA64MMFR0_TGRAN4_SUPPORTED_MAX
-#define ID_AA64MMFR0_TGRAN_2_SHIFT		ID_AA64MMFR0_TGRAN4_2_SHIFT
+#define ID_AA64MMFR0_EL1_TGRAN_SHIFT		ID_AA64MMFR0_EL1_TGRAN4_SHIFT
+#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN	ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MIN
+#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX	ID_AA64MMFR0_EL1_TGRAN4_SUPPORTED_MAX
+#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT		ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT
 #elif defined(CONFIG_ARM64_16K_PAGES)
-#define ID_AA64MMFR0_TGRAN_SHIFT		ID_AA64MMFR0_TGRAN16_SHIFT
-#define ID_AA64MMFR0_TGRAN_SUPPORTED_MIN	ID_AA64MMFR0_TGRAN16_SUPPORTED_MIN
-#define ID_AA64MMFR0_TGRAN_SUPPORTED_MAX	ID_AA64MMFR0_TGRAN16_SUPPORTED_MAX
-#define ID_AA64MMFR0_TGRAN_2_SHIFT		ID_AA64MMFR0_TGRAN16_2_SHIFT
+#define ID_AA64MMFR0_EL1_TGRAN_SHIFT		ID_AA64MMFR0_EL1_TGRAN16_SHIFT
+#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN	ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MIN
+#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX	ID_AA64MMFR0_EL1_TGRAN16_SUPPORTED_MAX
+#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT		ID_AA64MMFR0_EL1_TGRAN16_2_SHIFT
 #elif defined(CONFIG_ARM64_64K_PAGES)
-#define ID_AA64MMFR0_TGRAN_SHIFT		ID_AA64MMFR0_TGRAN64_SHIFT
-#define ID_AA64MMFR0_TGRAN_SUPPORTED_MIN	ID_AA64MMFR0_TGRAN64_SUPPORTED_MIN
-#define ID_AA64MMFR0_TGRAN_SUPPORTED_MAX	ID_AA64MMFR0_TGRAN64_SUPPORTED_MAX
-#define ID_AA64MMFR0_TGRAN_2_SHIFT		ID_AA64MMFR0_TGRAN64_2_SHIFT
+#define ID_AA64MMFR0_EL1_TGRAN_SHIFT		ID_AA64MMFR0_EL1_TGRAN64_SHIFT
+#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN	ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MIN
+#define ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX	ID_AA64MMFR0_EL1_TGRAN64_SUPPORTED_MAX
+#define ID_AA64MMFR0_EL1_TGRAN_2_SHIFT		ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT
 #endif
 
 #define MVFR2_FPMISC_SHIFT		4
@@ -1028,9 +872,6 @@
 #define TRFCR_ELx_ExTRE			BIT(1)
 #define TRFCR_ELx_E0TRE			BIT(0)
 
-/* HCRX_EL2 definitions */
-#define HCRX_EL2_SMPME_MASK		(1 << 5)
-
 /* GIC Hypervisor interface registers */
 /* ICH_MISR_EL2 bit definitions */
 #define ICH_MISR_EOI		(1 << 0)
diff --git a/arch/arm64/include/asm/system_misc.h b/arch/arm64/include/asm/system_misc.h
index 0eb7709422e2..c34344256762 100644
--- a/arch/arm64/include/asm/system_misc.h
+++ b/arch/arm64/include/asm/system_misc.h
@@ -18,7 +18,7 @@
 
 struct pt_regs;
 
-void die(const char *msg, struct pt_regs *regs, int err);
+void die(const char *msg, struct pt_regs *regs, long err);
 
 struct siginfo;
 void arm64_notify_die(const char *str, struct pt_regs *regs,
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index 2fc9f0861769..5c7b2f9d5913 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -203,9 +203,11 @@ static inline void uaccess_enable_privileged(void)
 }
 
 /*
- * Sanitise a uaccess pointer such that it becomes NULL if above the maximum
- * user address. In case the pointer is tagged (has the top byte set), untag
- * the pointer before checking.
+ * Sanitize a uaccess pointer such that it cannot reach any kernel address.
+ *
+ * Clearing bit 55 ensures the pointer cannot address any portion of the TTBR1
+ * address range (i.e. any kernel address), and either the pointer falls within
+ * the TTBR0 address range or must cause a fault.
  */
 #define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr)
 static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
@@ -213,14 +215,12 @@ static inline void __user *__uaccess_mask_ptr(const void __user *ptr)
 	void __user *safe_ptr;
 
 	asm volatile(
-	"	bics	xzr, %3, %2\n"
-	"	csel	%0, %1, xzr, eq\n"
-	: "=&r" (safe_ptr)
-	: "r" (ptr), "r" (TASK_SIZE_MAX - 1),
-	  "r" (untagged_addr(ptr))
-	: "cc");
-
-	csdb();
+	"	bic	%0, %1, %2\n"
+	: "=r" (safe_ptr)
+	: "r" (ptr),
+	  "i" (BIT(55))
+	);
+
 	return safe_ptr;
 }
 
diff --git a/arch/arm64/include/asm/vdso.h b/arch/arm64/include/asm/vdso.h
index f99dcb94b438..b4ae32109932 100644
--- a/arch/arm64/include/asm/vdso.h
+++ b/arch/arm64/include/asm/vdso.h
@@ -26,6 +26,9 @@
 	(void *)(vdso_offset_##name - VDSO_LBASE + (unsigned long)(base)); \
 })
 
+extern char vdso_start[], vdso_end[];
+extern char vdso32_start[], vdso32_end[];
+
 #endif /* !__ASSEMBLY__ */
 
 #endif /* __ASM_VDSO_H */
diff --git a/arch/arm64/include/asm/vdso/gettimeofday.h b/arch/arm64/include/asm/vdso/gettimeofday.h
index 4f7a629df81f..764d13e2916c 100644
--- a/arch/arm64/include/asm/vdso/gettimeofday.h
+++ b/arch/arm64/include/asm/vdso/gettimeofday.h
@@ -7,8 +7,10 @@
 
 #ifndef __ASSEMBLY__
 
+#include <asm/alternative.h>
 #include <asm/barrier.h>
 #include <asm/unistd.h>
+#include <asm/sysreg.h>
 
 #define VDSO_HAS_CLOCK_GETRES		1
 
@@ -78,11 +80,20 @@ static __always_inline u64 __arch_get_hw_counter(s32 clock_mode,
 		return 0;
 
 	/*
-	 * This isb() is required to prevent that the counter value
+	 * If FEAT_ECV is available, use the self-synchronizing counter.
+	 * Otherwise the isb is required to prevent that the counter value
 	 * is speculated.
-	 */
-	isb();
-	asm volatile("mrs %0, cntvct_el0" : "=r" (res) :: "memory");
+	*/
+	asm volatile(
+	ALTERNATIVE("isb\n"
+		    "mrs %0, cntvct_el0",
+		    "nop\n"
+		    __mrs_s("%0", SYS_CNTVCTSS_EL0),
+		    ARM64_HAS_ECV)
+	: "=r" (res)
+	:
+	: "memory");
+
 	arch_counter_enforce_ordering(res);
 
 	return res;
diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index 1ad2568a2569..9b245da6f507 100644
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -92,5 +92,6 @@
 #define HWCAP2_SME_FA64		(1 << 30)
 #define HWCAP2_WFXT		(1UL << 31)
 #define HWCAP2_EBF16		(1UL << 32)
+#define HWCAP2_SVE_EBF16	(1UL << 33)
 
 #endif /* _UAPI__ASM_HWCAP_H */
diff --git a/arch/arm64/include/uapi/asm/perf_regs.h b/arch/arm64/include/uapi/asm/perf_regs.h
index d54daafa89e3..86e556429e0e 100644
--- a/arch/arm64/include/uapi/asm/perf_regs.h
+++ b/arch/arm64/include/uapi/asm/perf_regs.h
@@ -37,5 +37,12 @@ enum perf_event_arm_regs {
 	PERF_REG_ARM64_SP,
 	PERF_REG_ARM64_PC,
 	PERF_REG_ARM64_MAX,
+
+	/* Extended/pseudo registers */
+	PERF_REG_ARM64_VG = 46,				/* SVE Vector Granule */
+	PERF_REG_ARM64_EXTENDED_MAX
 };
+
+#define PERF_REG_EXTENDED_MASK	(1ULL << PERF_REG_ARM64_VG)
+
 #endif /* _ASM_ARM64_PERF_REGS_H */
diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
index 1add7b01efa7..38a0b0291edb 100644
--- a/arch/arm64/kernel/Makefile
+++ b/arch/arm64/kernel/Makefile
@@ -45,6 +45,7 @@ $(obj)/%.stub.o: $(obj)/%.o FORCE
 obj-$(CONFIG_COMPAT)			+= sys32.o signal32.o			\
 					   sys_compat.o
 obj-$(CONFIG_COMPAT)			+= sigreturn32.o
+obj-$(CONFIG_COMPAT_ALIGNMENT_FIXUPS)	+= compat_alignment.o
 obj-$(CONFIG_KUSER_HELPERS)		+= kuser32.o
 obj-$(CONFIG_FUNCTION_TRACER)		+= ftrace.o entry-ftrace.o
 obj-$(CONFIG_MODULES)			+= module.o
diff --git a/arch/arm64/kernel/acpi_parking_protocol.c b/arch/arm64/kernel/acpi_parking_protocol.c
index bfeeb5319abf..b1990e38aed0 100644
--- a/arch/arm64/kernel/acpi_parking_protocol.c
+++ b/arch/arm64/kernel/acpi_parking_protocol.c
@@ -99,7 +99,7 @@ static int acpi_parking_protocol_cpu_boot(unsigned int cpu)
 	 * that read this address need to convert this address to the
 	 * Boot-Loader's endianness before jumping.
 	 */
-	writeq_relaxed(__pa_symbol(function_nocfi(secondary_entry)),
+	writeq_relaxed(__pa_symbol(secondary_entry),
 		       &mailbox->entry_point);
 	writel_relaxed(cpu_entry->gic_cpu_id, &mailbox->cpu_id);
 
diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c
index 9bcaa5eacf16..91263d09ea65 100644
--- a/arch/arm64/kernel/alternative.c
+++ b/arch/arm64/kernel/alternative.c
@@ -10,17 +10,23 @@
 
 #include <linux/init.h>
 #include <linux/cpu.h>
+#include <linux/elf.h>
 #include <asm/cacheflush.h>
 #include <asm/alternative.h>
 #include <asm/cpufeature.h>
 #include <asm/insn.h>
+#include <asm/module.h>
 #include <asm/sections.h>
+#include <asm/vdso.h>
 #include <linux/stop_machine.h>
 
 #define __ALT_PTR(a, f)		((void *)&(a)->f + (a)->f)
 #define ALT_ORIG_PTR(a)		__ALT_PTR(a, orig_offset)
 #define ALT_REPL_PTR(a)		__ALT_PTR(a, alt_offset)
 
+#define ALT_CAP(a)		((a)->cpufeature & ~ARM64_CB_BIT)
+#define ALT_HAS_CB(a)		((a)->cpufeature & ARM64_CB_BIT)
+
 /* Volatile, as we may be patching the guts of READ_ONCE() */
 static volatile int all_alternatives_applied;
 
@@ -133,7 +139,8 @@ static void clean_dcache_range_nopatch(u64 start, u64 end)
 	} while (cur += d_size, cur < end);
 }
 
-static void __nocfi __apply_alternatives(struct alt_region *region, bool is_module,
+static void __apply_alternatives(const struct alt_region *region,
+				 bool is_module,
 				 unsigned long *feature_mask)
 {
 	struct alt_instr *alt;
@@ -142,30 +149,27 @@ static void __nocfi __apply_alternatives(struct alt_region *region, bool is_modu
 
 	for (alt = region->begin; alt < region->end; alt++) {
 		int nr_inst;
+		int cap = ALT_CAP(alt);
 
-		if (!test_bit(alt->cpufeature, feature_mask))
+		if (!test_bit(cap, feature_mask))
 			continue;
 
-		/* Use ARM64_CB_PATCH as an unconditional patch */
-		if (alt->cpufeature < ARM64_CB_PATCH &&
-		    !cpus_have_cap(alt->cpufeature))
+		if (!cpus_have_cap(cap))
 			continue;
 
-		if (alt->cpufeature == ARM64_CB_PATCH)
+		if (ALT_HAS_CB(alt))
 			BUG_ON(alt->alt_len != 0);
 		else
 			BUG_ON(alt->alt_len != alt->orig_len);
 
-		pr_info_once("patching kernel code\n");
-
 		origptr = ALT_ORIG_PTR(alt);
 		updptr = is_module ? origptr : lm_alias(origptr);
 		nr_inst = alt->orig_len / AARCH64_INSN_SIZE;
 
-		if (alt->cpufeature < ARM64_CB_PATCH)
-			alt_cb = patch_alternative;
-		else
+		if (ALT_HAS_CB(alt))
 			alt_cb  = ALT_REPL_PTR(alt);
+		else
+			alt_cb = patch_alternative;
 
 		alt_cb(alt, origptr, updptr, nr_inst);
 
@@ -192,30 +196,55 @@ static void __nocfi __apply_alternatives(struct alt_region *region, bool is_modu
 	}
 }
 
+void apply_alternatives_vdso(void)
+{
+	struct alt_region region;
+	const struct elf64_hdr *hdr;
+	const struct elf64_shdr *shdr;
+	const struct elf64_shdr *alt;
+	DECLARE_BITMAP(all_capabilities, ARM64_NCAPS);
+
+	bitmap_fill(all_capabilities, ARM64_NCAPS);
+
+	hdr = (struct elf64_hdr *)vdso_start;
+	shdr = (void *)hdr + hdr->e_shoff;
+	alt = find_section(hdr, shdr, ".altinstructions");
+	if (!alt)
+		return;
+
+	region = (struct alt_region){
+		.begin	= (void *)hdr + alt->sh_offset,
+		.end	= (void *)hdr + alt->sh_offset + alt->sh_size,
+	};
+
+	__apply_alternatives(&region, false, &all_capabilities[0]);
+}
+
+static const struct alt_region kernel_alternatives = {
+	.begin	= (struct alt_instr *)__alt_instructions,
+	.end	= (struct alt_instr *)__alt_instructions_end,
+};
+
 /*
  * We might be patching the stop_machine state machine, so implement a
  * really simple polling protocol here.
  */
 static int __apply_alternatives_multi_stop(void *unused)
 {
-	struct alt_region region = {
-		.begin	= (struct alt_instr *)__alt_instructions,
-		.end	= (struct alt_instr *)__alt_instructions_end,
-	};
-
 	/* We always have a CPU 0 at this point (__init) */
 	if (smp_processor_id()) {
 		while (!all_alternatives_applied)
 			cpu_relax();
 		isb();
 	} else {
-		DECLARE_BITMAP(remaining_capabilities, ARM64_NPATCHABLE);
+		DECLARE_BITMAP(remaining_capabilities, ARM64_NCAPS);
 
 		bitmap_complement(remaining_capabilities, boot_capabilities,
-				  ARM64_NPATCHABLE);
+				  ARM64_NCAPS);
 
 		BUG_ON(all_alternatives_applied);
-		__apply_alternatives(&region, false, remaining_capabilities);
+		__apply_alternatives(&kernel_alternatives, false,
+				     remaining_capabilities);
 		/* Barriers provided by the cache flushing */
 		all_alternatives_applied = 1;
 	}
@@ -225,6 +254,9 @@ static int __apply_alternatives_multi_stop(void *unused)
 
 void __init apply_alternatives_all(void)
 {
+	pr_info("applying system-wide alternatives\n");
+
+	apply_alternatives_vdso();
 	/* better not try code patching on a live SMP system */
 	stop_machine(__apply_alternatives_multi_stop, NULL, cpu_online_mask);
 }
@@ -236,15 +268,13 @@ void __init apply_alternatives_all(void)
  */
 void __init apply_boot_alternatives(void)
 {
-	struct alt_region region = {
-		.begin	= (struct alt_instr *)__alt_instructions,
-		.end	= (struct alt_instr *)__alt_instructions_end,
-	};
-
 	/* If called on non-boot cpu things could go wrong */
 	WARN_ON(smp_processor_id() != 0);
 
-	__apply_alternatives(&region, false, &boot_capabilities[0]);
+	pr_info("applying boot alternatives\n");
+
+	__apply_alternatives(&kernel_alternatives, false,
+			     &boot_capabilities[0]);
 }
 
 #ifdef CONFIG_MODULES
@@ -254,10 +284,18 @@ void apply_alternatives_module(void *start, size_t length)
 		.begin	= start,
 		.end	= start + length,
 	};
-	DECLARE_BITMAP(all_capabilities, ARM64_NPATCHABLE);
+	DECLARE_BITMAP(all_capabilities, ARM64_NCAPS);
 
-	bitmap_fill(all_capabilities, ARM64_NPATCHABLE);
+	bitmap_fill(all_capabilities, ARM64_NCAPS);
 
 	__apply_alternatives(&region, true, &all_capabilities[0]);
 }
 #endif
+
+noinstr void alt_cb_patch_nops(struct alt_instr *alt, __le32 *origptr,
+			       __le32 *updptr, int nr_inst)
+{
+	for (int i = 0; i < nr_inst; i++)
+		updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
+}
+EXPORT_SYMBOL(alt_cb_patch_nops);
diff --git a/arch/arm64/kernel/compat_alignment.c b/arch/arm64/kernel/compat_alignment.c
new file mode 100644
index 000000000000..5edec2f49ec9
--- /dev/null
+++ b/arch/arm64/kernel/compat_alignment.c
@@ -0,0 +1,387 @@
+// SPDX-License-Identifier: GPL-2.0-only
+// based on arch/arm/mm/alignment.c
+
+#include <linux/compiler.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/perf_event.h>
+#include <linux/uaccess.h>
+
+#include <asm/exception.h>
+#include <asm/ptrace.h>
+#include <asm/traps.h>
+
+/*
+ * 32-bit misaligned trap handler (c) 1998 San Mehat (CCC) -July 1998
+ *
+ * Speed optimisations and better fault handling by Russell King.
+ */
+#define CODING_BITS(i)	(i & 0x0e000000)
+
+#define LDST_P_BIT(i)	(i & (1 << 24))		/* Preindex		*/
+#define LDST_U_BIT(i)	(i & (1 << 23))		/* Add offset		*/
+#define LDST_W_BIT(i)	(i & (1 << 21))		/* Writeback		*/
+#define LDST_L_BIT(i)	(i & (1 << 20))		/* Load			*/
+
+#define LDST_P_EQ_U(i)	((((i) ^ ((i) >> 1)) & (1 << 23)) == 0)
+
+#define LDSTHD_I_BIT(i)	(i & (1 << 22))		/* double/half-word immed */
+
+#define RN_BITS(i)	((i >> 16) & 15)	/* Rn			*/
+#define RD_BITS(i)	((i >> 12) & 15)	/* Rd			*/
+#define RM_BITS(i)	(i & 15)		/* Rm			*/
+
+#define REGMASK_BITS(i)	(i & 0xffff)
+
+#define BAD_INSTR 	0xdeadc0de
+
+/* Thumb-2 32 bit format per ARMv7 DDI0406A A6.3, either f800h,e800h,f800h */
+#define IS_T32(hi16) \
+	(((hi16) & 0xe000) == 0xe000 && ((hi16) & 0x1800))
+
+union offset_union {
+	unsigned long un;
+	  signed long sn;
+};
+
+#define TYPE_ERROR	0
+#define TYPE_FAULT	1
+#define TYPE_LDST	2
+#define TYPE_DONE	3
+
+static void
+do_alignment_finish_ldst(unsigned long addr, u32 instr, struct pt_regs *regs,
+			 union offset_union offset)
+{
+	if (!LDST_U_BIT(instr))
+		offset.un = -offset.un;
+
+	if (!LDST_P_BIT(instr))
+		addr += offset.un;
+
+	if (!LDST_P_BIT(instr) || LDST_W_BIT(instr))
+		regs->regs[RN_BITS(instr)] = addr;
+}
+
+static int
+do_alignment_ldrdstrd(unsigned long addr, u32 instr, struct pt_regs *regs)
+{
+	unsigned int rd = RD_BITS(instr);
+	unsigned int rd2;
+	int load;
+
+	if ((instr & 0xfe000000) == 0xe8000000) {
+		/* ARMv7 Thumb-2 32-bit LDRD/STRD */
+		rd2 = (instr >> 8) & 0xf;
+		load = !!(LDST_L_BIT(instr));
+	} else if (((rd & 1) == 1) || (rd == 14)) {
+		return TYPE_ERROR;
+	} else {
+		load = ((instr & 0xf0) == 0xd0);
+		rd2 = rd + 1;
+	}
+
+	if (load) {
+		unsigned int val, val2;
+
+		if (get_user(val, (u32 __user *)addr) ||
+		    get_user(val2, (u32 __user *)(addr + 4)))
+			return TYPE_FAULT;
+		regs->regs[rd] = val;
+		regs->regs[rd2] = val2;
+	} else {
+		if (put_user(regs->regs[rd], (u32 __user *)addr) ||
+		    put_user(regs->regs[rd2], (u32 __user *)(addr + 4)))
+			return TYPE_FAULT;
+	}
+	return TYPE_LDST;
+}
+
+/*
+ * LDM/STM alignment handler.
+ *
+ * There are 4 variants of this instruction:
+ *
+ * B = rn pointer before instruction, A = rn pointer after instruction
+ *              ------ increasing address ----->
+ *	        |    | r0 | r1 | ... | rx |    |
+ * PU = 01             B                    A
+ * PU = 11        B                    A
+ * PU = 00        A                    B
+ * PU = 10             A                    B
+ */
+static int
+do_alignment_ldmstm(unsigned long addr, u32 instr, struct pt_regs *regs)
+{
+	unsigned int rd, rn, nr_regs, regbits;
+	unsigned long eaddr, newaddr;
+	unsigned int val;
+
+	/* count the number of registers in the mask to be transferred */
+	nr_regs = hweight16(REGMASK_BITS(instr)) * 4;
+
+	rn = RN_BITS(instr);
+	newaddr = eaddr = regs->regs[rn];
+
+	if (!LDST_U_BIT(instr))
+		nr_regs = -nr_regs;
+	newaddr += nr_regs;
+	if (!LDST_U_BIT(instr))
+		eaddr = newaddr;
+
+	if (LDST_P_EQ_U(instr))	/* U = P */
+		eaddr += 4;
+
+	for (regbits = REGMASK_BITS(instr), rd = 0; regbits;
+	     regbits >>= 1, rd += 1)
+		if (regbits & 1) {
+			if (LDST_L_BIT(instr)) {
+				if (get_user(val, (u32 __user *)eaddr))
+					return TYPE_FAULT;
+				if (rd < 15)
+					regs->regs[rd] = val;
+				else
+					regs->pc = val;
+			} else {
+				/*
+				 * The PC register has a bias of +8 in ARM mode
+				 * and +4 in Thumb mode. This means that a read
+				 * of the value of PC should account for this.
+				 * Since Thumb does not permit STM instructions
+				 * to refer to PC, just add 8 here.
+				 */
+				val = (rd < 15) ? regs->regs[rd] : regs->pc + 8;
+				if (put_user(val, (u32 __user *)eaddr))
+					return TYPE_FAULT;
+			}
+			eaddr += 4;
+		}
+
+	if (LDST_W_BIT(instr))
+		regs->regs[rn] = newaddr;
+
+	return TYPE_DONE;
+}
+
+/*
+ * Convert Thumb multi-word load/store instruction forms to equivalent ARM
+ * instructions so we can reuse ARM userland alignment fault fixups for Thumb.
+ *
+ * This implementation was initially based on the algorithm found in
+ * gdb/sim/arm/thumbemu.c. It is basically just a code reduction of same
+ * to convert only Thumb ld/st instruction forms to equivalent ARM forms.
+ *
+ * NOTES:
+ * 1. Comments below refer to ARM ARM DDI0100E Thumb Instruction sections.
+ * 2. If for some reason we're passed an non-ld/st Thumb instruction to
+ *    decode, we return 0xdeadc0de. This should never happen under normal
+ *    circumstances but if it does, we've got other problems to deal with
+ *    elsewhere and we obviously can't fix those problems here.
+ */
+
+static unsigned long thumb2arm(u16 tinstr)
+{
+	u32 L = (tinstr & (1<<11)) >> 11;
+
+	switch ((tinstr & 0xf800) >> 11) {
+	/* 6.6.1 Format 1: */
+	case 0xc000 >> 11:				/* 7.1.51 STMIA */
+	case 0xc800 >> 11:				/* 7.1.25 LDMIA */
+		{
+			u32 Rn = (tinstr & (7<<8)) >> 8;
+			u32 W = ((L<<Rn) & (tinstr&255)) ? 0 : 1<<21;
+
+			return 0xe8800000 | W | (L<<20) | (Rn<<16) |
+				(tinstr&255);
+		}
+
+	/* 6.6.1 Format 2: */
+	case 0xb000 >> 11:				/* 7.1.48 PUSH */
+	case 0xb800 >> 11:				/* 7.1.47 POP */
+		if ((tinstr & (3 << 9)) == 0x0400) {
+			static const u32 subset[4] = {
+				0xe92d0000,	/* STMDB sp!,{registers} */
+				0xe92d4000,	/* STMDB sp!,{registers,lr} */
+				0xe8bd0000,	/* LDMIA sp!,{registers} */
+				0xe8bd8000	/* LDMIA sp!,{registers,pc} */
+			};
+			return subset[(L<<1) | ((tinstr & (1<<8)) >> 8)] |
+			    (tinstr & 255);		/* register_list */
+		}
+		fallthrough;	/* for illegal instruction case */
+
+	default:
+		return BAD_INSTR;
+	}
+}
+
+/*
+ * Convert Thumb-2 32 bit LDM, STM, LDRD, STRD to equivalent instruction
+ * handlable by ARM alignment handler, also find the corresponding handler,
+ * so that we can reuse ARM userland alignment fault fixups for Thumb.
+ *
+ * @pinstr: original Thumb-2 instruction; returns new handlable instruction
+ * @regs: register context.
+ * @poffset: return offset from faulted addr for later writeback
+ *
+ * NOTES:
+ * 1. Comments below refer to ARMv7 DDI0406A Thumb Instruction sections.
+ * 2. Register name Rt from ARMv7 is same as Rd from ARMv6 (Rd is Rt)
+ */
+static void *
+do_alignment_t32_to_handler(u32 *pinstr, struct pt_regs *regs,
+			    union offset_union *poffset)
+{
+	u32 instr = *pinstr;
+	u16 tinst1 = (instr >> 16) & 0xffff;
+	u16 tinst2 = instr & 0xffff;
+
+	switch (tinst1 & 0xffe0) {
+	/* A6.3.5 Load/Store multiple */
+	case 0xe880:		/* STM/STMIA/STMEA,LDM/LDMIA, PUSH/POP T2 */
+	case 0xe8a0:		/* ...above writeback version */
+	case 0xe900:		/* STMDB/STMFD, LDMDB/LDMEA */
+	case 0xe920:		/* ...above writeback version */
+		/* no need offset decision since handler calculates it */
+		return do_alignment_ldmstm;
+
+	case 0xf840:		/* POP/PUSH T3 (single register) */
+		if (RN_BITS(instr) == 13 && (tinst2 & 0x09ff) == 0x0904) {
+			u32 L = !!(LDST_L_BIT(instr));
+			const u32 subset[2] = {
+				0xe92d0000,	/* STMDB sp!,{registers} */
+				0xe8bd0000,	/* LDMIA sp!,{registers} */
+			};
+			*pinstr = subset[L] | (1<<RD_BITS(instr));
+			return do_alignment_ldmstm;
+		}
+		/* Else fall through for illegal instruction case */
+		break;
+
+	/* A6.3.6 Load/store double, STRD/LDRD(immed, lit, reg) */
+	case 0xe860:
+	case 0xe960:
+	case 0xe8e0:
+	case 0xe9e0:
+		poffset->un = (tinst2 & 0xff) << 2;
+		fallthrough;
+
+	case 0xe940:
+	case 0xe9c0:
+		return do_alignment_ldrdstrd;
+
+	/*
+	 * No need to handle load/store instructions up to word size
+	 * since ARMv6 and later CPUs can perform unaligned accesses.
+	 */
+	default:
+		break;
+	}
+	return NULL;
+}
+
+static int alignment_get_arm(struct pt_regs *regs, __le32 __user *ip, u32 *inst)
+{
+	__le32 instr = 0;
+	int fault;
+
+	fault = get_user(instr, ip);
+	if (fault)
+		return fault;
+
+	*inst = __le32_to_cpu(instr);
+	return 0;
+}
+
+static int alignment_get_thumb(struct pt_regs *regs, __le16 __user *ip, u16 *inst)
+{
+	__le16 instr = 0;
+	int fault;
+
+	fault = get_user(instr, ip);
+	if (fault)
+		return fault;
+
+	*inst = __le16_to_cpu(instr);
+	return 0;
+}
+
+int do_compat_alignment_fixup(unsigned long addr, struct pt_regs *regs)
+{
+	union offset_union offset;
+	unsigned long instrptr;
+	int (*handler)(unsigned long addr, u32 instr, struct pt_regs *regs);
+	unsigned int type;
+	u32 instr = 0;
+	u16 tinstr = 0;
+	int isize = 4;
+	int thumb2_32b = 0;
+	int fault;
+
+	instrptr = instruction_pointer(regs);
+
+	if (compat_thumb_mode(regs)) {
+		__le16 __user *ptr = (__le16 __user *)(instrptr & ~1);
+
+		fault = alignment_get_thumb(regs, ptr, &tinstr);
+		if (!fault) {
+			if (IS_T32(tinstr)) {
+				/* Thumb-2 32-bit */
+				u16 tinst2;
+				fault = alignment_get_thumb(regs, ptr + 1, &tinst2);
+				instr = ((u32)tinstr << 16) | tinst2;
+				thumb2_32b = 1;
+			} else {
+				isize = 2;
+				instr = thumb2arm(tinstr);
+			}
+		}
+	} else {
+		fault = alignment_get_arm(regs, (__le32 __user *)instrptr, &instr);
+	}
+
+	if (fault)
+		return 1;
+
+	switch (CODING_BITS(instr)) {
+	case 0x00000000:	/* 3.13.4 load/store instruction extensions */
+		if (LDSTHD_I_BIT(instr))
+			offset.un = (instr & 0xf00) >> 4 | (instr & 15);
+		else
+			offset.un = regs->regs[RM_BITS(instr)];
+
+		if ((instr & 0x001000f0) == 0x000000d0 || /* LDRD */
+		    (instr & 0x001000f0) == 0x000000f0)   /* STRD */
+			handler = do_alignment_ldrdstrd;
+		else
+			return 1;
+		break;
+
+	case 0x08000000:	/* ldm or stm, or thumb-2 32bit instruction */
+		if (thumb2_32b) {
+			offset.un = 0;
+			handler = do_alignment_t32_to_handler(&instr, regs, &offset);
+		} else {
+			offset.un = 0;
+			handler = do_alignment_ldmstm;
+		}
+		break;
+
+	default:
+		return 1;
+	}
+
+	type = handler(addr, instr, regs);
+
+	if (type == TYPE_ERROR || type == TYPE_FAULT)
+		return 1;
+
+	if (type == TYPE_LDST)
+		do_alignment_finish_ldst(addr, instr, regs, offset);
+
+	perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, regs->pc);
+	arm64_skip_faulting_instruction(regs, isize);
+
+	return 0;
+}
diff --git a/arch/arm64/kernel/cpu-reset.S b/arch/arm64/kernel/cpu-reset.S
index 48a8af97faa9..6b752fe89745 100644
--- a/arch/arm64/kernel/cpu-reset.S
+++ b/arch/arm64/kernel/cpu-reset.S
@@ -8,6 +8,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/assembler.h>
 #include <asm/sysreg.h>
 #include <asm/virt.h>
@@ -28,7 +29,7 @@
  * branch to what would be the reset vector. It must be executed with the
  * flat identity mapping.
  */
-SYM_CODE_START(cpu_soft_restart)
+SYM_TYPED_FUNC_START(cpu_soft_restart)
 	mov_q	x12, INIT_SCTLR_EL1_MMU_OFF
 	pre_disable_mmu_workaround
 	/*
@@ -47,6 +48,6 @@ SYM_CODE_START(cpu_soft_restart)
 	mov	x1, x3				// arg1
 	mov	x2, x4				// arg2
 	br	x8
-SYM_CODE_END(cpu_soft_restart)
+SYM_FUNC_END(cpu_soft_restart)
 
 .popsection
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 53b973b6059f..58ca4f6b25d6 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -121,6 +121,22 @@ cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused)
 	sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0);
 }
 
+static DEFINE_RAW_SPINLOCK(reg_user_mask_modification);
+static void __maybe_unused
+cpu_clear_bf16_from_user_emulation(const struct arm64_cpu_capabilities *__unused)
+{
+	struct arm64_ftr_reg *regp;
+
+	regp = get_arm64_ftr_reg(SYS_ID_AA64ISAR1_EL1);
+	if (!regp)
+		return;
+
+	raw_spin_lock(&reg_user_mask_modification);
+	if (regp->user_mask & ID_AA64ISAR1_EL1_BF16_MASK)
+		regp->user_mask &= ~ID_AA64ISAR1_EL1_BF16_MASK;
+	raw_spin_unlock(&reg_user_mask_modification);
+}
+
 #define CAP_MIDR_RANGE(model, v_min, r_min, v_max, r_max)	\
 	.matches = is_affected_midr_range,			\
 	.midr_range = MIDR_RANGE(model, v_min, r_min, v_max, r_max)
@@ -692,6 +708,16 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
 		.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
 	},
 #endif
+#ifdef CONFIG_ARM64_ERRATUM_2658417
+	{
+		.desc = "ARM erratum 2658417",
+		.capability = ARM64_WORKAROUND_2658417,
+		/* Cortex-A510 r0p0 - r1p1 */
+		ERRATA_MIDR_RANGE(MIDR_CORTEX_A510, 0, 0, 1, 1),
+		MIDR_FIXED(MIDR_CPU_VAR_REV(1,1), BIT(25)),
+		.cpu_enable = cpu_clear_bf16_from_user_emulation,
+	},
+#endif
 	{
 	}
 };
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index af4de817d712..6062454a9067 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -108,8 +108,7 @@ DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
 EXPORT_SYMBOL(cpu_hwcaps);
 static struct arm64_cpu_capabilities const __ro_after_init *cpu_hwcaps_ptrs[ARM64_NCAPS];
 
-/* Need also bit for ARM64_CB_PATCH */
-DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE);
+DECLARE_BITMAP(boot_capabilities, ARM64_NCAPS);
 
 bool arm64_use_ng_mappings = false;
 EXPORT_SYMBOL(arm64_use_ng_mappings);
@@ -134,31 +133,12 @@ DEFINE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);
  */
 static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly;
 
-/*
- * Flag to indicate if we have computed the system wide
- * capabilities based on the boot time active CPUs. This
- * will be used to determine if a new booting CPU should
- * go through the verification process to make sure that it
- * supports the system capabilities, without using a hotplug
- * notifier. This is also used to decide if we could use
- * the fast path for checking constant CPU caps.
- */
-DEFINE_STATIC_KEY_FALSE(arm64_const_caps_ready);
-EXPORT_SYMBOL(arm64_const_caps_ready);
-static inline void finalize_system_capabilities(void)
-{
-	static_branch_enable(&arm64_const_caps_ready);
-}
-
 void dump_cpu_features(void)
 {
 	/* file-wide pr_fmt adds "CPU features: " prefix */
 	pr_emerg("0x%*pb\n", ARM64_NCAPS, &cpu_hwcaps);
 }
 
-DEFINE_STATIC_KEY_ARRAY_FALSE(cpu_hwcap_keys, ARM64_NCAPS);
-EXPORT_SYMBOL(cpu_hwcap_keys);
-
 #define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
 	{						\
 		.sign = SIGNED,				\
@@ -243,35 +223,35 @@ static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
 };
 
 static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_DIT_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_AMU_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_MPAM_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SEL2_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV3_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV2_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_DIT_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_AMU_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_MPAM_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SEL2_SHIFT, 4, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
-				   FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_RAS_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_GIC_SHIFT, 4, 0),
-	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_ASIMD_SHIFT, 4, ID_AA64PFR0_ASIMD_NI),
-	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_FP_SHIFT, 4, ID_AA64PFR0_FP_NI),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL3_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL2_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SHIFT, 4, ID_AA64PFR0_ELx_64BIT_ONLY),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL0_SHIFT, 4, ID_AA64PFR0_ELx_64BIT_ONLY),
+				   FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SVE_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_RAS_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_GIC_SHIFT, 4, 0),
+	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_AdvSIMD_SHIFT, 4, ID_AA64PFR0_EL1_AdvSIMD_NI),
+	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_FP_SHIFT, 4, ID_AA64PFR0_EL1_FP_NI),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL3_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL2_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL1_SHIFT, 4, ID_AA64PFR0_EL1_ELx_64BIT_ONLY),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL0_SHIFT, 4, ID_AA64PFR0_EL1_ELx_64BIT_ONLY),
 	ARM64_FTR_END,
 };
 
 static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
-		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SME_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_MPAMFRAC_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_RASFRAC_SHIFT, 4, 0),
+		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_SME_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MPAM_frac_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_RAS_frac_SHIFT, 4, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_MTE),
-		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_MTE_SHIFT, 4, ID_AA64PFR1_MTE_NI),
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR1_SSBS_SHIFT, 4, ID_AA64PFR1_SSBS_PSTATE_NI),
+		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MTE_SHIFT, 4, ID_AA64PFR1_EL1_MTE_NI),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_SSBS_SHIFT, 4, ID_AA64PFR1_EL1_SSBS_NI),
 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_BTI),
-				    FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_BT_SHIFT, 4, 0),
+				    FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_BT_SHIFT, 4, 0),
 	ARM64_FTR_END,
 };
 
@@ -316,9 +296,9 @@ static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = {
 };
 
 static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ECV_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_FGT_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EXS_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_FGT_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_EXS_SHIFT, 4, 0),
 	/*
 	 * Page size not being supported at Stage-2 is not fatal. You
 	 * just give up KVM if PAGE_SIZE isn't supported there. Go fix
@@ -334,9 +314,9 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
 	 * fields are inconsistent across vCPUs, then it isn't worth
 	 * trying to bring KVM up.
 	 */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN4_2_SHIFT, 4, 1),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN64_2_SHIFT, 4, 1),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_TGRAN16_2_SHIFT, 4, 1),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT, 4, 1),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT, 4, 1),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN16_2_SHIFT, 4, 1),
 	/*
 	 * We already refuse to boot CPUs that don't support our configured
 	 * page size, so we can only detect mismatches for a page size other
@@ -344,55 +324,55 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
 	 * exist in the wild so, even though we don't like it, we'll have to go
 	 * along with it and treat them as non-strict.
 	 */
-	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN4_SHIFT, 4, ID_AA64MMFR0_TGRAN4_NI),
-	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN64_SHIFT, 4, ID_AA64MMFR0_TGRAN64_NI),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_TGRAN16_SHIFT, 4, ID_AA64MMFR0_TGRAN16_NI),
+	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN4_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN4_NI),
+	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN64_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN64_NI),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN16_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN16_NI),
 
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL0_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_BIGENDEL0_SHIFT, 4, 0),
 	/* Linux shouldn't care about secure memory */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_SNSMEM_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_BIGENDEL_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_ASID_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_SNSMEM_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_BIGEND_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT, 4, 0),
 	/*
 	 * Differing PARange is fine as long as all peripherals and memory are mapped
 	 * within the minimum PARange of all CPUs
 	 */
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_PARANGE_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_PARANGE_SHIFT, 4, 0),
 	ARM64_FTR_END,
 };
 
 static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_TIDCP1_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_AFP_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_ETS_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_TWED_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_XNX_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64MMFR1_SPECSEI_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_PAN_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_LOR_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HPD_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VHE_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_VMIDBITS_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_HADBS_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TIDCP1_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_ETS_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TWED_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_XNX_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64MMFR1_EL1_SpecSEI_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_PAN_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_LO_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HPDS_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_VH_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_VMIDBits_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HAFDBS_SHIFT, 4, 0),
 	ARM64_FTR_END,
 };
 
 static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_E0PD_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EVT_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_BBM_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_TTL_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_FWB_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IDS_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_AT_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_ST_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_NV_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CCIDX_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LVA_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_IESB_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_LSM_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_UAO_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_CNP_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_E0PD_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_EVT_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_BBM_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_TTL_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_FWB_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_IDS_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_AT_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_ST_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_NV_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_CCIDX_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_VARange_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_IESB_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_LSM_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_UAO_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_CnP_SHIFT, 4, 0),
 	ARM64_FTR_END,
 };
 
@@ -434,17 +414,17 @@ static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
 };
 
 static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
-	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_DOUBLELOCK_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_PMSVER_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_CTX_CMPS_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_WRPS_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_BRPS_SHIFT, 4, 0),
+	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_DoubleLock_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_PMSVer_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_CTX_CMPs_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_WRPs_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_BRPs_SHIFT, 4, 0),
 	/*
 	 * We can instantiate multiple PMU instances with different levels
 	 * of support.
 	 */
-	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_PMUVER_SHIFT, 4, 0),
-	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_DEBUGVER_SHIFT, 4, 0x6),
+	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_EL1_PMUVer_SHIFT, 4, 0),
+	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_EL1_DebugVer_SHIFT, 4, 0x6),
 	ARM64_FTR_END,
 };
 
@@ -750,7 +730,7 @@ static struct arm64_ftr_reg *get_arm64_ftr_reg_nowarn(u32 sys_id)
  * returns - Upon success,  matching ftr_reg entry for id.
  *         - NULL on failure but with an WARN_ON().
  */
-static struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
+struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
 {
 	struct arm64_ftr_reg *reg;
 
@@ -1392,6 +1372,12 @@ u64 __read_sysreg_by_encoding(u32 sys_id)
 #include <linux/irqchip/arm-gic-v3.h>
 
 static bool
+has_always(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	return true;
+}
+
+static bool
 feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
 {
 	int val = cpuid_feature_extract_field_width(reg, entry->field_pos,
@@ -1401,20 +1387,43 @@ feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
 	return val >= entry->min_field_value;
 }
 
-static bool
-has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
+static u64
+read_scoped_sysreg(const struct arm64_cpu_capabilities *entry, int scope)
 {
-	u64 val;
-
 	WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
 	if (scope == SCOPE_SYSTEM)
-		val = read_sanitised_ftr_reg(entry->sys_reg);
+		return read_sanitised_ftr_reg(entry->sys_reg);
 	else
-		val = __read_sysreg_by_encoding(entry->sys_reg);
+		return __read_sysreg_by_encoding(entry->sys_reg);
+}
+
+static bool
+has_user_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	int mask;
+	struct arm64_ftr_reg *regp;
+	u64 val = read_scoped_sysreg(entry, scope);
+
+	regp = get_arm64_ftr_reg(entry->sys_reg);
+	if (!regp)
+		return false;
+
+	mask = cpuid_feature_extract_unsigned_field_width(regp->user_mask,
+							  entry->field_pos,
+							  entry->field_width);
+	if (!mask)
+		return false;
 
 	return feature_matches(val, entry);
 }
 
+static bool
+has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
+{
+	u64 val = read_scoped_sysreg(entry, scope);
+	return feature_matches(val, entry);
+}
+
 const struct cpumask *system_32bit_el0_cpumask(void)
 {
 	if (!system_supports_32bit_el0())
@@ -1492,7 +1501,7 @@ static bool has_no_fpsimd(const struct arm64_cpu_capabilities *entry, int __unus
 	u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
 
 	return cpuid_feature_extract_signed_field(pfr0,
-					ID_AA64PFR0_FP_SHIFT) < 0;
+					ID_AA64PFR0_EL1_FP_SHIFT) < 0;
 }
 
 static bool has_cache_idc(const struct arm64_cpu_capabilities *entry,
@@ -1571,7 +1580,7 @@ bool kaslr_requires_kpti(void)
 	if (IS_ENABLED(CONFIG_ARM64_E0PD)) {
 		u64 mmfr2 = read_sysreg_s(SYS_ID_AA64MMFR2_EL1);
 		if (cpuid_feature_extract_unsigned_field(mmfr2,
-						ID_AA64MMFR2_E0PD_SHIFT))
+						ID_AA64MMFR2_EL1_E0PD_SHIFT))
 			return false;
 	}
 
@@ -1685,7 +1694,7 @@ static phys_addr_t kpti_ng_pgd_alloc(int shift)
 	return kpti_ng_temp_alloc;
 }
 
-static void __nocfi
+static void
 kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
 {
 	typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long);
@@ -1713,7 +1722,7 @@ kpti_install_ng_mappings(const struct arm64_cpu_capabilities *__unused)
 	if (arm64_use_ng_mappings)
 		return;
 
-	remap_fn = (void *)__pa_symbol(function_nocfi(idmap_kpti_install_ng_mappings));
+	remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
 
 	if (!cpu) {
 		alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
@@ -2034,7 +2043,8 @@ static void bti_enable(const struct arm64_cpu_capabilities *__unused)
 static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap)
 {
 	sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0);
-	isb();
+
+	mte_cpu_setup();
 
 	/*
 	 * Clear the tags in the zero page. This needs to be done via the
@@ -2088,12 +2098,22 @@ cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap)
 
 static const struct arm64_cpu_capabilities arm64_features[] = {
 	{
+		.capability = ARM64_ALWAYS_BOOT,
+		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
+		.matches = has_always,
+	},
+	{
+		.capability = ARM64_ALWAYS_SYSTEM,
+		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
+		.matches = has_always,
+	},
+	{
 		.desc = "GIC system register CPU interface",
 		.capability = ARM64_HAS_SYSREG_GIC_CPUIF,
 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
 		.matches = has_useable_gicv3_cpuif,
 		.sys_reg = SYS_ID_AA64PFR0_EL1,
-		.field_pos = ID_AA64PFR0_GIC_SHIFT,
+		.field_pos = ID_AA64PFR0_EL1_GIC_SHIFT,
 		.field_width = 4,
 		.sign = FTR_UNSIGNED,
 		.min_field_value = 1,
@@ -2104,7 +2124,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
 		.matches = has_cpuid_feature,
 		.sys_reg = SYS_ID_AA64MMFR0_EL1,
-		.field_pos = ID_AA64MMFR0_ECV_SHIFT,
+		.field_pos = ID_AA64MMFR0_EL1_ECV_SHIFT,
 		.field_width = 4,
 		.sign = FTR_UNSIGNED,
 		.min_field_value = 1,
@@ -2116,7 +2136,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
 		.matches = has_cpuid_feature,
 		.sys_reg = SYS_ID_AA64MMFR1_EL1,
-		.field_pos = ID_AA64MMFR1_PAN_SHIFT,
+		.field_pos = ID_AA64MMFR1_EL1_PAN_SHIFT,
 		.field_width = 4,
 		.sign = FTR_UNSIGNED,
 		.min_field_value = 1,
@@ -2130,7 +2150,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
 		.matches = has_cpuid_feature,
 		.sys_reg = SYS_ID_AA64MMFR1_EL1,
-		.field_pos = ID_AA64MMFR1_PAN_SHIFT,
+		.field_pos = ID_AA64MMFR1_EL1_PAN_SHIFT,
 		.field_width = 4,
 		.sign = FTR_UNSIGNED,
 		.min_field_value = 3,
@@ -2168,9 +2188,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.matches = has_32bit_el0,
 		.sys_reg = SYS_ID_AA64PFR0_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64PFR0_EL0_SHIFT,
+		.field_pos = ID_AA64PFR0_EL1_EL0_SHIFT,
 		.field_width = 4,
-		.min_field_value = ID_AA64PFR0_ELx_32BIT_64BIT,
+		.min_field_value = ID_AA64PFR0_EL1_ELx_32BIT_64BIT,
 	},
 #ifdef CONFIG_KVM
 	{
@@ -2180,9 +2200,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.matches = has_cpuid_feature,
 		.sys_reg = SYS_ID_AA64PFR0_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64PFR0_EL1_SHIFT,
+		.field_pos = ID_AA64PFR0_EL1_EL1_SHIFT,
 		.field_width = 4,
-		.min_field_value = ID_AA64PFR0_ELx_32BIT_64BIT,
+		.min_field_value = ID_AA64PFR0_EL1_ELx_32BIT_64BIT,
 	},
 	{
 		.desc = "Protected KVM",
@@ -2201,7 +2221,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		 * more details.
 		 */
 		.sys_reg = SYS_ID_AA64PFR0_EL1,
-		.field_pos = ID_AA64PFR0_CSV3_SHIFT,
+		.field_pos = ID_AA64PFR0_EL1_CSV3_SHIFT,
 		.field_width = 4,
 		.min_field_value = 1,
 		.matches = unmap_kernel_at_el0,
@@ -2244,9 +2264,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.capability = ARM64_SVE,
 		.sys_reg = SYS_ID_AA64PFR0_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64PFR0_SVE_SHIFT,
+		.field_pos = ID_AA64PFR0_EL1_SVE_SHIFT,
 		.field_width = 4,
-		.min_field_value = ID_AA64PFR0_SVE,
+		.min_field_value = ID_AA64PFR0_EL1_SVE_IMP,
 		.matches = has_cpuid_feature,
 		.cpu_enable = sve_kernel_enable,
 	},
@@ -2259,9 +2279,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.matches = has_cpuid_feature,
 		.sys_reg = SYS_ID_AA64PFR0_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64PFR0_RAS_SHIFT,
+		.field_pos = ID_AA64PFR0_EL1_RAS_SHIFT,
 		.field_width = 4,
-		.min_field_value = ID_AA64PFR0_RAS_V1,
+		.min_field_value = ID_AA64PFR0_EL1_RAS_IMP,
 		.cpu_enable = cpu_clear_disr,
 	},
 #endif /* CONFIG_ARM64_RAS_EXTN */
@@ -2278,9 +2298,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.matches = has_amu,
 		.sys_reg = SYS_ID_AA64PFR0_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64PFR0_AMU_SHIFT,
+		.field_pos = ID_AA64PFR0_EL1_AMU_SHIFT,
 		.field_width = 4,
-		.min_field_value = ID_AA64PFR0_AMU,
+		.min_field_value = ID_AA64PFR0_EL1_AMU_IMP,
 		.cpu_enable = cpu_amu_enable,
 	},
 #endif /* CONFIG_ARM64_AMU_EXTN */
@@ -2303,7 +2323,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.capability = ARM64_HAS_STAGE2_FWB,
 		.sys_reg = SYS_ID_AA64MMFR2_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64MMFR2_FWB_SHIFT,
+		.field_pos = ID_AA64MMFR2_EL1_FWB_SHIFT,
 		.field_width = 4,
 		.min_field_value = 1,
 		.matches = has_cpuid_feature,
@@ -2314,7 +2334,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.capability = ARM64_HAS_ARMv8_4_TTL,
 		.sys_reg = SYS_ID_AA64MMFR2_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64MMFR2_TTL_SHIFT,
+		.field_pos = ID_AA64MMFR2_EL1_TTL_SHIFT,
 		.field_width = 4,
 		.min_field_value = 1,
 		.matches = has_cpuid_feature,
@@ -2344,7 +2364,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.capability = ARM64_HW_DBM,
 		.sys_reg = SYS_ID_AA64MMFR1_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64MMFR1_HADBS_SHIFT,
+		.field_pos = ID_AA64MMFR1_EL1_HAFDBS_SHIFT,
 		.field_width = 4,
 		.min_field_value = 2,
 		.matches = has_hw_dbm,
@@ -2367,10 +2387,10 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
 		.matches = has_cpuid_feature,
 		.sys_reg = SYS_ID_AA64PFR1_EL1,
-		.field_pos = ID_AA64PFR1_SSBS_SHIFT,
+		.field_pos = ID_AA64PFR1_EL1_SSBS_SHIFT,
 		.field_width = 4,
 		.sign = FTR_UNSIGNED,
-		.min_field_value = ID_AA64PFR1_SSBS_PSTATE_ONLY,
+		.min_field_value = ID_AA64PFR1_EL1_SSBS_IMP,
 	},
 #ifdef CONFIG_ARM64_CNP
 	{
@@ -2380,7 +2400,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.matches = has_useable_cnp,
 		.sys_reg = SYS_ID_AA64MMFR2_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64MMFR2_CNP_SHIFT,
+		.field_pos = ID_AA64MMFR2_EL1_CnP_SHIFT,
 		.field_width = 4,
 		.min_field_value = 1,
 		.cpu_enable = cpu_enable_cnp,
@@ -2485,7 +2505,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
 		.matches = can_use_gic_priorities,
 		.sys_reg = SYS_ID_AA64PFR0_EL1,
-		.field_pos = ID_AA64PFR0_GIC_SHIFT,
+		.field_pos = ID_AA64PFR0_EL1_GIC_SHIFT,
 		.field_width = 4,
 		.sign = FTR_UNSIGNED,
 		.min_field_value = 1,
@@ -2499,7 +2519,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.sys_reg = SYS_ID_AA64MMFR2_EL1,
 		.sign = FTR_UNSIGNED,
 		.field_width = 4,
-		.field_pos = ID_AA64MMFR2_E0PD_SHIFT,
+		.field_pos = ID_AA64MMFR2_EL1_E0PD_SHIFT,
 		.matches = has_cpuid_feature,
 		.min_field_value = 1,
 		.cpu_enable = cpu_enable_e0pd,
@@ -2528,9 +2548,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.matches = has_cpuid_feature,
 		.cpu_enable = bti_enable,
 		.sys_reg = SYS_ID_AA64PFR1_EL1,
-		.field_pos = ID_AA64PFR1_BT_SHIFT,
+		.field_pos = ID_AA64PFR1_EL1_BT_SHIFT,
 		.field_width = 4,
-		.min_field_value = ID_AA64PFR1_BT_BTI,
+		.min_field_value = ID_AA64PFR1_EL1_BT_IMP,
 		.sign = FTR_UNSIGNED,
 	},
 #endif
@@ -2541,9 +2561,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
 		.matches = has_cpuid_feature,
 		.sys_reg = SYS_ID_AA64PFR1_EL1,
-		.field_pos = ID_AA64PFR1_MTE_SHIFT,
+		.field_pos = ID_AA64PFR1_EL1_MTE_SHIFT,
 		.field_width = 4,
-		.min_field_value = ID_AA64PFR1_MTE,
+		.min_field_value = ID_AA64PFR1_EL1_MTE_MTE2,
 		.sign = FTR_UNSIGNED,
 		.cpu_enable = cpu_enable_mte,
 	},
@@ -2553,9 +2573,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
 		.matches = has_cpuid_feature,
 		.sys_reg = SYS_ID_AA64PFR1_EL1,
-		.field_pos = ID_AA64PFR1_MTE_SHIFT,
+		.field_pos = ID_AA64PFR1_EL1_MTE_SHIFT,
 		.field_width = 4,
-		.min_field_value = ID_AA64PFR1_MTE_ASYMM,
+		.min_field_value = ID_AA64PFR1_EL1_MTE_MTE3,
 		.sign = FTR_UNSIGNED,
 	},
 #endif /* CONFIG_ARM64_MTE */
@@ -2577,9 +2597,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.capability = ARM64_SME,
 		.sys_reg = SYS_ID_AA64PFR1_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64PFR1_SME_SHIFT,
+		.field_pos = ID_AA64PFR1_EL1_SME_SHIFT,
 		.field_width = 4,
-		.min_field_value = ID_AA64PFR1_SME,
+		.min_field_value = ID_AA64PFR1_EL1_SME_IMP,
 		.matches = has_cpuid_feature,
 		.cpu_enable = sme_kernel_enable,
 	},
@@ -2614,9 +2634,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
 		.sys_reg = SYS_ID_AA64MMFR1_EL1,
 		.sign = FTR_UNSIGNED,
-		.field_pos = ID_AA64MMFR1_TIDCP1_SHIFT,
+		.field_pos = ID_AA64MMFR1_EL1_TIDCP1_SHIFT,
 		.field_width = 4,
-		.min_field_value = ID_AA64MMFR1_TIDCP1_IMP,
+		.min_field_value = ID_AA64MMFR1_EL1_TIDCP1_IMP,
 		.matches = has_cpuid_feature,
 		.cpu_enable = cpu_trap_el0_impdef,
 	},
@@ -2624,7 +2644,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 };
 
 #define HWCAP_CPUID_MATCH(reg, field, width, s, min_value)			\
-		.matches = has_cpuid_feature,					\
+		.matches = has_user_cpuid_feature,					\
 		.sys_reg = reg,							\
 		.field_pos = field,						\
 		.field_width = width,						\
@@ -2708,11 +2728,11 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_EL1_TS_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FLAGM),
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_EL1_TS_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2),
 	HWCAP_CAP(SYS_ID_AA64ISAR0_EL1, ID_AA64ISAR0_EL1_RNDR_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RNG),
-	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, 4, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_FP),
-	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_FP_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FPHP),
-	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, 4, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_ASIMD),
-	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_ASIMD_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP),
-	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_DIT_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DIT),
+	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_FP_SHIFT, 4, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_FP),
+	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_FP_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_FPHP),
+	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_AdvSIMD_SHIFT, 4, FTR_SIGNED, 0, CAP_HWCAP, KERNEL_HWCAP_ASIMD),
+	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_AdvSIMD_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP),
+	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_DIT_SHIFT, 4, FTR_SIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DIT),
 	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_DPB_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DCPOP),
 	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_DPB_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_DCPODP),
 	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_JSCVT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_JSCVT),
@@ -2725,38 +2745,39 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
 	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_BF16_SHIFT, 4, FTR_UNSIGNED, 2, CAP_HWCAP, KERNEL_HWCAP_EBF16),
 	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_DGH_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_DGH),
 	HWCAP_CAP(SYS_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_I8MM_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_I8MM),
-	HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_AT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_USCAT),
+	HWCAP_CAP(SYS_ID_AA64MMFR2_EL1, ID_AA64MMFR2_EL1_AT_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_USCAT),
 #ifdef CONFIG_ARM64_SVE
-	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_SVE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR0_SVE, CAP_HWCAP, KERNEL_HWCAP_SVE),
+	HWCAP_CAP(SYS_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_SVE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR0_EL1_SVE_IMP, CAP_HWCAP, KERNEL_HWCAP_SVE),
 	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SVEver_SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2),
 	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_AES_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEAES),
 	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_AES_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_AES_PMULL128, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL),
 	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_BitPerm_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_BitPerm_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM),
 	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_BF16_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_BF16_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16),
+	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_BF16_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_BF16_EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16),
 	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SHA3_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SHA3_IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3),
 	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_SM4_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_SM4_IMP, CAP_HWCAP, KERNEL_HWCAP_SVESM4),
 	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_I8MM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_I8MM_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEI8MM),
 	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_F32MM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_F32MM_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF32MM),
 	HWCAP_CAP(SYS_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_F64MM_SHIFT, 4, FTR_UNSIGNED, ID_AA64ZFR0_EL1_F64MM_IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF64MM),
 #endif
-	HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SSBS_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_SSBS_PSTATE_INSNS, CAP_HWCAP, KERNEL_HWCAP_SSBS),
+	HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_SSBS_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_EL1_SSBS_SSBS2, CAP_HWCAP, KERNEL_HWCAP_SSBS),
 #ifdef CONFIG_ARM64_BTI
-	HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_BT_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_BT_BTI, CAP_HWCAP, KERNEL_HWCAP_BTI),
+	HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_BT_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_EL1_BT_IMP, CAP_HWCAP, KERNEL_HWCAP_BTI),
 #endif
 #ifdef CONFIG_ARM64_PTR_AUTH
 	HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, KERNEL_HWCAP_PACA),
 	HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, KERNEL_HWCAP_PACG),
 #endif
 #ifdef CONFIG_ARM64_MTE
-	HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_MTE, CAP_HWCAP, KERNEL_HWCAP_MTE),
-	HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_MTE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_MTE_ASYMM, CAP_HWCAP, KERNEL_HWCAP_MTE3),
+	HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_MTE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_EL1_MTE_MTE2, CAP_HWCAP, KERNEL_HWCAP_MTE),
+	HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_MTE_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_EL1_MTE_MTE3, CAP_HWCAP, KERNEL_HWCAP_MTE3),
 #endif /* CONFIG_ARM64_MTE */
-	HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV),
-	HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP),
+	HWCAP_CAP(SYS_ID_AA64MMFR0_EL1, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_ECV),
+	HWCAP_CAP(SYS_ID_AA64MMFR1_EL1, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_AFP),
 	HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_RPRES_SHIFT, 4, FTR_UNSIGNED, 1, CAP_HWCAP, KERNEL_HWCAP_RPRES),
 	HWCAP_CAP(SYS_ID_AA64ISAR2_EL1, ID_AA64ISAR2_EL1_WFxT_SHIFT, 4, FTR_UNSIGNED, ID_AA64ISAR2_EL1_WFxT_IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT),
 #ifdef CONFIG_ARM64_SME
-	HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_SME_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_SME, CAP_HWCAP, KERNEL_HWCAP_SME),
+	HWCAP_CAP(SYS_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_SME_SHIFT, 4, FTR_UNSIGNED, ID_AA64PFR1_EL1_SME_IMP, CAP_HWCAP, KERNEL_HWCAP_SME),
 	HWCAP_CAP(SYS_ID_AA64SMFR0_EL1, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, FTR_UNSIGNED, ID_AA64SMFR0_EL1_FA64_IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64),
 	HWCAP_CAP(SYS_ID_AA64SMFR0_EL1, ID_AA64SMFR0_EL1_I16I64_SHIFT, 4, FTR_UNSIGNED, ID_AA64SMFR0_EL1_I16I64_IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I64),
 	HWCAP_CAP(SYS_ID_AA64SMFR0_EL1, ID_AA64SMFR0_EL1_F64F64_SHIFT, 1, FTR_UNSIGNED, ID_AA64SMFR0_EL1_F64F64_IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F64F64),
@@ -2929,9 +2950,6 @@ static void __init enable_cpu_capabilities(u16 scope_mask)
 		if (!cpus_have_cap(num))
 			continue;
 
-		/* Ensure cpus_have_const_cap(num) works */
-		static_branch_enable(&cpu_hwcap_keys[num]);
-
 		if (boot_scope && caps->cpu_enable)
 			/*
 			 * Capabilities with SCOPE_BOOT_CPU scope are finalised
@@ -3102,7 +3120,7 @@ static void verify_hyp_capabilities(void)
 
 	/* Verify IPA range */
 	parange = cpuid_feature_extract_unsigned_field(mmfr0,
-				ID_AA64MMFR0_PARANGE_SHIFT);
+				ID_AA64MMFR0_EL1_PARANGE_SHIFT);
 	ipa_max = id_aa64mmfr0_parange_to_phys_shift(parange);
 	if (ipa_max < get_kvm_ipa_limit()) {
 		pr_crit("CPU%d: IPA range mismatch\n", smp_processor_id());
@@ -3253,9 +3271,6 @@ void __init setup_cpu_features(void)
 	sme_setup();
 	minsigstksz_setup();
 
-	/* Advertise that we have computed the system capabilities */
-	finalize_system_capabilities();
-
 	/*
 	 * Check for sane CTR_EL0.CWG value.
 	 */
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index d7702f39b4d3..28d4f442b0bc 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -115,6 +115,7 @@ static const char *const hwcap_str[] = {
 	[KERNEL_HWCAP_SME_FA64]		= "smefa64",
 	[KERNEL_HWCAP_WFXT]		= "wfxt",
 	[KERNEL_HWCAP_EBF16]		= "ebf16",
+	[KERNEL_HWCAP_SVE_EBF16]	= "sveebf16",
 };
 
 #ifdef CONFIG_COMPAT
diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c
index bf9fe71589bc..3da09778267e 100644
--- a/arch/arm64/kernel/debug-monitors.c
+++ b/arch/arm64/kernel/debug-monitors.c
@@ -28,7 +28,7 @@
 u8 debug_monitors_arch(void)
 {
 	return cpuid_feature_extract_unsigned_field(read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1),
-						ID_AA64DFR0_DEBUGVER_SHIFT);
+						ID_AA64DFR0_EL1_DebugVer_SHIFT);
 }
 
 /*
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index c75ca36b4a49..9173fad279af 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -379,11 +379,20 @@ static void noinstr el1_pc(struct pt_regs *regs, unsigned long esr)
 	exit_to_kernel_mode(regs);
 }
 
-static void noinstr el1_undef(struct pt_regs *regs)
+static void noinstr el1_undef(struct pt_regs *regs, unsigned long esr)
 {
 	enter_from_kernel_mode(regs);
 	local_daif_inherit(regs);
-	do_undefinstr(regs);
+	do_undefinstr(regs, esr);
+	local_daif_mask();
+	exit_to_kernel_mode(regs);
+}
+
+static void noinstr el1_bti(struct pt_regs *regs, unsigned long esr)
+{
+	enter_from_kernel_mode(regs);
+	local_daif_inherit(regs);
+	do_el1_bti(regs, esr);
 	local_daif_mask();
 	exit_to_kernel_mode(regs);
 }
@@ -402,7 +411,7 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr)
 {
 	enter_from_kernel_mode(regs);
 	local_daif_inherit(regs);
-	do_ptrauth_fault(regs, esr);
+	do_el1_fpac(regs, esr);
 	local_daif_mask();
 	exit_to_kernel_mode(regs);
 }
@@ -425,7 +434,10 @@ asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
 		break;
 	case ESR_ELx_EC_SYS64:
 	case ESR_ELx_EC_UNKNOWN:
-		el1_undef(regs);
+		el1_undef(regs, esr);
+		break;
+	case ESR_ELx_EC_BTI:
+		el1_bti(regs, esr);
 		break;
 	case ESR_ELx_EC_BREAKPT_CUR:
 	case ESR_ELx_EC_SOFTSTP_CUR:
@@ -582,11 +594,11 @@ static void noinstr el0_sp(struct pt_regs *regs, unsigned long esr)
 	exit_to_user_mode(regs);
 }
 
-static void noinstr el0_undef(struct pt_regs *regs)
+static void noinstr el0_undef(struct pt_regs *regs, unsigned long esr)
 {
 	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
-	do_undefinstr(regs);
+	do_undefinstr(regs, esr);
 	exit_to_user_mode(regs);
 }
 
@@ -594,7 +606,7 @@ static void noinstr el0_bti(struct pt_regs *regs)
 {
 	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
-	do_bti(regs);
+	do_el0_bti(regs);
 	exit_to_user_mode(regs);
 }
 
@@ -629,7 +641,7 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr)
 {
 	enter_from_user_mode(regs);
 	local_daif_restore(DAIF_PROCCTX);
-	do_ptrauth_fault(regs, esr);
+	do_el0_fpac(regs, esr);
 	exit_to_user_mode(regs);
 }
 
@@ -670,7 +682,7 @@ asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
 		el0_pc(regs, esr);
 		break;
 	case ESR_ELx_EC_UNKNOWN:
-		el0_undef(regs);
+		el0_undef(regs, esr);
 		break;
 	case ESR_ELx_EC_BTI:
 		el0_bti(regs);
@@ -788,7 +800,7 @@ asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
 	case ESR_ELx_EC_CP14_MR:
 	case ESR_ELx_EC_CP14_LS:
 	case ESR_ELx_EC_CP14_64:
-		el0_undef(regs);
+		el0_undef(regs, esr);
 		break;
 	case ESR_ELx_EC_CP15_32:
 	case ESR_ELx_EC_CP15_64:
diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 2d73b3e793b2..e28137d64b76 100644
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -114,7 +114,7 @@
 	 * them if required.
 	 */
 	.macro	apply_ssbd, state, tmp1, tmp2
-alternative_cb	spectre_v4_patch_fw_mitigation_enable
+alternative_cb	ARM64_ALWAYS_SYSTEM, spectre_v4_patch_fw_mitigation_enable
 	b	.L__asm_ssbd_skip\@		// Patched to NOP
 alternative_cb_end
 	ldr_this_cpu	\tmp2, arm64_ssbd_callback_required, \tmp1
@@ -123,7 +123,7 @@ alternative_cb_end
 	tbnz	\tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@
 	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_2
 	mov	w1, #\state
-alternative_cb	smccc_patch_fw_mitigation_conduit
+alternative_cb	ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit
 	nop					// Patched to SMC/HVC #0
 alternative_cb_end
 .L__asm_ssbd_skip\@:
@@ -175,7 +175,7 @@ alternative_else_nop_endif
 
 	.macro mte_set_kernel_gcr, tmp, tmp2
 #ifdef CONFIG_KASAN_HW_TAGS
-alternative_cb	kasan_hw_tags_enable
+alternative_cb	ARM64_ALWAYS_SYSTEM, kasan_hw_tags_enable
 	b	1f
 alternative_cb_end
 	mov	\tmp, KERNEL_GCR_EL1
@@ -186,7 +186,7 @@ alternative_cb_end
 
 	.macro mte_set_user_gcr, tsk, tmp, tmp2
 #ifdef CONFIG_KASAN_HW_TAGS
-alternative_cb	kasan_hw_tags_enable
+alternative_cb	ARM64_ALWAYS_SYSTEM, kasan_hw_tags_enable
 	b	1f
 alternative_cb_end
 	ldr	\tmp, [\tsk, #THREAD_MTE_CTRL]
diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c
index ea5dc7c90f46..8745175f4a75 100644
--- a/arch/arm64/kernel/ftrace.c
+++ b/arch/arm64/kernel/ftrace.c
@@ -56,7 +56,7 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
 	unsigned long pc;
 	u32 new;
 
-	pc = (unsigned long)function_nocfi(ftrace_call);
+	pc = (unsigned long)ftrace_call;
 	new = aarch64_insn_gen_branch_imm(pc, (unsigned long)func,
 					  AARCH64_INSN_BRANCH_LINK);
 
@@ -217,11 +217,26 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
 	unsigned long pc = rec->ip;
 	u32 old = 0, new;
 
+	new = aarch64_insn_gen_nop();
+
+	/*
+	 * When using mcount, callsites in modules may have been initalized to
+	 * call an arbitrary module PLT (which redirects to the _mcount stub)
+	 * rather than the ftrace PLT we'll use at runtime (which redirects to
+	 * the ftrace trampoline). We can ignore the old PLT when initializing
+	 * the callsite.
+	 *
+	 * Note: 'mod' is only set at module load time.
+	 */
+	if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) &&
+	    IS_ENABLED(CONFIG_ARM64_MODULE_PLTS) && mod) {
+		return aarch64_insn_patch_text_nosync((void *)pc, new);
+	}
+
 	if (!ftrace_find_callable_addr(rec, mod, &addr))
 		return -EINVAL;
 
 	old = aarch64_insn_gen_branch_imm(pc, addr, AARCH64_INSN_BRANCH_LINK);
-	new = aarch64_insn_gen_nop();
 
 	return ftrace_modify_code(pc, old, new, true);
 }
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 814b6587ccb7..2196aad7b55b 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -99,7 +99,7 @@ SYM_CODE_START(primary_entry)
 	 */
 #if VA_BITS > 48
 	mrs_s	x0, SYS_ID_AA64MMFR2_EL1
-	tst	x0, #0xf << ID_AA64MMFR2_LVA_SHIFT
+	tst	x0, #0xf << ID_AA64MMFR2_EL1_VARange_SHIFT
 	mov	x0, #VA_BITS
 	mov	x25, #VA_BITS_MIN
 	csel	x25, x25, x0, eq
@@ -658,10 +658,10 @@ SYM_FUNC_END(__secondary_too_slow)
  */
 SYM_FUNC_START(__enable_mmu)
 	mrs	x3, ID_AA64MMFR0_EL1
-	ubfx	x3, x3, #ID_AA64MMFR0_TGRAN_SHIFT, 4
-	cmp     x3, #ID_AA64MMFR0_TGRAN_SUPPORTED_MIN
+	ubfx	x3, x3, #ID_AA64MMFR0_EL1_TGRAN_SHIFT, 4
+	cmp     x3, #ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN
 	b.lt    __no_granule_support
-	cmp     x3, #ID_AA64MMFR0_TGRAN_SUPPORTED_MAX
+	cmp     x3, #ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX
 	b.gt    __no_granule_support
 	phys_to_ttbr x2, x2
 	msr	ttbr0_el1, x2			// load TTBR0
@@ -679,7 +679,7 @@ SYM_FUNC_START(__cpu_secondary_check52bitva)
 	b.ne	2f
 
 	mrs_s	x0, SYS_ID_AA64MMFR2_EL1
-	and	x0, x0, #(0xf << ID_AA64MMFR2_LVA_SHIFT)
+	and	x0, x0, #(0xf << ID_AA64MMFR2_EL1_VARange_SHIFT)
 	cbnz	x0, 2f
 
 	update_early_cpu_boot_status \
diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S
index 12c7fad02ae5..2ee18c860f2a 100644
--- a/arch/arm64/kernel/hyp-stub.S
+++ b/arch/arm64/kernel/hyp-stub.S
@@ -98,7 +98,7 @@ SYM_CODE_START_LOCAL(elx_sync)
 SYM_CODE_END(elx_sync)
 
 SYM_CODE_START_LOCAL(__finalise_el2)
-	check_override id_aa64pfr0 ID_AA64PFR0_SVE_SHIFT .Linit_sve .Lskip_sve
+	check_override id_aa64pfr0 ID_AA64PFR0_EL1_SVE_SHIFT .Linit_sve .Lskip_sve
 
 .Linit_sve:	/* SVE register access */
 	mrs	x0, cptr_el2			// Disable SVE traps
@@ -109,7 +109,7 @@ SYM_CODE_START_LOCAL(__finalise_el2)
 	msr_s	SYS_ZCR_EL2, x1			// length for EL1.
 
 .Lskip_sve:
-	check_override id_aa64pfr1 ID_AA64PFR1_SME_SHIFT .Linit_sme .Lskip_sme
+	check_override id_aa64pfr1 ID_AA64PFR1_EL1_SME_SHIFT .Linit_sme .Lskip_sme
 
 .Linit_sme:	/* SME register access and priority mapping */
 	mrs	x0, cptr_el2			// Disable SME traps
@@ -142,7 +142,7 @@ SYM_CODE_START_LOCAL(__finalise_el2)
 	msr_s	SYS_SMPRIMAP_EL2, xzr		// Make all priorities equal
 
 	mrs	x1, id_aa64mmfr1_el1		// HCRX_EL2 present?
-	ubfx	x1, x1, #ID_AA64MMFR1_HCX_SHIFT, #4
+	ubfx	x1, x1, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4
 	cbz	x1, .Lskip_sme
 
 	mrs_s	x1, SYS_HCRX_EL2
@@ -157,7 +157,7 @@ SYM_CODE_START_LOCAL(__finalise_el2)
 	tbnz	x1, #0, 1f
 
 	// Needs to be VHE capable, obviously
-	check_override id_aa64mmfr1 ID_AA64MMFR1_VHE_SHIFT 2f 1f
+	check_override id_aa64mmfr1 ID_AA64MMFR1_EL1_VH_SHIFT 2f 1f
 
 1:	mov_q	x0, HVC_STUB_ERR
 	eret
diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c
index 1b0542c69738..95133765ed29 100644
--- a/arch/arm64/kernel/idreg-override.c
+++ b/arch/arm64/kernel/idreg-override.c
@@ -50,7 +50,7 @@ static const struct ftr_set_desc mmfr1 __initconst = {
 	.name		= "id_aa64mmfr1",
 	.override	= &id_aa64mmfr1_override,
 	.fields		= {
-		FIELD("vh", ID_AA64MMFR1_VHE_SHIFT, mmfr1_vh_filter),
+		FIELD("vh", ID_AA64MMFR1_EL1_VH_SHIFT, mmfr1_vh_filter),
 		{}
 	},
 };
@@ -74,7 +74,7 @@ static const struct ftr_set_desc pfr0 __initconst = {
 	.name		= "id_aa64pfr0",
 	.override	= &id_aa64pfr0_override,
 	.fields		= {
-	        FIELD("sve", ID_AA64PFR0_SVE_SHIFT, pfr0_sve_filter),
+	        FIELD("sve", ID_AA64PFR0_EL1_SVE_SHIFT, pfr0_sve_filter),
 		{}
 	},
 };
@@ -98,9 +98,9 @@ static const struct ftr_set_desc pfr1 __initconst = {
 	.name		= "id_aa64pfr1",
 	.override	= &id_aa64pfr1_override,
 	.fields		= {
-		FIELD("bt", ID_AA64PFR1_BT_SHIFT, NULL ),
-		FIELD("mte", ID_AA64PFR1_MTE_SHIFT, NULL),
-		FIELD("sme", ID_AA64PFR1_SME_SHIFT, pfr1_sme_filter),
+		FIELD("bt", ID_AA64PFR1_EL1_BT_SHIFT, NULL ),
+		FIELD("mte", ID_AA64PFR1_EL1_MTE_SHIFT, NULL),
+		FIELD("sme", ID_AA64PFR1_EL1_SME_SHIFT, pfr1_sme_filter),
 		{}
 	},
 };
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index afa69e04e75e..4aaa5f3d1f65 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -73,6 +73,7 @@ KVM_NVHE_ALIAS(spectre_bhb_patch_loop_iter);
 KVM_NVHE_ALIAS(spectre_bhb_patch_loop_mitigation_enable);
 KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);
 KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);
+KVM_NVHE_ALIAS(alt_cb_patch_nops);
 
 /* Global kernel state accessed by nVHE hyp code. */
 KVM_NVHE_ALIAS(kvm_vgic_global_state);
@@ -89,10 +90,6 @@ KVM_NVHE_ALIAS(__icache_flags);
 /* VMID bits set by the KVM VMID allocator */
 KVM_NVHE_ALIAS(kvm_arm_vmid_bits);
 
-/* Kernel symbols needed for cpus_have_final/const_caps checks. */
-KVM_NVHE_ALIAS(arm64_const_caps_ready);
-KVM_NVHE_ALIAS(cpu_hwcap_keys);
-
 /* Static keys which are set if a vGIC trap should be handled in hyp. */
 KVM_NVHE_ALIAS(vgic_v2_cpuif_trap);
 KVM_NVHE_ALIAS(vgic_v3_cpuif_trap);
diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index bda49430c9ea..38dbd3828f13 100644
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -21,7 +21,9 @@
 #include <linux/seq_file.h>
 #include <linux/vmalloc.h>
 #include <asm/daifflags.h>
+#include <asm/exception.h>
 #include <asm/vmap_stack.h>
+#include <asm/softirq_stack.h>
 
 /* Only access this in an NMI enter/exit */
 DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
@@ -71,6 +73,18 @@ static void init_irq_stacks(void)
 }
 #endif
 
+#ifndef CONFIG_PREEMPT_RT
+static void ____do_softirq(struct pt_regs *regs)
+{
+	__do_softirq();
+}
+
+void do_softirq_own_stack(void)
+{
+	call_on_irq_stack(NULL, ____do_softirq);
+}
+#endif
+
 static void default_handle_irq(struct pt_regs *regs)
 {
 	panic("IRQ taken without a root IRQ handler\n");
diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c
index 19c2d487cb08..ce3d40120f72 100644
--- a/arch/arm64/kernel/machine_kexec.c
+++ b/arch/arm64/kernel/machine_kexec.c
@@ -204,7 +204,7 @@ void machine_kexec(struct kimage *kimage)
 		typeof(cpu_soft_restart) *restart;
 
 		cpu_install_idmap();
-		restart = (void *)__pa_symbol(function_nocfi(cpu_soft_restart));
+		restart = (void *)__pa_symbol(cpu_soft_restart);
 		restart(is_hyp_nvhe(), kimage->start, kimage->arch.dtb_mem,
 			0, 0);
 	} else {
diff --git a/arch/arm64/kernel/module-plts.c b/arch/arm64/kernel/module-plts.c
index a3d0494f25a9..5a0a8f552a61 100644
--- a/arch/arm64/kernel/module-plts.c
+++ b/arch/arm64/kernel/module-plts.c
@@ -37,7 +37,8 @@ struct plt_entry get_plt_entry(u64 dst, void *pc)
 	return plt;
 }
 
-bool plt_entries_equal(const struct plt_entry *a, const struct plt_entry *b)
+static bool plt_entries_equal(const struct plt_entry *a,
+			      const struct plt_entry *b)
 {
 	u64 p, q;
 
diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index f2d4bb14bfab..76b41e4ca9fa 100644
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -476,21 +476,6 @@ overflow:
 	return -ENOEXEC;
 }
 
-static const Elf_Shdr *find_section(const Elf_Ehdr *hdr,
-				    const Elf_Shdr *sechdrs,
-				    const char *name)
-{
-	const Elf_Shdr *s, *se;
-	const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
-
-	for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++) {
-		if (strcmp(name, secstrs + s->sh_name) == 0)
-			return s;
-	}
-
-	return NULL;
-}
-
 static inline void __init_plt(struct plt_entry *plt, unsigned long addr)
 {
 	*plt = get_plt_entry(addr, plt);
diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c
index b2b730233274..aca88470fb69 100644
--- a/arch/arm64/kernel/mte.c
+++ b/arch/arm64/kernel/mte.c
@@ -285,6 +285,49 @@ void mte_thread_switch(struct task_struct *next)
 	mte_check_tfsr_el1();
 }
 
+void mte_cpu_setup(void)
+{
+	u64 rgsr;
+
+	/*
+	 * CnP must be enabled only after the MAIR_EL1 register has been set
+	 * up. Inconsistent MAIR_EL1 between CPUs sharing the same TLB may
+	 * lead to the wrong memory type being used for a brief window during
+	 * CPU power-up.
+	 *
+	 * CnP is not a boot feature so MTE gets enabled before CnP, but let's
+	 * make sure that is the case.
+	 */
+	BUG_ON(read_sysreg(ttbr0_el1) & TTBR_CNP_BIT);
+	BUG_ON(read_sysreg(ttbr1_el1) & TTBR_CNP_BIT);
+
+	/* Normal Tagged memory type at the corresponding MAIR index */
+	sysreg_clear_set(mair_el1,
+			 MAIR_ATTRIDX(MAIR_ATTR_MASK, MT_NORMAL_TAGGED),
+			 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_TAGGED,
+				      MT_NORMAL_TAGGED));
+
+	write_sysreg_s(KERNEL_GCR_EL1, SYS_GCR_EL1);
+
+	/*
+	 * If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then
+	 * RGSR_EL1.SEED must be non-zero for IRG to produce
+	 * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we
+	 * must initialize it.
+	 */
+	rgsr = (read_sysreg(CNTVCT_EL0) & SYS_RGSR_EL1_SEED_MASK) <<
+	       SYS_RGSR_EL1_SEED_SHIFT;
+	if (rgsr == 0)
+		rgsr = 1 << SYS_RGSR_EL1_SEED_SHIFT;
+	write_sysreg_s(rgsr, SYS_RGSR_EL1);
+
+	/* clear any pending tag check faults in TFSR*_EL1 */
+	write_sysreg_s(0, SYS_TFSR_EL1);
+	write_sysreg_s(0, SYS_TFSRE0_EL1);
+
+	local_flush_tlb_all();
+}
+
 void mte_suspend_enter(void)
 {
 	if (!system_supports_mte())
@@ -301,6 +344,14 @@ void mte_suspend_enter(void)
 	mte_check_tfsr_el1();
 }
 
+void mte_suspend_exit(void)
+{
+	if (!system_supports_mte())
+		return;
+
+	mte_cpu_setup();
+}
+
 long set_mte_ctrl(struct task_struct *task, unsigned long arg)
 {
 	u64 mte_ctrl = (~((arg & PR_MTE_TAG_MASK) >> PR_MTE_TAG_SHIFT) &
diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index cb69ff1e6138..7b0643fe2f13 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -390,7 +390,7 @@ static const struct attribute_group armv8_pmuv3_caps_attr_group = {
  */
 static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu)
 {
-	return (cpu_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_5);
+	return (cpu_pmu->pmuver >= ID_AA64DFR0_EL1_PMUVer_V3P5);
 }
 
 static inline bool armv8pmu_event_has_user_read(struct perf_event *event)
@@ -1145,8 +1145,8 @@ static void __armv8pmu_probe_pmu(void *info)
 
 	dfr0 = read_sysreg(id_aa64dfr0_el1);
 	pmuver = cpuid_feature_extract_unsigned_field(dfr0,
-			ID_AA64DFR0_PMUVER_SHIFT);
-	if (pmuver == ID_AA64DFR0_PMUVER_IMP_DEF || pmuver == 0)
+			ID_AA64DFR0_EL1_PMUVer_SHIFT);
+	if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF || pmuver == 0)
 		return;
 
 	cpu_pmu->pmuver = pmuver;
@@ -1172,7 +1172,7 @@ static void __armv8pmu_probe_pmu(void *info)
 			     pmceid, ARMV8_PMUV3_MAX_COMMON_EVENTS);
 
 	/* store PMMIR_EL1 register for sysfs */
-	if (pmuver >= ID_AA64DFR0_PMUVER_8_4 && (pmceid_raw[1] & BIT(31)))
+	if (pmuver >= ID_AA64DFR0_EL1_PMUVer_V3P4 && (pmceid_raw[1] & BIT(31)))
 		cpu_pmu->reg_pmmir = read_cpuid(PMMIR_EL1);
 	else
 		cpu_pmu->reg_pmmir = 0;
diff --git a/arch/arm64/kernel/perf_regs.c b/arch/arm64/kernel/perf_regs.c
index f6f58e6265df..b4eece3eb17d 100644
--- a/arch/arm64/kernel/perf_regs.c
+++ b/arch/arm64/kernel/perf_regs.c
@@ -9,9 +9,27 @@
 #include <asm/perf_regs.h>
 #include <asm/ptrace.h>
 
+static u64 perf_ext_regs_value(int idx)
+{
+	switch (idx) {
+	case PERF_REG_ARM64_VG:
+		if (WARN_ON_ONCE(!system_supports_sve()))
+			return 0;
+
+		/*
+		 * Vector granule is current length in bits of SVE registers
+		 * divided by 64.
+		 */
+		return (task_get_sve_vl(current) * 8) / 64;
+	default:
+		WARN_ON_ONCE(true);
+		return 0;
+	}
+}
+
 u64 perf_reg_value(struct pt_regs *regs, int idx)
 {
-	if (WARN_ON_ONCE((u32)idx >= PERF_REG_ARM64_MAX))
+	if (WARN_ON_ONCE((u32)idx >= PERF_REG_ARM64_EXTENDED_MAX))
 		return 0;
 
 	/*
@@ -51,6 +69,9 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 	if ((u32)idx == PERF_REG_ARM64_PC)
 		return regs->pc;
 
+	if ((u32)idx >= PERF_REG_ARM64_MAX)
+		return perf_ext_regs_value(idx);
+
 	return regs->regs[idx];
 }
 
@@ -58,7 +79,12 @@ u64 perf_reg_value(struct pt_regs *regs, int idx)
 
 int perf_reg_validate(u64 mask)
 {
-	if (!mask || mask & REG_RESERVED)
+	u64 reserved_mask = REG_RESERVED;
+
+	if (system_supports_sve())
+		reserved_mask &= ~(1ULL << PERF_REG_ARM64_VG);
+
+	if (!mask || mask & reserved_mask)
 		return -EINVAL;
 
 	return 0;
diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c
index d1d182320245..c9e4d0720285 100644
--- a/arch/arm64/kernel/probes/kprobes.c
+++ b/arch/arm64/kernel/probes/kprobes.c
@@ -44,13 +44,28 @@ post_kprobe_handler(struct kprobe *, struct kprobe_ctlblk *, struct pt_regs *);
 static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
 {
 	kprobe_opcode_t *addr = p->ainsn.api.insn;
-	void *addrs[] = {addr, addr + 1};
-	u32 insns[] = {p->opcode, BRK64_OPCODE_KPROBES_SS};
 
-	/* prepare insn slot */
-	aarch64_insn_patch_text(addrs, insns, 2);
-
-	flush_icache_range((uintptr_t)addr, (uintptr_t)(addr + MAX_INSN_SIZE));
+	/*
+	 * Prepare insn slot, Mark Rutland points out it depends on a coupe of
+	 * subtleties:
+	 *
+	 * - That the I-cache maintenance for these instructions is complete
+	 *   *before* the kprobe BRK is written (and aarch64_insn_patch_text_nosync()
+	 *   ensures this, but just omits causing a Context-Synchronization-Event
+	 *   on all CPUS).
+	 *
+	 * - That the kprobe BRK results in an exception (and consequently a
+	 *   Context-Synchronoization-Event), which ensures that the CPU will
+	 *   fetch thesingle-step slot instructions *after* this, ensuring that
+	 *   the new instructions are used
+	 *
+	 * It supposes to place ISB after patching to guarantee I-cache maintenance
+	 * is observed on all CPUS, however, single-step slot is installed in
+	 * the BRK exception handler, so it is unnecessary to generate
+	 * Contex-Synchronization-Event via ISB again.
+	 */
+	aarch64_insn_patch_text_nosync(addr, p->opcode);
+	aarch64_insn_patch_text_nosync(addr + 1, BRK64_OPCODE_KPROBES_SS);
 
 	/*
 	 * Needs restoring of return address after stepping xol.
diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
index 40be3a7c2c53..a8ea1637b137 100644
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -168,7 +168,7 @@ static enum mitigation_state spectre_v2_get_cpu_hw_mitigation_state(void)
 
 	/* If the CPU has CSV2 set, we're safe */
 	pfr0 = read_cpuid(ID_AA64PFR0_EL1);
-	if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_CSV2_SHIFT))
+	if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_CSV2_SHIFT))
 		return SPECTRE_UNAFFECTED;
 
 	/* Alternatively, we have a list of unaffected CPUs */
@@ -586,7 +586,7 @@ void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt,
 	if (spectre_v4_mitigations_off())
 		return;
 
-	if (cpus_have_final_cap(ARM64_SSBS))
+	if (cpus_have_cap(ARM64_SSBS))
 		return;
 
 	if (spectre_v4_mitigations_dynamic())
@@ -945,7 +945,7 @@ static bool supports_ecbhb(int scope)
 		mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
 
 	return cpuid_feature_extract_unsigned_field(mmfr1,
-						    ID_AA64MMFR1_ECBHB_SHIFT);
+						    ID_AA64MMFR1_EL1_ECBHB_SHIFT);
 }
 
 bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
@@ -988,6 +988,14 @@ static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
 	isb();
 }
 
+static bool __read_mostly __nospectre_bhb;
+static int __init parse_spectre_bhb_param(char *str)
+{
+	__nospectre_bhb = true;
+	return 0;
+}
+early_param("nospectre_bhb", parse_spectre_bhb_param);
+
 void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
 {
 	bp_hardening_cb_t cpu_cb;
@@ -1001,7 +1009,7 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
 		/* No point mitigating Spectre-BHB alone. */
 	} else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
 		pr_info_once("spectre-bhb mitigation disabled by compile time option\n");
-	} else if (cpu_mitigations_off()) {
+	} else if (cpu_mitigations_off() || __nospectre_bhb) {
 		pr_info_once("spectre-bhb mitigation disabled by command line option\n");
 	} else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
 		state = SPECTRE_MITIGATED;
diff --git a/arch/arm64/kernel/psci.c b/arch/arm64/kernel/psci.c
index ab7f4c476104..29a8e444db83 100644
--- a/arch/arm64/kernel/psci.c
+++ b/arch/arm64/kernel/psci.c
@@ -38,7 +38,7 @@ static int __init cpu_psci_cpu_prepare(unsigned int cpu)
 
 static int cpu_psci_cpu_boot(unsigned int cpu)
 {
-	phys_addr_t pa_secondary_entry = __pa_symbol(function_nocfi(secondary_entry));
+	phys_addr_t pa_secondary_entry = __pa_symbol(secondary_entry);
 	int err = psci_ops.cpu_on(cpu_logical_map(cpu), pa_secondary_entry);
 	if (err)
 		pr_err("failed to boot CPU%d (%d)\n", cpu, err);
diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c
index 041d2ae5c30a..c2fb5755bbec 100644
--- a/arch/arm64/kernel/ptrace.c
+++ b/arch/arm64/kernel/ptrace.c
@@ -121,7 +121,7 @@ static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr)
 {
 	return ((addr & ~(THREAD_SIZE - 1))  ==
 		(kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) ||
-		on_irq_stack(addr, sizeof(unsigned long), NULL);
+		on_irq_stack(addr, sizeof(unsigned long));
 }
 
 /**
@@ -666,10 +666,18 @@ static int fpr_set(struct task_struct *target, const struct user_regset *regset,
 static int tls_get(struct task_struct *target, const struct user_regset *regset,
 		   struct membuf to)
 {
+	int ret;
+
 	if (target == current)
 		tls_preserve_current_state();
 
-	return membuf_store(&to, target->thread.uw.tp_value);
+	ret = membuf_store(&to, target->thread.uw.tp_value);
+	if (system_supports_tpidr2())
+		ret = membuf_store(&to, target->thread.tpidr2_el0);
+	else
+		ret = membuf_zero(&to, sizeof(u64));
+
+	return ret;
 }
 
 static int tls_set(struct task_struct *target, const struct user_regset *regset,
@@ -677,13 +685,20 @@ static int tls_set(struct task_struct *target, const struct user_regset *regset,
 		   const void *kbuf, const void __user *ubuf)
 {
 	int ret;
-	unsigned long tls = target->thread.uw.tp_value;
+	unsigned long tls[2];
 
-	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &tls, 0, -1);
+	tls[0] = target->thread.uw.tp_value;
+	if (system_supports_sme())
+		tls[1] = target->thread.tpidr2_el0;
+
+	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, tls, 0, count);
 	if (ret)
 		return ret;
 
-	target->thread.uw.tp_value = tls;
+	target->thread.uw.tp_value = tls[0];
+	if (system_supports_sme())
+		target->thread.tpidr2_el0 = tls[1];
+
 	return ret;
 }
 
@@ -1390,7 +1405,7 @@ static const struct user_regset aarch64_regsets[] = {
 	},
 	[REGSET_TLS] = {
 		.core_note_type = NT_ARM_TLS,
-		.n = 1,
+		.n = 2,
 		.size = sizeof(void *),
 		.align = sizeof(void *),
 		.regset_get = tls_get,
diff --git a/arch/arm64/kernel/reloc_test_core.c b/arch/arm64/kernel/reloc_test_core.c
index e87a2b7f20f6..99f2ffe9fc05 100644
--- a/arch/arm64/kernel/reloc_test_core.c
+++ b/arch/arm64/kernel/reloc_test_core.c
@@ -48,7 +48,7 @@ static struct {
 	{ "R_AARCH64_PREL16",		relative_data16, (u64)&sym64_rel },
 };
 
-static int reloc_test_init(void)
+static int __init reloc_test_init(void)
 {
 	int i;
 
@@ -67,7 +67,7 @@ static int reloc_test_init(void)
 	return 0;
 }
 
-static void reloc_test_exit(void)
+static void __exit reloc_test_exit(void)
 {
 }
 
diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c
index d20620a1c51a..d56e170e1ca7 100644
--- a/arch/arm64/kernel/sdei.c
+++ b/arch/arm64/kernel/sdei.c
@@ -162,38 +162,6 @@ static int init_sdei_scs(void)
 	return err;
 }
 
-static bool on_sdei_normal_stack(unsigned long sp, unsigned long size,
-				 struct stack_info *info)
-{
-	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
-	unsigned long high = low + SDEI_STACK_SIZE;
-
-	return on_stack(sp, size, low, high, STACK_TYPE_SDEI_NORMAL, info);
-}
-
-static bool on_sdei_critical_stack(unsigned long sp, unsigned long size,
-				   struct stack_info *info)
-{
-	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
-	unsigned long high = low + SDEI_STACK_SIZE;
-
-	return on_stack(sp, size, low, high, STACK_TYPE_SDEI_CRITICAL, info);
-}
-
-bool _on_sdei_stack(unsigned long sp, unsigned long size, struct stack_info *info)
-{
-	if (!IS_ENABLED(CONFIG_VMAP_STACK))
-		return false;
-
-	if (on_sdei_critical_stack(sp, size, info))
-		return true;
-
-	if (on_sdei_normal_stack(sp, size, info))
-		return true;
-
-	return false;
-}
-
 unsigned long sdei_arch_get_entry_point(int conduit)
 {
 	/*
diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c
index 7e1624ecab3c..49029eace3ad 100644
--- a/arch/arm64/kernel/smp_spin_table.c
+++ b/arch/arm64/kernel/smp_spin_table.c
@@ -66,7 +66,7 @@ static int smp_spin_table_cpu_init(unsigned int cpu)
 static int smp_spin_table_cpu_prepare(unsigned int cpu)
 {
 	__le64 __iomem *release_addr;
-	phys_addr_t pa_holding_pen = __pa_symbol(function_nocfi(secondary_holding_pen));
+	phys_addr_t pa_holding_pen = __pa_symbol(secondary_holding_pen);
 
 	if (!cpu_release_addr[cpu])
 		return -ENODEV;
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
index ce190ee18a20..634279b3b03d 100644
--- a/arch/arm64/kernel/stacktrace.c
+++ b/arch/arm64/kernel/stacktrace.c
@@ -68,31 +68,6 @@ static inline void unwind_init_from_task(struct unwind_state *state,
 }
 
 /*
- * We can only safely access per-cpu stacks from current in a non-preemptible
- * context.
- */
-static bool on_accessible_stack(const struct task_struct *tsk,
-				unsigned long sp, unsigned long size,
-				struct stack_info *info)
-{
-	if (info)
-		info->type = STACK_TYPE_UNKNOWN;
-
-	if (on_task_stack(tsk, sp, size, info))
-		return true;
-	if (tsk != current || preemptible())
-		return false;
-	if (on_irq_stack(sp, size, info))
-		return true;
-	if (on_overflow_stack(sp, size, info))
-		return true;
-	if (on_sdei_stack(sp, size, info))
-		return true;
-
-	return false;
-}
-
-/*
  * Unwind from one frame record (A) to the next frame record (B).
  *
  * We terminate early if the location of B indicates a malformed chain of frame
@@ -103,14 +78,13 @@ static int notrace unwind_next(struct unwind_state *state)
 {
 	struct task_struct *tsk = state->task;
 	unsigned long fp = state->fp;
-	struct stack_info info;
 	int err;
 
 	/* Final frame; nothing to unwind */
 	if (fp == (unsigned long)task_pt_regs(tsk)->stackframe)
 		return -ENOENT;
 
-	err = unwind_next_common(state, &info, on_accessible_stack, NULL);
+	err = unwind_next_frame_record(state);
 	if (err)
 		return err;
 
@@ -190,11 +164,47 @@ void show_stack(struct task_struct *tsk, unsigned long *sp, const char *loglvl)
 	barrier();
 }
 
+/*
+ * Per-cpu stacks are only accessible when unwinding the current task in a
+ * non-preemptible context.
+ */
+#define STACKINFO_CPU(name)					\
+	({							\
+		((task == current) && !preemptible())		\
+			? stackinfo_get_##name()		\
+			: stackinfo_get_unknown();		\
+	})
+
+/*
+ * SDEI stacks are only accessible when unwinding the current task in an NMI
+ * context.
+ */
+#define STACKINFO_SDEI(name)					\
+	({							\
+		((task == current) && in_nmi())			\
+			? stackinfo_get_sdei_##name()		\
+			: stackinfo_get_unknown();		\
+	})
+
 noinline notrace void arch_stack_walk(stack_trace_consume_fn consume_entry,
 			      void *cookie, struct task_struct *task,
 			      struct pt_regs *regs)
 {
-	struct unwind_state state;
+	struct stack_info stacks[] = {
+		stackinfo_get_task(task),
+		STACKINFO_CPU(irq),
+#if defined(CONFIG_VMAP_STACK)
+		STACKINFO_CPU(overflow),
+#endif
+#if defined(CONFIG_VMAP_STACK) && defined(CONFIG_ARM_SDE_INTERFACE)
+		STACKINFO_SDEI(normal),
+		STACKINFO_SDEI(critical),
+#endif
+	};
+	struct unwind_state state = {
+		.stacks = stacks,
+		.nr_stacks = ARRAY_SIZE(stacks),
+	};
 
 	if (regs) {
 		if (task != current)
diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index 9135fe0f3df5..8b02d310838f 100644
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -43,6 +43,8 @@ void notrace __cpu_suspend_exit(void)
 {
 	unsigned int cpu = smp_processor_id();
 
+	mte_suspend_exit();
+
 	/*
 	 * We are resuming from reset with the idmap active in TTBR0_EL1.
 	 * We must uninstall the idmap and restore the expected MMU
diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c
index b7fed33981f7..23d281ed7621 100644
--- a/arch/arm64/kernel/traps.c
+++ b/arch/arm64/kernel/traps.c
@@ -26,6 +26,7 @@
 #include <linux/syscalls.h>
 #include <linux/mm_types.h>
 #include <linux/kasan.h>
+#include <linux/cfi.h>
 
 #include <asm/atomic.h>
 #include <asm/bug.h>
@@ -180,12 +181,12 @@ static void dump_kernel_instr(const char *lvl, struct pt_regs *regs)
 
 #define S_SMP " SMP"
 
-static int __die(const char *str, int err, struct pt_regs *regs)
+static int __die(const char *str, long err, struct pt_regs *regs)
 {
 	static int die_counter;
 	int ret;
 
-	pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n",
+	pr_emerg("Internal error: %s: %016lx [#%d]" S_PREEMPT S_SMP "\n",
 		 str, err, ++die_counter);
 
 	/* trap and error numbers are mostly meaningless on ARM */
@@ -206,7 +207,7 @@ static DEFINE_RAW_SPINLOCK(die_lock);
 /*
  * This function is protected against re-entrancy.
  */
-void die(const char *str, struct pt_regs *regs, int err)
+void die(const char *str, struct pt_regs *regs, long err)
 {
 	int ret;
 	unsigned long flags;
@@ -485,7 +486,7 @@ void arm64_notify_segfault(unsigned long addr)
 	force_signal_inject(SIGSEGV, code, addr, 0);
 }
 
-void do_undefinstr(struct pt_regs *regs)
+void do_undefinstr(struct pt_regs *regs, unsigned long esr)
 {
 	/* check for AArch32 breakpoint instructions */
 	if (!aarch32_break_handler(regs))
@@ -494,28 +495,38 @@ void do_undefinstr(struct pt_regs *regs)
 	if (call_undef_hook(regs) == 0)
 		return;
 
-	BUG_ON(!user_mode(regs));
+	if (!user_mode(regs))
+		die("Oops - Undefined instruction", regs, esr);
+
 	force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
 }
 NOKPROBE_SYMBOL(do_undefinstr);
 
-void do_bti(struct pt_regs *regs)
+void do_el0_bti(struct pt_regs *regs)
 {
-	BUG_ON(!user_mode(regs));
 	force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
 }
-NOKPROBE_SYMBOL(do_bti);
 
-void do_ptrauth_fault(struct pt_regs *regs, unsigned long esr)
+void do_el1_bti(struct pt_regs *regs, unsigned long esr)
+{
+	die("Oops - BTI", regs, esr);
+}
+NOKPROBE_SYMBOL(do_el1_bti);
+
+void do_el0_fpac(struct pt_regs *regs, unsigned long esr)
+{
+	force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr);
+}
+
+void do_el1_fpac(struct pt_regs *regs, unsigned long esr)
 {
 	/*
-	 * Unexpected FPAC exception or pointer authentication failure in
-	 * the kernel: kill the task before it does any more harm.
+	 * Unexpected FPAC exception in the kernel: kill the task before it
+	 * does any more harm.
 	 */
-	BUG_ON(!user_mode(regs));
-	force_signal_inject(SIGILL, ILL_ILLOPN, regs->pc, esr);
+	die("Oops - FPAC", regs, esr);
 }
-NOKPROBE_SYMBOL(do_ptrauth_fault);
+NOKPROBE_SYMBOL(do_el1_fpac)
 
 #define __user_cache_maint(insn, address, res)			\
 	if (address >= TASK_SIZE_MAX) {				\
@@ -758,7 +769,7 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs)
 		hook_base = cp15_64_hooks;
 		break;
 	default:
-		do_undefinstr(regs);
+		do_undefinstr(regs, esr);
 		return;
 	}
 
@@ -773,7 +784,7 @@ void do_cp15instr(unsigned long esr, struct pt_regs *regs)
 	 * EL0. Fall back to our usual undefined instruction handler
 	 * so that we handle these consistently.
 	 */
-	do_undefinstr(regs);
+	do_undefinstr(regs, esr);
 }
 NOKPROBE_SYMBOL(do_cp15instr);
 #endif
@@ -793,7 +804,7 @@ void do_sysinstr(unsigned long esr, struct pt_regs *regs)
 	 * back to our usual undefined instruction handler so that we handle
 	 * these consistently.
 	 */
-	do_undefinstr(regs);
+	do_undefinstr(regs, esr);
 }
 NOKPROBE_SYMBOL(do_sysinstr);
 
@@ -970,7 +981,7 @@ static int bug_handler(struct pt_regs *regs, unsigned long esr)
 {
 	switch (report_bug(regs->pc, regs)) {
 	case BUG_TRAP_TYPE_BUG:
-		die("Oops - BUG", regs, 0);
+		die("Oops - BUG", regs, esr);
 		break;
 
 	case BUG_TRAP_TYPE_WARN:
@@ -991,6 +1002,38 @@ static struct break_hook bug_break_hook = {
 	.imm = BUG_BRK_IMM,
 };
 
+#ifdef CONFIG_CFI_CLANG
+static int cfi_handler(struct pt_regs *regs, unsigned long esr)
+{
+	unsigned long target;
+	u32 type;
+
+	target = pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TARGET, esr));
+	type = (u32)pt_regs_read_reg(regs, FIELD_GET(CFI_BRK_IMM_TYPE, esr));
+
+	switch (report_cfi_failure(regs, regs->pc, &target, type)) {
+	case BUG_TRAP_TYPE_BUG:
+		die("Oops - CFI", regs, 0);
+		break;
+
+	case BUG_TRAP_TYPE_WARN:
+		break;
+
+	default:
+		return DBG_HOOK_ERROR;
+	}
+
+	arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
+	return DBG_HOOK_HANDLED;
+}
+
+static struct break_hook cfi_break_hook = {
+	.fn = cfi_handler,
+	.imm = CFI_BRK_IMM_BASE,
+	.mask = CFI_BRK_IMM_MASK,
+};
+#endif /* CONFIG_CFI_CLANG */
+
 static int reserved_fault_handler(struct pt_regs *regs, unsigned long esr)
 {
 	pr_err("%s generated an invalid instruction at %pS!\n",
@@ -1038,7 +1081,7 @@ static int kasan_handler(struct pt_regs *regs, unsigned long esr)
 	 * This is something that might be fixed at some point in the future.
 	 */
 	if (!recover)
-		die("Oops - KASAN", regs, 0);
+		die("Oops - KASAN", regs, esr);
 
 	/* If thread survives, skip over the brk instruction and continue: */
 	arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
@@ -1052,6 +1095,9 @@ static struct break_hook kasan_break_hook = {
 };
 #endif
 
+
+#define esr_comment(esr) ((esr) & ESR_ELx_BRK64_ISS_COMMENT_MASK)
+
 /*
  * Initial handler for AArch64 BRK exceptions
  * This handler only used until debug_traps_init().
@@ -1059,10 +1105,12 @@ static struct break_hook kasan_break_hook = {
 int __init early_brk64(unsigned long addr, unsigned long esr,
 		struct pt_regs *regs)
 {
+#ifdef CONFIG_CFI_CLANG
+	if ((esr_comment(esr) & ~CFI_BRK_IMM_MASK) == CFI_BRK_IMM_BASE)
+		return cfi_handler(regs, esr) != DBG_HOOK_HANDLED;
+#endif
 #ifdef CONFIG_KASAN_SW_TAGS
-	unsigned long comment = esr & ESR_ELx_BRK64_ISS_COMMENT_MASK;
-
-	if ((comment & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
+	if ((esr_comment(esr) & ~KASAN_BRK_MASK) == KASAN_BRK_IMM)
 		return kasan_handler(regs, esr) != DBG_HOOK_HANDLED;
 #endif
 	return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
@@ -1071,6 +1119,9 @@ int __init early_brk64(unsigned long addr, unsigned long esr,
 void __init trap_init(void)
 {
 	register_kernel_break_hook(&bug_break_hook);
+#ifdef CONFIG_CFI_CLANG
+	register_kernel_break_hook(&cfi_break_hook);
+#endif
 	register_kernel_break_hook(&fault_break_hook);
 #ifdef CONFIG_KASAN_SW_TAGS
 	register_kernel_break_hook(&kasan_break_hook);
diff --git a/arch/arm64/kernel/vdso.c b/arch/arm64/kernel/vdso.c
index a61fc4f989b3..ac93a2ee9c07 100644
--- a/arch/arm64/kernel/vdso.c
+++ b/arch/arm64/kernel/vdso.c
@@ -29,9 +29,6 @@
 #include <asm/signal32.h>
 #include <asm/vdso.h>
 
-extern char vdso_start[], vdso_end[];
-extern char vdso32_start[], vdso32_end[];
-
 enum vdso_abi {
 	VDSO_ABI_AA64,
 	VDSO_ABI_AA32,
diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile
index bafbf78fab77..619e2dc7ee14 100644
--- a/arch/arm64/kernel/vdso/Makefile
+++ b/arch/arm64/kernel/vdso/Makefile
@@ -40,7 +40,8 @@ ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO
 # kernel with CONFIG_WERROR enabled.
 CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) \
 				$(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) \
-				$(CC_FLAGS_LTO) -Wmissing-prototypes -Wmissing-declarations
+				$(CC_FLAGS_LTO) $(CC_FLAGS_CFI) \
+				-Wmissing-prototypes -Wmissing-declarations
 KASAN_SANITIZE			:= n
 KCSAN_SANITIZE			:= n
 UBSAN_SANITIZE			:= n
diff --git a/arch/arm64/kernel/vdso/vdso.lds.S b/arch/arm64/kernel/vdso/vdso.lds.S
index e69fb4aaaf3e..6028f1fe2d1c 100644
--- a/arch/arm64/kernel/vdso/vdso.lds.S
+++ b/arch/arm64/kernel/vdso/vdso.lds.S
@@ -48,6 +48,13 @@ SECTIONS
 	PROVIDE (_etext = .);
 	PROVIDE (etext = .);
 
+	. = ALIGN(4);
+	.altinstructions : {
+		__alt_instructions = .;
+		*(.altinstructions)
+		__alt_instructions_end = .;
+	}
+
 	.dynamic	: { *(.dynamic) }		:text	:dynamic
 
 	.rela.dyn	: ALIGN(8) { *(.rela .rela*) }
diff --git a/arch/arm64/kvm/debug.c b/arch/arm64/kvm/debug.c
index 0b28d7db7c76..3f7563d768e2 100644
--- a/arch/arm64/kvm/debug.c
+++ b/arch/arm64/kvm/debug.c
@@ -295,12 +295,12 @@ void kvm_arch_vcpu_load_debug_state_flags(struct kvm_vcpu *vcpu)
 	 * If SPE is present on this CPU and is available at current EL,
 	 * we may need to check if the host state needs to be saved.
 	 */
-	if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_PMSVER_SHIFT) &&
+	if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_PMSVer_SHIFT) &&
 	    !(read_sysreg_s(SYS_PMBIDR_EL1) & BIT(SYS_PMBIDR_EL1_P_SHIFT)))
 		vcpu_set_flag(vcpu, DEBUG_STATE_SAVE_SPE);
 
 	/* Check if we have TRBE implemented and available at the host */
-	if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRBE_SHIFT) &&
+	if (cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceBuffer_SHIFT) &&
 	    !(read_sysreg_s(SYS_TRBIDR_EL1) & TRBIDR_PROG))
 		vcpu_set_flag(vcpu, DEBUG_STATE_SAVE_TRBE);
 }
diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S
index 7839d075729b..8f3f93fa119e 100644
--- a/arch/arm64/kvm/hyp/hyp-entry.S
+++ b/arch/arm64/kvm/hyp/hyp-entry.S
@@ -196,7 +196,7 @@ SYM_CODE_END(__kvm_hyp_vector)
 	sub	sp, sp, #(8 * 4)
 	stp	x2, x3, [sp, #(8 * 0)]
 	stp	x0, x1, [sp, #(8 * 2)]
-	alternative_cb spectre_bhb_patch_wa3
+	alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_wa3
 	/* Patched to mov WA3 when supported */
 	mov	w0, #ARM_SMCCC_ARCH_WORKAROUND_1
 	alternative_cb_end
@@ -216,7 +216,7 @@ SYM_CODE_END(__kvm_hyp_vector)
 	mitigate_spectre_bhb_clear_insn
 	.endif
 	.if \indirect != 0
-	alternative_cb  kvm_patch_vector_branch
+	alternative_cb ARM64_ALWAYS_SYSTEM, kvm_patch_vector_branch
 	/*
 	 * For ARM64_SPECTRE_V3A configurations, these NOPs get replaced with:
 	 *
diff --git a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h
index fa6e466ed57f..07edfc7524c9 100644
--- a/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/fixed_config.h
@@ -35,9 +35,9 @@
  * - Data Independent Timing
  */
 #define PVM_ID_AA64PFR0_ALLOW (\
-	ARM64_FEATURE_MASK(ID_AA64PFR0_FP) | \
-	ARM64_FEATURE_MASK(ID_AA64PFR0_ASIMD) | \
-	ARM64_FEATURE_MASK(ID_AA64PFR0_DIT) \
+	ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_FP) | \
+	ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AdvSIMD) | \
+	ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_DIT) \
 	)
 
 /*
@@ -49,11 +49,11 @@
  *	Supported by KVM
  */
 #define PVM_ID_AA64PFR0_RESTRICT_UNSIGNED (\
-	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL0), ID_AA64PFR0_ELx_64BIT_ONLY) | \
-	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1), ID_AA64PFR0_ELx_64BIT_ONLY) | \
-	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL2), ID_AA64PFR0_ELx_64BIT_ONLY) | \
-	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL3), ID_AA64PFR0_ELx_64BIT_ONLY) | \
-	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_RAS), ID_AA64PFR0_RAS_V1) \
+	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \
+	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \
+	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL2), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \
+	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL3), ID_AA64PFR0_EL1_ELx_64BIT_ONLY) | \
+	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), ID_AA64PFR0_EL1_RAS_IMP) \
 	)
 
 /*
@@ -62,8 +62,8 @@
  * - Speculative Store Bypassing
  */
 #define PVM_ID_AA64PFR1_ALLOW (\
-	ARM64_FEATURE_MASK(ID_AA64PFR1_BT) | \
-	ARM64_FEATURE_MASK(ID_AA64PFR1_SSBS) \
+	ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_BT) | \
+	ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SSBS) \
 	)
 
 /*
@@ -74,10 +74,10 @@
  * - Non-context synchronizing exception entry and exit
  */
 #define PVM_ID_AA64MMFR0_ALLOW (\
-	ARM64_FEATURE_MASK(ID_AA64MMFR0_BIGENDEL) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR0_SNSMEM) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR0_BIGENDEL0) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR0_EXS) \
+	ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_BIGEND) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_SNSMEM) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_BIGENDEL0) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_EXS) \
 	)
 
 /*
@@ -86,8 +86,8 @@
  * - 16-bit ASID
  */
 #define PVM_ID_AA64MMFR0_RESTRICT_UNSIGNED (\
-	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64MMFR0_PARANGE), ID_AA64MMFR0_PARANGE_40) | \
-	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64MMFR0_ASID), ID_AA64MMFR0_ASID_16) \
+	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_PARANGE), ID_AA64MMFR0_EL1_PARANGE_40) | \
+	FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_ASIDBITS), ID_AA64MMFR0_EL1_ASIDBITS_16) \
 	)
 
 /*
@@ -100,12 +100,12 @@
  * - Enhanced Translation Synchronization
  */
 #define PVM_ID_AA64MMFR1_ALLOW (\
-	ARM64_FEATURE_MASK(ID_AA64MMFR1_HADBS) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR1_VMIDBITS) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR1_HPD) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR1_PAN) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR1_SPECSEI) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR1_ETS) \
+	ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_HAFDBS) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_VMIDBits) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_HPDS) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_PAN) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_SpecSEI) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_ETS) \
 	)
 
 /*
@@ -120,14 +120,14 @@
  * - E0PDx mechanism
  */
 #define PVM_ID_AA64MMFR2_ALLOW (\
-	ARM64_FEATURE_MASK(ID_AA64MMFR2_CNP) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR2_UAO) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR2_IESB) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR2_AT) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR2_IDS) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR2_TTL) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR2_BBM) | \
-	ARM64_FEATURE_MASK(ID_AA64MMFR2_E0PD) \
+	ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_CnP) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_UAO) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_IESB) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_AT) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_IDS) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_TTL) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_BBM) | \
+	ARM64_FEATURE_MASK(ID_AA64MMFR2_EL1_E0PD) \
 	)
 
 /*
diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c
index 99c8d8b73e70..85d3b7ae720f 100644
--- a/arch/arm64/kvm/hyp/nvhe/pkvm.c
+++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c
@@ -20,35 +20,35 @@ static void pvm_init_traps_aa64pfr0(struct kvm_vcpu *vcpu)
 	u64 cptr_set = 0;
 
 	/* Protected KVM does not support AArch32 guests. */
-	BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL0),
-		PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_ELx_64BIT_ONLY);
-	BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1),
-		PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_ELx_64BIT_ONLY);
+	BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL0),
+		PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_ELx_64BIT_ONLY);
+	BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1),
+		PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) != ID_AA64PFR0_EL1_ELx_64BIT_ONLY);
 
 	/*
 	 * Linux guests assume support for floating-point and Advanced SIMD. Do
 	 * not change the trapping behavior for these from the KVM default.
 	 */
-	BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_FP),
+	BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_FP),
 				PVM_ID_AA64PFR0_ALLOW));
-	BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_ASIMD),
+	BUILD_BUG_ON(!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AdvSIMD),
 				PVM_ID_AA64PFR0_ALLOW));
 
 	/* Trap RAS unless all current versions are supported */
-	if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_RAS), feature_ids) <
-	    ID_AA64PFR0_RAS_V1P1) {
+	if (FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_RAS), feature_ids) <
+	    ID_AA64PFR0_EL1_RAS_V1P1) {
 		hcr_set |= HCR_TERR | HCR_TEA;
 		hcr_clear |= HCR_FIEN;
 	}
 
 	/* Trap AMU */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_AMU), feature_ids)) {
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU), feature_ids)) {
 		hcr_clear |= HCR_AMVOFFEN;
 		cptr_set |= CPTR_EL2_TAM;
 	}
 
 	/* Trap SVE */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_SVE), feature_ids))
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE), feature_ids))
 		cptr_set |= CPTR_EL2_TZ;
 
 	vcpu->arch.hcr_el2 |= hcr_set;
@@ -66,7 +66,7 @@ static void pvm_init_traps_aa64pfr1(struct kvm_vcpu *vcpu)
 	u64 hcr_clear = 0;
 
 	/* Memory Tagging: Trap and Treat as Untagged if not supported. */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_MTE), feature_ids)) {
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE), feature_ids)) {
 		hcr_set |= HCR_TID5;
 		hcr_clear |= HCR_DCT | HCR_ATA;
 	}
@@ -86,32 +86,32 @@ static void pvm_init_traps_aa64dfr0(struct kvm_vcpu *vcpu)
 	u64 cptr_set = 0;
 
 	/* Trap/constrain PMU */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_PMUVER), feature_ids)) {
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMUVer), feature_ids)) {
 		mdcr_set |= MDCR_EL2_TPM | MDCR_EL2_TPMCR;
 		mdcr_clear |= MDCR_EL2_HPME | MDCR_EL2_MTPME |
 			      MDCR_EL2_HPMN_MASK;
 	}
 
 	/* Trap Debug */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_DEBUGVER), feature_ids))
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), feature_ids))
 		mdcr_set |= MDCR_EL2_TDRA | MDCR_EL2_TDA | MDCR_EL2_TDE;
 
 	/* Trap OS Double Lock */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_DOUBLELOCK), feature_ids))
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DoubleLock), feature_ids))
 		mdcr_set |= MDCR_EL2_TDOSA;
 
 	/* Trap SPE */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_PMSVER), feature_ids)) {
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer), feature_ids)) {
 		mdcr_set |= MDCR_EL2_TPMS;
 		mdcr_clear |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT;
 	}
 
 	/* Trap Trace Filter */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_TRACE_FILT), feature_ids))
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceFilt), feature_ids))
 		mdcr_set |= MDCR_EL2_TTRF;
 
 	/* Trap Trace */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_TRACEVER), feature_ids))
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_TraceVer), feature_ids))
 		cptr_set |= CPTR_EL2_TTA;
 
 	vcpu->arch.mdcr_el2 |= mdcr_set;
@@ -128,7 +128,7 @@ static void pvm_init_traps_aa64mmfr0(struct kvm_vcpu *vcpu)
 	u64 mdcr_set = 0;
 
 	/* Trap Debug Communications Channel registers */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_FGT), feature_ids))
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR0_EL1_FGT), feature_ids))
 		mdcr_set |= MDCR_EL2_TDCC;
 
 	vcpu->arch.mdcr_el2 |= mdcr_set;
@@ -143,7 +143,7 @@ static void pvm_init_traps_aa64mmfr1(struct kvm_vcpu *vcpu)
 	u64 hcr_set = 0;
 
 	/* Trap LOR */
-	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_LOR), feature_ids))
+	if (!FIELD_GET(ARM64_FEATURE_MASK(ID_AA64MMFR1_EL1_LO), feature_ids))
 		hcr_set |= HCR_TLOR;
 
 	vcpu->arch.hcr_el2 |= hcr_set;
diff --git a/arch/arm64/kvm/hyp/nvhe/stacktrace.c b/arch/arm64/kvm/hyp/nvhe/stacktrace.c
index 58f645ad66bc..ed6b58b19cfa 100644
--- a/arch/arm64/kvm/hyp/nvhe/stacktrace.c
+++ b/arch/arm64/kvm/hyp/nvhe/stacktrace.c
@@ -39,41 +39,32 @@ static void hyp_prepare_backtrace(unsigned long fp, unsigned long pc)
 
 DEFINE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)], pkvm_stacktrace);
 
-static bool on_overflow_stack(unsigned long sp, unsigned long size,
-			      struct stack_info *info)
+static struct stack_info stackinfo_get_overflow(void)
 {
 	unsigned long low = (unsigned long)this_cpu_ptr(overflow_stack);
 	unsigned long high = low + OVERFLOW_STACK_SIZE;
 
-	return on_stack(sp, size, low, high, STACK_TYPE_OVERFLOW, info);
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
 }
 
-static bool on_hyp_stack(unsigned long sp, unsigned long size,
-			      struct stack_info *info)
+static struct stack_info stackinfo_get_hyp(void)
 {
 	struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params);
 	unsigned long high = params->stack_hyp_va;
 	unsigned long low = high - PAGE_SIZE;
 
-	return on_stack(sp, size, low, high, STACK_TYPE_HYP, info);
-}
-
-static bool on_accessible_stack(const struct task_struct *tsk,
-				unsigned long sp, unsigned long size,
-				struct stack_info *info)
-{
-	if (info)
-		info->type = STACK_TYPE_UNKNOWN;
-
-	return (on_overflow_stack(sp, size, info) ||
-		on_hyp_stack(sp, size, info));
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
 }
 
 static int unwind_next(struct unwind_state *state)
 {
-	struct stack_info info;
-
-	return unwind_next_common(state, &info, on_accessible_stack, NULL);
+	return unwind_next_frame_record(state);
 }
 
 static void notrace unwind(struct unwind_state *state,
@@ -129,7 +120,14 @@ static bool pkvm_save_backtrace_entry(void *arg, unsigned long where)
  */
 static void pkvm_save_backtrace(unsigned long fp, unsigned long pc)
 {
-	struct unwind_state state;
+	struct stack_info stacks[] = {
+		stackinfo_get_overflow(),
+		stackinfo_get_hyp(),
+	};
+	struct unwind_state state = {
+		.stacks = stacks,
+		.nr_stacks = ARRAY_SIZE(stacks),
+	};
 	int idx = 0;
 
 	kvm_nvhe_unwind_init(&state, fp, pc);
diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c
index e20fa4475dac..0f9ac25afdf4 100644
--- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c
+++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c
@@ -92,9 +92,9 @@ static u64 get_pvm_id_aa64pfr0(const struct kvm_vcpu *vcpu)
 		PVM_ID_AA64PFR0_RESTRICT_UNSIGNED);
 
 	/* Spectre and Meltdown mitigation in KVM */
-	set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2),
+	set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2),
 			       (u64)kvm->arch.pfr0_csv2);
-	set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3),
+	set_mask |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3),
 			       (u64)kvm->arch.pfr0_csv3);
 
 	return (id_aa64pfr0_el1_sys_val & allow_mask) | set_mask;
@@ -106,7 +106,7 @@ static u64 get_pvm_id_aa64pfr1(const struct kvm_vcpu *vcpu)
 	u64 allow_mask = PVM_ID_AA64PFR1_ALLOW;
 
 	if (!kvm_has_mte(kvm))
-		allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_MTE);
+		allow_mask &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE);
 
 	return id_aa64pfr1_el1_sys_val & allow_mask;
 }
@@ -281,8 +281,8 @@ static bool pvm_access_id_aarch32(struct kvm_vcpu *vcpu,
 	 * No support for AArch32 guests, therefore, pKVM has no sanitized copy
 	 * of AArch32 feature id registers.
 	 */
-	BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1),
-		     PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) > ID_AA64PFR0_ELx_64BIT_ONLY);
+	BUILD_BUG_ON(FIELD_GET(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_EL1),
+		     PVM_ID_AA64PFR0_RESTRICT_UNSIGNED) > ID_AA64PFR0_EL1_ELx_64BIT_ONLY);
 
 	return pvm_access_raz_wi(vcpu, p, r);
 }
diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index 2cb3867eb7c2..cdf8e76b0be1 100644
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -61,7 +61,7 @@ struct kvm_pgtable_walk_data {
 
 static bool kvm_phys_is_valid(u64 phys)
 {
-	return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_PARANGE_MAX));
+	return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_EL1_PARANGE_MAX));
 }
 
 static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level)
diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c
index 11c43bed5f97..0003c7d37533 100644
--- a/arch/arm64/kvm/pmu-emul.c
+++ b/arch/arm64/kvm/pmu-emul.c
@@ -33,12 +33,12 @@ static u32 kvm_pmu_event_mask(struct kvm *kvm)
 	pmuver = kvm->arch.arm_pmu->pmuver;
 
 	switch (pmuver) {
-	case ID_AA64DFR0_PMUVER_8_0:
+	case ID_AA64DFR0_EL1_PMUVer_IMP:
 		return GENMASK(9, 0);
-	case ID_AA64DFR0_PMUVER_8_1:
-	case ID_AA64DFR0_PMUVER_8_4:
-	case ID_AA64DFR0_PMUVER_8_5:
-	case ID_AA64DFR0_PMUVER_8_7:
+	case ID_AA64DFR0_EL1_PMUVer_V3P1:
+	case ID_AA64DFR0_EL1_PMUVer_V3P4:
+	case ID_AA64DFR0_EL1_PMUVer_V3P5:
+	case ID_AA64DFR0_EL1_PMUVer_V3P7:
 		return GENMASK(15, 0);
 	default:		/* Shouldn't be here, just for sanity */
 		WARN_ONCE(1, "Unknown PMU version %d\n", pmuver);
@@ -774,7 +774,7 @@ void kvm_host_pmu_init(struct arm_pmu *pmu)
 {
 	struct arm_pmu_entry *entry;
 
-	if (pmu->pmuver == 0 || pmu->pmuver == ID_AA64DFR0_PMUVER_IMP_DEF)
+	if (pmu->pmuver == 0 || pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
 		return;
 
 	mutex_lock(&arm_pmus_lock);
@@ -828,7 +828,7 @@ static struct arm_pmu *kvm_pmu_probe_armpmu(void)
 	if (event->pmu) {
 		pmu = to_arm_pmu(event->pmu);
 		if (pmu->pmuver == 0 ||
-		    pmu->pmuver == ID_AA64DFR0_PMUVER_IMP_DEF)
+		    pmu->pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
 			pmu = NULL;
 	}
 
@@ -856,7 +856,7 @@ u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1)
 		 * Don't advertise STALL_SLOT, as PMMIR_EL0 is handled
 		 * as RAZ
 		 */
-		if (vcpu->kvm->arch.arm_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_4)
+		if (vcpu->kvm->arch.arm_pmu->pmuver >= ID_AA64DFR0_EL1_PMUVer_V3P4)
 			val &= ~BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32);
 		base = 32;
 	}
diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c
index 0e08fbe68715..5ae18472205a 100644
--- a/arch/arm64/kvm/reset.c
+++ b/arch/arm64/kvm/reset.c
@@ -359,7 +359,7 @@ int kvm_set_ipa_limit(void)
 
 	mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
 	parange = cpuid_feature_extract_unsigned_field(mmfr0,
-				ID_AA64MMFR0_PARANGE_SHIFT);
+				ID_AA64MMFR0_EL1_PARANGE_SHIFT);
 	/*
 	 * IPA size beyond 48 bits could not be supported
 	 * on either 4K or 16K page size. Hence let's cap
@@ -367,20 +367,20 @@ int kvm_set_ipa_limit(void)
 	 * on the system.
 	 */
 	if (PAGE_SIZE != SZ_64K)
-		parange = min(parange, (unsigned int)ID_AA64MMFR0_PARANGE_48);
+		parange = min(parange, (unsigned int)ID_AA64MMFR0_EL1_PARANGE_48);
 
 	/*
 	 * Check with ARMv8.5-GTG that our PAGE_SIZE is supported at
 	 * Stage-2. If not, things will stop very quickly.
 	 */
-	switch (cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_TGRAN_2_SHIFT)) {
-	case ID_AA64MMFR0_TGRAN_2_SUPPORTED_NONE:
+	switch (cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_EL1_TGRAN_2_SHIFT)) {
+	case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_NONE:
 		kvm_err("PAGE_SIZE not supported at Stage-2, giving up\n");
 		return -EINVAL;
-	case ID_AA64MMFR0_TGRAN_2_SUPPORTED_DEFAULT:
+	case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_DEFAULT:
 		kvm_debug("PAGE_SIZE supported at Stage-2 (default)\n");
 		break;
-	case ID_AA64MMFR0_TGRAN_2_SUPPORTED_MIN ... ID_AA64MMFR0_TGRAN_2_SUPPORTED_MAX:
+	case ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MIN ... ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_MAX:
 		kvm_debug("PAGE_SIZE supported at Stage-2 (advertised)\n");
 		break;
 	default:
diff --git a/arch/arm64/kvm/stacktrace.c b/arch/arm64/kvm/stacktrace.c
index 949d19d603fb..3ace5b75813b 100644
--- a/arch/arm64/kvm/stacktrace.c
+++ b/arch/arm64/kvm/stacktrace.c
@@ -21,6 +21,54 @@
 
 #include <asm/stacktrace/nvhe.h>
 
+static struct stack_info stackinfo_get_overflow(void)
+{
+	struct kvm_nvhe_stacktrace_info *stacktrace_info
+				= this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
+	unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base;
+	unsigned long high = low + OVERFLOW_STACK_SIZE;
+
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
+}
+
+static struct stack_info stackinfo_get_overflow_kern_va(void)
+{
+	unsigned long low = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack);
+	unsigned long high = low + OVERFLOW_STACK_SIZE;
+
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
+}
+
+static struct stack_info stackinfo_get_hyp(void)
+{
+	struct kvm_nvhe_stacktrace_info *stacktrace_info
+				= this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
+	unsigned long low = (unsigned long)stacktrace_info->stack_base;
+	unsigned long high = low + PAGE_SIZE;
+
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
+}
+
+static struct stack_info stackinfo_get_hyp_kern_va(void)
+{
+	unsigned long low = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_page);
+	unsigned long high = low + PAGE_SIZE;
+
+	return (struct stack_info) {
+		.low = low,
+		.high = high,
+	};
+}
+
 /*
  * kvm_nvhe_stack_kern_va - Convert KVM nVHE HYP stack addresses to a kernel VAs
  *
@@ -34,73 +82,45 @@
  * Returns true on success and updates @addr to its corresponding kernel VA;
  * otherwise returns false.
  */
-static bool kvm_nvhe_stack_kern_va(unsigned long *addr,
-				   enum stack_type type)
+static bool kvm_nvhe_stack_kern_va(unsigned long *addr, unsigned long size)
 {
-	struct kvm_nvhe_stacktrace_info *stacktrace_info;
-	unsigned long hyp_base, kern_base, hyp_offset;
+	struct stack_info stack_hyp, stack_kern;
 
-	stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
+	stack_hyp = stackinfo_get_hyp();
+	stack_kern = stackinfo_get_hyp_kern_va();
+	if (stackinfo_on_stack(&stack_hyp, *addr, size))
+		goto found;
 
-	switch (type) {
-	case STACK_TYPE_HYP:
-		kern_base = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_page);
-		hyp_base = (unsigned long)stacktrace_info->stack_base;
-		break;
-	case STACK_TYPE_OVERFLOW:
-		kern_base = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack);
-		hyp_base = (unsigned long)stacktrace_info->overflow_stack_base;
-		break;
-	default:
-		return false;
-	}
+	stack_hyp = stackinfo_get_overflow();
+	stack_kern = stackinfo_get_overflow_kern_va();
+	if (stackinfo_on_stack(&stack_hyp, *addr, size))
+		goto found;
 
-	hyp_offset = *addr - hyp_base;
-
-	*addr = kern_base + hyp_offset;
+	return false;
 
+found:
+	*addr = *addr - stack_hyp.low + stack_kern.low;
 	return true;
 }
 
-static bool on_overflow_stack(unsigned long sp, unsigned long size,
-			      struct stack_info *info)
-{
-	struct kvm_nvhe_stacktrace_info *stacktrace_info
-				= this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
-	unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base;
-	unsigned long high = low + OVERFLOW_STACK_SIZE;
-
-	return on_stack(sp, size, low, high, STACK_TYPE_OVERFLOW, info);
-}
-
-static bool on_hyp_stack(unsigned long sp, unsigned long size,
-			 struct stack_info *info)
-{
-	struct kvm_nvhe_stacktrace_info *stacktrace_info
-				= this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
-	unsigned long low = (unsigned long)stacktrace_info->stack_base;
-	unsigned long high = low + PAGE_SIZE;
-
-	return on_stack(sp, size, low, high, STACK_TYPE_HYP, info);
-}
-
-static bool on_accessible_stack(const struct task_struct *tsk,
-				unsigned long sp, unsigned long size,
-				struct stack_info *info)
+/*
+ * Convert a KVN nVHE HYP frame record address to a kernel VA
+ */
+static bool kvm_nvhe_stack_kern_record_va(unsigned long *addr)
 {
-	if (info)
-		info->type = STACK_TYPE_UNKNOWN;
-
-	return (on_overflow_stack(sp, size, info) ||
-		on_hyp_stack(sp, size, info));
+	return kvm_nvhe_stack_kern_va(addr, 16);
 }
 
 static int unwind_next(struct unwind_state *state)
 {
-	struct stack_info info;
-
-	return unwind_next_common(state, &info, on_accessible_stack,
-				  kvm_nvhe_stack_kern_va);
+	/*
+	 * The FP is in the hypervisor VA space. Convert it to the kernel VA
+	 * space so it can be unwound by the regular unwind functions.
+	 */
+	if (!kvm_nvhe_stack_kern_record_va(&state->fp))
+		return -EINVAL;
+
+	return unwind_next_frame_record(state);
 }
 
 static void unwind(struct unwind_state *state,
@@ -158,7 +178,14 @@ static void kvm_nvhe_dump_backtrace_end(void)
 static void hyp_dump_backtrace(unsigned long hyp_offset)
 {
 	struct kvm_nvhe_stacktrace_info *stacktrace_info;
-	struct unwind_state state;
+	struct stack_info stacks[] = {
+		stackinfo_get_overflow_kern_va(),
+		stackinfo_get_hyp_kern_va(),
+	};
+	struct unwind_state state = {
+		.stacks = stacks,
+		.nr_stacks = ARRAY_SIZE(stacks),
+	};
 
 	stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
 
diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c
index 3234f50b8c4b..2ef1121ab844 100644
--- a/arch/arm64/kvm/sys_regs.c
+++ b/arch/arm64/kvm/sys_regs.c
@@ -273,7 +273,7 @@ static bool trap_loregion(struct kvm_vcpu *vcpu,
 	u64 val = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
 	u32 sr = reg_to_encoding(r);
 
-	if (!(val & (0xfUL << ID_AA64MMFR1_LOR_SHIFT))) {
+	if (!(val & (0xfUL << ID_AA64MMFR1_EL1_LO_SHIFT))) {
 		kvm_inject_undefined(vcpu);
 		return false;
 	}
@@ -1077,22 +1077,22 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu,
 	switch (id) {
 	case SYS_ID_AA64PFR0_EL1:
 		if (!vcpu_has_sve(vcpu))
-			val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_SVE);
-		val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_AMU);
-		val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2);
-		val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2);
-		val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3);
-		val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3);
+			val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_SVE);
+		val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_AMU);
+		val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2);
+		val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV2), (u64)vcpu->kvm->arch.pfr0_csv2);
+		val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3);
+		val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_CSV3), (u64)vcpu->kvm->arch.pfr0_csv3);
 		if (kvm_vgic_global_state.type == VGIC_V3) {
-			val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_GIC);
-			val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_GIC), 1);
+			val &= ~ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_GIC);
+			val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64PFR0_EL1_GIC), 1);
 		}
 		break;
 	case SYS_ID_AA64PFR1_EL1:
 		if (!kvm_has_mte(vcpu->kvm))
-			val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_MTE);
+			val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE);
 
-		val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_SME);
+		val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SME);
 		break;
 	case SYS_ID_AA64ISAR1_EL1:
 		if (!vcpu_has_ptrauth(vcpu))
@@ -1110,14 +1110,14 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu,
 		break;
 	case SYS_ID_AA64DFR0_EL1:
 		/* Limit debug to ARMv8.0 */
-		val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_DEBUGVER);
-		val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_DEBUGVER), 6);
+		val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer);
+		val |= FIELD_PREP(ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_DebugVer), 6);
 		/* Limit guests to PMUv3 for ARMv8.4 */
 		val = cpuid_feature_cap_perfmon_field(val,
-						      ID_AA64DFR0_PMUVER_SHIFT,
-						      kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_PMUVER_8_4 : 0);
+						      ID_AA64DFR0_EL1_PMUVer_SHIFT,
+						      kvm_vcpu_has_pmu(vcpu) ? ID_AA64DFR0_EL1_PMUVer_V3P4 : 0);
 		/* Hide SPE from guests */
-		val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_PMSVER);
+		val &= ~ARM64_FEATURE_MASK(ID_AA64DFR0_EL1_PMSVer);
 		break;
 	case SYS_ID_DFR0_EL1:
 		/* Limit guests to PMUv3 for ARMv8.4 */
@@ -1196,21 +1196,21 @@ static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
 	 * it doesn't promise more than what is actually provided (the
 	 * guest could otherwise be covered in ectoplasmic residue).
 	 */
-	csv2 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV2_SHIFT);
+	csv2 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_EL1_CSV2_SHIFT);
 	if (csv2 > 1 ||
 	    (csv2 && arm64_get_spectre_v2_state() != SPECTRE_UNAFFECTED))
 		return -EINVAL;
 
 	/* Same thing for CSV3 */
-	csv3 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_CSV3_SHIFT);
+	csv3 = cpuid_feature_extract_unsigned_field(val, ID_AA64PFR0_EL1_CSV3_SHIFT);
 	if (csv3 > 1 ||
 	    (csv3 && arm64_get_meltdown_state() != SPECTRE_UNAFFECTED))
 		return -EINVAL;
 
 	/* We can only differ with CSV[23], and anything else is an error */
 	val ^= read_id_reg(vcpu, rd, false);
-	val &= ~((0xFUL << ID_AA64PFR0_CSV2_SHIFT) |
-		 (0xFUL << ID_AA64PFR0_CSV3_SHIFT));
+	val &= ~((0xFUL << ID_AA64PFR0_EL1_CSV2_SHIFT) |
+		 (0xFUL << ID_AA64PFR0_EL1_CSV3_SHIFT));
 	if (val)
 		return -EINVAL;
 
@@ -1825,11 +1825,11 @@ static bool trap_dbgdidr(struct kvm_vcpu *vcpu,
 	} else {
 		u64 dfr = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
 		u64 pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
-		u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL3_SHIFT);
+		u32 el3 = !!cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR0_EL1_EL3_SHIFT);
 
-		p->regval = ((((dfr >> ID_AA64DFR0_WRPS_SHIFT) & 0xf) << 28) |
-			     (((dfr >> ID_AA64DFR0_BRPS_SHIFT) & 0xf) << 24) |
-			     (((dfr >> ID_AA64DFR0_CTX_CMPS_SHIFT) & 0xf) << 20)
+		p->regval = ((((dfr >> ID_AA64DFR0_EL1_WRPs_SHIFT) & 0xf) << 28) |
+			     (((dfr >> ID_AA64DFR0_EL1_BRPs_SHIFT) & 0xf) << 24) |
+			     (((dfr >> ID_AA64DFR0_EL1_CTX_CMPs_SHIFT) & 0xf) << 20)
 			     | (6 << 16) | (1 << 15) | (el3 << 14) | (el3 << 12));
 		return true;
 	}
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index acdb7b3cc97d..91b22a014610 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -169,7 +169,7 @@ void __init kvm_update_va_mask(struct alt_instr *alt,
 		 * dictates it and we don't have any spare bits in the
 		 * address), NOP everything after masking the kernel VA.
 		 */
-		if (has_vhe() || (!tag_val && i > 0)) {
+		if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN) || (!tag_val && i > 0)) {
 			updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
 			continue;
 		}
@@ -193,7 +193,8 @@ void kvm_patch_vector_branch(struct alt_instr *alt,
 
 	BUG_ON(nr_inst != 4);
 
-	if (!cpus_have_const_cap(ARM64_SPECTRE_V3A) || WARN_ON_ONCE(has_vhe()))
+	if (!cpus_have_cap(ARM64_SPECTRE_V3A) ||
+	    WARN_ON_ONCE(cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN)))
 		return;
 
 	/*
diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index b8b4cf0bcf39..e1e0dca01839 100644
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -43,17 +43,17 @@ static u32 get_cpu_asid_bits(void)
 {
 	u32 asid;
 	int fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR0_EL1),
-						ID_AA64MMFR0_ASID_SHIFT);
+						ID_AA64MMFR0_EL1_ASIDBITS_SHIFT);
 
 	switch (fld) {
 	default:
 		pr_warn("CPU%d: Unknown ASID size (%d); assuming 8-bit\n",
 					smp_processor_id(),  fld);
 		fallthrough;
-	case ID_AA64MMFR0_ASID_8:
+	case ID_AA64MMFR0_EL1_ASIDBITS_8:
 		asid = 8;
 		break;
-	case ID_AA64MMFR0_ASID_16:
+	case ID_AA64MMFR0_EL1_ASIDBITS_16:
 		asid = 16;
 	}
 
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index 599cf81f5685..83a512a6ff0d 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -36,7 +36,7 @@ void arch_dma_prep_coherent(struct page *page, size_t size)
 {
 	unsigned long start = (unsigned long)page_address(page);
 
-	dcache_clean_inval_poc(start, start + size);
+	dcache_clean_poc(start, start + size);
 }
 
 #ifdef CONFIG_IOMMU_DMA
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index c33f1fad2745..5b391490e045 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -691,6 +691,9 @@ static int __kprobes do_translation_fault(unsigned long far,
 static int do_alignment_fault(unsigned long far, unsigned long esr,
 			      struct pt_regs *regs)
 {
+	if (IS_ENABLED(CONFIG_COMPAT_ALIGNMENT_FIXUPS) &&
+	    compat_user_mode(regs))
+		return do_compat_alignment_fixup(far, regs);
 	do_bad_area(far, esr, regs);
 	return 0;
 }
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index b9af30be813e..4b4651ee47f2 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -360,7 +360,7 @@ void __init arm64_memblock_init(void)
 		extern u16 memstart_offset_seed;
 		u64 mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
 		int parange = cpuid_feature_extract_unsigned_field(
-					mmfr0, ID_AA64MMFR0_PARANGE_SHIFT);
+					mmfr0, ID_AA64MMFR0_EL1_PARANGE_SHIFT);
 		s64 range = linear_region_size -
 			    BIT(id_aa64mmfr0_parange_to_phys_shift(parange));
 
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index eb489302c28a..9a7c38965154 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -539,7 +539,7 @@ static void __init map_mem(pgd_t *pgdp)
 	 */
 	BUILD_BUG_ON(pgd_index(direct_map_end - 1) == pgd_index(direct_map_end));
 
-	if (can_set_direct_map() || IS_ENABLED(CONFIG_KFENCE))
+	if (can_set_direct_map())
 		flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
 
 	/*
@@ -690,7 +690,7 @@ static bool arm64_early_this_cpu_has_bti(void)
 
 	pfr1 = __read_sysreg_by_encoding(SYS_ID_AA64PFR1_EL1);
 	return cpuid_feature_extract_unsigned_field(pfr1,
-						    ID_AA64PFR1_BT_SHIFT);
+						    ID_AA64PFR1_EL1_BT_SHIFT);
 }
 
 /*
@@ -1184,14 +1184,6 @@ static void free_empty_tables(unsigned long addr, unsigned long end,
 }
 #endif
 
-#if !ARM64_KERNEL_USES_PMD_MAPS
-int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
-		struct vmem_altmap *altmap)
-{
-	WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
-	return vmemmap_populate_basepages(start, end, node, altmap);
-}
-#else	/* !ARM64_KERNEL_USES_PMD_MAPS */
 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
 		struct vmem_altmap *altmap)
 {
@@ -1203,6 +1195,10 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
 	pmd_t *pmdp;
 
 	WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
+
+	if (!ARM64_KERNEL_USES_PMD_MAPS)
+		return vmemmap_populate_basepages(start, end, node, altmap);
+
 	do {
 		next = pmd_addr_end(addr, end);
 
@@ -1236,7 +1232,6 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
 
 	return 0;
 }
-#endif	/* !ARM64_KERNEL_USES_PMD_MAPS */
 
 #ifdef CONFIG_MEMORY_HOTPLUG
 void vmemmap_free(unsigned long start, unsigned long end,
@@ -1551,11 +1546,7 @@ int arch_add_memory(int nid, u64 start, u64 size,
 
 	VM_BUG_ON(!mhp_range_allowed(start, size, true));
 
-	/*
-	 * KFENCE requires linear map to be mapped at page granularity, so that
-	 * it is possible to protect/unprotect single pages in the KFENCE pool.
-	 */
-	if (can_set_direct_map() || IS_ENABLED(CONFIG_KFENCE))
+	if (can_set_direct_map())
 		flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
 
 	__create_pgd_mapping(swapper_pg_dir, start, __phys_to_virt(start),
diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c
index 64e985eaa52d..d107c3d434e2 100644
--- a/arch/arm64/mm/pageattr.c
+++ b/arch/arm64/mm/pageattr.c
@@ -21,7 +21,13 @@ bool rodata_full __ro_after_init = IS_ENABLED(CONFIG_RODATA_FULL_DEFAULT_ENABLED
 
 bool can_set_direct_map(void)
 {
-	return rodata_full || debug_pagealloc_enabled();
+	/*
+	 * rodata_full, DEBUG_PAGEALLOC and KFENCE require linear map to be
+	 * mapped at page granularity, so that it is possible to
+	 * protect/unprotect single pages.
+	 */
+	return rodata_full || debug_pagealloc_enabled() ||
+		IS_ENABLED(CONFIG_KFENCE);
 }
 
 static int change_page_range(pte_t *ptep, unsigned long addr, void *data)
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 7837a69524c5..b9ecbbae1e1a 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -10,6 +10,7 @@
 #include <linux/init.h>
 #include <linux/linkage.h>
 #include <linux/pgtable.h>
+#include <linux/cfi_types.h>
 #include <asm/assembler.h>
 #include <asm/asm-offsets.h>
 #include <asm/asm_pointer_auth.h>
@@ -48,17 +49,19 @@
 
 #ifdef CONFIG_KASAN_HW_TAGS
 #define TCR_MTE_FLAGS TCR_TCMA1 | TCR_TBI1 | TCR_TBID1
-#else
+#elif defined(CONFIG_ARM64_MTE)
 /*
  * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on
  * TBI being enabled at EL1.
  */
 #define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1
+#else
+#define TCR_MTE_FLAGS 0
 #endif
 
 /*
  * Default MAIR_EL1. MT_NORMAL_TAGGED is initially mapped as Normal memory and
- * changed during __cpu_setup to Normal Tagged if the system supports MTE.
+ * changed during mte_cpu_setup to Normal Tagged if the system supports MTE.
  */
 #define MAIR_EL1_SET							\
 	(MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) |	\
@@ -185,7 +188,7 @@ SYM_FUNC_END(cpu_do_resume)
  * This is the low-level counterpart to cpu_replace_ttbr1, and should not be
  * called by anything else. It can only be executed from a TTBR0 mapping.
  */
-SYM_FUNC_START(idmap_cpu_replace_ttbr1)
+SYM_TYPED_FUNC_START(idmap_cpu_replace_ttbr1)
 	save_and_disable_daif flags=x2
 
 	__idmap_cpu_set_reserved_ttbr1 x1, x3
@@ -253,7 +256,7 @@ SYM_FUNC_END(idmap_cpu_replace_ttbr1)
 SYM_DATA(__idmap_kpti_flag, .long 1)
 	.popsection
 
-SYM_FUNC_START(idmap_kpti_install_ng_mappings)
+SYM_TYPED_FUNC_START(idmap_kpti_install_ng_mappings)
 	cpu		.req	w0
 	temp_pte	.req	x0
 	num_cpus	.req	w1
@@ -426,46 +429,8 @@ SYM_FUNC_START(__cpu_setup)
 	mov_q	mair, MAIR_EL1_SET
 	mov_q	tcr, TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \
 			TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \
-			TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS
-
-#ifdef CONFIG_ARM64_MTE
-	/*
-	 * Update MAIR_EL1, GCR_EL1 and TFSR*_EL1 if MTE is supported
-	 * (ID_AA64PFR1_EL1[11:8] > 1).
-	 */
-	mrs	x10, ID_AA64PFR1_EL1
-	ubfx	x10, x10, #ID_AA64PFR1_MTE_SHIFT, #4
-	cmp	x10, #ID_AA64PFR1_MTE
-	b.lt	1f
-
-	/* Normal Tagged memory type at the corresponding MAIR index */
-	mov	x10, #MAIR_ATTR_NORMAL_TAGGED
-	bfi	mair, x10, #(8 *  MT_NORMAL_TAGGED), #8
+			TCR_TBI0 | TCR_A1 | TCR_KASAN_SW_FLAGS | TCR_MTE_FLAGS
 
-	mov	x10, #KERNEL_GCR_EL1
-	msr_s	SYS_GCR_EL1, x10
-
-	/*
-	 * If GCR_EL1.RRND=1 is implemented the same way as RRND=0, then
-	 * RGSR_EL1.SEED must be non-zero for IRG to produce
-	 * pseudorandom numbers. As RGSR_EL1 is UNKNOWN out of reset, we
-	 * must initialize it.
-	 */
-	mrs	x10, CNTVCT_EL0
-	ands	x10, x10, #SYS_RGSR_EL1_SEED_MASK
-	csinc	x10, x10, xzr, ne
-	lsl	x10, x10, #SYS_RGSR_EL1_SEED_SHIFT
-	msr_s	SYS_RGSR_EL1, x10
-
-	/* clear any pending tag check faults in TFSR*_EL1 */
-	msr_s	SYS_TFSR_EL1, xzr
-	msr_s	SYS_TFSRE0_EL1, xzr
-
-	/* set the TCR_EL1 bits */
-	mov_q	x10, TCR_MTE_FLAGS
-	orr	tcr, tcr, x10
-1:
-#endif
 	tcr_clear_errata_bits tcr, x9, x5
 
 #ifdef CONFIG_ARM64_VA_BITS_52
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 389623ae5a91..30f76178608b 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -1970,7 +1970,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
 				u32 flags, struct bpf_tramp_links *tlinks,
 				void *orig_call)
 {
-	int ret;
+	int i, ret;
 	int nargs = m->nr_args;
 	int max_insns = ((long)image_end - (long)image) / AARCH64_INSN_SIZE;
 	struct jit_ctx ctx = {
@@ -1982,6 +1982,12 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image,
 	if (nargs > 8)
 		return -ENOTSUPP;
 
+	/* don't support struct argument */
+	for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) {
+		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
+			return -ENOTSUPP;
+	}
+
 	ret = prepare_trampoline(&ctx, im, tlinks, orig_call, nargs, flags);
 	if (ret < 0)
 		return ret;
diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index 63b2484ce6c3..f1c0347ec31a 100644
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -2,6 +2,8 @@
 #
 # Internal CPU capabilities constants, keep this list sorted
 
+ALWAYS_BOOT
+ALWAYS_SYSTEM
 BTI
 # Unreliable: use system_supports_32bit_el0() instead.
 HAS_32BIT_EL0_DO_NOT_USE
@@ -68,6 +70,7 @@ WORKAROUND_2038923
 WORKAROUND_2064142
 WORKAROUND_2077057
 WORKAROUND_2457168
+WORKAROUND_2658417
 WORKAROUND_TRBE_OVERWRITE_FILL_MODE
 WORKAROUND_TSB_FLUSH_FAILURE
 WORKAROUND_TRBE_WRITE_OUT_OF_RANGE
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 9ae483ec1e56..7f1fb36f208c 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -46,6 +46,127 @@
 # feature that introduces them (eg, FEAT_LS64_ACCDATA introduces enumeration
 # item ACCDATA) though it may be more taseful to do something else.
 
+Sysreg	ID_AA64PFR0_EL1	3	0	0	4	0
+Enum	63:60	CSV3
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	59:56	CSV2
+	0b0000	NI
+	0b0001	IMP
+	0b0010	CSV2_2
+	0b0011	CSV2_3
+EndEnum
+Enum	55:52	RME
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	51:48	DIT
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	47:44	AMU
+	0b0000	NI
+	0b0001	IMP
+	0b0010	V1P1
+EndEnum
+Enum	43:40	MPAM
+	0b0000	0
+	0b0001	1
+EndEnum
+Enum	39:36	SEL2
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	35:32	SVE
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	31:28	RAS
+	0b0000	NI
+	0b0001	IMP
+	0b0010	V1P1
+EndEnum
+Enum	27:24	GIC
+	0b0000	NI
+	0b0001	IMP
+	0b0010	V4P1
+EndEnum
+Enum	23:20	AdvSIMD
+	0b0000	IMP
+	0b0001	FP16
+	0b1111	NI
+EndEnum
+Enum	19:16	FP
+	0b0000	IMP
+	0b0001	FP16
+	0b1111	NI
+EndEnum
+Enum	15:12	EL3
+	0b0000	NI
+	0b0001	IMP
+	0b0010	AARCH32
+EndEnum
+Enum	11:8	EL2
+	0b0000	NI
+	0b0001	IMP
+	0b0010	AARCH32
+EndEnum
+Enum	7:4	EL1
+	0b0001	IMP
+	0b0010	AARCH32
+EndEnum
+Enum	3:0	EL0
+	0b0001	IMP
+	0b0010	AARCH32
+EndEnum
+EndSysreg
+
+Sysreg	ID_AA64PFR1_EL1	3	0	0	4	1
+Res0	63:40
+Enum	39:36	NMI
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	35:32	CSV2_frac
+	0b0000	NI
+	0b0001	CSV2_1p1
+	0b0010	CSV2_1p2
+EndEnum
+Enum	31:28	RNDR_trap
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	27:24	SME
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Res0	23:20
+Enum	19:16	MPAM_frac
+	0b0000	MINOR_0
+	0b0001	MINOR_1
+EndEnum
+Enum	15:12	RAS_frac
+	0b0000	NI
+	0b0001	RASv1p1
+EndEnum
+Enum	11:8	MTE
+	0b0000	NI
+	0b0001	IMP
+	0b0010	MTE2
+	0b0011	MTE3
+EndEnum
+Enum	7:4	SSBS
+	0b0000	NI
+	0b0001	IMP
+	0b0010	SSBS2
+EndEnum
+Enum	3:0	BT
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+EndSysreg
+
 Sysreg	ID_AA64ZFR0_EL1	3	0	0	4	4
 Res0	63:60
 Enum	59:56	F64MM
@@ -98,7 +219,9 @@ Enum	63	FA64
 	0b1	IMP
 EndEnum
 Res0	62:60
-Field	59:56	SMEver
+Enum	59:56	SMEver
+	0b0000	IMP
+EndEnum
 Enum	55:52	I16I64
 	0b0000	NI
 	0b1111	IMP
@@ -129,6 +252,89 @@ EndEnum
 Res0	31:0
 EndSysreg
 
+Sysreg	ID_AA64DFR0_EL1	3	0	0	5	0
+Enum	63:60	HPMN0
+	0b0000	UNPREDICTABLE
+	0b0001	DEF
+EndEnum
+Res0	59:56
+Enum	55:52	BRBE
+	0b0000	NI
+	0b0001	IMP
+	0b0010	BRBE_V1P1
+EndEnum
+Enum	51:48	MTPMU
+	0b0000	NI_IMPDEF
+	0b0001	IMP
+	0b1111	NI
+EndEnum
+Enum	47:44	TraceBuffer
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	43:40	TraceFilt
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	39:36	DoubleLock
+	0b0000	IMP
+	0b1111	NI
+EndEnum
+Enum	35:32	PMSVer
+	0b0000	NI
+	0b0001	IMP
+	0b0010	V1P1
+	0b0011	V1P2
+	0b0100	V1P3
+EndEnum
+Field	31:28	CTX_CMPs
+Res0	27:24
+Field	23:20	WRPs
+Res0	19:16
+Field	15:12	BRPs
+Enum	11:8	PMUVer
+	0b0000	NI
+	0b0001	IMP
+	0b0100	V3P1
+	0b0101	V3P4
+	0b0110	V3P5
+	0b0111	V3P7
+	0b1000	V3P8
+	0b1111	IMP_DEF
+EndEnum
+Enum	7:4	TraceVer
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	3:0	DebugVer
+	0b0110	IMP
+	0b0111	VHE
+	0b1000	V8P2
+	0b1001	V8P4
+	0b1010	V8P8
+EndEnum
+EndSysreg
+
+Sysreg	ID_AA64DFR1_EL1	3	0	0	5	1
+Res0	63:0
+EndSysreg
+
+Sysreg	ID_AA64AFR0_EL1	3	0	0	5	4
+Res0	63:32
+Field	31:28	IMPDEF7
+Field	27:24	IMPDEF6
+Field	23:20	IMPDEF5
+Field	19:16	IMPDEF4
+Field	15:12	IMPDEF3
+Field	11:8	IMPDEF2
+Field	7:4	IMPDEF1
+Field	3:0	IMPDEF0
+EndSysreg
+
+Sysreg	ID_AA64AFR1_EL1	3	0	0	5	5
+Res0	63:0
+EndSysreg
+
 Sysreg	ID_AA64ISAR0_EL1	3	0	0	6	0
 Enum	63:60	RNDR
 	0b0000	NI
@@ -313,6 +519,217 @@ Enum	3:0	WFxT
 EndEnum
 EndSysreg
 
+Sysreg	ID_AA64MMFR0_EL1	3	0	0	7	0
+Enum	63:60	ECV
+	0b0000	NI
+	0b0001	IMP
+	0b0010	CNTPOFF
+EndEnum
+Enum	59:56	FGT
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Res0	55:48
+Enum	47:44	EXS
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	43:40	TGRAN4_2
+	0b0000	TGRAN4
+	0b0001	NI
+	0b0010	IMP
+	0b0011	52_BIT
+EndEnum
+Enum	39:36	TGRAN64_2
+	0b0000	TGRAN64
+	0b0001	NI
+	0b0010	IMP
+EndEnum
+Enum	35:32	TGRAN16_2
+	0b0000	TGRAN16
+	0b0001	NI
+	0b0010	IMP
+	0b0011	52_BIT
+EndEnum
+Enum	31:28	TGRAN4
+	0b0000	IMP
+	0b0001	52_BIT
+	0b1111	NI
+EndEnum
+Enum	27:24	TGRAN64
+	0b0000	IMP
+	0b1111	NI
+EndEnum
+Enum	23:20	TGRAN16
+	0b0000	NI
+	0b0001	IMP
+	0b0010	52_BIT
+EndEnum
+Enum	19:16	BIGENDEL0
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	15:12	SNSMEM
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	11:8	BIGEND
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	7:4	ASIDBITS
+	0b0000	8
+	0b0010	16
+EndEnum
+Enum	3:0	PARANGE
+	0b0000	32
+	0b0001	36
+	0b0010	40
+	0b0011	42
+	0b0100	44
+	0b0101	48
+	0b0110	52
+EndEnum
+EndSysreg
+
+Sysreg	ID_AA64MMFR1_EL1	3	0	0	7	1
+Enum	63:60	ECBHB
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	59:56	CMOW
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	55:52	TIDCP1
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	51:48	nTLBPA
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	47:44	AFP
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	43:40	HCX
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	39:36	ETS
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	35:32	TWED
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	31:28	XNX
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	27:24	SpecSEI
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	23:20	PAN
+	0b0000	NI
+	0b0001	IMP
+	0b0010	PAN2
+	0b0011	PAN3
+EndEnum
+Enum	19:16	LO
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	15:12	HPDS
+	0b0000	NI
+	0b0001	IMP
+	0b0010	HPDS2
+EndEnum
+Enum	11:8	VH
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	7:4	VMIDBits
+	0b0000	8
+	0b0010	16
+EndEnum
+Enum	3:0	HAFDBS
+	0b0000	NI
+	0b0001	AF
+	0b0010	DBM
+EndEnum
+EndSysreg
+
+Sysreg	ID_AA64MMFR2_EL1	3	0	0	7	2
+Enum	63:60	E0PD
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	59:56	EVT
+	0b0000	NI
+	0b0001	IMP
+	0b0010	TTLBxS
+EndEnum
+Enum	55:52	BBM
+	0b0000	0
+	0b0001	1
+	0b0010	2
+EndEnum
+Enum	51:48	TTL
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Res0	47:44
+Enum	43:40	FWB
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	39:36	IDS
+	0b0000	0x0
+	0b0001	0x18
+EndEnum
+Enum	35:32	AT
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	31:28	ST
+	0b0000	39
+	0b0001	48_47
+EndEnum
+Enum	27:24	NV
+	0b0000	NI
+	0b0001	IMP
+	0b0010	NV2
+EndEnum
+Enum	23:20	CCIDX
+	0b0000	32
+	0b0001	64
+EndEnum
+Enum	19:16	VARange
+	0b0000	48
+	0b0001	52
+EndEnum
+Enum	15:12	IESB
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	11:8	LSM
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	7:4	UAO
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+Enum	3:0	CnP
+	0b0000	NI
+	0b0001	IMP
+EndEnum
+EndSysreg
+
 Sysreg	SCTLR_EL1	3	0	1	0	0
 Field	63	TIDCP
 Field	62	SPINMASK
@@ -427,6 +844,12 @@ Sysreg	SMCR_EL1	3	0	1	2	6
 Fields	SMCR_ELx
 EndSysreg
 
+Sysreg	ALLINT	3	0	4	3	0
+Res0	63:14
+Field	13	ALLINT
+Res0	12:0
+EndSysreg
+
 Sysreg	FAR_EL1	3	0	6	0	0
 Field	63:0	ADDR
 EndSysreg
@@ -440,6 +863,14 @@ Sysreg	CONTEXTIDR_EL1	3	0	13	0	1
 Fields	CONTEXTIDR_ELx
 EndSysreg
 
+Sysreg	TPIDR_EL1	3	0	13	0	4
+Field	63:0	ThreadID
+EndSysreg
+
+Sysreg	SCXTNUM_EL1	3	0	13	0	7
+Field	63:0	SoftwareContextNumber
+EndSysreg
+
 Sysreg	CLIDR_EL1	3	1	0	0	1
 Res0	63:47
 Field	46:33	Ttypen
@@ -514,6 +945,22 @@ Sysreg	ZCR_EL2	3	4	1	2	0
 Fields	ZCR_ELx
 EndSysreg
 
+Sysreg	HCRX_EL2	3	4	1	2	2
+Res0	63:12
+Field	11	MSCEn
+Field	10	MCE2
+Field	9	CMOW
+Field	8	VFNMI
+Field	7	VINMI
+Field	6	TALLINT
+Field	5	SMPME
+Field	4	FGTnXS
+Field	3	FnXS
+Field	2	EnASR
+Field	1	EnALS
+Field	0	EnAS0
+EndSysreg
+
 Sysreg	SMPRIMAP_EL2	3	4	1	2	5
 Field	63:60	P15
 Field	59:56	P14
diff --git a/arch/hexagon/include/asm/io.h b/arch/hexagon/include/asm/io.h
index c33241425a5c..46a099de85b7 100644
--- a/arch/hexagon/include/asm/io.h
+++ b/arch/hexagon/include/asm/io.h
@@ -308,6 +308,31 @@ static inline void outsl(unsigned long port, const void *buffer, int count)
 	}
 }
 
+/*
+ * These defines are necessary to use the generic io.h for filling in
+ * the missing parts of the API contract. This is because the platform
+ * uses (inline) functions rather than defines and the generic helper
+ * fills in the undefined.
+ */
+#define virt_to_phys virt_to_phys
+#define phys_to_virt phys_to_virt
+#define memset_io memset_io
+#define memcpy_fromio memcpy_fromio
+#define memcpy_toio memcpy_toio
+#define readb readb
+#define readw readw
+#define readl readl
+#define writeb writeb
+#define writew writew
+#define writel writel
+#define insb insb
+#define insw insw
+#define insl insl
+#define outsb outsb
+#define outsw outsw
+#define outsl outsl
+#include <asm-generic/io.h>
+
 #endif /* __KERNEL__ */
 
 #endif
diff --git a/arch/ia64/mm/numa.c b/arch/ia64/mm/numa.c
index d6579ec3ea32..4c7b1f50e3b7 100644
--- a/arch/ia64/mm/numa.c
+++ b/arch/ia64/mm/numa.c
@@ -75,5 +75,6 @@ int memory_add_physaddr_to_nid(u64 addr)
 		return 0;
 	return nid;
 }
+EXPORT_SYMBOL(memory_add_physaddr_to_nid);
 #endif
 #endif
diff --git a/arch/loongarch/Kconfig b/arch/loongarch/Kconfig
index c7dd6ad779af..551dd99e98b8 100644
--- a/arch/loongarch/Kconfig
+++ b/arch/loongarch/Kconfig
@@ -10,7 +10,6 @@ config LOONGARCH
 	select ARCH_ENABLE_MEMORY_HOTPLUG
 	select ARCH_ENABLE_MEMORY_HOTREMOVE
 	select ARCH_HAS_ACPI_TABLE_UPGRADE	if ACPI
-	select ARCH_HAS_PHYS_TO_DMA
 	select ARCH_HAS_PTE_SPECIAL
 	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
 	select ARCH_INLINE_READ_LOCK if !PREEMPTION
diff --git a/arch/loongarch/include/asm/loongson.h b/arch/loongarch/include/asm/loongson.h
index 6e8f6972ceb6..00db93edae1b 100644
--- a/arch/loongarch/include/asm/loongson.h
+++ b/arch/loongarch/include/asm/loongson.h
@@ -14,8 +14,6 @@
 #include <asm/addrspace.h>
 #include <asm/bootinfo.h>
 
-extern const struct plat_smp_ops loongson3_smp_ops;
-
 #define LOONGSON_REG(x) \
 	(*(volatile u32 *)((char *)TO_UNCACHE(LOONGSON_REG_BASE) + (x)))
 
diff --git a/arch/loongarch/kernel/dma.c b/arch/loongarch/kernel/dma.c
index 8c9b5314a13e..7a9c6a9dd2d0 100644
--- a/arch/loongarch/kernel/dma.c
+++ b/arch/loongarch/kernel/dma.c
@@ -2,39 +2,29 @@
 /*
  * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
  */
-#include <linux/init.h>
+#include <linux/acpi.h>
 #include <linux/dma-direct.h>
-#include <linux/dma-mapping.h>
-#include <linux/dma-map-ops.h>
-#include <linux/swiotlb.h>
 
-#include <asm/bootinfo.h>
-#include <asm/dma.h>
-#include <asm/loongson.h>
-
-/*
- * We extract 4bit node id (bit 44~47) from Loongson-3's
- * 48bit physical address space and embed it into 40bit.
- */
-
-static int node_id_offset;
-
-dma_addr_t phys_to_dma(struct device *dev, phys_addr_t paddr)
-{
-	long nid = (paddr >> 44) & 0xf;
-
-	return ((nid << 44) ^ paddr) | (nid << node_id_offset);
-}
-
-phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
+void acpi_arch_dma_setup(struct device *dev)
 {
-	long nid = (daddr >> node_id_offset) & 0xf;
+	int ret;
+	u64 mask, end = 0;
+	const struct bus_dma_region *map = NULL;
+
+	ret = acpi_dma_get_range(dev, &map);
+	if (!ret && map) {
+		const struct bus_dma_region *r = map;
+
+		for (end = 0; r->size; r++) {
+			if (r->dma_start + r->size - 1 > end)
+				end = r->dma_start + r->size - 1;
+		}
+
+		mask = DMA_BIT_MASK(ilog2(end) + 1);
+		dev->bus_dma_limit = end;
+		dev->dma_range_map = map;
+		dev->coherent_dma_mask = min(dev->coherent_dma_mask, mask);
+		*dev->dma_mask = min(*dev->dma_mask, mask);
+	}
 
-	return ((nid << node_id_offset) ^ daddr) | (nid << 44);
-}
-
-void __init plat_swiotlb_setup(void)
-{
-	swiotlb_init(true, SWIOTLB_VERBOSE);
-	node_id_offset = ((readl(LS7A_DMA_CFG) & LS7A_DMA_NODE_MASK) >> LS7A_DMA_NODE_SHF) + 36;
 }
diff --git a/arch/loongarch/kernel/head.S b/arch/loongarch/kernel/head.S
index c60eb66793e3..331864369e49 100644
--- a/arch/loongarch/kernel/head.S
+++ b/arch/loongarch/kernel/head.S
@@ -14,6 +14,8 @@
 
 	__REF
 
+	.align 12
+
 SYM_CODE_START(kernel_entry)			# kernel entry point
 
 	/* Config direct window and set PG */
diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c
index 8f5c2f9a1a83..d97c69dbe553 100644
--- a/arch/loongarch/kernel/setup.c
+++ b/arch/loongarch/kernel/setup.c
@@ -247,7 +247,7 @@ static void __init arch_mem_init(char **cmdline_p)
 	sparse_init();
 	memblock_set_bottom_up(true);
 
-	plat_swiotlb_setup();
+	swiotlb_init(true, SWIOTLB_VERBOSE);
 
 	dma_contiguous_reserve(PFN_PHYS(max_low_pfn));
 
diff --git a/arch/loongarch/kernel/traps.c b/arch/loongarch/kernel/traps.c
index aa1c95aaf595..5010e95cef84 100644
--- a/arch/loongarch/kernel/traps.c
+++ b/arch/loongarch/kernel/traps.c
@@ -461,11 +461,9 @@ asmlinkage void noinstr do_watch(struct pt_regs *regs)
 
 asmlinkage void noinstr do_ri(struct pt_regs *regs)
 {
-	int status = -1;
+	int status = SIGILL;
 	unsigned int opcode = 0;
 	unsigned int __user *era = (unsigned int __user *)exception_era(regs);
-	unsigned long old_era = regs->csr_era;
-	unsigned long old_ra = regs->regs[1];
 	irqentry_state_t state = irqentry_enter(regs);
 
 	local_irq_enable();
@@ -477,21 +475,12 @@ asmlinkage void noinstr do_ri(struct pt_regs *regs)
 
 	die_if_kernel("Reserved instruction in kernel code", regs);
 
-	compute_return_era(regs);
-
 	if (unlikely(get_user(opcode, era) < 0)) {
 		status = SIGSEGV;
 		current->thread.error_code = 1;
 	}
 
-	if (status < 0)
-		status = SIGILL;
-
-	if (unlikely(status > 0)) {
-		regs->csr_era = old_era;		/* Undo skip-over.  */
-		regs->regs[1] = old_ra;
-		force_sig(status);
-	}
+	force_sig(status);
 
 out:
 	local_irq_disable();
diff --git a/arch/m68k/Kconfig b/arch/m68k/Kconfig
index b06faf6c0b27..7bff88118507 100644
--- a/arch/m68k/Kconfig
+++ b/arch/m68k/Kconfig
@@ -87,7 +87,7 @@ config MMU_SUN3
 
 config KEXEC
 	bool "kexec system call"
-	depends on M68KCLASSIC
+	depends on M68KCLASSIC && MMU
 	select KEXEC_CORE
 	help
 	  kexec is a system call that implements the ability to shutdown your
diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig
index a6a886a89be2..e2038d9499e4 100644
--- a/arch/m68k/configs/amiga_defconfig
+++ b/arch/m68k/configs/amiga_defconfig
@@ -84,7 +84,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -573,9 +572,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -594,6 +593,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig
index bffd24c2755e..ddd201259e43 100644
--- a/arch/m68k/configs/apollo_defconfig
+++ b/arch/m68k/configs/apollo_defconfig
@@ -80,7 +80,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -530,9 +529,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -551,6 +550,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig
index 0013425b1e08..d9f783707387 100644
--- a/arch/m68k/configs/atari_defconfig
+++ b/arch/m68k/configs/atari_defconfig
@@ -87,7 +87,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -550,9 +549,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -571,6 +570,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig
index 42d969697f7f..68957c6bcff1 100644
--- a/arch/m68k/configs/bvme6000_defconfig
+++ b/arch/m68k/configs/bvme6000_defconfig
@@ -77,7 +77,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -522,9 +521,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -543,6 +542,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig
index 97d6d9acb395..825c6a02fa9d 100644
--- a/arch/m68k/configs/hp300_defconfig
+++ b/arch/m68k/configs/hp300_defconfig
@@ -79,7 +79,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -532,9 +531,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -553,6 +552,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig
index 8cbfc1c659a3..17f64c562bf1 100644
--- a/arch/m68k/configs/mac_defconfig
+++ b/arch/m68k/configs/mac_defconfig
@@ -78,7 +78,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -552,9 +551,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -573,6 +572,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig
index 9f45fe60757f..f5f4c572b694 100644
--- a/arch/m68k/configs/multi_defconfig
+++ b/arch/m68k/configs/multi_defconfig
@@ -98,7 +98,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -638,9 +637,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -659,6 +658,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig
index 4736cfacf6a2..b4a0bbef7e39 100644
--- a/arch/m68k/configs/mvme147_defconfig
+++ b/arch/m68k/configs/mvme147_defconfig
@@ -76,7 +76,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -521,9 +520,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -542,6 +541,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig
index 638cd38aa7d2..c6a6d5926793 100644
--- a/arch/m68k/configs/mvme16x_defconfig
+++ b/arch/m68k/configs/mvme16x_defconfig
@@ -77,7 +77,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -522,9 +521,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -543,6 +542,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig
index ec8b6bb70ebd..49c9c89f0caf 100644
--- a/arch/m68k/configs/q40_defconfig
+++ b/arch/m68k/configs/q40_defconfig
@@ -78,7 +78,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -539,9 +538,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -560,6 +559,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig
index 7d8dc578d59c..9b44eeb9c07f 100644
--- a/arch/m68k/configs/sun3_defconfig
+++ b/arch/m68k/configs/sun3_defconfig
@@ -74,7 +74,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -521,9 +520,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -542,6 +541,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig
index 96290aee5302..d2ffb0a65b44 100644
--- a/arch/m68k/configs/sun3x_defconfig
+++ b/arch/m68k/configs/sun3x_defconfig
@@ -74,7 +74,6 @@ CONFIG_NETFILTER=y
 CONFIG_NETFILTER_NETLINK_HOOK=m
 CONFIG_NF_CONNTRACK=m
 CONFIG_NF_CONNTRACK_ZONES=y
-# CONFIG_NF_CONNTRACK_PROCFS is not set
 # CONFIG_NF_CT_PROTO_DCCP is not set
 CONFIG_NF_CONNTRACK_AMANDA=m
 CONFIG_NF_CONNTRACK_FTP=m
@@ -520,9 +519,9 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_KEYWRAP=m
 CONFIG_CRYPTO_ADIANTUM=m
+CONFIG_CRYPTO_HCTR2=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
-CONFIG_CRYPTO_BLAKE2S=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_RMD160=m
@@ -541,6 +540,7 @@ CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SEED=m
+CONFIG_CRYPTO_ARIA=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_SM4_GENERIC=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/m68k/include/uapi/asm/bootinfo-virt.h b/arch/m68k/include/uapi/asm/bootinfo-virt.h
index b091ee9b06e0..7dbcd7bec103 100644
--- a/arch/m68k/include/uapi/asm/bootinfo-virt.h
+++ b/arch/m68k/include/uapi/asm/bootinfo-virt.h
@@ -13,13 +13,8 @@
 #define BI_VIRT_VIRTIO_BASE	0x8004
 #define BI_VIRT_CTRL_BASE	0x8005
 
-/*
- * A random seed used to initialize the RNG. Record format:
- *
- *   - length       [ 2 bytes, 16-bit big endian ]
- *   - seed data    [ `length` bytes, padded to preserve 2-byte alignment ]
- */
-#define BI_VIRT_RNG_SEED	0x8006
+/* No longer used -- replaced with BI_RNG_SEED -- but don't reuse this index:
+ * #define BI_VIRT_RNG_SEED	0x8006 */
 
 #define VIRT_BOOTI_VERSION	MK_BI_VERSION(2, 0)
 
diff --git a/arch/m68k/include/uapi/asm/bootinfo.h b/arch/m68k/include/uapi/asm/bootinfo.h
index 95ecf3ae4c49..024e87d7095f 100644
--- a/arch/m68k/include/uapi/asm/bootinfo.h
+++ b/arch/m68k/include/uapi/asm/bootinfo.h
@@ -64,6 +64,13 @@ struct mem_info {
 					/* (struct mem_info) */
 #define BI_COMMAND_LINE		0x0007	/* kernel command line parameters */
 					/* (string) */
+/*
+ * A random seed used to initialize the RNG. Record format:
+ *
+ *   - length       [ 2 bytes, 16-bit big endian ]
+ *   - seed data    [ `length` bytes, padded to preserve 4-byte struct alignment ]
+ */
+#define BI_RNG_SEED		0x0008
 
 
     /*
diff --git a/arch/m68k/kernel/setup_mm.c b/arch/m68k/kernel/setup_mm.c
index e62fa8f2149b..3a2bb2e8fdad 100644
--- a/arch/m68k/kernel/setup_mm.c
+++ b/arch/m68k/kernel/setup_mm.c
@@ -25,6 +25,7 @@
 #include <linux/module.h>
 #include <linux/nvram.h>
 #include <linux/initrd.h>
+#include <linux/random.h>
 
 #include <asm/bootinfo.h>
 #include <asm/byteorder.h>
@@ -109,10 +110,9 @@ extern void paging_init(void);
 
 static void __init m68k_parse_bootinfo(const struct bi_record *record)
 {
+	const struct bi_record *first_record = record;
 	uint16_t tag;
 
-	save_bootinfo(record);
-
 	while ((tag = be16_to_cpu(record->tag)) != BI_LAST) {
 		int unknown = 0;
 		const void *data = record->data;
@@ -148,10 +148,21 @@ static void __init m68k_parse_bootinfo(const struct bi_record *record)
 			break;
 
 		case BI_COMMAND_LINE:
-			strlcpy(m68k_command_line, data,
+			strscpy(m68k_command_line, data,
 				sizeof(m68k_command_line));
 			break;
 
+		case BI_RNG_SEED: {
+			u16 len = be16_to_cpup(data);
+			add_bootloader_randomness(data + 2, len);
+			/*
+			 * Zero the data to preserve forward secrecy, and zero the
+			 * length to prevent kexec from using it.
+			 */
+			memzero_explicit((void *)data, len + 2);
+			break;
+		}
+
 		default:
 			if (MACH_IS_AMIGA)
 				unknown = amiga_parse_bootinfo(record);
@@ -182,6 +193,8 @@ static void __init m68k_parse_bootinfo(const struct bi_record *record)
 		record = (struct bi_record *)((unsigned long)record + size);
 	}
 
+	save_bootinfo(first_record);
+
 	m68k_realnum_memory = m68k_num_memory;
 #ifdef CONFIG_SINGLE_MEMORY_CHUNK
 	if (m68k_num_memory > 1) {
diff --git a/arch/m68k/virt/config.c b/arch/m68k/virt/config.c
index 4ab22946ff68..632ba200ad42 100644
--- a/arch/m68k/virt/config.c
+++ b/arch/m68k/virt/config.c
@@ -2,7 +2,6 @@
 
 #include <linux/reboot.h>
 #include <linux/serial_core.h>
-#include <linux/random.h>
 #include <clocksource/timer-goldfish.h>
 
 #include <asm/bootinfo.h>
@@ -93,16 +92,6 @@ int __init virt_parse_bootinfo(const struct bi_record *record)
 		data += 4;
 		virt_bi_data.virtio.irq = be32_to_cpup(data);
 		break;
-	case BI_VIRT_RNG_SEED: {
-		u16 len = be16_to_cpup(data);
-		add_bootloader_randomness(data + 2, len);
-		/*
-		 * Zero the data to preserve forward secrecy, and zero the
-		 * length to prevent kexec from using it.
-		 */
-		memzero_explicit((void *)data, len + 2);
-		break;
-	}
 	default:
 		unknown = 1;
 		break;
diff --git a/arch/mips/bcm47xx/prom.c b/arch/mips/bcm47xx/prom.c
index ab203e66ba0d..a9bea411d928 100644
--- a/arch/mips/bcm47xx/prom.c
+++ b/arch/mips/bcm47xx/prom.c
@@ -86,7 +86,7 @@ static __init void prom_init_mem(void)
 			pr_debug("Assume 128MB RAM\n");
 			break;
 		}
-		if (!memcmp(prom_init, prom_init + mem, 32))
+		if (!memcmp((void *)prom_init, (void *)prom_init + mem, 32))
 			break;
 	}
 	lowmem = mem;
@@ -159,7 +159,7 @@ void __init bcm47xx_prom_highmem_init(void)
 
 	off = EXTVBASE + __pa(off);
 	for (extmem = 128 << 20; extmem < 512 << 20; extmem <<= 1) {
-		if (!memcmp(prom_init, (void *)(off + extmem), 16))
+		if (!memcmp((void *)prom_init, (void *)(off + extmem), 16))
 			break;
 	}
 	extmem -= lowmem;
diff --git a/arch/mips/boot/dts/brcm/bcm63268.dtsi b/arch/mips/boot/dts/brcm/bcm63268.dtsi
index c3ce49ec675f..8926417a8fbc 100644
--- a/arch/mips/boot/dts/brcm/bcm63268.dtsi
+++ b/arch/mips/boot/dts/brcm/bcm63268.dtsi
@@ -105,14 +105,20 @@
 			interrupts = <2>, <3>;
 		};
 
-		wdt: watchdog@1000009c {
-			compatible = "brcm,bcm7038-wdt";
-			reg = <0x1000009c 0xc>;
+		timer-mfd@10000080 {
+			compatible = "brcm,bcm7038-twd", "simple-mfd", "syscon";
+			reg = <0x10000080 0x30>;
+			ranges = <0x0 0x10000080 0x30>;
 
-			clocks = <&periph_osc>;
-			clock-names = "refclk";
+			wdt: watchdog@1c {
+				compatible = "brcm,bcm7038-wdt";
+				reg = <0x1c 0xc>;
 
-			timeout-sec = <30>;
+				clocks = <&periph_osc>;
+				clock-names = "refclk";
+
+				timeout-sec = <30>;
+			};
 		};
 
 		uart0: serial@10000180 {
diff --git a/arch/mips/boot/dts/lantiq/Makefile b/arch/mips/boot/dts/lantiq/Makefile
index f5dfc06242b9..ae6e3e21ebeb 100644
--- a/arch/mips/boot/dts/lantiq/Makefile
+++ b/arch/mips/boot/dts/lantiq/Makefile
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-dtb-$(CONFIG_DT_EASY50712)	+= easy50712.dtb
+dtb-$(CONFIG_DT_EASY50712)	+= danube_easy50712.dtb
 
 obj-$(CONFIG_BUILTIN_DTB)	+= $(addsuffix .o, $(dtb-y))
diff --git a/arch/mips/boot/dts/lantiq/easy50712.dts b/arch/mips/boot/dts/lantiq/danube_easy50712.dts
index 1ce20b7d05cb..1ce20b7d05cb 100644
--- a/arch/mips/boot/dts/lantiq/easy50712.dts
+++ b/arch/mips/boot/dts/lantiq/danube_easy50712.dts
diff --git a/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc1.dts b/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc1.dts
index 24eebc5a85b1..0128bd8fa7ed 100644
--- a/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc1.dts
+++ b/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc1.dts
@@ -20,12 +20,6 @@
 		bootargs = "console=ttyS0,57600";
 	};
 
-	palmbus: palmbus@1e000000 {
-		i2c@900 {
-			status = "okay";
-		};
-	};
-
 	gpio-keys {
 		compatible = "gpio-keys";
 
@@ -53,7 +47,7 @@
 	};
 };
 
-&sdhci {
+&mmc {
 	status = "okay";
 };
 
diff --git a/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc2.dts b/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc2.dts
index 34006e667780..7515555388ae 100644
--- a/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc2.dts
+++ b/arch/mips/boot/dts/ralink/mt7621-gnubee-gb-pc2.dts
@@ -20,12 +20,6 @@
 		bootargs = "console=ttyS0,57600";
 	};
 
-	palmbus: palmbus@1e000000 {
-		i2c@900 {
-			status = "okay";
-		};
-	};
-
 	gpio-keys {
 		compatible = "gpio-keys";
 
@@ -35,9 +29,45 @@
 			linux,code = <KEY_RESTART>;
 		};
 	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+
+		ethblack-green {
+			label = "green:ethblack";
+			gpios = <&gpio 3 GPIO_ACTIVE_LOW>;
+		};
+
+		ethblue-green {
+			label = "green:ethblue";
+			gpios = <&gpio 4 GPIO_ACTIVE_LOW>;
+		};
+
+		ethyellow-green {
+			label = "green:ethyellow";
+			gpios = <&gpio 15 GPIO_ACTIVE_LOW>;
+		};
+
+		ethyellow-orange {
+			label = "orange:ethyellow";
+			gpios = <&gpio 13 GPIO_ACTIVE_LOW>;
+		};
+
+		power {
+			label = "green:power";
+			gpios = <&gpio 6 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "default-on";
+		};
+
+		system {
+			label = "green:system";
+			gpios = <&gpio 8 GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "disk-activity";
+		};
+	};
 };
 
-&sdhci {
+&mmc {
 	status = "okay";
 };
 
@@ -83,12 +113,12 @@
 
 &gmac1 {
 	status = "okay";
-	phy-handle = <&ethphy7>;
+	phy-handle = <&ethphy5>;
 };
 
 &mdio {
-	ethphy7: ethernet-phy@7 {
-		reg = <7>;
+	ethphy5: ethernet-phy@5 {
+		reg = <5>;
 		phy-mode = "rgmii-rxid";
 	};
 };
diff --git a/arch/mips/boot/dts/ralink/mt7621.dtsi b/arch/mips/boot/dts/ralink/mt7621.dtsi
index ee46ace0bcc1..f3f4c1f26e01 100644
--- a/arch/mips/boot/dts/ralink/mt7621.dtsi
+++ b/arch/mips/boot/dts/ralink/mt7621.dtsi
@@ -33,11 +33,6 @@
 		compatible = "mti,cpu-interrupt-controller";
 	};
 
-	aliases {
-		serial0 = &uartlite;
-	};
-
-
 	mmc_fixed_3v3: regulator-3v3 {
 		compatible = "regulator-fixed";
 		regulator-name = "mmc_power";
@@ -110,17 +105,16 @@
 			pinctrl-0 = <&i2c_pins>;
 		};
 
-		memc: syscon@5000 {
+		memc: memory-controller@5000 {
 			compatible = "mediatek,mt7621-memc", "syscon";
 			reg = <0x5000 0x1000>;
 		};
 
-		uartlite: uartlite@c00 {
+		serial0: serial@c00 {
 			compatible = "ns16550a";
 			reg = <0xc00 0x100>;
 
 			clocks = <&sysc MT7621_CLK_UART1>;
-			clock-names = "uart1";
 
 			interrupt-parent = <&gic>;
 			interrupts = <GIC_SHARED 26 IRQ_TYPE_LEVEL_HIGH>;
@@ -236,7 +230,7 @@
 		};
 	};
 
-	sdhci: sdhci@1e130000 {
+	mmc: mmc@1e130000 {
 		status = "disabled";
 
 		compatible = "mediatek,mt7620-mmc";
@@ -262,8 +256,8 @@
 		interrupts = <GIC_SHARED 20 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
-	xhci: xhci@1e1c0000 {
-		compatible = "mediatek,mt8173-xhci";
+	usb: usb@1e1c0000 {
+		compatible = "mediatek,mt8173-xhci", "mediatek,mtk-xhci";
 		reg = <0x1e1c0000 0x1000
 		       0x1e1d0700 0x0100>;
 		reg-names = "mac", "ippc";
@@ -338,23 +332,22 @@
 		gmac1: mac@1 {
 			compatible = "mediatek,eth-mac";
 			reg = <1>;
-			status = "off";
-			phy-mode = "rgmii-rxid";
+			status = "disabled";
+			phy-mode = "rgmii";
 		};
 
 		mdio: mdio-bus {
 			#address-cells = <1>;
 			#size-cells = <0>;
 
-			switch0: switch0@0 {
+			switch0: switch@1f {
 				compatible = "mediatek,mt7621";
-				reg = <0>;
+				reg = <0x1f>;
 				mediatek,mcm;
 				resets = <&sysc MT7621_RST_MCM>;
 				reset-names = "mcm";
 				interrupt-controller;
 				#interrupt-cells = <1>;
-				interrupt-parent = <&gic>;
 				interrupts = <GIC_SHARED 23 IRQ_TYPE_LEVEL_HIGH>;
 
 				ports {
@@ -362,31 +355,31 @@
 					#size-cells = <0>;
 
 					port@0 {
-						status = "off";
+						status = "disabled";
 						reg = <0>;
 						label = "lan0";
 					};
 
 					port@1 {
-						status = "off";
+						status = "disabled";
 						reg = <1>;
 						label = "lan1";
 					};
 
 					port@2 {
-						status = "off";
+						status = "disabled";
 						reg = <2>;
 						label = "lan2";
 					};
 
 					port@3 {
-						status = "off";
+						status = "disabled";
 						reg = <3>;
 						label = "lan3";
 					};
 
 					port@4 {
-						status = "off";
+						status = "disabled";
 						reg = <4>;
 						label = "lan4";
 					};
diff --git a/arch/mips/cavium-octeon/oct_ilm.c b/arch/mips/cavium-octeon/oct_ilm.c
index 6a4694538bb6..dc05262e85ff 100644
--- a/arch/mips/cavium-octeon/oct_ilm.c
+++ b/arch/mips/cavium-octeon/oct_ilm.c
@@ -28,7 +28,7 @@ struct latency_info {
 static struct latency_info li;
 static struct dentry *dir;
 
-static int show_latency(struct seq_file *m, void *v)
+static int oct_ilm_show(struct seq_file *m, void *v)
 {
 	u64 cpuclk, avg, max, min;
 	struct latency_info curr_li = li;
@@ -43,18 +43,7 @@ static int show_latency(struct seq_file *m, void *v)
 		   curr_li.interrupt_cnt, avg, max, min);
 	return 0;
 }
-
-static int oct_ilm_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, show_latency, NULL);
-}
-
-static const struct file_operations oct_ilm_ops = {
-	.open = oct_ilm_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(oct_ilm);
 
 static int reset_statistics(void *data, u64 value)
 {
@@ -67,7 +56,7 @@ DEFINE_DEBUGFS_ATTRIBUTE(reset_statistics_ops, NULL, reset_statistics, "%llu\n")
 static void init_debugfs(void)
 {
 	dir = debugfs_create_dir("oct_ilm", 0);
-	debugfs_create_file("statistics", 0222, dir, NULL, &oct_ilm_ops);
+	debugfs_create_file("statistics", 0222, dir, NULL, &oct_ilm_fops);
 	debugfs_create_file("reset", 0222, dir, NULL, &reset_statistics_ops);
 }
 
diff --git a/arch/mips/cavium-octeon/setup.c b/arch/mips/cavium-octeon/setup.c
index e7f994393ae8..a71727f7a608 100644
--- a/arch/mips/cavium-octeon/setup.c
+++ b/arch/mips/cavium-octeon/setup.c
@@ -530,7 +530,7 @@ void octeon_user_io_init(void)
 	/* Get the current settings for CP0_CVMMEMCTL_REG */
 	cvmmemctl.u64 = read_c0_cvmmemctl();
 	/* R/W If set, marked write-buffer entries time out the same
-	 * as as other entries; if clear, marked write-buffer entries
+	 * as other entries; if clear, marked write-buffer entries
 	 * use the maximum timeout. */
 	cvmmemctl.s.dismarkwblongto = 1;
 	/* R/W If set, a merged store does not clear the write-buffer
diff --git a/arch/mips/configs/ar7_defconfig b/arch/mips/configs/ar7_defconfig
index cf9c6329b807..ed4a6388791e 100644
--- a/arch/mips/configs/ar7_defconfig
+++ b/arch/mips/configs/ar7_defconfig
@@ -32,9 +32,6 @@ CONFIG_IP_ROUTE_MULTIPATH=y
 CONFIG_IP_ROUTE_VERBOSE=y
 CONFIG_IP_MROUTE=y
 CONFIG_SYN_COOKIES=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_INET_DIAG is not set
 CONFIG_TCP_CONG_ADVANCED=y
 # CONFIG_TCP_CONG_BIC is not set
@@ -117,7 +114,6 @@ CONFIG_JFFS2_SUMMARY=y
 CONFIG_JFFS2_COMPRESSION_OPTIONS=y
 CONFIG_SQUASHFS=y
 # CONFIG_CRYPTO_HW is not set
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_STRIP_ASM_SYMS=y
 CONFIG_DEBUG_FS=y
 CONFIG_CMDLINE_BOOL=y
diff --git a/arch/mips/configs/ath25_defconfig b/arch/mips/configs/ath25_defconfig
index 7143441f5476..afd1c16242e9 100644
--- a/arch/mips/configs/ath25_defconfig
+++ b/arch/mips/configs/ath25_defconfig
@@ -29,9 +29,6 @@ CONFIG_UNIX=y
 CONFIG_INET=y
 CONFIG_IP_MULTICAST=y
 CONFIG_IP_ADVANCED_ROUTER=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_IPV6 is not set
 CONFIG_CFG80211=m
 CONFIG_MAC80211=m
@@ -108,7 +105,6 @@ CONFIG_SQUASHFS_XZ=y
 # CONFIG_XZ_DEC_ARMTHUMB is not set
 # CONFIG_XZ_DEC_SPARC is not set
 CONFIG_PRINTK_TIME=y
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_STRIP_ASM_SYMS=y
 CONFIG_DEBUG_FS=y
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/mips/configs/ath79_defconfig b/arch/mips/configs/ath79_defconfig
index 96622a2ad333..0b741716c852 100644
--- a/arch/mips/configs/ath79_defconfig
+++ b/arch/mips/configs/ath79_defconfig
@@ -10,12 +10,6 @@ CONFIG_EMBEDDED=y
 # CONFIG_SLUB_DEBUG is not set
 # CONFIG_COMPAT_BRK is not set
 CONFIG_ATH79=y
-CONFIG_ATH79_MACH_AP121=y
-CONFIG_ATH79_MACH_AP136=y
-CONFIG_ATH79_MACH_AP81=y
-CONFIG_ATH79_MACH_DB120=y
-CONFIG_ATH79_MACH_PB44=y
-CONFIG_ATH79_MACH_UBNT_XM=y
 CONFIG_HZ_100=y
 # CONFIG_SECCOMP is not set
 CONFIG_PCI=y
@@ -29,9 +23,6 @@ CONFIG_UNIX=y
 CONFIG_INET=y
 CONFIG_IP_MULTICAST=y
 CONFIG_IP_ADVANCED_ROUTER=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_IPV6 is not set
 CONFIG_CFG80211=m
 CONFIG_MAC80211=m
@@ -92,7 +83,6 @@ CONFIG_LEDS_GPIO=y
 # CONFIG_DNOTIFY is not set
 # CONFIG_PROC_PAGE_MONITOR is not set
 CONFIG_CRC_ITU_T=m
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_STRIP_ASM_SYMS=y
 CONFIG_DEBUG_FS=y
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/mips/configs/bcm63xx_defconfig b/arch/mips/configs/bcm63xx_defconfig
index 861f680184b9..34d0ca638ef0 100644
--- a/arch/mips/configs/bcm63xx_defconfig
+++ b/arch/mips/configs/bcm63xx_defconfig
@@ -24,9 +24,6 @@ CONFIG_PCMCIA_BCM63XX=y
 CONFIG_NET=y
 CONFIG_UNIX=y
 CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_INET_DIAG is not set
 # CONFIG_IPV6 is not set
 CONFIG_CFG80211=y
diff --git a/arch/mips/configs/bigsur_defconfig b/arch/mips/configs/bigsur_defconfig
index d83e7d600b0a..d15961f00ece 100644
--- a/arch/mips/configs/bigsur_defconfig
+++ b/arch/mips/configs/bigsur_defconfig
@@ -49,8 +49,6 @@ CONFIG_IP_PIMSM_V2=y
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
 CONFIG_INET_IPCOMP=m
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
 CONFIG_TCP_MD5SIG=y
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_ROUTE_INFO=y
@@ -59,7 +57,6 @@ CONFIG_INET6_AH=m
 CONFIG_INET6_ESP=m
 CONFIG_INET6_IPCOMP=m
 CONFIG_IPV6_MIP6=m
-CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m
 CONFIG_IPV6_SIT_6RD=y
 CONFIG_IPV6_TUNNEL=m
 CONFIG_IPV6_MULTIPLE_TABLES=y
@@ -101,7 +98,6 @@ CONFIG_BAYCOM_SER_HDX=m
 CONFIG_YAM=m
 CONFIG_FW_LOADER=m
 CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_EEPROM_LEGACY=y
 CONFIG_EEPROM_MAX6875=y
@@ -230,12 +226,8 @@ CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_RMD128=m
 CONFIG_CRYPTO_RMD160=m
-CONFIG_CRYPTO_RMD256=m
-CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
@@ -243,7 +235,6 @@ CONFIG_CRYPTO_CAMELLIA=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
-CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/mips/configs/bmips_be_defconfig b/arch/mips/configs/bmips_be_defconfig
index 032bb51defe8..daef132d000b 100644
--- a/arch/mips/configs/bmips_be_defconfig
+++ b/arch/mips/configs/bmips_be_defconfig
@@ -17,9 +17,6 @@ CONFIG_PACKET=y
 CONFIG_PACKET_DIAG=y
 CONFIG_UNIX=y
 CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_INET_DIAG is not set
 CONFIG_CFG80211=y
 CONFIG_NL80211_TESTMODE=y
diff --git a/arch/mips/configs/bmips_stb_defconfig b/arch/mips/configs/bmips_stb_defconfig
index 5956fb95c19f..cd0dc37c3d84 100644
--- a/arch/mips/configs/bmips_stb_defconfig
+++ b/arch/mips/configs/bmips_stb_defconfig
@@ -12,7 +12,6 @@ CONFIG_HIGHMEM=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_SMP=y
 CONFIG_NR_CPUS=4
-CONFIG_CC_STACKPROTECTOR_STRONG=y
 # CONFIG_SECCOMP is not set
 CONFIG_MIPS_O32_FP64_SUPPORT=y
 # CONFIG_RD_GZIP is not set
@@ -21,8 +20,6 @@ CONFIG_MIPS_O32_FP64_SUPPORT=y
 CONFIG_RD_XZ=y
 # CONFIG_RD_LZO is not set
 # CONFIG_RD_LZ4 is not set
-# CONFIG_IOSCHED_DEADLINE is not set
-# CONFIG_IOSCHED_CFQ is not set
 CONFIG_PCI=y
 CONFIG_PCI_MSI=y
 CONFIG_PCIEASPM_POWERSAVE=y
@@ -30,7 +27,6 @@ CONFIG_PCIEPORTBUS=y
 CONFIG_PCIE_BRCMSTB=y
 CONFIG_CPU_FREQ=y
 CONFIG_CPU_FREQ_STAT=y
-CONFIG_CPU_FREQ_STAT_DETAILS=y
 CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
 CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
 CONFIG_CPU_FREQ_GOV_POWERSAVE=y
@@ -44,15 +40,11 @@ CONFIG_PACKET=y
 CONFIG_PACKET_DIAG=y
 CONFIG_UNIX=y
 CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_INET_DIAG is not set
 CONFIG_CFG80211=y
 CONFIG_NL80211_TESTMODE=y
 CONFIG_WIRELESS=y
 CONFIG_MAC80211=y
-CONFIG_NL80211=y
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 # CONFIG_STANDALONE is not set
@@ -70,10 +62,6 @@ CONFIG_IP_PNP_RARP=y
 CONFIG_IP_MROUTE=y
 CONFIG_IP_PIMSM_V1=y
 CONFIG_IP_PIMSM_V2=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
-# CONFIG_INET_LRO is not set
 CONFIG_INET_UDP_DIAG=y
 CONFIG_TCP_CONG_ADVANCED=y
 CONFIG_TCP_CONG_BIC=y
@@ -93,7 +81,6 @@ CONFIG_NET_SWITCHDEV=y
 CONFIG_DMA_CMA=y
 CONFIG_CMA_ALIGNMENT=12
 CONFIG_SPI=y
-CONFIG_SPI_BRCMSTB=y
 CONFIG_MTD=y
 CONFIG_MTD_CMDLINE_PARTS=y
 CONFIG_MTD_BLOCK=y
@@ -105,14 +92,11 @@ CONFIG_MTD_CFI_STAA=y
 CONFIG_MTD_ROM=y
 CONFIG_MTD_ABSENT=y
 CONFIG_MTD_PHYSMAP_OF=y
-CONFIG_MTD_M25P80=y
-CONFIG_MTD_NAND=y
 CONFIG_MTD_NAND_BRCMNAND=y
 CONFIG_MTD_SPI_NOR=y
 # CONFIG_MTD_SPI_NOR_USE_4K_SECTORS is not set
 CONFIG_MTD_UBI=y
 CONFIG_MTD_UBI_GLUEBI=y
-CONFIG_PROC_DEVICETREE=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=8192
@@ -120,7 +104,6 @@ CONFIG_BLK_DEV_RAM_SIZE=8192
 CONFIG_SCSI=y
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_SG=y
-CONFIG_SCSI_MULTI_LUN=y
 # CONFIG_SCSI_LOWLEVEL is not set
 CONFIG_NETDEVICES=y
 CONFIG_VLAN_8021Q=y
@@ -135,7 +118,6 @@ CONFIG_INPUT_UINPUT=y
 # CONFIG_SERIO is not set
 CONFIG_VT=y
 CONFIG_VT_HW_CONSOLE_BINDING=y
-# CONFIG_DEVKMEM is not set
 CONFIG_SERIAL_8250=y
 # CONFIG_SERIAL_8250_DEPRECATED_OPTIONS is not set
 CONFIG_SERIAL_8250_CONSOLE=y
@@ -203,17 +185,14 @@ CONFIG_CMDLINE="earlycon"
 CONFIG_MIPS_CMDLINE_DTB_EXTEND=y
 # CONFIG_MIPS_CMDLINE_FROM_BOOTLOADER is not set
 # CONFIG_CRYPTO_HW is not set
-CONFIG_DT_BCM974XX=y
 CONFIG_FW_CFE=y
 CONFIG_ATA=y
 CONFIG_SATA_AHCI_PLATFORM=y
-CONFIG_AHCI_BRCMSTB=y
 CONFIG_GENERIC_PHY=y
 CONFIG_GPIOLIB=y
 CONFIG_GPIO_SYSFS=y
 CONFIG_PHY_BRCM_USB=y
 CONFIG_PHY_BRCM_SATA=y
-CONFIG_PM_RUNTIME=y
 CONFIG_PM_DEBUG=y
 CONFIG_SYSVIPC=y
 CONFIG_FUNCTION_GRAPH_TRACER=y
@@ -227,3 +206,5 @@ CONFIG_FTRACE_SYSCALLS=y
 CONFIG_TRACER_SNAPSHOT=y
 CONFIG_TRACER_SNAPSHOT_PER_CPU_SWAP=y
 CONFIG_STACK_TRACER=y
+CONFIG_AHCI_BRCM=y
+CONFIG_MTD_RAW_NAND=y
diff --git a/arch/mips/configs/cavium_octeon_defconfig b/arch/mips/configs/cavium_octeon_defconfig
index 97ceaf080c0c..a2311495af79 100644
--- a/arch/mips/configs/cavium_octeon_defconfig
+++ b/arch/mips/configs/cavium_octeon_defconfig
@@ -71,7 +71,6 @@ CONFIG_NETDEVICES=y
 # CONFIG_NET_VENDOR_DEC is not set
 # CONFIG_NET_VENDOR_DLINK is not set
 # CONFIG_NET_VENDOR_EMULEX is not set
-# CONFIG_NET_VENDOR_HP is not set
 # CONFIG_NET_VENDOR_INTEL is not set
 # CONFIG_NET_VENDOR_MARVELL is not set
 # CONFIG_NET_VENDOR_MELLANOX is not set
diff --git a/arch/mips/configs/db1xxx_defconfig b/arch/mips/configs/db1xxx_defconfig
index b8bd66300996..a8b62df3c021 100644
--- a/arch/mips/configs/db1xxx_defconfig
+++ b/arch/mips/configs/db1xxx_defconfig
@@ -61,7 +61,6 @@ CONFIG_INET6_AH=y
 CONFIG_INET6_ESP=y
 CONFIG_INET6_IPCOMP=y
 CONFIG_IPV6_MIP6=y
-CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=y
 CONFIG_IPV6_VTI=y
 CONFIG_IPV6_SIT_6RD=y
 CONFIG_IPV6_GRE=y
diff --git a/arch/mips/configs/decstation_64_defconfig b/arch/mips/configs/decstation_64_defconfig
index 0021427a1bbe..49ec1575234e 100644
--- a/arch/mips/configs/decstation_64_defconfig
+++ b/arch/mips/configs/decstation_64_defconfig
@@ -37,9 +37,6 @@ CONFIG_SYN_COOKIES=y
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
 CONFIG_INET_IPCOMP=m
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
-CONFIG_INET_XFRM_MODE_BEET=m
 CONFIG_TCP_MD5SIG=y
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_ROUTE_INFO=y
@@ -47,14 +44,11 @@ CONFIG_INET6_AH=m
 CONFIG_INET6_ESP=m
 CONFIG_INET6_IPCOMP=m
 CONFIG_IPV6_MIP6=m
-CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m
 CONFIG_IPV6_MULTIPLE_TABLES=y
 CONFIG_IPV6_SUBTREES=y
 CONFIG_NETWORK_SECMARK=y
 CONFIG_IP_SCTP=m
 CONFIG_VLAN_8021Q=m
-CONFIG_DECNET=m
-CONFIG_DECNET_ROUTER=y
 # CONFIG_WIRELESS is not set
 # CONFIG_UEVENT_HELPER is not set
 # CONFIG_FW_LOADER is not set
@@ -79,7 +73,6 @@ CONFIG_NETDEVICES=y
 CONFIG_DECLANCE=y
 # CONFIG_NET_VENDOR_AQUANTIA is not set
 # CONFIG_NET_VENDOR_ARC is not set
-# CONFIG_NET_VENDOR_AURORA is not set
 # CONFIG_NET_VENDOR_BROADCOM is not set
 # CONFIG_NET_VENDOR_CADENCE is not set
 # CONFIG_NET_VENDOR_CAVIUM is not set
@@ -193,12 +186,8 @@ CONFIG_CRYPTO_CRC32=m
 CONFIG_CRYPTO_CRCT10DIF=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_RMD128=m
 CONFIG_CRYPTO_RMD160=m
-CONFIG_CRYPTO_RMD256=m
-CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARC4=m
@@ -208,7 +197,6 @@ CONFIG_CRYPTO_CAST5=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
-CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/mips/configs/decstation_defconfig b/arch/mips/configs/decstation_defconfig
index 7a97a0818ce4..5cec4c096e2c 100644
--- a/arch/mips/configs/decstation_defconfig
+++ b/arch/mips/configs/decstation_defconfig
@@ -33,9 +33,6 @@ CONFIG_SYN_COOKIES=y
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
 CONFIG_INET_IPCOMP=m
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
-CONFIG_INET_XFRM_MODE_BEET=m
 CONFIG_TCP_MD5SIG=y
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_ROUTE_INFO=y
@@ -43,14 +40,11 @@ CONFIG_INET6_AH=m
 CONFIG_INET6_ESP=m
 CONFIG_INET6_IPCOMP=m
 CONFIG_IPV6_MIP6=m
-CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m
 CONFIG_IPV6_MULTIPLE_TABLES=y
 CONFIG_IPV6_SUBTREES=y
 CONFIG_NETWORK_SECMARK=y
 CONFIG_IP_SCTP=m
 CONFIG_VLAN_8021Q=m
-CONFIG_DECNET=m
-CONFIG_DECNET_ROUTER=y
 # CONFIG_WIRELESS is not set
 # CONFIG_UEVENT_HELPER is not set
 # CONFIG_FW_LOADER is not set
@@ -75,7 +69,6 @@ CONFIG_NETDEVICES=y
 CONFIG_DECLANCE=y
 # CONFIG_NET_VENDOR_AQUANTIA is not set
 # CONFIG_NET_VENDOR_ARC is not set
-# CONFIG_NET_VENDOR_AURORA is not set
 # CONFIG_NET_VENDOR_BROADCOM is not set
 # CONFIG_NET_VENDOR_CADENCE is not set
 # CONFIG_NET_VENDOR_CAVIUM is not set
@@ -188,12 +181,8 @@ CONFIG_CRYPTO_CRC32=m
 CONFIG_CRYPTO_CRCT10DIF=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_RMD128=m
 CONFIG_CRYPTO_RMD160=m
-CONFIG_CRYPTO_RMD256=m
-CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARC4=m
@@ -203,7 +192,6 @@ CONFIG_CRYPTO_CAST5=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
-CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/mips/configs/decstation_r4k_defconfig b/arch/mips/configs/decstation_r4k_defconfig
index a0643363526d..af37e26d9b5b 100644
--- a/arch/mips/configs/decstation_r4k_defconfig
+++ b/arch/mips/configs/decstation_r4k_defconfig
@@ -32,9 +32,6 @@ CONFIG_SYN_COOKIES=y
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
 CONFIG_INET_IPCOMP=m
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
-CONFIG_INET_XFRM_MODE_BEET=m
 CONFIG_TCP_MD5SIG=y
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_ROUTE_INFO=y
@@ -42,14 +39,11 @@ CONFIG_INET6_AH=m
 CONFIG_INET6_ESP=m
 CONFIG_INET6_IPCOMP=m
 CONFIG_IPV6_MIP6=m
-CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m
 CONFIG_IPV6_MULTIPLE_TABLES=y
 CONFIG_IPV6_SUBTREES=y
 CONFIG_NETWORK_SECMARK=y
 CONFIG_IP_SCTP=m
 CONFIG_VLAN_8021Q=m
-CONFIG_DECNET=m
-CONFIG_DECNET_ROUTER=y
 # CONFIG_WIRELESS is not set
 # CONFIG_UEVENT_HELPER is not set
 # CONFIG_FW_LOADER is not set
@@ -74,7 +68,6 @@ CONFIG_NETDEVICES=y
 CONFIG_DECLANCE=y
 # CONFIG_NET_VENDOR_AQUANTIA is not set
 # CONFIG_NET_VENDOR_ARC is not set
-# CONFIG_NET_VENDOR_AURORA is not set
 # CONFIG_NET_VENDOR_BROADCOM is not set
 # CONFIG_NET_VENDOR_CADENCE is not set
 # CONFIG_NET_VENDOR_CAVIUM is not set
@@ -188,12 +181,8 @@ CONFIG_CRYPTO_CRC32=m
 CONFIG_CRYPTO_CRCT10DIF=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_RMD128=m
 CONFIG_CRYPTO_RMD160=m
-CONFIG_CRYPTO_RMD256=m
-CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_ARC4=m
@@ -203,7 +192,6 @@ CONFIG_CRYPTO_CAST5=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
-CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/mips/configs/fuloong2e_defconfig b/arch/mips/configs/fuloong2e_defconfig
index ba47c5e929b7..843d6a5a4f61 100644
--- a/arch/mips/configs/fuloong2e_defconfig
+++ b/arch/mips/configs/fuloong2e_defconfig
@@ -35,8 +35,6 @@ CONFIG_IP_MULTICAST=y
 CONFIG_IP_PNP=y
 CONFIG_IP_PNP_BOOTP=y
 CONFIG_NET_IPIP=m
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
 # CONFIG_INET_DIAG is not set
 # CONFIG_IPV6 is not set
 CONFIG_NETFILTER=y
@@ -92,7 +90,6 @@ CONFIG_MTD_CFI_AMDSTD=m
 CONFIG_MTD_CFI_STAA=m
 CONFIG_MTD_PHYSMAP=m
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_RAM=m
 CONFIG_CDROM_PKTCDVD=m
 CONFIG_ATA_OVER_ETH=m
@@ -159,7 +156,6 @@ CONFIG_USB_MOUSE=y
 CONFIG_USB=y
 CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
 CONFIG_USB_OTG_PRODUCTLIST=y
-CONFIG_USB_WUSB_CBAF=m
 CONFIG_USB_C67X00_HCD=m
 CONFIG_USB_EHCI_HCD=y
 CONFIG_USB_EHCI_ROOT_HUB_TT=y
@@ -219,15 +215,10 @@ CONFIG_CRYPTO_CTS=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_HMAC=y
-CONFIG_CRYPTO_RMD128=m
 CONFIG_CRYPTO_RMD160=m
-CONFIG_CRYPTO_RMD256=m
-CONFIG_CRYPTO_RMD320=m
-CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_DEFLATE=m
 CONFIG_CRYPTO_LZO=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC_CCITT=y
 CONFIG_CRC7=m
-# CONFIG_ENABLE_MUST_CHECK is not set
diff --git a/arch/mips/configs/generic/board-ocelot.config b/arch/mips/configs/generic/board-ocelot.config
index 510709565404..8cfbafa532e0 100644
--- a/arch/mips/configs/generic/board-ocelot.config
+++ b/arch/mips/configs/generic/board-ocelot.config
@@ -25,7 +25,6 @@ CONFIG_NETDEVICES=y
 CONFIG_NET_SWITCHDEV=y
 CONFIG_NET_DSA=y
 CONFIG_MSCC_OCELOT_SWITCH=y
-CONFIG_MSCC_OCELOT_SWITCH_OCELOT=y
 CONFIG_MDIO_MSCC_MIIM=y
 CONFIG_MICROSEMI_PHY=y
 
diff --git a/arch/mips/configs/gpr_defconfig b/arch/mips/configs/gpr_defconfig
index d82f4ebf687f..eb755650f821 100644
--- a/arch/mips/configs/gpr_defconfig
+++ b/arch/mips/configs/gpr_defconfig
@@ -29,9 +29,6 @@ CONFIG_IP_ROUTE_VERBOSE=y
 CONFIG_IP_PNP=y
 CONFIG_IP_PNP_BOOTP=y
 CONFIG_SYN_COOKIES=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_IPV6 is not set
 CONFIG_NETWORK_SECMARK=y
 CONFIG_NETFILTER=y
@@ -69,7 +66,6 @@ CONFIG_IP_NF_RAW=m
 CONFIG_IP_NF_ARPTABLES=m
 CONFIG_IP_NF_ARPFILTER=m
 CONFIG_IP_NF_ARP_MANGLE=m
-CONFIG_DECNET_NF_GRABULATOR=m
 CONFIG_BRIDGE_NF_EBTABLES=m
 CONFIG_BRIDGE_EBT_BROUTE=m
 CONFIG_BRIDGE_EBT_T_FILTER=m
@@ -99,7 +95,6 @@ CONFIG_ATM_MPOA=m
 CONFIG_ATM_BR2684=m
 CONFIG_BRIDGE=m
 CONFIG_VLAN_8021Q=m
-CONFIG_DECNET=m
 CONFIG_LLC2=m
 CONFIG_ATALK=m
 CONFIG_DEV_APPLETALK=m
@@ -220,9 +215,6 @@ CONFIG_HDLC_X25=m
 CONFIG_PCI200SYN=m
 CONFIG_WANXL=m
 CONFIG_FARSYNC=m
-CONFIG_DSCC4=m
-CONFIG_DSCC4_PCISYNC=y
-CONFIG_DSCC4_PCI_RST=y
 CONFIG_LAPBETHER=m
 # CONFIG_INPUT_KEYBOARD is not set
 # CONFIG_INPUT_MOUSE is not set
@@ -288,7 +280,6 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_SHA512=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
@@ -299,7 +290,6 @@ CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_DEFLATE=m
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_MAGIC_SYSRQ=y
 CONFIG_CMDLINE_BOOL=y
 CONFIG_CMDLINE="console=ttyS0,115200 root=/dev/nfs rw ip=auto"
diff --git a/arch/mips/configs/ip22_defconfig b/arch/mips/configs/ip22_defconfig
index 70a4ba90f491..13df29901237 100644
--- a/arch/mips/configs/ip22_defconfig
+++ b/arch/mips/configs/ip22_defconfig
@@ -36,9 +36,6 @@ CONFIG_IP_PNP_BOOTP=y
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
 CONFIG_INET_IPCOMP=m
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
-CONFIG_INET_XFRM_MODE_BEET=m
 CONFIG_TCP_MD5SIG=y
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_ROUTE_INFO=y
@@ -47,7 +44,6 @@ CONFIG_INET6_AH=m
 CONFIG_INET6_ESP=m
 CONFIG_INET6_IPCOMP=m
 CONFIG_IPV6_MIP6=m
-CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m
 CONFIG_IPV6_TUNNEL=m
 CONFIG_IPV6_MULTIPLE_TABLES=y
 CONFIG_IPV6_SUBTREES=y
@@ -226,7 +222,6 @@ CONFIG_SERIO_RAW=m
 CONFIG_VT_HW_CONSOLE_BINDING=y
 CONFIG_SERIAL_IP22_ZILOG=m
 # CONFIG_HW_RANDOM is not set
-CONFIG_RAW_DRIVER=m
 # CONFIG_HWMON is not set
 CONFIG_THERMAL=y
 CONFIG_WATCHDOG=y
@@ -320,11 +315,7 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_XCBC=m
-CONFIG_CRYPTO_RMD128=m
 CONFIG_CRYPTO_RMD160=m
-CONFIG_CRYPTO_RMD256=m
-CONFIG_CRYPTO_RMD320=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
@@ -333,7 +324,6 @@ CONFIG_CRYPTO_CAST5=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
-CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/mips/configs/ip27_defconfig b/arch/mips/configs/ip27_defconfig
index 821630ac1be7..3e86f8106ba0 100644
--- a/arch/mips/configs/ip27_defconfig
+++ b/arch/mips/configs/ip27_defconfig
@@ -33,9 +33,6 @@ CONFIG_NET_KEY_MIGRATE=y
 CONFIG_INET=y
 CONFIG_IP_MULTICAST=y
 CONFIG_IP_PNP=y
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
-CONFIG_INET_XFRM_MODE_BEET=m
 CONFIG_TCP_MD5SIG=y
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_ROUTE_INFO=y
@@ -44,10 +41,6 @@ CONFIG_INET6_AH=m
 CONFIG_INET6_ESP=m
 CONFIG_INET6_IPCOMP=m
 CONFIG_IPV6_MIP6=m
-CONFIG_INET6_XFRM_MODE_TRANSPORT=m
-CONFIG_INET6_XFRM_MODE_TUNNEL=m
-CONFIG_INET6_XFRM_MODE_BEET=m
-CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m
 CONFIG_IPV6_SIT=m
 CONFIG_IPV6_SIT_6RD=y
 CONFIG_IPV6_TUNNEL=m
@@ -92,7 +85,6 @@ CONFIG_CFG80211=m
 CONFIG_MAC80211=m
 CONFIG_RFKILL=m
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_CDROM_PKTCDVD=m
 CONFIG_ATA_OVER_ETH=m
 CONFIG_SCSI=y
@@ -115,7 +107,6 @@ CONFIG_SCSI_AIC94XX=m
 # CONFIG_AIC94XX_DEBUG is not set
 CONFIG_SCSI_MVSAS=m
 # CONFIG_SCSI_MVSAS_DEBUG is not set
-CONFIG_SCSI_DPT_I2O=m
 CONFIG_SCSI_MPT2SAS=m
 CONFIG_LIBFC=m
 CONFIG_SCSI_QLOGIC_1280=y
@@ -126,8 +117,6 @@ CONFIG_SCSI_DH_RDAC=m
 CONFIG_SCSI_DH_HP_SW=m
 CONFIG_SCSI_DH_EMC=m
 CONFIG_SCSI_DH_ALUA=m
-CONFIG_SCSI_OSD_INITIATOR=m
-CONFIG_SCSI_OSD_ULD=m
 CONFIG_MD=y
 CONFIG_BLK_DEV_MD=y
 CONFIG_MD_LINEAR=m
@@ -166,7 +155,6 @@ CONFIG_JME=m
 CONFIG_MLX4_EN=m
 # CONFIG_MLX4_DEBUG is not set
 CONFIG_KS8851_MLL=m
-CONFIG_VXGE=m
 CONFIG_AX88796=m
 CONFIG_AX88796_93CX6=y
 CONFIG_ETHOC=m
@@ -264,7 +252,6 @@ CONFIG_I2C_VIAPRO=m
 CONFIG_I2C_OCORES=m
 CONFIG_I2C_PCA_PLATFORM=m
 CONFIG_I2C_SIMTEC=m
-CONFIG_I2C_PARPORT_LIGHT=m
 CONFIG_I2C_TAOS_EVM=m
 CONFIG_I2C_STUB=m
 # CONFIG_HWMON is not set
@@ -309,7 +296,6 @@ CONFIG_TMPFS=y
 CONFIG_TMPFS_POSIX_ACL=y
 CONFIG_SQUASHFS=m
 CONFIG_OMFS_FS=m
-CONFIG_EXOFS_FS=m
 CONFIG_NFS_FS=y
 CONFIG_SECURITYFS=y
 CONFIG_CRYPTO_CRYPTD=m
@@ -321,12 +307,8 @@ CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_VMAC=m
 CONFIG_CRYPTO_MD4=m
-CONFIG_CRYPTO_RMD128=m
 CONFIG_CRYPTO_RMD160=m
-CONFIG_CRYPTO_RMD256=m
-CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA512=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
@@ -335,7 +317,6 @@ CONFIG_CRYPTO_CAST5=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
-CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/mips/configs/ip28_defconfig b/arch/mips/configs/ip28_defconfig
index 0921ef38e9fb..ba13eea0509f 100644
--- a/arch/mips/configs/ip28_defconfig
+++ b/arch/mips/configs/ip28_defconfig
@@ -29,9 +29,6 @@ CONFIG_IP_MULTICAST=y
 CONFIG_IP_PNP=y
 CONFIG_IP_PNP_DHCP=y
 CONFIG_IP_PNP_BOOTP=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 CONFIG_TCP_MD5SIG=y
 # CONFIG_IPV6 is not set
 CONFIG_SCSI=y
diff --git a/arch/mips/configs/ip32_defconfig b/arch/mips/configs/ip32_defconfig
index 74020aa3440b..8ced2224c328 100644
--- a/arch/mips/configs/ip32_defconfig
+++ b/arch/mips/configs/ip32_defconfig
@@ -43,7 +43,6 @@ CONFIG_IPV6_TUNNEL=m
 CONFIG_NETWORK_SECMARK=y
 CONFIG_CONNECTOR=y
 CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_RAID_ATTRS=y
 CONFIG_SCSI=y
@@ -165,7 +164,6 @@ CONFIG_CRYPTO_MICHAEL_MIC=y
 CONFIG_CRYPTO_SHA1=y
 CONFIG_CRYPTO_SHA256=y
 CONFIG_CRYPTO_SHA512=y
-CONFIG_CRYPTO_TGR192=y
 CONFIG_CRYPTO_WP512=y
 CONFIG_CRYPTO_ANUBIS=y
 CONFIG_CRYPTO_ARC4=y
diff --git a/arch/mips/configs/jazz_defconfig b/arch/mips/configs/jazz_defconfig
index 843f360da5f2..106b21cb677f 100644
--- a/arch/mips/configs/jazz_defconfig
+++ b/arch/mips/configs/jazz_defconfig
@@ -32,7 +32,6 @@ CONFIG_PARPORT_1284=y
 CONFIG_DEVTMPFS=y
 CONFIG_BLK_DEV_FD=m
 CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=m
 CONFIG_CDROM_PKTCDVD=m
diff --git a/arch/mips/configs/lemote2f_defconfig b/arch/mips/configs/lemote2f_defconfig
index 791894c4d8fb..7e598d338979 100644
--- a/arch/mips/configs/lemote2f_defconfig
+++ b/arch/mips/configs/lemote2f_defconfig
@@ -43,9 +43,6 @@ CONFIG_IP_MROUTE=y
 CONFIG_IP_PIMSM_V1=y
 CONFIG_IP_PIMSM_V2=y
 CONFIG_SYN_COOKIES=y
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
-CONFIG_INET_XFRM_MODE_BEET=m
 CONFIG_TCP_CONG_ADVANCED=y
 CONFIG_TCP_CONG_BIC=y
 CONFIG_DEFAULT_BIC=y
@@ -77,7 +74,6 @@ CONFIG_MAC80211_LEDS=y
 CONFIG_RFKILL=m
 CONFIG_RFKILL_INPUT=y
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=8192
 CONFIG_BLK_DEV_SD=y
@@ -312,12 +308,8 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_XTS=m
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_RMD128=m
 CONFIG_CRYPTO_RMD160=m
-CONFIG_CRYPTO_RMD256=m
-CONFIG_CRYPTO_RMD320=m
 CONFIG_CRYPTO_SHA1=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
@@ -326,7 +318,6 @@ CONFIG_CRYPTO_CAST5=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_FCRYPT=m
 CONFIG_CRYPTO_KHAZAD=m
-CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SEED=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
diff --git a/arch/mips/configs/loongson1b_defconfig b/arch/mips/configs/loongson1b_defconfig
index 25e70423e17d..68207b31dc20 100644
--- a/arch/mips/configs/loongson1b_defconfig
+++ b/arch/mips/configs/loongson1b_defconfig
@@ -28,9 +28,6 @@ CONFIG_INET=y
 CONFIG_IP_PNP=y
 CONFIG_IP_PNP_DHCP=y
 CONFIG_SYN_COOKIES=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_INET_DIAG is not set
 # CONFIG_IPV6 is not set
 # CONFIG_WIRELESS is not set
@@ -115,7 +112,6 @@ CONFIG_NLS_ISO8859_1=m
 # CONFIG_CRYPTO_ECHAINIV is not set
 # CONFIG_CRYPTO_HW is not set
 CONFIG_DYNAMIC_DEBUG=y
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_DEBUG_FS=y
 CONFIG_MAGIC_SYSRQ=y
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/mips/configs/loongson1c_defconfig b/arch/mips/configs/loongson1c_defconfig
index 3a158d4d2fab..c3910a9dee9e 100644
--- a/arch/mips/configs/loongson1c_defconfig
+++ b/arch/mips/configs/loongson1c_defconfig
@@ -29,9 +29,6 @@ CONFIG_INET=y
 CONFIG_IP_PNP=y
 CONFIG_IP_PNP_DHCP=y
 CONFIG_SYN_COOKIES=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_INET_DIAG is not set
 # CONFIG_IPV6 is not set
 # CONFIG_WIRELESS is not set
@@ -116,7 +113,6 @@ CONFIG_NLS_ISO8859_1=m
 # CONFIG_CRYPTO_ECHAINIV is not set
 # CONFIG_CRYPTO_HW is not set
 CONFIG_DYNAMIC_DEBUG=y
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_DEBUG_FS=y
 CONFIG_MAGIC_SYSRQ=y
 # CONFIG_SCHED_DEBUG is not set
diff --git a/arch/mips/configs/loongson2k_defconfig b/arch/mips/configs/loongson2k_defconfig
index e948ca487e2d..728bef666f7a 100644
--- a/arch/mips/configs/loongson2k_defconfig
+++ b/arch/mips/configs/loongson2k_defconfig
@@ -95,7 +95,6 @@ CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 CONFIG_MTD=m
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=8192
 CONFIG_RAID_ATTRS=m
@@ -229,7 +228,6 @@ CONFIG_SERIAL_8250_RSA=y
 CONFIG_SERIAL_OF_PLATFORM=y
 CONFIG_SERIAL_NONSTANDARD=y
 CONFIG_HW_RANDOM=y
-CONFIG_RAW_DRIVER=m
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_PIIX4=y
 CONFIG_GPIO_LOONGSON=y
@@ -336,7 +334,6 @@ CONFIG_DEFAULT_SECURITY_DAC=y
 CONFIG_CRYPTO_SEQIV=m
 CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_MD5=y
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_BLOWFISH=m
 CONFIG_CRYPTO_CAST5=m
diff --git a/arch/mips/configs/loongson3_defconfig b/arch/mips/configs/loongson3_defconfig
index 25ecd15bc952..aca66a5f330d 100644
--- a/arch/mips/configs/loongson3_defconfig
+++ b/arch/mips/configs/loongson3_defconfig
@@ -143,7 +143,6 @@ CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 CONFIG_MTD=m
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=y
 CONFIG_BLK_DEV_RAM=y
 CONFIG_BLK_DEV_RAM_SIZE=8192
 CONFIG_VIRTIO_BLK=y
@@ -268,7 +267,6 @@ CONFIG_SERIAL_OF_PLATFORM=y
 CONFIG_SERIAL_NONSTANDARD=y
 CONFIG_VIRTIO_CONSOLE=y
 CONFIG_HW_RANDOM=y
-CONFIG_RAW_DRIVER=m
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_PIIX4=y
 CONFIG_GPIO_LOONGSON=y
diff --git a/arch/mips/configs/malta_defconfig b/arch/mips/configs/malta_defconfig
index 7a5bdd236a2a..265d38dffbf6 100644
--- a/arch/mips/configs/malta_defconfig
+++ b/arch/mips/configs/malta_defconfig
@@ -42,8 +42,6 @@ CONFIG_SYN_COOKIES=y
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
 CONFIG_INET_IPCOMP=m
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
 CONFIG_TCP_MD5SIG=y
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_ROUTE_INFO=y
@@ -229,7 +227,6 @@ CONFIG_MTD_UBI=m
 CONFIG_MTD_UBI_GLUEBI=m
 CONFIG_BLK_DEV_FD=m
 CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_CDROM_PKTCDVD=m
@@ -237,7 +234,6 @@ CONFIG_ATA_OVER_ETH=m
 CONFIG_RAID_ATTRS=m
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
-CONFIG_CHR_DEV_OSST=m
 CONFIG_BLK_DEV_SR=y
 CONFIG_CHR_DEV_SG=m
 CONFIG_SCSI_CONSTANTS=y
@@ -408,7 +404,6 @@ CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_SHA512=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/mips/configs/malta_kvm_defconfig b/arch/mips/configs/malta_kvm_defconfig
index b5ba08d7ab57..1d2b248c7cd3 100644
--- a/arch/mips/configs/malta_kvm_defconfig
+++ b/arch/mips/configs/malta_kvm_defconfig
@@ -46,8 +46,6 @@ CONFIG_SYN_COOKIES=y
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
 CONFIG_INET_IPCOMP=m
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
 CONFIG_TCP_MD5SIG=y
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_ROUTE_INFO=y
@@ -233,7 +231,6 @@ CONFIG_MTD_UBI=m
 CONFIG_MTD_UBI_GLUEBI=m
 CONFIG_BLK_DEV_FD=m
 CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_CDROM_PKTCDVD=m
@@ -241,7 +238,6 @@ CONFIG_ATA_OVER_ETH=m
 CONFIG_RAID_ATTRS=m
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
-CONFIG_CHR_DEV_OSST=m
 CONFIG_BLK_DEV_SR=y
 CONFIG_CHR_DEV_SG=m
 CONFIG_SCSI_CONSTANTS=y
@@ -415,7 +411,6 @@ CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_SHA512=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/mips/configs/malta_qemu_32r6_defconfig b/arch/mips/configs/malta_qemu_32r6_defconfig
index 6fb9bc29f4a0..fd63a2b152f6 100644
--- a/arch/mips/configs/malta_qemu_32r6_defconfig
+++ b/arch/mips/configs/malta_qemu_32r6_defconfig
@@ -76,7 +76,6 @@ CONFIG_NET_ACT_POLICE=y
 # CONFIG_WIRELESS is not set
 CONFIG_DEVTMPFS=y
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_SG=y
 # CONFIG_SCSI_LOWLEVEL is not set
@@ -98,7 +97,6 @@ CONFIG_PCNET32=y
 # CONFIG_NET_VENDOR_DEC is not set
 # CONFIG_NET_VENDOR_DLINK is not set
 # CONFIG_NET_VENDOR_EMULEX is not set
-# CONFIG_NET_VENDOR_HP is not set
 # CONFIG_NET_VENDOR_INTEL is not set
 # CONFIG_NET_VENDOR_MARVELL is not set
 # CONFIG_NET_VENDOR_MELLANOX is not set
@@ -172,7 +170,6 @@ CONFIG_NLS_ISO8859_1=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/mips/configs/maltaaprp_defconfig b/arch/mips/configs/maltaaprp_defconfig
index eb72df528243..1f07e354c954 100644
--- a/arch/mips/configs/maltaaprp_defconfig
+++ b/arch/mips/configs/maltaaprp_defconfig
@@ -78,7 +78,6 @@ CONFIG_NET_ACT_POLICE=y
 # CONFIG_WIRELESS is not set
 CONFIG_DEVTMPFS=y
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_SG=y
 # CONFIG_SCSI_LOWLEVEL is not set
@@ -100,7 +99,6 @@ CONFIG_PCNET32=y
 # CONFIG_NET_VENDOR_DEC is not set
 # CONFIG_NET_VENDOR_DLINK is not set
 # CONFIG_NET_VENDOR_EMULEX is not set
-# CONFIG_NET_VENDOR_HP is not set
 # CONFIG_NET_VENDOR_INTEL is not set
 # CONFIG_NET_VENDOR_MARVELL is not set
 # CONFIG_NET_VENDOR_MELLANOX is not set
@@ -173,7 +171,6 @@ CONFIG_NLS_ISO8859_1=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/mips/configs/maltasmvp_defconfig b/arch/mips/configs/maltasmvp_defconfig
index 1fb40d310f49..5cd3eca236de 100644
--- a/arch/mips/configs/maltasmvp_defconfig
+++ b/arch/mips/configs/maltasmvp_defconfig
@@ -79,7 +79,6 @@ CONFIG_NET_ACT_POLICE=y
 # CONFIG_WIRELESS is not set
 CONFIG_DEVTMPFS=y
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_SG=y
 # CONFIG_SCSI_LOWLEVEL is not set
@@ -99,7 +98,6 @@ CONFIG_PCNET32=y
 # CONFIG_NET_VENDOR_DEC is not set
 # CONFIG_NET_VENDOR_DLINK is not set
 # CONFIG_NET_VENDOR_EMULEX is not set
-# CONFIG_NET_VENDOR_HP is not set
 # CONFIG_NET_VENDOR_INTEL is not set
 # CONFIG_NET_VENDOR_MARVELL is not set
 # CONFIG_NET_VENDOR_MELLANOX is not set
@@ -174,7 +172,6 @@ CONFIG_NLS_ISO8859_1=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/mips/configs/maltasmvp_eva_defconfig b/arch/mips/configs/maltasmvp_eva_defconfig
index 75cb778c6149..45688e742a15 100644
--- a/arch/mips/configs/maltasmvp_eva_defconfig
+++ b/arch/mips/configs/maltasmvp_eva_defconfig
@@ -80,7 +80,6 @@ CONFIG_NET_ACT_POLICE=y
 # CONFIG_WIRELESS is not set
 CONFIG_DEVTMPFS=y
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_SG=y
 # CONFIG_SCSI_LOWLEVEL is not set
@@ -102,7 +101,6 @@ CONFIG_PCNET32=y
 # CONFIG_NET_VENDOR_DEC is not set
 # CONFIG_NET_VENDOR_DLINK is not set
 # CONFIG_NET_VENDOR_EMULEX is not set
-# CONFIG_NET_VENDOR_HP is not set
 # CONFIG_NET_VENDOR_INTEL is not set
 # CONFIG_NET_VENDOR_MARVELL is not set
 # CONFIG_NET_VENDOR_MELLANOX is not set
@@ -176,7 +174,6 @@ CONFIG_NLS_ISO8859_1=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/mips/configs/maltaup_defconfig b/arch/mips/configs/maltaup_defconfig
index 7b4f247dc60c..136f965784db 100644
--- a/arch/mips/configs/maltaup_defconfig
+++ b/arch/mips/configs/maltaup_defconfig
@@ -77,7 +77,6 @@ CONFIG_NET_ACT_POLICE=y
 # CONFIG_WIRELESS is not set
 CONFIG_DEVTMPFS=y
 CONFIG_BLK_DEV_LOOP=y
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_SG=y
 # CONFIG_SCSI_LOWLEVEL is not set
@@ -99,7 +98,6 @@ CONFIG_PCNET32=y
 # CONFIG_NET_VENDOR_DEC is not set
 # CONFIG_NET_VENDOR_DLINK is not set
 # CONFIG_NET_VENDOR_EMULEX is not set
-# CONFIG_NET_VENDOR_HP is not set
 # CONFIG_NET_VENDOR_INTEL is not set
 # CONFIG_NET_VENDOR_MARVELL is not set
 # CONFIG_NET_VENDOR_MELLANOX is not set
@@ -172,7 +170,6 @@ CONFIG_NLS_ISO8859_1=m
 CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/mips/configs/maltaup_xpa_defconfig b/arch/mips/configs/maltaup_xpa_defconfig
index 8d58653f1b4e..75b8da8d9927 100644
--- a/arch/mips/configs/maltaup_xpa_defconfig
+++ b/arch/mips/configs/maltaup_xpa_defconfig
@@ -45,8 +45,6 @@ CONFIG_SYN_COOKIES=y
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
 CONFIG_INET_IPCOMP=m
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
 CONFIG_TCP_MD5SIG=y
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_ROUTE_INFO=y
@@ -231,7 +229,6 @@ CONFIG_MTD_UBI=m
 CONFIG_MTD_UBI_GLUEBI=m
 CONFIG_BLK_DEV_FD=m
 CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_NBD=m
 CONFIG_BLK_DEV_RAM=y
 CONFIG_CDROM_PKTCDVD=m
@@ -239,7 +236,6 @@ CONFIG_ATA_OVER_ETH=m
 CONFIG_RAID_ATTRS=m
 CONFIG_BLK_DEV_SD=y
 CONFIG_CHR_DEV_ST=m
-CONFIG_CHR_DEV_OSST=m
 CONFIG_BLK_DEV_SR=y
 CONFIG_CHR_DEV_SG=m
 CONFIG_SCSI_CONSTANTS=y
@@ -414,7 +410,6 @@ CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_SHA512=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/mips/configs/mtx1_defconfig b/arch/mips/configs/mtx1_defconfig
index 4194e79b435c..edf9634aa8ee 100644
--- a/arch/mips/configs/mtx1_defconfig
+++ b/arch/mips/configs/mtx1_defconfig
@@ -58,13 +58,9 @@ CONFIG_SYN_COOKIES=y
 CONFIG_INET_AH=m
 CONFIG_INET_ESP=m
 CONFIG_INET_IPCOMP=m
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
-CONFIG_INET_XFRM_MODE_BEET=m
 CONFIG_INET6_AH=m
 CONFIG_INET6_ESP=m
 CONFIG_INET6_IPCOMP=m
-CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m
 CONFIG_IPV6_TUNNEL=m
 CONFIG_NETWORK_SECMARK=y
 CONFIG_NETFILTER=y
@@ -116,7 +112,6 @@ CONFIG_IP6_NF_FILTER=m
 CONFIG_IP6_NF_TARGET_REJECT=m
 CONFIG_IP6_NF_MANGLE=m
 CONFIG_IP6_NF_RAW=m
-CONFIG_DECNET_NF_GRABULATOR=m
 CONFIG_BRIDGE_NF_EBTABLES=m
 CONFIG_BRIDGE_EBT_BROUTE=m
 CONFIG_BRIDGE_EBT_T_FILTER=m
@@ -146,7 +141,6 @@ CONFIG_ATM_MPOA=m
 CONFIG_ATM_BR2684=m
 CONFIG_BRIDGE=m
 CONFIG_VLAN_8021Q=m
-CONFIG_DECNET=m
 CONFIG_LLC2=m
 CONFIG_ATALK=m
 CONFIG_DEV_APPLETALK=m
@@ -284,7 +278,6 @@ CONFIG_PCMCIA_XIRCOM=m
 CONFIG_DL2K=m
 CONFIG_SUNDANCE=m
 CONFIG_PCMCIA_FMVJ18X=m
-CONFIG_HP100=m
 CONFIG_E100=m
 CONFIG_E1000=m
 CONFIG_IXGB=m
@@ -368,9 +361,6 @@ CONFIG_HDLC_X25=m
 CONFIG_PCI200SYN=m
 CONFIG_WANXL=m
 CONFIG_FARSYNC=m
-CONFIG_DSCC4=m
-CONFIG_DSCC4_PCISYNC=y
-CONFIG_DSCC4_PCI_RST=y
 CONFIG_LAPBETHER=m
 # CONFIG_KEYBOARD_ATKBD is not set
 CONFIG_KEYBOARD_GPIO=y
@@ -683,7 +673,6 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_MD5=y
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
@@ -692,5 +681,4 @@ CONFIG_CRYPTO_KHAZAD=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TEA=m
 CONFIG_CRYPTO_TWOFISH=m
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_MAGIC_SYSRQ=y
diff --git a/arch/mips/configs/omega2p_defconfig b/arch/mips/configs/omega2p_defconfig
index fc39ddf610a9..9c34daf83563 100644
--- a/arch/mips/configs/omega2p_defconfig
+++ b/arch/mips/configs/omega2p_defconfig
@@ -35,9 +35,6 @@ CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
 CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_INET_DIAG is not set
 # CONFIG_IPV6 is not set
 # CONFIG_WIRELESS is not set
diff --git a/arch/mips/configs/pic32mzda_defconfig b/arch/mips/configs/pic32mzda_defconfig
index fd567247adc7..48dd02d01ac1 100644
--- a/arch/mips/configs/pic32mzda_defconfig
+++ b/arch/mips/configs/pic32mzda_defconfig
@@ -45,7 +45,6 @@ CONFIG_KEYBOARD_GPIO_POLLED=m
 CONFIG_SERIAL_PIC32=y
 CONFIG_SERIAL_PIC32_CONSOLE=y
 CONFIG_HW_RANDOM=y
-CONFIG_RAW_DRIVER=m
 CONFIG_GPIO_SYSFS=y
 # CONFIG_HWMON is not set
 CONFIG_HIDRAW=y
diff --git a/arch/mips/configs/rb532_defconfig b/arch/mips/configs/rb532_defconfig
index 252d472387aa..93306f5e045b 100644
--- a/arch/mips/configs/rb532_defconfig
+++ b/arch/mips/configs/rb532_defconfig
@@ -33,9 +33,6 @@ CONFIG_IP_MULTIPLE_TABLES=y
 CONFIG_IP_ROUTE_MULTIPATH=y
 CONFIG_IP_ROUTE_VERBOSE=y
 CONFIG_SYN_COOKIES=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 CONFIG_INET_DIAG=m
 CONFIG_TCP_CONG_ADVANCED=y
 CONFIG_TCP_CONG_CUBIC=m
@@ -162,5 +159,4 @@ CONFIG_SQUASHFS=y
 CONFIG_CRYPTO_TEST=m
 # CONFIG_CRYPTO_HW is not set
 CONFIG_CRC16=m
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_STRIP_ASM_SYMS=y
diff --git a/arch/mips/configs/rbtx49xx_defconfig b/arch/mips/configs/rbtx49xx_defconfig
index f8212a813be7..30c195f28278 100644
--- a/arch/mips/configs/rbtx49xx_defconfig
+++ b/arch/mips/configs/rbtx49xx_defconfig
@@ -21,9 +21,6 @@ CONFIG_UNIX=y
 CONFIG_INET=y
 CONFIG_IP_MULTICAST=y
 CONFIG_IP_PNP=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_IPV6 is not set
 # CONFIG_WIRELESS is not set
 CONFIG_MTD=y
@@ -51,7 +48,6 @@ CONFIG_TC35815=y
 CONFIG_SERIAL_TXX9_CONSOLE=y
 CONFIG_SERIAL_TXX9_STDSERIAL=y
 CONFIG_SPI=y
-CONFIG_SPI_TXX9=y
 # CONFIG_HWMON is not set
 CONFIG_WATCHDOG=y
 CONFIG_TXX9_WDT=m
@@ -65,8 +61,6 @@ CONFIG_SND=m
 # CONFIG_SND_SPI is not set
 # CONFIG_SND_MIPS is not set
 CONFIG_SND_SOC=m
-CONFIG_SND_SOC_TXX9ACLC=m
-CONFIG_SND_SOC_TXX9ACLC_GENERIC=m
 # CONFIG_USB_SUPPORT is not set
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
@@ -78,7 +72,6 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_INTF_DEV_UIE_EMUL=y
 CONFIG_RTC_DRV_RS5C348=y
 CONFIG_RTC_DRV_DS1742=y
-CONFIG_RTC_DRV_TX4939=y
 CONFIG_DMADEVICES=y
 CONFIG_TXX9_DMAC=m
 # CONFIG_DNOTIFY is not set
diff --git a/arch/mips/configs/rm200_defconfig b/arch/mips/configs/rm200_defconfig
index 7d6f235e8ccb..9932a593e3c3 100644
--- a/arch/mips/configs/rm200_defconfig
+++ b/arch/mips/configs/rm200_defconfig
@@ -29,9 +29,6 @@ CONFIG_NET_IPIP=m
 CONFIG_IP_MROUTE=y
 CONFIG_IP_PIMSM_V1=y
 CONFIG_IP_PIMSM_V2=y
-CONFIG_INET_XFRM_MODE_TRANSPORT=m
-CONFIG_INET_XFRM_MODE_TUNNEL=m
-CONFIG_INET_XFRM_MODE_BEET=m
 CONFIG_TCP_MD5SIG=y
 CONFIG_IPV6_ROUTER_PREF=y
 CONFIG_IPV6_ROUTE_INFO=y
@@ -39,7 +36,6 @@ CONFIG_INET6_AH=m
 CONFIG_INET6_ESP=m
 CONFIG_INET6_IPCOMP=m
 CONFIG_IPV6_MIP6=m
-CONFIG_INET6_XFRM_MODE_ROUTEOPTIMIZATION=m
 CONFIG_IPV6_TUNNEL=m
 CONFIG_IPV6_MULTIPLE_TABLES=y
 CONFIG_IPV6_SUBTREES=y
@@ -116,7 +112,6 @@ CONFIG_IP6_NF_FILTER=m
 CONFIG_IP6_NF_TARGET_REJECT=m
 CONFIG_IP6_NF_MANGLE=m
 CONFIG_IP6_NF_RAW=m
-CONFIG_DECNET_NF_GRABULATOR=m
 CONFIG_BRIDGE_NF_EBTABLES=m
 CONFIG_BRIDGE_EBT_BROUTE=m
 CONFIG_BRIDGE_EBT_T_FILTER=m
@@ -137,7 +132,6 @@ CONFIG_BRIDGE_EBT_REDIRECT=m
 CONFIG_BRIDGE_EBT_SNAT=m
 CONFIG_BRIDGE_EBT_LOG=m
 CONFIG_BRIDGE=m
-CONFIG_DECNET=m
 CONFIG_NET_SCHED=y
 CONFIG_NET_SCH_CBQ=m
 CONFIG_NET_SCH_HTB=m
@@ -192,9 +186,7 @@ CONFIG_PARIDE_KTTI=m
 CONFIG_PARIDE_ON20=m
 CONFIG_PARIDE_ON26=m
 CONFIG_BLK_DEV_LOOP=m
-CONFIG_BLK_DEV_CRYPTOLOOP=m
 CONFIG_BLK_DEV_NBD=m
-CONFIG_BLK_DEV_SX8=m
 CONFIG_BLK_DEV_RAM=m
 CONFIG_CDROM_PKTCDVD=m
 CONFIG_ATA_OVER_ETH=m
@@ -400,7 +392,6 @@ CONFIG_CRYPTO_PCBC=m
 CONFIG_CRYPTO_HMAC=y
 CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_ANUBIS=m
 CONFIG_CRYPTO_BLOWFISH=m
diff --git a/arch/mips/configs/rt305x_defconfig b/arch/mips/configs/rt305x_defconfig
index eb359db15dba..bf017d493002 100644
--- a/arch/mips/configs/rt305x_defconfig
+++ b/arch/mips/configs/rt305x_defconfig
@@ -35,9 +35,6 @@ CONFIG_IP_ROUTE_VERBOSE=y
 CONFIG_IP_MROUTE=y
 CONFIG_IP_MROUTE_MULTIPLE_TABLES=y
 CONFIG_SYN_COOKIES=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_INET_DIAG is not set
 CONFIG_TCP_CONG_ADVANCED=y
 # CONFIG_TCP_CONG_BIC is not set
@@ -140,7 +137,6 @@ CONFIG_CRC32_SARWATE=y
 # CONFIG_XZ_DEC_ARMTHUMB is not set
 # CONFIG_XZ_DEC_SPARC is not set
 CONFIG_PRINTK_TIME=y
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_STRIP_ASM_SYMS=y
 CONFIG_DEBUG_FS=y
 CONFIG_MAGIC_SYSRQ=y
diff --git a/arch/mips/configs/sb1250_swarm_defconfig b/arch/mips/configs/sb1250_swarm_defconfig
index de94bf756a93..030186f89501 100644
--- a/arch/mips/configs/sb1250_swarm_defconfig
+++ b/arch/mips/configs/sb1250_swarm_defconfig
@@ -88,7 +88,6 @@ CONFIG_CRYPTO_XCBC=m
 CONFIG_CRYPTO_MD4=m
 CONFIG_CRYPTO_MICHAEL_MIC=m
 CONFIG_CRYPTO_SHA512=m
-CONFIG_CRYPTO_TGR192=m
 CONFIG_CRYPTO_WP512=m
 CONFIG_CRYPTO_BLOWFISH=m
 CONFIG_CRYPTO_CAMELLIA=m
@@ -96,7 +95,6 @@ CONFIG_CRYPTO_CAST5=m
 CONFIG_CRYPTO_CAST6=m
 CONFIG_CRYPTO_DES=m
 CONFIG_CRYPTO_FCRYPT=m
-CONFIG_CRYPTO_SALSA20=m
 CONFIG_CRYPTO_SERPENT=m
 CONFIG_CRYPTO_TWOFISH=m
 CONFIG_CRYPTO_DEFLATE=m
diff --git a/arch/mips/configs/vocore2_defconfig b/arch/mips/configs/vocore2_defconfig
index a14f8ea5c386..0722a3bf03c0 100644
--- a/arch/mips/configs/vocore2_defconfig
+++ b/arch/mips/configs/vocore2_defconfig
@@ -35,9 +35,6 @@ CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
 CONFIG_INET=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_INET_DIAG is not set
 # CONFIG_IPV6 is not set
 # CONFIG_WIRELESS is not set
diff --git a/arch/mips/configs/xway_defconfig b/arch/mips/configs/xway_defconfig
index eeb689f715cb..eb5acf1f24ae 100644
--- a/arch/mips/configs/xway_defconfig
+++ b/arch/mips/configs/xway_defconfig
@@ -37,9 +37,6 @@ CONFIG_IP_ROUTE_VERBOSE=y
 CONFIG_IP_MROUTE=y
 CONFIG_IP_MROUTE_MULTIPLE_TABLES=y
 CONFIG_SYN_COOKIES=y
-# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
-# CONFIG_INET_XFRM_MODE_TUNNEL is not set
-# CONFIG_INET_XFRM_MODE_BEET is not set
 # CONFIG_INET_DIAG is not set
 CONFIG_TCP_CONG_ADVANCED=y
 # CONFIG_TCP_CONG_BIC is not set
@@ -146,7 +143,6 @@ CONFIG_CRYPTO_ARC4=m
 CONFIG_CRC_ITU_T=m
 CONFIG_CRC32_SARWATE=y
 CONFIG_PRINTK_TIME=y
-# CONFIG_ENABLE_MUST_CHECK is not set
 CONFIG_STRIP_ASM_SYMS=y
 CONFIG_DEBUG_FS=y
 CONFIG_MAGIC_SYSRQ=y
diff --git a/arch/mips/include/asm/irq.h b/arch/mips/include/asm/irq.h
index 57561e0e6e8d..44f9824c1d8c 100644
--- a/arch/mips/include/asm/irq.h
+++ b/arch/mips/include/asm/irq.h
@@ -63,10 +63,6 @@ extern void do_domain_IRQ(struct irq_domain *domain, unsigned int irq);
 extern void arch_init_irq(void);
 extern void spurious_interrupt(void);
 
-extern int allocate_irqno(void);
-extern void alloc_legacy_irqno(void);
-extern void free_irqno(unsigned int irq);
-
 /*
  * Before R2 the timer and performance counter interrupts were both fixed to
  * IE7.	 Since R2 their number has to be read from the c0_intctl register.
diff --git a/arch/mips/include/asm/mach-ar7/ar7.h b/arch/mips/include/asm/mach-ar7/ar7.h
index cbe75ade3277..1e8621a6afa3 100644
--- a/arch/mips/include/asm/mach-ar7/ar7.h
+++ b/arch/mips/include/asm/mach-ar7/ar7.h
@@ -104,8 +104,6 @@ struct plat_dsl_data {
 	int reset_bit_sar;
 };
 
-extern int ar7_cpu_clock, ar7_bus_clock, ar7_dsp_clock;
-
 static inline int ar7_is_titan(void)
 {
 	return (readl((void *)KSEG1ADDR(AR7_REGS_GPIO + 0x24)) & 0xffff) ==
diff --git a/arch/mips/include/asm/octeon/cvmx-fpa.h b/arch/mips/include/asm/octeon/cvmx-fpa.h
index 29ae63606ab4..f6dfcca97f19 100644
--- a/arch/mips/include/asm/octeon/cvmx-fpa.h
+++ b/arch/mips/include/asm/octeon/cvmx-fpa.h
@@ -264,26 +264,6 @@ static inline void cvmx_fpa_free(void *ptr, uint64_t pool,
 }
 
 /**
- * Setup a FPA pool to control a new block of memory.
- * This can only be called once per pool. Make sure proper
- * locking enforces this.
- *
- * @pool:	Pool to initialize
- *		     0 <= pool < 8
- * @name:	Constant character string to name this pool.
- *		     String is not copied.
- * @buffer:	Pointer to the block of memory to use. This must be
- *		     accessible by all processors and external hardware.
- * @block_size: Size for each block controlled by the FPA
- * @num_blocks: Number of blocks
- *
- * Returns 0 on Success,
- *	   -1 on failure
- */
-extern int cvmx_fpa_setup_pool(uint64_t pool, const char *name, void *buffer,
-			       uint64_t block_size, uint64_t num_blocks);
-
-/**
  * Shutdown a Memory pool and validate that it had all of
  * the buffers originally placed in it. This should only be
  * called by one processor after all hardware has finished
diff --git a/arch/mips/include/asm/octeon/octeon.h b/arch/mips/include/asm/octeon/octeon.h
index 7e714aefc76d..5c1d726c702f 100644
--- a/arch/mips/include/asm/octeon/octeon.h
+++ b/arch/mips/include/asm/octeon/octeon.h
@@ -43,7 +43,6 @@ extern int octeon_get_southbridge_interrupt(void);
 extern int octeon_get_boot_coremask(void);
 extern int octeon_get_boot_num_arguments(void);
 extern const char *octeon_get_boot_argument(int arg);
-extern void octeon_hal_setup_reserved32(void);
 extern void octeon_user_io_init(void);
 
 extern void octeon_init_cvmcount(void);
diff --git a/arch/mips/include/asm/octeon/pci-octeon.h b/arch/mips/include/asm/octeon/pci-octeon.h
index b12d9a3fbfb6..2f46f6c6e3d0 100644
--- a/arch/mips/include/asm/octeon/pci-octeon.h
+++ b/arch/mips/include/asm/octeon/pci-octeon.h
@@ -64,6 +64,4 @@ enum octeon_dma_bar_type {
 extern enum octeon_dma_bar_type octeon_dma_bar_type;
 
 void octeon_pci_dma_init(void);
-extern char *octeon_swiotlb;
-
 #endif
diff --git a/arch/mips/include/asm/sibyte/sb1250.h b/arch/mips/include/asm/sibyte/sb1250.h
index dbde5f93f0dd..495b31925ed7 100644
--- a/arch/mips/include/asm/sibyte/sb1250.h
+++ b/arch/mips/include/asm/sibyte/sb1250.h
@@ -32,7 +32,6 @@ extern unsigned int soc_type;
 extern unsigned int periph_rev;
 extern unsigned int zbbus_mhz;
 
-extern void sb1250_time_init(void);
 extern void sb1250_mask_irq(int cpu, int irq);
 extern void sb1250_unmask_irq(int cpu, int irq);
 
diff --git a/arch/mips/include/asm/sni.h b/arch/mips/include/asm/sni.h
index 7dfa297ce597..7fb6656a6bfd 100644
--- a/arch/mips/include/asm/sni.h
+++ b/arch/mips/include/asm/sni.h
@@ -226,9 +226,6 @@ extern void sni_pcit_cplus_irq_init(void);
 extern void sni_rm200_irq_init(void);
 extern void sni_pcimt_irq_init(void);
 
-/* timer inits */
-extern void sni_cpu_time_init(void);
-
 /* eisa init for RM200/400 */
 #ifdef CONFIG_EISA
 extern int sni_eisa_root_init(void);
diff --git a/arch/mips/kernel/prom.c b/arch/mips/kernel/prom.c
index 7db6ff9aed7d..f88ce78e13e3 100644
--- a/arch/mips/kernel/prom.c
+++ b/arch/mips/kernel/prom.c
@@ -26,7 +26,7 @@ __init void mips_set_machine_name(const char *name)
 	if (name == NULL)
 		return;
 
-	strlcpy(mips_machine_name, name, sizeof(mips_machine_name));
+	strscpy(mips_machine_name, name, sizeof(mips_machine_name));
 	pr_info("MIPS: machine is %s\n", mips_get_machine_name());
 }
 
@@ -52,9 +52,9 @@ int __init __dt_register_buses(const char *bus0, const char *bus1)
 	if (!of_have_populated_dt())
 		panic("device tree not present");
 
-	strlcpy(of_ids[0].compatible, bus0, sizeof(of_ids[0].compatible));
+	strscpy(of_ids[0].compatible, bus0, sizeof(of_ids[0].compatible));
 	if (bus1) {
-		strlcpy(of_ids[1].compatible, bus1,
+		strscpy(of_ids[1].compatible, bus1,
 			sizeof(of_ids[1].compatible));
 	}
 
diff --git a/arch/mips/kernel/relocate.c b/arch/mips/kernel/relocate.c
index 56b51de2dc51..58fc8d089402 100644
--- a/arch/mips/kernel/relocate.c
+++ b/arch/mips/kernel/relocate.c
@@ -340,7 +340,7 @@ void *__init relocate_kernel(void)
 	early_init_dt_scan(fdt);
 	if (boot_command_line[0]) {
 		/* Boot command line was passed in device tree */
-		strlcpy(arcs_cmdline, boot_command_line, COMMAND_LINE_SIZE);
+		strscpy(arcs_cmdline, boot_command_line, COMMAND_LINE_SIZE);
 	}
 #endif /* CONFIG_USE_OF */
 
diff --git a/arch/mips/kernel/segment.c b/arch/mips/kernel/segment.c
index 0a9bd7b0983b..24560501c70d 100644
--- a/arch/mips/kernel/segment.c
+++ b/arch/mips/kernel/segment.c
@@ -46,7 +46,7 @@ static void build_segment_config(char *str, unsigned int cfg)
 		((cfg & MIPS_SEGCFG_EU) >> MIPS_SEGCFG_EU_SHIFT));
 }
 
-static int show_segments(struct seq_file *m, void *v)
+static int segments_show(struct seq_file *m, void *v)
 {
 	unsigned int segcfg;
 	char str[42];
@@ -80,18 +80,7 @@ static int show_segments(struct seq_file *m, void *v)
 
 	return 0;
 }
-
-static int segments_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, show_segments, NULL);
-}
-
-static const struct file_operations segments_fops = {
-	.open		= segments_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(segments);
 
 static int __init segments_info(void)
 {
diff --git a/arch/mips/kernel/setup.c b/arch/mips/kernel/setup.c
index 2ca156a5b231..39c79f67c7a3 100644
--- a/arch/mips/kernel/setup.c
+++ b/arch/mips/kernel/setup.c
@@ -42,6 +42,7 @@
 #include <asm/setup.h>
 #include <asm/smp-ops.h>
 #include <asm/prom.h>
+#include <asm/fw/fw.h>
 
 #ifdef CONFIG_MIPS_ELF_APPENDED_DTB
 char __section(".appended_dtb") __appended_dtb[0x100000];
@@ -756,6 +757,24 @@ static void __init prefill_possible_map(void)
 static inline void prefill_possible_map(void) {}
 #endif
 
+static void __init setup_rng_seed(void)
+{
+	char *rng_seed_hex = fw_getenv("rngseed");
+	u8 rng_seed[512];
+	size_t len;
+
+	if (!rng_seed_hex)
+		return;
+
+	len = min(sizeof(rng_seed), strlen(rng_seed_hex) / 2);
+	if (hex2bin(rng_seed, rng_seed_hex, len))
+		return;
+
+	add_bootloader_randomness(rng_seed, len);
+	memzero_explicit(rng_seed, len);
+	memzero_explicit(rng_seed_hex, len * 2);
+}
+
 void __init setup_arch(char **cmdline_p)
 {
 	cpu_probe();
@@ -786,6 +805,8 @@ void __init setup_arch(char **cmdline_p)
 	paging_init();
 
 	memblock_dump_all();
+
+	setup_rng_seed();
 }
 
 unsigned long kernelsp[NR_CPUS];
diff --git a/arch/mips/lantiq/prom.c b/arch/mips/lantiq/prom.c
index c731082a0c42..be4829cc7a3a 100644
--- a/arch/mips/lantiq/prom.c
+++ b/arch/mips/lantiq/prom.c
@@ -34,6 +34,14 @@ unsigned long physical_memsize = 0L;
  */
 static struct ltq_soc_info soc_info;
 
+/*
+ * These structs are used to override vsmp_init_secondary()
+ */
+#if defined(CONFIG_MIPS_MT_SMP)
+extern const struct plat_smp_ops vsmp_smp_ops;
+static struct plat_smp_ops lantiq_smp_ops;
+#endif
+
 const char *get_system_type(void)
 {
 	return soc_info.sys_type;
@@ -84,6 +92,17 @@ void __init plat_mem_setup(void)
 	__dt_setup_arch(dtb);
 }
 
+#if defined(CONFIG_MIPS_MT_SMP)
+static void lantiq_init_secondary(void)
+{
+	/*
+	 * MIPS CPU startup function vsmp_init_secondary() will only
+	 * enable some of the interrupts for the second CPU/VPE.
+	 */
+	set_c0_status(ST0_IM);
+}
+#endif
+
 void __init prom_init(void)
 {
 	/* call the soc specific detetcion code and get it to fill soc_info */
@@ -95,7 +114,10 @@ void __init prom_init(void)
 	prom_init_cmdline();
 
 #if defined(CONFIG_MIPS_MT_SMP)
-	if (register_vsmp_smp_ops())
-		panic("failed to register_vsmp_smp_ops()");
+	if (cpu_has_mipsmt) {
+		lantiq_smp_ops = vsmp_smp_ops;
+		lantiq_smp_ops.init_secondary = lantiq_init_secondary;
+		register_smp_ops(&lantiq_smp_ops);
+	}
 #endif
 }
diff --git a/arch/mips/lantiq/xway/vmmc.c b/arch/mips/lantiq/xway/vmmc.c
index 7a14da8d9d15..2796e87dfcae 100644
--- a/arch/mips/lantiq/xway/vmmc.c
+++ b/arch/mips/lantiq/xway/vmmc.c
@@ -4,9 +4,10 @@
  *  Copyright (C) 2012 John Crispin <john@phrozen.org>
  */
 
+#include <linux/err.h>
 #include <linux/export.h>
+#include <linux/gpio/consumer.h>
 #include <linux/of_platform.h>
-#include <linux/of_gpio.h>
 #include <linux/dma-mapping.h>
 
 #include <lantiq_soc.h>
@@ -25,23 +26,28 @@ EXPORT_SYMBOL(ltq_get_cp1_base);
 static int vmmc_probe(struct platform_device *pdev)
 {
 #define CP1_SIZE       (1 << 20)
+	struct gpio_desc *gpio;
 	int gpio_count;
 	dma_addr_t dma;
+	int error;
 
 	cp1_base =
 		(void *) CPHYSADDR(dma_alloc_coherent(&pdev->dev, CP1_SIZE,
 						    &dma, GFP_KERNEL));
 
-	gpio_count = of_gpio_count(pdev->dev.of_node);
+	gpio_count = gpiod_count(&pdev->dev, NULL);
 	while (gpio_count > 0) {
-		enum of_gpio_flags flags;
-		int gpio = of_get_gpio_flags(pdev->dev.of_node,
-					     --gpio_count, &flags);
-		if (gpio_request(gpio, "vmmc-relay"))
+		gpio = devm_gpiod_get_index(&pdev->dev,
+					    NULL, --gpio_count, GPIOD_OUT_HIGH);
+		error = PTR_ERR_OR_ZERO(gpio);
+		if (error) {
+			dev_err(&pdev->dev,
+				"failed to request GPIO idx %d: %d\n",
+				gpio_count, error);
 			continue;
-		dev_info(&pdev->dev, "requested GPIO %d\n", gpio);
-		gpio_direction_output(gpio,
-				      (flags & OF_GPIO_ACTIVE_LOW) ? (0) : (1));
+		}
+
+		gpiod_set_consumer_name(gpio, "vmmc-relay");
 	}
 
 	dev_info(&pdev->dev, "reserved %dMB at 0x%p", CP1_SIZE >> 20, cp1_base);
diff --git a/arch/mips/lib/bswapdi.c b/arch/mips/lib/bswapdi.c
index fcef74084492..88242dc7de17 100644
--- a/arch/mips/lib/bswapdi.c
+++ b/arch/mips/lib/bswapdi.c
@@ -1,17 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/export.h>
 #include <linux/compiler.h>
+#include <uapi/linux/swab.h>
+
+/* To silence -Wmissing-prototypes. */
+unsigned long long __bswapdi2(unsigned long long u);
 
 unsigned long long notrace __bswapdi2(unsigned long long u)
 {
-	return (((u) & 0xff00000000000000ull) >> 56) |
-	       (((u) & 0x00ff000000000000ull) >> 40) |
-	       (((u) & 0x0000ff0000000000ull) >> 24) |
-	       (((u) & 0x000000ff00000000ull) >>  8) |
-	       (((u) & 0x00000000ff000000ull) <<  8) |
-	       (((u) & 0x0000000000ff0000ull) << 24) |
-	       (((u) & 0x000000000000ff00ull) << 40) |
-	       (((u) & 0x00000000000000ffull) << 56);
+	return ___constant_swab64(u);
 }
-
 EXPORT_SYMBOL(__bswapdi2);
diff --git a/arch/mips/lib/bswapsi.c b/arch/mips/lib/bswapsi.c
index 22d8e4f6d66e..2ed655497de5 100644
--- a/arch/mips/lib/bswapsi.c
+++ b/arch/mips/lib/bswapsi.c
@@ -1,13 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/export.h>
 #include <linux/compiler.h>
+#include <uapi/linux/swab.h>
+
+/* To silence -Wmissing-prototypes. */
+unsigned int __bswapsi2(unsigned int u);
 
 unsigned int notrace __bswapsi2(unsigned int u)
 {
-	return (((u) & 0xff000000) >> 24) |
-	       (((u) & 0x00ff0000) >>  8) |
-	       (((u) & 0x0000ff00) <<  8) |
-	       (((u) & 0x000000ff) << 24);
+	return ___constant_swab32(u);
 }
-
 EXPORT_SYMBOL(__bswapsi2);
diff --git a/arch/mips/loongson2ef/common/pci.c b/arch/mips/loongson2ef/common/pci.c
index 200916925e95..7d9ea51e8c01 100644
--- a/arch/mips/loongson2ef/common/pci.c
+++ b/arch/mips/loongson2ef/common/pci.c
@@ -73,8 +73,6 @@ static void __init setup_pcimap(void)
 #endif
 }
 
-extern int sbx00_acpi_init(void);
-
 static int __init pcibios_init(void)
 {
 	setup_pcimap();
diff --git a/arch/mips/math-emu/cp1emu.c b/arch/mips/math-emu/cp1emu.c
index 587cf1d115e8..265bc57819df 100644
--- a/arch/mips/math-emu/cp1emu.c
+++ b/arch/mips/math-emu/cp1emu.c
@@ -1032,7 +1032,7 @@ static int cop1Emulate(struct pt_regs *xcp, struct mips_fpu_struct *ctx,
 	 */
 	if (dec_insn.micro_mips_mode) {
 		/*
-		 * If next instruction is a 16-bit instruction, then it
+		 * If next instruction is a 16-bit instruction, then
 		 * it cannot be a FPU instruction. This could happen
 		 * since we can be called for non-FPU instructions.
 		 */
diff --git a/arch/mips/net/bpf_jit_comp32.c b/arch/mips/net/bpf_jit_comp32.c
index 83c975d5cca2..ace5db3fbd17 100644
--- a/arch/mips/net/bpf_jit_comp32.c
+++ b/arch/mips/net/bpf_jit_comp32.c
@@ -1377,11 +1377,19 @@ void build_prologue(struct jit_context *ctx)
 	int stack, saved, locals, reserved;
 
 	/*
+	 * In the unlikely event that the TCC limit is raised to more
+	 * than 16 bits, it is clamped to the maximum value allowed for
+	 * the generated code (0xffff). It is better fail to compile
+	 * instead of degrading gracefully.
+	 */
+	BUILD_BUG_ON(MAX_TAIL_CALL_CNT > 0xffff);
+
+	/*
 	 * The first two instructions initialize TCC in the reserved (for us)
 	 * 16-byte area in the parent's stack frame. On a tail call, the
 	 * calling function jumps into the prologue after these instructions.
 	 */
-	emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT, 0xffff));
+	emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
 	emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP);
 
 	/*
diff --git a/arch/mips/net/bpf_jit_comp64.c b/arch/mips/net/bpf_jit_comp64.c
index 6475828ffb36..0e7c1bdcf914 100644
--- a/arch/mips/net/bpf_jit_comp64.c
+++ b/arch/mips/net/bpf_jit_comp64.c
@@ -548,11 +548,19 @@ void build_prologue(struct jit_context *ctx)
 	int stack, saved, locals, reserved;
 
 	/*
+	 * In the unlikely event that the TCC limit is raised to more
+	 * than 16 bits, it is clamped to the maximum value allowed for
+	 * the generated code (0xffff). It is better fail to compile
+	 * instead of degrading gracefully.
+	 */
+	BUILD_BUG_ON(MAX_TAIL_CALL_CNT > 0xffff);
+
+	/*
 	 * The first instruction initializes the tail call count register.
 	 * On a tail call, the calling function jumps into the prologue
 	 * after this instruction.
 	 */
-	emit(ctx, ori, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT, 0xffff));
+	emit(ctx, ori, tc, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
 
 	/* === Entry-point for tail calls === */
 
diff --git a/arch/mips/pci/pci-ar2315.c b/arch/mips/pci/pci-ar2315.c
index 30e0922f4cea..e17d862cfa4c 100644
--- a/arch/mips/pci/pci-ar2315.c
+++ b/arch/mips/pci/pci-ar2315.c
@@ -2,7 +2,7 @@
 /*
  */
 
-/**
+/*
  * Both AR2315 and AR2316 chips have PCI interface unit, which supports DMA
  * and interrupt. PCI interface supports MMIO access method, but does not
  * seem to support I/O ports.
diff --git a/arch/mips/pci/pci-lantiq.c b/arch/mips/pci/pci-lantiq.c
index 1ca42f482130..8d16cd021f60 100644
--- a/arch/mips/pci/pci-lantiq.c
+++ b/arch/mips/pci/pci-lantiq.c
@@ -9,11 +9,11 @@
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/mm.h>
 #include <linux/vmalloc.h>
 #include <linux/clk.h>
 #include <linux/of_platform.h>
-#include <linux/of_gpio.h>
 #include <linux/of_irq.h>
 #include <linux/of_pci.h>
 
@@ -62,7 +62,7 @@
 __iomem void *ltq_pci_mapped_cfg;
 static __iomem void *ltq_pci_membase;
 
-static int reset_gpio;
+static struct gpio_desc *reset_gpio;
 static struct clk *clk_pci, *clk_external;
 static struct resource pci_io_resource;
 static struct resource pci_mem_resource;
@@ -95,6 +95,7 @@ static int ltq_pci_startup(struct platform_device *pdev)
 	struct device_node *node = pdev->dev.of_node;
 	const __be32 *req_mask, *bus_clk;
 	u32 temp_buffer;
+	int error;
 
 	/* get our clocks */
 	clk_pci = clk_get(&pdev->dev, NULL);
@@ -123,17 +124,14 @@ static int ltq_pci_startup(struct platform_device *pdev)
 		clk_disable(clk_external);
 
 	/* setup reset gpio used by pci */
-	reset_gpio = of_get_named_gpio(node, "gpio-reset", 0);
-	if (gpio_is_valid(reset_gpio)) {
-		int ret = devm_gpio_request(&pdev->dev,
-						reset_gpio, "pci-reset");
-		if (ret) {
-			dev_err(&pdev->dev,
-				"failed to request gpio %d\n", reset_gpio);
-			return ret;
-		}
-		gpio_direction_output(reset_gpio, 1);
+	reset_gpio = devm_gpiod_get_optional(&pdev->dev, "reset",
+					     GPIOD_OUT_LOW);
+	error = PTR_ERR_OR_ZERO(reset_gpio);
+	if (error) {
+		dev_err(&pdev->dev, "failed to request gpio: %d\n", error);
+		return error;
 	}
+	gpiod_set_consumer_name(reset_gpio, "pci_reset");
 
 	/* enable auto-switching between PCI and EBU */
 	ltq_pci_w32(0xa, PCI_CR_CLK_CTRL);
@@ -195,11 +193,11 @@ static int ltq_pci_startup(struct platform_device *pdev)
 	ltq_ebu_w32(ltq_ebu_r32(LTQ_EBU_PCC_IEN) | 0x10, LTQ_EBU_PCC_IEN);
 
 	/* toggle reset pin */
-	if (gpio_is_valid(reset_gpio)) {
-		__gpio_set_value(reset_gpio, 0);
+	if (reset_gpio) {
+		gpiod_set_value_cansleep(reset_gpio, 1);
 		wmb();
 		mdelay(1);
-		__gpio_set_value(reset_gpio, 1);
+		gpiod_set_value_cansleep(reset_gpio, 0);
 	}
 	return 0;
 }
diff --git a/arch/mips/pic32/pic32mzda/init.c b/arch/mips/pic32/pic32mzda/init.c
index d9c8c4e46aff..08c46cf122d7 100644
--- a/arch/mips/pic32/pic32mzda/init.c
+++ b/arch/mips/pic32/pic32mzda/init.c
@@ -44,7 +44,7 @@ void __init plat_mem_setup(void)
 	pr_info(" builtin_cmdline  : %s\n", CONFIG_CMDLINE);
 #endif
 	if (dtb != __dtb_start)
-		strlcpy(arcs_cmdline, boot_command_line, COMMAND_LINE_SIZE);
+		strscpy(arcs_cmdline, boot_command_line, COMMAND_LINE_SIZE);
 
 #ifdef CONFIG_EARLY_PRINTK
 	fw_init_early_console(-1);
diff --git a/arch/mips/ralink/bootrom.c b/arch/mips/ralink/bootrom.c
index 94ca8379b83c..8c8cc0a81ed8 100644
--- a/arch/mips/ralink/bootrom.c
+++ b/arch/mips/ralink/bootrom.c
@@ -18,22 +18,11 @@ static int bootrom_show(struct seq_file *s, void *unused)
 
 	return 0;
 }
-
-static int bootrom_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, bootrom_show, NULL);
-}
-
-static const struct file_operations bootrom_file_ops = {
-	.open		= bootrom_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(bootrom);
 
 static int __init bootrom_setup(void)
 {
-	debugfs_create_file("bootrom", 0444, NULL, NULL, &bootrom_file_ops);
+	debugfs_create_file("bootrom", 0444, NULL, NULL, &bootrom_fops);
 	return 0;
 }
 
diff --git a/arch/mips/sgi-ip27/ip27-xtalk.c b/arch/mips/sgi-ip27/ip27-xtalk.c
index e762886d1dda..5143d1cf8984 100644
--- a/arch/mips/sgi-ip27/ip27-xtalk.c
+++ b/arch/mips/sgi-ip27/ip27-xtalk.c
@@ -27,15 +27,18 @@ static void bridge_platform_create(nasid_t nasid, int widget, int masterwid)
 {
 	struct xtalk_bridge_platform_data *bd;
 	struct sgi_w1_platform_data *wd;
-	struct platform_device *pdev;
+	struct platform_device *pdev_wd;
+	struct platform_device *pdev_bd;
 	struct resource w1_res;
 	unsigned long offset;
 
 	offset = NODE_OFFSET(nasid);
 
 	wd = kzalloc(sizeof(*wd), GFP_KERNEL);
-	if (!wd)
-		goto no_mem;
+	if (!wd) {
+		pr_warn("xtalk:n%d/%x bridge create out of memory\n", nasid, widget);
+		return;
+	}
 
 	snprintf(wd->dev_id, sizeof(wd->dev_id), "bridge-%012lx",
 		 offset + (widget << SWIN_SIZE_BITS));
@@ -46,24 +49,35 @@ static void bridge_platform_create(nasid_t nasid, int widget, int masterwid)
 	w1_res.end = w1_res.start + 3;
 	w1_res.flags = IORESOURCE_MEM;
 
-	pdev = platform_device_alloc("sgi_w1", PLATFORM_DEVID_AUTO);
-	if (!pdev) {
-		kfree(wd);
-		goto no_mem;
+	pdev_wd = platform_device_alloc("sgi_w1", PLATFORM_DEVID_AUTO);
+	if (!pdev_wd) {
+		pr_warn("xtalk:n%d/%x bridge create out of memory\n", nasid, widget);
+		goto err_kfree_wd;
+	}
+	if (platform_device_add_resources(pdev_wd, &w1_res, 1)) {
+		pr_warn("xtalk:n%d/%x bridge failed to add platform resources.\n", nasid, widget);
+		goto err_put_pdev_wd;
+	}
+	if (platform_device_add_data(pdev_wd, wd, sizeof(*wd))) {
+		pr_warn("xtalk:n%d/%x bridge failed to add platform data.\n", nasid, widget);
+		goto err_put_pdev_wd;
+	}
+	if (platform_device_add(pdev_wd)) {
+		pr_warn("xtalk:n%d/%x bridge failed to add platform device.\n", nasid, widget);
+		goto err_put_pdev_wd;
 	}
-	platform_device_add_resources(pdev, &w1_res, 1);
-	platform_device_add_data(pdev, wd, sizeof(*wd));
 	/* platform_device_add_data() duplicates the data */
 	kfree(wd);
-	platform_device_add(pdev);
 
 	bd = kzalloc(sizeof(*bd), GFP_KERNEL);
-	if (!bd)
-		goto no_mem;
-	pdev = platform_device_alloc("xtalk-bridge", PLATFORM_DEVID_AUTO);
-	if (!pdev) {
-		kfree(bd);
-		goto no_mem;
+	if (!bd) {
+		pr_warn("xtalk:n%d/%x bridge create out of memory\n", nasid, widget);
+		goto err_unregister_pdev_wd;
+	}
+	pdev_bd = platform_device_alloc("xtalk-bridge", PLATFORM_DEVID_AUTO);
+	if (!pdev_bd) {
+		pr_warn("xtalk:n%d/%x bridge create out of memory\n", nasid, widget);
+		goto err_kfree_bd;
 	}
 
 
@@ -84,15 +98,31 @@ static void bridge_platform_create(nasid_t nasid, int widget, int masterwid)
 	bd->io.flags	= IORESOURCE_IO;
 	bd->io_offset	= offset;
 
-	platform_device_add_data(pdev, bd, sizeof(*bd));
+	if (platform_device_add_data(pdev_bd, bd, sizeof(*bd))) {
+		pr_warn("xtalk:n%d/%x bridge failed to add platform data.\n", nasid, widget);
+		goto err_put_pdev_bd;
+	}
+	if (platform_device_add(pdev_bd)) {
+		pr_warn("xtalk:n%d/%x bridge failed to add platform device.\n", nasid, widget);
+		goto err_put_pdev_bd;
+	}
 	/* platform_device_add_data() duplicates the data */
 	kfree(bd);
-	platform_device_add(pdev);
 	pr_info("xtalk:n%d/%x bridge widget\n", nasid, widget);
 	return;
 
-no_mem:
-	pr_warn("xtalk:n%d/%x bridge create out of memory\n", nasid, widget);
+err_put_pdev_bd:
+	platform_device_put(pdev_bd);
+err_kfree_bd:
+	kfree(bd);
+err_unregister_pdev_wd:
+	platform_device_unregister(pdev_wd);
+	return;
+err_put_pdev_wd:
+	platform_device_put(pdev_wd);
+err_kfree_wd:
+	kfree(wd);
+	return;
 }
 
 static int probe_one_port(nasid_t nasid, int widget, int masterwid)
diff --git a/arch/mips/sgi-ip30/ip30-xtalk.c b/arch/mips/sgi-ip30/ip30-xtalk.c
index 8129524421cb..7ceb2b23ea1c 100644
--- a/arch/mips/sgi-ip30/ip30-xtalk.c
+++ b/arch/mips/sgi-ip30/ip30-xtalk.c
@@ -40,12 +40,15 @@ static void bridge_platform_create(int widget, int masterwid)
 {
 	struct xtalk_bridge_platform_data *bd;
 	struct sgi_w1_platform_data *wd;
-	struct platform_device *pdev;
+	struct platform_device *pdev_wd;
+	struct platform_device *pdev_bd;
 	struct resource w1_res;
 
 	wd = kzalloc(sizeof(*wd), GFP_KERNEL);
-	if (!wd)
-		goto no_mem;
+	if (!wd) {
+		pr_warn("xtalk:%x bridge create out of memory\n", widget);
+		return;
+	}
 
 	snprintf(wd->dev_id, sizeof(wd->dev_id), "bridge-%012lx",
 		 IP30_SWIN_BASE(widget));
@@ -56,24 +59,35 @@ static void bridge_platform_create(int widget, int masterwid)
 	w1_res.end = w1_res.start + 3;
 	w1_res.flags = IORESOURCE_MEM;
 
-	pdev = platform_device_alloc("sgi_w1", PLATFORM_DEVID_AUTO);
-	if (!pdev) {
-		kfree(wd);
-		goto no_mem;
+	pdev_wd = platform_device_alloc("sgi_w1", PLATFORM_DEVID_AUTO);
+	if (!pdev_wd) {
+		pr_warn("xtalk:%x bridge create out of memory\n", widget);
+		goto err_kfree_wd;
+	}
+	if (platform_device_add_resources(pdev_wd, &w1_res, 1)) {
+		pr_warn("xtalk:%x bridge failed to add platform resources.\n", widget);
+		goto err_put_pdev_wd;
+	}
+	if (platform_device_add_data(pdev_wd, wd, sizeof(*wd))) {
+		pr_warn("xtalk:%x bridge failed to add platform data.\n", widget);
+		goto err_put_pdev_wd;
+	}
+	if (platform_device_add(pdev_wd)) {
+		pr_warn("xtalk:%x bridge failed to add platform device.\n", widget);
+		goto err_put_pdev_wd;
 	}
-	platform_device_add_resources(pdev, &w1_res, 1);
-	platform_device_add_data(pdev, wd, sizeof(*wd));
 	/* platform_device_add_data() duplicates the data */
 	kfree(wd);
-	platform_device_add(pdev);
 
 	bd = kzalloc(sizeof(*bd), GFP_KERNEL);
-	if (!bd)
-		goto no_mem;
-	pdev = platform_device_alloc("xtalk-bridge", PLATFORM_DEVID_AUTO);
-	if (!pdev) {
-		kfree(bd);
-		goto no_mem;
+	if (!bd) {
+		pr_warn("xtalk:%x bridge create out of memory\n", widget);
+		goto err_unregister_pdev_wd;
+	}
+	pdev_bd = platform_device_alloc("xtalk-bridge", PLATFORM_DEVID_AUTO);
+	if (!pdev_bd) {
+		pr_warn("xtalk:%x bridge create out of memory\n", widget);
+		goto err_kfree_bd;
 	}
 
 	bd->bridge_addr	= IP30_RAW_SWIN_BASE(widget);
@@ -93,15 +107,31 @@ static void bridge_platform_create(int widget, int masterwid)
 	bd->io.flags	= IORESOURCE_IO;
 	bd->io_offset	= IP30_SWIN_BASE(widget);
 
-	platform_device_add_data(pdev, bd, sizeof(*bd));
+	if (platform_device_add_data(pdev_bd, bd, sizeof(*bd))) {
+		pr_warn("xtalk:%x bridge failed to add platform data.\n", widget);
+		goto err_put_pdev_bd;
+	}
+	if (platform_device_add(pdev_bd)) {
+		pr_warn("xtalk:%x bridge failed to add platform device.\n", widget);
+		goto err_put_pdev_bd;
+	}
 	/* platform_device_add_data() duplicates the data */
 	kfree(bd);
-	platform_device_add(pdev);
 	pr_info("xtalk:%x bridge widget\n", widget);
 	return;
 
-no_mem:
-	pr_warn("xtalk:%x bridge create out of memory\n", widget);
+err_put_pdev_bd:
+	platform_device_put(pdev_bd);
+err_kfree_bd:
+	kfree(bd);
+err_unregister_pdev_wd:
+	platform_device_unregister(pdev_wd);
+	return;
+err_put_pdev_wd:
+	platform_device_put(pdev_wd);
+err_kfree_wd:
+	kfree(wd);
+	return;
 }
 
 static unsigned int __init xbow_widget_active(s8 wid)
diff --git a/arch/mips/sibyte/sb1250/irq.c b/arch/mips/sibyte/sb1250/irq.c
index 86f49c48fc34..2f08ad267a11 100644
--- a/arch/mips/sibyte/sb1250/irq.c
+++ b/arch/mips/sibyte/sb1250/irq.c
@@ -262,12 +262,6 @@ void __init arch_init_irq(void)
 	__raw_writeq(tmp, IOADDR(A_IMR_REGISTER(0, R_IMR_INTERRUPT_MASK)));
 	__raw_writeq(tmp, IOADDR(A_IMR_REGISTER(1, R_IMR_INTERRUPT_MASK)));
 
-	/*
-	 * Note that the timer interrupts are also mapped, but this is
-	 * done in sb1250_time_init().	Also, the profiling driver
-	 * does its own management of IP7.
-	 */
-
 	/* Enable necessary IPs, disable the rest */
 	change_c0_status(ST0_IM, imask);
 }
diff --git a/arch/parisc/include/asm/io.h b/arch/parisc/include/asm/io.h
index 42ffb60a6ea9..c05e781be2f5 100644
--- a/arch/parisc/include/asm/io.h
+++ b/arch/parisc/include/asm/io.h
@@ -128,98 +128,16 @@ static inline void gsc_writeq(unsigned long long val, unsigned long addr)
 void __iomem *ioremap(unsigned long offset, unsigned long size);
 #define ioremap_wc			ioremap
 #define ioremap_uc			ioremap
+#define pci_iounmap			pci_iounmap
 
 extern void iounmap(const volatile void __iomem *addr);
 
-static inline unsigned char __raw_readb(const volatile void __iomem *addr)
-{
-	return (*(volatile unsigned char __force *) (addr));
-}
-static inline unsigned short __raw_readw(const volatile void __iomem *addr)
-{
-	return *(volatile unsigned short __force *) addr;
-}
-static inline unsigned int __raw_readl(const volatile void __iomem *addr)
-{
-	return *(volatile unsigned int __force *) addr;
-}
-static inline unsigned long long __raw_readq(const volatile void __iomem *addr)
-{
-	return *(volatile unsigned long long __force *) addr;
-}
-
-static inline void __raw_writeb(unsigned char b, volatile void __iomem *addr)
-{
-	*(volatile unsigned char __force *) addr = b;
-}
-static inline void __raw_writew(unsigned short b, volatile void __iomem *addr)
-{
-	*(volatile unsigned short __force *) addr = b;
-}
-static inline void __raw_writel(unsigned int b, volatile void __iomem *addr)
-{
-	*(volatile unsigned int __force *) addr = b;
-}
-static inline void __raw_writeq(unsigned long long b, volatile void __iomem *addr)
-{
-	*(volatile unsigned long long __force *) addr = b;
-}
-
-static inline unsigned char readb(const volatile void __iomem *addr)
-{
-	return __raw_readb(addr);
-}
-static inline unsigned short readw(const volatile void __iomem *addr)
-{
-	return le16_to_cpu((__le16 __force) __raw_readw(addr));
-}
-static inline unsigned int readl(const volatile void __iomem *addr)
-{
-	return le32_to_cpu((__le32 __force) __raw_readl(addr));
-}
-static inline unsigned long long readq(const volatile void __iomem *addr)
-{
-	return le64_to_cpu((__le64 __force) __raw_readq(addr));
-}
-
-static inline void writeb(unsigned char b, volatile void __iomem *addr)
-{
-	__raw_writeb(b, addr);
-}
-static inline void writew(unsigned short w, volatile void __iomem *addr)
-{
-	__raw_writew((__u16 __force) cpu_to_le16(w), addr);
-}
-static inline void writel(unsigned int l, volatile void __iomem *addr)
-{
-	__raw_writel((__u32 __force) cpu_to_le32(l), addr);
-}
-static inline void writeq(unsigned long long q, volatile void __iomem *addr)
-{
-	__raw_writeq((__u64 __force) cpu_to_le64(q), addr);
-}
-
-#define	readb	readb
-#define	readw	readw
-#define	readl	readl
-#define readq	readq
-#define writeb	writeb
-#define writew	writew
-#define writel	writel
-#define writeq	writeq
-
-#define readb_relaxed(addr)	readb(addr)
-#define readw_relaxed(addr)	readw(addr)
-#define readl_relaxed(addr)	readl(addr)
-#define readq_relaxed(addr)	readq(addr)
-#define writeb_relaxed(b, addr)	writeb(b, addr)
-#define writew_relaxed(w, addr)	writew(w, addr)
-#define writel_relaxed(l, addr)	writel(l, addr)
-#define writeq_relaxed(q, addr)	writeq(q, addr)
-
 void memset_io(volatile void __iomem *addr, unsigned char val, int count);
 void memcpy_fromio(void *dst, const volatile void __iomem *src, int count);
 void memcpy_toio(volatile void __iomem *dst, const void *src, int count);
+#define memset_io memset_io
+#define memcpy_fromio memcpy_fromio
+#define memcpy_toio memcpy_toio
 
 /* Port-space IO */
 
@@ -241,10 +159,15 @@ extern void eisa_out32(unsigned int data, unsigned short port);
 extern unsigned char inb(int addr);
 extern unsigned short inw(int addr);
 extern unsigned int inl(int addr);
-
 extern void outb(unsigned char b, int addr);
 extern void outw(unsigned short b, int addr);
 extern void outl(unsigned int b, int addr);
+#define inb inb
+#define inw inw
+#define inl inl
+#define outb outb
+#define outw outw
+#define outl outl
 #elif defined(CONFIG_EISA)
 #define inb eisa_in8
 #define inw eisa_in16
@@ -270,7 +193,9 @@ static inline int inl(unsigned long addr)
 	BUG();
 	return -1;
 }
-
+#define inb inb
+#define inw inw
+#define inl inl
 #define outb(x, y)	({(void)(x); (void)(y); BUG(); 0;})
 #define outw(x, y)	({(void)(x); (void)(y); BUG(); 0;})
 #define outl(x, y)	({(void)(x); (void)(y); BUG(); 0;})
@@ -285,7 +210,12 @@ extern void insl (unsigned long port, void *dst, unsigned long count);
 extern void outsb (unsigned long port, const void *src, unsigned long count);
 extern void outsw (unsigned long port, const void *src, unsigned long count);
 extern void outsl (unsigned long port, const void *src, unsigned long count);
-
+#define insb insb
+#define insw insw
+#define insl insl
+#define outsb outsb
+#define outsw outsw
+#define outsl outsl
 
 /* IO Port space is :      BBiiii   where BB is HBA number. */
 #define IO_SPACE_LIMIT 0x00ffffff
@@ -297,6 +227,7 @@ extern void outsl (unsigned long port, const void *src, unsigned long count);
  * value for either 32 or 64 bit mode */
 #define F_EXTEND(x) ((unsigned long)((x) | (0xffffffff00000000ULL)))
 
+#ifdef CONFIG_64BIT
 #define ioread64 ioread64
 #define ioread64be ioread64be
 #define iowrite64 iowrite64
@@ -305,8 +236,31 @@ extern u64 ioread64(const void __iomem *addr);
 extern u64 ioread64be(const void __iomem *addr);
 extern void iowrite64(u64 val, void __iomem *addr);
 extern void iowrite64be(u64 val, void __iomem *addr);
+#endif
 
 #include <asm-generic/iomap.h>
+/*
+ * These get provided from <asm-generic/iomap.h> since parisc does not
+ * select GENERIC_IOMAP.
+ */
+#define ioport_map ioport_map
+#define ioport_unmap ioport_unmap
+#define ioread8 ioread8
+#define ioread16 ioread16
+#define ioread32 ioread32
+#define ioread16be ioread16be
+#define ioread32be ioread32be
+#define iowrite8 iowrite8
+#define iowrite16 iowrite16
+#define iowrite32 iowrite32
+#define iowrite16be iowrite16be
+#define iowrite32be iowrite32be
+#define ioread8_rep ioread8_rep
+#define ioread16_rep ioread16_rep
+#define ioread32_rep ioread32_rep
+#define iowrite8_rep iowrite8_rep
+#define iowrite16_rep iowrite16_rep
+#define iowrite32_rep iowrite32_rep
 
 /*
  * Convert a physical pointer to a virtual kernel pointer for /dev/mem
@@ -316,4 +270,6 @@ extern void iowrite64be(u64 val, void __iomem *addr);
 
 extern int devmem_is_allowed(unsigned long pfn);
 
+#include <asm-generic/io.h>
+
 #endif
diff --git a/arch/parisc/lib/iomap.c b/arch/parisc/lib/iomap.c
index 860385058085..915c0c4da663 100644
--- a/arch/parisc/lib/iomap.c
+++ b/arch/parisc/lib/iomap.c
@@ -48,15 +48,19 @@ struct iomap_ops {
 	unsigned int (*read16be)(const void __iomem *);
 	unsigned int (*read32)(const void __iomem *);
 	unsigned int (*read32be)(const void __iomem *);
+#ifdef CONFIG_64BIT
 	u64 (*read64)(const void __iomem *);
 	u64 (*read64be)(const void __iomem *);
+#endif
 	void (*write8)(u8, void __iomem *);
 	void (*write16)(u16, void __iomem *);
 	void (*write16be)(u16, void __iomem *);
 	void (*write32)(u32, void __iomem *);
 	void (*write32be)(u32, void __iomem *);
+#ifdef CONFIG_64BIT
 	void (*write64)(u64, void __iomem *);
 	void (*write64be)(u64, void __iomem *);
+#endif
 	void (*read8r)(const void __iomem *, void *, unsigned long);
 	void (*read16r)(const void __iomem *, void *, unsigned long);
 	void (*read32r)(const void __iomem *, void *, unsigned long);
@@ -175,6 +179,7 @@ static unsigned int iomem_read32be(const void __iomem *addr)
 	return __raw_readl(addr);
 }
 
+#ifdef CONFIG_64BIT
 static u64 iomem_read64(const void __iomem *addr)
 {
 	return readq(addr);
@@ -184,6 +189,7 @@ static u64 iomem_read64be(const void __iomem *addr)
 {
 	return __raw_readq(addr);
 }
+#endif
 
 static void iomem_write8(u8 datum, void __iomem *addr)
 {
@@ -210,15 +216,17 @@ static void iomem_write32be(u32 datum, void __iomem *addr)
 	__raw_writel(datum, addr);
 }
 
+#ifdef CONFIG_64BIT
 static void iomem_write64(u64 datum, void __iomem *addr)
 {
-	writel(datum, addr);
+	writeq(datum, addr);
 }
 
 static void iomem_write64be(u64 datum, void __iomem *addr)
 {
-	__raw_writel(datum, addr);
+	__raw_writeq(datum, addr);
 }
+#endif
 
 static void iomem_read8r(const void __iomem *addr, void *dst, unsigned long count)
 {
@@ -274,15 +282,19 @@ static const struct iomap_ops iomem_ops = {
 	.read16be = iomem_read16be,
 	.read32 = iomem_read32,
 	.read32be = iomem_read32be,
+#ifdef CONFIG_64BIT
 	.read64 = iomem_read64,
 	.read64be = iomem_read64be,
+#endif
 	.write8 = iomem_write8,
 	.write16 = iomem_write16,
 	.write16be = iomem_write16be,
 	.write32 = iomem_write32,
 	.write32be = iomem_write32be,
+#ifdef CONFIG_64BIT
 	.write64 = iomem_write64,
 	.write64be = iomem_write64be,
+#endif
 	.read8r = iomem_read8r,
 	.read16r = iomem_read16r,
 	.read32r = iomem_read32r,
@@ -332,6 +344,7 @@ unsigned int ioread32be(const void __iomem *addr)
 	return *((u32 *)addr);
 }
 
+#ifdef CONFIG_64BIT
 u64 ioread64(const void __iomem *addr)
 {
 	if (unlikely(INDIRECT_ADDR(addr)))
@@ -345,26 +358,7 @@ u64 ioread64be(const void __iomem *addr)
 		return iomap_ops[ADDR_TO_REGION(addr)]->read64be(addr);
 	return *((u64 *)addr);
 }
-
-u64 ioread64_lo_hi(const void __iomem *addr)
-{
-	u32 low, high;
-
-	low = ioread32(addr);
-	high = ioread32(addr + sizeof(u32));
-
-	return low + ((u64)high << 32);
-}
-
-u64 ioread64_hi_lo(const void __iomem *addr)
-{
-	u32 low, high;
-
-	high = ioread32(addr + sizeof(u32));
-	low = ioread32(addr);
-
-	return low + ((u64)high << 32);
-}
+#endif
 
 void iowrite8(u8 datum, void __iomem *addr)
 {
@@ -411,6 +405,7 @@ void iowrite32be(u32 datum, void __iomem *addr)
 	}
 }
 
+#ifdef CONFIG_64BIT
 void iowrite64(u64 datum, void __iomem *addr)
 {
 	if (unlikely(INDIRECT_ADDR(addr))) {
@@ -428,18 +423,7 @@ void iowrite64be(u64 datum, void __iomem *addr)
 		*((u64 *)addr) = datum;
 	}
 }
-
-void iowrite64_lo_hi(u64 val, void __iomem *addr)
-{
-	iowrite32(val, addr);
-	iowrite32(val >> 32, addr + sizeof(u32));
-}
-
-void iowrite64_hi_lo(u64 val, void __iomem *addr)
-{
-	iowrite32(val >> 32, addr + sizeof(u32));
-	iowrite32(val, addr);
-}
+#endif
 
 /* Repeating interfaces */
 
@@ -544,19 +528,19 @@ EXPORT_SYMBOL(ioread16);
 EXPORT_SYMBOL(ioread16be);
 EXPORT_SYMBOL(ioread32);
 EXPORT_SYMBOL(ioread32be);
+#ifdef CONFIG_64BIT
 EXPORT_SYMBOL(ioread64);
 EXPORT_SYMBOL(ioread64be);
-EXPORT_SYMBOL(ioread64_lo_hi);
-EXPORT_SYMBOL(ioread64_hi_lo);
+#endif
 EXPORT_SYMBOL(iowrite8);
 EXPORT_SYMBOL(iowrite16);
 EXPORT_SYMBOL(iowrite16be);
 EXPORT_SYMBOL(iowrite32);
 EXPORT_SYMBOL(iowrite32be);
+#ifdef CONFIG_64BIT
 EXPORT_SYMBOL(iowrite64);
 EXPORT_SYMBOL(iowrite64be);
-EXPORT_SYMBOL(iowrite64_lo_hi);
-EXPORT_SYMBOL(iowrite64_hi_lo);
+#endif
 EXPORT_SYMBOL(ioread8_rep);
 EXPORT_SYMBOL(ioread16_rep);
 EXPORT_SYMBOL(ioread32_rep);
diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig
index 91967824272e..a24f484bfbd2 100644
--- a/arch/powerpc/configs/ppc6xx_defconfig
+++ b/arch/powerpc/configs/ppc6xx_defconfig
@@ -243,8 +243,6 @@ CONFIG_ATM_LANE=m
 CONFIG_ATM_BR2684=m
 CONFIG_BRIDGE=m
 CONFIG_VLAN_8021Q=m
-CONFIG_DECNET=m
-CONFIG_DECNET_ROUTER=y
 CONFIG_ATALK=m
 CONFIG_DEV_APPLETALK=m
 CONFIG_IPDDP=m
diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c
index 698274109c91..e712f80fe189 100644
--- a/arch/powerpc/mm/book3s64/radix_pgtable.c
+++ b/arch/powerpc/mm/book3s64/radix_pgtable.c
@@ -937,15 +937,6 @@ pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addre
 	pmd = *pmdp;
 	pmd_clear(pmdp);
 
-	/*
-	 * pmdp collapse_flush need to ensure that there are no parallel gup
-	 * walk after this call. This is needed so that we can have stable
-	 * page ref count when collapsing a page. We don't allow a collapse page
-	 * if we have gup taken on the page. We can ensure that by sending IPI
-	 * because gup walk happens with IRQ disabled.
-	 */
-	serialize_against_pte_lookup(vma->vm_mm);
-
 	radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
 
 	return pmd;
diff --git a/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c b/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c
index abb62fa630ef..77ed61306a73 100644
--- a/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c
+++ b/arch/powerpc/platforms/83xx/mcu_mpc8349emitx.c
@@ -178,7 +178,7 @@ err:
 	return ret;
 }
 
-static int mcu_remove(struct i2c_client *client)
+static void mcu_remove(struct i2c_client *client)
 {
 	struct mcu *mcu = i2c_get_clientdata(client);
 
@@ -193,7 +193,6 @@ static int mcu_remove(struct i2c_client *client)
 
 	mcu_gpiochip_remove(mcu);
 	kfree(mcu);
-	return 0;
 }
 
 static const struct i2c_device_id mcu_ids[] = {
diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c
index 320008528edd..dbcfe361831a 100644
--- a/arch/powerpc/platforms/cell/spufs/inode.c
+++ b/arch/powerpc/platforms/cell/spufs/inode.c
@@ -275,7 +275,7 @@ spufs_mkdir(struct inode *dir, struct dentry *dentry, unsigned int flags,
 	return ret;
 }
 
-static int spufs_context_open(struct path *path)
+static int spufs_context_open(const struct path *path)
 {
 	int ret;
 	struct file *filp;
@@ -491,7 +491,7 @@ out:
 	return ret;
 }
 
-static int spufs_gang_open(struct path *path)
+static int spufs_gang_open(const struct path *path)
 {
 	int ret;
 	struct file *filp;
@@ -536,7 +536,7 @@ static int spufs_create_gang(struct inode *inode,
 
 static struct file_system_type spufs_type;
 
-long spufs_create(struct path *path, struct dentry *dentry,
+long spufs_create(const struct path *path, struct dentry *dentry,
 		unsigned int flags, umode_t mode, struct file *filp)
 {
 	struct inode *dir = d_inode(path->dentry);
diff --git a/arch/powerpc/platforms/cell/spufs/spufs.h b/arch/powerpc/platforms/cell/spufs/spufs.h
index 23c6799cfa5a..af048b6dd30a 100644
--- a/arch/powerpc/platforms/cell/spufs/spufs.h
+++ b/arch/powerpc/platforms/cell/spufs/spufs.h
@@ -232,7 +232,7 @@ extern const struct spufs_tree_descr spufs_dir_debug_contents[];
 extern struct spufs_calls spufs_calls;
 struct coredump_params;
 long spufs_run_spu(struct spu_context *ctx, u32 *npc, u32 *status);
-long spufs_create(struct path *nd, struct dentry *dentry, unsigned int flags,
+long spufs_create(const struct path *nd, struct dentry *dentry, unsigned int flags,
 			umode_t mode, struct file *filp);
 /* ELF coredump callbacks for writing SPU ELF notes */
 extern int spufs_coredump_extra_notes_size(void);
diff --git a/arch/sh/include/asm/sections.h b/arch/sh/include/asm/sections.h
index 8edb824049b9..0cb0ca149ac3 100644
--- a/arch/sh/include/asm/sections.h
+++ b/arch/sh/include/asm/sections.h
@@ -4,7 +4,7 @@
 
 #include <asm-generic/sections.h>
 
-extern long __machvec_start, __machvec_end;
+extern char __machvec_start[], __machvec_end[];
 extern char __uncached_start, __uncached_end;
 extern char __start_eh_frame[], __stop_eh_frame[];
 
diff --git a/arch/sh/kernel/machvec.c b/arch/sh/kernel/machvec.c
index d606679a211e..57efaf5b82ae 100644
--- a/arch/sh/kernel/machvec.c
+++ b/arch/sh/kernel/machvec.c
@@ -20,8 +20,8 @@
 #define MV_NAME_SIZE 32
 
 #define for_each_mv(mv) \
-	for ((mv) = (struct sh_machine_vector *)&__machvec_start; \
-	     (mv) && (unsigned long)(mv) < (unsigned long)&__machvec_end; \
+	for ((mv) = (struct sh_machine_vector *)__machvec_start; \
+	     (mv) && (unsigned long)(mv) < (unsigned long)__machvec_end; \
 	     (mv)++)
 
 static struct sh_machine_vector * __init get_mv_byname(const char *name)
@@ -87,8 +87,8 @@ void __init sh_mv_setup(void)
 	if (!machvec_selected) {
 		unsigned long machvec_size;
 
-		machvec_size = ((unsigned long)&__machvec_end -
-				(unsigned long)&__machvec_start);
+		machvec_size = ((unsigned long)__machvec_end -
+				(unsigned long)__machvec_start);
 
 		/*
 		 * Sanity check for machvec section alignment. Ensure
@@ -102,7 +102,7 @@ void __init sh_mv_setup(void)
 		 * vector (usually the only one) from .machvec.init.
 		 */
 		if (machvec_size >= sizeof(struct sh_machine_vector))
-			sh_mv = *(struct sh_machine_vector *)&__machvec_start;
+			sh_mv = *(struct sh_machine_vector *)__machvec_start;
 	}
 
 	pr_notice("Booting machvec: %s\n", get_system_type());
diff --git a/arch/sparc/include/asm/io.h b/arch/sparc/include/asm/io.h
index 2eefa526b38f..2dad9be9ec75 100644
--- a/arch/sparc/include/asm/io.h
+++ b/arch/sparc/include/asm/io.h
@@ -19,4 +19,6 @@
 #define writel_be(__w, __addr)	__raw_writel(__w, __addr)
 #define writew_be(__l, __addr)	__raw_writew(__l, __addr)
 
+#include <asm-generic/io.h>
+
 #endif
diff --git a/arch/sparc/include/asm/io_64.h b/arch/sparc/include/asm/io_64.h
index 5ffa820dcd4d..9303270b22f3 100644
--- a/arch/sparc/include/asm/io_64.h
+++ b/arch/sparc/include/asm/io_64.h
@@ -9,6 +9,7 @@
 #include <asm/page.h>      /* IO address mapping routines need this */
 #include <asm/asi.h>
 #include <asm-generic/pci_iomap.h>
+#define pci_iomap pci_iomap
 
 /* BIO layer definitions. */
 extern unsigned long kern_base, kern_size;
@@ -239,38 +240,51 @@ static inline void outl(u32 l, unsigned long addr)
 void outsb(unsigned long, const void *, unsigned long);
 void outsw(unsigned long, const void *, unsigned long);
 void outsl(unsigned long, const void *, unsigned long);
+#define outsb outsb
+#define outsw outsw
+#define outsl outsl
 void insb(unsigned long, void *, unsigned long);
 void insw(unsigned long, void *, unsigned long);
 void insl(unsigned long, void *, unsigned long);
+#define insb insb
+#define insw insw
+#define insl insl
 
 static inline void readsb(void __iomem *port, void *buf, unsigned long count)
 {
 	insb((unsigned long __force)port, buf, count);
 }
+#define readsb readsb
+
 static inline void readsw(void __iomem *port, void *buf, unsigned long count)
 {
 	insw((unsigned long __force)port, buf, count);
 }
+#define readsw readsw
 
 static inline void readsl(void __iomem *port, void *buf, unsigned long count)
 {
 	insl((unsigned long __force)port, buf, count);
 }
+#define readsl readsl
 
 static inline void writesb(void __iomem *port, const void *buf, unsigned long count)
 {
 	outsb((unsigned long __force)port, buf, count);
 }
+#define writesb writesb
 
 static inline void writesw(void __iomem *port, const void *buf, unsigned long count)
 {
 	outsw((unsigned long __force)port, buf, count);
 }
+#define writesw writesw
 
 static inline void writesl(void __iomem *port, const void *buf, unsigned long count)
 {
 	outsl((unsigned long __force)port, buf, count);
 }
+#define writesl writesl
 
 #define ioread8_rep(p,d,l)	readsb(p,d,l)
 #define ioread16_rep(p,d,l)	readsw(p,d,l)
@@ -344,6 +358,7 @@ static inline void memset_io(volatile void __iomem *dst, int c, __kernel_size_t
 		d++;
 	}
 }
+#define memset_io memset_io
 
 static inline void sbus_memcpy_fromio(void *dst, const volatile void __iomem *src,
 				      __kernel_size_t n)
@@ -369,6 +384,7 @@ static inline void memcpy_fromio(void *dst, const volatile void __iomem *src,
 		src++;
 	}
 }
+#define memcpy_fromio memcpy_fromio
 
 static inline void sbus_memcpy_toio(volatile void __iomem *dst, const void *src,
 				    __kernel_size_t n)
@@ -395,6 +411,7 @@ static inline void memcpy_toio(volatile void __iomem *dst, const void *src,
 		d++;
 	}
 }
+#define memcpy_toio memcpy_toio
 
 #ifdef __KERNEL__
 
@@ -412,7 +429,9 @@ static inline void __iomem *ioremap(unsigned long offset, unsigned long size)
 static inline void __iomem *ioremap_np(unsigned long offset, unsigned long size)
 {
 	return NULL;
+
 }
+#define ioremap_np ioremap_np
 
 static inline void iounmap(volatile void __iomem *addr)
 {
@@ -432,10 +451,13 @@ static inline void iounmap(volatile void __iomem *addr)
 /* Create a virtual mapping cookie for an IO port range */
 void __iomem *ioport_map(unsigned long port, unsigned int nr);
 void ioport_unmap(void __iomem *);
+#define ioport_map ioport_map
+#define ioport_unmap ioport_unmap
 
 /* Create a virtual mapping cookie for a PCI BAR (memory or IO) */
 struct pci_dev;
 void pci_iounmap(struct pci_dev *dev, void __iomem *);
+#define pci_iounmap pci_iounmap
 
 static inline int sbus_can_dma_64bit(void)
 {
diff --git a/arch/sparc/include/asm/smp_32.h b/arch/sparc/include/asm/smp_32.h
index 856081761b0f..2cf7971d7f6c 100644
--- a/arch/sparc/include/asm/smp_32.h
+++ b/arch/sparc/include/asm/smp_32.h
@@ -33,9 +33,6 @@ extern volatile unsigned long cpu_callin_map[NR_CPUS];
 extern cpumask_t smp_commenced_mask;
 extern struct linux_prom_registers smp_penguin_ctable;
 
-typedef void (*smpfunc_t)(unsigned long, unsigned long, unsigned long,
-		       unsigned long, unsigned long);
-
 void cpu_panic(void);
 
 /*
@@ -57,7 +54,7 @@ void smp_bogo(struct seq_file *);
 void smp_info(struct seq_file *);
 
 struct sparc32_ipi_ops {
-	void (*cross_call)(smpfunc_t func, cpumask_t mask, unsigned long arg1,
+	void (*cross_call)(void *func, cpumask_t mask, unsigned long arg1,
 			   unsigned long arg2, unsigned long arg3,
 			   unsigned long arg4);
 	void (*resched)(int cpu);
@@ -66,28 +63,28 @@ struct sparc32_ipi_ops {
 };
 extern const struct sparc32_ipi_ops *sparc32_ipi_ops;
 
-static inline void xc0(smpfunc_t func)
+static inline void xc0(void *func)
 {
 	sparc32_ipi_ops->cross_call(func, *cpu_online_mask, 0, 0, 0, 0);
 }
 
-static inline void xc1(smpfunc_t func, unsigned long arg1)
+static inline void xc1(void *func, unsigned long arg1)
 {
 	sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, 0, 0, 0);
 }
-static inline void xc2(smpfunc_t func, unsigned long arg1, unsigned long arg2)
+static inline void xc2(void *func, unsigned long arg1, unsigned long arg2)
 {
 	sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, arg2, 0, 0);
 }
 
-static inline void xc3(smpfunc_t func, unsigned long arg1, unsigned long arg2,
+static inline void xc3(void *func, unsigned long arg1, unsigned long arg2,
 		       unsigned long arg3)
 {
 	sparc32_ipi_ops->cross_call(func, *cpu_online_mask,
 				    arg1, arg2, arg3, 0);
 }
 
-static inline void xc4(smpfunc_t func, unsigned long arg1, unsigned long arg2,
+static inline void xc4(void *func, unsigned long arg1, unsigned long arg2,
 		       unsigned long arg3, unsigned long arg4)
 {
 	sparc32_ipi_ops->cross_call(func, *cpu_online_mask,
diff --git a/arch/sparc/kernel/leon_smp.c b/arch/sparc/kernel/leon_smp.c
index 1eed26d423fb..991e9ad3d3e8 100644
--- a/arch/sparc/kernel/leon_smp.c
+++ b/arch/sparc/kernel/leon_smp.c
@@ -359,7 +359,7 @@ void leonsmp_ipi_interrupt(void)
 }
 
 static struct smp_funcall {
-	smpfunc_t func;
+	void *func;
 	unsigned long arg1;
 	unsigned long arg2;
 	unsigned long arg3;
@@ -372,7 +372,7 @@ static struct smp_funcall {
 static DEFINE_SPINLOCK(cross_call_lock);
 
 /* Cross calls must be serialized, at least currently. */
-static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
+static void leon_cross_call(void *func, cpumask_t mask, unsigned long arg1,
 			    unsigned long arg2, unsigned long arg3,
 			    unsigned long arg4)
 {
@@ -384,7 +384,7 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
 
 		{
 			/* If you make changes here, make sure gcc generates proper code... */
-			register smpfunc_t f asm("i0") = func;
+			register void *f asm("i0") = func;
 			register unsigned long a1 asm("i1") = arg1;
 			register unsigned long a2 asm("i2") = arg2;
 			register unsigned long a3 asm("i3") = arg3;
@@ -444,11 +444,13 @@ static void leon_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
 /* Running cross calls. */
 void leon_cross_call_irq(void)
 {
+	void (*func)(unsigned long, unsigned long, unsigned long, unsigned long,
+		     unsigned long) = ccall_info.func;
 	int i = smp_processor_id();
 
 	ccall_info.processors_in[i] = 1;
-	ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3,
-			ccall_info.arg4, ccall_info.arg5);
+	func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4,
+	     ccall_info.arg5);
 	ccall_info.processors_out[i] = 1;
 }
 
diff --git a/arch/sparc/kernel/sun4d_smp.c b/arch/sparc/kernel/sun4d_smp.c
index ff30f03beb7c..9a62a5cf3337 100644
--- a/arch/sparc/kernel/sun4d_smp.c
+++ b/arch/sparc/kernel/sun4d_smp.c
@@ -268,7 +268,7 @@ static void sun4d_ipi_resched(int cpu)
 }
 
 static struct smp_funcall {
-	smpfunc_t func;
+	void *func;
 	unsigned long arg1;
 	unsigned long arg2;
 	unsigned long arg3;
@@ -281,7 +281,7 @@ static struct smp_funcall {
 static DEFINE_SPINLOCK(cross_call_lock);
 
 /* Cross calls must be serialized, at least currently. */
-static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
+static void sun4d_cross_call(void *func, cpumask_t mask, unsigned long arg1,
 			     unsigned long arg2, unsigned long arg3,
 			     unsigned long arg4)
 {
@@ -296,7 +296,7 @@ static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
 			 * If you make changes here, make sure
 			 * gcc generates proper code...
 			 */
-			register smpfunc_t f asm("i0") = func;
+			register void *f asm("i0") = func;
 			register unsigned long a1 asm("i1") = arg1;
 			register unsigned long a2 asm("i2") = arg2;
 			register unsigned long a3 asm("i3") = arg3;
@@ -353,11 +353,13 @@ static void sun4d_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
 /* Running cross calls. */
 void smp4d_cross_call_irq(void)
 {
+	void (*func)(unsigned long, unsigned long, unsigned long, unsigned long,
+		     unsigned long) = ccall_info.func;
 	int i = hard_smp_processor_id();
 
 	ccall_info.processors_in[i] = 1;
-	ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3,
-			ccall_info.arg4, ccall_info.arg5);
+	func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4,
+	     ccall_info.arg5);
 	ccall_info.processors_out[i] = 1;
 }
 
diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c
index 228a6527082d..056df034e79e 100644
--- a/arch/sparc/kernel/sun4m_smp.c
+++ b/arch/sparc/kernel/sun4m_smp.c
@@ -157,7 +157,7 @@ static void sun4m_ipi_mask_one(int cpu)
 }
 
 static struct smp_funcall {
-	smpfunc_t func;
+	void *func;
 	unsigned long arg1;
 	unsigned long arg2;
 	unsigned long arg3;
@@ -170,7 +170,7 @@ static struct smp_funcall {
 static DEFINE_SPINLOCK(cross_call_lock);
 
 /* Cross calls must be serialized, at least currently. */
-static void sun4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
+static void sun4m_cross_call(void *func, cpumask_t mask, unsigned long arg1,
 			     unsigned long arg2, unsigned long arg3,
 			     unsigned long arg4)
 {
@@ -230,11 +230,13 @@ static void sun4m_cross_call(smpfunc_t func, cpumask_t mask, unsigned long arg1,
 /* Running cross calls. */
 void smp4m_cross_call_irq(void)
 {
+	void (*func)(unsigned long, unsigned long, unsigned long, unsigned long,
+		     unsigned long) = ccall_info.func;
 	int i = smp_processor_id();
 
 	ccall_info.processors_in[i] = 1;
-	ccall_info.func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3,
-			ccall_info.arg4, ccall_info.arg5);
+	func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4,
+	     ccall_info.arg5);
 	ccall_info.processors_out[i] = 1;
 }
 
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index a9aa6a92c7fe..13f027afc875 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -1636,19 +1636,19 @@ static void __init get_srmmu_type(void)
 /* Local cross-calls. */
 static void smp_flush_page_for_dma(unsigned long page)
 {
-	xc1((smpfunc_t) local_ops->page_for_dma, page);
+	xc1(local_ops->page_for_dma, page);
 	local_ops->page_for_dma(page);
 }
 
 static void smp_flush_cache_all(void)
 {
-	xc0((smpfunc_t) local_ops->cache_all);
+	xc0(local_ops->cache_all);
 	local_ops->cache_all();
 }
 
 static void smp_flush_tlb_all(void)
 {
-	xc0((smpfunc_t) local_ops->tlb_all);
+	xc0(local_ops->tlb_all);
 	local_ops->tlb_all();
 }
 
@@ -1659,7 +1659,7 @@ static void smp_flush_cache_mm(struct mm_struct *mm)
 		cpumask_copy(&cpu_mask, mm_cpumask(mm));
 		cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
 		if (!cpumask_empty(&cpu_mask))
-			xc1((smpfunc_t) local_ops->cache_mm, (unsigned long) mm);
+			xc1(local_ops->cache_mm, (unsigned long)mm);
 		local_ops->cache_mm(mm);
 	}
 }
@@ -1671,7 +1671,7 @@ static void smp_flush_tlb_mm(struct mm_struct *mm)
 		cpumask_copy(&cpu_mask, mm_cpumask(mm));
 		cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
 		if (!cpumask_empty(&cpu_mask)) {
-			xc1((smpfunc_t) local_ops->tlb_mm, (unsigned long) mm);
+			xc1(local_ops->tlb_mm, (unsigned long)mm);
 			if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm)
 				cpumask_copy(mm_cpumask(mm),
 					     cpumask_of(smp_processor_id()));
@@ -1691,8 +1691,8 @@ static void smp_flush_cache_range(struct vm_area_struct *vma,
 		cpumask_copy(&cpu_mask, mm_cpumask(mm));
 		cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
 		if (!cpumask_empty(&cpu_mask))
-			xc3((smpfunc_t) local_ops->cache_range,
-			    (unsigned long) vma, start, end);
+			xc3(local_ops->cache_range, (unsigned long)vma, start,
+			    end);
 		local_ops->cache_range(vma, start, end);
 	}
 }
@@ -1708,8 +1708,8 @@ static void smp_flush_tlb_range(struct vm_area_struct *vma,
 		cpumask_copy(&cpu_mask, mm_cpumask(mm));
 		cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
 		if (!cpumask_empty(&cpu_mask))
-			xc3((smpfunc_t) local_ops->tlb_range,
-			    (unsigned long) vma, start, end);
+			xc3(local_ops->tlb_range, (unsigned long)vma, start,
+			    end);
 		local_ops->tlb_range(vma, start, end);
 	}
 }
@@ -1723,8 +1723,7 @@ static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page)
 		cpumask_copy(&cpu_mask, mm_cpumask(mm));
 		cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
 		if (!cpumask_empty(&cpu_mask))
-			xc2((smpfunc_t) local_ops->cache_page,
-			    (unsigned long) vma, page);
+			xc2(local_ops->cache_page, (unsigned long)vma, page);
 		local_ops->cache_page(vma, page);
 	}
 }
@@ -1738,8 +1737,7 @@ static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
 		cpumask_copy(&cpu_mask, mm_cpumask(mm));
 		cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
 		if (!cpumask_empty(&cpu_mask))
-			xc2((smpfunc_t) local_ops->tlb_page,
-			    (unsigned long) vma, page);
+			xc2(local_ops->tlb_page, (unsigned long)vma, page);
 		local_ops->tlb_page(vma, page);
 	}
 }
@@ -1753,7 +1751,7 @@ static void smp_flush_page_to_ram(unsigned long page)
 	 * XXX This experiment failed, research further... -DaveM
 	 */
 #if 1
-	xc1((smpfunc_t) local_ops->page_to_ram, page);
+	xc1(local_ops->page_to_ram, page);
 #endif
 	local_ops->page_to_ram(page);
 }
@@ -1764,8 +1762,7 @@ static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr)
 	cpumask_copy(&cpu_mask, mm_cpumask(mm));
 	cpumask_clear_cpu(smp_processor_id(), &cpu_mask);
 	if (!cpumask_empty(&cpu_mask))
-		xc2((smpfunc_t) local_ops->sig_insns,
-		    (unsigned long) mm, insn_addr);
+		xc2(local_ops->sig_insns, (unsigned long)mm, insn_addr);
 	local_ops->sig_insns(mm, insn_addr);
 }
 
diff --git a/arch/um/Kconfig b/arch/um/Kconfig
index 78de31ac1da7..ad4ff3b0e91e 100644
--- a/arch/um/Kconfig
+++ b/arch/um/Kconfig
@@ -6,6 +6,7 @@ config UML
 	bool
 	default y
 	select ARCH_EPHEMERAL_INODES
+	select ARCH_HAS_FORTIFY_SOURCE
 	select ARCH_HAS_GCOV_PROFILE_ALL
 	select ARCH_HAS_KCOV
 	select ARCH_HAS_STRNCPY_FROM_USER
diff --git a/arch/um/os-Linux/user_syms.c b/arch/um/os-Linux/user_syms.c
index cb667c9225ab..fd575ecbcaec 100644
--- a/arch/um/os-Linux/user_syms.c
+++ b/arch/um/os-Linux/user_syms.c
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
+#define __NO_FORTIFY
 #include <linux/types.h>
 #include <linux/module.h>
 
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index f9920f1341c8..088af7c84e5d 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -107,6 +107,8 @@ config X86
 	select ARCH_SUPPORTS_PAGE_TABLE_CHECK	if X86_64
 	select ARCH_SUPPORTS_NUMA_BALANCING	if X86_64
 	select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP	if NR_CPUS <= 4096
+	select ARCH_SUPPORTS_CFI_CLANG		if X86_64
+	select ARCH_USES_CFI_TRAPS		if X86_64 && CFI_CLANG
 	select ARCH_SUPPORTS_LTO_CLANG
 	select ARCH_SUPPORTS_LTO_CLANG_THIN
 	select ARCH_USE_BUILTIN_BSWAP
@@ -257,6 +259,7 @@ config X86
 	select HAVE_STATIC_CALL_INLINE		if HAVE_OBJTOOL
 	select HAVE_PREEMPT_DYNAMIC_CALL
 	select HAVE_RSEQ
+	select HAVE_RUST			if X86_64
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UACCESS_VALIDATION		if HAVE_OBJTOOL
 	select HAVE_UNSTABLE_SCHED_CLOCK
@@ -284,6 +287,7 @@ config X86
 	select PROC_PID_ARCH_STATUS		if PROC_FS
 	select HAVE_ARCH_NODE_DEV_GROUP		if X86_SGX
 	imply IMA_SECURE_AND_OR_TRUSTED_BOOT    if EFI
+	select HAVE_DYNAMIC_FTRACE_NO_PATCHABLE
 
 config INSTRUCTION_DECODER
 	def_bool y
@@ -448,6 +452,11 @@ config X86_X2APIC
 	  This allows 32-bit apic IDs (so it can support very large systems),
 	  and accesses the local apic via MSRs not via mmio.
 
+	  Some Intel systems circa 2022 and later are locked into x2APIC mode
+	  and can not fall back to the legacy APIC modes if SGX or TDX are
+	  enabled in the BIOS.  They will be unable to boot without enabling
+	  this option.
+
 	  If you don't know what to do here, say N.
 
 config X86_MPPARSE
@@ -1919,7 +1928,7 @@ endchoice
 
 config X86_SGX
 	bool "Software Guard eXtensions (SGX)"
-	depends on X86_64 && CPU_SUP_INTEL
+	depends on X86_64 && CPU_SUP_INTEL && X86_X2APIC
 	depends on CRYPTO=y
 	depends on CRYPTO_SHA256=y
 	select SRCU
@@ -2569,7 +2578,7 @@ menuconfig APM
 
 	  1) make sure that you have enough swap space and that it is
 	  enabled.
-	  2) pass the "no-hlt" option to the kernel
+	  2) pass the "idle=poll" option to the kernel
 	  3) switch on floating point emulation in the kernel and pass
 	  the "no387" option to the kernel
 	  4) pass the "floppy=nodma" option to the kernel
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index bafbd905e6e7..2d7e640674c6 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -68,6 +68,7 @@ export BITS
 #    https://gcc.gnu.org/bugzilla/show_bug.cgi?id=53383
 #
 KBUILD_CFLAGS += -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx
+KBUILD_RUSTFLAGS += -Ctarget-feature=-sse,-sse2,-sse3,-ssse3,-sse4.1,-sse4.2,-avx,-avx2
 
 ifeq ($(CONFIG_X86_KERNEL_IBT),y)
 #
@@ -155,8 +156,17 @@ else
         cflags-$(CONFIG_GENERIC_CPU)	+= -mtune=generic
         KBUILD_CFLAGS += $(cflags-y)
 
+        rustflags-$(CONFIG_MK8)		+= -Ctarget-cpu=k8
+        rustflags-$(CONFIG_MPSC)	+= -Ctarget-cpu=nocona
+        rustflags-$(CONFIG_MCORE2)	+= -Ctarget-cpu=core2
+        rustflags-$(CONFIG_MATOM)	+= -Ctarget-cpu=atom
+        rustflags-$(CONFIG_GENERIC_CPU)	+= -Ztune-cpu=generic
+        KBUILD_RUSTFLAGS += $(rustflags-y)
+
         KBUILD_CFLAGS += -mno-red-zone
         KBUILD_CFLAGS += -mcmodel=kernel
+        KBUILD_RUSTFLAGS += -Cno-redzone=y
+        KBUILD_RUSTFLAGS += -Ccode-model=kernel
 endif
 
 #
diff --git a/arch/x86/boot/bitops.h b/arch/x86/boot/bitops.h
index 02e1dea11d94..8518ae214c9b 100644
--- a/arch/x86/boot/bitops.h
+++ b/arch/x86/boot/bitops.h
@@ -19,13 +19,13 @@
 
 static inline bool constant_test_bit(int nr, const void *addr)
 {
-	const u32 *p = (const u32 *)addr;
+	const u32 *p = addr;
 	return ((1UL << (nr & 31)) & (p[nr >> 5])) != 0;
 }
 static inline bool variable_test_bit(int nr, const void *addr)
 {
 	bool v;
-	const u32 *p = (const u32 *)addr;
+	const u32 *p = addr;
 
 	asm("btl %2,%1" CC_SET(c) : CC_OUT(c) (v) : "m" (*p), "Ir" (nr));
 	return v;
diff --git a/arch/x86/crypto/blowfish-x86_64-asm_64.S b/arch/x86/crypto/blowfish-x86_64-asm_64.S
index 802d71582689..4a43e072d2d1 100644
--- a/arch/x86/crypto/blowfish-x86_64-asm_64.S
+++ b/arch/x86/crypto/blowfish-x86_64-asm_64.S
@@ -6,6 +6,7 @@
  */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 
 .file "blowfish-x86_64-asm.S"
 .text
@@ -141,7 +142,7 @@ SYM_FUNC_START(__blowfish_enc_blk)
 	RET;
 SYM_FUNC_END(__blowfish_enc_blk)
 
-SYM_FUNC_START(blowfish_dec_blk)
+SYM_TYPED_FUNC_START(blowfish_dec_blk)
 	/* input:
 	 *	%rdi: ctx
 	 *	%rsi: dst
@@ -332,7 +333,7 @@ SYM_FUNC_START(__blowfish_enc_blk_4way)
 	RET;
 SYM_FUNC_END(__blowfish_enc_blk_4way)
 
-SYM_FUNC_START(blowfish_dec_blk_4way)
+SYM_TYPED_FUNC_START(blowfish_dec_blk_4way)
 	/* input:
 	 *	%rdi: ctx
 	 *	%rsi: dst
diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile
index 12f6c4d714cd..381d3333b996 100644
--- a/arch/x86/entry/vdso/Makefile
+++ b/arch/x86/entry/vdso/Makefile
@@ -91,7 +91,7 @@ ifneq ($(RETPOLINE_VDSO_CFLAGS),)
 endif
 endif
 
-$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
+$(vobjs): KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO) $(CC_FLAGS_CFI) $(RANDSTRUCT_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS)) $(CFL)
 $(vobjs): KBUILD_AFLAGS += -DBUILD_VDSO
 
 #
@@ -153,6 +153,7 @@ KBUILD_CFLAGS_32 := $(filter-out $(RANDSTRUCT_CFLAGS),$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out $(RETPOLINE_CFLAGS),$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_LTO),$(KBUILD_CFLAGS_32))
+KBUILD_CFLAGS_32 := $(filter-out $(CC_FLAGS_CFI),$(KBUILD_CFLAGS_32))
 KBUILD_CFLAGS_32 += -m32 -msoft-float -mregparm=0 -fpic
 KBUILD_CFLAGS_32 += -fno-stack-protector
 KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls)
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index c601939a74b1..c20d8cd47c48 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -2102,6 +2102,15 @@ static struct extra_reg intel_tnt_extra_regs[] __read_mostly = {
 	EVENT_EXTRA_END
 };
 
+EVENT_ATTR_STR(mem-loads,	mem_ld_grt,	"event=0xd0,umask=0x5,ldlat=3");
+EVENT_ATTR_STR(mem-stores,	mem_st_grt,	"event=0xd0,umask=0x6");
+
+static struct attribute *grt_mem_attrs[] = {
+	EVENT_PTR(mem_ld_grt),
+	EVENT_PTR(mem_st_grt),
+	NULL
+};
+
 static struct extra_reg intel_grt_extra_regs[] __read_mostly = {
 	/* must define OFFCORE_RSP_X first, see intel_fixup_er() */
 	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0),
@@ -5975,6 +5984,36 @@ __init int intel_pmu_init(void)
 		name = "Tremont";
 		break;
 
+	case INTEL_FAM6_ALDERLAKE_N:
+		x86_pmu.mid_ack = true;
+		memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,
+		       sizeof(hw_cache_event_ids));
+		memcpy(hw_cache_extra_regs, tnt_hw_cache_extra_regs,
+		       sizeof(hw_cache_extra_regs));
+		hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
+
+		x86_pmu.event_constraints = intel_slm_event_constraints;
+		x86_pmu.pebs_constraints = intel_grt_pebs_event_constraints;
+		x86_pmu.extra_regs = intel_grt_extra_regs;
+
+		x86_pmu.pebs_aliases = NULL;
+		x86_pmu.pebs_prec_dist = true;
+		x86_pmu.pebs_block = true;
+		x86_pmu.lbr_pt_coexist = true;
+		x86_pmu.flags |= PMU_FL_HAS_RSP_1;
+		x86_pmu.flags |= PMU_FL_INSTR_LATENCY;
+
+		intel_pmu_pebs_data_source_grt();
+		x86_pmu.pebs_latency_data = adl_latency_data_small;
+		x86_pmu.get_event_constraints = tnt_get_event_constraints;
+		x86_pmu.limit_period = spr_limit_period;
+		td_attr = tnt_events_attrs;
+		mem_attr = grt_mem_attrs;
+		extra_attr = nhm_format_attr;
+		pr_cont("Gracemont events, ");
+		name = "gracemont";
+		break;
+
 	case INTEL_FAM6_WESTMERE:
 	case INTEL_FAM6_WESTMERE_EP:
 	case INTEL_FAM6_WESTMERE_EX:
@@ -6317,7 +6356,6 @@ __init int intel_pmu_init(void)
 
 	case INTEL_FAM6_ALDERLAKE:
 	case INTEL_FAM6_ALDERLAKE_L:
-	case INTEL_FAM6_ALDERLAKE_N:
 	case INTEL_FAM6_RAPTORLAKE:
 	case INTEL_FAM6_RAPTORLAKE_P:
 		/*
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index de1f55d51784..ac973c6f82ad 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -110,13 +110,18 @@ void __init intel_pmu_pebs_data_source_skl(bool pmem)
 	__intel_pmu_pebs_data_source_skl(pmem, pebs_data_source);
 }
 
-static void __init intel_pmu_pebs_data_source_grt(u64 *data_source)
+static void __init __intel_pmu_pebs_data_source_grt(u64 *data_source)
 {
 	data_source[0x05] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HIT);
 	data_source[0x06] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOP, HITM);
 	data_source[0x08] = OP_LH | P(LVL, L3) | LEVEL(L3) | P(SNOOPX, FWD);
 }
 
+void __init intel_pmu_pebs_data_source_grt(void)
+{
+	__intel_pmu_pebs_data_source_grt(pebs_data_source);
+}
+
 void __init intel_pmu_pebs_data_source_adl(void)
 {
 	u64 *data_source;
@@ -127,7 +132,7 @@ void __init intel_pmu_pebs_data_source_adl(void)
 
 	data_source = x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX].pebs_data_source;
 	memcpy(data_source, pebs_data_source, sizeof(pebs_data_source));
-	intel_pmu_pebs_data_source_grt(data_source);
+	__intel_pmu_pebs_data_source_grt(data_source);
 }
 
 static u64 precise_store_data(u64 status)
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index ba3d24a6a4ec..266143abcbd8 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -1516,6 +1516,8 @@ void intel_pmu_pebs_data_source_skl(bool pmem);
 
 void intel_pmu_pebs_data_source_adl(void);
 
+void intel_pmu_pebs_data_source_grt(void);
+
 int intel_pmu_setup_lbr_filter(struct perf_event *event);
 
 void intel_pt_interrupt(void);
diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h
index e003a01b7c67..1dd14381bcb6 100644
--- a/arch/x86/include/asm/acrn.h
+++ b/arch/x86/include/asm/acrn.h
@@ -10,6 +10,15 @@
 /* Bit 0 indicates whether guest VM is privileged */
 #define	ACRN_FEATURE_PRIVILEGED_VM	BIT(0)
 
+/*
+ * Timing Information.
+ * This leaf returns the current TSC frequency in kHz.
+ *
+ * EAX: (Virtual) TSC frequency in kHz.
+ * EBX, ECX, EDX: RESERVED (reserved fields are set to zero).
+ */
+#define ACRN_CPUID_TIMING_INFO		0x40000010
+
 void acrn_setup_intr_handler(void (*handler)(void));
 void acrn_remove_intr_handler(void);
 
@@ -21,6 +30,11 @@ static inline u32 acrn_cpuid_base(void)
 	return 0;
 }
 
+static inline unsigned long acrn_get_tsc_khz(void)
+{
+	return cpuid_eax(ACRN_CPUID_TIMING_INFO);
+}
+
 /*
  * Hypercalls for ACRN
  *
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index 0fe9de58af31..2edf68475fec 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -247,17 +247,30 @@ arch_test_bit_acquire(unsigned long nr, const volatile unsigned long *addr)
 					  variable_test_bit(nr, addr);
 }
 
+static __always_inline unsigned long variable__ffs(unsigned long word)
+{
+	asm("rep; bsf %1,%0"
+		: "=r" (word)
+		: "rm" (word));
+	return word;
+}
+
 /**
  * __ffs - find first set bit in word
  * @word: The word to search
  *
  * Undefined if no bit exists, so code should check against 0 first.
  */
-static __always_inline unsigned long __ffs(unsigned long word)
+#define __ffs(word)				\
+	(__builtin_constant_p(word) ?		\
+	 (unsigned long)__builtin_ctzl(word) :	\
+	 variable__ffs(word))
+
+static __always_inline unsigned long variable_ffz(unsigned long word)
 {
 	asm("rep; bsf %1,%0"
 		: "=r" (word)
-		: "rm" (word));
+		: "r" (~word));
 	return word;
 }
 
@@ -267,13 +280,10 @@ static __always_inline unsigned long __ffs(unsigned long word)
  *
  * Undefined if no zero exists, so code should check against ~0UL first.
  */
-static __always_inline unsigned long ffz(unsigned long word)
-{
-	asm("rep; bsf %1,%0"
-		: "=r" (word)
-		: "r" (~word));
-	return word;
-}
+#define ffz(word)				\
+	(__builtin_constant_p(word) ?		\
+	 (unsigned long)__builtin_ctzl(~word) :	\
+	 variable_ffz(word))
 
 /*
  * __fls: find last set bit in word
@@ -292,18 +302,7 @@ static __always_inline unsigned long __fls(unsigned long word)
 #undef ADDR
 
 #ifdef __KERNEL__
-/**
- * ffs - find first set bit in word
- * @x: the word to search
- *
- * This is defined the same way as the libc and compiler builtin ffs
- * routines, therefore differs in spirit from the other bitops.
- *
- * ffs(value) returns 0 if value is 0 or the position of the first
- * set bit if value is nonzero. The first (least significant) bit
- * is at position 1.
- */
-static __always_inline int ffs(int x)
+static __always_inline int variable_ffs(int x)
 {
 	int r;
 
@@ -334,6 +333,19 @@ static __always_inline int ffs(int x)
 }
 
 /**
+ * ffs - find first set bit in word
+ * @x: the word to search
+ *
+ * This is defined the same way as the libc and compiler builtin ffs
+ * routines, therefore differs in spirit from the other bitops.
+ *
+ * ffs(value) returns 0 if value is 0 or the position of the first
+ * set bit if value is nonzero. The first (least significant) bit
+ * is at position 1.
+ */
+#define ffs(x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x))
+
+/**
  * fls - find last set bit in word
  * @x: the word to search
  *
diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h
new file mode 100644
index 000000000000..58dacd90daef
--- /dev/null
+++ b/arch/x86/include/asm/cfi.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_X86_CFI_H
+#define _ASM_X86_CFI_H
+
+/*
+ * Clang Control Flow Integrity (CFI) support.
+ *
+ * Copyright (C) 2022 Google LLC
+ */
+
+#include <linux/cfi.h>
+
+#ifdef CONFIG_CFI_CLANG
+enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
+#else
+static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
+{
+	return BUG_TRAP_TYPE_NONE;
+}
+#endif /* CONFIG_CFI_CLANG */
+
+#endif /* _ASM_X86_CFI_H */
diff --git a/arch/x86/include/asm/cpu.h b/arch/x86/include/asm/cpu.h
index 8cbf623f0ecf..b472ef76826a 100644
--- a/arch/x86/include/asm/cpu.h
+++ b/arch/x86/include/asm/cpu.h
@@ -94,4 +94,6 @@ static inline bool intel_cpu_signatures_match(unsigned int s1, unsigned int p1,
 	return p1 & p2;
 }
 
+extern u64 x86_read_arch_cap_msr(void);
+
 #endif /* _ASM_X86_CPU_H */
diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h
index aeb38023a703..5d75fe229342 100644
--- a/arch/x86/include/asm/intel-family.h
+++ b/arch/x86/include/asm/intel-family.h
@@ -115,6 +115,9 @@
 #define INTEL_FAM6_RAPTORLAKE_P		0xBA
 #define INTEL_FAM6_RAPTORLAKE_S		0xBF
 
+#define INTEL_FAM6_METEORLAKE		0xAC
+#define INTEL_FAM6_METEORLAKE_L		0xAA
+
 /* "Small Core" Processors (Atom) */
 
 #define INTEL_FAM6_ATOM_BONNELL		0x1C /* Diamondville, Pineview */
diff --git a/arch/x86/include/asm/linkage.h b/arch/x86/include/asm/linkage.h
index 73ca20049835..f484d656d34e 100644
--- a/arch/x86/include/asm/linkage.h
+++ b/arch/x86/include/asm/linkage.h
@@ -43,6 +43,18 @@
 
 #endif /* __ASSEMBLY__ */
 
+#define __CFI_TYPE(name)					\
+	SYM_START(__cfi_##name, SYM_L_LOCAL, SYM_A_NONE)	\
+	.fill 11, 1, 0x90 ASM_NL				\
+	.byte 0xb8 ASM_NL					\
+	.long __kcfi_typeid_##name ASM_NL			\
+	SYM_FUNC_END(__cfi_##name)
+
+/* SYM_TYPED_FUNC_START -- use for indirectly called globals, w/ CFI type */
+#define SYM_TYPED_FUNC_START(name)				\
+	SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)	\
+	ENDBR
+
 /* SYM_FUNC_START -- use for global functions */
 #define SYM_FUNC_START(name)				\
 	SYM_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)	\
diff --git a/arch/x86/include/asm/mc146818rtc.h b/arch/x86/include/asm/mc146818rtc.h
index 97198001e567..6115bb3d5795 100644
--- a/arch/x86/include/asm/mc146818rtc.h
+++ b/arch/x86/include/asm/mc146818rtc.h
@@ -95,7 +95,7 @@ static inline unsigned char current_lock_cmos_reg(void)
 unsigned char rtc_cmos_read(unsigned char addr);
 void rtc_cmos_write(unsigned char val, unsigned char addr);
 
-extern int mach_set_rtc_mmss(const struct timespec64 *now);
+extern int mach_set_cmos_time(const struct timespec64 *now);
 extern void mach_get_cmos_time(struct timespec64 *now);
 
 #define RTC_IRQ 8
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index cc73061e7255..6e986088817d 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -42,6 +42,7 @@
 #define MCI_STATUS_CEC_SHIFT	38           /* Corrected Error Count */
 #define MCI_STATUS_CEC_MASK	GENMASK_ULL(52,38)
 #define MCI_STATUS_CEC(c)	(((c) & MCI_STATUS_CEC_MASK) >> MCI_STATUS_CEC_SHIFT)
+#define MCI_STATUS_MSCOD(m)	(((m) >> 16) & 0xffff)
 
 /* AMD-specific bits */
 #define MCI_STATUS_TCC		BIT_ULL(55)  /* Task context corrupt */
diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
index 0c3d3440fe27..74ecc2bd6cd0 100644
--- a/arch/x86/include/asm/microcode.h
+++ b/arch/x86/include/asm/microcode.h
@@ -9,6 +9,7 @@
 struct ucode_patch {
 	struct list_head plist;
 	void *data;		/* Intel uses only this one */
+	unsigned int size;
 	u32 patch_id;
 	u16 equiv_cpu;
 };
@@ -32,9 +33,6 @@ enum ucode_state {
 };
 
 struct microcode_ops {
-	enum ucode_state (*request_microcode_user) (int cpu,
-				const void __user *buf, size_t size);
-
 	enum ucode_state (*request_microcode_fw) (int cpu, struct device *,
 						  bool refresh_fw);
 
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 6674bdb096f3..1e086b37a307 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -155,6 +155,11 @@
 						 * Return Stack Buffer Predictions.
 						 */
 
+#define ARCH_CAP_XAPIC_DISABLE		BIT(21)	/*
+						 * IA32_XAPIC_DISABLE_STATUS MSR
+						 * supported
+						 */
+
 #define MSR_IA32_FLUSH_CMD		0x0000010b
 #define L1D_FLUSH			BIT(0)	/*
 						 * Writeback and invalidate the
@@ -1054,4 +1059,12 @@
 #define MSR_IA32_HW_FEEDBACK_PTR        0x17d0
 #define MSR_IA32_HW_FEEDBACK_CONFIG     0x17d1
 
+/* x2APIC locked status */
+#define MSR_IA32_XAPIC_DISABLE_STATUS	0xBD
+#define LEGACY_XAPIC_DISABLED		BIT(0) /*
+						* x2APIC mode is locked and
+						* disabling x2APIC will cause
+						* a #GP
+						*/
+
 #endif /* _ASM_X86_MSR_INDEX_H */
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 964442b99245..2a0b8dd4ec33 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -743,6 +743,7 @@ extern void default_banner(void);
 	 word 771b;				\
 	 .byte ptype;				\
 	 .byte 772b-771b;			\
+	 _ASM_ALIGN;				\
 	.popsection
 
 
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 89df6c6617f5..f3d601574730 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -294,6 +294,7 @@ extern struct paravirt_patch_template pv_ops;
 	"  .byte " type "\n"				\
 	"  .byte 772b-771b\n"				\
 	"  .short " clobber "\n"			\
+	_ASM_ALIGN "\n"					\
 	".popsection\n"
 
 /* Generate patchable code, with the default asm parameters. */
@@ -328,7 +329,7 @@ int paravirt_disable_iospace(void);
  * Unfortunately, this is a relatively slow operation for modern CPUs,
  * because it cannot necessarily determine what the destination
  * address is.  In this case, the address is a runtime constant, so at
- * the very least we can patch the call to e a simple direct call, or
+ * the very least we can patch the call to a simple direct call, or,
  * ideally, patch an inline implementation into the callsite.  (Direct
  * calls are essentially free, because the call and return addresses
  * are completely predictable.)
@@ -339,10 +340,10 @@ int paravirt_disable_iospace(void);
  * on the stack.  All caller-save registers (eax,edx,ecx) are expected
  * to be modified (either clobbered or used for return values).
  * X86_64, on the other hand, already specifies a register-based calling
- * conventions, returning at %rax, with parameters going on %rdi, %rsi,
+ * conventions, returning at %rax, with parameters going in %rdi, %rsi,
  * %rdx, and %rcx. Note that for this reason, x86_64 does not need any
  * special handling for dealing with 4 arguments, unlike i386.
- * However, x86_64 also have to clobber all caller saved registers, which
+ * However, x86_64 also has to clobber all caller saved registers, which
  * unfortunately, are quite a bit (r8 - r11)
  *
  * The call instruction itself is marked by placing its start address
@@ -360,22 +361,22 @@ int paravirt_disable_iospace(void);
  * There are 5 sets of PVOP_* macros for dealing with 0-4 arguments.
  * It could be extended to more arguments, but there would be little
  * to be gained from that.  For each number of arguments, there are
- * the two VCALL and CALL variants for void and non-void functions.
+ * two VCALL and CALL variants for void and non-void functions.
  *
  * When there is a return value, the invoker of the macro must specify
  * the return type.  The macro then uses sizeof() on that type to
- * determine whether its a 32 or 64 bit value, and places the return
+ * determine whether it's a 32 or 64 bit value and places the return
  * in the right register(s) (just %eax for 32-bit, and %edx:%eax for
- * 64-bit). For x86_64 machines, it just returns at %rax regardless of
+ * 64-bit). For x86_64 machines, it just returns in %rax regardless of
  * the return value size.
  *
- * 64-bit arguments are passed as a pair of adjacent 32-bit arguments
+ * 64-bit arguments are passed as a pair of adjacent 32-bit arguments;
  * i386 also passes 64-bit arguments as a pair of adjacent 32-bit arguments
  * in low,high order
  *
  * Small structures are passed and returned in registers.  The macro
  * calling convention can't directly deal with this, so the wrapper
- * functions must do this.
+ * functions must do it.
  *
  * These PVOP_* macros are only defined within this header.  This
  * means that all uses must be wrapped in inline functions.  This also
@@ -414,8 +415,17 @@ int paravirt_disable_iospace(void);
 				"=c" (__ecx)
 #define PVOP_CALL_CLOBBERS	PVOP_VCALL_CLOBBERS, "=a" (__eax)
 
-/* void functions are still allowed [re]ax for scratch */
+/*
+ * void functions are still allowed [re]ax for scratch.
+ *
+ * The ZERO_CALL_USED REGS feature may end up zeroing out callee-saved
+ * registers. Make sure we model this with the appropriate clobbers.
+ */
+#ifdef CONFIG_ZERO_CALL_USED_REGS
+#define PVOP_VCALLEE_CLOBBERS	"=a" (__eax), PVOP_VCALL_CLOBBERS
+#else
 #define PVOP_VCALLEE_CLOBBERS	"=a" (__eax)
+#endif
 #define PVOP_CALLEE_CLOBBERS	PVOP_VCALLEE_CLOBBERS
 
 #define EXTRA_CLOBBERS	 , "r8", "r9", "r10", "r11"
diff --git a/arch/x86/include/asm/resctrl.h b/arch/x86/include/asm/resctrl.h
index d60ed0668a59..d24b04ebf950 100644
--- a/arch/x86/include/asm/resctrl.h
+++ b/arch/x86/include/asm/resctrl.h
@@ -81,6 +81,15 @@ static void __resctrl_sched_in(void)
 	}
 }
 
+static inline unsigned int resctrl_arch_round_mon_val(unsigned int val)
+{
+	unsigned int scale = boot_cpu_data.x86_cache_occ_scale;
+
+	/* h/w works in units of "boot_cpu_data.x86_cache_occ_scale" */
+	val /= scale;
+	return val * scale;
+}
+
 static inline void resctrl_sched_in(void)
 {
 	if (static_branch_likely(&rdt_enable_key))
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 81a0211a372d..a73bced40e24 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -21,16 +21,6 @@ DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id);
 DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_l2c_id);
 DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
 
-static inline struct cpumask *cpu_llc_shared_mask(int cpu)
-{
-	return per_cpu(cpu_llc_shared_map, cpu);
-}
-
-static inline struct cpumask *cpu_l2c_shared_mask(int cpu)
-{
-	return per_cpu(cpu_l2c_shared_map, cpu);
-}
-
 DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid);
 DECLARE_EARLY_PER_CPU_READ_MOSTLY(u32, x86_cpu_to_acpiid);
 DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);
@@ -172,6 +162,16 @@ extern int safe_smp_processor_id(void);
 # define safe_smp_processor_id()	smp_processor_id()
 #endif
 
+static inline struct cpumask *cpu_llc_shared_mask(int cpu)
+{
+	return per_cpu(cpu_llc_shared_map, cpu);
+}
+
+static inline struct cpumask *cpu_l2c_shared_mask(int cpu)
+{
+	return per_cpu(cpu_l2c_shared_map, cpu);
+}
+
 #else /* !CONFIG_SMP */
 #define wbinvd_on_cpu(cpu)     wbinvd()
 static inline int wbinvd_on_all_cpus(void)
@@ -179,6 +179,11 @@ static inline int wbinvd_on_all_cpus(void)
 	wbinvd();
 	return 0;
 }
+
+static inline struct cpumask *cpu_llc_shared_mask(int cpu)
+{
+	return (struct cpumask *)cpumask_of(0);
+}
 #endif /* CONFIG_SMP */
 
 extern unsigned disabled_cpus;
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 913e593a3b45..1ec6a9ea2328 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -448,7 +448,7 @@ do {									\
 
 #ifdef CONFIG_X86_32
 /*
- * Unlike the normal CMPXCHG, hardcode ECX for both success/fail and error.
+ * Unlike the normal CMPXCHG, use output GPR for both success/fail and error.
  * There are only six GPRs available and four (EAX, EBX, ECX, and EDX) are
  * hardcoded by CMPXCHG8B, leaving only ESI and EDI.  If the compiler uses
  * both ESI and EDI for the memory operand, compilation will fail if the error
@@ -461,11 +461,12 @@ do {									\
 	__typeof__(*(_ptr)) __new = (_new);				\
 	asm volatile("\n"						\
 		     "1: " LOCK_PREFIX "cmpxchg8b %[ptr]\n"		\
-		     "mov $0, %%ecx\n\t"				\
-		     "setz %%cl\n"					\
+		     "mov $0, %[result]\n\t"				\
+		     "setz %b[result]\n"				\
 		     "2:\n"						\
-		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG, %%ecx) \
-		     : [result]"=c" (__result),				\
+		     _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_EFAULT_REG,	\
+					   %[result])			\
+		     : [result] "=q" (__result),			\
 		       "+A" (__old),					\
 		       [ptr] "+m" (*_ptr)				\
 		     : "b" ((u32)__new),				\
@@ -502,9 +503,6 @@ strncpy_from_user(char *dst, const char __user *src, long count);
 
 extern __must_check long strnlen_user(const char __user *str, long n);
 
-unsigned long __must_check clear_user(void __user *mem, unsigned long len);
-unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
-
 #ifdef CONFIG_ARCH_HAS_COPY_MC
 unsigned long __must_check
 copy_mc_to_kernel(void *to, const void *from, unsigned len);
@@ -526,6 +524,8 @@ extern struct movsl_mask {
 #define ARCH_HAS_NOCACHE_UACCESS 1
 
 #ifdef CONFIG_X86_32
+unsigned long __must_check clear_user(void __user *mem, unsigned long len);
+unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
 # include <asm/uaccess_32.h>
 #else
 # include <asm/uaccess_64.h>
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index 45697e04d771..d13d71af5cf6 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -79,4 +79,49 @@ __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size)
 	kasan_check_write(dst, size);
 	return __copy_user_flushcache(dst, src, size);
 }
+
+/*
+ * Zero Userspace.
+ */
+
+__must_check unsigned long
+clear_user_original(void __user *addr, unsigned long len);
+__must_check unsigned long
+clear_user_rep_good(void __user *addr, unsigned long len);
+__must_check unsigned long
+clear_user_erms(void __user *addr, unsigned long len);
+
+static __always_inline __must_check unsigned long __clear_user(void __user *addr, unsigned long size)
+{
+	might_fault();
+	stac();
+
+	/*
+	 * No memory constraint because it doesn't change any memory gcc
+	 * knows about.
+	 */
+	asm volatile(
+		"1:\n\t"
+		ALTERNATIVE_3("rep stosb",
+			      "call clear_user_erms",	  ALT_NOT(X86_FEATURE_FSRM),
+			      "call clear_user_rep_good", ALT_NOT(X86_FEATURE_ERMS),
+			      "call clear_user_original", ALT_NOT(X86_FEATURE_REP_GOOD))
+		"2:\n"
+	       _ASM_EXTABLE_UA(1b, 2b)
+	       : "+c" (size), "+D" (addr), ASM_CALL_CONSTRAINT
+	       : "a" (0)
+		/* rep_good clobbers %rdx */
+	       : "rdx");
+
+	clac();
+
+	return size;
+}
+
+static __always_inline unsigned long clear_user(void __user *to, unsigned long n)
+{
+	if (access_ok(to, n))
+		return __clear_user(to, n);
+	return n;
+}
 #endif /* _ASM_X86_UACCESS_64_H */
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index a20a5ebfacd7..1286a73ebdbc 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -139,6 +139,8 @@ obj-$(CONFIG_UNWINDER_GUESS)		+= unwind_guess.o
 
 obj-$(CONFIG_AMD_MEM_ENCRYPT)		+= sev.o
 
+obj-$(CONFIG_CFI_CLANG)			+= cfi.o
+
 ###
 # 64 bit specific files
 ifeq ($(CONFIG_X86_64),y)
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 62f6b8b7c4a5..5cadcea035e0 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -453,6 +453,15 @@ static int patch_retpoline(void *addr, struct insn *insn, u8 *bytes)
 		return ret;
 	i += ret;
 
+	/*
+	 * The compiler is supposed to EMIT an INT3 after every unconditional
+	 * JMP instruction due to AMD BTC. However, if the compiler is too old
+	 * or SLS isn't enabled, we still need an INT3 after indirect JMPs
+	 * even on Intel.
+	 */
+	if (op == JMP32_INSN_OPCODE && i < insn->length)
+		bytes[i++] = INT3_INSN_OPCODE;
+
 	for (; i < insn->length;)
 		bytes[i++] = BYTES_NOP1;
 
@@ -1319,22 +1328,23 @@ struct bp_patching_desc {
 	atomic_t refs;
 };
 
-static struct bp_patching_desc *bp_desc;
+static struct bp_patching_desc bp_desc;
 
 static __always_inline
-struct bp_patching_desc *try_get_desc(struct bp_patching_desc **descp)
+struct bp_patching_desc *try_get_desc(void)
 {
-	/* rcu_dereference */
-	struct bp_patching_desc *desc = __READ_ONCE(*descp);
+	struct bp_patching_desc *desc = &bp_desc;
 
-	if (!desc || !arch_atomic_inc_not_zero(&desc->refs))
+	if (!arch_atomic_inc_not_zero(&desc->refs))
 		return NULL;
 
 	return desc;
 }
 
-static __always_inline void put_desc(struct bp_patching_desc *desc)
+static __always_inline void put_desc(void)
 {
+	struct bp_patching_desc *desc = &bp_desc;
+
 	smp_mb__before_atomic();
 	arch_atomic_dec(&desc->refs);
 }
@@ -1367,15 +1377,15 @@ noinstr int poke_int3_handler(struct pt_regs *regs)
 
 	/*
 	 * Having observed our INT3 instruction, we now must observe
-	 * bp_desc:
+	 * bp_desc with non-zero refcount:
 	 *
-	 *	bp_desc = desc			INT3
+	 *	bp_desc.refs = 1		INT3
 	 *	WMB				RMB
-	 *	write INT3			if (desc)
+	 *	write INT3			if (bp_desc.refs != 0)
 	 */
 	smp_rmb();
 
-	desc = try_get_desc(&bp_desc);
+	desc = try_get_desc();
 	if (!desc)
 		return 0;
 
@@ -1429,7 +1439,7 @@ noinstr int poke_int3_handler(struct pt_regs *regs)
 	ret = 1;
 
 out_put:
-	put_desc(desc);
+	put_desc();
 	return ret;
 }
 
@@ -1460,18 +1470,20 @@ static int tp_vec_nr;
  */
 static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries)
 {
-	struct bp_patching_desc desc = {
-		.vec = tp,
-		.nr_entries = nr_entries,
-		.refs = ATOMIC_INIT(1),
-	};
 	unsigned char int3 = INT3_INSN_OPCODE;
 	unsigned int i;
 	int do_sync;
 
 	lockdep_assert_held(&text_mutex);
 
-	smp_store_release(&bp_desc, &desc); /* rcu_assign_pointer */
+	bp_desc.vec = tp;
+	bp_desc.nr_entries = nr_entries;
+
+	/*
+	 * Corresponds to the implicit memory barrier in try_get_desc() to
+	 * ensure reading a non-zero refcount provides up to date bp_desc data.
+	 */
+	atomic_set_release(&bp_desc.refs, 1);
 
 	/*
 	 * Corresponding read barrier in int3 notifier for making sure the
@@ -1559,12 +1571,10 @@ static void text_poke_bp_batch(struct text_poke_loc *tp, unsigned int nr_entries
 		text_poke_sync();
 
 	/*
-	 * Remove and synchronize_rcu(), except we have a very primitive
-	 * refcount based completion.
+	 * Remove and wait for refs to be zero.
 	 */
-	WRITE_ONCE(bp_desc, NULL); /* RCU_INIT_POINTER */
-	if (!atomic_dec_and_test(&desc.refs))
-		atomic_cond_read_acquire(&desc.refs, !VAL);
+	if (!atomic_dec_and_test(&bp_desc.refs))
+		atomic_cond_read_acquire(&bp_desc.refs, !VAL);
 }
 
 static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c
index 194d54eed537..19a0207e529f 100644
--- a/arch/x86/kernel/amd_gart_64.c
+++ b/arch/x86/kernel/amd_gart_64.c
@@ -53,7 +53,7 @@ static u32 *iommu_gatt_base;		/* Remapping table */
  * of only flushing when an mapping is reused. With it true the GART is
  * flushed for every mapping. Problem is that doing the lazy flush seems
  * to trigger bugs with some popular PCI cards, in particular 3ware (but
- * has been also also seen with Qlogic at least).
+ * has been also seen with Qlogic at least).
  */
 static int iommu_fullflush = 1;
 
diff --git a/arch/x86/kernel/aperture_64.c b/arch/x86/kernel/aperture_64.c
index 7a5630d904b2..4feaa670d578 100644
--- a/arch/x86/kernel/aperture_64.c
+++ b/arch/x86/kernel/aperture_64.c
@@ -36,7 +36,7 @@
 /*
  * Using 512M as goal, in case kexec will load kernel_big
  * that will do the on-position decompress, and could overlap with
- * with the gart aperture that is used.
+ * the gart aperture that is used.
  * Sequence:
  * kernel_small
  * ==> kexec (with kdump trigger path or gart still enabled)
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 6d303d1d276c..c6876d3ea4b1 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -61,6 +61,7 @@
 #include <asm/cpu_device_id.h>
 #include <asm/intel-family.h>
 #include <asm/irq_regs.h>
+#include <asm/cpu.h>
 
 unsigned int num_processors;
 
@@ -1751,11 +1752,26 @@ EXPORT_SYMBOL_GPL(x2apic_mode);
 
 enum {
 	X2APIC_OFF,
-	X2APIC_ON,
 	X2APIC_DISABLED,
+	/* All states below here have X2APIC enabled */
+	X2APIC_ON,
+	X2APIC_ON_LOCKED
 };
 static int x2apic_state;
 
+static bool x2apic_hw_locked(void)
+{
+	u64 ia32_cap;
+	u64 msr;
+
+	ia32_cap = x86_read_arch_cap_msr();
+	if (ia32_cap & ARCH_CAP_XAPIC_DISABLE) {
+		rdmsrl(MSR_IA32_XAPIC_DISABLE_STATUS, msr);
+		return (msr & LEGACY_XAPIC_DISABLED);
+	}
+	return false;
+}
+
 static void __x2apic_disable(void)
 {
 	u64 msr;
@@ -1793,6 +1809,10 @@ static int __init setup_nox2apic(char *str)
 				apicid);
 			return 0;
 		}
+		if (x2apic_hw_locked()) {
+			pr_warn("APIC locked in x2apic mode, can't disable\n");
+			return 0;
+		}
 		pr_warn("x2apic already enabled.\n");
 		__x2apic_disable();
 	}
@@ -1807,10 +1827,18 @@ early_param("nox2apic", setup_nox2apic);
 void x2apic_setup(void)
 {
 	/*
-	 * If x2apic is not in ON state, disable it if already enabled
+	 * Try to make the AP's APIC state match that of the BSP,  but if the
+	 * BSP is unlocked and the AP is locked then there is a state mismatch.
+	 * Warn about the mismatch in case a GP fault occurs due to a locked AP
+	 * trying to be turned off.
+	 */
+	if (x2apic_state != X2APIC_ON_LOCKED && x2apic_hw_locked())
+		pr_warn("x2apic lock mismatch between BSP and AP.\n");
+	/*
+	 * If x2apic is not in ON or LOCKED state, disable it if already enabled
 	 * from BIOS.
 	 */
-	if (x2apic_state != X2APIC_ON) {
+	if (x2apic_state < X2APIC_ON) {
 		__x2apic_disable();
 		return;
 	}
@@ -1831,6 +1859,11 @@ static __init void x2apic_disable(void)
 	if (x2apic_id >= 255)
 		panic("Cannot disable x2apic, id: %08x\n", x2apic_id);
 
+	if (x2apic_hw_locked()) {
+		pr_warn("Cannot disable locked x2apic, id: %08x\n", x2apic_id);
+		return;
+	}
+
 	__x2apic_disable();
 	register_lapic_address(mp_lapic_addr);
 }
@@ -1889,7 +1922,10 @@ void __init check_x2apic(void)
 	if (x2apic_enabled()) {
 		pr_info("x2apic: enabled by BIOS, switching to x2apic ops\n");
 		x2apic_mode = 1;
-		x2apic_state = X2APIC_ON;
+		if (x2apic_hw_locked())
+			x2apic_state = X2APIC_ON_LOCKED;
+		else
+			x2apic_state = X2APIC_ON;
 	} else if (!boot_cpu_has(X86_FEATURE_X2APIC)) {
 		x2apic_state = X2APIC_DISABLED;
 	}
diff --git a/arch/x86/kernel/cfi.c b/arch/x86/kernel/cfi.c
new file mode 100644
index 000000000000..8674a5c0c031
--- /dev/null
+++ b/arch/x86/kernel/cfi.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Clang Control Flow Integrity (CFI) support.
+ *
+ * Copyright (C) 2022 Google LLC
+ */
+#include <asm/cfi.h>
+#include <asm/insn.h>
+#include <asm/insn-eval.h>
+#include <linux/string.h>
+
+/*
+ * Returns the target address and the expected type when regs->ip points
+ * to a compiler-generated CFI trap.
+ */
+static bool decode_cfi_insn(struct pt_regs *regs, unsigned long *target,
+			    u32 *type)
+{
+	char buffer[MAX_INSN_SIZE];
+	struct insn insn;
+	int offset = 0;
+
+	*target = *type = 0;
+
+	/*
+	 * The compiler generates the following instruction sequence
+	 * for indirect call checks:
+	 *
+	 *   movl    -<id>, %r10d       ; 6 bytes
+	 *   addl    -4(%reg), %r10d    ; 4 bytes
+	 *   je      .Ltmp1             ; 2 bytes
+	 *   ud2                        ; <- regs->ip
+	 *   .Ltmp1:
+	 *
+	 * We can decode the expected type and the target address from the
+	 * movl/addl instructions.
+	 */
+	if (copy_from_kernel_nofault(buffer, (void *)regs->ip - 12, MAX_INSN_SIZE))
+		return false;
+	if (insn_decode_kernel(&insn, &buffer[offset]))
+		return false;
+	if (insn.opcode.value != 0xBA)
+		return false;
+
+	*type = -(u32)insn.immediate.value;
+
+	if (copy_from_kernel_nofault(buffer, (void *)regs->ip - 6, MAX_INSN_SIZE))
+		return false;
+	if (insn_decode_kernel(&insn, &buffer[offset]))
+		return false;
+	if (insn.opcode.value != 0x3)
+		return false;
+
+	/* Read the target address from the register. */
+	offset = insn_get_modrm_rm_off(&insn, regs);
+	if (offset < 0)
+		return false;
+
+	*target = *(unsigned long *)((void *)regs + offset);
+
+	return true;
+}
+
+/*
+ * Checks if a ud2 trap is because of a CFI failure, and handles the trap
+ * if needed. Returns a bug_trap_type value similarly to report_bug.
+ */
+enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
+{
+	unsigned long target;
+	u32 type;
+
+	if (!is_cfi_trap(regs->ip))
+		return BUG_TRAP_TYPE_NONE;
+
+	if (!decode_cfi_insn(regs, &target, &type))
+		return report_cfi_failure_noaddr(regs, regs->ip);
+
+	return report_cfi_failure(regs, regs->ip, &target, type);
+}
+
+/*
+ * Ensure that __kcfi_typeid_ symbols are emitted for functions that may
+ * not be indirectly called with all configurations.
+ */
+__ADDRESSABLE(__memcpy)
diff --git a/arch/x86/kernel/cpu/acrn.c b/arch/x86/kernel/cpu/acrn.c
index 23f5f27b5a02..485441b7f030 100644
--- a/arch/x86/kernel/cpu/acrn.c
+++ b/arch/x86/kernel/cpu/acrn.c
@@ -28,6 +28,9 @@ static void __init acrn_init_platform(void)
 {
 	/* Setup the IDT for ACRN hypervisor callback */
 	alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_acrn_hv_callback);
+
+	x86_platform.calibrate_tsc = acrn_get_tsc_khz;
+	x86_platform.calibrate_cpu = acrn_get_tsc_khz;
 }
 
 static bool acrn_x2apic_available(void)
diff --git a/arch/x86/kernel/cpu/feat_ctl.c b/arch/x86/kernel/cpu/feat_ctl.c
index 993697e71854..03851240c3e3 100644
--- a/arch/x86/kernel/cpu/feat_ctl.c
+++ b/arch/x86/kernel/cpu/feat_ctl.c
@@ -1,11 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/tboot.h>
 
+#include <asm/cpu.h>
 #include <asm/cpufeature.h>
 #include <asm/msr-index.h>
 #include <asm/processor.h>
 #include <asm/vmx.h>
-#include "cpu.h"
 
 #undef pr_fmt
 #define pr_fmt(fmt)	"x86/cpu: " fmt
diff --git a/arch/x86/kernel/cpu/mce/apei.c b/arch/x86/kernel/cpu/mce/apei.c
index 717192915f28..8ed341714686 100644
--- a/arch/x86/kernel/cpu/mce/apei.c
+++ b/arch/x86/kernel/cpu/mce/apei.c
@@ -29,15 +29,26 @@
 void apei_mce_report_mem_error(int severity, struct cper_sec_mem_err *mem_err)
 {
 	struct mce m;
+	int lsb;
 
 	if (!(mem_err->validation_bits & CPER_MEM_VALID_PA))
 		return;
 
+	/*
+	 * Even if the ->validation_bits are set for address mask,
+	 * to be extra safe, check and reject an error radius '0',
+	 * and fall back to the default page size.
+	 */
+	if (mem_err->validation_bits & CPER_MEM_VALID_PA_MASK)
+		lsb = find_first_bit((void *)&mem_err->physical_addr_mask, PAGE_SHIFT);
+	else
+		lsb = PAGE_SHIFT;
+
 	mce_setup(&m);
 	m.bank = -1;
 	/* Fake a memory read error with unknown channel */
 	m.status = MCI_STATUS_VAL | MCI_STATUS_EN | MCI_STATUS_ADDRV | MCI_STATUS_MISCV | 0x9f;
-	m.misc = (MCI_MISC_ADDR_PHYS << 6) | PAGE_SHIFT;
+	m.misc = (MCI_MISC_ADDR_PHYS << 6) | lsb;
 
 	if (severity >= GHES_SEV_RECOVERABLE)
 		m.status |= MCI_STATUS_UC;
diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c
index 8b2fcdfa6d31..e7410e98fc1f 100644
--- a/arch/x86/kernel/cpu/microcode/amd.c
+++ b/arch/x86/kernel/cpu/microcode/amd.c
@@ -788,6 +788,7 @@ static int verify_and_add_patch(u8 family, u8 *fw, unsigned int leftover,
 		kfree(patch);
 		return -EINVAL;
 	}
+	patch->size = *patch_size;
 
 	mc_hdr      = (struct microcode_header_amd *)(fw + SECTION_HDR_SIZE);
 	proc_id     = mc_hdr->processor_rev_id;
@@ -869,7 +870,7 @@ load_microcode_amd(bool save, u8 family, const u8 *data, size_t size)
 		return ret;
 
 	memset(amd_ucode_patch, 0, PATCH_MAX_SIZE);
-	memcpy(amd_ucode_patch, p->data, min_t(u32, ksize(p->data), PATCH_MAX_SIZE));
+	memcpy(amd_ucode_patch, p->data, min_t(u32, p->size, PATCH_MAX_SIZE));
 
 	return ret;
 }
@@ -924,12 +925,6 @@ static enum ucode_state request_microcode_amd(int cpu, struct device *device,
 	return ret;
 }
 
-static enum ucode_state
-request_microcode_user(int cpu, const void __user *buf, size_t size)
-{
-	return UCODE_ERROR;
-}
-
 static void microcode_fini_cpu_amd(int cpu)
 {
 	struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
@@ -938,7 +933,6 @@ static void microcode_fini_cpu_amd(int cpu)
 }
 
 static struct microcode_ops microcode_amd_ops = {
-	.request_microcode_user           = request_microcode_user,
 	.request_microcode_fw             = request_microcode_amd,
 	.collect_cpu_info                 = collect_cpu_info_amd,
 	.apply_microcode                  = apply_microcode_amd,
diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c
index ad57e0e4d674..6a41cee242f6 100644
--- a/arch/x86/kernel/cpu/microcode/core.c
+++ b/arch/x86/kernel/cpu/microcode/core.c
@@ -491,7 +491,7 @@ wait_for_siblings:
  */
 static int microcode_reload_late(void)
 {
-	int ret;
+	int old = boot_cpu_data.microcode, ret;
 
 	pr_err("Attempting late microcode loading - it is dangerous and taints the kernel.\n");
 	pr_err("You should switch to early loading, if possible.\n");
@@ -503,7 +503,8 @@ static int microcode_reload_late(void)
 	if (ret == 0)
 		microcode_check();
 
-	pr_info("Reload completed, microcode revision: 0x%x\n", boot_cpu_data.microcode);
+	pr_info("Reload completed, microcode revision: 0x%x -> 0x%x\n",
+		old, boot_cpu_data.microcode);
 
 	return ret;
 }
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index 025c8f0cd948..1fcbd671f1df 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -916,24 +916,7 @@ static enum ucode_state request_microcode_fw(int cpu, struct device *device,
 	return ret;
 }
 
-static enum ucode_state
-request_microcode_user(int cpu, const void __user *buf, size_t size)
-{
-	struct iov_iter iter;
-	struct iovec iov;
-
-	if (is_blacklisted(cpu))
-		return UCODE_NFOUND;
-
-	iov.iov_base = (void __user *)buf;
-	iov.iov_len = size;
-	iov_iter_init(&iter, WRITE, &iov, 1, size);
-
-	return generic_load_microcode(cpu, &iter);
-}
-
 static struct microcode_ops microcode_intel_ops = {
-	.request_microcode_user		  = request_microcode_user,
 	.request_microcode_fw             = request_microcode_fw,
 	.collect_cpu_info                 = collect_cpu_info,
 	.apply_microcode                  = apply_microcode_intel,
diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
index bb1c3f5f60c8..de62b0b87ced 100644
--- a/arch/x86/kernel/cpu/resctrl/core.c
+++ b/arch/x86/kernel/cpu/resctrl/core.c
@@ -147,7 +147,6 @@ static inline void cache_alloc_hsw_probe(void)
 	r->cache.shareable_bits = 0xc0000;
 	r->cache.min_cbm_bits = 2;
 	r->alloc_capable = true;
-	r->alloc_enabled = true;
 
 	rdt_alloc_capable = true;
 }
@@ -211,7 +210,6 @@ static bool __get_mem_config_intel(struct rdt_resource *r)
 	thread_throttle_mode_init();
 
 	r->alloc_capable = true;
-	r->alloc_enabled = true;
 
 	return true;
 }
@@ -242,7 +240,6 @@ static bool __rdt_get_mem_config_amd(struct rdt_resource *r)
 	r->data_width = 4;
 
 	r->alloc_capable = true;
-	r->alloc_enabled = true;
 
 	return true;
 }
@@ -261,7 +258,6 @@ static void rdt_get_cache_alloc_cfg(int idx, struct rdt_resource *r)
 	r->cache.shareable_bits = ebx & r->default_ctrl;
 	r->data_width = (r->cache.cbm_len + 3) / 4;
 	r->alloc_capable = true;
-	r->alloc_enabled = true;
 }
 
 static void rdt_get_cdp_config(int level)
@@ -300,7 +296,7 @@ mba_wrmsr_amd(struct rdt_domain *d, struct msr_param *m, struct rdt_resource *r)
  * that can be written to QOS_MSRs.
  * There are currently no SKUs which support non linear delay values.
  */
-u32 delay_bw_map(unsigned long bw, struct rdt_resource *r)
+static u32 delay_bw_map(unsigned long bw, struct rdt_resource *r)
 {
 	if (r->membw.delay_linear)
 		return MAX_MBA_BW - bw;
@@ -401,7 +397,7 @@ struct rdt_domain *rdt_find_domain(struct rdt_resource *r, int id,
 	return NULL;
 }
 
-void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm)
+static void setup_default_ctrlval(struct rdt_resource *r, u32 *dc)
 {
 	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 	int i;
@@ -410,12 +406,17 @@ void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm)
 	 * Initialize the Control MSRs to having no control.
 	 * For Cache Allocation: Set all bits in cbm
 	 * For Memory Allocation: Set b/w requested to 100%
-	 * and the bandwidth in MBps to U32_MAX
 	 */
-	for (i = 0; i < hw_res->num_closid; i++, dc++, dm++) {
+	for (i = 0; i < hw_res->num_closid; i++, dc++)
 		*dc = r->default_ctrl;
-		*dm = MBA_MAX_MBPS;
-	}
+}
+
+static void domain_free(struct rdt_hw_domain *hw_dom)
+{
+	kfree(hw_dom->arch_mbm_total);
+	kfree(hw_dom->arch_mbm_local);
+	kfree(hw_dom->ctrl_val);
+	kfree(hw_dom);
 }
 
 static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d)
@@ -423,23 +424,15 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d)
 	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
 	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
 	struct msr_param m;
-	u32 *dc, *dm;
+	u32 *dc;
 
 	dc = kmalloc_array(hw_res->num_closid, sizeof(*hw_dom->ctrl_val),
 			   GFP_KERNEL);
 	if (!dc)
 		return -ENOMEM;
 
-	dm = kmalloc_array(hw_res->num_closid, sizeof(*hw_dom->mbps_val),
-			   GFP_KERNEL);
-	if (!dm) {
-		kfree(dc);
-		return -ENOMEM;
-	}
-
 	hw_dom->ctrl_val = dc;
-	hw_dom->mbps_val = dm;
-	setup_default_ctrlval(r, dc, dm);
+	setup_default_ctrlval(r, dc);
 
 	m.low = 0;
 	m.high = hw_res->num_closid;
@@ -447,39 +440,31 @@ static int domain_setup_ctrlval(struct rdt_resource *r, struct rdt_domain *d)
 	return 0;
 }
 
-static int domain_setup_mon_state(struct rdt_resource *r, struct rdt_domain *d)
+/**
+ * arch_domain_mbm_alloc() - Allocate arch private storage for the MBM counters
+ * @num_rmid:	The size of the MBM counter array
+ * @hw_dom:	The domain that owns the allocated arrays
+ */
+static int arch_domain_mbm_alloc(u32 num_rmid, struct rdt_hw_domain *hw_dom)
 {
 	size_t tsize;
 
-	if (is_llc_occupancy_enabled()) {
-		d->rmid_busy_llc = bitmap_zalloc(r->num_rmid, GFP_KERNEL);
-		if (!d->rmid_busy_llc)
-			return -ENOMEM;
-		INIT_DELAYED_WORK(&d->cqm_limbo, cqm_handle_limbo);
-	}
 	if (is_mbm_total_enabled()) {
-		tsize = sizeof(*d->mbm_total);
-		d->mbm_total = kcalloc(r->num_rmid, tsize, GFP_KERNEL);
-		if (!d->mbm_total) {
-			bitmap_free(d->rmid_busy_llc);
+		tsize = sizeof(*hw_dom->arch_mbm_total);
+		hw_dom->arch_mbm_total = kcalloc(num_rmid, tsize, GFP_KERNEL);
+		if (!hw_dom->arch_mbm_total)
 			return -ENOMEM;
-		}
 	}
 	if (is_mbm_local_enabled()) {
-		tsize = sizeof(*d->mbm_local);
-		d->mbm_local = kcalloc(r->num_rmid, tsize, GFP_KERNEL);
-		if (!d->mbm_local) {
-			bitmap_free(d->rmid_busy_llc);
-			kfree(d->mbm_total);
+		tsize = sizeof(*hw_dom->arch_mbm_local);
+		hw_dom->arch_mbm_local = kcalloc(num_rmid, tsize, GFP_KERNEL);
+		if (!hw_dom->arch_mbm_local) {
+			kfree(hw_dom->arch_mbm_total);
+			hw_dom->arch_mbm_total = NULL;
 			return -ENOMEM;
 		}
 	}
 
-	if (is_mbm_enabled()) {
-		INIT_DELAYED_WORK(&d->mbm_over, mbm_handle_overflow);
-		mbm_setup_overflow_handler(d, MBM_OVERFLOW_INTERVAL);
-	}
-
 	return 0;
 }
 
@@ -502,6 +487,7 @@ static void domain_add_cpu(int cpu, struct rdt_resource *r)
 	struct list_head *add_pos = NULL;
 	struct rdt_hw_domain *hw_dom;
 	struct rdt_domain *d;
+	int err;
 
 	d = rdt_find_domain(r, id, &add_pos);
 	if (IS_ERR(d)) {
@@ -527,25 +513,22 @@ static void domain_add_cpu(int cpu, struct rdt_resource *r)
 	rdt_domain_reconfigure_cdp(r);
 
 	if (r->alloc_capable && domain_setup_ctrlval(r, d)) {
-		kfree(hw_dom);
+		domain_free(hw_dom);
 		return;
 	}
 
-	if (r->mon_capable && domain_setup_mon_state(r, d)) {
-		kfree(hw_dom->ctrl_val);
-		kfree(hw_dom->mbps_val);
-		kfree(hw_dom);
+	if (r->mon_capable && arch_domain_mbm_alloc(r->num_rmid, hw_dom)) {
+		domain_free(hw_dom);
 		return;
 	}
 
 	list_add_tail(&d->list, add_pos);
 
-	/*
-	 * If resctrl is mounted, add
-	 * per domain monitor data directories.
-	 */
-	if (static_branch_unlikely(&rdt_mon_enable_key))
-		mkdir_mondata_subdir_allrdtgrp(r, d);
+	err = resctrl_online_domain(r, d);
+	if (err) {
+		list_del(&d->list);
+		domain_free(hw_dom);
+	}
 }
 
 static void domain_remove_cpu(int cpu, struct rdt_resource *r)
@@ -563,27 +546,8 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r)
 
 	cpumask_clear_cpu(cpu, &d->cpu_mask);
 	if (cpumask_empty(&d->cpu_mask)) {
-		/*
-		 * If resctrl is mounted, remove all the
-		 * per domain monitor data directories.
-		 */
-		if (static_branch_unlikely(&rdt_mon_enable_key))
-			rmdir_mondata_subdir_allrdtgrp(r, d->id);
+		resctrl_offline_domain(r, d);
 		list_del(&d->list);
-		if (r->mon_capable && is_mbm_enabled())
-			cancel_delayed_work(&d->mbm_over);
-		if (is_llc_occupancy_enabled() &&  has_busy_rmid(r, d)) {
-			/*
-			 * When a package is going down, forcefully
-			 * decrement rmid->ebusy. There is no way to know
-			 * that the L3 was flushed and hence may lead to
-			 * incorrect counts in rare scenarios, but leaving
-			 * the RMID as busy creates RMID leaks if the
-			 * package never comes back.
-			 */
-			__check_limbo(d, true);
-			cancel_delayed_work(&d->cqm_limbo);
-		}
 
 		/*
 		 * rdt_domain "d" is going to be freed below, so clear
@@ -591,13 +555,8 @@ static void domain_remove_cpu(int cpu, struct rdt_resource *r)
 		 */
 		if (d->plr)
 			d->plr->d = NULL;
+		domain_free(hw_dom);
 
-		kfree(hw_dom->ctrl_val);
-		kfree(hw_dom->mbps_val);
-		bitmap_free(d->rmid_busy_llc);
-		kfree(d->mbm_total);
-		kfree(d->mbm_local);
-		kfree(hw_dom);
 		return;
 	}
 
diff --git a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
index 87666275eed9..1dafbdc5ac31 100644
--- a/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
+++ b/arch/x86/kernel/cpu/resctrl/ctrlmondata.c
@@ -61,6 +61,7 @@ int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s,
 	     struct rdt_domain *d)
 {
 	struct resctrl_staged_config *cfg;
+	u32 closid = data->rdtgrp->closid;
 	struct rdt_resource *r = s->res;
 	unsigned long bw_val;
 
@@ -72,6 +73,12 @@ int parse_bw(struct rdt_parse_data *data, struct resctrl_schema *s,
 
 	if (!bw_validate(data->buf, &bw_val, r))
 		return -EINVAL;
+
+	if (is_mba_sc(r)) {
+		d->mbps_val[closid] = bw_val;
+		return 0;
+	}
+
 	cfg->new_ctrl = bw_val;
 	cfg->have_new_ctrl = true;
 
@@ -261,14 +268,13 @@ static u32 get_config_index(u32 closid, enum resctrl_conf_type type)
 
 static bool apply_config(struct rdt_hw_domain *hw_dom,
 			 struct resctrl_staged_config *cfg, u32 idx,
-			 cpumask_var_t cpu_mask, bool mba_sc)
+			 cpumask_var_t cpu_mask)
 {
 	struct rdt_domain *dom = &hw_dom->d_resctrl;
-	u32 *dc = !mba_sc ? hw_dom->ctrl_val : hw_dom->mbps_val;
 
-	if (cfg->new_ctrl != dc[idx]) {
+	if (cfg->new_ctrl != hw_dom->ctrl_val[idx]) {
 		cpumask_set_cpu(cpumask_any(&dom->cpu_mask), cpu_mask);
-		dc[idx] = cfg->new_ctrl;
+		hw_dom->ctrl_val[idx] = cfg->new_ctrl;
 
 		return true;
 	}
@@ -276,6 +282,27 @@ static bool apply_config(struct rdt_hw_domain *hw_dom,
 	return false;
 }
 
+int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d,
+			    u32 closid, enum resctrl_conf_type t, u32 cfg_val)
+{
+	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
+	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
+	u32 idx = get_config_index(closid, t);
+	struct msr_param msr_param;
+
+	if (!cpumask_test_cpu(smp_processor_id(), &d->cpu_mask))
+		return -EINVAL;
+
+	hw_dom->ctrl_val[idx] = cfg_val;
+
+	msr_param.res = r;
+	msr_param.low = idx;
+	msr_param.high = idx + 1;
+	hw_res->msr_update(d, &msr_param, r);
+
+	return 0;
+}
+
 int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)
 {
 	struct resctrl_staged_config *cfg;
@@ -284,14 +311,12 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)
 	enum resctrl_conf_type t;
 	cpumask_var_t cpu_mask;
 	struct rdt_domain *d;
-	bool mba_sc;
 	int cpu;
 	u32 idx;
 
 	if (!zalloc_cpumask_var(&cpu_mask, GFP_KERNEL))
 		return -ENOMEM;
 
-	mba_sc = is_mba_sc(r);
 	msr_param.res = NULL;
 	list_for_each_entry(d, &r->domains, list) {
 		hw_dom = resctrl_to_arch_dom(d);
@@ -301,7 +326,7 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)
 				continue;
 
 			idx = get_config_index(closid, t);
-			if (!apply_config(hw_dom, cfg, idx, cpu_mask, mba_sc))
+			if (!apply_config(hw_dom, cfg, idx, cpu_mask))
 				continue;
 
 			if (!msr_param.res) {
@@ -315,11 +340,7 @@ int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)
 		}
 	}
 
-	/*
-	 * Avoid writing the control msr with control values when
-	 * MBA software controller is enabled
-	 */
-	if (cpumask_empty(cpu_mask) || mba_sc)
+	if (cpumask_empty(cpu_mask))
 		goto done;
 	cpu = get_cpu();
 	/* Update resource control msr on this CPU if it's in cpu_mask. */
@@ -406,6 +427,14 @@ ssize_t rdtgroup_schemata_write(struct kernfs_open_file *of,
 
 	list_for_each_entry(s, &resctrl_schema_all, list) {
 		r = s->res;
+
+		/*
+		 * Writes to mba_sc resources update the software controller,
+		 * not the control MSR.
+		 */
+		if (is_mba_sc(r))
+			continue;
+
 		ret = resctrl_arch_update_domains(r, rdtgrp->closid);
 		if (ret)
 			goto out;
@@ -433,9 +462,7 @@ u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_domain *d,
 	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
 	u32 idx = get_config_index(closid, type);
 
-	if (!is_mba_sc(r))
-		return hw_dom->ctrl_val[idx];
-	return hw_dom->mbps_val[idx];
+	return hw_dom->ctrl_val[idx];
 }
 
 static void show_doms(struct seq_file *s, struct resctrl_schema *schema, int closid)
@@ -450,8 +477,12 @@ static void show_doms(struct seq_file *s, struct resctrl_schema *schema, int clo
 		if (sep)
 			seq_puts(s, ";");
 
-		ctrl_val = resctrl_arch_get_config(r, dom, closid,
-						   schema->conf_type);
+		if (is_mba_sc(r))
+			ctrl_val = dom->mbps_val[closid];
+		else
+			ctrl_val = resctrl_arch_get_config(r, dom, closid,
+							   schema->conf_type);
+
 		seq_printf(s, r->format_str, dom->id, max_data_width,
 			   ctrl_val);
 		sep = true;
@@ -518,7 +549,6 @@ void mon_event_read(struct rmid_read *rr, struct rdt_resource *r,
 int rdtgroup_mondata_show(struct seq_file *m, void *arg)
 {
 	struct kernfs_open_file *of = m->private;
-	struct rdt_hw_resource *hw_res;
 	u32 resid, evtid, domid;
 	struct rdtgroup *rdtgrp;
 	struct rdt_resource *r;
@@ -538,8 +568,7 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg)
 	domid = md.u.domid;
 	evtid = md.u.evtid;
 
-	hw_res = &rdt_resources_all[resid];
-	r = &hw_res->r_resctrl;
+	r = &rdt_resources_all[resid].r_resctrl;
 	d = rdt_find_domain(r, domid, NULL);
 	if (IS_ERR_OR_NULL(d)) {
 		ret = -ENOENT;
@@ -548,12 +577,12 @@ int rdtgroup_mondata_show(struct seq_file *m, void *arg)
 
 	mon_event_read(&rr, r, d, rdtgrp, evtid, false);
 
-	if (rr.val & RMID_VAL_ERROR)
+	if (rr.err == -EIO)
 		seq_puts(m, "Error\n");
-	else if (rr.val & RMID_VAL_UNAVAIL)
+	else if (rr.err == -EINVAL)
 		seq_puts(m, "Unavailable\n");
 	else
-		seq_printf(m, "%llu\n", rr.val * hw_res->mon_scale);
+		seq_printf(m, "%llu\n", rr.val);
 
 out:
 	rdtgroup_kn_unlock(of->kn);
diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h
index 1d647188a43b..5f7128686cfd 100644
--- a/arch/x86/kernel/cpu/resctrl/internal.h
+++ b/arch/x86/kernel/cpu/resctrl/internal.h
@@ -22,21 +22,12 @@
 
 #define L2_QOS_CDP_ENABLE		0x01ULL
 
-/*
- * Event IDs are used to program IA32_QM_EVTSEL before reading event
- * counter from IA32_QM_CTR
- */
-#define QOS_L3_OCCUP_EVENT_ID		0x01
-#define QOS_L3_MBM_TOTAL_EVENT_ID	0x02
-#define QOS_L3_MBM_LOCAL_EVENT_ID	0x03
-
 #define CQM_LIMBOCHECK_INTERVAL	1000
 
 #define MBM_CNTR_WIDTH_BASE		24
 #define MBM_OVERFLOW_INTERVAL		1000
 #define MAX_MBA_BW			100u
 #define MBA_IS_LINEAR			0x4
-#define MBA_MAX_MBPS			U32_MAX
 #define MAX_MBA_BW_AMD			0x800
 #define MBM_CNTR_WIDTH_OFFSET_AMD	20
 
@@ -74,7 +65,7 @@ DECLARE_STATIC_KEY_FALSE(rdt_mon_enable_key);
  * @list:		entry in &rdt_resource->evt_list
  */
 struct mon_evt {
-	u32			evtid;
+	enum resctrl_event_id	evtid;
 	char			*name;
 	struct list_head	list;
 };
@@ -91,9 +82,9 @@ struct mon_evt {
 union mon_data_bits {
 	void *priv;
 	struct {
-		unsigned int rid	: 10;
-		unsigned int evtid	: 8;
-		unsigned int domid	: 14;
+		unsigned int rid		: 10;
+		enum resctrl_event_id evtid	: 8;
+		unsigned int domid		: 14;
 	} u;
 };
 
@@ -101,12 +92,12 @@ struct rmid_read {
 	struct rdtgroup		*rgrp;
 	struct rdt_resource	*r;
 	struct rdt_domain	*d;
-	int			evtid;
+	enum resctrl_event_id	evtid;
 	bool			first;
+	int			err;
 	u64			val;
 };
 
-extern unsigned int resctrl_cqm_threshold;
 extern bool rdt_alloc_capable;
 extern bool rdt_mon_capable;
 extern unsigned int rdt_mon_features;
@@ -288,35 +279,45 @@ struct rftype {
 
 /**
  * struct mbm_state - status for each MBM counter in each domain
- * @chunks:	Total data moved (multiply by rdt_group.mon_scale to get bytes)
- * @prev_msr:	Value of IA32_QM_CTR for this RMID last time we read it
- * @prev_bw_msr:Value of previous IA32_QM_CTR for bandwidth counting
+ * @prev_bw_bytes: Previous bytes value read for bandwidth calculation
  * @prev_bw:	The most recent bandwidth in MBps
  * @delta_bw:	Difference between the current and previous bandwidth
  * @delta_comp:	Indicates whether to compute the delta_bw
  */
 struct mbm_state {
-	u64	chunks;
-	u64	prev_msr;
-	u64	prev_bw_msr;
+	u64	prev_bw_bytes;
 	u32	prev_bw;
 	u32	delta_bw;
 	bool	delta_comp;
 };
 
 /**
+ * struct arch_mbm_state - values used to compute resctrl_arch_rmid_read()s
+ *			   return value.
+ * @chunks:	Total data moved (multiply by rdt_group.mon_scale to get bytes)
+ * @prev_msr:	Value of IA32_QM_CTR last time it was read for the RMID used to
+ *		find this struct.
+ */
+struct arch_mbm_state {
+	u64	chunks;
+	u64	prev_msr;
+};
+
+/**
  * struct rdt_hw_domain - Arch private attributes of a set of CPUs that share
  *			  a resource
  * @d_resctrl:	Properties exposed to the resctrl file system
  * @ctrl_val:	array of cache or mem ctrl values (indexed by CLOSID)
- * @mbps_val:	When mba_sc is enabled, this holds the bandwidth in MBps
+ * @arch_mbm_total:	arch private state for MBM total bandwidth
+ * @arch_mbm_local:	arch private state for MBM local bandwidth
  *
  * Members of this structure are accessed via helpers that provide abstraction.
  */
 struct rdt_hw_domain {
 	struct rdt_domain		d_resctrl;
 	u32				*ctrl_val;
-	u32				*mbps_val;
+	struct arch_mbm_state		*arch_mbm_total;
+	struct arch_mbm_state		*arch_mbm_local;
 };
 
 static inline struct rdt_hw_domain *resctrl_to_arch_dom(struct rdt_domain *r)
@@ -459,14 +460,6 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_level l, bool enable);
 	for_each_rdt_resource(r)					      \
 		if (r->mon_capable)
 
-#define for_each_alloc_enabled_rdt_resource(r)				      \
-	for_each_rdt_resource(r)					      \
-		if (r->alloc_enabled)
-
-#define for_each_mon_enabled_rdt_resource(r)				      \
-	for_each_rdt_resource(r)					      \
-		if (r->mon_enabled)
-
 /* CPUID.(EAX=10H, ECX=ResID=1).EAX */
 union cpuid_0x10_1_eax {
 	struct {
@@ -530,10 +523,6 @@ void free_rmid(u32 rmid);
 int rdt_get_mon_l3_config(struct rdt_resource *r);
 void mon_event_count(void *info);
 int rdtgroup_mondata_show(struct seq_file *m, void *arg);
-void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
-				    unsigned int dom_id);
-void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
-				    struct rdt_domain *d);
 void mon_event_read(struct rmid_read *rr, struct rdt_resource *r,
 		    struct rdt_domain *d, struct rdtgroup *rdtgrp,
 		    int evtid, int first);
@@ -542,8 +531,6 @@ void mbm_setup_overflow_handler(struct rdt_domain *dom,
 void mbm_handle_overflow(struct work_struct *work);
 void __init intel_rdt_mbm_apply_quirk(void);
 bool is_mba_sc(struct rdt_resource *r);
-void setup_default_ctrlval(struct rdt_resource *r, u32 *dc, u32 *dm);
-u32 delay_bw_map(unsigned long bw, struct rdt_resource *r);
 void cqm_setup_limbo_handler(struct rdt_domain *dom, unsigned long delay_ms);
 void cqm_handle_limbo(struct work_struct *work);
 bool has_busy_rmid(struct rdt_resource *r, struct rdt_domain *d);
diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c
index eaf25a234ff5..efe0c30d3a12 100644
--- a/arch/x86/kernel/cpu/resctrl/monitor.c
+++ b/arch/x86/kernel/cpu/resctrl/monitor.c
@@ -16,8 +16,12 @@
  */
 
 #include <linux/module.h>
+#include <linux/sizes.h>
 #include <linux/slab.h>
+
 #include <asm/cpu_device_id.h>
+#include <asm/resctrl.h>
+
 #include "internal.h"
 
 struct rmid_entry {
@@ -37,8 +41,8 @@ static LIST_HEAD(rmid_free_lru);
  * @rmid_limbo_count     count of currently unused but (potentially)
  *     dirty RMIDs.
  *     This counts RMIDs that no one is currently using but that
- *     may have a occupancy value > intel_cqm_threshold. User can change
- *     the threshold occupancy value.
+ *     may have a occupancy value > resctrl_rmid_realloc_threshold. User can
+ *     change the threshold occupancy value.
  */
 static unsigned int rmid_limbo_count;
 
@@ -59,10 +63,15 @@ bool rdt_mon_capable;
 unsigned int rdt_mon_features;
 
 /*
- * This is the threshold cache occupancy at which we will consider an
+ * This is the threshold cache occupancy in bytes at which we will consider an
  * RMID available for re-allocation.
  */
-unsigned int resctrl_cqm_threshold;
+unsigned int resctrl_rmid_realloc_threshold;
+
+/*
+ * This is the maximum value for the reallocation threshold, in bytes.
+ */
+unsigned int resctrl_rmid_realloc_limit;
 
 #define CF(cf)	((unsigned long)(1048576 * (cf) + 0.5))
 
@@ -137,9 +146,54 @@ static inline struct rmid_entry *__rmid_entry(u32 rmid)
 	return entry;
 }
 
-static u64 __rmid_read(u32 rmid, u32 eventid)
+static struct arch_mbm_state *get_arch_mbm_state(struct rdt_hw_domain *hw_dom,
+						 u32 rmid,
+						 enum resctrl_event_id eventid)
 {
-	u64 val;
+	switch (eventid) {
+	case QOS_L3_OCCUP_EVENT_ID:
+		return NULL;
+	case QOS_L3_MBM_TOTAL_EVENT_ID:
+		return &hw_dom->arch_mbm_total[rmid];
+	case QOS_L3_MBM_LOCAL_EVENT_ID:
+		return &hw_dom->arch_mbm_local[rmid];
+	}
+
+	/* Never expect to get here */
+	WARN_ON_ONCE(1);
+
+	return NULL;
+}
+
+void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d,
+			     u32 rmid, enum resctrl_event_id eventid)
+{
+	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
+	struct arch_mbm_state *am;
+
+	am = get_arch_mbm_state(hw_dom, rmid, eventid);
+	if (am)
+		memset(am, 0, sizeof(*am));
+}
+
+static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width)
+{
+	u64 shift = 64 - width, chunks;
+
+	chunks = (cur_msr << shift) - (prev_msr << shift);
+	return chunks >> shift;
+}
+
+int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d,
+			   u32 rmid, enum resctrl_event_id eventid, u64 *val)
+{
+	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
+	struct rdt_hw_domain *hw_dom = resctrl_to_arch_dom(d);
+	struct arch_mbm_state *am;
+	u64 msr_val, chunks;
+
+	if (!cpumask_test_cpu(smp_processor_id(), &d->cpu_mask))
+		return -EINVAL;
 
 	/*
 	 * As per the SDM, when IA32_QM_EVTSEL.EvtID (bits 7:0) is configured
@@ -150,16 +204,26 @@ static u64 __rmid_read(u32 rmid, u32 eventid)
 	 * are error bits.
 	 */
 	wrmsr(MSR_IA32_QM_EVTSEL, eventid, rmid);
-	rdmsrl(MSR_IA32_QM_CTR, val);
-
-	return val;
-}
+	rdmsrl(MSR_IA32_QM_CTR, msr_val);
+
+	if (msr_val & RMID_VAL_ERROR)
+		return -EIO;
+	if (msr_val & RMID_VAL_UNAVAIL)
+		return -EINVAL;
+
+	am = get_arch_mbm_state(hw_dom, rmid, eventid);
+	if (am) {
+		am->chunks += mbm_overflow_count(am->prev_msr, msr_val,
+						 hw_res->mbm_width);
+		chunks = get_corrected_mbm_count(rmid, am->chunks);
+		am->prev_msr = msr_val;
+	} else {
+		chunks = msr_val;
+	}
 
-static bool rmid_dirty(struct rmid_entry *entry)
-{
-	u64 val = __rmid_read(entry->rmid, QOS_L3_OCCUP_EVENT_ID);
+	*val = chunks * hw_res->mon_scale;
 
-	return val >= resctrl_cqm_threshold;
+	return 0;
 }
 
 /*
@@ -170,11 +234,11 @@ static bool rmid_dirty(struct rmid_entry *entry)
  */
 void __check_limbo(struct rdt_domain *d, bool force_free)
 {
+	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
 	struct rmid_entry *entry;
-	struct rdt_resource *r;
 	u32 crmid = 1, nrmid;
-
-	r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
+	bool rmid_dirty;
+	u64 val = 0;
 
 	/*
 	 * Skip RMID 0 and start from RMID 1 and check all the RMIDs that
@@ -188,7 +252,15 @@ void __check_limbo(struct rdt_domain *d, bool force_free)
 			break;
 
 		entry = __rmid_entry(nrmid);
-		if (force_free || !rmid_dirty(entry)) {
+
+		if (resctrl_arch_rmid_read(r, d, entry->rmid,
+					   QOS_L3_OCCUP_EVENT_ID, &val)) {
+			rmid_dirty = true;
+		} else {
+			rmid_dirty = (val >= resctrl_rmid_realloc_threshold);
+		}
+
+		if (force_free || !rmid_dirty) {
 			clear_bit(entry->rmid, d->rmid_busy_llc);
 			if (!--entry->busy) {
 				rmid_limbo_count--;
@@ -227,19 +299,19 @@ int alloc_rmid(void)
 
 static void add_rmid_to_limbo(struct rmid_entry *entry)
 {
-	struct rdt_resource *r;
+	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
 	struct rdt_domain *d;
-	int cpu;
-	u64 val;
-
-	r = &rdt_resources_all[RDT_RESOURCE_L3].r_resctrl;
+	int cpu, err;
+	u64 val = 0;
 
 	entry->busy = 0;
 	cpu = get_cpu();
 	list_for_each_entry(d, &r->domains, list) {
 		if (cpumask_test_cpu(cpu, &d->cpu_mask)) {
-			val = __rmid_read(entry->rmid, QOS_L3_OCCUP_EVENT_ID);
-			if (val <= resctrl_cqm_threshold)
+			err = resctrl_arch_rmid_read(r, d, entry->rmid,
+						     QOS_L3_OCCUP_EVENT_ID,
+						     &val);
+			if (err || val <= resctrl_rmid_realloc_threshold)
 				continue;
 		}
 
@@ -277,24 +349,18 @@ void free_rmid(u32 rmid)
 		list_add_tail(&entry->list, &rmid_free_lru);
 }
 
-static u64 mbm_overflow_count(u64 prev_msr, u64 cur_msr, unsigned int width)
+static int __mon_event_count(u32 rmid, struct rmid_read *rr)
 {
-	u64 shift = 64 - width, chunks;
+	struct mbm_state *m;
+	u64 tval = 0;
 
-	chunks = (cur_msr << shift) - (prev_msr << shift);
-	return chunks >> shift;
-}
+	if (rr->first)
+		resctrl_arch_reset_rmid(rr->r, rr->d, rmid, rr->evtid);
 
-static u64 __mon_event_count(u32 rmid, struct rmid_read *rr)
-{
-	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(rr->r);
-	struct mbm_state *m;
-	u64 chunks, tval;
+	rr->err = resctrl_arch_rmid_read(rr->r, rr->d, rmid, rr->evtid, &tval);
+	if (rr->err)
+		return rr->err;
 
-	tval = __rmid_read(rmid, rr->evtid);
-	if (tval & (RMID_VAL_ERROR | RMID_VAL_UNAVAIL)) {
-		return tval;
-	}
 	switch (rr->evtid) {
 	case QOS_L3_OCCUP_EVENT_ID:
 		rr->val += tval;
@@ -308,48 +374,47 @@ static u64 __mon_event_count(u32 rmid, struct rmid_read *rr)
 	default:
 		/*
 		 * Code would never reach here because an invalid
-		 * event id would fail the __rmid_read.
+		 * event id would fail in resctrl_arch_rmid_read().
 		 */
-		return RMID_VAL_ERROR;
+		return -EINVAL;
 	}
 
 	if (rr->first) {
 		memset(m, 0, sizeof(struct mbm_state));
-		m->prev_bw_msr = m->prev_msr = tval;
 		return 0;
 	}
 
-	chunks = mbm_overflow_count(m->prev_msr, tval, hw_res->mbm_width);
-	m->chunks += chunks;
-	m->prev_msr = tval;
-
-	rr->val += get_corrected_mbm_count(rmid, m->chunks);
+	rr->val += tval;
 
 	return 0;
 }
 
 /*
+ * mbm_bw_count() - Update bw count from values previously read by
+ *		    __mon_event_count().
+ * @rmid:	The rmid used to identify the cached mbm_state.
+ * @rr:		The struct rmid_read populated by __mon_event_count().
+ *
  * Supporting function to calculate the memory bandwidth
- * and delta bandwidth in MBps.
+ * and delta bandwidth in MBps. The chunks value previously read by
+ * __mon_event_count() is compared with the chunks value from the previous
+ * invocation. This must be called once per second to maintain values in MBps.
  */
 static void mbm_bw_count(u32 rmid, struct rmid_read *rr)
 {
-	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(rr->r);
 	struct mbm_state *m = &rr->d->mbm_local[rmid];
-	u64 tval, cur_bw, chunks;
+	u64 cur_bw, bytes, cur_bytes;
 
-	tval = __rmid_read(rmid, rr->evtid);
-	if (tval & (RMID_VAL_ERROR | RMID_VAL_UNAVAIL))
-		return;
+	cur_bytes = rr->val;
+	bytes = cur_bytes - m->prev_bw_bytes;
+	m->prev_bw_bytes = cur_bytes;
 
-	chunks = mbm_overflow_count(m->prev_bw_msr, tval, hw_res->mbm_width);
-	cur_bw = (get_corrected_mbm_count(rmid, chunks) * hw_res->mon_scale) >> 20;
+	cur_bw = bytes / SZ_1M;
 
 	if (m->delta_comp)
 		m->delta_bw = abs(cur_bw - m->prev_bw);
 	m->delta_comp = false;
 	m->prev_bw = cur_bw;
-	m->prev_bw_msr = tval;
 }
 
 /*
@@ -361,11 +426,11 @@ void mon_event_count(void *info)
 	struct rdtgroup *rdtgrp, *entry;
 	struct rmid_read *rr = info;
 	struct list_head *head;
-	u64 ret_val;
+	int ret;
 
 	rdtgrp = rr->rgrp;
 
-	ret_val = __mon_event_count(rdtgrp->mon.rmid, rr);
+	ret = __mon_event_count(rdtgrp->mon.rmid, rr);
 
 	/*
 	 * For Ctrl groups read data from child monitor groups and
@@ -377,13 +442,17 @@ void mon_event_count(void *info)
 	if (rdtgrp->type == RDTCTRL_GROUP) {
 		list_for_each_entry(entry, head, mon.crdtgrp_list) {
 			if (__mon_event_count(entry->mon.rmid, rr) == 0)
-				ret_val = 0;
+				ret = 0;
 		}
 	}
 
-	/* Report error if none of rmid_reads are successful */
-	if (ret_val)
-		rr->val = ret_val;
+	/*
+	 * __mon_event_count() calls for newly created monitor groups may
+	 * report -EINVAL/Unavailable if the monitor hasn't seen any traffic.
+	 * Discard error if any of the monitor event reads succeeded.
+	 */
+	if (ret == 0)
+		rr->err = 0;
 }
 
 /*
@@ -420,10 +489,8 @@ void mon_event_count(void *info)
  */
 static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm)
 {
-	u32 closid, rmid, cur_msr, cur_msr_val, new_msr_val;
+	u32 closid, rmid, cur_msr_val, new_msr_val;
 	struct mbm_state *pmbm_data, *cmbm_data;
-	struct rdt_hw_resource *hw_r_mba;
-	struct rdt_hw_domain *hw_dom_mba;
 	u32 cur_bw, delta_bw, user_bw;
 	struct rdt_resource *r_mba;
 	struct rdt_domain *dom_mba;
@@ -433,8 +500,8 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm)
 	if (!is_mbm_local_enabled())
 		return;
 
-	hw_r_mba = &rdt_resources_all[RDT_RESOURCE_MBA];
-	r_mba = &hw_r_mba->r_resctrl;
+	r_mba = &rdt_resources_all[RDT_RESOURCE_MBA].r_resctrl;
+
 	closid = rgrp->closid;
 	rmid = rgrp->mon.rmid;
 	pmbm_data = &dom_mbm->mbm_local[rmid];
@@ -444,16 +511,13 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm)
 		pr_warn_once("Failure to get domain for MBA update\n");
 		return;
 	}
-	hw_dom_mba = resctrl_to_arch_dom(dom_mba);
 
 	cur_bw = pmbm_data->prev_bw;
-	user_bw = resctrl_arch_get_config(r_mba, dom_mba, closid, CDP_NONE);
+	user_bw = dom_mba->mbps_val[closid];
 	delta_bw = pmbm_data->delta_bw;
-	/*
-	 * resctrl_arch_get_config() chooses the mbps/ctrl value to return
-	 * based on is_mba_sc(). For now, reach into the hw_dom.
-	 */
-	cur_msr_val = hw_dom_mba->ctrl_val[closid];
+
+	/* MBA resource doesn't support CDP */
+	cur_msr_val = resctrl_arch_get_config(r_mba, dom_mba, closid, CDP_NONE);
 
 	/*
 	 * For Ctrl groups read data from child monitor groups.
@@ -488,9 +552,7 @@ static void update_mba_bw(struct rdtgroup *rgrp, struct rdt_domain *dom_mbm)
 		return;
 	}
 
-	cur_msr = hw_r_mba->msr_base + closid;
-	wrmsrl(cur_msr, delay_bw_map(new_msr_val, r_mba));
-	hw_dom_mba->ctrl_val[closid] = new_msr_val;
+	resctrl_arch_update_one(r_mba, dom_mba, closid, CDP_NONE, new_msr_val);
 
 	/*
 	 * Delta values are updated dynamically package wise for each
@@ -523,10 +585,12 @@ static void mbm_update(struct rdt_resource *r, struct rdt_domain *d, int rmid)
 	 */
 	if (is_mbm_total_enabled()) {
 		rr.evtid = QOS_L3_MBM_TOTAL_EVENT_ID;
+		rr.val = 0;
 		__mon_event_count(rmid, &rr);
 	}
 	if (is_mbm_local_enabled()) {
 		rr.evtid = QOS_L3_MBM_LOCAL_EVENT_ID;
+		rr.val = 0;
 		__mon_event_count(rmid, &rr);
 
 		/*
@@ -686,9 +750,10 @@ int rdt_get_mon_l3_config(struct rdt_resource *r)
 {
 	unsigned int mbm_offset = boot_cpu_data.x86_cache_mbm_width_offset;
 	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
-	unsigned int cl_size = boot_cpu_data.x86_cache_size;
+	unsigned int threshold;
 	int ret;
 
+	resctrl_rmid_realloc_limit = boot_cpu_data.x86_cache_size * 1024;
 	hw_res->mon_scale = boot_cpu_data.x86_cache_occ_scale;
 	r->num_rmid = boot_cpu_data.x86_cache_max_rmid + 1;
 	hw_res->mbm_width = MBM_CNTR_WIDTH_BASE;
@@ -705,10 +770,14 @@ int rdt_get_mon_l3_config(struct rdt_resource *r)
 	 *
 	 * For a 35MB LLC and 56 RMIDs, this is ~1.8% of the LLC.
 	 */
-	resctrl_cqm_threshold = cl_size * 1024 / r->num_rmid;
+	threshold = resctrl_rmid_realloc_limit / r->num_rmid;
 
-	/* h/w works in units of "boot_cpu_data.x86_cache_occ_scale" */
-	resctrl_cqm_threshold /= hw_res->mon_scale;
+	/*
+	 * Because num_rmid may not be a power of two, round the value
+	 * to the nearest multiple of hw_res->mon_scale so it matches a
+	 * value the hardware will measure. mon_scale may not be a power of 2.
+	 */
+	resctrl_rmid_realloc_threshold = resctrl_arch_round_mon_val(threshold);
 
 	ret = dom_data_init(r);
 	if (ret)
@@ -717,7 +786,6 @@ int rdt_get_mon_l3_config(struct rdt_resource *r)
 	l3_mon_evt_init(r);
 
 	r->mon_capable = true;
-	r->mon_enabled = true;
 
 	return 0;
 }
diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c
index db813f819ad6..d961ae3ed96e 100644
--- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c
+++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c
@@ -420,6 +420,7 @@ static int pseudo_lock_fn(void *_rdtgrp)
 	struct pseudo_lock_region *plr = rdtgrp->plr;
 	u32 rmid_p, closid_p;
 	unsigned long i;
+	u64 saved_msr;
 #ifdef CONFIG_KASAN
 	/*
 	 * The registers used for local register variables are also used
@@ -463,6 +464,7 @@ static int pseudo_lock_fn(void *_rdtgrp)
 	 * the buffer and evict pseudo-locked memory read earlier from the
 	 * cache.
 	 */
+	saved_msr = __rdmsr(MSR_MISC_FEATURE_CONTROL);
 	__wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0);
 	closid_p = this_cpu_read(pqr_state.cur_closid);
 	rmid_p = this_cpu_read(pqr_state.cur_rmid);
@@ -514,7 +516,7 @@ static int pseudo_lock_fn(void *_rdtgrp)
 	__wrmsr(IA32_PQR_ASSOC, rmid_p, closid_p);
 
 	/* Re-enable the hardware prefetcher(s) */
-	wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0);
+	wrmsrl(MSR_MISC_FEATURE_CONTROL, saved_msr);
 	local_irq_enable();
 
 	plr->thread_done = 1;
@@ -835,7 +837,7 @@ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d)
 	 * First determine which cpus have pseudo-locked regions
 	 * associated with them.
 	 */
-	for_each_alloc_enabled_rdt_resource(r) {
+	for_each_alloc_capable_rdt_resource(r) {
 		list_for_each_entry(d_i, &r->domains, list) {
 			if (d_i->plr)
 				cpumask_or(cpu_with_psl, cpu_with_psl,
@@ -871,6 +873,7 @@ bool rdtgroup_pseudo_locked_in_hierarchy(struct rdt_domain *d)
 static int measure_cycles_lat_fn(void *_plr)
 {
 	struct pseudo_lock_region *plr = _plr;
+	u32 saved_low, saved_high;
 	unsigned long i;
 	u64 start, end;
 	void *mem_r;
@@ -879,6 +882,7 @@ static int measure_cycles_lat_fn(void *_plr)
 	/*
 	 * Disable hardware prefetchers.
 	 */
+	rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
 	wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0);
 	mem_r = READ_ONCE(plr->kmem);
 	/*
@@ -895,7 +899,7 @@ static int measure_cycles_lat_fn(void *_plr)
 		end = rdtsc_ordered();
 		trace_pseudo_lock_mem_latency((u32)(end - start));
 	}
-	wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0);
+	wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
 	local_irq_enable();
 	plr->thread_done = 1;
 	wake_up_interruptible(&plr->lock_thread_wq);
@@ -940,6 +944,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr,
 	u64 hits_before = 0, hits_after = 0, miss_before = 0, miss_after = 0;
 	struct perf_event *miss_event, *hit_event;
 	int hit_pmcnum, miss_pmcnum;
+	u32 saved_low, saved_high;
 	unsigned int line_size;
 	unsigned int size;
 	unsigned long i;
@@ -973,6 +978,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr,
 	/*
 	 * Disable hardware prefetchers.
 	 */
+	rdmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
 	wrmsr(MSR_MISC_FEATURE_CONTROL, prefetch_disable_bits, 0x0);
 
 	/* Initialize rest of local variables */
@@ -1031,7 +1037,7 @@ static int measure_residency_fn(struct perf_event_attr *miss_attr,
 	 */
 	rmb();
 	/* Re-enable hardware prefetchers */
-	wrmsr(MSR_MISC_FEATURE_CONTROL, 0x0, 0x0);
+	wrmsr(MSR_MISC_FEATURE_CONTROL, saved_low, saved_high);
 	local_irq_enable();
 out_hit:
 	perf_event_release_kernel(hit_event);
diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
index f276aff521e8..e5a48f05e787 100644
--- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c
+++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
@@ -1030,10 +1030,7 @@ static int rdt_delay_linear_show(struct kernfs_open_file *of,
 static int max_threshold_occ_show(struct kernfs_open_file *of,
 				  struct seq_file *seq, void *v)
 {
-	struct rdt_resource *r = of->kn->parent->priv;
-	struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
-
-	seq_printf(seq, "%u\n", resctrl_cqm_threshold * hw_res->mon_scale);
+	seq_printf(seq, "%u\n", resctrl_rmid_realloc_threshold);
 
 	return 0;
 }
@@ -1055,7 +1052,6 @@ static int rdt_thread_throttle_mode_show(struct kernfs_open_file *of,
 static ssize_t max_threshold_occ_write(struct kernfs_open_file *of,
 				       char *buf, size_t nbytes, loff_t off)
 {
-	struct rdt_hw_resource *hw_res;
 	unsigned int bytes;
 	int ret;
 
@@ -1063,11 +1059,10 @@ static ssize_t max_threshold_occ_write(struct kernfs_open_file *of,
 	if (ret)
 		return ret;
 
-	if (bytes > (boot_cpu_data.x86_cache_size * 1024))
+	if (bytes > resctrl_rmid_realloc_limit)
 		return -EINVAL;
 
-	hw_res = resctrl_to_arch_res(of->kn->parent->priv);
-	resctrl_cqm_threshold = bytes / hw_res->mon_scale;
+	resctrl_rmid_realloc_threshold = resctrl_arch_round_mon_val(bytes);
 
 	return nbytes;
 }
@@ -1356,11 +1351,13 @@ static int rdtgroup_size_show(struct kernfs_open_file *of,
 			      struct seq_file *s, void *v)
 {
 	struct resctrl_schema *schema;
+	enum resctrl_conf_type type;
 	struct rdtgroup *rdtgrp;
 	struct rdt_resource *r;
 	struct rdt_domain *d;
 	unsigned int size;
 	int ret = 0;
+	u32 closid;
 	bool sep;
 	u32 ctrl;
 
@@ -1386,8 +1383,11 @@ static int rdtgroup_size_show(struct kernfs_open_file *of,
 		goto out;
 	}
 
+	closid = rdtgrp->closid;
+
 	list_for_each_entry(schema, &resctrl_schema_all, list) {
 		r = schema->res;
+		type = schema->conf_type;
 		sep = false;
 		seq_printf(s, "%*s:", max_name_width, schema->name);
 		list_for_each_entry(d, &r->domains, list) {
@@ -1396,9 +1396,12 @@ static int rdtgroup_size_show(struct kernfs_open_file *of,
 			if (rdtgrp->mode == RDT_MODE_PSEUDO_LOCKSETUP) {
 				size = 0;
 			} else {
-				ctrl = resctrl_arch_get_config(r, d,
-							       rdtgrp->closid,
-							       schema->conf_type);
+				if (is_mba_sc(r))
+					ctrl = d->mbps_val[closid];
+				else
+					ctrl = resctrl_arch_get_config(r, d,
+								       closid,
+								       type);
 				if (r->rid == RDT_RESOURCE_MBA)
 					size = ctrl;
 				else
@@ -1756,7 +1759,7 @@ static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn)
 	if (ret)
 		goto out_destroy;
 
-	/* loop over enabled controls, these are all alloc_enabled */
+	/* loop over enabled controls, these are all alloc_capable */
 	list_for_each_entry(s, &resctrl_schema_all, list) {
 		r = s->res;
 		fflags =  r->fflags | RF_CTRL_INFO;
@@ -1765,7 +1768,7 @@ static int rdtgroup_create_info_dir(struct kernfs_node *parent_kn)
 			goto out_destroy;
 	}
 
-	for_each_mon_enabled_rdt_resource(r) {
+	for_each_mon_capable_rdt_resource(r) {
 		fflags =  r->fflags | RF_MON_INFO;
 		sprintf(name, "%s_MON", r->name);
 		ret = rdtgroup_mkdir_info_resdir(r, name, fflags);
@@ -1889,26 +1892,61 @@ void rdt_domain_reconfigure_cdp(struct rdt_resource *r)
 		l3_qos_cfg_update(&hw_res->cdp_enabled);
 }
 
+static int mba_sc_domain_allocate(struct rdt_resource *r, struct rdt_domain *d)
+{
+	u32 num_closid = resctrl_arch_get_num_closid(r);
+	int cpu = cpumask_any(&d->cpu_mask);
+	int i;
+
+	d->mbps_val = kcalloc_node(num_closid, sizeof(*d->mbps_val),
+				   GFP_KERNEL, cpu_to_node(cpu));
+	if (!d->mbps_val)
+		return -ENOMEM;
+
+	for (i = 0; i < num_closid; i++)
+		d->mbps_val[i] = MBA_MAX_MBPS;
+
+	return 0;
+}
+
+static void mba_sc_domain_destroy(struct rdt_resource *r,
+				  struct rdt_domain *d)
+{
+	kfree(d->mbps_val);
+	d->mbps_val = NULL;
+}
+
 /*
- * Enable or disable the MBA software controller
- * which helps user specify bandwidth in MBps.
  * MBA software controller is supported only if
  * MBM is supported and MBA is in linear scale.
  */
+static bool supports_mba_mbps(void)
+{
+	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_MBA].r_resctrl;
+
+	return (is_mbm_local_enabled() &&
+		r->alloc_capable && is_mba_linear());
+}
+
+/*
+ * Enable or disable the MBA software controller
+ * which helps user specify bandwidth in MBps.
+ */
 static int set_mba_sc(bool mba_sc)
 {
 	struct rdt_resource *r = &rdt_resources_all[RDT_RESOURCE_MBA].r_resctrl;
-	struct rdt_hw_domain *hw_dom;
+	u32 num_closid = resctrl_arch_get_num_closid(r);
 	struct rdt_domain *d;
+	int i;
 
-	if (!is_mbm_enabled() || !is_mba_linear() ||
-	    mba_sc == is_mba_sc(r))
+	if (!supports_mba_mbps() || mba_sc == is_mba_sc(r))
 		return -EINVAL;
 
 	r->membw.mba_sc = mba_sc;
+
 	list_for_each_entry(d, &r->domains, list) {
-		hw_dom = resctrl_to_arch_dom(d);
-		setup_default_ctrlval(r, hw_dom->ctrl_val, hw_dom->mbps_val);
+		for (i = 0; i < num_closid; i++)
+			d->mbps_val[i] = MBA_MAX_MBPS;
 	}
 
 	return 0;
@@ -2106,7 +2144,7 @@ static int schemata_list_create(void)
 	struct rdt_resource *r;
 	int ret = 0;
 
-	for_each_alloc_enabled_rdt_resource(r) {
+	for_each_alloc_capable_rdt_resource(r) {
 		if (resctrl_arch_get_cdp_enabled(r->rid)) {
 			ret = schemata_list_add(r, CDP_CODE);
 			if (ret)
@@ -2261,7 +2299,7 @@ static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param)
 		ctx->enable_cdpl2 = true;
 		return 0;
 	case Opt_mba_mbps:
-		if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
+		if (!supports_mba_mbps())
 			return -EINVAL;
 		ctx->enable_mba_mbps = true;
 		return 0;
@@ -2452,7 +2490,7 @@ static void rdt_kill_sb(struct super_block *sb)
 	set_mba_sc(false);
 
 	/*Put everything back to default values. */
-	for_each_alloc_enabled_rdt_resource(r)
+	for_each_alloc_capable_rdt_resource(r)
 		reset_all_ctrls(r);
 	cdp_disable_all();
 	rmdir_all_sub();
@@ -2499,14 +2537,12 @@ static int mon_addfile(struct kernfs_node *parent_kn, const char *name,
  * Remove all subdirectories of mon_data of ctrl_mon groups
  * and monitor groups with given domain id.
  */
-void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r, unsigned int dom_id)
+static void rmdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
+					   unsigned int dom_id)
 {
 	struct rdtgroup *prgrp, *crgrp;
 	char name[32];
 
-	if (!r->mon_enabled)
-		return;
-
 	list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
 		sprintf(name, "mon_%s_%02d", r->name, dom_id);
 		kernfs_remove_by_name(prgrp->mon.mon_data_kn, name);
@@ -2565,16 +2601,13 @@ out_destroy:
  * Add all subdirectories of mon_data for "ctrl_mon" groups
  * and "monitor" groups with given domain id.
  */
-void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
-				    struct rdt_domain *d)
+static void mkdir_mondata_subdir_allrdtgrp(struct rdt_resource *r,
+					   struct rdt_domain *d)
 {
 	struct kernfs_node *parent_kn;
 	struct rdtgroup *prgrp, *crgrp;
 	struct list_head *head;
 
-	if (!r->mon_enabled)
-		return;
-
 	list_for_each_entry(prgrp, &rdt_all_groups, rdtgroup_list) {
 		parent_kn = prgrp->mon.mon_data_kn;
 		mkdir_mondata_subdir(parent_kn, d, r, prgrp);
@@ -2642,7 +2675,7 @@ static int mkdir_mondata_all(struct kernfs_node *parent_kn,
 	 * Create the subdirectories for each domain. Note that all events
 	 * in a domain like L3 are grouped into a resource whose domain is L3
 	 */
-	for_each_mon_enabled_rdt_resource(r) {
+	for_each_mon_capable_rdt_resource(r) {
 		ret = mkdir_mondata_subdir_alldom(kn, r, prgrp);
 		if (ret)
 			goto out_destroy;
@@ -2786,14 +2819,19 @@ static int rdtgroup_init_cat(struct resctrl_schema *s, u32 closid)
 }
 
 /* Initialize MBA resource with default values. */
-static void rdtgroup_init_mba(struct rdt_resource *r)
+static void rdtgroup_init_mba(struct rdt_resource *r, u32 closid)
 {
 	struct resctrl_staged_config *cfg;
 	struct rdt_domain *d;
 
 	list_for_each_entry(d, &r->domains, list) {
+		if (is_mba_sc(r)) {
+			d->mbps_val[closid] = MBA_MAX_MBPS;
+			continue;
+		}
+
 		cfg = &d->staged_config[CDP_NONE];
-		cfg->new_ctrl = is_mba_sc(r) ? MBA_MAX_MBPS : r->default_ctrl;
+		cfg->new_ctrl = r->default_ctrl;
 		cfg->have_new_ctrl = true;
 	}
 }
@@ -2808,7 +2846,9 @@ static int rdtgroup_init_alloc(struct rdtgroup *rdtgrp)
 	list_for_each_entry(s, &resctrl_schema_all, list) {
 		r = s->res;
 		if (r->rid == RDT_RESOURCE_MBA) {
-			rdtgroup_init_mba(r);
+			rdtgroup_init_mba(r, rdtgrp->closid);
+			if (is_mba_sc(r))
+				continue;
 		} else {
 			ret = rdtgroup_init_cat(s, rdtgrp->closid);
 			if (ret < 0)
@@ -3236,6 +3276,110 @@ out:
 	return ret;
 }
 
+static void domain_destroy_mon_state(struct rdt_domain *d)
+{
+	bitmap_free(d->rmid_busy_llc);
+	kfree(d->mbm_total);
+	kfree(d->mbm_local);
+}
+
+void resctrl_offline_domain(struct rdt_resource *r, struct rdt_domain *d)
+{
+	lockdep_assert_held(&rdtgroup_mutex);
+
+	if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA)
+		mba_sc_domain_destroy(r, d);
+
+	if (!r->mon_capable)
+		return;
+
+	/*
+	 * If resctrl is mounted, remove all the
+	 * per domain monitor data directories.
+	 */
+	if (static_branch_unlikely(&rdt_mon_enable_key))
+		rmdir_mondata_subdir_allrdtgrp(r, d->id);
+
+	if (is_mbm_enabled())
+		cancel_delayed_work(&d->mbm_over);
+	if (is_llc_occupancy_enabled() && has_busy_rmid(r, d)) {
+		/*
+		 * When a package is going down, forcefully
+		 * decrement rmid->ebusy. There is no way to know
+		 * that the L3 was flushed and hence may lead to
+		 * incorrect counts in rare scenarios, but leaving
+		 * the RMID as busy creates RMID leaks if the
+		 * package never comes back.
+		 */
+		__check_limbo(d, true);
+		cancel_delayed_work(&d->cqm_limbo);
+	}
+
+	domain_destroy_mon_state(d);
+}
+
+static int domain_setup_mon_state(struct rdt_resource *r, struct rdt_domain *d)
+{
+	size_t tsize;
+
+	if (is_llc_occupancy_enabled()) {
+		d->rmid_busy_llc = bitmap_zalloc(r->num_rmid, GFP_KERNEL);
+		if (!d->rmid_busy_llc)
+			return -ENOMEM;
+	}
+	if (is_mbm_total_enabled()) {
+		tsize = sizeof(*d->mbm_total);
+		d->mbm_total = kcalloc(r->num_rmid, tsize, GFP_KERNEL);
+		if (!d->mbm_total) {
+			bitmap_free(d->rmid_busy_llc);
+			return -ENOMEM;
+		}
+	}
+	if (is_mbm_local_enabled()) {
+		tsize = sizeof(*d->mbm_local);
+		d->mbm_local = kcalloc(r->num_rmid, tsize, GFP_KERNEL);
+		if (!d->mbm_local) {
+			bitmap_free(d->rmid_busy_llc);
+			kfree(d->mbm_total);
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
+int resctrl_online_domain(struct rdt_resource *r, struct rdt_domain *d)
+{
+	int err;
+
+	lockdep_assert_held(&rdtgroup_mutex);
+
+	if (supports_mba_mbps() && r->rid == RDT_RESOURCE_MBA)
+		/* RDT_RESOURCE_MBA is never mon_capable */
+		return mba_sc_domain_allocate(r, d);
+
+	if (!r->mon_capable)
+		return 0;
+
+	err = domain_setup_mon_state(r, d);
+	if (err)
+		return err;
+
+	if (is_mbm_enabled()) {
+		INIT_DELAYED_WORK(&d->mbm_over, mbm_handle_overflow);
+		mbm_setup_overflow_handler(d, MBM_OVERFLOW_INTERVAL);
+	}
+
+	if (is_llc_occupancy_enabled())
+		INIT_DELAYED_WORK(&d->cqm_limbo, cqm_handle_limbo);
+
+	/* If resctrl is mounted, add per domain monitor data directories. */
+	if (static_branch_unlikely(&rdt_mon_enable_key))
+		mkdir_mondata_subdir_allrdtgrp(r, d);
+
+	return 0;
+}
+
 /*
  * rdtgroup_init - rdtgroup initialization
  *
diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c
index 24c1bb8eb196..1ec20807de1e 100644
--- a/arch/x86/kernel/cpu/sgx/encl.c
+++ b/arch/x86/kernel/cpu/sgx/encl.c
@@ -12,6 +12,9 @@
 #include "encls.h"
 #include "sgx.h"
 
+static int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index,
+			    struct sgx_backing *backing);
+
 #define PCMDS_PER_PAGE (PAGE_SIZE / sizeof(struct sgx_pcmd))
 /*
  * 32 PCMD entries share a PCMD page. PCMD_FIRST_MASK is used to
@@ -344,8 +347,11 @@ static vm_fault_t sgx_encl_eaug_page(struct vm_area_struct *vma,
 	}
 
 	va_page = sgx_encl_grow(encl, false);
-	if (IS_ERR(va_page))
+	if (IS_ERR(va_page)) {
+		if (PTR_ERR(va_page) == -EBUSY)
+			vmret = VM_FAULT_NOPAGE;
 		goto err_out_epc;
+	}
 
 	if (va_page)
 		list_add(&va_page->list, &encl->va_pages);
@@ -906,15 +912,14 @@ const cpumask_t *sgx_encl_cpumask(struct sgx_encl *encl)
 static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl,
 					      pgoff_t index)
 {
-	struct inode *inode = encl->backing->f_path.dentry->d_inode;
-	struct address_space *mapping = inode->i_mapping;
+	struct address_space *mapping = encl->backing->f_mapping;
 	gfp_t gfpmask = mapping_gfp_mask(mapping);
 
 	return shmem_read_mapping_page_gfp(mapping, index, gfpmask);
 }
 
 /**
- * sgx_encl_get_backing() - Pin the backing storage
+ * __sgx_encl_get_backing() - Pin the backing storage
  * @encl:	an enclave pointer
  * @page_index:	enclave page index
  * @backing:	data for accessing backing storage for the page
@@ -926,7 +931,7 @@ static struct page *sgx_encl_get_backing_page(struct sgx_encl *encl,
  *   0 on success,
  *   -errno otherwise.
  */
-static int sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index,
+static int __sgx_encl_get_backing(struct sgx_encl *encl, unsigned long page_index,
 			 struct sgx_backing *backing)
 {
 	pgoff_t page_pcmd_off = sgx_encl_get_backing_page_pcmd_offset(encl, page_index);
@@ -1001,7 +1006,7 @@ static struct mem_cgroup *sgx_encl_get_mem_cgroup(struct sgx_encl *encl)
 }
 
 /**
- * sgx_encl_alloc_backing() - allocate a new backing storage page
+ * sgx_encl_alloc_backing() - create a new backing storage page
  * @encl:	an enclave pointer
  * @page_index:	enclave page index
  * @backing:	data for accessing backing storage for the page
@@ -1009,7 +1014,9 @@ static struct mem_cgroup *sgx_encl_get_mem_cgroup(struct sgx_encl *encl)
  * When called from ksgxd, sets the active memcg from one of the
  * mms in the enclave's mm_list prior to any backing page allocation,
  * in order to ensure that shmem page allocations are charged to the
- * enclave.
+ * enclave.  Create a backing page for loading data back into an EPC page with
+ * ELDU.  This function takes a reference on a new backing page which
+ * must be dropped with a corresponding call to sgx_encl_put_backing().
  *
  * Return:
  *   0 on success,
@@ -1022,7 +1029,7 @@ int sgx_encl_alloc_backing(struct sgx_encl *encl, unsigned long page_index,
 	struct mem_cgroup *memcg = set_active_memcg(encl_memcg);
 	int ret;
 
-	ret = sgx_encl_get_backing(encl, page_index, backing);
+	ret = __sgx_encl_get_backing(encl, page_index, backing);
 
 	set_active_memcg(memcg);
 	mem_cgroup_put(encl_memcg);
@@ -1040,15 +1047,17 @@ int sgx_encl_alloc_backing(struct sgx_encl *encl, unsigned long page_index,
  * It is the caller's responsibility to ensure that it is appropriate to use
  * sgx_encl_lookup_backing() rather than sgx_encl_alloc_backing(). If lookup is
  * not used correctly, this will cause an allocation which is not accounted for.
+ * This function takes a reference on an existing backing page which must be
+ * dropped with a corresponding call to sgx_encl_put_backing().
  *
  * Return:
  *   0 on success,
  *   -errno otherwise.
  */
-int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index,
+static int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index,
 			   struct sgx_backing *backing)
 {
-	return sgx_encl_get_backing(encl, page_index, backing);
+	return __sgx_encl_get_backing(encl, page_index, backing);
 }
 
 /**
diff --git a/arch/x86/kernel/cpu/sgx/encl.h b/arch/x86/kernel/cpu/sgx/encl.h
index a65a952116fd..f94ff14c9486 100644
--- a/arch/x86/kernel/cpu/sgx/encl.h
+++ b/arch/x86/kernel/cpu/sgx/encl.h
@@ -107,8 +107,6 @@ bool current_is_ksgxd(void);
 void sgx_encl_release(struct kref *ref);
 int sgx_encl_mm_add(struct sgx_encl *encl, struct mm_struct *mm);
 const cpumask_t *sgx_encl_cpumask(struct sgx_encl *encl);
-int sgx_encl_lookup_backing(struct sgx_encl *encl, unsigned long page_index,
-			    struct sgx_backing *backing);
 int sgx_encl_alloc_backing(struct sgx_encl *encl, unsigned long page_index,
 			   struct sgx_backing *backing);
 void sgx_encl_put_backing(struct sgx_backing *backing);
diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c
index 515e2a5f25bb..0aad028f04d4 100644
--- a/arch/x86/kernel/cpu/sgx/main.c
+++ b/arch/x86/kernel/cpu/sgx/main.c
@@ -49,9 +49,13 @@ static LIST_HEAD(sgx_dirty_page_list);
  * Reset post-kexec EPC pages to the uninitialized state. The pages are removed
  * from the input list, and made available for the page allocator. SECS pages
  * prepending their children in the input list are left intact.
+ *
+ * Return 0 when sanitization was successful or kthread was stopped, and the
+ * number of unsanitized pages otherwise.
  */
-static void __sgx_sanitize_pages(struct list_head *dirty_page_list)
+static unsigned long __sgx_sanitize_pages(struct list_head *dirty_page_list)
 {
+	unsigned long left_dirty = 0;
 	struct sgx_epc_page *page;
 	LIST_HEAD(dirty);
 	int ret;
@@ -59,7 +63,7 @@ static void __sgx_sanitize_pages(struct list_head *dirty_page_list)
 	/* dirty_page_list is thread-local, no need for a lock: */
 	while (!list_empty(dirty_page_list)) {
 		if (kthread_should_stop())
-			return;
+			return 0;
 
 		page = list_first_entry(dirty_page_list, struct sgx_epc_page, list);
 
@@ -92,12 +96,14 @@ static void __sgx_sanitize_pages(struct list_head *dirty_page_list)
 		} else {
 			/* The page is not yet clean - move to the dirty list. */
 			list_move_tail(&page->list, &dirty);
+			left_dirty++;
 		}
 
 		cond_resched();
 	}
 
 	list_splice(&dirty, dirty_page_list);
+	return left_dirty;
 }
 
 static bool sgx_reclaimer_age(struct sgx_epc_page *epc_page)
@@ -395,10 +401,7 @@ static int ksgxd(void *p)
 	 * required for SECS pages, whose child pages blocked EREMOVE.
 	 */
 	__sgx_sanitize_pages(&sgx_dirty_page_list);
-	__sgx_sanitize_pages(&sgx_dirty_page_list);
-
-	/* sanity check: */
-	WARN_ON(!list_empty(&sgx_dirty_page_list));
+	WARN_ON(__sgx_sanitize_pages(&sgx_dirty_page_list));
 
 	while (!kthread_should_stop()) {
 		if (try_to_freeze())
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index afae4dd77495..b3dba35f466e 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -128,7 +128,7 @@ void show_opcodes(struct pt_regs *regs, const char *loglvl)
 		/* No access to the user space stack of other tasks. Ignore. */
 		break;
 	default:
-		printk("%sCode: Unable to access opcode bytes at RIP 0x%lx.\n",
+		printk("%sCode: Unable to access opcode bytes at 0x%lx.\n",
 		       loglvl, prologue);
 		break;
 	}
diff --git a/arch/x86/kernel/early_printk.c b/arch/x86/kernel/early_printk.c
index 68b38925a74f..44f937015e1e 100644
--- a/arch/x86/kernel/early_printk.c
+++ b/arch/x86/kernel/early_printk.c
@@ -264,11 +264,11 @@ static __init void early_pci_serial_init(char *s)
 	bar0 = read_pci_config(bus, slot, func, PCI_BASE_ADDRESS_0);
 
 	/*
-	 * Verify it is a UART type device
+	 * Verify it is a 16550-UART type device
 	 */
 	if (((classcode >> 16 != PCI_CLASS_COMMUNICATION_MODEM) &&
 	     (classcode >> 16 != PCI_CLASS_COMMUNICATION_SERIAL)) ||
-	   (((classcode >> 8) & 0xff) != 0x02)) /* 16550 I/F at BAR0 */ {
+	    (((classcode >> 8) & 0xff) != PCI_SERIAL_16550_COMPATIBLE)) {
 		if (!force)
 			return;
 	}
@@ -276,22 +276,22 @@ static __init void early_pci_serial_init(char *s)
 	/*
 	 * Determine if it is IO or memory mapped
 	 */
-	if (bar0 & 0x01) {
+	if ((bar0 & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) {
 		/* it is IO mapped */
 		serial_in = io_serial_in;
 		serial_out = io_serial_out;
-		early_serial_base = bar0&0xfffffffc;
+		early_serial_base = bar0 & PCI_BASE_ADDRESS_IO_MASK;
 		write_pci_config(bus, slot, func, PCI_COMMAND,
-						cmdreg|PCI_COMMAND_IO);
+				 cmdreg|PCI_COMMAND_IO);
 	} else {
 		/* It is memory mapped - assume 32-bit alignment */
 		serial_in = mem32_serial_in;
 		serial_out = mem32_serial_out;
 		/* WARNING! assuming the address is always in the first 4G */
 		early_serial_base =
-			(unsigned long)early_ioremap(bar0 & 0xfffffff0, 0x10);
+			(unsigned long)early_ioremap(bar0 & PCI_BASE_ADDRESS_MEM_MASK, 0x10);
 		write_pci_config(bus, slot, func, PCI_COMMAND,
-						cmdreg|PCI_COMMAND_MEMORY);
+				 cmdreg|PCI_COMMAND_MEMORY);
 	}
 
 	/*
diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c
index 586f718b8e95..349046434513 100644
--- a/arch/x86/kernel/rtc.c
+++ b/arch/x86/kernel/rtc.c
@@ -4,11 +4,8 @@
  */
 #include <linux/platform_device.h>
 #include <linux/mc146818rtc.h>
-#include <linux/acpi.h>
-#include <linux/bcd.h>
 #include <linux/export.h>
 #include <linux/pnp.h>
-#include <linux/of.h>
 
 #include <asm/vsyscall.h>
 #include <asm/x86_init.h>
@@ -20,26 +17,23 @@
 /*
  * This is a special lock that is owned by the CPU and holds the index
  * register we are working with.  It is required for NMI access to the
- * CMOS/RTC registers.  See include/asm-i386/mc146818rtc.h for details.
+ * CMOS/RTC registers.  See arch/x86/include/asm/mc146818rtc.h for details.
  */
 volatile unsigned long cmos_lock;
 EXPORT_SYMBOL(cmos_lock);
 #endif /* CONFIG_X86_32 */
 
-/* For two digit years assume time is always after that */
-#define CMOS_YEARS_OFFS 2000
-
 DEFINE_SPINLOCK(rtc_lock);
 EXPORT_SYMBOL(rtc_lock);
 
 /*
- * In order to set the CMOS clock precisely, set_rtc_mmss has to be
+ * In order to set the CMOS clock precisely, mach_set_cmos_time has to be
  * called 500 ms after the second nowtime has started, because when
  * nowtime is written into the registers of the CMOS clock, it will
  * jump to the next second precisely 500 ms later. Check the Motorola
  * MC146818A or Dallas DS12887 data sheet for details.
  */
-int mach_set_rtc_mmss(const struct timespec64 *now)
+int mach_set_cmos_time(const struct timespec64 *now)
 {
 	unsigned long long nowtime = now->tv_sec;
 	struct rtc_time tm;
@@ -62,8 +56,7 @@ int mach_set_rtc_mmss(const struct timespec64 *now)
 
 void mach_get_cmos_time(struct timespec64 *now)
 {
-	unsigned int status, year, mon, day, hour, min, sec, century = 0;
-	unsigned long flags;
+	struct rtc_time tm;
 
 	/*
 	 * If pm_trace abused the RTC as storage, set the timespec to 0,
@@ -74,51 +67,13 @@ void mach_get_cmos_time(struct timespec64 *now)
 		return;
 	}
 
-	spin_lock_irqsave(&rtc_lock, flags);
-
-	/*
-	 * If UIP is clear, then we have >= 244 microseconds before
-	 * RTC registers will be updated.  Spec sheet says that this
-	 * is the reliable way to read RTC - registers. If UIP is set
-	 * then the register access might be invalid.
-	 */
-	while ((CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP))
-		cpu_relax();
-
-	sec = CMOS_READ(RTC_SECONDS);
-	min = CMOS_READ(RTC_MINUTES);
-	hour = CMOS_READ(RTC_HOURS);
-	day = CMOS_READ(RTC_DAY_OF_MONTH);
-	mon = CMOS_READ(RTC_MONTH);
-	year = CMOS_READ(RTC_YEAR);
-
-#ifdef CONFIG_ACPI
-	if (acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID &&
-	    acpi_gbl_FADT.century)
-		century = CMOS_READ(acpi_gbl_FADT.century);
-#endif
-
-	status = CMOS_READ(RTC_CONTROL);
-	WARN_ON_ONCE(RTC_ALWAYS_BCD && (status & RTC_DM_BINARY));
-
-	spin_unlock_irqrestore(&rtc_lock, flags);
-
-	if (RTC_ALWAYS_BCD || !(status & RTC_DM_BINARY)) {
-		sec = bcd2bin(sec);
-		min = bcd2bin(min);
-		hour = bcd2bin(hour);
-		day = bcd2bin(day);
-		mon = bcd2bin(mon);
-		year = bcd2bin(year);
+	if (mc146818_get_time(&tm)) {
+		pr_err("Unable to read current time from RTC\n");
+		now->tv_sec = now->tv_nsec = 0;
+		return;
 	}
 
-	if (century) {
-		century = bcd2bin(century);
-		year += century * 100;
-	} else
-		year += CMOS_YEARS_OFFS;
-
-	now->tv_sec = mktime64(year, mon, day, hour, min, sec);
+	now->tv_sec = rtc_tm_to_time64(&tm);
 	now->tv_nsec = 0;
 }
 
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index d62b2cb85cea..178015a820f0 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -63,6 +63,7 @@
 #include <asm/insn-eval.h>
 #include <asm/vdso.h>
 #include <asm/tdx.h>
+#include <asm/cfi.h>
 
 #ifdef CONFIG_X86_64
 #include <asm/x86_init.h>
@@ -313,7 +314,8 @@ static noinstr bool handle_bug(struct pt_regs *regs)
 	 */
 	if (regs->flags & X86_EFLAGS_IF)
 		raw_local_irq_enable();
-	if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) {
+	if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN ||
+	    handle_cfi_failure(regs) == BUG_TRAP_TYPE_WARN) {
 		regs->ip += LEN_UD2;
 		handled = true;
 	}
diff --git a/arch/x86/kernel/x86_init.c b/arch/x86/kernel/x86_init.c
index e84ee5cdbd8c..57353519bc11 100644
--- a/arch/x86/kernel/x86_init.c
+++ b/arch/x86/kernel/x86_init.c
@@ -138,7 +138,7 @@ struct x86_platform_ops x86_platform __ro_after_init = {
 	.calibrate_cpu			= native_calibrate_cpu_early,
 	.calibrate_tsc			= native_calibrate_tsc,
 	.get_wallclock			= mach_get_cmos_time,
-	.set_wallclock			= mach_set_rtc_mmss,
+	.set_wallclock			= mach_set_cmos_time,
 	.iommu_shutdown			= iommu_shutdown_noop,
 	.is_untracked_pat_range		= is_ISA_range,
 	.nmi_init			= default_nmi_init,
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 4c1c2c06e96b..2796dde06302 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -902,8 +902,6 @@ static inline int __do_cpuid_func(struct kvm_cpuid_array *array, u32 function)
 			entry->edx = 0;
 		}
 		break;
-	case 9:
-		break;
 	case 0xa: { /* Architectural Performance Monitoring */
 		union cpuid10_eax eax;
 		union cpuid10_edx edx;
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index ddd4367d4826..7eaf96064cb0 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -2570,7 +2570,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
 	 * bits which we consider mandatory enabled.
 	 * The CR0_READ_SHADOW is what L2 should have expected to read given
 	 * the specifications by L1; It's not enough to take
-	 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
+	 * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we
 	 * have more bits than L1 expected.
 	 */
 	vmx_set_cr0(vcpu, vmcs12->guest_cr0);
diff --git a/arch/x86/lib/clear_page_64.S b/arch/x86/lib/clear_page_64.S
index fe59b8ac4fcc..ecbfb4dd3b01 100644
--- a/arch/x86/lib/clear_page_64.S
+++ b/arch/x86/lib/clear_page_64.S
@@ -1,5 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 #include <linux/linkage.h>
+#include <asm/asm.h>
 #include <asm/export.h>
 
 /*
@@ -50,3 +51,140 @@ SYM_FUNC_START(clear_page_erms)
 	RET
 SYM_FUNC_END(clear_page_erms)
 EXPORT_SYMBOL_GPL(clear_page_erms)
+
+/*
+ * Default clear user-space.
+ * Input:
+ * rdi destination
+ * rcx count
+ *
+ * Output:
+ * rcx: uncleared bytes or 0 if successful.
+ */
+SYM_FUNC_START(clear_user_original)
+	/*
+	 * Copy only the lower 32 bits of size as that is enough to handle the rest bytes,
+	 * i.e., no need for a 'q' suffix and thus a REX prefix.
+	 */
+	mov %ecx,%eax
+	shr $3,%rcx
+	jz .Lrest_bytes
+
+	# do the qwords first
+	.p2align 4
+.Lqwords:
+	movq $0,(%rdi)
+	lea 8(%rdi),%rdi
+	dec %rcx
+	jnz .Lqwords
+
+.Lrest_bytes:
+	and $7,  %eax
+	jz .Lexit
+
+	# now do the rest bytes
+.Lbytes:
+	movb $0,(%rdi)
+	inc %rdi
+	dec %eax
+	jnz .Lbytes
+
+.Lexit:
+	/*
+	 * %rax still needs to be cleared in the exception case because this function is called
+	 * from inline asm and the compiler expects %rax to be zero when exiting the inline asm,
+	 * in case it might reuse it somewhere.
+	 */
+        xor %eax,%eax
+        RET
+
+.Lqwords_exception:
+        # convert remaining qwords back into bytes to return to caller
+        shl $3, %rcx
+        and $7, %eax
+        add %rax,%rcx
+        jmp .Lexit
+
+.Lbytes_exception:
+        mov %eax,%ecx
+        jmp .Lexit
+
+        _ASM_EXTABLE_UA(.Lqwords, .Lqwords_exception)
+        _ASM_EXTABLE_UA(.Lbytes, .Lbytes_exception)
+SYM_FUNC_END(clear_user_original)
+EXPORT_SYMBOL(clear_user_original)
+
+/*
+ * Alternative clear user-space when CPU feature X86_FEATURE_REP_GOOD is
+ * present.
+ * Input:
+ * rdi destination
+ * rcx count
+ *
+ * Output:
+ * rcx: uncleared bytes or 0 if successful.
+ */
+SYM_FUNC_START(clear_user_rep_good)
+	# call the original thing for less than a cacheline
+	cmp $64, %rcx
+	jb clear_user_original
+
+.Lprep:
+	# copy lower 32-bits for rest bytes
+	mov %ecx, %edx
+	shr $3, %rcx
+	jz .Lrep_good_rest_bytes
+
+.Lrep_good_qwords:
+	rep stosq
+
+.Lrep_good_rest_bytes:
+	and $7, %edx
+	jz .Lrep_good_exit
+
+.Lrep_good_bytes:
+	mov %edx, %ecx
+	rep stosb
+
+.Lrep_good_exit:
+	# see .Lexit comment above
+	xor %eax, %eax
+	RET
+
+.Lrep_good_qwords_exception:
+	# convert remaining qwords back into bytes to return to caller
+	shl $3, %rcx
+	and $7, %edx
+	add %rdx, %rcx
+	jmp .Lrep_good_exit
+
+	_ASM_EXTABLE_UA(.Lrep_good_qwords, .Lrep_good_qwords_exception)
+	_ASM_EXTABLE_UA(.Lrep_good_bytes, .Lrep_good_exit)
+SYM_FUNC_END(clear_user_rep_good)
+EXPORT_SYMBOL(clear_user_rep_good)
+
+/*
+ * Alternative clear user-space when CPU feature X86_FEATURE_ERMS is present.
+ * Input:
+ * rdi destination
+ * rcx count
+ *
+ * Output:
+ * rcx: uncleared bytes or 0 if successful.
+ *
+ */
+SYM_FUNC_START(clear_user_erms)
+	# call the original thing for less than a cacheline
+	cmp $64, %rcx
+	jb clear_user_original
+
+.Lerms_bytes:
+	rep stosb
+
+.Lerms_exit:
+	xorl %eax,%eax
+	RET
+
+	_ASM_EXTABLE_UA(.Lerms_bytes, .Lerms_exit)
+SYM_FUNC_END(clear_user_erms)
+EXPORT_SYMBOL(clear_user_erms)
diff --git a/arch/x86/lib/memcpy_64.S b/arch/x86/lib/memcpy_64.S
index d0d7b9bc6cad..dd8cd8831251 100644
--- a/arch/x86/lib/memcpy_64.S
+++ b/arch/x86/lib/memcpy_64.S
@@ -2,6 +2,7 @@
 /* Copyright 2002 Andi Kleen */
 
 #include <linux/linkage.h>
+#include <linux/cfi_types.h>
 #include <asm/errno.h>
 #include <asm/cpufeatures.h>
 #include <asm/alternative.h>
@@ -27,7 +28,7 @@
  * Output:
  * rax original destination
  */
-SYM_FUNC_START(__memcpy)
+SYM_TYPED_FUNC_START(__memcpy)
 	ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
 		      "jmp memcpy_erms", X86_FEATURE_ERMS
 
diff --git a/arch/x86/lib/usercopy.c b/arch/x86/lib/usercopy.c
index ad0139d25401..f1bb18617156 100644
--- a/arch/x86/lib/usercopy.c
+++ b/arch/x86/lib/usercopy.c
@@ -44,7 +44,7 @@ copy_from_user_nmi(void *to, const void __user *from, unsigned long n)
 	 * called from other contexts.
 	 */
 	pagefault_disable();
-	ret = __copy_from_user_inatomic(to, from, n);
+	ret = raw_copy_from_user(to, from, n);
 	pagefault_enable();
 
 	return ret;
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index 0ae6cf804197..6c1f8ac5e721 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -14,46 +14,6 @@
  * Zero Userspace
  */
 
-unsigned long __clear_user(void __user *addr, unsigned long size)
-{
-	long __d0;
-	might_fault();
-	/* no memory constraint because it doesn't change any memory gcc knows
-	   about */
-	stac();
-	asm volatile(
-		"	testq  %[size8],%[size8]\n"
-		"	jz     4f\n"
-		"	.align 16\n"
-		"0:	movq $0,(%[dst])\n"
-		"	addq   $8,%[dst]\n"
-		"	decl %%ecx ; jnz   0b\n"
-		"4:	movq  %[size1],%%rcx\n"
-		"	testl %%ecx,%%ecx\n"
-		"	jz     2f\n"
-		"1:	movb   $0,(%[dst])\n"
-		"	incq   %[dst]\n"
-		"	decl %%ecx ; jnz  1b\n"
-		"2:\n"
-
-		_ASM_EXTABLE_TYPE_REG(0b, 2b, EX_TYPE_UCOPY_LEN8, %[size1])
-		_ASM_EXTABLE_UA(1b, 2b)
-
-		: [size8] "=&c"(size), [dst] "=&D" (__d0)
-		: [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr));
-	clac();
-	return size;
-}
-EXPORT_SYMBOL(__clear_user);
-
-unsigned long clear_user(void __user *to, unsigned long n)
-{
-	if (access_ok(to, n))
-		return __clear_user(to, n);
-	return n;
-}
-EXPORT_SYMBOL(clear_user);
-
 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
 /**
  * clean_cache_range - write back a cache range with CLWB
diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index f8220fd2c169..829c1409ffbd 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -4,10 +4,12 @@ KCOV_INSTRUMENT_tlb.o			:= n
 KCOV_INSTRUMENT_mem_encrypt.o		:= n
 KCOV_INSTRUMENT_mem_encrypt_amd.o	:= n
 KCOV_INSTRUMENT_mem_encrypt_identity.o	:= n
+KCOV_INSTRUMENT_pgprot.o		:= n
 
 KASAN_SANITIZE_mem_encrypt.o		:= n
 KASAN_SANITIZE_mem_encrypt_amd.o	:= n
 KASAN_SANITIZE_mem_encrypt_identity.o	:= n
+KASAN_SANITIZE_pgprot.o		:= n
 
 # Disable KCSAN entirely, because otherwise we get warnings that some functions
 # reference __initdata sections.
@@ -17,6 +19,7 @@ ifdef CONFIG_FUNCTION_TRACER
 CFLAGS_REMOVE_mem_encrypt.o		= -pg
 CFLAGS_REMOVE_mem_encrypt_amd.o		= -pg
 CFLAGS_REMOVE_mem_encrypt_identity.o	= -pg
+CFLAGS_REMOVE_pgprot.o			= -pg
 endif
 
 obj-y				:=  init.o init_$(BITS).o fault.o ioremap.o extable.o mmap.o \
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index fa71a5d12e87..a498ae1fbe66 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -769,6 +769,8 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code,
 		unsigned long address, struct task_struct *tsk)
 {
 	const char *loglvl = task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG;
+	/* This is a racy snapshot, but it's better than nothing. */
+	int cpu = raw_smp_processor_id();
 
 	if (!unhandled_signal(tsk, SIGSEGV))
 		return;
@@ -782,6 +784,14 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code,
 
 	print_vma_addr(KERN_CONT " in ", regs->ip);
 
+	/*
+	 * Dump the likely CPU where the fatal segfault happened.
+	 * This can help identify faulty hardware.
+	 */
+	printk(KERN_CONT " likely on CPU %d (core %d, socket %d)", cpu,
+	       topology_core_id(cpu), topology_physical_package_id(cpu));
+
+
 	printk(KERN_CONT "\n");
 
 	show_opcodes(regs, loglvl);
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index c1f6c1c51d99..99620428ad78 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -419,7 +419,9 @@ static void emit_indirect_jump(u8 **pprog, int reg, u8 *ip)
 		OPTIMIZER_HIDE_VAR(reg);
 		emit_jump(&prog, &__x86_indirect_thunk_array[reg], ip);
 	} else {
-		EMIT2(0xFF, 0xE0 + reg);
+		EMIT2(0xFF, 0xE0 + reg);	/* jmp *%\reg */
+		if (IS_ENABLED(CONFIG_RETPOLINE) || IS_ENABLED(CONFIG_SLS))
+			EMIT1(0xCC);		/* int3 */
 	}
 
 	*pprog = prog;
@@ -662,7 +664,7 @@ static void emit_mov_imm64(u8 **pprog, u32 dst_reg,
 		 */
 		emit_mov_imm32(&prog, false, dst_reg, imm32_lo);
 	} else {
-		/* movabsq %rax, imm64 */
+		/* movabsq rax, imm64 */
 		EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg));
 		EMIT(imm32_lo, 4);
 		EMIT(imm32_hi, 4);
@@ -1751,34 +1753,60 @@ emit_jmp:
 static void save_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
 		      int stack_size)
 {
-	int i;
+	int i, j, arg_size, nr_regs;
 	/* Store function arguments to stack.
 	 * For a function that accepts two pointers the sequence will be:
 	 * mov QWORD PTR [rbp-0x10],rdi
 	 * mov QWORD PTR [rbp-0x8],rsi
 	 */
-	for (i = 0; i < min(nr_args, 6); i++)
-		emit_stx(prog, bytes_to_bpf_size(m->arg_size[i]),
-			 BPF_REG_FP,
-			 i == 5 ? X86_REG_R9 : BPF_REG_1 + i,
-			 -(stack_size - i * 8));
+	for (i = 0, j = 0; i < min(nr_args, 6); i++) {
+		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) {
+			nr_regs = (m->arg_size[i] + 7) / 8;
+			arg_size = 8;
+		} else {
+			nr_regs = 1;
+			arg_size = m->arg_size[i];
+		}
+
+		while (nr_regs) {
+			emit_stx(prog, bytes_to_bpf_size(arg_size),
+				 BPF_REG_FP,
+				 j == 5 ? X86_REG_R9 : BPF_REG_1 + j,
+				 -(stack_size - j * 8));
+			nr_regs--;
+			j++;
+		}
+	}
 }
 
 static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
 			 int stack_size)
 {
-	int i;
+	int i, j, arg_size, nr_regs;
 
 	/* Restore function arguments from stack.
 	 * For a function that accepts two pointers the sequence will be:
 	 * EMIT4(0x48, 0x8B, 0x7D, 0xF0); mov rdi,QWORD PTR [rbp-0x10]
 	 * EMIT4(0x48, 0x8B, 0x75, 0xF8); mov rsi,QWORD PTR [rbp-0x8]
 	 */
-	for (i = 0; i < min(nr_args, 6); i++)
-		emit_ldx(prog, bytes_to_bpf_size(m->arg_size[i]),
-			 i == 5 ? X86_REG_R9 : BPF_REG_1 + i,
-			 BPF_REG_FP,
-			 -(stack_size - i * 8));
+	for (i = 0, j = 0; i < min(nr_args, 6); i++) {
+		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) {
+			nr_regs = (m->arg_size[i] + 7) / 8;
+			arg_size = 8;
+		} else {
+			nr_regs = 1;
+			arg_size = m->arg_size[i];
+		}
+
+		while (nr_regs) {
+			emit_ldx(prog, bytes_to_bpf_size(arg_size),
+				 j == 5 ? X86_REG_R9 : BPF_REG_1 + j,
+				 BPF_REG_FP,
+				 -(stack_size - j * 8));
+			nr_regs--;
+			j++;
+		}
+	}
 }
 
 static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
@@ -1810,6 +1838,9 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
 	if (p->aux->sleepable) {
 		enter = __bpf_prog_enter_sleepable;
 		exit = __bpf_prog_exit_sleepable;
+	} else if (p->type == BPF_PROG_TYPE_STRUCT_OPS) {
+		enter = __bpf_prog_enter_struct_ops;
+		exit = __bpf_prog_exit_struct_ops;
 	} else if (p->expected_attach_type == BPF_LSM_CGROUP) {
 		enter = __bpf_prog_enter_lsm_cgroup;
 		exit = __bpf_prog_exit_lsm_cgroup;
@@ -2013,13 +2044,14 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
 int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end,
 				const struct btf_func_model *m, u32 flags,
 				struct bpf_tramp_links *tlinks,
-				void *orig_call)
+				void *func_addr)
 {
-	int ret, i, nr_args = m->nr_args;
+	int ret, i, nr_args = m->nr_args, extra_nregs = 0;
 	int regs_off, ip_off, args_off, stack_size = nr_args * 8, run_ctx_off;
 	struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
 	struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
 	struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
+	void *orig_call = func_addr;
 	u8 **branches = NULL;
 	u8 *prog;
 	bool save_ret;
@@ -2028,6 +2060,14 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 	if (nr_args > 6)
 		return -ENOTSUPP;
 
+	for (i = 0; i < MAX_BPF_FUNC_ARGS; i++) {
+		if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG)
+			extra_nregs += (m->arg_size[i] + 7) / 8 - 1;
+	}
+	if (nr_args + extra_nregs > 6)
+		return -ENOTSUPP;
+	stack_size += extra_nregs * 8;
+
 	/* Generated trampoline stack layout:
 	 *
 	 * RBP + 8         [ return address  ]
@@ -2040,7 +2080,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 	 *                 [ ...             ]
 	 * RBP - regs_off  [ reg_arg1        ]  program's ctx pointer
 	 *
-	 * RBP - args_off  [ args count      ]  always
+	 * RBP - args_off  [ arg regs count  ]  always
 	 *
 	 * RBP - ip_off    [ traced function ]  BPF_TRAMP_F_IP_ARG flag
 	 *
@@ -2083,21 +2123,19 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
 	EMIT4(0x48, 0x83, 0xEC, stack_size); /* sub rsp, stack_size */
 	EMIT1(0x53);		 /* push rbx */
 
-	/* Store number of arguments of the traced function:
-	 *   mov rax, nr_args
+	/* Store number of argument registers of the traced function:
+	 *   mov rax, nr_args + extra_nregs
 	 *   mov QWORD PTR [rbp - args_off], rax
 	 */
-	emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args);
+	emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_args + extra_nregs);
 	emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -args_off);
 
 	if (flags & BPF_TRAMP_F_IP_ARG) {
 		/* Store IP address of the traced function:
-		 * mov rax, QWORD PTR [rbp + 8]
-		 * sub rax, X86_PATCH_SIZE
+		 * movabsq rax, func_addr
 		 * mov QWORD PTR [rbp - ip_off], rax
 		 */
-		emit_ldx(&prog, BPF_DW, BPF_REG_0, BPF_REG_FP, 8);
-		EMIT4(0x48, 0x83, 0xe8, X86_PATCH_SIZE);
+		emit_mov_imm64(&prog, BPF_REG_0, (long) func_addr >> 32, (u32) (long) func_addr);
 		emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -ip_off);
 	}
 
@@ -2209,7 +2247,7 @@ cleanup:
 	return ret;
 }
 
-static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs)
+static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf)
 {
 	u8 *jg_reloc, *prog = *pprog;
 	int pivot, err, jg_bytes = 1;
@@ -2225,12 +2263,12 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs)
 		EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3),
 			    progs[a]);
 		err = emit_cond_near_jump(&prog,	/* je func */
-					  (void *)progs[a], prog,
+					  (void *)progs[a], image + (prog - buf),
 					  X86_JE);
 		if (err)
 			return err;
 
-		emit_indirect_jump(&prog, 2 /* rdx */, prog);
+		emit_indirect_jump(&prog, 2 /* rdx */, image + (prog - buf));
 
 		*pprog = prog;
 		return 0;
@@ -2255,7 +2293,7 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs)
 	jg_reloc = prog;
 
 	err = emit_bpf_dispatcher(&prog, a, a + pivot,	/* emit lower_part */
-				  progs);
+				  progs, image, buf);
 	if (err)
 		return err;
 
@@ -2269,7 +2307,7 @@ static int emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs)
 	emit_code(jg_reloc - jg_bytes, jg_offset, jg_bytes);
 
 	err = emit_bpf_dispatcher(&prog, a + pivot + 1,	/* emit upper_part */
-				  b, progs);
+				  b, progs, image, buf);
 	if (err)
 		return err;
 
@@ -2289,12 +2327,12 @@ static int cmp_ips(const void *a, const void *b)
 	return 0;
 }
 
-int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs)
+int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs)
 {
-	u8 *prog = image;
+	u8 *prog = buf;
 
 	sort(funcs, num_funcs, sizeof(funcs[0]), cmp_ips, NULL);
-	return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs);
+	return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf);
 }
 
 struct x64_jit_data {
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 6e598bd78eef..ebc98a68c400 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -169,7 +169,7 @@ static void __init do_add_efi_memmap(void)
 }
 
 /*
- * Given add_efi_memmap defaults to 0 and there there is no alternative
+ * Given add_efi_memmap defaults to 0 and there is no alternative
  * e820 mechanism for soft-reserved memory, import the full EFI memory
  * map if soft reservations are present and enabled. Otherwise, the
  * mechanism to disable the kernel's consideration of EFI_MEMORY_SP is
diff --git a/arch/x86/purgatory/Makefile b/arch/x86/purgatory/Makefile
index 31c634a22818..58a200dc762d 100644
--- a/arch/x86/purgatory/Makefile
+++ b/arch/x86/purgatory/Makefile
@@ -55,6 +55,10 @@ ifdef CONFIG_RETPOLINE
 PURGATORY_CFLAGS_REMOVE		+= $(RETPOLINE_CFLAGS)
 endif
 
+ifdef CONFIG_CFI_CLANG
+PURGATORY_CFLAGS_REMOVE		+= $(CC_FLAGS_CFI)
+endif
+
 CFLAGS_REMOVE_purgatory.o	+= $(PURGATORY_CFLAGS_REMOVE)
 CFLAGS_purgatory.o		+= $(PURGATORY_CFLAGS)
 
diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c
index e2c5b296120d..2925074b9a58 100644
--- a/arch/x86/tools/relocs.c
+++ b/arch/x86/tools/relocs.c
@@ -56,6 +56,7 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = {
 	"^(xen_irq_disable_direct_reloc$|"
 	"xen_save_fl_direct_reloc$|"
 	"VDSO|"
+	"__kcfi_typeid_|"
 	"__crc_)",
 
 /*
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index 0ed2e487a693..9b1a58dda935 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -765,6 +765,7 @@ static void xen_load_idt(const struct desc_ptr *desc)
 {
 	static DEFINE_SPINLOCK(lock);
 	static struct trap_info traps[257];
+	static const struct trap_info zero = { };
 	unsigned out;
 
 	trace_xen_cpu_load_idt(desc);
@@ -774,7 +775,7 @@ static void xen_load_idt(const struct desc_ptr *desc)
 	memcpy(this_cpu_ptr(&idt_desc), desc, sizeof(idt_desc));
 
 	out = xen_convert_trap_info(desc, traps, false);
-	memset(&traps[out], 0, sizeof(traps[0]));
+	traps[out] = zero;
 
 	xen_mc_flush();
 	if (HYPERVISOR_set_trap_table(traps))
diff --git a/block/bdev.c b/block/bdev.c
index ce05175e71ce..d699ecdb3260 100644
--- a/block/bdev.c
+++ b/block/bdev.c
@@ -26,6 +26,7 @@
 #include <linux/namei.h>
 #include <linux/part_stat.h>
 #include <linux/uaccess.h>
+#include <linux/stat.h>
 #include "../fs/internal.h"
 #include "blk.h"
 
@@ -1069,3 +1070,25 @@ void sync_bdevs(bool wait)
 	spin_unlock(&blockdev_superblock->s_inode_list_lock);
 	iput(old_inode);
 }
+
+/*
+ * Handle STATX_DIOALIGN for block devices.
+ *
+ * Note that the inode passed to this is the inode of a block device node file,
+ * not the block device's internal inode.  Therefore it is *not* valid to use
+ * I_BDEV() here; the block device has to be looked up by i_rdev instead.
+ */
+void bdev_statx_dioalign(struct inode *inode, struct kstat *stat)
+{
+	struct block_device *bdev;
+
+	bdev = blkdev_get_no_open(inode->i_rdev);
+	if (!bdev)
+		return;
+
+	stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
+	stat->dio_offset_align = bdev_logical_block_size(bdev);
+	stat->result_mask |= STATX_DIOALIGN;
+
+	blkdev_put_no_open(bdev);
+}
diff --git a/drivers/acpi/Kconfig b/drivers/acpi/Kconfig
index 7802d8846a8d..473241b5193f 100644
--- a/drivers/acpi/Kconfig
+++ b/drivers/acpi/Kconfig
@@ -27,9 +27,6 @@ menuconfig ACPI
 	  Management (APM) specification.  If both ACPI and APM support 
 	  are configured, ACPI is used.
 
-	  The project home page for the Linux ACPI subsystem is here:
-	  <https://01.org/linux-acpi>
-
 	  Linux support for ACPI is based on Intel Corporation's ACPI
 	  Component Architecture (ACPI CA).  For more information on the
 	  ACPI CA, see:
@@ -212,6 +209,7 @@ config ACPI_VIDEO
 	tristate "Video"
 	depends on BACKLIGHT_CLASS_DEVICE
 	depends on INPUT
+	depends on ACPI_WMI || !X86
 	select THERMAL
 	help
 	  This driver implements the ACPI Extensions For Display Adapters
@@ -347,7 +345,6 @@ config ACPI_CUSTOM_DSDT_FILE
 	depends on !STANDALONE
 	help
 	  This option supports a custom DSDT by linking it into the kernel.
-	  See Documentation/admin-guide/acpi/dsdt-override.rst
 
 	  Enter the full path name to the file which includes the AmlCode
 	  or dsdt_aml_code declaration.
diff --git a/drivers/acpi/ac.c b/drivers/acpi/ac.c
index c29e41bfcf35..bb9fe7984b1a 100644
--- a/drivers/acpi/ac.c
+++ b/drivers/acpi/ac.c
@@ -36,11 +36,6 @@ static int acpi_ac_add(struct acpi_device *device);
 static int acpi_ac_remove(struct acpi_device *device);
 static void acpi_ac_notify(struct acpi_device *device, u32 event);
 
-struct acpi_ac_bl {
-	const char *hid;
-	int hrv;
-};
-
 static const struct acpi_device_id ac_device_ids[] = {
 	{"ACPI0003", 0},
 	{"", 0},
diff --git a/drivers/acpi/acpi_amba.c b/drivers/acpi/acpi_amba.c
index ab8a4e0191b1..f5b443ab01c2 100644
--- a/drivers/acpi/acpi_amba.c
+++ b/drivers/acpi/acpi_amba.c
@@ -21,6 +21,7 @@
 
 static const struct acpi_device_id amba_id_list[] = {
 	{"ARMH0061", 0}, /* PL061 GPIO Device */
+	{"ARMH0330", 0}, /* ARM DMA Controller DMA-330 */
 	{"ARMHC500", 0}, /* ARM CoreSight ETM4x */
 	{"ARMHC501", 0}, /* ARM CoreSight ETR */
 	{"ARMHC502", 0}, /* ARM CoreSight STM */
@@ -48,6 +49,7 @@ static void amba_register_dummy_clk(void)
 static int amba_handler_attach(struct acpi_device *adev,
 				const struct acpi_device_id *id)
 {
+	struct acpi_device *parent = acpi_dev_parent(adev);
 	struct amba_device *dev;
 	struct resource_entry *rentry;
 	struct list_head resource_list;
@@ -97,8 +99,8 @@ static int amba_handler_attach(struct acpi_device *adev,
 	 * attached to it, that physical device should be the parent of
 	 * the amba device we are about to create.
 	 */
-	if (adev->parent)
-		dev->dev.parent = acpi_get_first_physical_node(adev->parent);
+	if (parent)
+		dev->dev.parent = acpi_get_first_physical_node(parent);
 
 	ACPI_COMPANION_SET(&dev->dev, adev);
 
diff --git a/drivers/acpi/acpi_apd.c b/drivers/acpi/acpi_apd.c
index ad245bbd965e..3bbe2276cac7 100644
--- a/drivers/acpi/acpi_apd.c
+++ b/drivers/acpi/acpi_apd.c
@@ -60,12 +60,6 @@ static int acpi_apd_setup(struct apd_private_data *pdata)
 }
 
 #ifdef CONFIG_X86_AMD_PLATFORM_DEVICE
-static int misc_check_res(struct acpi_resource *ares, void *data)
-{
-	struct resource res;
-
-	return !acpi_dev_resource_memory(ares, &res);
-}
 
 static int fch_misc_setup(struct apd_private_data *pdata)
 {
@@ -82,8 +76,7 @@ static int fch_misc_setup(struct apd_private_data *pdata)
 		return -ENOMEM;
 
 	INIT_LIST_HEAD(&resource_list);
-	ret = acpi_dev_get_resources(adev, &resource_list, misc_check_res,
-				     NULL);
+	ret = acpi_dev_get_memory_resources(adev, &resource_list);
 	if (ret < 0)
 		return -ENOENT;
 
diff --git a/drivers/acpi/acpi_fpdt.c b/drivers/acpi/acpi_fpdt.c
index 6922a44b3ce7..a2056c4c8cb7 100644
--- a/drivers/acpi/acpi_fpdt.c
+++ b/drivers/acpi/acpi_fpdt.c
@@ -143,6 +143,23 @@ static const struct attribute_group boot_attr_group = {
 
 static struct kobject *fpdt_kobj;
 
+#if defined CONFIG_X86 && defined CONFIG_PHYS_ADDR_T_64BIT
+#include <linux/processor.h>
+static bool fpdt_address_valid(u64 address)
+{
+	/*
+	 * On some systems the table contains invalid addresses
+	 * with unsuppored high address bits set, check for this.
+	 */
+	return !(address >> boot_cpu_data.x86_phys_bits);
+}
+#else
+static bool fpdt_address_valid(u64 address)
+{
+	return true;
+}
+#endif
+
 static int fpdt_process_subtable(u64 address, u32 subtable_type)
 {
 	struct fpdt_subtable_header *subtable_header;
@@ -151,6 +168,11 @@ static int fpdt_process_subtable(u64 address, u32 subtable_type)
 	u32 length, offset;
 	int result;
 
+	if (!fpdt_address_valid(address)) {
+		pr_info(FW_BUG "invalid physical address: 0x%llx!\n", address);
+		return -EINVAL;
+	}
+
 	subtable_header = acpi_os_map_memory(address, sizeof(*subtable_header));
 	if (!subtable_header)
 		return -ENOMEM;
diff --git a/drivers/acpi/acpi_lpss.c b/drivers/acpi/acpi_lpss.c
index c4d4d21391d7..f08ffa75f4a7 100644
--- a/drivers/acpi/acpi_lpss.c
+++ b/drivers/acpi/acpi_lpss.c
@@ -167,10 +167,10 @@ static struct pwm_lookup byt_pwm_lookup[] = {
 
 static void byt_pwm_setup(struct lpss_private_data *pdata)
 {
-	struct acpi_device *adev = pdata->adev;
+	u64 uid;
 
 	/* Only call pwm_add_table for the first PWM controller */
-	if (!adev->pnp.unique_id || strcmp(adev->pnp.unique_id, "1"))
+	if (acpi_dev_uid_to_integer(pdata->adev, &uid) || uid != 1)
 		return;
 
 	pwm_add_table(byt_pwm_lookup, ARRAY_SIZE(byt_pwm_lookup));
@@ -180,14 +180,13 @@ static void byt_pwm_setup(struct lpss_private_data *pdata)
 
 static void byt_i2c_setup(struct lpss_private_data *pdata)
 {
-	const char *uid_str = acpi_device_uid(pdata->adev);
 	acpi_handle handle = pdata->adev->handle;
 	unsigned long long shared_host = 0;
 	acpi_status status;
-	long uid = 0;
+	u64 uid;
 
-	/* Expected to always be true, but better safe then sorry */
-	if (uid_str && !kstrtol(uid_str, 10, &uid) && uid) {
+	/* Expected to always be successfull, but better safe then sorry */
+	if (!acpi_dev_uid_to_integer(pdata->adev, &uid) && uid) {
 		/* Detect I2C bus shared with PUNIT and ignore its d3 status */
 		status = acpi_evaluate_integer(handle, "_SEM", NULL, &shared_host);
 		if (ACPI_SUCCESS(status) && shared_host)
@@ -211,10 +210,10 @@ static struct pwm_lookup bsw_pwm_lookup[] = {
 
 static void bsw_pwm_setup(struct lpss_private_data *pdata)
 {
-	struct acpi_device *adev = pdata->adev;
+	u64 uid;
 
 	/* Only call pwm_add_table for the first PWM controller */
-	if (!adev->pnp.unique_id || strcmp(adev->pnp.unique_id, "1"))
+	if (acpi_dev_uid_to_integer(pdata->adev, &uid) || uid != 1)
 		return;
 
 	pwm_add_table(bsw_pwm_lookup, ARRAY_SIZE(bsw_pwm_lookup));
@@ -392,13 +391,6 @@ static const struct acpi_device_id acpi_lpss_device_ids[] = {
 
 #ifdef CONFIG_X86_INTEL_LPSS
 
-static int is_memory(struct acpi_resource *res, void *not_used)
-{
-	struct resource r;
-
-	return !acpi_dev_resource_memory(res, &r);
-}
-
 /* LPSS main clock device. */
 static struct platform_device *lpss_clk_dev;
 
@@ -659,29 +651,25 @@ static int acpi_lpss_create_device(struct acpi_device *adev,
 		return -ENOMEM;
 
 	INIT_LIST_HEAD(&resource_list);
-	ret = acpi_dev_get_resources(adev, &resource_list, is_memory, NULL);
+	ret = acpi_dev_get_memory_resources(adev, &resource_list);
 	if (ret < 0)
 		goto err_out;
 
-	list_for_each_entry(rentry, &resource_list, node)
-		if (resource_type(rentry->res) == IORESOURCE_MEM) {
-			if (dev_desc->prv_size_override)
-				pdata->mmio_size = dev_desc->prv_size_override;
-			else
-				pdata->mmio_size = resource_size(rentry->res);
-			pdata->mmio_base = ioremap(rentry->res->start,
-						   pdata->mmio_size);
-			break;
-		}
+	rentry = list_first_entry_or_null(&resource_list, struct resource_entry, node);
+	if (rentry) {
+		if (dev_desc->prv_size_override)
+			pdata->mmio_size = dev_desc->prv_size_override;
+		else
+			pdata->mmio_size = resource_size(rentry->res);
+		pdata->mmio_base = ioremap(rentry->res->start, pdata->mmio_size);
+	}
 
 	acpi_dev_free_resource_list(&resource_list);
 
 	if (!pdata->mmio_base) {
 		/* Avoid acpi_bus_attach() instantiating a pdev for this dev. */
 		adev->pnp.type.platform_id = 0;
-		/* Skip the device, but continue the namespace scan. */
-		ret = 0;
-		goto err_out;
+		goto out_free;
 	}
 
 	pdata->adev = adev;
@@ -692,11 +680,8 @@ static int acpi_lpss_create_device(struct acpi_device *adev,
 
 	if (dev_desc->flags & LPSS_CLK) {
 		ret = register_device_clock(adev, pdata);
-		if (ret) {
-			/* Skip the device, but continue the namespace scan. */
-			ret = 0;
-			goto err_out;
-		}
+		if (ret)
+			goto out_free;
 	}
 
 	/*
@@ -708,15 +693,19 @@ static int acpi_lpss_create_device(struct acpi_device *adev,
 
 	adev->driver_data = pdata;
 	pdev = acpi_create_platform_device(adev, dev_desc->properties);
-	if (!IS_ERR_OR_NULL(pdev)) {
-		acpi_lpss_create_device_links(adev, pdev);
-		return 1;
+	if (IS_ERR_OR_NULL(pdev)) {
+		adev->driver_data = NULL;
+		ret = PTR_ERR(pdev);
+		goto err_out;
 	}
 
-	ret = PTR_ERR(pdev);
-	adev->driver_data = NULL;
+	acpi_lpss_create_device_links(adev, pdev);
+	return 1;
 
- err_out:
+out_free:
+	/* Skip the device, but continue the namespace scan */
+	ret = 0;
+err_out:
 	kfree(pdata);
 	return ret;
 }
diff --git a/drivers/acpi/acpi_pcc.c b/drivers/acpi/acpi_pcc.c
index a12b55d81209..ee4ce5ba1fb2 100644
--- a/drivers/acpi/acpi_pcc.c
+++ b/drivers/acpi/acpi_pcc.c
@@ -23,6 +23,12 @@
 
 #include <acpi/pcc.h>
 
+/*
+ * Arbitrary retries in case the remote processor is slow to respond
+ * to PCC commands
+ */
+#define PCC_CMD_WAIT_RETRIES_NUM	500
+
 struct pcc_data {
 	struct pcc_mbox_chan *pcc_chan;
 	void __iomem *pcc_comm_addr;
@@ -63,6 +69,7 @@ acpi_pcc_address_space_setup(acpi_handle region_handle, u32 function,
 	if (IS_ERR(data->pcc_chan)) {
 		pr_err("Failed to find PCC channel for subspace %d\n",
 		       ctx->subspace_id);
+		kfree(data);
 		return AE_NOT_FOUND;
 	}
 
@@ -72,6 +79,8 @@ acpi_pcc_address_space_setup(acpi_handle region_handle, u32 function,
 	if (!data->pcc_comm_addr) {
 		pr_err("Failed to ioremap PCC comm region mem for %d\n",
 		       ctx->subspace_id);
+		pcc_mbox_free_channel(data->pcc_chan);
+		kfree(data);
 		return AE_NO_MEMORY;
 	}
 
@@ -86,6 +95,7 @@ acpi_pcc_address_space_handler(u32 function, acpi_physical_address addr,
 {
 	int ret;
 	struct pcc_data *data = region_context;
+	u64 usecs_lat;
 
 	reinit_completion(&data->done);
 
@@ -96,10 +106,22 @@ acpi_pcc_address_space_handler(u32 function, acpi_physical_address addr,
 	if (ret < 0)
 		return AE_ERROR;
 
-	if (data->pcc_chan->mchan->mbox->txdone_irq)
-		wait_for_completion(&data->done);
+	if (data->pcc_chan->mchan->mbox->txdone_irq) {
+		/*
+		 * pcc_chan->latency is just a Nominal value. In reality the remote
+		 * processor could be much slower to reply. So add an arbitrary
+		 * amount of wait on top of Nominal.
+		 */
+		usecs_lat = PCC_CMD_WAIT_RETRIES_NUM * data->pcc_chan->latency;
+		ret = wait_for_completion_timeout(&data->done,
+						  usecs_to_jiffies(usecs_lat));
+		if (ret == 0) {
+			pr_err("PCC command executed timeout!\n");
+			return AE_TIME;
+		}
+	}
 
-	mbox_client_txdone(data->pcc_chan->mchan, ret);
+	mbox_chan_txdone(data->pcc_chan->mchan, ret);
 
 	memcpy_fromio(value, data->pcc_comm_addr, data->ctx.length);
 
diff --git a/drivers/acpi/acpi_platform.c b/drivers/acpi/acpi_platform.c
index de3cbf152dee..fe00a5783f53 100644
--- a/drivers/acpi/acpi_platform.c
+++ b/drivers/acpi/acpi_platform.c
@@ -20,13 +20,13 @@
 #include "internal.h"
 
 static const struct acpi_device_id forbidden_id_list[] = {
+	{"ACPI0009", 0},	/* IOxAPIC */
+	{"ACPI000A", 0},	/* IOAPIC */
 	{"PNP0000",  0},	/* PIC */
 	{"PNP0100",  0},	/* Timer */
 	{"PNP0200",  0},	/* AT DMA Controller */
-	{"ACPI0009", 0},	/* IOxAPIC */
-	{"ACPI000A", 0},	/* IOAPIC */
 	{"SMB0001",  0},	/* ACPI SMBUS virtual device */
-	{"", 0},
+	{ }
 };
 
 static struct platform_device *acpi_platform_device_find_by_companion(struct acpi_device *adev)
@@ -78,7 +78,7 @@ static void acpi_platform_fill_resource(struct acpi_device *adev,
 	 * If the device has parent we need to take its resources into
 	 * account as well because this device might consume part of those.
 	 */
-	parent = acpi_get_first_physical_node(adev->parent);
+	parent = acpi_get_first_physical_node(acpi_dev_parent(adev));
 	if (parent && dev_is_pci(parent))
 		dest->parent = pci_find_resource(to_pci_dev(parent), dest);
 }
@@ -97,6 +97,7 @@ static void acpi_platform_fill_resource(struct acpi_device *adev,
 struct platform_device *acpi_create_platform_device(struct acpi_device *adev,
 						    const struct property_entry *properties)
 {
+	struct acpi_device *parent = acpi_dev_parent(adev);
 	struct platform_device *pdev = NULL;
 	struct platform_device_info pdevinfo;
 	struct resource_entry *rentry;
@@ -113,13 +114,11 @@ struct platform_device *acpi_create_platform_device(struct acpi_device *adev,
 
 	INIT_LIST_HEAD(&resource_list);
 	count = acpi_dev_get_resources(adev, &resource_list, NULL, NULL);
-	if (count < 0) {
+	if (count < 0)
 		return NULL;
-	} else if (count > 0) {
-		resources = kcalloc(count, sizeof(struct resource),
-				    GFP_KERNEL);
+	if (count > 0) {
+		resources = kcalloc(count, sizeof(*resources), GFP_KERNEL);
 		if (!resources) {
-			dev_err(&adev->dev, "No memory for resources\n");
 			acpi_dev_free_resource_list(&resource_list);
 			return ERR_PTR(-ENOMEM);
 		}
@@ -137,10 +136,9 @@ struct platform_device *acpi_create_platform_device(struct acpi_device *adev,
 	 * attached to it, that physical device should be the parent of the
 	 * platform device we are about to create.
 	 */
-	pdevinfo.parent = adev->parent ?
-		acpi_get_first_physical_node(adev->parent) : NULL;
+	pdevinfo.parent = parent ? acpi_get_first_physical_node(parent) : NULL;
 	pdevinfo.name = dev_name(&adev->dev);
-	pdevinfo.id = -1;
+	pdevinfo.id = PLATFORM_DEVID_NONE;
 	pdevinfo.res = resources;
 	pdevinfo.num_res = count;
 	pdevinfo.fwnode = acpi_fwnode_handle(adev);
diff --git a/drivers/acpi/acpi_video.c b/drivers/acpi/acpi_video.c
index 5cbe2196176d..32953646caeb 100644
--- a/drivers/acpi/acpi_video.c
+++ b/drivers/acpi/acpi_video.c
@@ -47,9 +47,6 @@ module_param(brightness_switch_enabled, bool, 0644);
 static bool allow_duplicates;
 module_param(allow_duplicates, bool, 0644);
 
-static int disable_backlight_sysfs_if = -1;
-module_param(disable_backlight_sysfs_if, int, 0444);
-
 #define REPORT_OUTPUT_KEY_EVENTS		0x01
 #define REPORT_BRIGHTNESS_KEY_EVENTS		0x02
 static int report_key_events = -1;
@@ -73,6 +70,16 @@ module_param(device_id_scheme, bool, 0444);
 static int only_lcd = -1;
 module_param(only_lcd, int, 0444);
 
+/*
+ * Display probing is known to take up to 5 seconds, so delay the fallback
+ * backlight registration by 5 seconds + 3 seconds for some extra margin.
+ */
+static int register_backlight_delay = 8;
+module_param(register_backlight_delay, int, 0444);
+MODULE_PARM_DESC(register_backlight_delay,
+	"Delay in seconds before doing fallback (non GPU driver triggered) "
+	"backlight registration, set to 0 to disable.");
+
 static bool may_report_brightness_keys;
 static int register_count;
 static DEFINE_MUTEX(register_count_mutex);
@@ -81,7 +88,9 @@ static LIST_HEAD(video_bus_head);
 static int acpi_video_bus_add(struct acpi_device *device);
 static int acpi_video_bus_remove(struct acpi_device *device);
 static void acpi_video_bus_notify(struct acpi_device *device, u32 event);
-void acpi_video_detect_exit(void);
+static void acpi_video_bus_register_backlight_work(struct work_struct *ignored);
+static DECLARE_DELAYED_WORK(video_bus_register_backlight_work,
+			    acpi_video_bus_register_backlight_work);
 
 /*
  * Indices in the _BCL method response: the first two items are special,
@@ -382,14 +391,6 @@ static int video_set_bqc_offset(const struct dmi_system_id *d)
 	return 0;
 }
 
-static int video_disable_backlight_sysfs_if(
-	const struct dmi_system_id *d)
-{
-	if (disable_backlight_sysfs_if == -1)
-		disable_backlight_sysfs_if = 1;
-	return 0;
-}
-
 static int video_set_device_id_scheme(const struct dmi_system_id *d)
 {
 	device_id_scheme = true;
@@ -463,40 +464,6 @@ static const struct dmi_system_id video_dmi_table[] = {
 	},
 
 	/*
-	 * Some machines have a broken acpi-video interface for brightness
-	 * control, but still need an acpi_video_device_lcd_set_level() call
-	 * on resume to turn the backlight power on.  We Enable backlight
-	 * control on these systems, but do not register a backlight sysfs
-	 * as brightness control does not work.
-	 */
-	{
-	 /* https://bugzilla.kernel.org/show_bug.cgi?id=21012 */
-	 .callback = video_disable_backlight_sysfs_if,
-	 .ident = "Toshiba Portege R700",
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
-		DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R700"),
-		},
-	},
-	{
-	 /* https://bugs.freedesktop.org/show_bug.cgi?id=82634 */
-	 .callback = video_disable_backlight_sysfs_if,
-	 .ident = "Toshiba Portege R830",
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
-		DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R830"),
-		},
-	},
-	{
-	 /* https://bugzilla.kernel.org/show_bug.cgi?id=21012 */
-	 .callback = video_disable_backlight_sysfs_if,
-	 .ident = "Toshiba Satellite R830",
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
-		DMI_MATCH(DMI_PRODUCT_NAME, "SATELLITE R830"),
-		},
-	},
-	/*
 	 * Some machine's _DOD IDs don't have bit 31(Device ID Scheme) set
 	 * but the IDs actually follow the Device ID Scheme.
 	 */
@@ -1758,9 +1725,6 @@ static void acpi_video_dev_register_backlight(struct acpi_video_device *device)
 	if (result)
 		return;
 
-	if (disable_backlight_sysfs_if > 0)
-		return;
-
 	name = kasprintf(GFP_KERNEL, "acpi_video%d", count);
 	if (!name)
 		return;
@@ -1859,8 +1823,6 @@ static int acpi_video_bus_register_backlight(struct acpi_video_bus *video)
 	if (video->backlight_registered)
 		return 0;
 
-	acpi_video_run_bcl_for_osi(video);
-
 	if (acpi_video_get_backlight_type() != acpi_backlight_video)
 		return 0;
 
@@ -2030,7 +1992,7 @@ static int acpi_video_bus_add(struct acpi_device *device)
 	acpi_status status;
 
 	status = acpi_walk_namespace(ACPI_TYPE_DEVICE,
-				device->parent->handle, 1,
+				acpi_dev_parent(device)->handle, 1,
 				acpi_video_bus_match, NULL,
 				device, NULL);
 	if (status == AE_ALREADY_EXISTS) {
@@ -2086,7 +2048,11 @@ static int acpi_video_bus_add(struct acpi_device *device)
 	list_add_tail(&video->entry, &video_bus_head);
 	mutex_unlock(&video_list_lock);
 
-	acpi_video_bus_register_backlight(video);
+	/*
+	 * The userspace visible backlight_device gets registered separately
+	 * from acpi_video_register_backlight().
+	 */
+	acpi_video_run_bcl_for_osi(video);
 	acpi_video_bus_add_notify_handler(video);
 
 	return 0;
@@ -2111,20 +2077,25 @@ static int acpi_video_bus_remove(struct acpi_device *device)
 
 	video = acpi_driver_data(device);
 
-	acpi_video_bus_remove_notify_handler(video);
-	acpi_video_bus_unregister_backlight(video);
-	acpi_video_bus_put_devices(video);
-
 	mutex_lock(&video_list_lock);
 	list_del(&video->entry);
 	mutex_unlock(&video_list_lock);
 
+	acpi_video_bus_remove_notify_handler(video);
+	acpi_video_bus_unregister_backlight(video);
+	acpi_video_bus_put_devices(video);
+
 	kfree(video->attached_array);
 	kfree(video);
 
 	return 0;
 }
 
+static void acpi_video_bus_register_backlight_work(struct work_struct *ignored)
+{
+	acpi_video_register_backlight();
+}
+
 static int __init is_i740(struct pci_dev *dev)
 {
 	if (dev->device == 0x00D1)
@@ -2235,6 +2206,18 @@ int acpi_video_register(void)
 	 */
 	register_count = 1;
 
+	/*
+	 * acpi_video_bus_add() skips registering the userspace visible
+	 * backlight_device. The intend is for this to be registered by the
+	 * drm/kms driver calling acpi_video_register_backlight() *after* it is
+	 * done setting up its own native backlight device. The delayed work
+	 * ensures that acpi_video_register_backlight() always gets called
+	 * eventually, in case there is no drm/kms driver or it is disabled.
+	 */
+	if (register_backlight_delay)
+		schedule_delayed_work(&video_bus_register_backlight_work,
+				      register_backlight_delay * HZ);
+
 leave:
 	mutex_unlock(&register_count_mutex);
 	return ret;
@@ -2245,6 +2228,7 @@ void acpi_video_unregister(void)
 {
 	mutex_lock(&register_count_mutex);
 	if (register_count) {
+		cancel_delayed_work_sync(&video_bus_register_backlight_work);
 		acpi_bus_unregister_driver(&acpi_video_bus);
 		register_count = 0;
 		may_report_brightness_keys = false;
@@ -2253,19 +2237,16 @@ void acpi_video_unregister(void)
 }
 EXPORT_SYMBOL(acpi_video_unregister);
 
-void acpi_video_unregister_backlight(void)
+void acpi_video_register_backlight(void)
 {
 	struct acpi_video_bus *video;
 
-	mutex_lock(&register_count_mutex);
-	if (register_count) {
-		mutex_lock(&video_list_lock);
-		list_for_each_entry(video, &video_bus_head, entry)
-			acpi_video_bus_unregister_backlight(video);
-		mutex_unlock(&video_list_lock);
-	}
-	mutex_unlock(&register_count_mutex);
+	mutex_lock(&video_list_lock);
+	list_for_each_entry(video, &video_bus_head, entry)
+		acpi_video_bus_register_backlight(video);
+	mutex_unlock(&video_list_lock);
 }
+EXPORT_SYMBOL(acpi_video_register_backlight);
 
 bool acpi_video_handles_brightness_key_presses(void)
 {
@@ -2302,7 +2283,6 @@ static int __init acpi_video_init(void)
 
 static void __exit acpi_video_exit(void)
 {
-	acpi_video_detect_exit();
 	acpi_video_unregister();
 }
 
diff --git a/drivers/acpi/apei/apei-base.c b/drivers/acpi/apei/apei-base.c
index 9f49272cad39..9b52482b4ed5 100644
--- a/drivers/acpi/apei/apei-base.c
+++ b/drivers/acpi/apei/apei-base.c
@@ -125,12 +125,9 @@ EXPORT_SYMBOL_GPL(apei_exec_write_register);
 int apei_exec_write_register_value(struct apei_exec_context *ctx,
 				   struct acpi_whea_header *entry)
 {
-	int rc;
-
 	ctx->value = entry->value;
-	rc = apei_exec_write_register(ctx, entry);
 
-	return rc;
+	return apei_exec_write_register(ctx, entry);
 }
 EXPORT_SYMBOL_GPL(apei_exec_write_register_value);
 
diff --git a/drivers/acpi/apei/bert.c b/drivers/acpi/apei/bert.c
index 45973aa6e06d..c23eb75866d0 100644
--- a/drivers/acpi/apei/bert.c
+++ b/drivers/acpi/apei/bert.c
@@ -90,6 +90,9 @@ static void __init bert_print_all(struct acpi_bert_region *region,
 
 	if (skipped)
 		pr_info(HW_ERR "Skipped %d error records\n", skipped);
+
+	if (printed + skipped)
+		pr_info("Total records found: %d\n", printed + skipped);
 }
 
 static int __init setup_bert_disable(char *str)
diff --git a/drivers/acpi/apei/erst.c b/drivers/acpi/apei/erst.c
index 31b077eedb58..247989060e29 100644
--- a/drivers/acpi/apei/erst.c
+++ b/drivers/acpi/apei/erst.c
@@ -1020,14 +1020,10 @@ static int reader_pos;
 
 static int erst_open_pstore(struct pstore_info *psi)
 {
-	int rc;
-
 	if (erst_disable)
 		return -ENODEV;
 
-	rc = erst_get_record_id_begin(&reader_pos);
-
-	return rc;
+	return erst_get_record_id_begin(&reader_pos);
 }
 
 static int erst_close_pstore(struct pstore_info *psi)
diff --git a/drivers/acpi/arm64/dma.c b/drivers/acpi/arm64/dma.c
index f16739ad3cc0..93d796531af3 100644
--- a/drivers/acpi/arm64/dma.c
+++ b/drivers/acpi/arm64/dma.c
@@ -4,11 +4,12 @@
 #include <linux/device.h>
 #include <linux/dma-direct.h>
 
-void acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size)
+void acpi_arch_dma_setup(struct device *dev)
 {
 	int ret;
 	u64 end, mask;
-	u64 dmaaddr = 0, size = 0, offset = 0;
+	u64 size = 0;
+	const struct bus_dma_region *map = NULL;
 
 	/*
 	 * If @dev is expected to be DMA-capable then the bus code that created
@@ -26,7 +27,19 @@ void acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size)
 	else
 		size = 1ULL << 32;
 
-	ret = acpi_dma_get_range(dev, &dmaaddr, &offset, &size);
+	ret = acpi_dma_get_range(dev, &map);
+	if (!ret && map) {
+		const struct bus_dma_region *r = map;
+
+		for (end = 0; r->size; r++) {
+			if (r->dma_start + r->size - 1 > end)
+				end = r->dma_start + r->size - 1;
+		}
+
+		size = end + 1;
+		dev->dma_range_map = map;
+	}
+
 	if (ret == -ENODEV)
 		ret = iort_dma_get_ranges(dev, &size);
 	if (!ret) {
@@ -34,17 +47,10 @@ void acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size)
 		 * Limit coherent and dma mask based on size retrieved from
 		 * firmware.
 		 */
-		end = dmaaddr + size - 1;
+		end = size - 1;
 		mask = DMA_BIT_MASK(ilog2(end) + 1);
 		dev->bus_dma_limit = end;
 		dev->coherent_dma_mask = min(dev->coherent_dma_mask, mask);
 		*dev->dma_mask = min(*dev->dma_mask, mask);
 	}
-
-	*dma_addr = dmaaddr;
-	*dma_size = size;
-
-	ret = dma_direct_set_offset(dev, dmaaddr + offset, dmaaddr, size);
-
-	dev_dbg(dev, "dma_offset(%#08llx)%s\n", offset, ret ? " failed!" : "");
 }
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index c0d20d997891..d466c8195314 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -456,7 +456,7 @@ out_free:
                              Notification Handling
    -------------------------------------------------------------------------- */
 
-/**
+/*
  * acpi_bus_notify
  * ---------------
  * Callback for all 'system-level' device notifications (values 0x00-0x7F).
@@ -511,7 +511,7 @@ static void acpi_bus_notify(acpi_handle handle, u32 type, void *data)
 		break;
 	}
 
-	adev = acpi_bus_get_acpi_device(handle);
+	adev = acpi_get_acpi_dev(handle);
 	if (!adev)
 		goto err;
 
@@ -524,14 +524,14 @@ static void acpi_bus_notify(acpi_handle handle, u32 type, void *data)
 	}
 
 	if (!hotplug_event) {
-		acpi_bus_put_acpi_device(adev);
+		acpi_put_acpi_dev(adev);
 		return;
 	}
 
 	if (ACPI_SUCCESS(acpi_hotplug_schedule(adev, type)))
 		return;
 
-	acpi_bus_put_acpi_device(adev);
+	acpi_put_acpi_dev(adev);
 
  err:
 	acpi_evaluate_ost(handle, type, ost_code, NULL);
@@ -802,7 +802,7 @@ static bool acpi_of_modalias(struct acpi_device *adev,
 
 	str = obj->string.pointer;
 	chr = strchr(str, ',');
-	strlcpy(modalias, chr ? chr + 1 : str, len);
+	strscpy(modalias, chr ? chr + 1 : str, len);
 
 	return true;
 }
@@ -822,7 +822,7 @@ void acpi_set_modalias(struct acpi_device *adev, const char *default_id,
 		       char *modalias, size_t len)
 {
 	if (!acpi_of_modalias(adev, modalias, len))
-		strlcpy(modalias, default_id, len);
+		strscpy(modalias, default_id, len);
 }
 EXPORT_SYMBOL_GPL(acpi_set_modalias);
 
@@ -925,12 +925,13 @@ static const void *acpi_of_device_get_match_data(const struct device *dev)
 
 const void *acpi_device_get_match_data(const struct device *dev)
 {
+	const struct acpi_device_id *acpi_ids = dev->driver->acpi_match_table;
 	const struct acpi_device_id *match;
 
-	if (!dev->driver->acpi_match_table)
+	if (!acpi_ids)
 		return acpi_of_device_get_match_data(dev);
 
-	match = acpi_match_device(dev->driver->acpi_match_table, dev);
+	match = acpi_match_device(acpi_ids, dev);
 	if (!match)
 		return NULL;
 
@@ -948,14 +949,13 @@ EXPORT_SYMBOL(acpi_match_device_ids);
 bool acpi_driver_match_device(struct device *dev,
 			      const struct device_driver *drv)
 {
-	if (!drv->acpi_match_table)
-		return acpi_of_match_device(ACPI_COMPANION(dev),
-					    drv->of_match_table,
-					    NULL);
-
-	return __acpi_match_device(acpi_companion_match(dev),
-				   drv->acpi_match_table, drv->of_match_table,
-				   NULL, NULL);
+	const struct acpi_device_id *acpi_ids = drv->acpi_match_table;
+	const struct of_device_id *of_ids = drv->of_match_table;
+
+	if (!acpi_ids)
+		return acpi_of_match_device(ACPI_COMPANION(dev), of_ids, NULL);
+
+	return __acpi_match_device(acpi_companion_match(dev), acpi_ids, of_ids, NULL, NULL);
 }
 EXPORT_SYMBOL_GPL(acpi_driver_match_device);
 
@@ -973,16 +973,13 @@ EXPORT_SYMBOL_GPL(acpi_driver_match_device);
  */
 int acpi_bus_register_driver(struct acpi_driver *driver)
 {
-	int ret;
-
 	if (acpi_disabled)
 		return -ENODEV;
 	driver->drv.name = driver->name;
 	driver->drv.bus = &acpi_bus_type;
 	driver->drv.owner = driver->owner;
 
-	ret = driver_register(&driver->drv);
-	return ret;
+	return driver_register(&driver->drv);
 }
 
 EXPORT_SYMBOL(acpi_bus_register_driver);
diff --git a/drivers/acpi/cppc_acpi.c b/drivers/acpi/cppc_acpi.c
index 1e15a9f25ae9..093675b1a1ff 100644
--- a/drivers/acpi/cppc_acpi.c
+++ b/drivers/acpi/cppc_acpi.c
@@ -424,6 +424,9 @@ bool acpi_cpc_valid(void)
 	struct cpc_desc *cpc_ptr;
 	int cpu;
 
+	if (acpi_disabled)
+		return false;
+
 	for_each_present_cpu(cpu) {
 		cpc_ptr = per_cpu(cpc_desc_ptr, cpu);
 		if (!cpc_ptr)
@@ -1241,6 +1244,48 @@ out_err:
 EXPORT_SYMBOL_GPL(cppc_get_perf_caps);
 
 /**
+ * cppc_perf_ctrs_in_pcc - Check if any perf counters are in a PCC region.
+ *
+ * CPPC has flexibility about how CPU performance counters are accessed.
+ * One of the choices is PCC regions, which can have a high access latency. This
+ * routine allows callers of cppc_get_perf_ctrs() to know this ahead of time.
+ *
+ * Return: true if any of the counters are in PCC regions, false otherwise
+ */
+bool cppc_perf_ctrs_in_pcc(void)
+{
+	int cpu;
+
+	for_each_present_cpu(cpu) {
+		struct cpc_register_resource *ref_perf_reg;
+		struct cpc_desc *cpc_desc;
+
+		cpc_desc = per_cpu(cpc_desc_ptr, cpu);
+
+		if (CPC_IN_PCC(&cpc_desc->cpc_regs[DELIVERED_CTR]) ||
+		    CPC_IN_PCC(&cpc_desc->cpc_regs[REFERENCE_CTR]) ||
+		    CPC_IN_PCC(&cpc_desc->cpc_regs[CTR_WRAP_TIME]))
+			return true;
+
+
+		ref_perf_reg = &cpc_desc->cpc_regs[REFERENCE_PERF];
+
+		/*
+		 * If reference perf register is not supported then we should
+		 * use the nominal perf value
+		 */
+		if (!CPC_SUPPORTED(ref_perf_reg))
+			ref_perf_reg = &cpc_desc->cpc_regs[NOMINAL_PERF];
+
+		if (CPC_IN_PCC(ref_perf_reg))
+			return true;
+	}
+
+	return false;
+}
+EXPORT_SYMBOL_GPL(cppc_perf_ctrs_in_pcc);
+
+/**
  * cppc_get_perf_ctrs - Read a CPU's performance feedback counters.
  * @cpunum: CPU from which to read counters.
  * @perf_fb_ctrs: ptr to cppc_perf_fb_ctrs. See cppc_acpi.h
diff --git a/drivers/acpi/device_pm.c b/drivers/acpi/device_pm.c
index 9dce1245689c..d594effe905f 100644
--- a/drivers/acpi/device_pm.c
+++ b/drivers/acpi/device_pm.c
@@ -75,15 +75,17 @@ static int acpi_dev_pm_explicit_get(struct acpi_device *device, int *state)
 int acpi_device_get_power(struct acpi_device *device, int *state)
 {
 	int result = ACPI_STATE_UNKNOWN;
+	struct acpi_device *parent;
 	int error;
 
 	if (!device || !state)
 		return -EINVAL;
 
+	parent = acpi_dev_parent(device);
+
 	if (!device->flags.power_manageable) {
 		/* TBD: Non-recursive algorithm for walking up hierarchy. */
-		*state = device->parent ?
-			device->parent->power.state : ACPI_STATE_D0;
+		*state = parent ? parent->power.state : ACPI_STATE_D0;
 		goto out;
 	}
 
@@ -122,10 +124,10 @@ int acpi_device_get_power(struct acpi_device *device, int *state)
 	 * point, the fact that the device is in D0 implies that the parent has
 	 * to be in D0 too, except if ignore_parent is set.
 	 */
-	if (!device->power.flags.ignore_parent && device->parent
-	    && device->parent->power.state == ACPI_STATE_UNKNOWN
-	    && result == ACPI_STATE_D0)
-		device->parent->power.state = ACPI_STATE_D0;
+	if (!device->power.flags.ignore_parent && parent &&
+	    parent->power.state == ACPI_STATE_UNKNOWN &&
+	    result == ACPI_STATE_D0)
+		parent->power.state = ACPI_STATE_D0;
 
 	*state = result;
 
@@ -191,13 +193,17 @@ int acpi_device_set_power(struct acpi_device *device, int state)
 		return -ENODEV;
 	}
 
-	if (!device->power.flags.ignore_parent && device->parent &&
-	    state < device->parent->power.state) {
-		acpi_handle_debug(device->handle,
-				  "Cannot transition to %s for parent in %s\n",
-				  acpi_power_state_string(state),
-				  acpi_power_state_string(device->parent->power.state));
-		return -ENODEV;
+	if (!device->power.flags.ignore_parent) {
+		struct acpi_device *parent;
+
+		parent = acpi_dev_parent(device);
+		if (parent && state < parent->power.state) {
+			acpi_handle_debug(device->handle,
+					  "Cannot transition to %s for parent in %s\n",
+					  acpi_power_state_string(state),
+					  acpi_power_state_string(parent->power.state));
+			return -ENODEV;
+		}
 	}
 
 	/*
@@ -497,7 +503,7 @@ static void acpi_pm_notify_handler(acpi_handle handle, u32 val, void *not_used)
 
 	acpi_handle_debug(handle, "Wake notify\n");
 
-	adev = acpi_bus_get_acpi_device(handle);
+	adev = acpi_get_acpi_dev(handle);
 	if (!adev)
 		return;
 
@@ -515,7 +521,7 @@ static void acpi_pm_notify_handler(acpi_handle handle, u32 val, void *not_used)
 
 	mutex_unlock(&acpi_pm_notifier_lock);
 
-	acpi_bus_put_acpi_device(adev);
+	acpi_put_acpi_dev(adev);
 }
 
 /**
@@ -1460,7 +1466,7 @@ EXPORT_SYMBOL_GPL(acpi_storage_d3);
  * not valid to ask for the ACPI power state of the device in that time frame.
  *
  * This function is intended to be used in a driver's probe or remove
- * function. See Documentation/firmware-guide/acpi/low-power-probe.rst for
+ * function. See Documentation/firmware-guide/acpi/non-d0-probe.rst for
  * more information.
  */
 bool acpi_dev_state_d0(struct device *dev)
diff --git a/drivers/acpi/dptf/Kconfig b/drivers/acpi/dptf/Kconfig
index 1e8c7ce89bf1..4b3fdc03e4ed 100644
--- a/drivers/acpi/dptf/Kconfig
+++ b/drivers/acpi/dptf/Kconfig
@@ -11,9 +11,6 @@ menuconfig ACPI_DPTF
 	  a coordinated approach for different policies to effect the hardware
 	  state of a system.
 
-	  For more information see:
-	  <https://01.org/intel%C2%AE-dynamic-platform-and-thermal-framework-dptf-chromium-os/overview>
-
 if ACPI_DPTF
 
 config DPTF_POWER
diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c
index c95e535035a0..9b42628cf21b 100644
--- a/drivers/acpi/ec.c
+++ b/drivers/acpi/ec.c
@@ -917,14 +917,10 @@ EXPORT_SYMBOL(ec_read);
 
 int ec_write(u8 addr, u8 val)
 {
-	int err;
-
 	if (!first_ec)
 		return -ENODEV;
 
-	err = acpi_ec_write(first_ec, addr, val);
-
-	return err;
+	return acpi_ec_write(first_ec, addr, val);
 }
 EXPORT_SYMBOL(ec_write);
 
diff --git a/drivers/acpi/fan_core.c b/drivers/acpi/fan_core.c
index b9a9a59ddcc1..52a0b303b70a 100644
--- a/drivers/acpi/fan_core.c
+++ b/drivers/acpi/fan_core.c
@@ -19,43 +19,12 @@
 
 #include "fan.h"
 
-MODULE_AUTHOR("Paul Diefenbaugh");
-MODULE_DESCRIPTION("ACPI Fan Driver");
-MODULE_LICENSE("GPL");
-
-static int acpi_fan_probe(struct platform_device *pdev);
-static int acpi_fan_remove(struct platform_device *pdev);
-
 static const struct acpi_device_id fan_device_ids[] = {
 	ACPI_FAN_DEVICE_IDS,
 	{"", 0},
 };
 MODULE_DEVICE_TABLE(acpi, fan_device_ids);
 
-#ifdef CONFIG_PM_SLEEP
-static int acpi_fan_suspend(struct device *dev);
-static int acpi_fan_resume(struct device *dev);
-static const struct dev_pm_ops acpi_fan_pm = {
-	.resume = acpi_fan_resume,
-	.freeze = acpi_fan_suspend,
-	.thaw = acpi_fan_resume,
-	.restore = acpi_fan_resume,
-};
-#define FAN_PM_OPS_PTR (&acpi_fan_pm)
-#else
-#define FAN_PM_OPS_PTR NULL
-#endif
-
-static struct platform_driver acpi_fan_driver = {
-	.probe = acpi_fan_probe,
-	.remove = acpi_fan_remove,
-	.driver = {
-		.name = "acpi-fan",
-		.acpi_match_table = fan_device_ids,
-		.pm = FAN_PM_OPS_PTR,
-	},
-};
-
 /* thermal cooling device callbacks */
 static int fan_get_max_state(struct thermal_cooling_device *cdev, unsigned long
 			     *state)
@@ -459,6 +428,33 @@ static int acpi_fan_resume(struct device *dev)
 
 	return result;
 }
+
+static const struct dev_pm_ops acpi_fan_pm = {
+	.resume = acpi_fan_resume,
+	.freeze = acpi_fan_suspend,
+	.thaw = acpi_fan_resume,
+	.restore = acpi_fan_resume,
+};
+#define FAN_PM_OPS_PTR (&acpi_fan_pm)
+
+#else
+
+#define FAN_PM_OPS_PTR NULL
+
 #endif
 
+static struct platform_driver acpi_fan_driver = {
+	.probe = acpi_fan_probe,
+	.remove = acpi_fan_remove,
+	.driver = {
+		.name = "acpi-fan",
+		.acpi_match_table = fan_device_ids,
+		.pm = FAN_PM_OPS_PTR,
+	},
+};
+
 module_platform_driver(acpi_fan_driver);
+
+MODULE_AUTHOR("Paul Diefenbaugh");
+MODULE_DESCRIPTION("ACPI Fan Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h
index 628bf8f18130..219c02df9a08 100644
--- a/drivers/acpi/internal.h
+++ b/drivers/acpi/internal.h
@@ -102,10 +102,10 @@ struct acpi_device_bus_id {
 	struct list_head node;
 };
 
-int acpi_device_add(struct acpi_device *device,
-		    void (*release)(struct device *));
 void acpi_init_device_object(struct acpi_device *device, acpi_handle handle,
-			     int type);
+			     int type, void (*release)(struct device *));
+int acpi_tie_acpi_dev(struct acpi_device *adev);
+int acpi_device_add(struct acpi_device *device);
 int acpi_device_setup_files(struct acpi_device *dev);
 void acpi_device_remove_files(struct acpi_device *dev);
 void acpi_device_add_finalize(struct acpi_device *device);
diff --git a/drivers/acpi/irq.c b/drivers/acpi/irq.c
index dabe45eba055..4db5bb587599 100644
--- a/drivers/acpi/irq.c
+++ b/drivers/acpi/irq.c
@@ -118,12 +118,12 @@ acpi_get_irq_source_fwhandle(const struct acpi_resource_source *source,
 	if (WARN_ON(ACPI_FAILURE(status)))
 		return NULL;
 
-	device = acpi_bus_get_acpi_device(handle);
+	device = acpi_get_acpi_dev(handle);
 	if (WARN_ON(!device))
 		return NULL;
 
 	result = &device->fwnode;
-	acpi_bus_put_acpi_device(device);
+	acpi_put_acpi_dev(device);
 	return result;
 }
 
diff --git a/drivers/acpi/numa/hmat.c b/drivers/acpi/numa/hmat.c
index c3d783aca196..23f49a2f4d14 100644
--- a/drivers/acpi/numa/hmat.c
+++ b/drivers/acpi/numa/hmat.c
@@ -9,7 +9,6 @@
  */
 
 #define pr_fmt(fmt) "acpi/hmat: " fmt
-#define dev_fmt(fmt) "acpi/hmat: " fmt
 
 #include <linux/acpi.h>
 #include <linux/bitops.h>
@@ -302,7 +301,7 @@ static __init int hmat_parse_locality(union acpi_subtable_headers *header,
 	u8 type, mem_hier;
 
 	if (hmat_loc->header.length < sizeof(*hmat_loc)) {
-		pr_notice("HMAT: Unexpected locality header length: %u\n",
+		pr_notice("Unexpected locality header length: %u\n",
 			 hmat_loc->header.length);
 		return -EINVAL;
 	}
@@ -314,12 +313,12 @@ static __init int hmat_parse_locality(union acpi_subtable_headers *header,
 	total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
 		     sizeof(*inits) * ipds + sizeof(*targs) * tpds;
 	if (hmat_loc->header.length < total_size) {
-		pr_notice("HMAT: Unexpected locality header length:%u, minimum required:%u\n",
+		pr_notice("Unexpected locality header length:%u, minimum required:%u\n",
 			 hmat_loc->header.length, total_size);
 		return -EINVAL;
 	}
 
-	pr_info("HMAT: Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
+	pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
 		hmat_loc->flags, hmat_data_type(type), ipds, tpds,
 		hmat_loc->entry_base_unit);
 
@@ -363,13 +362,13 @@ static __init int hmat_parse_cache(union acpi_subtable_headers *header,
 	u32 attrs;
 
 	if (cache->header.length < sizeof(*cache)) {
-		pr_notice("HMAT: Unexpected cache header length: %u\n",
+		pr_notice("Unexpected cache header length: %u\n",
 			 cache->header.length);
 		return -EINVAL;
 	}
 
 	attrs = cache->cache_attributes;
-	pr_info("HMAT: Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
+	pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
 		cache->memory_PD, cache->cache_size, attrs,
 		cache->number_of_SMBIOShandles);
 
@@ -424,24 +423,24 @@ static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *heade
 	struct memory_target *target = NULL;
 
 	if (p->header.length != sizeof(*p)) {
-		pr_notice("HMAT: Unexpected address range header length: %u\n",
+		pr_notice("Unexpected address range header length: %u\n",
 			 p->header.length);
 		return -EINVAL;
 	}
 
 	if (hmat_revision == 1)
-		pr_info("HMAT: Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
+		pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
 			p->reserved3, p->reserved4, p->flags, p->processor_PD,
 			p->memory_PD);
 	else
-		pr_info("HMAT: Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
+		pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
 			p->flags, p->processor_PD, p->memory_PD);
 
 	if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||
 	    hmat_revision > 1) {
 		target = find_mem_target(p->memory_PD);
 		if (!target) {
-			pr_debug("HMAT: Memory Domain missing from SRAT\n");
+			pr_debug("Memory Domain missing from SRAT\n");
 			return -EINVAL;
 		}
 	}
@@ -449,7 +448,7 @@ static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *heade
 		int p_node = pxm_to_node(p->processor_PD);
 
 		if (p_node == NUMA_NO_NODE) {
-			pr_debug("HMAT: Invalid Processor Domain\n");
+			pr_debug("Invalid Processor Domain\n");
 			return -EINVAL;
 		}
 		target->processor_pxm = p->processor_PD;
@@ -840,7 +839,7 @@ static __init int hmat_init(void)
 	case 2:
 		break;
 	default:
-		pr_notice("Ignoring HMAT: Unknown revision:%d\n", hmat_revision);
+		pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision);
 		goto out_put;
 	}
 
@@ -848,7 +847,7 @@ static __init int hmat_init(void)
 		if (acpi_table_parse_entries(ACPI_SIG_HMAT,
 					     sizeof(struct acpi_table_hmat), i,
 					     hmat_parse_subtable, 0) < 0) {
-			pr_notice("Ignoring HMAT: Invalid table");
+			pr_notice("Ignoring: Invalid table");
 			goto out_put;
 		}
 	}
diff --git a/drivers/acpi/osi.c b/drivers/acpi/osi.c
index 9f6853809138..d4405e1ca9b9 100644
--- a/drivers/acpi/osi.c
+++ b/drivers/acpi/osi.c
@@ -44,30 +44,6 @@ osi_setup_entries[OSI_STRING_ENTRIES_MAX] __initdata = {
 	{"Processor Device", true},
 	{"3.0 _SCP Extensions", true},
 	{"Processor Aggregator Device", true},
-	/*
-	 * Linux-Dell-Video is used by BIOS to disable RTD3 for NVidia graphics
-	 * cards as RTD3 is not supported by drivers now.  Systems with NVidia
-	 * cards will hang without RTD3 disabled.
-	 *
-	 * Once NVidia drivers officially support RTD3, this _OSI strings can
-	 * be removed if both new and old graphics cards are supported.
-	 */
-	{"Linux-Dell-Video", true},
-	/*
-	 * Linux-Lenovo-NV-HDMI-Audio is used by BIOS to power on NVidia's HDMI
-	 * audio device which is turned off for power-saving in Windows OS.
-	 * This power management feature observed on some Lenovo Thinkpad
-	 * systems which will not be able to output audio via HDMI without
-	 * a BIOS workaround.
-	 */
-	{"Linux-Lenovo-NV-HDMI-Audio", true},
-	/*
-	 * Linux-HPI-Hybrid-Graphics is used by BIOS to enable dGPU to
-	 * output video directly to external monitors on HP Inc. mobile
-	 * workstations as Nvidia and AMD VGA drivers provide limited
-	 * hybrid graphics supports.
-	 */
-	{"Linux-HPI-Hybrid-Graphics", true},
 };
 
 static u32 acpi_osi_handler(acpi_string interface, u32 supported)
diff --git a/drivers/acpi/pci_root.c b/drivers/acpi/pci_root.c
index d57cf8454b93..c8385ef54c37 100644
--- a/drivers/acpi/pci_root.c
+++ b/drivers/acpi/pci_root.c
@@ -312,76 +312,25 @@ struct acpi_handle_node {
  */
 struct pci_dev *acpi_get_pci_dev(acpi_handle handle)
 {
-	int dev, fn;
-	unsigned long long adr;
-	acpi_status status;
-	acpi_handle phandle;
-	struct pci_bus *pbus;
-	struct pci_dev *pdev = NULL;
-	struct acpi_handle_node *node, *tmp;
-	struct acpi_pci_root *root;
-	LIST_HEAD(device_list);
-
-	/*
-	 * Walk up the ACPI CA namespace until we reach a PCI root bridge.
-	 */
-	phandle = handle;
-	while (!acpi_is_root_bridge(phandle)) {
-		node = kzalloc(sizeof(struct acpi_handle_node), GFP_KERNEL);
-		if (!node)
-			goto out;
-
-		INIT_LIST_HEAD(&node->node);
-		node->handle = phandle;
-		list_add(&node->node, &device_list);
-
-		status = acpi_get_parent(phandle, &phandle);
-		if (ACPI_FAILURE(status))
-			goto out;
-	}
-
-	root = acpi_pci_find_root(phandle);
-	if (!root)
-		goto out;
+	struct acpi_device *adev = acpi_fetch_acpi_dev(handle);
+	struct acpi_device_physical_node *pn;
+	struct pci_dev *pci_dev = NULL;
 
-	pbus = root->bus;
-
-	/*
-	 * Now, walk back down the PCI device tree until we return to our
-	 * original handle. Assumes that everything between the PCI root
-	 * bridge and the device we're looking for must be a P2P bridge.
-	 */
-	list_for_each_entry(node, &device_list, node) {
-		acpi_handle hnd = node->handle;
-		status = acpi_evaluate_integer(hnd, "_ADR", NULL, &adr);
-		if (ACPI_FAILURE(status))
-			goto out;
-		dev = (adr >> 16) & 0xffff;
-		fn  = adr & 0xffff;
-
-		pdev = pci_get_slot(pbus, PCI_DEVFN(dev, fn));
-		if (!pdev || hnd == handle)
-			break;
+	if (!adev)
+		return NULL;
 
-		pbus = pdev->subordinate;
-		pci_dev_put(pdev);
+	mutex_lock(&adev->physical_node_lock);
 
-		/*
-		 * This function may be called for a non-PCI device that has a
-		 * PCI parent (eg. a disk under a PCI SATA controller).  In that
-		 * case pdev->subordinate will be NULL for the parent.
-		 */
-		if (!pbus) {
-			dev_dbg(&pdev->dev, "Not a PCI-to-PCI bridge\n");
-			pdev = NULL;
+	list_for_each_entry(pn, &adev->physical_node_list, node) {
+		if (dev_is_pci(pn->dev)) {
+			pci_dev = to_pci_dev(pn->dev);
 			break;
 		}
 	}
-out:
-	list_for_each_entry_safe(node, tmp, &device_list, node)
-		kfree(node);
 
-	return pdev;
+	mutex_unlock(&adev->physical_node_lock);
+
+	return pci_dev;
 }
 EXPORT_SYMBOL_GPL(acpi_get_pci_dev);
 
diff --git a/drivers/acpi/power.c b/drivers/acpi/power.c
index 8c4a73a1351e..f2588aba8421 100644
--- a/drivers/acpi/power.c
+++ b/drivers/acpi/power.c
@@ -944,13 +944,15 @@ struct acpi_device *acpi_add_power_resource(acpi_handle handle)
 		return NULL;
 
 	device = &resource->device;
-	acpi_init_device_object(device, handle, ACPI_BUS_TYPE_POWER);
+	acpi_init_device_object(device, handle, ACPI_BUS_TYPE_POWER,
+				acpi_release_power_resource);
 	mutex_init(&resource->resource_lock);
 	INIT_LIST_HEAD(&resource->list_node);
 	INIT_LIST_HEAD(&resource->dependents);
 	strcpy(acpi_device_name(device), ACPI_POWER_DEVICE_NAME);
 	strcpy(acpi_device_class(device), ACPI_POWER_CLASS);
 	device->power.state = ACPI_STATE_UNKNOWN;
+	device->flags.match_driver = true;
 
 	/* Evaluate the object to get the system level and resource order. */
 	status = acpi_evaluate_object(handle, NULL, NULL, &buffer);
@@ -967,8 +969,11 @@ struct acpi_device *acpi_add_power_resource(acpi_handle handle)
 
 	pr_info("%s [%s]\n", acpi_device_name(device), acpi_device_bid(device));
 
-	device->flags.match_driver = true;
-	result = acpi_device_add(device, acpi_release_power_resource);
+	result = acpi_tie_acpi_dev(device);
+	if (result)
+		goto err;
+
+	result = acpi_device_add(device);
 	if (result)
 		goto err;
 
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index 16a1663d02d4..acfabfe07c4f 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -531,10 +531,27 @@ static void wait_for_freeze(void)
 	/* No delay is needed if we are in guest */
 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR))
 		return;
+	/*
+	 * Modern (>=Nehalem) Intel systems use ACPI via intel_idle,
+	 * not this code.  Assume that any Intel systems using this
+	 * are ancient and may need the dummy wait.  This also assumes
+	 * that the motivating chipset issue was Intel-only.
+	 */
+	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
+		return;
 #endif
-	/* Dummy wait op - must do something useless after P_LVL2 read
-	   because chipsets cannot guarantee that STPCLK# signal
-	   gets asserted in time to freeze execution properly. */
+	/*
+	 * Dummy wait op - must do something useless after P_LVL2 read
+	 * because chipsets cannot guarantee that STPCLK# signal gets
+	 * asserted in time to freeze execution properly
+	 *
+	 * This workaround has been in place since the original ACPI
+	 * implementation was merged, circa 2002.
+	 *
+	 * If a profile is pointing to this instruction, please first
+	 * consider moving your system to a more modern idle
+	 * mechanism.
+	 */
 	inl(acpi_gbl_FADT.xpm_timer_block.address);
 }
 
@@ -787,7 +804,7 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)
 
 		state = &drv->states[count];
 		snprintf(state->name, CPUIDLE_NAME_LEN, "C%d", i);
-		strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
+		strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
 		state->exit_latency = cx->latency;
 		state->target_residency = cx->latency * latency_factor;
 		state->enter = acpi_idle_enter;
@@ -956,7 +973,7 @@ static int acpi_processor_evaluate_lpi(acpi_handle handle,
 
 		obj = pkg_elem + 9;
 		if (obj->type == ACPI_TYPE_STRING)
-			strlcpy(lpi_state->desc, obj->string.pointer,
+			strscpy(lpi_state->desc, obj->string.pointer,
 				ACPI_CX_DESC_LEN);
 
 		lpi_state->index = state_idx;
@@ -1022,7 +1039,7 @@ static bool combine_lpi_states(struct acpi_lpi_state *local,
 	result->arch_flags = parent->arch_flags;
 	result->index = parent->index;
 
-	strlcpy(result->desc, local->desc, ACPI_CX_DESC_LEN);
+	strscpy(result->desc, local->desc, ACPI_CX_DESC_LEN);
 	strlcat(result->desc, "+", ACPI_CX_DESC_LEN);
 	strlcat(result->desc, parent->desc, ACPI_CX_DESC_LEN);
 	return true;
@@ -1196,7 +1213,7 @@ static int acpi_processor_setup_lpi_states(struct acpi_processor *pr)
 
 		state = &drv->states[i];
 		snprintf(state->name, CPUIDLE_NAME_LEN, "LPI-%d", i);
-		strlcpy(state->desc, lpi->desc, CPUIDLE_DESC_LEN);
+		strscpy(state->desc, lpi->desc, CPUIDLE_DESC_LEN);
 		state->exit_latency = lpi->wake_latency;
 		state->target_residency = lpi->min_residency;
 		if (lpi->arch_flags)
diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c
index d4c168ce428c..b8d9eb9a433e 100644
--- a/drivers/acpi/property.c
+++ b/drivers/acpi/property.c
@@ -304,8 +304,10 @@ static void acpi_init_of_compatible(struct acpi_device *adev)
 		ret = acpi_dev_get_property(adev, "compatible",
 					    ACPI_TYPE_STRING, &of_compatible);
 		if (ret) {
-			if (adev->parent
-			    && adev->parent->flags.of_compatible_ok)
+			struct acpi_device *parent;
+
+			parent = acpi_dev_parent(adev);
+			if (parent && parent->flags.of_compatible_ok)
 				goto out;
 
 			return;
@@ -1267,10 +1269,11 @@ acpi_node_get_parent(const struct fwnode_handle *fwnode)
 		return to_acpi_data_node(fwnode)->parent;
 	}
 	if (is_acpi_device_node(fwnode)) {
-		struct device *dev = to_acpi_device_node(fwnode)->dev.parent;
+		struct acpi_device *parent;
 
-		if (dev)
-			return acpi_fwnode_handle(to_acpi_device(dev));
+		parent = acpi_dev_parent(to_acpi_device_node(fwnode));
+		if (parent)
+			return acpi_fwnode_handle(parent);
 	}
 
 	return NULL;
diff --git a/drivers/acpi/resource.c b/drivers/acpi/resource.c
index 510cdec375c4..514d89656dde 100644
--- a/drivers/acpi/resource.c
+++ b/drivers/acpi/resource.c
@@ -399,6 +399,31 @@ static const struct dmi_system_id medion_laptop[] = {
 	{ }
 };
 
+static const struct dmi_system_id asus_laptop[] = {
+	{
+		.ident = "Asus Vivobook K3402ZA",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_BOARD_NAME, "K3402ZA"),
+		},
+	},
+	{
+		.ident = "Asus Vivobook K3502ZA",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_BOARD_NAME, "K3502ZA"),
+		},
+	},
+	{
+		.ident = "Asus Vivobook S5402ZA",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_BOARD_NAME, "S5402ZA"),
+		},
+	},
+	{ }
+};
+
 struct irq_override_cmp {
 	const struct dmi_system_id *system;
 	unsigned char irq;
@@ -409,6 +434,7 @@ struct irq_override_cmp {
 
 static const struct irq_override_cmp skip_override_table[] = {
 	{ medion_laptop, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0 },
+	{ asus_laptop, 1, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_LOW, 0 },
 };
 
 static bool acpi_dev_irq_override(u32 gsi, u8 triggering, u8 polarity,
@@ -690,6 +716,9 @@ static int is_memory(struct acpi_resource *ares, void *not_used)
 
 	memset(&win, 0, sizeof(win));
 
+	if (acpi_dev_filter_resource_type(ares, IORESOURCE_MEM))
+		return 1;
+
 	return !(acpi_dev_resource_memory(ares, res)
 	       || acpi_dev_resource_address_space(ares, &win)
 	       || acpi_dev_resource_ext_address_space(ares, &win));
@@ -719,6 +748,23 @@ int acpi_dev_get_dma_resources(struct acpi_device *adev, struct list_head *list)
 EXPORT_SYMBOL_GPL(acpi_dev_get_dma_resources);
 
 /**
+ * acpi_dev_get_memory_resources - Get current memory resources of a device.
+ * @adev: ACPI device node to get the resources for.
+ * @list: Head of the resultant list of resources (must be empty).
+ *
+ * This is a helper function that locates all memory type resources of @adev
+ * with acpi_dev_get_resources().
+ *
+ * The number of resources in the output list is returned on success, an error
+ * code reflecting the error condition is returned otherwise.
+ */
+int acpi_dev_get_memory_resources(struct acpi_device *adev, struct list_head *list)
+{
+	return acpi_dev_get_resources(adev, list, is_memory, NULL);
+}
+EXPORT_SYMBOL_GPL(acpi_dev_get_memory_resources);
+
+/**
  * acpi_dev_filter_resource_type - Filter ACPI resource according to resource
  *				   types
  * @ares: Input ACPI resource object.
diff --git a/drivers/acpi/sbs.c b/drivers/acpi/sbs.c
index 4938010fcac7..e6a01a8df1b8 100644
--- a/drivers/acpi/sbs.c
+++ b/drivers/acpi/sbs.c
@@ -632,7 +632,7 @@ static int acpi_sbs_add(struct acpi_device *device)
 
 	mutex_init(&sbs->lock);
 
-	sbs->hc = acpi_driver_data(device->parent);
+	sbs->hc = acpi_driver_data(acpi_dev_parent(device));
 	sbs->device = device;
 	strcpy(acpi_device_name(device), ACPI_SBS_DEVICE_NAME);
 	strcpy(acpi_device_class(device), ACPI_SBS_CLASS);
diff --git a/drivers/acpi/sbshc.c b/drivers/acpi/sbshc.c
index 7c62e149a7a1..340e0b61587e 100644
--- a/drivers/acpi/sbshc.c
+++ b/drivers/acpi/sbshc.c
@@ -266,7 +266,7 @@ static int acpi_smbus_hc_add(struct acpi_device *device)
 	mutex_init(&hc->lock);
 	init_waitqueue_head(&hc->wait);
 
-	hc->ec = acpi_driver_data(device->parent);
+	hc->ec = acpi_driver_data(acpi_dev_parent(device));
 	hc->offset = (val >> 8) & 0xff;
 	hc->query_bit = val & 0xff;
 	device->driver_data = hc;
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index 42cec8120f18..558664d169fc 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -20,6 +20,7 @@
 #include <linux/platform_data/x86/apple.h>
 #include <linux/pgtable.h>
 #include <linux/crc32.h>
+#include <linux/dma-direct.h>
 
 #include "internal.h"
 
@@ -29,8 +30,6 @@ extern struct acpi_device *acpi_root;
 #define ACPI_BUS_HID			"LNXSYBUS"
 #define ACPI_BUS_DEVICE_NAME		"System Bus"
 
-#define ACPI_IS_ROOT_DEVICE(device)    (!(device)->parent)
-
 #define INVALID_ACPI_HANDLE	((acpi_handle)empty_zero_page)
 
 static const char *dummy_hid = "device";
@@ -429,7 +428,7 @@ void acpi_device_hotplug(struct acpi_device *adev, u32 src)
 	acpi_evaluate_ost(adev->handle, src, ost_code, NULL);
 
  out:
-	acpi_bus_put_acpi_device(adev);
+	acpi_put_acpi_dev(adev);
 	mutex_unlock(&acpi_scan_lock);
 	unlock_device_hotplug();
 }
@@ -599,11 +598,22 @@ static void get_acpi_device(void *dev)
 	acpi_dev_get(dev);
 }
 
-struct acpi_device *acpi_bus_get_acpi_device(acpi_handle handle)
+/**
+ * acpi_get_acpi_dev - Retrieve ACPI device object and reference count it.
+ * @handle: ACPI handle associated with the requested ACPI device object.
+ *
+ * Return a pointer to the ACPI device object associated with @handle and bump
+ * up that object's reference counter (under the ACPI Namespace lock), if
+ * present, or return NULL otherwise.
+ *
+ * The ACPI device object reference acquired by this function needs to be
+ * dropped via acpi_dev_put().
+ */
+struct acpi_device *acpi_get_acpi_dev(acpi_handle handle)
 {
 	return handle_to_device(handle, get_acpi_device);
 }
-EXPORT_SYMBOL_GPL(acpi_bus_get_acpi_device);
+EXPORT_SYMBOL_GPL(acpi_get_acpi_dev);
 
 static struct acpi_device_bus_id *acpi_device_bus_id_match(const char *dev_id)
 {
@@ -632,7 +642,7 @@ static int acpi_device_set_name(struct acpi_device *device,
 	return 0;
 }
 
-static int acpi_tie_acpi_dev(struct acpi_device *adev)
+int acpi_tie_acpi_dev(struct acpi_device *adev)
 {
 	acpi_handle handle = adev->handle;
 	acpi_status status;
@@ -662,8 +672,7 @@ static void acpi_store_pld_crc(struct acpi_device *adev)
 	ACPI_FREE(pld);
 }
 
-static int __acpi_device_add(struct acpi_device *device,
-			     void (*release)(struct device *))
+int acpi_device_add(struct acpi_device *device)
 {
 	struct acpi_device_bus_id *acpi_device_bus_id;
 	int result;
@@ -719,11 +728,6 @@ static int __acpi_device_add(struct acpi_device *device,
 
 	mutex_unlock(&acpi_device_lock);
 
-	if (device->parent)
-		device->dev.parent = &device->parent->dev;
-
-	device->dev.bus = &acpi_bus_type;
-	device->dev.release = release;
 	result = device_add(&device->dev);
 	if (result) {
 		dev_err(&device->dev, "Error registering device\n");
@@ -750,17 +754,6 @@ err_unlock:
 	return result;
 }
 
-int acpi_device_add(struct acpi_device *adev, void (*release)(struct device *))
-{
-	int ret;
-
-	ret = acpi_tie_acpi_dev(adev);
-	if (ret)
-		return ret;
-
-	return __acpi_device_add(adev, release);
-}
-
 /* --------------------------------------------------------------------------
                                  Device Enumeration
    -------------------------------------------------------------------------- */
@@ -805,10 +798,9 @@ static const char * const acpi_honor_dep_ids[] = {
 	NULL
 };
 
-static struct acpi_device *acpi_bus_get_parent(acpi_handle handle)
+static struct acpi_device *acpi_find_parent_acpi_dev(acpi_handle handle)
 {
-	struct acpi_device *device;
-	acpi_status status;
+	struct acpi_device *adev;
 
 	/*
 	 * Fixed hardware devices do not appear in the namespace and do not
@@ -819,13 +811,18 @@ static struct acpi_device *acpi_bus_get_parent(acpi_handle handle)
 		return acpi_root;
 
 	do {
+		acpi_status status;
+
 		status = acpi_get_parent(handle, &handle);
-		if (ACPI_FAILURE(status))
-			return status == AE_NULL_ENTRY ? NULL : acpi_root;
+		if (ACPI_FAILURE(status)) {
+			if (status != AE_NULL_ENTRY)
+				return acpi_root;
 
-		device = acpi_fetch_acpi_dev(handle);
-	} while (!device);
-	return device;
+			return NULL;
+		}
+		adev = acpi_fetch_acpi_dev(handle);
+	} while (!adev);
+	return adev;
 }
 
 acpi_status
@@ -1112,7 +1109,7 @@ static void acpi_device_get_busid(struct acpi_device *device)
 	 * The device's Bus ID is simply the object name.
 	 * TBD: Shouldn't this value be unique (within the ACPI namespace)?
 	 */
-	if (ACPI_IS_ROOT_DEVICE(device)) {
+	if (!acpi_dev_parent(device)) {
 		strcpy(device->pnp.bus_id, "ACPI");
 		return;
 	}
@@ -1467,25 +1464,21 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev)
  * acpi_dma_get_range() - Get device DMA parameters.
  *
  * @dev: device to configure
- * @dma_addr: pointer device DMA address result
- * @offset: pointer to the DMA offset result
- * @size: pointer to DMA range size result
+ * @map: pointer to DMA ranges result
  *
- * Evaluate DMA regions and return respectively DMA region start, offset
- * and size in dma_addr, offset and size on parsing success; it does not
- * update the passed in values on failure.
+ * Evaluate DMA regions and return pointer to DMA regions on
+ * parsing success; it does not update the passed in values on failure.
  *
  * Return 0 on success, < 0 on failure.
  */
-int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset,
-		       u64 *size)
+int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map)
 {
 	struct acpi_device *adev;
 	LIST_HEAD(list);
 	struct resource_entry *rentry;
 	int ret;
 	struct device *dma_dev = dev;
-	u64 len, dma_start = U64_MAX, dma_end = 0, dma_offset = 0;
+	struct bus_dma_region *r;
 
 	/*
 	 * Walk the device tree chasing an ACPI companion with a _DMA
@@ -1510,31 +1503,28 @@ int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset,
 
 	ret = acpi_dev_get_dma_resources(adev, &list);
 	if (ret > 0) {
+		r = kcalloc(ret + 1, sizeof(*r), GFP_KERNEL);
+		if (!r) {
+			ret = -ENOMEM;
+			goto out;
+		}
+
 		list_for_each_entry(rentry, &list, node) {
-			if (dma_offset && rentry->offset != dma_offset) {
+			if (rentry->res->start >= rentry->res->end) {
+				kfree(r);
 				ret = -EINVAL;
-				dev_warn(dma_dev, "Can't handle multiple windows with different offsets\n");
+				dev_dbg(dma_dev, "Invalid DMA regions configuration\n");
 				goto out;
 			}
-			dma_offset = rentry->offset;
 
-			/* Take lower and upper limits */
-			if (rentry->res->start < dma_start)
-				dma_start = rentry->res->start;
-			if (rentry->res->end > dma_end)
-				dma_end = rentry->res->end;
-		}
-
-		if (dma_start >= dma_end) {
-			ret = -EINVAL;
-			dev_dbg(dma_dev, "Invalid DMA regions configuration\n");
-			goto out;
+			r->cpu_start = rentry->res->start;
+			r->dma_start = rentry->res->start - rentry->offset;
+			r->size = resource_size(rentry->res);
+			r->offset = rentry->offset;
+			r++;
 		}
 
-		*dma_addr = dma_start - dma_offset;
-		len = dma_end - dma_start;
-		*size = max(len, len + 1);
-		*offset = dma_offset;
+		*map = r;
 	}
  out:
 	acpi_dev_free_resource_list(&list);
@@ -1624,20 +1614,19 @@ int acpi_dma_configure_id(struct device *dev, enum dev_dma_attr attr,
 			  const u32 *input_id)
 {
 	const struct iommu_ops *iommu;
-	u64 dma_addr = 0, size = 0;
 
 	if (attr == DEV_DMA_NOT_SUPPORTED) {
 		set_dma_ops(dev, &dma_dummy_ops);
 		return 0;
 	}
 
-	acpi_arch_dma_setup(dev, &dma_addr, &size);
+	acpi_arch_dma_setup(dev);
 
 	iommu = acpi_iommu_configure_id(dev, input_id);
 	if (PTR_ERR(iommu) == -EPROBE_DEFER)
 		return -EPROBE_DEFER;
 
-	arch_setup_dma_ops(dev, dma_addr, size,
+	arch_setup_dma_ops(dev, 0, U64_MAX,
 				iommu, attr == DEV_DMA_COHERENT);
 
 	return 0;
@@ -1648,7 +1637,7 @@ static void acpi_init_coherency(struct acpi_device *adev)
 {
 	unsigned long long cca = 0;
 	acpi_status status;
-	struct acpi_device *parent = adev->parent;
+	struct acpi_device *parent = acpi_dev_parent(adev);
 
 	if (parent && parent->flags.cca_seen) {
 		/*
@@ -1692,7 +1681,7 @@ static int acpi_check_serial_bus_slave(struct acpi_resource *ares, void *data)
 
 static bool acpi_is_indirect_io_slave(struct acpi_device *device)
 {
-	struct acpi_device *parent = device->parent;
+	struct acpi_device *parent = acpi_dev_parent(device);
 	static const struct acpi_device_id indirect_io_hosts[] = {
 		{"HISI0191", 0},
 		{}
@@ -1762,12 +1751,16 @@ static bool acpi_device_enumeration_by_parent(struct acpi_device *device)
 }
 
 void acpi_init_device_object(struct acpi_device *device, acpi_handle handle,
-			     int type)
+			     int type, void (*release)(struct device *))
 {
+	struct acpi_device *parent = acpi_find_parent_acpi_dev(handle);
+
 	INIT_LIST_HEAD(&device->pnp.ids);
 	device->device_type = type;
 	device->handle = handle;
-	device->parent = acpi_bus_get_parent(handle);
+	device->dev.parent = parent ? &parent->dev : NULL;
+	device->dev.release = release;
+	device->dev.bus = &acpi_bus_type;
 	fwnode_init(&device->fwnode, &acpi_device_fwnode_ops);
 	acpi_set_device_status(device, ACPI_STA_DEFAULT);
 	acpi_device_get_busid(device);
@@ -1821,7 +1814,7 @@ static int acpi_add_single_object(struct acpi_device **child,
 	if (!device)
 		return -ENOMEM;
 
-	acpi_init_device_object(device, handle, type);
+	acpi_init_device_object(device, handle, type, acpi_device_release);
 	/*
 	 * Getting the status is delayed till here so that we can call
 	 * acpi_bus_get_status() and use its quirk handling.  Note that
@@ -1851,7 +1844,7 @@ static int acpi_add_single_object(struct acpi_device **child,
 		mutex_unlock(&acpi_dep_list_lock);
 
 	if (!result)
-		result = __acpi_device_add(device, acpi_device_release);
+		result = acpi_device_add(device);
 
 	if (result) {
 		acpi_device_release(&device->dev);
@@ -1862,8 +1855,8 @@ static int acpi_add_single_object(struct acpi_device **child,
 	acpi_device_add_finalize(device);
 
 	acpi_handle_debug(handle, "Added as %s, parent %s\n",
-			  dev_name(&device->dev), device->parent ?
-				dev_name(&device->parent->dev) : "(null)");
+			  dev_name(&device->dev), device->dev.parent ?
+				dev_name(device->dev.parent) : "(null)");
 
 	*child = device;
 	return 0;
@@ -2235,11 +2228,24 @@ ok:
 	return 0;
 }
 
-static int acpi_dev_get_first_consumer_dev_cb(struct acpi_dep_data *dep, void *data)
+static int acpi_dev_get_next_consumer_dev_cb(struct acpi_dep_data *dep, void *data)
 {
-	struct acpi_device *adev;
+	struct acpi_device **adev_p = data;
+	struct acpi_device *adev = *adev_p;
 
-	adev = acpi_bus_get_acpi_device(dep->consumer);
+	/*
+	 * If we're passed a 'previous' consumer device then we need to skip
+	 * any consumers until we meet the previous one, and then NULL @data
+	 * so the next one can be returned.
+	 */
+	if (adev) {
+		if (dep->consumer == adev->handle)
+			*adev_p = NULL;
+
+		return 0;
+	}
+
+	adev = acpi_get_acpi_dev(dep->consumer);
 	if (adev) {
 		*(struct acpi_device **)data = adev;
 		return 1;
@@ -2292,7 +2298,7 @@ static bool acpi_scan_clear_dep_queue(struct acpi_device *adev)
 
 static int acpi_scan_clear_dep(struct acpi_dep_data *dep, void *data)
 {
-	struct acpi_device *adev = acpi_bus_get_acpi_device(dep->consumer);
+	struct acpi_device *adev = acpi_get_acpi_dev(dep->consumer);
 
 	if (adev) {
 		adev->dep_unmet--;
@@ -2368,25 +2374,32 @@ bool acpi_dev_ready_for_enumeration(const struct acpi_device *device)
 EXPORT_SYMBOL_GPL(acpi_dev_ready_for_enumeration);
 
 /**
- * acpi_dev_get_first_consumer_dev - Return ACPI device dependent on @supplier
+ * acpi_dev_get_next_consumer_dev - Return the next adev dependent on @supplier
  * @supplier: Pointer to the dependee device
+ * @start: Pointer to the current dependent device
  *
- * Returns the first &struct acpi_device which declares itself dependent on
+ * Returns the next &struct acpi_device which declares itself dependent on
  * @supplier via the _DEP buffer, parsed from the acpi_dep_list.
  *
- * The caller is responsible for putting the reference to adev when it is no
- * longer needed.
+ * If the returned adev is not passed as @start to this function, the caller is
+ * responsible for putting the reference to adev when it is no longer needed.
  */
-struct acpi_device *acpi_dev_get_first_consumer_dev(struct acpi_device *supplier)
+struct acpi_device *acpi_dev_get_next_consumer_dev(struct acpi_device *supplier,
+						   struct acpi_device *start)
 {
-	struct acpi_device *adev = NULL;
+	struct acpi_device *adev = start;
 
 	acpi_walk_dep_device_list(supplier->handle,
-				  acpi_dev_get_first_consumer_dev_cb, &adev);
+				  acpi_dev_get_next_consumer_dev_cb, &adev);
+
+	acpi_dev_put(start);
+
+	if (adev == start)
+		return NULL;
 
 	return adev;
 }
-EXPORT_SYMBOL_GPL(acpi_dev_get_first_consumer_dev);
+EXPORT_SYMBOL_GPL(acpi_dev_get_next_consumer_dev);
 
 /**
  * acpi_bus_scan - Add ACPI device node objects in a given namespace scope.
diff --git a/drivers/acpi/sleep.h b/drivers/acpi/sleep.h
index 7fe41ee489d6..d960a238be4e 100644
--- a/drivers/acpi/sleep.h
+++ b/drivers/acpi/sleep.h
@@ -18,6 +18,7 @@ static inline acpi_status acpi_set_waking_vector(u32 wakeup_address)
 extern int acpi_s2idle_begin(void);
 extern int acpi_s2idle_prepare(void);
 extern int acpi_s2idle_prepare_late(void);
+extern void acpi_s2idle_check(void);
 extern bool acpi_s2idle_wake(void);
 extern void acpi_s2idle_restore_early(void);
 extern void acpi_s2idle_restore(void);
diff --git a/drivers/acpi/utils.c b/drivers/acpi/utils.c
index 5a7b8065e77f..2ea14648a661 100644
--- a/drivers/acpi/utils.c
+++ b/drivers/acpi/utils.c
@@ -794,6 +794,30 @@ bool acpi_dev_hid_uid_match(struct acpi_device *adev,
 EXPORT_SYMBOL(acpi_dev_hid_uid_match);
 
 /**
+ * acpi_dev_uid_to_integer - treat ACPI device _UID as integer
+ * @adev: ACPI device to get _UID from
+ * @integer: output buffer for integer
+ *
+ * Considers _UID as integer and converts it to @integer.
+ *
+ * Returns 0 on success, or negative error code otherwise.
+ */
+int acpi_dev_uid_to_integer(struct acpi_device *adev, u64 *integer)
+{
+	const char *uid;
+
+	if (!adev)
+		return -ENODEV;
+
+	uid = acpi_device_uid(adev);
+	if (!uid)
+		return -ENODATA;
+
+	return kstrtou64(uid, 0, integer);
+}
+EXPORT_SYMBOL(acpi_dev_uid_to_integer);
+
+/**
  * acpi_dev_found - Detect presence of a given ACPI device in the namespace.
  * @hid: Hardware ID of the device.
  *
@@ -878,7 +902,7 @@ bool acpi_dev_present(const char *hid, const char *uid, s64 hrv)
 	struct acpi_dev_match_info match = {};
 	struct device *dev;
 
-	strlcpy(match.hid[0].id, hid, sizeof(match.hid[0].id));
+	strscpy(match.hid[0].id, hid, sizeof(match.hid[0].id));
 	match.uid = uid;
 	match.hrv = hrv;
 
@@ -911,7 +935,7 @@ acpi_dev_get_next_match_dev(struct acpi_device *adev, const char *hid, const cha
 	struct acpi_dev_match_info match = {};
 	struct device *dev;
 
-	strlcpy(match.hid[0].id, hid, sizeof(match.hid[0].id));
+	strscpy(match.hid[0].id, hid, sizeof(match.hid[0].id));
 	match.uid = uid;
 	match.hrv = hrv;
 
@@ -961,7 +985,7 @@ EXPORT_SYMBOL(acpi_video_backlight_string);
 
 static int __init acpi_backlight(char *str)
 {
-	strlcpy(acpi_video_backlight_string, str,
+	strscpy(acpi_video_backlight_string, str,
 		sizeof(acpi_video_backlight_string));
 	return 1;
 }
diff --git a/drivers/acpi/video_detect.c b/drivers/acpi/video_detect.c
index 5d7f38016a24..0d9064a9804c 100644
--- a/drivers/acpi/video_detect.c
+++ b/drivers/acpi/video_detect.c
@@ -17,8 +17,9 @@
  * Otherwise vendor specific drivers like thinkpad_acpi, asus-laptop,
  * sony_acpi,... can take care about backlight brightness.
  *
- * Backlight drivers can use acpi_video_get_backlight_type() to determine
- * which driver should handle the backlight.
+ * Backlight drivers can use acpi_video_get_backlight_type() to determine which
+ * driver should handle the backlight. RAW/GPU-driver backlight drivers must
+ * use the acpi_video_backlight_use_native() helper for this.
  *
  * If CONFIG_ACPI_VIDEO is neither set as "compiled in" (y) nor as a module (m)
  * this file will not be compiled and acpi_video_get_backlight_type() will
@@ -27,20 +28,16 @@
 
 #include <linux/export.h>
 #include <linux/acpi.h>
+#include <linux/apple-gmux.h>
 #include <linux/backlight.h>
 #include <linux/dmi.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/platform_data/x86/nvidia-wmi-ec-backlight.h>
 #include <linux/types.h>
 #include <linux/workqueue.h>
 #include <acpi/video.h>
 
-void acpi_video_unregister_backlight(void);
-
-static bool backlight_notifier_registered;
-static struct notifier_block backlight_nb;
-static struct work_struct backlight_notify_work;
-
 static enum acpi_backlight_type acpi_backlight_cmdline = acpi_backlight_undef;
 static enum acpi_backlight_type acpi_backlight_dmi = acpi_backlight_undef;
 
@@ -78,6 +75,36 @@ find_video(acpi_handle handle, u32 lvl, void *context, void **rv)
 	return AE_OK;
 }
 
+/* This depends on ACPI_WMI which is X86 only */
+#ifdef CONFIG_X86
+static bool nvidia_wmi_ec_supported(void)
+{
+	struct wmi_brightness_args args = {
+		.mode = WMI_BRIGHTNESS_MODE_GET,
+		.val = 0,
+		.ret = 0,
+	};
+	struct acpi_buffer buf = { (acpi_size)sizeof(args), &args };
+	acpi_status status;
+
+	status = wmi_evaluate_method(WMI_BRIGHTNESS_GUID, 0,
+				     WMI_BRIGHTNESS_METHOD_SOURCE, &buf, &buf);
+	if (ACPI_FAILURE(status))
+		return false;
+
+	/*
+	 * If brightness is handled by the EC then nvidia-wmi-ec-backlight
+	 * should be used, else the GPU driver(s) should be used.
+	 */
+	return args.ret == WMI_BRIGHTNESS_SOURCE_EC;
+}
+#else
+static bool nvidia_wmi_ec_supported(void)
+{
+	return false;
+}
+#endif
+
 /* Force to use vendor driver when the ACPI device is known to be
  * buggy */
 static int video_detect_force_vendor(const struct dmi_system_id *d)
@@ -105,63 +132,143 @@ static int video_detect_force_none(const struct dmi_system_id *d)
 }
 
 static const struct dmi_system_id video_detect_dmi_table[] = {
-	/* On Samsung X360, the BIOS will set a flag (VDRV) if generic
-	 * ACPI backlight device is used. This flag will definitively break
-	 * the backlight interface (even the vendor interface) until next
-	 * reboot. It's why we should prevent video.ko from being used here
-	 * and we can't rely on a later call to acpi_video_unregister().
-	 */
 	{
+	 /* https://bugzilla.redhat.com/show_bug.cgi?id=1128309 */
 	 .callback = video_detect_force_vendor,
-	 /* X360 */
+	 /* Acer KAV80 */
 	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
-		DMI_MATCH(DMI_PRODUCT_NAME, "X360"),
-		DMI_MATCH(DMI_BOARD_NAME, "X360"),
+		DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "KAV80"),
 		},
 	},
 	{
-	.callback = video_detect_force_vendor,
-	/* Asus UL30VT */
-	.matches = {
+	 .callback = video_detect_force_vendor,
+	 /* Asus UL30VT */
+	 .matches = {
 		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK Computer Inc."),
 		DMI_MATCH(DMI_PRODUCT_NAME, "UL30VT"),
 		},
 	},
 	{
-	.callback = video_detect_force_vendor,
-	/* Asus UL30A */
-	.matches = {
+	 .callback = video_detect_force_vendor,
+	 /* Asus UL30A */
+	 .matches = {
 		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK Computer Inc."),
 		DMI_MATCH(DMI_PRODUCT_NAME, "UL30A"),
 		},
 	},
 	{
-	.callback = video_detect_force_vendor,
-	/* GIGABYTE GB-BXBT-2807 */
-	.matches = {
+	 .callback = video_detect_force_vendor,
+	 /* Asus X55U */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "X55U"),
+		},
+	},
+	{
+	 .callback = video_detect_force_vendor,
+	 /* Asus X101CH */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "X101CH"),
+		},
+	},
+	{
+	 .callback = video_detect_force_vendor,
+	 /* Asus X401U */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "X401U"),
+		},
+	},
+	{
+	 .callback = video_detect_force_vendor,
+	 /* Asus X501U */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "X501U"),
+		},
+	},
+	{
+	 .callback = video_detect_force_vendor,
+	 /* Asus 1015CX */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "1015CX"),
+		},
+	},
+	{
+	 .callback = video_detect_force_vendor,
+	 /* GIGABYTE GB-BXBT-2807 */
+	 .matches = {
 		DMI_MATCH(DMI_SYS_VENDOR, "GIGABYTE"),
 		DMI_MATCH(DMI_PRODUCT_NAME, "GB-BXBT-2807"),
 		},
 	},
 	{
-	.callback = video_detect_force_vendor,
-	/* Sony VPCEH3U1E */
-	.matches = {
+	 .callback = video_detect_force_vendor,
+	 /* Samsung N150/N210/N220 */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "N150/N210/N220"),
+		DMI_MATCH(DMI_BOARD_NAME, "N150/N210/N220"),
+		},
+	},
+	{
+	 .callback = video_detect_force_vendor,
+	 /* Samsung NF110/NF210/NF310 */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "NF110/NF210/NF310"),
+		DMI_MATCH(DMI_BOARD_NAME, "NF110/NF210/NF310"),
+		},
+	},
+	{
+	 .callback = video_detect_force_vendor,
+	 /* Samsung NC210 */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "NC210/NC110"),
+		DMI_MATCH(DMI_BOARD_NAME, "NC210/NC110"),
+		},
+	},
+	{
+	 .callback = video_detect_force_vendor,
+	 /* Sony VPCEH3U1E */
+	 .matches = {
 		DMI_MATCH(DMI_SYS_VENDOR, "Sony Corporation"),
 		DMI_MATCH(DMI_PRODUCT_NAME, "VPCEH3U1E"),
 		},
 	},
 	{
-	.callback = video_detect_force_vendor,
-	/* Xiaomi Mi Pad 2 */
-	.matches = {
+	 .callback = video_detect_force_vendor,
+	 /* Xiaomi Mi Pad 2 */
+	 .matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "Xiaomi Inc"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "Mipad2"),
 		},
 	},
 
 	/*
+	 * Toshiba models with Transflective display, these need to use
+	 * the toshiba_acpi vendor driver for proper Transflective handling.
+	 */
+	{
+	 .callback = video_detect_force_vendor,
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R500"),
+		},
+	},
+	{
+	 .callback = video_detect_force_vendor,
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R600"),
+		},
+	},
+
+	/*
 	 * These models have a working acpi_video backlight control, and using
 	 * native backlight causes a regression where backlight does not work
 	 * when userspace is not handling brightness key events. Disable
@@ -390,6 +497,41 @@ static const struct dmi_system_id video_detect_dmi_table[] = {
 		},
 	},
 	{
+	 /* https://bugzilla.redhat.com/show_bug.cgi?id=1012674 */
+	 .callback = video_detect_force_native,
+	 /* Acer Aspire 5741 */
+	 .matches = {
+		DMI_MATCH(DMI_BOARD_VENDOR, "Acer"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "Aspire 5741"),
+		},
+	},
+	{
+	 /* https://bugzilla.kernel.org/show_bug.cgi?id=42993 */
+	 .callback = video_detect_force_native,
+	 /* Acer Aspire 5750 */
+	 .matches = {
+		DMI_MATCH(DMI_BOARD_VENDOR, "Acer"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "Aspire 5750"),
+		},
+	},
+	{
+	 /* https://bugzilla.kernel.org/show_bug.cgi?id=42833 */
+	 .callback = video_detect_force_native,
+	 /* Acer Extensa 5235 */
+	 .matches = {
+		DMI_MATCH(DMI_BOARD_VENDOR, "Acer"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "Extensa 5235"),
+		},
+	},
+	{
+	 .callback = video_detect_force_native,
+	 /* Acer TravelMate 4750 */
+	 .matches = {
+		DMI_MATCH(DMI_BOARD_VENDOR, "Acer"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "TravelMate 4750"),
+		},
+	},
+	{
 	 /* https://bugzilla.kernel.org/show_bug.cgi?id=207835 */
 	 .callback = video_detect_force_native,
 	 /* Acer TravelMate 5735Z */
@@ -400,120 +542,109 @@ static const struct dmi_system_id video_detect_dmi_table[] = {
 		},
 	},
 	{
-	.callback = video_detect_force_native,
-	/* ASUSTeK COMPUTER INC. GA401 */
-	.matches = {
+	 /* https://bugzilla.kernel.org/show_bug.cgi?id=36322 */
+	 .callback = video_detect_force_native,
+	 /* Acer TravelMate 5760 */
+	 .matches = {
+		DMI_MATCH(DMI_BOARD_VENDOR, "Acer"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "TravelMate 5760"),
+		},
+	},
+	{
+	 .callback = video_detect_force_native,
+	 /* ASUSTeK COMPUTER INC. GA401 */
+	 .matches = {
 		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
 		DMI_MATCH(DMI_PRODUCT_NAME, "GA401"),
 		},
 	},
 	{
-	.callback = video_detect_force_native,
-	/* ASUSTeK COMPUTER INC. GA502 */
-	.matches = {
+	 .callback = video_detect_force_native,
+	 /* ASUSTeK COMPUTER INC. GA502 */
+	 .matches = {
 		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
 		DMI_MATCH(DMI_PRODUCT_NAME, "GA502"),
 		},
 	},
 	{
-	.callback = video_detect_force_native,
-	/* ASUSTeK COMPUTER INC. GA503 */
-	.matches = {
+	 .callback = video_detect_force_native,
+	 /* ASUSTeK COMPUTER INC. GA503 */
+	 .matches = {
 		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
 		DMI_MATCH(DMI_PRODUCT_NAME, "GA503"),
 		},
 	},
-	/*
-	 * Clevo NL5xRU and NL5xNU/TUXEDO Aura 15 Gen1 and Gen2 have both a
-	 * working native and video interface. However the default detection
-	 * mechanism first registers the video interface before unregistering
-	 * it again and switching to the native interface during boot. This
-	 * results in a dangling SBIOS request for backlight change for some
-	 * reason, causing the backlight to switch to ~2% once per boot on the
-	 * first power cord connect or disconnect event. Setting the native
-	 * interface explicitly circumvents this buggy behaviour, by avoiding
-	 * the unregistering process.
-	 */
 	{
-	.callback = video_detect_force_native,
-	.ident = "Clevo NL5xRU",
-	.matches = {
-		DMI_MATCH(DMI_BOARD_NAME, "NL5xRU"),
+	 .callback = video_detect_force_native,
+	 /* Asus UX303UB */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "UX303UB"),
 		},
 	},
 	{
-	.callback = video_detect_force_native,
-	.ident = "Clevo NL5xRU",
-	.matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "TUXEDO"),
-		DMI_MATCH(DMI_BOARD_NAME, "AURA1501"),
+	 .callback = video_detect_force_native,
+	 /* Samsung N150P */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "N150P"),
+		DMI_MATCH(DMI_BOARD_NAME, "N150P"),
 		},
 	},
 	{
-	.callback = video_detect_force_native,
-	.ident = "Clevo NL5xRU",
-	.matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "TUXEDO"),
-		DMI_MATCH(DMI_BOARD_NAME, "EDUBOOK1502"),
+	 .callback = video_detect_force_native,
+	 /* Samsung N145P/N250P/N260P */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "N145P/N250P/N260P"),
+		DMI_MATCH(DMI_BOARD_NAME, "N145P/N250P/N260P"),
 		},
 	},
 	{
-	.callback = video_detect_force_native,
-	.ident = "Clevo NL5xNU",
-	.matches = {
-		DMI_MATCH(DMI_BOARD_NAME, "NL5xNU"),
+	 .callback = video_detect_force_native,
+	 /* Samsung N250P */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
+		DMI_MATCH(DMI_PRODUCT_NAME, "N250P"),
+		DMI_MATCH(DMI_BOARD_NAME, "N250P"),
 		},
 	},
+
 	/*
-	 * The TongFang PF5PU1G, PF4NU1F, PF5NU1G, and PF5LUXG/TUXEDO BA15 Gen10,
-	 * Pulse 14/15 Gen1, and Pulse 15 Gen2 have the same problem as the Clevo
-	 * NL5xRU and NL5xNU/TUXEDO Aura 15 Gen1 and Gen2. See the description
-	 * above.
+	 * These Toshibas have a broken acpi-video interface for brightness
+	 * control. They also have an issue where the panel is off after
+	 * suspend until a special firmware call is made to turn it back
+	 * on. This is handled by the toshiba_acpi kernel module, so that
+	 * module must be enabled for these models to work correctly.
 	 */
 	{
-	.callback = video_detect_force_native,
-	.ident = "TongFang PF5PU1G",
-	.matches = {
-		DMI_MATCH(DMI_BOARD_NAME, "PF5PU1G"),
-		},
-	},
-	{
-	.callback = video_detect_force_native,
-	.ident = "TongFang PF4NU1F",
-	.matches = {
-		DMI_MATCH(DMI_BOARD_NAME, "PF4NU1F"),
-		},
-	},
-	{
-	.callback = video_detect_force_native,
-	.ident = "TongFang PF4NU1F",
-	.matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "TUXEDO"),
-		DMI_MATCH(DMI_BOARD_NAME, "PULSE1401"),
-		},
-	},
-	{
-	.callback = video_detect_force_native,
-	.ident = "TongFang PF5NU1G",
-	.matches = {
-		DMI_MATCH(DMI_BOARD_NAME, "PF5NU1G"),
+	 /* https://bugzilla.kernel.org/show_bug.cgi?id=21012 */
+	 .callback = video_detect_force_native,
+	 /* Toshiba Portégé R700 */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R700"),
 		},
 	},
 	{
-	.callback = video_detect_force_native,
-	.ident = "TongFang PF5NU1G",
-	.matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "TUXEDO"),
-		DMI_MATCH(DMI_BOARD_NAME, "PULSE1501"),
+	 /* Portégé: https://bugs.freedesktop.org/show_bug.cgi?id=82634 */
+	 /* Satellite: https://bugzilla.kernel.org/show_bug.cgi?id=21012 */
+	 .callback = video_detect_force_native,
+	 /* Toshiba Satellite/Portégé R830 */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "R830"),
 		},
 	},
 	{
-	.callback = video_detect_force_native,
-	.ident = "TongFang PF5LUXG",
-	.matches = {
-		DMI_MATCH(DMI_BOARD_NAME, "PF5LUXG"),
+	 .callback = video_detect_force_native,
+	 /* Toshiba Satellite/Portégé Z830 */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "Z830"),
 		},
 	},
+
 	/*
 	 * Desktops which falsely report a backlight and which our heuristics
 	 * for this do not catch.
@@ -537,43 +668,15 @@ static const struct dmi_system_id video_detect_dmi_table[] = {
 	{ },
 };
 
-/* This uses a workqueue to avoid various locking ordering issues */
-static void acpi_video_backlight_notify_work(struct work_struct *work)
-{
-	if (acpi_video_get_backlight_type() != acpi_backlight_video)
-		acpi_video_unregister_backlight();
-}
-
-static int acpi_video_backlight_notify(struct notifier_block *nb,
-				       unsigned long val, void *bd)
-{
-	struct backlight_device *backlight = bd;
-
-	/* A raw bl registering may change video -> native */
-	if (backlight->props.type == BACKLIGHT_RAW &&
-	    val == BACKLIGHT_REGISTERED)
-		schedule_work(&backlight_notify_work);
-
-	return NOTIFY_OK;
-}
-
 /*
  * Determine which type of backlight interface to use on this system,
  * First check cmdline, then dmi quirks, then do autodetect.
- *
- * The autodetect order is:
- * 1) Is the acpi-video backlight interface supported ->
- *  no, use a vendor interface
- * 2) Is this a win8 "ready" BIOS and do we have a native interface ->
- *  yes, use a native interface
- * 3) Else use the acpi-video interface
- *
- * Arguably the native on win8 check should be done first, but that would
- * be a behavior change, which may causes issues.
  */
-enum acpi_backlight_type acpi_video_get_backlight_type(void)
+static enum acpi_backlight_type __acpi_video_get_backlight_type(bool native)
 {
 	static DEFINE_MUTEX(init_mutex);
+	static bool nvidia_wmi_ec_present;
+	static bool native_available;
 	static bool init_done;
 	static long video_caps;
 
@@ -585,48 +688,60 @@ enum acpi_backlight_type acpi_video_get_backlight_type(void)
 		acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
 				    ACPI_UINT32_MAX, find_video, NULL,
 				    &video_caps, NULL);
-		INIT_WORK(&backlight_notify_work,
-			  acpi_video_backlight_notify_work);
-		backlight_nb.notifier_call = acpi_video_backlight_notify;
-		backlight_nb.priority = 0;
-		if (backlight_register_notifier(&backlight_nb) == 0)
-			backlight_notifier_registered = true;
+		nvidia_wmi_ec_present = nvidia_wmi_ec_supported();
 		init_done = true;
 	}
+	if (native)
+		native_available = true;
 	mutex_unlock(&init_mutex);
 
+	/*
+	 * The below heuristics / detection steps are in order of descending
+	 * presedence. The commandline takes presedence over anything else.
+	 */
 	if (acpi_backlight_cmdline != acpi_backlight_undef)
 		return acpi_backlight_cmdline;
 
+	/* DMI quirks override any autodetection. */
 	if (acpi_backlight_dmi != acpi_backlight_undef)
 		return acpi_backlight_dmi;
 
-	if (!(video_caps & ACPI_VIDEO_BACKLIGHT))
-		return acpi_backlight_vendor;
+	/* Special cases such as nvidia_wmi_ec and apple gmux. */
+	if (nvidia_wmi_ec_present)
+		return acpi_backlight_nvidia_wmi_ec;
 
-	if (acpi_osi_is_win8() && backlight_device_get_by_type(BACKLIGHT_RAW))
-		return acpi_backlight_native;
+	if (apple_gmux_present())
+		return acpi_backlight_apple_gmux;
+
+	/* On systems with ACPI video use either native or ACPI video. */
+	if (video_caps & ACPI_VIDEO_BACKLIGHT) {
+		/*
+		 * Windows 8 and newer no longer use the ACPI video interface,
+		 * so it often does not work. If the ACPI tables are written
+		 * for win8 and native brightness ctl is available, use that.
+		 *
+		 * The native check deliberately is inside the if acpi-video
+		 * block on older devices without acpi-video support native
+		 * is usually not the best choice.
+		 */
+		if (acpi_osi_is_win8() && native_available)
+			return acpi_backlight_native;
+		else
+			return acpi_backlight_video;
+	}
 
-	return acpi_backlight_video;
+	/* No ACPI video (old hw), use vendor specific fw methods. */
+	return acpi_backlight_vendor;
 }
-EXPORT_SYMBOL(acpi_video_get_backlight_type);
 
-/*
- * Set the preferred backlight interface type based on DMI info.
- * This function allows DMI blacklists to be implemented by external
- * platform drivers instead of putting a big blacklist in video_detect.c
- */
-void acpi_video_set_dmi_backlight_type(enum acpi_backlight_type type)
+enum acpi_backlight_type acpi_video_get_backlight_type(void)
 {
-	acpi_backlight_dmi = type;
-	/* Remove acpi-video backlight interface if it is no longer desired */
-	if (acpi_video_get_backlight_type() != acpi_backlight_video)
-		acpi_video_unregister_backlight();
+	return __acpi_video_get_backlight_type(false);
 }
-EXPORT_SYMBOL(acpi_video_set_dmi_backlight_type);
+EXPORT_SYMBOL(acpi_video_get_backlight_type);
 
-void __exit acpi_video_detect_exit(void)
+bool acpi_video_backlight_use_native(void)
 {
-	if (backlight_notifier_registered)
-		backlight_unregister_notifier(&backlight_nb);
+	return __acpi_video_get_backlight_type(true) == acpi_backlight_native;
 }
+EXPORT_SYMBOL(acpi_video_backlight_use_native);
diff --git a/drivers/acpi/x86/apple.c b/drivers/acpi/x86/apple.c
index c285c91a5e9c..8812ecd03d55 100644
--- a/drivers/acpi/x86/apple.c
+++ b/drivers/acpi/x86/apple.c
@@ -8,6 +8,7 @@
 #include <linux/bitmap.h>
 #include <linux/platform_data/x86/apple.h>
 #include <linux/uuid.h>
+#include "../internal.h"
 
 /* Apple _DSM device properties GUID */
 static const guid_t apple_prp_guid =
diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c
index f9ac12b778e6..42f249070c09 100644
--- a/drivers/acpi/x86/s2idle.c
+++ b/drivers/acpi/x86/s2idle.c
@@ -17,6 +17,7 @@
 
 #include <linux/acpi.h>
 #include <linux/device.h>
+#include <linux/dmi.h>
 #include <linux/suspend.h>
 
 #include "../sleep.h"
@@ -27,6 +28,10 @@ static bool sleep_no_lps0 __read_mostly;
 module_param(sleep_no_lps0, bool, 0644);
 MODULE_PARM_DESC(sleep_no_lps0, "Do not use the special LPS0 device interface");
 
+static bool prefer_microsoft_dsm_guid __read_mostly;
+module_param(prefer_microsoft_dsm_guid, bool, 0644);
+MODULE_PARM_DESC(prefer_microsoft_dsm_guid, "Prefer using Microsoft GUID in LPS0 device _DSM evaluation");
+
 static const struct acpi_device_id lps0_device_ids[] = {
 	{"PNP0D80", },
 	{"", },
@@ -363,40 +368,132 @@ out:
 	return ret;
 }
 
+struct amd_lps0_hid_device_data {
+	const unsigned int rev_id;
+	const bool check_off_by_one;
+	const bool prefer_amd_guid;
+};
+
+static const struct amd_lps0_hid_device_data amd_picasso = {
+	.rev_id = 0,
+	.check_off_by_one = true,
+	.prefer_amd_guid = false,
+};
+
+static const struct amd_lps0_hid_device_data amd_cezanne = {
+	.rev_id = 0,
+	.check_off_by_one = false,
+	.prefer_amd_guid = false,
+};
+
+static const struct amd_lps0_hid_device_data amd_rembrandt = {
+	.rev_id = 2,
+	.check_off_by_one = false,
+	.prefer_amd_guid = true,
+};
+
+static const struct acpi_device_id amd_hid_ids[] = {
+	{"AMD0004",	(kernel_ulong_t)&amd_picasso,	},
+	{"AMD0005",	(kernel_ulong_t)&amd_picasso,	},
+	{"AMDI0005",	(kernel_ulong_t)&amd_picasso,	},
+	{"AMDI0006",	(kernel_ulong_t)&amd_cezanne,	},
+	{"AMDI0007",	(kernel_ulong_t)&amd_rembrandt,	},
+	{}
+};
+
+static int lps0_prefer_microsoft(const struct dmi_system_id *id)
+{
+	pr_debug("Preferring Microsoft GUID.\n");
+	prefer_microsoft_dsm_guid = true;
+	return 0;
+}
+
+static const struct dmi_system_id s2idle_dmi_table[] __initconst = {
+	{
+		/*
+		 * ASUS TUF Gaming A17 FA707RE
+		 * https://bugzilla.kernel.org/show_bug.cgi?id=216101
+		 */
+		.callback = lps0_prefer_microsoft,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ASUS TUF Gaming A17"),
+		},
+	},
+	{
+		/* ASUS ROG Zephyrus G14 (2022) */
+		.callback = lps0_prefer_microsoft,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ROG Zephyrus G14 GA402"),
+		},
+	},
+	{
+		/*
+		 * Lenovo Yoga Slim 7 Pro X 14ARH7
+		 * https://bugzilla.kernel.org/show_bug.cgi?id=216473 : 82V2
+		 * https://bugzilla.kernel.org/show_bug.cgi?id=216438 : 82TL
+		 */
+		.callback = lps0_prefer_microsoft,
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "82"),
+		},
+	},
+	{
+		/*
+		 * ASUSTeK COMPUTER INC. ROG Flow X13 GV301RE_GV301RE
+		 * https://gitlab.freedesktop.org/drm/amd/-/issues/2148
+		 */
+		.callback = lps0_prefer_microsoft,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ROG Flow X13 GV301"),
+		},
+	},
+	{
+		/*
+		 * ASUSTeK COMPUTER INC. ROG Flow X16 GV601RW_GV601RW
+		 * https://gitlab.freedesktop.org/drm/amd/-/issues/2148
+		 */
+		.callback = lps0_prefer_microsoft,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ROG Flow X16 GV601"),
+		},
+	},
+	{}
+};
+
 static int lps0_device_attach(struct acpi_device *adev,
 			      const struct acpi_device_id *not_used)
 {
 	if (lps0_device_handle)
 		return 0;
 
+	lps0_dsm_func_mask_microsoft = validate_dsm(adev->handle,
+						    ACPI_LPS0_DSM_UUID_MICROSOFT, 0,
+						    &lps0_dsm_guid_microsoft);
 	if (acpi_s2idle_vendor_amd()) {
-		/* AMD0004, AMD0005, AMDI0005:
-		 * - Should use rev_id 0x0
-		 * - function mask > 0x3: Should use AMD method, but has off by one bug
-		 * - function mask = 0x3: Should use Microsoft method
-		 * AMDI0006:
-		 * - should use rev_id 0x0
-		 * - function mask = 0x3: Should use Microsoft method
-		 * AMDI0007:
-		 * - Should use rev_id 0x2
-		 * - Should only use AMD method
-		 */
-		const char *hid = acpi_device_hid(adev);
-		rev_id = strcmp(hid, "AMDI0007") ? 0 : 2;
+		static const struct acpi_device_id *dev_id;
+		const struct amd_lps0_hid_device_data *data;
+
+		for (dev_id = &amd_hid_ids[0]; dev_id->id[0]; dev_id++)
+			if (acpi_dev_hid_uid_match(adev, dev_id->id, NULL))
+				break;
+		if (dev_id->id[0])
+			data = (const struct amd_lps0_hid_device_data *) dev_id->driver_data;
+		else
+			data = &amd_rembrandt;
+		rev_id = data->rev_id;
 		lps0_dsm_func_mask = validate_dsm(adev->handle,
 					ACPI_LPS0_DSM_UUID_AMD, rev_id, &lps0_dsm_guid);
-		lps0_dsm_func_mask_microsoft = validate_dsm(adev->handle,
-					ACPI_LPS0_DSM_UUID_MICROSOFT, 0,
-					&lps0_dsm_guid_microsoft);
-		if (lps0_dsm_func_mask > 0x3 && (!strcmp(hid, "AMD0004") ||
-						 !strcmp(hid, "AMD0005") ||
-						 !strcmp(hid, "AMDI0005"))) {
+		if (lps0_dsm_func_mask > 0x3 && data->check_off_by_one) {
 			lps0_dsm_func_mask = (lps0_dsm_func_mask << 1) | 0x1;
 			acpi_handle_debug(adev->handle, "_DSM UUID %s: Adjusted function mask: 0x%x\n",
 					  ACPI_LPS0_DSM_UUID_AMD, lps0_dsm_func_mask);
-		} else if (lps0_dsm_func_mask_microsoft > 0 &&
-				(!strcmp(hid, "AMDI0007") ||
-				 !strcmp(hid, "AMDI0008"))) {
+		} else if (lps0_dsm_func_mask_microsoft > 0 && data->prefer_amd_guid &&
+				!prefer_microsoft_dsm_guid) {
 			lps0_dsm_func_mask_microsoft = -EINVAL;
 			acpi_handle_debug(adev->handle, "_DSM Using AMD method\n");
 		}
@@ -404,7 +501,8 @@ static int lps0_device_attach(struct acpi_device *adev,
 		rev_id = 1;
 		lps0_dsm_func_mask = validate_dsm(adev->handle,
 					ACPI_LPS0_DSM_UUID, rev_id, &lps0_dsm_guid);
-		lps0_dsm_func_mask_microsoft = -EINVAL;
+		if (!prefer_microsoft_dsm_guid)
+			lps0_dsm_func_mask_microsoft = -EINVAL;
 	}
 
 	if (lps0_dsm_func_mask < 0 && lps0_dsm_func_mask_microsoft < 0)
@@ -486,6 +584,19 @@ int acpi_s2idle_prepare_late(void)
 	return 0;
 }
 
+void acpi_s2idle_check(void)
+{
+	struct acpi_s2idle_dev_ops *handler;
+
+	if (!lps0_device_handle || sleep_no_lps0)
+		return;
+
+	list_for_each_entry(handler, &lps0_s2idle_devops_head, list_node) {
+		if (handler->check)
+			handler->check();
+	}
+}
+
 void acpi_s2idle_restore_early(void)
 {
 	struct acpi_s2idle_dev_ops *handler;
@@ -527,14 +638,16 @@ static const struct platform_s2idle_ops acpi_s2idle_ops_lps0 = {
 	.begin = acpi_s2idle_begin,
 	.prepare = acpi_s2idle_prepare,
 	.prepare_late = acpi_s2idle_prepare_late,
+	.check = acpi_s2idle_check,
 	.wake = acpi_s2idle_wake,
 	.restore_early = acpi_s2idle_restore_early,
 	.restore = acpi_s2idle_restore,
 	.end = acpi_s2idle_end,
 };
 
-void acpi_s2idle_setup(void)
+void __init acpi_s2idle_setup(void)
 {
+	dmi_check_system(s2idle_dmi_table);
 	acpi_scan_add_handler(&lps0_handler);
 	s2idle_set_ops(&acpi_s2idle_ops_lps0);
 }
diff --git a/drivers/acpi/x86/utils.c b/drivers/acpi/x86/utils.c
index 664070fc8349..f8a2cbdc0ce2 100644
--- a/drivers/acpi/x86/utils.c
+++ b/drivers/acpi/x86/utils.c
@@ -207,9 +207,26 @@ static const struct x86_cpu_id storage_d3_cpu_ids[] = {
 	{}
 };
 
+static const struct dmi_system_id force_storage_d3_dmi[] = {
+	{
+		/*
+		 * _ADR is ambiguous between GPP1.DEV0 and GPP1.NVME
+		 * but .NVME is needed to get StorageD3Enable node
+		 * https://bugzilla.kernel.org/show_bug.cgi?id=216440
+		 */
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 14 7425 2-in-1"),
+		}
+	},
+	{}
+};
+
 bool force_storage_d3(void)
 {
-	return x86_match_cpu(storage_d3_cpu_ids);
+	const struct dmi_system_id *dmi_id = dmi_first_match(force_storage_d3_dmi);
+
+	return dmi_id || x86_match_cpu(storage_d3_cpu_ids);
 }
 
 /*
@@ -351,11 +368,17 @@ int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *s
 	struct acpi_device *adev = ACPI_COMPANION(controller_parent);
 	const struct dmi_system_id *dmi_id;
 	long quirks = 0;
+	u64 uid;
+	int ret;
 
 	*skip = false;
 
-	/* !dev_is_platform() to not match on PNP enumerated debug UARTs */
-	if (!adev || !adev->pnp.unique_id || !dev_is_platform(controller_parent))
+	ret = acpi_dev_uid_to_integer(adev, &uid);
+	if (ret)
+		return 0;
+
+	/* to not match on PNP enumerated debug UARTs */
+	if (!dev_is_platform(controller_parent))
 		return 0;
 
 	dmi_id = dmi_first_match(acpi_quirk_skip_dmi_ids);
@@ -363,10 +386,10 @@ int acpi_quirk_skip_serdev_enumeration(struct device *controller_parent, bool *s
 		quirks = (unsigned long)dmi_id->driver_data;
 
 	if (quirks & ACPI_QUIRK_UART1_TTY_UART2_SKIP) {
-		if (!strcmp(adev->pnp.unique_id, "1"))
+		if (uid == 1)
 			return -ENODEV; /* Create tty cdev instead of serdev */
 
-		if (!strcmp(adev->pnp.unique_id, "2"))
+		if (uid == 2)
 			*skip = true;
 	}
 
diff --git a/drivers/ata/ahci_imx.c b/drivers/ata/ahci_imx.c
index 79aa9f285312..b734e069034d 100644
--- a/drivers/ata/ahci_imx.c
+++ b/drivers/ata/ahci_imx.c
@@ -327,7 +327,7 @@ static int read_adc_sum(void *dev, u16 rtune_ctl_reg, void __iomem * mmio)
 }
 
 /* SATA AHCI temperature monitor */
-static int sata_ahci_read_temperature(void *dev, int *temp)
+static int __sata_ahci_read_temperature(void *dev, int *temp)
 {
 	u16 mpll_test_reg, rtune_ctl_reg, dac_ctl_reg, read_sum;
 	u32 str1, str2, str3, str4;
@@ -416,6 +416,11 @@ static int sata_ahci_read_temperature(void *dev, int *temp)
 	return 0;
 }
 
+static int sata_ahci_read_temperature(struct thermal_zone_device *tz, int *temp)
+{
+	return __sata_ahci_read_temperature(tz->devdata, temp);
+}
+
 static ssize_t sata_ahci_show_temp(struct device *dev,
 				   struct device_attribute *da,
 				   char *buf)
@@ -423,14 +428,14 @@ static ssize_t sata_ahci_show_temp(struct device *dev,
 	unsigned int temp = 0;
 	int err;
 
-	err = sata_ahci_read_temperature(dev, &temp);
+	err = __sata_ahci_read_temperature(dev, &temp);
 	if (err < 0)
 		return err;
 
 	return sprintf(buf, "%u\n", temp);
 }
 
-static const struct thermal_zone_of_device_ops fsl_sata_ahci_of_thermal_ops = {
+static const struct thermal_zone_device_ops fsl_sata_ahci_of_thermal_ops = {
 	.get_temp = sata_ahci_read_temperature,
 };
 
@@ -1131,8 +1136,8 @@ static int imx_ahci_probe(struct platform_device *pdev)
 			ret = PTR_ERR(hwmon_dev);
 			goto disable_clk;
 		}
-		devm_thermal_zone_of_sensor_register(hwmon_dev, 0, hwmon_dev,
-					     &fsl_sata_ahci_of_thermal_ops);
+		devm_thermal_of_zone_register(hwmon_dev, 0, hwmon_dev,
+					      &fsl_sata_ahci_of_thermal_ops);
 		dev_info(dev, "%s: sensor 'sata_ahci'\n", dev_name(hwmon_dev));
 	}
 
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index 826d41f341e4..c9a9aa607b62 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -3988,6 +3988,10 @@ static const struct ata_blacklist_entry ata_device_blacklist [] = {
 	{ "PIONEER DVD-RW  DVR-212D",	NULL,	ATA_HORKAGE_NOSETXFER },
 	{ "PIONEER DVD-RW  DVR-216D",	NULL,	ATA_HORKAGE_NOSETXFER },
 
+	/* These specific Pioneer models have LPM issues */
+	{ "PIONEER BD-RW   BDR-207M",	NULL,	ATA_HORKAGE_NOLPM },
+	{ "PIONEER BD-RW   BDR-205",	NULL,	ATA_HORKAGE_NOLPM },
+
 	/* Crucial BX100 SSD 500GB has broken LPM support */
 	{ "CT500BX100SSD1",		NULL,	ATA_HORKAGE_NOLPM },
 
diff --git a/drivers/ata/libata-sata.c b/drivers/ata/libata-sata.c
index 7a5fe41aa5ae..13b9d0fdd42c 100644
--- a/drivers/ata/libata-sata.c
+++ b/drivers/ata/libata-sata.c
@@ -1018,26 +1018,25 @@ DEVICE_ATTR(sw_activity, S_IWUSR | S_IRUGO, ata_scsi_activity_show,
 EXPORT_SYMBOL_GPL(dev_attr_sw_activity);
 
 /**
- *	__ata_change_queue_depth - helper for ata_scsi_change_queue_depth
- *	@ap: ATA port to which the device change the queue depth
+ *	ata_change_queue_depth - Set a device maximum queue depth
+ *	@ap: ATA port of the target device
+ *	@dev: target ATA device
  *	@sdev: SCSI device to configure queue depth for
  *	@queue_depth: new queue depth
  *
- *	libsas and libata have different approaches for associating a sdev to
- *	its ata_port.
+ *	Helper to set a device maximum queue depth, usable with both libsas
+ *	and libata.
  *
  */
-int __ata_change_queue_depth(struct ata_port *ap, struct scsi_device *sdev,
-			     int queue_depth)
+int ata_change_queue_depth(struct ata_port *ap, struct ata_device *dev,
+			   struct scsi_device *sdev, int queue_depth)
 {
-	struct ata_device *dev;
 	unsigned long flags;
 
-	if (queue_depth < 1 || queue_depth == sdev->queue_depth)
+	if (!dev || !ata_dev_enabled(dev))
 		return sdev->queue_depth;
 
-	dev = ata_scsi_find_dev(ap, sdev);
-	if (!dev || !ata_dev_enabled(dev))
+	if (queue_depth < 1 || queue_depth == sdev->queue_depth)
 		return sdev->queue_depth;
 
 	/* NCQ enabled? */
@@ -1059,7 +1058,7 @@ int __ata_change_queue_depth(struct ata_port *ap, struct scsi_device *sdev,
 
 	return scsi_change_queue_depth(sdev, queue_depth);
 }
-EXPORT_SYMBOL_GPL(__ata_change_queue_depth);
+EXPORT_SYMBOL_GPL(ata_change_queue_depth);
 
 /**
  *	ata_scsi_change_queue_depth - SCSI callback for queue depth config
@@ -1080,7 +1079,8 @@ int ata_scsi_change_queue_depth(struct scsi_device *sdev, int queue_depth)
 {
 	struct ata_port *ap = ata_shost_to_port(sdev->host);
 
-	return __ata_change_queue_depth(ap, sdev, queue_depth);
+	return ata_change_queue_depth(ap, ata_scsi_find_dev(ap, sdev),
+				      sdev, queue_depth);
 }
 EXPORT_SYMBOL_GPL(ata_scsi_change_queue_depth);
 
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index 29e2f55c6faa..ff9602a0e54e 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -1055,6 +1055,7 @@ EXPORT_SYMBOL_GPL(ata_scsi_dma_need_drain);
 int ata_scsi_dev_config(struct scsi_device *sdev, struct ata_device *dev)
 {
 	struct request_queue *q = sdev->request_queue;
+	int depth = 1;
 
 	if (!ata_id_has_unload(dev->id))
 		dev->flags |= ATA_DFLAG_NO_UNLOAD;
@@ -1100,13 +1101,10 @@ int ata_scsi_dev_config(struct scsi_device *sdev, struct ata_device *dev)
 	if (dev->flags & ATA_DFLAG_AN)
 		set_bit(SDEV_EVT_MEDIA_CHANGE, sdev->supported_events);
 
-	if (dev->flags & ATA_DFLAG_NCQ) {
-		int depth;
-
+	if (dev->flags & ATA_DFLAG_NCQ)
 		depth = min(sdev->host->can_queue, ata_id_queue_depth(dev->id));
-		depth = min(ATA_MAX_QUEUE, depth);
-		scsi_change_queue_depth(sdev, depth);
-	}
+	depth = min(ATA_MAX_QUEUE, depth);
+	scsi_change_queue_depth(sdev, depth);
 
 	if (dev->flags & ATA_DFLAG_TRUSTED)
 		sdev->security_supported = 1;
diff --git a/drivers/auxdisplay/ht16k33.c b/drivers/auxdisplay/ht16k33.c
index 4fab3b2c7023..02425991c159 100644
--- a/drivers/auxdisplay/ht16k33.c
+++ b/drivers/auxdisplay/ht16k33.c
@@ -775,7 +775,7 @@ static int ht16k33_probe(struct i2c_client *client)
 	return err;
 }
 
-static int ht16k33_remove(struct i2c_client *client)
+static void ht16k33_remove(struct i2c_client *client)
 {
 	struct ht16k33_priv *priv = i2c_get_clientdata(client);
 	struct ht16k33_fbdev *fbdev = &priv->fbdev;
@@ -796,8 +796,6 @@ static int ht16k33_remove(struct i2c_client *client)
 		device_remove_file(&client->dev, &dev_attr_map_seg14);
 		break;
 	}
-
-	return 0;
 }
 
 static const struct i2c_device_id ht16k33_i2c_match[] = {
diff --git a/drivers/auxdisplay/lcd2s.c b/drivers/auxdisplay/lcd2s.c
index e465108d9998..135831a16514 100644
--- a/drivers/auxdisplay/lcd2s.c
+++ b/drivers/auxdisplay/lcd2s.c
@@ -340,13 +340,12 @@ fail1:
 	return err;
 }
 
-static int lcd2s_i2c_remove(struct i2c_client *i2c)
+static void lcd2s_i2c_remove(struct i2c_client *i2c)
 {
 	struct lcd2s_data *lcd2s = i2c_get_clientdata(i2c);
 
 	charlcd_unregister(lcd2s->charlcd);
 	charlcd_free(lcd2s->charlcd);
-	return 0;
 }
 
 static const struct i2c_device_id lcd2s_i2c_id[] = {
diff --git a/drivers/base/arch_topology.c b/drivers/base/arch_topology.c
index 46cbe4471e78..dd90591e51ba 100644
--- a/drivers/base/arch_topology.c
+++ b/drivers/base/arch_topology.c
@@ -353,7 +353,7 @@ void topology_init_cpu_capacity_cppc(void)
 	struct cppc_perf_caps perf_caps;
 	int cpu;
 
-	if (likely(acpi_disabled || !acpi_cpc_valid()))
+	if (likely(!acpi_cpc_valid()))
 		return;
 
 	raw_capacity = kcalloc(num_possible_cpus(), sizeof(*raw_capacity),
diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c
index 997be3ac20a7..b52049098d4e 100644
--- a/drivers/base/power/runtime.c
+++ b/drivers/base/power/runtime.c
@@ -792,10 +792,13 @@ static int rpm_resume(struct device *dev, int rpmflags)
 		DEFINE_WAIT(wait);
 
 		if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
-			if (dev->power.runtime_status == RPM_SUSPENDING)
+			if (dev->power.runtime_status == RPM_SUSPENDING) {
 				dev->power.deferred_resume = true;
-			else
+				if (rpmflags & RPM_NOWAIT)
+					retval = -EINPROGRESS;
+			} else {
 				retval = -EINPROGRESS;
+			}
 			goto out;
 		}
 
diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
index e3befa2c1b66..7cc0c0cf8eaa 100644
--- a/drivers/base/power/wakeup.c
+++ b/drivers/base/power/wakeup.c
@@ -944,6 +944,8 @@ void pm_system_irq_wakeup(unsigned int irq_number)
 	else
 		irq_number = 0;
 
+	pm_pr_dbg("Triggering wakeup from IRQ %d\n", irq_number);
+
 	raw_spin_unlock_irqrestore(&wakeup_irq_lock, flags);
 
 	if (irq_number)
diff --git a/drivers/base/regmap/regmap-mmio.c b/drivers/base/regmap/regmap-mmio.c
index 71f16be7e717..3ccdd86a97e7 100644
--- a/drivers/base/regmap/regmap-mmio.c
+++ b/drivers/base/regmap/regmap-mmio.c
@@ -10,13 +10,14 @@
 #include <linux/module.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
+#include <linux/swab.h>
 
 #include "internal.h"
 
 struct regmap_mmio_context {
 	void __iomem *regs;
 	unsigned int val_bytes;
-	bool relaxed_mmio;
+	bool big_endian;
 
 	bool attached_clk;
 	struct clk *clk;
@@ -33,9 +34,6 @@ static int regmap_mmio_regbits_check(size_t reg_bits)
 	case 8:
 	case 16:
 	case 32:
-#ifdef CONFIG_64BIT
-	case 64:
-#endif
 		return 0;
 	default:
 		return -EINVAL;
@@ -50,18 +48,13 @@ static int regmap_mmio_get_min_stride(size_t val_bits)
 	case 8:
 		/* The core treats 0 as 1 */
 		min_stride = 0;
-		return 0;
+		break;
 	case 16:
 		min_stride = 2;
 		break;
 	case 32:
 		min_stride = 4;
 		break;
-#ifdef CONFIG_64BIT
-	case 64:
-		min_stride = 8;
-		break;
-#endif
 	default:
 		return -EINVAL;
 	}
@@ -83,6 +76,12 @@ static void regmap_mmio_write8_relaxed(struct regmap_mmio_context *ctx,
 	writeb_relaxed(val, ctx->regs + reg);
 }
 
+static void regmap_mmio_iowrite8(struct regmap_mmio_context *ctx,
+				 unsigned int reg, unsigned int val)
+{
+	iowrite8(val, ctx->regs + reg);
+}
+
 static void regmap_mmio_write16le(struct regmap_mmio_context *ctx,
 				  unsigned int reg,
 				  unsigned int val)
@@ -97,10 +96,22 @@ static void regmap_mmio_write16le_relaxed(struct regmap_mmio_context *ctx,
 	writew_relaxed(val, ctx->regs + reg);
 }
 
+static void regmap_mmio_iowrite16le(struct regmap_mmio_context *ctx,
+				    unsigned int reg, unsigned int val)
+{
+	iowrite16(val, ctx->regs + reg);
+}
+
 static void regmap_mmio_write16be(struct regmap_mmio_context *ctx,
 				  unsigned int reg,
 				  unsigned int val)
 {
+	writew(swab16(val), ctx->regs + reg);
+}
+
+static void regmap_mmio_iowrite16be(struct regmap_mmio_context *ctx,
+				    unsigned int reg, unsigned int val)
+{
 	iowrite16be(val, ctx->regs + reg);
 }
 
@@ -118,28 +129,24 @@ static void regmap_mmio_write32le_relaxed(struct regmap_mmio_context *ctx,
 	writel_relaxed(val, ctx->regs + reg);
 }
 
-static void regmap_mmio_write32be(struct regmap_mmio_context *ctx,
-				  unsigned int reg,
-				  unsigned int val)
+static void regmap_mmio_iowrite32le(struct regmap_mmio_context *ctx,
+				    unsigned int reg, unsigned int val)
 {
-	iowrite32be(val, ctx->regs + reg);
+	iowrite32(val, ctx->regs + reg);
 }
 
-#ifdef CONFIG_64BIT
-static void regmap_mmio_write64le(struct regmap_mmio_context *ctx,
+static void regmap_mmio_write32be(struct regmap_mmio_context *ctx,
 				  unsigned int reg,
 				  unsigned int val)
 {
-	writeq(val, ctx->regs + reg);
+	writel(swab32(val), ctx->regs + reg);
 }
 
-static void regmap_mmio_write64le_relaxed(struct regmap_mmio_context *ctx,
-				  unsigned int reg,
-				  unsigned int val)
+static void regmap_mmio_iowrite32be(struct regmap_mmio_context *ctx,
+				    unsigned int reg, unsigned int val)
 {
-	writeq_relaxed(val, ctx->regs + reg);
+	iowrite32be(val, ctx->regs + reg);
 }
-#endif
 
 static int regmap_mmio_write(void *context, unsigned int reg, unsigned int val)
 {
@@ -160,6 +167,83 @@ static int regmap_mmio_write(void *context, unsigned int reg, unsigned int val)
 	return 0;
 }
 
+static int regmap_mmio_noinc_write(void *context, unsigned int reg,
+				   const void *val, size_t val_count)
+{
+	struct regmap_mmio_context *ctx = context;
+	int ret = 0;
+	int i;
+
+	if (!IS_ERR(ctx->clk)) {
+		ret = clk_enable(ctx->clk);
+		if (ret < 0)
+			return ret;
+	}
+
+	/*
+	 * There are no native, assembly-optimized write single register
+	 * operations for big endian, so fall back to emulation if this
+	 * is needed. (Single bytes are fine, they are not affected by
+	 * endianness.)
+	 */
+	if (ctx->big_endian && (ctx->val_bytes > 1)) {
+		switch (ctx->val_bytes) {
+		case 2:
+		{
+			const u16 *valp = (const u16 *)val;
+			for (i = 0; i < val_count; i++)
+				writew(swab16(valp[i]), ctx->regs + reg);
+			goto out_clk;
+		}
+		case 4:
+		{
+			const u32 *valp = (const u32 *)val;
+			for (i = 0; i < val_count; i++)
+				writel(swab32(valp[i]), ctx->regs + reg);
+			goto out_clk;
+		}
+#ifdef CONFIG_64BIT
+		case 8:
+		{
+			const u64 *valp = (const u64 *)val;
+			for (i = 0; i < val_count; i++)
+				writeq(swab64(valp[i]), ctx->regs + reg);
+			goto out_clk;
+		}
+#endif
+		default:
+			ret = -EINVAL;
+			goto out_clk;
+		}
+	}
+
+	switch (ctx->val_bytes) {
+	case 1:
+		writesb(ctx->regs + reg, (const u8 *)val, val_count);
+		break;
+	case 2:
+		writesw(ctx->regs + reg, (const u16 *)val, val_count);
+		break;
+	case 4:
+		writesl(ctx->regs + reg, (const u32 *)val, val_count);
+		break;
+#ifdef CONFIG_64BIT
+	case 8:
+		writesq(ctx->regs + reg, (const u64 *)val, val_count);
+		break;
+#endif
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+out_clk:
+	if (!IS_ERR(ctx->clk))
+		clk_disable(ctx->clk);
+
+	return ret;
+}
+
 static unsigned int regmap_mmio_read8(struct regmap_mmio_context *ctx,
 				      unsigned int reg)
 {
@@ -172,6 +256,12 @@ static unsigned int regmap_mmio_read8_relaxed(struct regmap_mmio_context *ctx,
 	return readb_relaxed(ctx->regs + reg);
 }
 
+static unsigned int regmap_mmio_ioread8(struct regmap_mmio_context *ctx,
+					unsigned int reg)
+{
+	return ioread8(ctx->regs + reg);
+}
+
 static unsigned int regmap_mmio_read16le(struct regmap_mmio_context *ctx,
 				         unsigned int reg)
 {
@@ -184,9 +274,21 @@ static unsigned int regmap_mmio_read16le_relaxed(struct regmap_mmio_context *ctx
 	return readw_relaxed(ctx->regs + reg);
 }
 
+static unsigned int regmap_mmio_ioread16le(struct regmap_mmio_context *ctx,
+					   unsigned int reg)
+{
+	return ioread16(ctx->regs + reg);
+}
+
 static unsigned int regmap_mmio_read16be(struct regmap_mmio_context *ctx,
 				         unsigned int reg)
 {
+	return swab16(readw(ctx->regs + reg));
+}
+
+static unsigned int regmap_mmio_ioread16be(struct regmap_mmio_context *ctx,
+					   unsigned int reg)
+{
 	return ioread16be(ctx->regs + reg);
 }
 
@@ -202,25 +304,23 @@ static unsigned int regmap_mmio_read32le_relaxed(struct regmap_mmio_context *ctx
 	return readl_relaxed(ctx->regs + reg);
 }
 
-static unsigned int regmap_mmio_read32be(struct regmap_mmio_context *ctx,
-				         unsigned int reg)
+static unsigned int regmap_mmio_ioread32le(struct regmap_mmio_context *ctx,
+					   unsigned int reg)
 {
-	return ioread32be(ctx->regs + reg);
+	return ioread32(ctx->regs + reg);
 }
 
-#ifdef CONFIG_64BIT
-static unsigned int regmap_mmio_read64le(struct regmap_mmio_context *ctx,
+static unsigned int regmap_mmio_read32be(struct regmap_mmio_context *ctx,
 				         unsigned int reg)
 {
-	return readq(ctx->regs + reg);
+	return swab32(readl(ctx->regs + reg));
 }
 
-static unsigned int regmap_mmio_read64le_relaxed(struct regmap_mmio_context *ctx,
-						 unsigned int reg)
+static unsigned int regmap_mmio_ioread32be(struct regmap_mmio_context *ctx,
+					   unsigned int reg)
 {
-	return readq_relaxed(ctx->regs + reg);
+	return ioread32be(ctx->regs + reg);
 }
-#endif
 
 static int regmap_mmio_read(void *context, unsigned int reg, unsigned int *val)
 {
@@ -241,6 +341,71 @@ static int regmap_mmio_read(void *context, unsigned int reg, unsigned int *val)
 	return 0;
 }
 
+static int regmap_mmio_noinc_read(void *context, unsigned int reg,
+				  void *val, size_t val_count)
+{
+	struct regmap_mmio_context *ctx = context;
+	int ret = 0;
+
+	if (!IS_ERR(ctx->clk)) {
+		ret = clk_enable(ctx->clk);
+		if (ret < 0)
+			return ret;
+	}
+
+	switch (ctx->val_bytes) {
+	case 1:
+		readsb(ctx->regs + reg, (u8 *)val, val_count);
+		break;
+	case 2:
+		readsw(ctx->regs + reg, (u16 *)val, val_count);
+		break;
+	case 4:
+		readsl(ctx->regs + reg, (u32 *)val, val_count);
+		break;
+#ifdef CONFIG_64BIT
+	case 8:
+		readsq(ctx->regs + reg, (u64 *)val, val_count);
+		break;
+#endif
+	default:
+		ret = -EINVAL;
+		goto out_clk;
+	}
+
+	/*
+	 * There are no native, assembly-optimized write single register
+	 * operations for big endian, so fall back to emulation if this
+	 * is needed. (Single bytes are fine, they are not affected by
+	 * endianness.)
+	 */
+	if (ctx->big_endian && (ctx->val_bytes > 1)) {
+		switch (ctx->val_bytes) {
+		case 2:
+			swab16_array(val, val_count);
+			break;
+		case 4:
+			swab32_array(val, val_count);
+			break;
+#ifdef CONFIG_64BIT
+		case 8:
+			swab64_array(val, val_count);
+			break;
+#endif
+		default:
+			ret = -EINVAL;
+			break;
+		}
+	}
+
+out_clk:
+	if (!IS_ERR(ctx->clk))
+		clk_disable(ctx->clk);
+
+	return ret;
+}
+
+
 static void regmap_mmio_free_context(void *context)
 {
 	struct regmap_mmio_context *ctx = context;
@@ -257,6 +422,8 @@ static const struct regmap_bus regmap_mmio = {
 	.fast_io = true,
 	.reg_write = regmap_mmio_write,
 	.reg_read = regmap_mmio_read,
+	.reg_noinc_write = regmap_mmio_noinc_write,
+	.reg_noinc_read = regmap_mmio_noinc_read,
 	.free_context = regmap_mmio_free_context,
 	.val_format_endian_default = REGMAP_ENDIAN_LITTLE,
 };
@@ -284,13 +451,15 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev,
 	if (config->reg_stride < min_stride)
 		return ERR_PTR(-EINVAL);
 
+	if (config->use_relaxed_mmio && config->io_port)
+		return ERR_PTR(-EINVAL);
+
 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
 		return ERR_PTR(-ENOMEM);
 
 	ctx->regs = regs;
 	ctx->val_bytes = config->val_bits / 8;
-	ctx->relaxed_mmio = config->use_relaxed_mmio;
 	ctx->clk = ERR_PTR(-ENODEV);
 
 	switch (regmap_get_val_endian(dev, &regmap_mmio, config)) {
@@ -301,7 +470,10 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev,
 #endif
 		switch (config->val_bits) {
 		case 8:
-			if (ctx->relaxed_mmio) {
+			if (config->io_port) {
+				ctx->reg_read = regmap_mmio_ioread8;
+				ctx->reg_write = regmap_mmio_iowrite8;
+			} else if (config->use_relaxed_mmio) {
 				ctx->reg_read = regmap_mmio_read8_relaxed;
 				ctx->reg_write = regmap_mmio_write8_relaxed;
 			} else {
@@ -310,7 +482,10 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev,
 			}
 			break;
 		case 16:
-			if (ctx->relaxed_mmio) {
+			if (config->io_port) {
+				ctx->reg_read = regmap_mmio_ioread16le;
+				ctx->reg_write = regmap_mmio_iowrite16le;
+			} else if (config->use_relaxed_mmio) {
 				ctx->reg_read = regmap_mmio_read16le_relaxed;
 				ctx->reg_write = regmap_mmio_write16le_relaxed;
 			} else {
@@ -319,7 +494,10 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev,
 			}
 			break;
 		case 32:
-			if (ctx->relaxed_mmio) {
+			if (config->io_port) {
+				ctx->reg_read = regmap_mmio_ioread32le;
+				ctx->reg_write = regmap_mmio_iowrite32le;
+			} else if (config->use_relaxed_mmio) {
 				ctx->reg_read = regmap_mmio_read32le_relaxed;
 				ctx->reg_write = regmap_mmio_write32le_relaxed;
 			} else {
@@ -327,17 +505,6 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev,
 				ctx->reg_write = regmap_mmio_write32le;
 			}
 			break;
-#ifdef CONFIG_64BIT
-		case 64:
-			if (ctx->relaxed_mmio) {
-				ctx->reg_read = regmap_mmio_read64le_relaxed;
-				ctx->reg_write = regmap_mmio_write64le_relaxed;
-			} else {
-				ctx->reg_read = regmap_mmio_read64le;
-				ctx->reg_write = regmap_mmio_write64le;
-			}
-			break;
-#endif
 		default:
 			ret = -EINVAL;
 			goto err_free;
@@ -347,18 +514,34 @@ static struct regmap_mmio_context *regmap_mmio_gen_context(struct device *dev,
 #ifdef __BIG_ENDIAN
 	case REGMAP_ENDIAN_NATIVE:
 #endif
+		ctx->big_endian = true;
 		switch (config->val_bits) {
 		case 8:
-			ctx->reg_read = regmap_mmio_read8;
-			ctx->reg_write = regmap_mmio_write8;
+			if (config->io_port) {
+				ctx->reg_read = regmap_mmio_ioread8;
+				ctx->reg_write = regmap_mmio_iowrite8;
+			} else {
+				ctx->reg_read = regmap_mmio_read8;
+				ctx->reg_write = regmap_mmio_write8;
+			}
 			break;
 		case 16:
-			ctx->reg_read = regmap_mmio_read16be;
-			ctx->reg_write = regmap_mmio_write16be;
+			if (config->io_port) {
+				ctx->reg_read = regmap_mmio_ioread16be;
+				ctx->reg_write = regmap_mmio_iowrite16be;
+			} else {
+				ctx->reg_read = regmap_mmio_read16be;
+				ctx->reg_write = regmap_mmio_write16be;
+			}
 			break;
 		case 32:
-			ctx->reg_read = regmap_mmio_read32be;
-			ctx->reg_write = regmap_mmio_write32be;
+			if (config->io_port) {
+				ctx->reg_read = regmap_mmio_ioread32be;
+				ctx->reg_write = regmap_mmio_iowrite32be;
+			} else {
+				ctx->reg_read = regmap_mmio_read32be;
+				ctx->reg_write = regmap_mmio_write32be;
+			}
 			break;
 		default:
 			ret = -EINVAL;
diff --git a/drivers/base/regmap/regmap-spi-avmm.c b/drivers/base/regmap/regmap-spi-avmm.c
index ad1da83e849f..4c2b94b3e30b 100644
--- a/drivers/base/regmap/regmap-spi-avmm.c
+++ b/drivers/base/regmap/regmap-spi-avmm.c
@@ -7,6 +7,7 @@
 #include <linux/module.h>
 #include <linux/regmap.h>
 #include <linux/spi/spi.h>
+#include <linux/swab.h>
 
 /*
  * This driver implements the regmap operations for a generic SPI
@@ -162,19 +163,12 @@ struct spi_avmm_bridge {
 	/* bridge buffer used in translation between protocol layers */
 	char trans_buf[TRANS_BUF_SIZE];
 	char phy_buf[PHY_BUF_SIZE];
-	void (*swap_words)(char *buf, unsigned int len);
+	void (*swap_words)(void *buf, unsigned int len);
 };
 
-static void br_swap_words_32(char *buf, unsigned int len)
+static void br_swap_words_32(void *buf, unsigned int len)
 {
-	u32 *p = (u32 *)buf;
-	unsigned int count;
-
-	count = len / 4;
-	while (count--) {
-		*p = swab32p(p);
-		p++;
-	}
+	swab32_array(buf, len / 4);
 }
 
 /*
diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c
index fee221c5008c..c6d6d53e8cd3 100644
--- a/drivers/base/regmap/regmap.c
+++ b/drivers/base/regmap/regmap.c
@@ -288,15 +288,9 @@ static void regmap_format_16_native(void *buf, unsigned int val,
 	memcpy(buf, &v, sizeof(v));
 }
 
-static void regmap_format_24(void *buf, unsigned int val, unsigned int shift)
+static void regmap_format_24_be(void *buf, unsigned int val, unsigned int shift)
 {
-	u8 *b = buf;
-
-	val <<= shift;
-
-	b[0] = val >> 16;
-	b[1] = val >> 8;
-	b[2] = val;
+	put_unaligned_be24(val << shift, buf);
 }
 
 static void regmap_format_32_be(void *buf, unsigned int val, unsigned int shift)
@@ -380,14 +374,9 @@ static unsigned int regmap_parse_16_native(const void *buf)
 	return v;
 }
 
-static unsigned int regmap_parse_24(const void *buf)
+static unsigned int regmap_parse_24_be(const void *buf)
 {
-	const u8 *b = buf;
-	unsigned int ret = b[2];
-	ret |= ((unsigned int)b[1]) << 8;
-	ret |= ((unsigned int)b[0]) << 16;
-
-	return ret;
+	return get_unaligned_be24(buf);
 }
 
 static unsigned int regmap_parse_32_be(const void *buf)
@@ -991,9 +980,13 @@ struct regmap *__regmap_init(struct device *dev,
 		break;
 
 	case 24:
-		if (reg_endian != REGMAP_ENDIAN_BIG)
+		switch (reg_endian) {
+		case REGMAP_ENDIAN_BIG:
+			map->format.format_reg = regmap_format_24_be;
+			break;
+		default:
 			goto err_hwlock;
-		map->format.format_reg = regmap_format_24;
+		}
 		break;
 
 	case 32:
@@ -1064,10 +1057,14 @@ struct regmap *__regmap_init(struct device *dev,
 		}
 		break;
 	case 24:
-		if (val_endian != REGMAP_ENDIAN_BIG)
+		switch (val_endian) {
+		case REGMAP_ENDIAN_BIG:
+			map->format.format_val = regmap_format_24_be;
+			map->format.parse_val = regmap_parse_24_be;
+			break;
+		default:
 			goto err_hwlock;
-		map->format.format_val = regmap_format_24;
-		map->format.parse_val = regmap_parse_24;
+		}
 		break;
 	case 32:
 		switch (val_endian) {
@@ -2132,6 +2129,99 @@ int regmap_raw_write(struct regmap *map, unsigned int reg,
 }
 EXPORT_SYMBOL_GPL(regmap_raw_write);
 
+static int regmap_noinc_readwrite(struct regmap *map, unsigned int reg,
+				  void *val, unsigned int val_len, bool write)
+{
+	size_t val_bytes = map->format.val_bytes;
+	size_t val_count = val_len / val_bytes;
+	unsigned int lastval;
+	u8 *u8p;
+	u16 *u16p;
+	u32 *u32p;
+#ifdef CONFIG_64BIT
+	u64 *u64p;
+#endif
+	int ret;
+	int i;
+
+	switch (val_bytes) {
+	case 1:
+		u8p = val;
+		if (write)
+			lastval = (unsigned int)u8p[val_count - 1];
+		break;
+	case 2:
+		u16p = val;
+		if (write)
+			lastval = (unsigned int)u16p[val_count - 1];
+		break;
+	case 4:
+		u32p = val;
+		if (write)
+			lastval = (unsigned int)u32p[val_count - 1];
+		break;
+#ifdef CONFIG_64BIT
+	case 8:
+		u64p = val;
+		if (write)
+			lastval = (unsigned int)u64p[val_count - 1];
+		break;
+#endif
+	default:
+		return -EINVAL;
+	}
+
+	/*
+	 * Update the cache with the last value we write, the rest is just
+	 * gone down in the hardware FIFO. We can't cache FIFOs. This makes
+	 * sure a single read from the cache will work.
+	 */
+	if (write) {
+		if (!map->cache_bypass && !map->defer_caching) {
+			ret = regcache_write(map, reg, lastval);
+			if (ret != 0)
+				return ret;
+			if (map->cache_only) {
+				map->cache_dirty = true;
+				return 0;
+			}
+		}
+		ret = map->bus->reg_noinc_write(map->bus_context, reg, val, val_count);
+	} else {
+		ret = map->bus->reg_noinc_read(map->bus_context, reg, val, val_count);
+	}
+
+	if (!ret && regmap_should_log(map)) {
+		dev_info(map->dev, "%x %s [", reg, write ? "<=" : "=>");
+		for (i = 0; i < val_count; i++) {
+			switch (val_bytes) {
+			case 1:
+				pr_cont("%x", u8p[i]);
+				break;
+			case 2:
+				pr_cont("%x", u16p[i]);
+				break;
+			case 4:
+				pr_cont("%x", u32p[i]);
+				break;
+#ifdef CONFIG_64BIT
+			case 8:
+				pr_cont("%llx", u64p[i]);
+				break;
+#endif
+			default:
+				break;
+			}
+			if (i == (val_count - 1))
+				pr_cont("]\n");
+			else
+				pr_cont(",");
+		}
+	}
+
+	return 0;
+}
+
 /**
  * regmap_noinc_write(): Write data from a register without incrementing the
  *			register number
@@ -2159,9 +2249,8 @@ int regmap_noinc_write(struct regmap *map, unsigned int reg,
 	size_t write_len;
 	int ret;
 
-	if (!map->write)
-		return -ENOTSUPP;
-
+	if (!map->write && !(map->bus && map->bus->reg_noinc_write))
+		return -EINVAL;
 	if (val_len % map->format.val_bytes)
 		return -EINVAL;
 	if (!IS_ALIGNED(reg, map->reg_stride))
@@ -2176,6 +2265,15 @@ int regmap_noinc_write(struct regmap *map, unsigned int reg,
 		goto out_unlock;
 	}
 
+	/*
+	 * Use the accelerated operation if we can. The val drops the const
+	 * typing in order to facilitate code reuse in regmap_noinc_readwrite().
+	 */
+	if (map->bus->reg_noinc_write) {
+		ret = regmap_noinc_readwrite(map, reg, (void *)val, val_len, true);
+		goto out_unlock;
+	}
+
 	while (val_len) {
 		if (map->max_raw_write && map->max_raw_write < val_len)
 			write_len = map->max_raw_write;
@@ -2350,6 +2448,10 @@ out:
 
 		kfree(wval);
 	}
+
+	if (!ret)
+		trace_regmap_bulk_write(map, reg, val, val_bytes * val_count);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(regmap_bulk_write);
@@ -2946,6 +3048,22 @@ int regmap_noinc_read(struct regmap *map, unsigned int reg,
 		goto out_unlock;
 	}
 
+	/* Use the accelerated operation if we can */
+	if (map->bus->reg_noinc_read) {
+		/*
+		 * We have not defined the FIFO semantics for cache, as the
+		 * cache is just one value deep. Should we return the last
+		 * written value? Just avoid this by always reading the FIFO
+		 * even when using cache. Cache only will not work.
+		 */
+		if (map->cache_only) {
+			ret = -EBUSY;
+			goto out_unlock;
+		}
+		ret = regmap_noinc_readwrite(map, reg, val, val_len, false);
+		goto out_unlock;
+	}
+
 	while (val_len) {
 		if (map->max_raw_read && map->max_raw_read < val_len)
 			read_len = map->max_raw_read;
@@ -3095,6 +3213,9 @@ out:
 		map->unlock(map->lock_arg);
 	}
 
+	if (!ret)
+		trace_regmap_bulk_read(map, reg, val, val_bytes * val_count);
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(regmap_bulk_read);
diff --git a/drivers/base/regmap/trace.h b/drivers/base/regmap/trace.h
index 9abee14df9ee..704e106e5dbd 100644
--- a/drivers/base/regmap/trace.h
+++ b/drivers/base/regmap/trace.h
@@ -32,9 +32,7 @@ DECLARE_EVENT_CLASS(regmap_reg,
 		__entry->val = val;
 	),
 
-	TP_printk("%s reg=%x val=%x", __get_str(name),
-		  (unsigned int)__entry->reg,
-		  (unsigned int)__entry->val)
+	TP_printk("%s reg=%x val=%x", __get_str(name), __entry->reg, __entry->val)
 );
 
 DEFINE_EVENT(regmap_reg, regmap_reg_write,
@@ -43,7 +41,6 @@ DEFINE_EVENT(regmap_reg, regmap_reg_write,
 		 unsigned int val),
 
 	TP_ARGS(map, reg, val)
-
 );
 
 DEFINE_EVENT(regmap_reg, regmap_reg_read,
@@ -52,7 +49,6 @@ DEFINE_EVENT(regmap_reg, regmap_reg_read,
 		 unsigned int val),
 
 	TP_ARGS(map, reg, val)
-
 );
 
 DEFINE_EVENT(regmap_reg, regmap_reg_read_cache,
@@ -61,7 +57,47 @@ DEFINE_EVENT(regmap_reg, regmap_reg_read_cache,
 		 unsigned int val),
 
 	TP_ARGS(map, reg, val)
+);
+
+DECLARE_EVENT_CLASS(regmap_bulk,
+
+	TP_PROTO(struct regmap *map, unsigned int reg,
+		 const void *val, int val_len),
+
+	TP_ARGS(map, reg, val, val_len),
+
+	TP_STRUCT__entry(
+		__string(name, regmap_name(map))
+		__field(unsigned int, reg)
+		__dynamic_array(char, buf, val_len)
+		__field(int, val_len)
+	),
+
+	TP_fast_assign(
+		__assign_str(name, regmap_name(map));
+		__entry->reg = reg;
+		__entry->val_len = val_len;
+		memcpy(__get_dynamic_array(buf), val, val_len);
+	),
 
+	TP_printk("%s reg=%x val=%s", __get_str(name), __entry->reg,
+		  __print_hex(__get_dynamic_array(buf), __entry->val_len))
+);
+
+DEFINE_EVENT(regmap_bulk, regmap_bulk_write,
+
+	TP_PROTO(struct regmap *map, unsigned int reg,
+		 const void *val, int val_len),
+
+	TP_ARGS(map, reg, val, val_len)
+);
+
+DEFINE_EVENT(regmap_bulk, regmap_bulk_read,
+
+	TP_PROTO(struct regmap *map, unsigned int reg,
+		 const void *val, int val_len),
+
+	TP_ARGS(map, reg, val, val_len)
 );
 
 DECLARE_EVENT_CLASS(regmap_block,
@@ -82,9 +118,7 @@ DECLARE_EVENT_CLASS(regmap_block,
 		__entry->count = count;
 	),
 
-	TP_printk("%s reg=%x count=%d", __get_str(name),
-		  (unsigned int)__entry->reg,
-		  (int)__entry->count)
+	TP_printk("%s reg=%x count=%d", __get_str(name), __entry->reg, __entry->count)
 );
 
 DEFINE_EVENT(regmap_block, regmap_hw_read_start,
@@ -154,8 +188,7 @@ DECLARE_EVENT_CLASS(regmap_bool,
 		__entry->flag = flag;
 	),
 
-	TP_printk("%s flag=%d", __get_str(name),
-		  (int)__entry->flag)
+	TP_printk("%s flag=%d", __get_str(name), __entry->flag)
 );
 
 DEFINE_EVENT(regmap_bool, regmap_cache_only,
@@ -163,7 +196,6 @@ DEFINE_EVENT(regmap_bool, regmap_cache_only,
 	TP_PROTO(struct regmap *map, bool flag),
 
 	TP_ARGS(map, flag)
-
 );
 
 DEFINE_EVENT(regmap_bool, regmap_cache_bypass,
@@ -171,7 +203,6 @@ DEFINE_EVENT(regmap_bool, regmap_cache_bypass,
 	TP_PROTO(struct regmap *map, bool flag),
 
 	TP_ARGS(map, flag)
-
 );
 
 DECLARE_EVENT_CLASS(regmap_async,
@@ -203,7 +234,6 @@ DEFINE_EVENT(regmap_async, regmap_async_io_complete,
 	TP_PROTO(struct regmap *map),
 
 	TP_ARGS(map)
-
 );
 
 DEFINE_EVENT(regmap_async, regmap_async_complete_start,
@@ -211,7 +241,6 @@ DEFINE_EVENT(regmap_async, regmap_async_complete_start,
 	TP_PROTO(struct regmap *map),
 
 	TP_ARGS(map)
-
 );
 
 DEFINE_EVENT(regmap_async, regmap_async_complete_done,
@@ -219,7 +248,6 @@ DEFINE_EVENT(regmap_async, regmap_async_complete_done,
 	TP_PROTO(struct regmap *map),
 
 	TP_ARGS(map)
-
 );
 
 TRACE_EVENT(regcache_drop_region,
@@ -241,8 +269,7 @@ TRACE_EVENT(regcache_drop_region,
 		__entry->to = to;
 	),
 
-	TP_printk("%s %u-%u", __get_str(name), (unsigned int)__entry->from,
-		  (unsigned int)__entry->to)
+	TP_printk("%s %u-%u", __get_str(name), __entry->from, __entry->to)
 );
 
 #endif /* _TRACE_REGMAP_H */
diff --git a/drivers/bcma/driver_mips.c b/drivers/bcma/driver_mips.c
index 12aca34e8db0..4f01e6b17bb9 100644
--- a/drivers/bcma/driver_mips.c
+++ b/drivers/bcma/driver_mips.c
@@ -30,7 +30,7 @@ enum bcma_boot_dev {
 	BCMA_BOOT_DEV_NAND,
 };
 
-/* The 47162a0 hangs when reading MIPS DMP registers registers */
+/* The 47162a0 hangs when reading MIPS DMP registers */
 static inline bool bcma_core_mips_bcm47162a0_quirk(struct bcma_device *dev)
 {
 	return dev->bus->chipinfo.id == BCMA_CHIP_ID_BCM47162 &&
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 2a709daefbc4..6cec9ce23fd3 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -2322,6 +2322,7 @@ static struct genl_family nbd_genl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= nbd_connect_genl_ops,
 	.n_small_ops	= ARRAY_SIZE(nbd_connect_genl_ops),
+	.resv_start_op	= NBD_CMD_STATUS + 1,
 	.maxattr	= NBD_ATTR_MAX,
 	.policy = nbd_attr_policy,
 	.mcgrps		= nbd_mcast_grps,
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index 30255fcaf181..dd9a05174726 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -322,14 +322,14 @@ static blk_status_t virtblk_prep_rq(struct blk_mq_hw_ctx *hctx,
 	if (unlikely(status))
 		return status;
 
-	blk_mq_start_request(req);
-
 	vbr->sg_table.nents = virtblk_map_data(hctx, req, vbr);
 	if (unlikely(vbr->sg_table.nents < 0)) {
 		virtblk_cleanup_cmd(req);
 		return BLK_STS_RESOURCE;
 	}
 
+	blk_mq_start_request(req);
+
 	return BLK_STS_OK;
 }
 
@@ -391,8 +391,7 @@ static bool virtblk_prep_rq_batch(struct request *req)
 }
 
 static bool virtblk_add_req_batch(struct virtio_blk_vq *vq,
-					struct request **rqlist,
-					struct request **requeue_list)
+					struct request **rqlist)
 {
 	unsigned long flags;
 	int err;
@@ -408,7 +407,7 @@ static bool virtblk_add_req_batch(struct virtio_blk_vq *vq,
 		if (err) {
 			virtblk_unmap_data(req, vbr);
 			virtblk_cleanup_cmd(req);
-			rq_list_add(requeue_list, req);
+			blk_mq_requeue_request(req, true);
 		}
 	}
 
@@ -436,7 +435,7 @@ static void virtio_queue_rqs(struct request **rqlist)
 
 		if (!next || req->mq_hctx != next->mq_hctx) {
 			req->rq_next = NULL;
-			kick = virtblk_add_req_batch(vq, rqlist, &requeue_list);
+			kick = virtblk_add_req_batch(vq, rqlist);
 			if (kick)
 				virtqueue_notify(vq->vq);
 
diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c
index 818681c89db8..a657e9a3e96a 100644
--- a/drivers/bluetooth/btintel.c
+++ b/drivers/bluetooth/btintel.c
@@ -449,6 +449,7 @@ static int btintel_version_info_tlv(struct hci_dev *hdev,
 	case 0x17:	/* TyP */
 	case 0x18:	/* Slr */
 	case 0x19:	/* Slr-F */
+	case 0x1b:      /* Mgr */
 		break;
 	default:
 		bt_dev_err(hdev, "Unsupported Intel hardware variant (0x%x)",
@@ -2330,6 +2331,7 @@ static void btintel_set_msft_opcode(struct hci_dev *hdev, u8 hw_variant)
 	case 0x17:
 	case 0x18:
 	case 0x19:
+	case 0x1b:
 		hci_set_msft_opcode(hdev, 0xFC1E);
 		break;
 	default:
@@ -2439,15 +2441,20 @@ static int btintel_setup_combined(struct hci_dev *hdev)
 					       INTEL_ROM_LEGACY_NO_WBS_SUPPORT))
 				set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED,
 					&hdev->quirks);
+			if (ver.hw_variant == 0x08 && ver.fw_variant == 0x22)
+				set_bit(HCI_QUIRK_VALID_LE_STATES,
+					&hdev->quirks);
 
 			err = btintel_legacy_rom_setup(hdev, &ver);
 			break;
 		case 0x0b:      /* SfP */
-		case 0x0c:      /* WsP */
 		case 0x11:      /* JfP */
 		case 0x12:      /* ThP */
 		case 0x13:      /* HrP */
 		case 0x14:      /* CcP */
+			set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks);
+			fallthrough;
+		case 0x0c:	/* WsP */
 			/* Apply the device specific HCI quirks
 			 *
 			 * All Legacy bootloader devices support WBS
@@ -2455,11 +2462,6 @@ static int btintel_setup_combined(struct hci_dev *hdev)
 			set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED,
 				&hdev->quirks);
 
-			/* Valid LE States quirk for JfP/ThP familiy */
-			if (ver.hw_variant == 0x11 || ver.hw_variant == 0x12)
-				set_bit(HCI_QUIRK_VALID_LE_STATES,
-					&hdev->quirks);
-
 			/* Setup MSFT Extension support */
 			btintel_set_msft_opcode(hdev, ver.hw_variant);
 
@@ -2530,9 +2532,8 @@ static int btintel_setup_combined(struct hci_dev *hdev)
 		 */
 		set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED, &hdev->quirks);
 
-		/* Valid LE States quirk for JfP/ThP familiy */
-		if (ver.hw_variant == 0x11 || ver.hw_variant == 0x12)
-			set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks);
+		/* Set Valid LE States quirk */
+		set_bit(HCI_QUIRK_VALID_LE_STATES, &hdev->quirks);
 
 		/* Setup MSFT Extension support */
 		btintel_set_msft_opcode(hdev, ver.hw_variant);
@@ -2542,6 +2543,7 @@ static int btintel_setup_combined(struct hci_dev *hdev)
 	case 0x17:
 	case 0x18:
 	case 0x19:
+	case 0x1b:
 		/* Display version information of TLV type */
 		btintel_version_info_tlv(hdev, &ver_tlv);
 
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index 15caa6469538..271963805a38 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -426,6 +426,8 @@ static const struct usb_device_id blacklist_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x04ca, 0x4006), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0cb8, 0xc549), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
 
 	/* Realtek 8852CE Bluetooth devices */
 	{ USB_DEVICE(0x04ca, 0x4007), .driver_info = BTUSB_REALTEK |
@@ -438,6 +440,8 @@ static const struct usb_device_id blacklist_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3586), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3592), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
 
 	/* Realtek Bluetooth devices */
 	{ USB_VENDOR_AND_INTERFACE_INFO(0x0bda, 0xe0, 0x01, 0x01),
@@ -466,6 +470,9 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x0489, 0xe0c8), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH |
 						     BTUSB_VALID_LE_STATES },
+	{ USB_DEVICE(0x0489, 0xe0e0), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH |
+						     BTUSB_VALID_LE_STATES },
 	{ USB_DEVICE(0x04ca, 0x3802), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH |
 						     BTUSB_VALID_LE_STATES },
@@ -478,9 +485,18 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x13d3, 0x3567), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH |
 						     BTUSB_VALID_LE_STATES },
+	{ USB_DEVICE(0x13d3, 0x3578), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH |
+						     BTUSB_VALID_LE_STATES },
+	{ USB_DEVICE(0x13d3, 0x3583), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH |
+						     BTUSB_VALID_LE_STATES },
 	{ USB_DEVICE(0x0489, 0xe0cd), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH |
 						     BTUSB_VALID_LE_STATES },
+	{ USB_DEVICE(0x0e8d, 0x0608), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH |
+						     BTUSB_VALID_LE_STATES },
 
 	/* MediaTek MT7922A Bluetooth devices */
 	{ USB_DEVICE(0x0489, 0xe0d8), .driver_info = BTUSB_MEDIATEK |
@@ -516,19 +532,17 @@ static const struct usb_device_id blacklist_table[] = {
 	{ USB_DEVICE(0x0bda, 0xb009), .driver_info = BTUSB_REALTEK },
 	{ USB_DEVICE(0x2ff8, 0xb011), .driver_info = BTUSB_REALTEK },
 
-	/* Additional Realtek 8761B Bluetooth devices */
+	/* Additional Realtek 8761BUV Bluetooth devices */
 	{ USB_DEVICE(0x2357, 0x0604), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Additional Realtek 8761BU Bluetooth devices */
 	{ USB_DEVICE(0x0b05, 0x190e), .driver_info = BTUSB_REALTEK |
 	  					     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x2550, 0x8761), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
-
-	/* Additional Realtek 8761BUV Bluetooth devices */
 	{ USB_DEVICE(0x0bda, 0x8771), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x7392, 0xc611), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
 
 	/* Additional Realtek 8821AE Bluetooth devices */
 	{ USB_DEVICE(0x0b05, 0x17dc), .driver_info = BTUSB_REALTEK },
@@ -2477,15 +2491,29 @@ static int btusb_mtk_hci_wmt_sync(struct hci_dev *hdev,
 
 	set_bit(BTUSB_TX_WAIT_VND_EVT, &data->flags);
 
+	/* WMT cmd/event doesn't follow up the generic HCI cmd/event handling,
+	 * it needs constantly polling control pipe until the host received the
+	 * WMT event, thus, we should require to specifically acquire PM counter
+	 * on the USB to prevent the interface from entering auto suspended
+	 * while WMT cmd/event in progress.
+	 */
+	err = usb_autopm_get_interface(data->intf);
+	if (err < 0)
+		goto err_free_wc;
+
 	err = __hci_cmd_send(hdev, 0xfc6f, hlen, wc);
 
 	if (err < 0) {
 		clear_bit(BTUSB_TX_WAIT_VND_EVT, &data->flags);
+		usb_autopm_put_interface(data->intf);
 		goto err_free_wc;
 	}
 
 	/* Submit control IN URB on demand to process the WMT event */
 	err = btusb_mtk_submit_wmt_recv_urb(hdev);
+
+	usb_autopm_put_interface(data->intf);
+
 	if (err < 0)
 		goto err_free_wc;
 
diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c
index f537673ede17..865112e96ff9 100644
--- a/drivers/bluetooth/hci_ldisc.c
+++ b/drivers/bluetooth/hci_ldisc.c
@@ -493,6 +493,11 @@ static int hci_uart_tty_open(struct tty_struct *tty)
 		BT_ERR("Can't allocate control structure");
 		return -ENFILE;
 	}
+	if (percpu_init_rwsem(&hu->proto_lock)) {
+		BT_ERR("Can't allocate semaphore structure");
+		kfree(hu);
+		return -ENOMEM;
+	}
 
 	tty->disc_data = hu;
 	hu->tty = tty;
@@ -505,8 +510,6 @@ static int hci_uart_tty_open(struct tty_struct *tty)
 	INIT_WORK(&hu->init_ready, hci_uart_init_work);
 	INIT_WORK(&hu->write_work, hci_uart_write_work);
 
-	percpu_init_rwsem(&hu->proto_lock);
-
 	/* Flush any pending characters in the driver */
 	tty_driver_flush_buffer(tty);
 
diff --git a/drivers/bluetooth/hci_serdev.c b/drivers/bluetooth/hci_serdev.c
index c0e5f42ec6b7..f16fd79bc02b 100644
--- a/drivers/bluetooth/hci_serdev.c
+++ b/drivers/bluetooth/hci_serdev.c
@@ -310,11 +310,12 @@ int hci_uart_register_device(struct hci_uart *hu,
 
 	serdev_device_set_client_ops(hu->serdev, &hci_serdev_client_ops);
 
+	if (percpu_init_rwsem(&hu->proto_lock))
+		return -ENOMEM;
+
 	err = serdev_device_open(hu->serdev);
 	if (err)
-		return err;
-
-	percpu_init_rwsem(&hu->proto_lock);
+		goto err_rwsem;
 
 	err = p->open(hu);
 	if (err)
@@ -389,6 +390,8 @@ err_alloc:
 	p->close(hu);
 err_open:
 	serdev_device_close(hu->serdev);
+err_rwsem:
+	percpu_free_rwsem(&hu->proto_lock);
 	return err;
 }
 EXPORT_SYMBOL_GPL(hci_uart_register_device);
@@ -410,5 +413,6 @@ void hci_uart_unregister_device(struct hci_uart *hu)
 		clear_bit(HCI_UART_PROTO_READY, &hu->flags);
 		serdev_device_close(hu->serdev);
 	}
+	percpu_free_rwsem(&hu->proto_lock);
 }
 EXPORT_SYMBOL_GPL(hci_uart_unregister_device);
diff --git a/drivers/bus/hisi_lpc.c b/drivers/bus/hisi_lpc.c
index 2e564803e786..5b65a48f17e7 100644
--- a/drivers/bus/hisi_lpc.c
+++ b/drivers/bus/hisi_lpc.c
@@ -85,7 +85,7 @@ static int wait_lpc_idle(void __iomem *mbase, unsigned int waitcnt)
 		ndelay(LPC_NSEC_PERWAIT);
 	} while (--waitcnt);
 
-	return -ETIME;
+	return -ETIMEDOUT;
 }
 
 /*
@@ -347,7 +347,7 @@ static int hisi_lpc_acpi_xlat_io_res(struct acpi_device *adev,
 	unsigned long sys_port;
 	resource_size_t len = resource_size(res);
 
-	sys_port = logic_pio_trans_hwaddr(&host->fwnode, res->start, len);
+	sys_port = logic_pio_trans_hwaddr(acpi_fwnode_handle(host), res->start, len);
 	if (sys_port == ~0UL)
 		return -EFAULT;
 
@@ -472,9 +472,7 @@ static int hisi_lpc_acpi_clear_enumerated(struct acpi_device *adev, void *not_us
 
 struct hisi_lpc_acpi_cell {
 	const char *hid;
-	const char *name;
-	void *pdata;
-	size_t pdata_size;
+	const struct platform_device_info *pdevinfo;
 };
 
 static void hisi_lpc_acpi_remove(struct device *hostdev)
@@ -505,28 +503,45 @@ static int hisi_lpc_acpi_add_child(struct acpi_device *child, void *data)
 		/* ipmi */
 		{
 			.hid = "IPI0001",
-			.name = "hisi-lpc-ipmi",
+			.pdevinfo = (struct platform_device_info []) {
+				{
+					.parent = hostdev,
+					.fwnode = acpi_fwnode_handle(child),
+					.name = "hisi-lpc-ipmi",
+					.id = PLATFORM_DEVID_AUTO,
+					.res = res,
+					.num_res = num_res,
+				},
+			},
 		},
 		/* 8250-compatible uart */
 		{
 			.hid = "HISI1031",
-			.name = "serial8250",
-			.pdata = (struct plat_serial8250_port []) {
+			.pdevinfo = (struct platform_device_info []) {
 				{
-					.iobase = res->start,
-					.uartclk = 1843200,
-					.iotype = UPIO_PORT,
-					.flags = UPF_BOOT_AUTOCONF,
+					.parent = hostdev,
+					.fwnode = acpi_fwnode_handle(child),
+					.name = "serial8250",
+					.id = PLATFORM_DEVID_AUTO,
+					.res = res,
+					.num_res = num_res,
+					.data = (struct plat_serial8250_port []) {
+						{
+							.iobase = res->start,
+							.uartclk = 1843200,
+							.iotype = UPIO_PORT,
+							.flags = UPF_BOOT_AUTOCONF,
+						},
+						{}
+					},
+					.size_data =  2 * sizeof(struct plat_serial8250_port),
 				},
-				{}
 			},
-			.pdata_size = 2 *
-				sizeof(struct plat_serial8250_port),
 		},
 		{}
 	};
 
-	for (; cell && cell->name; cell++) {
+	for (; cell && cell->hid; cell++) {
 		if (!strcmp(cell->hid, hid)) {
 			found = true;
 			break;
@@ -540,31 +555,12 @@ static int hisi_lpc_acpi_add_child(struct acpi_device *child, void *data)
 		return 0;
 	}
 
-	pdev = platform_device_alloc(cell->name, PLATFORM_DEVID_AUTO);
-	if (!pdev)
-		return -ENOMEM;
-
-	pdev->dev.parent = hostdev;
-	ACPI_COMPANION_SET(&pdev->dev, child);
-
-	ret = platform_device_add_resources(pdev, res, num_res);
-	if (ret)
-		goto fail;
-
-	ret = platform_device_add_data(pdev, cell->pdata, cell->pdata_size);
-	if (ret)
-		goto fail;
-
-	ret = platform_device_add(pdev);
-	if (ret)
-		goto fail;
+	pdev = platform_device_register_full(cell->pdevinfo);
+	if (IS_ERR(pdev))
+		return PTR_ERR(pdev);
 
 	acpi_device_set_enumerated(child);
 	return 0;
-
-fail:
-	platform_device_put(pdev);
-	return ret;
 }
 
 /*
@@ -589,11 +585,6 @@ static int hisi_lpc_acpi_probe(struct device *hostdev)
 
 	return ret;
 }
-
-static const struct acpi_device_id hisi_lpc_acpi_match[] = {
-	{"HISI0191"},
-	{}
-};
 #else
 static int hisi_lpc_acpi_probe(struct device *dev)
 {
@@ -615,11 +606,9 @@ static void hisi_lpc_acpi_remove(struct device *hostdev)
 static int hisi_lpc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct acpi_device *acpi_device = ACPI_COMPANION(dev);
 	struct logic_pio_hwaddr *range;
 	struct hisi_lpc_dev *lpcdev;
 	resource_size_t io_end;
-	struct resource *res;
 	int ret;
 
 	lpcdev = devm_kzalloc(dev, sizeof(*lpcdev), GFP_KERNEL);
@@ -628,8 +617,7 @@ static int hisi_lpc_probe(struct platform_device *pdev)
 
 	spin_lock_init(&lpcdev->cycle_lock);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	lpcdev->membase = devm_ioremap_resource(dev, res);
+	lpcdev->membase = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(lpcdev->membase))
 		return PTR_ERR(lpcdev->membase);
 
@@ -637,7 +625,7 @@ static int hisi_lpc_probe(struct platform_device *pdev)
 	if (!range)
 		return -ENOMEM;
 
-	range->fwnode = dev->fwnode;
+	range->fwnode = dev_fwnode(dev);
 	range->flags = LOGIC_PIO_INDIRECT;
 	range->size = PIO_INDIRECT_SIZE;
 	range->hostdata = lpcdev;
@@ -651,7 +639,7 @@ static int hisi_lpc_probe(struct platform_device *pdev)
 	}
 
 	/* register the LPC host PIO resources */
-	if (acpi_device)
+	if (is_acpi_device_node(range->fwnode))
 		ret = hisi_lpc_acpi_probe(dev);
 	else
 		ret = of_platform_populate(dev->of_node, NULL, NULL, dev);
@@ -672,11 +660,10 @@ static int hisi_lpc_probe(struct platform_device *pdev)
 static int hisi_lpc_remove(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct acpi_device *acpi_device = ACPI_COMPANION(dev);
 	struct hisi_lpc_dev *lpcdev = dev_get_drvdata(dev);
 	struct logic_pio_hwaddr *range = lpcdev->io_host;
 
-	if (acpi_device)
+	if (is_acpi_device_node(range->fwnode))
 		hisi_lpc_acpi_remove(dev);
 	else
 		of_platform_depopulate(dev);
@@ -692,11 +679,16 @@ static const struct of_device_id hisi_lpc_of_match[] = {
 	{}
 };
 
+static const struct acpi_device_id hisi_lpc_acpi_match[] = {
+	{"HISI0191"},
+	{}
+};
+
 static struct platform_driver hisi_lpc_driver = {
 	.driver = {
 		.name           = DRV_NAME,
 		.of_match_table = hisi_lpc_of_match,
-		.acpi_match_table = ACPI_PTR(hisi_lpc_acpi_match),
+		.acpi_match_table = hisi_lpc_acpi_match,
 	},
 	.probe = hisi_lpc_probe,
 	.remove = hisi_lpc_remove,
diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c
index 9e545f2a5a26..fa2246da63c1 100644
--- a/drivers/bus/mhi/host/pci_generic.c
+++ b/drivers/bus/mhi/host/pci_generic.c
@@ -507,6 +507,8 @@ static const struct mhi_channel_config mhi_telit_fn990_channels[] = {
 	MHI_CHANNEL_CONFIG_DL(13, "MBIM", 32, 0),
 	MHI_CHANNEL_CONFIG_UL(32, "DUN", 32, 0),
 	MHI_CHANNEL_CONFIG_DL(33, "DUN", 32, 0),
+	MHI_CHANNEL_CONFIG_UL(92, "DUN2", 32, 1),
+	MHI_CHANNEL_CONFIG_DL(93, "DUN2", 32, 1),
 	MHI_CHANNEL_CONFIG_HW_UL(100, "IP_HW0_MBIM", 128, 2),
 	MHI_CHANNEL_CONFIG_HW_DL(101, "IP_HW0_MBIM", 128, 3),
 };
diff --git a/drivers/char/ipmi/ipmb_dev_int.c b/drivers/char/ipmi/ipmb_dev_int.c
index db40037eb347..a0e9e80d92ee 100644
--- a/drivers/char/ipmi/ipmb_dev_int.c
+++ b/drivers/char/ipmi/ipmb_dev_int.c
@@ -341,14 +341,12 @@ static int ipmb_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int ipmb_remove(struct i2c_client *client)
+static void ipmb_remove(struct i2c_client *client)
 {
 	struct ipmb_dev *ipmb_dev = i2c_get_clientdata(client);
 
 	i2c_slave_unregister(client);
 	misc_deregister(&ipmb_dev->miscdev);
-
-	return 0;
 }
 
 static const struct i2c_device_id ipmb_id[] = {
diff --git a/drivers/char/ipmi/ipmi_ipmb.c b/drivers/char/ipmi/ipmi_ipmb.c
index ab19b4b3317e..25c010c9ec25 100644
--- a/drivers/char/ipmi/ipmi_ipmb.c
+++ b/drivers/char/ipmi/ipmi_ipmb.c
@@ -424,7 +424,7 @@ static void ipmi_ipmb_request_events(void *send_info)
 	/* We don't fetch events here. */
 }
 
-static int ipmi_ipmb_remove(struct i2c_client *client)
+static void ipmi_ipmb_remove(struct i2c_client *client)
 {
 	struct ipmi_ipmb_dev *iidev = i2c_get_clientdata(client);
 
@@ -438,8 +438,6 @@ static int ipmi_ipmb_remove(struct i2c_client *client)
 	ipmi_ipmb_stop_thread(iidev);
 
 	ipmi_unregister_smi(iidev->intf);
-
-	return 0;
 }
 
 static int ipmi_ipmb_probe(struct i2c_client *client)
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
index fc742ee9c046..13da021e7c6b 100644
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -1281,13 +1281,13 @@ static void shutdown_ssif(void *send_info)
 	}
 }
 
-static int ssif_remove(struct i2c_client *client)
+static void ssif_remove(struct i2c_client *client)
 {
 	struct ssif_info *ssif_info = i2c_get_clientdata(client);
 	struct ssif_addr_info *addr_info;
 
 	if (!ssif_info)
-		return 0;
+		return;
 
 	/*
 	 * After this point, we won't deliver anything asychronously
@@ -1303,8 +1303,6 @@ static int ssif_remove(struct i2c_client *client)
 	}
 
 	kfree(ssif_info);
-
-	return 0;
 }
 
 static int read_response(struct i2c_client *client, unsigned char *resp)
diff --git a/drivers/char/tpm/st33zp24/i2c.c b/drivers/char/tpm/st33zp24/i2c.c
index 3170d59d660c..a3aa411389e7 100644
--- a/drivers/char/tpm/st33zp24/i2c.c
+++ b/drivers/char/tpm/st33zp24/i2c.c
@@ -264,13 +264,11 @@ static int st33zp24_i2c_probe(struct i2c_client *client,
  * @param: client, the i2c_client description (TPM I2C description).
  * @return: 0 in case of success.
  */
-static int st33zp24_i2c_remove(struct i2c_client *client)
+static void st33zp24_i2c_remove(struct i2c_client *client)
 {
 	struct tpm_chip *chip = i2c_get_clientdata(client);
 
 	st33zp24_remove(chip);
-
-	return 0;
 }
 
 static const struct i2c_device_id st33zp24_i2c_id[] = {
diff --git a/drivers/char/tpm/tpm_i2c_atmel.c b/drivers/char/tpm/tpm_i2c_atmel.c
index d5ac85558214..4be3677c1463 100644
--- a/drivers/char/tpm/tpm_i2c_atmel.c
+++ b/drivers/char/tpm/tpm_i2c_atmel.c
@@ -179,12 +179,11 @@ static int i2c_atmel_probe(struct i2c_client *client,
 	return tpm_chip_register(chip);
 }
 
-static int i2c_atmel_remove(struct i2c_client *client)
+static void i2c_atmel_remove(struct i2c_client *client)
 {
 	struct device *dev = &(client->dev);
 	struct tpm_chip *chip = dev_get_drvdata(dev);
 	tpm_chip_unregister(chip);
-	return 0;
 }
 
 static const struct i2c_device_id i2c_atmel_id[] = {
diff --git a/drivers/char/tpm/tpm_i2c_infineon.c b/drivers/char/tpm/tpm_i2c_infineon.c
index a19d32cb4e94..fd3c3661e646 100644
--- a/drivers/char/tpm/tpm_i2c_infineon.c
+++ b/drivers/char/tpm/tpm_i2c_infineon.c
@@ -706,15 +706,13 @@ static int tpm_tis_i2c_probe(struct i2c_client *client,
 	return rc;
 }
 
-static int tpm_tis_i2c_remove(struct i2c_client *client)
+static void tpm_tis_i2c_remove(struct i2c_client *client)
 {
 	struct tpm_chip *chip = tpm_dev.chip;
 
 	tpm_chip_unregister(chip);
 	release_locality(chip, tpm_dev.locality, 1);
 	tpm_dev.client = NULL;
-
-	return 0;
 }
 
 static struct i2c_driver tpm_tis_i2c_driver = {
diff --git a/drivers/char/tpm/tpm_i2c_nuvoton.c b/drivers/char/tpm/tpm_i2c_nuvoton.c
index b77c18e38662..95c37350cc8e 100644
--- a/drivers/char/tpm/tpm_i2c_nuvoton.c
+++ b/drivers/char/tpm/tpm_i2c_nuvoton.c
@@ -622,12 +622,11 @@ static int i2c_nuvoton_probe(struct i2c_client *client,
 	return tpm_chip_register(chip);
 }
 
-static int i2c_nuvoton_remove(struct i2c_client *client)
+static void i2c_nuvoton_remove(struct i2c_client *client)
 {
 	struct tpm_chip *chip = i2c_get_clientdata(client);
 
 	tpm_chip_unregister(chip);
-	return 0;
 }
 
 static const struct i2c_device_id i2c_nuvoton_id[] = {
diff --git a/drivers/char/tpm/tpm_tis_i2c.c b/drivers/char/tpm/tpm_tis_i2c.c
index ba0911b1d1ff..0692510dfcab 100644
--- a/drivers/char/tpm/tpm_tis_i2c.c
+++ b/drivers/char/tpm/tpm_tis_i2c.c
@@ -351,13 +351,12 @@ static int tpm_tis_i2c_probe(struct i2c_client *dev,
 				 NULL);
 }
 
-static int tpm_tis_i2c_remove(struct i2c_client *client)
+static void tpm_tis_i2c_remove(struct i2c_client *client)
 {
 	struct tpm_chip *chip = i2c_get_clientdata(client);
 
 	tpm_chip_unregister(chip);
 	tpm_tis_remove(chip);
-	return 0;
 }
 
 static const struct i2c_device_id tpm_tis_i2c_id[] = {
diff --git a/drivers/char/tpm/tpm_tis_i2c_cr50.c b/drivers/char/tpm/tpm_tis_i2c_cr50.c
index 974479a1ec5a..77cea5b31c6e 100644
--- a/drivers/char/tpm/tpm_tis_i2c_cr50.c
+++ b/drivers/char/tpm/tpm_tis_i2c_cr50.c
@@ -763,20 +763,18 @@ static int tpm_cr50_i2c_probe(struct i2c_client *client)
  * - 0:		Success.
  * - -errno:	A POSIX error code.
  */
-static int tpm_cr50_i2c_remove(struct i2c_client *client)
+static void tpm_cr50_i2c_remove(struct i2c_client *client)
 {
 	struct tpm_chip *chip = i2c_get_clientdata(client);
 	struct device *dev = &client->dev;
 
 	if (!chip) {
 		dev_crit(dev, "Could not get client data at remove, memory corruption ahead\n");
-		return 0;
+		return;
 	}
 
 	tpm_chip_unregister(chip);
 	tpm_cr50_release_locality(chip, true);
-
-	return 0;
 }
 
 static SIMPLE_DEV_PM_OPS(cr50_i2c_pm, tpm_pm_suspend, tpm_pm_resume);
diff --git a/drivers/clk/bcm/clk-iproc-pll.c b/drivers/clk/bcm/clk-iproc-pll.c
index 1a098db12062..680f9d8d357c 100644
--- a/drivers/clk/bcm/clk-iproc-pll.c
+++ b/drivers/clk/bcm/clk-iproc-pll.c
@@ -726,6 +726,7 @@ void iproc_pll_clk_setup(struct device_node *node,
 	const char *parent_name;
 	struct iproc_clk *iclk_array;
 	struct clk_hw_onecell_data *clk_data;
+	const char *clk_name;
 
 	if (WARN_ON(!pll_ctrl) || WARN_ON(!clk_ctrl))
 		return;
@@ -773,7 +774,12 @@ void iproc_pll_clk_setup(struct device_node *node,
 	iclk = &iclk_array[0];
 	iclk->pll = pll;
 
-	init.name = node->name;
+	ret = of_property_read_string_index(node, "clock-output-names",
+					    0, &clk_name);
+	if (WARN_ON(ret))
+		goto err_pll_register;
+
+	init.name = clk_name;
 	init.ops = &iproc_pll_ops;
 	init.flags = 0;
 	parent_name = of_clk_get_parent_name(node, 0);
@@ -793,13 +799,11 @@ void iproc_pll_clk_setup(struct device_node *node,
 		goto err_pll_register;
 
 	clk_data->hws[0] = &iclk->hw;
+	parent_name = clk_name;
 
 	/* now initialize and register all leaf clocks */
 	for (i = 1; i < num_clks; i++) {
-		const char *clk_name;
-
 		memset(&init, 0, sizeof(init));
-		parent_name = node->name;
 
 		ret = of_property_read_string_index(node, "clock-output-names",
 						    i, &clk_name);
diff --git a/drivers/clk/clk-cdce706.c b/drivers/clk/clk-cdce706.c
index 5467d941ddfd..1449d0537674 100644
--- a/drivers/clk/clk-cdce706.c
+++ b/drivers/clk/clk-cdce706.c
@@ -665,10 +665,9 @@ static int cdce706_probe(struct i2c_client *client)
 				      cdce);
 }
 
-static int cdce706_remove(struct i2c_client *client)
+static void cdce706_remove(struct i2c_client *client)
 {
 	of_clk_del_provider(client->dev.of_node);
-	return 0;
 }
 
 
diff --git a/drivers/clk/clk-cs2000-cp.c b/drivers/clk/clk-cs2000-cp.c
index aa5c72bab83e..320d39922206 100644
--- a/drivers/clk/clk-cs2000-cp.c
+++ b/drivers/clk/clk-cs2000-cp.c
@@ -557,7 +557,7 @@ static int cs2000_version_print(struct cs2000_priv *priv)
 	return 0;
 }
 
-static int cs2000_remove(struct i2c_client *client)
+static void cs2000_remove(struct i2c_client *client)
 {
 	struct cs2000_priv *priv = i2c_get_clientdata(client);
 	struct device *dev = priv_to_dev(priv);
@@ -566,8 +566,6 @@ static int cs2000_remove(struct i2c_client *client)
 	of_clk_del_provider(np);
 
 	clk_hw_unregister(&priv->hw);
-
-	return 0;
 }
 
 static int cs2000_probe(struct i2c_client *client)
diff --git a/drivers/clk/clk-si514.c b/drivers/clk/clk-si514.c
index 4481c4303534..c028fa103bed 100644
--- a/drivers/clk/clk-si514.c
+++ b/drivers/clk/clk-si514.c
@@ -370,10 +370,9 @@ static int si514_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int si514_remove(struct i2c_client *client)
+static void si514_remove(struct i2c_client *client)
 {
 	of_clk_del_provider(client->dev.of_node);
-	return 0;
 }
 
 static const struct i2c_device_id si514_id[] = {
diff --git a/drivers/clk/clk-si5341.c b/drivers/clk/clk-si5341.c
index 4bca73212662..0e528d7ba656 100644
--- a/drivers/clk/clk-si5341.c
+++ b/drivers/clk/clk-si5341.c
@@ -1796,7 +1796,7 @@ cleanup:
 	return err;
 }
 
-static int si5341_remove(struct i2c_client *client)
+static void si5341_remove(struct i2c_client *client)
 {
 	struct clk_si5341 *data = i2c_get_clientdata(client);
 	int i;
@@ -1807,8 +1807,6 @@ static int si5341_remove(struct i2c_client *client)
 		if (data->clk[i].vddo_reg)
 			regulator_disable(data->clk[i].vddo_reg);
 	}
-
-	return 0;
 }
 
 static const struct i2c_device_id si5341_id[] = {
diff --git a/drivers/clk/clk-si5351.c b/drivers/clk/clk-si5351.c
index b9f088c4ba2f..9e939c98a455 100644
--- a/drivers/clk/clk-si5351.c
+++ b/drivers/clk/clk-si5351.c
@@ -1651,11 +1651,9 @@ static int si5351_i2c_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int si5351_i2c_remove(struct i2c_client *client)
+static void si5351_i2c_remove(struct i2c_client *client)
 {
 	of_clk_del_provider(client->dev.of_node);
-
-	return 0;
 }
 
 static struct i2c_driver si5351_driver = {
diff --git a/drivers/clk/clk-si570.c b/drivers/clk/clk-si570.c
index 1ff8f32f734d..0a6d70c49726 100644
--- a/drivers/clk/clk-si570.c
+++ b/drivers/clk/clk-si570.c
@@ -498,10 +498,9 @@ static int si570_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int si570_remove(struct i2c_client *client)
+static void si570_remove(struct i2c_client *client)
 {
 	of_clk_del_provider(client->dev.of_node);
-	return 0;
 }
 
 static const struct of_device_id clk_si570_of_match[] = {
diff --git a/drivers/clk/clk-tps68470.c b/drivers/clk/clk-tps68470.c
index e5fbefd6ac2d..38f44b5b9b1b 100644
--- a/drivers/clk/clk-tps68470.c
+++ b/drivers/clk/clk-tps68470.c
@@ -200,7 +200,9 @@ static int tps68470_clk_probe(struct platform_device *pdev)
 		.flags = CLK_SET_RATE_GATE,
 	};
 	struct tps68470_clkdata *tps68470_clkdata;
+	struct tps68470_clk_consumer *consumer;
 	int ret;
+	int i;
 
 	tps68470_clkdata = devm_kzalloc(&pdev->dev, sizeof(*tps68470_clkdata),
 					GFP_KERNEL);
@@ -223,10 +225,13 @@ static int tps68470_clk_probe(struct platform_device *pdev)
 		return ret;
 
 	if (pdata) {
-		ret = devm_clk_hw_register_clkdev(&pdev->dev,
-						  &tps68470_clkdata->clkout_hw,
-						  pdata->consumer_con_id,
-						  pdata->consumer_dev_name);
+		for (i = 0; i < pdata->n_consumers; i++) {
+			consumer = &pdata->consumers[i];
+			ret = devm_clk_hw_register_clkdev(&pdev->dev,
+							  &tps68470_clkdata->clkout_hw,
+							  consumer->consumer_con_id,
+							  consumer->consumer_dev_name);
+		}
 	}
 
 	return ret;
diff --git a/drivers/clk/clk-versaclock5.c b/drivers/clk/clk-versaclock5.c
index e7be3e54b9be..657493ecce4c 100644
--- a/drivers/clk/clk-versaclock5.c
+++ b/drivers/clk/clk-versaclock5.c
@@ -1138,7 +1138,7 @@ err_clk:
 	return ret;
 }
 
-static int vc5_remove(struct i2c_client *client)
+static void vc5_remove(struct i2c_client *client)
 {
 	struct vc5_driver_data *vc5 = i2c_get_clientdata(client);
 
@@ -1146,8 +1146,6 @@ static int vc5_remove(struct i2c_client *client)
 
 	if (vc5->chip_info->flags & VC5_HAS_INTERNAL_XTAL)
 		clk_unregister_fixed_rate(vc5->pin_xin);
-
-	return 0;
 }
 
 static int __maybe_unused vc5_suspend(struct device *dev)
diff --git a/drivers/clk/imx/clk-imx6sx.c b/drivers/clk/imx/clk-imx6sx.c
index fc1bd23d4583..598f3cf4eba4 100644
--- a/drivers/clk/imx/clk-imx6sx.c
+++ b/drivers/clk/imx/clk-imx6sx.c
@@ -280,13 +280,13 @@ static void __init imx6sx_clocks_init(struct device_node *ccm_node)
 	hws[IMX6SX_CLK_SSI3_SEL]           = imx_clk_hw_mux("ssi3_sel",         base + 0x1c,  14,     2,      ssi_sels,          ARRAY_SIZE(ssi_sels));
 	hws[IMX6SX_CLK_SSI2_SEL]           = imx_clk_hw_mux("ssi2_sel",         base + 0x1c,  12,     2,      ssi_sels,          ARRAY_SIZE(ssi_sels));
 	hws[IMX6SX_CLK_SSI1_SEL]           = imx_clk_hw_mux("ssi1_sel",         base + 0x1c,  10,     2,      ssi_sels,          ARRAY_SIZE(ssi_sels));
-	hws[IMX6SX_CLK_QSPI1_SEL]          = imx_clk_hw_mux_flags("qspi1_sel", base + 0x1c,  7, 3, qspi1_sels, ARRAY_SIZE(qspi1_sels), CLK_SET_RATE_PARENT);
+	hws[IMX6SX_CLK_QSPI1_SEL]          = imx_clk_hw_mux("qspi1_sel",        base + 0x1c,  7,      3,      qspi1_sels,        ARRAY_SIZE(qspi1_sels));
 	hws[IMX6SX_CLK_PERCLK_SEL]         = imx_clk_hw_mux("perclk_sel",       base + 0x1c,  6,      1,      perclk_sels,       ARRAY_SIZE(perclk_sels));
 	hws[IMX6SX_CLK_VID_SEL]            = imx_clk_hw_mux("vid_sel",          base + 0x20,  21,     3,      vid_sels,          ARRAY_SIZE(vid_sels));
 	hws[IMX6SX_CLK_ESAI_SEL]           = imx_clk_hw_mux("esai_sel",         base + 0x20,  19,     2,      audio_sels,        ARRAY_SIZE(audio_sels));
 	hws[IMX6SX_CLK_CAN_SEL]            = imx_clk_hw_mux("can_sel",          base + 0x20,  8,      2,      can_sels,          ARRAY_SIZE(can_sels));
 	hws[IMX6SX_CLK_UART_SEL]           = imx_clk_hw_mux("uart_sel",         base + 0x24,  6,      1,      uart_sels,         ARRAY_SIZE(uart_sels));
-	hws[IMX6SX_CLK_QSPI2_SEL]          = imx_clk_hw_mux_flags("qspi2_sel", base + 0x2c, 15, 3, qspi2_sels, ARRAY_SIZE(qspi2_sels), CLK_SET_RATE_PARENT);
+	hws[IMX6SX_CLK_QSPI2_SEL]          = imx_clk_hw_mux("qspi2_sel",        base + 0x2c,  15,     3,      qspi2_sels,        ARRAY_SIZE(qspi2_sels));
 	hws[IMX6SX_CLK_SPDIF_SEL]          = imx_clk_hw_mux("spdif_sel",        base + 0x30,  20,     2,      audio_sels,        ARRAY_SIZE(audio_sels));
 	hws[IMX6SX_CLK_AUDIO_SEL]          = imx_clk_hw_mux("audio_sel",        base + 0x30,  7,      2,      audio_sels,        ARRAY_SIZE(audio_sels));
 	hws[IMX6SX_CLK_ENET_PRE_SEL]       = imx_clk_hw_mux("enet_pre_sel",     base + 0x34,  15,     3,      enet_pre_sels,     ARRAY_SIZE(enet_pre_sels));
diff --git a/drivers/clk/imx/clk-imx93.c b/drivers/clk/imx/clk-imx93.c
index f5c9fa40491c..dcc41d178238 100644
--- a/drivers/clk/imx/clk-imx93.c
+++ b/drivers/clk/imx/clk-imx93.c
@@ -332,7 +332,7 @@ static struct platform_driver imx93_clk_driver = {
 	.driver = {
 		.name = "imx93-ccm",
 		.suppress_bind_attrs = true,
-		.of_match_table = of_match_ptr(imx93_clk_of_match),
+		.of_match_table = imx93_clk_of_match,
 	},
 };
 module_platform_driver(imx93_clk_driver);
diff --git a/drivers/clk/ingenic/tcu.c b/drivers/clk/ingenic/tcu.c
index 201bf6e6b6e0..d5544cbc5c48 100644
--- a/drivers/clk/ingenic/tcu.c
+++ b/drivers/clk/ingenic/tcu.c
@@ -101,15 +101,11 @@ static bool ingenic_tcu_enable_regs(struct clk_hw *hw)
 	bool enabled = false;
 
 	/*
-	 * If the SoC has no global TCU clock, we must ungate the channel's
-	 * clock to be able to access its registers.
-	 * If we have a TCU clock, it will be enabled automatically as it has
-	 * been attached to the regmap.
+	 * According to the programming manual, a timer channel's registers can
+	 * only be accessed when the channel's stop bit is clear.
 	 */
-	if (!tcu->clk) {
-		enabled = !!ingenic_tcu_is_enabled(hw);
-		regmap_write(tcu->map, TCU_REG_TSCR, BIT(info->gate_bit));
-	}
+	enabled = !!ingenic_tcu_is_enabled(hw);
+	regmap_write(tcu->map, TCU_REG_TSCR, BIT(info->gate_bit));
 
 	return enabled;
 }
@@ -120,8 +116,7 @@ static void ingenic_tcu_disable_regs(struct clk_hw *hw)
 	const struct ingenic_tcu_clk_info *info = tcu_clk->info;
 	struct ingenic_tcu *tcu = tcu_clk->tcu;
 
-	if (!tcu->clk)
-		regmap_write(tcu->map, TCU_REG_TSSR, BIT(info->gate_bit));
+	regmap_write(tcu->map, TCU_REG_TSSR, BIT(info->gate_bit));
 }
 
 static u8 ingenic_tcu_get_parent(struct clk_hw *hw)
diff --git a/drivers/clk/microchip/clk-mpfs.c b/drivers/clk/microchip/clk-mpfs.c
index 070c3b896559..b6b89413e090 100644
--- a/drivers/clk/microchip/clk-mpfs.c
+++ b/drivers/clk/microchip/clk-mpfs.c
@@ -239,6 +239,11 @@ static const struct clk_ops mpfs_clk_cfg_ops = {
 	.hw.init = CLK_HW_INIT(_name, _parent, &mpfs_clk_cfg_ops, 0),			\
 }
 
+#define CLK_CPU_OFFSET		0u
+#define CLK_AXI_OFFSET		1u
+#define CLK_AHB_OFFSET		2u
+#define CLK_RTCREF_OFFSET	3u
+
 static struct mpfs_cfg_hw_clock mpfs_cfg_clks[] = {
 	CLK_CFG(CLK_CPU, "clk_cpu", "clk_msspll", 0, 2, mpfs_div_cpu_axi_table, 0,
 		REG_CLOCK_CONFIG_CR),
@@ -362,7 +367,7 @@ static const struct clk_ops mpfs_periph_clk_ops = {
 				  _flags),					\
 }
 
-#define PARENT_CLK(PARENT) (&mpfs_cfg_clks[CLK_##PARENT].hw)
+#define PARENT_CLK(PARENT) (&mpfs_cfg_clks[CLK_##PARENT##_OFFSET].hw)
 
 /*
  * Critical clocks:
@@ -370,6 +375,8 @@ static const struct clk_ops mpfs_periph_clk_ops = {
  *   trap handler
  * - CLK_MMUART0: reserved by the hss
  * - CLK_DDRC: provides clock to the ddr subsystem
+ * - CLK_RTC: the onboard RTC's AHB bus clock must be kept running as the rtc will stop
+ *   if the AHB interface clock is disabled
  * - CLK_FICx: these provide the processor side clocks to the "FIC" (Fabric InterConnect)
  *   clock domain crossers which provide the interface to the FPGA fabric. Disabling them
  *   causes the FPGA fabric to go into reset.
@@ -394,7 +401,7 @@ static struct mpfs_periph_hw_clock mpfs_periph_clks[] = {
 	CLK_PERIPH(CLK_CAN0, "clk_periph_can0", PARENT_CLK(AHB), 14, 0),
 	CLK_PERIPH(CLK_CAN1, "clk_periph_can1", PARENT_CLK(AHB), 15, 0),
 	CLK_PERIPH(CLK_USB, "clk_periph_usb", PARENT_CLK(AHB), 16, 0),
-	CLK_PERIPH(CLK_RTC, "clk_periph_rtc", PARENT_CLK(AHB), 18, 0),
+	CLK_PERIPH(CLK_RTC, "clk_periph_rtc", PARENT_CLK(AHB), 18, CLK_IS_CRITICAL),
 	CLK_PERIPH(CLK_QSPI, "clk_periph_qspi", PARENT_CLK(AHB), 19, 0),
 	CLK_PERIPH(CLK_GPIO0, "clk_periph_gpio0", PARENT_CLK(AHB), 20, 0),
 	CLK_PERIPH(CLK_GPIO1, "clk_periph_gpio1", PARENT_CLK(AHB), 21, 0),
diff --git a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
index 30056da3e0af..42568c616181 100644
--- a/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
+++ b/drivers/clk/sunxi-ng/ccu-sun50i-h6.c
@@ -1191,9 +1191,13 @@ static int sun50i_h6_ccu_probe(struct platform_device *pdev)
 	if (IS_ERR(reg))
 		return PTR_ERR(reg);
 
-	/* Force PLL_GPU output divider bits to 0 */
+	/*
+	 * Force PLL_GPU output divider bits to 0 and adjust
+	 * multiplier to sensible default value of 432 MHz.
+	 */
 	val = readl(reg + SUN50I_H6_PLL_GPU_REG);
-	val &= ~BIT(0);
+	val &= ~(GENMASK(15, 8) | BIT(0));
+	val |= 17 << 8;
 	writel(val, reg + SUN50I_H6_PLL_GPU_REG);
 
 	/* Force GPU_CLK divider bits to 0 */
diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86
index 55516043b656..310779b07daf 100644
--- a/drivers/cpufreq/Kconfig.x86
+++ b/drivers/cpufreq/Kconfig.x86
@@ -51,6 +51,21 @@ config X86_AMD_PSTATE
 
 	  If in doubt, say N.
 
+config X86_AMD_PSTATE_UT
+	tristate "selftest for AMD Processor P-State driver"
+	depends on X86 && ACPI_PROCESSOR
+	default n
+	help
+	  This kernel module is used for testing. It's safe to say M here.
+
+	  It can also be built-in without X86_AMD_PSTATE enabled.
+	  Currently, only tests for amd-pstate are supported. If X86_AMD_PSTATE
+	  is set disabled, it can tell the users test can only run on amd-pstate
+	  driver, please set X86_AMD_PSTATE enabled.
+	  In the future, comparison tests will be added. It can set amd-pstate
+	  disabled and set acpi-cpufreq enabled to run test cases, then compare
+	  the test results.
+
 config X86_ACPI_CPUFREQ
 	tristate "ACPI Processor P-States driver"
 	depends on ACPI_PROCESSOR
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 285de70af877..49b98c62c5af 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -30,6 +30,7 @@ amd_pstate-y				:= amd-pstate.o amd-pstate-trace.o
 
 obj-$(CONFIG_X86_ACPI_CPUFREQ)		+= acpi-cpufreq.o
 obj-$(CONFIG_X86_AMD_PSTATE)		+= amd_pstate.o
+obj-$(CONFIG_X86_AMD_PSTATE_UT)		+= amd-pstate-ut.o
 obj-$(CONFIG_X86_POWERNOW_K8)		+= powernow-k8.o
 obj-$(CONFIG_X86_PCC_CPUFREQ)		+= pcc-cpufreq.o
 obj-$(CONFIG_X86_POWERNOW_K6)		+= powernow-k6.o
diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c
new file mode 100644
index 000000000000..e4a5b4d90f83
--- /dev/null
+++ b/drivers/cpufreq/amd-pstate-ut.c
@@ -0,0 +1,293 @@
+// SPDX-License-Identifier: GPL-1.0-or-later
+/*
+ * AMD Processor P-state Frequency Driver Unit Test
+ *
+ * Copyright (C) 2022 Advanced Micro Devices, Inc. All Rights Reserved.
+ *
+ * Author: Meng Li <li.meng@amd.com>
+ *
+ * The AMD P-State Unit Test is a test module for testing the amd-pstate
+ * driver. 1) It can help all users to verify their processor support
+ * (SBIOS/Firmware or Hardware). 2) Kernel can have a basic function
+ * test to avoid the kernel regression during the update. 3) We can
+ * introduce more functional or performance tests to align the result
+ * together, it will benefit power and performance scale optimization.
+ *
+ * This driver implements basic framework with plans to enhance it with
+ * additional test cases to improve the depth and coverage of the test.
+ *
+ * See Documentation/admin-guide/pm/amd-pstate.rst Unit Tests for
+ * amd-pstate to get more detail.
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/fs.h>
+#include <linux/amd-pstate.h>
+
+#include <acpi/cppc_acpi.h>
+
+/*
+ * Abbreviations:
+ * amd_pstate_ut: used as a shortform for AMD P-State unit test.
+ * It helps to keep variable names smaller, simpler
+ */
+enum amd_pstate_ut_result {
+	AMD_PSTATE_UT_RESULT_PASS,
+	AMD_PSTATE_UT_RESULT_FAIL,
+};
+
+struct amd_pstate_ut_struct {
+	const char *name;
+	void (*func)(u32 index);
+	enum amd_pstate_ut_result result;
+};
+
+/*
+ * Kernel module for testing the AMD P-State unit test
+ */
+static void amd_pstate_ut_acpi_cpc_valid(u32 index);
+static void amd_pstate_ut_check_enabled(u32 index);
+static void amd_pstate_ut_check_perf(u32 index);
+static void amd_pstate_ut_check_freq(u32 index);
+
+static struct amd_pstate_ut_struct amd_pstate_ut_cases[] = {
+	{"amd_pstate_ut_acpi_cpc_valid",   amd_pstate_ut_acpi_cpc_valid   },
+	{"amd_pstate_ut_check_enabled",    amd_pstate_ut_check_enabled    },
+	{"amd_pstate_ut_check_perf",       amd_pstate_ut_check_perf       },
+	{"amd_pstate_ut_check_freq",       amd_pstate_ut_check_freq       }
+};
+
+static bool get_shared_mem(void)
+{
+	bool result = false;
+	char path[] = "/sys/module/amd_pstate/parameters/shared_mem";
+	char buf[5] = {0};
+	struct file *filp = NULL;
+	loff_t pos = 0;
+	ssize_t ret;
+
+	if (!boot_cpu_has(X86_FEATURE_CPPC)) {
+		filp = filp_open(path, O_RDONLY, 0);
+		if (IS_ERR(filp))
+			pr_err("%s unable to open %s file!\n", __func__, path);
+		else {
+			ret = kernel_read(filp, &buf, sizeof(buf), &pos);
+			if (ret < 0)
+				pr_err("%s read %s file fail ret=%ld!\n",
+					__func__, path, (long)ret);
+			filp_close(filp, NULL);
+		}
+
+		if ('Y' == *buf)
+			result = true;
+	}
+
+	return result;
+}
+
+/*
+ * check the _CPC object is present in SBIOS.
+ */
+static void amd_pstate_ut_acpi_cpc_valid(u32 index)
+{
+	if (acpi_cpc_valid())
+		amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+	else {
+		amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+		pr_err("%s the _CPC object is not present in SBIOS!\n", __func__);
+	}
+}
+
+static void amd_pstate_ut_pstate_enable(u32 index)
+{
+	int ret = 0;
+	u64 cppc_enable = 0;
+
+	ret = rdmsrl_safe(MSR_AMD_CPPC_ENABLE, &cppc_enable);
+	if (ret) {
+		amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+		pr_err("%s rdmsrl_safe MSR_AMD_CPPC_ENABLE ret=%d error!\n", __func__, ret);
+		return;
+	}
+	if (cppc_enable)
+		amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+	else {
+		amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+		pr_err("%s amd pstate must be enabled!\n", __func__);
+	}
+}
+
+/*
+ * check if amd pstate is enabled
+ */
+static void amd_pstate_ut_check_enabled(u32 index)
+{
+	if (get_shared_mem())
+		amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+	else
+		amd_pstate_ut_pstate_enable(index);
+}
+
+/*
+ * check if performance values are reasonable.
+ * highest_perf >= nominal_perf > lowest_nonlinear_perf > lowest_perf > 0
+ */
+static void amd_pstate_ut_check_perf(u32 index)
+{
+	int cpu = 0, ret = 0;
+	u32 highest_perf = 0, nominal_perf = 0, lowest_nonlinear_perf = 0, lowest_perf = 0;
+	u64 cap1 = 0;
+	struct cppc_perf_caps cppc_perf;
+	struct cpufreq_policy *policy = NULL;
+	struct amd_cpudata *cpudata = NULL;
+
+	highest_perf = amd_get_highest_perf();
+
+	for_each_possible_cpu(cpu) {
+		policy = cpufreq_cpu_get(cpu);
+		if (!policy)
+			break;
+		cpudata = policy->driver_data;
+
+		if (get_shared_mem()) {
+			ret = cppc_get_perf_caps(cpu, &cppc_perf);
+			if (ret) {
+				amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+				pr_err("%s cppc_get_perf_caps ret=%d error!\n", __func__, ret);
+				return;
+			}
+
+			nominal_perf = cppc_perf.nominal_perf;
+			lowest_nonlinear_perf = cppc_perf.lowest_nonlinear_perf;
+			lowest_perf = cppc_perf.lowest_perf;
+		} else {
+			ret = rdmsrl_safe_on_cpu(cpu, MSR_AMD_CPPC_CAP1, &cap1);
+			if (ret) {
+				amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+				pr_err("%s read CPPC_CAP1 ret=%d error!\n", __func__, ret);
+				return;
+			}
+
+			nominal_perf = AMD_CPPC_NOMINAL_PERF(cap1);
+			lowest_nonlinear_perf = AMD_CPPC_LOWNONLIN_PERF(cap1);
+			lowest_perf = AMD_CPPC_LOWEST_PERF(cap1);
+		}
+
+		if ((highest_perf != READ_ONCE(cpudata->highest_perf)) ||
+			(nominal_perf != READ_ONCE(cpudata->nominal_perf)) ||
+			(lowest_nonlinear_perf != READ_ONCE(cpudata->lowest_nonlinear_perf)) ||
+			(lowest_perf != READ_ONCE(cpudata->lowest_perf))) {
+			amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+			pr_err("%s cpu%d highest=%d %d nominal=%d %d lowest_nonlinear=%d %d lowest=%d %d, they should be equal!\n",
+				__func__, cpu, highest_perf, cpudata->highest_perf,
+				nominal_perf, cpudata->nominal_perf,
+				lowest_nonlinear_perf, cpudata->lowest_nonlinear_perf,
+				lowest_perf, cpudata->lowest_perf);
+			return;
+		}
+
+		if (!((highest_perf >= nominal_perf) &&
+			(nominal_perf > lowest_nonlinear_perf) &&
+			(lowest_nonlinear_perf > lowest_perf) &&
+			(lowest_perf > 0))) {
+			amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+			pr_err("%s cpu%d highest=%d >= nominal=%d > lowest_nonlinear=%d > lowest=%d > 0, the formula is incorrect!\n",
+				__func__, cpu, highest_perf, nominal_perf,
+				lowest_nonlinear_perf, lowest_perf);
+			return;
+		}
+	}
+
+	amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+}
+
+/*
+ * Check if frequency values are reasonable.
+ * max_freq >= nominal_freq > lowest_nonlinear_freq > min_freq > 0
+ * check max freq when set support boost mode.
+ */
+static void amd_pstate_ut_check_freq(u32 index)
+{
+	int cpu = 0;
+	struct cpufreq_policy *policy = NULL;
+	struct amd_cpudata *cpudata = NULL;
+
+	for_each_possible_cpu(cpu) {
+		policy = cpufreq_cpu_get(cpu);
+		if (!policy)
+			break;
+		cpudata = policy->driver_data;
+
+		if (!((cpudata->max_freq >= cpudata->nominal_freq) &&
+			(cpudata->nominal_freq > cpudata->lowest_nonlinear_freq) &&
+			(cpudata->lowest_nonlinear_freq > cpudata->min_freq) &&
+			(cpudata->min_freq > 0))) {
+			amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+			pr_err("%s cpu%d max=%d >= nominal=%d > lowest_nonlinear=%d > min=%d > 0, the formula is incorrect!\n",
+				__func__, cpu, cpudata->max_freq, cpudata->nominal_freq,
+				cpudata->lowest_nonlinear_freq, cpudata->min_freq);
+			return;
+		}
+
+		if (cpudata->min_freq != policy->min) {
+			amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+			pr_err("%s cpu%d cpudata_min_freq=%d policy_min=%d, they should be equal!\n",
+				__func__, cpu, cpudata->min_freq, policy->min);
+			return;
+		}
+
+		if (cpudata->boost_supported) {
+			if ((policy->max == cpudata->max_freq) ||
+					(policy->max == cpudata->nominal_freq))
+				amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+			else {
+				amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+				pr_err("%s cpu%d policy_max=%d should be equal cpu_max=%d or cpu_nominal=%d !\n",
+					__func__, cpu, policy->max, cpudata->max_freq,
+					cpudata->nominal_freq);
+				return;
+			}
+		} else {
+			amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_FAIL;
+			pr_err("%s cpu%d must support boost!\n", __func__, cpu);
+			return;
+		}
+	}
+
+	amd_pstate_ut_cases[index].result = AMD_PSTATE_UT_RESULT_PASS;
+}
+
+static int __init amd_pstate_ut_init(void)
+{
+	u32 i = 0, arr_size = ARRAY_SIZE(amd_pstate_ut_cases);
+
+	for (i = 0; i < arr_size; i++) {
+		amd_pstate_ut_cases[i].func(i);
+		switch (amd_pstate_ut_cases[i].result) {
+		case AMD_PSTATE_UT_RESULT_PASS:
+			pr_info("%-4d %-20s\t success!\n", i+1, amd_pstate_ut_cases[i].name);
+			break;
+		case AMD_PSTATE_UT_RESULT_FAIL:
+		default:
+			pr_info("%-4d %-20s\t fail!\n", i+1, amd_pstate_ut_cases[i].name);
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static void __exit amd_pstate_ut_exit(void)
+{
+}
+
+module_init(amd_pstate_ut_init);
+module_exit(amd_pstate_ut_exit);
+
+MODULE_AUTHOR("Meng Li <li.meng@amd.com>");
+MODULE_DESCRIPTION("AMD P-state driver Test module");
+MODULE_LICENSE("GPL");
diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c
index 9ac75c1cde9c..ace7d50cf2ac 100644
--- a/drivers/cpufreq/amd-pstate.c
+++ b/drivers/cpufreq/amd-pstate.c
@@ -36,6 +36,7 @@
 #include <linux/delay.h>
 #include <linux/uaccess.h>
 #include <linux/static_call.h>
+#include <linux/amd-pstate.h>
 
 #include <acpi/processor.h>
 #include <acpi/cppc_acpi.h>
@@ -46,8 +47,8 @@
 #include <asm/cpu_device_id.h>
 #include "amd-pstate-trace.h"
 
-#define AMD_PSTATE_TRANSITION_LATENCY	0x20000
-#define AMD_PSTATE_TRANSITION_DELAY	500
+#define AMD_PSTATE_TRANSITION_LATENCY	20000
+#define AMD_PSTATE_TRANSITION_DELAY	1000
 
 /*
  * TODO: We need more time to fine tune processors with shared memory solution
@@ -65,65 +66,6 @@ MODULE_PARM_DESC(shared_mem,
 
 static struct cpufreq_driver amd_pstate_driver;
 
-/**
- * struct  amd_aperf_mperf
- * @aperf: actual performance frequency clock count
- * @mperf: maximum performance frequency clock count
- * @tsc:   time stamp counter
- */
-struct amd_aperf_mperf {
-	u64 aperf;
-	u64 mperf;
-	u64 tsc;
-};
-
-/**
- * struct amd_cpudata - private CPU data for AMD P-State
- * @cpu: CPU number
- * @req: constraint request to apply
- * @cppc_req_cached: cached performance request hints
- * @highest_perf: the maximum performance an individual processor may reach,
- *		  assuming ideal conditions
- * @nominal_perf: the maximum sustained performance level of the processor,
- *		  assuming ideal operating conditions
- * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power
- *			   savings are achieved
- * @lowest_perf: the absolute lowest performance level of the processor
- * @max_freq: the frequency that mapped to highest_perf
- * @min_freq: the frequency that mapped to lowest_perf
- * @nominal_freq: the frequency that mapped to nominal_perf
- * @lowest_nonlinear_freq: the frequency that mapped to lowest_nonlinear_perf
- * @cur: Difference of Aperf/Mperf/tsc count between last and current sample
- * @prev: Last Aperf/Mperf/tsc count value read from register
- * @freq: current cpu frequency value
- * @boost_supported: check whether the Processor or SBIOS supports boost mode
- *
- * The amd_cpudata is key private data for each CPU thread in AMD P-State, and
- * represents all the attributes and goals that AMD P-State requests at runtime.
- */
-struct amd_cpudata {
-	int	cpu;
-
-	struct	freq_qos_request req[2];
-	u64	cppc_req_cached;
-
-	u32	highest_perf;
-	u32	nominal_perf;
-	u32	lowest_nonlinear_perf;
-	u32	lowest_perf;
-
-	u32	max_freq;
-	u32	min_freq;
-	u32	nominal_freq;
-	u32	lowest_nonlinear_freq;
-
-	struct amd_aperf_mperf cur;
-	struct amd_aperf_mperf prev;
-
-	u64 freq;
-	bool	boost_supported;
-};
-
 static inline int pstate_enable(bool enable)
 {
 	return wrmsrl_safe(MSR_AMD_CPPC_ENABLE, enable);
@@ -152,6 +94,7 @@ static inline int amd_pstate_enable(bool enable)
 static int pstate_init_perf(struct amd_cpudata *cpudata)
 {
 	u64 cap1;
+	u32 highest_perf;
 
 	int ret = rdmsrl_safe_on_cpu(cpudata->cpu, MSR_AMD_CPPC_CAP1,
 				     &cap1);
@@ -163,7 +106,11 @@ static int pstate_init_perf(struct amd_cpudata *cpudata)
 	 *
 	 * CPPC entry doesn't indicate the highest performance in some ASICs.
 	 */
-	WRITE_ONCE(cpudata->highest_perf, amd_get_highest_perf());
+	highest_perf = amd_get_highest_perf();
+	if (highest_perf > AMD_CPPC_HIGHEST_PERF(cap1))
+		highest_perf = AMD_CPPC_HIGHEST_PERF(cap1);
+
+	WRITE_ONCE(cpudata->highest_perf, highest_perf);
 
 	WRITE_ONCE(cpudata->nominal_perf, AMD_CPPC_NOMINAL_PERF(cap1));
 	WRITE_ONCE(cpudata->lowest_nonlinear_perf, AMD_CPPC_LOWNONLIN_PERF(cap1));
@@ -175,12 +122,17 @@ static int pstate_init_perf(struct amd_cpudata *cpudata)
 static int cppc_init_perf(struct amd_cpudata *cpudata)
 {
 	struct cppc_perf_caps cppc_perf;
+	u32 highest_perf;
 
 	int ret = cppc_get_perf_caps(cpudata->cpu, &cppc_perf);
 	if (ret)
 		return ret;
 
-	WRITE_ONCE(cpudata->highest_perf, amd_get_highest_perf());
+	highest_perf = amd_get_highest_perf();
+	if (highest_perf > cppc_perf.highest_perf)
+		highest_perf = cppc_perf.highest_perf;
+
+	WRITE_ONCE(cpudata->highest_perf, highest_perf);
 
 	WRITE_ONCE(cpudata->nominal_perf, cppc_perf.nominal_perf);
 	WRITE_ONCE(cpudata->lowest_nonlinear_perf,
@@ -269,6 +221,7 @@ static void amd_pstate_update(struct amd_cpudata *cpudata, u32 min_perf,
 	u64 prev = READ_ONCE(cpudata->cppc_req_cached);
 	u64 value = prev;
 
+	des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
 	value &= ~AMD_CPPC_MIN_PERF(~0L);
 	value |= AMD_CPPC_MIN_PERF(min_perf);
 
@@ -312,7 +265,7 @@ static int amd_pstate_target(struct cpufreq_policy *policy,
 		return -ENODEV;
 
 	cap_perf = READ_ONCE(cpudata->highest_perf);
-	min_perf = READ_ONCE(cpudata->lowest_nonlinear_perf);
+	min_perf = READ_ONCE(cpudata->lowest_perf);
 	max_perf = cap_perf;
 
 	freqs.old = policy->cur;
@@ -357,8 +310,6 @@ static void amd_pstate_adjust_perf(unsigned int cpu,
 	if (max_perf < min_perf)
 		max_perf = min_perf;
 
-	des_perf = clamp_t(unsigned long, des_perf, min_perf, max_perf);
-
 	amd_pstate_update(cpudata, min_perf, des_perf, max_perf, true);
 }
 
@@ -555,9 +506,7 @@ free_cpudata1:
 
 static int amd_pstate_cpu_exit(struct cpufreq_policy *policy)
 {
-	struct amd_cpudata *cpudata;
-
-	cpudata = policy->driver_data;
+	struct amd_cpudata *cpudata = policy->driver_data;
 
 	freq_qos_remove_request(&cpudata->req[1]);
 	freq_qos_remove_request(&cpudata->req[0]);
@@ -599,9 +548,7 @@ static ssize_t show_amd_pstate_max_freq(struct cpufreq_policy *policy,
 					char *buf)
 {
 	int max_freq;
-	struct amd_cpudata *cpudata;
-
-	cpudata = policy->driver_data;
+	struct amd_cpudata *cpudata = policy->driver_data;
 
 	max_freq = amd_get_max_freq(cpudata);
 	if (max_freq < 0)
@@ -614,9 +561,7 @@ static ssize_t show_amd_pstate_lowest_nonlinear_freq(struct cpufreq_policy *poli
 						     char *buf)
 {
 	int freq;
-	struct amd_cpudata *cpudata;
-
-	cpudata = policy->driver_data;
+	struct amd_cpudata *cpudata = policy->driver_data;
 
 	freq = amd_get_lowest_nonlinear_freq(cpudata);
 	if (freq < 0)
@@ -662,7 +607,7 @@ static struct cpufreq_driver amd_pstate_driver = {
 	.resume		= amd_pstate_cpu_resume,
 	.set_boost	= amd_pstate_set_boost,
 	.name		= "amd-pstate",
-	.attr           = amd_pstate_attr,
+	.attr		= amd_pstate_attr,
 };
 
 static int __init amd_pstate_init(void)
@@ -673,7 +618,7 @@ static int __init amd_pstate_init(void)
 		return -ENODEV;
 
 	if (!acpi_cpc_valid()) {
-		pr_debug("the _CPC object is not present in SBIOS\n");
+		pr_warn_once("the _CPC object is not present in SBIOS or ACPI disabled\n");
 		return -ENODEV;
 	}
 
diff --git a/drivers/cpufreq/bmips-cpufreq.c b/drivers/cpufreq/bmips-cpufreq.c
index f7c23fa468f0..39221a9a187a 100644
--- a/drivers/cpufreq/bmips-cpufreq.c
+++ b/drivers/cpufreq/bmips-cpufreq.c
@@ -156,7 +156,7 @@ static struct cpufreq_driver bmips_cpufreq_driver = {
 	.name		= BMIPS_CPUFREQ_PREFIX,
 };
 
-static int __init bmips_cpufreq_probe(void)
+static int __init bmips_cpufreq_driver_init(void)
 {
 	struct cpufreq_compat *cc;
 	struct device_node *np;
@@ -176,7 +176,13 @@ static int __init bmips_cpufreq_probe(void)
 
 	return cpufreq_register_driver(&bmips_cpufreq_driver);
 }
-device_initcall(bmips_cpufreq_probe);
+module_init(bmips_cpufreq_driver_init);
+
+static void __exit bmips_cpufreq_driver_exit(void)
+{
+	cpufreq_unregister_driver(&bmips_cpufreq_driver);
+}
+module_exit(bmips_cpufreq_driver_exit);
 
 MODULE_AUTHOR("Markus Mayer <mmayer@broadcom.com>");
 MODULE_DESCRIPTION("CPUfreq driver for Broadcom BMIPS SoCs");
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index 24eaf0ec344d..432dfb4e8027 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -63,7 +63,15 @@ static struct cppc_workaround_oem_info wa_info[] = {
 
 static struct cpufreq_driver cppc_cpufreq_driver;
 
+static enum {
+	FIE_UNSET = -1,
+	FIE_ENABLED,
+	FIE_DISABLED
+} fie_disabled = FIE_UNSET;
+
 #ifdef CONFIG_ACPI_CPPC_CPUFREQ_FIE
+module_param(fie_disabled, int, 0444);
+MODULE_PARM_DESC(fie_disabled, "Disable Frequency Invariance Engine (FIE)");
 
 /* Frequency invariance support */
 struct cppc_freq_invariance {
@@ -158,7 +166,7 @@ static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy)
 	struct cppc_freq_invariance *cppc_fi;
 	int cpu, ret;
 
-	if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
+	if (fie_disabled)
 		return;
 
 	for_each_cpu(cpu, policy->cpus) {
@@ -199,7 +207,7 @@ static void cppc_cpufreq_cpu_fie_exit(struct cpufreq_policy *policy)
 	struct cppc_freq_invariance *cppc_fi;
 	int cpu;
 
-	if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
+	if (fie_disabled)
 		return;
 
 	/* policy->cpus will be empty here, use related_cpus instead */
@@ -229,7 +237,15 @@ static void __init cppc_freq_invariance_init(void)
 	};
 	int ret;
 
-	if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
+	if (fie_disabled != FIE_ENABLED && fie_disabled != FIE_DISABLED) {
+		fie_disabled = FIE_ENABLED;
+		if (cppc_perf_ctrs_in_pcc()) {
+			pr_info("FIE not enabled on systems with registers in PCC\n");
+			fie_disabled = FIE_DISABLED;
+		}
+	}
+
+	if (fie_disabled)
 		return;
 
 	kworker_fie = kthread_create_worker(0, "cppc_fie");
@@ -247,7 +263,7 @@ static void __init cppc_freq_invariance_init(void)
 
 static void cppc_freq_invariance_exit(void)
 {
-	if (cppc_cpufreq_driver.get == hisi_cppc_cpufreq_get_rate)
+	if (fie_disabled)
 		return;
 
 	kthread_destroy_worker(kworker_fie);
@@ -936,6 +952,7 @@ static void cppc_check_hisi_workaround(void)
 		    wa_info[i].oem_revision == tbl->oem_revision) {
 			/* Overwrite the get() callback */
 			cppc_cpufreq_driver.get = hisi_cppc_cpufreq_get_rate;
+			fie_disabled = FIE_DISABLED;
 			break;
 		}
 	}
@@ -947,7 +964,7 @@ static int __init cppc_cpufreq_init(void)
 {
 	int ret;
 
-	if ((acpi_disabled) || !acpi_cpc_valid())
+	if (!acpi_cpc_valid())
 		return -ENODEV;
 
 	cppc_check_hisi_workaround();
diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c
index 2c96de3f2d83..6ac3800db450 100644
--- a/drivers/cpufreq/cpufreq-dt-platdev.c
+++ b/drivers/cpufreq/cpufreq-dt-platdev.c
@@ -146,6 +146,7 @@ static const struct of_device_id blocklist[] __initconst = {
 	{ .compatible = "qcom,sc8180x", },
 	{ .compatible = "qcom,sc8280xp", },
 	{ .compatible = "qcom,sdm845", },
+	{ .compatible = "qcom,sm6115", },
 	{ .compatible = "qcom,sm6350", },
 	{ .compatible = "qcom,sm8150", },
 	{ .compatible = "qcom,sm8250", },
diff --git a/drivers/cpufreq/highbank-cpufreq.c b/drivers/cpufreq/highbank-cpufreq.c
index ac57cddc5f2f..a45864701143 100644
--- a/drivers/cpufreq/highbank-cpufreq.c
+++ b/drivers/cpufreq/highbank-cpufreq.c
@@ -55,7 +55,7 @@ static struct notifier_block hb_cpufreq_clk_nb = {
 	.notifier_call = hb_cpufreq_clk_notify,
 };
 
-static int hb_cpufreq_driver_init(void)
+static int __init hb_cpufreq_driver_init(void)
 {
 	struct platform_device_info devinfo = { .name = "cpufreq-dt", };
 	struct device *cpu_dev;
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 57cdb3679885..fc3ebeb0bbe5 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -2416,6 +2416,7 @@ static const struct x86_cpu_id intel_pstate_cpu_ids[] = {
 	X86_MATCH(SKYLAKE_X,		core_funcs),
 	X86_MATCH(COMETLAKE,		core_funcs),
 	X86_MATCH(ICELAKE_X,		core_funcs),
+	X86_MATCH(TIGERLAKE,		core_funcs),
 	{}
 };
 MODULE_DEVICE_TABLE(x86cpu, intel_pstate_cpu_ids);
diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c
index d5ef3c66c762..833589bc95e4 100644
--- a/drivers/cpufreq/qcom-cpufreq-hw.c
+++ b/drivers/cpufreq/qcom-cpufreq-hw.c
@@ -13,6 +13,7 @@
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
 #include <linux/pm_opp.h>
+#include <linux/pm_qos.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/units.h>
@@ -56,6 +57,8 @@ struct qcom_cpufreq_data {
 	struct cpufreq_policy *policy;
 
 	bool per_core_dcvs;
+
+	struct freq_qos_request throttle_freq_req;
 };
 
 static unsigned long cpu_hw_rate, xo_rate;
@@ -316,14 +319,16 @@ static void qcom_lmh_dcvs_notify(struct qcom_cpufreq_data *data)
 	if (IS_ERR(opp)) {
 		dev_warn(dev, "Can't find the OPP for throttling: %pe!\n", opp);
 	} else {
-		throttled_freq = freq_hz / HZ_PER_KHZ;
-
-		/* Update thermal pressure (the boost frequencies are accepted) */
-		arch_update_thermal_pressure(policy->related_cpus, throttled_freq);
-
 		dev_pm_opp_put(opp);
 	}
 
+	throttled_freq = freq_hz / HZ_PER_KHZ;
+
+	freq_qos_update_request(&data->throttle_freq_req, throttled_freq);
+
+	/* Update thermal pressure (the boost frequencies are accepted) */
+	arch_update_thermal_pressure(policy->related_cpus, throttled_freq);
+
 	/*
 	 * In the unlikely case policy is unregistered do not enable
 	 * polling or h/w interrupt
@@ -413,6 +418,14 @@ static int qcom_cpufreq_hw_lmh_init(struct cpufreq_policy *policy, int index)
 	if (data->throttle_irq < 0)
 		return data->throttle_irq;
 
+	ret = freq_qos_add_request(&policy->constraints,
+				   &data->throttle_freq_req, FREQ_QOS_MAX,
+				   FREQ_QOS_MAX_DEFAULT_VALUE);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Failed to add freq constraint (%d)\n", ret);
+		return ret;
+	}
+
 	data->cancel_throttle = false;
 	data->policy = policy;
 
@@ -479,6 +492,7 @@ static void qcom_cpufreq_hw_lmh_exit(struct qcom_cpufreq_data *data)
 	if (data->throttle_irq <= 0)
 		return;
 
+	freq_qos_remove_request(&data->throttle_freq_req);
 	free_irq(data->throttle_irq, data);
 }
 
diff --git a/drivers/cpufreq/sti-cpufreq.c b/drivers/cpufreq/sti-cpufreq.c
index a67df90848c2..1a63aeea8711 100644
--- a/drivers/cpufreq/sti-cpufreq.c
+++ b/drivers/cpufreq/sti-cpufreq.c
@@ -252,7 +252,7 @@ static int sti_cpufreq_fetch_syscon_registers(void)
 	return 0;
 }
 
-static int sti_cpufreq_init(void)
+static int __init sti_cpufreq_init(void)
 {
 	int ret;
 
diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c
index 1216046cf4c2..c2004cae3f02 100644
--- a/drivers/cpufreq/tegra194-cpufreq.c
+++ b/drivers/cpufreq/tegra194-cpufreq.c
@@ -38,14 +38,6 @@
 /* cpufreq transisition latency */
 #define TEGRA_CPUFREQ_TRANSITION_LATENCY (300 * 1000) /* unit in nanoseconds */
 
-enum cluster {
-	CLUSTER0,
-	CLUSTER1,
-	CLUSTER2,
-	CLUSTER3,
-	MAX_CLUSTERS,
-};
-
 struct tegra_cpu_ctr {
 	u32 cpu;
 	u32 coreclk_cnt, last_coreclk_cnt;
@@ -67,12 +59,12 @@ struct tegra_cpufreq_ops {
 struct tegra_cpufreq_soc {
 	struct tegra_cpufreq_ops *ops;
 	int maxcpus_per_cluster;
+	unsigned int num_clusters;
 	phys_addr_t actmon_cntr_base;
 };
 
 struct tegra194_cpufreq_data {
 	void __iomem *regs;
-	size_t num_clusters;
 	struct cpufreq_frequency_table **tables;
 	const struct tegra_cpufreq_soc *soc;
 };
@@ -166,6 +158,14 @@ static const struct tegra_cpufreq_soc tegra234_cpufreq_soc = {
 	.ops = &tegra234_cpufreq_ops,
 	.actmon_cntr_base = 0x9000,
 	.maxcpus_per_cluster = 4,
+	.num_clusters = 3,
+};
+
+static const struct tegra_cpufreq_soc tegra239_cpufreq_soc = {
+	.ops = &tegra234_cpufreq_ops,
+	.actmon_cntr_base = 0x4000,
+	.maxcpus_per_cluster = 8,
+	.num_clusters = 1,
 };
 
 static void tegra194_get_cpu_cluster_id(u32 cpu, u32 *cpuid, u32 *clusterid)
@@ -314,11 +314,7 @@ static void tegra194_get_cpu_ndiv_sysreg(void *ndiv)
 
 static int tegra194_get_cpu_ndiv(u32 cpu, u32 cpuid, u32 clusterid, u64 *ndiv)
 {
-	int ret;
-
-	ret = smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, &ndiv, true);
-
-	return ret;
+	return smp_call_function_single(cpu, tegra194_get_cpu_ndiv_sysreg, &ndiv, true);
 }
 
 static void tegra194_set_cpu_ndiv_sysreg(void *data)
@@ -382,7 +378,7 @@ static int tegra194_cpufreq_init(struct cpufreq_policy *policy)
 
 	data->soc->ops->get_cpu_cluster_id(policy->cpu, NULL, &clusterid);
 
-	if (clusterid >= data->num_clusters || !data->tables[clusterid])
+	if (clusterid >= data->soc->num_clusters || !data->tables[clusterid])
 		return -EINVAL;
 
 	start_cpu = rounddown(policy->cpu, maxcpus_per_cluster);
@@ -433,6 +429,7 @@ static struct tegra_cpufreq_ops tegra194_cpufreq_ops = {
 static const struct tegra_cpufreq_soc tegra194_cpufreq_soc = {
 	.ops = &tegra194_cpufreq_ops,
 	.maxcpus_per_cluster = 2,
+	.num_clusters = 4,
 };
 
 static void tegra194_cpufreq_free_resources(void)
@@ -525,15 +522,14 @@ static int tegra194_cpufreq_probe(struct platform_device *pdev)
 
 	soc = of_device_get_match_data(&pdev->dev);
 
-	if (soc->ops && soc->maxcpus_per_cluster) {
+	if (soc->ops && soc->maxcpus_per_cluster && soc->num_clusters) {
 		data->soc = soc;
 	} else {
 		dev_err(&pdev->dev, "soc data missing\n");
 		return -EINVAL;
 	}
 
-	data->num_clusters = MAX_CLUSTERS;
-	data->tables = devm_kcalloc(&pdev->dev, data->num_clusters,
+	data->tables = devm_kcalloc(&pdev->dev, data->soc->num_clusters,
 				    sizeof(*data->tables), GFP_KERNEL);
 	if (!data->tables)
 		return -ENOMEM;
@@ -558,7 +554,7 @@ static int tegra194_cpufreq_probe(struct platform_device *pdev)
 		goto put_bpmp;
 	}
 
-	for (i = 0; i < data->num_clusters; i++) {
+	for (i = 0; i < data->soc->num_clusters; i++) {
 		data->tables[i] = init_freq_table(pdev, bpmp, i);
 		if (IS_ERR(data->tables[i])) {
 			err = PTR_ERR(data->tables[i]);
@@ -590,6 +586,7 @@ static int tegra194_cpufreq_remove(struct platform_device *pdev)
 static const struct of_device_id tegra194_cpufreq_of_match[] = {
 	{ .compatible = "nvidia,tegra194-ccplex", .data = &tegra194_cpufreq_soc },
 	{ .compatible = "nvidia,tegra234-ccplex-cluster", .data = &tegra234_cpufreq_soc },
+	{ .compatible = "nvidia,tegra239-ccplex-cluster", .data = &tegra239_cpufreq_soc },
 	{ /* sentinel */ }
 };
 
diff --git a/drivers/cpufreq/ti-cpufreq.c b/drivers/cpufreq/ti-cpufreq.c
index df85a77d476b..f64180dd2005 100644
--- a/drivers/cpufreq/ti-cpufreq.c
+++ b/drivers/cpufreq/ti-cpufreq.c
@@ -398,7 +398,7 @@ fail_put_node:
 	return ret;
 }
 
-static int ti_cpufreq_init(void)
+static int __init ti_cpufreq_init(void)
 {
 	const struct of_device_id *match;
 
diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c
index 74068742cef3..9acde71558d5 100644
--- a/drivers/cpuidle/coupled.c
+++ b/drivers/cpuidle/coupled.c
@@ -54,7 +54,7 @@
  * variable is not locked.  It is only written from the cpu that
  * it stores (or by the on/offlining cpu if that cpu is offline),
  * and only read after all the cpus are ready for the coupled idle
- * state are are no longer updating it.
+ * state are no longer updating it.
  *
  * Three atomic counters are used.  alive_count tracks the number
  * of cpus in the coupled set that are currently or soon will be
diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c
index c32c600b3cf8..0b5461b3d7dd 100644
--- a/drivers/cpuidle/cpuidle-powernv.c
+++ b/drivers/cpuidle/cpuidle-powernv.c
@@ -233,8 +233,8 @@ static inline void add_powernv_state(int index, const char *name,
 				     unsigned int exit_latency,
 				     u64 psscr_val, u64 psscr_mask)
 {
-	strlcpy(powernv_states[index].name, name, CPUIDLE_NAME_LEN);
-	strlcpy(powernv_states[index].desc, name, CPUIDLE_NAME_LEN);
+	strscpy(powernv_states[index].name, name, CPUIDLE_NAME_LEN);
+	strscpy(powernv_states[index].desc, name, CPUIDLE_NAME_LEN);
 	powernv_states[index].flags = flags;
 	powernv_states[index].target_residency = target_residency;
 	powernv_states[index].exit_latency = exit_latency;
diff --git a/drivers/cpuidle/cpuidle-psci-domain.c b/drivers/cpuidle/cpuidle-psci-domain.c
index 3db4fca1172b..821984947ed9 100644
--- a/drivers/cpuidle/cpuidle-psci-domain.c
+++ b/drivers/cpuidle/cpuidle-psci-domain.c
@@ -124,10 +124,8 @@ static bool psci_pd_try_set_osi_mode(void)
 		return false;
 
 	ret = psci_set_osi_mode(true);
-	if (ret) {
-		pr_warn("failed to enable OSI mode: %d\n", ret);
+	if (ret)
 		return false;
-	}
 
 	return true;
 }
diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c
index 29acaf48e575..0d0f9751ff8f 100644
--- a/drivers/cpuidle/governor.c
+++ b/drivers/cpuidle/governor.c
@@ -63,12 +63,11 @@ int cpuidle_switch_governor(struct cpuidle_governor *gov)
 
 	cpuidle_curr_governor = gov;
 
-	if (gov) {
-		list_for_each_entry(dev, &cpuidle_detected_devices, device_list)
-			cpuidle_enable_device(dev);
-		cpuidle_install_idle_handler();
-		printk(KERN_INFO "cpuidle: using governor %s\n", gov->name);
-	}
+	list_for_each_entry(dev, &cpuidle_detected_devices, device_list)
+		cpuidle_enable_device(dev);
+
+	cpuidle_install_idle_handler();
+	pr_info("cpuidle: using governor %s\n", gov->name);
 
 	return 0;
 }
diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c
index a4b13d326cfc..82bf15d49561 100644
--- a/drivers/crypto/atmel-ecc.c
+++ b/drivers/crypto/atmel-ecc.c
@@ -343,7 +343,7 @@ static int atmel_ecc_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int atmel_ecc_remove(struct i2c_client *client)
+static void atmel_ecc_remove(struct i2c_client *client)
 {
 	struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
 
@@ -358,7 +358,7 @@ static int atmel_ecc_remove(struct i2c_client *client)
 		 * accessing the freed memory.
 		 */
 		dev_emerg(&client->dev, "Device is busy, expect memory corruption.\n");
-		return 0;
+		return;
 	}
 
 	crypto_unregister_kpp(&atmel_ecdh_nist_p256);
@@ -366,8 +366,6 @@ static int atmel_ecc_remove(struct i2c_client *client)
 	spin_lock(&driver_data.i2c_list_lock);
 	list_del(&i2c_priv->i2c_client_list_node);
 	spin_unlock(&driver_data.i2c_list_lock);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c
index e4087bdd2475..a84b657598c6 100644
--- a/drivers/crypto/atmel-sha204a.c
+++ b/drivers/crypto/atmel-sha204a.c
@@ -116,18 +116,16 @@ static int atmel_sha204a_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int atmel_sha204a_remove(struct i2c_client *client)
+static void atmel_sha204a_remove(struct i2c_client *client)
 {
 	struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client);
 
 	if (atomic_read(&i2c_priv->tfm_count)) {
 		dev_emerg(&client->dev, "Device is busy, will remove it anyhow\n");
-		return 0;
+		return;
 	}
 
 	kfree((void *)i2c_priv->hwrng.priv);
-
-	return 0;
 }
 
 static const struct of_device_id atmel_sha204a_dt_ids[] = {
diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c
index 2a60d0525cde..168195672e2e 100644
--- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c
+++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c
@@ -56,6 +56,10 @@ static void virtio_crypto_akcipher_finalize_req(
 	struct virtio_crypto_akcipher_request *vc_akcipher_req,
 	struct akcipher_request *req, int err)
 {
+	kfree(vc_akcipher_req->src_buf);
+	kfree(vc_akcipher_req->dst_buf);
+	vc_akcipher_req->src_buf = NULL;
+	vc_akcipher_req->dst_buf = NULL;
 	virtcrypto_clear_request(&vc_akcipher_req->base);
 
 	crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err);
diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c
index 9a88faaf8b27..39ac069cabc7 100644
--- a/drivers/devfreq/event/rockchip-dfi.c
+++ b/drivers/devfreq/event/rockchip-dfi.c
@@ -189,10 +189,9 @@ static int rockchip_dfi_probe(struct platform_device *pdev)
 		return PTR_ERR(data->regs);
 
 	data->clk = devm_clk_get(dev, "pclk_ddr_mon");
-	if (IS_ERR(data->clk)) {
-		dev_err(dev, "Cannot get the clk dmc_clk\n");
-		return PTR_ERR(data->clk);
-	}
+	if (IS_ERR(data->clk))
+		return dev_err_probe(dev, PTR_ERR(data->clk),
+				     "Cannot get the clk pclk_ddr_mon\n");
 
 	/* try to find the optional reference to the pmu syscon */
 	node = of_parse_phandle(np, "rockchip,pmu", 0);
diff --git a/drivers/devfreq/mtk-cci-devfreq.c b/drivers/devfreq/mtk-cci-devfreq.c
index 71abb3fbd042..e5458ada5197 100644
--- a/drivers/devfreq/mtk-cci-devfreq.c
+++ b/drivers/devfreq/mtk-cci-devfreq.c
@@ -291,9 +291,13 @@ static int mtk_ccifreq_probe(struct platform_device *pdev)
 	}
 
 	drv->sram_reg = devm_regulator_get_optional(dev, "sram");
-	if (IS_ERR(drv->sram_reg))
+	if (IS_ERR(drv->sram_reg)) {
+		ret = PTR_ERR(drv->sram_reg);
+		if (ret == -EPROBE_DEFER)
+			goto out_free_resources;
+
 		drv->sram_reg = NULL;
-	else {
+	} else {
 		ret = regulator_enable(drv->sram_reg);
 		if (ret) {
 			dev_err(dev, "failed to enable sram regulator\n");
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index efb4990b29e1..dd0f83ee505b 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -53,7 +53,7 @@ static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen)
 		ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN);
 	spin_unlock(&dmabuf->name_lock);
 
-	return dynamic_dname(dentry, buffer, buflen, "/%s:%s",
+	return dynamic_dname(buffer, buflen, "/%s:%s",
 			     dentry->d_name.name, ret > 0 ? name : "");
 }
 
@@ -531,11 +531,11 @@ static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags)
 	 * value.
 	 */
 	inode->i_ino = atomic64_add_return(1, &dmabuf_inode);
+	flags &= O_ACCMODE | O_NONBLOCK;
 	file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf",
 				 flags, &dma_buf_fops);
 	if (IS_ERR(file))
 		goto err_alloc_file;
-	file->f_flags = flags & (O_ACCMODE | O_NONBLOCK);
 	file->private_data = dmabuf;
 	file->f_path.dentry->d_fsdata = dmabuf;
 
diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c
index 066400ed8841..406b4e26f538 100644
--- a/drivers/dma-buf/dma-fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -136,6 +136,10 @@ struct dma_fence *dma_fence_get_stub(void)
 			       &dma_fence_stub_ops,
 			       &dma_fence_stub_lock,
 			       0, 0);
+
+		set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+			&dma_fence_stub.flags);
+
 		dma_fence_signal_locked(&dma_fence_stub);
 	}
 	spin_unlock(&dma_fence_stub_lock);
@@ -161,6 +165,10 @@ struct dma_fence *dma_fence_allocate_private_stub(void)
 		       &dma_fence_stub_ops,
 		       &dma_fence_stub_lock,
 		       0, 0);
+
+	set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
+		&dma_fence_stub.flags);
+
 	dma_fence_signal(fence);
 
 	return fence;
@@ -500,6 +508,8 @@ dma_fence_wait_timeout(struct dma_fence *fence, bool intr, signed long timeout)
 
 	__dma_fence_might_wait();
 
+	dma_fence_enable_sw_signaling(fence);
+
 	trace_dma_fence_wait_start(fence);
 	if (fence->ops->wait)
 		ret = fence->ops->wait(fence, intr, timeout);
@@ -601,9 +611,6 @@ void dma_fence_enable_sw_signaling(struct dma_fence *fence)
 {
 	unsigned long flags;
 
-	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
-		return;
-
 	spin_lock_irqsave(fence->lock, flags);
 	__dma_fence_enable_signaling(fence);
 	spin_unlock_irqrestore(fence->lock, flags);
@@ -756,19 +763,16 @@ dma_fence_default_wait(struct dma_fence *fence, bool intr, signed long timeout)
 	unsigned long flags;
 	signed long ret = timeout ? timeout : 1;
 
-	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
-		return ret;
-
 	spin_lock_irqsave(fence->lock, flags);
 
+	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
+		goto out;
+
 	if (intr && signal_pending(current)) {
 		ret = -ERESTARTSYS;
 		goto out;
 	}
 
-	if (!__dma_fence_enable_signaling(fence))
-		goto out;
-
 	if (!timeout) {
 		ret = 0;
 		goto out;
diff --git a/drivers/dma-buf/st-dma-fence-chain.c b/drivers/dma-buf/st-dma-fence-chain.c
index 8ce1ea59d31b..0a9b099d0518 100644
--- a/drivers/dma-buf/st-dma-fence-chain.c
+++ b/drivers/dma-buf/st-dma-fence-chain.c
@@ -87,6 +87,8 @@ static int sanitycheck(void *arg)
 	if (!chain)
 		err = -ENOMEM;
 
+	dma_fence_enable_sw_signaling(chain);
+
 	dma_fence_signal(f);
 	dma_fence_put(f);
 
@@ -143,6 +145,8 @@ static int fence_chains_init(struct fence_chains *fc, unsigned int count,
 		}
 
 		fc->tail = fc->chains[i];
+
+		dma_fence_enable_sw_signaling(fc->chains[i]);
 	}
 
 	fc->chain_length = i;
diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
index 4105d5ea8dde..f0cee984b6c7 100644
--- a/drivers/dma-buf/st-dma-fence-unwrap.c
+++ b/drivers/dma-buf/st-dma-fence-unwrap.c
@@ -102,6 +102,8 @@ static int sanitycheck(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	array = mock_array(1, f);
 	if (!array)
 		return -ENOMEM;
@@ -124,12 +126,16 @@ static int unwrap_array(void *arg)
 	if (!f1)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f1);
+
 	f2 = mock_fence();
 	if (!f2) {
 		dma_fence_put(f1);
 		return -ENOMEM;
 	}
 
+	dma_fence_enable_sw_signaling(f2);
+
 	array = mock_array(2, f1, f2);
 	if (!array)
 		return -ENOMEM;
@@ -164,12 +170,16 @@ static int unwrap_chain(void *arg)
 	if (!f1)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f1);
+
 	f2 = mock_fence();
 	if (!f2) {
 		dma_fence_put(f1);
 		return -ENOMEM;
 	}
 
+	dma_fence_enable_sw_signaling(f2);
+
 	chain = mock_chain(f1, f2);
 	if (!chain)
 		return -ENOMEM;
@@ -204,12 +214,16 @@ static int unwrap_chain_array(void *arg)
 	if (!f1)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f1);
+
 	f2 = mock_fence();
 	if (!f2) {
 		dma_fence_put(f1);
 		return -ENOMEM;
 	}
 
+	dma_fence_enable_sw_signaling(f2);
+
 	array = mock_array(2, f1, f2);
 	if (!array)
 		return -ENOMEM;
@@ -248,12 +262,16 @@ static int unwrap_merge(void *arg)
 	if (!f1)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f1);
+
 	f2 = mock_fence();
 	if (!f2) {
 		err = -ENOMEM;
 		goto error_put_f1;
 	}
 
+	dma_fence_enable_sw_signaling(f2);
+
 	f3 = dma_fence_unwrap_merge(f1, f2);
 	if (!f3) {
 		err = -ENOMEM;
@@ -296,10 +314,14 @@ static int unwrap_merge_complex(void *arg)
 	if (!f1)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f1);
+
 	f2 = mock_fence();
 	if (!f2)
 		goto error_put_f1;
 
+	dma_fence_enable_sw_signaling(f2);
+
 	f3 = dma_fence_unwrap_merge(f1, f2);
 	if (!f3)
 		goto error_put_f2;
diff --git a/drivers/dma-buf/st-dma-fence.c b/drivers/dma-buf/st-dma-fence.c
index c8a12d7ad71a..fb6e0a6ae2c9 100644
--- a/drivers/dma-buf/st-dma-fence.c
+++ b/drivers/dma-buf/st-dma-fence.c
@@ -102,6 +102,8 @@ static int sanitycheck(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	dma_fence_signal(f);
 	dma_fence_put(f);
 
@@ -117,6 +119,8 @@ static int test_signaling(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	if (dma_fence_is_signaled(f)) {
 		pr_err("Fence unexpectedly signaled on creation\n");
 		goto err_free;
@@ -190,6 +194,8 @@ static int test_late_add_callback(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	dma_fence_signal(f);
 
 	if (!dma_fence_add_callback(f, &cb.cb, simple_callback)) {
@@ -282,6 +288,8 @@ static int test_status(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	if (dma_fence_get_status(f)) {
 		pr_err("Fence unexpectedly has signaled status on creation\n");
 		goto err_free;
@@ -308,6 +316,8 @@ static int test_error(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	dma_fence_set_error(f, -EIO);
 
 	if (dma_fence_get_status(f)) {
@@ -337,6 +347,8 @@ static int test_wait(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	if (dma_fence_wait_timeout(f, false, 0) != -ETIME) {
 		pr_err("Wait reported complete before being signaled\n");
 		goto err_free;
@@ -379,6 +391,8 @@ static int test_wait_timeout(void *arg)
 	if (!wt.f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(wt.f);
+
 	if (dma_fence_wait_timeout(wt.f, false, 1) != -ETIME) {
 		pr_err("Wait reported complete before being signaled\n");
 		goto err_free;
@@ -458,6 +472,8 @@ static int thread_signal_callback(void *arg)
 			break;
 		}
 
+		dma_fence_enable_sw_signaling(f1);
+
 		rcu_assign_pointer(t->fences[t->id], f1);
 		smp_wmb();
 
diff --git a/drivers/dma-buf/st-dma-resv.c b/drivers/dma-buf/st-dma-resv.c
index 813779e3c9be..15dbea1462ed 100644
--- a/drivers/dma-buf/st-dma-resv.c
+++ b/drivers/dma-buf/st-dma-resv.c
@@ -45,6 +45,8 @@ static int sanitycheck(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	dma_fence_signal(f);
 	dma_fence_put(f);
 
@@ -69,6 +71,8 @@ static int test_signaling(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
 	if (r) {
@@ -114,6 +118,8 @@ static int test_for_each(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
 	if (r) {
@@ -173,6 +179,8 @@ static int test_for_each_unlocked(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
 	if (r) {
@@ -244,6 +252,8 @@ static int test_get_fences(void *arg)
 	if (!f)
 		return -ENOMEM;
 
+	dma_fence_enable_sw_signaling(f);
+
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
 	if (r) {
diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c
index 3ebec19a8e02..af57799c86ce 100644
--- a/drivers/dma-buf/sync_file.c
+++ b/drivers/dma-buf/sync_file.c
@@ -132,7 +132,7 @@ EXPORT_SYMBOL(sync_file_get_fence);
 char *sync_file_get_name(struct sync_file *sync_file, char *buf, int len)
 {
 	if (sync_file->user_name[0]) {
-		strlcpy(buf, sync_file->user_name, len);
+		strscpy(buf, sync_file->user_name, len);
 	} else {
 		struct dma_fence *fence = sync_file->fence;
 
@@ -172,7 +172,7 @@ static struct sync_file *sync_file_merge(const char *name, struct sync_file *a,
 		return NULL;
 	}
 	sync_file->fence = fence;
-	strlcpy(sync_file->user_name, name, sizeof(sync_file->user_name));
+	strscpy(sync_file->user_name, name, sizeof(sync_file->user_name));
 	return sync_file;
 }
 
@@ -262,9 +262,9 @@ err_put_fd:
 static int sync_fill_fence_info(struct dma_fence *fence,
 				 struct sync_fence_info *info)
 {
-	strlcpy(info->obj_name, fence->ops->get_timeline_name(fence),
+	strscpy(info->obj_name, fence->ops->get_timeline_name(fence),
 		sizeof(info->obj_name));
-	strlcpy(info->driver_name, fence->ops->get_driver_name(fence),
+	strscpy(info->driver_name, fence->ops->get_driver_name(fence),
 		sizeof(info->driver_name));
 
 	info->status = dma_fence_get_status(fence);
diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c
index 38e8767ec371..2bcdb935a3ac 100644
--- a/drivers/dma-buf/udmabuf.c
+++ b/drivers/dma-buf/udmabuf.c
@@ -124,17 +124,20 @@ static int begin_cpu_udmabuf(struct dma_buf *buf,
 {
 	struct udmabuf *ubuf = buf->priv;
 	struct device *dev = ubuf->device->this_device;
+	int ret = 0;
 
 	if (!ubuf->sg) {
 		ubuf->sg = get_sg_table(dev, buf, direction);
-		if (IS_ERR(ubuf->sg))
-			return PTR_ERR(ubuf->sg);
+		if (IS_ERR(ubuf->sg)) {
+			ret = PTR_ERR(ubuf->sg);
+			ubuf->sg = NULL;
+		}
 	} else {
 		dma_sync_sg_for_cpu(dev, ubuf->sg->sgl, ubuf->sg->nents,
 				    direction);
 	}
 
-	return 0;
+	return ret;
 }
 
 static int end_cpu_udmabuf(struct dma_buf *buf,
@@ -210,7 +213,7 @@ static long udmabuf_create(struct miscdevice *device,
 		memfd = fget(list[i].memfd);
 		if (!memfd)
 			goto err;
-		mapping = file_inode(memfd)->i_mapping;
+		mapping = memfd->f_mapping;
 		if (!shmem_mapping(mapping) && !is_file_hugepages(memfd))
 			goto err;
 		seals = memfd_fcntl(memfd, F_GET_SEALS, 0);
diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c
index eb58644bb019..6faeb2ab3960 100644
--- a/drivers/edac/edac_mc.c
+++ b/drivers/edac/edac_mc.c
@@ -103,7 +103,6 @@ static void edac_mc_dump_dimm(struct dimm_info *dimm)
 	edac_dbg(4, "  dimm->label = '%s'\n", dimm->label);
 	edac_dbg(4, "  dimm->nr_pages = 0x%x\n", dimm->nr_pages);
 	edac_dbg(4, "  dimm->grain = %d\n", dimm->grain);
-	edac_dbg(4, "  dimm->nr_pages = 0x%x\n", dimm->nr_pages);
 }
 
 static void edac_mc_dump_csrow(struct csrow_info *csrow)
diff --git a/drivers/edac/edac_module.h b/drivers/edac/edac_module.h
index 96f6de0c8ff6..50ed9f2425bb 100644
--- a/drivers/edac/edac_module.h
+++ b/drivers/edac/edac_module.h
@@ -28,13 +28,9 @@ void edac_mc_sysfs_exit(void);
 extern int edac_create_sysfs_mci_device(struct mem_ctl_info *mci,
 					const struct attribute_group **groups);
 extern void edac_remove_sysfs_mci_device(struct mem_ctl_info *mci);
-extern int edac_get_log_ue(void);
-extern int edac_get_log_ce(void);
-extern int edac_get_panic_on_ue(void);
 extern int edac_mc_get_log_ue(void);
 extern int edac_mc_get_log_ce(void);
 extern int edac_mc_get_panic_on_ue(void);
-extern int edac_get_poll_msec(void);
 extern unsigned int edac_mc_get_poll_msec(void);
 
 unsigned edac_dimm_info_location(struct dimm_info *dimm, char *buf,
diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c
index 6cf50ee0b77c..a22ea053f8e1 100644
--- a/drivers/edac/i10nm_base.c
+++ b/drivers/edac/i10nm_base.c
@@ -74,31 +74,47 @@ static struct list_head *i10nm_edac_list;
 
 static struct res_config *res_cfg;
 static int retry_rd_err_log;
+static int decoding_via_mca;
+static bool mem_cfg_2lm;
 
 static u32 offsets_scrub_icx[]  = {0x22c60, 0x22c54, 0x22c5c, 0x22c58, 0x22c28, 0x20ed8};
 static u32 offsets_scrub_spr[]  = {0x22c60, 0x22c54, 0x22f08, 0x22c58, 0x22c28, 0x20ed8};
+static u32 offsets_scrub_spr_hbm0[]  = {0x2860, 0x2854, 0x2b08, 0x2858, 0x2828, 0x0ed8};
+static u32 offsets_scrub_spr_hbm1[]  = {0x2c60, 0x2c54, 0x2f08, 0x2c58, 0x2c28, 0x0fa8};
 static u32 offsets_demand_icx[] = {0x22e54, 0x22e60, 0x22e64, 0x22e58, 0x22e5c, 0x20ee0};
 static u32 offsets_demand_spr[] = {0x22e54, 0x22e60, 0x22f10, 0x22e58, 0x22e5c, 0x20ee0};
+static u32 offsets_demand2_spr[] = {0x22c70, 0x22d80, 0x22f18, 0x22d58, 0x22c64, 0x20f10};
+static u32 offsets_demand_spr_hbm0[] = {0x2a54, 0x2a60, 0x2b10, 0x2a58, 0x2a5c, 0x0ee0};
+static u32 offsets_demand_spr_hbm1[] = {0x2e54, 0x2e60, 0x2f10, 0x2e58, 0x2e5c, 0x0fb0};
 
-static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable)
+static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable,
+				      u32 *offsets_scrub, u32 *offsets_demand,
+				      u32 *offsets_demand2)
 {
-	u32 s, d;
+	u32 s, d, d2;
 
-	if (!imc->mbase)
-		return;
-
-	s = I10NM_GET_REG32(imc, chan, res_cfg->offsets_scrub[0]);
-	d = I10NM_GET_REG32(imc, chan, res_cfg->offsets_demand[0]);
+	s = I10NM_GET_REG32(imc, chan, offsets_scrub[0]);
+	d = I10NM_GET_REG32(imc, chan, offsets_demand[0]);
+	if (offsets_demand2)
+		d2 = I10NM_GET_REG32(imc, chan, offsets_demand2[0]);
 
 	if (enable) {
 		/* Save default configurations */
 		imc->chan[chan].retry_rd_err_log_s = s;
 		imc->chan[chan].retry_rd_err_log_d = d;
+		if (offsets_demand2)
+			imc->chan[chan].retry_rd_err_log_d2 = d2;
 
 		s &= ~RETRY_RD_ERR_LOG_NOOVER_UC;
 		s |=  RETRY_RD_ERR_LOG_EN;
 		d &= ~RETRY_RD_ERR_LOG_NOOVER_UC;
 		d |=  RETRY_RD_ERR_LOG_EN;
+
+		if (offsets_demand2) {
+			d2 &= ~RETRY_RD_ERR_LOG_UC;
+			d2 |=  RETRY_RD_ERR_LOG_NOOVER;
+			d2 |=  RETRY_RD_ERR_LOG_EN;
+		}
 	} else {
 		/* Restore default configurations */
 		if (imc->chan[chan].retry_rd_err_log_s & RETRY_RD_ERR_LOG_UC)
@@ -113,23 +129,55 @@ static void __enable_retry_rd_err_log(struct skx_imc *imc, int chan, bool enable
 			d |=  RETRY_RD_ERR_LOG_NOOVER;
 		if (!(imc->chan[chan].retry_rd_err_log_d & RETRY_RD_ERR_LOG_EN))
 			d &= ~RETRY_RD_ERR_LOG_EN;
+
+		if (offsets_demand2) {
+			if (imc->chan[chan].retry_rd_err_log_d2 & RETRY_RD_ERR_LOG_UC)
+				d2 |=  RETRY_RD_ERR_LOG_UC;
+			if (!(imc->chan[chan].retry_rd_err_log_d2 & RETRY_RD_ERR_LOG_NOOVER))
+				d2 &=  ~RETRY_RD_ERR_LOG_NOOVER;
+			if (!(imc->chan[chan].retry_rd_err_log_d2 & RETRY_RD_ERR_LOG_EN))
+				d2 &= ~RETRY_RD_ERR_LOG_EN;
+		}
 	}
 
-	I10NM_SET_REG32(imc, chan, res_cfg->offsets_scrub[0], s);
-	I10NM_SET_REG32(imc, chan, res_cfg->offsets_demand[0], d);
+	I10NM_SET_REG32(imc, chan, offsets_scrub[0], s);
+	I10NM_SET_REG32(imc, chan, offsets_demand[0], d);
+	if (offsets_demand2)
+		I10NM_SET_REG32(imc, chan, offsets_demand2[0], d2);
 }
 
 static void enable_retry_rd_err_log(bool enable)
 {
+	struct skx_imc *imc;
 	struct skx_dev *d;
 	int i, j;
 
 	edac_dbg(2, "\n");
 
 	list_for_each_entry(d, i10nm_edac_list, list)
-		for (i = 0; i < I10NM_NUM_IMC; i++)
-			for (j = 0; j < I10NM_NUM_CHANNELS; j++)
-				__enable_retry_rd_err_log(&d->imc[i], j, enable);
+		for (i = 0; i < I10NM_NUM_IMC; i++) {
+			imc = &d->imc[i];
+			if (!imc->mbase)
+				continue;
+
+			for (j = 0; j < I10NM_NUM_CHANNELS; j++) {
+				if (imc->hbm_mc) {
+					__enable_retry_rd_err_log(imc, j, enable,
+								  res_cfg->offsets_scrub_hbm0,
+								  res_cfg->offsets_demand_hbm0,
+								  NULL);
+					__enable_retry_rd_err_log(imc, j, enable,
+								  res_cfg->offsets_scrub_hbm1,
+								  res_cfg->offsets_demand_hbm1,
+								  NULL);
+				} else {
+					__enable_retry_rd_err_log(imc, j, enable,
+								  res_cfg->offsets_scrub,
+								  res_cfg->offsets_demand,
+								  res_cfg->offsets_demand2);
+				}
+			}
+	}
 }
 
 static void show_retry_rd_err_log(struct decoded_addr *res, char *msg,
@@ -138,14 +186,33 @@ static void show_retry_rd_err_log(struct decoded_addr *res, char *msg,
 	struct skx_imc *imc = &res->dev->imc[res->imc];
 	u32 log0, log1, log2, log3, log4;
 	u32 corr0, corr1, corr2, corr3;
+	u32 lxg0, lxg1, lxg3, lxg4;
+	u32 *xffsets = NULL;
 	u64 log2a, log5;
+	u64 lxg2a, lxg5;
 	u32 *offsets;
-	int n;
+	int n, pch;
 
 	if (!imc->mbase)
 		return;
 
-	offsets = scrub_err ? res_cfg->offsets_scrub : res_cfg->offsets_demand;
+	if (imc->hbm_mc) {
+		pch = res->cs & 1;
+
+		if (pch)
+			offsets = scrub_err ? res_cfg->offsets_scrub_hbm1 :
+					      res_cfg->offsets_demand_hbm1;
+		else
+			offsets = scrub_err ? res_cfg->offsets_scrub_hbm0 :
+					      res_cfg->offsets_demand_hbm0;
+	} else {
+		if (scrub_err) {
+			offsets = res_cfg->offsets_scrub;
+		} else {
+			offsets = res_cfg->offsets_demand;
+			xffsets = res_cfg->offsets_demand2;
+		}
+	}
 
 	log0 = I10NM_GET_REG32(imc, res->channel, offsets[0]);
 	log1 = I10NM_GET_REG32(imc, res->channel, offsets[1]);
@@ -153,20 +220,52 @@ static void show_retry_rd_err_log(struct decoded_addr *res, char *msg,
 	log4 = I10NM_GET_REG32(imc, res->channel, offsets[4]);
 	log5 = I10NM_GET_REG64(imc, res->channel, offsets[5]);
 
+	if (xffsets) {
+		lxg0 = I10NM_GET_REG32(imc, res->channel, xffsets[0]);
+		lxg1 = I10NM_GET_REG32(imc, res->channel, xffsets[1]);
+		lxg3 = I10NM_GET_REG32(imc, res->channel, xffsets[3]);
+		lxg4 = I10NM_GET_REG32(imc, res->channel, xffsets[4]);
+		lxg5 = I10NM_GET_REG64(imc, res->channel, xffsets[5]);
+	}
+
 	if (res_cfg->type == SPR) {
 		log2a = I10NM_GET_REG64(imc, res->channel, offsets[2]);
-		n = snprintf(msg, len, " retry_rd_err_log[%.8x %.8x %.16llx %.8x %.8x %.16llx]",
+		n = snprintf(msg, len, " retry_rd_err_log[%.8x %.8x %.16llx %.8x %.8x %.16llx",
 			     log0, log1, log2a, log3, log4, log5);
+
+		if (len - n > 0) {
+			if (xffsets) {
+				lxg2a = I10NM_GET_REG64(imc, res->channel, xffsets[2]);
+				n += snprintf(msg + n, len - n, " %.8x %.8x %.16llx %.8x %.8x %.16llx]",
+					     lxg0, lxg1, lxg2a, lxg3, lxg4, lxg5);
+			} else {
+				n += snprintf(msg + n, len - n, "]");
+			}
+		}
 	} else {
 		log2 = I10NM_GET_REG32(imc, res->channel, offsets[2]);
 		n = snprintf(msg, len, " retry_rd_err_log[%.8x %.8x %.8x %.8x %.8x %.16llx]",
 			     log0, log1, log2, log3, log4, log5);
 	}
 
-	corr0 = I10NM_GET_REG32(imc, res->channel, 0x22c18);
-	corr1 = I10NM_GET_REG32(imc, res->channel, 0x22c1c);
-	corr2 = I10NM_GET_REG32(imc, res->channel, 0x22c20);
-	corr3 = I10NM_GET_REG32(imc, res->channel, 0x22c24);
+	if (imc->hbm_mc) {
+		if (pch) {
+			corr0 = I10NM_GET_REG32(imc, res->channel, 0x2c18);
+			corr1 = I10NM_GET_REG32(imc, res->channel, 0x2c1c);
+			corr2 = I10NM_GET_REG32(imc, res->channel, 0x2c20);
+			corr3 = I10NM_GET_REG32(imc, res->channel, 0x2c24);
+		} else {
+			corr0 = I10NM_GET_REG32(imc, res->channel, 0x2818);
+			corr1 = I10NM_GET_REG32(imc, res->channel, 0x281c);
+			corr2 = I10NM_GET_REG32(imc, res->channel, 0x2820);
+			corr3 = I10NM_GET_REG32(imc, res->channel, 0x2824);
+		}
+	} else {
+		corr0 = I10NM_GET_REG32(imc, res->channel, 0x22c18);
+		corr1 = I10NM_GET_REG32(imc, res->channel, 0x22c1c);
+		corr2 = I10NM_GET_REG32(imc, res->channel, 0x22c20);
+		corr3 = I10NM_GET_REG32(imc, res->channel, 0x22c24);
+	}
 
 	if (len - n > 0)
 		snprintf(msg + n, len - n,
@@ -177,9 +276,16 @@ static void show_retry_rd_err_log(struct decoded_addr *res, char *msg,
 			 corr3 & 0xffff, corr3 >> 16);
 
 	/* Clear status bits */
-	if (retry_rd_err_log == 2 && (log0 & RETRY_RD_ERR_LOG_OVER_UC_V)) {
-		log0 &= ~RETRY_RD_ERR_LOG_OVER_UC_V;
-		I10NM_SET_REG32(imc, res->channel, offsets[0], log0);
+	if (retry_rd_err_log == 2) {
+		if (log0 & RETRY_RD_ERR_LOG_OVER_UC_V) {
+			log0 &= ~RETRY_RD_ERR_LOG_OVER_UC_V;
+			I10NM_SET_REG32(imc, res->channel, offsets[0], log0);
+		}
+
+		if (xffsets && (lxg0 & RETRY_RD_ERR_LOG_OVER_UC_V)) {
+			lxg0 &= ~RETRY_RD_ERR_LOG_OVER_UC_V;
+			I10NM_SET_REG32(imc, res->channel, xffsets[0], lxg0);
+		}
 	}
 }
 
@@ -231,6 +337,103 @@ static bool i10nm_check_2lm(struct res_config *cfg)
 	return false;
 }
 
+/*
+ * Check whether the error comes from DDRT by ICX/Tremont model specific error code.
+ * Refer to SDM vol3B 16.11.3 Intel IMC MC error codes for IA32_MCi_STATUS.
+ */
+static bool i10nm_mscod_is_ddrt(u32 mscod)
+{
+	switch (mscod) {
+	case 0x0106: case 0x0107:
+	case 0x0800: case 0x0804:
+	case 0x0806 ... 0x0808:
+	case 0x080a ... 0x080e:
+	case 0x0810: case 0x0811:
+	case 0x0816: case 0x081e:
+	case 0x081f:
+		return true;
+	}
+
+	return false;
+}
+
+static bool i10nm_mc_decode_available(struct mce *mce)
+{
+	u8 bank;
+
+	if (!decoding_via_mca || mem_cfg_2lm)
+		return false;
+
+	if ((mce->status & (MCI_STATUS_MISCV | MCI_STATUS_ADDRV))
+			!= (MCI_STATUS_MISCV | MCI_STATUS_ADDRV))
+		return false;
+
+	bank = mce->bank;
+
+	switch (res_cfg->type) {
+	case I10NM:
+		if (bank < 13 || bank > 26)
+			return false;
+
+		/* DDRT errors can't be decoded from MCA bank registers */
+		if (MCI_MISC_ECC_MODE(mce->misc) == MCI_MISC_ECC_DDRT)
+			return false;
+
+		if (i10nm_mscod_is_ddrt(MCI_STATUS_MSCOD(mce->status)))
+			return false;
+
+		/* Check whether one of {13,14,17,18,21,22,25,26} */
+		return ((bank - 13) & BIT(1)) == 0;
+	default:
+		return false;
+	}
+}
+
+static bool i10nm_mc_decode(struct decoded_addr *res)
+{
+	struct mce *m = res->mce;
+	struct skx_dev *d;
+	u8 bank;
+
+	if (!i10nm_mc_decode_available(m))
+		return false;
+
+	list_for_each_entry(d, i10nm_edac_list, list) {
+		if (d->imc[0].src_id == m->socketid) {
+			res->socket = m->socketid;
+			res->dev = d;
+			break;
+		}
+	}
+
+	switch (res_cfg->type) {
+	case I10NM:
+		bank = m->bank - 13;
+		res->imc = bank / 4;
+		res->channel = bank % 2;
+		break;
+	default:
+		return false;
+	}
+
+	if (!res->dev) {
+		skx_printk(KERN_ERR, "No device for src_id %d imc %d\n",
+			   m->socketid, res->imc);
+		return false;
+	}
+
+	res->column       = GET_BITFIELD(m->misc, 9, 18) << 2;
+	res->row          = GET_BITFIELD(m->misc, 19, 39);
+	res->bank_group   = GET_BITFIELD(m->misc, 40, 41);
+	res->bank_address = GET_BITFIELD(m->misc, 42, 43);
+	res->bank_group  |= GET_BITFIELD(m->misc, 44, 44) << 2;
+	res->rank         = GET_BITFIELD(m->misc, 56, 58);
+	res->dimm         = res->rank >> 2;
+	res->rank         = res->rank % 4;
+
+	return true;
+}
+
 static int i10nm_get_ddr_munits(void)
 {
 	struct pci_dev *mdev;
@@ -420,7 +623,12 @@ static struct res_config spr_cfg = {
 	.sad_all_devfn		= PCI_DEVFN(10, 0),
 	.sad_all_offset		= 0x300,
 	.offsets_scrub		= offsets_scrub_spr,
+	.offsets_scrub_hbm0	= offsets_scrub_spr_hbm0,
+	.offsets_scrub_hbm1	= offsets_scrub_spr_hbm1,
 	.offsets_demand		= offsets_demand_spr,
+	.offsets_demand2	= offsets_demand2_spr,
+	.offsets_demand_hbm0	= offsets_demand_spr_hbm0,
+	.offsets_demand_hbm1	= offsets_demand_spr_hbm1,
 };
 
 static const struct x86_cpu_id i10nm_cpuids[] = {
@@ -574,7 +782,8 @@ static int __init i10nm_init(void)
 		return -ENODEV;
 	}
 
-	skx_set_mem_cfg(i10nm_check_2lm(cfg));
+	mem_cfg_2lm = i10nm_check_2lm(cfg);
+	skx_set_mem_cfg(mem_cfg_2lm);
 
 	rc = i10nm_get_ddr_munits();
 
@@ -626,9 +835,11 @@ static int __init i10nm_init(void)
 	setup_i10nm_debug();
 
 	if (retry_rd_err_log && res_cfg->offsets_scrub && res_cfg->offsets_demand) {
-		skx_set_decode(NULL, show_retry_rd_err_log);
+		skx_set_decode(i10nm_mc_decode, show_retry_rd_err_log);
 		if (retry_rd_err_log == 2)
 			enable_retry_rd_err_log(true);
+	} else {
+		skx_set_decode(i10nm_mc_decode, NULL);
 	}
 
 	i10nm_printk(KERN_INFO, "%s\n", I10NM_REVISION);
@@ -658,6 +869,34 @@ static void __exit i10nm_exit(void)
 module_init(i10nm_init);
 module_exit(i10nm_exit);
 
+static int set_decoding_via_mca(const char *buf, const struct kernel_param *kp)
+{
+	unsigned long val;
+	int ret;
+
+	ret = kstrtoul(buf, 0, &val);
+
+	if (ret || val > 1)
+		return -EINVAL;
+
+	if (val && mem_cfg_2lm) {
+		i10nm_printk(KERN_NOTICE, "Decoding errors via MCA banks for 2LM isn't supported yet\n");
+		return -EIO;
+	}
+
+	ret = param_set_int(buf, kp);
+
+	return ret;
+}
+
+static const struct kernel_param_ops decoding_via_mca_param_ops = {
+	.set = set_decoding_via_mca,
+	.get = param_get_int,
+};
+
+module_param_cb(decoding_via_mca, &decoding_via_mca_param_ops, &decoding_via_mca, 0644);
+MODULE_PARM_DESC(decoding_via_mca, "decoding_via_mca: 0=off(default), 1=enable");
+
 module_param(retry_rd_err_log, int, 0444);
 MODULE_PARM_DESC(retry_rd_err_log, "retry_rd_err_log: 0=off(default), 1=bios(Linux doesn't reset any control bits, but just reports values.), 2=linux(Linux tries to take control and resets mode bits, clear valid/UC bits after reading.)");
 
diff --git a/drivers/edac/i7300_edac.c b/drivers/edac/i7300_edac.c
index 4f28b8c8d378..61adaa872ba7 100644
--- a/drivers/edac/i7300_edac.c
+++ b/drivers/edac/i7300_edac.c
@@ -1193,7 +1193,7 @@ static int __init i7300_init(void)
 }
 
 /**
- * i7300_init() - Unregisters the driver
+ * i7300_exit() - Unregisters the driver
  */
 static void __exit i7300_exit(void)
 {
diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c
index 9a9ff5ad611a..9ef13570f2e5 100644
--- a/drivers/edac/ie31200_edac.c
+++ b/drivers/edac/ie31200_edac.c
@@ -20,11 +20,15 @@
  * 0c08: Xeon E3-1200 v3 Processor DRAM Controller
  * 1918: Xeon E3-1200 v5 Skylake Host Bridge/DRAM Registers
  * 5918: Xeon E3-1200 Xeon E3-1200 v6/7th Gen Core Processor Host Bridge/DRAM Registers
+ * 190f: 6th Gen Core Dual-Core Processor Host Bridge/DRAM Registers
+ * 191f: 6th Gen Core Quad-Core Processor Host Bridge/DRAM Registers
  * 3e..: 8th/9th Gen Core Processor Host Bridge/DRAM Registers
  *
  * Based on Intel specification:
  * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/xeon-e3-1200v3-vol-2-datasheet.pdf
  * http://www.intel.com/content/www/us/en/processors/xeon/xeon-e3-1200-family-vol-2-datasheet.html
+ * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/desktop-6th-gen-core-family-datasheet-vol-2.pdf
+ * https://www.intel.com/content/dam/www/public/us/en/documents/datasheets/xeon-e3-1200v6-vol-2-datasheet.pdf
  * https://www.intel.com/content/www/us/en/processors/core/7th-gen-core-family-mobile-h-processor-lines-datasheet-vol-2.html
  * https://www.intel.com/content/www/us/en/products/docs/processors/core/8th-gen-core-family-datasheet-vol-2.html
  *
@@ -53,15 +57,17 @@
 #define ie31200_printk(level, fmt, arg...) \
 	edac_printk(level, "ie31200", fmt, ##arg)
 
-#define PCI_DEVICE_ID_INTEL_IE31200_HB_1 0x0108
-#define PCI_DEVICE_ID_INTEL_IE31200_HB_2 0x010c
-#define PCI_DEVICE_ID_INTEL_IE31200_HB_3 0x0150
-#define PCI_DEVICE_ID_INTEL_IE31200_HB_4 0x0158
-#define PCI_DEVICE_ID_INTEL_IE31200_HB_5 0x015c
-#define PCI_DEVICE_ID_INTEL_IE31200_HB_6 0x0c04
-#define PCI_DEVICE_ID_INTEL_IE31200_HB_7 0x0c08
-#define PCI_DEVICE_ID_INTEL_IE31200_HB_8 0x1918
-#define PCI_DEVICE_ID_INTEL_IE31200_HB_9 0x5918
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_1  0x0108
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_2  0x010c
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_3  0x0150
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_4  0x0158
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_5  0x015c
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_6  0x0c04
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_7  0x0c08
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_8  0x190F
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_9  0x1918
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_10 0x191F
+#define PCI_DEVICE_ID_INTEL_IE31200_HB_11 0x5918
 
 /* Coffee Lake-S */
 #define PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_MASK 0x3e00
@@ -80,6 +86,8 @@
 #define DEVICE_ID_SKYLAKE_OR_LATER(did)                                        \
 	(((did) == PCI_DEVICE_ID_INTEL_IE31200_HB_8) ||                        \
 	 ((did) == PCI_DEVICE_ID_INTEL_IE31200_HB_9) ||                        \
+	 ((did) == PCI_DEVICE_ID_INTEL_IE31200_HB_10) ||                       \
+	 ((did) == PCI_DEVICE_ID_INTEL_IE31200_HB_11) ||                       \
 	 (((did) & PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_MASK) ==                 \
 	  PCI_DEVICE_ID_INTEL_IE31200_HB_CFL_MASK))
 
@@ -577,6 +585,8 @@ static const struct pci_device_id ie31200_pci_tbl[] = {
 	{ PCI_VEND_DEV(INTEL, IE31200_HB_7),      PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
 	{ PCI_VEND_DEV(INTEL, IE31200_HB_8),      PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
 	{ PCI_VEND_DEV(INTEL, IE31200_HB_9),      PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+	{ PCI_VEND_DEV(INTEL, IE31200_HB_10),     PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
+	{ PCI_VEND_DEV(INTEL, IE31200_HB_11),     PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
 	{ PCI_VEND_DEV(INTEL, IE31200_HB_CFL_1),  PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
 	{ PCI_VEND_DEV(INTEL, IE31200_HB_CFL_2),  PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
 	{ PCI_VEND_DEV(INTEL, IE31200_HB_CFL_3),  PCI_ANY_ID, PCI_ANY_ID, 0, 0, IE31200 },
diff --git a/drivers/edac/ppc4xx_edac.c b/drivers/edac/ppc4xx_edac.c
index 0bc670778c99..046969b4e82e 100644
--- a/drivers/edac/ppc4xx_edac.c
+++ b/drivers/edac/ppc4xx_edac.c
@@ -178,11 +178,6 @@ struct ppc4xx_ecc_status {
 	u32 wmirq;
 };
 
-/* Function Prototypes */
-
-static int ppc4xx_edac_probe(struct platform_device *device);
-static int ppc4xx_edac_remove(struct platform_device *device);
-
 /* Global Variables */
 
 /*
@@ -197,15 +192,6 @@ static const struct of_device_id ppc4xx_edac_match[] = {
 };
 MODULE_DEVICE_TABLE(of, ppc4xx_edac_match);
 
-static struct platform_driver ppc4xx_edac_driver = {
-	.probe			= ppc4xx_edac_probe,
-	.remove			= ppc4xx_edac_remove,
-	.driver = {
-		.name = PPC4XX_EDAC_MODULE_NAME,
-		.of_match_table = ppc4xx_edac_match,
-	},
-};
-
 /*
  * TODO: The row and channel parameters likely need to be dynamically
  * set based on the aforementioned variant controller realizations.
@@ -1391,6 +1377,15 @@ ppc4xx_edac_opstate_init(void)
 			     EDAC_OPSTATE_UNKNOWN_STR)));
 }
 
+static struct platform_driver ppc4xx_edac_driver = {
+	.probe			= ppc4xx_edac_probe,
+	.remove			= ppc4xx_edac_remove,
+	.driver = {
+		.name = PPC4XX_EDAC_MODULE_NAME,
+		.of_match_table = ppc4xx_edac_match,
+	},
+};
+
 /**
  * ppc4xx_edac_init - driver/module insertion entry point
  *
diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c
index 9678ab97c7ac..8e39370fdb5c 100644
--- a/drivers/edac/sb_edac.c
+++ b/drivers/edac/sb_edac.c
@@ -335,6 +335,12 @@ struct sbridge_info {
 struct sbridge_channel {
 	u32		ranks;
 	u32		dimms;
+	struct dimm {
+		u32 rowbits;
+		u32 colbits;
+		u32 bank_xor_enable;
+		u32 amap_fine;
+	} dimm[MAX_DIMMS];
 };
 
 struct pci_id_descr {
@@ -1603,7 +1609,7 @@ static int __populate_dimms(struct mem_ctl_info *mci,
 		banks = 8;
 
 	for (i = 0; i < channels; i++) {
-		u32 mtr;
+		u32 mtr, amap = 0;
 
 		int max_dimms_per_channel;
 
@@ -1615,6 +1621,7 @@ static int __populate_dimms(struct mem_ctl_info *mci,
 			max_dimms_per_channel = ARRAY_SIZE(mtr_regs);
 			if (!pvt->pci_tad[i])
 				continue;
+			pci_read_config_dword(pvt->pci_tad[i], 0x8c, &amap);
 		}
 
 		for (j = 0; j < max_dimms_per_channel; j++) {
@@ -1627,6 +1634,7 @@ static int __populate_dimms(struct mem_ctl_info *mci,
 					mtr_regs[j], &mtr);
 			}
 			edac_dbg(4, "Channel #%d  MTR%d = %x\n", i, j, mtr);
+
 			if (IS_DIMM_PRESENT(mtr)) {
 				if (!IS_ECC_ENABLED(pvt->info.mcmtr)) {
 					sbridge_printk(KERN_ERR, "CPU SrcID #%d, Ha #%d, Channel #%d has DIMMs, but ECC is disabled\n",
@@ -1661,6 +1669,11 @@ static int __populate_dimms(struct mem_ctl_info *mci,
 				dimm->dtype = pvt->info.get_width(pvt, mtr);
 				dimm->mtype = mtype;
 				dimm->edac_mode = mode;
+				pvt->channel[i].dimm[j].rowbits = order_base_2(rows);
+				pvt->channel[i].dimm[j].colbits = order_base_2(cols);
+				pvt->channel[i].dimm[j].bank_xor_enable =
+						GET_BITFIELD(pvt->info.mcmtr, 9, 9);
+				pvt->channel[i].dimm[j].amap_fine = GET_BITFIELD(amap, 0, 0);
 				snprintf(dimm->label, sizeof(dimm->label),
 						 "CPU_SrcID#%u_Ha#%u_Chan#%u_DIMM#%u",
 						 pvt->sbridge_dev->source_id, pvt->sbridge_dev->dom, i, j);
@@ -1922,6 +1935,99 @@ static struct mem_ctl_info *get_mci_for_node_id(u8 node_id, u8 ha)
 	return NULL;
 }
 
+static u8 sb_close_row[] = {
+	15, 16, 17, 18, 20, 21, 22, 28, 10, 11, 12, 13, 29, 30, 31, 32, 33
+};
+
+static u8 sb_close_column[] = {
+	3, 4, 5, 14, 19, 23, 24, 25, 26, 27
+};
+
+static u8 sb_open_row[] = {
+	14, 15, 16, 20, 28, 21, 22, 23, 24, 25, 26, 27, 29, 30, 31, 32, 33
+};
+
+static u8 sb_open_column[] = {
+	3, 4, 5, 6, 7, 8, 9, 10, 11, 12
+};
+
+static u8 sb_open_fine_column[] = {
+	3, 4, 5, 7, 8, 9, 10, 11, 12, 13
+};
+
+static int sb_bits(u64 addr, int nbits, u8 *bits)
+{
+	int i, res = 0;
+
+	for (i = 0; i < nbits; i++)
+		res |= ((addr >> bits[i]) & 1) << i;
+	return res;
+}
+
+static int sb_bank_bits(u64 addr, int b0, int b1, int do_xor, int x0, int x1)
+{
+	int ret = GET_BITFIELD(addr, b0, b0) | (GET_BITFIELD(addr, b1, b1) << 1);
+
+	if (do_xor)
+		ret ^= GET_BITFIELD(addr, x0, x0) | (GET_BITFIELD(addr, x1, x1) << 1);
+
+	return ret;
+}
+
+static bool sb_decode_ddr4(struct mem_ctl_info *mci, int ch, u8 rank,
+			   u64 rank_addr, char *msg)
+{
+	int dimmno = 0;
+	int row, col, bank_address, bank_group;
+	struct sbridge_pvt *pvt;
+	u32 bg0 = 0, rowbits = 0, colbits = 0;
+	u32 amap_fine = 0, bank_xor_enable = 0;
+
+	dimmno = (rank < 12) ? rank / 4 : 2;
+	pvt = mci->pvt_info;
+	amap_fine =  pvt->channel[ch].dimm[dimmno].amap_fine;
+	bg0 = amap_fine ? 6 : 13;
+	rowbits = pvt->channel[ch].dimm[dimmno].rowbits;
+	colbits = pvt->channel[ch].dimm[dimmno].colbits;
+	bank_xor_enable = pvt->channel[ch].dimm[dimmno].bank_xor_enable;
+
+	if (pvt->is_lockstep) {
+		pr_warn_once("LockStep row/column decode is not supported yet!\n");
+		msg[0] = '\0';
+		return false;
+	}
+
+	if (pvt->is_close_pg) {
+		row = sb_bits(rank_addr, rowbits, sb_close_row);
+		col = sb_bits(rank_addr, colbits, sb_close_column);
+		col |= 0x400; /* C10 is autoprecharge, always set */
+		bank_address = sb_bank_bits(rank_addr, 8, 9, bank_xor_enable, 22, 28);
+		bank_group = sb_bank_bits(rank_addr, 6, 7, bank_xor_enable, 20, 21);
+	} else {
+		row = sb_bits(rank_addr, rowbits, sb_open_row);
+		if (amap_fine)
+			col = sb_bits(rank_addr, colbits, sb_open_fine_column);
+		else
+			col = sb_bits(rank_addr, colbits, sb_open_column);
+		bank_address = sb_bank_bits(rank_addr, 18, 19, bank_xor_enable, 22, 23);
+		bank_group = sb_bank_bits(rank_addr, bg0, 17, bank_xor_enable, 20, 21);
+	}
+
+	row &= (1u << rowbits) - 1;
+
+	sprintf(msg, "row:0x%x col:0x%x bank_addr:%d bank_group:%d",
+		row, col, bank_address, bank_group);
+	return true;
+}
+
+static bool sb_decode_ddr3(struct mem_ctl_info *mci, int ch, u8 rank,
+			   u64 rank_addr, char *msg)
+{
+	pr_warn_once("DDR3 row/column decode not support yet!\n");
+	msg[0] = '\0';
+	return false;
+}
+
 static int get_memory_error_data(struct mem_ctl_info *mci,
 				 u64 addr,
 				 u8 *socket, u8 *ha,
@@ -1937,12 +2043,13 @@ static int get_memory_error_data(struct mem_ctl_info *mci,
 	int			interleave_mode, shiftup = 0;
 	unsigned int		sad_interleave[MAX_INTERLEAVE];
 	u32			reg, dram_rule;
-	u8			ch_way, sck_way, pkg, sad_ha = 0;
+	u8			ch_way, sck_way, pkg, sad_ha = 0, rankid = 0;
 	u32			tad_offset;
 	u32			rir_way;
 	u32			mb, gb;
 	u64			ch_addr, offset, limit = 0, prv = 0;
-
+	u64			rank_addr;
+	enum mem_type		mtype;
 
 	/*
 	 * Step 0) Check if the address is at special memory ranges
@@ -2226,6 +2333,28 @@ static int get_memory_error_data(struct mem_ctl_info *mci,
 	pci_read_config_dword(pvt->pci_tad[base_ch], rir_offset[n_rir][idx], &reg);
 	*rank = RIR_RNK_TGT(pvt->info.type, reg);
 
+	if (pvt->info.type == BROADWELL) {
+		if (pvt->is_close_pg)
+			shiftup = 6;
+		else
+			shiftup = 13;
+
+		rank_addr = ch_addr >> shiftup;
+		rank_addr /= (1 << rir_way);
+		rank_addr <<= shiftup;
+		rank_addr |= ch_addr & GENMASK_ULL(shiftup - 1, 0);
+		rank_addr -= RIR_OFFSET(pvt->info.type, reg);
+
+		mtype = pvt->info.get_memory_type(pvt);
+		rankid = *rank;
+		if (mtype == MEM_DDR4 || mtype == MEM_RDDR4)
+			sb_decode_ddr4(mci, base_ch, rankid, rank_addr, msg);
+		else
+			sb_decode_ddr3(mci, base_ch, rankid, rank_addr, msg);
+	} else {
+		msg[0] = '\0';
+	}
+
 	edac_dbg(0, "RIR#%d: channel address 0x%08Lx < 0x%08Lx, RIR interleave %d, index %d\n",
 		 n_rir,
 		 ch_addr,
@@ -2950,7 +3079,7 @@ static void sbridge_mce_output_error(struct mem_ctl_info *mci,
 	struct mem_ctl_info *new_mci;
 	struct sbridge_pvt *pvt = mci->pvt_info;
 	enum hw_event_mc_err_type tp_event;
-	char *optype, msg[256];
+	char *optype, msg[256], msg_full[512];
 	bool ripv = GET_BITFIELD(m->mcgstatus, 0, 0);
 	bool overflow = GET_BITFIELD(m->status, 62, 62);
 	bool uncorrected_error = GET_BITFIELD(m->status, 61, 61);
@@ -3089,18 +3218,17 @@ static void sbridge_mce_output_error(struct mem_ctl_info *mci,
 	 */
 	if (!pvt->is_lockstep && !pvt->is_cur_addr_mirrored && !pvt->is_close_pg)
 		channel = first_channel;
-
-	snprintf(msg, sizeof(msg),
-		 "%s%s area:%s err_code:%04x:%04x socket:%d ha:%d channel_mask:%ld rank:%d",
+	snprintf(msg_full, sizeof(msg_full),
+		 "%s%s area:%s err_code:%04x:%04x socket:%d ha:%d channel_mask:%ld rank:%d %s",
 		 overflow ? " OVERFLOW" : "",
 		 (uncorrected_error && recoverable) ? " recoverable" : "",
 		 area_type,
 		 mscod, errcode,
 		 socket, ha,
 		 channel_mask,
-		 rank);
+		 rank, msg);
 
-	edac_dbg(0, "%s\n", msg);
+	edac_dbg(0, "%s\n", msg_full);
 
 	/* FIXME: need support for channel mask */
 
@@ -3111,7 +3239,7 @@ static void sbridge_mce_output_error(struct mem_ctl_info *mci,
 	edac_mc_handle_error(tp_event, mci, core_err_cnt,
 			     m->addr >> PAGE_SHIFT, m->addr & ~PAGE_MASK, 0,
 			     channel, dimm, -1,
-			     optype, msg);
+			     optype, msg_full);
 	return;
 err_parsing:
 	edac_mc_handle_error(tp_event, mci, core_err_cnt, 0, 0, 0,
diff --git a/drivers/edac/skx_base.c b/drivers/edac/skx_base.c
index 1abc020d49ab..7e2762f62eec 100644
--- a/drivers/edac/skx_base.c
+++ b/drivers/edac/skx_base.c
@@ -714,8 +714,13 @@ static int __init skx_init(void)
 
 	skx_set_decode(skx_decode, skx_show_retry_rd_err_log);
 
-	if (nvdimm_count && skx_adxl_get() == -ENODEV)
-		skx_printk(KERN_NOTICE, "Only decoding DDR4 address!\n");
+	if (nvdimm_count && skx_adxl_get() != -ENODEV) {
+		skx_set_decode(NULL, skx_show_retry_rd_err_log);
+	} else {
+		if (nvdimm_count)
+			skx_printk(KERN_NOTICE, "Only decoding DDR4 address!\n");
+		skx_set_decode(skx_decode, skx_show_retry_rd_err_log);
+	}
 
 	/* Ensure that the OPSTATE is set correctly for POLL or NMI */
 	opstate_init();
diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c
index 19c17c5198c5..f0f8e98f6efb 100644
--- a/drivers/edac/skx_common.c
+++ b/drivers/edac/skx_common.c
@@ -27,9 +27,11 @@ static const char * const component_names[] = {
 	[INDEX_MEMCTRL]		= "MemoryControllerId",
 	[INDEX_CHANNEL]		= "ChannelId",
 	[INDEX_DIMM]		= "DimmSlotId",
+	[INDEX_CS]		= "ChipSelect",
 	[INDEX_NM_MEMCTRL]	= "NmMemoryControllerId",
 	[INDEX_NM_CHANNEL]	= "NmChannelId",
 	[INDEX_NM_DIMM]		= "NmDimmSlotId",
+	[INDEX_NM_CS]		= "NmChipSelect",
 };
 
 static int component_indices[ARRAY_SIZE(component_names)];
@@ -40,7 +42,7 @@ static char *adxl_msg;
 static unsigned long adxl_nm_bitmap;
 
 static char skx_msg[MSG_SIZE];
-static skx_decode_f skx_decode;
+static skx_decode_f driver_decode;
 static skx_show_retry_log_f skx_show_retry_rd_err_log;
 static u64 skx_tolm, skx_tohm;
 static LIST_HEAD(dev_edac_list);
@@ -139,10 +141,13 @@ static bool skx_adxl_decode(struct decoded_addr *res, bool error_in_1st_level_me
 			       (int)adxl_values[component_indices[INDEX_NM_CHANNEL]] : -1;
 		res->dimm    = (adxl_nm_bitmap & BIT_NM_DIMM) ?
 			       (int)adxl_values[component_indices[INDEX_NM_DIMM]] : -1;
+		res->cs      = (adxl_nm_bitmap & BIT_NM_CS) ?
+			       (int)adxl_values[component_indices[INDEX_NM_CS]] : -1;
 	} else {
 		res->imc     = (int)adxl_values[component_indices[INDEX_MEMCTRL]];
 		res->channel = (int)adxl_values[component_indices[INDEX_CHANNEL]];
 		res->dimm    = (int)adxl_values[component_indices[INDEX_DIMM]];
+		res->cs      = (int)adxl_values[component_indices[INDEX_CS]];
 	}
 
 	if (res->imc > NUM_IMC - 1 || res->imc < 0) {
@@ -173,6 +178,8 @@ static bool skx_adxl_decode(struct decoded_addr *res, bool error_in_1st_level_me
 			break;
 	}
 
+	res->decoded_by_adxl = true;
+
 	return true;
 }
 
@@ -183,7 +190,7 @@ void skx_set_mem_cfg(bool mem_cfg_2lm)
 
 void skx_set_decode(skx_decode_f decode, skx_show_retry_log_f show_retry_log)
 {
-	skx_decode = decode;
+	driver_decode = decode;
 	skx_show_retry_rd_err_log = show_retry_log;
 }
 
@@ -591,19 +598,19 @@ static void skx_mce_output_error(struct mem_ctl_info *mci,
 			break;
 		}
 	}
-	if (adxl_component_count) {
+	if (res->decoded_by_adxl) {
 		len = snprintf(skx_msg, MSG_SIZE, "%s%s err_code:0x%04x:0x%04x %s",
 			 overflow ? " OVERFLOW" : "",
 			 (uncorrected_error && recoverable) ? " recoverable" : "",
 			 mscod, errcode, adxl_msg);
 	} else {
 		len = snprintf(skx_msg, MSG_SIZE,
-			 "%s%s err_code:0x%04x:0x%04x socket:%d imc:%d rank:%d bg:%d ba:%d row:0x%x col:0x%x",
+			 "%s%s err_code:0x%04x:0x%04x ProcessorSocketId:0x%x MemoryControllerId:0x%x PhysicalRankId:0x%x Row:0x%x Column:0x%x Bank:0x%x BankGroup:0x%x",
 			 overflow ? " OVERFLOW" : "",
 			 (uncorrected_error && recoverable) ? " recoverable" : "",
 			 mscod, errcode,
 			 res->socket, res->imc, res->rank,
-			 res->bank_group, res->bank_address, res->row, res->column);
+			 res->row, res->column, res->bank_address, res->bank_group);
 	}
 
 	if (skx_show_retry_rd_err_log)
@@ -649,13 +656,14 @@ int skx_mce_check_error(struct notifier_block *nb, unsigned long val,
 		return NOTIFY_DONE;
 
 	memset(&res, 0, sizeof(res));
+	res.mce  = mce;
 	res.addr = mce->addr;
 
-	if (adxl_component_count) {
-		if (!skx_adxl_decode(&res, skx_error_in_1st_level_mem(mce)))
+	/* Try driver decoder first */
+	if (!(driver_decode && driver_decode(&res))) {
+		/* Then try firmware decoder (ACPI DSM methods) */
+		if (!(adxl_component_count && skx_adxl_decode(&res, skx_error_in_1st_level_mem(mce))))
 			return NOTIFY_DONE;
-	} else if (!skx_decode || !skx_decode(&res)) {
-		return NOTIFY_DONE;
 	}
 
 	mci = res.dev->imc[res.imc].mci;
diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h
index 03ac067a80b9..0cbadd3d2cd3 100644
--- a/drivers/edac/skx_common.h
+++ b/drivers/edac/skx_common.h
@@ -10,6 +10,7 @@
 #define _SKX_COMM_EDAC_H
 
 #include <linux/bits.h>
+#include <asm/mce.h>
 
 #define MSG_SIZE		1024
 
@@ -52,6 +53,9 @@
 #define IS_DIMM_PRESENT(r)		GET_BITFIELD(r, 15, 15)
 #define IS_NVDIMM_PRESENT(r, i)		GET_BITFIELD(r, i, i)
 
+#define MCI_MISC_ECC_MODE(m)	(((m) >> 59) & 15)
+#define MCI_MISC_ECC_DDRT	8	/* read from DDRT */
+
 /*
  * Each cpu socket contains some pci devices that provide global
  * information, and also some that are local to each of the two
@@ -82,6 +86,7 @@ struct skx_dev {
 			struct pci_dev	*edev;
 			u32 retry_rd_err_log_s;
 			u32 retry_rd_err_log_d;
+			u32 retry_rd_err_log_d2;
 			struct skx_dimm {
 				u8 close_pg;
 				u8 bank_xor_enable;
@@ -108,18 +113,22 @@ enum {
 	INDEX_MEMCTRL,
 	INDEX_CHANNEL,
 	INDEX_DIMM,
+	INDEX_CS,
 	INDEX_NM_FIRST,
 	INDEX_NM_MEMCTRL = INDEX_NM_FIRST,
 	INDEX_NM_CHANNEL,
 	INDEX_NM_DIMM,
+	INDEX_NM_CS,
 	INDEX_MAX
 };
 
 #define BIT_NM_MEMCTRL	BIT_ULL(INDEX_NM_MEMCTRL)
 #define BIT_NM_CHANNEL	BIT_ULL(INDEX_NM_CHANNEL)
 #define BIT_NM_DIMM	BIT_ULL(INDEX_NM_DIMM)
+#define BIT_NM_CS	BIT_ULL(INDEX_NM_CS)
 
 struct decoded_addr {
+	struct mce *mce;
 	struct skx_dev *dev;
 	u64	addr;
 	int	socket;
@@ -129,6 +138,7 @@ struct decoded_addr {
 	int	sktways;
 	int	chanways;
 	int	dimm;
+	int	cs;
 	int	rank;
 	int	channel_rank;
 	u64	rank_address;
@@ -136,6 +146,7 @@ struct decoded_addr {
 	int	column;
 	int	bank_address;
 	int	bank_group;
+	bool	decoded_by_adxl;
 };
 
 struct res_config {
@@ -154,7 +165,12 @@ struct res_config {
 	int sad_all_offset;
 	/* Offsets of retry_rd_err_log registers */
 	u32 *offsets_scrub;
+	u32 *offsets_scrub_hbm0;
+	u32 *offsets_scrub_hbm1;
 	u32 *offsets_demand;
+	u32 *offsets_demand2;
+	u32 *offsets_demand_hbm0;
+	u32 *offsets_demand_hbm1;
 };
 
 typedef int (*get_dimm_config_f)(struct mem_ctl_info *mci,
diff --git a/drivers/edac/wq.c b/drivers/edac/wq.c
index d021d287eaec..ad3f516627c5 100644
--- a/drivers/edac/wq.c
+++ b/drivers/edac/wq.c
@@ -37,7 +37,6 @@ int edac_workqueue_setup(void)
 
 void edac_workqueue_teardown(void)
 {
-	flush_workqueue(wq);
 	destroy_workqueue(wq);
 	wq = NULL;
 }
diff --git a/drivers/extcon/extcon-rt8973a.c b/drivers/extcon/extcon-rt8973a.c
index 02ba770acb27..e6e448f6ea2f 100644
--- a/drivers/extcon/extcon-rt8973a.c
+++ b/drivers/extcon/extcon-rt8973a.c
@@ -646,13 +646,11 @@ static int rt8973a_muic_i2c_probe(struct i2c_client *i2c,
 	return 0;
 }
 
-static int rt8973a_muic_i2c_remove(struct i2c_client *i2c)
+static void rt8973a_muic_i2c_remove(struct i2c_client *i2c)
 {
 	struct rt8973a_muic_info *info = i2c_get_clientdata(i2c);
 
 	regmap_del_irq_chip(info->irq, info->irq_data);
-
-	return 0;
 }
 
 static const struct of_device_id rt8973a_dt_match[] = {
diff --git a/drivers/firmware/arm_ffa/bus.c b/drivers/firmware/arm_ffa/bus.c
index 641a91819088..99d439480612 100644
--- a/drivers/firmware/arm_ffa/bus.c
+++ b/drivers/firmware/arm_ffa/bus.c
@@ -167,7 +167,8 @@ bool ffa_device_is_valid(struct ffa_device *ffa_dev)
 	return valid;
 }
 
-struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id)
+struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id,
+				       const struct ffa_ops *ops)
 {
 	int ret;
 	struct device *dev;
@@ -183,6 +184,7 @@ struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id)
 	dev_set_name(&ffa_dev->dev, "arm-ffa-%04x", vm_id);
 
 	ffa_dev->vm_id = vm_id;
+	ffa_dev->ops = ops;
 	uuid_copy(&ffa_dev->uuid, uuid);
 
 	ret = device_register(&ffa_dev->dev);
diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c
index ec731e9e942b..d5e86ef40b89 100644
--- a/drivers/firmware/arm_ffa/driver.c
+++ b/drivers/firmware/arm_ffa/driver.c
@@ -163,6 +163,7 @@ struct ffa_drv_info {
 	struct mutex tx_lock; /* lock to protect Tx buffer */
 	void *rx_buffer;
 	void *tx_buffer;
+	bool mem_ops_native;
 };
 
 static struct ffa_drv_info *drv_info;
@@ -263,18 +264,24 @@ static int ffa_rxtx_unmap(u16 vm_id)
 	return 0;
 }
 
+#define PARTITION_INFO_GET_RETURN_COUNT_ONLY	BIT(0)
+
 /* buffer must be sizeof(struct ffa_partition_info) * num_partitions */
 static int
 __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
 			 struct ffa_partition_info *buffer, int num_partitions)
 {
-	int count;
+	int idx, count, flags = 0, sz, buf_sz;
 	ffa_value_t partition_info;
 
+	if (!buffer || !num_partitions) /* Just get the count for now */
+		flags = PARTITION_INFO_GET_RETURN_COUNT_ONLY;
+
 	mutex_lock(&drv_info->rx_lock);
 	invoke_ffa_fn((ffa_value_t){
 		      .a0 = FFA_PARTITION_INFO_GET,
 		      .a1 = uuid0, .a2 = uuid1, .a3 = uuid2, .a4 = uuid3,
+		      .a5 = flags,
 		      }, &partition_info);
 
 	if (partition_info.a0 == FFA_ERROR) {
@@ -284,8 +291,19 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
 
 	count = partition_info.a2;
 
+	if (drv_info->version > FFA_VERSION_1_0) {
+		buf_sz = sz = partition_info.a3;
+		if (sz > sizeof(*buffer))
+			buf_sz = sizeof(*buffer);
+	} else {
+		/* FFA_VERSION_1_0 lacks size in the response */
+		buf_sz = sz = 8;
+	}
+
 	if (buffer && count <= num_partitions)
-		memcpy(buffer, drv_info->rx_buffer, sizeof(*buffer) * count);
+		for (idx = 0; idx < count; idx++)
+			memcpy(buffer + idx, drv_info->rx_buffer + idx * sz,
+			       buf_sz);
 
 	ffa_rx_release();
 
@@ -571,6 +589,39 @@ static int ffa_memory_reclaim(u64 g_handle, u32 flags)
 	return 0;
 }
 
+static int ffa_features(u32 func_feat_id, u32 input_props,
+			u32 *if_props_1, u32 *if_props_2)
+{
+	ffa_value_t id;
+
+	if (!ARM_SMCCC_IS_FAST_CALL(func_feat_id) && input_props) {
+		pr_err("%s: Invalid Parameters: %x, %x", __func__,
+		       func_feat_id, input_props);
+		return ffa_to_linux_errno(FFA_RET_INVALID_PARAMETERS);
+	}
+
+	invoke_ffa_fn((ffa_value_t){
+		.a0 = FFA_FEATURES, .a1 = func_feat_id, .a2 = input_props,
+		}, &id);
+
+	if (id.a0 == FFA_ERROR)
+		return ffa_to_linux_errno((int)id.a2);
+
+	if (if_props_1)
+		*if_props_1 = id.a2;
+	if (if_props_2)
+		*if_props_2 = id.a3;
+
+	return 0;
+}
+
+static void ffa_set_up_mem_ops_native_flag(void)
+{
+	if (!ffa_features(FFA_FN_NATIVE(MEM_LEND), 0, NULL, NULL) ||
+	    !ffa_features(FFA_FN_NATIVE(MEM_SHARE), 0, NULL, NULL))
+		drv_info->mem_ops_native = true;
+}
+
 static u32 ffa_api_version_get(void)
 {
 	return drv_info->version;
@@ -597,11 +648,19 @@ static int ffa_partition_info_get(const char *uuid_str,
 	return 0;
 }
 
-static void ffa_mode_32bit_set(struct ffa_device *dev)
+static void _ffa_mode_32bit_set(struct ffa_device *dev)
 {
 	dev->mode_32bit = true;
 }
 
+static void ffa_mode_32bit_set(struct ffa_device *dev)
+{
+	if (drv_info->version > FFA_VERSION_1_0)
+		return;
+
+	_ffa_mode_32bit_set(dev);
+}
+
 static int ffa_sync_send_receive(struct ffa_device *dev,
 				 struct ffa_send_direct_data *data)
 {
@@ -609,17 +668,15 @@ static int ffa_sync_send_receive(struct ffa_device *dev,
 				       dev->mode_32bit, data);
 }
 
-static int
-ffa_memory_share(struct ffa_device *dev, struct ffa_mem_ops_args *args)
+static int ffa_memory_share(struct ffa_mem_ops_args *args)
 {
-	if (dev->mode_32bit)
-		return ffa_memory_ops(FFA_MEM_SHARE, args);
+	if (drv_info->mem_ops_native)
+		return ffa_memory_ops(FFA_FN_NATIVE(MEM_SHARE), args);
 
-	return ffa_memory_ops(FFA_FN_NATIVE(MEM_SHARE), args);
+	return ffa_memory_ops(FFA_MEM_SHARE, args);
 }
 
-static int
-ffa_memory_lend(struct ffa_device *dev, struct ffa_mem_ops_args *args)
+static int ffa_memory_lend(struct ffa_mem_ops_args *args)
 {
 	/* Note that upon a successful MEM_LEND request the caller
 	 * must ensure that the memory region specified is not accessed
@@ -628,36 +685,47 @@ ffa_memory_lend(struct ffa_device *dev, struct ffa_mem_ops_args *args)
 	 * however on systems without a hypervisor the responsibility
 	 * falls to the calling kernel driver to prevent access.
 	 */
-	if (dev->mode_32bit)
-		return ffa_memory_ops(FFA_MEM_LEND, args);
+	if (drv_info->mem_ops_native)
+		return ffa_memory_ops(FFA_FN_NATIVE(MEM_LEND), args);
 
-	return ffa_memory_ops(FFA_FN_NATIVE(MEM_LEND), args);
+	return ffa_memory_ops(FFA_MEM_LEND, args);
 }
 
-static const struct ffa_dev_ops ffa_ops = {
+static const struct ffa_info_ops ffa_drv_info_ops = {
 	.api_version_get = ffa_api_version_get,
 	.partition_info_get = ffa_partition_info_get,
+};
+
+static const struct ffa_msg_ops ffa_drv_msg_ops = {
 	.mode_32bit_set = ffa_mode_32bit_set,
 	.sync_send_receive = ffa_sync_send_receive,
+};
+
+static const struct ffa_mem_ops ffa_drv_mem_ops = {
 	.memory_reclaim = ffa_memory_reclaim,
 	.memory_share = ffa_memory_share,
 	.memory_lend = ffa_memory_lend,
 };
 
-const struct ffa_dev_ops *ffa_dev_ops_get(struct ffa_device *dev)
-{
-	if (ffa_device_is_valid(dev))
-		return &ffa_ops;
-
-	return NULL;
-}
-EXPORT_SYMBOL_GPL(ffa_dev_ops_get);
+static const struct ffa_ops ffa_drv_ops = {
+	.info_ops = &ffa_drv_info_ops,
+	.msg_ops = &ffa_drv_msg_ops,
+	.mem_ops = &ffa_drv_mem_ops,
+};
 
 void ffa_device_match_uuid(struct ffa_device *ffa_dev, const uuid_t *uuid)
 {
 	int count, idx;
 	struct ffa_partition_info *pbuf, *tpbuf;
 
+	/*
+	 * FF-A v1.1 provides UUID for each partition as part of the discovery
+	 * API, the discovered UUID must be populated in the device's UUID and
+	 * there is no need to copy the same from the driver table.
+	 */
+	if (drv_info->version > FFA_VERSION_1_0)
+		return;
+
 	count = ffa_partition_probe(uuid, &pbuf);
 	if (count <= 0)
 		return;
@@ -671,6 +739,7 @@ void ffa_device_match_uuid(struct ffa_device *ffa_dev, const uuid_t *uuid)
 static void ffa_setup_partitions(void)
 {
 	int count, idx;
+	uuid_t uuid;
 	struct ffa_device *ffa_dev;
 	struct ffa_partition_info *pbuf, *tpbuf;
 
@@ -681,19 +750,24 @@ static void ffa_setup_partitions(void)
 	}
 
 	for (idx = 0, tpbuf = pbuf; idx < count; idx++, tpbuf++) {
-		/* Note that the &uuid_null parameter will require
+		import_uuid(&uuid, (u8 *)tpbuf->uuid);
+
+		/* Note that if the UUID will be uuid_null, that will require
 		 * ffa_device_match() to find the UUID of this partition id
-		 * with help of ffa_device_match_uuid(). Once the FF-A spec
-		 * is updated to provide correct UUID here for each partition
-		 * as part of the discovery API, we need to pass the
-		 * discovered UUID here instead.
+		 * with help of ffa_device_match_uuid(). FF-A v1.1 and above
+		 * provides UUID here for each partition as part of the
+		 * discovery API and the same is passed.
 		 */
-		ffa_dev = ffa_device_register(&uuid_null, tpbuf->id);
+		ffa_dev = ffa_device_register(&uuid, tpbuf->id, &ffa_drv_ops);
 		if (!ffa_dev) {
 			pr_err("%s: failed to register partition ID 0x%x\n",
 			       __func__, tpbuf->id);
 			continue;
 		}
+
+		if (drv_info->version > FFA_VERSION_1_0 &&
+		    !(tpbuf->properties & FFA_PARTITION_AARCH64_EXEC))
+			_ffa_mode_32bit_set(ffa_dev);
 	}
 	kfree(pbuf);
 }
@@ -751,6 +825,8 @@ static int __init ffa_init(void)
 
 	ffa_setup_partitions();
 
+	ffa_set_up_mem_ops_native_flag();
+
 	return 0;
 free_pages:
 	if (drv_info->tx_buffer)
diff --git a/drivers/firmware/arm_scmi/scmi_pm_domain.c b/drivers/firmware/arm_scmi/scmi_pm_domain.c
index 4e27c3d66a83..0e05a79de82d 100644
--- a/drivers/firmware/arm_scmi/scmi_pm_domain.c
+++ b/drivers/firmware/arm_scmi/scmi_pm_domain.c
@@ -8,7 +8,6 @@
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/module.h>
-#include <linux/pm_clock.h>
 #include <linux/pm_domain.h>
 #include <linux/scmi_protocol.h>
 
@@ -53,27 +52,6 @@ static int scmi_pd_power_off(struct generic_pm_domain *domain)
 	return scmi_pd_power(domain, false);
 }
 
-static int scmi_pd_attach_dev(struct generic_pm_domain *pd, struct device *dev)
-{
-	int ret;
-
-	ret = pm_clk_create(dev);
-	if (ret)
-		return ret;
-
-	ret = of_pm_clk_add_clks(dev);
-	if (ret >= 0)
-		return 0;
-
-	pm_clk_destroy(dev);
-	return ret;
-}
-
-static void scmi_pd_detach_dev(struct generic_pm_domain *pd, struct device *dev)
-{
-	pm_clk_destroy(dev);
-}
-
 static int scmi_pm_domain_probe(struct scmi_device *sdev)
 {
 	int num_domains, i;
@@ -124,10 +102,6 @@ static int scmi_pm_domain_probe(struct scmi_device *sdev)
 		scmi_pd->genpd.name = scmi_pd->name;
 		scmi_pd->genpd.power_off = scmi_pd_power_off;
 		scmi_pd->genpd.power_on = scmi_pd_power_on;
-		scmi_pd->genpd.attach_dev = scmi_pd_attach_dev;
-		scmi_pd->genpd.detach_dev = scmi_pd_detach_dev;
-		scmi_pd->genpd.flags = GENPD_FLAG_PM_CLK |
-				       GENPD_FLAG_ACTIVE_WAKEUP;
 
 		pm_genpd_init(&scmi_pd->genpd, NULL,
 			      state == SCMI_POWER_STATE_GENERIC_OFF);
diff --git a/drivers/firmware/efi/dev-path-parser.c b/drivers/firmware/efi/dev-path-parser.c
index eb9c65f97841..f80d87c199c3 100644
--- a/drivers/firmware/efi/dev-path-parser.c
+++ b/drivers/firmware/efi/dev-path-parser.c
@@ -15,9 +15,11 @@
 static long __init parse_acpi_path(const struct efi_dev_path *node,
 				   struct device *parent, struct device **child)
 {
-	char hid[ACPI_ID_LEN], uid[11]; /* UINT_MAX + null byte */
 	struct acpi_device *adev;
 	struct device *phys_dev;
+	char hid[ACPI_ID_LEN];
+	u64 uid;
+	int ret;
 
 	if (node->header.length != 12)
 		return -EINVAL;
@@ -27,12 +29,12 @@ static long __init parse_acpi_path(const struct efi_dev_path *node,
 		'A' + ((node->acpi.hid >>  5) & 0x1f) - 1,
 		'A' + ((node->acpi.hid >>  0) & 0x1f) - 1,
 			node->acpi.hid >> 16);
-	sprintf(uid, "%u", node->acpi.uid);
 
 	for_each_acpi_dev_match(adev, hid, NULL, -1) {
-		if (adev->pnp.unique_id && !strcmp(adev->pnp.unique_id, uid))
+		ret = acpi_dev_uid_to_integer(adev, &uid);
+		if (ret == 0 && node->acpi.uid == uid)
 			break;
-		if (!adev->pnp.unique_id && node->acpi.uid == 0)
+		if (ret == -ENODATA && node->acpi.uid == 0)
 			break;
 	}
 	if (!adev)
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index 2c67f71f2375..b43fdb319fd4 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -46,6 +46,8 @@ KBUILD_CFLAGS := $(filter-out $(RANDSTRUCT_CFLAGS), $(KBUILD_CFLAGS))
 
 # remove SCS flags from all objects in this directory
 KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
+# disable CFI
+KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_CFI), $(KBUILD_CFLAGS))
 # disable LTO
 KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_LTO), $(KBUILD_CFLAGS))
 
diff --git a/drivers/firmware/efi/libstub/arm64-stub.c b/drivers/firmware/efi/libstub/arm64-stub.c
index 577173ee1f83..60973e84d7ab 100644
--- a/drivers/firmware/efi/libstub/arm64-stub.c
+++ b/drivers/firmware/efi/libstub/arm64-stub.c
@@ -23,8 +23,8 @@ efi_status_t check_platform_features(void)
 	if (IS_ENABLED(CONFIG_ARM64_4K_PAGES))
 		return EFI_SUCCESS;
 
-	tg = (read_cpuid(ID_AA64MMFR0_EL1) >> ID_AA64MMFR0_TGRAN_SHIFT) & 0xf;
-	if (tg < ID_AA64MMFR0_TGRAN_SUPPORTED_MIN || tg > ID_AA64MMFR0_TGRAN_SUPPORTED_MAX) {
+	tg = (read_cpuid(ID_AA64MMFR0_EL1) >> ID_AA64MMFR0_EL1_TGRAN_SHIFT) & 0xf;
+	if (tg < ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MIN || tg > ID_AA64MMFR0_EL1_TGRAN_SUPPORTED_MAX) {
 		if (IS_ENABLED(CONFIG_ARM64_64K_PAGES))
 			efi_err("This 64 KB granular kernel is not supported by your CPU\n");
 		else
diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
index cfb448eabdaa..e7bcfca4159f 100644
--- a/drivers/firmware/psci/psci.c
+++ b/drivers/firmware/psci/psci.c
@@ -9,6 +9,7 @@
 #include <linux/acpi.h>
 #include <linux/arm-smccc.h>
 #include <linux/cpuidle.h>
+#include <linux/debugfs.h>
 #include <linux/errno.h>
 #include <linux/linkage.h>
 #include <linux/of.h>
@@ -163,6 +164,8 @@ int psci_set_osi_mode(bool enable)
 			PSCI_1_0_SUSPEND_MODE_PC;
 
 	err = invoke_psci_fn(PSCI_1_0_FN_SET_SUSPEND_MODE, suspend_mode, 0, 0);
+	if (err < 0)
+		pr_warn("failed to set %s mode: %d\n", enable ? "OSI" : "PC", err);
 	return psci_to_linux_errno(err);
 }
 
@@ -274,7 +277,7 @@ static void set_conduit(enum arm_smccc_conduit conduit)
 	psci_conduit = conduit;
 }
 
-static int get_set_conduit_method(struct device_node *np)
+static int get_set_conduit_method(const struct device_node *np)
 {
 	const char *method;
 
@@ -324,17 +327,130 @@ static void psci_sys_poweroff(void)
 	invoke_psci_fn(PSCI_0_2_FN_SYSTEM_OFF, 0, 0, 0);
 }
 
-static int __init psci_features(u32 psci_func_id)
+static int psci_features(u32 psci_func_id)
 {
 	return invoke_psci_fn(PSCI_1_0_FN_PSCI_FEATURES,
 			      psci_func_id, 0, 0);
 }
 
+#ifdef CONFIG_DEBUG_FS
+
+#define PSCI_ID(ver, _name) \
+	{ .fn = PSCI_##ver##_FN_##_name, .name = #_name, }
+#define PSCI_ID_NATIVE(ver, _name) \
+	{ .fn = PSCI_FN_NATIVE(ver, _name), .name = #_name, }
+
+/* A table of all optional functions */
+static const struct {
+	u32 fn;
+	const char *name;
+} psci_fn_ids[] = {
+	PSCI_ID_NATIVE(0_2, MIGRATE),
+	PSCI_ID(0_2, MIGRATE_INFO_TYPE),
+	PSCI_ID_NATIVE(0_2, MIGRATE_INFO_UP_CPU),
+	PSCI_ID(1_0, CPU_FREEZE),
+	PSCI_ID_NATIVE(1_0, CPU_DEFAULT_SUSPEND),
+	PSCI_ID_NATIVE(1_0, NODE_HW_STATE),
+	PSCI_ID_NATIVE(1_0, SYSTEM_SUSPEND),
+	PSCI_ID(1_0, SET_SUSPEND_MODE),
+	PSCI_ID_NATIVE(1_0, STAT_RESIDENCY),
+	PSCI_ID_NATIVE(1_0, STAT_COUNT),
+	PSCI_ID_NATIVE(1_1, SYSTEM_RESET2),
+	PSCI_ID(1_1, MEM_PROTECT),
+	PSCI_ID_NATIVE(1_1, MEM_PROTECT_CHECK_RANGE),
+};
+
+static int psci_debugfs_read(struct seq_file *s, void *data)
+{
+	int feature, type, i;
+	u32 ver;
+
+	ver = psci_ops.get_version();
+	seq_printf(s, "PSCIv%d.%d\n",
+		   PSCI_VERSION_MAJOR(ver),
+		   PSCI_VERSION_MINOR(ver));
+
+	/* PSCI_FEATURES is available only starting from 1.0 */
+	if (PSCI_VERSION_MAJOR(ver) < 1)
+		return 0;
+
+	feature = psci_features(ARM_SMCCC_VERSION_FUNC_ID);
+	if (feature != PSCI_RET_NOT_SUPPORTED) {
+		ver = invoke_psci_fn(ARM_SMCCC_VERSION_FUNC_ID, 0, 0, 0);
+		seq_printf(s, "SMC Calling Convention v%d.%d\n",
+			   PSCI_VERSION_MAJOR(ver),
+			   PSCI_VERSION_MINOR(ver));
+	} else {
+		seq_puts(s, "SMC Calling Convention v1.0 is assumed\n");
+	}
+
+	feature = psci_features(PSCI_FN_NATIVE(0_2, CPU_SUSPEND));
+	if (feature < 0) {
+		seq_printf(s, "PSCI_FEATURES(CPU_SUSPEND) error (%d)\n", feature);
+	} else {
+		seq_printf(s, "OSI is %ssupported\n",
+			   (feature & BIT(0)) ? "" : "not ");
+		seq_printf(s, "%s StateID format is used\n",
+			   (feature & BIT(1)) ? "Extended" : "Original");
+	}
+
+	type = psci_ops.migrate_info_type();
+	if (type == PSCI_0_2_TOS_UP_MIGRATE ||
+	    type == PSCI_0_2_TOS_UP_NO_MIGRATE) {
+		unsigned long cpuid;
+
+		seq_printf(s, "Trusted OS %smigrate capable\n",
+			   type == PSCI_0_2_TOS_UP_NO_MIGRATE ? "not " : "");
+		cpuid = psci_migrate_info_up_cpu();
+		seq_printf(s, "Trusted OS resident on physical CPU 0x%lx (#%d)\n",
+			   cpuid, resident_cpu);
+	} else if (type == PSCI_0_2_TOS_MP) {
+		seq_puts(s, "Trusted OS migration not required\n");
+	} else {
+		if (type != PSCI_RET_NOT_SUPPORTED)
+			seq_printf(s, "MIGRATE_INFO_TYPE returned unknown type (%d)\n", type);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(psci_fn_ids); i++) {
+		feature = psci_features(psci_fn_ids[i].fn);
+		if (feature == PSCI_RET_NOT_SUPPORTED)
+			continue;
+		if (feature < 0)
+			seq_printf(s, "PSCI_FEATURES(%s) error (%d)\n",
+				   psci_fn_ids[i].name, feature);
+		else
+			seq_printf(s, "%s is supported\n", psci_fn_ids[i].name);
+	}
+
+	return 0;
+}
+
+static int psci_debugfs_open(struct inode *inode, struct file *f)
+{
+	return single_open(f, psci_debugfs_read, NULL);
+}
+
+static const struct file_operations psci_debugfs_ops = {
+	.owner = THIS_MODULE,
+	.open = psci_debugfs_open,
+	.release = single_release,
+	.read = seq_read,
+	.llseek = seq_lseek
+};
+
+static int __init psci_debugfs_init(void)
+{
+	return PTR_ERR_OR_ZERO(debugfs_create_file("psci", 0444, NULL, NULL,
+						   &psci_debugfs_ops));
+}
+late_initcall(psci_debugfs_init)
+#endif
+
 #ifdef CONFIG_CPU_IDLE
 static int psci_suspend_finisher(unsigned long state)
 {
 	u32 power_state = state;
-	phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume));
+	phys_addr_t pa_cpu_resume = __pa_symbol(cpu_resume);
 
 	return psci_ops.cpu_suspend(power_state, pa_cpu_resume);
 }
@@ -359,7 +475,7 @@ int psci_cpu_suspend_enter(u32 state)
 
 static int psci_system_suspend(unsigned long unused)
 {
-	phys_addr_t pa_cpu_resume = __pa_symbol(function_nocfi(cpu_resume));
+	phys_addr_t pa_cpu_resume = __pa_symbol(cpu_resume);
 
 	return invoke_psci_fn(PSCI_FN_NATIVE(1_0, SYSTEM_SUSPEND),
 			      pa_cpu_resume, 0, 0);
@@ -528,7 +644,7 @@ typedef int (*psci_initcall_t)(const struct device_node *);
  *
  * Probe based on PSCI PSCI_VERSION function
  */
-static int __init psci_0_2_init(struct device_node *np)
+static int __init psci_0_2_init(const struct device_node *np)
 {
 	int err;
 
@@ -549,7 +665,7 @@ static int __init psci_0_2_init(struct device_node *np)
 /*
  * PSCI < v0.2 get PSCI Function IDs via DT.
  */
-static int __init psci_0_1_init(struct device_node *np)
+static int __init psci_0_1_init(const struct device_node *np)
 {
 	u32 id;
 	int err;
@@ -585,7 +701,7 @@ static int __init psci_0_1_init(struct device_node *np)
 	return 0;
 }
 
-static int __init psci_1_0_init(struct device_node *np)
+static int __init psci_1_0_init(const struct device_node *np)
 {
 	int err;
 
diff --git a/drivers/firmware/qcom_scm.h b/drivers/firmware/qcom_scm.h
index 0d51eef2472f..db3d08a01209 100644
--- a/drivers/firmware/qcom_scm.h
+++ b/drivers/firmware/qcom_scm.h
@@ -129,8 +129,6 @@ extern int scm_legacy_call(struct device *dev, const struct qcom_scm_desc *desc,
 #define QCOM_SCM_SMMU_CONFIG_ERRATA1		0x03
 #define QCOM_SCM_SMMU_CONFIG_ERRATA1_CLIENT_ALL	0x02
 
-extern void __qcom_scm_init(void);
-
 /* common error codes */
 #define QCOM_SCM_V2_EBUSY	-12
 #define QCOM_SCM_ENOMEM		-5
diff --git a/drivers/firmware/sysfb.c b/drivers/firmware/sysfb.c
index 1f276f108cc9..3fd3563d962b 100644
--- a/drivers/firmware/sysfb.c
+++ b/drivers/firmware/sysfb.c
@@ -94,6 +94,10 @@ static __init int sysfb_init(void)
 		name = "efi-framebuffer";
 	else if (si->orig_video_isVGA == VIDEO_TYPE_VLFB)
 		name = "vesa-framebuffer";
+	else if (si->orig_video_isVGA == VIDEO_TYPE_VGAC)
+		name = "vga-framebuffer";
+	else if (si->orig_video_isVGA == VIDEO_TYPE_EGAC)
+		name = "ega-framebuffer";
 	else
 		name = "platform-framebuffer";
 
diff --git a/drivers/firmware/tegra/bpmp-debugfs.c b/drivers/firmware/tegra/bpmp-debugfs.c
index 0c440afd5224..9d3874cdaaee 100644
--- a/drivers/firmware/tegra/bpmp-debugfs.c
+++ b/drivers/firmware/tegra/bpmp-debugfs.c
@@ -377,18 +377,11 @@ static ssize_t bpmp_debug_store(struct file *file, const char __user *buf,
 	if (!filename)
 		return -ENOENT;
 
-	databuf = kmalloc(count, GFP_KERNEL);
-	if (!databuf)
-		return -ENOMEM;
-
-	if (copy_from_user(databuf, buf, count)) {
-		err = -EFAULT;
-		goto free_ret;
-	}
+	databuf = memdup_user(buf, count);
+	if (IS_ERR(databuf))
+		return PTR_ERR(databuf);
 
 	err = mrq_debug_write(bpmp, filename, databuf, count);
-
-free_ret:
 	kfree(databuf);
 
 	return err ?: count;
diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c
index d1f652802181..ff5cabe70a2b 100644
--- a/drivers/firmware/xilinx/zynqmp.c
+++ b/drivers/firmware/xilinx/zynqmp.c
@@ -1312,6 +1312,37 @@ int zynqmp_pm_get_feature_config(enum pm_feature_config_id id,
 }
 
 /**
+ * zynqmp_pm_set_sd_config - PM call to set value of SD config registers
+ * @node:	SD node ID
+ * @config:	The config type of SD registers
+ * @value:	Value to be set
+ *
+ * Return:	Returns 0 on success or error value on failure.
+ */
+int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, node, IOCTL_SET_SD_CONFIG,
+				   config, value, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_sd_config);
+
+/**
+ * zynqmp_pm_set_gem_config - PM call to set value of GEM config registers
+ * @node:	GEM node ID
+ * @config:	The config type of GEM registers
+ * @value:	Value to be set
+ *
+ * Return:	Returns 0 on success or error value on failure.
+ */
+int zynqmp_pm_set_gem_config(u32 node, enum pm_gem_config_type config,
+			     u32 value)
+{
+	return zynqmp_pm_invoke_fn(PM_IOCTL, node, IOCTL_SET_GEM_CONFIG,
+				   config, value, NULL);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_set_gem_config);
+
+/**
  * struct zynqmp_pm_shutdown_scope - Struct for shutdown scope
  * @subtype:	Shutdown subtype
  * @name:	Matching string for scope argument
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index 0642f579196f..3f64345fe40b 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -874,10 +874,11 @@ config GPIO_104_IDI_48
 	  module parameter.
 
 config GPIO_F7188X
-	tristate "F71869, F71869A, F71882FG, F71889F and F81866 GPIO support"
+	tristate "Fintek and Nuvoton Super-I/O GPIO support"
 	help
 	  This option enables support for GPIOs found on Fintek Super-I/O
 	  chips F71869, F71869A, F71882FG, F71889F and F81866.
+	  As well as Nuvoton Super-I/O chip NCT6116D.
 
 	  To compile this driver as a module, choose M here: the module will
 	  be called f7188x-gpio.
diff --git a/drivers/gpio/gpio-adp5588.c b/drivers/gpio/gpio-adp5588.c
index d49f12560cde..9b562dbbd733 100644
--- a/drivers/gpio/gpio-adp5588.c
+++ b/drivers/gpio/gpio-adp5588.c
@@ -409,14 +409,12 @@ static int adp5588_gpio_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int adp5588_gpio_remove(struct i2c_client *client)
+static void adp5588_gpio_remove(struct i2c_client *client)
 {
 	struct adp5588_gpio *dev = i2c_get_clientdata(client);
 
 	if (dev->client->irq)
 		free_irq(dev->client->irq, dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id adp5588_gpio_id[] = {
diff --git a/drivers/gpio/gpio-f7188x.c b/drivers/gpio/gpio-f7188x.c
index 18a3147f5a42..9effa7769bef 100644
--- a/drivers/gpio/gpio-f7188x.c
+++ b/drivers/gpio/gpio-f7188x.c
@@ -1,12 +1,15 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * GPIO driver for Fintek Super-I/O F71869, F71869A, F71882, F71889 and F81866
+ * GPIO driver for Fintek and Nuvoton Super-I/O chips
  *
  * Copyright (C) 2010-2013 LaCie
  *
  * Author: Simon Guinot <simon.guinot@sequanux.org>
  */
 
+#define DRVNAME "gpio-f7188x"
+#define pr_fmt(fmt) DRVNAME ": " fmt
+
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/platform_device.h>
@@ -14,30 +17,41 @@
 #include <linux/gpio/driver.h>
 #include <linux/bitops.h>
 
-#define DRVNAME "gpio-f7188x"
-
 /*
  * Super-I/O registers
  */
 #define SIO_LDSEL		0x07	/* Logical device select */
 #define SIO_DEVID		0x20	/* Device ID (2 bytes) */
-#define SIO_DEVREV		0x22	/* Device revision */
-#define SIO_MANID		0x23	/* Fintek ID (2 bytes) */
 
-#define SIO_LD_GPIO		0x06	/* GPIO logical device */
 #define SIO_UNLOCK_KEY		0x87	/* Key to enable Super-I/O */
 #define SIO_LOCK_KEY		0xAA	/* Key to disable Super-I/O */
 
-#define SIO_FINTEK_ID		0x1934	/* Manufacturer ID */
+/*
+ * Fintek devices.
+ */
+#define SIO_FINTEK_DEVREV	0x22	/* Fintek Device revision */
+#define SIO_FINTEK_MANID	0x23    /* Fintek ID (2 bytes) */
+
+#define SIO_FINTEK_ID		0x1934  /* Manufacturer ID */
+
 #define SIO_F71869_ID		0x0814	/* F71869 chipset ID */
 #define SIO_F71869A_ID		0x1007	/* F71869A chipset ID */
 #define SIO_F71882_ID		0x0541	/* F71882 chipset ID */
 #define SIO_F71889_ID		0x0909	/* F71889 chipset ID */
 #define SIO_F71889A_ID		0x1005	/* F71889A chipset ID */
 #define SIO_F81866_ID		0x1010	/* F81866 chipset ID */
-#define SIO_F81804_ID		0x1502  /* F81804 chipset ID, same for f81966 */
+#define SIO_F81804_ID		0x1502  /* F81804 chipset ID, same for F81966 */
 #define SIO_F81865_ID		0x0704	/* F81865 chipset ID */
 
+#define SIO_LD_GPIO_FINTEK	0x06	/* GPIO logical device */
+
+/*
+ * Nuvoton devices.
+ */
+#define SIO_NCT6116D_ID		0xD283  /* NCT6116D chipset ID */
+
+#define SIO_LD_GPIO_NUVOTON	0x07	/* GPIO logical device */
+
 
 enum chips {
 	f71869,
@@ -48,6 +62,7 @@ enum chips {
 	f81866,
 	f81804,
 	f81865,
+	nct6116d,
 };
 
 static const char * const f7188x_names[] = {
@@ -59,10 +74,12 @@ static const char * const f7188x_names[] = {
 	"f81866",
 	"f81804",
 	"f81865",
+	"nct6116d",
 };
 
 struct f7188x_sio {
 	int addr;
+	int device;
 	enum chips type;
 };
 
@@ -110,7 +127,7 @@ static inline int superio_enter(int base)
 {
 	/* Don't step on other drivers' I/O space by accident. */
 	if (!request_muxed_region(base, 2, DRVNAME)) {
-		pr_err(DRVNAME "I/O address 0x%04x already in use\n", base);
+		pr_err("I/O address 0x%04x already in use\n", base);
 		return -EBUSY;
 	}
 
@@ -146,10 +163,10 @@ static void f7188x_gpio_set(struct gpio_chip *chip, unsigned offset, int value);
 static int f7188x_gpio_set_config(struct gpio_chip *chip, unsigned offset,
 				  unsigned long config);
 
-#define F7188X_GPIO_BANK(_base, _ngpio, _regbase)			\
+#define F7188X_GPIO_BANK(_base, _ngpio, _regbase, _label)			\
 	{								\
 		.chip = {						\
-			.label            = DRVNAME,			\
+			.label            = _label,			\
 			.owner            = THIS_MODULE,		\
 			.get_direction    = f7188x_gpio_get_direction,	\
 			.direction_input  = f7188x_gpio_direction_in,	\
@@ -164,94 +181,108 @@ static int f7188x_gpio_set_config(struct gpio_chip *chip, unsigned offset,
 		.regbase = _regbase,					\
 	}
 
-#define gpio_dir(base) (base + 0)
-#define gpio_data_out(base) (base + 1)
-#define gpio_data_in(base) (base + 2)
+#define f7188x_gpio_dir(base) ((base) + 0)
+#define f7188x_gpio_data_out(base) ((base) + 1)
+#define f7188x_gpio_data_in(base) ((base) + 2)
 /* Output mode register (0:open drain 1:push-pull). */
-#define gpio_out_mode(base) (base + 3)
+#define f7188x_gpio_out_mode(base) ((base) + 3)
+
+#define f7188x_gpio_dir_invert(type)	((type) == nct6116d)
+#define f7188x_gpio_data_single(type)	((type) == nct6116d)
 
 static struct f7188x_gpio_bank f71869_gpio_bank[] = {
-	F7188X_GPIO_BANK(0, 6, 0xF0),
-	F7188X_GPIO_BANK(10, 8, 0xE0),
-	F7188X_GPIO_BANK(20, 8, 0xD0),
-	F7188X_GPIO_BANK(30, 8, 0xC0),
-	F7188X_GPIO_BANK(40, 8, 0xB0),
-	F7188X_GPIO_BANK(50, 5, 0xA0),
-	F7188X_GPIO_BANK(60, 6, 0x90),
+	F7188X_GPIO_BANK(0, 6, 0xF0, DRVNAME "-0"),
+	F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"),
+	F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"),
+	F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"),
+	F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"),
+	F7188X_GPIO_BANK(50, 5, 0xA0, DRVNAME "-5"),
+	F7188X_GPIO_BANK(60, 6, 0x90, DRVNAME "-6"),
 };
 
 static struct f7188x_gpio_bank f71869a_gpio_bank[] = {
-	F7188X_GPIO_BANK(0, 6, 0xF0),
-	F7188X_GPIO_BANK(10, 8, 0xE0),
-	F7188X_GPIO_BANK(20, 8, 0xD0),
-	F7188X_GPIO_BANK(30, 8, 0xC0),
-	F7188X_GPIO_BANK(40, 8, 0xB0),
-	F7188X_GPIO_BANK(50, 5, 0xA0),
-	F7188X_GPIO_BANK(60, 8, 0x90),
-	F7188X_GPIO_BANK(70, 8, 0x80),
+	F7188X_GPIO_BANK(0, 6, 0xF0, DRVNAME "-0"),
+	F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"),
+	F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"),
+	F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"),
+	F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"),
+	F7188X_GPIO_BANK(50, 5, 0xA0, DRVNAME "-5"),
+	F7188X_GPIO_BANK(60, 8, 0x90, DRVNAME "-6"),
+	F7188X_GPIO_BANK(70, 8, 0x80, DRVNAME "-7"),
 };
 
 static struct f7188x_gpio_bank f71882_gpio_bank[] = {
-	F7188X_GPIO_BANK(0, 8, 0xF0),
-	F7188X_GPIO_BANK(10, 8, 0xE0),
-	F7188X_GPIO_BANK(20, 8, 0xD0),
-	F7188X_GPIO_BANK(30, 4, 0xC0),
-	F7188X_GPIO_BANK(40, 4, 0xB0),
+	F7188X_GPIO_BANK(0, 8, 0xF0, DRVNAME "-0"),
+	F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"),
+	F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"),
+	F7188X_GPIO_BANK(30, 4, 0xC0, DRVNAME "-3"),
+	F7188X_GPIO_BANK(40, 4, 0xB0, DRVNAME "-4"),
 };
 
 static struct f7188x_gpio_bank f71889a_gpio_bank[] = {
-	F7188X_GPIO_BANK(0, 7, 0xF0),
-	F7188X_GPIO_BANK(10, 7, 0xE0),
-	F7188X_GPIO_BANK(20, 8, 0xD0),
-	F7188X_GPIO_BANK(30, 8, 0xC0),
-	F7188X_GPIO_BANK(40, 8, 0xB0),
-	F7188X_GPIO_BANK(50, 5, 0xA0),
-	F7188X_GPIO_BANK(60, 8, 0x90),
-	F7188X_GPIO_BANK(70, 8, 0x80),
+	F7188X_GPIO_BANK(0, 7, 0xF0, DRVNAME "-0"),
+	F7188X_GPIO_BANK(10, 7, 0xE0, DRVNAME "-1"),
+	F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"),
+	F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"),
+	F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"),
+	F7188X_GPIO_BANK(50, 5, 0xA0, DRVNAME "-5"),
+	F7188X_GPIO_BANK(60, 8, 0x90, DRVNAME "-6"),
+	F7188X_GPIO_BANK(70, 8, 0x80, DRVNAME "-7"),
 };
 
 static struct f7188x_gpio_bank f71889_gpio_bank[] = {
-	F7188X_GPIO_BANK(0, 7, 0xF0),
-	F7188X_GPIO_BANK(10, 7, 0xE0),
-	F7188X_GPIO_BANK(20, 8, 0xD0),
-	F7188X_GPIO_BANK(30, 8, 0xC0),
-	F7188X_GPIO_BANK(40, 8, 0xB0),
-	F7188X_GPIO_BANK(50, 5, 0xA0),
-	F7188X_GPIO_BANK(60, 8, 0x90),
-	F7188X_GPIO_BANK(70, 8, 0x80),
+	F7188X_GPIO_BANK(0, 7, 0xF0, DRVNAME "-0"),
+	F7188X_GPIO_BANK(10, 7, 0xE0, DRVNAME "-1"),
+	F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"),
+	F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"),
+	F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"),
+	F7188X_GPIO_BANK(50, 5, 0xA0, DRVNAME "-5"),
+	F7188X_GPIO_BANK(60, 8, 0x90, DRVNAME "-6"),
+	F7188X_GPIO_BANK(70, 8, 0x80, DRVNAME "-7"),
 };
 
 static struct f7188x_gpio_bank f81866_gpio_bank[] = {
-	F7188X_GPIO_BANK(0, 8, 0xF0),
-	F7188X_GPIO_BANK(10, 8, 0xE0),
-	F7188X_GPIO_BANK(20, 8, 0xD0),
-	F7188X_GPIO_BANK(30, 8, 0xC0),
-	F7188X_GPIO_BANK(40, 8, 0xB0),
-	F7188X_GPIO_BANK(50, 8, 0xA0),
-	F7188X_GPIO_BANK(60, 8, 0x90),
-	F7188X_GPIO_BANK(70, 8, 0x80),
-	F7188X_GPIO_BANK(80, 8, 0x88),
+	F7188X_GPIO_BANK(0, 8, 0xF0, DRVNAME "-0"),
+	F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"),
+	F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"),
+	F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"),
+	F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"),
+	F7188X_GPIO_BANK(50, 8, 0xA0, DRVNAME "-5"),
+	F7188X_GPIO_BANK(60, 8, 0x90, DRVNAME "-6"),
+	F7188X_GPIO_BANK(70, 8, 0x80, DRVNAME "-7"),
+	F7188X_GPIO_BANK(80, 8, 0x88, DRVNAME "-8"),
 };
 
 
 static struct f7188x_gpio_bank f81804_gpio_bank[] = {
-	F7188X_GPIO_BANK(0, 8, 0xF0),
-	F7188X_GPIO_BANK(10, 8, 0xE0),
-	F7188X_GPIO_BANK(20, 8, 0xD0),
-	F7188X_GPIO_BANK(50, 8, 0xA0),
-	F7188X_GPIO_BANK(60, 8, 0x90),
-	F7188X_GPIO_BANK(70, 8, 0x80),
-	F7188X_GPIO_BANK(90, 8, 0x98),
+	F7188X_GPIO_BANK(0, 8, 0xF0, DRVNAME "-0"),
+	F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"),
+	F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"),
+	F7188X_GPIO_BANK(50, 8, 0xA0, DRVNAME "-3"),
+	F7188X_GPIO_BANK(60, 8, 0x90, DRVNAME "-4"),
+	F7188X_GPIO_BANK(70, 8, 0x80, DRVNAME "-5"),
+	F7188X_GPIO_BANK(90, 8, 0x98, DRVNAME "-6"),
 };
 
 static struct f7188x_gpio_bank f81865_gpio_bank[] = {
-	F7188X_GPIO_BANK(0, 8, 0xF0),
-	F7188X_GPIO_BANK(10, 8, 0xE0),
-	F7188X_GPIO_BANK(20, 8, 0xD0),
-	F7188X_GPIO_BANK(30, 8, 0xC0),
-	F7188X_GPIO_BANK(40, 8, 0xB0),
-	F7188X_GPIO_BANK(50, 8, 0xA0),
-	F7188X_GPIO_BANK(60, 5, 0x90),
+	F7188X_GPIO_BANK(0, 8, 0xF0, DRVNAME "-0"),
+	F7188X_GPIO_BANK(10, 8, 0xE0, DRVNAME "-1"),
+	F7188X_GPIO_BANK(20, 8, 0xD0, DRVNAME "-2"),
+	F7188X_GPIO_BANK(30, 8, 0xC0, DRVNAME "-3"),
+	F7188X_GPIO_BANK(40, 8, 0xB0, DRVNAME "-4"),
+	F7188X_GPIO_BANK(50, 8, 0xA0, DRVNAME "-5"),
+	F7188X_GPIO_BANK(60, 5, 0x90, DRVNAME "-6"),
+};
+
+static struct f7188x_gpio_bank nct6116d_gpio_bank[] = {
+	F7188X_GPIO_BANK(0, 8, 0xE0, DRVNAME "-0"),
+	F7188X_GPIO_BANK(10, 8, 0xE4, DRVNAME "-1"),
+	F7188X_GPIO_BANK(20, 8, 0xE8, DRVNAME "-2"),
+	F7188X_GPIO_BANK(30, 8, 0xEC, DRVNAME "-3"),
+	F7188X_GPIO_BANK(40, 8, 0xF0, DRVNAME "-4"),
+	F7188X_GPIO_BANK(50, 8, 0xF4, DRVNAME "-5"),
+	F7188X_GPIO_BANK(60, 8, 0xF8, DRVNAME "-6"),
+	F7188X_GPIO_BANK(70, 1, 0xFC, DRVNAME "-7"),
 };
 
 static int f7188x_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
@@ -264,13 +295,16 @@ static int f7188x_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
 	err = superio_enter(sio->addr);
 	if (err)
 		return err;
-	superio_select(sio->addr, SIO_LD_GPIO);
+	superio_select(sio->addr, sio->device);
 
-	dir = superio_inb(sio->addr, gpio_dir(bank->regbase));
+	dir = superio_inb(sio->addr, f7188x_gpio_dir(bank->regbase));
 
 	superio_exit(sio->addr);
 
-	if (dir & 1 << offset)
+	if (f7188x_gpio_dir_invert(sio->type))
+		dir = ~dir;
+
+	if (dir & BIT(offset))
 		return GPIO_LINE_DIRECTION_OUT;
 
 	return GPIO_LINE_DIRECTION_IN;
@@ -286,11 +320,15 @@ static int f7188x_gpio_direction_in(struct gpio_chip *chip, unsigned offset)
 	err = superio_enter(sio->addr);
 	if (err)
 		return err;
-	superio_select(sio->addr, SIO_LD_GPIO);
+	superio_select(sio->addr, sio->device);
 
-	dir = superio_inb(sio->addr, gpio_dir(bank->regbase));
-	dir &= ~BIT(offset);
-	superio_outb(sio->addr, gpio_dir(bank->regbase), dir);
+	dir = superio_inb(sio->addr, f7188x_gpio_dir(bank->regbase));
+
+	if (f7188x_gpio_dir_invert(sio->type))
+		dir |= BIT(offset);
+	else
+		dir &= ~BIT(offset);
+	superio_outb(sio->addr, f7188x_gpio_dir(bank->regbase), dir);
 
 	superio_exit(sio->addr);
 
@@ -307,14 +345,14 @@ static int f7188x_gpio_get(struct gpio_chip *chip, unsigned offset)
 	err = superio_enter(sio->addr);
 	if (err)
 		return err;
-	superio_select(sio->addr, SIO_LD_GPIO);
+	superio_select(sio->addr, sio->device);
 
-	dir = superio_inb(sio->addr, gpio_dir(bank->regbase));
+	dir = superio_inb(sio->addr, f7188x_gpio_dir(bank->regbase));
 	dir = !!(dir & BIT(offset));
-	if (dir)
-		data = superio_inb(sio->addr, gpio_data_out(bank->regbase));
+	if (f7188x_gpio_data_single(sio->type) || dir)
+		data = superio_inb(sio->addr, f7188x_gpio_data_out(bank->regbase));
 	else
-		data = superio_inb(sio->addr, gpio_data_in(bank->regbase));
+		data = superio_inb(sio->addr, f7188x_gpio_data_in(bank->regbase));
 
 	superio_exit(sio->addr);
 
@@ -332,18 +370,21 @@ static int f7188x_gpio_direction_out(struct gpio_chip *chip,
 	err = superio_enter(sio->addr);
 	if (err)
 		return err;
-	superio_select(sio->addr, SIO_LD_GPIO);
+	superio_select(sio->addr, sio->device);
 
-	data_out = superio_inb(sio->addr, gpio_data_out(bank->regbase));
+	data_out = superio_inb(sio->addr, f7188x_gpio_data_out(bank->regbase));
 	if (value)
 		data_out |= BIT(offset);
 	else
 		data_out &= ~BIT(offset);
-	superio_outb(sio->addr, gpio_data_out(bank->regbase), data_out);
+	superio_outb(sio->addr, f7188x_gpio_data_out(bank->regbase), data_out);
 
-	dir = superio_inb(sio->addr, gpio_dir(bank->regbase));
-	dir |= BIT(offset);
-	superio_outb(sio->addr, gpio_dir(bank->regbase), dir);
+	dir = superio_inb(sio->addr, f7188x_gpio_dir(bank->regbase));
+	if (f7188x_gpio_dir_invert(sio->type))
+		dir &= ~BIT(offset);
+	else
+		dir |= BIT(offset);
+	superio_outb(sio->addr, f7188x_gpio_dir(bank->regbase), dir);
 
 	superio_exit(sio->addr);
 
@@ -360,14 +401,14 @@ static void f7188x_gpio_set(struct gpio_chip *chip, unsigned offset, int value)
 	err = superio_enter(sio->addr);
 	if (err)
 		return;
-	superio_select(sio->addr, SIO_LD_GPIO);
+	superio_select(sio->addr, sio->device);
 
-	data_out = superio_inb(sio->addr, gpio_data_out(bank->regbase));
+	data_out = superio_inb(sio->addr, f7188x_gpio_data_out(bank->regbase));
 	if (value)
 		data_out |= BIT(offset);
 	else
 		data_out &= ~BIT(offset);
-	superio_outb(sio->addr, gpio_data_out(bank->regbase), data_out);
+	superio_outb(sio->addr, f7188x_gpio_data_out(bank->regbase), data_out);
 
 	superio_exit(sio->addr);
 }
@@ -388,14 +429,14 @@ static int f7188x_gpio_set_config(struct gpio_chip *chip, unsigned offset,
 	err = superio_enter(sio->addr);
 	if (err)
 		return err;
-	superio_select(sio->addr, SIO_LD_GPIO);
+	superio_select(sio->addr, sio->device);
 
-	data = superio_inb(sio->addr, gpio_out_mode(bank->regbase));
+	data = superio_inb(sio->addr, f7188x_gpio_out_mode(bank->regbase));
 	if (param == PIN_CONFIG_DRIVE_OPEN_DRAIN)
 		data &= ~BIT(offset);
 	else
 		data |= BIT(offset);
-	superio_outb(sio->addr, gpio_out_mode(bank->regbase), data);
+	superio_outb(sio->addr, f7188x_gpio_out_mode(bank->regbase), data);
 
 	superio_exit(sio->addr);
 	return 0;
@@ -449,6 +490,10 @@ static int f7188x_gpio_probe(struct platform_device *pdev)
 		data->nr_bank = ARRAY_SIZE(f81865_gpio_bank);
 		data->bank = f81865_gpio_bank;
 		break;
+	case nct6116d:
+		data->nr_bank = ARRAY_SIZE(nct6116d_gpio_bank);
+		data->bank = nct6116d_gpio_bank;
+		break;
 	default:
 		return -ENODEV;
 	}
@@ -479,18 +524,15 @@ static int __init f7188x_find(int addr, struct f7188x_sio *sio)
 {
 	int err;
 	u16 devid;
+	u16 manid;
 
 	err = superio_enter(addr);
 	if (err)
 		return err;
 
 	err = -ENODEV;
-	devid = superio_inw(addr, SIO_MANID);
-	if (devid != SIO_FINTEK_ID) {
-		pr_debug(DRVNAME ": Not a Fintek device at 0x%08x\n", addr);
-		goto err;
-	}
 
+	sio->device = SIO_LD_GPIO_FINTEK;
 	devid = superio_inw(addr, SIO_DEVID);
 	switch (devid) {
 	case SIO_F71869_ID:
@@ -517,17 +559,30 @@ static int __init f7188x_find(int addr, struct f7188x_sio *sio)
 	case SIO_F81865_ID:
 		sio->type = f81865;
 		break;
+	case SIO_NCT6116D_ID:
+		sio->device = SIO_LD_GPIO_NUVOTON;
+		sio->type = nct6116d;
+		break;
 	default:
-		pr_info(DRVNAME ": Unsupported Fintek device 0x%04x\n", devid);
+		pr_info("Unsupported Fintek device 0x%04x\n", devid);
 		goto err;
 	}
+
+	/* double check manufacturer where possible */
+	if (sio->type != nct6116d) {
+		manid = superio_inw(addr, SIO_FINTEK_MANID);
+		if (manid != SIO_FINTEK_ID) {
+			pr_debug("Not a Fintek device at 0x%08x\n", addr);
+			goto err;
+		}
+	}
+
 	sio->addr = addr;
 	err = 0;
 
-	pr_info(DRVNAME ": Found %s at %#x, revision %d\n",
-		f7188x_names[sio->type],
-		(unsigned int) addr,
-		(int) superio_inb(addr, SIO_DEVREV));
+	pr_info("Found %s at %#x\n", f7188x_names[sio->type], (unsigned int)addr);
+	if (sio->type != nct6116d)
+		pr_info("   revision %d\n", superio_inb(addr, SIO_FINTEK_DEVREV));
 
 err:
 	superio_exit(addr);
@@ -548,13 +603,13 @@ f7188x_gpio_device_add(const struct f7188x_sio *sio)
 	err = platform_device_add_data(f7188x_gpio_pdev,
 				       sio, sizeof(*sio));
 	if (err) {
-		pr_err(DRVNAME "Platform data allocation failed\n");
+		pr_err("Platform data allocation failed\n");
 		goto err;
 	}
 
 	err = platform_device_add(f7188x_gpio_pdev);
 	if (err) {
-		pr_err(DRVNAME "Device addition failed\n");
+		pr_err("Device addition failed\n");
 		goto err;
 	}
 
diff --git a/drivers/gpio/gpio-max7300.c b/drivers/gpio/gpio-max7300.c
index b2b547dd6e84..43da381a4d7e 100644
--- a/drivers/gpio/gpio-max7300.c
+++ b/drivers/gpio/gpio-max7300.c
@@ -48,11 +48,9 @@ static int max7300_probe(struct i2c_client *client,
 	return __max730x_probe(ts);
 }
 
-static int max7300_remove(struct i2c_client *client)
+static void max7300_remove(struct i2c_client *client)
 {
 	__max730x_remove(&client->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id max7300_id[] = {
diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c
index aa126ab80f0c..1bb317b8dcce 100644
--- a/drivers/gpio/gpio-mvebu.c
+++ b/drivers/gpio/gpio-mvebu.c
@@ -790,8 +790,12 @@ static int mvebu_pwm_probe(struct platform_device *pdev,
 	u32 offset;
 	u32 set;
 
-	if (of_device_is_compatible(mvchip->chip.of_node,
-				    "marvell,armada-370-gpio")) {
+	if (mvchip->soc_variant == MVEBU_GPIO_SOC_VARIANT_A8K) {
+		int ret = of_property_read_u32(dev->of_node,
+					       "marvell,pwm-offset", &offset);
+		if (ret < 0)
+			return 0;
+	} else {
 		/*
 		 * There are only two sets of PWM configuration registers for
 		 * all the GPIO lines on those SoCs which this driver reserves
@@ -801,13 +805,6 @@ static int mvebu_pwm_probe(struct platform_device *pdev,
 		if (!platform_get_resource_byname(pdev, IORESOURCE_MEM, "pwm"))
 			return 0;
 		offset = 0;
-	} else if (mvchip->soc_variant == MVEBU_GPIO_SOC_VARIANT_A8K) {
-		int ret = of_property_read_u32(dev->of_node,
-					       "marvell,pwm-offset", &offset);
-		if (ret < 0)
-			return 0;
-	} else {
-		return 0;
 	}
 
 	if (IS_ERR(mvchip->clk))
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index 2925f4d8cef3..cf9bf3fcaee2 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -1101,24 +1101,17 @@ err_exit:
 	return ret;
 }
 
-static int pca953x_remove(struct i2c_client *client)
+static void pca953x_remove(struct i2c_client *client)
 {
 	struct pca953x_platform_data *pdata = dev_get_platdata(&client->dev);
 	struct pca953x_chip *chip = i2c_get_clientdata(client);
-	int ret;
 
 	if (pdata && pdata->teardown) {
-		ret = pdata->teardown(client, chip->gpio_chip.base,
-				      chip->gpio_chip.ngpio, pdata->context);
-		if (ret < 0)
-			dev_err(&client->dev, "teardown failed, %d\n", ret);
-	} else {
-		ret = 0;
+		pdata->teardown(client, chip->gpio_chip.base,
+				chip->gpio_chip.ngpio, pdata->context);
 	}
 
 	regulator_disable(chip->regulator);
-
-	return ret;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c
index 59cc27e4de51..e98ea47d7237 100644
--- a/drivers/gpio/gpio-pcf857x.c
+++ b/drivers/gpio/gpio-pcf857x.c
@@ -399,7 +399,7 @@ fail:
 	return status;
 }
 
-static int pcf857x_remove(struct i2c_client *client)
+static void pcf857x_remove(struct i2c_client *client)
 {
 	struct pcf857x_platform_data	*pdata = dev_get_platdata(&client->dev);
 	struct pcf857x			*gpio = i2c_get_clientdata(client);
@@ -407,8 +407,6 @@ static int pcf857x_remove(struct i2c_client *client)
 	if (pdata && pdata->teardown)
 		pdata->teardown(client, gpio->chip.base, gpio->chip.ngpio,
 				pdata->context);
-
-	return 0;
 }
 
 static void pcf857x_shutdown(struct i2c_client *client)
diff --git a/drivers/gpio/gpio-tpic2810.c b/drivers/gpio/gpio-tpic2810.c
index a09b1e69b072..d642c35cb97c 100644
--- a/drivers/gpio/gpio-tpic2810.c
+++ b/drivers/gpio/gpio-tpic2810.c
@@ -126,13 +126,11 @@ static int tpic2810_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tpic2810_remove(struct i2c_client *client)
+static void tpic2810_remove(struct i2c_client *client)
 {
 	struct tpic2810 *gpio = i2c_get_clientdata(client);
 
 	gpiochip_remove(&gpio->chip);
-
-	return 0;
 }
 
 static const struct i2c_device_id tpic2810_id_table[] = {
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 6c2256e8474b..198ba846d34b 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -31,6 +31,7 @@ menuconfig DRM
 config DRM_MIPI_DBI
 	tristate
 	depends on DRM
+	select DRM_KMS_HELPER
 
 config DRM_MIPI_DSI
 	bool
@@ -50,10 +51,9 @@ config DRM_DEBUG_MM
 
 	  If in doubt, say "N".
 
-config DRM_DEBUG_SELFTEST
-	tristate "kselftests for DRM"
-	depends on DRM
-	depends on DEBUG_KERNEL
+config DRM_KUNIT_TEST
+	tristate "KUnit tests for DRM" if !KUNIT_ALL_TESTS
+	depends on DRM && KUNIT
 	select PRIME_NUMBERS
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HELPER
@@ -61,19 +61,6 @@ config DRM_DEBUG_SELFTEST
 	select DRM_KMS_HELPER
 	select DRM_BUDDY
 	select DRM_EXPORT_FOR_TESTS if m
-	default n
-	help
-	  This option provides kernel modules that can be used to run
-	  various selftests on parts of the DRM api. This option is not
-	  useful for distributions or general kernels, but only for kernel
-	  developers working on DRM and associated drivers.
-
-	  If in doubt, say "N".
-
-config DRM_KUNIT_TEST
-	tristate "KUnit tests for DRM" if !KUNIT_ALL_TESTS
-	depends on DRM && KUNIT=y
-	select DRM_KMS_HELPER
 	default KUNIT_ALL_TESTS
 	help
 	  This builds unit tests for DRM. This option is not useful for
@@ -214,11 +201,11 @@ config DRM_TTM_HELPER
 	help
 	  Helpers for ttm-based gem objects
 
-config DRM_GEM_CMA_HELPER
+config DRM_GEM_DMA_HELPER
 	tristate
 	depends on DRM
 	help
-	  Choose this if you need the GEM CMA helper functions
+	  Choose this if you need the GEM DMA helper functions
 
 config DRM_GEM_SHMEM_HELPER
 	tristate
@@ -248,6 +235,13 @@ config DRM_RADEON
 	select HWMON
 	select BACKLIGHT_CLASS_DEVICE
 	select INTERVAL_TREE
+	# radeon depends on ACPI_VIDEO when ACPI is enabled, for select to work
+	# ACPI_VIDEO's dependencies must also be selected.
+	select INPUT if ACPI
+	select ACPI_VIDEO if ACPI
+	# On x86 ACPI_VIDEO also needs ACPI_WMI
+	select X86_PLATFORM_DEVICES if ACPI && X86
+	select ACPI_WMI if ACPI && X86
 	help
 	  Choose this option if you have an ATI Radeon graphics card.  There
 	  are both PCI and AGP versions.  You don't need to choose this to
@@ -273,6 +267,13 @@ config DRM_AMDGPU
 	select BACKLIGHT_CLASS_DEVICE
 	select INTERVAL_TREE
 	select DRM_BUDDY
+	# amdgpu depends on ACPI_VIDEO when ACPI is enabled, for select to work
+	# ACPI_VIDEO's dependencies must also be selected.
+	select INPUT if ACPI
+	select ACPI_VIDEO if ACPI
+	# On x86 ACPI_VIDEO also needs ACPI_WMI
+	select X86_PLATFORM_DEVICES if ACPI && X86
+	select ACPI_WMI if ACPI && X86
 	help
 	  Choose this option if you have a recent AMD Radeon graphics card.
 
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index e7af358e6dda..25d0ba310509 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -40,9 +40,9 @@ obj-$(CONFIG_DRM_PANEL_ORIENTATION_QUIRKS) += drm_panel_orientation_quirks.o
 
 obj-$(CONFIG_DRM_BUDDY) += drm_buddy.o
 
-drm_cma_helper-y := drm_gem_cma_helper.o
-drm_cma_helper-$(CONFIG_DRM_KMS_HELPER) += drm_fb_cma_helper.o
-obj-$(CONFIG_DRM_GEM_CMA_HELPER) += drm_cma_helper.o
+drm_dma_helper-y := drm_gem_dma_helper.o
+drm_dma_helper-$(CONFIG_DRM_KMS_HELPER) += drm_fb_dma_helper.o
+obj-$(CONFIG_DRM_GEM_DMA_HELPER) += drm_dma_helper.o
 
 drm_shmem_helper-y := drm_gem_shmem_helper.o
 obj-$(CONFIG_DRM_GEM_SHMEM_HELPER) += drm_shmem_helper.o
@@ -75,7 +75,6 @@ obj-$(CONFIG_DRM_KMS_HELPER) += drm_kms_helper.o
 # Drivers and the rest
 #
 
-obj-$(CONFIG_DRM_DEBUG_SELFTEST) += selftests/
 obj-$(CONFIG_DRM_KUNIT_TEST) += tests/
 
 obj-$(CONFIG_DRM_MIPI_DBI) += drm_mipi_dbi.o
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index 5a283d12f8e1..6ad39cf71bdd 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -75,7 +75,7 @@ amdgpu-y += \
 	vi.o mxgpu_vi.o nbio_v6_1.o soc15.o emu_soc.o mxgpu_ai.o nbio_v7_0.o vega10_reg_init.o \
 	vega20_reg_init.o nbio_v7_4.o nbio_v2_3.o nv.o arct_reg_init.o mxgpu_nv.o \
 	nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o \
-	nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o
+	sienna_cichlid.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o
 
 # add DF block
 amdgpu-y += \
@@ -89,7 +89,7 @@ amdgpu-y += \
 	gfxhub_v1_0.o mmhub_v1_0.o gmc_v9_0.o gfxhub_v1_1.o mmhub_v9_4.o \
 	gfxhub_v2_0.o mmhub_v2_0.o gmc_v10_0.o gfxhub_v2_1.o mmhub_v2_3.o \
 	mmhub_v1_7.o gfxhub_v3_0.o mmhub_v3_0.o mmhub_v3_0_2.o gmc_v11_0.o \
-	mmhub_v3_0_1.o
+	mmhub_v3_0_1.o gfxhub_v3_0_3.o
 
 # add UMC block
 amdgpu-y += \
@@ -134,7 +134,8 @@ amdgpu-y += \
 	gfx_v9_4_2.o \
 	gfx_v10_0.o \
 	imu_v11_0.o \
-	gfx_v11_0.o
+	gfx_v11_0.o \
+	imu_v11_0_3.o
 
 # add async DMA block
 amdgpu-y += \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index d597e2656c47..ae9371b172e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -274,6 +274,9 @@ extern int amdgpu_vcnfw_log;
 #define AMDGPU_RESET_VCE			(1 << 13)
 #define AMDGPU_RESET_VCE1			(1 << 14)
 
+#define AMDGPU_RESET_LEVEL_SOFT_RECOVERY (1 << 0)
+#define AMDGPU_RESET_LEVEL_MODE2 (1 << 1)
+
 /* max cursor sizes (in pixels) */
 #define CIK_CURSOR_WIDTH 128
 #define CIK_CURSOR_HEIGHT 128
@@ -882,6 +885,7 @@ struct amdgpu_device {
 	u64				fence_context;
 	unsigned			num_rings;
 	struct amdgpu_ring		*rings[AMDGPU_MAX_RINGS];
+	struct dma_fence __rcu		*gang_submit;
 	bool				ib_pool_ready;
 	struct amdgpu_sa_manager	ib_pools[AMDGPU_IB_POOL_MAX];
 	struct amdgpu_sched		gpu_sched[AMDGPU_HW_IP_NUM][AMDGPU_RING_PRIO_MAX];
@@ -1060,6 +1064,9 @@ struct amdgpu_device {
 	uint32_t                        scpm_status;
 
 	struct work_struct		reset_work;
+
+	uint32_t						amdgpu_reset_level_mask;
+	bool                            job_hang;
 };
 
 static inline struct amdgpu_device *drm_to_adev(struct drm_device *ddev)
@@ -1288,6 +1295,8 @@ u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
 				u32 reg);
 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
 				u32 reg, u32 v);
+struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
+					    struct dma_fence *gang);
 
 /* atpx handler */
 #if defined(CONFIG_VGA_SWITCHEROO)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
index 130060834b4e..b14800ac179e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2012 Advanced Micro Devices, Inc.
  *
@@ -849,6 +850,7 @@ int amdgpu_acpi_init(struct amdgpu_device *adev)
 		if (amdgpu_device_has_dc_support(adev)) {
 #if defined(CONFIG_DRM_AMD_DC)
 			struct amdgpu_display_manager *dm = &adev->dm;
+
 			if (dm->backlight_dev[0])
 				atif->bd = dm->backlight_dev[0];
 #endif
@@ -863,6 +865,7 @@ int amdgpu_acpi_init(struct amdgpu_device *adev)
 				if ((enc->devices & (ATOM_DEVICE_LCD_SUPPORT)) &&
 				    enc->enc_priv) {
 					struct amdgpu_encoder_atom_dig *dig = enc->enc_priv;
+
 					if (dig->bl_dev) {
 						atif->bd = dig->bl_dev;
 						break;
@@ -919,9 +922,9 @@ static bool amdgpu_atif_pci_probe_handle(struct pci_dev *pdev)
 		return false;
 
 	status = acpi_get_handle(dhandle, "ATIF", &atif_handle);
-	if (ACPI_FAILURE(status)) {
+	if (ACPI_FAILURE(status))
 		return false;
-	}
+
 	amdgpu_acpi_priv.atif.handle = atif_handle;
 	acpi_get_name(amdgpu_acpi_priv.atif.handle, ACPI_FULL_PATHNAME, &buffer);
 	DRM_DEBUG_DRIVER("Found ATIF handle %s\n", acpi_method_name);
@@ -954,9 +957,9 @@ static bool amdgpu_atcs_pci_probe_handle(struct pci_dev *pdev)
 		return false;
 
 	status = acpi_get_handle(dhandle, "ATCS", &atcs_handle);
-	if (ACPI_FAILURE(status)) {
+	if (ACPI_FAILURE(status))
 		return false;
-	}
+
 	amdgpu_acpi_priv.atcs.handle = atcs_handle;
 	acpi_get_name(amdgpu_acpi_priv.atcs.handle, ACPI_FULL_PATHNAME, &buffer);
 	DRM_DEBUG_DRIVER("Found ATCS handle %s\n", acpi_method_name);
@@ -1050,6 +1053,10 @@ bool amdgpu_acpi_should_gpu_reset(struct amdgpu_device *adev)
 {
 	if (adev->flags & AMD_IS_APU)
 		return false;
+
+	if (amdgpu_sriov_vf(adev))
+		return false;
+
 	return pm_suspend_target_state != PM_SUSPEND_TO_IDLE;
 }
 
@@ -1066,6 +1073,12 @@ bool amdgpu_acpi_is_s0ix_active(struct amdgpu_device *adev)
 	    (pm_suspend_target_state != PM_SUSPEND_TO_IDLE))
 		return false;
 
+	/*
+	 * If ACPI_FADT_LOW_POWER_S0 is not set in the FADT, it is generally
+	 * risky to do any special firmware-related preparations for entering
+	 * S0ix even though the system is suspending to idle, so return false
+	 * in that case.
+	 */
 	if (!(acpi_gbl_FADT.flags & ACPI_FADT_LOW_POWER_S0)) {
 		dev_warn_once(adev->dev,
 			      "Power consumption will be higher as BIOS has not been configured for suspend-to-idle.\n"
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 5e53a5293935..9e98f3866edc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2014 Advanced Micro Devices, Inc.
  *
@@ -130,11 +131,13 @@ static void amdgpu_amdkfd_reset_work(struct work_struct *work)
 						  kfd.reset_work);
 
 	struct amdgpu_reset_context reset_context;
+
 	memset(&reset_context, 0, sizeof(reset_context));
 
 	reset_context.method = AMD_RESET_METHOD_NONE;
 	reset_context.reset_req_dev = adev;
 	clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+	clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags);
 
 	amdgpu_device_gpu_recover(adev, NULL, &reset_context);
 }
@@ -683,6 +686,7 @@ int amdgpu_amdkfd_submit_ib(struct amdgpu_device *adev,
 	ib->length_dw = ib_len;
 	/* This works for NO_HWS. TODO: need to handle without knowing VMID */
 	job->vmid = vmid;
+	job->num_ibs = 1;
 
 	ret = amdgpu_ib_schedule(ring, 1, ib, job, &f);
 
@@ -752,11 +756,7 @@ void amdgpu_amdkfd_ras_poison_consumption_handler(struct amdgpu_device *adev, bo
 {
 	struct ras_err_data err_data = {0, 0, 0, NULL};
 
-	/* CPU MCA will handle page retirement if connected_to_cpu is 1 */
-	if (!adev->gmc.xgmi.connected_to_cpu)
-		amdgpu_umc_poison_handler(adev, &err_data, reset);
-	else if (reset)
-		amdgpu_amdkfd_gpu_reset(adev);
+	amdgpu_umc_poison_handler(adev, &err_data, reset);
 }
 
 bool amdgpu_amdkfd_ras_query_utcl2_poison_status(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 2170db83e41d..978d3970b5cc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2014-2018 Advanced Micro Devices, Inc.
  *
@@ -297,7 +298,7 @@ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
 	 */
 	replacement = dma_fence_get_stub();
 	dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context,
-				replacement, DMA_RESV_USAGE_READ);
+				replacement, DMA_RESV_USAGE_BOOKKEEP);
 	dma_fence_put(replacement);
 	return 0;
 }
@@ -1390,8 +1391,9 @@ static int init_kfd_vm(struct amdgpu_vm *vm, void **process_info,
 	ret = dma_resv_reserve_fences(vm->root.bo->tbo.base.resv, 1);
 	if (ret)
 		goto reserve_shared_fail;
-	amdgpu_bo_fence(vm->root.bo,
-			&vm->process_info->eviction_fence->base, true);
+	dma_resv_add_fence(vm->root.bo->tbo.base.resv,
+			   &vm->process_info->eviction_fence->base,
+			   DMA_RESV_USAGE_BOOKKEEP);
 	amdgpu_bo_unreserve(vm->root.bo);
 
 	/* Update process info */
@@ -1612,6 +1614,7 @@ size_t amdgpu_amdkfd_get_available_memory(struct amdgpu_device *adev)
 	uint64_t reserved_for_pt =
 		ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size);
 	size_t available;
+
 	spin_lock(&kfd_mem_limit.mem_limit_lock);
 	available = adev->gmc.real_vram_size
 		- adev->kfd.vram_used_aligned
@@ -1987,9 +1990,9 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
 	}
 
 	if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm) && !bo->tbo.pin_count)
-		amdgpu_bo_fence(bo,
-				&avm->process_info->eviction_fence->base,
-				true);
+		dma_resv_add_fence(bo->tbo.base.resv,
+				   &avm->process_info->eviction_fence->base,
+				   DMA_RESV_USAGE_BOOKKEEP);
 	ret = unreserve_bo_and_vms(&ctx, false, false);
 
 	goto out;
@@ -2216,7 +2219,7 @@ int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct amdgpu_device *adev,
 {
 	if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {
 		*mem = *adev->gmc.vm_fault_info;
-		mb();
+		mb(); /* make sure read happened */
 		atomic_set(&adev->gmc.vm_fault_info_updated, 0);
 	}
 	return 0;
@@ -2758,15 +2761,18 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
 		if (mem->bo->tbo.pin_count)
 			continue;
 
-		amdgpu_bo_fence(mem->bo,
-			&process_info->eviction_fence->base, true);
+		dma_resv_add_fence(mem->bo->tbo.base.resv,
+				   &process_info->eviction_fence->base,
+				   DMA_RESV_USAGE_BOOKKEEP);
 	}
 	/* Attach eviction fence to PD / PT BOs */
 	list_for_each_entry(peer_vm, &process_info->vm_list_head,
 			    vm_list_node) {
 		struct amdgpu_bo *bo = peer_vm->root.bo;
 
-		amdgpu_bo_fence(bo, &process_info->eviction_fence->base, true);
+		dma_resv_add_fence(bo->tbo.base.resv,
+				   &process_info->eviction_fence->base,
+				   DMA_RESV_USAGE_BOOKKEEP);
 	}
 
 validate_map_fail:
@@ -2820,7 +2826,9 @@ int amdgpu_amdkfd_add_gws_to_process(void *info, void *gws, struct kgd_mem **mem
 	ret = dma_resv_reserve_fences(gws_bo->tbo.base.resv, 1);
 	if (ret)
 		goto reserve_shared_fail;
-	amdgpu_bo_fence(gws_bo, &process_info->eviction_fence->base, true);
+	dma_resv_add_fence(gws_bo->tbo.base.resv,
+			   &process_info->eviction_fence->base,
+			   DMA_RESV_USAGE_BOOKKEEP);
 	amdgpu_bo_unreserve(gws_bo);
 	mutex_unlock(&(*mem)->process_info->lock);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index b7933c2ce765..491d4846fc02 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -1674,10 +1674,12 @@ amdgpu_connector_add(struct amdgpu_device *adev,
 						   adev->mode_info.dither_property,
 						   AMDGPU_FMT_DITHER_DISABLE);
 
-			if (amdgpu_audio != 0)
+			if (amdgpu_audio != 0) {
 				drm_object_attach_property(&amdgpu_connector->base.base,
 							   adev->mode_info.audio_property,
 							   AMDGPU_AUDIO_AUTO);
+				amdgpu_connector->audio = AMDGPU_AUDIO_AUTO;
+			}
 
 			subpixel_order = SubPixelHorizontalRGB;
 			connector->interlace_allowed = true;
@@ -1799,6 +1801,7 @@ amdgpu_connector_add(struct amdgpu_device *adev,
 				drm_object_attach_property(&amdgpu_connector->base.base,
 							   adev->mode_info.audio_property,
 							   AMDGPU_AUDIO_AUTO);
+				amdgpu_connector->audio = AMDGPU_AUDIO_AUTO;
 			}
 			drm_object_attach_property(&amdgpu_connector->base.base,
 						   adev->mode_info.dither_property,
@@ -1852,6 +1855,7 @@ amdgpu_connector_add(struct amdgpu_device *adev,
 				drm_object_attach_property(&amdgpu_connector->base.base,
 							   adev->mode_info.audio_property,
 							   AMDGPU_AUDIO_AUTO);
+				amdgpu_connector->audio = AMDGPU_AUDIO_AUTO;
 			}
 			drm_object_attach_property(&amdgpu_connector->base.base,
 						   adev->mode_info.dither_property,
@@ -1902,6 +1906,7 @@ amdgpu_connector_add(struct amdgpu_device *adev,
 				drm_object_attach_property(&amdgpu_connector->base.base,
 							   adev->mode_info.audio_property,
 							   AMDGPU_AUDIO_AUTO);
+				amdgpu_connector->audio = AMDGPU_AUDIO_AUTO;
 			}
 			drm_object_attach_property(&amdgpu_connector->base.base,
 						   adev->mode_info.dither_property,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index b7bae833c804..1bbd39b3b0fc 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -39,9 +39,82 @@
 #include "amdgpu_gem.h"
 #include "amdgpu_ras.h"
 
-static int amdgpu_cs_user_fence_chunk(struct amdgpu_cs_parser *p,
-				      struct drm_amdgpu_cs_chunk_fence *data,
-				      uint32_t *offset)
+static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p,
+				 struct amdgpu_device *adev,
+				 struct drm_file *filp,
+				 union drm_amdgpu_cs *cs)
+{
+	struct amdgpu_fpriv *fpriv = filp->driver_priv;
+
+	if (cs->in.num_chunks == 0)
+		return -EINVAL;
+
+	memset(p, 0, sizeof(*p));
+	p->adev = adev;
+	p->filp = filp;
+
+	p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
+	if (!p->ctx)
+		return -EINVAL;
+
+	if (atomic_read(&p->ctx->guilty)) {
+		amdgpu_ctx_put(p->ctx);
+		return -ECANCELED;
+	}
+	return 0;
+}
+
+static int amdgpu_cs_job_idx(struct amdgpu_cs_parser *p,
+			     struct drm_amdgpu_cs_chunk_ib *chunk_ib)
+{
+	struct drm_sched_entity *entity;
+	unsigned int i;
+	int r;
+
+	r = amdgpu_ctx_get_entity(p->ctx, chunk_ib->ip_type,
+				  chunk_ib->ip_instance,
+				  chunk_ib->ring, &entity);
+	if (r)
+		return r;
+
+	/*
+	 * Abort if there is no run queue associated with this entity.
+	 * Possibly because of disabled HW IP.
+	 */
+	if (entity->rq == NULL)
+		return -EINVAL;
+
+	/* Check if we can add this IB to some existing job */
+	for (i = 0; i < p->gang_size; ++i)
+		if (p->entities[i] == entity)
+			return i;
+
+	/* If not increase the gang size if possible */
+	if (i == AMDGPU_CS_GANG_SIZE)
+		return -EINVAL;
+
+	p->entities[i] = entity;
+	p->gang_size = i + 1;
+	return i;
+}
+
+static int amdgpu_cs_p1_ib(struct amdgpu_cs_parser *p,
+			   struct drm_amdgpu_cs_chunk_ib *chunk_ib,
+			   unsigned int *num_ibs)
+{
+	int r;
+
+	r = amdgpu_cs_job_idx(p, chunk_ib);
+	if (r < 0)
+		return r;
+
+	++(num_ibs[r]);
+	return 0;
+}
+
+static int amdgpu_cs_p1_user_fence(struct amdgpu_cs_parser *p,
+				   struct drm_amdgpu_cs_chunk_fence *data,
+				   uint32_t *offset)
 {
 	struct drm_gem_object *gobj;
 	struct amdgpu_bo *bo;
@@ -80,11 +153,11 @@ error_unref:
 	return r;
 }
 
-static int amdgpu_cs_bo_handles_chunk(struct amdgpu_cs_parser *p,
-				      struct drm_amdgpu_bo_list_in *data)
+static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p,
+				   struct drm_amdgpu_bo_list_in *data)
 {
+	struct drm_amdgpu_bo_list_entry *info;
 	int r;
-	struct drm_amdgpu_bo_list_entry *info = NULL;
 
 	r = amdgpu_bo_create_list_entry_array(data, &info);
 	if (r)
@@ -104,38 +177,25 @@ error_free:
 	return r;
 }
 
-static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs *cs)
+/* Copy the data from userspace and go over it the first time */
+static int amdgpu_cs_pass1(struct amdgpu_cs_parser *p,
+			   union drm_amdgpu_cs *cs)
 {
 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
+	unsigned int num_ibs[AMDGPU_CS_GANG_SIZE] = { };
 	struct amdgpu_vm *vm = &fpriv->vm;
 	uint64_t *chunk_array_user;
 	uint64_t *chunk_array;
-	unsigned size, num_ibs = 0;
 	uint32_t uf_offset = 0;
-	int i;
+	unsigned int size;
 	int ret;
+	int i;
 
-	if (cs->in.num_chunks == 0)
-		return -EINVAL;
-
-	chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t), GFP_KERNEL);
+	chunk_array = kvmalloc_array(cs->in.num_chunks, sizeof(uint64_t),
+				     GFP_KERNEL);
 	if (!chunk_array)
 		return -ENOMEM;
 
-	p->ctx = amdgpu_ctx_get(fpriv, cs->in.ctx_id);
-	if (!p->ctx) {
-		ret = -EINVAL;
-		goto free_chunk;
-	}
-
-	mutex_lock(&p->ctx->lock);
-
-	/* skip guilty context job */
-	if (atomic_read(&p->ctx->guilty) == 1) {
-		ret = -ECANCELED;
-		goto free_chunk;
-	}
-
 	/* get chunks */
 	chunk_array_user = u64_to_user_ptr(cs->in.chunks);
 	if (copy_from_user(chunk_array, chunk_array_user,
@@ -170,7 +230,8 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs
 		size = p->chunks[i].length_dw;
 		cdata = u64_to_user_ptr(user_chunk.chunk_data);
 
-		p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
+		p->chunks[i].kdata = kvmalloc_array(size, sizeof(uint32_t),
+						    GFP_KERNEL);
 		if (p->chunks[i].kdata == NULL) {
 			ret = -ENOMEM;
 			i--;
@@ -182,36 +243,35 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs
 			goto free_partial_kdata;
 		}
 
+		/* Assume the worst on the following checks */
+		ret = -EINVAL;
 		switch (p->chunks[i].chunk_id) {
 		case AMDGPU_CHUNK_ID_IB:
-			++num_ibs;
+			if (size < sizeof(struct drm_amdgpu_cs_chunk_ib))
+				goto free_partial_kdata;
+
+			ret = amdgpu_cs_p1_ib(p, p->chunks[i].kdata, num_ibs);
+			if (ret)
+				goto free_partial_kdata;
 			break;
 
 		case AMDGPU_CHUNK_ID_FENCE:
-			size = sizeof(struct drm_amdgpu_cs_chunk_fence);
-			if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
-				ret = -EINVAL;
+			if (size < sizeof(struct drm_amdgpu_cs_chunk_fence))
 				goto free_partial_kdata;
-			}
 
-			ret = amdgpu_cs_user_fence_chunk(p, p->chunks[i].kdata,
-							 &uf_offset);
+			ret = amdgpu_cs_p1_user_fence(p, p->chunks[i].kdata,
+						      &uf_offset);
 			if (ret)
 				goto free_partial_kdata;
-
 			break;
 
 		case AMDGPU_CHUNK_ID_BO_HANDLES:
-			size = sizeof(struct drm_amdgpu_bo_list_in);
-			if (p->chunks[i].length_dw * sizeof(uint32_t) < size) {
-				ret = -EINVAL;
+			if (size < sizeof(struct drm_amdgpu_bo_list_in))
 				goto free_partial_kdata;
-			}
 
-			ret = amdgpu_cs_bo_handles_chunk(p, p->chunks[i].kdata);
+			ret = amdgpu_cs_p1_bo_handles(p, p->chunks[i].kdata);
 			if (ret)
 				goto free_partial_kdata;
-
 			break;
 
 		case AMDGPU_CHUNK_ID_DEPENDENCIES:
@@ -223,22 +283,32 @@ static int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, union drm_amdgpu_cs
 			break;
 
 		default:
-			ret = -EINVAL;
 			goto free_partial_kdata;
 		}
 	}
 
-	ret = amdgpu_job_alloc(p->adev, num_ibs, &p->job, vm);
-	if (ret)
-		goto free_all_kdata;
+	if (!p->gang_size)
+		return -EINVAL;
+
+	for (i = 0; i < p->gang_size; ++i) {
+		ret = amdgpu_job_alloc(p->adev, num_ibs[i], &p->jobs[i], vm);
+		if (ret)
+			goto free_all_kdata;
+
+		ret = drm_sched_job_init(&p->jobs[i]->base, p->entities[i],
+					 &fpriv->vm);
+		if (ret)
+			goto free_all_kdata;
+	}
+	p->gang_leader = p->jobs[p->gang_size - 1];
 
-	if (p->ctx->vram_lost_counter != p->job->vram_lost_counter) {
+	if (p->ctx->vram_lost_counter != p->gang_leader->vram_lost_counter) {
 		ret = -ECANCELED;
 		goto free_all_kdata;
 	}
 
 	if (p->uf_entry.tv.bo)
-		p->job->uf_addr = uf_offset;
+		p->gang_leader->uf_addr = uf_offset;
 	kvfree(chunk_array);
 
 	/* Use this opportunity to fill in task info for the vm */
@@ -260,6 +330,297 @@ free_chunk:
 	return ret;
 }
 
+static int amdgpu_cs_p2_ib(struct amdgpu_cs_parser *p,
+			   struct amdgpu_cs_chunk *chunk,
+			   unsigned int *ce_preempt,
+			   unsigned int *de_preempt)
+{
+	struct drm_amdgpu_cs_chunk_ib *chunk_ib = chunk->kdata;
+	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
+	struct amdgpu_vm *vm = &fpriv->vm;
+	struct amdgpu_ring *ring;
+	struct amdgpu_job *job;
+	struct amdgpu_ib *ib;
+	int r;
+
+	r = amdgpu_cs_job_idx(p, chunk_ib);
+	if (r < 0)
+		return r;
+
+	job = p->jobs[r];
+	ring = amdgpu_job_ring(job);
+	ib = &job->ibs[job->num_ibs++];
+
+	/* MM engine doesn't support user fences */
+	if (p->uf_entry.tv.bo && ring->funcs->no_user_fence)
+		return -EINVAL;
+
+	if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
+	    chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
+		if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
+			(*ce_preempt)++;
+		else
+			(*de_preempt)++;
+
+		/* Each GFX command submit allows only 1 IB max
+		 * preemptible for CE & DE */
+		if (*ce_preempt > 1 || *de_preempt > 1)
+			return -EINVAL;
+	}
+
+	if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE)
+		job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT;
+
+	r =  amdgpu_ib_get(p->adev, vm, ring->funcs->parse_cs ?
+			   chunk_ib->ib_bytes : 0,
+			   AMDGPU_IB_POOL_DELAYED, ib);
+	if (r) {
+		DRM_ERROR("Failed to get ib !\n");
+		return r;
+	}
+
+	ib->gpu_addr = chunk_ib->va_start;
+	ib->length_dw = chunk_ib->ib_bytes / 4;
+	ib->flags = chunk_ib->flags;
+	return 0;
+}
+
+static int amdgpu_cs_p2_dependencies(struct amdgpu_cs_parser *p,
+				     struct amdgpu_cs_chunk *chunk)
+{
+	struct drm_amdgpu_cs_chunk_dep *deps = chunk->kdata;
+	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
+	unsigned num_deps;
+	int i, r;
+
+	num_deps = chunk->length_dw * 4 /
+		sizeof(struct drm_amdgpu_cs_chunk_dep);
+
+	for (i = 0; i < num_deps; ++i) {
+		struct amdgpu_ctx *ctx;
+		struct drm_sched_entity *entity;
+		struct dma_fence *fence;
+
+		ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id);
+		if (ctx == NULL)
+			return -EINVAL;
+
+		r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type,
+					  deps[i].ip_instance,
+					  deps[i].ring, &entity);
+		if (r) {
+			amdgpu_ctx_put(ctx);
+			return r;
+		}
+
+		fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle);
+		amdgpu_ctx_put(ctx);
+
+		if (IS_ERR(fence))
+			return PTR_ERR(fence);
+		else if (!fence)
+			continue;
+
+		if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) {
+			struct drm_sched_fence *s_fence;
+			struct dma_fence *old = fence;
+
+			s_fence = to_drm_sched_fence(fence);
+			fence = dma_fence_get(&s_fence->scheduled);
+			dma_fence_put(old);
+		}
+
+		r = amdgpu_sync_fence(&p->gang_leader->sync, fence);
+		dma_fence_put(fence);
+		if (r)
+			return r;
+	}
+	return 0;
+}
+
+static int amdgpu_syncobj_lookup_and_add(struct amdgpu_cs_parser *p,
+					 uint32_t handle, u64 point,
+					 u64 flags)
+{
+	struct dma_fence *fence;
+	int r;
+
+	r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence);
+	if (r) {
+		DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n",
+			  handle, point, r);
+		return r;
+	}
+
+	r = amdgpu_sync_fence(&p->gang_leader->sync, fence);
+	dma_fence_put(fence);
+
+	return r;
+}
+
+static int amdgpu_cs_p2_syncobj_in(struct amdgpu_cs_parser *p,
+				   struct amdgpu_cs_chunk *chunk)
+{
+	struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;
+	unsigned num_deps;
+	int i, r;
+
+	num_deps = chunk->length_dw * 4 /
+		sizeof(struct drm_amdgpu_cs_chunk_sem);
+	for (i = 0; i < num_deps; ++i) {
+		r = amdgpu_syncobj_lookup_and_add(p, deps[i].handle, 0, 0);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static int amdgpu_cs_p2_syncobj_timeline_wait(struct amdgpu_cs_parser *p,
+					      struct amdgpu_cs_chunk *chunk)
+{
+	struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;
+	unsigned num_deps;
+	int i, r;
+
+	num_deps = chunk->length_dw * 4 /
+		sizeof(struct drm_amdgpu_cs_chunk_syncobj);
+	for (i = 0; i < num_deps; ++i) {
+		r = amdgpu_syncobj_lookup_and_add(p, syncobj_deps[i].handle,
+						  syncobj_deps[i].point,
+						  syncobj_deps[i].flags);
+		if (r)
+			return r;
+	}
+
+	return 0;
+}
+
+static int amdgpu_cs_p2_syncobj_out(struct amdgpu_cs_parser *p,
+				    struct amdgpu_cs_chunk *chunk)
+{
+	struct drm_amdgpu_cs_chunk_sem *deps = chunk->kdata;
+	unsigned num_deps;
+	int i;
+
+	num_deps = chunk->length_dw * 4 /
+		sizeof(struct drm_amdgpu_cs_chunk_sem);
+
+	if (p->post_deps)
+		return -EINVAL;
+
+	p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
+				     GFP_KERNEL);
+	p->num_post_deps = 0;
+
+	if (!p->post_deps)
+		return -ENOMEM;
+
+
+	for (i = 0; i < num_deps; ++i) {
+		p->post_deps[i].syncobj =
+			drm_syncobj_find(p->filp, deps[i].handle);
+		if (!p->post_deps[i].syncobj)
+			return -EINVAL;
+		p->post_deps[i].chain = NULL;
+		p->post_deps[i].point = 0;
+		p->num_post_deps++;
+	}
+
+	return 0;
+}
+
+static int amdgpu_cs_p2_syncobj_timeline_signal(struct amdgpu_cs_parser *p,
+						struct amdgpu_cs_chunk *chunk)
+{
+	struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps = chunk->kdata;
+	unsigned num_deps;
+	int i;
+
+	num_deps = chunk->length_dw * 4 /
+		sizeof(struct drm_amdgpu_cs_chunk_syncobj);
+
+	if (p->post_deps)
+		return -EINVAL;
+
+	p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
+				     GFP_KERNEL);
+	p->num_post_deps = 0;
+
+	if (!p->post_deps)
+		return -ENOMEM;
+
+	for (i = 0; i < num_deps; ++i) {
+		struct amdgpu_cs_post_dep *dep = &p->post_deps[i];
+
+		dep->chain = NULL;
+		if (syncobj_deps[i].point) {
+			dep->chain = dma_fence_chain_alloc();
+			if (!dep->chain)
+				return -ENOMEM;
+		}
+
+		dep->syncobj = drm_syncobj_find(p->filp,
+						syncobj_deps[i].handle);
+		if (!dep->syncobj) {
+			dma_fence_chain_free(dep->chain);
+			return -EINVAL;
+		}
+		dep->point = syncobj_deps[i].point;
+		p->num_post_deps++;
+	}
+
+	return 0;
+}
+
+static int amdgpu_cs_pass2(struct amdgpu_cs_parser *p)
+{
+	unsigned int ce_preempt = 0, de_preempt = 0;
+	int i, r;
+
+	for (i = 0; i < p->nchunks; ++i) {
+		struct amdgpu_cs_chunk *chunk;
+
+		chunk = &p->chunks[i];
+
+		switch (chunk->chunk_id) {
+		case AMDGPU_CHUNK_ID_IB:
+			r = amdgpu_cs_p2_ib(p, chunk, &ce_preempt, &de_preempt);
+			if (r)
+				return r;
+			break;
+		case AMDGPU_CHUNK_ID_DEPENDENCIES:
+		case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
+			r = amdgpu_cs_p2_dependencies(p, chunk);
+			if (r)
+				return r;
+			break;
+		case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
+			r = amdgpu_cs_p2_syncobj_in(p, chunk);
+			if (r)
+				return r;
+			break;
+		case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
+			r = amdgpu_cs_p2_syncobj_out(p, chunk);
+			if (r)
+				return r;
+			break;
+		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
+			r = amdgpu_cs_p2_syncobj_timeline_wait(p, chunk);
+			if (r)
+				return r;
+			break;
+		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
+			r = amdgpu_cs_p2_syncobj_timeline_signal(p, chunk);
+			if (r)
+				return r;
+			break;
+		}
+	}
+
+	return 0;
+}
+
 /* Convert microseconds to bytes. */
 static u64 us_to_bytes(struct amdgpu_device *adev, s64 us)
 {
@@ -495,9 +856,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_bo_list_entry *e;
 	struct list_head duplicates;
-	struct amdgpu_bo *gds;
-	struct amdgpu_bo *gws;
-	struct amdgpu_bo *oa;
+	unsigned int i;
 	int r;
 
 	INIT_LIST_HEAD(&p->validated);
@@ -581,16 +940,6 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 		e->bo_va = amdgpu_vm_bo_find(vm, bo);
 	}
 
-	/* Move fence waiting after getting reservation lock of
-	 * PD root. Then there is no need on a ctx mutex lock.
-	 */
-	r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entity);
-	if (unlikely(r != 0)) {
-		if (r != -ERESTARTSYS)
-			DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n");
-		goto error_validate;
-	}
-
 	amdgpu_cs_get_threshold_for_moves(p->adev, &p->bytes_moved_threshold,
 					  &p->bytes_moved_vis_threshold);
 	p->bytes_moved = 0;
@@ -611,197 +960,139 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
 	if (r)
 		goto error_validate;
 
-	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
-				     p->bytes_moved_vis);
+	if (p->uf_entry.tv.bo) {
+		struct amdgpu_bo *uf = ttm_to_amdgpu_bo(p->uf_entry.tv.bo);
 
-	gds = p->bo_list->gds_obj;
-	gws = p->bo_list->gws_obj;
-	oa = p->bo_list->oa_obj;
+		r = amdgpu_ttm_alloc_gart(&uf->tbo);
+		if (r)
+			goto error_validate;
 
-	if (gds) {
-		p->job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT;
-		p->job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT;
-	}
-	if (gws) {
-		p->job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT;
-		p->job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT;
-	}
-	if (oa) {
-		p->job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT;
-		p->job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT;
+		p->gang_leader->uf_addr += amdgpu_bo_gpu_offset(uf);
 	}
 
-	if (!r && p->uf_entry.tv.bo) {
-		struct amdgpu_bo *uf = ttm_to_amdgpu_bo(p->uf_entry.tv.bo);
+	amdgpu_cs_report_moved_bytes(p->adev, p->bytes_moved,
+				     p->bytes_moved_vis);
 
-		r = amdgpu_ttm_alloc_gart(&uf->tbo);
-		p->job->uf_addr += amdgpu_bo_gpu_offset(uf);
-	}
+	for (i = 0; i < p->gang_size; ++i)
+		amdgpu_job_set_resources(p->jobs[i], p->bo_list->gds_obj,
+					 p->bo_list->gws_obj,
+					 p->bo_list->oa_obj);
+	return 0;
 
 error_validate:
-	if (r)
-		ttm_eu_backoff_reservation(&p->ticket, &p->validated);
+	ttm_eu_backoff_reservation(&p->ticket, &p->validated);
 
 out_free_user_pages:
-	if (r) {
-		amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
-			struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
+	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
+		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
 
-			if (!e->user_pages)
-				continue;
-			amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
-			kvfree(e->user_pages);
-			e->user_pages = NULL;
-		}
-		mutex_unlock(&p->bo_list->bo_list_mutex);
+		if (!e->user_pages)
+			continue;
+		amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm);
+		kvfree(e->user_pages);
+		e->user_pages = NULL;
 	}
 	return r;
 }
 
-static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
+static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *p)
 {
-	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
-	struct amdgpu_bo_list_entry *e;
-	int r;
+	int i, j;
 
-	list_for_each_entry(e, &p->validated, tv.head) {
-		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
-		struct dma_resv *resv = bo->tbo.base.resv;
-		enum amdgpu_sync_mode sync_mode;
+	if (!trace_amdgpu_cs_enabled())
+		return;
 
-		sync_mode = amdgpu_bo_explicit_sync(bo) ?
-			AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER;
-		r = amdgpu_sync_resv(p->adev, &p->job->sync, resv, sync_mode,
-				     &fpriv->vm);
-		if (r)
-			return r;
+	for (i = 0; i < p->gang_size; ++i) {
+		struct amdgpu_job *job = p->jobs[i];
+
+		for (j = 0; j < job->num_ibs; ++j)
+			trace_amdgpu_cs(p, job, &job->ibs[j]);
 	}
-	return 0;
 }
 
-/**
- * amdgpu_cs_parser_fini() - clean parser states
- * @parser:	parser structure holding parsing context.
- * @error:	error number
- * @backoff:	indicator to backoff the reservation
- *
- * If error is set then unvalidate buffer, otherwise just free memory
- * used by parsing context.
- **/
-static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser, int error,
-				  bool backoff)
+static int amdgpu_cs_patch_ibs(struct amdgpu_cs_parser *p,
+			       struct amdgpu_job *job)
 {
-	unsigned i;
+	struct amdgpu_ring *ring = amdgpu_job_ring(job);
+	unsigned int i;
+	int r;
 
-	if (error && backoff) {
-		ttm_eu_backoff_reservation(&parser->ticket,
-					   &parser->validated);
-		mutex_unlock(&parser->bo_list->bo_list_mutex);
-	}
+	/* Only for UVD/VCE VM emulation */
+	if (!ring->funcs->parse_cs && !ring->funcs->patch_cs_in_place)
+		return 0;
 
-	for (i = 0; i < parser->num_post_deps; i++) {
-		drm_syncobj_put(parser->post_deps[i].syncobj);
-		kfree(parser->post_deps[i].chain);
-	}
-	kfree(parser->post_deps);
+	for (i = 0; i < job->num_ibs; ++i) {
+		struct amdgpu_ib *ib = &job->ibs[i];
+		struct amdgpu_bo_va_mapping *m;
+		struct amdgpu_bo *aobj;
+		uint64_t va_start;
+		uint8_t *kptr;
 
-	dma_fence_put(parser->fence);
+		va_start = ib->gpu_addr & AMDGPU_GMC_HOLE_MASK;
+		r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
+		if (r) {
+			DRM_ERROR("IB va_start is invalid\n");
+			return r;
+		}
 
-	if (parser->ctx) {
-		mutex_unlock(&parser->ctx->lock);
-		amdgpu_ctx_put(parser->ctx);
+		if ((va_start + ib->length_dw * 4) >
+		    (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
+			DRM_ERROR("IB va_start+ib_bytes is invalid\n");
+			return -EINVAL;
+		}
+
+		/* the IB should be reserved at this point */
+		r = amdgpu_bo_kmap(aobj, (void **)&kptr);
+		if (r) {
+			return r;
+		}
+
+		kptr += va_start - (m->start * AMDGPU_GPU_PAGE_SIZE);
+
+		if (ring->funcs->parse_cs) {
+			memcpy(ib->ptr, kptr, ib->length_dw * 4);
+			amdgpu_bo_kunmap(aobj);
+
+			r = amdgpu_ring_parse_cs(ring, p, job, ib);
+			if (r)
+				return r;
+		} else {
+			ib->ptr = (uint32_t *)kptr;
+			r = amdgpu_ring_patch_cs_in_place(ring, p, job, ib);
+			amdgpu_bo_kunmap(aobj);
+			if (r)
+				return r;
+		}
 	}
-	if (parser->bo_list)
-		amdgpu_bo_list_put(parser->bo_list);
 
-	for (i = 0; i < parser->nchunks; i++)
-		kvfree(parser->chunks[i].kdata);
-	kvfree(parser->chunks);
-	if (parser->job)
-		amdgpu_job_free(parser->job);
-	if (parser->uf_entry.tv.bo) {
-		struct amdgpu_bo *uf = ttm_to_amdgpu_bo(parser->uf_entry.tv.bo);
+	return 0;
+}
 
-		amdgpu_bo_unref(&uf);
+static int amdgpu_cs_patch_jobs(struct amdgpu_cs_parser *p)
+{
+	unsigned int i;
+	int r;
+
+	for (i = 0; i < p->gang_size; ++i) {
+		r = amdgpu_cs_patch_ibs(p, p->jobs[i]);
+		if (r)
+			return r;
 	}
+	return 0;
 }
 
 static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 {
-	struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched);
 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
+	struct amdgpu_job *job = p->gang_leader;
 	struct amdgpu_device *adev = p->adev;
 	struct amdgpu_vm *vm = &fpriv->vm;
 	struct amdgpu_bo_list_entry *e;
 	struct amdgpu_bo_va *bo_va;
 	struct amdgpu_bo *bo;
+	unsigned int i;
 	int r;
 
-	/* Only for UVD/VCE VM emulation */
-	if (ring->funcs->parse_cs || ring->funcs->patch_cs_in_place) {
-		unsigned i, j;
-
-		for (i = 0, j = 0; i < p->nchunks && j < p->job->num_ibs; i++) {
-			struct drm_amdgpu_cs_chunk_ib *chunk_ib;
-			struct amdgpu_bo_va_mapping *m;
-			struct amdgpu_bo *aobj = NULL;
-			struct amdgpu_cs_chunk *chunk;
-			uint64_t offset, va_start;
-			struct amdgpu_ib *ib;
-			uint8_t *kptr;
-
-			chunk = &p->chunks[i];
-			ib = &p->job->ibs[j];
-			chunk_ib = chunk->kdata;
-
-			if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
-				continue;
-
-			va_start = chunk_ib->va_start & AMDGPU_GMC_HOLE_MASK;
-			r = amdgpu_cs_find_mapping(p, va_start, &aobj, &m);
-			if (r) {
-				DRM_ERROR("IB va_start is invalid\n");
-				return r;
-			}
-
-			if ((va_start + chunk_ib->ib_bytes) >
-			    (m->last + 1) * AMDGPU_GPU_PAGE_SIZE) {
-				DRM_ERROR("IB va_start+ib_bytes is invalid\n");
-				return -EINVAL;
-			}
-
-			/* the IB should be reserved at this point */
-			r = amdgpu_bo_kmap(aobj, (void **)&kptr);
-			if (r) {
-				return r;
-			}
-
-			offset = m->start * AMDGPU_GPU_PAGE_SIZE;
-			kptr += va_start - offset;
-
-			if (ring->funcs->parse_cs) {
-				memcpy(ib->ptr, kptr, chunk_ib->ib_bytes);
-				amdgpu_bo_kunmap(aobj);
-
-				r = amdgpu_ring_parse_cs(ring, p, p->job, ib);
-				if (r)
-					return r;
-			} else {
-				ib->ptr = (uint32_t *)kptr;
-				r = amdgpu_ring_patch_cs_in_place(ring, p, p->job, ib);
-				amdgpu_bo_kunmap(aobj);
-				if (r)
-					return r;
-			}
-
-			j++;
-		}
-	}
-
-	if (!p->job->vm)
-		return amdgpu_cs_sync_rings(p);
-
-
 	r = amdgpu_vm_clear_freed(adev, vm, NULL);
 	if (r)
 		return r;
@@ -810,18 +1101,18 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 	if (r)
 		return r;
 
-	r = amdgpu_sync_fence(&p->job->sync, fpriv->prt_va->last_pt_update);
+	r = amdgpu_sync_fence(&job->sync, fpriv->prt_va->last_pt_update);
 	if (r)
 		return r;
 
-	if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
+	if (fpriv->csa_va) {
 		bo_va = fpriv->csa_va;
 		BUG_ON(!bo_va);
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (r)
 			return r;
 
-		r = amdgpu_sync_fence(&p->job->sync, bo_va->last_pt_update);
+		r = amdgpu_sync_fence(&job->sync, bo_va->last_pt_update);
 		if (r)
 			return r;
 	}
@@ -840,7 +1131,7 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 		if (r)
 			return r;
 
-		r = amdgpu_sync_fence(&p->job->sync, bo_va->last_pt_update);
+		r = amdgpu_sync_fence(&job->sync, bo_va->last_pt_update);
 		if (r)
 			return r;
 	}
@@ -853,11 +1144,18 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 	if (r)
 		return r;
 
-	r = amdgpu_sync_fence(&p->job->sync, vm->last_update);
+	r = amdgpu_sync_fence(&job->sync, vm->last_update);
 	if (r)
 		return r;
 
-	p->job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo);
+	for (i = 0; i < p->gang_size; ++i) {
+		job = p->jobs[i];
+
+		if (!job->vm)
+			continue;
+
+		job->vm_pd_addr = amdgpu_gmc_pd_addr(vm->root.bo);
+	}
 
 	if (amdgpu_vm_debug) {
 		/* Invalidate all BOs to test for userspace bugs */
@@ -872,331 +1170,40 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 		}
 	}
 
-	return amdgpu_cs_sync_rings(p);
-}
-
-static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
-			     struct amdgpu_cs_parser *parser)
-{
-	struct amdgpu_fpriv *fpriv = parser->filp->driver_priv;
-	struct amdgpu_vm *vm = &fpriv->vm;
-	int r, ce_preempt = 0, de_preempt = 0;
-	struct amdgpu_ring *ring;
-	int i, j;
-
-	for (i = 0, j = 0; i < parser->nchunks && j < parser->job->num_ibs; i++) {
-		struct amdgpu_cs_chunk *chunk;
-		struct amdgpu_ib *ib;
-		struct drm_amdgpu_cs_chunk_ib *chunk_ib;
-		struct drm_sched_entity *entity;
-
-		chunk = &parser->chunks[i];
-		ib = &parser->job->ibs[j];
-		chunk_ib = (struct drm_amdgpu_cs_chunk_ib *)chunk->kdata;
-
-		if (chunk->chunk_id != AMDGPU_CHUNK_ID_IB)
-			continue;
-
-		if (chunk_ib->ip_type == AMDGPU_HW_IP_GFX &&
-		    (amdgpu_mcbp || amdgpu_sriov_vf(adev))) {
-			if (chunk_ib->flags & AMDGPU_IB_FLAG_PREEMPT) {
-				if (chunk_ib->flags & AMDGPU_IB_FLAG_CE)
-					ce_preempt++;
-				else
-					de_preempt++;
-			}
-
-			/* each GFX command submit allows 0 or 1 IB preemptible for CE & DE */
-			if (ce_preempt > 1 || de_preempt > 1)
-				return -EINVAL;
-		}
-
-		r = amdgpu_ctx_get_entity(parser->ctx, chunk_ib->ip_type,
-					  chunk_ib->ip_instance, chunk_ib->ring,
-					  &entity);
-		if (r)
-			return r;
-
-		if (chunk_ib->flags & AMDGPU_IB_FLAG_PREAMBLE)
-			parser->job->preamble_status |=
-				AMDGPU_PREAMBLE_IB_PRESENT;
-
-		if (parser->entity && parser->entity != entity)
-			return -EINVAL;
-
-		/* Return if there is no run queue associated with this entity.
-		 * Possibly because of disabled HW IP*/
-		if (entity->rq == NULL)
-			return -EINVAL;
-
-		parser->entity = entity;
-
-		ring = to_amdgpu_ring(entity->rq->sched);
-		r =  amdgpu_ib_get(adev, vm, ring->funcs->parse_cs ?
-				   chunk_ib->ib_bytes : 0,
-				   AMDGPU_IB_POOL_DELAYED, ib);
-		if (r) {
-			DRM_ERROR("Failed to get ib !\n");
-			return r;
-		}
-
-		ib->gpu_addr = chunk_ib->va_start;
-		ib->length_dw = chunk_ib->ib_bytes / 4;
-		ib->flags = chunk_ib->flags;
-
-		j++;
-	}
-
-	/* MM engine doesn't support user fences */
-	ring = to_amdgpu_ring(parser->entity->rq->sched);
-	if (parser->job->uf_addr && ring->funcs->no_user_fence)
-		return -EINVAL;
-
 	return 0;
 }
 
-static int amdgpu_cs_process_fence_dep(struct amdgpu_cs_parser *p,
-				       struct amdgpu_cs_chunk *chunk)
+static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
 {
 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
-	unsigned num_deps;
-	int i, r;
-	struct drm_amdgpu_cs_chunk_dep *deps;
-
-	deps = (struct drm_amdgpu_cs_chunk_dep *)chunk->kdata;
-	num_deps = chunk->length_dw * 4 /
-		sizeof(struct drm_amdgpu_cs_chunk_dep);
-
-	for (i = 0; i < num_deps; ++i) {
-		struct amdgpu_ctx *ctx;
-		struct drm_sched_entity *entity;
-		struct dma_fence *fence;
-
-		ctx = amdgpu_ctx_get(fpriv, deps[i].ctx_id);
-		if (ctx == NULL)
-			return -EINVAL;
-
-		r = amdgpu_ctx_get_entity(ctx, deps[i].ip_type,
-					  deps[i].ip_instance,
-					  deps[i].ring, &entity);
-		if (r) {
-			amdgpu_ctx_put(ctx);
-			return r;
-		}
-
-		fence = amdgpu_ctx_get_fence(ctx, entity, deps[i].handle);
-		amdgpu_ctx_put(ctx);
-
-		if (IS_ERR(fence))
-			return PTR_ERR(fence);
-		else if (!fence)
-			continue;
-
-		if (chunk->chunk_id == AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES) {
-			struct drm_sched_fence *s_fence;
-			struct dma_fence *old = fence;
-
-			s_fence = to_drm_sched_fence(fence);
-			fence = dma_fence_get(&s_fence->scheduled);
-			dma_fence_put(old);
-		}
-
-		r = amdgpu_sync_fence(&p->job->sync, fence);
-		dma_fence_put(fence);
-		if (r)
-			return r;
-	}
-	return 0;
-}
-
-static int amdgpu_syncobj_lookup_and_add_to_sync(struct amdgpu_cs_parser *p,
-						 uint32_t handle, u64 point,
-						 u64 flags)
-{
-	struct dma_fence *fence;
+	struct amdgpu_job *leader = p->gang_leader;
+	struct amdgpu_bo_list_entry *e;
+	unsigned int i;
 	int r;
 
-	r = drm_syncobj_find_fence(p->filp, handle, point, flags, &fence);
-	if (r) {
-		DRM_ERROR("syncobj %u failed to find fence @ %llu (%d)!\n",
-			  handle, point, r);
-		return r;
-	}
-
-	r = amdgpu_sync_fence(&p->job->sync, fence);
-	dma_fence_put(fence);
-
-	return r;
-}
-
-static int amdgpu_cs_process_syncobj_in_dep(struct amdgpu_cs_parser *p,
-					    struct amdgpu_cs_chunk *chunk)
-{
-	struct drm_amdgpu_cs_chunk_sem *deps;
-	unsigned num_deps;
-	int i, r;
+	list_for_each_entry(e, &p->validated, tv.head) {
+		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
+		struct dma_resv *resv = bo->tbo.base.resv;
+		enum amdgpu_sync_mode sync_mode;
 
-	deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata;
-	num_deps = chunk->length_dw * 4 /
-		sizeof(struct drm_amdgpu_cs_chunk_sem);
-	for (i = 0; i < num_deps; ++i) {
-		r = amdgpu_syncobj_lookup_and_add_to_sync(p, deps[i].handle,
-							  0, 0);
+		sync_mode = amdgpu_bo_explicit_sync(bo) ?
+			AMDGPU_SYNC_EXPLICIT : AMDGPU_SYNC_NE_OWNER;
+		r = amdgpu_sync_resv(p->adev, &leader->sync, resv, sync_mode,
+				     &fpriv->vm);
 		if (r)
 			return r;
 	}
 
-	return 0;
-}
-
-
-static int amdgpu_cs_process_syncobj_timeline_in_dep(struct amdgpu_cs_parser *p,
-						     struct amdgpu_cs_chunk *chunk)
-{
-	struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps;
-	unsigned num_deps;
-	int i, r;
-
-	syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata;
-	num_deps = chunk->length_dw * 4 /
-		sizeof(struct drm_amdgpu_cs_chunk_syncobj);
-	for (i = 0; i < num_deps; ++i) {
-		r = amdgpu_syncobj_lookup_and_add_to_sync(p,
-							  syncobj_deps[i].handle,
-							  syncobj_deps[i].point,
-							  syncobj_deps[i].flags);
+	for (i = 0; i < p->gang_size - 1; ++i) {
+		r = amdgpu_sync_clone(&leader->sync, &p->jobs[i]->sync);
 		if (r)
 			return r;
 	}
 
-	return 0;
-}
-
-static int amdgpu_cs_process_syncobj_out_dep(struct amdgpu_cs_parser *p,
-					     struct amdgpu_cs_chunk *chunk)
-{
-	struct drm_amdgpu_cs_chunk_sem *deps;
-	unsigned num_deps;
-	int i;
-
-	deps = (struct drm_amdgpu_cs_chunk_sem *)chunk->kdata;
-	num_deps = chunk->length_dw * 4 /
-		sizeof(struct drm_amdgpu_cs_chunk_sem);
-
-	if (p->post_deps)
-		return -EINVAL;
-
-	p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
-				     GFP_KERNEL);
-	p->num_post_deps = 0;
-
-	if (!p->post_deps)
-		return -ENOMEM;
-
-
-	for (i = 0; i < num_deps; ++i) {
-		p->post_deps[i].syncobj =
-			drm_syncobj_find(p->filp, deps[i].handle);
-		if (!p->post_deps[i].syncobj)
-			return -EINVAL;
-		p->post_deps[i].chain = NULL;
-		p->post_deps[i].point = 0;
-		p->num_post_deps++;
-	}
-
-	return 0;
-}
-
-
-static int amdgpu_cs_process_syncobj_timeline_out_dep(struct amdgpu_cs_parser *p,
-						      struct amdgpu_cs_chunk *chunk)
-{
-	struct drm_amdgpu_cs_chunk_syncobj *syncobj_deps;
-	unsigned num_deps;
-	int i;
-
-	syncobj_deps = (struct drm_amdgpu_cs_chunk_syncobj *)chunk->kdata;
-	num_deps = chunk->length_dw * 4 /
-		sizeof(struct drm_amdgpu_cs_chunk_syncobj);
-
-	if (p->post_deps)
-		return -EINVAL;
-
-	p->post_deps = kmalloc_array(num_deps, sizeof(*p->post_deps),
-				     GFP_KERNEL);
-	p->num_post_deps = 0;
-
-	if (!p->post_deps)
-		return -ENOMEM;
-
-	for (i = 0; i < num_deps; ++i) {
-		struct amdgpu_cs_post_dep *dep = &p->post_deps[i];
-
-		dep->chain = NULL;
-		if (syncobj_deps[i].point) {
-			dep->chain = dma_fence_chain_alloc();
-			if (!dep->chain)
-				return -ENOMEM;
-		}
-
-		dep->syncobj = drm_syncobj_find(p->filp,
-						syncobj_deps[i].handle);
-		if (!dep->syncobj) {
-			dma_fence_chain_free(dep->chain);
-			return -EINVAL;
-		}
-		dep->point = syncobj_deps[i].point;
-		p->num_post_deps++;
-	}
-
-	return 0;
-}
-
-static int amdgpu_cs_dependencies(struct amdgpu_device *adev,
-				  struct amdgpu_cs_parser *p)
-{
-	int i, r;
-
-	/* TODO: Investigate why we still need the context lock */
-	mutex_unlock(&p->ctx->lock);
-
-	for (i = 0; i < p->nchunks; ++i) {
-		struct amdgpu_cs_chunk *chunk;
-
-		chunk = &p->chunks[i];
+	r = amdgpu_ctx_wait_prev_fence(p->ctx, p->entities[p->gang_size - 1]);
+	if (r && r != -ERESTARTSYS)
+		DRM_ERROR("amdgpu_ctx_wait_prev_fence failed.\n");
 
-		switch (chunk->chunk_id) {
-		case AMDGPU_CHUNK_ID_DEPENDENCIES:
-		case AMDGPU_CHUNK_ID_SCHEDULED_DEPENDENCIES:
-			r = amdgpu_cs_process_fence_dep(p, chunk);
-			if (r)
-				goto out;
-			break;
-		case AMDGPU_CHUNK_ID_SYNCOBJ_IN:
-			r = amdgpu_cs_process_syncobj_in_dep(p, chunk);
-			if (r)
-				goto out;
-			break;
-		case AMDGPU_CHUNK_ID_SYNCOBJ_OUT:
-			r = amdgpu_cs_process_syncobj_out_dep(p, chunk);
-			if (r)
-				goto out;
-			break;
-		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_WAIT:
-			r = amdgpu_cs_process_syncobj_timeline_in_dep(p, chunk);
-			if (r)
-				goto out;
-			break;
-		case AMDGPU_CHUNK_ID_SYNCOBJ_TIMELINE_SIGNAL:
-			r = amdgpu_cs_process_syncobj_timeline_out_dep(p, chunk);
-			if (r)
-				goto out;
-			break;
-		}
-	}
-
-out:
-	mutex_lock(&p->ctx->lock);
 	return r;
 }
 
@@ -1221,20 +1228,28 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 			    union drm_amdgpu_cs *cs)
 {
 	struct amdgpu_fpriv *fpriv = p->filp->driver_priv;
-	struct drm_sched_entity *entity = p->entity;
+	struct amdgpu_job *leader = p->gang_leader;
 	struct amdgpu_bo_list_entry *e;
-	struct amdgpu_job *job;
+	unsigned int i;
 	uint64_t seq;
 	int r;
 
-	job = p->job;
-	p->job = NULL;
+	for (i = 0; i < p->gang_size; ++i)
+		drm_sched_job_arm(&p->jobs[i]->base);
 
-	r = drm_sched_job_init(&job->base, entity, &fpriv->vm);
-	if (r)
-		goto error_unlock;
+	for (i = 0; i < (p->gang_size - 1); ++i) {
+		struct dma_fence *fence;
 
-	drm_sched_job_arm(&job->base);
+		fence = &p->jobs[i]->base.s_fence->scheduled;
+		r = amdgpu_sync_fence(&leader->sync, fence);
+		if (r)
+			goto error_cleanup;
+	}
+
+	if (p->gang_size > 1) {
+		for (i = 0; i < p->gang_size; ++i)
+			amdgpu_job_set_gang_leader(p->jobs[i], leader);
+	}
 
 	/* No memory allocation is allowed while holding the notifier lock.
 	 * The lock is held until amdgpu_cs_submit is finished and fence is
@@ -1245,6 +1260,7 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 	/* If userptr are invalidated after amdgpu_cs_parser_bos(), return
 	 * -EAGAIN, drmIoctl in libdrm will restart the amdgpu_cs_ioctl.
 	 */
+	r = 0;
 	amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) {
 		struct amdgpu_bo *bo = ttm_to_amdgpu_bo(e->tv.bo);
 
@@ -1252,67 +1268,96 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 	}
 	if (r) {
 		r = -EAGAIN;
-		goto error_abort;
+		goto error_unlock;
 	}
 
-	p->fence = dma_fence_get(&job->base.s_fence->finished);
+	p->fence = dma_fence_get(&leader->base.s_fence->finished);
+	list_for_each_entry(e, &p->validated, tv.head) {
+
+		/* Everybody except for the gang leader uses READ */
+		for (i = 0; i < (p->gang_size - 1); ++i) {
+			dma_resv_add_fence(e->tv.bo->base.resv,
+					   &p->jobs[i]->base.s_fence->finished,
+					   DMA_RESV_USAGE_READ);
+		}
 
-	seq = amdgpu_ctx_add_fence(p->ctx, entity, p->fence);
+		/* The gang leader is remembered as writer */
+		e->tv.num_shared = 0;
+	}
+
+	seq = amdgpu_ctx_add_fence(p->ctx, p->entities[p->gang_size - 1],
+				   p->fence);
 	amdgpu_cs_post_dependencies(p);
 
-	if ((job->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) &&
+	if ((leader->preamble_status & AMDGPU_PREAMBLE_IB_PRESENT) &&
 	    !p->ctx->preamble_presented) {
-		job->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
+		leader->preamble_status |= AMDGPU_PREAMBLE_IB_PRESENT_FIRST;
 		p->ctx->preamble_presented = true;
 	}
 
 	cs->out.handle = seq;
-	job->uf_sequence = seq;
-
-	amdgpu_job_free_resources(job);
+	leader->uf_sequence = seq;
 
-	trace_amdgpu_cs_ioctl(job);
 	amdgpu_vm_bo_trace_cs(&fpriv->vm, &p->ticket);
-	drm_sched_entity_push_job(&job->base);
+	for (i = 0; i < p->gang_size; ++i) {
+		amdgpu_job_free_resources(p->jobs[i]);
+		trace_amdgpu_cs_ioctl(p->jobs[i]);
+		drm_sched_entity_push_job(&p->jobs[i]->base);
+		p->jobs[i] = NULL;
+	}
 
 	amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm);
-
-	/* Make sure all BOs are remembered as writers */
-	amdgpu_bo_list_for_each_entry(e, p->bo_list)
-		e->tv.num_shared = 0;
-
 	ttm_eu_fence_buffer_objects(&p->ticket, &p->validated, p->fence);
+
 	mutex_unlock(&p->adev->notifier_lock);
 	mutex_unlock(&p->bo_list->bo_list_mutex);
-
 	return 0;
 
-error_abort:
-	drm_sched_job_cleanup(&job->base);
+error_unlock:
 	mutex_unlock(&p->adev->notifier_lock);
 
-error_unlock:
-	amdgpu_job_free(job);
+error_cleanup:
+	for (i = 0; i < p->gang_size; ++i)
+		drm_sched_job_cleanup(&p->jobs[i]->base);
 	return r;
 }
 
-static void trace_amdgpu_cs_ibs(struct amdgpu_cs_parser *parser)
+/* Cleanup the parser structure */
+static void amdgpu_cs_parser_fini(struct amdgpu_cs_parser *parser)
 {
-	int i;
+	unsigned i;
 
-	if (!trace_amdgpu_cs_enabled())
-		return;
+	for (i = 0; i < parser->num_post_deps; i++) {
+		drm_syncobj_put(parser->post_deps[i].syncobj);
+		kfree(parser->post_deps[i].chain);
+	}
+	kfree(parser->post_deps);
+
+	dma_fence_put(parser->fence);
+
+	if (parser->ctx)
+		amdgpu_ctx_put(parser->ctx);
+	if (parser->bo_list)
+		amdgpu_bo_list_put(parser->bo_list);
+
+	for (i = 0; i < parser->nchunks; i++)
+		kvfree(parser->chunks[i].kdata);
+	kvfree(parser->chunks);
+	for (i = 0; i < parser->gang_size; ++i) {
+		if (parser->jobs[i])
+			amdgpu_job_free(parser->jobs[i]);
+	}
+	if (parser->uf_entry.tv.bo) {
+		struct amdgpu_bo *uf = ttm_to_amdgpu_bo(parser->uf_entry.tv.bo);
 
-	for (i = 0; i < parser->job->num_ibs; i++)
-		trace_amdgpu_cs(parser, i);
+		amdgpu_bo_unref(&uf);
+	}
 }
 
 int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
-	union drm_amdgpu_cs *cs = data;
-	struct amdgpu_cs_parser parser = {};
-	bool reserved_buffers = false;
+	struct amdgpu_cs_parser parser;
 	int r;
 
 	if (amdgpu_ras_intr_triggered())
@@ -1321,25 +1366,20 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 	if (!adev->accel_working)
 		return -EBUSY;
 
-	parser.adev = adev;
-	parser.filp = filp;
-
-	r = amdgpu_cs_parser_init(&parser, data);
+	r = amdgpu_cs_parser_init(&parser, adev, filp, data);
 	if (r) {
 		if (printk_ratelimit())
 			DRM_ERROR("Failed to initialize parser %d!\n", r);
-		goto out;
+		return r;
 	}
 
-	r = amdgpu_cs_ib_fill(adev, &parser);
+	r = amdgpu_cs_pass1(&parser, data);
 	if (r)
-		goto out;
+		goto error_fini;
 
-	r = amdgpu_cs_dependencies(adev, &parser);
-	if (r) {
-		DRM_ERROR("Failed in the dependencies handling %d!\n", r);
-		goto out;
-	}
+	r = amdgpu_cs_pass2(&parser);
+	if (r)
+		goto error_fini;
 
 	r = amdgpu_cs_parser_bos(&parser, data);
 	if (r) {
@@ -1347,22 +1387,36 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 			DRM_ERROR("Not enough memory for command submission!\n");
 		else if (r != -ERESTARTSYS && r != -EAGAIN)
 			DRM_ERROR("Failed to process the buffer list %d!\n", r);
-		goto out;
+		goto error_fini;
 	}
 
-	reserved_buffers = true;
+	r = amdgpu_cs_patch_jobs(&parser);
+	if (r)
+		goto error_backoff;
+
+	r = amdgpu_cs_vm_handling(&parser);
+	if (r)
+		goto error_backoff;
+
+	r = amdgpu_cs_sync_rings(&parser);
+	if (r)
+		goto error_backoff;
 
 	trace_amdgpu_cs_ibs(&parser);
 
-	r = amdgpu_cs_vm_handling(&parser);
+	r = amdgpu_cs_submit(&parser, data);
 	if (r)
-		goto out;
+		goto error_backoff;
 
-	r = amdgpu_cs_submit(&parser, cs);
+	amdgpu_cs_parser_fini(&parser);
+	return 0;
 
-out:
-	amdgpu_cs_parser_fini(&parser, r, reserved_buffers);
+error_backoff:
+	ttm_eu_backoff_reservation(&parser.ticket, &parser.validated);
+	mutex_unlock(&parser.bo_list->bo_list_mutex);
 
+error_fini:
+	amdgpu_cs_parser_fini(&parser);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.h
index 30ecc4917f81..cbaa19b2b8a3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.h
@@ -27,6 +27,8 @@
 #include "amdgpu_bo_list.h"
 #include "amdgpu_ring.h"
 
+#define AMDGPU_CS_GANG_SIZE	4
+
 struct amdgpu_bo_va_mapping;
 
 struct amdgpu_cs_chunk {
@@ -50,9 +52,11 @@ struct amdgpu_cs_parser {
 	unsigned		nchunks;
 	struct amdgpu_cs_chunk	*chunks;
 
-	/* scheduler job object */
-	struct amdgpu_job	*job;
-	struct drm_sched_entity	*entity;
+	/* scheduler job objects */
+	unsigned int		gang_size;
+	struct drm_sched_entity	*entities[AMDGPU_CS_GANG_SIZE];
+	struct amdgpu_job	*jobs[AMDGPU_CS_GANG_SIZE];
+	struct amdgpu_job	*gang_leader;
 
 	/* buffer objects */
 	struct ww_acquire_ctx		ticket;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 8ee4e8491f39..f6d9d5da53cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -315,7 +315,6 @@ static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority,
 	kref_init(&ctx->refcount);
 	ctx->mgr = mgr;
 	spin_lock_init(&ctx->ring_lock);
-	mutex_init(&ctx->lock);
 
 	ctx->reset_counter = atomic_read(&mgr->adev->gpu_reset_counter);
 	ctx->reset_counter_query = ctx->reset_counter;
@@ -402,12 +401,11 @@ static void amdgpu_ctx_fini(struct kref *ref)
 		}
 	}
 
-	if (drm_dev_enter(&adev->ddev, &idx)) {
+	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
 		amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate);
 		drm_dev_exit(idx);
 	}
 
-	mutex_destroy(&ctx->lock);
 	kfree(ctx);
 }
 
@@ -848,7 +846,7 @@ void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr,
 
 	mgr->adev = adev;
 	mutex_init(&mgr->lock);
-	idr_init(&mgr->ctx_handles);
+	idr_init_base(&mgr->ctx_handles, 1);
 
 	for (i = 0; i < AMDGPU_HW_IP_NUM; ++i)
 		atomic64_set(&mgr->time_spend[i], 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
index cc7c8afff414..0fa0e56daf67 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.h
@@ -53,7 +53,6 @@ struct amdgpu_ctx {
 	bool				preamble_presented;
 	int32_t				init_priority;
 	int32_t				override_priority;
-	struct mutex			lock;
 	atomic_t			guilty;
 	unsigned long			ras_counter_ce;
 	unsigned long			ras_counter_ue;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index cb00c7d6f50b..6066aebf491c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -1043,6 +1043,157 @@ err:
 }
 
 /**
+ * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
+ *
+ * @f: open file handle
+ * @buf: User buffer to store read data in
+ * @size: Number of bytes to read
+ * @pos:  Offset to seek to
+ *
+ * Read the last residency value logged. It doesn't auto update, one needs to
+ * stop logging before getting the current value.
+ */
+static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
+						    size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+	if (r < 0) {
+		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+		return r;
+	}
+
+	while (size) {
+		uint32_t value;
+
+		r = amdgpu_get_gfx_off_residency(adev, &value);
+		if (r)
+			goto out;
+
+		r = put_user(value, (uint32_t *)buf);
+		if (r)
+			goto out;
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	r = result;
+out:
+	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+
+	return r;
+}
+
+/**
+ * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
+ *
+ * @f: open file handle
+ * @buf: User buffer to write data from
+ * @size: Number of bytes to write
+ * @pos:  Offset to seek to
+ *
+ * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
+ */
+static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
+						     size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+	if (r < 0) {
+		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+		return r;
+	}
+
+	while (size) {
+		u32 value;
+
+		r = get_user(value, (uint32_t *)buf);
+		if (r)
+			goto out;
+
+		amdgpu_set_gfx_off_residency(adev, value ? true : false);
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	r = result;
+out:
+	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+
+	return r;
+}
+
+
+/**
+ * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
+ *
+ * @f: open file handle
+ * @buf: User buffer to store read data in
+ * @size: Number of bytes to read
+ * @pos:  Offset to seek to
+ */
+static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
+						size_t size, loff_t *pos)
+{
+	struct amdgpu_device *adev = file_inode(f)->i_private;
+	ssize_t result = 0;
+	int r;
+
+	if (size & 0x3 || *pos & 0x3)
+		return -EINVAL;
+
+	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
+	if (r < 0) {
+		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+		return r;
+	}
+
+	while (size) {
+		u64 value = 0;
+
+		r = amdgpu_get_gfx_off_entrycount(adev, &value);
+		if (r)
+			goto out;
+
+		r = put_user(value, (u64 *)buf);
+		if (r)
+			goto out;
+
+		result += 4;
+		buf += 4;
+		*pos += 4;
+		size -= 4;
+	}
+
+	r = result;
+out:
+	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
+	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
+
+	return r;
+}
+
+/**
  * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
  *
  * @f: open file handle
@@ -1249,6 +1400,19 @@ static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
 	.llseek = default_llseek
 };
 
+static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_gfxoff_count_read,
+	.llseek = default_llseek
+};
+
+static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
+	.owner = THIS_MODULE,
+	.read = amdgpu_debugfs_gfxoff_residency_read,
+	.write = amdgpu_debugfs_gfxoff_residency_write,
+	.llseek = default_llseek
+};
+
 static const struct file_operations *debugfs_regs[] = {
 	&amdgpu_debugfs_regs_fops,
 	&amdgpu_debugfs_regs2_fops,
@@ -1261,6 +1425,8 @@ static const struct file_operations *debugfs_regs[] = {
 	&amdgpu_debugfs_gpr_fops,
 	&amdgpu_debugfs_gfxoff_fops,
 	&amdgpu_debugfs_gfxoff_status_fops,
+	&amdgpu_debugfs_gfxoff_count_fops,
+	&amdgpu_debugfs_gfxoff_residency_fops,
 };
 
 static const char *debugfs_regs_names[] = {
@@ -1275,6 +1441,8 @@ static const char *debugfs_regs_names[] = {
 	"amdgpu_gpr",
 	"amdgpu_gfxoff",
 	"amdgpu_gfxoff_status",
+	"amdgpu_gfxoff_count",
+	"amdgpu_gfxoff_residency",
 };
 
 /**
@@ -1786,6 +1954,8 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)
 		return PTR_ERR(ent);
 	}
 
+	debugfs_create_u32("amdgpu_reset_level", 0600, root, &adev->amdgpu_reset_level_mask);
+
 	/* Register debugfs entries for amdgpu_ttm */
 	amdgpu_ttm_debugfs_init(adev);
 	amdgpu_debugfs_pm_init(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index be7aff2d4a57..ab8f970b2849 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2459,19 +2459,21 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 	 */
 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
 		if (amdgpu_xgmi_add_device(adev) == 0) {
-			struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
+			if (!amdgpu_sriov_vf(adev)) {
+				struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
 
-			if (!hive->reset_domain ||
-			    !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
-				r = -ENOENT;
+				if (!hive->reset_domain ||
+				    !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
+					r = -ENOENT;
+					amdgpu_put_xgmi_hive(hive);
+					goto init_failed;
+				}
+
+				/* Drop the early temporary reset domain we created for device */
+				amdgpu_reset_put_reset_domain(adev->reset_domain);
+				adev->reset_domain = hive->reset_domain;
 				amdgpu_put_xgmi_hive(hive);
-				goto init_failed;
 			}
-
-			/* Drop the early temporary reset domain we created for device */
-			amdgpu_reset_put_reset_domain(adev->reset_domain);
-			adev->reset_domain = hive->reset_domain;
-			amdgpu_put_xgmi_hive(hive);
 		}
 	}
 
@@ -3152,7 +3154,8 @@ static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
 			continue;
 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
-		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
+		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
+		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
 
 			r = adev->ip_blocks[i].version->funcs->resume(adev);
 			if (r) {
@@ -3509,6 +3512,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	adev->gmc.gart_size = 512 * 1024 * 1024;
 	adev->accel_working = false;
 	adev->num_rings = 0;
+	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
 	adev->mman.buffer_funcs = NULL;
 	adev->mman.buffer_funcs_ring = NULL;
 	adev->vm_manager.vm_pte_funcs = NULL;
@@ -3587,6 +3591,8 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
 
 	adev->gfx.gfx_off_req_count = 1;
+	adev->gfx.gfx_off_residency = 0;
+	adev->gfx.gfx_off_entrycount = 0;
 	adev->pm.ac_power = power_supply_is_system_supplied() > 0;
 
 	atomic_set(&adev->throttling_logging_enabled, 1);
@@ -3975,8 +3981,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
 
 	amdgpu_gart_dummy_page_fini(adev);
 
-	if (drm_dev_is_unplugged(adev_to_drm(adev)))
-		amdgpu_device_unmap_mmio(adev);
+	amdgpu_device_unmap_mmio(adev);
 
 }
 
@@ -3989,6 +3994,7 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
 	release_firmware(adev->firmware.gpu_info_fw);
 	adev->firmware.gpu_info_fw = NULL;
 	adev->accel_working = false;
+	dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
 
 	amdgpu_reset_fini(adev);
 
@@ -4064,12 +4070,20 @@ static void amdgpu_device_evict_resources(struct amdgpu_device *adev)
 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
+	int r = 0;
 
 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
 	adev->in_suspend = true;
 
+	if (amdgpu_sriov_vf(adev)) {
+		amdgpu_virt_fini_data_exchange(adev);
+		r = amdgpu_virt_request_full_gpu(adev, false);
+		if (r)
+			return r;
+	}
+
 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
 		DRM_WARN("smart shift update failed\n");
 
@@ -4093,6 +4107,9 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
 
 	amdgpu_device_ip_suspend_phase2(adev);
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_release_full_gpu(adev, false);
+
 	return 0;
 }
 
@@ -4111,6 +4128,12 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
 	struct amdgpu_device *adev = drm_to_adev(dev);
 	int r = 0;
 
+	if (amdgpu_sriov_vf(adev)) {
+		r = amdgpu_virt_request_full_gpu(adev, true);
+		if (r)
+			return r;
+	}
+
 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 
@@ -4125,6 +4148,13 @@ int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
 	}
 
 	r = amdgpu_device_ip_resume(adev);
+
+	/* no matter what r is, always need to properly release full GPU */
+	if (amdgpu_sriov_vf(adev)) {
+		amdgpu_virt_init_data_exchange(adev);
+		amdgpu_virt_release_full_gpu(adev, true);
+	}
+
 	if (r) {
 		dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
 		return r;
@@ -4517,14 +4547,15 @@ bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
  */
 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
 {
-	if (!amdgpu_device_ip_check_soft_reset(adev)) {
-		dev_info(adev->dev, "Timeout, but no hardware hang detected.\n");
-		return false;
-	}
 
 	if (amdgpu_gpu_recovery == 0)
 		goto disabled;
 
+	if (!amdgpu_device_ip_check_soft_reset(adev)) {
+		dev_info(adev->dev,"Timeout, but no hardware hang detected.\n");
+		return false;
+	}
+
 	if (amdgpu_sriov_vf(adev))
 		return true;
 
@@ -4649,7 +4680,7 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 		if (!need_full_reset)
 			need_full_reset = amdgpu_device_ip_need_full_reset(adev);
 
-		if (!need_full_reset) {
+		if (!need_full_reset && amdgpu_gpu_recovery) {
 			amdgpu_device_ip_pre_soft_reset(adev);
 			r = amdgpu_device_ip_soft_reset(adev);
 			amdgpu_device_ip_post_soft_reset(adev);
@@ -4745,6 +4776,7 @@ int amdgpu_do_asic_reset(struct list_head *device_list_handle,
 	struct amdgpu_device *tmp_adev = NULL;
 	bool need_full_reset, skip_hw_reset, vram_lost = false;
 	int r = 0;
+	bool gpu_reset_for_dev_remove = 0;
 
 	/* Try reset handler method first */
 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
@@ -4764,6 +4796,10 @@ int amdgpu_do_asic_reset(struct list_head *device_list_handle,
 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
 	skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
 
+	gpu_reset_for_dev_remove =
+		test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
+			test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
+
 	/*
 	 * ASIC reset has to be done on all XGMI hive nodes ASAP
 	 * to allow proper links negotiation in FW (within 1 sec)
@@ -4808,6 +4844,18 @@ int amdgpu_do_asic_reset(struct list_head *device_list_handle,
 		amdgpu_ras_intr_cleared();
 	}
 
+	/* Since the mode1 reset affects base ip blocks, the
+	 * phase1 ip blocks need to be resumed. Otherwise there
+	 * will be a BIOS signature error and the psp bootloader
+	 * can't load kdb on the next amdgpu install.
+	 */
+	if (gpu_reset_for_dev_remove) {
+		list_for_each_entry(tmp_adev, device_list_handle, reset_list)
+			amdgpu_device_ip_resume_phase1(tmp_adev);
+
+		goto end;
+	}
+
 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
 		if (need_full_reset) {
 			/* post card */
@@ -5047,6 +5095,7 @@ static void amdgpu_device_recheck_guilty_jobs(
 
 			/* set guilty */
 			drm_sched_increase_karma(s_job);
+			amdgpu_reset_prepare_hwcontext(adev, reset_context);
 retry:
 			/* do hw reset */
 			if (amdgpu_sriov_vf(adev)) {
@@ -5129,6 +5178,11 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	bool need_emergency_restart = false;
 	bool audio_suspended = false;
 	int tmp_vram_lost_counter;
+	bool gpu_reset_for_dev_remove = false;
+
+	gpu_reset_for_dev_remove =
+			test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
+				test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
 
 	/*
 	 * Special case: RAS triggered and full reset isn't supported
@@ -5156,6 +5210,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 
 	reset_context->job = job;
 	reset_context->hive = hive;
+
 	/*
 	 * Build list of devices to reset.
 	 * In case we are in XGMI hive mode, resort the device list
@@ -5163,8 +5218,11 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	 */
 	INIT_LIST_HEAD(&device_list);
 	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
-		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
+		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
 			list_add_tail(&tmp_adev->reset_list, &device_list);
+			if (gpu_reset_for_dev_remove && adev->shutdown)
+				tmp_adev->shutdown = true;
+		}
 		if (!list_is_first(&adev->reset_list, &device_list))
 			list_rotate_to_front(&adev->reset_list, &device_list);
 		device_list_handle = &device_list;
@@ -5247,6 +5305,10 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 
 retry:	/* Rest of adevs pre asic reset from XGMI hive. */
 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
+		if (gpu_reset_for_dev_remove) {
+			/* Workaroud for ASICs need to disable SMC first */
+			amdgpu_device_smu_fini_early(tmp_adev);
+		}
 		r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
 		/*TODO Should we stop ?*/
 		if (r) {
@@ -5275,8 +5337,14 @@ retry:	/* Rest of adevs pre asic reset from XGMI hive. */
 			amdgpu_ras_resume(adev);
 	} else {
 		r = amdgpu_do_asic_reset(device_list_handle, reset_context);
-		if (r && r == -EAGAIN)
+		if (r && r == -EAGAIN) {
+			set_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context->flags);
+			adev->asic_reset_res = 0;
 			goto retry;
+		}
+
+		if (!r && gpu_reset_for_dev_remove)
+			goto recover_end;
 	}
 
 skip_hw_reset:
@@ -5350,6 +5418,7 @@ skip_sched_resume:
 		amdgpu_device_unset_mp1_state(tmp_adev);
 	}
 
+recover_end:
 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
 					    reset_list);
 	amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
@@ -5532,9 +5601,9 @@ bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
 		~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
 	resource_size_t aper_limit =
 		adev->gmc.aper_base + adev->gmc.aper_size - 1;
-	bool p2p_access = !adev->gmc.xgmi.connected_to_cpu &&
-			  !(pci_p2pdma_distance_many(adev->pdev,
-					&peer_adev->dev, 1, true) < 0);
+	bool p2p_access =
+		!adev->gmc.xgmi.connected_to_cpu &&
+		!(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
 
 	return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
 		adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
@@ -5708,6 +5777,7 @@ pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
 	reset_context.reset_req_dev = adev;
 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
 	set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
+	set_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags);
 
 	adev->no_hw_access = true;
 	r = amdgpu_device_pre_asic_reset(adev, &reset_context);
@@ -5917,3 +5987,36 @@ void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
 	(void)RREG32(data);
 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 }
+
+/**
+ * amdgpu_device_switch_gang - switch to a new gang
+ * @adev: amdgpu_device pointer
+ * @gang: the gang to switch to
+ *
+ * Try to switch to a new gang.
+ * Returns: NULL if we switched to the new gang or a reference to the current
+ * gang leader.
+ */
+struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
+					    struct dma_fence *gang)
+{
+	struct dma_fence *old = NULL;
+
+	do {
+		dma_fence_put(old);
+		rcu_read_lock();
+		old = dma_fence_get_rcu_safe(&adev->gang_submit);
+		rcu_read_unlock();
+
+		if (old == gang)
+			break;
+
+		if (!dma_fence_is_signaled(old))
+			return old;
+
+	} while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
+			 old, gang) != old);
+
+	dma_fence_put(old);
+	return NULL;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index 95d34590cad1..3993e6134914 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -229,7 +229,7 @@ static int amdgpu_discovery_read_binary_from_file(struct amdgpu_device *adev, ui
 		return r;
 	}
 
-	memcpy((u8 *)binary, (u8 *)fw->data, adev->mman.discovery_tmr_size);
+	memcpy((u8 *)binary, (u8 *)fw->data, fw->size);
 	release_firmware(fw);
 
 	return 0;
@@ -1506,6 +1506,7 @@ static int amdgpu_discovery_set_common_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(11, 0, 0):
 	case IP_VERSION(11, 0, 1):
 	case IP_VERSION(11, 0, 2):
+	case IP_VERSION(11, 0, 3):
 		amdgpu_device_ip_block_add(adev, &soc21_common_ip_block);
 		break;
 	default:
@@ -1549,6 +1550,7 @@ static int amdgpu_discovery_set_gmc_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(11, 0, 0):
 	case IP_VERSION(11, 0, 1):
 	case IP_VERSION(11, 0, 2):
+	case IP_VERSION(11, 0, 3):
 		amdgpu_device_ip_block_add(adev, &gmc_v11_0_ip_block);
 		break;
 	default:
@@ -1633,6 +1635,7 @@ static int amdgpu_discovery_set_psp_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(13, 0, 5):
 	case IP_VERSION(13, 0, 7):
 	case IP_VERSION(13, 0, 8):
+	case IP_VERSION(13, 0, 10):
 		amdgpu_device_ip_block_add(adev, &psp_v13_0_ip_block);
 		break;
 	case IP_VERSION(13, 0, 4):
@@ -1682,6 +1685,7 @@ static int amdgpu_discovery_set_smu_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(13, 0, 5):
 	case IP_VERSION(13, 0, 7):
 	case IP_VERSION(13, 0, 8):
+	case IP_VERSION(13, 0, 10):
 		amdgpu_device_ip_block_add(adev, &smu_v13_0_ip_block);
 		break;
 	default:
@@ -1780,6 +1784,7 @@ static int amdgpu_discovery_set_gc_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(11, 0, 0):
 	case IP_VERSION(11, 0, 1):
 	case IP_VERSION(11, 0, 2):
+	case IP_VERSION(11, 0, 3):
 		amdgpu_device_ip_block_add(adev, &gfx_v11_0_ip_block);
 		break;
 	default:
@@ -1823,6 +1828,7 @@ static int amdgpu_discovery_set_sdma_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(6, 0, 0):
 	case IP_VERSION(6, 0, 1):
 	case IP_VERSION(6, 0, 2):
+	case IP_VERSION(6, 0, 3):
 		amdgpu_device_ip_block_add(adev, &sdma_v6_0_ip_block);
 		break;
 	default:
@@ -1903,7 +1909,8 @@ static int amdgpu_discovery_set_mm_ip_blocks(struct amdgpu_device *adev)
 		case IP_VERSION(4, 0, 2):
 		case IP_VERSION(4, 0, 4):
 			amdgpu_device_ip_block_add(adev, &vcn_v4_0_ip_block);
-			amdgpu_device_ip_block_add(adev, &jpeg_v4_0_ip_block);
+			if (!amdgpu_sriov_vf(adev))
+				amdgpu_device_ip_block_add(adev, &jpeg_v4_0_ip_block);
 			break;
 		default:
 			dev_err(adev->dev,
@@ -1940,6 +1947,7 @@ static int amdgpu_discovery_set_mes_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(11, 0, 0):
 	case IP_VERSION(11, 0, 1):
 	case IP_VERSION(11, 0, 2):
+	case IP_VERSION(11, 0, 3):
 		amdgpu_device_ip_block_add(adev, &mes_v11_0_ip_block);
 		adev->enable_mes = true;
 		adev->enable_mes_kiq = true;
@@ -2165,6 +2173,7 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 		break;
 	case IP_VERSION(11, 0, 0):
 	case IP_VERSION(11, 0, 2):
+	case IP_VERSION(11, 0, 3):
 		adev->family = AMDGPU_FAMILY_GC_11_0_0;
 		break;
 	case IP_VERSION(11, 0, 1):
@@ -2234,7 +2243,10 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 		break;
 	case IP_VERSION(4, 3, 0):
 	case IP_VERSION(4, 3, 1):
-		adev->nbio.funcs = &nbio_v4_3_funcs;
+		if (amdgpu_sriov_vf(adev))
+			adev->nbio.funcs = &nbio_v4_3_sriov_funcs;
+		else
+			adev->nbio.funcs = &nbio_v4_3_funcs;
 		adev->nbio.hdp_flush_reg = &nbio_v4_3_hdp_flush_reg;
 		break;
 	case IP_VERSION(7, 7, 0):
@@ -2332,6 +2344,7 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(6, 0, 0):
 	case IP_VERSION(6, 0, 1):
 	case IP_VERSION(6, 0, 2):
+	case IP_VERSION(6, 0, 3):
 		adev->lsdma.funcs = &lsdma_v6_0_funcs;
 		break;
 	default:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
index 782cbca37538..7bd8e33b14be 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dma_buf.c
@@ -58,7 +58,7 @@ static int amdgpu_dma_buf_attach(struct dma_buf *dmabuf,
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	int r;
 
-	if (pci_p2pdma_distance_many(adev->pdev, &attach->dev, 1, true) < 0)
+	if (pci_p2pdma_distance(adev->pdev, attach->dev, false) < 0)
 		attach->peer2peer = false;
 
 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 429fcdf28836..16f6a313335e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -102,9 +102,10 @@
  * - 3.46.0 - To enable hot plug amdgpu tests in libdrm
  * - 3.47.0 - Add AMDGPU_GEM_CREATE_DISCARDABLE and AMDGPU_VM_NOALLOC flags
  * - 3.48.0 - Add IP discovery version info to HW INFO
+ *   3.49.0 - Add gang submit into CS IOCTL
  */
 #define KMS_DRIVER_MAJOR	3
-#define KMS_DRIVER_MINOR	48
+#define KMS_DRIVER_MINOR	49
 #define KMS_DRIVER_PATCHLEVEL	0
 
 int amdgpu_vram_limit;
@@ -2181,15 +2182,46 @@ amdgpu_pci_remove(struct pci_dev *pdev)
 	struct drm_device *dev = pci_get_drvdata(pdev);
 	struct amdgpu_device *adev = drm_to_adev(dev);
 
-	drm_dev_unplug(dev);
-
 	if (adev->pm.rpm_mode != AMDGPU_RUNPM_NONE) {
 		pm_runtime_get_sync(dev->dev);
 		pm_runtime_forbid(dev->dev);
 	}
 
+	if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 2)) {
+		bool need_to_reset_gpu = false;
+
+		if (adev->gmc.xgmi.num_physical_nodes > 1) {
+			struct amdgpu_hive_info *hive;
+
+			hive = amdgpu_get_xgmi_hive(adev);
+			if (hive->device_remove_count == 0)
+				need_to_reset_gpu = true;
+			hive->device_remove_count++;
+			amdgpu_put_xgmi_hive(hive);
+		} else {
+			need_to_reset_gpu = true;
+		}
+
+		/* Workaround for ASICs need to reset SMU.
+		 * Called only when the first device is removed.
+		 */
+		if (need_to_reset_gpu) {
+			struct amdgpu_reset_context reset_context;
+
+			adev->shutdown = true;
+			memset(&reset_context, 0, sizeof(reset_context));
+			reset_context.method = AMD_RESET_METHOD_NONE;
+			reset_context.reset_req_dev = adev;
+			set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+			set_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context.flags);
+			amdgpu_device_gpu_recover(adev, NULL, &reset_context);
+		}
+	}
+
 	amdgpu_driver_unload_kms(dev);
 
+	drm_dev_unplug(dev);
+
 	/*
 	 * Flush any in flight DMA operations from device.
 	 * Clear the Bus Master Enable bit and then wait on the PCIe Device
@@ -2563,8 +2595,11 @@ static int amdgpu_pmops_runtime_resume(struct device *dev)
 		amdgpu_device_baco_exit(drm_dev);
 	}
 	ret = amdgpu_device_resume(drm_dev, false);
-	if (ret)
+	if (ret) {
+		if (amdgpu_device_supports_px(drm_dev))
+			pci_disable_device(pdev);
 		return ret;
+	}
 
 	if (amdgpu_device_supports_px(drm_dev))
 		drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 8adeb7469f1e..d0d99ed607dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -400,7 +400,6 @@ unsigned amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
 	/* We are not protected by ring lock when reading the last sequence
 	 * but it's ok to report slightly wrong fence count here.
 	 */
-	amdgpu_fence_process(ring);
 	emitted = 0x100000000ull;
 	emitted -= atomic_read(&ring->fence_drv.last_seq);
 	emitted += READ_ONCE(ring->fence_drv.sync_seq);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 222d3d7ea076..9546adc8a76f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -23,6 +23,7 @@
  *
  */
 
+#include <linux/firmware.h>
 #include "amdgpu.h"
 #include "amdgpu_gfx.h"
 #include "amdgpu_rlc.h"
@@ -477,7 +478,7 @@ int amdgpu_gfx_disable_kcq(struct amdgpu_device *adev)
 		kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.compute_ring[i],
 					   RESET_QUEUES, 0, 0);
 
-	if (adev->gfx.kiq.ring.sched.ready)
+	if (adev->gfx.kiq.ring.sched.ready && !adev->job_hang)
 		r = amdgpu_ring_test_helper(kiq_ring);
 	spin_unlock(&adev->gfx.kiq.ring_lock);
 
@@ -610,6 +611,45 @@ unlock:
 	mutex_unlock(&adev->gfx.gfx_off_mutex);
 }
 
+int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value)
+{
+	int r = 0;
+
+	mutex_lock(&adev->gfx.gfx_off_mutex);
+
+	r = amdgpu_dpm_set_residency_gfxoff(adev, value);
+
+	mutex_unlock(&adev->gfx.gfx_off_mutex);
+
+	return r;
+}
+
+int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *value)
+{
+	int r = 0;
+
+	mutex_lock(&adev->gfx.gfx_off_mutex);
+
+	r = amdgpu_dpm_get_residency_gfxoff(adev, value);
+
+	mutex_unlock(&adev->gfx.gfx_off_mutex);
+
+	return r;
+}
+
+int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value)
+{
+	int r = 0;
+
+	mutex_lock(&adev->gfx.gfx_off_mutex);
+
+	r = amdgpu_dpm_get_entrycount_gfxoff(adev, value);
+
+	mutex_unlock(&adev->gfx.gfx_off_mutex);
+
+	return r;
+}
+
 int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value)
 {
 
@@ -826,3 +866,142 @@ int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev)
 	}
 	return amdgpu_num_kcq;
 }
+
+void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev,
+				  uint32_t ucode_id)
+{
+	const struct gfx_firmware_header_v1_0 *cp_hdr;
+	const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0;
+	struct amdgpu_firmware_info *info = NULL;
+	const struct firmware *ucode_fw;
+	unsigned int fw_size;
+
+	switch (ucode_id) {
+	case AMDGPU_UCODE_ID_CP_PFP:
+		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+			adev->gfx.pfp_fw->data;
+		adev->gfx.pfp_fw_version =
+			le32_to_cpu(cp_hdr->header.ucode_version);
+		adev->gfx.pfp_feature_version =
+			le32_to_cpu(cp_hdr->ucode_feature_version);
+		ucode_fw = adev->gfx.pfp_fw;
+		fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_PFP:
+		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
+			adev->gfx.pfp_fw->data;
+		adev->gfx.pfp_fw_version =
+			le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
+		adev->gfx.pfp_feature_version =
+			le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
+		ucode_fw = adev->gfx.pfp_fw;
+		fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK:
+	case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK:
+		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
+			adev->gfx.pfp_fw->data;
+		ucode_fw = adev->gfx.pfp_fw;
+		fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
+		break;
+	case AMDGPU_UCODE_ID_CP_ME:
+		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+			adev->gfx.me_fw->data;
+		adev->gfx.me_fw_version =
+			le32_to_cpu(cp_hdr->header.ucode_version);
+		adev->gfx.me_feature_version =
+			le32_to_cpu(cp_hdr->ucode_feature_version);
+		ucode_fw = adev->gfx.me_fw;
+		fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_ME:
+		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
+			adev->gfx.me_fw->data;
+		adev->gfx.me_fw_version =
+			le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
+		adev->gfx.me_feature_version =
+			le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
+		ucode_fw = adev->gfx.me_fw;
+		fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK:
+	case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK:
+		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
+			adev->gfx.me_fw->data;
+		ucode_fw = adev->gfx.me_fw;
+		fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
+		break;
+	case AMDGPU_UCODE_ID_CP_CE:
+		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+			adev->gfx.ce_fw->data;
+		adev->gfx.ce_fw_version =
+			le32_to_cpu(cp_hdr->header.ucode_version);
+		adev->gfx.ce_feature_version =
+			le32_to_cpu(cp_hdr->ucode_feature_version);
+		ucode_fw = adev->gfx.ce_fw;
+		fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes);
+		break;
+	case AMDGPU_UCODE_ID_CP_MEC1:
+		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+			adev->gfx.mec_fw->data;
+		adev->gfx.mec_fw_version =
+			le32_to_cpu(cp_hdr->header.ucode_version);
+		adev->gfx.mec_feature_version =
+			le32_to_cpu(cp_hdr->ucode_feature_version);
+		ucode_fw = adev->gfx.mec_fw;
+		fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
+			  le32_to_cpu(cp_hdr->jt_size) * 4;
+		break;
+	case AMDGPU_UCODE_ID_CP_MEC1_JT:
+		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+			adev->gfx.mec_fw->data;
+		ucode_fw = adev->gfx.mec_fw;
+		fw_size = le32_to_cpu(cp_hdr->jt_size) * 4;
+		break;
+	case AMDGPU_UCODE_ID_CP_MEC2:
+		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+			adev->gfx.mec2_fw->data;
+		adev->gfx.mec2_fw_version =
+			le32_to_cpu(cp_hdr->header.ucode_version);
+		adev->gfx.mec2_feature_version =
+			le32_to_cpu(cp_hdr->ucode_feature_version);
+		ucode_fw = adev->gfx.mec2_fw;
+		fw_size = le32_to_cpu(cp_hdr->header.ucode_size_bytes) -
+			  le32_to_cpu(cp_hdr->jt_size) * 4;
+		break;
+	case AMDGPU_UCODE_ID_CP_MEC2_JT:
+		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
+			adev->gfx.mec2_fw->data;
+		ucode_fw = adev->gfx.mec2_fw;
+		fw_size = le32_to_cpu(cp_hdr->jt_size) * 4;
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_MEC:
+		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
+			adev->gfx.mec_fw->data;
+		adev->gfx.mec_fw_version =
+			le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
+		adev->gfx.mec_feature_version =
+			le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
+		ucode_fw = adev->gfx.mec_fw;
+		fw_size = le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes);
+		break;
+	case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
+	case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
+	case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
+	case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
+		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)
+			adev->gfx.mec_fw->data;
+		ucode_fw = adev->gfx.mec_fw;
+		fw_size = le32_to_cpu(cp_hdr_v2_0->data_size_bytes);
+		break;
+	default:
+		break;
+	}
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		info = &adev->firmware.ucode[ucode_id];
+		info->ucode_id = ucode_id;
+		info->fw = ucode_fw;
+		adev->firmware.fw_size += ALIGN(fw_size, PAGE_SIZE);
+	}
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index 23a696d38390..832b3807f1d6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -304,6 +304,10 @@ struct amdgpu_gfx {
 	uint32_t			rlc_srlg_feature_version;
 	uint32_t			rlc_srls_fw_version;
 	uint32_t			rlc_srls_feature_version;
+	uint32_t			rlcp_ucode_version;
+	uint32_t			rlcp_ucode_feature_version;
+	uint32_t			rlcv_ucode_version;
+	uint32_t			rlcv_ucode_feature_version;
 	uint32_t			mec_feature_version;
 	uint32_t			mec2_feature_version;
 	bool				mec_fw_write_wait;
@@ -332,10 +336,12 @@ struct amdgpu_gfx {
 	uint32_t                        srbm_soft_reset;
 
 	/* gfx off */
-	bool                            gfx_off_state; /* true: enabled, false: disabled */
-	struct mutex                    gfx_off_mutex;
-	uint32_t                        gfx_off_req_count; /* default 1, enable gfx off: dec 1, disable gfx off: add 1 */
-	struct delayed_work             gfx_off_delay_work;
+	bool                            gfx_off_state;      /* true: enabled, false: disabled */
+	struct mutex                    gfx_off_mutex;      /* mutex to change gfxoff state */
+	uint32_t                        gfx_off_req_count;  /* default 1, enable gfx off: dec 1, disable gfx off: add 1 */
+	struct delayed_work             gfx_off_delay_work; /* async work to set gfx block off */
+	uint32_t                        gfx_off_residency;  /* last logged residency */
+	uint64_t                        gfx_off_entrycount; /* count of times GPU has get into GFXOFF state */
 
 	/* pipe reservation */
 	struct mutex			pipe_reserve_mutex;
@@ -407,6 +413,10 @@ bool amdgpu_gfx_is_me_queue_enabled(struct amdgpu_device *adev, int me,
 void amdgpu_gfx_off_ctrl(struct amdgpu_device *adev, bool enable);
 int amdgpu_get_gfx_off_status(struct amdgpu_device *adev, uint32_t *value);
 int amdgpu_gfx_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block);
+void amdgpu_gfx_ras_fini(struct amdgpu_device *adev);
+int amdgpu_get_gfx_off_entrycount(struct amdgpu_device *adev, u64 *value);
+int amdgpu_get_gfx_off_residency(struct amdgpu_device *adev, u32 *residency);
+int amdgpu_set_gfx_off_residency(struct amdgpu_device *adev, bool value);
 int amdgpu_gfx_process_ras_data_cb(struct amdgpu_device *adev,
 		void *err_data,
 		struct amdgpu_iv_entry *entry);
@@ -416,4 +426,6 @@ int amdgpu_gfx_cp_ecc_error_irq(struct amdgpu_device *adev,
 uint32_t amdgpu_kiq_rreg(struct amdgpu_device *adev, uint32_t reg);
 void amdgpu_kiq_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v);
 int amdgpu_gfx_get_num_kcq(struct amdgpu_device *adev);
+void amdgpu_gfx_cp_init_microcode(struct amdgpu_device *adev, uint32_t ucode_id);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h
index beabab515836..c7b44aeb671b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfxhub.h
@@ -35,6 +35,9 @@ struct amdgpu_gfxhub_funcs {
 	void (*init)(struct amdgpu_device *adev);
 	int (*get_xgmi_info)(struct amdgpu_device *adev);
 	void (*utcl2_harvest)(struct amdgpu_device *adev);
+	void (*mode2_save_regs)(struct amdgpu_device *adev);
+	void (*mode2_restore_regs)(struct amdgpu_device *adev);
+	void (*halt)(struct amdgpu_device *adev);
 };
 
 struct amdgpu_gfxhub {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index aebc384531ac..34233a74248c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -572,45 +572,15 @@ void amdgpu_gmc_tmz_set(struct amdgpu_device *adev)
 void amdgpu_gmc_noretry_set(struct amdgpu_device *adev)
 {
 	struct amdgpu_gmc *gmc = &adev->gmc;
-
-	switch (adev->ip_versions[GC_HWIP][0]) {
-	case IP_VERSION(9, 0, 1):
-	case IP_VERSION(9, 3, 0):
-	case IP_VERSION(9, 4, 0):
-	case IP_VERSION(9, 4, 1):
-	case IP_VERSION(9, 4, 2):
-	case IP_VERSION(10, 3, 3):
-	case IP_VERSION(10, 3, 4):
-	case IP_VERSION(10, 3, 5):
-	case IP_VERSION(10, 3, 6):
-	case IP_VERSION(10, 3, 7):
-		/*
-		 * noretry = 0 will cause kfd page fault tests fail
-		 * for some ASICs, so set default to 1 for these ASICs.
-		 */
-		if (amdgpu_noretry == -1)
-			gmc->noretry = 1;
-		else
-			gmc->noretry = amdgpu_noretry;
-		break;
-	default:
-		/* Raven currently has issues with noretry
-		 * regardless of what we decide for other
-		 * asics, we should leave raven with
-		 * noretry = 0 until we root cause the
-		 * issues.
-		 *
-		 * default this to 0 for now, but we may want
-		 * to change this in the future for certain
-		 * GPUs as it can increase performance in
-		 * certain cases.
-		 */
-		if (amdgpu_noretry == -1)
-			gmc->noretry = 0;
-		else
-			gmc->noretry = amdgpu_noretry;
-		break;
-	}
+	uint32_t gc_ver = adev->ip_versions[GC_HWIP][0];
+	bool noretry_default = (gc_ver == IP_VERSION(9, 0, 1) ||
+				gc_ver == IP_VERSION(9, 3, 0) ||
+				gc_ver == IP_VERSION(9, 4, 0) ||
+				gc_ver == IP_VERSION(9, 4, 1) ||
+				gc_ver == IP_VERSION(9, 4, 2) ||
+				gc_ver >= IP_VERSION(10, 3, 0));
+
+	gmc->noretry = (amdgpu_noretry == -1) ? noretry_default : amdgpu_noretry;
 }
 
 void amdgpu_gmc_set_vm_fault_masks(struct amdgpu_device *adev, int hub_type,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 008eaca27151..0305b660cd17 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -264,6 +264,32 @@ struct amdgpu_gmc {
 	u64 mall_size;
 	/* number of UMC instances */
 	int num_umc;
+	/* mode2 save restore */
+	u64 VM_L2_CNTL;
+	u64 VM_L2_CNTL2;
+	u64 VM_DUMMY_PAGE_FAULT_CNTL;
+	u64 VM_DUMMY_PAGE_FAULT_ADDR_LO32;
+	u64 VM_DUMMY_PAGE_FAULT_ADDR_HI32;
+	u64 VM_L2_PROTECTION_FAULT_CNTL;
+	u64 VM_L2_PROTECTION_FAULT_CNTL2;
+	u64 VM_L2_PROTECTION_FAULT_MM_CNTL3;
+	u64 VM_L2_PROTECTION_FAULT_MM_CNTL4;
+	u64 VM_L2_PROTECTION_FAULT_ADDR_LO32;
+	u64 VM_L2_PROTECTION_FAULT_ADDR_HI32;
+	u64 VM_DEBUG;
+	u64 VM_L2_MM_GROUP_RT_CLASSES;
+	u64 VM_L2_BANK_SELECT_RESERVED_CID;
+	u64 VM_L2_BANK_SELECT_RESERVED_CID2;
+	u64 VM_L2_CACHE_PARITY_CNTL;
+	u64 VM_L2_IH_LOG_CNTL;
+	u64 VM_CONTEXT_CNTL[16];
+	u64 VM_CONTEXT_PAGE_TABLE_BASE_ADDR_LO32[16];
+	u64 VM_CONTEXT_PAGE_TABLE_BASE_ADDR_HI32[16];
+	u64 VM_CONTEXT_PAGE_TABLE_START_ADDR_LO32[16];
+	u64 VM_CONTEXT_PAGE_TABLE_START_ADDR_HI32[16];
+	u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32[16];
+	u64 VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32[16];
+	u64 MC_VM_MX_L1_TLB_CNTL;
 };
 
 #define amdgpu_gmc_flush_gpu_tlb(adev, vmid, vmhub, type) ((adev)->gmc.gmc_funcs->flush_gpu_tlb((adev), (vmid), (vmhub), (type)))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index 8c6b2284cf56..1f3302aebeff 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -205,6 +205,42 @@ void amdgpu_gtt_mgr_recover(struct amdgpu_gtt_mgr *mgr)
 }
 
 /**
+ * amdgpu_gtt_mgr_intersects - test for intersection
+ *
+ * @man: Our manager object
+ * @res: The resource to test
+ * @place: The place for the new allocation
+ * @size: The size of the new allocation
+ *
+ * Simplified intersection test, only interesting if we need GART or not.
+ */
+static bool amdgpu_gtt_mgr_intersects(struct ttm_resource_manager *man,
+				      struct ttm_resource *res,
+				      const struct ttm_place *place,
+				      size_t size)
+{
+	return !place->lpfn || amdgpu_gtt_mgr_has_gart_addr(res);
+}
+
+/**
+ * amdgpu_gtt_mgr_compatible - test for compatibility
+ *
+ * @man: Our manager object
+ * @res: The resource to test
+ * @place: The place for the new allocation
+ * @size: The size of the new allocation
+ *
+ * Simplified compatibility test.
+ */
+static bool amdgpu_gtt_mgr_compatible(struct ttm_resource_manager *man,
+				      struct ttm_resource *res,
+				      const struct ttm_place *place,
+				      size_t size)
+{
+	return !place->lpfn || amdgpu_gtt_mgr_has_gart_addr(res);
+}
+
+/**
  * amdgpu_gtt_mgr_debug - dump VRAM table
  *
  * @man: TTM memory type manager
@@ -225,6 +261,8 @@ static void amdgpu_gtt_mgr_debug(struct ttm_resource_manager *man,
 static const struct ttm_resource_manager_func amdgpu_gtt_mgr_func = {
 	.alloc = amdgpu_gtt_mgr_new,
 	.free = amdgpu_gtt_mgr_del,
+	.intersects = amdgpu_gtt_mgr_intersects,
+	.compatible = amdgpu_gtt_mgr_compatible,
 	.debug = amdgpu_gtt_mgr_debug
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index c2fd6f3076a6..46c99331d7f1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -49,6 +49,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 	}
 
 	memset(&ti, 0, sizeof(struct amdgpu_task_info));
+	adev->job_hang = true;
 
 	if (amdgpu_gpu_recovery &&
 	    amdgpu_ring_soft_recovery(ring, job->vmid, s_job->s_fence->parent)) {
@@ -71,6 +72,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 		reset_context.method = AMD_RESET_METHOD_NONE;
 		reset_context.reset_req_dev = adev;
 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+		clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags);
 
 		r = amdgpu_device_gpu_recover(ring->adev, job, &reset_context);
 		if (r)
@@ -82,6 +84,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 	}
 
 exit:
+	adev->job_hang = false;
 	drm_dev_exit(idx);
 	return DRM_GPU_SCHED_STAT_NOMINAL;
 }
@@ -102,7 +105,6 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
 	 */
 	(*job)->base.sched = &adev->rings[0]->sched;
 	(*job)->vm = vm;
-	(*job)->num_ibs = num_ibs;
 
 	amdgpu_sync_create(&(*job)->sync);
 	amdgpu_sync_create(&(*job)->sched_sync);
@@ -122,6 +124,7 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
 	if (r)
 		return r;
 
+	(*job)->num_ibs = 1;
 	r = amdgpu_ib_get(adev, NULL, size, pool_type, &(*job)->ibs[0]);
 	if (r)
 		kfree(*job);
@@ -129,6 +132,23 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
 	return r;
 }
 
+void amdgpu_job_set_resources(struct amdgpu_job *job, struct amdgpu_bo *gds,
+			      struct amdgpu_bo *gws, struct amdgpu_bo *oa)
+{
+	if (gds) {
+		job->gds_base = amdgpu_bo_gpu_offset(gds) >> PAGE_SHIFT;
+		job->gds_size = amdgpu_bo_size(gds) >> PAGE_SHIFT;
+	}
+	if (gws) {
+		job->gws_base = amdgpu_bo_gpu_offset(gws) >> PAGE_SHIFT;
+		job->gws_size = amdgpu_bo_size(gws) >> PAGE_SHIFT;
+	}
+	if (oa) {
+		job->oa_base = amdgpu_bo_gpu_offset(oa) >> PAGE_SHIFT;
+		job->oa_size = amdgpu_bo_size(oa) >> PAGE_SHIFT;
+	}
+}
+
 void amdgpu_job_free_resources(struct amdgpu_job *job)
 {
 	struct amdgpu_ring *ring = to_amdgpu_ring(job->base.sched);
@@ -153,11 +173,29 @@ static void amdgpu_job_free_cb(struct drm_sched_job *s_job)
 	dma_fence_put(&job->hw_fence);
 }
 
+void amdgpu_job_set_gang_leader(struct amdgpu_job *job,
+				struct amdgpu_job *leader)
+{
+	struct dma_fence *fence = &leader->base.s_fence->scheduled;
+
+	WARN_ON(job->gang_submit);
+
+	/*
+	 * Don't add a reference when we are the gang leader to avoid circle
+	 * dependency.
+	 */
+	if (job != leader)
+		dma_fence_get(fence);
+	job->gang_submit = fence;
+}
+
 void amdgpu_job_free(struct amdgpu_job *job)
 {
 	amdgpu_job_free_resources(job);
 	amdgpu_sync_free(&job->sync);
 	amdgpu_sync_free(&job->sched_sync);
+	if (job->gang_submit != &job->base.s_fence->scheduled)
+		dma_fence_put(job->gang_submit);
 
 	if (!job->hw_fence.ops)
 		kfree(job);
@@ -227,12 +265,16 @@ static struct dma_fence *amdgpu_job_dependency(struct drm_sched_job *sched_job,
 		fence = amdgpu_sync_get_fence(&job->sync);
 	}
 
+	if (!fence && job->gang_submit)
+		fence = amdgpu_device_switch_gang(ring->adev, job->gang_submit);
+
 	return fence;
 }
 
 static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
 {
 	struct amdgpu_ring *ring = to_amdgpu_ring(sched_job->sched);
+	struct amdgpu_device *adev = ring->adev;
 	struct dma_fence *fence = NULL, *finished;
 	struct amdgpu_job *job;
 	int r = 0;
@@ -244,8 +286,10 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
 
 	trace_amdgpu_sched_run_job(job);
 
-	if (job->vram_lost_counter != atomic_read(&ring->adev->vram_lost_counter))
-		dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */
+	/* Skip job if VRAM is lost and never resubmit gangs */
+	if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter) ||
+	    (job->job_run_counter && job->gang_submit))
+		dma_fence_set_error(finished, -ECANCELED);
 
 	if (finished->error < 0) {
 		DRM_INFO("Skip scheduling IBs!\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index babc0af751c2..ab7b150e5d50 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -50,6 +50,7 @@ struct amdgpu_job {
 	struct amdgpu_sync	sync;
 	struct amdgpu_sync	sched_sync;
 	struct dma_fence	hw_fence;
+	struct dma_fence	*gang_submit;
 	uint32_t		preamble_status;
 	uint32_t                preemption_status;
 	bool                    vm_needs_flush;
@@ -72,11 +73,20 @@ struct amdgpu_job {
 	struct amdgpu_ib	ibs[];
 };
 
+static inline struct amdgpu_ring *amdgpu_job_ring(struct amdgpu_job *job)
+{
+	return to_amdgpu_ring(job->base.entity->rq->sched);
+}
+
 int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
 		     struct amdgpu_job **job, struct amdgpu_vm *vm);
 int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
 		enum amdgpu_ib_pool_type pool, struct amdgpu_job **job);
+void amdgpu_job_set_resources(struct amdgpu_job *job, struct amdgpu_bo *gds,
+			      struct amdgpu_bo *gws, struct amdgpu_bo *oa);
 void amdgpu_job_free_resources(struct amdgpu_job *job);
+void amdgpu_job_set_gang_leader(struct amdgpu_job *job,
+				struct amdgpu_job *leader);
 void amdgpu_job_free(struct amdgpu_job *job);
 int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity,
 		      void *owner, struct dma_fence **f);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 1369c25448dc..fe23e09eec98 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -247,6 +247,14 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info,
 		fw_info->ver = adev->gfx.rlc_srls_fw_version;
 		fw_info->feature = adev->gfx.rlc_srls_feature_version;
 		break;
+	case AMDGPU_INFO_FW_GFX_RLCP:
+		fw_info->ver = adev->gfx.rlcp_ucode_version;
+		fw_info->feature = adev->gfx.rlcp_ucode_feature_version;
+		break;
+	case AMDGPU_INFO_FW_GFX_RLCV:
+		fw_info->ver = adev->gfx.rlcv_ucode_version;
+		fw_info->feature = adev->gfx.rlcv_ucode_feature_version;
+		break;
 	case AMDGPU_INFO_FW_GFX_MEC:
 		if (query_fw->index == 0) {
 			fw_info->ver = adev->gfx.mec_fw_version;
@@ -328,6 +336,14 @@ static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info,
 		fw_info->ver = adev->psp.cap_fw_version;
 		fw_info->feature = adev->psp.cap_feature_version;
 		break;
+	case AMDGPU_INFO_FW_MES_KIQ:
+		fw_info->ver = adev->mes.ucode_fw_version[0];
+		fw_info->feature = 0;
+		break;
+	case AMDGPU_INFO_FW_MES:
+		fw_info->ver = adev->mes.ucode_fw_version[1];
+		fw_info->feature = 0;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -1160,7 +1176,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 	}
 
 	mutex_init(&fpriv->bo_list_lock);
-	idr_init(&fpriv->bo_list_handles);
+	idr_init_base(&fpriv->bo_list_handles, 1);
 
 	amdgpu_ctx_mgr_init(&fpriv->ctx_mgr, adev);
 
@@ -1469,6 +1485,22 @@ static int amdgpu_debugfs_firmware_info_show(struct seq_file *m, void *unused)
 	seq_printf(m, "RLC SRLS feature version: %u, firmware version: 0x%08x\n",
 		   fw_info.feature, fw_info.ver);
 
+	/* RLCP */
+	query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLCP;
+	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+	if (ret)
+		return ret;
+	seq_printf(m, "RLCP feature version: %u, firmware version: 0x%08x\n",
+		   fw_info.feature, fw_info.ver);
+
+	/* RLCV */
+        query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLCV;
+	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+	if (ret)
+		return ret;
+	seq_printf(m, "RLCV feature version: %u, firmware version: 0x%08x\n",
+		   fw_info.feature, fw_info.ver);
+
 	/* MEC */
 	query_fw.fw_type = AMDGPU_INFO_FW_GFX_MEC;
 	query_fw.index = 0;
@@ -1581,6 +1613,22 @@ static int amdgpu_debugfs_firmware_info_show(struct seq_file *m, void *unused)
 				fw_info.feature, fw_info.ver);
 	}
 
+	/* MES_KIQ */
+	query_fw.fw_type = AMDGPU_INFO_FW_MES_KIQ;
+	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+	if (ret)
+		return ret;
+	seq_printf(m, "MES_KIQ feature version: %u, firmware version: 0x%08x\n",
+		   fw_info.feature, fw_info.ver);
+
+	/* MES */
+	query_fw.fw_type = AMDGPU_INFO_FW_MES;
+	ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+	if (ret)
+		return ret;
+	seq_printf(m, "MES feature version: %u, firmware version: 0x%08x\n",
+		   fw_info.feature, fw_info.ver);
+
 	seq_printf(m, "VBIOS version: %s\n", ctx->vbios_version);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index 7b46f6bf4187..ad980f4b66e1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -222,6 +222,8 @@ struct mes_add_queue_input {
 	uint64_t	tba_addr;
 	uint64_t	tma_addr;
 	uint32_t	is_kfd_process;
+	uint32_t	is_aql_queue;
+	uint32_t	queue_size;
 };
 
 struct mes_remove_queue_input {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index d788a00043a5..37322550d750 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -38,7 +38,6 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 4570ad449390..e6a9b9fc9e0b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -591,7 +591,7 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 	if (!bp->destroy)
 		bp->destroy = &amdgpu_bo_destroy;
 
-	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, size, bp->type,
+	r = ttm_bo_init_reserved(&adev->mman.bdev, &bo->tbo, bp->type,
 				 &bo->placement, page_align, &ctx,  NULL,
 				 bp->resv, bp->destroy);
 	if (unlikely(r != 0))
@@ -1309,7 +1309,7 @@ void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
 	if (bo->base.resv == &bo->base._resv)
 		amdgpu_amdkfd_remove_fence_on_pt_pd_bos(abo);
 
-	if (bo->resource->mem_type != TTM_PL_VRAM ||
+	if (!bo->resource || bo->resource->mem_type != TTM_PL_VRAM ||
 	    !(abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE) ||
 	    adev->in_suspend || adev->shutdown)
 		return;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index c9dec2434f37..effa7df3ddbf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -138,6 +138,7 @@ static int psp_early_init(void *handle)
 	case IP_VERSION(13, 0, 3):
 	case IP_VERSION(13, 0, 5):
 	case IP_VERSION(13, 0, 8):
+	case IP_VERSION(13, 0, 10):
 		psp_v13_0_set_psp_funcs(psp);
 		psp->autoload_supported = true;
 		break;
@@ -327,23 +328,32 @@ static int psp_init_sriov_microcode(struct psp_context *psp)
 
 	switch (adev->ip_versions[MP0_HWIP][0]) {
 	case IP_VERSION(9, 0, 0):
+		adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 		ret = psp_init_cap_microcode(psp, "vega10");
 		break;
 	case IP_VERSION(11, 0, 9):
+		adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 		ret = psp_init_cap_microcode(psp, "navi12");
 		break;
 	case IP_VERSION(11, 0, 7):
+		adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 		ret = psp_init_cap_microcode(psp, "sienna_cichlid");
 		break;
 	case IP_VERSION(13, 0, 2):
+		adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 		ret = psp_init_cap_microcode(psp, "aldebaran");
 		ret &= psp_init_ta_microcode(psp, "aldebaran");
 		break;
+	case IP_VERSION(13, 0, 0):
+		adev->virt.autoload_ucode_id = 0;
+		break;
+	case IP_VERSION(13, 0, 10):
+		adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MES1_DATA;
+		break;
 	default:
 		BUG();
 		break;
 	}
-
 	return ret;
 }
 
@@ -501,6 +511,11 @@ static int psp_sw_fini(void *handle)
 	kfree(cmd);
 	cmd = NULL;
 
+	if (psp->km_ring.ring_mem)
+		amdgpu_bo_free_kernel(&adev->firmware.rbuf,
+				      &psp->km_ring.ring_mem_mc_addr,
+				      (void **)&psp->km_ring.ring_mem);
+
 	amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 			      &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 	amdgpu_bo_free_kernel(&psp->fence_buf_bo,
@@ -769,6 +784,7 @@ static bool psp_skip_tmr(struct psp_context *psp)
 	case IP_VERSION(11, 0, 9):
 	case IP_VERSION(11, 0, 7):
 	case IP_VERSION(13, 0, 2):
+	case IP_VERSION(13, 0, 10):
 		return true;
 	default:
 		return false;
@@ -815,7 +831,7 @@ static int psp_tmr_unload(struct psp_context *psp)
 	struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 
 	psp_prep_tmr_unload_cmd_buf(psp, cmd);
-	DRM_INFO("free PSP TMR buffer\n");
+	dev_info(psp->adev->dev, "free PSP TMR buffer\n");
 
 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
 				 psp->fence_buf_mc_addr);
@@ -2044,6 +2060,15 @@ static int psp_hw_start(struct psp_context *psp)
 			}
 		}
 
+		if ((is_psp_fw_valid(psp->ras_drv)) &&
+		    (psp->funcs->bootloader_load_ras_drv != NULL)) {
+			ret = psp_bootloader_load_ras_drv(psp);
+			if (ret) {
+				DRM_ERROR("PSP load ras_drv failed!\n");
+				return ret;
+			}
+		}
+
 		if ((is_psp_fw_valid(psp->sos)) &&
 		    (psp->funcs->bootloader_load_sos != NULL)) {
 			ret = psp_bootloader_load_sos(psp);
@@ -2414,20 +2439,7 @@ static bool fw_load_skip_check(struct psp_context *psp,
 		return true;
 
 	if (amdgpu_sriov_vf(psp->adev) &&
-	   (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA4
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA5
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA6
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA7
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
-	    || ucode->ucode_id == AMDGPU_UCODE_ID_SMC))
-		/*skip ucode loading in SRIOV VF */
+	    amdgpu_virt_fw_load_skip_check(psp->adev, ucode->ucode_id))
 		return true;
 
 	if (psp->autoload_supported &&
@@ -2501,7 +2513,7 @@ static int psp_load_non_psp_fw(struct psp_context *psp)
 
 		/* Start rlc autoload after psp recieved all the gfx firmware */
 		if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
-		    AMDGPU_UCODE_ID_CP_MEC2 : AMDGPU_UCODE_ID_RLC_G)) {
+		    adev->virt.autoload_ucode_id : AMDGPU_UCODE_ID_RLC_G)) {
 			ret = psp_rlc_autoload_start(psp);
 			if (ret) {
 				DRM_ERROR("Failed to start rlc autoload\n");
@@ -3042,6 +3054,12 @@ static int parse_sos_bin_descriptor(struct psp_context *psp,
 		psp->dbg_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
 		psp->dbg_drv.start_addr         = ucode_start_addr;
 		break;
+	case PSP_FW_TYPE_PSP_RAS_DRV:
+		psp->ras_drv.fw_version         = le32_to_cpu(desc->fw_version);
+		psp->ras_drv.feature_version    = le32_to_cpu(desc->fw_version);
+		psp->ras_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
+		psp->ras_drv.start_addr         = ucode_start_addr;
+		break;
 	default:
 		dev_warn(psp->adev->dev, "Unsupported PSP FW type: %d\n", desc->fw_type);
 		break;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index e593e8c2a54d..58ce3ebb446c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -72,6 +72,7 @@ enum psp_bootloader_cmd {
 	PSP_BL__LOAD_SOCDRV             = 0xB0000,
 	PSP_BL__LOAD_DBGDRV             = 0xC0000,
 	PSP_BL__LOAD_INTFDRV		= 0xD0000,
+	PSP_BL__LOAD_RASDRV		    = 0xE0000,
 	PSP_BL__DRAM_LONG_TRAIN		= 0x100000,
 	PSP_BL__DRAM_SHORT_TRAIN	= 0x200000,
 	PSP_BL__LOAD_TOS_SPL_TABLE	= 0x10000000,
@@ -115,6 +116,7 @@ struct psp_funcs
 	int (*bootloader_load_soc_drv)(struct psp_context *psp);
 	int (*bootloader_load_intf_drv)(struct psp_context *psp);
 	int (*bootloader_load_dbg_drv)(struct psp_context *psp);
+	int (*bootloader_load_ras_drv)(struct psp_context *psp);
 	int (*bootloader_load_sos)(struct psp_context *psp);
 	int (*ring_init)(struct psp_context *psp, enum psp_ring_type ring_type);
 	int (*ring_create)(struct psp_context *psp,
@@ -324,6 +326,7 @@ struct psp_context
 	struct psp_bin_desc		soc_drv;
 	struct psp_bin_desc		intf_drv;
 	struct psp_bin_desc		dbg_drv;
+	struct psp_bin_desc		ras_drv;
 
 	/* tmr buffer */
 	struct amdgpu_bo		*tmr_bo;
@@ -404,6 +407,9 @@ struct amdgpu_psp_funcs {
 		((psp)->funcs->bootloader_load_intf_drv ? (psp)->funcs->bootloader_load_intf_drv((psp)) : 0)
 #define psp_bootloader_load_dbg_drv(psp) \
 		((psp)->funcs->bootloader_load_dbg_drv ? (psp)->funcs->bootloader_load_dbg_drv((psp)) : 0)
+#define psp_bootloader_load_ras_drv(psp) \
+		((psp)->funcs->bootloader_load_ras_drv ? \
+		(psp)->funcs->bootloader_load_ras_drv((psp)) : 0)
 #define psp_bootloader_load_sos(psp) \
 		((psp)->funcs->bootloader_load_sos ? (psp)->funcs->bootloader_load_sos((psp)) : 0)
 #define psp_smu_reload_quirk(psp) \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 12c6f97945a5..ccebd8e2a2d8 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -1950,6 +1950,7 @@ static void amdgpu_ras_do_recovery(struct work_struct *work)
 		reset_context.method = AMD_RESET_METHOD_NONE;
 		reset_context.reset_req_dev = adev;
 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+		clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags);
 
 		amdgpu_device_gpu_recover(ras->adev, NULL, &reset_context);
 	}
@@ -2719,7 +2720,8 @@ int amdgpu_ras_pre_fini(struct amdgpu_device *adev)
 
 
 	/* Need disable ras on all IPs here before ip [hw/sw]fini */
-	amdgpu_ras_disable_all_features(adev, 0);
+	if (con->features)
+		amdgpu_ras_disable_all_features(adev, 0);
 	amdgpu_ras_recovery_fini(adev);
 	return 0;
 }
@@ -2832,11 +2834,8 @@ static int amdgpu_bad_page_notifier(struct notifier_block *nb,
 	struct mce *m = (struct mce *)data;
 	struct amdgpu_device *adev = NULL;
 	uint32_t gpu_id = 0;
-	uint32_t umc_inst = 0;
-	uint32_t ch_inst, channel_index = 0;
+	uint32_t umc_inst = 0, ch_inst = 0;
 	struct ras_err_data err_data = {0, 0, 0, NULL};
-	struct eeprom_table_record err_rec;
-	uint64_t retired_page;
 
 	/*
 	 * If the error was generated in UMC_V2, which belongs to GPU UMCs,
@@ -2875,21 +2874,22 @@ static int amdgpu_bad_page_notifier(struct notifier_block *nb,
 	dev_info(adev->dev, "Uncorrectable error detected in UMC inst: %d, chan_idx: %d",
 			     umc_inst, ch_inst);
 
+	err_data.err_addr =
+		kcalloc(adev->umc.max_ras_err_cnt_per_query,
+			sizeof(struct eeprom_table_record), GFP_KERNEL);
+	if(!err_data.err_addr) {
+		dev_warn(adev->dev, "Failed to alloc memory for "
+				"umc error address record in mca notifier!\n");
+		return NOTIFY_DONE;
+	}
+
 	/*
 	 * Translate UMC channel address to Physical address
 	 */
-	channel_index =
-		adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num
-					  + ch_inst];
-
-	retired_page = ADDR_OF_8KB_BLOCK(m->addr) |
-			ADDR_OF_256B_BLOCK(channel_index) |
-			OFFSET_IN_256B_BLOCK(m->addr);
-
-	memset(&err_rec, 0x0, sizeof(struct eeprom_table_record));
-	err_data.err_addr = &err_rec;
-	amdgpu_umc_fill_error_record(&err_data, m->addr,
-			retired_page, channel_index, umc_inst);
+	if (adev->umc.ras &&
+	    adev->umc.ras->convert_ras_error_address)
+		adev->umc.ras->convert_ras_error_address(adev,
+			&err_data, 0, ch_inst, umc_inst, m->addr);
 
 	if (amdgpu_bad_page_threshold != 0) {
 		amdgpu_ras_add_bad_pages(adev, err_data.err_addr,
@@ -2897,6 +2897,7 @@ static int amdgpu_bad_page_notifier(struct notifier_block *nb,
 		amdgpu_ras_save_bad_pages(adev);
 	}
 
+	kfree(err_data.err_addr);
 	return NOTIFY_OK;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
index c4283987bb1e..84c241b9a2a1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
@@ -38,6 +38,7 @@
 #define EEPROM_I2C_MADDR_ARCTURUS_D342  0x0
 #define EEPROM_I2C_MADDR_SIENNA_CICHLID 0x0
 #define EEPROM_I2C_MADDR_ALDEBARAN      0x0
+#define EEPROM_I2C_MADDR_SMU_13_0_0     (0x54UL << 16)
 
 /*
  * The 2 macros bellow represent the actual size in bytes that
@@ -156,6 +157,15 @@ static bool __get_eeprom_i2c_addr(struct amdgpu_device *adev,
 		return false;
 	}
 
+	switch (adev->ip_versions[MP1_HWIP][0]) {
+	case IP_VERSION(13, 0, 0):
+		control->i2c_address = EEPROM_I2C_MADDR_SMU_13_0_0;
+		break;
+
+	default:
+		break;
+	}
+
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
index 32c86a0b145c..9da5ead50c90 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
@@ -23,6 +23,7 @@
 
 #include "amdgpu_reset.h"
 #include "aldebaran.h"
+#include "sienna_cichlid.h"
 
 int amdgpu_reset_add_handler(struct amdgpu_reset_control *reset_ctl,
 			     struct amdgpu_reset_handler *handler)
@@ -36,10 +37,15 @@ int amdgpu_reset_init(struct amdgpu_device *adev)
 {
 	int ret = 0;
 
+	adev->amdgpu_reset_level_mask = 0x1;
+
 	switch (adev->ip_versions[MP1_HWIP][0]) {
 	case IP_VERSION(13, 0, 2):
 		ret = aldebaran_reset_init(adev);
 		break;
+	case IP_VERSION(11, 0, 7):
+		ret = sienna_cichlid_reset_init(adev);
+		break;
 	default:
 		break;
 	}
@@ -55,6 +61,9 @@ int amdgpu_reset_fini(struct amdgpu_device *adev)
 	case IP_VERSION(13, 0, 2):
 		ret = aldebaran_reset_fini(adev);
 		break;
+	case IP_VERSION(11, 0, 7):
+		ret = sienna_cichlid_reset_fini(adev);
+		break;
 	default:
 		break;
 	}
@@ -67,6 +76,12 @@ int amdgpu_reset_prepare_hwcontext(struct amdgpu_device *adev,
 {
 	struct amdgpu_reset_handler *reset_handler = NULL;
 
+	if (!(adev->amdgpu_reset_level_mask & AMDGPU_RESET_LEVEL_MODE2))
+		return -ENOSYS;
+
+	if (test_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context->flags))
+		return -ENOSYS;
+
 	if (adev->reset_cntl && adev->reset_cntl->get_reset_handler)
 		reset_handler = adev->reset_cntl->get_reset_handler(
 			adev->reset_cntl, reset_context);
@@ -83,6 +98,12 @@ int amdgpu_reset_perform_reset(struct amdgpu_device *adev,
 	int ret;
 	struct amdgpu_reset_handler *reset_handler = NULL;
 
+	if (!(adev->amdgpu_reset_level_mask & AMDGPU_RESET_LEVEL_MODE2))
+		return -ENOSYS;
+
+	if (test_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context->flags))
+		return -ENOSYS;
+
 	if (adev->reset_cntl)
 		reset_handler = adev->reset_cntl->get_reset_handler(
 			adev->reset_cntl, reset_context);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
index ffda1560c648..f5318fedf2f0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
@@ -30,6 +30,8 @@ enum AMDGPU_RESET_FLAGS {
 
 	AMDGPU_NEED_FULL_RESET = 0,
 	AMDGPU_SKIP_HW_RESET = 1,
+	AMDGPU_SKIP_MODE2_RESET = 2,
+	AMDGPU_RESET_FOR_DEVICE_REMOVE = 3,
 };
 
 struct amdgpu_reset_context {
@@ -111,7 +113,8 @@ static inline bool amdgpu_reset_get_reset_domain(struct amdgpu_reset_domain *dom
 
 static inline void amdgpu_reset_put_reset_domain(struct amdgpu_reset_domain *domain)
 {
-	kref_put(&domain->refcount, amdgpu_reset_destroy_reset_domain);
+	if (domain)
+		kref_put(&domain->refcount, amdgpu_reset_destroy_reset_domain);
 }
 
 static inline bool amdgpu_reset_domain_schedule(struct amdgpu_reset_domain *domain,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index d3558c34d406..3e316b013fd9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -405,6 +405,9 @@ bool amdgpu_ring_soft_recovery(struct amdgpu_ring *ring, unsigned int vmid,
 {
 	ktime_t deadline = ktime_add_us(ktime_get(), 10000);
 
+	if (!(ring->adev->amdgpu_reset_level_mask & AMDGPU_RESET_LEVEL_SOFT_RECOVERY))
+		return false;
+
 	if (amdgpu_sriov_vf(ring->adev) || !ring->funcs->soft_recovery || !fence)
 		return false;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c
index 6373bfb47d55..012b72d00e04 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.c
@@ -272,3 +272,275 @@ void amdgpu_gfx_rlc_fini(struct amdgpu_device *adev)
 			      &adev->gfx.rlc.cp_table_gpu_addr,
 			      (void **)&adev->gfx.rlc.cp_table_ptr);
 }
+
+static int amdgpu_gfx_rlc_init_microcode_v2_0(struct amdgpu_device *adev)
+{
+	const struct common_firmware_header *common_hdr;
+	const struct rlc_firmware_header_v2_0 *rlc_hdr;
+	struct amdgpu_firmware_info *info;
+	unsigned int *tmp;
+	unsigned int i;
+
+	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
+
+	adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
+	adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
+	adev->gfx.rlc.save_and_restore_offset =
+		le32_to_cpu(rlc_hdr->save_and_restore_offset);
+	adev->gfx.rlc.clear_state_descriptor_offset =
+		le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
+	adev->gfx.rlc.avail_scratch_ram_locations =
+		le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
+	adev->gfx.rlc.reg_restore_list_size =
+		le32_to_cpu(rlc_hdr->reg_restore_list_size);
+	adev->gfx.rlc.reg_list_format_start =
+		le32_to_cpu(rlc_hdr->reg_list_format_start);
+	adev->gfx.rlc.reg_list_format_separate_start =
+		le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
+	adev->gfx.rlc.starting_offsets_start =
+		le32_to_cpu(rlc_hdr->starting_offsets_start);
+	adev->gfx.rlc.reg_list_format_size_bytes =
+		le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
+	adev->gfx.rlc.reg_list_size_bytes =
+		le32_to_cpu(rlc_hdr->reg_list_size_bytes);
+	adev->gfx.rlc.register_list_format =
+		kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
+			adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
+	if (!adev->gfx.rlc.register_list_format) {
+		dev_err(adev->dev, "failed to allocate memory for rlc register_list_format\n");
+		return -ENOMEM;
+	}
+
+	tmp = (unsigned int *)((uintptr_t)rlc_hdr +
+			le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
+	for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++)
+		adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
+
+	adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
+
+	tmp = (unsigned int *)((uintptr_t)rlc_hdr +
+			le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
+	for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
+		adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
+		info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
+		info->fw = adev->gfx.rlc_fw;
+		if (info->fw) {
+			common_hdr = (const struct common_firmware_header *)info->fw->data;
+			adev->firmware.fw_size +=
+				ALIGN(le32_to_cpu(common_hdr->ucode_size_bytes), PAGE_SIZE);
+		}
+	}
+
+	return 0;
+}
+
+static void amdgpu_gfx_rlc_init_microcode_v2_1(struct amdgpu_device *adev)
+{
+	const struct rlc_firmware_header_v2_1 *rlc_hdr;
+	struct amdgpu_firmware_info *info;
+
+	rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
+	adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
+	adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
+	adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
+	adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
+	adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
+	adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
+	adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
+	adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
+	adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
+	adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
+	adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
+	adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
+	adev->gfx.rlc.reg_list_format_direct_reg_list_length =
+		le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		if (adev->gfx.rlc.save_restore_list_cntl_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
+		}
+
+		if (adev->gfx.rlc.save_restore_list_gpm_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
+		}
+
+		if (adev->gfx.rlc.save_restore_list_srm_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
+		}
+	}
+}
+
+static void amdgpu_gfx_rlc_init_microcode_v2_2(struct amdgpu_device *adev)
+{
+	const struct rlc_firmware_header_v2_2 *rlc_hdr;
+	struct amdgpu_firmware_info *info;
+
+	rlc_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
+	adev->gfx.rlc.rlc_iram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_iram_ucode_size_bytes);
+	adev->gfx.rlc.rlc_iram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_iram_ucode_offset_bytes);
+	adev->gfx.rlc.rlc_dram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_dram_ucode_size_bytes);
+	adev->gfx.rlc.rlc_dram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_dram_ucode_offset_bytes);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		if (adev->gfx.rlc.rlc_iram_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_IRAM];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_IRAM;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.rlc_iram_ucode_size_bytes, PAGE_SIZE);
+		}
+
+		if (adev->gfx.rlc.rlc_dram_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_DRAM];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_DRAM;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.rlc_dram_ucode_size_bytes, PAGE_SIZE);
+		}
+	}
+}
+
+static void amdgpu_gfx_rlc_init_microcode_v2_3(struct amdgpu_device *adev)
+{
+	const struct rlc_firmware_header_v2_3 *rlc_hdr;
+	struct amdgpu_firmware_info *info;
+
+	rlc_hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
+	adev->gfx.rlcp_ucode_version = le32_to_cpu(rlc_hdr->rlcp_ucode_version);
+	adev->gfx.rlcp_ucode_feature_version = le32_to_cpu(rlc_hdr->rlcp_ucode_feature_version);
+	adev->gfx.rlc.rlcp_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlcp_ucode_size_bytes);
+	adev->gfx.rlc.rlcp_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlcp_ucode_offset_bytes);
+
+	adev->gfx.rlcv_ucode_version = le32_to_cpu(rlc_hdr->rlcv_ucode_version);
+	adev->gfx.rlcv_ucode_feature_version = le32_to_cpu(rlc_hdr->rlcv_ucode_feature_version);
+	adev->gfx.rlc.rlcv_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlcv_ucode_size_bytes);
+	adev->gfx.rlc.rlcv_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlcv_ucode_offset_bytes);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		if (adev->gfx.rlc.rlcp_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_P];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_P;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.rlcp_ucode_size_bytes, PAGE_SIZE);
+		}
+
+		if (adev->gfx.rlc.rlcv_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_V];
+			info->ucode_id = AMDGPU_UCODE_ID_RLC_V;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.rlcv_ucode_size_bytes, PAGE_SIZE);
+		}
+	}
+}
+
+static void amdgpu_gfx_rlc_init_microcode_v2_4(struct amdgpu_device *adev)
+{
+	const struct rlc_firmware_header_v2_4 *rlc_hdr;
+	struct amdgpu_firmware_info *info;
+
+	rlc_hdr = (const struct rlc_firmware_header_v2_4 *)adev->gfx.rlc_fw->data;
+	adev->gfx.rlc.global_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->global_tap_delays_ucode_size_bytes);
+	adev->gfx.rlc.global_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->global_tap_delays_ucode_offset_bytes);
+	adev->gfx.rlc.se0_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se0_tap_delays_ucode_size_bytes);
+	adev->gfx.rlc.se0_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se0_tap_delays_ucode_offset_bytes);
+	adev->gfx.rlc.se1_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se1_tap_delays_ucode_size_bytes);
+	adev->gfx.rlc.se1_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se1_tap_delays_ucode_offset_bytes);
+	adev->gfx.rlc.se2_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se2_tap_delays_ucode_size_bytes);
+	adev->gfx.rlc.se2_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se2_tap_delays_ucode_offset_bytes);
+	adev->gfx.rlc.se3_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se3_tap_delays_ucode_size_bytes);
+	adev->gfx.rlc.se3_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se3_tap_delays_ucode_offset_bytes);
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		if (adev->gfx.rlc.global_tap_delays_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS];
+			info->ucode_id = AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.global_tap_delays_ucode_size_bytes, PAGE_SIZE);
+		}
+
+		if (adev->gfx.rlc.se0_tap_delays_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE0_TAP_DELAYS];
+			info->ucode_id = AMDGPU_UCODE_ID_SE0_TAP_DELAYS;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.se0_tap_delays_ucode_size_bytes, PAGE_SIZE);
+		}
+
+		if (adev->gfx.rlc.se1_tap_delays_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE1_TAP_DELAYS];
+			info->ucode_id = AMDGPU_UCODE_ID_SE1_TAP_DELAYS;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.se1_tap_delays_ucode_size_bytes, PAGE_SIZE);
+		}
+
+		if (adev->gfx.rlc.se2_tap_delays_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE2_TAP_DELAYS];
+			info->ucode_id = AMDGPU_UCODE_ID_SE2_TAP_DELAYS;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.se2_tap_delays_ucode_size_bytes, PAGE_SIZE);
+		}
+
+		if (adev->gfx.rlc.se3_tap_delays_ucode_size_bytes) {
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE3_TAP_DELAYS];
+			info->ucode_id = AMDGPU_UCODE_ID_SE3_TAP_DELAYS;
+			info->fw = adev->gfx.rlc_fw;
+			adev->firmware.fw_size +=
+				ALIGN(adev->gfx.rlc.se3_tap_delays_ucode_size_bytes, PAGE_SIZE);
+		}
+	}
+}
+
+int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev,
+				  uint16_t version_major,
+				  uint16_t version_minor)
+{
+	int err;
+
+	if (version_major < 2) {
+		/* only support rlc_hdr v2.x and onwards */
+		dev_err(adev->dev, "unsupported rlc fw hdr\n");
+		return -EINVAL;
+	}
+
+	/* is_rlc_v2_1 is still used in APU code path */
+	if (version_major == 2 && version_minor == 1)
+		adev->gfx.rlc.is_rlc_v2_1 = true;
+
+	if (version_minor >= 0) {
+		err = amdgpu_gfx_rlc_init_microcode_v2_0(adev);
+		if (err) {
+			dev_err(adev->dev, "fail to init rlc v2_0 microcode\n");
+			return err;
+		}
+	}
+	if (version_minor >= 1)
+		amdgpu_gfx_rlc_init_microcode_v2_1(adev);
+	if (version_minor >= 2)
+		amdgpu_gfx_rlc_init_microcode_v2_2(adev);
+	if (version_minor == 3)
+		amdgpu_gfx_rlc_init_microcode_v2_3(adev);
+	if (version_minor == 4)
+		amdgpu_gfx_rlc_init_microcode_v2_4(adev);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
index 03ac36b2c2cf..23f060db9255 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_rlc.h
@@ -267,5 +267,7 @@ int amdgpu_gfx_rlc_init_csb(struct amdgpu_device *adev);
 int amdgpu_gfx_rlc_init_cpt(struct amdgpu_device *adev);
 void amdgpu_gfx_rlc_setup_cp_table(struct amdgpu_device *adev);
 void amdgpu_gfx_rlc_fini(struct amdgpu_device *adev);
-
+int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev,
+				  uint16_t version_major,
+				  uint16_t version_minor);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
index 42c1f050542f..3949b7e3907f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
@@ -21,6 +21,7 @@
  *
  */
 
+#include <linux/firmware.h>
 #include "amdgpu.h"
 #include "amdgpu_sdma.h"
 #include "amdgpu_ras.h"
@@ -150,3 +151,135 @@ int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
 	amdgpu_ras_interrupt_dispatch(adev, &ih_data);
 	return 0;
 }
+
+static int amdgpu_sdma_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
+{
+	int err = 0;
+	uint16_t version_major;
+	const struct common_firmware_header *header = NULL;
+	const struct sdma_firmware_header_v1_0 *hdr;
+	const struct sdma_firmware_header_v2_0 *hdr_v2;
+
+	err = amdgpu_ucode_validate(sdma_inst->fw);
+	if (err)
+		return err;
+
+	header = (const struct common_firmware_header *)
+		sdma_inst->fw->data;
+	version_major = le16_to_cpu(header->header_version_major);
+
+	switch (version_major) {
+	case 1:
+		hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;
+		sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
+		sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
+		break;
+	case 2:
+		hdr_v2 = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data;
+		sdma_inst->fw_version = le32_to_cpu(hdr_v2->header.ucode_version);
+		sdma_inst->feature_version = le32_to_cpu(hdr_v2->ucode_feature_version);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (sdma_inst->feature_version >= 20)
+		sdma_inst->burst_nop = true;
+
+	return 0;
+}
+
+void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
+				  bool duplicate)
+{
+	int i;
+
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		release_firmware(adev->sdma.instance[i].fw);
+		if (duplicate)
+			break;
+	}
+
+	memset((void *)adev->sdma.instance, 0,
+	       sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
+}
+
+int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
+			       char *fw_name, u32 instance,
+			       bool duplicate)
+{
+	struct amdgpu_firmware_info *info = NULL;
+	const struct common_firmware_header *header = NULL;
+	int err = 0, i;
+	const struct sdma_firmware_header_v2_0 *sdma_hdr;
+	uint16_t version_major;
+
+	err = request_firmware(&adev->sdma.instance[instance].fw, fw_name, adev->dev);
+	if (err)
+		goto out;
+
+	header = (const struct common_firmware_header *)
+		adev->sdma.instance[instance].fw->data;
+	version_major = le16_to_cpu(header->header_version_major);
+
+	if ((duplicate && instance) || (!duplicate && version_major > 1))
+		return -EINVAL;
+
+	err = amdgpu_sdma_init_inst_ctx(&adev->sdma.instance[instance]);
+	if (err)
+		goto out;
+
+	if (duplicate) {
+		for (i = 1; i < adev->sdma.num_instances; i++)
+			memcpy((void *)&adev->sdma.instance[i],
+			       (void *)&adev->sdma.instance[0],
+			       sizeof(struct amdgpu_sdma_instance));
+	}
+
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	DRM_DEBUG("psp_load == '%s'\n",
+		  adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+		switch (version_major) {
+		case 1:
+			for (i = 0; i < adev->sdma.num_instances; i++) {
+				if (!duplicate && (instance != i))
+					continue;
+				else {
+					info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
+					info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
+					info->fw = adev->sdma.instance[i].fw;
+					adev->firmware.fw_size +=
+						ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
+				}
+			}
+			break;
+		case 2:
+			sdma_hdr = (const struct sdma_firmware_header_v2_0 *)
+				adev->sdma.instance[0].fw->data;
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0];
+			info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0;
+			info->fw = adev->sdma.instance[0].fw;
+			adev->firmware.fw_size +=
+				ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE);
+			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1];
+			info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1;
+			info->fw = adev->sdma.instance[0].fw;
+			adev->firmware.fw_size +=
+				ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);
+			break;
+		default:
+			return -EINVAL;
+		}
+	}
+
+out:
+	if (err) {
+		DRM_ERROR("SDMA: Failed to init firmware \"%s\"\n", fw_name);
+		amdgpu_sdma_destroy_inst_ctx(adev, duplicate);
+	}
+	return err;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
index 53ac3ebae8d6..d2d88279fefb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.h
@@ -124,4 +124,8 @@ int amdgpu_sdma_process_ras_data_cb(struct amdgpu_device *adev,
 int amdgpu_sdma_process_ecc_irq(struct amdgpu_device *adev,
 				      struct amdgpu_irq_src *source,
 				      struct amdgpu_iv_entry *entry);
+int amdgpu_sdma_init_microcode(struct amdgpu_device *adev,
+        char *fw_name, u32 instance, bool duplicate);
+void amdgpu_sdma_destroy_inst_ctx(struct amdgpu_device *adev,
+        bool duplicate);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index 504af1b93bfa..090e66a1b284 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: MIT
 /*
  * Copyright 2014 Advanced Micro Devices, Inc.
  * All Rights Reserved.
@@ -315,6 +316,7 @@ struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
 	struct hlist_node *tmp;
 	struct dma_fence *f;
 	int i;
+
 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
 
 		f = e->fence;
@@ -392,7 +394,7 @@ void amdgpu_sync_free(struct amdgpu_sync *sync)
 {
 	struct amdgpu_sync_entry *e;
 	struct hlist_node *tmp;
-	unsigned i;
+	unsigned int i;
 
 	hash_for_each_safe(sync->fences, i, tmp, e, node) {
 		hash_del(&e->node);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 06dfcf297a8d..5e6ddc7e101c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -140,8 +140,10 @@ TRACE_EVENT(amdgpu_bo_create,
 );
 
 TRACE_EVENT(amdgpu_cs,
-	    TP_PROTO(struct amdgpu_cs_parser *p, int i),
-	    TP_ARGS(p, i),
+	    TP_PROTO(struct amdgpu_cs_parser *p,
+		     struct amdgpu_job *job,
+		     struct amdgpu_ib *ib),
+	    TP_ARGS(p, job, ib),
 	    TP_STRUCT__entry(
 			     __field(struct amdgpu_bo_list *, bo_list)
 			     __field(u32, ring)
@@ -151,10 +153,10 @@ TRACE_EVENT(amdgpu_cs,
 
 	    TP_fast_assign(
 			   __entry->bo_list = p->bo_list;
-			   __entry->ring = to_amdgpu_ring(p->entity->rq->sched)->idx;
-			   __entry->dw = p->job->ibs[i].length_dw;
+			   __entry->ring = to_amdgpu_ring(job->base.sched)->idx;
+			   __entry->dw = ib->length_dw;
 			   __entry->fences = amdgpu_fence_count_emitted(
-				to_amdgpu_ring(p->entity->rq->sched));
+				to_amdgpu_ring(job->base.sched));
 			   ),
 	    TP_printk("bo_list=%p, ring=%u, dw=%u, fences=%u",
 		      __entry->bo_list, __entry->ring, __entry->dw,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 134575a3893c..b1c455329023 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -471,7 +471,8 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo, bool evict,
 
 	adev = amdgpu_ttm_adev(bo->bdev);
 
-	if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
+	if (!old_mem || (old_mem->mem_type == TTM_PL_SYSTEM &&
+			 bo->ttm == NULL)) {
 		ttm_bo_move_null(bo, new_mem);
 		goto out;
 	}
@@ -1329,11 +1330,12 @@ uint64_t amdgpu_ttm_tt_pte_flags(struct amdgpu_device *adev, struct ttm_tt *ttm,
 static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 					    const struct ttm_place *place)
 {
-	unsigned long num_pages = bo->resource->num_pages;
 	struct dma_resv_iter resv_cursor;
-	struct amdgpu_res_cursor cursor;
 	struct dma_fence *f;
 
+	if (!amdgpu_bo_is_amdgpu_bo(bo))
+		return ttm_bo_eviction_valuable(bo, place);
+
 	/* Swapout? */
 	if (bo->resource->mem_type == TTM_PL_SYSTEM)
 		return true;
@@ -1352,40 +1354,20 @@ static bool amdgpu_ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 			return false;
 	}
 
-	switch (bo->resource->mem_type) {
-	case AMDGPU_PL_PREEMPT:
-		/* Preemptible BOs don't own system resources managed by the
-		 * driver (pages, VRAM, GART space). They point to resources
-		 * owned by someone else (e.g. pageable memory in user mode
-		 * or a DMABuf). They are used in a preemptible context so we
-		 * can guarantee no deadlocks and good QoS in case of MMU
-		 * notifiers or DMABuf move notifiers from the resource owner.
-		 */
+	/* Preemptible BOs don't own system resources managed by the
+	 * driver (pages, VRAM, GART space). They point to resources
+	 * owned by someone else (e.g. pageable memory in user mode
+	 * or a DMABuf). They are used in a preemptible context so we
+	 * can guarantee no deadlocks and good QoS in case of MMU
+	 * notifiers or DMABuf move notifiers from the resource owner.
+	 */
+	if (bo->resource->mem_type == AMDGPU_PL_PREEMPT)
 		return false;
-	case TTM_PL_TT:
-		if (amdgpu_bo_is_amdgpu_bo(bo) &&
-		    amdgpu_bo_encrypted(ttm_to_amdgpu_bo(bo)))
-			return false;
-		return true;
 
-	case TTM_PL_VRAM:
-		/* Check each drm MM node individually */
-		amdgpu_res_first(bo->resource, 0, (u64)num_pages << PAGE_SHIFT,
-				 &cursor);
-		while (cursor.remaining) {
-			if (place->fpfn < PFN_DOWN(cursor.start + cursor.size)
-			    && !(place->lpfn &&
-				 place->lpfn <= PFN_DOWN(cursor.start)))
-				return true;
-
-			amdgpu_res_next(&cursor, cursor.size);
-		}
+	if (bo->resource->mem_type == TTM_PL_TT &&
+	    amdgpu_bo_encrypted(ttm_to_amdgpu_bo(bo)))
 		return false;
 
-	default:
-		break;
-	}
-
 	return ttm_bo_eviction_valuable(bo, place);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
index 939c8614f0e3..dd0bc649a57d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
@@ -164,70 +164,138 @@ void amdgpu_ucode_print_rlc_hdr(const struct common_firmware_header *hdr)
 	} else if (version_major == 2) {
 		const struct rlc_firmware_header_v2_0 *rlc_hdr =
 			container_of(hdr, struct rlc_firmware_header_v2_0, header);
+		const struct rlc_firmware_header_v2_1 *rlc_hdr_v2_1 =
+			container_of(rlc_hdr, struct rlc_firmware_header_v2_1, v2_0);
+		const struct rlc_firmware_header_v2_2 *rlc_hdr_v2_2 =
+			container_of(rlc_hdr_v2_1, struct rlc_firmware_header_v2_2, v2_1);
+		const struct rlc_firmware_header_v2_3 *rlc_hdr_v2_3 =
+			container_of(rlc_hdr_v2_2, struct rlc_firmware_header_v2_3, v2_2);
+		const struct rlc_firmware_header_v2_4 *rlc_hdr_v2_4 =
+			container_of(rlc_hdr_v2_3, struct rlc_firmware_header_v2_4, v2_3);
 
-		DRM_DEBUG("ucode_feature_version: %u\n",
-			  le32_to_cpu(rlc_hdr->ucode_feature_version));
-		DRM_DEBUG("jt_offset: %u\n", le32_to_cpu(rlc_hdr->jt_offset));
-		DRM_DEBUG("jt_size: %u\n", le32_to_cpu(rlc_hdr->jt_size));
-		DRM_DEBUG("save_and_restore_offset: %u\n",
-			  le32_to_cpu(rlc_hdr->save_and_restore_offset));
-		DRM_DEBUG("clear_state_descriptor_offset: %u\n",
-			  le32_to_cpu(rlc_hdr->clear_state_descriptor_offset));
-		DRM_DEBUG("avail_scratch_ram_locations: %u\n",
-			  le32_to_cpu(rlc_hdr->avail_scratch_ram_locations));
-		DRM_DEBUG("reg_restore_list_size: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_restore_list_size));
-		DRM_DEBUG("reg_list_format_start: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_format_start));
-		DRM_DEBUG("reg_list_format_separate_start: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_format_separate_start));
-		DRM_DEBUG("starting_offsets_start: %u\n",
-			  le32_to_cpu(rlc_hdr->starting_offsets_start));
-		DRM_DEBUG("reg_list_format_size_bytes: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_format_size_bytes));
-		DRM_DEBUG("reg_list_format_array_offset_bytes: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
-		DRM_DEBUG("reg_list_size_bytes: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_size_bytes));
-		DRM_DEBUG("reg_list_array_offset_bytes: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
-		DRM_DEBUG("reg_list_format_separate_size_bytes: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_format_separate_size_bytes));
-		DRM_DEBUG("reg_list_format_separate_array_offset_bytes: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_format_separate_array_offset_bytes));
-		DRM_DEBUG("reg_list_separate_size_bytes: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_separate_size_bytes));
-		DRM_DEBUG("reg_list_separate_array_offset_bytes: %u\n",
-			  le32_to_cpu(rlc_hdr->reg_list_separate_array_offset_bytes));
-		if (version_minor == 1) {
-			const struct rlc_firmware_header_v2_1 *v2_1 =
-				container_of(rlc_hdr, struct rlc_firmware_header_v2_1, v2_0);
+		switch (version_minor) {
+		case 0:
+			/* rlc_hdr v2_0 */
+			DRM_DEBUG("ucode_feature_version: %u\n",
+				  le32_to_cpu(rlc_hdr->ucode_feature_version));
+			DRM_DEBUG("jt_offset: %u\n", le32_to_cpu(rlc_hdr->jt_offset));
+			DRM_DEBUG("jt_size: %u\n", le32_to_cpu(rlc_hdr->jt_size));
+			DRM_DEBUG("save_and_restore_offset: %u\n",
+				  le32_to_cpu(rlc_hdr->save_and_restore_offset));
+			DRM_DEBUG("clear_state_descriptor_offset: %u\n",
+				  le32_to_cpu(rlc_hdr->clear_state_descriptor_offset));
+			DRM_DEBUG("avail_scratch_ram_locations: %u\n",
+				  le32_to_cpu(rlc_hdr->avail_scratch_ram_locations));
+			DRM_DEBUG("reg_restore_list_size: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_restore_list_size));
+			DRM_DEBUG("reg_list_format_start: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_list_format_start));
+			DRM_DEBUG("reg_list_format_separate_start: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_list_format_separate_start));
+			DRM_DEBUG("starting_offsets_start: %u\n",
+				  le32_to_cpu(rlc_hdr->starting_offsets_start));
+			DRM_DEBUG("reg_list_format_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_list_format_size_bytes));
+			DRM_DEBUG("reg_list_format_array_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
+			DRM_DEBUG("reg_list_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_list_size_bytes));
+			DRM_DEBUG("reg_list_array_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
+			DRM_DEBUG("reg_list_format_separate_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_list_format_separate_size_bytes));
+			DRM_DEBUG("reg_list_format_separate_array_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_list_format_separate_array_offset_bytes));
+			DRM_DEBUG("reg_list_separate_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_list_separate_size_bytes));
+			DRM_DEBUG("reg_list_separate_array_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr->reg_list_separate_array_offset_bytes));
+			break;
+		case 1:
+			/* rlc_hdr v2_1 */
 			DRM_DEBUG("reg_list_format_direct_reg_list_length: %u\n",
-				  le32_to_cpu(v2_1->reg_list_format_direct_reg_list_length));
+				  le32_to_cpu(rlc_hdr_v2_1->reg_list_format_direct_reg_list_length));
 			DRM_DEBUG("save_restore_list_cntl_ucode_ver: %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_cntl_ucode_ver));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_cntl_ucode_ver));
 			DRM_DEBUG("save_restore_list_cntl_feature_ver: %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_cntl_feature_ver));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_cntl_feature_ver));
 			DRM_DEBUG("save_restore_list_cntl_size_bytes %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_cntl_size_bytes));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_cntl_size_bytes));
 			DRM_DEBUG("save_restore_list_cntl_offset_bytes: %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_cntl_offset_bytes));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_cntl_offset_bytes));
 			DRM_DEBUG("save_restore_list_gpm_ucode_ver: %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_gpm_ucode_ver));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_gpm_ucode_ver));
 			DRM_DEBUG("save_restore_list_gpm_feature_ver: %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_gpm_feature_ver));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_gpm_feature_ver));
 			DRM_DEBUG("save_restore_list_gpm_size_bytes %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_gpm_size_bytes));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_gpm_size_bytes));
 			DRM_DEBUG("save_restore_list_gpm_offset_bytes: %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_gpm_offset_bytes));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_gpm_offset_bytes));
 			DRM_DEBUG("save_restore_list_srm_ucode_ver: %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_srm_ucode_ver));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_srm_ucode_ver));
 			DRM_DEBUG("save_restore_list_srm_feature_ver: %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_srm_feature_ver));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_srm_feature_ver));
 			DRM_DEBUG("save_restore_list_srm_size_bytes %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_srm_size_bytes));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_srm_size_bytes));
 			DRM_DEBUG("save_restore_list_srm_offset_bytes: %u\n",
-				  le32_to_cpu(v2_1->save_restore_list_srm_offset_bytes));
+				  le32_to_cpu(rlc_hdr_v2_1->save_restore_list_srm_offset_bytes));
+			break;
+		case 2:
+			/* rlc_hdr v2_2 */
+			DRM_DEBUG("rlc_iram_ucode_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_2->rlc_iram_ucode_size_bytes));
+			DRM_DEBUG("rlc_iram_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_2->rlc_iram_ucode_offset_bytes));
+			DRM_DEBUG("rlc_dram_ucode_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_2->rlc_dram_ucode_size_bytes));
+			DRM_DEBUG("rlc_dram_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_2->rlc_dram_ucode_offset_bytes));
+			break;
+		case 3:
+			/* rlc_hdr v2_3 */
+			DRM_DEBUG("rlcp_ucode_version: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_3->rlcp_ucode_version));
+			DRM_DEBUG("rlcp_ucode_feature_version: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_3->rlcp_ucode_feature_version));
+			DRM_DEBUG("rlcp_ucode_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_3->rlcp_ucode_size_bytes));
+			DRM_DEBUG("rlcp_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_3->rlcp_ucode_offset_bytes));
+			DRM_DEBUG("rlcv_ucode_version: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_3->rlcv_ucode_version));
+			DRM_DEBUG("rlcv_ucode_feature_version: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_3->rlcv_ucode_feature_version));
+			DRM_DEBUG("rlcv_ucode_size_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_3->rlcv_ucode_size_bytes));
+			DRM_DEBUG("rlcv_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_3->rlcv_ucode_offset_bytes));
+			break;
+		case 4:
+			/* rlc_hdr v2_4 */
+			DRM_DEBUG("global_tap_delays_ucode_size_bytes :%u\n",
+				  le32_to_cpu(rlc_hdr_v2_4->global_tap_delays_ucode_size_bytes));
+			DRM_DEBUG("global_tap_delays_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_4->global_tap_delays_ucode_offset_bytes));
+			DRM_DEBUG("se0_tap_delays_ucode_size_bytes :%u\n",
+				  le32_to_cpu(rlc_hdr_v2_4->se0_tap_delays_ucode_size_bytes));
+			DRM_DEBUG("se0_tap_delays_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_4->se0_tap_delays_ucode_offset_bytes));
+			DRM_DEBUG("se1_tap_delays_ucode_size_bytes :%u\n",
+				  le32_to_cpu(rlc_hdr_v2_4->se1_tap_delays_ucode_size_bytes));
+			DRM_DEBUG("se1_tap_delays_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_4->se1_tap_delays_ucode_offset_bytes));
+			DRM_DEBUG("se2_tap_delays_ucode_size_bytes :%u\n",
+				  le32_to_cpu(rlc_hdr_v2_4->se2_tap_delays_ucode_size_bytes));
+			DRM_DEBUG("se2_tap_delays_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_4->se2_tap_delays_ucode_offset_bytes));
+			DRM_DEBUG("se3_tap_delays_ucode_size_bytes :%u\n",
+				  le32_to_cpu(rlc_hdr_v2_4->se3_tap_delays_ucode_size_bytes));
+			DRM_DEBUG("se3_tap_delays_ucode_offset_bytes: %u\n",
+				  le32_to_cpu(rlc_hdr_v2_4->se3_tap_delays_ucode_offset_bytes));
+			break;
+		default:
+			DRM_ERROR("Unknown RLC v2 ucode: v2.%u\n", version_minor);
+			break;
 		}
 	} else {
 		DRM_ERROR("Unknown RLC ucode version: %u.%u\n", version_major, version_minor);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
index 96b6cf4c4d54..1c36235b4539 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h
@@ -124,6 +124,7 @@ enum psp_fw_type {
 	PSP_FW_TYPE_PSP_SOC_DRV,
 	PSP_FW_TYPE_PSP_INTF_DRV,
 	PSP_FW_TYPE_PSP_DBG_DRV,
+	PSP_FW_TYPE_PSP_RAS_DRV,
 };
 
 /* version_major=2, version_minor=0 */
@@ -260,8 +261,12 @@ struct rlc_firmware_header_v2_2 {
 /* version_major=2, version_minor=3 */
 struct rlc_firmware_header_v2_3 {
     struct rlc_firmware_header_v2_2 v2_2;
+    uint32_t rlcp_ucode_version;
+    uint32_t rlcp_ucode_feature_version;
     uint32_t rlcp_ucode_size_bytes;
     uint32_t rlcp_ucode_offset_bytes;
+    uint32_t rlcv_ucode_version;
+    uint32_t rlcv_ucode_feature_version;
     uint32_t rlcv_ucode_size_bytes;
     uint32_t rlcv_ucode_offset_bytes;
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
index 3629d8f292ef..2fb4951a6433 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.h
@@ -22,6 +22,8 @@
 #define __AMDGPU_UMC_H__
 #include "amdgpu_ras.h"
 
+#define UMC_INVALID_ADDR 0x1ULL
+
 /*
  * (addr / 256) * 4096, the higher 26 bits in ErrorAddr
  * is the index of 4KB block
@@ -51,6 +53,10 @@ struct amdgpu_umc_ras {
 	struct amdgpu_ras_block_object ras_block;
 	void (*err_cnt_init)(struct amdgpu_device *adev);
 	bool (*query_ras_poison_mode)(struct amdgpu_device *adev);
+	void (*convert_ras_error_address)(struct amdgpu_device *adev,
+						 struct ras_err_data *err_data,
+						 uint32_t umc_reg_offset, uint32_t ch_inst,
+						 uint32_t umc_inst, uint64_t mca_addr);
 	void (*ecc_info_query_ras_error_count)(struct amdgpu_device *adev,
 				      void *ras_error_status);
 	void (*ecc_info_query_ras_error_address)(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index f36e4f08db6d..0b52af415b28 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -191,7 +191,7 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev)
 		fw_name = FIRMWARE_VCN4_0_2;
 		if ((adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) &&
 			(adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG))
-			adev->vcn.indirect_sram = false;
+			adev->vcn.indirect_sram = true;
 		break;
 	case IP_VERSION(4, 0, 4):
 		fw_name = FIRMWARE_VCN4_0_4;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
index 60c608144480..253ea6b159df 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.h
@@ -161,6 +161,8 @@
 #define AMDGPU_VCN_SW_RING_FLAG		(1 << 9)
 #define AMDGPU_VCN_FW_LOGGING_FLAG	(1 << 10)
 #define AMDGPU_VCN_SMU_VERSION_INFO_FLAG (1 << 11)
+#define AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG (1 << 11)
+#define AMDGPU_VCN_VF_RB_SETUP_FLAG (1 << 14)
 
 #define AMDGPU_VCN_IB_FLAG_DECODE_BUFFER	0x00000001
 #define AMDGPU_VCN_CMD_FLAG_MSG_BUFFER		0x00000001
@@ -170,6 +172,9 @@
 #define VCN_CODEC_DISABLE_MASK_HEVC (1 << 2)
 #define VCN_CODEC_DISABLE_MASK_H264 (1 << 3)
 
+#define AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU (0)
+#define AMDGPU_VCN_SMU_DPM_INTERFACE_APU (1)
+
 enum fw_queue_mode {
 	FW_QUEUE_RING_RESET = 1,
 	FW_QUEUE_DPG_HOLD_OFF = 2,
@@ -317,12 +322,26 @@ struct amdgpu_fw_shared {
 	struct amdgpu_fw_shared_smu_interface_info smu_interface_info;
 };
 
+struct amdgpu_fw_shared_rb_setup {
+	uint32_t is_rb_enabled_flags;
+	uint32_t rb_addr_lo;
+	uint32_t rb_addr_hi;
+	uint32_t  rb_size;
+	uint32_t  rb4_addr_lo;
+	uint32_t  rb4_addr_hi;
+	uint32_t  rb4_size;
+	uint32_t  reserved[6];
+};
+
 struct amdgpu_vcn4_fw_shared {
 	uint32_t present_flag_0;
 	uint8_t pad[12];
 	struct amdgpu_fw_shared_unified_queue_struct sq;
 	uint8_t pad1[8];
 	struct amdgpu_fw_shared_fw_logging fw_log;
+	uint8_t pad2[20];
+	struct amdgpu_fw_shared_rb_setup rb_setup;
+	struct amdgpu_fw_shared_smu_interface_info smu_dpm_interface;
 };
 
 struct amdgpu_vcn_fwlog {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index 9be57389301b..e4af40b9a8aa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -690,7 +690,6 @@ void amdgpu_virt_exchange_data(struct amdgpu_device *adev)
 	}
 }
 
-
 void amdgpu_detect_virtualization(struct amdgpu_device *adev)
 {
 	uint32_t reg;
@@ -707,6 +706,7 @@ void amdgpu_detect_virtualization(struct amdgpu_device *adev)
 	case CHIP_SIENNA_CICHLID:
 	case CHIP_ARCTURUS:
 	case CHIP_ALDEBARAN:
+	case CHIP_IP_DISCOVERY:
 		reg = RREG32(mmRCC_IOV_FUNC_IDENTIFIER);
 		break;
 	default: /* other chip doesn't support SRIOV */
@@ -750,6 +750,7 @@ void amdgpu_detect_virtualization(struct amdgpu_device *adev)
 		case CHIP_NAVI10:
 		case CHIP_NAVI12:
 		case CHIP_SIENNA_CICHLID:
+		case CHIP_IP_DISCOVERY:
 			nv_set_virt_ops(adev);
 			/* try send GPU_INIT_DATA request to host */
 			amdgpu_virt_request_init_data(adev);
@@ -807,6 +808,60 @@ enum amdgpu_sriov_vf_mode amdgpu_virt_get_sriov_vf_mode(struct amdgpu_device *ad
 	return mode;
 }
 
+bool amdgpu_virt_fw_load_skip_check(struct amdgpu_device *adev, uint32_t ucode_id)
+{
+	switch (adev->ip_versions[MP0_HWIP][0]) {
+	case IP_VERSION(13, 0, 0):
+		/* no vf autoload, white list */
+		if (ucode_id == AMDGPU_UCODE_ID_VCN1 ||
+		    ucode_id == AMDGPU_UCODE_ID_VCN)
+			return false;
+		else
+			return true;
+	case IP_VERSION(13, 0, 10):
+		/* white list */
+		if (ucode_id == AMDGPU_UCODE_ID_CAP
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK
+		|| ucode_id == AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK
+		|| ucode_id == AMDGPU_UCODE_ID_CP_MES
+		|| ucode_id == AMDGPU_UCODE_ID_CP_MES_DATA
+		|| ucode_id == AMDGPU_UCODE_ID_CP_MES1
+		|| ucode_id == AMDGPU_UCODE_ID_CP_MES1_DATA
+		|| ucode_id == AMDGPU_UCODE_ID_VCN1
+		|| ucode_id == AMDGPU_UCODE_ID_VCN)
+			return false;
+		else
+			return true;
+	default:
+		/* lagacy black list */
+		if (ucode_id == AMDGPU_UCODE_ID_SDMA0
+		    || ucode_id == AMDGPU_UCODE_ID_SDMA1
+		    || ucode_id == AMDGPU_UCODE_ID_SDMA2
+		    || ucode_id == AMDGPU_UCODE_ID_SDMA3
+		    || ucode_id == AMDGPU_UCODE_ID_SDMA4
+		    || ucode_id == AMDGPU_UCODE_ID_SDMA5
+		    || ucode_id == AMDGPU_UCODE_ID_SDMA6
+		    || ucode_id == AMDGPU_UCODE_ID_SDMA7
+		    || ucode_id == AMDGPU_UCODE_ID_RLC_G
+		    || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL
+		    || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM
+		    || ucode_id == AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM
+		    || ucode_id == AMDGPU_UCODE_ID_SMC)
+			return true;
+		else
+			return false;
+	}
+}
+
 void amdgpu_virt_update_sriov_video_codec(struct amdgpu_device *adev,
 			struct amdgpu_video_codec_info *encode, uint32_t encode_array_size,
 			struct amdgpu_video_codec_info *decode, uint32_t decode_array_size)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 239f232f9c02..d94c31e68a14 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -253,6 +253,9 @@ struct amdgpu_virt {
 	uint32_t decode_max_frame_pixels;
 	uint32_t encode_max_dimension_pixels;
 	uint32_t encode_max_frame_pixels;
+
+	/* the ucode id to signal the autoload */
+	uint32_t autoload_ucode_id;
 };
 
 struct amdgpu_video_codec_info;
@@ -343,4 +346,6 @@ void amdgpu_sriov_wreg(struct amdgpu_device *adev,
 		       u32 acc_flags, u32 hwip);
 u32 amdgpu_sriov_rreg(struct amdgpu_device *adev,
 		      u32 offset, u32 acc_flags, u32 hwip);
+bool amdgpu_virt_fw_load_skip_check(struct amdgpu_device *adev,
+			uint32_t ucode_id);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
index 576849e95296..f4b5301ea2a0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
@@ -282,8 +282,8 @@ static int amdgpu_vkms_plane_atomic_check(struct drm_plane *plane,
 		return PTR_ERR(crtc_state);
 
 	ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, true);
 	if (ret != 0)
 		return ret;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 690fd4f639f1..83b0c5d86e48 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -183,10 +183,12 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 	struct amdgpu_bo *bo = vm_bo->bo;
 
 	vm_bo->moved = true;
+	spin_lock(&vm_bo->vm->status_lock);
 	if (bo->tbo.type == ttm_bo_type_kernel)
 		list_move(&vm_bo->vm_status, &vm->evicted);
 	else
 		list_move_tail(&vm_bo->vm_status, &vm->evicted);
+	spin_unlock(&vm_bo->vm->status_lock);
 }
 /**
  * amdgpu_vm_bo_moved - vm_bo is moved
@@ -198,7 +200,9 @@ static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
  */
 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 {
+	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
+	spin_unlock(&vm_bo->vm->status_lock);
 }
 
 /**
@@ -211,7 +215,9 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
  */
 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
 {
+	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
+	spin_unlock(&vm_bo->vm->status_lock);
 	vm_bo->moved = false;
 }
 
@@ -225,9 +231,9 @@ static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
  */
 static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
 {
-	spin_lock(&vm_bo->vm->invalidated_lock);
+	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
-	spin_unlock(&vm_bo->vm->invalidated_lock);
+	spin_unlock(&vm_bo->vm->status_lock);
 }
 
 /**
@@ -240,10 +246,13 @@ static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
  */
 static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
 {
-	if (vm_bo->bo->parent)
+	if (vm_bo->bo->parent) {
+		spin_lock(&vm_bo->vm->status_lock);
 		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
-	else
+		spin_unlock(&vm_bo->vm->status_lock);
+	} else {
 		amdgpu_vm_bo_idle(vm_bo);
+	}
 }
 
 /**
@@ -256,9 +265,9 @@ static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
  */
 static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
 {
-	spin_lock(&vm_bo->vm->invalidated_lock);
+	spin_lock(&vm_bo->vm->status_lock);
 	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
-	spin_unlock(&vm_bo->vm->invalidated_lock);
+	spin_unlock(&vm_bo->vm->status_lock);
 }
 
 /**
@@ -363,12 +372,20 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			      int (*validate)(void *p, struct amdgpu_bo *bo),
 			      void *param)
 {
-	struct amdgpu_vm_bo_base *bo_base, *tmp;
+	struct amdgpu_vm_bo_base *bo_base;
+	struct amdgpu_bo *shadow;
+	struct amdgpu_bo *bo;
 	int r;
 
-	list_for_each_entry_safe(bo_base, tmp, &vm->evicted, vm_status) {
-		struct amdgpu_bo *bo = bo_base->bo;
-		struct amdgpu_bo *shadow = amdgpu_bo_shadowed(bo);
+	spin_lock(&vm->status_lock);
+	while (!list_empty(&vm->evicted)) {
+		bo_base = list_first_entry(&vm->evicted,
+					   struct amdgpu_vm_bo_base,
+					   vm_status);
+		spin_unlock(&vm->status_lock);
+
+		bo = bo_base->bo;
+		shadow = amdgpu_bo_shadowed(bo);
 
 		r = validate(param, bo);
 		if (r)
@@ -385,7 +402,9 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
 			amdgpu_vm_bo_relocated(bo_base);
 		}
+		spin_lock(&vm->status_lock);
 	}
+	spin_unlock(&vm->status_lock);
 
 	amdgpu_vm_eviction_lock(vm);
 	vm->evicting = false;
@@ -406,13 +425,18 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
  */
 bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 {
+	bool empty;
 	bool ret;
 
 	amdgpu_vm_eviction_lock(vm);
 	ret = !vm->evicting;
 	amdgpu_vm_eviction_unlock(vm);
 
-	return ret && list_empty(&vm->evicted);
+	spin_lock(&vm->status_lock);
+	empty = list_empty(&vm->evicted);
+	spin_unlock(&vm->status_lock);
+
+	return ret && empty;
 }
 
 /**
@@ -680,9 +704,14 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	struct amdgpu_vm_update_params params;
 	struct amdgpu_vm_bo_base *entry;
 	bool flush_tlb_needed = false;
+	LIST_HEAD(relocated);
 	int r, idx;
 
-	if (list_empty(&vm->relocated))
+	spin_lock(&vm->status_lock);
+	list_splice_init(&vm->relocated, &relocated);
+	spin_unlock(&vm->status_lock);
+
+	if (list_empty(&relocated))
 		return 0;
 
 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
@@ -697,7 +726,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (r)
 		goto error;
 
-	list_for_each_entry(entry, &vm->relocated, vm_status) {
+	list_for_each_entry(entry, &relocated, vm_status) {
 		/* vm_flush_needed after updating moved PDEs */
 		flush_tlb_needed |= entry->moved;
 
@@ -713,9 +742,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (flush_tlb_needed)
 		atomic64_inc(&vm->tlb_seq);
 
-	while (!list_empty(&vm->relocated)) {
-		entry = list_first_entry(&vm->relocated,
-					 struct amdgpu_vm_bo_base,
+	while (!list_empty(&relocated)) {
+		entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
 					 vm_status);
 		amdgpu_vm_bo_idle(entry);
 	}
@@ -912,6 +940,7 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
 {
 	struct amdgpu_bo_va *bo_va, *tmp;
 
+	spin_lock(&vm->status_lock);
 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
@@ -936,7 +965,6 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
 				gtt_mem, cpu_mem);
 	}
-	spin_lock(&vm->invalidated_lock);
 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
@@ -949,7 +977,7 @@ void amdgpu_vm_get_memory(struct amdgpu_vm *vm, uint64_t *vram_mem,
 		amdgpu_bo_get_memory(bo_va->base.bo, vram_mem,
 				gtt_mem, cpu_mem);
 	}
-	spin_unlock(&vm->invalidated_lock);
+	spin_unlock(&vm->status_lock);
 }
 /**
  * amdgpu_vm_bo_update - update all BO mappings in the vm page table
@@ -1278,24 +1306,29 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 			   struct amdgpu_vm *vm)
 {
-	struct amdgpu_bo_va *bo_va, *tmp;
+	struct amdgpu_bo_va *bo_va;
 	struct dma_resv *resv;
 	bool clear;
 	int r;
 
-	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
+	spin_lock(&vm->status_lock);
+	while (!list_empty(&vm->moved)) {
+		bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
+					 base.vm_status);
+		spin_unlock(&vm->status_lock);
+
 		/* Per VM BOs never need to bo cleared in the page tables */
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (r)
 			return r;
+		spin_lock(&vm->status_lock);
 	}
 
-	spin_lock(&vm->invalidated_lock);
 	while (!list_empty(&vm->invalidated)) {
 		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
 					 base.vm_status);
 		resv = bo_va->base.bo->tbo.base.resv;
-		spin_unlock(&vm->invalidated_lock);
+		spin_unlock(&vm->status_lock);
 
 		/* Try to reserve the BO to avoid clearing its ptes */
 		if (!amdgpu_vm_debug && dma_resv_trylock(resv))
@@ -1310,9 +1343,9 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 
 		if (!clear)
 			dma_resv_unlock(resv);
-		spin_lock(&vm->invalidated_lock);
+		spin_lock(&vm->status_lock);
 	}
-	spin_unlock(&vm->invalidated_lock);
+	spin_unlock(&vm->status_lock);
 
 	return 0;
 }
@@ -1387,7 +1420,7 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
 
 	if (bo && bo->tbo.base.resv == vm->root.bo->tbo.base.resv &&
 	    !bo_va->base.moved) {
-		list_move(&bo_va->base.vm_status, &vm->moved);
+		amdgpu_vm_bo_moved(&bo_va->base);
 	}
 	trace_amdgpu_vm_bo_map(bo_va, mapping);
 }
@@ -1763,9 +1796,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
 		}
 	}
 
-	spin_lock(&vm->invalidated_lock);
+	spin_lock(&vm->status_lock);
 	list_del(&bo_va->base.vm_status);
-	spin_unlock(&vm->invalidated_lock);
+	spin_unlock(&vm->status_lock);
 
 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
 		list_del(&mapping->list);
@@ -2019,9 +2052,11 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 	INIT_LIST_HEAD(&vm->moved);
 	INIT_LIST_HEAD(&vm->idle);
 	INIT_LIST_HEAD(&vm->invalidated);
-	spin_lock_init(&vm->invalidated_lock);
+	spin_lock_init(&vm->status_lock);
 	INIT_LIST_HEAD(&vm->freed);
 	INIT_LIST_HEAD(&vm->done);
+	INIT_LIST_HEAD(&vm->pt_freed);
+	INIT_WORK(&vm->pt_free_work, amdgpu_vm_pt_free_work);
 
 	/* create scheduler entities for page table updates */
 	r = drm_sched_entity_init(&vm->immediate, DRM_SCHED_PRIORITY_NORMAL,
@@ -2223,6 +2258,8 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 
 	amdgpu_amdkfd_gpuvm_destroy_cb(adev, vm);
 
+	flush_work(&vm->pt_free_work);
+
 	root = amdgpu_bo_ref(vm->root.bo);
 	amdgpu_bo_reserve(root, true);
 	amdgpu_vm_set_pasid(adev, vm, 0);
@@ -2547,6 +2584,7 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
 	unsigned int total_done_objs = 0;
 	unsigned int id = 0;
 
+	spin_lock(&vm->status_lock);
 	seq_puts(m, "\tIdle BOs:\n");
 	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
 		if (!bo_va->base.bo)
@@ -2584,7 +2622,6 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
 	id = 0;
 
 	seq_puts(m, "\tInvalidated BOs:\n");
-	spin_lock(&vm->invalidated_lock);
 	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
 		if (!bo_va->base.bo)
 			continue;
@@ -2599,7 +2636,7 @@ void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
 			continue;
 		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
 	}
-	spin_unlock(&vm->invalidated_lock);
+	spin_unlock(&vm->status_lock);
 	total_done_objs = id;
 
 	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 9ecb7f663e19..83acb7bd80fe 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -254,6 +254,9 @@ struct amdgpu_vm {
 	bool			evicting;
 	unsigned int		saved_flags;
 
+	/* Lock to protect vm_bo add/del/move on all lists of vm */
+	spinlock_t		status_lock;
+
 	/* BOs who needs a validation */
 	struct list_head	evicted;
 
@@ -268,7 +271,6 @@ struct amdgpu_vm {
 
 	/* regular invalidated BOs, but not yet updated in the PT */
 	struct list_head	invalidated;
-	spinlock_t		invalidated_lock;
 
 	/* BO mappings freed, but not yet updated in the PT */
 	struct list_head	freed;
@@ -276,6 +278,10 @@ struct amdgpu_vm {
 	/* BOs which are invalidated, has been updated in the PTs */
 	struct list_head        done;
 
+	/* PT BOs scheduled to free and fill with zero if vm_resv is not hold */
+	struct list_head	pt_freed;
+	struct work_struct	pt_free_work;
+
 	/* contains the page directory */
 	struct amdgpu_vm_bo_base     root;
 	struct dma_fence	*last_update;
@@ -471,6 +477,7 @@ int amdgpu_vm_pde_update(struct amdgpu_vm_update_params *params,
 int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 			  uint64_t start, uint64_t end,
 			  uint64_t dst, uint64_t flags);
+void amdgpu_vm_pt_free_work(struct work_struct *work);
 
 #if defined(CONFIG_DEBUG_FS)
 void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index 88de9f0d4728..358b91243e37 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -637,10 +637,34 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
 	}
 	ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
 	entry->bo->vm_bo = NULL;
+
+	spin_lock(&entry->vm->status_lock);
 	list_del(&entry->vm_status);
+	spin_unlock(&entry->vm->status_lock);
 	amdgpu_bo_unref(&entry->bo);
 }
 
+void amdgpu_vm_pt_free_work(struct work_struct *work)
+{
+	struct amdgpu_vm_bo_base *entry, *next;
+	struct amdgpu_vm *vm;
+	LIST_HEAD(pt_freed);
+
+	vm = container_of(work, struct amdgpu_vm, pt_free_work);
+
+	spin_lock(&vm->status_lock);
+	list_splice_init(&vm->pt_freed, &pt_freed);
+	spin_unlock(&vm->status_lock);
+
+	/* flush_work in amdgpu_vm_fini ensure vm->root.bo is valid. */
+	amdgpu_bo_reserve(vm->root.bo, true);
+
+	list_for_each_entry_safe(entry, next, &pt_freed, vm_status)
+		amdgpu_vm_pt_free(entry);
+
+	amdgpu_bo_unreserve(vm->root.bo);
+}
+
 /**
  * amdgpu_vm_pt_free_dfs - free PD/PT levels
  *
@@ -652,11 +676,24 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
  */
 static void amdgpu_vm_pt_free_dfs(struct amdgpu_device *adev,
 				  struct amdgpu_vm *vm,
-				  struct amdgpu_vm_pt_cursor *start)
+				  struct amdgpu_vm_pt_cursor *start,
+				  bool unlocked)
 {
 	struct amdgpu_vm_pt_cursor cursor;
 	struct amdgpu_vm_bo_base *entry;
 
+	if (unlocked) {
+		spin_lock(&vm->status_lock);
+		for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
+			list_move(&entry->vm_status, &vm->pt_freed);
+
+		if (start)
+			list_move(&start->entry->vm_status, &vm->pt_freed);
+		spin_unlock(&vm->status_lock);
+		schedule_work(&vm->pt_free_work);
+		return;
+	}
+
 	for_each_amdgpu_vm_pt_dfs_safe(adev, vm, start, cursor, entry)
 		amdgpu_vm_pt_free(entry);
 
@@ -673,7 +710,7 @@ static void amdgpu_vm_pt_free_dfs(struct amdgpu_device *adev,
  */
 void amdgpu_vm_pt_free_root(struct amdgpu_device *adev, struct amdgpu_vm *vm)
 {
-	amdgpu_vm_pt_free_dfs(adev, vm, NULL);
+	amdgpu_vm_pt_free_dfs(adev, vm, NULL, false);
 }
 
 /**
@@ -966,7 +1003,8 @@ int amdgpu_vm_ptes_update(struct amdgpu_vm_update_params *params,
 				if (cursor.entry->bo) {
 					params->table_freed = true;
 					amdgpu_vm_pt_free_dfs(adev, params->vm,
-							      &cursor);
+							      &cursor,
+							      params->unlocked);
 				}
 				amdgpu_vm_pt_next(adev, &cursor);
 			}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
index 1fd3cbca20a2..2b0669c464f6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -112,7 +112,8 @@ static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,
 		swap(p->vm->last_unlocked, tmp);
 		dma_fence_put(tmp);
 	} else {
-		amdgpu_bo_fence(p->vm->root.bo, f, true);
+		dma_resv_add_fence(p->vm->root.bo->tbo.base.resv, f,
+				   DMA_RESV_USAGE_BOOKKEEP);
 	}
 
 	if (fence && !p->immediate)
@@ -211,12 +212,15 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p,
 	int r;
 
 	/* Wait for PD/PT moves to be completed */
-	dma_resv_for_each_fence(&cursor, bo->tbo.base.resv,
-				DMA_RESV_USAGE_KERNEL, fence) {
+	dma_resv_iter_begin(&cursor, bo->tbo.base.resv, DMA_RESV_USAGE_KERNEL);
+	dma_resv_for_each_fence_unlocked(&cursor, fence) {
 		r = amdgpu_sync_fence(&p->job->sync, fence);
-		if (r)
+		if (r) {
+			dma_resv_iter_end(&cursor);
 			return r;
+		}
 	}
+	dma_resv_iter_end(&cursor);
 
 	do {
 		ndw = p->num_dw_left;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
index 28ec5f8ac1c1..73a517bcf5c1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c
@@ -721,6 +721,72 @@ uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr)
 }
 
 /**
+ * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection
+ *
+ * @man: TTM memory type manager
+ * @res: The resource to test
+ * @place: The place to test against
+ * @size: Size of the new allocation
+ *
+ * Test each drm buddy block for intersection for eviction decision.
+ */
+static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man,
+				       struct ttm_resource *res,
+				       const struct ttm_place *place,
+				       size_t size)
+{
+	struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
+	struct drm_buddy_block *block;
+
+	/* Check each drm buddy block individually */
+	list_for_each_entry(block, &mgr->blocks, link) {
+		unsigned long fpfn =
+			amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
+		unsigned long lpfn = fpfn +
+			(amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);
+
+		if (place->fpfn < lpfn &&
+		    (!place->lpfn || place->lpfn > fpfn))
+			return true;
+	}
+
+	return false;
+}
+
+/**
+ * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility
+ *
+ * @man: TTM memory type manager
+ * @res: The resource to test
+ * @place: The place to test against
+ * @size: Size of the new allocation
+ *
+ * Test each drm buddy block for placement compatibility.
+ */
+static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man,
+				       struct ttm_resource *res,
+				       const struct ttm_place *place,
+				       size_t size)
+{
+	struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
+	struct drm_buddy_block *block;
+
+	/* Check each drm buddy block individually */
+	list_for_each_entry(block, &mgr->blocks, link) {
+		unsigned long fpfn =
+			amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
+		unsigned long lpfn = fpfn +
+			(amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);
+
+		if (fpfn < place->fpfn ||
+		    (place->lpfn && lpfn > place->lpfn))
+			return false;
+	}
+
+	return true;
+}
+
+/**
  * amdgpu_vram_mgr_debug - dump VRAM table
  *
  * @man: TTM memory type manager
@@ -753,6 +819,8 @@ static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,
 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = {
 	.alloc	= amdgpu_vram_mgr_new,
 	.free	= amdgpu_vram_mgr_del,
+	.intersects = amdgpu_vram_mgr_intersects,
+	.compatible = amdgpu_vram_mgr_compatible,
 	.debug	= amdgpu_vram_mgr_debug
 };
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
index f2aebbf3fbe3..47159e9a0884 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.c
@@ -392,12 +392,20 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
 	}
 
 	/**
+	 * Only init hive->reset_domain for none SRIOV configuration. For SRIOV,
+	 * Host driver decide how to reset the GPU either through FLR or chain reset.
+	 * Guest side will get individual notifications from the host for the FLR
+	 * if necessary.
+	 */
+	if (!amdgpu_sriov_vf(adev)) {
+	/**
 	 * Avoid recreating reset domain when hive is reconstructed for the case
-	 * of reset the devices in the XGMI hive during probe for SRIOV
+	 * of reset the devices in the XGMI hive during probe for passthrough GPU
 	 * See https://www.spinics.net/lists/amd-gfx/msg58836.html
 	 */
-	if (adev->reset_domain->type != XGMI_HIVE) {
-		hive->reset_domain = amdgpu_reset_create_reset_domain(XGMI_HIVE, "amdgpu-reset-hive");
+		if (adev->reset_domain->type != XGMI_HIVE) {
+			hive->reset_domain =
+				amdgpu_reset_create_reset_domain(XGMI_HIVE, "amdgpu-reset-hive");
 			if (!hive->reset_domain) {
 				dev_err(adev->dev, "XGMI: failed initializing reset domain for xgmi hive\n");
 				ret = -ENOMEM;
@@ -406,9 +414,10 @@ struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev)
 				hive = NULL;
 				goto pro_end;
 			}
-	} else {
-		amdgpu_reset_get_reset_domain(adev->reset_domain);
-		hive->reset_domain = adev->reset_domain;
+		} else {
+			amdgpu_reset_get_reset_domain(adev->reset_domain);
+			hive->reset_domain = adev->reset_domain;
+		}
 	}
 
 	hive->hive_id = adev->gmc.xgmi.hive_id;
@@ -504,6 +513,9 @@ int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_dev
 {
 	int ret;
 
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
 	/* Each psp need to set the latest topology */
 	ret = psp_xgmi_set_topology_info(&adev->psp,
 					 atomic_read(&hive->number_devices),
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
index 552e6fb55aa8..30dcc1681b4e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xgmi.h
@@ -43,6 +43,7 @@ struct amdgpu_hive_info {
 	} pstate;
 
 	struct amdgpu_reset_domain *reset_domain;
+	uint32_t device_remove_count;
 };
 
 struct amdgpu_pcs_ras_field {
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
index fa7421afb9a6..6be9ac2b9c5b 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
@@ -26,6 +26,8 @@
 
 #include <linux/pci.h>
 
+#include <acpi/video.h>
+
 #include <drm/drm_crtc_helper.h>
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
@@ -182,7 +184,12 @@ void amdgpu_atombios_encoder_init_backlight(struct amdgpu_encoder *amdgpu_encode
 		return;
 
 	if (!(adev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU))
-		return;
+		goto register_acpi_backlight;
+
+	if (!acpi_video_backlight_use_native()) {
+		drm_info(dev, "Skipping amdgpu atom DIG backlight registration\n");
+		goto register_acpi_backlight;
+	}
 
 	pdata = kmalloc(sizeof(struct amdgpu_backlight_privdata), GFP_KERNEL);
 	if (!pdata) {
@@ -218,6 +225,11 @@ void amdgpu_atombios_encoder_init_backlight(struct amdgpu_encoder *amdgpu_encode
 error:
 	kfree(pdata);
 	return;
+
+register_acpi_backlight:
+	/* Try registering an ACPI video backlight device instead. */
+	acpi_video_register_backlight();
+	return;
 }
 
 void
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index a3cd5c1e8529..af94ac580d3e 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -3943,56 +3943,6 @@ static void gfx_v10_0_check_fw_write_wait(struct amdgpu_device *adev)
 		DRM_WARN_ONCE("CP firmware version too old, please update!");
 }
 
-
-static void gfx_v10_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
-{
-	const struct rlc_firmware_header_v2_1 *rlc_hdr;
-
-	rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
-	adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
-	adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
-	adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
-	adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
-	adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
-	adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
-	adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
-	adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
-	adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
-	adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
-	adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
-	adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
-	adev->gfx.rlc.reg_list_format_direct_reg_list_length =
-			le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
-}
-
-static void gfx_v10_0_init_rlc_iram_dram_microcode(struct amdgpu_device *adev)
-{
-	const struct rlc_firmware_header_v2_2 *rlc_hdr;
-
-	rlc_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
-	adev->gfx.rlc.rlc_iram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_iram_ucode_size_bytes);
-	adev->gfx.rlc.rlc_iram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_iram_ucode_offset_bytes);
-	adev->gfx.rlc.rlc_dram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_dram_ucode_size_bytes);
-	adev->gfx.rlc.rlc_dram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_dram_ucode_offset_bytes);
-}
-
-static void gfx_v10_0_init_tap_delays_microcode(struct amdgpu_device *adev)
-{
-	const struct rlc_firmware_header_v2_4 *rlc_hdr;
-
-	rlc_hdr = (const struct rlc_firmware_header_v2_4 *)adev->gfx.rlc_fw->data;
-	adev->gfx.rlc.global_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->global_tap_delays_ucode_size_bytes);
-	adev->gfx.rlc.global_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->global_tap_delays_ucode_offset_bytes);
-	adev->gfx.rlc.se0_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se0_tap_delays_ucode_size_bytes);
-	adev->gfx.rlc.se0_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se0_tap_delays_ucode_offset_bytes);
-	adev->gfx.rlc.se1_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se1_tap_delays_ucode_size_bytes);
-	adev->gfx.rlc.se1_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se1_tap_delays_ucode_offset_bytes);
-	adev->gfx.rlc.se2_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se2_tap_delays_ucode_size_bytes);
-	adev->gfx.rlc.se2_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se2_tap_delays_ucode_offset_bytes);
-	adev->gfx.rlc.se3_tap_delays_ucode_size_bytes = le32_to_cpu(rlc_hdr->se3_tap_delays_ucode_size_bytes);
-	adev->gfx.rlc.se3_tap_delays_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->se3_tap_delays_ucode_offset_bytes);
-}
-
 static bool gfx_v10_0_navi10_gfxoff_should_enable(struct amdgpu_device *adev)
 {
 	bool ret = false;
@@ -4028,12 +3978,7 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev)
 	char fw_name[40];
 	char *wks = "";
 	int err;
-	struct amdgpu_firmware_info *info = NULL;
-	const struct common_firmware_header *header = NULL;
-	const struct gfx_firmware_header_v1_0 *cp_hdr;
 	const struct rlc_firmware_header_v2_0 *rlc_hdr;
-	unsigned int *tmp = NULL;
-	unsigned int i = 0;
 	uint16_t version_major;
 	uint16_t version_minor;
 
@@ -4091,9 +4036,7 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev)
 	err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
 	if (err)
 		goto out;
-	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
-	adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-	adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me%s.bin", chip_name, wks);
 	err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
@@ -4102,9 +4045,7 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev)
 	err = amdgpu_ucode_validate(adev->gfx.me_fw);
 	if (err)
 		goto out;
-	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
-	adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-	adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce%s.bin", chip_name, wks);
 	err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
@@ -4113,69 +4054,27 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev)
 	err = amdgpu_ucode_validate(adev->gfx.ce_fw);
 	if (err)
 		goto out;
-	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
-	adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-	adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE);
 
 	if (!amdgpu_sriov_vf(adev)) {
 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
 		err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
 		if (err)
 			goto out;
+		/* don't check this.  There are apparently firmwares in the wild with
+		 * incorrect size in the header
+		 */
 		err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
+		if (err)
+			dev_dbg(adev->dev,
+				"gfx10: amdgpu_ucode_validate() failed \"%s\"\n",
+				fw_name);
 		rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 		version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 		version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
-
-		adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
-		adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
-		adev->gfx.rlc.save_and_restore_offset =
-			le32_to_cpu(rlc_hdr->save_and_restore_offset);
-		adev->gfx.rlc.clear_state_descriptor_offset =
-			le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
-		adev->gfx.rlc.avail_scratch_ram_locations =
-			le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
-		adev->gfx.rlc.reg_restore_list_size =
-			le32_to_cpu(rlc_hdr->reg_restore_list_size);
-		adev->gfx.rlc.reg_list_format_start =
-			le32_to_cpu(rlc_hdr->reg_list_format_start);
-		adev->gfx.rlc.reg_list_format_separate_start =
-			le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
-		adev->gfx.rlc.starting_offsets_start =
-			le32_to_cpu(rlc_hdr->starting_offsets_start);
-		adev->gfx.rlc.reg_list_format_size_bytes =
-			le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
-		adev->gfx.rlc.reg_list_size_bytes =
-			le32_to_cpu(rlc_hdr->reg_list_size_bytes);
-		adev->gfx.rlc.register_list_format =
-			kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
-					adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
-		if (!adev->gfx.rlc.register_list_format) {
-			err = -ENOMEM;
+		err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
+		if (err)
 			goto out;
-		}
-
-		tmp = (unsigned int *)((uintptr_t)rlc_hdr +
-							   le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
-		for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++)
-			adev->gfx.rlc.register_list_format[i] =	le32_to_cpu(tmp[i]);
-
-		adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
-
-		tmp = (unsigned int *)((uintptr_t)rlc_hdr +
-							   le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
-		for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
-			adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
-
-		if (version_major == 2) {
-			if (version_minor >= 1)
-				gfx_v10_0_init_rlc_ext_microcode(adev);
-			if (version_minor >= 2)
-				gfx_v10_0_init_rlc_iram_dram_microcode(adev);
-			if (version_minor == 4) {
-				gfx_v10_0_init_tap_delays_microcode(adev);
-			}
-		}
 	}
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec%s.bin", chip_name, wks);
@@ -4185,9 +4084,8 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev)
 	err = amdgpu_ucode_validate(adev->gfx.mec_fw);
 	if (err)
 		goto out;
-	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
-	adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-	adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2%s.bin", chip_name, wks);
 	err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
@@ -4195,164 +4093,18 @@ static int gfx_v10_0_init_microcode(struct amdgpu_device *adev)
 		err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
 		if (err)
 			goto out;
-		cp_hdr = (const struct gfx_firmware_header_v1_0 *)
-		adev->gfx.mec2_fw->data;
-		adev->gfx.mec2_fw_version =
-		le32_to_cpu(cp_hdr->header.ucode_version);
-		adev->gfx.mec2_feature_version =
-		le32_to_cpu(cp_hdr->ucode_feature_version);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT);
 	} else {
 		err = 0;
 		adev->gfx.mec2_fw = NULL;
 	}
 
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
-		info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
-		info->fw = adev->gfx.pfp_fw;
-		header = (const struct common_firmware_header *)info->fw->data;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
-		info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
-		info->fw = adev->gfx.me_fw;
-		header = (const struct common_firmware_header *)info->fw->data;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
-		info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
-		info->fw = adev->gfx.ce_fw;
-		header = (const struct common_firmware_header *)info->fw->data;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
-		info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
-		info->fw = adev->gfx.rlc_fw;
-		if (info->fw) {
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-		}
-		if (adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
-		    adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
-		    adev->gfx.rlc.save_restore_list_srm_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
-
-			if (adev->gfx.rlc.rlc_iram_ucode_size_bytes &&
-			    adev->gfx.rlc.rlc_dram_ucode_size_bytes) {
-				info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_IRAM];
-				info->ucode_id = AMDGPU_UCODE_ID_RLC_IRAM;
-				info->fw = adev->gfx.rlc_fw;
-				adev->firmware.fw_size +=
-					ALIGN(adev->gfx.rlc.rlc_iram_ucode_size_bytes, PAGE_SIZE);
-
-				info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_DRAM];
-				info->ucode_id = AMDGPU_UCODE_ID_RLC_DRAM;
-				info->fw = adev->gfx.rlc_fw;
-				adev->firmware.fw_size +=
-					ALIGN(adev->gfx.rlc.rlc_dram_ucode_size_bytes, PAGE_SIZE);
-			}
-
-		}
-
-		if (adev->gfx.rlc.global_tap_delays_ucode_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS];
-			info->ucode_id = AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.global_tap_delays_ucode_size_bytes, PAGE_SIZE);
-		}
-
-		if (adev->gfx.rlc.se0_tap_delays_ucode_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE0_TAP_DELAYS];
-			info->ucode_id = AMDGPU_UCODE_ID_SE0_TAP_DELAYS;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.se0_tap_delays_ucode_size_bytes, PAGE_SIZE);
-		}
-
-		if (adev->gfx.rlc.se1_tap_delays_ucode_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE1_TAP_DELAYS];
-			info->ucode_id = AMDGPU_UCODE_ID_SE1_TAP_DELAYS;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.se1_tap_delays_ucode_size_bytes, PAGE_SIZE);
-		}
-
-		if (adev->gfx.rlc.se2_tap_delays_ucode_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE2_TAP_DELAYS];
-			info->ucode_id = AMDGPU_UCODE_ID_SE2_TAP_DELAYS;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.se2_tap_delays_ucode_size_bytes, PAGE_SIZE);
-		}
-
-		if (adev->gfx.rlc.se3_tap_delays_ucode_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SE3_TAP_DELAYS];
-			info->ucode_id = AMDGPU_UCODE_ID_SE3_TAP_DELAYS;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.se3_tap_delays_ucode_size_bytes, PAGE_SIZE);
-		}
-
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
-		info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
-		info->fw = adev->gfx.mec_fw;
-		header = (const struct common_firmware_header *)info->fw->data;
-		cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(header->ucode_size_bytes) -
-			      le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
-
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
-		info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
-		info->fw = adev->gfx.mec_fw;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
-
-		if (adev->gfx.mec2_fw) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
-			info->fw = adev->gfx.mec2_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(header->ucode_size_bytes) -
-				      le32_to_cpu(cp_hdr->jt_size) * 4,
-				      PAGE_SIZE);
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
-			info->fw = adev->gfx.mec2_fw;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
-				      PAGE_SIZE);
-		}
-	}
-
 	gfx_v10_0_check_fw_write_wait(adev);
 out:
 	if (err) {
 		dev_err(adev->dev,
-			"gfx10: Failed to load firmware \"%s\"\n",
+			"gfx10: Failed to init firmware \"%s\"\n",
 			fw_name);
 		release_firmware(adev->gfx.pfp_fw);
 		adev->gfx.pfp_fw = NULL;
@@ -5981,6 +5733,9 @@ static int gfx_v10_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
 		WREG32_SOC15(GC, 0, mmCP_ME_CNTL, tmp);
 	}
 
+	if (adev->job_hang && !enable)
+		return 0;
+
 	for (i = 0; i < adev->usec_timeout; i++) {
 		if (RREG32_SOC15(GC, 0, mmCP_STAT) == 0)
 			break;
@@ -7579,8 +7334,10 @@ static int gfx_v10_0_kiq_disable_kgq(struct amdgpu_device *adev)
 	for (i = 0; i < adev->gfx.num_gfx_rings; i++)
 		kiq->pmf->kiq_unmap_queues(kiq_ring, &adev->gfx.gfx_ring[i],
 					   PREEMPT_QUEUES, 0, 0);
-
-	return amdgpu_ring_test_helper(kiq_ring);
+	if (!adev->job_hang)
+		return amdgpu_ring_test_helper(kiq_ring);
+	else
+		return 0;
 }
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index f6b1bb40e503..251109723ab6 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -73,21 +73,10 @@ MODULE_FIRMWARE("amdgpu/gc_11_0_2_pfp.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_2_me.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mec.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_2_rlc.bin");
-
-static const struct soc15_reg_golden golden_settings_gc_11_0[] =
-{
-	/* Pending on emulation bring up */
-};
-
-static const struct soc15_reg_golden golden_settings_gc_11_0_0[] =
-{
-	/* Pending on emulation bring up */
-};
-
-static const struct soc15_reg_golden golden_settings_gc_rlc_spm_11_0[] =
-{
-	/* Pending on emulation bring up */
-};
+MODULE_FIRMWARE("amdgpu/gc_11_0_3_pfp.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_3_me.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_3_mec.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_3_rlc.bin");
 
 static const struct soc15_reg_golden golden_settings_gc_11_0_1[] =
 {
@@ -269,42 +258,17 @@ static void gfx_v11_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
 	adev->gfx.kiq.pmf = &gfx_v11_0_kiq_pm4_funcs;
 }
 
-static void gfx_v11_0_init_spm_golden_registers(struct amdgpu_device *adev)
-{
-	switch (adev->ip_versions[GC_HWIP][0]) {
-	case IP_VERSION(11, 0, 0):
-		soc15_program_register_sequence(adev,
-						golden_settings_gc_rlc_spm_11_0,
-						(const u32)ARRAY_SIZE(golden_settings_gc_rlc_spm_11_0));
-		break;
-	default:
-		break;
-	}
-}
-
 static void gfx_v11_0_init_golden_registers(struct amdgpu_device *adev)
 {
 	switch (adev->ip_versions[GC_HWIP][0]) {
-	case IP_VERSION(11, 0, 0):
-		soc15_program_register_sequence(adev,
-						golden_settings_gc_11_0,
-						(const u32)ARRAY_SIZE(golden_settings_gc_11_0));
-		soc15_program_register_sequence(adev,
-						golden_settings_gc_11_0_0,
-						(const u32)ARRAY_SIZE(golden_settings_gc_11_0_0));
-		break;
 	case IP_VERSION(11, 0, 1):
 		soc15_program_register_sequence(adev,
-						golden_settings_gc_11_0,
-						(const u32)ARRAY_SIZE(golden_settings_gc_11_0));
-		soc15_program_register_sequence(adev,
 						golden_settings_gc_11_0_1,
 						(const u32)ARRAY_SIZE(golden_settings_gc_11_0_1));
 		break;
 	default:
 		break;
 	}
-	gfx_v11_0_init_spm_golden_registers(adev);
 }
 
 static void gfx_v11_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
@@ -474,61 +438,12 @@ static void gfx_v11_0_free_microcode(struct amdgpu_device *adev)
 	kfree(adev->gfx.rlc.register_list_format);
 }
 
-static void gfx_v11_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
-{
-	const struct rlc_firmware_header_v2_1 *rlc_hdr;
-
-	rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
-	adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
-	adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
-	adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
-	adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
-	adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
-	adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
-	adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
-	adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
-	adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
-	adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
-	adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
-	adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
-	adev->gfx.rlc.reg_list_format_direct_reg_list_length =
-			le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
-}
-
-static void gfx_v11_0_init_rlc_iram_dram_microcode(struct amdgpu_device *adev)
-{
-	const struct rlc_firmware_header_v2_2 *rlc_hdr;
-
-	rlc_hdr = (const struct rlc_firmware_header_v2_2 *)adev->gfx.rlc_fw->data;
-	adev->gfx.rlc.rlc_iram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_iram_ucode_size_bytes);
-	adev->gfx.rlc.rlc_iram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_iram_ucode_offset_bytes);
-	adev->gfx.rlc.rlc_dram_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlc_dram_ucode_size_bytes);
-	adev->gfx.rlc.rlc_dram_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlc_dram_ucode_offset_bytes);
-}
-
-static void gfx_v11_0_init_rlcp_rlcv_microcode(struct amdgpu_device *adev)
-{
-	const struct rlc_firmware_header_v2_3 *rlc_hdr;
-
-	rlc_hdr = (const struct rlc_firmware_header_v2_3 *)adev->gfx.rlc_fw->data;
-	adev->gfx.rlc.rlcp_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlcp_ucode_size_bytes);
-	adev->gfx.rlc.rlcp_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlcp_ucode_offset_bytes);
-	adev->gfx.rlc.rlcv_ucode_size_bytes = le32_to_cpu(rlc_hdr->rlcv_ucode_size_bytes);
-	adev->gfx.rlc.rlcv_ucode = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->rlcv_ucode_offset_bytes);
-}
-
 static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 {
 	char fw_name[40];
 	char ucode_prefix[30];
 	int err;
-	struct amdgpu_firmware_info *info = NULL;
-	const struct common_firmware_header *header = NULL;
-	const struct gfx_firmware_header_v1_0 *cp_hdr;
-	const struct gfx_firmware_header_v2_0 *cp_hdr_v2_0;
 	const struct rlc_firmware_header_v2_0 *rlc_hdr;
-	unsigned int *tmp = NULL;
-	unsigned int i = 0;
 	uint16_t version_major;
 	uint16_t version_minor;
 
@@ -549,14 +464,11 @@ static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 				adev->gfx.pfp_fw->data, 2, 0);
 	if (adev->gfx.rs64_enable) {
 		dev_info(adev->dev, "CP RS64 enable\n");
-		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.pfp_fw->data;
-		adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
-		adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
-		
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK);
 	} else {
-		cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
-		adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-		adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
 	}
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", ucode_prefix);
@@ -567,14 +479,11 @@ static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 	if (err)
 		goto out;
 	if (adev->gfx.rs64_enable) {
-		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.me_fw->data;
-		adev->gfx.me_fw_version = le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
-		adev->gfx.me_feature_version = le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
-		
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK);
 	} else {
-		cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
-		adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-		adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
 	}
 
 	if (!amdgpu_sriov_vf(adev)) {
@@ -583,58 +492,14 @@ static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 		if (err)
 			goto out;
 		err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
+		if (err)
+			goto out;
 		rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 		version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 		version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
-
-		adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
-		adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
-		adev->gfx.rlc.save_and_restore_offset =
-			le32_to_cpu(rlc_hdr->save_and_restore_offset);
-		adev->gfx.rlc.clear_state_descriptor_offset =
-			le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
-		adev->gfx.rlc.avail_scratch_ram_locations =
-			le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
-		adev->gfx.rlc.reg_restore_list_size =
-			le32_to_cpu(rlc_hdr->reg_restore_list_size);
-		adev->gfx.rlc.reg_list_format_start =
-			le32_to_cpu(rlc_hdr->reg_list_format_start);
-		adev->gfx.rlc.reg_list_format_separate_start =
-			le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
-		adev->gfx.rlc.starting_offsets_start =
-			le32_to_cpu(rlc_hdr->starting_offsets_start);
-		adev->gfx.rlc.reg_list_format_size_bytes =
-			le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
-		adev->gfx.rlc.reg_list_size_bytes =
-			le32_to_cpu(rlc_hdr->reg_list_size_bytes);
-		adev->gfx.rlc.register_list_format =
-			kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
-					adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
-		if (!adev->gfx.rlc.register_list_format) {
-			err = -ENOMEM;
+		err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
+		if (err)
 			goto out;
-		}
-
-		tmp = (unsigned int *)((uintptr_t)rlc_hdr +
-							   le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
-		for (i = 0 ; i < (rlc_hdr->reg_list_format_size_bytes >> 2); i++)
-			adev->gfx.rlc.register_list_format[i] =	le32_to_cpu(tmp[i]);
-
-		adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
-
-		tmp = (unsigned int *)((uintptr_t)rlc_hdr +
-							   le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
-		for (i = 0 ; i < (rlc_hdr->reg_list_size_bytes >> 2); i++)
-			adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
-
-		if (version_major == 2) {
-			if (version_minor >= 1)
-				gfx_v11_0_init_rlc_ext_microcode(adev);
-			if (version_minor >= 2)
-				gfx_v11_0_init_rlc_iram_dram_microcode(adev);
-			if (version_minor == 3)
-				gfx_v11_0_init_rlcp_rlcv_microcode(adev);
-		}
 	}
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", ucode_prefix);
@@ -645,190 +510,23 @@ static int gfx_v11_0_init_microcode(struct amdgpu_device *adev)
 	if (err)
 		goto out;
 	if (adev->gfx.rs64_enable) {
-		cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
-		adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr_v2_0->header.ucode_version);
-		adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr_v2_0->ucode_feature_version);
-		
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK);
 	} else {
-		cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
-		adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-		adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
+		amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
 	}
 
 	/* only one MEC for gfx 11.0.0. */
 	adev->gfx.mec2_fw = NULL;
 
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		if (adev->gfx.rs64_enable) {
-			cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.pfp_fw->data;
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_PFP];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_PFP;
-			info->fw = adev->gfx.pfp_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes), PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK;
-			info->fw = adev->gfx.pfp_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK;
-			info->fw = adev->gfx.pfp_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
-
-			cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.me_fw->data;
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_ME];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_ME;
-			info->fw = adev->gfx.me_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes), PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK;
-			info->fw = adev->gfx.me_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK;
-			info->fw = adev->gfx.me_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
-
-			cp_hdr_v2_0 = (const struct gfx_firmware_header_v2_0 *)adev->gfx.mec_fw->data;
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC;
-			info->fw = adev->gfx.mec_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->ucode_size_bytes), PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK;
-			info->fw = adev->gfx.mec_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK;
-			info->fw = adev->gfx.mec_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK;
-			info->fw = adev->gfx.mec_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK;
-			info->fw = adev->gfx.mec_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr_v2_0->data_size_bytes), PAGE_SIZE);
-		} else {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
-			info->fw = adev->gfx.pfp_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
-			info->fw = adev->gfx.me_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
-			info->fw = adev->gfx.mec_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(header->ucode_size_bytes) -
-				      le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
-			info->fw = adev->gfx.mec_fw;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
-		}
-
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
-		info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
-		info->fw = adev->gfx.rlc_fw;
-		if (info->fw) {
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-		}
-		if (adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
-		    adev->gfx.rlc.save_restore_list_srm_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
-		}
-
-		if (adev->gfx.rlc.rlc_iram_ucode_size_bytes &&
-		    adev->gfx.rlc.rlc_dram_ucode_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_IRAM];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_IRAM;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.rlc_iram_ucode_size_bytes, PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_DRAM];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_DRAM;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.rlc_dram_ucode_size_bytes, PAGE_SIZE);
-		}
-
-		if (adev->gfx.rlc.rlcp_ucode_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_P];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_P;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.rlcp_ucode_size_bytes, PAGE_SIZE);
-		}
-
-		if (adev->gfx.rlc.rlcv_ucode_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_V];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_V;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.rlcv_ucode_size_bytes, PAGE_SIZE);
-		}
-	}
-
 out:
 	if (err) {
 		dev_err(adev->dev,
-			"gfx11: Failed to load firmware \"%s\"\n",
+			"gfx11: Failed to init firmware \"%s\"\n",
 			fw_name);
 		release_firmware(adev->gfx.pfp_fw);
 		adev->gfx.pfp_fw = NULL;
@@ -1140,7 +838,6 @@ static const struct amdgpu_gfx_funcs gfx_v11_0_gfx_funcs = {
 	.read_wave_sgprs = &gfx_v11_0_read_wave_sgprs,
 	.read_wave_vgprs = &gfx_v11_0_read_wave_vgprs,
 	.select_me_pipe_q = &gfx_v11_0_select_me_pipe_q,
-	.init_spm_golden = &gfx_v11_0_init_spm_golden_registers,
 	.update_perfmon_mgcg = &gfx_v11_0_update_perf_clk,
 };
 
@@ -1151,6 +848,7 @@ static int gfx_v11_0_gpu_early_init(struct amdgpu_device *adev)
 	switch (adev->ip_versions[GC_HWIP][0]) {
 	case IP_VERSION(11, 0, 0):
 	case IP_VERSION(11, 0, 2):
+	case IP_VERSION(11, 0, 3):
 		adev->gfx.config.max_hw_contexts = 8;
 		adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
 		adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
@@ -1586,6 +1284,7 @@ static int gfx_v11_0_sw_init(void *handle)
 	case IP_VERSION(11, 0, 0):
 	case IP_VERSION(11, 0, 1):
 	case IP_VERSION(11, 0, 2):
+	case IP_VERSION(11, 0, 3):
 		adev->gfx.me.num_me = 1;
 		adev->gfx.me.num_pipe_per_me = 1;
 		adev->gfx.me.num_queue_per_pipe = 1;
@@ -2760,6 +2459,21 @@ static void gfx_v11_0_config_gfx_rs64(struct amdgpu_device *adev)
 					mec_hdr->ucode_start_addr_hi >> 2);
 	}
 	soc21_grbm_select(adev, 0, 0, 0, 0);
+
+	/* reset mec pipe */
+	tmp = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 1);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 1);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 1);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 1);
+	WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
+
+	/* clear mec pipe reset */
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE0_RESET, 0);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE1_RESET, 0);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE2_RESET, 0);
+	tmp = REG_SET_FIELD(tmp, CP_MEC_RS64_CNTL, MEC_PIPE3_RESET, 0);
+	WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, tmp);
 }
 
 static int gfx_v11_0_wait_for_rlc_autoload_complete(struct amdgpu_device *adev)
@@ -5260,6 +4974,8 @@ static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
 {
 	u32 reg, data;
 
+	amdgpu_gfx_off_ctrl(adev, false);
+
 	reg = SOC15_REG_OFFSET(GC, 0, regRLC_SPM_MC_CNTL);
 	if (amdgpu_sriov_is_pp_one_vf(adev))
 		data = RREG32_NO_KIQ(reg);
@@ -5273,6 +4989,8 @@ static void gfx_v11_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
 		WREG32_SOC15_NO_KIQ(GC, 0, regRLC_SPM_MC_CNTL, data);
 	else
 		WREG32_SOC15(GC, 0, regRLC_SPM_MC_CNTL, data);
+
+	amdgpu_gfx_off_ctrl(adev, true);
 }
 
 static const struct amdgpu_rlc_funcs gfx_v11_0_rlc_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index fc9c1043244c..0320be4a5fc6 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -126,6 +126,8 @@ MODULE_FIRMWARE("amdgpu/green_sardine_rlc.bin");
 MODULE_FIRMWARE("amdgpu/aldebaran_mec.bin");
 MODULE_FIRMWARE("amdgpu/aldebaran_mec2.bin");
 MODULE_FIRMWARE("amdgpu/aldebaran_rlc.bin");
+MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec.bin");
+MODULE_FIRMWARE("amdgpu/aldebaran_sjt_mec2.bin");
 
 #define mmTCP_CHAN_STEER_0_ARCT								0x0b03
 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX							0
@@ -1089,27 +1091,6 @@ static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
 	kfree(adev->gfx.rlc.register_list_format);
 }
 
-static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
-{
-	const struct rlc_firmware_header_v2_1 *rlc_hdr;
-
-	rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
-	adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
-	adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
-	adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
-	adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
-	adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
-	adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
-	adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
-	adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
-	adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
-	adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
-	adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
-	adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
-	adev->gfx.rlc.reg_list_format_direct_reg_list_length =
-			le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
-}
-
 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
 {
 	adev->gfx.me_fw_write_wait = false;
@@ -1271,9 +1252,6 @@ static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
 {
 	char fw_name[30];
 	int err;
-	struct amdgpu_firmware_info *info = NULL;
-	const struct common_firmware_header *header = NULL;
-	const struct gfx_firmware_header_v1_0 *cp_hdr;
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
 	err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
@@ -1282,9 +1260,7 @@ static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
 	err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
 	if (err)
 		goto out;
-	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
-	adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-	adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_PFP);
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
 	err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
@@ -1293,9 +1269,7 @@ static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
 	err = amdgpu_ucode_validate(adev->gfx.me_fw);
 	if (err)
 		goto out;
-	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
-	adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-	adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_ME);
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
 	err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
@@ -1304,37 +1278,12 @@ static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
 	err = amdgpu_ucode_validate(adev->gfx.ce_fw);
 	if (err)
 		goto out;
-	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
-	adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-	adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
-
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
-		info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
-		info->fw = adev->gfx.pfp_fw;
-		header = (const struct common_firmware_header *)info->fw->data;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
-		info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
-		info->fw = adev->gfx.me_fw;
-		header = (const struct common_firmware_header *)info->fw->data;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
-		info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
-		info->fw = adev->gfx.ce_fw;
-		header = (const struct common_firmware_header *)info->fw->data;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-	}
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_CE);
 
 out:
 	if (err) {
 		dev_err(adev->dev,
-			"gfx9: Failed to load firmware \"%s\"\n",
+			"gfx9: Failed to init firmware \"%s\"\n",
 			fw_name);
 		release_firmware(adev->gfx.pfp_fw);
 		adev->gfx.pfp_fw = NULL;
@@ -1351,11 +1300,7 @@ static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
 {
 	char fw_name[30];
 	int err;
-	struct amdgpu_firmware_info *info = NULL;
-	const struct common_firmware_header *header = NULL;
 	const struct rlc_firmware_header_v2_0 *rlc_hdr;
-	unsigned int *tmp = NULL;
-	unsigned int i = 0;
 	uint16_t version_major;
 	uint16_t version_minor;
 	uint32_t smu_version;
@@ -1384,92 +1329,17 @@ static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
 	if (err)
 		goto out;
 	err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
+	if (err)
+		goto out;
 	rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
 
 	version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
 	version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
-	if (version_major == 2 && version_minor == 1)
-		adev->gfx.rlc.is_rlc_v2_1 = true;
-
-	adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
-	adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
-	adev->gfx.rlc.save_and_restore_offset =
-			le32_to_cpu(rlc_hdr->save_and_restore_offset);
-	adev->gfx.rlc.clear_state_descriptor_offset =
-			le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
-	adev->gfx.rlc.avail_scratch_ram_locations =
-			le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
-	adev->gfx.rlc.reg_restore_list_size =
-			le32_to_cpu(rlc_hdr->reg_restore_list_size);
-	adev->gfx.rlc.reg_list_format_start =
-			le32_to_cpu(rlc_hdr->reg_list_format_start);
-	adev->gfx.rlc.reg_list_format_separate_start =
-			le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
-	adev->gfx.rlc.starting_offsets_start =
-			le32_to_cpu(rlc_hdr->starting_offsets_start);
-	adev->gfx.rlc.reg_list_format_size_bytes =
-			le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
-	adev->gfx.rlc.reg_list_size_bytes =
-			le32_to_cpu(rlc_hdr->reg_list_size_bytes);
-	adev->gfx.rlc.register_list_format =
-			kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
-				adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
-	if (!adev->gfx.rlc.register_list_format) {
-		err = -ENOMEM;
-		goto out;
-	}
-
-	tmp = (unsigned int *)((uintptr_t)rlc_hdr +
-			le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
-	for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
-		adev->gfx.rlc.register_list_format[i] =	le32_to_cpu(tmp[i]);
-
-	adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
-
-	tmp = (unsigned int *)((uintptr_t)rlc_hdr +
-			le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
-	for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
-		adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
-
-	if (adev->gfx.rlc.is_rlc_v2_1)
-		gfx_v9_0_init_rlc_ext_microcode(adev);
-
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
-		info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
-		info->fw = adev->gfx.rlc_fw;
-		header = (const struct common_firmware_header *)info->fw->data;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-
-		if (adev->gfx.rlc.is_rlc_v2_1 &&
-		    adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
-		    adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
-		    adev->gfx.rlc.save_restore_list_srm_size_bytes) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
-
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
-			info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
-			info->fw = adev->gfx.rlc_fw;
-			adev->firmware.fw_size +=
-				ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
-		}
-	}
-
+	err = amdgpu_gfx_rlc_init_microcode(adev, version_major, version_minor);
 out:
 	if (err) {
 		dev_err(adev->dev,
-			"gfx9: Failed to load firmware \"%s\"\n",
+			"gfx9: Failed to init firmware \"%s\"\n",
 			fw_name);
 		release_firmware(adev->gfx.rlc_fw);
 		adev->gfx.rlc_fw = NULL;
@@ -1492,35 +1362,34 @@ static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
 {
 	char fw_name[30];
 	int err;
-	struct amdgpu_firmware_info *info = NULL;
-	const struct common_firmware_header *header = NULL;
-	const struct gfx_firmware_header_v1_0 *cp_hdr;
 
-	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
+	if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
+		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sjt_mec.bin", chip_name);
+	else
+		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
+
 	err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
 	if (err)
 		goto out;
 	err = amdgpu_ucode_validate(adev->gfx.mec_fw);
 	if (err)
 		goto out;
-	cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
-	adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
-	adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
-
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1);
+	amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC1_JT);
 
 	if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
-		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
+		if (amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_ALDEBARAN))
+			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sjt_mec2.bin", chip_name);
+		else
+			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
+
 		err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
 		if (!err) {
 			err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
 			if (err)
 				goto out;
-			cp_hdr = (const struct gfx_firmware_header_v1_0 *)
-			adev->gfx.mec2_fw->data;
-			adev->gfx.mec2_fw_version =
-			le32_to_cpu(cp_hdr->header.ucode_version);
-			adev->gfx.mec2_feature_version =
-			le32_to_cpu(cp_hdr->ucode_feature_version);
+			amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2);
+			amdgpu_gfx_cp_init_microcode(adev, AMDGPU_UCODE_ID_CP_MEC2_JT);
 		} else {
 			err = 0;
 			adev->gfx.mec2_fw = NULL;
@@ -1530,49 +1399,12 @@ static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
 		adev->gfx.mec2_feature_version = adev->gfx.mec_feature_version;
 	}
 
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
-		info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
-		info->fw = adev->gfx.mec_fw;
-		header = (const struct common_firmware_header *)info->fw->data;
-		cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
-
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
-		info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
-		info->fw = adev->gfx.mec_fw;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
-
-		if (adev->gfx.mec2_fw) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
-			info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
-			info->fw = adev->gfx.mec2_fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
-
-			/* TODO: Determine if MEC2 JT FW loading can be removed
-				 for all GFX V9 asic and above */
-			if (gfx_v9_0_load_mec2_fw_bin_support(adev)) {
-				info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
-				info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
-				info->fw = adev->gfx.mec2_fw;
-				adev->firmware.fw_size +=
-					ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
-					PAGE_SIZE);
-			}
-		}
-	}
-
 out:
 	gfx_v9_0_check_if_need_gfxoff(adev);
 	gfx_v9_0_check_fw_write_wait(adev);
 	if (err) {
 		dev_err(adev->dev,
-			"gfx9: Failed to load firmware \"%s\"\n",
+			"gfx9: Failed to init firmware \"%s\"\n",
 			fw_name);
 		release_firmware(adev->gfx.mec_fw);
 		adev->gfx.mec_fw = NULL;
@@ -5597,7 +5429,7 @@ static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigne
 	BUG_ON(offset > ring->buf_mask);
 	BUG_ON(ring->ring[offset] != 0x55aa55aa);
 
-	cur = (ring->wptr & ring->buf_mask) - 1;
+	cur = (ring->wptr - 1) & ring->buf_mask;
 	if (likely(cur > offset))
 		ring->ring[offset] = cur - offset;
 	else
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
index d8c531581116..8cf53e039c11 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v2_1.c
@@ -576,6 +576,111 @@ static void gfxhub_v2_1_utcl2_harvest(struct amdgpu_device *adev)
 	}
 }
 
+static void gfxhub_v2_1_save_regs(struct amdgpu_device *adev)
+{
+	int i;
+	adev->gmc.VM_L2_CNTL = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL);
+	adev->gmc.VM_L2_CNTL2 = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2);
+	adev->gmc.VM_DUMMY_PAGE_FAULT_CNTL = RREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_CNTL);
+	adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_LO32 = RREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_ADDR_LO32);
+	adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_HI32 = RREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_ADDR_HI32);
+	adev->gmc.VM_L2_PROTECTION_FAULT_CNTL = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL);
+	adev->gmc.VM_L2_PROTECTION_FAULT_CNTL2 = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL2);
+	adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL3 = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_MM_CNTL3);
+	adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL4 = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_MM_CNTL4);
+	adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_LO32 = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_ADDR_LO32);
+	adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_HI32 = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_ADDR_HI32);
+	adev->gmc.VM_DEBUG = RREG32_SOC15(GC, 0, mmGCVM_DEBUG);
+	adev->gmc.VM_L2_MM_GROUP_RT_CLASSES = RREG32_SOC15(GC, 0, mmGCVM_L2_MM_GROUP_RT_CLASSES);
+	adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID = RREG32_SOC15(GC, 0, mmGCVM_L2_BANK_SELECT_RESERVED_CID);
+	adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID2 = RREG32_SOC15(GC, 0, mmGCVM_L2_BANK_SELECT_RESERVED_CID2);
+	adev->gmc.VM_L2_CACHE_PARITY_CNTL = RREG32_SOC15(GC, 0, mmGCVM_L2_CACHE_PARITY_CNTL);
+	adev->gmc.VM_L2_IH_LOG_CNTL = RREG32_SOC15(GC, 0, mmGCVM_L2_IH_LOG_CNTL);
+
+	for (i = 0; i <= 15; i++) {
+		adev->gmc.VM_CONTEXT_CNTL[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL, i);
+		adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_LO32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, i * 2);
+		adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_HI32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, i * 2);
+		adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_LO32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, i * 2);
+		adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_HI32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, i * 2);
+		adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, i * 2);
+		adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, i * 2);
+	}
+
+	adev->gmc.MC_VM_MX_L1_TLB_CNTL = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL);
+}
+
+static void gfxhub_v2_1_restore_regs(struct amdgpu_device *adev)
+{
+	int i;
+	WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL, adev->gmc.VM_L2_CNTL);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2, adev->gmc.VM_L2_CNTL2);
+	WREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_CNTL, adev->gmc.VM_DUMMY_PAGE_FAULT_CNTL);
+	WREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_ADDR_LO32, adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_LO32);
+	WREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_ADDR_HI32, adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_HI32);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL, adev->gmc.VM_L2_PROTECTION_FAULT_CNTL);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL2, adev->gmc.VM_L2_PROTECTION_FAULT_CNTL2);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_MM_CNTL3, adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL3);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_MM_CNTL4, adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL4);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_ADDR_LO32, adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_LO32);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_ADDR_HI32, adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_HI32);
+	WREG32_SOC15(GC, 0, mmGCVM_DEBUG, adev->gmc.VM_DEBUG);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_MM_GROUP_RT_CLASSES, adev->gmc.VM_L2_MM_GROUP_RT_CLASSES);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_BANK_SELECT_RESERVED_CID, adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_BANK_SELECT_RESERVED_CID2, adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID2);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_CACHE_PARITY_CNTL, adev->gmc.VM_L2_CACHE_PARITY_CNTL);
+	WREG32_SOC15(GC, 0, mmGCVM_L2_IH_LOG_CNTL, adev->gmc.VM_L2_IH_LOG_CNTL);
+
+	for (i = 0; i <= 15; i++) {
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL, i, adev->gmc.VM_CONTEXT_CNTL[i]);
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_LO32[i]);
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_HI32[i]);
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_LO32[i]);
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_HI32[i]);
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32[i]);
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32[i]);
+	}
+
+	WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE, adev->gmc.vram_start >> 24);
+	WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_TOP, adev->gmc.vram_end >> 24);
+	WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, adev->gmc.MC_VM_MX_L1_TLB_CNTL);
+}
+
+static void gfxhub_v2_1_halt(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+	int i;
+	uint32_t tmp;
+	int time = 1000;
+
+	gfxhub_v2_1_set_fault_enable_default(adev, false);
+
+	for (i = 0; i <= 14; i++) {
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
+				    i * hub->ctx_addr_distance, ~0);
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
+				    i * hub->ctx_addr_distance, ~0);
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
+				    i * hub->ctx_addr_distance,
+				    0);
+		WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
+				    i * hub->ctx_addr_distance,
+				    0);
+	}
+	tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
+	while ((tmp & (GRBM_STATUS2__EA_BUSY_MASK |
+		      GRBM_STATUS2__EA_LINK_BUSY_MASK)) != 0 &&
+	       time) {
+		udelay(100);
+		time--;
+		tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
+	}
+
+	if (!time) {
+		DRM_WARN("failed to wait for GRBM(EA) idle\n");
+	}
+}
+
 const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = {
 	.get_fb_location = gfxhub_v2_1_get_fb_location,
 	.get_mc_fb_offset = gfxhub_v2_1_get_mc_fb_offset,
@@ -586,4 +691,7 @@ const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = {
 	.init = gfxhub_v2_1_init,
 	.get_xgmi_info = gfxhub_v2_1_get_xgmi_info,
 	.utcl2_harvest = gfxhub_v2_1_utcl2_harvest,
+	.mode2_save_regs = gfxhub_v2_1_save_regs,
+	.mode2_restore_regs = gfxhub_v2_1_restore_regs,
+	.halt = gfxhub_v2_1_halt,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
new file mode 100644
index 000000000000..5d3fffd4929f
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.c
@@ -0,0 +1,511 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "amdgpu.h"
+#include "gfxhub_v3_0_3.h"
+
+#include "gc/gc_11_0_3_offset.h"
+#include "gc/gc_11_0_3_sh_mask.h"
+#include "navi10_enum.h"
+#include "soc15_common.h"
+
+#define regGCVM_L2_CNTL3_DEFAULT		0x80100007
+#define regGCVM_L2_CNTL4_DEFAULT		0x000000c1
+#define regGCVM_L2_CNTL5_DEFAULT		0x00003fe0
+
+static const char *gfxhub_client_ids[] = {
+	"CB/DB",
+	"Reserved",
+	"GE1",
+	"GE2",
+	"CPF",
+	"CPC",
+	"CPG",
+	"RLC",
+	"TCP",
+	"SQC (inst)",
+	"SQC (data)",
+	"SQG",
+	"Reserved",
+	"SDMA0",
+	"SDMA1",
+	"GCR",
+	"SDMA2",
+	"SDMA3",
+};
+
+static uint32_t gfxhub_v3_0_3_get_invalidate_req(unsigned int vmid,
+					       uint32_t flush_type)
+{
+	u32 req = 0;
+
+	/* invalidate using legacy mode on vmid*/
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
+	req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
+			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
+
+	return req;
+}
+
+static void
+gfxhub_v3_0_3_print_l2_protection_fault_status(struct amdgpu_device *adev,
+					     uint32_t status)
+{
+	u32 cid = REG_GET_FIELD(status,
+				GCVM_L2_PROTECTION_FAULT_STATUS, CID);
+
+	dev_err(adev->dev,
+		"GCVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
+		status);
+	dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
+		cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid],
+		cid);
+	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
+		REG_GET_FIELD(status,
+		GCVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
+	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
+		REG_GET_FIELD(status,
+		GCVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
+	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
+		REG_GET_FIELD(status,
+		GCVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
+	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
+		REG_GET_FIELD(status,
+		GCVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
+	dev_err(adev->dev, "\t RW: 0x%lx\n",
+		REG_GET_FIELD(status,
+		GCVM_L2_PROTECTION_FAULT_STATUS, RW));
+}
+
+static u64 gfxhub_v3_0_3_get_fb_location(struct amdgpu_device *adev)
+{
+	u64 base = RREG32_SOC15(GC, 0, regGCMC_VM_FB_LOCATION_BASE);
+
+	base &= GCMC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
+	base <<= 24;
+
+	return base;
+}
+
+static u64 gfxhub_v3_0_3_get_mc_fb_offset(struct amdgpu_device *adev)
+{
+	return (u64)RREG32_SOC15(GC, 0, regGCMC_VM_FB_OFFSET) << 24;
+}
+
+static void gfxhub_v3_0_3_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid,
+				uint64_t page_table_base)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+
+	WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
+			    hub->ctx_addr_distance * vmid,
+			    lower_32_bits(page_table_base));
+
+	WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
+			    hub->ctx_addr_distance * vmid,
+			    upper_32_bits(page_table_base));
+}
+
+static void gfxhub_v3_0_3_init_gart_aperture_regs(struct amdgpu_device *adev)
+{
+	uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
+
+	gfxhub_v3_0_3_setup_vm_pt_regs(adev, 0, pt_base);
+
+	WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
+		     (u32)(adev->gmc.gart_start >> 12));
+	WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
+		     (u32)(adev->gmc.gart_start >> 44));
+
+	WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
+		     (u32)(adev->gmc.gart_end >> 12));
+	WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
+		     (u32)(adev->gmc.gart_end >> 44));
+}
+
+static void gfxhub_v3_0_3_init_system_aperture_regs(struct amdgpu_device *adev)
+{
+	uint64_t value;
+
+	/* Disable AGP. */
+	WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BASE, 0);
+	WREG32_SOC15(GC, 0, regGCMC_VM_AGP_TOP, 0);
+	WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BOT, 0x00FFFFFF);
+
+	/* Program the system aperture low logical page number. */
+	WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR,
+		     adev->gmc.vram_start >> 18);
+	WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
+		     adev->gmc.vram_end >> 18);
+
+	/* Set default page address. */
+	value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start
+		+ adev->vm_manager.vram_base_offset;
+	WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
+		     (u32)(value >> 12));
+	WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
+		     (u32)(value >> 44));
+
+	/* Program "protection fault". */
+	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
+		     (u32)(adev->dummy_page_addr >> 12));
+	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
+		     (u32)((u64)adev->dummy_page_addr >> 44));
+
+	WREG32_FIELD15_PREREG(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
+		       ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
+}
+
+
+static void gfxhub_v3_0_3_init_tlb_regs(struct amdgpu_device *adev)
+{
+	uint32_t tmp;
+
+	/* Setup TLB control */
+	tmp = RREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL);
+
+	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
+	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
+	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+			    ENABLE_ADVANCED_DRIVER_MODEL, 1);
+	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+			    SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
+	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
+	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+			    MTYPE, MTYPE_UC); /* UC, uncached */
+
+	WREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL, tmp);
+}
+
+static void gfxhub_v3_0_3_init_cache_regs(struct amdgpu_device *adev)
+{
+	uint32_t tmp;
+
+	/* These registers are not accessible to VF-SRIOV.
+	 * The PF will program them instead.
+	 */
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	/* Setup L2 cache */
+	tmp = RREG32_SOC15(GC, 0, regGCVM_L2_CNTL);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 1);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+			    ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
+	/* XXX for emulation, Refer to closed source code.*/
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
+			    L2_PDE0_CACHE_TAG_GENERATION_MODE, 0);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
+	WREG32_SOC15(GC, 0, regGCVM_L2_CNTL, tmp);
+
+	tmp = RREG32_SOC15(GC, 0, regGCVM_L2_CNTL2);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
+	WREG32_SOC15(GC, 0, regGCVM_L2_CNTL2, tmp);
+
+	tmp = regGCVM_L2_CNTL3_DEFAULT;
+	if (adev->gmc.translate_further) {
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 12);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3,
+				    L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
+	} else {
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 9);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3,
+				    L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
+	}
+	WREG32_SOC15(GC, 0, regGCVM_L2_CNTL3, tmp);
+
+	tmp = regGCVM_L2_CNTL4_DEFAULT;
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
+	WREG32_SOC15(GC, 0, regGCVM_L2_CNTL4, tmp);
+
+	tmp = regGCVM_L2_CNTL5_DEFAULT;
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL5, L2_CACHE_SMALLK_FRAGMENT_SIZE, 0);
+	WREG32_SOC15(GC, 0, regGCVM_L2_CNTL5, tmp);
+}
+
+static void gfxhub_v3_0_3_enable_system_domain(struct amdgpu_device *adev)
+{
+	uint32_t tmp;
+
+	tmp = RREG32_SOC15(GC, 0, regGCVM_CONTEXT0_CNTL);
+	tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
+	tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
+	tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL,
+			    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
+	WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_CNTL, tmp);
+}
+
+static void gfxhub_v3_0_3_disable_identity_aperture(struct amdgpu_device *adev)
+{
+	/* These registers are not accessible to VF-SRIOV.
+	 * The PF will program them instead.
+	 */
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
+		     0xFFFFFFFF);
+	WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
+		     0x0000000F);
+
+	WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
+		     0);
+	WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
+		     0);
+
+	WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
+	WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);
+
+}
+
+static void gfxhub_v3_0_3_setup_vmid_config(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+	int i;
+	uint32_t tmp;
+
+	for (i = 0; i <= 14; i++) {
+		tmp = RREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_CNTL, i);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
+				    adev->vm_manager.num_level);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+				VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+				READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+				EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+				PAGE_TABLE_BLOCK_SIZE,
+				adev->vm_manager.block_size - 9);
+		/* Send no-retry XNACK on fault to suppress VM fault storm. */
+		tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
+				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
+				    !amdgpu_noretry);
+		WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_CNTL,
+				    i * hub->ctx_distance, tmp);
+		WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
+				    i * hub->ctx_addr_distance, 0);
+		WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
+				    i * hub->ctx_addr_distance, 0);
+		WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
+				    i * hub->ctx_addr_distance,
+				    lower_32_bits(adev->vm_manager.max_pfn - 1));
+		WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
+				    i * hub->ctx_addr_distance,
+				    upper_32_bits(adev->vm_manager.max_pfn - 1));
+	}
+
+	hub->vm_cntx_cntl = tmp;
+}
+
+static void gfxhub_v3_0_3_program_invalidation(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+	unsigned i;
+
+	for (i = 0 ; i < 18; ++i) {
+		WREG32_SOC15_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
+				    i * hub->eng_addr_distance, 0xffffffff);
+		WREG32_SOC15_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
+				    i * hub->eng_addr_distance, 0x1f);
+	}
+}
+
+static int gfxhub_v3_0_3_gart_enable(struct amdgpu_device *adev)
+{
+	if (amdgpu_sriov_vf(adev)) {
+		/*
+		 * GCMC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
+		 * VF copy registers so vbios post doesn't program them, for
+		 * SRIOV driver need to program them
+		 */
+		WREG32_SOC15(GC, 0, regGCMC_VM_FB_LOCATION_BASE,
+			     adev->gmc.vram_start >> 24);
+		WREG32_SOC15(GC, 0, regGCMC_VM_FB_LOCATION_TOP,
+			     adev->gmc.vram_end >> 24);
+	}
+
+	/* GART Enable. */
+	gfxhub_v3_0_3_init_gart_aperture_regs(adev);
+	gfxhub_v3_0_3_init_system_aperture_regs(adev);
+	gfxhub_v3_0_3_init_tlb_regs(adev);
+	gfxhub_v3_0_3_init_cache_regs(adev);
+
+	gfxhub_v3_0_3_enable_system_domain(adev);
+	gfxhub_v3_0_3_disable_identity_aperture(adev);
+	gfxhub_v3_0_3_setup_vmid_config(adev);
+	gfxhub_v3_0_3_program_invalidation(adev);
+
+	return 0;
+}
+
+static void gfxhub_v3_0_3_gart_disable(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+	u32 tmp;
+	u32 i;
+
+	/* Disable all tables */
+	for (i = 0; i < 16; i++)
+		WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT0_CNTL,
+				    i * hub->ctx_distance, 0);
+
+	/* Setup TLB control */
+	tmp = RREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL);
+	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
+	tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
+			    ENABLE_ADVANCED_DRIVER_MODEL, 0);
+	WREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL, tmp);
+
+	/* Setup L2 cache */
+	WREG32_FIELD15_PREREG(GC, 0, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0);
+	WREG32_SOC15(GC, 0, regGCVM_L2_CNTL3, 0);
+}
+
+/**
+ * gfxhub_v3_0_3_set_fault_enable_default - update GART/VM fault handling
+ *
+ * @adev: amdgpu_device pointer
+ * @value: true redirects VM faults to the default page
+ */
+static void gfxhub_v3_0_3_set_fault_enable_default(struct amdgpu_device *adev,
+					  bool value)
+{
+	u32 tmp;
+
+	/* These registers are not accessible to VF-SRIOV.
+	 * The PF will program them instead.
+	 */
+	if (amdgpu_sriov_vf(adev))
+		return;
+
+	tmp = RREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
+			    value);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+			    EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
+	if (!value) {
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+				CRASH_ON_NO_RETRY_FAULT, 1);
+		tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
+				CRASH_ON_RETRY_FAULT, 1);
+	}
+	WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL, tmp);
+}
+
+static const struct amdgpu_vmhub_funcs gfxhub_v3_0_3_vmhub_funcs = {
+	.print_l2_protection_fault_status = gfxhub_v3_0_3_print_l2_protection_fault_status,
+	.get_invalidate_req = gfxhub_v3_0_3_get_invalidate_req,
+};
+
+static void gfxhub_v3_0_3_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0];
+
+	hub->ctx0_ptb_addr_lo32 =
+		SOC15_REG_OFFSET(GC, 0,
+				 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
+	hub->ctx0_ptb_addr_hi32 =
+		SOC15_REG_OFFSET(GC, 0,
+				 regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
+	hub->vm_inv_eng0_sem =
+		SOC15_REG_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_SEM);
+	hub->vm_inv_eng0_req =
+		SOC15_REG_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_REQ);
+	hub->vm_inv_eng0_ack =
+		SOC15_REG_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_ACK);
+	hub->vm_context0_cntl =
+		SOC15_REG_OFFSET(GC, 0, regGCVM_CONTEXT0_CNTL);
+	hub->vm_l2_pro_fault_status =
+		SOC15_REG_OFFSET(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS);
+	hub->vm_l2_pro_fault_cntl =
+		SOC15_REG_OFFSET(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL);
+
+	hub->ctx_distance = regGCVM_CONTEXT1_CNTL - regGCVM_CONTEXT0_CNTL;
+	hub->ctx_addr_distance = regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 -
+		regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
+	hub->eng_distance = regGCVM_INVALIDATE_ENG1_REQ -
+		regGCVM_INVALIDATE_ENG0_REQ;
+	hub->eng_addr_distance = regGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 -
+		regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
+
+	hub->vm_cntx_cntl_vm_fault = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
+		GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
+
+	hub->vmhub_funcs = &gfxhub_v3_0_3_vmhub_funcs;
+}
+
+const struct amdgpu_gfxhub_funcs gfxhub_v3_0_3_funcs = {
+	.get_fb_location = gfxhub_v3_0_3_get_fb_location,
+	.get_mc_fb_offset = gfxhub_v3_0_3_get_mc_fb_offset,
+	.setup_vm_pt_regs = gfxhub_v3_0_3_setup_vm_pt_regs,
+	.gart_enable = gfxhub_v3_0_3_gart_enable,
+	.gart_disable = gfxhub_v3_0_3_gart_disable,
+	.set_fault_enable_default = gfxhub_v3_0_3_set_fault_enable_default,
+	.init = gfxhub_v3_0_3_init,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.h b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.h
new file mode 100644
index 000000000000..6153bd5e3083
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/gfxhub_v3_0_3.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __GFXHUB_V3_0_3_H__
+#define __GFXHUB_V3_0_3_H__
+
+extern const struct amdgpu_gfxhub_funcs gfxhub_v3_0_3_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
index 1471bfb9ae38..846ccb6cf07d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
@@ -39,6 +39,7 @@
 #include "soc15_common.h"
 #include "nbio_v4_3.h"
 #include "gfxhub_v3_0.h"
+#include "gfxhub_v3_0_3.h"
 #include "mmhub_v3_0.h"
 #include "mmhub_v3_0_1.h"
 #include "mmhub_v3_0_2.h"
@@ -233,7 +234,8 @@ static void gmc_v11_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
 
 	/* Issue additional private vm invalidation to MMHUB */
 	if ((vmhub != AMDGPU_GFXHUB_0) &&
-	    (hub->vm_l2_bank_select_reserved_cid2)) {
+	    (hub->vm_l2_bank_select_reserved_cid2) &&
+		!amdgpu_sriov_vf(adev)) {
 		inv_req = RREG32_NO_KIQ(hub->vm_l2_bank_select_reserved_cid2);
 		/* bit 25: RSERVED_CACHE_PRIVATE_INVALIDATION */
 		inv_req |= (1 << 25);
@@ -590,7 +592,14 @@ static void gmc_v11_0_set_mmhub_funcs(struct amdgpu_device *adev)
 
 static void gmc_v11_0_set_gfxhub_funcs(struct amdgpu_device *adev)
 {
-	adev->gfxhub.funcs = &gfxhub_v3_0_funcs;
+	switch (adev->ip_versions[GC_HWIP][0]) {
+	case IP_VERSION(11, 0, 3):
+		adev->gfxhub.funcs = &gfxhub_v3_0_3_funcs;
+		break;
+	default:
+		adev->gfxhub.funcs = &gfxhub_v3_0_funcs;
+		break;
+	}
 }
 
 static int gmc_v11_0_early_init(void *handle)
@@ -640,7 +649,10 @@ static void gmc_v11_0_vram_gtt_location(struct amdgpu_device *adev,
 	amdgpu_gmc_gart_location(adev, mc);
 
 	/* base offset of vram pages */
-	adev->vm_manager.vram_base_offset = adev->mmhub.funcs->get_mc_fb_offset(adev);
+	if (amdgpu_sriov_vf(adev))
+		adev->vm_manager.vram_base_offset = 0;
+	else
+		adev->vm_manager.vram_base_offset = adev->mmhub.funcs->get_mc_fb_offset(adev);
 }
 
 /**
@@ -732,6 +744,7 @@ static int gmc_v11_0_sw_init(void *handle)
 	case IP_VERSION(11, 0, 0):
 	case IP_VERSION(11, 0, 1):
 	case IP_VERSION(11, 0, 2):
+	case IP_VERSION(11, 0, 3):
 		adev->num_vmhubs = 2;
 		/*
 		 * To fulfill 4-level page support,
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c b/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
index 085e613f3646..7cd79a3844b2 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v6_0.c
@@ -105,7 +105,13 @@ force_update_wptr_for_self_int(struct amdgpu_device *adev,
 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
 				   RB_USED_INT_THRESHOLD, threshold);
 
-	WREG32_SOC15(OSSSYS, 0, regIH_RB_CNTL_RING1, ih_rb_cntl);
+	if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) {
+		if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1, ih_rb_cntl))
+			return;
+	} else {
+		WREG32_SOC15(OSSSYS, 0, regIH_RB_CNTL_RING1, ih_rb_cntl);
+	}
+
 	WREG32_SOC15(OSSSYS, 0, regIH_CNTL2, ih_cntl);
 }
 
@@ -132,7 +138,13 @@ static int ih_v6_0_toggle_ring_interrupts(struct amdgpu_device *adev,
 	/* enable_intr field is only valid in ring0 */
 	if (ih == &adev->irq.ih)
 		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0));
-	WREG32(ih_regs->ih_rb_cntl, tmp);
+
+	if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) {
+		if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp))
+			return -ETIMEDOUT;
+	} else {
+		WREG32(ih_regs->ih_rb_cntl, tmp);
+	}
 
 	if (enable) {
 		ih->enabled = true;
@@ -242,7 +254,15 @@ static int ih_v6_0_enable_ring(struct amdgpu_device *adev,
 		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 0);
 		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_FULL_DRAIN_ENABLE, 1);
 	}
-	WREG32(ih_regs->ih_rb_cntl, tmp);
+
+	if (amdgpu_sriov_vf(adev) && amdgpu_sriov_reg_indirect_ih(adev)) {
+		if (psp_reg_program(&adev->psp, ih_regs->psp_reg_id, tmp)) {
+			DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
+			return -ETIMEDOUT;
+		}
+	} else {
+		WREG32(ih_regs->ih_rb_cntl, tmp);
+	}
 
 	if (ih == &adev->irq.ih) {
 		/* set the ih ring 0 writeback address whether it's enabled or not */
diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c b/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c
index 76383baa3929..95548c512f4f 100644
--- a/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/imu_v11_0.c
@@ -26,12 +26,15 @@
 #include "amdgpu_imu.h"
 #include "amdgpu_dpm.h"
 
+#include "imu_v11_0_3.h"
+
 #include "gc/gc_11_0_0_offset.h"
 #include "gc/gc_11_0_0_sh_mask.h"
 
 MODULE_FIRMWARE("amdgpu/gc_11_0_0_imu.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_1_imu.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_2_imu.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_3_imu.bin");
 
 static int imu_v11_0_init_microcode(struct amdgpu_device *adev)
 {
@@ -360,6 +363,9 @@ static void imu_v11_0_program_rlc_ram(struct amdgpu_device *adev)
 		program_imu_rlc_ram(adev, imu_rlc_ram_golden_11_0_2,
 				(const u32)ARRAY_SIZE(imu_rlc_ram_golden_11_0_2));
 		break;
+	case IP_VERSION(11, 0, 3):
+		imu_v11_0_3_program_rlc_ram(adev);
+		break;
 	default:
 		BUG();
 		break;
diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v11_0_3.c b/drivers/gpu/drm/amd/amdgpu/imu_v11_0_3.c
new file mode 100644
index 000000000000..fc69c1a29e23
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/imu_v11_0_3.c
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "amdgpu.h"
+#include "amdgpu_imu.h"
+#include "imu_v11_0_3.h"
+
+#include "gc/gc_11_0_3_offset.h"
+#include "gc/gc_11_0_3_sh_mask.h"
+
+static const struct imu_rlc_ram_golden imu_rlc_ram_golden_11_0_3[] = {
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_IO_RD_COMBINE_FLUSH, 0x00055555, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_IO_WR_COMBINE_FLUSH, 0x00055555, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_DRAM_COMBINE_FLUSH, 0x00555555, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_MISC2, 0x00001ffe, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_SDP_CREDITS, 0x003f3fff, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_SDP_TAG_RESERVE1, 0x00000000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_SDP_VCC_RESERVE0, 0x00041000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_SDP_VCC_RESERVE1, 0x00000000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_SDP_VCD_RESERVE0, 0x00040000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_SDP_VCD_RESERVE1, 0x00000000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_MISC, 0x00000017, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGUS_SDP_ENABLE, 0x00000001, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_SDP_CREDITS, 0x003f3fbf, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_SDP_TAG_RESERVE0, 0x10200800, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_SDP_TAG_RESERVE1, 0x00000088, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_SDP_VCC_RESERVE0, 0x1d041040, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_SDP_VCC_RESERVE1, 0x80000000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_SDP_IO_PRIORITY, 0x88888888, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_MAM_CTRL, 0x0000d800, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_SDP_ARB_FINAL, 0x000007ff, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_DRAM_PAGE_BURST, 0x20080200, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_SDP_ENABLE, 0x00000001, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2, 0x00020000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_APT_CNTL, 0x0000000c, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_CACHEABLE_DRAM_ADDRESS_END, 0x000fffff, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCEA_MISC, 0x0c48bff0, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_GC_SA_UNIT_DISABLE, 0x00fffc01, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_GC_PRIM_CONFIG, 0x000fffe1, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_RB_BACKEND_DISABLE, 0xffffff01, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG, 0xfffe0001, 0x40000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG, 0xfffe0001, 0x42000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG, 0xffff0001, 0x44000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG, 0xffff0001, 0x46000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG, 0xffff0001, 0x48000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_GC_SHADER_ARRAY_CONFIG, 0xffff0001, 0x4A000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCGTS_TCC_DISABLE, 0x00000001, 0x00000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_GC_SHADER_RATE_CONFIG, 0x00000001, 0x00000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCC_GC_EDC_CONFIG, 0x00000001, 0x00000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL, 0x00000500, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR, 0x00000001, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0x00000000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_LOCAL_FB_ADDRESS_START, 0x00000000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_LOCAL_FB_ADDRESS_END, 0x000005ff, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_FB_LOCATION_BASE, 0x00006000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_FB_LOCATION_TOP, 0x000065ff, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_CONTEXT0_CNTL, 0x00000000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_CONTEXT1_CNTL, 0x00000000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_NB_TOP_OF_DRAM_SLOT1, 0xff800000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_NB_LOWER_TOP_OF_DRAM2, 0x00000001, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_NB_UPPER_TOP_OF_DRAM2, 0x00000fff, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL, 0x00001ffc, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL, 0x00000551, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_L2_CNTL, 0x00080603, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_L2_CNTL2, 0x00000003, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_L2_CNTL3, 0x00100003, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_L2_CNTL5, 0x00003fe0, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_CONTEXT0_CNTL, 0x00000001, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES, 0x00000c00, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_CONTEXT1_CNTL, 0x00000001, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES, 0x00000c00, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGB_ADDR_CONFIG, 0x00000444, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGL2_PIPE_STEER_0, 0x54105410, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGL2_PIPE_STEER_2, 0x76323276, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGB_ADDR_CONFIG, 0x00000244, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCUTCL2_HARVEST_BYPASS_GROUPS, 0x00000006, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_APT_CNTL, 0x0000000c, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_AGP_BASE, 0x00000000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_AGP_BOT, 0x00000002, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCMC_VM_AGP_TOP, 0x00000000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL2, 0x00020000, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regSDMA0_UCODE_SELFLOAD_CONTROL, 0x00000210, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regSDMA1_UCODE_SELFLOAD_CONTROL, 0x00000210, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCPC_PSP_DEBUG, CPC_PSP_DEBUG__GPA_OVERRIDE_MASK, 0xe0000000),
+	IMU_RLC_RAM_GOLDEN_VALUE(GC, 0, regCPG_PSP_DEBUG, CPG_PSP_DEBUG__GPA_OVERRIDE_MASK, 0xe0000000),
+};
+
+static void program_rlc_ram_register_setting(struct amdgpu_device *adev,
+					     const struct imu_rlc_ram_golden *regs,
+					     const u32 array_size)
+{
+	const struct imu_rlc_ram_golden *entry;
+	u32 reg, data;
+	int i;
+
+	for (i = 0; i < array_size; ++i) {
+		entry = &regs[i];
+		reg =  adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
+		reg |= entry->addr_mask;
+
+		data = entry->data;
+		if (entry->reg == regGCMC_VM_AGP_BASE)
+			data = 0x00ffffff;
+		else if (entry->reg == regGCMC_VM_AGP_TOP)
+			data = 0x0;
+		else if (entry->reg == regGCMC_VM_FB_LOCATION_BASE)
+			data = adev->gmc.vram_start >> 24;
+		else if (entry->reg == regGCMC_VM_FB_LOCATION_TOP)
+			data = adev->gmc.vram_end >> 24;
+
+		WREG32_SOC15(GC, 0, regGFX_IMU_RLC_RAM_ADDR_HIGH, 0);
+		WREG32_SOC15(GC, 0, regGFX_IMU_RLC_RAM_ADDR_LOW, reg);
+		WREG32_SOC15(GC, 0, regGFX_IMU_RLC_RAM_DATA, data);
+	}
+	//Indicate the latest entry
+	WREG32_SOC15(GC, 0, regGFX_IMU_RLC_RAM_ADDR_HIGH, 0);
+	WREG32_SOC15(GC, 0, regGFX_IMU_RLC_RAM_ADDR_LOW, 0);
+	WREG32_SOC15(GC, 0, regGFX_IMU_RLC_RAM_DATA, 0);
+}
+
+void imu_v11_0_3_program_rlc_ram(struct amdgpu_device *adev)
+{
+	program_rlc_ram_register_setting(adev,
+					 imu_rlc_ram_golden_11_0_3,
+					 (const u32)ARRAY_SIZE(imu_rlc_ram_golden_11_0_3));
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v11_0_3.h b/drivers/gpu/drm/amd/amdgpu/imu_v11_0_3.h
new file mode 100644
index 000000000000..702be568f26b
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/imu_v11_0_3.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __IMU_V11_0_3_H__
+#define __IMU_V11_0_3_H__
+
+void imu_v11_0_3_program_rlc_ram(struct amdgpu_device *adev);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
index cc3fdbbcd314..5cec6b259b7f 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v11_0.c
@@ -38,6 +38,8 @@ MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_1_mes1.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes.bin");
 MODULE_FIRMWARE("amdgpu/gc_11_0_2_mes1.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes.bin");
+MODULE_FIRMWARE("amdgpu/gc_11_0_3_mes1.bin");
 
 static int mes_v11_0_hw_fini(void *handle);
 static int mes_v11_0_kiq_hw_init(struct amdgpu_device *adev);
@@ -185,6 +187,19 @@ static int mes_v11_0_add_hw_queue(struct amdgpu_mes *mes,
 	mes_add_queue_pkt.is_kfd_process = input->is_kfd_process;
 	mes_add_queue_pkt.trap_en = 1;
 
+	/* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
+	mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
+	mes_add_queue_pkt.gds_size = input->queue_size;
+
+	if (!(((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 4) &&
+		  (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0)) &&
+		  (adev->ip_versions[GC_HWIP][0] <= IP_VERSION(11, 0, 3))))
+		mes_add_queue_pkt.trap_en = 1;
+
+	/* For KFD, gds_size is re-used for queue size (needed in MES for AQL queues) */
+	mes_add_queue_pkt.is_aql_queue = input->is_aql_queue;
+	mes_add_queue_pkt.gds_size = input->queue_size;
+
 	return mes_v11_0_submit_pkt_and_poll_completion(mes,
 			&mes_add_queue_pkt, sizeof(mes_add_queue_pkt),
 			offsetof(union MESAPI__ADD_QUEUE, api_status));
diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c
index bc11b2de37ae..a1d26c4d80b8 100644
--- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0.c
@@ -169,17 +169,17 @@ static void mmhub_v3_0_init_system_aperture_regs(struct amdgpu_device *adev)
 	uint64_t value;
 	uint32_t tmp;
 
-	/* Disable AGP. */
-	WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_BASE, 0);
-	WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_TOP, 0);
-	WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_BOT, 0x00FFFFFF);
-
 	if (!amdgpu_sriov_vf(adev)) {
 		/*
 		 * the new L1 policy will block SRIOV guest from writing
 		 * these regs, and they will be programed at host.
 		 * so skip programing these regs.
 		 */
+		/* Disable AGP. */
+		WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_BASE, 0);
+		WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_TOP, 0);
+		WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_BOT, 0x00FFFFFF);
+
 		/* Program the system aperture low logical page number. */
 		WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR,
 			     adev->gmc.vram_start >> 18);
diff --git a/drivers/gpu/drm/amd/amdgpu/mmsch_v4_0.h b/drivers/gpu/drm/amd/amdgpu/mmsch_v4_0.h
new file mode 100644
index 000000000000..f772bb499f3e
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/mmsch_v4_0.h
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __MMSCH_V4_0_H__
+#define __MMSCH_V4_0_H__
+
+#include "amdgpu_vcn.h"
+
+#define MMSCH_VERSION_MAJOR	4
+#define MMSCH_VERSION_MINOR	0
+#define MMSCH_VERSION	(MMSCH_VERSION_MAJOR << 16 | MMSCH_VERSION_MINOR)
+
+#define RB_ENABLED (1 << 0)
+#define RB4_ENABLED (1 << 1)
+#define MMSCH_DOORBELL_OFFSET 0x8
+
+#define MMSCH_VF_ENGINE_STATUS__PASS 0x1
+
+#define MMSCH_VF_MAILBOX_RESP__OK 0x1
+#define MMSCH_VF_MAILBOX_RESP__INCOMPLETE 0x2
+
+enum mmsch_v4_0_command_type {
+	MMSCH_COMMAND__DIRECT_REG_WRITE = 0,
+	MMSCH_COMMAND__DIRECT_REG_POLLING = 2,
+	MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE = 3,
+	MMSCH_COMMAND__INDIRECT_REG_WRITE = 8,
+	MMSCH_COMMAND__END = 0xf
+};
+
+struct mmsch_v4_0_table_info {
+	uint32_t init_status;
+	uint32_t table_offset;
+	uint32_t table_size;
+};
+
+struct mmsch_v4_0_init_header {
+	uint32_t version;
+	uint32_t total_size;
+	struct mmsch_v4_0_table_info inst[AMDGPU_MAX_VCN_INSTANCES];
+	struct mmsch_v4_0_table_info jpegdec;
+};
+
+struct mmsch_v4_0_cmd_direct_reg_header {
+	uint32_t reg_offset   : 28;
+	uint32_t command_type : 4;
+};
+
+struct mmsch_v4_0_cmd_indirect_reg_header {
+	uint32_t reg_offset    : 20;
+	uint32_t reg_idx_space : 8;
+	uint32_t command_type  : 4;
+};
+
+struct mmsch_v4_0_cmd_direct_write {
+	struct mmsch_v4_0_cmd_direct_reg_header cmd_header;
+	uint32_t reg_value;
+};
+
+struct mmsch_v4_0_cmd_direct_read_modify_write {
+	struct mmsch_v4_0_cmd_direct_reg_header cmd_header;
+	uint32_t write_data;
+	uint32_t mask_value;
+};
+
+struct mmsch_v4_0_cmd_direct_polling {
+	struct mmsch_v4_0_cmd_direct_reg_header cmd_header;
+	uint32_t mask_value;
+	uint32_t wait_value;
+};
+
+struct mmsch_v4_0_cmd_end {
+	struct mmsch_v4_0_cmd_direct_reg_header cmd_header;
+};
+
+struct mmsch_v4_0_cmd_indirect_write {
+	struct mmsch_v4_0_cmd_indirect_reg_header cmd_header;
+	uint32_t reg_value;
+};
+
+#define MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(reg, mask, data) { \
+	size = sizeof(struct mmsch_v4_0_cmd_direct_read_modify_write); \
+	size_dw = size / 4; \
+	direct_rd_mod_wt.cmd_header.reg_offset = reg; \
+	direct_rd_mod_wt.mask_value = mask; \
+	direct_rd_mod_wt.write_data = data; \
+	memcpy((void *)table_loc, &direct_rd_mod_wt, size); \
+	table_loc += size_dw; \
+	table_size += size_dw; \
+}
+
+#define MMSCH_V4_0_INSERT_DIRECT_WT(reg, value) { \
+	size = sizeof(struct mmsch_v4_0_cmd_direct_write); \
+	size_dw = size / 4; \
+	direct_wt.cmd_header.reg_offset = reg; \
+	direct_wt.reg_value = value; \
+	memcpy((void *)table_loc, &direct_wt, size); \
+	table_loc += size_dw; \
+	table_size += size_dw; \
+}
+
+#define MMSCH_V4_0_INSERT_DIRECT_POLL(reg, mask, wait) { \
+	size = sizeof(struct mmsch_v4_0_cmd_direct_polling); \
+	size_dw = size / 4; \
+	direct_poll.cmd_header.reg_offset = reg; \
+	direct_poll.mask_value = mask; \
+	direct_poll.wait_value = wait; \
+	memcpy((void *)table_loc, &direct_poll, size); \
+	table_loc += size_dw; \
+	table_size += size_dw; \
+}
+
+#define MMSCH_V4_0_INSERT_END() { \
+	size = sizeof(struct mmsch_v4_0_cmd_end); \
+	size_dw = size / 4; \
+	memcpy((void *)table_loc, &end, size); \
+	table_loc += size_dw; \
+	table_size += size_dw; \
+}
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
index 12906ba74462..a2f04b249132 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_ai.c
@@ -290,6 +290,7 @@ flr_done:
 		reset_context.method = AMD_RESET_METHOD_NONE;
 		reset_context.reset_req_dev = adev;
 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+		clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags);
 
 		amdgpu_device_gpu_recover(adev, NULL, &reset_context);
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
index e07757eea7ad..a977f0027928 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_nv.c
@@ -317,6 +317,7 @@ flr_done:
 		reset_context.method = AMD_RESET_METHOD_NONE;
 		reset_context.reset_req_dev = adev;
 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+		clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags);
 
 		amdgpu_device_gpu_recover(adev, NULL, &reset_context);
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
index 288c414babdf..fd14fa9b9cd7 100644
--- a/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/mxgpu_vi.c
@@ -529,6 +529,7 @@ static void xgpu_vi_mailbox_flr_work(struct work_struct *work)
 		reset_context.method = AMD_RESET_METHOD_NONE;
 		reset_context.reset_req_dev = adev;
 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+		clear_bit(AMDGPU_SKIP_MODE2_RESET, &reset_context.flags);
 
 		amdgpu_device_gpu_recover(adev, NULL, &reset_context);
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
index 982a89f841d5..15eb3658d70e 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.c
@@ -488,3 +488,47 @@ const struct amdgpu_nbio_funcs nbio_v4_3_funcs = {
 	.get_rom_offset = nbio_v4_3_get_rom_offset,
 	.program_aspm = nbio_v4_3_program_aspm,
 };
+
+
+static void nbio_v4_3_sriov_ih_doorbell_range(struct amdgpu_device *adev,
+					bool use_doorbell, int doorbell_index)
+{
+}
+
+static void nbio_v4_3_sriov_sdma_doorbell_range(struct amdgpu_device *adev, int instance,
+					  bool use_doorbell, int doorbell_index,
+					  int doorbell_size)
+{
+}
+
+static void nbio_v4_3_sriov_vcn_doorbell_range(struct amdgpu_device *adev, bool use_doorbell,
+					 int doorbell_index, int instance)
+{
+}
+
+static void nbio_v4_3_sriov_gc_doorbell_init(struct amdgpu_device *adev)
+{
+}
+
+const struct amdgpu_nbio_funcs nbio_v4_3_sriov_funcs = {
+	.get_hdp_flush_req_offset = nbio_v4_3_get_hdp_flush_req_offset,
+	.get_hdp_flush_done_offset = nbio_v4_3_get_hdp_flush_done_offset,
+	.get_pcie_index_offset = nbio_v4_3_get_pcie_index_offset,
+	.get_pcie_data_offset = nbio_v4_3_get_pcie_data_offset,
+	.get_rev_id = nbio_v4_3_get_rev_id,
+	.mc_access_enable = nbio_v4_3_mc_access_enable,
+	.get_memsize = nbio_v4_3_get_memsize,
+	.sdma_doorbell_range = nbio_v4_3_sriov_sdma_doorbell_range,
+	.vcn_doorbell_range = nbio_v4_3_sriov_vcn_doorbell_range,
+	.gc_doorbell_init = nbio_v4_3_sriov_gc_doorbell_init,
+	.enable_doorbell_aperture = nbio_v4_3_enable_doorbell_aperture,
+	.enable_doorbell_selfring_aperture = nbio_v4_3_enable_doorbell_selfring_aperture,
+	.ih_doorbell_range = nbio_v4_3_sriov_ih_doorbell_range,
+	.update_medium_grain_clock_gating = nbio_v4_3_update_medium_grain_clock_gating,
+	.update_medium_grain_light_sleep = nbio_v4_3_update_medium_grain_light_sleep,
+	.get_clockgating_state = nbio_v4_3_get_clockgating_state,
+	.ih_control = nbio_v4_3_ih_control,
+	.init_registers = nbio_v4_3_init_registers,
+	.remap_hdp_registers = nbio_v4_3_remap_hdp_registers,
+	.get_rom_offset = nbio_v4_3_get_rom_offset,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.h b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.h
index ade43661d7a9..711999ceedf4 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.h
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v4_3.h
@@ -28,5 +28,6 @@
 
 extern const struct nbio_hdp_flush_reg nbio_v4_3_hdp_flush_reg;
 extern const struct amdgpu_nbio_funcs nbio_v4_3_funcs;
+extern const struct amdgpu_nbio_funcs nbio_v4_3_sriov_funcs;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
index a75a286e1ecf..21d822b1d589 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
@@ -44,6 +44,7 @@ MODULE_FIRMWARE("amdgpu/psp_13_0_0_sos.bin");
 MODULE_FIRMWARE("amdgpu/psp_13_0_0_ta.bin");
 MODULE_FIRMWARE("amdgpu/psp_13_0_7_sos.bin");
 MODULE_FIRMWARE("amdgpu/psp_13_0_7_ta.bin");
+MODULE_FIRMWARE("amdgpu/psp_13_0_10_sos.bin");
 
 /* For large FW files the time to complete can be very long */
 #define USBC_PD_POLLING_LIMIT_S 240
@@ -109,6 +110,7 @@ static int psp_v13_0_init_microcode(struct psp_context *psp)
 		break;
 	case IP_VERSION(13, 0, 0):
 	case IP_VERSION(13, 0, 7):
+	case IP_VERSION(13, 0, 10):
 		err = psp_init_sos_microcode(psp, chip_name);
 		if (err)
 			return err;
@@ -222,6 +224,12 @@ static int psp_v13_0_bootloader_load_dbg_drv(struct psp_context *psp)
 	return psp_v13_0_bootloader_load_component(psp, &psp->dbg_drv, PSP_BL__LOAD_DBGDRV);
 }
 
+static int psp_v13_0_bootloader_load_ras_drv(struct psp_context *psp)
+{
+	return psp_v13_0_bootloader_load_component(psp, &psp->ras_drv, PSP_BL__LOAD_RASDRV);
+}
+
+
 static int psp_v13_0_bootloader_load_sos(struct psp_context *psp)
 {
 	int ret;
@@ -718,6 +726,7 @@ static const struct psp_funcs psp_v13_0_funcs = {
 	.bootloader_load_soc_drv = psp_v13_0_bootloader_load_soc_drv,
 	.bootloader_load_intf_drv = psp_v13_0_bootloader_load_intf_drv,
 	.bootloader_load_dbg_drv = psp_v13_0_bootloader_load_dbg_drv,
+	.bootloader_load_ras_drv = psp_v13_0_bootloader_load_ras_drv,
 	.bootloader_load_sos = psp_v13_0_bootloader_load_sos,
 	.ring_init = psp_v13_0_ring_init,
 	.ring_create = psp_v13_0_ring_create,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index 56424f75dd2c..7241a9fb0121 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -561,44 +561,6 @@ static void sdma_v4_0_setup_ulv(struct amdgpu_device *adev)
 	}
 }
 
-static int sdma_v4_0_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
-{
-	int err = 0;
-	const struct sdma_firmware_header_v1_0 *hdr;
-
-	err = amdgpu_ucode_validate(sdma_inst->fw);
-	if (err)
-		return err;
-
-	hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;
-	sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
-	sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
-
-	if (sdma_inst->feature_version >= 20)
-		sdma_inst->burst_nop = true;
-
-	return 0;
-}
-
-static void sdma_v4_0_destroy_inst_ctx(struct amdgpu_device *adev)
-{
-	int i;
-
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		release_firmware(adev->sdma.instance[i].fw);
-		adev->sdma.instance[i].fw = NULL;
-
-		/* arcturus shares the same FW memory across
-		   all SDMA isntances */
-		if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 2) ||
-		    adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 0))
-			break;
-	}
-
-	memset((void *)adev->sdma.instance, 0,
-		sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
-}
-
 /**
  * sdma_v4_0_init_microcode - load ucode images from disk
  *
@@ -615,9 +577,7 @@ static int sdma_v4_0_init_microcode(struct amdgpu_device *adev)
 {
 	const char *chip_name;
 	char fw_name[30];
-	int err = 0, i;
-	struct amdgpu_firmware_info *info = NULL;
-	const struct common_firmware_header *header = NULL;
+	int ret, i;
 
 	DRM_DEBUG("\n");
 
@@ -656,58 +616,25 @@ static int sdma_v4_0_init_microcode(struct amdgpu_device *adev)
 		BUG();
 	}
 
-	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name);
-
-	err = request_firmware(&adev->sdma.instance[0].fw, fw_name, adev->dev);
-	if (err)
-		goto out;
-
-	err = sdma_v4_0_init_inst_ctx(&adev->sdma.instance[0]);
-	if (err)
-		goto out;
-
-	for (i = 1; i < adev->sdma.num_instances; i++) {
+	for (i = 0; i < adev->sdma.num_instances; i++) {
+		if (i == 0)
+			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name);
+		else
+			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma%d.bin", chip_name, i);
 		if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 2) ||
                     adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 0)) {
 			/* Acturus & Aldebaran will leverage the same FW memory
 			   for every SDMA instance */
-			memcpy((void *)&adev->sdma.instance[i],
-			       (void *)&adev->sdma.instance[0],
-			       sizeof(struct amdgpu_sdma_instance));
-		}
-		else {
-			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma%d.bin", chip_name, i);
-
-			err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev);
-			if (err)
-				goto out;
-
-			err = sdma_v4_0_init_inst_ctx(&adev->sdma.instance[i]);
-			if (err)
-				goto out;
-		}
-	}
-
-	DRM_DEBUG("psp_load == '%s'\n",
-		adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
-
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		for (i = 0; i < adev->sdma.num_instances; i++) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
-			info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
-			info->fw = adev->sdma.instance[i].fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
+			ret = amdgpu_sdma_init_microcode(adev, fw_name, 0, true);
+			break;
+		} else {
+			ret = amdgpu_sdma_init_microcode(adev, fw_name, i, false);
+			if (ret)
+				return ret;
 		}
 	}
 
-out:
-	if (err) {
-		DRM_ERROR("sdma_v4_0: Failed to load firmware \"%s\"\n", fw_name);
-		sdma_v4_0_destroy_inst_ctx(adev);
-	}
-	return err;
+	return ret;
 }
 
 /**
@@ -2000,14 +1927,17 @@ static int sdma_v4_0_sw_fini(void *handle)
 			amdgpu_ring_fini(&adev->sdma.instance[i].page);
 	}
 
-	sdma_v4_0_destroy_inst_ctx(adev);
+	if (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 2, 0) ||
+            adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(4, 4, 0))
+		amdgpu_sdma_destroy_inst_ctx(adev, true);
+	else
+		amdgpu_sdma_destroy_inst_ctx(adev, false);
 
 	return 0;
 }
 
 static int sdma_v4_0_hw_init(void *handle)
 {
-	int r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
 	if (adev->flags & AMD_IS_APU)
@@ -2016,9 +1946,7 @@ static int sdma_v4_0_hw_init(void *handle)
 	if (!amdgpu_sriov_vf(adev))
 		sdma_v4_0_init_golden_registers(adev);
 
-	r = sdma_v4_0_start(adev);
-
-	return r;
+	return sdma_v4_0_start(adev);
 }
 
 static int sdma_v4_0_hw_fini(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
index a019ac92edb7..c05c3eebde4c 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
@@ -240,10 +240,7 @@ static int sdma_v5_0_init_microcode(struct amdgpu_device *adev)
 {
 	const char *chip_name;
 	char fw_name[40];
-	int err = 0, i;
-	struct amdgpu_firmware_info *info = NULL;
-	const struct common_firmware_header *header = NULL;
-	const struct sdma_firmware_header_v1_0 *hdr;
+	int ret, i;
 
 	if (amdgpu_sriov_vf(adev) && (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(5, 0, 5)))
 		return 0;
@@ -272,38 +269,12 @@ static int sdma_v5_0_init_microcode(struct amdgpu_device *adev)
 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma.bin", chip_name);
 		else
 			snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sdma1.bin", chip_name);
-		err = request_firmware(&adev->sdma.instance[i].fw, fw_name, adev->dev);
-		if (err)
-			goto out;
-		err = amdgpu_ucode_validate(adev->sdma.instance[i].fw);
-		if (err)
-			goto out;
-		hdr = (const struct sdma_firmware_header_v1_0 *)adev->sdma.instance[i].fw->data;
-		adev->sdma.instance[i].fw_version = le32_to_cpu(hdr->header.ucode_version);
-		adev->sdma.instance[i].feature_version = le32_to_cpu(hdr->ucode_feature_version);
-		if (adev->sdma.instance[i].feature_version >= 20)
-			adev->sdma.instance[i].burst_nop = true;
-		DRM_DEBUG("psp_load == '%s'\n",
-				adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
-
-		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
-			info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
-			info->fw = adev->sdma.instance[i].fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-		}
-	}
-out:
-	if (err) {
-		DRM_ERROR("sdma_v5_0: Failed to load firmware \"%s\"\n", fw_name);
-		for (i = 0; i < adev->sdma.num_instances; i++) {
-			release_firmware(adev->sdma.instance[i].fw);
-			adev->sdma.instance[i].fw = NULL;
-		}
+		ret = amdgpu_sdma_init_microcode(adev, fw_name, i, false);
+		if (ret)
+			return ret;
 	}
-	return err;
+	
+	return ret;
 }
 
 static unsigned sdma_v5_0_ring_init_cond_exec(struct amdgpu_ring *ring)
@@ -1465,12 +1436,10 @@ static int sdma_v5_0_sw_fini(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int i;
 
-	for (i = 0; i < adev->sdma.num_instances; i++) {
-		release_firmware(adev->sdma.instance[i].fw);
-		adev->sdma.instance[i].fw = NULL;
-
+	for (i = 0; i < adev->sdma.num_instances; i++)
 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
-	}
+
+	amdgpu_sdma_destroy_inst_ctx(adev, false);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
index 83c6ccaaa9e4..f136fec7b4f4 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
@@ -89,33 +89,6 @@ static u32 sdma_v5_2_get_reg_offset(struct amdgpu_device *adev, u32 instance, u3
 	return base + internal_offset;
 }
 
-static int sdma_v5_2_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
-{
-	int err = 0;
-	const struct sdma_firmware_header_v1_0 *hdr;
-
-	err = amdgpu_ucode_validate(sdma_inst->fw);
-	if (err)
-		return err;
-
-	hdr = (const struct sdma_firmware_header_v1_0 *)sdma_inst->fw->data;
-	sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
-	sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
-
-	if (sdma_inst->feature_version >= 20)
-		sdma_inst->burst_nop = true;
-
-	return 0;
-}
-
-static void sdma_v5_2_destroy_inst_ctx(struct amdgpu_device *adev)
-{
-	release_firmware(adev->sdma.instance[0].fw);
-
-	memset((void *)adev->sdma.instance, 0,
-	       sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
-}
-
 /**
  * sdma_v5_2_init_microcode - load ucode images from disk
  *
@@ -132,9 +105,6 @@ static int sdma_v5_2_init_microcode(struct amdgpu_device *adev)
 {
 	const char *chip_name;
 	char fw_name[40];
-	int err = 0, i;
-	struct amdgpu_firmware_info *info = NULL;
-	const struct common_firmware_header *header = NULL;
 
 	DRM_DEBUG("\n");
 
@@ -169,42 +139,7 @@ static int sdma_v5_2_init_microcode(struct amdgpu_device *adev)
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s.bin", chip_name);
 
-	err = request_firmware(&adev->sdma.instance[0].fw, fw_name, adev->dev);
-	if (err)
-		goto out;
-
-	err = sdma_v5_2_init_inst_ctx(&adev->sdma.instance[0]);
-	if (err)
-		goto out;
-
-	for (i = 1; i < adev->sdma.num_instances; i++)
-		memcpy((void *)&adev->sdma.instance[i],
-		       (void *)&adev->sdma.instance[0],
-		       sizeof(struct amdgpu_sdma_instance));
-
-	if (amdgpu_sriov_vf(adev) && (adev->ip_versions[SDMA0_HWIP][0] == IP_VERSION(5, 2, 0)))
-		return 0;
-
-	DRM_DEBUG("psp_load == '%s'\n",
-		  adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
-
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		for (i = 0; i < adev->sdma.num_instances; i++) {
-			info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA0 + i];
-			info->ucode_id = AMDGPU_UCODE_ID_SDMA0 + i;
-			info->fw = adev->sdma.instance[i].fw;
-			header = (const struct common_firmware_header *)info->fw->data;
-			adev->firmware.fw_size +=
-				ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
-		}
-	}
-
-out:
-	if (err) {
-		DRM_ERROR("sdma_v5_2: Failed to load firmware \"%s\"\n", fw_name);
-		sdma_v5_2_destroy_inst_ctx(adev);
-	}
-	return err;
+	return amdgpu_sdma_init_microcode(adev, fw_name, 0, true);
 }
 
 static unsigned sdma_v5_2_ring_init_cond_exec(struct amdgpu_ring *ring)
@@ -1406,19 +1341,16 @@ static int sdma_v5_2_sw_fini(void *handle)
 	for (i = 0; i < adev->sdma.num_instances; i++)
 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
 
-	sdma_v5_2_destroy_inst_ctx(adev);
+	amdgpu_sdma_destroy_inst_ctx(adev, true);
 
 	return 0;
 }
 
 static int sdma_v5_2_hw_init(void *handle)
 {
-	int r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	r = sdma_v5_2_start(adev);
-
-	return r;
+	return sdma_v5_2_start(adev);
 }
 
 static int sdma_v5_2_hw_fini(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index 0200cb3a31a4..db51230163c5 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -47,6 +47,7 @@
 MODULE_FIRMWARE("amdgpu/sdma_6_0_0.bin");
 MODULE_FIRMWARE("amdgpu/sdma_6_0_1.bin");
 MODULE_FIRMWARE("amdgpu/sdma_6_0_2.bin");
+MODULE_FIRMWARE("amdgpu/sdma_6_0_3.bin");
 
 #define SDMA1_REG_OFFSET 0x600
 #define SDMA0_HYP_DEC_REG_START 0x5880
@@ -77,33 +78,6 @@ static u32 sdma_v6_0_get_reg_offset(struct amdgpu_device *adev, u32 instance, u3
 	return base + internal_offset;
 }
 
-static int sdma_v6_0_init_inst_ctx(struct amdgpu_sdma_instance *sdma_inst)
-{
-	int err = 0;
-	const struct sdma_firmware_header_v2_0 *hdr;
-
-	err = amdgpu_ucode_validate(sdma_inst->fw);
-	if (err)
-		return err;
-
-	hdr = (const struct sdma_firmware_header_v2_0 *)sdma_inst->fw->data;
-	sdma_inst->fw_version = le32_to_cpu(hdr->header.ucode_version);
-	sdma_inst->feature_version = le32_to_cpu(hdr->ucode_feature_version);
-
-	if (sdma_inst->feature_version >= 20)
-		sdma_inst->burst_nop = true;
-
-	return 0;
-}
-
-static void sdma_v6_0_destroy_inst_ctx(struct amdgpu_device *adev)
-{
-	release_firmware(adev->sdma.instance[0].fw);
-
-	memset((void*)adev->sdma.instance, 0,
-	       sizeof(struct amdgpu_sdma_instance) * AMDGPU_MAX_SDMA_INSTANCES);
-}
-
 /**
  * sdma_v6_0_init_microcode - load ucode images from disk
  *
@@ -113,16 +87,10 @@ static void sdma_v6_0_destroy_inst_ctx(struct amdgpu_device *adev)
  * the driver (not loaded into hw).
  * Returns 0 on success, error on failure.
  */
-
-// emulation only, won't work on real chip
-// sdma 6.0.0 real chip need to use PSP to load firmware
 static int sdma_v6_0_init_microcode(struct amdgpu_device *adev)
 {
 	char fw_name[30];
 	char ucode_prefix[30];
-	int err = 0, i;
-	struct amdgpu_firmware_info *info = NULL;
-	const struct sdma_firmware_header_v2_0 *sdma_hdr;
 
 	DRM_DEBUG("\n");
 
@@ -130,43 +98,7 @@ static int sdma_v6_0_init_microcode(struct amdgpu_device *adev)
 
 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s.bin", ucode_prefix);
 
-	err = request_firmware(&adev->sdma.instance[0].fw, fw_name, adev->dev);
-	if (err)
-		goto out;
-
-	err = sdma_v6_0_init_inst_ctx(&adev->sdma.instance[0]);
-	if (err)
-		goto out;
-
-	for (i = 1; i < adev->sdma.num_instances; i++) {
-		memcpy((void*)&adev->sdma.instance[i],
-		       (void*)&adev->sdma.instance[0],
-		       sizeof(struct amdgpu_sdma_instance));
-	}
-
-	DRM_DEBUG("psp_load == '%s'\n",
-		  adev->firmware.load_type == AMDGPU_FW_LOAD_PSP ? "true" : "false");
-
-	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
-		sdma_hdr = (const struct sdma_firmware_header_v2_0 *)adev->sdma.instance[0].fw->data;
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH0];
-		info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH0;
-		info->fw = adev->sdma.instance[0].fw;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(sdma_hdr->ctx_ucode_size_bytes), PAGE_SIZE);
-		info = &adev->firmware.ucode[AMDGPU_UCODE_ID_SDMA_UCODE_TH1];
-		info->ucode_id = AMDGPU_UCODE_ID_SDMA_UCODE_TH1;
-		info->fw = adev->sdma.instance[0].fw;
-		adev->firmware.fw_size +=
-			ALIGN(le32_to_cpu(sdma_hdr->ctl_ucode_size_bytes), PAGE_SIZE);
-	}
-
-out:
-	if (err) {
-		DRM_ERROR("sdma_v6_0: Failed to load firmware \"%s\"\n", fw_name);
-		sdma_v6_0_destroy_inst_ctx(adev);
-	}
-	return err;
+	return amdgpu_sdma_init_microcode(adev, fw_name, 0, true);
 }
 
 static unsigned sdma_v6_0_ring_init_cond_exec(struct amdgpu_ring *ring)
@@ -559,7 +491,8 @@ static int sdma_v6_0_gfx_resume(struct amdgpu_device *adev)
 	for (i = 0; i < adev->sdma.num_instances; i++) {
 		ring = &adev->sdma.instance[i].ring;
 
-		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0);
+		if (!amdgpu_sriov_vf(adev))
+			WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_SEM_WAIT_FAIL_TIMER_CNTL), 0);
 
 		/* Set ring buffer size in dwords */
 		rb_bufsz = order_base_2(ring->ring_size / 4);
@@ -593,7 +526,10 @@ static int sdma_v6_0_gfx_resume(struct amdgpu_device *adev)
 		       lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
 
 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
-		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 0);
+		if (amdgpu_sriov_vf(adev))
+			rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 1);
+		else
+			rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, WPTR_POLL_ENABLE, 0);
 		rb_cntl = REG_SET_FIELD(rb_cntl, SDMA0_QUEUE0_RB_CNTL, F32_WPTR_POLL_ENABLE, 1);
 
 		WREG32_SOC15_IP(GC, sdma_v6_0_get_reg_offset(adev, i, regSDMA0_QUEUE0_RB_BASE), ring->gpu_addr >> 8);
@@ -1365,19 +1301,16 @@ static int sdma_v6_0_sw_fini(void *handle)
 	for (i = 0; i < adev->sdma.num_instances; i++)
 		amdgpu_ring_fini(&adev->sdma.instance[i].ring);
 
-	sdma_v6_0_destroy_inst_ctx(adev);
+	amdgpu_sdma_destroy_inst_ctx(adev, true);
 
 	return 0;
 }
 
 static int sdma_v6_0_hw_init(void *handle)
 {
-	int r;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
-	r = sdma_v6_0_start(adev);
-
-	return r;
+	return sdma_v6_0_start(adev);
 }
 
 static int sdma_v6_0_hw_fini(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c b/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c
new file mode 100644
index 000000000000..7aa570c1ce4a
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.c
@@ -0,0 +1,303 @@
+/*
+ * Copyright 2021 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#include "sienna_cichlid.h"
+#include "amdgpu_reset.h"
+#include "amdgpu_amdkfd.h"
+#include "amdgpu_dpm.h"
+#include "amdgpu_job.h"
+#include "amdgpu_ring.h"
+#include "amdgpu_ras.h"
+#include "amdgpu_psp.h"
+#include "amdgpu_xgmi.h"
+
+static struct amdgpu_reset_handler *
+sienna_cichlid_get_reset_handler(struct amdgpu_reset_control *reset_ctl,
+			    struct amdgpu_reset_context *reset_context)
+{
+	struct amdgpu_reset_handler *handler;
+	struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle;
+
+	if (reset_context->method != AMD_RESET_METHOD_NONE) {
+		list_for_each_entry(handler, &reset_ctl->reset_handlers,
+				     handler_list) {
+			if (handler->reset_method == reset_context->method)
+				return handler;
+		}
+	} else {
+		list_for_each_entry(handler, &reset_ctl->reset_handlers,
+				     handler_list) {
+			if (handler->reset_method == AMD_RESET_METHOD_MODE2 &&
+			    adev->pm.fw_version >= 0x3a5500 &&
+			    !amdgpu_sriov_vf(adev)) {
+				reset_context->method = AMD_RESET_METHOD_MODE2;
+				return handler;
+			}
+		}
+	}
+
+	return NULL;
+}
+
+static int sienna_cichlid_mode2_suspend_ip(struct amdgpu_device *adev)
+{
+	int r, i;
+
+	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
+	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
+
+	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
+		if (!(adev->ip_blocks[i].version->type ==
+			      AMD_IP_BLOCK_TYPE_GFX ||
+		      adev->ip_blocks[i].version->type ==
+			      AMD_IP_BLOCK_TYPE_SDMA))
+			continue;
+
+		r = adev->ip_blocks[i].version->funcs->suspend(adev);
+
+		if (r) {
+			dev_err(adev->dev,
+				"suspend of IP block <%s> failed %d\n",
+				adev->ip_blocks[i].version->funcs->name, r);
+			return r;
+		}
+		adev->ip_blocks[i].status.hw = false;
+	}
+
+	return r;
+}
+
+static int
+sienna_cichlid_mode2_prepare_hwcontext(struct amdgpu_reset_control *reset_ctl,
+				  struct amdgpu_reset_context *reset_context)
+{
+	int r = 0;
+	struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle;
+
+	if (!amdgpu_sriov_vf(adev)) {
+		if (adev->gfxhub.funcs->mode2_save_regs)
+			adev->gfxhub.funcs->mode2_save_regs(adev);
+		if (adev->gfxhub.funcs->halt)
+			adev->gfxhub.funcs->halt(adev);
+		r = sienna_cichlid_mode2_suspend_ip(adev);
+	}
+
+	return r;
+}
+
+static void sienna_cichlid_async_reset(struct work_struct *work)
+{
+	struct amdgpu_reset_handler *handler;
+	struct amdgpu_reset_control *reset_ctl =
+		container_of(work, struct amdgpu_reset_control, reset_work);
+	struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle;
+
+	list_for_each_entry(handler, &reset_ctl->reset_handlers,
+			     handler_list) {
+		if (handler->reset_method == reset_ctl->active_reset) {
+			dev_dbg(adev->dev, "Resetting device\n");
+			handler->do_reset(adev);
+			break;
+		}
+	}
+}
+
+static int sienna_cichlid_mode2_reset(struct amdgpu_device *adev)
+{
+	/* disable BM */
+	pci_clear_master(adev->pdev);
+	return amdgpu_dpm_mode2_reset(adev);
+}
+
+static int
+sienna_cichlid_mode2_perform_reset(struct amdgpu_reset_control *reset_ctl,
+			      struct amdgpu_reset_context *reset_context)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)reset_ctl->handle;
+	int r;
+
+	r = sienna_cichlid_mode2_reset(adev);
+	if (r) {
+		dev_err(adev->dev,
+			"ASIC reset failed with error, %d ", r);
+	}
+	return r;
+}
+
+static int sienna_cichlid_mode2_restore_ip(struct amdgpu_device *adev)
+{
+	int i, r;
+	struct psp_context *psp = &adev->psp;
+
+	r = psp_rlc_autoload_start(psp);
+	if (r) {
+		dev_err(adev->dev, "Failed to start rlc autoload\n");
+		return r;
+	}
+
+	/* Reinit GFXHUB */
+	if (adev->gfxhub.funcs->mode2_restore_regs)
+		adev->gfxhub.funcs->mode2_restore_regs(adev);
+	adev->gfxhub.funcs->init(adev);
+	r = adev->gfxhub.funcs->gart_enable(adev);
+	if (r) {
+		dev_err(adev->dev, "GFXHUB gart reenable failed after reset\n");
+		return r;
+	}
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
+			r = adev->ip_blocks[i].version->funcs->resume(adev);
+			if (r) {
+				dev_err(adev->dev,
+					"resume of IP block <%s> failed %d\n",
+					adev->ip_blocks[i].version->funcs->name, r);
+				return r;
+			}
+
+			adev->ip_blocks[i].status.hw = true;
+		}
+	}
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!(adev->ip_blocks[i].version->type ==
+			      AMD_IP_BLOCK_TYPE_GFX ||
+		      adev->ip_blocks[i].version->type ==
+			      AMD_IP_BLOCK_TYPE_SDMA))
+			continue;
+		r = adev->ip_blocks[i].version->funcs->resume(adev);
+		if (r) {
+			dev_err(adev->dev,
+				"resume of IP block <%s> failed %d\n",
+				adev->ip_blocks[i].version->funcs->name, r);
+			return r;
+		}
+
+		adev->ip_blocks[i].status.hw = true;
+	}
+
+	for (i = 0; i < adev->num_ip_blocks; i++) {
+		if (!(adev->ip_blocks[i].version->type ==
+			      AMD_IP_BLOCK_TYPE_GFX ||
+		      adev->ip_blocks[i].version->type ==
+			      AMD_IP_BLOCK_TYPE_SDMA))
+			continue;
+
+		if (adev->ip_blocks[i].version->funcs->late_init) {
+			r = adev->ip_blocks[i].version->funcs->late_init(
+				(void *)adev);
+			if (r) {
+				dev_err(adev->dev,
+					"late_init of IP block <%s> failed %d after reset\n",
+					adev->ip_blocks[i].version->funcs->name,
+					r);
+				return r;
+			}
+		}
+		adev->ip_blocks[i].status.late_initialized = true;
+	}
+
+	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
+	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
+
+	return r;
+}
+
+static int
+sienna_cichlid_mode2_restore_hwcontext(struct amdgpu_reset_control *reset_ctl,
+				  struct amdgpu_reset_context *reset_context)
+{
+	int r;
+	struct amdgpu_device *tmp_adev = (struct amdgpu_device *)reset_ctl->handle;
+
+	dev_info(tmp_adev->dev,
+			"GPU reset succeeded, trying to resume\n");
+	r = sienna_cichlid_mode2_restore_ip(tmp_adev);
+	if (r)
+		goto end;
+
+	/*
+	* Add this ASIC as tracked as reset was already
+	* complete successfully.
+	*/
+	amdgpu_register_gpu_instance(tmp_adev);
+
+	/* Resume RAS */
+	amdgpu_ras_resume(tmp_adev);
+
+	amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
+
+	r = amdgpu_ib_ring_tests(tmp_adev);
+	if (r) {
+		dev_err(tmp_adev->dev,
+			"ib ring test failed (%d).\n", r);
+		r = -EAGAIN;
+		goto end;
+	}
+
+end:
+	if (r)
+		return -EAGAIN;
+	else
+		return r;
+}
+
+static struct amdgpu_reset_handler sienna_cichlid_mode2_handler = {
+	.reset_method		= AMD_RESET_METHOD_MODE2,
+	.prepare_env		= NULL,
+	.prepare_hwcontext	= sienna_cichlid_mode2_prepare_hwcontext,
+	.perform_reset		= sienna_cichlid_mode2_perform_reset,
+	.restore_hwcontext	= sienna_cichlid_mode2_restore_hwcontext,
+	.restore_env		= NULL,
+	.do_reset		= sienna_cichlid_mode2_reset,
+};
+
+int sienna_cichlid_reset_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_reset_control *reset_ctl;
+
+	reset_ctl = kzalloc(sizeof(*reset_ctl), GFP_KERNEL);
+	if (!reset_ctl)
+		return -ENOMEM;
+
+	reset_ctl->handle = adev;
+	reset_ctl->async_reset = sienna_cichlid_async_reset;
+	reset_ctl->active_reset = AMD_RESET_METHOD_NONE;
+	reset_ctl->get_reset_handler = sienna_cichlid_get_reset_handler;
+
+	INIT_LIST_HEAD(&reset_ctl->reset_handlers);
+	INIT_WORK(&reset_ctl->reset_work, reset_ctl->async_reset);
+	/* Only mode2 is handled through reset control now */
+	amdgpu_reset_add_handler(reset_ctl, &sienna_cichlid_mode2_handler);
+
+	adev->reset_cntl = reset_ctl;
+
+	return 0;
+}
+
+int sienna_cichlid_reset_fini(struct amdgpu_device *adev)
+{
+	kfree(adev->reset_cntl);
+	adev->reset_cntl = NULL;
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dml_wrapper.h b/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.h
index 5dcfbd8e2697..5213b162dacd 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dml_wrapper.h
+++ b/drivers/gpu/drm/amd/amdgpu/sienna_cichlid.h
@@ -1,5 +1,5 @@
 /*
- * Copyright 2015 Advanced Micro Devices, Inc.
+ * Copyright 2021 Advanced Micro Devices, Inc.
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -19,16 +19,14 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
  *
- * Authors: AMD
- *
  */
 
-#ifndef DML_WRAPPER_H_
-#define DML_WRAPPER_H_
+#ifndef __SIENNA_CICHLID_H__
+#define __SIENNA_CICHLID_H__
 
-#include "dc.h"
-#include "dml/display_mode_vba.h"
+#include "amdgpu.h"
 
-bool dml_validate(struct dc *dc, struct dc_state *context, bool fast_validate);
+int sienna_cichlid_reset_init(struct amdgpu_device *adev);
+int sienna_cichlid_reset_fini(struct amdgpu_device *adev);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c
index 2e50db3b761e..16b757664a35 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc21.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc21.c
@@ -179,7 +179,7 @@ void soc21_grbm_select(struct amdgpu_device *adev,
 	grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, VMID, vmid);
 	grbm_gfx_cntl = REG_SET_FIELD(grbm_gfx_cntl, GRBM_GFX_CNTL, QUEUEID, queue);
 
-	WREG32(SOC15_REG_OFFSET(GC, 0, regGRBM_GFX_CNTL), grbm_gfx_cntl);
+	WREG32_SOC15(GC, 0, regGRBM_GFX_CNTL, grbm_gfx_cntl);
 }
 
 static void soc21_vga_set_state(struct amdgpu_device *adev, bool state)
@@ -583,6 +583,10 @@ static int soc21_common_early_init(void *handle)
 			AMD_PG_SUPPORT_JPEG |
 			AMD_PG_SUPPORT_ATHUB |
 			AMD_PG_SUPPORT_MMHUB;
+		if (amdgpu_sriov_vf(adev)) {
+			adev->cg_flags = 0;
+			adev->pg_flags = 0;
+		}
 		adev->external_rev_id = adev->rev_id + 0x1; // TODO: need update
 		break;
 	case IP_VERSION(11, 0, 2):
@@ -625,9 +629,23 @@ static int soc21_common_early_init(void *handle)
 			AMD_CG_SUPPORT_JPEG_MGCG;
 		adev->pg_flags =
 			AMD_PG_SUPPORT_GFX_PG |
+			AMD_PG_SUPPORT_VCN_DPG |
 			AMD_PG_SUPPORT_JPEG;
 		adev->external_rev_id = adev->rev_id + 0x1;
 		break;
+	case IP_VERSION(11, 0, 3):
+		adev->cg_flags = AMD_CG_SUPPORT_VCN_MGCG |
+			AMD_CG_SUPPORT_JPEG_MGCG;
+		adev->pg_flags = AMD_PG_SUPPORT_VCN |
+			AMD_PG_SUPPORT_VCN_DPG |
+			AMD_PG_SUPPORT_JPEG;
+		if (amdgpu_sriov_vf(adev)) {
+			/* hypervisor control CG and PG enablement */
+			adev->cg_flags = 0;
+			adev->pg_flags = 0;
+		}
+		adev->external_rev_id = adev->rev_id + 0x20;
+		break;
 	default:
 		/* FIXME: not supported yet */
 		return -EINVAL;
diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c
index bf7524f16b66..a0d19b768346 100644
--- a/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c
+++ b/drivers/gpu/drm/amd/amdgpu/umc_v6_7.c
@@ -452,41 +452,47 @@ static void umc_v6_7_query_ras_error_count(struct amdgpu_device *adev,
 
 static void umc_v6_7_query_error_address(struct amdgpu_device *adev,
 					 struct ras_err_data *err_data,
-					 uint32_t umc_reg_offset,
-					 uint32_t ch_inst,
-					 uint32_t umc_inst)
+					 uint32_t umc_reg_offset, uint32_t ch_inst,
+					 uint32_t umc_inst, uint64_t mca_addr)
 {
 	uint32_t mc_umc_status_addr;
 	uint32_t channel_index;
-	uint64_t mc_umc_status, mc_umc_addrt0;
+	uint64_t mc_umc_status = 0, mc_umc_addrt0;
 	uint64_t err_addr, soc_pa, retired_page, column;
 
-	mc_umc_status_addr =
-		SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
-	mc_umc_addrt0 =
-		SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_ADDRT0);
+	if (mca_addr == UMC_INVALID_ADDR) {
+		mc_umc_status_addr =
+			SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
+		mc_umc_addrt0 =
+			SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_ADDRT0);
 
-	mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4);
+		mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4);
 
-	if (mc_umc_status == 0)
-		return;
+		if (mc_umc_status == 0)
+			return;
 
-	if (!err_data->err_addr) {
-		/* clear umc status */
-		WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
-		return;
+		if (!err_data->err_addr) {
+			/* clear umc status */
+			WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
+			return;
+		}
 	}
 
 	channel_index =
 		adev->umc.channel_idx_tbl[umc_inst * adev->umc.channel_inst_num + ch_inst];
 
 	/* calculate error address if ue/ce error is detected */
-	if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
+	if ((REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
 	    (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC) == 1 ||
-	    REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) {
-
-		err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4);
-		err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
+	    REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)) ||
+	    mca_addr != UMC_INVALID_ADDR) {
+		if (mca_addr == UMC_INVALID_ADDR) {
+			err_addr = RREG64_PCIE((mc_umc_addrt0 + umc_reg_offset) * 4);
+			err_addr =
+				REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr);
+		} else {
+			err_addr = mca_addr;
+		}
 
 		/* translate umc channel address to soc pa, 3 parts are included */
 		soc_pa = ADDR_OF_8KB_BLOCK(err_addr) |
@@ -501,7 +507,8 @@ static void umc_v6_7_query_error_address(struct amdgpu_device *adev,
 
 		/* we only save ue error information currently, ce is skipped */
 		if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, UECC)
-				== 1) {
+				== 1 ||
+		    mca_addr != UMC_INVALID_ADDR) {
 			/* loop for all possibilities of [C4 C3 C2] */
 			for (column = 0; column < UMC_V6_7_NA_MAP_PA_NUM; column++) {
 				retired_page = soc_pa | (column << UMC_V6_7_PA_C2_BIT);
@@ -519,7 +526,8 @@ static void umc_v6_7_query_error_address(struct amdgpu_device *adev,
 	}
 
 	/* clear umc status */
-	WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
+	if (mca_addr == UMC_INVALID_ADDR)
+		WREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL);
 }
 
 static void umc_v6_7_query_ras_error_address(struct amdgpu_device *adev,
@@ -540,9 +548,8 @@ static void umc_v6_7_query_ras_error_address(struct amdgpu_device *adev,
 							 ch_inst);
 		umc_v6_7_query_error_address(adev,
 					     err_data,
-					     umc_reg_offset,
-					     ch_inst,
-					     umc_inst);
+					     umc_reg_offset, ch_inst,
+					     umc_inst, UMC_INVALID_ADDR);
 	}
 }
 
@@ -583,4 +590,5 @@ struct amdgpu_umc_ras umc_v6_7_ras = {
 	.query_ras_poison_mode = umc_v6_7_query_ras_poison_mode,
 	.ecc_info_query_ras_error_count = umc_v6_7_ecc_info_query_ras_error_count,
 	.ecc_info_query_ras_error_address = umc_v6_7_ecc_info_query_ras_error_address,
+	.convert_ras_error_address = umc_v6_7_query_error_address,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c b/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c
index 36a2053f2e8b..a8cbda81828d 100644
--- a/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c
+++ b/drivers/gpu/drm/amd/amdgpu/umc_v8_10.c
@@ -101,22 +101,16 @@ static void umc_v8_10_query_correctable_error_count(struct amdgpu_device *adev,
 						   uint32_t umc_reg_offset,
 						   unsigned long *error_count)
 {
-	uint32_t ecc_err_cnt, ecc_err_cnt_addr;
 	uint64_t mc_umc_status;
 	uint32_t mc_umc_status_addr;
 
 	/* UMC 8_10 registers */
-	ecc_err_cnt_addr =
-		SOC15_REG_OFFSET(UMC, 0, regUMCCH0_0_GeccErrCnt);
 	mc_umc_status_addr =
 		SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0);
 
-	ecc_err_cnt = RREG32_PCIE((ecc_err_cnt_addr + umc_reg_offset) * 4);
-	*error_count +=
-		(REG_GET_FIELD(ecc_err_cnt, UMCCH0_0_GeccErrCnt, GeccErrCnt) -
-		 UMC_V8_10_CE_CNT_INIT);
-
-	/* Check for SRAM correctable error, MCUMC_STATUS is a 64 bit register */
+	/* Rely on MCUMC_STATUS for correctable error counter
+	 * MCUMC_STATUS is a 64 bit register
+	 */
 	mc_umc_status = RREG64_PCIE((mc_umc_status_addr + umc_reg_offset) * 4);
 	if (REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, Val) == 1 &&
 	    REG_GET_FIELD(mc_umc_status, MCA_UMC_UMC0_MCUMC_STATUST0, CECC) == 1)
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
index 39405f0db824..9c8b5fd99037 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
@@ -1761,21 +1761,23 @@ static const struct amdgpu_ring_funcs vcn_v3_0_dec_sw_ring_vm_funcs = {
 	.emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper,
 };
 
-static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p)
+static int vcn_v3_0_limit_sched(struct amdgpu_cs_parser *p,
+				struct amdgpu_job *job)
 {
 	struct drm_gpu_scheduler **scheds;
 
 	/* The create msg must be in the first IB submitted */
-	if (atomic_read(&p->entity->fence_seq))
+	if (atomic_read(&job->base.entity->fence_seq))
 		return -EINVAL;
 
 	scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_DEC]
 		[AMDGPU_RING_PRIO_DEFAULT].sched;
-	drm_sched_entity_modify_sched(p->entity, scheds, 1);
+	drm_sched_entity_modify_sched(job->base.entity, scheds, 1);
 	return 0;
 }
 
-static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr)
+static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
+			    uint64_t addr)
 {
 	struct ttm_operation_ctx ctx = { false, false };
 	struct amdgpu_bo_va_mapping *map;
@@ -1846,7 +1848,7 @@ static int vcn_v3_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr)
 		if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
 			continue;
 
-		r = vcn_v3_0_limit_sched(p);
+		r = vcn_v3_0_limit_sched(p, job);
 		if (r)
 			goto out;
 	}
@@ -1860,7 +1862,7 @@ static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
 					   struct amdgpu_job *job,
 					   struct amdgpu_ib *ib)
 {
-	struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched);
+	struct amdgpu_ring *ring = amdgpu_job_ring(job);
 	uint32_t msg_lo = 0, msg_hi = 0;
 	unsigned i;
 	int r;
@@ -1879,7 +1881,8 @@ static int vcn_v3_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
 			msg_hi = val;
 		} else if (reg == PACKET0(p->adev->vcn.internal.cmd, 0) &&
 			   val == 0) {
-			r = vcn_v3_0_dec_msg(p, ((u64)msg_hi) << 32 | msg_lo);
+			r = vcn_v3_0_dec_msg(p, job,
+					     ((u64)msg_hi) << 32 | msg_lo);
 			if (r)
 				return r;
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
index fb2d74f30448..897a5ce9c9da 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
@@ -30,6 +30,7 @@
 #include "soc15d.h"
 #include "soc15_hw_ip.h"
 #include "vcn_v2_0.h"
+#include "mmsch_v4_0.h"
 
 #include "vcn/vcn_4_0_0_offset.h"
 #include "vcn/vcn_4_0_0_sh_mask.h"
@@ -45,6 +46,8 @@
 #define VCN_VID_SOC_ADDRESS_2_0							0x1fb00
 #define VCN1_VID_SOC_ADDRESS_3_0						0x48300
 
+#define VCN_HARVEST_MMSCH								0
+
 #define RDECODE_MSG_CREATE							0x00000000
 #define RDECODE_MESSAGE_CREATE							0x00000001
 
@@ -53,12 +56,14 @@ static int amdgpu_ih_clientid_vcns[] = {
 	SOC15_IH_CLIENTID_VCN1
 };
 
+static int vcn_v4_0_start_sriov(struct amdgpu_device *adev);
 static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev);
 static void vcn_v4_0_set_irq_funcs(struct amdgpu_device *adev);
 static int vcn_v4_0_set_powergating_state(void *handle,
         enum amd_powergating_state state);
 static int vcn_v4_0_pause_dpg_mode(struct amdgpu_device *adev,
         int inst_idx, struct dpg_pause_state *new_state);
+static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring);
 
 /**
  * vcn_v4_0_early_init - set function pointers
@@ -71,6 +76,9 @@ static int vcn_v4_0_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
+	if (amdgpu_sriov_vf(adev))
+		adev->vcn.harvest_config = VCN_HARVEST_MMSCH;
+
 	/* re-use enc ring as unified ring */
 	adev->vcn.num_enc_rings = 1;
 
@@ -92,6 +100,7 @@ static int vcn_v4_0_sw_init(void *handle)
 	struct amdgpu_ring *ring;
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int i, r;
+	int vcn_doorbell_index = 0;
 
 	r = amdgpu_vcn_sw_init(adev);
 	if (r)
@@ -103,6 +112,12 @@ static int vcn_v4_0_sw_init(void *handle)
 	if (r)
 		return r;
 
+	if (amdgpu_sriov_vf(adev)) {
+		vcn_doorbell_index = adev->doorbell_index.vcn.vcn_ring0_1 - MMSCH_DOORBELL_OFFSET;
+		/* get DWORD offset */
+		vcn_doorbell_index = vcn_doorbell_index << 1;
+	}
+
 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 		volatile struct amdgpu_vcn4_fw_shared *fw_shared;
 
@@ -119,7 +134,10 @@ static int vcn_v4_0_sw_init(void *handle)
 
 		ring = &adev->vcn.inst[i].ring_enc[0];
 		ring->use_doorbell = true;
-		ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;
+		if (amdgpu_sriov_vf(adev))
+			ring->doorbell_index = vcn_doorbell_index + i * (adev->vcn.num_enc_rings + 1) + 1;
+		else
+			ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 2 + 8 * i;
 
 		sprintf(ring->name, "vcn_unified_%d", i);
 
@@ -132,10 +150,23 @@ static int vcn_v4_0_sw_init(void *handle)
 		fw_shared->present_flag_0 = cpu_to_le32(AMDGPU_FW_SHARED_FLAG_0_UNIFIED_QUEUE);
 		fw_shared->sq.is_enabled = 1;
 
+		fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_SMU_DPM_INTERFACE_FLAG);
+		fw_shared->smu_dpm_interface.smu_interface_type = (adev->flags & AMD_IS_APU) ?
+			AMDGPU_VCN_SMU_DPM_INTERFACE_APU : AMDGPU_VCN_SMU_DPM_INTERFACE_DGPU;
+
+		if (amdgpu_sriov_vf(adev))
+			fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
+
 		if (amdgpu_vcnfw_log)
 			amdgpu_vcn_fwlog_init(&adev->vcn.inst[i]);
 	}
 
+	if (amdgpu_sriov_vf(adev)) {
+		r = amdgpu_virt_alloc_mm_table(adev);
+		if (r)
+			return r;
+	}
+
 	if (adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG)
 		adev->vcn.pause_dpg_mode = vcn_v4_0_pause_dpg_mode;
 
@@ -169,6 +200,9 @@ static int vcn_v4_0_sw_fini(void *handle)
 		drm_dev_exit(idx);
 	}
 
+	if (amdgpu_sriov_vf(adev))
+		amdgpu_virt_free_mm_table(adev);
+
 	r = amdgpu_vcn_suspend(adev);
 	if (r)
 		return r;
@@ -191,18 +225,42 @@ static int vcn_v4_0_hw_init(void *handle)
 	struct amdgpu_ring *ring;
 	int i, r;
 
-	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
-		if (adev->vcn.harvest_config & (1 << i))
-			continue;
+	if (amdgpu_sriov_vf(adev)) {
+		r = vcn_v4_0_start_sriov(adev);
+		if (r)
+			goto done;
 
-		ring = &adev->vcn.inst[i].ring_enc[0];
+		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
+			if (adev->vcn.harvest_config & (1 << i))
+				continue;
 
-		adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
-				((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
+			ring = &adev->vcn.inst[i].ring_enc[0];
+			if (amdgpu_vcn_is_disabled_vcn(adev, VCN_ENCODE_RING, i)) {
+				ring->sched.ready = false;
+				ring->no_scheduler = true;
+				dev_info(adev->dev, "ring %s is disabled by hypervisor\n", ring->name);
+			} else {
+				ring->wptr = 0;
+				ring->wptr_old = 0;
+				vcn_v4_0_unified_ring_set_wptr(ring);
+				ring->sched.ready = true;
+			}
+		}
+	} else {
+		for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
+			if (adev->vcn.harvest_config & (1 << i))
+				continue;
 
-		r = amdgpu_ring_test_helper(ring);
-		if (r)
-			goto done;
+			ring = &adev->vcn.inst[i].ring_enc[0];
+
+			adev->nbio.funcs->vcn_doorbell_range(adev, ring->use_doorbell,
+					((adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 8 * i), i);
+
+			r = amdgpu_ring_test_helper(ring);
+			if (r)
+				goto done;
+
+		}
 	}
 
 done:
@@ -230,12 +288,14 @@ static int vcn_v4_0_hw_fini(void *handle)
 	for (i = 0; i < adev->vcn.num_vcn_inst; ++i) {
 		if (adev->vcn.harvest_config & (1 << i))
 			continue;
-
-		if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
+		if (!amdgpu_sriov_vf(adev)) {
+			if ((adev->pg_flags & AMD_PG_SUPPORT_VCN_DPG) ||
                         (adev->vcn.cur_state != AMD_PG_STATE_GATE &&
                                 RREG32_SOC15(VCN, i, regUVD_STATUS))) {
                         vcn_v4_0_set_powergating_state(adev, AMD_PG_STATE_GATE);
+			}
 		}
+
 	}
 
 	return 0;
@@ -1107,6 +1167,214 @@ static int vcn_v4_0_start(struct amdgpu_device *adev)
 	return 0;
 }
 
+static int vcn_v4_0_start_sriov(struct amdgpu_device *adev)
+{
+	int i;
+	struct amdgpu_ring *ring_enc;
+	uint64_t cache_addr;
+	uint64_t rb_enc_addr;
+	uint64_t ctx_addr;
+	uint32_t param, resp, expected;
+	uint32_t offset, cache_size;
+	uint32_t tmp, timeout;
+
+	struct amdgpu_mm_table *table = &adev->virt.mm_table;
+	uint32_t *table_loc;
+	uint32_t table_size;
+	uint32_t size, size_dw;
+	uint32_t init_status;
+	uint32_t enabled_vcn;
+
+	struct mmsch_v4_0_cmd_direct_write
+		direct_wt = { {0} };
+	struct mmsch_v4_0_cmd_direct_read_modify_write
+		direct_rd_mod_wt = { {0} };
+	struct mmsch_v4_0_cmd_end end = { {0} };
+	struct mmsch_v4_0_init_header header;
+
+	volatile struct amdgpu_vcn4_fw_shared *fw_shared;
+	volatile struct amdgpu_fw_shared_rb_setup *rb_setup;
+
+	direct_wt.cmd_header.command_type =
+		MMSCH_COMMAND__DIRECT_REG_WRITE;
+	direct_rd_mod_wt.cmd_header.command_type =
+		MMSCH_COMMAND__DIRECT_REG_READ_MODIFY_WRITE;
+	end.cmd_header.command_type =
+		MMSCH_COMMAND__END;
+
+	header.version = MMSCH_VERSION;
+	header.total_size = sizeof(struct mmsch_v4_0_init_header) >> 2;
+	for (i = 0; i < AMDGPU_MAX_VCN_INSTANCES; i++) {
+		header.inst[i].init_status = 0;
+		header.inst[i].table_offset = 0;
+		header.inst[i].table_size = 0;
+	}
+
+	table_loc = (uint32_t *)table->cpu_addr;
+	table_loc += header.total_size;
+	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
+		if (adev->vcn.harvest_config & (1 << i))
+			continue;
+
+		table_size = 0;
+
+		MMSCH_V4_0_INSERT_DIRECT_RD_MOD_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_STATUS),
+			~UVD_STATUS__UVD_BUSY, UVD_STATUS__UVD_BUSY);
+
+		cache_size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4);
+
+		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+			MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+				regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
+				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_lo);
+			MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+				regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
+				adev->firmware.ucode[AMDGPU_UCODE_ID_VCN + i].tmr_mc_addr_hi);
+			offset = 0;
+			MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+				regUVD_VCPU_CACHE_OFFSET0),
+				0);
+		} else {
+			MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+				regUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW),
+				lower_32_bits(adev->vcn.inst[i].gpu_addr));
+			MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+				regUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH),
+				upper_32_bits(adev->vcn.inst[i].gpu_addr));
+			offset = cache_size;
+			MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+				regUVD_VCPU_CACHE_OFFSET0),
+				AMDGPU_UVD_FIRMWARE_OFFSET >> 3);
+		}
+
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_VCPU_CACHE_SIZE0),
+			cache_size);
+
+		cache_addr = adev->vcn.inst[i].gpu_addr + offset;
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW),
+			lower_32_bits(cache_addr));
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH),
+			upper_32_bits(cache_addr));
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_VCPU_CACHE_OFFSET1),
+			0);
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_VCPU_CACHE_SIZE1),
+			AMDGPU_VCN_STACK_SIZE);
+
+		cache_addr = adev->vcn.inst[i].gpu_addr + offset +
+			AMDGPU_VCN_STACK_SIZE;
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW),
+			lower_32_bits(cache_addr));
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH),
+			upper_32_bits(cache_addr));
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_VCPU_CACHE_OFFSET2),
+			0);
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_VCPU_CACHE_SIZE2),
+			AMDGPU_VCN_CONTEXT_SIZE);
+
+		fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
+		rb_setup = &fw_shared->rb_setup;
+
+		ring_enc = &adev->vcn.inst[i].ring_enc[0];
+		ring_enc->wptr = 0;
+		rb_enc_addr = ring_enc->gpu_addr;
+
+		rb_setup->is_rb_enabled_flags |= RB_ENABLED;
+		rb_setup->rb_addr_lo = lower_32_bits(rb_enc_addr);
+		rb_setup->rb_addr_hi = upper_32_bits(rb_enc_addr);
+		rb_setup->rb_size = ring_enc->ring_size / 4;
+		fw_shared->present_flag_0 |= cpu_to_le32(AMDGPU_VCN_VF_RB_SETUP_FLAG);
+
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_LMI_VCPU_NC0_64BIT_BAR_LOW),
+			lower_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_LMI_VCPU_NC0_64BIT_BAR_HIGH),
+			upper_32_bits(adev->vcn.inst[i].fw_shared.gpu_addr));
+		MMSCH_V4_0_INSERT_DIRECT_WT(SOC15_REG_OFFSET(VCN, i,
+			regUVD_VCPU_NONCACHE_SIZE0),
+			AMDGPU_GPU_PAGE_ALIGN(sizeof(struct amdgpu_vcn4_fw_shared)));
+
+		/* add end packet */
+		MMSCH_V4_0_INSERT_END();
+
+		/* refine header */
+		header.inst[i].init_status = 0;
+		header.inst[i].table_offset = header.total_size;
+		header.inst[i].table_size = table_size;
+		header.total_size += table_size;
+	}
+
+	/* Update init table header in memory */
+	size = sizeof(struct mmsch_v4_0_init_header);
+	table_loc = (uint32_t *)table->cpu_addr;
+	memcpy((void *)table_loc, &header, size);
+
+	/* message MMSCH (in VCN[0]) to initialize this client
+	 * 1, write to mmsch_vf_ctx_addr_lo/hi register with GPU mc addr
+	 * of memory descriptor location
+	 */
+	ctx_addr = table->gpu_addr;
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_LO, lower_32_bits(ctx_addr));
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_ADDR_HI, upper_32_bits(ctx_addr));
+
+	/* 2, update vmid of descriptor */
+	tmp = RREG32_SOC15(VCN, 0, regMMSCH_VF_VMID);
+	tmp &= ~MMSCH_VF_VMID__VF_CTX_VMID_MASK;
+	/* use domain0 for MM scheduler */
+	tmp |= (0 << MMSCH_VF_VMID__VF_CTX_VMID__SHIFT);
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_VMID, tmp);
+
+	/* 3, notify mmsch about the size of this descriptor */
+	size = header.total_size;
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_CTX_SIZE, size);
+
+	/* 4, set resp to zero */
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP, 0);
+
+	/* 5, kick off the initialization and wait until
+	 * MMSCH_VF_MAILBOX_RESP becomes non-zero
+	 */
+	param = 0x00000001;
+	WREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_HOST, param);
+	tmp = 0;
+	timeout = 1000;
+	resp = 0;
+	expected = MMSCH_VF_MAILBOX_RESP__OK;
+	while (resp != expected) {
+		resp = RREG32_SOC15(VCN, 0, regMMSCH_VF_MAILBOX_RESP);
+		if (resp != 0)
+			break;
+
+		udelay(10);
+		tmp = tmp + 10;
+		if (tmp >= timeout) {
+			DRM_ERROR("failed to init MMSCH. TIME-OUT after %d usec"\
+				" waiting for regMMSCH_VF_MAILBOX_RESP "\
+				"(expected=0x%08x, readback=0x%08x)\n",
+				tmp, expected, resp);
+			return -EBUSY;
+		}
+	}
+	enabled_vcn = amdgpu_vcn_is_disabled_vcn(adev, VCN_DECODE_RING, 0) ? 1 : 0;
+	init_status = ((struct mmsch_v4_0_init_header *)(table_loc))->inst[enabled_vcn].init_status;
+	if (resp != expected && resp != MMSCH_VF_MAILBOX_RESP__INCOMPLETE
+	&& init_status != MMSCH_VF_ENGINE_STATUS__PASS)
+		DRM_ERROR("MMSCH init status is incorrect! readback=0x%08x, header init "\
+			"status for VCN%x: 0x%x\n", resp, enabled_vcn, init_status);
+
+	return 0;
+}
+
 /**
  * vcn_v4_0_stop_dpg_mode - VCN stop with dpg mode
  *
@@ -1327,21 +1595,23 @@ static void vcn_v4_0_unified_ring_set_wptr(struct amdgpu_ring *ring)
 	}
 }
 
-static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p)
+static int vcn_v4_0_limit_sched(struct amdgpu_cs_parser *p,
+				struct amdgpu_job *job)
 {
 	struct drm_gpu_scheduler **scheds;
 
 	/* The create msg must be in the first IB submitted */
-	if (atomic_read(&p->entity->fence_seq))
+	if (atomic_read(&job->base.entity->fence_seq))
 		return -EINVAL;
 
 	scheds = p->adev->gpu_sched[AMDGPU_HW_IP_VCN_ENC]
 		[AMDGPU_RING_PRIO_0].sched;
-	drm_sched_entity_modify_sched(p->entity, scheds, 1);
+	drm_sched_entity_modify_sched(job->base.entity, scheds, 1);
 	return 0;
 }
 
-static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr)
+static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, struct amdgpu_job *job,
+			    uint64_t addr)
 {
 	struct ttm_operation_ctx ctx = { false, false };
 	struct amdgpu_bo_va_mapping *map;
@@ -1412,7 +1682,7 @@ static int vcn_v4_0_dec_msg(struct amdgpu_cs_parser *p, uint64_t addr)
 		if (create[0] == 0x7 || create[0] == 0x10 || create[0] == 0x11)
 			continue;
 
-		r = vcn_v4_0_limit_sched(p);
+		r = vcn_v4_0_limit_sched(p, job);
 		if (r)
 			goto out;
 	}
@@ -1425,32 +1695,34 @@ out:
 #define RADEON_VCN_ENGINE_TYPE_DECODE                                 (0x00000003)
 
 static int vcn_v4_0_ring_patch_cs_in_place(struct amdgpu_cs_parser *p,
-				struct amdgpu_job *job,
-				struct amdgpu_ib *ib)
+					   struct amdgpu_job *job,
+					   struct amdgpu_ib *ib)
 {
-	struct amdgpu_ring *ring = to_amdgpu_ring(p->entity->rq->sched);
-	struct amdgpu_vcn_decode_buffer *decode_buffer = NULL;
+	struct amdgpu_ring *ring = amdgpu_job_ring(job);
+	struct amdgpu_vcn_decode_buffer *decode_buffer;
+	uint64_t addr;
 	uint32_t val;
-	int r = 0;
 
 	/* The first instance can decode anything */
 	if (!ring->me)
-		return r;
+		return 0;
 
 	/* unified queue ib header has 8 double words. */
 	if (ib->length_dw < 8)
-		return r;
+		return 0;
 
 	val = amdgpu_ib_get_value(ib, 6); //RADEON_VCN_ENGINE_TYPE
+	if (val != RADEON_VCN_ENGINE_TYPE_DECODE)
+		return 0;
 
-	if (val == RADEON_VCN_ENGINE_TYPE_DECODE) {
-		decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[10];
+	decode_buffer = (struct amdgpu_vcn_decode_buffer *)&ib->ptr[10];
 
-		if (decode_buffer->valid_buf_flag  & 0x1)
-			r = vcn_v4_0_dec_msg(p, ((u64)decode_buffer->msg_buffer_address_hi) << 32 |
-						decode_buffer->msg_buffer_address_lo);
-	}
-	return r;
+	if (!(decode_buffer->valid_buf_flag  & 0x1))
+		return 0;
+
+	addr = ((u64)decode_buffer->msg_buffer_address_hi) << 32 |
+		decode_buffer->msg_buffer_address_lo;
+	return vcn_v4_0_dec_msg(p, job, addr);
 }
 
 static const struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = {
@@ -1596,6 +1868,15 @@ static int vcn_v4_0_set_powergating_state(void *handle, enum amd_powergating_sta
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	int ret;
 
+	/* for SRIOV, guest should not control VCN Power-gating
+	 * MMSCH FW should control Power-gating and clock-gating
+	 * guest should avoid touching CGC and PG
+	 */
+	if (amdgpu_sriov_vf(adev)) {
+		adev->vcn.cur_state = AMD_PG_STATE_UNGATE;
+		return 0;
+	}
+
 	if(state == adev->vcn.cur_state)
 		return 0;
 
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
index 60a81649cf12..c7118843db05 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h
@@ -742,7 +742,7 @@ static const uint32_t cwsr_trap_nv1x_hex[] = {
 	0xbf88fffe, 0x877aff7f,
 	0x04000000, 0x8f7a857a,
 	0x886d7a6d, 0xb97b02dc,
-	0x8f7b997b, 0xb97a2a05,
+	0x8f7b997b, 0xb97a3a05,
 	0x807a817a, 0xbf0d997b,
 	0xbf850002, 0x8f7a897a,
 	0xbf820001, 0x8f7a8a7a,
@@ -819,7 +819,7 @@ static const uint32_t cwsr_trap_nv1x_hex[] = {
 	0xbefe037c, 0xbefc0370,
 	0xf4611c7a, 0xf8000000,
 	0x80708470, 0xbefc037e,
-	0xb9702a05, 0x80708170,
+	0xb9703a05, 0x80708170,
 	0xbf0d9973, 0xbf850002,
 	0x8f708970, 0xbf820001,
 	0x8f708a70, 0xb97a1e06,
@@ -1069,7 +1069,7 @@ static const uint32_t cwsr_trap_nv1x_hex[] = {
 	0xb9f9f816, 0x876f7bff,
 	0xfffff800, 0x906f8b6f,
 	0xb9efa2c3, 0xb9f3f801,
-	0xb96e2a05, 0x806e816e,
+	0xb96e3a05, 0x806e816e,
 	0xbf0d9972, 0xbf850002,
 	0x8f6e896e, 0xbf820001,
 	0x8f6e8a6e, 0xb96f1e06,
@@ -2114,7 +2114,7 @@ static const uint32_t cwsr_trap_gfx10_hex[] = {
 	0x007a0000, 0x7e000280,
 	0xbefe037a, 0xbeff037b,
 	0xb97b02dc, 0x8f7b997b,
-	0xb97a2a05, 0x807a817a,
+	0xb97a3a05, 0x807a817a,
 	0xbf0d997b, 0xbf850002,
 	0x8f7a897a, 0xbf820001,
 	0x8f7a8a7a, 0xb97b1e06,
@@ -2157,7 +2157,7 @@ static const uint32_t cwsr_trap_gfx10_hex[] = {
 	0x01000000, 0xe0704100,
 	0x705d0100, 0xe0704200,
 	0x705d0200, 0xe0704300,
-	0x705d0300, 0xb9702a05,
+	0x705d0300, 0xb9703a05,
 	0x80708170, 0xbf0d9973,
 	0xbf850002, 0x8f708970,
 	0xbf820001, 0x8f708a70,
@@ -2189,7 +2189,7 @@ static const uint32_t cwsr_trap_gfx10_hex[] = {
 	0xbefe03ff, 0x0000ffff,
 	0xbeff0380, 0xe0704000,
 	0x705d0200, 0xbefe03c1,
-	0xb9702a05, 0x80708170,
+	0xb9703a05, 0x80708170,
 	0xbf0d9973, 0xbf850002,
 	0x8f708970, 0xbf820001,
 	0x8f708a70, 0xb97a1e06,
@@ -2475,7 +2475,7 @@ static const uint32_t cwsr_trap_gfx10_hex[] = {
 	0xb9ef4803, 0x876f7bff,
 	0xfffff800, 0x906f8b6f,
 	0xb9efa2c3, 0xb9f3f801,
-	0xb96e2a05, 0x806e816e,
+	0xb96e3a05, 0x806e816e,
 	0xbf0d9972, 0xbf850002,
 	0x8f6e896e, 0xbf820001,
 	0x8f6e8a6e, 0xb96f1e06,
@@ -2494,438 +2494,441 @@ static const uint32_t cwsr_trap_gfx10_hex[] = {
 	0xbf9f0000, 0xbf9f0000,
 	0xbf9f0000, 0x00000000,
 };
-
 static const uint32_t cwsr_trap_gfx11_hex[] = {
-	0xbfa00001, 0xbfa0021b,
+	0xbfa00001, 0xbfa0021e,
 	0xb0804006, 0xb8f8f802,
-	0x91788678, 0xb8fbf803,
-	0x8b6eff78, 0x00002000,
-	0xbfa10009, 0x8b6eff6d,
-	0x00ff0000, 0xbfa2001e,
-	0x8b6eff7b, 0x00000400,
-	0xbfa20041, 0xbf830010,
-	0xb8fbf803, 0xbfa0fffa,
-	0x8b6eff7b, 0x00000900,
-	0xbfa20015, 0x8b6eff7b,
-	0x000071ff, 0xbfa10008,
-	0x8b6fff7b, 0x00007080,
-	0xbfa10001, 0xbeee1287,
-	0xb8eff801, 0x846e8c6e,
-	0x8b6e6f6e, 0xbfa2000a,
+	0x9178ff78, 0x00020006,
+	0xb8fbf803, 0xbf0d9f6d,
+	0xbfa20006, 0x8b6eff78,
+	0x00002000, 0xbfa10009,
 	0x8b6eff6d, 0x00ff0000,
-	0xbfa20007, 0xb8eef801,
-	0x8b6eff6e, 0x00000800,
-	0xbfa20003, 0x8b6eff7b,
-	0x00000400, 0xbfa20026,
-	0xbefa4d82, 0xbf89fc07,
-	0x84fa887a, 0xf4005bbd,
-	0xf8000010, 0xbf89fc07,
-	0x846e976e, 0x9177ff77,
-	0x00800000, 0x8c776e77,
-	0xf4045bbd, 0xf8000000,
-	0xbf89fc07, 0xf4045ebd,
-	0xf8000008, 0xbf89fc07,
-	0x8bee6e6e, 0xbfa10001,
-	0xbe80486e, 0x8b6eff6d,
-	0x01ff0000, 0xbfa20005,
-	0x8c78ff78, 0x00002000,
-	0x80ec886c, 0x82ed806d,
-	0xbfa00005, 0x8b6eff6d,
-	0x01000000, 0xbfa20002,
-	0x806c846c, 0x826d806d,
-	0x8b6dff6d, 0x0000ffff,
-	0x8bfe7e7e, 0x8bea6a6a,
-	0xb978f802, 0xbe804a6c,
-	0x8b6dff6d, 0x0000ffff,
-	0xbefa0080, 0xb97a0283,
-	0xbeee007e, 0xbeef007f,
-	0xbefe0180, 0xbefe4d84,
-	0xbf89fc07, 0x8b7aff7f,
-	0x04000000, 0x847a857a,
-	0x8c6d7a6d, 0xbefa007e,
-	0x8b7bff7f, 0x0000ffff,
-	0xbefe00c1, 0xbeff00c1,
-	0xdca6c000, 0x007a0000,
-	0x7e000280, 0xbefe007a,
-	0xbeff007b, 0xb8fb02dc,
-	0x847b997b, 0xb8fa3b05,
-	0x807a817a, 0xbf0d997b,
-	0xbfa20002, 0x847a897a,
-	0xbfa00001, 0x847a8a7a,
-	0xb8fb1e06, 0x847b8a7b,
-	0x807a7b7a, 0x8b7bff7f,
-	0x0000ffff, 0x807aff7a,
-	0x00000200, 0x807a7e7a,
-	0x827b807b, 0xd7610000,
-	0x00010870, 0xd7610000,
-	0x00010a71, 0xd7610000,
-	0x00010c72, 0xd7610000,
-	0x00010e73, 0xd7610000,
-	0x00011074, 0xd7610000,
-	0x00011275, 0xd7610000,
-	0x00011476, 0xd7610000,
-	0x00011677, 0xd7610000,
-	0x00011a79, 0xd7610000,
-	0x00011c7e, 0xd7610000,
-	0x00011e7f, 0xbefe00ff,
-	0x00003fff, 0xbeff0080,
-	0xdca6c040, 0x007a0000,
-	0xd760007a, 0x00011d00,
-	0xd760007b, 0x00011f00,
+	0xbfa2001e, 0x8b6eff7b,
+	0x00000400, 0xbfa20041,
+	0xbf830010, 0xb8fbf803,
+	0xbfa0fffa, 0x8b6eff7b,
+	0x00000900, 0xbfa20015,
+	0x8b6eff7b, 0x000071ff,
+	0xbfa10008, 0x8b6fff7b,
+	0x00007080, 0xbfa10001,
+	0xbeee1287, 0xb8eff801,
+	0x846e8c6e, 0x8b6e6f6e,
+	0xbfa2000a, 0x8b6eff6d,
+	0x00ff0000, 0xbfa20007,
+	0xb8eef801, 0x8b6eff6e,
+	0x00000800, 0xbfa20003,
+	0x8b6eff7b, 0x00000400,
+	0xbfa20026, 0xbefa4d82,
+	0xbf89fc07, 0x84fa887a,
+	0xf4005bbd, 0xf8000010,
+	0xbf89fc07, 0x846e976e,
+	0x9177ff77, 0x00800000,
+	0x8c776e77, 0xf4045bbd,
+	0xf8000000, 0xbf89fc07,
+	0xf4045ebd, 0xf8000008,
+	0xbf89fc07, 0x8bee6e6e,
+	0xbfa10001, 0xbe80486e,
+	0x8b6eff6d, 0x01ff0000,
+	0xbfa20005, 0x8c78ff78,
+	0x00002000, 0x80ec886c,
+	0x82ed806d, 0xbfa00005,
+	0x8b6eff6d, 0x01000000,
+	0xbfa20002, 0x806c846c,
+	0x826d806d, 0x8b6dff6d,
+	0x0000ffff, 0x8bfe7e7e,
+	0x8bea6a6a, 0xb978f802,
+	0xbe804a6c, 0x8b6dff6d,
+	0x0000ffff, 0xbefa0080,
+	0xb97a0283, 0xbeee007e,
+	0xbeef007f, 0xbefe0180,
+	0xbefe4d84, 0xbf89fc07,
+	0x8b7aff7f, 0x04000000,
+	0x847a857a, 0x8c6d7a6d,
+	0xbefa007e, 0x8b7bff7f,
+	0x0000ffff, 0xbefe00c1,
+	0xbeff00c1, 0xdca6c000,
+	0x007a0000, 0x7e000280,
 	0xbefe007a, 0xbeff007b,
-	0xbef4007e, 0x8b75ff7f,
-	0x0000ffff, 0x8c75ff75,
-	0x00040000, 0xbef60080,
-	0xbef700ff, 0x10807fac,
-	0xbef1007d, 0xbef00080,
-	0xb8f302dc, 0x84739973,
-	0xbefe00c1, 0x857d9973,
-	0x8b7d817d, 0xbf06817d,
-	0xbfa20002, 0xbeff0080,
-	0xbfa00002, 0xbeff00c1,
-	0xbfa00009, 0xbef600ff,
-	0x01000000, 0xe0685080,
-	0x701d0100, 0xe0685100,
-	0x701d0200, 0xe0685180,
-	0x701d0300, 0xbfa00008,
+	0xb8fb02dc, 0x847b997b,
+	0xb8fa3b05, 0x807a817a,
+	0xbf0d997b, 0xbfa20002,
+	0x847a897a, 0xbfa00001,
+	0x847a8a7a, 0xb8fb1e06,
+	0x847b8a7b, 0x807a7b7a,
+	0x8b7bff7f, 0x0000ffff,
+	0x807aff7a, 0x00000200,
+	0x807a7e7a, 0x827b807b,
+	0xd7610000, 0x00010870,
+	0xd7610000, 0x00010a71,
+	0xd7610000, 0x00010c72,
+	0xd7610000, 0x00010e73,
+	0xd7610000, 0x00011074,
+	0xd7610000, 0x00011275,
+	0xd7610000, 0x00011476,
+	0xd7610000, 0x00011677,
+	0xd7610000, 0x00011a79,
+	0xd7610000, 0x00011c7e,
+	0xd7610000, 0x00011e7f,
+	0xbefe00ff, 0x00003fff,
+	0xbeff0080, 0xdca6c040,
+	0x007a0000, 0xd760007a,
+	0x00011d00, 0xd760007b,
+	0x00011f00, 0xbefe007a,
+	0xbeff007b, 0xbef4007e,
+	0x8b75ff7f, 0x0000ffff,
+	0x8c75ff75, 0x00040000,
+	0xbef60080, 0xbef700ff,
+	0x10807fac, 0xbef1007d,
+	0xbef00080, 0xb8f302dc,
+	0x84739973, 0xbefe00c1,
+	0x857d9973, 0x8b7d817d,
+	0xbf06817d, 0xbfa20002,
+	0xbeff0080, 0xbfa00002,
+	0xbeff00c1, 0xbfa00009,
 	0xbef600ff, 0x01000000,
-	0xe0685100, 0x701d0100,
-	0xe0685200, 0x701d0200,
-	0xe0685300, 0x701d0300,
+	0xe0685080, 0x701d0100,
+	0xe0685100, 0x701d0200,
+	0xe0685180, 0x701d0300,
+	0xbfa00008, 0xbef600ff,
+	0x01000000, 0xe0685100,
+	0x701d0100, 0xe0685200,
+	0x701d0200, 0xe0685300,
+	0x701d0300, 0xb8f03b05,
+	0x80708170, 0xbf0d9973,
+	0xbfa20002, 0x84708970,
+	0xbfa00001, 0x84708a70,
+	0xb8fa1e06, 0x847a8a7a,
+	0x80707a70, 0x8070ff70,
+	0x00000200, 0xbef600ff,
+	0x01000000, 0x7e000280,
+	0x7e020280, 0x7e040280,
+	0xbefd0080, 0xd7610002,
+	0x0000fa71, 0x807d817d,
+	0xd7610002, 0x0000fa6c,
+	0x807d817d, 0x917aff6d,
+	0x80000000, 0xd7610002,
+	0x0000fa7a, 0x807d817d,
+	0xd7610002, 0x0000fa6e,
+	0x807d817d, 0xd7610002,
+	0x0000fa6f, 0x807d817d,
+	0xd7610002, 0x0000fa78,
+	0x807d817d, 0xb8faf803,
+	0xd7610002, 0x0000fa7a,
+	0x807d817d, 0xd7610002,
+	0x0000fa7b, 0x807d817d,
+	0xb8f1f801, 0xd7610002,
+	0x0000fa71, 0x807d817d,
+	0xb8f1f814, 0xd7610002,
+	0x0000fa71, 0x807d817d,
+	0xb8f1f815, 0xd7610002,
+	0x0000fa71, 0x807d817d,
+	0xbefe00ff, 0x0000ffff,
+	0xbeff0080, 0xe0685000,
+	0x701d0200, 0xbefe00c1,
 	0xb8f03b05, 0x80708170,
 	0xbf0d9973, 0xbfa20002,
 	0x84708970, 0xbfa00001,
 	0x84708a70, 0xb8fa1e06,
 	0x847a8a7a, 0x80707a70,
-	0x8070ff70, 0x00000200,
 	0xbef600ff, 0x01000000,
-	0x7e000280, 0x7e020280,
-	0x7e040280, 0xbefd0080,
-	0xd7610002, 0x0000fa71,
-	0x807d817d, 0xd7610002,
-	0x0000fa6c, 0x807d817d,
-	0x917aff6d, 0x80000000,
-	0xd7610002, 0x0000fa7a,
-	0x807d817d, 0xd7610002,
-	0x0000fa6e, 0x807d817d,
-	0xd7610002, 0x0000fa6f,
-	0x807d817d, 0xd7610002,
-	0x0000fa78, 0x807d817d,
-	0xb8faf803, 0xd7610002,
-	0x0000fa7a, 0x807d817d,
-	0xd7610002, 0x0000fa7b,
-	0x807d817d, 0xb8f1f801,
-	0xd7610002, 0x0000fa71,
-	0x807d817d, 0xb8f1f814,
-	0xd7610002, 0x0000fa71,
-	0x807d817d, 0xb8f1f815,
-	0xd7610002, 0x0000fa71,
-	0x807d817d, 0xbefe00ff,
-	0x0000ffff, 0xbeff0080,
-	0xe0685000, 0x701d0200,
-	0xbefe00c1, 0xb8f03b05,
-	0x80708170, 0xbf0d9973,
-	0xbfa20002, 0x84708970,
-	0xbfa00001, 0x84708a70,
-	0xb8fa1e06, 0x847a8a7a,
-	0x80707a70, 0xbef600ff,
-	0x01000000, 0xbef90080,
-	0xbefd0080, 0xbf800000,
-	0xbe804100, 0xbe824102,
-	0xbe844104, 0xbe864106,
-	0xbe884108, 0xbe8a410a,
-	0xbe8c410c, 0xbe8e410e,
-	0xd7610002, 0x0000f200,
-	0x80798179, 0xd7610002,
-	0x0000f201, 0x80798179,
-	0xd7610002, 0x0000f202,
-	0x80798179, 0xd7610002,
-	0x0000f203, 0x80798179,
-	0xd7610002, 0x0000f204,
+	0xbef90080, 0xbefd0080,
+	0xbf800000, 0xbe804100,
+	0xbe824102, 0xbe844104,
+	0xbe864106, 0xbe884108,
+	0xbe8a410a, 0xbe8c410c,
+	0xbe8e410e, 0xd7610002,
+	0x0000f200, 0x80798179,
+	0xd7610002, 0x0000f201,
 	0x80798179, 0xd7610002,
-	0x0000f205, 0x80798179,
-	0xd7610002, 0x0000f206,
+	0x0000f202, 0x80798179,
+	0xd7610002, 0x0000f203,
 	0x80798179, 0xd7610002,
-	0x0000f207, 0x80798179,
-	0xd7610002, 0x0000f208,
+	0x0000f204, 0x80798179,
+	0xd7610002, 0x0000f205,
 	0x80798179, 0xd7610002,
-	0x0000f209, 0x80798179,
-	0xd7610002, 0x0000f20a,
+	0x0000f206, 0x80798179,
+	0xd7610002, 0x0000f207,
 	0x80798179, 0xd7610002,
-	0x0000f20b, 0x80798179,
-	0xd7610002, 0x0000f20c,
+	0x0000f208, 0x80798179,
+	0xd7610002, 0x0000f209,
 	0x80798179, 0xd7610002,
-	0x0000f20d, 0x80798179,
-	0xd7610002, 0x0000f20e,
+	0x0000f20a, 0x80798179,
+	0xd7610002, 0x0000f20b,
 	0x80798179, 0xd7610002,
-	0x0000f20f, 0x80798179,
-	0xbf06a079, 0xbfa10006,
-	0xe0685000, 0x701d0200,
-	0x8070ff70, 0x00000080,
-	0xbef90080, 0x7e040280,
-	0x807d907d, 0xbf0aff7d,
-	0x00000060, 0xbfa2ffbc,
-	0xbe804100, 0xbe824102,
-	0xbe844104, 0xbe864106,
-	0xbe884108, 0xbe8a410a,
-	0xd7610002, 0x0000f200,
+	0x0000f20c, 0x80798179,
+	0xd7610002, 0x0000f20d,
 	0x80798179, 0xd7610002,
-	0x0000f201, 0x80798179,
-	0xd7610002, 0x0000f202,
+	0x0000f20e, 0x80798179,
+	0xd7610002, 0x0000f20f,
+	0x80798179, 0xbf06a079,
+	0xbfa10006, 0xe0685000,
+	0x701d0200, 0x8070ff70,
+	0x00000080, 0xbef90080,
+	0x7e040280, 0x807d907d,
+	0xbf0aff7d, 0x00000060,
+	0xbfa2ffbc, 0xbe804100,
+	0xbe824102, 0xbe844104,
+	0xbe864106, 0xbe884108,
+	0xbe8a410a, 0xd7610002,
+	0x0000f200, 0x80798179,
+	0xd7610002, 0x0000f201,
 	0x80798179, 0xd7610002,
-	0x0000f203, 0x80798179,
-	0xd7610002, 0x0000f204,
+	0x0000f202, 0x80798179,
+	0xd7610002, 0x0000f203,
 	0x80798179, 0xd7610002,
-	0x0000f205, 0x80798179,
-	0xd7610002, 0x0000f206,
+	0x0000f204, 0x80798179,
+	0xd7610002, 0x0000f205,
 	0x80798179, 0xd7610002,
-	0x0000f207, 0x80798179,
-	0xd7610002, 0x0000f208,
+	0x0000f206, 0x80798179,
+	0xd7610002, 0x0000f207,
 	0x80798179, 0xd7610002,
-	0x0000f209, 0x80798179,
-	0xd7610002, 0x0000f20a,
+	0x0000f208, 0x80798179,
+	0xd7610002, 0x0000f209,
 	0x80798179, 0xd7610002,
-	0x0000f20b, 0x80798179,
-	0xe0685000, 0x701d0200,
+	0x0000f20a, 0x80798179,
+	0xd7610002, 0x0000f20b,
+	0x80798179, 0xe0685000,
+	0x701d0200, 0xbefe00c1,
+	0x857d9973, 0x8b7d817d,
+	0xbf06817d, 0xbfa20002,
+	0xbeff0080, 0xbfa00001,
+	0xbeff00c1, 0xb8fb4306,
+	0x8b7bc17b, 0xbfa10044,
+	0xbfbd0000, 0x8b7aff6d,
+	0x80000000, 0xbfa10040,
+	0x847b867b, 0x847b827b,
+	0xbef6007b, 0xb8f03b05,
+	0x80708170, 0xbf0d9973,
+	0xbfa20002, 0x84708970,
+	0xbfa00001, 0x84708a70,
+	0xb8fa1e06, 0x847a8a7a,
+	0x80707a70, 0x8070ff70,
+	0x00000200, 0x8070ff70,
+	0x00000080, 0xbef600ff,
+	0x01000000, 0xd71f0000,
+	0x000100c1, 0xd7200000,
+	0x000200c1, 0x16000084,
+	0x857d9973, 0x8b7d817d,
+	0xbf06817d, 0xbefd0080,
+	0xbfa20012, 0xbe8300ff,
+	0x00000080, 0xbf800000,
+	0xbf800000, 0xbf800000,
+	0xd8d80000, 0x01000000,
+	0xbf890000, 0xe0685000,
+	0x701d0100, 0x807d037d,
+	0x80700370, 0xd5250000,
+	0x0001ff00, 0x00000080,
+	0xbf0a7b7d, 0xbfa2fff4,
+	0xbfa00011, 0xbe8300ff,
+	0x00000100, 0xbf800000,
+	0xbf800000, 0xbf800000,
+	0xd8d80000, 0x01000000,
+	0xbf890000, 0xe0685000,
+	0x701d0100, 0x807d037d,
+	0x80700370, 0xd5250000,
+	0x0001ff00, 0x00000100,
+	0xbf0a7b7d, 0xbfa2fff4,
 	0xbefe00c1, 0x857d9973,
 	0x8b7d817d, 0xbf06817d,
-	0xbfa20002, 0xbeff0080,
-	0xbfa00001, 0xbeff00c1,
-	0xb8fb4306, 0x8b7bc17b,
-	0xbfa10044, 0xbfbd0000,
-	0x8b7aff6d, 0x80000000,
-	0xbfa10040, 0x847b867b,
-	0x847b827b, 0xbef6007b,
-	0xb8f03b05, 0x80708170,
-	0xbf0d9973, 0xbfa20002,
-	0x84708970, 0xbfa00001,
-	0x84708a70, 0xb8fa1e06,
-	0x847a8a7a, 0x80707a70,
-	0x8070ff70, 0x00000200,
-	0x8070ff70, 0x00000080,
-	0xbef600ff, 0x01000000,
-	0xd71f0000, 0x000100c1,
-	0xd7200000, 0x000200c1,
-	0x16000084, 0x857d9973,
+	0xbfa20004, 0xbef000ff,
+	0x00000200, 0xbeff0080,
+	0xbfa00003, 0xbef000ff,
+	0x00000400, 0xbeff00c1,
+	0xb8fb3b05, 0x807b817b,
+	0x847b827b, 0x857d9973,
 	0x8b7d817d, 0xbf06817d,
-	0xbefd0080, 0xbfa20012,
-	0xbe8300ff, 0x00000080,
-	0xbf800000, 0xbf800000,
-	0xbf800000, 0xd8d80000,
-	0x01000000, 0xbf890000,
-	0xe0685000, 0x701d0100,
-	0x807d037d, 0x80700370,
-	0xd5250000, 0x0001ff00,
-	0x00000080, 0xbf0a7b7d,
-	0xbfa2fff4, 0xbfa00011,
-	0xbe8300ff, 0x00000100,
-	0xbf800000, 0xbf800000,
-	0xbf800000, 0xd8d80000,
-	0x01000000, 0xbf890000,
-	0xe0685000, 0x701d0100,
-	0x807d037d, 0x80700370,
-	0xd5250000, 0x0001ff00,
-	0x00000100, 0xbf0a7b7d,
-	0xbfa2fff4, 0xbefe00c1,
-	0x857d9973, 0x8b7d817d,
-	0xbf06817d, 0xbfa20004,
-	0xbef000ff, 0x00000200,
-	0xbeff0080, 0xbfa00003,
-	0xbef000ff, 0x00000400,
-	0xbeff00c1, 0xb8fb3b05,
-	0x807b817b, 0x847b827b,
-	0x857d9973, 0x8b7d817d,
-	0xbf06817d, 0xbfa20017,
+	0xbfa20017, 0xbef600ff,
+	0x01000000, 0xbefd0084,
+	0xbf0a7b7d, 0xbfa10037,
+	0x7e008700, 0x7e028701,
+	0x7e048702, 0x7e068703,
+	0xe0685000, 0x701d0000,
+	0xe0685080, 0x701d0100,
+	0xe0685100, 0x701d0200,
+	0xe0685180, 0x701d0300,
+	0x807d847d, 0x8070ff70,
+	0x00000200, 0xbf0a7b7d,
+	0xbfa2ffef, 0xbfa00025,
 	0xbef600ff, 0x01000000,
 	0xbefd0084, 0xbf0a7b7d,
-	0xbfa10037, 0x7e008700,
+	0xbfa10011, 0x7e008700,
 	0x7e028701, 0x7e048702,
 	0x7e068703, 0xe0685000,
-	0x701d0000, 0xe0685080,
-	0x701d0100, 0xe0685100,
-	0x701d0200, 0xe0685180,
+	0x701d0000, 0xe0685100,
+	0x701d0100, 0xe0685200,
+	0x701d0200, 0xe0685300,
 	0x701d0300, 0x807d847d,
-	0x8070ff70, 0x00000200,
+	0x8070ff70, 0x00000400,
 	0xbf0a7b7d, 0xbfa2ffef,
-	0xbfa00025, 0xbef600ff,
-	0x01000000, 0xbefd0084,
-	0xbf0a7b7d, 0xbfa10011,
-	0x7e008700, 0x7e028701,
-	0x7e048702, 0x7e068703,
+	0xb8fb1e06, 0x8b7bc17b,
+	0xbfa1000c, 0x847b837b,
+	0x807b7d7b, 0xbefe00c1,
+	0xbeff0080, 0x7e008700,
 	0xe0685000, 0x701d0000,
-	0xe0685100, 0x701d0100,
-	0xe0685200, 0x701d0200,
-	0xe0685300, 0x701d0300,
-	0x807d847d, 0x8070ff70,
-	0x00000400, 0xbf0a7b7d,
-	0xbfa2ffef, 0xb8fb1e06,
-	0x8b7bc17b, 0xbfa1000c,
-	0x847b837b, 0x807b7d7b,
-	0xbefe00c1, 0xbeff0080,
-	0x7e008700, 0xe0685000,
-	0x701d0000, 0x807d817d,
-	0x8070ff70, 0x00000080,
-	0xbf0a7b7d, 0xbfa2fff8,
-	0xbfa00141, 0xbef4007e,
-	0x8b75ff7f, 0x0000ffff,
-	0x8c75ff75, 0x00040000,
-	0xbef60080, 0xbef700ff,
-	0x10807fac, 0xb8f202dc,
-	0x84729972, 0x8b6eff7f,
-	0x04000000, 0xbfa1003a,
+	0x807d817d, 0x8070ff70,
+	0x00000080, 0xbf0a7b7d,
+	0xbfa2fff8, 0xbfa00146,
+	0xbef4007e, 0x8b75ff7f,
+	0x0000ffff, 0x8c75ff75,
+	0x00040000, 0xbef60080,
+	0xbef700ff, 0x10807fac,
+	0xb8f202dc, 0x84729972,
+	0x8b6eff7f, 0x04000000,
+	0xbfa1003a, 0xbefe00c1,
+	0x857d9972, 0x8b7d817d,
+	0xbf06817d, 0xbfa20002,
+	0xbeff0080, 0xbfa00001,
+	0xbeff00c1, 0xb8ef4306,
+	0x8b6fc16f, 0xbfa1002f,
+	0x846f866f, 0x846f826f,
+	0xbef6006f, 0xb8f83b05,
+	0x80788178, 0xbf0d9972,
+	0xbfa20002, 0x84788978,
+	0xbfa00001, 0x84788a78,
+	0xb8ee1e06, 0x846e8a6e,
+	0x80786e78, 0x8078ff78,
+	0x00000200, 0x8078ff78,
+	0x00000080, 0xbef600ff,
+	0x01000000, 0x857d9972,
+	0x8b7d817d, 0xbf06817d,
+	0xbefd0080, 0xbfa2000c,
+	0xe0500000, 0x781d0000,
+	0xbf8903f7, 0xdac00000,
+	0x00000000, 0x807dff7d,
+	0x00000080, 0x8078ff78,
+	0x00000080, 0xbf0a6f7d,
+	0xbfa2fff5, 0xbfa0000b,
+	0xe0500000, 0x781d0000,
+	0xbf8903f7, 0xdac00000,
+	0x00000000, 0x807dff7d,
+	0x00000100, 0x8078ff78,
+	0x00000100, 0xbf0a6f7d,
+	0xbfa2fff5, 0xbef80080,
 	0xbefe00c1, 0x857d9972,
 	0x8b7d817d, 0xbf06817d,
 	0xbfa20002, 0xbeff0080,
 	0xbfa00001, 0xbeff00c1,
-	0xb8ef4306, 0x8b6fc16f,
-	0xbfa1002f, 0x846f866f,
-	0x846f826f, 0xbef6006f,
-	0xb8f83b05, 0x80788178,
-	0xbf0d9972, 0xbfa20002,
-	0x84788978, 0xbfa00001,
-	0x84788a78, 0xb8ee1e06,
-	0x846e8a6e, 0x80786e78,
+	0xb8ef3b05, 0x806f816f,
+	0x846f826f, 0x857d9972,
+	0x8b7d817d, 0xbf06817d,
+	0xbfa20024, 0xbef600ff,
+	0x01000000, 0xbeee0078,
 	0x8078ff78, 0x00000200,
-	0x8078ff78, 0x00000080,
-	0xbef600ff, 0x01000000,
-	0x857d9972, 0x8b7d817d,
-	0xbf06817d, 0xbefd0080,
-	0xbfa2000c, 0xe0500000,
-	0x781d0000, 0xbf8903f7,
-	0xdac00000, 0x00000000,
-	0x807dff7d, 0x00000080,
-	0x8078ff78, 0x00000080,
-	0xbf0a6f7d, 0xbfa2fff5,
-	0xbfa0000b, 0xe0500000,
-	0x781d0000, 0xbf8903f7,
-	0xdac00000, 0x00000000,
-	0x807dff7d, 0x00000100,
-	0x8078ff78, 0x00000100,
-	0xbf0a6f7d, 0xbfa2fff5,
-	0xbef80080, 0xbefe00c1,
-	0x857d9972, 0x8b7d817d,
-	0xbf06817d, 0xbfa20002,
-	0xbeff0080, 0xbfa00001,
-	0xbeff00c1, 0xb8ef3b05,
-	0x806f816f, 0x846f826f,
-	0x857d9972, 0x8b7d817d,
-	0xbf06817d, 0xbfa20024,
-	0xbef600ff, 0x01000000,
-	0xbeee0078, 0x8078ff78,
-	0x00000200, 0xbefd0084,
-	0xbf0a6f7d, 0xbfa10050,
+	0xbefd0084, 0xbf0a6f7d,
+	0xbfa10050, 0xe0505000,
+	0x781d0000, 0xe0505080,
+	0x781d0100, 0xe0505100,
+	0x781d0200, 0xe0505180,
+	0x781d0300, 0xbf8903f7,
+	0x7e008500, 0x7e028501,
+	0x7e048502, 0x7e068503,
+	0x807d847d, 0x8078ff78,
+	0x00000200, 0xbf0a6f7d,
+	0xbfa2ffee, 0xe0505000,
+	0x6e1d0000, 0xe0505080,
+	0x6e1d0100, 0xe0505100,
+	0x6e1d0200, 0xe0505180,
+	0x6e1d0300, 0xbf8903f7,
+	0xbfa00034, 0xbef600ff,
+	0x01000000, 0xbeee0078,
+	0x8078ff78, 0x00000400,
+	0xbefd0084, 0xbf0a6f7d,
+	0xbfa10012, 0xe0505000,
+	0x781d0000, 0xe0505100,
+	0x781d0100, 0xe0505200,
+	0x781d0200, 0xe0505300,
+	0x781d0300, 0xbf8903f7,
+	0x7e008500, 0x7e028501,
+	0x7e048502, 0x7e068503,
+	0x807d847d, 0x8078ff78,
+	0x00000400, 0xbf0a6f7d,
+	0xbfa2ffee, 0xb8ef1e06,
+	0x8b6fc16f, 0xbfa1000e,
+	0x846f836f, 0x806f7d6f,
+	0xbefe00c1, 0xbeff0080,
 	0xe0505000, 0x781d0000,
-	0xe0505080, 0x781d0100,
-	0xe0505100, 0x781d0200,
-	0xe0505180, 0x781d0300,
 	0xbf8903f7, 0x7e008500,
-	0x7e028501, 0x7e048502,
-	0x7e068503, 0x807d847d,
-	0x8078ff78, 0x00000200,
-	0xbf0a6f7d, 0xbfa2ffee,
+	0x807d817d, 0x8078ff78,
+	0x00000080, 0xbf0a6f7d,
+	0xbfa2fff7, 0xbeff00c1,
 	0xe0505000, 0x6e1d0000,
-	0xe0505080, 0x6e1d0100,
-	0xe0505100, 0x6e1d0200,
-	0xe0505180, 0x6e1d0300,
-	0xbf8903f7, 0xbfa00034,
-	0xbef600ff, 0x01000000,
-	0xbeee0078, 0x8078ff78,
-	0x00000400, 0xbefd0084,
-	0xbf0a6f7d, 0xbfa10012,
-	0xe0505000, 0x781d0000,
-	0xe0505100, 0x781d0100,
-	0xe0505200, 0x781d0200,
-	0xe0505300, 0x781d0300,
-	0xbf8903f7, 0x7e008500,
-	0x7e028501, 0x7e048502,
-	0x7e068503, 0x807d847d,
-	0x8078ff78, 0x00000400,
-	0xbf0a6f7d, 0xbfa2ffee,
-	0xb8ef1e06, 0x8b6fc16f,
-	0xbfa1000e, 0x846f836f,
-	0x806f7d6f, 0xbefe00c1,
-	0xbeff0080, 0xe0505000,
-	0x781d0000, 0xbf8903f7,
-	0x7e008500, 0x807d817d,
-	0x8078ff78, 0x00000080,
-	0xbf0a6f7d, 0xbfa2fff7,
-	0xbeff00c1, 0xe0505000,
-	0x6e1d0000, 0xe0505100,
-	0x6e1d0100, 0xe0505200,
-	0x6e1d0200, 0xe0505300,
-	0x6e1d0300, 0xbf8903f7,
+	0xe0505100, 0x6e1d0100,
+	0xe0505200, 0x6e1d0200,
+	0xe0505300, 0x6e1d0300,
+	0xbf8903f7, 0xb8f83b05,
+	0x80788178, 0xbf0d9972,
+	0xbfa20002, 0x84788978,
+	0xbfa00001, 0x84788a78,
+	0xb8ee1e06, 0x846e8a6e,
+	0x80786e78, 0x8078ff78,
+	0x00000200, 0x80f8ff78,
+	0x00000050, 0xbef600ff,
+	0x01000000, 0xbefd00ff,
+	0x0000006c, 0x80f89078,
+	0xf428403a, 0xf0000000,
+	0xbf89fc07, 0x80fd847d,
+	0xbf800000, 0xbe804300,
+	0xbe824302, 0x80f8a078,
+	0xf42c403a, 0xf0000000,
+	0xbf89fc07, 0x80fd887d,
+	0xbf800000, 0xbe804300,
+	0xbe824302, 0xbe844304,
+	0xbe864306, 0x80f8c078,
+	0xf430403a, 0xf0000000,
+	0xbf89fc07, 0x80fd907d,
+	0xbf800000, 0xbe804300,
+	0xbe824302, 0xbe844304,
+	0xbe864306, 0xbe884308,
+	0xbe8a430a, 0xbe8c430c,
+	0xbe8e430e, 0xbf06807d,
+	0xbfa1fff0, 0xb980f801,
+	0x00000000, 0xbfbd0000,
 	0xb8f83b05, 0x80788178,
 	0xbf0d9972, 0xbfa20002,
 	0x84788978, 0xbfa00001,
 	0x84788a78, 0xb8ee1e06,
 	0x846e8a6e, 0x80786e78,
 	0x8078ff78, 0x00000200,
-	0x80f8ff78, 0x00000050,
 	0xbef600ff, 0x01000000,
-	0xbefd00ff, 0x0000006c,
-	0x80f89078, 0xf428403a,
-	0xf0000000, 0xbf89fc07,
-	0x80fd847d, 0xbf800000,
-	0xbe804300, 0xbe824302,
-	0x80f8a078, 0xf42c403a,
-	0xf0000000, 0xbf89fc07,
-	0x80fd887d, 0xbf800000,
-	0xbe804300, 0xbe824302,
-	0xbe844304, 0xbe864306,
-	0x80f8c078, 0xf430403a,
-	0xf0000000, 0xbf89fc07,
-	0x80fd907d, 0xbf800000,
-	0xbe804300, 0xbe824302,
-	0xbe844304, 0xbe864306,
-	0xbe884308, 0xbe8a430a,
-	0xbe8c430c, 0xbe8e430e,
-	0xbf06807d, 0xbfa1fff0,
-	0xb980f801, 0x00000000,
-	0xbfbd0000, 0xb8f83b05,
-	0x80788178, 0xbf0d9972,
-	0xbfa20002, 0x84788978,
-	0xbfa00001, 0x84788a78,
-	0xb8ee1e06, 0x846e8a6e,
-	0x80786e78, 0x8078ff78,
-	0x00000200, 0xbef600ff,
-	0x01000000, 0xf4205bfa,
+	0xf4205bfa, 0xf0000000,
+	0x80788478, 0xf4205b3a,
 	0xf0000000, 0x80788478,
-	0xf4205b3a, 0xf0000000,
-	0x80788478, 0xf4205b7a,
+	0xf4205b7a, 0xf0000000,
+	0x80788478, 0xf4205c3a,
 	0xf0000000, 0x80788478,
-	0xf4205c3a, 0xf0000000,
-	0x80788478, 0xf4205c7a,
+	0xf4205c7a, 0xf0000000,
+	0x80788478, 0xf4205eba,
 	0xf0000000, 0x80788478,
-	0xf4205eba, 0xf0000000,
-	0x80788478, 0xf4205efa,
+	0xf4205efa, 0xf0000000,
+	0x80788478, 0xf4205e7a,
 	0xf0000000, 0x80788478,
-	0xf4205e7a, 0xf0000000,
-	0x80788478, 0xf4205cfa,
+	0xf4205cfa, 0xf0000000,
+	0x80788478, 0xf4205bba,
 	0xf0000000, 0x80788478,
+	0xbf89fc07, 0xb96ef814,
 	0xf4205bba, 0xf0000000,
 	0x80788478, 0xbf89fc07,
-	0xb96ef814, 0xf4205bba,
-	0xf0000000, 0x80788478,
-	0xbf89fc07, 0xb96ef815,
-	0xbefd006f, 0xbefe0070,
-	0xbeff0071, 0x8b6f7bff,
-	0x000003ff, 0xb96f4803,
-	0x8b6f7bff, 0xfffff800,
-	0x856f8b6f, 0xb96fa2c3,
-	0xb973f801, 0xb8ee3b05,
-	0x806e816e, 0xbf0d9972,
-	0xbfa20002, 0x846e896e,
-	0xbfa00001, 0x846e8a6e,
-	0xb8ef1e06, 0x846f8a6f,
-	0x806e6f6e, 0x806eff6e,
-	0x00000200, 0x806e746e,
-	0x826f8075, 0x8b6fff6f,
-	0x0000ffff, 0xf4085c37,
-	0xf8000050, 0xf4085d37,
-	0xf8000060, 0xf4005e77,
-	0xf8000074, 0xbf89fc07,
-	0x8b6dff6d, 0x0000ffff,
-	0x8bfe7e7e, 0x8bea6a6a,
+	0xb96ef815, 0xbefd006f,
+	0xbefe0070, 0xbeff0071,
+	0x8b6f7bff, 0x000003ff,
+	0xb96f4803, 0x8b6f7bff,
+	0xfffff800, 0x856f8b6f,
+	0xb96fa2c3, 0xb973f801,
+	0xb8ee3b05, 0x806e816e,
+	0xbf0d9972, 0xbfa20002,
+	0x846e896e, 0xbfa00001,
+	0x846e8a6e, 0xb8ef1e06,
+	0x846f8a6f, 0x806e6f6e,
+	0x806eff6e, 0x00000200,
+	0x806e746e, 0x826f8075,
+	0x8b6fff6f, 0x0000ffff,
+	0xf4085c37, 0xf8000050,
+	0xf4085d37, 0xf8000060,
+	0xf4005e77, 0xf8000074,
+	0xbf89fc07, 0x8b6dff6d,
+	0x0000ffff, 0x8bfe7e7e,
+	0x8bea6a6a, 0xb8eef802,
+	0xbf0d866e, 0xbfa20002,
+	0xb97af802, 0xbe80486c,
 	0xb97af802, 0xbe804a6c,
 	0xbfb00000, 0xbf9f0000,
 	0xbf9f0000, 0xbf9f0000,
diff --git a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm
index 250ab007399b..0f81670f6f9c 100644
--- a/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm
+++ b/drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx10.asm
@@ -43,12 +43,14 @@
 #define HAVE_XNACK (ASIC_FAMILY < CHIP_SIENNA_CICHLID)
 #define HAVE_SENDMSG_RTN (ASIC_FAMILY >= CHIP_PLUM_BONITO)
 #define HAVE_BUFFER_LDS_LOAD (ASIC_FAMILY < CHIP_PLUM_BONITO)
+#define SW_SA_TRAP (ASIC_FAMILY >= CHIP_PLUM_BONITO)
 
 var SINGLE_STEP_MISSED_WORKAROUND		= 1	//workaround for lost MODE.DEBUG_EN exception when SAVECTX raised
 
 var SQ_WAVE_STATUS_SPI_PRIO_MASK		= 0x00000006
 var SQ_WAVE_STATUS_HALT_MASK			= 0x2000
 var SQ_WAVE_STATUS_ECC_ERR_MASK			= 0x20000
+var SQ_WAVE_STATUS_TRAP_EN_SHIFT		= 6
 
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT		= 12
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE		= 9
@@ -183,6 +185,13 @@ L_SKIP_RESTORE:
 
 	s_getreg_b32	s_save_trapsts, hwreg(HW_REG_TRAPSTS)
 
+#if SW_SA_TRAP
+	// If ttmp1[31] is set then trap may occur early.
+	// Spin wait until SAVECTX exception is raised.
+	s_bitcmp1_b32	s_save_pc_hi, 31
+	s_cbranch_scc1  L_CHECK_SAVE
+#endif
+
 	s_and_b32       ttmp2, s_save_status, SQ_WAVE_STATUS_HALT_MASK
 	s_cbranch_scc0	L_NOT_HALTED
 
@@ -1061,8 +1070,20 @@ L_RESTORE_HWREG:
 	s_and_b32	s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff		//pc[47:32] //Do it here in order not to affect STATUS
 	s_and_b64	exec, exec, exec					// Restore STATUS.EXECZ, not writable by s_setreg_b32
 	s_and_b64	vcc, vcc, vcc						// Restore STATUS.VCCZ, not writable by s_setreg_b32
+
+#if SW_SA_TRAP
+	// If traps are enabled then return to the shader with PRIV=0.
+	// Otherwise retain PRIV=1 for subsequent context save requests.
+	s_getreg_b32	s_restore_tmp, hwreg(HW_REG_STATUS)
+	s_bitcmp1_b32	s_restore_tmp, SQ_WAVE_STATUS_TRAP_EN_SHIFT
+	s_cbranch_scc1	L_RETURN_WITHOUT_PRIV
+
 	s_setreg_b32	hwreg(HW_REG_STATUS), s_restore_status			// SCC is included, which is changed by previous salu
+	s_setpc_b64	[s_restore_pc_lo, s_restore_pc_hi]
+L_RETURN_WITHOUT_PRIV:
+#endif
 
+	s_setreg_b32	hwreg(HW_REG_STATUS), s_restore_status			// SCC is included, which is changed by previous salu
 	s_rfe_b64	s_restore_pc_lo						//Return to the main shader program and resume execution
 
 L_END_PGM:
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index dc774ddf3445..5feaba6a77de 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -327,6 +327,12 @@ static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
 		goto err_bind_process;
 	}
 
+	if (!pdd->doorbell_index &&
+	    kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
+		err = -ENOMEM;
+		goto err_alloc_doorbells;
+	}
+
 	/* Starting with GFX11, wptr BOs must be mapped to GART for MES to determine work
 	 * on unmapped queues for usermode queue oversubscription (no aggregated doorbell)
 	 */
@@ -404,6 +410,7 @@ err_create_queue:
 	if (wptr_bo)
 		amdgpu_amdkfd_free_gtt_mem(dev->adev, wptr_bo);
 err_wptr_map_gart:
+err_alloc_doorbells:
 err_bind_process:
 err_pdd:
 	mutex_unlock(&p->mutex);
@@ -869,14 +876,11 @@ static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
 				void *data)
 {
 	struct kfd_ioctl_wait_events_args *args = data;
-	int err;
 
-	err = kfd_wait_on_events(p, args->num_events,
+	return kfd_wait_on_events(p, args->num_events,
 			(void __user *)args->events_ptr,
 			(args->wait_for_all != 0),
 			&args->timeout, &args->wait_result);
-
-	return err;
 }
 static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
 					struct kfd_process *p, void *data)
@@ -1092,6 +1096,10 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
 			goto err_unlock;
 		}
 		offset = kfd_get_process_doorbells(pdd);
+		if (!offset) {
+			err = -ENOMEM;
+			goto err_unlock;
+		}
 	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
 		if (args->size != PAGE_SIZE) {
 			err = -EINVAL;
@@ -1576,6 +1584,8 @@ static int kfd_ioctl_smi_events(struct file *filep,
 	return kfd_smi_event_open(pdd->dev, &args->anon_fd);
 }
 
+#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
+
 static int kfd_ioctl_set_xnack_mode(struct file *filep,
 				    struct kfd_process *p, void *data)
 {
@@ -1586,22 +1596,29 @@ static int kfd_ioctl_set_xnack_mode(struct file *filep,
 	if (args->xnack_enabled >= 0) {
 		if (!list_empty(&p->pqm.queues)) {
 			pr_debug("Process has user queues running\n");
-			mutex_unlock(&p->mutex);
-			return -EBUSY;
+			r = -EBUSY;
+			goto out_unlock;
 		}
-		if (args->xnack_enabled && !kfd_process_xnack_mode(p, true))
+
+		if (p->xnack_enabled == args->xnack_enabled)
+			goto out_unlock;
+
+		if (args->xnack_enabled && !kfd_process_xnack_mode(p, true)) {
 			r = -EPERM;
-		else
-			p->xnack_enabled = args->xnack_enabled;
+			goto out_unlock;
+		}
+
+		r = svm_range_switch_xnack_reserve_mem(p, args->xnack_enabled);
 	} else {
 		args->xnack_enabled = p->xnack_enabled;
 	}
+
+out_unlock:
 	mutex_unlock(&p->mutex);
 
 	return r;
 }
 
-#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
 {
 	struct kfd_ioctl_svm_args *args = data;
@@ -1621,6 +1638,11 @@ static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
 	return r;
 }
 #else
+static int kfd_ioctl_set_xnack_mode(struct file *filep,
+				    struct kfd_process *p, void *data)
+{
+	return -EPERM;
+}
 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
 {
 	return -EPERM;
@@ -2145,6 +2167,12 @@ static int criu_restore_devices(struct kfd_process *p,
 			ret = PTR_ERR(pdd);
 			goto exit;
 		}
+
+		if (!pdd->doorbell_index &&
+		    kfd_alloc_process_doorbells(pdd->dev, &pdd->doorbell_index) < 0) {
+			ret = -ENOMEM;
+			goto exit;
+		}
 	}
 
 	/*
@@ -2173,6 +2201,8 @@ static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
 			return -EINVAL;
 
 		offset = kfd_get_process_doorbells(pdd);
+		if (!offset)
+			return -ENOMEM;
 	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
 		/* MMIO BOs need remapped bus address */
 		if (bo_bucket->size != PAGE_SIZE) {
@@ -2847,7 +2877,6 @@ static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
 		      struct vm_area_struct *vma)
 {
 	phys_addr_t address;
-	int ret;
 
 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
 		return -EINVAL;
@@ -2867,12 +2896,11 @@ static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
 		 process->pasid, (unsigned long long) vma->vm_start,
 		 address, vma->vm_flags, PAGE_SIZE);
 
-	ret = io_remap_pfn_range(vma,
+	return io_remap_pfn_range(vma,
 				vma->vm_start,
 				address >> PAGE_SHIFT,
 				PAGE_SIZE,
 				vma->vm_page_prot);
-	return ret;
 }
 
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
index a5409531a2fd..cd5f8b219bf9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_crat.c
@@ -1522,6 +1522,7 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev,
 		case IP_VERSION(11, 0, 0):
 		case IP_VERSION(11, 0, 1):
 		case IP_VERSION(11, 0, 2):
+		case IP_VERSION(11, 0, 3):
 			pcache_info = cache_info;
 			num_of_cache_types =
 				kfd_fill_gpu_cache_info_from_gfx_config(kdev, pcache_info);
@@ -2283,7 +2284,7 @@ static int kfd_create_vcrat_image_gpu(void *pcrat_image,
 
 	/* Fill in Subtype: IO_LINKS
 	 *  Only direct links are added here which is Link from GPU to
-	 *  to its NUMA node. Indirect links are added by userspace.
+	 *  its NUMA node. Indirect links are added by userspace.
 	 */
 	sub_type_hdr = (typeof(sub_type_hdr))((char *)sub_type_hdr +
 		cache_mem_filled);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index 22c0929d410b..65a1d4f9004b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -91,6 +91,7 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
 	case IP_VERSION(6, 0, 0):
 	case IP_VERSION(6, 0, 1):
 	case IP_VERSION(6, 0, 2):
+	case IP_VERSION(6, 0, 3):
 		kfd->device_info.num_sdma_queues_per_engine = 8;
 		break;
 	default:
@@ -103,6 +104,7 @@ static void kfd_device_info_set_sdma_info(struct kfd_dev *kfd)
 	switch (sdma_version) {
 	case IP_VERSION(6, 0, 0):
 	case IP_VERSION(6, 0, 2):
+	case IP_VERSION(6, 0, 3):
 		/* Reserve 1 for paging and 1 for gfx */
 		kfd->device_info.num_reserved_sdma_queues_per_engine = 2;
 		/* BIT(0)=engine-0 queue-0; BIT(1)=engine-1 queue-0; BIT(2)=engine-0 queue-1; ... */
@@ -150,6 +152,7 @@ static void kfd_device_info_set_event_interrupt_class(struct kfd_dev *kfd)
 	case IP_VERSION(11, 0, 0):
 	case IP_VERSION(11, 0, 1):
 	case IP_VERSION(11, 0, 2):
+	case IP_VERSION(11, 0, 3):
 		kfd->device_info.event_interrupt_class = &event_interrupt_class_v11;
 		break;
 	default:
@@ -399,6 +402,11 @@ struct kfd_dev *kgd2kfd_probe(struct amdgpu_device *adev, bool vf)
 			gfx_target_version = 110002;
 			f2g = &gfx_v11_kfd2kgd;
 			break;
+		case IP_VERSION(11, 0, 3):
+			/* Note: Compiler version is 11.0.1 while HW version is 11.0.3 */
+			gfx_target_version = 110001;
+			f2g = &gfx_v11_kfd2kgd;
+			break;
 		default:
 			break;
 		}
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index e83725a28106..ecb4c3abc629 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -205,6 +205,8 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 	}
 
 	queue_input.is_kfd_process = 1;
+	queue_input.is_aql_queue = (q->properties.format == KFD_QUEUE_FORMAT_AQL);
+	queue_input.queue_size = q->properties.queue_size >> 2;
 
 	queue_input.paging = false;
 	queue_input.tba_addr = qpd->tba_addr;
@@ -1240,6 +1242,24 @@ static void init_interrupts(struct device_queue_manager *dqm)
 			dqm->dev->kfd2kgd->init_interrupts(dqm->dev->adev, i);
 }
 
+static void init_sdma_bitmaps(struct device_queue_manager *dqm)
+{
+	unsigned int num_sdma_queues =
+		min_t(unsigned int, sizeof(dqm->sdma_bitmap)*8,
+		      get_num_sdma_queues(dqm));
+	unsigned int num_xgmi_sdma_queues =
+		min_t(unsigned int, sizeof(dqm->xgmi_sdma_bitmap)*8,
+		      get_num_xgmi_sdma_queues(dqm));
+
+	if (num_sdma_queues)
+		dqm->sdma_bitmap = GENMASK_ULL(num_sdma_queues-1, 0);
+	if (num_xgmi_sdma_queues)
+		dqm->xgmi_sdma_bitmap = GENMASK_ULL(num_xgmi_sdma_queues-1, 0);
+
+	dqm->sdma_bitmap &= ~get_reserved_sdma_queues_bitmap(dqm);
+	pr_info("sdma_bitmap: %llx\n", dqm->sdma_bitmap);
+}
+
 static int initialize_nocpsch(struct device_queue_manager *dqm)
 {
 	int pipe, queue;
@@ -1268,11 +1288,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm)
 
 	memset(dqm->vmid_pasid, 0, sizeof(dqm->vmid_pasid));
 
-	dqm->sdma_bitmap = ~0ULL >> (64 - get_num_sdma_queues(dqm));
-	dqm->sdma_bitmap &= ~(get_reserved_sdma_queues_bitmap(dqm));
-	pr_info("sdma_bitmap: %llx\n", dqm->sdma_bitmap);
-
-	dqm->xgmi_sdma_bitmap = ~0ULL >> (64 - get_num_xgmi_sdma_queues(dqm));
+	init_sdma_bitmaps(dqm);
 
 	return 0;
 }
@@ -1450,9 +1466,6 @@ static int set_sched_resources(struct device_queue_manager *dqm)
 
 static int initialize_cpsch(struct device_queue_manager *dqm)
 {
-	uint64_t num_sdma_queues;
-	uint64_t num_xgmi_sdma_queues;
-
 	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
 
 	mutex_init(&dqm->lock_hidden);
@@ -1461,24 +1474,10 @@ static int initialize_cpsch(struct device_queue_manager *dqm)
 	dqm->active_cp_queue_count = 0;
 	dqm->gws_queue_count = 0;
 	dqm->active_runlist = false;
-
-	num_sdma_queues = get_num_sdma_queues(dqm);
-	if (num_sdma_queues >= BITS_PER_TYPE(dqm->sdma_bitmap))
-		dqm->sdma_bitmap = ULLONG_MAX;
-	else
-		dqm->sdma_bitmap = (BIT_ULL(num_sdma_queues) - 1);
-
-	dqm->sdma_bitmap &= ~(get_reserved_sdma_queues_bitmap(dqm));
-	pr_info("sdma_bitmap: %llx\n", dqm->sdma_bitmap);
-
-	num_xgmi_sdma_queues = get_num_xgmi_sdma_queues(dqm);
-	if (num_xgmi_sdma_queues >= BITS_PER_TYPE(dqm->xgmi_sdma_bitmap))
-		dqm->xgmi_sdma_bitmap = ULLONG_MAX;
-	else
-		dqm->xgmi_sdma_bitmap = (BIT_ULL(num_xgmi_sdma_queues) - 1);
-
 	INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
 
+	init_sdma_bitmaps(dqm);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
index cb3d2ccc5100..cd4e61bf0493 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c
@@ -157,6 +157,8 @@ int kfd_doorbell_mmap(struct kfd_dev *dev, struct kfd_process *process,
 
 	/* Calculate physical address of doorbell */
 	address = kfd_get_process_doorbells(pdd);
+	if (!address)
+		return -ENOMEM;
 	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
 				VM_DONTDUMP | VM_PFNMAP;
 
@@ -275,6 +277,13 @@ uint64_t kfd_get_number_elems(struct kfd_dev *kfd)
 
 phys_addr_t kfd_get_process_doorbells(struct kfd_process_device *pdd)
 {
+	if (!pdd->doorbell_index) {
+		int r = kfd_alloc_process_doorbells(pdd->dev,
+						    &pdd->doorbell_index);
+		if (r)
+			return 0;
+	}
+
 	return pdd->dev->doorbell_base +
 		pdd->doorbell_index * kfd_doorbell_process_slice(pdd->dev);
 }
@@ -294,6 +303,9 @@ int kfd_alloc_process_doorbells(struct kfd_dev *kfd, unsigned int *doorbell_inde
 	if (r > 0)
 		*doorbell_index = r;
 
+	if (r < 0)
+		pr_err("Failed to allocate process doorbells\n");
+
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
index a6fcbeeb7428..0d53f6067422 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_int_process_v11.c
@@ -350,11 +350,11 @@ static void event_interrupt_wq_v11(struct kfd_dev *dev,
 				print_sq_intr_info_inst(context_id0, context_id1);
 				sq_int_priv = REG_GET_FIELD(context_id0,
 						SQ_INTERRUPT_WORD_WAVE_CTXID0, PRIV);
-				if (sq_int_priv /*&& (kfd_set_dbg_ev_from_interrupt(dev, pasid,
+				/*if (sq_int_priv && (kfd_set_dbg_ev_from_interrupt(dev, pasid,
 						KFD_CTXID0_DOORBELL_ID(context_id0),
 						KFD_CTXID0_TRAP_CODE(context_id0),
-						NULL, 0))*/)
-					return;
+						NULL, 0)))
+					return;*/
 				break;
 			case SQ_INTERRUPT_WORD_ENCODING_ERROR:
 				print_sq_intr_info_error(context_id0, context_id1);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index b059a77b6081..c70c026c9a93 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -322,12 +322,13 @@ svm_migrate_copy_to_vram(struct amdgpu_device *adev, struct svm_range *prange,
 	for (i = j = 0; i < npages; i++) {
 		struct page *spage;
 
+		dst[i] = cursor.start + (j << PAGE_SHIFT);
+		migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
+		svm_migrate_get_vram_page(prange, migrate->dst[i]);
+		migrate->dst[i] = migrate_pfn(migrate->dst[i]);
+
 		spage = migrate_pfn_to_page(migrate->src[i]);
 		if (spage && !is_zone_device_page(spage)) {
-			dst[i] = cursor.start + (j << PAGE_SHIFT);
-			migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
-			svm_migrate_get_vram_page(prange, migrate->dst[i]);
-			migrate->dst[i] = migrate_pfn(migrate->dst[i]);
 			src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
 					      DMA_TO_DEVICE);
 			r = dma_mapping_error(dev, src[i]);
@@ -522,9 +523,6 @@ svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
 	pr_debug("svms 0x%p [0x%lx 0x%lx] to gpu 0x%x\n", prange->svms,
 		 prange->start, prange->last, best_loc);
 
-	/* FIXME: workaround for page locking bug with invalid pages */
-	svm_range_prefault(prange, mm, SVM_ADEV_PGMAP_OWNER(adev));
-
 	start = prange->start << PAGE_SHIFT;
 	end = (prange->last + 1) << PAGE_SHIFT;
 
@@ -886,7 +884,7 @@ svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
 static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
 {
 	unsigned long addr = vmf->address;
-	struct vm_area_struct *vma;
+	struct svm_range_bo *svm_bo;
 	enum svm_work_list_ops op;
 	struct svm_range *parent;
 	struct svm_range *prange;
@@ -894,29 +892,42 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
 	struct mm_struct *mm;
 	int r = 0;
 
-	vma = vmf->vma;
-	mm = vma->vm_mm;
+	svm_bo = vmf->page->zone_device_data;
+	if (!svm_bo) {
+		pr_debug("failed get device page at addr 0x%lx\n", addr);
+		return VM_FAULT_SIGBUS;
+	}
+	if (!mmget_not_zero(svm_bo->eviction_fence->mm)) {
+		pr_debug("addr 0x%lx of process mm is destroyed\n", addr);
+		return VM_FAULT_SIGBUS;
+	}
+
+	mm = svm_bo->eviction_fence->mm;
+	if (mm != vmf->vma->vm_mm)
+		pr_debug("addr 0x%lx is COW mapping in child process\n", addr);
 
-	p = kfd_lookup_process_by_mm(vma->vm_mm);
+	p = kfd_lookup_process_by_mm(mm);
 	if (!p) {
 		pr_debug("failed find process at fault address 0x%lx\n", addr);
-		return VM_FAULT_SIGBUS;
+		r = VM_FAULT_SIGBUS;
+		goto out_mmput;
 	}
 	if (READ_ONCE(p->svms.faulting_task) == current) {
 		pr_debug("skipping ram migration\n");
-		kfd_unref_process(p);
-		return 0;
+		r = 0;
+		goto out_unref_process;
 	}
-	addr >>= PAGE_SHIFT;
+
 	pr_debug("CPU page fault svms 0x%p address 0x%lx\n", &p->svms, addr);
+	addr >>= PAGE_SHIFT;
 
 	mutex_lock(&p->svms.lock);
 
 	prange = svm_range_from_addr(&p->svms, addr, &parent);
 	if (!prange) {
-		pr_debug("cannot find svm range at 0x%lx\n", addr);
+		pr_debug("failed get range svms 0x%p addr 0x%lx\n", &p->svms, addr);
 		r = -EFAULT;
-		goto out;
+		goto out_unlock_svms;
 	}
 
 	mutex_lock(&parent->migrate_mutex);
@@ -938,10 +949,11 @@ static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
 		goto out_unlock_prange;
 	}
 
-	r = svm_migrate_vram_to_ram(prange, mm, KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU);
+	r = svm_migrate_vram_to_ram(prange, vmf->vma->vm_mm,
+				    KFD_MIGRATE_TRIGGER_PAGEFAULT_CPU);
 	if (r)
-		pr_debug("failed %d migrate 0x%p [0x%lx 0x%lx] to ram\n", r,
-			 prange, prange->start, prange->last);
+		pr_debug("failed %d migrate svms 0x%p range 0x%p [0x%lx 0x%lx]\n",
+			 r, prange->svms, prange, prange->start, prange->last);
 
 	/* xnack on, update mapping on GPUs with ACCESS_IN_PLACE */
 	if (p->xnack_enabled && parent == prange)
@@ -955,9 +967,12 @@ out_unlock_prange:
 	if (prange != parent)
 		mutex_unlock(&prange->migrate_mutex);
 	mutex_unlock(&parent->migrate_mutex);
-out:
+out_unlock_svms:
 	mutex_unlock(&p->svms.lock);
+out_unref_process:
 	kfd_unref_process(p);
+out_mmput:
+	mmput(mm);
 
 	pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
index b8e14c2cc295..26b53b6d673e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
@@ -126,6 +126,10 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 	m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF;
 	m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF;
 	m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se4 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se5 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se6 = 0xFFFFFFFF;
+	m->compute_static_thread_mgmt_se7 = 0xFFFFFFFF;
 
 	m->cp_hqd_persistent_state = CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK |
 			0x55 << CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT;
@@ -177,14 +181,6 @@ static int load_mqd(struct mqd_manager *mm, void *mqd,
 	return r;
 }
 
-static int hiq_load_mqd_kiq(struct mqd_manager *mm, void *mqd,
-			    uint32_t pipe_id, uint32_t queue_id,
-			    struct queue_properties *p, struct mm_struct *mms)
-{
-	return mm->dev->kfd2kgd->hiq_mqd_load(mm->dev->adev, mqd, pipe_id,
-					      queue_id, p->doorbell_off);
-}
-
 static void update_mqd(struct mqd_manager *mm, void *mqd,
 		       struct queue_properties *q,
 		       struct mqd_update_info *minfo)
@@ -256,31 +252,6 @@ static uint32_t read_doorbell_id(void *mqd)
 	return m->queue_doorbell_id0;
 }
 
-static int destroy_mqd(struct mqd_manager *mm, void *mqd,
-		       enum kfd_preempt_type type,
-		       unsigned int timeout, uint32_t pipe_id,
-		       uint32_t queue_id)
-{
-	return mm->dev->kfd2kgd->hqd_destroy
-		(mm->dev->adev, mqd, type, timeout,
-		 pipe_id, queue_id);
-}
-
-static void free_mqd(struct mqd_manager *mm, void *mqd,
-			struct kfd_mem_obj *mqd_mem_obj)
-{
-	kfd_gtt_sa_free(mm->dev, mqd_mem_obj);
-}
-
-static bool is_occupied(struct mqd_manager *mm, void *mqd,
-			uint64_t queue_address,	uint32_t pipe_id,
-			uint32_t queue_id)
-{
-	return mm->dev->kfd2kgd->hqd_is_occupied(
-		mm->dev->adev, queue_address,
-		pipe_id, queue_id);
-}
-
 static int get_wave_state(struct mqd_manager *mm, void *mqd,
 			  void __user *ctl_stack,
 			  u32 *ctl_stack_used_size,
@@ -349,15 +320,6 @@ static void init_mqd_sdma(struct mqd_manager *mm, void **mqd,
 	mm->update_mqd(mm, m, q, NULL);
 }
 
-static int load_mqd_sdma(struct mqd_manager *mm, void *mqd,
-		uint32_t pipe_id, uint32_t queue_id,
-		struct queue_properties *p, struct mm_struct *mms)
-{
-	return mm->dev->kfd2kgd->hqd_sdma_load(mm->dev->adev, mqd,
-					       (uint32_t __user *)p->write_ptr,
-					       mms);
-}
-
 #define SDMA_RLC_DUMMY_DEFAULT 0xf
 
 static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
@@ -389,25 +351,6 @@ static void update_mqd_sdma(struct mqd_manager *mm, void *mqd,
 	q->is_active = QUEUE_IS_ACTIVE(*q);
 }
 
-/*
- *  * preempt type here is ignored because there is only one way
- *  * to preempt sdma queue
- */
-static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd,
-		enum kfd_preempt_type type,
-		unsigned int timeout, uint32_t pipe_id,
-		uint32_t queue_id)
-{
-	return mm->dev->kfd2kgd->hqd_sdma_destroy(mm->dev->adev, mqd, timeout);
-}
-
-static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd,
-		uint64_t queue_address, uint32_t pipe_id,
-		uint32_t queue_id)
-{
-	return mm->dev->kfd2kgd->hqd_sdma_is_occupied(mm->dev->adev, mqd);
-}
-
 #if defined(CONFIG_DEBUG_FS)
 
 static int debugfs_show_mqd(struct seq_file *m, void *data)
@@ -445,11 +388,11 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
 		pr_debug("%s@%i\n", __func__, __LINE__);
 		mqd->allocate_mqd = allocate_mqd;
 		mqd->init_mqd = init_mqd;
-		mqd->free_mqd = free_mqd;
+		mqd->free_mqd = kfd_free_mqd_cp;
 		mqd->load_mqd = load_mqd;
 		mqd->update_mqd = update_mqd;
-		mqd->destroy_mqd = destroy_mqd;
-		mqd->is_occupied = is_occupied;
+		mqd->destroy_mqd = kfd_destroy_mqd_cp;
+		mqd->is_occupied = kfd_is_occupied_cp;
 		mqd->mqd_size = sizeof(struct v11_compute_mqd);
 		mqd->get_wave_state = get_wave_state;
 #if defined(CONFIG_DEBUG_FS)
@@ -462,10 +405,10 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
 		mqd->allocate_mqd = allocate_hiq_mqd;
 		mqd->init_mqd = init_mqd_hiq;
 		mqd->free_mqd = free_mqd_hiq_sdma;
-		mqd->load_mqd = hiq_load_mqd_kiq;
+		mqd->load_mqd = kfd_hiq_load_mqd_kiq;
 		mqd->update_mqd = update_mqd;
-		mqd->destroy_mqd = destroy_mqd;
-		mqd->is_occupied = is_occupied;
+		mqd->destroy_mqd = kfd_destroy_mqd_cp;
+		mqd->is_occupied = kfd_is_occupied_cp;
 		mqd->mqd_size = sizeof(struct v11_compute_mqd);
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
@@ -476,11 +419,11 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
 	case KFD_MQD_TYPE_DIQ:
 		mqd->allocate_mqd = allocate_mqd;
 		mqd->init_mqd = init_mqd_hiq;
-		mqd->free_mqd = free_mqd;
+		mqd->free_mqd = kfd_free_mqd_cp;
 		mqd->load_mqd = load_mqd;
 		mqd->update_mqd = update_mqd;
-		mqd->destroy_mqd = destroy_mqd;
-		mqd->is_occupied = is_occupied;
+		mqd->destroy_mqd = kfd_destroy_mqd_cp;
+		mqd->is_occupied = kfd_is_occupied_cp;
 		mqd->mqd_size = sizeof(struct v11_compute_mqd);
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd;
@@ -491,10 +434,10 @@ struct mqd_manager *mqd_manager_init_v11(enum KFD_MQD_TYPE type,
 		mqd->allocate_mqd = allocate_sdma_mqd;
 		mqd->init_mqd = init_mqd_sdma;
 		mqd->free_mqd = free_mqd_hiq_sdma;
-		mqd->load_mqd = load_mqd_sdma;
+		mqd->load_mqd = kfd_load_mqd_sdma;
 		mqd->update_mqd = update_mqd_sdma;
-		mqd->destroy_mqd = destroy_mqd_sdma;
-		mqd->is_occupied = is_occupied_sdma;
+		mqd->destroy_mqd = kfd_destroy_mqd_sdma;
+		mqd->is_occupied = kfd_is_occupied_sdma;
 		mqd->mqd_size = sizeof(struct v11_sdma_mqd);
 #if defined(CONFIG_DEBUG_FS)
 		mqd->debugfs_show_mqd = debugfs_show_mqd_sdma;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 6c83a519b3a1..951b63677248 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -1499,11 +1499,6 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
 	if (!pdd)
 		return NULL;
 
-	if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
-		pr_err("Failed to alloc doorbell for pdd\n");
-		goto err_free_pdd;
-	}
-
 	if (init_doorbell_bitmap(&pdd->qpd, dev)) {
 		pr_err("Failed to init doorbell for process\n");
 		goto err_free_pdd;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
index 6e3e7f54381b..5137476ec18e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c
@@ -857,6 +857,13 @@ int kfd_criu_restore_queue(struct kfd_process *p,
 		ret = -EINVAL;
 		goto exit;
 	}
+
+	if (!pdd->doorbell_index &&
+	    kfd_alloc_process_doorbells(pdd->dev, &pdd->doorbell_index) < 0) {
+		ret = -ENOMEM;
+		goto exit;
+	}
+
 	/* data stored in this order: mqd, ctl_stack */
 	mqd = q_extra_data;
 	ctl_stack = mqd + q_data->mqd_size;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
index 11074cc8c333..f5913ba22174 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c
@@ -278,7 +278,7 @@ static void svm_range_free(struct svm_range *prange, bool update_mem_usage)
 	svm_range_free_dma_mappings(prange);
 
 	if (update_mem_usage && !p->xnack_enabled) {
-		pr_debug("unreserve mem limit: %lld\n", size);
+		pr_debug("unreserve prange 0x%p size: 0x%llx\n", prange, size);
 		amdgpu_amdkfd_unreserve_mem_limit(NULL, size,
 					KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
 	}
@@ -2956,6 +2956,64 @@ out:
 	return r;
 }
 
+int
+svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled)
+{
+	struct svm_range *prange, *pchild;
+	uint64_t reserved_size = 0;
+	uint64_t size;
+	int r = 0;
+
+	pr_debug("switching xnack from %d to %d\n", p->xnack_enabled, xnack_enabled);
+
+	mutex_lock(&p->svms.lock);
+
+	list_for_each_entry(prange, &p->svms.list, list) {
+		svm_range_lock(prange);
+		list_for_each_entry(pchild, &prange->child_list, child_list) {
+			size = (pchild->last - pchild->start + 1) << PAGE_SHIFT;
+			if (xnack_enabled) {
+				amdgpu_amdkfd_unreserve_mem_limit(NULL, size,
+						KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+			} else {
+				r = amdgpu_amdkfd_reserve_mem_limit(NULL, size,
+						KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+				if (r)
+					goto out_unlock;
+				reserved_size += size;
+			}
+		}
+
+		size = (prange->last - prange->start + 1) << PAGE_SHIFT;
+		if (xnack_enabled) {
+			amdgpu_amdkfd_unreserve_mem_limit(NULL, size,
+						KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+		} else {
+			r = amdgpu_amdkfd_reserve_mem_limit(NULL, size,
+						KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+			if (r)
+				goto out_unlock;
+			reserved_size += size;
+		}
+out_unlock:
+		svm_range_unlock(prange);
+		if (r)
+			break;
+	}
+
+	if (r)
+		amdgpu_amdkfd_unreserve_mem_limit(NULL, reserved_size,
+						KFD_IOC_ALLOC_MEM_FLAGS_USERPTR);
+	else
+		/* Change xnack mode must be inside svms lock, to avoid race with
+		 * svm_range_deferred_list_work unreserve memory in parallel.
+		 */
+		p->xnack_enabled = xnack_enabled;
+
+	mutex_unlock(&p->svms.lock);
+	return r;
+}
+
 void svm_range_list_fini(struct kfd_process *p)
 {
 	struct svm_range *prange;
@@ -3181,28 +3239,6 @@ out:
 	return best_loc;
 }
 
-/* FIXME: This is a workaround for page locking bug when some pages are
- * invalid during migration to VRAM
- */
-void svm_range_prefault(struct svm_range *prange, struct mm_struct *mm,
-			void *owner)
-{
-	struct hmm_range *hmm_range;
-	int r;
-
-	if (prange->validated_once)
-		return;
-
-	r = amdgpu_hmm_range_get_pages(&prange->notifier, mm, NULL,
-				       prange->start << PAGE_SHIFT,
-				       prange->npages, &hmm_range,
-				       false, true, owner);
-	if (!r) {
-		amdgpu_hmm_range_get_pages_done(hmm_range);
-		prange->validated_once = true;
-	}
-}
-
 /* svm_range_trigger_migration - start page migration if prefetch loc changed
  * @mm: current process mm_struct
  * @prange: svm range structure
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
index cfac13ad06ef..7a33b93f9df6 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.h
@@ -181,8 +181,6 @@ void schedule_deferred_list_work(struct svm_range_list *svms);
 void svm_range_dma_unmap(struct device *dev, dma_addr_t *dma_addr,
 			 unsigned long offset, unsigned long npages);
 void svm_range_free_dma_mappings(struct svm_range *prange);
-void svm_range_prefault(struct svm_range *prange, struct mm_struct *mm,
-			void *owner);
 int svm_range_get_info(struct kfd_process *p, uint32_t *num_svm_ranges,
 		       uint64_t *svm_priv_data_size);
 int kfd_criu_checkpoint_svm(struct kfd_process *p,
@@ -205,6 +203,7 @@ void svm_range_list_lock_and_flush_work(struct svm_range_list *svms, struct mm_s
 void svm_range_bo_unref_async(struct svm_range_bo *svm_bo);
 
 void svm_range_set_max_pages(struct amdgpu_device *adev);
+int svm_range_switch_xnack_reserve_mem(struct kfd_process *p, bool xnack_enabled);
 
 #else
 
diff --git a/drivers/gpu/drm/amd/display/Kconfig b/drivers/gpu/drm/amd/display/Kconfig
index 413d8c6d592f..6925e0280dbe 100644
--- a/drivers/gpu/drm/amd/display/Kconfig
+++ b/drivers/gpu/drm/amd/display/Kconfig
@@ -28,7 +28,6 @@ config DRM_AMD_DC_SI
 	bool "AMD DC support for Southern Islands ASICs"
 	depends on DRM_AMDGPU_SI
 	depends on DRM_AMD_DC
-	default n
 	help
 	  Choose this option to enable new AMD DC support for SI asics
 	  by default. This includes Tahiti, Pitcairn, Cape Verde, Oland.
@@ -43,7 +42,6 @@ config DEBUG_KERNEL_DC
 
 config DRM_AMD_SECURE_DISPLAY
         bool "Enable secure display support"
-        default n
         depends on DEBUG_FS
         depends on DRM_AMD_DC_DCN
         help
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index 1efe7fa5bc58..4c73727e0b7d 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -88,6 +88,9 @@
 #include <drm/drm_vblank.h>
 #include <drm/drm_audio_component.h>
 #include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_plane_helper.h>
+
+#include <acpi/video.h>
 
 #include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
 
@@ -97,8 +100,6 @@
 #include "soc15_common.h"
 #include "vega10_ip_offset.h"
 
-#include "soc15_common.h"
-
 #include "gc/gc_11_0_0_offset.h"
 #include "gc/gc_11_0_0_sh_mask.h"
 
@@ -1295,13 +1296,21 @@ static struct hpd_rx_irq_offload_work_queue *hpd_rx_irq_create_workqueue(struct
 
 		if (hpd_rx_offload_wq[i].wq == NULL) {
 			DRM_ERROR("create amdgpu_dm_hpd_rx_offload_wq fail!");
-			return NULL;
+			goto out_err;
 		}
 
 		spin_lock_init(&hpd_rx_offload_wq[i].offload_lock);
 	}
 
 	return hpd_rx_offload_wq;
+
+out_err:
+	for (i = 0; i < max_caps; i++) {
+		if (hpd_rx_offload_wq[i].wq)
+			destroy_workqueue(hpd_rx_offload_wq[i].wq);
+	}
+	kfree(hpd_rx_offload_wq);
+	return NULL;
 }
 
 struct amdgpu_stutter_quirk {
@@ -1529,7 +1538,6 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 
 	if (amdgpu_dc_debug_mask & DC_DISABLE_DSC) {
 		adev->dm.dc->debug.disable_dsc = true;
-		adev->dm.dc->debug.disable_dsc_edp = true;
 	}
 
 	if (amdgpu_dc_debug_mask & DC_DISABLE_CLOCK_GATING)
@@ -2807,20 +2815,18 @@ static const struct drm_mode_config_funcs amdgpu_dm_mode_funcs = {
 };
 
 static struct drm_mode_config_helper_funcs amdgpu_dm_mode_config_helperfuncs = {
-	.atomic_commit_tail = amdgpu_dm_atomic_commit_tail
+	.atomic_commit_tail = amdgpu_dm_atomic_commit_tail,
+	.atomic_commit_setup = drm_dp_mst_atomic_setup_commit,
 };
 
 static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector)
 {
-	u32 max_avg, min_cll, max, min, q, r;
 	struct amdgpu_dm_backlight_caps *caps;
 	struct amdgpu_display_manager *dm;
 	struct drm_connector *conn_base;
 	struct amdgpu_device *adev;
 	struct dc_link *link = NULL;
-	static const u8 pre_computed_values[] = {
-		50, 51, 52, 53, 55, 56, 57, 58, 59, 61, 62, 63, 65, 66, 68, 69,
-		71, 72, 74, 75, 77, 79, 81, 82, 84, 86, 88, 90, 92, 94, 96, 98};
+	struct drm_luminance_range_info *luminance_range;
 	int i;
 
 	if (!aconnector || !aconnector->dc_link)
@@ -2842,8 +2848,6 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector)
 	caps = &dm->backlight_caps[i];
 	caps->ext_caps = &aconnector->dc_link->dpcd_sink_ext_caps;
 	caps->aux_support = false;
-	max_avg = conn_base->hdr_sink_metadata.hdmi_type1.max_fall;
-	min_cll = conn_base->hdr_sink_metadata.hdmi_type1.min_cll;
 
 	if (caps->ext_caps->bits.oled == 1 /*||
 	    caps->ext_caps->bits.sdr_aux_backlight_control == 1 ||
@@ -2855,31 +2859,9 @@ static void update_connector_ext_caps(struct amdgpu_dm_connector *aconnector)
 	else if (amdgpu_backlight == 1)
 		caps->aux_support = true;
 
-	/* From the specification (CTA-861-G), for calculating the maximum
-	 * luminance we need to use:
-	 *	Luminance = 50*2**(CV/32)
-	 * Where CV is a one-byte value.
-	 * For calculating this expression we may need float point precision;
-	 * to avoid this complexity level, we take advantage that CV is divided
-	 * by a constant. From the Euclids division algorithm, we know that CV
-	 * can be written as: CV = 32*q + r. Next, we replace CV in the
-	 * Luminance expression and get 50*(2**q)*(2**(r/32)), hence we just
-	 * need to pre-compute the value of r/32. For pre-computing the values
-	 * We just used the following Ruby line:
-	 *	(0...32).each {|cv| puts (50*2**(cv/32.0)).round}
-	 * The results of the above expressions can be verified at
-	 * pre_computed_values.
-	 */
-	q = max_avg >> 5;
-	r = max_avg % 32;
-	max = (1 << q) * pre_computed_values[r];
-
-	// min luminance: maxLum * (CV/255)^2 / 100
-	q = DIV_ROUND_CLOSEST(min_cll, 255);
-	min = max * DIV_ROUND_CLOSEST((q * q), 100);
-
-	caps->aux_max_input_signal = max;
-	caps->aux_min_input_signal = min;
+	luminance_range = &conn_base->display_info.luminance_range;
+	caps->aux_min_input_signal = luminance_range->min_luminance;
+	caps->aux_max_input_signal = luminance_range->max_luminance;
 }
 
 void amdgpu_dm_update_connector_after_detect(
@@ -4058,6 +4040,13 @@ amdgpu_dm_register_backlight_device(struct amdgpu_display_manager *dm)
 	amdgpu_dm_update_backlight_caps(dm, dm->num_of_edps);
 	dm->brightness[dm->num_of_edps] = AMDGPU_MAX_BL_LEVEL;
 
+	if (!acpi_video_backlight_use_native()) {
+		drm_info(adev_to_drm(dm->adev), "Skipping amdgpu DM backlight registration\n");
+		/* Try registering an ACPI video backlight device instead. */
+		acpi_video_register_backlight();
+		return;
+	}
+
 	props.max_brightness = AMDGPU_MAX_BL_LEVEL;
 	props.brightness = AMDGPU_MAX_BL_LEVEL;
 	props.type = BACKLIGHT_RAW;
@@ -5629,7 +5618,8 @@ static void apply_dsc_policy_for_stream(struct amdgpu_dm_connector *aconnector,
 	dc_dsc_policy_set_enable_dsc_when_not_needed(
 		aconnector->dsc_settings.dsc_force_enable == DSC_CLK_FORCE_ENABLE);
 
-	if (aconnector->dc_link && sink->sink_signal == SIGNAL_TYPE_EDP && !dc->debug.disable_dsc_edp &&
+	if (aconnector->dc_link && sink->sink_signal == SIGNAL_TYPE_EDP &&
+	    !aconnector->dc_link->panel_config.dsc.disable_dsc_edp &&
 	    dc->caps.edp_dsc_support && aconnector->dsc_settings.dsc_force_enable != DSC_CLK_FORCE_DISABLE) {
 
 		apply_dsc_policy_for_edp(aconnector, sink, stream, dsc_caps, max_dsc_target_bpp_limit_override);
@@ -6321,10 +6311,17 @@ amdgpu_dm_connector_atomic_check(struct drm_connector *conn,
 		drm_atomic_get_old_connector_state(state, conn);
 	struct drm_crtc *crtc = new_con_state->crtc;
 	struct drm_crtc_state *new_crtc_state;
+	struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(conn);
 	int ret;
 
 	trace_amdgpu_dm_connector_atomic_check(new_con_state);
 
+	if (conn->connector_type == DRM_MODE_CONNECTOR_DisplayPort) {
+		ret = drm_dp_mst_root_conn_atomic_check(new_con_state, &aconn->mst_mgr);
+		if (ret < 0)
+			return ret;
+	}
+
 	if (!crtc)
 		return 0;
 
@@ -6408,6 +6405,7 @@ static int dm_encoder_helper_atomic_check(struct drm_encoder *encoder,
 	const struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
 	struct drm_dp_mst_topology_mgr *mst_mgr;
 	struct drm_dp_mst_port *mst_port;
+	struct drm_dp_mst_topology_state *mst_state;
 	enum dc_color_depth color_depth;
 	int clock, bpp = 0;
 	bool is_y420 = false;
@@ -6421,6 +6419,13 @@ static int dm_encoder_helper_atomic_check(struct drm_encoder *encoder,
 	if (!crtc_state->connectors_changed && !crtc_state->mode_changed)
 		return 0;
 
+	mst_state = drm_atomic_get_mst_topology_state(state, mst_mgr);
+	if (IS_ERR(mst_state))
+		return PTR_ERR(mst_state);
+
+	if (!mst_state->pbn_div)
+		mst_state->pbn_div = dm_mst_get_pbn_divider(aconnector->mst_port->dc_link);
+
 	if (!state->duplicated) {
 		int max_bpc = conn_state->max_requested_bpc;
 		is_y420 = drm_mode_is_420_also(&connector->display_info, adjusted_mode) &&
@@ -6432,11 +6437,10 @@ static int dm_encoder_helper_atomic_check(struct drm_encoder *encoder,
 		clock = adjusted_mode->clock;
 		dm_new_connector_state->pbn = drm_dp_calc_pbn_mode(clock, bpp, false);
 	}
-	dm_new_connector_state->vcpi_slots = drm_dp_atomic_find_vcpi_slots(state,
-									   mst_mgr,
-									   mst_port,
-									   dm_new_connector_state->pbn,
-									   dm_mst_get_pbn_divider(aconnector->dc_link));
+
+	dm_new_connector_state->vcpi_slots =
+		drm_dp_atomic_find_time_slots(state, mst_mgr, mst_port,
+					      dm_new_connector_state->pbn);
 	if (dm_new_connector_state->vcpi_slots < 0) {
 		DRM_DEBUG_ATOMIC("failed finding vcpi slots: %d\n", (int)dm_new_connector_state->vcpi_slots);
 		return dm_new_connector_state->vcpi_slots;
@@ -6506,18 +6510,12 @@ static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_state *state,
 			dm_conn_state->pbn = pbn;
 			dm_conn_state->vcpi_slots = slot_num;
 
-			drm_dp_mst_atomic_enable_dsc(state,
-						     aconnector->port,
-						     dm_conn_state->pbn,
-						     0,
+			drm_dp_mst_atomic_enable_dsc(state, aconnector->port, dm_conn_state->pbn,
 						     false);
 			continue;
 		}
 
-		vcpi = drm_dp_mst_atomic_enable_dsc(state,
-						    aconnector->port,
-						    pbn, pbn_div,
-						    true);
+		vcpi = drm_dp_mst_atomic_enable_dsc(state, aconnector->port, pbn, true);
 		if (vcpi < 0)
 			return vcpi;
 
@@ -7387,11 +7385,6 @@ static void update_freesync_state_on_stream(
 		&vrr_infopacket,
 		pack_sdp_v1_3);
 
-	new_crtc_state->freesync_timing_changed |=
-		(memcmp(&acrtc->dm_irq_params.vrr_params.adjust,
-			&vrr_params.adjust,
-			sizeof(vrr_params.adjust)) != 0);
-
 	new_crtc_state->freesync_vrr_info_changed |=
 		(memcmp(&new_crtc_state->vrr_infopacket,
 			&vrr_infopacket,
@@ -7400,7 +7393,6 @@ static void update_freesync_state_on_stream(
 	acrtc->dm_irq_params.vrr_params = vrr_params;
 	new_crtc_state->vrr_infopacket = vrr_infopacket;
 
-	new_stream->adjust = acrtc->dm_irq_params.vrr_params.adjust;
 	new_stream->vrr_infopacket = vrr_infopacket;
 
 	if (new_crtc_state->freesync_vrr_info_changed)
@@ -7463,10 +7455,6 @@ static void update_stream_irq_parameters(
 				      new_stream,
 				      &config, &vrr_params);
 
-	new_crtc_state->freesync_timing_changed |=
-		(memcmp(&acrtc->dm_irq_params.vrr_params.adjust,
-			&vrr_params.adjust, sizeof(vrr_params.adjust)) != 0);
-
 	new_crtc_state->freesync_config = config;
 	/* Copy state for access from DM IRQ handler */
 	acrtc->dm_irq_params.freesync_config = config;
@@ -7992,6 +7980,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 		DRM_ERROR("Waiting for fences timed out!");
 
 	drm_atomic_helper_update_legacy_modeset_state(dev, state);
+	drm_dp_mst_atomic_wait_for_dependencies(state);
 
 	dm_state = dm_atomic_get_new_state(state);
 	if (dm_state && dm_state->context) {
@@ -8390,7 +8379,6 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 		dc_release_state(dc_state_temp);
 }
 
-
 static int dm_force_atomic_commit(struct drm_connector *connector)
 {
 	int ret = 0;
@@ -9321,6 +9309,7 @@ static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm
 
 /**
  * amdgpu_dm_atomic_check() - Atomic check implementation for AMDgpu DM.
+ *
  * @dev: The DRM device
  * @state: The atomic state to commit
  *
@@ -9361,8 +9350,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 	struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state;
 #if defined(CONFIG_DRM_AMD_DC_DCN)
 	struct dsc_mst_fairness_vars vars[MAX_PIPES];
-	struct drm_dp_mst_topology_state *mst_state;
-	struct drm_dp_mst_topology_mgr *mgr;
 #endif
 
 	trace_amdgpu_dm_atomic_check_begin(state);
@@ -9379,9 +9366,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		struct dm_connector_state *dm_new_con_state = to_dm_connector_state(new_con_state);
 
 		/* Skip connectors that are disabled or part of modeset already. */
-		if (!old_con_state->crtc && !new_con_state->crtc)
-			continue;
-
 		if (!new_con_state->crtc)
 			continue;
 
@@ -9609,33 +9593,6 @@ static int amdgpu_dm_atomic_check(struct drm_device *dev,
 		lock_and_validation_needed = true;
 	}
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
-	/* set the slot info for each mst_state based on the link encoding format */
-	for_each_new_mst_mgr_in_state(state, mgr, mst_state, i) {
-		struct amdgpu_dm_connector *aconnector;
-		struct drm_connector *connector;
-		struct drm_connector_list_iter iter;
-		u8 link_coding_cap;
-
-		if (!mgr->mst_state )
-			continue;
-
-		drm_connector_list_iter_begin(dev, &iter);
-		drm_for_each_connector_iter(connector, &iter) {
-			int id = connector->index;
-
-			if (id == mst_state->mgr->conn_base_id) {
-				aconnector = to_amdgpu_dm_connector(connector);
-				link_coding_cap = dc_link_dp_mst_decide_link_encoding_format(aconnector->dc_link);
-				drm_dp_mst_update_slots(mst_state, link_coding_cap);
-
-				break;
-			}
-		}
-		drm_connector_list_iter_end(&iter);
-
-	}
-#endif
 	/**
 	 * Streams and planes are reset when there are changes that affect
 	 * bandwidth. Anything that affects bandwidth needs to go through
@@ -9943,8 +9900,19 @@ static int parse_hdmi_amd_vsdb(struct amdgpu_dm_connector *aconnector,
 	return valid_vsdb_found ? i : -ENODEV;
 }
 
+/**
+ * amdgpu_dm_update_freesync_caps - Update Freesync capabilities
+ *
+ * @connector: Connector to query.
+ * @edid: EDID from monitor
+ *
+ * Amdgpu supports Freesync in DP and HDMI displays, and it is required to keep
+ * track of some of the display information in the internal data struct used by
+ * amdgpu_dm. This function checks which type of connector we need to set the
+ * FreeSync parameters.
+ */
 void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
-					struct edid *edid)
+				    struct edid *edid)
 {
 	int i = 0;
 	struct detailed_timing *timing;
@@ -9957,8 +9925,8 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
 
 	struct drm_device *dev = connector->dev;
 	struct amdgpu_device *adev = drm_to_adev(dev);
-	bool freesync_capable = false;
 	struct amdgpu_hdmi_vsdb_info vsdb_info = {0};
+	bool freesync_capable = false;
 
 	if (!connector->state) {
 		DRM_ERROR("%s - Connector has no state", __func__);
@@ -9987,7 +9955,6 @@ void amdgpu_dm_update_freesync_caps(struct drm_connector *connector,
 	if (!adev->dm.freesync_module)
 		goto update;
 
-
 	if (sink->sink_signal == SIGNAL_TYPE_DISPLAY_PORT
 		|| sink->sink_signal == SIGNAL_TYPE_EDP) {
 		bool edid_check_required = false;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index 90b306a1dd68..b5ce15c43bcc 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -598,6 +598,10 @@ struct amdgpu_dm_connector {
 	 * The 'current' sink is in dc_link->sink. */
 	struct dc_sink *dc_sink;
 	struct dc_link *dc_link;
+
+	/**
+	 * @dc_em_sink: Reference to the emulated (virtual) sink.
+	 */
 	struct dc_sink *dc_em_sink;
 
 	/* DM only */
@@ -610,7 +614,16 @@ struct amdgpu_dm_connector {
 	struct amdgpu_i2c_adapter *i2c;
 
 	/* Monitor range limits */
-	int min_vfreq ;
+	/**
+	 * @min_vfreq: Minimal frequency supported by the display in Hz. This
+	 * value is set to zero when there is no FreeSync support.
+	 */
+	int min_vfreq;
+
+	/**
+	 * @max_vfreq: Maximum frequency supported by the display in Hz. This
+	 * value is set to zero when there is no FreeSync support.
+	 */
 	int max_vfreq ;
 	int pixel_clock_mhz;
 
@@ -668,7 +681,6 @@ struct dm_crtc_state {
 
 	int crc_skip_count;
 
-	bool freesync_timing_changed;
 	bool freesync_vrr_info_changed;
 
 	bool dsc_force_changed;
@@ -705,11 +717,34 @@ struct dm_connector_state {
 	uint64_t pbn;
 };
 
+/**
+ * struct amdgpu_hdmi_vsdb_info - Keep track of the VSDB info
+ *
+ * AMDGPU supports FreeSync over HDMI by using the VSDB section, and this
+ * struct is useful to keep track of the display-specific information about
+ * FreeSync.
+ */
 struct amdgpu_hdmi_vsdb_info {
-	unsigned int amd_vsdb_version;		/* VSDB version, should be used to determine which VSIF to send */
-	bool freesync_supported;		/* FreeSync Supported */
-	unsigned int min_refresh_rate_hz;	/* FreeSync Minimum Refresh Rate in Hz */
-	unsigned int max_refresh_rate_hz;	/* FreeSync Maximum Refresh Rate in Hz */
+	/**
+	 * @amd_vsdb_version: Vendor Specific Data Block Version, should be
+	 * used to determine which Vendor Specific InfoFrame (VSIF) to send.
+	 */
+	unsigned int amd_vsdb_version;
+
+	/**
+	 * @freesync_supported: FreeSync Supported.
+	 */
+	bool freesync_supported;
+
+	/**
+	 * @min_refresh_rate_hz: FreeSync Minimum Refresh Rate in Hz.
+	 */
+	unsigned int min_refresh_rate_hz;
+
+	/**
+	 * @max_refresh_rate_hz: FreeSync Maximum Refresh Rate in Hz
+	 */
+	unsigned int max_refresh_rate_hz;
 };
 
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
index a71177305bcd..a4cb23d059bd 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
@@ -29,7 +29,9 @@
 #include "modules/color/color_gamma.h"
 #include "basics/conversion.h"
 
-/*
+/**
+ * DOC: overview
+ *
  * The DC interface to HW gives us the following color management blocks
  * per pipe (surface):
  *
@@ -71,8 +73,8 @@
 
 #define MAX_DRM_LUT_VALUE 0xFFFF
 
-/*
- * Initialize the color module.
+/**
+ * amdgpu_dm_init_color_mod - Initialize the color module.
  *
  * We're not using the full color module, only certain components.
  * Only call setup functions for components that we need.
@@ -82,7 +84,14 @@ void amdgpu_dm_init_color_mod(void)
 	setup_x_points_distribution();
 }
 
-/* Extracts the DRM lut and lut size from a blob. */
+/**
+ * __extract_blob_lut - Extracts the DRM lut and lut size from a blob.
+ * @blob: DRM color mgmt property blob
+ * @size: lut size
+ *
+ * Returns:
+ * DRM LUT or NULL
+ */
 static const struct drm_color_lut *
 __extract_blob_lut(const struct drm_property_blob *blob, uint32_t *size)
 {
@@ -90,13 +99,18 @@ __extract_blob_lut(const struct drm_property_blob *blob, uint32_t *size)
 	return blob ? (struct drm_color_lut *)blob->data : NULL;
 }
 
-/*
- * Return true if the given lut is a linear mapping of values, i.e. it acts
- * like a bypass LUT.
+/**
+ * __is_lut_linear - check if the given lut is a linear mapping of values
+ * @lut: given lut to check values
+ * @size: lut size
  *
  * It is considered linear if the lut represents:
- * f(a) = (0xFF00/MAX_COLOR_LUT_ENTRIES-1)a; for integer a in
- *                                           [0, MAX_COLOR_LUT_ENTRIES)
+ * f(a) = (0xFF00/MAX_COLOR_LUT_ENTRIES-1)a; for integer a in [0,
+ * MAX_COLOR_LUT_ENTRIES)
+ *
+ * Returns:
+ * True if the given lut is a linear mapping of values, i.e. it acts like a
+ * bypass LUT. Otherwise, false.
  */
 static bool __is_lut_linear(const struct drm_color_lut *lut, uint32_t size)
 {
@@ -119,9 +133,13 @@ static bool __is_lut_linear(const struct drm_color_lut *lut, uint32_t size)
 	return true;
 }
 
-/*
- * Convert the drm_color_lut to dc_gamma. The conversion depends on the size
- * of the lut - whether or not it's legacy.
+/**
+ * __drm_lut_to_dc_gamma - convert the drm_color_lut to dc_gamma.
+ * @lut: DRM lookup table for color conversion
+ * @gamma: DC gamma to set entries
+ * @is_legacy: legacy or atomic gamma
+ *
+ * The conversion depends on the size of the lut - whether or not it's legacy.
  */
 static void __drm_lut_to_dc_gamma(const struct drm_color_lut *lut,
 				  struct dc_gamma *gamma, bool is_legacy)
@@ -154,8 +172,11 @@ static void __drm_lut_to_dc_gamma(const struct drm_color_lut *lut,
 	}
 }
 
-/*
- * Converts a DRM CTM to a DC CSC float matrix.
+/**
+ * __drm_ctm_to_dc_matrix - converts a DRM CTM to a DC CSC float matrix
+ * @ctm: DRM color transformation matrix
+ * @matrix: DC CSC float matrix
+ *
  * The matrix needs to be a 3x4 (12 entry) matrix.
  */
 static void __drm_ctm_to_dc_matrix(const struct drm_color_ctm *ctm,
@@ -189,7 +210,18 @@ static void __drm_ctm_to_dc_matrix(const struct drm_color_ctm *ctm,
 	}
 }
 
-/* Calculates the legacy transfer function - only for sRGB input space. */
+/**
+ * __set_legacy_tf - Calculates the legacy transfer function
+ * @func: transfer function
+ * @lut: lookup table that defines the color space
+ * @lut_size: size of respective lut
+ * @has_rom: if ROM can be used for hardcoded curve
+ *
+ * Only for sRGB input space
+ *
+ * Returns:
+ * 0 in case of success, -ENOMEM if fails
+ */
 static int __set_legacy_tf(struct dc_transfer_func *func,
 			   const struct drm_color_lut *lut, uint32_t lut_size,
 			   bool has_rom)
@@ -218,7 +250,16 @@ static int __set_legacy_tf(struct dc_transfer_func *func,
 	return res ? 0 : -ENOMEM;
 }
 
-/* Calculates the output transfer function based on expected input space. */
+/**
+ * __set_output_tf - calculates the output transfer function based on expected input space.
+ * @func: transfer function
+ * @lut: lookup table that defines the color space
+ * @lut_size: size of respective lut
+ * @has_rom: if ROM can be used for hardcoded curve
+ *
+ * Returns:
+ * 0 in case of success. -ENOMEM if fails.
+ */
 static int __set_output_tf(struct dc_transfer_func *func,
 			   const struct drm_color_lut *lut, uint32_t lut_size,
 			   bool has_rom)
@@ -262,7 +303,16 @@ static int __set_output_tf(struct dc_transfer_func *func,
 	return res ? 0 : -ENOMEM;
 }
 
-/* Caculates the input transfer function based on expected input space. */
+/**
+ * __set_input_tf - calculates the input transfer function based on expected
+ * input space.
+ * @func: transfer function
+ * @lut: lookup table that defines the color space
+ * @lut_size: size of respective lut.
+ *
+ * Returns:
+ * 0 in case of success. -ENOMEM if fails.
+ */
 static int __set_input_tf(struct dc_transfer_func *func,
 			  const struct drm_color_lut *lut, uint32_t lut_size)
 {
@@ -285,13 +335,14 @@ static int __set_input_tf(struct dc_transfer_func *func,
 }
 
 /**
- * amdgpu_dm_verify_lut_sizes
+ * amdgpu_dm_verify_lut_sizes - verifies if DRM luts match the hw supported sizes
  * @crtc_state: the DRM CRTC state
  *
- * Verifies that the Degamma and Gamma LUTs attached to the |crtc_state| are of
- * the expected size.
+ * Verifies that the Degamma and Gamma LUTs attached to the &crtc_state
+ * are of the expected size.
  *
- * Returns 0 on success.
+ * Returns:
+ * 0 on success. -EINVAL if any lut sizes are invalid.
  */
 int amdgpu_dm_verify_lut_sizes(const struct drm_crtc_state *crtc_state)
 {
@@ -327,9 +378,9 @@ int amdgpu_dm_verify_lut_sizes(const struct drm_crtc_state *crtc_state)
  * of the HW blocks as long as the CRTC CTM always comes before the
  * CRTC RGM and after the CRTC DGM.
  *
- * The CRTC RGM block will be placed in the RGM LUT block if it is non-linear.
- * The CRTC DGM block will be placed in the DGM LUT block if it is non-linear.
- * The CRTC CTM will be placed in the gamut remap block if it is non-linear.
+ * - The CRTC RGM block will be placed in the RGM LUT block if it is non-linear.
+ * - The CRTC DGM block will be placed in the DGM LUT block if it is non-linear.
+ * - The CRTC CTM will be placed in the gamut remap block if it is non-linear.
  *
  * The RGM block is typically more fully featured and accurate across
  * all ASICs - DCE can't support a custom non-linear CRTC DGM.
@@ -338,7 +389,8 @@ int amdgpu_dm_verify_lut_sizes(const struct drm_crtc_state *crtc_state)
  * management at once we have to either restrict the usage of CRTC properties
  * or blend adjustments together.
  *
- * Returns 0 on success.
+ * Returns:
+ * 0 on success. Error code if setup fails.
  */
 int amdgpu_dm_update_crtc_color_mgmt(struct dm_crtc_state *crtc)
 {
@@ -393,7 +445,7 @@ int amdgpu_dm_update_crtc_color_mgmt(struct dm_crtc_state *crtc)
 		if (r)
 			return r;
 	} else if (has_regamma) {
-		/* CRTC RGM goes into RGM LUT. */
+		/* If atomic regamma, CRTC RGM goes into RGM LUT. */
 		stream->out_transfer_func->type = TF_TYPE_DISTRIBUTED_POINTS;
 		stream->out_transfer_func->tf = TRANSFER_FUNCTION_LINEAR;
 
@@ -450,9 +502,10 @@ int amdgpu_dm_update_crtc_color_mgmt(struct dm_crtc_state *crtc)
  *
  * Update the underlying dc_stream_state's input transfer function (ITF) in
  * preparation for hardware commit. The transfer function used depends on
- * the prepartion done on the stream for color management.
+ * the preparation done on the stream for color management.
  *
- * Returns 0 on success.
+ * Returns:
+ * 0 on success. -ENOMEM if mem allocation fails.
  */
 int amdgpu_dm_update_plane_color_mgmt(struct dm_crtc_state *crtc,
 				      struct dc_plane_state *dc_plane_state)
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
index a0154a5f7183..f0b01c8dc4a6 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c
@@ -27,6 +27,7 @@
 #include <linux/acpi.h>
 #include <linux/i2c.h>
 
+#include <drm/drm_atomic.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/amdgpu_drm.h>
 #include <drm/drm_edid.h>
@@ -153,41 +154,28 @@ enum dc_edid_status dm_helpers_parse_edid_caps(
 	return result;
 }
 
-static void get_payload_table(
-		struct amdgpu_dm_connector *aconnector,
-		struct dp_mst_stream_allocation_table *proposed_table)
+static void
+fill_dc_mst_payload_table_from_drm(struct drm_dp_mst_topology_state *mst_state,
+				   struct amdgpu_dm_connector *aconnector,
+				   struct dc_dp_mst_stream_allocation_table *table)
 {
-	int i;
-	struct drm_dp_mst_topology_mgr *mst_mgr =
-			&aconnector->mst_port->mst_mgr;
-
-	mutex_lock(&mst_mgr->payload_lock);
-
-	proposed_table->stream_count = 0;
-
-	/* number of active streams */
-	for (i = 0; i < mst_mgr->max_payloads; i++) {
-		if (mst_mgr->payloads[i].num_slots == 0)
-			break; /* end of vcp_id table */
-
-		ASSERT(mst_mgr->payloads[i].payload_state !=
-				DP_PAYLOAD_DELETE_LOCAL);
-
-		if (mst_mgr->payloads[i].payload_state == DP_PAYLOAD_LOCAL ||
-			mst_mgr->payloads[i].payload_state ==
-					DP_PAYLOAD_REMOTE) {
-
-			struct dp_mst_stream_allocation *sa =
-					&proposed_table->stream_allocations[
-						proposed_table->stream_count];
-
-			sa->slot_count = mst_mgr->payloads[i].num_slots;
-			sa->vcp_id = mst_mgr->proposed_vcpis[i]->vcpi;
-			proposed_table->stream_count++;
-		}
+	struct dc_dp_mst_stream_allocation_table new_table = { 0 };
+	struct dc_dp_mst_stream_allocation *sa;
+	struct drm_dp_mst_atomic_payload *payload;
+
+	/* Fill payload info*/
+	list_for_each_entry(payload, &mst_state->payloads, next) {
+		if (payload->delete)
+			continue;
+
+		sa = &new_table.stream_allocations[new_table.stream_count];
+		sa->slot_count = payload->time_slots;
+		sa->vcp_id = payload->vcpi;
+		new_table.stream_count++;
 	}
 
-	mutex_unlock(&mst_mgr->payload_lock);
+	/* Overwrite the old table */
+	*table = new_table;
 }
 
 void dm_helpers_dp_update_branch_info(
@@ -201,15 +189,13 @@ void dm_helpers_dp_update_branch_info(
 bool dm_helpers_dp_mst_write_payload_allocation_table(
 		struct dc_context *ctx,
 		const struct dc_stream_state *stream,
-		struct dp_mst_stream_allocation_table *proposed_table,
+		struct dc_dp_mst_stream_allocation_table *proposed_table,
 		bool enable)
 {
 	struct amdgpu_dm_connector *aconnector;
-	struct dm_connector_state *dm_conn_state;
+	struct drm_dp_mst_topology_state *mst_state;
+	struct drm_dp_mst_atomic_payload *payload;
 	struct drm_dp_mst_topology_mgr *mst_mgr;
-	struct drm_dp_mst_port *mst_port;
-	bool ret;
-	u8 link_coding_cap = DP_8b_10b_ENCODING;
 
 	aconnector = (struct amdgpu_dm_connector *)stream->dm_stream_context;
 	/* Accessing the connector state is required for vcpi_slots allocation
@@ -220,40 +206,21 @@ bool dm_helpers_dp_mst_write_payload_allocation_table(
 	if (!aconnector || !aconnector->mst_port)
 		return false;
 
-	dm_conn_state = to_dm_connector_state(aconnector->base.state);
-
 	mst_mgr = &aconnector->mst_port->mst_mgr;
-
-	if (!mst_mgr->mst_state)
-		return false;
-
-	mst_port = aconnector->port;
-
-#if defined(CONFIG_DRM_AMD_DC_DCN)
-	link_coding_cap = dc_link_dp_mst_decide_link_encoding_format(aconnector->dc_link);
-#endif
-
-	if (enable) {
-
-		ret = drm_dp_mst_allocate_vcpi(mst_mgr, mst_port,
-					       dm_conn_state->pbn,
-					       dm_conn_state->vcpi_slots);
-		if (!ret)
-			return false;
-
-	} else {
-		drm_dp_mst_reset_vcpi_slots(mst_mgr, mst_port);
-	}
+	mst_state = to_drm_dp_mst_topology_state(mst_mgr->base.state);
 
 	/* It's OK for this to fail */
-	drm_dp_update_payload_part1(mst_mgr, (link_coding_cap == DP_CAP_ANSI_128B132B) ? 0:1);
+	payload = drm_atomic_get_mst_payload_state(mst_state, aconnector->port);
+	if (enable)
+		drm_dp_add_payload_part1(mst_mgr, mst_state, payload);
+	else
+		drm_dp_remove_payload(mst_mgr, mst_state, payload);
 
 	/* mst_mgr->->payloads are VC payload notify MST branch using DPCD or
 	 * AUX message. The sequence is slot 1-63 allocated sequence for each
 	 * stream. AMD ASIC stream slot allocation should follow the same
 	 * sequence. copy DRM MST allocation to dc */
-
-	get_payload_table(aconnector, proposed_table);
+	fill_dc_mst_payload_table_from_drm(mst_state, aconnector, proposed_table);
 
 	return true;
 }
@@ -310,8 +277,9 @@ bool dm_helpers_dp_mst_send_payload_allocation(
 		bool enable)
 {
 	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_topology_state *mst_state;
 	struct drm_dp_mst_topology_mgr *mst_mgr;
-	struct drm_dp_mst_port *mst_port;
+	struct drm_dp_mst_atomic_payload *payload;
 	enum mst_progress_status set_flag = MST_ALLOCATE_NEW_PAYLOAD;
 	enum mst_progress_status clr_flag = MST_CLEAR_ALLOCATED_PAYLOAD;
 
@@ -320,19 +288,16 @@ bool dm_helpers_dp_mst_send_payload_allocation(
 	if (!aconnector || !aconnector->mst_port)
 		return false;
 
-	mst_port = aconnector->port;
-
 	mst_mgr = &aconnector->mst_port->mst_mgr;
+	mst_state = to_drm_dp_mst_topology_state(mst_mgr->base.state);
 
-	if (!mst_mgr->mst_state)
-		return false;
-
+	payload = drm_atomic_get_mst_payload_state(mst_state, aconnector->port);
 	if (!enable) {
 		set_flag = MST_CLEAR_ALLOCATED_PAYLOAD;
 		clr_flag = MST_ALLOCATE_NEW_PAYLOAD;
 	}
 
-	if (drm_dp_update_payload_part2(mst_mgr)) {
+	if (enable && drm_dp_add_payload_part2(mst_mgr, mst_state->base.state, payload)) {
 		amdgpu_dm_set_mst_status(&aconnector->mst_status,
 			set_flag, false);
 	} else {
@@ -342,9 +307,6 @@ bool dm_helpers_dp_mst_send_payload_allocation(
 			clr_flag, false);
 	}
 
-	if (!enable)
-		drm_dp_mst_deallocate_vcpi(mst_mgr, mst_port);
-
 	return true;
 }
 
@@ -729,8 +691,14 @@ bool dm_helpers_dp_write_dsc_enable(
 		const struct dc_stream_state *stream,
 		bool enable)
 {
-	uint8_t enable_dsc = enable ? 1 : 0;
+	static const uint8_t DSC_DISABLE;
+	static const uint8_t DSC_DECODING = 0x01;
+	static const uint8_t DSC_PASSTHROUGH = 0x02;
+
 	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_port *port;
+	uint8_t enable_dsc = enable ? DSC_DECODING : DSC_DISABLE;
+	uint8_t enable_passthrough = enable ? DSC_PASSTHROUGH : DSC_DISABLE;
 	uint8_t ret = 0;
 
 	if (!stream)
@@ -750,8 +718,39 @@ bool dm_helpers_dp_write_dsc_enable(
 				aconnector->dsc_aux, stream, enable_dsc);
 #endif
 
-		ret = drm_dp_dpcd_write(aconnector->dsc_aux, DP_DSC_ENABLE, &enable_dsc, 1);
-		DC_LOG_DC("Send DSC %s to MST RX\n", enable_dsc ? "enable" : "disable");
+		port = aconnector->port;
+
+		if (enable) {
+			if (port->passthrough_aux) {
+				ret = drm_dp_dpcd_write(port->passthrough_aux,
+							DP_DSC_ENABLE,
+							&enable_passthrough, 1);
+				DC_LOG_DC("Sent DSC pass-through enable to virtual dpcd port, ret = %u\n",
+					  ret);
+			}
+
+			ret = drm_dp_dpcd_write(aconnector->dsc_aux,
+						DP_DSC_ENABLE, &enable_dsc, 1);
+			DC_LOG_DC("Sent DSC decoding enable to %s port, ret = %u\n",
+				  (port->passthrough_aux) ? "remote RX" :
+				  "virtual dpcd",
+				  ret);
+		} else {
+			ret = drm_dp_dpcd_write(aconnector->dsc_aux,
+						DP_DSC_ENABLE, &enable_dsc, 1);
+			DC_LOG_DC("Sent DSC decoding disable to %s port, ret = %u\n",
+				  (port->passthrough_aux) ? "remote RX" :
+				  "virtual dpcd",
+				  ret);
+
+			if (port->passthrough_aux) {
+				ret = drm_dp_dpcd_write(port->passthrough_aux,
+							DP_DSC_ENABLE,
+							&enable_passthrough, 1);
+				DC_LOG_DC("Sent DSC pass-through disable to virtual dpcd port, ret = %u\n",
+					  ret);
+			}
+		}
 	}
 
 	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT || stream->signal == SIGNAL_TYPE_EDP) {
@@ -768,7 +767,7 @@ bool dm_helpers_dp_write_dsc_enable(
 #endif
 	}
 
-	return (ret > 0);
+	return ret;
 }
 
 bool dm_helpers_is_dp_sink_present(struct dc_link *link)
@@ -879,6 +878,34 @@ void dm_helpers_smu_timeout(struct dc_context *ctx, unsigned int msg_id, unsigne
 	//amdgpu_device_gpu_recover(dc_context->driver-context, NULL);
 }
 
+void dm_helpers_init_panel_settings(
+	struct dc_context *ctx,
+	struct dc_panel_config *panel_config,
+	struct dc_sink *sink)
+{
+	// Extra Panel Power Sequence
+	panel_config->pps.extra_t3_ms = sink->edid_caps.panel_patch.extra_t3_ms;
+	panel_config->pps.extra_t7_ms = sink->edid_caps.panel_patch.extra_t7_ms;
+	panel_config->pps.extra_delay_backlight_off = sink->edid_caps.panel_patch.extra_delay_backlight_off;
+	panel_config->pps.extra_post_t7_ms = 0;
+	panel_config->pps.extra_pre_t11_ms = 0;
+	panel_config->pps.extra_t12_ms = sink->edid_caps.panel_patch.extra_t12_ms;
+	panel_config->pps.extra_post_OUI_ms = 0;
+	// Feature DSC
+	panel_config->dsc.disable_dsc_edp = false;
+	panel_config->dsc.force_dsc_edp_policy = 0;
+}
+
+void dm_helpers_override_panel_settings(
+	struct dc_context *ctx,
+	struct dc_panel_config *panel_config)
+{
+	// Feature DSC
+	if (amdgpu_dc_debug_mask & DC_DISABLE_DSC) {
+		panel_config->dsc.disable_dsc_edp = true;
+	}
+}
+
 void *dm_helpers_allocate_gpu_mem(
 		struct dc_context *ctx,
 		enum dc_gpu_mem_alloc_type type,
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
index 2e74ccf7df5b..6ff96b4bdda5 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
@@ -36,6 +36,7 @@
 #include "dm_helpers.h"
 
 #include "dc_link_ddc.h"
+#include "dc_link_dp.h"
 #include "ddc_service_types.h"
 #include "dpcd_defs.h"
 
@@ -447,34 +448,13 @@ dm_dp_mst_detect(struct drm_connector *connector,
 }
 
 static int dm_dp_mst_atomic_check(struct drm_connector *connector,
-				struct drm_atomic_state *state)
+				  struct drm_atomic_state *state)
 {
-	struct drm_connector_state *new_conn_state =
-			drm_atomic_get_new_connector_state(state, connector);
-	struct drm_connector_state *old_conn_state =
-			drm_atomic_get_old_connector_state(state, connector);
 	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
-	struct drm_crtc_state *new_crtc_state;
-	struct drm_dp_mst_topology_mgr *mst_mgr;
-	struct drm_dp_mst_port *mst_port;
+	struct drm_dp_mst_topology_mgr *mst_mgr = &aconnector->mst_port->mst_mgr;
+	struct drm_dp_mst_port *mst_port = aconnector->port;
 
-	mst_port = aconnector->port;
-	mst_mgr = &aconnector->mst_port->mst_mgr;
-
-	if (!old_conn_state->crtc)
-		return 0;
-
-	if (new_conn_state->crtc) {
-		new_crtc_state = drm_atomic_get_new_crtc_state(state, new_conn_state->crtc);
-		if (!new_crtc_state ||
-		    !drm_atomic_crtc_needs_modeset(new_crtc_state) ||
-		    new_crtc_state->enable)
-			return 0;
-		}
-
-	return drm_dp_atomic_release_vcpi_slots(state,
-						mst_mgr,
-						mst_port);
+	return drm_dp_atomic_release_time_slots(state, mst_mgr, mst_port);
 }
 
 static const struct drm_connector_helper_funcs dm_dp_mst_connector_helper_funcs = {
@@ -618,15 +598,8 @@ void amdgpu_dm_initialize_dp_connector(struct amdgpu_display_manager *dm,
 
 	dc_link_dp_get_max_link_enc_cap(aconnector->dc_link, &max_link_enc_cap);
 	aconnector->mst_mgr.cbs = &dm_mst_cbs;
-	drm_dp_mst_topology_mgr_init(
-		&aconnector->mst_mgr,
-		adev_to_drm(dm->adev),
-		&aconnector->dm_dp_aux.aux,
-		16,
-		4,
-		max_link_enc_cap.lane_count,
-		drm_dp_bw_code_to_link_rate(max_link_enc_cap.link_rate),
-		aconnector->connector_id);
+	drm_dp_mst_topology_mgr_init(&aconnector->mst_mgr, adev_to_drm(dm->adev),
+				     &aconnector->dm_dp_aux.aux, 16, 4, aconnector->connector_id);
 
 	drm_connector_attach_dp_subconnector_property(&aconnector->base);
 }
@@ -731,6 +704,7 @@ static int bpp_x16_from_pbn(struct dsc_mst_fairness_params param, int pbn)
 }
 
 static bool increase_dsc_bpp(struct drm_atomic_state *state,
+			     struct drm_dp_mst_topology_state *mst_state,
 			     struct dc_link *dc_link,
 			     struct dsc_mst_fairness_params *params,
 			     struct dsc_mst_fairness_vars *vars,
@@ -743,12 +717,9 @@ static bool increase_dsc_bpp(struct drm_atomic_state *state,
 	int min_initial_slack;
 	int next_index;
 	int remaining_to_increase = 0;
-	int pbn_per_timeslot;
 	int link_timeslots_used;
 	int fair_pbn_alloc;
 
-	pbn_per_timeslot = dm_mst_get_pbn_divider(dc_link);
-
 	for (i = 0; i < count; i++) {
 		if (vars[i + k].dsc_enabled) {
 			initial_slack[i] =
@@ -779,46 +750,43 @@ static bool increase_dsc_bpp(struct drm_atomic_state *state,
 		link_timeslots_used = 0;
 
 		for (i = 0; i < count; i++)
-			link_timeslots_used += DIV_ROUND_UP(vars[i + k].pbn, pbn_per_timeslot);
+			link_timeslots_used += DIV_ROUND_UP(vars[i + k].pbn, mst_state->pbn_div);
 
-		fair_pbn_alloc = (63 - link_timeslots_used) / remaining_to_increase * pbn_per_timeslot;
+		fair_pbn_alloc =
+			(63 - link_timeslots_used) / remaining_to_increase * mst_state->pbn_div;
 
 		if (initial_slack[next_index] > fair_pbn_alloc) {
 			vars[next_index].pbn += fair_pbn_alloc;
-			if (drm_dp_atomic_find_vcpi_slots(state,
+			if (drm_dp_atomic_find_time_slots(state,
 							  params[next_index].port->mgr,
 							  params[next_index].port,
-							  vars[next_index].pbn,
-							  pbn_per_timeslot) < 0)
+							  vars[next_index].pbn) < 0)
 				return false;
 			if (!drm_dp_mst_atomic_check(state)) {
 				vars[next_index].bpp_x16 = bpp_x16_from_pbn(params[next_index], vars[next_index].pbn);
 			} else {
 				vars[next_index].pbn -= fair_pbn_alloc;
-				if (drm_dp_atomic_find_vcpi_slots(state,
+				if (drm_dp_atomic_find_time_slots(state,
 								  params[next_index].port->mgr,
 								  params[next_index].port,
-								  vars[next_index].pbn,
-								  pbn_per_timeslot) < 0)
+								  vars[next_index].pbn) < 0)
 					return false;
 			}
 		} else {
 			vars[next_index].pbn += initial_slack[next_index];
-			if (drm_dp_atomic_find_vcpi_slots(state,
+			if (drm_dp_atomic_find_time_slots(state,
 							  params[next_index].port->mgr,
 							  params[next_index].port,
-							  vars[next_index].pbn,
-							  pbn_per_timeslot) < 0)
+							  vars[next_index].pbn) < 0)
 				return false;
 			if (!drm_dp_mst_atomic_check(state)) {
 				vars[next_index].bpp_x16 = params[next_index].bw_range.max_target_bpp_x16;
 			} else {
 				vars[next_index].pbn -= initial_slack[next_index];
-				if (drm_dp_atomic_find_vcpi_slots(state,
+				if (drm_dp_atomic_find_time_slots(state,
 								  params[next_index].port->mgr,
 								  params[next_index].port,
-								  vars[next_index].pbn,
-								  pbn_per_timeslot) < 0)
+								  vars[next_index].pbn) < 0)
 					return false;
 			}
 		}
@@ -872,11 +840,10 @@ static bool try_disable_dsc(struct drm_atomic_state *state,
 			break;
 
 		vars[next_index].pbn = kbps_to_peak_pbn(params[next_index].bw_range.stream_kbps);
-		if (drm_dp_atomic_find_vcpi_slots(state,
+		if (drm_dp_atomic_find_time_slots(state,
 						  params[next_index].port->mgr,
 						  params[next_index].port,
-						  vars[next_index].pbn,
-						  dm_mst_get_pbn_divider(dc_link)) < 0)
+						  vars[next_index].pbn) < 0)
 			return false;
 
 		if (!drm_dp_mst_atomic_check(state)) {
@@ -884,11 +851,10 @@ static bool try_disable_dsc(struct drm_atomic_state *state,
 			vars[next_index].bpp_x16 = 0;
 		} else {
 			vars[next_index].pbn = kbps_to_peak_pbn(params[next_index].bw_range.max_kbps);
-			if (drm_dp_atomic_find_vcpi_slots(state,
+			if (drm_dp_atomic_find_time_slots(state,
 							  params[next_index].port->mgr,
 							  params[next_index].port,
-							  vars[next_index].pbn,
-							  dm_mst_get_pbn_divider(dc_link)) < 0)
+							  vars[next_index].pbn) < 0)
 				return false;
 		}
 
@@ -902,17 +868,27 @@ static bool compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
 					     struct dc_state *dc_state,
 					     struct dc_link *dc_link,
 					     struct dsc_mst_fairness_vars *vars,
+					     struct drm_dp_mst_topology_mgr *mgr,
 					     int *link_vars_start_index)
 {
-	int i, k;
 	struct dc_stream_state *stream;
 	struct dsc_mst_fairness_params params[MAX_PIPES];
 	struct amdgpu_dm_connector *aconnector;
+	struct drm_dp_mst_topology_state *mst_state = drm_atomic_get_mst_topology_state(state, mgr);
 	int count = 0;
+	int i, k;
 	bool debugfs_overwrite = false;
 
 	memset(params, 0, sizeof(params));
 
+	if (IS_ERR(mst_state))
+		return false;
+
+	mst_state->pbn_div = dm_mst_get_pbn_divider(dc_link);
+#if defined(CONFIG_DRM_AMD_DC_DCN)
+	drm_dp_mst_update_slots(mst_state, dc_link_dp_mst_decide_link_encoding_format(dc_link));
+#endif
+
 	/* Set up params */
 	for (i = 0; i < dc_state->stream_count; i++) {
 		struct dc_dsc_policy dsc_policy = {0};
@@ -971,11 +947,8 @@ static bool compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
 		vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.stream_kbps);
 		vars[i + k].dsc_enabled = false;
 		vars[i + k].bpp_x16 = 0;
-		if (drm_dp_atomic_find_vcpi_slots(state,
-						 params[i].port->mgr,
-						 params[i].port,
-						 vars[i + k].pbn,
-						 dm_mst_get_pbn_divider(dc_link)) < 0)
+		if (drm_dp_atomic_find_time_slots(state, params[i].port->mgr, params[i].port,
+						  vars[i + k].pbn) < 0)
 			return false;
 	}
 	if (!drm_dp_mst_atomic_check(state) && !debugfs_overwrite) {
@@ -989,21 +962,15 @@ static bool compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
 			vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.min_kbps);
 			vars[i + k].dsc_enabled = true;
 			vars[i + k].bpp_x16 = params[i].bw_range.min_target_bpp_x16;
-			if (drm_dp_atomic_find_vcpi_slots(state,
-							  params[i].port->mgr,
-							  params[i].port,
-							  vars[i + k].pbn,
-							  dm_mst_get_pbn_divider(dc_link)) < 0)
+			if (drm_dp_atomic_find_time_slots(state, params[i].port->mgr,
+							  params[i].port, vars[i + k].pbn) < 0)
 				return false;
 		} else {
 			vars[i + k].pbn = kbps_to_peak_pbn(params[i].bw_range.stream_kbps);
 			vars[i + k].dsc_enabled = false;
 			vars[i + k].bpp_x16 = 0;
-			if (drm_dp_atomic_find_vcpi_slots(state,
-							  params[i].port->mgr,
-							  params[i].port,
-							  vars[i + k].pbn,
-							  dm_mst_get_pbn_divider(dc_link)) < 0)
+			if (drm_dp_atomic_find_time_slots(state, params[i].port->mgr,
+							  params[i].port, vars[i + k].pbn) < 0)
 				return false;
 		}
 	}
@@ -1011,7 +978,7 @@ static bool compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
 		return false;
 
 	/* Optimize degree of compression */
-	if (!increase_dsc_bpp(state, dc_link, params, vars, count, k))
+	if (!increase_dsc_bpp(state, mst_state, dc_link, params, vars, count, k))
 		return false;
 
 	if (!try_disable_dsc(state, dc_link, params, vars, count, k))
@@ -1157,8 +1124,9 @@ bool compute_mst_dsc_configs_for_state(struct drm_atomic_state *state,
 			continue;
 
 		mutex_lock(&aconnector->mst_mgr.lock);
-		if (!compute_mst_dsc_configs_for_link(state, dc_state, stream->link,
-			vars, &link_vars_start_index)) {
+		if (!compute_mst_dsc_configs_for_link(state, dc_state, stream->link, vars,
+						      &aconnector->mst_mgr,
+						      &link_vars_start_index)) {
 			mutex_unlock(&aconnector->mst_mgr.lock);
 			return false;
 		}
@@ -1216,10 +1184,8 @@ static bool
 			continue;
 
 		mutex_lock(&aconnector->mst_mgr.lock);
-		if (!compute_mst_dsc_configs_for_link(state,
-						      dc_state,
-						      stream->link,
-						      vars,
+		if (!compute_mst_dsc_configs_for_link(state, dc_state, stream->link, vars,
+						      &aconnector->mst_mgr,
 						      &link_vars_start_index)) {
 			mutex_unlock(&aconnector->mst_mgr.lock);
 			return false;
@@ -1386,19 +1352,90 @@ clean_exit:
 	return (ret == 0);
 }
 
-#endif
+static unsigned int kbps_from_pbn(unsigned int pbn)
+{
+	unsigned int kbps = pbn;
+
+	kbps *= (1000000 / PEAK_FACTOR_X1000);
+	kbps *= 8;
+	kbps *= 54;
+	kbps /= 64;
+
+	return kbps;
+}
+
+static bool is_dsc_common_config_possible(struct dc_stream_state *stream,
+					  struct dc_dsc_bw_range *bw_range)
+{
+	struct dc_dsc_policy dsc_policy = {0};
+
+	dc_dsc_get_policy_for_timing(&stream->timing, 0, &dsc_policy);
+	dc_dsc_compute_bandwidth_range(stream->sink->ctx->dc->res_pool->dscs[0],
+				       stream->sink->ctx->dc->debug.dsc_min_slice_height_override,
+				       dsc_policy.min_target_bpp * 16,
+				       dsc_policy.max_target_bpp * 16,
+				       &stream->sink->dsc_caps.dsc_dec_caps,
+				       &stream->timing, bw_range);
+
+	return bw_range->max_target_bpp_x16 && bw_range->min_target_bpp_x16;
+}
+#endif /* CONFIG_DRM_AMD_DC_DCN */
 
 enum dc_status dm_dp_mst_is_port_support_mode(
 	struct amdgpu_dm_connector *aconnector,
 	struct dc_stream_state *stream)
 {
 	int bpp, pbn, branch_max_throughput_mps = 0;
+#if defined(CONFIG_DRM_AMD_DC_DCN)
+	struct dc_link_settings cur_link_settings;
+	unsigned int end_to_end_bw_in_kbps = 0;
+	unsigned int upper_link_bw_in_kbps = 0, down_link_bw_in_kbps = 0;
+	unsigned int max_compressed_bw_in_kbps = 0;
+	struct dc_dsc_bw_range bw_range = {0};
 
-	/* check if mode could be supported within fUll_pbn */
-	bpp = convert_dc_color_depth_into_bpc(stream->timing.display_color_depth) * 3;
-	pbn = drm_dp_calc_pbn_mode(stream->timing.pix_clk_100hz / 10, bpp, false);
-	if (pbn > aconnector->port->full_pbn)
-		return DC_FAIL_BANDWIDTH_VALIDATE;
+	/*
+	 * check if the mode could be supported if DSC pass-through is supported
+	 * AND check if there enough bandwidth available to support the mode
+	 * with DSC enabled.
+	 */
+	if (is_dsc_common_config_possible(stream, &bw_range) &&
+	    aconnector->port->passthrough_aux) {
+		mutex_lock(&aconnector->mst_mgr.lock);
+
+		cur_link_settings = stream->link->verified_link_cap;
+
+		upper_link_bw_in_kbps = dc_link_bandwidth_kbps(aconnector->dc_link,
+							       &cur_link_settings
+							       );
+		down_link_bw_in_kbps = kbps_from_pbn(aconnector->port->full_pbn);
+
+		/* pick the bottleneck */
+		end_to_end_bw_in_kbps = min(upper_link_bw_in_kbps,
+					    down_link_bw_in_kbps);
+
+		mutex_unlock(&aconnector->mst_mgr.lock);
+
+		/*
+		 * use the maximum dsc compression bandwidth as the required
+		 * bandwidth for the mode
+		 */
+		max_compressed_bw_in_kbps = bw_range.min_kbps;
+
+		if (end_to_end_bw_in_kbps < max_compressed_bw_in_kbps) {
+			DRM_DEBUG_DRIVER("Mode does not fit into DSC pass-through bandwidth validation\n");
+			return DC_FAIL_BANDWIDTH_VALIDATE;
+		}
+	} else {
+#endif
+		/* check if mode could be supported within full_pbn */
+		bpp = convert_dc_color_depth_into_bpc(stream->timing.display_color_depth) * 3;
+		pbn = drm_dp_calc_pbn_mode(stream->timing.pix_clk_100hz / 10, bpp, false);
+
+		if (pbn > aconnector->port->full_pbn)
+			return DC_FAIL_BANDWIDTH_VALIDATE;
+#if defined(CONFIG_DRM_AMD_DC_DCN)
+	}
+#endif
 
 	/* check is mst dsc output bandwidth branch_overall_throughput_0_mps */
 	switch (stream->timing.pixel_encoding) {
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
index 987bde4dca3d..dfd3be49eac8 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
@@ -1563,7 +1563,7 @@ int dm_drm_plane_get_property(struct drm_plane *plane,
 static const struct drm_plane_funcs dm_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.destroy	= drm_primary_helper_destroy,
+	.destroy	= drm_plane_helper_destroy,
 	.reset = dm_drm_plane_reset,
 	.atomic_duplicate_state = dm_drm_plane_duplicate_state,
 	.atomic_destroy_state = dm_drm_plane_destroy_state,
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
index c8da18e45b0e..8ca10ab3dfc1 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
@@ -170,7 +170,13 @@ bool amdgpu_dm_psr_enable(struct dc_stream_state *stream)
 					   &stream, 1,
 					   &params);
 
-	power_opt |= psr_power_opt_z10_static_screen;
+	/*
+	 * Only enable static-screen optimizations for PSR1. For PSR SU, this
+	 * causes vstartup interrupt issues, used by amdgpu_dm to send vblank
+	 * events.
+	 */
+	if (link->psr_settings.psr_version < DC_PSR_VERSION_SU_1)
+		power_opt |= psr_power_opt_z10_static_screen;
 
 	return dc_link_set_psr_allow_active(link, &psr_enable, false, false, &power_opt);
 }
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
index 09fbb7ad5362..53b077b40d72 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
@@ -24,6 +24,7 @@
  */
 
 #include "dm_services.h"
+#include "core_types.h"
 
 #include "ObjectID.h"
 #include "atomfirmware.h"
@@ -44,25 +45,6 @@
 
 #include "bios_parser_common.h"
 
-/* Temporarily add in defines until ObjectID.h patch is updated in a few days */
-#ifndef GENERIC_OBJECT_ID_BRACKET_LAYOUT
-#define GENERIC_OBJECT_ID_BRACKET_LAYOUT          0x05
-#endif /* GENERIC_OBJECT_ID_BRACKET_LAYOUT */
-
-#ifndef GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1
-#define GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1	\
-	(GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\
-	GRAPH_OBJECT_ENUM_ID1 << ENUM_ID_SHIFT |\
-	GENERIC_OBJECT_ID_BRACKET_LAYOUT << OBJECT_ID_SHIFT)
-#endif /* GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1 */
-
-#ifndef GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID2
-#define GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID2	\
-	(GRAPH_OBJECT_TYPE_GENERIC << OBJECT_TYPE_SHIFT |\
-	GRAPH_OBJECT_ENUM_ID2 << ENUM_ID_SHIFT |\
-	GENERIC_OBJECT_ID_BRACKET_LAYOUT << OBJECT_ID_SHIFT)
-#endif /* GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID2 */
-
 #define DC_LOGGER \
 	bp->base.ctx->logger
 
@@ -868,6 +850,8 @@ static enum bp_result get_ss_info_v4_1(
 				disp_cntl_tbl->dvi_ss_rate_10hz * 10;
 		if (disp_cntl_tbl->dvi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE)
 			ss_info->type.CENTER_MODE = true;
+
+		DC_LOG_BIOS("AS_SIGNAL_TYPE_DVI ss_percentage: %d\n", ss_info->spread_spectrum_percentage);
 		break;
 	case AS_SIGNAL_TYPE_HDMI:
 		ss_info->spread_spectrum_percentage =
@@ -876,6 +860,8 @@ static enum bp_result get_ss_info_v4_1(
 				disp_cntl_tbl->hdmi_ss_rate_10hz * 10;
 		if (disp_cntl_tbl->hdmi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE)
 			ss_info->type.CENTER_MODE = true;
+
+		DC_LOG_BIOS("AS_SIGNAL_TYPE_HDMI ss_percentage: %d\n", ss_info->spread_spectrum_percentage);
 		break;
 	/* TODO LVDS not support anymore? */
 	case AS_SIGNAL_TYPE_DISPLAY_PORT:
@@ -885,6 +871,8 @@ static enum bp_result get_ss_info_v4_1(
 				disp_cntl_tbl->dp_ss_rate_10hz * 10;
 		if (disp_cntl_tbl->dp_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE)
 			ss_info->type.CENTER_MODE = true;
+
+		DC_LOG_BIOS("AS_SIGNAL_TYPE_DISPLAY_PORT ss_percentage: %d\n", ss_info->spread_spectrum_percentage);
 		break;
 	case AS_SIGNAL_TYPE_GPU_PLL:
 		/* atom_firmware: DAL only get data from dce_info table.
@@ -898,13 +886,15 @@ static enum bp_result get_ss_info_v4_1(
 				      DATA_TABLES(smu_info));
 		if (!smu_info)
 			return BP_RESULT_BADBIOSTABLE;
-
+		DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", smu_info->gpuclk_ss_percentage);
 		ss_info->spread_spectrum_percentage =
 				smu_info->waflclk_ss_percentage;
 		ss_info->spread_spectrum_range =
 				smu_info->gpuclk_ss_rate_10hz * 10;
 		if (smu_info->waflclk_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE)
 			ss_info->type.CENTER_MODE = true;
+
+		DC_LOG_BIOS("AS_SIGNAL_TYPE_XGMI ss_percentage: %d\n", ss_info->spread_spectrum_percentage);
 		break;
 	default:
 		result = BP_RESULT_UNSUPPORTED;
@@ -941,6 +931,7 @@ static enum bp_result get_ss_info_v4_2(
 	if (!smu_info)
 		return BP_RESULT_BADBIOSTABLE;
 
+	DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", smu_info->gpuclk_ss_percentage);
 	ss_info->type.STEP_AND_DELAY_INFO = false;
 	ss_info->spread_percentage_divider = 1000;
 	/* BIOS no longer uses target clock.  Always enable for now */
@@ -954,6 +945,8 @@ static enum bp_result get_ss_info_v4_2(
 				disp_cntl_tbl->dvi_ss_rate_10hz * 10;
 		if (disp_cntl_tbl->dvi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE)
 			ss_info->type.CENTER_MODE = true;
+
+		DC_LOG_BIOS("AS_SIGNAL_TYPE_DVI ss_percentage: %d\n", ss_info->spread_spectrum_percentage);
 		break;
 	case AS_SIGNAL_TYPE_HDMI:
 		ss_info->spread_spectrum_percentage =
@@ -962,6 +955,8 @@ static enum bp_result get_ss_info_v4_2(
 				disp_cntl_tbl->hdmi_ss_rate_10hz * 10;
 		if (disp_cntl_tbl->hdmi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE)
 			ss_info->type.CENTER_MODE = true;
+
+		DC_LOG_BIOS("AS_SIGNAL_TYPE_HDMI ss_percentage: %d\n", ss_info->spread_spectrum_percentage);
 		break;
 	/* TODO LVDS not support anymore? */
 	case AS_SIGNAL_TYPE_DISPLAY_PORT:
@@ -971,6 +966,8 @@ static enum bp_result get_ss_info_v4_2(
 				smu_info->gpuclk_ss_rate_10hz * 10;
 		if (smu_info->gpuclk_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE)
 			ss_info->type.CENTER_MODE = true;
+
+		DC_LOG_BIOS("AS_SIGNAL_TYPE_DISPLAY_PORT ss_percentage: %d\n", ss_info->spread_spectrum_percentage);
 		break;
 	case AS_SIGNAL_TYPE_GPU_PLL:
 		/* atom_firmware: DAL only get data from dce_info table.
@@ -1019,6 +1016,8 @@ static enum bp_result get_ss_info_v4_5(
 				disp_cntl_tbl->dvi_ss_rate_10hz * 10;
 		if (disp_cntl_tbl->dvi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE)
 			ss_info->type.CENTER_MODE = true;
+
+		DC_LOG_BIOS("AS_SIGNAL_TYPE_DVI ss_percentage: %d\n", ss_info->spread_spectrum_percentage);
 		break;
 	case AS_SIGNAL_TYPE_HDMI:
 		ss_info->spread_spectrum_percentage =
@@ -1027,6 +1026,8 @@ static enum bp_result get_ss_info_v4_5(
 				disp_cntl_tbl->hdmi_ss_rate_10hz * 10;
 		if (disp_cntl_tbl->hdmi_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE)
 			ss_info->type.CENTER_MODE = true;
+
+		DC_LOG_BIOS("AS_SIGNAL_TYPE_HDMI ss_percentage: %d\n", ss_info->spread_spectrum_percentage);
 		break;
 	case AS_SIGNAL_TYPE_DISPLAY_PORT:
 		ss_info->spread_spectrum_percentage =
@@ -1035,6 +1036,8 @@ static enum bp_result get_ss_info_v4_5(
 				disp_cntl_tbl->dp_ss_rate_10hz * 10;
 		if (disp_cntl_tbl->dp_ss_mode & ATOM_SS_CENTRE_SPREAD_MODE)
 			ss_info->type.CENTER_MODE = true;
+
+		DC_LOG_BIOS("AS_SIGNAL_TYPE_DISPLAY_PORT ss_percentage: %d\n", ss_info->spread_spectrum_percentage);
 		break;
 	case AS_SIGNAL_TYPE_GPU_PLL:
 		/* atom_smu_info_v4_0 does not have fields for SS for SMU Display PLL anymore.
@@ -1372,7 +1375,7 @@ static enum bp_result bios_parser_get_lttpr_interop(
 	default:
 		break;
 	}
-
+	DC_LOG_BIOS("DCE_INFO_CAPS_VBIOS_LTTPR_TRANSPARENT_ENABLE: %d tbl_revision.major = %d tbl_revision.minor = %d\n", *dce_caps, tbl_revision.major, tbl_revision.minor);
 	return result;
 }
 
@@ -1388,6 +1391,7 @@ static enum bp_result bios_parser_get_lttpr_caps(
 	if (!DATA_TABLES(dce_info))
 		return BP_RESULT_UNSUPPORTED;
 
+	*dce_caps  = 0;
 	header = GET_IMAGE(struct atom_common_table_header,
 						DATA_TABLES(dce_info));
 	get_atom_data_table_revision(header, &tbl_revision);
@@ -1421,7 +1425,11 @@ static enum bp_result bios_parser_get_lttpr_caps(
 	default:
 		break;
 	}
-
+	DC_LOG_BIOS("DCE_INFO_CAPS_LTTPR_SUPPORT_ENABLE: %d tbl_revision.major = %d tbl_revision.minor = %d\n", *dce_caps, tbl_revision.major, tbl_revision.minor);
+	if (dcb->ctx->dc->config.force_bios_enable_lttpr && *dce_caps == 0) {
+		*dce_caps = 1;
+		DC_LOG_BIOS("DCE_INFO_CAPS_VBIOS_LTTPR_TRANSPARENT_ENABLE: forced enabled");
+	}
 	return result;
 }
 
@@ -1859,7 +1867,7 @@ static enum bp_result get_firmware_info_v3_2(
 		/* Vega12 */
 		smu_info_v3_2 = GET_IMAGE(struct atom_smu_info_v3_2,
 							DATA_TABLES(smu_info));
-
+		DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", smu_info_v3_2->gpuclk_ss_percentage);
 		if (!smu_info_v3_2)
 			return BP_RESULT_BADBIOSTABLE;
 
@@ -1868,7 +1876,7 @@ static enum bp_result get_firmware_info_v3_2(
 		/* Vega20 */
 		smu_info_v3_3 = GET_IMAGE(struct atom_smu_info_v3_3,
 							DATA_TABLES(smu_info));
-
+		DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", smu_info_v3_3->gpuclk_ss_percentage);
 		if (!smu_info_v3_3)
 			return BP_RESULT_BADBIOSTABLE;
 
@@ -2010,7 +2018,7 @@ static enum bp_result get_firmware_info_v3_4(
 
 			if (!smu_info_v3_5)
 				return BP_RESULT_BADBIOSTABLE;
-
+			DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", smu_info_v3_5->gpuclk_ss_percentage);
 			info->default_engine_clk = smu_info_v3_5->bootup_dcefclk_10khz * 10;
 			break;
 
@@ -2416,6 +2424,7 @@ static enum bp_result get_integrated_info_v11(
 	info_v11 = GET_IMAGE(struct atom_integrated_system_info_v1_11,
 					DATA_TABLES(integratedsysteminfo));
 
+	DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", info_v11->gpuclk_ss_percentage);
 	if (info_v11 == NULL)
 		return BP_RESULT_BADBIOSTABLE;
 
@@ -2630,6 +2639,7 @@ static enum bp_result get_integrated_info_v2_1(
 
 	info_v2_1 = GET_IMAGE(struct atom_integrated_system_info_v2_1,
 					DATA_TABLES(integratedsysteminfo));
+	DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", info_v2_1->gpuclk_ss_percentage);
 
 	if (info_v2_1 == NULL)
 		return BP_RESULT_BADBIOSTABLE;
@@ -2791,6 +2801,8 @@ static enum bp_result get_integrated_info_v2_2(
 	info_v2_2 = GET_IMAGE(struct atom_integrated_system_info_v2_2,
 					DATA_TABLES(integratedsysteminfo));
 
+	DC_LOG_BIOS("gpuclk_ss_percentage (unit of 0.001 percent): %d\n", info_v2_2->gpuclk_ss_percentage);
+
 	if (info_v2_2 == NULL)
 		return BP_RESULT_BADBIOSTABLE;
 
@@ -2942,6 +2954,27 @@ static enum bp_result construct_integrated_info(
 		default:
 			return result;
 		}
+		if (result == BP_RESULT_OK) {
+
+			DC_LOG_BIOS("edp1:\n"
+						"\tedp_pwr_on_off_delay = %d\n"
+						"\tedp_pwr_on_vary_bl_to_blon = %d\n"
+						"\tedp_pwr_down_bloff_to_vary_bloff = %d\n"
+						"\tedp_bootup_bl_level = %d\n",
+						info->edp1_info.edp_pwr_on_off_delay,
+						info->edp1_info.edp_pwr_on_vary_bl_to_blon,
+						info->edp1_info.edp_pwr_down_bloff_to_vary_bloff,
+						info->edp1_info.edp_bootup_bl_level);
+			DC_LOG_BIOS("edp2:\n"
+						"\tedp_pwr_on_off_delayv = %d\n"
+						"\tedp_pwr_on_vary_bl_to_blon = %d\n"
+						"\tedp_pwr_down_bloff_to_vary_bloff = %d\n"
+						"\tedp_bootup_bl_level = %d\n",
+						info->edp2_info.edp_pwr_on_off_delay,
+						info->edp2_info.edp_pwr_on_vary_bl_to_blon,
+						info->edp2_info.edp_pwr_down_bloff_to_vary_bloff,
+						info->edp2_info.edp_bootup_bl_level);
+		}
 	}
 
 	if (result != BP_RESULT_OK)
@@ -2967,13 +3000,22 @@ static enum bp_result construct_integrated_info(
 						info->ext_disp_conn_info.path[i].ext_encoder_obj_id.id,
 						info->ext_disp_conn_info.path[i].caps
 						);
+			if (info->ext_disp_conn_info.path[i].caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN)
+				DC_LOG_BIOS("BIOS EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN on path %d\n", i);
+			else if (bp->base.ctx->dc->config.force_bios_fixed_vs) {
+				info->ext_disp_conn_info.path[i].caps |= EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN;
+				DC_LOG_BIOS("driver forced EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN on path %d\n", i);
+			}
 		}
-
 		// Log the Checksum and Voltage Swing
 		DC_LOG_BIOS("Integrated info table CHECKSUM: %d\n"
 					"Integrated info table FIX_DP_VOLTAGE_SWING: %d\n",
 					info->ext_disp_conn_info.checksum,
 					info->ext_disp_conn_info.fixdpvoltageswing);
+		if (bp->base.ctx->dc->config.force_bios_fixed_vs && info->ext_disp_conn_info.fixdpvoltageswing == 0) {
+			info->ext_disp_conn_info.fixdpvoltageswing = bp->base.ctx->dc->config.force_bios_fixed_vs & 0xF;
+			DC_LOG_BIOS("driver forced fixdpvoltageswing = %d\n", info->ext_disp_conn_info.fixdpvoltageswing);
+		}
 	}
 	/* Sort voltage table from low to high*/
 	for (i = 1; i < NUMBER_OF_DISP_CLK_VOLTAGE; ++i) {
@@ -3319,6 +3361,7 @@ static enum bp_result bios_get_board_layout_info(
 	struct bios_parser *bp;
 
 	static enum bp_result record_result;
+	unsigned int max_slots;
 
 	const unsigned int slot_index_to_vbios_id[MAX_BOARD_SLOTS] = {
 		GENERICOBJECT_BRACKET_LAYOUT_ENUM_ID1,
@@ -3335,8 +3378,14 @@ static enum bp_result bios_get_board_layout_info(
 	}
 
 	board_layout_info->num_of_slots = 0;
+	max_slots = MAX_BOARD_SLOTS;
+
+	// Assume single slot on v1_5
+	if (bp->object_info_tbl.revision.minor == 5) {
+		max_slots = 1;
+	}
 
-	for (i = 0; i < MAX_BOARD_SLOTS; ++i) {
+	for (i = 0; i < max_slots; ++i) {
 		record_result = get_bracket_layout_record(dcb,
 			slot_index_to_vbios_id[i],
 			&board_layout_info->slots[i]);
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
index 23a299c929a1..c1eaf571407a 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
@@ -48,6 +48,11 @@
 
 #include "dc_dmub_srv.h"
 
+#include "logger_types.h"
+#undef DC_LOGGER
+#define DC_LOGGER \
+	clk_mgr->base.base.ctx->logger
+
 #include "yellow_carp_offset.h"
 
 #define regCLK1_CLK_PLL_REQ			0x0237
@@ -738,8 +743,49 @@ void dcn31_clk_mgr_construct(
 	clk_mgr->base.base.bw_params = &dcn31_bw_params;
 
 	if (clk_mgr->base.base.ctx->dc->debug.pstate_enabled) {
+		int i;
+
 		dcn31_get_dpm_table_from_smu(&clk_mgr->base, &smu_dpm_clks);
 
+		DC_LOG_SMU("NumDcfClkLevelsEnabled: %d\n"
+				   "NumDispClkLevelsEnabled: %d\n"
+				   "NumSocClkLevelsEnabled: %d\n"
+				   "VcnClkLevelsEnabled: %d\n"
+				   "NumDfPst atesEnabled: %d\n"
+				   "MinGfxClk: %d\n"
+				   "MaxGfxClk: %d\n",
+				   smu_dpm_clks.dpm_clks->NumDcfClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->NumDispClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->NumSocClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->VcnClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->NumDfPstatesEnabled,
+				   smu_dpm_clks.dpm_clks->MinGfxClk,
+				   smu_dpm_clks.dpm_clks->MaxGfxClk);
+		for (i = 0; i < smu_dpm_clks.dpm_clks->NumDcfClkLevelsEnabled; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->DcfClocks[%d] = %d\n",
+					   i,
+					   smu_dpm_clks.dpm_clks->DcfClocks[i]);
+		}
+		for (i = 0; i < smu_dpm_clks.dpm_clks->NumDispClkLevelsEnabled; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->DispClocks[%d] = %d\n",
+					   i, smu_dpm_clks.dpm_clks->DispClocks[i]);
+		}
+		for (i = 0; i < smu_dpm_clks.dpm_clks->NumSocClkLevelsEnabled; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->SocClocks[%d] = %d\n",
+					   i, smu_dpm_clks.dpm_clks->SocClocks[i]);
+		}
+		for (i = 0; i < NUM_SOC_VOLTAGE_LEVELS; i++)
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->SocVoltage[%d] = %d\n",
+					   i, smu_dpm_clks.dpm_clks->SocVoltage[i]);
+
+		for (i = 0; i < NUM_DF_PSTATE_LEVELS; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks.DfPstateTable[%d].FClk = %d\n"
+					   "smu_dpm_clks.dpm_clks->DfPstateTable[%d].MemClk= %d\n"
+					   "smu_dpm_clks.dpm_clks->DfPstateTable[%d].Voltage = %d\n",
+					   i, smu_dpm_clks.dpm_clks->DfPstateTable[i].FClk,
+					   i, smu_dpm_clks.dpm_clks->DfPstateTable[i].MemClk,
+					   i, smu_dpm_clks.dpm_clks->DfPstateTable[i].Voltage);
+		}
 		if (ctx->dc_bios && ctx->dc_bios->integrated_info) {
 			dcn31_clk_mgr_helper_populate_bw_params(
 					&clk_mgr->base,
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
index 8559dcd80af0..1131c6d73f6c 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
@@ -51,6 +51,13 @@
 #include "dc_link_dp.h"
 #include "dcn314_smu.h"
 
+
+#include "logger_types.h"
+#undef DC_LOGGER
+#define DC_LOGGER \
+	clk_mgr->base.base.ctx->logger
+
+
 #define MAX_INSTANCE                                        7
 #define MAX_SEGMENT                                         8
 
@@ -130,11 +137,20 @@ static void dcn314_disable_otg_wa(struct clk_mgr *clk_mgr_base, struct dc_state
 		if (pipe->top_pipe || pipe->prev_odm_pipe)
 			continue;
 		if (pipe->stream && (pipe->stream->dpms_off || dc_is_virtual_signal(pipe->stream->signal))) {
+			struct stream_encoder *stream_enc = pipe->stream_res.stream_enc;
+
 			if (disable) {
+				if (stream_enc && stream_enc->funcs->disable_fifo)
+					pipe->stream_res.stream_enc->funcs->disable_fifo(stream_enc);
+
 				pipe->stream_res.tg->funcs->immediate_disable_crtc(pipe->stream_res.tg);
 				reset_sync_context_for_pipe(dc, context, i);
-			} else
+			} else {
 				pipe->stream_res.tg->funcs->enable_crtc(pipe->stream_res.tg);
+
+				if (stream_enc && stream_enc->funcs->enable_fifo)
+					pipe->stream_res.stream_enc->funcs->enable_fifo(stream_enc);
+			}
 		}
 	}
 }
@@ -614,7 +630,7 @@ static void dcn314_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *cl
 		bw_params->clk_table.entries[i].dppclk_mhz = max_dppclk;
 		bw_params->clk_table.entries[i].wck_ratio = convert_wck_ratio(
 			clock_table->DfPstateTable[min_pstate].WckRatio);
-	};
+	}
 
 	/* Make sure to include at least one entry at highest pstate */
 	if (max_pstate != min_pstate || i == 0) {
@@ -777,7 +793,48 @@ void dcn314_clk_mgr_construct(
 	clk_mgr->base.base.bw_params = &dcn314_bw_params;
 
 	if (clk_mgr->base.base.ctx->dc->debug.pstate_enabled) {
+		int i;
+
 		dcn314_get_dpm_table_from_smu(&clk_mgr->base, &smu_dpm_clks);
+		DC_LOG_SMU("NumDcfClkLevelsEnabled: %d\n"
+				   "NumDispClkLevelsEnabled: %d\n"
+				   "NumSocClkLevelsEnabled: %d\n"
+				   "VcnClkLevelsEnabled: %d\n"
+				   "NumDfPst atesEnabled: %d\n"
+				   "MinGfxClk: %d\n"
+				   "MaxGfxClk: %d\n",
+				   smu_dpm_clks.dpm_clks->NumDcfClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->NumDispClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->NumSocClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->VcnClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->NumDfPstatesEnabled,
+				   smu_dpm_clks.dpm_clks->MinGfxClk,
+				   smu_dpm_clks.dpm_clks->MaxGfxClk);
+		for (i = 0; i < smu_dpm_clks.dpm_clks->NumDcfClkLevelsEnabled; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->DcfClocks[%d] = %d\n",
+					   i,
+					   smu_dpm_clks.dpm_clks->DcfClocks[i]);
+		}
+		for (i = 0; i < smu_dpm_clks.dpm_clks->NumDispClkLevelsEnabled; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->DispClocks[%d] = %d\n",
+					   i, smu_dpm_clks.dpm_clks->DispClocks[i]);
+		}
+		for (i = 0; i < smu_dpm_clks.dpm_clks->NumSocClkLevelsEnabled; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->SocClocks[%d] = %d\n",
+					   i, smu_dpm_clks.dpm_clks->SocClocks[i]);
+		}
+		for (i = 0; i < NUM_SOC_VOLTAGE_LEVELS; i++)
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->SocVoltage[%d] = %d\n",
+					   i, smu_dpm_clks.dpm_clks->SocVoltage[i]);
+
+		for (i = 0; i < NUM_DF_PSTATE_LEVELS; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks.DfPstateTable[%d].FClk = %d\n"
+					   "smu_dpm_clks.dpm_clks->DfPstateTable[%d].MemClk= %d\n"
+					   "smu_dpm_clks.dpm_clks->DfPstateTable[%d].Voltage = %d\n",
+					   i, smu_dpm_clks.dpm_clks->DfPstateTable[i].FClk,
+					   i, smu_dpm_clks.dpm_clks->DfPstateTable[i].MemClk,
+					   i, smu_dpm_clks.dpm_clks->DfPstateTable[i].Voltage);
+		}
 
 		if (ctx->dc_bios && ctx->dc_bios->integrated_info && ctx->dc->config.use_default_clock_table == false) {
 			dcn314_clk_mgr_helper_populate_bw_params(
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
index 98ad8e0fd2d8..893991a0eb97 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
@@ -41,6 +41,11 @@
 
 #include "dc_dmub_srv.h"
 
+#include "logger_types.h"
+#undef DC_LOGGER
+#define DC_LOGGER \
+	clk_mgr->base.base.ctx->logger
+
 #include "dc_link_dp.h"
 
 #define TO_CLK_MGR_DCN315(clk_mgr)\
@@ -514,7 +519,7 @@ static void dcn315_clk_mgr_helper_populate_bw_params(
 		bw_params->clk_table.entries[i].dispclk_mhz = clock_table->DispClocks[i];
 		bw_params->clk_table.entries[i].dppclk_mhz = clock_table->DppClocks[i];
 		bw_params->clk_table.entries[i].wck_ratio = 1;
-	};
+	}
 
 	/* Make sure to include at least one entry and highest pstate */
 	if (max_pstate != min_pstate || i == 0) {
@@ -672,7 +677,48 @@ void dcn315_clk_mgr_construct(
 	clk_mgr->base.base.bw_params = &dcn315_bw_params;
 
 	if (clk_mgr->base.base.ctx->dc->debug.pstate_enabled) {
+		int i;
+
 		dcn315_get_dpm_table_from_smu(&clk_mgr->base, &smu_dpm_clks);
+		DC_LOG_SMU("NumDcfClkLevelsEnabled: %d\n"
+				   "NumDispClkLevelsEnabled: %d\n"
+				   "NumSocClkLevelsEnabled: %d\n"
+				   "VcnClkLevelsEnabled: %d\n"
+				   "NumDfPst atesEnabled: %d\n"
+				   "MinGfxClk: %d\n"
+				   "MaxGfxClk: %d\n",
+				   smu_dpm_clks.dpm_clks->NumDcfClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->NumDispClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->NumSocClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->VcnClkLevelsEnabled,
+				   smu_dpm_clks.dpm_clks->NumDfPstatesEnabled,
+				   smu_dpm_clks.dpm_clks->MinGfxClk,
+				   smu_dpm_clks.dpm_clks->MaxGfxClk);
+		for (i = 0; i < smu_dpm_clks.dpm_clks->NumDcfClkLevelsEnabled; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->DcfClocks[%d] = %d\n",
+					   i,
+					   smu_dpm_clks.dpm_clks->DcfClocks[i]);
+		}
+		for (i = 0; i < smu_dpm_clks.dpm_clks->NumDispClkLevelsEnabled; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->DispClocks[%d] = %d\n",
+					   i, smu_dpm_clks.dpm_clks->DispClocks[i]);
+		}
+		for (i = 0; i < smu_dpm_clks.dpm_clks->NumSocClkLevelsEnabled; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->SocClocks[%d] = %d\n",
+					   i, smu_dpm_clks.dpm_clks->SocClocks[i]);
+		}
+		for (i = 0; i < NUM_SOC_VOLTAGE_LEVELS; i++)
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks->SocVoltage[%d] = %d\n",
+					   i, smu_dpm_clks.dpm_clks->SocVoltage[i]);
+
+		for (i = 0; i < NUM_DF_PSTATE_LEVELS; i++) {
+			DC_LOG_SMU("smu_dpm_clks.dpm_clks.DfPstateTable[%d].FClk = %d\n"
+					   "smu_dpm_clks.dpm_clks->DfPstateTable[%d].MemClk= %d\n"
+					   "smu_dpm_clks.dpm_clks->DfPstateTable[%d].Voltage = %d\n",
+					   i, smu_dpm_clks.dpm_clks->DfPstateTable[i].FClk,
+					   i, smu_dpm_clks.dpm_clks->DfPstateTable[i].MemClk,
+					   i, smu_dpm_clks.dpm_clks->DfPstateTable[i].Voltage);
+		}
 
 		if (ctx->dc_bios && ctx->dc_bios->integrated_info) {
 			dcn315_clk_mgr_helper_populate_bw_params(
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
index c6785969eb1a..f0f3f66629cc 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
@@ -156,12 +156,14 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base)
 {
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	unsigned int num_levels;
+	unsigned int num_dcfclk_levels, num_dtbclk_levels, num_dispclk_levels;
 
 	memset(&(clk_mgr_base->clks), 0, sizeof(struct dc_clocks));
 	clk_mgr_base->clks.p_state_change_support = true;
 	clk_mgr_base->clks.prev_p_state_change_support = true;
 	clk_mgr_base->clks.fclk_prev_p_state_change_support = true;
 	clk_mgr->smu_present = false;
+	clk_mgr->dpm_present = false;
 
 	if (!clk_mgr_base->bw_params)
 		return;
@@ -179,6 +181,7 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base)
 	dcn32_init_single_clock(clk_mgr, PPCLK_DCFCLK,
 			&clk_mgr_base->bw_params->clk_table.entries[0].dcfclk_mhz,
 			&num_levels);
+	num_dcfclk_levels = num_levels;
 
 	/* SOCCLK */
 	dcn32_init_single_clock(clk_mgr, PPCLK_SOCCLK,
@@ -189,11 +192,16 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base)
 		dcn32_init_single_clock(clk_mgr, PPCLK_DTBCLK,
 				&clk_mgr_base->bw_params->clk_table.entries[0].dtbclk_mhz,
 				&num_levels);
+	num_dtbclk_levels = num_levels;
 
 	/* DISPCLK */
 	dcn32_init_single_clock(clk_mgr, PPCLK_DISPCLK,
 			&clk_mgr_base->bw_params->clk_table.entries[0].dispclk_mhz,
 			&num_levels);
+	num_dispclk_levels = num_levels;
+
+	if (num_dcfclk_levels && num_dtbclk_levels && num_dispclk_levels)
+		clk_mgr->dpm_present = true;
 
 	if (clk_mgr_base->ctx->dc->debug.min_disp_clk_khz) {
 		unsigned int i;
@@ -658,6 +666,12 @@ static void dcn32_get_memclk_states_from_smu(struct clk_mgr *clk_mgr_base)
 			&num_levels);
 	clk_mgr_base->bw_params->clk_table.num_entries = num_levels ? num_levels : 1;
 
+	if (clk_mgr->dpm_present && !num_levels)
+		clk_mgr->dpm_present = false;
+
+	if (!clk_mgr->dpm_present)
+		dcn32_patch_dpm_table(clk_mgr_base->bw_params);
+
 	DC_FP_START();
 	/* Refresh bounding box */
 	clk_mgr_base->ctx->dc->res_pool->funcs->update_bw_bounding_box(
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index fb22c3d70528..258ba5a872b1 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -401,6 +401,9 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 {
 	int i;
 
+	if (memcmp(adjust, &stream->adjust, sizeof(struct dc_crtc_timing_adjust)) == 0)
+		return true;
+
 	stream->adjust.v_total_max = adjust->v_total_max;
 	stream->adjust.v_total_mid = adjust->v_total_mid;
 	stream->adjust.v_total_mid_frame_num = adjust->v_total_mid_frame_num;
@@ -638,14 +641,17 @@ bool dc_stream_configure_crc(struct dc *dc, struct dc_stream_state *stream,
 
 /**
  * dc_stream_get_crc() - Get CRC values for the given stream.
- * @dc: DC object
+ *
+ * @dc: DC object.
  * @stream: The DC stream state of the stream to get CRCs from.
- * @r_cr: CRC value for the first of the 3 channels stored here.
- * @g_y:  CRC value for the second of the 3 channels stored here.
- * @b_cb: CRC value for the third of the 3 channels stored here.
+ * @r_cr: CRC value for the red component.
+ * @g_y:  CRC value for the green component.
+ * @b_cb: CRC value for the blue component.
  *
  * dc_stream_configure_crc needs to be called beforehand to enable CRCs.
- * Return false if stream is not found, or if CRCs are not enabled.
+ *
+ * Return:
+ * false if stream is not found, or if CRCs are not enabled.
  */
 bool dc_stream_get_crc(struct dc *dc, struct dc_stream_state *stream,
 		       uint32_t *r_cr, uint32_t *g_y, uint32_t *b_cb)
@@ -1195,7 +1201,7 @@ static void wait_for_no_pipes_pending(struct dc *dc, struct dc_state *context)
 		int count = 0;
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
 
-		if (!pipe->plane_state)
+		if (!pipe->plane_state || pipe->stream->mall_stream_config.type == SUBVP_PHANTOM)
 			continue;
 
 		/* Timeout 100 ms */
@@ -1744,6 +1750,9 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 		context->stream_count == 0)
 		dc->hwss.prepare_bandwidth(dc, context);
 
+	if (dc->debug.enable_double_buffered_dsc_pg_support)
+		dc->hwss.update_dsc_pg(dc, context, false);
+
 	disable_dangling_plane(dc, context);
 	/* re-program planes for existing stream, in case we need to
 	 * free up plane resource for later use
@@ -1834,6 +1843,9 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 		dc->hwss.optimize_bandwidth(dc, context);
 	}
 
+	if (dc->debug.enable_double_buffered_dsc_pg_support)
+		dc->hwss.update_dsc_pg(dc, context, true);
+
 	if (dc->ctx->dce_version >= DCE_VERSION_MAX)
 		TRACE_DCN_CLOCK_STATE(&context->bw_ctx.bw.dcn.clk);
 	else
@@ -1997,6 +2009,9 @@ void dc_post_update_surfaces_to_stream(struct dc *dc)
 
 	dc->hwss.optimize_bandwidth(dc, context);
 
+	if (dc->debug.enable_double_buffered_dsc_pg_support)
+		dc->hwss.update_dsc_pg(dc, context, true);
+
 	dc->optimized_required = false;
 	dc->wm_optimized_required = false;
 }
@@ -2316,9 +2331,13 @@ static enum surface_update_type det_surface_update(const struct dc *dc,
 	type = get_scaling_info_update_type(u);
 	elevate_update_type(&overall_type, type);
 
-	if (u->flip_addr)
+	if (u->flip_addr) {
 		update_flags->bits.addr_update = 1;
-
+		if (u->flip_addr->address.tmz_surface != u->surface->address.tmz_surface) {
+			update_flags->bits.tmz_changed = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL);
+		}
+	}
 	if (u->in_transfer_func)
 		update_flags->bits.in_transfer_func_change = 1;
 
@@ -2753,11 +2772,8 @@ static void copy_stream_update_to_stream(struct dc *dc,
 	if (update->abm_level)
 		stream->abm_level = *update->abm_level;
 
-	if (update->periodic_interrupt0)
-		stream->periodic_interrupt0 = *update->periodic_interrupt0;
-
-	if (update->periodic_interrupt1)
-		stream->periodic_interrupt1 = *update->periodic_interrupt1;
+	if (update->periodic_interrupt)
+		stream->periodic_interrupt = *update->periodic_interrupt;
 
 	if (update->gamut_remap)
 		stream->gamut_remap_matrix = *update->gamut_remap;
@@ -2842,16 +2858,6 @@ static void copy_stream_update_to_stream(struct dc *dc,
 	}
 }
 
-void dc_reset_state(struct dc *dc, struct dc_state *context)
-{
-	dc_resource_state_destruct(context);
-
-	/* clear the structure, but don't reset the reference count */
-	memset(context, 0, offsetof(struct dc_state, refcount));
-
-	init_state(dc, context);
-}
-
 static bool update_planes_and_stream_state(struct dc *dc,
 		struct dc_surface_update *srf_updates, int surface_count,
 		struct dc_stream_state *stream,
@@ -2987,13 +2993,8 @@ static void commit_planes_do_stream_update(struct dc *dc,
 
 		if (!pipe_ctx->top_pipe &&  !pipe_ctx->prev_odm_pipe && pipe_ctx->stream == stream) {
 
-			if (stream_update->periodic_interrupt0 &&
-					dc->hwss.setup_periodic_interrupt)
-				dc->hwss.setup_periodic_interrupt(dc, pipe_ctx, VLINE0);
-
-			if (stream_update->periodic_interrupt1 &&
-					dc->hwss.setup_periodic_interrupt)
-				dc->hwss.setup_periodic_interrupt(dc, pipe_ctx, VLINE1);
+			if (stream_update->periodic_interrupt && dc->hwss.setup_periodic_interrupt)
+				dc->hwss.setup_periodic_interrupt(dc, pipe_ctx);
 
 			if ((stream_update->hdr_static_metadata && !stream->use_dynamic_meta) ||
 					stream_update->vrr_infopacket ||
@@ -3071,7 +3072,6 @@ static void commit_planes_do_stream_update(struct dc *dc,
 				} else {
 					if (get_seamless_boot_stream_count(context) == 0)
 						dc->hwss.prepare_bandwidth(dc, dc->current_state);
-
 					core_link_enable_stream(dc->current_state, pipe_ctx);
 				}
 			}
@@ -3099,11 +3099,9 @@ static void commit_planes_do_stream_update(struct dc *dc,
 
 static bool dc_dmub_should_send_dirty_rect_cmd(struct dc *dc, struct dc_stream_state *stream)
 {
-	if (stream->link->psr_settings.psr_version == DC_PSR_VERSION_SU_1)
-		return true;
-
-	if (stream->link->psr_settings.psr_version == DC_PSR_VERSION_1 &&
-	    dc->debug.enable_sw_cntl_psr)
+	if ((stream->link->psr_settings.psr_version == DC_PSR_VERSION_SU_1
+			|| stream->link->psr_settings.psr_version == DC_PSR_VERSION_1)
+			&& stream->ctx->dce_version >= DCN_VERSION_3_1)
 		return true;
 
 	return false;
@@ -3205,6 +3203,9 @@ static void commit_planes_for_stream(struct dc *dc,
 		if (get_seamless_boot_stream_count(context) == 0)
 			dc->hwss.prepare_bandwidth(dc, context);
 
+		if (dc->debug.enable_double_buffered_dsc_pg_support)
+			dc->hwss.update_dsc_pg(dc, context, false);
+
 		context_clock_trace(dc, context);
 	}
 
@@ -3319,10 +3320,6 @@ static void commit_planes_for_stream(struct dc *dc,
 		if (dc->hwss.program_front_end_for_ctx)
 			dc->hwss.program_front_end_for_ctx(dc, context);
 
-		if (update_type != UPDATE_TYPE_FAST)
-			if (dc->hwss.commit_subvp_config)
-				dc->hwss.commit_subvp_config(dc, context);
-
 		if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
 			dc->hwss.interdependent_update_lock(dc, context, false);
 		} else {
@@ -3330,16 +3327,15 @@ static void commit_planes_for_stream(struct dc *dc,
 		}
 		dc->hwss.post_unlock_program_front_end(dc, context);
 
+		if (update_type != UPDATE_TYPE_FAST)
+			if (dc->hwss.commit_subvp_config)
+				dc->hwss.commit_subvp_config(dc, context);
+
 		/* Since phantom pipe programming is moved to post_unlock_program_front_end,
 		 * move the SubVP lock to after the phantom pipes have been setup
 		 */
-		if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
-			if (dc->hwss.subvp_pipe_control_lock)
-				dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use);
-		} else {
-			if (dc->hwss.subvp_pipe_control_lock)
-				dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use);
-		}
+		if (dc->hwss.subvp_pipe_control_lock)
+			dc->hwss.subvp_pipe_control_lock(dc, context, false, should_lock_all_pipes, NULL, subvp_prev_use);
 		return;
 	}
 
@@ -3463,10 +3459,6 @@ static void commit_planes_for_stream(struct dc *dc,
 
 	}
 
-	if (update_type != UPDATE_TYPE_FAST)
-		if (dc->hwss.commit_subvp_config)
-			dc->hwss.commit_subvp_config(dc, context);
-
 	if (should_lock_all_pipes && dc->hwss.interdependent_update_lock) {
 		dc->hwss.interdependent_update_lock(dc, context, false);
 	} else {
@@ -3504,6 +3496,10 @@ static void commit_planes_for_stream(struct dc *dc,
 	if (update_type != UPDATE_TYPE_FAST)
 		dc->hwss.post_unlock_program_front_end(dc, context);
 
+	if (update_type != UPDATE_TYPE_FAST)
+		if (dc->hwss.commit_subvp_config)
+			dc->hwss.commit_subvp_config(dc, context);
+
 	/* Since phantom pipe programming is moved to post_unlock_program_front_end,
 	 * move the SubVP lock to after the phantom pipes have been setup
 	 */
@@ -3533,19 +3529,72 @@ static void commit_planes_for_stream(struct dc *dc,
 	}
 }
 
+/* Determines if the incoming context requires a applying transition state with unnecessary
+ * pipe splitting and ODM disabled, due to hardware limitations. In a case where
+ * the OPP associated with an MPCC might change due to plane additions, this function
+ * returns true.
+ */
+static bool could_mpcc_tree_change_for_active_pipes(struct dc *dc,
+		struct dc_stream_state *stream,
+		int surface_count,
+		bool *is_plane_addition)
+{
+
+	struct dc_stream_status *cur_stream_status = stream_get_status(dc->current_state, stream);
+	bool force_minimal_pipe_splitting = false;
+
+	*is_plane_addition = false;
+
+	if (cur_stream_status &&
+			dc->current_state->stream_count > 0 &&
+			dc->debug.pipe_split_policy != MPC_SPLIT_AVOID) {
+		/* determine if minimal transition is required due to MPC*/
+		if (surface_count > 0) {
+			if (cur_stream_status->plane_count > surface_count) {
+				force_minimal_pipe_splitting = true;
+			} else if (cur_stream_status->plane_count < surface_count) {
+				force_minimal_pipe_splitting = true;
+				*is_plane_addition = true;
+			}
+		}
+	}
+
+	if (cur_stream_status &&
+			dc->current_state->stream_count == 1 &&
+			dc->debug.enable_single_display_2to1_odm_policy) {
+		/* determine if minimal transition is required due to dynamic ODM*/
+		if (surface_count > 0) {
+			if (cur_stream_status->plane_count > 2 && cur_stream_status->plane_count > surface_count) {
+				force_minimal_pipe_splitting = true;
+			} else if (surface_count > 2 && cur_stream_status->plane_count < surface_count) {
+				force_minimal_pipe_splitting = true;
+				*is_plane_addition = true;
+			}
+		}
+	}
+
+	return force_minimal_pipe_splitting;
+}
+
 static bool commit_minimal_transition_state(struct dc *dc,
 		struct dc_state *transition_base_context)
 {
 	struct dc_state *transition_context = dc_create_state(dc);
-	enum pipe_split_policy tmp_policy;
+	enum pipe_split_policy tmp_mpc_policy;
+	bool temp_dynamic_odm_policy;
 	enum dc_status ret = DC_ERROR_UNEXPECTED;
 	unsigned int i, j;
 
 	if (!transition_context)
 		return false;
 
-	tmp_policy = dc->debug.pipe_split_policy;
-	dc->debug.pipe_split_policy = MPC_SPLIT_AVOID;
+	if (!dc->config.is_vmin_only_asic) {
+		tmp_mpc_policy = dc->debug.pipe_split_policy;
+		dc->debug.pipe_split_policy = MPC_SPLIT_AVOID;
+	}
+
+	temp_dynamic_odm_policy = dc->debug.enable_single_display_2to1_odm_policy;
+	dc->debug.enable_single_display_2to1_odm_policy = false;
 
 	dc_resource_state_copy_construct(transition_base_context, transition_context);
 
@@ -3567,19 +3616,22 @@ static bool commit_minimal_transition_state(struct dc *dc,
 		ret = dc_commit_state_no_check(dc, transition_context);
 	}
 
-	//always release as dc_commit_state_no_check retains in good case
+	/*always release as dc_commit_state_no_check retains in good case*/
 	dc_release_state(transition_context);
 
-	//restore previous pipe split policy
-	dc->debug.pipe_split_policy = tmp_policy;
+	/*restore previous pipe split and odm policy*/
+	if (!dc->config.is_vmin_only_asic)
+		dc->debug.pipe_split_policy = tmp_mpc_policy;
+
+	dc->debug.enable_single_display_2to1_odm_policy = temp_dynamic_odm_policy;
 
 	if (ret != DC_OK) {
-		//this should never happen
+		/*this should never happen*/
 		BREAK_TO_DEBUGGER();
 		return false;
 	}
 
-	//force full surface update
+	/*force full surface update*/
 	for (i = 0; i < dc->current_state->stream_count; i++) {
 		for (j = 0; j < dc->current_state->stream_status[i].plane_count; j++) {
 			dc->current_state->stream_status[i].plane_states[j]->update_flags.raw = 0xFFFFFFFF;
@@ -3602,22 +3654,14 @@ bool dc_update_planes_and_stream(struct dc *dc,
 	 * cause underflow. Apply stream configuration with minimal pipe
 	 * split first to avoid unsupported transitions for active pipes.
 	 */
-	bool force_minimal_pipe_splitting = false;
-	bool is_plane_addition = false;
-
-	struct dc_stream_status *cur_stream_status = stream_get_status(dc->current_state, stream);
+	bool force_minimal_pipe_splitting;
+	bool is_plane_addition;
 
-	if (cur_stream_status &&
-			dc->current_state->stream_count > 0 &&
-			dc->debug.pipe_split_policy != MPC_SPLIT_AVOID) {
-		/* determine if minimal transition is required */
-		if (cur_stream_status->plane_count > surface_count) {
-			force_minimal_pipe_splitting = true;
-		} else if (cur_stream_status->plane_count < surface_count) {
-			force_minimal_pipe_splitting = true;
-			is_plane_addition = true;
-		}
-	}
+	force_minimal_pipe_splitting = could_mpcc_tree_change_for_active_pipes(
+			dc,
+			stream,
+			surface_count,
+			&is_plane_addition);
 
 	/* on plane addition, minimal state is the current one */
 	if (force_minimal_pipe_splitting && is_plane_addition &&
@@ -3634,7 +3678,7 @@ bool dc_update_planes_and_stream(struct dc *dc,
 			&context))
 		return false;
 
-	/* on plane addition, minimal state is the new one */
+	/* on plane removal, minimal state is the new one */
 	if (force_minimal_pipe_splitting && !is_plane_addition) {
 		if (!commit_minimal_transition_state(dc, context)) {
 			dc_release_state(context);
@@ -4021,7 +4065,7 @@ struct dc_sink *dc_link_add_remote_sink(
 	 * Treat device as no EDID device if EDID
 	 * parsing fails
 	 */
-	if (edid_status != EDID_OK) {
+	if (edid_status != EDID_OK && edid_status != EDID_PARTIAL_VALID) {
 		dc_sink->dc_edid.length = 0;
 		dm_error("Bad EDID, status%d!\n", edid_status);
 	}
@@ -4276,8 +4320,8 @@ void dc_mclk_switch_using_fw_based_vblank_stretch_shut_down(struct dc *dc)
 /*
  *****************************************************************************
  * Function: dc_is_dmub_outbox_supported -
- * 
- * @brief 
+ *
+ * @brief
  *      Checks whether DMUB FW supports outbox notifications, if supported
  *		DM should register outbox interrupt prior to actually enabling interrupts
  *		via dc_enable_dmub_outbox
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index 2a8007928210..7c2e3b8dc26a 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -402,6 +402,44 @@ void get_hdr_visual_confirm_color(
 	}
 }
 
+void get_subvp_visual_confirm_color(
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		struct tg_color *color)
+{
+	uint32_t color_value = MAX_TG_COLOR_VALUE;
+	bool enable_subvp = false;
+	int i;
+
+	if (!dc->ctx || !dc->ctx->dmub_srv || !pipe_ctx)
+		return;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream && pipe->stream->mall_stream_config.paired_stream &&
+		    pipe->stream->mall_stream_config.type == SUBVP_MAIN) {
+			/* SubVP enable - red */
+			color->color_r_cr = color_value;
+			enable_subvp = true;
+
+			if (pipe_ctx->stream == pipe->stream)
+				return;
+			break;
+		}
+	}
+
+	if (enable_subvp && pipe_ctx->stream->mall_stream_config.type == SUBVP_NONE) {
+		color->color_r_cr = 0;
+		if (pipe_ctx->stream->ignore_msa_timing_param == 1)
+			/* SubVP enable and DRR on - green */
+			color->color_g_y = color_value;
+		else
+			/* SubVP enable and No DRR - blue */
+			color->color_b_cb = color_value;
+	}
+}
+
 void get_surface_tile_visual_confirm_color(
 		struct pipe_ctx *pipe_ctx,
 		struct tg_color *color)
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
index 66d2ae7aacf5..3d19fb92333b 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c
@@ -832,8 +832,9 @@ static bool discover_dp_mst_topology(struct dc_link *link, enum dc_detect_reason
 	LINK_INFO("link=%d, mst branch is now Connected\n",
 		  link->link_index);
 
-	apply_dpia_mst_dsc_always_on_wa(link);
 	link->type = dc_connection_mst_branch;
+	apply_dpia_mst_dsc_always_on_wa(link);
+
 	dm_helpers_dp_update_branch_info(link->ctx, link);
 	if (dm_helpers_dp_mst_start_top_mgr(link->ctx,
 			link, (reason == DETECT_REASON_BOOT || reason == DETECT_REASON_RESUMEFROMS3S4))) {
@@ -847,20 +848,13 @@ static bool discover_dp_mst_topology(struct dc_link *link, enum dc_detect_reason
 
 bool reset_cur_dp_mst_topology(struct dc_link *link)
 {
-	bool result = false;
 	DC_LOGGER_INIT(link->ctx->logger);
 
 	LINK_INFO("link=%d, mst branch is now Disconnected\n",
 		  link->link_index);
 
 	revert_dpia_mst_dsc_always_on_wa(link);
-	result = dm_helpers_dp_mst_stop_top_mgr(link->ctx, link);
-
-	link->mst_stream_alloc_table.stream_count = 0;
-	memset(link->mst_stream_alloc_table.stream_allocations,
-			0,
-			sizeof(link->mst_stream_alloc_table.stream_allocations));
-	return result;
+	return dm_helpers_dp_mst_stop_top_mgr(link->ctx, link);
 }
 
 static bool should_prepare_phy_clocks_for_link_verification(const struct dc *dc,
@@ -1311,6 +1305,14 @@ static bool detect_link_and_local_sink(struct dc_link *link,
 				sink->edid_caps.audio_modes[i].sample_rate,
 				sink->edid_caps.audio_modes[i].sample_size);
 		}
+
+		if (link->connector_signal == SIGNAL_TYPE_EDP) {
+			// Init dc_panel_config
+			dm_helpers_init_panel_settings(dc_ctx, &link->panel_config, sink);
+			// Override dc_panel_config if system has specific settings
+			dm_helpers_override_panel_settings(dc_ctx, &link->panel_config);
+		}
+
 	} else {
 		/* From Connected-to-Disconnected. */
 		link->type = dc_connection_none;
@@ -1975,7 +1977,7 @@ static enum dc_status enable_link_dp(struct dc_state *state,
 	int i;
 	bool apply_seamless_boot_optimization = false;
 	uint32_t bl_oled_enable_delay = 50; // in ms
-	const uint32_t post_oui_delay = 30; // 30ms
+	uint32_t post_oui_delay = 30; // 30ms
 	/* Reduce link bandwidth between failed link training attempts. */
 	bool do_fallback = false;
 
@@ -2022,8 +2024,10 @@ static enum dc_status enable_link_dp(struct dc_state *state,
 
 	// during mode switch we do DP_SET_POWER off then on, and OUI is lost
 	dpcd_set_source_specific_data(link);
-	if (link->dpcd_sink_ext_caps.raw != 0)
+	if (link->dpcd_sink_ext_caps.raw != 0) {
+		post_oui_delay += link->panel_config.pps.extra_post_OUI_ms;
 		msleep(post_oui_delay);
+	}
 
 	// similarly, mode switch can cause loss of cable ID
 	dpcd_write_cable_id_to_dprx(link);
@@ -2069,11 +2073,7 @@ static enum dc_status enable_link_edp(
 		struct dc_state *state,
 		struct pipe_ctx *pipe_ctx)
 {
-	enum dc_status status;
-
-	status = enable_link_dp(state, pipe_ctx);
-
-	return status;
+	return enable_link_dp(state, pipe_ctx);
 }
 
 static enum dc_status enable_link_dp_mst(
@@ -2639,9 +2639,8 @@ static void disable_link(struct dc_link *link, const struct link_resource *link_
 				dp_set_fec_ready(link, link_res, false);
 			}
 		}
-	} else {
-		if (signal != SIGNAL_TYPE_VIRTUAL)
-			link->link_enc->funcs->disable_output(link->link_enc, signal);
+	} else if (signal != SIGNAL_TYPE_VIRTUAL) {
+		link->dc->hwss.disable_link_output(link, link_res, signal);
 	}
 
 	if (signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
@@ -2663,6 +2662,7 @@ static void enable_link_hdmi(struct pipe_ctx *pipe_ctx)
 	bool is_over_340mhz = false;
 	bool is_vga_mode = (stream->timing.h_addressable == 640)
 			&& (stream->timing.v_addressable == 480);
+	struct dc *dc = pipe_ctx->stream->ctx->dc;
 
 	if (stream->phy_pix_clk == 0)
 		stream->phy_pix_clk = stream->timing.pix_clk_100hz / 10;
@@ -2702,11 +2702,12 @@ static void enable_link_hdmi(struct pipe_ctx *pipe_ctx)
 	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422)
 		display_color_depth = COLOR_DEPTH_888;
 
-	link->link_enc->funcs->enable_tmds_output(
-			link->link_enc,
+	dc->hwss.enable_tmds_link_output(
+			link,
+			&pipe_ctx->link_res,
+			pipe_ctx->stream->signal,
 			pipe_ctx->clock_source->id,
 			display_color_depth,
-			pipe_ctx->stream->signal,
 			stream->phy_pix_clk);
 
 	if (dc_is_hdmi_signal(pipe_ctx->stream->signal))
@@ -2717,15 +2718,16 @@ static void enable_link_lvds(struct pipe_ctx *pipe_ctx)
 {
 	struct dc_stream_state *stream = pipe_ctx->stream;
 	struct dc_link *link = stream->link;
+	struct dc *dc = stream->ctx->dc;
 
 	if (stream->phy_pix_clk == 0)
 		stream->phy_pix_clk = stream->timing.pix_clk_100hz / 10;
 
 	memset(&stream->link->cur_link_settings, 0,
 			sizeof(struct dc_link_settings));
-
-	link->link_enc->funcs->enable_lvds_output(
-			link->link_enc,
+	dc->hwss.enable_lvds_link_output(
+			link,
+			&pipe_ctx->link_res,
 			pipe_ctx->clock_source->id,
 			stream->phy_pix_clk);
 
@@ -3516,7 +3518,7 @@ static void update_mst_stream_alloc_table(
 	struct dc_link *link,
 	struct stream_encoder *stream_enc,
 	struct hpo_dp_stream_encoder *hpo_dp_stream_enc, // TODO: Rename stream_enc to dio_stream_enc?
-	const struct dp_mst_stream_allocation_table *proposed_table)
+	const struct dc_dp_mst_stream_allocation_table *proposed_table)
 {
 	struct link_mst_stream_allocation work_table[MAX_CONTROLLER_NUM] = { 0 };
 	struct link_mst_stream_allocation *dc_alloc;
@@ -3563,6 +3565,35 @@ static void update_mst_stream_alloc_table(
 				work_table[i];
 }
 
+static void remove_stream_from_alloc_table(
+		struct dc_link *link,
+		struct stream_encoder *dio_stream_enc,
+		struct hpo_dp_stream_encoder *hpo_dp_stream_enc)
+{
+	int i = 0;
+	struct link_mst_stream_allocation_table *table =
+			&link->mst_stream_alloc_table;
+
+	if (hpo_dp_stream_enc) {
+		for (; i < table->stream_count; i++)
+			if (hpo_dp_stream_enc == table->stream_allocations[i].hpo_dp_stream_enc)
+				break;
+	} else {
+		for (; i < table->stream_count; i++)
+			if (dio_stream_enc == table->stream_allocations[i].stream_enc)
+				break;
+	}
+
+	if (i < table->stream_count) {
+		i++;
+		for (; i < table->stream_count; i++)
+			table->stream_allocations[i-1] = table->stream_allocations[i];
+		memset(&table->stream_allocations[table->stream_count-1], 0,
+				sizeof(struct link_mst_stream_allocation));
+		table->stream_count--;
+	}
+}
+
 static void dc_log_vcp_x_y(const struct dc_link *link, struct fixed31_32 avg_time_slots_per_mtp)
 {
 	const uint32_t VCP_Y_PRECISION = 1000;
@@ -3679,7 +3710,7 @@ enum dc_status dc_link_allocate_mst_payload(struct pipe_ctx *pipe_ctx)
 {
 	struct dc_stream_state *stream = pipe_ctx->stream;
 	struct dc_link *link = stream->link;
-	struct dp_mst_stream_allocation_table proposed_table = {0};
+	struct dc_dp_mst_stream_allocation_table proposed_table = {0};
 	struct fixed31_32 avg_time_slots_per_mtp;
 	struct fixed31_32 pbn;
 	struct fixed31_32 pbn_per_slot;
@@ -3784,7 +3815,7 @@ enum dc_status dc_link_reduce_mst_payload(struct pipe_ctx *pipe_ctx, uint32_t bw
 	struct fixed31_32 avg_time_slots_per_mtp;
 	struct fixed31_32 pbn;
 	struct fixed31_32 pbn_per_slot;
-	struct dp_mst_stream_allocation_table proposed_table = {0};
+	struct dc_dp_mst_stream_allocation_table proposed_table = {0};
 	uint8_t i;
 	const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res);
 	DC_LOGGER_INIT(link->ctx->logger);
@@ -3873,7 +3904,7 @@ enum dc_status dc_link_increase_mst_payload(struct pipe_ctx *pipe_ctx, uint32_t
 	struct fixed31_32 avg_time_slots_per_mtp;
 	struct fixed31_32 pbn;
 	struct fixed31_32 pbn_per_slot;
-	struct dp_mst_stream_allocation_table proposed_table = {0};
+	struct dc_dp_mst_stream_allocation_table proposed_table = {0};
 	uint8_t i;
 	enum act_return_status ret;
 	const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res);
@@ -3957,7 +3988,7 @@ static enum dc_status deallocate_mst_payload(struct pipe_ctx *pipe_ctx)
 {
 	struct dc_stream_state *stream = pipe_ctx->stream;
 	struct dc_link *link = stream->link;
-	struct dp_mst_stream_allocation_table proposed_table = {0};
+	struct dc_dp_mst_stream_allocation_table proposed_table = {0};
 	struct fixed31_32 avg_time_slots_per_mtp = dc_fixpt_from_int(0);
 	int i;
 	bool mst_mode = (link->type == dc_connection_mst_branch);
@@ -3980,26 +4011,32 @@ static enum dc_status deallocate_mst_payload(struct pipe_ctx *pipe_ctx)
 				&empty_link_settings,
 				avg_time_slots_per_mtp);
 
-	/* TODO: which component is responsible for remove payload table? */
 	if (mst_mode) {
+		/* when link is in mst mode, reply on mst manager to remove
+		 * payload
+		 */
 		if (dm_helpers_dp_mst_write_payload_allocation_table(
 				stream->ctx,
 				stream,
 				&proposed_table,
-				false)) {
+				false))
 
 			update_mst_stream_alloc_table(
-						link,
-						pipe_ctx->stream_res.stream_enc,
-						pipe_ctx->stream_res.hpo_dp_stream_enc,
-						&proposed_table);
-		}
-		else {
-				DC_LOG_WARNING("Failed to update"
-						"MST allocation table for"
-						"pipe idx:%d\n",
-						pipe_ctx->pipe_idx);
-		}
+					link,
+					pipe_ctx->stream_res.stream_enc,
+					pipe_ctx->stream_res.hpo_dp_stream_enc,
+					&proposed_table);
+		else
+			DC_LOG_WARNING("Failed to update"
+					"MST allocation table for"
+					"pipe idx:%d\n",
+					pipe_ctx->pipe_idx);
+	} else {
+		/* when link is no longer in mst mode (mst hub unplugged),
+		 * remove payload with default dc logic
+		 */
+		remove_stream_from_alloc_table(link, pipe_ctx->stream_res.stream_enc,
+				pipe_ctx->stream_res.hpo_dp_stream_enc);
 	}
 
 	DC_LOG_MST("%s"
@@ -4303,8 +4340,9 @@ void core_link_enable_stream(
 		 */
 		if (pipe_ctx->stream->timing.flags.DSC) {
 			if (dc_is_dp_signal(pipe_ctx->stream->signal) ||
-					dc_is_virtual_signal(pipe_ctx->stream->signal))
-				dp_set_dsc_enable(pipe_ctx, true);
+				dc_is_virtual_signal(pipe_ctx->stream->signal))
+			dp_set_dsc_enable(pipe_ctx, true);
+
 		}
 
 		status = enable_link(state, pipe_ctx);
@@ -4736,7 +4774,7 @@ bool dc_link_should_enable_fec(const struct dc_link *link)
 	else if (link->connector_signal == SIGNAL_TYPE_EDP
 			&& (link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.
 			 dsc_support.DSC_SUPPORT == false
-				|| link->dc->debug.disable_dsc_edp
+				|| link->panel_config.dsc.disable_dsc_edp
 				|| !link->dc->caps.edp_dsc_support))
 		force_disable = true;
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
index d01d2eeed813..651231387043 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c
@@ -35,6 +35,8 @@
 #include "dc_link_ddc.h"
 #include "dce/dce_aux.h"
 #include "dmub/inc/dmub_cmd.h"
+#include "link_dpcd.h"
+#include "include/dal_asic_id.h"
 
 #define DC_LOGGER_INIT(logger)
 
@@ -683,6 +685,21 @@ bool dc_link_aux_try_to_configure_timeout(struct ddc_service *ddc,
 	bool result = false;
 	struct ddc *ddc_pin = ddc->ddc_pin;
 
+	if ((ddc->link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) &&
+			!ddc->link->dc->debug.disable_fixed_vs_aux_timeout_wa &&
+			ASICREV_IS_YELLOW_CARP(ddc->ctx->asic_id.hw_internal_rev)) {
+		/* Fixed VS workaround for AUX timeout */
+		const uint32_t fixed_vs_address = 0xF004F;
+		const uint8_t fixed_vs_data[4] = {0x1, 0x22, 0x63, 0xc};
+
+		core_link_write_dpcd(ddc->link,
+				fixed_vs_address,
+				fixed_vs_data,
+				sizeof(fixed_vs_data));
+
+		timeout = 3072;
+	}
+
 	/* Do not try to access nonexistent DDC pin. */
 	if (ddc->link->ep_type != DISPLAY_ENDPOINT_PHY)
 		return true;
@@ -691,6 +708,7 @@ bool dc_link_aux_try_to_configure_timeout(struct ddc_service *ddc,
 		ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en]->funcs->configure_timeout(ddc, timeout);
 		result = true;
 	}
+
 	return result;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
index 780f7f4c28b6..c57df45e83ff 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c
@@ -526,9 +526,9 @@ uint8_t dc_dp_initialize_scrambling_data_symbols(
 	return disable_scrabled_data_symbols;
 }
 
-static inline bool is_repeater(struct dc_link *link, uint32_t offset)
+static inline bool is_repeater(const struct link_training_settings *lt_settings, uint32_t offset)
 {
-	return (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && (offset != 0);
+	return (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && (offset != 0);
 }
 
 static void dpcd_set_lt_pattern_and_lane_settings(
@@ -545,7 +545,7 @@ static void dpcd_set_lt_pattern_and_lane_settings(
 	bool edp_workaround = false; /* TODO link_prop.INTERNAL */
 	dpcd_base_lt_offset = DP_TRAINING_PATTERN_SET;
 
-	if (is_repeater(link, offset))
+	if (is_repeater(lt_settings, offset))
 		dpcd_base_lt_offset = DP_TRAINING_PATTERN_SET_PHY_REPEATER1 +
 			((DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE) * (offset - 1));
 
@@ -561,7 +561,7 @@ static void dpcd_set_lt_pattern_and_lane_settings(
 	dpcd_lt_buffer[DP_TRAINING_PATTERN_SET - DP_TRAINING_PATTERN_SET]
 		= dpcd_pattern.raw;
 
-	if (is_repeater(link, offset)) {
+	if (is_repeater(lt_settings, offset)) {
 		DC_LOG_HW_LINK_TRAINING("%s\n LTTPR Repeater ID: %d\n 0x%X pattern = %x\n",
 			__func__,
 			offset,
@@ -584,7 +584,7 @@ static void dpcd_set_lt_pattern_and_lane_settings(
 		lt_settings->dpcd_lane_settings,
 		size_in_bytes);
 
-	if (is_repeater(link, offset)) {
+	if (is_repeater(lt_settings, offset)) {
 		if (dp_get_link_encoding_format(&lt_settings->link_settings) ==
 				DP_128b_132b_ENCODING)
 			DC_LOG_HW_LINK_TRAINING("%s:\n LTTPR Repeater ID: %d\n"
@@ -873,7 +873,7 @@ enum dc_status dp_get_lane_status_and_lane_adjust(
 	uint32_t lane;
 	enum dc_status status;
 
-	if (is_repeater(link, offset)) {
+	if (is_repeater(link_training_setting, offset)) {
 		lane01_status_address =
 				DP_LANE0_1_STATUS_PHY_REPEATER1 +
 				((DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE) * (offset - 1));
@@ -906,7 +906,7 @@ enum dc_status dp_get_lane_status_and_lane_adjust(
 
 	ln_align->raw = dpcd_buf[2];
 
-	if (is_repeater(link, offset)) {
+	if (is_repeater(link_training_setting, offset)) {
 		DC_LOG_HW_LINK_TRAINING("%s:\n LTTPR Repeater ID: %d\n"
 				" 0x%X Lane01Status = %x\n 0x%X Lane23Status = %x\n ",
 			__func__,
@@ -954,7 +954,7 @@ enum dc_status dpcd_set_lane_settings(
 
 	lane0_set_address = DP_TRAINING_LANE0_SET;
 
-	if (is_repeater(link, offset))
+	if (is_repeater(link_training_setting, offset))
 		lane0_set_address = DP_TRAINING_LANE0_SET_PHY_REPEATER1 +
 		((DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE) * (offset - 1));
 
@@ -963,7 +963,7 @@ enum dc_status dpcd_set_lane_settings(
 		(uint8_t *)(link_training_setting->dpcd_lane_settings),
 		link_training_setting->link_settings.lane_count);
 
-	if (is_repeater(link, offset)) {
+	if (is_repeater(link_training_setting, offset)) {
 		if (dp_get_link_encoding_format(&link_training_setting->link_settings) ==
 				DP_128b_132b_ENCODING)
 			DC_LOG_HW_LINK_TRAINING("%s:\n LTTPR Repeater ID: %d\n"
@@ -1172,7 +1172,7 @@ static enum link_training_result perform_channel_equalization_sequence(
 	/* Note: also check that TPS4 is a supported feature*/
 	tr_pattern = lt_settings->pattern_for_eq;
 
-	if (is_repeater(link, offset) && dp_get_link_encoding_format(&lt_settings->link_settings) == DP_8b_10b_ENCODING)
+	if (is_repeater(lt_settings, offset) && dp_get_link_encoding_format(&lt_settings->link_settings) == DP_8b_10b_ENCODING)
 		tr_pattern = DP_TRAINING_PATTERN_SEQUENCE_4;
 
 	dp_set_hw_training_pattern(link, link_res, tr_pattern, offset);
@@ -1198,7 +1198,7 @@ static enum link_training_result perform_channel_equalization_sequence(
 		/* 3. wait for receiver to lock-on*/
 		wait_time_microsec = lt_settings->eq_pattern_time;
 
-		if (is_repeater(link, offset))
+		if (is_repeater(lt_settings, offset))
 			wait_time_microsec =
 					dp_translate_training_aux_read_interval(
 						link->dpcd_caps.lttpr_caps.aux_rd_interval[offset - 1]);
@@ -1469,7 +1469,6 @@ static inline void decide_8b_10b_training_settings(
 	 */
 	lt_settings->link_settings.link_spread = link->dp_ss_off ?
 			LINK_SPREAD_DISABLED : LINK_SPREAD_05_DOWNSPREAD_30KHZ;
-	lt_settings->lttpr_mode = link->lttpr_mode;
 	lt_settings->cr_pattern_time = get_cr_training_aux_rd_interval(link, link_setting);
 	lt_settings->eq_pattern_time = get_eq_training_aux_rd_interval(link, link_setting);
 	lt_settings->pattern_for_cr = decide_cr_training_pattern(link_setting);
@@ -1478,6 +1477,7 @@ static inline void decide_8b_10b_training_settings(
 	lt_settings->should_set_fec_ready = true;
 	lt_settings->disallow_per_lane_settings = true;
 	lt_settings->always_match_dpcd_with_hw_lane_settings = true;
+	lt_settings->lttpr_mode = dp_decide_8b_10b_lttpr_mode(link);
 	dp_hw_to_dpcd_lane_settings(lt_settings, lt_settings->hw_lane_settings, lt_settings->dpcd_lane_settings);
 }
 
@@ -1501,9 +1501,8 @@ static inline void decide_128b_132b_training_settings(struct dc_link *link,
 	lt_settings->cds_pattern_time = 2500;
 	lt_settings->cds_wait_time_limit = (dp_convert_to_count(
 			link->dpcd_caps.lttpr_caps.phy_repeater_cnt) + 1) * 20000;
-	lt_settings->lttpr_mode = dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt) ?
-			LTTPR_MODE_NON_TRANSPARENT : LTTPR_MODE_TRANSPARENT;
 	lt_settings->disallow_per_lane_settings = true;
+	lt_settings->lttpr_mode = dp_decide_128b_132b_lttpr_mode(link);
 	dp_hw_to_dpcd_lane_settings(lt_settings,
 			lt_settings->hw_lane_settings, lt_settings->dpcd_lane_settings);
 }
@@ -1543,7 +1542,7 @@ static void override_training_settings(
 		lt_settings->ffe_preset = overrides->ffe_preset;
 	/* Override HW lane settings with BIOS forced values if present */
 	if (link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN &&
-			link->lttpr_mode == LTTPR_MODE_TRANSPARENT) {
+			lt_settings->lttpr_mode == LTTPR_MODE_TRANSPARENT) {
 		lt_settings->voltage_swing = &link->bios_forced_drive_settings.VOLTAGE_SWING;
 		lt_settings->pre_emphasis = &link->bios_forced_drive_settings.PRE_EMPHASIS;
 		lt_settings->always_match_dpcd_with_hw_lane_settings = false;
@@ -1584,6 +1583,15 @@ static void override_training_settings(
 
 	if (link->preferred_training_settings.fec_enable != NULL)
 		lt_settings->should_set_fec_ready = *link->preferred_training_settings.fec_enable;
+
+	#if defined(CONFIG_DRM_AMD_DC_DCN)
+	/* Check DP tunnel LTTPR mode debug option. */
+	if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA && link->dc->debug.dpia_debug.bits.force_non_lttpr)
+		lt_settings->lttpr_mode = LTTPR_MODE_NON_LTTPR;
+
+#endif
+	dp_get_lttpr_mode_override(link, &lt_settings->lttpr_mode);
+
 }
 
 uint8_t dp_convert_to_count(uint8_t lttpr_repeater_count)
@@ -1649,7 +1657,7 @@ static enum dc_status configure_lttpr_mode_non_transparent(
 		link->dpcd_caps.lttpr_caps.mode = repeater_mode;
 	}
 
-	if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
+	if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
 
 		DC_LOG_HW_LINK_TRAINING("%s\n Set LTTPR to Non Transparent Mode\n", __func__);
 
@@ -2099,7 +2107,7 @@ static enum link_training_result dp_perform_8b_10b_link_training(
 	/* 1. set link rate, lane count and spread. */
 	dpcd_set_link_settings(link, lt_settings);
 
-	if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
+	if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
 
 		/* 2. perform link training (set link training done
 		 *  to false is done as well)
@@ -2216,7 +2224,7 @@ static enum link_training_result perform_fixed_vs_pe_nontransparent_training_seq
 
 	link->vendor_specific_lttpr_link_rate_wa = target_rate;
 
-	if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
+	if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
 
 		/* 2. perform link training (set link training done
 		 *  to false is done as well)
@@ -2288,7 +2296,7 @@ static enum link_training_result dp_perform_fixed_vs_pe_training_sequence(
 	ASSERT(dp_get_link_encoding_format(&lt_settings->link_settings) ==
 			DP_8b_10b_ENCODING);
 
-	if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
+	if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
 		status = perform_fixed_vs_pe_nontransparent_training_sequence(link, link_res, lt_settings);
 		return status;
 	}
@@ -2635,6 +2643,7 @@ enum link_training_result dc_link_dp_perform_link_training(
 			link,
 			link_settings,
 			&lt_settings);
+
 	override_training_settings(
 			link,
 			&link->preferred_training_settings,
@@ -2652,7 +2661,7 @@ enum link_training_result dc_link_dp_perform_link_training(
 	 * Per DP specs starting from here, DPTX device shall not issue
 	 * Non-LT AUX transactions inside training mode.
 	 */
-	if (link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN)
+	if (link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN && encoding == DP_8b_10b_ENCODING)
 		status = dp_perform_fixed_vs_pe_training_sequence(link, link_res, &lt_settings);
 	else if (encoding == DP_8b_10b_ENCODING)
 		status = dp_perform_8b_10b_link_training(link, link_res, &lt_settings);
@@ -3086,7 +3095,7 @@ struct dc_link_settings dp_get_max_link_cap(struct dc_link *link)
 	 * account for lttpr repeaters cap
 	 * notes: repeaters do not snoop in the DPRX Capabilities addresses (3.6.3).
 	 */
-	if (link->lttpr_mode != LTTPR_MODE_NON_LTTPR) {
+	if (dp_is_lttpr_present(link)) {
 		if (link->dpcd_caps.lttpr_caps.max_lane_count < max_link_cap.lane_count)
 			max_link_cap.lane_count = link->dpcd_caps.lttpr_caps.max_lane_count;
 		lttpr_max_link_rate = get_lttpr_max_link_rate(link);
@@ -3240,7 +3249,7 @@ static bool dp_verify_link_cap(
 	cur_link_settings = max_link_settings;
 
 	/* Grant extended timeout request */
-	if ((link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && (link->dpcd_caps.lttpr_caps.max_ext_timeout > 0)) {
+	if (dp_is_lttpr_present(link) && link->dpcd_caps.lttpr_caps.max_ext_timeout > 0) {
 		uint8_t grant = link->dpcd_caps.lttpr_caps.max_ext_timeout & 0x80;
 
 		core_link_write_dpcd(link, DP_PHY_REPEATER_EXTENDED_WAIT_TIMEOUT, &grant, sizeof(grant));
@@ -3749,7 +3758,7 @@ static bool decide_edp_link_settings_with_dsc(struct dc_link *link,
 
 	unsigned int policy = 0;
 
-	policy = link->ctx->dc->debug.force_dsc_edp_policy;
+	policy = link->panel_config.dsc.force_dsc_edp_policy;
 	if (max_link_rate == LINK_RATE_UNKNOWN)
 		max_link_rate = link->verified_link_cap.link_rate;
 	/*
@@ -3915,7 +3924,7 @@ bool decide_link_settings(struct dc_stream_state *stream,
 		if (stream->timing.flags.DSC) {
 			enum dc_link_rate max_link_rate = LINK_RATE_UNKNOWN;
 
-			if (link->ctx->dc->debug.force_dsc_edp_policy) {
+			if (link->panel_config.dsc.force_dsc_edp_policy) {
 				/* calculate link max link rate cap*/
 				struct dc_link_settings tmp_link_setting;
 				struct dc_crtc_timing tmp_timing = stream->timing;
@@ -4101,8 +4110,13 @@ static void dp_test_send_phy_test_pattern(struct dc_link *link)
 			&dpcd_lane_adjustment[0].raw,
 			sizeof(dpcd_lane_adjustment));
 
+	/* prepare link training settings */
+	link_training_settings.link_settings = link->cur_link_settings;
+
+	link_training_settings.lttpr_mode = dp_decide_lttpr_mode(link, &link->cur_link_settings);
+
 	if ((link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) &&
-			link->lttpr_mode == LTTPR_MODE_TRANSPARENT)
+			link_training_settings.lttpr_mode == LTTPR_MODE_TRANSPARENT)
 		dp_fixed_vs_pe_read_lane_adjust(
 				link,
 				link_training_settings.dpcd_lane_settings);
@@ -4209,9 +4223,6 @@ static void dp_test_send_phy_test_pattern(struct dc_link *link)
 				test_pattern_size);
 	}
 
-	/* prepare link training settings */
-	link_training_settings.link_settings = link->cur_link_settings;
-
 	for (lane = 0; lane <
 		(unsigned int)(link->cur_link_settings.lane_count);
 		lane++) {
@@ -4524,17 +4535,15 @@ void dc_link_dp_handle_link_loss(struct dc_link *link)
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx && pipe_ctx->stream && !pipe_ctx->stream->dpms_off &&
-				pipe_ctx->stream->link == link && !pipe_ctx->prev_odm_pipe) {
+				pipe_ctx->stream->link == link && !pipe_ctx->prev_odm_pipe)
 			core_link_disable_stream(pipe_ctx);
-		}
 	}
 
 	for (i = 0; i < MAX_PIPES; i++) {
 		pipe_ctx = &link->dc->current_state->res_ctx.pipe_ctx[i];
 		if (pipe_ctx && pipe_ctx->stream && !pipe_ctx->stream->dpms_off &&
-				pipe_ctx->stream->link == link && !pipe_ctx->prev_odm_pipe) {
+				pipe_ctx->stream->link == link && !pipe_ctx->prev_odm_pipe)
 			core_link_enable_stream(link->dc->current_state, pipe_ctx);
-		}
 	}
 }
 
@@ -5023,129 +5032,136 @@ static bool dpcd_read_sink_ext_caps(struct dc_link *link)
 	return true;
 }
 
-/* Logic to determine LTTPR mode */
-static void determine_lttpr_mode(struct dc_link *link)
+bool dp_retrieve_lttpr_cap(struct dc_link *link)
 {
-	bool allow_lttpr_non_transparent_mode = 0;
-	bool vbios_lttpr_enable = link->dc->caps.vbios_lttpr_enable;
+	uint8_t lttpr_dpcd_data[8];
+	enum dc_status status = DC_ERROR_UNEXPECTED;
+	bool is_lttpr_present = false;
+
+	/* Logic to determine LTTPR support*/
 	bool vbios_lttpr_interop = link->dc->caps.vbios_lttpr_aware;
 
+	if (!vbios_lttpr_interop || !link->dc->caps.extended_aux_timeout_support)
+		return false;
 
-	if ((link->dc->config.allow_lttpr_non_transparent_mode.bits.DP2_0 &&
-			link->dpcd_caps.channel_coding_cap.bits.DP_128b_132b_SUPPORTED)) {
-		allow_lttpr_non_transparent_mode = 1;
-	} else if (link->dc->config.allow_lttpr_non_transparent_mode.bits.DP1_4A &&
-			!link->dpcd_caps.channel_coding_cap.bits.DP_128b_132b_SUPPORTED) {
-		allow_lttpr_non_transparent_mode = 1;
+	/* By reading LTTPR capability, RX assumes that we will enable
+	 * LTTPR extended aux timeout if LTTPR is present.
+	 */
+	status = core_link_read_dpcd(link,
+			DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV,
+			lttpr_dpcd_data,
+			sizeof(lttpr_dpcd_data));
+
+	link->dpcd_caps.lttpr_caps.revision.raw =
+			lttpr_dpcd_data[DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV -
+							DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
+
+	link->dpcd_caps.lttpr_caps.max_link_rate =
+			lttpr_dpcd_data[DP_MAX_LINK_RATE_PHY_REPEATER -
+							DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
+
+	link->dpcd_caps.lttpr_caps.phy_repeater_cnt =
+			lttpr_dpcd_data[DP_PHY_REPEATER_CNT -
+							DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
+
+	link->dpcd_caps.lttpr_caps.max_lane_count =
+			lttpr_dpcd_data[DP_MAX_LANE_COUNT_PHY_REPEATER -
+							DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
+
+	link->dpcd_caps.lttpr_caps.mode =
+			lttpr_dpcd_data[DP_PHY_REPEATER_MODE -
+							DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
+
+	link->dpcd_caps.lttpr_caps.max_ext_timeout =
+			lttpr_dpcd_data[DP_PHY_REPEATER_EXTENDED_WAIT_TIMEOUT -
+							DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
+	link->dpcd_caps.lttpr_caps.main_link_channel_coding.raw =
+			lttpr_dpcd_data[DP_MAIN_LINK_CHANNEL_CODING_PHY_REPEATER -
+							DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
+
+	link->dpcd_caps.lttpr_caps.supported_128b_132b_rates.raw =
+			lttpr_dpcd_data[DP_PHY_REPEATER_128B132B_RATES -
+							DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
+
+	/* If this chip cap is set, at least one retimer must exist in the chain
+	 * Override count to 1 if we receive a known bad count (0 or an invalid value)
+	 */
+	if (link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN &&
+			(dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt) == 0)) {
+		ASSERT(0);
+		link->dpcd_caps.lttpr_caps.phy_repeater_cnt = 0x80;
 	}
 
-	link->lttpr_mode = LTTPR_MODE_NON_LTTPR;
-	if (vbios_lttpr_enable && vbios_lttpr_interop)
-		link->lttpr_mode = LTTPR_MODE_NON_TRANSPARENT;
-	else if (!vbios_lttpr_enable && vbios_lttpr_interop) {
-		if (allow_lttpr_non_transparent_mode)
-			link->lttpr_mode = LTTPR_MODE_NON_TRANSPARENT;
-		else
-			link->lttpr_mode = LTTPR_MODE_TRANSPARENT;
-	} else if (!vbios_lttpr_enable && !vbios_lttpr_interop) {
-		if (!allow_lttpr_non_transparent_mode || !link->dc->caps.extended_aux_timeout_support)
-			link->lttpr_mode = LTTPR_MODE_NON_LTTPR;
-		else
-			link->lttpr_mode = LTTPR_MODE_NON_TRANSPARENT;
-	}
+	/* Attempt to train in LTTPR transparent mode if repeater count exceeds 8. */
+	is_lttpr_present = dp_is_lttpr_present(link);
 
-#if defined(CONFIG_DRM_AMD_DC_DCN)
-	/* Check DP tunnel LTTPR mode debug option. */
-	if (link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA &&
-	    link->dc->debug.dpia_debug.bits.force_non_lttpr)
-		link->lttpr_mode = LTTPR_MODE_NON_LTTPR;
-#endif
+	if (is_lttpr_present)
+		CONN_DATA_DETECT(link, lttpr_dpcd_data, sizeof(lttpr_dpcd_data), "LTTPR Caps: ");
+
+	return is_lttpr_present;
 }
 
-bool dp_retrieve_lttpr_cap(struct dc_link *link)
+bool dp_is_lttpr_present(struct dc_link *link)
 {
-	uint8_t lttpr_dpcd_data[8];
-	enum dc_status status = DC_ERROR_UNEXPECTED;
-	bool is_lttpr_present = false;
+	return (dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt) != 0 &&
+			link->dpcd_caps.lttpr_caps.max_lane_count > 0 &&
+			link->dpcd_caps.lttpr_caps.max_lane_count <= 4 &&
+			link->dpcd_caps.lttpr_caps.revision.raw >= 0x14);
+}
 
-	memset(lttpr_dpcd_data, '\0', sizeof(lttpr_dpcd_data));
+enum lttpr_mode dp_decide_lttpr_mode(struct dc_link *link, struct dc_link_settings *link_setting)
+{
+	enum dp_link_encoding encoding = dp_get_link_encoding_format(link_setting);
 
-	/* Logic to determine LTTPR mode*/
-	determine_lttpr_mode(link);
+	if (encoding == DP_8b_10b_ENCODING)
+		return dp_decide_8b_10b_lttpr_mode(link);
+	else if (encoding == DP_128b_132b_ENCODING)
+		return dp_decide_128b_132b_lttpr_mode(link);
 
-	if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT || link->lttpr_mode == LTTPR_MODE_TRANSPARENT) {
-		if ((link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) &&
-				!link->dc->debug.disable_fixed_vs_aux_timeout_wa) {
-			/* Fixed VS workaround for AUX timeout */
-			const uint32_t fixed_vs_address = 0xF004F;
-			const uint8_t fixed_vs_data[4] = {0x1, 0x22, 0x63, 0xc};
+	ASSERT(0);
+	return LTTPR_MODE_NON_LTTPR;
+}
 
-			core_link_write_dpcd(
-					link,
-					fixed_vs_address,
-					fixed_vs_data,
-					sizeof(fixed_vs_data));
-		}
+void dp_get_lttpr_mode_override(struct dc_link *link, enum lttpr_mode *override)
+{
+	if (!dp_is_lttpr_present(link))
+		return;
 
-		/* By reading LTTPR capability, RX assumes that we will enable
-		 * LTTPR extended aux timeout if LTTPR is present.
-		 */
-		status = core_link_read_dpcd(
-				link,
-				DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV,
-				lttpr_dpcd_data,
-				sizeof(lttpr_dpcd_data));
-
-		link->dpcd_caps.lttpr_caps.revision.raw =
-				lttpr_dpcd_data[DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV -
-								DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
-
-		link->dpcd_caps.lttpr_caps.max_link_rate =
-				lttpr_dpcd_data[DP_MAX_LINK_RATE_PHY_REPEATER -
-								DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
-
-		link->dpcd_caps.lttpr_caps.phy_repeater_cnt =
-				lttpr_dpcd_data[DP_PHY_REPEATER_CNT -
-								DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
-
-		link->dpcd_caps.lttpr_caps.max_lane_count =
-				lttpr_dpcd_data[DP_MAX_LANE_COUNT_PHY_REPEATER -
-								DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
-
-		link->dpcd_caps.lttpr_caps.mode =
-				lttpr_dpcd_data[DP_PHY_REPEATER_MODE -
-								DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
-
-		link->dpcd_caps.lttpr_caps.max_ext_timeout =
-				lttpr_dpcd_data[DP_PHY_REPEATER_EXTENDED_WAIT_TIMEOUT -
-								DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
-		link->dpcd_caps.lttpr_caps.main_link_channel_coding.raw =
-				lttpr_dpcd_data[DP_MAIN_LINK_CHANNEL_CODING_PHY_REPEATER -
-								DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
-
-		link->dpcd_caps.lttpr_caps.supported_128b_132b_rates.raw =
-				lttpr_dpcd_data[DP_PHY_REPEATER_128B132B_RATES -
-								DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
-
-		/* If this chip cap is set, at least one retimer must exist in the chain
-		 * Override count to 1 if we receive a known bad count (0 or an invalid value) */
-		if (link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN &&
-				(dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt) == 0)) {
-			ASSERT(0);
-			link->dpcd_caps.lttpr_caps.phy_repeater_cnt = 0x80;
-		}
+	if (link->dc->debug.lttpr_mode_override == LTTPR_MODE_TRANSPARENT) {
+		*override = LTTPR_MODE_TRANSPARENT;
+	} else if (link->dc->debug.lttpr_mode_override == LTTPR_MODE_NON_TRANSPARENT) {
+		*override = LTTPR_MODE_NON_TRANSPARENT;
+	} else if (link->dc->debug.lttpr_mode_override == LTTPR_MODE_NON_LTTPR) {
+		*override = LTTPR_MODE_NON_LTTPR;
+	}
+}
 
-		/* Attempt to train in LTTPR transparent mode if repeater count exceeds 8. */
-		is_lttpr_present = (link->dpcd_caps.lttpr_caps.max_lane_count > 0 &&
-				link->dpcd_caps.lttpr_caps.max_lane_count <= 4 &&
-				link->dpcd_caps.lttpr_caps.revision.raw >= 0x14);
-		if (is_lttpr_present) {
-			CONN_DATA_DETECT(link, lttpr_dpcd_data, sizeof(lttpr_dpcd_data), "LTTPR Caps: ");
-			configure_lttpr_mode_transparent(link);
-		} else
-			link->lttpr_mode = LTTPR_MODE_NON_LTTPR;
+enum lttpr_mode dp_decide_8b_10b_lttpr_mode(struct dc_link *link)
+{
+	bool is_lttpr_present = dp_is_lttpr_present(link);
+	bool vbios_lttpr_force_non_transparent = link->dc->caps.vbios_lttpr_enable;
+	bool vbios_lttpr_aware = link->dc->caps.vbios_lttpr_aware;
+
+	if (!is_lttpr_present)
+		return LTTPR_MODE_NON_LTTPR;
+
+	if (vbios_lttpr_aware) {
+		if (vbios_lttpr_force_non_transparent)
+			return LTTPR_MODE_NON_TRANSPARENT;
+		else
+			return LTTPR_MODE_TRANSPARENT;
 	}
-	return is_lttpr_present;
+
+	if (link->dc->config.allow_lttpr_non_transparent_mode.bits.DP1_4A &&
+			link->dc->caps.extended_aux_timeout_support)
+		return LTTPR_MODE_NON_TRANSPARENT;
+
+	return LTTPR_MODE_NON_LTTPR;
+}
+
+enum lttpr_mode dp_decide_128b_132b_lttpr_mode(struct dc_link *link)
+{
+	return dp_is_lttpr_present(link) ? LTTPR_MODE_NON_TRANSPARENT : LTTPR_MODE_NON_LTTPR;
 }
 
 static bool get_usbc_cable_id(struct dc_link *link, union dp_cable_id *cable_id)
@@ -5207,13 +5223,16 @@ static enum dc_status wa_try_to_wake_dprx(struct dc_link *link, uint64_t timeout
 	uint64_t current_ts = 0;
 	uint64_t time_taken_ms = 0;
 	enum dc_connection_type type = dc_connection_none;
+	bool lttpr_present;
+	bool vbios_lttpr_interop = link->dc->caps.vbios_lttpr_aware;
 
-	determine_lttpr_mode(link);
+	lttpr_present = dp_is_lttpr_present(link) ||
+			(!vbios_lttpr_interop || !link->dc->caps.extended_aux_timeout_support);
 
 	/* Issue an AUX read to test DPRX responsiveness. If LTTPR is supported the first read is expected to
 	 * be to determine LTTPR capabilities. Otherwise trying to read power state should be an innocuous AUX read.
 	 */
-	if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT || link->lttpr_mode == LTTPR_MODE_TRANSPARENT)
+	if (lttpr_present)
 		status = core_link_read_dpcd(
 				link,
 				DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV,
@@ -5281,6 +5300,7 @@ static bool retrieve_link_cap(struct dc_link *link)
 	union dp_downstream_port_present ds_port = { 0 };
 	enum dc_status status = DC_ERROR_UNEXPECTED;
 	uint32_t read_dpcd_retry_cnt = 3;
+	uint32_t aux_channel_retry_cnt = 0;
 	int i;
 	struct dp_sink_hw_fw_revision dp_hw_fw_revision;
 	const uint32_t post_oui_delay = 30; // 30ms
@@ -5308,20 +5328,46 @@ static bool retrieve_link_cap(struct dc_link *link)
 		status = wa_try_to_wake_dprx(link, timeout_ms);
 	}
 
+	while (status != DC_OK && aux_channel_retry_cnt < 10) {
+		status = core_link_read_dpcd(link, DP_SET_POWER,
+				&dpcd_power_state, sizeof(dpcd_power_state));
+
+		/* Delay 1 ms if AUX CH is in power down state. Based on spec
+		 * section 2.3.1.2, if AUX CH may be powered down due to
+		 * write to DPCD 600h = 2. Sink AUX CH is monitoring differential
+		 * signal and may need up to 1 ms before being able to reply.
+		 */
+		if (status != DC_OK || dpcd_power_state == DP_SET_POWER_D3) {
+			udelay(1000);
+			aux_channel_retry_cnt++;
+		}
+	}
+
+	/* If aux channel is not active, return false and trigger another detect*/
+	if (status != DC_OK) {
+		dpcd_power_state = DP_SET_POWER_D0;
+		status = core_link_write_dpcd(
+				link,
+				DP_SET_POWER,
+				&dpcd_power_state,
+				sizeof(dpcd_power_state));
+
+		dpcd_power_state = DP_SET_POWER_D3;
+		status = core_link_write_dpcd(
+				link,
+				DP_SET_POWER,
+				&dpcd_power_state,
+				sizeof(dpcd_power_state));
+		return false;
+	}
+
 	is_lttpr_present = dp_retrieve_lttpr_cap(link);
-	/* Read DP tunneling information. */
-	status = dpcd_get_tunneling_device_data(link);
 
-	status = core_link_read_dpcd(link, DP_SET_POWER,
-			&dpcd_power_state, sizeof(dpcd_power_state));
+	if (is_lttpr_present)
+		configure_lttpr_mode_transparent(link);
 
-	/* Delay 1 ms if AUX CH is in power down state. Based on spec
-	 * section 2.3.1.2, if AUX CH may be powered down due to
-	 * write to DPCD 600h = 2. Sink AUX CH is monitoring differential
-	 * signal and may need up to 1 ms before being able to reply.
-	 */
-	if (status != DC_OK || dpcd_power_state == DP_SET_POWER_D3)
-		udelay(1000);
+	/* Read DP tunneling information. */
+	status = dpcd_get_tunneling_device_data(link);
 
 	dpcd_set_source_specific_data(link);
 	/* Sink may need to configure internals based on vendor, so allow some
@@ -6071,7 +6117,7 @@ bool dc_link_dp_set_test_pattern(
 		/* Set DPCD Lane Settings before running test pattern */
 		if (p_link_settings != NULL) {
 			if ((link->chip_caps & EXT_DISPLAY_PATH_CAPS__DP_FIXED_VS_EN) &&
-					link->lttpr_mode == LTTPR_MODE_TRANSPARENT) {
+					p_link_settings->lttpr_mode == LTTPR_MODE_TRANSPARENT) {
 				dp_fixed_vs_pe_set_retimer_lane_settings(
 						link,
 						p_link_settings->dpcd_lane_settings,
@@ -7048,68 +7094,16 @@ void dp_enable_link_phy(
 	enum clock_source_id clock_source,
 	const struct dc_link_settings *link_settings)
 {
-	struct dc  *dc = link->ctx->dc;
-	struct dmcu *dmcu = dc->res_pool->dmcu;
-	struct pipe_ctx *pipes =
-			link->dc->current_state->res_ctx.pipe_ctx;
-	struct clock_source *dp_cs =
-			link->dc->res_pool->dp_clock_source;
-	const struct link_hwss *link_hwss = get_link_hwss(link, link_res);
-	unsigned int i;
-
-	if (link->connector_signal == SIGNAL_TYPE_EDP) {
-		if (!link->dc->config.edp_no_power_sequencing)
-			link->dc->hwss.edp_power_control(link, true);
-		link->dc->hwss.edp_wait_for_hpd_ready(link, true);
-	}
-
-	/* If the current pixel clock source is not DTO(happens after
-	 * switching from HDMI passive dongle to DP on the same connector),
-	 * switch the pixel clock source to DTO.
-	 */
-	for (i = 0; i < MAX_PIPES; i++) {
-		if (pipes[i].stream != NULL &&
-			pipes[i].stream->link == link) {
-			if (pipes[i].clock_source != NULL &&
-					pipes[i].clock_source->id != CLOCK_SOURCE_ID_DP_DTO) {
-				pipes[i].clock_source = dp_cs;
-				pipes[i].stream_res.pix_clk_params.requested_pix_clk_100hz =
-						pipes[i].stream->timing.pix_clk_100hz;
-				pipes[i].clock_source->funcs->program_pix_clk(
-							pipes[i].clock_source,
-							&pipes[i].stream_res.pix_clk_params,
-							dp_get_link_encoding_format(link_settings),
-							&pipes[i].pll_settings);
-			}
-		}
-	}
-
 	link->cur_link_settings = *link_settings;
-
-	if (dp_get_link_encoding_format(link_settings) == DP_8b_10b_ENCODING) {
-		if (dc->clk_mgr->funcs->notify_link_rate_change)
-			dc->clk_mgr->funcs->notify_link_rate_change(dc->clk_mgr, link);
-	}
-
-	if (dmcu != NULL && dmcu->funcs->lock_phy)
-		dmcu->funcs->lock_phy(dmcu);
-
-	if (link_hwss->ext.enable_dp_link_output)
-		link_hwss->ext.enable_dp_link_output(link, link_res, signal,
-				clock_source, link_settings);
-
-	if (dmcu != NULL && dmcu->funcs->unlock_phy)
-		dmcu->funcs->unlock_phy(dmcu);
-
-	dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_LINK_PHY);
+	link->dc->hwss.enable_dp_link_output(link, link_res, signal,
+			clock_source, link_settings);
 	dp_receiver_power_ctrl(link, true);
 }
 
 void edp_add_delay_for_T9(struct dc_link *link)
 {
-	if (link->local_sink &&
-			link->local_sink->edid_caps.panel_patch.extra_delay_backlight_off > 0)
-		udelay(link->local_sink->edid_caps.panel_patch.extra_delay_backlight_off * 1000);
+	if (link && link->panel_config.pps.extra_delay_backlight_off > 0)
+		udelay(link->panel_config.pps.extra_delay_backlight_off * 1000);
 }
 
 bool edp_receiver_ready_T9(struct dc_link *link)
@@ -7165,9 +7159,8 @@ bool edp_receiver_ready_T7(struct dc_link *link)
 		} while (time_taken_in_ns < 50 * 1000000); //MAx T7 is 50ms
 	}
 
-	if (link->local_sink &&
-			link->local_sink->edid_caps.panel_patch.extra_t7_ms > 0)
-		udelay(link->local_sink->edid_caps.panel_patch.extra_t7_ms * 1000);
+	if (link && link->panel_config.pps.extra_t7_ms > 0)
+		udelay(link->panel_config.pps.extra_t7_ms * 1000);
 
 	return result;
 }
@@ -7176,29 +7169,11 @@ void dp_disable_link_phy(struct dc_link *link, const struct link_resource *link_
 		enum signal_type signal)
 {
 	struct dc  *dc = link->ctx->dc;
-	struct dmcu *dmcu = dc->res_pool->dmcu;
-	const struct link_hwss *link_hwss = get_link_hwss(link, link_res);
 
 	if (!link->wa_flags.dp_keep_receiver_powered)
 		dp_receiver_power_ctrl(link, false);
 
-	if (signal == SIGNAL_TYPE_EDP) {
-		if (link->dc->hwss.edp_backlight_control)
-			link->dc->hwss.edp_backlight_control(link, false);
-		if (link_hwss->ext.disable_dp_link_output)
-			link_hwss->ext.disable_dp_link_output(link, link_res, signal);
-		link->dc->hwss.edp_power_control(link, false);
-	} else {
-		if (dmcu != NULL && dmcu->funcs->lock_phy)
-			dmcu->funcs->lock_phy(dmcu);
-		if (link_hwss->ext.disable_dp_link_output)
-			link_hwss->ext.disable_dp_link_output(link, link_res, signal);
-		if (dmcu != NULL && dmcu->funcs->unlock_phy)
-			dmcu->funcs->unlock_phy(dmcu);
-	}
-
-	dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
-
+	dc->hwss.disable_link_output(link, link_res, signal);
 	/* Clear current link setting.*/
 	memset(&link->cur_link_settings, 0,
 			sizeof(link->cur_link_settings));
@@ -7264,7 +7239,7 @@ void dp_set_hw_lane_settings(
 {
 	const struct link_hwss *link_hwss = get_link_hwss(link, link_res);
 
-	if ((link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && !is_immediate_downstream(link, offset))
+	if ((link_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) && !is_immediate_downstream(link, offset))
 		return;
 
 	if (link_hwss->ext.set_dp_lane_settings)
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c
index 468e39589ed8..74e36b34d3f7 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_dpia.c
@@ -115,12 +115,14 @@ static enum link_training_result dpia_configure_link(
 	DC_LOG_HW_LINK_TRAINING("%s\n DPIA(%d) configuring\n - LTTPR mode(%d)\n",
 				__func__,
 				link->link_id.enum_id - ENUM_ID_1,
-				link->lttpr_mode);
+				lt_settings->lttpr_mode);
 
 	dp_decide_training_settings(link,
 		link_setting,
 		lt_settings);
 
+	dp_get_lttpr_mode_override(link, &lt_settings->lttpr_mode);
+
 	status = dpcd_configure_channel_coding(link, lt_settings);
 	if (status != DC_OK && link->is_hpd_pending)
 		return LINK_TRAINING_ABORT;
@@ -178,7 +180,7 @@ static uint8_t dpia_build_set_config_data(enum dpia_set_config_type type,
 
 	switch (type) {
 	case DPIA_SET_CFG_SET_LINK:
-		data.set_link.mode = link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT ? 1 : 0;
+		data.set_link.mode = lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT ? 1 : 0;
 		break;
 	case DPIA_SET_CFG_SET_PHY_TEST_MODE:
 		break;
@@ -553,7 +555,7 @@ static enum link_training_result dpia_training_cr_phase(
 {
 	enum link_training_result result = LINK_TRAINING_CR_FAIL_LANE0;
 
-	if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT)
+	if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT)
 		result = dpia_training_cr_non_transparent(link, link_res, lt_settings, hop);
 	else
 		result = dpia_training_cr_transparent(link, link_res, lt_settings);
@@ -830,7 +832,7 @@ static enum link_training_result dpia_training_eq_phase(
 {
 	enum link_training_result result;
 
-	if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT)
+	if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT)
 		result = dpia_training_eq_non_transparent(link, link_res, lt_settings, hop);
 	else
 		result = dpia_training_eq_transparent(link, link_res, lt_settings);
@@ -870,13 +872,14 @@ static enum dc_status dpcd_clear_lt_pattern(struct dc_link *link, uint32_t hop)
  * @param hop The Hop in display path. DPRX = 0.
  */
 static enum link_training_result dpia_training_end(struct dc_link *link,
+		struct link_training_settings *lt_settings,
 		uint32_t hop)
 {
 	enum link_training_result result = LINK_TRAINING_SUCCESS;
 	uint8_t repeater_cnt = 0; /* Number of hops/repeaters in display path. */
 	enum dc_status status;
 
-	if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
+	if (lt_settings->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT) {
 		repeater_cnt = dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt);
 
 		if (hop == repeater_cnt) { /* DPTX-to-DPIA */
@@ -916,7 +919,7 @@ static enum link_training_result dpia_training_end(struct dc_link *link,
 				link->link_id.enum_id - ENUM_ID_1,
 				hop,
 				result,
-				link->lttpr_mode);
+				lt_settings->lttpr_mode);
 
 	return result;
 }
@@ -928,7 +931,9 @@ static enum link_training_result dpia_training_end(struct dc_link *link,
  * @param link DPIA link being trained.
  * @param hop The Hop in display path. DPRX = 0.
  */
-static void dpia_training_abort(struct dc_link *link, uint32_t hop)
+static void dpia_training_abort(struct dc_link *link,
+	struct link_training_settings *lt_settings,
+	uint32_t hop)
 {
 	uint8_t data = 0;
 	uint32_t dpcd_tps_offset = DP_TRAINING_PATTERN_SET;
@@ -936,7 +941,7 @@ static void dpia_training_abort(struct dc_link *link, uint32_t hop)
 	DC_LOG_HW_LINK_TRAINING("%s\n DPIA(%d) aborting\n - LTTPR mode(%d)\n - HPD(%d)\n",
 				__func__,
 				link->link_id.enum_id - ENUM_ID_1,
-				link->lttpr_mode,
+				lt_settings->lttpr_mode,
 				link->is_hpd_pending);
 
 	/* Abandon clean-up if sink unplugged. */
@@ -964,12 +969,16 @@ enum link_training_result dc_link_dpia_perform_link_training(
 	uint8_t repeater_cnt = 0; /* Number of hops/repeaters in display path. */
 	int8_t repeater_id; /* Current hop. */
 
+	struct dc_link_settings link_settings = *link_setting; // non-const copy to pass in
+
+	lt_settings.lttpr_mode = dp_decide_lttpr_mode(link, &link_settings);
+
 	/* Configure link as prescribed in link_setting and set LTTPR mode. */
 	result = dpia_configure_link(link, link_res, link_setting, &lt_settings);
 	if (result != LINK_TRAINING_SUCCESS)
 		return result;
 
-	if (link->lttpr_mode == LTTPR_MODE_NON_TRANSPARENT)
+	if (lt_settings.lttpr_mode == LTTPR_MODE_NON_TRANSPARENT)
 		repeater_cnt = dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt);
 
 	/* Train each hop in turn starting with the one closest to DPTX.
@@ -987,7 +996,7 @@ enum link_training_result dc_link_dpia_perform_link_training(
 			break;
 
 		/* Stop training hop. */
-		result = dpia_training_end(link, repeater_id);
+		result = dpia_training_end(link, &lt_settings, repeater_id);
 		if (result != LINK_TRAINING_SUCCESS)
 			break;
 	}
@@ -1001,9 +1010,9 @@ enum link_training_result dc_link_dpia_perform_link_training(
 		msleep(5);
 		result = dp_check_link_loss_status(link, &lt_settings);
 	} else if (result == LINK_TRAINING_ABORT) {
-		dpia_training_abort(link, repeater_id);
+		dpia_training_abort(link, &lt_settings, repeater_id);
 	} else {
-		dpia_training_end(link, repeater_id);
+		dpia_training_end(link, &lt_settings, repeater_id);
 	}
 	return result;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index ccf7bd3d90fe..8ee0d946bb2f 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -1904,9 +1904,6 @@ bool dc_is_stream_unchanged(
 	if (memcmp(&old_stream->audio_info, &stream->audio_info, sizeof(stream->audio_info)) != 0)
 		return false;
 
-	if (old_stream->odm_2to1_policy_applied != stream->odm_2to1_policy_applied)
-		return false;
-
 	return true;
 }
 
@@ -3665,3 +3662,25 @@ const struct link_hwss *get_link_hwss(const struct dc_link *link,
 	else
 		return get_virtual_link_hwss();
 }
+
+bool is_h_timing_divisible_by_2(struct dc_stream_state *stream)
+{
+	bool divisible = false;
+	uint16_t h_blank_start = 0;
+	uint16_t h_blank_end = 0;
+
+	if (stream) {
+		h_blank_start = stream->timing.h_total - stream->timing.h_front_porch;
+		h_blank_end = h_blank_start - stream->timing.h_addressable;
+
+		/* HTOTAL, Hblank start/end, and Hsync start/end all must be
+		 * divisible by 2 in order for the horizontal timing params
+		 * to be considered divisible by 2. Hsync start is always 0.
+		 */
+		divisible = (stream->timing.h_total % 2 == 0) &&
+				(h_blank_start % 2 == 0) &&
+				(h_blank_end % 2 == 0) &&
+				(stream->timing.h_sync_width % 2 == 0);
+	}
+	return divisible;
+}
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index 0c85ab5933b4..ae13887756bf 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -30,6 +30,7 @@
 #include "resource.h"
 #include "ipp.h"
 #include "timing_generator.h"
+#include "dc_dmub_srv.h"
 
 #define DC_LOGGER dc->ctx->logger
 
@@ -519,7 +520,7 @@ bool dc_stream_remove_writeback(struct dc *dc,
 	}
 
 	/* remove writeback info for disabled writeback pipes from stream */
-	for (i = 0, j = 0; i < stream->num_wb_info; i++) {
+	for (i = 0, j = 0; i < stream->num_wb_info && j < MAX_DWB_PIPES; i++) {
 		if (stream->writeback_info[i].wb_enabled) {
 			if (i != j)
 				/* trim the array */
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index dbf8158b832e..2ecf36e6329b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -47,7 +47,7 @@ struct aux_payload;
 struct set_config_cmd_payload;
 struct dmub_notification;
 
-#define DC_VER "3.2.198"
+#define DC_VER "3.2.205"
 
 #define MAX_SURFACES 3
 #define MAX_PLANES 6
@@ -118,7 +118,26 @@ struct dc_plane_cap {
 	uint32_t min_height;
 };
 
-// Color management caps (DPP and MPC)
+/**
+ * DOC: color-management-caps
+ *
+ * **Color management caps (DPP and MPC)**
+ *
+ * Modules/color calculates various color operations which are translated to
+ * abstracted HW. DCE 5-12 had almost no important changes, but starting with
+ * DCN1, every new generation comes with fairly major differences in color
+ * pipeline. Therefore, we abstract color pipe capabilities so modules/DM can
+ * decide mapping to HW block based on logical capabilities.
+ */
+
+/**
+ * struct rom_curve_caps - predefined transfer function caps for degamma and regamma
+ * @srgb: RGB color space transfer func
+ * @bt2020: BT.2020 transfer func
+ * @gamma2_2: standard gamma
+ * @pq: perceptual quantizer transfer function
+ * @hlg: hybrid log–gamma transfer function
+ */
 struct rom_curve_caps {
 	uint16_t srgb : 1;
 	uint16_t bt2020 : 1;
@@ -127,36 +146,68 @@ struct rom_curve_caps {
 	uint16_t hlg : 1;
 };
 
+/**
+ * struct dpp_color_caps - color pipeline capabilities for display pipe and
+ * plane blocks
+ *
+ * @dcn_arch: all DCE generations treated the same
+ * @input_lut_shared: shared with DGAM. Input LUT is different than most LUTs,
+ * just plain 256-entry lookup
+ * @icsc: input color space conversion
+ * @dgam_ram: programmable degamma LUT
+ * @post_csc: post color space conversion, before gamut remap
+ * @gamma_corr: degamma correction
+ * @hw_3d_lut: 3D LUT support. It implies a shaper LUT before. It may be shared
+ * with MPC by setting mpc:shared_3d_lut flag
+ * @ogam_ram: programmable out/blend gamma LUT
+ * @ocsc: output color space conversion
+ * @dgam_rom_for_yuv: pre-defined degamma LUT for YUV planes
+ * @dgam_rom_caps: pre-definied curve caps for degamma 1D LUT
+ * @ogam_rom_caps: pre-definied curve caps for regamma 1D LUT
+ *
+ * Note: hdr_mult and gamut remap (CTM) are always available in DPP (in that order)
+ */
 struct dpp_color_caps {
-	uint16_t dcn_arch : 1; // all DCE generations treated the same
-	// input lut is different than most LUTs, just plain 256-entry lookup
-	uint16_t input_lut_shared : 1; // shared with DGAM
+	uint16_t dcn_arch : 1;
+	uint16_t input_lut_shared : 1;
 	uint16_t icsc : 1;
 	uint16_t dgam_ram : 1;
-	uint16_t post_csc : 1; // before gamut remap
+	uint16_t post_csc : 1;
 	uint16_t gamma_corr : 1;
-
-	// hdr_mult and gamut remap always available in DPP (in that order)
-	// 3d lut implies shaper LUT,
-	// it may be shared with MPC - check MPC:shared_3d_lut flag
 	uint16_t hw_3d_lut : 1;
-	uint16_t ogam_ram : 1; // blnd gam
+	uint16_t ogam_ram : 1;
 	uint16_t ocsc : 1;
 	uint16_t dgam_rom_for_yuv : 1;
 	struct rom_curve_caps dgam_rom_caps;
 	struct rom_curve_caps ogam_rom_caps;
 };
 
+/**
+ * struct mpc_color_caps - color pipeline capabilities for multiple pipe and
+ * plane combined blocks
+ *
+ * @gamut_remap: color transformation matrix
+ * @ogam_ram: programmable out gamma LUT
+ * @ocsc: output color space conversion matrix
+ * @num_3dluts: MPC 3D LUT; always assumes a preceding shaper LUT
+ * @shared_3d_lut: shared 3D LUT flag. Can be either DPP or MPC, but single
+ * instance
+ * @ogam_rom_caps: pre-definied curve caps for regamma 1D LUT
+ */
 struct mpc_color_caps {
 	uint16_t gamut_remap : 1;
 	uint16_t ogam_ram : 1;
 	uint16_t ocsc : 1;
-	uint16_t num_3dluts : 3; //3d lut always assumes a preceding shaper LUT
-	uint16_t shared_3d_lut:1; //can be in either DPP or MPC, but single instance
-
+	uint16_t num_3dluts : 3;
+	uint16_t shared_3d_lut:1;
 	struct rom_curve_caps ogam_rom_caps;
 };
 
+/**
+ * struct dc_color_caps - color pipes capabilities for DPP and MPC hw blocks
+ * @dpp: color pipes caps for DPP
+ * @mpc: color pipes caps for MPC
+ */
 struct dc_color_caps {
 	struct dpp_color_caps dpp;
 	struct mpc_color_caps mpc;
@@ -350,10 +401,14 @@ struct dc_config {
 	uint8_t  vblank_alignment_max_frame_time_diff;
 	bool is_asymmetric_memory;
 	bool is_single_rank_dimm;
+	bool is_vmin_only_asic;
 	bool use_pipe_ctx_sync_logic;
 	bool ignore_dpref_ss;
 	bool enable_mipi_converter_optimization;
 	bool use_default_clock_table;
+	bool force_bios_enable_lttpr;
+	uint8_t force_bios_fixed_vs;
+
 };
 
 enum visual_confirm {
@@ -365,6 +420,7 @@ enum visual_confirm {
 	VISUAL_CONFIRM_SWAPCHAIN = 6,
 	VISUAL_CONFIRM_FAMS = 7,
 	VISUAL_CONFIRM_SWIZZLE = 9,
+	VISUAL_CONFIRM_SUBVP = 14,
 };
 
 enum dc_psr_power_opts {
@@ -386,9 +442,31 @@ enum dcc_option {
 	DCC_HALF_REQ_DISALBE = 2,
 };
 
+/**
+ * enum pipe_split_policy - Pipe split strategy supported by DCN
+ *
+ * This enum is used to define the pipe split policy supported by DCN. By
+ * default, DC favors MPC_SPLIT_DYNAMIC.
+ */
 enum pipe_split_policy {
+	/**
+	 * @MPC_SPLIT_DYNAMIC: DC will automatically decide how to split the
+	 * pipe in order to bring the best trade-off between performance and
+	 * power consumption. This is the recommended option.
+	 */
 	MPC_SPLIT_DYNAMIC = 0,
+
+	/**
+	 * @MPC_SPLIT_DYNAMIC: Avoid pipe split, which means that DC will not
+	 * try any sort of split optimization.
+	 */
 	MPC_SPLIT_AVOID = 1,
+
+	/**
+	 * @MPC_SPLIT_DYNAMIC: With this option, DC will only try to optimize
+	 * the pipe utilization when using a single display; if the user
+	 * connects to a second display, DC will avoid pipe split.
+	 */
 	MPC_SPLIT_AVOID_MULT_DISP = 2,
 };
 
@@ -623,6 +701,14 @@ struct dc_state;
 struct resource_pool;
 struct dce_hwseq;
 
+/**
+ * struct dc_debug_options - DC debug struct
+ *
+ * This struct provides a simple mechanism for developers to change some
+ * configurations, enable/disable features, and activate extra debug options.
+ * This can be very handy to narrow down whether some specific feature is
+ * causing an issue or not.
+ */
 struct dc_debug_options {
 	bool native422_support;
 	bool disable_dsc;
@@ -642,6 +728,11 @@ struct dc_debug_options {
 	bool disable_stutter;
 	bool use_max_lb;
 	enum dcc_option disable_dcc;
+
+	/**
+	 * @pipe_split_policy: Define which pipe split policy is used by the
+	 * display core.
+	 */
 	enum pipe_split_policy pipe_split_policy;
 	bool force_single_disp_pipe_split;
 	bool voltage_align_fclk;
@@ -715,8 +806,6 @@ struct dc_debug_options {
 	bool validate_dml_output;
 	bool enable_dmcub_surface_flip;
 	bool usbc_combo_phy_reset_wa;
-	bool disable_dsc_edp;
-	unsigned int  force_dsc_edp_policy;
 	bool enable_dram_clock_change_one_display_vactive;
 	/* TODO - remove once tested */
 	bool legacy_dp2_lt;
@@ -740,12 +829,14 @@ struct dc_debug_options {
 	int crb_alloc_policy_min_disp_count;
 	bool disable_z10;
 	bool enable_z9_disable_interface;
-	bool enable_sw_cntl_psr;
 	union dpia_debug_options dpia_debug;
 	bool disable_fixed_vs_aux_timeout_wa;
 	bool force_disable_subvp;
 	bool force_subvp_mclk_switch;
 	bool allow_sw_cursor_fallback;
+	unsigned int force_subvp_num_ways;
+	unsigned int force_mall_ss_num_ways;
+	bool alloc_extra_way_for_cursor;
 	bool force_usr_allow;
 	/* uses value at boot and disables switch */
 	bool disable_dtb_ref_clk_switch;
@@ -759,7 +850,9 @@ struct dc_debug_options {
 	bool use_legacy_soc_bb_mechanism;
 	bool exit_idle_opt_for_cursor_updates;
 	bool enable_single_display_2to1_odm_policy;
+	bool enable_double_buffered_dsc_pg_support;
 	bool enable_dp_dig_pixel_rate_div_policy;
+	enum lttpr_mode lttpr_mode_override;
 };
 
 struct gpu_info_soc_bounding_box_v1_0;
@@ -815,6 +908,17 @@ struct dc {
 
 	uint32_t *dcn_reg_offsets;
 	uint32_t *nbio_reg_offsets;
+
+	/* Scratch memory */
+	struct {
+		struct {
+			/*
+			 * For matching clock_limits table in driver with table
+			 * from PMFW.
+			 */
+			struct _vcs_dpi_voltage_scaling_st clock_limits[DC__VOLTAGE_STATES];
+		} update_bw_bounding_box;
+	} scratch;
 };
 
 enum frame_buffer_mode {
@@ -1018,6 +1122,7 @@ union surface_update_flags {
 		uint32_t clock_change:1;
 		uint32_t stereo_format_change:1;
 		uint32_t lut_3d:1;
+		uint32_t tmz_changed:1;
 		uint32_t full_update:1;
 	} bits;
 
@@ -1086,6 +1191,7 @@ struct dc_plane_state {
 	/* private to dc_surface.c */
 	enum dc_irq_source irq_source;
 	struct kref refcount;
+	struct tg_color visual_confirm_color;
 };
 
 struct dc_plane_info {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
index 52a61b3e5a8b..89d7d3fd3321 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
@@ -323,11 +323,13 @@ bool dc_dmub_srv_p_state_delegate(struct dc *dc, bool should_manage_pstate, stru
 	struct dmub_cmd_fw_assisted_mclk_switch_config *config_data = &cmd.fw_assisted_mclk_switch.config_data;
 	int i = 0;
 	int ramp_up_num_steps = 1; // TODO: Ramp is currently disabled. Reenable it.
-	uint8_t visual_confirm_enabled = dc->debug.visual_confirm == VISUAL_CONFIRM_FAMS;
+	uint8_t visual_confirm_enabled;
 
 	if (dc == NULL)
 		return false;
 
+	visual_confirm_enabled = dc->debug.visual_confirm == VISUAL_CONFIRM_FAMS;
+
 	// Format command.
 	cmd.fw_assisted_mclk_switch.header.type = DMUB_CMD__FW_ASSISTED_MCLK_SWITCH;
 	cmd.fw_assisted_mclk_switch.header.sub_type = DMUB_CMD__FAMS_SETUP_FW_CTRL;
@@ -387,6 +389,37 @@ void dc_dmub_srv_query_caps_cmd(struct dmub_srv *dmub)
 	}
 }
 
+void dc_dmub_srv_get_visual_confirm_color_cmd(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+	union dmub_rb_cmd cmd = { 0 };
+	enum dmub_status status;
+	unsigned int panel_inst = 0;
+
+	dc_get_edp_link_panel_inst(dc, pipe_ctx->stream->link, &panel_inst);
+
+	memset(&cmd, 0, sizeof(cmd));
+
+	// Prepare fw command
+	cmd.visual_confirm_color.header.type = DMUB_CMD__GET_VISUAL_CONFIRM_COLOR;
+	cmd.visual_confirm_color.header.sub_type = 0;
+	cmd.visual_confirm_color.header.ret_status = 1;
+	cmd.visual_confirm_color.header.payload_bytes = sizeof(struct dmub_cmd_visual_confirm_color_data);
+	cmd.visual_confirm_color.visual_confirm_color_data.visual_confirm_color.panel_inst = panel_inst;
+
+	// Send command to fw
+	status = dmub_srv_cmd_with_reply_data(dc->ctx->dmub_srv->dmub, &cmd);
+
+	ASSERT(status == DMUB_STATUS_OK);
+
+	// If command was processed, copy feature caps to dmub srv
+	if (status == DMUB_STATUS_OK &&
+		cmd.visual_confirm_color.header.ret_status == 0) {
+		memcpy(&dc->ctx->dmub_srv->dmub->visual_confirm_color,
+			&cmd.visual_confirm_color.visual_confirm_color_data,
+			sizeof(struct dmub_visual_confirm_color));
+	}
+}
+
 #ifdef CONFIG_DRM_AMD_DC_DCN
 /**
  * ***********************************************************************************************
@@ -602,7 +635,7 @@ static void populate_subvp_cmd_pipe_info(struct dc *dc,
 			&cmd->fw_assisted_mclk_switch_v2.config_data.pipe_data[cmd_pipe_index];
 	struct dc_crtc_timing *main_timing = &subvp_pipe->stream->timing;
 	struct dc_crtc_timing *phantom_timing = &subvp_pipe->stream->mall_stream_config.paired_stream->timing;
-	uint32_t out_num, out_den;
+	uint32_t out_num_stream, out_den_stream, out_num_plane, out_den_plane, out_num, out_den;
 
 	pipe_data->mode = SUBVP;
 	pipe_data->pipe_config.subvp_data.pix_clk_100hz = subvp_pipe->stream->timing.pix_clk_100hz;
@@ -619,11 +652,16 @@ static void populate_subvp_cmd_pipe_info(struct dc *dc,
 	/* Calculate the scaling factor from the src and dst height.
 	 * e.g. If 3840x2160 being downscaled to 1920x1080, the scaling factor is 1/2.
 	 * Reduce the fraction 1080/2160 = 1/2 for the "scaling factor"
+	 *
+	 * Make sure to combine stream and plane scaling together.
 	 */
-	reduce_fraction(subvp_pipe->stream->src.height, subvp_pipe->stream->dst.height, &out_num, &out_den);
-	// TODO: Uncomment below lines once DMCUB include headers are promoted
-	//pipe_data->pipe_config.subvp_data.scale_factor_numerator = out_num;
-	//pipe_data->pipe_config.subvp_data.scale_factor_denominator = out_den;
+	reduce_fraction(subvp_pipe->stream->src.height, subvp_pipe->stream->dst.height,
+			&out_num_stream, &out_den_stream);
+	reduce_fraction(subvp_pipe->plane_state->src_rect.height, subvp_pipe->plane_state->dst_rect.height,
+			&out_num_plane, &out_den_plane);
+	reduce_fraction(out_num_stream * out_num_plane, out_den_stream * out_den_plane, &out_num, &out_den);
+	pipe_data->pipe_config.subvp_data.scale_factor_numerator = out_num;
+	pipe_data->pipe_config.subvp_data.scale_factor_denominator = out_den;
 
 	// Prefetch lines is equal to VACTIVE + BP + VSYNC
 	pipe_data->pipe_config.subvp_data.prefetch_lines =
@@ -636,12 +674,28 @@ static void populate_subvp_cmd_pipe_info(struct dc *dc,
 	pipe_data->pipe_config.subvp_data.processing_delay_lines =
 			div64_u64(((uint64_t)(dc->caps.subvp_fw_processing_delay_us) * ((uint64_t)phantom_timing->pix_clk_100hz * 100) +
 					((uint64_t)phantom_timing->h_total * 1000000 - 1)), ((uint64_t)phantom_timing->h_total * 1000000));
+
+	if (subvp_pipe->bottom_pipe) {
+		pipe_data->pipe_config.subvp_data.main_split_pipe_index = subvp_pipe->bottom_pipe->pipe_idx;
+	} else if (subvp_pipe->next_odm_pipe) {
+		pipe_data->pipe_config.subvp_data.main_split_pipe_index = subvp_pipe->next_odm_pipe->pipe_idx;
+	} else {
+		pipe_data->pipe_config.subvp_data.main_split_pipe_index = 0;
+	}
+
 	// Find phantom pipe index based on phantom stream
 	for (j = 0; j < dc->res_pool->pipe_count; j++) {
 		struct pipe_ctx *phantom_pipe = &context->res_ctx.pipe_ctx[j];
 
 		if (phantom_pipe->stream == subvp_pipe->stream->mall_stream_config.paired_stream) {
 			pipe_data->pipe_config.subvp_data.phantom_pipe_index = phantom_pipe->pipe_idx;
+			if (phantom_pipe->bottom_pipe) {
+				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = phantom_pipe->bottom_pipe->pipe_idx;
+			} else if (phantom_pipe->next_odm_pipe) {
+				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = phantom_pipe->next_odm_pipe->pipe_idx;
+			} else {
+				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = 0;
+			}
 			break;
 		}
 	}
@@ -686,7 +740,9 @@ void dc_dmub_setup_subvp_dmub_command(struct dc *dc,
 		if (!pipe->stream)
 			continue;
 
-		if (pipe->plane_state && !pipe->top_pipe &&
+		/* For SubVP pipe count, only count the top most (ODM / MPC) pipe
+		 */
+		if (pipe->plane_state && !pipe->top_pipe && !pipe->prev_odm_pipe &&
 				pipe->stream->mall_stream_config.type == SUBVP_MAIN)
 			subvp_pipes[subvp_count++] = pipe;
 	}
@@ -699,7 +755,12 @@ void dc_dmub_setup_subvp_dmub_command(struct dc *dc,
 			if (!pipe->stream)
 				continue;
 
+			/* When populating subvp cmd info, only pass in the top most (ODM / MPC) pipe.
+			 * Any ODM or MPC splits being used in SubVP will be handled internally in
+			 * populate_subvp_cmd_pipe_info
+			 */
 			if (pipe->plane_state && pipe->stream->mall_stream_config.paired_stream &&
+					!pipe->top_pipe && !pipe->prev_odm_pipe &&
 					pipe->stream->mall_stream_config.type == SUBVP_MAIN) {
 				populate_subvp_cmd_pipe_info(dc, context, &cmd, pipe, cmd_pipe_index++);
 			} else if (pipe->plane_state && pipe->stream->mall_stream_config.type == SUBVP_NONE) {
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
index 159782cd6659..7e438345b1a8 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
@@ -78,12 +78,14 @@ void dc_dmub_srv_set_drr_manual_trigger_cmd(struct dc *dc, uint32_t tg_inst);
 bool dc_dmub_srv_p_state_delegate(struct dc *dc, bool enable_pstate, struct dc_state *context);
 
 void dc_dmub_srv_query_caps_cmd(struct dmub_srv *dmub);
+void dc_dmub_srv_get_visual_confirm_color_cmd(struct dc *dc, struct pipe_ctx *pipe_ctx);
 void dc_dmub_srv_clear_inbox0_ack(struct dc_dmub_srv *dmub_srv);
 void dc_dmub_srv_wait_for_inbox0_ack(struct dc_dmub_srv *dmub_srv);
 void dc_dmub_srv_send_inbox0_cmd(struct dc_dmub_srv *dmub_srv, union dmub_inbox0_data_register data);
 
 bool dc_dmub_srv_get_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv, struct dmub_diagnostic_data *dmub_oca);
 
+void dc_dmub_setup_subvp_dmub_command(struct dc *dc, struct dc_state *context, bool enable);
 void dc_dmub_srv_log_diagnostic_data(struct dc_dmub_srv *dc_dmub_srv);
 
 #endif /* _DMUB_DC_SRV_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index 584aaf6967fd..848db8676adf 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -417,19 +417,43 @@ enum dc_scan_direction {
 	SCAN_DIRECTION_VERTICAL = 2,    /* 90, 270 rotation */
 };
 
+/**
+ * struct dc_cursor_position: Hardware cursor data.
+ *
+ * This struct keeps the action information related to the cursor that will be
+ * sent and received from our DC core.
+ */
 struct dc_cursor_position {
+	/**
+	 * @x: It represents the top left abscissa coordinate of the cursor.
+	 */
 	uint32_t x;
+
+	/**
+	 * @y: It is the top ordinate of the cursor coordinate.
+	 */
 	uint32_t y;
 
+	/**
+	 * @x_hotspot: Define the abscissa point where mouse click happens.
+	 */
 	uint32_t x_hotspot;
+
+	/**
+	 * @y_hotspot: Define the ordinate point where mouse click happens.
+	 */
 	uint32_t y_hotspot;
 
-	/*
-	 * This parameter indicates whether HW cursor should be enabled
+	/**
+	 * @enable: This parameter indicates whether hardware cursor should be
+	 * enabled.
 	 */
 	bool enable;
 
-	/* Translate cursor x/y by the source rectangle for each plane. */
+	/**
+	 * @translate_by_source: Translate cursor x/y by the source rectangle
+	 * for each plane.
+	 */
 	bool translate_by_source;
 };
 
@@ -494,7 +518,9 @@ struct dc_gamma {
 /* Used by both ipp amd opp functions*/
 /* TODO: to be consolidated with enum color_space */
 
-/*
+/**
+ * enum dc_cursor_color_format - DC cursor programming mode
+ *
  * This enum is for programming CURSOR_MODE register field. What this register
  * should be programmed to depends on OS requested cursor shape flags and what
  * we stored in the cursor surface.
@@ -530,17 +556,39 @@ union dc_cursor_attribute_flags {
 };
 
 struct dc_cursor_attributes {
+	/**
+	 * @address: This field represents the framebuffer address associated
+	 * with the cursor. It is important to highlight that this address is
+	 * divided into a high and low parts.
+	 */
 	PHYSICAL_ADDRESS_LOC address;
+
+	/**
+	 * @pitch: Cursor line stride.
+	 */
 	uint32_t pitch;
 
-	/* Width and height should correspond to cursor surface width x heigh */
+	/**
+	 * @width: Width should correspond to cursor surface width.
+	 */
 	uint32_t width;
+	/**
+	 * @heigh: Height should correspond to cursor surface heigh.
+	 */
 	uint32_t height;
 
+	/**
+	 * @color_format: DC cursor programming mode.
+	 */
 	enum dc_cursor_color_format color_format;
-	uint32_t sdr_white_level; // for boosting (SDR) cursor in HDR mode
+	/**
+	 * @sdr_white_level: Boosting (SDR) cursor in HDR mode.
+	 */
+	uint32_t sdr_white_level;
 
-	/* In case we support HW Cursor rotation in the future */
+	/**
+	 * @rotation_angle: In case we support HW Cursor rotation in the future
+	 */
 	enum dc_rotation_angle rotation_angle;
 
 	union dc_cursor_attribute_flags attribute_flags;
@@ -764,22 +812,108 @@ struct dc_dsc_config {
 	bool is_dp; /* indicate if DSC is applied based on DP's capability */
 	uint32_t mst_pbn; /* pbn of display on dsc mst hub */
 };
+
+/**
+ * struct dc_crtc_timing - Timing parameters used to configure DCN blocks
+ *
+ * DCN provides multiple signals and parameters that can be used to adjust
+ * timing parameters, this struct aggregate multiple of these values for easy
+ * access. In this struct, fields prefixed with h_* are related to horizontal
+ * timing, and v_* to vertical timing. Keep in mind that when we talk about
+ * vertical timings, the values, in general, are described in the number of
+ * lines; on the other hand, the horizontal values are in pixels.
+ */
 struct dc_crtc_timing {
+	/**
+	 * @h_total: The total number of pixels from the rising edge of HSync
+	 * until the rising edge of the current HSync.
+	 */
 	uint32_t h_total;
+
+	/**
+	 * @h_border_left: The black pixels related to the left border
+	 */
 	uint32_t h_border_left;
+
+	/**
+	 * @h_addressable: It is the range of pixels displayed horizontally.
+	 * For example, if the display resolution is 3840@2160, the horizontal
+	 * addressable area is 3840.
+	 */
 	uint32_t h_addressable;
+
+	/**
+	 * @h_border_right: The black pixels related to the right border
+	 */
 	uint32_t h_border_right;
+
+	/**
+	 * @h_front_porch: Period (in pixels) between HBlank start and the
+	 * rising edge of HSync.
+	 */
 	uint32_t h_front_porch;
+
+	/**
+	 * @h_sync_width: HSync duration in pixels.
+	 */
 	uint32_t h_sync_width;
 
+	/**
+	 * @v_total: It is the total number of lines from the rising edge of
+	 * the previous VSync until the rising edge of the current VSync.
+	 *
+	 *          |--------------------------|
+	 *          +-+        V_TOTAL         +-+
+	 *          | |                        | |
+	 * VSync ---+ +--------- // -----------+ +---
+	 */
 	uint32_t v_total;
+
+	/**
+	 * @v_border_top: The black border on the top.
+	 */
 	uint32_t v_border_top;
+
+	/**
+	 * @v_addressable: It is the range of the scanout at which the
+	 * framebuffer is displayed. For example, if the display resolution is
+	 * 3840@2160, the addressable area is 2160 lines, or if the resolution
+	 * is 1920x1080, the addressable area is 1080 lines.
+	 */
 	uint32_t v_addressable;
+
+	/**
+	 * @v_border_bottom: The black border on the bottom.
+	 */
 	uint32_t v_border_bottom;
+
+	/**
+	 * @v_front_porch: Period (in lines) between VBlank start and rising
+	 * edge of VSync.
+	 *                  +-+
+	 * VSync            | |
+	 *        ----------+ +--------...
+	 *          +------------------...
+	 * VBlank   |
+	 *        --+
+	 *          |-------|
+	 *        v_front_porch
+	 */
 	uint32_t v_front_porch;
+
+	/**
+	 * @v_sync_width: VSync signal width in lines.
+	 */
 	uint32_t v_sync_width;
 
+	/**
+	 * @pix_clk_100hz: Pipe pixel precision
+	 *
+	 * This field is used to communicate pixel clocks with 100 Hz accuracy
+	 * from dc_crtc_timing to BIOS command table.
+	 */
 	uint32_t pix_clk_100hz;
+
 	uint32_t min_refresh_in_uhz;
 
 	uint32_t vic;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_link.h b/drivers/gpu/drm/amd/display/dc/dc_link.h
index 9544abf75e84..bf5f9e2773bc 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_link.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_link.h
@@ -113,6 +113,32 @@ struct psr_settings {
 	unsigned int psr_power_opt;
 };
 
+/* To split out "global" and "per-panel" config settings.
+ * Add a struct dc_panel_config under dc_link
+ */
+struct dc_panel_config {
+	// extra panel power sequence parameters
+	struct pps {
+		unsigned int extra_t3_ms;
+		unsigned int extra_t7_ms;
+		unsigned int extra_delay_backlight_off;
+		unsigned int extra_post_t7_ms;
+		unsigned int extra_pre_t11_ms;
+		unsigned int extra_t12_ms;
+		unsigned int extra_post_OUI_ms;
+	} pps;
+	// ABM
+	struct varib {
+		unsigned int varibright_feature_enable;
+		unsigned int def_varibright_level;
+		unsigned int abm_config_setting;
+	} varib;
+	// edp DSC
+	struct dsc {
+		bool disable_dsc_edp;
+		unsigned int force_dsc_edp_policy;
+	} dsc;
+};
 /*
  * A link contains one or more sinks and their connected status.
  * The currently active signal type (HDMI, DP-SST, DP-MST) is also reported.
@@ -131,7 +157,6 @@ struct dc_link {
 	bool link_state_valid;
 	bool aux_access_disabled;
 	bool sync_lt_in_progress;
-	enum lttpr_mode lttpr_mode;
 	bool is_internal_display;
 
 	/* TODO: Rename. Flag an endpoint as having a programmable mapping to a
@@ -224,6 +249,7 @@ struct dc_link {
 		bool dpia_mst_dsc_always_on;
 		/* Forced DPIA into TBT3 compatibility mode. */
 		bool dpia_forced_tbt3_mode;
+		bool dongle_mode_timing_override;
 	} wa_flags;
 	struct link_mst_stream_allocation_table mst_stream_alloc_table;
 
@@ -232,6 +258,8 @@ struct dc_link {
 
 	struct gpio *hpd_gpio;
 	enum dc_link_fec_state fec_state;
+	struct dc_panel_config panel_config;
+	struct phy_state phy_state;
 };
 
 const struct dc_link_status *dc_link_get_status(const struct dc_link *dc_link);
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index f87f852d4829..9e6025c98db9 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -212,8 +212,7 @@ struct dc_stream_state {
 	/* DMCU info */
 	unsigned int abm_level;
 
-	struct periodic_interrupt_config periodic_interrupt0;
-	struct periodic_interrupt_config periodic_interrupt1;
+	struct periodic_interrupt_config periodic_interrupt;
 
 	/* from core_stream struct */
 	struct dc_context *ctx;
@@ -268,8 +267,6 @@ struct dc_stream_state {
 	bool has_non_synchronizable_pclk;
 	bool vblank_synchronized;
 	struct mall_stream_config mall_stream_config;
-
-	bool odm_2to1_policy_applied;
 };
 
 #define ABM_LEVEL_IMMEDIATE_DISABLE 255
@@ -283,8 +280,7 @@ struct dc_stream_update {
 	struct dc_info_packet *hdr_static_metadata;
 	unsigned int *abm_level;
 
-	struct periodic_interrupt_config *periodic_interrupt0;
-	struct periodic_interrupt_config *periodic_interrupt1;
+	struct periodic_interrupt_config *periodic_interrupt;
 
 	struct dc_info_packet *vrr_infopacket;
 	struct dc_info_packet *vsc_infopacket;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
index bdb6bac8dd97..c94a966c6612 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_audio.c
@@ -300,7 +300,7 @@ static void set_high_bit_rate_capable(
 	AZ_REG_WRITE(AZALIA_F0_CODEC_PIN_CONTROL_RESPONSE_HBR, value);
 }
 
-/* set video latency in in ms/2+1 */
+/* set video latency in ms/2+1 */
 static void set_video_latency(
 	struct audio *audio,
 	int latency_in_ms)
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
index 919c2c2ba84b..32782ef9ef77 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
@@ -814,12 +814,6 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
 								"dce_aux_transfer_with_retries: AUX_RET_SUCCESS: AUX_TRANSACTION_REPLY_I2C_OVER_AUX_DEFER");
 
 				retry_on_defer = true;
-				fallthrough;
-			case AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK:
-				if (*payload->reply == AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK)
-					DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION,
-								LOG_FLAG_I2cAux_DceAux,
-								"dce_aux_transfer_with_retries: AUX_RET_SUCCESS: AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK");
 
 				if (aux_defer_retries >= AUX_MIN_DEFER_RETRIES
 						&& defer_time_in_ms >= AUX_MAX_DEFER_TIMEOUT_MS) {
@@ -848,7 +842,11 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
 					}
 				}
 				break;
-
+			case AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK:
+				DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION,
+							LOG_FLAG_I2cAux_DceAux,
+							"dce_aux_transfer_with_retries: FAILURE: AUX_TRANSACTION_REPLY_I2C_OVER_AUX_NACK");
+				goto fail;
 			case AUX_TRANSACTION_REPLY_I2C_DEFER:
 				DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION,
 							LOG_FLAG_I2cAux_DceAux,
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
index 0df06740ec39..bec5e9f787fc 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
@@ -393,17 +393,18 @@ static bool dmub_psr_copy_settings(struct dmub_psr *dmub,
 	if (copy_settings_data->dsc_enable_status &&
 		link->dpcd_caps.sink_dev_id == DP_DEVICE_ID_38EC11 &&
 		!memcmp(link->dpcd_caps.sink_dev_id_str, DP_SINK_DEVICE_STR_ID_1,
-			sizeof(link->dpcd_caps.sink_dev_id_str)))
+			sizeof(DP_SINK_DEVICE_STR_ID_1)))
 		link->psr_settings.force_ffu_mode = 1;
 	else
 		link->psr_settings.force_ffu_mode = 0;
 	copy_settings_data->force_ffu_mode = link->psr_settings.force_ffu_mode;
 
 	if (link->fec_state == dc_link_fec_enabled &&
+		link->dpcd_caps.sink_dev_id == DP_DEVICE_ID_38EC11 &&
 		(!memcmp(link->dpcd_caps.sink_dev_id_str, DP_SINK_DEVICE_STR_ID_1,
-			sizeof(link->dpcd_caps.sink_dev_id_str)) ||
+			sizeof(DP_SINK_DEVICE_STR_ID_1)) ||
 		!memcmp(link->dpcd_caps.sink_dev_id_str, DP_SINK_DEVICE_STR_ID_2,
-			sizeof(link->dpcd_caps.sink_dev_id_str))))
+			sizeof(DP_SINK_DEVICE_STR_ID_2))))
 		copy_settings_data->debug.bitfields.force_wakeup_by_tps3 = 1;
 	else
 		copy_settings_data->debug.bitfields.force_wakeup_by_tps3 = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
index aea49334021c..d260eaa1509e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c
@@ -722,7 +722,6 @@ void dce110_edp_wait_for_hpd_ready(
 	struct dc_context *ctx = link->ctx;
 	struct graphics_object_id connector = link->link_enc->connector;
 	struct gpio *hpd;
-	struct dc_sink *sink = link->local_sink;
 	bool edp_hpd_high = false;
 	uint32_t time_elapsed = 0;
 	uint32_t timeout = power_up ?
@@ -755,9 +754,9 @@ void dce110_edp_wait_for_hpd_ready(
 		return;
 	}
 
-	if (sink != NULL) {
-		if (sink->edid_caps.panel_patch.extra_t3_ms > 0) {
-			int extra_t3_in_ms = sink->edid_caps.panel_patch.extra_t3_ms;
+	if (link != NULL) {
+		if (link->panel_config.pps.extra_t3_ms > 0) {
+			int extra_t3_in_ms = link->panel_config.pps.extra_t3_ms;
 
 			msleep(extra_t3_in_ms);
 		}
@@ -842,7 +841,7 @@ void dce110_edp_power_control(
 			/* add time defined by a patch, if any (usually patch extra_t12_ms is 0) */
 			if (link->local_sink != NULL)
 				remaining_min_edp_poweroff_time_ms +=
-					link->local_sink->edid_caps.panel_patch.extra_t12_ms;
+					link->panel_config.pps.extra_t12_ms;
 
 			/* Adjust remaining_min_edp_poweroff_time_ms if this is not the first time. */
 			if (dp_trace_get_edp_poweroff_timestamp(link) != 0) {
@@ -946,7 +945,7 @@ void dce110_edp_wait_for_T12(
 						current_ts,
 						dp_trace_get_edp_poweroff_timestamp(link)), 1000000);
 
-		t12_duration += link->local_sink->edid_caps.panel_patch.extra_t12_ms; // Add extra T12
+		t12_duration += link->panel_config.pps.extra_t12_ms; // Add extra T12
 
 		if (time_since_edp_poweroff_ms < t12_duration)
 			msleep(t12_duration - time_since_edp_poweroff_ms);
@@ -965,6 +964,8 @@ void dce110_edp_backlight_control(
 	struct dc_context *ctx = link->ctx;
 	struct bp_transmitter_control cntl = { 0 };
 	uint8_t panel_instance;
+	unsigned int pre_T11_delay = OLED_PRE_T11_DELAY;
+	unsigned int post_T7_delay = OLED_POST_T7_DELAY;
 
 	if (dal_graphics_object_id_get_connector_id(link->link_enc->connector)
 		!= CONNECTOR_ID_EDP) {
@@ -1043,8 +1044,10 @@ void dce110_edp_backlight_control(
 
 	link_transmitter_control(ctx->dc_bios, &cntl);
 
-	if (enable && link->dpcd_sink_ext_caps.bits.oled)
-		msleep(OLED_POST_T7_DELAY);
+	if (enable && link->dpcd_sink_ext_caps.bits.oled) {
+		post_T7_delay += link->panel_config.pps.extra_post_t7_ms;
+		msleep(post_T7_delay);
+	}
 
 	if (link->dpcd_sink_ext_caps.bits.oled ||
 		link->dpcd_sink_ext_caps.bits.hdr_aux_backlight_control == 1 ||
@@ -1066,8 +1069,10 @@ void dce110_edp_backlight_control(
 			DC_LOG_DC("edp_receiver_ready_T9 skipped\n");
 	}
 
-	if (!enable && link->dpcd_sink_ext_caps.bits.oled)
-		msleep(OLED_PRE_T11_DELAY);
+	if (!enable && link->dpcd_sink_ext_caps.bits.oled) {
+		pre_T11_delay += link->panel_config.pps.extra_pre_t11_ms;
+		msleep(pre_T11_delay);
+	}
 }
 
 void dce110_enable_audio_stream(struct pipe_ctx *pipe_ctx)
@@ -1441,6 +1446,14 @@ static enum dc_status dce110_enable_stream_timing(
 			return DC_ERROR_UNEXPECTED;
 		}
 
+		if (dc_is_hdmi_tmds_signal(stream->signal)) {
+			stream->link->phy_state.symclk_ref_cnts.otg = 1;
+			if (stream->link->phy_state.symclk_state == SYMCLK_OFF_TX_OFF)
+				stream->link->phy_state.symclk_state = SYMCLK_ON_TX_OFF;
+			else
+				stream->link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
+		}
+
 		pipe_ctx->stream_res.tg->funcs->program_timing(
 				pipe_ctx->stream_res.tg,
 				&stream->timing,
@@ -2114,6 +2127,7 @@ static void dce110_reset_hw_ctx_wrap(
 				BREAK_TO_DEBUGGER();
 			}
 			pipe_ctx_old->stream_res.tg->funcs->disable_crtc(pipe_ctx_old->stream_res.tg);
+			pipe_ctx_old->stream->link->phy_state.symclk_ref_cnts.otg = 0;
 			pipe_ctx_old->plane_res.mi->funcs->free_mem_input(
 					pipe_ctx_old->plane_res.mi, dc->current_state->stream_count);
 
@@ -2164,8 +2178,7 @@ static void dce110_setup_audio_dto(
 			continue;
 		if (pipe_ctx->stream->signal != SIGNAL_TYPE_HDMI_TYPE_A)
 			continue;
-		if (pipe_ctx->stream_res.audio != NULL &&
-			pipe_ctx->stream_res.audio->enabled == false) {
+		if (pipe_ctx->stream_res.audio != NULL) {
 			struct audio_output audio_output;
 
 			build_audio_output(context, pipe_ctx, &audio_output);
@@ -2205,8 +2218,7 @@ static void dce110_setup_audio_dto(
 			if (!dc_is_dp_signal(pipe_ctx->stream->signal))
 				continue;
 
-			if (pipe_ctx->stream_res.audio != NULL &&
-				pipe_ctx->stream_res.audio->enabled == false) {
+			if (pipe_ctx->stream_res.audio != NULL) {
 				struct audio_output audio_output;
 
 				build_audio_output(context, pipe_ctx, &audio_output);
@@ -2994,6 +3006,124 @@ void dce110_set_pipe(struct pipe_ctx *pipe_ctx)
 		abm->funcs->set_pipe(abm, otg_inst, panel_cntl->inst);
 }
 
+void dce110_enable_lvds_link_output(struct dc_link *link,
+		const struct link_resource *link_res,
+		enum clock_source_id clock_source,
+		uint32_t pixel_clock)
+{
+	link->link_enc->funcs->enable_lvds_output(
+			link->link_enc,
+			clock_source,
+			pixel_clock);
+	link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
+}
+
+void dce110_enable_tmds_link_output(struct dc_link *link,
+		const struct link_resource *link_res,
+		enum signal_type signal,
+		enum clock_source_id clock_source,
+		enum dc_color_depth color_depth,
+		uint32_t pixel_clock)
+{
+	link->link_enc->funcs->enable_tmds_output(
+			link->link_enc,
+			clock_source,
+			color_depth,
+			signal,
+			pixel_clock);
+	link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
+}
+
+void dce110_enable_dp_link_output(
+		struct dc_link *link,
+		const struct link_resource *link_res,
+		enum signal_type signal,
+		enum clock_source_id clock_source,
+		const struct dc_link_settings *link_settings)
+{
+	struct dc  *dc = link->ctx->dc;
+	struct dmcu *dmcu = dc->res_pool->dmcu;
+	struct pipe_ctx *pipes =
+			link->dc->current_state->res_ctx.pipe_ctx;
+	struct clock_source *dp_cs =
+			link->dc->res_pool->dp_clock_source;
+	const struct link_hwss *link_hwss = get_link_hwss(link, link_res);
+	unsigned int i;
+
+
+	if (link->connector_signal == SIGNAL_TYPE_EDP) {
+		if (!link->dc->config.edp_no_power_sequencing)
+			link->dc->hwss.edp_power_control(link, true);
+		link->dc->hwss.edp_wait_for_hpd_ready(link, true);
+	}
+
+	/* If the current pixel clock source is not DTO(happens after
+	 * switching from HDMI passive dongle to DP on the same connector),
+	 * switch the pixel clock source to DTO.
+	 */
+
+	for (i = 0; i < MAX_PIPES; i++) {
+		if (pipes[i].stream != NULL &&
+				pipes[i].stream->link == link) {
+			if (pipes[i].clock_source != NULL &&
+					pipes[i].clock_source->id != CLOCK_SOURCE_ID_DP_DTO) {
+				pipes[i].clock_source = dp_cs;
+				pipes[i].stream_res.pix_clk_params.requested_pix_clk_100hz =
+						pipes[i].stream->timing.pix_clk_100hz;
+				pipes[i].clock_source->funcs->program_pix_clk(
+						pipes[i].clock_source,
+						&pipes[i].stream_res.pix_clk_params,
+						dp_get_link_encoding_format(link_settings),
+						&pipes[i].pll_settings);
+			}
+		}
+	}
+
+	if (dp_get_link_encoding_format(link_settings) == DP_8b_10b_ENCODING) {
+		if (dc->clk_mgr->funcs->notify_link_rate_change)
+			dc->clk_mgr->funcs->notify_link_rate_change(dc->clk_mgr, link);
+	}
+
+	if (dmcu != NULL && dmcu->funcs->lock_phy)
+		dmcu->funcs->lock_phy(dmcu);
+
+	if (link_hwss->ext.enable_dp_link_output)
+		link_hwss->ext.enable_dp_link_output(link, link_res, signal,
+				clock_source, link_settings);
+
+	link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
+
+	if (dmcu != NULL && dmcu->funcs->unlock_phy)
+		dmcu->funcs->unlock_phy(dmcu);
+
+	dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_LINK_PHY);
+}
+
+void dce110_disable_link_output(struct dc_link *link,
+		const struct link_resource *link_res,
+		enum signal_type signal)
+{
+	struct dc *dc = link->ctx->dc;
+	const struct link_hwss *link_hwss = get_link_hwss(link, link_res);
+	struct dmcu *dmcu = dc->res_pool->dmcu;
+
+	if (signal == SIGNAL_TYPE_EDP &&
+			link->dc->hwss.edp_backlight_control)
+		link->dc->hwss.edp_backlight_control(link, false);
+	else if (dmcu != NULL && dmcu->funcs->lock_phy)
+		dmcu->funcs->lock_phy(dmcu);
+
+	link_hwss->disable_link_output(link, link_res, signal);
+	link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+
+	if (signal == SIGNAL_TYPE_EDP &&
+			link->dc->hwss.edp_backlight_control)
+		link->dc->hwss.edp_power_control(link, false);
+	else if (dmcu != NULL && dmcu->funcs->lock_phy)
+		dmcu->funcs->unlock_phy(dmcu);
+	dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
+}
+
 static const struct hw_sequencer_funcs dce110_funcs = {
 	.program_gamut_remap = program_gamut_remap,
 	.program_output_csc = program_output_csc,
@@ -3033,6 +3163,10 @@ static const struct hw_sequencer_funcs dce110_funcs = {
 	.set_backlight_level = dce110_set_backlight_level,
 	.set_abm_immediate_disable = dce110_set_abm_immediate_disable,
 	.set_pipe = dce110_set_pipe,
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dce110_disable_link_output,
 };
 
 static const struct hwseq_private_funcs dce110_private_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
index b6f3843d3d05..758f4b3b0087 100644
--- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.h
@@ -90,6 +90,24 @@ bool dce110_set_backlight_level(struct pipe_ctx *pipe_ctx,
 		uint32_t frame_ramp);
 void dce110_set_abm_immediate_disable(struct pipe_ctx *pipe_ctx);
 void dce110_set_pipe(struct pipe_ctx *pipe_ctx);
-
+void dce110_disable_link_output(struct dc_link *link,
+		const struct link_resource *link_res,
+		enum signal_type signal);
+void dce110_enable_lvds_link_output(struct dc_link *link,
+		const struct link_resource *link_res,
+		enum clock_source_id clock_source,
+		uint32_t pixel_clock);
+void dce110_enable_tmds_link_output(struct dc_link *link,
+		const struct link_resource *link_res,
+		enum signal_type signal,
+		enum clock_source_id clock_source,
+		enum dc_color_depth color_depth,
+		uint32_t pixel_clock);
+void dce110_enable_dp_link_output(
+		struct dc_link *link,
+		const struct link_resource *link_res,
+		enum signal_type signal,
+		enum clock_source_id clock_source,
+		const struct dc_link_settings *link_settings);
 #endif /* __DC_HWSS_DCE110_H__ */
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
index db7ca4b0cdb9..897f412f539e 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp.c
@@ -448,11 +448,12 @@ void dpp1_set_cursor_position(
 			src_y_offset = pos->y - pos->x_hotspot - param->viewport.y;
 		}
 	} else if (param->rotation == ROTATION_ANGLE_180) {
-		src_x_offset = pos->x - param->viewport.x;
+		if (!param->mirror)
+			src_x_offset = pos->x - param->viewport.x;
+
 		src_y_offset = pos->y - param->viewport.y;
 	}
 
-
 	if (src_x_offset >= (int)param->viewport.width)
 		cur_en = 0;  /* not visible beyond right edge*/
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
index 564e061ccb58..52e201e9b091 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hubp.c
@@ -1208,13 +1208,10 @@ void hubp1_cursor_set_position(
 			src_y_offset = pos->y - pos->x_hotspot - param->viewport.y;
 		}
 	} else if (param->rotation == ROTATION_ANGLE_180) {
-		src_x_offset = pos->x - param->viewport.x;
-		src_y_offset = pos->y - param->viewport.y;
-	}
+		if (!param->mirror)
+			src_x_offset = pos->x - param->viewport.x;
 
-	if (param->mirror) {
-		x_hotspot = param->viewport.width - x_hotspot;
-		src_x_offset = param->viewport.x + param->viewport.width - src_x_offset;
+		src_y_offset = pos->y - param->viewport.y;
 	}
 
 	dst_x_offset = (src_x_offset >= 0) ? src_x_offset : 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
index 5b5d952b2b8c..72521749c01d 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c
@@ -899,6 +899,14 @@ enum dc_status dcn10_enable_stream_timing(
 		return DC_ERROR_UNEXPECTED;
 	}
 
+	if (dc_is_hdmi_tmds_signal(stream->signal)) {
+		stream->link->phy_state.symclk_ref_cnts.otg = 1;
+		if (stream->link->phy_state.symclk_state == SYMCLK_OFF_TX_OFF)
+			stream->link->phy_state.symclk_state = SYMCLK_ON_TX_OFF;
+		else
+			stream->link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
+	}
+
 	pipe_ctx->stream_res.tg->funcs->program_timing(
 			pipe_ctx->stream_res.tg,
 			&stream->timing,
@@ -1017,6 +1025,7 @@ static void dcn10_reset_back_end_for_pipe(
 		if (pipe_ctx->stream_res.tg->funcs->set_drr)
 			pipe_ctx->stream_res.tg->funcs->set_drr(
 					pipe_ctx->stream_res.tg, NULL);
+		pipe_ctx->stream->link->phy_state.symclk_ref_cnts.otg = 0;
 	}
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++)
@@ -2151,8 +2160,8 @@ static int dcn10_align_pixel_clocks(struct dc *dc, int group_size,
 				dc->res_pool->dp_clock_source->funcs->get_pixel_clk_frequency_100hz(
 					dc->res_pool->dp_clock_source,
 					grouped_pipes[i]->stream_res.tg->inst, &pclk);
-					grouped_pipes[i]->stream->timing.pix_clk_100hz =
-						pclk*get_clock_divider(grouped_pipes[i], false);
+				grouped_pipes[i]->stream->timing.pix_clk_100hz =
+					pclk*get_clock_divider(grouped_pipes[i], false);
 				if (master == -1)
 					master = i;
 			}
@@ -2199,14 +2208,14 @@ void dcn10_enable_vblanks_synchronization(
 	if (master >= 0) {
 		for (i = 0; i < group_size; i++) {
 			if (i != master && !grouped_pipes[i]->stream->has_non_synchronizable_pclk)
-			grouped_pipes[i]->stream_res.tg->funcs->align_vblanks(
-				grouped_pipes[master]->stream_res.tg,
-				grouped_pipes[i]->stream_res.tg,
-				grouped_pipes[master]->stream->timing.pix_clk_100hz,
-				grouped_pipes[i]->stream->timing.pix_clk_100hz,
-				get_clock_divider(grouped_pipes[master], false),
-				get_clock_divider(grouped_pipes[i], false));
-				grouped_pipes[i]->stream->vblank_synchronized = true;
+				grouped_pipes[i]->stream_res.tg->funcs->align_vblanks(
+					grouped_pipes[master]->stream_res.tg,
+					grouped_pipes[i]->stream_res.tg,
+					grouped_pipes[master]->stream->timing.pix_clk_100hz,
+					grouped_pipes[i]->stream->timing.pix_clk_100hz,
+					get_clock_divider(grouped_pipes[master], false),
+					get_clock_divider(grouped_pipes[i], false));
+			grouped_pipes[i]->stream->vblank_synchronized = true;
 		}
 		grouped_pipes[master]->stream->vblank_synchronized = true;
 		DC_SYNC_INFO("Sync complete\n");
@@ -2539,8 +2548,10 @@ void dcn10_update_visual_confirm_color(struct dc *dc, struct pipe_ctx *pipe_ctx,
 		color_space_to_black_color(
 				dc, pipe_ctx->stream->output_color_space, color);
 
-	if (mpc->funcs->set_bg_color)
+	if (mpc->funcs->set_bg_color) {
+		memcpy(&pipe_ctx->plane_state->visual_confirm_color, color, sizeof(struct tg_color));
 		mpc->funcs->set_bg_color(mpc, color, mpcc_id);
+	}
 }
 
 void dcn10_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
@@ -3340,11 +3351,11 @@ static bool dcn10_dmub_should_update_cursor_data(
 	if (pipe_ctx->plane_state->address.type == PLN_ADDR_TYPE_VIDEO_PROGRESSIVE)
 		return false;
 
-	if (pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_SU_1)
-		return true;
+	if (dcn10_can_pipe_disable_cursor(pipe_ctx))
+		return false;
 
-	if (pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_1 &&
-	    debug->enable_sw_cntl_psr)
+	if ((pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_SU_1 || pipe_ctx->stream->link->psr_settings.psr_version == DC_PSR_VERSION_1)
+			&& pipe_ctx->stream->ctx->dce_version >= DCN_VERSION_3_1)
 		return true;
 
 	return false;
@@ -3468,8 +3479,7 @@ void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
 		.rotation = pipe_ctx->plane_state->rotation,
 		.mirror = pipe_ctx->plane_state->horizontal_mirror
 	};
-	bool pipe_split_on = (pipe_ctx->top_pipe != NULL) ||
-		(pipe_ctx->bottom_pipe != NULL);
+	bool pipe_split_on = false;
 	bool odm_combine_on = (pipe_ctx->next_odm_pipe != NULL) ||
 		(pipe_ctx->prev_odm_pipe != NULL);
 
@@ -3478,6 +3488,13 @@ void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	int x_pos = pos_cpy.x;
 	int y_pos = pos_cpy.y;
 
+	if ((pipe_ctx->top_pipe != NULL) || (pipe_ctx->bottom_pipe != NULL)) {
+		if ((pipe_ctx->plane_state->src_rect.width != pipe_ctx->plane_res.scl_data.viewport.width) ||
+			(pipe_ctx->plane_state->src_rect.height != pipe_ctx->plane_res.scl_data.viewport.height)) {
+			pipe_split_on = true;
+		}
+	}
+
 	/**
 	 * DC cursor is stream space, HW cursor is plane space and drawn
 	 * as part of the framebuffer.
@@ -3549,8 +3566,36 @@ void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	if (pos_cpy.enable && dcn10_can_pipe_disable_cursor(pipe_ctx))
 		pos_cpy.enable = false;
 
+
+	if (param.rotation == ROTATION_ANGLE_0) {
+		int viewport_width =
+			pipe_ctx->plane_res.scl_data.viewport.width;
+		int viewport_x =
+			pipe_ctx->plane_res.scl_data.viewport.x;
+
+		if (param.mirror) {
+			if (pipe_split_on || odm_combine_on) {
+				if (pos_cpy.x >= viewport_width + viewport_x) {
+					pos_cpy.x = 2 * viewport_width
+							- pos_cpy.x + 2 * viewport_x;
+				} else {
+					uint32_t temp_x = pos_cpy.x;
+
+					pos_cpy.x = 2 * viewport_x - pos_cpy.x;
+					if (temp_x >= viewport_x +
+						(int)hubp->curs_attr.width || pos_cpy.x
+						<= (int)hubp->curs_attr.width +
+						pipe_ctx->plane_state->src_rect.x) {
+						pos_cpy.x = temp_x + viewport_width;
+					}
+				}
+			} else {
+				pos_cpy.x = viewport_width - pos_cpy.x + 2 * viewport_x;
+			}
+		}
+	}
 	// Swap axis and mirror horizontally
-	if (param.rotation == ROTATION_ANGLE_90) {
+	else if (param.rotation == ROTATION_ANGLE_90) {
 		uint32_t temp_x = pos_cpy.x;
 
 		pos_cpy.x = pipe_ctx->plane_res.scl_data.viewport.width -
@@ -3621,23 +3666,25 @@ void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
 		int viewport_x =
 			pipe_ctx->plane_res.scl_data.viewport.x;
 
-		if (pipe_split_on || odm_combine_on) {
-			if (pos_cpy.x >= viewport_width + viewport_x) {
-				pos_cpy.x = 2 * viewport_width
-						- pos_cpy.x + 2 * viewport_x;
-			} else {
-				uint32_t temp_x = pos_cpy.x;
-
-				pos_cpy.x = 2 * viewport_x - pos_cpy.x;
-				if (temp_x >= viewport_x +
-					(int)hubp->curs_attr.width || pos_cpy.x
-					<= (int)hubp->curs_attr.width +
-					pipe_ctx->plane_state->src_rect.x) {
-					pos_cpy.x = temp_x + viewport_width;
+		if (!param.mirror) {
+			if (pipe_split_on || odm_combine_on) {
+				if (pos_cpy.x >= viewport_width + viewport_x) {
+					pos_cpy.x = 2 * viewport_width
+							- pos_cpy.x + 2 * viewport_x;
+				} else {
+					uint32_t temp_x = pos_cpy.x;
+
+					pos_cpy.x = 2 * viewport_x - pos_cpy.x;
+					if (temp_x >= viewport_x +
+						(int)hubp->curs_attr.width || pos_cpy.x
+						<= (int)hubp->curs_attr.width +
+						pipe_ctx->plane_state->src_rect.x) {
+						pos_cpy.x = temp_x + viewport_width;
+					}
 				}
+			} else {
+				pos_cpy.x = viewport_width - pos_cpy.x + 2 * viewport_x;
 			}
-		} else {
-			pos_cpy.x = viewport_width - pos_cpy.x + 2 * viewport_x;
 		}
 
 		/**
@@ -3738,7 +3785,6 @@ int dcn10_get_vupdate_offset_from_vsync(struct pipe_ctx *pipe_ctx)
 	int vesa_sync_start;
 	int asic_blank_end;
 	int interlace_factor;
-	int vertical_line_start;
 
 	patched_crtc_timing = *dc_crtc_timing;
 	apply_front_porch_workaround(&patched_crtc_timing);
@@ -3754,10 +3800,8 @@ int dcn10_get_vupdate_offset_from_vsync(struct pipe_ctx *pipe_ctx)
 			patched_crtc_timing.v_border_top)
 			* interlace_factor;
 
-	vertical_line_start = asic_blank_end -
+	return asic_blank_end -
 			pipe_ctx->pipe_dlg_param.vstartup_start + 1;
-
-	return vertical_line_start;
 }
 
 void dcn10_calc_vupdate_position(
@@ -3768,7 +3812,7 @@ void dcn10_calc_vupdate_position(
 {
 	const struct dc_crtc_timing *dc_crtc_timing = &pipe_ctx->stream->timing;
 	int vline_int_offset_from_vupdate =
-			pipe_ctx->stream->periodic_interrupt0.lines_offset;
+			pipe_ctx->stream->periodic_interrupt.lines_offset;
 	int vupdate_offset_from_vsync = dc->hwss.get_vupdate_offset_from_vsync(pipe_ctx);
 	int start_position;
 
@@ -3793,18 +3837,10 @@ void dcn10_calc_vupdate_position(
 static void dcn10_cal_vline_position(
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx,
-		enum vline_select vline,
 		uint32_t *start_line,
 		uint32_t *end_line)
 {
-	enum vertical_interrupt_ref_point ref_point = INVALID_POINT;
-
-	if (vline == VLINE0)
-		ref_point = pipe_ctx->stream->periodic_interrupt0.ref_point;
-	else if (vline == VLINE1)
-		ref_point = pipe_ctx->stream->periodic_interrupt1.ref_point;
-
-	switch (ref_point) {
+	switch (pipe_ctx->stream->periodic_interrupt.ref_point) {
 	case START_V_UPDATE:
 		dcn10_calc_vupdate_position(
 				dc,
@@ -3813,7 +3849,9 @@ static void dcn10_cal_vline_position(
 				end_line);
 		break;
 	case START_V_SYNC:
-		// Suppose to do nothing because vsync is 0;
+		// vsync is line 0 so start_line is just the requested line offset
+		*start_line = pipe_ctx->stream->periodic_interrupt.lines_offset;
+		*end_line = *start_line + 2;
 		break;
 	default:
 		ASSERT(0);
@@ -3823,24 +3861,15 @@ static void dcn10_cal_vline_position(
 
 void dcn10_setup_periodic_interrupt(
 		struct dc *dc,
-		struct pipe_ctx *pipe_ctx,
-		enum vline_select vline)
+		struct pipe_ctx *pipe_ctx)
 {
 	struct timing_generator *tg = pipe_ctx->stream_res.tg;
+	uint32_t start_line = 0;
+	uint32_t end_line = 0;
 
-	if (vline == VLINE0) {
-		uint32_t start_line = 0;
-		uint32_t end_line = 0;
-
-		dcn10_cal_vline_position(dc, pipe_ctx, vline, &start_line, &end_line);
+	dcn10_cal_vline_position(dc, pipe_ctx, &start_line, &end_line);
 
-		tg->funcs->setup_vertical_interrupt0(tg, start_line, end_line);
-
-	} else if (vline == VLINE1) {
-		pipe_ctx->stream_res.tg->funcs->setup_vertical_interrupt1(
-				tg,
-				pipe_ctx->stream->periodic_interrupt1.lines_offset);
-	}
+	tg->funcs->setup_vertical_interrupt0(tg, start_line, end_line);
 }
 
 void dcn10_setup_vupdate_interrupt(struct dc *dc, struct pipe_ctx *pipe_ctx)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
index 9ae07c77fdc0..0ef7bf7ddb75 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h
@@ -175,8 +175,7 @@ void dcn10_set_cursor_attribute(struct pipe_ctx *pipe_ctx);
 void dcn10_set_cursor_sdr_white_level(struct pipe_ctx *pipe_ctx);
 void dcn10_setup_periodic_interrupt(
 		struct dc *dc,
-		struct pipe_ctx *pipe_ctx,
-		enum vline_select vline);
+		struct pipe_ctx *pipe_ctx);
 enum dc_status dcn10_set_clock(struct dc *dc,
 		enum dc_clock_type clock_type,
 		uint32_t clk_khz,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c
index 10e613ec7d24..f2371c948822 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c
@@ -82,6 +82,10 @@ static const struct hw_sequencer_funcs dcn10_funcs = {
 	.set_backlight_level = dce110_set_backlight_level,
 	.set_abm_immediate_disable = dce110_set_abm_immediate_disable,
 	.set_pipe = dce110_set_pipe,
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dce110_disable_link_output,
 	.get_dcc_en_bits = dcn10_get_dcc_en_bits,
 	.update_visual_confirm_color = dcn10_update_visual_confirm_color,
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
index 3fc300cd1ce9..ea7739255119 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.c
@@ -312,6 +312,20 @@ void optc1_program_timing(
 	}
 }
 
+/**
+ * optc1_set_vtg_params - Set Vertical Timing Generator (VTG) parameters
+ *
+ * @optc: timing_generator struct used to extract the optc parameters
+ * @dc_crtc_timing: Timing parameters configured
+ * @program_fp2: Boolean value indicating if FP2 will be programmed or not
+ *
+ * OTG is responsible for generating the global sync signals, including
+ * vertical timing information for each HUBP in the dcfclk domain. Each VTG is
+ * associated with one OTG that provides HUBP with vertical timing information
+ * (i.e., there is 1:1 correspondence between OTG and VTG). This function is
+ * responsible for setting the OTG parameters to the VTG during the pipe
+ * programming.
+ */
 void optc1_set_vtg_params(struct timing_generator *optc,
 		const struct dc_crtc_timing *dc_crtc_timing, bool program_fp2)
 {
@@ -1072,7 +1086,7 @@ static void optc1_set_test_pattern(
 				src_color[index] >> (src_bpc - dst_bpc);
 		/* CRTC_TEST_PATTERN_DATA has 16 bits,
 		 * lowest 6 are hardwired to ZERO
-		 * color bits should be left aligned aligned to MSB
+		 * color bits should be left aligned to MSB
 		 * XXXXXXXXXX000000 for 10 bit,
 		 * XXXXXXXX00000000 for 8 bit and XXXXXX0000000000 for 6
 		 */
@@ -1379,6 +1393,12 @@ void optc1_read_otg_state(struct optc *optc1,
 	REG_GET(OPTC_INPUT_GLOBAL_CONTROL,
 			OPTC_UNDERFLOW_OCCURRED_STATUS, &s->underflow_occurred_status);
 
+	REG_GET(OTG_VERTICAL_INTERRUPT1_CONTROL,
+			OTG_VERTICAL_INTERRUPT1_INT_ENABLE, &s->vertical_interrupt1_en);
+
+	REG_GET(OTG_VERTICAL_INTERRUPT1_POSITION,
+				OTG_VERTICAL_INTERRUPT1_LINE_START, &s->vertical_interrupt1_line);
+
 	REG_GET(OTG_VERTICAL_INTERRUPT2_CONTROL,
 			OTG_VERTICAL_INTERRUPT2_INT_ENABLE, &s->vertical_interrupt2_en);
 
@@ -1498,8 +1518,23 @@ bool optc1_configure_crc(struct timing_generator *optc,
 	return true;
 }
 
+/**
+ * optc1_get_crc - Capture CRC result per component
+ *
+ * @optc: timing_generator instance.
+ * @r_cr: 16-bit primary CRC signature for red data.
+ * @g_y: 16-bit primary CRC signature for green data.
+ * @b_cb: 16-bit primary CRC signature for blue data.
+ *
+ * This function reads the CRC signature from the OPTC registers. Notice that
+ * we have three registers to keep the CRC result per color component (RGB).
+ *
+ * Returns:
+ * If CRC is disabled, return false; otherwise, return true, and the CRC
+ * results in the parameters.
+ */
 bool optc1_get_crc(struct timing_generator *optc,
-		    uint32_t *r_cr, uint32_t *g_y, uint32_t *b_cb)
+		   uint32_t *r_cr, uint32_t *g_y, uint32_t *b_cb)
 {
 	uint32_t field = 0;
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
@@ -1510,12 +1545,14 @@ bool optc1_get_crc(struct timing_generator *optc,
 	if (!field)
 		return false;
 
+	/* OTG_CRC0_DATA_RG has the CRC16 results for the red and green component */
 	REG_GET_2(OTG_CRC0_DATA_RG,
-			CRC0_R_CR, r_cr,
-			CRC0_G_Y, g_y);
+		  CRC0_R_CR, r_cr,
+		  CRC0_G_Y, g_y);
 
+	/* OTG_CRC0_DATA_B has the CRC16 results for the blue component */
 	REG_GET(OTG_CRC0_DATA_B,
-			CRC0_B_CB, b_cb);
+		CRC0_B_CB, b_cb);
 
 	return true;
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
index 3fe5882ed018..6323ca6dc3b3 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_optc.h
@@ -583,6 +583,8 @@ struct dcn_otg_state {
 	uint32_t underflow_occurred_status;
 	uint32_t otg_enabled;
 	uint32_t blank_enabled;
+	uint32_t vertical_interrupt1_en;
+	uint32_t vertical_interrupt1_line;
 	uint32_t vertical_interrupt2_en;
 	uint32_t vertical_interrupt2_line;
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
index 174eebbe8b4f..831080b9eb87 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_resource.c
@@ -1495,6 +1495,24 @@ static bool dcn10_resource_construct(
 	/* Other architectures we build for build this with soft-float */
 	dcn10_resource_construct_fp(dc);
 
+	if (!dc->config.is_vmin_only_asic)
+		if (ASICREV_IS_RAVEN2(dc->ctx->asic_id.hw_internal_rev))
+			switch (dc->ctx->asic_id.pci_revision_id) {
+			case PRID_DALI_DE:
+			case PRID_DALI_DF:
+			case PRID_DALI_E3:
+			case PRID_DALI_E4:
+			case PRID_POLLOCK_94:
+			case PRID_POLLOCK_95:
+			case PRID_POLLOCK_E9:
+			case PRID_POLLOCK_EA:
+			case PRID_POLLOCK_EB:
+				dc->config.is_vmin_only_asic = true;
+				break;
+			default:
+				break;
+			}
+
 	pool->base.pp_smu = dcn10_pp_smu_create(ctx);
 
 	/*
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
index 2b9d3e63191b..915a20461c77 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dccg.h
@@ -274,6 +274,7 @@ struct dccg_registers {
 	uint32_t DSCCLK2_DTO_PARAM;
 	uint32_t DPSTREAMCLK_ROOT_GATE_DISABLE;
 	uint32_t DPSTREAMCLK_GATE_DISABLE;
+	uint32_t DCCG_GATE_DISABLE_CNTL;
 	uint32_t DCCG_GATE_DISABLE_CNTL2;
 	uint32_t DCCG_GATE_DISABLE_CNTL3;
 	uint32_t HDMISTREAMCLK0_DTO_PARAM;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c
index 9570c2118ccc..b1ec0e6f7f58 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hubp.c
@@ -987,13 +987,10 @@ void hubp2_cursor_set_position(
 			src_y_offset = pos->y - pos->x_hotspot - param->viewport.y;
 		}
 	} else if (param->rotation == ROTATION_ANGLE_180) {
-		src_x_offset = pos->x - param->viewport.x;
-		src_y_offset = pos->y - param->viewport.y;
-	}
+		if (!param->mirror)
+			src_x_offset = pos->x - param->viewport.x;
 
-	if (param->mirror) {
-		x_hotspot = param->viewport.width - x_hotspot;
-		src_x_offset = param->viewport.x + param->viewport.width - src_x_offset;
+		src_y_offset = pos->y - param->viewport.y;
 	}
 
 	dst_x_offset = (src_x_offset >= 0) ? src_x_offset : 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
index 598ce872a8d7..e1d271fe9e64 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c
@@ -706,6 +706,14 @@ enum dc_status dcn20_enable_stream_timing(
 		return DC_ERROR_UNEXPECTED;
 	}
 
+	if (dc_is_hdmi_tmds_signal(stream->signal)) {
+		stream->link->phy_state.symclk_ref_cnts.otg = 1;
+		if (stream->link->phy_state.symclk_state == SYMCLK_OFF_TX_OFF)
+			stream->link->phy_state.symclk_state = SYMCLK_ON_TX_OFF;
+		else
+			stream->link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
+	}
+
 	if (dc->hwseq->funcs.PLAT_58856_wa && (!dc_is_dp_signal(stream->signal)))
 		dc->hwseq->funcs.PLAT_58856_wa(context, pipe_ctx);
 
@@ -1899,8 +1907,14 @@ void dcn20_post_unlock_program_front_end(
 			 * can underflow due to HUBP_VTG_SEL programming if done in the regular front end
 			 * programming sequence).
 			 */
-			if (pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_PHANTOM)
+			while (pipe) {
+				if (pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
+					if (dc->hwss.update_phantom_vp_position)
+						dc->hwss.update_phantom_vp_position(dc, context, pipe);
 					dcn20_program_pipe(dc, pipe, context);
+				}
+				pipe = pipe->bottom_pipe;
+			}
 		}
 	}
 
@@ -2347,7 +2361,9 @@ static void dcn20_reset_back_end_for_pipe(
 		struct dc_state *context)
 {
 	int i;
-	struct dc_link *link;
+	struct dc_link *link = pipe_ctx->stream->link;
+	const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res);
+
 	DC_LOGGER_INIT(dc->ctx->logger);
 	if (pipe_ctx->stream_res.stream_enc == NULL) {
 		pipe_ctx->stream = NULL;
@@ -2355,7 +2371,6 @@ static void dcn20_reset_back_end_for_pipe(
 	}
 
 	if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment)) {
-		link = pipe_ctx->stream->link;
 		/* DPMS may already disable or */
 		/* dpms_off status is incorrect due to fastboot
 		 * feature. When system resume from S4 with second
@@ -2404,6 +2419,16 @@ static void dcn20_reset_back_end_for_pipe(
 		if (pipe_ctx->stream_res.tg->funcs->set_drr)
 			pipe_ctx->stream_res.tg->funcs->set_drr(
 					pipe_ctx->stream_res.tg, NULL);
+		/* TODO - convert symclk_ref_cnts for otg to a bit map to solve
+		 * the case where the same symclk is shared across multiple otg
+		 * instances
+		 */
+		link->phy_state.symclk_ref_cnts.otg = 0;
+		if (link->phy_state.symclk_state == SYMCLK_ON_TX_OFF) {
+			link_hwss->disable_link_output(link,
+					&pipe_ctx->link_res, pipe_ctx->stream->signal);
+			link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+		}
 	}
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++)
@@ -2463,9 +2488,13 @@ void dcn20_update_visual_confirm_color(struct dc *dc, struct pipe_ctx *pipe_ctx,
 		get_mpctree_visual_confirm_color(pipe_ctx, color);
 	else if (dc->debug.visual_confirm == VISUAL_CONFIRM_SWIZZLE)
 		get_surface_tile_visual_confirm_color(pipe_ctx, color);
+	else if (dc->debug.visual_confirm == VISUAL_CONFIRM_SUBVP)
+		get_subvp_visual_confirm_color(dc, pipe_ctx, color);
 
-	if (mpc->funcs->set_bg_color)
+	if (mpc->funcs->set_bg_color) {
+		memcpy(&pipe_ctx->plane_state->visual_confirm_color, color, sizeof(struct tg_color));
 		mpc->funcs->set_bg_color(mpc, color, mpcc_id);
+	}
 }
 
 void dcn20_update_mpcc(struct dc *dc, struct pipe_ctx *pipe_ctx)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c
index 91e4885b743e..7c5817c426fa 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_init.c
@@ -96,6 +96,10 @@ static const struct hw_sequencer_funcs dcn20_funcs = {
 #ifndef TRIM_FSFT
 	.optimize_timing_for_fsft = dcn20_optimize_timing_for_fsft,
 #endif
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dce110_disable_link_output,
 	.set_disp_pattern_generator = dcn20_set_disp_pattern_generator,
 	.get_dcc_en_bits = dcn10_get_dcc_en_bits,
 	.update_visual_confirm_color = dcn20_update_visual_confirm_color
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c
index 694260c10a01..ccd91792991b 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_mmhubbub.c
@@ -215,7 +215,8 @@ void mmhubbub2_config_mcif_irq(struct mcif_wb *mcif_wb,
 	REG_UPDATE(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN,  params->sw_overrun_int_en);
 
 	REG_UPDATE(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN,  params->vce_int_en);
-	REG_UPDATE(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN,  params->vce_slice_int_en);
+	if (mcif_wb20->mcif_wb_mask->MCIF_WB_BUFMGR_VCE_SLICE_INT_EN)
+		REG_UPDATE(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN,  params->vce_slice_int_en);
 }
 
 void mmhubbub2_enable_mcif(struct mcif_wb *mcif_wb)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c
index 05b3fba9ccce..61bcfa03c4e7 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_hwseq.c
@@ -82,7 +82,7 @@ static bool patch_address_for_sbs_tb_stereo(
 	return false;
 }
 
-static void gpu_addr_to_uma(struct dce_hwseq *hwseq,
+static bool gpu_addr_to_uma(struct dce_hwseq *hwseq,
 		PHYSICAL_ADDRESS_LOC *addr)
 {
 	bool is_in_uma;
@@ -98,6 +98,7 @@ static void gpu_addr_to_uma(struct dce_hwseq *hwseq,
 	} else {
 		is_in_uma = false;
 	}
+	return is_in_uma;
 }
 
 static void plane_address_in_gpu_space_to_uma(struct dce_hwseq *hwseq,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_init.c b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_init.c
index 1826dd7f3da1..9c16633e473a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn201/dcn201_init.c
@@ -86,6 +86,10 @@ static const struct hw_sequencer_funcs dcn201_funcs = {
 	.set_backlight_level = dce110_set_backlight_level,
 	.set_abm_immediate_disable = dce110_set_abm_immediate_disable,
 	.set_pipe = dce110_set_pipe,
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dce110_disable_link_output,
 	.set_disp_pattern_generator = dcn20_set_disp_pattern_generator,
 	.update_visual_confirm_color = dcn20_update_visual_confirm_color,
 };
diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c
index b270f0b194dc..fe1a8e2e08ef 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_init.c
@@ -99,6 +99,10 @@ static const struct hw_sequencer_funcs dcn21_funcs = {
 #ifndef TRIM_FSFT
 	.optimize_timing_for_fsft = dcn20_optimize_timing_for_fsft,
 #endif
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dce110_disable_link_output,
 	.is_abm_supported = dcn21_is_abm_supported,
 	.set_disp_pattern_generator = dcn20_set_disp_pattern_generator,
 	.get_dcc_en_bits = dcn10_get_dcc_en_bits,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c
index fb59fed8f425..8c5045711264 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_hwseq.c
@@ -939,13 +939,32 @@ bool dcn30_does_plane_fit_in_mall(struct dc *dc, struct dc_plane_state *plane, s
 
 void dcn30_hardware_release(struct dc *dc)
 {
+	bool subvp_in_use = false;
+	uint32_t i;
+
 	dc_dmub_srv_p_state_delegate(dc, false, NULL);
+	dc_dmub_setup_subvp_dmub_command(dc, dc->current_state, false);
+
+	/* SubVP treated the same way as FPO. If driver disable and
+	 * we are using a SubVP config, disable and force on DCN side
+	 * to prevent P-State hang on driver enable.
+	 */
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
 
+		if (!pipe->stream)
+			continue;
+
+		if (pipe->stream->mall_stream_config.type == SUBVP_MAIN) {
+			subvp_in_use = true;
+			break;
+		}
+	}
 	/* If pstate unsupported, or still supported
 	 * by firmware, force it supported by dcn
 	 */
 	if (dc->current_state)
-		if ((!dc->clk_mgr->clks.p_state_change_support ||
+		if ((!dc->clk_mgr->clks.p_state_change_support || subvp_in_use ||
 				dc->current_state->bw_ctx.bw.dcn.clk.fw_based_mclk_switching) &&
 				dc->res_pool->hubbub->funcs->force_pstate_change_control)
 			dc->res_pool->hubbub->funcs->force_pstate_change_control(
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c
index 4c06e6e1ba4a..3216d10c58ba 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_init.c
@@ -100,6 +100,10 @@ static const struct hw_sequencer_funcs dcn30_funcs = {
 	.set_abm_immediate_disable = dcn21_set_abm_immediate_disable,
 	.hardware_release = dcn30_hardware_release,
 	.set_pipe = dcn21_set_pipe,
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dce110_disable_link_output,
 	.set_disp_pattern_generator = dcn30_set_disp_pattern_generator,
 	.get_dcc_en_bits = dcn10_get_dcc_en_bits,
 	.update_visual_confirm_color = dcn20_update_visual_confirm_color,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.h b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.h
index f2580e65196c..7446e54bf5aa 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mmhubbub.h
@@ -227,11 +227,7 @@
 	SF(MCIF_WB0_MCIF_WB_BUF_3_ADDR_C, MCIF_WB_BUF_3_ADDR_C, mask_sh),\
 	SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_Y, MCIF_WB_BUF_4_ADDR_Y, mask_sh),\
 	SF(MCIF_WB0_MCIF_WB_BUF_4_ADDR_C, MCIF_WB_BUF_4_ADDR_C, mask_sh),\
-	SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK_IGNORE, mask_sh),\
-	SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN, mask_sh),\
-	SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_ACK, mask_sh),\
 	SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, mask_sh),\
-	SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK, mask_sh),\
 	SF(MCIF_WB0_MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_SLICE_SIZE, mask_sh),\
 	SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_REFRESH_WATERMARK, mask_sh),\
 	SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_WATERMARK_MASK, mask_sh),\
@@ -363,11 +359,7 @@
 	SF(MCIF_WB_BUF_3_ADDR_C, MCIF_WB_BUF_3_ADDR_C, mask_sh),\
 	SF(MCIF_WB_BUF_4_ADDR_Y, MCIF_WB_BUF_4_ADDR_Y, mask_sh),\
 	SF(MCIF_WB_BUF_4_ADDR_C, MCIF_WB_BUF_4_ADDR_C, mask_sh),\
-	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK_IGNORE, mask_sh),\
-	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN, mask_sh),\
-	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_ACK, mask_sh),\
 	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, mask_sh),\
-	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK, mask_sh),\
 	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_SLICE_SIZE, mask_sh),\
 	SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_REFRESH_WATERMARK, mask_sh),\
 	SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_WATERMARK_MASK, mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
index 64320e0ca446..3a3b2ac791c7 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c
@@ -724,7 +724,8 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.dwb_fi_phase = -1, // -1 = disable,
 	.dmub_command_table = true,
 	.disable_psr = false,
-	.use_max_lb = true
+	.use_max_lb = true,
+	.exit_idle_opt_for_cursor_updates = true
 };
 
 static const struct dc_debug_options debug_defaults_diags = {
@@ -1916,7 +1917,7 @@ static int get_refresh_rate(struct dc_state *context)
  */
 #define V_SCALE (10000 / MAX_STRETCHED_V_BLANK)
 
-int get_frame_rate_at_max_stretch_100hz(struct dc_state *context)
+static int get_frame_rate_at_max_stretch_100hz(struct dc_state *context)
 {
 	struct dc_crtc_timing *timing = NULL;
 	uint32_t sec_per_100_lines;
@@ -1946,7 +1947,7 @@ int get_frame_rate_at_max_stretch_100hz(struct dc_state *context)
 	return scaled_refresh_rate;
 }
 
-bool is_refresh_rate_support_mclk_switch_using_fw_based_vblank_stretch(struct dc_state *context)
+static bool is_refresh_rate_support_mclk_switch_using_fw_based_vblank_stretch(struct dc_state *context)
 {
 	int refresh_rate_max_stretch_100hz;
 	int min_refresh_100hz;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c
index 3d42a1a337ec..6192851c59ed 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_init.c
@@ -99,6 +99,10 @@ static const struct hw_sequencer_funcs dcn301_funcs = {
 	.set_backlight_level = dcn21_set_backlight_level,
 	.set_abm_immediate_disable = dcn21_set_abm_immediate_disable,
 	.set_pipe = dcn21_set_pipe,
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dce110_disable_link_output,
 	.set_disp_pattern_generator = dcn30_set_disp_pattern_generator,
 	.get_dcc_en_bits = dcn10_get_dcc_en_bits,
 	.optimize_pwr_state = dcn21_optimize_pwr_state,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
index db172677d613..559e563d5bc1 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn301/dcn301_resource.c
@@ -634,7 +634,7 @@ static const struct dcn20_vmid_mask vmid_masks = {
 		DCN20_VMID_MASK_SH_LIST(_MASK)
 };
 
-static const struct resource_caps res_cap_dcn301 = {
+static struct resource_caps res_cap_dcn301 = {
 	.num_timing_generator = 4,
 	.num_opp = 4,
 	.num_video_plane = 4,
@@ -700,6 +700,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.dwb_fi_phase = -1, // -1 = disable
 	.dmub_command_table = true,
 	.use_max_lb = false,
+	.exit_idle_opt_for_cursor_updates = true
 };
 
 static const struct dc_debug_options debug_defaults_diags = {
@@ -1429,6 +1430,8 @@ static bool dcn301_resource_construct(
 
 	ctx->dc_bios->regs = &bios_regs;
 
+	if (dc->ctx->asic_id.chip_id == DEVICE_ID_VGH_1435)
+		res_cap_dcn301.num_pll = 2;
 	pool->base.res_cap = &res_cap_dcn301;
 
 	pool->base.funcs = &dcn301_res_pool_funcs;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
index 4fab537e822f..b925b6ddde5a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn302/dcn302_resource.c
@@ -93,7 +93,8 @@ static const struct dc_debug_options debug_defaults_drv = {
 		.underflow_assert_delay_us = 0xFFFFFFFF,
 		.dwb_fi_phase = -1, // -1 = disable,
 		.dmub_command_table = true,
-		.use_max_lb = true
+		.use_max_lb = true,
+		.exit_idle_opt_for_cursor_updates = true
 };
 
 static const struct dc_debug_options debug_defaults_diags = {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
index d97076648acb..527d5c902878 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn303/dcn303_resource.c
@@ -77,6 +77,7 @@ static const struct dc_debug_options debug_defaults_drv = {
 		.underflow_assert_delay_us = 0xFFFFFFFF,
 		.dwb_fi_phase = -1, // -1 = disable,
 		.dmub_command_table = true,
+		.exit_idle_opt_for_cursor_updates = true,
 		.disable_idle_power_optimizations = false,
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c
index 23621ff08c90..52fb2bf3d578 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hpo_dp_stream_encoder.c
@@ -150,9 +150,9 @@ static void dcn31_hpo_dp_stream_enc_dp_blank(
 	 * 10us*5000=50ms. This covers 41.7ms of minimum 24 Hz mode +
 	 * a little more because we may not trust delay accuracy.
 	 */
-	//REG_WAIT(DP_SYM32_ENC_VID_STREAM_CONTROL,
-	//		VID_STREAM_STATUS, 0,
-	//		10, 5000);
+	REG_WAIT(DP_SYM32_ENC_VID_STREAM_CONTROL,
+			VID_STREAM_STATUS, 0,
+			10, 5000);
 
 	/* Disable SDP tranmission */
 	REG_UPDATE(DP_SYM32_ENC_SDP_CONTROL,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c
index 51c5f3685470..6360dc9502e7 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.c
@@ -876,7 +876,7 @@ static bool hubbub31_get_dcc_compression_cap(struct hubbub *hubbub,
 	return true;
 }
 
-static int hubbub31_init_dchub_sys_ctx(struct hubbub *hubbub,
+int hubbub31_init_dchub_sys_ctx(struct hubbub *hubbub,
 		struct dcn_hubbub_phys_addr_config *pa_config)
 {
 	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.h
index e3a654bf04e8..70c60de448ac 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hubbub.h
@@ -122,6 +122,8 @@
 	HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_PIPE, mask_sh), \
 	HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_INTERRUPT_STATUS, mask_sh)
 
+int hubbub31_init_dchub_sys_ctx(struct hubbub *hubbub,
+		struct dcn_hubbub_phys_addr_config *pa_config);
 
 void hubbub31_construct(struct dcn20_hubbub *hubbub3,
 	struct dc_context *ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c
index 1ed1404e969d..bdf101547484 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_hwseq.c
@@ -535,11 +535,11 @@ static void dcn31_reset_back_end_for_pipe(
 			pipe_ctx->stream_res.tg,
 			OPTC_DSC_DISABLED, 0, 0);
 	pipe_ctx->stream_res.tg->funcs->disable_crtc(pipe_ctx->stream_res.tg);
-
 	pipe_ctx->stream_res.tg->funcs->enable_optc_clock(pipe_ctx->stream_res.tg, false);
 	if (pipe_ctx->stream_res.tg->funcs->set_odm_bypass)
 		pipe_ctx->stream_res.tg->funcs->set_odm_bypass(
 				pipe_ctx->stream_res.tg, &pipe_ctx->stream->timing);
+	pipe_ctx->stream->link->phy_state.symclk_ref_cnts.otg = 0;
 
 	if (pipe_ctx->stream_res.tg->funcs->set_drr)
 		pipe_ctx->stream_res.tg->funcs->set_drr(
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c
index e708f07fe75a..3a32810bbe38 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_init.c
@@ -100,6 +100,10 @@ static const struct hw_sequencer_funcs dcn31_funcs = {
 	.set_backlight_level = dcn21_set_backlight_level,
 	.set_abm_immediate_disable = dcn21_set_abm_immediate_disable,
 	.set_pipe = dcn21_set_pipe,
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dce110_disable_link_output,
 	.z10_restore = dcn31_z10_restore,
 	.z10_save_init = dcn31_z10_save_init,
 	.set_disp_pattern_generator = dcn30_set_disp_pattern_generator,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
index aedff18aff56..8c1a6fb36306 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn31/dcn31_resource.c
@@ -889,9 +889,8 @@ static const struct dc_debug_options debug_defaults_drv = {
 	},
 	.disable_z10 = true,
 	.optimize_edp_link_rate = true,
-	.enable_sw_cntl_psr = true,
 	.enable_z9_disable_interface = true, /* Allow support for the PMFW interface for disable Z9*/
-	.dml_hostvm_override = DML_HOSTVM_OVERRIDE_FALSE,
+	.dml_hostvm_override = DML_HOSTVM_NO_OVERRIDE,
 };
 
 static const struct dc_debug_options debug_defaults_diags = {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c
index 232cc15979dd..1bd7e0f327d8 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.c
@@ -45,6 +45,48 @@
 #define DC_LOGGER \
 	dccg->ctx->logger
 
+static void dccg314_get_pixel_rate_div(
+		struct dccg *dccg,
+		uint32_t otg_inst,
+		enum pixel_rate_div *k1,
+		enum pixel_rate_div *k2)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+	uint32_t val_k1 = PIXEL_RATE_DIV_NA, val_k2 = PIXEL_RATE_DIV_NA;
+
+	*k1 = PIXEL_RATE_DIV_NA;
+	*k2 = PIXEL_RATE_DIV_NA;
+
+	switch (otg_inst) {
+	case 0:
+		REG_GET_2(OTG_PIXEL_RATE_DIV,
+			OTG0_PIXEL_RATE_DIVK1, &val_k1,
+			OTG0_PIXEL_RATE_DIVK2, &val_k2);
+		break;
+	case 1:
+		REG_GET_2(OTG_PIXEL_RATE_DIV,
+			OTG1_PIXEL_RATE_DIVK1, &val_k1,
+			OTG1_PIXEL_RATE_DIVK2, &val_k2);
+		break;
+	case 2:
+		REG_GET_2(OTG_PIXEL_RATE_DIV,
+			OTG2_PIXEL_RATE_DIVK1, &val_k1,
+			OTG2_PIXEL_RATE_DIVK2, &val_k2);
+		break;
+	case 3:
+		REG_GET_2(OTG_PIXEL_RATE_DIV,
+			OTG3_PIXEL_RATE_DIVK1, &val_k1,
+			OTG3_PIXEL_RATE_DIVK2, &val_k2);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		return;
+	}
+
+	*k1 = (enum pixel_rate_div)val_k1;
+	*k2 = (enum pixel_rate_div)val_k2;
+}
+
 static void dccg314_set_pixel_rate_div(
 		struct dccg *dccg,
 		uint32_t otg_inst,
@@ -52,6 +94,11 @@ static void dccg314_set_pixel_rate_div(
 		enum pixel_rate_div k2)
 {
 	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+	enum pixel_rate_div cur_k1 = PIXEL_RATE_DIV_NA, cur_k2 = PIXEL_RATE_DIV_NA;
+
+	dccg314_get_pixel_rate_div(dccg, otg_inst, &cur_k1, &cur_k2);
+	if (k1 == PIXEL_RATE_DIV_NA || k2 == PIXEL_RATE_DIV_NA || (k1 == cur_k1 && k2 == cur_k2))
+		return;
 
 	switch (otg_inst) {
 	case 0:
@@ -137,7 +184,7 @@ static void dccg314_set_dtbclk_p_src(
 }
 
 /* Controls the generation of pixel valid for OTG in (OTG -> HPO case) */
-void dccg314_set_dtbclk_dto(
+static void dccg314_set_dtbclk_dto(
 		struct dccg *dccg,
 		const struct dtbclk_dto_params *params)
 {
@@ -181,7 +228,7 @@ void dccg314_set_dtbclk_dto(
 	}
 }
 
-void dccg314_set_dpstreamclk(
+static void dccg314_set_dpstreamclk(
 		struct dccg *dccg,
 		enum streamclk_source src,
 		int otg_inst,
@@ -220,7 +267,7 @@ void dccg314_set_dpstreamclk(
 	}
 }
 
-void dccg314_set_valid_pixel_rate(
+static void dccg314_set_valid_pixel_rate(
 		struct dccg *dccg,
 		int ref_dtbclk_khz,
 		int otg_inst,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h
index 9a4a9efc0203..6a35986307af 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dccg.h
@@ -63,34 +63,28 @@
 	DCCG_SRII(PHASE, DTBCLK_DTO, 3),\
 	SR(DCCG_AUDIO_DTBCLK_DTO_MODULO),\
 	SR(DCCG_AUDIO_DTBCLK_DTO_PHASE),\
+	SR(DCCG_AUDIO_DTO_SOURCE),\
+	SR(DENTIST_DISPCLK_CNTL),\
+	SR(DSCCLK0_DTO_PARAM),\
+	SR(DSCCLK1_DTO_PARAM),\
+	SR(DSCCLK2_DTO_PARAM),\
+	SR(DSCCLK_DTO_CTRL),\
+	SR(DCCG_GATE_DISABLE_CNTL2),\
+	SR(DCCG_GATE_DISABLE_CNTL3),\
+	SR(HDMISTREAMCLK0_DTO_PARAM),\
 	SR(OTG_PIXEL_RATE_DIV),\
 	SR(DTBCLK_P_CNTL),\
 	SR(DCCG_AUDIO_DTO_SOURCE)
 
-
-#define DCCG_MASK_SH_LIST_DCN314(mask_sh) \
-	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 0, mask_sh),\
+#define DCCG_MASK_SH_LIST_DCN314_COMMON(mask_sh) \
 	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 0, mask_sh),\
-	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 1, mask_sh),\
 	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 1, mask_sh),\
-	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 2, mask_sh),\
 	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 2, mask_sh),\
-	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 3, mask_sh),\
 	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_DB_EN, DPPCLK, 3, mask_sh),\
 	DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_PHASE, mask_sh),\
 	DCCG_SF(DPPCLK0_DTO_PARAM, DPPCLK0_DTO_MODULO, mask_sh),\
 	DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_EN, mask_sh),\
 	DCCG_SF(HDMICHARCLK0_CLOCK_CNTL, HDMICHARCLK0_SRC_SEL, mask_sh),\
-	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_EN, mask_sh),\
-	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_SRC_SEL, mask_sh),\
-	DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_EN, mask_sh),\
-	DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_SRC_SEL, mask_sh),\
-	DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_FORCE_EN, mask_sh),\
-	DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_FORCE_SRC_SEL, mask_sh),\
-	DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_FORCE_EN, mask_sh),\
-	DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_FORCE_SRC_SEL, mask_sh),\
-	DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_FORCE_EN, mask_sh),\
-	DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_FORCE_SRC_SEL, mask_sh),\
 	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK0_EN, mask_sh),\
 	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK1_EN, mask_sh),\
 	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK2_EN, mask_sh),\
@@ -100,7 +94,6 @@
 	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK2_SRC_SEL, mask_sh),\
 	DCCG_SF(DPSTREAMCLK_CNTL, DPSTREAMCLK3_SRC_SEL, mask_sh),\
 	DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_EN, mask_sh),\
-	DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_DTO_FORCE_DIS, mask_sh),\
 	DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_SRC_SEL, mask_sh),\
 	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE0_SRC_SEL, mask_sh),\
 	DCCG_SF(SYMCLK32_SE_CNTL, SYMCLK32_SE1_SRC_SEL, mask_sh),\
@@ -148,7 +141,48 @@
 	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P3_SRC_SEL, mask_sh),\
 	DCCG_SF(DTBCLK_P_CNTL, DTBCLK_P3_EN, mask_sh),\
 	DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO0_SOURCE_SEL, mask_sh),\
-	DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO_SEL, mask_sh)
+	DCCG_SF(DCCG_AUDIO_DTO_SOURCE, DCCG_AUDIO_DTO_SEL, mask_sh),\
+	DCCG_SF(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_CHG_MODE, mask_sh),\
+	DCCG_SF(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_PHASE, mask_sh),\
+	DCCG_SF(DSCCLK0_DTO_PARAM, DSCCLK0_DTO_MODULO, mask_sh),\
+	DCCG_SF(DSCCLK1_DTO_PARAM, DSCCLK1_DTO_PHASE, mask_sh),\
+	DCCG_SF(DSCCLK1_DTO_PARAM, DSCCLK1_DTO_MODULO, mask_sh),\
+	DCCG_SF(DSCCLK2_DTO_PARAM, DSCCLK2_DTO_PHASE, mask_sh),\
+	DCCG_SF(DSCCLK2_DTO_PARAM, DSCCLK2_DTO_MODULO, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE2_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_SE3_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE0_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL3, SYMCLK32_ROOT_LE1_GATE_DISABLE, mask_sh),\
+	DCCG_SF(HDMISTREAMCLK0_DTO_PARAM, HDMISTREAMCLK0_DTO_PHASE, mask_sh),\
+	DCCG_SF(HDMISTREAMCLK0_DTO_PARAM, HDMISTREAMCLK0_DTO_MODULO, mask_sh)
+
+#define DCCG_MASK_SH_LIST_DCN314(mask_sh) \
+	DCCG_MASK_SH_LIST_DCN314_COMMON(mask_sh),\
+	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 0, mask_sh),\
+	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 1, mask_sh),\
+	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 2, mask_sh),\
+	DCCG_SFI(DPPCLK_DTO_CTRL, DTO_ENABLE, DPPCLK, 3, mask_sh),\
+	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_EN, mask_sh),\
+	DCCG_SF(PHYASYMCLK_CLOCK_CNTL, PHYASYMCLK_FORCE_SRC_SEL, mask_sh),\
+	DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_EN, mask_sh),\
+	DCCG_SF(PHYBSYMCLK_CLOCK_CNTL, PHYBSYMCLK_FORCE_SRC_SEL, mask_sh),\
+	DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_FORCE_EN, mask_sh),\
+	DCCG_SF(PHYCSYMCLK_CLOCK_CNTL, PHYCSYMCLK_FORCE_SRC_SEL, mask_sh),\
+	DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_FORCE_EN, mask_sh),\
+	DCCG_SF(PHYDSYMCLK_CLOCK_CNTL, PHYDSYMCLK_FORCE_SRC_SEL, mask_sh),\
+	DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_FORCE_EN, mask_sh),\
+	DCCG_SF(PHYESYMCLK_CLOCK_CNTL, PHYESYMCLK_FORCE_SRC_SEL, mask_sh),\
+	DCCG_SF(HDMISTREAMCLK_CNTL, HDMISTREAMCLK0_DTO_FORCE_DIS, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK0_DTO_ENABLE, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK1_DTO_ENABLE, mask_sh),\
+	DCCG_SF(DSCCLK_DTO_CTRL, DSCCLK2_DTO_ENABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYASYMCLK_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYBSYMCLK_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYCSYMCLK_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYDSYMCLK_GATE_DISABLE, mask_sh),\
+	DCCG_SF(DCCG_GATE_DISABLE_CNTL2, PHYESYMCLK_GATE_DISABLE, mask_sh)
 
 struct dccg *dccg314_create(
 	struct dc_context *ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c
index 06d8638db696..0d2ffb692957 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_dio_stream_encoder.c
@@ -56,7 +56,8 @@ static void enc314_enable_fifo(struct stream_encoder *enc)
 
 	/* TODO: Confirm if we need to wait for DIG_SYMCLK_FE_ON */
 	REG_WAIT(DIG_FE_CNTL, DIG_SYMCLK_FE_ON, 1, 10, 5000);
-	REG_UPDATE_2(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 1, DIG_FIFO_READ_START_LEVEL, 0x7);
+	REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_READ_START_LEVEL, 0x7);
+	REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 1);
 	REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 1, 10, 5000);
 	REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_RESET, 0);
 	REG_WAIT(DIG_FIFO_CTRL0, DIG_FIFO_RESET_DONE, 0, 10, 5000);
@@ -80,7 +81,7 @@ static void enc314_dp_set_odm_combine(
 }
 
 /* setup stream encoder in dvi mode */
-void enc314_stream_encoder_dvi_set_stream_attribute(
+static void enc314_stream_encoder_dvi_set_stream_attribute(
 	struct stream_encoder *enc,
 	struct dc_crtc_timing *crtc_timing,
 	bool is_dual_link)
@@ -261,6 +262,16 @@ static bool is_two_pixels_per_containter(const struct dc_crtc_timing *timing)
 	return two_pix;
 }
 
+void enc314_stream_encoder_dp_blank(
+	struct dc_link *link,
+	struct stream_encoder *enc)
+{
+	/* New to DCN314 - disable the FIFO before VID stream disable. */
+	enc314_disable_fifo(enc);
+
+	enc1_stream_encoder_dp_blank(link, enc);
+}
+
 static void enc314_stream_encoder_dp_unblank(
 		struct dc_link *link,
 		struct stream_encoder *enc,
@@ -316,15 +327,11 @@ static void enc314_stream_encoder_dp_unblank(
 	/* switch DP encoder to CRTC data, but reset it the fifo first. It may happen
 	 * that it overflows during mode transition, and sometimes doesn't recover.
 	 */
-	REG_UPDATE(DIG_FIFO_CTRL0, DIG_FIFO_READ_START_LEVEL, 0x7);
 	REG_UPDATE(DP_STEER_FIFO, DP_STEER_FIFO_RESET, 1);
 	udelay(10);
 
 	REG_UPDATE(DP_STEER_FIFO, DP_STEER_FIFO_RESET, 0);
 
-	/* DIG Resync FIFO now needs to be explicitly enabled. */
-	enc314_enable_fifo(enc);
-
 	/* wait 100us for DIG/DP logic to prime
 	 * (i.e. a few video lines)
 	 */
@@ -340,6 +347,12 @@ static void enc314_stream_encoder_dp_unblank(
 
 	REG_UPDATE(DP_VID_STREAM_CNTL, DP_VID_STREAM_ENABLE, true);
 
+	/*
+	 * DIG Resync FIFO now needs to be explicitly enabled.
+	 * This should come after DP_VID_STREAM_ENABLE per HW docs.
+	 */
+	enc314_enable_fifo(enc);
+
 	dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_DP_VID_STREAM);
 }
 
@@ -408,7 +421,7 @@ static const struct stream_encoder_funcs dcn314_str_enc_funcs = {
 	.stop_dp_info_packets =
 		enc1_stream_encoder_stop_dp_info_packets,
 	.dp_blank =
-		enc1_stream_encoder_dp_blank,
+		enc314_stream_encoder_dp_blank,
 	.dp_unblank =
 		enc314_stream_encoder_dp_unblank,
 	.audio_mute_control = enc3_audio_mute_control,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c
index f4d1b83979fe..588c1c71241f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_hwseq.c
@@ -348,6 +348,9 @@ unsigned int dcn314_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsig
 	two_pix_per_container = optc2_is_two_pixels_per_containter(&stream->timing);
 	odm_combine_factor = get_odm_config(pipe_ctx, NULL);
 
+	if (pipe_ctx->stream->signal == SIGNAL_TYPE_VIRTUAL)
+		return odm_combine_factor;
+
 	if (is_dp_128b_132b_signal(pipe_ctx)) {
 		*k2_div = PIXEL_RATE_DIV_BY_1;
 	} else if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal) || dc_is_dvi_signal(pipe_ctx->stream->signal)) {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c
index 72a563a4c3e8..5b6c2d94ec71 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_init.c
@@ -102,6 +102,10 @@ static const struct hw_sequencer_funcs dcn314_funcs = {
 	.set_backlight_level = dcn21_set_backlight_level,
 	.set_abm_immediate_disable = dcn21_set_abm_immediate_disable,
 	.set_pipe = dcn21_set_pipe,
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dce110_disable_link_output,
 	.z10_restore = dcn31_z10_restore,
 	.z10_save_init = dcn31_z10_save_init,
 	.set_disp_pattern_generator = dcn30_set_disp_pattern_generator,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c
index 38aa28ec6b13..47eb162f1a75 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_optc.c
@@ -150,7 +150,7 @@ static bool optc314_disable_crtc(struct timing_generator *optc)
 	return true;
 }
 
-void optc314_phantom_crtc_post_enable(struct timing_generator *optc)
+static void optc314_phantom_crtc_post_enable(struct timing_generator *optc)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c
index 44ac1c2aabf5..24ec71cbd3e3 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn314/dcn314_resource.c
@@ -915,7 +915,6 @@ static const struct dc_debug_options debug_defaults_drv = {
 		}
 	},
 	.optimize_edp_link_rate = true,
-	.enable_sw_cntl_psr = true,
 	.seamless_boot_odm_combine = true
 };
 
@@ -1647,6 +1646,7 @@ static struct clock_source *dcn31_clock_source_create(
 	}
 
 	BREAK_TO_DEBUGGER();
+	kfree(clk_src);
 	return NULL;
 }
 
@@ -1719,6 +1719,7 @@ static struct clock_source *dcn30_clock_source_create(
 	}
 
 	BREAK_TO_DEBUGGER();
+	kfree(clk_src);
 	return NULL;
 }
 
@@ -1818,8 +1819,6 @@ static bool dcn314_resource_construct(
 
 	if (dc->ctx->dce_environment == DCE_ENV_PRODUCTION_DRV)
 		dc->debug = debug_defaults_drv;
-	else if (dc->ctx->dce_environment == DCE_ENV_FPGA_MAXIMUS)
-		dc->debug = debug_defaults_diags;
 	else
 		dc->debug = debug_defaults_diags;
 	// Init the vm_helper
diff --git a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c
index 7463b12ae4a3..eebb42c9ddd6 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn315/dcn315_resource.c
@@ -886,7 +886,6 @@ static const struct dc_debug_options debug_defaults_drv = {
 		}
 	},
 	.optimize_edp_link_rate = true,
-	.enable_sw_cntl_psr = true,
 	.psr_power_use_phy_fsm = 0,
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c
index d56a212e065c..f4b52a35ad84 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn316/dcn316_resource.c
@@ -886,7 +886,6 @@ static const struct dc_debug_options debug_defaults_drv = {
 		}
 	},
 	.optimize_edp_link_rate = true,
-	.enable_sw_cntl_psr = true,
 };
 
 static const struct dc_debug_options debug_defaults_diags = {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c
index 0d5e8a441512..e4daed44ef5f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dccg.c
@@ -42,6 +42,48 @@
 #define DC_LOGGER \
 	dccg->ctx->logger
 
+static void dccg32_get_pixel_rate_div(
+		struct dccg *dccg,
+		uint32_t otg_inst,
+		enum pixel_rate_div *k1,
+		enum pixel_rate_div *k2)
+{
+	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
+	uint32_t val_k1 = PIXEL_RATE_DIV_NA, val_k2 = PIXEL_RATE_DIV_NA;
+
+	*k1 = PIXEL_RATE_DIV_NA;
+	*k2 = PIXEL_RATE_DIV_NA;
+
+	switch (otg_inst) {
+	case 0:
+		REG_GET_2(OTG_PIXEL_RATE_DIV,
+			OTG0_PIXEL_RATE_DIVK1, &val_k1,
+			OTG0_PIXEL_RATE_DIVK2, &val_k2);
+		break;
+	case 1:
+		REG_GET_2(OTG_PIXEL_RATE_DIV,
+			OTG1_PIXEL_RATE_DIVK1, &val_k1,
+			OTG1_PIXEL_RATE_DIVK2, &val_k2);
+		break;
+	case 2:
+		REG_GET_2(OTG_PIXEL_RATE_DIV,
+			OTG2_PIXEL_RATE_DIVK1, &val_k1,
+			OTG2_PIXEL_RATE_DIVK2, &val_k2);
+		break;
+	case 3:
+		REG_GET_2(OTG_PIXEL_RATE_DIV,
+			OTG3_PIXEL_RATE_DIVK1, &val_k1,
+			OTG3_PIXEL_RATE_DIVK2, &val_k2);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		return;
+	}
+
+	*k1 = (enum pixel_rate_div)val_k1;
+	*k2 = (enum pixel_rate_div)val_k2;
+}
+
 static void dccg32_set_pixel_rate_div(
 		struct dccg *dccg,
 		uint32_t otg_inst,
@@ -50,6 +92,17 @@ static void dccg32_set_pixel_rate_div(
 {
 	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
 
+	enum pixel_rate_div cur_k1 = PIXEL_RATE_DIV_NA, cur_k2 = PIXEL_RATE_DIV_NA;
+
+	// Don't program 0xF into the register field. Not valid since
+	// K1 / K2 field is only 1 / 2 bits wide
+	if (k1 == PIXEL_RATE_DIV_NA || k2 == PIXEL_RATE_DIV_NA)
+		return;
+
+	dccg32_get_pixel_rate_div(dccg, otg_inst, &cur_k1, &cur_k2);
+	if (k1 == cur_k1 && k2 == cur_k2)
+		return;
+
 	switch (otg_inst) {
 	case 0:
 		REG_UPDATE_2(OTG_PIXEL_RATE_DIV,
@@ -133,7 +186,7 @@ static void dccg32_set_dtbclk_p_src(
 }
 
 /* Controls the generation of pixel valid for OTG in (OTG -> HPO case) */
-void dccg32_set_dtbclk_dto(
+static void dccg32_set_dtbclk_dto(
 		struct dccg *dccg,
 		const struct dtbclk_dto_params *params)
 {
@@ -208,7 +261,7 @@ static void dccg32_get_dccg_ref_freq(struct dccg *dccg,
 	return;
 }
 
-void dccg32_set_dpstreamclk(
+static void dccg32_set_dpstreamclk(
 		struct dccg *dccg,
 		enum streamclk_source src,
 		int otg_inst,
@@ -245,7 +298,7 @@ void dccg32_set_dpstreamclk(
 	}
 }
 
-void dccg32_otg_add_pixel(struct dccg *dccg,
+static void dccg32_otg_add_pixel(struct dccg *dccg,
 		uint32_t otg_inst)
 {
 	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
@@ -254,7 +307,7 @@ void dccg32_otg_add_pixel(struct dccg *dccg,
 			OTG_ADD_PIXEL[otg_inst], 1);
 }
 
-void dccg32_otg_drop_pixel(struct dccg *dccg,
+static void dccg32_otg_drop_pixel(struct dccg *dccg,
 		uint32_t otg_inst)
 {
 	struct dcn_dccg *dccg_dcn = TO_DCN_DCCG(dccg);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c
index d6855d4f749b..fdae6aa89908 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_link_encoder.c
@@ -118,7 +118,7 @@ void dcn32_link_encoder_enable_dp_output(
 	}
 }
 
-bool dcn32_link_encoder_is_in_alt_mode(struct link_encoder *enc)
+static bool dcn32_link_encoder_is_in_alt_mode(struct link_encoder *enc)
 {
 	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
 	uint32_t dp_alt_mode_disable = 0;
@@ -133,7 +133,7 @@ bool dcn32_link_encoder_is_in_alt_mode(struct link_encoder *enc)
 	return is_usb_c_alt_mode;
 }
 
-void dcn32_link_encoder_get_max_link_cap(struct link_encoder *enc,
+static void dcn32_link_encoder_get_max_link_cap(struct link_encoder *enc,
 	struct dc_link_settings *link_settings)
 {
 	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c
index 38a48983f663..0e9dce414641 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dio_stream_encoder.c
@@ -60,7 +60,7 @@ static void enc32_dp_set_odm_combine(
 }
 
 /* setup stream encoder in dvi mode */
-void enc32_stream_encoder_dvi_set_stream_attribute(
+static void enc32_stream_encoder_dvi_set_stream_attribute(
 	struct stream_encoder *enc,
 	struct dc_crtc_timing *crtc_timing,
 	bool is_dual_link)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.c
index f349cbe2a0f0..dcf12a0b031c 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_dpp.c
@@ -31,7 +31,7 @@
 #include "dcn30/dcn30_cm_common.h"
 
 /* Compute the maximum number of lines that we can fit in the line buffer */
-void dscl32_calc_lb_num_partitions(
+static void dscl32_calc_lb_num_partitions(
 		const struct scaler_data *scl_data,
 		enum lb_memory_config lb_config,
 		int *num_part_y,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c
index 99eb239bbc7b..f6d3da475835 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.c
@@ -68,7 +68,7 @@ static void dcn32_init_crb(struct hubbub *hubbub)
 	REG_UPDATE(DCHUBBUB_DEBUG_CTRL_0, DET_DEPTH, 0x47F);
 }
 
-static void dcn32_program_det_size(struct hubbub *hubbub, int hubp_inst, unsigned int det_buffer_size_in_kbyte)
+void dcn32_program_det_size(struct hubbub *hubbub, int hubp_inst, unsigned int det_buffer_size_in_kbyte)
 {
 	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
 
@@ -98,9 +98,13 @@ static void dcn32_program_det_size(struct hubbub *hubbub, int hubp_inst, unsigne
 	default:
 		break;
 	}
-	/* Should never be hit, if it is we have an erroneous hw config*/
-	ASSERT(hubbub2->det0_size + hubbub2->det1_size + hubbub2->det2_size
-			+ hubbub2->det3_size + hubbub2->compbuf_size_segments <= hubbub2->crb_size_segs);
+	if (hubbub2->det0_size + hubbub2->det1_size + hubbub2->det2_size
+			+ hubbub2->det3_size + hubbub2->compbuf_size_segments > hubbub2->crb_size_segs) {
+		/* This may happen during seamless transition from ODM 2:1 to ODM4:1 */
+		DC_LOG_WARNING("CRB Config Warning: DET size (%d,%d,%d,%d) + Compbuf size (%d) >  CRB segments (%d)\n",
+						hubbub2->det0_size, hubbub2->det1_size, hubbub2->det2_size, hubbub2->det3_size,
+						hubbub2->compbuf_size_segments, hubbub2->crb_size_segs);
+	}
 }
 
 static void dcn32_program_compbuf_size(struct hubbub *hubbub, unsigned int compbuf_size_kb, bool safe_to_increase)
@@ -140,7 +144,7 @@ static uint32_t convert_and_clamp(
 	return ret_val;
 }
 
-static bool hubbub32_program_urgent_watermarks(
+bool hubbub32_program_urgent_watermarks(
 		struct hubbub *hubbub,
 		struct dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
@@ -330,7 +334,7 @@ static bool hubbub32_program_urgent_watermarks(
 	return wm_pending;
 }
 
-static bool hubbub32_program_stutter_watermarks(
+bool hubbub32_program_stutter_watermarks(
 		struct hubbub *hubbub,
 		struct dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
@@ -476,7 +480,7 @@ static bool hubbub32_program_stutter_watermarks(
 }
 
 
-static bool hubbub32_program_pstate_watermarks(
+bool hubbub32_program_pstate_watermarks(
 		struct hubbub *hubbub,
 		struct dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
@@ -629,7 +633,7 @@ static bool hubbub32_program_pstate_watermarks(
 }
 
 
-static bool hubbub32_program_usr_watermarks(
+bool hubbub32_program_usr_watermarks(
 		struct hubbub *hubbub,
 		struct dcn_watermark_set *watermarks,
 		unsigned int refclk_mhz,
@@ -769,7 +773,7 @@ static bool hubbub32_program_watermarks(
 }
 
 /* Copy values from WM set A to all other sets */
-void hubbub32_init_watermarks(struct hubbub *hubbub)
+static void hubbub32_init_watermarks(struct hubbub *hubbub)
 {
 	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
 	uint32_t reg;
@@ -820,7 +824,7 @@ void hubbub32_init_watermarks(struct hubbub *hubbub)
 	REG_WRITE(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D, reg);
 }
 
-void hubbub32_wm_read_state(struct hubbub *hubbub,
+static void hubbub32_wm_read_state(struct hubbub *hubbub,
 		struct dcn_hubbub_wm *wm)
 {
 	struct dcn20_hubbub *hubbub2 = TO_DCN20_HUBBUB(hubbub);
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h
index 3bae6e558971..cda94e0e31bf 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hubbub.h
@@ -161,6 +161,35 @@
 	HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_PIPE, mask_sh), \
 	HUBBUB_SF(DCN_VM_FAULT_STATUS, DCN_VM_ERROR_INTERRUPT_STATUS, mask_sh)
 
+bool hubbub32_program_urgent_watermarks(
+		struct hubbub *hubbub,
+		struct dcn_watermark_set *watermarks,
+		unsigned int refclk_mhz,
+		bool safe_to_lower);
+
+bool hubbub32_program_stutter_watermarks(
+		struct hubbub *hubbub,
+		struct dcn_watermark_set *watermarks,
+		unsigned int refclk_mhz,
+		bool safe_to_lower);
+
+bool hubbub32_program_pstate_watermarks(
+		struct hubbub *hubbub,
+		struct dcn_watermark_set *watermarks,
+		unsigned int refclk_mhz,
+		bool safe_to_lower);
+
+bool hubbub32_program_usr_watermarks(
+		struct hubbub *hubbub,
+		struct dcn_watermark_set *watermarks,
+		unsigned int refclk_mhz,
+		bool safe_to_lower);
+
+void hubbub32_force_usr_retraining_allow(struct hubbub *hubbub, bool allow);
+
+void hubbub32_force_wm_propagate_to_pipes(struct hubbub *hubbub);
+
+void dcn32_program_det_size(struct hubbub *hubbub, int hubp_inst, unsigned int det_buffer_size_in_kbyte);
 
 void hubbub32_construct(struct dcn20_hubbub *hubbub2,
 	struct dc_context *ctx,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c
index 344fe7535df5..a750343ca521 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.c
@@ -49,6 +49,7 @@
 #include "dcn20/dcn20_optc.h"
 #include "dmub_subvp_state.h"
 #include "dce/dmub_hw_lock_mgr.h"
+#include "dcn32_resource.h"
 #include "dc_link_dp.h"
 #include "dmub/inc/dmub_subvp_state.h"
 
@@ -198,42 +199,6 @@ static bool dcn32_check_no_memory_request_for_cab(struct dc *dc)
 	return false;
 }
 
-/* This function takes in the start address and surface size to be cached in CAB
- * and calculates the total number of cache lines required to store the surface.
- * The number of cache lines used for each surface is calculated independently of
- * one another. For example, if there is a primary surface(1), meta surface(2), and
- * cursor(3), this function should be called 3 times to calculate the number of cache
- * lines used for each of those surfaces.
- */
-static uint32_t dcn32_cache_lines_for_surface(struct dc *dc, uint32_t surface_size, uint64_t start_address)
-{
-	uint32_t lines_used = 1;
-	uint32_t num_cached_bytes = 0;
-	uint32_t remaining_size = 0;
-	uint32_t cache_line_size = dc->caps.cache_line_size;
-	uint32_t remainder = 0;
-
-	/* 1. Calculate surface size minus the number of bytes stored
-	 * in the first cache line (all bytes in first cache line might
-	 * not be fully used).
-	 */
-	div_u64_rem(start_address, cache_line_size, &remainder);
-	num_cached_bytes = cache_line_size - remainder;
-	remaining_size = surface_size - num_cached_bytes;
-
-	/* 2. Calculate number of cache lines that will be fully used with
-	 * the remaining number of bytes to be stored.
-	 */
-	lines_used += (remaining_size / cache_line_size);
-
-	/* 3. Check if we need an extra line due to the remaining size not being
-	 * a multiple of CACHE_LINE_SIZE.
-	 */
-	if (remaining_size % cache_line_size > 0)
-		lines_used++;
-
-	return lines_used;
-}
 
 /* This function loops through every surface that needs to be cached in CAB for SS,
  * and calculates the total number of ways required to store all surfaces (primary,
@@ -241,94 +206,116 @@ static uint32_t dcn32_cache_lines_for_surface(struct dc *dc, uint32_t surface_si
  */
 static uint32_t dcn32_calculate_cab_allocation(struct dc *dc, struct dc_state *ctx)
 {
-	uint8_t i, j;
+	uint8_t i;
+	int j;
 	struct dc_stream_state *stream = NULL;
 	struct dc_plane_state *plane = NULL;
-	uint32_t surface_size = 0;
 	uint32_t cursor_size = 0;
-	uint32_t cache_lines_used = 0;
 	uint32_t total_lines = 0;
 	uint32_t lines_per_way = 0;
-	uint32_t num_ways = 0;
-	uint32_t prev_addr_low = 0;
+	uint8_t num_ways = 0;
+	uint8_t bytes_per_pixel = 0;
+	uint8_t cursor_bpp = 0;
+	uint16_t mblk_width = 0;
+	uint16_t mblk_height = 0;
+	uint16_t mall_alloc_width_blk_aligned = 0;
+	uint16_t mall_alloc_height_blk_aligned = 0;
+	uint16_t num_mblks = 0;
+	uint32_t bytes_in_mall = 0;
+	uint32_t cache_lines_used = 0;
+	uint32_t cache_lines_per_plane = 0;
 
-	for (i = 0; i < ctx->stream_count; i++) {
-		stream = ctx->streams[i];
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
 
-		// Don't include PSR surface in the total surface size for CAB allocation
-		if (stream->link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED)
+		if (!pipe->stream || !pipe->plane_state ||
+				pipe->stream->link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED ||
+				pipe->stream->mall_stream_config.type == SUBVP_PHANTOM)
 			continue;
 
-		if (ctx->stream_status[i].plane_count == 0)
-			continue;
+		bytes_per_pixel = pipe->plane_state->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? 8 : 4;
+		mblk_width = DCN3_2_MBLK_WIDTH;
+		mblk_height = bytes_per_pixel == 4 ? DCN3_2_MBLK_HEIGHT_4BPE : DCN3_2_MBLK_HEIGHT_8BPE;
 
-		// For each stream, loop through each plane to calculate the number of cache
-		// lines required to store the surface in CAB
-		for (j = 0; j < ctx->stream_status[i].plane_count; j++) {
-			plane = ctx->stream_status[i].plane_states[j];
+		/* full_vp_width_blk_aligned = FLOOR(vp_x_start + full_vp_width + blk_width - 1, blk_width) -
+		 * FLOOR(vp_x_start, blk_width)
+		 *
+		 * mall_alloc_width_blk_aligned_l/c = full_vp_width_blk_aligned_l/c
+		 */
+		mall_alloc_width_blk_aligned = ((pipe->plane_res.scl_data.viewport.x +
+				pipe->plane_res.scl_data.viewport.width + mblk_width - 1) / mblk_width * mblk_width) -
+						(pipe->plane_res.scl_data.viewport.x / mblk_width * mblk_width);
+
+		/* full_vp_height_blk_aligned = FLOOR(vp_y_start + full_vp_height + blk_height - 1, blk_height) -
+		 * FLOOR(vp_y_start, blk_height)
+		 *
+		 * mall_alloc_height_blk_aligned_l/c = full_vp_height_blk_aligned_l/c
+		 */
+		mall_alloc_height_blk_aligned = ((pipe->plane_res.scl_data.viewport.y +
+				pipe->plane_res.scl_data.viewport.height + mblk_height - 1) / mblk_height * mblk_height) -
+						(pipe->plane_res.scl_data.viewport.y / mblk_height * mblk_height);
 
-			// Calculate total surface size
-			if (prev_addr_low != plane->address.grph.addr.u.low_part) {
-				/* if plane address are different from prev FB, then userspace allocated separate FBs*/
-				surface_size += plane->plane_size.surface_pitch *
-					plane->plane_size.surface_size.height *
-					(plane->format >= SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616 ? 8 : 4);
+		num_mblks = ((mall_alloc_width_blk_aligned + mblk_width - 1) / mblk_width) *
+				((mall_alloc_height_blk_aligned + mblk_height - 1) / mblk_height);
 
-				prev_addr_low = plane->address.grph.addr.u.low_part;
-			} else {
-				/* We have the same fb for all the planes.
-				 * Xorg always creates one giant fb that holds all surfaces,
-				 * so allocating it once is sufficient.
-				 * */
-				continue;
-			}
-			// Convert surface size + starting address to number of cache lines required
-			// (alignment accounted for)
-			cache_lines_used += dcn32_cache_lines_for_surface(dc, surface_size,
-					plane->address.grph.addr.quad_part);
-
-			if (plane->address.grph.meta_addr.quad_part) {
-				// Meta surface
-				cache_lines_used += dcn32_cache_lines_for_surface(dc, surface_size,
-						plane->address.grph.meta_addr.quad_part);
-			}
-		}
+		/* For DCC:
+		 * meta_num_mblk = CEILING(full_mblk_width_ub_l*full_mblk_height_ub_l*Bpe/256/mblk_bytes, 1)
+		 */
+		if (pipe->plane_state->dcc.enable)
+			num_mblks += (mall_alloc_width_blk_aligned * mall_alloc_width_blk_aligned * bytes_per_pixel +
+					(256 * DCN3_2_MALL_MBLK_SIZE_BYTES) - 1) / (256 * DCN3_2_MALL_MBLK_SIZE_BYTES);
 
-		// Include cursor size for CAB allocation
-		for (j = 0; j < dc->res_pool->pipe_count; j++) {
-			struct pipe_ctx *pipe = &ctx->res_ctx.pipe_ctx[j];
-			struct hubp *hubp = pipe->plane_res.hubp;
+		bytes_in_mall = num_mblks * DCN3_2_MALL_MBLK_SIZE_BYTES;
 
-			if (pipe->stream && pipe->plane_state && hubp)
-				/* Find the cursor plane and use the exact size instead of
-				 * using the max for calculation
-				 */
-				if (hubp->curs_attr.width > 0) {
-					cursor_size = hubp->curs_attr.width * hubp->curs_attr.height;
-					break;
-				}
-		}
+		/* (cache lines used is total bytes / cache_line size. Add +2 for worst case alignment
+		 * (MALL is 64-byte aligned)
+		 */
+		cache_lines_per_plane = bytes_in_mall / dc->caps.cache_line_size + 2;
+		cache_lines_used += cache_lines_per_plane;
+	}
 
-		switch (stream->cursor_attributes.color_format) {
-		case CURSOR_MODE_MONO:
-			cursor_size /= 2;
-			break;
-		case CURSOR_MODE_COLOR_1BIT_AND:
-		case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA:
-		case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA:
-			cursor_size *= 4;
-			break;
+	// Include cursor size for CAB allocation
+	for (j = 0; j < dc->res_pool->pipe_count; j++) {
+		struct pipe_ctx *pipe = &ctx->res_ctx.pipe_ctx[j];
+		struct hubp *hubp = pipe->plane_res.hubp;
 
-		case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED:
-		case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED:
-			cursor_size *= 8;
-			break;
-		}
+		if (pipe->stream && pipe->plane_state && hubp)
+			/* Find the cursor plane and use the exact size instead of
+			using the max for calculation */
 
-		if (stream->cursor_position.enable && plane->address.grph.cursor_cache_addr.quad_part) {
-			cache_lines_used += dcn32_cache_lines_for_surface(dc, cursor_size,
-					plane->address.grph.cursor_cache_addr.quad_part);
-		}
+		if (hubp->curs_attr.width > 0) {
+				// Round cursor width to next multiple of 64
+				cursor_size = (((hubp->curs_attr.width + 63) / 64) * 64) * hubp->curs_attr.height;
+
+				switch (pipe->stream->cursor_attributes.color_format) {
+				case CURSOR_MODE_MONO:
+					cursor_size /= 2;
+					cursor_bpp = 4;
+					break;
+				case CURSOR_MODE_COLOR_1BIT_AND:
+				case CURSOR_MODE_COLOR_PRE_MULTIPLIED_ALPHA:
+				case CURSOR_MODE_COLOR_UN_PRE_MULTIPLIED_ALPHA:
+					cursor_size *= 4;
+					cursor_bpp = 4;
+					break;
+
+				case CURSOR_MODE_COLOR_64BIT_FP_PRE_MULTIPLIED:
+				case CURSOR_MODE_COLOR_64BIT_FP_UN_PRE_MULTIPLIED:
+					cursor_size *= 8;
+					cursor_bpp = 8;
+					break;
+				}
+
+				if (pipe->stream->cursor_position.enable && !dc->debug.alloc_extra_way_for_cursor &&
+						cursor_size > 16384) {
+					/* cursor_num_mblk = CEILING(num_cursors*cursor_width*cursor_width*cursor_Bpe/mblk_bytes, 1)
+					 */
+					cache_lines_used += (((hubp->curs_attr.width * hubp->curs_attr.height * cursor_bpp +
+										DCN3_2_MALL_MBLK_SIZE_BYTES - 1) / DCN3_2_MALL_MBLK_SIZE_BYTES) *
+										DCN3_2_MALL_MBLK_SIZE_BYTES) / dc->caps.cache_line_size + 2;
+				}
+				break;
+			}
 	}
 
 	// Convert number of cache lines required to number of ways
@@ -345,8 +332,8 @@ static uint32_t dcn32_calculate_cab_allocation(struct dc *dc, struct dc_state *c
 			plane = ctx->stream_status[i].plane_states[j];
 
 			if (stream->cursor_position.enable && plane &&
-				!plane->address.grph.cursor_cache_addr.quad_part &&
-				cursor_size > 16384) {
+					dc->debug.alloc_extra_way_for_cursor &&
+					cursor_size > 16384) {
 				/* Cursor caching is not supported since it won't be on the same line.
 				 * So we need an extra line to accommodate it. With large cursors and a single 4k monitor
 				 * this case triggers corruption. If we're at the edge, then dont trigger display refresh
@@ -358,7 +345,9 @@ static uint32_t dcn32_calculate_cab_allocation(struct dc *dc, struct dc_state *c
 			}
 		}
 	}
-
+	if (dc->debug.force_mall_ss_num_ways > 0) {
+		num_ways = dc->debug.force_mall_ss_num_ways;
+	}
 	return num_ways;
 }
 
@@ -367,7 +356,7 @@ bool dcn32_apply_idle_power_optimizations(struct dc *dc, bool enable)
 	union dmub_rb_cmd cmd;
 	uint8_t ways, i;
 	int j;
-	bool stereo_in_use = false;
+	bool mall_ss_unsupported = false;
 	struct dc_plane_state *plane = NULL;
 
 	if (!dc->ctx->dmub_srv)
@@ -398,22 +387,23 @@ bool dcn32_apply_idle_power_optimizations(struct dc *dc, bool enable)
 			 */
 			ways = dcn32_calculate_cab_allocation(dc, dc->current_state);
 
-			/* MALL not supported with Stereo3D. If any plane is using stereo,
-			 * don't try to enter MALL.
+			/* MALL not supported with Stereo3D or TMZ surface. If any plane is using stereo,
+			 * or TMZ surface, don't try to enter MALL.
 			 */
 			for (i = 0; i < dc->current_state->stream_count; i++) {
 				for (j = 0; j < dc->current_state->stream_status[i].plane_count; j++) {
 					plane = dc->current_state->stream_status[i].plane_states[j];
 
-					if (plane->address.type == PLN_ADDR_TYPE_GRPH_STEREO) {
-						stereo_in_use = true;
+					if (plane->address.type == PLN_ADDR_TYPE_GRPH_STEREO ||
+							plane->address.tmz_surface) {
+						mall_ss_unsupported = true;
 						break;
 					}
 				}
-				if (stereo_in_use)
+				if (mall_ss_unsupported)
 					break;
 			}
-			if (ways <= dc->caps.cache_num_ways && !stereo_in_use) {
+			if (ways <= dc->caps.cache_num_ways && !mall_ss_unsupported) {
 				memset(&cmd, 0, sizeof(cmd));
 				cmd.cab.header.type = DMUB_CMD__CAB_FOR_SS;
 				cmd.cab.header.sub_type = DMUB_CMD__CAB_DCN_SS_FIT_IN_CAB;
@@ -451,7 +441,6 @@ bool dcn32_apply_idle_power_optimizations(struct dc *dc, bool enable)
  */
 void dcn32_commit_subvp_config(struct dc *dc, struct dc_state *context)
 {
-/*
 	int i;
 	bool enable_subvp = false;
 
@@ -469,7 +458,6 @@ void dcn32_commit_subvp_config(struct dc *dc, struct dc_state *context)
 		}
 	}
 	dc_dmub_setup_subvp_dmub_command(dc, context, enable_subvp);
-*/
 }
 
 /* Sub-Viewport DMUB lock needs to be acquired by driver whenever SubVP is active and:
@@ -675,9 +663,9 @@ bool dcn32_set_output_transfer_func(struct dc *dc,
 					stream->out_transfer_func,
 					&mpc->blender_params, false))
 				params = &mpc->blender_params;
-		 /* there are no ROM LUTs in OUTGAM */
-		if (stream->out_transfer_func->type == TF_TYPE_PREDEFINED)
-			BREAK_TO_DEBUGGER();
+			/* there are no ROM LUTs in OUTGAM */
+			if (stream->out_transfer_func->type == TF_TYPE_PREDEFINED)
+				BREAK_TO_DEBUGGER();
 		}
 	}
 
@@ -773,7 +761,8 @@ void dcn32_update_mall_sel(struct dc *dc, struct dc_state *context)
 				hubp->funcs->hubp_update_mall_sel(hubp,
 					num_ways <= dc->caps.cache_num_ways &&
 					pipe->stream->link->psr_settings.psr_version == DC_PSR_VERSION_UNSUPPORTED &&
-					pipe->plane_state->address.type !=  PLN_ADDR_TYPE_GRPH_STEREO ? 2 : 0,
+					pipe->plane_state->address.type !=  PLN_ADDR_TYPE_GRPH_STEREO &&
+					!pipe->plane_state->address.tmz_surface ? 2 : 0,
 							cache_cursor);
 			}
 		}
@@ -883,6 +872,7 @@ void dcn32_init_hw(struct dc *dc)
 		if (link->link_enc->funcs->is_dig_enabled &&
 			link->link_enc->funcs->is_dig_enabled(link->link_enc)) {
 			link->link_status.link_active = true;
+			link->phy_state.symclk_state = SYMCLK_ON_TX_ON;
 			if (link->link_enc->funcs->fec_is_active &&
 					link->link_enc->funcs->fec_is_active(link->link_enc))
 				link->fec_state = dc_link_fec_enabled;
@@ -1181,6 +1171,9 @@ unsigned int dcn32_calculate_dccg_k1_k2_values(struct pipe_ctx *pipe_ctx, unsign
 	two_pix_per_container = optc2_is_two_pixels_per_containter(&stream->timing);
 	odm_combine_factor = get_odm_config(pipe_ctx, NULL);
 
+	if (pipe_ctx->stream->signal == SIGNAL_TYPE_VIRTUAL)
+		return odm_combine_factor;
+
 	if (is_dp_128b_132b_signal(pipe_ctx)) {
 		*k2_div = PIXEL_RATE_DIV_BY_1;
 	} else if (dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal) || dc_is_dvi_signal(pipe_ctx->stream->signal)) {
@@ -1274,3 +1267,155 @@ bool dcn32_is_dp_dig_pixel_rate_div_policy(struct pipe_ctx *pipe_ctx)
 		return true;
 	return false;
 }
+
+static void apply_symclk_on_tx_off_wa(struct dc_link *link)
+{
+	/* There are use cases where SYMCLK is referenced by OTG. For instance
+	 * for TMDS signal, OTG relies SYMCLK even if TX video output is off.
+	 * However current link interface will power off PHY when disabling link
+	 * output. This will turn off SYMCLK generated by PHY. The workaround is
+	 * to identify such case where SYMCLK is still in use by OTG when we
+	 * power off PHY. When this is detected, we will temporarily power PHY
+	 * back on and move PHY's SYMCLK state to SYMCLK_ON_TX_OFF by calling
+	 * program_pix_clk interface. When OTG is disabled, we will then power
+	 * off PHY by calling disable link output again.
+	 *
+	 * In future dcn generations, we plan to rework transmitter control
+	 * interface so that we could have an option to set SYMCLK ON TX OFF
+	 * state in one step without this workaround
+	 */
+
+	struct dc *dc = link->ctx->dc;
+	struct pipe_ctx *pipe_ctx = NULL;
+	uint8_t i;
+
+	if (link->phy_state.symclk_ref_cnts.otg > 0) {
+		for (i = 0; i < MAX_PIPES; i++) {
+			pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
+			if (pipe_ctx->stream && pipe_ctx->stream->link == link && pipe_ctx->top_pipe == NULL) {
+				pipe_ctx->clock_source->funcs->program_pix_clk(
+						pipe_ctx->clock_source,
+						&pipe_ctx->stream_res.pix_clk_params,
+						dp_get_link_encoding_format(&pipe_ctx->link_config.dp_link_settings),
+						&pipe_ctx->pll_settings);
+				link->phy_state.symclk_state = SYMCLK_ON_TX_OFF;
+				break;
+			}
+		}
+	}
+}
+
+void dcn32_disable_link_output(struct dc_link *link,
+		const struct link_resource *link_res,
+		enum signal_type signal)
+{
+	struct dc *dc = link->ctx->dc;
+	const struct link_hwss *link_hwss = get_link_hwss(link, link_res);
+	struct dmcu *dmcu = dc->res_pool->dmcu;
+
+	if (signal == SIGNAL_TYPE_EDP &&
+			link->dc->hwss.edp_backlight_control)
+		link->dc->hwss.edp_backlight_control(link, false);
+	else if (dmcu != NULL && dmcu->funcs->lock_phy)
+		dmcu->funcs->lock_phy(dmcu);
+
+	link_hwss->disable_link_output(link, link_res, signal);
+	link->phy_state.symclk_state = SYMCLK_OFF_TX_OFF;
+
+	if (signal == SIGNAL_TYPE_EDP &&
+			link->dc->hwss.edp_backlight_control)
+		link->dc->hwss.edp_power_control(link, false);
+	else if (dmcu != NULL && dmcu->funcs->lock_phy)
+		dmcu->funcs->unlock_phy(dmcu);
+
+	dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_DISABLE_LINK_PHY);
+
+	apply_symclk_on_tx_off_wa(link);
+}
+
+/* For SubVP the main pipe can have a viewport position change
+ * without a full update. In this case we must also update the
+ * viewport positions for the phantom pipe accordingly.
+ */
+void dcn32_update_phantom_vp_position(struct dc *dc,
+		struct dc_state *context,
+		struct pipe_ctx *phantom_pipe)
+{
+	uint32_t i;
+	struct dc_plane_state *phantom_plane = phantom_pipe->plane_state;
+
+	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+
+		if (pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_MAIN &&
+				pipe->stream->mall_stream_config.paired_stream == phantom_pipe->stream) {
+			if (pipe->plane_state && pipe->plane_state->update_flags.bits.position_change) {
+
+				phantom_plane->src_rect.x = pipe->plane_state->src_rect.x;
+				phantom_plane->src_rect.y = pipe->plane_state->src_rect.y;
+				phantom_plane->clip_rect.x = pipe->plane_state->clip_rect.x;
+				phantom_plane->dst_rect.x = pipe->plane_state->dst_rect.x;
+				phantom_plane->dst_rect.y = pipe->plane_state->dst_rect.y;
+
+				phantom_pipe->plane_state->update_flags.bits.position_change = 1;
+				resource_build_scaling_params(phantom_pipe);
+				return;
+			}
+		}
+	}
+}
+
+bool dcn32_dsc_pg_status(
+		struct dce_hwseq *hws,
+		unsigned int dsc_inst)
+{
+	uint32_t pwr_status = 0;
+
+	switch (dsc_inst) {
+	case 0: /* DSC0 */
+		REG_GET(DOMAIN16_PG_STATUS,
+				DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	case 1: /* DSC1 */
+
+		REG_GET(DOMAIN17_PG_STATUS,
+				DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	case 2: /* DSC2 */
+		REG_GET(DOMAIN18_PG_STATUS,
+				DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	case 3: /* DSC3 */
+		REG_GET(DOMAIN19_PG_STATUS,
+				DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
+		break;
+	default:
+		BREAK_TO_DEBUGGER();
+		break;
+	}
+
+	return pwr_status == 0 ? true : false;
+}
+
+void dcn32_update_dsc_pg(struct dc *dc,
+		struct dc_state *context,
+		bool safe_to_disable)
+{
+	struct dce_hwseq *hws = dc->hwseq;
+	int i;
+
+	for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++) {
+		struct display_stream_compressor *dsc = dc->res_pool->dscs[i];
+		bool is_dsc_ungated = hws->funcs.dsc_pg_status(hws, dsc->inst);
+
+		if (context->res_ctx.is_dsc_acquired[i]) {
+			if (!is_dsc_ungated) {
+				hws->funcs.dsc_pg_control(hws, dsc->inst, true);
+			}
+		} else if (safe_to_disable) {
+			if (is_dsc_ungated) {
+				hws->funcs.dsc_pg_control(hws, dsc->inst, false);
+			}
+		}
+	}
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h
index 083f3aeb54f0..ac3657a5b9ea 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_hwseq.h
@@ -84,4 +84,20 @@ void dcn32_unblank_stream(struct pipe_ctx *pipe_ctx,
 
 bool dcn32_is_dp_dig_pixel_rate_div_policy(struct pipe_ctx *pipe_ctx);
 
+void dcn32_disable_link_output(struct dc_link *link,
+		const struct link_resource *link_res,
+		enum signal_type signal);
+
+void dcn32_update_phantom_vp_position(struct dc *dc,
+		struct dc_state *context,
+		struct pipe_ctx *phantom_pipe);
+
+bool dcn32_dsc_pg_status(
+		struct dce_hwseq *hws,
+		unsigned int dsc_inst);
+
+void dcn32_update_dsc_pg(struct dc *dc,
+		struct dc_state *context,
+		bool safe_to_disable);
+
 #endif /* __DC_HWSS_DCN32_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c
index c279a25ea293..45a949ba6f3f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_init.c
@@ -99,11 +99,17 @@ static const struct hw_sequencer_funcs dcn32_funcs = {
 	.set_abm_immediate_disable = dcn21_set_abm_immediate_disable,
 	.hardware_release = dcn30_hardware_release,
 	.set_pipe = dcn21_set_pipe,
+	.enable_lvds_link_output = dce110_enable_lvds_link_output,
+	.enable_tmds_link_output = dce110_enable_tmds_link_output,
+	.enable_dp_link_output = dce110_enable_dp_link_output,
+	.disable_link_output = dcn32_disable_link_output,
 	.set_disp_pattern_generator = dcn30_set_disp_pattern_generator,
 	.get_dcc_en_bits = dcn10_get_dcc_en_bits,
 	.commit_subvp_config = dcn32_commit_subvp_config,
 	.subvp_pipe_control_lock = dcn32_subvp_pipe_control_lock,
 	.update_visual_confirm_color = dcn20_update_visual_confirm_color,
+	.update_phantom_vp_position = dcn32_update_phantom_vp_position,
+	.update_dsc_pg = dcn32_update_dsc_pg,
 };
 
 static const struct hwseq_private_funcs dcn32_private_funcs = {
@@ -133,6 +139,7 @@ static const struct hwseq_private_funcs dcn32_private_funcs = {
 	.program_all_writeback_pipes_in_tree = dcn30_program_all_writeback_pipes_in_tree,
 	.update_odm = dcn32_update_odm,
 	.dsc_pg_control = dcn32_dsc_pg_control,
+	.dsc_pg_status = dcn32_dsc_pg_status,
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
 	.verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high,
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c
index adf93cc8359c..41b0baf8e183 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.c
@@ -100,7 +100,7 @@ static void mmhubbub32_warmup_mcif(struct mcif_wb *mcif_wb,
 	REG_UPDATE(MMHUBBUB_WARMUP_CONTROL_STATUS, MMHUBBUB_WARMUP_EN, false);
 }
 
-void mmhubbub32_config_mcif_buf(struct mcif_wb *mcif_wb,
+static void mmhubbub32_config_mcif_buf(struct mcif_wb *mcif_wb,
 		struct mcif_buf_params *params,
 		unsigned int dest_height)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.h
index 22355051f5f7..e460cf8d9041 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mmhubbub.h
@@ -90,7 +90,6 @@
 	SF(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_OVERRUN_INT_EN, mask_sh),\
 	SF(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUFMGR_SW_LOCK, mask_sh),\
 	SF(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB_BUF_ADDR_FENCE_EN, mask_sh),\
-	SF(MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_VCE_INT_STATUS, mask_sh),\
 	SF(MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_SW_INT_STATUS, mask_sh),\
 	SF(MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_SW_OVERRUN_INT_STATUS, mask_sh),\
 	SF(MCIF_WB_BUFMGR_STATUS, MCIF_WB_BUFMGR_CUR_BUF, mask_sh),\
@@ -101,7 +100,6 @@
 	SF(MCIF_WB_BUF_PITCH, MCIF_WB_BUF_CHROMA_PITCH, mask_sh),\
 	SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_ACTIVE, mask_sh),\
 	SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_SW_LOCKED, mask_sh),\
-	SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_VCE_LOCKED, mask_sh),\
 	SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_OVERFLOW, mask_sh),\
 	SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_DISABLE, mask_sh),\
 	SF(MCIF_WB_BUF_1_STATUS, MCIF_WB_BUF_1_MODE, mask_sh),\
@@ -116,7 +114,6 @@
 	SF(MCIF_WB_BUF_1_STATUS2, MCIF_WB_BUF_1_C_OVERRUN, mask_sh),\
 	SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_ACTIVE, mask_sh),\
 	SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_SW_LOCKED, mask_sh),\
-	SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_VCE_LOCKED, mask_sh),\
 	SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_OVERFLOW, mask_sh),\
 	SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_DISABLE, mask_sh),\
 	SF(MCIF_WB_BUF_2_STATUS, MCIF_WB_BUF_2_MODE, mask_sh),\
@@ -131,7 +128,6 @@
 	SF(MCIF_WB_BUF_2_STATUS2, MCIF_WB_BUF_2_C_OVERRUN, mask_sh),\
 	SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_ACTIVE, mask_sh),\
 	SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_SW_LOCKED, mask_sh),\
-	SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_VCE_LOCKED, mask_sh),\
 	SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_OVERFLOW, mask_sh),\
 	SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_DISABLE, mask_sh),\
 	SF(MCIF_WB_BUF_3_STATUS, MCIF_WB_BUF_3_MODE, mask_sh),\
@@ -146,7 +142,6 @@
 	SF(MCIF_WB_BUF_3_STATUS2, MCIF_WB_BUF_3_C_OVERRUN, mask_sh),\
 	SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_ACTIVE, mask_sh),\
 	SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_SW_LOCKED, mask_sh),\
-	SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_VCE_LOCKED, mask_sh),\
 	SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_OVERFLOW, mask_sh),\
 	SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_DISABLE, mask_sh),\
 	SF(MCIF_WB_BUF_4_STATUS, MCIF_WB_BUF_4_MODE, mask_sh),\
@@ -172,11 +167,6 @@
 	SF(MCIF_WB_BUF_3_ADDR_C, MCIF_WB_BUF_3_ADDR_C, mask_sh),\
 	SF(MCIF_WB_BUF_4_ADDR_Y, MCIF_WB_BUF_4_ADDR_Y, mask_sh),\
 	SF(MCIF_WB_BUF_4_ADDR_C, MCIF_WB_BUF_4_ADDR_C, mask_sh),\
-	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK_IGNORE, mask_sh),\
-	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_EN, mask_sh),\
-	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_INT_ACK, mask_sh),\
-	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_SLICE_INT_EN, mask_sh),\
-	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_VCE_LOCK, mask_sh),\
 	SF(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB_BUFMGR_SLICE_SIZE, mask_sh),\
 	SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_REFRESH_WATERMARK, mask_sh),\
 	SF(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, NB_PSTATE_CHANGE_WATERMARK_MASK, mask_sh),\
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c
index 357bd2461bc9..4edd0655965b 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_mpc.c
@@ -701,7 +701,7 @@ static void mpc32_power_on_shaper_3dlut(
 }
 
 
-bool mpc32_program_shaper(
+static bool mpc32_program_shaper(
 		struct mpc *mpc,
 		const struct pwl_params *params,
 		uint32_t mpcc_id)
@@ -726,7 +726,7 @@ bool mpc32_program_shaper(
 	else
 		next_mode = LUT_RAM_A;
 
-	mpc32_configure_shaper_lut(mpc, next_mode == LUT_RAM_A ? true:false, mpcc_id);
+	mpc32_configure_shaper_lut(mpc, next_mode == LUT_RAM_A, mpcc_id);
 
 	if (next_mode == LUT_RAM_A)
 		mpc32_program_shaper_luta_settings(mpc, params, mpcc_id);
@@ -897,7 +897,7 @@ static void mpc32_set_3dlut_mode(
 }
 
 
-bool mpc32_program_3dlut(
+static bool mpc32_program_3dlut(
 		struct mpc *mpc,
 		const struct tetrahedral_params *params,
 		int mpcc_id)
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c
index 1fad7b48bd5b..ec3989d37086 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_optc.c
@@ -156,7 +156,7 @@ static bool optc32_disable_crtc(struct timing_generator *optc)
 	return true;
 }
 
-void optc32_phantom_crtc_post_enable(struct timing_generator *optc)
+static void optc32_phantom_crtc_post_enable(struct timing_generator *optc)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 
@@ -190,7 +190,7 @@ static void optc32_set_odm_bypass(struct timing_generator *optc,
 	optc1->opp_count = 1;
 }
 
-void optc32_setup_manual_trigger(struct timing_generator *optc)
+static void optc32_setup_manual_trigger(struct timing_generator *optc)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(optc);
 	struct dc *dc = optc->ctx->dc;
@@ -215,7 +215,7 @@ void optc32_setup_manual_trigger(struct timing_generator *optc)
 	}
 }
 
-void optc32_set_drr(
+static void optc32_set_drr(
 	struct timing_generator *optc,
 	const struct drr_params *params)
 {
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c
index c3b783cea8a0..05de97ea855f 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.c
@@ -90,29 +90,6 @@
 #include "dcn20/dcn20_vmid.h"
 #include "dml/dcn32/dcn32_fpu.h"
 
-#define DCN_BASE__INST0_SEG1                       0x000000C0
-#define DCN_BASE__INST0_SEG2                       0x000034C0
-#define DCN_BASE__INST0_SEG3                       0x00009000
-#define NBIO_BASE__INST0_SEG1                      0x00000014
-
-#define MAX_INSTANCE                                        6
-#define MAX_SEGMENT                                         6
-
-struct IP_BASE_INSTANCE {
-	unsigned int segment[MAX_SEGMENT];
-};
-
-struct IP_BASE {
-	struct IP_BASE_INSTANCE instance[MAX_INSTANCE];
-};
-
-static const struct IP_BASE DCN_BASE = { { { { 0x00000012, 0x000000C0, 0x000034C0, 0x00009000, 0x02403C00, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } } } };
-
 #define DC_LOGGER_INIT(logger)
 
 enum dcn32_clk_src_array_id {
@@ -131,79 +108,103 @@ enum dcn32_clk_src_array_id {
 /* DCN */
 /* TODO awful hack. fixup dcn20_dwb.h */
 #undef BASE_INNER
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
 
 #define BASE(seg) BASE_INNER(seg)
 
 #define SR(reg_name)\
-		.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+		REG_STRUCT.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
 					reg ## reg_name
+#define SR_ARR(reg_name, id) \
+	REG_STRUCT[id].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
+
+#define SR_ARR_INIT(reg_name, id, value) \
+	REG_STRUCT[id].reg_name = value
 
 #define SRI(reg_name, block, id)\
-	.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
+	REG_STRUCT.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI_ARR(reg_name, block, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SR_ARR_I2C(reg_name, id) \
+	REG_STRUCT[id-1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
+
+#define SRI_ARR_I2C(reg_name, block, id)\
+	REG_STRUCT[id-1].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI_ARR_ALPHABET(reg_name, block, index, id)\
+	REG_STRUCT[index].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
 
 #define SRI2(reg_name, block, id)\
-	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
-					reg ## reg_name
+	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +	\
+		reg ## reg_name
+#define SRI2_ARR(reg_name, block, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) +	\
+		reg ## reg_name
 
 #define SRIR(var_name, reg_name, block, id)\
 	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
+		reg ## block ## id ## _ ## reg_name
 
 #define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
 					reg ## block ## id ## _ ## reg_name
 
+#define SRII_ARR_2(reg_name, block, id, inst)\
+	REG_STRUCT[inst].reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
 #define SRII_MPC_RMU(reg_name, block, id)\
 	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
+		reg ## block ## id ## _ ## reg_name
 
 #define SRII_DWB(reg_name, temp_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## temp_name
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## temp_name
 
 #define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
+	REG_STRUCT.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
 
 #define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					reg ## reg_name ## _ ## block ## id
+	REG_STRUCT.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+		reg ## reg_name ## _ ## block ## id
 
 /* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIO_BASE__INST0_SEG ## seg
+#define NBIO_BASE_INNER(seg) ctx->nbio_reg_offsets[seg]
 
 #define NBIO_BASE(seg) \
 	NBIO_BASE_INNER(seg)
 
 #define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
-					regBIF_BX0_ ## reg_name
+	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
+			regBIF_BX0_ ## reg_name
+#define NBIO_SR_ARR(reg_name, id)\
+	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
+		regBIF_BX0_ ## reg_name
 
 #undef CTX
 #define CTX ctx
 #define REG(reg_name) \
-	(DCN_BASE.instance[0].segment[reg ## reg_name ## _BASE_IDX] + reg ## reg_name)
+	(ctx->dcn_reg_offsets[reg ## reg_name ## _BASE_IDX] + reg ## reg_name)
 
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
+static struct bios_registers bios_regs;
 
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN3_0(index, pllid),\
-}
+#define bios_regs_init() \
+		( \
+		NBIO_SR(BIOS_SCRATCH_3),\
+		NBIO_SR(BIOS_SCRATCH_6)\
+		)
 
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E)
-};
+#define clk_src_regs_init(index, pllid)\
+	CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)
+
+static struct dce110_clk_src_regs clk_src_regs[5];
 
 static const struct dce110_clk_src_shift cs_shift = {
 		CS_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
@@ -213,17 +214,10 @@ static const struct dce110_clk_src_mask cs_mask = {
 		CS_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
 };
 
-#define abm_regs(id)\
-[id] = {\
-		ABM_DCN32_REG_LIST(id)\
-}
+#define abm_regs_init(id)\
+		ABM_DCN32_REG_LIST_RI(id)
 
-static const struct dce_abm_registers abm_regs[] = {
-		abm_regs(0),
-		abm_regs(1),
-		abm_regs(2),
-		abm_regs(3),
-};
+static struct dce_abm_registers abm_regs[4];
 
 static const struct dce_abm_shift abm_shift = {
 		ABM_MASK_SH_LIST_DCN32(__SHIFT)
@@ -233,18 +227,10 @@ static const struct dce_abm_mask abm_mask = {
 		ABM_MASK_SH_LIST_DCN32(_MASK)
 };
 
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
+#define audio_regs_init(id)\
+		AUD_COMMON_REG_LIST_RI(id)
 
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4)
-};
+static struct dce_audio_registers audio_regs[5];
 
 #define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
 		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
@@ -259,23 +245,10 @@ static const struct dce_audio_mask audio_mask = {
 		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
 };
 
-#define vpg_regs(id)\
-[id] = {\
-	VPG_DCN3_REG_LIST(id)\
-}
+#define vpg_regs_init(id)\
+	VPG_DCN3_REG_LIST_RI(id)
 
-static const struct dcn30_vpg_registers vpg_regs[] = {
-	vpg_regs(0),
-	vpg_regs(1),
-	vpg_regs(2),
-	vpg_regs(3),
-	vpg_regs(4),
-	vpg_regs(5),
-	vpg_regs(6),
-	vpg_regs(7),
-	vpg_regs(8),
-	vpg_regs(9),
-};
+static struct dcn30_vpg_registers vpg_regs[10];
 
 static const struct dcn30_vpg_shift vpg_shift = {
 	DCN3_VPG_MASK_SH_LIST(__SHIFT)
@@ -285,19 +258,10 @@ static const struct dcn30_vpg_mask vpg_mask = {
 	DCN3_VPG_MASK_SH_LIST(_MASK)
 };
 
-#define afmt_regs(id)\
-[id] = {\
-	AFMT_DCN3_REG_LIST(id)\
-}
+#define afmt_regs_init(id)\
+	AFMT_DCN3_REG_LIST_RI(id)
 
-static const struct dcn30_afmt_registers afmt_regs[] = {
-	afmt_regs(0),
-	afmt_regs(1),
-	afmt_regs(2),
-	afmt_regs(3),
-	afmt_regs(4),
-	afmt_regs(5)
-};
+static struct dcn30_afmt_registers afmt_regs[6];
 
 static const struct dcn30_afmt_shift afmt_shift = {
 	DCN3_AFMT_MASK_SH_LIST(__SHIFT)
@@ -307,17 +271,10 @@ static const struct dcn30_afmt_mask afmt_mask = {
 	DCN3_AFMT_MASK_SH_LIST(_MASK)
 };
 
-#define apg_regs(id)\
-[id] = {\
-	APG_DCN31_REG_LIST(id)\
-}
+#define apg_regs_init(id)\
+	APG_DCN31_REG_LIST_RI(id)
 
-static const struct dcn31_apg_registers apg_regs[] = {
-	apg_regs(0),
-	apg_regs(1),
-	apg_regs(2),
-	apg_regs(3)
-};
+static struct dcn31_apg_registers apg_regs[4];
 
 static const struct dcn31_apg_shift apg_shift = {
 	DCN31_APG_MASK_SH_LIST(__SHIFT)
@@ -327,18 +284,10 @@ static const struct dcn31_apg_mask apg_mask = {
 		DCN31_APG_MASK_SH_LIST(_MASK)
 };
 
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN32_REG_LIST(id)\
-}
+#define stream_enc_regs_init(id)\
+	SE_DCN32_REG_LIST_RI(id)
 
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4)
-};
+static struct dcn10_stream_enc_registers stream_enc_regs[5];
 
 static const struct dcn10_stream_encoder_shift se_shift = {
 		SE_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
@@ -349,46 +298,24 @@ static const struct dcn10_stream_encoder_mask se_mask = {
 };
 
 
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
+#define aux_regs_init(id)\
+	DCN2_AUX_REG_LIST_RI(id)
 
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4)
-};
+static struct dcn10_link_enc_aux_registers link_enc_aux_regs[5];
 
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
+#define hpd_regs_init(id)\
+	HPD_REG_LIST_RI(id)
 
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4)
-};
+static struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[5];
 
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN31_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid), \
+#define link_regs_init(id, phyid)\
+	( \
+	LE_DCN31_REG_LIST_RI(id), \
+	UNIPHY_DCN2_REG_LIST_RI(id, phyid)\
+	)
 	/*DPCS_DCN31_REG_LIST(id),*/ \
-}
 
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-	link_regs(2, C),
-	link_regs(3, D),
-	link_regs(4, E)
-};
+static struct dcn10_link_enc_registers link_enc_regs[5];
 
 static const struct dcn10_link_enc_shift le_shift = {
 	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
@@ -401,17 +328,10 @@ static const struct dcn10_link_enc_mask le_mask = {
 	//DPCS_DCN31_MASK_SH_LIST(_MASK)
 };
 
-#define hpo_dp_stream_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST(id)\
-}
+#define hpo_dp_stream_encoder_reg_init(id)\
+	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
 
-static const struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[] = {
-	hpo_dp_stream_encoder_reg_list(0),
-	hpo_dp_stream_encoder_reg_list(1),
-	hpo_dp_stream_encoder_reg_list(2),
-	hpo_dp_stream_encoder_reg_list(3),
-};
+static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
 
 static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
 	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
@@ -422,20 +342,14 @@ static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
 };
 
 
-#define hpo_dp_link_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_LINK_ENC_REG_LIST(id),\
-	/*DCN3_1_RDPCSTX_REG_LIST(0),*/\
-	/*DCN3_1_RDPCSTX_REG_LIST(1),*/\
-	/*DCN3_1_RDPCSTX_REG_LIST(2),*/\
-	/*DCN3_1_RDPCSTX_REG_LIST(3),*/\
-	/*DCN3_1_RDPCSTX_REG_LIST(4)*/\
-}
+#define hpo_dp_link_encoder_reg_init(id)\
+	DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)
+	/*DCN3_1_RDPCSTX_REG_LIST(0),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(1),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(2),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(3),*/
 
-static const struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[] = {
-	hpo_dp_link_encoder_reg_list(0),
-	hpo_dp_link_encoder_reg_list(1),
-};
+static struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[2];
 
 static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
 	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
@@ -445,17 +359,10 @@ static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
 	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
 };
 
-#define dpp_regs(id)\
-[id] = {\
-	DPP_REG_LIST_DCN30_COMMON(id),\
-}
+#define dpp_regs_init(id)\
+	DPP_REG_LIST_DCN30_COMMON_RI(id)
 
-static const struct dcn3_dpp_registers dpp_regs[] = {
-	dpp_regs(0),
-	dpp_regs(1),
-	dpp_regs(2),
-	dpp_regs(3)
-};
+static struct dcn3_dpp_registers dpp_regs[4];
 
 static const struct dcn3_dpp_shift tf_shift = {
 		DPP_REG_LIST_SH_MASK_DCN30_COMMON(__SHIFT)
@@ -466,17 +373,10 @@ static const struct dcn3_dpp_mask tf_mask = {
 };
 
 
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN30(id),\
-}
+#define opp_regs_init(id)\
+	OPP_REG_LIST_DCN30_RI(id)
 
-static const struct dcn20_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3)
-};
+static struct dcn20_opp_registers opp_regs[4];
 
 static const struct dcn20_opp_shift opp_shift = {
 	OPP_MASK_SH_LIST_DCN20(__SHIFT)
@@ -486,21 +386,16 @@ static const struct dcn20_opp_mask opp_mask = {
 	OPP_MASK_SH_LIST_DCN20(_MASK)
 };
 
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUXN_IMPCAL = 0, \
-	.AUXP_IMPCAL = 0, \
-	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
+#define aux_engine_regs_init(id)\
+	( \
+	AUX_COMMON_REG_LIST0_RI(id), \
+	SR_ARR_INIT(AUXN_IMPCAL, id, 0), \
+	SR_ARR_INIT(AUXP_IMPCAL, id, 0), \
+	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK), \
+	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK)\
+	)
 
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4)
-};
+static struct dce110_aux_registers aux_engine_regs[5];
 
 static const struct dce110_aux_registers_shift aux_shift = {
 	DCN_AUX_MASK_SH_LIST(__SHIFT)
@@ -510,15 +405,10 @@ static const struct dce110_aux_registers_mask aux_mask = {
 	DCN_AUX_MASK_SH_LIST(_MASK)
 };
 
+#define dwbc_regs_dcn3_init(id)\
+	DWBC_COMMON_REG_LIST_DCN30_RI(id)
 
-#define dwbc_regs_dcn3(id)\
-[id] = {\
-	DWBC_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_dwbc_registers dwbc30_regs[] = {
-	dwbc_regs_dcn3(0),
-};
+static struct dcn30_dwbc_registers dwbc30_regs[1];
 
 static const struct dcn30_dwbc_shift dwbc30_shift = {
 	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
@@ -528,14 +418,10 @@ static const struct dcn30_dwbc_mask dwbc30_mask = {
 	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
 };
 
-#define mcif_wb_regs_dcn3(id)\
-[id] = {\
-	MCIF_WB_COMMON_REG_LIST_DCN32(id),\
-}
+#define mcif_wb_regs_dcn3_init(id)\
+	MCIF_WB_COMMON_REG_LIST_DCN32_RI(id)
 
-static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
-	mcif_wb_regs_dcn3(0)
-};
+static struct dcn30_mmhubbub_registers mcif_wb30_regs[1];
 
 static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
 	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
@@ -545,17 +431,10 @@ static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
 	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(_MASK)
 };
 
-#define dsc_regsDCN20(id)\
-[id] = {\
-	DSC_REG_LIST_DCN20(id)\
-}
+#define dsc_regsDCN20_init(id)\
+	DSC_REG_LIST_DCN20_RI(id)
 
-static const struct dcn20_dsc_registers dsc_regs[] = {
-	dsc_regsDCN20(0),
-	dsc_regsDCN20(1),
-	dsc_regsDCN20(2),
-	dsc_regsDCN20(3)
-};
+static struct dcn20_dsc_registers dsc_regs[4];
 
 static const struct dcn20_dsc_shift dsc_shift = {
 	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
@@ -565,17 +444,18 @@ static const struct dcn20_dsc_mask dsc_mask = {
 	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
 };
 
-static const struct dcn30_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN3_2(0),
-		MPC_REG_LIST_DCN3_2(1),
-		MPC_REG_LIST_DCN3_2(2),
-		MPC_REG_LIST_DCN3_2(3),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
-		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
-};
+static struct dcn30_mpc_registers mpc_regs;
+
+#define dcn_mpc_regs_init() \
+	MPC_REG_LIST_DCN3_2_RI(0),\
+	MPC_REG_LIST_DCN3_2_RI(1),\
+	MPC_REG_LIST_DCN3_2_RI(2),\
+	MPC_REG_LIST_DCN3_2_RI(3),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\
+	MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)
 
 static const struct dcn30_mpc_shift mpc_shift = {
 	MPC_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
@@ -585,19 +465,10 @@ static const struct dcn30_mpc_mask mpc_mask = {
 	MPC_COMMON_MASK_SH_LIST_DCN32(_MASK)
 };
 
-#define optc_regs(id)\
-[id] = {OPTC_COMMON_REG_LIST_DCN3_2(id)}
-
-//#ifdef DIAGS_BUILD
-//static struct dcn_optc_registers optc_regs[] = {
-//#else
-static const struct dcn_optc_registers optc_regs[] = {
-//#endif
-	optc_regs(0),
-	optc_regs(1),
-	optc_regs(2),
-	optc_regs(3)
-};
+#define optc_regs_init(id)\
+	OPTC_COMMON_REG_LIST_DCN3_2_RI(id)
+
+static struct dcn_optc_registers optc_regs[4];
 
 static const struct dcn_optc_shift optc_shift = {
 	OPTC_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
@@ -607,17 +478,10 @@ static const struct dcn_optc_mask optc_mask = {
 	OPTC_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
 };
 
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN32(id)\
-}
+#define hubp_regs_init(id)\
+	HUBP_REG_LIST_DCN32_RI(id)
 
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3)
-};
+static struct dcn_hubp2_registers hubp_regs[4];
 
 
 static const struct dcn_hubp2_shift hubp_shift = {
@@ -627,9 +491,10 @@ static const struct dcn_hubp2_shift hubp_shift = {
 static const struct dcn_hubp2_mask hubp_mask = {
 		HUBP_MASK_SH_LIST_DCN32(_MASK)
 };
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN32(0)
-};
+
+static struct dcn_hubbub_registers hubbub_reg;
+#define hubbub_reg_init()\
+		HUBBUB_REG_LIST_DCN32_RI(0)
 
 static const struct dcn_hubbub_shift hubbub_shift = {
 		HUBBUB_MASK_SH_LIST_DCN32(__SHIFT)
@@ -639,9 +504,10 @@ static const struct dcn_hubbub_mask hubbub_mask = {
 		HUBBUB_MASK_SH_LIST_DCN32(_MASK)
 };
 
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN32()
-};
+static struct dccg_registers dccg_regs;
+
+#define dccg_regs_init()\
+	DCCG_REG_LIST_DCN32_RI()
 
 static const struct dccg_shift dccg_shift = {
 		DCCG_MASK_SH_LIST_DCN32(__SHIFT)
@@ -714,9 +580,10 @@ static const struct dccg_mask dccg_mask = {
 	SR(AZALIA_AUDIO_DTO), \
 	SR(AZALIA_CONTROLLER_CLOCK_GATING)
 
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN32_REG_LIST()
-};
+static struct dce_hwseq_registers hwseq_reg;
+
+#define hwseq_reg_init()\
+	HWSEQ_DCN32_REG_LIST()
 
 #define HWSEQ_DCN32_MASK_SH_LIST(mask_sh)\
 	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
@@ -759,29 +626,10 @@ static const struct dce_hwseq_shift hwseq_shift = {
 static const struct dce_hwseq_mask hwseq_mask = {
 		HWSEQ_DCN32_MASK_SH_LIST(_MASK)
 };
-#define vmid_regs(id)\
-[id] = {\
-		DCN20_VMID_REG_LIST(id)\
-}
+#define vmid_regs_init(id)\
+		DCN20_VMID_REG_LIST_RI(id)
 
-static const struct dcn_vmid_registers vmid_regs[] = {
-	vmid_regs(0),
-	vmid_regs(1),
-	vmid_regs(2),
-	vmid_regs(3),
-	vmid_regs(4),
-	vmid_regs(5),
-	vmid_regs(6),
-	vmid_regs(7),
-	vmid_regs(8),
-	vmid_regs(9),
-	vmid_regs(10),
-	vmid_regs(11),
-	vmid_regs(12),
-	vmid_regs(13),
-	vmid_regs(14),
-	vmid_regs(15)
-};
+static struct dcn_vmid_registers vmid_regs[16];
 
 static const struct dcn20_vmid_shift vmid_shifts = {
 		DCN20_VMID_MASK_SH_LIST(__SHIFT)
@@ -870,8 +718,12 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.force_disable_subvp = false,
 	.exit_idle_opt_for_cursor_updates = true,
 	.enable_single_display_2to1_odm_policy = true,
+
+	/* Must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions*/
+	.enable_double_buffered_dsc_pg_support = true,
 	.enable_dp_dig_pixel_rate_div_policy = 1,
 	.allow_sw_cursor_fallback = false,
+	.alloc_extra_way_for_cursor = true,
 };
 
 static const struct dc_debug_options debug_defaults_diags = {
@@ -904,6 +756,14 @@ static struct dce_aux *dcn32_aux_engine_create(
 	if (!aux_engine)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT aux_engine_regs
+	aux_engine_regs_init(0),
+	aux_engine_regs_init(1),
+	aux_engine_regs_init(2),
+	aux_engine_regs_init(3),
+	aux_engine_regs_init(4);
+
 	dce110_aux_engine_construct(aux_engine, ctx, inst,
 				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
 				    &aux_engine_regs[inst],
@@ -913,15 +773,10 @@ static struct dce_aux *dcn32_aux_engine_create(
 
 	return &aux_engine->base;
 }
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-};
+#define i2c_inst_regs_init(id)\
+	I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dce_i2c_registers i2c_hw_regs[5];
 
 static const struct dce_i2c_shift i2c_shifts = {
 		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
@@ -941,6 +796,14 @@ static struct dce_i2c_hw *dcn32_i2c_hw_create(
 	if (!dce_i2c_hw)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT i2c_hw_regs
+	i2c_inst_regs_init(1),
+	i2c_inst_regs_init(2),
+	i2c_inst_regs_init(3),
+	i2c_inst_regs_init(4),
+	i2c_inst_regs_init(5);
+
 	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
 				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
 
@@ -980,6 +843,29 @@ static struct hubbub *dcn32_hubbub_create(struct dc_context *ctx)
 	if (!hubbub2)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT hubbub_reg
+	hubbub_reg_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT vmid_regs
+	vmid_regs_init(0),
+	vmid_regs_init(1),
+	vmid_regs_init(2),
+	vmid_regs_init(3),
+	vmid_regs_init(4),
+	vmid_regs_init(5),
+	vmid_regs_init(6),
+	vmid_regs_init(7),
+	vmid_regs_init(8),
+	vmid_regs_init(9),
+	vmid_regs_init(10),
+	vmid_regs_init(11),
+	vmid_regs_init(12),
+	vmid_regs_init(13),
+	vmid_regs_init(14),
+	vmid_regs_init(15);
+
 	hubbub32_construct(hubbub2, ctx,
 			&hubbub_reg,
 			&hubbub_shift,
@@ -1012,6 +898,13 @@ static struct hubp *dcn32_hubp_create(
 	if (!hubp2)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT hubp_regs
+	hubp_regs_init(0),
+	hubp_regs_init(1),
+	hubp_regs_init(2),
+	hubp_regs_init(3);
+
 	if (hubp32_construct(hubp2, ctx, inst,
 			&hubp_regs[inst], &hubp_shift, &hubp_mask))
 		return &hubp2->base;
@@ -1037,6 +930,13 @@ static struct dpp *dcn32_dpp_create(
 	if (!dpp3)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT dpp_regs
+	dpp_regs_init(0),
+	dpp_regs_init(1),
+	dpp_regs_init(2),
+	dpp_regs_init(3);
+
 	if (dpp32_construct(dpp3, ctx, inst,
 			&dpp_regs[inst], &tf_shift, &tf_mask))
 		return &dpp3->base;
@@ -1057,6 +957,10 @@ static struct mpc *dcn32_mpc_create(
 	if (!mpc30)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT mpc_regs
+	dcn_mpc_regs_init();
+
 	dcn32_mpc_construct(mpc30, ctx,
 			&mpc_regs,
 			&mpc_shift,
@@ -1078,6 +982,13 @@ static struct output_pixel_processor *dcn32_opp_create(
 		return NULL;
 	}
 
+#undef REG_STRUCT
+#define REG_STRUCT opp_regs
+	opp_regs_init(0),
+	opp_regs_init(1),
+	opp_regs_init(2),
+	opp_regs_init(3);
+
 	dcn20_opp_construct(opp2, ctx, inst,
 			&opp_regs[inst], &opp_shift, &opp_mask);
 	return &opp2->base;
@@ -1094,6 +1005,13 @@ static struct timing_generator *dcn32_timing_generator_create(
 	if (!tgn10)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT optc_regs
+	optc_regs_init(0),
+	optc_regs_init(1),
+	optc_regs_init(2),
+	optc_regs_init(3);
+
 	tgn10->base.inst = instance;
 	tgn10->base.ctx = ctx;
 
@@ -1128,6 +1046,30 @@ static struct link_encoder *dcn32_link_encoder_create(
 	if (!enc20)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_aux_regs
+	aux_regs_init(0),
+	aux_regs_init(1),
+	aux_regs_init(2),
+	aux_regs_init(3),
+	aux_regs_init(4);
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_hpd_regs
+	hpd_regs_init(0),
+	hpd_regs_init(1),
+	hpd_regs_init(2),
+	hpd_regs_init(3),
+	hpd_regs_init(4);
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_regs
+	link_regs_init(0, A),
+	link_regs_init(1, B),
+	link_regs_init(2, C),
+	link_regs_init(3, D),
+	link_regs_init(4, E);
+
 	dcn32_link_encoder_construct(enc20,
 			enc_init_data,
 			&link_enc_feature,
@@ -1157,7 +1099,7 @@ static void read_dce_straps(
 	struct dc_context *ctx,
 	struct resource_straps *straps)
 {
-	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
+	generic_reg_get(ctx, ctx->dcn_reg_offsets[regDC_PINSTRAPS_BASE_IDX] + regDC_PINSTRAPS,
 		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
 
 }
@@ -1165,6 +1107,15 @@ static void read_dce_straps(
 static struct audio *dcn32_create_audio(
 		struct dc_context *ctx, unsigned int inst)
 {
+
+#undef REG_STRUCT
+#define REG_STRUCT audio_regs
+	audio_regs_init(0),
+	audio_regs_init(1),
+	audio_regs_init(2),
+	audio_regs_init(3),
+	audio_regs_init(4);
+
 	return dce_audio_create(ctx, inst,
 			&audio_regs[inst], &audio_shift, &audio_mask);
 }
@@ -1178,6 +1129,19 @@ static struct vpg *dcn32_vpg_create(
 	if (!vpg3)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT vpg_regs
+	vpg_regs_init(0),
+	vpg_regs_init(1),
+	vpg_regs_init(2),
+	vpg_regs_init(3),
+	vpg_regs_init(4),
+	vpg_regs_init(5),
+	vpg_regs_init(6),
+	vpg_regs_init(7),
+	vpg_regs_init(8),
+	vpg_regs_init(9);
+
 	vpg3_construct(vpg3, ctx, inst,
 			&vpg_regs[inst],
 			&vpg_shift,
@@ -1195,6 +1159,15 @@ static struct afmt *dcn32_afmt_create(
 	if (!afmt3)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT afmt_regs
+	afmt_regs_init(0),
+	afmt_regs_init(1),
+	afmt_regs_init(2),
+	afmt_regs_init(3),
+	afmt_regs_init(4),
+	afmt_regs_init(5);
+
 	afmt3_construct(afmt3, ctx, inst,
 			&afmt_regs[inst],
 			&afmt_shift,
@@ -1212,6 +1185,13 @@ static struct apg *dcn31_apg_create(
 	if (!apg31)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT apg_regs
+	apg_regs_init(0),
+	apg_regs_init(1),
+	apg_regs_init(2),
+	apg_regs_init(3);
+
 	apg31_construct(apg31, ctx, inst,
 			&apg_regs[inst],
 			&apg_shift,
@@ -1248,6 +1228,14 @@ static struct stream_encoder *dcn32_stream_encoder_create(
 		return NULL;
 	}
 
+#undef REG_STRUCT
+#define REG_STRUCT stream_enc_regs
+	stream_enc_regs_init(0),
+	stream_enc_regs_init(1),
+	stream_enc_regs_init(2),
+	stream_enc_regs_init(3),
+	stream_enc_regs_init(4);
+
 	dcn32_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
 					eng_id, vpg, afmt,
 					&stream_enc_regs[eng_id],
@@ -1298,6 +1286,13 @@ static struct hpo_dp_stream_encoder *dcn32_hpo_dp_stream_encoder_create(
 		return NULL;
 	}
 
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_stream_enc_regs
+	hpo_dp_stream_encoder_reg_init(0),
+	hpo_dp_stream_encoder_reg_init(1),
+	hpo_dp_stream_encoder_reg_init(2),
+	hpo_dp_stream_encoder_reg_init(3);
+
 	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
 					hpo_dp_inst, eng_id, vpg, apg,
 					&hpo_dp_stream_enc_regs[hpo_dp_inst],
@@ -1315,6 +1310,11 @@ static struct hpo_dp_link_encoder *dcn32_hpo_dp_link_encoder_create(
 	/* allocate HPO link encoder */
 	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
 
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_link_enc_regs
+	hpo_dp_link_encoder_reg_init(0),
+	hpo_dp_link_encoder_reg_init(1);
+
 	hpo_dp_link_encoder32_construct(hpo_dp_enc31, ctx, inst,
 					&hpo_dp_link_enc_regs[inst],
 					&hpo_dp_le_shift, &hpo_dp_le_mask);
@@ -1327,6 +1327,10 @@ static struct dce_hwseq *dcn32_hwseq_create(
 {
 	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
 
+#undef REG_STRUCT
+#define REG_STRUCT hwseq_reg
+	hwseq_reg_init();
+
 	if (hws) {
 		hws->ctx = ctx;
 		hws->regs = &hwseq_reg;
@@ -1518,6 +1522,10 @@ static bool dcn32_dwbc_create(struct dc_context *ctx, struct resource_pool *pool
 			return false;
 		}
 
+#undef REG_STRUCT
+#define REG_STRUCT dwbc30_regs
+		dwbc_regs_dcn3_init(0);
+
 		dcn30_dwbc_construct(dwbc30, ctx,
 				&dwbc30_regs[i],
 				&dwbc30_shift,
@@ -1543,6 +1551,10 @@ static bool dcn32_mmhubbub_create(struct dc_context *ctx, struct resource_pool *
 			return false;
 		}
 
+#undef REG_STRUCT
+#define REG_STRUCT mcif_wb30_regs
+		mcif_wb_regs_dcn3_init(0);
+
 		dcn32_mmhubbub_construct(mcif_wb30, ctx,
 				&mcif_wb30_regs[i],
 				&mcif_wb30_shift,
@@ -1565,6 +1577,13 @@ static struct display_stream_compressor *dcn32_dsc_create(
 		return NULL;
 	}
 
+#undef REG_STRUCT
+#define REG_STRUCT dsc_regs
+	dsc_regsDCN20_init(0),
+	dsc_regsDCN20_init(1),
+	dsc_regsDCN20_init(2),
+	dsc_regsDCN20_init(3);
+
 	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
 
 	dsc->max_image_width = 6016;
@@ -1702,13 +1721,26 @@ bool dcn32_remove_phantom_pipes(struct dc *dc, struct dc_state *context)
 {
 	int i;
 	bool removed_pipe = false;
+	struct dc_plane_state *phantom_plane = NULL;
+	struct dc_stream_state *phantom_stream = NULL;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
 		// build scaling params for phantom pipes
 		if (pipe->plane_state && pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
+			phantom_plane = pipe->plane_state;
+			phantom_stream = pipe->stream;
+
 			dc_rem_all_planes_for_stream(dc, pipe->stream, context);
 			dc_remove_stream_from_ctx(dc, context, pipe->stream);
+
+			/* Ref count is incremented on allocation and also when added to the context.
+			 * Therefore we must call release for the the phantom plane and stream once
+			 * they are removed from the ctx to finally decrement the refcount to 0 to free.
+			 */
+			dc_plane_state_release(phantom_plane);
+			dc_stream_release(phantom_stream);
+
 			removed_pipe = true;
 		}
 
@@ -1808,12 +1840,6 @@ validate_out:
 	return out;
 }
 
-
-static bool is_dual_plane(enum surface_pixel_format format)
-{
-	return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA;
-}
-
 int dcn32_populate_dml_pipes_from_context(
 	struct dc *dc, struct dc_state *context,
 	display_e2e_pipe_params_st *pipes,
@@ -1822,12 +1848,37 @@ int dcn32_populate_dml_pipes_from_context(
 	int i, pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe;
-	bool subvp_in_use = false, is_pipe_split_expected[MAX_PIPES];
-	int plane_count = 0;
+	bool subvp_in_use = false;
+	uint8_t is_pipe_split_expected[MAX_PIPES] = {0};
 	struct dc_crtc_timing *timing;
 
 	dcn20_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
 
+	/* Determine whether we will apply ODM 2to1 policy:
+	 * Applies to single display and where the number of planes is less than 3.
+	 * For 3 plane case ( 2 MPO planes ), we will not set the policy for the MPO pipes.
+	 *
+	 * Apply pipe split policy first so we can predict the pipe split correctly
+	 * (dcn32_predict_pipe_split).
+	 */
+	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+		if (!res_ctx->pipe_ctx[i].stream)
+			continue;
+		pipe = &res_ctx->pipe_ctx[i];
+		timing = &pipe->stream->timing;
+
+		pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_dal;
+		if (context->stream_count == 1 &&
+				context->stream_status[0].plane_count <= 1 &&
+				!dc_is_hdmi_signal(res_ctx->pipe_ctx[i].stream->signal) &&
+				is_h_timing_divisible_by_2(res_ctx->pipe_ctx[i].stream) &&
+				pipe->stream->timing.pix_clk_100hz * 100 > DCN3_2_VMIN_DISPCLK_HZ &&
+				dc->debug.enable_single_display_2to1_odm_policy) {
+			pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_2to1;
+		}
+		pipe_cnt++;
+	}
+
 	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
 
 		if (!res_ctx->pipe_ctx[i].stream)
@@ -1880,59 +1931,18 @@ int dcn32_populate_dml_pipes_from_context(
 			}
 		}
 
-		/* Calculate the number of planes we have so we can determine
-		 *  whether to apply ODM 2to1 policy or not
-		 */
-		if (pipe->stream && !pipe->prev_odm_pipe &&
-				(!pipe->top_pipe || pipe->top_pipe->plane_state != pipe->plane_state))
-			++plane_count;
-
 		DC_FP_START();
-		is_pipe_split_expected[i] = dcn32_predict_pipe_split(context, pipes[i].pipe, i);
+		is_pipe_split_expected[i] = dcn32_predict_pipe_split(context, &pipes[pipe_cnt]);
 		DC_FP_END();
 
 		pipe_cnt++;
 	}
 
-	/* Determine whether we will apply ODM 2to1 policy
-	 * Applies to single display and where the number of planes is less than 3
-	 * For 3 plane case ( 2 MPO planes ), we will not set the policy for the MPO pipes
-	 */
-	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
-		if (!res_ctx->pipe_ctx[i].stream)
-			continue;
-		pipe = &res_ctx->pipe_ctx[i];
-		timing = &pipe->stream->timing;
-
-		pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_dal;
-		res_ctx->pipe_ctx[i].stream->odm_2to1_policy_applied  = false;
-		if (context->stream_count == 1 && timing->dsc_cfg.num_slices_h != 1) {
-			if (dc->debug.enable_single_display_2to1_odm_policy) {
-				if (!((plane_count > 2) && pipe->top_pipe))
-					pipes[pipe_cnt].pipe.dest.odm_combine_policy = dm_odm_combine_policy_2to1;
-			}
-			res_ctx->pipe_ctx[i].stream->odm_2to1_policy_applied = true;
-		}
-		pipe_cnt++;
-	}
-
 	/* For DET allocation, we don't want to use DML policy (not optimal for utilizing all
 	 * the DET available for each pipe). Use the DET override input to maintain our driver
 	 * policy.
 	 */
-	if (pipe_cnt == 1 && !is_pipe_split_expected[0]) {
-		pipes[0].pipe.src.det_size_override = DCN3_2_MAX_DET_SIZE;
-		if (pipe->plane_state && !dc->debug.disable_z9_mpc) {
-			if (!is_dual_plane(pipe->plane_state->format)) {
-				pipes[0].pipe.src.det_size_override = DCN3_2_DEFAULT_DET_SIZE;
-				pipes[0].pipe.src.unbounded_req_mode = true;
-				if (pipe->plane_state->src_rect.width >= 5120 &&
-					pipe->plane_state->src_rect.height >= 2880)
-					pipes[0].pipe.src.det_size_override = 320; // 5K or higher
-			}
-		}
-	} else
-		dcn32_determine_det_override(context, pipes, is_pipe_split_expected, dc->res_pool->pipe_count);
+	dcn32_set_det_allocations(dc, context, pipes);
 
 	// In general cases we want to keep the dram clock change requirement
 	// (prefer configs that support MCLK switch). Only override to false
@@ -2003,6 +2013,28 @@ static bool dcn32_resource_construct(
 	uint32_t pipe_fuses = 0;
 	uint32_t num_pipes  = 4;
 
+	#undef REG_STRUCT
+	#define REG_STRUCT bios_regs
+		bios_regs_init();
+
+	#undef REG_STRUCT
+	#define REG_STRUCT clk_src_regs
+		clk_src_regs_init(0, A),
+		clk_src_regs_init(1, B),
+		clk_src_regs_init(2, C),
+		clk_src_regs_init(3, D),
+		clk_src_regs_init(4, E);
+	#undef REG_STRUCT
+	#define REG_STRUCT abm_regs
+		abm_regs_init(0),
+		abm_regs_init(1),
+		abm_regs_init(2),
+		abm_regs_init(3);
+
+	#undef REG_STRUCT
+	#define REG_STRUCT dccg_regs
+		dccg_regs_init();
+
 	DC_FP_START();
 
 	ctx->dc_bios->regs = &bios_regs;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h
index cf15d0e5e9b4..55945cca2260 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource.h
@@ -28,11 +28,16 @@
 
 #include "core_types.h"
 
+#define DCN3_2_DEFAULT_DET_SIZE 256
+#define DCN3_2_MAX_DET_SIZE 1152
+#define DCN3_2_MIN_DET_SIZE 128
+#define DCN3_2_MIN_COMPBUF_SIZE_KB 128
 #define DCN3_2_DET_SEG_SIZE 64
 #define DCN3_2_MALL_MBLK_SIZE_BYTES 65536 // 64 * 1024
 #define DCN3_2_MBLK_WIDTH 128
 #define DCN3_2_MBLK_HEIGHT_4BPE 128
 #define DCN3_2_MBLK_HEIGHT_8BPE 64
+#define DCN3_2_VMIN_DISPCLK_HZ 717000000
 
 #define TO_DCN32_RES_POOL(pool)\
 	container_of(pool, struct dcn32_resource_pool, base)
@@ -109,7 +114,1167 @@ struct pipe_ctx *dcn32_acquire_idle_pipe_for_head_pipe_in_layer(
 		struct dc_stream_state *stream,
 		struct pipe_ctx *head_pipe);
 
-void dcn32_determine_det_override(struct dc_state *context, display_e2e_pipe_params_st *pipes,
-		bool *is_pipe_split_expected, int pipe_cnt);
+void dcn32_determine_det_override(struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes);
+
+void dcn32_set_det_allocations(struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes);
+/* definitions for run time init of reg offsets */
+
+/* CLK SRC */
+#define CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)                             \
+  (                                                                            \
+  SRI_ARR_ALPHABET(PIXCLK_RESYNC_CNTL, PHYPLL, index, pllid),                  \
+      SRII_ARR_2(PHASE, DP_DTO, 0, index),                                     \
+      SRII_ARR_2(PHASE, DP_DTO, 1, index),                                     \
+      SRII_ARR_2(PHASE, DP_DTO, 2, index),                                     \
+      SRII_ARR_2(PHASE, DP_DTO, 3, index),                                     \
+      SRII_ARR_2(MODULO, DP_DTO, 0, index),                                    \
+      SRII_ARR_2(MODULO, DP_DTO, 1, index),                                    \
+      SRII_ARR_2(MODULO, DP_DTO, 2, index),                                    \
+      SRII_ARR_2(MODULO, DP_DTO, 3, index),                                    \
+      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 0, index),                              \
+      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 1, index),                              \
+      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 2, index),                              \
+      SRII_ARR_2(PIXEL_RATE_CNTL, OTG, 3, index)                               \
+  )
+
+/* ABM */
+#define ABM_DCN32_REG_LIST_RI(id)                                              \
+  ( \
+  SRI_ARR(DC_ABM1_HG_SAMPLE_RATE, ABM, id),                                    \
+      SRI_ARR(DC_ABM1_LS_SAMPLE_RATE, ABM, id),                                \
+      SRI_ARR(BL1_PWM_BL_UPDATE_SAMPLE_RATE, ABM, id),                         \
+      SRI_ARR(DC_ABM1_HG_MISC_CTRL, ABM, id),                                  \
+      SRI_ARR(DC_ABM1_IPCSC_COEFF_SEL, ABM, id),                               \
+      SRI_ARR(BL1_PWM_CURRENT_ABM_LEVEL, ABM, id),                             \
+      SRI_ARR(BL1_PWM_TARGET_ABM_LEVEL, ABM, id),                              \
+      SRI_ARR(BL1_PWM_USER_LEVEL, ABM, id),                                    \
+      SRI_ARR(DC_ABM1_LS_MIN_MAX_PIXEL_VALUE_THRES, ABM, id),                  \
+      SRI_ARR(DC_ABM1_HGLS_REG_READ_PROGRESS, ABM, id),                        \
+      SRI_ARR(DC_ABM1_ACE_OFFSET_SLOPE_0, ABM, id),                            \
+      SRI_ARR(DC_ABM1_ACE_THRES_12, ABM, id), NBIO_SR_ARR(BIOS_SCRATCH_2, id)  \
+  )
+
+/* Audio */
+#define AUD_COMMON_REG_LIST_RI(id)                                             \
+  ( \
+  SRI_ARR(AZALIA_F0_CODEC_ENDPOINT_INDEX, AZF0ENDPOINT, id),                   \
+      SRI_ARR(AZALIA_F0_CODEC_ENDPOINT_DATA, AZF0ENDPOINT, id),                \
+      SR_ARR(AZALIA_F0_CODEC_FUNCTION_PARAMETER_STREAM_FORMATS, id),           \
+      SR_ARR(AZALIA_F0_CODEC_FUNCTION_PARAMETER_SUPPORTED_SIZE_RATES, id),     \
+      SR_ARR(AZALIA_F0_CODEC_FUNCTION_PARAMETER_POWER_STATES, id),             \
+      SR_ARR(DCCG_AUDIO_DTO_SOURCE, id), SR_ARR(DCCG_AUDIO_DTO0_MODULE, id),   \
+      SR_ARR(DCCG_AUDIO_DTO0_PHASE, id), SR_ARR(DCCG_AUDIO_DTO1_MODULE, id),   \
+      SR_ARR(DCCG_AUDIO_DTO1_PHASE, id)                                        \
+  )
+
+/* VPG */
+
+#define VPG_DCN3_REG_LIST_RI(id)                                               \
+  ( \
+  SRI_ARR(VPG_GENERIC_STATUS, VPG, id),                                        \
+      SRI_ARR(VPG_GENERIC_PACKET_ACCESS_CTRL, VPG, id),                        \
+      SRI_ARR(VPG_GENERIC_PACKET_DATA, VPG, id),                               \
+      SRI_ARR(VPG_GSP_FRAME_UPDATE_CTRL, VPG, id),                             \
+      SRI_ARR(VPG_GSP_IMMEDIATE_UPDATE_CTRL, VPG, id)                          \
+  )
+
+/* AFMT */
+#define AFMT_DCN3_REG_LIST_RI(id)                                              \
+  ( \
+  SRI_ARR(AFMT_INFOFRAME_CONTROL0, AFMT, id),                                  \
+      SRI_ARR(AFMT_VBI_PACKET_CONTROL, AFMT, id),                              \
+      SRI_ARR(AFMT_AUDIO_PACKET_CONTROL, AFMT, id),                            \
+      SRI_ARR(AFMT_AUDIO_PACKET_CONTROL2, AFMT, id),                           \
+      SRI_ARR(AFMT_AUDIO_SRC_CONTROL, AFMT, id),                               \
+      SRI_ARR(AFMT_60958_0, AFMT, id), SRI_ARR(AFMT_60958_1, AFMT, id),        \
+      SRI_ARR(AFMT_60958_2, AFMT, id), SRI_ARR(AFMT_MEM_PWR, AFMT, id)         \
+  )
+
+/* APG */
+#define APG_DCN31_REG_LIST_RI(id)                                              \
+  (\
+  SRI_ARR(APG_CONTROL, APG, id), SRI_ARR(APG_CONTROL2, APG, id),               \
+      SRI_ARR(APG_MEM_PWR, APG, id), SRI_ARR(APG_DBG_GEN_CONTROL, APG, id)     \
+  )
+
+/* Stream encoder */
+#define SE_DCN32_REG_LIST_RI(id)                                               \
+  ( \
+  SRI_ARR(AFMT_CNTL, DIG, id), SRI_ARR(DIG_FE_CNTL, DIG, id),                  \
+      SRI_ARR(HDMI_CONTROL, DIG, id), SRI_ARR(HDMI_DB_CONTROL, DIG, id),       \
+      SRI_ARR(HDMI_GC, DIG, id),                                               \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL0, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL1, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL2, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL3, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL4, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL5, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL6, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL7, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL8, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL9, DIG, id),                          \
+      SRI_ARR(HDMI_GENERIC_PACKET_CONTROL10, DIG, id),                         \
+      SRI_ARR(HDMI_INFOFRAME_CONTROL0, DIG, id),                               \
+      SRI_ARR(HDMI_INFOFRAME_CONTROL1, DIG, id),                               \
+      SRI_ARR(HDMI_VBI_PACKET_CONTROL, DIG, id),                               \
+      SRI_ARR(HDMI_AUDIO_PACKET_CONTROL, DIG, id),                             \
+      SRI_ARR(HDMI_ACR_PACKET_CONTROL, DIG, id),                               \
+      SRI_ARR(HDMI_ACR_32_0, DIG, id), SRI_ARR(HDMI_ACR_32_1, DIG, id),        \
+      SRI_ARR(HDMI_ACR_44_0, DIG, id), SRI_ARR(HDMI_ACR_44_1, DIG, id),        \
+      SRI_ARR(HDMI_ACR_48_0, DIG, id), SRI_ARR(HDMI_ACR_48_1, DIG, id),        \
+      SRI_ARR(DP_DB_CNTL, DP, id), SRI_ARR(DP_MSA_MISC, DP, id),               \
+      SRI_ARR(DP_MSA_VBID_MISC, DP, id), SRI_ARR(DP_MSA_COLORIMETRY, DP, id),  \
+      SRI_ARR(DP_MSA_TIMING_PARAM1, DP, id),                                   \
+      SRI_ARR(DP_MSA_TIMING_PARAM2, DP, id),                                   \
+      SRI_ARR(DP_MSA_TIMING_PARAM3, DP, id),                                   \
+      SRI_ARR(DP_MSA_TIMING_PARAM4, DP, id),                                   \
+      SRI_ARR(DP_MSE_RATE_CNTL, DP, id), SRI_ARR(DP_MSE_RATE_UPDATE, DP, id),  \
+      SRI_ARR(DP_PIXEL_FORMAT, DP, id), SRI_ARR(DP_SEC_CNTL, DP, id),          \
+      SRI_ARR(DP_SEC_CNTL1, DP, id), SRI_ARR(DP_SEC_CNTL2, DP, id),            \
+      SRI_ARR(DP_SEC_CNTL5, DP, id), SRI_ARR(DP_SEC_CNTL6, DP, id),            \
+      SRI_ARR(DP_STEER_FIFO, DP, id), SRI_ARR(DP_VID_M, DP, id),               \
+      SRI_ARR(DP_VID_N, DP, id), SRI_ARR(DP_VID_STREAM_CNTL, DP, id),          \
+      SRI_ARR(DP_VID_TIMING, DP, id), SRI_ARR(DP_SEC_AUD_N, DP, id),           \
+      SRI_ARR(DP_SEC_TIMESTAMP, DP, id), SRI_ARR(DP_DSC_CNTL, DP, id),         \
+      SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id),                           \
+      SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id),                          \
+      SRI_ARR(DP_SEC_FRAMING4, DP, id), SRI_ARR(DP_GSP11_CNTL, DP, id),        \
+      SRI_ARR(DME_CONTROL, DME, id),                                           \
+      SRI_ARR(DP_SEC_METADATA_TRANSMISSION, DP, id),                           \
+      SRI_ARR(HDMI_METADATA_PACKET_CONTROL, DIG, id),                          \
+      SRI_ARR(DIG_FE_CNTL, DIG, id), SRI_ARR(DIG_CLOCK_PATTERN, DIG, id),      \
+      SRI_ARR(DIG_FIFO_CTRL0, DIG, id)                                         \
+  )
+
+/* Aux regs */
+
+#define AUX_REG_LIST_RI(id)                                                    \
+  ( \
+  SRI_ARR(AUX_CONTROL, DP_AUX, id), SRI_ARR(AUX_DPHY_RX_CONTROL0, DP_AUX, id), \
+      SRI_ARR(AUX_DPHY_RX_CONTROL1, DP_AUX, id)                                \
+  )
+
+#define DCN2_AUX_REG_LIST_RI(id)                                               \
+  ( \
+  AUX_REG_LIST_RI(id), SRI_ARR(AUX_DPHY_TX_CONTROL, DP_AUX, id)                \
+  )
+
+/* HDP */
+#define HPD_REG_LIST_RI(id) SRI_ARR(DC_HPD_CONTROL, HPD, id)
+
+/* Link encoder */
+#define LE_DCN3_REG_LIST_RI(id)                                                \
+  ( \
+  SRI_ARR(DIG_BE_CNTL, DIG, id), SRI_ARR(DIG_BE_EN_CNTL, DIG, id),             \
+      SRI_ARR(TMDS_CTL_BITS, DIG, id),                                         \
+      SRI_ARR(TMDS_DCBALANCER_CONTROL, DIG, id), SRI_ARR(DP_CONFIG, DP, id),   \
+      SRI_ARR(DP_DPHY_CNTL, DP, id), SRI_ARR(DP_DPHY_PRBS_CNTL, DP, id),       \
+      SRI_ARR(DP_DPHY_SCRAM_CNTL, DP, id), SRI_ARR(DP_DPHY_SYM0, DP, id),      \
+      SRI_ARR(DP_DPHY_SYM1, DP, id), SRI_ARR(DP_DPHY_SYM2, DP, id),            \
+      SRI_ARR(DP_DPHY_TRAINING_PATTERN_SEL, DP, id),                           \
+      SRI_ARR(DP_LINK_CNTL, DP, id), SRI_ARR(DP_LINK_FRAMING_CNTL, DP, id),    \
+      SRI_ARR(DP_MSE_SAT0, DP, id), SRI_ARR(DP_MSE_SAT1, DP, id),              \
+      SRI_ARR(DP_MSE_SAT2, DP, id), SRI_ARR(DP_MSE_SAT_UPDATE, DP, id),        \
+      SRI_ARR(DP_SEC_CNTL, DP, id), SRI_ARR(DP_VID_STREAM_CNTL, DP, id),       \
+      SRI_ARR(DP_DPHY_FAST_TRAINING, DP, id), SRI_ARR(DP_SEC_CNTL1, DP, id),   \
+      SRI_ARR(DP_DPHY_BS_SR_SWAP_CNTL, DP, id),                                \
+      SRI_ARR(DP_DPHY_HBR2_PATTERN_CONTROL, DP, id)                            \
+  )
+
+#define LE_DCN31_REG_LIST_RI(id)                                               \
+  ( \
+  LE_DCN3_REG_LIST_RI(id), SRI_ARR(DP_DPHY_INTERNAL_CTRL, DP, id),             \
+      SR_ARR(DIO_LINKA_CNTL, id), SR_ARR(DIO_LINKB_CNTL, id),                  \
+      SR_ARR(DIO_LINKC_CNTL, id), SR_ARR(DIO_LINKD_CNTL, id),                  \
+      SR_ARR(DIO_LINKE_CNTL, id), SR_ARR(DIO_LINKF_CNTL, id)                   \
+  )
+
+#define UNIPHY_DCN2_REG_LIST_RI(id, phyid)                                     \
+  ( \
+  SRI_ARR_ALPHABET(CLOCK_ENABLE, SYMCLK, id, phyid),                           \
+      SRI_ARR_ALPHABET(CHANNEL_XBAR_CNTL, UNIPHY, id, phyid)                   \
+  )
+
+/* HPO DP stream encoder */
+#define DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)                               \
+  ( \
+  SR_ARR(DP_STREAM_MAPPER_CONTROL0, id),                                       \
+      SR_ARR(DP_STREAM_MAPPER_CONTROL1, id),                                   \
+      SR_ARR(DP_STREAM_MAPPER_CONTROL2, id),                                   \
+      SR_ARR(DP_STREAM_MAPPER_CONTROL3, id),                                   \
+      SRI_ARR(DP_STREAM_ENC_CLOCK_CONTROL, DP_STREAM_ENC, id),                 \
+      SRI_ARR(DP_STREAM_ENC_INPUT_MUX_CONTROL, DP_STREAM_ENC, id),             \
+      SRI_ARR(DP_STREAM_ENC_AUDIO_CONTROL, DP_STREAM_ENC, id),                 \
+      SRI_ARR(DP_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0, DP_STREAM_ENC, id), \
+      SRI_ARR(DP_SYM32_ENC_CONTROL, DP_SYM32_ENC, id),                         \
+      SRI_ARR(DP_SYM32_ENC_VID_PIXEL_FORMAT, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_VID_PIXEL_FORMAT_DOUBLE_BUFFER_CONTROL, DP_SYM32_ENC, id), \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA0, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA1, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA2, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA3, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA4, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA5, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA6, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA7, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA8, DP_SYM32_ENC, id),                        \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA_CONTROL, DP_SYM32_ENC, id),                 \
+      SRI_ARR(DP_SYM32_ENC_VID_MSA_DOUBLE_BUFFER_CONTROL, DP_SYM32_ENC, id),   \
+      SRI_ARR(DP_SYM32_ENC_VID_FIFO_CONTROL, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_VID_STREAM_CONTROL, DP_SYM32_ENC, id),              \
+      SRI_ARR(DP_SYM32_ENC_VID_VBID_CONTROL, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_SDP_CONTROL, DP_SYM32_ENC, id),                     \
+      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL0, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL2, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL3, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL5, DP_SYM32_ENC, id),                \
+      SRI_ARR(DP_SYM32_ENC_SDP_GSP_CONTROL11, DP_SYM32_ENC, id),               \
+      SRI_ARR(DP_SYM32_ENC_SDP_METADATA_PACKET_CONTROL, DP_SYM32_ENC, id),     \
+      SRI_ARR(DP_SYM32_ENC_SDP_AUDIO_CONTROL0, DP_SYM32_ENC, id),              \
+      SRI_ARR(DP_SYM32_ENC_VID_CRC_CONTROL, DP_SYM32_ENC, id),                 \
+      SRI_ARR(DP_SYM32_ENC_HBLANK_CONTROL, DP_SYM32_ENC, id)                   \
+  )
+
+/* HPO DP link encoder regs */
+#define DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)                                 \
+  ( \
+  SRI_ARR(DP_LINK_ENC_CLOCK_CONTROL, DP_LINK_ENC, id),                         \
+      SRI_ARR(DP_DPHY_SYM32_CONTROL, DP_DPHY_SYM32, id),                       \
+      SRI_ARR(DP_DPHY_SYM32_STATUS, DP_DPHY_SYM32, id),                        \
+      SRI_ARR(DP_DPHY_SYM32_TP_CONFIG, DP_DPHY_SYM32, id),                     \
+      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED0, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED1, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED2, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_TP_PRBS_SEED3, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_TP_SQ_PULSE, DP_DPHY_SYM32, id),                   \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM0, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM1, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM2, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM3, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM4, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM5, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM6, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM7, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM8, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM9, DP_DPHY_SYM32, id),                    \
+      SRI_ARR(DP_DPHY_SYM32_TP_CUSTOM10, DP_DPHY_SYM32, id),                   \
+      SRI_ARR(DP_DPHY_SYM32_SAT_VC0, DP_DPHY_SYM32, id),                       \
+      SRI_ARR(DP_DPHY_SYM32_SAT_VC1, DP_DPHY_SYM32, id),                       \
+      SRI_ARR(DP_DPHY_SYM32_SAT_VC2, DP_DPHY_SYM32, id),                       \
+      SRI_ARR(DP_DPHY_SYM32_SAT_VC3, DP_DPHY_SYM32, id),                       \
+      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL0, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL1, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL2, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_VC_RATE_CNTL3, DP_DPHY_SYM32, id),                 \
+      SRI_ARR(DP_DPHY_SYM32_SAT_UPDATE, DP_DPHY_SYM32, id)                     \
+  )
+
+/* DPP */
+#define DPP_REG_LIST_DCN30_COMMON_RI(id)                                       \
+  ( \
+  SRI_ARR(CM_DEALPHA, CM, id), SRI_ARR(CM_MEM_PWR_STATUS, CM, id),             \
+      SRI_ARR(CM_BIAS_CR_R, CM, id), SRI_ARR(CM_BIAS_Y_G_CB_B, CM, id),        \
+      SRI_ARR(PRE_DEGAM, CNVC_CFG, id), SRI_ARR(CM_GAMCOR_CONTROL, CM, id),    \
+      SRI_ARR(CM_GAMCOR_LUT_CONTROL, CM, id),                                  \
+      SRI_ARR(CM_GAMCOR_LUT_INDEX, CM, id),                                    \
+      SRI_ARR(CM_GAMCOR_LUT_INDEX, CM, id),                                    \
+      SRI_ARR(CM_GAMCOR_LUT_DATA, CM, id),                                     \
+      SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_B, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_G, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMB_START_CNTL_R, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_B, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_G, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMB_START_SLOPE_CNTL_R, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_B, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_B, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_G, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_G, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL1_R, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_END_CNTL2_R, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMB_REGION_0_1, CM, id),                              \
+      SRI_ARR(CM_GAMCOR_RAMB_REGION_32_33, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMB_OFFSET_B, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMB_OFFSET_G, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMB_OFFSET_R, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_B, CM, id),                       \
+      SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_G, CM, id),                       \
+      SRI_ARR(CM_GAMCOR_RAMB_START_BASE_CNTL_R, CM, id),                       \
+      SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_B, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_G, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMA_START_CNTL_R, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_B, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_G, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMA_START_SLOPE_CNTL_R, CM, id),                      \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_B, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_B, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_G, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_G, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL1_R, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_END_CNTL2_R, CM, id),                             \
+      SRI_ARR(CM_GAMCOR_RAMA_REGION_0_1, CM, id),                              \
+      SRI_ARR(CM_GAMCOR_RAMA_REGION_32_33, CM, id),                            \
+      SRI_ARR(CM_GAMCOR_RAMA_OFFSET_B, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMA_OFFSET_G, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMA_OFFSET_R, CM, id),                                \
+      SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_B, CM, id),                       \
+      SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_G, CM, id),                       \
+      SRI_ARR(CM_GAMCOR_RAMA_START_BASE_CNTL_R, CM, id),                       \
+      SRI_ARR(CM_GAMUT_REMAP_CONTROL, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C11_C12, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C13_C14, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C21_C22, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C23_C24, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C31_C32, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_C33_C34, CM, id),                                 \
+      SRI_ARR(CM_GAMUT_REMAP_B_C11_C12, CM, id),                               \
+      SRI_ARR(CM_GAMUT_REMAP_B_C13_C14, CM, id),                               \
+      SRI_ARR(CM_GAMUT_REMAP_B_C21_C22, CM, id),                               \
+      SRI_ARR(CM_GAMUT_REMAP_B_C23_C24, CM, id),                               \
+      SRI_ARR(CM_GAMUT_REMAP_B_C31_C32, CM, id),                               \
+      SRI_ARR(CM_GAMUT_REMAP_B_C33_C34, CM, id),                               \
+      SRI_ARR(DSCL_EXT_OVERSCAN_LEFT_RIGHT, DSCL, id),                         \
+      SRI_ARR(DSCL_EXT_OVERSCAN_TOP_BOTTOM, DSCL, id),                         \
+      SRI_ARR(OTG_H_BLANK, DSCL, id), SRI_ARR(OTG_V_BLANK, DSCL, id),          \
+      SRI_ARR(SCL_MODE, DSCL, id), SRI_ARR(LB_DATA_FORMAT, DSCL, id),          \
+      SRI_ARR(LB_MEMORY_CTRL, DSCL, id), SRI_ARR(DSCL_AUTOCAL, DSCL, id),      \
+      SRI_ARR(SCL_TAP_CONTROL, DSCL, id),                                      \
+      SRI_ARR(SCL_COEF_RAM_TAP_SELECT, DSCL, id),                              \
+      SRI_ARR(SCL_COEF_RAM_TAP_DATA, DSCL, id),                                \
+      SRI_ARR(DSCL_2TAP_CONTROL, DSCL, id), SRI_ARR(MPC_SIZE, DSCL, id),       \
+      SRI_ARR(SCL_HORZ_FILTER_SCALE_RATIO, DSCL, id),                          \
+      SRI_ARR(SCL_VERT_FILTER_SCALE_RATIO, DSCL, id),                          \
+      SRI_ARR(SCL_HORZ_FILTER_SCALE_RATIO_C, DSCL, id),                        \
+      SRI_ARR(SCL_VERT_FILTER_SCALE_RATIO_C, DSCL, id),                        \
+      SRI_ARR(SCL_HORZ_FILTER_INIT, DSCL, id),                                 \
+      SRI_ARR(SCL_HORZ_FILTER_INIT_C, DSCL, id),                               \
+      SRI_ARR(SCL_VERT_FILTER_INIT, DSCL, id),                                 \
+      SRI_ARR(SCL_VERT_FILTER_INIT_C, DSCL, id),                               \
+      SRI_ARR(RECOUT_START, DSCL, id), SRI_ARR(RECOUT_SIZE, DSCL, id),         \
+      SRI_ARR(PRE_DEALPHA, CNVC_CFG, id), SRI_ARR(PRE_REALPHA, CNVC_CFG, id),  \
+      SRI_ARR(PRE_CSC_MODE, CNVC_CFG, id),                                     \
+      SRI_ARR(PRE_CSC_C11_C12, CNVC_CFG, id),                                  \
+      SRI_ARR(PRE_CSC_C33_C34, CNVC_CFG, id),                                  \
+      SRI_ARR(PRE_CSC_B_C11_C12, CNVC_CFG, id),                                \
+      SRI_ARR(PRE_CSC_B_C33_C34, CNVC_CFG, id),                                \
+      SRI_ARR(CM_POST_CSC_CONTROL, CM, id),                                    \
+      SRI_ARR(CM_POST_CSC_C11_C12, CM, id),                                    \
+      SRI_ARR(CM_POST_CSC_C33_C34, CM, id),                                    \
+      SRI_ARR(CM_POST_CSC_B_C11_C12, CM, id),                                  \
+      SRI_ARR(CM_POST_CSC_B_C33_C34, CM, id),                                  \
+      SRI_ARR(CM_MEM_PWR_CTRL, CM, id), SRI_ARR(CM_CONTROL, CM, id),           \
+      SRI_ARR(FORMAT_CONTROL, CNVC_CFG, id),                                   \
+      SRI_ARR(CNVC_SURFACE_PIXEL_FORMAT, CNVC_CFG, id),                        \
+      SRI_ARR(CURSOR0_CONTROL, CNVC_CUR, id),                                  \
+      SRI_ARR(CURSOR0_COLOR0, CNVC_CUR, id),                                   \
+      SRI_ARR(CURSOR0_COLOR1, CNVC_CUR, id),                                   \
+      SRI_ARR(CURSOR0_FP_SCALE_BIAS, CNVC_CUR, id),                            \
+      SRI_ARR(DPP_CONTROL, DPP_TOP, id), SRI_ARR(CM_HDR_MULT_COEF, CM, id),    \
+      SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                   \
+      SRI_ARR(ALPHA_2BIT_LUT, CNVC_CFG, id),                                   \
+      SRI_ARR(FCNV_FP_BIAS_R, CNVC_CFG, id),                                   \
+      SRI_ARR(FCNV_FP_BIAS_G, CNVC_CFG, id),                                   \
+      SRI_ARR(FCNV_FP_BIAS_B, CNVC_CFG, id),                                   \
+      SRI_ARR(FCNV_FP_SCALE_R, CNVC_CFG, id),                                  \
+      SRI_ARR(FCNV_FP_SCALE_G, CNVC_CFG, id),                                  \
+      SRI_ARR(FCNV_FP_SCALE_B, CNVC_CFG, id),                                  \
+      SRI_ARR(COLOR_KEYER_CONTROL, CNVC_CFG, id),                              \
+      SRI_ARR(COLOR_KEYER_ALPHA, CNVC_CFG, id),                                \
+      SRI_ARR(COLOR_KEYER_RED, CNVC_CFG, id),                                  \
+      SRI_ARR(COLOR_KEYER_GREEN, CNVC_CFG, id),                                \
+      SRI_ARR(COLOR_KEYER_BLUE, CNVC_CFG, id),                                 \
+      SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                   \
+      SRI_ARR(OBUF_MEM_PWR_CTRL, DSCL, id),                                    \
+      SRI_ARR(DSCL_MEM_PWR_STATUS, DSCL, id),                                  \
+      SRI_ARR(DSCL_MEM_PWR_CTRL, DSCL, id)                                     \
+  )
+
+/* OPP */
+#define OPP_REG_LIST_DCN_RI(id)                                                \
+  ( \
+  SRI_ARR(FMT_BIT_DEPTH_CONTROL, FMT, id), SRI_ARR(FMT_CONTROL, FMT, id),      \
+      SRI_ARR(FMT_DITHER_RAND_R_SEED, FMT, id),                                \
+      SRI_ARR(FMT_DITHER_RAND_G_SEED, FMT, id),                                \
+      SRI_ARR(FMT_DITHER_RAND_B_SEED, FMT, id),                                \
+      SRI_ARR(FMT_CLAMP_CNTL, FMT, id),                                        \
+      SRI_ARR(FMT_DYNAMIC_EXP_CNTL, FMT, id),                                  \
+      SRI_ARR(FMT_MAP420_MEMORY_CONTROL, FMT, id),                             \
+      SRI_ARR(OPPBUF_CONTROL, OPPBUF, id),                                     \
+      SRI_ARR(OPPBUF_3D_PARAMETERS_0, OPPBUF, id),                             \
+      SRI_ARR(OPPBUF_3D_PARAMETERS_1, OPPBUF, id),                             \
+      SRI_ARR(OPP_PIPE_CONTROL, OPP_PIPE, id)                                  \
+  )
+
+#define OPP_REG_LIST_DCN10_RI(id) OPP_REG_LIST_DCN_RI(id)
+
+#define OPP_DPG_REG_LIST_RI(id)                                                \
+  ( \
+  SRI_ARR(DPG_CONTROL, DPG, id), SRI_ARR(DPG_DIMENSIONS, DPG, id),             \
+      SRI_ARR(DPG_OFFSET_SEGMENT, DPG, id), SRI_ARR(DPG_COLOUR_B_CB, DPG, id), \
+      SRI_ARR(DPG_COLOUR_G_Y, DPG, id), SRI_ARR(DPG_COLOUR_R_CR, DPG, id),     \
+      SRI_ARR(DPG_RAMP_CONTROL, DPG, id), SRI_ARR(DPG_STATUS, DPG, id)         \
+  )
+
+#define OPP_REG_LIST_DCN30_RI(id)                                              \
+  ( \
+  OPP_REG_LIST_DCN10_RI(id), OPP_DPG_REG_LIST_RI(id),                          \
+      SRI_ARR(FMT_422_CONTROL, FMT, id)                                        \
+  )
+
+/* Aux engine regs */
+#define AUX_COMMON_REG_LIST0_RI(id)                                            \
+  ( \
+  SRI_ARR(AUX_CONTROL, DP_AUX, id), SRI_ARR(AUX_ARB_CONTROL, DP_AUX, id),      \
+      SRI_ARR(AUX_SW_DATA, DP_AUX, id), SRI_ARR(AUX_SW_CONTROL, DP_AUX, id),   \
+      SRI_ARR(AUX_INTERRUPT_CONTROL, DP_AUX, id),                              \
+      SRI_ARR(AUX_DPHY_RX_CONTROL1, DP_AUX, id),                               \
+      SRI_ARR(AUX_SW_STATUS, DP_AUX, id)                                       \
+  )
+
+/* DWBC */
+#define DWBC_COMMON_REG_LIST_DCN30_RI(id)                                      \
+  ( \
+  SR_ARR(DWB_ENABLE_CLK_CTRL, id), SR_ARR(DWB_MEM_PWR_CTRL, id),               \
+      SR_ARR(FC_MODE_CTRL, id), SR_ARR(FC_FLOW_CTRL, id),                      \
+      SR_ARR(FC_WINDOW_START, id), SR_ARR(FC_WINDOW_SIZE, id),                 \
+      SR_ARR(FC_SOURCE_SIZE, id), SR_ARR(DWB_UPDATE_CTRL, id),                 \
+      SR_ARR(DWB_CRC_CTRL, id), SR_ARR(DWB_CRC_MASK_R_G, id),                  \
+      SR_ARR(DWB_CRC_MASK_B_A, id), SR_ARR(DWB_CRC_VAL_R_G, id),               \
+      SR_ARR(DWB_CRC_VAL_B_A, id), SR_ARR(DWB_OUT_CTRL, id),                   \
+      SR_ARR(DWB_MMHUBBUB_BACKPRESSURE_CNT_EN, id),                            \
+      SR_ARR(DWB_MMHUBBUB_BACKPRESSURE_CNT, id),                               \
+      SR_ARR(DWB_HOST_READ_CONTROL, id), SR_ARR(DWB_SOFT_RESET, id),           \
+      SR_ARR(DWB_HDR_MULT_COEF, id), SR_ARR(DWB_GAMUT_REMAP_MODE, id),         \
+      SR_ARR(DWB_GAMUT_REMAP_COEF_FORMAT, id),                                 \
+      SR_ARR(DWB_GAMUT_REMAPA_C11_C12, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPA_C13_C14, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPA_C21_C22, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPA_C23_C24, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPA_C31_C32, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPA_C33_C34, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C11_C12, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C13_C14, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C21_C22, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C23_C24, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C31_C32, id),                                    \
+      SR_ARR(DWB_GAMUT_REMAPB_C33_C34, id), SR_ARR(DWB_OGAM_CONTROL, id),      \
+      SR_ARR(DWB_OGAM_LUT_INDEX, id), SR_ARR(DWB_OGAM_LUT_DATA, id),           \
+      SR_ARR(DWB_OGAM_LUT_CONTROL, id),                                        \
+      SR_ARR(DWB_OGAM_RAMA_START_CNTL_B, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_START_CNTL_G, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_START_CNTL_R, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_START_BASE_CNTL_B, id),                             \
+      SR_ARR(DWB_OGAM_RAMA_START_SLOPE_CNTL_B, id),                            \
+      SR_ARR(DWB_OGAM_RAMA_START_BASE_CNTL_G, id),                             \
+      SR_ARR(DWB_OGAM_RAMA_START_SLOPE_CNTL_G, id),                            \
+      SR_ARR(DWB_OGAM_RAMA_START_BASE_CNTL_R, id),                             \
+      SR_ARR(DWB_OGAM_RAMA_START_SLOPE_CNTL_R, id),                            \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL1_B, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL2_B, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL1_G, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL2_G, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL1_R, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_END_CNTL2_R, id),                                   \
+      SR_ARR(DWB_OGAM_RAMA_OFFSET_B, id), SR_ARR(DWB_OGAM_RAMA_OFFSET_G, id),  \
+      SR_ARR(DWB_OGAM_RAMA_OFFSET_R, id),                                      \
+      SR_ARR(DWB_OGAM_RAMA_REGION_0_1, id),                                    \
+      SR_ARR(DWB_OGAM_RAMA_REGION_2_3, id),                                    \
+      SR_ARR(DWB_OGAM_RAMA_REGION_4_5, id),                                    \
+      SR_ARR(DWB_OGAM_RAMA_REGION_6_7, id),                                    \
+      SR_ARR(DWB_OGAM_RAMA_REGION_8_9, id),                                    \
+      SR_ARR(DWB_OGAM_RAMA_REGION_10_11, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_12_13, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_14_15, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_16_17, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_18_19, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_20_21, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_22_23, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_24_25, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_26_27, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_28_29, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_30_31, id),                                  \
+      SR_ARR(DWB_OGAM_RAMA_REGION_32_33, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_START_CNTL_B, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_START_CNTL_G, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_START_CNTL_R, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_START_BASE_CNTL_B, id),                             \
+      SR_ARR(DWB_OGAM_RAMB_START_SLOPE_CNTL_B, id),                            \
+      SR_ARR(DWB_OGAM_RAMB_START_BASE_CNTL_G, id),                             \
+      SR_ARR(DWB_OGAM_RAMB_START_SLOPE_CNTL_G, id),                            \
+      SR_ARR(DWB_OGAM_RAMB_START_BASE_CNTL_R, id),                             \
+      SR_ARR(DWB_OGAM_RAMB_START_SLOPE_CNTL_R, id),                            \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL1_B, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL2_B, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL1_G, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL2_G, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL1_R, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_END_CNTL2_R, id),                                   \
+      SR_ARR(DWB_OGAM_RAMB_OFFSET_B, id), SR_ARR(DWB_OGAM_RAMB_OFFSET_G, id),  \
+      SR_ARR(DWB_OGAM_RAMB_OFFSET_R, id),                                      \
+      SR_ARR(DWB_OGAM_RAMB_REGION_0_1, id),                                    \
+      SR_ARR(DWB_OGAM_RAMB_REGION_2_3, id),                                    \
+      SR_ARR(DWB_OGAM_RAMB_REGION_4_5, id),                                    \
+      SR_ARR(DWB_OGAM_RAMB_REGION_6_7, id),                                    \
+      SR_ARR(DWB_OGAM_RAMB_REGION_8_9, id),                                    \
+      SR_ARR(DWB_OGAM_RAMB_REGION_10_11, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_12_13, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_14_15, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_16_17, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_18_19, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_20_21, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_22_23, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_24_25, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_26_27, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_28_29, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_30_31, id),                                  \
+      SR_ARR(DWB_OGAM_RAMB_REGION_32_33, id)                                   \
+  )
+
+/* MCIF */
+
+#define MCIF_WB_COMMON_REG_LIST_DCN32_RI(inst)                                 \
+  ( \
+  SRI2_ARR(MCIF_WB_BUFMGR_SW_CONTROL, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUFMGR_STATUS, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUF_PITCH, MCIF_WB, inst),                              \
+      SRI2_ARR(MCIF_WB_BUF_1_STATUS, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_1_STATUS2, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUF_2_STATUS, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_2_STATUS2, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUF_3_STATUS, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_3_STATUS2, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUF_4_STATUS, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_4_STATUS2, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_ARBITRATION_CONTROL, MCIF_WB, inst),                    \
+      SRI2_ARR(MCIF_WB_SCLK_CHANGE, MCIF_WB, inst),                            \
+      SRI2_ARR(MCIF_WB_TEST_DEBUG_INDEX, MCIF_WB, inst),                       \
+      SRI2_ARR(MCIF_WB_TEST_DEBUG_DATA, MCIF_WB, inst),                        \
+      SRI2_ARR(MCIF_WB_BUF_1_ADDR_Y, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_1_ADDR_C, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_2_ADDR_Y, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_2_ADDR_C, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_3_ADDR_Y, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_3_ADDR_C, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_4_ADDR_Y, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUF_4_ADDR_C, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_BUFMGR_VCE_CONTROL, MCIF_WB, inst),                     \
+      SRI2_ARR(MCIF_WB_NB_PSTATE_LATENCY_WATERMARK, MMHUBBUB, inst),           \
+      SRI2_ARR(MCIF_WB_NB_PSTATE_CONTROL, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_WATERMARK, MMHUBBUB, inst),                             \
+      SRI2_ARR(MCIF_WB_CLOCK_GATER_CONTROL, MCIF_WB, inst),                    \
+      SRI2_ARR(MCIF_WB_SELF_REFRESH_CONTROL, MCIF_WB, inst),                   \
+      SRI2_ARR(MULTI_LEVEL_QOS_CTRL, MCIF_WB, inst),                           \
+      SRI2_ARR(MCIF_WB_SECURITY_LEVEL, MCIF_WB, inst),                         \
+      SRI2_ARR(MCIF_WB_BUF_LUMA_SIZE, MCIF_WB, inst),                          \
+      SRI2_ARR(MCIF_WB_BUF_CHROMA_SIZE, MCIF_WB, inst),                        \
+      SRI2_ARR(MCIF_WB_BUF_1_ADDR_Y_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_1_ADDR_C_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_2_ADDR_Y_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_2_ADDR_C_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_3_ADDR_Y_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_3_ADDR_C_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_4_ADDR_Y_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_4_ADDR_C_HIGH, MCIF_WB, inst),                      \
+      SRI2_ARR(MCIF_WB_BUF_1_RESOLUTION, MCIF_WB, inst),                       \
+      SRI2_ARR(MCIF_WB_BUF_2_RESOLUTION, MCIF_WB, inst),                       \
+      SRI2_ARR(MCIF_WB_BUF_3_RESOLUTION, MCIF_WB, inst),                       \
+      SRI2_ARR(MCIF_WB_BUF_4_RESOLUTION, MCIF_WB, inst),                       \
+      SRI2_ARR(MMHUBBUB_MEM_PWR_CNTL, MMHUBBUB, inst),                         \
+      SRI2_ARR(MMHUBBUB_WARMUP_ADDR_REGION, MMHUBBUB, inst),                   \
+      SRI2_ARR(MMHUBBUB_WARMUP_BASE_ADDR_HIGH, MMHUBBUB, inst),                \
+      SRI2_ARR(MMHUBBUB_WARMUP_BASE_ADDR_LOW, MMHUBBUB, inst),                 \
+      SRI2_ARR(MMHUBBUB_WARMUP_CONTROL_STATUS, MMHUBBUB, inst)                 \
+  )
+
+/* DSC */
+
+#define DSC_REG_LIST_DCN20_RI(id)                                              \
+  ( \
+  SRI_ARR(DSC_TOP_CONTROL, DSC_TOP, id),                                       \
+      SRI_ARR(DSC_DEBUG_CONTROL, DSC_TOP, id),                                 \
+      SRI_ARR(DSCC_CONFIG0, DSCC, id), SRI_ARR(DSCC_CONFIG1, DSCC, id),        \
+      SRI_ARR(DSCC_STATUS, DSCC, id),                                          \
+      SRI_ARR(DSCC_INTERRUPT_CONTROL_STATUS, DSCC, id),                        \
+      SRI_ARR(DSCC_PPS_CONFIG0, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG1, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG2, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG3, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG4, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG5, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG6, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG7, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG8, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG9, DSCC, id),                                     \
+      SRI_ARR(DSCC_PPS_CONFIG10, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG11, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG12, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG13, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG14, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG15, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG16, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG17, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG18, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG19, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG20, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG21, DSCC, id),                                    \
+      SRI_ARR(DSCC_PPS_CONFIG22, DSCC, id),                                    \
+      SRI_ARR(DSCC_MEM_POWER_CONTROL, DSCC, id),                               \
+      SRI_ARR(DSCC_R_Y_SQUARED_ERROR_LOWER, DSCC, id),                         \
+      SRI_ARR(DSCC_R_Y_SQUARED_ERROR_UPPER, DSCC, id),                         \
+      SRI_ARR(DSCC_G_CB_SQUARED_ERROR_LOWER, DSCC, id),                        \
+      SRI_ARR(DSCC_G_CB_SQUARED_ERROR_UPPER, DSCC, id),                        \
+      SRI_ARR(DSCC_B_CR_SQUARED_ERROR_LOWER, DSCC, id),                        \
+      SRI_ARR(DSCC_B_CR_SQUARED_ERROR_UPPER, DSCC, id),                        \
+      SRI_ARR(DSCC_MAX_ABS_ERROR0, DSCC, id),                                  \
+      SRI_ARR(DSCC_MAX_ABS_ERROR1, DSCC, id),                                  \
+      SRI_ARR(DSCC_RATE_BUFFER0_MAX_FULLNESS_LEVEL, DSCC, id),                 \
+      SRI_ARR(DSCC_RATE_BUFFER1_MAX_FULLNESS_LEVEL, DSCC, id),                 \
+      SRI_ARR(DSCC_RATE_BUFFER2_MAX_FULLNESS_LEVEL, DSCC, id),                 \
+      SRI_ARR(DSCC_RATE_BUFFER3_MAX_FULLNESS_LEVEL, DSCC, id),                 \
+      SRI_ARR(DSCC_RATE_CONTROL_BUFFER0_MAX_FULLNESS_LEVEL, DSCC, id),         \
+      SRI_ARR(DSCC_RATE_CONTROL_BUFFER1_MAX_FULLNESS_LEVEL, DSCC, id),         \
+      SRI_ARR(DSCC_RATE_CONTROL_BUFFER2_MAX_FULLNESS_LEVEL, DSCC, id),         \
+      SRI_ARR(DSCC_RATE_CONTROL_BUFFER3_MAX_FULLNESS_LEVEL, DSCC, id),         \
+      SRI_ARR(DSCCIF_CONFIG0, DSCCIF, id),                                     \
+      SRI_ARR(DSCCIF_CONFIG1, DSCCIF, id),                                     \
+      SRI_ARR(DSCRM_DSC_FORWARD_CONFIG, DSCRM, id)                             \
+  )
+
+/* MPC */
+
+#define MPC_DWB_MUX_REG_LIST_DCN3_0_RI(inst)                                   \
+  SRII_DWB(DWB_MUX, MUX, MPC_DWB, inst)
+
+#define MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0_RI(inst)                            \
+  ( \
+  SRII(MUX, MPC_OUT, inst), VUPDATE_SRII(CUR, VUPDATE_LOCK_SET, inst)          \
+  )
+
+#define MPC_OUT_MUX_REG_LIST_DCN3_0_RI(inst)                                   \
+  ( \
+  MPC_OUT_MUX_COMMON_REG_LIST_DCN1_0_RI(inst), SRII(CSC_MODE, MPC_OUT, inst),  \
+      SRII(CSC_C11_C12_A, MPC_OUT, inst), SRII(CSC_C33_C34_A, MPC_OUT, inst),  \
+      SRII(CSC_C11_C12_B, MPC_OUT, inst), SRII(CSC_C33_C34_B, MPC_OUT, inst),  \
+      SRII(DENORM_CONTROL, MPC_OUT, inst),                                     \
+      SRII(DENORM_CLAMP_G_Y, MPC_OUT, inst),                                   \
+      SRII(DENORM_CLAMP_B_CB, MPC_OUT, inst), SR(MPC_OUT_CSC_COEF_FORMAT)      \
+  )
+
+#define MPC_COMMON_REG_LIST_DCN1_0_RI(inst)                                    \
+  ( \
+  SRII(MPCC_TOP_SEL, MPCC, inst), SRII(MPCC_BOT_SEL, MPCC, inst),              \
+      SRII(MPCC_CONTROL, MPCC, inst), SRII(MPCC_STATUS, MPCC, inst),           \
+      SRII(MPCC_OPP_ID, MPCC, inst), SRII(MPCC_BG_G_Y, MPCC, inst),            \
+      SRII(MPCC_BG_R_CR, MPCC, inst), SRII(MPCC_BG_B_CB, MPCC, inst),          \
+      SRII(MPCC_SM_CONTROL, MPCC, inst),                                       \
+      SRII(MPCC_UPDATE_LOCK_SEL, MPCC, inst)                                   \
+  )
+
+#define MPC_REG_LIST_DCN3_0_RI(inst)                                           \
+  ( \
+  MPC_COMMON_REG_LIST_DCN1_0_RI(inst), SRII(MPCC_TOP_GAIN, MPCC, inst),        \
+      SRII(MPCC_BOT_GAIN_INSIDE, MPCC, inst),                                  \
+      SRII(MPCC_BOT_GAIN_OUTSIDE, MPCC, inst),                                 \
+      SRII(MPCC_MEM_PWR_CTRL, MPCC, inst),                                     \
+      SRII(MPCC_OGAM_LUT_INDEX, MPCC_OGAM, inst),                              \
+      SRII(MPCC_OGAM_LUT_DATA, MPCC_OGAM, inst),                               \
+      SRII(MPCC_GAMUT_REMAP_COEF_FORMAT, MPCC_OGAM, inst),                     \
+      SRII(MPCC_GAMUT_REMAP_MODE, MPCC_OGAM, inst),                            \
+      SRII(MPC_GAMUT_REMAP_C11_C12_A, MPCC_OGAM, inst),                        \
+      SRII(MPC_GAMUT_REMAP_C33_C34_A, MPCC_OGAM, inst),                        \
+      SRII(MPC_GAMUT_REMAP_C11_C12_B, MPCC_OGAM, inst),                        \
+      SRII(MPC_GAMUT_REMAP_C33_C34_B, MPCC_OGAM, inst),                        \
+      SRII(MPCC_OGAM_RAMA_START_CNTL_B, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMA_START_CNTL_G, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMA_START_CNTL_R, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMA_START_SLOPE_CNTL_B, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMA_START_SLOPE_CNTL_G, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMA_START_SLOPE_CNTL_R, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMA_END_CNTL1_B, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_END_CNTL2_B, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_END_CNTL1_G, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_END_CNTL2_G, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_END_CNTL1_R, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_END_CNTL2_R, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMA_REGION_0_1, MPCC_OGAM, inst),                        \
+      SRII(MPCC_OGAM_RAMA_REGION_32_33, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMA_OFFSET_B, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMA_OFFSET_G, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMA_OFFSET_R, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMA_START_BASE_CNTL_B, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_RAMA_START_BASE_CNTL_G, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_RAMA_START_BASE_CNTL_R, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_RAMB_START_CNTL_B, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMB_START_CNTL_G, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMB_START_CNTL_R, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMB_START_SLOPE_CNTL_B, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMB_START_SLOPE_CNTL_G, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMB_START_SLOPE_CNTL_R, MPCC_OGAM, inst),                \
+      SRII(MPCC_OGAM_RAMB_END_CNTL1_B, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_END_CNTL2_B, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_END_CNTL1_G, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_END_CNTL2_G, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_END_CNTL1_R, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_END_CNTL2_R, MPCC_OGAM, inst),                       \
+      SRII(MPCC_OGAM_RAMB_REGION_0_1, MPCC_OGAM, inst),                        \
+      SRII(MPCC_OGAM_RAMB_REGION_32_33, MPCC_OGAM, inst),                      \
+      SRII(MPCC_OGAM_RAMB_OFFSET_B, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMB_OFFSET_G, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMB_OFFSET_R, MPCC_OGAM, inst),                          \
+      SRII(MPCC_OGAM_RAMB_START_BASE_CNTL_B, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_RAMB_START_BASE_CNTL_G, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_RAMB_START_BASE_CNTL_R, MPCC_OGAM, inst),                 \
+      SRII(MPCC_OGAM_CONTROL, MPCC_OGAM, inst),                                \
+      SRII(MPCC_OGAM_LUT_CONTROL, MPCC_OGAM, inst)                             \
+  )
+
+#define MPC_REG_LIST_DCN3_2_RI(inst) \
+	MPC_REG_LIST_DCN3_0_RI(inst),\
+	SRII(MPCC_MOVABLE_CM_LOCATION_CONTROL, MPCC, inst),\
+	SRII(MPCC_MCM_SHAPER_CONTROL, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_OFFSET_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_OFFSET_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_OFFSET_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_SCALE_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_SCALE_G_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_LUT_INDEX, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_LUT_DATA, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_START_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_END_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_0_1, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_2_3, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_4_5, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_6_7, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_8_9, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_10_11, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_12_13, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_14_15, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_16_17, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_18_19, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_20_21, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_22_23, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_24_25, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_26_27, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_28_29, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_30_31, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMA_REGION_32_33, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_START_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_END_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_0_1, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_2_3, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_4_5, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_6_7, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_8_9, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_10_11, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_12_13, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_14_15, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_16_17, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_18_19, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_20_21, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_22_23, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_24_25, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_26_27, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_28_29, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_30_31, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_SHAPER_RAMB_REGION_32_33, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_MODE, MPCC_MCM, inst), /*TODO: may need to add other 3DLUT regs*/\
+	SRII(MPCC_MCM_3DLUT_INDEX, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_DATA, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_DATA_30BIT, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_READ_WRITE_CONTROL, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_OUT_NORM_FACTOR, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_OUT_OFFSET_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_OUT_OFFSET_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_3DLUT_OUT_OFFSET_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_CONTROL, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_LUT_INDEX, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_LUT_DATA, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_LUT_CONTROL, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_SLOPE_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_START_BASE_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL1_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_END_CNTL2_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_OFFSET_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_0_1, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_2_3, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_4_5, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_6_7, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_8_9, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_10_11, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_12_13, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_14_15, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_16_17, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_18_19, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_20_21, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_22_23, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_24_25, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_26_27, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_28_29, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_30_31, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMA_REGION_32_33, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_SLOPE_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_START_BASE_CNTL_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL1_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_END_CNTL2_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_B, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_G, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_OFFSET_R, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_0_1, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_2_3, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_4_5, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_6_7, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_8_9, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_10_11, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_12_13, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_14_15, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_16_17, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_18_19, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_20_21, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_22_23, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_24_25, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_26_27, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_28_29, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_30_31, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_1DLUT_RAMB_REGION_32_33, MPCC_MCM, inst),\
+	SRII(MPCC_MCM_MEM_PWR_CTRL, MPCC_MCM, inst)
+
+/* OPTC */
+
+#define OPTC_COMMON_REG_LIST_DCN3_2_RI(inst)                                   \
+  ( \
+  SRI_ARR(OTG_VSTARTUP_PARAM, OTG, inst),                                      \
+      SRI_ARR(OTG_VUPDATE_PARAM, OTG, inst),                                   \
+      SRI_ARR(OTG_VREADY_PARAM, OTG, inst),                                    \
+      SRI_ARR(OTG_MASTER_UPDATE_LOCK, OTG, inst),                              \
+      SRI_ARR(OTG_GLOBAL_CONTROL0, OTG, inst),                                 \
+      SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),                                 \
+      SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),                                 \
+      SRI_ARR(OTG_GLOBAL_CONTROL4, OTG, inst),                                 \
+      SRI_ARR(OTG_DOUBLE_BUFFER_CONTROL, OTG, inst),                           \
+      SRI_ARR(OTG_H_TOTAL, OTG, inst),                                         \
+      SRI_ARR(OTG_H_BLANK_START_END, OTG, inst),                               \
+      SRI_ARR(OTG_H_SYNC_A, OTG, inst), SRI_ARR(OTG_H_SYNC_A_CNTL, OTG, inst), \
+      SRI_ARR(OTG_H_TIMING_CNTL, OTG, inst), SRI_ARR(OTG_V_TOTAL, OTG, inst),  \
+      SRI_ARR(OTG_V_BLANK_START_END, OTG, inst),                               \
+      SRI_ARR(OTG_V_SYNC_A, OTG, inst), SRI_ARR(OTG_V_SYNC_A_CNTL, OTG, inst), \
+      SRI_ARR(OTG_CONTROL, OTG, inst), SRI_ARR(OTG_STEREO_CONTROL, OTG, inst), \
+      SRI_ARR(OTG_3D_STRUCTURE_CONTROL, OTG, inst),                            \
+      SRI_ARR(OTG_STEREO_STATUS, OTG, inst),                                   \
+      SRI_ARR(OTG_V_TOTAL_MAX, OTG, inst),                                     \
+      SRI_ARR(OTG_V_TOTAL_MIN, OTG, inst),                                     \
+      SRI_ARR(OTG_V_TOTAL_CONTROL, OTG, inst),                                 \
+      SRI_ARR(OTG_TRIGA_CNTL, OTG, inst),                                      \
+      SRI_ARR(OTG_FORCE_COUNT_NOW_CNTL, OTG, inst),                            \
+      SRI_ARR(OTG_STATIC_SCREEN_CONTROL, OTG, inst),                           \
+      SRI_ARR(OTG_STATUS_FRAME_COUNT, OTG, inst),                              \
+      SRI_ARR(OTG_STATUS, OTG, inst), SRI_ARR(OTG_STATUS_POSITION, OTG, inst), \
+      SRI_ARR(OTG_NOM_VERT_POSITION, OTG, inst),                               \
+      SRI_ARR(OTG_M_CONST_DTO0, OTG, inst),                                    \
+      SRI_ARR(OTG_M_CONST_DTO1, OTG, inst),                                    \
+      SRI_ARR(OTG_CLOCK_CONTROL, OTG, inst),                                   \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT0_CONTROL, OTG, inst),                     \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT0_POSITION, OTG, inst),                    \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT1_CONTROL, OTG, inst),                     \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT1_POSITION, OTG, inst),                    \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT2_CONTROL, OTG, inst),                     \
+      SRI_ARR(OTG_VERTICAL_INTERRUPT2_POSITION, OTG, inst),                    \
+      SRI_ARR(OPTC_INPUT_CLOCK_CONTROL, ODM, inst),                            \
+      SRI_ARR(OPTC_DATA_SOURCE_SELECT, ODM, inst),                             \
+      SRI_ARR(OPTC_INPUT_GLOBAL_CONTROL, ODM, inst),                           \
+      SRI_ARR(CONTROL, VTG, inst), SRI_ARR(OTG_VERT_SYNC_CONTROL, OTG, inst),  \
+      SRI_ARR(OTG_GSL_CONTROL, OTG, inst), SRI_ARR(OTG_CRC_CNTL, OTG, inst),   \
+      SRI_ARR(OTG_CRC0_DATA_RG, OTG, inst),                                    \
+      SRI_ARR(OTG_CRC0_DATA_B, OTG, inst),                                     \
+      SRI_ARR(OTG_CRC0_WINDOWA_X_CONTROL, OTG, inst),                          \
+      SRI_ARR(OTG_CRC0_WINDOWA_Y_CONTROL, OTG, inst),                          \
+      SRI_ARR(OTG_CRC0_WINDOWB_X_CONTROL, OTG, inst),                          \
+      SRI_ARR(OTG_CRC0_WINDOWB_Y_CONTROL, OTG, inst),                          \
+      SR_ARR(GSL_SOURCE_SELECT, inst),                                         \
+      SRI_ARR(OTG_TRIGA_MANUAL_TRIG, OTG, inst),                               \
+      SRI_ARR(OTG_GLOBAL_CONTROL1, OTG, inst),                                 \
+      SRI_ARR(OTG_GLOBAL_CONTROL2, OTG, inst),                                 \
+      SRI_ARR(OTG_GSL_WINDOW_X, OTG, inst),                                    \
+      SRI_ARR(OTG_GSL_WINDOW_Y, OTG, inst),                                    \
+      SRI_ARR(OTG_VUPDATE_KEEPOUT, OTG, inst),                                 \
+      SRI_ARR(OTG_DSC_START_POSITION, OTG, inst),                              \
+      SRI_ARR(OTG_DRR_TRIGGER_WINDOW, OTG, inst),                              \
+      SRI_ARR(OTG_DRR_V_TOTAL_CHANGE, OTG, inst),                              \
+      SRI_ARR(OPTC_DATA_FORMAT_CONTROL, ODM, inst),                            \
+      SRI_ARR(OPTC_BYTES_PER_PIXEL, ODM, inst),                                \
+      SRI_ARR(OPTC_WIDTH_CONTROL, ODM, inst),                                  \
+      SRI_ARR(OPTC_MEMORY_CONFIG, ODM, inst),                                  \
+      SRI_ARR(OTG_DRR_CONTROL, OTG, inst)                                      \
+  )
+
+/* HUBP */
+
+#define HUBP_REG_LIST_DCN_VM_RI(id)                                            \
+  ( \
+  SRI_ARR(NOM_PARAMETERS_0, HUBPREQ, id),                                      \
+      SRI_ARR(NOM_PARAMETERS_1, HUBPREQ, id),                                  \
+      SRI_ARR(NOM_PARAMETERS_2, HUBPREQ, id),                                  \
+      SRI_ARR(NOM_PARAMETERS_3, HUBPREQ, id),                                  \
+      SRI_ARR(DCN_VM_MX_L1_TLB_CNTL, HUBPREQ, id)                              \
+  )
+
+#define HUBP_REG_LIST_DCN_RI(id)                                               \
+  ( \
+  SRI_ARR(DCHUBP_CNTL, HUBP, id), SRI_ARR(HUBPREQ_DEBUG_DB, HUBP, id),         \
+      SRI_ARR(HUBPREQ_DEBUG, HUBP, id), SRI_ARR(DCSURF_ADDR_CONFIG, HUBP, id), \
+      SRI_ARR(DCSURF_TILING_CONFIG, HUBP, id),                                 \
+      SRI_ARR(DCSURF_SURFACE_PITCH, HUBPREQ, id),                              \
+      SRI_ARR(DCSURF_SURFACE_PITCH_C, HUBPREQ, id),                            \
+      SRI_ARR(DCSURF_SURFACE_CONFIG, HUBP, id),                                \
+      SRI_ARR(DCSURF_FLIP_CONTROL, HUBPREQ, id),                               \
+      SRI_ARR(DCSURF_PRI_VIEWPORT_DIMENSION, HUBP, id),                        \
+      SRI_ARR(DCSURF_PRI_VIEWPORT_START, HUBP, id),                            \
+      SRI_ARR(DCSURF_SEC_VIEWPORT_DIMENSION, HUBP, id),                        \
+      SRI_ARR(DCSURF_SEC_VIEWPORT_START, HUBP, id),                            \
+      SRI_ARR(DCSURF_PRI_VIEWPORT_DIMENSION_C, HUBP, id),                      \
+      SRI_ARR(DCSURF_PRI_VIEWPORT_START_C, HUBP, id),                          \
+      SRI_ARR(DCSURF_SEC_VIEWPORT_DIMENSION_C, HUBP, id),                      \
+      SRI_ARR(DCSURF_SEC_VIEWPORT_START_C, HUBP, id),                          \
+      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH, HUBPREQ, id),               \
+      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS, HUBPREQ, id),                    \
+      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH, HUBPREQ, id),             \
+      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS, HUBPREQ, id),                  \
+      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH, HUBPREQ, id),          \
+      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS, HUBPREQ, id),               \
+      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH, HUBPREQ, id),        \
+      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS, HUBPREQ, id),             \
+      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),             \
+      SRI_ARR(DCSURF_PRIMARY_SURFACE_ADDRESS_C, HUBPREQ, id),                  \
+      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),           \
+      SRI_ARR(DCSURF_SECONDARY_SURFACE_ADDRESS_C, HUBPREQ, id),                \
+      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),        \
+      SRI_ARR(DCSURF_PRIMARY_META_SURFACE_ADDRESS_C, HUBPREQ, id),             \
+      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_HIGH_C, HUBPREQ, id),      \
+      SRI_ARR(DCSURF_SECONDARY_META_SURFACE_ADDRESS_C, HUBPREQ, id),           \
+      SRI_ARR(DCSURF_SURFACE_INUSE, HUBPREQ, id),                              \
+      SRI_ARR(DCSURF_SURFACE_INUSE_HIGH, HUBPREQ, id),                         \
+      SRI_ARR(DCSURF_SURFACE_INUSE_C, HUBPREQ, id),                            \
+      SRI_ARR(DCSURF_SURFACE_INUSE_HIGH_C, HUBPREQ, id),                       \
+      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE, HUBPREQ, id),                     \
+      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_HIGH, HUBPREQ, id),                \
+      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_C, HUBPREQ, id),                   \
+      SRI_ARR(DCSURF_SURFACE_EARLIEST_INUSE_HIGH_C, HUBPREQ, id),              \
+      SRI_ARR(DCSURF_SURFACE_CONTROL, HUBPREQ, id),                            \
+      SRI_ARR(DCSURF_SURFACE_FLIP_INTERRUPT, HUBPREQ, id),                     \
+      SRI_ARR(HUBPRET_CONTROL, HUBPRET, id),                                   \
+      SRI_ARR(HUBPRET_READ_LINE_STATUS, HUBPRET, id),                          \
+      SRI_ARR(DCN_EXPANSION_MODE, HUBPREQ, id),                                \
+      SRI_ARR(DCHUBP_REQ_SIZE_CONFIG, HUBP, id),                               \
+      SRI_ARR(DCHUBP_REQ_SIZE_CONFIG_C, HUBP, id),                             \
+      SRI_ARR(BLANK_OFFSET_0, HUBPREQ, id),                                    \
+      SRI_ARR(BLANK_OFFSET_1, HUBPREQ, id),                                    \
+      SRI_ARR(DST_DIMENSIONS, HUBPREQ, id),                                    \
+      SRI_ARR(DST_AFTER_SCALER, HUBPREQ, id),                                  \
+      SRI_ARR(VBLANK_PARAMETERS_0, HUBPREQ, id),                               \
+      SRI_ARR(REF_FREQ_TO_PIX_FREQ, HUBPREQ, id),                              \
+      SRI_ARR(VBLANK_PARAMETERS_1, HUBPREQ, id),                               \
+      SRI_ARR(VBLANK_PARAMETERS_3, HUBPREQ, id),                               \
+      SRI_ARR(NOM_PARAMETERS_4, HUBPREQ, id),                                  \
+      SRI_ARR(NOM_PARAMETERS_5, HUBPREQ, id),                                  \
+      SRI_ARR(PER_LINE_DELIVERY_PRE, HUBPREQ, id),                             \
+      SRI_ARR(PER_LINE_DELIVERY, HUBPREQ, id),                                 \
+      SRI_ARR(VBLANK_PARAMETERS_2, HUBPREQ, id),                               \
+      SRI_ARR(VBLANK_PARAMETERS_4, HUBPREQ, id),                               \
+      SRI_ARR(NOM_PARAMETERS_6, HUBPREQ, id),                                  \
+      SRI_ARR(NOM_PARAMETERS_7, HUBPREQ, id),                                  \
+      SRI_ARR(DCN_TTU_QOS_WM, HUBPREQ, id),                                    \
+      SRI_ARR(DCN_GLOBAL_TTU_CNTL, HUBPREQ, id),                               \
+      SRI_ARR(DCN_SURF0_TTU_CNTL0, HUBPREQ, id),                               \
+      SRI_ARR(DCN_SURF0_TTU_CNTL1, HUBPREQ, id),                               \
+      SRI_ARR(DCN_SURF1_TTU_CNTL0, HUBPREQ, id),                               \
+      SRI_ARR(DCN_SURF1_TTU_CNTL1, HUBPREQ, id),                               \
+      SRI_ARR(DCN_CUR0_TTU_CNTL0, HUBPREQ, id),                                \
+      SRI_ARR(DCN_CUR0_TTU_CNTL1, HUBPREQ, id),                                \
+      SRI_ARR(HUBP_CLK_CNTL, HUBP, id)                                         \
+  )
+
+#define HUBP_REG_LIST_DCN2_COMMON_RI(id)                                       \
+  ( \
+  HUBP_REG_LIST_DCN_RI(id), HUBP_REG_LIST_DCN_VM_RI(id),                       \
+      SRI_ARR(PREFETCH_SETTINGS, HUBPREQ, id),                                 \
+      SRI_ARR(PREFETCH_SETTINGS_C, HUBPREQ, id),                               \
+      SRI_ARR(DCN_VM_SYSTEM_APERTURE_LOW_ADDR, HUBPREQ, id),                   \
+      SRI_ARR(DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, HUBPREQ, id),                  \
+      SRI_ARR(CURSOR_SETTINGS, HUBPREQ, id),                                   \
+      SRI_ARR(CURSOR_SURFACE_ADDRESS_HIGH, CURSOR0_, id),                      \
+      SRI_ARR(CURSOR_SURFACE_ADDRESS, CURSOR0_, id),                           \
+      SRI_ARR(CURSOR_SIZE, CURSOR0_, id),                                      \
+      SRI_ARR(CURSOR_CONTROL, CURSOR0_, id),                                   \
+      SRI_ARR(CURSOR_POSITION, CURSOR0_, id),                                  \
+      SRI_ARR(CURSOR_HOT_SPOT, CURSOR0_, id),                                  \
+      SRI_ARR(CURSOR_DST_OFFSET, CURSOR0_, id),                                \
+      SRI_ARR(DMDATA_ADDRESS_HIGH, CURSOR0_, id),                              \
+      SRI_ARR(DMDATA_ADDRESS_LOW, CURSOR0_, id),                               \
+      SRI_ARR(DMDATA_CNTL, CURSOR0_, id),                                      \
+      SRI_ARR(DMDATA_SW_CNTL, CURSOR0_, id),                                   \
+      SRI_ARR(DMDATA_QOS_CNTL, CURSOR0_, id),                                  \
+      SRI_ARR(DMDATA_SW_DATA, CURSOR0_, id),                                   \
+      SRI_ARR(DMDATA_STATUS, CURSOR0_, id),                                    \
+      SRI_ARR(FLIP_PARAMETERS_0, HUBPREQ, id),                                 \
+      SRI_ARR(FLIP_PARAMETERS_1, HUBPREQ, id),                                 \
+      SRI_ARR(FLIP_PARAMETERS_2, HUBPREQ, id),                                 \
+      SRI_ARR(DCN_CUR1_TTU_CNTL0, HUBPREQ, id),                                \
+      SRI_ARR(DCN_CUR1_TTU_CNTL1, HUBPREQ, id),                                \
+      SRI_ARR(DCSURF_FLIP_CONTROL2, HUBPREQ, id),                              \
+      SRI_ARR(VMID_SETTINGS_0, HUBPREQ, id)                                    \
+  )
+
+#define HUBP_REG_LIST_DCN21_RI(id)                                             \
+  ( \
+  HUBP_REG_LIST_DCN2_COMMON_RI(id), SRI_ARR(FLIP_PARAMETERS_3, HUBPREQ, id),   \
+      SRI_ARR(FLIP_PARAMETERS_4, HUBPREQ, id),                                 \
+      SRI_ARR(FLIP_PARAMETERS_5, HUBPREQ, id),                                 \
+      SRI_ARR(FLIP_PARAMETERS_6, HUBPREQ, id),                                 \
+      SRI_ARR(VBLANK_PARAMETERS_5, HUBPREQ, id),                               \
+      SRI_ARR(VBLANK_PARAMETERS_6, HUBPREQ, id)                                \
+  )
+
+#define HUBP_REG_LIST_DCN30_RI(id)                                             \
+  ( \
+  HUBP_REG_LIST_DCN21_RI(id), SRI_ARR(DCN_DMDATA_VM_CNTL, HUBPREQ, id)         \
+  )
+
+#define HUBP_REG_LIST_DCN32_RI(id)                                             \
+  ( \
+  HUBP_REG_LIST_DCN30_RI(id), SRI_ARR(DCHUBP_MALL_CONFIG, HUBP, id),           \
+      SRI_ARR(DCHUBP_VMPG_CONFIG, HUBP, id),                                   \
+      SRI_ARR(UCLK_PSTATE_FORCE, HUBPREQ, id)                                  \
+  )
+
+/* HUBBUB */
+
+#define HUBBUB_REG_LIST_DCN32_RI(id)                                           \
+  ( \
+  SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_A),                                   \
+      SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_B),                               \
+      SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_C),                               \
+      SR(DCHUBBUB_ARB_DATA_URGENCY_WATERMARK_D),                               \
+      SR(DCHUBBUB_ARB_WATERMARK_CHANGE_CNTL),                                  \
+      SR(DCHUBBUB_ARB_DRAM_STATE_CNTL), SR(DCHUBBUB_ARB_SAT_LEVEL),            \
+      SR(DCHUBBUB_ARB_DF_REQ_OUTSTAND), SR(DCHUBBUB_GLOBAL_TIMER_CNTL),        \
+      SR(DCHUBBUB_SOFT_RESET), SR(DCHUBBUB_CRC_CTRL),                          \
+      SR(DCN_VM_FB_LOCATION_BASE), SR(DCN_VM_FB_LOCATION_TOP),                 \
+      SR(DCN_VM_FB_OFFSET), SR(DCN_VM_AGP_BOT), SR(DCN_VM_AGP_TOP),            \
+      SR(DCN_VM_AGP_BASE), HUBBUB_SR_WATERMARK_REG_LIST(),                     \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_A), SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_B),  \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_C), SR(DCHUBBUB_ARB_FRAC_URG_BW_NOM_D),  \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_A),                                     \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_B),                                     \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_C),                                     \
+      SR(DCHUBBUB_ARB_FRAC_URG_BW_FLIP_D),                                     \
+      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_A),                            \
+      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_B),                            \
+      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_C),                            \
+      SR(DCHUBBUB_ARB_REFCYC_PER_TRIP_TO_MEMORY_D), SR(DCHUBBUB_DET0_CTRL),    \
+      SR(DCHUBBUB_DET1_CTRL), SR(DCHUBBUB_DET2_CTRL), SR(DCHUBBUB_DET3_CTRL),  \
+      SR(DCHUBBUB_COMPBUF_CTRL), SR(COMPBUF_RESERVED_SPACE),                   \
+      SR(DCHUBBUB_DEBUG_CTRL_0),                                               \
+      SR(DCHUBBUB_ARB_USR_RETRAINING_CNTL),                                    \
+      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_A),                             \
+      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_B),                             \
+      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_C),                             \
+      SR(DCHUBBUB_ARB_USR_RETRAINING_WATERMARK_D),                             \
+      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_A),                         \
+      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_B),                         \
+      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_C),                         \
+      SR(DCHUBBUB_ARB_UCLK_PSTATE_CHANGE_WATERMARK_D),                         \
+      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_A),                         \
+      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_B),                         \
+      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_C),                         \
+      SR(DCHUBBUB_ARB_FCLK_PSTATE_CHANGE_WATERMARK_D),                         \
+      SR(DCN_VM_FAULT_ADDR_MSB), SR(DCN_VM_FAULT_ADDR_LSB),                    \
+      SR(DCN_VM_FAULT_CNTL), SR(DCN_VM_FAULT_STATUS)                           \
+  )
+
+/* DCCG */
+
+#define DCCG_REG_LIST_DCN32_RI()                                               \
+  ( \
+  SR(DPPCLK_DTO_CTRL), DCCG_SRII(DTO_PARAM, DPPCLK, 0),                        \
+      DCCG_SRII(DTO_PARAM, DPPCLK, 1), DCCG_SRII(DTO_PARAM, DPPCLK, 2),        \
+      DCCG_SRII(DTO_PARAM, DPPCLK, 3), DCCG_SRII(CLOCK_CNTL, HDMICHARCLK, 0),  \
+      SR(PHYASYMCLK_CLOCK_CNTL), SR(PHYBSYMCLK_CLOCK_CNTL),                    \
+      SR(PHYCSYMCLK_CLOCK_CNTL), SR(PHYDSYMCLK_CLOCK_CNTL),                    \
+      SR(PHYESYMCLK_CLOCK_CNTL), SR(DPSTREAMCLK_CNTL), SR(HDMISTREAMCLK_CNTL), \
+      SR(SYMCLK32_SE_CNTL), SR(SYMCLK32_LE_CNTL),                              \
+      DCCG_SRII(PIXEL_RATE_CNTL, OTG, 0), DCCG_SRII(PIXEL_RATE_CNTL, OTG, 1),  \
+      DCCG_SRII(PIXEL_RATE_CNTL, OTG, 2), DCCG_SRII(PIXEL_RATE_CNTL, OTG, 3),  \
+      DCCG_SRII(MODULO, DTBCLK_DTO, 0), DCCG_SRII(MODULO, DTBCLK_DTO, 1),      \
+      DCCG_SRII(MODULO, DTBCLK_DTO, 2), DCCG_SRII(MODULO, DTBCLK_DTO, 3),      \
+      DCCG_SRII(PHASE, DTBCLK_DTO, 0), DCCG_SRII(PHASE, DTBCLK_DTO, 1),        \
+      DCCG_SRII(PHASE, DTBCLK_DTO, 2), DCCG_SRII(PHASE, DTBCLK_DTO, 3),        \
+      SR(DCCG_AUDIO_DTBCLK_DTO_MODULO), SR(DCCG_AUDIO_DTBCLK_DTO_PHASE),       \
+      SR(OTG_PIXEL_RATE_DIV), SR(DTBCLK_P_CNTL), SR(DCCG_AUDIO_DTO_SOURCE)     \
+  )
+
+/* VMID */
+#define DCN20_VMID_REG_LIST_RI(id)                                             \
+  ( \
+  SRI_ARR(CNTL, DCN_VM_CONTEXT, id),                                           \
+      SRI_ARR(PAGE_TABLE_BASE_ADDR_HI32, DCN_VM_CONTEXT, id),                  \
+      SRI_ARR(PAGE_TABLE_BASE_ADDR_LO32, DCN_VM_CONTEXT, id),                  \
+      SRI_ARR(PAGE_TABLE_START_ADDR_HI32, DCN_VM_CONTEXT, id),                 \
+      SRI_ARR(PAGE_TABLE_START_ADDR_LO32, DCN_VM_CONTEXT, id),                 \
+      SRI_ARR(PAGE_TABLE_END_ADDR_HI32, DCN_VM_CONTEXT, id),                   \
+      SRI_ARR(PAGE_TABLE_END_ADDR_LO32, DCN_VM_CONTEXT, id)                    \
+  )
+
+/* I2C HW */
+
+#define I2C_HW_ENGINE_COMMON_REG_LIST_RI(id)                                   \
+  ( \
+      SRI_ARR_I2C(SETUP, DC_I2C_DDC, id), SRI_ARR_I2C(SPEED, DC_I2C_DDC, id),  \
+      SRI_ARR_I2C(HW_STATUS, DC_I2C_DDC, id),                                  \
+      SR_ARR_I2C(DC_I2C_ARBITRATION, id),                                      \
+      SR_ARR_I2C(DC_I2C_CONTROL, id), SR_ARR_I2C(DC_I2C_SW_STATUS, id),        \
+      SR_ARR_I2C(DC_I2C_TRANSACTION0, id), SR_ARR_I2C(DC_I2C_TRANSACTION1, id),\
+      SR_ARR_I2C(DC_I2C_TRANSACTION2, id), SR_ARR_I2C(DC_I2C_TRANSACTION3, id),\
+      SR_ARR_I2C(DC_I2C_DATA, id), SR_ARR_I2C(MICROSECOND_TIME_BASE_DIV, id)          \
+  )
+
+#define I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)                             \
+  ( \
+      I2C_HW_ENGINE_COMMON_REG_LIST_RI(id), SR_ARR_I2C(DIO_MEM_PWR_CTRL, id),  \
+      SR_ARR_I2C(DIO_MEM_PWR_STATUS, id)                                           \
+  )
 
 #endif /* _DCN32_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
index 1f195c5b3377..a2a70a1572b7 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn32/dcn32_resource_helpers.c
@@ -28,6 +28,11 @@
 #include "dcn20/dcn20_resource.h"
 #include "dml/dcn32/display_mode_vba_util_32.h"
 
+static bool is_dual_plane(enum surface_pixel_format format)
+{
+	return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA;
+}
+
 /**
  * ********************************************************************************************
  * dcn32_helper_calculate_num_ways_for_subvp: Calculate number of ways needed for SubVP
@@ -54,22 +59,27 @@ uint32_t dcn32_helper_calculate_num_ways_for_subvp(struct dc *dc, struct dc_stat
 	uint32_t num_mblks = 0;
 	uint32_t cache_lines_per_plane = 0;
 	uint32_t i = 0, j = 0;
-	uint32_t mblk_width = 0;
-	uint32_t mblk_height = 0;
+	uint16_t mblk_width = 0;
+	uint16_t mblk_height = 0;
 	uint32_t full_vp_width_blk_aligned = 0;
 	uint32_t full_vp_height_blk_aligned = 0;
 	uint32_t mall_alloc_width_blk_aligned = 0;
 	uint32_t mall_alloc_height_blk_aligned = 0;
-	uint32_t full_vp_height = 0;
+	uint16_t full_vp_height = 0;
+	bool subvp_in_use = false;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
 
-		// Find the phantom pipes
-		if (pipe->stream && pipe->plane_state && !pipe->top_pipe && !pipe->prev_odm_pipe &&
+		/* Find the phantom pipes.
+		 * - For pipe split case we need to loop through the bottom and next ODM
+		 *   pipes or only half the viewport size is counted
+		 */
+		if (pipe->stream && pipe->plane_state &&
 				pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
 			struct pipe_ctx *main_pipe = NULL;
 
+			subvp_in_use = true;
 			/* Get full viewport height from main pipe (required for MBLK calculation) */
 			for (j = 0; j < dc->res_pool->pipe_count; j++) {
 				main_pipe = &context->res_ctx.pipe_ctx[j];
@@ -116,9 +126,9 @@ uint32_t dcn32_helper_calculate_num_ways_for_subvp(struct dc *dc, struct dc_stat
 			// (MALL is 64-byte aligned)
 			cache_lines_per_plane = bytes_in_mall / dc->caps.cache_line_size + 2;
 
-			// For DCC we must cache the meat surface, so double cache lines required
+			/* For DCC divide by 256 */
 			if (pipe->plane_state->dcc.enable)
-				cache_lines_per_plane *= 2;
+				cache_lines_per_plane = cache_lines_per_plane + (cache_lines_per_plane / 256) + 1;
 			cache_lines_used += cache_lines_per_plane;
 		}
 	}
@@ -129,6 +139,9 @@ uint32_t dcn32_helper_calculate_num_ways_for_subvp(struct dc *dc, struct dc_stat
 	if (cache_lines_used % lines_per_way > 0)
 		num_ways++;
 
+	if (subvp_in_use && dc->debug.force_subvp_num_ways > 0)
+		num_ways = dc->debug.force_subvp_num_ways;
+
 	return num_ways;
 }
 
@@ -220,36 +233,133 @@ bool dcn32_mpo_in_use(struct dc_state *context)
 	return false;
 }
 
-void dcn32_determine_det_override(struct dc_state *context, display_e2e_pipe_params_st *pipes,
-		bool *is_pipe_split_expected, int pipe_cnt)
+/**
+ * *******************************************************************************************
+ * dcn32_determine_det_override: Determine DET allocation for each pipe
+ *
+ * This function determines how much DET to allocate for each pipe. The total number of
+ * DET segments will be split equally among each of the streams, and after that the DET
+ * segments per stream will be split equally among the planes for the given stream.
+ *
+ * If there is a plane that's driven by more than 1 pipe (i.e. pipe split), then the
+ * number of DET for that given plane will be split among the pipes driving that plane.
+ *
+ *
+ * High level algorithm:
+ * 1. Split total DET among number of streams
+ * 2. For each stream, split DET among the planes
+ * 3. For each plane, check if there is a pipe split. If yes, split the DET allocation
+ *    among those pipes.
+ * 4. Assign the DET override to the DML pipes.
+ *
+ * @param [in]: dc: Current DC state
+ * @param [in]: context: New DC state to be programmed
+ * @param [in]: pipes: Array of DML pipes
+ *
+ * @return: void
+ *
+ * *******************************************************************************************
+ */
+void dcn32_determine_det_override(struct dc *dc,
+		struct dc_state *context,
+		display_e2e_pipe_params_st *pipes)
 {
-	int i, j, count, stream_segments, pipe_segments[MAX_PIPES];
+	uint32_t i, j, k;
+	uint8_t pipe_plane_count, stream_segments, plane_segments, pipe_segments[MAX_PIPES] = {0};
+	uint8_t pipe_counted[MAX_PIPES] = {0};
+	uint8_t pipe_cnt = 0;
+	struct dc_plane_state *current_plane = NULL;
+	uint8_t stream_count = 0;
 
-	if (context->stream_count > 0) {
-		stream_segments = 18 / context->stream_count;
+	for (i = 0; i < context->stream_count; i++) {
+		/* Don't count SubVP streams for DET allocation */
+		if (context->streams[i]->mall_stream_config.type != SUBVP_PHANTOM) {
+			stream_count++;
+		}
+	}
+
+	if (stream_count > 0) {
+		stream_segments = 18 / stream_count;
 		for (i = 0; i < context->stream_count; i++) {
-			count = 0;
-			for (j = 0; j < pipe_cnt; j++) {
-				if (context->res_ctx.pipe_ctx[j].stream == context->streams[i]) {
-					count++;
-					if (is_pipe_split_expected[j])
-						count++;
+			if (context->streams[i]->mall_stream_config.type == SUBVP_PHANTOM)
+				continue;
+			if (context->stream_status[i].plane_count > 0)
+				plane_segments = stream_segments / context->stream_status[i].plane_count;
+			else
+				plane_segments = stream_segments;
+			for (j = 0; j < dc->res_pool->pipe_count; j++) {
+				pipe_plane_count = 0;
+				if (context->res_ctx.pipe_ctx[j].stream == context->streams[i] &&
+						pipe_counted[j] != 1) {
+					/* Note: pipe_plane_count indicates the number of pipes to be used for a
+					 * given plane. e.g. pipe_plane_count = 1 means single pipe (i.e. not split),
+					 * pipe_plane_count = 2 means 2:1 split, etc.
+					 */
+					pipe_plane_count++;
+					pipe_counted[j] = 1;
+					current_plane = context->res_ctx.pipe_ctx[j].plane_state;
+					for (k = 0; k < dc->res_pool->pipe_count; k++) {
+						if (k != j && context->res_ctx.pipe_ctx[k].stream == context->streams[i] &&
+								context->res_ctx.pipe_ctx[k].plane_state == current_plane) {
+							pipe_plane_count++;
+							pipe_counted[k] = 1;
+						}
+					}
+
+					pipe_segments[j] = plane_segments / pipe_plane_count;
+					for (k = 0; k < dc->res_pool->pipe_count; k++) {
+						if (k != j && context->res_ctx.pipe_ctx[k].stream == context->streams[i] &&
+								context->res_ctx.pipe_ctx[k].plane_state == current_plane) {
+							pipe_segments[k] = plane_segments / pipe_plane_count;
+						}
+					}
 				}
 			}
-			pipe_segments[i] = stream_segments / count;
 		}
 
-		for (i = 0; i < pipe_cnt; i++) {
-			pipes[i].pipe.src.det_size_override = 0;
-			for (j = 0; j < context->stream_count; j++) {
-				if (context->res_ctx.pipe_ctx[i].stream == context->streams[j]) {
-					pipes[i].pipe.src.det_size_override = pipe_segments[j] * DCN3_2_DET_SEG_SIZE;
-					break;
-				}
-			}
+		for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+			if (!context->res_ctx.pipe_ctx[i].stream)
+				continue;
+			pipes[pipe_cnt].pipe.src.det_size_override = pipe_segments[i] * DCN3_2_DET_SEG_SIZE;
+			pipe_cnt++;
 		}
 	} else {
-		for (i = 0; i < pipe_cnt; i++)
+		for (i = 0; i < dc->res_pool->pipe_count; i++)
 			pipes[i].pipe.src.det_size_override = 4 * DCN3_2_DET_SEG_SIZE; //DCN3_2_DEFAULT_DET_SIZE
 	}
 }
+
+void dcn32_set_det_allocations(struct dc *dc, struct dc_state *context,
+	display_e2e_pipe_params_st *pipes)
+{
+	int i, pipe_cnt;
+	struct resource_context *res_ctx = &context->res_ctx;
+	struct pipe_ctx *pipe;
+
+	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
+
+		if (!res_ctx->pipe_ctx[i].stream)
+			continue;
+
+		pipe = &res_ctx->pipe_ctx[i];
+		pipe_cnt++;
+	}
+
+	/* For DET allocation, we don't want to use DML policy (not optimal for utilizing all
+	 * the DET available for each pipe). Use the DET override input to maintain our driver
+	 * policy.
+	 */
+	if (pipe_cnt == 1) {
+		pipes[0].pipe.src.det_size_override = DCN3_2_MAX_DET_SIZE;
+		if (pipe->plane_state && !dc->debug.disable_z9_mpc && pipe->plane_state->tiling_info.gfx9.swizzle != DC_SW_LINEAR) {
+			if (!is_dual_plane(pipe->plane_state->format)) {
+				pipes[0].pipe.src.det_size_override = DCN3_2_DEFAULT_DET_SIZE;
+				pipes[0].pipe.src.unbounded_req_mode = true;
+				if (pipe->plane_state->src_rect.width >= 5120 &&
+					pipe->plane_state->src_rect.height >= 2880)
+					pipes[0].pipe.src.det_size_override = 320; // 5K or higher
+			}
+		}
+	} else
+		dcn32_determine_det_override(dc, context, pipes);
+}
diff --git a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c
index 7309eed33a61..aed0f689cbbf 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn321/dcn321_resource.c
@@ -93,31 +93,6 @@
 #include "vm_helper.h"
 #include "dcn20/dcn20_vmid.h"
 
-#define DCN_BASE__INST0_SEG1                       0x000000C0
-#define DCN_BASE__INST0_SEG2                       0x000034C0
-#define DCN_BASE__INST0_SEG3                       0x00009000
-#define NBIO_BASE__INST0_SEG1                      0x00000014
-
-#define MAX_INSTANCE                                        8
-#define MAX_SEGMENT                                         6
-
-struct IP_BASE_INSTANCE {
-	unsigned int segment[MAX_SEGMENT];
-};
-
-struct IP_BASE {
-	struct IP_BASE_INSTANCE instance[MAX_INSTANCE];
-};
-
-static const struct IP_BASE DCN_BASE = { { { { 0x00000012, 0x000000C0, 0x000034C0, 0x00009000, 0x02403C00, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } },
-					{ { 0, 0, 0, 0, 0, 0 } } } };
-
 #define DC_LOGGER_INIT(logger)
 #define fixed16_to_double(x) (((double)x) / ((double) (1 << 16)))
 #define fixed16_to_double_to_cpu(x) fixed16_to_double(le32_to_cpu(x))
@@ -138,78 +113,102 @@ enum dcn321_clk_src_array_id {
 /* DCN */
 /* TODO awful hack. fixup dcn20_dwb.h */
 #undef BASE_INNER
-#define BASE_INNER(seg) DCN_BASE__INST0_SEG ## seg
+#define BASE_INNER(seg) ctx->dcn_reg_offsets[seg]
 
 #define BASE(seg) BASE_INNER(seg)
 
 #define SR(reg_name)\
-		.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
-					reg ## reg_name
+	REG_STRUCT.reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+		reg ## reg_name
+#define SR_ARR(reg_name, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) +  \
+		reg ## reg_name
+#define SR_ARR_INIT(reg_name, id, value)\
+	REG_STRUCT[id].reg_name =  value
 
 #define SRI(reg_name, block, id)\
-	.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
+	REG_STRUCT.reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI_ARR(reg_name, block, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SR_ARR_I2C(reg_name, id) \
+	REG_STRUCT[id-1].reg_name = BASE(reg##reg_name##_BASE_IDX) + reg##reg_name
+
+#define SRI_ARR_I2C(reg_name, block, id)\
+	REG_STRUCT[id-1].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRI_ARR_ALPHABET(reg_name, block, index, id)\
+	REG_STRUCT[index].reg_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
 
 #define SRI2(reg_name, block, id)\
 	.reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
-					reg ## reg_name
+		reg ## reg_name
+#define SRI2_ARR(reg_name, block, id)\
+	REG_STRUCT[id].reg_name = BASE(reg ## reg_name ## _BASE_IDX) + \
+		reg ## reg_name
 
 #define SRIR(var_name, reg_name, block, id)\
 	.var_name = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
+		reg ## block ## id ## _ ## reg_name
 
 #define SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
+
+#define SRII_ARR_2(reg_name, block, id, inst)\
+	REG_STRUCT[inst].reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
 
 #define SRII_MPC_RMU(reg_name, block, id)\
 	.RMU##_##reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
+		reg ## block ## id ## _ ## reg_name
 
 #define SRII_DWB(reg_name, temp_name, block, id)\
-	.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## temp_name
+	REG_STRUCT.reg_name[id] = BASE(reg ## block ## id ## _ ## temp_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## temp_name
 
 #define DCCG_SRII(reg_name, block, id)\
-	.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
-					reg ## block ## id ## _ ## reg_name
+	REG_STRUCT.block ## _ ## reg_name[id] = BASE(reg ## block ## id ## _ ## reg_name ## _BASE_IDX) + \
+		reg ## block ## id ## _ ## reg_name
 
 #define VUPDATE_SRII(reg_name, block, id)\
-	.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
-					reg ## reg_name ## _ ## block ## id
+	REG_STRUCT.reg_name[id] = BASE(reg ## reg_name ## _ ## block ## id ## _BASE_IDX) + \
+		reg ## reg_name ## _ ## block ## id
 
 /* NBIO */
-#define NBIO_BASE_INNER(seg) \
-	NBIO_BASE__INST0_SEG ## seg
+#define NBIO_BASE_INNER(seg) ctx->nbio_reg_offsets[seg]
 
 #define NBIO_BASE(seg) \
 	NBIO_BASE_INNER(seg)
 
 #define NBIO_SR(reg_name)\
-		.reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
-					regBIF_BX0_ ## reg_name
+	REG_STRUCT.reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
+		regBIF_BX0_ ## reg_name
+#define NBIO_SR_ARR(reg_name, id)\
+	REG_STRUCT[id].reg_name = NBIO_BASE(regBIF_BX0_ ## reg_name ## _BASE_IDX) + \
+		regBIF_BX0_ ## reg_name
 
 #define CTX ctx
 #define REG(reg_name) \
-	(DCN_BASE.instance[0].segment[reg ## reg_name ## _BASE_IDX] + reg ## reg_name)
+	(ctx->dcn_reg_offsets[reg ## reg_name ## _BASE_IDX] + reg ## reg_name)
 
-static const struct bios_registers bios_regs = {
-		NBIO_SR(BIOS_SCRATCH_3),
-		NBIO_SR(BIOS_SCRATCH_6)
-};
+static struct bios_registers bios_regs;
 
-#define clk_src_regs(index, pllid)\
-[index] = {\
-	CS_COMMON_REG_LIST_DCN3_0(index, pllid),\
-}
+#define bios_regs_init() \
+		( \
+		NBIO_SR(BIOS_SCRATCH_3),\
+		NBIO_SR(BIOS_SCRATCH_6)\
+		)
 
-static const struct dce110_clk_src_regs clk_src_regs[] = {
-	clk_src_regs(0, A),
-	clk_src_regs(1, B),
-	clk_src_regs(2, C),
-	clk_src_regs(3, D),
-	clk_src_regs(4, E)
-};
+#define clk_src_regs_init(index, pllid)\
+	CS_COMMON_REG_LIST_DCN3_0_RI(index, pllid)
+
+static struct dce110_clk_src_regs clk_src_regs[5];
 
 static const struct dce110_clk_src_shift cs_shift = {
 		CS_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
@@ -219,17 +218,10 @@ static const struct dce110_clk_src_mask cs_mask = {
 		CS_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
 };
 
-#define abm_regs(id)\
-[id] = {\
-		ABM_DCN32_REG_LIST(id)\
-}
+#define abm_regs_init(id)\
+		ABM_DCN32_REG_LIST_RI(id)
 
-static const struct dce_abm_registers abm_regs[] = {
-		abm_regs(0),
-		abm_regs(1),
-		abm_regs(2),
-		abm_regs(3),
-};
+static struct dce_abm_registers abm_regs[4];
 
 static const struct dce_abm_shift abm_shift = {
 		ABM_MASK_SH_LIST_DCN32(__SHIFT)
@@ -239,18 +231,10 @@ static const struct dce_abm_mask abm_mask = {
 		ABM_MASK_SH_LIST_DCN32(_MASK)
 };
 
-#define audio_regs(id)\
-[id] = {\
-		AUD_COMMON_REG_LIST(id)\
-}
+#define audio_regs_init(id)\
+		AUD_COMMON_REG_LIST_RI(id)
 
-static const struct dce_audio_registers audio_regs[] = {
-	audio_regs(0),
-	audio_regs(1),
-	audio_regs(2),
-	audio_regs(3),
-	audio_regs(4)
-};
+static struct dce_audio_registers audio_regs[5];
 
 #define DCE120_AUD_COMMON_MASK_SH_LIST(mask_sh)\
 		SF(AZF0ENDPOINT0_AZALIA_F0_CODEC_ENDPOINT_INDEX, AZALIA_ENDPOINT_REG_INDEX, mask_sh),\
@@ -265,23 +249,10 @@ static const struct dce_audio_mask audio_mask = {
 		DCE120_AUD_COMMON_MASK_SH_LIST(_MASK)
 };
 
-#define vpg_regs(id)\
-[id] = {\
-	VPG_DCN3_REG_LIST(id)\
-}
+#define vpg_regs_init(id)\
+	VPG_DCN3_REG_LIST_RI(id)
 
-static const struct dcn30_vpg_registers vpg_regs[] = {
-	vpg_regs(0),
-	vpg_regs(1),
-	vpg_regs(2),
-	vpg_regs(3),
-	vpg_regs(4),
-	vpg_regs(5),
-	vpg_regs(6),
-	vpg_regs(7),
-	vpg_regs(8),
-	vpg_regs(9),
-};
+static struct dcn30_vpg_registers vpg_regs[10];
 
 static const struct dcn30_vpg_shift vpg_shift = {
 	DCN3_VPG_MASK_SH_LIST(__SHIFT)
@@ -291,19 +262,10 @@ static const struct dcn30_vpg_mask vpg_mask = {
 	DCN3_VPG_MASK_SH_LIST(_MASK)
 };
 
-#define afmt_regs(id)\
-[id] = {\
-	AFMT_DCN3_REG_LIST(id)\
-}
+#define afmt_regs_init(id)\
+	AFMT_DCN3_REG_LIST_RI(id)
 
-static const struct dcn30_afmt_registers afmt_regs[] = {
-	afmt_regs(0),
-	afmt_regs(1),
-	afmt_regs(2),
-	afmt_regs(3),
-	afmt_regs(4),
-	afmt_regs(5)
-};
+static struct dcn30_afmt_registers afmt_regs[6];
 
 static const struct dcn30_afmt_shift afmt_shift = {
 	DCN3_AFMT_MASK_SH_LIST(__SHIFT)
@@ -313,17 +275,10 @@ static const struct dcn30_afmt_mask afmt_mask = {
 	DCN3_AFMT_MASK_SH_LIST(_MASK)
 };
 
-#define apg_regs(id)\
-[id] = {\
-	APG_DCN31_REG_LIST(id)\
-}
+#define apg_regs_init(id)\
+	APG_DCN31_REG_LIST_RI(id)
 
-static const struct dcn31_apg_registers apg_regs[] = {
-	apg_regs(0),
-	apg_regs(1),
-	apg_regs(2),
-	apg_regs(3)
-};
+static struct dcn31_apg_registers apg_regs[4];
 
 static const struct dcn31_apg_shift apg_shift = {
 	DCN31_APG_MASK_SH_LIST(__SHIFT)
@@ -333,18 +288,10 @@ static const struct dcn31_apg_mask apg_mask = {
 		DCN31_APG_MASK_SH_LIST(_MASK)
 };
 
-#define stream_enc_regs(id)\
-[id] = {\
-	SE_DCN32_REG_LIST(id)\
-}
+#define stream_enc_regs_init(id)\
+	SE_DCN32_REG_LIST_RI(id)
 
-static const struct dcn10_stream_enc_registers stream_enc_regs[] = {
-	stream_enc_regs(0),
-	stream_enc_regs(1),
-	stream_enc_regs(2),
-	stream_enc_regs(3),
-	stream_enc_regs(4)
-};
+static struct dcn10_stream_enc_registers stream_enc_regs[5];
 
 static const struct dcn10_stream_encoder_shift se_shift = {
 		SE_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
@@ -355,46 +302,24 @@ static const struct dcn10_stream_encoder_mask se_mask = {
 };
 
 
-#define aux_regs(id)\
-[id] = {\
-	DCN2_AUX_REG_LIST(id)\
-}
+#define aux_regs_init(id)\
+	DCN2_AUX_REG_LIST_RI(id)
 
-static const struct dcn10_link_enc_aux_registers link_enc_aux_regs[] = {
-		aux_regs(0),
-		aux_regs(1),
-		aux_regs(2),
-		aux_regs(3),
-		aux_regs(4)
-};
+static struct dcn10_link_enc_aux_registers link_enc_aux_regs[5];
 
-#define hpd_regs(id)\
-[id] = {\
-	HPD_REG_LIST(id)\
-}
+#define hpd_regs_init(id)\
+	HPD_REG_LIST_RI(id)
 
-static const struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[] = {
-		hpd_regs(0),
-		hpd_regs(1),
-		hpd_regs(2),
-		hpd_regs(3),
-		hpd_regs(4)
-};
+static struct dcn10_link_enc_hpd_registers link_enc_hpd_regs[5];
 
-#define link_regs(id, phyid)\
-[id] = {\
-	LE_DCN31_REG_LIST(id), \
-	UNIPHY_DCN2_REG_LIST(phyid), \
+#define link_regs_init(id, phyid)\
+	( \
+	LE_DCN31_REG_LIST_RI(id), \
+	UNIPHY_DCN2_REG_LIST_RI(id, phyid)\
+	)
 	/*DPCS_DCN31_REG_LIST(id),*/ \
-}
 
-static const struct dcn10_link_enc_registers link_enc_regs[] = {
-	link_regs(0, A),
-	link_regs(1, B),
-	link_regs(2, C),
-	link_regs(3, D),
-	link_regs(4, E)
-};
+static struct dcn10_link_enc_registers link_enc_regs[5];
 
 static const struct dcn10_link_enc_shift le_shift = {
 	LINK_ENCODER_MASK_SH_LIST_DCN31(__SHIFT), \
@@ -406,17 +331,10 @@ static const struct dcn10_link_enc_mask le_mask = {
 //	DPCS_DCN31_MASK_SH_LIST(_MASK)
 };
 
-#define hpo_dp_stream_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST(id)\
-}
+#define hpo_dp_stream_encoder_reg_init(id)\
+	DCN3_1_HPO_DP_STREAM_ENC_REG_LIST_RI(id)
 
-static const struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[] = {
-	hpo_dp_stream_encoder_reg_list(0),
-	hpo_dp_stream_encoder_reg_list(1),
-	hpo_dp_stream_encoder_reg_list(2),
-	hpo_dp_stream_encoder_reg_list(3),
-};
+static struct dcn31_hpo_dp_stream_encoder_registers hpo_dp_stream_enc_regs[4];
 
 static const struct dcn31_hpo_dp_stream_encoder_shift hpo_dp_se_shift = {
 	DCN3_1_HPO_DP_STREAM_ENC_MASK_SH_LIST(__SHIFT)
@@ -427,20 +345,14 @@ static const struct dcn31_hpo_dp_stream_encoder_mask hpo_dp_se_mask = {
 };
 
 
-#define hpo_dp_link_encoder_reg_list(id)\
-[id] = {\
-	DCN3_1_HPO_DP_LINK_ENC_REG_LIST(id),\
-	/*DCN3_1_RDPCSTX_REG_LIST(0),*/\
-	/*DCN3_1_RDPCSTX_REG_LIST(1),*/\
-	/*DCN3_1_RDPCSTX_REG_LIST(2),*/\
-	/*DCN3_1_RDPCSTX_REG_LIST(3),*/\
-	/*DCN3_1_RDPCSTX_REG_LIST(4)*/\
-}
+#define hpo_dp_link_encoder_reg_init(id)\
+	DCN3_1_HPO_DP_LINK_ENC_REG_LIST_RI(id)
+	/*DCN3_1_RDPCSTX_REG_LIST(0),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(1),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(2),*/
+	/*DCN3_1_RDPCSTX_REG_LIST(3),*/
 
-static const struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[] = {
-	hpo_dp_link_encoder_reg_list(0),
-	hpo_dp_link_encoder_reg_list(1),
-};
+static struct dcn31_hpo_dp_link_encoder_registers hpo_dp_link_enc_regs[2];
 
 static const struct dcn31_hpo_dp_link_encoder_shift hpo_dp_le_shift = {
 	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(__SHIFT)
@@ -450,17 +362,10 @@ static const struct dcn31_hpo_dp_link_encoder_mask hpo_dp_le_mask = {
 	DCN3_2_HPO_DP_LINK_ENC_MASK_SH_LIST(_MASK)
 };
 
-#define dpp_regs(id)\
-[id] = {\
-	DPP_REG_LIST_DCN30_COMMON(id),\
-}
+#define dpp_regs_init(id)\
+	DPP_REG_LIST_DCN30_COMMON_RI(id)
 
-static const struct dcn3_dpp_registers dpp_regs[] = {
-	dpp_regs(0),
-	dpp_regs(1),
-	dpp_regs(2),
-	dpp_regs(3)
-};
+static struct dcn3_dpp_registers dpp_regs[4];
 
 static const struct dcn3_dpp_shift tf_shift = {
 		DPP_REG_LIST_SH_MASK_DCN30_COMMON(__SHIFT)
@@ -471,17 +376,10 @@ static const struct dcn3_dpp_mask tf_mask = {
 };
 
 
-#define opp_regs(id)\
-[id] = {\
-	OPP_REG_LIST_DCN30(id),\
-}
+#define opp_regs_init(id)\
+	OPP_REG_LIST_DCN30_RI(id)
 
-static const struct dcn20_opp_registers opp_regs[] = {
-	opp_regs(0),
-	opp_regs(1),
-	opp_regs(2),
-	opp_regs(3)
-};
+static struct dcn20_opp_registers opp_regs[4];
 
 static const struct dcn20_opp_shift opp_shift = {
 	OPP_MASK_SH_LIST_DCN20(__SHIFT)
@@ -491,21 +389,15 @@ static const struct dcn20_opp_mask opp_mask = {
 	OPP_MASK_SH_LIST_DCN20(_MASK)
 };
 
-#define aux_engine_regs(id)\
-[id] = {\
-	AUX_COMMON_REG_LIST0(id), \
-	.AUXN_IMPCAL = 0, \
-	.AUXP_IMPCAL = 0, \
-	.AUX_RESET_MASK = DP_AUX0_AUX_CONTROL__AUX_RESET_MASK, \
-}
+#define aux_engine_regs_init(id) \
+	( \
+	AUX_COMMON_REG_LIST0_RI(id), SR_ARR_INIT(AUXN_IMPCAL, id, 0), \
+	SR_ARR_INIT(AUXP_IMPCAL, id, 0), \
+	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK), \
+	SR_ARR_INIT(AUX_RESET_MASK, id, DP_AUX0_AUX_CONTROL__AUX_RESET_MASK)\
+	)
 
-static const struct dce110_aux_registers aux_engine_regs[] = {
-		aux_engine_regs(0),
-		aux_engine_regs(1),
-		aux_engine_regs(2),
-		aux_engine_regs(3),
-		aux_engine_regs(4)
-};
+static struct dce110_aux_registers aux_engine_regs[5];
 
 static const struct dce110_aux_registers_shift aux_shift = {
 	DCN_AUX_MASK_SH_LIST(__SHIFT)
@@ -515,15 +407,10 @@ static const struct dce110_aux_registers_mask aux_mask = {
 	DCN_AUX_MASK_SH_LIST(_MASK)
 };
 
+#define dwbc_regs_dcn3_init(id)\
+	DWBC_COMMON_REG_LIST_DCN30_RI(id)
 
-#define dwbc_regs_dcn3(id)\
-[id] = {\
-	DWBC_COMMON_REG_LIST_DCN30(id),\
-}
-
-static const struct dcn30_dwbc_registers dwbc30_regs[] = {
-	dwbc_regs_dcn3(0),
-};
+static struct dcn30_dwbc_registers dwbc30_regs[1];
 
 static const struct dcn30_dwbc_shift dwbc30_shift = {
 	DWBC_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
@@ -533,14 +420,10 @@ static const struct dcn30_dwbc_mask dwbc30_mask = {
 	DWBC_COMMON_MASK_SH_LIST_DCN30(_MASK)
 };
 
-#define mcif_wb_regs_dcn3(id)\
-[id] = {\
-	MCIF_WB_COMMON_REG_LIST_DCN32(id),\
-}
+#define mcif_wb_regs_dcn3_init(id)\
+	MCIF_WB_COMMON_REG_LIST_DCN32_RI(id)
 
-static const struct dcn30_mmhubbub_registers mcif_wb30_regs[] = {
-	mcif_wb_regs_dcn3(0)
-};
+static struct dcn30_mmhubbub_registers mcif_wb30_regs[1];
 
 static const struct dcn30_mmhubbub_shift mcif_wb30_shift = {
 	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
@@ -550,17 +433,10 @@ static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
 	MCIF_WB_COMMON_MASK_SH_LIST_DCN32(_MASK)
 };
 
-#define dsc_regsDCN20(id)\
-[id] = {\
-	DSC_REG_LIST_DCN20(id)\
-}
+#define dsc_regsDCN20_init(id)\
+	DSC_REG_LIST_DCN20_RI(id)
 
-static const struct dcn20_dsc_registers dsc_regs[] = {
-	dsc_regsDCN20(0),
-	dsc_regsDCN20(1),
-	dsc_regsDCN20(2),
-	dsc_regsDCN20(3)
-};
+static struct dcn20_dsc_registers dsc_regs[4];
 
 static const struct dcn20_dsc_shift dsc_shift = {
 	DSC_REG_LIST_SH_MASK_DCN20(__SHIFT)
@@ -570,17 +446,17 @@ static const struct dcn20_dsc_mask dsc_mask = {
 	DSC_REG_LIST_SH_MASK_DCN20(_MASK)
 };
 
-static const struct dcn30_mpc_registers mpc_regs = {
-		MPC_REG_LIST_DCN3_2(0),
-		MPC_REG_LIST_DCN3_2(1),
-		MPC_REG_LIST_DCN3_2(2),
-		MPC_REG_LIST_DCN3_2(3),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(0),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(1),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(2),
-		MPC_OUT_MUX_REG_LIST_DCN3_0(3),
-		MPC_DWB_MUX_REG_LIST_DCN3_0(0),
-};
+static struct dcn30_mpc_registers mpc_regs;
+#define dcn_mpc_regs_init()\
+	MPC_REG_LIST_DCN3_2_RI(0),\
+	MPC_REG_LIST_DCN3_2_RI(1),\
+	MPC_REG_LIST_DCN3_2_RI(2),\
+	MPC_REG_LIST_DCN3_2_RI(3),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(0),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(1),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(2),\
+	MPC_OUT_MUX_REG_LIST_DCN3_0_RI(3),\
+	MPC_DWB_MUX_REG_LIST_DCN3_0_RI(0)
 
 static const struct dcn30_mpc_shift mpc_shift = {
 	MPC_COMMON_MASK_SH_LIST_DCN32(__SHIFT)
@@ -590,15 +466,10 @@ static const struct dcn30_mpc_mask mpc_mask = {
 	MPC_COMMON_MASK_SH_LIST_DCN32(_MASK)
 };
 
-#define optc_regs(id)\
-[id] = {OPTC_COMMON_REG_LIST_DCN3_2(id)}
+#define optc_regs_init(id)\
+	OPTC_COMMON_REG_LIST_DCN3_2_RI(id)
 
-static const struct dcn_optc_registers optc_regs[] = {
-	optc_regs(0),
-	optc_regs(1),
-	optc_regs(2),
-	optc_regs(3)
-};
+static struct dcn_optc_registers optc_regs[4];
 
 static const struct dcn_optc_shift optc_shift = {
 	OPTC_COMMON_MASK_SH_LIST_DCN3_2(__SHIFT)
@@ -608,18 +479,10 @@ static const struct dcn_optc_mask optc_mask = {
 	OPTC_COMMON_MASK_SH_LIST_DCN3_2(_MASK)
 };
 
-#define hubp_regs(id)\
-[id] = {\
-	HUBP_REG_LIST_DCN32(id)\
-}
-
-static const struct dcn_hubp2_registers hubp_regs[] = {
-		hubp_regs(0),
-		hubp_regs(1),
-		hubp_regs(2),
-		hubp_regs(3)
-};
+#define hubp_regs_init(id) \
+	HUBP_REG_LIST_DCN32_RI(id)
 
+static struct dcn_hubp2_registers hubp_regs[4];
 
 static const struct dcn_hubp2_shift hubp_shift = {
 		HUBP_MASK_SH_LIST_DCN32(__SHIFT)
@@ -628,9 +491,10 @@ static const struct dcn_hubp2_shift hubp_shift = {
 static const struct dcn_hubp2_mask hubp_mask = {
 		HUBP_MASK_SH_LIST_DCN32(_MASK)
 };
-static const struct dcn_hubbub_registers hubbub_reg = {
-		HUBBUB_REG_LIST_DCN32(0)
-};
+
+static struct dcn_hubbub_registers hubbub_reg;
+#define hubbub_reg_init()\
+		HUBBUB_REG_LIST_DCN32_RI(0)
 
 static const struct dcn_hubbub_shift hubbub_shift = {
 		HUBBUB_MASK_SH_LIST_DCN32(__SHIFT)
@@ -640,9 +504,10 @@ static const struct dcn_hubbub_mask hubbub_mask = {
 		HUBBUB_MASK_SH_LIST_DCN32(_MASK)
 };
 
-static const struct dccg_registers dccg_regs = {
-		DCCG_REG_LIST_DCN32()
-};
+static struct dccg_registers dccg_regs;
+
+#define dccg_regs_init()\
+	DCCG_REG_LIST_DCN32_RI()
 
 static const struct dccg_shift dccg_shift = {
 		DCCG_MASK_SH_LIST_DCN32(__SHIFT)
@@ -715,9 +580,10 @@ static const struct dccg_mask dccg_mask = {
 	SR(AZALIA_AUDIO_DTO), \
 	SR(AZALIA_CONTROLLER_CLOCK_GATING)
 
-static const struct dce_hwseq_registers hwseq_reg = {
-		HWSEQ_DCN32_REG_LIST()
-};
+static struct dce_hwseq_registers hwseq_reg;
+
+#define hwseq_reg_init()\
+	HWSEQ_DCN32_REG_LIST()
 
 #define HWSEQ_DCN32_MASK_SH_LIST(mask_sh)\
 	HWSEQ_DCN_MASK_SH_LIST(mask_sh), \
@@ -760,29 +626,10 @@ static const struct dce_hwseq_shift hwseq_shift = {
 static const struct dce_hwseq_mask hwseq_mask = {
 		HWSEQ_DCN32_MASK_SH_LIST(_MASK)
 };
-#define vmid_regs(id)\
-[id] = {\
-		DCN20_VMID_REG_LIST(id)\
-}
+#define vmid_regs_init(id)\
+		DCN20_VMID_REG_LIST_RI(id)
 
-static const struct dcn_vmid_registers vmid_regs[] = {
-	vmid_regs(0),
-	vmid_regs(1),
-	vmid_regs(2),
-	vmid_regs(3),
-	vmid_regs(4),
-	vmid_regs(5),
-	vmid_regs(6),
-	vmid_regs(7),
-	vmid_regs(8),
-	vmid_regs(9),
-	vmid_regs(10),
-	vmid_regs(11),
-	vmid_regs(12),
-	vmid_regs(13),
-	vmid_regs(14),
-	vmid_regs(15)
-};
+static struct dcn_vmid_registers vmid_regs[16];
 
 static const struct dcn20_vmid_shift vmid_shifts = {
 		DCN20_VMID_MASK_SH_LIST(__SHIFT)
@@ -871,8 +718,12 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.force_disable_subvp = false,
 	.exit_idle_opt_for_cursor_updates = true,
 	.enable_single_display_2to1_odm_policy = true,
+
+	/*must match enable_single_display_2to1_odm_policy to support dynamic ODM transitions*/
+	.enable_double_buffered_dsc_pg_support = true,
 	.enable_dp_dig_pixel_rate_div_policy = 1,
 	.allow_sw_cursor_fallback = false,
+	.alloc_extra_way_for_cursor = true,
 };
 
 static const struct dc_debug_options debug_defaults_diags = {
@@ -906,6 +757,14 @@ static struct dce_aux *dcn321_aux_engine_create(
 	if (!aux_engine)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT aux_engine_regs
+	aux_engine_regs_init(0),
+	aux_engine_regs_init(1),
+	aux_engine_regs_init(2),
+	aux_engine_regs_init(3),
+	aux_engine_regs_init(4);
+
 	dce110_aux_engine_construct(aux_engine, ctx, inst,
 				    SW_AUX_TIMEOUT_PERIOD_MULTIPLIER * AUX_TIMEOUT_PERIOD,
 				    &aux_engine_regs[inst],
@@ -915,15 +774,10 @@ static struct dce_aux *dcn321_aux_engine_create(
 
 	return &aux_engine->base;
 }
-#define i2c_inst_regs(id) { I2C_HW_ENGINE_COMMON_REG_LIST_DCN30(id) }
-
-static const struct dce_i2c_registers i2c_hw_regs[] = {
-		i2c_inst_regs(1),
-		i2c_inst_regs(2),
-		i2c_inst_regs(3),
-		i2c_inst_regs(4),
-		i2c_inst_regs(5),
-};
+#define i2c_inst_regs_init(id)\
+	I2C_HW_ENGINE_COMMON_REG_LIST_DCN30_RI(id)
+
+static struct dce_i2c_registers i2c_hw_regs[5];
 
 static const struct dce_i2c_shift i2c_shifts = {
 		I2C_COMMON_MASK_SH_LIST_DCN30(__SHIFT)
@@ -943,6 +797,14 @@ static struct dce_i2c_hw *dcn321_i2c_hw_create(
 	if (!dce_i2c_hw)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT i2c_hw_regs
+	i2c_inst_regs_init(1),
+	i2c_inst_regs_init(2),
+	i2c_inst_regs_init(3),
+	i2c_inst_regs_init(4),
+	i2c_inst_regs_init(5);
+
 	dcn2_i2c_hw_construct(dce_i2c_hw, ctx, inst,
 				    &i2c_hw_regs[inst], &i2c_shifts, &i2c_masks);
 
@@ -982,6 +844,29 @@ static struct hubbub *dcn321_hubbub_create(struct dc_context *ctx)
 	if (!hubbub2)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT hubbub_reg
+	hubbub_reg_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT vmid_regs
+	vmid_regs_init(0),
+	vmid_regs_init(1),
+	vmid_regs_init(2),
+	vmid_regs_init(3),
+	vmid_regs_init(4),
+	vmid_regs_init(5),
+	vmid_regs_init(6),
+	vmid_regs_init(7),
+	vmid_regs_init(8),
+	vmid_regs_init(9),
+	vmid_regs_init(10),
+	vmid_regs_init(11),
+	vmid_regs_init(12),
+	vmid_regs_init(13),
+	vmid_regs_init(14),
+	vmid_regs_init(15);
+
 	hubbub32_construct(hubbub2, ctx,
 			&hubbub_reg,
 			&hubbub_shift,
@@ -1014,6 +899,13 @@ static struct hubp *dcn321_hubp_create(
 	if (!hubp2)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT hubp_regs
+	hubp_regs_init(0),
+	hubp_regs_init(1),
+	hubp_regs_init(2),
+	hubp_regs_init(3);
+
 	if (hubp32_construct(hubp2, ctx, inst,
 			&hubp_regs[inst], &hubp_shift, &hubp_mask))
 		return &hubp2->base;
@@ -1039,6 +931,13 @@ static struct dpp *dcn321_dpp_create(
 	if (!dpp3)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT dpp_regs
+	dpp_regs_init(0),
+	dpp_regs_init(1),
+	dpp_regs_init(2),
+	dpp_regs_init(3);
+
 	if (dpp32_construct(dpp3, ctx, inst,
 			&dpp_regs[inst], &tf_shift, &tf_mask))
 		return &dpp3->base;
@@ -1059,6 +958,10 @@ static struct mpc *dcn321_mpc_create(
 	if (!mpc30)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT mpc_regs
+	dcn_mpc_regs_init();
+
 	dcn32_mpc_construct(mpc30, ctx,
 			&mpc_regs,
 			&mpc_shift,
@@ -1080,6 +983,13 @@ static struct output_pixel_processor *dcn321_opp_create(
 		return NULL;
 	}
 
+#undef REG_STRUCT
+#define REG_STRUCT opp_regs
+	opp_regs_init(0),
+	opp_regs_init(1),
+	opp_regs_init(2),
+	opp_regs_init(3);
+
 	dcn20_opp_construct(opp2, ctx, inst,
 			&opp_regs[inst], &opp_shift, &opp_mask);
 	return &opp2->base;
@@ -1096,6 +1006,13 @@ static struct timing_generator *dcn321_timing_generator_create(
 	if (!tgn10)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT optc_regs
+	optc_regs_init(0),
+	optc_regs_init(1),
+	optc_regs_init(2),
+	optc_regs_init(3);
+
 	tgn10->base.inst = instance;
 	tgn10->base.ctx = ctx;
 
@@ -1130,6 +1047,30 @@ static struct link_encoder *dcn321_link_encoder_create(
 	if (!enc20)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_aux_regs
+	aux_regs_init(0),
+	aux_regs_init(1),
+	aux_regs_init(2),
+	aux_regs_init(3),
+	aux_regs_init(4);
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_hpd_regs
+	hpd_regs_init(0),
+	hpd_regs_init(1),
+	hpd_regs_init(2),
+	hpd_regs_init(3),
+	hpd_regs_init(4);
+
+#undef REG_STRUCT
+#define REG_STRUCT link_enc_regs
+	link_regs_init(0, A),
+	link_regs_init(1, B),
+	link_regs_init(2, C),
+	link_regs_init(3, D),
+	link_regs_init(4, E);
+
 	dcn321_link_encoder_construct(enc20,
 			enc_init_data,
 			&link_enc_feature,
@@ -1146,7 +1087,7 @@ static void read_dce_straps(
 	struct dc_context *ctx,
 	struct resource_straps *straps)
 {
-	generic_reg_get(ctx, regDC_PINSTRAPS + BASE(regDC_PINSTRAPS_BASE_IDX),
+	generic_reg_get(ctx, ctx->dcn_reg_offsets[regDC_PINSTRAPS_BASE_IDX] + regDC_PINSTRAPS,
 		FN(DC_PINSTRAPS, DC_PINSTRAPS_AUDIO), &straps->dc_pinstraps_audio);
 
 }
@@ -1154,6 +1095,15 @@ static void read_dce_straps(
 static struct audio *dcn321_create_audio(
 		struct dc_context *ctx, unsigned int inst)
 {
+
+#undef REG_STRUCT
+#define REG_STRUCT audio_regs
+	audio_regs_init(0),
+	audio_regs_init(1),
+	audio_regs_init(2),
+	audio_regs_init(3),
+	audio_regs_init(4);
+
 	return dce_audio_create(ctx, inst,
 			&audio_regs[inst], &audio_shift, &audio_mask);
 }
@@ -1167,6 +1117,19 @@ static struct vpg *dcn321_vpg_create(
 	if (!vpg3)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT vpg_regs
+	vpg_regs_init(0),
+	vpg_regs_init(1),
+	vpg_regs_init(2),
+	vpg_regs_init(3),
+	vpg_regs_init(4),
+	vpg_regs_init(5),
+	vpg_regs_init(6),
+	vpg_regs_init(7),
+	vpg_regs_init(8),
+	vpg_regs_init(9);
+
 	vpg3_construct(vpg3, ctx, inst,
 			&vpg_regs[inst],
 			&vpg_shift,
@@ -1184,6 +1147,15 @@ static struct afmt *dcn321_afmt_create(
 	if (!afmt3)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT afmt_regs
+	afmt_regs_init(0),
+	afmt_regs_init(1),
+	afmt_regs_init(2),
+	afmt_regs_init(3),
+	afmt_regs_init(4),
+	afmt_regs_init(5);
+
 	afmt3_construct(afmt3, ctx, inst,
 			&afmt_regs[inst],
 			&afmt_shift,
@@ -1201,6 +1173,13 @@ static struct apg *dcn321_apg_create(
 	if (!apg31)
 		return NULL;
 
+#undef REG_STRUCT
+#define REG_STRUCT apg_regs
+	apg_regs_init(0),
+	apg_regs_init(1),
+	apg_regs_init(2),
+	apg_regs_init(3);
+
 	apg31_construct(apg31, ctx, inst,
 			&apg_regs[inst],
 			&apg_shift,
@@ -1237,6 +1216,14 @@ static struct stream_encoder *dcn321_stream_encoder_create(
 		return NULL;
 	}
 
+#undef REG_STRUCT
+#define REG_STRUCT stream_enc_regs
+	stream_enc_regs_init(0),
+	stream_enc_regs_init(1),
+	stream_enc_regs_init(2),
+	stream_enc_regs_init(3),
+	stream_enc_regs_init(4);
+
 	dcn32_dio_stream_encoder_construct(enc1, ctx, ctx->dc_bios,
 					eng_id, vpg, afmt,
 					&stream_enc_regs[eng_id],
@@ -1287,6 +1274,13 @@ static struct hpo_dp_stream_encoder *dcn321_hpo_dp_stream_encoder_create(
 		return NULL;
 	}
 
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_stream_enc_regs
+	hpo_dp_stream_encoder_reg_init(0),
+	hpo_dp_stream_encoder_reg_init(1),
+	hpo_dp_stream_encoder_reg_init(2),
+	hpo_dp_stream_encoder_reg_init(3);
+
 	dcn31_hpo_dp_stream_encoder_construct(hpo_dp_enc31, ctx, ctx->dc_bios,
 					hpo_dp_inst, eng_id, vpg, apg,
 					&hpo_dp_stream_enc_regs[hpo_dp_inst],
@@ -1304,6 +1298,11 @@ static struct hpo_dp_link_encoder *dcn321_hpo_dp_link_encoder_create(
 	/* allocate HPO link encoder */
 	hpo_dp_enc31 = kzalloc(sizeof(struct dcn31_hpo_dp_link_encoder), GFP_KERNEL);
 
+#undef REG_STRUCT
+#define REG_STRUCT hpo_dp_link_enc_regs
+	hpo_dp_link_encoder_reg_init(0),
+	hpo_dp_link_encoder_reg_init(1);
+
 	hpo_dp_link_encoder32_construct(hpo_dp_enc31, ctx, inst,
 					&hpo_dp_link_enc_regs[inst],
 					&hpo_dp_le_shift, &hpo_dp_le_mask);
@@ -1316,6 +1315,10 @@ static struct dce_hwseq *dcn321_hwseq_create(
 {
 	struct dce_hwseq *hws = kzalloc(sizeof(struct dce_hwseq), GFP_KERNEL);
 
+#undef REG_STRUCT
+#define REG_STRUCT hwseq_reg
+	hwseq_reg_init();
+
 	if (hws) {
 		hws->ctx = ctx;
 		hws->regs = &hwseq_reg;
@@ -1506,6 +1509,10 @@ static bool dcn321_dwbc_create(struct dc_context *ctx, struct resource_pool *poo
 			return false;
 		}
 
+#undef REG_STRUCT
+#define REG_STRUCT dwbc30_regs
+		dwbc_regs_dcn3_init(0);
+
 		dcn30_dwbc_construct(dwbc30, ctx,
 				&dwbc30_regs[i],
 				&dwbc30_shift,
@@ -1531,6 +1538,10 @@ static bool dcn321_mmhubbub_create(struct dc_context *ctx, struct resource_pool
 			return false;
 		}
 
+#undef REG_STRUCT
+#define REG_STRUCT mcif_wb30_regs
+		mcif_wb_regs_dcn3_init(0);
+
 		dcn32_mmhubbub_construct(mcif_wb30, ctx,
 				&mcif_wb30_regs[i],
 				&mcif_wb30_shift,
@@ -1553,6 +1564,13 @@ static struct display_stream_compressor *dcn321_dsc_create(
 		return NULL;
 	}
 
+#undef REG_STRUCT
+#define REG_STRUCT dsc_regs
+	dsc_regsDCN20_init(0),
+	dsc_regsDCN20_init(1),
+	dsc_regsDCN20_init(2),
+	dsc_regsDCN20_init(3);
+
 	dsc2_construct(dsc, ctx, inst, &dsc_regs[inst], &dsc_shift, &dsc_mask);
 
 	dsc->max_image_width = 6016;
@@ -1617,6 +1635,30 @@ static bool dcn321_resource_construct(
 	uint32_t pipe_fuses = 0;
 	uint32_t num_pipes  = 4;
 
+#undef REG_STRUCT
+#define REG_STRUCT bios_regs
+	bios_regs_init();
+
+#undef REG_STRUCT
+#define REG_STRUCT clk_src_regs
+	clk_src_regs_init(0, A),
+	clk_src_regs_init(1, B),
+	clk_src_regs_init(2, C),
+	clk_src_regs_init(3, D),
+	clk_src_regs_init(4, E);
+
+#undef REG_STRUCT
+#define REG_STRUCT abm_regs
+	abm_regs_init(0),
+	abm_regs_init(1),
+	abm_regs_init(2),
+	abm_regs_init(3);
+
+#undef REG_STRUCT
+#define REG_STRUCT dccg_regs
+		dccg_regs_init();
+
+
 	ctx->dc_bios->regs = &bios_regs;
 
 	pool->base.res_cap = &res_cap_dcn321;
diff --git a/drivers/gpu/drm/amd/display/dc/dm_helpers.h b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
index fb6a2d7b6470..e3e5c39895a3 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_helpers.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
@@ -33,7 +33,7 @@
 #include "dc_types.h"
 #include "dc.h"
 
-struct dp_mst_stream_allocation_table;
+struct dc_dp_mst_stream_allocation_table;
 struct aux_payload;
 enum aux_return_code_type;
 
@@ -77,7 +77,7 @@ void dm_helpers_dp_update_branch_info(
 bool dm_helpers_dp_mst_write_payload_allocation_table(
 		struct dc_context *ctx,
 		const struct dc_stream_state *stream,
-		struct dp_mst_stream_allocation_table *proposed_table,
+		struct dc_dp_mst_stream_allocation_table *proposed_table,
 		bool enable);
 
 /*
@@ -171,7 +171,13 @@ void dm_helpers_smu_timeout(struct dc_context *ctx, unsigned int msg_id, unsigne
 // 0x1 = Result_OK, 0xFE = Result_UnkmownCmd, 0x0 = Status_Busy
 #define IS_SMU_TIMEOUT(result) \
 	(result == 0x0)
-
+void dm_helpers_init_panel_settings(
+	struct dc_context *ctx,
+	struct dc_panel_config *config,
+	struct dc_sink *sink);
+void dm_helpers_override_panel_settings(
+	struct dc_context *ctx,
+	struct dc_panel_config *config);
 int dm_helper_dmub_aux_transfer_sync(
 		struct dc_context *ctx,
 		const struct dc_link *link,
diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/bw_fixed.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/bw_fixed.c
index 6ca288fb5fb9..3aa8dd0acd5e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/calcs/bw_fixed.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/bw_fixed.c
@@ -25,12 +25,11 @@
 #include "dm_services.h"
 #include "bw_fixed.h"
 
+#define MAX_I64 \
+	((int64_t)((1ULL << 63) - 1))
 
 #define MIN_I64 \
-	(int64_t)(-(1LL << 63))
-
-#define MAX_I64 \
-	(int64_t)((1ULL << 63) - 1)
+	(-MAX_I64 - 1)
 
 #define FRACTIONAL_PART_MASK \
 	((1ULL << BW_FIXED_BITS_PER_FRACTIONAL_PART) - 1)
@@ -49,6 +48,7 @@ static uint64_t abs_i64(int64_t arg)
 struct bw_fixed bw_int_to_fixed_nonconst(int64_t value)
 {
 	struct bw_fixed res;
+
 	ASSERT(value < BW_FIXED_MAX_I32 && value > BW_FIXED_MIN_I32);
 	res.value = value << BW_FIXED_BITS_PER_FRACTIONAL_PART;
 	return res;
@@ -78,14 +78,12 @@ struct bw_fixed bw_frc_to_fixed(int64_t numerator, int64_t denominator)
 	{
 		uint32_t i = BW_FIXED_BITS_PER_FRACTIONAL_PART;
 
-		do
-		{
+		do {
 			remainder <<= 1;
 
 			res_value <<= 1;
 
-			if (remainder >= arg2_value)
-			{
+			if (remainder >= arg2_value) {
 				res_value |= 1;
 				remainder -= arg2_value;
 			}
diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c
index 41284e263325..288d22a16cf2 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c
@@ -526,10 +526,10 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 				}
 				if (v->max_swath_height_c[k] > 0.0) {
 					v->swath_width_granularity_c = 256.0 /dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / v->max_swath_height_c[k];
-				}
-				v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width_yper_state[i][j][k] / 2.0 - 1.0, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pixel_in_detc[k] * v->max_swath_height_c[k];
-				if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) {
-					v->rounded_up_max_swath_size_bytes_c =dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0) + 256;
+					v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width_yper_state[i][j][k] / 2.0 - 1.0, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pixel_in_detc[k] * v->max_swath_height_c[k];
+					if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) {
+						v->rounded_up_max_swath_size_bytes_c = dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0) + 256;
+					}
 				}
 				if (v->rounded_up_max_swath_size_bytes_y + v->rounded_up_max_swath_size_bytes_c <= v->det_buffer_size_in_kbyte * 1024.0 / 2.0) {
 					v->swath_height_yper_state[i][j][k] = v->max_swath_height_y[k];
@@ -552,14 +552,14 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 					v->lines_in_det_chroma = v->det_buffer_size_in_kbyte * 1024.0 / 3.0 / v->byte_per_pixel_in_dety[k] / (v->swath_width_yper_state[i][j][k] / 2.0);
 				}
 				v->effective_lb_latency_hiding_source_lines_luma =dcn_bw_min2(v->max_line_buffer_lines,dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_yper_state[i][j][k] /dcn_bw_max2(v->h_ratio[k], 1.0)), 1.0)) - (v->vtaps[k] - 1.0);
-				v->effective_lb_latency_hiding_source_lines_chroma =dcn_bw_min2(v->max_line_buffer_lines,dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_yper_state[i][j][k] / 2.0 /dcn_bw_max2(v->h_ratio[k] / 2.0, 1.0)), 1.0)) - (v->vta_pschroma[k] - 1.0);
 				v->effective_detlb_lines_luma =dcn_bw_floor2(v->lines_in_det_luma +dcn_bw_min2(v->lines_in_det_luma * v->required_dispclk[i][j] * v->byte_per_pixel_in_dety[k] * v->pscl_factor[k] / v->return_bw_per_state[i], v->effective_lb_latency_hiding_source_lines_luma), v->swath_height_yper_state[i][j][k]);
-				v->effective_detlb_lines_chroma =dcn_bw_floor2(v->lines_in_det_chroma +dcn_bw_min2(v->lines_in_det_chroma * v->required_dispclk[i][j] * v->byte_per_pixel_in_detc[k] * v->pscl_factor_chroma[k] / v->return_bw_per_state[i], v->effective_lb_latency_hiding_source_lines_chroma), v->swath_height_cper_state[i][j][k]);
 				if (v->byte_per_pixel_in_detc[k] == 0.0) {
 					v->urgent_latency_support_us_per_state[i][j][k] = v->effective_detlb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_detlb_lines_luma * v->swath_width_yper_state[i][j][k] *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]);
 				}
 				else {
-					v->urgent_latency_support_us_per_state[i][j][k] =dcn_bw_min2(v->effective_detlb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_detlb_lines_luma * v->swath_width_yper_state[i][j][k] *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]), v->effective_detlb_lines_chroma * (v->htotal[k] / v->pixel_clock[k]) / (v->v_ratio[k] / 2.0) - v->effective_detlb_lines_chroma * v->swath_width_yper_state[i][j][k] / 2.0 *dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]));
+					v->effective_lb_latency_hiding_source_lines_chroma = dcn_bw_min2(v->max_line_buffer_lines, dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_yper_state[i][j][k] / 2.0 / dcn_bw_max2(v->h_ratio[k] / 2.0, 1.0)), 1.0)) - (v->vta_pschroma[k] - 1.0);
+					v->effective_detlb_lines_chroma = dcn_bw_floor2(v->lines_in_det_chroma + dcn_bw_min2(v->lines_in_det_chroma * v->required_dispclk[i][j] * v->byte_per_pixel_in_detc[k] * v->pscl_factor_chroma[k] / v->return_bw_per_state[i], v->effective_lb_latency_hiding_source_lines_chroma), v->swath_height_cper_state[i][j][k]);
+					v->urgent_latency_support_us_per_state[i][j][k] = dcn_bw_min2(v->effective_detlb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_detlb_lines_luma * v->swath_width_yper_state[i][j][k] * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]), v->effective_detlb_lines_chroma * (v->htotal[k] / v->pixel_clock[k]) / (v->v_ratio[k] / 2.0) - v->effective_detlb_lines_chroma * v->swath_width_yper_state[i][j][k] / 2.0 * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]));
 				}
 			}
 		}
@@ -1146,10 +1146,10 @@ void display_pipe_configuration(struct dcn_bw_internal_vars *v)
 		}
 		if (v->maximum_swath_height_c > 0.0) {
 			v->swath_width_granularity_c = 256.0 /dcn_bw_ceil2(v->byte_per_pix_detc, 2.0) / v->maximum_swath_height_c;
-		}
-		v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width / 2.0 - 1.0, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pix_detc * v->maximum_swath_height_c;
-		if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) {
-			v->rounded_up_max_swath_size_bytes_c =dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0) + 256;
+			v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width / 2.0 - 1.0, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pix_detc * v->maximum_swath_height_c;
+			if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) {
+				v->rounded_up_max_swath_size_bytes_c = dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0) + 256;
+			}
 		}
 		if (v->rounded_up_max_swath_size_bytes_y + v->rounded_up_max_swath_size_bytes_c <= v->det_buffer_size_in_kbyte * 1024.0 / 2.0) {
 			v->swath_height_y[k] = v->maximum_swath_height_y;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c
index 07d18e78de49..cac72413a097 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c
@@ -23,6 +23,7 @@
  *
  */
 
+#include "os_types.h"
 #include "dcn_calc_math.h"
 
 #define isNaN(number) ((number) != (number))
@@ -69,8 +70,8 @@ float dcn_bw_max2(const float arg1, const float arg2)
 
 float dcn_bw_floor2(const float arg, const float significance)
 {
-	if (significance == 0)
-		return 0;
+	ASSERT(significance != 0);
+
 	return ((int) (arg / significance)) * significance;
 }
 float dcn_bw_floor(const float arg)
@@ -80,17 +81,14 @@ float dcn_bw_floor(const float arg)
 
 float dcn_bw_ceil(const float arg)
 {
-	float flr = dcn_bw_floor2(arg, 1);
-
-	return flr + 0.00001 >= arg ? arg : flr + 1;
+	return (int) (arg + 0.99999);
 }
 
 float dcn_bw_ceil2(const float arg, const float significance)
 {
-	float flr = dcn_bw_floor2(arg, significance);
-	if (significance == 0)
-		return 0;
-	return flr + 0.00001 >= arg ? arg : flr + significance;
+	ASSERT(significance != 0);
+
+	return ((int) (arg / significance + 0.99999)) * significance;
 }
 
 float dcn_bw_max3(float v1, float v2, float v3)
diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
index db3b16b77034..d46adc849d2a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
@@ -736,30 +736,13 @@ static void hack_bounding_box(struct dcn_bw_internal_vars *v,
 		hack_force_pipe_split(v, context->streams[0]->timing.pix_clk_100hz);
 }
 
-static unsigned int get_highest_allowed_voltage_level(uint32_t chip_family,
-						      uint32_t hw_internal_rev,
-						      uint32_t pci_revision_id)
+static unsigned int get_highest_allowed_voltage_level(bool is_vmin_only_asic)
 {
 	/* for low power RV2 variants, the highest voltage level we want is 0 */
-	if ((chip_family == FAMILY_RV) &&
-	     ASICREV_IS_RAVEN2(hw_internal_rev))
-		switch (pci_revision_id) {
-		case PRID_DALI_DE:
-		case PRID_DALI_DF:
-		case PRID_DALI_E3:
-		case PRID_DALI_E4:
-		case PRID_POLLOCK_94:
-		case PRID_POLLOCK_95:
-		case PRID_POLLOCK_E9:
-		case PRID_POLLOCK_EA:
-		case PRID_POLLOCK_EB:
-			return 0;
-		default:
-			break;
-		}
-
-	/* we are ok with all levels */
-	return 4;
+	if (is_vmin_only_asic)
+		return 0;
+	else	/* we are ok with all levels */
+		return 4;
 }
 
 bool dcn_validate_bandwidth(
@@ -1323,10 +1306,7 @@ bool dcn_validate_bandwidth(
 	PERFORMANCE_TRACE_END();
 	BW_VAL_TRACE_FINISH();
 
-	if (bw_limit_pass && v->voltage_level <= get_highest_allowed_voltage_level(
-							dc->ctx->asic_id.chip_family,
-							dc->ctx->asic_id.hw_internal_rev,
-							dc->ctx->asic_id.pci_revision_id))
+	if (bw_limit_pass && v->voltage_level <= get_highest_allowed_voltage_level(dc->config.is_vmin_only_asic))
 		return true;
 	else
 		return false;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
index d34e0f1314d9..d680f1c5b69f 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
@@ -2234,6 +2234,7 @@ static struct _vcs_dpi_voltage_scaling_st construct_low_pstate_lvl(struct clk_li
 
 void dcn21_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
 {
+	struct _vcs_dpi_voltage_scaling_st *s = dc->scratch.update_bw_bounding_box.clock_limits;
 	struct dcn21_resource_pool *pool = TO_DCN21_RES_POOL(dc->res_pool);
 	struct clk_limit_table *clk_table = &bw_params->clk_table;
 	unsigned int i, closest_clk_lvl = 0, k = 0;
@@ -2247,8 +2248,7 @@ void dcn21_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params
 
 	ASSERT(clk_table->num_entries);
 	/* Copy dcn2_1_soc.clock_limits to clock_limits to avoid copying over null states later */
-	memcpy(&dcn2_1_soc._clock_tmp, &dcn2_1_soc.clock_limits,
-	       sizeof(dcn2_1_soc.clock_limits));
+	memcpy(s, dcn2_1_soc.clock_limits, sizeof(dcn2_1_soc.clock_limits));
 
 	for (i = 0; i < clk_table->num_entries; i++) {
 		/* loop backwards*/
@@ -2263,25 +2263,25 @@ void dcn21_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params
 		if (i == 1)
 			k++;
 
-		dcn2_1_soc._clock_tmp[k].state = k;
-		dcn2_1_soc._clock_tmp[k].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;
-		dcn2_1_soc._clock_tmp[k].fabricclk_mhz = clk_table->entries[i].fclk_mhz;
-		dcn2_1_soc._clock_tmp[k].socclk_mhz = clk_table->entries[i].socclk_mhz;
-		dcn2_1_soc._clock_tmp[k].dram_speed_mts = clk_table->entries[i].memclk_mhz * 2;
-
-		dcn2_1_soc._clock_tmp[k].dispclk_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].dispclk_mhz;
-		dcn2_1_soc._clock_tmp[k].dppclk_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].dppclk_mhz;
-		dcn2_1_soc._clock_tmp[k].dram_bw_per_chan_gbps = dcn2_1_soc.clock_limits[closest_clk_lvl].dram_bw_per_chan_gbps;
-		dcn2_1_soc._clock_tmp[k].dscclk_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].dscclk_mhz;
-		dcn2_1_soc._clock_tmp[k].dtbclk_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].dtbclk_mhz;
-		dcn2_1_soc._clock_tmp[k].phyclk_d18_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].phyclk_d18_mhz;
-		dcn2_1_soc._clock_tmp[k].phyclk_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].phyclk_mhz;
+		s[k].state = k;
+		s[k].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;
+		s[k].fabricclk_mhz = clk_table->entries[i].fclk_mhz;
+		s[k].socclk_mhz = clk_table->entries[i].socclk_mhz;
+		s[k].dram_speed_mts = clk_table->entries[i].memclk_mhz * 2;
+
+		s[k].dispclk_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].dispclk_mhz;
+		s[k].dppclk_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].dppclk_mhz;
+		s[k].dram_bw_per_chan_gbps =
+			dcn2_1_soc.clock_limits[closest_clk_lvl].dram_bw_per_chan_gbps;
+		s[k].dscclk_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].dscclk_mhz;
+		s[k].dtbclk_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].dtbclk_mhz;
+		s[k].phyclk_d18_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].phyclk_d18_mhz;
+		s[k].phyclk_mhz = dcn2_1_soc.clock_limits[closest_clk_lvl].phyclk_mhz;
 
 		k++;
 	}
 
-	memcpy(&dcn2_1_soc.clock_limits, &dcn2_1_soc._clock_tmp,
-	       sizeof(dcn2_1_soc.clock_limits));
+	memcpy(dcn2_1_soc.clock_limits, s, sizeof(dcn2_1_soc.clock_limits));
 
 	if (clk_table->num_entries) {
 		dcn2_1_soc.num_states = clk_table->num_entries + 1;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c
index 63bbdf8b8678..edd098c7eb92 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c
@@ -4478,17 +4478,17 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 						locals->EffectiveLBLatencyHidingSourceLinesLuma),
 						locals->SwathHeightYPerState[i][j][k]);
 
-				locals->EffectiveDETLBLinesChroma = dml_floor(locals->LinesInDETChroma + dml_min(
-						locals->LinesInDETChroma * locals->RequiredDISPCLK[i][j] * locals->BytePerPixelInDETC[k] *
-						locals->PSCL_FACTOR_CHROMA[k] / locals->ReturnBWPerState[i][0],
-						locals->EffectiveLBLatencyHidingSourceLinesChroma),
-						locals->SwathHeightCPerState[i][j][k]);
 
 				if (locals->BytePerPixelInDETC[k] == 0) {
 					locals->UrgentLatencySupportUsPerState[i][j][k] = locals->EffectiveDETLBLinesLuma * (locals->HTotal[k] / locals->PixelClock[k])
 							/ locals->VRatio[k] - locals->EffectiveDETLBLinesLuma * locals->SwathWidthYPerState[i][j][k] *
 								dml_ceil(locals->BytePerPixelInDETY[k], 1) / (locals->ReturnBWPerState[i][0] / locals->NoOfDPP[i][j][k]);
 				} else {
+					locals->EffectiveDETLBLinesChroma = dml_floor(locals->LinesInDETChroma + dml_min(
+							locals->LinesInDETChroma * locals->RequiredDISPCLK[i][j] * locals->BytePerPixelInDETC[k] *
+							locals->PSCL_FACTOR_CHROMA[k] / locals->ReturnBWPerState[i][0],
+							locals->EffectiveLBLatencyHidingSourceLinesChroma),
+							locals->SwathHeightCPerState[i][j][k]);
 					locals->UrgentLatencySupportUsPerState[i][j][k] = dml_min(
 						locals->EffectiveDETLBLinesLuma * (locals->HTotal[k] / locals->PixelClock[k])
 						/ locals->VRatio[k] - locals->EffectiveDETLBLinesLuma * locals->SwathWidthYPerState[i][j][k] *
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c
index 8a7485e21d53..1d84ae50311d 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c
@@ -806,10 +806,12 @@ static bool CalculatePrefetchSchedule(
 
 	if (myPipe->SourceScan == dm_horz) {
 		*swath_width_luma_ub = dml_ceil(SwathWidthY - 1, myPipe->BlockWidth256BytesY) + myPipe->BlockWidth256BytesY;
-		*swath_width_chroma_ub = dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockWidth256BytesC) + myPipe->BlockWidth256BytesC;
+		if (myPipe->BlockWidth256BytesC > 0)
+			*swath_width_chroma_ub = dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockWidth256BytesC) + myPipe->BlockWidth256BytesC;
 	} else {
 		*swath_width_luma_ub = dml_ceil(SwathWidthY - 1, myPipe->BlockHeight256BytesY) + myPipe->BlockHeight256BytesY;
-		*swath_width_chroma_ub = dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockHeight256BytesC) + myPipe->BlockHeight256BytesC;
+		if (myPipe->BlockWidth256BytesC > 0)
+			*swath_width_chroma_ub = dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockHeight256BytesC) + myPipe->BlockHeight256BytesC;
 	}
 
 	prefetch_bw_oto = (PrefetchSourceLinesY * *swath_width_luma_ub * dml_ceil(BytePerPixelDETY, 1) + PrefetchSourceLinesC * *swath_width_chroma_ub * dml_ceil(BytePerPixelDETC, 2)) / Tsw_oto;
@@ -2634,7 +2636,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					&mode_lib->vba.SrcActiveDrainRate,
 					&mode_lib->vba.TInitXFill,
 					&mode_lib->vba.TslvChk);
-					locals->XFCRemoteSurfaceFlipLatency[k] =
+			locals->XFCRemoteSurfaceFlipLatency[k] =
 					dml_floor(
 							mode_lib->vba.XFCRemoteSurfaceFlipDelay
 									/ (mode_lib->vba.HTotal[k]
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
index 1cb858dd6ea0..479e2c1a1301 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
@@ -396,64 +396,10 @@ static void CalculateUrgentBurstFactor(
 
 static void UseMinimumDCFCLK(
 		struct display_mode_lib *mode_lib,
-		int MaxInterDCNTileRepeaters,
+		struct vba_vars_st *v,
 		int MaxPrefetchMode,
-		double FinalDRAMClockChangeLatency,
-		double SREnterPlusExitTime,
-		int ReturnBusWidth,
-		int RoundTripPingLatencyCycles,
-		int ReorderingBytes,
-		int PixelChunkSizeInKByte,
-		int MetaChunkSize,
-		bool GPUVMEnable,
-		int GPUVMMaxPageTableLevels,
-		bool HostVMEnable,
-		int NumberOfActivePlanes,
-		double HostVMMinPageSize,
-		int HostVMMaxNonCachedPageTableLevels,
-		bool DynamicMetadataVMEnabled,
-		enum immediate_flip_requirement ImmediateFlipRequirement,
-		bool ProgressiveToInterlaceUnitInOPP,
-		double MaxAveragePercentOfIdealSDPPortBWDisplayCanUseInNormalSystemOperation,
-		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData,
-		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
-		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly,
-		int VTotal[],
-		int VActive[],
-		int DynamicMetadataTransmittedBytes[],
-		int DynamicMetadataLinesBeforeActiveRequired[],
-		bool Interlace[],
-		double RequiredDPPCLK[][2][DC__NUM_DPP__MAX],
-		double RequiredDISPCLK[][2],
-		double UrgLatency[],
-		unsigned int NoOfDPP[][2][DC__NUM_DPP__MAX],
-		double ProjectedDCFCLKDeepSleep[][2],
-		double MaximumVStartup[][2][DC__NUM_DPP__MAX],
-		double TotalVActivePixelBandwidth[][2],
-		double TotalVActiveCursorBandwidth[][2],
-		double TotalMetaRowBandwidth[][2],
-		double TotalDPTERowBandwidth[][2],
-		unsigned int TotalNumberOfActiveDPP[][2],
-		unsigned int TotalNumberOfDCCActiveDPP[][2],
-		int dpte_group_bytes[],
-		double PrefetchLinesY[][2][DC__NUM_DPP__MAX],
-		double PrefetchLinesC[][2][DC__NUM_DPP__MAX],
-		unsigned int swath_width_luma_ub_all_states[][2][DC__NUM_DPP__MAX],
-		unsigned int swath_width_chroma_ub_all_states[][2][DC__NUM_DPP__MAX],
-		int BytePerPixelY[],
-		int BytePerPixelC[],
-		int HTotal[],
-		double PixelClock[],
-		double PDEAndMetaPTEBytesPerFrame[][2][DC__NUM_DPP__MAX],
-		double DPTEBytesPerRow[][2][DC__NUM_DPP__MAX],
-		double MetaRowBytes[][2][DC__NUM_DPP__MAX],
-		bool DynamicMetadataEnable[],
-		double VActivePixelBandwidth[][2][DC__NUM_DPP__MAX],
-		double VActiveCursorBandwidth[][2][DC__NUM_DPP__MAX],
-		double ReadBandwidthLuma[],
-		double ReadBandwidthChroma[],
-		double DCFCLKPerState[],
-		double DCFCLKState[][2]);
+		int ReorderingBytes);
+
 static void CalculatePixelDeliveryTimes(
 		unsigned int NumberOfActivePlanes,
 		double VRatio[],
@@ -4692,66 +4638,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 	}
 
 	if (v->UseMinimumRequiredDCFCLK == true) {
-		UseMinimumDCFCLK(
-				mode_lib,
-				v->MaxInterDCNTileRepeaters,
-				MaxPrefetchMode,
-				v->FinalDRAMClockChangeLatency,
-				v->SREnterPlusExitTime,
-				v->ReturnBusWidth,
-				v->RoundTripPingLatencyCycles,
-				ReorderingBytes,
-				v->PixelChunkSizeInKByte,
-				v->MetaChunkSize,
-				v->GPUVMEnable,
-				v->GPUVMMaxPageTableLevels,
-				v->HostVMEnable,
-				v->NumberOfActivePlanes,
-				v->HostVMMinPageSize,
-				v->HostVMMaxNonCachedPageTableLevels,
-				v->DynamicMetadataVMEnabled,
-				v->ImmediateFlipRequirement[0],
-				v->ProgressiveToInterlaceUnitInOPP,
-				v->MaxAveragePercentOfIdealSDPPortBWDisplayCanUseInNormalSystemOperation,
-				v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData,
-				v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
-				v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly,
-				v->VTotal,
-				v->VActive,
-				v->DynamicMetadataTransmittedBytes,
-				v->DynamicMetadataLinesBeforeActiveRequired,
-				v->Interlace,
-				v->RequiredDPPCLK,
-				v->RequiredDISPCLK,
-				v->UrgLatency,
-				v->NoOfDPP,
-				v->ProjectedDCFCLKDeepSleep,
-				v->MaximumVStartup,
-				v->TotalVActivePixelBandwidth,
-				v->TotalVActiveCursorBandwidth,
-				v->TotalMetaRowBandwidth,
-				v->TotalDPTERowBandwidth,
-				v->TotalNumberOfActiveDPP,
-				v->TotalNumberOfDCCActiveDPP,
-				v->dpte_group_bytes,
-				v->PrefetchLinesY,
-				v->PrefetchLinesC,
-				v->swath_width_luma_ub_all_states,
-				v->swath_width_chroma_ub_all_states,
-				v->BytePerPixelY,
-				v->BytePerPixelC,
-				v->HTotal,
-				v->PixelClock,
-				v->PDEAndMetaPTEBytesPerFrame,
-				v->DPTEBytesPerRow,
-				v->MetaRowBytes,
-				v->DynamicMetadataEnable,
-				v->VActivePixelBandwidth,
-				v->VActiveCursorBandwidth,
-				v->ReadBandwidthLuma,
-				v->ReadBandwidthChroma,
-				v->DCFCLKPerState,
-				v->DCFCLKState);
+		UseMinimumDCFCLK(mode_lib, v, MaxPrefetchMode, ReorderingBytes);
 
 		if (v->ClampMinDCFCLK) {
 			/* Clamp calculated values to actual minimum */
@@ -6435,10 +6322,6 @@ static void CalculateSwathWidth(
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 		enum odm_combine_mode MainPlaneODMCombine = 0;
-		surface_width_ub_l = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
-		surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
-		surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
-		surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
 
 		if (SourceScan[k] != dm_vert) {
 			SwathWidthSingleDPPY[k] = ViewportWidth[k];
@@ -6478,8 +6361,6 @@ static void CalculateSwathWidth(
 
 		surface_width_ub_l  = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
 		surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
-		surface_width_ub_c  = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
-		surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
 
 		if (SourceScan[k] != dm_vert) {
 			MaximumSwathHeightY[k] = Read256BytesBlockHeightY[k];
@@ -6487,6 +6368,7 @@ static void CalculateSwathWidth(
 			swath_width_luma_ub[k] = dml_min(surface_width_ub_l, (long) dml_ceil(SwathWidthY[k] - 1,
 					Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]);
 			if (BytePerPixC[k] > 0) {
+				surface_width_ub_c  = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
 				swath_width_chroma_ub[k] = dml_min(surface_width_ub_c, (long) dml_ceil(SwathWidthC[k] - 1,
 						Read256BytesBlockWidthC[k]) + Read256BytesBlockWidthC[k]);
 			} else {
@@ -6498,6 +6380,7 @@ static void CalculateSwathWidth(
 			swath_width_luma_ub[k] = dml_min(surface_height_ub_l, (long) dml_ceil(SwathWidthY[k] - 1,
 					Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]);
 			if (BytePerPixC[k] > 0) {
+				surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
 				swath_width_chroma_ub[k] = dml_min(surface_height_ub_c, (long) dml_ceil(SwathWidthC[k] - 1,
 						Read256BytesBlockHeightC[k]) + Read256BytesBlockHeightC[k]);
 			} else {
@@ -6612,74 +6495,19 @@ static double CalculateUrgentLatency(
 
 static noinline_for_stack void UseMinimumDCFCLK(
 		struct display_mode_lib *mode_lib,
-		int MaxInterDCNTileRepeaters,
+		struct vba_vars_st *v,
 		int MaxPrefetchMode,
-		double FinalDRAMClockChangeLatency,
-		double SREnterPlusExitTime,
-		int ReturnBusWidth,
-		int RoundTripPingLatencyCycles,
-		int ReorderingBytes,
-		int PixelChunkSizeInKByte,
-		int MetaChunkSize,
-		bool GPUVMEnable,
-		int GPUVMMaxPageTableLevels,
-		bool HostVMEnable,
-		int NumberOfActivePlanes,
-		double HostVMMinPageSize,
-		int HostVMMaxNonCachedPageTableLevels,
-		bool DynamicMetadataVMEnabled,
-		enum immediate_flip_requirement ImmediateFlipRequirement,
-		bool ProgressiveToInterlaceUnitInOPP,
-		double MaxAveragePercentOfIdealSDPPortBWDisplayCanUseInNormalSystemOperation,
-		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData,
-		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
-		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly,
-		int VTotal[],
-		int VActive[],
-		int DynamicMetadataTransmittedBytes[],
-		int DynamicMetadataLinesBeforeActiveRequired[],
-		bool Interlace[],
-		double RequiredDPPCLK[][2][DC__NUM_DPP__MAX],
-		double RequiredDISPCLK[][2],
-		double UrgLatency[],
-		unsigned int NoOfDPP[][2][DC__NUM_DPP__MAX],
-		double ProjectedDCFCLKDeepSleep[][2],
-		double MaximumVStartup[][2][DC__NUM_DPP__MAX],
-		double TotalVActivePixelBandwidth[][2],
-		double TotalVActiveCursorBandwidth[][2],
-		double TotalMetaRowBandwidth[][2],
-		double TotalDPTERowBandwidth[][2],
-		unsigned int TotalNumberOfActiveDPP[][2],
-		unsigned int TotalNumberOfDCCActiveDPP[][2],
-		int dpte_group_bytes[],
-		double PrefetchLinesY[][2][DC__NUM_DPP__MAX],
-		double PrefetchLinesC[][2][DC__NUM_DPP__MAX],
-		unsigned int swath_width_luma_ub_all_states[][2][DC__NUM_DPP__MAX],
-		unsigned int swath_width_chroma_ub_all_states[][2][DC__NUM_DPP__MAX],
-		int BytePerPixelY[],
-		int BytePerPixelC[],
-		int HTotal[],
-		double PixelClock[],
-		double PDEAndMetaPTEBytesPerFrame[][2][DC__NUM_DPP__MAX],
-		double DPTEBytesPerRow[][2][DC__NUM_DPP__MAX],
-		double MetaRowBytes[][2][DC__NUM_DPP__MAX],
-		bool DynamicMetadataEnable[],
-		double VActivePixelBandwidth[][2][DC__NUM_DPP__MAX],
-		double VActiveCursorBandwidth[][2][DC__NUM_DPP__MAX],
-		double ReadBandwidthLuma[],
-		double ReadBandwidthChroma[],
-		double DCFCLKPerState[],
-		double DCFCLKState[][2])
+		int ReorderingBytes)
 {
 	double   NormalEfficiency = 0;
 	double   PTEEfficiency = 0;
 	double   TotalMaxPrefetchFlipDPTERowBandwidth[DC__VOLTAGE_STATES][2] = { { 0 } };
 	unsigned int i, j, k;
 
-	NormalEfficiency =  (HostVMEnable == true ? PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData
-			: PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly) / 100.0;
-	PTEEfficiency =  (HostVMEnable == true ? PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly
-			/ PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData : 1.0);
+	NormalEfficiency =  (v->HostVMEnable == true ? v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData
+			: v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelDataOnly) / 100.0;
+	PTEEfficiency =  (v->HostVMEnable == true ? v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly
+			/ v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData : 1.0);
 	for (i = 0; i < mode_lib->soc.num_states; ++i) {
 		for (j = 0; j <= 1; ++j) {
 			double PixelDCFCLKCyclesRequiredInPrefetch[DC__NUM_DPP__MAX] = { 0 };
@@ -6697,58 +6525,58 @@ static noinline_for_stack void UseMinimumDCFCLK(
 			double MinimumTvmPlus2Tr0 = 0;
 
 			TotalMaxPrefetchFlipDPTERowBandwidth[i][j] = 0;
-			for (k = 0; k < NumberOfActivePlanes; ++k) {
+			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 				TotalMaxPrefetchFlipDPTERowBandwidth[i][j] = TotalMaxPrefetchFlipDPTERowBandwidth[i][j]
-					+ NoOfDPP[i][j][k] * DPTEBytesPerRow[i][j][k] / (15.75 * HTotal[k] / PixelClock[k]);
+					+ v->NoOfDPP[i][j][k] * v->DPTEBytesPerRow[i][j][k] / (15.75 * v->HTotal[k] / v->PixelClock[k]);
 			}
 
-			for (k = 0; k <= NumberOfActivePlanes - 1; ++k) {
-				NoOfDPPState[k] = NoOfDPP[i][j][k];
+			for (k = 0; k <= v->NumberOfActivePlanes - 1; ++k) {
+				NoOfDPPState[k] = v->NoOfDPP[i][j][k];
 			}
 
-			MinimumTWait = CalculateTWait(MaxPrefetchMode, FinalDRAMClockChangeLatency, UrgLatency[i], SREnterPlusExitTime);
-			NonDPTEBandwidth = TotalVActivePixelBandwidth[i][j] + TotalVActiveCursorBandwidth[i][j] + TotalMetaRowBandwidth[i][j];
-			DPTEBandwidth =  (HostVMEnable == true || ImmediateFlipRequirement == dm_immediate_flip_required) ?
-					TotalMaxPrefetchFlipDPTERowBandwidth[i][j] : TotalDPTERowBandwidth[i][j];
-			DCFCLKRequiredForAverageBandwidth = dml_max3(ProjectedDCFCLKDeepSleep[i][j],
-					(NonDPTEBandwidth + TotalDPTERowBandwidth[i][j]) / ReturnBusWidth / (MaxAveragePercentOfIdealSDPPortBWDisplayCanUseInNormalSystemOperation / 100),
-					(NonDPTEBandwidth + DPTEBandwidth / PTEEfficiency) / NormalEfficiency / ReturnBusWidth);
-
-			ExtraLatencyBytes = CalculateExtraLatencyBytes(ReorderingBytes, TotalNumberOfActiveDPP[i][j], PixelChunkSizeInKByte, TotalNumberOfDCCActiveDPP[i][j],
-					MetaChunkSize, GPUVMEnable, HostVMEnable, NumberOfActivePlanes, NoOfDPPState, dpte_group_bytes,
-					PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData, PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
-					HostVMMinPageSize, HostVMMaxNonCachedPageTableLevels);
-			ExtraLatencyCycles = RoundTripPingLatencyCycles + 32 + ExtraLatencyBytes / NormalEfficiency / ReturnBusWidth;
-			for (k = 0; k < NumberOfActivePlanes; ++k) {
+			MinimumTWait = CalculateTWait(MaxPrefetchMode, v->FinalDRAMClockChangeLatency, v->UrgLatency[i], v->SREnterPlusExitTime);
+			NonDPTEBandwidth = v->TotalVActivePixelBandwidth[i][j] + v->TotalVActiveCursorBandwidth[i][j] + v->TotalMetaRowBandwidth[i][j];
+			DPTEBandwidth =  (v->HostVMEnable == true || v->ImmediateFlipRequirement[0] == dm_immediate_flip_required) ?
+					TotalMaxPrefetchFlipDPTERowBandwidth[i][j] : v->TotalDPTERowBandwidth[i][j];
+			DCFCLKRequiredForAverageBandwidth = dml_max3(v->ProjectedDCFCLKDeepSleep[i][j],
+					(NonDPTEBandwidth + v->TotalDPTERowBandwidth[i][j]) / v->ReturnBusWidth / (v->MaxAveragePercentOfIdealSDPPortBWDisplayCanUseInNormalSystemOperation / 100),
+					(NonDPTEBandwidth + DPTEBandwidth / PTEEfficiency) / NormalEfficiency / v->ReturnBusWidth);
+
+			ExtraLatencyBytes = CalculateExtraLatencyBytes(ReorderingBytes, v->TotalNumberOfActiveDPP[i][j], v->PixelChunkSizeInKByte, v->TotalNumberOfDCCActiveDPP[i][j],
+					v->MetaChunkSize, v->GPUVMEnable, v->HostVMEnable, v->NumberOfActivePlanes, NoOfDPPState, v->dpte_group_bytes,
+					v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData, v->PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
+					v->HostVMMinPageSize, v->HostVMMaxNonCachedPageTableLevels);
+			ExtraLatencyCycles = v->RoundTripPingLatencyCycles + 32 + ExtraLatencyBytes / NormalEfficiency / v->ReturnBusWidth;
+			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 				double DCFCLKCyclesRequiredInPrefetch = { 0 };
 				double ExpectedPrefetchBWAcceleration = { 0 };
 				double PrefetchTime = { 0 };
 
-				PixelDCFCLKCyclesRequiredInPrefetch[k] = (PrefetchLinesY[i][j][k] * swath_width_luma_ub_all_states[i][j][k] * BytePerPixelY[k]
-					+ PrefetchLinesC[i][j][k] * swath_width_chroma_ub_all_states[i][j][k] * BytePerPixelC[k]) / NormalEfficiency / ReturnBusWidth;
-				DCFCLKCyclesRequiredInPrefetch = 2 * ExtraLatencyCycles / NoOfDPPState[k] + PDEAndMetaPTEBytesPerFrame[i][j][k] / PTEEfficiency
-					/ NormalEfficiency / ReturnBusWidth *  (GPUVMMaxPageTableLevels > 2 ? 1 : 0) + 2 * DPTEBytesPerRow[i][j][k] / PTEEfficiency
-					/ NormalEfficiency / ReturnBusWidth + 2 * MetaRowBytes[i][j][k] / NormalEfficiency / ReturnBusWidth + PixelDCFCLKCyclesRequiredInPrefetch[k];
-				PrefetchPixelLinesTime[k] = dml_max(PrefetchLinesY[i][j][k], PrefetchLinesC[i][j][k]) * HTotal[k] / PixelClock[k];
-				ExpectedPrefetchBWAcceleration = (VActivePixelBandwidth[i][j][k] + VActiveCursorBandwidth[i][j][k]) / (ReadBandwidthLuma[k] + ReadBandwidthChroma[k]);
-				DynamicMetadataVMExtraLatency[k] = (GPUVMEnable == true && DynamicMetadataEnable[k] == true && DynamicMetadataVMEnabled == true) ?
-						UrgLatency[i] * GPUVMMaxPageTableLevels *  (HostVMEnable == true ? HostVMMaxNonCachedPageTableLevels + 1 : 1) : 0;
-				PrefetchTime = (MaximumVStartup[i][j][k] - 1) * HTotal[k] / PixelClock[k] - MinimumTWait - UrgLatency[i] * ((GPUVMMaxPageTableLevels <= 2 ? GPUVMMaxPageTableLevels
-						: GPUVMMaxPageTableLevels - 2) * (HostVMEnable == true ? HostVMMaxNonCachedPageTableLevels + 1 : 1) - 1) - DynamicMetadataVMExtraLatency[k];
+				PixelDCFCLKCyclesRequiredInPrefetch[k] = (v->PrefetchLinesY[i][j][k] * v->swath_width_luma_ub_all_states[i][j][k] * v->BytePerPixelY[k]
+					+ v->PrefetchLinesC[i][j][k] * v->swath_width_chroma_ub_all_states[i][j][k] * v->BytePerPixelC[k]) / NormalEfficiency / v->ReturnBusWidth;
+				DCFCLKCyclesRequiredInPrefetch = 2 * ExtraLatencyCycles / NoOfDPPState[k] + v->PDEAndMetaPTEBytesPerFrame[i][j][k] / PTEEfficiency
+					/ NormalEfficiency / v->ReturnBusWidth *  (v->GPUVMMaxPageTableLevels > 2 ? 1 : 0) + 2 * v->DPTEBytesPerRow[i][j][k] / PTEEfficiency
+					/ NormalEfficiency / v->ReturnBusWidth + 2 * v->MetaRowBytes[i][j][k] / NormalEfficiency / v->ReturnBusWidth + PixelDCFCLKCyclesRequiredInPrefetch[k];
+				PrefetchPixelLinesTime[k] = dml_max(v->PrefetchLinesY[i][j][k], v->PrefetchLinesC[i][j][k]) * v->HTotal[k] / v->PixelClock[k];
+				ExpectedPrefetchBWAcceleration = (v->VActivePixelBandwidth[i][j][k] + v->VActiveCursorBandwidth[i][j][k]) / (v->ReadBandwidthLuma[k] + v->ReadBandwidthChroma[k]);
+				DynamicMetadataVMExtraLatency[k] = (v->GPUVMEnable == true && v->DynamicMetadataEnable[k] == true && v->DynamicMetadataVMEnabled == true) ?
+						v->UrgLatency[i] * v->GPUVMMaxPageTableLevels *  (v->HostVMEnable == true ? v->HostVMMaxNonCachedPageTableLevels + 1 : 1) : 0;
+				PrefetchTime = (v->MaximumVStartup[i][j][k] - 1) * v->HTotal[k] / v->PixelClock[k] - MinimumTWait - v->UrgLatency[i] * ((v->GPUVMMaxPageTableLevels <= 2 ? v->GPUVMMaxPageTableLevels
+						: v->GPUVMMaxPageTableLevels - 2) * (v->HostVMEnable == true ? v->HostVMMaxNonCachedPageTableLevels + 1 : 1) - 1) - DynamicMetadataVMExtraLatency[k];
 
 				if (PrefetchTime > 0) {
 					double ExpectedVRatioPrefetch = { 0 };
 					ExpectedVRatioPrefetch = PrefetchPixelLinesTime[k] / (PrefetchTime * PixelDCFCLKCyclesRequiredInPrefetch[k] / DCFCLKCyclesRequiredInPrefetch);
 					DCFCLKRequiredForPeakBandwidthPerPlane[k] = NoOfDPPState[k] * PixelDCFCLKCyclesRequiredInPrefetch[k] / PrefetchPixelLinesTime[k]
 						* dml_max(1.0, ExpectedVRatioPrefetch) * dml_max(1.0, ExpectedVRatioPrefetch / 4) * ExpectedPrefetchBWAcceleration;
-					if (HostVMEnable == true || ImmediateFlipRequirement == dm_immediate_flip_required) {
+					if (v->HostVMEnable == true || v->ImmediateFlipRequirement[0] == dm_immediate_flip_required) {
 						DCFCLKRequiredForPeakBandwidthPerPlane[k] = DCFCLKRequiredForPeakBandwidthPerPlane[k]
-							+ NoOfDPPState[k] * DPTEBandwidth / PTEEfficiency / NormalEfficiency / ReturnBusWidth;
+							+ NoOfDPPState[k] * DPTEBandwidth / PTEEfficiency / NormalEfficiency / v->ReturnBusWidth;
 					}
 				} else {
-					DCFCLKRequiredForPeakBandwidthPerPlane[k] = DCFCLKPerState[i];
+					DCFCLKRequiredForPeakBandwidthPerPlane[k] = v->DCFCLKPerState[i];
 				}
-				if (DynamicMetadataEnable[k] == true) {
+				if (v->DynamicMetadataEnable[k] == true) {
 					double TsetupPipe = { 0 };
 					double TdmbfPipe = { 0 };
 					double TdmsksPipe = { 0 };
@@ -6756,49 +6584,49 @@ static noinline_for_stack void UseMinimumDCFCLK(
 					double AllowedTimeForUrgentExtraLatency = { 0 };
 
 					CalculateDynamicMetadataParameters(
-							MaxInterDCNTileRepeaters,
-							RequiredDPPCLK[i][j][k],
-							RequiredDISPCLK[i][j],
-							ProjectedDCFCLKDeepSleep[i][j],
-							PixelClock[k],
-							HTotal[k],
-							VTotal[k] - VActive[k],
-							DynamicMetadataTransmittedBytes[k],
-							DynamicMetadataLinesBeforeActiveRequired[k],
-							Interlace[k],
-							ProgressiveToInterlaceUnitInOPP,
+							v->MaxInterDCNTileRepeaters,
+							v->RequiredDPPCLK[i][j][k],
+							v->RequiredDISPCLK[i][j],
+							v->ProjectedDCFCLKDeepSleep[i][j],
+							v->PixelClock[k],
+							v->HTotal[k],
+							v->VTotal[k] - v->VActive[k],
+							v->DynamicMetadataTransmittedBytes[k],
+							v->DynamicMetadataLinesBeforeActiveRequired[k],
+							v->Interlace[k],
+							v->ProgressiveToInterlaceUnitInOPP,
 							&TsetupPipe,
 							&TdmbfPipe,
 							&TdmecPipe,
 							&TdmsksPipe);
-					AllowedTimeForUrgentExtraLatency = MaximumVStartup[i][j][k] * HTotal[k] / PixelClock[k] - MinimumTWait - TsetupPipe
+					AllowedTimeForUrgentExtraLatency = v->MaximumVStartup[i][j][k] * v->HTotal[k] / v->PixelClock[k] - MinimumTWait - TsetupPipe
 							- TdmbfPipe - TdmecPipe - TdmsksPipe - DynamicMetadataVMExtraLatency[k];
 					if (AllowedTimeForUrgentExtraLatency > 0) {
 						DCFCLKRequiredForPeakBandwidthPerPlane[k] = dml_max(DCFCLKRequiredForPeakBandwidthPerPlane[k],
 								ExtraLatencyCycles / AllowedTimeForUrgentExtraLatency);
 					} else {
-						DCFCLKRequiredForPeakBandwidthPerPlane[k] = DCFCLKPerState[i];
+						DCFCLKRequiredForPeakBandwidthPerPlane[k] = v->DCFCLKPerState[i];
 					}
 				}
 			}
 			DCFCLKRequiredForPeakBandwidth = 0;
-			for (k = 0; k <= NumberOfActivePlanes - 1; ++k) {
+			for (k = 0; k <= v->NumberOfActivePlanes - 1; ++k) {
 				DCFCLKRequiredForPeakBandwidth = DCFCLKRequiredForPeakBandwidth + DCFCLKRequiredForPeakBandwidthPerPlane[k];
 			}
-			MinimumTvmPlus2Tr0 = UrgLatency[i] * (GPUVMEnable == true ? (HostVMEnable == true ?
-					(GPUVMMaxPageTableLevels + 2) * (HostVMMaxNonCachedPageTableLevels + 1) - 1 : GPUVMMaxPageTableLevels + 1) : 0);
-			for (k = 0; k < NumberOfActivePlanes; ++k) {
+			MinimumTvmPlus2Tr0 = v->UrgLatency[i] * (v->GPUVMEnable == true ? (v->HostVMEnable == true ?
+					(v->GPUVMMaxPageTableLevels + 2) * (v->HostVMMaxNonCachedPageTableLevels + 1) - 1 : v->GPUVMMaxPageTableLevels + 1) : 0);
+			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 				double MaximumTvmPlus2Tr0PlusTsw = { 0 };
-				MaximumTvmPlus2Tr0PlusTsw = (MaximumVStartup[i][j][k] - 2) * HTotal[k] / PixelClock[k] - MinimumTWait - DynamicMetadataVMExtraLatency[k];
+				MaximumTvmPlus2Tr0PlusTsw = (v->MaximumVStartup[i][j][k] - 2) * v->HTotal[k] / v->PixelClock[k] - MinimumTWait - DynamicMetadataVMExtraLatency[k];
 				if (MaximumTvmPlus2Tr0PlusTsw <= MinimumTvmPlus2Tr0 + PrefetchPixelLinesTime[k] / 4) {
-					DCFCLKRequiredForPeakBandwidth = DCFCLKPerState[i];
+					DCFCLKRequiredForPeakBandwidth = v->DCFCLKPerState[i];
 				} else {
 					DCFCLKRequiredForPeakBandwidth = dml_max3(DCFCLKRequiredForPeakBandwidth, 2 * ExtraLatencyCycles
 							/ (MaximumTvmPlus2Tr0PlusTsw - MinimumTvmPlus2Tr0 - PrefetchPixelLinesTime[k] / 4),
 						(2 * ExtraLatencyCycles + PixelDCFCLKCyclesRequiredInPrefetch[k]) / (MaximumTvmPlus2Tr0PlusTsw - MinimumTvmPlus2Tr0));
 				}
 			}
-			DCFCLKState[i][j] = dml_min(DCFCLKPerState[i], 1.05 * (1 + mode_lib->vba.PercentMarginOverMinimumRequiredDCFCLK / 100)
+			v->DCFCLKState[i][j] = dml_min(v->DCFCLKPerState[i], 1.05 * (1 + mode_lib->vba.PercentMarginOverMinimumRequiredDCFCLK / 100)
 					* dml_max(DCFCLKRequiredForAverageBandwidth, DCFCLKRequiredForPeakBandwidth));
 		}
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c
index d211cf6d234c..422f17aefd4a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c
@@ -322,6 +322,7 @@ static void calculate_wm_set_for_vlevel(int vlevel,
 
 void dcn301_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
 {
+	struct _vcs_dpi_voltage_scaling_st *s = dc->scratch.update_bw_bounding_box.clock_limits;
 	struct dcn301_resource_pool *pool = TO_DCN301_RES_POOL(dc->res_pool);
 	struct clk_limit_table *clk_table = &bw_params->clk_table;
 	unsigned int i, closest_clk_lvl;
@@ -329,8 +330,7 @@ void dcn301_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param
 
 	dc_assert_fp_enabled();
 
-	memcpy(&dcn3_01_soc._clock_tmp, &dcn3_01_soc.clock_limits,
-	       sizeof(dcn3_01_soc.clock_limits));
+	memcpy(s, dcn3_01_soc.clock_limits, sizeof(dcn3_01_soc.clock_limits));
 
 	/* Default clock levels are used for diags, which may lead to overclocking. */
 	if (!IS_DIAG_DC(dc->ctx->dce_environment)) {
@@ -348,35 +348,42 @@ void dcn301_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param
 				}
 			}
 
-			dcn3_01_soc._clock_tmp[i].state = i;
-			dcn3_01_soc._clock_tmp[i].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;
-			dcn3_01_soc._clock_tmp[i].fabricclk_mhz = clk_table->entries[i].fclk_mhz;
-			dcn3_01_soc._clock_tmp[i].socclk_mhz = clk_table->entries[i].socclk_mhz;
-			dcn3_01_soc._clock_tmp[i].dram_speed_mts = clk_table->entries[i].memclk_mhz * 2;
-
-			dcn3_01_soc._clock_tmp[i].dispclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dispclk_mhz;
-			dcn3_01_soc._clock_tmp[i].dppclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dppclk_mhz;
-			dcn3_01_soc._clock_tmp[i].dram_bw_per_chan_gbps = dcn3_01_soc.clock_limits[closest_clk_lvl].dram_bw_per_chan_gbps;
-			dcn3_01_soc._clock_tmp[i].dscclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dscclk_mhz;
-			dcn3_01_soc._clock_tmp[i].dtbclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dtbclk_mhz;
-			dcn3_01_soc._clock_tmp[i].phyclk_d18_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].phyclk_d18_mhz;
-			dcn3_01_soc._clock_tmp[i].phyclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].phyclk_mhz;
+			s[i].state = i;
+			s[i].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;
+			s[i].fabricclk_mhz = clk_table->entries[i].fclk_mhz;
+			s[i].socclk_mhz = clk_table->entries[i].socclk_mhz;
+			s[i].dram_speed_mts = clk_table->entries[i].memclk_mhz * 2;
+
+			s[i].dispclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dispclk_mhz;
+			s[i].dppclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dppclk_mhz;
+			s[i].dram_bw_per_chan_gbps =
+				dcn3_01_soc.clock_limits[closest_clk_lvl].dram_bw_per_chan_gbps;
+			s[i].dscclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dscclk_mhz;
+			s[i].dtbclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].dtbclk_mhz;
+			s[i].phyclk_d18_mhz =
+				dcn3_01_soc.clock_limits[closest_clk_lvl].phyclk_d18_mhz;
+			s[i].phyclk_mhz = dcn3_01_soc.clock_limits[closest_clk_lvl].phyclk_mhz;
 		}
 
 		if (clk_table->num_entries) {
 			dcn3_01_soc.num_states = clk_table->num_entries;
 			/* duplicate last level */
-			dcn3_01_soc._clock_tmp[dcn3_01_soc.num_states] = dcn3_01_soc.clock_limits[dcn3_01_soc.num_states - 1];
-			dcn3_01_soc._clock_tmp[dcn3_01_soc.num_states].state = dcn3_01_soc.num_states;
+			s[dcn3_01_soc.num_states] =
+				dcn3_01_soc.clock_limits[dcn3_01_soc.num_states - 1];
+			s[dcn3_01_soc.num_states].state = dcn3_01_soc.num_states;
 		}
 	}
 
-	memcpy(&dcn3_01_soc.clock_limits, &dcn3_01_soc._clock_tmp,
-	       sizeof(dcn3_01_soc.clock_limits));
+	memcpy(dcn3_01_soc.clock_limits, s, sizeof(dcn3_01_soc.clock_limits));
 
 	dcn3_01_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0;
 	dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0;
 
+	if ((int)(dcn3_01_soc.dram_clock_change_latency_us * 1000)
+				!= dc->debug.dram_clock_change_latency_ns
+			&& dc->debug.dram_clock_change_latency_ns) {
+		dcn3_01_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000.0;
+	}
 	dml_init_instance(&dc->dml, &dcn3_01_soc, &dcn3_01_ip, DML_PROJECT_DCN30);
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
index fa7b0291ce4d..b6e99eefe869 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
@@ -598,14 +598,14 @@ void dcn31_calculate_wm_and_dlg_fp(
 
 void dcn31_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
 {
+	struct _vcs_dpi_voltage_scaling_st *s = dc->scratch.update_bw_bounding_box.clock_limits;
 	struct clk_limit_table *clk_table = &bw_params->clk_table;
 	unsigned int i, closest_clk_lvl;
 	int j;
 
 	dc_assert_fp_enabled();
 
-	memcpy(&dcn3_1_soc._clock_tmp, &dcn3_1_soc.clock_limits,
-	       sizeof(dcn3_1_soc.clock_limits));
+	memcpy(s, dcn3_1_soc.clock_limits, sizeof(dcn3_1_soc.clock_limits));
 
 	// Default clock levels are used for diags, which may lead to overclocking.
 	if (!IS_DIAG_DC(dc->ctx->dce_environment)) {
@@ -634,38 +634,46 @@ void dcn31_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params
 				}
 			}
 
-			dcn3_1_soc._clock_tmp[i].state = i;
+			s[i].state = i;
 
 			/* Clocks dependent on voltage level. */
-			dcn3_1_soc._clock_tmp[i].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;
-			dcn3_1_soc._clock_tmp[i].fabricclk_mhz = clk_table->entries[i].fclk_mhz;
-			dcn3_1_soc._clock_tmp[i].socclk_mhz = clk_table->entries[i].socclk_mhz;
-			dcn3_1_soc._clock_tmp[i].dram_speed_mts = clk_table->entries[i].memclk_mhz * 2 * clk_table->entries[i].wck_ratio;
+			s[i].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;
+			s[i].fabricclk_mhz = clk_table->entries[i].fclk_mhz;
+			s[i].socclk_mhz = clk_table->entries[i].socclk_mhz;
+			s[i].dram_speed_mts = clk_table->entries[i].memclk_mhz *
+				2 * clk_table->entries[i].wck_ratio;
 
 			/* Clocks independent of voltage level. */
-			dcn3_1_soc._clock_tmp[i].dispclk_mhz = max_dispclk_mhz ? max_dispclk_mhz :
+			s[i].dispclk_mhz = max_dispclk_mhz ? max_dispclk_mhz :
 				dcn3_1_soc.clock_limits[closest_clk_lvl].dispclk_mhz;
 
-			dcn3_1_soc._clock_tmp[i].dppclk_mhz = max_dppclk_mhz ? max_dppclk_mhz :
+			s[i].dppclk_mhz = max_dppclk_mhz ? max_dppclk_mhz :
 				dcn3_1_soc.clock_limits[closest_clk_lvl].dppclk_mhz;
 
-			dcn3_1_soc._clock_tmp[i].dram_bw_per_chan_gbps = dcn3_1_soc.clock_limits[closest_clk_lvl].dram_bw_per_chan_gbps;
-			dcn3_1_soc._clock_tmp[i].dscclk_mhz = dcn3_1_soc.clock_limits[closest_clk_lvl].dscclk_mhz;
-			dcn3_1_soc._clock_tmp[i].dtbclk_mhz = dcn3_1_soc.clock_limits[closest_clk_lvl].dtbclk_mhz;
-			dcn3_1_soc._clock_tmp[i].phyclk_d18_mhz = dcn3_1_soc.clock_limits[closest_clk_lvl].phyclk_d18_mhz;
-			dcn3_1_soc._clock_tmp[i].phyclk_mhz = dcn3_1_soc.clock_limits[closest_clk_lvl].phyclk_mhz;
+			s[i].dram_bw_per_chan_gbps =
+				dcn3_1_soc.clock_limits[closest_clk_lvl].dram_bw_per_chan_gbps;
+			s[i].dscclk_mhz = dcn3_1_soc.clock_limits[closest_clk_lvl].dscclk_mhz;
+			s[i].dtbclk_mhz = dcn3_1_soc.clock_limits[closest_clk_lvl].dtbclk_mhz;
+			s[i].phyclk_d18_mhz =
+				dcn3_1_soc.clock_limits[closest_clk_lvl].phyclk_d18_mhz;
+			s[i].phyclk_mhz = dcn3_1_soc.clock_limits[closest_clk_lvl].phyclk_mhz;
 		}
 		if (clk_table->num_entries) {
 			dcn3_1_soc.num_states = clk_table->num_entries;
 		}
 	}
 
-	memcpy(&dcn3_1_soc.clock_limits, &dcn3_1_soc._clock_tmp,
-	       sizeof(dcn3_1_soc.clock_limits));
+	memcpy(dcn3_1_soc.clock_limits, s, sizeof(dcn3_1_soc.clock_limits));
 
 	dcn3_1_soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0;
 	dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0;
 
+	if ((int)(dcn3_1_soc.dram_clock_change_latency_us * 1000)
+				!= dc->debug.dram_clock_change_latency_ns
+			&& dc->debug.dram_clock_change_latency_ns) {
+		dcn3_1_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000;
+	}
+
 	if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
 		dml_init_instance(&dc->dml, &dcn3_1_soc, &dcn3_1_ip, DML_PROJECT_DCN31);
 	else
@@ -724,6 +732,12 @@ void dcn315_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param
 	 */
 	dcn3_15_soc.dispclk_dppclk_vco_speed_mhz = max_dispclk_mhz * 2;
 
+	if ((int)(dcn3_15_soc.dram_clock_change_latency_us * 1000)
+				!= dc->debug.dram_clock_change_latency_ns
+			&& dc->debug.dram_clock_change_latency_ns) {
+		dcn3_15_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000;
+	}
+
 	if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
 		dml_init_instance(&dc->dml, &dcn3_15_soc, &dcn3_15_ip, DML_PROJECT_DCN31);
 	else
@@ -732,6 +746,7 @@ void dcn315_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param
 
 void dcn316_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
 {
+	struct _vcs_dpi_voltage_scaling_st *s = dc->scratch.update_bw_bounding_box.clock_limits;
 	struct clk_limit_table *clk_table = &bw_params->clk_table;
 	unsigned int i, closest_clk_lvl;
 	int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
@@ -739,8 +754,7 @@ void dcn316_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param
 
 	dc_assert_fp_enabled();
 
-	memcpy(&dcn3_16_soc._clock_tmp, &dcn3_16_soc.clock_limits,
-	       sizeof(dcn3_16_soc.clock_limits));
+	memcpy(s, dcn3_16_soc.clock_limits, sizeof(dcn3_16_soc.clock_limits));
 
 	// Default clock levels are used for diags, which may lead to overclocking.
 	if (!IS_DIAG_DC(dc->ctx->dce_environment)) {
@@ -762,7 +776,8 @@ void dcn316_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param
 		for (i = 0; i < clk_table->num_entries; i++) {
 			/* loop backwards*/
 			for (closest_clk_lvl = 0, j = dcn3_16_soc.num_states - 1; j >= 0; j--) {
-				if ((unsigned int) dcn3_16_soc.clock_limits[j].dcfclk_mhz <= clk_table->entries[i].dcfclk_mhz) {
+				if ((unsigned int) dcn3_16_soc.clock_limits[j].dcfclk_mhz <=
+				    clk_table->entries[i].dcfclk_mhz) {
 					closest_clk_lvl = j;
 					break;
 				}
@@ -773,44 +788,53 @@ void dcn316_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_param
 				closest_clk_lvl = dcn3_16_soc.num_states - 1;
 			}
 
-			dcn3_16_soc._clock_tmp[i].state = i;
+			s[i].state = i;
 
 			/* Clocks dependent on voltage level. */
-			dcn3_16_soc._clock_tmp[i].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;
+			s[i].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;
 			if (clk_table->num_entries == 1 &&
-			    dcn3_16_soc._clock_tmp[i].dcfclk_mhz < dcn3_16_soc.clock_limits[closest_clk_lvl].dcfclk_mhz) {
+			    s[i].dcfclk_mhz <
+			    dcn3_16_soc.clock_limits[closest_clk_lvl].dcfclk_mhz) {
 				/*SMU fix not released yet*/
-				dcn3_16_soc._clock_tmp[i].dcfclk_mhz = dcn3_16_soc.clock_limits[closest_clk_lvl].dcfclk_mhz;
+				s[i].dcfclk_mhz =
+					dcn3_16_soc.clock_limits[closest_clk_lvl].dcfclk_mhz;
 			}
-			dcn3_16_soc._clock_tmp[i].fabricclk_mhz = clk_table->entries[i].fclk_mhz;
-			dcn3_16_soc._clock_tmp[i].socclk_mhz = clk_table->entries[i].socclk_mhz;
-			dcn3_16_soc._clock_tmp[i].dram_speed_mts = clk_table->entries[i].memclk_mhz * 2 * clk_table->entries[i].wck_ratio;
+			s[i].fabricclk_mhz = clk_table->entries[i].fclk_mhz;
+			s[i].socclk_mhz = clk_table->entries[i].socclk_mhz;
+			s[i].dram_speed_mts = clk_table->entries[i].memclk_mhz *
+				2 * clk_table->entries[i].wck_ratio;
 
 			/* Clocks independent of voltage level. */
-			dcn3_16_soc._clock_tmp[i].dispclk_mhz = max_dispclk_mhz ? max_dispclk_mhz :
+			s[i].dispclk_mhz = max_dispclk_mhz ? max_dispclk_mhz :
 				dcn3_16_soc.clock_limits[closest_clk_lvl].dispclk_mhz;
 
-			dcn3_16_soc._clock_tmp[i].dppclk_mhz = max_dppclk_mhz ? max_dppclk_mhz :
+			s[i].dppclk_mhz = max_dppclk_mhz ? max_dppclk_mhz :
 				dcn3_16_soc.clock_limits[closest_clk_lvl].dppclk_mhz;
 
-			dcn3_16_soc._clock_tmp[i].dram_bw_per_chan_gbps = dcn3_16_soc.clock_limits[closest_clk_lvl].dram_bw_per_chan_gbps;
-			dcn3_16_soc._clock_tmp[i].dscclk_mhz = dcn3_16_soc.clock_limits[closest_clk_lvl].dscclk_mhz;
-			dcn3_16_soc._clock_tmp[i].dtbclk_mhz = dcn3_16_soc.clock_limits[closest_clk_lvl].dtbclk_mhz;
-			dcn3_16_soc._clock_tmp[i].phyclk_d18_mhz = dcn3_16_soc.clock_limits[closest_clk_lvl].phyclk_d18_mhz;
-			dcn3_16_soc._clock_tmp[i].phyclk_mhz = dcn3_16_soc.clock_limits[closest_clk_lvl].phyclk_mhz;
+			s[i].dram_bw_per_chan_gbps =
+				dcn3_16_soc.clock_limits[closest_clk_lvl].dram_bw_per_chan_gbps;
+			s[i].dscclk_mhz = dcn3_16_soc.clock_limits[closest_clk_lvl].dscclk_mhz;
+			s[i].dtbclk_mhz = dcn3_16_soc.clock_limits[closest_clk_lvl].dtbclk_mhz;
+			s[i].phyclk_d18_mhz =
+				dcn3_16_soc.clock_limits[closest_clk_lvl].phyclk_d18_mhz;
+			s[i].phyclk_mhz = dcn3_16_soc.clock_limits[closest_clk_lvl].phyclk_mhz;
 		}
 		if (clk_table->num_entries) {
 			dcn3_16_soc.num_states = clk_table->num_entries;
 		}
 	}
 
-	memcpy(&dcn3_16_soc.clock_limits, &dcn3_16_soc._clock_tmp,
-	       sizeof(dcn3_16_soc.clock_limits));
+	memcpy(dcn3_16_soc.clock_limits, s, sizeof(dcn3_16_soc.clock_limits));
 
 	if (max_dispclk_mhz) {
 		dcn3_16_soc.dispclk_dppclk_vco_speed_mhz = max_dispclk_mhz * 2;
 		dc->dml.soc.dispclk_dppclk_vco_speed_mhz = max_dispclk_mhz * 2;
 	}
+	if ((int)(dcn3_16_soc.dram_clock_change_latency_us * 1000)
+				!= dc->debug.dram_clock_change_latency_ns
+			&& dc->debug.dram_clock_change_latency_ns) {
+		dcn3_16_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000;
+	}
 
 	if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
 		dml_init_instance(&dc->dml, &dcn3_16_soc, &dcn3_16_ip, DML_PROJECT_DCN31);
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
index 8ca66f1644dc..8dfe639b6508 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
@@ -1051,10 +1051,10 @@ static bool CalculatePrefetchSchedule(
 		bytes_pp = myPipe->BytePerPixelY + myPipe->BytePerPixelC;
 	/*rev 99*/
 	prefetch_bw_pr = dml_min(1, bytes_pp * myPipe->PixelClock / (double) myPipe->DPPPerPlane);
-    max_Tsw = dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime;
+	max_Tsw = dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime;
 	prefetch_sw_bytes = PrefetchSourceLinesY * swath_width_luma_ub * myPipe->BytePerPixelY + PrefetchSourceLinesC * swath_width_chroma_ub * myPipe->BytePerPixelC;
 	prefetch_bw_oto = dml_max(bytes_pp * myPipe->PixelClock / myPipe->DPPPerPlane, prefetch_sw_bytes / (dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime));
-    prefetch_bw_oto = dml_max(prefetch_bw_pr, prefetch_sw_bytes / max_Tsw);
+	prefetch_bw_oto = dml_max(prefetch_bw_pr, prefetch_sw_bytes / max_Tsw);
 
 	min_Lsw = dml_max(1, dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) / max_vratio_pre);
 	Lsw_oto = dml_ceil(4 * dml_max(prefetch_sw_bytes / prefetch_bw_oto / LineTime, min_Lsw), 1) / 4;
@@ -6711,8 +6711,6 @@ static void CalculateSwathWidth(
 		{
 		int surface_width_ub_l = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
 		int surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
-		int surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
-		int surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
 
 #ifdef __DML_VBA_DEBUG__
 		dml_print("DML::%s: k=%d surface_width_ub_l=%0d\n", __func__, k, surface_width_ub_l);
@@ -6723,6 +6721,8 @@ static void CalculateSwathWidth(
 			MaximumSwathHeightC[k] = Read256BytesBlockHeightC[k];
 			swath_width_luma_ub[k] = dml_min(surface_width_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]);
 			if (BytePerPixC[k] > 0) {
+				int surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
+
 				swath_width_chroma_ub[k] = dml_min(
 						surface_width_ub_c,
 						(int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockWidthC[k]) + Read256BytesBlockWidthC[k]);
@@ -6734,6 +6734,8 @@ static void CalculateSwathWidth(
 			MaximumSwathHeightC[k] = Read256BytesBlockWidthC[k];
 			swath_width_luma_ub[k] = dml_min(surface_height_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]);
 			if (BytePerPixC[k] > 0) {
+				int surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
+
 				swath_width_chroma_ub[k] = dml_min(
 						surface_height_ub_c,
 						(int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockHeightC[k]) + Read256BytesBlockHeightC[k]);
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
index 4bb3b31ea7e0..cf420ad2b8dc 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
@@ -264,6 +264,11 @@ void dcn314_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p
 		dc->dml.soc.dispclk_dppclk_vco_speed_mhz = max_dispclk_mhz * 2;
 	}
 
+	if ((int)(dcn3_14_soc.dram_clock_change_latency_us * 1000)
+				!= dc->debug.dram_clock_change_latency_ns
+			&& dc->debug.dram_clock_change_latency_ns) {
+		dcn3_14_soc.dram_clock_change_latency_us = dc->debug.dram_clock_change_latency_ns / 1000;
+	}
 	if (!IS_FPGA_MAXIMUS_DC(dc->ctx->dce_environment))
 		dml_init_instance(&dc->dml, &dcn3_14_soc, &dcn3_14_ip, DML_PROJECT_DCN314);
 	else
@@ -318,6 +323,8 @@ int dcn314_populate_dml_pipes_from_context_fpu(struct dc *dc, struct dc_state *c
 		pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_luma = 0;
 		pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_chroma = 0;
 		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
+		pipes[pipe_cnt].pipe.dest.vblank_nom =
+				dcn3_14_ip.VBlankNomDefaultUS / (timing->h_total / (timing->pix_clk_100hz / 10000.0));
 		pipes[pipe_cnt].pipe.src.dcc_rate = 3;
 		pipes[pipe_cnt].dout.dsc_input_bpc = 0;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
index ee821c4fb5dd..0d12fd079cd6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
@@ -1306,7 +1306,7 @@ static bool CalculatePrefetchSchedule(
 		//                      - ((NumberOfCursors > 0 || GPUVMEnable || DCCEnable) ?
 				- ((GPUVMEnable || myPipe->DCCEnable) ? (*DestinationLinesToRequestVMInVBlank + 2 * *DestinationLinesToRequestRowInVBlank) : 0.0); // TODO: Did someone else add this??
 #else
-				LinesToRequestPrefetchPixelData = *DestinationLinesForPrefetch - *DestinationLinesToRequestVMInVBlank - 2 * *DestinationLinesToRequestRowInVBlank;
+		LinesToRequestPrefetchPixelData = *DestinationLinesForPrefetch - *DestinationLinesToRequestVMInVBlank - 2 * *DestinationLinesToRequestRowInVBlank;
 #endif
 
 #ifdef __DML_VBA_DEBUG__
@@ -6825,8 +6825,6 @@ static void CalculateSwathWidth(
 		{
 		int surface_width_ub_l = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
 		int surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
-		int surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
-		int surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
 
 #ifdef __DML_VBA_DEBUG__
 		dml_print("DML::%s: k=%d surface_width_ub_l=%0d\n", __func__, k, surface_width_ub_l);
@@ -6837,6 +6835,8 @@ static void CalculateSwathWidth(
 			MaximumSwathHeightC[k] = Read256BytesBlockHeightC[k];
 			swath_width_luma_ub[k] = dml_min(surface_width_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]);
 			if (BytePerPixC[k] > 0) {
+				int surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
+
 				swath_width_chroma_ub[k] = dml_min(
 						surface_width_ub_c,
 						(int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockWidthC[k]) + Read256BytesBlockWidthC[k]);
@@ -6848,6 +6848,8 @@ static void CalculateSwathWidth(
 			MaximumSwathHeightC[k] = Read256BytesBlockWidthC[k];
 			swath_width_luma_ub[k] = dml_min(surface_height_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]);
 			if (BytePerPixC[k] > 0) {
+				int surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
+
 				swath_width_chroma_ub[k] = dml_min(
 						surface_height_ub_c,
 						(int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockHeightC[k]) + Read256BytesBlockHeightC[k]);
@@ -7183,7 +7185,7 @@ static unsigned int CalculateMaxVStartup(
 	double line_time_us = HTotal / PixelClock;
 	unsigned int vblank_actual = VTotal - VActive;
 	unsigned int vblank_nom_default_in_line = dml_floor(VBlankNomDefaultUS / line_time_us, 1.0);
-	unsigned int vblank_nom_input = dml_min(VBlankNom, vblank_nom_default_in_line);
+	unsigned int vblank_nom_input = VBlankNom; //dml_min(VBlankNom, vblank_nom_default_in_line);
 	unsigned int vblank_avail = vblank_nom_input == 0 ? vblank_nom_default_in_line : vblank_nom_input;
 
 	vblank_size = (unsigned int) dml_min(vblank_actual, vblank_avail);
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
index f43686997917..0571700f53f9 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
@@ -121,8 +121,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_2_soc = {
 		},
 	},
 	.num_states = 1,
-	.sr_exit_time_us = 20.16,
-	.sr_enter_plus_exit_time_us = 27.13,
+	.sr_exit_time_us = 42.97,
+	.sr_enter_plus_exit_time_us = 49.94,
 	.sr_exit_z8_time_us = 285.0,
 	.sr_enter_plus_exit_z8_time_us = 320,
 	.writeback_latency_us = 12.0,
@@ -330,41 +330,92 @@ void dcn32_helper_populate_phantom_dlg_params(struct dc *dc,
 	}
 }
 
-bool dcn32_predict_pipe_split(struct dc_state *context, display_pipe_params_st pipe, int index)
+/**
+ * *******************************************************************************************
+ * dcn32_predict_pipe_split: Predict if pipe split will occur for a given DML pipe
+ *
+ * This function takes in a DML pipe (pipe_e2e) and predicts if pipe split is required (both
+ * ODM and MPC). For pipe split, ODM combine is determined by the ODM mode, and MPC combine is
+ * determined by DPPClk requirements
+ *
+ * This function follows the same policy as DML:
+ * - Check for ODM combine requirements / policy first
+ * - MPC combine is only chosen if there is no ODM combine requirements / policy in place, and
+ *   MPC is required
+ *
+ * @param [in]: context: New DC state to be programmed
+ * @param [in]: pipe_e2e: DML pipe end to end context
+ *
+ * @return: Number of splits expected (1 for 2:1 split, 3 for 4:1 split, 0 for no splits).
+ *
+ * *******************************************************************************************
+ */
+uint8_t dcn32_predict_pipe_split(struct dc_state *context,
+				  display_e2e_pipe_params_st *pipe_e2e)
 {
 	double pscl_throughput;
 	double pscl_throughput_chroma;
 	double dpp_clk_single_dpp, clock;
 	double clk_frequency = 0.0;
 	double vco_speed = context->bw_ctx.dml.soc.dispclk_dppclk_vco_speed_mhz;
+	bool total_available_pipes_support = false;
+	uint32_t number_of_dpp = 0;
+	enum odm_combine_mode odm_mode = dm_odm_combine_mode_disabled;
+	double req_dispclk_per_surface = 0;
+	uint8_t num_splits = 0;
 
 	dc_assert_fp_enabled();
 
-	dml32_CalculateSinglePipeDPPCLKAndSCLThroughput(pipe.scale_ratio_depth.hscl_ratio,
-							pipe.scale_ratio_depth.hscl_ratio_c,
-							pipe.scale_ratio_depth.vscl_ratio,
-							pipe.scale_ratio_depth.vscl_ratio_c,
-							context->bw_ctx.dml.ip.max_dchub_pscl_bw_pix_per_clk,
-							context->bw_ctx.dml.ip.max_pscl_lb_bw_pix_per_clk,
-							pipe.dest.pixel_rate_mhz,
-							pipe.src.source_format,
-							pipe.scale_taps.htaps,
-							pipe.scale_taps.htaps_c,
-							pipe.scale_taps.vtaps,
-							pipe.scale_taps.vtaps_c,
-							/* Output */
-							&pscl_throughput, &pscl_throughput_chroma,
-							&dpp_clk_single_dpp);
+	dml32_CalculateODMMode(context->bw_ctx.dml.ip.maximum_pixels_per_line_per_dsc_unit,
+			pipe_e2e->pipe.dest.hactive,
+			pipe_e2e->dout.output_format,
+			pipe_e2e->dout.output_type,
+			pipe_e2e->pipe.dest.odm_combine_policy,
+			context->bw_ctx.dml.soc.clock_limits[context->bw_ctx.dml.soc.num_states - 1].dispclk_mhz,
+			context->bw_ctx.dml.soc.clock_limits[context->bw_ctx.dml.soc.num_states - 1].dispclk_mhz,
+			pipe_e2e->dout.dsc_enable != 0,
+			0, /* TotalNumberOfActiveDPP can be 0 since we're predicting pipe split requirement */
+			context->bw_ctx.dml.ip.max_num_dpp,
+			pipe_e2e->pipe.dest.pixel_rate_mhz,
+			context->bw_ctx.dml.soc.dcn_downspread_percent,
+			context->bw_ctx.dml.ip.dispclk_ramp_margin_percent,
+			context->bw_ctx.dml.soc.dispclk_dppclk_vco_speed_mhz,
+			pipe_e2e->dout.dsc_slices,
+			/* Output */
+			&total_available_pipes_support,
+			&number_of_dpp,
+			&odm_mode,
+			&req_dispclk_per_surface);
+
+	dml32_CalculateSinglePipeDPPCLKAndSCLThroughput(pipe_e2e->pipe.scale_ratio_depth.hscl_ratio,
+			pipe_e2e->pipe.scale_ratio_depth.hscl_ratio_c,
+			pipe_e2e->pipe.scale_ratio_depth.vscl_ratio,
+			pipe_e2e->pipe.scale_ratio_depth.vscl_ratio_c,
+			context->bw_ctx.dml.ip.max_dchub_pscl_bw_pix_per_clk,
+			context->bw_ctx.dml.ip.max_pscl_lb_bw_pix_per_clk,
+			pipe_e2e->pipe.dest.pixel_rate_mhz,
+			pipe_e2e->pipe.src.source_format,
+			pipe_e2e->pipe.scale_taps.htaps,
+			pipe_e2e->pipe.scale_taps.htaps_c,
+			pipe_e2e->pipe.scale_taps.vtaps,
+			pipe_e2e->pipe.scale_taps.vtaps_c,
+			/* Output */
+			&pscl_throughput, &pscl_throughput_chroma,
+			&dpp_clk_single_dpp);
 
 	clock = dpp_clk_single_dpp * (1 + context->bw_ctx.dml.soc.dcn_downspread_percent / 100);
 
 	if (clock > 0)
-		clk_frequency = vco_speed * 4.0 / ((int)(vco_speed * 4.0));
+		clk_frequency = vco_speed * 4.0 / ((int)(vco_speed * 4.0) / clock);
 
-	if (clk_frequency > context->bw_ctx.dml.soc.clock_limits[index].dppclk_mhz)
-		return true;
-	else
-		return false;
+	if (odm_mode == dm_odm_combine_mode_2to1)
+		num_splits = 1;
+	else if (odm_mode == dm_odm_combine_mode_4to1)
+		num_splits = 3;
+	else if (clk_frequency > context->bw_ctx.dml.soc.clock_limits[context->bw_ctx.dml.soc.num_states - 1].dppclk_mhz)
+		num_splits = 1;
+
+	return num_splits;
 }
 
 static float calculate_net_bw_in_kbytes_sec(struct _vcs_dpi_voltage_scaling_st *entry)
@@ -604,6 +655,7 @@ static bool dcn32_assign_subvp_pipe(struct dc *dc,
 	bool valid_assignment_found = false;
 	unsigned int free_pipes = dcn32_get_num_free_pipes(dc, context);
 	bool current_assignment_freesync = false;
+	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
 
 	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
@@ -617,8 +669,16 @@ static bool dcn32_assign_subvp_pipe(struct dc *dc,
 		refresh_rate = (pipe->stream->timing.pix_clk_100hz * 100 +
 				pipe->stream->timing.v_total * pipe->stream->timing.h_total - 1)
 				/ (double)(pipe->stream->timing.v_total * pipe->stream->timing.h_total);
+		/* SubVP pipe candidate requirements:
+		 * - Refresh rate < 120hz
+		 * - Not able to switch in vactive naturally (switching in active means the
+		 *   DET provides enough buffer to hide the P-State switch latency -- trying
+		 *   to combine this with SubVP can cause issues with the scheduling).
+		 * - Not TMZ surface
+		 */
 		if (pipe->plane_state && !pipe->top_pipe &&
-				pipe->stream->mall_stream_config.type == SUBVP_NONE && refresh_rate < 120) {
+				pipe->stream->mall_stream_config.type == SUBVP_NONE && refresh_rate < 120 && !pipe->plane_state->address.tmz_surface &&
+				vba->ActiveDRAMClockChangeLatencyMarginPerState[vba->VoltageLevel][vba->maxMpcComb][vba->pipe_plane[pipe_idx]] <= 0) {
 			while (pipe) {
 				num_pipes++;
 				pipe = pipe->bottom_pipe;
@@ -1042,8 +1102,10 @@ static void dcn32_full_validate_bw_helper(struct dc *dc,
 
 	*vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, *pipe_cnt);
 	/* This may adjust vlevel and maxMpcComb */
-	if (*vlevel < context->bw_ctx.dml.soc.num_states)
+	if (*vlevel < context->bw_ctx.dml.soc.num_states) {
 		*vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge);
+		vba->VoltageLevel = *vlevel;
+	}
 
 	/* Conditions for setting up phantom pipes for SubVP:
 	 * 1. Not force disable SubVP
@@ -1058,8 +1120,10 @@ static void dcn32_full_validate_bw_helper(struct dc *dc,
 	    dc->debug.force_subvp_mclk_switch)) {
 
 		dcn32_merge_pipes_for_subvp(dc, context);
-		// to re-initialize viewport after the pipe merge
-		for (int i = 0; i < dc->res_pool->pipe_count; i++) {
+		memset(merge, 0, MAX_PIPES * sizeof(bool));
+
+		/* to re-initialize viewport after the pipe merge */
+		for (i = 0; i < dc->res_pool->pipe_count; i++) {
 			struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
 
 			if (!pipe_ctx->plane_state || !pipe_ctx->stream)
@@ -1135,17 +1199,31 @@ static void dcn32_full_validate_bw_helper(struct dc *dc,
 			dc->res_pool->funcs->remove_phantom_pipes(dc, context);
 			vba->DRAMClockChangeSupport[*vlevel][vba->maxMpcComb] = dm_dram_clock_change_unsupported;
 			*pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, false);
-		} else {
-			// only call dcn20_validate_apply_pipe_split_flags if we found a supported config
-			memset(split, 0, MAX_PIPES * sizeof(int));
-			memset(merge, 0, MAX_PIPES * sizeof(bool));
-			*vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge);
 
+			*vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, *pipe_cnt);
+			/* This may adjust vlevel and maxMpcComb */
+			if (*vlevel < context->bw_ctx.dml.soc.num_states) {
+				*vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge);
+				vba->VoltageLevel = *vlevel;
+			}
+		} else {
 			// Most populate phantom DLG params before programming hardware / timing for phantom pipe
 			DC_FP_START();
 			dcn32_helper_populate_phantom_dlg_params(dc, context, pipes, *pipe_cnt);
 			DC_FP_END();
 
+			/* Call validate_apply_pipe_split flags after calling DML getters for
+			 * phantom dlg params, or some of the VBA params indicating pipe split
+			 * can be overwritten by the getters.
+			 *
+			 * When setting up SubVP config, all pipes are merged before attempting to
+			 * add phantom pipes. If pipe split (ODM / MPC) is required, both the main
+			 * and phantom pipes will be split in the regular pipe splitting sequence.
+			 */
+			memset(split, 0, MAX_PIPES * sizeof(int));
+			memset(merge, 0, MAX_PIPES * sizeof(bool));
+			*vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge);
+			vba->VoltageLevel = *vlevel;
 			// Note: We can't apply the phantom pipes to hardware at this time. We have to wait
 			// until driver has acquired the DMCUB lock to do it safely.
 		}
@@ -1469,6 +1547,8 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 			memset(split, 0, sizeof(split));
 			memset(merge, 0, sizeof(merge));
 			vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
+			// dcn20_validate_apply_pipe_split_flags can modify voltage level outside of DML
+			vba->VoltageLevel = vlevel;
 		}
 	}
 
@@ -1511,6 +1591,28 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 			if (pipe->next_odm_pipe)
 				pipe->next_odm_pipe->prev_odm_pipe = pipe->prev_odm_pipe;
 
+			/*2:1ODM+MPC Split MPO to Single Pipe + MPC Split MPO*/
+			if (pipe->bottom_pipe) {
+				if (pipe->bottom_pipe->prev_odm_pipe || pipe->bottom_pipe->next_odm_pipe) {
+					/*MPC split rules will handle this case*/
+					pipe->bottom_pipe->top_pipe = NULL;
+				} else {
+					if (pipe->prev_odm_pipe->bottom_pipe) {
+						/* 3 plane MPO*/
+						pipe->bottom_pipe->top_pipe = pipe->prev_odm_pipe->bottom_pipe;
+						pipe->prev_odm_pipe->bottom_pipe->bottom_pipe = pipe->bottom_pipe;
+					} else {
+						/* 2 plane MPO*/
+						pipe->bottom_pipe->top_pipe = pipe->prev_odm_pipe;
+						pipe->prev_odm_pipe->bottom_pipe = pipe->bottom_pipe;
+					}
+				}
+			}
+
+			if (pipe->top_pipe) {
+				pipe->top_pipe->bottom_pipe = NULL;
+			}
+
 			pipe->bottom_pipe = NULL;
 			pipe->next_odm_pipe = NULL;
 			pipe->plane_state = NULL;
@@ -1643,8 +1745,20 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 		goto validate_fail;
 	}
 
-	if (repopulate_pipes)
+	if (repopulate_pipes) {
 		pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, fast_validate);
+
+		/* repopulate_pipes = 1 means the pipes were either split or merged. In this case
+		 * we have to re-calculate the DET allocation and run through DML once more to
+		 * ensure all the params are calculated correctly. We do not need to run the
+		 * pipe split check again after this call (pipes are already split / merged).
+		 * */
+		if (!fast_validate) {
+			context->bw_ctx.dml.soc.allow_for_pstate_or_stutter_in_vblank_final =
+						dm_prefetch_support_uclk_fclk_and_stutter_if_possible;
+			vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
+		}
+	}
 	*vlevel_out = vlevel;
 	*pipe_cnt_out = pipe_cnt;
 
@@ -1829,7 +1943,11 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
 	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
 	context->bw_ctx.bw.dcn.watermarks.c.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.fclk_pstate_change_ns = get_fclk_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	/* On DCN32/321, PMFW will set PSTATE_CHANGE_TYPE = 1 (FCLK) for UCLK dummy p-state.
+	 * In this case we must program FCLK WM Set C to use the UCLK dummy p-state WM
+	 * value.
+	 */
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.fclk_pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
 	context->bw_ctx.bw.dcn.watermarks.c.usr_retraining_ns = get_usr_retraining_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
 
 	if ((!pstate_en) && (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].valid)) {
@@ -1926,6 +2044,45 @@ static void remove_entry_from_table_at_index(struct _vcs_dpi_voltage_scaling_st
 	memset(&table[--(*num_entries)], 0, sizeof(struct _vcs_dpi_voltage_scaling_st));
 }
 
+void dcn32_patch_dpm_table(struct clk_bw_params *bw_params)
+{
+	int i;
+	unsigned int max_dcfclk_mhz = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0,
+			max_phyclk_mhz = 0, max_dtbclk_mhz = 0, max_fclk_mhz = 0, max_uclk_mhz = 0;
+
+	for (i = 0; i < MAX_NUM_DPM_LVL; i++) {
+		if (bw_params->clk_table.entries[i].dcfclk_mhz > max_dcfclk_mhz)
+			max_dcfclk_mhz = bw_params->clk_table.entries[i].dcfclk_mhz;
+		if (bw_params->clk_table.entries[i].fclk_mhz > max_fclk_mhz)
+			max_fclk_mhz = bw_params->clk_table.entries[i].fclk_mhz;
+		if (bw_params->clk_table.entries[i].memclk_mhz > max_uclk_mhz)
+			max_uclk_mhz = bw_params->clk_table.entries[i].memclk_mhz;
+		if (bw_params->clk_table.entries[i].dispclk_mhz > max_dispclk_mhz)
+			max_dispclk_mhz = bw_params->clk_table.entries[i].dispclk_mhz;
+		if (bw_params->clk_table.entries[i].dppclk_mhz > max_dppclk_mhz)
+			max_dppclk_mhz = bw_params->clk_table.entries[i].dppclk_mhz;
+		if (bw_params->clk_table.entries[i].phyclk_mhz > max_phyclk_mhz)
+			max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz;
+		if (bw_params->clk_table.entries[i].dtbclk_mhz > max_dtbclk_mhz)
+			max_dtbclk_mhz = bw_params->clk_table.entries[i].dtbclk_mhz;
+	}
+
+	/* Scan through clock values we currently have and if they are 0,
+	 *  then populate it with dcn3_2_soc.clock_limits[] value.
+	 *
+	 * Do it for DCFCLK, DISPCLK, DTBCLK and UCLK as any of those being
+	 *  0, will cause it to skip building the clock table.
+	 */
+	if (max_dcfclk_mhz == 0)
+		bw_params->clk_table.entries[0].dcfclk_mhz = dcn3_2_soc.clock_limits[0].dcfclk_mhz;
+	if (max_dispclk_mhz == 0)
+		bw_params->clk_table.entries[0].dispclk_mhz = dcn3_2_soc.clock_limits[0].dispclk_mhz;
+	if (max_dtbclk_mhz == 0)
+		bw_params->clk_table.entries[0].dtbclk_mhz = dcn3_2_soc.clock_limits[0].dtbclk_mhz;
+	if (max_uclk_mhz == 0)
+		bw_params->clk_table.entries[0].memclk_mhz = dcn3_2_soc.clock_limits[0].dram_speed_mts / 16;
+}
+
 static int build_synthetic_soc_states(struct clk_bw_params *bw_params,
 		struct _vcs_dpi_voltage_scaling_st *table, unsigned int *num_entries)
 {
@@ -2160,6 +2317,7 @@ void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa
 		if ((int)(dcn3_2_soc.urgent_latency_us * 1000) != dc->bb_overrides.urgent_latency_ns
 			&& dc->bb_overrides.urgent_latency_ns) {
 			dcn3_2_soc.urgent_latency_us = dc->bb_overrides.urgent_latency_ns / 1000.0;
+			dcn3_2_soc.urgent_latency_pixel_data_only_us = dc->bb_overrides.urgent_latency_ns / 1000.0;
 		}
 
 		if ((int)(dcn3_2_soc.dram_clock_change_latency_us * 1000)
@@ -2189,13 +2347,16 @@ void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa
 
 			if (dc->ctx->dc_bios->funcs->get_soc_bb_info(dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) {
 				if (bb_info.dram_clock_change_latency_100ns > 0)
-					dcn3_2_soc.dram_clock_change_latency_us = bb_info.dram_clock_change_latency_100ns * 10;
+					dcn3_2_soc.dram_clock_change_latency_us =
+						bb_info.dram_clock_change_latency_100ns * 10;
 
-			if (bb_info.dram_sr_enter_exit_latency_100ns > 0)
-				dcn3_2_soc.sr_enter_plus_exit_time_us = bb_info.dram_sr_enter_exit_latency_100ns * 10;
+				if (bb_info.dram_sr_enter_exit_latency_100ns > 0)
+					dcn3_2_soc.sr_enter_plus_exit_time_us =
+						bb_info.dram_sr_enter_exit_latency_100ns * 10;
 
-			if (bb_info.dram_sr_exit_latency_100ns > 0)
-				dcn3_2_soc.sr_exit_time_us = bb_info.dram_sr_exit_latency_100ns * 10;
+				if (bb_info.dram_sr_exit_latency_100ns > 0)
+					dcn3_2_soc.sr_exit_time_us =
+						bb_info.dram_sr_exit_latency_100ns * 10;
 			}
 		}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h
index 6ce221098979..3a3dc2ce4c73 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.h
@@ -29,11 +29,6 @@
 
 #include "clk_mgr_internal.h"
 
-#define DCN3_2_DEFAULT_DET_SIZE 256
-#define DCN3_2_MAX_DET_SIZE 1152
-#define DCN3_2_MIN_DET_SIZE 128
-#define DCN3_2_MIN_COMPBUF_SIZE_KB 128
-
 void dcn32_build_wm_range_table_fpu(struct clk_mgr_internal *clk_mgr);
 
 void dcn32_helper_populate_phantom_dlg_params(struct dc *dc,
@@ -41,9 +36,8 @@ void dcn32_helper_populate_phantom_dlg_params(struct dc *dc,
 					      display_e2e_pipe_params_st *pipes,
 					      int pipe_cnt);
 
-bool dcn32_predict_pipe_split(struct dc_state *context,
-			      display_pipe_params_st pipe,
-			      int index);
+uint8_t dcn32_predict_pipe_split(struct dc_state *context,
+				  display_e2e_pipe_params_st *pipe_e2e);
 
 void insert_entry_into_table_sorted(struct _vcs_dpi_voltage_scaling_st *table,
 				    unsigned int *num_entries,
@@ -77,4 +71,6 @@ int dcn32_find_dummy_latency_index_for_fw_based_mclk_switch(struct dc *dc,
 							    int pipe_cnt,
 							    int vlevel);
 
+void dcn32_patch_dpm_table(struct clk_bw_params *bw_params);
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c
index 6980f698eb23..75be1e1ce543 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c
@@ -677,9 +677,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				dml_ceil((double) v->WritebackDelay[mode_lib->vba.VoltageLevel][k]
 				/ (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]), 1));
 
-	// Clamp to max OTG vstartup register limit
-	if (v->MaxVStartupLines[k] > 1023)
-		v->MaxVStartupLines[k] = 1023;
+		// Clamp to max OTG vstartup register limit
+		if (v->MaxVStartupLines[k] > 1023)
+			v->MaxVStartupLines[k] = 1023;
 
 #ifdef __DML_VBA_DEBUG__
 		dml_print("DML::%s: k=%d MaxVStartupLines = %d\n", __func__, k, v->MaxVStartupLines[k]);
@@ -2004,6 +2004,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading,
 						mode_lib->vba.DISPCLKRampingMargin,
 						mode_lib->vba.DISPCLKDPPCLKVCOSpeed,
+						mode_lib->vba.NumberOfDSCSlices[k],
 
 						/* Output */
 						&v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.TotalAvailablePipesSupportNoDSC,
@@ -2026,6 +2027,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						mode_lib->vba.DISPCLKDPPCLKDSCCLKDownSpreading,
 						mode_lib->vba.DISPCLKRampingMargin,
 						mode_lib->vba.DISPCLKDPPCLKVCOSpeed,
+						mode_lib->vba.NumberOfDSCSlices[k],
 
 						/* Output */
 						&v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.TotalAvailablePipesSupportDSC,
@@ -3529,7 +3531,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						&v->FCLKChangeSupport[i][j],
 						&v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.dummy_single2[1], // double *MinActiveFCLKChangeLatencySupported
 						&mode_lib->vba.USRRetrainingSupport[i][j],
-						mode_lib->vba.ActiveDRAMClockChangeLatencyMargin);
+						mode_lib->vba.ActiveDRAMClockChangeLatencyMarginPerState[i][j]);
 			}
 		}
 	} // End of Prefetch Check
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
index 365d290bba99..ad66e241f9ae 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
@@ -721,8 +721,8 @@ void dml32_CalculateSwathWidth(
 
 	unsigned int surface_width_ub_l;
 	unsigned int surface_height_ub_l;
-	unsigned int surface_width_ub_c;
-	unsigned int surface_height_ub_c;
+	unsigned int surface_width_ub_c = 0;
+	unsigned int surface_height_ub_c = 0;
 
 #ifdef __DML_VBA_DEBUG__
 	dml_print("DML::%s: ForceSingleDPP = %d\n", __func__, ForceSingleDPP);
@@ -786,21 +786,6 @@ void dml32_CalculateSwathWidth(
 
 		surface_width_ub_l  = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
 		surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
-		surface_width_ub_c  = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
-		surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
-
-#ifdef __DML_VBA_DEBUG__
-		dml_print("DML::%s: k=%d surface_width_ub_l=%0d\n", __func__, k, surface_width_ub_l);
-		dml_print("DML::%s: k=%d surface_height_ub_l=%0d\n", __func__, k, surface_height_ub_l);
-		dml_print("DML::%s: k=%d surface_width_ub_c=%0d\n", __func__, k, surface_width_ub_c);
-		dml_print("DML::%s: k=%d surface_height_ub_c=%0d\n", __func__, k, surface_height_ub_c);
-		dml_print("DML::%s: k=%d Read256BytesBlockWidthY=%0d\n", __func__, k, Read256BytesBlockWidthY[k]);
-		dml_print("DML::%s: k=%d Read256BytesBlockHeightY=%0d\n", __func__, k, Read256BytesBlockHeightY[k]);
-		dml_print("DML::%s: k=%d Read256BytesBlockWidthC=%0d\n", __func__, k, Read256BytesBlockWidthC[k]);
-		dml_print("DML::%s: k=%d Read256BytesBlockHeightC=%0d\n", __func__, k, Read256BytesBlockHeightC[k]);
-		dml_print("DML::%s: k=%d ViewportStationary=%0d\n", __func__, k, ViewportStationary[k]);
-		dml_print("DML::%s: k=%d DPPPerSurface=%0d\n", __func__, k, DPPPerSurface[k]);
-#endif
 
 		if (!IsVertical(SourceRotation[k])) {
 			MaximumSwathHeightY[k] = Read256BytesBlockHeightY[k];
@@ -820,6 +805,7 @@ void dml32_CalculateSwathWidth(
 								Read256BytesBlockWidthY[k]);
 			}
 			if (BytePerPixC[k] > 0) {
+				surface_width_ub_c  = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
 				if (ViewportStationary[k] && DPPPerSurface[k] == 1) {
 					swath_width_chroma_ub[k] = dml_min(surface_width_ub_c,
 							dml_floor(ViewportXStartC[k] + SwathWidthC[k] +
@@ -850,6 +836,7 @@ void dml32_CalculateSwathWidth(
 						Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]);
 			}
 			if (BytePerPixC[k] > 0) {
+				surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
 				if (ViewportStationary[k] && DPPPerSurface[k] == 1) {
 					swath_width_chroma_ub[k] = dml_min(surface_height_ub_c,
 							dml_floor(ViewportYStartC[k] + SwathWidthC[k] +
@@ -868,6 +855,16 @@ void dml32_CalculateSwathWidth(
 		}
 
 #ifdef __DML_VBA_DEBUG__
+		dml_print("DML::%s: k=%d surface_width_ub_l=%0d\n", __func__, k, surface_width_ub_l);
+		dml_print("DML::%s: k=%d surface_height_ub_l=%0d\n", __func__, k, surface_height_ub_l);
+		dml_print("DML::%s: k=%d surface_width_ub_c=%0d\n", __func__, k, surface_width_ub_c);
+		dml_print("DML::%s: k=%d surface_height_ub_c=%0d\n", __func__, k, surface_height_ub_c);
+		dml_print("DML::%s: k=%d Read256BytesBlockWidthY=%0d\n", __func__, k, Read256BytesBlockWidthY[k]);
+		dml_print("DML::%s: k=%d Read256BytesBlockHeightY=%0d\n", __func__, k, Read256BytesBlockHeightY[k]);
+		dml_print("DML::%s: k=%d Read256BytesBlockWidthC=%0d\n", __func__, k, Read256BytesBlockWidthC[k]);
+		dml_print("DML::%s: k=%d Read256BytesBlockHeightC=%0d\n", __func__, k, Read256BytesBlockHeightC[k]);
+		dml_print("DML::%s: k=%d ViewportStationary=%0d\n", __func__, k, ViewportStationary[k]);
+		dml_print("DML::%s: k=%d DPPPerSurface=%0d\n", __func__, k, DPPPerSurface[k]);
 		dml_print("DML::%s: k=%d swath_width_luma_ub=%0d\n", __func__, k, swath_width_luma_ub[k]);
 		dml_print("DML::%s: k=%d swath_width_chroma_ub=%0d\n", __func__, k, swath_width_chroma_ub[k]);
 		dml_print("DML::%s: k=%d MaximumSwathHeightY=%0d\n", __func__, k, MaximumSwathHeightY[k]);
@@ -1196,6 +1193,7 @@ void dml32_CalculateODMMode(
 		double DISPCLKDPPCLKDSCCLKDownSpreading,
 		double DISPCLKRampingMargin,
 		double DISPCLKDPPCLKVCOSpeed,
+		unsigned int NumberOfDSCSlices,
 
 		/* Output */
 		bool *TotalAvailablePipesSupport,
@@ -1231,7 +1229,8 @@ void dml32_CalculateODMMode(
 
 	if (!(Output == dm_hdmi || Output == dm_dp || Output == dm_edp) && (ODMUse == dm_odm_combine_policy_4to1 ||
 			((SurfaceRequiredDISPCLKWithODMCombineTwoToOne > StateDispclk ||
-					(DSCEnable && (HActive > 2 * MaximumPixelsPerLinePerDSCUnit)))))) {
+					(DSCEnable && (HActive > 2 * MaximumPixelsPerLinePerDSCUnit))
+					|| NumberOfDSCSlices > 8)))) {
 		if (TotalNumberOfActiveDPP + 4 <= MaxNumDPP) {
 			*ODMMode = dm_odm_combine_mode_4to1;
 			*RequiredDISPCLKPerSurface = SurfaceRequiredDISPCLKWithODMCombineFourToOne;
@@ -1242,7 +1241,8 @@ void dml32_CalculateODMMode(
 	} else if (Output != dm_hdmi && (ODMUse == dm_odm_combine_policy_2to1 ||
 			(((SurfaceRequiredDISPCLKWithoutODMCombine > StateDispclk &&
 					SurfaceRequiredDISPCLKWithODMCombineTwoToOne <= StateDispclk) ||
-					(DSCEnable && (HActive > MaximumPixelsPerLinePerDSCUnit)))))) {
+					(DSCEnable && (HActive > MaximumPixelsPerLinePerDSCUnit))
+					|| (NumberOfDSCSlices <= 8 && NumberOfDSCSlices > 4))))) {
 		if (TotalNumberOfActiveDPP + 2 <= MaxNumDPP) {
 			*ODMMode = dm_odm_combine_mode_2to1;
 			*RequiredDISPCLKPerSurface = SurfaceRequiredDISPCLKWithODMCombineTwoToOne;
@@ -1896,7 +1896,7 @@ void dml32_CalculateSurfaceSizeInMall(
 		if (UseMALLForStaticScreen[k] == dm_use_mall_static_screen_enable)
 			TotalSurfaceSizeInMALL = TotalSurfaceSizeInMALL + SurfaceSizeInMALL[k];
 	}
-	*ExceededMALLSize =  (TotalSurfaceSizeInMALL <= MALLAllocatedForDCN * 1024 * 1024 ? false : true);
+	*ExceededMALLSize =  (TotalSurfaceSizeInMALL > MALLAllocatedForDCN * 1024 * 1024);
 } // CalculateSurfaceSizeInMall
 
 void dml32_CalculateVMRowAndSwath(
@@ -4260,7 +4260,7 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport(
 	double ActiveClockChangeLatencyHidingY;
 	double ActiveClockChangeLatencyHidingC;
 	double ActiveClockChangeLatencyHiding;
-    double EffectiveDETBufferSizeY;
+	double EffectiveDETBufferSizeY;
 	double     ActiveFCLKChangeLatencyMargin[DC__NUM_DPP__MAX];
 	double     USRRetrainingLatencyMargin[DC__NUM_DPP__MAX];
 	double TotalPixelBW = 0.0;
@@ -4643,10 +4643,6 @@ void dml32_CalculateMinAndMaxPrefetchMode(
 	} else if (AllowForPStateChangeOrStutterInVBlankFinal == dm_prefetch_support_uclk_fclk_and_stutter) {
 		*MinPrefetchMode = 0;
 		*MaxPrefetchMode = 0;
-	} else if (AllowForPStateChangeOrStutterInVBlankFinal ==
-			dm_prefetch_support_uclk_fclk_and_stutter_if_possible) {
-		*MinPrefetchMode = 0;
-		*MaxPrefetchMode = 3;
 	} else {
 		*MinPrefetchMode = 0;
 		*MaxPrefetchMode = 3;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h
index 0b427d89b3c5..55cead0d4237 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h
@@ -228,6 +228,7 @@ void dml32_CalculateODMMode(
 		double DISPCLKDPPCLKDSCCLKDownSpreading,
 		double DISPCLKRampingMargin,
 		double DISPCLKDPPCLKVCOSpeed,
+		unsigned int NumberOfDSCSlices,
 
 		/* Output */
 		bool *TotalAvailablePipesSupport,
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c
index c87091683b5d..dd90f241e906 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c
@@ -489,6 +489,7 @@ void dcn321_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p
 		if ((int)(dcn3_21_soc.urgent_latency_us * 1000) != dc->bb_overrides.urgent_latency_ns
 			&& dc->bb_overrides.urgent_latency_ns) {
 			dcn3_21_soc.urgent_latency_us = dc->bb_overrides.urgent_latency_ns / 1000.0;
+			dcn3_21_soc.urgent_latency_pixel_data_only_us = dc->bb_overrides.urgent_latency_ns / 1000.0;
 		}
 
 		if ((int)(dcn3_21_soc.dram_clock_change_latency_us * 1000)
@@ -518,13 +519,16 @@ void dcn321_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p
 
 			if (dc->ctx->dc_bios->funcs->get_soc_bb_info(dc->ctx->dc_bios, &bb_info) == BP_RESULT_OK) {
 				if (bb_info.dram_clock_change_latency_100ns > 0)
-					dcn3_21_soc.dram_clock_change_latency_us = bb_info.dram_clock_change_latency_100ns * 10;
+					dcn3_21_soc.dram_clock_change_latency_us =
+						bb_info.dram_clock_change_latency_100ns * 10;
 
-			if (bb_info.dram_sr_enter_exit_latency_100ns > 0)
-				dcn3_21_soc.sr_enter_plus_exit_time_us = bb_info.dram_sr_enter_exit_latency_100ns * 10;
+				if (bb_info.dram_sr_enter_exit_latency_100ns > 0)
+					dcn3_21_soc.sr_enter_plus_exit_time_us =
+						bb_info.dram_sr_enter_exit_latency_100ns * 10;
 
-			if (bb_info.dram_sr_exit_latency_100ns > 0)
-				dcn3_21_soc.sr_exit_time_us = bb_info.dram_sr_exit_latency_100ns * 10;
+				if (bb_info.dram_sr_exit_latency_100ns > 0)
+					dcn3_21_soc.sr_exit_time_us =
+						bb_info.dram_sr_exit_latency_100ns * 10;
 			}
 		}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
index e8b094006d95..f33a8879b05a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_structs.h
@@ -26,6 +26,16 @@
 #include "dc_features.h"
 #include "display_mode_enums.h"
 
+/**
+ * DOC: overview
+ *
+ * Most of the DML code is automatically generated and tested via hardware
+ * description language. Usually, we use the reference _vcs_dpi in the code
+ * where VCS means "Verilog Compiled Simulator" and DPI stands for "Direct
+ * Programmer Interface". In other words, those structs can be used to
+ * interface with Verilog with other languages such as C.
+ */
+
 #ifndef __DISPLAY_MODE_STRUCTS_H__
 #define __DISPLAY_MODE_STRUCTS_H__
 
@@ -159,13 +169,20 @@ struct _vcs_dpi_voltage_scaling_st {
 	double dtbclk_mhz;
 };
 
+/**
+ * _vcs_dpi_soc_bounding_box_st: SOC definitions
+ *
+ * This struct maintains the SOC Bounding Box information for the ASIC; it
+ * defines things such as clock, voltage, performance, etc. Usually, we load
+ * these values from VBIOS; if something goes wrong, we use some hard-coded
+ * values, which will enable the ASIC to light up with limitations.
+ */
 struct _vcs_dpi_soc_bounding_box_st {
 	struct _vcs_dpi_voltage_scaling_st clock_limits[DC__VOLTAGE_STATES];
-	/*
-	 * This is a temporary stash for updating @clock_limits with the PMFW
-	 * clock table. Do not use outside of *update_bw_boudning_box functions.
+	/**
+	 * @num_states: It represents the total of Display Power Management
+	 * (DPM) supported by the specific ASIC.
 	 */
-	struct _vcs_dpi_voltage_scaling_st _clock_tmp[DC__VOLTAGE_STATES];
 	unsigned int num_states;
 	double sr_exit_time_us;
 	double sr_enter_plus_exit_time_us;
@@ -231,6 +248,14 @@ struct _vcs_dpi_soc_bounding_box_st {
 	enum self_refresh_affinity allow_dram_self_refresh_or_dram_clock_change_in_vblank;
 };
 
+/**
+ * @_vcs_dpi_ip_params_st: IP configuraion for DCN blocks
+ *
+ * In this struct you can find the DCN configuration associated to the specific
+ * ASIC. For example, here we can save how many DPPs the ASIC is using and it
+ * is available.
+ *
+ */
 struct _vcs_dpi_ip_params_st {
 	bool use_min_dcfclk;
 	bool clamp_min_dcfclk;
@@ -283,6 +308,9 @@ struct _vcs_dpi_ip_params_st {
 	unsigned int writeback_line_buffer_chroma_buffer_size;
 
 	unsigned int max_page_table_levels;
+	/**
+	 * @max_num_dpp: Maximum number of DPP supported in the target ASIC.
+	 */
 	unsigned int max_num_dpp;
 	unsigned int max_num_otg;
 	unsigned int cursor_chunk_size;
@@ -482,6 +510,7 @@ struct _vcs_dpi_display_pipe_dest_params_st {
 	unsigned int htotal;
 	unsigned int vtotal;
 	unsigned int vfront_porch;
+	unsigned int vblank_nom;
 	unsigned int vactive;
 	unsigned int hactive;
 	unsigned int vstartup_start;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
index 503e7d984ff0..03924aed8d5c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
@@ -597,6 +597,7 @@ static void fetch_pipe_params(struct display_mode_lib *mode_lib)
 		mode_lib->vba.HTotal[mode_lib->vba.NumberOfActivePlanes] = dst->htotal;
 		mode_lib->vba.VTotal[mode_lib->vba.NumberOfActivePlanes] = dst->vtotal;
 		mode_lib->vba.VFrontPorch[mode_lib->vba.NumberOfActivePlanes] = dst->vfront_porch;
+		mode_lib->vba.VBlankNom[mode_lib->vba.NumberOfActivePlanes] = dst->vblank_nom;
 		mode_lib->vba.DCCFractionOfZeroSizeRequestsLuma[mode_lib->vba.NumberOfActivePlanes] = src->dcc_fraction_of_zs_req_luma;
 		mode_lib->vba.DCCFractionOfZeroSizeRequestsChroma[mode_lib->vba.NumberOfActivePlanes] = src->dcc_fraction_of_zs_req_chroma;
 		mode_lib->vba.DCCEnable[mode_lib->vba.NumberOfActivePlanes] =
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h
index 2051ddaa641a..630f3395e90a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h
@@ -312,6 +312,7 @@ struct vba_vars_st {
 	unsigned int ActiveDPPs;
 	unsigned int LBLatencyHidingSourceLinesY;
 	unsigned int LBLatencyHidingSourceLinesC;
+	double ActiveDRAMClockChangeLatencyMarginPerState[DC__VOLTAGE_STATES][2][DC__NUM_DPP__MAX];// DML doesn't save active margin per state
 	double ActiveDRAMClockChangeLatencyMargin[DC__NUM_DPP__MAX];
 	double CachedActiveDRAMClockChangeLatencyMargin[DC__NUM_DPP__MAX]; // Cache in dml_get_voltage_level for debug purposes only
 	double MinActiveDRAMClockChangeMargin;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h b/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
index 479d7d83220c..072bd0539605 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
@@ -76,14 +76,9 @@ static inline double dml_floor(double a, double granularity)
 
 static inline double dml_round(double a)
 {
-	double round_pt = 0.5;
-	double ceil = dml_ceil(a, 1);
-	double floor = dml_floor(a, 1);
+	const double round_pt = 0.5;
 
-	if (a - floor >= round_pt)
-		return ceil;
-	else
-		return floor;
+	return dml_floor(a + round_pt, 1);
 }
 
 /* float
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml_wrapper.c b/drivers/gpu/drm/amd/display/dc/dml/dml_wrapper.c
deleted file mode 100644
index b4b51e51fc25..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dml/dml_wrapper.c
+++ /dev/null
@@ -1,1884 +0,0 @@
-/*
- * Copyright 2017 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#include "resource.h"
-#include "core_types.h"
-#include "dsc.h"
-#include "clk_mgr.h"
-
-#ifndef DC_LOGGER_INIT
-#define DC_LOGGER_INIT
-#undef DC_LOG_WARNING
-#define DC_LOG_WARNING
-#endif
-
-#define DML_WRAPPER_TRANSLATION_
-#include "dml_wrapper_translation.c"
-#undef DML_WRAPPER_TRANSLATION_
-
-static bool is_dual_plane(enum surface_pixel_format format)
-{
-	return format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN || format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA;
-}
-
-static void build_clamping_params(struct dc_stream_state *stream)
-{
-	stream->clamping.clamping_level = CLAMPING_FULL_RANGE;
-	stream->clamping.c_depth = stream->timing.display_color_depth;
-	stream->clamping.pixel_encoding = stream->timing.pixel_encoding;
-}
-
-static void get_pixel_clock_parameters(
-	const struct pipe_ctx *pipe_ctx,
-	struct pixel_clk_params *pixel_clk_params)
-{
-	const struct dc_stream_state *stream = pipe_ctx->stream;
-
-	/*TODO: is this halved for YCbCr 420? in that case we might want to move
-	 * the pixel clock normalization for hdmi up to here instead of doing it
-	 * in pll_adjust_pix_clk
-	 */
-	pixel_clk_params->requested_pix_clk_100hz = stream->timing.pix_clk_100hz;
-	pixel_clk_params->encoder_object_id = stream->link->link_enc->id;
-	pixel_clk_params->signal_type = pipe_ctx->stream->signal;
-	pixel_clk_params->controller_id = pipe_ctx->stream_res.tg->inst + 1;
-	/* TODO: un-hardcode*/
-	pixel_clk_params->requested_sym_clk = LINK_RATE_LOW *
-						LINK_RATE_REF_FREQ_IN_KHZ;
-	pixel_clk_params->flags.ENABLE_SS = 0;
-	pixel_clk_params->color_depth =
-		stream->timing.display_color_depth;
-	pixel_clk_params->flags.DISPLAY_BLANKED = 1;
-	pixel_clk_params->flags.SUPPORT_YCBCR420 = (stream->timing.pixel_encoding ==
-			PIXEL_ENCODING_YCBCR420);
-	pixel_clk_params->pixel_encoding = stream->timing.pixel_encoding;
-	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR422) {
-		pixel_clk_params->color_depth = COLOR_DEPTH_888;
-	}
-	if (stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) {
-		pixel_clk_params->requested_pix_clk_100hz  = pixel_clk_params->requested_pix_clk_100hz / 2;
-	}
-	if (stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
-		pixel_clk_params->requested_pix_clk_100hz *= 2;
-
-}
-
-static void resource_build_bit_depth_reduction_params(struct dc_stream_state *stream,
-		struct bit_depth_reduction_params *fmt_bit_depth)
-{
-	enum dc_dither_option option = stream->dither_option;
-	enum dc_pixel_encoding pixel_encoding =
-			stream->timing.pixel_encoding;
-
-	memset(fmt_bit_depth, 0, sizeof(*fmt_bit_depth));
-
-	if (option == DITHER_OPTION_DEFAULT) {
-		switch (stream->timing.display_color_depth) {
-		case COLOR_DEPTH_666:
-			option = DITHER_OPTION_SPATIAL6;
-			break;
-		case COLOR_DEPTH_888:
-			option = DITHER_OPTION_SPATIAL8;
-			break;
-		case COLOR_DEPTH_101010:
-			option = DITHER_OPTION_SPATIAL10;
-			break;
-		default:
-			option = DITHER_OPTION_DISABLE;
-		}
-	}
-
-	if (option == DITHER_OPTION_DISABLE)
-		return;
-
-	if (option == DITHER_OPTION_TRUN6) {
-		fmt_bit_depth->flags.TRUNCATE_ENABLED = 1;
-		fmt_bit_depth->flags.TRUNCATE_DEPTH = 0;
-	} else if (option == DITHER_OPTION_TRUN8 ||
-			option == DITHER_OPTION_TRUN8_SPATIAL6 ||
-			option == DITHER_OPTION_TRUN8_FM6) {
-		fmt_bit_depth->flags.TRUNCATE_ENABLED = 1;
-		fmt_bit_depth->flags.TRUNCATE_DEPTH = 1;
-	} else if (option == DITHER_OPTION_TRUN10        ||
-			option == DITHER_OPTION_TRUN10_SPATIAL6   ||
-			option == DITHER_OPTION_TRUN10_SPATIAL8   ||
-			option == DITHER_OPTION_TRUN10_FM8     ||
-			option == DITHER_OPTION_TRUN10_FM6     ||
-			option == DITHER_OPTION_TRUN10_SPATIAL8_FM6) {
-		fmt_bit_depth->flags.TRUNCATE_ENABLED = 1;
-		fmt_bit_depth->flags.TRUNCATE_DEPTH = 2;
-	}
-
-	/* special case - Formatter can only reduce by 4 bits at most.
-	 * When reducing from 12 to 6 bits,
-	 * HW recommends we use trunc with round mode
-	 * (if we did nothing, trunc to 10 bits would be used)
-	 * note that any 12->10 bit reduction is ignored prior to DCE8,
-	 * as the input was 10 bits.
-	 */
-	if (option == DITHER_OPTION_SPATIAL6_FRAME_RANDOM ||
-			option == DITHER_OPTION_SPATIAL6 ||
-			option == DITHER_OPTION_FM6) {
-		fmt_bit_depth->flags.TRUNCATE_ENABLED = 1;
-		fmt_bit_depth->flags.TRUNCATE_DEPTH = 2;
-		fmt_bit_depth->flags.TRUNCATE_MODE = 1;
-	}
-
-	/* spatial dither
-	 * note that spatial modes 1-3 are never used
-	 */
-	if (option == DITHER_OPTION_SPATIAL6_FRAME_RANDOM            ||
-			option == DITHER_OPTION_SPATIAL6 ||
-			option == DITHER_OPTION_TRUN10_SPATIAL6      ||
-			option == DITHER_OPTION_TRUN8_SPATIAL6) {
-		fmt_bit_depth->flags.SPATIAL_DITHER_ENABLED = 1;
-		fmt_bit_depth->flags.SPATIAL_DITHER_DEPTH = 0;
-		fmt_bit_depth->flags.HIGHPASS_RANDOM = 1;
-		fmt_bit_depth->flags.RGB_RANDOM =
-				(pixel_encoding == PIXEL_ENCODING_RGB) ? 1 : 0;
-	} else if (option == DITHER_OPTION_SPATIAL8_FRAME_RANDOM            ||
-			option == DITHER_OPTION_SPATIAL8 ||
-			option == DITHER_OPTION_SPATIAL8_FM6        ||
-			option == DITHER_OPTION_TRUN10_SPATIAL8      ||
-			option == DITHER_OPTION_TRUN10_SPATIAL8_FM6) {
-		fmt_bit_depth->flags.SPATIAL_DITHER_ENABLED = 1;
-		fmt_bit_depth->flags.SPATIAL_DITHER_DEPTH = 1;
-		fmt_bit_depth->flags.HIGHPASS_RANDOM = 1;
-		fmt_bit_depth->flags.RGB_RANDOM =
-				(pixel_encoding == PIXEL_ENCODING_RGB) ? 1 : 0;
-	} else if (option == DITHER_OPTION_SPATIAL10_FRAME_RANDOM ||
-			option == DITHER_OPTION_SPATIAL10 ||
-			option == DITHER_OPTION_SPATIAL10_FM8 ||
-			option == DITHER_OPTION_SPATIAL10_FM6) {
-		fmt_bit_depth->flags.SPATIAL_DITHER_ENABLED = 1;
-		fmt_bit_depth->flags.SPATIAL_DITHER_DEPTH = 2;
-		fmt_bit_depth->flags.HIGHPASS_RANDOM = 1;
-		fmt_bit_depth->flags.RGB_RANDOM =
-				(pixel_encoding == PIXEL_ENCODING_RGB) ? 1 : 0;
-	}
-
-	if (option == DITHER_OPTION_SPATIAL6 ||
-			option == DITHER_OPTION_SPATIAL8 ||
-			option == DITHER_OPTION_SPATIAL10) {
-		fmt_bit_depth->flags.FRAME_RANDOM = 0;
-	} else {
-		fmt_bit_depth->flags.FRAME_RANDOM = 1;
-	}
-
-	//////////////////////
-	//// temporal dither
-	//////////////////////
-	if (option == DITHER_OPTION_FM6           ||
-			option == DITHER_OPTION_SPATIAL8_FM6     ||
-			option == DITHER_OPTION_SPATIAL10_FM6     ||
-			option == DITHER_OPTION_TRUN10_FM6     ||
-			option == DITHER_OPTION_TRUN8_FM6      ||
-			option == DITHER_OPTION_TRUN10_SPATIAL8_FM6) {
-		fmt_bit_depth->flags.FRAME_MODULATION_ENABLED = 1;
-		fmt_bit_depth->flags.FRAME_MODULATION_DEPTH = 0;
-	} else if (option == DITHER_OPTION_FM8        ||
-			option == DITHER_OPTION_SPATIAL10_FM8  ||
-			option == DITHER_OPTION_TRUN10_FM8) {
-		fmt_bit_depth->flags.FRAME_MODULATION_ENABLED = 1;
-		fmt_bit_depth->flags.FRAME_MODULATION_DEPTH = 1;
-	} else if (option == DITHER_OPTION_FM10) {
-		fmt_bit_depth->flags.FRAME_MODULATION_ENABLED = 1;
-		fmt_bit_depth->flags.FRAME_MODULATION_DEPTH = 2;
-	}
-
-	fmt_bit_depth->pixel_encoding = pixel_encoding;
-}
-
-/* Move this after the above function as VS complains about
- * declaration issues for resource_build_bit_depth_reduction_params.
- */
-
-static enum dc_status build_pipe_hw_param(struct pipe_ctx *pipe_ctx)
-{
-
-	get_pixel_clock_parameters(pipe_ctx, &pipe_ctx->stream_res.pix_clk_params);
-
-	if (pipe_ctx->clock_source)
-		pipe_ctx->clock_source->funcs->get_pix_clk_dividers(
-			pipe_ctx->clock_source,
-			&pipe_ctx->stream_res.pix_clk_params,
-			&pipe_ctx->pll_settings);
-
-	pipe_ctx->stream->clamping.pixel_encoding = pipe_ctx->stream->timing.pixel_encoding;
-
-	resource_build_bit_depth_reduction_params(pipe_ctx->stream,
-		&pipe_ctx->stream->bit_depth_params);
-	build_clamping_params(pipe_ctx->stream);
-
-	return DC_OK;
-}
-
-bool dml_validate_dsc(struct dc *dc, struct dc_state *new_ctx)
-{
-	int i;
-
-	/* Validate DSC config, dsc count validation is already done */
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe_ctx = &new_ctx->res_ctx.pipe_ctx[i];
-		struct dc_stream_state *stream = pipe_ctx->stream;
-		struct dsc_config dsc_cfg;
-		struct pipe_ctx *odm_pipe;
-		int opp_cnt = 1;
-
-		for (odm_pipe = pipe_ctx->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
-			opp_cnt++;
-
-		/* Only need to validate top pipe */
-		if (pipe_ctx->top_pipe || pipe_ctx->prev_odm_pipe || !stream || !stream->timing.flags.DSC)
-			continue;
-
-		dsc_cfg.pic_width = (stream->timing.h_addressable + stream->timing.h_border_left
-				+ stream->timing.h_border_right) / opp_cnt;
-		dsc_cfg.pic_height = stream->timing.v_addressable + stream->timing.v_border_top
-				+ stream->timing.v_border_bottom;
-		dsc_cfg.pixel_encoding = stream->timing.pixel_encoding;
-		dsc_cfg.color_depth = stream->timing.display_color_depth;
-		dsc_cfg.is_odm = pipe_ctx->next_odm_pipe ? true : false;
-		dsc_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
-		dsc_cfg.dc_dsc_cfg.num_slices_h /= opp_cnt;
-
-		if (pipe_ctx->stream_res.dsc && !pipe_ctx->stream_res.dsc->funcs->dsc_validate_stream(pipe_ctx->stream_res.dsc, &dsc_cfg))
-			return false;
-	}
-	return true;
-}
-
-enum dc_status dml_build_mapped_resource(const struct dc *dc, struct dc_state *context, struct dc_stream_state *stream)
-{
-	enum dc_status status = DC_OK;
-	struct pipe_ctx *pipe_ctx = resource_get_head_pipe_for_stream(&context->res_ctx, stream);
-
-	if (!pipe_ctx)
-		return DC_ERROR_UNEXPECTED;
-
-
-	status = build_pipe_hw_param(pipe_ctx);
-
-	return status;
-}
-
-void dml_acquire_dsc(const struct dc *dc,
-			struct resource_context *res_ctx,
-			struct display_stream_compressor **dsc,
-			int pipe_idx)
-{
-	int i;
-	const struct resource_pool *pool = dc->res_pool;
-	struct display_stream_compressor *dsc_old = dc->current_state->res_ctx.pipe_ctx[pipe_idx].stream_res.dsc;
-
-	ASSERT(*dsc == NULL); /* If this ASSERT fails, dsc was not released properly */
-	*dsc = NULL;
-
-	/* Always do 1-to-1 mapping when number of DSCs is same as number of pipes */
-	if (pool->res_cap->num_dsc == pool->res_cap->num_opp) {
-		*dsc = pool->dscs[pipe_idx];
-		res_ctx->is_dsc_acquired[pipe_idx] = true;
-		return;
-	}
-
-	/* Return old DSC to avoid the need for redo it */
-	if (dsc_old && !res_ctx->is_dsc_acquired[dsc_old->inst]) {
-		*dsc = dsc_old;
-		res_ctx->is_dsc_acquired[dsc_old->inst] = true;
-		return ;
-	}
-
-	/* Find first free DSC */
-	for (i = 0; i < pool->res_cap->num_dsc; i++)
-		if (!res_ctx->is_dsc_acquired[i]) {
-			*dsc = pool->dscs[i];
-			res_ctx->is_dsc_acquired[i] = true;
-			break;
-		}
-}
-
-static bool dml_split_stream_for_mpc_or_odm(
-		const struct dc *dc,
-		struct resource_context *res_ctx,
-		struct pipe_ctx *pri_pipe,
-		struct pipe_ctx *sec_pipe,
-		bool odm)
-{
-	int pipe_idx = sec_pipe->pipe_idx;
-	const struct resource_pool *pool = dc->res_pool;
-
-	*sec_pipe = *pri_pipe;
-
-	sec_pipe->pipe_idx = pipe_idx;
-	sec_pipe->plane_res.mi = pool->mis[pipe_idx];
-	sec_pipe->plane_res.hubp = pool->hubps[pipe_idx];
-	sec_pipe->plane_res.ipp = pool->ipps[pipe_idx];
-	sec_pipe->plane_res.xfm = pool->transforms[pipe_idx];
-	sec_pipe->plane_res.dpp = pool->dpps[pipe_idx];
-	sec_pipe->plane_res.mpcc_inst = pool->dpps[pipe_idx]->inst;
-	sec_pipe->stream_res.dsc = NULL;
-	if (odm) {
-		if (pri_pipe->next_odm_pipe) {
-			ASSERT(pri_pipe->next_odm_pipe != sec_pipe);
-			sec_pipe->next_odm_pipe = pri_pipe->next_odm_pipe;
-			sec_pipe->next_odm_pipe->prev_odm_pipe = sec_pipe;
-		}
-		if (pri_pipe->top_pipe && pri_pipe->top_pipe->next_odm_pipe) {
-			pri_pipe->top_pipe->next_odm_pipe->bottom_pipe = sec_pipe;
-			sec_pipe->top_pipe = pri_pipe->top_pipe->next_odm_pipe;
-		}
-		if (pri_pipe->bottom_pipe && pri_pipe->bottom_pipe->next_odm_pipe) {
-			pri_pipe->bottom_pipe->next_odm_pipe->top_pipe = sec_pipe;
-			sec_pipe->bottom_pipe = pri_pipe->bottom_pipe->next_odm_pipe;
-		}
-		pri_pipe->next_odm_pipe = sec_pipe;
-		sec_pipe->prev_odm_pipe = pri_pipe;
-		ASSERT(sec_pipe->top_pipe == NULL);
-
-		if (!sec_pipe->top_pipe)
-			sec_pipe->stream_res.opp = pool->opps[pipe_idx];
-		else
-			sec_pipe->stream_res.opp = sec_pipe->top_pipe->stream_res.opp;
-		if (sec_pipe->stream->timing.flags.DSC == 1) {
-			dml_acquire_dsc(dc, res_ctx, &sec_pipe->stream_res.dsc, pipe_idx);
-			ASSERT(sec_pipe->stream_res.dsc);
-			if (sec_pipe->stream_res.dsc == NULL)
-				return false;
-		}
-	} else {
-		if (pri_pipe->bottom_pipe) {
-			ASSERT(pri_pipe->bottom_pipe != sec_pipe);
-			sec_pipe->bottom_pipe = pri_pipe->bottom_pipe;
-			sec_pipe->bottom_pipe->top_pipe = sec_pipe;
-		}
-		pri_pipe->bottom_pipe = sec_pipe;
-		sec_pipe->top_pipe = pri_pipe;
-
-		ASSERT(pri_pipe->plane_state);
-	}
-
-	return true;
-}
-
-static struct pipe_ctx *dml_find_split_pipe(
-		struct dc *dc,
-		struct dc_state *context,
-		int old_index)
-{
-	struct pipe_ctx *pipe = NULL;
-	int i;
-
-	if (old_index >= 0 && context->res_ctx.pipe_ctx[old_index].stream == NULL) {
-		pipe = &context->res_ctx.pipe_ctx[old_index];
-		pipe->pipe_idx = old_index;
-	}
-
-	if (!pipe)
-		for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
-			if (dc->current_state->res_ctx.pipe_ctx[i].top_pipe == NULL
-					&& dc->current_state->res_ctx.pipe_ctx[i].prev_odm_pipe == NULL) {
-				if (context->res_ctx.pipe_ctx[i].stream == NULL) {
-					pipe = &context->res_ctx.pipe_ctx[i];
-					pipe->pipe_idx = i;
-					break;
-				}
-			}
-		}
-
-	/*
-	 * May need to fix pipes getting tossed from 1 opp to another on flip
-	 * Add for debugging transient underflow during topology updates:
-	 * ASSERT(pipe);
-	 */
-	if (!pipe)
-		for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
-			if (context->res_ctx.pipe_ctx[i].stream == NULL) {
-				pipe = &context->res_ctx.pipe_ctx[i];
-				pipe->pipe_idx = i;
-				break;
-			}
-		}
-
-	return pipe;
-}
-
-static void dml_release_dsc(struct resource_context *res_ctx,
-			const struct resource_pool *pool,
-			struct display_stream_compressor **dsc)
-{
-	int i;
-
-	for (i = 0; i < pool->res_cap->num_dsc; i++)
-		if (pool->dscs[i] == *dsc) {
-			res_ctx->is_dsc_acquired[i] = false;
-			*dsc = NULL;
-			break;
-		}
-}
-
-static int dml_get_num_mpc_splits(struct pipe_ctx *pipe)
-{
-	int mpc_split_count = 0;
-	struct pipe_ctx *other_pipe = pipe->bottom_pipe;
-
-	while (other_pipe && other_pipe->plane_state == pipe->plane_state) {
-		mpc_split_count++;
-		other_pipe = other_pipe->bottom_pipe;
-	}
-	other_pipe = pipe->top_pipe;
-	while (other_pipe && other_pipe->plane_state == pipe->plane_state) {
-		mpc_split_count++;
-		other_pipe = other_pipe->top_pipe;
-	}
-
-	return mpc_split_count;
-}
-
-static bool dml_enough_pipes_for_subvp(struct dc *dc,
-		struct dc_state *context)
-{
-	int i = 0;
-	int num_pipes = 0;
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		if (pipe->stream && pipe->plane_state)
-			num_pipes++;
-	}
-
-	// Sub-VP only possible if the number of "real" pipes is
-	// less than or equal to half the number of available pipes
-	if (num_pipes * 2 > dc->res_pool->pipe_count)
-		return false;
-
-	return true;
-}
-
-static int dml_validate_apply_pipe_split_flags(
-		struct dc *dc,
-		struct dc_state *context,
-		int vlevel,
-		int *split,
-		bool *merge)
-{
-	int i, pipe_idx, vlevel_split;
-	int plane_count = 0;
-	bool force_split = false;
-	bool avoid_split = dc->debug.pipe_split_policy == MPC_SPLIT_AVOID;
-	struct vba_vars_st *v = &context->bw_ctx.dml.vba;
-	int max_mpc_comb = v->maxMpcComb;
-
-	if (context->stream_count > 1) {
-		if (dc->debug.pipe_split_policy == MPC_SPLIT_AVOID_MULT_DISP)
-			avoid_split = true;
-	} else if (dc->debug.force_single_disp_pipe_split)
-			force_split = true;
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		/**
-		 * Workaround for avoiding pipe-split in cases where we'd split
-		 * planes that are too small, resulting in splits that aren't
-		 * valid for the scaler.
-		 */
-		if (pipe->plane_state &&
-		    (pipe->plane_state->dst_rect.width <= 16 ||
-		     pipe->plane_state->dst_rect.height <= 16 ||
-		     pipe->plane_state->src_rect.width <= 16 ||
-		     pipe->plane_state->src_rect.height <= 16))
-			avoid_split = true;
-
-		/* TODO: fix dc bugs and remove this split threshold thing */
-		if (pipe->stream && !pipe->prev_odm_pipe &&
-				(!pipe->top_pipe || pipe->top_pipe->plane_state != pipe->plane_state))
-			++plane_count;
-	}
-	if (plane_count > dc->res_pool->pipe_count / 2)
-		avoid_split = true;
-
-	/* W/A: Mode timing with borders may not work well with pipe split, avoid for this corner case */
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct dc_crtc_timing timing;
-
-		if (!pipe->stream)
-			continue;
-		else {
-			timing = pipe->stream->timing;
-			if (timing.h_border_left + timing.h_border_right
-					+ timing.v_border_top + timing.v_border_bottom > 0) {
-				avoid_split = true;
-				break;
-			}
-		}
-	}
-
-	/* Avoid split loop looks for lowest voltage level that allows most unsplit pipes possible */
-	if (avoid_split) {
-		for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-			if (!context->res_ctx.pipe_ctx[i].stream)
-				continue;
-
-			for (vlevel_split = vlevel; vlevel <= context->bw_ctx.dml.soc.num_states; vlevel++)
-				if (v->NoOfDPP[vlevel][0][pipe_idx] == 1 &&
-						v->ModeSupport[vlevel][0])
-					break;
-			/* Impossible to not split this pipe */
-			if (vlevel > context->bw_ctx.dml.soc.num_states)
-				vlevel = vlevel_split;
-			else
-				max_mpc_comb = 0;
-			pipe_idx++;
-		}
-		v->maxMpcComb = max_mpc_comb;
-	}
-
-	/* Split loop sets which pipe should be split based on dml outputs and dc flags */
-	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		int pipe_plane = v->pipe_plane[pipe_idx];
-		bool split4mpc = context->stream_count == 1 && plane_count == 1
-				&& dc->config.enable_4to1MPC && dc->res_pool->pipe_count >= 4;
-
-		if (!context->res_ctx.pipe_ctx[i].stream)
-			continue;
-
-		if (split4mpc || v->NoOfDPP[vlevel][max_mpc_comb][pipe_plane] == 4)
-			split[i] = 4;
-		else if (force_split || v->NoOfDPP[vlevel][max_mpc_comb][pipe_plane] == 2)
-				split[i] = 2;
-
-		if ((pipe->stream->view_format ==
-				VIEW_3D_FORMAT_SIDE_BY_SIDE ||
-				pipe->stream->view_format ==
-				VIEW_3D_FORMAT_TOP_AND_BOTTOM) &&
-				(pipe->stream->timing.timing_3d_format ==
-				TIMING_3D_FORMAT_TOP_AND_BOTTOM ||
-				 pipe->stream->timing.timing_3d_format ==
-				TIMING_3D_FORMAT_SIDE_BY_SIDE))
-			split[i] = 2;
-		if (dc->debug.force_odm_combine & (1 << pipe->stream_res.tg->inst)) {
-			split[i] = 2;
-			v->ODMCombineEnablePerState[vlevel][pipe_plane] = dm_odm_combine_mode_2to1;
-		}
-		if (dc->debug.force_odm_combine_4to1 & (1 << pipe->stream_res.tg->inst)) {
-			split[i] = 4;
-			v->ODMCombineEnablePerState[vlevel][pipe_plane] = dm_odm_combine_mode_4to1;
-		}
-		/*420 format workaround*/
-		if (pipe->stream->timing.h_addressable > 7680 &&
-				pipe->stream->timing.pixel_encoding == PIXEL_ENCODING_YCBCR420) {
-			split[i] = 4;
-		}
-
-		v->ODMCombineEnabled[pipe_plane] =
-			v->ODMCombineEnablePerState[vlevel][pipe_plane];
-
-		if (v->ODMCombineEnabled[pipe_plane] == dm_odm_combine_mode_disabled) {
-			if (dml_get_num_mpc_splits(pipe) == 1) {
-				/*If need split for mpc but 2 way split already*/
-				if (split[i] == 4)
-					split[i] = 2; /* 2 -> 4 MPC */
-				else if (split[i] == 2)
-					split[i] = 0; /* 2 -> 2 MPC */
-				else if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state)
-					merge[i] = true; /* 2 -> 1 MPC */
-			} else if (dml_get_num_mpc_splits(pipe) == 3) {
-				/*If need split for mpc but 4 way split already*/
-				if (split[i] == 2 && ((pipe->top_pipe && !pipe->top_pipe->top_pipe)
-						|| !pipe->bottom_pipe)) {
-					merge[i] = true; /* 4 -> 2 MPC */
-				} else if (split[i] == 0 && pipe->top_pipe &&
-						pipe->top_pipe->plane_state == pipe->plane_state)
-					merge[i] = true; /* 4 -> 1 MPC */
-				split[i] = 0;
-			} else if (dml_get_num_mpc_splits(pipe)) {
-				/* ODM -> MPC transition */
-				if (pipe->prev_odm_pipe) {
-					split[i] = 0;
-					merge[i] = true;
-				}
-			}
-		} else {
-			if (dml_get_num_mpc_splits(pipe) == 1) {
-				/*If need split for odm but 2 way split already*/
-				if (split[i] == 4)
-					split[i] = 2; /* 2 -> 4 ODM */
-				else if (split[i] == 2)
-					split[i] = 0; /* 2 -> 2 ODM */
-				else if (pipe->prev_odm_pipe) {
-					ASSERT(0); /* NOT expected yet */
-					merge[i] = true; /* exit ODM */
-				}
-			} else if (dml_get_num_mpc_splits(pipe) == 3) {
-				/*If need split for odm but 4 way split already*/
-				if (split[i] == 2 && ((pipe->prev_odm_pipe && !pipe->prev_odm_pipe->prev_odm_pipe)
-						|| !pipe->next_odm_pipe)) {
-					ASSERT(0); /* NOT expected yet */
-					merge[i] = true; /* 4 -> 2 ODM */
-				} else if (split[i] == 0 && pipe->prev_odm_pipe) {
-					ASSERT(0); /* NOT expected yet */
-					merge[i] = true; /* exit ODM */
-				}
-				split[i] = 0;
-			} else if (dml_get_num_mpc_splits(pipe)) {
-				/* MPC -> ODM transition */
-				ASSERT(0); /* NOT expected yet */
-				if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state) {
-					split[i] = 0;
-					merge[i] = true;
-				}
-			}
-		}
-
-		/* Adjust dppclk when split is forced, do not bother with dispclk */
-		if (split[i] != 0 && v->NoOfDPP[vlevel][max_mpc_comb][pipe_idx] == 1)
-			v->RequiredDPPCLK[vlevel][max_mpc_comb][pipe_idx] /= 2;
-		pipe_idx++;
-	}
-
-	return vlevel;
-}
-
-static void dml_set_phantom_stream_timing(struct dc *dc,
-		struct dc_state *context,
-		struct pipe_ctx *ref_pipe,
-		struct dc_stream_state *phantom_stream)
-{
-	// phantom_vactive = blackout (latency + margin) + fw_processing_delays + pstate allow width
-	uint32_t phantom_vactive_us = context->bw_ctx.dml.soc.dram_clock_change_latency_us + 60 +
-					dc->caps.subvp_fw_processing_delay_us +
-					dc->caps.subvp_pstate_allow_width_us;
-	uint32_t phantom_vactive = ((double)phantom_vactive_us/1000000) *
-					(ref_pipe->stream->timing.pix_clk_100hz * 100) /
-					(double)ref_pipe->stream->timing.h_total;
-	uint32_t phantom_bp = ref_pipe->pipe_dlg_param.vstartup_start;
-
-	phantom_stream->dst.y = 0;
-	phantom_stream->dst.height = phantom_vactive;
-	phantom_stream->src.y = 0;
-	phantom_stream->src.height = phantom_vactive;
-
-	phantom_stream->timing.v_addressable = phantom_vactive;
-	phantom_stream->timing.v_front_porch = 1;
-	phantom_stream->timing.v_total = phantom_stream->timing.v_addressable +
-						phantom_stream->timing.v_front_porch +
-						phantom_stream->timing.v_sync_width +
-						phantom_bp;
-}
-
-static struct dc_stream_state *dml_enable_phantom_stream(struct dc *dc,
-		struct dc_state *context,
-		struct pipe_ctx *ref_pipe)
-{
-	struct dc_stream_state *phantom_stream = NULL;
-
-	phantom_stream = dc_create_stream_for_sink(ref_pipe->stream->sink);
-	phantom_stream->signal = SIGNAL_TYPE_VIRTUAL;
-	phantom_stream->dpms_off = true;
-	phantom_stream->mall_stream_config.type = SUBVP_PHANTOM;
-	phantom_stream->mall_stream_config.paired_stream = ref_pipe->stream;
-	ref_pipe->stream->mall_stream_config.type = SUBVP_MAIN;
-	ref_pipe->stream->mall_stream_config.paired_stream = phantom_stream;
-
-	/* stream has limited viewport and small timing */
-	memcpy(&phantom_stream->timing, &ref_pipe->stream->timing, sizeof(phantom_stream->timing));
-	memcpy(&phantom_stream->src, &ref_pipe->stream->src, sizeof(phantom_stream->src));
-	memcpy(&phantom_stream->dst, &ref_pipe->stream->dst, sizeof(phantom_stream->dst));
-	dml_set_phantom_stream_timing(dc, context, ref_pipe, phantom_stream);
-
-	dc_add_stream_to_ctx(dc, context, phantom_stream);
-	dc->hwss.apply_ctx_to_hw(dc, context);
-	return phantom_stream;
-}
-
-static void dml_enable_phantom_plane(struct dc *dc,
-		struct dc_state *context,
-		struct dc_stream_state *phantom_stream,
-		struct pipe_ctx *main_pipe)
-{
-	struct dc_plane_state *phantom_plane = NULL;
-	struct dc_plane_state *prev_phantom_plane = NULL;
-	struct pipe_ctx *curr_pipe = main_pipe;
-
-	while (curr_pipe) {
-		if (curr_pipe->top_pipe && curr_pipe->top_pipe->plane_state == curr_pipe->plane_state)
-			phantom_plane = prev_phantom_plane;
-		else
-			phantom_plane = dc_create_plane_state(dc);
-
-		memcpy(&phantom_plane->address, &curr_pipe->plane_state->address, sizeof(phantom_plane->address));
-		memcpy(&phantom_plane->scaling_quality, &curr_pipe->plane_state->scaling_quality,
-				sizeof(phantom_plane->scaling_quality));
-		memcpy(&phantom_plane->src_rect, &curr_pipe->plane_state->src_rect, sizeof(phantom_plane->src_rect));
-		memcpy(&phantom_plane->dst_rect, &curr_pipe->plane_state->dst_rect, sizeof(phantom_plane->dst_rect));
-		memcpy(&phantom_plane->clip_rect, &curr_pipe->plane_state->clip_rect, sizeof(phantom_plane->clip_rect));
-		memcpy(&phantom_plane->plane_size, &curr_pipe->plane_state->plane_size,
-				sizeof(phantom_plane->plane_size));
-		memcpy(&phantom_plane->tiling_info, &curr_pipe->plane_state->tiling_info,
-				sizeof(phantom_plane->tiling_info));
-		memcpy(&phantom_plane->dcc, &curr_pipe->plane_state->dcc, sizeof(phantom_plane->dcc));
-		/* Currently compat_level is undefined in dc_state
-		* phantom_plane->compat_level = curr_pipe->plane_state->compat_level;
-		*/
-		phantom_plane->format = curr_pipe->plane_state->format;
-		phantom_plane->rotation = curr_pipe->plane_state->rotation;
-		phantom_plane->visible = curr_pipe->plane_state->visible;
-
-		/* Shadow pipe has small viewport. */
-		phantom_plane->clip_rect.y = 0;
-		phantom_plane->clip_rect.height = phantom_stream->timing.v_addressable;
-
-		dc_add_plane_to_context(dc, phantom_stream, phantom_plane, context);
-
-		curr_pipe = curr_pipe->bottom_pipe;
-		prev_phantom_plane = phantom_plane;
-	}
-}
-
-static void dml_add_phantom_pipes(struct dc *dc, struct dc_state *context)
-{
-	int i = 0;
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct dc_stream_state *ref_stream = pipe->stream;
-		// Only construct phantom stream for top pipes that have plane enabled
-		if (!pipe->top_pipe && pipe->plane_state && pipe->stream &&
-				pipe->stream->mall_stream_config.type == SUBVP_NONE) {
-			struct dc_stream_state *phantom_stream = NULL;
-
-			phantom_stream = dml_enable_phantom_stream(dc, context, pipe);
-			dml_enable_phantom_plane(dc, context, phantom_stream, pipe);
-		}
-	}
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		if (pipe->plane_state && pipe->stream &&
-				pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
-			pipe->stream->use_dynamic_meta = false;
-			pipe->plane_state->flip_immediate = false;
-			if (!resource_build_scaling_params(pipe)) {
-				// Log / remove phantom pipes since failed to build scaling params
-			}
-		}
-	}
-}
-
-static void dml_remove_phantom_pipes(struct dc *dc, struct dc_state *context)
-{
-	int i;
-	bool removed_pipe = false;
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		// build scaling params for phantom pipes
-		if (pipe->plane_state && pipe->stream && pipe->stream->mall_stream_config.type == SUBVP_PHANTOM) {
-			dc_rem_all_planes_for_stream(dc, pipe->stream, context);
-			dc_remove_stream_from_ctx(dc, context, pipe->stream);
-			removed_pipe = true;
-		}
-
-		// Clear all phantom stream info
-		if (pipe->stream) {
-			pipe->stream->mall_stream_config.type = SUBVP_NONE;
-			pipe->stream->mall_stream_config.paired_stream = NULL;
-		}
-	}
-	if (removed_pipe)
-		dc->hwss.apply_ctx_to_hw(dc, context);
-}
-
-/*
- * If the input state contains no upstream planes for a particular pipe (i.e. only timing)
- * we need to populate some "conservative" plane information as DML cannot handle "no planes"
- */
-static void populate_default_plane_from_timing(const struct dc_crtc_timing *timing, struct _vcs_dpi_display_pipe_params_st *pipe)
-{
-	pipe->src.is_hsplit = pipe->dest.odm_combine != dm_odm_combine_mode_disabled;
-	pipe->src.source_scan = dm_horz;
-	pipe->src.sw_mode = dm_sw_4kb_s;
-	pipe->src.macro_tile_size = dm_64k_tile;
-	pipe->src.viewport_width = timing->h_addressable;
-	if (pipe->src.viewport_width > 1920)
-		pipe->src.viewport_width = 1920;
-	pipe->src.viewport_height = timing->v_addressable;
-	if (pipe->src.viewport_height > 1080)
-		pipe->src.viewport_height = 1080;
-	pipe->src.surface_height_y = pipe->src.viewport_height;
-	pipe->src.surface_width_y = pipe->src.viewport_width;
-	pipe->src.surface_height_c = pipe->src.viewport_height;
-	pipe->src.surface_width_c = pipe->src.viewport_width;
-	pipe->src.data_pitch = ((pipe->src.viewport_width + 255) / 256) * 256;
-	pipe->src.source_format = dm_444_32;
-	pipe->dest.recout_width = pipe->src.viewport_width;
-	pipe->dest.recout_height = pipe->src.viewport_height;
-	pipe->dest.full_recout_width = pipe->dest.recout_width;
-	pipe->dest.full_recout_height = pipe->dest.recout_height;
-	pipe->scale_ratio_depth.lb_depth = dm_lb_16;
-	pipe->scale_ratio_depth.hscl_ratio = 1.0;
-	pipe->scale_ratio_depth.vscl_ratio = 1.0;
-	pipe->scale_ratio_depth.scl_enable = 0;
-	pipe->scale_taps.htaps = 1;
-	pipe->scale_taps.vtaps = 1;
-	pipe->dest.vtotal_min = timing->v_total;
-	pipe->dest.vtotal_max = timing->v_total;
-
-	if (pipe->dest.odm_combine == dm_odm_combine_mode_2to1) {
-		pipe->src.viewport_width /= 2;
-		pipe->dest.recout_width /= 2;
-	} else if (pipe->dest.odm_combine == dm_odm_combine_mode_4to1) {
-		pipe->src.viewport_width /= 4;
-		pipe->dest.recout_width /= 4;
-	}
-
-	pipe->src.dcc = false;
-	pipe->src.dcc_rate = 1;
-}
-
-/*
- * If the pipe is not blending (i.e. pipe_ctx->top pipe == null) then its
- * hsplit group is equal to its own pipe ID
- * Otherwise, all pipes part of the same blending tree have the same hsplit group
- * ID as the top most pipe
- *
- * If the pipe ctx is ODM combined, then similar logic follows
- */
-static void populate_hsplit_group_from_dc_pipe_ctx (const struct pipe_ctx *dc_pipe_ctx, struct _vcs_dpi_display_e2e_pipe_params_st *e2e_pipe)
-{
-	e2e_pipe->pipe.src.hsplit_grp = dc_pipe_ctx->pipe_idx;
-
-	if (dc_pipe_ctx->top_pipe && dc_pipe_ctx->top_pipe->plane_state
-			== dc_pipe_ctx->plane_state) {
-		struct pipe_ctx *first_pipe = dc_pipe_ctx->top_pipe;
-		int split_idx = 0;
-
-		while (first_pipe->top_pipe && first_pipe->top_pipe->plane_state
-				== dc_pipe_ctx->plane_state) {
-			first_pipe = first_pipe->top_pipe;
-			split_idx++;
-		}
-
-		/* Treat 4to1 mpc combine as an mpo of 2 2-to-1 combines */
-		if (split_idx == 0)
-			e2e_pipe->pipe.src.hsplit_grp = first_pipe->pipe_idx;
-		else if (split_idx == 1)
-			e2e_pipe->pipe.src.hsplit_grp = dc_pipe_ctx->pipe_idx;
-		else if (split_idx == 2)
-			e2e_pipe->pipe.src.hsplit_grp = dc_pipe_ctx->top_pipe->pipe_idx;
-
-	} else if (dc_pipe_ctx->prev_odm_pipe) {
-		struct pipe_ctx *first_pipe = dc_pipe_ctx->prev_odm_pipe;
-
-		while (first_pipe->prev_odm_pipe)
-			first_pipe = first_pipe->prev_odm_pipe;
-		e2e_pipe->pipe.src.hsplit_grp = first_pipe->pipe_idx;
-	}
-}
-
-static void populate_dml_from_dc_pipe_ctx (const struct pipe_ctx *dc_pipe_ctx, struct _vcs_dpi_display_e2e_pipe_params_st *e2e_pipe, int always_scale)
-{
-	const struct dc_plane_state *pln = dc_pipe_ctx->plane_state;
-	const struct scaler_data *scl = &dc_pipe_ctx->plane_res.scl_data;
-
-	e2e_pipe->pipe.src.immediate_flip = pln->flip_immediate;
-	e2e_pipe->pipe.src.is_hsplit = (dc_pipe_ctx->bottom_pipe && dc_pipe_ctx->bottom_pipe->plane_state == pln)
-			|| (dc_pipe_ctx->top_pipe && dc_pipe_ctx->top_pipe->plane_state == pln)
-			|| e2e_pipe->pipe.dest.odm_combine != dm_odm_combine_mode_disabled;
-
-	/* stereo is not split */
-	if (pln->stereo_format == PLANE_STEREO_FORMAT_SIDE_BY_SIDE ||
-		pln->stereo_format == PLANE_STEREO_FORMAT_TOP_AND_BOTTOM) {
-		e2e_pipe->pipe.src.is_hsplit = false;
-		e2e_pipe->pipe.src.hsplit_grp = dc_pipe_ctx->pipe_idx;
-	}
-
-	e2e_pipe->pipe.src.source_scan = pln->rotation == ROTATION_ANGLE_90
-			|| pln->rotation == ROTATION_ANGLE_270 ? dm_vert : dm_horz;
-	e2e_pipe->pipe.src.viewport_y_y = scl->viewport.y;
-	e2e_pipe->pipe.src.viewport_y_c = scl->viewport_c.y;
-	e2e_pipe->pipe.src.viewport_width = scl->viewport.width;
-	e2e_pipe->pipe.src.viewport_width_c = scl->viewport_c.width;
-	e2e_pipe->pipe.src.viewport_height = scl->viewport.height;
-	e2e_pipe->pipe.src.viewport_height_c = scl->viewport_c.height;
-	e2e_pipe->pipe.src.viewport_width_max = pln->src_rect.width;
-	e2e_pipe->pipe.src.viewport_height_max = pln->src_rect.height;
-	e2e_pipe->pipe.src.surface_width_y = pln->plane_size.surface_size.width;
-	e2e_pipe->pipe.src.surface_height_y = pln->plane_size.surface_size.height;
-	e2e_pipe->pipe.src.surface_width_c = pln->plane_size.chroma_size.width;
-	e2e_pipe->pipe.src.surface_height_c = pln->plane_size.chroma_size.height;
-
-	if (pln->format == SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA
-			|| pln->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN) {
-		e2e_pipe->pipe.src.data_pitch = pln->plane_size.surface_pitch;
-		e2e_pipe->pipe.src.data_pitch_c = pln->plane_size.chroma_pitch;
-		e2e_pipe->pipe.src.meta_pitch = pln->dcc.meta_pitch;
-		e2e_pipe->pipe.src.meta_pitch_c = pln->dcc.meta_pitch_c;
-	} else {
-		e2e_pipe->pipe.src.data_pitch = pln->plane_size.surface_pitch;
-		e2e_pipe->pipe.src.meta_pitch = pln->dcc.meta_pitch;
-	}
-	e2e_pipe->pipe.src.dcc = pln->dcc.enable;
-	e2e_pipe->pipe.src.dcc_rate = 1;
-	e2e_pipe->pipe.dest.recout_width = scl->recout.width;
-	e2e_pipe->pipe.dest.recout_height = scl->recout.height;
-	e2e_pipe->pipe.dest.full_recout_height = scl->recout.height;
-	e2e_pipe->pipe.dest.full_recout_width = scl->recout.width;
-	if (e2e_pipe->pipe.dest.odm_combine == dm_odm_combine_mode_2to1)
-		e2e_pipe->pipe.dest.full_recout_width *= 2;
-	else if (e2e_pipe->pipe.dest.odm_combine == dm_odm_combine_mode_4to1)
-		e2e_pipe->pipe.dest.full_recout_width *= 4;
-	else {
-		struct pipe_ctx *split_pipe = dc_pipe_ctx->bottom_pipe;
-
-		while (split_pipe && split_pipe->plane_state == pln) {
-			e2e_pipe->pipe.dest.full_recout_width += split_pipe->plane_res.scl_data.recout.width;
-			split_pipe = split_pipe->bottom_pipe;
-		}
-		split_pipe = dc_pipe_ctx->top_pipe;
-		while (split_pipe && split_pipe->plane_state == pln) {
-			e2e_pipe->pipe.dest.full_recout_width += split_pipe->plane_res.scl_data.recout.width;
-			split_pipe = split_pipe->top_pipe;
-		}
-	}
-
-	e2e_pipe->pipe.scale_ratio_depth.lb_depth = dm_lb_16;
-	e2e_pipe->pipe.scale_ratio_depth.hscl_ratio = (double) scl->ratios.horz.value / (1ULL<<32);
-	e2e_pipe->pipe.scale_ratio_depth.hscl_ratio_c = (double) scl->ratios.horz_c.value / (1ULL<<32);
-	e2e_pipe->pipe.scale_ratio_depth.vscl_ratio = (double) scl->ratios.vert.value / (1ULL<<32);
-	e2e_pipe->pipe.scale_ratio_depth.vscl_ratio_c = (double) scl->ratios.vert_c.value / (1ULL<<32);
-	e2e_pipe->pipe.scale_ratio_depth.scl_enable =
-			scl->ratios.vert.value != dc_fixpt_one.value
-			|| scl->ratios.horz.value != dc_fixpt_one.value
-			|| scl->ratios.vert_c.value != dc_fixpt_one.value
-			|| scl->ratios.horz_c.value != dc_fixpt_one.value /*Lb only or Full scl*/
-			|| always_scale; /*support always scale*/
-	e2e_pipe->pipe.scale_taps.htaps = scl->taps.h_taps;
-	e2e_pipe->pipe.scale_taps.htaps_c = scl->taps.h_taps_c;
-	e2e_pipe->pipe.scale_taps.vtaps = scl->taps.v_taps;
-	e2e_pipe->pipe.scale_taps.vtaps_c = scl->taps.v_taps_c;
-
-	/* Currently compat_level is not defined. Commenting it until further resolution
-	 * if (pln->compat_level == DC_LEGACY_TILING_ADDR_GEN_TWO) {
-		swizzle_to_dml_params(pln->tiling_info.gfx9.swizzle,
-				&e2e_pipe->pipe.src.sw_mode);
-		e2e_pipe->pipe.src.macro_tile_size =
-				swizzle_mode_to_macro_tile_size(pln->tiling_info.gfx9.swizzle);
-	} else {
-		gfx10array_mode_to_dml_params(pln->tiling_info.gfx10compatible.array_mode,
-				pln->compat_level,
-				&e2e_pipe->pipe.src.sw_mode);
-		e2e_pipe->pipe.src.macro_tile_size = dm_4k_tile;
-	}*/
-
-	e2e_pipe->pipe.src.source_format = dc_source_format_to_dml_source_format(pln->format);
-}
-
-static void populate_dml_cursor_parameters_from_dc_pipe_ctx (const struct pipe_ctx *dc_pipe_ctx, struct _vcs_dpi_display_e2e_pipe_params_st *e2e_pipe)
-{
-	/*
-	* For graphic plane, cursor number is 1, nv12 is 0
-	* bw calculations due to cursor on/off
-	*/
-	if (dc_pipe_ctx->plane_state &&
-			(dc_pipe_ctx->plane_state->address.type == PLN_ADDR_TYPE_VIDEO_PROGRESSIVE ||
-			dc_pipe_ctx->stream->mall_stream_config.type == SUBVP_PHANTOM))
-		e2e_pipe->pipe.src.num_cursors = 0;
-	else
-		e2e_pipe->pipe.src.num_cursors = 1;
-
-	e2e_pipe->pipe.src.cur0_src_width = 256;
-	e2e_pipe->pipe.src.cur0_bpp = dm_cur_32bit;
-}
-
-static int populate_dml_pipes_from_context_base(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		bool fast_validate)
-{
-	int pipe_cnt, i;
-	bool synchronized_vblank = true;
-	struct resource_context *res_ctx = &context->res_ctx;
-
-	for (i = 0, pipe_cnt = -1; i < dc->res_pool->pipe_count; i++) {
-		if (!res_ctx->pipe_ctx[i].stream)
-			continue;
-
-		if (pipe_cnt < 0) {
-			pipe_cnt = i;
-			continue;
-		}
-
-		if (res_ctx->pipe_ctx[pipe_cnt].stream == res_ctx->pipe_ctx[i].stream)
-			continue;
-
-		if (dc->debug.disable_timing_sync ||
-			(!resource_are_streams_timing_synchronizable(
-				res_ctx->pipe_ctx[pipe_cnt].stream,
-				res_ctx->pipe_ctx[i].stream) &&
-			!resource_are_vblanks_synchronizable(
-				res_ctx->pipe_ctx[pipe_cnt].stream,
-				res_ctx->pipe_ctx[i].stream))) {
-			synchronized_vblank = false;
-			break;
-		}
-	}
-
-	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
-		struct dc_crtc_timing *timing = &res_ctx->pipe_ctx[i].stream->timing;
-
-		struct audio_check aud_check = {0};
-		if (!res_ctx->pipe_ctx[i].stream)
-			continue;
-
-		/* todo:
-		pipes[pipe_cnt].pipe.src.dynamic_metadata_enable = 0;
-		pipes[pipe_cnt].pipe.src.dcc = 0;
-		pipes[pipe_cnt].pipe.src.vm = 0;*/
-
-		pipes[pipe_cnt].clks_cfg.refclk_mhz = dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000.0;
-
-		pipes[pipe_cnt].dout.dsc_enable = res_ctx->pipe_ctx[i].stream->timing.flags.DSC;
-		/* todo: rotation?*/
-		pipes[pipe_cnt].dout.dsc_slices = res_ctx->pipe_ctx[i].stream->timing.dsc_cfg.num_slices_h;
-		if (res_ctx->pipe_ctx[i].stream->use_dynamic_meta) {
-			pipes[pipe_cnt].pipe.src.dynamic_metadata_enable = true;
-			/* 1/2 vblank */
-			pipes[pipe_cnt].pipe.src.dynamic_metadata_lines_before_active =
-				(timing->v_total - timing->v_addressable
-					- timing->v_border_top - timing->v_border_bottom) / 2;
-			/* 36 bytes dp, 32 hdmi */
-			pipes[pipe_cnt].pipe.src.dynamic_metadata_xmit_bytes =
-				dc_is_dp_signal(res_ctx->pipe_ctx[i].stream->signal) ? 36 : 32;
-		}
-		pipes[pipe_cnt].pipe.dest.synchronized_vblank_all_planes = synchronized_vblank;
-
-		dc_timing_to_dml_timing(timing, &pipes[pipe_cnt].pipe.dest);
-		pipes[pipe_cnt].pipe.dest.vtotal_min = res_ctx->pipe_ctx[i].stream->adjust.v_total_min;
-		pipes[pipe_cnt].pipe.dest.vtotal_max = res_ctx->pipe_ctx[i].stream->adjust.v_total_max;
-
-		pipes[pipe_cnt].pipe.dest.otg_inst = res_ctx->pipe_ctx[i].stream_res.tg->inst;
-
-		pipes[pipe_cnt].pipe.dest.odm_combine = get_dml_odm_combine(&res_ctx->pipe_ctx[i]);
-
-		populate_hsplit_group_from_dc_pipe_ctx(&res_ctx->pipe_ctx[i], &pipes[pipe_cnt]);
-
-		pipes[pipe_cnt].dout.dp_lanes = 4;
-		pipes[pipe_cnt].dout.is_virtual = 0;
-		pipes[pipe_cnt].dout.output_type = get_dml_output_type(res_ctx->pipe_ctx[i].stream->signal);
-		if (pipes[pipe_cnt].dout.output_type < 0) {
-			pipes[pipe_cnt].dout.output_type = dm_dp;
-			pipes[pipe_cnt].dout.is_virtual = 1;
-		}
-
-		populate_color_depth_and_encoding_from_timing(&res_ctx->pipe_ctx[i].stream->timing, &pipes[pipe_cnt].dout);
-
-		if (res_ctx->pipe_ctx[i].stream->timing.flags.DSC)
-			pipes[pipe_cnt].dout.output_bpp = res_ctx->pipe_ctx[i].stream->timing.dsc_cfg.bits_per_pixel / 16.0;
-
-		/* todo: default max for now, until there is logic reflecting this in dc*/
-		pipes[pipe_cnt].dout.dsc_input_bpc = 12;
-		/*fill up the audio sample rate (unit in kHz)*/
-		get_audio_check(&res_ctx->pipe_ctx[i].stream->audio_info, &aud_check);
-		pipes[pipe_cnt].dout.max_audio_sample_rate = aud_check.max_audiosample_rate / 1000;
-
-		populate_dml_cursor_parameters_from_dc_pipe_ctx(&res_ctx->pipe_ctx[i], &pipes[pipe_cnt]);
-
-		if (!res_ctx->pipe_ctx[i].plane_state) {
-			populate_default_plane_from_timing(timing, &pipes[pipe_cnt].pipe);
-		} else {
-			populate_dml_from_dc_pipe_ctx(&res_ctx->pipe_ctx[i], &pipes[pipe_cnt], dc->debug.always_scale);
-		}
-
-		pipe_cnt++;
-	}
-
-	/* populate writeback information */
-	if (dc->res_pool)
-		dc->res_pool->funcs->populate_dml_writeback_from_context(dc, res_ctx, pipes);
-
-	return pipe_cnt;
-}
-
-static int dml_populate_dml_pipes_from_context(
-	struct dc *dc, struct dc_state *context,
-	display_e2e_pipe_params_st *pipes,
-	bool fast_validate)
-{
-	int i, pipe_cnt;
-	struct resource_context *res_ctx = &context->res_ctx;
-	struct pipe_ctx *pipe = NULL;	// Fix potentially uninitialized error from VS
-
-	populate_dml_pipes_from_context_base(dc, context, pipes, fast_validate);
-
-	for (i = 0, pipe_cnt = 0; i < dc->res_pool->pipe_count; i++) {
-		struct dc_crtc_timing *timing;
-
-		if (!res_ctx->pipe_ctx[i].stream)
-			continue;
-		pipe = &res_ctx->pipe_ctx[i];
-		timing = &pipe->stream->timing;
-
-		pipes[pipe_cnt].pipe.src.gpuvm = true;
-		pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_luma = 0;
-		pipes[pipe_cnt].pipe.src.dcc_fraction_of_zs_req_chroma = 0;
-		pipes[pipe_cnt].pipe.dest.vfront_porch = timing->v_front_porch;
-
-		pipes[pipe_cnt].dout.dsc_input_bpc = 0;
-		if (pipes[pipe_cnt].dout.dsc_enable) {
-			switch (timing->display_color_depth) {
-			case COLOR_DEPTH_888:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 8;
-				break;
-			case COLOR_DEPTH_101010:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 10;
-				break;
-			case COLOR_DEPTH_121212:
-				pipes[pipe_cnt].dout.dsc_input_bpc = 12;
-				break;
-			default:
-				ASSERT(0);
-				break;
-			}
-		}
-		pipe_cnt++;
-	}
-	dc->config.enable_4to1MPC = false;
-	if (pipe_cnt == 1 && pipe->plane_state && !dc->debug.disable_z9_mpc) {
-		if (is_dual_plane(pipe->plane_state->format)
-				&& pipe->plane_state->src_rect.width <= 1920 && pipe->plane_state->src_rect.height <= 1080) {
-			dc->config.enable_4to1MPC = true;
-		} else if (!is_dual_plane(pipe->plane_state->format)) {
-			context->bw_ctx.dml.ip.det_buffer_size_kbytes = 192;
-			pipes[0].pipe.src.unbounded_req_mode = true;
-		}
-	}
-
-	return pipe_cnt;
-}
-
-static void dml_full_validate_bw_helper(struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int *vlevel,
-		int *split,
-		bool *merge,
-		int *pipe_cnt)
-{
-	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
-
-	/*
-	 * DML favors voltage over p-state, but we're more interested in
-	 * supporting p-state over voltage. We can't support p-state in
-	 * prefetch mode > 0 so try capping the prefetch mode to start.
-	 */
-	context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
-		dm_allow_self_refresh_and_mclk_switch;
-	*vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, *pipe_cnt);
-	/* This may adjust vlevel and maxMpcComb */
-	if (*vlevel < context->bw_ctx.dml.soc.num_states)
-		*vlevel = dml_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge);
-
-	/* Conditions for setting up phantom pipes for SubVP:
-	 * 1. Not force disable SubVP
-	 * 2. Full update (i.e. !fast_validate)
-	 * 3. Enough pipes are available to support SubVP (TODO: Which pipes will use VACTIVE / VBLANK / SUBVP?)
-	 * 4. Display configuration passes validation
-	 * 5. (Config doesn't support MCLK in VACTIVE/VBLANK || dc->debug.force_subvp_mclk_switch)
-	 */
-	if (!dc->debug.force_disable_subvp &&
-			dml_enough_pipes_for_subvp(dc, context) &&
-			*vlevel < context->bw_ctx.dml.soc.num_states &&
-			(vba->DRAMClockChangeSupport[*vlevel][vba->maxMpcComb] == dm_dram_clock_change_unsupported ||
-			dc->debug.force_subvp_mclk_switch)) {
-
-		dml_add_phantom_pipes(dc, context);
-
-		 /* Create input to DML based on new context which includes phantom pipes
-		  * TODO: Input to DML should mark which pipes are phantom
-		  */
-		*pipe_cnt = dml_populate_dml_pipes_from_context(dc, context, pipes, false);
-		*vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, *pipe_cnt);
-		if (*vlevel < context->bw_ctx.dml.soc.num_states) {
-			memset(split, 0, MAX_PIPES * sizeof(*split));
-			memset(merge, 0, MAX_PIPES * sizeof(*merge));
-			*vlevel = dml_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge);
-		}
-
-		// If SubVP pipe config is unsupported (or cannot be used for UCLK switching)
-		// remove phantom pipes and repopulate dml pipes
-		if (*vlevel == context->bw_ctx.dml.soc.num_states ||
-				vba->DRAMClockChangeSupport[*vlevel][vba->maxMpcComb] == dm_dram_clock_change_unsupported) {
-			dml_remove_phantom_pipes(dc, context);
-			*pipe_cnt = dml_populate_dml_pipes_from_context(dc, context, pipes, false);
-		}
-	}
-}
-
-static void dcn20_adjust_adaptive_sync_v_startup(
-		const struct dc_crtc_timing *dc_crtc_timing, int *vstartup_start)
-{
-	struct dc_crtc_timing patched_crtc_timing;
-	uint32_t asic_blank_end   = 0;
-	uint32_t asic_blank_start = 0;
-	uint32_t newVstartup	  = 0;
-
-	patched_crtc_timing = *dc_crtc_timing;
-
-	if (patched_crtc_timing.flags.INTERLACE == 1) {
-		if (patched_crtc_timing.v_front_porch < 2)
-			patched_crtc_timing.v_front_porch = 2;
-	} else {
-		if (patched_crtc_timing.v_front_porch < 1)
-			patched_crtc_timing.v_front_porch = 1;
-	}
-
-	/* blank_start = frame end - front porch */
-	asic_blank_start = patched_crtc_timing.v_total -
-					patched_crtc_timing.v_front_porch;
-
-	/* blank_end = blank_start - active */
-	asic_blank_end = asic_blank_start -
-					patched_crtc_timing.v_border_bottom -
-					patched_crtc_timing.v_addressable -
-					patched_crtc_timing.v_border_top;
-
-	newVstartup = asic_blank_end + (patched_crtc_timing.v_total - asic_blank_start);
-
-	*vstartup_start = ((newVstartup > *vstartup_start) ? newVstartup : *vstartup_start);
-}
-
-static bool is_dp_128b_132b_signal(struct pipe_ctx *pipe_ctx)
-{
-	return (pipe_ctx->stream_res.hpo_dp_stream_enc &&
-			pipe_ctx->link_res.hpo_dp_link_enc &&
-			dc_is_dp_signal(pipe_ctx->stream->signal));
-}
-
-static bool is_dtbclk_required(struct dc *dc, struct dc_state *context)
-{
-	int i;
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		if (!context->res_ctx.pipe_ctx[i].stream)
-			continue;
-		if (is_dp_128b_132b_signal(&context->res_ctx.pipe_ctx[i]))
-			return true;
-	}
-	return false;
-}
-
-static void dml_update_soc_for_wm_a(struct dc *dc, struct dc_state *context)
-{
-	if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].valid) {
-		context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.pstate_latency_us;
-		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.sr_enter_plus_exit_time_us;
-		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.sr_exit_time_us;
-		context->bw_ctx.dml.soc.fclk_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.fclk_change_latency_us;
-	}
-}
-
-static bool dml_internal_validate(
-		struct dc *dc,
-		struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int *pipe_cnt_out,
-		int *vlevel_out,
-		bool fast_validate)
-{
-	bool out = false;
-	bool repopulate_pipes = false;
-	int split[MAX_PIPES] = { 0 };
-	bool merge[MAX_PIPES] = { false };
-	bool newly_split[MAX_PIPES] = { false };
-	int pipe_cnt, i, pipe_idx, vlevel;
-	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
-
-	ASSERT(pipes);
-	if (!pipes)
-		return false;
-
-	// For each full update, remove all existing phantom pipes first
-	dml_remove_phantom_pipes(dc, context);
-
-	dml_update_soc_for_wm_a(dc, context);
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		if (pipe->plane_state) {
-			// On initial pass through DML, we intend to use MALL for SS on all
-			// (non-PSR) surfaces with none using MALL for P-State
-			// 'mall_plane_config': is not a member of 'dc_plane_state' - commenting it out till mall_plane_config gets supported in dc_plant_state
-			//if (pipe->stream && pipe->stream->link->psr_settings.psr_version == DC_PSR_VERSION_UNSUPPORTED)
-			//	pipe->plane_state->mall_plane_config.use_mall_for_ss = true;
-		}
-	}
-	pipe_cnt = dml_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
-
-	if (!pipe_cnt) {
-		out = true;
-		goto validate_out;
-	}
-
-	dml_log_pipe_params(&context->bw_ctx.dml, pipes, pipe_cnt);
-
-	if (!fast_validate) {
-		dml_full_validate_bw_helper(dc, context, pipes, &vlevel, split, merge, &pipe_cnt);
-	}
-
-	if (fast_validate || vlevel == context->bw_ctx.dml.soc.num_states ||
-			vba->DRAMClockChangeSupport[vlevel][vba->maxMpcComb] == dm_dram_clock_change_unsupported) {
-		/*
-		 * If mode is unsupported or there's still no p-state support then
-		 * fall back to favoring voltage.
-		 *
-		 * We don't actually support prefetch mode 2, so require that we
-		 * at least support prefetch mode 1.
-		 */
-		context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
-			dm_allow_self_refresh;
-
-		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
-		if (vlevel < context->bw_ctx.dml.soc.num_states) {
-			memset(split, 0, sizeof(split));
-			memset(merge, 0, sizeof(merge));
-			vlevel = dml_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
-		}
-	}
-
-	dml_log_mode_support_params(&context->bw_ctx.dml);
-
-	if (vlevel == context->bw_ctx.dml.soc.num_states)
-		goto validate_fail;
-
-	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *mpo_pipe = pipe->bottom_pipe;
-
-		if (!pipe->stream)
-			continue;
-
-		/* We only support full screen mpo with ODM */
-		if (vba->ODMCombineEnabled[vba->pipe_plane[pipe_idx]] != dm_odm_combine_mode_disabled
-				&& pipe->plane_state && mpo_pipe
-				&& memcmp(&mpo_pipe->plane_res.scl_data.recout,
-						&pipe->plane_res.scl_data.recout,
-						sizeof(struct rect)) != 0) {
-			ASSERT(mpo_pipe->plane_state != pipe->plane_state);
-			goto validate_fail;
-		}
-		pipe_idx++;
-	}
-
-	/* merge pipes if necessary */
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		/*skip pipes that don't need merging*/
-		if (!merge[i])
-			continue;
-
-		/* if ODM merge we ignore mpc tree, mpo pipes will have their own flags */
-		if (pipe->prev_odm_pipe) {
-			/*split off odm pipe*/
-			pipe->prev_odm_pipe->next_odm_pipe = pipe->next_odm_pipe;
-			if (pipe->next_odm_pipe)
-				pipe->next_odm_pipe->prev_odm_pipe = pipe->prev_odm_pipe;
-
-			pipe->bottom_pipe = NULL;
-			pipe->next_odm_pipe = NULL;
-			pipe->plane_state = NULL;
-			pipe->stream = NULL;
-			pipe->top_pipe = NULL;
-			pipe->prev_odm_pipe = NULL;
-			if (pipe->stream_res.dsc)
-				dml_release_dsc(&context->res_ctx, dc->res_pool, &pipe->stream_res.dsc);
-			memset(&pipe->plane_res, 0, sizeof(pipe->plane_res));
-			memset(&pipe->stream_res, 0, sizeof(pipe->stream_res));
-			repopulate_pipes = true;
-		} else if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state) {
-			struct pipe_ctx *top_pipe = pipe->top_pipe;
-			struct pipe_ctx *bottom_pipe = pipe->bottom_pipe;
-
-			top_pipe->bottom_pipe = bottom_pipe;
-			if (bottom_pipe)
-				bottom_pipe->top_pipe = top_pipe;
-
-			pipe->top_pipe = NULL;
-			pipe->bottom_pipe = NULL;
-			pipe->plane_state = NULL;
-			pipe->stream = NULL;
-			memset(&pipe->plane_res, 0, sizeof(pipe->plane_res));
-			memset(&pipe->stream_res, 0, sizeof(pipe->stream_res));
-			repopulate_pipes = true;
-		} else
-			ASSERT(0); /* Should never try to merge master pipe */
-
-	}
-
-	for (i = 0, pipe_idx = -1; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
-		struct pipe_ctx *hsplit_pipe = NULL;
-		bool odm;
-		int old_index = -1;
-
-		if (!pipe->stream || newly_split[i])
-			continue;
-
-		pipe_idx++;
-		odm = vba->ODMCombineEnabled[vba->pipe_plane[pipe_idx]] != dm_odm_combine_mode_disabled;
-
-		if (!pipe->plane_state && !odm)
-			continue;
-
-		if (split[i]) {
-			if (odm) {
-				if (split[i] == 4 && old_pipe->next_odm_pipe && old_pipe->next_odm_pipe->next_odm_pipe)
-					old_index = old_pipe->next_odm_pipe->next_odm_pipe->pipe_idx;
-				else if (old_pipe->next_odm_pipe)
-					old_index = old_pipe->next_odm_pipe->pipe_idx;
-			} else {
-				if (split[i] == 4 && old_pipe->bottom_pipe && old_pipe->bottom_pipe->bottom_pipe &&
-						old_pipe->bottom_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
-					old_index = old_pipe->bottom_pipe->bottom_pipe->pipe_idx;
-				else if (old_pipe->bottom_pipe &&
-						old_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
-					old_index = old_pipe->bottom_pipe->pipe_idx;
-			}
-			hsplit_pipe = dml_find_split_pipe(dc, context, old_index);
-			ASSERT(hsplit_pipe);
-			if (!hsplit_pipe)
-				goto validate_fail;
-
-			if (!dml_split_stream_for_mpc_or_odm(
-					dc, &context->res_ctx,
-					pipe, hsplit_pipe, odm))
-				goto validate_fail;
-
-			newly_split[hsplit_pipe->pipe_idx] = true;
-			repopulate_pipes = true;
-		}
-		if (split[i] == 4) {
-			struct pipe_ctx *pipe_4to1;
-
-			if (odm && old_pipe->next_odm_pipe)
-				old_index = old_pipe->next_odm_pipe->pipe_idx;
-			else if (!odm && old_pipe->bottom_pipe &&
-						old_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
-				old_index = old_pipe->bottom_pipe->pipe_idx;
-			else
-				old_index = -1;
-			pipe_4to1 = dml_find_split_pipe(dc, context, old_index);
-			ASSERT(pipe_4to1);
-			if (!pipe_4to1)
-				goto validate_fail;
-			if (!dml_split_stream_for_mpc_or_odm(
-					dc, &context->res_ctx,
-					pipe, pipe_4to1, odm))
-				goto validate_fail;
-			newly_split[pipe_4to1->pipe_idx] = true;
-
-			if (odm && old_pipe->next_odm_pipe && old_pipe->next_odm_pipe->next_odm_pipe
-					&& old_pipe->next_odm_pipe->next_odm_pipe->next_odm_pipe)
-				old_index = old_pipe->next_odm_pipe->next_odm_pipe->next_odm_pipe->pipe_idx;
-			else if (!odm && old_pipe->bottom_pipe && old_pipe->bottom_pipe->bottom_pipe &&
-					old_pipe->bottom_pipe->bottom_pipe->bottom_pipe &&
-					old_pipe->bottom_pipe->bottom_pipe->bottom_pipe->plane_state == old_pipe->plane_state)
-				old_index = old_pipe->bottom_pipe->bottom_pipe->bottom_pipe->pipe_idx;
-			else
-				old_index = -1;
-			pipe_4to1 = dml_find_split_pipe(dc, context, old_index);
-			ASSERT(pipe_4to1);
-			if (!pipe_4to1)
-				goto validate_fail;
-			if (!dml_split_stream_for_mpc_or_odm(
-					dc, &context->res_ctx,
-					hsplit_pipe, pipe_4to1, odm))
-				goto validate_fail;
-			newly_split[pipe_4to1->pipe_idx] = true;
-		}
-		if (odm)
-			dml_build_mapped_resource(dc, context, pipe->stream);
-	}
-
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-
-		if (pipe->plane_state) {
-			if (!resource_build_scaling_params(pipe))
-				goto validate_fail;
-		}
-	}
-
-	/* Actual dsc count per stream dsc validation*/
-	if (!dml_validate_dsc(dc, context)) {
-		vba->ValidationStatus[vba->soc.num_states] = DML_FAIL_DSC_VALIDATION_FAILURE;
-		goto validate_fail;
-	}
-
-	if (repopulate_pipes)
-		pipe_cnt = dml_populate_dml_pipes_from_context(dc, context, pipes, fast_validate);
-	*vlevel_out = vlevel;
-	*pipe_cnt_out = pipe_cnt;
-
-	out = true;
-	goto validate_out;
-
-validate_fail:
-	out = false;
-
-validate_out:
-	return out;
-}
-
-static void dml_calculate_dlg_params(
-		struct dc *dc, struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt,
-		int vlevel)
-{
-	int i, pipe_idx;
-	int plane_count;
-
-	/* Writeback MCIF_WB arbitration parameters */
-	if (dc->res_pool)
-		dc->res_pool->funcs->set_mcif_arb_params(dc, context, pipes, pipe_cnt);
-
-	context->bw_ctx.bw.dcn.clk.dispclk_khz = context->bw_ctx.dml.vba.DISPCLK * 1000;
-	context->bw_ctx.bw.dcn.clk.dcfclk_khz = context->bw_ctx.dml.vba.DCFCLK * 1000;
-	context->bw_ctx.bw.dcn.clk.socclk_khz = context->bw_ctx.dml.vba.SOCCLK * 1000;
-	context->bw_ctx.bw.dcn.clk.dramclk_khz = context->bw_ctx.dml.vba.DRAMSpeed * 1000 / 16;
-	context->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz = context->bw_ctx.dml.vba.DCFCLKDeepSleep * 1000;
-	context->bw_ctx.bw.dcn.clk.fclk_khz = context->bw_ctx.dml.vba.FabricClock * 1000;
-	context->bw_ctx.bw.dcn.clk.p_state_change_support =
-		context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb]
-							!= dm_dram_clock_change_unsupported;
-
-	context->bw_ctx.bw.dcn.clk.dppclk_khz = 0;
-	/* 'z9_support': is not a member of 'dc_clocks' - Commenting out till we have this support in dc_clocks
-	 * context->bw_ctx.bw.dcn.clk.z9_support = (context->bw_ctx.dml.vba.StutterPeriod > 5000.0) ?
-			DCN_Z9_SUPPORT_ALLOW : DCN_Z9_SUPPORT_DISALLOW;
-	*/
-	plane_count = 0;
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		if (context->res_ctx.pipe_ctx[i].plane_state)
-			plane_count++;
-	}
-
-	/* Commented out as per above error for now.
-	if (plane_count == 0)
-		context->bw_ctx.bw.dcn.clk.z9_support = DCN_Z9_SUPPORT_ALLOW;
-	*/
-	context->bw_ctx.bw.dcn.clk.dtbclk_en = is_dtbclk_required(dc, context);
-	context->bw_ctx.bw.dcn.clk.fclk_p_state_change_support =
-		context->bw_ctx.dml.vba.FCLKChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
-	if (context->bw_ctx.bw.dcn.clk.dispclk_khz < dc->debug.min_disp_clk_khz)
-		context->bw_ctx.bw.dcn.clk.dispclk_khz = dc->debug.min_disp_clk_khz;
-
-	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-		if (!context->res_ctx.pipe_ctx[i].stream)
-			continue;
-		pipes[pipe_idx].pipe.dest.vstartup_start = get_vstartup(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
-		pipes[pipe_idx].pipe.dest.vupdate_offset = get_vupdate_offset(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
-		pipes[pipe_idx].pipe.dest.vupdate_width = get_vupdate_width(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
-		pipes[pipe_idx].pipe.dest.vready_offset = get_vready_offset(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
-		if (context->res_ctx.pipe_ctx[i].stream->mall_stream_config.type == SUBVP_PHANTOM) {
-			// Phantom pipe requires that DET_SIZE = 0 and no unbounded requests
-			context->res_ctx.pipe_ctx[i].det_buffer_size_kb = 0;
-			context->res_ctx.pipe_ctx[i].unbounded_req = false;
-		} else {
-			context->res_ctx.pipe_ctx[i].det_buffer_size_kb = context->bw_ctx.dml.ip.det_buffer_size_kbytes;
-			context->res_ctx.pipe_ctx[i].unbounded_req = pipes[pipe_idx].pipe.src.unbounded_req_mode;
-		}
-
-		if (context->bw_ctx.bw.dcn.clk.dppclk_khz < pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000)
-			context->bw_ctx.bw.dcn.clk.dppclk_khz = pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000;
-		context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz =
-						pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000;
-		context->res_ctx.pipe_ctx[i].pipe_dlg_param = pipes[pipe_idx].pipe.dest;
-		pipe_idx++;
-	}
-	/*save a original dppclock copy*/
-	context->bw_ctx.bw.dcn.clk.bw_dppclk_khz = context->bw_ctx.bw.dcn.clk.dppclk_khz;
-	context->bw_ctx.bw.dcn.clk.bw_dispclk_khz = context->bw_ctx.bw.dcn.clk.dispclk_khz;
-	context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz = context->bw_ctx.dml.soc.clock_limits[vlevel].dppclk_mhz * 1000;
-	context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz = context->bw_ctx.dml.soc.clock_limits[vlevel].dispclk_mhz * 1000;
-	context->bw_ctx.bw.dcn.compbuf_size_kb = context->bw_ctx.dml.ip.config_return_buffer_size_in_kbytes
-						- context->bw_ctx.dml.ip.det_buffer_size_kbytes * pipe_idx;
-
-	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-		bool cstate_en = context->bw_ctx.dml.vba.PrefetchMode[vlevel][context->bw_ctx.dml.vba.maxMpcComb] != 2;
-
-		if (!context->res_ctx.pipe_ctx[i].stream)
-			continue;
-
-		context->bw_ctx.dml.funcs.rq_dlg_get_dlg_reg(&context->bw_ctx.dml,
-				&context->res_ctx.pipe_ctx[i].dlg_regs,
-				&context->res_ctx.pipe_ctx[i].ttu_regs,
-				pipes,
-				pipe_cnt,
-				pipe_idx,
-				cstate_en,
-				context->bw_ctx.bw.dcn.clk.p_state_change_support,
-				false, false, true);
-
-		context->bw_ctx.dml.funcs.rq_dlg_get_rq_reg(&context->bw_ctx.dml,
-				&context->res_ctx.pipe_ctx[i].rq_regs,
-				&pipes[pipe_idx].pipe);
-		pipe_idx++;
-	}
-}
-
-static void dml_calculate_wm_and_dlg(
-		struct dc *dc, struct dc_state *context,
-		display_e2e_pipe_params_st *pipes,
-		int pipe_cnt,
-		int vlevel)
-{
-	int i, pipe_idx, vlevel_temp = 0;
-
-	double dcfclk = context->bw_ctx.dml.soc.clock_limits[0].dcfclk_mhz;
-	double dcfclk_from_validation = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
-	unsigned int min_dram_speed_mts = context->bw_ctx.dml.vba.DRAMSpeed;
-	bool pstate_en = context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb] !=
-			dm_dram_clock_change_unsupported;
-
-	/* Set B:
-	 * For Set B calculations use clocks from clock_limits[2] when available i.e. when SMU is present,
-	 * otherwise use arbitrary low value from spreadsheet for DCFCLK as lower is safer for watermark
-	 * calculations to cover bootup clocks.
-	 * DCFCLK: soc.clock_limits[2] when available
-	 * UCLK: soc.clock_limits[2] when available
-	 */
-	if (context->bw_ctx.dml.soc.num_states > 2) {
-		vlevel_temp = 2;
-		dcfclk = context->bw_ctx.dml.soc.clock_limits[2].dcfclk_mhz;
-	} else
-		dcfclk = 615; //DCFCLK Vmin_lv
-
-	pipes[0].clks_cfg.voltage = vlevel_temp;
-	pipes[0].clks_cfg.dcfclk_mhz = dcfclk;
-	pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel_temp].socclk_mhz;
-
-	if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].valid) {
-		context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.pstate_latency_us;
-		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.sr_enter_plus_exit_time_us;
-		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.sr_exit_time_us;
-	}
-	context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.fclk_pstate_change_ns = get_fclk_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	//context->bw_ctx.bw.dcn.watermarks.b.usr_retraining_ns = get_wm_usr_retraining(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-
-	/* Temporary, to have some fclk_pstate_change_ns and usr_retraining_ns wm values until DML is implemented */
-	//context->bw_ctx.bw.dcn.watermarks.b.usr_retraining = context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns / 8;
-
-	/* Set D:
-	 * All clocks min.
-	 * DCFCLK: Min, as reported by PM FW when available
-	 * UCLK  : Min, as reported by PM FW when available
-	 * sr_enter_exit/sr_exit should be lower than used for DRAM (TBD after bringup or later, use as decided in Clk Mgr)
-	 */
-
-	if (context->bw_ctx.dml.soc.num_states > 2) {
-		vlevel_temp = 0;
-		dcfclk = dc->clk_mgr->bw_params->clk_table.entries[0].dcfclk_mhz;
-	} else
-		dcfclk = 615; //DCFCLK Vmin_lv
-
-	pipes[0].clks_cfg.voltage = vlevel_temp;
-	pipes[0].clks_cfg.dcfclk_mhz = dcfclk;
-	pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel_temp].socclk_mhz;
-
-	if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].valid) {
-		context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].dml_input.pstate_latency_us;
-		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].dml_input.sr_enter_plus_exit_time_us;
-		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_D].dml_input.sr_exit_time_us;
-	}
-	context->bw_ctx.bw.dcn.watermarks.d.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.fclk_pstate_change_ns = get_fclk_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	//context->bw_ctx.bw.dcn.watermarks.d.usr_retraining_ns = get_wm_usr_retraining(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-
-	/* Temporary, to have some fclk_pstate_change_ns and usr_retraining_ns wm values until DML is implemented */
-	//context->bw_ctx.bw.dcn.watermarks.d.usr_retraining = context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns / 8;
-	/* Set C, for Dummy P-State:
-	 * All clocks min.
-	 * DCFCLK: Min, as reported by PM FW, when available
-	 * UCLK  : Min,  as reported by PM FW, when available
-	 * pstate latency as per UCLK state dummy pstate latency
-	 */
-	if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].valid) {
-		unsigned int min_dram_speed_mts_margin = 160;
-
-		if ((!pstate_en))
-			min_dram_speed_mts = dc->clk_mgr->bw_params->clk_table.entries[dc->clk_mgr->bw_params->clk_table.num_entries - 1].memclk_mhz * 16;
-
-		/* find largest table entry that is lower than dram speed, but lower than DPM0 still uses DPM0 */
-		for (i = 3; i > 0; i--)
-			if (min_dram_speed_mts + min_dram_speed_mts_margin > dc->clk_mgr->bw_params->dummy_pstate_table[i].dram_speed_mts)
-				break;
-
-		context->bw_ctx.dml.soc.dram_clock_change_latency_us = dc->clk_mgr->bw_params->dummy_pstate_table[i].dummy_pstate_latency_us;
-		context->bw_ctx.dml.soc.dummy_pstate_latency_us = dc->clk_mgr->bw_params->dummy_pstate_table[i].dummy_pstate_latency_us;
-		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_enter_plus_exit_time_us;
-		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_exit_time_us;
-	}
-	context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.fclk_pstate_change_ns = get_fclk_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	//context->bw_ctx.bw.dcn.watermarks.c.usr_retraining_ns = get_wm_usr_retraining(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-
-	/* Temporary, to have some fclk_pstate_change_ns and usr_retraining_ns wm values until DML is implemented */
-	//context->bw_ctx.bw.dcn.watermarks.c.usr_retraining = context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns / 8;
-	if ((!pstate_en) && (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].valid)) {
-		/* The only difference between A and C is p-state latency, if p-state is not supported
-		 * with full p-state latency we want to calculate DLG based on dummy p-state latency,
-		 * Set A p-state watermark set to 0 previously, when p-state unsupported, for now keep as previous implementation.
-		 */
-		context->bw_ctx.bw.dcn.watermarks.a = context->bw_ctx.bw.dcn.watermarks.c;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = 0;
-	} else {
-		/* Set A:
-		 * All clocks min.
-		 * DCFCLK: Min, as reported by PM FW, when available
-		 * UCLK: Min, as reported by PM FW, when available
-		 */
-		dml_update_soc_for_wm_a(dc, context);
-		context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	}
-
-	pipes[0].clks_cfg.voltage = vlevel;
-	pipes[0].clks_cfg.dcfclk_mhz = dcfclk_from_validation;
-	pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].socclk_mhz;
-
-	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-		if (!context->res_ctx.pipe_ctx[i].stream)
-			continue;
-
-		pipes[pipe_idx].clks_cfg.dispclk_mhz = get_dispclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt);
-		pipes[pipe_idx].clks_cfg.dppclk_mhz = get_dppclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
-
-		if (dc->config.forced_clocks) {
-			pipes[pipe_idx].clks_cfg.dispclk_mhz = context->bw_ctx.dml.soc.clock_limits[0].dispclk_mhz;
-			pipes[pipe_idx].clks_cfg.dppclk_mhz = context->bw_ctx.dml.soc.clock_limits[0].dppclk_mhz;
-		}
-		if (dc->debug.min_disp_clk_khz > pipes[pipe_idx].clks_cfg.dispclk_mhz * 1000)
-			pipes[pipe_idx].clks_cfg.dispclk_mhz = dc->debug.min_disp_clk_khz / 1000.0;
-		if (dc->debug.min_dpp_clk_khz > pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000)
-			pipes[pipe_idx].clks_cfg.dppclk_mhz = dc->debug.min_dpp_clk_khz / 1000.0;
-
-		pipe_idx++;
-	}
-
-	context->perf_params.stutter_period_us = context->bw_ctx.dml.vba.StutterPeriod;
-
-	dml_calculate_dlg_params(dc, context, pipes, pipe_cnt, vlevel);
-
-	if (!pstate_en)
-		/* Restore full p-state latency */
-		context->bw_ctx.dml.soc.dram_clock_change_latency_us =
-				dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.pstate_latency_us;
-}
-
-bool dml_validate(struct dc *dc,
-		struct dc_state *context,
-		bool fast_validate)
-{
-	bool out = false;
-
-	BW_VAL_TRACE_SETUP();
-
-	int vlevel = 0;
-	int pipe_cnt = 0;
-	display_e2e_pipe_params_st *pipes = context->bw_ctx.dml.dml_pipe_state;
-	DC_LOGGER_INIT(dc->ctx->logger);
-
-	BW_VAL_TRACE_COUNT();
-
-	out = dml_internal_validate(dc, context, pipes, &pipe_cnt, &vlevel, fast_validate);
-
-	if (pipe_cnt == 0)
-		goto validate_out;
-
-	if (!out)
-		goto validate_fail;
-
-	BW_VAL_TRACE_END_VOLTAGE_LEVEL();
-
-	if (fast_validate) {
-		BW_VAL_TRACE_SKIP(fast);
-		goto validate_out;
-	}
-
-	dml_calculate_wm_and_dlg(dc, context, pipes, pipe_cnt, vlevel);
-
-	BW_VAL_TRACE_END_WATERMARKS();
-
-	goto validate_out;
-
-validate_fail:
-	DC_LOG_WARNING("Mode Validation Warning: %s failed validation.\n",
-		dml_get_status_message(context->bw_ctx.dml.vba.ValidationStatus[context->bw_ctx.dml.vba.soc.num_states]));
-
-	BW_VAL_TRACE_SKIP(fail);
-	out = false;
-
-validate_out:
-	BW_VAL_TRACE_FINISH();
-
-	return out;
-}
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml_wrapper_translation.c b/drivers/gpu/drm/amd/display/dc/dml/dml_wrapper_translation.c
deleted file mode 100644
index 4ec5310a2962..000000000000
--- a/drivers/gpu/drm/amd/display/dc/dml/dml_wrapper_translation.c
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Copyright 2017 Advanced Micro Devices, Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: AMD
- *
- */
-
-#ifdef DML_WRAPPER_TRANSLATION_
-
-static void gfx10array_mode_to_dml_params(
-		enum array_mode_values array_mode,
-		enum legacy_tiling_compat_level compat_level,
-		unsigned int *sw_mode)
-{
-	switch (array_mode) {
-	case DC_ARRAY_LINEAR_ALLIGNED:
-	case DC_ARRAY_LINEAR_GENERAL:
-		*sw_mode = dm_sw_linear;
-		break;
-	case DC_ARRAY_2D_TILED_THIN1:
-// DC_LEGACY_TILING_ADDR_GEN_ZERO - undefined as per current code hence removed
-#if 0
-		if (compat_level == DC_LEGACY_TILING_ADDR_GEN_ZERO)
-			*sw_mode = dm_sw_gfx7_2d_thin_l_vp;
-		else
-			*sw_mode = dm_sw_gfx7_2d_thin_gl;
-#endif
-		break;
-	default:
-		ASSERT(0); /* Not supported */
-		break;
-	}
-}
-
-static void swizzle_to_dml_params(
-		enum swizzle_mode_values swizzle,
-		unsigned int *sw_mode)
-{
-	switch (swizzle) {
-	case DC_SW_LINEAR:
-		*sw_mode = dm_sw_linear;
-		break;
-	case DC_SW_4KB_S:
-		*sw_mode = dm_sw_4kb_s;
-		break;
-	case DC_SW_4KB_S_X:
-		*sw_mode = dm_sw_4kb_s_x;
-		break;
-	case DC_SW_4KB_D:
-		*sw_mode = dm_sw_4kb_d;
-		break;
-	case DC_SW_4KB_D_X:
-		*sw_mode = dm_sw_4kb_d_x;
-		break;
-	case DC_SW_64KB_S:
-		*sw_mode = dm_sw_64kb_s;
-		break;
-	case DC_SW_64KB_S_X:
-		*sw_mode = dm_sw_64kb_s_x;
-		break;
-	case DC_SW_64KB_S_T:
-		*sw_mode = dm_sw_64kb_s_t;
-		break;
-	case DC_SW_64KB_D:
-		*sw_mode = dm_sw_64kb_d;
-		break;
-	case DC_SW_64KB_D_X:
-		*sw_mode = dm_sw_64kb_d_x;
-		break;
-	case DC_SW_64KB_D_T:
-		*sw_mode = dm_sw_64kb_d_t;
-		break;
-	case DC_SW_64KB_R_X:
-		*sw_mode = dm_sw_64kb_r_x;
-		break;
-	case DC_SW_VAR_S:
-		*sw_mode = dm_sw_var_s;
-		break;
-	case DC_SW_VAR_S_X:
-		*sw_mode = dm_sw_var_s_x;
-		break;
-	case DC_SW_VAR_D:
-		*sw_mode = dm_sw_var_d;
-		break;
-	case DC_SW_VAR_D_X:
-		*sw_mode = dm_sw_var_d_x;
-		break;
-
-	default:
-		ASSERT(0); /* Not supported */
-		break;
-	}
-}
-
-static void dc_timing_to_dml_timing(const struct dc_crtc_timing *timing, struct _vcs_dpi_display_pipe_dest_params_st *dest)
-{
-	dest->hblank_start = timing->h_total - timing->h_front_porch;
-	dest->hblank_end = dest->hblank_start
-			- timing->h_addressable
-			- timing->h_border_left
-			- timing->h_border_right;
-	dest->vblank_start = timing->v_total - timing->v_front_porch;
-	dest->vblank_end = dest->vblank_start
-			- timing->v_addressable
-			- timing->v_border_top
-			- timing->v_border_bottom;
-	dest->htotal = timing->h_total;
-	dest->vtotal = timing->v_total;
-	dest->hactive = timing->h_addressable;
-	dest->vactive = timing->v_addressable;
-	dest->interlaced = timing->flags.INTERLACE;
-	dest->pixel_rate_mhz = timing->pix_clk_100hz/10000.0;
-	if (timing->timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
-		dest->pixel_rate_mhz *= 2;
-}
-
-static enum odm_combine_mode get_dml_odm_combine(const struct pipe_ctx *pipe)
-{
-	int odm_split_count = 0;
-	enum odm_combine_mode combine_mode = dm_odm_combine_mode_disabled;
-	struct pipe_ctx *next_pipe = pipe->next_odm_pipe;
-
-	// Traverse pipe tree to determine odm split count
-	while (next_pipe) {
-		odm_split_count++;
-		next_pipe = next_pipe->next_odm_pipe;
-	}
-	pipe = pipe->prev_odm_pipe;
-	while (pipe) {
-		odm_split_count++;
-		pipe = pipe->prev_odm_pipe;
-	}
-
-	// Translate split to DML odm combine factor
-	switch (odm_split_count) {
-	case 1:
-		combine_mode = dm_odm_combine_mode_2to1;
-		break;
-	case 3:
-		combine_mode = dm_odm_combine_mode_4to1;
-		break;
-	default:
-		combine_mode = dm_odm_combine_mode_disabled;
-	}
-
-	return combine_mode;
-}
-
-static int get_dml_output_type(enum signal_type dc_signal)
-{
-	int dml_output_type = -1;
-
-	switch (dc_signal) {
-	case SIGNAL_TYPE_DISPLAY_PORT_MST:
-	case SIGNAL_TYPE_DISPLAY_PORT:
-		dml_output_type = dm_dp;
-		break;
-	case SIGNAL_TYPE_EDP:
-		dml_output_type = dm_edp;
-		break;
-	case SIGNAL_TYPE_HDMI_TYPE_A:
-	case SIGNAL_TYPE_DVI_SINGLE_LINK:
-	case SIGNAL_TYPE_DVI_DUAL_LINK:
-		dml_output_type = dm_hdmi;
-		break;
-	default:
-		break;
-	}
-
-	return dml_output_type;
-}
-
-static void populate_color_depth_and_encoding_from_timing(const struct dc_crtc_timing *timing, struct _vcs_dpi_display_output_params_st *dout)
-{
-	int output_bpc = 0;
-
-	switch (timing->display_color_depth) {
-	case COLOR_DEPTH_666:
-		output_bpc = 6;
-		break;
-	case COLOR_DEPTH_888:
-		output_bpc = 8;
-		break;
-	case COLOR_DEPTH_101010:
-		output_bpc = 10;
-		break;
-	case COLOR_DEPTH_121212:
-		output_bpc = 12;
-		break;
-	case COLOR_DEPTH_141414:
-		output_bpc = 14;
-		break;
-	case COLOR_DEPTH_161616:
-		output_bpc = 16;
-		break;
-	case COLOR_DEPTH_999:
-		output_bpc = 9;
-		break;
-	case COLOR_DEPTH_111111:
-		output_bpc = 11;
-		break;
-	default:
-		output_bpc = 8;
-		break;
-	}
-
-	switch (timing->pixel_encoding) {
-	case PIXEL_ENCODING_RGB:
-	case PIXEL_ENCODING_YCBCR444:
-		dout->output_format = dm_444;
-		dout->output_bpp = output_bpc * 3;
-		break;
-	case PIXEL_ENCODING_YCBCR420:
-		dout->output_format = dm_420;
-		dout->output_bpp = (output_bpc * 3.0) / 2;
-		break;
-	case PIXEL_ENCODING_YCBCR422:
-		if (timing->flags.DSC && !timing->dsc_cfg.ycbcr422_simple)
-			dout->output_format = dm_n422;
-		else
-			dout->output_format = dm_s422;
-		dout->output_bpp = output_bpc * 2;
-		break;
-	default:
-		dout->output_format = dm_444;
-		dout->output_bpp = output_bpc * 3;
-	}
-}
-
-static enum source_format_class dc_source_format_to_dml_source_format(enum surface_pixel_format dc_format)
-{
-	enum source_format_class dml_format = dm_444_32;
-
-	switch (dc_format) {
-	case SURFACE_PIXEL_FORMAT_VIDEO_420_YCbCr:
-	case SURFACE_PIXEL_FORMAT_VIDEO_420_YCrCb:
-		dml_format = dm_420_8;
-		break;
-	case SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCbCr:
-	case SURFACE_PIXEL_FORMAT_VIDEO_420_10bpc_YCrCb:
-		dml_format = dm_420_10;
-		break;
-	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616:
-	case SURFACE_PIXEL_FORMAT_GRPH_ARGB16161616F:
-	case SURFACE_PIXEL_FORMAT_GRPH_ABGR16161616F:
-		dml_format = dm_444_64;
-		break;
-	case SURFACE_PIXEL_FORMAT_GRPH_ARGB1555:
-	case SURFACE_PIXEL_FORMAT_GRPH_RGB565:
-		dml_format = dm_444_16;
-		break;
-	case SURFACE_PIXEL_FORMAT_GRPH_PALETA_256_COLORS:
-		dml_format = dm_444_8;
-		break;
-	case SURFACE_PIXEL_FORMAT_GRPH_RGBE_ALPHA:
-		dml_format = dm_rgbe_alpha;
-		break;
-	default:
-		dml_format = dm_444_32;
-		break;
-	}
-
-	return dml_format;
-}
-
-#endif
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index b3d0a4ea2446..8919a2092ac5 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -399,6 +399,10 @@ struct pipe_ctx {
 	struct dc_stream_state *stream;
 
 	struct plane_resource plane_res;
+
+	/**
+	 * @stream_res: Reference to DCN resource components such OPP and DSC.
+	 */
 	struct stream_resource stream_res;
 	struct link_resource link_res;
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h b/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h
index 6682d9e181c6..b304d450b038 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h
@@ -194,6 +194,11 @@ enum dc_status dpcd_configure_lttpr_mode(
 
 enum dp_link_encoding dp_get_link_encoding_format(const struct dc_link_settings *link_settings);
 bool dp_retrieve_lttpr_cap(struct dc_link *link);
+bool dp_is_lttpr_present(struct dc_link *link);
+enum lttpr_mode dp_decide_lttpr_mode(struct dc_link *link, struct dc_link_settings *link_setting);
+void dp_get_lttpr_mode_override(struct dc_link *link, enum lttpr_mode *override);
+enum lttpr_mode dp_decide_8b_10b_lttpr_mode(struct dc_link *link);
+enum lttpr_mode dp_decide_128b_132b_lttpr_mode(struct dc_link *link);
 bool dpcd_write_128b_132b_sst_payload_allocation_table(
 		const struct dc_stream_state *stream,
 		struct dc_link *link,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
index 68c2ed434d2c..cff5fd55a0ad 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr_internal.h
@@ -340,6 +340,8 @@ struct clk_mgr_internal {
 	bool smu_present;
 	void *wm_range_table;
 	long long wm_range_table_addr;
+
+	bool dpm_present;
 };
 
 struct clk_mgr_internal_funcs {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
index d89bd55f110f..cd2be729846b 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hw_shared.h
@@ -268,6 +268,20 @@ enum dc_lut_mode {
 	LUT_RAM_B
 };
 
+enum symclk_state {
+	SYMCLK_OFF_TX_OFF,
+	SYMCLK_ON_TX_ON,
+	SYMCLK_ON_TX_OFF,
+};
+
+struct phy_state {
+	struct {
+		uint8_t otg		: 1;
+		uint8_t reserved	: 7;
+	} symclk_ref_cnts;
+	enum symclk_state symclk_state;
+};
+
 /**
  * speakersToChannels
  *
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
index 5097037e3962..8d86159d9de0 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
@@ -22,6 +22,16 @@
  *
  */
 
+/**
+ * DOC: mpc-overview
+ *
+ * Multiple Pipe/Plane Combined (MPC) is a component in the hardware pipeline
+ * that performs blending of multiple planes, using global and per-pixel alpha.
+ * It also performs post-blending color correction operations according to the
+ * hardware capabilities, such as color transformation matrix and gamma 1D and
+ * 3D LUT.
+ */
+
 #ifndef __DC_MPCC_H__
 #define __DC_MPCC_H__
 
@@ -48,14 +58,39 @@ enum mpcc_blend_mode {
 	MPCC_BLEND_MODE_TOP_BOT_BLENDING
 };
 
+/**
+ * enum mpcc_alpha_blend_mode - define the alpha blend mode regarding pixel
+ * alpha and plane alpha values
+ */
 enum mpcc_alpha_blend_mode {
+	/**
+	 * @MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA: per pixel alpha using DPP
+	 * alpha value
+	 */
 	MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA,
+	/**
+	 * @MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA_COMBINED_GLOBAL_GAIN: per
+	 * pixel alpha using DPP alpha value multiplied by a global gain (plane
+	 * alpha)
+	 */
 	MPCC_ALPHA_BLEND_MODE_PER_PIXEL_ALPHA_COMBINED_GLOBAL_GAIN,
+	/**
+	 * @MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA: global alpha value, ignores
+	 * pixel alpha and consider only plane alpha
+	 */
 	MPCC_ALPHA_BLEND_MODE_GLOBAL_ALPHA
 };
 
-/*
- * MPCC blending configuration
+/**
+ * struct mpcc_blnd_cfg - MPCC blending configuration
+ *
+ * @black_color: background color
+ * @alpha_mode: alpha blend mode (MPCC_ALPHA_BLND_MODE)
+ * @pre_multiplied_alpha: whether pixel color values were pre-multiplied by the
+ * alpha channel (MPCC_ALPHA_MULTIPLIED_MODE)
+ * @global_gain: used when blend mode considers both pixel alpha and plane
+ * alpha value and assumes the global alpha value.
+ * @global_alpha: plane alpha value
  */
 struct mpcc_blnd_cfg {
 	struct tg_color black_color;	/* background color */
@@ -107,8 +142,15 @@ struct mpc_dwb_flow_control {
 	int flow_ctrl_cnt1;
 };
 
-/*
- * MPCC connection and blending configuration for a single MPCC instance.
+/**
+ * struct mpcc - MPCC connection and blending configuration for a single MPCC instance.
+ * @mpcc_id: MPCC physical instance
+ * @dpp_id: DPP input to this MPCC
+ * @mpcc_bot: pointer to bottom layer MPCC. NULL when not connected.
+ * @blnd_cfg: the blending configuration for this MPCC
+ * @sm_cfg: stereo mix setting for this MPCC
+ * @shared_bottom: if MPCC output to both OPP and DWB endpoints, true. Otherwise, false.
+ *
  * This struct is used as a node in an MPC tree.
  */
 struct mpcc {
@@ -120,8 +162,12 @@ struct mpcc {
 	bool shared_bottom;		/* TRUE if MPCC output to both OPP and DWB endpoints, else FALSE */
 };
 
-/*
- * MPC tree represents all MPCC connections for a pipe.
+/**
+ * struct mpc_tree - MPC tree represents all MPCC connections for a pipe.
+ *
+ * @opp_id: the OPP instance that owns this MPC tree
+ * @opp_list: the top MPCC layer of the MPC tree that outputs to OPP endpoint
+ *
  */
 struct mpc_tree {
 	int opp_id;			/* The OPP instance that owns this MPC tree */
@@ -149,13 +195,18 @@ struct mpcc_state {
 	uint32_t busy;
 };
 
+/**
+ * struct mpc_funcs - funcs
+ */
 struct mpc_funcs {
 	void (*read_mpcc_state)(
 			struct mpc *mpc,
 			int mpcc_inst,
 			struct mpcc_state *s);
 
-	/*
+	/**
+	 * @insert_plane:
+	 *
 	 * Insert DPP into MPC tree based on specified blending position.
 	 * Only used for planes that are part of blending chain for OPP output
 	 *
@@ -180,7 +231,9 @@ struct mpc_funcs {
 			int dpp_id,
 			int mpcc_id);
 
-	/*
+	/**
+	 * @remove_mpcc:
+	 *
 	 * Remove a specified MPCC from the MPC tree.
 	 *
 	 * Parameters:
@@ -195,7 +248,9 @@ struct mpc_funcs {
 			struct mpc_tree *tree,
 			struct mpcc *mpcc);
 
-	/*
+	/**
+	 * @mpc_init:
+	 *
 	 * Reset the MPCC HW status by disconnecting all muxes.
 	 *
 	 * Parameters:
@@ -208,7 +263,9 @@ struct mpc_funcs {
 			struct mpc *mpc,
 			unsigned int mpcc_id);
 
-	/*
+	/**
+	 * @update_blending:
+	 *
 	 * Update the blending configuration for a specified MPCC.
 	 *
 	 * Parameters:
@@ -223,7 +280,9 @@ struct mpc_funcs {
 		struct mpcc_blnd_cfg *blnd_cfg,
 		int mpcc_id);
 
-	/*
+	/**
+	 * @cursor_lock:
+	 *
 	 * Lock cursor updates for the specified OPP.
 	 * OPP defines the set of MPCC that are locked together for cursor.
 	 *
@@ -239,8 +298,10 @@ struct mpc_funcs {
 			int opp_id,
 			bool lock);
 
-	/*
-	 * Add DPP into 'secondary' MPC tree based on specified blending position.
+	/**
+	 * @insert_plane_to_secondary:
+	 *
+	 * Add DPP into secondary MPC tree based on specified blending position.
 	 * Only used for planes that are part of blending chain for DWB output
 	 *
 	 * Parameters:
@@ -264,7 +325,9 @@ struct mpc_funcs {
 			int dpp_id,
 			int mpcc_id);
 
-	/*
+	/**
+	 * @remove_mpcc_from_secondary:
+	 *
 	 * Remove a specified DPP from the 'secondary' MPC tree.
 	 *
 	 * Parameters:
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
index 4cfa733cf96f..72eef7a5ed83 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
@@ -137,7 +137,13 @@ struct crc_params {
 	bool enable;
 };
 
+/**
+ * struct timing_generator - Entry point to Output Timing Generator feature.
+ */
 struct timing_generator {
+	/**
+	 * @funcs: Timing generator control functions
+	 */
 	const struct timing_generator_funcs *funcs;
 	struct dc_bios *bp;
 	struct dc_context *ctx;
@@ -148,7 +154,9 @@ struct dc_crtc_timing;
 
 struct drr_params;
 
-
+/**
+ * struct timing_generator_funcs - Control timing generator on a given device.
+ */
 struct timing_generator_funcs {
 	bool (*validate_timing)(struct timing_generator *tg,
 							const struct dc_crtc_timing *timing);
@@ -273,8 +281,8 @@ struct timing_generator_funcs {
 			       const struct crc_params *params);
 
 	/**
-	 * Get CRCs for the given timing generator. Return false if CRCs are
-	 * not enabled (via configure_crc).
+	 * @get_crc: Get CRCs for the given timing generator. Return false if
+	 * CRCs are not enabled (via configure_crc).
 	 */
 	bool (*get_crc)(struct timing_generator *tg,
 			uint32_t *r_cr, uint32_t *g_y, uint32_t *b_cb);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
index ccb3c719fc4d..d04b68dad413 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer.h
@@ -32,11 +32,6 @@
 #include "inc/hw/link_encoder.h"
 #include "core_status.h"
 
-enum vline_select {
-	VLINE0,
-	VLINE1
-};
-
 struct pipe_ctx;
 struct dc_state;
 struct dc_stream_status;
@@ -48,6 +43,7 @@ struct dc_phy_addr_space_config;
 struct dc_virtual_addr_space_config;
 struct dpp;
 struct dce_hwseq;
+struct link_resource;
 
 struct hw_sequencer_funcs {
 	void (*hardware_release)(struct dc *dc);
@@ -88,6 +84,7 @@ struct hw_sequencer_funcs {
 		struct pipe_ctx *pipe_ctx, bool enableTripleBuffer);
 	void (*update_pending_status)(struct pipe_ctx *pipe_ctx);
 	void (*power_down)(struct dc *dc);
+	void (*update_dsc_pg)(struct dc *dc, struct dc_state *context, bool safe_to_disable);
 
 	/* Pipe Lock Related */
 	void (*pipe_control_lock)(struct dc *dc,
@@ -116,8 +113,7 @@ struct hw_sequencer_funcs {
 			int group_index, int group_size,
 			struct pipe_ctx *grouped_pipes[]);
 	void (*setup_periodic_interrupt)(struct dc *dc,
-			struct pipe_ctx *pipe_ctx,
-			enum vline_select vline);
+			struct pipe_ctx *pipe_ctx);
 	void (*set_drr)(struct pipe_ctx **pipe_ctx, int num_pipes,
 			struct dc_crtc_timing_adjust adjust);
 	void (*set_static_screen_control)(struct pipe_ctx **pipe_ctx,
@@ -218,6 +214,25 @@ struct hw_sequencer_funcs {
 
 	void (*set_pipe)(struct pipe_ctx *pipe_ctx);
 
+	void (*enable_dp_link_output)(struct dc_link *link,
+			const struct link_resource *link_res,
+			enum signal_type signal,
+			enum clock_source_id clock_source,
+			const struct dc_link_settings *link_settings);
+	void (*enable_tmds_link_output)(struct dc_link *link,
+			const struct link_resource *link_res,
+			enum signal_type signal,
+			enum clock_source_id clock_source,
+			enum dc_color_depth color_depth,
+			uint32_t pixel_clock);
+	void (*enable_lvds_link_output)(struct dc_link *link,
+			const struct link_resource *link_res,
+			enum clock_source_id clock_source,
+			uint32_t pixel_clock);
+	void (*disable_link_output)(struct dc_link *link,
+			const struct link_resource *link_res,
+			enum signal_type signal);
+
 	void (*get_dcc_en_bits)(struct dc *dc, int *dcc_en_bits);
 
 	/* Idle Optimization Related */
@@ -245,6 +260,10 @@ struct hw_sequencer_funcs {
 			struct tg_color *color,
 			int mpcc_id);
 
+	void (*update_phantom_vp_position)(struct dc *dc,
+			struct dc_state *context,
+			struct pipe_ctx *phantom_pipe);
+
 	void (*commit_subvp_config)(struct dc *dc, struct dc_state *context);
 	void (*subvp_pipe_control_lock)(struct dc *dc,
 			struct dc_state *context,
@@ -271,6 +290,11 @@ void get_surface_visual_confirm_color(
 		const struct pipe_ctx *pipe_ctx,
 		struct tg_color *color);
 
+void get_subvp_visual_confirm_color(
+	struct dc *dc,
+	struct pipe_ctx *pipe_ctx,
+	struct tg_color *color);
+
 void get_hdr_visual_confirm_color(
 		struct pipe_ctx *pipe_ctx,
 		struct tg_color *color);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h
index 1cdea0efe5c1..a4d61bb724b6 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw_sequencer_private.h
@@ -124,6 +124,8 @@ struct hwseq_private_funcs {
 	void (*dsc_pg_control)(struct dce_hwseq *hws,
 			unsigned int dsc_inst,
 			bool power_on);
+	bool (*dsc_pg_status)(struct dce_hwseq *hws,
+			unsigned int dsc_inst);
 	void (*update_odm)(struct dc *dc, struct dc_state *context,
 			struct pipe_ctx *pipe_ctx);
 	void (*program_all_writeback_pipes_in_tree)(struct dc *dc,
diff --git a/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h b/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h
index 3482a877b6af..89964c980b87 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/link_hwss.h
@@ -55,9 +55,6 @@ struct link_hwss_ext {
 			enum signal_type signal,
 			enum clock_source_id clock_source,
 			const struct dc_link_settings *link_settings);
-	void (*disable_dp_link_output)(struct dc_link *link,
-			const struct link_resource *link_res,
-			enum signal_type signal);
 	void (*set_dp_link_test_pattern)(struct dc_link *link,
 			const struct link_resource *link_res,
 			struct encoder_set_dp_phy_pattern_param *tp_params);
@@ -79,6 +76,9 @@ struct link_hwss {
 	void (*setup_stream_encoder)(struct pipe_ctx *pipe_ctx);
 	void (*reset_stream_encoder)(struct pipe_ctx *pipe_ctx);
 	void (*setup_stream_attribute)(struct pipe_ctx *pipe_ctx);
+	void (*disable_link_output)(struct dc_link *link,
+			const struct link_resource *link_res,
+			enum signal_type signal);
 };
 #endif /* __DC_LINK_HWSS_H__ */
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h
index 7614125c92c7..c37d1141febe 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/resource.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h
@@ -228,4 +228,6 @@ uint8_t resource_transmitter_to_phy_idx(const struct dc *dc, enum transmitter tr
 const struct link_hwss *get_link_hwss(const struct dc_link *link,
 		const struct link_resource *link_res);
 
+bool is_h_timing_divisible_by_2(struct dc_stream_state *stream);
+
 #endif /* DRIVERS_GPU_DRM_AMD_DC_DEV_DC_INC_RESOURCE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c
index 5e92019539c8..4227adbc646a 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.c
@@ -130,7 +130,7 @@ void enable_dio_dp_link_output(struct dc_link *link,
 	dp_source_sequence_trace(link, DPCD_SOURCE_SEQ_AFTER_ENABLE_LINK_PHY);
 }
 
-void disable_dio_dp_link_output(struct dc_link *link,
+void disable_dio_link_output(struct dc_link *link,
 		const struct link_resource *link_res,
 		enum signal_type signal)
 {
@@ -174,10 +174,10 @@ static const struct link_hwss dio_link_hwss = {
 	.setup_stream_encoder = setup_dio_stream_encoder,
 	.reset_stream_encoder = reset_dio_stream_encoder,
 	.setup_stream_attribute = setup_dio_stream_attribute,
+	.disable_link_output = disable_dio_link_output,
 	.ext = {
 		.set_throttled_vcp_size = set_dio_throttled_vcp_size,
 		.enable_dp_link_output = enable_dio_dp_link_output,
-		.disable_dp_link_output = disable_dio_dp_link_output,
 		.set_dp_link_test_pattern = set_dio_dp_link_test_pattern,
 		.set_dp_lane_settings = set_dio_dp_lane_settings,
 		.update_stream_allocation_table = update_dio_stream_allocation_table,
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h
index 08f22b32df48..126d37f847a1 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h
+++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dio.h
@@ -40,7 +40,7 @@ void enable_dio_dp_link_output(struct dc_link *link,
 		enum signal_type signal,
 		enum clock_source_id clock_source,
 		const struct dc_link_settings *link_settings);
-void disable_dio_dp_link_output(struct dc_link *link,
+void disable_dio_link_output(struct dc_link *link,
 		const struct link_resource *link_res,
 		enum signal_type signal);
 void set_dio_dp_link_test_pattern(struct dc_link *link,
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c
index 89d4e8159138..64f7ea6a9aa3 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_dpia.c
@@ -56,10 +56,10 @@ static const struct link_hwss dpia_link_hwss = {
 	.setup_stream_encoder = setup_dio_stream_encoder,
 	.reset_stream_encoder = reset_dio_stream_encoder,
 	.setup_stream_attribute = setup_dio_stream_attribute,
+	.disable_link_output = disable_dio_link_output,
 	.ext = {
 		.set_throttled_vcp_size = set_dio_throttled_vcp_size,
 		.enable_dp_link_output = enable_dio_dp_link_output,
-		.disable_dp_link_output = disable_dio_dp_link_output,
 		.set_dp_link_test_pattern = set_dio_dp_link_test_pattern,
 		.set_dp_lane_settings = set_dio_dp_lane_settings,
 		.update_stream_allocation_table = update_dpia_stream_allocation_table,
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c b/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c
index 226af06278ce..7d3147175ca2 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_hwss_hpo_dp.c
@@ -266,11 +266,11 @@ static const struct link_hwss hpo_dp_link_hwss = {
 	.setup_stream_encoder = setup_hpo_dp_stream_encoder,
 	.reset_stream_encoder = reset_hpo_dp_stream_encoder,
 	.setup_stream_attribute = setup_hpo_dp_stream_attribute,
+	.disable_link_output = disable_hpo_dp_link_output,
 	.ext = {
 		.set_throttled_vcp_size = set_hpo_dp_throttled_vcp_size,
 		.set_hblank_min_symbol_width = set_hpo_dp_hblank_min_symbol_width,
 		.enable_dp_link_output = enable_hpo_dp_link_output,
-		.disable_dp_link_output = disable_hpo_dp_link_output,
 		.set_dp_link_test_pattern  = set_hpo_dp_link_test_pattern,
 		.set_dp_lane_settings = set_hpo_dp_lane_settings,
 		.update_stream_allocation_table = update_hpo_dp_stream_allocation_table,
diff --git a/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_hwss.c b/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_hwss.c
index 501173ce270e..9522fe0b36c9 100644
--- a/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_hwss.c
+++ b/drivers/gpu/drm/amd/display/dc/virtual/virtual_link_hwss.c
@@ -36,10 +36,18 @@ void virtual_setup_stream_attribute(struct pipe_ctx *pipe_ctx)
 void virtual_reset_stream_encoder(struct pipe_ctx *pipe_ctx)
 {
 }
+
+void virtual_disable_link_output(struct dc_link *link,
+	const struct link_resource *link_res,
+	enum signal_type signal)
+{
+}
+
 static const struct link_hwss virtual_link_hwss = {
 	.setup_stream_encoder = virtual_setup_stream_encoder,
 	.reset_stream_encoder = virtual_reset_stream_encoder,
 	.setup_stream_attribute = virtual_setup_stream_attribute,
+	.disable_link_output = virtual_disable_link_output,
 };
 
 const struct link_hwss *get_virtual_link_hwss(void)
diff --git a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
index ced176d17bae..f34c45b19fcb 100644
--- a/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dmub/dmub_srv.h
@@ -441,6 +441,7 @@ struct dmub_srv {
 
 	/* Feature capabilities reported by fw */
 	struct dmub_feature_caps feature_caps;
+	struct dmub_visual_confirm_color visual_confirm_color;
 };
 
 /**
diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
index d7f3619352f0..5d1aadade8a5 100644
--- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
+++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
@@ -234,8 +234,7 @@ union dmub_psr_debug_flags {
 };
 
 /**
- * DMUB feature capabilities.
- * After DMUB init, driver will query FW capabilities prior to enabling certain features.
+ * DMUB visual confirm color
  */
 struct dmub_feature_caps {
 	/**
@@ -246,6 +245,16 @@ struct dmub_feature_caps {
 	uint8_t reserved[6];
 };
 
+struct dmub_visual_confirm_color {
+	/**
+	 * Maximum 10 bits color value
+	 */
+	uint16_t color_r_cr;
+	uint16_t color_g_y;
+	uint16_t color_b_cb;
+	uint16_t panel_inst;
+};
+
 #if defined(__cplusplus)
 }
 #endif
@@ -645,6 +654,10 @@ enum dmub_cmd_type {
 	 */
 	DMUB_CMD__QUERY_FEATURE_CAPS = 6,
 	/**
+	 * Command type used to get visual confirm color.
+	 */
+	DMUB_CMD__GET_VISUAL_CONFIRM_COLOR = 8,
+	/**
 	 * Command type used for all PSR commands.
 	 */
 	DMUB_CMD__PSR = 64,
@@ -747,6 +760,11 @@ enum dmub_cmd_dpia_type {
 	DMUB_CMD__DPIA_MST_ALLOC_SLOTS = 2,
 };
 
+enum dmub_cmd_header_sub_type {
+	DMUB_CMD__SUB_TYPE_GENERAL         = 0,
+	DMUB_CMD__SUB_TYPE_CURSOR_POSITION = 1
+};
+
 #pragma pack(push, 1)
 
 /**
@@ -976,8 +994,17 @@ struct dmub_cmd_fw_assisted_mclk_switch_pipe_data_v2 {
 			uint16_t vtotal;
 			uint8_t main_pipe_index;
 			uint8_t phantom_pipe_index;
+			/* Since the microschedule is calculated in terms of OTG lines,
+			 * include any scaling factors to make sure when we get accurate
+			 * conversion when programming MALL_START_LINE (which is in terms
+			 * of HUBP lines). If 4K is being downscaled to 1080p, scale factor
+			 * is 1/2 (numerator = 1, denominator = 2).
+			 */
+			uint8_t scale_factor_numerator;
+			uint8_t scale_factor_denominator;
 			uint8_t is_drr;
-			uint8_t padding;
+			uint8_t main_split_pipe_index;
+			uint8_t phantom_split_pipe_index;
 		} subvp_data;
 
 		struct {
@@ -999,7 +1026,11 @@ struct dmub_cmd_fw_assisted_mclk_switch_pipe_data_v2 {
 		} vblank_data;
 	} pipe_config;
 
-	enum mclk_switch_mode mode;
+	/* - subvp_data in the union (pipe_config) takes up 27 bytes.
+	 * - Make the "mode" field a uint8_t instead of enum so we only use 1 byte (only
+	 *   for the DMCUB command, cast to enum once we populate the DMCUB subvp state).
+	 */
+	uint8_t mode; // enum mclk_switch_mode
 };
 
 /**
@@ -2766,6 +2797,31 @@ struct dmub_rb_cmd_query_feature_caps {
 	struct dmub_cmd_query_feature_caps_data query_feature_caps_data;
 };
 
+/**
+ * Data passed from driver to FW in a DMUB_CMD__GET_VISUAL_CONFIRM_COLOR command.
+ */
+struct dmub_cmd_visual_confirm_color_data {
+	/**
+	 * DMUB feature capabilities.
+	 * After DMUB init, driver will query FW capabilities prior to enabling certain features.
+	 */
+struct dmub_visual_confirm_color visual_confirm_color;
+};
+
+/**
+ * Definition of a DMUB_CMD__GET_VISUAL_CONFIRM_COLOR command.
+ */
+struct dmub_rb_cmd_get_visual_confirm_color {
+ /**
+	 * Command header.
+	 */
+	struct dmub_cmd_header header;
+	/**
+	 * Data passed from driver to FW in a DMUB_CMD__GET_VISUAL_CONFIRM_COLOR command.
+	 */
+	struct dmub_cmd_visual_confirm_color_data visual_confirm_color_data;
+};
+
 struct dmub_optc_state {
 	uint32_t v_total_max;
 	uint32_t v_total_min;
@@ -3138,6 +3194,11 @@ union dmub_rb_cmd {
 	 * Definition of a DMUB_CMD__QUERY_FEATURE_CAPS command.
 	 */
 	struct dmub_rb_cmd_query_feature_caps query_feature_caps;
+
+	/**
+	 * Definition of a DMUB_CMD__GET_VISUAL_CONFIRM_COLOR command.
+	 */
+	struct dmub_rb_cmd_get_visual_confirm_color visual_confirm_color;
 	struct dmub_rb_cmd_drr_update drr_update;
 	struct dmub_rb_cmd_fw_assisted_mclk_switch fw_assisted_mclk_switch;
 
diff --git a/drivers/gpu/drm/amd/display/include/dal_asic_id.h b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
index 9f3558c0ef11..c3089c673975 100644
--- a/drivers/gpu/drm/amd/display/include/dal_asic_id.h
+++ b/drivers/gpu/drm/amd/display/include/dal_asic_id.h
@@ -215,6 +215,7 @@ enum {
 #define DEVICE_ID_NV_143F 0x143F
 #define FAMILY_VGH 144
 #define DEVICE_ID_VGH_163F 0x163F
+#define DEVICE_ID_VGH_1435 0x1435
 #define VANGOGH_A0 0x01
 #define VANGOGH_UNKNOWN 0xFF
 
diff --git a/drivers/gpu/drm/amd/display/include/ddc_service_types.h b/drivers/gpu/drm/amd/display/include/ddc_service_types.h
index 05096c644a60..a7ba5bd8dc16 100644
--- a/drivers/gpu/drm/amd/display/include/ddc_service_types.h
+++ b/drivers/gpu/drm/amd/display/include/ddc_service_types.h
@@ -128,8 +128,8 @@ struct av_sync_data {
 	uint8_t aud_del_ins3;/* DPCD 0002Dh */
 };
 
-static const uint8_t DP_SINK_DEVICE_STR_ID_1[] = {7, 1, 8, 7, 3, 0};
-static const uint8_t DP_SINK_DEVICE_STR_ID_2[] = {7, 1, 8, 7, 5, 0};
+static const uint8_t DP_SINK_DEVICE_STR_ID_1[] = {7, 1, 8, 7, 3};
+static const uint8_t DP_SINK_DEVICE_STR_ID_2[] = {7, 1, 8, 7, 5};
 
 static const u8 DP_SINK_BRANCH_DEV_NAME_7580[] = "7580\x80u";
 
diff --git a/drivers/gpu/drm/amd/display/include/link_service_types.h b/drivers/gpu/drm/amd/display/include/link_service_types.h
index 79fabc51c991..d1e91d31d151 100644
--- a/drivers/gpu/drm/amd/display/include/link_service_types.h
+++ b/drivers/gpu/drm/amd/display/include/link_service_types.h
@@ -83,6 +83,7 @@ enum link_training_result {
 };
 
 enum lttpr_mode {
+	LTTPR_MODE_UNKNOWN,
 	LTTPR_MODE_NON_LTTPR,
 	LTTPR_MODE_TRANSPARENT,
 	LTTPR_MODE_NON_TRANSPARENT,
@@ -246,8 +247,16 @@ union dpcd_training_lane_set {
 };
 
 
+/* AMD's copy of various payload data for MST. We have two copies of the payload table (one in DRM,
+ * one in DC) since DRM's MST helpers can't be accessed here. This stream allocation table should
+ * _ONLY_ be filled out from DM and then passed to DC, do NOT use these for _any_ kind of atomic
+ * state calculations in DM, or you will break something.
+ */
+
+struct drm_dp_mst_port;
+
 /* DP MST stream allocation (payload bandwidth number) */
-struct dp_mst_stream_allocation {
+struct dc_dp_mst_stream_allocation {
 	uint8_t vcp_id;
 	/* number of slots required for the DP stream in
 	 * transport packet */
@@ -255,11 +264,11 @@ struct dp_mst_stream_allocation {
 };
 
 /* DP MST stream allocation table */
-struct dp_mst_stream_allocation_table {
+struct dc_dp_mst_stream_allocation_table {
 	/* number of DP video streams */
 	int stream_count;
 	/* array of stream allocations */
-	struct dp_mst_stream_allocation stream_allocations[MAX_CONTROLLER_NUM];
+	struct dc_dp_mst_stream_allocation stream_allocations[MAX_CONTROLLER_NUM];
 };
 
 #endif /*__DAL_LINK_SERVICE_TYPES_H__*/
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h
index f21554a1c86c..3973110f149c 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_offset.h
@@ -3129,6 +3129,8 @@
 #define mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32_BASE_IDX                                          0
 #define mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32                                                   0x15cc
 #define mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32_BASE_IDX                                          0
+#define mmGCVM_DEBUG                                                                                   0x15cd
+#define mmGCVM_DEBUG_BASE_IDX                                                                          0
 #define mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32                                             0x15ce
 #define mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32_BASE_IDX                                    0
 #define mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32                                             0x15cf
@@ -3151,6 +3153,8 @@
 #define mmGCVM_L2_BANK_SELECT_RESERVED_CID2_BASE_IDX                                                   0
 #define mmGCVM_L2_CACHE_PARITY_CNTL                                                                    0x15d8
 #define mmGCVM_L2_CACHE_PARITY_CNTL_BASE_IDX                                                           0
+#define mmGCVM_L2_IH_LOG_CNTL                                                                          0x15d9
+#define mmGCVM_L2_IH_LOG_CNTL_BASE_IDX                                                                 0
 #define mmGCVM_L2_CNTL5                                                                                0x15dc
 #define mmGCVM_L2_CNTL5_BASE_IDX                                                                       0
 #define mmGCVM_L2_GCR_CNTL                                                                             0x15dd
@@ -9796,14 +9800,118 @@
 
 // addressBlock: gc_pwrdec
 // base address: 0x3c000
+#define mmCGTS_RD_CTRL_REG                                                                             0x5004
+#define mmCGTS_RD_CTRL_REG_BASE_IDX                                                                    1
+#define mmCGTS_RD_REG                                                                                  0x5005
+#define mmCGTS_RD_REG_BASE_IDX                                                                         1
+#define mmCGTS_TCC_DISABLE                                                                             0x5006
+#define mmCGTS_TCC_DISABLE_BASE_IDX                                                                    1
+#define mmCGTS_USER_TCC_DISABLE                                                                        0x5007
+#define mmCGTS_USER_TCC_DISABLE_BASE_IDX                                                               1
+#define mmCGTS_STATUS_REG                                                                              0x5008
+#define mmCGTS_STATUS_REG_BASE_IDX                                                                     1
+#define mmCGTT_SPI_CGTSSM_CLK_CTRL                                                                     0x5009
+#define mmCGTT_SPI_CGTSSM_CLK_CTRL_BASE_IDX                                                            1
+#define mmCGTT_SPI_PS_CLK_CTRL                                                                         0x507d
+#define mmCGTT_SPI_PS_CLK_CTRL_BASE_IDX                                                                1
+#define mmCGTT_SPIS_CLK_CTRL                                                                           0x507e
+#define mmCGTT_SPIS_CLK_CTRL_BASE_IDX                                                                  1
+#define mmCGTT_SPI_CLK_CTRL                                                                            0x5080
+#define mmCGTT_SPI_CLK_CTRL_BASE_IDX                                                                   1
+#define mmCGTT_PC_CLK_CTRL                                                                             0x5081
+#define mmCGTT_PC_CLK_CTRL_BASE_IDX                                                                    1
+#define mmCGTT_BCI_CLK_CTRL                                                                            0x5082
+#define mmCGTT_BCI_CLK_CTRL_BASE_IDX                                                                   1
+#define mmCGTT_VGT_CLK_CTRL                                                                            0x5084
+#define mmCGTT_VGT_CLK_CTRL_BASE_IDX                                                                   1
+#define mmCGTT_IA_CLK_CTRL                                                                             0x5085
+#define mmCGTT_IA_CLK_CTRL_BASE_IDX                                                                    1
+#define mmCGTT_WD_CLK_CTRL                                                                             0x5086
+#define mmCGTT_WD_CLK_CTRL_BASE_IDX                                                                    1
+#define mmCGTT_GS_NGG_CLK_CTRL                                                                         0x5087
+#define mmCGTT_GS_NGG_CLK_CTRL_BASE_IDX                                                                1
+#define mmCGTT_PA_CLK_CTRL                                                                             0x5088
+#define mmCGTT_PA_CLK_CTRL_BASE_IDX                                                                    1
+#define mmCGTT_SC_CLK_CTRL0                                                                            0x5089
+#define mmCGTT_SC_CLK_CTRL0_BASE_IDX                                                                   1
+#define mmCGTT_SC_CLK_CTRL1                                                                            0x508a
+#define mmCGTT_SC_CLK_CTRL1_BASE_IDX                                                                   1
+#define mmCGTT_SC_CLK_CTRL2                                                                            0x508b
+#define mmCGTT_SC_CLK_CTRL2_BASE_IDX                                                                   1
+#define mmCGTT_SQ_CLK_CTRL                                                                             0x508c
+#define mmCGTT_SQ_CLK_CTRL_BASE_IDX                                                                    1
+#define mmCGTT_SQG_CLK_CTRL                                                                            0x508d
+#define mmCGTT_SQG_CLK_CTRL_BASE_IDX                                                                   1
 #define mmSQ_ALU_CLK_CTRL                                                                              0x508e
 #define mmSQ_ALU_CLK_CTRL_BASE_IDX                                                                     1
 #define mmSQ_TEX_CLK_CTRL                                                                              0x508f
 #define mmSQ_TEX_CLK_CTRL_BASE_IDX                                                                     1
 #define mmSQ_LDS_CLK_CTRL                                                                              0x5090
 #define mmSQ_LDS_CLK_CTRL_BASE_IDX                                                                     1
+#define mmCGTT_SX_CLK_CTRL0                                                                            0x5094
+#define mmCGTT_SX_CLK_CTRL0_BASE_IDX                                                                   1
+#define mmCGTT_SX_CLK_CTRL1                                                                            0x5095
+#define mmCGTT_SX_CLK_CTRL1_BASE_IDX                                                                   1
+#define mmCGTT_SX_CLK_CTRL2                                                                            0x5096
+#define mmCGTT_SX_CLK_CTRL2_BASE_IDX                                                                   1
+#define mmCGTT_SX_CLK_CTRL3                                                                            0x5097
+#define mmCGTT_SX_CLK_CTRL3_BASE_IDX                                                                   1
+#define mmCGTT_SX_CLK_CTRL4                                                                            0x5098
+#define mmCGTT_SX_CLK_CTRL4_BASE_IDX                                                                   1
+#define mmTD_CGTT_CTRL                                                                                 0x509c
+#define mmTD_CGTT_CTRL_BASE_IDX                                                                        1
+#define mmTA_CGTT_CTRL                                                                                 0x509d
+#define mmTA_CGTT_CTRL_BASE_IDX                                                                        1
+#define mmCGTT_TCPI_CLK_CTRL                                                                           0x5109
+#define mmCGTT_TCPI_CLK_CTRL_BASE_IDX                                                                  1
+#define mmCGTT_GDS_CLK_CTRL                                                                            0x50a0
+#define mmCGTT_GDS_CLK_CTRL_BASE_IDX                                                                   1
+#define mmDB_CGTT_CLK_CTRL_0                                                                           0x50a4
+#define mmDB_CGTT_CLK_CTRL_0_BASE_IDX                                                                  1
+#define mmCB_CGTT_SCLK_CTRL                                                                            0x50a8
+#define mmCB_CGTT_SCLK_CTRL_BASE_IDX                                                                   1
+#define mmGL2C_CGTT_SCLK_CTRL                                                                          0x50fc
+#define mmGL2C_CGTT_SCLK_CTRL_BASE_IDX                                                                 1
+#define mmGL2A_CGTT_SCLK_CTRL                                                                          0x50ac
+#define mmGL2A_CGTT_SCLK_CTRL_BASE_IDX                                                                 1
+#define mmGL2A_CGTT_SCLK_CTRL_1                                                                        0x50ad
+#define mmGL2A_CGTT_SCLK_CTRL_1_BASE_IDX                                                               1
+#define mmCGTT_CP_CLK_CTRL                                                                             0x50b0
+#define mmCGTT_CP_CLK_CTRL_BASE_IDX                                                                    1
+#define mmCGTT_CPF_CLK_CTRL                                                                            0x50b1
+#define mmCGTT_CPF_CLK_CTRL_BASE_IDX                                                                   1
+#define mmCGTT_CPC_CLK_CTRL                                                                            0x50b2
+#define mmCGTT_CPC_CLK_CTRL_BASE_IDX                                                                   1
+#define mmCGTT_RLC_CLK_CTRL                                                                            0x50b5
+#define mmCGTT_RLC_CLK_CTRL_BASE_IDX                                                                   1
 #define mmRLC_GFX_RM_CNTL                                                                              0x50b6
 #define mmRLC_GFX_RM_CNTL_BASE_IDX                                                                     1
+#define mmRMI_CGTT_SCLK_CTRL                                                                           0x50c0
+#define mmRMI_CGTT_SCLK_CTRL_BASE_IDX                                                                  1
+#define mmCGTT_TCPF_CLK_CTRL                                                                           0x5111
+#define mmCGTT_TCPF_CLK_CTRL_BASE_IDX                                                                  1
+#define mmGCR_CGTT_SCLK_CTRL                                                                           0x50c2
+#define mmGCR_CGTT_SCLK_CTRL_BASE_IDX                                                                  1
+#define mmUTCL1_CGTT_CLK_CTRL                                                                          0x50c3
+#define mmUTCL1_CGTT_CLK_CTRL_BASE_IDX                                                                 1
+#define mmGCEA_CGTT_CLK_CTRL                                                                           0x50c4
+#define mmGCEA_CGTT_CLK_CTRL_BASE_IDX                                                                  1
+#define mmSE_CAC_CGTT_CLK_CTRL                                                                         0x50d0
+#define mmSE_CAC_CGTT_CLK_CTRL_BASE_IDX                                                                1
+#define mmGC_CAC_CGTT_CLK_CTRL                                                                         0x50d8
+#define mmGC_CAC_CGTT_CLK_CTRL_BASE_IDX                                                                1
+#define mmGRBM_CGTT_CLK_CNTL                                                                           0x50e0
+#define mmGRBM_CGTT_CLK_CNTL_BASE_IDX                                                                  1
+#define mmGUS_CGTT_CLK_CTRL                                                                            0x50f4
+#define mmGUS_CGTT_CLK_CTRL_BASE_IDX                                                                   1
+#define mmCGTT_PH_CLK_CTRL0                                                                            0x50f8
+#define mmCGTT_PH_CLK_CTRL0_BASE_IDX                                                                   1
+#define mmCGTT_PH_CLK_CTRL1                                                                            0x50f9
+#define mmCGTT_PH_CLK_CTRL1_BASE_IDX                                                                   1
+#define mmCGTT_PH_CLK_CTRL2                                                                            0x50fa
+#define mmCGTT_PH_CLK_CTRL2_BASE_IDX                                                                   1
+#define mmCGTT_PH_CLK_CTRL3                                                                            0x50fb
+#define mmCGTT_PH_CLK_CTRL3_BASE_IDX                                                                   1
 
 
 // addressBlock: gc_hypdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h
index a827b0ff8905..d4e8ff22ecb8 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_10_3_0_sh_mask.h
@@ -34547,6 +34547,503 @@
 
 
 // addressBlock: gc_pwrdec
+//CGTS_RD_CTRL_REG
+#define CGTS_RD_CTRL_REG__ROW_MUX_SEL__SHIFT                                                                  0x0
+#define CGTS_RD_CTRL_REG__REG_MUX_SEL__SHIFT                                                                  0x4
+#define CGTS_RD_CTRL_REG__ROW_MUX_SEL_MASK                                                                    0x0000000FL
+#define CGTS_RD_CTRL_REG__REG_MUX_SEL_MASK                                                                    0x000000F0L
+//CGTS_RD_REG
+#define CGTS_RD_REG__READ_DATA__SHIFT                                                                         0x0
+#define CGTS_RD_REG__READ_DATA_MASK                                                                           0xFFFFFFFFL
+//CGTS_TCC_DISABLE
+#define CGTS_TCC_DISABLE__HI_TCC_DISABLE__SHIFT                                                               0x8
+#define CGTS_TCC_DISABLE__TCC_DISABLE__SHIFT                                                                  0x10
+#define CGTS_TCC_DISABLE__HI_TCC_DISABLE_MASK                                                                 0x0000FF00L
+#define CGTS_TCC_DISABLE__TCC_DISABLE_MASK                                                                    0xFFFF0000L
+//CGTS_USER_TCC_DISABLE
+#define CGTS_USER_TCC_DISABLE__HI_TCC_DISABLE__SHIFT                                                          0x8
+#define CGTS_USER_TCC_DISABLE__TCC_DISABLE__SHIFT                                                             0x10
+#define CGTS_USER_TCC_DISABLE__HI_TCC_DISABLE_MASK                                                            0x0000FF00L
+#define CGTS_USER_TCC_DISABLE__TCC_DISABLE_MASK                                                               0xFFFF0000L
+//CGTS_STATUS_REG
+#define CGTS_STATUS_REG__SA0_QUAD0_MGCG_ENABLED__SHIFT                                                        0x0
+#define CGTS_STATUS_REG__SA0_QUAD0_CG_STATUS__SHIFT                                                           0x1
+#define CGTS_STATUS_REG__SA1_QUAD0_MGCG_ENABLED__SHIFT                                                        0x8
+#define CGTS_STATUS_REG__SA1_QUAD0_CG_STATUS__SHIFT                                                           0x9
+#define CGTS_STATUS_REG__SA0_QUAD0_MGCG_ENABLED_MASK                                                          0x00000001L
+#define CGTS_STATUS_REG__SA0_QUAD0_CG_STATUS_MASK                                                             0x00000006L
+#define CGTS_STATUS_REG__SA1_QUAD0_MGCG_ENABLED_MASK                                                          0x00000100L
+#define CGTS_STATUS_REG__SA1_QUAD0_CG_STATUS_MASK                                                             0x00000600L
+//CGTT_SPI_CGTSSM_CLK_CTRL
+#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP3_OVERRIDE__SHIFT                                                        0x1b
+#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP2_OVERRIDE__SHIFT                                                        0x1c
+#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP1_OVERRIDE__SHIFT                                                        0x1d
+#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP0_OVERRIDE__SHIFT                                                        0x1e
+#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP3_OVERRIDE_MASK                                                          0x08000000L
+#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP2_OVERRIDE_MASK                                                          0x10000000L
+#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP1_OVERRIDE_MASK                                                          0x20000000L
+#define CGTT_SPI_CGTSSM_CLK_CTRL__GRP0_OVERRIDE_MASK                                                          0x40000000L
+//CGTT_SPI_PS_CLK_CTRL
+#define CGTT_SPI_PS_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
+#define CGTT_SPI_PS_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                     0x10
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                     0x11
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                     0x12
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                     0x13
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                     0x14
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                     0x15
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                     0x16
+#define CGTT_SPI_PS_CLK_CTRL__GRP6_OVERRIDE__SHIFT                                                            0x18
+#define CGTT_SPI_PS_CLK_CTRL__GRP5_OVERRIDE__SHIFT                                                            0x19
+#define CGTT_SPI_PS_CLK_CTRL__GRP4_OVERRIDE__SHIFT                                                            0x1a
+#define CGTT_SPI_PS_CLK_CTRL__GRP3_OVERRIDE__SHIFT                                                            0x1b
+#define CGTT_SPI_PS_CLK_CTRL__GRP2_OVERRIDE__SHIFT                                                            0x1c
+#define CGTT_SPI_PS_CLK_CTRL__GRP1_OVERRIDE__SHIFT                                                            0x1d
+#define CGTT_SPI_PS_CLK_CTRL__GRP0_OVERRIDE__SHIFT                                                            0x1e
+#define CGTT_SPI_PS_CLK_CTRL__REG_OVERRIDE__SHIFT                                                             0x1f
+#define CGTT_SPI_PS_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
+#define CGTT_SPI_PS_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                       0x00010000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                       0x00020000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                       0x00040000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                       0x00080000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                       0x00100000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                       0x00200000L
+#define CGTT_SPI_PS_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                       0x00400000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP6_OVERRIDE_MASK                                                              0x01000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP5_OVERRIDE_MASK                                                              0x02000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP4_OVERRIDE_MASK                                                              0x04000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP3_OVERRIDE_MASK                                                              0x08000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP2_OVERRIDE_MASK                                                              0x10000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP1_OVERRIDE_MASK                                                              0x20000000L
+#define CGTT_SPI_PS_CLK_CTRL__GRP0_OVERRIDE_MASK                                                              0x40000000L
+#define CGTT_SPI_PS_CLK_CTRL__REG_OVERRIDE_MASK                                                               0x80000000L
+//CGTT_SPIS_CLK_CTRL
+#define CGTT_SPIS_CLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
+#define CGTT_SPIS_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x10
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x11
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x12
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x13
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x14
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x15
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x16
+#define CGTT_SPIS_CLK_CTRL__GRP6_OVERRIDE__SHIFT                                                              0x18
+#define CGTT_SPIS_CLK_CTRL__GRP5_OVERRIDE__SHIFT                                                              0x19
+#define CGTT_SPIS_CLK_CTRL__GRP4_OVERRIDE__SHIFT                                                              0x1a
+#define CGTT_SPIS_CLK_CTRL__GRP3_OVERRIDE__SHIFT                                                              0x1b
+#define CGTT_SPIS_CLK_CTRL__GRP2_OVERRIDE__SHIFT                                                              0x1c
+#define CGTT_SPIS_CLK_CTRL__GRP1_OVERRIDE__SHIFT                                                              0x1d
+#define CGTT_SPIS_CLK_CTRL__GRP0_OVERRIDE__SHIFT                                                              0x1e
+#define CGTT_SPIS_CLK_CTRL__REG_OVERRIDE__SHIFT                                                               0x1f
+#define CGTT_SPIS_CLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
+#define CGTT_SPIS_CLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00010000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00020000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00040000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00080000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00100000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00200000L
+#define CGTT_SPIS_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00400000L
+#define CGTT_SPIS_CLK_CTRL__GRP6_OVERRIDE_MASK                                                                0x01000000L
+#define CGTT_SPIS_CLK_CTRL__GRP5_OVERRIDE_MASK                                                                0x02000000L
+#define CGTT_SPIS_CLK_CTRL__GRP4_OVERRIDE_MASK                                                                0x04000000L
+#define CGTT_SPIS_CLK_CTRL__GRP3_OVERRIDE_MASK                                                                0x08000000L
+#define CGTT_SPIS_CLK_CTRL__GRP2_OVERRIDE_MASK                                                                0x10000000L
+#define CGTT_SPIS_CLK_CTRL__GRP1_OVERRIDE_MASK                                                                0x20000000L
+#define CGTT_SPIS_CLK_CTRL__GRP0_OVERRIDE_MASK                                                                0x40000000L
+#define CGTT_SPIS_CLK_CTRL__REG_OVERRIDE_MASK                                                                 0x80000000L
+//CGTT_SPI_CLK_CTRL
+#define CGTT_SPI_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x10
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x11
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x12
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x13
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x14
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x15
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x16
+#define CGTT_SPI_CLK_CTRL__GRP6_OVERRIDE__SHIFT                                                               0x18
+#define CGTT_SPI_CLK_CTRL__GRP5_OVERRIDE__SHIFT                                                               0x19
+#define CGTT_SPI_CLK_CTRL__GRP4_OVERRIDE__SHIFT                                                               0x1a
+#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE__SHIFT                                                               0x1b
+#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE__SHIFT                                                               0x1c
+#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE__SHIFT                                                               0x1d
+#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE__SHIFT                                                               0x1e
+#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_SPI_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SPI_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00010000L
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00020000L
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00040000L
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00080000L
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00100000L
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00200000L
+#define CGTT_SPI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00400000L
+#define CGTT_SPI_CLK_CTRL__GRP6_OVERRIDE_MASK                                                                 0x01000000L
+#define CGTT_SPI_CLK_CTRL__GRP5_OVERRIDE_MASK                                                                 0x02000000L
+#define CGTT_SPI_CLK_CTRL__GRP4_OVERRIDE_MASK                                                                 0x04000000L
+#define CGTT_SPI_CLK_CTRL__GRP3_OVERRIDE_MASK                                                                 0x08000000L
+#define CGTT_SPI_CLK_CTRL__GRP2_OVERRIDE_MASK                                                                 0x10000000L
+#define CGTT_SPI_CLK_CTRL__GRP1_OVERRIDE_MASK                                                                 0x20000000L
+#define CGTT_SPI_CLK_CTRL__GRP0_OVERRIDE_MASK                                                                 0x40000000L
+#define CGTT_SPI_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//CGTT_PC_CLK_CTRL
+#define CGTT_PC_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_PC_CLK_CTRL__PC_RAM_FGCG_OVERRIDE__SHIFT                                                         0x11
+#define CGTT_PC_CLK_CTRL__PC_WRITE_CLK_EN_OVERRIDE__SHIFT                                                     0xd
+#define CGTT_PC_CLK_CTRL__PC_READ_CLK_EN_OVERRIDE__SHIFT                                                      0xe
+#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE__SHIFT                                                               0x1b
+#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE__SHIFT                                                               0x1c
+#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE__SHIFT                                                               0x1d
+#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE__SHIFT                                                               0x1e
+#define CGTT_PC_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_PC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_PC_CLK_CTRL__PC_RAM_FGCG_OVERRIDE_MASK                                                           0x00020000L
+#define CGTT_PC_CLK_CTRL__PC_WRITE_CLK_EN_OVERRIDE_MASK                                                       0x00002000L
+#define CGTT_PC_CLK_CTRL__PC_READ_CLK_EN_OVERRIDE_MASK                                                        0x00004000L
+#define CGTT_PC_CLK_CTRL__CORE3_OVERRIDE_MASK                                                                 0x08000000L
+#define CGTT_PC_CLK_CTRL__CORE2_OVERRIDE_MASK                                                                 0x10000000L
+#define CGTT_PC_CLK_CTRL__CORE1_OVERRIDE_MASK                                                                 0x20000000L
+#define CGTT_PC_CLK_CTRL__CORE0_OVERRIDE_MASK                                                                 0x40000000L
+//CGTT_BCI_CLK_CTRL
+#define CGTT_BCI_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_BCI_CLK_CTRL__RESERVED__SHIFT                                                                    0xc
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE__SHIFT                                                              0x18
+#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE__SHIFT                                                              0x19
+#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE__SHIFT                                                              0x1a
+#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE__SHIFT                                                              0x1b
+#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE__SHIFT                                                              0x1c
+#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE__SHIFT                                                              0x1d
+#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE__SHIFT                                                              0x1e
+#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_BCI_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_BCI_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_BCI_CLK_CTRL__RESERVED_MASK                                                                      0x0000F000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_BCI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_BCI_CLK_CTRL__CORE6_OVERRIDE_MASK                                                                0x01000000L
+#define CGTT_BCI_CLK_CTRL__CORE5_OVERRIDE_MASK                                                                0x02000000L
+#define CGTT_BCI_CLK_CTRL__CORE4_OVERRIDE_MASK                                                                0x04000000L
+#define CGTT_BCI_CLK_CTRL__CORE3_OVERRIDE_MASK                                                                0x08000000L
+#define CGTT_BCI_CLK_CTRL__CORE2_OVERRIDE_MASK                                                                0x10000000L
+#define CGTT_BCI_CLK_CTRL__CORE1_OVERRIDE_MASK                                                                0x20000000L
+#define CGTT_BCI_CLK_CTRL__CORE0_OVERRIDE_MASK                                                                0x40000000L
+#define CGTT_BCI_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//CGTT_VGT_CLK_CTRL
+#define CGTT_VGT_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE__SHIFT                                                                 0xf
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE__SHIFT                                                               0x1c
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                               0x1d
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_VGT_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE_MASK                                                                   0x00008000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE_MASK                                                                 0x10000000L
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE_MASK                                                                 0x20000000L
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//CGTT_IA_CLK_CTRL
+#define CGTT_IA_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x18
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE__SHIFT                                                                  0x19
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1e
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_IA_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x01000000L
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE_MASK                                                                    0x02000000L
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x40000000L
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_WD_CLK_CTRL
+#define CGTT_WD_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_WD_CLK_CTRL__PERF_ENABLE__SHIFT                                                                  0xf
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x18
+#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1d
+#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE__SHIFT                                                          0x1e
+#define CGTT_WD_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_WD_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_WD_CLK_CTRL__PERF_ENABLE_MASK                                                                    0x00008000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_WD_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x01000000L
+#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x20000000L
+#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE_MASK                                                            0x40000000L
+#define CGTT_WD_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_GS_NGG_CLK_CTRL
+#define CGTT_GS_NGG_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
+#define CGTT_GS_NGG_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
+#define CGTT_GS_NGG_CLK_CTRL__PERF_ENABLE__SHIFT                                                              0xf
+#define CGTT_GS_NGG_CLK_CTRL__DBG_ENABLE__SHIFT                                                               0x10
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                     0x11
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                     0x12
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                     0x13
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                     0x14
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                     0x15
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                     0x16
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                     0x17
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                           0x18
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                           0x19
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                           0x1a
+#define CGTT_GS_NGG_CLK_CTRL__PRIMGEN_OVERRIDE__SHIFT                                                         0x1c
+#define CGTT_GS_NGG_CLK_CTRL__GS1_OVERRIDE__SHIFT                                                             0x1d
+#define CGTT_GS_NGG_CLK_CTRL__GS0_OVERRIDE__SHIFT                                                             0x1e
+#define CGTT_GS_NGG_CLK_CTRL__REG_OVERRIDE__SHIFT                                                             0x1f
+#define CGTT_GS_NGG_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
+#define CGTT_GS_NGG_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
+#define CGTT_GS_NGG_CLK_CTRL__PERF_ENABLE_MASK                                                                0x00008000L
+#define CGTT_GS_NGG_CLK_CTRL__DBG_ENABLE_MASK                                                                 0x00010000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                       0x00020000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                       0x00040000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                       0x00080000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                       0x00100000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                       0x00200000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                       0x00400000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                       0x00800000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                             0x01000000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                             0x02000000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                             0x04000000L
+#define CGTT_GS_NGG_CLK_CTRL__PRIMGEN_OVERRIDE_MASK                                                           0x10000000L
+#define CGTT_GS_NGG_CLK_CTRL__GS1_OVERRIDE_MASK                                                               0x20000000L
+#define CGTT_GS_NGG_CLK_CTRL__GS0_OVERRIDE_MASK                                                               0x40000000L
+#define CGTT_GS_NGG_CLK_CTRL__REG_OVERRIDE_MASK                                                               0x80000000L
+//CGTT_PA_CLK_CTRL
+#define CGTT_PA_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x18
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                               0x19
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                               0x1a
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                               0x1b
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                               0x1c
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE__SHIFT                                                              0x1d
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE__SHIFT                                                              0x1e
+#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE__SHIFT                                                             0x1f
+#define CGTT_PA_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_PA_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x01000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                 0x02000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                 0x04000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                 0x08000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                 0x10000000L
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE_MASK                                                                0x20000000L
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE_MASK                                                                0x40000000L
+#define CGTT_PA_CLK_CTRL__REG_CLK_OVERRIDE_MASK                                                               0x80000000L
+//CGTT_SC_CLK_CTRL0
+#define CGTT_SC_CLK_CTRL0__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_STALL_OVERRIDE__SHIFT                                              0x10
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x11
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x12
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x13
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x14
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x15
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x16
+#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE__SHIFT                                                      0x17
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_OVERRIDE__SHIFT                                                    0x18
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT                                                              0x19
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT                                                              0x1a
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT                                                              0x1b
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT                                                              0x1c
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT                                                              0x1d
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT                                                              0x1e
+#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE__SHIFT                                                            0x1f
+#define CGTT_SC_CLK_CTRL0__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_STALL_OVERRIDE_MASK                                                0x00010000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE5_MASK                                                          0x00020000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE4_MASK                                                          0x00040000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE3_MASK                                                          0x00080000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE2_MASK                                                          0x00100000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE1_MASK                                                          0x00200000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE0_MASK                                                          0x00400000L
+#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE_MASK                                                        0x00800000L
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_OVERRIDE_MASK                                                      0x01000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE5_MASK                                                                0x02000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE4_MASK                                                                0x04000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE3_MASK                                                                0x08000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE2_MASK                                                                0x10000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE1_MASK                                                                0x20000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE0_MASK                                                                0x40000000L
+#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE_MASK                                                              0x80000000L
+//CGTT_SC_CLK_CTRL1
+#define CGTT_SC_CLK_CTRL1__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SC_CLK_CTRL1__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE0__SHIFT                                             0x10
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE__SHIFT                                              0x11
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE__SHIFT                                              0x12
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_STALL_OVERRIDE__SHIFT                                     0x13
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_STALL_OVERRIDE__SHIFT                                           0x14
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE__SHIFT                                            0x15
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE__SHIFT                                                      0x16
+#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_STALL_OVERRIDE__SHIFT                                                 0x17
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE0__SHIFT                                                   0x18
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE__SHIFT                                                    0x19
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE__SHIFT                                                    0x1a
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_OVERRIDE__SHIFT                                           0x1b
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE__SHIFT                                                 0x1c
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE__SHIFT                                                  0x1d
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_OVERRIDE__SHIFT                                                            0x1e
+#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_OVERRIDE__SHIFT                                                       0x1f
+#define CGTT_SC_CLK_CTRL1__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SC_CLK_CTRL1__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE0_MASK                                               0x00010000L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE_MASK                                                0x00020000L
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE_MASK                                                0x00040000L
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_STALL_OVERRIDE_MASK                                       0x00080000L
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_STALL_OVERRIDE_MASK                                             0x00100000L
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE_MASK                                              0x00200000L
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE_MASK                                                        0x00400000L
+#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_STALL_OVERRIDE_MASK                                                   0x00800000L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE0_MASK                                                     0x01000000L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE_MASK                                                      0x02000000L
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE_MASK                                                      0x04000000L
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_OVERRIDE_MASK                                             0x08000000L
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE_MASK                                                   0x10000000L
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE_MASK                                                    0x20000000L
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_OVERRIDE_MASK                                                              0x40000000L
+#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_OVERRIDE_MASK                                                         0x80000000L
+//CGTT_SC_CLK_CTRL2
+#define CGTT_SC_CLK_CTRL2__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SC_CLK_CTRL2__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SC_CLK_CTRL2__DBR_CLK_OVERRIDE__SHIFT                                                            0x1a
+#define CGTT_SC_CLK_CTRL2__SCF_SCB_INTF_CLK_OVERRIDE__SHIFT                                                   0x1b
+#define CGTT_SC_CLK_CTRL2__SC_PKR_INTF_CLK_OVERRIDE__SHIFT                                                    0x1c
+#define CGTT_SC_CLK_CTRL2__SC_DB_INTF_CLK_OVERRIDE__SHIFT                                                     0x1d
+#define CGTT_SC_CLK_CTRL2__PA_SC_INTF_CLK_OVERRIDE__SHIFT                                                     0x1e
+#define CGTT_SC_CLK_CTRL2__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SC_CLK_CTRL2__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SC_CLK_CTRL2__DBR_CLK_OVERRIDE_MASK                                                              0x04000000L
+#define CGTT_SC_CLK_CTRL2__SCF_SCB_INTF_CLK_OVERRIDE_MASK                                                     0x08000000L
+#define CGTT_SC_CLK_CTRL2__SC_PKR_INTF_CLK_OVERRIDE_MASK                                                      0x10000000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_INTF_CLK_OVERRIDE_MASK                                                       0x20000000L
+#define CGTT_SC_CLK_CTRL2__PA_SC_INTF_CLK_OVERRIDE_MASK                                                       0x40000000L
+//CGTT_SQ_CLK_CTRL
+#define CGTT_SQ_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1e
+#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_SQ_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_SQ_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_SQ_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_SQ_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x40000000L
+#define CGTT_SQ_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_SQG_CLK_CTRL
+#define CGTT_SQG_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE__SHIFT                                                             0x1c
+#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE__SHIFT                                                            0x1d
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                               0x1e
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_SQG_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE_MASK                                                               0x10000000L
+#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE_MASK                                                              0x20000000L
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE_MASK                                                                 0x40000000L
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
 //SQ_ALU_CLK_CTRL
 #define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA0__SHIFT                                                              0x0
 #define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA1__SHIFT                                                              0x10
@@ -34562,12 +35059,982 @@
 #define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA1__SHIFT                                                              0x10
 #define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA0_MASK                                                                0x0000FFFFL
 #define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA1_MASK                                                                0xFFFF0000L
+//CGTT_SX_CLK_CTRL0
+#define CGTT_SX_CLK_CTRL0__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SX_CLK_CTRL0__RESERVED__SHIFT                                                                    0xc
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT                                                              0x1f
+#define CGTT_SX_CLK_CTRL0__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SX_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SX_CLK_CTRL0__RESERVED_MASK                                                                      0x0000F000L
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_SX_CLK_CTRL0__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+#define CGTT_SX_CLK_CTRL0__SOFT_OVERRIDE0_MASK                                                                0x80000000L
+//CGTT_SX_CLK_CTRL1
+#define CGTT_SX_CLK_CTRL1__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SX_CLK_CTRL1__RESERVED__SHIFT                                                                    0xc
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0__SHIFT                                                              0x1f
+#define CGTT_SX_CLK_CTRL1__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SX_CLK_CTRL1__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SX_CLK_CTRL1__RESERVED_MASK                                                                      0x0000F000L
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_SX_CLK_CTRL1__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+#define CGTT_SX_CLK_CTRL1__SOFT_OVERRIDE0_MASK                                                                0x80000000L
+//CGTT_SX_CLK_CTRL2
+#define CGTT_SX_CLK_CTRL2__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SX_CLK_CTRL2__RESERVED__SHIFT                                                                    0xd
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0__SHIFT                                                              0x1f
+#define CGTT_SX_CLK_CTRL2__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SX_CLK_CTRL2__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SX_CLK_CTRL2__RESERVED_MASK                                                                      0x0000E000L
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_SX_CLK_CTRL2__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+#define CGTT_SX_CLK_CTRL2__SOFT_OVERRIDE0_MASK                                                                0x80000000L
+//CGTT_SX_CLK_CTRL3
+#define CGTT_SX_CLK_CTRL3__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SX_CLK_CTRL3__RESERVED__SHIFT                                                                    0xd
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0__SHIFT                                                              0x1f
+#define CGTT_SX_CLK_CTRL3__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SX_CLK_CTRL3__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SX_CLK_CTRL3__RESERVED_MASK                                                                      0x0000E000L
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_SX_CLK_CTRL3__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+#define CGTT_SX_CLK_CTRL3__SOFT_OVERRIDE0_MASK                                                                0x80000000L
+//CGTT_SX_CLK_CTRL4
+#define CGTT_SX_CLK_CTRL4__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SX_CLK_CTRL4__RESERVED__SHIFT                                                                    0xc
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0__SHIFT                                                              0x1f
+#define CGTT_SX_CLK_CTRL4__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SX_CLK_CTRL4__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SX_CLK_CTRL4__RESERVED_MASK                                                                      0x0000F000L
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_SX_CLK_CTRL4__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+#define CGTT_SX_CLK_CTRL4__SOFT_OVERRIDE0_MASK                                                                0x80000000L
+//TD_CGTT_CTRL
+#define TD_CGTT_CTRL__ON_DELAY__SHIFT                                                                         0x0
+#define TD_CGTT_CTRL__OFF_HYSTERESIS__SHIFT                                                                   0x4
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                             0x10
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                             0x11
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                             0x12
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                             0x13
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                             0x14
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                             0x15
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                             0x16
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                             0x17
+#define TD_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT                                                                   0x18
+#define TD_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT                                                                   0x19
+#define TD_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT                                                                   0x1a
+#define TD_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT                                                                   0x1b
+#define TD_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT                                                                   0x1c
+#define TD_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT                                                                   0x1d
+#define TD_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT                                                                   0x1e
+#define TD_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT                                                                   0x1f
+#define TD_CGTT_CTRL__ON_DELAY_MASK                                                                           0x0000000FL
+#define TD_CGTT_CTRL__OFF_HYSTERESIS_MASK                                                                     0x00000FF0L
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                               0x00010000L
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                               0x00020000L
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                               0x00040000L
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                               0x00080000L
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                               0x00100000L
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                               0x00200000L
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                               0x00400000L
+#define TD_CGTT_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                               0x00800000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE7_MASK                                                                     0x01000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE6_MASK                                                                     0x02000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE5_MASK                                                                     0x04000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE4_MASK                                                                     0x08000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE3_MASK                                                                     0x10000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE2_MASK                                                                     0x20000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE1_MASK                                                                     0x40000000L
+#define TD_CGTT_CTRL__SOFT_OVERRIDE0_MASK                                                                     0x80000000L
+//TA_CGTT_CTRL
+#define TA_CGTT_CTRL__ON_DELAY__SHIFT                                                                         0x0
+#define TA_CGTT_CTRL__OFF_HYSTERESIS__SHIFT                                                                   0x4
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                             0x10
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                             0x11
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                             0x12
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                             0x13
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                             0x14
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                             0x15
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                             0x16
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                             0x17
+#define TA_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT                                                                   0x18
+#define TA_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT                                                                   0x19
+#define TA_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT                                                                   0x1a
+#define TA_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT                                                                   0x1b
+#define TA_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT                                                                   0x1c
+#define TA_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT                                                                   0x1d
+#define TA_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT                                                                   0x1e
+#define TA_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT                                                                   0x1f
+#define TA_CGTT_CTRL__ON_DELAY_MASK                                                                           0x0000000FL
+#define TA_CGTT_CTRL__OFF_HYSTERESIS_MASK                                                                     0x00000FF0L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                               0x00010000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                               0x00020000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                               0x00040000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                               0x00080000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                               0x00100000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                               0x00200000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                               0x00400000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                               0x00800000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE7_MASK                                                                     0x01000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE6_MASK                                                                     0x02000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE5_MASK                                                                     0x04000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE4_MASK                                                                     0x08000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE3_MASK                                                                     0x10000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE2_MASK                                                                     0x20000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE1_MASK                                                                     0x40000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE0_MASK                                                                     0x80000000L
+//CGTT_TCPI_CLK_CTRL
+#define CGTT_TCPI_CLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
+#define CGTT_TCPI_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define CGTT_TCPI_CLK_CTRL__SPARE__SHIFT                                                                      0xc
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0xf
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x10
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x11
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x12
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x13
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x14
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x15
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x16
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                             0x17
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x18
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x19
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1a
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1b
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1c
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1d
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1e
+#define CGTT_TCPI_CLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
+#define CGTT_TCPI_CLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define CGTT_TCPI_CLK_CTRL__SPARE_MASK                                                                        0x00007000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00008000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00010000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00020000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00040000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00080000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00100000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00200000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00400000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                               0x00800000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x01000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x02000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x04000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x08000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x10000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x20000000L
+#define CGTT_TCPI_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x40000000L
+//CGTT_GDS_CLK_CTRL
+#define CGTT_GDS_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_GDS_CLK_CTRL__UNUSED__SHIFT                                                                      0xc
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                              0x1f
+#define CGTT_GDS_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_GDS_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_GDS_CLK_CTRL__UNUSED_MASK                                                                        0x0000F000L
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_GDS_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+#define CGTT_GDS_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                                0x80000000L
+//DB_CGTT_CLK_CTRL_0
+#define DB_CGTT_CLK_CTRL_0__ON_DELAY__SHIFT                                                                   0x0
+#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define DB_CGTT_CLK_CTRL_0__RESERVED__SHIFT                                                                   0xc
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7__SHIFT                                                             0x18
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6__SHIFT                                                             0x19
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5__SHIFT                                                             0x1a
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4__SHIFT                                                             0x1b
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3__SHIFT                                                             0x1c
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2__SHIFT                                                             0x1d
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define DB_CGTT_CLK_CTRL_0__ON_DELAY_MASK                                                                     0x0000000FL
+#define DB_CGTT_CLK_CTRL_0__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define DB_CGTT_CLK_CTRL_0__RESERVED_MASK                                                                     0x0000F000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7_MASK                                                               0x01000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6_MASK                                                               0x02000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5_MASK                                                               0x04000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4_MASK                                                               0x08000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3_MASK                                                               0x10000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2_MASK                                                               0x20000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+//CB_CGTT_SCLK_CTRL
+#define CB_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                              0x1f
+#define CB_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                                0x80000000L
+//GL2C_CGTT_SCLK_CTRL
+#define GL2C_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define GL2C_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                      0x10
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                      0x11
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                      0x12
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                      0x13
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                      0x14
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                      0x15
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                      0x16
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                      0x17
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                            0x18
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                            0x19
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                            0x1a
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                            0x1b
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                            0x1c
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                            0x1d
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                            0x1e
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                            0x1f
+#define GL2C_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define GL2C_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                        0x00010000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                        0x00020000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                        0x00040000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                        0x00080000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                        0x00100000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                        0x00200000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                        0x00400000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                        0x00800000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK                                                              0x01000000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                              0x02000000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                              0x04000000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                              0x08000000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                              0x10000000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                              0x20000000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                              0x40000000L
+#define GL2C_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                              0x80000000L
+//GL2A_CGTT_SCLK_CTRL
+#define GL2A_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define GL2A_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                      0x10
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                      0x11
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                      0x12
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                      0x13
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                      0x14
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                      0x15
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                      0x16
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                      0x17
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                            0x18
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                            0x19
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                            0x1a
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                            0x1b
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                            0x1c
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                            0x1d
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                            0x1e
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                            0x1f
+#define GL2A_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define GL2A_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                        0x00010000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                        0x00020000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                        0x00040000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                        0x00080000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                        0x00100000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                        0x00200000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                        0x00400000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                        0x00800000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK                                                              0x01000000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                              0x02000000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                              0x04000000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                              0x08000000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                              0x10000000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                              0x20000000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                              0x40000000L
+#define GL2A_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                              0x80000000L
+//GL2A_CGTT_SCLK_CTRL_1
+#define GL2A_CGTT_SCLK_CTRL_1__ON_DELAY__SHIFT                                                                0x0
+#define GL2A_CGTT_SCLK_CTRL_1__OFF_HYSTERESIS__SHIFT                                                          0x4
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE7__SHIFT                                                    0x10
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE6__SHIFT                                                    0x11
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE5__SHIFT                                                    0x12
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE4__SHIFT                                                    0x13
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE3__SHIFT                                                    0x14
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE2__SHIFT                                                    0x15
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE1__SHIFT                                                    0x16
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE0__SHIFT                                                    0x17
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE7__SHIFT                                                          0x18
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE6__SHIFT                                                          0x19
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE5__SHIFT                                                          0x1a
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE4__SHIFT                                                          0x1b
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE3__SHIFT                                                          0x1c
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE2__SHIFT                                                          0x1d
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE1__SHIFT                                                          0x1e
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE0__SHIFT                                                          0x1f
+#define GL2A_CGTT_SCLK_CTRL_1__ON_DELAY_MASK                                                                  0x0000000FL
+#define GL2A_CGTT_SCLK_CTRL_1__OFF_HYSTERESIS_MASK                                                            0x00000FF0L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE7_MASK                                                      0x00010000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE6_MASK                                                      0x00020000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE5_MASK                                                      0x00040000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE4_MASK                                                      0x00080000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE3_MASK                                                      0x00100000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE2_MASK                                                      0x00200000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE1_MASK                                                      0x00400000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_STALL_OVERRIDE0_MASK                                                      0x00800000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE7_MASK                                                            0x01000000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE6_MASK                                                            0x02000000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE5_MASK                                                            0x04000000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE4_MASK                                                            0x08000000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE3_MASK                                                            0x10000000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE2_MASK                                                            0x20000000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE1_MASK                                                            0x40000000L
+#define GL2A_CGTT_SCLK_CTRL_1__SOFT_OVERRIDE0_MASK                                                            0x80000000L
+//CGTT_CP_CLK_CTRL
+#define CGTT_CP_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                                0xf
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                        0x1d
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                            0x1e
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                            0x1f
+#define CGTT_CP_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                  0x00008000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                          0x20000000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                              0x40000000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                              0x80000000L
+//CGTT_CPF_CLK_CTRL
+#define CGTT_CPF_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_CPF_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                               0xf
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1a
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PRT__SHIFT                                                           0x1b
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_CMP__SHIFT                                                           0x1c
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_GFX__SHIFT                                                           0x1d
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
+#define CGTT_CPF_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_CPF_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                 0x00008000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x04000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PRT_MASK                                                             0x08000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_CMP_MASK                                                             0x10000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_GFX_MASK                                                             0x20000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
+//CGTT_CPC_CLK_CTRL
+#define CGTT_CPC_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                               0xf
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1d
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
+#define CGTT_CPC_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                 0x00008000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x20000000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
+//CGTT_RLC_CLK_CTRL
+#define CGTT_RLC_CLK_CTRL__RESERVED__SHIFT                                                                    0x0
+#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
+#define CGTT_RLC_CLK_CTRL__RESERVED_MASK                                                                      0x0000000FL
+#define CGTT_RLC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_RLC_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+#define CGTT_RLC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
 //RLC_GFX_RM_CNTL
 #define RLC_GFX_RM_CNTL__RLC_GFX_RM_VALID__SHIFT                                                              0x0
 #define RLC_GFX_RM_CNTL__RESERVED__SHIFT                                                                      0x1
 #define RLC_GFX_RM_CNTL__RLC_GFX_RM_VALID_MASK                                                                0x00000001L
 #define RLC_GFX_RM_CNTL__RESERVED_MASK                                                                        0xFFFFFFFEL
-
+//RMI_CGTT_SCLK_CTRL
+#define RMI_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
+#define RMI_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define RMI_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
+#define RMI_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define RMI_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+//CGTT_TCPF_CLK_CTRL
+#define CGTT_TCPF_CLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
+#define CGTT_TCPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define CGTT_TCPF_CLK_CTRL__SPARE__SHIFT                                                                      0xc
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0xf
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x10
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x11
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x12
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x13
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x14
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x15
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x16
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                             0x17
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x18
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x19
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1a
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1b
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1c
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1d
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1e
+#define CGTT_TCPF_CLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
+#define CGTT_TCPF_CLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define CGTT_TCPF_CLK_CTRL__SPARE_MASK                                                                        0x00007000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00008000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00010000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00020000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00040000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00080000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00100000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00200000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00400000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                               0x00800000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x01000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x02000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x04000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x08000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x10000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x20000000L
+#define CGTT_TCPF_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x40000000L
+//GCR_CGTT_SCLK_CTRL
+#define GCR_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
+#define GCR_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x19
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x1a
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x1b
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x1c
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x1d
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1e
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x1f
+#define GCR_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
+#define GCR_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                               0x02000000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                               0x04000000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                               0x08000000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                               0x10000000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                               0x20000000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                               0x40000000L
+#define GCR_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                               0x80000000L
+//UTCL1_CGTT_CLK_CTRL
+#define UTCL1_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                  0x0
+#define UTCL1_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                            0x4
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                      0x10
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                      0x11
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                      0x12
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                      0x13
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                      0x14
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                      0x15
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                      0x16
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                      0x17
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                            0x19
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                            0x1a
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                            0x1b
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                            0x1c
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                            0x1d
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                            0x1e
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                            0x1f
+#define UTCL1_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                    0x0000000FL
+#define UTCL1_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                              0x00000FF0L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                        0x00010000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                        0x00020000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                        0x00040000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                        0x00080000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                        0x00100000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                        0x00200000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                        0x00400000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                        0x00800000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                              0x02000000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                              0x04000000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE4_MASK                                                              0x08000000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE3_MASK                                                              0x10000000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE2_MASK                                                              0x20000000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE1_MASK                                                              0x40000000L
+#define UTCL1_CGTT_CLK_CTRL__SOFT_OVERRIDE0_MASK                                                              0x80000000L
+//GCEA_CGTT_CLK_CTRL
+#define GCEA_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                   0x0
+#define GCEA_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define GCEA_CGTT_CLK_CTRL__SPARE0__SHIFT                                                                     0xc
+#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE__SHIFT                                                  0x14
+#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ__SHIFT                                                   0x15
+#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN__SHIFT                                                 0x16
+#define GCEA_CGTT_CLK_CTRL__SPARE1__SHIFT                                                                     0x17
+#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE__SHIFT                                                        0x1c
+#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ__SHIFT                                                         0x1d
+#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                       0x1e
+#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                     0x1f
+#define GCEA_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                     0x0000000FL
+#define GCEA_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define GCEA_CGTT_CLK_CTRL__SPARE0_MASK                                                                       0x000FF000L
+#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE_MASK                                                    0x00100000L
+#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ_MASK                                                     0x00200000L
+#define GCEA_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN_MASK                                                   0x00400000L
+#define GCEA_CGTT_CLK_CTRL__SPARE1_MASK                                                                       0x0F800000L
+#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE_MASK                                                          0x10000000L
+#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ_MASK                                                           0x20000000L
+#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                         0x40000000L
+#define GCEA_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                       0x80000000L
+//SE_CAC_CGTT_CLK_CTRL
+#define SE_CAC_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
+#define SE_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
+#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                        0x1e
+#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                        0x1f
+#define SE_CAC_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
+#define SE_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
+#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                          0x40000000L
+#define SE_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                          0x80000000L
+//GC_CAC_CGTT_CLK_CTRL
+#define GC_CAC_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
+#define GC_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
+#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                        0x1e
+#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                        0x1f
+#define GC_CAC_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
+#define GC_CAC_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
+#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                          0x40000000L
+#define GC_CAC_CGTT_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                          0x80000000L
+//GRBM_CGTT_CLK_CNTL
+#define GRBM_CGTT_CLK_CNTL__ON_DELAY__SHIFT                                                                   0x0
+#define GRBM_CGTT_CLK_CNTL__OFF_HYSTERESIS__SHIFT                                                             0x4
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE7__SHIFT                                                       0x10
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE6__SHIFT                                                       0x11
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE5__SHIFT                                                       0x12
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE4__SHIFT                                                       0x13
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE3__SHIFT                                                       0x14
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE2__SHIFT                                                       0x15
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE1__SHIFT                                                       0x16
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE0__SHIFT                                                       0x17
+#define GRBM_CGTT_CLK_CNTL__SOFT_OVERRIDE_DYN__SHIFT                                                          0x1e
+#define GRBM_CGTT_CLK_CNTL__ON_DELAY_MASK                                                                     0x0000000FL
+#define GRBM_CGTT_CLK_CNTL__OFF_HYSTERESIS_MASK                                                               0x00000FF0L
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE7_MASK                                                         0x00010000L
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE6_MASK                                                         0x00020000L
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE5_MASK                                                         0x00040000L
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE4_MASK                                                         0x00080000L
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE3_MASK                                                         0x00100000L
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE2_MASK                                                         0x00200000L
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE1_MASK                                                         0x00400000L
+#define GRBM_CGTT_CLK_CNTL__SOFT_STALL_OVERRIDE0_MASK                                                         0x00800000L
+#define GRBM_CGTT_CLK_CNTL__SOFT_OVERRIDE_DYN_MASK                                                            0x40000000L
+//GUS_CGTT_CLK_CTRL
+#define GUS_CGTT_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define GUS_CGTT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define GUS_CGTT_CLK_CTRL__SPARE0__SHIFT                                                                      0xc
+#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_DRAM__SHIFT                                                    0x13
+#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE__SHIFT                                                   0x14
+#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ__SHIFT                                                    0x15
+#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN__SHIFT                                                  0x16
+#define GUS_CGTT_CLK_CTRL__SPARE1__SHIFT                                                                      0x17
+#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_DRAM__SHIFT                                                          0x1b
+#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE__SHIFT                                                         0x1c
+#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ__SHIFT                                                          0x1d
+#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                        0x1e
+#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                      0x1f
+#define GUS_CGTT_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define GUS_CGTT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define GUS_CGTT_CLK_CTRL__SPARE0_MASK                                                                        0x0007F000L
+#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_DRAM_MASK                                                      0x00080000L
+#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_WRITE_MASK                                                     0x00100000L
+#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_READ_MASK                                                      0x00200000L
+#define GUS_CGTT_CLK_CTRL__SOFT_STALL_OVERRIDE_RETURN_MASK                                                    0x00400000L
+#define GUS_CGTT_CLK_CTRL__SPARE1_MASK                                                                        0x07800000L
+#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_DRAM_MASK                                                            0x08000000L
+#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_WRITE_MASK                                                           0x10000000L
+#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_READ_MASK                                                            0x20000000L
+#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                          0x40000000L
+#define GUS_CGTT_CLK_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                        0x80000000L
+//CGTT_PH_CLK_CTRL0
+#define CGTT_PH_CLK_CTRL0__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL0__PERFMON_CLK_OVERRIDE__SHIFT                                                        0x1e
+#define CGTT_PH_CLK_CTRL0__REG_CLK_OVERRIDE__SHIFT                                                            0x1f
+#define CGTT_PH_CLK_CTRL0__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_PH_CLK_CTRL0__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL0__PERFMON_CLK_OVERRIDE_MASK                                                          0x40000000L
+#define CGTT_PH_CLK_CTRL0__REG_CLK_OVERRIDE_MASK                                                              0x80000000L
+//CGTT_PH_CLK_CTRL1
+#define CGTT_PH_CLK_CTRL1__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL1__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_PH_CLK_CTRL1__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL1__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+//CGTT_PH_CLK_CTRL2
+#define CGTT_PH_CLK_CTRL2__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL2__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_PH_CLK_CTRL2__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL2__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+//CGTT_PH_CLK_CTRL3
+#define CGTT_PH_CLK_CTRL3__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL3__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_PH_CLK_CTRL3__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL3__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE1_MASK                                                                0x40000000L
 
 // addressBlock: gc_hypdec
 //CP_HYP_PFP_UCODE_ADDR
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_3_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_3_offset.h
new file mode 100644
index 000000000000..3b95a59b196c
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_3_offset.h
@@ -0,0 +1,12086 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _gc_11_0_3_OFFSET_HEADER
+#define _gc_11_0_3_OFFSET_HEADER
+
+
+
+// addressBlock: gc_sdma0_sdma0dec
+// base address: 0x4980
+#define regSDMA0_DEC_START                                                                              0x0000
+#define regSDMA0_DEC_START_BASE_IDX                                                                     0
+#define regSDMA0_F32_MISC_CNTL                                                                          0x000b
+#define regSDMA0_F32_MISC_CNTL_BASE_IDX                                                                 0
+#define regSDMA0_GLOBAL_TIMESTAMP_LO                                                                    0x000f
+#define regSDMA0_GLOBAL_TIMESTAMP_LO_BASE_IDX                                                           0
+#define regSDMA0_GLOBAL_TIMESTAMP_HI                                                                    0x0010
+#define regSDMA0_GLOBAL_TIMESTAMP_HI_BASE_IDX                                                           0
+#define regSDMA0_POWER_CNTL                                                                             0x001a
+#define regSDMA0_POWER_CNTL_BASE_IDX                                                                    0
+#define regSDMA0_CNTL                                                                                   0x001c
+#define regSDMA0_CNTL_BASE_IDX                                                                          0
+#define regSDMA0_CHICKEN_BITS                                                                           0x001d
+#define regSDMA0_CHICKEN_BITS_BASE_IDX                                                                  0
+#define regSDMA0_GB_ADDR_CONFIG                                                                         0x001e
+#define regSDMA0_GB_ADDR_CONFIG_BASE_IDX                                                                0
+#define regSDMA0_GB_ADDR_CONFIG_READ                                                                    0x001f
+#define regSDMA0_GB_ADDR_CONFIG_READ_BASE_IDX                                                           0
+#define regSDMA0_RB_RPTR_FETCH                                                                          0x0020
+#define regSDMA0_RB_RPTR_FETCH_BASE_IDX                                                                 0
+#define regSDMA0_RB_RPTR_FETCH_HI                                                                       0x0021
+#define regSDMA0_RB_RPTR_FETCH_HI_BASE_IDX                                                              0
+#define regSDMA0_SEM_WAIT_FAIL_TIMER_CNTL                                                               0x0022
+#define regSDMA0_SEM_WAIT_FAIL_TIMER_CNTL_BASE_IDX                                                      0
+#define regSDMA0_IB_OFFSET_FETCH                                                                        0x0023
+#define regSDMA0_IB_OFFSET_FETCH_BASE_IDX                                                               0
+#define regSDMA0_PROGRAM                                                                                0x0024
+#define regSDMA0_PROGRAM_BASE_IDX                                                                       0
+#define regSDMA0_STATUS_REG                                                                             0x0025
+#define regSDMA0_STATUS_REG_BASE_IDX                                                                    0
+#define regSDMA0_STATUS1_REG                                                                            0x0026
+#define regSDMA0_STATUS1_REG_BASE_IDX                                                                   0
+#define regSDMA0_CNTL1                                                                                  0x0027
+#define regSDMA0_CNTL1_BASE_IDX                                                                         0
+#define regSDMA0_HBM_PAGE_CONFIG                                                                        0x0028
+#define regSDMA0_HBM_PAGE_CONFIG_BASE_IDX                                                               0
+#define regSDMA0_UCODE_CHECKSUM                                                                         0x0029
+#define regSDMA0_UCODE_CHECKSUM_BASE_IDX                                                                0
+#define regSDMA0_FREEZE                                                                                 0x002b
+#define regSDMA0_FREEZE_BASE_IDX                                                                        0
+#define regSDMA0_PROCESS_QUANTUM0                                                                       0x002c
+#define regSDMA0_PROCESS_QUANTUM0_BASE_IDX                                                              0
+#define regSDMA0_PROCESS_QUANTUM1                                                                       0x002d
+#define regSDMA0_PROCESS_QUANTUM1_BASE_IDX                                                              0
+#define regSDMA0_WATCHDOG_CNTL                                                                          0x002e
+#define regSDMA0_WATCHDOG_CNTL_BASE_IDX                                                                 0
+#define regSDMA0_QUEUE_STATUS0                                                                          0x002f
+#define regSDMA0_QUEUE_STATUS0_BASE_IDX                                                                 0
+#define regSDMA0_EDC_CONFIG                                                                             0x0032
+#define regSDMA0_EDC_CONFIG_BASE_IDX                                                                    0
+#define regSDMA0_BA_THRESHOLD                                                                           0x0033
+#define regSDMA0_BA_THRESHOLD_BASE_IDX                                                                  0
+#define regSDMA0_ID                                                                                     0x0034
+#define regSDMA0_ID_BASE_IDX                                                                            0
+#define regSDMA0_VERSION                                                                                0x0035
+#define regSDMA0_VERSION_BASE_IDX                                                                       0
+#define regSDMA0_EDC_COUNTER                                                                            0x0036
+#define regSDMA0_EDC_COUNTER_BASE_IDX                                                                   0
+#define regSDMA0_EDC_COUNTER_CLEAR                                                                      0x0037
+#define regSDMA0_EDC_COUNTER_CLEAR_BASE_IDX                                                             0
+#define regSDMA0_STATUS2_REG                                                                            0x0038
+#define regSDMA0_STATUS2_REG_BASE_IDX                                                                   0
+#define regSDMA0_ATOMIC_CNTL                                                                            0x0039
+#define regSDMA0_ATOMIC_CNTL_BASE_IDX                                                                   0
+#define regSDMA0_ATOMIC_PREOP_LO                                                                        0x003a
+#define regSDMA0_ATOMIC_PREOP_LO_BASE_IDX                                                               0
+#define regSDMA0_ATOMIC_PREOP_HI                                                                        0x003b
+#define regSDMA0_ATOMIC_PREOP_HI_BASE_IDX                                                               0
+#define regSDMA0_UTCL1_CNTL                                                                             0x003c
+#define regSDMA0_UTCL1_CNTL_BASE_IDX                                                                    0
+#define regSDMA0_UTCL1_WATERMK                                                                          0x003d
+#define regSDMA0_UTCL1_WATERMK_BASE_IDX                                                                 0
+#define regSDMA0_UTCL1_TIMEOUT                                                                          0x003e
+#define regSDMA0_UTCL1_TIMEOUT_BASE_IDX                                                                 0
+#define regSDMA0_UTCL1_PAGE                                                                             0x003f
+#define regSDMA0_UTCL1_PAGE_BASE_IDX                                                                    0
+#define regSDMA0_UTCL1_RD_STATUS                                                                        0x0040
+#define regSDMA0_UTCL1_RD_STATUS_BASE_IDX                                                               0
+#define regSDMA0_UTCL1_WR_STATUS                                                                        0x0041
+#define regSDMA0_UTCL1_WR_STATUS_BASE_IDX                                                               0
+#define regSDMA0_UTCL1_INV0                                                                             0x0042
+#define regSDMA0_UTCL1_INV0_BASE_IDX                                                                    0
+#define regSDMA0_UTCL1_INV1                                                                             0x0043
+#define regSDMA0_UTCL1_INV1_BASE_IDX                                                                    0
+#define regSDMA0_UTCL1_INV2                                                                             0x0044
+#define regSDMA0_UTCL1_INV2_BASE_IDX                                                                    0
+#define regSDMA0_UTCL1_RD_XNACK0                                                                        0x0045
+#define regSDMA0_UTCL1_RD_XNACK0_BASE_IDX                                                               0
+#define regSDMA0_UTCL1_RD_XNACK1                                                                        0x0046
+#define regSDMA0_UTCL1_RD_XNACK1_BASE_IDX                                                               0
+#define regSDMA0_UTCL1_WR_XNACK0                                                                        0x0047
+#define regSDMA0_UTCL1_WR_XNACK0_BASE_IDX                                                               0
+#define regSDMA0_UTCL1_WR_XNACK1                                                                        0x0048
+#define regSDMA0_UTCL1_WR_XNACK1_BASE_IDX                                                               0
+#define regSDMA0_RELAX_ORDERING_LUT                                                                     0x004a
+#define regSDMA0_RELAX_ORDERING_LUT_BASE_IDX                                                            0
+#define regSDMA0_CHICKEN_BITS_2                                                                         0x004b
+#define regSDMA0_CHICKEN_BITS_2_BASE_IDX                                                                0
+#define regSDMA0_STATUS3_REG                                                                            0x004c
+#define regSDMA0_STATUS3_REG_BASE_IDX                                                                   0
+#define regSDMA0_PHYSICAL_ADDR_LO                                                                       0x004d
+#define regSDMA0_PHYSICAL_ADDR_LO_BASE_IDX                                                              0
+#define regSDMA0_PHYSICAL_ADDR_HI                                                                       0x004e
+#define regSDMA0_PHYSICAL_ADDR_HI_BASE_IDX                                                              0
+#define regSDMA0_GLOBAL_QUANTUM                                                                         0x004f
+#define regSDMA0_GLOBAL_QUANTUM_BASE_IDX                                                                0
+#define regSDMA0_ERROR_LOG                                                                              0x0050
+#define regSDMA0_ERROR_LOG_BASE_IDX                                                                     0
+#define regSDMA0_PUB_DUMMY_REG0                                                                         0x0051
+#define regSDMA0_PUB_DUMMY_REG0_BASE_IDX                                                                0
+#define regSDMA0_PUB_DUMMY_REG1                                                                         0x0052
+#define regSDMA0_PUB_DUMMY_REG1_BASE_IDX                                                                0
+#define regSDMA0_PUB_DUMMY_REG2                                                                         0x0053
+#define regSDMA0_PUB_DUMMY_REG2_BASE_IDX                                                                0
+#define regSDMA0_PUB_DUMMY_REG3                                                                         0x0054
+#define regSDMA0_PUB_DUMMY_REG3_BASE_IDX                                                                0
+#define regSDMA0_F32_COUNTER                                                                            0x0055
+#define regSDMA0_F32_COUNTER_BASE_IDX                                                                   0
+#define regSDMA0_CRD_CNTL                                                                               0x005b
+#define regSDMA0_CRD_CNTL_BASE_IDX                                                                      0
+#define regSDMA0_RLC_CGCG_CTRL                                                                          0x005c
+#define regSDMA0_RLC_CGCG_CTRL_BASE_IDX                                                                 0
+#define regSDMA0_GPU_IOV_VIOLATION_LOG                                                                  0x005d
+#define regSDMA0_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                         0
+#define regSDMA0_AQL_STATUS                                                                             0x005f
+#define regSDMA0_AQL_STATUS_BASE_IDX                                                                    0
+#define regSDMA0_EA_DBIT_ADDR_DATA                                                                      0x0060
+#define regSDMA0_EA_DBIT_ADDR_DATA_BASE_IDX                                                             0
+#define regSDMA0_EA_DBIT_ADDR_INDEX                                                                     0x0061
+#define regSDMA0_EA_DBIT_ADDR_INDEX_BASE_IDX                                                            0
+#define regSDMA0_TLBI_GCR_CNTL                                                                          0x0062
+#define regSDMA0_TLBI_GCR_CNTL_BASE_IDX                                                                 0
+#define regSDMA0_TILING_CONFIG                                                                          0x0063
+#define regSDMA0_TILING_CONFIG_BASE_IDX                                                                 0
+#define regSDMA0_HASH                                                                                   0x0064
+#define regSDMA0_HASH_BASE_IDX                                                                          0
+#define regSDMA0_INT_STATUS                                                                             0x0070
+#define regSDMA0_INT_STATUS_BASE_IDX                                                                    0
+#define regSDMA0_GPU_IOV_VIOLATION_LOG2                                                                 0x0071
+#define regSDMA0_GPU_IOV_VIOLATION_LOG2_BASE_IDX                                                        0
+#define regSDMA0_HOLE_ADDR_LO                                                                           0x0072
+#define regSDMA0_HOLE_ADDR_LO_BASE_IDX                                                                  0
+#define regSDMA0_HOLE_ADDR_HI                                                                           0x0073
+#define regSDMA0_HOLE_ADDR_HI_BASE_IDX                                                                  0
+#define regSDMA0_CLOCK_GATING_STATUS                                                                    0x0075
+#define regSDMA0_CLOCK_GATING_STATUS_BASE_IDX                                                           0
+#define regSDMA0_STATUS4_REG                                                                            0x0076
+#define regSDMA0_STATUS4_REG_BASE_IDX                                                                   0
+#define regSDMA0_SCRATCH_RAM_DATA                                                                       0x0077
+#define regSDMA0_SCRATCH_RAM_DATA_BASE_IDX                                                              0
+#define regSDMA0_SCRATCH_RAM_ADDR                                                                       0x0078
+#define regSDMA0_SCRATCH_RAM_ADDR_BASE_IDX                                                              0
+#define regSDMA0_TIMESTAMP_CNTL                                                                         0x0079
+#define regSDMA0_TIMESTAMP_CNTL_BASE_IDX                                                                0
+#define regSDMA0_STATUS5_REG                                                                            0x007a
+#define regSDMA0_STATUS5_REG_BASE_IDX                                                                   0
+#define regSDMA0_QUEUE_RESET_REQ                                                                        0x007b
+#define regSDMA0_QUEUE_RESET_REQ_BASE_IDX                                                               0
+#define regSDMA0_STATUS6_REG                                                                            0x007c
+#define regSDMA0_STATUS6_REG_BASE_IDX                                                                   0
+#define regSDMA0_UCODE1_CHECKSUM                                                                        0x007d
+#define regSDMA0_UCODE1_CHECKSUM_BASE_IDX                                                               0
+#define regSDMA0_CE_CTRL                                                                                0x007e
+#define regSDMA0_CE_CTRL_BASE_IDX                                                                       0
+#define regSDMA0_FED_STATUS                                                                             0x007f
+#define regSDMA0_FED_STATUS_BASE_IDX                                                                    0
+#define regSDMA0_QUEUE0_RB_CNTL                                                                         0x0080
+#define regSDMA0_QUEUE0_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE0_RB_BASE                                                                         0x0081
+#define regSDMA0_QUEUE0_RB_BASE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE0_RB_BASE_HI                                                                      0x0082
+#define regSDMA0_QUEUE0_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE0_RB_RPTR                                                                         0x0083
+#define regSDMA0_QUEUE0_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE0_RB_RPTR_HI                                                                      0x0084
+#define regSDMA0_QUEUE0_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE0_RB_WPTR                                                                         0x0085
+#define regSDMA0_QUEUE0_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE0_RB_WPTR_HI                                                                      0x0086
+#define regSDMA0_QUEUE0_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE0_RB_RPTR_ADDR_HI                                                                 0x0088
+#define regSDMA0_QUEUE0_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA0_QUEUE0_RB_RPTR_ADDR_LO                                                                 0x0089
+#define regSDMA0_QUEUE0_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA0_QUEUE0_IB_CNTL                                                                         0x008a
+#define regSDMA0_QUEUE0_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE0_IB_RPTR                                                                         0x008b
+#define regSDMA0_QUEUE0_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE0_IB_OFFSET                                                                       0x008c
+#define regSDMA0_QUEUE0_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA0_QUEUE0_IB_BASE_LO                                                                      0x008d
+#define regSDMA0_QUEUE0_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA0_QUEUE0_IB_BASE_HI                                                                      0x008e
+#define regSDMA0_QUEUE0_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE0_IB_SIZE                                                                         0x008f
+#define regSDMA0_QUEUE0_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE0_SKIP_CNTL                                                                       0x0090
+#define regSDMA0_QUEUE0_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA0_QUEUE0_CONTEXT_STATUS                                                                  0x0091
+#define regSDMA0_QUEUE0_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA0_QUEUE0_DOORBELL                                                                        0x0092
+#define regSDMA0_QUEUE0_DOORBELL_BASE_IDX                                                               0
+#define regSDMA0_QUEUE0_DOORBELL_LOG                                                                    0x00a9
+#define regSDMA0_QUEUE0_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_DOORBELL_OFFSET                                                                 0x00ab
+#define regSDMA0_QUEUE0_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA0_QUEUE0_CSA_ADDR_LO                                                                     0x00ac
+#define regSDMA0_QUEUE0_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA0_QUEUE0_CSA_ADDR_HI                                                                     0x00ad
+#define regSDMA0_QUEUE0_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA0_QUEUE0_SCHEDULE_CNTL                                                                   0x00ae
+#define regSDMA0_QUEUE0_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA0_QUEUE0_IB_SUB_REMAIN                                                                   0x00af
+#define regSDMA0_QUEUE0_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA0_QUEUE0_PREEMPT                                                                         0x00b0
+#define regSDMA0_QUEUE0_PREEMPT_BASE_IDX                                                                0
+#define regSDMA0_QUEUE0_DUMMY_REG                                                                       0x00b1
+#define regSDMA0_QUEUE0_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI                                                            0x00b2
+#define regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO                                                            0x00b3
+#define regSDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_QUEUE0_RB_AQL_CNTL                                                                     0x00b4
+#define regSDMA0_QUEUE0_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE0_MINOR_PTR_UPDATE                                                                0x00b5
+#define regSDMA0_QUEUE0_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA0_QUEUE0_RB_PREEMPT                                                                      0x00b6
+#define regSDMA0_QUEUE0_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA0_QUEUE0_MIDCMD_DATA0                                                                    0x00c0
+#define regSDMA0_QUEUE0_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_MIDCMD_DATA1                                                                    0x00c1
+#define regSDMA0_QUEUE0_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_MIDCMD_DATA2                                                                    0x00c2
+#define regSDMA0_QUEUE0_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_MIDCMD_DATA3                                                                    0x00c3
+#define regSDMA0_QUEUE0_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_MIDCMD_DATA4                                                                    0x00c4
+#define regSDMA0_QUEUE0_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_MIDCMD_DATA5                                                                    0x00c5
+#define regSDMA0_QUEUE0_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_MIDCMD_DATA6                                                                    0x00c6
+#define regSDMA0_QUEUE0_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_MIDCMD_DATA7                                                                    0x00c7
+#define regSDMA0_QUEUE0_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_MIDCMD_DATA8                                                                    0x00c8
+#define regSDMA0_QUEUE0_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_MIDCMD_DATA9                                                                    0x00c9
+#define regSDMA0_QUEUE0_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA0_QUEUE0_MIDCMD_DATA10                                                                   0x00ca
+#define regSDMA0_QUEUE0_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA0_QUEUE0_MIDCMD_CNTL                                                                     0x00cb
+#define regSDMA0_QUEUE0_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE1_RB_CNTL                                                                         0x00d8
+#define regSDMA0_QUEUE1_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE1_RB_BASE                                                                         0x00d9
+#define regSDMA0_QUEUE1_RB_BASE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE1_RB_BASE_HI                                                                      0x00da
+#define regSDMA0_QUEUE1_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE1_RB_RPTR                                                                         0x00db
+#define regSDMA0_QUEUE1_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE1_RB_RPTR_HI                                                                      0x00dc
+#define regSDMA0_QUEUE1_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE1_RB_WPTR                                                                         0x00dd
+#define regSDMA0_QUEUE1_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE1_RB_WPTR_HI                                                                      0x00de
+#define regSDMA0_QUEUE1_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE1_RB_RPTR_ADDR_HI                                                                 0x00e0
+#define regSDMA0_QUEUE1_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA0_QUEUE1_RB_RPTR_ADDR_LO                                                                 0x00e1
+#define regSDMA0_QUEUE1_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA0_QUEUE1_IB_CNTL                                                                         0x00e2
+#define regSDMA0_QUEUE1_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE1_IB_RPTR                                                                         0x00e3
+#define regSDMA0_QUEUE1_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE1_IB_OFFSET                                                                       0x00e4
+#define regSDMA0_QUEUE1_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA0_QUEUE1_IB_BASE_LO                                                                      0x00e5
+#define regSDMA0_QUEUE1_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA0_QUEUE1_IB_BASE_HI                                                                      0x00e6
+#define regSDMA0_QUEUE1_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE1_IB_SIZE                                                                         0x00e7
+#define regSDMA0_QUEUE1_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE1_SKIP_CNTL                                                                       0x00e8
+#define regSDMA0_QUEUE1_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA0_QUEUE1_CONTEXT_STATUS                                                                  0x00e9
+#define regSDMA0_QUEUE1_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA0_QUEUE1_DOORBELL                                                                        0x00ea
+#define regSDMA0_QUEUE1_DOORBELL_BASE_IDX                                                               0
+#define regSDMA0_QUEUE1_DOORBELL_LOG                                                                    0x0101
+#define regSDMA0_QUEUE1_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_DOORBELL_OFFSET                                                                 0x0103
+#define regSDMA0_QUEUE1_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA0_QUEUE1_CSA_ADDR_LO                                                                     0x0104
+#define regSDMA0_QUEUE1_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA0_QUEUE1_CSA_ADDR_HI                                                                     0x0105
+#define regSDMA0_QUEUE1_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA0_QUEUE1_SCHEDULE_CNTL                                                                   0x0106
+#define regSDMA0_QUEUE1_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA0_QUEUE1_IB_SUB_REMAIN                                                                   0x0107
+#define regSDMA0_QUEUE1_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA0_QUEUE1_PREEMPT                                                                         0x0108
+#define regSDMA0_QUEUE1_PREEMPT_BASE_IDX                                                                0
+#define regSDMA0_QUEUE1_DUMMY_REG                                                                       0x0109
+#define regSDMA0_QUEUE1_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA0_QUEUE1_RB_WPTR_POLL_ADDR_HI                                                            0x010a
+#define regSDMA0_QUEUE1_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_QUEUE1_RB_WPTR_POLL_ADDR_LO                                                            0x010b
+#define regSDMA0_QUEUE1_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_QUEUE1_RB_AQL_CNTL                                                                     0x010c
+#define regSDMA0_QUEUE1_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE1_MINOR_PTR_UPDATE                                                                0x010d
+#define regSDMA0_QUEUE1_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA0_QUEUE1_RB_PREEMPT                                                                      0x010e
+#define regSDMA0_QUEUE1_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA0_QUEUE1_MIDCMD_DATA0                                                                    0x0118
+#define regSDMA0_QUEUE1_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_MIDCMD_DATA1                                                                    0x0119
+#define regSDMA0_QUEUE1_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_MIDCMD_DATA2                                                                    0x011a
+#define regSDMA0_QUEUE1_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_MIDCMD_DATA3                                                                    0x011b
+#define regSDMA0_QUEUE1_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_MIDCMD_DATA4                                                                    0x011c
+#define regSDMA0_QUEUE1_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_MIDCMD_DATA5                                                                    0x011d
+#define regSDMA0_QUEUE1_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_MIDCMD_DATA6                                                                    0x011e
+#define regSDMA0_QUEUE1_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_MIDCMD_DATA7                                                                    0x011f
+#define regSDMA0_QUEUE1_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_MIDCMD_DATA8                                                                    0x0120
+#define regSDMA0_QUEUE1_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_MIDCMD_DATA9                                                                    0x0121
+#define regSDMA0_QUEUE1_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA0_QUEUE1_MIDCMD_DATA10                                                                   0x0122
+#define regSDMA0_QUEUE1_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA0_QUEUE1_MIDCMD_CNTL                                                                     0x0123
+#define regSDMA0_QUEUE1_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE2_RB_CNTL                                                                         0x0130
+#define regSDMA0_QUEUE2_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE2_RB_BASE                                                                         0x0131
+#define regSDMA0_QUEUE2_RB_BASE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE2_RB_BASE_HI                                                                      0x0132
+#define regSDMA0_QUEUE2_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE2_RB_RPTR                                                                         0x0133
+#define regSDMA0_QUEUE2_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE2_RB_RPTR_HI                                                                      0x0134
+#define regSDMA0_QUEUE2_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE2_RB_WPTR                                                                         0x0135
+#define regSDMA0_QUEUE2_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE2_RB_WPTR_HI                                                                      0x0136
+#define regSDMA0_QUEUE2_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE2_RB_RPTR_ADDR_HI                                                                 0x0138
+#define regSDMA0_QUEUE2_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA0_QUEUE2_RB_RPTR_ADDR_LO                                                                 0x0139
+#define regSDMA0_QUEUE2_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA0_QUEUE2_IB_CNTL                                                                         0x013a
+#define regSDMA0_QUEUE2_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE2_IB_RPTR                                                                         0x013b
+#define regSDMA0_QUEUE2_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE2_IB_OFFSET                                                                       0x013c
+#define regSDMA0_QUEUE2_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA0_QUEUE2_IB_BASE_LO                                                                      0x013d
+#define regSDMA0_QUEUE2_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA0_QUEUE2_IB_BASE_HI                                                                      0x013e
+#define regSDMA0_QUEUE2_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE2_IB_SIZE                                                                         0x013f
+#define regSDMA0_QUEUE2_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE2_SKIP_CNTL                                                                       0x0140
+#define regSDMA0_QUEUE2_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA0_QUEUE2_CONTEXT_STATUS                                                                  0x0141
+#define regSDMA0_QUEUE2_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA0_QUEUE2_DOORBELL                                                                        0x0142
+#define regSDMA0_QUEUE2_DOORBELL_BASE_IDX                                                               0
+#define regSDMA0_QUEUE2_DOORBELL_LOG                                                                    0x0159
+#define regSDMA0_QUEUE2_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_DOORBELL_OFFSET                                                                 0x015b
+#define regSDMA0_QUEUE2_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA0_QUEUE2_CSA_ADDR_LO                                                                     0x015c
+#define regSDMA0_QUEUE2_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA0_QUEUE2_CSA_ADDR_HI                                                                     0x015d
+#define regSDMA0_QUEUE2_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA0_QUEUE2_SCHEDULE_CNTL                                                                   0x015e
+#define regSDMA0_QUEUE2_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA0_QUEUE2_IB_SUB_REMAIN                                                                   0x015f
+#define regSDMA0_QUEUE2_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA0_QUEUE2_PREEMPT                                                                         0x0160
+#define regSDMA0_QUEUE2_PREEMPT_BASE_IDX                                                                0
+#define regSDMA0_QUEUE2_DUMMY_REG                                                                       0x0161
+#define regSDMA0_QUEUE2_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA0_QUEUE2_RB_WPTR_POLL_ADDR_HI                                                            0x0162
+#define regSDMA0_QUEUE2_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_QUEUE2_RB_WPTR_POLL_ADDR_LO                                                            0x0163
+#define regSDMA0_QUEUE2_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_QUEUE2_RB_AQL_CNTL                                                                     0x0164
+#define regSDMA0_QUEUE2_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE2_MINOR_PTR_UPDATE                                                                0x0165
+#define regSDMA0_QUEUE2_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA0_QUEUE2_RB_PREEMPT                                                                      0x0166
+#define regSDMA0_QUEUE2_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA0_QUEUE2_MIDCMD_DATA0                                                                    0x0170
+#define regSDMA0_QUEUE2_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_MIDCMD_DATA1                                                                    0x0171
+#define regSDMA0_QUEUE2_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_MIDCMD_DATA2                                                                    0x0172
+#define regSDMA0_QUEUE2_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_MIDCMD_DATA3                                                                    0x0173
+#define regSDMA0_QUEUE2_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_MIDCMD_DATA4                                                                    0x0174
+#define regSDMA0_QUEUE2_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_MIDCMD_DATA5                                                                    0x0175
+#define regSDMA0_QUEUE2_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_MIDCMD_DATA6                                                                    0x0176
+#define regSDMA0_QUEUE2_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_MIDCMD_DATA7                                                                    0x0177
+#define regSDMA0_QUEUE2_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_MIDCMD_DATA8                                                                    0x0178
+#define regSDMA0_QUEUE2_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_MIDCMD_DATA9                                                                    0x0179
+#define regSDMA0_QUEUE2_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA0_QUEUE2_MIDCMD_DATA10                                                                   0x017a
+#define regSDMA0_QUEUE2_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA0_QUEUE2_MIDCMD_CNTL                                                                     0x017b
+#define regSDMA0_QUEUE2_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE3_RB_CNTL                                                                         0x0188
+#define regSDMA0_QUEUE3_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE3_RB_BASE                                                                         0x0189
+#define regSDMA0_QUEUE3_RB_BASE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE3_RB_BASE_HI                                                                      0x018a
+#define regSDMA0_QUEUE3_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE3_RB_RPTR                                                                         0x018b
+#define regSDMA0_QUEUE3_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE3_RB_RPTR_HI                                                                      0x018c
+#define regSDMA0_QUEUE3_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE3_RB_WPTR                                                                         0x018d
+#define regSDMA0_QUEUE3_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE3_RB_WPTR_HI                                                                      0x018e
+#define regSDMA0_QUEUE3_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE3_RB_RPTR_ADDR_HI                                                                 0x0190
+#define regSDMA0_QUEUE3_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA0_QUEUE3_RB_RPTR_ADDR_LO                                                                 0x0191
+#define regSDMA0_QUEUE3_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA0_QUEUE3_IB_CNTL                                                                         0x0192
+#define regSDMA0_QUEUE3_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE3_IB_RPTR                                                                         0x0193
+#define regSDMA0_QUEUE3_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE3_IB_OFFSET                                                                       0x0194
+#define regSDMA0_QUEUE3_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA0_QUEUE3_IB_BASE_LO                                                                      0x0195
+#define regSDMA0_QUEUE3_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA0_QUEUE3_IB_BASE_HI                                                                      0x0196
+#define regSDMA0_QUEUE3_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE3_IB_SIZE                                                                         0x0197
+#define regSDMA0_QUEUE3_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE3_SKIP_CNTL                                                                       0x0198
+#define regSDMA0_QUEUE3_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA0_QUEUE3_CONTEXT_STATUS                                                                  0x0199
+#define regSDMA0_QUEUE3_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA0_QUEUE3_DOORBELL                                                                        0x019a
+#define regSDMA0_QUEUE3_DOORBELL_BASE_IDX                                                               0
+#define regSDMA0_QUEUE3_DOORBELL_LOG                                                                    0x01b1
+#define regSDMA0_QUEUE3_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_DOORBELL_OFFSET                                                                 0x01b3
+#define regSDMA0_QUEUE3_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA0_QUEUE3_CSA_ADDR_LO                                                                     0x01b4
+#define regSDMA0_QUEUE3_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA0_QUEUE3_CSA_ADDR_HI                                                                     0x01b5
+#define regSDMA0_QUEUE3_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA0_QUEUE3_SCHEDULE_CNTL                                                                   0x01b6
+#define regSDMA0_QUEUE3_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA0_QUEUE3_IB_SUB_REMAIN                                                                   0x01b7
+#define regSDMA0_QUEUE3_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA0_QUEUE3_PREEMPT                                                                         0x01b8
+#define regSDMA0_QUEUE3_PREEMPT_BASE_IDX                                                                0
+#define regSDMA0_QUEUE3_DUMMY_REG                                                                       0x01b9
+#define regSDMA0_QUEUE3_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA0_QUEUE3_RB_WPTR_POLL_ADDR_HI                                                            0x01ba
+#define regSDMA0_QUEUE3_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_QUEUE3_RB_WPTR_POLL_ADDR_LO                                                            0x01bb
+#define regSDMA0_QUEUE3_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_QUEUE3_RB_AQL_CNTL                                                                     0x01bc
+#define regSDMA0_QUEUE3_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE3_MINOR_PTR_UPDATE                                                                0x01bd
+#define regSDMA0_QUEUE3_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA0_QUEUE3_RB_PREEMPT                                                                      0x01be
+#define regSDMA0_QUEUE3_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA0_QUEUE3_MIDCMD_DATA0                                                                    0x01c8
+#define regSDMA0_QUEUE3_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_MIDCMD_DATA1                                                                    0x01c9
+#define regSDMA0_QUEUE3_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_MIDCMD_DATA2                                                                    0x01ca
+#define regSDMA0_QUEUE3_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_MIDCMD_DATA3                                                                    0x01cb
+#define regSDMA0_QUEUE3_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_MIDCMD_DATA4                                                                    0x01cc
+#define regSDMA0_QUEUE3_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_MIDCMD_DATA5                                                                    0x01cd
+#define regSDMA0_QUEUE3_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_MIDCMD_DATA6                                                                    0x01ce
+#define regSDMA0_QUEUE3_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_MIDCMD_DATA7                                                                    0x01cf
+#define regSDMA0_QUEUE3_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_MIDCMD_DATA8                                                                    0x01d0
+#define regSDMA0_QUEUE3_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_MIDCMD_DATA9                                                                    0x01d1
+#define regSDMA0_QUEUE3_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA0_QUEUE3_MIDCMD_DATA10                                                                   0x01d2
+#define regSDMA0_QUEUE3_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA0_QUEUE3_MIDCMD_CNTL                                                                     0x01d3
+#define regSDMA0_QUEUE3_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE4_RB_CNTL                                                                         0x01e0
+#define regSDMA0_QUEUE4_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE4_RB_BASE                                                                         0x01e1
+#define regSDMA0_QUEUE4_RB_BASE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE4_RB_BASE_HI                                                                      0x01e2
+#define regSDMA0_QUEUE4_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE4_RB_RPTR                                                                         0x01e3
+#define regSDMA0_QUEUE4_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE4_RB_RPTR_HI                                                                      0x01e4
+#define regSDMA0_QUEUE4_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE4_RB_WPTR                                                                         0x01e5
+#define regSDMA0_QUEUE4_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE4_RB_WPTR_HI                                                                      0x01e6
+#define regSDMA0_QUEUE4_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE4_RB_RPTR_ADDR_HI                                                                 0x01e8
+#define regSDMA0_QUEUE4_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA0_QUEUE4_RB_RPTR_ADDR_LO                                                                 0x01e9
+#define regSDMA0_QUEUE4_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA0_QUEUE4_IB_CNTL                                                                         0x01ea
+#define regSDMA0_QUEUE4_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE4_IB_RPTR                                                                         0x01eb
+#define regSDMA0_QUEUE4_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE4_IB_OFFSET                                                                       0x01ec
+#define regSDMA0_QUEUE4_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA0_QUEUE4_IB_BASE_LO                                                                      0x01ed
+#define regSDMA0_QUEUE4_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA0_QUEUE4_IB_BASE_HI                                                                      0x01ee
+#define regSDMA0_QUEUE4_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE4_IB_SIZE                                                                         0x01ef
+#define regSDMA0_QUEUE4_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE4_SKIP_CNTL                                                                       0x01f0
+#define regSDMA0_QUEUE4_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA0_QUEUE4_CONTEXT_STATUS                                                                  0x01f1
+#define regSDMA0_QUEUE4_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA0_QUEUE4_DOORBELL                                                                        0x01f2
+#define regSDMA0_QUEUE4_DOORBELL_BASE_IDX                                                               0
+#define regSDMA0_QUEUE4_DOORBELL_LOG                                                                    0x0209
+#define regSDMA0_QUEUE4_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_DOORBELL_OFFSET                                                                 0x020b
+#define regSDMA0_QUEUE4_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA0_QUEUE4_CSA_ADDR_LO                                                                     0x020c
+#define regSDMA0_QUEUE4_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA0_QUEUE4_CSA_ADDR_HI                                                                     0x020d
+#define regSDMA0_QUEUE4_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA0_QUEUE4_SCHEDULE_CNTL                                                                   0x020e
+#define regSDMA0_QUEUE4_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA0_QUEUE4_IB_SUB_REMAIN                                                                   0x020f
+#define regSDMA0_QUEUE4_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA0_QUEUE4_PREEMPT                                                                         0x0210
+#define regSDMA0_QUEUE4_PREEMPT_BASE_IDX                                                                0
+#define regSDMA0_QUEUE4_DUMMY_REG                                                                       0x0211
+#define regSDMA0_QUEUE4_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA0_QUEUE4_RB_WPTR_POLL_ADDR_HI                                                            0x0212
+#define regSDMA0_QUEUE4_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_QUEUE4_RB_WPTR_POLL_ADDR_LO                                                            0x0213
+#define regSDMA0_QUEUE4_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_QUEUE4_RB_AQL_CNTL                                                                     0x0214
+#define regSDMA0_QUEUE4_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE4_MINOR_PTR_UPDATE                                                                0x0215
+#define regSDMA0_QUEUE4_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA0_QUEUE4_RB_PREEMPT                                                                      0x0216
+#define regSDMA0_QUEUE4_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA0_QUEUE4_MIDCMD_DATA0                                                                    0x0220
+#define regSDMA0_QUEUE4_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_MIDCMD_DATA1                                                                    0x0221
+#define regSDMA0_QUEUE4_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_MIDCMD_DATA2                                                                    0x0222
+#define regSDMA0_QUEUE4_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_MIDCMD_DATA3                                                                    0x0223
+#define regSDMA0_QUEUE4_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_MIDCMD_DATA4                                                                    0x0224
+#define regSDMA0_QUEUE4_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_MIDCMD_DATA5                                                                    0x0225
+#define regSDMA0_QUEUE4_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_MIDCMD_DATA6                                                                    0x0226
+#define regSDMA0_QUEUE4_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_MIDCMD_DATA7                                                                    0x0227
+#define regSDMA0_QUEUE4_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_MIDCMD_DATA8                                                                    0x0228
+#define regSDMA0_QUEUE4_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_MIDCMD_DATA9                                                                    0x0229
+#define regSDMA0_QUEUE4_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA0_QUEUE4_MIDCMD_DATA10                                                                   0x022a
+#define regSDMA0_QUEUE4_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA0_QUEUE4_MIDCMD_CNTL                                                                     0x022b
+#define regSDMA0_QUEUE4_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE5_RB_CNTL                                                                         0x0238
+#define regSDMA0_QUEUE5_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE5_RB_BASE                                                                         0x0239
+#define regSDMA0_QUEUE5_RB_BASE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE5_RB_BASE_HI                                                                      0x023a
+#define regSDMA0_QUEUE5_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE5_RB_RPTR                                                                         0x023b
+#define regSDMA0_QUEUE5_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE5_RB_RPTR_HI                                                                      0x023c
+#define regSDMA0_QUEUE5_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE5_RB_WPTR                                                                         0x023d
+#define regSDMA0_QUEUE5_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE5_RB_WPTR_HI                                                                      0x023e
+#define regSDMA0_QUEUE5_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE5_RB_RPTR_ADDR_HI                                                                 0x0240
+#define regSDMA0_QUEUE5_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA0_QUEUE5_RB_RPTR_ADDR_LO                                                                 0x0241
+#define regSDMA0_QUEUE5_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA0_QUEUE5_IB_CNTL                                                                         0x0242
+#define regSDMA0_QUEUE5_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE5_IB_RPTR                                                                         0x0243
+#define regSDMA0_QUEUE5_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE5_IB_OFFSET                                                                       0x0244
+#define regSDMA0_QUEUE5_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA0_QUEUE5_IB_BASE_LO                                                                      0x0245
+#define regSDMA0_QUEUE5_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA0_QUEUE5_IB_BASE_HI                                                                      0x0246
+#define regSDMA0_QUEUE5_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE5_IB_SIZE                                                                         0x0247
+#define regSDMA0_QUEUE5_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE5_SKIP_CNTL                                                                       0x0248
+#define regSDMA0_QUEUE5_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA0_QUEUE5_CONTEXT_STATUS                                                                  0x0249
+#define regSDMA0_QUEUE5_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA0_QUEUE5_DOORBELL                                                                        0x024a
+#define regSDMA0_QUEUE5_DOORBELL_BASE_IDX                                                               0
+#define regSDMA0_QUEUE5_DOORBELL_LOG                                                                    0x0261
+#define regSDMA0_QUEUE5_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_DOORBELL_OFFSET                                                                 0x0263
+#define regSDMA0_QUEUE5_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA0_QUEUE5_CSA_ADDR_LO                                                                     0x0264
+#define regSDMA0_QUEUE5_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA0_QUEUE5_CSA_ADDR_HI                                                                     0x0265
+#define regSDMA0_QUEUE5_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA0_QUEUE5_SCHEDULE_CNTL                                                                   0x0266
+#define regSDMA0_QUEUE5_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA0_QUEUE5_IB_SUB_REMAIN                                                                   0x0267
+#define regSDMA0_QUEUE5_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA0_QUEUE5_PREEMPT                                                                         0x0268
+#define regSDMA0_QUEUE5_PREEMPT_BASE_IDX                                                                0
+#define regSDMA0_QUEUE5_DUMMY_REG                                                                       0x0269
+#define regSDMA0_QUEUE5_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA0_QUEUE5_RB_WPTR_POLL_ADDR_HI                                                            0x026a
+#define regSDMA0_QUEUE5_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_QUEUE5_RB_WPTR_POLL_ADDR_LO                                                            0x026b
+#define regSDMA0_QUEUE5_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_QUEUE5_RB_AQL_CNTL                                                                     0x026c
+#define regSDMA0_QUEUE5_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE5_MINOR_PTR_UPDATE                                                                0x026d
+#define regSDMA0_QUEUE5_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA0_QUEUE5_RB_PREEMPT                                                                      0x026e
+#define regSDMA0_QUEUE5_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA0_QUEUE5_MIDCMD_DATA0                                                                    0x0278
+#define regSDMA0_QUEUE5_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_MIDCMD_DATA1                                                                    0x0279
+#define regSDMA0_QUEUE5_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_MIDCMD_DATA2                                                                    0x027a
+#define regSDMA0_QUEUE5_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_MIDCMD_DATA3                                                                    0x027b
+#define regSDMA0_QUEUE5_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_MIDCMD_DATA4                                                                    0x027c
+#define regSDMA0_QUEUE5_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_MIDCMD_DATA5                                                                    0x027d
+#define regSDMA0_QUEUE5_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_MIDCMD_DATA6                                                                    0x027e
+#define regSDMA0_QUEUE5_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_MIDCMD_DATA7                                                                    0x027f
+#define regSDMA0_QUEUE5_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_MIDCMD_DATA8                                                                    0x0280
+#define regSDMA0_QUEUE5_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_MIDCMD_DATA9                                                                    0x0281
+#define regSDMA0_QUEUE5_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA0_QUEUE5_MIDCMD_DATA10                                                                   0x0282
+#define regSDMA0_QUEUE5_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA0_QUEUE5_MIDCMD_CNTL                                                                     0x0283
+#define regSDMA0_QUEUE5_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE6_RB_CNTL                                                                         0x0290
+#define regSDMA0_QUEUE6_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE6_RB_BASE                                                                         0x0291
+#define regSDMA0_QUEUE6_RB_BASE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE6_RB_BASE_HI                                                                      0x0292
+#define regSDMA0_QUEUE6_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE6_RB_RPTR                                                                         0x0293
+#define regSDMA0_QUEUE6_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE6_RB_RPTR_HI                                                                      0x0294
+#define regSDMA0_QUEUE6_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE6_RB_WPTR                                                                         0x0295
+#define regSDMA0_QUEUE6_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE6_RB_WPTR_HI                                                                      0x0296
+#define regSDMA0_QUEUE6_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE6_RB_RPTR_ADDR_HI                                                                 0x0298
+#define regSDMA0_QUEUE6_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA0_QUEUE6_RB_RPTR_ADDR_LO                                                                 0x0299
+#define regSDMA0_QUEUE6_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA0_QUEUE6_IB_CNTL                                                                         0x029a
+#define regSDMA0_QUEUE6_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE6_IB_RPTR                                                                         0x029b
+#define regSDMA0_QUEUE6_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE6_IB_OFFSET                                                                       0x029c
+#define regSDMA0_QUEUE6_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA0_QUEUE6_IB_BASE_LO                                                                      0x029d
+#define regSDMA0_QUEUE6_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA0_QUEUE6_IB_BASE_HI                                                                      0x029e
+#define regSDMA0_QUEUE6_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE6_IB_SIZE                                                                         0x029f
+#define regSDMA0_QUEUE6_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE6_SKIP_CNTL                                                                       0x02a0
+#define regSDMA0_QUEUE6_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA0_QUEUE6_CONTEXT_STATUS                                                                  0x02a1
+#define regSDMA0_QUEUE6_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA0_QUEUE6_DOORBELL                                                                        0x02a2
+#define regSDMA0_QUEUE6_DOORBELL_BASE_IDX                                                               0
+#define regSDMA0_QUEUE6_DOORBELL_LOG                                                                    0x02b9
+#define regSDMA0_QUEUE6_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_DOORBELL_OFFSET                                                                 0x02bb
+#define regSDMA0_QUEUE6_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA0_QUEUE6_CSA_ADDR_LO                                                                     0x02bc
+#define regSDMA0_QUEUE6_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA0_QUEUE6_CSA_ADDR_HI                                                                     0x02bd
+#define regSDMA0_QUEUE6_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA0_QUEUE6_SCHEDULE_CNTL                                                                   0x02be
+#define regSDMA0_QUEUE6_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA0_QUEUE6_IB_SUB_REMAIN                                                                   0x02bf
+#define regSDMA0_QUEUE6_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA0_QUEUE6_PREEMPT                                                                         0x02c0
+#define regSDMA0_QUEUE6_PREEMPT_BASE_IDX                                                                0
+#define regSDMA0_QUEUE6_DUMMY_REG                                                                       0x02c1
+#define regSDMA0_QUEUE6_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA0_QUEUE6_RB_WPTR_POLL_ADDR_HI                                                            0x02c2
+#define regSDMA0_QUEUE6_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_QUEUE6_RB_WPTR_POLL_ADDR_LO                                                            0x02c3
+#define regSDMA0_QUEUE6_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_QUEUE6_RB_AQL_CNTL                                                                     0x02c4
+#define regSDMA0_QUEUE6_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE6_MINOR_PTR_UPDATE                                                                0x02c5
+#define regSDMA0_QUEUE6_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA0_QUEUE6_RB_PREEMPT                                                                      0x02c6
+#define regSDMA0_QUEUE6_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA0_QUEUE6_MIDCMD_DATA0                                                                    0x02d0
+#define regSDMA0_QUEUE6_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_MIDCMD_DATA1                                                                    0x02d1
+#define regSDMA0_QUEUE6_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_MIDCMD_DATA2                                                                    0x02d2
+#define regSDMA0_QUEUE6_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_MIDCMD_DATA3                                                                    0x02d3
+#define regSDMA0_QUEUE6_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_MIDCMD_DATA4                                                                    0x02d4
+#define regSDMA0_QUEUE6_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_MIDCMD_DATA5                                                                    0x02d5
+#define regSDMA0_QUEUE6_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_MIDCMD_DATA6                                                                    0x02d6
+#define regSDMA0_QUEUE6_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_MIDCMD_DATA7                                                                    0x02d7
+#define regSDMA0_QUEUE6_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_MIDCMD_DATA8                                                                    0x02d8
+#define regSDMA0_QUEUE6_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_MIDCMD_DATA9                                                                    0x02d9
+#define regSDMA0_QUEUE6_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA0_QUEUE6_MIDCMD_DATA10                                                                   0x02da
+#define regSDMA0_QUEUE6_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA0_QUEUE6_MIDCMD_CNTL                                                                     0x02db
+#define regSDMA0_QUEUE6_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE7_RB_CNTL                                                                         0x02e8
+#define regSDMA0_QUEUE7_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE7_RB_BASE                                                                         0x02e9
+#define regSDMA0_QUEUE7_RB_BASE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE7_RB_BASE_HI                                                                      0x02ea
+#define regSDMA0_QUEUE7_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE7_RB_RPTR                                                                         0x02eb
+#define regSDMA0_QUEUE7_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE7_RB_RPTR_HI                                                                      0x02ec
+#define regSDMA0_QUEUE7_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE7_RB_WPTR                                                                         0x02ed
+#define regSDMA0_QUEUE7_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE7_RB_WPTR_HI                                                                      0x02ee
+#define regSDMA0_QUEUE7_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE7_RB_RPTR_ADDR_HI                                                                 0x02f0
+#define regSDMA0_QUEUE7_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA0_QUEUE7_RB_RPTR_ADDR_LO                                                                 0x02f1
+#define regSDMA0_QUEUE7_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA0_QUEUE7_IB_CNTL                                                                         0x02f2
+#define regSDMA0_QUEUE7_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA0_QUEUE7_IB_RPTR                                                                         0x02f3
+#define regSDMA0_QUEUE7_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA0_QUEUE7_IB_OFFSET                                                                       0x02f4
+#define regSDMA0_QUEUE7_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA0_QUEUE7_IB_BASE_LO                                                                      0x02f5
+#define regSDMA0_QUEUE7_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA0_QUEUE7_IB_BASE_HI                                                                      0x02f6
+#define regSDMA0_QUEUE7_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA0_QUEUE7_IB_SIZE                                                                         0x02f7
+#define regSDMA0_QUEUE7_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA0_QUEUE7_SKIP_CNTL                                                                       0x02f8
+#define regSDMA0_QUEUE7_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA0_QUEUE7_CONTEXT_STATUS                                                                  0x02f9
+#define regSDMA0_QUEUE7_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA0_QUEUE7_DOORBELL                                                                        0x02fa
+#define regSDMA0_QUEUE7_DOORBELL_BASE_IDX                                                               0
+#define regSDMA0_QUEUE7_DOORBELL_LOG                                                                    0x0311
+#define regSDMA0_QUEUE7_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_DOORBELL_OFFSET                                                                 0x0313
+#define regSDMA0_QUEUE7_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA0_QUEUE7_CSA_ADDR_LO                                                                     0x0314
+#define regSDMA0_QUEUE7_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA0_QUEUE7_CSA_ADDR_HI                                                                     0x0315
+#define regSDMA0_QUEUE7_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA0_QUEUE7_SCHEDULE_CNTL                                                                   0x0316
+#define regSDMA0_QUEUE7_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA0_QUEUE7_IB_SUB_REMAIN                                                                   0x0317
+#define regSDMA0_QUEUE7_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA0_QUEUE7_PREEMPT                                                                         0x0318
+#define regSDMA0_QUEUE7_PREEMPT_BASE_IDX                                                                0
+#define regSDMA0_QUEUE7_DUMMY_REG                                                                       0x0319
+#define regSDMA0_QUEUE7_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA0_QUEUE7_RB_WPTR_POLL_ADDR_HI                                                            0x031a
+#define regSDMA0_QUEUE7_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA0_QUEUE7_RB_WPTR_POLL_ADDR_LO                                                            0x031b
+#define regSDMA0_QUEUE7_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA0_QUEUE7_RB_AQL_CNTL                                                                     0x031c
+#define regSDMA0_QUEUE7_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA0_QUEUE7_MINOR_PTR_UPDATE                                                                0x031d
+#define regSDMA0_QUEUE7_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA0_QUEUE7_RB_PREEMPT                                                                      0x031e
+#define regSDMA0_QUEUE7_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA0_QUEUE7_MIDCMD_DATA0                                                                    0x0328
+#define regSDMA0_QUEUE7_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_MIDCMD_DATA1                                                                    0x0329
+#define regSDMA0_QUEUE7_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_MIDCMD_DATA2                                                                    0x032a
+#define regSDMA0_QUEUE7_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_MIDCMD_DATA3                                                                    0x032b
+#define regSDMA0_QUEUE7_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_MIDCMD_DATA4                                                                    0x032c
+#define regSDMA0_QUEUE7_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_MIDCMD_DATA5                                                                    0x032d
+#define regSDMA0_QUEUE7_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_MIDCMD_DATA6                                                                    0x032e
+#define regSDMA0_QUEUE7_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_MIDCMD_DATA7                                                                    0x032f
+#define regSDMA0_QUEUE7_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_MIDCMD_DATA8                                                                    0x0330
+#define regSDMA0_QUEUE7_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_MIDCMD_DATA9                                                                    0x0331
+#define regSDMA0_QUEUE7_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA0_QUEUE7_MIDCMD_DATA10                                                                   0x0332
+#define regSDMA0_QUEUE7_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA0_QUEUE7_MIDCMD_CNTL                                                                     0x0333
+#define regSDMA0_QUEUE7_MIDCMD_CNTL_BASE_IDX                                                            0
+
+
+// addressBlock: gc_sdma0_sdma1dec
+// base address: 0x6180
+#define regSDMA1_DEC_START                                                                              0x0600
+#define regSDMA1_DEC_START_BASE_IDX                                                                     0
+#define regSDMA1_F32_MISC_CNTL                                                                          0x060b
+#define regSDMA1_F32_MISC_CNTL_BASE_IDX                                                                 0
+#define regSDMA1_GLOBAL_TIMESTAMP_LO                                                                    0x060f
+#define regSDMA1_GLOBAL_TIMESTAMP_LO_BASE_IDX                                                           0
+#define regSDMA1_GLOBAL_TIMESTAMP_HI                                                                    0x0610
+#define regSDMA1_GLOBAL_TIMESTAMP_HI_BASE_IDX                                                           0
+#define regSDMA1_POWER_CNTL                                                                             0x061a
+#define regSDMA1_POWER_CNTL_BASE_IDX                                                                    0
+#define regSDMA1_CNTL                                                                                   0x061c
+#define regSDMA1_CNTL_BASE_IDX                                                                          0
+#define regSDMA1_CHICKEN_BITS                                                                           0x061d
+#define regSDMA1_CHICKEN_BITS_BASE_IDX                                                                  0
+#define regSDMA1_GB_ADDR_CONFIG                                                                         0x061e
+#define regSDMA1_GB_ADDR_CONFIG_BASE_IDX                                                                0
+#define regSDMA1_GB_ADDR_CONFIG_READ                                                                    0x061f
+#define regSDMA1_GB_ADDR_CONFIG_READ_BASE_IDX                                                           0
+#define regSDMA1_RB_RPTR_FETCH                                                                          0x0620
+#define regSDMA1_RB_RPTR_FETCH_BASE_IDX                                                                 0
+#define regSDMA1_RB_RPTR_FETCH_HI                                                                       0x0621
+#define regSDMA1_RB_RPTR_FETCH_HI_BASE_IDX                                                              0
+#define regSDMA1_SEM_WAIT_FAIL_TIMER_CNTL                                                               0x0622
+#define regSDMA1_SEM_WAIT_FAIL_TIMER_CNTL_BASE_IDX                                                      0
+#define regSDMA1_IB_OFFSET_FETCH                                                                        0x0623
+#define regSDMA1_IB_OFFSET_FETCH_BASE_IDX                                                               0
+#define regSDMA1_PROGRAM                                                                                0x0624
+#define regSDMA1_PROGRAM_BASE_IDX                                                                       0
+#define regSDMA1_STATUS_REG                                                                             0x0625
+#define regSDMA1_STATUS_REG_BASE_IDX                                                                    0
+#define regSDMA1_STATUS1_REG                                                                            0x0626
+#define regSDMA1_STATUS1_REG_BASE_IDX                                                                   0
+#define regSDMA1_CNTL1                                                                                  0x0627
+#define regSDMA1_CNTL1_BASE_IDX                                                                         0
+#define regSDMA1_HBM_PAGE_CONFIG                                                                        0x0628
+#define regSDMA1_HBM_PAGE_CONFIG_BASE_IDX                                                               0
+#define regSDMA1_UCODE_CHECKSUM                                                                         0x0629
+#define regSDMA1_UCODE_CHECKSUM_BASE_IDX                                                                0
+#define regSDMA1_FREEZE                                                                                 0x062b
+#define regSDMA1_FREEZE_BASE_IDX                                                                        0
+#define regSDMA1_PROCESS_QUANTUM0                                                                       0x062c
+#define regSDMA1_PROCESS_QUANTUM0_BASE_IDX                                                              0
+#define regSDMA1_PROCESS_QUANTUM1                                                                       0x062d
+#define regSDMA1_PROCESS_QUANTUM1_BASE_IDX                                                              0
+#define regSDMA1_WATCHDOG_CNTL                                                                          0x062e
+#define regSDMA1_WATCHDOG_CNTL_BASE_IDX                                                                 0
+#define regSDMA1_QUEUE_STATUS0                                                                          0x062f
+#define regSDMA1_QUEUE_STATUS0_BASE_IDX                                                                 0
+#define regSDMA1_EDC_CONFIG                                                                             0x0632
+#define regSDMA1_EDC_CONFIG_BASE_IDX                                                                    0
+#define regSDMA1_BA_THRESHOLD                                                                           0x0633
+#define regSDMA1_BA_THRESHOLD_BASE_IDX                                                                  0
+#define regSDMA1_ID                                                                                     0x0634
+#define regSDMA1_ID_BASE_IDX                                                                            0
+#define regSDMA1_VERSION                                                                                0x0635
+#define regSDMA1_VERSION_BASE_IDX                                                                       0
+#define regSDMA1_EDC_COUNTER                                                                            0x0636
+#define regSDMA1_EDC_COUNTER_BASE_IDX                                                                   0
+#define regSDMA1_EDC_COUNTER_CLEAR                                                                      0x0637
+#define regSDMA1_EDC_COUNTER_CLEAR_BASE_IDX                                                             0
+#define regSDMA1_STATUS2_REG                                                                            0x0638
+#define regSDMA1_STATUS2_REG_BASE_IDX                                                                   0
+#define regSDMA1_ATOMIC_CNTL                                                                            0x0639
+#define regSDMA1_ATOMIC_CNTL_BASE_IDX                                                                   0
+#define regSDMA1_ATOMIC_PREOP_LO                                                                        0x063a
+#define regSDMA1_ATOMIC_PREOP_LO_BASE_IDX                                                               0
+#define regSDMA1_ATOMIC_PREOP_HI                                                                        0x063b
+#define regSDMA1_ATOMIC_PREOP_HI_BASE_IDX                                                               0
+#define regSDMA1_UTCL1_CNTL                                                                             0x063c
+#define regSDMA1_UTCL1_CNTL_BASE_IDX                                                                    0
+#define regSDMA1_UTCL1_WATERMK                                                                          0x063d
+#define regSDMA1_UTCL1_WATERMK_BASE_IDX                                                                 0
+#define regSDMA1_UTCL1_TIMEOUT                                                                          0x063e
+#define regSDMA1_UTCL1_TIMEOUT_BASE_IDX                                                                 0
+#define regSDMA1_UTCL1_PAGE                                                                             0x063f
+#define regSDMA1_UTCL1_PAGE_BASE_IDX                                                                    0
+#define regSDMA1_UTCL1_RD_STATUS                                                                        0x0640
+#define regSDMA1_UTCL1_RD_STATUS_BASE_IDX                                                               0
+#define regSDMA1_UTCL1_WR_STATUS                                                                        0x0641
+#define regSDMA1_UTCL1_WR_STATUS_BASE_IDX                                                               0
+#define regSDMA1_UTCL1_INV0                                                                             0x0642
+#define regSDMA1_UTCL1_INV0_BASE_IDX                                                                    0
+#define regSDMA1_UTCL1_INV1                                                                             0x0643
+#define regSDMA1_UTCL1_INV1_BASE_IDX                                                                    0
+#define regSDMA1_UTCL1_INV2                                                                             0x0644
+#define regSDMA1_UTCL1_INV2_BASE_IDX                                                                    0
+#define regSDMA1_UTCL1_RD_XNACK0                                                                        0x0645
+#define regSDMA1_UTCL1_RD_XNACK0_BASE_IDX                                                               0
+#define regSDMA1_UTCL1_RD_XNACK1                                                                        0x0646
+#define regSDMA1_UTCL1_RD_XNACK1_BASE_IDX                                                               0
+#define regSDMA1_UTCL1_WR_XNACK0                                                                        0x0647
+#define regSDMA1_UTCL1_WR_XNACK0_BASE_IDX                                                               0
+#define regSDMA1_UTCL1_WR_XNACK1                                                                        0x0648
+#define regSDMA1_UTCL1_WR_XNACK1_BASE_IDX                                                               0
+#define regSDMA1_RELAX_ORDERING_LUT                                                                     0x064a
+#define regSDMA1_RELAX_ORDERING_LUT_BASE_IDX                                                            0
+#define regSDMA1_CHICKEN_BITS_2                                                                         0x064b
+#define regSDMA1_CHICKEN_BITS_2_BASE_IDX                                                                0
+#define regSDMA1_STATUS3_REG                                                                            0x064c
+#define regSDMA1_STATUS3_REG_BASE_IDX                                                                   0
+#define regSDMA1_PHYSICAL_ADDR_LO                                                                       0x064d
+#define regSDMA1_PHYSICAL_ADDR_LO_BASE_IDX                                                              0
+#define regSDMA1_PHYSICAL_ADDR_HI                                                                       0x064e
+#define regSDMA1_PHYSICAL_ADDR_HI_BASE_IDX                                                              0
+#define regSDMA1_GLOBAL_QUANTUM                                                                         0x064f
+#define regSDMA1_GLOBAL_QUANTUM_BASE_IDX                                                                0
+#define regSDMA1_ERROR_LOG                                                                              0x0650
+#define regSDMA1_ERROR_LOG_BASE_IDX                                                                     0
+#define regSDMA1_PUB_DUMMY_REG0                                                                         0x0651
+#define regSDMA1_PUB_DUMMY_REG0_BASE_IDX                                                                0
+#define regSDMA1_PUB_DUMMY_REG1                                                                         0x0652
+#define regSDMA1_PUB_DUMMY_REG1_BASE_IDX                                                                0
+#define regSDMA1_PUB_DUMMY_REG2                                                                         0x0653
+#define regSDMA1_PUB_DUMMY_REG2_BASE_IDX                                                                0
+#define regSDMA1_PUB_DUMMY_REG3                                                                         0x0654
+#define regSDMA1_PUB_DUMMY_REG3_BASE_IDX                                                                0
+#define regSDMA1_F32_COUNTER                                                                            0x0655
+#define regSDMA1_F32_COUNTER_BASE_IDX                                                                   0
+#define regSDMA1_CRD_CNTL                                                                               0x065b
+#define regSDMA1_CRD_CNTL_BASE_IDX                                                                      0
+#define regSDMA1_RLC_CGCG_CTRL                                                                          0x065c
+#define regSDMA1_RLC_CGCG_CTRL_BASE_IDX                                                                 0
+#define regSDMA1_GPU_IOV_VIOLATION_LOG                                                                  0x065d
+#define regSDMA1_GPU_IOV_VIOLATION_LOG_BASE_IDX                                                         0
+#define regSDMA1_AQL_STATUS                                                                             0x065f
+#define regSDMA1_AQL_STATUS_BASE_IDX                                                                    0
+#define regSDMA1_EA_DBIT_ADDR_DATA                                                                      0x0660
+#define regSDMA1_EA_DBIT_ADDR_DATA_BASE_IDX                                                             0
+#define regSDMA1_EA_DBIT_ADDR_INDEX                                                                     0x0661
+#define regSDMA1_EA_DBIT_ADDR_INDEX_BASE_IDX                                                            0
+#define regSDMA1_TLBI_GCR_CNTL                                                                          0x0662
+#define regSDMA1_TLBI_GCR_CNTL_BASE_IDX                                                                 0
+#define regSDMA1_TILING_CONFIG                                                                          0x0663
+#define regSDMA1_TILING_CONFIG_BASE_IDX                                                                 0
+#define regSDMA1_HASH                                                                                   0x0664
+#define regSDMA1_HASH_BASE_IDX                                                                          0
+#define regSDMA1_INT_STATUS                                                                             0x0670
+#define regSDMA1_INT_STATUS_BASE_IDX                                                                    0
+#define regSDMA1_GPU_IOV_VIOLATION_LOG2                                                                 0x0671
+#define regSDMA1_GPU_IOV_VIOLATION_LOG2_BASE_IDX                                                        0
+#define regSDMA1_HOLE_ADDR_LO                                                                           0x0672
+#define regSDMA1_HOLE_ADDR_LO_BASE_IDX                                                                  0
+#define regSDMA1_HOLE_ADDR_HI                                                                           0x0673
+#define regSDMA1_HOLE_ADDR_HI_BASE_IDX                                                                  0
+#define regSDMA1_CLOCK_GATING_STATUS                                                                    0x0675
+#define regSDMA1_CLOCK_GATING_STATUS_BASE_IDX                                                           0
+#define regSDMA1_STATUS4_REG                                                                            0x0676
+#define regSDMA1_STATUS4_REG_BASE_IDX                                                                   0
+#define regSDMA1_SCRATCH_RAM_DATA                                                                       0x0677
+#define regSDMA1_SCRATCH_RAM_DATA_BASE_IDX                                                              0
+#define regSDMA1_SCRATCH_RAM_ADDR                                                                       0x0678
+#define regSDMA1_SCRATCH_RAM_ADDR_BASE_IDX                                                              0
+#define regSDMA1_TIMESTAMP_CNTL                                                                         0x0679
+#define regSDMA1_TIMESTAMP_CNTL_BASE_IDX                                                                0
+#define regSDMA1_STATUS5_REG                                                                            0x067a
+#define regSDMA1_STATUS5_REG_BASE_IDX                                                                   0
+#define regSDMA1_QUEUE_RESET_REQ                                                                        0x067b
+#define regSDMA1_QUEUE_RESET_REQ_BASE_IDX                                                               0
+#define regSDMA1_STATUS6_REG                                                                            0x067c
+#define regSDMA1_STATUS6_REG_BASE_IDX                                                                   0
+#define regSDMA1_UCODE1_CHECKSUM                                                                        0x067d
+#define regSDMA1_UCODE1_CHECKSUM_BASE_IDX                                                               0
+#define regSDMA1_CE_CTRL                                                                                0x067e
+#define regSDMA1_CE_CTRL_BASE_IDX                                                                       0
+#define regSDMA1_FED_STATUS                                                                             0x067f
+#define regSDMA1_FED_STATUS_BASE_IDX                                                                    0
+#define regSDMA1_QUEUE0_RB_CNTL                                                                         0x0680
+#define regSDMA1_QUEUE0_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE0_RB_BASE                                                                         0x0681
+#define regSDMA1_QUEUE0_RB_BASE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE0_RB_BASE_HI                                                                      0x0682
+#define regSDMA1_QUEUE0_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE0_RB_RPTR                                                                         0x0683
+#define regSDMA1_QUEUE0_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE0_RB_RPTR_HI                                                                      0x0684
+#define regSDMA1_QUEUE0_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE0_RB_WPTR                                                                         0x0685
+#define regSDMA1_QUEUE0_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE0_RB_WPTR_HI                                                                      0x0686
+#define regSDMA1_QUEUE0_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE0_RB_RPTR_ADDR_HI                                                                 0x0688
+#define regSDMA1_QUEUE0_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA1_QUEUE0_RB_RPTR_ADDR_LO                                                                 0x0689
+#define regSDMA1_QUEUE0_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA1_QUEUE0_IB_CNTL                                                                         0x068a
+#define regSDMA1_QUEUE0_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE0_IB_RPTR                                                                         0x068b
+#define regSDMA1_QUEUE0_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE0_IB_OFFSET                                                                       0x068c
+#define regSDMA1_QUEUE0_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA1_QUEUE0_IB_BASE_LO                                                                      0x068d
+#define regSDMA1_QUEUE0_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA1_QUEUE0_IB_BASE_HI                                                                      0x068e
+#define regSDMA1_QUEUE0_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE0_IB_SIZE                                                                         0x068f
+#define regSDMA1_QUEUE0_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE0_SKIP_CNTL                                                                       0x0690
+#define regSDMA1_QUEUE0_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA1_QUEUE0_CONTEXT_STATUS                                                                  0x0691
+#define regSDMA1_QUEUE0_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA1_QUEUE0_DOORBELL                                                                        0x0692
+#define regSDMA1_QUEUE0_DOORBELL_BASE_IDX                                                               0
+#define regSDMA1_QUEUE0_DOORBELL_LOG                                                                    0x06a9
+#define regSDMA1_QUEUE0_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_DOORBELL_OFFSET                                                                 0x06ab
+#define regSDMA1_QUEUE0_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA1_QUEUE0_CSA_ADDR_LO                                                                     0x06ac
+#define regSDMA1_QUEUE0_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA1_QUEUE0_CSA_ADDR_HI                                                                     0x06ad
+#define regSDMA1_QUEUE0_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA1_QUEUE0_SCHEDULE_CNTL                                                                   0x06ae
+#define regSDMA1_QUEUE0_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA1_QUEUE0_IB_SUB_REMAIN                                                                   0x06af
+#define regSDMA1_QUEUE0_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA1_QUEUE0_PREEMPT                                                                         0x06b0
+#define regSDMA1_QUEUE0_PREEMPT_BASE_IDX                                                                0
+#define regSDMA1_QUEUE0_DUMMY_REG                                                                       0x06b1
+#define regSDMA1_QUEUE0_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA1_QUEUE0_RB_WPTR_POLL_ADDR_HI                                                            0x06b2
+#define regSDMA1_QUEUE0_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_QUEUE0_RB_WPTR_POLL_ADDR_LO                                                            0x06b3
+#define regSDMA1_QUEUE0_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_QUEUE0_RB_AQL_CNTL                                                                     0x06b4
+#define regSDMA1_QUEUE0_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE0_MINOR_PTR_UPDATE                                                                0x06b5
+#define regSDMA1_QUEUE0_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA1_QUEUE0_RB_PREEMPT                                                                      0x06b6
+#define regSDMA1_QUEUE0_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA1_QUEUE0_MIDCMD_DATA0                                                                    0x06c0
+#define regSDMA1_QUEUE0_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_MIDCMD_DATA1                                                                    0x06c1
+#define regSDMA1_QUEUE0_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_MIDCMD_DATA2                                                                    0x06c2
+#define regSDMA1_QUEUE0_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_MIDCMD_DATA3                                                                    0x06c3
+#define regSDMA1_QUEUE0_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_MIDCMD_DATA4                                                                    0x06c4
+#define regSDMA1_QUEUE0_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_MIDCMD_DATA5                                                                    0x06c5
+#define regSDMA1_QUEUE0_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_MIDCMD_DATA6                                                                    0x06c6
+#define regSDMA1_QUEUE0_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_MIDCMD_DATA7                                                                    0x06c7
+#define regSDMA1_QUEUE0_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_MIDCMD_DATA8                                                                    0x06c8
+#define regSDMA1_QUEUE0_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_MIDCMD_DATA9                                                                    0x06c9
+#define regSDMA1_QUEUE0_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA1_QUEUE0_MIDCMD_DATA10                                                                   0x06ca
+#define regSDMA1_QUEUE0_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA1_QUEUE0_MIDCMD_CNTL                                                                     0x06cb
+#define regSDMA1_QUEUE0_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE1_RB_CNTL                                                                         0x06d8
+#define regSDMA1_QUEUE1_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE1_RB_BASE                                                                         0x06d9
+#define regSDMA1_QUEUE1_RB_BASE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE1_RB_BASE_HI                                                                      0x06da
+#define regSDMA1_QUEUE1_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE1_RB_RPTR                                                                         0x06db
+#define regSDMA1_QUEUE1_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE1_RB_RPTR_HI                                                                      0x06dc
+#define regSDMA1_QUEUE1_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE1_RB_WPTR                                                                         0x06dd
+#define regSDMA1_QUEUE1_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE1_RB_WPTR_HI                                                                      0x06de
+#define regSDMA1_QUEUE1_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE1_RB_RPTR_ADDR_HI                                                                 0x06e0
+#define regSDMA1_QUEUE1_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA1_QUEUE1_RB_RPTR_ADDR_LO                                                                 0x06e1
+#define regSDMA1_QUEUE1_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA1_QUEUE1_IB_CNTL                                                                         0x06e2
+#define regSDMA1_QUEUE1_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE1_IB_RPTR                                                                         0x06e3
+#define regSDMA1_QUEUE1_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE1_IB_OFFSET                                                                       0x06e4
+#define regSDMA1_QUEUE1_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA1_QUEUE1_IB_BASE_LO                                                                      0x06e5
+#define regSDMA1_QUEUE1_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA1_QUEUE1_IB_BASE_HI                                                                      0x06e6
+#define regSDMA1_QUEUE1_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE1_IB_SIZE                                                                         0x06e7
+#define regSDMA1_QUEUE1_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE1_SKIP_CNTL                                                                       0x06e8
+#define regSDMA1_QUEUE1_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA1_QUEUE1_CONTEXT_STATUS                                                                  0x06e9
+#define regSDMA1_QUEUE1_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA1_QUEUE1_DOORBELL                                                                        0x06ea
+#define regSDMA1_QUEUE1_DOORBELL_BASE_IDX                                                               0
+#define regSDMA1_QUEUE1_DOORBELL_LOG                                                                    0x0701
+#define regSDMA1_QUEUE1_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_DOORBELL_OFFSET                                                                 0x0703
+#define regSDMA1_QUEUE1_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA1_QUEUE1_CSA_ADDR_LO                                                                     0x0704
+#define regSDMA1_QUEUE1_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA1_QUEUE1_CSA_ADDR_HI                                                                     0x0705
+#define regSDMA1_QUEUE1_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA1_QUEUE1_SCHEDULE_CNTL                                                                   0x0706
+#define regSDMA1_QUEUE1_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA1_QUEUE1_IB_SUB_REMAIN                                                                   0x0707
+#define regSDMA1_QUEUE1_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA1_QUEUE1_PREEMPT                                                                         0x0708
+#define regSDMA1_QUEUE1_PREEMPT_BASE_IDX                                                                0
+#define regSDMA1_QUEUE1_DUMMY_REG                                                                       0x0709
+#define regSDMA1_QUEUE1_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA1_QUEUE1_RB_WPTR_POLL_ADDR_HI                                                            0x070a
+#define regSDMA1_QUEUE1_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_QUEUE1_RB_WPTR_POLL_ADDR_LO                                                            0x070b
+#define regSDMA1_QUEUE1_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_QUEUE1_RB_AQL_CNTL                                                                     0x070c
+#define regSDMA1_QUEUE1_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE1_MINOR_PTR_UPDATE                                                                0x070d
+#define regSDMA1_QUEUE1_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA1_QUEUE1_RB_PREEMPT                                                                      0x070e
+#define regSDMA1_QUEUE1_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA1_QUEUE1_MIDCMD_DATA0                                                                    0x0718
+#define regSDMA1_QUEUE1_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_MIDCMD_DATA1                                                                    0x0719
+#define regSDMA1_QUEUE1_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_MIDCMD_DATA2                                                                    0x071a
+#define regSDMA1_QUEUE1_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_MIDCMD_DATA3                                                                    0x071b
+#define regSDMA1_QUEUE1_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_MIDCMD_DATA4                                                                    0x071c
+#define regSDMA1_QUEUE1_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_MIDCMD_DATA5                                                                    0x071d
+#define regSDMA1_QUEUE1_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_MIDCMD_DATA6                                                                    0x071e
+#define regSDMA1_QUEUE1_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_MIDCMD_DATA7                                                                    0x071f
+#define regSDMA1_QUEUE1_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_MIDCMD_DATA8                                                                    0x0720
+#define regSDMA1_QUEUE1_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_MIDCMD_DATA9                                                                    0x0721
+#define regSDMA1_QUEUE1_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA1_QUEUE1_MIDCMD_DATA10                                                                   0x0722
+#define regSDMA1_QUEUE1_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA1_QUEUE1_MIDCMD_CNTL                                                                     0x0723
+#define regSDMA1_QUEUE1_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE2_RB_CNTL                                                                         0x0730
+#define regSDMA1_QUEUE2_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE2_RB_BASE                                                                         0x0731
+#define regSDMA1_QUEUE2_RB_BASE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE2_RB_BASE_HI                                                                      0x0732
+#define regSDMA1_QUEUE2_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE2_RB_RPTR                                                                         0x0733
+#define regSDMA1_QUEUE2_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE2_RB_RPTR_HI                                                                      0x0734
+#define regSDMA1_QUEUE2_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE2_RB_WPTR                                                                         0x0735
+#define regSDMA1_QUEUE2_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE2_RB_WPTR_HI                                                                      0x0736
+#define regSDMA1_QUEUE2_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE2_RB_RPTR_ADDR_HI                                                                 0x0738
+#define regSDMA1_QUEUE2_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA1_QUEUE2_RB_RPTR_ADDR_LO                                                                 0x0739
+#define regSDMA1_QUEUE2_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA1_QUEUE2_IB_CNTL                                                                         0x073a
+#define regSDMA1_QUEUE2_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE2_IB_RPTR                                                                         0x073b
+#define regSDMA1_QUEUE2_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE2_IB_OFFSET                                                                       0x073c
+#define regSDMA1_QUEUE2_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA1_QUEUE2_IB_BASE_LO                                                                      0x073d
+#define regSDMA1_QUEUE2_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA1_QUEUE2_IB_BASE_HI                                                                      0x073e
+#define regSDMA1_QUEUE2_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE2_IB_SIZE                                                                         0x073f
+#define regSDMA1_QUEUE2_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE2_SKIP_CNTL                                                                       0x0740
+#define regSDMA1_QUEUE2_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA1_QUEUE2_CONTEXT_STATUS                                                                  0x0741
+#define regSDMA1_QUEUE2_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA1_QUEUE2_DOORBELL                                                                        0x0742
+#define regSDMA1_QUEUE2_DOORBELL_BASE_IDX                                                               0
+#define regSDMA1_QUEUE2_DOORBELL_LOG                                                                    0x0759
+#define regSDMA1_QUEUE2_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_DOORBELL_OFFSET                                                                 0x075b
+#define regSDMA1_QUEUE2_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA1_QUEUE2_CSA_ADDR_LO                                                                     0x075c
+#define regSDMA1_QUEUE2_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA1_QUEUE2_CSA_ADDR_HI                                                                     0x075d
+#define regSDMA1_QUEUE2_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA1_QUEUE2_SCHEDULE_CNTL                                                                   0x075e
+#define regSDMA1_QUEUE2_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA1_QUEUE2_IB_SUB_REMAIN                                                                   0x075f
+#define regSDMA1_QUEUE2_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA1_QUEUE2_PREEMPT                                                                         0x0760
+#define regSDMA1_QUEUE2_PREEMPT_BASE_IDX                                                                0
+#define regSDMA1_QUEUE2_DUMMY_REG                                                                       0x0761
+#define regSDMA1_QUEUE2_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA1_QUEUE2_RB_WPTR_POLL_ADDR_HI                                                            0x0762
+#define regSDMA1_QUEUE2_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_QUEUE2_RB_WPTR_POLL_ADDR_LO                                                            0x0763
+#define regSDMA1_QUEUE2_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_QUEUE2_RB_AQL_CNTL                                                                     0x0764
+#define regSDMA1_QUEUE2_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE2_MINOR_PTR_UPDATE                                                                0x0765
+#define regSDMA1_QUEUE2_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA1_QUEUE2_RB_PREEMPT                                                                      0x0766
+#define regSDMA1_QUEUE2_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA1_QUEUE2_MIDCMD_DATA0                                                                    0x0770
+#define regSDMA1_QUEUE2_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_MIDCMD_DATA1                                                                    0x0771
+#define regSDMA1_QUEUE2_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_MIDCMD_DATA2                                                                    0x0772
+#define regSDMA1_QUEUE2_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_MIDCMD_DATA3                                                                    0x0773
+#define regSDMA1_QUEUE2_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_MIDCMD_DATA4                                                                    0x0774
+#define regSDMA1_QUEUE2_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_MIDCMD_DATA5                                                                    0x0775
+#define regSDMA1_QUEUE2_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_MIDCMD_DATA6                                                                    0x0776
+#define regSDMA1_QUEUE2_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_MIDCMD_DATA7                                                                    0x0777
+#define regSDMA1_QUEUE2_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_MIDCMD_DATA8                                                                    0x0778
+#define regSDMA1_QUEUE2_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_MIDCMD_DATA9                                                                    0x0779
+#define regSDMA1_QUEUE2_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA1_QUEUE2_MIDCMD_DATA10                                                                   0x077a
+#define regSDMA1_QUEUE2_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA1_QUEUE2_MIDCMD_CNTL                                                                     0x077b
+#define regSDMA1_QUEUE2_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE3_RB_CNTL                                                                         0x0788
+#define regSDMA1_QUEUE3_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE3_RB_BASE                                                                         0x0789
+#define regSDMA1_QUEUE3_RB_BASE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE3_RB_BASE_HI                                                                      0x078a
+#define regSDMA1_QUEUE3_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE3_RB_RPTR                                                                         0x078b
+#define regSDMA1_QUEUE3_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE3_RB_RPTR_HI                                                                      0x078c
+#define regSDMA1_QUEUE3_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE3_RB_WPTR                                                                         0x078d
+#define regSDMA1_QUEUE3_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE3_RB_WPTR_HI                                                                      0x078e
+#define regSDMA1_QUEUE3_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE3_RB_RPTR_ADDR_HI                                                                 0x0790
+#define regSDMA1_QUEUE3_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA1_QUEUE3_RB_RPTR_ADDR_LO                                                                 0x0791
+#define regSDMA1_QUEUE3_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA1_QUEUE3_IB_CNTL                                                                         0x0792
+#define regSDMA1_QUEUE3_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE3_IB_RPTR                                                                         0x0793
+#define regSDMA1_QUEUE3_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE3_IB_OFFSET                                                                       0x0794
+#define regSDMA1_QUEUE3_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA1_QUEUE3_IB_BASE_LO                                                                      0x0795
+#define regSDMA1_QUEUE3_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA1_QUEUE3_IB_BASE_HI                                                                      0x0796
+#define regSDMA1_QUEUE3_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE3_IB_SIZE                                                                         0x0797
+#define regSDMA1_QUEUE3_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE3_SKIP_CNTL                                                                       0x0798
+#define regSDMA1_QUEUE3_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA1_QUEUE3_CONTEXT_STATUS                                                                  0x0799
+#define regSDMA1_QUEUE3_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA1_QUEUE3_DOORBELL                                                                        0x079a
+#define regSDMA1_QUEUE3_DOORBELL_BASE_IDX                                                               0
+#define regSDMA1_QUEUE3_DOORBELL_LOG                                                                    0x07b1
+#define regSDMA1_QUEUE3_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_DOORBELL_OFFSET                                                                 0x07b3
+#define regSDMA1_QUEUE3_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA1_QUEUE3_CSA_ADDR_LO                                                                     0x07b4
+#define regSDMA1_QUEUE3_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA1_QUEUE3_CSA_ADDR_HI                                                                     0x07b5
+#define regSDMA1_QUEUE3_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA1_QUEUE3_SCHEDULE_CNTL                                                                   0x07b6
+#define regSDMA1_QUEUE3_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA1_QUEUE3_IB_SUB_REMAIN                                                                   0x07b7
+#define regSDMA1_QUEUE3_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA1_QUEUE3_PREEMPT                                                                         0x07b8
+#define regSDMA1_QUEUE3_PREEMPT_BASE_IDX                                                                0
+#define regSDMA1_QUEUE3_DUMMY_REG                                                                       0x07b9
+#define regSDMA1_QUEUE3_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA1_QUEUE3_RB_WPTR_POLL_ADDR_HI                                                            0x07ba
+#define regSDMA1_QUEUE3_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_QUEUE3_RB_WPTR_POLL_ADDR_LO                                                            0x07bb
+#define regSDMA1_QUEUE3_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_QUEUE3_RB_AQL_CNTL                                                                     0x07bc
+#define regSDMA1_QUEUE3_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE3_MINOR_PTR_UPDATE                                                                0x07bd
+#define regSDMA1_QUEUE3_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA1_QUEUE3_RB_PREEMPT                                                                      0x07be
+#define regSDMA1_QUEUE3_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA1_QUEUE3_MIDCMD_DATA0                                                                    0x07c8
+#define regSDMA1_QUEUE3_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_MIDCMD_DATA1                                                                    0x07c9
+#define regSDMA1_QUEUE3_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_MIDCMD_DATA2                                                                    0x07ca
+#define regSDMA1_QUEUE3_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_MIDCMD_DATA3                                                                    0x07cb
+#define regSDMA1_QUEUE3_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_MIDCMD_DATA4                                                                    0x07cc
+#define regSDMA1_QUEUE3_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_MIDCMD_DATA5                                                                    0x07cd
+#define regSDMA1_QUEUE3_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_MIDCMD_DATA6                                                                    0x07ce
+#define regSDMA1_QUEUE3_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_MIDCMD_DATA7                                                                    0x07cf
+#define regSDMA1_QUEUE3_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_MIDCMD_DATA8                                                                    0x07d0
+#define regSDMA1_QUEUE3_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_MIDCMD_DATA9                                                                    0x07d1
+#define regSDMA1_QUEUE3_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA1_QUEUE3_MIDCMD_DATA10                                                                   0x07d2
+#define regSDMA1_QUEUE3_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA1_QUEUE3_MIDCMD_CNTL                                                                     0x07d3
+#define regSDMA1_QUEUE3_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE4_RB_CNTL                                                                         0x07e0
+#define regSDMA1_QUEUE4_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE4_RB_BASE                                                                         0x07e1
+#define regSDMA1_QUEUE4_RB_BASE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE4_RB_BASE_HI                                                                      0x07e2
+#define regSDMA1_QUEUE4_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE4_RB_RPTR                                                                         0x07e3
+#define regSDMA1_QUEUE4_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE4_RB_RPTR_HI                                                                      0x07e4
+#define regSDMA1_QUEUE4_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE4_RB_WPTR                                                                         0x07e5
+#define regSDMA1_QUEUE4_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE4_RB_WPTR_HI                                                                      0x07e6
+#define regSDMA1_QUEUE4_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE4_RB_RPTR_ADDR_HI                                                                 0x07e8
+#define regSDMA1_QUEUE4_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA1_QUEUE4_RB_RPTR_ADDR_LO                                                                 0x07e9
+#define regSDMA1_QUEUE4_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA1_QUEUE4_IB_CNTL                                                                         0x07ea
+#define regSDMA1_QUEUE4_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE4_IB_RPTR                                                                         0x07eb
+#define regSDMA1_QUEUE4_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE4_IB_OFFSET                                                                       0x07ec
+#define regSDMA1_QUEUE4_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA1_QUEUE4_IB_BASE_LO                                                                      0x07ed
+#define regSDMA1_QUEUE4_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA1_QUEUE4_IB_BASE_HI                                                                      0x07ee
+#define regSDMA1_QUEUE4_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE4_IB_SIZE                                                                         0x07ef
+#define regSDMA1_QUEUE4_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE4_SKIP_CNTL                                                                       0x07f0
+#define regSDMA1_QUEUE4_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA1_QUEUE4_CONTEXT_STATUS                                                                  0x07f1
+#define regSDMA1_QUEUE4_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA1_QUEUE4_DOORBELL                                                                        0x07f2
+#define regSDMA1_QUEUE4_DOORBELL_BASE_IDX                                                               0
+#define regSDMA1_QUEUE4_DOORBELL_LOG                                                                    0x0809
+#define regSDMA1_QUEUE4_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_DOORBELL_OFFSET                                                                 0x080b
+#define regSDMA1_QUEUE4_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA1_QUEUE4_CSA_ADDR_LO                                                                     0x080c
+#define regSDMA1_QUEUE4_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA1_QUEUE4_CSA_ADDR_HI                                                                     0x080d
+#define regSDMA1_QUEUE4_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA1_QUEUE4_SCHEDULE_CNTL                                                                   0x080e
+#define regSDMA1_QUEUE4_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA1_QUEUE4_IB_SUB_REMAIN                                                                   0x080f
+#define regSDMA1_QUEUE4_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA1_QUEUE4_PREEMPT                                                                         0x0810
+#define regSDMA1_QUEUE4_PREEMPT_BASE_IDX                                                                0
+#define regSDMA1_QUEUE4_DUMMY_REG                                                                       0x0811
+#define regSDMA1_QUEUE4_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA1_QUEUE4_RB_WPTR_POLL_ADDR_HI                                                            0x0812
+#define regSDMA1_QUEUE4_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_QUEUE4_RB_WPTR_POLL_ADDR_LO                                                            0x0813
+#define regSDMA1_QUEUE4_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_QUEUE4_RB_AQL_CNTL                                                                     0x0814
+#define regSDMA1_QUEUE4_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE4_MINOR_PTR_UPDATE                                                                0x0815
+#define regSDMA1_QUEUE4_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA1_QUEUE4_RB_PREEMPT                                                                      0x0816
+#define regSDMA1_QUEUE4_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA1_QUEUE4_MIDCMD_DATA0                                                                    0x0820
+#define regSDMA1_QUEUE4_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_MIDCMD_DATA1                                                                    0x0821
+#define regSDMA1_QUEUE4_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_MIDCMD_DATA2                                                                    0x0822
+#define regSDMA1_QUEUE4_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_MIDCMD_DATA3                                                                    0x0823
+#define regSDMA1_QUEUE4_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_MIDCMD_DATA4                                                                    0x0824
+#define regSDMA1_QUEUE4_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_MIDCMD_DATA5                                                                    0x0825
+#define regSDMA1_QUEUE4_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_MIDCMD_DATA6                                                                    0x0826
+#define regSDMA1_QUEUE4_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_MIDCMD_DATA7                                                                    0x0827
+#define regSDMA1_QUEUE4_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_MIDCMD_DATA8                                                                    0x0828
+#define regSDMA1_QUEUE4_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_MIDCMD_DATA9                                                                    0x0829
+#define regSDMA1_QUEUE4_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA1_QUEUE4_MIDCMD_DATA10                                                                   0x082a
+#define regSDMA1_QUEUE4_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA1_QUEUE4_MIDCMD_CNTL                                                                     0x082b
+#define regSDMA1_QUEUE4_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE5_RB_CNTL                                                                         0x0838
+#define regSDMA1_QUEUE5_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE5_RB_BASE                                                                         0x0839
+#define regSDMA1_QUEUE5_RB_BASE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE5_RB_BASE_HI                                                                      0x083a
+#define regSDMA1_QUEUE5_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE5_RB_RPTR                                                                         0x083b
+#define regSDMA1_QUEUE5_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE5_RB_RPTR_HI                                                                      0x083c
+#define regSDMA1_QUEUE5_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE5_RB_WPTR                                                                         0x083d
+#define regSDMA1_QUEUE5_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE5_RB_WPTR_HI                                                                      0x083e
+#define regSDMA1_QUEUE5_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE5_RB_RPTR_ADDR_HI                                                                 0x0840
+#define regSDMA1_QUEUE5_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA1_QUEUE5_RB_RPTR_ADDR_LO                                                                 0x0841
+#define regSDMA1_QUEUE5_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA1_QUEUE5_IB_CNTL                                                                         0x0842
+#define regSDMA1_QUEUE5_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE5_IB_RPTR                                                                         0x0843
+#define regSDMA1_QUEUE5_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE5_IB_OFFSET                                                                       0x0844
+#define regSDMA1_QUEUE5_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA1_QUEUE5_IB_BASE_LO                                                                      0x0845
+#define regSDMA1_QUEUE5_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA1_QUEUE5_IB_BASE_HI                                                                      0x0846
+#define regSDMA1_QUEUE5_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE5_IB_SIZE                                                                         0x0847
+#define regSDMA1_QUEUE5_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE5_SKIP_CNTL                                                                       0x0848
+#define regSDMA1_QUEUE5_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA1_QUEUE5_CONTEXT_STATUS                                                                  0x0849
+#define regSDMA1_QUEUE5_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA1_QUEUE5_DOORBELL                                                                        0x084a
+#define regSDMA1_QUEUE5_DOORBELL_BASE_IDX                                                               0
+#define regSDMA1_QUEUE5_DOORBELL_LOG                                                                    0x0861
+#define regSDMA1_QUEUE5_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_DOORBELL_OFFSET                                                                 0x0863
+#define regSDMA1_QUEUE5_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA1_QUEUE5_CSA_ADDR_LO                                                                     0x0864
+#define regSDMA1_QUEUE5_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA1_QUEUE5_CSA_ADDR_HI                                                                     0x0865
+#define regSDMA1_QUEUE5_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA1_QUEUE5_SCHEDULE_CNTL                                                                   0x0866
+#define regSDMA1_QUEUE5_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA1_QUEUE5_IB_SUB_REMAIN                                                                   0x0867
+#define regSDMA1_QUEUE5_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA1_QUEUE5_PREEMPT                                                                         0x0868
+#define regSDMA1_QUEUE5_PREEMPT_BASE_IDX                                                                0
+#define regSDMA1_QUEUE5_DUMMY_REG                                                                       0x0869
+#define regSDMA1_QUEUE5_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA1_QUEUE5_RB_WPTR_POLL_ADDR_HI                                                            0x086a
+#define regSDMA1_QUEUE5_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_QUEUE5_RB_WPTR_POLL_ADDR_LO                                                            0x086b
+#define regSDMA1_QUEUE5_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_QUEUE5_RB_AQL_CNTL                                                                     0x086c
+#define regSDMA1_QUEUE5_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE5_MINOR_PTR_UPDATE                                                                0x086d
+#define regSDMA1_QUEUE5_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA1_QUEUE5_RB_PREEMPT                                                                      0x086e
+#define regSDMA1_QUEUE5_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA1_QUEUE5_MIDCMD_DATA0                                                                    0x0878
+#define regSDMA1_QUEUE5_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_MIDCMD_DATA1                                                                    0x0879
+#define regSDMA1_QUEUE5_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_MIDCMD_DATA2                                                                    0x087a
+#define regSDMA1_QUEUE5_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_MIDCMD_DATA3                                                                    0x087b
+#define regSDMA1_QUEUE5_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_MIDCMD_DATA4                                                                    0x087c
+#define regSDMA1_QUEUE5_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_MIDCMD_DATA5                                                                    0x087d
+#define regSDMA1_QUEUE5_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_MIDCMD_DATA6                                                                    0x087e
+#define regSDMA1_QUEUE5_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_MIDCMD_DATA7                                                                    0x087f
+#define regSDMA1_QUEUE5_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_MIDCMD_DATA8                                                                    0x0880
+#define regSDMA1_QUEUE5_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_MIDCMD_DATA9                                                                    0x0881
+#define regSDMA1_QUEUE5_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA1_QUEUE5_MIDCMD_DATA10                                                                   0x0882
+#define regSDMA1_QUEUE5_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA1_QUEUE5_MIDCMD_CNTL                                                                     0x0883
+#define regSDMA1_QUEUE5_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE6_RB_CNTL                                                                         0x0890
+#define regSDMA1_QUEUE6_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE6_RB_BASE                                                                         0x0891
+#define regSDMA1_QUEUE6_RB_BASE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE6_RB_BASE_HI                                                                      0x0892
+#define regSDMA1_QUEUE6_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE6_RB_RPTR                                                                         0x0893
+#define regSDMA1_QUEUE6_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE6_RB_RPTR_HI                                                                      0x0894
+#define regSDMA1_QUEUE6_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE6_RB_WPTR                                                                         0x0895
+#define regSDMA1_QUEUE6_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE6_RB_WPTR_HI                                                                      0x0896
+#define regSDMA1_QUEUE6_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE6_RB_RPTR_ADDR_HI                                                                 0x0898
+#define regSDMA1_QUEUE6_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA1_QUEUE6_RB_RPTR_ADDR_LO                                                                 0x0899
+#define regSDMA1_QUEUE6_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA1_QUEUE6_IB_CNTL                                                                         0x089a
+#define regSDMA1_QUEUE6_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE6_IB_RPTR                                                                         0x089b
+#define regSDMA1_QUEUE6_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE6_IB_OFFSET                                                                       0x089c
+#define regSDMA1_QUEUE6_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA1_QUEUE6_IB_BASE_LO                                                                      0x089d
+#define regSDMA1_QUEUE6_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA1_QUEUE6_IB_BASE_HI                                                                      0x089e
+#define regSDMA1_QUEUE6_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE6_IB_SIZE                                                                         0x089f
+#define regSDMA1_QUEUE6_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE6_SKIP_CNTL                                                                       0x08a0
+#define regSDMA1_QUEUE6_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA1_QUEUE6_CONTEXT_STATUS                                                                  0x08a1
+#define regSDMA1_QUEUE6_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA1_QUEUE6_DOORBELL                                                                        0x08a2
+#define regSDMA1_QUEUE6_DOORBELL_BASE_IDX                                                               0
+#define regSDMA1_QUEUE6_DOORBELL_LOG                                                                    0x08b9
+#define regSDMA1_QUEUE6_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_DOORBELL_OFFSET                                                                 0x08bb
+#define regSDMA1_QUEUE6_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA1_QUEUE6_CSA_ADDR_LO                                                                     0x08bc
+#define regSDMA1_QUEUE6_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA1_QUEUE6_CSA_ADDR_HI                                                                     0x08bd
+#define regSDMA1_QUEUE6_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA1_QUEUE6_SCHEDULE_CNTL                                                                   0x08be
+#define regSDMA1_QUEUE6_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA1_QUEUE6_IB_SUB_REMAIN                                                                   0x08bf
+#define regSDMA1_QUEUE6_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA1_QUEUE6_PREEMPT                                                                         0x08c0
+#define regSDMA1_QUEUE6_PREEMPT_BASE_IDX                                                                0
+#define regSDMA1_QUEUE6_DUMMY_REG                                                                       0x08c1
+#define regSDMA1_QUEUE6_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA1_QUEUE6_RB_WPTR_POLL_ADDR_HI                                                            0x08c2
+#define regSDMA1_QUEUE6_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_QUEUE6_RB_WPTR_POLL_ADDR_LO                                                            0x08c3
+#define regSDMA1_QUEUE6_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_QUEUE6_RB_AQL_CNTL                                                                     0x08c4
+#define regSDMA1_QUEUE6_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE6_MINOR_PTR_UPDATE                                                                0x08c5
+#define regSDMA1_QUEUE6_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA1_QUEUE6_RB_PREEMPT                                                                      0x08c6
+#define regSDMA1_QUEUE6_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA1_QUEUE6_MIDCMD_DATA0                                                                    0x08d0
+#define regSDMA1_QUEUE6_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_MIDCMD_DATA1                                                                    0x08d1
+#define regSDMA1_QUEUE6_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_MIDCMD_DATA2                                                                    0x08d2
+#define regSDMA1_QUEUE6_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_MIDCMD_DATA3                                                                    0x08d3
+#define regSDMA1_QUEUE6_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_MIDCMD_DATA4                                                                    0x08d4
+#define regSDMA1_QUEUE6_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_MIDCMD_DATA5                                                                    0x08d5
+#define regSDMA1_QUEUE6_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_MIDCMD_DATA6                                                                    0x08d6
+#define regSDMA1_QUEUE6_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_MIDCMD_DATA7                                                                    0x08d7
+#define regSDMA1_QUEUE6_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_MIDCMD_DATA8                                                                    0x08d8
+#define regSDMA1_QUEUE6_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_MIDCMD_DATA9                                                                    0x08d9
+#define regSDMA1_QUEUE6_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA1_QUEUE6_MIDCMD_DATA10                                                                   0x08da
+#define regSDMA1_QUEUE6_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA1_QUEUE6_MIDCMD_CNTL                                                                     0x08db
+#define regSDMA1_QUEUE6_MIDCMD_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE7_RB_CNTL                                                                         0x08e8
+#define regSDMA1_QUEUE7_RB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE7_RB_BASE                                                                         0x08e9
+#define regSDMA1_QUEUE7_RB_BASE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE7_RB_BASE_HI                                                                      0x08ea
+#define regSDMA1_QUEUE7_RB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE7_RB_RPTR                                                                         0x08eb
+#define regSDMA1_QUEUE7_RB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE7_RB_RPTR_HI                                                                      0x08ec
+#define regSDMA1_QUEUE7_RB_RPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE7_RB_WPTR                                                                         0x08ed
+#define regSDMA1_QUEUE7_RB_WPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE7_RB_WPTR_HI                                                                      0x08ee
+#define regSDMA1_QUEUE7_RB_WPTR_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE7_RB_RPTR_ADDR_HI                                                                 0x08f0
+#define regSDMA1_QUEUE7_RB_RPTR_ADDR_HI_BASE_IDX                                                        0
+#define regSDMA1_QUEUE7_RB_RPTR_ADDR_LO                                                                 0x08f1
+#define regSDMA1_QUEUE7_RB_RPTR_ADDR_LO_BASE_IDX                                                        0
+#define regSDMA1_QUEUE7_IB_CNTL                                                                         0x08f2
+#define regSDMA1_QUEUE7_IB_CNTL_BASE_IDX                                                                0
+#define regSDMA1_QUEUE7_IB_RPTR                                                                         0x08f3
+#define regSDMA1_QUEUE7_IB_RPTR_BASE_IDX                                                                0
+#define regSDMA1_QUEUE7_IB_OFFSET                                                                       0x08f4
+#define regSDMA1_QUEUE7_IB_OFFSET_BASE_IDX                                                              0
+#define regSDMA1_QUEUE7_IB_BASE_LO                                                                      0x08f5
+#define regSDMA1_QUEUE7_IB_BASE_LO_BASE_IDX                                                             0
+#define regSDMA1_QUEUE7_IB_BASE_HI                                                                      0x08f6
+#define regSDMA1_QUEUE7_IB_BASE_HI_BASE_IDX                                                             0
+#define regSDMA1_QUEUE7_IB_SIZE                                                                         0x08f7
+#define regSDMA1_QUEUE7_IB_SIZE_BASE_IDX                                                                0
+#define regSDMA1_QUEUE7_SKIP_CNTL                                                                       0x08f8
+#define regSDMA1_QUEUE7_SKIP_CNTL_BASE_IDX                                                              0
+#define regSDMA1_QUEUE7_CONTEXT_STATUS                                                                  0x08f9
+#define regSDMA1_QUEUE7_CONTEXT_STATUS_BASE_IDX                                                         0
+#define regSDMA1_QUEUE7_DOORBELL                                                                        0x08fa
+#define regSDMA1_QUEUE7_DOORBELL_BASE_IDX                                                               0
+#define regSDMA1_QUEUE7_DOORBELL_LOG                                                                    0x0911
+#define regSDMA1_QUEUE7_DOORBELL_LOG_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_DOORBELL_OFFSET                                                                 0x0913
+#define regSDMA1_QUEUE7_DOORBELL_OFFSET_BASE_IDX                                                        0
+#define regSDMA1_QUEUE7_CSA_ADDR_LO                                                                     0x0914
+#define regSDMA1_QUEUE7_CSA_ADDR_LO_BASE_IDX                                                            0
+#define regSDMA1_QUEUE7_CSA_ADDR_HI                                                                     0x0915
+#define regSDMA1_QUEUE7_CSA_ADDR_HI_BASE_IDX                                                            0
+#define regSDMA1_QUEUE7_SCHEDULE_CNTL                                                                   0x0916
+#define regSDMA1_QUEUE7_SCHEDULE_CNTL_BASE_IDX                                                          0
+#define regSDMA1_QUEUE7_IB_SUB_REMAIN                                                                   0x0917
+#define regSDMA1_QUEUE7_IB_SUB_REMAIN_BASE_IDX                                                          0
+#define regSDMA1_QUEUE7_PREEMPT                                                                         0x0918
+#define regSDMA1_QUEUE7_PREEMPT_BASE_IDX                                                                0
+#define regSDMA1_QUEUE7_DUMMY_REG                                                                       0x0919
+#define regSDMA1_QUEUE7_DUMMY_REG_BASE_IDX                                                              0
+#define regSDMA1_QUEUE7_RB_WPTR_POLL_ADDR_HI                                                            0x091a
+#define regSDMA1_QUEUE7_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                   0
+#define regSDMA1_QUEUE7_RB_WPTR_POLL_ADDR_LO                                                            0x091b
+#define regSDMA1_QUEUE7_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                   0
+#define regSDMA1_QUEUE7_RB_AQL_CNTL                                                                     0x091c
+#define regSDMA1_QUEUE7_RB_AQL_CNTL_BASE_IDX                                                            0
+#define regSDMA1_QUEUE7_MINOR_PTR_UPDATE                                                                0x091d
+#define regSDMA1_QUEUE7_MINOR_PTR_UPDATE_BASE_IDX                                                       0
+#define regSDMA1_QUEUE7_RB_PREEMPT                                                                      0x091e
+#define regSDMA1_QUEUE7_RB_PREEMPT_BASE_IDX                                                             0
+#define regSDMA1_QUEUE7_MIDCMD_DATA0                                                                    0x0928
+#define regSDMA1_QUEUE7_MIDCMD_DATA0_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_MIDCMD_DATA1                                                                    0x0929
+#define regSDMA1_QUEUE7_MIDCMD_DATA1_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_MIDCMD_DATA2                                                                    0x092a
+#define regSDMA1_QUEUE7_MIDCMD_DATA2_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_MIDCMD_DATA3                                                                    0x092b
+#define regSDMA1_QUEUE7_MIDCMD_DATA3_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_MIDCMD_DATA4                                                                    0x092c
+#define regSDMA1_QUEUE7_MIDCMD_DATA4_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_MIDCMD_DATA5                                                                    0x092d
+#define regSDMA1_QUEUE7_MIDCMD_DATA5_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_MIDCMD_DATA6                                                                    0x092e
+#define regSDMA1_QUEUE7_MIDCMD_DATA6_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_MIDCMD_DATA7                                                                    0x092f
+#define regSDMA1_QUEUE7_MIDCMD_DATA7_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_MIDCMD_DATA8                                                                    0x0930
+#define regSDMA1_QUEUE7_MIDCMD_DATA8_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_MIDCMD_DATA9                                                                    0x0931
+#define regSDMA1_QUEUE7_MIDCMD_DATA9_BASE_IDX                                                           0
+#define regSDMA1_QUEUE7_MIDCMD_DATA10                                                                   0x0932
+#define regSDMA1_QUEUE7_MIDCMD_DATA10_BASE_IDX                                                          0
+#define regSDMA1_QUEUE7_MIDCMD_CNTL                                                                     0x0933
+#define regSDMA1_QUEUE7_MIDCMD_CNTL_BASE_IDX                                                            0
+
+
+// addressBlock: gc_sdma0_sdma0hypdec
+// base address: 0x3e200
+#define regSDMA0_UCODE_ADDR                                                                             0x5880
+#define regSDMA0_UCODE_ADDR_BASE_IDX                                                                    1
+#define regSDMA0_UCODE_DATA                                                                             0x5881
+#define regSDMA0_UCODE_DATA_BASE_IDX                                                                    1
+#define regSDMA0_UCODE_SELFLOAD_CONTROL                                                                 0x5882
+#define regSDMA0_UCODE_SELFLOAD_CONTROL_BASE_IDX                                                        1
+#define regSDMA0_BROADCAST_UCODE_ADDR                                                                   0x5886
+#define regSDMA0_BROADCAST_UCODE_ADDR_BASE_IDX                                                          1
+#define regSDMA0_BROADCAST_UCODE_DATA                                                                   0x5887
+#define regSDMA0_BROADCAST_UCODE_DATA_BASE_IDX                                                          1
+#define regSDMA0_VM_CTX_LO                                                                              0x588c
+#define regSDMA0_VM_CTX_LO_BASE_IDX                                                                     1
+#define regSDMA0_VM_CTX_HI                                                                              0x588d
+#define regSDMA0_VM_CTX_HI_BASE_IDX                                                                     1
+#define regSDMA0_ACTIVE_FCN_ID                                                                          0x588e
+#define regSDMA0_ACTIVE_FCN_ID_BASE_IDX                                                                 1
+#define regSDMA0_VM_CTX_CNTL                                                                            0x588f
+#define regSDMA0_VM_CTX_CNTL_BASE_IDX                                                                   1
+#define regSDMA0_VIRT_RESET_REQ                                                                         0x5890
+#define regSDMA0_VIRT_RESET_REQ_BASE_IDX                                                                1
+#define regSDMA0_CONTEXT_REG_TYPE0                                                                      0x5891
+#define regSDMA0_CONTEXT_REG_TYPE0_BASE_IDX                                                             1
+#define regSDMA0_CONTEXT_REG_TYPE1                                                                      0x5892
+#define regSDMA0_CONTEXT_REG_TYPE1_BASE_IDX                                                             1
+#define regSDMA0_CONTEXT_REG_TYPE2                                                                      0x5893
+#define regSDMA0_CONTEXT_REG_TYPE2_BASE_IDX                                                             1
+#define regSDMA0_PUB_REG_TYPE0                                                                          0x5894
+#define regSDMA0_PUB_REG_TYPE0_BASE_IDX                                                                 1
+#define regSDMA0_PUB_REG_TYPE1                                                                          0x5895
+#define regSDMA0_PUB_REG_TYPE1_BASE_IDX                                                                 1
+#define regSDMA0_PUB_REG_TYPE2                                                                          0x5896
+#define regSDMA0_PUB_REG_TYPE2_BASE_IDX                                                                 1
+#define regSDMA0_PUB_REG_TYPE3                                                                          0x5897
+#define regSDMA0_PUB_REG_TYPE3_BASE_IDX                                                                 1
+#define regSDMA0_VM_CNTL                                                                                0x5899
+#define regSDMA0_VM_CNTL_BASE_IDX                                                                       1
+#define regSDMA0_F32_CNTL                                                                               0x589a
+#define regSDMA0_F32_CNTL_BASE_IDX                                                                      1
+
+
+// addressBlock: gc_sdma0_sdma1hypdec
+// base address: 0x3e280
+#define regSDMA1_UCODE_ADDR                                                                             0x58a0
+#define regSDMA1_UCODE_ADDR_BASE_IDX                                                                    1
+#define regSDMA1_UCODE_DATA                                                                             0x58a1
+#define regSDMA1_UCODE_DATA_BASE_IDX                                                                    1
+#define regSDMA1_UCODE_SELFLOAD_CONTROL                                                                 0x58a2
+#define regSDMA1_UCODE_SELFLOAD_CONTROL_BASE_IDX                                                        1
+#define regSDMA1_BROADCAST_UCODE_ADDR                                                                   0x58a6
+#define regSDMA1_BROADCAST_UCODE_ADDR_BASE_IDX                                                          1
+#define regSDMA1_BROADCAST_UCODE_DATA                                                                   0x58a7
+#define regSDMA1_BROADCAST_UCODE_DATA_BASE_IDX                                                          1
+#define regSDMA1_VM_CTX_LO                                                                              0x58ac
+#define regSDMA1_VM_CTX_LO_BASE_IDX                                                                     1
+#define regSDMA1_VM_CTX_HI                                                                              0x58ad
+#define regSDMA1_VM_CTX_HI_BASE_IDX                                                                     1
+#define regSDMA1_ACTIVE_FCN_ID                                                                          0x58ae
+#define regSDMA1_ACTIVE_FCN_ID_BASE_IDX                                                                 1
+#define regSDMA1_VM_CTX_CNTL                                                                            0x58af
+#define regSDMA1_VM_CTX_CNTL_BASE_IDX                                                                   1
+#define regSDMA1_VIRT_RESET_REQ                                                                         0x58b0
+#define regSDMA1_VIRT_RESET_REQ_BASE_IDX                                                                1
+#define regSDMA1_CONTEXT_REG_TYPE0                                                                      0x58b1
+#define regSDMA1_CONTEXT_REG_TYPE0_BASE_IDX                                                             1
+#define regSDMA1_CONTEXT_REG_TYPE1                                                                      0x58b2
+#define regSDMA1_CONTEXT_REG_TYPE1_BASE_IDX                                                             1
+#define regSDMA1_CONTEXT_REG_TYPE2                                                                      0x58b3
+#define regSDMA1_CONTEXT_REG_TYPE2_BASE_IDX                                                             1
+#define regSDMA1_PUB_REG_TYPE0                                                                          0x58b4
+#define regSDMA1_PUB_REG_TYPE0_BASE_IDX                                                                 1
+#define regSDMA1_PUB_REG_TYPE1                                                                          0x58b5
+#define regSDMA1_PUB_REG_TYPE1_BASE_IDX                                                                 1
+#define regSDMA1_PUB_REG_TYPE2                                                                          0x58b6
+#define regSDMA1_PUB_REG_TYPE2_BASE_IDX                                                                 1
+#define regSDMA1_PUB_REG_TYPE3                                                                          0x58b7
+#define regSDMA1_PUB_REG_TYPE3_BASE_IDX                                                                 1
+#define regSDMA1_VM_CNTL                                                                                0x58b9
+#define regSDMA1_VM_CNTL_BASE_IDX                                                                       1
+#define regSDMA1_F32_CNTL                                                                               0x58ba
+#define regSDMA1_F32_CNTL_BASE_IDX                                                                      1
+
+
+// addressBlock: gc_sdma0_sdma0perfsdec
+// base address: 0x37880
+#define regSDMA0_PERFCNT_PERFCOUNTER0_CFG                                                               0x3e20
+#define regSDMA0_PERFCNT_PERFCOUNTER0_CFG_BASE_IDX                                                      1
+#define regSDMA0_PERFCNT_PERFCOUNTER1_CFG                                                               0x3e21
+#define regSDMA0_PERFCNT_PERFCOUNTER1_CFG_BASE_IDX                                                      1
+#define regSDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL                                                          0x3e22
+#define regSDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                 1
+#define regSDMA0_PERFCNT_MISC_CNTL                                                                      0x3e23
+#define regSDMA0_PERFCNT_MISC_CNTL_BASE_IDX                                                             1
+#define regSDMA0_PERFCOUNTER0_SELECT                                                                    0x3e24
+#define regSDMA0_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regSDMA0_PERFCOUNTER0_SELECT1                                                                   0x3e25
+#define regSDMA0_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regSDMA0_PERFCOUNTER1_SELECT                                                                    0x3e26
+#define regSDMA0_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regSDMA0_PERFCOUNTER1_SELECT1                                                                   0x3e27
+#define regSDMA0_PERFCOUNTER1_SELECT1_BASE_IDX                                                          1
+
+
+// addressBlock: gc_sdma0_sdma1perfsdec
+// base address: 0x378b0
+#define regSDMA1_PERFCNT_PERFCOUNTER0_CFG                                                               0x3e2c
+#define regSDMA1_PERFCNT_PERFCOUNTER0_CFG_BASE_IDX                                                      1
+#define regSDMA1_PERFCNT_PERFCOUNTER1_CFG                                                               0x3e2d
+#define regSDMA1_PERFCNT_PERFCOUNTER1_CFG_BASE_IDX                                                      1
+#define regSDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL                                                          0x3e2e
+#define regSDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                 1
+#define regSDMA1_PERFCNT_MISC_CNTL                                                                      0x3e2f
+#define regSDMA1_PERFCNT_MISC_CNTL_BASE_IDX                                                             1
+#define regSDMA1_PERFCOUNTER0_SELECT                                                                    0x3e30
+#define regSDMA1_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regSDMA1_PERFCOUNTER0_SELECT1                                                                   0x3e31
+#define regSDMA1_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regSDMA1_PERFCOUNTER1_SELECT                                                                    0x3e32
+#define regSDMA1_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regSDMA1_PERFCOUNTER1_SELECT1                                                                   0x3e33
+#define regSDMA1_PERFCOUNTER1_SELECT1_BASE_IDX                                                          1
+
+
+// addressBlock: gc_sdma0_sdma0perfddec
+// base address: 0x35980
+#define regSDMA0_PERFCNT_PERFCOUNTER_LO                                                                 0x3660
+#define regSDMA0_PERFCNT_PERFCOUNTER_LO_BASE_IDX                                                        1
+#define regSDMA0_PERFCNT_PERFCOUNTER_HI                                                                 0x3661
+#define regSDMA0_PERFCNT_PERFCOUNTER_HI_BASE_IDX                                                        1
+#define regSDMA0_PERFCOUNTER0_LO                                                                        0x3662
+#define regSDMA0_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regSDMA0_PERFCOUNTER0_HI                                                                        0x3663
+#define regSDMA0_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regSDMA0_PERFCOUNTER1_LO                                                                        0x3664
+#define regSDMA0_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regSDMA0_PERFCOUNTER1_HI                                                                        0x3665
+#define regSDMA0_PERFCOUNTER1_HI_BASE_IDX                                                               1
+
+
+// addressBlock: gc_sdma0_sdma1perfddec
+// base address: 0x359b0
+#define regSDMA1_PERFCNT_PERFCOUNTER_LO                                                                 0x366c
+#define regSDMA1_PERFCNT_PERFCOUNTER_LO_BASE_IDX                                                        1
+#define regSDMA1_PERFCNT_PERFCOUNTER_HI                                                                 0x366d
+#define regSDMA1_PERFCNT_PERFCOUNTER_HI_BASE_IDX                                                        1
+#define regSDMA1_PERFCOUNTER0_LO                                                                        0x366e
+#define regSDMA1_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regSDMA1_PERFCOUNTER0_HI                                                                        0x366f
+#define regSDMA1_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regSDMA1_PERFCOUNTER1_LO                                                                        0x3670
+#define regSDMA1_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regSDMA1_PERFCOUNTER1_HI                                                                        0x3671
+#define regSDMA1_PERFCOUNTER1_HI_BASE_IDX                                                               1
+
+
+// addressBlock: gc_grbmdec
+// base address: 0x8000
+#define regGRBM_CNTL                                                                                    0x0da0
+#define regGRBM_CNTL_BASE_IDX                                                                           0
+#define regGRBM_SKEW_CNTL                                                                               0x0da1
+#define regGRBM_SKEW_CNTL_BASE_IDX                                                                      0
+#define regGRBM_STATUS2                                                                                 0x0da2
+#define regGRBM_STATUS2_BASE_IDX                                                                        0
+#define regGRBM_PWR_CNTL                                                                                0x0da3
+#define regGRBM_PWR_CNTL_BASE_IDX                                                                       0
+#define regGRBM_STATUS                                                                                  0x0da4
+#define regGRBM_STATUS_BASE_IDX                                                                         0
+#define regGRBM_STATUS_SE0                                                                              0x0da5
+#define regGRBM_STATUS_SE0_BASE_IDX                                                                     0
+#define regGRBM_STATUS_SE1                                                                              0x0da6
+#define regGRBM_STATUS_SE1_BASE_IDX                                                                     0
+#define regGRBM_STATUS3                                                                                 0x0da7
+#define regGRBM_STATUS3_BASE_IDX                                                                        0
+#define regGRBM_SOFT_RESET                                                                              0x0da8
+#define regGRBM_SOFT_RESET_BASE_IDX                                                                     0
+#define regGRBM_GFX_CLKEN_CNTL                                                                          0x0dac
+#define regGRBM_GFX_CLKEN_CNTL_BASE_IDX                                                                 0
+#define regGRBM_WAIT_IDLE_CLOCKS                                                                        0x0dad
+#define regGRBM_WAIT_IDLE_CLOCKS_BASE_IDX                                                               0
+#define regGRBM_STATUS_SE2                                                                              0x0dae
+#define regGRBM_STATUS_SE2_BASE_IDX                                                                     0
+#define regGRBM_READ_ERROR                                                                              0x0db6
+#define regGRBM_READ_ERROR_BASE_IDX                                                                     0
+#define regGRBM_READ_ERROR2                                                                             0x0db7
+#define regGRBM_READ_ERROR2_BASE_IDX                                                                    0
+#define regGRBM_INT_CNTL                                                                                0x0db8
+#define regGRBM_INT_CNTL_BASE_IDX                                                                       0
+#define regGRBM_TRAP_OP                                                                                 0x0db9
+#define regGRBM_TRAP_OP_BASE_IDX                                                                        0
+#define regGRBM_TRAP_ADDR                                                                               0x0dba
+#define regGRBM_TRAP_ADDR_BASE_IDX                                                                      0
+#define regGRBM_TRAP_ADDR_MSK                                                                           0x0dbb
+#define regGRBM_TRAP_ADDR_MSK_BASE_IDX                                                                  0
+#define regGRBM_TRAP_WD                                                                                 0x0dbc
+#define regGRBM_TRAP_WD_BASE_IDX                                                                        0
+#define regGRBM_TRAP_WD_MSK                                                                             0x0dbd
+#define regGRBM_TRAP_WD_MSK_BASE_IDX                                                                    0
+#define regGRBM_DSM_BYPASS                                                                              0x0dbe
+#define regGRBM_DSM_BYPASS_BASE_IDX                                                                     0
+#define regGRBM_WRITE_ERROR                                                                             0x0dbf
+#define regGRBM_WRITE_ERROR_BASE_IDX                                                                    0
+#define regGRBM_CHIP_REVISION                                                                           0x0dc1
+#define regGRBM_CHIP_REVISION_BASE_IDX                                                                  0
+#define regGRBM_RSMU_CFG                                                                                0x0dc3
+#define regGRBM_RSMU_CFG_BASE_IDX                                                                       0
+#define regGRBM_IH_CREDIT                                                                               0x0dc4
+#define regGRBM_IH_CREDIT_BASE_IDX                                                                      0
+#define regGRBM_PWR_CNTL2                                                                               0x0dc5
+#define regGRBM_PWR_CNTL2_BASE_IDX                                                                      0
+#define regGRBM_UTCL2_INVAL_RANGE_START                                                                 0x0dc6
+#define regGRBM_UTCL2_INVAL_RANGE_START_BASE_IDX                                                        0
+#define regGRBM_UTCL2_INVAL_RANGE_END                                                                   0x0dc7
+#define regGRBM_UTCL2_INVAL_RANGE_END_BASE_IDX                                                          0
+#define regGRBM_RSMU_READ_ERROR                                                                         0x0dc8
+#define regGRBM_RSMU_READ_ERROR_BASE_IDX                                                                0
+#define regGRBM_INVALID_PIPE                                                                            0x0dc9
+#define regGRBM_INVALID_PIPE_BASE_IDX                                                                   0
+#define regGRBM_FENCE_RANGE0                                                                            0x0dca
+#define regGRBM_FENCE_RANGE0_BASE_IDX                                                                   0
+#define regGRBM_FENCE_RANGE1                                                                            0x0dcb
+#define regGRBM_FENCE_RANGE1_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG0                                                                            0x0de0
+#define regGRBM_SCRATCH_REG0_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG1                                                                            0x0de1
+#define regGRBM_SCRATCH_REG1_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG2                                                                            0x0de2
+#define regGRBM_SCRATCH_REG2_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG3                                                                            0x0de3
+#define regGRBM_SCRATCH_REG3_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG4                                                                            0x0de4
+#define regGRBM_SCRATCH_REG4_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG5                                                                            0x0de5
+#define regGRBM_SCRATCH_REG5_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG6                                                                            0x0de6
+#define regGRBM_SCRATCH_REG6_BASE_IDX                                                                   0
+#define regGRBM_SCRATCH_REG7                                                                            0x0de7
+#define regGRBM_SCRATCH_REG7_BASE_IDX                                                                   0
+#define regVIOLATION_DATA_ASYNC_VF_PROG                                                                 0x0df1
+#define regVIOLATION_DATA_ASYNC_VF_PROG_BASE_IDX                                                        0
+
+
+// addressBlock: gc_cpdec
+// base address: 0x8200
+#define regCP_CPC_DEBUG_CNTL                                                                            0x0e20
+#define regCP_CPC_DEBUG_CNTL_BASE_IDX                                                                   0
+#define regCP_CPF_DEBUG_CNTL                                                                            0x0e22
+#define regCP_CPF_DEBUG_CNTL_BASE_IDX                                                                   0
+#define regCP_CPC_STATUS                                                                                0x0e24
+#define regCP_CPC_STATUS_BASE_IDX                                                                       0
+#define regCP_CPC_BUSY_STAT                                                                             0x0e25
+#define regCP_CPC_BUSY_STAT_BASE_IDX                                                                    0
+#define regCP_CPC_STALLED_STAT1                                                                         0x0e26
+#define regCP_CPC_STALLED_STAT1_BASE_IDX                                                                0
+#define regCP_CPF_STATUS                                                                                0x0e27
+#define regCP_CPF_STATUS_BASE_IDX                                                                       0
+#define regCP_CPF_BUSY_STAT                                                                             0x0e28
+#define regCP_CPF_BUSY_STAT_BASE_IDX                                                                    0
+#define regCP_CPF_STALLED_STAT1                                                                         0x0e29
+#define regCP_CPF_STALLED_STAT1_BASE_IDX                                                                0
+#define regCP_CPC_BUSY_STAT2                                                                            0x0e2a
+#define regCP_CPC_BUSY_STAT2_BASE_IDX                                                                   0
+#define regCP_CPC_GRBM_FREE_COUNT                                                                       0x0e2b
+#define regCP_CPC_GRBM_FREE_COUNT_BASE_IDX                                                              0
+#define regCP_CPC_PRIV_VIOLATION_ADDR                                                                   0x0e2c
+#define regCP_CPC_PRIV_VIOLATION_ADDR_BASE_IDX                                                          0
+#define regCP_MEC_ME1_HEADER_DUMP                                                                       0x0e2e
+#define regCP_MEC_ME1_HEADER_DUMP_BASE_IDX                                                              0
+#define regCP_MEC_ME2_HEADER_DUMP                                                                       0x0e2f
+#define regCP_MEC_ME2_HEADER_DUMP_BASE_IDX                                                              0
+#define regCP_CPC_SCRATCH_INDEX                                                                         0x0e30
+#define regCP_CPC_SCRATCH_INDEX_BASE_IDX                                                                0
+#define regCP_CPC_SCRATCH_DATA                                                                          0x0e31
+#define regCP_CPC_SCRATCH_DATA_BASE_IDX                                                                 0
+#define regCP_CPF_GRBM_FREE_COUNT                                                                       0x0e32
+#define regCP_CPF_GRBM_FREE_COUNT_BASE_IDX                                                              0
+#define regCP_CPF_BUSY_STAT2                                                                            0x0e33
+#define regCP_CPF_BUSY_STAT2_BASE_IDX                                                                   0
+#define regCP_CPC_HALT_HYST_COUNT                                                                       0x0e47
+#define regCP_CPC_HALT_HYST_COUNT_BASE_IDX                                                              0
+#define regCP_STALLED_STAT3                                                                             0x0f3c
+#define regCP_STALLED_STAT3_BASE_IDX                                                                    0
+#define regCP_STALLED_STAT1                                                                             0x0f3d
+#define regCP_STALLED_STAT1_BASE_IDX                                                                    0
+#define regCP_STALLED_STAT2                                                                             0x0f3e
+#define regCP_STALLED_STAT2_BASE_IDX                                                                    0
+#define regCP_BUSY_STAT                                                                                 0x0f3f
+#define regCP_BUSY_STAT_BASE_IDX                                                                        0
+#define regCP_STAT                                                                                      0x0f40
+#define regCP_STAT_BASE_IDX                                                                             0
+#define regCP_ME_HEADER_DUMP                                                                            0x0f41
+#define regCP_ME_HEADER_DUMP_BASE_IDX                                                                   0
+#define regCP_PFP_HEADER_DUMP                                                                           0x0f42
+#define regCP_PFP_HEADER_DUMP_BASE_IDX                                                                  0
+#define regCP_GRBM_FREE_COUNT                                                                           0x0f43
+#define regCP_GRBM_FREE_COUNT_BASE_IDX                                                                  0
+#define regCP_PFP_INSTR_PNTR                                                                            0x0f45
+#define regCP_PFP_INSTR_PNTR_BASE_IDX                                                                   0
+#define regCP_ME_INSTR_PNTR                                                                             0x0f46
+#define regCP_ME_INSTR_PNTR_BASE_IDX                                                                    0
+#define regCP_MEC1_INSTR_PNTR                                                                           0x0f48
+#define regCP_MEC1_INSTR_PNTR_BASE_IDX                                                                  0
+#define regCP_MEC2_INSTR_PNTR                                                                           0x0f49
+#define regCP_MEC2_INSTR_PNTR_BASE_IDX                                                                  0
+#define regCP_CSF_STAT                                                                                  0x0f54
+#define regCP_CSF_STAT_BASE_IDX                                                                         0
+#define regCP_CNTX_STAT                                                                                 0x0f58
+#define regCP_CNTX_STAT_BASE_IDX                                                                        0
+#define regCP_ME_PREEMPTION                                                                             0x0f59
+#define regCP_ME_PREEMPTION_BASE_IDX                                                                    0
+#define regCP_RB1_RPTR                                                                                  0x0f5f
+#define regCP_RB1_RPTR_BASE_IDX                                                                         0
+#define regCP_RB0_RPTR                                                                                  0x0f60
+#define regCP_RB0_RPTR_BASE_IDX                                                                         0
+#define regCP_RB_RPTR                                                                                   0x0f60
+#define regCP_RB_RPTR_BASE_IDX                                                                          0
+#define regCP_RB_WPTR_DELAY                                                                             0x0f61
+#define regCP_RB_WPTR_DELAY_BASE_IDX                                                                    0
+#define regCP_RB_WPTR_POLL_CNTL                                                                         0x0f62
+#define regCP_RB_WPTR_POLL_CNTL_BASE_IDX                                                                0
+#define regCP_ROQ1_THRESHOLDS                                                                           0x0f75
+#define regCP_ROQ1_THRESHOLDS_BASE_IDX                                                                  0
+#define regCP_ROQ2_THRESHOLDS                                                                           0x0f76
+#define regCP_ROQ2_THRESHOLDS_BASE_IDX                                                                  0
+#define regCP_STQ_THRESHOLDS                                                                            0x0f77
+#define regCP_STQ_THRESHOLDS_BASE_IDX                                                                   0
+#define regCP_MEQ_THRESHOLDS                                                                            0x0f79
+#define regCP_MEQ_THRESHOLDS_BASE_IDX                                                                   0
+#define regCP_ROQ_AVAIL                                                                                 0x0f7a
+#define regCP_ROQ_AVAIL_BASE_IDX                                                                        0
+#define regCP_STQ_AVAIL                                                                                 0x0f7b
+#define regCP_STQ_AVAIL_BASE_IDX                                                                        0
+#define regCP_ROQ2_AVAIL                                                                                0x0f7c
+#define regCP_ROQ2_AVAIL_BASE_IDX                                                                       0
+#define regCP_MEQ_AVAIL                                                                                 0x0f7d
+#define regCP_MEQ_AVAIL_BASE_IDX                                                                        0
+#define regCP_CMD_INDEX                                                                                 0x0f7e
+#define regCP_CMD_INDEX_BASE_IDX                                                                        0
+#define regCP_CMD_DATA                                                                                  0x0f7f
+#define regCP_CMD_DATA_BASE_IDX                                                                         0
+#define regCP_ROQ_RB_STAT                                                                               0x0f80
+#define regCP_ROQ_RB_STAT_BASE_IDX                                                                      0
+#define regCP_ROQ_IB1_STAT                                                                              0x0f81
+#define regCP_ROQ_IB1_STAT_BASE_IDX                                                                     0
+#define regCP_ROQ_IB2_STAT                                                                              0x0f82
+#define regCP_ROQ_IB2_STAT_BASE_IDX                                                                     0
+#define regCP_STQ_STAT                                                                                  0x0f83
+#define regCP_STQ_STAT_BASE_IDX                                                                         0
+#define regCP_STQ_WR_STAT                                                                               0x0f84
+#define regCP_STQ_WR_STAT_BASE_IDX                                                                      0
+#define regCP_MEQ_STAT                                                                                  0x0f85
+#define regCP_MEQ_STAT_BASE_IDX                                                                         0
+#define regCP_ROQ3_THRESHOLDS                                                                           0x0f8c
+#define regCP_ROQ3_THRESHOLDS_BASE_IDX                                                                  0
+#define regCP_ROQ_DB_STAT                                                                               0x0f8d
+#define regCP_ROQ_DB_STAT_BASE_IDX                                                                      0
+#define regCP_INT_STAT_DEBUG                                                                            0x0f97
+#define regCP_INT_STAT_DEBUG_BASE_IDX                                                                   0
+#define regCP_DEBUG_CNTL                                                                                0x0f98
+#define regCP_DEBUG_CNTL_BASE_IDX                                                                       0
+#define regCP_PRIV_VIOLATION_ADDR                                                                       0x0f9a
+#define regCP_PRIV_VIOLATION_ADDR_BASE_IDX                                                              0
+
+
+// addressBlock: gc_padec
+// base address: 0x8800
+#define regVGT_DMA_DATA_FIFO_DEPTH                                                                      0x0fcd
+#define regVGT_DMA_DATA_FIFO_DEPTH_BASE_IDX                                                             0
+#define regVGT_DMA_REQ_FIFO_DEPTH                                                                       0x0fce
+#define regVGT_DMA_REQ_FIFO_DEPTH_BASE_IDX                                                              0
+#define regVGT_DRAW_INIT_FIFO_DEPTH                                                                     0x0fcf
+#define regVGT_DRAW_INIT_FIFO_DEPTH_BASE_IDX                                                            0
+#define regVGT_MC_LAT_CNTL                                                                              0x0fd6
+#define regVGT_MC_LAT_CNTL_BASE_IDX                                                                     0
+#define regIA_UTCL1_STATUS_2                                                                            0x0fd7
+#define regIA_UTCL1_STATUS_2_BASE_IDX                                                                   0
+#define regWD_CNTL_STATUS                                                                               0x0fdf
+#define regWD_CNTL_STATUS_BASE_IDX                                                                      0
+#define regCC_GC_PRIM_CONFIG                                                                            0x0fe0
+#define regCC_GC_PRIM_CONFIG_BASE_IDX                                                                   0
+#define regWD_QOS                                                                                       0x0fe2
+#define regWD_QOS_BASE_IDX                                                                              0
+#define regWD_UTCL1_CNTL                                                                                0x0fe3
+#define regWD_UTCL1_CNTL_BASE_IDX                                                                       0
+#define regWD_UTCL1_STATUS                                                                              0x0fe4
+#define regWD_UTCL1_STATUS_BASE_IDX                                                                     0
+#define regIA_UTCL1_CNTL                                                                                0x0fe6
+#define regIA_UTCL1_CNTL_BASE_IDX                                                                       0
+#define regIA_UTCL1_STATUS                                                                              0x0fe7
+#define regIA_UTCL1_STATUS_BASE_IDX                                                                     0
+#define regCC_GC_SA_UNIT_DISABLE                                                                        0x0fe9
+#define regCC_GC_SA_UNIT_DISABLE_BASE_IDX                                                               0
+#define regGE_RATE_CNTL_1                                                                               0x0ff4
+#define regGE_RATE_CNTL_1_BASE_IDX                                                                      0
+#define regGE_RATE_CNTL_2                                                                               0x0ff5
+#define regGE_RATE_CNTL_2_BASE_IDX                                                                      0
+#define regVGT_SYS_CONFIG                                                                               0x1003
+#define regVGT_SYS_CONFIG_BASE_IDX                                                                      0
+#define regGE_PRIV_CONTROL                                                                              0x1004
+#define regGE_PRIV_CONTROL_BASE_IDX                                                                     0
+#define regGE_STATUS                                                                                    0x1005
+#define regGE_STATUS_BASE_IDX                                                                           0
+#define regVGT_GS_MAX_WAVE_ID                                                                           0x1009
+#define regVGT_GS_MAX_WAVE_ID_BASE_IDX                                                                  0
+#define regGFX_PIPE_CONTROL                                                                             0x100d
+#define regGFX_PIPE_CONTROL_BASE_IDX                                                                    0
+#define regCC_GC_SHADER_ARRAY_CONFIG                                                                    0x100f
+#define regCC_GC_SHADER_ARRAY_CONFIG_BASE_IDX                                                           0
+#define regGE2_SE_CNTL_STATUS                                                                           0x1011
+#define regGE2_SE_CNTL_STATUS_BASE_IDX                                                                  0
+#define regVGT_RESET_DEBUG                                                                              0x1014
+#define regVGT_RESET_DEBUG_BASE_IDX                                                                     0
+#define regGE_SPI_IF_SAFE_REG                                                                           0x1018
+#define regGE_SPI_IF_SAFE_REG_BASE_IDX                                                                  0
+#define regGE_PA_IF_SAFE_REG                                                                            0x1019
+#define regGE_PA_IF_SAFE_REG_BASE_IDX                                                                   0
+#define regPA_CL_CNTL_STATUS                                                                            0x1024
+#define regPA_CL_CNTL_STATUS_BASE_IDX                                                                   0
+#define regPA_CL_ENHANCE                                                                                0x1025
+#define regPA_CL_ENHANCE_BASE_IDX                                                                       0
+#define regPA_CL_RESET_DEBUG                                                                            0x1026
+#define regPA_CL_RESET_DEBUG_BASE_IDX                                                                   0
+#define regPA_SU_CNTL_STATUS                                                                            0x1034
+#define regPA_SU_CNTL_STATUS_BASE_IDX                                                                   0
+#define regPA_SC_FIFO_DEPTH_CNTL                                                                        0x1035
+#define regPA_SC_FIFO_DEPTH_CNTL_BASE_IDX                                                               0
+
+
+// addressBlock: gc_sqdec
+// base address: 0x8c00
+#define regSQ_CONFIG                                                                                    0x10a0
+#define regSQ_CONFIG_BASE_IDX                                                                           0
+#define regSQC_CONFIG                                                                                   0x10a1
+#define regSQC_CONFIG_BASE_IDX                                                                          0
+#define regLDS_CONFIG                                                                                   0x10a2
+#define regLDS_CONFIG_BASE_IDX                                                                          0
+#define regSQ_RANDOM_WAVE_PRI                                                                           0x10a3
+#define regSQ_RANDOM_WAVE_PRI_BASE_IDX                                                                  0
+#define regSQG_STATUS                                                                                   0x10a4
+#define regSQG_STATUS_BASE_IDX                                                                          0
+#define regSQ_FIFO_SIZES                                                                                0x10a5
+#define regSQ_FIFO_SIZES_BASE_IDX                                                                       0
+#define regSQ_DSM_CNTL                                                                                  0x10a6
+#define regSQ_DSM_CNTL_BASE_IDX                                                                         0
+#define regSQ_DSM_CNTL2                                                                                 0x10a7
+#define regSQ_DSM_CNTL2_BASE_IDX                                                                        0
+#define regSP_CONFIG                                                                                    0x10ab
+#define regSP_CONFIG_BASE_IDX                                                                           0
+#define regSQ_ARB_CONFIG                                                                                0x10ac
+#define regSQ_ARB_CONFIG_BASE_IDX                                                                       0
+#define regSQ_DEBUG_HOST_TRAP_STATUS                                                                    0x10b6
+#define regSQ_DEBUG_HOST_TRAP_STATUS_BASE_IDX                                                           0
+#define regSQG_GL1H_STATUS                                                                              0x10b9
+#define regSQG_GL1H_STATUS_BASE_IDX                                                                     0
+#define regSQG_CONFIG                                                                                   0x10ba
+#define regSQG_CONFIG_BASE_IDX                                                                          0
+#define regSQ_PERF_SNAPSHOT_CTRL                                                                        0x10bb
+#define regSQ_PERF_SNAPSHOT_CTRL_BASE_IDX                                                               0
+#define regCC_GC_SHADER_RATE_CONFIG                                                                     0x10bc
+#define regCC_GC_SHADER_RATE_CONFIG_BASE_IDX                                                            0
+#define regSQ_INTERRUPT_AUTO_MASK                                                                       0x10be
+#define regSQ_INTERRUPT_AUTO_MASK_BASE_IDX                                                              0
+#define regSQ_INTERRUPT_MSG_CTRL                                                                        0x10bf
+#define regSQ_INTERRUPT_MSG_CTRL_BASE_IDX                                                               0
+#define regSQ_WATCH0_ADDR_H                                                                             0x10d0
+#define regSQ_WATCH0_ADDR_H_BASE_IDX                                                                    0
+#define regSQ_WATCH0_ADDR_L                                                                             0x10d1
+#define regSQ_WATCH0_ADDR_L_BASE_IDX                                                                    0
+#define regSQ_WATCH0_CNTL                                                                               0x10d2
+#define regSQ_WATCH0_CNTL_BASE_IDX                                                                      0
+#define regSQ_WATCH1_ADDR_H                                                                             0x10d3
+#define regSQ_WATCH1_ADDR_H_BASE_IDX                                                                    0
+#define regSQ_WATCH1_ADDR_L                                                                             0x10d4
+#define regSQ_WATCH1_ADDR_L_BASE_IDX                                                                    0
+#define regSQ_WATCH1_CNTL                                                                               0x10d5
+#define regSQ_WATCH1_CNTL_BASE_IDX                                                                      0
+#define regSQ_WATCH2_ADDR_H                                                                             0x10d6
+#define regSQ_WATCH2_ADDR_H_BASE_IDX                                                                    0
+#define regSQ_WATCH2_ADDR_L                                                                             0x10d7
+#define regSQ_WATCH2_ADDR_L_BASE_IDX                                                                    0
+#define regSQ_WATCH2_CNTL                                                                               0x10d8
+#define regSQ_WATCH2_CNTL_BASE_IDX                                                                      0
+#define regSQ_WATCH3_ADDR_H                                                                             0x10d9
+#define regSQ_WATCH3_ADDR_H_BASE_IDX                                                                    0
+#define regSQ_WATCH3_ADDR_L                                                                             0x10da
+#define regSQ_WATCH3_ADDR_L_BASE_IDX                                                                    0
+#define regSQ_WATCH3_CNTL                                                                               0x10db
+#define regSQ_WATCH3_CNTL_BASE_IDX                                                                      0
+#define regSQ_IND_INDEX                                                                                 0x1118
+#define regSQ_IND_INDEX_BASE_IDX                                                                        0
+#define regSQ_IND_DATA                                                                                  0x1119
+#define regSQ_IND_DATA_BASE_IDX                                                                         0
+#define regSQ_CMD                                                                                       0x111b
+#define regSQ_CMD_BASE_IDX                                                                              0
+#define regSQC_MISC_CONFIG                                                                              0x1179
+#define regSQC_MISC_CONFIG_BASE_IDX                                                                     0
+
+
+// addressBlock: gc_shsdec
+// base address: 0x9000
+#define regSX_DEBUG_BUSY                                                                                0x11b4
+#define regSX_DEBUG_BUSY_BASE_IDX                                                                       0
+#define regSX_DEBUG_BUSY_2                                                                              0x11b5
+#define regSX_DEBUG_BUSY_2_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_3                                                                              0x11b6
+#define regSX_DEBUG_BUSY_3_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_4                                                                              0x11b7
+#define regSX_DEBUG_BUSY_4_BASE_IDX                                                                     0
+#define regSX_DEBUG_1                                                                                   0x11b8
+#define regSX_DEBUG_1_BASE_IDX                                                                          0
+#define regSX_DEBUG_BUSY_5                                                                              0x11b9
+#define regSX_DEBUG_BUSY_5_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_6                                                                              0x11ba
+#define regSX_DEBUG_BUSY_6_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_7                                                                              0x11bb
+#define regSX_DEBUG_BUSY_7_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_8                                                                              0x11bc
+#define regSX_DEBUG_BUSY_8_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_9                                                                              0x11bd
+#define regSX_DEBUG_BUSY_9_BASE_IDX                                                                     0
+#define regSX_DEBUG_BUSY_10                                                                             0x11be
+#define regSX_DEBUG_BUSY_10_BASE_IDX                                                                    0
+#define regSPI_PS_MAX_WAVE_ID                                                                           0x11da
+#define regSPI_PS_MAX_WAVE_ID_BASE_IDX                                                                  0
+#define regSPI_GFX_CNTL                                                                                 0x11dc
+#define regSPI_GFX_CNTL_BASE_IDX                                                                        0
+#define regSPI_DEBUG_READ                                                                               0x11e2
+#define regSPI_DEBUG_READ_BASE_IDX                                                                      0
+#define regSPI_DSM_CNTL                                                                                 0x11e3
+#define regSPI_DSM_CNTL_BASE_IDX                                                                        0
+#define regSPI_DSM_CNTL2                                                                                0x11e4
+#define regSPI_DSM_CNTL2_BASE_IDX                                                                       0
+#define regSPI_EDC_CNT                                                                                  0x11e5
+#define regSPI_EDC_CNT_BASE_IDX                                                                         0
+#define regSPI_DEBUG_BUSY                                                                               0x11f0
+#define regSPI_DEBUG_BUSY_BASE_IDX                                                                      0
+#define regSPI_CONFIG_PS_CU_EN                                                                          0x11f2
+#define regSPI_CONFIG_PS_CU_EN_BASE_IDX                                                                 0
+#define regSPI_WF_LIFETIME_CNTL                                                                         0x124a
+#define regSPI_WF_LIFETIME_CNTL_BASE_IDX                                                                0
+#define regSPI_WF_LIFETIME_LIMIT_0                                                                      0x124b
+#define regSPI_WF_LIFETIME_LIMIT_0_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_1                                                                      0x124c
+#define regSPI_WF_LIFETIME_LIMIT_1_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_2                                                                      0x124d
+#define regSPI_WF_LIFETIME_LIMIT_2_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_3                                                                      0x124e
+#define regSPI_WF_LIFETIME_LIMIT_3_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_4                                                                      0x124f
+#define regSPI_WF_LIFETIME_LIMIT_4_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_LIMIT_5                                                                      0x1250
+#define regSPI_WF_LIFETIME_LIMIT_5_BASE_IDX                                                             0
+#define regSPI_WF_LIFETIME_STATUS_0                                                                     0x1255
+#define regSPI_WF_LIFETIME_STATUS_0_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_2                                                                     0x1257
+#define regSPI_WF_LIFETIME_STATUS_2_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_4                                                                     0x1259
+#define regSPI_WF_LIFETIME_STATUS_4_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_6                                                                     0x125b
+#define regSPI_WF_LIFETIME_STATUS_6_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_7                                                                     0x125c
+#define regSPI_WF_LIFETIME_STATUS_7_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_9                                                                     0x125e
+#define regSPI_WF_LIFETIME_STATUS_9_BASE_IDX                                                            0
+#define regSPI_WF_LIFETIME_STATUS_11                                                                    0x1260
+#define regSPI_WF_LIFETIME_STATUS_11_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_13                                                                    0x1262
+#define regSPI_WF_LIFETIME_STATUS_13_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_14                                                                    0x1263
+#define regSPI_WF_LIFETIME_STATUS_14_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_15                                                                    0x1264
+#define regSPI_WF_LIFETIME_STATUS_15_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_16                                                                    0x1265
+#define regSPI_WF_LIFETIME_STATUS_16_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_17                                                                    0x1266
+#define regSPI_WF_LIFETIME_STATUS_17_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_18                                                                    0x1267
+#define regSPI_WF_LIFETIME_STATUS_18_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_19                                                                    0x1268
+#define regSPI_WF_LIFETIME_STATUS_19_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_STATUS_20                                                                    0x1269
+#define regSPI_WF_LIFETIME_STATUS_20_BASE_IDX                                                           0
+#define regSPI_WF_LIFETIME_DEBUG                                                                        0x126a
+#define regSPI_WF_LIFETIME_DEBUG_BASE_IDX                                                               0
+#define regSPI_WF_LIFETIME_STATUS_21                                                                    0x126b
+#define regSPI_WF_LIFETIME_STATUS_21_BASE_IDX                                                           0
+#define regSPI_LB_CTR_CTRL                                                                              0x1274
+#define regSPI_LB_CTR_CTRL_BASE_IDX                                                                     0
+#define regSPI_LB_WGP_MASK                                                                              0x1275
+#define regSPI_LB_WGP_MASK_BASE_IDX                                                                     0
+#define regSPI_LB_DATA_REG                                                                              0x1276
+#define regSPI_LB_DATA_REG_BASE_IDX                                                                     0
+#define regSPI_PG_ENABLE_STATIC_WGP_MASK                                                                0x1277
+#define regSPI_PG_ENABLE_STATIC_WGP_MASK_BASE_IDX                                                       0
+#define regSPI_GDS_CREDITS                                                                              0x1278
+#define regSPI_GDS_CREDITS_BASE_IDX                                                                     0
+#define regSPI_SX_EXPORT_BUFFER_SIZES                                                                   0x1279
+#define regSPI_SX_EXPORT_BUFFER_SIZES_BASE_IDX                                                          0
+#define regSPI_SX_SCOREBOARD_BUFFER_SIZES                                                               0x127a
+#define regSPI_SX_SCOREBOARD_BUFFER_SIZES_BASE_IDX                                                      0
+#define regSPI_CSQ_WF_ACTIVE_STATUS                                                                     0x127b
+#define regSPI_CSQ_WF_ACTIVE_STATUS_BASE_IDX                                                            0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_0                                                                    0x127c
+#define regSPI_CSQ_WF_ACTIVE_COUNT_0_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_1                                                                    0x127d
+#define regSPI_CSQ_WF_ACTIVE_COUNT_1_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_2                                                                    0x127e
+#define regSPI_CSQ_WF_ACTIVE_COUNT_2_BASE_IDX                                                           0
+#define regSPI_CSQ_WF_ACTIVE_COUNT_3                                                                    0x127f
+#define regSPI_CSQ_WF_ACTIVE_COUNT_3_BASE_IDX                                                           0
+#define regSPI_LB_DATA_WAVES                                                                            0x1284
+#define regSPI_LB_DATA_WAVES_BASE_IDX                                                                   0
+#define regSPI_LB_DATA_PERWGP_WAVE_HSGS                                                                 0x1285
+#define regSPI_LB_DATA_PERWGP_WAVE_HSGS_BASE_IDX                                                        0
+#define regSPI_LB_DATA_PERWGP_WAVE_CS                                                                   0x1287
+#define regSPI_LB_DATA_PERWGP_WAVE_CS_BASE_IDX                                                          0
+#define regSPIS_DEBUG_READ                                                                              0x128a
+#define regSPIS_DEBUG_READ_BASE_IDX                                                                     0
+#define regBCI_DEBUG_READ                                                                               0x128b
+#define regBCI_DEBUG_READ_BASE_IDX                                                                      0
+#define regSPI_P0_TRAP_SCREEN_PSBA_LO                                                                   0x128c
+#define regSPI_P0_TRAP_SCREEN_PSBA_LO_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_PSBA_HI                                                                   0x128d
+#define regSPI_P0_TRAP_SCREEN_PSBA_HI_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_PSMA_LO                                                                   0x128e
+#define regSPI_P0_TRAP_SCREEN_PSMA_LO_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_PSMA_HI                                                                   0x128f
+#define regSPI_P0_TRAP_SCREEN_PSMA_HI_BASE_IDX                                                          0
+#define regSPI_P0_TRAP_SCREEN_GPR_MIN                                                                   0x1290
+#define regSPI_P0_TRAP_SCREEN_GPR_MIN_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSBA_LO                                                                   0x1291
+#define regSPI_P1_TRAP_SCREEN_PSBA_LO_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSBA_HI                                                                   0x1292
+#define regSPI_P1_TRAP_SCREEN_PSBA_HI_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSMA_LO                                                                   0x1293
+#define regSPI_P1_TRAP_SCREEN_PSMA_LO_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_PSMA_HI                                                                   0x1294
+#define regSPI_P1_TRAP_SCREEN_PSMA_HI_BASE_IDX                                                          0
+#define regSPI_P1_TRAP_SCREEN_GPR_MIN                                                                   0x1295
+#define regSPI_P1_TRAP_SCREEN_GPR_MIN_BASE_IDX                                                          0
+
+
+// addressBlock: gc_tpdec
+// base address: 0x9400
+#define regTD_CNTL                                                                                      0x12c5
+#define regTD_CNTL_BASE_IDX                                                                             0
+#define regTD_STATUS                                                                                    0x12c6
+#define regTD_STATUS_BASE_IDX                                                                           0
+#define regTD_POWER_CNTL                                                                                0x12ca
+#define regTD_POWER_CNTL_BASE_IDX                                                                       0
+#define regTD_CNTL2                                                                                     0x12cb
+#define regTD_CNTL2_BASE_IDX                                                                            0
+#define regTD_DSM_CNTL                                                                                  0x12cf
+#define regTD_DSM_CNTL_BASE_IDX                                                                         0
+#define regTD_DSM_CNTL2                                                                                 0x12d0
+#define regTD_DSM_CNTL2_BASE_IDX                                                                        0
+#define regTD_SCRATCH                                                                                   0x12d3
+#define regTD_SCRATCH_BASE_IDX                                                                          0
+#define regTA_CNTL                                                                                      0x12e1
+#define regTA_CNTL_BASE_IDX                                                                             0
+#define regTA_CNTL_AUX                                                                                  0x12e2
+#define regTA_CNTL_AUX_BASE_IDX                                                                         0
+#define regTA_CNTL2                                                                                     0x12e5
+#define regTA_CNTL2_BASE_IDX                                                                            0
+#define regTA_STATUS                                                                                    0x12e8
+#define regTA_STATUS_BASE_IDX                                                                           0
+#define regTA_SCRATCH                                                                                   0x1304
+#define regTA_SCRATCH_BASE_IDX                                                                          0
+
+
+// addressBlock: gc_gdsdec
+// base address: 0x9700
+#define regGDS_CONFIG                                                                                   0x1360
+#define regGDS_CONFIG_BASE_IDX                                                                          0
+#define regGDS_CNTL_STATUS                                                                              0x1361
+#define regGDS_CNTL_STATUS_BASE_IDX                                                                     0
+#define regGDS_ENHANCE                                                                                  0x1362
+#define regGDS_ENHANCE_BASE_IDX                                                                         0
+#define regGDS_PROTECTION_FAULT                                                                         0x1363
+#define regGDS_PROTECTION_FAULT_BASE_IDX                                                                0
+#define regGDS_VM_PROTECTION_FAULT                                                                      0x1364
+#define regGDS_VM_PROTECTION_FAULT_BASE_IDX                                                             0
+#define regGDS_EDC_CNT                                                                                  0x1365
+#define regGDS_EDC_CNT_BASE_IDX                                                                         0
+#define regGDS_EDC_GRBM_CNT                                                                             0x1366
+#define regGDS_EDC_GRBM_CNT_BASE_IDX                                                                    0
+#define regGDS_EDC_OA_DED                                                                               0x1367
+#define regGDS_EDC_OA_DED_BASE_IDX                                                                      0
+#define regGDS_DSM_CNTL                                                                                 0x136a
+#define regGDS_DSM_CNTL_BASE_IDX                                                                        0
+#define regGDS_EDC_OA_PHY_CNT                                                                           0x136b
+#define regGDS_EDC_OA_PHY_CNT_BASE_IDX                                                                  0
+#define regGDS_EDC_OA_PIPE_CNT                                                                          0x136c
+#define regGDS_EDC_OA_PIPE_CNT_BASE_IDX                                                                 0
+#define regGDS_DSM_CNTL2                                                                                0x136d
+#define regGDS_DSM_CNTL2_BASE_IDX                                                                       0
+
+
+// addressBlock: gc_rbdec
+// base address: 0x9800
+#define regDB_DEBUG                                                                                     0x13ac
+#define regDB_DEBUG_BASE_IDX                                                                            0
+#define regDB_DEBUG2                                                                                    0x13ad
+#define regDB_DEBUG2_BASE_IDX                                                                           0
+#define regDB_DEBUG3                                                                                    0x13ae
+#define regDB_DEBUG3_BASE_IDX                                                                           0
+#define regDB_DEBUG4                                                                                    0x13af
+#define regDB_DEBUG4_BASE_IDX                                                                           0
+#define regDB_ETILE_STUTTER_CONTROL                                                                     0x13b0
+#define regDB_ETILE_STUTTER_CONTROL_BASE_IDX                                                            0
+#define regDB_LTILE_STUTTER_CONTROL                                                                     0x13b1
+#define regDB_LTILE_STUTTER_CONTROL_BASE_IDX                                                            0
+#define regDB_EQUAD_STUTTER_CONTROL                                                                     0x13b2
+#define regDB_EQUAD_STUTTER_CONTROL_BASE_IDX                                                            0
+#define regDB_LQUAD_STUTTER_CONTROL                                                                     0x13b3
+#define regDB_LQUAD_STUTTER_CONTROL_BASE_IDX                                                            0
+#define regDB_CREDIT_LIMIT                                                                              0x13b4
+#define regDB_CREDIT_LIMIT_BASE_IDX                                                                     0
+#define regDB_WATERMARKS                                                                                0x13b5
+#define regDB_WATERMARKS_BASE_IDX                                                                       0
+#define regDB_SUBTILE_CONTROL                                                                           0x13b6
+#define regDB_SUBTILE_CONTROL_BASE_IDX                                                                  0
+#define regDB_FREE_CACHELINES                                                                           0x13b7
+#define regDB_FREE_CACHELINES_BASE_IDX                                                                  0
+#define regDB_FIFO_DEPTH1                                                                               0x13b8
+#define regDB_FIFO_DEPTH1_BASE_IDX                                                                      0
+#define regDB_FIFO_DEPTH2                                                                               0x13b9
+#define regDB_FIFO_DEPTH2_BASE_IDX                                                                      0
+#define regDB_LAST_OF_BURST_CONFIG                                                                      0x13ba
+#define regDB_LAST_OF_BURST_CONFIG_BASE_IDX                                                             0
+#define regDB_RING_CONTROL                                                                              0x13bb
+#define regDB_RING_CONTROL_BASE_IDX                                                                     0
+#define regDB_MEM_ARB_WATERMARKS                                                                        0x13bc
+#define regDB_MEM_ARB_WATERMARKS_BASE_IDX                                                               0
+#define regDB_FIFO_DEPTH3                                                                               0x13bd
+#define regDB_FIFO_DEPTH3_BASE_IDX                                                                      0
+#define regDB_DEBUG6                                                                                    0x13be
+#define regDB_DEBUG6_BASE_IDX                                                                           0
+#define regDB_EXCEPTION_CONTROL                                                                         0x13bf
+#define regDB_EXCEPTION_CONTROL_BASE_IDX                                                                0
+#define regDB_DEBUG7                                                                                    0x13d0
+#define regDB_DEBUG7_BASE_IDX                                                                           0
+#define regDB_DEBUG5                                                                                    0x13d1
+#define regDB_DEBUG5_BASE_IDX                                                                           0
+#define regDB_FGCG_SRAMS_CLK_CTRL                                                                       0x13d7
+#define regDB_FGCG_SRAMS_CLK_CTRL_BASE_IDX                                                              0
+#define regDB_FGCG_INTERFACES_CLK_CTRL                                                                  0x13d8
+#define regDB_FGCG_INTERFACES_CLK_CTRL_BASE_IDX                                                         0
+#define regDB_FIFO_DEPTH4                                                                               0x13d9
+#define regDB_FIFO_DEPTH4_BASE_IDX                                                                      0
+#define regCC_RB_REDUNDANCY                                                                             0x13dc
+#define regCC_RB_REDUNDANCY_BASE_IDX                                                                    0
+#define regCC_RB_BACKEND_DISABLE                                                                        0x13dd
+#define regCC_RB_BACKEND_DISABLE_BASE_IDX                                                               0
+#define regGB_ADDR_CONFIG                                                                               0x13de
+#define regGB_ADDR_CONFIG_BASE_IDX                                                                      0
+#define regGB_BACKEND_MAP                                                                               0x13df
+#define regGB_BACKEND_MAP_BASE_IDX                                                                      0
+#define regGB_GPU_ID                                                                                    0x13e0
+#define regGB_GPU_ID_BASE_IDX                                                                           0
+#define regCC_RB_DAISY_CHAIN                                                                            0x13e1
+#define regCC_RB_DAISY_CHAIN_BASE_IDX                                                                   0
+#define regGB_ADDR_CONFIG_READ                                                                          0x13e2
+#define regGB_ADDR_CONFIG_READ_BASE_IDX                                                                 0
+#define regCB_HW_CONTROL_4                                                                              0x1422
+#define regCB_HW_CONTROL_4_BASE_IDX                                                                     0
+#define regCB_HW_CONTROL_3                                                                              0x1423
+#define regCB_HW_CONTROL_3_BASE_IDX                                                                     0
+#define regCB_HW_CONTROL                                                                                0x1424
+#define regCB_HW_CONTROL_BASE_IDX                                                                       0
+#define regCB_HW_CONTROL_1                                                                              0x1425
+#define regCB_HW_CONTROL_1_BASE_IDX                                                                     0
+#define regCB_HW_CONTROL_2                                                                              0x1426
+#define regCB_HW_CONTROL_2_BASE_IDX                                                                     0
+#define regCB_DCC_CONFIG                                                                                0x1427
+#define regCB_DCC_CONFIG_BASE_IDX                                                                       0
+#define regCB_HW_MEM_ARBITER_RD                                                                         0x1428
+#define regCB_HW_MEM_ARBITER_RD_BASE_IDX                                                                0
+#define regCB_HW_MEM_ARBITER_WR                                                                         0x1429
+#define regCB_HW_MEM_ARBITER_WR_BASE_IDX                                                                0
+#define regCB_FGCG_SRAM_OVERRIDE                                                                        0x142a
+#define regCB_FGCG_SRAM_OVERRIDE_BASE_IDX                                                               0
+#define regCB_DCC_CONFIG2                                                                               0x142b
+#define regCB_DCC_CONFIG2_BASE_IDX                                                                      0
+#define regCHICKEN_BITS                                                                                 0x142d
+#define regCHICKEN_BITS_BASE_IDX                                                                        0
+#define regCB_CACHE_EVICT_POINTS                                                                        0x142e
+#define regCB_CACHE_EVICT_POINTS_BASE_IDX                                                               0
+
+
+// addressBlock: gc_gceadec
+// base address: 0xa800
+#define regGCEA_DRAM_RD_CLI2GRP_MAP0                                                                    0x17a0
+#define regGCEA_DRAM_RD_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regGCEA_DRAM_RD_CLI2GRP_MAP1                                                                    0x17a1
+#define regGCEA_DRAM_RD_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regGCEA_DRAM_WR_CLI2GRP_MAP0                                                                    0x17a2
+#define regGCEA_DRAM_WR_CLI2GRP_MAP0_BASE_IDX                                                           0
+#define regGCEA_DRAM_WR_CLI2GRP_MAP1                                                                    0x17a3
+#define regGCEA_DRAM_WR_CLI2GRP_MAP1_BASE_IDX                                                           0
+#define regGCEA_DRAM_RD_GRP2VC_MAP                                                                      0x17a4
+#define regGCEA_DRAM_RD_GRP2VC_MAP_BASE_IDX                                                             0
+#define regGCEA_DRAM_WR_GRP2VC_MAP                                                                      0x17a5
+#define regGCEA_DRAM_WR_GRP2VC_MAP_BASE_IDX                                                             0
+#define regGCEA_DRAM_RD_LAZY                                                                            0x17a6
+#define regGCEA_DRAM_RD_LAZY_BASE_IDX                                                                   0
+#define regGCEA_DRAM_WR_LAZY                                                                            0x17a7
+#define regGCEA_DRAM_WR_LAZY_BASE_IDX                                                                   0
+#define regGCEA_DRAM_RD_CAM_CNTL                                                                        0x17a8
+#define regGCEA_DRAM_RD_CAM_CNTL_BASE_IDX                                                               0
+#define regGCEA_DRAM_WR_CAM_CNTL                                                                        0x17a9
+#define regGCEA_DRAM_WR_CAM_CNTL_BASE_IDX                                                               0
+#define regGCEA_DRAM_PAGE_BURST                                                                         0x17aa
+#define regGCEA_DRAM_PAGE_BURST_BASE_IDX                                                                0
+#define regGCEA_DRAM_RD_PRI_AGE                                                                         0x17ab
+#define regGCEA_DRAM_RD_PRI_AGE_BASE_IDX                                                                0
+#define regGCEA_DRAM_WR_PRI_AGE                                                                         0x17ac
+#define regGCEA_DRAM_WR_PRI_AGE_BASE_IDX                                                                0
+#define regGCEA_DRAM_RD_PRI_QUEUING                                                                     0x17ad
+#define regGCEA_DRAM_RD_PRI_QUEUING_BASE_IDX                                                            0
+#define regGCEA_DRAM_WR_PRI_QUEUING                                                                     0x17ae
+#define regGCEA_DRAM_WR_PRI_QUEUING_BASE_IDX                                                            0
+#define regGCEA_DRAM_RD_PRI_FIXED                                                                       0x17af
+#define regGCEA_DRAM_RD_PRI_FIXED_BASE_IDX                                                              0
+#define regGCEA_DRAM_WR_PRI_FIXED                                                                       0x17b0
+#define regGCEA_DRAM_WR_PRI_FIXED_BASE_IDX                                                              0
+#define regGCEA_DRAM_RD_PRI_URGENCY                                                                     0x17b1
+#define regGCEA_DRAM_RD_PRI_URGENCY_BASE_IDX                                                            0
+#define regGCEA_DRAM_WR_PRI_URGENCY                                                                     0x17b2
+#define regGCEA_DRAM_WR_PRI_URGENCY_BASE_IDX                                                            0
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI1                                                                  0x17b3
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI2                                                                  0x17b4
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI3                                                                  0x17b5
+#define regGCEA_DRAM_RD_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI1                                                                  0x17b6
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI1_BASE_IDX                                                         0
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI2                                                                  0x17b7
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI2_BASE_IDX                                                         0
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI3                                                                  0x17b8
+#define regGCEA_DRAM_WR_PRI_QUANT_PRI3_BASE_IDX                                                         0
+#define regGCEA_IO_RD_CLI2GRP_MAP0                                                                      0x187d
+#define regGCEA_IO_RD_CLI2GRP_MAP0_BASE_IDX                                                             0
+#define regGCEA_IO_RD_CLI2GRP_MAP1                                                                      0x187e
+#define regGCEA_IO_RD_CLI2GRP_MAP1_BASE_IDX                                                             0
+#define regGCEA_IO_WR_CLI2GRP_MAP0                                                                      0x187f
+#define regGCEA_IO_WR_CLI2GRP_MAP0_BASE_IDX                                                             0
+#define regGCEA_IO_WR_CLI2GRP_MAP1                                                                      0x1880
+#define regGCEA_IO_WR_CLI2GRP_MAP1_BASE_IDX                                                             0
+#define regGCEA_IO_RD_COMBINE_FLUSH                                                                     0x1881
+#define regGCEA_IO_RD_COMBINE_FLUSH_BASE_IDX                                                            0
+#define regGCEA_IO_WR_COMBINE_FLUSH                                                                     0x1882
+#define regGCEA_IO_WR_COMBINE_FLUSH_BASE_IDX                                                            0
+#define regGCEA_IO_GROUP_BURST                                                                          0x1883
+#define regGCEA_IO_GROUP_BURST_BASE_IDX                                                                 0
+#define regGCEA_IO_RD_PRI_AGE                                                                           0x1884
+#define regGCEA_IO_RD_PRI_AGE_BASE_IDX                                                                  0
+#define regGCEA_IO_WR_PRI_AGE                                                                           0x1885
+#define regGCEA_IO_WR_PRI_AGE_BASE_IDX                                                                  0
+#define regGCEA_IO_RD_PRI_QUEUING                                                                       0x1886
+#define regGCEA_IO_RD_PRI_QUEUING_BASE_IDX                                                              0
+#define regGCEA_IO_WR_PRI_QUEUING                                                                       0x1887
+#define regGCEA_IO_WR_PRI_QUEUING_BASE_IDX                                                              0
+#define regGCEA_IO_RD_PRI_FIXED                                                                         0x1888
+#define regGCEA_IO_RD_PRI_FIXED_BASE_IDX                                                                0
+#define regGCEA_IO_WR_PRI_FIXED                                                                         0x1889
+#define regGCEA_IO_WR_PRI_FIXED_BASE_IDX                                                                0
+#define regGCEA_IO_RD_PRI_URGENCY                                                                       0x188a
+#define regGCEA_IO_RD_PRI_URGENCY_BASE_IDX                                                              0
+#define regGCEA_IO_WR_PRI_URGENCY                                                                       0x188b
+#define regGCEA_IO_WR_PRI_URGENCY_BASE_IDX                                                              0
+#define regGCEA_IO_RD_PRI_URGENCY_MASKING                                                               0x188c
+#define regGCEA_IO_RD_PRI_URGENCY_MASKING_BASE_IDX                                                      0
+#define regGCEA_IO_WR_PRI_URGENCY_MASKING                                                               0x188d
+#define regGCEA_IO_WR_PRI_URGENCY_MASKING_BASE_IDX                                                      0
+#define regGCEA_IO_RD_PRI_QUANT_PRI1                                                                    0x188e
+#define regGCEA_IO_RD_PRI_QUANT_PRI1_BASE_IDX                                                           0
+#define regGCEA_IO_RD_PRI_QUANT_PRI2                                                                    0x188f
+#define regGCEA_IO_RD_PRI_QUANT_PRI2_BASE_IDX                                                           0
+#define regGCEA_IO_RD_PRI_QUANT_PRI3                                                                    0x1890
+#define regGCEA_IO_RD_PRI_QUANT_PRI3_BASE_IDX                                                           0
+#define regGCEA_IO_WR_PRI_QUANT_PRI1                                                                    0x1891
+#define regGCEA_IO_WR_PRI_QUANT_PRI1_BASE_IDX                                                           0
+#define regGCEA_IO_WR_PRI_QUANT_PRI2                                                                    0x1892
+#define regGCEA_IO_WR_PRI_QUANT_PRI2_BASE_IDX                                                           0
+#define regGCEA_IO_WR_PRI_QUANT_PRI3                                                                    0x1893
+#define regGCEA_IO_WR_PRI_QUANT_PRI3_BASE_IDX                                                           0
+#define regGCEA_SDP_ARB_DRAM                                                                            0x1894
+#define regGCEA_SDP_ARB_DRAM_BASE_IDX                                                                   0
+#define regGCEA_SDP_ARB_FINAL                                                                           0x1896
+#define regGCEA_SDP_ARB_FINAL_BASE_IDX                                                                  0
+#define regGCEA_SDP_DRAM_PRIORITY                                                                       0x1897
+#define regGCEA_SDP_DRAM_PRIORITY_BASE_IDX                                                              0
+#define regGCEA_SDP_IO_PRIORITY                                                                         0x1899
+#define regGCEA_SDP_IO_PRIORITY_BASE_IDX                                                                0
+#define regGCEA_SDP_CREDITS                                                                             0x189a
+#define regGCEA_SDP_CREDITS_BASE_IDX                                                                    0
+#define regGCEA_SDP_TAG_RESERVE0                                                                        0x189b
+#define regGCEA_SDP_TAG_RESERVE0_BASE_IDX                                                               0
+#define regGCEA_SDP_TAG_RESERVE1                                                                        0x189c
+#define regGCEA_SDP_TAG_RESERVE1_BASE_IDX                                                               0
+#define regGCEA_SDP_VCC_RESERVE0                                                                        0x189d
+#define regGCEA_SDP_VCC_RESERVE0_BASE_IDX                                                               0
+#define regGCEA_SDP_VCC_RESERVE1                                                                        0x189e
+#define regGCEA_SDP_VCC_RESERVE1_BASE_IDX                                                               0
+#define regGCEA_SDP_VCD_RESERVE0                                                                        0x189f
+#define regGCEA_SDP_VCD_RESERVE0_BASE_IDX                                                               0
+
+
+// addressBlock: gc_gceadec2
+// base address: 0x9c00
+#define regGCEA_SDP_VCD_RESERVE1                                                                        0x14a0
+#define regGCEA_SDP_VCD_RESERVE1_BASE_IDX                                                               0
+#define regGCEA_SDP_REQ_CNTL                                                                            0x14a1
+#define regGCEA_SDP_REQ_CNTL_BASE_IDX                                                                   0
+#define regGCEA_MISC                                                                                    0x14a2
+#define regGCEA_MISC_BASE_IDX                                                                           0
+#define regGCEA_LATENCY_SAMPLING                                                                        0x14a3
+#define regGCEA_LATENCY_SAMPLING_BASE_IDX                                                               0
+#define regGCEA_MAM_CTRL2                                                                               0x14a9
+#define regGCEA_MAM_CTRL2_BASE_IDX                                                                      0
+#define regGCEA_MAM_CTRL                                                                                0x14ab
+#define regGCEA_MAM_CTRL_BASE_IDX                                                                       0
+#define regGCEA_EDC_CNT                                                                                 0x14b2
+#define regGCEA_EDC_CNT_BASE_IDX                                                                        0
+#define regGCEA_EDC_CNT2                                                                                0x14b3
+#define regGCEA_EDC_CNT2_BASE_IDX                                                                       0
+#define regGCEA_DSM_CNTL                                                                                0x14b4
+#define regGCEA_DSM_CNTL_BASE_IDX                                                                       0
+#define regGCEA_DSM_CNTLA                                                                               0x14b5
+#define regGCEA_DSM_CNTLA_BASE_IDX                                                                      0
+#define regGCEA_DSM_CNTLB                                                                               0x14b6
+#define regGCEA_DSM_CNTLB_BASE_IDX                                                                      0
+#define regGCEA_DSM_CNTL2                                                                               0x14b7
+#define regGCEA_DSM_CNTL2_BASE_IDX                                                                      0
+#define regGCEA_DSM_CNTL2A                                                                              0x14b8
+#define regGCEA_DSM_CNTL2A_BASE_IDX                                                                     0
+#define regGCEA_DSM_CNTL2B                                                                              0x14b9
+#define regGCEA_DSM_CNTL2B_BASE_IDX                                                                     0
+#define regGCEA_GL2C_XBR_CREDITS                                                                        0x14ba
+#define regGCEA_GL2C_XBR_CREDITS_BASE_IDX                                                               0
+#define regGCEA_GL2C_XBR_MAXBURST                                                                       0x14bb
+#define regGCEA_GL2C_XBR_MAXBURST_BASE_IDX                                                              0
+#define regGCEA_PROBE_CNTL                                                                              0x14bc
+#define regGCEA_PROBE_CNTL_BASE_IDX                                                                     0
+#define regGCEA_PROBE_MAP                                                                               0x14bd
+#define regGCEA_PROBE_MAP_BASE_IDX                                                                      0
+#define regGCEA_ERR_STATUS                                                                              0x14be
+#define regGCEA_ERR_STATUS_BASE_IDX                                                                     0
+#define regGCEA_MISC2                                                                                   0x14bf
+#define regGCEA_MISC2_BASE_IDX                                                                          0
+
+
+// addressBlock: gc_gceadec3
+// base address: 0x9dc0
+#define regGCEA_SDP_BACKDOOR_CMDCREDITS0                                                                0x1512
+#define regGCEA_SDP_BACKDOOR_CMDCREDITS0_BASE_IDX                                                       0
+#define regGCEA_SDP_BACKDOOR_CMDCREDITS1                                                                0x1513
+#define regGCEA_SDP_BACKDOOR_CMDCREDITS1_BASE_IDX                                                       0
+#define regGCEA_SDP_BACKDOOR_DATACREDITS0                                                               0x1514
+#define regGCEA_SDP_BACKDOOR_DATACREDITS0_BASE_IDX                                                      0
+#define regGCEA_SDP_BACKDOOR_DATACREDITS1                                                               0x1515
+#define regGCEA_SDP_BACKDOOR_DATACREDITS1_BASE_IDX                                                      0
+#define regGCEA_SDP_BACKDOOR_MISCCREDITS                                                                0x1516
+#define regGCEA_SDP_BACKDOOR_MISCCREDITS_BASE_IDX                                                       0
+#define regGCEA_RRET_MEM_RESERVE                                                                        0x1518
+#define regGCEA_RRET_MEM_RESERVE_BASE_IDX                                                               0
+#define regGCEA_EDC_CNT3                                                                                0x151a
+#define regGCEA_EDC_CNT3_BASE_IDX                                                                       0
+#define regGCEA_SDP_ENABLE                                                                              0x151e
+#define regGCEA_SDP_ENABLE_BASE_IDX                                                                     0
+
+
+// addressBlock: gc_spipdec2
+// base address: 0x9c80
+#define regSPI_PQEV_CTRL                                                                                0x14c0
+#define regSPI_PQEV_CTRL_BASE_IDX                                                                       0
+#define regSPI_EXP_THROTTLE_CTRL                                                                        0x14c3
+#define regSPI_EXP_THROTTLE_CTRL_BASE_IDX                                                               0
+
+
+// addressBlock: gc_rmi_rmidec
+// base address: 0x2e200
+#define regRMI_GENERAL_CNTL                                                                             0x1880
+#define regRMI_GENERAL_CNTL_BASE_IDX                                                                    1
+#define regRMI_GENERAL_CNTL1                                                                            0x1881
+#define regRMI_GENERAL_CNTL1_BASE_IDX                                                                   1
+#define regRMI_GENERAL_STATUS                                                                           0x1882
+#define regRMI_GENERAL_STATUS_BASE_IDX                                                                  1
+#define regRMI_SUBBLOCK_STATUS0                                                                         0x1883
+#define regRMI_SUBBLOCK_STATUS0_BASE_IDX                                                                1
+#define regRMI_SUBBLOCK_STATUS1                                                                         0x1884
+#define regRMI_SUBBLOCK_STATUS1_BASE_IDX                                                                1
+#define regRMI_SUBBLOCK_STATUS2                                                                         0x1885
+#define regRMI_SUBBLOCK_STATUS2_BASE_IDX                                                                1
+#define regRMI_SUBBLOCK_STATUS3                                                                         0x1886
+#define regRMI_SUBBLOCK_STATUS3_BASE_IDX                                                                1
+#define regRMI_XBAR_CONFIG                                                                              0x1887
+#define regRMI_XBAR_CONFIG_BASE_IDX                                                                     1
+#define regRMI_PROBE_POP_LOGIC_CNTL                                                                     0x1888
+#define regRMI_PROBE_POP_LOGIC_CNTL_BASE_IDX                                                            1
+#define regRMI_UTC_XNACK_N_MISC_CNTL                                                                    0x1889
+#define regRMI_UTC_XNACK_N_MISC_CNTL_BASE_IDX                                                           1
+#define regRMI_DEMUX_CNTL                                                                               0x188a
+#define regRMI_DEMUX_CNTL_BASE_IDX                                                                      1
+#define regRMI_UTCL1_CNTL1                                                                              0x188b
+#define regRMI_UTCL1_CNTL1_BASE_IDX                                                                     1
+#define regRMI_UTCL1_CNTL2                                                                              0x188c
+#define regRMI_UTCL1_CNTL2_BASE_IDX                                                                     1
+#define regRMI_UTC_UNIT_CONFIG                                                                          0x188d
+#define regRMI_UTC_UNIT_CONFIG_BASE_IDX                                                                 1
+#define regRMI_TCIW_FORMATTER0_CNTL                                                                     0x188e
+#define regRMI_TCIW_FORMATTER0_CNTL_BASE_IDX                                                            1
+#define regRMI_TCIW_FORMATTER1_CNTL                                                                     0x188f
+#define regRMI_TCIW_FORMATTER1_CNTL_BASE_IDX                                                            1
+#define regRMI_SCOREBOARD_CNTL                                                                          0x1890
+#define regRMI_SCOREBOARD_CNTL_BASE_IDX                                                                 1
+#define regRMI_SCOREBOARD_STATUS0                                                                       0x1891
+#define regRMI_SCOREBOARD_STATUS0_BASE_IDX                                                              1
+#define regRMI_SCOREBOARD_STATUS1                                                                       0x1892
+#define regRMI_SCOREBOARD_STATUS1_BASE_IDX                                                              1
+#define regRMI_SCOREBOARD_STATUS2                                                                       0x1893
+#define regRMI_SCOREBOARD_STATUS2_BASE_IDX                                                              1
+#define regRMI_XBAR_ARBITER_CONFIG                                                                      0x1894
+#define regRMI_XBAR_ARBITER_CONFIG_BASE_IDX                                                             1
+#define regRMI_XBAR_ARBITER_CONFIG_1                                                                    0x1895
+#define regRMI_XBAR_ARBITER_CONFIG_1_BASE_IDX                                                           1
+#define regRMI_CLOCK_CNTRL                                                                              0x1896
+#define regRMI_CLOCK_CNTRL_BASE_IDX                                                                     1
+#define regRMI_UTCL1_STATUS                                                                             0x1897
+#define regRMI_UTCL1_STATUS_BASE_IDX                                                                    1
+#define regRMI_RB_GLX_CID_MAP                                                                           0x1898
+#define regRMI_RB_GLX_CID_MAP_BASE_IDX                                                                  1
+#define regRMI_XNACK_DEBUG                                                                              0x189e
+#define regRMI_XNACK_DEBUG_BASE_IDX                                                                     1
+#define regRMI_SPARE                                                                                    0x189f
+#define regRMI_SPARE_BASE_IDX                                                                           1
+#define regRMI_SPARE_1                                                                                  0x18a0
+#define regRMI_SPARE_1_BASE_IDX                                                                         1
+#define regRMI_SPARE_2                                                                                  0x18a1
+#define regRMI_SPARE_2_BASE_IDX                                                                         1
+#define regCC_RMI_REDUNDANCY                                                                            0x18a2
+#define regCC_RMI_REDUNDANCY_BASE_IDX                                                                   1
+
+
+// addressBlock: gc_pmmdec
+// base address: 0x9f80
+#define regGCR_PIO_CNTL                                                                                 0x1580
+#define regGCR_PIO_CNTL_BASE_IDX                                                                        0
+#define regGCR_PIO_DATA                                                                                 0x1581
+#define regGCR_PIO_DATA_BASE_IDX                                                                        0
+#define regPMM_CNTL                                                                                     0x1582
+#define regPMM_CNTL_BASE_IDX                                                                            0
+#define regPMM_STATUS                                                                                   0x1583
+#define regPMM_STATUS_BASE_IDX                                                                          0
+
+
+// addressBlock: gc_utcl1dec
+// base address: 0x9fb0
+#define regUTCL1_CTRL_1                                                                                 0x158c
+#define regUTCL1_CTRL_1_BASE_IDX                                                                        0
+#define regUTCL1_ALOG                                                                                   0x158f
+#define regUTCL1_ALOG_BASE_IDX                                                                          0
+#define regUTCL1_STATUS                                                                                 0x1594
+#define regUTCL1_STATUS_BASE_IDX                                                                        0
+
+
+// addressBlock: gc_gcvmsharedpfdec
+// base address: 0xa000
+#define regGCMC_VM_NB_MMIOBASE                                                                          0x15a0
+#define regGCMC_VM_NB_MMIOBASE_BASE_IDX                                                                 0
+#define regGCMC_VM_NB_MMIOLIMIT                                                                         0x15a1
+#define regGCMC_VM_NB_MMIOLIMIT_BASE_IDX                                                                0
+#define regGCMC_VM_NB_PCI_CTRL                                                                          0x15a2
+#define regGCMC_VM_NB_PCI_CTRL_BASE_IDX                                                                 0
+#define regGCMC_VM_NB_PCI_ARB                                                                           0x15a3
+#define regGCMC_VM_NB_PCI_ARB_BASE_IDX                                                                  0
+#define regGCMC_VM_NB_TOP_OF_DRAM_SLOT1                                                                 0x15a4
+#define regGCMC_VM_NB_TOP_OF_DRAM_SLOT1_BASE_IDX                                                        0
+#define regGCMC_VM_NB_LOWER_TOP_OF_DRAM2                                                                0x15a5
+#define regGCMC_VM_NB_LOWER_TOP_OF_DRAM2_BASE_IDX                                                       0
+#define regGCMC_VM_NB_UPPER_TOP_OF_DRAM2                                                                0x15a6
+#define regGCMC_VM_NB_UPPER_TOP_OF_DRAM2_BASE_IDX                                                       0
+#define regGCMC_VM_FB_OFFSET                                                                            0x15a7
+#define regGCMC_VM_FB_OFFSET_BASE_IDX                                                                   0
+#define regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB                                                     0x15a8
+#define regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB_BASE_IDX                                            0
+#define regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB                                                     0x15a9
+#define regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB_BASE_IDX                                            0
+#define regGCMC_VM_STEERING                                                                             0x15aa
+#define regGCMC_VM_STEERING_BASE_IDX                                                                    0
+#define regGCMC_SHARED_VIRT_RESET_REQ                                                                   0x15ab
+#define regGCMC_SHARED_VIRT_RESET_REQ_BASE_IDX                                                          0
+#define regGCMC_MEM_POWER_LS                                                                            0x15ac
+#define regGCMC_MEM_POWER_LS_BASE_IDX                                                                   0
+#define regGCMC_VM_CACHEABLE_DRAM_ADDRESS_START                                                         0x15ad
+#define regGCMC_VM_CACHEABLE_DRAM_ADDRESS_START_BASE_IDX                                                0
+#define regGCMC_VM_CACHEABLE_DRAM_ADDRESS_END                                                           0x15ae
+#define regGCMC_VM_CACHEABLE_DRAM_ADDRESS_END_BASE_IDX                                                  0
+#define regGCMC_VM_LOCAL_SYSMEM_ADDRESS_START                                                           0x15af
+#define regGCMC_VM_LOCAL_SYSMEM_ADDRESS_START_BASE_IDX                                                  0
+#define regGCMC_VM_LOCAL_SYSMEM_ADDRESS_END                                                             0x15b0
+#define regGCMC_VM_LOCAL_SYSMEM_ADDRESS_END_BASE_IDX                                                    0
+#define regGCMC_VM_APT_CNTL                                                                             0x15b1
+#define regGCMC_VM_APT_CNTL_BASE_IDX                                                                    0
+#define regGCMC_VM_LOCAL_FB_ADDRESS_START                                                               0x15b2
+#define regGCMC_VM_LOCAL_FB_ADDRESS_START_BASE_IDX                                                      0
+#define regGCMC_VM_LOCAL_FB_ADDRESS_END                                                                 0x15b3
+#define regGCMC_VM_LOCAL_FB_ADDRESS_END_BASE_IDX                                                        0
+#define regGCMC_VM_LOCAL_FB_ADDRESS_LOCK_CNTL                                                           0x15b4
+#define regGCMC_VM_LOCAL_FB_ADDRESS_LOCK_CNTL_BASE_IDX                                                  0
+#define regGCUTCL2_ICG_CTRL                                                                             0x15b5
+#define regGCUTCL2_ICG_CTRL_BASE_IDX                                                                    0
+#define regGCMC_SHARED_ACTIVE_FCN_ID                                                                    0x15b6
+#define regGCMC_SHARED_ACTIVE_FCN_ID_BASE_IDX                                                           0
+#define regGCUTCL2_CGTT_BUSY_CTRL                                                                       0x15b7
+#define regGCUTCL2_CGTT_BUSY_CTRL_BASE_IDX                                                              0
+#define regGCMC_VM_FB_NOALLOC_CNTL                                                                      0x15b8
+#define regGCMC_VM_FB_NOALLOC_CNTL_BASE_IDX                                                             0
+#define regGCUTCL2_HARVEST_BYPASS_GROUPS                                                                0x15b9
+#define regGCUTCL2_HARVEST_BYPASS_GROUPS_BASE_IDX                                                       0
+#define regGCUTCL2_GROUP_RET_FAULT_STATUS                                                               0x15bb
+#define regGCUTCL2_GROUP_RET_FAULT_STATUS_BASE_IDX                                                      0
+
+
+// addressBlock: gc_gcvml2pfdec
+// base address: 0xa080
+#define regGCVM_L2_CNTL                                                                                 0x15c0
+#define regGCVM_L2_CNTL_BASE_IDX                                                                        0
+#define regGCVM_L2_CNTL2                                                                                0x15c1
+#define regGCVM_L2_CNTL2_BASE_IDX                                                                       0
+#define regGCVM_L2_CNTL3                                                                                0x15c2
+#define regGCVM_L2_CNTL3_BASE_IDX                                                                       0
+#define regGCVM_L2_STATUS                                                                               0x15c3
+#define regGCVM_L2_STATUS_BASE_IDX                                                                      0
+#define regGCVM_DUMMY_PAGE_FAULT_CNTL                                                                   0x15c4
+#define regGCVM_DUMMY_PAGE_FAULT_CNTL_BASE_IDX                                                          0
+#define regGCVM_DUMMY_PAGE_FAULT_ADDR_LO32                                                              0x15c5
+#define regGCVM_DUMMY_PAGE_FAULT_ADDR_LO32_BASE_IDX                                                     0
+#define regGCVM_DUMMY_PAGE_FAULT_ADDR_HI32                                                              0x15c6
+#define regGCVM_DUMMY_PAGE_FAULT_ADDR_HI32_BASE_IDX                                                     0
+#define regGCVM_INVALIDATE_CNTL                                                                         0x15c7
+#define regGCVM_INVALIDATE_CNTL_BASE_IDX                                                                0
+#define regGCVM_L2_PROTECTION_FAULT_CNTL                                                                0x15c8
+#define regGCVM_L2_PROTECTION_FAULT_CNTL_BASE_IDX                                                       0
+#define regGCVM_L2_PROTECTION_FAULT_CNTL2                                                               0x15c9
+#define regGCVM_L2_PROTECTION_FAULT_CNTL2_BASE_IDX                                                      0
+#define regGCVM_L2_PROTECTION_FAULT_MM_CNTL3                                                            0x15ca
+#define regGCVM_L2_PROTECTION_FAULT_MM_CNTL3_BASE_IDX                                                   0
+#define regGCVM_L2_PROTECTION_FAULT_MM_CNTL4                                                            0x15cb
+#define regGCVM_L2_PROTECTION_FAULT_MM_CNTL4_BASE_IDX                                                   0
+#define regGCVM_L2_PROTECTION_FAULT_STATUS                                                              0x15cc
+#define regGCVM_L2_PROTECTION_FAULT_STATUS_BASE_IDX                                                     0
+#define regGCVM_L2_PROTECTION_FAULT_ADDR_LO32                                                           0x15cd
+#define regGCVM_L2_PROTECTION_FAULT_ADDR_LO32_BASE_IDX                                                  0
+#define regGCVM_L2_PROTECTION_FAULT_ADDR_HI32                                                           0x15ce
+#define regGCVM_L2_PROTECTION_FAULT_ADDR_HI32_BASE_IDX                                                  0
+#define regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32                                                   0x15cf
+#define regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32_BASE_IDX                                          0
+#define regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32                                                   0x15d0
+#define regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32_BASE_IDX                                          0
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32                                             0x15d2
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32_BASE_IDX                                    0
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32                                             0x15d3
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32_BASE_IDX                                    0
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32                                            0x15d4
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32_BASE_IDX                                   0
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32                                            0x15d5
+#define regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32_BASE_IDX                                   0
+#define regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32                                                0x15d6
+#define regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32_BASE_IDX                                       0
+#define regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32                                                0x15d7
+#define regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32_BASE_IDX                                       0
+#define regGCVM_L2_CNTL4                                                                                0x15d8
+#define regGCVM_L2_CNTL4_BASE_IDX                                                                       0
+#define regGCVM_L2_MM_GROUP_RT_CLASSES                                                                  0x15d9
+#define regGCVM_L2_MM_GROUP_RT_CLASSES_BASE_IDX                                                         0
+#define regGCVM_L2_BANK_SELECT_RESERVED_CID                                                             0x15da
+#define regGCVM_L2_BANK_SELECT_RESERVED_CID_BASE_IDX                                                    0
+#define regGCVM_L2_BANK_SELECT_RESERVED_CID2                                                            0x15db
+#define regGCVM_L2_BANK_SELECT_RESERVED_CID2_BASE_IDX                                                   0
+#define regGCVM_L2_CACHE_PARITY_CNTL                                                                    0x15dc
+#define regGCVM_L2_CACHE_PARITY_CNTL_BASE_IDX                                                           0
+#define regGCVM_L2_ICG_CTRL                                                                             0x15dd
+#define regGCVM_L2_ICG_CTRL_BASE_IDX                                                                    0
+#define regGCVM_L2_CNTL5                                                                                0x15de
+#define regGCVM_L2_CNTL5_BASE_IDX                                                                       0
+#define regGCVM_L2_GCR_CNTL                                                                             0x15df
+#define regGCVM_L2_GCR_CNTL_BASE_IDX                                                                    0
+#define regGCVML2_WALKER_MACRO_THROTTLE_TIME                                                            0x15e0
+#define regGCVML2_WALKER_MACRO_THROTTLE_TIME_BASE_IDX                                                   0
+#define regGCVML2_WALKER_MACRO_THROTTLE_FETCH_LIMIT                                                     0x15e1
+#define regGCVML2_WALKER_MACRO_THROTTLE_FETCH_LIMIT_BASE_IDX                                            0
+#define regGCVML2_WALKER_MICRO_THROTTLE_TIME                                                            0x15e2
+#define regGCVML2_WALKER_MICRO_THROTTLE_TIME_BASE_IDX                                                   0
+#define regGCVML2_WALKER_MICRO_THROTTLE_FETCH_LIMIT                                                     0x15e3
+#define regGCVML2_WALKER_MICRO_THROTTLE_FETCH_LIMIT_BASE_IDX                                            0
+#define regGCVM_L2_CGTT_BUSY_CTRL                                                                       0x15e4
+#define regGCVM_L2_CGTT_BUSY_CTRL_BASE_IDX                                                              0
+#define regGCVM_L2_PTE_CACHE_DUMP_CNTL                                                                  0x15e5
+#define regGCVM_L2_PTE_CACHE_DUMP_CNTL_BASE_IDX                                                         0
+#define regGCVM_L2_PTE_CACHE_DUMP_READ                                                                  0x15e6
+#define regGCVM_L2_PTE_CACHE_DUMP_READ_BASE_IDX                                                         0
+#define regGCVM_L2_BANK_SELECT_MASKS                                                                    0x15e9
+#define regGCVM_L2_BANK_SELECT_MASKS_BASE_IDX                                                           0
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC                                                          0x15ea
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC_BASE_IDX                                                 0
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC                                               0x15eb
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC_BASE_IDX                                      0
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC                                             0x15ec
+#define regGCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC_BASE_IDX                                    0
+#define regGCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT                                                      0x15ed
+#define regGCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT_BASE_IDX                                             0
+#define regGCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ                                                      0x15ee
+#define regGCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ_BASE_IDX                                             0
+
+
+// addressBlock: gc_gcatcl2dec
+// base address: 0xa300
+#define regGC_ATC_L2_CNTL                                                                               0x1660
+#define regGC_ATC_L2_CNTL_BASE_IDX                                                                      0
+#define regGC_ATC_L2_CNTL2                                                                              0x1661
+#define regGC_ATC_L2_CNTL2_BASE_IDX                                                                     0
+#define regGC_ATC_L2_CACHE_DATA0                                                                        0x1664
+#define regGC_ATC_L2_CACHE_DATA0_BASE_IDX                                                               0
+#define regGC_ATC_L2_CACHE_DATA1                                                                        0x1665
+#define regGC_ATC_L2_CACHE_DATA1_BASE_IDX                                                               0
+#define regGC_ATC_L2_CACHE_DATA2                                                                        0x1666
+#define regGC_ATC_L2_CACHE_DATA2_BASE_IDX                                                               0
+#define regGC_ATC_L2_CNTL3                                                                              0x1667
+#define regGC_ATC_L2_CNTL3_BASE_IDX                                                                     0
+#define regGC_ATC_L2_STATUS                                                                             0x1668
+#define regGC_ATC_L2_STATUS_BASE_IDX                                                                    0
+#define regGC_ATC_L2_STATUS2                                                                            0x1669
+#define regGC_ATC_L2_STATUS2_BASE_IDX                                                                   0
+#define regGC_ATC_L2_MISC_CG                                                                            0x166a
+#define regGC_ATC_L2_MISC_CG_BASE_IDX                                                                   0
+#define regGC_ATC_L2_MEM_POWER_LS                                                                       0x166b
+#define regGC_ATC_L2_MEM_POWER_LS_BASE_IDX                                                              0
+#define regGC_ATC_L2_SDPPORT_CTRL                                                                       0x166f
+#define regGC_ATC_L2_SDPPORT_CTRL_BASE_IDX                                                              0
+
+
+// addressBlock: gc_gcl2tlbpfdec
+// base address: 0xa380
+#define regGCL2TLB_TLB0_STATUS                                                                          0x1681
+#define regGCL2TLB_TLB0_STATUS_BASE_IDX                                                                 0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO                                               0x1683
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO_BASE_IDX                                      0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI                                               0x1684
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI_BASE_IDX                                      0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO                                              0x1685
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO_BASE_IDX                                     0
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI                                              0x1686
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI_BASE_IDX                                     0
+
+
+// addressBlock: gc_gcvmsharedvcdec
+// base address: 0xa3a0
+#define regGCMC_VM_FB_LOCATION_BASE                                                                     0x1688
+#define regGCMC_VM_FB_LOCATION_BASE_BASE_IDX                                                            0
+#define regGCMC_VM_FB_LOCATION_TOP                                                                      0x1689
+#define regGCMC_VM_FB_LOCATION_TOP_BASE_IDX                                                             0
+#define regGCMC_VM_AGP_TOP                                                                              0x168a
+#define regGCMC_VM_AGP_TOP_BASE_IDX                                                                     0
+#define regGCMC_VM_AGP_BOT                                                                              0x168b
+#define regGCMC_VM_AGP_BOT_BASE_IDX                                                                     0
+#define regGCMC_VM_AGP_BASE                                                                             0x168c
+#define regGCMC_VM_AGP_BASE_BASE_IDX                                                                    0
+#define regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR                                                             0x168d
+#define regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR_BASE_IDX                                                    0
+#define regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR                                                            0x168e
+#define regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR_BASE_IDX                                                   0
+#define regGCMC_VM_MX_L1_TLB_CNTL                                                                       0x168f
+#define regGCMC_VM_MX_L1_TLB_CNTL_BASE_IDX                                                              0
+
+
+// addressBlock: gc_gcvml2vcdec
+// base address: 0xa3e0
+#define regGCVM_CONTEXT0_CNTL                                                                           0x1698
+#define regGCVM_CONTEXT0_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT1_CNTL                                                                           0x1699
+#define regGCVM_CONTEXT1_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT2_CNTL                                                                           0x169a
+#define regGCVM_CONTEXT2_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT3_CNTL                                                                           0x169b
+#define regGCVM_CONTEXT3_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT4_CNTL                                                                           0x169c
+#define regGCVM_CONTEXT4_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT5_CNTL                                                                           0x169d
+#define regGCVM_CONTEXT5_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT6_CNTL                                                                           0x169e
+#define regGCVM_CONTEXT6_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT7_CNTL                                                                           0x169f
+#define regGCVM_CONTEXT7_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT8_CNTL                                                                           0x16a0
+#define regGCVM_CONTEXT8_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT9_CNTL                                                                           0x16a1
+#define regGCVM_CONTEXT9_CNTL_BASE_IDX                                                                  0
+#define regGCVM_CONTEXT10_CNTL                                                                          0x16a2
+#define regGCVM_CONTEXT10_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXT11_CNTL                                                                          0x16a3
+#define regGCVM_CONTEXT11_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXT12_CNTL                                                                          0x16a4
+#define regGCVM_CONTEXT12_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXT13_CNTL                                                                          0x16a5
+#define regGCVM_CONTEXT13_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXT14_CNTL                                                                          0x16a6
+#define regGCVM_CONTEXT14_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXT15_CNTL                                                                          0x16a7
+#define regGCVM_CONTEXT15_CNTL_BASE_IDX                                                                 0
+#define regGCVM_CONTEXTS_DISABLE                                                                        0x16a8
+#define regGCVM_CONTEXTS_DISABLE_BASE_IDX                                                               0
+#define regGCVM_INVALIDATE_ENG0_SEM                                                                     0x16a9
+#define regGCVM_INVALIDATE_ENG0_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG1_SEM                                                                     0x16aa
+#define regGCVM_INVALIDATE_ENG1_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG2_SEM                                                                     0x16ab
+#define regGCVM_INVALIDATE_ENG2_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG3_SEM                                                                     0x16ac
+#define regGCVM_INVALIDATE_ENG3_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG4_SEM                                                                     0x16ad
+#define regGCVM_INVALIDATE_ENG4_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG5_SEM                                                                     0x16ae
+#define regGCVM_INVALIDATE_ENG5_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG6_SEM                                                                     0x16af
+#define regGCVM_INVALIDATE_ENG6_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG7_SEM                                                                     0x16b0
+#define regGCVM_INVALIDATE_ENG7_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG8_SEM                                                                     0x16b1
+#define regGCVM_INVALIDATE_ENG8_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG9_SEM                                                                     0x16b2
+#define regGCVM_INVALIDATE_ENG9_SEM_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG10_SEM                                                                    0x16b3
+#define regGCVM_INVALIDATE_ENG10_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG11_SEM                                                                    0x16b4
+#define regGCVM_INVALIDATE_ENG11_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG12_SEM                                                                    0x16b5
+#define regGCVM_INVALIDATE_ENG12_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG13_SEM                                                                    0x16b6
+#define regGCVM_INVALIDATE_ENG13_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG14_SEM                                                                    0x16b7
+#define regGCVM_INVALIDATE_ENG14_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG15_SEM                                                                    0x16b8
+#define regGCVM_INVALIDATE_ENG15_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG16_SEM                                                                    0x16b9
+#define regGCVM_INVALIDATE_ENG16_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG17_SEM                                                                    0x16ba
+#define regGCVM_INVALIDATE_ENG17_SEM_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG0_REQ                                                                     0x16bb
+#define regGCVM_INVALIDATE_ENG0_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG1_REQ                                                                     0x16bc
+#define regGCVM_INVALIDATE_ENG1_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG2_REQ                                                                     0x16bd
+#define regGCVM_INVALIDATE_ENG2_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG3_REQ                                                                     0x16be
+#define regGCVM_INVALIDATE_ENG3_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG4_REQ                                                                     0x16bf
+#define regGCVM_INVALIDATE_ENG4_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG5_REQ                                                                     0x16c0
+#define regGCVM_INVALIDATE_ENG5_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG6_REQ                                                                     0x16c1
+#define regGCVM_INVALIDATE_ENG6_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG7_REQ                                                                     0x16c2
+#define regGCVM_INVALIDATE_ENG7_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG8_REQ                                                                     0x16c3
+#define regGCVM_INVALIDATE_ENG8_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG9_REQ                                                                     0x16c4
+#define regGCVM_INVALIDATE_ENG9_REQ_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG10_REQ                                                                    0x16c5
+#define regGCVM_INVALIDATE_ENG10_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG11_REQ                                                                    0x16c6
+#define regGCVM_INVALIDATE_ENG11_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG12_REQ                                                                    0x16c7
+#define regGCVM_INVALIDATE_ENG12_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG13_REQ                                                                    0x16c8
+#define regGCVM_INVALIDATE_ENG13_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG14_REQ                                                                    0x16c9
+#define regGCVM_INVALIDATE_ENG14_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG15_REQ                                                                    0x16ca
+#define regGCVM_INVALIDATE_ENG15_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG16_REQ                                                                    0x16cb
+#define regGCVM_INVALIDATE_ENG16_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG17_REQ                                                                    0x16cc
+#define regGCVM_INVALIDATE_ENG17_REQ_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG0_ACK                                                                     0x16cd
+#define regGCVM_INVALIDATE_ENG0_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG1_ACK                                                                     0x16ce
+#define regGCVM_INVALIDATE_ENG1_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG2_ACK                                                                     0x16cf
+#define regGCVM_INVALIDATE_ENG2_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG3_ACK                                                                     0x16d0
+#define regGCVM_INVALIDATE_ENG3_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG4_ACK                                                                     0x16d1
+#define regGCVM_INVALIDATE_ENG4_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG5_ACK                                                                     0x16d2
+#define regGCVM_INVALIDATE_ENG5_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG6_ACK                                                                     0x16d3
+#define regGCVM_INVALIDATE_ENG6_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG7_ACK                                                                     0x16d4
+#define regGCVM_INVALIDATE_ENG7_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG8_ACK                                                                     0x16d5
+#define regGCVM_INVALIDATE_ENG8_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG9_ACK                                                                     0x16d6
+#define regGCVM_INVALIDATE_ENG9_ACK_BASE_IDX                                                            0
+#define regGCVM_INVALIDATE_ENG10_ACK                                                                    0x16d7
+#define regGCVM_INVALIDATE_ENG10_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG11_ACK                                                                    0x16d8
+#define regGCVM_INVALIDATE_ENG11_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG12_ACK                                                                    0x16d9
+#define regGCVM_INVALIDATE_ENG12_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG13_ACK                                                                    0x16da
+#define regGCVM_INVALIDATE_ENG13_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG14_ACK                                                                    0x16db
+#define regGCVM_INVALIDATE_ENG14_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG15_ACK                                                                    0x16dc
+#define regGCVM_INVALIDATE_ENG15_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG16_ACK                                                                    0x16dd
+#define regGCVM_INVALIDATE_ENG16_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG17_ACK                                                                    0x16de
+#define regGCVM_INVALIDATE_ENG17_ACK_BASE_IDX                                                           0
+#define regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32                                                         0x16df
+#define regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32                                                         0x16e0
+#define regGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32                                                         0x16e1
+#define regGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG1_ADDR_RANGE_HI32                                                         0x16e2
+#define regGCVM_INVALIDATE_ENG1_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32                                                         0x16e3
+#define regGCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG2_ADDR_RANGE_HI32                                                         0x16e4
+#define regGCVM_INVALIDATE_ENG2_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32                                                         0x16e5
+#define regGCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG3_ADDR_RANGE_HI32                                                         0x16e6
+#define regGCVM_INVALIDATE_ENG3_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32                                                         0x16e7
+#define regGCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG4_ADDR_RANGE_HI32                                                         0x16e8
+#define regGCVM_INVALIDATE_ENG4_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32                                                         0x16e9
+#define regGCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG5_ADDR_RANGE_HI32                                                         0x16ea
+#define regGCVM_INVALIDATE_ENG5_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32                                                         0x16eb
+#define regGCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG6_ADDR_RANGE_HI32                                                         0x16ec
+#define regGCVM_INVALIDATE_ENG6_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32                                                         0x16ed
+#define regGCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG7_ADDR_RANGE_HI32                                                         0x16ee
+#define regGCVM_INVALIDATE_ENG7_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32                                                         0x16ef
+#define regGCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG8_ADDR_RANGE_HI32                                                         0x16f0
+#define regGCVM_INVALIDATE_ENG8_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32                                                         0x16f1
+#define regGCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG9_ADDR_RANGE_HI32                                                         0x16f2
+#define regGCVM_INVALIDATE_ENG9_ADDR_RANGE_HI32_BASE_IDX                                                0
+#define regGCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32                                                        0x16f3
+#define regGCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG10_ADDR_RANGE_HI32                                                        0x16f4
+#define regGCVM_INVALIDATE_ENG10_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32                                                        0x16f5
+#define regGCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG11_ADDR_RANGE_HI32                                                        0x16f6
+#define regGCVM_INVALIDATE_ENG11_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32                                                        0x16f7
+#define regGCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG12_ADDR_RANGE_HI32                                                        0x16f8
+#define regGCVM_INVALIDATE_ENG12_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32                                                        0x16f9
+#define regGCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG13_ADDR_RANGE_HI32                                                        0x16fa
+#define regGCVM_INVALIDATE_ENG13_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32                                                        0x16fb
+#define regGCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG14_ADDR_RANGE_HI32                                                        0x16fc
+#define regGCVM_INVALIDATE_ENG14_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32                                                        0x16fd
+#define regGCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG15_ADDR_RANGE_HI32                                                        0x16fe
+#define regGCVM_INVALIDATE_ENG15_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32                                                        0x16ff
+#define regGCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG16_ADDR_RANGE_HI32                                                        0x1700
+#define regGCVM_INVALIDATE_ENG16_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32                                                        0x1701
+#define regGCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32_BASE_IDX                                               0
+#define regGCVM_INVALIDATE_ENG17_ADDR_RANGE_HI32                                                        0x1702
+#define regGCVM_INVALIDATE_ENG17_ADDR_RANGE_HI32_BASE_IDX                                               0
+#define regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32                                                      0x1703
+#define regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32                                                      0x1704
+#define regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32                                                      0x1705
+#define regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32                                                      0x1706
+#define regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32                                                      0x1707
+#define regGCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32                                                      0x1708
+#define regGCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32                                                      0x1709
+#define regGCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32                                                      0x170a
+#define regGCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32                                                      0x170b
+#define regGCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32                                                      0x170c
+#define regGCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32                                                      0x170d
+#define regGCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32                                                      0x170e
+#define regGCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32                                                      0x170f
+#define regGCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32                                                      0x1710
+#define regGCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32                                                      0x1711
+#define regGCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32                                                      0x1712
+#define regGCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32                                                      0x1713
+#define regGCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32                                                      0x1714
+#define regGCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32                                                      0x1715
+#define regGCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32                                                      0x1716
+#define regGCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32                                                     0x1717
+#define regGCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32                                                     0x1718
+#define regGCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32                                                     0x1719
+#define regGCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32                                                     0x171a
+#define regGCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32                                                     0x171b
+#define regGCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32                                                     0x171c
+#define regGCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32                                                     0x171d
+#define regGCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32                                                     0x171e
+#define regGCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32                                                     0x171f
+#define regGCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32                                                     0x1720
+#define regGCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32                                                     0x1721
+#define regGCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32                                                     0x1722
+#define regGCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32                                                     0x1723
+#define regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32                                                     0x1724
+#define regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32                                                     0x1725
+#define regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32                                                     0x1726
+#define regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32                                                     0x1727
+#define regGCVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32                                                     0x1728
+#define regGCVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32                                                     0x1729
+#define regGCVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32                                                     0x172a
+#define regGCVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32                                                     0x172b
+#define regGCVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32                                                     0x172c
+#define regGCVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32                                                     0x172d
+#define regGCVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32                                                     0x172e
+#define regGCVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32                                                     0x172f
+#define regGCVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32                                                     0x1730
+#define regGCVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32                                                     0x1731
+#define regGCVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32                                                     0x1732
+#define regGCVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32                                                     0x1733
+#define regGCVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32                                                     0x1734
+#define regGCVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32                                                     0x1735
+#define regGCVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                            0
+#define regGCVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32                                                     0x1736
+#define regGCVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                            0
+#define regGCVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32                                                    0x1737
+#define regGCVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32                                                    0x1738
+#define regGCVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32                                                    0x1739
+#define regGCVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32                                                    0x173a
+#define regGCVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32                                                    0x173b
+#define regGCVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32                                                    0x173c
+#define regGCVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32                                                    0x173d
+#define regGCVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32                                                    0x173e
+#define regGCVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32                                                    0x173f
+#define regGCVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32                                                    0x1740
+#define regGCVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32                                                    0x1741
+#define regGCVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32_BASE_IDX                                           0
+#define regGCVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32                                                    0x1742
+#define regGCVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32_BASE_IDX                                           0
+#define regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32                                                       0x1743
+#define regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32                                                       0x1744
+#define regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32                                                       0x1745
+#define regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32                                                       0x1746
+#define regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32                                                       0x1747
+#define regGCVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32                                                       0x1748
+#define regGCVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32                                                       0x1749
+#define regGCVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32                                                       0x174a
+#define regGCVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32                                                       0x174b
+#define regGCVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32                                                       0x174c
+#define regGCVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32                                                       0x174d
+#define regGCVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32                                                       0x174e
+#define regGCVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32                                                       0x174f
+#define regGCVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32                                                       0x1750
+#define regGCVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32                                                       0x1751
+#define regGCVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32                                                       0x1752
+#define regGCVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32                                                       0x1753
+#define regGCVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32                                                       0x1754
+#define regGCVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32                                                       0x1755
+#define regGCVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                              0
+#define regGCVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32                                                       0x1756
+#define regGCVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                              0
+#define regGCVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32                                                      0x1757
+#define regGCVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32                                                      0x1758
+#define regGCVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32                                                      0x1759
+#define regGCVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32                                                      0x175a
+#define regGCVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32                                                      0x175b
+#define regGCVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32                                                      0x175c
+#define regGCVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32                                                      0x175d
+#define regGCVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32                                                      0x175e
+#define regGCVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32                                                      0x175f
+#define regGCVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32                                                      0x1760
+#define regGCVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32                                                      0x1761
+#define regGCVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32_BASE_IDX                                             0
+#define regGCVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32                                                      0x1762
+#define regGCVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32_BASE_IDX                                             0
+#define regGCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                                    0x1763
+#define regGCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                           0
+#define regGCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1764
+#define regGCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1765
+#define regGCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1766
+#define regGCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1767
+#define regGCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1768
+#define regGCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x1769
+#define regGCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x176a
+#define regGCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x176b
+#define regGCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x176c
+#define regGCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                           0x176d
+#define regGCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                  0
+#define regGCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x176e
+#define regGCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+#define regGCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x176f
+#define regGCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+#define regGCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x1770
+#define regGCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+#define regGCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x1771
+#define regGCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+#define regGCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x1772
+#define regGCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+#define regGCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES                                          0x1773
+#define regGCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES_BASE_IDX                                 0
+
+
+// addressBlock: gc_gcvml2perfddec
+// base address: 0x35380
+#define regGCVML2_PERFCOUNTER2_0_LO                                                                     0x34e0
+#define regGCVML2_PERFCOUNTER2_0_LO_BASE_IDX                                                            1
+#define regGCVML2_PERFCOUNTER2_1_LO                                                                     0x34e1
+#define regGCVML2_PERFCOUNTER2_1_LO_BASE_IDX                                                            1
+#define regGCVML2_PERFCOUNTER2_0_HI                                                                     0x34e2
+#define regGCVML2_PERFCOUNTER2_0_HI_BASE_IDX                                                            1
+#define regGCVML2_PERFCOUNTER2_1_HI                                                                     0x34e3
+#define regGCVML2_PERFCOUNTER2_1_HI_BASE_IDX                                                            1
+
+
+// addressBlock: gc_gcvml2prdec
+// base address: 0x35390
+#define regGCMC_VM_L2_PERFCOUNTER_LO                                                                    0x34e4
+#define regGCMC_VM_L2_PERFCOUNTER_LO_BASE_IDX                                                           1
+#define regGCMC_VM_L2_PERFCOUNTER_HI                                                                    0x34e5
+#define regGCMC_VM_L2_PERFCOUNTER_HI_BASE_IDX                                                           1
+#define regGCUTCL2_PERFCOUNTER_LO                                                                       0x34e6
+#define regGCUTCL2_PERFCOUNTER_LO_BASE_IDX                                                              1
+#define regGCUTCL2_PERFCOUNTER_HI                                                                       0x34e7
+#define regGCUTCL2_PERFCOUNTER_HI_BASE_IDX                                                              1
+
+
+// addressBlock: gc_gcatcl2perfddec
+// base address: 0x353d0
+#define regGC_ATC_L2_PERFCOUNTER2_LO                                                                    0x34f4
+#define regGC_ATC_L2_PERFCOUNTER2_LO_BASE_IDX                                                           1
+#define regGC_ATC_L2_PERFCOUNTER2_HI                                                                    0x34f5
+#define regGC_ATC_L2_PERFCOUNTER2_HI_BASE_IDX                                                           1
+
+
+// addressBlock: gc_gcatcl2pfcntrdec
+// base address: 0x353e0
+#define regGC_ATC_L2_PERFCOUNTER_LO                                                                     0x34f8
+#define regGC_ATC_L2_PERFCOUNTER_LO_BASE_IDX                                                            1
+#define regGC_ATC_L2_PERFCOUNTER_HI                                                                     0x34f9
+#define regGC_ATC_L2_PERFCOUNTER_HI_BASE_IDX                                                            1
+
+
+// addressBlock: gc_gcl2tlbprdec
+// base address: 0x353e8
+#define regGCL2TLB_PERFCOUNTER_LO                                                                       0x34fa
+#define regGCL2TLB_PERFCOUNTER_LO_BASE_IDX                                                              1
+#define regGCL2TLB_PERFCOUNTER_HI                                                                       0x34fb
+#define regGCL2TLB_PERFCOUNTER_HI_BASE_IDX                                                              1
+
+
+// addressBlock: gc_gcvml2perfsdec
+// base address: 0x37480
+#define regGCVML2_PERFCOUNTER2_0_SELECT                                                                 0x3d20
+#define regGCVML2_PERFCOUNTER2_0_SELECT_BASE_IDX                                                        1
+#define regGCVML2_PERFCOUNTER2_1_SELECT                                                                 0x3d21
+#define regGCVML2_PERFCOUNTER2_1_SELECT_BASE_IDX                                                        1
+#define regGCVML2_PERFCOUNTER2_0_SELECT1                                                                0x3d22
+#define regGCVML2_PERFCOUNTER2_0_SELECT1_BASE_IDX                                                       1
+#define regGCVML2_PERFCOUNTER2_1_SELECT1                                                                0x3d23
+#define regGCVML2_PERFCOUNTER2_1_SELECT1_BASE_IDX                                                       1
+#define regGCVML2_PERFCOUNTER2_0_MODE                                                                   0x3d24
+#define regGCVML2_PERFCOUNTER2_0_MODE_BASE_IDX                                                          1
+#define regGCVML2_PERFCOUNTER2_1_MODE                                                                   0x3d25
+#define regGCVML2_PERFCOUNTER2_1_MODE_BASE_IDX                                                          1
+
+
+// addressBlock: gc_gcvml2pldec
+// base address: 0x374c0
+#define regGCMC_VM_L2_PERFCOUNTER0_CFG                                                                  0x3d30
+#define regGCMC_VM_L2_PERFCOUNTER0_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER1_CFG                                                                  0x3d31
+#define regGCMC_VM_L2_PERFCOUNTER1_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER2_CFG                                                                  0x3d32
+#define regGCMC_VM_L2_PERFCOUNTER2_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER3_CFG                                                                  0x3d33
+#define regGCMC_VM_L2_PERFCOUNTER3_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER4_CFG                                                                  0x3d34
+#define regGCMC_VM_L2_PERFCOUNTER4_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER5_CFG                                                                  0x3d35
+#define regGCMC_VM_L2_PERFCOUNTER5_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER6_CFG                                                                  0x3d36
+#define regGCMC_VM_L2_PERFCOUNTER6_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER7_CFG                                                                  0x3d37
+#define regGCMC_VM_L2_PERFCOUNTER7_CFG_BASE_IDX                                                         1
+#define regGCMC_VM_L2_PERFCOUNTER_RSLT_CNTL                                                             0x3d38
+#define regGCMC_VM_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                    1
+#define regGCUTCL2_PERFCOUNTER0_CFG                                                                     0x3d39
+#define regGCUTCL2_PERFCOUNTER0_CFG_BASE_IDX                                                            1
+#define regGCUTCL2_PERFCOUNTER1_CFG                                                                     0x3d3a
+#define regGCUTCL2_PERFCOUNTER1_CFG_BASE_IDX                                                            1
+#define regGCUTCL2_PERFCOUNTER2_CFG                                                                     0x3d3b
+#define regGCUTCL2_PERFCOUNTER2_CFG_BASE_IDX                                                            1
+#define regGCUTCL2_PERFCOUNTER3_CFG                                                                     0x3d3c
+#define regGCUTCL2_PERFCOUNTER3_CFG_BASE_IDX                                                            1
+#define regGCUTCL2_PERFCOUNTER_RSLT_CNTL                                                                0x3d3d
+#define regGCUTCL2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                       1
+
+
+// addressBlock: gc_gcatcl2perfsdec
+// base address: 0x37500
+#define regGC_ATC_L2_PERFCOUNTER2_SELECT                                                                0x3d40
+#define regGC_ATC_L2_PERFCOUNTER2_SELECT_BASE_IDX                                                       1
+#define regGC_ATC_L2_PERFCOUNTER2_SELECT1                                                               0x3d41
+#define regGC_ATC_L2_PERFCOUNTER2_SELECT1_BASE_IDX                                                      1
+#define regGC_ATC_L2_PERFCOUNTER2_MODE                                                                  0x3d42
+#define regGC_ATC_L2_PERFCOUNTER2_MODE_BASE_IDX                                                         1
+
+
+// addressBlock: gc_gcatcl2pfcntldec
+// base address: 0x37510
+#define regGC_ATC_L2_PERFCOUNTER0_CFG                                                                   0x3d44
+#define regGC_ATC_L2_PERFCOUNTER0_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER1_CFG                                                                   0x3d45
+#define regGC_ATC_L2_PERFCOUNTER1_CFG_BASE_IDX                                                          1
+#define regGC_ATC_L2_PERFCOUNTER_RSLT_CNTL                                                              0x3d46
+#define regGC_ATC_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                     1
+
+
+// addressBlock: gc_gcl2tlbpldec
+// base address: 0x37528
+#define regGCL2TLB_PERFCOUNTER0_CFG                                                                     0x3d4a
+#define regGCL2TLB_PERFCOUNTER0_CFG_BASE_IDX                                                            1
+#define regGCL2TLB_PERFCOUNTER1_CFG                                                                     0x3d4b
+#define regGCL2TLB_PERFCOUNTER1_CFG_BASE_IDX                                                            1
+#define regGCL2TLB_PERFCOUNTER2_CFG                                                                     0x3d4c
+#define regGCL2TLB_PERFCOUNTER2_CFG_BASE_IDX                                                            1
+#define regGCL2TLB_PERFCOUNTER3_CFG                                                                     0x3d4d
+#define regGCL2TLB_PERFCOUNTER3_CFG_BASE_IDX                                                            1
+#define regGCL2TLB_PERFCOUNTER_RSLT_CNTL                                                                0x3d4e
+#define regGCL2TLB_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                       1
+
+
+// addressBlock: gc_gcvml2pspdec
+// base address: 0x3f900
+#define regGCUTCL2_TRANSLATION_BYPASS_BY_VMID                                                           0x5e41
+#define regGCUTCL2_TRANSLATION_BYPASS_BY_VMID_BASE_IDX                                                  1
+#define regGCVM_IOMMU_GPU_HOST_TRANSLATION_ENABLE                                                       0x5e43
+#define regGCVM_IOMMU_GPU_HOST_TRANSLATION_ENABLE_BASE_IDX                                              1
+#define regGCVM_IOMMU_CONTROL_REGISTER                                                                  0x5e44
+#define regGCVM_IOMMU_CONTROL_REGISTER_BASE_IDX                                                         1
+#define regGCVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER                                         0x5e45
+#define regGCVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER_BASE_IDX                                1
+#define regGCVM_IOMMU_MMIO_CNTRL_1                                                                      0x5e46
+#define regGCVM_IOMMU_MMIO_CNTRL_1_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_LO_0                                                                       0x5e47
+#define regGCMC_VM_MARC_BASE_LO_0_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_LO_1                                                                       0x5e48
+#define regGCMC_VM_MARC_BASE_LO_1_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_LO_2                                                                       0x5e49
+#define regGCMC_VM_MARC_BASE_LO_2_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_LO_3                                                                       0x5e4a
+#define regGCMC_VM_MARC_BASE_LO_3_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_LO_4                                                                       0x5e4b
+#define regGCMC_VM_MARC_BASE_LO_4_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_LO_5                                                                       0x5e4c
+#define regGCMC_VM_MARC_BASE_LO_5_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_LO_6                                                                       0x5e4d
+#define regGCMC_VM_MARC_BASE_LO_6_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_LO_7                                                                       0x5e4e
+#define regGCMC_VM_MARC_BASE_LO_7_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_LO_8                                                                       0x5e4f
+#define regGCMC_VM_MARC_BASE_LO_8_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_LO_9                                                                       0x5e50
+#define regGCMC_VM_MARC_BASE_LO_9_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_LO_10                                                                      0x5e51
+#define regGCMC_VM_MARC_BASE_LO_10_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_LO_11                                                                      0x5e52
+#define regGCMC_VM_MARC_BASE_LO_11_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_LO_12                                                                      0x5e53
+#define regGCMC_VM_MARC_BASE_LO_12_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_LO_13                                                                      0x5e54
+#define regGCMC_VM_MARC_BASE_LO_13_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_LO_14                                                                      0x5e55
+#define regGCMC_VM_MARC_BASE_LO_14_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_LO_15                                                                      0x5e56
+#define regGCMC_VM_MARC_BASE_LO_15_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_HI_0                                                                       0x5e57
+#define regGCMC_VM_MARC_BASE_HI_0_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_HI_1                                                                       0x5e58
+#define regGCMC_VM_MARC_BASE_HI_1_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_HI_2                                                                       0x5e59
+#define regGCMC_VM_MARC_BASE_HI_2_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_HI_3                                                                       0x5e5a
+#define regGCMC_VM_MARC_BASE_HI_3_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_HI_4                                                                       0x5e5b
+#define regGCMC_VM_MARC_BASE_HI_4_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_HI_5                                                                       0x5e5c
+#define regGCMC_VM_MARC_BASE_HI_5_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_HI_6                                                                       0x5e5d
+#define regGCMC_VM_MARC_BASE_HI_6_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_HI_7                                                                       0x5e5e
+#define regGCMC_VM_MARC_BASE_HI_7_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_HI_8                                                                       0x5e5f
+#define regGCMC_VM_MARC_BASE_HI_8_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_HI_9                                                                       0x5e60
+#define regGCMC_VM_MARC_BASE_HI_9_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_BASE_HI_10                                                                      0x5e61
+#define regGCMC_VM_MARC_BASE_HI_10_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_HI_11                                                                      0x5e62
+#define regGCMC_VM_MARC_BASE_HI_11_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_HI_12                                                                      0x5e63
+#define regGCMC_VM_MARC_BASE_HI_12_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_HI_13                                                                      0x5e64
+#define regGCMC_VM_MARC_BASE_HI_13_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_HI_14                                                                      0x5e65
+#define regGCMC_VM_MARC_BASE_HI_14_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_BASE_HI_15                                                                      0x5e66
+#define regGCMC_VM_MARC_BASE_HI_15_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_0                                                                      0x5e67
+#define regGCMC_VM_MARC_RELOC_LO_0_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_1                                                                      0x5e68
+#define regGCMC_VM_MARC_RELOC_LO_1_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_2                                                                      0x5e69
+#define regGCMC_VM_MARC_RELOC_LO_2_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_3                                                                      0x5e6a
+#define regGCMC_VM_MARC_RELOC_LO_3_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_4                                                                      0x5e6b
+#define regGCMC_VM_MARC_RELOC_LO_4_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_5                                                                      0x5e6c
+#define regGCMC_VM_MARC_RELOC_LO_5_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_6                                                                      0x5e6d
+#define regGCMC_VM_MARC_RELOC_LO_6_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_7                                                                      0x5e6e
+#define regGCMC_VM_MARC_RELOC_LO_7_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_8                                                                      0x5e6f
+#define regGCMC_VM_MARC_RELOC_LO_8_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_9                                                                      0x5e70
+#define regGCMC_VM_MARC_RELOC_LO_9_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_LO_10                                                                     0x5e71
+#define regGCMC_VM_MARC_RELOC_LO_10_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_LO_11                                                                     0x5e72
+#define regGCMC_VM_MARC_RELOC_LO_11_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_LO_12                                                                     0x5e73
+#define regGCMC_VM_MARC_RELOC_LO_12_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_LO_13                                                                     0x5e74
+#define regGCMC_VM_MARC_RELOC_LO_13_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_LO_14                                                                     0x5e75
+#define regGCMC_VM_MARC_RELOC_LO_14_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_LO_15                                                                     0x5e76
+#define regGCMC_VM_MARC_RELOC_LO_15_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_HI_0                                                                      0x5e77
+#define regGCMC_VM_MARC_RELOC_HI_0_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_HI_1                                                                      0x5e78
+#define regGCMC_VM_MARC_RELOC_HI_1_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_HI_2                                                                      0x5e79
+#define regGCMC_VM_MARC_RELOC_HI_2_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_HI_3                                                                      0x5e7a
+#define regGCMC_VM_MARC_RELOC_HI_3_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_HI_4                                                                      0x5e7b
+#define regGCMC_VM_MARC_RELOC_HI_4_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_HI_5                                                                      0x5e7c
+#define regGCMC_VM_MARC_RELOC_HI_5_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_HI_6                                                                      0x5e7d
+#define regGCMC_VM_MARC_RELOC_HI_6_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_HI_7                                                                      0x5e7e
+#define regGCMC_VM_MARC_RELOC_HI_7_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_HI_8                                                                      0x5e7f
+#define regGCMC_VM_MARC_RELOC_HI_8_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_HI_9                                                                      0x5e80
+#define regGCMC_VM_MARC_RELOC_HI_9_BASE_IDX                                                             1
+#define regGCMC_VM_MARC_RELOC_HI_10                                                                     0x5e81
+#define regGCMC_VM_MARC_RELOC_HI_10_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_HI_11                                                                     0x5e82
+#define regGCMC_VM_MARC_RELOC_HI_11_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_HI_12                                                                     0x5e83
+#define regGCMC_VM_MARC_RELOC_HI_12_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_HI_13                                                                     0x5e84
+#define regGCMC_VM_MARC_RELOC_HI_13_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_HI_14                                                                     0x5e85
+#define regGCMC_VM_MARC_RELOC_HI_14_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_RELOC_HI_15                                                                     0x5e86
+#define regGCMC_VM_MARC_RELOC_HI_15_BASE_IDX                                                            1
+#define regGCMC_VM_MARC_LEN_LO_0                                                                        0x5e87
+#define regGCMC_VM_MARC_LEN_LO_0_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_LO_1                                                                        0x5e88
+#define regGCMC_VM_MARC_LEN_LO_1_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_LO_2                                                                        0x5e89
+#define regGCMC_VM_MARC_LEN_LO_2_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_LO_3                                                                        0x5e8a
+#define regGCMC_VM_MARC_LEN_LO_3_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_LO_4                                                                        0x5e8b
+#define regGCMC_VM_MARC_LEN_LO_4_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_LO_5                                                                        0x5e8c
+#define regGCMC_VM_MARC_LEN_LO_5_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_LO_6                                                                        0x5e8d
+#define regGCMC_VM_MARC_LEN_LO_6_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_LO_7                                                                        0x5e8e
+#define regGCMC_VM_MARC_LEN_LO_7_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_LO_8                                                                        0x5e8f
+#define regGCMC_VM_MARC_LEN_LO_8_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_LO_9                                                                        0x5e90
+#define regGCMC_VM_MARC_LEN_LO_9_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_LO_10                                                                       0x5e91
+#define regGCMC_VM_MARC_LEN_LO_10_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_LO_11                                                                       0x5e92
+#define regGCMC_VM_MARC_LEN_LO_11_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_LO_12                                                                       0x5e93
+#define regGCMC_VM_MARC_LEN_LO_12_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_LO_13                                                                       0x5e94
+#define regGCMC_VM_MARC_LEN_LO_13_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_LO_14                                                                       0x5e95
+#define regGCMC_VM_MARC_LEN_LO_14_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_LO_15                                                                       0x5e96
+#define regGCMC_VM_MARC_LEN_LO_15_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_HI_0                                                                        0x5e97
+#define regGCMC_VM_MARC_LEN_HI_0_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_HI_1                                                                        0x5e98
+#define regGCMC_VM_MARC_LEN_HI_1_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_HI_2                                                                        0x5e99
+#define regGCMC_VM_MARC_LEN_HI_2_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_HI_3                                                                        0x5e9a
+#define regGCMC_VM_MARC_LEN_HI_3_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_HI_4                                                                        0x5e9b
+#define regGCMC_VM_MARC_LEN_HI_4_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_HI_5                                                                        0x5e9c
+#define regGCMC_VM_MARC_LEN_HI_5_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_HI_6                                                                        0x5e9d
+#define regGCMC_VM_MARC_LEN_HI_6_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_HI_7                                                                        0x5e9e
+#define regGCMC_VM_MARC_LEN_HI_7_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_HI_8                                                                        0x5e9f
+#define regGCMC_VM_MARC_LEN_HI_8_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_HI_9                                                                        0x5ea0
+#define regGCMC_VM_MARC_LEN_HI_9_BASE_IDX                                                               1
+#define regGCMC_VM_MARC_LEN_HI_10                                                                       0x5ea1
+#define regGCMC_VM_MARC_LEN_HI_10_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_HI_11                                                                       0x5ea2
+#define regGCMC_VM_MARC_LEN_HI_11_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_HI_12                                                                       0x5ea3
+#define regGCMC_VM_MARC_LEN_HI_12_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_HI_13                                                                       0x5ea4
+#define regGCMC_VM_MARC_LEN_HI_13_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_HI_14                                                                       0x5ea5
+#define regGCMC_VM_MARC_LEN_HI_14_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_LEN_HI_15                                                                       0x5ea6
+#define regGCMC_VM_MARC_LEN_HI_15_BASE_IDX                                                              1
+#define regGCMC_VM_MARC_PFVF_MAPPING_0                                                                  0x5ea7
+#define regGCMC_VM_MARC_PFVF_MAPPING_0_BASE_IDX                                                         1
+#define regGCMC_VM_MARC_PFVF_MAPPING_1                                                                  0x5ea8
+#define regGCMC_VM_MARC_PFVF_MAPPING_1_BASE_IDX                                                         1
+#define regGCMC_VM_MARC_PFVF_MAPPING_2                                                                  0x5ea9
+#define regGCMC_VM_MARC_PFVF_MAPPING_2_BASE_IDX                                                         1
+#define regGCMC_VM_MARC_PFVF_MAPPING_3                                                                  0x5eaa
+#define regGCMC_VM_MARC_PFVF_MAPPING_3_BASE_IDX                                                         1
+#define regGCMC_VM_MARC_PFVF_MAPPING_4                                                                  0x5eab
+#define regGCMC_VM_MARC_PFVF_MAPPING_4_BASE_IDX                                                         1
+#define regGCMC_VM_MARC_PFVF_MAPPING_5                                                                  0x5eac
+#define regGCMC_VM_MARC_PFVF_MAPPING_5_BASE_IDX                                                         1
+#define regGCMC_VM_MARC_PFVF_MAPPING_6                                                                  0x5ead
+#define regGCMC_VM_MARC_PFVF_MAPPING_6_BASE_IDX                                                         1
+#define regGCMC_VM_MARC_PFVF_MAPPING_7                                                                  0x5eae
+#define regGCMC_VM_MARC_PFVF_MAPPING_7_BASE_IDX                                                         1
+#define regGCMC_VM_MARC_PFVF_MAPPING_8                                                                  0x5eaf
+#define regGCMC_VM_MARC_PFVF_MAPPING_8_BASE_IDX                                                         1
+#define regGCMC_VM_MARC_PFVF_MAPPING_9                                                                  0x5eb0
+#define regGCMC_VM_MARC_PFVF_MAPPING_9_BASE_IDX                                                         1
+#define regGCMC_VM_MARC_PFVF_MAPPING_10                                                                 0x5eb1
+#define regGCMC_VM_MARC_PFVF_MAPPING_10_BASE_IDX                                                        1
+#define regGCMC_VM_MARC_PFVF_MAPPING_11                                                                 0x5eb2
+#define regGCMC_VM_MARC_PFVF_MAPPING_11_BASE_IDX                                                        1
+#define regGCMC_VM_MARC_PFVF_MAPPING_12                                                                 0x5eb3
+#define regGCMC_VM_MARC_PFVF_MAPPING_12_BASE_IDX                                                        1
+#define regGCMC_VM_MARC_PFVF_MAPPING_13                                                                 0x5eb4
+#define regGCMC_VM_MARC_PFVF_MAPPING_13_BASE_IDX                                                        1
+#define regGCMC_VM_MARC_PFVF_MAPPING_14                                                                 0x5eb5
+#define regGCMC_VM_MARC_PFVF_MAPPING_14_BASE_IDX                                                        1
+#define regGCMC_VM_MARC_PFVF_MAPPING_15                                                                 0x5eb6
+#define regGCMC_VM_MARC_PFVF_MAPPING_15_BASE_IDX                                                        1
+#define regGCUTC_TRANSLATION_FAULT_CNTL0                                                                0x5eb7
+#define regGCUTC_TRANSLATION_FAULT_CNTL0_BASE_IDX                                                       1
+#define regGCUTC_TRANSLATION_FAULT_CNTL1                                                                0x5eb8
+#define regGCUTC_TRANSLATION_FAULT_CNTL1_BASE_IDX                                                       1
+
+
+// addressBlock: gc_gcl2tlbpspdec
+// base address: 0x3fb10
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_CNTL                                                     0x5ec4
+#define regGCUTC_GPUVA_VMID_TRANSLATION_ASSIST_CNTL_BASE_IDX                                            1
+
+
+// addressBlock: gc_shdec
+// base address: 0xb000
+#define regSPI_SHADER_PGM_RSRC4_PS                                                                      0x19a1
+#define regSPI_SHADER_PGM_RSRC4_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_CHKSUM_PS                                                                     0x19a6
+#define regSPI_SHADER_PGM_CHKSUM_PS_BASE_IDX                                                            0
+#define regSPI_SHADER_PGM_RSRC3_PS                                                                      0x19a7
+#define regSPI_SHADER_PGM_RSRC3_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_LO_PS                                                                         0x19a8
+#define regSPI_SHADER_PGM_LO_PS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_PS                                                                         0x19a9
+#define regSPI_SHADER_PGM_HI_PS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC1_PS                                                                      0x19aa
+#define regSPI_SHADER_PGM_RSRC1_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC2_PS                                                                      0x19ab
+#define regSPI_SHADER_PGM_RSRC2_PS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_PS_0                                                                    0x19ac
+#define regSPI_SHADER_USER_DATA_PS_0_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_1                                                                    0x19ad
+#define regSPI_SHADER_USER_DATA_PS_1_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_2                                                                    0x19ae
+#define regSPI_SHADER_USER_DATA_PS_2_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_3                                                                    0x19af
+#define regSPI_SHADER_USER_DATA_PS_3_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_4                                                                    0x19b0
+#define regSPI_SHADER_USER_DATA_PS_4_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_5                                                                    0x19b1
+#define regSPI_SHADER_USER_DATA_PS_5_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_6                                                                    0x19b2
+#define regSPI_SHADER_USER_DATA_PS_6_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_7                                                                    0x19b3
+#define regSPI_SHADER_USER_DATA_PS_7_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_8                                                                    0x19b4
+#define regSPI_SHADER_USER_DATA_PS_8_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_9                                                                    0x19b5
+#define regSPI_SHADER_USER_DATA_PS_9_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_PS_10                                                                   0x19b6
+#define regSPI_SHADER_USER_DATA_PS_10_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_11                                                                   0x19b7
+#define regSPI_SHADER_USER_DATA_PS_11_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_12                                                                   0x19b8
+#define regSPI_SHADER_USER_DATA_PS_12_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_13                                                                   0x19b9
+#define regSPI_SHADER_USER_DATA_PS_13_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_14                                                                   0x19ba
+#define regSPI_SHADER_USER_DATA_PS_14_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_15                                                                   0x19bb
+#define regSPI_SHADER_USER_DATA_PS_15_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_16                                                                   0x19bc
+#define regSPI_SHADER_USER_DATA_PS_16_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_17                                                                   0x19bd
+#define regSPI_SHADER_USER_DATA_PS_17_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_18                                                                   0x19be
+#define regSPI_SHADER_USER_DATA_PS_18_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_19                                                                   0x19bf
+#define regSPI_SHADER_USER_DATA_PS_19_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_20                                                                   0x19c0
+#define regSPI_SHADER_USER_DATA_PS_20_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_21                                                                   0x19c1
+#define regSPI_SHADER_USER_DATA_PS_21_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_22                                                                   0x19c2
+#define regSPI_SHADER_USER_DATA_PS_22_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_23                                                                   0x19c3
+#define regSPI_SHADER_USER_DATA_PS_23_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_24                                                                   0x19c4
+#define regSPI_SHADER_USER_DATA_PS_24_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_25                                                                   0x19c5
+#define regSPI_SHADER_USER_DATA_PS_25_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_26                                                                   0x19c6
+#define regSPI_SHADER_USER_DATA_PS_26_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_27                                                                   0x19c7
+#define regSPI_SHADER_USER_DATA_PS_27_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_28                                                                   0x19c8
+#define regSPI_SHADER_USER_DATA_PS_28_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_29                                                                   0x19c9
+#define regSPI_SHADER_USER_DATA_PS_29_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_30                                                                   0x19ca
+#define regSPI_SHADER_USER_DATA_PS_30_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_PS_31                                                                   0x19cb
+#define regSPI_SHADER_USER_DATA_PS_31_BASE_IDX                                                          0
+#define regSPI_SHADER_REQ_CTRL_PS                                                                       0x19d0
+#define regSPI_SHADER_REQ_CTRL_PS_BASE_IDX                                                              0
+#define regSPI_SHADER_USER_ACCUM_PS_0                                                                   0x19d2
+#define regSPI_SHADER_USER_ACCUM_PS_0_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_ACCUM_PS_1                                                                   0x19d3
+#define regSPI_SHADER_USER_ACCUM_PS_1_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_ACCUM_PS_2                                                                   0x19d4
+#define regSPI_SHADER_USER_ACCUM_PS_2_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_ACCUM_PS_3                                                                   0x19d5
+#define regSPI_SHADER_USER_ACCUM_PS_3_BASE_IDX                                                          0
+#define regSPI_SHADER_PGM_CHKSUM_GS                                                                     0x1a20
+#define regSPI_SHADER_PGM_CHKSUM_GS_BASE_IDX                                                            0
+#define regSPI_SHADER_PGM_RSRC4_GS                                                                      0x1a21
+#define regSPI_SHADER_PGM_RSRC4_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_ADDR_LO_GS                                                              0x1a22
+#define regSPI_SHADER_USER_DATA_ADDR_LO_GS_BASE_IDX                                                     0
+#define regSPI_SHADER_USER_DATA_ADDR_HI_GS                                                              0x1a23
+#define regSPI_SHADER_USER_DATA_ADDR_HI_GS_BASE_IDX                                                     0
+#define regSPI_SHADER_PGM_LO_ES_GS                                                                      0x1a24
+#define regSPI_SHADER_PGM_LO_ES_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_HI_ES_GS                                                                      0x1a25
+#define regSPI_SHADER_PGM_HI_ES_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC3_GS                                                                      0x1a27
+#define regSPI_SHADER_PGM_RSRC3_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_LO_GS                                                                         0x1a28
+#define regSPI_SHADER_PGM_LO_GS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_GS                                                                         0x1a29
+#define regSPI_SHADER_PGM_HI_GS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC1_GS                                                                      0x1a2a
+#define regSPI_SHADER_PGM_RSRC1_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC2_GS                                                                      0x1a2b
+#define regSPI_SHADER_PGM_RSRC2_GS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_GS_0                                                                    0x1a2c
+#define regSPI_SHADER_USER_DATA_GS_0_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_1                                                                    0x1a2d
+#define regSPI_SHADER_USER_DATA_GS_1_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_2                                                                    0x1a2e
+#define regSPI_SHADER_USER_DATA_GS_2_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_3                                                                    0x1a2f
+#define regSPI_SHADER_USER_DATA_GS_3_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_4                                                                    0x1a30
+#define regSPI_SHADER_USER_DATA_GS_4_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_5                                                                    0x1a31
+#define regSPI_SHADER_USER_DATA_GS_5_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_6                                                                    0x1a32
+#define regSPI_SHADER_USER_DATA_GS_6_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_7                                                                    0x1a33
+#define regSPI_SHADER_USER_DATA_GS_7_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_8                                                                    0x1a34
+#define regSPI_SHADER_USER_DATA_GS_8_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_9                                                                    0x1a35
+#define regSPI_SHADER_USER_DATA_GS_9_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_GS_10                                                                   0x1a36
+#define regSPI_SHADER_USER_DATA_GS_10_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_11                                                                   0x1a37
+#define regSPI_SHADER_USER_DATA_GS_11_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_12                                                                   0x1a38
+#define regSPI_SHADER_USER_DATA_GS_12_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_13                                                                   0x1a39
+#define regSPI_SHADER_USER_DATA_GS_13_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_14                                                                   0x1a3a
+#define regSPI_SHADER_USER_DATA_GS_14_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_15                                                                   0x1a3b
+#define regSPI_SHADER_USER_DATA_GS_15_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_16                                                                   0x1a3c
+#define regSPI_SHADER_USER_DATA_GS_16_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_17                                                                   0x1a3d
+#define regSPI_SHADER_USER_DATA_GS_17_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_18                                                                   0x1a3e
+#define regSPI_SHADER_USER_DATA_GS_18_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_19                                                                   0x1a3f
+#define regSPI_SHADER_USER_DATA_GS_19_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_20                                                                   0x1a40
+#define regSPI_SHADER_USER_DATA_GS_20_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_21                                                                   0x1a41
+#define regSPI_SHADER_USER_DATA_GS_21_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_22                                                                   0x1a42
+#define regSPI_SHADER_USER_DATA_GS_22_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_23                                                                   0x1a43
+#define regSPI_SHADER_USER_DATA_GS_23_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_24                                                                   0x1a44
+#define regSPI_SHADER_USER_DATA_GS_24_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_25                                                                   0x1a45
+#define regSPI_SHADER_USER_DATA_GS_25_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_26                                                                   0x1a46
+#define regSPI_SHADER_USER_DATA_GS_26_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_27                                                                   0x1a47
+#define regSPI_SHADER_USER_DATA_GS_27_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_28                                                                   0x1a48
+#define regSPI_SHADER_USER_DATA_GS_28_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_29                                                                   0x1a49
+#define regSPI_SHADER_USER_DATA_GS_29_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_30                                                                   0x1a4a
+#define regSPI_SHADER_USER_DATA_GS_30_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_GS_31                                                                   0x1a4b
+#define regSPI_SHADER_USER_DATA_GS_31_BASE_IDX                                                          0
+#define regSPI_SHADER_GS_MESHLET_DIM                                                                    0x1a4c
+#define regSPI_SHADER_GS_MESHLET_DIM_BASE_IDX                                                           0
+#define regSPI_SHADER_GS_MESHLET_EXP_ALLOC                                                              0x1a4d
+#define regSPI_SHADER_GS_MESHLET_EXP_ALLOC_BASE_IDX                                                     0
+#define regSPI_SHADER_REQ_CTRL_ESGS                                                                     0x1a50
+#define regSPI_SHADER_REQ_CTRL_ESGS_BASE_IDX                                                            0
+#define regSPI_SHADER_USER_ACCUM_ESGS_0                                                                 0x1a52
+#define regSPI_SHADER_USER_ACCUM_ESGS_0_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_ESGS_1                                                                 0x1a53
+#define regSPI_SHADER_USER_ACCUM_ESGS_1_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_ESGS_2                                                                 0x1a54
+#define regSPI_SHADER_USER_ACCUM_ESGS_2_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_ESGS_3                                                                 0x1a55
+#define regSPI_SHADER_USER_ACCUM_ESGS_3_BASE_IDX                                                        0
+#define regSPI_SHADER_PGM_LO_ES                                                                         0x1a68
+#define regSPI_SHADER_PGM_LO_ES_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_ES                                                                         0x1a69
+#define regSPI_SHADER_PGM_HI_ES_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_CHKSUM_HS                                                                     0x1aa0
+#define regSPI_SHADER_PGM_CHKSUM_HS_BASE_IDX                                                            0
+#define regSPI_SHADER_PGM_RSRC4_HS                                                                      0x1aa1
+#define regSPI_SHADER_PGM_RSRC4_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_ADDR_LO_HS                                                              0x1aa2
+#define regSPI_SHADER_USER_DATA_ADDR_LO_HS_BASE_IDX                                                     0
+#define regSPI_SHADER_USER_DATA_ADDR_HI_HS                                                              0x1aa3
+#define regSPI_SHADER_USER_DATA_ADDR_HI_HS_BASE_IDX                                                     0
+#define regSPI_SHADER_PGM_LO_LS_HS                                                                      0x1aa4
+#define regSPI_SHADER_PGM_LO_LS_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_HI_LS_HS                                                                      0x1aa5
+#define regSPI_SHADER_PGM_HI_LS_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC3_HS                                                                      0x1aa7
+#define regSPI_SHADER_PGM_RSRC3_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_LO_HS                                                                         0x1aa8
+#define regSPI_SHADER_PGM_LO_HS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_HS                                                                         0x1aa9
+#define regSPI_SHADER_PGM_HI_HS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_RSRC1_HS                                                                      0x1aaa
+#define regSPI_SHADER_PGM_RSRC1_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_PGM_RSRC2_HS                                                                      0x1aab
+#define regSPI_SHADER_PGM_RSRC2_HS_BASE_IDX                                                             0
+#define regSPI_SHADER_USER_DATA_HS_0                                                                    0x1aac
+#define regSPI_SHADER_USER_DATA_HS_0_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_1                                                                    0x1aad
+#define regSPI_SHADER_USER_DATA_HS_1_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_2                                                                    0x1aae
+#define regSPI_SHADER_USER_DATA_HS_2_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_3                                                                    0x1aaf
+#define regSPI_SHADER_USER_DATA_HS_3_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_4                                                                    0x1ab0
+#define regSPI_SHADER_USER_DATA_HS_4_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_5                                                                    0x1ab1
+#define regSPI_SHADER_USER_DATA_HS_5_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_6                                                                    0x1ab2
+#define regSPI_SHADER_USER_DATA_HS_6_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_7                                                                    0x1ab3
+#define regSPI_SHADER_USER_DATA_HS_7_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_8                                                                    0x1ab4
+#define regSPI_SHADER_USER_DATA_HS_8_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_9                                                                    0x1ab5
+#define regSPI_SHADER_USER_DATA_HS_9_BASE_IDX                                                           0
+#define regSPI_SHADER_USER_DATA_HS_10                                                                   0x1ab6
+#define regSPI_SHADER_USER_DATA_HS_10_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_11                                                                   0x1ab7
+#define regSPI_SHADER_USER_DATA_HS_11_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_12                                                                   0x1ab8
+#define regSPI_SHADER_USER_DATA_HS_12_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_13                                                                   0x1ab9
+#define regSPI_SHADER_USER_DATA_HS_13_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_14                                                                   0x1aba
+#define regSPI_SHADER_USER_DATA_HS_14_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_15                                                                   0x1abb
+#define regSPI_SHADER_USER_DATA_HS_15_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_16                                                                   0x1abc
+#define regSPI_SHADER_USER_DATA_HS_16_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_17                                                                   0x1abd
+#define regSPI_SHADER_USER_DATA_HS_17_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_18                                                                   0x1abe
+#define regSPI_SHADER_USER_DATA_HS_18_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_19                                                                   0x1abf
+#define regSPI_SHADER_USER_DATA_HS_19_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_20                                                                   0x1ac0
+#define regSPI_SHADER_USER_DATA_HS_20_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_21                                                                   0x1ac1
+#define regSPI_SHADER_USER_DATA_HS_21_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_22                                                                   0x1ac2
+#define regSPI_SHADER_USER_DATA_HS_22_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_23                                                                   0x1ac3
+#define regSPI_SHADER_USER_DATA_HS_23_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_24                                                                   0x1ac4
+#define regSPI_SHADER_USER_DATA_HS_24_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_25                                                                   0x1ac5
+#define regSPI_SHADER_USER_DATA_HS_25_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_26                                                                   0x1ac6
+#define regSPI_SHADER_USER_DATA_HS_26_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_27                                                                   0x1ac7
+#define regSPI_SHADER_USER_DATA_HS_27_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_28                                                                   0x1ac8
+#define regSPI_SHADER_USER_DATA_HS_28_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_29                                                                   0x1ac9
+#define regSPI_SHADER_USER_DATA_HS_29_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_30                                                                   0x1aca
+#define regSPI_SHADER_USER_DATA_HS_30_BASE_IDX                                                          0
+#define regSPI_SHADER_USER_DATA_HS_31                                                                   0x1acb
+#define regSPI_SHADER_USER_DATA_HS_31_BASE_IDX                                                          0
+#define regSPI_SHADER_REQ_CTRL_LSHS                                                                     0x1ad0
+#define regSPI_SHADER_REQ_CTRL_LSHS_BASE_IDX                                                            0
+#define regSPI_SHADER_USER_ACCUM_LSHS_0                                                                 0x1ad2
+#define regSPI_SHADER_USER_ACCUM_LSHS_0_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_LSHS_1                                                                 0x1ad3
+#define regSPI_SHADER_USER_ACCUM_LSHS_1_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_LSHS_2                                                                 0x1ad4
+#define regSPI_SHADER_USER_ACCUM_LSHS_2_BASE_IDX                                                        0
+#define regSPI_SHADER_USER_ACCUM_LSHS_3                                                                 0x1ad5
+#define regSPI_SHADER_USER_ACCUM_LSHS_3_BASE_IDX                                                        0
+#define regSPI_SHADER_PGM_LO_LS                                                                         0x1ae8
+#define regSPI_SHADER_PGM_LO_LS_BASE_IDX                                                                0
+#define regSPI_SHADER_PGM_HI_LS                                                                         0x1ae9
+#define regSPI_SHADER_PGM_HI_LS_BASE_IDX                                                                0
+#define regCOMPUTE_DISPATCH_INITIATOR                                                                   0x1ba0
+#define regCOMPUTE_DISPATCH_INITIATOR_BASE_IDX                                                          0
+#define regCOMPUTE_DIM_X                                                                                0x1ba1
+#define regCOMPUTE_DIM_X_BASE_IDX                                                                       0
+#define regCOMPUTE_DIM_Y                                                                                0x1ba2
+#define regCOMPUTE_DIM_Y_BASE_IDX                                                                       0
+#define regCOMPUTE_DIM_Z                                                                                0x1ba3
+#define regCOMPUTE_DIM_Z_BASE_IDX                                                                       0
+#define regCOMPUTE_START_X                                                                              0x1ba4
+#define regCOMPUTE_START_X_BASE_IDX                                                                     0
+#define regCOMPUTE_START_Y                                                                              0x1ba5
+#define regCOMPUTE_START_Y_BASE_IDX                                                                     0
+#define regCOMPUTE_START_Z                                                                              0x1ba6
+#define regCOMPUTE_START_Z_BASE_IDX                                                                     0
+#define regCOMPUTE_NUM_THREAD_X                                                                         0x1ba7
+#define regCOMPUTE_NUM_THREAD_X_BASE_IDX                                                                0
+#define regCOMPUTE_NUM_THREAD_Y                                                                         0x1ba8
+#define regCOMPUTE_NUM_THREAD_Y_BASE_IDX                                                                0
+#define regCOMPUTE_NUM_THREAD_Z                                                                         0x1ba9
+#define regCOMPUTE_NUM_THREAD_Z_BASE_IDX                                                                0
+#define regCOMPUTE_PIPELINESTAT_ENABLE                                                                  0x1baa
+#define regCOMPUTE_PIPELINESTAT_ENABLE_BASE_IDX                                                         0
+#define regCOMPUTE_PERFCOUNT_ENABLE                                                                     0x1bab
+#define regCOMPUTE_PERFCOUNT_ENABLE_BASE_IDX                                                            0
+#define regCOMPUTE_PGM_LO                                                                               0x1bac
+#define regCOMPUTE_PGM_LO_BASE_IDX                                                                      0
+#define regCOMPUTE_PGM_HI                                                                               0x1bad
+#define regCOMPUTE_PGM_HI_BASE_IDX                                                                      0
+#define regCOMPUTE_DISPATCH_PKT_ADDR_LO                                                                 0x1bae
+#define regCOMPUTE_DISPATCH_PKT_ADDR_LO_BASE_IDX                                                        0
+#define regCOMPUTE_DISPATCH_PKT_ADDR_HI                                                                 0x1baf
+#define regCOMPUTE_DISPATCH_PKT_ADDR_HI_BASE_IDX                                                        0
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_LO                                                             0x1bb0
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_LO_BASE_IDX                                                    0
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_HI                                                             0x1bb1
+#define regCOMPUTE_DISPATCH_SCRATCH_BASE_HI_BASE_IDX                                                    0
+#define regCOMPUTE_PGM_RSRC1                                                                            0x1bb2
+#define regCOMPUTE_PGM_RSRC1_BASE_IDX                                                                   0
+#define regCOMPUTE_PGM_RSRC2                                                                            0x1bb3
+#define regCOMPUTE_PGM_RSRC2_BASE_IDX                                                                   0
+#define regCOMPUTE_VMID                                                                                 0x1bb4
+#define regCOMPUTE_VMID_BASE_IDX                                                                        0
+#define regCOMPUTE_RESOURCE_LIMITS                                                                      0x1bb5
+#define regCOMPUTE_RESOURCE_LIMITS_BASE_IDX                                                             0
+#define regCOMPUTE_DESTINATION_EN_SE0                                                                   0x1bb6
+#define regCOMPUTE_DESTINATION_EN_SE0_BASE_IDX                                                          0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE0                                                               0x1bb6
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE0_BASE_IDX                                                      0
+#define regCOMPUTE_DESTINATION_EN_SE1                                                                   0x1bb7
+#define regCOMPUTE_DESTINATION_EN_SE1_BASE_IDX                                                          0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE1                                                               0x1bb7
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE1_BASE_IDX                                                      0
+#define regCOMPUTE_TMPRING_SIZE                                                                         0x1bb8
+#define regCOMPUTE_TMPRING_SIZE_BASE_IDX                                                                0
+#define regCOMPUTE_DESTINATION_EN_SE2                                                                   0x1bb9
+#define regCOMPUTE_DESTINATION_EN_SE2_BASE_IDX                                                          0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE2                                                               0x1bb9
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE2_BASE_IDX                                                      0
+#define regCOMPUTE_DESTINATION_EN_SE3                                                                   0x1bba
+#define regCOMPUTE_DESTINATION_EN_SE3_BASE_IDX                                                          0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE3                                                               0x1bba
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE3_BASE_IDX                                                      0
+#define regCOMPUTE_RESTART_X                                                                            0x1bbb
+#define regCOMPUTE_RESTART_X_BASE_IDX                                                                   0
+#define regCOMPUTE_RESTART_Y                                                                            0x1bbc
+#define regCOMPUTE_RESTART_Y_BASE_IDX                                                                   0
+#define regCOMPUTE_RESTART_Z                                                                            0x1bbd
+#define regCOMPUTE_RESTART_Z_BASE_IDX                                                                   0
+#define regCOMPUTE_THREAD_TRACE_ENABLE                                                                  0x1bbe
+#define regCOMPUTE_THREAD_TRACE_ENABLE_BASE_IDX                                                         0
+#define regCOMPUTE_MISC_RESERVED                                                                        0x1bbf
+#define regCOMPUTE_MISC_RESERVED_BASE_IDX                                                               0
+#define regCOMPUTE_DISPATCH_ID                                                                          0x1bc0
+#define regCOMPUTE_DISPATCH_ID_BASE_IDX                                                                 0
+#define regCOMPUTE_THREADGROUP_ID                                                                       0x1bc1
+#define regCOMPUTE_THREADGROUP_ID_BASE_IDX                                                              0
+#define regCOMPUTE_REQ_CTRL                                                                             0x1bc2
+#define regCOMPUTE_REQ_CTRL_BASE_IDX                                                                    0
+#define regCOMPUTE_USER_ACCUM_0                                                                         0x1bc4
+#define regCOMPUTE_USER_ACCUM_0_BASE_IDX                                                                0
+#define regCOMPUTE_USER_ACCUM_1                                                                         0x1bc5
+#define regCOMPUTE_USER_ACCUM_1_BASE_IDX                                                                0
+#define regCOMPUTE_USER_ACCUM_2                                                                         0x1bc6
+#define regCOMPUTE_USER_ACCUM_2_BASE_IDX                                                                0
+#define regCOMPUTE_USER_ACCUM_3                                                                         0x1bc7
+#define regCOMPUTE_USER_ACCUM_3_BASE_IDX                                                                0
+#define regCOMPUTE_PGM_RSRC3                                                                            0x1bc8
+#define regCOMPUTE_PGM_RSRC3_BASE_IDX                                                                   0
+#define regCOMPUTE_DDID_INDEX                                                                           0x1bc9
+#define regCOMPUTE_DDID_INDEX_BASE_IDX                                                                  0
+#define regCOMPUTE_SHADER_CHKSUM                                                                        0x1bca
+#define regCOMPUTE_SHADER_CHKSUM_BASE_IDX                                                               0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE4                                                               0x1bcb
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE4_BASE_IDX                                                      0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE5                                                               0x1bcc
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE5_BASE_IDX                                                      0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE6                                                               0x1bcd
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE6_BASE_IDX                                                      0
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE7                                                               0x1bce
+#define regCOMPUTE_STATIC_THREAD_MGMT_SE7_BASE_IDX                                                      0
+#define regCOMPUTE_DISPATCH_INTERLEAVE                                                                  0x1bcf
+#define regCOMPUTE_DISPATCH_INTERLEAVE_BASE_IDX                                                         0
+#define regCOMPUTE_RELAUNCH                                                                             0x1bd0
+#define regCOMPUTE_RELAUNCH_BASE_IDX                                                                    0
+#define regCOMPUTE_WAVE_RESTORE_ADDR_LO                                                                 0x1bd1
+#define regCOMPUTE_WAVE_RESTORE_ADDR_LO_BASE_IDX                                                        0
+#define regCOMPUTE_WAVE_RESTORE_ADDR_HI                                                                 0x1bd2
+#define regCOMPUTE_WAVE_RESTORE_ADDR_HI_BASE_IDX                                                        0
+#define regCOMPUTE_RELAUNCH2                                                                            0x1bd3
+#define regCOMPUTE_RELAUNCH2_BASE_IDX                                                                   0
+#define regCOMPUTE_USER_DATA_0                                                                          0x1be0
+#define regCOMPUTE_USER_DATA_0_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_1                                                                          0x1be1
+#define regCOMPUTE_USER_DATA_1_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_2                                                                          0x1be2
+#define regCOMPUTE_USER_DATA_2_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_3                                                                          0x1be3
+#define regCOMPUTE_USER_DATA_3_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_4                                                                          0x1be4
+#define regCOMPUTE_USER_DATA_4_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_5                                                                          0x1be5
+#define regCOMPUTE_USER_DATA_5_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_6                                                                          0x1be6
+#define regCOMPUTE_USER_DATA_6_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_7                                                                          0x1be7
+#define regCOMPUTE_USER_DATA_7_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_8                                                                          0x1be8
+#define regCOMPUTE_USER_DATA_8_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_9                                                                          0x1be9
+#define regCOMPUTE_USER_DATA_9_BASE_IDX                                                                 0
+#define regCOMPUTE_USER_DATA_10                                                                         0x1bea
+#define regCOMPUTE_USER_DATA_10_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_11                                                                         0x1beb
+#define regCOMPUTE_USER_DATA_11_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_12                                                                         0x1bec
+#define regCOMPUTE_USER_DATA_12_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_13                                                                         0x1bed
+#define regCOMPUTE_USER_DATA_13_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_14                                                                         0x1bee
+#define regCOMPUTE_USER_DATA_14_BASE_IDX                                                                0
+#define regCOMPUTE_USER_DATA_15                                                                         0x1bef
+#define regCOMPUTE_USER_DATA_15_BASE_IDX                                                                0
+#define regCOMPUTE_DISPATCH_TUNNEL                                                                      0x1c1d
+#define regCOMPUTE_DISPATCH_TUNNEL_BASE_IDX                                                             0
+#define regCOMPUTE_DISPATCH_END                                                                         0x1c1e
+#define regCOMPUTE_DISPATCH_END_BASE_IDX                                                                0
+#define regCOMPUTE_NOWHERE                                                                              0x1c1f
+#define regCOMPUTE_NOWHERE_BASE_IDX                                                                     0
+#define regSH_RESERVED_REG0                                                                             0x1c20
+#define regSH_RESERVED_REG0_BASE_IDX                                                                    0
+#define regSH_RESERVED_REG1                                                                             0x1c21
+#define regSH_RESERVED_REG1_BASE_IDX                                                                    0
+
+
+// addressBlock: gc_cppdec
+// base address: 0xc080
+#define regCP_CU_MASK_ADDR_LO                                                                           0x1dd2
+#define regCP_CU_MASK_ADDR_LO_BASE_IDX                                                                  0
+#define regCP_CU_MASK_ADDR_HI                                                                           0x1dd3
+#define regCP_CU_MASK_ADDR_HI_BASE_IDX                                                                  0
+#define regCP_CU_MASK_CNTL                                                                              0x1dd4
+#define regCP_CU_MASK_CNTL_BASE_IDX                                                                     0
+#define regCP_EOPQ_WAIT_TIME                                                                            0x1dd5
+#define regCP_EOPQ_WAIT_TIME_BASE_IDX                                                                   0
+#define regCP_CPC_MGCG_SYNC_CNTL                                                                        0x1dd6
+#define regCP_CPC_MGCG_SYNC_CNTL_BASE_IDX                                                               0
+#define regCPC_INT_INFO                                                                                 0x1dd7
+#define regCPC_INT_INFO_BASE_IDX                                                                        0
+#define regCP_VIRT_STATUS                                                                               0x1dd8
+#define regCP_VIRT_STATUS_BASE_IDX                                                                      0
+#define regCPC_INT_ADDR                                                                                 0x1dd9
+#define regCPC_INT_ADDR_BASE_IDX                                                                        0
+#define regCPC_INT_PASID                                                                                0x1dda
+#define regCPC_INT_PASID_BASE_IDX                                                                       0
+#define regCP_GFX_ERROR                                                                                 0x1ddb
+#define regCP_GFX_ERROR_BASE_IDX                                                                        0
+#define regCPG_UTCL1_CNTL                                                                               0x1ddc
+#define regCPG_UTCL1_CNTL_BASE_IDX                                                                      0
+#define regCPC_UTCL1_CNTL                                                                               0x1ddd
+#define regCPC_UTCL1_CNTL_BASE_IDX                                                                      0
+#define regCPF_UTCL1_CNTL                                                                               0x1dde
+#define regCPF_UTCL1_CNTL_BASE_IDX                                                                      0
+#define regCP_AQL_SMM_STATUS                                                                            0x1ddf
+#define regCP_AQL_SMM_STATUS_BASE_IDX                                                                   0
+#define regCP_RB0_BASE                                                                                  0x1de0
+#define regCP_RB0_BASE_BASE_IDX                                                                         0
+#define regCP_RB_BASE                                                                                   0x1de0
+#define regCP_RB_BASE_BASE_IDX                                                                          0
+#define regCP_RB0_CNTL                                                                                  0x1de1
+#define regCP_RB0_CNTL_BASE_IDX                                                                         0
+#define regCP_RB_CNTL                                                                                   0x1de1
+#define regCP_RB_CNTL_BASE_IDX                                                                          0
+#define regCP_RB_RPTR_WR                                                                                0x1de2
+#define regCP_RB_RPTR_WR_BASE_IDX                                                                       0
+#define regCP_RB0_RPTR_ADDR                                                                             0x1de3
+#define regCP_RB0_RPTR_ADDR_BASE_IDX                                                                    0
+#define regCP_RB_RPTR_ADDR                                                                              0x1de3
+#define regCP_RB_RPTR_ADDR_BASE_IDX                                                                     0
+#define regCP_RB0_RPTR_ADDR_HI                                                                          0x1de4
+#define regCP_RB0_RPTR_ADDR_HI_BASE_IDX                                                                 0
+#define regCP_RB_RPTR_ADDR_HI                                                                           0x1de4
+#define regCP_RB_RPTR_ADDR_HI_BASE_IDX                                                                  0
+#define regCP_RB0_BUFSZ_MASK                                                                            0x1de5
+#define regCP_RB0_BUFSZ_MASK_BASE_IDX                                                                   0
+#define regCP_RB_BUFSZ_MASK                                                                             0x1de5
+#define regCP_RB_BUFSZ_MASK_BASE_IDX                                                                    0
+#define regGC_PRIV_MODE                                                                                 0x1de8
+#define regGC_PRIV_MODE_BASE_IDX                                                                        0
+#define regCP_INT_CNTL                                                                                  0x1de9
+#define regCP_INT_CNTL_BASE_IDX                                                                         0
+#define regCP_INT_STATUS                                                                                0x1dea
+#define regCP_INT_STATUS_BASE_IDX                                                                       0
+#define regCP_DEVICE_ID                                                                                 0x1deb
+#define regCP_DEVICE_ID_BASE_IDX                                                                        0
+#define regCP_ME0_PIPE_PRIORITY_CNTS                                                                    0x1dec
+#define regCP_ME0_PIPE_PRIORITY_CNTS_BASE_IDX                                                           0
+#define regCP_RING_PRIORITY_CNTS                                                                        0x1dec
+#define regCP_RING_PRIORITY_CNTS_BASE_IDX                                                               0
+#define regCP_ME0_PIPE0_PRIORITY                                                                        0x1ded
+#define regCP_ME0_PIPE0_PRIORITY_BASE_IDX                                                               0
+#define regCP_RING0_PRIORITY                                                                            0x1ded
+#define regCP_RING0_PRIORITY_BASE_IDX                                                                   0
+#define regCP_ME0_PIPE1_PRIORITY                                                                        0x1dee
+#define regCP_ME0_PIPE1_PRIORITY_BASE_IDX                                                               0
+#define regCP_RING1_PRIORITY                                                                            0x1dee
+#define regCP_RING1_PRIORITY_BASE_IDX                                                                   0
+#define regCP_FATAL_ERROR                                                                               0x1df0
+#define regCP_FATAL_ERROR_BASE_IDX                                                                      0
+#define regCP_RB_VMID                                                                                   0x1df1
+#define regCP_RB_VMID_BASE_IDX                                                                          0
+#define regCP_ME0_PIPE0_VMID                                                                            0x1df2
+#define regCP_ME0_PIPE0_VMID_BASE_IDX                                                                   0
+#define regCP_ME0_PIPE1_VMID                                                                            0x1df3
+#define regCP_ME0_PIPE1_VMID_BASE_IDX                                                                   0
+#define regCP_RB0_WPTR                                                                                  0x1df4
+#define regCP_RB0_WPTR_BASE_IDX                                                                         0
+#define regCP_RB_WPTR                                                                                   0x1df4
+#define regCP_RB_WPTR_BASE_IDX                                                                          0
+#define regCP_RB0_WPTR_HI                                                                               0x1df5
+#define regCP_RB0_WPTR_HI_BASE_IDX                                                                      0
+#define regCP_RB_WPTR_HI                                                                                0x1df5
+#define regCP_RB_WPTR_HI_BASE_IDX                                                                       0
+#define regCP_RB1_WPTR                                                                                  0x1df6
+#define regCP_RB1_WPTR_BASE_IDX                                                                         0
+#define regCP_RB1_WPTR_HI                                                                               0x1df7
+#define regCP_RB1_WPTR_HI_BASE_IDX                                                                      0
+#define regCP_PROCESS_QUANTUM                                                                           0x1df9
+#define regCP_PROCESS_QUANTUM_BASE_IDX                                                                  0
+#define regCP_RB_DOORBELL_RANGE_LOWER                                                                   0x1dfa
+#define regCP_RB_DOORBELL_RANGE_LOWER_BASE_IDX                                                          0
+#define regCP_RB_DOORBELL_RANGE_UPPER                                                                   0x1dfb
+#define regCP_RB_DOORBELL_RANGE_UPPER_BASE_IDX                                                          0
+#define regCP_MEC_DOORBELL_RANGE_LOWER                                                                  0x1dfc
+#define regCP_MEC_DOORBELL_RANGE_LOWER_BASE_IDX                                                         0
+#define regCP_MEC_DOORBELL_RANGE_UPPER                                                                  0x1dfd
+#define regCP_MEC_DOORBELL_RANGE_UPPER_BASE_IDX                                                         0
+#define regCPG_UTCL1_ERROR                                                                              0x1dfe
+#define regCPG_UTCL1_ERROR_BASE_IDX                                                                     0
+#define regCPC_UTCL1_ERROR                                                                              0x1dff
+#define regCPC_UTCL1_ERROR_BASE_IDX                                                                     0
+#define regCP_RB1_BASE                                                                                  0x1e00
+#define regCP_RB1_BASE_BASE_IDX                                                                         0
+#define regCP_RB1_CNTL                                                                                  0x1e01
+#define regCP_RB1_CNTL_BASE_IDX                                                                         0
+#define regCP_RB1_RPTR_ADDR                                                                             0x1e02
+#define regCP_RB1_RPTR_ADDR_BASE_IDX                                                                    0
+#define regCP_RB1_RPTR_ADDR_HI                                                                          0x1e03
+#define regCP_RB1_RPTR_ADDR_HI_BASE_IDX                                                                 0
+#define regCP_RB1_BUFSZ_MASK                                                                            0x1e04
+#define regCP_RB1_BUFSZ_MASK_BASE_IDX                                                                   0
+#define regCP_INT_CNTL_RING0                                                                            0x1e0a
+#define regCP_INT_CNTL_RING0_BASE_IDX                                                                   0
+#define regCP_INT_CNTL_RING1                                                                            0x1e0b
+#define regCP_INT_CNTL_RING1_BASE_IDX                                                                   0
+#define regCP_INT_STATUS_RING0                                                                          0x1e0d
+#define regCP_INT_STATUS_RING0_BASE_IDX                                                                 0
+#define regCP_INT_STATUS_RING1                                                                          0x1e0e
+#define regCP_INT_STATUS_RING1_BASE_IDX                                                                 0
+#define regCP_ME_F32_INTERRUPT                                                                          0x1e13
+#define regCP_ME_F32_INTERRUPT_BASE_IDX                                                                 0
+#define regCP_PFP_F32_INTERRUPT                                                                         0x1e14
+#define regCP_PFP_F32_INTERRUPT_BASE_IDX                                                                0
+#define regCP_MEC1_F32_INTERRUPT                                                                        0x1e16
+#define regCP_MEC1_F32_INTERRUPT_BASE_IDX                                                               0
+#define regCP_MEC2_F32_INTERRUPT                                                                        0x1e17
+#define regCP_MEC2_F32_INTERRUPT_BASE_IDX                                                               0
+#define regCP_PWR_CNTL                                                                                  0x1e18
+#define regCP_PWR_CNTL_BASE_IDX                                                                         0
+#define regCP_ECC_FIRSTOCCURRENCE                                                                       0x1e1a
+#define regCP_ECC_FIRSTOCCURRENCE_BASE_IDX                                                              0
+#define regCP_ECC_FIRSTOCCURRENCE_RING0                                                                 0x1e1b
+#define regCP_ECC_FIRSTOCCURRENCE_RING0_BASE_IDX                                                        0
+#define regCP_ECC_FIRSTOCCURRENCE_RING1                                                                 0x1e1c
+#define regCP_ECC_FIRSTOCCURRENCE_RING1_BASE_IDX                                                        0
+#define regGB_EDC_MODE                                                                                  0x1e1e
+#define regGB_EDC_MODE_BASE_IDX                                                                         0
+#define regCP_DEBUG                                                                                     0x1e1f
+#define regCP_DEBUG_BASE_IDX                                                                            0
+#define regCP_CPF_DEBUG                                                                                 0x1e20
+#define regCP_CPF_DEBUG_BASE_IDX                                                                        0
+#define regCP_CPC_DEBUG                                                                                 0x1e21
+#define regCP_CPC_DEBUG_BASE_IDX                                                                        0
+#define regCP_PQ_WPTR_POLL_CNTL                                                                         0x1e23
+#define regCP_PQ_WPTR_POLL_CNTL_BASE_IDX                                                                0
+#define regCP_PQ_WPTR_POLL_CNTL1                                                                        0x1e24
+#define regCP_PQ_WPTR_POLL_CNTL1_BASE_IDX                                                               0
+#define regCP_ME1_PIPE0_INT_CNTL                                                                        0x1e25
+#define regCP_ME1_PIPE0_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE1_INT_CNTL                                                                        0x1e26
+#define regCP_ME1_PIPE1_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE2_INT_CNTL                                                                        0x1e27
+#define regCP_ME1_PIPE2_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE3_INT_CNTL                                                                        0x1e28
+#define regCP_ME1_PIPE3_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME2_PIPE0_INT_CNTL                                                                        0x1e29
+#define regCP_ME2_PIPE0_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME2_PIPE1_INT_CNTL                                                                        0x1e2a
+#define regCP_ME2_PIPE1_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME2_PIPE2_INT_CNTL                                                                        0x1e2b
+#define regCP_ME2_PIPE2_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME2_PIPE3_INT_CNTL                                                                        0x1e2c
+#define regCP_ME2_PIPE3_INT_CNTL_BASE_IDX                                                               0
+#define regCP_ME1_PIPE0_INT_STATUS                                                                      0x1e2d
+#define regCP_ME1_PIPE0_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_PIPE1_INT_STATUS                                                                      0x1e2e
+#define regCP_ME1_PIPE1_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_PIPE2_INT_STATUS                                                                      0x1e2f
+#define regCP_ME1_PIPE2_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_PIPE3_INT_STATUS                                                                      0x1e30
+#define regCP_ME1_PIPE3_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME2_PIPE0_INT_STATUS                                                                      0x1e31
+#define regCP_ME2_PIPE0_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME2_PIPE1_INT_STATUS                                                                      0x1e32
+#define regCP_ME2_PIPE1_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME2_PIPE2_INT_STATUS                                                                      0x1e33
+#define regCP_ME2_PIPE2_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME2_PIPE3_INT_STATUS                                                                      0x1e34
+#define regCP_ME2_PIPE3_INT_STATUS_BASE_IDX                                                             0
+#define regCP_ME1_INT_STAT_DEBUG                                                                        0x1e35
+#define regCP_ME1_INT_STAT_DEBUG_BASE_IDX                                                               0
+#define regCP_ME2_INT_STAT_DEBUG                                                                        0x1e36
+#define regCP_ME2_INT_STAT_DEBUG_BASE_IDX                                                               0
+#define regCP_GFX_QUEUE_INDEX                                                                           0x1e37
+#define regCP_GFX_QUEUE_INDEX_BASE_IDX                                                                  0
+#define regCC_GC_EDC_CONFIG                                                                             0x1e38
+#define regCC_GC_EDC_CONFIG_BASE_IDX                                                                    0
+#define regCP_ME1_PIPE_PRIORITY_CNTS                                                                    0x1e39
+#define regCP_ME1_PIPE_PRIORITY_CNTS_BASE_IDX                                                           0
+#define regCP_ME1_PIPE0_PRIORITY                                                                        0x1e3a
+#define regCP_ME1_PIPE0_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME1_PIPE1_PRIORITY                                                                        0x1e3b
+#define regCP_ME1_PIPE1_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME1_PIPE2_PRIORITY                                                                        0x1e3c
+#define regCP_ME1_PIPE2_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME1_PIPE3_PRIORITY                                                                        0x1e3d
+#define regCP_ME1_PIPE3_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME2_PIPE_PRIORITY_CNTS                                                                    0x1e3e
+#define regCP_ME2_PIPE_PRIORITY_CNTS_BASE_IDX                                                           0
+#define regCP_ME2_PIPE0_PRIORITY                                                                        0x1e3f
+#define regCP_ME2_PIPE0_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME2_PIPE1_PRIORITY                                                                        0x1e40
+#define regCP_ME2_PIPE1_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME2_PIPE2_PRIORITY                                                                        0x1e41
+#define regCP_ME2_PIPE2_PRIORITY_BASE_IDX                                                               0
+#define regCP_ME2_PIPE3_PRIORITY                                                                        0x1e42
+#define regCP_ME2_PIPE3_PRIORITY_BASE_IDX                                                               0
+#define regCP_PFP_PRGRM_CNTR_START                                                                      0x1e44
+#define regCP_PFP_PRGRM_CNTR_START_BASE_IDX                                                             0
+#define regCP_ME_PRGRM_CNTR_START                                                                       0x1e45
+#define regCP_ME_PRGRM_CNTR_START_BASE_IDX                                                              0
+#define regCP_MEC1_PRGRM_CNTR_START                                                                     0x1e46
+#define regCP_MEC1_PRGRM_CNTR_START_BASE_IDX                                                            0
+#define regCP_MEC2_PRGRM_CNTR_START                                                                     0x1e47
+#define regCP_MEC2_PRGRM_CNTR_START_BASE_IDX                                                            0
+#define regCP_PFP_INTR_ROUTINE_START                                                                    0x1e49
+#define regCP_PFP_INTR_ROUTINE_START_BASE_IDX                                                           0
+#define regCP_ME_INTR_ROUTINE_START                                                                     0x1e4a
+#define regCP_ME_INTR_ROUTINE_START_BASE_IDX                                                            0
+#define regCP_MEC1_INTR_ROUTINE_START                                                                   0x1e4b
+#define regCP_MEC1_INTR_ROUTINE_START_BASE_IDX                                                          0
+#define regCP_MEC2_INTR_ROUTINE_START                                                                   0x1e4c
+#define regCP_MEC2_INTR_ROUTINE_START_BASE_IDX                                                          0
+#define regCP_CONTEXT_CNTL                                                                              0x1e4d
+#define regCP_CONTEXT_CNTL_BASE_IDX                                                                     0
+#define regCP_MAX_CONTEXT                                                                               0x1e4e
+#define regCP_MAX_CONTEXT_BASE_IDX                                                                      0
+#define regCP_IQ_WAIT_TIME1                                                                             0x1e4f
+#define regCP_IQ_WAIT_TIME1_BASE_IDX                                                                    0
+#define regCP_IQ_WAIT_TIME2                                                                             0x1e50
+#define regCP_IQ_WAIT_TIME2_BASE_IDX                                                                    0
+#define regCP_RB0_BASE_HI                                                                               0x1e51
+#define regCP_RB0_BASE_HI_BASE_IDX                                                                      0
+#define regCP_RB1_BASE_HI                                                                               0x1e52
+#define regCP_RB1_BASE_HI_BASE_IDX                                                                      0
+#define regCP_VMID_RESET                                                                                0x1e53
+#define regCP_VMID_RESET_BASE_IDX                                                                       0
+#define regCPC_INT_CNTL                                                                                 0x1e54
+#define regCPC_INT_CNTL_BASE_IDX                                                                        0
+#define regCPC_INT_STATUS                                                                               0x1e55
+#define regCPC_INT_STATUS_BASE_IDX                                                                      0
+#define regCP_VMID_PREEMPT                                                                              0x1e56
+#define regCP_VMID_PREEMPT_BASE_IDX                                                                     0
+#define regCPC_INT_CNTX_ID                                                                              0x1e57
+#define regCPC_INT_CNTX_ID_BASE_IDX                                                                     0
+#define regCP_PQ_STATUS                                                                                 0x1e58
+#define regCP_PQ_STATUS_BASE_IDX                                                                        0
+#define regCP_PFP_PRGRM_CNTR_START_HI                                                                   0x1e59
+#define regCP_PFP_PRGRM_CNTR_START_HI_BASE_IDX                                                          0
+#define regCP_MAX_DRAW_COUNT                                                                            0x1e5c
+#define regCP_MAX_DRAW_COUNT_BASE_IDX                                                                   0
+#define regCP_MEC1_F32_INT_DIS                                                                          0x1e5d
+#define regCP_MEC1_F32_INT_DIS_BASE_IDX                                                                 0
+#define regCP_MEC2_F32_INT_DIS                                                                          0x1e5e
+#define regCP_MEC2_F32_INT_DIS_BASE_IDX                                                                 0
+#define regCP_VMID_STATUS                                                                               0x1e5f
+#define regCP_VMID_STATUS_BASE_IDX                                                                      0
+#define regCPC_SUSPEND_CTX_SAVE_BASE_ADDR_LO                                                            0x1e60
+#define regCPC_SUSPEND_CTX_SAVE_BASE_ADDR_LO_BASE_IDX                                                   0
+#define regCPC_SUSPEND_CTX_SAVE_BASE_ADDR_HI                                                            0x1e61
+#define regCPC_SUSPEND_CTX_SAVE_BASE_ADDR_HI_BASE_IDX                                                   0
+#define regCPC_SUSPEND_CTX_SAVE_CONTROL                                                                 0x1e62
+#define regCPC_SUSPEND_CTX_SAVE_CONTROL_BASE_IDX                                                        0
+#define regCPC_SUSPEND_CNTL_STACK_OFFSET                                                                0x1e63
+#define regCPC_SUSPEND_CNTL_STACK_OFFSET_BASE_IDX                                                       0
+#define regCPC_SUSPEND_CNTL_STACK_SIZE                                                                  0x1e64
+#define regCPC_SUSPEND_CNTL_STACK_SIZE_BASE_IDX                                                         0
+#define regCPC_SUSPEND_WG_STATE_OFFSET                                                                  0x1e65
+#define regCPC_SUSPEND_WG_STATE_OFFSET_BASE_IDX                                                         0
+#define regCPC_SUSPEND_CTX_SAVE_SIZE                                                                    0x1e66
+#define regCPC_SUSPEND_CTX_SAVE_SIZE_BASE_IDX                                                           0
+#define regCPC_OS_PIPES                                                                                 0x1e67
+#define regCPC_OS_PIPES_BASE_IDX                                                                        0
+#define regCP_SUSPEND_RESUME_REQ                                                                        0x1e68
+#define regCP_SUSPEND_RESUME_REQ_BASE_IDX                                                               0
+#define regCP_SUSPEND_CNTL                                                                              0x1e69
+#define regCP_SUSPEND_CNTL_BASE_IDX                                                                     0
+#define regCP_IQ_WAIT_TIME3                                                                             0x1e6a
+#define regCP_IQ_WAIT_TIME3_BASE_IDX                                                                    0
+#define regCPC_DDID_BASE_ADDR_LO                                                                        0x1e6b
+#define regCPC_DDID_BASE_ADDR_LO_BASE_IDX                                                               0
+#define regCP_DDID_BASE_ADDR_LO                                                                         0x1e6b
+#define regCP_DDID_BASE_ADDR_LO_BASE_IDX                                                                0
+#define regCPC_DDID_BASE_ADDR_HI                                                                        0x1e6c
+#define regCPC_DDID_BASE_ADDR_HI_BASE_IDX                                                               0
+#define regCP_DDID_BASE_ADDR_HI                                                                         0x1e6c
+#define regCP_DDID_BASE_ADDR_HI_BASE_IDX                                                                0
+#define regCPC_DDID_CNTL                                                                                0x1e6d
+#define regCPC_DDID_CNTL_BASE_IDX                                                                       0
+#define regCP_DDID_CNTL                                                                                 0x1e6d
+#define regCP_DDID_CNTL_BASE_IDX                                                                        0
+#define regCP_GFX_DDID_INFLIGHT_COUNT                                                                   0x1e6e
+#define regCP_GFX_DDID_INFLIGHT_COUNT_BASE_IDX                                                          0
+#define regCP_GFX_DDID_WPTR                                                                             0x1e6f
+#define regCP_GFX_DDID_WPTR_BASE_IDX                                                                    0
+#define regCP_GFX_DDID_RPTR                                                                             0x1e70
+#define regCP_GFX_DDID_RPTR_BASE_IDX                                                                    0
+#define regCP_GFX_DDID_DELTA_RPT_COUNT                                                                  0x1e71
+#define regCP_GFX_DDID_DELTA_RPT_COUNT_BASE_IDX                                                         0
+#define regCP_GFX_HPD_STATUS0                                                                           0x1e72
+#define regCP_GFX_HPD_STATUS0_BASE_IDX                                                                  0
+#define regCP_GFX_HPD_CONTROL0                                                                          0x1e73
+#define regCP_GFX_HPD_CONTROL0_BASE_IDX                                                                 0
+#define regCP_GFX_HPD_OSPRE_FENCE_ADDR_LO                                                               0x1e74
+#define regCP_GFX_HPD_OSPRE_FENCE_ADDR_LO_BASE_IDX                                                      0
+#define regCP_GFX_HPD_OSPRE_FENCE_ADDR_HI                                                               0x1e75
+#define regCP_GFX_HPD_OSPRE_FENCE_ADDR_HI_BASE_IDX                                                      0
+#define regCP_GFX_HPD_OSPRE_FENCE_DATA_LO                                                               0x1e76
+#define regCP_GFX_HPD_OSPRE_FENCE_DATA_LO_BASE_IDX                                                      0
+#define regCP_GFX_HPD_OSPRE_FENCE_DATA_HI                                                               0x1e77
+#define regCP_GFX_HPD_OSPRE_FENCE_DATA_HI_BASE_IDX                                                      0
+#define regCP_GFX_INDEX_MUTEX                                                                           0x1e78
+#define regCP_GFX_INDEX_MUTEX_BASE_IDX                                                                  0
+#define regCP_ME_PRGRM_CNTR_START_HI                                                                    0x1e79
+#define regCP_ME_PRGRM_CNTR_START_HI_BASE_IDX                                                           0
+#define regCP_PFP_INTR_ROUTINE_START_HI                                                                 0x1e7a
+#define regCP_PFP_INTR_ROUTINE_START_HI_BASE_IDX                                                        0
+#define regCP_ME_INTR_ROUTINE_START_HI                                                                  0x1e7b
+#define regCP_ME_INTR_ROUTINE_START_HI_BASE_IDX                                                         0
+#define regCP_GFX_MQD_BASE_ADDR                                                                         0x1e7e
+#define regCP_GFX_MQD_BASE_ADDR_BASE_IDX                                                                0
+#define regCP_GFX_MQD_BASE_ADDR_HI                                                                      0x1e7f
+#define regCP_GFX_MQD_BASE_ADDR_HI_BASE_IDX                                                             0
+#define regCP_GFX_HQD_ACTIVE                                                                            0x1e80
+#define regCP_GFX_HQD_ACTIVE_BASE_IDX                                                                   0
+#define regCP_GFX_HQD_VMID                                                                              0x1e81
+#define regCP_GFX_HQD_VMID_BASE_IDX                                                                     0
+#define regCP_GFX_HQD_QUEUE_PRIORITY                                                                    0x1e84
+#define regCP_GFX_HQD_QUEUE_PRIORITY_BASE_IDX                                                           0
+#define regCP_GFX_HQD_QUANTUM                                                                           0x1e85
+#define regCP_GFX_HQD_QUANTUM_BASE_IDX                                                                  0
+#define regCP_GFX_HQD_BASE                                                                              0x1e86
+#define regCP_GFX_HQD_BASE_BASE_IDX                                                                     0
+#define regCP_GFX_HQD_BASE_HI                                                                           0x1e87
+#define regCP_GFX_HQD_BASE_HI_BASE_IDX                                                                  0
+#define regCP_GFX_HQD_RPTR                                                                              0x1e88
+#define regCP_GFX_HQD_RPTR_BASE_IDX                                                                     0
+#define regCP_GFX_HQD_RPTR_ADDR                                                                         0x1e89
+#define regCP_GFX_HQD_RPTR_ADDR_BASE_IDX                                                                0
+#define regCP_GFX_HQD_RPTR_ADDR_HI                                                                      0x1e8a
+#define regCP_GFX_HQD_RPTR_ADDR_HI_BASE_IDX                                                             0
+#define regCP_RB_WPTR_POLL_ADDR_LO                                                                      0x1e8b
+#define regCP_RB_WPTR_POLL_ADDR_LO_BASE_IDX                                                             0
+#define regCP_RB_WPTR_POLL_ADDR_HI                                                                      0x1e8c
+#define regCP_RB_WPTR_POLL_ADDR_HI_BASE_IDX                                                             0
+#define regCP_RB_DOORBELL_CONTROL                                                                       0x1e8d
+#define regCP_RB_DOORBELL_CONTROL_BASE_IDX                                                              0
+#define regCP_GFX_HQD_OFFSET                                                                            0x1e8e
+#define regCP_GFX_HQD_OFFSET_BASE_IDX                                                                   0
+#define regCP_GFX_HQD_CNTL                                                                              0x1e8f
+#define regCP_GFX_HQD_CNTL_BASE_IDX                                                                     0
+#define regCP_GFX_HQD_CSMD_RPTR                                                                         0x1e90
+#define regCP_GFX_HQD_CSMD_RPTR_BASE_IDX                                                                0
+#define regCP_GFX_HQD_WPTR                                                                              0x1e91
+#define regCP_GFX_HQD_WPTR_BASE_IDX                                                                     0
+#define regCP_GFX_HQD_WPTR_HI                                                                           0x1e92
+#define regCP_GFX_HQD_WPTR_HI_BASE_IDX                                                                  0
+#define regCP_GFX_HQD_DEQUEUE_REQUEST                                                                   0x1e93
+#define regCP_GFX_HQD_DEQUEUE_REQUEST_BASE_IDX                                                          0
+#define regCP_GFX_HQD_MAPPED                                                                            0x1e94
+#define regCP_GFX_HQD_MAPPED_BASE_IDX                                                                   0
+#define regCP_GFX_HQD_QUE_MGR_CONTROL                                                                   0x1e95
+#define regCP_GFX_HQD_QUE_MGR_CONTROL_BASE_IDX                                                          0
+#define regCP_GFX_HQD_IQ_TIMER                                                                          0x1e96
+#define regCP_GFX_HQD_IQ_TIMER_BASE_IDX                                                                 0
+#define regCP_GFX_HQD_HQ_STATUS0                                                                        0x1e98
+#define regCP_GFX_HQD_HQ_STATUS0_BASE_IDX                                                               0
+#define regCP_GFX_HQD_HQ_CONTROL0                                                                       0x1e99
+#define regCP_GFX_HQD_HQ_CONTROL0_BASE_IDX                                                              0
+#define regCP_GFX_MQD_CONTROL                                                                           0x1e9a
+#define regCP_GFX_MQD_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_GFX_CONTROL                                                                           0x1e9f
+#define regCP_HQD_GFX_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_GFX_STATUS                                                                            0x1ea0
+#define regCP_HQD_GFX_STATUS_BASE_IDX                                                                   0
+#define regCP_DMA_WATCH0_ADDR_LO                                                                        0x1ec0
+#define regCP_DMA_WATCH0_ADDR_LO_BASE_IDX                                                               0
+#define regCP_DMA_WATCH0_ADDR_HI                                                                        0x1ec1
+#define regCP_DMA_WATCH0_ADDR_HI_BASE_IDX                                                               0
+#define regCP_DMA_WATCH0_MASK                                                                           0x1ec2
+#define regCP_DMA_WATCH0_MASK_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH0_CNTL                                                                           0x1ec3
+#define regCP_DMA_WATCH0_CNTL_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH1_ADDR_LO                                                                        0x1ec4
+#define regCP_DMA_WATCH1_ADDR_LO_BASE_IDX                                                               0
+#define regCP_DMA_WATCH1_ADDR_HI                                                                        0x1ec5
+#define regCP_DMA_WATCH1_ADDR_HI_BASE_IDX                                                               0
+#define regCP_DMA_WATCH1_MASK                                                                           0x1ec6
+#define regCP_DMA_WATCH1_MASK_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH1_CNTL                                                                           0x1ec7
+#define regCP_DMA_WATCH1_CNTL_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH2_ADDR_LO                                                                        0x1ec8
+#define regCP_DMA_WATCH2_ADDR_LO_BASE_IDX                                                               0
+#define regCP_DMA_WATCH2_ADDR_HI                                                                        0x1ec9
+#define regCP_DMA_WATCH2_ADDR_HI_BASE_IDX                                                               0
+#define regCP_DMA_WATCH2_MASK                                                                           0x1eca
+#define regCP_DMA_WATCH2_MASK_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH2_CNTL                                                                           0x1ecb
+#define regCP_DMA_WATCH2_CNTL_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH3_ADDR_LO                                                                        0x1ecc
+#define regCP_DMA_WATCH3_ADDR_LO_BASE_IDX                                                               0
+#define regCP_DMA_WATCH3_ADDR_HI                                                                        0x1ecd
+#define regCP_DMA_WATCH3_ADDR_HI_BASE_IDX                                                               0
+#define regCP_DMA_WATCH3_MASK                                                                           0x1ece
+#define regCP_DMA_WATCH3_MASK_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH3_CNTL                                                                           0x1ecf
+#define regCP_DMA_WATCH3_CNTL_BASE_IDX                                                                  0
+#define regCP_DMA_WATCH_STAT_ADDR_LO                                                                    0x1ed0
+#define regCP_DMA_WATCH_STAT_ADDR_LO_BASE_IDX                                                           0
+#define regCP_DMA_WATCH_STAT_ADDR_HI                                                                    0x1ed1
+#define regCP_DMA_WATCH_STAT_ADDR_HI_BASE_IDX                                                           0
+#define regCP_DMA_WATCH_STAT                                                                            0x1ed2
+#define regCP_DMA_WATCH_STAT_BASE_IDX                                                                   0
+#define regCP_PFP_JT_STAT                                                                               0x1ed3
+#define regCP_PFP_JT_STAT_BASE_IDX                                                                      0
+#define regCP_MEC_JT_STAT                                                                               0x1ed5
+#define regCP_MEC_JT_STAT_BASE_IDX                                                                      0
+#define regCP_CPC_BUSY_HYSTERESIS                                                                       0x1edb
+#define regCP_CPC_BUSY_HYSTERESIS_BASE_IDX                                                              0
+#define regCP_CPF_BUSY_HYSTERESIS1                                                                      0x1edc
+#define regCP_CPF_BUSY_HYSTERESIS1_BASE_IDX                                                             0
+#define regCP_CPF_BUSY_HYSTERESIS2                                                                      0x1edd
+#define regCP_CPF_BUSY_HYSTERESIS2_BASE_IDX                                                             0
+#define regCP_CPG_BUSY_HYSTERESIS1                                                                      0x1ede
+#define regCP_CPG_BUSY_HYSTERESIS1_BASE_IDX                                                             0
+#define regCP_CPG_BUSY_HYSTERESIS2                                                                      0x1edf
+#define regCP_CPG_BUSY_HYSTERESIS2_BASE_IDX                                                             0
+#define regCP_RB_DOORBELL_CLEAR                                                                         0x1f28
+#define regCP_RB_DOORBELL_CLEAR_BASE_IDX                                                                0
+#define regCP_RB0_ACTIVE                                                                                0x1f40
+#define regCP_RB0_ACTIVE_BASE_IDX                                                                       0
+#define regCP_RB_ACTIVE                                                                                 0x1f40
+#define regCP_RB_ACTIVE_BASE_IDX                                                                        0
+#define regCP_RB1_ACTIVE                                                                                0x1f41
+#define regCP_RB1_ACTIVE_BASE_IDX                                                                       0
+#define regCP_RB_STATUS                                                                                 0x1f43
+#define regCP_RB_STATUS_BASE_IDX                                                                        0
+#define regCPG_RCIU_CAM_INDEX                                                                           0x1f44
+#define regCPG_RCIU_CAM_INDEX_BASE_IDX                                                                  0
+#define regCPG_RCIU_CAM_DATA                                                                            0x1f45
+#define regCPG_RCIU_CAM_DATA_BASE_IDX                                                                   0
+#define regCPG_RCIU_CAM_DATA_PHASE0                                                                     0x1f45
+#define regCPG_RCIU_CAM_DATA_PHASE0_BASE_IDX                                                            0
+#define regCPG_RCIU_CAM_DATA_PHASE1                                                                     0x1f45
+#define regCPG_RCIU_CAM_DATA_PHASE1_BASE_IDX                                                            0
+#define regCPG_RCIU_CAM_DATA_PHASE2                                                                     0x1f45
+#define regCPG_RCIU_CAM_DATA_PHASE2_BASE_IDX                                                            0
+#define regCP_GPU_TIMESTAMP_OFFSET_LO                                                                   0x1f4c
+#define regCP_GPU_TIMESTAMP_OFFSET_LO_BASE_IDX                                                          0
+#define regCP_GPU_TIMESTAMP_OFFSET_HI                                                                   0x1f4d
+#define regCP_GPU_TIMESTAMP_OFFSET_HI_BASE_IDX                                                          0
+#define regCP_SDMA_DMA_DONE                                                                             0x1f4e
+#define regCP_SDMA_DMA_DONE_BASE_IDX                                                                    0
+#define regCP_PFP_SDMA_CS                                                                               0x1f4f
+#define regCP_PFP_SDMA_CS_BASE_IDX                                                                      0
+#define regCP_ME_SDMA_CS                                                                                0x1f50
+#define regCP_ME_SDMA_CS_BASE_IDX                                                                       0
+#define regCPF_GCR_CNTL                                                                                 0x1f53
+#define regCPF_GCR_CNTL_BASE_IDX                                                                        0
+#define regCPG_UTCL1_STATUS                                                                             0x1f54
+#define regCPG_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regCPC_UTCL1_STATUS                                                                             0x1f55
+#define regCPC_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regCPF_UTCL1_STATUS                                                                             0x1f56
+#define regCPF_UTCL1_STATUS_BASE_IDX                                                                    0
+#define regCP_SD_CNTL                                                                                   0x1f57
+#define regCP_SD_CNTL_BASE_IDX                                                                          0
+#define regCP_SOFT_RESET_CNTL                                                                           0x1f59
+#define regCP_SOFT_RESET_CNTL_BASE_IDX                                                                  0
+#define regCP_CPC_GFX_CNTL                                                                              0x1f5a
+#define regCP_CPC_GFX_CNTL_BASE_IDX                                                                     0
+
+
+// addressBlock: gc_spipdec
+// base address: 0xc700
+#define regSPI_ARB_PRIORITY                                                                             0x1f60
+#define regSPI_ARB_PRIORITY_BASE_IDX                                                                    0
+#define regSPI_ARB_CYCLES_0                                                                             0x1f61
+#define regSPI_ARB_CYCLES_0_BASE_IDX                                                                    0
+#define regSPI_ARB_CYCLES_1                                                                             0x1f62
+#define regSPI_ARB_CYCLES_1_BASE_IDX                                                                    0
+#define regSPI_WCL_PIPE_PERCENT_GFX                                                                     0x1f67
+#define regSPI_WCL_PIPE_PERCENT_GFX_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_HP3D                                                                    0x1f68
+#define regSPI_WCL_PIPE_PERCENT_HP3D_BASE_IDX                                                           0
+#define regSPI_WCL_PIPE_PERCENT_CS0                                                                     0x1f69
+#define regSPI_WCL_PIPE_PERCENT_CS0_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS1                                                                     0x1f6a
+#define regSPI_WCL_PIPE_PERCENT_CS1_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS2                                                                     0x1f6b
+#define regSPI_WCL_PIPE_PERCENT_CS2_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS3                                                                     0x1f6c
+#define regSPI_WCL_PIPE_PERCENT_CS3_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS4                                                                     0x1f6d
+#define regSPI_WCL_PIPE_PERCENT_CS4_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS5                                                                     0x1f6e
+#define regSPI_WCL_PIPE_PERCENT_CS5_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS6                                                                     0x1f6f
+#define regSPI_WCL_PIPE_PERCENT_CS6_BASE_IDX                                                            0
+#define regSPI_WCL_PIPE_PERCENT_CS7                                                                     0x1f70
+#define regSPI_WCL_PIPE_PERCENT_CS7_BASE_IDX                                                            0
+#define regSPI_USER_ACCUM_VMID_CNTL                                                                     0x1f71
+#define regSPI_USER_ACCUM_VMID_CNTL_BASE_IDX                                                            0
+#define regSPI_GDBG_PER_VMID_CNTL                                                                       0x1f72
+#define regSPI_GDBG_PER_VMID_CNTL_BASE_IDX                                                              0
+#define regSPI_COMPUTE_QUEUE_RESET                                                                      0x1f73
+#define regSPI_COMPUTE_QUEUE_RESET_BASE_IDX                                                             0
+#define regSPI_COMPUTE_WF_CTX_SAVE                                                                      0x1f74
+#define regSPI_COMPUTE_WF_CTX_SAVE_BASE_IDX                                                             0
+
+
+// addressBlock: gc_cpphqddec
+// base address: 0xc800
+#define regCP_HPD_UTCL1_CNTL                                                                            0x1fa3
+#define regCP_HPD_UTCL1_CNTL_BASE_IDX                                                                   0
+#define regCP_HPD_UTCL1_ERROR                                                                           0x1fa7
+#define regCP_HPD_UTCL1_ERROR_BASE_IDX                                                                  0
+#define regCP_HPD_UTCL1_ERROR_ADDR                                                                      0x1fa8
+#define regCP_HPD_UTCL1_ERROR_ADDR_BASE_IDX                                                             0
+#define regCP_MQD_BASE_ADDR                                                                             0x1fa9
+#define regCP_MQD_BASE_ADDR_BASE_IDX                                                                    0
+#define regCP_MQD_BASE_ADDR_HI                                                                          0x1faa
+#define regCP_MQD_BASE_ADDR_HI_BASE_IDX                                                                 0
+#define regCP_HQD_ACTIVE                                                                                0x1fab
+#define regCP_HQD_ACTIVE_BASE_IDX                                                                       0
+#define regCP_HQD_VMID                                                                                  0x1fac
+#define regCP_HQD_VMID_BASE_IDX                                                                         0
+#define regCP_HQD_PERSISTENT_STATE                                                                      0x1fad
+#define regCP_HQD_PERSISTENT_STATE_BASE_IDX                                                             0
+#define regCP_HQD_PIPE_PRIORITY                                                                         0x1fae
+#define regCP_HQD_PIPE_PRIORITY_BASE_IDX                                                                0
+#define regCP_HQD_QUEUE_PRIORITY                                                                        0x1faf
+#define regCP_HQD_QUEUE_PRIORITY_BASE_IDX                                                               0
+#define regCP_HQD_QUANTUM                                                                               0x1fb0
+#define regCP_HQD_QUANTUM_BASE_IDX                                                                      0
+#define regCP_HQD_PQ_BASE                                                                               0x1fb1
+#define regCP_HQD_PQ_BASE_BASE_IDX                                                                      0
+#define regCP_HQD_PQ_BASE_HI                                                                            0x1fb2
+#define regCP_HQD_PQ_BASE_HI_BASE_IDX                                                                   0
+#define regCP_HQD_PQ_RPTR                                                                               0x1fb3
+#define regCP_HQD_PQ_RPTR_BASE_IDX                                                                      0
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR                                                                   0x1fb4
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR_BASE_IDX                                                          0
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR_HI                                                                0x1fb5
+#define regCP_HQD_PQ_RPTR_REPORT_ADDR_HI_BASE_IDX                                                       0
+#define regCP_HQD_PQ_WPTR_POLL_ADDR                                                                     0x1fb6
+#define regCP_HQD_PQ_WPTR_POLL_ADDR_BASE_IDX                                                            0
+#define regCP_HQD_PQ_WPTR_POLL_ADDR_HI                                                                  0x1fb7
+#define regCP_HQD_PQ_WPTR_POLL_ADDR_HI_BASE_IDX                                                         0
+#define regCP_HQD_PQ_DOORBELL_CONTROL                                                                   0x1fb8
+#define regCP_HQD_PQ_DOORBELL_CONTROL_BASE_IDX                                                          0
+#define regCP_HQD_PQ_CONTROL                                                                            0x1fba
+#define regCP_HQD_PQ_CONTROL_BASE_IDX                                                                   0
+#define regCP_HQD_IB_BASE_ADDR                                                                          0x1fbb
+#define regCP_HQD_IB_BASE_ADDR_BASE_IDX                                                                 0
+#define regCP_HQD_IB_BASE_ADDR_HI                                                                       0x1fbc
+#define regCP_HQD_IB_BASE_ADDR_HI_BASE_IDX                                                              0
+#define regCP_HQD_IB_RPTR                                                                               0x1fbd
+#define regCP_HQD_IB_RPTR_BASE_IDX                                                                      0
+#define regCP_HQD_IB_CONTROL                                                                            0x1fbe
+#define regCP_HQD_IB_CONTROL_BASE_IDX                                                                   0
+#define regCP_HQD_IQ_TIMER                                                                              0x1fbf
+#define regCP_HQD_IQ_TIMER_BASE_IDX                                                                     0
+#define regCP_HQD_IQ_RPTR                                                                               0x1fc0
+#define regCP_HQD_IQ_RPTR_BASE_IDX                                                                      0
+#define regCP_HQD_DEQUEUE_REQUEST                                                                       0x1fc1
+#define regCP_HQD_DEQUEUE_REQUEST_BASE_IDX                                                              0
+#define regCP_HQD_DMA_OFFLOAD                                                                           0x1fc2
+#define regCP_HQD_DMA_OFFLOAD_BASE_IDX                                                                  0
+#define regCP_HQD_OFFLOAD                                                                               0x1fc2
+#define regCP_HQD_OFFLOAD_BASE_IDX                                                                      0
+#define regCP_HQD_SEMA_CMD                                                                              0x1fc3
+#define regCP_HQD_SEMA_CMD_BASE_IDX                                                                     0
+#define regCP_HQD_MSG_TYPE                                                                              0x1fc4
+#define regCP_HQD_MSG_TYPE_BASE_IDX                                                                     0
+#define regCP_HQD_ATOMIC0_PREOP_LO                                                                      0x1fc5
+#define regCP_HQD_ATOMIC0_PREOP_LO_BASE_IDX                                                             0
+#define regCP_HQD_ATOMIC0_PREOP_HI                                                                      0x1fc6
+#define regCP_HQD_ATOMIC0_PREOP_HI_BASE_IDX                                                             0
+#define regCP_HQD_ATOMIC1_PREOP_LO                                                                      0x1fc7
+#define regCP_HQD_ATOMIC1_PREOP_LO_BASE_IDX                                                             0
+#define regCP_HQD_ATOMIC1_PREOP_HI                                                                      0x1fc8
+#define regCP_HQD_ATOMIC1_PREOP_HI_BASE_IDX                                                             0
+#define regCP_HQD_HQ_SCHEDULER0                                                                         0x1fc9
+#define regCP_HQD_HQ_SCHEDULER0_BASE_IDX                                                                0
+#define regCP_HQD_HQ_STATUS0                                                                            0x1fc9
+#define regCP_HQD_HQ_STATUS0_BASE_IDX                                                                   0
+#define regCP_HQD_HQ_CONTROL0                                                                           0x1fca
+#define regCP_HQD_HQ_CONTROL0_BASE_IDX                                                                  0
+#define regCP_HQD_HQ_SCHEDULER1                                                                         0x1fca
+#define regCP_HQD_HQ_SCHEDULER1_BASE_IDX                                                                0
+#define regCP_MQD_CONTROL                                                                               0x1fcb
+#define regCP_MQD_CONTROL_BASE_IDX                                                                      0
+#define regCP_HQD_HQ_STATUS1                                                                            0x1fcc
+#define regCP_HQD_HQ_STATUS1_BASE_IDX                                                                   0
+#define regCP_HQD_HQ_CONTROL1                                                                           0x1fcd
+#define regCP_HQD_HQ_CONTROL1_BASE_IDX                                                                  0
+#define regCP_HQD_EOP_BASE_ADDR                                                                         0x1fce
+#define regCP_HQD_EOP_BASE_ADDR_BASE_IDX                                                                0
+#define regCP_HQD_EOP_BASE_ADDR_HI                                                                      0x1fcf
+#define regCP_HQD_EOP_BASE_ADDR_HI_BASE_IDX                                                             0
+#define regCP_HQD_EOP_CONTROL                                                                           0x1fd0
+#define regCP_HQD_EOP_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_EOP_RPTR                                                                              0x1fd1
+#define regCP_HQD_EOP_RPTR_BASE_IDX                                                                     0
+#define regCP_HQD_EOP_WPTR                                                                              0x1fd2
+#define regCP_HQD_EOP_WPTR_BASE_IDX                                                                     0
+#define regCP_HQD_EOP_EVENTS                                                                            0x1fd3
+#define regCP_HQD_EOP_EVENTS_BASE_IDX                                                                   0
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_LO                                                                 0x1fd4
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_LO_BASE_IDX                                                        0
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_HI                                                                 0x1fd5
+#define regCP_HQD_CTX_SAVE_BASE_ADDR_HI_BASE_IDX                                                        0
+#define regCP_HQD_CTX_SAVE_CONTROL                                                                      0x1fd6
+#define regCP_HQD_CTX_SAVE_CONTROL_BASE_IDX                                                             0
+#define regCP_HQD_CNTL_STACK_OFFSET                                                                     0x1fd7
+#define regCP_HQD_CNTL_STACK_OFFSET_BASE_IDX                                                            0
+#define regCP_HQD_CNTL_STACK_SIZE                                                                       0x1fd8
+#define regCP_HQD_CNTL_STACK_SIZE_BASE_IDX                                                              0
+#define regCP_HQD_WG_STATE_OFFSET                                                                       0x1fd9
+#define regCP_HQD_WG_STATE_OFFSET_BASE_IDX                                                              0
+#define regCP_HQD_CTX_SAVE_SIZE                                                                         0x1fda
+#define regCP_HQD_CTX_SAVE_SIZE_BASE_IDX                                                                0
+#define regCP_HQD_GDS_RESOURCE_STATE                                                                    0x1fdb
+#define regCP_HQD_GDS_RESOURCE_STATE_BASE_IDX                                                           0
+#define regCP_HQD_ERROR                                                                                 0x1fdc
+#define regCP_HQD_ERROR_BASE_IDX                                                                        0
+#define regCP_HQD_EOP_WPTR_MEM                                                                          0x1fdd
+#define regCP_HQD_EOP_WPTR_MEM_BASE_IDX                                                                 0
+#define regCP_HQD_AQL_CONTROL                                                                           0x1fde
+#define regCP_HQD_AQL_CONTROL_BASE_IDX                                                                  0
+#define regCP_HQD_PQ_WPTR_LO                                                                            0x1fdf
+#define regCP_HQD_PQ_WPTR_LO_BASE_IDX                                                                   0
+#define regCP_HQD_PQ_WPTR_HI                                                                            0x1fe0
+#define regCP_HQD_PQ_WPTR_HI_BASE_IDX                                                                   0
+#define regCP_HQD_SUSPEND_CNTL_STACK_OFFSET                                                             0x1fe1
+#define regCP_HQD_SUSPEND_CNTL_STACK_OFFSET_BASE_IDX                                                    0
+#define regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT                                                             0x1fe2
+#define regCP_HQD_SUSPEND_CNTL_STACK_DW_CNT_BASE_IDX                                                    0
+#define regCP_HQD_SUSPEND_WG_STATE_OFFSET                                                               0x1fe3
+#define regCP_HQD_SUSPEND_WG_STATE_OFFSET_BASE_IDX                                                      0
+#define regCP_HQD_DDID_RPTR                                                                             0x1fe4
+#define regCP_HQD_DDID_RPTR_BASE_IDX                                                                    0
+#define regCP_HQD_DDID_WPTR                                                                             0x1fe5
+#define regCP_HQD_DDID_WPTR_BASE_IDX                                                                    0
+#define regCP_HQD_DDID_INFLIGHT_COUNT                                                                   0x1fe6
+#define regCP_HQD_DDID_INFLIGHT_COUNT_BASE_IDX                                                          0
+#define regCP_HQD_DDID_DELTA_RPT_COUNT                                                                  0x1fe7
+#define regCP_HQD_DDID_DELTA_RPT_COUNT_BASE_IDX                                                         0
+#define regCP_HQD_DEQUEUE_STATUS                                                                        0x1fe8
+#define regCP_HQD_DEQUEUE_STATUS_BASE_IDX                                                               0
+
+
+// addressBlock: gc_tcpdec
+// base address: 0xca80
+#define regTCP_WATCH0_ADDR_H                                                                            0x2048
+#define regTCP_WATCH0_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH0_ADDR_L                                                                            0x2049
+#define regTCP_WATCH0_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH0_CNTL                                                                              0x204a
+#define regTCP_WATCH0_CNTL_BASE_IDX                                                                     0
+#define regTCP_WATCH1_ADDR_H                                                                            0x204b
+#define regTCP_WATCH1_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH1_ADDR_L                                                                            0x204c
+#define regTCP_WATCH1_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH1_CNTL                                                                              0x204d
+#define regTCP_WATCH1_CNTL_BASE_IDX                                                                     0
+#define regTCP_WATCH2_ADDR_H                                                                            0x204e
+#define regTCP_WATCH2_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH2_ADDR_L                                                                            0x204f
+#define regTCP_WATCH2_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH2_CNTL                                                                              0x2050
+#define regTCP_WATCH2_CNTL_BASE_IDX                                                                     0
+#define regTCP_WATCH3_ADDR_H                                                                            0x2051
+#define regTCP_WATCH3_ADDR_H_BASE_IDX                                                                   0
+#define regTCP_WATCH3_ADDR_L                                                                            0x2052
+#define regTCP_WATCH3_ADDR_L_BASE_IDX                                                                   0
+#define regTCP_WATCH3_CNTL                                                                              0x2053
+#define regTCP_WATCH3_CNTL_BASE_IDX                                                                     0
+
+
+// addressBlock: gc_gdspdec
+// base address: 0xcc00
+#define regGDS_VMID0_BASE                                                                               0x20a0
+#define regGDS_VMID0_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID0_SIZE                                                                               0x20a1
+#define regGDS_VMID0_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID1_BASE                                                                               0x20a2
+#define regGDS_VMID1_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID1_SIZE                                                                               0x20a3
+#define regGDS_VMID1_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID2_BASE                                                                               0x20a4
+#define regGDS_VMID2_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID2_SIZE                                                                               0x20a5
+#define regGDS_VMID2_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID3_BASE                                                                               0x20a6
+#define regGDS_VMID3_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID3_SIZE                                                                               0x20a7
+#define regGDS_VMID3_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID4_BASE                                                                               0x20a8
+#define regGDS_VMID4_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID4_SIZE                                                                               0x20a9
+#define regGDS_VMID4_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID5_BASE                                                                               0x20aa
+#define regGDS_VMID5_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID5_SIZE                                                                               0x20ab
+#define regGDS_VMID5_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID6_BASE                                                                               0x20ac
+#define regGDS_VMID6_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID6_SIZE                                                                               0x20ad
+#define regGDS_VMID6_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID7_BASE                                                                               0x20ae
+#define regGDS_VMID7_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID7_SIZE                                                                               0x20af
+#define regGDS_VMID7_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID8_BASE                                                                               0x20b0
+#define regGDS_VMID8_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID8_SIZE                                                                               0x20b1
+#define regGDS_VMID8_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID9_BASE                                                                               0x20b2
+#define regGDS_VMID9_BASE_BASE_IDX                                                                      0
+#define regGDS_VMID9_SIZE                                                                               0x20b3
+#define regGDS_VMID9_SIZE_BASE_IDX                                                                      0
+#define regGDS_VMID10_BASE                                                                              0x20b4
+#define regGDS_VMID10_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID10_SIZE                                                                              0x20b5
+#define regGDS_VMID10_SIZE_BASE_IDX                                                                     0
+#define regGDS_VMID11_BASE                                                                              0x20b6
+#define regGDS_VMID11_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID11_SIZE                                                                              0x20b7
+#define regGDS_VMID11_SIZE_BASE_IDX                                                                     0
+#define regGDS_VMID12_BASE                                                                              0x20b8
+#define regGDS_VMID12_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID12_SIZE                                                                              0x20b9
+#define regGDS_VMID12_SIZE_BASE_IDX                                                                     0
+#define regGDS_VMID13_BASE                                                                              0x20ba
+#define regGDS_VMID13_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID13_SIZE                                                                              0x20bb
+#define regGDS_VMID13_SIZE_BASE_IDX                                                                     0
+#define regGDS_VMID14_BASE                                                                              0x20bc
+#define regGDS_VMID14_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID14_SIZE                                                                              0x20bd
+#define regGDS_VMID14_SIZE_BASE_IDX                                                                     0
+#define regGDS_VMID15_BASE                                                                              0x20be
+#define regGDS_VMID15_BASE_BASE_IDX                                                                     0
+#define regGDS_VMID15_SIZE                                                                              0x20bf
+#define regGDS_VMID15_SIZE_BASE_IDX                                                                     0
+#define regGDS_GWS_VMID0                                                                                0x20c0
+#define regGDS_GWS_VMID0_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID1                                                                                0x20c1
+#define regGDS_GWS_VMID1_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID2                                                                                0x20c2
+#define regGDS_GWS_VMID2_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID3                                                                                0x20c3
+#define regGDS_GWS_VMID3_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID4                                                                                0x20c4
+#define regGDS_GWS_VMID4_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID5                                                                                0x20c5
+#define regGDS_GWS_VMID5_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID6                                                                                0x20c6
+#define regGDS_GWS_VMID6_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID7                                                                                0x20c7
+#define regGDS_GWS_VMID7_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID8                                                                                0x20c8
+#define regGDS_GWS_VMID8_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID9                                                                                0x20c9
+#define regGDS_GWS_VMID9_BASE_IDX                                                                       0
+#define regGDS_GWS_VMID10                                                                               0x20ca
+#define regGDS_GWS_VMID10_BASE_IDX                                                                      0
+#define regGDS_GWS_VMID11                                                                               0x20cb
+#define regGDS_GWS_VMID11_BASE_IDX                                                                      0
+#define regGDS_GWS_VMID12                                                                               0x20cc
+#define regGDS_GWS_VMID12_BASE_IDX                                                                      0
+#define regGDS_GWS_VMID13                                                                               0x20cd
+#define regGDS_GWS_VMID13_BASE_IDX                                                                      0
+#define regGDS_GWS_VMID14                                                                               0x20ce
+#define regGDS_GWS_VMID14_BASE_IDX                                                                      0
+#define regGDS_GWS_VMID15                                                                               0x20cf
+#define regGDS_GWS_VMID15_BASE_IDX                                                                      0
+#define regGDS_OA_VMID0                                                                                 0x20d0
+#define regGDS_OA_VMID0_BASE_IDX                                                                        0
+#define regGDS_OA_VMID1                                                                                 0x20d1
+#define regGDS_OA_VMID1_BASE_IDX                                                                        0
+#define regGDS_OA_VMID2                                                                                 0x20d2
+#define regGDS_OA_VMID2_BASE_IDX                                                                        0
+#define regGDS_OA_VMID3                                                                                 0x20d3
+#define regGDS_OA_VMID3_BASE_IDX                                                                        0
+#define regGDS_OA_VMID4                                                                                 0x20d4
+#define regGDS_OA_VMID4_BASE_IDX                                                                        0
+#define regGDS_OA_VMID5                                                                                 0x20d5
+#define regGDS_OA_VMID5_BASE_IDX                                                                        0
+#define regGDS_OA_VMID6                                                                                 0x20d6
+#define regGDS_OA_VMID6_BASE_IDX                                                                        0
+#define regGDS_OA_VMID7                                                                                 0x20d7
+#define regGDS_OA_VMID7_BASE_IDX                                                                        0
+#define regGDS_OA_VMID8                                                                                 0x20d8
+#define regGDS_OA_VMID8_BASE_IDX                                                                        0
+#define regGDS_OA_VMID9                                                                                 0x20d9
+#define regGDS_OA_VMID9_BASE_IDX                                                                        0
+#define regGDS_OA_VMID10                                                                                0x20da
+#define regGDS_OA_VMID10_BASE_IDX                                                                       0
+#define regGDS_OA_VMID11                                                                                0x20db
+#define regGDS_OA_VMID11_BASE_IDX                                                                       0
+#define regGDS_OA_VMID12                                                                                0x20dc
+#define regGDS_OA_VMID12_BASE_IDX                                                                       0
+#define regGDS_OA_VMID13                                                                                0x20dd
+#define regGDS_OA_VMID13_BASE_IDX                                                                       0
+#define regGDS_OA_VMID14                                                                                0x20de
+#define regGDS_OA_VMID14_BASE_IDX                                                                       0
+#define regGDS_OA_VMID15                                                                                0x20df
+#define regGDS_OA_VMID15_BASE_IDX                                                                       0
+#define regGDS_GWS_RESET0                                                                               0x20e4
+#define regGDS_GWS_RESET0_BASE_IDX                                                                      0
+#define regGDS_GWS_RESET1                                                                               0x20e5
+#define regGDS_GWS_RESET1_BASE_IDX                                                                      0
+#define regGDS_GWS_RESOURCE_RESET                                                                       0x20e6
+#define regGDS_GWS_RESOURCE_RESET_BASE_IDX                                                              0
+#define regGDS_COMPUTE_MAX_WAVE_ID                                                                      0x20e8
+#define regGDS_COMPUTE_MAX_WAVE_ID_BASE_IDX                                                             0
+#define regGDS_OA_RESET_MASK                                                                            0x20e9
+#define regGDS_OA_RESET_MASK_BASE_IDX                                                                   0
+#define regGDS_OA_RESET                                                                                 0x20ea
+#define regGDS_OA_RESET_BASE_IDX                                                                        0
+#define regGDS_CS_CTXSW_STATUS                                                                          0x20ed
+#define regGDS_CS_CTXSW_STATUS_BASE_IDX                                                                 0
+#define regGDS_CS_CTXSW_CNT0                                                                            0x20ee
+#define regGDS_CS_CTXSW_CNT0_BASE_IDX                                                                   0
+#define regGDS_CS_CTXSW_CNT1                                                                            0x20ef
+#define regGDS_CS_CTXSW_CNT1_BASE_IDX                                                                   0
+#define regGDS_CS_CTXSW_CNT2                                                                            0x20f0
+#define regGDS_CS_CTXSW_CNT2_BASE_IDX                                                                   0
+#define regGDS_CS_CTXSW_CNT3                                                                            0x20f1
+#define regGDS_CS_CTXSW_CNT3_BASE_IDX                                                                   0
+#define regGDS_GFX_CTXSW_STATUS                                                                         0x20f2
+#define regGDS_GFX_CTXSW_STATUS_BASE_IDX                                                                0
+#define regGDS_PS_CTXSW_CNT0                                                                            0x20f7
+#define regGDS_PS_CTXSW_CNT0_BASE_IDX                                                                   0
+#define regGDS_PS_CTXSW_CNT1                                                                            0x20f8
+#define regGDS_PS_CTXSW_CNT1_BASE_IDX                                                                   0
+#define regGDS_PS_CTXSW_CNT2                                                                            0x20f9
+#define regGDS_PS_CTXSW_CNT2_BASE_IDX                                                                   0
+#define regGDS_PS_CTXSW_CNT3                                                                            0x20fa
+#define regGDS_PS_CTXSW_CNT3_BASE_IDX                                                                   0
+#define regGDS_PS_CTXSW_IDX                                                                             0x20fb
+#define regGDS_PS_CTXSW_IDX_BASE_IDX                                                                    0
+#define regGDS_GS_CTXSW_CNT0                                                                            0x2117
+#define regGDS_GS_CTXSW_CNT0_BASE_IDX                                                                   0
+#define regGDS_GS_CTXSW_CNT1                                                                            0x2118
+#define regGDS_GS_CTXSW_CNT1_BASE_IDX                                                                   0
+#define regGDS_GS_CTXSW_CNT2                                                                            0x2119
+#define regGDS_GS_CTXSW_CNT2_BASE_IDX                                                                   0
+#define regGDS_GS_CTXSW_CNT3                                                                            0x211a
+#define regGDS_GS_CTXSW_CNT3_BASE_IDX                                                                   0
+#define regGDS_MEMORY_CLEAN                                                                             0x211f
+#define regGDS_MEMORY_CLEAN_BASE_IDX                                                                    0
+
+
+// addressBlock: gc_rasdec
+// base address: 0xce00
+#define regRAS_SIGNATURE_CONTROL                                                                        0x2120
+#define regRAS_SIGNATURE_CONTROL_BASE_IDX                                                               0
+#define regRAS_SIGNATURE_MASK                                                                           0x2121
+#define regRAS_SIGNATURE_MASK_BASE_IDX                                                                  0
+#define regRAS_SX_SIGNATURE0                                                                            0x2122
+#define regRAS_SX_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_SX_SIGNATURE1                                                                            0x2123
+#define regRAS_SX_SIGNATURE1_BASE_IDX                                                                   0
+#define regRAS_SX_SIGNATURE2                                                                            0x2124
+#define regRAS_SX_SIGNATURE2_BASE_IDX                                                                   0
+#define regRAS_SX_SIGNATURE3                                                                            0x2125
+#define regRAS_SX_SIGNATURE3_BASE_IDX                                                                   0
+#define regRAS_DB_SIGNATURE0                                                                            0x212b
+#define regRAS_DB_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_PA_SIGNATURE0                                                                            0x212c
+#define regRAS_PA_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE0                                                                            0x212f
+#define regRAS_SC_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE1                                                                            0x2130
+#define regRAS_SC_SIGNATURE1_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE2                                                                            0x2131
+#define regRAS_SC_SIGNATURE2_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE3                                                                            0x2132
+#define regRAS_SC_SIGNATURE3_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE4                                                                            0x2133
+#define regRAS_SC_SIGNATURE4_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE5                                                                            0x2134
+#define regRAS_SC_SIGNATURE5_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE6                                                                            0x2135
+#define regRAS_SC_SIGNATURE6_BASE_IDX                                                                   0
+#define regRAS_SC_SIGNATURE7                                                                            0x2136
+#define regRAS_SC_SIGNATURE7_BASE_IDX                                                                   0
+#define regRAS_SPI_SIGNATURE0                                                                           0x2139
+#define regRAS_SPI_SIGNATURE0_BASE_IDX                                                                  0
+#define regRAS_SPI_SIGNATURE1                                                                           0x213a
+#define regRAS_SPI_SIGNATURE1_BASE_IDX                                                                  0
+#define regRAS_CB_SIGNATURE0                                                                            0x213d
+#define regRAS_CB_SIGNATURE0_BASE_IDX                                                                   0
+#define regRAS_BCI_SIGNATURE0                                                                           0x213e
+#define regRAS_BCI_SIGNATURE0_BASE_IDX                                                                  0
+#define regRAS_BCI_SIGNATURE1                                                                           0x213f
+#define regRAS_BCI_SIGNATURE1_BASE_IDX                                                                  0
+
+
+// addressBlock: gc_gusdec
+// base address: 0x33000
+#define regGUS_IO_RD_COMBINE_FLUSH                                                                      0x2c00
+#define regGUS_IO_RD_COMBINE_FLUSH_BASE_IDX                                                             1
+#define regGUS_IO_WR_COMBINE_FLUSH                                                                      0x2c01
+#define regGUS_IO_WR_COMBINE_FLUSH_BASE_IDX                                                             1
+#define regGUS_IO_RD_PRI_AGE_RATE                                                                       0x2c02
+#define regGUS_IO_RD_PRI_AGE_RATE_BASE_IDX                                                              1
+#define regGUS_IO_WR_PRI_AGE_RATE                                                                       0x2c03
+#define regGUS_IO_WR_PRI_AGE_RATE_BASE_IDX                                                              1
+#define regGUS_IO_RD_PRI_AGE_COEFF                                                                      0x2c04
+#define regGUS_IO_RD_PRI_AGE_COEFF_BASE_IDX                                                             1
+#define regGUS_IO_WR_PRI_AGE_COEFF                                                                      0x2c05
+#define regGUS_IO_WR_PRI_AGE_COEFF_BASE_IDX                                                             1
+#define regGUS_IO_RD_PRI_QUEUING                                                                        0x2c06
+#define regGUS_IO_RD_PRI_QUEUING_BASE_IDX                                                               1
+#define regGUS_IO_WR_PRI_QUEUING                                                                        0x2c07
+#define regGUS_IO_WR_PRI_QUEUING_BASE_IDX                                                               1
+#define regGUS_IO_RD_PRI_FIXED                                                                          0x2c08
+#define regGUS_IO_RD_PRI_FIXED_BASE_IDX                                                                 1
+#define regGUS_IO_WR_PRI_FIXED                                                                          0x2c09
+#define regGUS_IO_WR_PRI_FIXED_BASE_IDX                                                                 1
+#define regGUS_IO_RD_PRI_URGENCY_COEFF                                                                  0x2c0a
+#define regGUS_IO_RD_PRI_URGENCY_COEFF_BASE_IDX                                                         1
+#define regGUS_IO_WR_PRI_URGENCY_COEFF                                                                  0x2c0b
+#define regGUS_IO_WR_PRI_URGENCY_COEFF_BASE_IDX                                                         1
+#define regGUS_IO_RD_PRI_URGENCY_MODE                                                                   0x2c0c
+#define regGUS_IO_RD_PRI_URGENCY_MODE_BASE_IDX                                                          1
+#define regGUS_IO_WR_PRI_URGENCY_MODE                                                                   0x2c0d
+#define regGUS_IO_WR_PRI_URGENCY_MODE_BASE_IDX                                                          1
+#define regGUS_IO_RD_PRI_QUANT_PRI1                                                                     0x2c0e
+#define regGUS_IO_RD_PRI_QUANT_PRI1_BASE_IDX                                                            1
+#define regGUS_IO_RD_PRI_QUANT_PRI2                                                                     0x2c0f
+#define regGUS_IO_RD_PRI_QUANT_PRI2_BASE_IDX                                                            1
+#define regGUS_IO_RD_PRI_QUANT_PRI3                                                                     0x2c10
+#define regGUS_IO_RD_PRI_QUANT_PRI3_BASE_IDX                                                            1
+#define regGUS_IO_RD_PRI_QUANT_PRI4                                                                     0x2c11
+#define regGUS_IO_RD_PRI_QUANT_PRI4_BASE_IDX                                                            1
+#define regGUS_IO_WR_PRI_QUANT_PRI1                                                                     0x2c12
+#define regGUS_IO_WR_PRI_QUANT_PRI1_BASE_IDX                                                            1
+#define regGUS_IO_WR_PRI_QUANT_PRI2                                                                     0x2c13
+#define regGUS_IO_WR_PRI_QUANT_PRI2_BASE_IDX                                                            1
+#define regGUS_IO_WR_PRI_QUANT_PRI3                                                                     0x2c14
+#define regGUS_IO_WR_PRI_QUANT_PRI3_BASE_IDX                                                            1
+#define regGUS_IO_WR_PRI_QUANT_PRI4                                                                     0x2c15
+#define regGUS_IO_WR_PRI_QUANT_PRI4_BASE_IDX                                                            1
+#define regGUS_IO_RD_PRI_QUANT1_PRI1                                                                    0x2c16
+#define regGUS_IO_RD_PRI_QUANT1_PRI1_BASE_IDX                                                           1
+#define regGUS_IO_RD_PRI_QUANT1_PRI2                                                                    0x2c17
+#define regGUS_IO_RD_PRI_QUANT1_PRI2_BASE_IDX                                                           1
+#define regGUS_IO_RD_PRI_QUANT1_PRI3                                                                    0x2c18
+#define regGUS_IO_RD_PRI_QUANT1_PRI3_BASE_IDX                                                           1
+#define regGUS_IO_RD_PRI_QUANT1_PRI4                                                                    0x2c19
+#define regGUS_IO_RD_PRI_QUANT1_PRI4_BASE_IDX                                                           1
+#define regGUS_IO_WR_PRI_QUANT1_PRI1                                                                    0x2c1a
+#define regGUS_IO_WR_PRI_QUANT1_PRI1_BASE_IDX                                                           1
+#define regGUS_IO_WR_PRI_QUANT1_PRI2                                                                    0x2c1b
+#define regGUS_IO_WR_PRI_QUANT1_PRI2_BASE_IDX                                                           1
+#define regGUS_IO_WR_PRI_QUANT1_PRI3                                                                    0x2c1c
+#define regGUS_IO_WR_PRI_QUANT1_PRI3_BASE_IDX                                                           1
+#define regGUS_IO_WR_PRI_QUANT1_PRI4                                                                    0x2c1d
+#define regGUS_IO_WR_PRI_QUANT1_PRI4_BASE_IDX                                                           1
+#define regGUS_DRAM_COMBINE_FLUSH                                                                       0x2c1e
+#define regGUS_DRAM_COMBINE_FLUSH_BASE_IDX                                                              1
+#define regGUS_DRAM_COMBINE_RD_WR_EN                                                                    0x2c1f
+#define regGUS_DRAM_COMBINE_RD_WR_EN_BASE_IDX                                                           1
+#define regGUS_DRAM_PRI_AGE_RATE                                                                        0x2c20
+#define regGUS_DRAM_PRI_AGE_RATE_BASE_IDX                                                               1
+#define regGUS_DRAM_PRI_AGE_COEFF                                                                       0x2c21
+#define regGUS_DRAM_PRI_AGE_COEFF_BASE_IDX                                                              1
+#define regGUS_DRAM_PRI_QUEUING                                                                         0x2c22
+#define regGUS_DRAM_PRI_QUEUING_BASE_IDX                                                                1
+#define regGUS_DRAM_PRI_FIXED                                                                           0x2c23
+#define regGUS_DRAM_PRI_FIXED_BASE_IDX                                                                  1
+#define regGUS_DRAM_PRI_URGENCY_COEFF                                                                   0x2c24
+#define regGUS_DRAM_PRI_URGENCY_COEFF_BASE_IDX                                                          1
+#define regGUS_DRAM_PRI_URGENCY_MODE                                                                    0x2c25
+#define regGUS_DRAM_PRI_URGENCY_MODE_BASE_IDX                                                           1
+#define regGUS_DRAM_PRI_QUANT_PRI1                                                                      0x2c26
+#define regGUS_DRAM_PRI_QUANT_PRI1_BASE_IDX                                                             1
+#define regGUS_DRAM_PRI_QUANT_PRI2                                                                      0x2c27
+#define regGUS_DRAM_PRI_QUANT_PRI2_BASE_IDX                                                             1
+#define regGUS_DRAM_PRI_QUANT_PRI3                                                                      0x2c28
+#define regGUS_DRAM_PRI_QUANT_PRI3_BASE_IDX                                                             1
+#define regGUS_DRAM_PRI_QUANT_PRI4                                                                      0x2c29
+#define regGUS_DRAM_PRI_QUANT_PRI4_BASE_IDX                                                             1
+#define regGUS_DRAM_PRI_QUANT_PRI5                                                                      0x2c2a
+#define regGUS_DRAM_PRI_QUANT_PRI5_BASE_IDX                                                             1
+#define regGUS_DRAM_PRI_QUANT1_PRI1                                                                     0x2c2b
+#define regGUS_DRAM_PRI_QUANT1_PRI1_BASE_IDX                                                            1
+#define regGUS_DRAM_PRI_QUANT1_PRI2                                                                     0x2c2c
+#define regGUS_DRAM_PRI_QUANT1_PRI2_BASE_IDX                                                            1
+#define regGUS_DRAM_PRI_QUANT1_PRI3                                                                     0x2c2d
+#define regGUS_DRAM_PRI_QUANT1_PRI3_BASE_IDX                                                            1
+#define regGUS_DRAM_PRI_QUANT1_PRI4                                                                     0x2c2e
+#define regGUS_DRAM_PRI_QUANT1_PRI4_BASE_IDX                                                            1
+#define regGUS_DRAM_PRI_QUANT1_PRI5                                                                     0x2c2f
+#define regGUS_DRAM_PRI_QUANT1_PRI5_BASE_IDX                                                            1
+#define regGUS_IO_GROUP_BURST                                                                           0x2c30
+#define regGUS_IO_GROUP_BURST_BASE_IDX                                                                  1
+#define regGUS_DRAM_GROUP_BURST                                                                         0x2c31
+#define regGUS_DRAM_GROUP_BURST_BASE_IDX                                                                1
+#define regGUS_SDP_ARB_FINAL                                                                            0x2c32
+#define regGUS_SDP_ARB_FINAL_BASE_IDX                                                                   1
+#define regGUS_SDP_QOS_VC_PRIORITY                                                                      0x2c33
+#define regGUS_SDP_QOS_VC_PRIORITY_BASE_IDX                                                             1
+#define regGUS_SDP_CREDITS                                                                              0x2c34
+#define regGUS_SDP_CREDITS_BASE_IDX                                                                     1
+#define regGUS_SDP_TAG_RESERVE0                                                                         0x2c35
+#define regGUS_SDP_TAG_RESERVE0_BASE_IDX                                                                1
+#define regGUS_SDP_TAG_RESERVE1                                                                         0x2c36
+#define regGUS_SDP_TAG_RESERVE1_BASE_IDX                                                                1
+#define regGUS_SDP_VCC_RESERVE0                                                                         0x2c37
+#define regGUS_SDP_VCC_RESERVE0_BASE_IDX                                                                1
+#define regGUS_SDP_VCC_RESERVE1                                                                         0x2c38
+#define regGUS_SDP_VCC_RESERVE1_BASE_IDX                                                                1
+#define regGUS_SDP_VCD_RESERVE0                                                                         0x2c39
+#define regGUS_SDP_VCD_RESERVE0_BASE_IDX                                                                1
+#define regGUS_SDP_VCD_RESERVE1                                                                         0x2c3a
+#define regGUS_SDP_VCD_RESERVE1_BASE_IDX                                                                1
+#define regGUS_SDP_REQ_CNTL                                                                             0x2c3b
+#define regGUS_SDP_REQ_CNTL_BASE_IDX                                                                    1
+#define regGUS_MISC                                                                                     0x2c3c
+#define regGUS_MISC_BASE_IDX                                                                            1
+#define regGUS_LATENCY_SAMPLING                                                                         0x2c3d
+#define regGUS_LATENCY_SAMPLING_BASE_IDX                                                                1
+#define regGUS_ERR_STATUS                                                                               0x2c3e
+#define regGUS_ERR_STATUS_BASE_IDX                                                                      1
+#define regGUS_MISC2                                                                                    0x2c3f
+#define regGUS_MISC2_BASE_IDX                                                                           1
+#define regGUS_SDP_BACKDOOR_CMDCREDITS0                                                                 0x2c40
+#define regGUS_SDP_BACKDOOR_CMDCREDITS0_BASE_IDX                                                        1
+#define regGUS_SDP_BACKDOOR_CMDCREDITS1                                                                 0x2c41
+#define regGUS_SDP_BACKDOOR_CMDCREDITS1_BASE_IDX                                                        1
+#define regGUS_SDP_BACKDOOR_DATACREDITS0                                                                0x2c42
+#define regGUS_SDP_BACKDOOR_DATACREDITS0_BASE_IDX                                                       1
+#define regGUS_SDP_BACKDOOR_DATACREDITS1                                                                0x2c43
+#define regGUS_SDP_BACKDOOR_DATACREDITS1_BASE_IDX                                                       1
+#define regGUS_SDP_BACKDOOR_MISCCREDITS                                                                 0x2c44
+#define regGUS_SDP_BACKDOOR_MISCCREDITS_BASE_IDX                                                        1
+#define regGUS_SDP_ENABLE                                                                               0x2c45
+#define regGUS_SDP_ENABLE_BASE_IDX                                                                      1
+#define regGUS_L1_CH0_CMD_IN                                                                            0x2c46
+#define regGUS_L1_CH0_CMD_IN_BASE_IDX                                                                   1
+#define regGUS_L1_CH0_CMD_OUT                                                                           0x2c47
+#define regGUS_L1_CH0_CMD_OUT_BASE_IDX                                                                  1
+#define regGUS_L1_CH0_DATA_IN                                                                           0x2c48
+#define regGUS_L1_CH0_DATA_IN_BASE_IDX                                                                  1
+#define regGUS_L1_CH0_DATA_OUT                                                                          0x2c49
+#define regGUS_L1_CH0_DATA_OUT_BASE_IDX                                                                 1
+#define regGUS_L1_CH0_DATA_U_IN                                                                         0x2c4a
+#define regGUS_L1_CH0_DATA_U_IN_BASE_IDX                                                                1
+#define regGUS_L1_CH0_DATA_U_OUT                                                                        0x2c4b
+#define regGUS_L1_CH0_DATA_U_OUT_BASE_IDX                                                               1
+#define regGUS_L1_CH1_CMD_IN                                                                            0x2c4c
+#define regGUS_L1_CH1_CMD_IN_BASE_IDX                                                                   1
+#define regGUS_L1_CH1_CMD_OUT                                                                           0x2c4d
+#define regGUS_L1_CH1_CMD_OUT_BASE_IDX                                                                  1
+#define regGUS_L1_CH1_DATA_IN                                                                           0x2c4e
+#define regGUS_L1_CH1_DATA_IN_BASE_IDX                                                                  1
+#define regGUS_L1_CH1_DATA_OUT                                                                          0x2c4f
+#define regGUS_L1_CH1_DATA_OUT_BASE_IDX                                                                 1
+#define regGUS_L1_CH1_DATA_U_IN                                                                         0x2c50
+#define regGUS_L1_CH1_DATA_U_IN_BASE_IDX                                                                1
+#define regGUS_L1_CH1_DATA_U_OUT                                                                        0x2c51
+#define regGUS_L1_CH1_DATA_U_OUT_BASE_IDX                                                               1
+#define regGUS_L1_SA0_CMD_IN                                                                            0x2c52
+#define regGUS_L1_SA0_CMD_IN_BASE_IDX                                                                   1
+#define regGUS_L1_SA0_CMD_OUT                                                                           0x2c53
+#define regGUS_L1_SA0_CMD_OUT_BASE_IDX                                                                  1
+#define regGUS_L1_SA0_DATA_IN                                                                           0x2c54
+#define regGUS_L1_SA0_DATA_IN_BASE_IDX                                                                  1
+#define regGUS_L1_SA0_DATA_OUT                                                                          0x2c55
+#define regGUS_L1_SA0_DATA_OUT_BASE_IDX                                                                 1
+#define regGUS_L1_SA0_DATA_U_IN                                                                         0x2c56
+#define regGUS_L1_SA0_DATA_U_IN_BASE_IDX                                                                1
+#define regGUS_L1_SA0_DATA_U_OUT                                                                        0x2c57
+#define regGUS_L1_SA0_DATA_U_OUT_BASE_IDX                                                               1
+#define regGUS_L1_SA1_CMD_IN                                                                            0x2c58
+#define regGUS_L1_SA1_CMD_IN_BASE_IDX                                                                   1
+#define regGUS_L1_SA1_CMD_OUT                                                                           0x2c59
+#define regGUS_L1_SA1_CMD_OUT_BASE_IDX                                                                  1
+#define regGUS_L1_SA1_DATA_IN                                                                           0x2c5a
+#define regGUS_L1_SA1_DATA_IN_BASE_IDX                                                                  1
+#define regGUS_L1_SA1_DATA_OUT                                                                          0x2c5b
+#define regGUS_L1_SA1_DATA_OUT_BASE_IDX                                                                 1
+#define regGUS_L1_SA1_DATA_U_IN                                                                         0x2c5c
+#define regGUS_L1_SA1_DATA_U_IN_BASE_IDX                                                                1
+#define regGUS_L1_SA1_DATA_U_OUT                                                                        0x2c5d
+#define regGUS_L1_SA1_DATA_U_OUT_BASE_IDX                                                               1
+#define regGUS_L1_SA2_CMD_IN                                                                            0x2c5e
+#define regGUS_L1_SA2_CMD_IN_BASE_IDX                                                                   1
+#define regGUS_L1_SA2_CMD_OUT                                                                           0x2c5f
+#define regGUS_L1_SA2_CMD_OUT_BASE_IDX                                                                  1
+#define regGUS_L1_SA2_DATA_IN                                                                           0x2c60
+#define regGUS_L1_SA2_DATA_IN_BASE_IDX                                                                  1
+#define regGUS_L1_SA2_DATA_OUT                                                                          0x2c61
+#define regGUS_L1_SA2_DATA_OUT_BASE_IDX                                                                 1
+#define regGUS_L1_SA2_DATA_U_IN                                                                         0x2c62
+#define regGUS_L1_SA2_DATA_U_IN_BASE_IDX                                                                1
+#define regGUS_L1_SA2_DATA_U_OUT                                                                        0x2c63
+#define regGUS_L1_SA2_DATA_U_OUT_BASE_IDX                                                               1
+#define regGUS_L1_SA3_CMD_IN                                                                            0x2c64
+#define regGUS_L1_SA3_CMD_IN_BASE_IDX                                                                   1
+#define regGUS_L1_SA3_CMD_OUT                                                                           0x2c65
+#define regGUS_L1_SA3_CMD_OUT_BASE_IDX                                                                  1
+#define regGUS_L1_SA3_DATA_IN                                                                           0x2c66
+#define regGUS_L1_SA3_DATA_IN_BASE_IDX                                                                  1
+#define regGUS_L1_SA3_DATA_OUT                                                                          0x2c67
+#define regGUS_L1_SA3_DATA_OUT_BASE_IDX                                                                 1
+#define regGUS_L1_SA3_DATA_U_IN                                                                         0x2c68
+#define regGUS_L1_SA3_DATA_U_IN_BASE_IDX                                                                1
+#define regGUS_L1_SA3_DATA_U_OUT                                                                        0x2c69
+#define regGUS_L1_SA3_DATA_U_OUT_BASE_IDX                                                               1
+#define regGUS_MISC3                                                                                    0x2c6a
+#define regGUS_MISC3_BASE_IDX                                                                           1
+#define regGUS_WRRSP_FIFO_CNTL                                                                          0x2c6b
+#define regGUS_WRRSP_FIFO_CNTL_BASE_IDX                                                                 1
+
+
+// addressBlock: gc_gfxdec0
+// base address: 0x28000
+#define regDB_RENDER_CONTROL                                                                            0x0000
+#define regDB_RENDER_CONTROL_BASE_IDX                                                                   1
+#define regDB_COUNT_CONTROL                                                                             0x0001
+#define regDB_COUNT_CONTROL_BASE_IDX                                                                    1
+#define regDB_DEPTH_VIEW                                                                                0x0002
+#define regDB_DEPTH_VIEW_BASE_IDX                                                                       1
+#define regDB_RENDER_OVERRIDE                                                                           0x0003
+#define regDB_RENDER_OVERRIDE_BASE_IDX                                                                  1
+#define regDB_RENDER_OVERRIDE2                                                                          0x0004
+#define regDB_RENDER_OVERRIDE2_BASE_IDX                                                                 1
+#define regDB_HTILE_DATA_BASE                                                                           0x0005
+#define regDB_HTILE_DATA_BASE_BASE_IDX                                                                  1
+#define regDB_DEPTH_SIZE_XY                                                                             0x0007
+#define regDB_DEPTH_SIZE_XY_BASE_IDX                                                                    1
+#define regDB_DEPTH_BOUNDS_MIN                                                                          0x0008
+#define regDB_DEPTH_BOUNDS_MIN_BASE_IDX                                                                 1
+#define regDB_DEPTH_BOUNDS_MAX                                                                          0x0009
+#define regDB_DEPTH_BOUNDS_MAX_BASE_IDX                                                                 1
+#define regDB_STENCIL_CLEAR                                                                             0x000a
+#define regDB_STENCIL_CLEAR_BASE_IDX                                                                    1
+#define regDB_DEPTH_CLEAR                                                                               0x000b
+#define regDB_DEPTH_CLEAR_BASE_IDX                                                                      1
+#define regPA_SC_SCREEN_SCISSOR_TL                                                                      0x000c
+#define regPA_SC_SCREEN_SCISSOR_TL_BASE_IDX                                                             1
+#define regPA_SC_SCREEN_SCISSOR_BR                                                                      0x000d
+#define regPA_SC_SCREEN_SCISSOR_BR_BASE_IDX                                                             1
+#define regDB_RESERVED_REG_2                                                                            0x000f
+#define regDB_RESERVED_REG_2_BASE_IDX                                                                   1
+#define regDB_Z_INFO                                                                                    0x0010
+#define regDB_Z_INFO_BASE_IDX                                                                           1
+#define regDB_STENCIL_INFO                                                                              0x0011
+#define regDB_STENCIL_INFO_BASE_IDX                                                                     1
+#define regDB_Z_READ_BASE                                                                               0x0012
+#define regDB_Z_READ_BASE_BASE_IDX                                                                      1
+#define regDB_STENCIL_READ_BASE                                                                         0x0013
+#define regDB_STENCIL_READ_BASE_BASE_IDX                                                                1
+#define regDB_Z_WRITE_BASE                                                                              0x0014
+#define regDB_Z_WRITE_BASE_BASE_IDX                                                                     1
+#define regDB_STENCIL_WRITE_BASE                                                                        0x0015
+#define regDB_STENCIL_WRITE_BASE_BASE_IDX                                                               1
+#define regDB_RESERVED_REG_1                                                                            0x0016
+#define regDB_RESERVED_REG_1_BASE_IDX                                                                   1
+#define regDB_RESERVED_REG_3                                                                            0x0017
+#define regDB_RESERVED_REG_3_BASE_IDX                                                                   1
+#define regDB_Z_READ_BASE_HI                                                                            0x001a
+#define regDB_Z_READ_BASE_HI_BASE_IDX                                                                   1
+#define regDB_STENCIL_READ_BASE_HI                                                                      0x001b
+#define regDB_STENCIL_READ_BASE_HI_BASE_IDX                                                             1
+#define regDB_Z_WRITE_BASE_HI                                                                           0x001c
+#define regDB_Z_WRITE_BASE_HI_BASE_IDX                                                                  1
+#define regDB_STENCIL_WRITE_BASE_HI                                                                     0x001d
+#define regDB_STENCIL_WRITE_BASE_HI_BASE_IDX                                                            1
+#define regDB_HTILE_DATA_BASE_HI                                                                        0x001e
+#define regDB_HTILE_DATA_BASE_HI_BASE_IDX                                                               1
+#define regDB_RMI_L2_CACHE_CONTROL                                                                      0x001f
+#define regDB_RMI_L2_CACHE_CONTROL_BASE_IDX                                                             1
+#define regTA_BC_BASE_ADDR                                                                              0x0020
+#define regTA_BC_BASE_ADDR_BASE_IDX                                                                     1
+#define regTA_BC_BASE_ADDR_HI                                                                           0x0021
+#define regTA_BC_BASE_ADDR_HI_BASE_IDX                                                                  1
+#define regCOHER_DEST_BASE_HI_0                                                                         0x007a
+#define regCOHER_DEST_BASE_HI_0_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_HI_1                                                                         0x007b
+#define regCOHER_DEST_BASE_HI_1_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_HI_2                                                                         0x007c
+#define regCOHER_DEST_BASE_HI_2_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_HI_3                                                                         0x007d
+#define regCOHER_DEST_BASE_HI_3_BASE_IDX                                                                1
+#define regCOHER_DEST_BASE_2                                                                            0x007e
+#define regCOHER_DEST_BASE_2_BASE_IDX                                                                   1
+#define regCOHER_DEST_BASE_3                                                                            0x007f
+#define regCOHER_DEST_BASE_3_BASE_IDX                                                                   1
+#define regPA_SC_WINDOW_OFFSET                                                                          0x0080
+#define regPA_SC_WINDOW_OFFSET_BASE_IDX                                                                 1
+#define regPA_SC_WINDOW_SCISSOR_TL                                                                      0x0081
+#define regPA_SC_WINDOW_SCISSOR_TL_BASE_IDX                                                             1
+#define regPA_SC_WINDOW_SCISSOR_BR                                                                      0x0082
+#define regPA_SC_WINDOW_SCISSOR_BR_BASE_IDX                                                             1
+#define regPA_SC_CLIPRECT_RULE                                                                          0x0083
+#define regPA_SC_CLIPRECT_RULE_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_0_TL                                                                          0x0084
+#define regPA_SC_CLIPRECT_0_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_0_BR                                                                          0x0085
+#define regPA_SC_CLIPRECT_0_BR_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_1_TL                                                                          0x0086
+#define regPA_SC_CLIPRECT_1_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_1_BR                                                                          0x0087
+#define regPA_SC_CLIPRECT_1_BR_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_2_TL                                                                          0x0088
+#define regPA_SC_CLIPRECT_2_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_2_BR                                                                          0x0089
+#define regPA_SC_CLIPRECT_2_BR_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_3_TL                                                                          0x008a
+#define regPA_SC_CLIPRECT_3_TL_BASE_IDX                                                                 1
+#define regPA_SC_CLIPRECT_3_BR                                                                          0x008b
+#define regPA_SC_CLIPRECT_3_BR_BASE_IDX                                                                 1
+#define regPA_SC_EDGERULE                                                                               0x008c
+#define regPA_SC_EDGERULE_BASE_IDX                                                                      1
+#define regPA_SU_HARDWARE_SCREEN_OFFSET                                                                 0x008d
+#define regPA_SU_HARDWARE_SCREEN_OFFSET_BASE_IDX                                                        1
+#define regCB_TARGET_MASK                                                                               0x008e
+#define regCB_TARGET_MASK_BASE_IDX                                                                      1
+#define regCB_SHADER_MASK                                                                               0x008f
+#define regCB_SHADER_MASK_BASE_IDX                                                                      1
+#define regPA_SC_GENERIC_SCISSOR_TL                                                                     0x0090
+#define regPA_SC_GENERIC_SCISSOR_TL_BASE_IDX                                                            1
+#define regPA_SC_GENERIC_SCISSOR_BR                                                                     0x0091
+#define regPA_SC_GENERIC_SCISSOR_BR_BASE_IDX                                                            1
+#define regCOHER_DEST_BASE_0                                                                            0x0092
+#define regCOHER_DEST_BASE_0_BASE_IDX                                                                   1
+#define regCOHER_DEST_BASE_1                                                                            0x0093
+#define regCOHER_DEST_BASE_1_BASE_IDX                                                                   1
+#define regPA_SC_VPORT_SCISSOR_0_TL                                                                     0x0094
+#define regPA_SC_VPORT_SCISSOR_0_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_0_BR                                                                     0x0095
+#define regPA_SC_VPORT_SCISSOR_0_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_1_TL                                                                     0x0096
+#define regPA_SC_VPORT_SCISSOR_1_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_1_BR                                                                     0x0097
+#define regPA_SC_VPORT_SCISSOR_1_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_2_TL                                                                     0x0098
+#define regPA_SC_VPORT_SCISSOR_2_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_2_BR                                                                     0x0099
+#define regPA_SC_VPORT_SCISSOR_2_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_3_TL                                                                     0x009a
+#define regPA_SC_VPORT_SCISSOR_3_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_3_BR                                                                     0x009b
+#define regPA_SC_VPORT_SCISSOR_3_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_4_TL                                                                     0x009c
+#define regPA_SC_VPORT_SCISSOR_4_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_4_BR                                                                     0x009d
+#define regPA_SC_VPORT_SCISSOR_4_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_5_TL                                                                     0x009e
+#define regPA_SC_VPORT_SCISSOR_5_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_5_BR                                                                     0x009f
+#define regPA_SC_VPORT_SCISSOR_5_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_6_TL                                                                     0x00a0
+#define regPA_SC_VPORT_SCISSOR_6_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_6_BR                                                                     0x00a1
+#define regPA_SC_VPORT_SCISSOR_6_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_7_TL                                                                     0x00a2
+#define regPA_SC_VPORT_SCISSOR_7_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_7_BR                                                                     0x00a3
+#define regPA_SC_VPORT_SCISSOR_7_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_8_TL                                                                     0x00a4
+#define regPA_SC_VPORT_SCISSOR_8_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_8_BR                                                                     0x00a5
+#define regPA_SC_VPORT_SCISSOR_8_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_9_TL                                                                     0x00a6
+#define regPA_SC_VPORT_SCISSOR_9_TL_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_9_BR                                                                     0x00a7
+#define regPA_SC_VPORT_SCISSOR_9_BR_BASE_IDX                                                            1
+#define regPA_SC_VPORT_SCISSOR_10_TL                                                                    0x00a8
+#define regPA_SC_VPORT_SCISSOR_10_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_10_BR                                                                    0x00a9
+#define regPA_SC_VPORT_SCISSOR_10_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_11_TL                                                                    0x00aa
+#define regPA_SC_VPORT_SCISSOR_11_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_11_BR                                                                    0x00ab
+#define regPA_SC_VPORT_SCISSOR_11_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_12_TL                                                                    0x00ac
+#define regPA_SC_VPORT_SCISSOR_12_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_12_BR                                                                    0x00ad
+#define regPA_SC_VPORT_SCISSOR_12_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_13_TL                                                                    0x00ae
+#define regPA_SC_VPORT_SCISSOR_13_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_13_BR                                                                    0x00af
+#define regPA_SC_VPORT_SCISSOR_13_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_14_TL                                                                    0x00b0
+#define regPA_SC_VPORT_SCISSOR_14_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_14_BR                                                                    0x00b1
+#define regPA_SC_VPORT_SCISSOR_14_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_15_TL                                                                    0x00b2
+#define regPA_SC_VPORT_SCISSOR_15_TL_BASE_IDX                                                           1
+#define regPA_SC_VPORT_SCISSOR_15_BR                                                                    0x00b3
+#define regPA_SC_VPORT_SCISSOR_15_BR_BASE_IDX                                                           1
+#define regPA_SC_VPORT_ZMIN_0                                                                           0x00b4
+#define regPA_SC_VPORT_ZMIN_0_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_0                                                                           0x00b5
+#define regPA_SC_VPORT_ZMAX_0_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_1                                                                           0x00b6
+#define regPA_SC_VPORT_ZMIN_1_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_1                                                                           0x00b7
+#define regPA_SC_VPORT_ZMAX_1_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_2                                                                           0x00b8
+#define regPA_SC_VPORT_ZMIN_2_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_2                                                                           0x00b9
+#define regPA_SC_VPORT_ZMAX_2_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_3                                                                           0x00ba
+#define regPA_SC_VPORT_ZMIN_3_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_3                                                                           0x00bb
+#define regPA_SC_VPORT_ZMAX_3_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_4                                                                           0x00bc
+#define regPA_SC_VPORT_ZMIN_4_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_4                                                                           0x00bd
+#define regPA_SC_VPORT_ZMAX_4_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_5                                                                           0x00be
+#define regPA_SC_VPORT_ZMIN_5_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_5                                                                           0x00bf
+#define regPA_SC_VPORT_ZMAX_5_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_6                                                                           0x00c0
+#define regPA_SC_VPORT_ZMIN_6_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_6                                                                           0x00c1
+#define regPA_SC_VPORT_ZMAX_6_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_7                                                                           0x00c2
+#define regPA_SC_VPORT_ZMIN_7_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_7                                                                           0x00c3
+#define regPA_SC_VPORT_ZMAX_7_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_8                                                                           0x00c4
+#define regPA_SC_VPORT_ZMIN_8_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_8                                                                           0x00c5
+#define regPA_SC_VPORT_ZMAX_8_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_9                                                                           0x00c6
+#define regPA_SC_VPORT_ZMIN_9_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMAX_9                                                                           0x00c7
+#define regPA_SC_VPORT_ZMAX_9_BASE_IDX                                                                  1
+#define regPA_SC_VPORT_ZMIN_10                                                                          0x00c8
+#define regPA_SC_VPORT_ZMIN_10_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_10                                                                          0x00c9
+#define regPA_SC_VPORT_ZMAX_10_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_11                                                                          0x00ca
+#define regPA_SC_VPORT_ZMIN_11_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_11                                                                          0x00cb
+#define regPA_SC_VPORT_ZMAX_11_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_12                                                                          0x00cc
+#define regPA_SC_VPORT_ZMIN_12_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_12                                                                          0x00cd
+#define regPA_SC_VPORT_ZMAX_12_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_13                                                                          0x00ce
+#define regPA_SC_VPORT_ZMIN_13_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_13                                                                          0x00cf
+#define regPA_SC_VPORT_ZMAX_13_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_14                                                                          0x00d0
+#define regPA_SC_VPORT_ZMIN_14_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_14                                                                          0x00d1
+#define regPA_SC_VPORT_ZMAX_14_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMIN_15                                                                          0x00d2
+#define regPA_SC_VPORT_ZMIN_15_BASE_IDX                                                                 1
+#define regPA_SC_VPORT_ZMAX_15                                                                          0x00d3
+#define regPA_SC_VPORT_ZMAX_15_BASE_IDX                                                                 1
+#define regPA_SC_RASTER_CONFIG                                                                          0x00d4
+#define regPA_SC_RASTER_CONFIG_BASE_IDX                                                                 1
+#define regPA_SC_RASTER_CONFIG_1                                                                        0x00d5
+#define regPA_SC_RASTER_CONFIG_1_BASE_IDX                                                               1
+#define regPA_SC_SCREEN_EXTENT_CONTROL                                                                  0x00d6
+#define regPA_SC_SCREEN_EXTENT_CONTROL_BASE_IDX                                                         1
+#define regPA_SC_TILE_STEERING_OVERRIDE                                                                 0x00d7
+#define regPA_SC_TILE_STEERING_OVERRIDE_BASE_IDX                                                        1
+#define regCP_PERFMON_CNTX_CNTL                                                                         0x00d8
+#define regCP_PERFMON_CNTX_CNTL_BASE_IDX                                                                1
+#define regCP_PIPEID                                                                                    0x00d9
+#define regCP_PIPEID_BASE_IDX                                                                           1
+#define regCP_RINGID                                                                                    0x00d9
+#define regCP_RINGID_BASE_IDX                                                                           1
+#define regCP_VMID                                                                                      0x00da
+#define regCP_VMID_BASE_IDX                                                                             1
+#define regCONTEXT_RESERVED_REG0                                                                        0x00db
+#define regCONTEXT_RESERVED_REG0_BASE_IDX                                                               1
+#define regCONTEXT_RESERVED_REG1                                                                        0x00dc
+#define regCONTEXT_RESERVED_REG1_BASE_IDX                                                               1
+#define regPA_SC_VRS_OVERRIDE_CNTL                                                                      0x00f4
+#define regPA_SC_VRS_OVERRIDE_CNTL_BASE_IDX                                                             1
+#define regPA_SC_VRS_RATE_FEEDBACK_BASE                                                                 0x00f5
+#define regPA_SC_VRS_RATE_FEEDBACK_BASE_BASE_IDX                                                        1
+#define regPA_SC_VRS_RATE_FEEDBACK_BASE_EXT                                                             0x00f6
+#define regPA_SC_VRS_RATE_FEEDBACK_BASE_EXT_BASE_IDX                                                    1
+#define regPA_SC_VRS_RATE_FEEDBACK_SIZE_XY                                                              0x00f7
+#define regPA_SC_VRS_RATE_FEEDBACK_SIZE_XY_BASE_IDX                                                     1
+#define regPA_SC_VRS_RATE_CACHE_CNTL                                                                    0x00f9
+#define regPA_SC_VRS_RATE_CACHE_CNTL_BASE_IDX                                                           1
+#define regPA_SC_VRS_RATE_BASE                                                                          0x00fc
+#define regPA_SC_VRS_RATE_BASE_BASE_IDX                                                                 1
+#define regPA_SC_VRS_RATE_BASE_EXT                                                                      0x00fd
+#define regPA_SC_VRS_RATE_BASE_EXT_BASE_IDX                                                             1
+#define regPA_SC_VRS_RATE_SIZE_XY                                                                       0x00fe
+#define regPA_SC_VRS_RATE_SIZE_XY_BASE_IDX                                                              1
+#define regVGT_MULTI_PRIM_IB_RESET_INDX                                                                 0x0103
+#define regVGT_MULTI_PRIM_IB_RESET_INDX_BASE_IDX                                                        1
+#define regCB_RMI_GL2_CACHE_CONTROL                                                                     0x0104
+#define regCB_RMI_GL2_CACHE_CONTROL_BASE_IDX                                                            1
+#define regCB_BLEND_RED                                                                                 0x0105
+#define regCB_BLEND_RED_BASE_IDX                                                                        1
+#define regCB_BLEND_GREEN                                                                               0x0106
+#define regCB_BLEND_GREEN_BASE_IDX                                                                      1
+#define regCB_BLEND_BLUE                                                                                0x0107
+#define regCB_BLEND_BLUE_BASE_IDX                                                                       1
+#define regCB_BLEND_ALPHA                                                                               0x0108
+#define regCB_BLEND_ALPHA_BASE_IDX                                                                      1
+#define regCB_FDCC_CONTROL                                                                              0x0109
+#define regCB_FDCC_CONTROL_BASE_IDX                                                                     1
+#define regCB_COVERAGE_OUT_CONTROL                                                                      0x010a
+#define regCB_COVERAGE_OUT_CONTROL_BASE_IDX                                                             1
+#define regDB_STENCIL_CONTROL                                                                           0x010b
+#define regDB_STENCIL_CONTROL_BASE_IDX                                                                  1
+#define regDB_STENCILREFMASK                                                                            0x010c
+#define regDB_STENCILREFMASK_BASE_IDX                                                                   1
+#define regDB_STENCILREFMASK_BF                                                                         0x010d
+#define regDB_STENCILREFMASK_BF_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XSCALE                                                                           0x010f
+#define regPA_CL_VPORT_XSCALE_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_XOFFSET                                                                          0x0110
+#define regPA_CL_VPORT_XOFFSET_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_YSCALE                                                                           0x0111
+#define regPA_CL_VPORT_YSCALE_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_YOFFSET                                                                          0x0112
+#define regPA_CL_VPORT_YOFFSET_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_ZSCALE                                                                           0x0113
+#define regPA_CL_VPORT_ZSCALE_BASE_IDX                                                                  1
+#define regPA_CL_VPORT_ZOFFSET                                                                          0x0114
+#define regPA_CL_VPORT_ZOFFSET_BASE_IDX                                                                 1
+#define regPA_CL_VPORT_XSCALE_1                                                                         0x0115
+#define regPA_CL_VPORT_XSCALE_1_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_1                                                                        0x0116
+#define regPA_CL_VPORT_XOFFSET_1_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_1                                                                         0x0117
+#define regPA_CL_VPORT_YSCALE_1_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_1                                                                        0x0118
+#define regPA_CL_VPORT_YOFFSET_1_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_1                                                                         0x0119
+#define regPA_CL_VPORT_ZSCALE_1_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_1                                                                        0x011a
+#define regPA_CL_VPORT_ZOFFSET_1_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_2                                                                         0x011b
+#define regPA_CL_VPORT_XSCALE_2_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_2                                                                        0x011c
+#define regPA_CL_VPORT_XOFFSET_2_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_2                                                                         0x011d
+#define regPA_CL_VPORT_YSCALE_2_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_2                                                                        0x011e
+#define regPA_CL_VPORT_YOFFSET_2_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_2                                                                         0x011f
+#define regPA_CL_VPORT_ZSCALE_2_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_2                                                                        0x0120
+#define regPA_CL_VPORT_ZOFFSET_2_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_3                                                                         0x0121
+#define regPA_CL_VPORT_XSCALE_3_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_3                                                                        0x0122
+#define regPA_CL_VPORT_XOFFSET_3_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_3                                                                         0x0123
+#define regPA_CL_VPORT_YSCALE_3_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_3                                                                        0x0124
+#define regPA_CL_VPORT_YOFFSET_3_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_3                                                                         0x0125
+#define regPA_CL_VPORT_ZSCALE_3_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_3                                                                        0x0126
+#define regPA_CL_VPORT_ZOFFSET_3_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_4                                                                         0x0127
+#define regPA_CL_VPORT_XSCALE_4_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_4                                                                        0x0128
+#define regPA_CL_VPORT_XOFFSET_4_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_4                                                                         0x0129
+#define regPA_CL_VPORT_YSCALE_4_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_4                                                                        0x012a
+#define regPA_CL_VPORT_YOFFSET_4_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_4                                                                         0x012b
+#define regPA_CL_VPORT_ZSCALE_4_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_4                                                                        0x012c
+#define regPA_CL_VPORT_ZOFFSET_4_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_5                                                                         0x012d
+#define regPA_CL_VPORT_XSCALE_5_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_5                                                                        0x012e
+#define regPA_CL_VPORT_XOFFSET_5_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_5                                                                         0x012f
+#define regPA_CL_VPORT_YSCALE_5_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_5                                                                        0x0130
+#define regPA_CL_VPORT_YOFFSET_5_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_5                                                                         0x0131
+#define regPA_CL_VPORT_ZSCALE_5_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_5                                                                        0x0132
+#define regPA_CL_VPORT_ZOFFSET_5_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_6                                                                         0x0133
+#define regPA_CL_VPORT_XSCALE_6_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_6                                                                        0x0134
+#define regPA_CL_VPORT_XOFFSET_6_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_6                                                                         0x0135
+#define regPA_CL_VPORT_YSCALE_6_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_6                                                                        0x0136
+#define regPA_CL_VPORT_YOFFSET_6_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_6                                                                         0x0137
+#define regPA_CL_VPORT_ZSCALE_6_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_6                                                                        0x0138
+#define regPA_CL_VPORT_ZOFFSET_6_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_7                                                                         0x0139
+#define regPA_CL_VPORT_XSCALE_7_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_7                                                                        0x013a
+#define regPA_CL_VPORT_XOFFSET_7_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_7                                                                         0x013b
+#define regPA_CL_VPORT_YSCALE_7_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_7                                                                        0x013c
+#define regPA_CL_VPORT_YOFFSET_7_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_7                                                                         0x013d
+#define regPA_CL_VPORT_ZSCALE_7_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_7                                                                        0x013e
+#define regPA_CL_VPORT_ZOFFSET_7_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_8                                                                         0x013f
+#define regPA_CL_VPORT_XSCALE_8_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_8                                                                        0x0140
+#define regPA_CL_VPORT_XOFFSET_8_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_8                                                                         0x0141
+#define regPA_CL_VPORT_YSCALE_8_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_8                                                                        0x0142
+#define regPA_CL_VPORT_YOFFSET_8_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_8                                                                         0x0143
+#define regPA_CL_VPORT_ZSCALE_8_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_8                                                                        0x0144
+#define regPA_CL_VPORT_ZOFFSET_8_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_9                                                                         0x0145
+#define regPA_CL_VPORT_XSCALE_9_BASE_IDX                                                                1
+#define regPA_CL_VPORT_XOFFSET_9                                                                        0x0146
+#define regPA_CL_VPORT_XOFFSET_9_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YSCALE_9                                                                         0x0147
+#define regPA_CL_VPORT_YSCALE_9_BASE_IDX                                                                1
+#define regPA_CL_VPORT_YOFFSET_9                                                                        0x0148
+#define regPA_CL_VPORT_YOFFSET_9_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZSCALE_9                                                                         0x0149
+#define regPA_CL_VPORT_ZSCALE_9_BASE_IDX                                                                1
+#define regPA_CL_VPORT_ZOFFSET_9                                                                        0x014a
+#define regPA_CL_VPORT_ZOFFSET_9_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XSCALE_10                                                                        0x014b
+#define regPA_CL_VPORT_XSCALE_10_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_10                                                                       0x014c
+#define regPA_CL_VPORT_XOFFSET_10_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_10                                                                        0x014d
+#define regPA_CL_VPORT_YSCALE_10_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_10                                                                       0x014e
+#define regPA_CL_VPORT_YOFFSET_10_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_10                                                                        0x014f
+#define regPA_CL_VPORT_ZSCALE_10_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_10                                                                       0x0150
+#define regPA_CL_VPORT_ZOFFSET_10_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE_11                                                                        0x0151
+#define regPA_CL_VPORT_XSCALE_11_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_11                                                                       0x0152
+#define regPA_CL_VPORT_XOFFSET_11_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_11                                                                        0x0153
+#define regPA_CL_VPORT_YSCALE_11_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_11                                                                       0x0154
+#define regPA_CL_VPORT_YOFFSET_11_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_11                                                                        0x0155
+#define regPA_CL_VPORT_ZSCALE_11_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_11                                                                       0x0156
+#define regPA_CL_VPORT_ZOFFSET_11_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE_12                                                                        0x0157
+#define regPA_CL_VPORT_XSCALE_12_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_12                                                                       0x0158
+#define regPA_CL_VPORT_XOFFSET_12_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_12                                                                        0x0159
+#define regPA_CL_VPORT_YSCALE_12_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_12                                                                       0x015a
+#define regPA_CL_VPORT_YOFFSET_12_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_12                                                                        0x015b
+#define regPA_CL_VPORT_ZSCALE_12_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_12                                                                       0x015c
+#define regPA_CL_VPORT_ZOFFSET_12_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE_13                                                                        0x015d
+#define regPA_CL_VPORT_XSCALE_13_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_13                                                                       0x015e
+#define regPA_CL_VPORT_XOFFSET_13_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_13                                                                        0x015f
+#define regPA_CL_VPORT_YSCALE_13_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_13                                                                       0x0160
+#define regPA_CL_VPORT_YOFFSET_13_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_13                                                                        0x0161
+#define regPA_CL_VPORT_ZSCALE_13_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_13                                                                       0x0162
+#define regPA_CL_VPORT_ZOFFSET_13_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE_14                                                                        0x0163
+#define regPA_CL_VPORT_XSCALE_14_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_14                                                                       0x0164
+#define regPA_CL_VPORT_XOFFSET_14_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_14                                                                        0x0165
+#define regPA_CL_VPORT_YSCALE_14_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_14                                                                       0x0166
+#define regPA_CL_VPORT_YOFFSET_14_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_14                                                                        0x0167
+#define regPA_CL_VPORT_ZSCALE_14_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_14                                                                       0x0168
+#define regPA_CL_VPORT_ZOFFSET_14_BASE_IDX                                                              1
+#define regPA_CL_VPORT_XSCALE_15                                                                        0x0169
+#define regPA_CL_VPORT_XSCALE_15_BASE_IDX                                                               1
+#define regPA_CL_VPORT_XOFFSET_15                                                                       0x016a
+#define regPA_CL_VPORT_XOFFSET_15_BASE_IDX                                                              1
+#define regPA_CL_VPORT_YSCALE_15                                                                        0x016b
+#define regPA_CL_VPORT_YSCALE_15_BASE_IDX                                                               1
+#define regPA_CL_VPORT_YOFFSET_15                                                                       0x016c
+#define regPA_CL_VPORT_YOFFSET_15_BASE_IDX                                                              1
+#define regPA_CL_VPORT_ZSCALE_15                                                                        0x016d
+#define regPA_CL_VPORT_ZSCALE_15_BASE_IDX                                                               1
+#define regPA_CL_VPORT_ZOFFSET_15                                                                       0x016e
+#define regPA_CL_VPORT_ZOFFSET_15_BASE_IDX                                                              1
+#define regPA_CL_UCP_0_X                                                                                0x016f
+#define regPA_CL_UCP_0_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_0_Y                                                                                0x0170
+#define regPA_CL_UCP_0_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_0_Z                                                                                0x0171
+#define regPA_CL_UCP_0_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_0_W                                                                                0x0172
+#define regPA_CL_UCP_0_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_X                                                                                0x0173
+#define regPA_CL_UCP_1_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_Y                                                                                0x0174
+#define regPA_CL_UCP_1_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_Z                                                                                0x0175
+#define regPA_CL_UCP_1_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_1_W                                                                                0x0176
+#define regPA_CL_UCP_1_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_X                                                                                0x0177
+#define regPA_CL_UCP_2_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_Y                                                                                0x0178
+#define regPA_CL_UCP_2_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_Z                                                                                0x0179
+#define regPA_CL_UCP_2_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_2_W                                                                                0x017a
+#define regPA_CL_UCP_2_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_X                                                                                0x017b
+#define regPA_CL_UCP_3_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_Y                                                                                0x017c
+#define regPA_CL_UCP_3_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_Z                                                                                0x017d
+#define regPA_CL_UCP_3_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_3_W                                                                                0x017e
+#define regPA_CL_UCP_3_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_X                                                                                0x017f
+#define regPA_CL_UCP_4_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_Y                                                                                0x0180
+#define regPA_CL_UCP_4_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_Z                                                                                0x0181
+#define regPA_CL_UCP_4_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_4_W                                                                                0x0182
+#define regPA_CL_UCP_4_W_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_X                                                                                0x0183
+#define regPA_CL_UCP_5_X_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_Y                                                                                0x0184
+#define regPA_CL_UCP_5_Y_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_Z                                                                                0x0185
+#define regPA_CL_UCP_5_Z_BASE_IDX                                                                       1
+#define regPA_CL_UCP_5_W                                                                                0x0186
+#define regPA_CL_UCP_5_W_BASE_IDX                                                                       1
+#define regPA_CL_PROG_NEAR_CLIP_Z                                                                       0x0187
+#define regPA_CL_PROG_NEAR_CLIP_Z_BASE_IDX                                                              1
+#define regPA_RATE_CNTL                                                                                 0x0188
+#define regPA_RATE_CNTL_BASE_IDX                                                                        1
+#define regSPI_PS_INPUT_CNTL_0                                                                          0x0191
+#define regSPI_PS_INPUT_CNTL_0_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_1                                                                          0x0192
+#define regSPI_PS_INPUT_CNTL_1_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_2                                                                          0x0193
+#define regSPI_PS_INPUT_CNTL_2_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_3                                                                          0x0194
+#define regSPI_PS_INPUT_CNTL_3_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_4                                                                          0x0195
+#define regSPI_PS_INPUT_CNTL_4_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_5                                                                          0x0196
+#define regSPI_PS_INPUT_CNTL_5_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_6                                                                          0x0197
+#define regSPI_PS_INPUT_CNTL_6_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_7                                                                          0x0198
+#define regSPI_PS_INPUT_CNTL_7_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_8                                                                          0x0199
+#define regSPI_PS_INPUT_CNTL_8_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_9                                                                          0x019a
+#define regSPI_PS_INPUT_CNTL_9_BASE_IDX                                                                 1
+#define regSPI_PS_INPUT_CNTL_10                                                                         0x019b
+#define regSPI_PS_INPUT_CNTL_10_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_11                                                                         0x019c
+#define regSPI_PS_INPUT_CNTL_11_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_12                                                                         0x019d
+#define regSPI_PS_INPUT_CNTL_12_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_13                                                                         0x019e
+#define regSPI_PS_INPUT_CNTL_13_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_14                                                                         0x019f
+#define regSPI_PS_INPUT_CNTL_14_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_15                                                                         0x01a0
+#define regSPI_PS_INPUT_CNTL_15_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_16                                                                         0x01a1
+#define regSPI_PS_INPUT_CNTL_16_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_17                                                                         0x01a2
+#define regSPI_PS_INPUT_CNTL_17_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_18                                                                         0x01a3
+#define regSPI_PS_INPUT_CNTL_18_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_19                                                                         0x01a4
+#define regSPI_PS_INPUT_CNTL_19_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_20                                                                         0x01a5
+#define regSPI_PS_INPUT_CNTL_20_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_21                                                                         0x01a6
+#define regSPI_PS_INPUT_CNTL_21_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_22                                                                         0x01a7
+#define regSPI_PS_INPUT_CNTL_22_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_23                                                                         0x01a8
+#define regSPI_PS_INPUT_CNTL_23_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_24                                                                         0x01a9
+#define regSPI_PS_INPUT_CNTL_24_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_25                                                                         0x01aa
+#define regSPI_PS_INPUT_CNTL_25_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_26                                                                         0x01ab
+#define regSPI_PS_INPUT_CNTL_26_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_27                                                                         0x01ac
+#define regSPI_PS_INPUT_CNTL_27_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_28                                                                         0x01ad
+#define regSPI_PS_INPUT_CNTL_28_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_29                                                                         0x01ae
+#define regSPI_PS_INPUT_CNTL_29_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_30                                                                         0x01af
+#define regSPI_PS_INPUT_CNTL_30_BASE_IDX                                                                1
+#define regSPI_PS_INPUT_CNTL_31                                                                         0x01b0
+#define regSPI_PS_INPUT_CNTL_31_BASE_IDX                                                                1
+#define regSPI_VS_OUT_CONFIG                                                                            0x01b1
+#define regSPI_VS_OUT_CONFIG_BASE_IDX                                                                   1
+#define regSPI_PS_INPUT_ENA                                                                             0x01b3
+#define regSPI_PS_INPUT_ENA_BASE_IDX                                                                    1
+#define regSPI_PS_INPUT_ADDR                                                                            0x01b4
+#define regSPI_PS_INPUT_ADDR_BASE_IDX                                                                   1
+#define regSPI_INTERP_CONTROL_0                                                                         0x01b5
+#define regSPI_INTERP_CONTROL_0_BASE_IDX                                                                1
+#define regSPI_PS_IN_CONTROL                                                                            0x01b6
+#define regSPI_PS_IN_CONTROL_BASE_IDX                                                                   1
+#define regSPI_BARYC_CNTL                                                                               0x01b8
+#define regSPI_BARYC_CNTL_BASE_IDX                                                                      1
+#define regSPI_TMPRING_SIZE                                                                             0x01ba
+#define regSPI_TMPRING_SIZE_BASE_IDX                                                                    1
+#define regSPI_GFX_SCRATCH_BASE_LO                                                                      0x01bb
+#define regSPI_GFX_SCRATCH_BASE_LO_BASE_IDX                                                             1
+#define regSPI_GFX_SCRATCH_BASE_HI                                                                      0x01bc
+#define regSPI_GFX_SCRATCH_BASE_HI_BASE_IDX                                                             1
+#define regSPI_SHADER_IDX_FORMAT                                                                        0x01c2
+#define regSPI_SHADER_IDX_FORMAT_BASE_IDX                                                               1
+#define regSPI_SHADER_POS_FORMAT                                                                        0x01c3
+#define regSPI_SHADER_POS_FORMAT_BASE_IDX                                                               1
+#define regSPI_SHADER_Z_FORMAT                                                                          0x01c4
+#define regSPI_SHADER_Z_FORMAT_BASE_IDX                                                                 1
+#define regSPI_SHADER_COL_FORMAT                                                                        0x01c5
+#define regSPI_SHADER_COL_FORMAT_BASE_IDX                                                               1
+#define regSX_PS_DOWNCONVERT_CONTROL                                                                    0x01d4
+#define regSX_PS_DOWNCONVERT_CONTROL_BASE_IDX                                                           1
+#define regSX_PS_DOWNCONVERT                                                                            0x01d5
+#define regSX_PS_DOWNCONVERT_BASE_IDX                                                                   1
+#define regSX_BLEND_OPT_EPSILON                                                                         0x01d6
+#define regSX_BLEND_OPT_EPSILON_BASE_IDX                                                                1
+#define regSX_BLEND_OPT_CONTROL                                                                         0x01d7
+#define regSX_BLEND_OPT_CONTROL_BASE_IDX                                                                1
+#define regSX_MRT0_BLEND_OPT                                                                            0x01d8
+#define regSX_MRT0_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT1_BLEND_OPT                                                                            0x01d9
+#define regSX_MRT1_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT2_BLEND_OPT                                                                            0x01da
+#define regSX_MRT2_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT3_BLEND_OPT                                                                            0x01db
+#define regSX_MRT3_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT4_BLEND_OPT                                                                            0x01dc
+#define regSX_MRT4_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT5_BLEND_OPT                                                                            0x01dd
+#define regSX_MRT5_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT6_BLEND_OPT                                                                            0x01de
+#define regSX_MRT6_BLEND_OPT_BASE_IDX                                                                   1
+#define regSX_MRT7_BLEND_OPT                                                                            0x01df
+#define regSX_MRT7_BLEND_OPT_BASE_IDX                                                                   1
+#define regCB_BLEND0_CONTROL                                                                            0x01e0
+#define regCB_BLEND0_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND1_CONTROL                                                                            0x01e1
+#define regCB_BLEND1_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND2_CONTROL                                                                            0x01e2
+#define regCB_BLEND2_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND3_CONTROL                                                                            0x01e3
+#define regCB_BLEND3_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND4_CONTROL                                                                            0x01e4
+#define regCB_BLEND4_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND5_CONTROL                                                                            0x01e5
+#define regCB_BLEND5_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND6_CONTROL                                                                            0x01e6
+#define regCB_BLEND6_CONTROL_BASE_IDX                                                                   1
+#define regCB_BLEND7_CONTROL                                                                            0x01e7
+#define regCB_BLEND7_CONTROL_BASE_IDX                                                                   1
+#define regGFX_COPY_STATE                                                                               0x01f4
+#define regGFX_COPY_STATE_BASE_IDX                                                                      1
+#define regPA_CL_POINT_X_RAD                                                                            0x01f5
+#define regPA_CL_POINT_X_RAD_BASE_IDX                                                                   1
+#define regPA_CL_POINT_Y_RAD                                                                            0x01f6
+#define regPA_CL_POINT_Y_RAD_BASE_IDX                                                                   1
+#define regPA_CL_POINT_SIZE                                                                             0x01f7
+#define regPA_CL_POINT_SIZE_BASE_IDX                                                                    1
+#define regPA_CL_POINT_CULL_RAD                                                                         0x01f8
+#define regPA_CL_POINT_CULL_RAD_BASE_IDX                                                                1
+#define regVGT_DMA_BASE_HI                                                                              0x01f9
+#define regVGT_DMA_BASE_HI_BASE_IDX                                                                     1
+#define regVGT_DMA_BASE                                                                                 0x01fa
+#define regVGT_DMA_BASE_BASE_IDX                                                                        1
+#define regVGT_DRAW_INITIATOR                                                                           0x01fc
+#define regVGT_DRAW_INITIATOR_BASE_IDX                                                                  1
+#define regVGT_EVENT_ADDRESS_REG                                                                        0x01fe
+#define regVGT_EVENT_ADDRESS_REG_BASE_IDX                                                               1
+#define regGE_MAX_OUTPUT_PER_SUBGROUP                                                                   0x01ff
+#define regGE_MAX_OUTPUT_PER_SUBGROUP_BASE_IDX                                                          1
+#define regDB_DEPTH_CONTROL                                                                             0x0200
+#define regDB_DEPTH_CONTROL_BASE_IDX                                                                    1
+#define regDB_EQAA                                                                                      0x0201
+#define regDB_EQAA_BASE_IDX                                                                             1
+#define regCB_COLOR_CONTROL                                                                             0x0202
+#define regCB_COLOR_CONTROL_BASE_IDX                                                                    1
+#define regDB_SHADER_CONTROL                                                                            0x0203
+#define regDB_SHADER_CONTROL_BASE_IDX                                                                   1
+#define regPA_CL_CLIP_CNTL                                                                              0x0204
+#define regPA_CL_CLIP_CNTL_BASE_IDX                                                                     1
+#define regPA_SU_SC_MODE_CNTL                                                                           0x0205
+#define regPA_SU_SC_MODE_CNTL_BASE_IDX                                                                  1
+#define regPA_CL_VTE_CNTL                                                                               0x0206
+#define regPA_CL_VTE_CNTL_BASE_IDX                                                                      1
+#define regPA_CL_VS_OUT_CNTL                                                                            0x0207
+#define regPA_CL_VS_OUT_CNTL_BASE_IDX                                                                   1
+#define regPA_CL_NANINF_CNTL                                                                            0x0208
+#define regPA_CL_NANINF_CNTL_BASE_IDX                                                                   1
+#define regPA_SU_LINE_STIPPLE_CNTL                                                                      0x0209
+#define regPA_SU_LINE_STIPPLE_CNTL_BASE_IDX                                                             1
+#define regPA_SU_LINE_STIPPLE_SCALE                                                                     0x020a
+#define regPA_SU_LINE_STIPPLE_SCALE_BASE_IDX                                                            1
+#define regPA_SU_PRIM_FILTER_CNTL                                                                       0x020b
+#define regPA_SU_PRIM_FILTER_CNTL_BASE_IDX                                                              1
+#define regPA_SU_SMALL_PRIM_FILTER_CNTL                                                                 0x020c
+#define regPA_SU_SMALL_PRIM_FILTER_CNTL_BASE_IDX                                                        1
+#define regPA_CL_NGG_CNTL                                                                               0x020e
+#define regPA_CL_NGG_CNTL_BASE_IDX                                                                      1
+#define regPA_SU_OVER_RASTERIZATION_CNTL                                                                0x020f
+#define regPA_SU_OVER_RASTERIZATION_CNTL_BASE_IDX                                                       1
+#define regPA_STEREO_CNTL                                                                               0x0210
+#define regPA_STEREO_CNTL_BASE_IDX                                                                      1
+#define regPA_STATE_STEREO_X                                                                            0x0211
+#define regPA_STATE_STEREO_X_BASE_IDX                                                                   1
+#define regPA_CL_VRS_CNTL                                                                               0x0212
+#define regPA_CL_VRS_CNTL_BASE_IDX                                                                      1
+#define regPA_SU_POINT_SIZE                                                                             0x0280
+#define regPA_SU_POINT_SIZE_BASE_IDX                                                                    1
+#define regPA_SU_POINT_MINMAX                                                                           0x0281
+#define regPA_SU_POINT_MINMAX_BASE_IDX                                                                  1
+#define regPA_SU_LINE_CNTL                                                                              0x0282
+#define regPA_SU_LINE_CNTL_BASE_IDX                                                                     1
+#define regPA_SC_LINE_STIPPLE                                                                           0x0283
+#define regPA_SC_LINE_STIPPLE_BASE_IDX                                                                  1
+#define regVGT_HOS_MAX_TESS_LEVEL                                                                       0x0286
+#define regVGT_HOS_MAX_TESS_LEVEL_BASE_IDX                                                              1
+#define regVGT_HOS_MIN_TESS_LEVEL                                                                       0x0287
+#define regVGT_HOS_MIN_TESS_LEVEL_BASE_IDX                                                              1
+#define regPA_SC_MODE_CNTL_0                                                                            0x0292
+#define regPA_SC_MODE_CNTL_0_BASE_IDX                                                                   1
+#define regPA_SC_MODE_CNTL_1                                                                            0x0293
+#define regPA_SC_MODE_CNTL_1_BASE_IDX                                                                   1
+#define regVGT_ENHANCE                                                                                  0x0294
+#define regVGT_ENHANCE_BASE_IDX                                                                         1
+#define regIA_ENHANCE                                                                                   0x029c
+#define regIA_ENHANCE_BASE_IDX                                                                          1
+#define regVGT_DMA_SIZE                                                                                 0x029d
+#define regVGT_DMA_SIZE_BASE_IDX                                                                        1
+#define regVGT_DMA_MAX_SIZE                                                                             0x029e
+#define regVGT_DMA_MAX_SIZE_BASE_IDX                                                                    1
+#define regVGT_DMA_INDEX_TYPE                                                                           0x029f
+#define regVGT_DMA_INDEX_TYPE_BASE_IDX                                                                  1
+#define regWD_ENHANCE                                                                                   0x02a0
+#define regWD_ENHANCE_BASE_IDX                                                                          1
+#define regVGT_PRIMITIVEID_EN                                                                           0x02a1
+#define regVGT_PRIMITIVEID_EN_BASE_IDX                                                                  1
+#define regVGT_DMA_NUM_INSTANCES                                                                        0x02a2
+#define regVGT_DMA_NUM_INSTANCES_BASE_IDX                                                               1
+#define regVGT_PRIMITIVEID_RESET                                                                        0x02a3
+#define regVGT_PRIMITIVEID_RESET_BASE_IDX                                                               1
+#define regVGT_EVENT_INITIATOR                                                                          0x02a4
+#define regVGT_EVENT_INITIATOR_BASE_IDX                                                                 1
+#define regVGT_DRAW_PAYLOAD_CNTL                                                                        0x02a6
+#define regVGT_DRAW_PAYLOAD_CNTL_BASE_IDX                                                               1
+#define regVGT_ESGS_RING_ITEMSIZE                                                                       0x02ab
+#define regVGT_ESGS_RING_ITEMSIZE_BASE_IDX                                                              1
+#define regVGT_REUSE_OFF                                                                                0x02ad
+#define regVGT_REUSE_OFF_BASE_IDX                                                                       1
+#define regDB_HTILE_SURFACE                                                                             0x02af
+#define regDB_HTILE_SURFACE_BASE_IDX                                                                    1
+#define regDB_SRESULTS_COMPARE_STATE0                                                                   0x02b0
+#define regDB_SRESULTS_COMPARE_STATE0_BASE_IDX                                                          1
+#define regDB_SRESULTS_COMPARE_STATE1                                                                   0x02b1
+#define regDB_SRESULTS_COMPARE_STATE1_BASE_IDX                                                          1
+#define regDB_PRELOAD_CONTROL                                                                           0x02b2
+#define regDB_PRELOAD_CONTROL_BASE_IDX                                                                  1
+#define regVGT_STRMOUT_DRAW_OPAQUE_OFFSET                                                               0x02ca
+#define regVGT_STRMOUT_DRAW_OPAQUE_OFFSET_BASE_IDX                                                      1
+#define regVGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE                                                   0x02cb
+#define regVGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE_BASE_IDX                                          1
+#define regVGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE                                                        0x02cc
+#define regVGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE_BASE_IDX                                               1
+#define regVGT_GS_MAX_VERT_OUT                                                                          0x02ce
+#define regVGT_GS_MAX_VERT_OUT_BASE_IDX                                                                 1
+#define regGE_NGG_SUBGRP_CNTL                                                                           0x02d3
+#define regGE_NGG_SUBGRP_CNTL_BASE_IDX                                                                  1
+#define regVGT_TESS_DISTRIBUTION                                                                        0x02d4
+#define regVGT_TESS_DISTRIBUTION_BASE_IDX                                                               1
+#define regVGT_SHADER_STAGES_EN                                                                         0x02d5
+#define regVGT_SHADER_STAGES_EN_BASE_IDX                                                                1
+#define regVGT_LS_HS_CONFIG                                                                             0x02d6
+#define regVGT_LS_HS_CONFIG_BASE_IDX                                                                    1
+#define regVGT_TF_PARAM                                                                                 0x02db
+#define regVGT_TF_PARAM_BASE_IDX                                                                        1
+#define regDB_ALPHA_TO_MASK                                                                             0x02dc
+#define regDB_ALPHA_TO_MASK_BASE_IDX                                                                    1
+#define regPA_SU_POLY_OFFSET_DB_FMT_CNTL                                                                0x02de
+#define regPA_SU_POLY_OFFSET_DB_FMT_CNTL_BASE_IDX                                                       1
+#define regPA_SU_POLY_OFFSET_CLAMP                                                                      0x02df
+#define regPA_SU_POLY_OFFSET_CLAMP_BASE_IDX                                                             1
+#define regPA_SU_POLY_OFFSET_FRONT_SCALE                                                                0x02e0
+#define regPA_SU_POLY_OFFSET_FRONT_SCALE_BASE_IDX                                                       1
+#define regPA_SU_POLY_OFFSET_FRONT_OFFSET                                                               0x02e1
+#define regPA_SU_POLY_OFFSET_FRONT_OFFSET_BASE_IDX                                                      1
+#define regPA_SU_POLY_OFFSET_BACK_SCALE                                                                 0x02e2
+#define regPA_SU_POLY_OFFSET_BACK_SCALE_BASE_IDX                                                        1
+#define regPA_SU_POLY_OFFSET_BACK_OFFSET                                                                0x02e3
+#define regPA_SU_POLY_OFFSET_BACK_OFFSET_BASE_IDX                                                       1
+#define regVGT_GS_INSTANCE_CNT                                                                          0x02e4
+#define regVGT_GS_INSTANCE_CNT_BASE_IDX                                                                 1
+#define regPA_SC_CENTROID_PRIORITY_0                                                                    0x02f5
+#define regPA_SC_CENTROID_PRIORITY_0_BASE_IDX                                                           1
+#define regPA_SC_CENTROID_PRIORITY_1                                                                    0x02f6
+#define regPA_SC_CENTROID_PRIORITY_1_BASE_IDX                                                           1
+#define regPA_SC_LINE_CNTL                                                                              0x02f7
+#define regPA_SC_LINE_CNTL_BASE_IDX                                                                     1
+#define regPA_SC_AA_CONFIG                                                                              0x02f8
+#define regPA_SC_AA_CONFIG_BASE_IDX                                                                     1
+#define regPA_SU_VTX_CNTL                                                                               0x02f9
+#define regPA_SU_VTX_CNTL_BASE_IDX                                                                      1
+#define regPA_CL_GB_VERT_CLIP_ADJ                                                                       0x02fa
+#define regPA_CL_GB_VERT_CLIP_ADJ_BASE_IDX                                                              1
+#define regPA_CL_GB_VERT_DISC_ADJ                                                                       0x02fb
+#define regPA_CL_GB_VERT_DISC_ADJ_BASE_IDX                                                              1
+#define regPA_CL_GB_HORZ_CLIP_ADJ                                                                       0x02fc
+#define regPA_CL_GB_HORZ_CLIP_ADJ_BASE_IDX                                                              1
+#define regPA_CL_GB_HORZ_DISC_ADJ                                                                       0x02fd
+#define regPA_CL_GB_HORZ_DISC_ADJ_BASE_IDX                                                              1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0                                                            0x02fe
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1                                                            0x02ff
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2                                                            0x0300
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3                                                            0x0301
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0                                                            0x0302
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1                                                            0x0303
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2                                                            0x0304
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3                                                            0x0305
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0                                                            0x0306
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1                                                            0x0307
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2                                                            0x0308
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3                                                            0x0309
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0                                                            0x030a
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1                                                            0x030b
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2                                                            0x030c
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2_BASE_IDX                                                   1
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3                                                            0x030d
+#define regPA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3_BASE_IDX                                                   1
+#define regPA_SC_AA_MASK_X0Y0_X1Y0                                                                      0x030e
+#define regPA_SC_AA_MASK_X0Y0_X1Y0_BASE_IDX                                                             1
+#define regPA_SC_AA_MASK_X0Y1_X1Y1                                                                      0x030f
+#define regPA_SC_AA_MASK_X0Y1_X1Y1_BASE_IDX                                                             1
+#define regPA_SC_SHADER_CONTROL                                                                         0x0310
+#define regPA_SC_SHADER_CONTROL_BASE_IDX                                                                1
+#define regPA_SC_BINNER_CNTL_0                                                                          0x0311
+#define regPA_SC_BINNER_CNTL_0_BASE_IDX                                                                 1
+#define regPA_SC_BINNER_CNTL_1                                                                          0x0312
+#define regPA_SC_BINNER_CNTL_1_BASE_IDX                                                                 1
+#define regPA_SC_CONSERVATIVE_RASTERIZATION_CNTL                                                        0x0313
+#define regPA_SC_CONSERVATIVE_RASTERIZATION_CNTL_BASE_IDX                                               1
+#define regPA_SC_NGG_MODE_CNTL                                                                          0x0314
+#define regPA_SC_NGG_MODE_CNTL_BASE_IDX                                                                 1
+#define regPA_SC_BINNER_CNTL_2                                                                          0x0315
+#define regPA_SC_BINNER_CNTL_2_BASE_IDX                                                                 1
+#define regCB_COLOR0_BASE                                                                               0x0318
+#define regCB_COLOR0_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR0_VIEW                                                                               0x031b
+#define regCB_COLOR0_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR0_INFO                                                                               0x031c
+#define regCB_COLOR0_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR0_ATTRIB                                                                             0x031d
+#define regCB_COLOR0_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR0_FDCC_CONTROL                                                                       0x031e
+#define regCB_COLOR0_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR0_DCC_BASE                                                                           0x0325
+#define regCB_COLOR0_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR1_BASE                                                                               0x0327
+#define regCB_COLOR1_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR1_VIEW                                                                               0x032a
+#define regCB_COLOR1_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR1_INFO                                                                               0x032b
+#define regCB_COLOR1_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR1_ATTRIB                                                                             0x032c
+#define regCB_COLOR1_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR1_FDCC_CONTROL                                                                       0x032d
+#define regCB_COLOR1_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR1_DCC_BASE                                                                           0x0334
+#define regCB_COLOR1_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR2_BASE                                                                               0x0336
+#define regCB_COLOR2_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR2_VIEW                                                                               0x0339
+#define regCB_COLOR2_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR2_INFO                                                                               0x033a
+#define regCB_COLOR2_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR2_ATTRIB                                                                             0x033b
+#define regCB_COLOR2_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR2_FDCC_CONTROL                                                                       0x033c
+#define regCB_COLOR2_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR2_DCC_BASE                                                                           0x0343
+#define regCB_COLOR2_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR3_BASE                                                                               0x0345
+#define regCB_COLOR3_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR3_VIEW                                                                               0x0348
+#define regCB_COLOR3_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR3_INFO                                                                               0x0349
+#define regCB_COLOR3_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR3_ATTRIB                                                                             0x034a
+#define regCB_COLOR3_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR3_FDCC_CONTROL                                                                       0x034b
+#define regCB_COLOR3_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR3_DCC_BASE                                                                           0x0352
+#define regCB_COLOR3_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR4_BASE                                                                               0x0354
+#define regCB_COLOR4_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR4_VIEW                                                                               0x0357
+#define regCB_COLOR4_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR4_INFO                                                                               0x0358
+#define regCB_COLOR4_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR4_ATTRIB                                                                             0x0359
+#define regCB_COLOR4_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR4_FDCC_CONTROL                                                                       0x035a
+#define regCB_COLOR4_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR4_DCC_BASE                                                                           0x0361
+#define regCB_COLOR4_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR5_BASE                                                                               0x0363
+#define regCB_COLOR5_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR5_VIEW                                                                               0x0366
+#define regCB_COLOR5_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR5_INFO                                                                               0x0367
+#define regCB_COLOR5_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR5_ATTRIB                                                                             0x0368
+#define regCB_COLOR5_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR5_FDCC_CONTROL                                                                       0x0369
+#define regCB_COLOR5_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR5_DCC_BASE                                                                           0x0370
+#define regCB_COLOR5_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR6_BASE                                                                               0x0372
+#define regCB_COLOR6_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR6_VIEW                                                                               0x0375
+#define regCB_COLOR6_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR6_INFO                                                                               0x0376
+#define regCB_COLOR6_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR6_ATTRIB                                                                             0x0377
+#define regCB_COLOR6_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR6_FDCC_CONTROL                                                                       0x0378
+#define regCB_COLOR6_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR6_DCC_BASE                                                                           0x037f
+#define regCB_COLOR6_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR7_BASE                                                                               0x0381
+#define regCB_COLOR7_BASE_BASE_IDX                                                                      1
+#define regCB_COLOR7_VIEW                                                                               0x0384
+#define regCB_COLOR7_VIEW_BASE_IDX                                                                      1
+#define regCB_COLOR7_INFO                                                                               0x0385
+#define regCB_COLOR7_INFO_BASE_IDX                                                                      1
+#define regCB_COLOR7_ATTRIB                                                                             0x0386
+#define regCB_COLOR7_ATTRIB_BASE_IDX                                                                    1
+#define regCB_COLOR7_FDCC_CONTROL                                                                       0x0387
+#define regCB_COLOR7_FDCC_CONTROL_BASE_IDX                                                              1
+#define regCB_COLOR7_DCC_BASE                                                                           0x038e
+#define regCB_COLOR7_DCC_BASE_BASE_IDX                                                                  1
+#define regCB_COLOR0_BASE_EXT                                                                           0x0390
+#define regCB_COLOR0_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR1_BASE_EXT                                                                           0x0391
+#define regCB_COLOR1_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR2_BASE_EXT                                                                           0x0392
+#define regCB_COLOR2_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR3_BASE_EXT                                                                           0x0393
+#define regCB_COLOR3_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR4_BASE_EXT                                                                           0x0394
+#define regCB_COLOR4_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR5_BASE_EXT                                                                           0x0395
+#define regCB_COLOR5_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR6_BASE_EXT                                                                           0x0396
+#define regCB_COLOR6_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR7_BASE_EXT                                                                           0x0397
+#define regCB_COLOR7_BASE_EXT_BASE_IDX                                                                  1
+#define regCB_COLOR0_DCC_BASE_EXT                                                                       0x03a8
+#define regCB_COLOR0_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR1_DCC_BASE_EXT                                                                       0x03a9
+#define regCB_COLOR1_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR2_DCC_BASE_EXT                                                                       0x03aa
+#define regCB_COLOR2_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR3_DCC_BASE_EXT                                                                       0x03ab
+#define regCB_COLOR3_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR4_DCC_BASE_EXT                                                                       0x03ac
+#define regCB_COLOR4_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR5_DCC_BASE_EXT                                                                       0x03ad
+#define regCB_COLOR5_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR6_DCC_BASE_EXT                                                                       0x03ae
+#define regCB_COLOR6_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR7_DCC_BASE_EXT                                                                       0x03af
+#define regCB_COLOR7_DCC_BASE_EXT_BASE_IDX                                                              1
+#define regCB_COLOR0_ATTRIB2                                                                            0x03b0
+#define regCB_COLOR0_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR1_ATTRIB2                                                                            0x03b1
+#define regCB_COLOR1_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR2_ATTRIB2                                                                            0x03b2
+#define regCB_COLOR2_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR3_ATTRIB2                                                                            0x03b3
+#define regCB_COLOR3_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR4_ATTRIB2                                                                            0x03b4
+#define regCB_COLOR4_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR5_ATTRIB2                                                                            0x03b5
+#define regCB_COLOR5_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR6_ATTRIB2                                                                            0x03b6
+#define regCB_COLOR6_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR7_ATTRIB2                                                                            0x03b7
+#define regCB_COLOR7_ATTRIB2_BASE_IDX                                                                   1
+#define regCB_COLOR0_ATTRIB3                                                                            0x03b8
+#define regCB_COLOR0_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR1_ATTRIB3                                                                            0x03b9
+#define regCB_COLOR1_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR2_ATTRIB3                                                                            0x03ba
+#define regCB_COLOR2_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR3_ATTRIB3                                                                            0x03bb
+#define regCB_COLOR3_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR4_ATTRIB3                                                                            0x03bc
+#define regCB_COLOR4_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR5_ATTRIB3                                                                            0x03bd
+#define regCB_COLOR5_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR6_ATTRIB3                                                                            0x03be
+#define regCB_COLOR6_ATTRIB3_BASE_IDX                                                                   1
+#define regCB_COLOR7_ATTRIB3                                                                            0x03bf
+#define regCB_COLOR7_ATTRIB3_BASE_IDX                                                                   1
+
+
+// addressBlock: gc_pfvf_cpdec
+// base address: 0x2a000
+#define regCONFIG_RESERVED_REG0                                                                         0x0800
+#define regCONFIG_RESERVED_REG0_BASE_IDX                                                                1
+#define regCONFIG_RESERVED_REG1                                                                         0x0801
+#define regCONFIG_RESERVED_REG1_BASE_IDX                                                                1
+#define regCP_MEC_CNTL                                                                                  0x0802
+#define regCP_MEC_CNTL_BASE_IDX                                                                         1
+#define regCP_ME_CNTL                                                                                   0x0803
+#define regCP_ME_CNTL_BASE_IDX                                                                          1
+
+
+// addressBlock: gc_pfvf_grbmdec
+// base address: 0x2a400
+#define regGRBM_GFX_CNTL                                                                                0x0900
+#define regGRBM_GFX_CNTL_BASE_IDX                                                                       1
+#define regGRBM_NOWHERE                                                                                 0x0901
+#define regGRBM_NOWHERE_BASE_IDX                                                                        1
+
+
+// addressBlock: gc_pfvf_padec
+// base address: 0x2a500
+#define regPA_SC_VRS_SURFACE_CNTL                                                                       0x0940
+#define regPA_SC_VRS_SURFACE_CNTL_BASE_IDX                                                              1
+#define regPA_SC_ENHANCE                                                                                0x0941
+#define regPA_SC_ENHANCE_BASE_IDX                                                                       1
+#define regPA_SC_ENHANCE_1                                                                              0x0942
+#define regPA_SC_ENHANCE_1_BASE_IDX                                                                     1
+#define regPA_SC_ENHANCE_2                                                                              0x0943
+#define regPA_SC_ENHANCE_2_BASE_IDX                                                                     1
+#define regPA_SC_ENHANCE_3                                                                              0x0944
+#define regPA_SC_ENHANCE_3_BASE_IDX                                                                     1
+#define regPA_SC_BINNER_CNTL_OVERRIDE                                                                   0x0946
+#define regPA_SC_BINNER_CNTL_OVERRIDE_BASE_IDX                                                          1
+#define regPA_SC_PBB_OVERRIDE_FLAG                                                                      0x0947
+#define regPA_SC_PBB_OVERRIDE_FLAG_BASE_IDX                                                             1
+#define regPA_SC_DSM_CNTL                                                                               0x0948
+#define regPA_SC_DSM_CNTL_BASE_IDX                                                                      1
+#define regPA_SC_TILE_STEERING_CREST_OVERRIDE                                                           0x0949
+#define regPA_SC_TILE_STEERING_CREST_OVERRIDE_BASE_IDX                                                  1
+#define regPA_SC_FIFO_SIZE                                                                              0x094a
+#define regPA_SC_FIFO_SIZE_BASE_IDX                                                                     1
+#define regPA_SC_IF_FIFO_SIZE                                                                           0x094b
+#define regPA_SC_IF_FIFO_SIZE_BASE_IDX                                                                  1
+#define regPA_SC_PACKER_WAVE_ID_CNTL                                                                    0x094c
+#define regPA_SC_PACKER_WAVE_ID_CNTL_BASE_IDX                                                           1
+#define regPA_SC_ATM_CNTL                                                                               0x094d
+#define regPA_SC_ATM_CNTL_BASE_IDX                                                                      1
+#define regPA_SC_PKR_WAVE_TABLE_CNTL                                                                    0x094e
+#define regPA_SC_PKR_WAVE_TABLE_CNTL_BASE_IDX                                                           1
+#define regPA_SC_FORCE_EOV_MAX_CNTS                                                                     0x094f
+#define regPA_SC_FORCE_EOV_MAX_CNTS_BASE_IDX                                                            1
+#define regPA_SC_BINNER_EVENT_CNTL_0                                                                    0x0950
+#define regPA_SC_BINNER_EVENT_CNTL_0_BASE_IDX                                                           1
+#define regPA_SC_BINNER_EVENT_CNTL_1                                                                    0x0951
+#define regPA_SC_BINNER_EVENT_CNTL_1_BASE_IDX                                                           1
+#define regPA_SC_BINNER_EVENT_CNTL_2                                                                    0x0952
+#define regPA_SC_BINNER_EVENT_CNTL_2_BASE_IDX                                                           1
+#define regPA_SC_BINNER_EVENT_CNTL_3                                                                    0x0953
+#define regPA_SC_BINNER_EVENT_CNTL_3_BASE_IDX                                                           1
+#define regPA_SC_BINNER_TIMEOUT_COUNTER                                                                 0x0954
+#define regPA_SC_BINNER_TIMEOUT_COUNTER_BASE_IDX                                                        1
+#define regPA_SC_BINNER_PERF_CNTL_0                                                                     0x0955
+#define regPA_SC_BINNER_PERF_CNTL_0_BASE_IDX                                                            1
+#define regPA_SC_BINNER_PERF_CNTL_1                                                                     0x0956
+#define regPA_SC_BINNER_PERF_CNTL_1_BASE_IDX                                                            1
+#define regPA_SC_BINNER_PERF_CNTL_2                                                                     0x0957
+#define regPA_SC_BINNER_PERF_CNTL_2_BASE_IDX                                                            1
+#define regPA_SC_BINNER_PERF_CNTL_3                                                                     0x0958
+#define regPA_SC_BINNER_PERF_CNTL_3_BASE_IDX                                                            1
+#define regPA_SC_P3D_TRAP_SCREEN_HV_LOCK                                                                0x095b
+#define regPA_SC_P3D_TRAP_SCREEN_HV_LOCK_BASE_IDX                                                       1
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_LOCK                                                               0x095c
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_LOCK_BASE_IDX                                                      1
+#define regPA_SC_TRAP_SCREEN_HV_LOCK                                                                    0x095d
+#define regPA_SC_TRAP_SCREEN_HV_LOCK_BASE_IDX                                                           1
+#define regPA_PH_INTERFACE_FIFO_SIZE                                                                    0x095e
+#define regPA_PH_INTERFACE_FIFO_SIZE_BASE_IDX                                                           1
+#define regPA_PH_ENHANCE                                                                                0x095f
+#define regPA_PH_ENHANCE_BASE_IDX                                                                       1
+#define regPA_SC_VRS_SURFACE_CNTL_1                                                                     0x0960
+#define regPA_SC_VRS_SURFACE_CNTL_1_BASE_IDX                                                            1
+
+
+// addressBlock: gc_pfvf_sqdec
+// base address: 0x2a780
+#define regSQ_RUNTIME_CONFIG                                                                            0x09e0
+#define regSQ_RUNTIME_CONFIG_BASE_IDX                                                                   1
+#define regSQ_DEBUG_STS_GLOBAL                                                                          0x09e1
+#define regSQ_DEBUG_STS_GLOBAL_BASE_IDX                                                                 1
+#define regSQ_DEBUG_STS_GLOBAL2                                                                         0x09e2
+#define regSQ_DEBUG_STS_GLOBAL2_BASE_IDX                                                                1
+#define regSH_MEM_BASES                                                                                 0x09e3
+#define regSH_MEM_BASES_BASE_IDX                                                                        1
+#define regSH_MEM_CONFIG                                                                                0x09e4
+#define regSH_MEM_CONFIG_BASE_IDX                                                                       1
+#define regSQ_DEBUG                                                                                     0x09e5
+#define regSQ_DEBUG_BASE_IDX                                                                            1
+#define regSQ_SHADER_TBA_LO                                                                             0x09e6
+#define regSQ_SHADER_TBA_LO_BASE_IDX                                                                    1
+#define regSQ_SHADER_TBA_HI                                                                             0x09e7
+#define regSQ_SHADER_TBA_HI_BASE_IDX                                                                    1
+#define regSQ_SHADER_TMA_LO                                                                             0x09e8
+#define regSQ_SHADER_TMA_LO_BASE_IDX                                                                    1
+#define regSQ_SHADER_TMA_HI                                                                             0x09e9
+#define regSQ_SHADER_TMA_HI_BASE_IDX                                                                    1
+
+
+// addressBlock: gc_pfonly_cpdec
+// base address: 0x2e000
+#define regCP_DEBUG_2                                                                                   0x1800
+#define regCP_DEBUG_2_BASE_IDX                                                                          1
+#define regCP_FETCHER_SOURCE                                                                            0x1801
+#define regCP_FETCHER_SOURCE_BASE_IDX                                                                   1
+#define regCP_DFY_CNTL                                                                                  0x1804
+#define regCP_DFY_CNTL_BASE_IDX                                                                         1
+#define regCP_DFY_STAT                                                                                  0x1805
+#define regCP_DFY_STAT_BASE_IDX                                                                         1
+#define regCP_DFY_ADDR_HI                                                                               0x1806
+#define regCP_DFY_ADDR_HI_BASE_IDX                                                                      1
+#define regCP_DFY_ADDR_LO                                                                               0x1807
+#define regCP_DFY_ADDR_LO_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_0                                                                                0x1808
+#define regCP_DFY_DATA_0_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_1                                                                                0x1809
+#define regCP_DFY_DATA_1_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_2                                                                                0x180a
+#define regCP_DFY_DATA_2_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_3                                                                                0x180b
+#define regCP_DFY_DATA_3_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_4                                                                                0x180c
+#define regCP_DFY_DATA_4_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_5                                                                                0x180d
+#define regCP_DFY_DATA_5_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_6                                                                                0x180e
+#define regCP_DFY_DATA_6_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_7                                                                                0x180f
+#define regCP_DFY_DATA_7_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_8                                                                                0x1810
+#define regCP_DFY_DATA_8_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_9                                                                                0x1811
+#define regCP_DFY_DATA_9_BASE_IDX                                                                       1
+#define regCP_DFY_DATA_10                                                                               0x1812
+#define regCP_DFY_DATA_10_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_11                                                                               0x1813
+#define regCP_DFY_DATA_11_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_12                                                                               0x1814
+#define regCP_DFY_DATA_12_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_13                                                                               0x1815
+#define regCP_DFY_DATA_13_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_14                                                                               0x1816
+#define regCP_DFY_DATA_14_BASE_IDX                                                                      1
+#define regCP_DFY_DATA_15                                                                               0x1817
+#define regCP_DFY_DATA_15_BASE_IDX                                                                      1
+#define regCP_DFY_CMD                                                                                   0x1818
+#define regCP_DFY_CMD_BASE_IDX                                                                          1
+
+
+// addressBlock: gc_pfonly_cpphqddec
+// base address: 0x2e080
+#define regCP_HPD_MES_ROQ_OFFSETS                                                                       0x1821
+#define regCP_HPD_MES_ROQ_OFFSETS_BASE_IDX                                                              1
+#define regCP_HPD_ROQ_OFFSETS                                                                           0x1821
+#define regCP_HPD_ROQ_OFFSETS_BASE_IDX                                                                  1
+#define regCP_HPD_STATUS0                                                                               0x1822
+#define regCP_HPD_STATUS0_BASE_IDX                                                                      1
+
+
+// addressBlock: gc_pfonly_didtdec
+// base address: 0x2e400
+#define regDIDT_INDEX_AUTO_INCR_EN                                                                      0x1900
+#define regDIDT_INDEX_AUTO_INCR_EN_BASE_IDX                                                             1
+#define regDIDT_EDC_CTRL                                                                                0x1901
+#define regDIDT_EDC_CTRL_BASE_IDX                                                                       1
+#define regDIDT_EDC_THROTTLE_CTRL                                                                       0x1902
+#define regDIDT_EDC_THROTTLE_CTRL_BASE_IDX                                                              1
+#define regDIDT_EDC_THRESHOLD                                                                           0x1903
+#define regDIDT_EDC_THRESHOLD_BASE_IDX                                                                  1
+#define regDIDT_EDC_STALL_PATTERN_1_2                                                                   0x1904
+#define regDIDT_EDC_STALL_PATTERN_1_2_BASE_IDX                                                          1
+#define regDIDT_EDC_STALL_PATTERN_3_4                                                                   0x1905
+#define regDIDT_EDC_STALL_PATTERN_3_4_BASE_IDX                                                          1
+#define regDIDT_EDC_STALL_PATTERN_5_6                                                                   0x1906
+#define regDIDT_EDC_STALL_PATTERN_5_6_BASE_IDX                                                          1
+#define regDIDT_EDC_STALL_PATTERN_7                                                                     0x1907
+#define regDIDT_EDC_STALL_PATTERN_7_BASE_IDX                                                            1
+#define regDIDT_EDC_STATUS                                                                              0x1908
+#define regDIDT_EDC_STATUS_BASE_IDX                                                                     1
+#define regDIDT_EDC_DYNAMIC_THRESHOLD_RO                                                                0x1909
+#define regDIDT_EDC_DYNAMIC_THRESHOLD_RO_BASE_IDX                                                       1
+#define regDIDT_EDC_OVERFLOW                                                                            0x190a
+#define regDIDT_EDC_OVERFLOW_BASE_IDX                                                                   1
+#define regDIDT_EDC_ROLLING_POWER_DELTA                                                                 0x190b
+#define regDIDT_EDC_ROLLING_POWER_DELTA_BASE_IDX                                                        1
+#define regDIDT_IND_INDEX                                                                               0x190c
+#define regDIDT_IND_INDEX_BASE_IDX                                                                      1
+#define regDIDT_IND_DATA                                                                                0x190d
+#define regDIDT_IND_DATA_BASE_IDX                                                                       1
+
+
+// addressBlock: gc_pfonly_spidec
+// base address: 0x2e500
+#define regSPI_CDBG_SYS_GFX                                                                             0x1940
+#define regSPI_CDBG_SYS_GFX_BASE_IDX                                                                    1
+#define regSPI_CDBG_SYS_HP3D                                                                            0x1941
+#define regSPI_CDBG_SYS_HP3D_BASE_IDX                                                                   1
+#define regSPI_CDBG_SYS_CS0                                                                             0x1942
+#define regSPI_CDBG_SYS_CS0_BASE_IDX                                                                    1
+#define regSPI_GDBG_WAVE_CNTL                                                                           0x1943
+#define regSPI_GDBG_WAVE_CNTL_BASE_IDX                                                                  1
+#define regSPI_GDBG_TRAP_CONFIG                                                                         0x1944
+#define regSPI_GDBG_TRAP_CONFIG_BASE_IDX                                                                1
+#define regSPI_GDBG_WAVE_CNTL3                                                                          0x1945
+#define regSPI_GDBG_WAVE_CNTL3_BASE_IDX                                                                 1
+#define regSPI_RESET_DEBUG                                                                              0x1946
+#define regSPI_RESET_DEBUG_BASE_IDX                                                                     1
+#define regSPI_ARB_CNTL_0                                                                               0x1949
+#define regSPI_ARB_CNTL_0_BASE_IDX                                                                      1
+#define regSPI_FEATURE_CTRL                                                                             0x194a
+#define regSPI_FEATURE_CTRL_BASE_IDX                                                                    1
+#define regSPI_SHADER_RSRC_LIMIT_CTRL                                                                   0x194b
+#define regSPI_SHADER_RSRC_LIMIT_CTRL_BASE_IDX                                                          1
+#define regSPI_COMPUTE_WF_CTX_SAVE_STATUS                                                               0x194e
+#define regSPI_COMPUTE_WF_CTX_SAVE_STATUS_BASE_IDX                                                      1
+
+
+// addressBlock: gc_pfonly_tcpdec
+// base address: 0x2e680
+#define regTCP_INVALIDATE                                                                               0x19a0
+#define regTCP_INVALIDATE_BASE_IDX                                                                      1
+#define regTCP_STATUS                                                                                   0x19a1
+#define regTCP_STATUS_BASE_IDX                                                                          1
+#define regTCP_CNTL                                                                                     0x19a2
+#define regTCP_CNTL_BASE_IDX                                                                            1
+#define regTCP_CNTL2                                                                                    0x19a3
+#define regTCP_CNTL2_BASE_IDX                                                                           1
+#define regTCP_CREDIT                                                                                   0x19a4
+#define regTCP_CREDIT_BASE_IDX                                                                          1
+
+
+// addressBlock: gc_pfonly_gdsdec
+// base address: 0x2e6c0
+#define regGDS_ENHANCE2                                                                                 0x19b0
+#define regGDS_ENHANCE2_BASE_IDX                                                                        1
+#define regGDS_OA_CGPG_RESTORE                                                                          0x19b1
+#define regGDS_OA_CGPG_RESTORE_BASE_IDX                                                                 1
+
+
+// addressBlock: gc_pfonly_utcl1dec
+// base address: 0x2e600
+#define regUTCL1_CTRL_0                                                                                 0x1980
+#define regUTCL1_CTRL_0_BASE_IDX                                                                        1
+#define regUTCL1_UTCL0_INVREQ_DISABLE                                                                   0x1984
+#define regUTCL1_UTCL0_INVREQ_DISABLE_BASE_IDX                                                          1
+#define regUTCL1_CTRL_2                                                                                 0x1985
+#define regUTCL1_CTRL_2_BASE_IDX                                                                        1
+#define regUTCL1_FIFO_SIZING                                                                            0x1986
+#define regUTCL1_FIFO_SIZING_BASE_IDX                                                                   1
+#define regGCRD_SA0_TARGETS_DISABLE                                                                     0x1987
+#define regGCRD_SA0_TARGETS_DISABLE_BASE_IDX                                                            1
+#define regGCRD_SA1_TARGETS_DISABLE                                                                     0x1989
+#define regGCRD_SA1_TARGETS_DISABLE_BASE_IDX                                                            1
+#define regGCRD_CREDIT_SAFE                                                                             0x198a
+#define regGCRD_CREDIT_SAFE_BASE_IDX                                                                    1
+
+
+// addressBlock: gc_pfonly_pmmdec
+// base address: 0x2e640
+#define regGCR_GENERAL_CNTL                                                                             0x1990
+#define regGCR_GENERAL_CNTL_BASE_IDX                                                                    1
+#define regGCR_TARGET_DISABLE                                                                           0x1991
+#define regGCR_TARGET_DISABLE_BASE_IDX                                                                  1
+#define regGCR_CMD_STATUS                                                                               0x1992
+#define regGCR_CMD_STATUS_BASE_IDX                                                                      1
+#define regGCR_SPARE                                                                                    0x1993
+#define regGCR_SPARE_BASE_IDX                                                                           1
+#define regPMM_CNTL2                                                                                    0x1999
+#define regPMM_CNTL2_BASE_IDX                                                                           1
+
+
+// addressBlock: gc_pfonly_gccacdec
+// base address: 0x2eb40
+#define regGC_CAC_CTRL_1                                                                                0x1ad0
+#define regGC_CAC_CTRL_1_BASE_IDX                                                                       1
+#define regGC_CAC_CTRL_2                                                                                0x1ad1
+#define regGC_CAC_CTRL_2_BASE_IDX                                                                       1
+#define regGC_CAC_AGGR_LOWER                                                                            0x1ad2
+#define regGC_CAC_AGGR_LOWER_BASE_IDX                                                                   1
+#define regGC_CAC_AGGR_UPPER                                                                            0x1ad3
+#define regGC_CAC_AGGR_UPPER_BASE_IDX                                                                   1
+#define regSE0_CAC_AGGR_LOWER                                                                           0x1ad4
+#define regSE0_CAC_AGGR_LOWER_BASE_IDX                                                                  1
+#define regSE0_CAC_AGGR_UPPER                                                                           0x1ad5
+#define regSE0_CAC_AGGR_UPPER_BASE_IDX                                                                  1
+#define regSE1_CAC_AGGR_LOWER                                                                           0x1ad6
+#define regSE1_CAC_AGGR_LOWER_BASE_IDX                                                                  1
+#define regSE1_CAC_AGGR_UPPER                                                                           0x1ad7
+#define regSE1_CAC_AGGR_UPPER_BASE_IDX                                                                  1
+#define regSE2_CAC_AGGR_LOWER                                                                           0x1ad8
+#define regSE2_CAC_AGGR_LOWER_BASE_IDX                                                                  1
+#define regSE2_CAC_AGGR_UPPER                                                                           0x1ad9
+#define regSE2_CAC_AGGR_UPPER_BASE_IDX                                                                  1
+#define regGC_CAC_AGGR_GFXCLK_CYCLE                                                                     0x1ae4
+#define regGC_CAC_AGGR_GFXCLK_CYCLE_BASE_IDX                                                            1
+#define regSE0_CAC_AGGR_GFXCLK_CYCLE                                                                    0x1ae5
+#define regSE0_CAC_AGGR_GFXCLK_CYCLE_BASE_IDX                                                           1
+#define regSE1_CAC_AGGR_GFXCLK_CYCLE                                                                    0x1ae6
+#define regSE1_CAC_AGGR_GFXCLK_CYCLE_BASE_IDX                                                           1
+#define regSE2_CAC_AGGR_GFXCLK_CYCLE                                                                    0x1ae7
+#define regSE2_CAC_AGGR_GFXCLK_CYCLE_BASE_IDX                                                           1
+#define regGC_EDC_CTRL                                                                                  0x1aed
+#define regGC_EDC_CTRL_BASE_IDX                                                                         1
+#define regGC_EDC_THRESHOLD                                                                             0x1aee
+#define regGC_EDC_THRESHOLD_BASE_IDX                                                                    1
+#define regGC_EDC_STRETCH_CTRL                                                                          0x1aef
+#define regGC_EDC_STRETCH_CTRL_BASE_IDX                                                                 1
+#define regGC_EDC_STRETCH_THRESHOLD                                                                     0x1af0
+#define regGC_EDC_STRETCH_THRESHOLD_BASE_IDX                                                            1
+#define regEDC_HYSTERESIS_CNTL                                                                          0x1af1
+#define regEDC_HYSTERESIS_CNTL_BASE_IDX                                                                 1
+#define regGC_THROTTLE_CTRL                                                                             0x1af2
+#define regGC_THROTTLE_CTRL_BASE_IDX                                                                    1
+#define regGC_THROTTLE_CTRL1                                                                            0x1af3
+#define regGC_THROTTLE_CTRL1_BASE_IDX                                                                   1
+#define regPCC_STALL_PATTERN_CTRL                                                                       0x1af4
+#define regPCC_STALL_PATTERN_CTRL_BASE_IDX                                                              1
+#define regPWRBRK_STALL_PATTERN_CTRL                                                                    0x1af5
+#define regPWRBRK_STALL_PATTERN_CTRL_BASE_IDX                                                           1
+#define regPCC_STALL_PATTERN_1_2                                                                        0x1af6
+#define regPCC_STALL_PATTERN_1_2_BASE_IDX                                                               1
+#define regPCC_STALL_PATTERN_3_4                                                                        0x1af7
+#define regPCC_STALL_PATTERN_3_4_BASE_IDX                                                               1
+#define regPCC_STALL_PATTERN_5_6                                                                        0x1af8
+#define regPCC_STALL_PATTERN_5_6_BASE_IDX                                                               1
+#define regPCC_STALL_PATTERN_7                                                                          0x1af9
+#define regPCC_STALL_PATTERN_7_BASE_IDX                                                                 1
+#define regPWRBRK_STALL_PATTERN_1_2                                                                     0x1afa
+#define regPWRBRK_STALL_PATTERN_1_2_BASE_IDX                                                            1
+#define regPWRBRK_STALL_PATTERN_3_4                                                                     0x1afb
+#define regPWRBRK_STALL_PATTERN_3_4_BASE_IDX                                                            1
+#define regPWRBRK_STALL_PATTERN_5_6                                                                     0x1afc
+#define regPWRBRK_STALL_PATTERN_5_6_BASE_IDX                                                            1
+#define regPWRBRK_STALL_PATTERN_7                                                                       0x1afd
+#define regPWRBRK_STALL_PATTERN_7_BASE_IDX                                                              1
+#define regDIDT_STALL_PATTERN_CTRL                                                                      0x1afe
+#define regDIDT_STALL_PATTERN_CTRL_BASE_IDX                                                             1
+#define regDIDT_STALL_PATTERN_1_2                                                                       0x1aff
+#define regDIDT_STALL_PATTERN_1_2_BASE_IDX                                                              1
+#define regDIDT_STALL_PATTERN_3_4                                                                       0x1b00
+#define regDIDT_STALL_PATTERN_3_4_BASE_IDX                                                              1
+#define regDIDT_STALL_PATTERN_5_6                                                                       0x1b01
+#define regDIDT_STALL_PATTERN_5_6_BASE_IDX                                                              1
+#define regDIDT_STALL_PATTERN_7                                                                         0x1b02
+#define regDIDT_STALL_PATTERN_7_BASE_IDX                                                                1
+#define regPCC_PWRBRK_HYSTERESIS_CTRL                                                                   0x1b03
+#define regPCC_PWRBRK_HYSTERESIS_CTRL_BASE_IDX                                                          1
+#define regEDC_STRETCH_PERF_COUNTER                                                                     0x1b04
+#define regEDC_STRETCH_PERF_COUNTER_BASE_IDX                                                            1
+#define regEDC_UNSTRETCH_PERF_COUNTER                                                                   0x1b05
+#define regEDC_UNSTRETCH_PERF_COUNTER_BASE_IDX                                                          1
+#define regEDC_STRETCH_NUM_PERF_COUNTER                                                                 0x1b06
+#define regEDC_STRETCH_NUM_PERF_COUNTER_BASE_IDX                                                        1
+#define regGC_EDC_STATUS                                                                                0x1b07
+#define regGC_EDC_STATUS_BASE_IDX                                                                       1
+#define regGC_EDC_OVERFLOW                                                                              0x1b08
+#define regGC_EDC_OVERFLOW_BASE_IDX                                                                     1
+#define regGC_EDC_ROLLING_POWER_DELTA                                                                   0x1b09
+#define regGC_EDC_ROLLING_POWER_DELTA_BASE_IDX                                                          1
+#define regGC_THROTTLE_STATUS                                                                           0x1b0a
+#define regGC_THROTTLE_STATUS_BASE_IDX                                                                  1
+#define regEDC_PERF_COUNTER                                                                             0x1b0b
+#define regEDC_PERF_COUNTER_BASE_IDX                                                                    1
+#define regPCC_PERF_COUNTER                                                                             0x1b0c
+#define regPCC_PERF_COUNTER_BASE_IDX                                                                    1
+#define regPWRBRK_PERF_COUNTER                                                                          0x1b0d
+#define regPWRBRK_PERF_COUNTER_BASE_IDX                                                                 1
+#define regEDC_HYSTERESIS_STAT                                                                          0x1b0e
+#define regEDC_HYSTERESIS_STAT_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_CP_0                                                                           0x1b10
+#define regGC_CAC_WEIGHT_CP_0_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_CP_1                                                                           0x1b11
+#define regGC_CAC_WEIGHT_CP_1_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_EA_0                                                                           0x1b12
+#define regGC_CAC_WEIGHT_EA_0_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_EA_1                                                                           0x1b13
+#define regGC_CAC_WEIGHT_EA_1_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_EA_2                                                                           0x1b14
+#define regGC_CAC_WEIGHT_EA_2_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_UTCL2_ROUTER_0                                                                 0x1b15
+#define regGC_CAC_WEIGHT_UTCL2_ROUTER_0_BASE_IDX                                                        1
+#define regGC_CAC_WEIGHT_UTCL2_ROUTER_1                                                                 0x1b16
+#define regGC_CAC_WEIGHT_UTCL2_ROUTER_1_BASE_IDX                                                        1
+#define regGC_CAC_WEIGHT_UTCL2_ROUTER_2                                                                 0x1b17
+#define regGC_CAC_WEIGHT_UTCL2_ROUTER_2_BASE_IDX                                                        1
+#define regGC_CAC_WEIGHT_UTCL2_ROUTER_3                                                                 0x1b18
+#define regGC_CAC_WEIGHT_UTCL2_ROUTER_3_BASE_IDX                                                        1
+#define regGC_CAC_WEIGHT_UTCL2_ROUTER_4                                                                 0x1b19
+#define regGC_CAC_WEIGHT_UTCL2_ROUTER_4_BASE_IDX                                                        1
+#define regGC_CAC_WEIGHT_UTCL2_VML2_0                                                                   0x1b1a
+#define regGC_CAC_WEIGHT_UTCL2_VML2_0_BASE_IDX                                                          1
+#define regGC_CAC_WEIGHT_UTCL2_VML2_1                                                                   0x1b1b
+#define regGC_CAC_WEIGHT_UTCL2_VML2_1_BASE_IDX                                                          1
+#define regGC_CAC_WEIGHT_UTCL2_VML2_2                                                                   0x1b1c
+#define regGC_CAC_WEIGHT_UTCL2_VML2_2_BASE_IDX                                                          1
+#define regGC_CAC_WEIGHT_UTCL2_WALKER_0                                                                 0x1b1d
+#define regGC_CAC_WEIGHT_UTCL2_WALKER_0_BASE_IDX                                                        1
+#define regGC_CAC_WEIGHT_UTCL2_WALKER_1                                                                 0x1b1e
+#define regGC_CAC_WEIGHT_UTCL2_WALKER_1_BASE_IDX                                                        1
+#define regGC_CAC_WEIGHT_UTCL2_WALKER_2                                                                 0x1b1f
+#define regGC_CAC_WEIGHT_UTCL2_WALKER_2_BASE_IDX                                                        1
+#define regGC_CAC_WEIGHT_GDS_0                                                                          0x1b20
+#define regGC_CAC_WEIGHT_GDS_0_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_GDS_1                                                                          0x1b21
+#define regGC_CAC_WEIGHT_GDS_1_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_GDS_2                                                                          0x1b22
+#define regGC_CAC_WEIGHT_GDS_2_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_GE_0                                                                           0x1b23
+#define regGC_CAC_WEIGHT_GE_0_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_GE_1                                                                           0x1b24
+#define regGC_CAC_WEIGHT_GE_1_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_GE_2                                                                           0x1b25
+#define regGC_CAC_WEIGHT_GE_2_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_GE_3                                                                           0x1b26
+#define regGC_CAC_WEIGHT_GE_3_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_PMM_0                                                                          0x1b2e
+#define regGC_CAC_WEIGHT_PMM_0_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_GL2C_0                                                                         0x1b2f
+#define regGC_CAC_WEIGHT_GL2C_0_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_GL2C_1                                                                         0x1b30
+#define regGC_CAC_WEIGHT_GL2C_1_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_GL2C_2                                                                         0x1b31
+#define regGC_CAC_WEIGHT_GL2C_2_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_PH_0                                                                           0x1b32
+#define regGC_CAC_WEIGHT_PH_0_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_PH_1                                                                           0x1b33
+#define regGC_CAC_WEIGHT_PH_1_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_PH_2                                                                           0x1b34
+#define regGC_CAC_WEIGHT_PH_2_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_PH_3                                                                           0x1b35
+#define regGC_CAC_WEIGHT_PH_3_BASE_IDX                                                                  1
+#define regGC_CAC_WEIGHT_SDMA_0                                                                         0x1b36
+#define regGC_CAC_WEIGHT_SDMA_0_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_1                                                                         0x1b37
+#define regGC_CAC_WEIGHT_SDMA_1_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_2                                                                         0x1b38
+#define regGC_CAC_WEIGHT_SDMA_2_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_3                                                                         0x1b39
+#define regGC_CAC_WEIGHT_SDMA_3_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_4                                                                         0x1b3a
+#define regGC_CAC_WEIGHT_SDMA_4_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_SDMA_5                                                                         0x1b3b
+#define regGC_CAC_WEIGHT_SDMA_5_BASE_IDX                                                                1
+#define regGC_CAC_WEIGHT_CHC_0                                                                          0x1b3c
+#define regGC_CAC_WEIGHT_CHC_0_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_CHC_1                                                                          0x1b3d
+#define regGC_CAC_WEIGHT_CHC_1_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_GUS_0                                                                          0x1b3e
+#define regGC_CAC_WEIGHT_GUS_0_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_GUS_1                                                                          0x1b3f
+#define regGC_CAC_WEIGHT_GUS_1_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_RLC_0                                                                          0x1b40
+#define regGC_CAC_WEIGHT_RLC_0_BASE_IDX                                                                 1
+#define regGC_CAC_WEIGHT_GRBM_0                                                                         0x1b44
+#define regGC_CAC_WEIGHT_GRBM_0_BASE_IDX                                                                1
+#define regGC_EDC_CLK_MONITOR_CTRL                                                                      0x1b56
+#define regGC_EDC_CLK_MONITOR_CTRL_BASE_IDX                                                             1
+#define regGC_CAC_IND_INDEX                                                                             0x1b58
+#define regGC_CAC_IND_INDEX_BASE_IDX                                                                    1
+#define regGC_CAC_IND_DATA                                                                              0x1b59
+#define regGC_CAC_IND_DATA_BASE_IDX                                                                     1
+#define regSE_CAC_CTRL_1                                                                                0x1b70
+#define regSE_CAC_CTRL_1_BASE_IDX                                                                       1
+#define regSE_CAC_CTRL_2                                                                                0x1b71
+#define regSE_CAC_CTRL_2_BASE_IDX                                                                       1
+#define regSE_CAC_WEIGHT_TA_0                                                                           0x1b72
+#define regSE_CAC_WEIGHT_TA_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_TD_0                                                                           0x1b73
+#define regSE_CAC_WEIGHT_TD_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_TD_1                                                                           0x1b74
+#define regSE_CAC_WEIGHT_TD_1_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_TD_2                                                                           0x1b75
+#define regSE_CAC_WEIGHT_TD_2_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_TD_3                                                                           0x1b76
+#define regSE_CAC_WEIGHT_TD_3_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_TD_4                                                                           0x1b77
+#define regSE_CAC_WEIGHT_TD_4_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_TD_5                                                                           0x1b78
+#define regSE_CAC_WEIGHT_TD_5_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_TCP_0                                                                          0x1b79
+#define regSE_CAC_WEIGHT_TCP_0_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_1                                                                          0x1b7a
+#define regSE_CAC_WEIGHT_TCP_1_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_2                                                                          0x1b7b
+#define regSE_CAC_WEIGHT_TCP_2_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_TCP_3                                                                          0x1b7c
+#define regSE_CAC_WEIGHT_TCP_3_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_SQ_0                                                                           0x1b7d
+#define regSE_CAC_WEIGHT_SQ_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SQ_1                                                                           0x1b7e
+#define regSE_CAC_WEIGHT_SQ_1_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SQ_2                                                                           0x1b7f
+#define regSE_CAC_WEIGHT_SQ_2_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SP_0                                                                           0x1b80
+#define regSE_CAC_WEIGHT_SP_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SP_1                                                                           0x1b81
+#define regSE_CAC_WEIGHT_SP_1_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_LDS_0                                                                          0x1b82
+#define regSE_CAC_WEIGHT_LDS_0_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_LDS_1                                                                          0x1b83
+#define regSE_CAC_WEIGHT_LDS_1_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_LDS_2                                                                          0x1b84
+#define regSE_CAC_WEIGHT_LDS_2_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_LDS_3                                                                          0x1b85
+#define regSE_CAC_WEIGHT_LDS_3_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_SQC_0                                                                          0x1b87
+#define regSE_CAC_WEIGHT_SQC_0_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_SQC_1                                                                          0x1b88
+#define regSE_CAC_WEIGHT_SQC_1_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_CU_0                                                                           0x1b89
+#define regSE_CAC_WEIGHT_CU_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_BCI_0                                                                          0x1b8a
+#define regSE_CAC_WEIGHT_BCI_0_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_CB_0                                                                           0x1b8b
+#define regSE_CAC_WEIGHT_CB_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_CB_1                                                                           0x1b8c
+#define regSE_CAC_WEIGHT_CB_1_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_CB_2                                                                           0x1b8d
+#define regSE_CAC_WEIGHT_CB_2_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_CB_3                                                                           0x1b8e
+#define regSE_CAC_WEIGHT_CB_3_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_CB_4                                                                           0x1b8f
+#define regSE_CAC_WEIGHT_CB_4_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_CB_5                                                                           0x1b90
+#define regSE_CAC_WEIGHT_CB_5_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_CB_6                                                                           0x1b91
+#define regSE_CAC_WEIGHT_CB_6_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_CB_7                                                                           0x1b92
+#define regSE_CAC_WEIGHT_CB_7_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_CB_8                                                                           0x1b93
+#define regSE_CAC_WEIGHT_CB_8_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_CB_9                                                                           0x1b94
+#define regSE_CAC_WEIGHT_CB_9_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_CB_10                                                                          0x1b95
+#define regSE_CAC_WEIGHT_CB_10_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_CB_11                                                                          0x1b96
+#define regSE_CAC_WEIGHT_CB_11_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_DB_0                                                                           0x1b97
+#define regSE_CAC_WEIGHT_DB_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_DB_1                                                                           0x1b98
+#define regSE_CAC_WEIGHT_DB_1_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_DB_2                                                                           0x1b99
+#define regSE_CAC_WEIGHT_DB_2_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_DB_3                                                                           0x1b9a
+#define regSE_CAC_WEIGHT_DB_3_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_DB_4                                                                           0x1b9b
+#define regSE_CAC_WEIGHT_DB_4_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_RMI_0                                                                          0x1b9c
+#define regSE_CAC_WEIGHT_RMI_0_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_RMI_1                                                                          0x1b9d
+#define regSE_CAC_WEIGHT_RMI_1_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_SX_0                                                                           0x1b9e
+#define regSE_CAC_WEIGHT_SX_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SXRB_0                                                                         0x1b9f
+#define regSE_CAC_WEIGHT_SXRB_0_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_UTCL1_0                                                                        0x1ba0
+#define regSE_CAC_WEIGHT_UTCL1_0_BASE_IDX                                                               1
+#define regSE_CAC_WEIGHT_GL1C_0                                                                         0x1ba1
+#define regSE_CAC_WEIGHT_GL1C_0_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_GL1C_1                                                                         0x1ba2
+#define regSE_CAC_WEIGHT_GL1C_1_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_GL1C_2                                                                         0x1ba3
+#define regSE_CAC_WEIGHT_GL1C_2_BASE_IDX                                                                1
+#define regSE_CAC_WEIGHT_SPI_0                                                                          0x1ba4
+#define regSE_CAC_WEIGHT_SPI_0_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_SPI_1                                                                          0x1ba5
+#define regSE_CAC_WEIGHT_SPI_1_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_SPI_2                                                                          0x1ba6
+#define regSE_CAC_WEIGHT_SPI_2_BASE_IDX                                                                 1
+#define regSE_CAC_WEIGHT_PC_0                                                                           0x1ba7
+#define regSE_CAC_WEIGHT_PC_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_PA_0                                                                           0x1ba8
+#define regSE_CAC_WEIGHT_PA_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_PA_1                                                                           0x1ba9
+#define regSE_CAC_WEIGHT_PA_1_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_PA_2                                                                           0x1baa
+#define regSE_CAC_WEIGHT_PA_2_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_PA_3                                                                           0x1bab
+#define regSE_CAC_WEIGHT_PA_3_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SC_0                                                                           0x1bac
+#define regSE_CAC_WEIGHT_SC_0_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SC_1                                                                           0x1bad
+#define regSE_CAC_WEIGHT_SC_1_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SC_2                                                                           0x1bae
+#define regSE_CAC_WEIGHT_SC_2_BASE_IDX                                                                  1
+#define regSE_CAC_WEIGHT_SC_3                                                                           0x1baf
+#define regSE_CAC_WEIGHT_SC_3_BASE_IDX                                                                  1
+#define regSE_CAC_WINDOW_AGGR_VALUE                                                                     0x1bb0
+#define regSE_CAC_WINDOW_AGGR_VALUE_BASE_IDX                                                            1
+#define regSE_CAC_WINDOW_GFXCLK_CYCLE                                                                   0x1bb1
+#define regSE_CAC_WINDOW_GFXCLK_CYCLE_BASE_IDX                                                          1
+#define regSE_CAC_IND_INDEX                                                                             0x1bce
+#define regSE_CAC_IND_INDEX_BASE_IDX                                                                    1
+#define regSE_CAC_IND_DATA                                                                              0x1bcf
+#define regSE_CAC_IND_DATA_BASE_IDX                                                                     1
+
+
+// addressBlock: gc_pfonly2_spidec
+// base address: 0x2f000
+#define regSPI_RESOURCE_RESERVE_CU_0                                                                    0x1c00
+#define regSPI_RESOURCE_RESERVE_CU_0_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_1                                                                    0x1c01
+#define regSPI_RESOURCE_RESERVE_CU_1_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_2                                                                    0x1c02
+#define regSPI_RESOURCE_RESERVE_CU_2_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_3                                                                    0x1c03
+#define regSPI_RESOURCE_RESERVE_CU_3_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_4                                                                    0x1c04
+#define regSPI_RESOURCE_RESERVE_CU_4_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_5                                                                    0x1c05
+#define regSPI_RESOURCE_RESERVE_CU_5_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_6                                                                    0x1c06
+#define regSPI_RESOURCE_RESERVE_CU_6_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_7                                                                    0x1c07
+#define regSPI_RESOURCE_RESERVE_CU_7_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_8                                                                    0x1c08
+#define regSPI_RESOURCE_RESERVE_CU_8_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_9                                                                    0x1c09
+#define regSPI_RESOURCE_RESERVE_CU_9_BASE_IDX                                                           1
+#define regSPI_RESOURCE_RESERVE_CU_10                                                                   0x1c0a
+#define regSPI_RESOURCE_RESERVE_CU_10_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_CU_11                                                                   0x1c0b
+#define regSPI_RESOURCE_RESERVE_CU_11_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_CU_12                                                                   0x1c0c
+#define regSPI_RESOURCE_RESERVE_CU_12_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_CU_13                                                                   0x1c0d
+#define regSPI_RESOURCE_RESERVE_CU_13_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_CU_14                                                                   0x1c0e
+#define regSPI_RESOURCE_RESERVE_CU_14_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_CU_15                                                                   0x1c0f
+#define regSPI_RESOURCE_RESERVE_CU_15_BASE_IDX                                                          1
+#define regSPI_RESOURCE_RESERVE_EN_CU_0                                                                 0x1c10
+#define regSPI_RESOURCE_RESERVE_EN_CU_0_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_1                                                                 0x1c11
+#define regSPI_RESOURCE_RESERVE_EN_CU_1_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_2                                                                 0x1c12
+#define regSPI_RESOURCE_RESERVE_EN_CU_2_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_3                                                                 0x1c13
+#define regSPI_RESOURCE_RESERVE_EN_CU_3_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_4                                                                 0x1c14
+#define regSPI_RESOURCE_RESERVE_EN_CU_4_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_5                                                                 0x1c15
+#define regSPI_RESOURCE_RESERVE_EN_CU_5_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_6                                                                 0x1c16
+#define regSPI_RESOURCE_RESERVE_EN_CU_6_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_7                                                                 0x1c17
+#define regSPI_RESOURCE_RESERVE_EN_CU_7_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_8                                                                 0x1c18
+#define regSPI_RESOURCE_RESERVE_EN_CU_8_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_9                                                                 0x1c19
+#define regSPI_RESOURCE_RESERVE_EN_CU_9_BASE_IDX                                                        1
+#define regSPI_RESOURCE_RESERVE_EN_CU_10                                                                0x1c1a
+#define regSPI_RESOURCE_RESERVE_EN_CU_10_BASE_IDX                                                       1
+#define regSPI_RESOURCE_RESERVE_EN_CU_11                                                                0x1c1b
+#define regSPI_RESOURCE_RESERVE_EN_CU_11_BASE_IDX                                                       1
+#define regSPI_RESOURCE_RESERVE_EN_CU_12                                                                0x1c1c
+#define regSPI_RESOURCE_RESERVE_EN_CU_12_BASE_IDX                                                       1
+#define regSPI_RESOURCE_RESERVE_EN_CU_13                                                                0x1c1d
+#define regSPI_RESOURCE_RESERVE_EN_CU_13_BASE_IDX                                                       1
+#define regSPI_RESOURCE_RESERVE_EN_CU_14                                                                0x1c1e
+#define regSPI_RESOURCE_RESERVE_EN_CU_14_BASE_IDX                                                       1
+#define regSPI_RESOURCE_RESERVE_EN_CU_15                                                                0x1c1f
+#define regSPI_RESOURCE_RESERVE_EN_CU_15_BASE_IDX                                                       1
+
+
+// addressBlock: gc_gfxudec
+// base address: 0x30000
+#define regCP_EOP_DONE_ADDR_LO                                                                          0x2000
+#define regCP_EOP_DONE_ADDR_LO_BASE_IDX                                                                 1
+#define regCP_EOP_DONE_ADDR_HI                                                                          0x2001
+#define regCP_EOP_DONE_ADDR_HI_BASE_IDX                                                                 1
+#define regCP_EOP_DONE_DATA_LO                                                                          0x2002
+#define regCP_EOP_DONE_DATA_LO_BASE_IDX                                                                 1
+#define regCP_EOP_DONE_DATA_HI                                                                          0x2003
+#define regCP_EOP_DONE_DATA_HI_BASE_IDX                                                                 1
+#define regCP_EOP_LAST_FENCE_LO                                                                         0x2004
+#define regCP_EOP_LAST_FENCE_LO_BASE_IDX                                                                1
+#define regCP_EOP_LAST_FENCE_HI                                                                         0x2005
+#define regCP_EOP_LAST_FENCE_HI_BASE_IDX                                                                1
+#define regCP_PIPE_STATS_ADDR_LO                                                                        0x2018
+#define regCP_PIPE_STATS_ADDR_LO_BASE_IDX                                                               1
+#define regCP_PIPE_STATS_ADDR_HI                                                                        0x2019
+#define regCP_PIPE_STATS_ADDR_HI_BASE_IDX                                                               1
+#define regCP_VGT_IAVERT_COUNT_LO                                                                       0x201a
+#define regCP_VGT_IAVERT_COUNT_LO_BASE_IDX                                                              1
+#define regCP_VGT_IAVERT_COUNT_HI                                                                       0x201b
+#define regCP_VGT_IAVERT_COUNT_HI_BASE_IDX                                                              1
+#define regCP_VGT_IAPRIM_COUNT_LO                                                                       0x201c
+#define regCP_VGT_IAPRIM_COUNT_LO_BASE_IDX                                                              1
+#define regCP_VGT_IAPRIM_COUNT_HI                                                                       0x201d
+#define regCP_VGT_IAPRIM_COUNT_HI_BASE_IDX                                                              1
+#define regCP_VGT_GSPRIM_COUNT_LO                                                                       0x201e
+#define regCP_VGT_GSPRIM_COUNT_LO_BASE_IDX                                                              1
+#define regCP_VGT_GSPRIM_COUNT_HI                                                                       0x201f
+#define regCP_VGT_GSPRIM_COUNT_HI_BASE_IDX                                                              1
+#define regCP_VGT_VSINVOC_COUNT_LO                                                                      0x2020
+#define regCP_VGT_VSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_VSINVOC_COUNT_HI                                                                      0x2021
+#define regCP_VGT_VSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_GSINVOC_COUNT_LO                                                                      0x2022
+#define regCP_VGT_GSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_GSINVOC_COUNT_HI                                                                      0x2023
+#define regCP_VGT_GSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_HSINVOC_COUNT_LO                                                                      0x2024
+#define regCP_VGT_HSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_HSINVOC_COUNT_HI                                                                      0x2025
+#define regCP_VGT_HSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_DSINVOC_COUNT_LO                                                                      0x2026
+#define regCP_VGT_DSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_DSINVOC_COUNT_HI                                                                      0x2027
+#define regCP_VGT_DSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_PA_CINVOC_COUNT_LO                                                                        0x2028
+#define regCP_PA_CINVOC_COUNT_LO_BASE_IDX                                                               1
+#define regCP_PA_CINVOC_COUNT_HI                                                                        0x2029
+#define regCP_PA_CINVOC_COUNT_HI_BASE_IDX                                                               1
+#define regCP_PA_CPRIM_COUNT_LO                                                                         0x202a
+#define regCP_PA_CPRIM_COUNT_LO_BASE_IDX                                                                1
+#define regCP_PA_CPRIM_COUNT_HI                                                                         0x202b
+#define regCP_PA_CPRIM_COUNT_HI_BASE_IDX                                                                1
+#define regCP_SC_PSINVOC_COUNT0_LO                                                                      0x202c
+#define regCP_SC_PSINVOC_COUNT0_LO_BASE_IDX                                                             1
+#define regCP_SC_PSINVOC_COUNT0_HI                                                                      0x202d
+#define regCP_SC_PSINVOC_COUNT0_HI_BASE_IDX                                                             1
+#define regCP_SC_PSINVOC_COUNT1_LO                                                                      0x202e
+#define regCP_SC_PSINVOC_COUNT1_LO_BASE_IDX                                                             1
+#define regCP_SC_PSINVOC_COUNT1_HI                                                                      0x202f
+#define regCP_SC_PSINVOC_COUNT1_HI_BASE_IDX                                                             1
+#define regCP_VGT_CSINVOC_COUNT_LO                                                                      0x2030
+#define regCP_VGT_CSINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_CSINVOC_COUNT_HI                                                                      0x2031
+#define regCP_VGT_CSINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_VGT_ASINVOC_COUNT_LO                                                                      0x2032
+#define regCP_VGT_ASINVOC_COUNT_LO_BASE_IDX                                                             1
+#define regCP_VGT_ASINVOC_COUNT_HI                                                                      0x2033
+#define regCP_VGT_ASINVOC_COUNT_HI_BASE_IDX                                                             1
+#define regCP_PIPE_STATS_CONTROL                                                                        0x203d
+#define regCP_PIPE_STATS_CONTROL_BASE_IDX                                                               1
+#define regSCRATCH_REG0                                                                                 0x2040
+#define regSCRATCH_REG0_BASE_IDX                                                                        1
+#define regSCRATCH_REG1                                                                                 0x2041
+#define regSCRATCH_REG1_BASE_IDX                                                                        1
+#define regSCRATCH_REG2                                                                                 0x2042
+#define regSCRATCH_REG2_BASE_IDX                                                                        1
+#define regSCRATCH_REG3                                                                                 0x2043
+#define regSCRATCH_REG3_BASE_IDX                                                                        1
+#define regSCRATCH_REG4                                                                                 0x2044
+#define regSCRATCH_REG4_BASE_IDX                                                                        1
+#define regSCRATCH_REG5                                                                                 0x2045
+#define regSCRATCH_REG5_BASE_IDX                                                                        1
+#define regSCRATCH_REG6                                                                                 0x2046
+#define regSCRATCH_REG6_BASE_IDX                                                                        1
+#define regSCRATCH_REG7                                                                                 0x2047
+#define regSCRATCH_REG7_BASE_IDX                                                                        1
+#define regSCRATCH_REG_ATOMIC                                                                           0x2048
+#define regSCRATCH_REG_ATOMIC_BASE_IDX                                                                  1
+#define regSCRATCH_REG_CMPSWAP_ATOMIC                                                                   0x2048
+#define regSCRATCH_REG_CMPSWAP_ATOMIC_BASE_IDX                                                          1
+#define regCP_APPEND_DDID_CNT                                                                           0x204b
+#define regCP_APPEND_DDID_CNT_BASE_IDX                                                                  1
+#define regCP_APPEND_DATA_HI                                                                            0x204c
+#define regCP_APPEND_DATA_HI_BASE_IDX                                                                   1
+#define regCP_APPEND_LAST_CS_FENCE_HI                                                                   0x204d
+#define regCP_APPEND_LAST_CS_FENCE_HI_BASE_IDX                                                          1
+#define regCP_APPEND_LAST_PS_FENCE_HI                                                                   0x204e
+#define regCP_APPEND_LAST_PS_FENCE_HI_BASE_IDX                                                          1
+#define regCP_PFP_ATOMIC_PREOP_LO                                                                       0x2052
+#define regCP_PFP_ATOMIC_PREOP_LO_BASE_IDX                                                              1
+#define regCP_PFP_ATOMIC_PREOP_HI                                                                       0x2053
+#define regCP_PFP_ATOMIC_PREOP_HI_BASE_IDX                                                              1
+#define regCP_PFP_GDS_ATOMIC0_PREOP_LO                                                                  0x2054
+#define regCP_PFP_GDS_ATOMIC0_PREOP_LO_BASE_IDX                                                         1
+#define regCP_PFP_GDS_ATOMIC0_PREOP_HI                                                                  0x2055
+#define regCP_PFP_GDS_ATOMIC0_PREOP_HI_BASE_IDX                                                         1
+#define regCP_PFP_GDS_ATOMIC1_PREOP_LO                                                                  0x2056
+#define regCP_PFP_GDS_ATOMIC1_PREOP_LO_BASE_IDX                                                         1
+#define regCP_PFP_GDS_ATOMIC1_PREOP_HI                                                                  0x2057
+#define regCP_PFP_GDS_ATOMIC1_PREOP_HI_BASE_IDX                                                         1
+#define regCP_APPEND_ADDR_LO                                                                            0x2058
+#define regCP_APPEND_ADDR_LO_BASE_IDX                                                                   1
+#define regCP_APPEND_ADDR_HI                                                                            0x2059
+#define regCP_APPEND_ADDR_HI_BASE_IDX                                                                   1
+#define regCP_APPEND_DATA                                                                               0x205a
+#define regCP_APPEND_DATA_BASE_IDX                                                                      1
+#define regCP_APPEND_DATA_LO                                                                            0x205a
+#define regCP_APPEND_DATA_LO_BASE_IDX                                                                   1
+#define regCP_APPEND_LAST_CS_FENCE                                                                      0x205b
+#define regCP_APPEND_LAST_CS_FENCE_BASE_IDX                                                             1
+#define regCP_APPEND_LAST_CS_FENCE_LO                                                                   0x205b
+#define regCP_APPEND_LAST_CS_FENCE_LO_BASE_IDX                                                          1
+#define regCP_APPEND_LAST_PS_FENCE                                                                      0x205c
+#define regCP_APPEND_LAST_PS_FENCE_BASE_IDX                                                             1
+#define regCP_APPEND_LAST_PS_FENCE_LO                                                                   0x205c
+#define regCP_APPEND_LAST_PS_FENCE_LO_BASE_IDX                                                          1
+#define regCP_ATOMIC_PREOP_LO                                                                           0x205d
+#define regCP_ATOMIC_PREOP_LO_BASE_IDX                                                                  1
+#define regCP_ME_ATOMIC_PREOP_LO                                                                        0x205d
+#define regCP_ME_ATOMIC_PREOP_LO_BASE_IDX                                                               1
+#define regCP_ATOMIC_PREOP_HI                                                                           0x205e
+#define regCP_ATOMIC_PREOP_HI_BASE_IDX                                                                  1
+#define regCP_ME_ATOMIC_PREOP_HI                                                                        0x205e
+#define regCP_ME_ATOMIC_PREOP_HI_BASE_IDX                                                               1
+#define regCP_GDS_ATOMIC0_PREOP_LO                                                                      0x205f
+#define regCP_GDS_ATOMIC0_PREOP_LO_BASE_IDX                                                             1
+#define regCP_ME_GDS_ATOMIC0_PREOP_LO                                                                   0x205f
+#define regCP_ME_GDS_ATOMIC0_PREOP_LO_BASE_IDX                                                          1
+#define regCP_GDS_ATOMIC0_PREOP_HI                                                                      0x2060
+#define regCP_GDS_ATOMIC0_PREOP_HI_BASE_IDX                                                             1
+#define regCP_ME_GDS_ATOMIC0_PREOP_HI                                                                   0x2060
+#define regCP_ME_GDS_ATOMIC0_PREOP_HI_BASE_IDX                                                          1
+#define regCP_GDS_ATOMIC1_PREOP_LO                                                                      0x2061
+#define regCP_GDS_ATOMIC1_PREOP_LO_BASE_IDX                                                             1
+#define regCP_ME_GDS_ATOMIC1_PREOP_LO                                                                   0x2061
+#define regCP_ME_GDS_ATOMIC1_PREOP_LO_BASE_IDX                                                          1
+#define regCP_GDS_ATOMIC1_PREOP_HI                                                                      0x2062
+#define regCP_GDS_ATOMIC1_PREOP_HI_BASE_IDX                                                             1
+#define regCP_ME_GDS_ATOMIC1_PREOP_HI                                                                   0x2062
+#define regCP_ME_GDS_ATOMIC1_PREOP_HI_BASE_IDX                                                          1
+#define regCP_ME_MC_WADDR_LO                                                                            0x2069
+#define regCP_ME_MC_WADDR_LO_BASE_IDX                                                                   1
+#define regCP_ME_MC_WADDR_HI                                                                            0x206a
+#define regCP_ME_MC_WADDR_HI_BASE_IDX                                                                   1
+#define regCP_ME_MC_WDATA_LO                                                                            0x206b
+#define regCP_ME_MC_WDATA_LO_BASE_IDX                                                                   1
+#define regCP_ME_MC_WDATA_HI                                                                            0x206c
+#define regCP_ME_MC_WDATA_HI_BASE_IDX                                                                   1
+#define regCP_ME_MC_RADDR_LO                                                                            0x206d
+#define regCP_ME_MC_RADDR_LO_BASE_IDX                                                                   1
+#define regCP_ME_MC_RADDR_HI                                                                            0x206e
+#define regCP_ME_MC_RADDR_HI_BASE_IDX                                                                   1
+#define regCP_SEM_WAIT_TIMER                                                                            0x206f
+#define regCP_SEM_WAIT_TIMER_BASE_IDX                                                                   1
+#define regCP_SIG_SEM_ADDR_LO                                                                           0x2070
+#define regCP_SIG_SEM_ADDR_LO_BASE_IDX                                                                  1
+#define regCP_SIG_SEM_ADDR_HI                                                                           0x2071
+#define regCP_SIG_SEM_ADDR_HI_BASE_IDX                                                                  1
+#define regCP_WAIT_REG_MEM_TIMEOUT                                                                      0x2074
+#define regCP_WAIT_REG_MEM_TIMEOUT_BASE_IDX                                                             1
+#define regCP_WAIT_SEM_ADDR_LO                                                                          0x2075
+#define regCP_WAIT_SEM_ADDR_LO_BASE_IDX                                                                 1
+#define regCP_WAIT_SEM_ADDR_HI                                                                          0x2076
+#define regCP_WAIT_SEM_ADDR_HI_BASE_IDX                                                                 1
+#define regCP_DMA_PFP_CONTROL                                                                           0x2077
+#define regCP_DMA_PFP_CONTROL_BASE_IDX                                                                  1
+#define regCP_DMA_ME_CONTROL                                                                            0x2078
+#define regCP_DMA_ME_CONTROL_BASE_IDX                                                                   1
+#define regCP_DMA_ME_SRC_ADDR                                                                           0x2080
+#define regCP_DMA_ME_SRC_ADDR_BASE_IDX                                                                  1
+#define regCP_DMA_ME_SRC_ADDR_HI                                                                        0x2081
+#define regCP_DMA_ME_SRC_ADDR_HI_BASE_IDX                                                               1
+#define regCP_DMA_ME_DST_ADDR                                                                           0x2082
+#define regCP_DMA_ME_DST_ADDR_BASE_IDX                                                                  1
+#define regCP_DMA_ME_DST_ADDR_HI                                                                        0x2083
+#define regCP_DMA_ME_DST_ADDR_HI_BASE_IDX                                                               1
+#define regCP_DMA_ME_COMMAND                                                                            0x2084
+#define regCP_DMA_ME_COMMAND_BASE_IDX                                                                   1
+#define regCP_DMA_PFP_SRC_ADDR                                                                          0x2085
+#define regCP_DMA_PFP_SRC_ADDR_BASE_IDX                                                                 1
+#define regCP_DMA_PFP_SRC_ADDR_HI                                                                       0x2086
+#define regCP_DMA_PFP_SRC_ADDR_HI_BASE_IDX                                                              1
+#define regCP_DMA_PFP_DST_ADDR                                                                          0x2087
+#define regCP_DMA_PFP_DST_ADDR_BASE_IDX                                                                 1
+#define regCP_DMA_PFP_DST_ADDR_HI                                                                       0x2088
+#define regCP_DMA_PFP_DST_ADDR_HI_BASE_IDX                                                              1
+#define regCP_DMA_PFP_COMMAND                                                                           0x2089
+#define regCP_DMA_PFP_COMMAND_BASE_IDX                                                                  1
+#define regCP_DMA_CNTL                                                                                  0x208a
+#define regCP_DMA_CNTL_BASE_IDX                                                                         1
+#define regCP_DMA_READ_TAGS                                                                             0x208b
+#define regCP_DMA_READ_TAGS_BASE_IDX                                                                    1
+#define regCP_PFP_IB_CONTROL                                                                            0x208d
+#define regCP_PFP_IB_CONTROL_BASE_IDX                                                                   1
+#define regCP_PFP_LOAD_CONTROL                                                                          0x208e
+#define regCP_PFP_LOAD_CONTROL_BASE_IDX                                                                 1
+#define regCP_SCRATCH_INDEX                                                                             0x208f
+#define regCP_SCRATCH_INDEX_BASE_IDX                                                                    1
+#define regCP_SCRATCH_DATA                                                                              0x2090
+#define regCP_SCRATCH_DATA_BASE_IDX                                                                     1
+#define regCP_RB_OFFSET                                                                                 0x2091
+#define regCP_RB_OFFSET_BASE_IDX                                                                        1
+#define regCP_IB1_OFFSET                                                                                0x2092
+#define regCP_IB1_OFFSET_BASE_IDX                                                                       1
+#define regCP_IB2_OFFSET                                                                                0x2093
+#define regCP_IB2_OFFSET_BASE_IDX                                                                       1
+#define regCP_IB1_PREAMBLE_BEGIN                                                                        0x2094
+#define regCP_IB1_PREAMBLE_BEGIN_BASE_IDX                                                               1
+#define regCP_IB1_PREAMBLE_END                                                                          0x2095
+#define regCP_IB1_PREAMBLE_END_BASE_IDX                                                                 1
+#define regCP_IB2_PREAMBLE_BEGIN                                                                        0x2096
+#define regCP_IB2_PREAMBLE_BEGIN_BASE_IDX                                                               1
+#define regCP_IB2_PREAMBLE_END                                                                          0x2097
+#define regCP_IB2_PREAMBLE_END_BASE_IDX                                                                 1
+#define regCP_DMA_ME_CMD_ADDR_LO                                                                        0x209c
+#define regCP_DMA_ME_CMD_ADDR_LO_BASE_IDX                                                               1
+#define regCP_DMA_ME_CMD_ADDR_HI                                                                        0x209d
+#define regCP_DMA_ME_CMD_ADDR_HI_BASE_IDX                                                               1
+#define regCP_DMA_PFP_CMD_ADDR_LO                                                                       0x209e
+#define regCP_DMA_PFP_CMD_ADDR_LO_BASE_IDX                                                              1
+#define regCP_DMA_PFP_CMD_ADDR_HI                                                                       0x209f
+#define regCP_DMA_PFP_CMD_ADDR_HI_BASE_IDX                                                              1
+#define regCP_APPEND_CMD_ADDR_LO                                                                        0x20a0
+#define regCP_APPEND_CMD_ADDR_LO_BASE_IDX                                                               1
+#define regCP_APPEND_CMD_ADDR_HI                                                                        0x20a1
+#define regCP_APPEND_CMD_ADDR_HI_BASE_IDX                                                               1
+#define regUCONFIG_RESERVED_REG0                                                                        0x20a2
+#define regUCONFIG_RESERVED_REG0_BASE_IDX                                                               1
+#define regUCONFIG_RESERVED_REG1                                                                        0x20a3
+#define regUCONFIG_RESERVED_REG1_BASE_IDX                                                               1
+#define regCP_PA_MSPRIM_COUNT_LO                                                                        0x20a4
+#define regCP_PA_MSPRIM_COUNT_LO_BASE_IDX                                                               1
+#define regCP_PA_MSPRIM_COUNT_HI                                                                        0x20a5
+#define regCP_PA_MSPRIM_COUNT_HI_BASE_IDX                                                               1
+#define regCP_GE_MSINVOC_COUNT_LO                                                                       0x20a6
+#define regCP_GE_MSINVOC_COUNT_LO_BASE_IDX                                                              1
+#define regCP_GE_MSINVOC_COUNT_HI                                                                       0x20a7
+#define regCP_GE_MSINVOC_COUNT_HI_BASE_IDX                                                              1
+#define regCP_IB1_CMD_BUFSZ                                                                             0x20c0
+#define regCP_IB1_CMD_BUFSZ_BASE_IDX                                                                    1
+#define regCP_IB2_CMD_BUFSZ                                                                             0x20c1
+#define regCP_IB2_CMD_BUFSZ_BASE_IDX                                                                    1
+#define regCP_ST_CMD_BUFSZ                                                                              0x20c2
+#define regCP_ST_CMD_BUFSZ_BASE_IDX                                                                     1
+#define regCP_IB1_BASE_LO                                                                               0x20cc
+#define regCP_IB1_BASE_LO_BASE_IDX                                                                      1
+#define regCP_IB1_BASE_HI                                                                               0x20cd
+#define regCP_IB1_BASE_HI_BASE_IDX                                                                      1
+#define regCP_IB1_BUFSZ                                                                                 0x20ce
+#define regCP_IB1_BUFSZ_BASE_IDX                                                                        1
+#define regCP_IB2_BASE_LO                                                                               0x20cf
+#define regCP_IB2_BASE_LO_BASE_IDX                                                                      1
+#define regCP_IB2_BASE_HI                                                                               0x20d0
+#define regCP_IB2_BASE_HI_BASE_IDX                                                                      1
+#define regCP_IB2_BUFSZ                                                                                 0x20d1
+#define regCP_IB2_BUFSZ_BASE_IDX                                                                        1
+#define regCP_ST_BASE_LO                                                                                0x20d2
+#define regCP_ST_BASE_LO_BASE_IDX                                                                       1
+#define regCP_ST_BASE_HI                                                                                0x20d3
+#define regCP_ST_BASE_HI_BASE_IDX                                                                       1
+#define regCP_ST_BUFSZ                                                                                  0x20d4
+#define regCP_ST_BUFSZ_BASE_IDX                                                                         1
+#define regCP_EOP_DONE_EVENT_CNTL                                                                       0x20d5
+#define regCP_EOP_DONE_EVENT_CNTL_BASE_IDX                                                              1
+#define regCP_EOP_DONE_DATA_CNTL                                                                        0x20d6
+#define regCP_EOP_DONE_DATA_CNTL_BASE_IDX                                                               1
+#define regCP_EOP_DONE_CNTX_ID                                                                          0x20d7
+#define regCP_EOP_DONE_CNTX_ID_BASE_IDX                                                                 1
+#define regCP_DB_BASE_LO                                                                                0x20d8
+#define regCP_DB_BASE_LO_BASE_IDX                                                                       1
+#define regCP_DB_BASE_HI                                                                                0x20d9
+#define regCP_DB_BASE_HI_BASE_IDX                                                                       1
+#define regCP_DB_BUFSZ                                                                                  0x20da
+#define regCP_DB_BUFSZ_BASE_IDX                                                                         1
+#define regCP_DB_CMD_BUFSZ                                                                              0x20db
+#define regCP_DB_CMD_BUFSZ_BASE_IDX                                                                     1
+#define regCP_PFP_COMPLETION_STATUS                                                                     0x20ec
+#define regCP_PFP_COMPLETION_STATUS_BASE_IDX                                                            1
+#define regCP_PRED_NOT_VISIBLE                                                                          0x20ee
+#define regCP_PRED_NOT_VISIBLE_BASE_IDX                                                                 1
+#define regCP_PFP_METADATA_BASE_ADDR                                                                    0x20f0
+#define regCP_PFP_METADATA_BASE_ADDR_BASE_IDX                                                           1
+#define regCP_PFP_METADATA_BASE_ADDR_HI                                                                 0x20f1
+#define regCP_PFP_METADATA_BASE_ADDR_HI_BASE_IDX                                                        1
+#define regCP_DRAW_INDX_INDR_ADDR                                                                       0x20f4
+#define regCP_DRAW_INDX_INDR_ADDR_BASE_IDX                                                              1
+#define regCP_DRAW_INDX_INDR_ADDR_HI                                                                    0x20f5
+#define regCP_DRAW_INDX_INDR_ADDR_HI_BASE_IDX                                                           1
+#define regCP_DISPATCH_INDR_ADDR                                                                        0x20f6
+#define regCP_DISPATCH_INDR_ADDR_BASE_IDX                                                               1
+#define regCP_DISPATCH_INDR_ADDR_HI                                                                     0x20f7
+#define regCP_DISPATCH_INDR_ADDR_HI_BASE_IDX                                                            1
+#define regCP_INDEX_BASE_ADDR                                                                           0x20f8
+#define regCP_INDEX_BASE_ADDR_BASE_IDX                                                                  1
+#define regCP_INDEX_BASE_ADDR_HI                                                                        0x20f9
+#define regCP_INDEX_BASE_ADDR_HI_BASE_IDX                                                               1
+#define regCP_INDEX_TYPE                                                                                0x20fa
+#define regCP_INDEX_TYPE_BASE_IDX                                                                       1
+#define regCP_GDS_BKUP_ADDR                                                                             0x20fb
+#define regCP_GDS_BKUP_ADDR_BASE_IDX                                                                    1
+#define regCP_GDS_BKUP_ADDR_HI                                                                          0x20fc
+#define regCP_GDS_BKUP_ADDR_HI_BASE_IDX                                                                 1
+#define regCP_SAMPLE_STATUS                                                                             0x20fd
+#define regCP_SAMPLE_STATUS_BASE_IDX                                                                    1
+#define regCP_ME_COHER_CNTL                                                                             0x20fe
+#define regCP_ME_COHER_CNTL_BASE_IDX                                                                    1
+#define regCP_ME_COHER_SIZE                                                                             0x20ff
+#define regCP_ME_COHER_SIZE_BASE_IDX                                                                    1
+#define regCP_ME_COHER_SIZE_HI                                                                          0x2100
+#define regCP_ME_COHER_SIZE_HI_BASE_IDX                                                                 1
+#define regCP_ME_COHER_BASE                                                                             0x2101
+#define regCP_ME_COHER_BASE_BASE_IDX                                                                    1
+#define regCP_ME_COHER_BASE_HI                                                                          0x2102
+#define regCP_ME_COHER_BASE_HI_BASE_IDX                                                                 1
+#define regCP_ME_COHER_STATUS                                                                           0x2103
+#define regCP_ME_COHER_STATUS_BASE_IDX                                                                  1
+#define regRLC_GPM_PERF_COUNT_0                                                                         0x2140
+#define regRLC_GPM_PERF_COUNT_0_BASE_IDX                                                                1
+#define regRLC_GPM_PERF_COUNT_1                                                                         0x2141
+#define regRLC_GPM_PERF_COUNT_1_BASE_IDX                                                                1
+#define regGRBM_GFX_INDEX                                                                               0x2200
+#define regGRBM_GFX_INDEX_BASE_IDX                                                                      1
+#define regVGT_PRIMITIVE_TYPE                                                                           0x2242
+#define regVGT_PRIMITIVE_TYPE_BASE_IDX                                                                  1
+#define regVGT_INDEX_TYPE                                                                               0x2243
+#define regVGT_INDEX_TYPE_BASE_IDX                                                                      1
+#define regGE_MIN_VTX_INDX                                                                              0x2249
+#define regGE_MIN_VTX_INDX_BASE_IDX                                                                     1
+#define regGE_INDX_OFFSET                                                                               0x224a
+#define regGE_INDX_OFFSET_BASE_IDX                                                                      1
+#define regGE_MULTI_PRIM_IB_RESET_EN                                                                    0x224b
+#define regGE_MULTI_PRIM_IB_RESET_EN_BASE_IDX                                                           1
+#define regVGT_NUM_INDICES                                                                              0x224c
+#define regVGT_NUM_INDICES_BASE_IDX                                                                     1
+#define regVGT_NUM_INSTANCES                                                                            0x224d
+#define regVGT_NUM_INSTANCES_BASE_IDX                                                                   1
+#define regVGT_TF_RING_SIZE                                                                             0x224e
+#define regVGT_TF_RING_SIZE_BASE_IDX                                                                    1
+#define regVGT_HS_OFFCHIP_PARAM                                                                         0x224f
+#define regVGT_HS_OFFCHIP_PARAM_BASE_IDX                                                                1
+#define regVGT_TF_MEMORY_BASE                                                                           0x2250
+#define regVGT_TF_MEMORY_BASE_BASE_IDX                                                                  1
+#define regGE_MAX_VTX_INDX                                                                              0x2259
+#define regGE_MAX_VTX_INDX_BASE_IDX                                                                     1
+#define regVGT_INSTANCE_BASE_ID                                                                         0x225a
+#define regVGT_INSTANCE_BASE_ID_BASE_IDX                                                                1
+#define regGE_CNTL                                                                                      0x225b
+#define regGE_CNTL_BASE_IDX                                                                             1
+#define regGE_USER_VGPR1                                                                                0x225c
+#define regGE_USER_VGPR1_BASE_IDX                                                                       1
+#define regGE_USER_VGPR2                                                                                0x225d
+#define regGE_USER_VGPR2_BASE_IDX                                                                       1
+#define regGE_USER_VGPR3                                                                                0x225e
+#define regGE_USER_VGPR3_BASE_IDX                                                                       1
+#define regGE_STEREO_CNTL                                                                               0x225f
+#define regGE_STEREO_CNTL_BASE_IDX                                                                      1
+#define regGE_PC_ALLOC                                                                                  0x2260
+#define regGE_PC_ALLOC_BASE_IDX                                                                         1
+#define regVGT_TF_MEMORY_BASE_HI                                                                        0x2261
+#define regVGT_TF_MEMORY_BASE_HI_BASE_IDX                                                               1
+#define regGE_USER_VGPR_EN                                                                              0x2262
+#define regGE_USER_VGPR_EN_BASE_IDX                                                                     1
+#define regGE_GS_FAST_LAUNCH_WG_DIM                                                                     0x2264
+#define regGE_GS_FAST_LAUNCH_WG_DIM_BASE_IDX                                                            1
+#define regGE_GS_FAST_LAUNCH_WG_DIM_1                                                                   0x2265
+#define regGE_GS_FAST_LAUNCH_WG_DIM_1_BASE_IDX                                                          1
+#define regVGT_GS_OUT_PRIM_TYPE                                                                         0x2266
+#define regVGT_GS_OUT_PRIM_TYPE_BASE_IDX                                                                1
+#define regPA_SU_LINE_STIPPLE_VALUE                                                                     0x2280
+#define regPA_SU_LINE_STIPPLE_VALUE_BASE_IDX                                                            1
+#define regPA_SC_LINE_STIPPLE_STATE                                                                     0x2281
+#define regPA_SC_LINE_STIPPLE_STATE_BASE_IDX                                                            1
+#define regPA_SC_SCREEN_EXTENT_MIN_0                                                                    0x2284
+#define regPA_SC_SCREEN_EXTENT_MIN_0_BASE_IDX                                                           1
+#define regPA_SC_SCREEN_EXTENT_MAX_0                                                                    0x2285
+#define regPA_SC_SCREEN_EXTENT_MAX_0_BASE_IDX                                                           1
+#define regPA_SC_SCREEN_EXTENT_MIN_1                                                                    0x2286
+#define regPA_SC_SCREEN_EXTENT_MIN_1_BASE_IDX                                                           1
+#define regPA_SC_SCREEN_EXTENT_MAX_1                                                                    0x228b
+#define regPA_SC_SCREEN_EXTENT_MAX_1_BASE_IDX                                                           1
+#define regPA_SC_P3D_TRAP_SCREEN_HV_EN                                                                  0x22a0
+#define regPA_SC_P3D_TRAP_SCREEN_HV_EN_BASE_IDX                                                         1
+#define regPA_SC_P3D_TRAP_SCREEN_H                                                                      0x22a1
+#define regPA_SC_P3D_TRAP_SCREEN_H_BASE_IDX                                                             1
+#define regPA_SC_P3D_TRAP_SCREEN_V                                                                      0x22a2
+#define regPA_SC_P3D_TRAP_SCREEN_V_BASE_IDX                                                             1
+#define regPA_SC_P3D_TRAP_SCREEN_OCCURRENCE                                                             0x22a3
+#define regPA_SC_P3D_TRAP_SCREEN_OCCURRENCE_BASE_IDX                                                    1
+#define regPA_SC_P3D_TRAP_SCREEN_COUNT                                                                  0x22a4
+#define regPA_SC_P3D_TRAP_SCREEN_COUNT_BASE_IDX                                                         1
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_EN                                                                 0x22a8
+#define regPA_SC_HP3D_TRAP_SCREEN_HV_EN_BASE_IDX                                                        1
+#define regPA_SC_HP3D_TRAP_SCREEN_H                                                                     0x22a9
+#define regPA_SC_HP3D_TRAP_SCREEN_H_BASE_IDX                                                            1
+#define regPA_SC_HP3D_TRAP_SCREEN_V                                                                     0x22aa
+#define regPA_SC_HP3D_TRAP_SCREEN_V_BASE_IDX                                                            1
+#define regPA_SC_HP3D_TRAP_SCREEN_OCCURRENCE                                                            0x22ab
+#define regPA_SC_HP3D_TRAP_SCREEN_OCCURRENCE_BASE_IDX                                                   1
+#define regPA_SC_HP3D_TRAP_SCREEN_COUNT                                                                 0x22ac
+#define regPA_SC_HP3D_TRAP_SCREEN_COUNT_BASE_IDX                                                        1
+#define regPA_SC_TRAP_SCREEN_HV_EN                                                                      0x22b0
+#define regPA_SC_TRAP_SCREEN_HV_EN_BASE_IDX                                                             1
+#define regPA_SC_TRAP_SCREEN_H                                                                          0x22b1
+#define regPA_SC_TRAP_SCREEN_H_BASE_IDX                                                                 1
+#define regPA_SC_TRAP_SCREEN_V                                                                          0x22b2
+#define regPA_SC_TRAP_SCREEN_V_BASE_IDX                                                                 1
+#define regPA_SC_TRAP_SCREEN_OCCURRENCE                                                                 0x22b3
+#define regPA_SC_TRAP_SCREEN_OCCURRENCE_BASE_IDX                                                        1
+#define regPA_SC_TRAP_SCREEN_COUNT                                                                      0x22b4
+#define regPA_SC_TRAP_SCREEN_COUNT_BASE_IDX                                                             1
+#define regSQ_THREAD_TRACE_USERDATA_0                                                                   0x2340
+#define regSQ_THREAD_TRACE_USERDATA_0_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_1                                                                   0x2341
+#define regSQ_THREAD_TRACE_USERDATA_1_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_2                                                                   0x2342
+#define regSQ_THREAD_TRACE_USERDATA_2_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_3                                                                   0x2343
+#define regSQ_THREAD_TRACE_USERDATA_3_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_4                                                                   0x2344
+#define regSQ_THREAD_TRACE_USERDATA_4_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_5                                                                   0x2345
+#define regSQ_THREAD_TRACE_USERDATA_5_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_6                                                                   0x2346
+#define regSQ_THREAD_TRACE_USERDATA_6_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_USERDATA_7                                                                   0x2347
+#define regSQ_THREAD_TRACE_USERDATA_7_BASE_IDX                                                          1
+#define regSQC_CACHES                                                                                   0x2348
+#define regSQC_CACHES_BASE_IDX                                                                          1
+#define regTA_CS_BC_BASE_ADDR                                                                           0x2380
+#define regTA_CS_BC_BASE_ADDR_BASE_IDX                                                                  1
+#define regTA_CS_BC_BASE_ADDR_HI                                                                        0x2381
+#define regTA_CS_BC_BASE_ADDR_HI_BASE_IDX                                                               1
+#define regDB_OCCLUSION_COUNT0_LOW                                                                      0x23c0
+#define regDB_OCCLUSION_COUNT0_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT0_HI                                                                       0x23c1
+#define regDB_OCCLUSION_COUNT0_HI_BASE_IDX                                                              1
+#define regDB_OCCLUSION_COUNT1_LOW                                                                      0x23c2
+#define regDB_OCCLUSION_COUNT1_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT1_HI                                                                       0x23c3
+#define regDB_OCCLUSION_COUNT1_HI_BASE_IDX                                                              1
+#define regDB_OCCLUSION_COUNT2_LOW                                                                      0x23c4
+#define regDB_OCCLUSION_COUNT2_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT2_HI                                                                       0x23c5
+#define regDB_OCCLUSION_COUNT2_HI_BASE_IDX                                                              1
+#define regDB_OCCLUSION_COUNT3_LOW                                                                      0x23c6
+#define regDB_OCCLUSION_COUNT3_LOW_BASE_IDX                                                             1
+#define regDB_OCCLUSION_COUNT3_HI                                                                       0x23c7
+#define regDB_OCCLUSION_COUNT3_HI_BASE_IDX                                                              1
+#define regGDS_RD_ADDR                                                                                  0x2400
+#define regGDS_RD_ADDR_BASE_IDX                                                                         1
+#define regGDS_RD_DATA                                                                                  0x2401
+#define regGDS_RD_DATA_BASE_IDX                                                                         1
+#define regGDS_RD_BURST_ADDR                                                                            0x2402
+#define regGDS_RD_BURST_ADDR_BASE_IDX                                                                   1
+#define regGDS_RD_BURST_COUNT                                                                           0x2403
+#define regGDS_RD_BURST_COUNT_BASE_IDX                                                                  1
+#define regGDS_RD_BURST_DATA                                                                            0x2404
+#define regGDS_RD_BURST_DATA_BASE_IDX                                                                   1
+#define regGDS_WR_ADDR                                                                                  0x2405
+#define regGDS_WR_ADDR_BASE_IDX                                                                         1
+#define regGDS_WR_DATA                                                                                  0x2406
+#define regGDS_WR_DATA_BASE_IDX                                                                         1
+#define regGDS_WR_BURST_ADDR                                                                            0x2407
+#define regGDS_WR_BURST_ADDR_BASE_IDX                                                                   1
+#define regGDS_WR_BURST_DATA                                                                            0x2408
+#define regGDS_WR_BURST_DATA_BASE_IDX                                                                   1
+#define regGDS_WRITE_COMPLETE                                                                           0x2409
+#define regGDS_WRITE_COMPLETE_BASE_IDX                                                                  1
+#define regGDS_ATOM_CNTL                                                                                0x240a
+#define regGDS_ATOM_CNTL_BASE_IDX                                                                       1
+#define regGDS_ATOM_COMPLETE                                                                            0x240b
+#define regGDS_ATOM_COMPLETE_BASE_IDX                                                                   1
+#define regGDS_ATOM_BASE                                                                                0x240c
+#define regGDS_ATOM_BASE_BASE_IDX                                                                       1
+#define regGDS_ATOM_SIZE                                                                                0x240d
+#define regGDS_ATOM_SIZE_BASE_IDX                                                                       1
+#define regGDS_ATOM_OFFSET0                                                                             0x240e
+#define regGDS_ATOM_OFFSET0_BASE_IDX                                                                    1
+#define regGDS_ATOM_OFFSET1                                                                             0x240f
+#define regGDS_ATOM_OFFSET1_BASE_IDX                                                                    1
+#define regGDS_ATOM_DST                                                                                 0x2410
+#define regGDS_ATOM_DST_BASE_IDX                                                                        1
+#define regGDS_ATOM_OP                                                                                  0x2411
+#define regGDS_ATOM_OP_BASE_IDX                                                                         1
+#define regGDS_ATOM_SRC0                                                                                0x2412
+#define regGDS_ATOM_SRC0_BASE_IDX                                                                       1
+#define regGDS_ATOM_SRC0_U                                                                              0x2413
+#define regGDS_ATOM_SRC0_U_BASE_IDX                                                                     1
+#define regGDS_ATOM_SRC1                                                                                0x2414
+#define regGDS_ATOM_SRC1_BASE_IDX                                                                       1
+#define regGDS_ATOM_SRC1_U                                                                              0x2415
+#define regGDS_ATOM_SRC1_U_BASE_IDX                                                                     1
+#define regGDS_ATOM_READ0                                                                               0x2416
+#define regGDS_ATOM_READ0_BASE_IDX                                                                      1
+#define regGDS_ATOM_READ0_U                                                                             0x2417
+#define regGDS_ATOM_READ0_U_BASE_IDX                                                                    1
+#define regGDS_ATOM_READ1                                                                               0x2418
+#define regGDS_ATOM_READ1_BASE_IDX                                                                      1
+#define regGDS_ATOM_READ1_U                                                                             0x2419
+#define regGDS_ATOM_READ1_U_BASE_IDX                                                                    1
+#define regGDS_GWS_RESOURCE_CNTL                                                                        0x241a
+#define regGDS_GWS_RESOURCE_CNTL_BASE_IDX                                                               1
+#define regGDS_GWS_RESOURCE                                                                             0x241b
+#define regGDS_GWS_RESOURCE_BASE_IDX                                                                    1
+#define regGDS_GWS_RESOURCE_CNT                                                                         0x241c
+#define regGDS_GWS_RESOURCE_CNT_BASE_IDX                                                                1
+#define regGDS_OA_CNTL                                                                                  0x241d
+#define regGDS_OA_CNTL_BASE_IDX                                                                         1
+#define regGDS_OA_COUNTER                                                                               0x241e
+#define regGDS_OA_COUNTER_BASE_IDX                                                                      1
+#define regGDS_OA_ADDRESS                                                                               0x241f
+#define regGDS_OA_ADDRESS_BASE_IDX                                                                      1
+#define regGDS_OA_INCDEC                                                                                0x2420
+#define regGDS_OA_INCDEC_BASE_IDX                                                                       1
+#define regGDS_OA_RING_SIZE                                                                             0x2421
+#define regGDS_OA_RING_SIZE_BASE_IDX                                                                    1
+#define regGDS_STRMOUT_DWORDS_WRITTEN_0                                                                 0x2422
+#define regGDS_STRMOUT_DWORDS_WRITTEN_0_BASE_IDX                                                        1
+#define regGDS_STRMOUT_DWORDS_WRITTEN_1                                                                 0x2423
+#define regGDS_STRMOUT_DWORDS_WRITTEN_1_BASE_IDX                                                        1
+#define regGDS_STRMOUT_DWORDS_WRITTEN_2                                                                 0x2424
+#define regGDS_STRMOUT_DWORDS_WRITTEN_2_BASE_IDX                                                        1
+#define regGDS_STRMOUT_DWORDS_WRITTEN_3                                                                 0x2425
+#define regGDS_STRMOUT_DWORDS_WRITTEN_3_BASE_IDX                                                        1
+#define regGDS_GS_0                                                                                     0x2426
+#define regGDS_GS_0_BASE_IDX                                                                            1
+#define regGDS_GS_1                                                                                     0x2427
+#define regGDS_GS_1_BASE_IDX                                                                            1
+#define regGDS_GS_2                                                                                     0x2428
+#define regGDS_GS_2_BASE_IDX                                                                            1
+#define regGDS_GS_3                                                                                     0x2429
+#define regGDS_GS_3_BASE_IDX                                                                            1
+#define regGDS_STRMOUT_PRIMS_NEEDED_0_LO                                                                0x242a
+#define regGDS_STRMOUT_PRIMS_NEEDED_0_LO_BASE_IDX                                                       1
+#define regGDS_STRMOUT_PRIMS_NEEDED_0_HI                                                                0x242b
+#define regGDS_STRMOUT_PRIMS_NEEDED_0_HI_BASE_IDX                                                       1
+#define regGDS_STRMOUT_PRIMS_WRITTEN_0_LO                                                               0x242c
+#define regGDS_STRMOUT_PRIMS_WRITTEN_0_LO_BASE_IDX                                                      1
+#define regGDS_STRMOUT_PRIMS_WRITTEN_0_HI                                                               0x242d
+#define regGDS_STRMOUT_PRIMS_WRITTEN_0_HI_BASE_IDX                                                      1
+#define regGDS_STRMOUT_PRIMS_NEEDED_1_LO                                                                0x242e
+#define regGDS_STRMOUT_PRIMS_NEEDED_1_LO_BASE_IDX                                                       1
+#define regGDS_STRMOUT_PRIMS_NEEDED_1_HI                                                                0x242f
+#define regGDS_STRMOUT_PRIMS_NEEDED_1_HI_BASE_IDX                                                       1
+#define regGDS_STRMOUT_PRIMS_WRITTEN_1_LO                                                               0x2430
+#define regGDS_STRMOUT_PRIMS_WRITTEN_1_LO_BASE_IDX                                                      1
+#define regGDS_STRMOUT_PRIMS_WRITTEN_1_HI                                                               0x2431
+#define regGDS_STRMOUT_PRIMS_WRITTEN_1_HI_BASE_IDX                                                      1
+#define regGDS_STRMOUT_PRIMS_NEEDED_2_LO                                                                0x2432
+#define regGDS_STRMOUT_PRIMS_NEEDED_2_LO_BASE_IDX                                                       1
+#define regGDS_STRMOUT_PRIMS_NEEDED_2_HI                                                                0x2433
+#define regGDS_STRMOUT_PRIMS_NEEDED_2_HI_BASE_IDX                                                       1
+#define regGDS_STRMOUT_PRIMS_WRITTEN_2_LO                                                               0x2434
+#define regGDS_STRMOUT_PRIMS_WRITTEN_2_LO_BASE_IDX                                                      1
+#define regGDS_STRMOUT_PRIMS_WRITTEN_2_HI                                                               0x2435
+#define regGDS_STRMOUT_PRIMS_WRITTEN_2_HI_BASE_IDX                                                      1
+#define regGDS_STRMOUT_PRIMS_NEEDED_3_LO                                                                0x2436
+#define regGDS_STRMOUT_PRIMS_NEEDED_3_LO_BASE_IDX                                                       1
+#define regGDS_STRMOUT_PRIMS_NEEDED_3_HI                                                                0x2437
+#define regGDS_STRMOUT_PRIMS_NEEDED_3_HI_BASE_IDX                                                       1
+#define regGDS_STRMOUT_PRIMS_WRITTEN_3_LO                                                               0x2438
+#define regGDS_STRMOUT_PRIMS_WRITTEN_3_LO_BASE_IDX                                                      1
+#define regGDS_STRMOUT_PRIMS_WRITTEN_3_HI                                                               0x2439
+#define regGDS_STRMOUT_PRIMS_WRITTEN_3_HI_BASE_IDX                                                      1
+#define regSPI_CONFIG_CNTL                                                                              0x2440
+#define regSPI_CONFIG_CNTL_BASE_IDX                                                                     1
+#define regSPI_CONFIG_CNTL_1                                                                            0x2441
+#define regSPI_CONFIG_CNTL_1_BASE_IDX                                                                   1
+#define regSPI_CONFIG_CNTL_2                                                                            0x2442
+#define regSPI_CONFIG_CNTL_2_BASE_IDX                                                                   1
+#define regSPI_WAVE_LIMIT_CNTL                                                                          0x2443
+#define regSPI_WAVE_LIMIT_CNTL_BASE_IDX                                                                 1
+#define regSPI_GS_THROTTLE_CNTL1                                                                        0x2444
+#define regSPI_GS_THROTTLE_CNTL1_BASE_IDX                                                               1
+#define regSPI_GS_THROTTLE_CNTL2                                                                        0x2445
+#define regSPI_GS_THROTTLE_CNTL2_BASE_IDX                                                               1
+#define regSPI_ATTRIBUTE_RING_BASE                                                                      0x2446
+#define regSPI_ATTRIBUTE_RING_BASE_BASE_IDX                                                             1
+#define regSPI_ATTRIBUTE_RING_SIZE                                                                      0x2447
+#define regSPI_ATTRIBUTE_RING_SIZE_BASE_IDX                                                             1
+
+
+// addressBlock: gc_cprs64dec
+// base address: 0x32000
+#define regCP_MES_PRGRM_CNTR_START                                                                      0x2800
+#define regCP_MES_PRGRM_CNTR_START_BASE_IDX                                                             1
+#define regCP_MES_INTR_ROUTINE_START                                                                    0x2801
+#define regCP_MES_INTR_ROUTINE_START_BASE_IDX                                                           1
+#define regCP_MES_MTVEC_LO                                                                              0x2801
+#define regCP_MES_MTVEC_LO_BASE_IDX                                                                     1
+#define regCP_MES_INTR_ROUTINE_START_HI                                                                 0x2802
+#define regCP_MES_INTR_ROUTINE_START_HI_BASE_IDX                                                        1
+#define regCP_MES_MTVEC_HI                                                                              0x2802
+#define regCP_MES_MTVEC_HI_BASE_IDX                                                                     1
+#define regCP_MES_CNTL                                                                                  0x2807
+#define regCP_MES_CNTL_BASE_IDX                                                                         1
+#define regCP_MES_PIPE_PRIORITY_CNTS                                                                    0x2808
+#define regCP_MES_PIPE_PRIORITY_CNTS_BASE_IDX                                                           1
+#define regCP_MES_PIPE0_PRIORITY                                                                        0x2809
+#define regCP_MES_PIPE0_PRIORITY_BASE_IDX                                                               1
+#define regCP_MES_PIPE1_PRIORITY                                                                        0x280a
+#define regCP_MES_PIPE1_PRIORITY_BASE_IDX                                                               1
+#define regCP_MES_PIPE2_PRIORITY                                                                        0x280b
+#define regCP_MES_PIPE2_PRIORITY_BASE_IDX                                                               1
+#define regCP_MES_PIPE3_PRIORITY                                                                        0x280c
+#define regCP_MES_PIPE3_PRIORITY_BASE_IDX                                                               1
+#define regCP_MES_HEADER_DUMP                                                                           0x280d
+#define regCP_MES_HEADER_DUMP_BASE_IDX                                                                  1
+#define regCP_MES_MIE_LO                                                                                0x280e
+#define regCP_MES_MIE_LO_BASE_IDX                                                                       1
+#define regCP_MES_MIE_HI                                                                                0x280f
+#define regCP_MES_MIE_HI_BASE_IDX                                                                       1
+#define regCP_MES_INTERRUPT                                                                             0x2810
+#define regCP_MES_INTERRUPT_BASE_IDX                                                                    1
+#define regCP_MES_SCRATCH_INDEX                                                                         0x2811
+#define regCP_MES_SCRATCH_INDEX_BASE_IDX                                                                1
+#define regCP_MES_SCRATCH_DATA                                                                          0x2812
+#define regCP_MES_SCRATCH_DATA_BASE_IDX                                                                 1
+#define regCP_MES_INSTR_PNTR                                                                            0x2813
+#define regCP_MES_INSTR_PNTR_BASE_IDX                                                                   1
+#define regCP_MES_MSCRATCH_HI                                                                           0x2814
+#define regCP_MES_MSCRATCH_HI_BASE_IDX                                                                  1
+#define regCP_MES_MSCRATCH_LO                                                                           0x2815
+#define regCP_MES_MSCRATCH_LO_BASE_IDX                                                                  1
+#define regCP_MES_MSTATUS_LO                                                                            0x2816
+#define regCP_MES_MSTATUS_LO_BASE_IDX                                                                   1
+#define regCP_MES_MSTATUS_HI                                                                            0x2817
+#define regCP_MES_MSTATUS_HI_BASE_IDX                                                                   1
+#define regCP_MES_MEPC_LO                                                                               0x2818
+#define regCP_MES_MEPC_LO_BASE_IDX                                                                      1
+#define regCP_MES_MEPC_HI                                                                               0x2819
+#define regCP_MES_MEPC_HI_BASE_IDX                                                                      1
+#define regCP_MES_MCAUSE_LO                                                                             0x281a
+#define regCP_MES_MCAUSE_LO_BASE_IDX                                                                    1
+#define regCP_MES_MCAUSE_HI                                                                             0x281b
+#define regCP_MES_MCAUSE_HI_BASE_IDX                                                                    1
+#define regCP_MES_MBADADDR_LO                                                                           0x281c
+#define regCP_MES_MBADADDR_LO_BASE_IDX                                                                  1
+#define regCP_MES_MBADADDR_HI                                                                           0x281d
+#define regCP_MES_MBADADDR_HI_BASE_IDX                                                                  1
+#define regCP_MES_MIP_LO                                                                                0x281e
+#define regCP_MES_MIP_LO_BASE_IDX                                                                       1
+#define regCP_MES_MIP_HI                                                                                0x281f
+#define regCP_MES_MIP_HI_BASE_IDX                                                                       1
+#define regCP_MES_IC_OP_CNTL                                                                            0x2820
+#define regCP_MES_IC_OP_CNTL_BASE_IDX                                                                   1
+#define regCP_MES_MCYCLE_LO                                                                             0x2826
+#define regCP_MES_MCYCLE_LO_BASE_IDX                                                                    1
+#define regCP_MES_MCYCLE_HI                                                                             0x2827
+#define regCP_MES_MCYCLE_HI_BASE_IDX                                                                    1
+#define regCP_MES_MTIME_LO                                                                              0x2828
+#define regCP_MES_MTIME_LO_BASE_IDX                                                                     1
+#define regCP_MES_MTIME_HI                                                                              0x2829
+#define regCP_MES_MTIME_HI_BASE_IDX                                                                     1
+#define regCP_MES_MINSTRET_LO                                                                           0x282a
+#define regCP_MES_MINSTRET_LO_BASE_IDX                                                                  1
+#define regCP_MES_MINSTRET_HI                                                                           0x282b
+#define regCP_MES_MINSTRET_HI_BASE_IDX                                                                  1
+#define regCP_MES_MISA_LO                                                                               0x282c
+#define regCP_MES_MISA_LO_BASE_IDX                                                                      1
+#define regCP_MES_MISA_HI                                                                               0x282d
+#define regCP_MES_MISA_HI_BASE_IDX                                                                      1
+#define regCP_MES_MVENDORID_LO                                                                          0x282e
+#define regCP_MES_MVENDORID_LO_BASE_IDX                                                                 1
+#define regCP_MES_MVENDORID_HI                                                                          0x282f
+#define regCP_MES_MVENDORID_HI_BASE_IDX                                                                 1
+#define regCP_MES_MARCHID_LO                                                                            0x2830
+#define regCP_MES_MARCHID_LO_BASE_IDX                                                                   1
+#define regCP_MES_MARCHID_HI                                                                            0x2831
+#define regCP_MES_MARCHID_HI_BASE_IDX                                                                   1
+#define regCP_MES_MIMPID_LO                                                                             0x2832
+#define regCP_MES_MIMPID_LO_BASE_IDX                                                                    1
+#define regCP_MES_MIMPID_HI                                                                             0x2833
+#define regCP_MES_MIMPID_HI_BASE_IDX                                                                    1
+#define regCP_MES_MHARTID_LO                                                                            0x2834
+#define regCP_MES_MHARTID_LO_BASE_IDX                                                                   1
+#define regCP_MES_MHARTID_HI                                                                            0x2835
+#define regCP_MES_MHARTID_HI_BASE_IDX                                                                   1
+#define regCP_MES_DC_BASE_CNTL                                                                          0x2836
+#define regCP_MES_DC_BASE_CNTL_BASE_IDX                                                                 1
+#define regCP_MES_DC_OP_CNTL                                                                            0x2837
+#define regCP_MES_DC_OP_CNTL_BASE_IDX                                                                   1
+#define regCP_MES_MTIMECMP_LO                                                                           0x2838
+#define regCP_MES_MTIMECMP_LO_BASE_IDX                                                                  1
+#define regCP_MES_MTIMECMP_HI                                                                           0x2839
+#define regCP_MES_MTIMECMP_HI_BASE_IDX                                                                  1
+#define regCP_MES_PROCESS_QUANTUM_PIPE0                                                                 0x283a
+#define regCP_MES_PROCESS_QUANTUM_PIPE0_BASE_IDX                                                        1
+#define regCP_MES_PROCESS_QUANTUM_PIPE1                                                                 0x283b
+#define regCP_MES_PROCESS_QUANTUM_PIPE1_BASE_IDX                                                        1
+#define regCP_MES_DOORBELL_CONTROL1                                                                     0x283c
+#define regCP_MES_DOORBELL_CONTROL1_BASE_IDX                                                            1
+#define regCP_MES_DOORBELL_CONTROL2                                                                     0x283d
+#define regCP_MES_DOORBELL_CONTROL2_BASE_IDX                                                            1
+#define regCP_MES_DOORBELL_CONTROL3                                                                     0x283e
+#define regCP_MES_DOORBELL_CONTROL3_BASE_IDX                                                            1
+#define regCP_MES_DOORBELL_CONTROL4                                                                     0x283f
+#define regCP_MES_DOORBELL_CONTROL4_BASE_IDX                                                            1
+#define regCP_MES_DOORBELL_CONTROL5                                                                     0x2840
+#define regCP_MES_DOORBELL_CONTROL5_BASE_IDX                                                            1
+#define regCP_MES_DOORBELL_CONTROL6                                                                     0x2841
+#define regCP_MES_DOORBELL_CONTROL6_BASE_IDX                                                            1
+#define regCP_MES_GP0_LO                                                                                0x2843
+#define regCP_MES_GP0_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP0_HI                                                                                0x2844
+#define regCP_MES_GP0_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP1_LO                                                                                0x2845
+#define regCP_MES_GP1_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP1_HI                                                                                0x2846
+#define regCP_MES_GP1_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP2_LO                                                                                0x2847
+#define regCP_MES_GP2_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP2_HI                                                                                0x2848
+#define regCP_MES_GP2_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP3_LO                                                                                0x2849
+#define regCP_MES_GP3_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP3_HI                                                                                0x284a
+#define regCP_MES_GP3_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP4_LO                                                                                0x284b
+#define regCP_MES_GP4_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP4_HI                                                                                0x284c
+#define regCP_MES_GP4_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP5_LO                                                                                0x284d
+#define regCP_MES_GP5_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP5_HI                                                                                0x284e
+#define regCP_MES_GP5_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP6_LO                                                                                0x284f
+#define regCP_MES_GP6_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP6_HI                                                                                0x2850
+#define regCP_MES_GP6_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP7_LO                                                                                0x2851
+#define regCP_MES_GP7_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP7_HI                                                                                0x2852
+#define regCP_MES_GP7_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP8_LO                                                                                0x2853
+#define regCP_MES_GP8_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP8_HI                                                                                0x2854
+#define regCP_MES_GP8_HI_BASE_IDX                                                                       1
+#define regCP_MES_GP9_LO                                                                                0x2855
+#define regCP_MES_GP9_LO_BASE_IDX                                                                       1
+#define regCP_MES_GP9_HI                                                                                0x2856
+#define regCP_MES_GP9_HI_BASE_IDX                                                                       1
+#define regCP_MES_LOCAL_BASE0_LO                                                                        0x2883
+#define regCP_MES_LOCAL_BASE0_LO_BASE_IDX                                                               1
+#define regCP_MES_LOCAL_BASE0_HI                                                                        0x2884
+#define regCP_MES_LOCAL_BASE0_HI_BASE_IDX                                                               1
+#define regCP_MES_LOCAL_MASK0_LO                                                                        0x2885
+#define regCP_MES_LOCAL_MASK0_LO_BASE_IDX                                                               1
+#define regCP_MES_LOCAL_MASK0_HI                                                                        0x2886
+#define regCP_MES_LOCAL_MASK0_HI_BASE_IDX                                                               1
+#define regCP_MES_LOCAL_APERTURE                                                                        0x2887
+#define regCP_MES_LOCAL_APERTURE_BASE_IDX                                                               1
+#define regCP_MES_LOCAL_INSTR_BASE_LO                                                                   0x2888
+#define regCP_MES_LOCAL_INSTR_BASE_LO_BASE_IDX                                                          1
+#define regCP_MES_LOCAL_INSTR_BASE_HI                                                                   0x2889
+#define regCP_MES_LOCAL_INSTR_BASE_HI_BASE_IDX                                                          1
+#define regCP_MES_LOCAL_INSTR_MASK_LO                                                                   0x288a
+#define regCP_MES_LOCAL_INSTR_MASK_LO_BASE_IDX                                                          1
+#define regCP_MES_LOCAL_INSTR_MASK_HI                                                                   0x288b
+#define regCP_MES_LOCAL_INSTR_MASK_HI_BASE_IDX                                                          1
+#define regCP_MES_LOCAL_INSTR_APERTURE                                                                  0x288c
+#define regCP_MES_LOCAL_INSTR_APERTURE_BASE_IDX                                                         1
+#define regCP_MES_LOCAL_SCRATCH_APERTURE                                                                0x288d
+#define regCP_MES_LOCAL_SCRATCH_APERTURE_BASE_IDX                                                       1
+#define regCP_MES_LOCAL_SCRATCH_BASE_LO                                                                 0x288e
+#define regCP_MES_LOCAL_SCRATCH_BASE_LO_BASE_IDX                                                        1
+#define regCP_MES_LOCAL_SCRATCH_BASE_HI                                                                 0x288f
+#define regCP_MES_LOCAL_SCRATCH_BASE_HI_BASE_IDX                                                        1
+#define regCP_MES_PERFCOUNT_CNTL                                                                        0x2899
+#define regCP_MES_PERFCOUNT_CNTL_BASE_IDX                                                               1
+#define regCP_MES_PENDING_INTERRUPT                                                                     0x289a
+#define regCP_MES_PENDING_INTERRUPT_BASE_IDX                                                            1
+#define regCP_MES_PRGRM_CNTR_START_HI                                                                   0x289d
+#define regCP_MES_PRGRM_CNTR_START_HI_BASE_IDX                                                          1
+#define regCP_MES_INTERRUPT_DATA_16                                                                     0x289f
+#define regCP_MES_INTERRUPT_DATA_16_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_17                                                                     0x28a0
+#define regCP_MES_INTERRUPT_DATA_17_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_18                                                                     0x28a1
+#define regCP_MES_INTERRUPT_DATA_18_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_19                                                                     0x28a2
+#define regCP_MES_INTERRUPT_DATA_19_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_20                                                                     0x28a3
+#define regCP_MES_INTERRUPT_DATA_20_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_21                                                                     0x28a4
+#define regCP_MES_INTERRUPT_DATA_21_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_22                                                                     0x28a5
+#define regCP_MES_INTERRUPT_DATA_22_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_23                                                                     0x28a6
+#define regCP_MES_INTERRUPT_DATA_23_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_24                                                                     0x28a7
+#define regCP_MES_INTERRUPT_DATA_24_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_25                                                                     0x28a8
+#define regCP_MES_INTERRUPT_DATA_25_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_26                                                                     0x28a9
+#define regCP_MES_INTERRUPT_DATA_26_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_27                                                                     0x28aa
+#define regCP_MES_INTERRUPT_DATA_27_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_28                                                                     0x28ab
+#define regCP_MES_INTERRUPT_DATA_28_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_29                                                                     0x28ac
+#define regCP_MES_INTERRUPT_DATA_29_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_30                                                                     0x28ad
+#define regCP_MES_INTERRUPT_DATA_30_BASE_IDX                                                            1
+#define regCP_MES_INTERRUPT_DATA_31                                                                     0x28ae
+#define regCP_MES_INTERRUPT_DATA_31_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE0_BASE                                                                     0x28af
+#define regCP_MES_DC_APERTURE0_BASE_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE0_MASK                                                                     0x28b0
+#define regCP_MES_DC_APERTURE0_MASK_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE0_CNTL                                                                     0x28b1
+#define regCP_MES_DC_APERTURE0_CNTL_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE1_BASE                                                                     0x28b2
+#define regCP_MES_DC_APERTURE1_BASE_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE1_MASK                                                                     0x28b3
+#define regCP_MES_DC_APERTURE1_MASK_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE1_CNTL                                                                     0x28b4
+#define regCP_MES_DC_APERTURE1_CNTL_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE2_BASE                                                                     0x28b5
+#define regCP_MES_DC_APERTURE2_BASE_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE2_MASK                                                                     0x28b6
+#define regCP_MES_DC_APERTURE2_MASK_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE2_CNTL                                                                     0x28b7
+#define regCP_MES_DC_APERTURE2_CNTL_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE3_BASE                                                                     0x28b8
+#define regCP_MES_DC_APERTURE3_BASE_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE3_MASK                                                                     0x28b9
+#define regCP_MES_DC_APERTURE3_MASK_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE3_CNTL                                                                     0x28ba
+#define regCP_MES_DC_APERTURE3_CNTL_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE4_BASE                                                                     0x28bb
+#define regCP_MES_DC_APERTURE4_BASE_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE4_MASK                                                                     0x28bc
+#define regCP_MES_DC_APERTURE4_MASK_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE4_CNTL                                                                     0x28bd
+#define regCP_MES_DC_APERTURE4_CNTL_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE5_BASE                                                                     0x28be
+#define regCP_MES_DC_APERTURE5_BASE_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE5_MASK                                                                     0x28bf
+#define regCP_MES_DC_APERTURE5_MASK_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE5_CNTL                                                                     0x28c0
+#define regCP_MES_DC_APERTURE5_CNTL_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE6_BASE                                                                     0x28c1
+#define regCP_MES_DC_APERTURE6_BASE_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE6_MASK                                                                     0x28c2
+#define regCP_MES_DC_APERTURE6_MASK_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE6_CNTL                                                                     0x28c3
+#define regCP_MES_DC_APERTURE6_CNTL_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE7_BASE                                                                     0x28c4
+#define regCP_MES_DC_APERTURE7_BASE_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE7_MASK                                                                     0x28c5
+#define regCP_MES_DC_APERTURE7_MASK_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE7_CNTL                                                                     0x28c6
+#define regCP_MES_DC_APERTURE7_CNTL_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE8_BASE                                                                     0x28c7
+#define regCP_MES_DC_APERTURE8_BASE_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE8_MASK                                                                     0x28c8
+#define regCP_MES_DC_APERTURE8_MASK_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE8_CNTL                                                                     0x28c9
+#define regCP_MES_DC_APERTURE8_CNTL_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE9_BASE                                                                     0x28ca
+#define regCP_MES_DC_APERTURE9_BASE_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE9_MASK                                                                     0x28cb
+#define regCP_MES_DC_APERTURE9_MASK_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE9_CNTL                                                                     0x28cc
+#define regCP_MES_DC_APERTURE9_CNTL_BASE_IDX                                                            1
+#define regCP_MES_DC_APERTURE10_BASE                                                                    0x28cd
+#define regCP_MES_DC_APERTURE10_BASE_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE10_MASK                                                                    0x28ce
+#define regCP_MES_DC_APERTURE10_MASK_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE10_CNTL                                                                    0x28cf
+#define regCP_MES_DC_APERTURE10_CNTL_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE11_BASE                                                                    0x28d0
+#define regCP_MES_DC_APERTURE11_BASE_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE11_MASK                                                                    0x28d1
+#define regCP_MES_DC_APERTURE11_MASK_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE11_CNTL                                                                    0x28d2
+#define regCP_MES_DC_APERTURE11_CNTL_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE12_BASE                                                                    0x28d3
+#define regCP_MES_DC_APERTURE12_BASE_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE12_MASK                                                                    0x28d4
+#define regCP_MES_DC_APERTURE12_MASK_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE12_CNTL                                                                    0x28d5
+#define regCP_MES_DC_APERTURE12_CNTL_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE13_BASE                                                                    0x28d6
+#define regCP_MES_DC_APERTURE13_BASE_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE13_MASK                                                                    0x28d7
+#define regCP_MES_DC_APERTURE13_MASK_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE13_CNTL                                                                    0x28d8
+#define regCP_MES_DC_APERTURE13_CNTL_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE14_BASE                                                                    0x28d9
+#define regCP_MES_DC_APERTURE14_BASE_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE14_MASK                                                                    0x28da
+#define regCP_MES_DC_APERTURE14_MASK_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE14_CNTL                                                                    0x28db
+#define regCP_MES_DC_APERTURE14_CNTL_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE15_BASE                                                                    0x28dc
+#define regCP_MES_DC_APERTURE15_BASE_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE15_MASK                                                                    0x28dd
+#define regCP_MES_DC_APERTURE15_MASK_BASE_IDX                                                           1
+#define regCP_MES_DC_APERTURE15_CNTL                                                                    0x28de
+#define regCP_MES_DC_APERTURE15_CNTL_BASE_IDX                                                           1
+#define regCP_MEC_RS64_PRGRM_CNTR_START                                                                 0x2900
+#define regCP_MEC_RS64_PRGRM_CNTR_START_BASE_IDX                                                        1
+#define regCP_MEC_MTVEC_LO                                                                              0x2901
+#define regCP_MEC_MTVEC_LO_BASE_IDX                                                                     1
+#define regCP_MEC_MTVEC_HI                                                                              0x2902
+#define regCP_MEC_MTVEC_HI_BASE_IDX                                                                     1
+#define regCP_MEC_ISA_CNTL                                                                              0x2903
+#define regCP_MEC_ISA_CNTL_BASE_IDX                                                                     1
+#define regCP_MEC_RS64_CNTL                                                                             0x2904
+#define regCP_MEC_RS64_CNTL_BASE_IDX                                                                    1
+#define regCP_MEC_MIE_LO                                                                                0x2905
+#define regCP_MEC_MIE_LO_BASE_IDX                                                                       1
+#define regCP_MEC_MIE_HI                                                                                0x2906
+#define regCP_MEC_MIE_HI_BASE_IDX                                                                       1
+#define regCP_MEC_RS64_INTERRUPT                                                                        0x2907
+#define regCP_MEC_RS64_INTERRUPT_BASE_IDX                                                               1
+#define regCP_MEC_RS64_INSTR_PNTR                                                                       0x2908
+#define regCP_MEC_RS64_INSTR_PNTR_BASE_IDX                                                              1
+#define regCP_MEC_MIP_LO                                                                                0x2909
+#define regCP_MEC_MIP_LO_BASE_IDX                                                                       1
+#define regCP_MEC_MIP_HI                                                                                0x290a
+#define regCP_MEC_MIP_HI_BASE_IDX                                                                       1
+#define regCP_MEC_DC_BASE_CNTL                                                                          0x290b
+#define regCP_MEC_DC_BASE_CNTL_BASE_IDX                                                                 1
+#define regCP_MEC_DC_OP_CNTL                                                                            0x290c
+#define regCP_MEC_DC_OP_CNTL_BASE_IDX                                                                   1
+#define regCP_MEC_MTIMECMP_LO                                                                           0x290d
+#define regCP_MEC_MTIMECMP_LO_BASE_IDX                                                                  1
+#define regCP_MEC_MTIMECMP_HI                                                                           0x290e
+#define regCP_MEC_MTIMECMP_HI_BASE_IDX                                                                  1
+#define regCP_MEC_GP0_LO                                                                                0x2910
+#define regCP_MEC_GP0_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP0_HI                                                                                0x2911
+#define regCP_MEC_GP0_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP1_LO                                                                                0x2912
+#define regCP_MEC_GP1_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP1_HI                                                                                0x2913
+#define regCP_MEC_GP1_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP2_LO                                                                                0x2914
+#define regCP_MEC_GP2_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP2_HI                                                                                0x2915
+#define regCP_MEC_GP2_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP3_LO                                                                                0x2916
+#define regCP_MEC_GP3_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP3_HI                                                                                0x2917
+#define regCP_MEC_GP3_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP4_LO                                                                                0x2918
+#define regCP_MEC_GP4_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP4_HI                                                                                0x2919
+#define regCP_MEC_GP4_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP5_LO                                                                                0x291a
+#define regCP_MEC_GP5_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP5_HI                                                                                0x291b
+#define regCP_MEC_GP5_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP6_LO                                                                                0x291c
+#define regCP_MEC_GP6_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP6_HI                                                                                0x291d
+#define regCP_MEC_GP6_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP7_LO                                                                                0x291e
+#define regCP_MEC_GP7_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP7_HI                                                                                0x291f
+#define regCP_MEC_GP7_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP8_LO                                                                                0x2920
+#define regCP_MEC_GP8_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP8_HI                                                                                0x2921
+#define regCP_MEC_GP8_HI_BASE_IDX                                                                       1
+#define regCP_MEC_GP9_LO                                                                                0x2922
+#define regCP_MEC_GP9_LO_BASE_IDX                                                                       1
+#define regCP_MEC_GP9_HI                                                                                0x2923
+#define regCP_MEC_GP9_HI_BASE_IDX                                                                       1
+#define regCP_MEC_LOCAL_BASE0_LO                                                                        0x2927
+#define regCP_MEC_LOCAL_BASE0_LO_BASE_IDX                                                               1
+#define regCP_MEC_LOCAL_BASE0_HI                                                                        0x2928
+#define regCP_MEC_LOCAL_BASE0_HI_BASE_IDX                                                               1
+#define regCP_MEC_LOCAL_MASK0_LO                                                                        0x2929
+#define regCP_MEC_LOCAL_MASK0_LO_BASE_IDX                                                               1
+#define regCP_MEC_LOCAL_MASK0_HI                                                                        0x292a
+#define regCP_MEC_LOCAL_MASK0_HI_BASE_IDX                                                               1
+#define regCP_MEC_LOCAL_APERTURE                                                                        0x292b
+#define regCP_MEC_LOCAL_APERTURE_BASE_IDX                                                               1
+#define regCP_MEC_LOCAL_INSTR_BASE_LO                                                                   0x292c
+#define regCP_MEC_LOCAL_INSTR_BASE_LO_BASE_IDX                                                          1
+#define regCP_MEC_LOCAL_INSTR_BASE_HI                                                                   0x292d
+#define regCP_MEC_LOCAL_INSTR_BASE_HI_BASE_IDX                                                          1
+#define regCP_MEC_LOCAL_INSTR_MASK_LO                                                                   0x292e
+#define regCP_MEC_LOCAL_INSTR_MASK_LO_BASE_IDX                                                          1
+#define regCP_MEC_LOCAL_INSTR_MASK_HI                                                                   0x292f
+#define regCP_MEC_LOCAL_INSTR_MASK_HI_BASE_IDX                                                          1
+#define regCP_MEC_LOCAL_INSTR_APERTURE                                                                  0x2930
+#define regCP_MEC_LOCAL_INSTR_APERTURE_BASE_IDX                                                         1
+#define regCP_MEC_LOCAL_SCRATCH_APERTURE                                                                0x2931
+#define regCP_MEC_LOCAL_SCRATCH_APERTURE_BASE_IDX                                                       1
+#define regCP_MEC_LOCAL_SCRATCH_BASE_LO                                                                 0x2932
+#define regCP_MEC_LOCAL_SCRATCH_BASE_LO_BASE_IDX                                                        1
+#define regCP_MEC_LOCAL_SCRATCH_BASE_HI                                                                 0x2933
+#define regCP_MEC_LOCAL_SCRATCH_BASE_HI_BASE_IDX                                                        1
+#define regCP_MEC_RS64_PERFCOUNT_CNTL                                                                   0x2934
+#define regCP_MEC_RS64_PERFCOUNT_CNTL_BASE_IDX                                                          1
+#define regCP_MEC_RS64_PENDING_INTERRUPT                                                                0x2935
+#define regCP_MEC_RS64_PENDING_INTERRUPT_BASE_IDX                                                       1
+#define regCP_MEC_RS64_PRGRM_CNTR_START_HI                                                              0x2938
+#define regCP_MEC_RS64_PRGRM_CNTR_START_HI_BASE_IDX                                                     1
+#define regCP_MEC_RS64_INTERRUPT_DATA_16                                                                0x293a
+#define regCP_MEC_RS64_INTERRUPT_DATA_16_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_17                                                                0x293b
+#define regCP_MEC_RS64_INTERRUPT_DATA_17_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_18                                                                0x293c
+#define regCP_MEC_RS64_INTERRUPT_DATA_18_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_19                                                                0x293d
+#define regCP_MEC_RS64_INTERRUPT_DATA_19_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_20                                                                0x293e
+#define regCP_MEC_RS64_INTERRUPT_DATA_20_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_21                                                                0x293f
+#define regCP_MEC_RS64_INTERRUPT_DATA_21_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_22                                                                0x2940
+#define regCP_MEC_RS64_INTERRUPT_DATA_22_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_23                                                                0x2941
+#define regCP_MEC_RS64_INTERRUPT_DATA_23_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_24                                                                0x2942
+#define regCP_MEC_RS64_INTERRUPT_DATA_24_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_25                                                                0x2943
+#define regCP_MEC_RS64_INTERRUPT_DATA_25_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_26                                                                0x2944
+#define regCP_MEC_RS64_INTERRUPT_DATA_26_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_27                                                                0x2945
+#define regCP_MEC_RS64_INTERRUPT_DATA_27_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_28                                                                0x2946
+#define regCP_MEC_RS64_INTERRUPT_DATA_28_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_29                                                                0x2947
+#define regCP_MEC_RS64_INTERRUPT_DATA_29_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_30                                                                0x2948
+#define regCP_MEC_RS64_INTERRUPT_DATA_30_BASE_IDX                                                       1
+#define regCP_MEC_RS64_INTERRUPT_DATA_31                                                                0x2949
+#define regCP_MEC_RS64_INTERRUPT_DATA_31_BASE_IDX                                                       1
+#define regCP_MEC_DC_APERTURE0_BASE                                                                     0x294a
+#define regCP_MEC_DC_APERTURE0_BASE_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE0_MASK                                                                     0x294b
+#define regCP_MEC_DC_APERTURE0_MASK_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE0_CNTL                                                                     0x294c
+#define regCP_MEC_DC_APERTURE0_CNTL_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE1_BASE                                                                     0x294d
+#define regCP_MEC_DC_APERTURE1_BASE_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE1_MASK                                                                     0x294e
+#define regCP_MEC_DC_APERTURE1_MASK_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE1_CNTL                                                                     0x294f
+#define regCP_MEC_DC_APERTURE1_CNTL_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE2_BASE                                                                     0x2950
+#define regCP_MEC_DC_APERTURE2_BASE_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE2_MASK                                                                     0x2951
+#define regCP_MEC_DC_APERTURE2_MASK_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE2_CNTL                                                                     0x2952
+#define regCP_MEC_DC_APERTURE2_CNTL_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE3_BASE                                                                     0x2953
+#define regCP_MEC_DC_APERTURE3_BASE_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE3_MASK                                                                     0x2954
+#define regCP_MEC_DC_APERTURE3_MASK_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE3_CNTL                                                                     0x2955
+#define regCP_MEC_DC_APERTURE3_CNTL_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE4_BASE                                                                     0x2956
+#define regCP_MEC_DC_APERTURE4_BASE_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE4_MASK                                                                     0x2957
+#define regCP_MEC_DC_APERTURE4_MASK_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE4_CNTL                                                                     0x2958
+#define regCP_MEC_DC_APERTURE4_CNTL_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE5_BASE                                                                     0x2959
+#define regCP_MEC_DC_APERTURE5_BASE_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE5_MASK                                                                     0x295a
+#define regCP_MEC_DC_APERTURE5_MASK_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE5_CNTL                                                                     0x295b
+#define regCP_MEC_DC_APERTURE5_CNTL_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE6_BASE                                                                     0x295c
+#define regCP_MEC_DC_APERTURE6_BASE_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE6_MASK                                                                     0x295d
+#define regCP_MEC_DC_APERTURE6_MASK_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE6_CNTL                                                                     0x295e
+#define regCP_MEC_DC_APERTURE6_CNTL_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE7_BASE                                                                     0x295f
+#define regCP_MEC_DC_APERTURE7_BASE_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE7_MASK                                                                     0x2960
+#define regCP_MEC_DC_APERTURE7_MASK_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE7_CNTL                                                                     0x2961
+#define regCP_MEC_DC_APERTURE7_CNTL_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE8_BASE                                                                     0x2962
+#define regCP_MEC_DC_APERTURE8_BASE_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE8_MASK                                                                     0x2963
+#define regCP_MEC_DC_APERTURE8_MASK_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE8_CNTL                                                                     0x2964
+#define regCP_MEC_DC_APERTURE8_CNTL_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE9_BASE                                                                     0x2965
+#define regCP_MEC_DC_APERTURE9_BASE_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE9_MASK                                                                     0x2966
+#define regCP_MEC_DC_APERTURE9_MASK_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE9_CNTL                                                                     0x2967
+#define regCP_MEC_DC_APERTURE9_CNTL_BASE_IDX                                                            1
+#define regCP_MEC_DC_APERTURE10_BASE                                                                    0x2968
+#define regCP_MEC_DC_APERTURE10_BASE_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE10_MASK                                                                    0x2969
+#define regCP_MEC_DC_APERTURE10_MASK_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE10_CNTL                                                                    0x296a
+#define regCP_MEC_DC_APERTURE10_CNTL_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE11_BASE                                                                    0x296b
+#define regCP_MEC_DC_APERTURE11_BASE_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE11_MASK                                                                    0x296c
+#define regCP_MEC_DC_APERTURE11_MASK_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE11_CNTL                                                                    0x296d
+#define regCP_MEC_DC_APERTURE11_CNTL_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE12_BASE                                                                    0x296e
+#define regCP_MEC_DC_APERTURE12_BASE_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE12_MASK                                                                    0x296f
+#define regCP_MEC_DC_APERTURE12_MASK_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE12_CNTL                                                                    0x2970
+#define regCP_MEC_DC_APERTURE12_CNTL_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE13_BASE                                                                    0x2971
+#define regCP_MEC_DC_APERTURE13_BASE_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE13_MASK                                                                    0x2972
+#define regCP_MEC_DC_APERTURE13_MASK_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE13_CNTL                                                                    0x2973
+#define regCP_MEC_DC_APERTURE13_CNTL_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE14_BASE                                                                    0x2974
+#define regCP_MEC_DC_APERTURE14_BASE_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE14_MASK                                                                    0x2975
+#define regCP_MEC_DC_APERTURE14_MASK_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE14_CNTL                                                                    0x2976
+#define regCP_MEC_DC_APERTURE14_CNTL_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE15_BASE                                                                    0x2977
+#define regCP_MEC_DC_APERTURE15_BASE_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE15_MASK                                                                    0x2978
+#define regCP_MEC_DC_APERTURE15_MASK_BASE_IDX                                                           1
+#define regCP_MEC_DC_APERTURE15_CNTL                                                                    0x2979
+#define regCP_MEC_DC_APERTURE15_CNTL_BASE_IDX                                                           1
+#define regCP_CPC_IC_OP_CNTL                                                                            0x297a
+#define regCP_CPC_IC_OP_CNTL_BASE_IDX                                                                   1
+#define regCP_GFX_CNTL                                                                                  0x2a00
+#define regCP_GFX_CNTL_BASE_IDX                                                                         1
+#define regCP_GFX_RS64_INTERRUPT0                                                                       0x2a01
+#define regCP_GFX_RS64_INTERRUPT0_BASE_IDX                                                              1
+#define regCP_GFX_RS64_INTR_EN0                                                                         0x2a02
+#define regCP_GFX_RS64_INTR_EN0_BASE_IDX                                                                1
+#define regCP_GFX_RS64_INTR_EN1                                                                         0x2a03
+#define regCP_GFX_RS64_INTR_EN1_BASE_IDX                                                                1
+#define regCP_GFX_RS64_DC_BASE_CNTL                                                                     0x2a08
+#define regCP_GFX_RS64_DC_BASE_CNTL_BASE_IDX                                                            1
+#define regCP_GFX_RS64_DC_OP_CNTL                                                                       0x2a09
+#define regCP_GFX_RS64_DC_OP_CNTL_BASE_IDX                                                              1
+#define regCP_GFX_RS64_LOCAL_BASE0_LO                                                                   0x2a0a
+#define regCP_GFX_RS64_LOCAL_BASE0_LO_BASE_IDX                                                          1
+#define regCP_GFX_RS64_LOCAL_BASE0_HI                                                                   0x2a0b
+#define regCP_GFX_RS64_LOCAL_BASE0_HI_BASE_IDX                                                          1
+#define regCP_GFX_RS64_LOCAL_MASK0_LO                                                                   0x2a0c
+#define regCP_GFX_RS64_LOCAL_MASK0_LO_BASE_IDX                                                          1
+#define regCP_GFX_RS64_LOCAL_MASK0_HI                                                                   0x2a0d
+#define regCP_GFX_RS64_LOCAL_MASK0_HI_BASE_IDX                                                          1
+#define regCP_GFX_RS64_LOCAL_APERTURE                                                                   0x2a0e
+#define regCP_GFX_RS64_LOCAL_APERTURE_BASE_IDX                                                          1
+#define regCP_GFX_RS64_LOCAL_INSTR_BASE_LO                                                              0x2a0f
+#define regCP_GFX_RS64_LOCAL_INSTR_BASE_LO_BASE_IDX                                                     1
+#define regCP_GFX_RS64_LOCAL_INSTR_BASE_HI                                                              0x2a10
+#define regCP_GFX_RS64_LOCAL_INSTR_BASE_HI_BASE_IDX                                                     1
+#define regCP_GFX_RS64_LOCAL_INSTR_MASK_LO                                                              0x2a11
+#define regCP_GFX_RS64_LOCAL_INSTR_MASK_LO_BASE_IDX                                                     1
+#define regCP_GFX_RS64_LOCAL_INSTR_MASK_HI                                                              0x2a12
+#define regCP_GFX_RS64_LOCAL_INSTR_MASK_HI_BASE_IDX                                                     1
+#define regCP_GFX_RS64_LOCAL_INSTR_APERTURE                                                             0x2a13
+#define regCP_GFX_RS64_LOCAL_INSTR_APERTURE_BASE_IDX                                                    1
+#define regCP_GFX_RS64_LOCAL_SCRATCH_APERTURE                                                           0x2a14
+#define regCP_GFX_RS64_LOCAL_SCRATCH_APERTURE_BASE_IDX                                                  1
+#define regCP_GFX_RS64_LOCAL_SCRATCH_BASE_LO                                                            0x2a15
+#define regCP_GFX_RS64_LOCAL_SCRATCH_BASE_LO_BASE_IDX                                                   1
+#define regCP_GFX_RS64_LOCAL_SCRATCH_BASE_HI                                                            0x2a16
+#define regCP_GFX_RS64_LOCAL_SCRATCH_BASE_HI_BASE_IDX                                                   1
+#define regCP_GFX_RS64_PERFCOUNT_CNTL0                                                                  0x2a1a
+#define regCP_GFX_RS64_PERFCOUNT_CNTL0_BASE_IDX                                                         1
+#define regCP_GFX_RS64_PERFCOUNT_CNTL1                                                                  0x2a1b
+#define regCP_GFX_RS64_PERFCOUNT_CNTL1_BASE_IDX                                                         1
+#define regCP_GFX_RS64_MIP_LO0                                                                          0x2a1c
+#define regCP_GFX_RS64_MIP_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_MIP_LO1                                                                          0x2a1d
+#define regCP_GFX_RS64_MIP_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_MIP_HI0                                                                          0x2a1e
+#define regCP_GFX_RS64_MIP_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_MIP_HI1                                                                          0x2a1f
+#define regCP_GFX_RS64_MIP_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_MTIMECMP_LO0                                                                     0x2a20
+#define regCP_GFX_RS64_MTIMECMP_LO0_BASE_IDX                                                            1
+#define regCP_GFX_RS64_MTIMECMP_LO1                                                                     0x2a21
+#define regCP_GFX_RS64_MTIMECMP_LO1_BASE_IDX                                                            1
+#define regCP_GFX_RS64_MTIMECMP_HI0                                                                     0x2a22
+#define regCP_GFX_RS64_MTIMECMP_HI0_BASE_IDX                                                            1
+#define regCP_GFX_RS64_MTIMECMP_HI1                                                                     0x2a23
+#define regCP_GFX_RS64_MTIMECMP_HI1_BASE_IDX                                                            1
+#define regCP_GFX_RS64_GP0_LO0                                                                          0x2a24
+#define regCP_GFX_RS64_GP0_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP0_LO1                                                                          0x2a25
+#define regCP_GFX_RS64_GP0_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP0_HI0                                                                          0x2a26
+#define regCP_GFX_RS64_GP0_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP0_HI1                                                                          0x2a27
+#define regCP_GFX_RS64_GP0_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP1_LO0                                                                          0x2a28
+#define regCP_GFX_RS64_GP1_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP1_LO1                                                                          0x2a29
+#define regCP_GFX_RS64_GP1_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP1_HI0                                                                          0x2a2a
+#define regCP_GFX_RS64_GP1_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP1_HI1                                                                          0x2a2b
+#define regCP_GFX_RS64_GP1_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP2_LO0                                                                          0x2a2c
+#define regCP_GFX_RS64_GP2_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP2_LO1                                                                          0x2a2d
+#define regCP_GFX_RS64_GP2_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP2_HI0                                                                          0x2a2e
+#define regCP_GFX_RS64_GP2_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP2_HI1                                                                          0x2a2f
+#define regCP_GFX_RS64_GP2_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP3_LO0                                                                          0x2a30
+#define regCP_GFX_RS64_GP3_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP3_LO1                                                                          0x2a31
+#define regCP_GFX_RS64_GP3_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP3_HI0                                                                          0x2a32
+#define regCP_GFX_RS64_GP3_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP3_HI1                                                                          0x2a33
+#define regCP_GFX_RS64_GP3_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP4_LO0                                                                          0x2a34
+#define regCP_GFX_RS64_GP4_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP4_LO1                                                                          0x2a35
+#define regCP_GFX_RS64_GP4_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP4_HI0                                                                          0x2a36
+#define regCP_GFX_RS64_GP4_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP4_HI1                                                                          0x2a37
+#define regCP_GFX_RS64_GP4_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP5_LO0                                                                          0x2a38
+#define regCP_GFX_RS64_GP5_LO0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP5_LO1                                                                          0x2a39
+#define regCP_GFX_RS64_GP5_LO1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP5_HI0                                                                          0x2a3a
+#define regCP_GFX_RS64_GP5_HI0_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP5_HI1                                                                          0x2a3b
+#define regCP_GFX_RS64_GP5_HI1_BASE_IDX                                                                 1
+#define regCP_GFX_RS64_GP6_LO                                                                           0x2a3c
+#define regCP_GFX_RS64_GP6_LO_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP6_HI                                                                           0x2a3d
+#define regCP_GFX_RS64_GP6_HI_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP7_LO                                                                           0x2a3e
+#define regCP_GFX_RS64_GP7_LO_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP7_HI                                                                           0x2a3f
+#define regCP_GFX_RS64_GP7_HI_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP8_LO                                                                           0x2a40
+#define regCP_GFX_RS64_GP8_LO_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP8_HI                                                                           0x2a41
+#define regCP_GFX_RS64_GP8_HI_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP9_LO                                                                           0x2a42
+#define regCP_GFX_RS64_GP9_LO_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_GP9_HI                                                                           0x2a43
+#define regCP_GFX_RS64_GP9_HI_BASE_IDX                                                                  1
+#define regCP_GFX_RS64_INSTR_PNTR0                                                                      0x2a44
+#define regCP_GFX_RS64_INSTR_PNTR0_BASE_IDX                                                             1
+#define regCP_GFX_RS64_INSTR_PNTR1                                                                      0x2a45
+#define regCP_GFX_RS64_INSTR_PNTR1_BASE_IDX                                                             1
+#define regCP_GFX_RS64_PENDING_INTERRUPT0                                                               0x2a46
+#define regCP_GFX_RS64_PENDING_INTERRUPT0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_PENDING_INTERRUPT1                                                               0x2a47
+#define regCP_GFX_RS64_PENDING_INTERRUPT1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE0_BASE0                                                               0x2a49
+#define regCP_GFX_RS64_DC_APERTURE0_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE0_MASK0                                                               0x2a4a
+#define regCP_GFX_RS64_DC_APERTURE0_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE0_CNTL0                                                               0x2a4b
+#define regCP_GFX_RS64_DC_APERTURE0_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_BASE0                                                               0x2a4c
+#define regCP_GFX_RS64_DC_APERTURE1_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_MASK0                                                               0x2a4d
+#define regCP_GFX_RS64_DC_APERTURE1_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_CNTL0                                                               0x2a4e
+#define regCP_GFX_RS64_DC_APERTURE1_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_BASE0                                                               0x2a4f
+#define regCP_GFX_RS64_DC_APERTURE2_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_MASK0                                                               0x2a50
+#define regCP_GFX_RS64_DC_APERTURE2_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_CNTL0                                                               0x2a51
+#define regCP_GFX_RS64_DC_APERTURE2_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_BASE0                                                               0x2a52
+#define regCP_GFX_RS64_DC_APERTURE3_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_MASK0                                                               0x2a53
+#define regCP_GFX_RS64_DC_APERTURE3_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_CNTL0                                                               0x2a54
+#define regCP_GFX_RS64_DC_APERTURE3_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_BASE0                                                               0x2a55
+#define regCP_GFX_RS64_DC_APERTURE4_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_MASK0                                                               0x2a56
+#define regCP_GFX_RS64_DC_APERTURE4_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_CNTL0                                                               0x2a57
+#define regCP_GFX_RS64_DC_APERTURE4_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_BASE0                                                               0x2a58
+#define regCP_GFX_RS64_DC_APERTURE5_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_MASK0                                                               0x2a59
+#define regCP_GFX_RS64_DC_APERTURE5_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_CNTL0                                                               0x2a5a
+#define regCP_GFX_RS64_DC_APERTURE5_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_BASE0                                                               0x2a5b
+#define regCP_GFX_RS64_DC_APERTURE6_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_MASK0                                                               0x2a5c
+#define regCP_GFX_RS64_DC_APERTURE6_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_CNTL0                                                               0x2a5d
+#define regCP_GFX_RS64_DC_APERTURE6_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_BASE0                                                               0x2a5e
+#define regCP_GFX_RS64_DC_APERTURE7_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_MASK0                                                               0x2a5f
+#define regCP_GFX_RS64_DC_APERTURE7_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_CNTL0                                                               0x2a60
+#define regCP_GFX_RS64_DC_APERTURE7_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_BASE0                                                               0x2a61
+#define regCP_GFX_RS64_DC_APERTURE8_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_MASK0                                                               0x2a62
+#define regCP_GFX_RS64_DC_APERTURE8_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_CNTL0                                                               0x2a63
+#define regCP_GFX_RS64_DC_APERTURE8_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_BASE0                                                               0x2a64
+#define regCP_GFX_RS64_DC_APERTURE9_BASE0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_MASK0                                                               0x2a65
+#define regCP_GFX_RS64_DC_APERTURE9_MASK0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_CNTL0                                                               0x2a66
+#define regCP_GFX_RS64_DC_APERTURE9_CNTL0_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE10_BASE0                                                              0x2a67
+#define regCP_GFX_RS64_DC_APERTURE10_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE10_MASK0                                                              0x2a68
+#define regCP_GFX_RS64_DC_APERTURE10_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE10_CNTL0                                                              0x2a69
+#define regCP_GFX_RS64_DC_APERTURE10_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_BASE0                                                              0x2a6a
+#define regCP_GFX_RS64_DC_APERTURE11_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_MASK0                                                              0x2a6b
+#define regCP_GFX_RS64_DC_APERTURE11_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_CNTL0                                                              0x2a6c
+#define regCP_GFX_RS64_DC_APERTURE11_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_BASE0                                                              0x2a6d
+#define regCP_GFX_RS64_DC_APERTURE12_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_MASK0                                                              0x2a6e
+#define regCP_GFX_RS64_DC_APERTURE12_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_CNTL0                                                              0x2a6f
+#define regCP_GFX_RS64_DC_APERTURE12_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_BASE0                                                              0x2a70
+#define regCP_GFX_RS64_DC_APERTURE13_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_MASK0                                                              0x2a71
+#define regCP_GFX_RS64_DC_APERTURE13_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_CNTL0                                                              0x2a72
+#define regCP_GFX_RS64_DC_APERTURE13_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_BASE0                                                              0x2a73
+#define regCP_GFX_RS64_DC_APERTURE14_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_MASK0                                                              0x2a74
+#define regCP_GFX_RS64_DC_APERTURE14_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_CNTL0                                                              0x2a75
+#define regCP_GFX_RS64_DC_APERTURE14_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_BASE0                                                              0x2a76
+#define regCP_GFX_RS64_DC_APERTURE15_BASE0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_MASK0                                                              0x2a77
+#define regCP_GFX_RS64_DC_APERTURE15_MASK0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_CNTL0                                                              0x2a78
+#define regCP_GFX_RS64_DC_APERTURE15_CNTL0_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE0_BASE1                                                               0x2a79
+#define regCP_GFX_RS64_DC_APERTURE0_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE0_MASK1                                                               0x2a7a
+#define regCP_GFX_RS64_DC_APERTURE0_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE0_CNTL1                                                               0x2a7b
+#define regCP_GFX_RS64_DC_APERTURE0_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_BASE1                                                               0x2a7c
+#define regCP_GFX_RS64_DC_APERTURE1_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_MASK1                                                               0x2a7d
+#define regCP_GFX_RS64_DC_APERTURE1_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE1_CNTL1                                                               0x2a7e
+#define regCP_GFX_RS64_DC_APERTURE1_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_BASE1                                                               0x2a7f
+#define regCP_GFX_RS64_DC_APERTURE2_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_MASK1                                                               0x2a80
+#define regCP_GFX_RS64_DC_APERTURE2_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE2_CNTL1                                                               0x2a81
+#define regCP_GFX_RS64_DC_APERTURE2_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_BASE1                                                               0x2a82
+#define regCP_GFX_RS64_DC_APERTURE3_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_MASK1                                                               0x2a83
+#define regCP_GFX_RS64_DC_APERTURE3_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE3_CNTL1                                                               0x2a84
+#define regCP_GFX_RS64_DC_APERTURE3_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_BASE1                                                               0x2a85
+#define regCP_GFX_RS64_DC_APERTURE4_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_MASK1                                                               0x2a86
+#define regCP_GFX_RS64_DC_APERTURE4_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE4_CNTL1                                                               0x2a87
+#define regCP_GFX_RS64_DC_APERTURE4_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_BASE1                                                               0x2a88
+#define regCP_GFX_RS64_DC_APERTURE5_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_MASK1                                                               0x2a89
+#define regCP_GFX_RS64_DC_APERTURE5_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE5_CNTL1                                                               0x2a8a
+#define regCP_GFX_RS64_DC_APERTURE5_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_BASE1                                                               0x2a8b
+#define regCP_GFX_RS64_DC_APERTURE6_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_MASK1                                                               0x2a8c
+#define regCP_GFX_RS64_DC_APERTURE6_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE6_CNTL1                                                               0x2a8d
+#define regCP_GFX_RS64_DC_APERTURE6_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_BASE1                                                               0x2a8e
+#define regCP_GFX_RS64_DC_APERTURE7_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_MASK1                                                               0x2a8f
+#define regCP_GFX_RS64_DC_APERTURE7_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE7_CNTL1                                                               0x2a90
+#define regCP_GFX_RS64_DC_APERTURE7_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_BASE1                                                               0x2a91
+#define regCP_GFX_RS64_DC_APERTURE8_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_MASK1                                                               0x2a92
+#define regCP_GFX_RS64_DC_APERTURE8_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE8_CNTL1                                                               0x2a93
+#define regCP_GFX_RS64_DC_APERTURE8_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_BASE1                                                               0x2a94
+#define regCP_GFX_RS64_DC_APERTURE9_BASE1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_MASK1                                                               0x2a95
+#define regCP_GFX_RS64_DC_APERTURE9_MASK1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE9_CNTL1                                                               0x2a96
+#define regCP_GFX_RS64_DC_APERTURE9_CNTL1_BASE_IDX                                                      1
+#define regCP_GFX_RS64_DC_APERTURE10_BASE1                                                              0x2a97
+#define regCP_GFX_RS64_DC_APERTURE10_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE10_MASK1                                                              0x2a98
+#define regCP_GFX_RS64_DC_APERTURE10_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE10_CNTL1                                                              0x2a99
+#define regCP_GFX_RS64_DC_APERTURE10_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_BASE1                                                              0x2a9a
+#define regCP_GFX_RS64_DC_APERTURE11_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_MASK1                                                              0x2a9b
+#define regCP_GFX_RS64_DC_APERTURE11_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE11_CNTL1                                                              0x2a9c
+#define regCP_GFX_RS64_DC_APERTURE11_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_BASE1                                                              0x2a9d
+#define regCP_GFX_RS64_DC_APERTURE12_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_MASK1                                                              0x2a9e
+#define regCP_GFX_RS64_DC_APERTURE12_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE12_CNTL1                                                              0x2a9f
+#define regCP_GFX_RS64_DC_APERTURE12_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_BASE1                                                              0x2aa0
+#define regCP_GFX_RS64_DC_APERTURE13_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_MASK1                                                              0x2aa1
+#define regCP_GFX_RS64_DC_APERTURE13_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE13_CNTL1                                                              0x2aa2
+#define regCP_GFX_RS64_DC_APERTURE13_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_BASE1                                                              0x2aa3
+#define regCP_GFX_RS64_DC_APERTURE14_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_MASK1                                                              0x2aa4
+#define regCP_GFX_RS64_DC_APERTURE14_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE14_CNTL1                                                              0x2aa5
+#define regCP_GFX_RS64_DC_APERTURE14_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_BASE1                                                              0x2aa6
+#define regCP_GFX_RS64_DC_APERTURE15_BASE1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_MASK1                                                              0x2aa7
+#define regCP_GFX_RS64_DC_APERTURE15_MASK1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_DC_APERTURE15_CNTL1                                                              0x2aa8
+#define regCP_GFX_RS64_DC_APERTURE15_CNTL1_BASE_IDX                                                     1
+#define regCP_GFX_RS64_INTERRUPT1                                                                       0x2aac
+#define regCP_GFX_RS64_INTERRUPT1_BASE_IDX                                                              1
+
+
+// addressBlock: gc_gl1dec
+// base address: 0x33400
+#define regGL1_ARB_CTRL                                                                                 0x2d00
+#define regGL1_ARB_CTRL_BASE_IDX                                                                        1
+#define regGL1_DRAM_BURST_MASK                                                                          0x2d02
+#define regGL1_DRAM_BURST_MASK_BASE_IDX                                                                 1
+#define regGL1_ARB_STATUS                                                                               0x2d03
+#define regGL1_ARB_STATUS_BASE_IDX                                                                      1
+#define regGL1_DRAM_BURST_CTRL                                                                          0x2d04
+#define regGL1_DRAM_BURST_CTRL_BASE_IDX                                                                 1
+#define regGL1I_GL1R_REP_FGCG_OVERRIDE                                                                  0x2d05
+#define regGL1I_GL1R_REP_FGCG_OVERRIDE_BASE_IDX                                                         1
+#define regGL1C_CTRL                                                                                    0x2d40
+#define regGL1C_CTRL_BASE_IDX                                                                           1
+#define regGL1C_STATUS                                                                                  0x2d41
+#define regGL1C_STATUS_BASE_IDX                                                                         1
+#define regGL1C_UTCL0_CNTL2                                                                             0x2d43
+#define regGL1C_UTCL0_CNTL2_BASE_IDX                                                                    1
+#define regGL1C_UTCL0_STATUS                                                                            0x2d44
+#define regGL1C_UTCL0_STATUS_BASE_IDX                                                                   1
+#define regGL1C_UTCL0_RETRY                                                                             0x2d45
+#define regGL1C_UTCL0_RETRY_BASE_IDX                                                                    1
+#define regGL1C_CTRL2                                                                                   0x2d46
+#define regGL1C_CTRL2_BASE_IDX                                                                          1
+
+
+// addressBlock: gc_chdec
+// base address: 0x33600
+#define regCH_ARB_CTRL                                                                                  0x2d80
+#define regCH_ARB_CTRL_BASE_IDX                                                                         1
+#define regCH_DRAM_BURST_MASK                                                                           0x2d82
+#define regCH_DRAM_BURST_MASK_BASE_IDX                                                                  1
+#define regCH_ARB_STATUS                                                                                0x2d83
+#define regCH_ARB_STATUS_BASE_IDX                                                                       1
+#define regCH_DRAM_BURST_CTRL                                                                           0x2d84
+#define regCH_DRAM_BURST_CTRL_BASE_IDX                                                                  1
+#define regCHA_CHC_CREDITS                                                                              0x2d88
+#define regCHA_CHC_CREDITS_BASE_IDX                                                                     1
+#define regCHA_CLIENT_FREE_DELAY                                                                        0x2d89
+#define regCHA_CLIENT_FREE_DELAY_BASE_IDX                                                               1
+#define regCHI_CHR_REP_FGCG_OVERRIDE                                                                    0x2d8c
+#define regCHI_CHR_REP_FGCG_OVERRIDE_BASE_IDX                                                           1
+#define regCH_VC5_ENABLE                                                                                0x2d94
+#define regCH_VC5_ENABLE_BASE_IDX                                                                       1
+#define regCHC_CTRL                                                                                     0x2dc0
+#define regCHC_CTRL_BASE_IDX                                                                            1
+#define regCHC_STATUS                                                                                   0x2dc1
+#define regCHC_STATUS_BASE_IDX                                                                          1
+#define regCHCG_CTRL                                                                                    0x2dc2
+#define regCHCG_CTRL_BASE_IDX                                                                           1
+#define regCHCG_STATUS                                                                                  0x2dc3
+#define regCHCG_STATUS_BASE_IDX                                                                         1
+
+
+// addressBlock: gc_gl2dec
+// base address: 0x33800
+#define regGL2C_CTRL                                                                                    0x2e00
+#define regGL2C_CTRL_BASE_IDX                                                                           1
+#define regGL2C_CTRL2                                                                                   0x2e01
+#define regGL2C_CTRL2_BASE_IDX                                                                          1
+#define regGL2C_STATUS                                                                                  0x2e02
+#define regGL2C_STATUS_BASE_IDX                                                                         1
+#define regGL2C_ADDR_MATCH_MASK                                                                         0x2e03
+#define regGL2C_ADDR_MATCH_MASK_BASE_IDX                                                                1
+#define regGL2C_ADDR_MATCH_SIZE                                                                         0x2e04
+#define regGL2C_ADDR_MATCH_SIZE_BASE_IDX                                                                1
+#define regGL2C_WBINVL2                                                                                 0x2e05
+#define regGL2C_WBINVL2_BASE_IDX                                                                        1
+#define regGL2C_SOFT_RESET                                                                              0x2e06
+#define regGL2C_SOFT_RESET_BASE_IDX                                                                     1
+#define regGL2C_CM_CTRL0                                                                                0x2e07
+#define regGL2C_CM_CTRL0_BASE_IDX                                                                       1
+#define regGL2C_CM_CTRL1                                                                                0x2e08
+#define regGL2C_CM_CTRL1_BASE_IDX                                                                       1
+#define regGL2C_CM_STALL                                                                                0x2e09
+#define regGL2C_CM_STALL_BASE_IDX                                                                       1
+#define regGL2C_CM_CTRL2                                                                                0x2e0b
+#define regGL2C_CM_CTRL2_BASE_IDX                                                                       1
+#define regGL2C_CTRL3                                                                                   0x2e0c
+#define regGL2C_CTRL3_BASE_IDX                                                                          1
+#define regGL2C_LB_CTR_CTRL                                                                             0x2e0d
+#define regGL2C_LB_CTR_CTRL_BASE_IDX                                                                    1
+#define regGL2C_LB_DATA0                                                                                0x2e0e
+#define regGL2C_LB_DATA0_BASE_IDX                                                                       1
+#define regGL2C_LB_DATA1                                                                                0x2e0f
+#define regGL2C_LB_DATA1_BASE_IDX                                                                       1
+#define regGL2C_LB_DATA2                                                                                0x2e10
+#define regGL2C_LB_DATA2_BASE_IDX                                                                       1
+#define regGL2C_LB_DATA3                                                                                0x2e11
+#define regGL2C_LB_DATA3_BASE_IDX                                                                       1
+#define regGL2C_LB_CTR_SEL0                                                                             0x2e12
+#define regGL2C_LB_CTR_SEL0_BASE_IDX                                                                    1
+#define regGL2C_LB_CTR_SEL1                                                                             0x2e13
+#define regGL2C_LB_CTR_SEL1_BASE_IDX                                                                    1
+#define regGL2C_CTRL4                                                                                   0x2e17
+#define regGL2C_CTRL4_BASE_IDX                                                                          1
+#define regGL2C_DISCARD_STALL_CTRL                                                                      0x2e18
+#define regGL2C_DISCARD_STALL_CTRL_BASE_IDX                                                             1
+#define regGL2A_ADDR_MATCH_CTRL                                                                         0x2e20
+#define regGL2A_ADDR_MATCH_CTRL_BASE_IDX                                                                1
+#define regGL2A_ADDR_MATCH_MASK                                                                         0x2e21
+#define regGL2A_ADDR_MATCH_MASK_BASE_IDX                                                                1
+#define regGL2A_ADDR_MATCH_SIZE                                                                         0x2e22
+#define regGL2A_ADDR_MATCH_SIZE_BASE_IDX                                                                1
+#define regGL2A_PRIORITY_CTRL                                                                           0x2e23
+#define regGL2A_PRIORITY_CTRL_BASE_IDX                                                                  1
+#define regGL2A_CTRL                                                                                    0x2e24
+#define regGL2A_CTRL_BASE_IDX                                                                           1
+#define regGL2A_RESP_THROTTLE_CTRL                                                                      0x2e2a
+#define regGL2A_RESP_THROTTLE_CTRL_BASE_IDX                                                             1
+
+
+// addressBlock: gc_gl1hdec
+// base address: 0x33900
+#define regGL1H_ARB_CTRL                                                                                0x2e40
+#define regGL1H_ARB_CTRL_BASE_IDX                                                                       1
+#define regGL1H_GL1_CREDITS                                                                             0x2e41
+#define regGL1H_GL1_CREDITS_BASE_IDX                                                                    1
+#define regGL1H_BURST_MASK                                                                              0x2e42
+#define regGL1H_BURST_MASK_BASE_IDX                                                                     1
+#define regGL1H_BURST_CTRL                                                                              0x2e43
+#define regGL1H_BURST_CTRL_BASE_IDX                                                                     1
+#define regGL1H_ARB_STATUS                                                                              0x2e44
+#define regGL1H_ARB_STATUS_BASE_IDX                                                                     1
+
+
+// addressBlock: gc_perfddec
+// base address: 0x34000
+#define regCPG_PERFCOUNTER1_LO                                                                          0x3000
+#define regCPG_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCPG_PERFCOUNTER1_HI                                                                          0x3001
+#define regCPG_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCPG_PERFCOUNTER0_LO                                                                          0x3002
+#define regCPG_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCPG_PERFCOUNTER0_HI                                                                          0x3003
+#define regCPG_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER1_LO                                                                          0x3004
+#define regCPC_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER1_HI                                                                          0x3005
+#define regCPC_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER0_LO                                                                          0x3006
+#define regCPC_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCPC_PERFCOUNTER0_HI                                                                          0x3007
+#define regCPC_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER1_LO                                                                          0x3008
+#define regCPF_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER1_HI                                                                          0x3009
+#define regCPF_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER0_LO                                                                          0x300a
+#define regCPF_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCPF_PERFCOUNTER0_HI                                                                          0x300b
+#define regCPF_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCPF_LATENCY_STATS_DATA                                                                       0x300c
+#define regCPF_LATENCY_STATS_DATA_BASE_IDX                                                              1
+#define regCPG_LATENCY_STATS_DATA                                                                       0x300d
+#define regCPG_LATENCY_STATS_DATA_BASE_IDX                                                              1
+#define regCPC_LATENCY_STATS_DATA                                                                       0x300e
+#define regCPC_LATENCY_STATS_DATA_BASE_IDX                                                              1
+#define regGRBM_PERFCOUNTER0_LO                                                                         0x3040
+#define regGRBM_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGRBM_PERFCOUNTER0_HI                                                                         0x3041
+#define regGRBM_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGRBM_PERFCOUNTER1_LO                                                                         0x3043
+#define regGRBM_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGRBM_PERFCOUNTER1_HI                                                                         0x3044
+#define regGRBM_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGRBM_SE0_PERFCOUNTER_LO                                                                      0x3045
+#define regGRBM_SE0_PERFCOUNTER_LO_BASE_IDX                                                             1
+#define regGRBM_SE0_PERFCOUNTER_HI                                                                      0x3046
+#define regGRBM_SE0_PERFCOUNTER_HI_BASE_IDX                                                             1
+#define regGRBM_SE1_PERFCOUNTER_LO                                                                      0x3047
+#define regGRBM_SE1_PERFCOUNTER_LO_BASE_IDX                                                             1
+#define regGRBM_SE1_PERFCOUNTER_HI                                                                      0x3048
+#define regGRBM_SE1_PERFCOUNTER_HI_BASE_IDX                                                             1
+#define regGRBM_SE2_PERFCOUNTER_LO                                                                      0x3049
+#define regGRBM_SE2_PERFCOUNTER_LO_BASE_IDX                                                             1
+#define regGRBM_SE2_PERFCOUNTER_HI                                                                      0x304a
+#define regGRBM_SE2_PERFCOUNTER_HI_BASE_IDX                                                             1
+#define regGRBM_SE3_PERFCOUNTER_LO                                                                      0x304b
+#define regGRBM_SE3_PERFCOUNTER_LO_BASE_IDX                                                             1
+#define regGRBM_SE3_PERFCOUNTER_HI                                                                      0x304c
+#define regGRBM_SE3_PERFCOUNTER_HI_BASE_IDX                                                             1
+#define regGE1_PERFCOUNTER0_LO                                                                          0x30a4
+#define regGE1_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER0_HI                                                                          0x30a5
+#define regGE1_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER1_LO                                                                          0x30a6
+#define regGE1_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER1_HI                                                                          0x30a7
+#define regGE1_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER2_LO                                                                          0x30a8
+#define regGE1_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER2_HI                                                                          0x30a9
+#define regGE1_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER3_LO                                                                          0x30aa
+#define regGE1_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regGE1_PERFCOUNTER3_HI                                                                          0x30ab
+#define regGE1_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regGE2_DIST_PERFCOUNTER0_LO                                                                     0x30ac
+#define regGE2_DIST_PERFCOUNTER0_LO_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER0_HI                                                                     0x30ad
+#define regGE2_DIST_PERFCOUNTER0_HI_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER1_LO                                                                     0x30ae
+#define regGE2_DIST_PERFCOUNTER1_LO_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER1_HI                                                                     0x30af
+#define regGE2_DIST_PERFCOUNTER1_HI_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER2_LO                                                                     0x30b0
+#define regGE2_DIST_PERFCOUNTER2_LO_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER2_HI                                                                     0x30b1
+#define regGE2_DIST_PERFCOUNTER2_HI_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER3_LO                                                                     0x30b2
+#define regGE2_DIST_PERFCOUNTER3_LO_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER3_HI                                                                     0x30b3
+#define regGE2_DIST_PERFCOUNTER3_HI_BASE_IDX                                                            1
+#define regGE2_SE_PERFCOUNTER0_LO                                                                       0x30b4
+#define regGE2_SE_PERFCOUNTER0_LO_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER0_HI                                                                       0x30b5
+#define regGE2_SE_PERFCOUNTER0_HI_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER1_LO                                                                       0x30b6
+#define regGE2_SE_PERFCOUNTER1_LO_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER1_HI                                                                       0x30b7
+#define regGE2_SE_PERFCOUNTER1_HI_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER2_LO                                                                       0x30b8
+#define regGE2_SE_PERFCOUNTER2_LO_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER2_HI                                                                       0x30b9
+#define regGE2_SE_PERFCOUNTER2_HI_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER3_LO                                                                       0x30ba
+#define regGE2_SE_PERFCOUNTER3_LO_BASE_IDX                                                              1
+#define regGE2_SE_PERFCOUNTER3_HI                                                                       0x30bb
+#define regGE2_SE_PERFCOUNTER3_HI_BASE_IDX                                                              1
+#define regPA_SU_PERFCOUNTER0_LO                                                                        0x3100
+#define regPA_SU_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER0_HI                                                                        0x3101
+#define regPA_SU_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER1_LO                                                                        0x3102
+#define regPA_SU_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER1_HI                                                                        0x3103
+#define regPA_SU_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER2_LO                                                                        0x3104
+#define regPA_SU_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER2_HI                                                                        0x3105
+#define regPA_SU_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER3_LO                                                                        0x3106
+#define regPA_SU_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regPA_SU_PERFCOUNTER3_HI                                                                        0x3107
+#define regPA_SU_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER0_LO                                                                        0x3140
+#define regPA_SC_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER0_HI                                                                        0x3141
+#define regPA_SC_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER1_LO                                                                        0x3142
+#define regPA_SC_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER1_HI                                                                        0x3143
+#define regPA_SC_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER2_LO                                                                        0x3144
+#define regPA_SC_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER2_HI                                                                        0x3145
+#define regPA_SC_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER3_LO                                                                        0x3146
+#define regPA_SC_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER3_HI                                                                        0x3147
+#define regPA_SC_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER4_LO                                                                        0x3148
+#define regPA_SC_PERFCOUNTER4_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER4_HI                                                                        0x3149
+#define regPA_SC_PERFCOUNTER4_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER5_LO                                                                        0x314a
+#define regPA_SC_PERFCOUNTER5_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER5_HI                                                                        0x314b
+#define regPA_SC_PERFCOUNTER5_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER6_LO                                                                        0x314c
+#define regPA_SC_PERFCOUNTER6_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER6_HI                                                                        0x314d
+#define regPA_SC_PERFCOUNTER6_HI_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER7_LO                                                                        0x314e
+#define regPA_SC_PERFCOUNTER7_LO_BASE_IDX                                                               1
+#define regPA_SC_PERFCOUNTER7_HI                                                                        0x314f
+#define regPA_SC_PERFCOUNTER7_HI_BASE_IDX                                                               1
+#define regSPI_PERFCOUNTER0_HI                                                                          0x3180
+#define regSPI_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER0_LO                                                                          0x3181
+#define regSPI_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER1_HI                                                                          0x3182
+#define regSPI_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER1_LO                                                                          0x3183
+#define regSPI_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER2_HI                                                                          0x3184
+#define regSPI_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER2_LO                                                                          0x3185
+#define regSPI_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER3_HI                                                                          0x3186
+#define regSPI_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER3_LO                                                                          0x3187
+#define regSPI_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER4_HI                                                                          0x3188
+#define regSPI_PERFCOUNTER4_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER4_LO                                                                          0x3189
+#define regSPI_PERFCOUNTER4_LO_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER5_HI                                                                          0x318a
+#define regSPI_PERFCOUNTER5_HI_BASE_IDX                                                                 1
+#define regSPI_PERFCOUNTER5_LO                                                                          0x318b
+#define regSPI_PERFCOUNTER5_LO_BASE_IDX                                                                 1
+#define regPC_PERFCOUNTER0_HI                                                                           0x318c
+#define regPC_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER0_LO                                                                           0x318d
+#define regPC_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER1_HI                                                                           0x318e
+#define regPC_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER1_LO                                                                           0x318f
+#define regPC_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER2_HI                                                                           0x3190
+#define regPC_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER2_LO                                                                           0x3191
+#define regPC_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER3_HI                                                                           0x3192
+#define regPC_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regPC_PERFCOUNTER3_LO                                                                           0x3193
+#define regPC_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER0_LO                                                                           0x31c0
+#define regSQ_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER1_LO                                                                           0x31c2
+#define regSQ_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER2_LO                                                                           0x31c4
+#define regSQ_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER3_LO                                                                           0x31c6
+#define regSQ_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER4_LO                                                                           0x31c8
+#define regSQ_PERFCOUNTER4_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER5_LO                                                                           0x31ca
+#define regSQ_PERFCOUNTER5_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER6_LO                                                                           0x31cc
+#define regSQ_PERFCOUNTER6_LO_BASE_IDX                                                                  1
+#define regSQ_PERFCOUNTER7_LO                                                                           0x31ce
+#define regSQ_PERFCOUNTER7_LO_BASE_IDX                                                                  1
+#define regSQG_PERFCOUNTER0_LO                                                                          0x31e4
+#define regSQG_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER0_HI                                                                          0x31e5
+#define regSQG_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER1_LO                                                                          0x31e6
+#define regSQG_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER1_HI                                                                          0x31e7
+#define regSQG_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER2_LO                                                                          0x31e8
+#define regSQG_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER2_HI                                                                          0x31e9
+#define regSQG_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER3_LO                                                                          0x31ea
+#define regSQG_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER3_HI                                                                          0x31eb
+#define regSQG_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER4_LO                                                                          0x31ec
+#define regSQG_PERFCOUNTER4_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER4_HI                                                                          0x31ed
+#define regSQG_PERFCOUNTER4_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER5_LO                                                                          0x31ee
+#define regSQG_PERFCOUNTER5_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER5_HI                                                                          0x31ef
+#define regSQG_PERFCOUNTER5_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER6_LO                                                                          0x31f0
+#define regSQG_PERFCOUNTER6_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER6_HI                                                                          0x31f1
+#define regSQG_PERFCOUNTER6_HI_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER7_LO                                                                          0x31f2
+#define regSQG_PERFCOUNTER7_LO_BASE_IDX                                                                 1
+#define regSQG_PERFCOUNTER7_HI                                                                          0x31f3
+#define regSQG_PERFCOUNTER7_HI_BASE_IDX                                                                 1
+#define regSX_PERFCOUNTER0_LO                                                                           0x3240
+#define regSX_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER0_HI                                                                           0x3241
+#define regSX_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER1_LO                                                                           0x3242
+#define regSX_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER1_HI                                                                           0x3243
+#define regSX_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER2_LO                                                                           0x3244
+#define regSX_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER2_HI                                                                           0x3245
+#define regSX_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER3_LO                                                                           0x3246
+#define regSX_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regSX_PERFCOUNTER3_HI                                                                           0x3247
+#define regSX_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regGCEA_PERFCOUNTER2_LO                                                                         0x3260
+#define regGCEA_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regGCEA_PERFCOUNTER2_HI                                                                         0x3261
+#define regGCEA_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regGCEA_PERFCOUNTER_LO                                                                          0x3262
+#define regGCEA_PERFCOUNTER_LO_BASE_IDX                                                                 1
+#define regGCEA_PERFCOUNTER_HI                                                                          0x3263
+#define regGCEA_PERFCOUNTER_HI_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER0_LO                                                                          0x3280
+#define regGDS_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER0_HI                                                                          0x3281
+#define regGDS_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER1_LO                                                                          0x3282
+#define regGDS_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER1_HI                                                                          0x3283
+#define regGDS_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER2_LO                                                                          0x3284
+#define regGDS_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER2_HI                                                                          0x3285
+#define regGDS_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER3_LO                                                                          0x3286
+#define regGDS_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regGDS_PERFCOUNTER3_HI                                                                          0x3287
+#define regGDS_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regTA_PERFCOUNTER0_LO                                                                           0x32c0
+#define regTA_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regTA_PERFCOUNTER0_HI                                                                           0x32c1
+#define regTA_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regTA_PERFCOUNTER1_LO                                                                           0x32c2
+#define regTA_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regTA_PERFCOUNTER1_HI                                                                           0x32c3
+#define regTA_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER0_LO                                                                           0x3300
+#define regTD_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER0_HI                                                                           0x3301
+#define regTD_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER1_LO                                                                           0x3302
+#define regTD_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regTD_PERFCOUNTER1_HI                                                                           0x3303
+#define regTD_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regTCP_PERFCOUNTER0_LO                                                                          0x3340
+#define regTCP_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER0_HI                                                                          0x3341
+#define regTCP_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER1_LO                                                                          0x3342
+#define regTCP_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER1_HI                                                                          0x3343
+#define regTCP_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER2_LO                                                                          0x3344
+#define regTCP_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER2_HI                                                                          0x3345
+#define regTCP_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER3_LO                                                                          0x3346
+#define regTCP_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER3_HI                                                                          0x3347
+#define regTCP_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regTCP_PERFCOUNTER_FILTER                                                                       0x3348
+#define regTCP_PERFCOUNTER_FILTER_BASE_IDX                                                              1
+#define regTCP_PERFCOUNTER_FILTER2                                                                      0x3349
+#define regTCP_PERFCOUNTER_FILTER2_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER_FILTER_EN                                                                    0x334a
+#define regTCP_PERFCOUNTER_FILTER_EN_BASE_IDX                                                           1
+#define regGL2C_PERFCOUNTER0_LO                                                                         0x3380
+#define regGL2C_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER0_HI                                                                         0x3381
+#define regGL2C_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER1_LO                                                                         0x3382
+#define regGL2C_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER1_HI                                                                         0x3383
+#define regGL2C_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER2_LO                                                                         0x3384
+#define regGL2C_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER2_HI                                                                         0x3385
+#define regGL2C_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER3_LO                                                                         0x3386
+#define regGL2C_PERFCOUNTER3_LO_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER3_HI                                                                         0x3387
+#define regGL2C_PERFCOUNTER3_HI_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER0_LO                                                                         0x3390
+#define regGL2A_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER0_HI                                                                         0x3391
+#define regGL2A_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER1_LO                                                                         0x3392
+#define regGL2A_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER1_HI                                                                         0x3393
+#define regGL2A_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER2_LO                                                                         0x3394
+#define regGL2A_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER2_HI                                                                         0x3395
+#define regGL2A_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER3_LO                                                                         0x3396
+#define regGL2A_PERFCOUNTER3_LO_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER3_HI                                                                         0x3397
+#define regGL2A_PERFCOUNTER3_HI_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER0_LO                                                                         0x33a0
+#define regGL1C_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER0_HI                                                                         0x33a1
+#define regGL1C_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER1_LO                                                                         0x33a2
+#define regGL1C_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER1_HI                                                                         0x33a3
+#define regGL1C_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER2_LO                                                                         0x33a4
+#define regGL1C_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER2_HI                                                                         0x33a5
+#define regGL1C_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER3_LO                                                                         0x33a6
+#define regGL1C_PERFCOUNTER3_LO_BASE_IDX                                                                1
+#define regGL1C_PERFCOUNTER3_HI                                                                         0x33a7
+#define regGL1C_PERFCOUNTER3_HI_BASE_IDX                                                                1
+#define regCHC_PERFCOUNTER0_LO                                                                          0x33c0
+#define regCHC_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER0_HI                                                                          0x33c1
+#define regCHC_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER1_LO                                                                          0x33c2
+#define regCHC_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER1_HI                                                                          0x33c3
+#define regCHC_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER2_LO                                                                          0x33c4
+#define regCHC_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER2_HI                                                                          0x33c5
+#define regCHC_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER3_LO                                                                          0x33c6
+#define regCHC_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regCHC_PERFCOUNTER3_HI                                                                          0x33c7
+#define regCHC_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regCHCG_PERFCOUNTER0_LO                                                                         0x33c8
+#define regCHCG_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regCHCG_PERFCOUNTER0_HI                                                                         0x33c9
+#define regCHCG_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regCHCG_PERFCOUNTER1_LO                                                                         0x33ca
+#define regCHCG_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regCHCG_PERFCOUNTER1_HI                                                                         0x33cb
+#define regCHCG_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regCHCG_PERFCOUNTER2_LO                                                                         0x33cc
+#define regCHCG_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regCHCG_PERFCOUNTER2_HI                                                                         0x33cd
+#define regCHCG_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regCHCG_PERFCOUNTER3_LO                                                                         0x33ce
+#define regCHCG_PERFCOUNTER3_LO_BASE_IDX                                                                1
+#define regCHCG_PERFCOUNTER3_HI                                                                         0x33cf
+#define regCHCG_PERFCOUNTER3_HI_BASE_IDX                                                                1
+#define regCB_PERFCOUNTER0_LO                                                                           0x3406
+#define regCB_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER0_HI                                                                           0x3407
+#define regCB_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER1_LO                                                                           0x3408
+#define regCB_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER1_HI                                                                           0x3409
+#define regCB_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER2_LO                                                                           0x340a
+#define regCB_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER2_HI                                                                           0x340b
+#define regCB_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER3_LO                                                                           0x340c
+#define regCB_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regCB_PERFCOUNTER3_HI                                                                           0x340d
+#define regCB_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER0_LO                                                                           0x3440
+#define regDB_PERFCOUNTER0_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER0_HI                                                                           0x3441
+#define regDB_PERFCOUNTER0_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER1_LO                                                                           0x3442
+#define regDB_PERFCOUNTER1_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER1_HI                                                                           0x3443
+#define regDB_PERFCOUNTER1_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER2_LO                                                                           0x3444
+#define regDB_PERFCOUNTER2_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER2_HI                                                                           0x3445
+#define regDB_PERFCOUNTER2_HI_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER3_LO                                                                           0x3446
+#define regDB_PERFCOUNTER3_LO_BASE_IDX                                                                  1
+#define regDB_PERFCOUNTER3_HI                                                                           0x3447
+#define regDB_PERFCOUNTER3_HI_BASE_IDX                                                                  1
+#define regRLC_PERFCOUNTER0_LO                                                                          0x3480
+#define regRLC_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regRLC_PERFCOUNTER0_HI                                                                          0x3481
+#define regRLC_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regRLC_PERFCOUNTER1_LO                                                                          0x3482
+#define regRLC_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regRLC_PERFCOUNTER1_HI                                                                          0x3483
+#define regRLC_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER0_LO                                                                          0x34c0
+#define regRMI_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER0_HI                                                                          0x34c1
+#define regRMI_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER1_LO                                                                          0x34c2
+#define regRMI_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER1_HI                                                                          0x34c3
+#define regRMI_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER2_LO                                                                          0x34c4
+#define regRMI_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER2_HI                                                                          0x34c5
+#define regRMI_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER3_LO                                                                          0x34c6
+#define regRMI_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regRMI_PERFCOUNTER3_HI                                                                          0x34c7
+#define regRMI_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regGCR_PERFCOUNTER0_LO                                                                          0x3520
+#define regGCR_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regGCR_PERFCOUNTER0_HI                                                                          0x3521
+#define regGCR_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regGCR_PERFCOUNTER1_LO                                                                          0x3522
+#define regGCR_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regGCR_PERFCOUNTER1_HI                                                                          0x3523
+#define regGCR_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regPA_PH_PERFCOUNTER0_LO                                                                        0x3580
+#define regPA_PH_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER0_HI                                                                        0x3581
+#define regPA_PH_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER1_LO                                                                        0x3582
+#define regPA_PH_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER1_HI                                                                        0x3583
+#define regPA_PH_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER2_LO                                                                        0x3584
+#define regPA_PH_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER2_HI                                                                        0x3585
+#define regPA_PH_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER3_LO                                                                        0x3586
+#define regPA_PH_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER3_HI                                                                        0x3587
+#define regPA_PH_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER4_LO                                                                        0x3588
+#define regPA_PH_PERFCOUNTER4_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER4_HI                                                                        0x3589
+#define regPA_PH_PERFCOUNTER4_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER5_LO                                                                        0x358a
+#define regPA_PH_PERFCOUNTER5_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER5_HI                                                                        0x358b
+#define regPA_PH_PERFCOUNTER5_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER6_LO                                                                        0x358c
+#define regPA_PH_PERFCOUNTER6_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER6_HI                                                                        0x358d
+#define regPA_PH_PERFCOUNTER6_HI_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER7_LO                                                                        0x358e
+#define regPA_PH_PERFCOUNTER7_LO_BASE_IDX                                                               1
+#define regPA_PH_PERFCOUNTER7_HI                                                                        0x358f
+#define regPA_PH_PERFCOUNTER7_HI_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER0_LO                                                                        0x35a0
+#define regUTCL1_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER0_HI                                                                        0x35a1
+#define regUTCL1_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER1_LO                                                                        0x35a2
+#define regUTCL1_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER1_HI                                                                        0x35a3
+#define regUTCL1_PERFCOUNTER1_HI_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER2_LO                                                                        0x35a4
+#define regUTCL1_PERFCOUNTER2_LO_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER2_HI                                                                        0x35a5
+#define regUTCL1_PERFCOUNTER2_HI_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER3_LO                                                                        0x35a6
+#define regUTCL1_PERFCOUNTER3_LO_BASE_IDX                                                               1
+#define regUTCL1_PERFCOUNTER3_HI                                                                        0x35a7
+#define regUTCL1_PERFCOUNTER3_HI_BASE_IDX                                                               1
+#define regGL1A_PERFCOUNTER0_LO                                                                         0x35c0
+#define regGL1A_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER0_HI                                                                         0x35c1
+#define regGL1A_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER1_LO                                                                         0x35c2
+#define regGL1A_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER1_HI                                                                         0x35c3
+#define regGL1A_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER2_LO                                                                         0x35c4
+#define regGL1A_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER2_HI                                                                         0x35c5
+#define regGL1A_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER3_LO                                                                         0x35c6
+#define regGL1A_PERFCOUNTER3_LO_BASE_IDX                                                                1
+#define regGL1A_PERFCOUNTER3_HI                                                                         0x35c7
+#define regGL1A_PERFCOUNTER3_HI_BASE_IDX                                                                1
+#define regGL1H_PERFCOUNTER0_LO                                                                         0x35d0
+#define regGL1H_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGL1H_PERFCOUNTER0_HI                                                                         0x35d1
+#define regGL1H_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGL1H_PERFCOUNTER1_LO                                                                         0x35d2
+#define regGL1H_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGL1H_PERFCOUNTER1_HI                                                                         0x35d3
+#define regGL1H_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGL1H_PERFCOUNTER2_LO                                                                         0x35d4
+#define regGL1H_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regGL1H_PERFCOUNTER2_HI                                                                         0x35d5
+#define regGL1H_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regGL1H_PERFCOUNTER3_LO                                                                         0x35d6
+#define regGL1H_PERFCOUNTER3_LO_BASE_IDX                                                                1
+#define regGL1H_PERFCOUNTER3_HI                                                                         0x35d7
+#define regGL1H_PERFCOUNTER3_HI_BASE_IDX                                                                1
+#define regCHA_PERFCOUNTER0_LO                                                                          0x3600
+#define regCHA_PERFCOUNTER0_LO_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER0_HI                                                                          0x3601
+#define regCHA_PERFCOUNTER0_HI_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER1_LO                                                                          0x3602
+#define regCHA_PERFCOUNTER1_LO_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER1_HI                                                                          0x3603
+#define regCHA_PERFCOUNTER1_HI_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER2_LO                                                                          0x3604
+#define regCHA_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER2_HI                                                                          0x3605
+#define regCHA_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER3_LO                                                                          0x3606
+#define regCHA_PERFCOUNTER3_LO_BASE_IDX                                                                 1
+#define regCHA_PERFCOUNTER3_HI                                                                          0x3607
+#define regCHA_PERFCOUNTER3_HI_BASE_IDX                                                                 1
+#define regGUS_PERFCOUNTER2_LO                                                                          0x3640
+#define regGUS_PERFCOUNTER2_LO_BASE_IDX                                                                 1
+#define regGUS_PERFCOUNTER2_HI                                                                          0x3641
+#define regGUS_PERFCOUNTER2_HI_BASE_IDX                                                                 1
+#define regGUS_PERFCOUNTER_LO                                                                           0x3642
+#define regGUS_PERFCOUNTER_LO_BASE_IDX                                                                  1
+#define regGUS_PERFCOUNTER_HI                                                                           0x3643
+#define regGUS_PERFCOUNTER_HI_BASE_IDX                                                                  1
+
+
+// addressBlock: gc_perfsdec
+// base address: 0x36000
+#define regCPG_PERFCOUNTER1_SELECT                                                                      0x3800
+#define regCPG_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCPG_PERFCOUNTER0_SELECT1                                                                     0x3801
+#define regCPG_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCPG_PERFCOUNTER0_SELECT                                                                      0x3802
+#define regCPG_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCPC_PERFCOUNTER1_SELECT                                                                      0x3803
+#define regCPC_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCPC_PERFCOUNTER0_SELECT1                                                                     0x3804
+#define regCPC_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCPF_PERFCOUNTER1_SELECT                                                                      0x3805
+#define regCPF_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCPF_PERFCOUNTER0_SELECT1                                                                     0x3806
+#define regCPF_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCPF_PERFCOUNTER0_SELECT                                                                      0x3807
+#define regCPF_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCP_PERFMON_CNTL                                                                              0x3808
+#define regCP_PERFMON_CNTL_BASE_IDX                                                                     1
+#define regCPC_PERFCOUNTER0_SELECT                                                                      0x3809
+#define regCPC_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCPF_TC_PERF_COUNTER_WINDOW_SELECT                                                            0x380a
+#define regCPF_TC_PERF_COUNTER_WINDOW_SELECT_BASE_IDX                                                   1
+#define regCPG_TC_PERF_COUNTER_WINDOW_SELECT                                                            0x380b
+#define regCPG_TC_PERF_COUNTER_WINDOW_SELECT_BASE_IDX                                                   1
+#define regCPF_LATENCY_STATS_SELECT                                                                     0x380c
+#define regCPF_LATENCY_STATS_SELECT_BASE_IDX                                                            1
+#define regCPG_LATENCY_STATS_SELECT                                                                     0x380d
+#define regCPG_LATENCY_STATS_SELECT_BASE_IDX                                                            1
+#define regCPC_LATENCY_STATS_SELECT                                                                     0x380e
+#define regCPC_LATENCY_STATS_SELECT_BASE_IDX                                                            1
+#define regCPC_TC_PERF_COUNTER_WINDOW_SELECT                                                            0x380f
+#define regCPC_TC_PERF_COUNTER_WINDOW_SELECT_BASE_IDX                                                   1
+#define regCP_DRAW_OBJECT                                                                               0x3810
+#define regCP_DRAW_OBJECT_BASE_IDX                                                                      1
+#define regCP_DRAW_OBJECT_COUNTER                                                                       0x3811
+#define regCP_DRAW_OBJECT_COUNTER_BASE_IDX                                                              1
+#define regCP_DRAW_WINDOW_MASK_HI                                                                       0x3812
+#define regCP_DRAW_WINDOW_MASK_HI_BASE_IDX                                                              1
+#define regCP_DRAW_WINDOW_HI                                                                            0x3813
+#define regCP_DRAW_WINDOW_HI_BASE_IDX                                                                   1
+#define regCP_DRAW_WINDOW_LO                                                                            0x3814
+#define regCP_DRAW_WINDOW_LO_BASE_IDX                                                                   1
+#define regCP_DRAW_WINDOW_CNTL                                                                          0x3815
+#define regCP_DRAW_WINDOW_CNTL_BASE_IDX                                                                 1
+#define regGRBM_PERFCOUNTER0_SELECT                                                                     0x3840
+#define regGRBM_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGRBM_PERFCOUNTER1_SELECT                                                                     0x3841
+#define regGRBM_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGRBM_SE0_PERFCOUNTER_SELECT                                                                  0x3842
+#define regGRBM_SE0_PERFCOUNTER_SELECT_BASE_IDX                                                         1
+#define regGRBM_SE1_PERFCOUNTER_SELECT                                                                  0x3843
+#define regGRBM_SE1_PERFCOUNTER_SELECT_BASE_IDX                                                         1
+#define regGRBM_SE2_PERFCOUNTER_SELECT                                                                  0x3844
+#define regGRBM_SE2_PERFCOUNTER_SELECT_BASE_IDX                                                         1
+#define regGRBM_SE3_PERFCOUNTER_SELECT                                                                  0x3845
+#define regGRBM_SE3_PERFCOUNTER_SELECT_BASE_IDX                                                         1
+#define regGRBM_PERFCOUNTER0_SELECT_HI                                                                  0x384d
+#define regGRBM_PERFCOUNTER0_SELECT_HI_BASE_IDX                                                         1
+#define regGRBM_PERFCOUNTER1_SELECT_HI                                                                  0x384e
+#define regGRBM_PERFCOUNTER1_SELECT_HI_BASE_IDX                                                         1
+#define regGE1_PERFCOUNTER0_SELECT                                                                      0x38a4
+#define regGE1_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regGE1_PERFCOUNTER0_SELECT1                                                                     0x38a5
+#define regGE1_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regGE1_PERFCOUNTER1_SELECT                                                                      0x38a6
+#define regGE1_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regGE1_PERFCOUNTER1_SELECT1                                                                     0x38a7
+#define regGE1_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regGE1_PERFCOUNTER2_SELECT                                                                      0x38a8
+#define regGE1_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regGE1_PERFCOUNTER2_SELECT1                                                                     0x38a9
+#define regGE1_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regGE1_PERFCOUNTER3_SELECT                                                                      0x38aa
+#define regGE1_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regGE1_PERFCOUNTER3_SELECT1                                                                     0x38ab
+#define regGE1_PERFCOUNTER3_SELECT1_BASE_IDX                                                            1
+#define regGE2_DIST_PERFCOUNTER0_SELECT                                                                 0x38ac
+#define regGE2_DIST_PERFCOUNTER0_SELECT_BASE_IDX                                                        1
+#define regGE2_DIST_PERFCOUNTER0_SELECT1                                                                0x38ad
+#define regGE2_DIST_PERFCOUNTER0_SELECT1_BASE_IDX                                                       1
+#define regGE2_DIST_PERFCOUNTER1_SELECT                                                                 0x38ae
+#define regGE2_DIST_PERFCOUNTER1_SELECT_BASE_IDX                                                        1
+#define regGE2_DIST_PERFCOUNTER1_SELECT1                                                                0x38af
+#define regGE2_DIST_PERFCOUNTER1_SELECT1_BASE_IDX                                                       1
+#define regGE2_DIST_PERFCOUNTER2_SELECT                                                                 0x38b0
+#define regGE2_DIST_PERFCOUNTER2_SELECT_BASE_IDX                                                        1
+#define regGE2_DIST_PERFCOUNTER2_SELECT1                                                                0x38b1
+#define regGE2_DIST_PERFCOUNTER2_SELECT1_BASE_IDX                                                       1
+#define regGE2_DIST_PERFCOUNTER3_SELECT                                                                 0x38b2
+#define regGE2_DIST_PERFCOUNTER3_SELECT_BASE_IDX                                                        1
+#define regGE2_DIST_PERFCOUNTER3_SELECT1                                                                0x38b3
+#define regGE2_DIST_PERFCOUNTER3_SELECT1_BASE_IDX                                                       1
+#define regGE2_SE_PERFCOUNTER0_SELECT                                                                   0x38b4
+#define regGE2_SE_PERFCOUNTER0_SELECT_BASE_IDX                                                          1
+#define regGE2_SE_PERFCOUNTER0_SELECT1                                                                  0x38b5
+#define regGE2_SE_PERFCOUNTER0_SELECT1_BASE_IDX                                                         1
+#define regGE2_SE_PERFCOUNTER1_SELECT                                                                   0x38b6
+#define regGE2_SE_PERFCOUNTER1_SELECT_BASE_IDX                                                          1
+#define regGE2_SE_PERFCOUNTER1_SELECT1                                                                  0x38b7
+#define regGE2_SE_PERFCOUNTER1_SELECT1_BASE_IDX                                                         1
+#define regGE2_SE_PERFCOUNTER2_SELECT                                                                   0x38b8
+#define regGE2_SE_PERFCOUNTER2_SELECT_BASE_IDX                                                          1
+#define regGE2_SE_PERFCOUNTER2_SELECT1                                                                  0x38b9
+#define regGE2_SE_PERFCOUNTER2_SELECT1_BASE_IDX                                                         1
+#define regGE2_SE_PERFCOUNTER3_SELECT                                                                   0x38ba
+#define regGE2_SE_PERFCOUNTER3_SELECT_BASE_IDX                                                          1
+#define regGE2_SE_PERFCOUNTER3_SELECT1                                                                  0x38bb
+#define regGE2_SE_PERFCOUNTER3_SELECT1_BASE_IDX                                                         1
+#define regPA_SU_PERFCOUNTER0_SELECT                                                                    0x3900
+#define regPA_SU_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER0_SELECT1                                                                   0x3901
+#define regPA_SU_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regPA_SU_PERFCOUNTER1_SELECT                                                                    0x3902
+#define regPA_SU_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER1_SELECT1                                                                   0x3903
+#define regPA_SU_PERFCOUNTER1_SELECT1_BASE_IDX                                                          1
+#define regPA_SU_PERFCOUNTER2_SELECT                                                                    0x3904
+#define regPA_SU_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER2_SELECT1                                                                   0x3905
+#define regPA_SU_PERFCOUNTER2_SELECT1_BASE_IDX                                                          1
+#define regPA_SU_PERFCOUNTER3_SELECT                                                                    0x3906
+#define regPA_SU_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regPA_SU_PERFCOUNTER3_SELECT1                                                                   0x3907
+#define regPA_SU_PERFCOUNTER3_SELECT1_BASE_IDX                                                          1
+#define regPA_SC_PERFCOUNTER0_SELECT                                                                    0x3940
+#define regPA_SC_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER0_SELECT1                                                                   0x3941
+#define regPA_SC_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regPA_SC_PERFCOUNTER1_SELECT                                                                    0x3942
+#define regPA_SC_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER2_SELECT                                                                    0x3943
+#define regPA_SC_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER3_SELECT                                                                    0x3944
+#define regPA_SC_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER4_SELECT                                                                    0x3945
+#define regPA_SC_PERFCOUNTER4_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER5_SELECT                                                                    0x3946
+#define regPA_SC_PERFCOUNTER5_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER6_SELECT                                                                    0x3947
+#define regPA_SC_PERFCOUNTER6_SELECT_BASE_IDX                                                           1
+#define regPA_SC_PERFCOUNTER7_SELECT                                                                    0x3948
+#define regPA_SC_PERFCOUNTER7_SELECT_BASE_IDX                                                           1
+#define regSPI_PERFCOUNTER0_SELECT                                                                      0x3980
+#define regSPI_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER1_SELECT                                                                      0x3981
+#define regSPI_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER2_SELECT                                                                      0x3982
+#define regSPI_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER3_SELECT                                                                      0x3983
+#define regSPI_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER0_SELECT1                                                                     0x3984
+#define regSPI_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER1_SELECT1                                                                     0x3985
+#define regSPI_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER2_SELECT1                                                                     0x3986
+#define regSPI_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER3_SELECT1                                                                     0x3987
+#define regSPI_PERFCOUNTER3_SELECT1_BASE_IDX                                                            1
+#define regSPI_PERFCOUNTER4_SELECT                                                                      0x3988
+#define regSPI_PERFCOUNTER4_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER5_SELECT                                                                      0x3989
+#define regSPI_PERFCOUNTER5_SELECT_BASE_IDX                                                             1
+#define regSPI_PERFCOUNTER_BINS                                                                         0x398a
+#define regSPI_PERFCOUNTER_BINS_BASE_IDX                                                                1
+#define regPC_PERFCOUNTER0_SELECT                                                                       0x398c
+#define regPC_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regPC_PERFCOUNTER1_SELECT                                                                       0x398d
+#define regPC_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regPC_PERFCOUNTER2_SELECT                                                                       0x398e
+#define regPC_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regPC_PERFCOUNTER3_SELECT                                                                       0x398f
+#define regPC_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regPC_PERFCOUNTER0_SELECT1                                                                      0x3990
+#define regPC_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regPC_PERFCOUNTER1_SELECT1                                                                      0x3991
+#define regPC_PERFCOUNTER1_SELECT1_BASE_IDX                                                             1
+#define regPC_PERFCOUNTER2_SELECT1                                                                      0x3992
+#define regPC_PERFCOUNTER2_SELECT1_BASE_IDX                                                             1
+#define regPC_PERFCOUNTER3_SELECT1                                                                      0x3993
+#define regPC_PERFCOUNTER3_SELECT1_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER0_SELECT                                                                       0x39c0
+#define regSQ_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER1_SELECT                                                                       0x39c1
+#define regSQ_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER2_SELECT                                                                       0x39c2
+#define regSQ_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER3_SELECT                                                                       0x39c3
+#define regSQ_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER4_SELECT                                                                       0x39c4
+#define regSQ_PERFCOUNTER4_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER5_SELECT                                                                       0x39c5
+#define regSQ_PERFCOUNTER5_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER6_SELECT                                                                       0x39c6
+#define regSQ_PERFCOUNTER6_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER7_SELECT                                                                       0x39c7
+#define regSQ_PERFCOUNTER7_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER8_SELECT                                                                       0x39c8
+#define regSQ_PERFCOUNTER8_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER9_SELECT                                                                       0x39c9
+#define regSQ_PERFCOUNTER9_SELECT_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER10_SELECT                                                                      0x39ca
+#define regSQ_PERFCOUNTER10_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER11_SELECT                                                                      0x39cb
+#define regSQ_PERFCOUNTER11_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER12_SELECT                                                                      0x39cc
+#define regSQ_PERFCOUNTER12_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER13_SELECT                                                                      0x39cd
+#define regSQ_PERFCOUNTER13_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER14_SELECT                                                                      0x39ce
+#define regSQ_PERFCOUNTER14_SELECT_BASE_IDX                                                             1
+#define regSQ_PERFCOUNTER15_SELECT                                                                      0x39cf
+#define regSQ_PERFCOUNTER15_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER0_SELECT                                                                      0x39d0
+#define regSQG_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER1_SELECT                                                                      0x39d1
+#define regSQG_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER2_SELECT                                                                      0x39d2
+#define regSQG_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER3_SELECT                                                                      0x39d3
+#define regSQG_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER4_SELECT                                                                      0x39d4
+#define regSQG_PERFCOUNTER4_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER5_SELECT                                                                      0x39d5
+#define regSQG_PERFCOUNTER5_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER6_SELECT                                                                      0x39d6
+#define regSQG_PERFCOUNTER6_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER7_SELECT                                                                      0x39d7
+#define regSQG_PERFCOUNTER7_SELECT_BASE_IDX                                                             1
+#define regSQG_PERFCOUNTER_CTRL                                                                         0x39d8
+#define regSQG_PERFCOUNTER_CTRL_BASE_IDX                                                                1
+#define regSQG_PERFCOUNTER_CTRL2                                                                        0x39da
+#define regSQG_PERFCOUNTER_CTRL2_BASE_IDX                                                               1
+#define regSQG_PERF_SAMPLE_FINISH                                                                       0x39db
+#define regSQG_PERF_SAMPLE_FINISH_BASE_IDX                                                              1
+#define regSQ_PERFCOUNTER_CTRL                                                                          0x39e0
+#define regSQ_PERFCOUNTER_CTRL_BASE_IDX                                                                 1
+#define regSQ_PERFCOUNTER_CTRL2                                                                         0x39e2
+#define regSQ_PERFCOUNTER_CTRL2_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_BUF0_BASE                                                                    0x39e8
+#define regSQ_THREAD_TRACE_BUF0_BASE_BASE_IDX                                                           1
+#define regSQ_THREAD_TRACE_BUF0_SIZE                                                                    0x39e9
+#define regSQ_THREAD_TRACE_BUF0_SIZE_BASE_IDX                                                           1
+#define regSQ_THREAD_TRACE_BUF1_BASE                                                                    0x39ea
+#define regSQ_THREAD_TRACE_BUF1_BASE_BASE_IDX                                                           1
+#define regSQ_THREAD_TRACE_BUF1_SIZE                                                                    0x39eb
+#define regSQ_THREAD_TRACE_BUF1_SIZE_BASE_IDX                                                           1
+#define regSQ_THREAD_TRACE_CTRL                                                                         0x39ec
+#define regSQ_THREAD_TRACE_CTRL_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_MASK                                                                         0x39ed
+#define regSQ_THREAD_TRACE_MASK_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_TOKEN_MASK                                                                   0x39ee
+#define regSQ_THREAD_TRACE_TOKEN_MASK_BASE_IDX                                                          1
+#define regSQ_THREAD_TRACE_WPTR                                                                         0x39ef
+#define regSQ_THREAD_TRACE_WPTR_BASE_IDX                                                                1
+#define regSQ_THREAD_TRACE_STATUS                                                                       0x39f4
+#define regSQ_THREAD_TRACE_STATUS_BASE_IDX                                                              1
+#define regSQ_THREAD_TRACE_STATUS2                                                                      0x39f5
+#define regSQ_THREAD_TRACE_STATUS2_BASE_IDX                                                             1
+#define regSQ_THREAD_TRACE_GFX_DRAW_CNTR                                                                0x39f6
+#define regSQ_THREAD_TRACE_GFX_DRAW_CNTR_BASE_IDX                                                       1
+#define regSQ_THREAD_TRACE_GFX_MARKER_CNTR                                                              0x39f7
+#define regSQ_THREAD_TRACE_GFX_MARKER_CNTR_BASE_IDX                                                     1
+#define regSQ_THREAD_TRACE_HP3D_DRAW_CNTR                                                               0x39f8
+#define regSQ_THREAD_TRACE_HP3D_DRAW_CNTR_BASE_IDX                                                      1
+#define regSQ_THREAD_TRACE_HP3D_MARKER_CNTR                                                             0x39f9
+#define regSQ_THREAD_TRACE_HP3D_MARKER_CNTR_BASE_IDX                                                    1
+#define regSQ_THREAD_TRACE_DROPPED_CNTR                                                                 0x39fa
+#define regSQ_THREAD_TRACE_DROPPED_CNTR_BASE_IDX                                                        1
+#define regGCEA_PERFCOUNTER2_SELECT                                                                     0x3a00
+#define regGCEA_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regGCEA_PERFCOUNTER2_SELECT1                                                                    0x3a01
+#define regGCEA_PERFCOUNTER2_SELECT1_BASE_IDX                                                           1
+#define regGCEA_PERFCOUNTER2_MODE                                                                       0x3a02
+#define regGCEA_PERFCOUNTER2_MODE_BASE_IDX                                                              1
+#define regGCEA_PERFCOUNTER0_CFG                                                                        0x3a03
+#define regGCEA_PERFCOUNTER0_CFG_BASE_IDX                                                               1
+#define regGCEA_PERFCOUNTER1_CFG                                                                        0x3a04
+#define regGCEA_PERFCOUNTER1_CFG_BASE_IDX                                                               1
+#define regGCEA_PERFCOUNTER_RSLT_CNTL                                                                   0x3a05
+#define regGCEA_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                          1
+#define regSX_PERFCOUNTER0_SELECT                                                                       0x3a40
+#define regSX_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER1_SELECT                                                                       0x3a41
+#define regSX_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER2_SELECT                                                                       0x3a42
+#define regSX_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER3_SELECT                                                                       0x3a43
+#define regSX_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regSX_PERFCOUNTER0_SELECT1                                                                      0x3a44
+#define regSX_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regSX_PERFCOUNTER1_SELECT1                                                                      0x3a45
+#define regSX_PERFCOUNTER1_SELECT1_BASE_IDX                                                             1
+#define regGDS_PERFCOUNTER0_SELECT                                                                      0x3a80
+#define regGDS_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regGDS_PERFCOUNTER1_SELECT                                                                      0x3a81
+#define regGDS_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regGDS_PERFCOUNTER2_SELECT                                                                      0x3a82
+#define regGDS_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regGDS_PERFCOUNTER3_SELECT                                                                      0x3a83
+#define regGDS_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regGDS_PERFCOUNTER0_SELECT1                                                                     0x3a84
+#define regGDS_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regGDS_PERFCOUNTER1_SELECT1                                                                     0x3a85
+#define regGDS_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regGDS_PERFCOUNTER2_SELECT1                                                                     0x3a86
+#define regGDS_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regGDS_PERFCOUNTER3_SELECT1                                                                     0x3a87
+#define regGDS_PERFCOUNTER3_SELECT1_BASE_IDX                                                            1
+#define regTA_PERFCOUNTER0_SELECT                                                                       0x3ac0
+#define regTA_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regTA_PERFCOUNTER0_SELECT1                                                                      0x3ac1
+#define regTA_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regTA_PERFCOUNTER1_SELECT                                                                       0x3ac2
+#define regTA_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regTD_PERFCOUNTER0_SELECT                                                                       0x3b00
+#define regTD_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regTD_PERFCOUNTER0_SELECT1                                                                      0x3b01
+#define regTD_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regTD_PERFCOUNTER1_SELECT                                                                       0x3b02
+#define regTD_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regTCP_PERFCOUNTER0_SELECT                                                                      0x3b40
+#define regTCP_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER0_SELECT1                                                                     0x3b41
+#define regTCP_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regTCP_PERFCOUNTER1_SELECT                                                                      0x3b42
+#define regTCP_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER1_SELECT1                                                                     0x3b43
+#define regTCP_PERFCOUNTER1_SELECT1_BASE_IDX                                                            1
+#define regTCP_PERFCOUNTER2_SELECT                                                                      0x3b44
+#define regTCP_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regTCP_PERFCOUNTER3_SELECT                                                                      0x3b45
+#define regTCP_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regGL2C_PERFCOUNTER0_SELECT                                                                     0x3b80
+#define regGL2C_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGL2C_PERFCOUNTER0_SELECT1                                                                    0x3b81
+#define regGL2C_PERFCOUNTER0_SELECT1_BASE_IDX                                                           1
+#define regGL2C_PERFCOUNTER1_SELECT                                                                     0x3b82
+#define regGL2C_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGL2C_PERFCOUNTER1_SELECT1                                                                    0x3b83
+#define regGL2C_PERFCOUNTER1_SELECT1_BASE_IDX                                                           1
+#define regGL2C_PERFCOUNTER2_SELECT                                                                     0x3b84
+#define regGL2C_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regGL2C_PERFCOUNTER3_SELECT                                                                     0x3b85
+#define regGL2C_PERFCOUNTER3_SELECT_BASE_IDX                                                            1
+#define regGL2A_PERFCOUNTER0_SELECT                                                                     0x3b90
+#define regGL2A_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGL2A_PERFCOUNTER0_SELECT1                                                                    0x3b91
+#define regGL2A_PERFCOUNTER0_SELECT1_BASE_IDX                                                           1
+#define regGL2A_PERFCOUNTER1_SELECT                                                                     0x3b92
+#define regGL2A_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGL2A_PERFCOUNTER1_SELECT1                                                                    0x3b93
+#define regGL2A_PERFCOUNTER1_SELECT1_BASE_IDX                                                           1
+#define regGL2A_PERFCOUNTER2_SELECT                                                                     0x3b94
+#define regGL2A_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regGL2A_PERFCOUNTER3_SELECT                                                                     0x3b95
+#define regGL2A_PERFCOUNTER3_SELECT_BASE_IDX                                                            1
+#define regGL1C_PERFCOUNTER0_SELECT                                                                     0x3ba0
+#define regGL1C_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGL1C_PERFCOUNTER0_SELECT1                                                                    0x3ba1
+#define regGL1C_PERFCOUNTER0_SELECT1_BASE_IDX                                                           1
+#define regGL1C_PERFCOUNTER1_SELECT                                                                     0x3ba2
+#define regGL1C_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGL1C_PERFCOUNTER2_SELECT                                                                     0x3ba3
+#define regGL1C_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regGL1C_PERFCOUNTER3_SELECT                                                                     0x3ba4
+#define regGL1C_PERFCOUNTER3_SELECT_BASE_IDX                                                            1
+#define regCHC_PERFCOUNTER0_SELECT                                                                      0x3bc0
+#define regCHC_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCHC_PERFCOUNTER0_SELECT1                                                                     0x3bc1
+#define regCHC_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCHC_PERFCOUNTER1_SELECT                                                                      0x3bc2
+#define regCHC_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCHC_PERFCOUNTER2_SELECT                                                                      0x3bc3
+#define regCHC_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regCHC_PERFCOUNTER3_SELECT                                                                      0x3bc4
+#define regCHC_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regCHCG_PERFCOUNTER0_SELECT                                                                     0x3bc6
+#define regCHCG_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regCHCG_PERFCOUNTER0_SELECT1                                                                    0x3bc7
+#define regCHCG_PERFCOUNTER0_SELECT1_BASE_IDX                                                           1
+#define regCHCG_PERFCOUNTER1_SELECT                                                                     0x3bc8
+#define regCHCG_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regCHCG_PERFCOUNTER2_SELECT                                                                     0x3bc9
+#define regCHCG_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regCHCG_PERFCOUNTER3_SELECT                                                                     0x3bca
+#define regCHCG_PERFCOUNTER3_SELECT_BASE_IDX                                                            1
+#define regCB_PERFCOUNTER_FILTER                                                                        0x3c00
+#define regCB_PERFCOUNTER_FILTER_BASE_IDX                                                               1
+#define regCB_PERFCOUNTER0_SELECT                                                                       0x3c01
+#define regCB_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regCB_PERFCOUNTER0_SELECT1                                                                      0x3c02
+#define regCB_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regCB_PERFCOUNTER1_SELECT                                                                       0x3c03
+#define regCB_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regCB_PERFCOUNTER2_SELECT                                                                       0x3c04
+#define regCB_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regCB_PERFCOUNTER3_SELECT                                                                       0x3c05
+#define regCB_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER0_SELECT                                                                       0x3c40
+#define regDB_PERFCOUNTER0_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER0_SELECT1                                                                      0x3c41
+#define regDB_PERFCOUNTER0_SELECT1_BASE_IDX                                                             1
+#define regDB_PERFCOUNTER1_SELECT                                                                       0x3c42
+#define regDB_PERFCOUNTER1_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER1_SELECT1                                                                      0x3c43
+#define regDB_PERFCOUNTER1_SELECT1_BASE_IDX                                                             1
+#define regDB_PERFCOUNTER2_SELECT                                                                       0x3c44
+#define regDB_PERFCOUNTER2_SELECT_BASE_IDX                                                              1
+#define regDB_PERFCOUNTER3_SELECT                                                                       0x3c46
+#define regDB_PERFCOUNTER3_SELECT_BASE_IDX                                                              1
+#define regRLC_SPM_PERFMON_CNTL                                                                         0x3c80
+#define regRLC_SPM_PERFMON_CNTL_BASE_IDX                                                                1
+#define regRLC_SPM_PERFMON_RING_BASE_LO                                                                 0x3c81
+#define regRLC_SPM_PERFMON_RING_BASE_LO_BASE_IDX                                                        1
+#define regRLC_SPM_PERFMON_RING_BASE_HI                                                                 0x3c82
+#define regRLC_SPM_PERFMON_RING_BASE_HI_BASE_IDX                                                        1
+#define regRLC_SPM_PERFMON_RING_SIZE                                                                    0x3c83
+#define regRLC_SPM_PERFMON_RING_SIZE_BASE_IDX                                                           1
+#define regRLC_SPM_RING_WRPTR                                                                           0x3c84
+#define regRLC_SPM_RING_WRPTR_BASE_IDX                                                                  1
+#define regRLC_SPM_RING_RDPTR                                                                           0x3c85
+#define regRLC_SPM_RING_RDPTR_BASE_IDX                                                                  1
+#define regRLC_SPM_SEGMENT_THRESHOLD                                                                    0x3c86
+#define regRLC_SPM_SEGMENT_THRESHOLD_BASE_IDX                                                           1
+#define regRLC_SPM_PERFMON_SEGMENT_SIZE                                                                 0x3c87
+#define regRLC_SPM_PERFMON_SEGMENT_SIZE_BASE_IDX                                                        1
+#define regRLC_SPM_GLOBAL_MUXSEL_ADDR                                                                   0x3c88
+#define regRLC_SPM_GLOBAL_MUXSEL_ADDR_BASE_IDX                                                          1
+#define regRLC_SPM_GLOBAL_MUXSEL_DATA                                                                   0x3c89
+#define regRLC_SPM_GLOBAL_MUXSEL_DATA_BASE_IDX                                                          1
+#define regRLC_SPM_SE_MUXSEL_ADDR                                                                       0x3c8a
+#define regRLC_SPM_SE_MUXSEL_ADDR_BASE_IDX                                                              1
+#define regRLC_SPM_SE_MUXSEL_DATA                                                                       0x3c8b
+#define regRLC_SPM_SE_MUXSEL_DATA_BASE_IDX                                                              1
+#define regRLC_SPM_ACCUM_DATARAM_ADDR                                                                   0x3c92
+#define regRLC_SPM_ACCUM_DATARAM_ADDR_BASE_IDX                                                          1
+#define regRLC_SPM_ACCUM_DATARAM_DATA                                                                   0x3c93
+#define regRLC_SPM_ACCUM_DATARAM_DATA_BASE_IDX                                                          1
+#define regRLC_SPM_ACCUM_SWA_DATARAM_ADDR                                                               0x3c94
+#define regRLC_SPM_ACCUM_SWA_DATARAM_ADDR_BASE_IDX                                                      1
+#define regRLC_SPM_ACCUM_SWA_DATARAM_DATA                                                               0x3c95
+#define regRLC_SPM_ACCUM_SWA_DATARAM_DATA_BASE_IDX                                                      1
+#define regRLC_SPM_ACCUM_CTRLRAM_ADDR                                                                   0x3c96
+#define regRLC_SPM_ACCUM_CTRLRAM_ADDR_BASE_IDX                                                          1
+#define regRLC_SPM_ACCUM_CTRLRAM_DATA                                                                   0x3c97
+#define regRLC_SPM_ACCUM_CTRLRAM_DATA_BASE_IDX                                                          1
+#define regRLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET                                                            0x3c98
+#define regRLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET_BASE_IDX                                                   1
+#define regRLC_SPM_ACCUM_STATUS                                                                         0x3c99
+#define regRLC_SPM_ACCUM_STATUS_BASE_IDX                                                                1
+#define regRLC_SPM_ACCUM_CTRL                                                                           0x3c9a
+#define regRLC_SPM_ACCUM_CTRL_BASE_IDX                                                                  1
+#define regRLC_SPM_ACCUM_MODE                                                                           0x3c9b
+#define regRLC_SPM_ACCUM_MODE_BASE_IDX                                                                  1
+#define regRLC_SPM_ACCUM_THRESHOLD                                                                      0x3c9c
+#define regRLC_SPM_ACCUM_THRESHOLD_BASE_IDX                                                             1
+#define regRLC_SPM_ACCUM_SAMPLES_REQUESTED                                                              0x3c9d
+#define regRLC_SPM_ACCUM_SAMPLES_REQUESTED_BASE_IDX                                                     1
+#define regRLC_SPM_ACCUM_DATARAM_WRCOUNT                                                                0x3c9e
+#define regRLC_SPM_ACCUM_DATARAM_WRCOUNT_BASE_IDX                                                       1
+#define regRLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS                                                     0x3c9f
+#define regRLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS_BASE_IDX                                            1
+#define regRLC_SPM_PAUSE                                                                                0x3ca2
+#define regRLC_SPM_PAUSE_BASE_IDX                                                                       1
+#define regRLC_SPM_STATUS                                                                               0x3ca3
+#define regRLC_SPM_STATUS_BASE_IDX                                                                      1
+#define regRLC_SPM_GFXCLOCK_LOWCOUNT                                                                    0x3ca4
+#define regRLC_SPM_GFXCLOCK_LOWCOUNT_BASE_IDX                                                           1
+#define regRLC_SPM_GFXCLOCK_HIGHCOUNT                                                                   0x3ca5
+#define regRLC_SPM_GFXCLOCK_HIGHCOUNT_BASE_IDX                                                          1
+#define regRLC_SPM_MODE                                                                                 0x3cad
+#define regRLC_SPM_MODE_BASE_IDX                                                                        1
+#define regRLC_SPM_RSPM_REQ_DATA_LO                                                                     0x3cae
+#define regRLC_SPM_RSPM_REQ_DATA_LO_BASE_IDX                                                            1
+#define regRLC_SPM_RSPM_REQ_DATA_HI                                                                     0x3caf
+#define regRLC_SPM_RSPM_REQ_DATA_HI_BASE_IDX                                                            1
+#define regRLC_SPM_RSPM_REQ_OP                                                                          0x3cb0
+#define regRLC_SPM_RSPM_REQ_OP_BASE_IDX                                                                 1
+#define regRLC_SPM_RSPM_RET_DATA                                                                        0x3cb1
+#define regRLC_SPM_RSPM_RET_DATA_BASE_IDX                                                               1
+#define regRLC_SPM_RSPM_RET_OP                                                                          0x3cb2
+#define regRLC_SPM_RSPM_RET_OP_BASE_IDX                                                                 1
+#define regRLC_SPM_SE_RSPM_REQ_DATA_LO                                                                  0x3cb3
+#define regRLC_SPM_SE_RSPM_REQ_DATA_LO_BASE_IDX                                                         1
+#define regRLC_SPM_SE_RSPM_REQ_DATA_HI                                                                  0x3cb4
+#define regRLC_SPM_SE_RSPM_REQ_DATA_HI_BASE_IDX                                                         1
+#define regRLC_SPM_SE_RSPM_REQ_OP                                                                       0x3cb5
+#define regRLC_SPM_SE_RSPM_REQ_OP_BASE_IDX                                                              1
+#define regRLC_SPM_SE_RSPM_RET_DATA                                                                     0x3cb6
+#define regRLC_SPM_SE_RSPM_RET_DATA_BASE_IDX                                                            1
+#define regRLC_SPM_SE_RSPM_RET_OP                                                                       0x3cb7
+#define regRLC_SPM_SE_RSPM_RET_OP_BASE_IDX                                                              1
+#define regRLC_SPM_RSPM_CMD                                                                             0x3cb8
+#define regRLC_SPM_RSPM_CMD_BASE_IDX                                                                    1
+#define regRLC_SPM_RSPM_CMD_ACK                                                                         0x3cb9
+#define regRLC_SPM_RSPM_CMD_ACK_BASE_IDX                                                                1
+#define regRLC_SPM_SPARE                                                                                0x3cbf
+#define regRLC_SPM_SPARE_BASE_IDX                                                                       1
+#define regRLC_PERFMON_CNTL                                                                             0x3cc0
+#define regRLC_PERFMON_CNTL_BASE_IDX                                                                    1
+#define regRLC_PERFCOUNTER0_SELECT                                                                      0x3cc1
+#define regRLC_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regRLC_PERFCOUNTER1_SELECT                                                                      0x3cc2
+#define regRLC_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regRLC_GPU_IOV_PERF_CNT_CNTL                                                                    0x3cc3
+#define regRLC_GPU_IOV_PERF_CNT_CNTL_BASE_IDX                                                           1
+#define regRLC_GPU_IOV_PERF_CNT_WR_ADDR                                                                 0x3cc4
+#define regRLC_GPU_IOV_PERF_CNT_WR_ADDR_BASE_IDX                                                        1
+#define regRLC_GPU_IOV_PERF_CNT_WR_DATA                                                                 0x3cc5
+#define regRLC_GPU_IOV_PERF_CNT_WR_DATA_BASE_IDX                                                        1
+#define regRLC_GPU_IOV_PERF_CNT_RD_ADDR                                                                 0x3cc6
+#define regRLC_GPU_IOV_PERF_CNT_RD_ADDR_BASE_IDX                                                        1
+#define regRLC_GPU_IOV_PERF_CNT_RD_DATA                                                                 0x3cc7
+#define regRLC_GPU_IOV_PERF_CNT_RD_DATA_BASE_IDX                                                        1
+#define regRMI_PERFCOUNTER0_SELECT                                                                      0x3d00
+#define regRMI_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regRMI_PERFCOUNTER0_SELECT1                                                                     0x3d01
+#define regRMI_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regRMI_PERFCOUNTER1_SELECT                                                                      0x3d02
+#define regRMI_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regRMI_PERFCOUNTER2_SELECT                                                                      0x3d03
+#define regRMI_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regRMI_PERFCOUNTER2_SELECT1                                                                     0x3d04
+#define regRMI_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regRMI_PERFCOUNTER3_SELECT                                                                      0x3d05
+#define regRMI_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regRMI_PERF_COUNTER_CNTL                                                                        0x3d06
+#define regRMI_PERF_COUNTER_CNTL_BASE_IDX                                                               1
+#define regGCR_PERFCOUNTER0_SELECT                                                                      0x3d60
+#define regGCR_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regGCR_PERFCOUNTER0_SELECT1                                                                     0x3d61
+#define regGCR_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regGCR_PERFCOUNTER1_SELECT                                                                      0x3d62
+#define regGCR_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regPA_PH_PERFCOUNTER0_SELECT                                                                    0x3d80
+#define regPA_PH_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER0_SELECT1                                                                   0x3d81
+#define regPA_PH_PERFCOUNTER0_SELECT1_BASE_IDX                                                          1
+#define regPA_PH_PERFCOUNTER1_SELECT                                                                    0x3d82
+#define regPA_PH_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER2_SELECT                                                                    0x3d83
+#define regPA_PH_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER3_SELECT                                                                    0x3d84
+#define regPA_PH_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER4_SELECT                                                                    0x3d85
+#define regPA_PH_PERFCOUNTER4_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER5_SELECT                                                                    0x3d86
+#define regPA_PH_PERFCOUNTER5_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER6_SELECT                                                                    0x3d87
+#define regPA_PH_PERFCOUNTER6_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER7_SELECT                                                                    0x3d88
+#define regPA_PH_PERFCOUNTER7_SELECT_BASE_IDX                                                           1
+#define regPA_PH_PERFCOUNTER1_SELECT1                                                                   0x3d90
+#define regPA_PH_PERFCOUNTER1_SELECT1_BASE_IDX                                                          1
+#define regPA_PH_PERFCOUNTER2_SELECT1                                                                   0x3d91
+#define regPA_PH_PERFCOUNTER2_SELECT1_BASE_IDX                                                          1
+#define regPA_PH_PERFCOUNTER3_SELECT1                                                                   0x3d92
+#define regPA_PH_PERFCOUNTER3_SELECT1_BASE_IDX                                                          1
+#define regUTCL1_PERFCOUNTER0_SELECT                                                                    0x3da0
+#define regUTCL1_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regUTCL1_PERFCOUNTER1_SELECT                                                                    0x3da1
+#define regUTCL1_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regUTCL1_PERFCOUNTER2_SELECT                                                                    0x3da2
+#define regUTCL1_PERFCOUNTER2_SELECT_BASE_IDX                                                           1
+#define regUTCL1_PERFCOUNTER3_SELECT                                                                    0x3da3
+#define regUTCL1_PERFCOUNTER3_SELECT_BASE_IDX                                                           1
+#define regGL1A_PERFCOUNTER0_SELECT                                                                     0x3dc0
+#define regGL1A_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGL1A_PERFCOUNTER0_SELECT1                                                                    0x3dc1
+#define regGL1A_PERFCOUNTER0_SELECT1_BASE_IDX                                                           1
+#define regGL1A_PERFCOUNTER1_SELECT                                                                     0x3dc2
+#define regGL1A_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGL1A_PERFCOUNTER2_SELECT                                                                     0x3dc3
+#define regGL1A_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regGL1A_PERFCOUNTER3_SELECT                                                                     0x3dc4
+#define regGL1A_PERFCOUNTER3_SELECT_BASE_IDX                                                            1
+#define regGL1H_PERFCOUNTER0_SELECT                                                                     0x3dd0
+#define regGL1H_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGL1H_PERFCOUNTER0_SELECT1                                                                    0x3dd1
+#define regGL1H_PERFCOUNTER0_SELECT1_BASE_IDX                                                           1
+#define regGL1H_PERFCOUNTER1_SELECT                                                                     0x3dd2
+#define regGL1H_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGL1H_PERFCOUNTER2_SELECT                                                                     0x3dd3
+#define regGL1H_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regGL1H_PERFCOUNTER3_SELECT                                                                     0x3dd4
+#define regGL1H_PERFCOUNTER3_SELECT_BASE_IDX                                                            1
+#define regCHA_PERFCOUNTER0_SELECT                                                                      0x3de0
+#define regCHA_PERFCOUNTER0_SELECT_BASE_IDX                                                             1
+#define regCHA_PERFCOUNTER0_SELECT1                                                                     0x3de1
+#define regCHA_PERFCOUNTER0_SELECT1_BASE_IDX                                                            1
+#define regCHA_PERFCOUNTER1_SELECT                                                                      0x3de2
+#define regCHA_PERFCOUNTER1_SELECT_BASE_IDX                                                             1
+#define regCHA_PERFCOUNTER2_SELECT                                                                      0x3de3
+#define regCHA_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regCHA_PERFCOUNTER3_SELECT                                                                      0x3de4
+#define regCHA_PERFCOUNTER3_SELECT_BASE_IDX                                                             1
+#define regGUS_PERFCOUNTER2_SELECT                                                                      0x3e00
+#define regGUS_PERFCOUNTER2_SELECT_BASE_IDX                                                             1
+#define regGUS_PERFCOUNTER2_SELECT1                                                                     0x3e01
+#define regGUS_PERFCOUNTER2_SELECT1_BASE_IDX                                                            1
+#define regGUS_PERFCOUNTER2_MODE                                                                        0x3e02
+#define regGUS_PERFCOUNTER2_MODE_BASE_IDX                                                               1
+#define regGUS_PERFCOUNTER0_CFG                                                                         0x3e03
+#define regGUS_PERFCOUNTER0_CFG_BASE_IDX                                                                1
+#define regGUS_PERFCOUNTER1_CFG                                                                         0x3e04
+#define regGUS_PERFCOUNTER1_CFG_BASE_IDX                                                                1
+#define regGUS_PERFCOUNTER_RSLT_CNTL                                                                    0x3e05
+#define regGUS_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                           1
+
+
+// addressBlock: gc_gdfll_gdfll_dec
+// base address: 0x3a000
+#define regGDFLL_EDC_HYSTERESIS_CNTL                                                                    0x4828
+#define regGDFLL_EDC_HYSTERESIS_CNTL_BASE_IDX                                                           1
+#define regGDFLL_EDC_HYSTERESIS_STAT                                                                    0x4829
+#define regGDFLL_EDC_HYSTERESIS_STAT_BASE_IDX                                                           1
+
+
+// addressBlock: gc_gdfll_se_gdfll_dec
+// base address: 0x3a300
+#define regGDFLL_SE_EDC_HYSTERESIS_CNTL                                                                 0x48e8
+#define regGDFLL_SE_EDC_HYSTERESIS_CNTL_BASE_IDX                                                        1
+#define regGDFLL_SE_EDC_HYSTERESIS_STAT                                                                 0x48e9
+#define regGDFLL_SE_EDC_HYSTERESIS_STAT_BASE_IDX                                                        1
+
+
+// addressBlock: gc_grtavfs_grtavfs_dec
+// base address: 0x3ac00
+#define regGRTAVFS_RTAVFS_REG_ADDR                                                                      0x4b00
+#define regGRTAVFS_RTAVFS_REG_ADDR_BASE_IDX                                                             1
+#define regGRTAVFS_RTAVFS_WR_DATA                                                                       0x4b01
+#define regGRTAVFS_RTAVFS_WR_DATA_BASE_IDX                                                              1
+#define regGRTAVFS_GENERAL_0                                                                            0x4b02
+#define regGRTAVFS_GENERAL_0_BASE_IDX                                                                   1
+#define regGRTAVFS_RTAVFS_RD_DATA                                                                       0x4b03
+#define regGRTAVFS_RTAVFS_RD_DATA_BASE_IDX                                                              1
+#define regGRTAVFS_RTAVFS_REG_CTRL                                                                      0x4b04
+#define regGRTAVFS_RTAVFS_REG_CTRL_BASE_IDX                                                             1
+#define regGRTAVFS_RTAVFS_REG_STATUS                                                                    0x4b05
+#define regGRTAVFS_RTAVFS_REG_STATUS_BASE_IDX                                                           1
+#define regGRTAVFS_TARG_FREQ                                                                            0x4b06
+#define regGRTAVFS_TARG_FREQ_BASE_IDX                                                                   1
+#define regGRTAVFS_TARG_VOLT                                                                            0x4b07
+#define regGRTAVFS_TARG_VOLT_BASE_IDX                                                                   1
+#define regGRTAVFS_SOFT_RESET                                                                           0x4b0c
+#define regGRTAVFS_SOFT_RESET_BASE_IDX                                                                  1
+#define regGRTAVFS_PSM_CNTL                                                                             0x4b0d
+#define regGRTAVFS_PSM_CNTL_BASE_IDX                                                                    1
+#define regGRTAVFS_CLK_CNTL                                                                             0x4b0e
+#define regGRTAVFS_CLK_CNTL_BASE_IDX                                                                    1
+
+
+// addressBlock: gc_grtavfs_se_grtavfs_dec
+// base address: 0x3ad00
+#define regGRTAVFS_SE_RTAVFS_REG_ADDR                                                                   0x4b40
+#define regGRTAVFS_SE_RTAVFS_REG_ADDR_BASE_IDX                                                          1
+#define regGRTAVFS_SE_RTAVFS_WR_DATA                                                                    0x4b41
+#define regGRTAVFS_SE_RTAVFS_WR_DATA_BASE_IDX                                                           1
+#define regGRTAVFS_SE_GENERAL_0                                                                         0x4b42
+#define regGRTAVFS_SE_GENERAL_0_BASE_IDX                                                                1
+#define regGRTAVFS_SE_RTAVFS_RD_DATA                                                                    0x4b43
+#define regGRTAVFS_SE_RTAVFS_RD_DATA_BASE_IDX                                                           1
+#define regGRTAVFS_SE_RTAVFS_REG_CTRL                                                                   0x4b44
+#define regGRTAVFS_SE_RTAVFS_REG_CTRL_BASE_IDX                                                          1
+#define regGRTAVFS_SE_RTAVFS_REG_STATUS                                                                 0x4b45
+#define regGRTAVFS_SE_RTAVFS_REG_STATUS_BASE_IDX                                                        1
+#define regGRTAVFS_SE_TARG_FREQ                                                                         0x4b46
+#define regGRTAVFS_SE_TARG_FREQ_BASE_IDX                                                                1
+#define regGRTAVFS_SE_TARG_VOLT                                                                         0x4b47
+#define regGRTAVFS_SE_TARG_VOLT_BASE_IDX                                                                1
+#define regGRTAVFS_SE_SOFT_RESET                                                                        0x4b4c
+#define regGRTAVFS_SE_SOFT_RESET_BASE_IDX                                                               1
+#define regGRTAVFS_SE_PSM_CNTL                                                                          0x4b4d
+#define regGRTAVFS_SE_PSM_CNTL_BASE_IDX                                                                 1
+#define regGRTAVFS_SE_CLK_CNTL                                                                          0x4b4e
+#define regGRTAVFS_SE_CLK_CNTL_BASE_IDX                                                                 1
+
+
+// addressBlock: gc_grtavfsdec
+// base address: 0x3ac00
+#define regRTAVFS_RTAVFS_REG_ADDR                                                                       0x4b00
+#define regRTAVFS_RTAVFS_REG_ADDR_BASE_IDX                                                              1
+#define regRTAVFS_RTAVFS_WR_DATA                                                                        0x4b01
+#define regRTAVFS_RTAVFS_WR_DATA_BASE_IDX                                                               1
+
+
+// addressBlock: gc_hypdec
+// base address: 0x3e000
+#define regGFX_PIPE_PRIORITY                                                                            0x587f
+#define regGFX_PIPE_PRIORITY_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_VF_ENABLE                                                                        0x5b00
+#define regRLC_GPU_IOV_VF_ENABLE_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_CFG_REG6                                                                         0x5b06
+#define regRLC_GPU_IOV_CFG_REG6_BASE_IDX                                                                1
+#define regRLC_SDMA0_STATUS                                                                             0x5b18
+#define regRLC_SDMA0_STATUS_BASE_IDX                                                                    1
+#define regRLC_SDMA1_STATUS                                                                             0x5b19
+#define regRLC_SDMA1_STATUS_BASE_IDX                                                                    1
+#define regRLC_SDMA2_STATUS                                                                             0x5b1a
+#define regRLC_SDMA2_STATUS_BASE_IDX                                                                    1
+#define regRLC_SDMA3_STATUS                                                                             0x5b1b
+#define regRLC_SDMA3_STATUS_BASE_IDX                                                                    1
+#define regRLC_SDMA0_BUSY_STATUS                                                                        0x5b1c
+#define regRLC_SDMA0_BUSY_STATUS_BASE_IDX                                                               1
+#define regRLC_SDMA1_BUSY_STATUS                                                                        0x5b1d
+#define regRLC_SDMA1_BUSY_STATUS_BASE_IDX                                                               1
+#define regRLC_SDMA2_BUSY_STATUS                                                                        0x5b1e
+#define regRLC_SDMA2_BUSY_STATUS_BASE_IDX                                                               1
+#define regRLC_SDMA3_BUSY_STATUS                                                                        0x5b1f
+#define regRLC_SDMA3_BUSY_STATUS_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_CFG_REG8                                                                         0x5b20
+#define regRLC_GPU_IOV_CFG_REG8_BASE_IDX                                                                1
+#define regRLC_RLCV_TIMER_INT_0                                                                         0x5b25
+#define regRLC_RLCV_TIMER_INT_0_BASE_IDX                                                                1
+#define regRLC_RLCV_TIMER_INT_1                                                                         0x5b26
+#define regRLC_RLCV_TIMER_INT_1_BASE_IDX                                                                1
+#define regRLC_RLCV_TIMER_CTRL                                                                          0x5b27
+#define regRLC_RLCV_TIMER_CTRL_BASE_IDX                                                                 1
+#define regRLC_RLCV_TIMER_STAT                                                                          0x5b28
+#define regRLC_RLCV_TIMER_STAT_BASE_IDX                                                                 1
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS                                                               0x5b2a
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_BASE_IDX                                                      1
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_SET                                                           0x5b2b
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_SET_BASE_IDX                                                  1
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_CLR                                                           0x5b2c
+#define regRLC_GPU_IOV_VF_DOORBELL_STATUS_CLR_BASE_IDX                                                  1
+#define regRLC_GPU_IOV_VF_MASK                                                                          0x5b2d
+#define regRLC_GPU_IOV_VF_MASK_BASE_IDX                                                                 1
+#define regRLC_HYP_SEMAPHORE_0                                                                          0x5b2e
+#define regRLC_HYP_SEMAPHORE_0_BASE_IDX                                                                 1
+#define regRLC_HYP_SEMAPHORE_1                                                                          0x5b2f
+#define regRLC_HYP_SEMAPHORE_1_BASE_IDX                                                                 1
+#define regRLC_BUSY_CLK_CNTL                                                                            0x5b30
+#define regRLC_BUSY_CLK_CNTL_BASE_IDX                                                                   1
+#define regRLC_CLK_CNTL                                                                                 0x5b31
+#define regRLC_CLK_CNTL_BASE_IDX                                                                        1
+#define regRLC_PACE_TIMER_STAT                                                                          0x5b33
+#define regRLC_PACE_TIMER_STAT_BASE_IDX                                                                 1
+#define regRLC_GPU_IOV_SCH_BLOCK                                                                        0x5b34
+#define regRLC_GPU_IOV_SCH_BLOCK_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_CFG_REG1                                                                         0x5b35
+#define regRLC_GPU_IOV_CFG_REG1_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_CFG_REG2                                                                         0x5b36
+#define regRLC_GPU_IOV_CFG_REG2_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_VM_BUSY_STATUS                                                                   0x5b37
+#define regRLC_GPU_IOV_VM_BUSY_STATUS_BASE_IDX                                                          1
+#define regRLC_GPU_IOV_SCH_0                                                                            0x5b38
+#define regRLC_GPU_IOV_SCH_0_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_ACTIVE_FCN_ID                                                                    0x5b39
+#define regRLC_GPU_IOV_ACTIVE_FCN_ID_BASE_IDX                                                           1
+#define regRLC_GPU_IOV_SCH_3                                                                            0x5b3a
+#define regRLC_GPU_IOV_SCH_3_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_SCH_1                                                                            0x5b3b
+#define regRLC_GPU_IOV_SCH_1_BASE_IDX                                                                   1
+#define regRLC_GPU_IOV_SCH_2                                                                            0x5b3c
+#define regRLC_GPU_IOV_SCH_2_BASE_IDX                                                                   1
+#define regRLC_PACE_INT_FORCE                                                                           0x5b3d
+#define regRLC_PACE_INT_FORCE_BASE_IDX                                                                  1
+#define regRLC_PACE_INT_CLEAR                                                                           0x5b3e
+#define regRLC_PACE_INT_CLEAR_BASE_IDX                                                                  1
+#define regRLC_GPU_IOV_INT_STAT                                                                         0x5b3f
+#define regRLC_GPU_IOV_INT_STAT_BASE_IDX                                                                1
+#define regRLC_IH_COOKIE                                                                                0x5b41
+#define regRLC_IH_COOKIE_BASE_IDX                                                                       1
+#define regRLC_IH_COOKIE_CNTL                                                                           0x5b42
+#define regRLC_IH_COOKIE_CNTL_BASE_IDX                                                                  1
+#define regRLC_HYP_RLCG_UCODE_CHKSUM                                                                    0x5b43
+#define regRLC_HYP_RLCG_UCODE_CHKSUM_BASE_IDX                                                           1
+#define regRLC_HYP_RLCP_UCODE_CHKSUM                                                                    0x5b44
+#define regRLC_HYP_RLCP_UCODE_CHKSUM_BASE_IDX                                                           1
+#define regRLC_HYP_RLCV_UCODE_CHKSUM                                                                    0x5b45
+#define regRLC_HYP_RLCV_UCODE_CHKSUM_BASE_IDX                                                           1
+#define regRLC_GPU_IOV_F32_CNTL                                                                         0x5b46
+#define regRLC_GPU_IOV_F32_CNTL_BASE_IDX                                                                1
+#define regRLC_GPU_IOV_F32_RESET                                                                        0x5b47
+#define regRLC_GPU_IOV_F32_RESET_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_UCODE_ADDR                                                                       0x5b48
+#define regRLC_GPU_IOV_UCODE_ADDR_BASE_IDX                                                              1
+#define regRLC_GPU_IOV_UCODE_DATA                                                                       0x5b49
+#define regRLC_GPU_IOV_UCODE_DATA_BASE_IDX                                                              1
+#define regRLC_GPU_IOV_SMU_RESPONSE                                                                     0x5b4a
+#define regRLC_GPU_IOV_SMU_RESPONSE_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_F32_INVALIDATE_CACHE                                                             0x5b4b
+#define regRLC_GPU_IOV_F32_INVALIDATE_CACHE_BASE_IDX                                                    1
+#define regRLC_GPU_IOV_VIRT_RESET_REQ                                                                   0x5b4c
+#define regRLC_GPU_IOV_VIRT_RESET_REQ_BASE_IDX                                                          1
+#define regRLC_GPU_IOV_RLC_RESPONSE                                                                     0x5b4d
+#define regRLC_GPU_IOV_RLC_RESPONSE_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_INT_DISABLE                                                                      0x5b4e
+#define regRLC_GPU_IOV_INT_DISABLE_BASE_IDX                                                             1
+#define regRLC_GPU_IOV_INT_FORCE                                                                        0x5b4f
+#define regRLC_GPU_IOV_INT_FORCE_BASE_IDX                                                               1
+#define regRLC_GPU_IOV_SCRATCH_ADDR                                                                     0x5b50
+#define regRLC_GPU_IOV_SCRATCH_ADDR_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SCRATCH_DATA                                                                     0x5b51
+#define regRLC_GPU_IOV_SCRATCH_DATA_BASE_IDX                                                            1
+#define regRLC_HYP_SEMAPHORE_2                                                                          0x5b52
+#define regRLC_HYP_SEMAPHORE_2_BASE_IDX                                                                 1
+#define regRLC_HYP_SEMAPHORE_3                                                                          0x5b53
+#define regRLC_HYP_SEMAPHORE_3_BASE_IDX                                                                 1
+#define regRLC_LX6_SCRATCH_ADDR                                                                         0x5b59
+#define regRLC_LX6_SCRATCH_ADDR_BASE_IDX                                                                1
+#define regRLC_LX6_CORE1_SCRATCH_ADDR                                                                   0x5b5b
+#define regRLC_LX6_CORE1_SCRATCH_ADDR_BASE_IDX                                                          1
+#define regRLC_GPM_UCODE_ADDR                                                                           0x5b60
+#define regRLC_GPM_UCODE_ADDR_BASE_IDX                                                                  1
+#define regRLC_GPM_UCODE_DATA                                                                           0x5b61
+#define regRLC_GPM_UCODE_DATA_BASE_IDX                                                                  1
+#define regRLC_GPM_IRAM_ADDR                                                                            0x5b62
+#define regRLC_GPM_IRAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_GPM_IRAM_DATA                                                                            0x5b63
+#define regRLC_GPM_IRAM_DATA_BASE_IDX                                                                   1
+#define regRLC_RLCP_IRAM_ADDR                                                                           0x5b64
+#define regRLC_RLCP_IRAM_ADDR_BASE_IDX                                                                  1
+#define regRLC_RLCP_IRAM_DATA                                                                           0x5b65
+#define regRLC_RLCP_IRAM_DATA_BASE_IDX                                                                  1
+#define regRLC_RLCV_IRAM_ADDR                                                                           0x5b66
+#define regRLC_RLCV_IRAM_ADDR_BASE_IDX                                                                  1
+#define regRLC_RLCV_IRAM_DATA                                                                           0x5b67
+#define regRLC_RLCV_IRAM_DATA_BASE_IDX                                                                  1
+#define regRLC_LX6_DRAM_ADDR                                                                            0x5b68
+#define regRLC_LX6_DRAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_LX6_DRAM_DATA                                                                            0x5b69
+#define regRLC_LX6_DRAM_DATA_BASE_IDX                                                                   1
+#define regRLC_LX6_IRAM_ADDR                                                                            0x5b6a
+#define regRLC_LX6_IRAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_LX6_IRAM_DATA                                                                            0x5b6b
+#define regRLC_LX6_IRAM_DATA_BASE_IDX                                                                   1
+#define regRLC_PACE_UCODE_ADDR                                                                          0x5b6c
+#define regRLC_PACE_UCODE_ADDR_BASE_IDX                                                                 1
+#define regRLC_PACE_UCODE_DATA                                                                          0x5b6d
+#define regRLC_PACE_UCODE_DATA_BASE_IDX                                                                 1
+#define regRLC_GPM_SCRATCH_ADDR                                                                         0x5b6e
+#define regRLC_GPM_SCRATCH_ADDR_BASE_IDX                                                                1
+#define regRLC_GPM_SCRATCH_DATA                                                                         0x5b6f
+#define regRLC_GPM_SCRATCH_DATA_BASE_IDX                                                                1
+#define regRLC_SRM_DRAM_ADDR                                                                            0x5b71
+#define regRLC_SRM_DRAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_SRM_DRAM_DATA                                                                            0x5b72
+#define regRLC_SRM_DRAM_DATA_BASE_IDX                                                                   1
+#define regRLC_SRM_ARAM_ADDR                                                                            0x5b73
+#define regRLC_SRM_ARAM_ADDR_BASE_IDX                                                                   1
+#define regRLC_SRM_ARAM_DATA                                                                            0x5b74
+#define regRLC_SRM_ARAM_DATA_BASE_IDX                                                                   1
+#define regRLC_PACE_SCRATCH_ADDR                                                                        0x5b77
+#define regRLC_PACE_SCRATCH_ADDR_BASE_IDX                                                               1
+#define regRLC_PACE_SCRATCH_DATA                                                                        0x5b78
+#define regRLC_PACE_SCRATCH_DATA_BASE_IDX                                                               1
+#define regRLC_GTS_OFFSET_LSB                                                                           0x5b79
+#define regRLC_GTS_OFFSET_LSB_BASE_IDX                                                                  1
+#define regRLC_GTS_OFFSET_MSB                                                                           0x5b7a
+#define regRLC_GTS_OFFSET_MSB_BASE_IDX                                                                  1
+#define regGL2_PIPE_STEER_0                                                                             0x5b80
+#define regGL2_PIPE_STEER_0_BASE_IDX                                                                    1
+#define regGL2_PIPE_STEER_1                                                                             0x5b81
+#define regGL2_PIPE_STEER_1_BASE_IDX                                                                    1
+#define regGL2_PIPE_STEER_2                                                                             0x5b82
+#define regGL2_PIPE_STEER_2_BASE_IDX                                                                    1
+#define regGL2_PIPE_STEER_3                                                                             0x5b83
+#define regGL2_PIPE_STEER_3_BASE_IDX                                                                    1
+#define regGL1_PIPE_STEER                                                                               0x5b84
+#define regGL1_PIPE_STEER_BASE_IDX                                                                      1
+#define regCH_PIPE_STEER                                                                                0x5b88
+#define regCH_PIPE_STEER_BASE_IDX                                                                       1
+#define regGC_USER_SHADER_ARRAY_CONFIG                                                                  0x5b90
+#define regGC_USER_SHADER_ARRAY_CONFIG_BASE_IDX                                                         1
+#define regGC_USER_PRIM_CONFIG                                                                          0x5b91
+#define regGC_USER_PRIM_CONFIG_BASE_IDX                                                                 1
+#define regGC_USER_SA_UNIT_DISABLE                                                                      0x5b92
+#define regGC_USER_SA_UNIT_DISABLE_BASE_IDX                                                             1
+#define regGC_USER_RB_REDUNDANCY                                                                        0x5b93
+#define regGC_USER_RB_REDUNDANCY_BASE_IDX                                                               1
+#define regGC_USER_RB_BACKEND_DISABLE                                                                   0x5b94
+#define regGC_USER_RB_BACKEND_DISABLE_BASE_IDX                                                          1
+#define regGC_USER_RMI_REDUNDANCY                                                                       0x5b95
+#define regGC_USER_RMI_REDUNDANCY_BASE_IDX                                                              1
+#define regCGTS_USER_TCC_DISABLE                                                                        0x5b96
+#define regCGTS_USER_TCC_DISABLE_BASE_IDX                                                               1
+#define regGC_USER_SHADER_RATE_CONFIG                                                                   0x5b97
+#define regGC_USER_SHADER_RATE_CONFIG_BASE_IDX                                                          1
+#define regRLC_GPU_IOV_SDMA0_STATUS                                                                     0x5bc0
+#define regRLC_GPU_IOV_SDMA0_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA1_STATUS                                                                     0x5bc1
+#define regRLC_GPU_IOV_SDMA1_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA2_STATUS                                                                     0x5bc2
+#define regRLC_GPU_IOV_SDMA2_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA3_STATUS                                                                     0x5bc3
+#define regRLC_GPU_IOV_SDMA3_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA4_STATUS                                                                     0x5bc4
+#define regRLC_GPU_IOV_SDMA4_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA5_STATUS                                                                     0x5bc5
+#define regRLC_GPU_IOV_SDMA5_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA6_STATUS                                                                     0x5bc6
+#define regRLC_GPU_IOV_SDMA6_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA7_STATUS                                                                     0x5bc7
+#define regRLC_GPU_IOV_SDMA7_STATUS_BASE_IDX                                                            1
+#define regRLC_GPU_IOV_SDMA0_BUSY_STATUS                                                                0x5bc8
+#define regRLC_GPU_IOV_SDMA0_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA1_BUSY_STATUS                                                                0x5bc9
+#define regRLC_GPU_IOV_SDMA1_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA2_BUSY_STATUS                                                                0x5bca
+#define regRLC_GPU_IOV_SDMA2_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA3_BUSY_STATUS                                                                0x5bcb
+#define regRLC_GPU_IOV_SDMA3_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA4_BUSY_STATUS                                                                0x5bcc
+#define regRLC_GPU_IOV_SDMA4_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA5_BUSY_STATUS                                                                0x5bcd
+#define regRLC_GPU_IOV_SDMA5_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA6_BUSY_STATUS                                                                0x5bce
+#define regRLC_GPU_IOV_SDMA6_BUSY_STATUS_BASE_IDX                                                       1
+#define regRLC_GPU_IOV_SDMA7_BUSY_STATUS                                                                0x5bcf
+#define regRLC_GPU_IOV_SDMA7_BUSY_STATUS_BASE_IDX                                                       1
+
+
+// addressBlock: gc_cphypdec
+// base address: 0x3e000
+#define regCP_HYP_PFP_UCODE_ADDR                                                                        0x5814
+#define regCP_HYP_PFP_UCODE_ADDR_BASE_IDX                                                               1
+#define regCP_PFP_UCODE_ADDR                                                                            0x5814
+#define regCP_PFP_UCODE_ADDR_BASE_IDX                                                                   1
+#define regCP_HYP_PFP_UCODE_DATA                                                                        0x5815
+#define regCP_HYP_PFP_UCODE_DATA_BASE_IDX                                                               1
+#define regCP_PFP_UCODE_DATA                                                                            0x5815
+#define regCP_PFP_UCODE_DATA_BASE_IDX                                                                   1
+#define regCP_HYP_ME_UCODE_ADDR                                                                         0x5816
+#define regCP_HYP_ME_UCODE_ADDR_BASE_IDX                                                                1
+#define regCP_ME_RAM_RADDR                                                                              0x5816
+#define regCP_ME_RAM_RADDR_BASE_IDX                                                                     1
+#define regCP_ME_RAM_WADDR                                                                              0x5816
+#define regCP_ME_RAM_WADDR_BASE_IDX                                                                     1
+#define regCP_HYP_ME_UCODE_DATA                                                                         0x5817
+#define regCP_HYP_ME_UCODE_DATA_BASE_IDX                                                                1
+#define regCP_ME_RAM_DATA                                                                               0x5817
+#define regCP_ME_RAM_DATA_BASE_IDX                                                                      1
+#define regCP_HYP_MEC1_UCODE_ADDR                                                                       0x581a
+#define regCP_HYP_MEC1_UCODE_ADDR_BASE_IDX                                                              1
+#define regCP_MEC_ME1_UCODE_ADDR                                                                        0x581a
+#define regCP_MEC_ME1_UCODE_ADDR_BASE_IDX                                                               1
+#define regCP_HYP_MEC1_UCODE_DATA                                                                       0x581b
+#define regCP_HYP_MEC1_UCODE_DATA_BASE_IDX                                                              1
+#define regCP_MEC_ME1_UCODE_DATA                                                                        0x581b
+#define regCP_MEC_ME1_UCODE_DATA_BASE_IDX                                                               1
+#define regCP_HYP_MEC2_UCODE_ADDR                                                                       0x581c
+#define regCP_HYP_MEC2_UCODE_ADDR_BASE_IDX                                                              1
+#define regCP_MEC_ME2_UCODE_ADDR                                                                        0x581c
+#define regCP_MEC_ME2_UCODE_ADDR_BASE_IDX                                                               1
+#define regCP_HYP_MEC2_UCODE_DATA                                                                       0x581d
+#define regCP_HYP_MEC2_UCODE_DATA_BASE_IDX                                                              1
+#define regCP_MEC_ME2_UCODE_DATA                                                                        0x581d
+#define regCP_MEC_ME2_UCODE_DATA_BASE_IDX                                                               1
+#define regCP_HYP_PFP_UCODE_CHKSUM                                                                      0x581e
+#define regCP_HYP_PFP_UCODE_CHKSUM_BASE_IDX                                                             1
+#define regCP_HYP_ME_UCODE_CHKSUM                                                                       0x5820
+#define regCP_HYP_ME_UCODE_CHKSUM_BASE_IDX                                                              1
+#define regCP_HYP_MEC_ME1_UCODE_CHKSUM                                                                  0x5821
+#define regCP_HYP_MEC_ME1_UCODE_CHKSUM_BASE_IDX                                                         1
+#define regCP_HYP_MEC_ME2_UCODE_CHKSUM                                                                  0x5822
+#define regCP_HYP_MEC_ME2_UCODE_CHKSUM_BASE_IDX                                                         1
+#define regCP_PFP_IC_BASE_LO                                                                            0x5840
+#define regCP_PFP_IC_BASE_LO_BASE_IDX                                                                   1
+#define regCP_PFP_IC_BASE_HI                                                                            0x5841
+#define regCP_PFP_IC_BASE_HI_BASE_IDX                                                                   1
+#define regCP_PFP_IC_BASE_CNTL                                                                          0x5842
+#define regCP_PFP_IC_BASE_CNTL_BASE_IDX                                                                 1
+#define regCP_PFP_IC_OP_CNTL                                                                            0x5843
+#define regCP_PFP_IC_OP_CNTL_BASE_IDX                                                                   1
+#define regCP_ME_IC_BASE_LO                                                                             0x5844
+#define regCP_ME_IC_BASE_LO_BASE_IDX                                                                    1
+#define regCP_ME_IC_BASE_HI                                                                             0x5845
+#define regCP_ME_IC_BASE_HI_BASE_IDX                                                                    1
+#define regCP_ME_IC_BASE_CNTL                                                                           0x5846
+#define regCP_ME_IC_BASE_CNTL_BASE_IDX                                                                  1
+#define regCP_ME_IC_OP_CNTL                                                                             0x5847
+#define regCP_ME_IC_OP_CNTL_BASE_IDX                                                                    1
+#define regCP_CPC_IC_BASE_LO                                                                            0x584c
+#define regCP_CPC_IC_BASE_LO_BASE_IDX                                                                   1
+#define regCP_CPC_IC_BASE_HI                                                                            0x584d
+#define regCP_CPC_IC_BASE_HI_BASE_IDX                                                                   1
+#define regCP_CPC_IC_BASE_CNTL                                                                          0x584e
+#define regCP_CPC_IC_BASE_CNTL_BASE_IDX                                                                 1
+#define regCP_MES_IC_BASE_LO                                                                            0x5850
+#define regCP_MES_IC_BASE_LO_BASE_IDX                                                                   1
+#define regCP_MES_MIBASE_LO                                                                             0x5850
+#define regCP_MES_MIBASE_LO_BASE_IDX                                                                    1
+#define regCP_MES_IC_BASE_HI                                                                            0x5851
+#define regCP_MES_IC_BASE_HI_BASE_IDX                                                                   1
+#define regCP_MES_MIBASE_HI                                                                             0x5851
+#define regCP_MES_MIBASE_HI_BASE_IDX                                                                    1
+#define regCP_MES_IC_BASE_CNTL                                                                          0x5852
+#define regCP_MES_IC_BASE_CNTL_BASE_IDX                                                                 1
+#define regCP_MES_DC_BASE_LO                                                                            0x5854
+#define regCP_MES_DC_BASE_LO_BASE_IDX                                                                   1
+#define regCP_MES_MDBASE_LO                                                                             0x5854
+#define regCP_MES_MDBASE_LO_BASE_IDX                                                                    1
+#define regCP_MES_DC_BASE_HI                                                                            0x5855
+#define regCP_MES_DC_BASE_HI_BASE_IDX                                                                   1
+#define regCP_MES_MDBASE_HI                                                                             0x5855
+#define regCP_MES_MDBASE_HI_BASE_IDX                                                                    1
+#define regCP_MES_MIBOUND_LO                                                                            0x585b
+#define regCP_MES_MIBOUND_LO_BASE_IDX                                                                   1
+#define regCP_MES_MIBOUND_HI                                                                            0x585c
+#define regCP_MES_MIBOUND_HI_BASE_IDX                                                                   1
+#define regCP_MES_MDBOUND_LO                                                                            0x585d
+#define regCP_MES_MDBOUND_LO_BASE_IDX                                                                   1
+#define regCP_MES_MDBOUND_HI                                                                            0x585e
+#define regCP_MES_MDBOUND_HI_BASE_IDX                                                                   1
+#define regCP_GFX_RS64_DC_BASE0_LO                                                                      0x5863
+#define regCP_GFX_RS64_DC_BASE0_LO_BASE_IDX                                                             1
+#define regCP_GFX_RS64_DC_BASE1_LO                                                                      0x5864
+#define regCP_GFX_RS64_DC_BASE1_LO_BASE_IDX                                                             1
+#define regCP_GFX_RS64_DC_BASE0_HI                                                                      0x5865
+#define regCP_GFX_RS64_DC_BASE0_HI_BASE_IDX                                                             1
+#define regCP_GFX_RS64_DC_BASE1_HI                                                                      0x5866
+#define regCP_GFX_RS64_DC_BASE1_HI_BASE_IDX                                                             1
+#define regCP_GFX_RS64_MIBOUND_LO                                                                       0x586c
+#define regCP_GFX_RS64_MIBOUND_LO_BASE_IDX                                                              1
+#define regCP_GFX_RS64_MIBOUND_HI                                                                       0x586d
+#define regCP_GFX_RS64_MIBOUND_HI_BASE_IDX                                                              1
+#define regCP_MEC_DC_BASE_LO                                                                            0x5870
+#define regCP_MEC_DC_BASE_LO_BASE_IDX                                                                   1
+#define regCP_MEC_MDBASE_LO                                                                             0x5870
+#define regCP_MEC_MDBASE_LO_BASE_IDX                                                                    1
+#define regCP_MEC_DC_BASE_HI                                                                            0x5871
+#define regCP_MEC_DC_BASE_HI_BASE_IDX                                                                   1
+#define regCP_MEC_MDBASE_HI                                                                             0x5871
+#define regCP_MEC_MDBASE_HI_BASE_IDX                                                                    1
+#define regCP_MEC_MIBOUND_LO                                                                            0x5872
+#define regCP_MEC_MIBOUND_LO_BASE_IDX                                                                   1
+#define regCP_MEC_MIBOUND_HI                                                                            0x5873
+#define regCP_MEC_MIBOUND_HI_BASE_IDX                                                                   1
+#define regCP_MEC_MDBOUND_LO                                                                            0x5874
+#define regCP_MEC_MDBOUND_LO_BASE_IDX                                                                   1
+#define regCP_MEC_MDBOUND_HI                                                                            0x5875
+#define regCP_MEC_MDBOUND_HI_BASE_IDX                                                                   1
+
+
+// addressBlock: gc_grbm_hypdec
+// base address: 0x3e800
+#define regGRBM_GFX_INDEX_SR_SELECT                                                                     0x5a00
+#define regGRBM_GFX_INDEX_SR_SELECT_BASE_IDX                                                            1
+#define regGRBM_GFX_INDEX_SR_DATA                                                                       0x5a01
+#define regGRBM_GFX_INDEX_SR_DATA_BASE_IDX                                                              1
+#define regGRBM_GFX_CNTL_SR_SELECT                                                                      0x5a02
+#define regGRBM_GFX_CNTL_SR_SELECT_BASE_IDX                                                             1
+#define regGRBM_GFX_CNTL_SR_DATA                                                                        0x5a03
+#define regGRBM_GFX_CNTL_SR_DATA_BASE_IDX                                                               1
+#define regGC_IH_COOKIE_0_PTR                                                                           0x5a07
+#define regGC_IH_COOKIE_0_PTR_BASE_IDX                                                                  1
+#define regGRBM_SE_REMAP_CNTL                                                                           0x5a08
+#define regGRBM_SE_REMAP_CNTL_BASE_IDX                                                                  1
+
+
+// addressBlock: gc_gcvmsharedhvdec
+// base address: 0x3ea00
+#define regGCMC_VM_FB_SIZE_OFFSET_VF0                                                                   0x5a80
+#define regGCMC_VM_FB_SIZE_OFFSET_VF0_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF1                                                                   0x5a81
+#define regGCMC_VM_FB_SIZE_OFFSET_VF1_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF2                                                                   0x5a82
+#define regGCMC_VM_FB_SIZE_OFFSET_VF2_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF3                                                                   0x5a83
+#define regGCMC_VM_FB_SIZE_OFFSET_VF3_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF4                                                                   0x5a84
+#define regGCMC_VM_FB_SIZE_OFFSET_VF4_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF5                                                                   0x5a85
+#define regGCMC_VM_FB_SIZE_OFFSET_VF5_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF6                                                                   0x5a86
+#define regGCMC_VM_FB_SIZE_OFFSET_VF6_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF7                                                                   0x5a87
+#define regGCMC_VM_FB_SIZE_OFFSET_VF7_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF8                                                                   0x5a88
+#define regGCMC_VM_FB_SIZE_OFFSET_VF8_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF9                                                                   0x5a89
+#define regGCMC_VM_FB_SIZE_OFFSET_VF9_BASE_IDX                                                          1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF10                                                                  0x5a8a
+#define regGCMC_VM_FB_SIZE_OFFSET_VF10_BASE_IDX                                                         1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF11                                                                  0x5a8b
+#define regGCMC_VM_FB_SIZE_OFFSET_VF11_BASE_IDX                                                         1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF12                                                                  0x5a8c
+#define regGCMC_VM_FB_SIZE_OFFSET_VF12_BASE_IDX                                                         1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF13                                                                  0x5a8d
+#define regGCMC_VM_FB_SIZE_OFFSET_VF13_BASE_IDX                                                         1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF14                                                                  0x5a8e
+#define regGCMC_VM_FB_SIZE_OFFSET_VF14_BASE_IDX                                                         1
+#define regGCMC_VM_FB_SIZE_OFFSET_VF15                                                                  0x5a8f
+#define regGCMC_VM_FB_SIZE_OFFSET_VF15_BASE_IDX                                                         1
+
+
+// addressBlock: gc_rlcdec
+// base address: 0x3b000
+#define regRLC_CNTL                                                                                     0x4c00
+#define regRLC_CNTL_BASE_IDX                                                                            1
+#define regRLC_F32_UCODE_VERSION                                                                        0x4c03
+#define regRLC_F32_UCODE_VERSION_BASE_IDX                                                               1
+#define regRLC_STAT                                                                                     0x4c04
+#define regRLC_STAT_BASE_IDX                                                                            1
+#define regRLC_REFCLOCK_TIMESTAMP_LSB                                                                   0x4c0c
+#define regRLC_REFCLOCK_TIMESTAMP_LSB_BASE_IDX                                                          1
+#define regRLC_REFCLOCK_TIMESTAMP_MSB                                                                   0x4c0d
+#define regRLC_REFCLOCK_TIMESTAMP_MSB_BASE_IDX                                                          1
+#define regRLC_GPM_TIMER_INT_0                                                                          0x4c0e
+#define regRLC_GPM_TIMER_INT_0_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_INT_1                                                                          0x4c0f
+#define regRLC_GPM_TIMER_INT_1_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_INT_2                                                                          0x4c10
+#define regRLC_GPM_TIMER_INT_2_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_INT_3                                                                          0x4c11
+#define regRLC_GPM_TIMER_INT_3_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_INT_4                                                                          0x4c12
+#define regRLC_GPM_TIMER_INT_4_BASE_IDX                                                                 1
+#define regRLC_GPM_TIMER_CTRL                                                                           0x4c13
+#define regRLC_GPM_TIMER_CTRL_BASE_IDX                                                                  1
+#define regRLC_GPM_TIMER_STAT                                                                           0x4c14
+#define regRLC_GPM_TIMER_STAT_BASE_IDX                                                                  1
+#define regRLC_GPM_LEGACY_INT_STAT                                                                      0x4c16
+#define regRLC_GPM_LEGACY_INT_STAT_BASE_IDX                                                             1
+#define regRLC_GPM_LEGACY_INT_CLEAR                                                                     0x4c17
+#define regRLC_GPM_LEGACY_INT_CLEAR_BASE_IDX                                                            1
+#define regRLC_INT_STAT                                                                                 0x4c18
+#define regRLC_INT_STAT_BASE_IDX                                                                        1
+#define regRLC_MGCG_CTRL                                                                                0x4c1a
+#define regRLC_MGCG_CTRL_BASE_IDX                                                                       1
+#define regRLC_JUMP_TABLE_RESTORE                                                                       0x4c1e
+#define regRLC_JUMP_TABLE_RESTORE_BASE_IDX                                                              1
+#define regRLC_PG_DELAY_2                                                                               0x4c1f
+#define regRLC_PG_DELAY_2_BASE_IDX                                                                      1
+#define regRLC_GPU_CLOCK_COUNT_LSB                                                                      0x4c24
+#define regRLC_GPU_CLOCK_COUNT_LSB_BASE_IDX                                                             1
+#define regRLC_GPU_CLOCK_COUNT_MSB                                                                      0x4c25
+#define regRLC_GPU_CLOCK_COUNT_MSB_BASE_IDX                                                             1
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT                                                                  0x4c26
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_BASE_IDX                                                         1
+#define regRLC_UCODE_CNTL                                                                               0x4c27
+#define regRLC_UCODE_CNTL_BASE_IDX                                                                      1
+#define regRLC_GPM_THREAD_RESET                                                                         0x4c28
+#define regRLC_GPM_THREAD_RESET_BASE_IDX                                                                1
+#define regRLC_GPM_CP_DMA_COMPLETE_T0                                                                   0x4c29
+#define regRLC_GPM_CP_DMA_COMPLETE_T0_BASE_IDX                                                          1
+#define regRLC_GPM_CP_DMA_COMPLETE_T1                                                                   0x4c2a
+#define regRLC_GPM_CP_DMA_COMPLETE_T1_BASE_IDX                                                          1
+#define regRLC_GPM_THREAD_INVALIDATE_CACHE                                                              0x4c2b
+#define regRLC_GPM_THREAD_INVALIDATE_CACHE_BASE_IDX                                                     1
+#define regRLC_CLK_COUNT_GFXCLK_LSB                                                                     0x4c30
+#define regRLC_CLK_COUNT_GFXCLK_LSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_GFXCLK_MSB                                                                     0x4c31
+#define regRLC_CLK_COUNT_GFXCLK_MSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_REFCLK_LSB                                                                     0x4c32
+#define regRLC_CLK_COUNT_REFCLK_LSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_REFCLK_MSB                                                                     0x4c33
+#define regRLC_CLK_COUNT_REFCLK_MSB_BASE_IDX                                                            1
+#define regRLC_CLK_COUNT_CTRL                                                                           0x4c34
+#define regRLC_CLK_COUNT_CTRL_BASE_IDX                                                                  1
+#define regRLC_CLK_COUNT_STAT                                                                           0x4c35
+#define regRLC_CLK_COUNT_STAT_BASE_IDX                                                                  1
+#define regRLC_RLCG_DOORBELL_CNTL                                                                       0x4c36
+#define regRLC_RLCG_DOORBELL_CNTL_BASE_IDX                                                              1
+#define regRLC_RLCG_DOORBELL_STAT                                                                       0x4c37
+#define regRLC_RLCG_DOORBELL_STAT_BASE_IDX                                                              1
+#define regRLC_RLCG_DOORBELL_0_DATA_LO                                                                  0x4c38
+#define regRLC_RLCG_DOORBELL_0_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_0_DATA_HI                                                                  0x4c39
+#define regRLC_RLCG_DOORBELL_0_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_1_DATA_LO                                                                  0x4c3a
+#define regRLC_RLCG_DOORBELL_1_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_1_DATA_HI                                                                  0x4c3b
+#define regRLC_RLCG_DOORBELL_1_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_2_DATA_LO                                                                  0x4c3c
+#define regRLC_RLCG_DOORBELL_2_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_2_DATA_HI                                                                  0x4c3d
+#define regRLC_RLCG_DOORBELL_2_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_3_DATA_LO                                                                  0x4c3e
+#define regRLC_RLCG_DOORBELL_3_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCG_DOORBELL_3_DATA_HI                                                                  0x4c3f
+#define regRLC_RLCG_DOORBELL_3_DATA_HI_BASE_IDX                                                         1
+#define regRLC_GPU_CLOCK_32_RES_SEL                                                                     0x4c41
+#define regRLC_GPU_CLOCK_32_RES_SEL_BASE_IDX                                                            1
+#define regRLC_GPU_CLOCK_32                                                                             0x4c42
+#define regRLC_GPU_CLOCK_32_BASE_IDX                                                                    1
+#define regRLC_PG_CNTL                                                                                  0x4c43
+#define regRLC_PG_CNTL_BASE_IDX                                                                         1
+#define regRLC_GPM_THREAD_PRIORITY                                                                      0x4c44
+#define regRLC_GPM_THREAD_PRIORITY_BASE_IDX                                                             1
+#define regRLC_GPM_THREAD_ENABLE                                                                        0x4c45
+#define regRLC_GPM_THREAD_ENABLE_BASE_IDX                                                               1
+#define regRLC_RLCG_DOORBELL_RANGE                                                                      0x4c47
+#define regRLC_RLCG_DOORBELL_RANGE_BASE_IDX                                                             1
+#define regRLC_CGTT_MGCG_OVERRIDE                                                                       0x4c48
+#define regRLC_CGTT_MGCG_OVERRIDE_BASE_IDX                                                              1
+#define regRLC_CGCG_CGLS_CTRL                                                                           0x4c49
+#define regRLC_CGCG_CGLS_CTRL_BASE_IDX                                                                  1
+#define regRLC_CGCG_RAMP_CTRL                                                                           0x4c4a
+#define regRLC_CGCG_RAMP_CTRL_BASE_IDX                                                                  1
+#define regRLC_DYN_PG_STATUS                                                                            0x4c4b
+#define regRLC_DYN_PG_STATUS_BASE_IDX                                                                   1
+#define regRLC_DYN_PG_REQUEST                                                                           0x4c4c
+#define regRLC_DYN_PG_REQUEST_BASE_IDX                                                                  1
+#define regRLC_PG_DELAY                                                                                 0x4c4d
+#define regRLC_PG_DELAY_BASE_IDX                                                                        1
+#define regRLC_WGP_STATUS                                                                               0x4c4e
+#define regRLC_WGP_STATUS_BASE_IDX                                                                      1
+#define regRLC_PG_ALWAYS_ON_WGP_MASK                                                                    0x4c53
+#define regRLC_PG_ALWAYS_ON_WGP_MASK_BASE_IDX                                                           1
+#define regRLC_MAX_PG_WGP                                                                               0x4c54
+#define regRLC_MAX_PG_WGP_BASE_IDX                                                                      1
+#define regRLC_AUTO_PG_CTRL                                                                             0x4c55
+#define regRLC_AUTO_PG_CTRL_BASE_IDX                                                                    1
+#define regRLC_SERDES_RD_INDEX                                                                          0x4c59
+#define regRLC_SERDES_RD_INDEX_BASE_IDX                                                                 1
+#define regRLC_SERDES_RD_DATA_0                                                                         0x4c5a
+#define regRLC_SERDES_RD_DATA_0_BASE_IDX                                                                1
+#define regRLC_SERDES_RD_DATA_1                                                                         0x4c5b
+#define regRLC_SERDES_RD_DATA_1_BASE_IDX                                                                1
+#define regRLC_SERDES_RD_DATA_2                                                                         0x4c5c
+#define regRLC_SERDES_RD_DATA_2_BASE_IDX                                                                1
+#define regRLC_SERDES_RD_DATA_3                                                                         0x4c5d
+#define regRLC_SERDES_RD_DATA_3_BASE_IDX                                                                1
+#define regRLC_SERDES_MASK                                                                              0x4c5e
+#define regRLC_SERDES_MASK_BASE_IDX                                                                     1
+#define regRLC_SERDES_CTRL                                                                              0x4c5f
+#define regRLC_SERDES_CTRL_BASE_IDX                                                                     1
+#define regRLC_SERDES_DATA                                                                              0x4c60
+#define regRLC_SERDES_DATA_BASE_IDX                                                                     1
+#define regRLC_SERDES_BUSY                                                                              0x4c61
+#define regRLC_SERDES_BUSY_BASE_IDX                                                                     1
+#define regRLC_GPM_GENERAL_0                                                                            0x4c63
+#define regRLC_GPM_GENERAL_0_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_1                                                                            0x4c64
+#define regRLC_GPM_GENERAL_1_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_2                                                                            0x4c65
+#define regRLC_GPM_GENERAL_2_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_3                                                                            0x4c66
+#define regRLC_GPM_GENERAL_3_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_4                                                                            0x4c67
+#define regRLC_GPM_GENERAL_4_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_5                                                                            0x4c68
+#define regRLC_GPM_GENERAL_5_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_6                                                                            0x4c69
+#define regRLC_GPM_GENERAL_6_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_7                                                                            0x4c6a
+#define regRLC_GPM_GENERAL_7_BASE_IDX                                                                   1
+#define regRLC_STATIC_PG_STATUS                                                                         0x4c6e
+#define regRLC_STATIC_PG_STATUS_BASE_IDX                                                                1
+#define regRLC_GPM_GENERAL_16                                                                           0x4c76
+#define regRLC_GPM_GENERAL_16_BASE_IDX                                                                  1
+#define regRLC_PG_DELAY_3                                                                               0x4c78
+#define regRLC_PG_DELAY_3_BASE_IDX                                                                      1
+#define regRLC_GPR_REG1                                                                                 0x4c79
+#define regRLC_GPR_REG1_BASE_IDX                                                                        1
+#define regRLC_GPR_REG2                                                                                 0x4c7a
+#define regRLC_GPR_REG2_BASE_IDX                                                                        1
+#define regRLC_GPM_INT_DISABLE_TH0                                                                      0x4c7c
+#define regRLC_GPM_INT_DISABLE_TH0_BASE_IDX                                                             1
+#define regRLC_GPM_LEGACY_INT_DISABLE                                                                   0x4c7d
+#define regRLC_GPM_LEGACY_INT_DISABLE_BASE_IDX                                                          1
+#define regRLC_GPM_INT_FORCE_TH0                                                                        0x4c7e
+#define regRLC_GPM_INT_FORCE_TH0_BASE_IDX                                                               1
+#define regRLC_SRM_CNTL                                                                                 0x4c80
+#define regRLC_SRM_CNTL_BASE_IDX                                                                        1
+#define regRLC_SRM_GPM_COMMAND_STATUS                                                                   0x4c88
+#define regRLC_SRM_GPM_COMMAND_STATUS_BASE_IDX                                                          1
+#define regRLC_SRM_INDEX_CNTL_ADDR_0                                                                    0x4c8b
+#define regRLC_SRM_INDEX_CNTL_ADDR_0_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_1                                                                    0x4c8c
+#define regRLC_SRM_INDEX_CNTL_ADDR_1_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_2                                                                    0x4c8d
+#define regRLC_SRM_INDEX_CNTL_ADDR_2_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_3                                                                    0x4c8e
+#define regRLC_SRM_INDEX_CNTL_ADDR_3_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_4                                                                    0x4c8f
+#define regRLC_SRM_INDEX_CNTL_ADDR_4_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_5                                                                    0x4c90
+#define regRLC_SRM_INDEX_CNTL_ADDR_5_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_6                                                                    0x4c91
+#define regRLC_SRM_INDEX_CNTL_ADDR_6_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_ADDR_7                                                                    0x4c92
+#define regRLC_SRM_INDEX_CNTL_ADDR_7_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_0                                                                    0x4c93
+#define regRLC_SRM_INDEX_CNTL_DATA_0_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_1                                                                    0x4c94
+#define regRLC_SRM_INDEX_CNTL_DATA_1_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_2                                                                    0x4c95
+#define regRLC_SRM_INDEX_CNTL_DATA_2_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_3                                                                    0x4c96
+#define regRLC_SRM_INDEX_CNTL_DATA_3_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_4                                                                    0x4c97
+#define regRLC_SRM_INDEX_CNTL_DATA_4_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_5                                                                    0x4c98
+#define regRLC_SRM_INDEX_CNTL_DATA_5_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_6                                                                    0x4c99
+#define regRLC_SRM_INDEX_CNTL_DATA_6_BASE_IDX                                                           1
+#define regRLC_SRM_INDEX_CNTL_DATA_7                                                                    0x4c9a
+#define regRLC_SRM_INDEX_CNTL_DATA_7_BASE_IDX                                                           1
+#define regRLC_SRM_STAT                                                                                 0x4c9b
+#define regRLC_SRM_STAT_BASE_IDX                                                                        1
+#define regRLC_GPM_GENERAL_8                                                                            0x4cad
+#define regRLC_GPM_GENERAL_8_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_9                                                                            0x4cae
+#define regRLC_GPM_GENERAL_9_BASE_IDX                                                                   1
+#define regRLC_GPM_GENERAL_10                                                                           0x4caf
+#define regRLC_GPM_GENERAL_10_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_11                                                                           0x4cb0
+#define regRLC_GPM_GENERAL_11_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_12                                                                           0x4cb1
+#define regRLC_GPM_GENERAL_12_BASE_IDX                                                                  1
+#define regRLC_GPM_UTCL1_CNTL_0                                                                         0x4cb2
+#define regRLC_GPM_UTCL1_CNTL_0_BASE_IDX                                                                1
+#define regRLC_GPM_UTCL1_CNTL_1                                                                         0x4cb3
+#define regRLC_GPM_UTCL1_CNTL_1_BASE_IDX                                                                1
+#define regRLC_GPM_UTCL1_CNTL_2                                                                         0x4cb4
+#define regRLC_GPM_UTCL1_CNTL_2_BASE_IDX                                                                1
+#define regRLC_SPM_UTCL1_CNTL                                                                           0x4cb5
+#define regRLC_SPM_UTCL1_CNTL_BASE_IDX                                                                  1
+#define regRLC_UTCL1_STATUS_2                                                                           0x4cb6
+#define regRLC_UTCL1_STATUS_2_BASE_IDX                                                                  1
+#define regRLC_SPM_UTCL1_ERROR_1                                                                        0x4cbc
+#define regRLC_SPM_UTCL1_ERROR_1_BASE_IDX                                                               1
+#define regRLC_SPM_UTCL1_ERROR_2                                                                        0x4cbd
+#define regRLC_SPM_UTCL1_ERROR_2_BASE_IDX                                                               1
+#define regRLC_GPM_UTCL1_TH0_ERROR_1                                                                    0x4cbe
+#define regRLC_GPM_UTCL1_TH0_ERROR_1_BASE_IDX                                                           1
+#define regRLC_GPM_UTCL1_TH0_ERROR_2                                                                    0x4cc0
+#define regRLC_GPM_UTCL1_TH0_ERROR_2_BASE_IDX                                                           1
+#define regRLC_GPM_UTCL1_TH1_ERROR_1                                                                    0x4cc1
+#define regRLC_GPM_UTCL1_TH1_ERROR_1_BASE_IDX                                                           1
+#define regRLC_GPM_UTCL1_TH1_ERROR_2                                                                    0x4cc2
+#define regRLC_GPM_UTCL1_TH1_ERROR_2_BASE_IDX                                                           1
+#define regRLC_GPM_UTCL1_TH2_ERROR_1                                                                    0x4cc3
+#define regRLC_GPM_UTCL1_TH2_ERROR_1_BASE_IDX                                                           1
+#define regRLC_GPM_UTCL1_TH2_ERROR_2                                                                    0x4cc4
+#define regRLC_GPM_UTCL1_TH2_ERROR_2_BASE_IDX                                                           1
+#define regRLC_CGCG_CGLS_CTRL_3D                                                                        0x4cc5
+#define regRLC_CGCG_CGLS_CTRL_3D_BASE_IDX                                                               1
+#define regRLC_CGCG_RAMP_CTRL_3D                                                                        0x4cc6
+#define regRLC_CGCG_RAMP_CTRL_3D_BASE_IDX                                                               1
+#define regRLC_SEMAPHORE_0                                                                              0x4cc7
+#define regRLC_SEMAPHORE_0_BASE_IDX                                                                     1
+#define regRLC_SEMAPHORE_1                                                                              0x4cc8
+#define regRLC_SEMAPHORE_1_BASE_IDX                                                                     1
+#define regRLC_SEMAPHORE_2                                                                              0x4cc9
+#define regRLC_SEMAPHORE_2_BASE_IDX                                                                     1
+#define regRLC_SEMAPHORE_3                                                                              0x4cca
+#define regRLC_SEMAPHORE_3_BASE_IDX                                                                     1
+#define regRLC_PACE_INT_STAT                                                                            0x4ccc
+#define regRLC_PACE_INT_STAT_BASE_IDX                                                                   1
+#define regRLC_UTCL1_STATUS                                                                             0x4cd4
+#define regRLC_UTCL1_STATUS_BASE_IDX                                                                    1
+#define regRLC_R2I_CNTL_0                                                                               0x4cd5
+#define regRLC_R2I_CNTL_0_BASE_IDX                                                                      1
+#define regRLC_R2I_CNTL_1                                                                               0x4cd6
+#define regRLC_R2I_CNTL_1_BASE_IDX                                                                      1
+#define regRLC_R2I_CNTL_2                                                                               0x4cd7
+#define regRLC_R2I_CNTL_2_BASE_IDX                                                                      1
+#define regRLC_R2I_CNTL_3                                                                               0x4cd8
+#define regRLC_R2I_CNTL_3_BASE_IDX                                                                      1
+#define regRLC_GPM_INT_STAT_TH0                                                                         0x4cdc
+#define regRLC_GPM_INT_STAT_TH0_BASE_IDX                                                                1
+#define regRLC_GPM_GENERAL_13                                                                           0x4cdd
+#define regRLC_GPM_GENERAL_13_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_14                                                                           0x4cde
+#define regRLC_GPM_GENERAL_14_BASE_IDX                                                                  1
+#define regRLC_GPM_GENERAL_15                                                                           0x4cdf
+#define regRLC_GPM_GENERAL_15_BASE_IDX                                                                  1
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_1                                                                0x4cea
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_1_BASE_IDX                                                       1
+#define regRLC_GPU_CLOCK_COUNT_LSB_2                                                                    0x4ceb
+#define regRLC_GPU_CLOCK_COUNT_LSB_2_BASE_IDX                                                           1
+#define regRLC_GPU_CLOCK_COUNT_MSB_2                                                                    0x4cec
+#define regRLC_GPU_CLOCK_COUNT_MSB_2_BASE_IDX                                                           1
+#define regRLC_PACE_INT_DISABLE                                                                         0x4ced
+#define regRLC_PACE_INT_DISABLE_BASE_IDX                                                                1
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_2                                                                0x4cef
+#define regRLC_CAPTURE_GPU_CLOCK_COUNT_2_BASE_IDX                                                       1
+#define regRLC_RLCV_DOORBELL_RANGE                                                                      0x4cf0
+#define regRLC_RLCV_DOORBELL_RANGE_BASE_IDX                                                             1
+#define regRLC_RLCV_DOORBELL_CNTL                                                                       0x4cf1
+#define regRLC_RLCV_DOORBELL_CNTL_BASE_IDX                                                              1
+#define regRLC_RLCV_DOORBELL_STAT                                                                       0x4cf2
+#define regRLC_RLCV_DOORBELL_STAT_BASE_IDX                                                              1
+#define regRLC_RLCV_DOORBELL_0_DATA_LO                                                                  0x4cf3
+#define regRLC_RLCV_DOORBELL_0_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_0_DATA_HI                                                                  0x4cf4
+#define regRLC_RLCV_DOORBELL_0_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_1_DATA_LO                                                                  0x4cf5
+#define regRLC_RLCV_DOORBELL_1_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_1_DATA_HI                                                                  0x4cf6
+#define regRLC_RLCV_DOORBELL_1_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_2_DATA_LO                                                                  0x4cf7
+#define regRLC_RLCV_DOORBELL_2_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_2_DATA_HI                                                                  0x4cf8
+#define regRLC_RLCV_DOORBELL_2_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_3_DATA_LO                                                                  0x4cf9
+#define regRLC_RLCV_DOORBELL_3_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCV_DOORBELL_3_DATA_HI                                                                  0x4cfa
+#define regRLC_RLCV_DOORBELL_3_DATA_HI_BASE_IDX                                                         1
+#define regRLC_GPU_CLOCK_COUNT_LSB_1                                                                    0x4cfb
+#define regRLC_GPU_CLOCK_COUNT_LSB_1_BASE_IDX                                                           1
+#define regRLC_GPU_CLOCK_COUNT_MSB_1                                                                    0x4cfc
+#define regRLC_GPU_CLOCK_COUNT_MSB_1_BASE_IDX                                                           1
+#define regRLC_RLCV_SPARE_INT                                                                           0x4d00
+#define regRLC_RLCV_SPARE_INT_BASE_IDX                                                                  1
+#define regRLC_FIREWALL_VIOLATION                                                                       0x4d02
+#define regRLC_FIREWALL_VIOLATION_BASE_IDX                                                              1
+#define regRLC_PACE_TIMER_INT_0                                                                         0x4d04
+#define regRLC_PACE_TIMER_INT_0_BASE_IDX                                                                1
+#define regRLC_PACE_TIMER_INT_1                                                                         0x4d05
+#define regRLC_PACE_TIMER_INT_1_BASE_IDX                                                                1
+#define regRLC_PACE_TIMER_CTRL                                                                          0x4d06
+#define regRLC_PACE_TIMER_CTRL_BASE_IDX                                                                 1
+#define regRLC_SMU_CLK_REQ                                                                              0x4d08
+#define regRLC_SMU_CLK_REQ_BASE_IDX                                                                     1
+#define regRLC_CP_STAT_INVAL_STAT                                                                       0x4d09
+#define regRLC_CP_STAT_INVAL_STAT_BASE_IDX                                                              1
+#define regRLC_CP_STAT_INVAL_CTRL                                                                       0x4d0a
+#define regRLC_CP_STAT_INVAL_CTRL_BASE_IDX                                                              1
+#define regRLC_SPARE                                                                                    0x4d0b
+#define regRLC_SPARE_BASE_IDX                                                                           1
+#define regRLC_SPP_CTRL                                                                                 0x4d0c
+#define regRLC_SPP_CTRL_BASE_IDX                                                                        1
+#define regRLC_SPP_SHADER_PROFILE_EN                                                                    0x4d0d
+#define regRLC_SPP_SHADER_PROFILE_EN_BASE_IDX                                                           1
+#define regRLC_SPP_SSF_CAPTURE_EN                                                                       0x4d0e
+#define regRLC_SPP_SSF_CAPTURE_EN_BASE_IDX                                                              1
+#define regRLC_SPP_SSF_THRESHOLD_0                                                                      0x4d0f
+#define regRLC_SPP_SSF_THRESHOLD_0_BASE_IDX                                                             1
+#define regRLC_SPP_SSF_THRESHOLD_1                                                                      0x4d10
+#define regRLC_SPP_SSF_THRESHOLD_1_BASE_IDX                                                             1
+#define regRLC_SPP_SSF_THRESHOLD_2                                                                      0x4d11
+#define regRLC_SPP_SSF_THRESHOLD_2_BASE_IDX                                                             1
+#define regRLC_SPP_INFLIGHT_RD_ADDR                                                                     0x4d12
+#define regRLC_SPP_INFLIGHT_RD_ADDR_BASE_IDX                                                            1
+#define regRLC_SPP_INFLIGHT_RD_DATA                                                                     0x4d13
+#define regRLC_SPP_INFLIGHT_RD_DATA_BASE_IDX                                                            1
+#define regRLC_SPP_PROF_INFO_1                                                                          0x4d18
+#define regRLC_SPP_PROF_INFO_1_BASE_IDX                                                                 1
+#define regRLC_SPP_PROF_INFO_2                                                                          0x4d19
+#define regRLC_SPP_PROF_INFO_2_BASE_IDX                                                                 1
+#define regRLC_SPP_GLOBAL_SH_ID                                                                         0x4d1a
+#define regRLC_SPP_GLOBAL_SH_ID_BASE_IDX                                                                1
+#define regRLC_SPP_GLOBAL_SH_ID_VALID                                                                   0x4d1b
+#define regRLC_SPP_GLOBAL_SH_ID_VALID_BASE_IDX                                                          1
+#define regRLC_SPP_STATUS                                                                               0x4d1c
+#define regRLC_SPP_STATUS_BASE_IDX                                                                      1
+#define regRLC_SPP_PVT_STAT_0                                                                           0x4d1d
+#define regRLC_SPP_PVT_STAT_0_BASE_IDX                                                                  1
+#define regRLC_SPP_PVT_STAT_1                                                                           0x4d1e
+#define regRLC_SPP_PVT_STAT_1_BASE_IDX                                                                  1
+#define regRLC_SPP_PVT_STAT_2                                                                           0x4d1f
+#define regRLC_SPP_PVT_STAT_2_BASE_IDX                                                                  1
+#define regRLC_SPP_PVT_STAT_3                                                                           0x4d20
+#define regRLC_SPP_PVT_STAT_3_BASE_IDX                                                                  1
+#define regRLC_SPP_PVT_LEVEL_MAX                                                                        0x4d21
+#define regRLC_SPP_PVT_LEVEL_MAX_BASE_IDX                                                               1
+#define regRLC_SPP_STALL_STATE_UPDATE                                                                   0x4d22
+#define regRLC_SPP_STALL_STATE_UPDATE_BASE_IDX                                                          1
+#define regRLC_SPP_PBB_INFO                                                                             0x4d23
+#define regRLC_SPP_PBB_INFO_BASE_IDX                                                                    1
+#define regRLC_SPP_RESET                                                                                0x4d24
+#define regRLC_SPP_RESET_BASE_IDX                                                                       1
+#define regRLC_RLCP_DOORBELL_RANGE                                                                      0x4d26
+#define regRLC_RLCP_DOORBELL_RANGE_BASE_IDX                                                             1
+#define regRLC_RLCP_DOORBELL_CNTL                                                                       0x4d27
+#define regRLC_RLCP_DOORBELL_CNTL_BASE_IDX                                                              1
+#define regRLC_RLCP_DOORBELL_STAT                                                                       0x4d28
+#define regRLC_RLCP_DOORBELL_STAT_BASE_IDX                                                              1
+#define regRLC_RLCP_DOORBELL_0_DATA_LO                                                                  0x4d29
+#define regRLC_RLCP_DOORBELL_0_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCP_DOORBELL_0_DATA_HI                                                                  0x4d2a
+#define regRLC_RLCP_DOORBELL_0_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCP_DOORBELL_1_DATA_LO                                                                  0x4d2b
+#define regRLC_RLCP_DOORBELL_1_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCP_DOORBELL_1_DATA_HI                                                                  0x4d2c
+#define regRLC_RLCP_DOORBELL_1_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCP_DOORBELL_2_DATA_LO                                                                  0x4d2d
+#define regRLC_RLCP_DOORBELL_2_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCP_DOORBELL_2_DATA_HI                                                                  0x4d2e
+#define regRLC_RLCP_DOORBELL_2_DATA_HI_BASE_IDX                                                         1
+#define regRLC_RLCP_DOORBELL_3_DATA_LO                                                                  0x4d2f
+#define regRLC_RLCP_DOORBELL_3_DATA_LO_BASE_IDX                                                         1
+#define regRLC_RLCP_DOORBELL_3_DATA_HI                                                                  0x4d30
+#define regRLC_RLCP_DOORBELL_3_DATA_HI_BASE_IDX                                                         1
+#define regRLC_CAC_MASK_CNTL                                                                            0x4d45
+#define regRLC_CAC_MASK_CNTL_BASE_IDX                                                                   1
+#define regRLC_POWER_RESIDENCY_CNTR_CTRL                                                                0x4d48
+#define regRLC_POWER_RESIDENCY_CNTR_CTRL_BASE_IDX                                                       1
+#define regRLC_CLK_RESIDENCY_CNTR_CTRL                                                                  0x4d49
+#define regRLC_CLK_RESIDENCY_CNTR_CTRL_BASE_IDX                                                         1
+#define regRLC_DS_RESIDENCY_CNTR_CTRL                                                                   0x4d4a
+#define regRLC_DS_RESIDENCY_CNTR_CTRL_BASE_IDX                                                          1
+#define regRLC_ULV_RESIDENCY_CNTR_CTRL                                                                  0x4d4b
+#define regRLC_ULV_RESIDENCY_CNTR_CTRL_BASE_IDX                                                         1
+#define regRLC_PCC_RESIDENCY_CNTR_CTRL                                                                  0x4d4c
+#define regRLC_PCC_RESIDENCY_CNTR_CTRL_BASE_IDX                                                         1
+#define regRLC_GENERAL_RESIDENCY_CNTR_CTRL                                                              0x4d4d
+#define regRLC_GENERAL_RESIDENCY_CNTR_CTRL_BASE_IDX                                                     1
+#define regRLC_POWER_RESIDENCY_EVENT_CNTR                                                               0x4d50
+#define regRLC_POWER_RESIDENCY_EVENT_CNTR_BASE_IDX                                                      1
+#define regRLC_CLK_RESIDENCY_EVENT_CNTR                                                                 0x4d51
+#define regRLC_CLK_RESIDENCY_EVENT_CNTR_BASE_IDX                                                        1
+#define regRLC_DS_RESIDENCY_EVENT_CNTR                                                                  0x4d52
+#define regRLC_DS_RESIDENCY_EVENT_CNTR_BASE_IDX                                                         1
+#define regRLC_ULV_RESIDENCY_EVENT_CNTR                                                                 0x4d53
+#define regRLC_ULV_RESIDENCY_EVENT_CNTR_BASE_IDX                                                        1
+#define regRLC_PCC_RESIDENCY_EVENT_CNTR                                                                 0x4d54
+#define regRLC_PCC_RESIDENCY_EVENT_CNTR_BASE_IDX                                                        1
+#define regRLC_GENERAL_RESIDENCY_EVENT_CNTR                                                             0x4d55
+#define regRLC_GENERAL_RESIDENCY_EVENT_CNTR_BASE_IDX                                                    1
+#define regRLC_POWER_RESIDENCY_REF_CNTR                                                                 0x4d58
+#define regRLC_POWER_RESIDENCY_REF_CNTR_BASE_IDX                                                        1
+#define regRLC_CLK_RESIDENCY_REF_CNTR                                                                   0x4d59
+#define regRLC_CLK_RESIDENCY_REF_CNTR_BASE_IDX                                                          1
+#define regRLC_DS_RESIDENCY_REF_CNTR                                                                    0x4d5a
+#define regRLC_DS_RESIDENCY_REF_CNTR_BASE_IDX                                                           1
+#define regRLC_ULV_RESIDENCY_REF_CNTR                                                                   0x4d5b
+#define regRLC_ULV_RESIDENCY_REF_CNTR_BASE_IDX                                                          1
+#define regRLC_PCC_RESIDENCY_REF_CNTR                                                                   0x4d5c
+#define regRLC_PCC_RESIDENCY_REF_CNTR_BASE_IDX                                                          1
+#define regRLC_GENERAL_RESIDENCY_REF_CNTR                                                               0x4d5d
+#define regRLC_GENERAL_RESIDENCY_REF_CNTR_BASE_IDX                                                      1
+#define regRLC_GFX_IH_CLIENT_CTRL                                                                       0x4d5e
+#define regRLC_GFX_IH_CLIENT_CTRL_BASE_IDX                                                              1
+#define regRLC_GFX_IH_ARBITER_STAT                                                                      0x4d5f
+#define regRLC_GFX_IH_ARBITER_STAT_BASE_IDX                                                             1
+#define regRLC_GFX_IH_CLIENT_SE_STAT_L                                                                  0x4d60
+#define regRLC_GFX_IH_CLIENT_SE_STAT_L_BASE_IDX                                                         1
+#define regRLC_GFX_IH_CLIENT_SE_STAT_H                                                                  0x4d61
+#define regRLC_GFX_IH_CLIENT_SE_STAT_H_BASE_IDX                                                         1
+#define regRLC_GFX_IH_CLIENT_SDMA_STAT                                                                  0x4d62
+#define regRLC_GFX_IH_CLIENT_SDMA_STAT_BASE_IDX                                                         1
+#define regRLC_GFX_IH_CLIENT_OTHER_STAT                                                                 0x4d63
+#define regRLC_GFX_IH_CLIENT_OTHER_STAT_BASE_IDX                                                        1
+#define regRLC_SPM_GLOBAL_DELAY_IND_ADDR                                                                0x4d64
+#define regRLC_SPM_GLOBAL_DELAY_IND_ADDR_BASE_IDX                                                       1
+#define regRLC_SPM_GLOBAL_DELAY_IND_DATA                                                                0x4d65
+#define regRLC_SPM_GLOBAL_DELAY_IND_DATA_BASE_IDX                                                       1
+#define regRLC_SPM_SE_DELAY_IND_ADDR                                                                    0x4d66
+#define regRLC_SPM_SE_DELAY_IND_ADDR_BASE_IDX                                                           1
+#define regRLC_SPM_SE_DELAY_IND_DATA                                                                    0x4d67
+#define regRLC_SPM_SE_DELAY_IND_DATA_BASE_IDX                                                           1
+#define regRLC_LX6_CNTL                                                                                 0x4d80
+#define regRLC_LX6_CNTL_BASE_IDX                                                                        1
+#define regRLC_XT_CORE_STATUS                                                                           0x4dd4
+#define regRLC_XT_CORE_STATUS_BASE_IDX                                                                  1
+#define regRLC_XT_CORE_INTERRUPT                                                                        0x4dd5
+#define regRLC_XT_CORE_INTERRUPT_BASE_IDX                                                               1
+#define regRLC_XT_CORE_FAULT_INFO                                                                       0x4dd6
+#define regRLC_XT_CORE_FAULT_INFO_BASE_IDX                                                              1
+#define regRLC_XT_CORE_ALT_RESET_VEC                                                                    0x4dd7
+#define regRLC_XT_CORE_ALT_RESET_VEC_BASE_IDX                                                           1
+#define regRLC_XT_CORE_RESERVED                                                                         0x4dd8
+#define regRLC_XT_CORE_RESERVED_BASE_IDX                                                                1
+#define regRLC_XT_INT_VEC_FORCE                                                                         0x4dd9
+#define regRLC_XT_INT_VEC_FORCE_BASE_IDX                                                                1
+#define regRLC_XT_INT_VEC_CLEAR                                                                         0x4dda
+#define regRLC_XT_INT_VEC_CLEAR_BASE_IDX                                                                1
+#define regRLC_XT_INT_VEC_MUX_SEL                                                                       0x4ddb
+#define regRLC_XT_INT_VEC_MUX_SEL_BASE_IDX                                                              1
+#define regRLC_XT_INT_VEC_MUX_INT_SEL                                                                   0x4ddc
+#define regRLC_XT_INT_VEC_MUX_INT_SEL_BASE_IDX                                                          1
+#define regRLC_GPU_CLOCK_COUNT_SPM_LSB                                                                  0x4de4
+#define regRLC_GPU_CLOCK_COUNT_SPM_LSB_BASE_IDX                                                         1
+#define regRLC_GPU_CLOCK_COUNT_SPM_MSB                                                                  0x4de5
+#define regRLC_GPU_CLOCK_COUNT_SPM_MSB_BASE_IDX                                                         1
+#define regRLC_SPM_THREAD_TRACE_CTRL                                                                    0x4de6
+#define regRLC_SPM_THREAD_TRACE_CTRL_BASE_IDX                                                           1
+#define regRLC_SPP_CAM_ADDR                                                                             0x4de8
+#define regRLC_SPP_CAM_ADDR_BASE_IDX                                                                    1
+#define regRLC_SPP_CAM_DATA                                                                             0x4de9
+#define regRLC_SPP_CAM_DATA_BASE_IDX                                                                    1
+#define regRLC_SPP_CAM_EXT_ADDR                                                                         0x4dea
+#define regRLC_SPP_CAM_EXT_ADDR_BASE_IDX                                                                1
+#define regRLC_SPP_CAM_EXT_DATA                                                                         0x4deb
+#define regRLC_SPP_CAM_EXT_DATA_BASE_IDX                                                                1
+#define regRLC_CPAXI_DOORBELL_MON_CTRL                                                                  0x4df1
+#define regRLC_CPAXI_DOORBELL_MON_CTRL_BASE_IDX                                                         1
+#define regRLC_CPAXI_DOORBELL_MON_STAT                                                                  0x4df2
+#define regRLC_CPAXI_DOORBELL_MON_STAT_BASE_IDX                                                         1
+#define regRLC_CPAXI_DOORBELL_MON_DATA_LSB                                                              0x4df3
+#define regRLC_CPAXI_DOORBELL_MON_DATA_LSB_BASE_IDX                                                     1
+#define regRLC_CPAXI_DOORBELL_MON_DATA_MSB                                                              0x4df4
+#define regRLC_CPAXI_DOORBELL_MON_DATA_MSB_BASE_IDX                                                     1
+#define regRLC_XT_DOORBELL_RANGE                                                                        0x4df5
+#define regRLC_XT_DOORBELL_RANGE_BASE_IDX                                                               1
+#define regRLC_XT_DOORBELL_CNTL                                                                         0x4df6
+#define regRLC_XT_DOORBELL_CNTL_BASE_IDX                                                                1
+#define regRLC_XT_DOORBELL_STAT                                                                         0x4df7
+#define regRLC_XT_DOORBELL_STAT_BASE_IDX                                                                1
+#define regRLC_XT_DOORBELL_0_DATA_LO                                                                    0x4df8
+#define regRLC_XT_DOORBELL_0_DATA_LO_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_0_DATA_HI                                                                    0x4df9
+#define regRLC_XT_DOORBELL_0_DATA_HI_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_1_DATA_LO                                                                    0x4dfa
+#define regRLC_XT_DOORBELL_1_DATA_LO_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_1_DATA_HI                                                                    0x4dfb
+#define regRLC_XT_DOORBELL_1_DATA_HI_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_2_DATA_LO                                                                    0x4dfc
+#define regRLC_XT_DOORBELL_2_DATA_LO_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_2_DATA_HI                                                                    0x4dfd
+#define regRLC_XT_DOORBELL_2_DATA_HI_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_3_DATA_LO                                                                    0x4dfe
+#define regRLC_XT_DOORBELL_3_DATA_LO_BASE_IDX                                                           1
+#define regRLC_XT_DOORBELL_3_DATA_HI                                                                    0x4dff
+#define regRLC_XT_DOORBELL_3_DATA_HI_BASE_IDX                                                           1
+#define regRLC_MEM_SLP_CNTL                                                                             0x4e00
+#define regRLC_MEM_SLP_CNTL_BASE_IDX                                                                    1
+#define regSMU_RLC_RESPONSE                                                                             0x4e01
+#define regSMU_RLC_RESPONSE_BASE_IDX                                                                    1
+#define regRLC_RLCV_SAFE_MODE                                                                           0x4e02
+#define regRLC_RLCV_SAFE_MODE_BASE_IDX                                                                  1
+#define regRLC_SMU_SAFE_MODE                                                                            0x4e03
+#define regRLC_SMU_SAFE_MODE_BASE_IDX                                                                   1
+#define regRLC_RLCV_COMMAND                                                                             0x4e04
+#define regRLC_RLCV_COMMAND_BASE_IDX                                                                    1
+#define regRLC_SMU_MESSAGE                                                                              0x4e05
+#define regRLC_SMU_MESSAGE_BASE_IDX                                                                     1
+#define regRLC_SMU_MESSAGE_1                                                                            0x4e06
+#define regRLC_SMU_MESSAGE_1_BASE_IDX                                                                   1
+#define regRLC_SMU_MESSAGE_2                                                                            0x4e07
+#define regRLC_SMU_MESSAGE_2_BASE_IDX                                                                   1
+#define regRLC_SRM_GPM_COMMAND                                                                          0x4e08
+#define regRLC_SRM_GPM_COMMAND_BASE_IDX                                                                 1
+#define regRLC_SRM_GPM_ABORT                                                                            0x4e09
+#define regRLC_SRM_GPM_ABORT_BASE_IDX                                                                   1
+#define regRLC_SMU_COMMAND                                                                              0x4e0a
+#define regRLC_SMU_COMMAND_BASE_IDX                                                                     1
+#define regRLC_SMU_ARGUMENT_1                                                                           0x4e0b
+#define regRLC_SMU_ARGUMENT_1_BASE_IDX                                                                  1
+#define regRLC_SMU_ARGUMENT_2                                                                           0x4e0c
+#define regRLC_SMU_ARGUMENT_2_BASE_IDX                                                                  1
+#define regRLC_SMU_ARGUMENT_3                                                                           0x4e0d
+#define regRLC_SMU_ARGUMENT_3_BASE_IDX                                                                  1
+#define regRLC_SMU_ARGUMENT_4                                                                           0x4e0e
+#define regRLC_SMU_ARGUMENT_4_BASE_IDX                                                                  1
+#define regRLC_SMU_ARGUMENT_5                                                                           0x4e0f
+#define regRLC_SMU_ARGUMENT_5_BASE_IDX                                                                  1
+#define regRLC_IMU_BOOTLOAD_ADDR_HI                                                                     0x4e10
+#define regRLC_IMU_BOOTLOAD_ADDR_HI_BASE_IDX                                                            1
+#define regRLC_IMU_BOOTLOAD_ADDR_LO                                                                     0x4e11
+#define regRLC_IMU_BOOTLOAD_ADDR_LO_BASE_IDX                                                            1
+#define regRLC_IMU_BOOTLOAD_SIZE                                                                        0x4e12
+#define regRLC_IMU_BOOTLOAD_SIZE_BASE_IDX                                                               1
+#define regRLC_IMU_MISC                                                                                 0x4e16
+#define regRLC_IMU_MISC_BASE_IDX                                                                        1
+#define regRLC_IMU_RESET_VECTOR                                                                         0x4e17
+#define regRLC_IMU_RESET_VECTOR_BASE_IDX                                                                1
+
+
+// addressBlock: gc_rlcsdec
+// base address: 0x3b980
+#define regRLC_RLCS_DEC_START                                                                           0x4e60
+#define regRLC_RLCS_DEC_START_BASE_IDX                                                                  1
+#define regRLC_RLCS_DEC_DUMP_ADDR                                                                       0x4e61
+#define regRLC_RLCS_DEC_DUMP_ADDR_BASE_IDX                                                              1
+#define regRLC_RLCS_EXCEPTION_REG_1                                                                     0x4e62
+#define regRLC_RLCS_EXCEPTION_REG_1_BASE_IDX                                                            1
+#define regRLC_RLCS_EXCEPTION_REG_2                                                                     0x4e63
+#define regRLC_RLCS_EXCEPTION_REG_2_BASE_IDX                                                            1
+#define regRLC_RLCS_EXCEPTION_REG_3                                                                     0x4e64
+#define regRLC_RLCS_EXCEPTION_REG_3_BASE_IDX                                                            1
+#define regRLC_RLCS_EXCEPTION_REG_4                                                                     0x4e65
+#define regRLC_RLCS_EXCEPTION_REG_4_BASE_IDX                                                            1
+#define regRLC_RLCS_CGCG_REQUEST                                                                        0x4e66
+#define regRLC_RLCS_CGCG_REQUEST_BASE_IDX                                                               1
+#define regRLC_RLCS_CGCG_STATUS                                                                         0x4e67
+#define regRLC_RLCS_CGCG_STATUS_BASE_IDX                                                                1
+#define regRLC_RLCS_SOC_DS_CNTL                                                                         0x4e68
+#define regRLC_RLCS_SOC_DS_CNTL_BASE_IDX                                                                1
+#define regRLC_RLCS_GFX_DS_CNTL                                                                         0x4e69
+#define regRLC_RLCS_GFX_DS_CNTL_BASE_IDX                                                                1
+#define regRLC_RLCS_GFX_DS_ALLOW_MASK_CNTL                                                              0x4e6a
+#define regRLC_RLCS_GFX_DS_ALLOW_MASK_CNTL_BASE_IDX                                                     1
+#define regRLC_GPM_STAT                                                                                 0x4e6b
+#define regRLC_GPM_STAT_BASE_IDX                                                                        1
+#define regRLC_RLCS_GPM_STAT                                                                            0x4e6b
+#define regRLC_RLCS_GPM_STAT_BASE_IDX                                                                   1
+#define regRLC_RLCS_ABORTED_PD_SEQUENCE                                                                 0x4e6c
+#define regRLC_RLCS_ABORTED_PD_SEQUENCE_BASE_IDX                                                        1
+#define regRLC_RLCS_DIDT_FORCE_STALL                                                                    0x4e6d
+#define regRLC_RLCS_DIDT_FORCE_STALL_BASE_IDX                                                           1
+#define regRLC_RLCS_IOV_CMD_STATUS                                                                      0x4e6e
+#define regRLC_RLCS_IOV_CMD_STATUS_BASE_IDX                                                             1
+#define regRLC_RLCS_IOV_CNTX_LOC_SIZE                                                                   0x4e6f
+#define regRLC_RLCS_IOV_CNTX_LOC_SIZE_BASE_IDX                                                          1
+#define regRLC_RLCS_IOV_SCH_BLOCK                                                                       0x4e70
+#define regRLC_RLCS_IOV_SCH_BLOCK_BASE_IDX                                                              1
+#define regRLC_RLCS_IOV_VM_BUSY_STATUS                                                                  0x4e71
+#define regRLC_RLCS_IOV_VM_BUSY_STATUS_BASE_IDX                                                         1
+#define regRLC_RLCS_GPM_STAT_2                                                                          0x4e72
+#define regRLC_RLCS_GPM_STAT_2_BASE_IDX                                                                 1
+#define regRLC_RLCS_GRBM_SOFT_RESET                                                                     0x4e73
+#define regRLC_RLCS_GRBM_SOFT_RESET_BASE_IDX                                                            1
+#define regRLC_RLCS_PG_CHANGE_STATUS                                                                    0x4e74
+#define regRLC_RLCS_PG_CHANGE_STATUS_BASE_IDX                                                           1
+#define regRLC_RLCS_PG_CHANGE_READ                                                                      0x4e75
+#define regRLC_RLCS_PG_CHANGE_READ_BASE_IDX                                                             1
+#define regRLC_RLCS_IH_SEMAPHORE                                                                        0x4e76
+#define regRLC_RLCS_IH_SEMAPHORE_BASE_IDX                                                               1
+#define regRLC_RLCS_IH_COOKIE_SEMAPHORE                                                                 0x4e77
+#define regRLC_RLCS_IH_COOKIE_SEMAPHORE_BASE_IDX                                                        1
+#define regRLC_RLCS_WGP_STATUS                                                                          0x4e78
+#define regRLC_RLCS_WGP_STATUS_BASE_IDX                                                                 1
+#define regRLC_RLCS_WGP_READ                                                                            0x4e79
+#define regRLC_RLCS_WGP_READ_BASE_IDX                                                                   1
+#define regRLC_RLCS_CP_INT_CTRL_1                                                                       0x4e7a
+#define regRLC_RLCS_CP_INT_CTRL_1_BASE_IDX                                                              1
+#define regRLC_RLCS_CP_INT_CTRL_2                                                                       0x4e7b
+#define regRLC_RLCS_CP_INT_CTRL_2_BASE_IDX                                                              1
+#define regRLC_RLCS_CP_INT_INFO_1                                                                       0x4e7c
+#define regRLC_RLCS_CP_INT_INFO_1_BASE_IDX                                                              1
+#define regRLC_RLCS_CP_INT_INFO_2                                                                       0x4e7d
+#define regRLC_RLCS_CP_INT_INFO_2_BASE_IDX                                                              1
+#define regRLC_RLCS_SPM_INT_CTRL                                                                        0x4e7e
+#define regRLC_RLCS_SPM_INT_CTRL_BASE_IDX                                                               1
+#define regRLC_RLCS_SPM_INT_INFO_1                                                                      0x4e7f
+#define regRLC_RLCS_SPM_INT_INFO_1_BASE_IDX                                                             1
+#define regRLC_RLCS_SPM_INT_INFO_2                                                                      0x4e80
+#define regRLC_RLCS_SPM_INT_INFO_2_BASE_IDX                                                             1
+#define regRLC_RLCS_DSM_TRIG                                                                            0x4e81
+#define regRLC_RLCS_DSM_TRIG_BASE_IDX                                                                   1
+#define regRLC_RLCS_BOOTLOAD_STATUS                                                                     0x4e82
+#define regRLC_RLCS_BOOTLOAD_STATUS_BASE_IDX                                                            1
+#define regRLC_RLCS_POWER_BRAKE_CNTL                                                                    0x4e83
+#define regRLC_RLCS_POWER_BRAKE_CNTL_BASE_IDX                                                           1
+#define regRLC_RLCS_POWER_BRAKE_CNTL_TH1                                                                0x4e84
+#define regRLC_RLCS_POWER_BRAKE_CNTL_TH1_BASE_IDX                                                       1
+#define regRLC_RLCS_GRBM_IDLE_BUSY_STAT                                                                 0x4e85
+#define regRLC_RLCS_GRBM_IDLE_BUSY_STAT_BASE_IDX                                                        1
+#define regRLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL                                                             0x4e86
+#define regRLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL_BASE_IDX                                                    1
+#define regRLC_RLCS_CMP_IDLE_CNTL                                                                       0x4e87
+#define regRLC_RLCS_CMP_IDLE_CNTL_BASE_IDX                                                              1
+#define regRLC_RLCS_GENERAL_0                                                                           0x4e88
+#define regRLC_RLCS_GENERAL_0_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_1                                                                           0x4e89
+#define regRLC_RLCS_GENERAL_1_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_2                                                                           0x4e8a
+#define regRLC_RLCS_GENERAL_2_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_3                                                                           0x4e8b
+#define regRLC_RLCS_GENERAL_3_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_4                                                                           0x4e8c
+#define regRLC_RLCS_GENERAL_4_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_5                                                                           0x4e8d
+#define regRLC_RLCS_GENERAL_5_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_6                                                                           0x4e8e
+#define regRLC_RLCS_GENERAL_6_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_7                                                                           0x4e8f
+#define regRLC_RLCS_GENERAL_7_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_8                                                                           0x4e90
+#define regRLC_RLCS_GENERAL_8_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_9                                                                           0x4e91
+#define regRLC_RLCS_GENERAL_9_BASE_IDX                                                                  1
+#define regRLC_RLCS_GENERAL_10                                                                          0x4e92
+#define regRLC_RLCS_GENERAL_10_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_11                                                                          0x4e93
+#define regRLC_RLCS_GENERAL_11_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_12                                                                          0x4e94
+#define regRLC_RLCS_GENERAL_12_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_13                                                                          0x4e95
+#define regRLC_RLCS_GENERAL_13_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_14                                                                          0x4e96
+#define regRLC_RLCS_GENERAL_14_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_15                                                                          0x4e97
+#define regRLC_RLCS_GENERAL_15_BASE_IDX                                                                 1
+#define regRLC_RLCS_GENERAL_16                                                                          0x4e98
+#define regRLC_RLCS_GENERAL_16_BASE_IDX                                                                 1
+#define regRLC_RLCS_AUXILIARY_REG_1                                                                     0x4ec5
+#define regRLC_RLCS_AUXILIARY_REG_1_BASE_IDX                                                            1
+#define regRLC_RLCS_AUXILIARY_REG_2                                                                     0x4ec6
+#define regRLC_RLCS_AUXILIARY_REG_2_BASE_IDX                                                            1
+#define regRLC_RLCS_AUXILIARY_REG_3                                                                     0x4ec7
+#define regRLC_RLCS_AUXILIARY_REG_3_BASE_IDX                                                            1
+#define regRLC_RLCS_AUXILIARY_REG_4                                                                     0x4ec8
+#define regRLC_RLCS_AUXILIARY_REG_4_BASE_IDX                                                            1
+#define regRLC_RLCS_SPM_SQTT_MODE                                                                       0x4ec9
+#define regRLC_RLCS_SPM_SQTT_MODE_BASE_IDX                                                              1
+#define regRLC_RLCS_CP_DMA_SRCID_OVER                                                                   0x4eca
+#define regRLC_RLCS_CP_DMA_SRCID_OVER_BASE_IDX                                                          1
+#define regRLC_RLCS_BOOTLOAD_ID_STATUS1                                                                 0x4ecb
+#define regRLC_RLCS_BOOTLOAD_ID_STATUS1_BASE_IDX                                                        1
+#define regRLC_RLCS_BOOTLOAD_ID_STATUS2                                                                 0x4ecc
+#define regRLC_RLCS_BOOTLOAD_ID_STATUS2_BASE_IDX                                                        1
+#define regRLC_RLCS_IMU_VIDCHG_CNTL                                                                     0x4ecd
+#define regRLC_RLCS_IMU_VIDCHG_CNTL_BASE_IDX                                                            1
+#define regRLC_RLCS_EDC_INT_CNTL                                                                        0x4ece
+#define regRLC_RLCS_EDC_INT_CNTL_BASE_IDX                                                               1
+#define regRLC_RLCS_KMD_LOG_CNTL1                                                                       0x4ecf
+#define regRLC_RLCS_KMD_LOG_CNTL1_BASE_IDX                                                              1
+#define regRLC_RLCS_KMD_LOG_CNTL2                                                                       0x4ed0
+#define regRLC_RLCS_KMD_LOG_CNTL2_BASE_IDX                                                              1
+#define regRLC_RLCS_GPM_LEGACY_INT_STAT                                                                 0x4ed1
+#define regRLC_RLCS_GPM_LEGACY_INT_STAT_BASE_IDX                                                        1
+#define regRLC_RLCS_GPM_LEGACY_INT_DISABLE                                                              0x4ed2
+#define regRLC_RLCS_GPM_LEGACY_INT_DISABLE_BASE_IDX                                                     1
+#define regRLC_RLCS_SRM_SRCID_CNTL                                                                      0x4ed3
+#define regRLC_RLCS_SRM_SRCID_CNTL_BASE_IDX                                                             1
+#define regRLC_RLCS_GCR_DATA_0                                                                          0x4ed4
+#define regRLC_RLCS_GCR_DATA_0_BASE_IDX                                                                 1
+#define regRLC_RLCS_GCR_DATA_1                                                                          0x4ed5
+#define regRLC_RLCS_GCR_DATA_1_BASE_IDX                                                                 1
+#define regRLC_RLCS_GCR_DATA_2                                                                          0x4ed6
+#define regRLC_RLCS_GCR_DATA_2_BASE_IDX                                                                 1
+#define regRLC_RLCS_GCR_DATA_3                                                                          0x4ed7
+#define regRLC_RLCS_GCR_DATA_3_BASE_IDX                                                                 1
+#define regRLC_RLCS_GCR_STATUS                                                                          0x4ed8
+#define regRLC_RLCS_GCR_STATUS_BASE_IDX                                                                 1
+#define regRLC_RLCS_PERFMON_CLK_CNTL_UCODE                                                              0x4ed9
+#define regRLC_RLCS_PERFMON_CLK_CNTL_UCODE_BASE_IDX                                                     1
+#define regRLC_RLCS_UTCL2_CNTL                                                                          0x4eda
+#define regRLC_RLCS_UTCL2_CNTL_BASE_IDX                                                                 1
+#define regRLC_RLCS_IMU_RLC_MSG_DATA0                                                                   0x4edb
+#define regRLC_RLCS_IMU_RLC_MSG_DATA0_BASE_IDX                                                          1
+#define regRLC_RLCS_IMU_RLC_MSG_DATA1                                                                   0x4edc
+#define regRLC_RLCS_IMU_RLC_MSG_DATA1_BASE_IDX                                                          1
+#define regRLC_RLCS_IMU_RLC_MSG_DATA2                                                                   0x4edd
+#define regRLC_RLCS_IMU_RLC_MSG_DATA2_BASE_IDX                                                          1
+#define regRLC_RLCS_IMU_RLC_MSG_DATA3                                                                   0x4ede
+#define regRLC_RLCS_IMU_RLC_MSG_DATA3_BASE_IDX                                                          1
+#define regRLC_RLCS_IMU_RLC_MSG_DATA4                                                                   0x4edf
+#define regRLC_RLCS_IMU_RLC_MSG_DATA4_BASE_IDX                                                          1
+#define regRLC_RLCS_IMU_RLC_MSG_CONTROL                                                                 0x4ee0
+#define regRLC_RLCS_IMU_RLC_MSG_CONTROL_BASE_IDX                                                        1
+#define regRLC_RLCS_IMU_RLC_MSG_CNTL                                                                    0x4ee1
+#define regRLC_RLCS_IMU_RLC_MSG_CNTL_BASE_IDX                                                           1
+#define regRLC_RLCS_RLC_IMU_MSG_DATA0                                                                   0x4ee2
+#define regRLC_RLCS_RLC_IMU_MSG_DATA0_BASE_IDX                                                          1
+#define regRLC_RLCS_RLC_IMU_MSG_CONTROL                                                                 0x4ee3
+#define regRLC_RLCS_RLC_IMU_MSG_CONTROL_BASE_IDX                                                        1
+#define regRLC_RLCS_RLC_IMU_MSG_CNTL                                                                    0x4ee4
+#define regRLC_RLCS_RLC_IMU_MSG_CNTL_BASE_IDX                                                           1
+#define regRLC_RLCS_IMU_RLC_TELEMETRY_DATA_0                                                            0x4ee5
+#define regRLC_RLCS_IMU_RLC_TELEMETRY_DATA_0_BASE_IDX                                                   1
+#define regRLC_RLCS_IMU_RLC_TELEMETRY_DATA_1                                                            0x4ee6
+#define regRLC_RLCS_IMU_RLC_TELEMETRY_DATA_1_BASE_IDX                                                   1
+#define regRLC_RLCS_IMU_RLC_MUTEX_CNTL                                                                  0x4ee7
+#define regRLC_RLCS_IMU_RLC_MUTEX_CNTL_BASE_IDX                                                         1
+#define regRLC_RLCS_IMU_RLC_STATUS                                                                      0x4ee8
+#define regRLC_RLCS_IMU_RLC_STATUS_BASE_IDX                                                             1
+#define regRLC_RLCS_RLC_IMU_STATUS                                                                      0x4ee9
+#define regRLC_RLCS_RLC_IMU_STATUS_BASE_IDX                                                             1
+#define regRLC_RLCS_IMU_RAM_DATA_1                                                                      0x4eea
+#define regRLC_RLCS_IMU_RAM_DATA_1_BASE_IDX                                                             1
+#define regRLC_RLCS_IMU_RAM_ADDR_1_LSB                                                                  0x4eeb
+#define regRLC_RLCS_IMU_RAM_ADDR_1_LSB_BASE_IDX                                                         1
+#define regRLC_RLCS_IMU_RAM_ADDR_1_MSB                                                                  0x4eec
+#define regRLC_RLCS_IMU_RAM_ADDR_1_MSB_BASE_IDX                                                         1
+#define regRLC_RLCS_IMU_RAM_DATA_0                                                                      0x4eed
+#define regRLC_RLCS_IMU_RAM_DATA_0_BASE_IDX                                                             1
+#define regRLC_RLCS_IMU_RAM_ADDR_0_LSB                                                                  0x4eee
+#define regRLC_RLCS_IMU_RAM_ADDR_0_LSB_BASE_IDX                                                         1
+#define regRLC_RLCS_IMU_RAM_ADDR_0_MSB                                                                  0x4eef
+#define regRLC_RLCS_IMU_RAM_ADDR_0_MSB_BASE_IDX                                                         1
+#define regRLC_RLCS_IMU_RAM_CNTL                                                                        0x4ef0
+#define regRLC_RLCS_IMU_RAM_CNTL_BASE_IDX                                                               1
+#define regRLC_RLCS_IMU_GFX_DOORBELL_FENCE                                                              0x4ef1
+#define regRLC_RLCS_IMU_GFX_DOORBELL_FENCE_BASE_IDX                                                     1
+#define regRLC_RLCS_SDMA_INT_CNTL_1                                                                     0x4ef3
+#define regRLC_RLCS_SDMA_INT_CNTL_1_BASE_IDX                                                            1
+#define regRLC_RLCS_SDMA_INT_CNTL_2                                                                     0x4ef4
+#define regRLC_RLCS_SDMA_INT_CNTL_2_BASE_IDX                                                            1
+#define regRLC_RLCS_SDMA_INT_STAT                                                                       0x4ef5
+#define regRLC_RLCS_SDMA_INT_STAT_BASE_IDX                                                              1
+#define regRLC_RLCS_SDMA_INT_INFO                                                                       0x4ef6
+#define regRLC_RLCS_SDMA_INT_INFO_BASE_IDX                                                              1
+#define regRLC_RLCS_PMM_CGCG_CNTL                                                                       0x4ef7
+#define regRLC_RLCS_PMM_CGCG_CNTL_BASE_IDX                                                              1
+#define regRLC_RLCS_GFX_MEM_POWER_CTRL_LO                                                               0x4ef8
+#define regRLC_RLCS_GFX_MEM_POWER_CTRL_LO_BASE_IDX                                                      1
+#define regRLC_RLCS_GFX_RM_CNTL                                                                         0x4efa
+#define regRLC_RLCS_GFX_RM_CNTL_BASE_IDX                                                                1
+#define regRLC_RLCS_IH_CTRL_1                                                                           0x4efb
+#define regRLC_RLCS_IH_CTRL_1_BASE_IDX                                                                  1
+#define regRLC_RLCS_IH_CTRL_2                                                                           0x4efc
+#define regRLC_RLCS_IH_CTRL_2_BASE_IDX                                                                  1
+#define regRLC_RLCS_IH_CTRL_3                                                                           0x4efd
+#define regRLC_RLCS_IH_CTRL_3_BASE_IDX                                                                  1
+#define regRLC_RLCS_IH_STATUS                                                                           0x4efe
+#define regRLC_RLCS_IH_STATUS_BASE_IDX                                                                  1
+#define regRLC_RLCS_DEC_END                                                                             0x4fff
+#define regRLC_RLCS_DEC_END_BASE_IDX                                                                    1
+
+
+// addressBlock: gc_pfvfdec_rlc
+// base address: 0x2a600
+#define regRLC_SAFE_MODE                                                                                0x0980
+#define regRLC_SAFE_MODE_BASE_IDX                                                                       1
+#define regRLC_SPM_SAMPLE_CNT                                                                           0x0981
+#define regRLC_SPM_SAMPLE_CNT_BASE_IDX                                                                  1
+#define regRLC_SPM_MC_CNTL                                                                              0x0982
+#define regRLC_SPM_MC_CNTL_BASE_IDX                                                                     1
+#define regRLC_SPM_INT_CNTL                                                                             0x0983
+#define regRLC_SPM_INT_CNTL_BASE_IDX                                                                    1
+#define regRLC_SPM_INT_STATUS                                                                           0x0984
+#define regRLC_SPM_INT_STATUS_BASE_IDX                                                                  1
+#define regRLC_SPM_INT_INFO_1                                                                           0x0985
+#define regRLC_SPM_INT_INFO_1_BASE_IDX                                                                  1
+#define regRLC_SPM_INT_INFO_2                                                                           0x0986
+#define regRLC_SPM_INT_INFO_2_BASE_IDX                                                                  1
+#define regRLC_CSIB_ADDR_LO                                                                             0x0987
+#define regRLC_CSIB_ADDR_LO_BASE_IDX                                                                    1
+#define regRLC_CSIB_ADDR_HI                                                                             0x0988
+#define regRLC_CSIB_ADDR_HI_BASE_IDX                                                                    1
+#define regRLC_CSIB_LENGTH                                                                              0x0989
+#define regRLC_CSIB_LENGTH_BASE_IDX                                                                     1
+#define regRLC_CP_SCHEDULERS                                                                            0x098a
+#define regRLC_CP_SCHEDULERS_BASE_IDX                                                                   1
+#define regRLC_CP_EOF_INT                                                                               0x098b
+#define regRLC_CP_EOF_INT_BASE_IDX                                                                      1
+#define regRLC_CP_EOF_INT_CNT                                                                           0x098c
+#define regRLC_CP_EOF_INT_CNT_BASE_IDX                                                                  1
+#define regRLC_SPARE_INT_0                                                                              0x098d
+#define regRLC_SPARE_INT_0_BASE_IDX                                                                     1
+#define regRLC_SPARE_INT_1                                                                              0x098e
+#define regRLC_SPARE_INT_1_BASE_IDX                                                                     1
+#define regRLC_SPARE_INT_2                                                                              0x098f
+#define regRLC_SPARE_INT_2_BASE_IDX                                                                     1
+#define regRLC_PACE_SPARE_INT                                                                           0x0990
+#define regRLC_PACE_SPARE_INT_BASE_IDX                                                                  1
+#define regRLC_PACE_SPARE_INT_1                                                                         0x0991
+#define regRLC_PACE_SPARE_INT_1_BASE_IDX                                                                1
+#define regRLC_RLCV_SPARE_INT_1                                                                         0x0992
+#define regRLC_RLCV_SPARE_INT_1_BASE_IDX                                                                1
+
+
+// addressBlock: gc_pwrdec
+// base address: 0x3c000
+#define regCGTS_TCC_DISABLE                                                                             0x5006
+#define regCGTS_TCC_DISABLE_BASE_IDX                                                                    1
+#define regCGTX_SPI_DEBUG_CLK_CTRL                                                                      0x507f
+#define regCGTX_SPI_DEBUG_CLK_CTRL_BASE_IDX                                                             1
+#define regCGTT_VGT_CLK_CTRL                                                                            0x5084
+#define regCGTT_VGT_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_IA_CLK_CTRL                                                                             0x5085
+#define regCGTT_IA_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_WD_CLK_CTRL                                                                             0x5086
+#define regCGTT_WD_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_GS_NGG_CLK_CTRL                                                                         0x5087
+#define regCGTT_GS_NGG_CLK_CTRL_BASE_IDX                                                                1
+#define regCGTT_PA_CLK_CTRL                                                                             0x5088
+#define regCGTT_PA_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_SC_CLK_CTRL0                                                                            0x5089
+#define regCGTT_SC_CLK_CTRL0_BASE_IDX                                                                   1
+#define regCGTT_SC_CLK_CTRL1                                                                            0x508a
+#define regCGTT_SC_CLK_CTRL1_BASE_IDX                                                                   1
+#define regCGTT_SC_CLK_CTRL2                                                                            0x508b
+#define regCGTT_SC_CLK_CTRL2_BASE_IDX                                                                   1
+#define regCGTT_SQG_CLK_CTRL                                                                            0x508d
+#define regCGTT_SQG_CLK_CTRL_BASE_IDX                                                                   1
+#define regSQ_ALU_CLK_CTRL                                                                              0x508e
+#define regSQ_ALU_CLK_CTRL_BASE_IDX                                                                     1
+#define regSQ_TEX_CLK_CTRL                                                                              0x508f
+#define regSQ_TEX_CLK_CTRL_BASE_IDX                                                                     1
+#define regSQ_LDS_CLK_CTRL                                                                              0x5090
+#define regSQ_LDS_CLK_CTRL_BASE_IDX                                                                     1
+#define regICG_SP_CLK_CTRL                                                                              0x5093
+#define regICG_SP_CLK_CTRL_BASE_IDX                                                                     1
+#define regTA_CGTT_CTRL                                                                                 0x509d
+#define regTA_CGTT_CTRL_BASE_IDX                                                                        1
+#define regDB_CGTT_CLK_CTRL_0                                                                           0x50a4
+#define regDB_CGTT_CLK_CTRL_0_BASE_IDX                                                                  1
+#define regCB_CGTT_SCLK_CTRL                                                                            0x50a8
+#define regCB_CGTT_SCLK_CTRL_BASE_IDX                                                                   1
+#define regGFX_ICG_GL2A_CTRL                                                                            0x50ac
+#define regGFX_ICG_GL2A_CTRL_BASE_IDX                                                                   1
+#define regCGTT_CP_CLK_CTRL                                                                             0x50b0
+#define regCGTT_CP_CLK_CTRL_BASE_IDX                                                                    1
+#define regCGTT_CPF_CLK_CTRL                                                                            0x50b1
+#define regCGTT_CPF_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_CPC_CLK_CTRL                                                                            0x50b2
+#define regCGTT_CPC_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_RLC_CLK_CTRL                                                                            0x50b5
+#define regCGTT_RLC_CLK_CTRL_BASE_IDX                                                                   1
+#define regCGTT_SC_CLK_CTRL3                                                                            0x50bc
+#define regCGTT_SC_CLK_CTRL3_BASE_IDX                                                                   1
+#define regCGTT_SC_CLK_CTRL4                                                                            0x50bd
+#define regCGTT_SC_CLK_CTRL4_BASE_IDX                                                                   1
+#define regGCEA_ICG_CTRL                                                                                0x50c4
+#define regGCEA_ICG_CTRL_BASE_IDX                                                                       1
+#define regGL1I_GL1R_MGCG_OVERRIDE                                                                      0x50e4
+#define regGL1I_GL1R_MGCG_OVERRIDE_BASE_IDX                                                             1
+#define regGL1H_ICG_CTRL                                                                                0x50e8
+#define regGL1H_ICG_CTRL_BASE_IDX                                                                       1
+#define regCHI_CHR_MGCG_OVERRIDE                                                                        0x50e9
+#define regCHI_CHR_MGCG_OVERRIDE_BASE_IDX                                                               1
+#define regICG_GL1C_CLK_CTRL                                                                            0x50ec
+#define regICG_GL1C_CLK_CTRL_BASE_IDX                                                                   1
+#define regICG_GL1A_CTRL                                                                                0x50f0
+#define regICG_GL1A_CTRL_BASE_IDX                                                                       1
+#define regICG_CHA_CTRL                                                                                 0x50f1
+#define regICG_CHA_CTRL_BASE_IDX                                                                        1
+#define regGUS_ICG_CTRL                                                                                 0x50f4
+#define regGUS_ICG_CTRL_BASE_IDX                                                                        1
+#define regCGTT_PH_CLK_CTRL0                                                                            0x50f8
+#define regCGTT_PH_CLK_CTRL0_BASE_IDX                                                                   1
+#define regCGTT_PH_CLK_CTRL1                                                                            0x50f9
+#define regCGTT_PH_CLK_CTRL1_BASE_IDX                                                                   1
+#define regCGTT_PH_CLK_CTRL2                                                                            0x50fa
+#define regCGTT_PH_CLK_CTRL2_BASE_IDX                                                                   1
+#define regCGTT_PH_CLK_CTRL3                                                                            0x50fb
+#define regCGTT_PH_CLK_CTRL3_BASE_IDX                                                                   1
+#define regGFX_ICG_GL2C_CTRL                                                                            0x50fc
+#define regGFX_ICG_GL2C_CTRL_BASE_IDX                                                                   1
+#define regGFX_ICG_GL2C_CTRL1                                                                           0x50fd
+#define regGFX_ICG_GL2C_CTRL1_BASE_IDX                                                                  1
+#define regICG_LDS_CLK_CTRL                                                                             0x5114
+#define regICG_LDS_CLK_CTRL_BASE_IDX                                                                    1
+#define regGFX_ICG_UTCL1_CTRL                                                                           0x511c
+#define regGFX_ICG_UTCL1_CTRL_BASE_IDX                                                                  1
+#define regICG_CHC_CLK_CTRL                                                                             0x5140
+#define regICG_CHC_CLK_CTRL_BASE_IDX                                                                    1
+#define regICG_CHCG_CLK_CTRL                                                                            0x5144
+#define regICG_CHCG_CLK_CTRL_BASE_IDX                                                                   1
+
+
+// addressBlock: gc_pspdec
+// base address: 0x3f000
+#define regCP_MES_DM_INDEX_ADDR                                                                         0x5c00
+#define regCP_MES_DM_INDEX_ADDR_BASE_IDX                                                                1
+#define regCP_MES_DM_INDEX_DATA                                                                         0x5c01
+#define regCP_MES_DM_INDEX_DATA_BASE_IDX                                                                1
+#define regCP_MEC_DM_INDEX_ADDR                                                                         0x5c02
+#define regCP_MEC_DM_INDEX_ADDR_BASE_IDX                                                                1
+#define regCP_MEC_DM_INDEX_DATA                                                                         0x5c03
+#define regCP_MEC_DM_INDEX_DATA_BASE_IDX                                                                1
+#define regCP_GFX_RS64_DM_INDEX_ADDR                                                                    0x5c04
+#define regCP_GFX_RS64_DM_INDEX_ADDR_BASE_IDX                                                           1
+#define regCP_GFX_RS64_DM_INDEX_DATA                                                                    0x5c05
+#define regCP_GFX_RS64_DM_INDEX_DATA_BASE_IDX                                                           1
+#define regCPG_PSP_DEBUG                                                                                0x5c10
+#define regCPG_PSP_DEBUG_BASE_IDX                                                                       1
+#define regCPC_PSP_DEBUG                                                                                0x5c11
+#define regCPC_PSP_DEBUG_BASE_IDX                                                                       1
+#define regGRBM_IOV_ERROR_FIFO                                                                          0x5e07
+#define regGRBM_IOV_ERROR_FIFO_BASE_IDX                                                                 1
+#define regGRBM_SEC_CNTL                                                                                0x5e0d
+#define regGRBM_SEC_CNTL_BASE_IDX                                                                       1
+#define regGRBM_CAM_INDEX                                                                               0x5e10
+#define regGRBM_CAM_INDEX_BASE_IDX                                                                      1
+#define regGRBM_HYP_CAM_INDEX                                                                           0x5e10
+#define regGRBM_HYP_CAM_INDEX_BASE_IDX                                                                  1
+#define regGRBM_CAM_DATA                                                                                0x5e11
+#define regGRBM_CAM_DATA_BASE_IDX                                                                       1
+#define regGRBM_HYP_CAM_DATA                                                                            0x5e11
+#define regGRBM_HYP_CAM_DATA_BASE_IDX                                                                   1
+#define regGRBM_CAM_DATA_UPPER                                                                          0x5e12
+#define regGRBM_CAM_DATA_UPPER_BASE_IDX                                                                 1
+#define regGRBM_HYP_CAM_DATA_UPPER                                                                      0x5e12
+#define regGRBM_HYP_CAM_DATA_UPPER_BASE_IDX                                                             1
+#define regRLC_FWL_FIRST_VIOL_ADDR                                                                      0x5f26
+#define regRLC_FWL_FIRST_VIOL_ADDR_BASE_IDX                                                             1
+
+
+// addressBlock: gc_gfx_imu_gfx_imudec
+// base address: 0x38000
+#define regGFX_IMU_C2PMSG_0                                                                             0x4000
+#define regGFX_IMU_C2PMSG_0_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_1                                                                             0x4001
+#define regGFX_IMU_C2PMSG_1_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_2                                                                             0x4002
+#define regGFX_IMU_C2PMSG_2_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_3                                                                             0x4003
+#define regGFX_IMU_C2PMSG_3_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_4                                                                             0x4004
+#define regGFX_IMU_C2PMSG_4_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_5                                                                             0x4005
+#define regGFX_IMU_C2PMSG_5_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_6                                                                             0x4006
+#define regGFX_IMU_C2PMSG_6_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_7                                                                             0x4007
+#define regGFX_IMU_C2PMSG_7_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_8                                                                             0x4008
+#define regGFX_IMU_C2PMSG_8_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_9                                                                             0x4009
+#define regGFX_IMU_C2PMSG_9_BASE_IDX                                                                    1
+#define regGFX_IMU_C2PMSG_10                                                                            0x400a
+#define regGFX_IMU_C2PMSG_10_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_11                                                                            0x400b
+#define regGFX_IMU_C2PMSG_11_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_12                                                                            0x400c
+#define regGFX_IMU_C2PMSG_12_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_13                                                                            0x400d
+#define regGFX_IMU_C2PMSG_13_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_14                                                                            0x400e
+#define regGFX_IMU_C2PMSG_14_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_15                                                                            0x400f
+#define regGFX_IMU_C2PMSG_15_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_16                                                                            0x4010
+#define regGFX_IMU_C2PMSG_16_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_17                                                                            0x4011
+#define regGFX_IMU_C2PMSG_17_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_18                                                                            0x4012
+#define regGFX_IMU_C2PMSG_18_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_19                                                                            0x4013
+#define regGFX_IMU_C2PMSG_19_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_20                                                                            0x4014
+#define regGFX_IMU_C2PMSG_20_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_21                                                                            0x4015
+#define regGFX_IMU_C2PMSG_21_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_22                                                                            0x4016
+#define regGFX_IMU_C2PMSG_22_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_23                                                                            0x4017
+#define regGFX_IMU_C2PMSG_23_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_24                                                                            0x4018
+#define regGFX_IMU_C2PMSG_24_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_25                                                                            0x4019
+#define regGFX_IMU_C2PMSG_25_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_26                                                                            0x401a
+#define regGFX_IMU_C2PMSG_26_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_27                                                                            0x401b
+#define regGFX_IMU_C2PMSG_27_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_28                                                                            0x401c
+#define regGFX_IMU_C2PMSG_28_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_29                                                                            0x401d
+#define regGFX_IMU_C2PMSG_29_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_30                                                                            0x401e
+#define regGFX_IMU_C2PMSG_30_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_31                                                                            0x401f
+#define regGFX_IMU_C2PMSG_31_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_32                                                                            0x4020
+#define regGFX_IMU_C2PMSG_32_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_33                                                                            0x4021
+#define regGFX_IMU_C2PMSG_33_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_34                                                                            0x4022
+#define regGFX_IMU_C2PMSG_34_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_35                                                                            0x4023
+#define regGFX_IMU_C2PMSG_35_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_36                                                                            0x4024
+#define regGFX_IMU_C2PMSG_36_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_37                                                                            0x4025
+#define regGFX_IMU_C2PMSG_37_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_38                                                                            0x4026
+#define regGFX_IMU_C2PMSG_38_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_39                                                                            0x4027
+#define regGFX_IMU_C2PMSG_39_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_40                                                                            0x4028
+#define regGFX_IMU_C2PMSG_40_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_41                                                                            0x4029
+#define regGFX_IMU_C2PMSG_41_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_42                                                                            0x402a
+#define regGFX_IMU_C2PMSG_42_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_43                                                                            0x402b
+#define regGFX_IMU_C2PMSG_43_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_44                                                                            0x402c
+#define regGFX_IMU_C2PMSG_44_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_45                                                                            0x402d
+#define regGFX_IMU_C2PMSG_45_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_46                                                                            0x402e
+#define regGFX_IMU_C2PMSG_46_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_47                                                                            0x402f
+#define regGFX_IMU_C2PMSG_47_BASE_IDX                                                                   1
+#define regGFX_IMU_MSG_FLAGS                                                                            0x403f
+#define regGFX_IMU_MSG_FLAGS_BASE_IDX                                                                   1
+#define regGFX_IMU_C2PMSG_ACCESS_CTRL0                                                                  0x4040
+#define regGFX_IMU_C2PMSG_ACCESS_CTRL0_BASE_IDX                                                         1
+#define regGFX_IMU_C2PMSG_ACCESS_CTRL1                                                                  0x4041
+#define regGFX_IMU_C2PMSG_ACCESS_CTRL1_BASE_IDX                                                         1
+#define regGFX_IMU_PWRMGT_IRQ_CTRL                                                                      0x4042
+#define regGFX_IMU_PWRMGT_IRQ_CTRL_BASE_IDX                                                             1
+#define regGFX_IMU_MP1_MUTEX                                                                            0x4043
+#define regGFX_IMU_MP1_MUTEX_BASE_IDX                                                                   1
+#define regGFX_IMU_RLC_DATA_4                                                                           0x4046
+#define regGFX_IMU_RLC_DATA_4_BASE_IDX                                                                  1
+#define regGFX_IMU_RLC_DATA_3                                                                           0x4047
+#define regGFX_IMU_RLC_DATA_3_BASE_IDX                                                                  1
+#define regGFX_IMU_RLC_DATA_2                                                                           0x4048
+#define regGFX_IMU_RLC_DATA_2_BASE_IDX                                                                  1
+#define regGFX_IMU_RLC_DATA_1                                                                           0x4049
+#define regGFX_IMU_RLC_DATA_1_BASE_IDX                                                                  1
+#define regGFX_IMU_RLC_DATA_0                                                                           0x404a
+#define regGFX_IMU_RLC_DATA_0_BASE_IDX                                                                  1
+#define regGFX_IMU_RLC_CMD                                                                              0x404b
+#define regGFX_IMU_RLC_CMD_BASE_IDX                                                                     1
+#define regGFX_IMU_RLC_MUTEX                                                                            0x404c
+#define regGFX_IMU_RLC_MUTEX_BASE_IDX                                                                   1
+#define regGFX_IMU_RLC_MSG_STATUS                                                                       0x404f
+#define regGFX_IMU_RLC_MSG_STATUS_BASE_IDX                                                              1
+#define regRLC_GFX_IMU_DATA_0                                                                           0x4052
+#define regRLC_GFX_IMU_DATA_0_BASE_IDX                                                                  1
+#define regRLC_GFX_IMU_CMD                                                                              0x4053
+#define regRLC_GFX_IMU_CMD_BASE_IDX                                                                     1
+#define regGFX_IMU_RLC_STATUS                                                                           0x4054
+#define regGFX_IMU_RLC_STATUS_BASE_IDX                                                                  1
+#define regGFX_IMU_STATUS                                                                               0x4055
+#define regGFX_IMU_STATUS_BASE_IDX                                                                      1
+#define regGFX_IMU_SOC_DATA                                                                             0x4059
+#define regGFX_IMU_SOC_DATA_BASE_IDX                                                                    1
+#define regGFX_IMU_SOC_ADDR                                                                             0x405a
+#define regGFX_IMU_SOC_ADDR_BASE_IDX                                                                    1
+#define regGFX_IMU_SOC_REQ                                                                              0x405b
+#define regGFX_IMU_SOC_REQ_BASE_IDX                                                                     1
+#define regGFX_IMU_VF_CTRL                                                                              0x405c
+#define regGFX_IMU_VF_CTRL_BASE_IDX                                                                     1
+#define regGFX_IMU_TELEMETRY                                                                            0x4060
+#define regGFX_IMU_TELEMETRY_BASE_IDX                                                                   1
+#define regGFX_IMU_TELEMETRY_DATA                                                                       0x4061
+#define regGFX_IMU_TELEMETRY_DATA_BASE_IDX                                                              1
+#define regGFX_IMU_TELEMETRY_TEMPERATURE                                                                0x4062
+#define regGFX_IMU_TELEMETRY_TEMPERATURE_BASE_IDX                                                       1
+#define regGFX_IMU_SCRATCH_0                                                                            0x4068
+#define regGFX_IMU_SCRATCH_0_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_1                                                                            0x4069
+#define regGFX_IMU_SCRATCH_1_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_2                                                                            0x406a
+#define regGFX_IMU_SCRATCH_2_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_3                                                                            0x406b
+#define regGFX_IMU_SCRATCH_3_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_4                                                                            0x406c
+#define regGFX_IMU_SCRATCH_4_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_5                                                                            0x406d
+#define regGFX_IMU_SCRATCH_5_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_6                                                                            0x406e
+#define regGFX_IMU_SCRATCH_6_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_7                                                                            0x406f
+#define regGFX_IMU_SCRATCH_7_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_8                                                                            0x4070
+#define regGFX_IMU_SCRATCH_8_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_9                                                                            0x4071
+#define regGFX_IMU_SCRATCH_9_BASE_IDX                                                                   1
+#define regGFX_IMU_SCRATCH_10                                                                           0x4072
+#define regGFX_IMU_SCRATCH_10_BASE_IDX                                                                  1
+#define regGFX_IMU_SCRATCH_11                                                                           0x4073
+#define regGFX_IMU_SCRATCH_11_BASE_IDX                                                                  1
+#define regGFX_IMU_SCRATCH_12                                                                           0x4074
+#define regGFX_IMU_SCRATCH_12_BASE_IDX                                                                  1
+#define regGFX_IMU_SCRATCH_13                                                                           0x4075
+#define regGFX_IMU_SCRATCH_13_BASE_IDX                                                                  1
+#define regGFX_IMU_SCRATCH_14                                                                           0x4076
+#define regGFX_IMU_SCRATCH_14_BASE_IDX                                                                  1
+#define regGFX_IMU_SCRATCH_15                                                                           0x4077
+#define regGFX_IMU_SCRATCH_15_BASE_IDX                                                                  1
+#define regGFX_IMU_FW_GTS_LO                                                                            0x4078
+#define regGFX_IMU_FW_GTS_LO_BASE_IDX                                                                   1
+#define regGFX_IMU_FW_GTS_HI                                                                            0x4079
+#define regGFX_IMU_FW_GTS_HI_BASE_IDX                                                                   1
+#define regGFX_IMU_GTS_OFFSET_LO                                                                        0x407a
+#define regGFX_IMU_GTS_OFFSET_LO_BASE_IDX                                                               1
+#define regGFX_IMU_GTS_OFFSET_HI                                                                        0x407b
+#define regGFX_IMU_GTS_OFFSET_HI_BASE_IDX                                                               1
+#define regGFX_IMU_RLC_GTS_OFFSET_LO                                                                    0x407c
+#define regGFX_IMU_RLC_GTS_OFFSET_LO_BASE_IDX                                                           1
+#define regGFX_IMU_RLC_GTS_OFFSET_HI                                                                    0x407d
+#define regGFX_IMU_RLC_GTS_OFFSET_HI_BASE_IDX                                                           1
+#define regGFX_IMU_CORE_INT_STATUS                                                                      0x407f
+#define regGFX_IMU_CORE_INT_STATUS_BASE_IDX                                                             1
+#define regGFX_IMU_PIC_INT_MASK                                                                         0x4080
+#define regGFX_IMU_PIC_INT_MASK_BASE_IDX                                                                1
+#define regGFX_IMU_PIC_INT_LVL                                                                          0x4081
+#define regGFX_IMU_PIC_INT_LVL_BASE_IDX                                                                 1
+#define regGFX_IMU_PIC_INT_EDGE                                                                         0x4082
+#define regGFX_IMU_PIC_INT_EDGE_BASE_IDX                                                                1
+#define regGFX_IMU_PIC_INT_PRI_0                                                                        0x4083
+#define regGFX_IMU_PIC_INT_PRI_0_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_1                                                                        0x4084
+#define regGFX_IMU_PIC_INT_PRI_1_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_2                                                                        0x4085
+#define regGFX_IMU_PIC_INT_PRI_2_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_3                                                                        0x4086
+#define regGFX_IMU_PIC_INT_PRI_3_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_4                                                                        0x4087
+#define regGFX_IMU_PIC_INT_PRI_4_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_5                                                                        0x4088
+#define regGFX_IMU_PIC_INT_PRI_5_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_6                                                                        0x4089
+#define regGFX_IMU_PIC_INT_PRI_6_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_PRI_7                                                                        0x408a
+#define regGFX_IMU_PIC_INT_PRI_7_BASE_IDX                                                               1
+#define regGFX_IMU_PIC_INT_STATUS                                                                       0x408b
+#define regGFX_IMU_PIC_INT_STATUS_BASE_IDX                                                              1
+#define regGFX_IMU_PIC_INTR                                                                             0x408c
+#define regGFX_IMU_PIC_INTR_BASE_IDX                                                                    1
+#define regGFX_IMU_PIC_INTR_ID                                                                          0x408d
+#define regGFX_IMU_PIC_INTR_ID_BASE_IDX                                                                 1
+#define regGFX_IMU_IH_CTRL_1                                                                            0x4090
+#define regGFX_IMU_IH_CTRL_1_BASE_IDX                                                                   1
+#define regGFX_IMU_IH_CTRL_2                                                                            0x4091
+#define regGFX_IMU_IH_CTRL_2_BASE_IDX                                                                   1
+#define regGFX_IMU_IH_CTRL_3                                                                            0x4092
+#define regGFX_IMU_IH_CTRL_3_BASE_IDX                                                                   1
+#define regGFX_IMU_IH_STATUS                                                                            0x4093
+#define regGFX_IMU_IH_STATUS_BASE_IDX                                                                   1
+#define regGFX_IMU_FUSESTRAP                                                                            0x4094
+#define regGFX_IMU_FUSESTRAP_BASE_IDX                                                                   1
+#define regGFX_IMU_SMUIO_VIDCHG_CTRL                                                                    0x4098
+#define regGFX_IMU_SMUIO_VIDCHG_CTRL_BASE_IDX                                                           1
+#define regGFX_IMU_GFXCLK_BYPASS_CTRL                                                                   0x409c
+#define regGFX_IMU_GFXCLK_BYPASS_CTRL_BASE_IDX                                                          1
+#define regGFX_IMU_CLK_CTRL                                                                             0x409d
+#define regGFX_IMU_CLK_CTRL_BASE_IDX                                                                    1
+#define regGFX_IMU_DOORBELL_CONTROL                                                                     0x409e
+#define regGFX_IMU_DOORBELL_CONTROL_BASE_IDX                                                            1
+#define regGFX_IMU_RLC_CG_CTRL                                                                          0x40a0
+#define regGFX_IMU_RLC_CG_CTRL_BASE_IDX                                                                 1
+#define regGFX_IMU_RLC_THROTTLE_GFX                                                                     0x40a1
+#define regGFX_IMU_RLC_THROTTLE_GFX_BASE_IDX                                                            1
+#define regGFX_IMU_RLC_RESET_VECTOR                                                                     0x40a2
+#define regGFX_IMU_RLC_RESET_VECTOR_BASE_IDX                                                            1
+#define regGFX_IMU_RLC_OVERRIDE                                                                         0x40a3
+#define regGFX_IMU_RLC_OVERRIDE_BASE_IDX                                                                1
+#define regGFX_IMU_DPM_CONTROL                                                                          0x40a8
+#define regGFX_IMU_DPM_CONTROL_BASE_IDX                                                                 1
+#define regGFX_IMU_DPM_ACC                                                                              0x40a9
+#define regGFX_IMU_DPM_ACC_BASE_IDX                                                                     1
+#define regGFX_IMU_DPM_REF_COUNTER                                                                      0x40aa
+#define regGFX_IMU_DPM_REF_COUNTER_BASE_IDX                                                             1
+#define regGFX_IMU_RLC_RAM_INDEX                                                                        0x40ac
+#define regGFX_IMU_RLC_RAM_INDEX_BASE_IDX                                                               1
+#define regGFX_IMU_RLC_RAM_ADDR_HIGH                                                                    0x40ad
+#define regGFX_IMU_RLC_RAM_ADDR_HIGH_BASE_IDX                                                           1
+#define regGFX_IMU_RLC_RAM_ADDR_LOW                                                                     0x40ae
+#define regGFX_IMU_RLC_RAM_ADDR_LOW_BASE_IDX                                                            1
+#define regGFX_IMU_RLC_RAM_DATA                                                                         0x40af
+#define regGFX_IMU_RLC_RAM_DATA_BASE_IDX                                                                1
+#define regGFX_IMU_FENCE_CTRL                                                                           0x40b0
+#define regGFX_IMU_FENCE_CTRL_BASE_IDX                                                                  1
+#define regGFX_IMU_FENCE_LOG_INIT                                                                       0x40b1
+#define regGFX_IMU_FENCE_LOG_INIT_BASE_IDX                                                              1
+#define regGFX_IMU_FENCE_LOG_ADDR                                                                       0x40b2
+#define regGFX_IMU_FENCE_LOG_ADDR_BASE_IDX                                                              1
+#define regGFX_IMU_PROGRAM_CTR                                                                          0x40b5
+#define regGFX_IMU_PROGRAM_CTR_BASE_IDX                                                                 1
+#define regGFX_IMU_CORE_CTRL                                                                            0x40b6
+#define regGFX_IMU_CORE_CTRL_BASE_IDX                                                                   1
+#define regGFX_IMU_CORE_STATUS                                                                          0x40b7
+#define regGFX_IMU_CORE_STATUS_BASE_IDX                                                                 1
+#define regGFX_IMU_PWROKRAW                                                                             0x40b8
+#define regGFX_IMU_PWROKRAW_BASE_IDX                                                                    1
+#define regGFX_IMU_PWROK                                                                                0x40b9
+#define regGFX_IMU_PWROK_BASE_IDX                                                                       1
+#define regGFX_IMU_GAP_PWROK                                                                            0x40ba
+#define regGFX_IMU_GAP_PWROK_BASE_IDX                                                                   1
+#define regGFX_IMU_RESETn                                                                               0x40bb
+#define regGFX_IMU_RESETn_BASE_IDX                                                                      1
+#define regGFX_IMU_GFX_RESET_CTRL                                                                       0x40bc
+#define regGFX_IMU_GFX_RESET_CTRL_BASE_IDX                                                              1
+#define regGFX_IMU_AEB_OVERRIDE                                                                         0x40bd
+#define regGFX_IMU_AEB_OVERRIDE_BASE_IDX                                                                1
+#define regGFX_IMU_VDCI_RESET_CTRL                                                                      0x40be
+#define regGFX_IMU_VDCI_RESET_CTRL_BASE_IDX                                                             1
+#define regGFX_IMU_GFX_ISO_CTRL                                                                         0x40bf
+#define regGFX_IMU_GFX_ISO_CTRL_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER0_CTRL0                                                                         0x40c0
+#define regGFX_IMU_TIMER0_CTRL0_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER0_CTRL1                                                                         0x40c1
+#define regGFX_IMU_TIMER0_CTRL1_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER0_CMP_AUTOINC                                                                   0x40c2
+#define regGFX_IMU_TIMER0_CMP_AUTOINC_BASE_IDX                                                          1
+#define regGFX_IMU_TIMER0_CMP_INTEN                                                                     0x40c3
+#define regGFX_IMU_TIMER0_CMP_INTEN_BASE_IDX                                                            1
+#define regGFX_IMU_TIMER0_CMP0                                                                          0x40c4
+#define regGFX_IMU_TIMER0_CMP0_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER0_CMP1                                                                          0x40c5
+#define regGFX_IMU_TIMER0_CMP1_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER0_CMP3                                                                          0x40c7
+#define regGFX_IMU_TIMER0_CMP3_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER0_VALUE                                                                         0x40c8
+#define regGFX_IMU_TIMER0_VALUE_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER1_CTRL0                                                                         0x40c9
+#define regGFX_IMU_TIMER1_CTRL0_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER1_CTRL1                                                                         0x40ca
+#define regGFX_IMU_TIMER1_CTRL1_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER1_CMP_AUTOINC                                                                   0x40cb
+#define regGFX_IMU_TIMER1_CMP_AUTOINC_BASE_IDX                                                          1
+#define regGFX_IMU_TIMER1_CMP_INTEN                                                                     0x40cc
+#define regGFX_IMU_TIMER1_CMP_INTEN_BASE_IDX                                                            1
+#define regGFX_IMU_TIMER1_CMP0                                                                          0x40cd
+#define regGFX_IMU_TIMER1_CMP0_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER1_CMP1                                                                          0x40ce
+#define regGFX_IMU_TIMER1_CMP1_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER1_CMP3                                                                          0x40d0
+#define regGFX_IMU_TIMER1_CMP3_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER1_VALUE                                                                         0x40d1
+#define regGFX_IMU_TIMER1_VALUE_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER2_CTRL0                                                                         0x40d2
+#define regGFX_IMU_TIMER2_CTRL0_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER2_CTRL1                                                                         0x40d3
+#define regGFX_IMU_TIMER2_CTRL1_BASE_IDX                                                                1
+#define regGFX_IMU_TIMER2_CMP_AUTOINC                                                                   0x40d4
+#define regGFX_IMU_TIMER2_CMP_AUTOINC_BASE_IDX                                                          1
+#define regGFX_IMU_TIMER2_CMP_INTEN                                                                     0x40d5
+#define regGFX_IMU_TIMER2_CMP_INTEN_BASE_IDX                                                            1
+#define regGFX_IMU_TIMER2_CMP0                                                                          0x40d6
+#define regGFX_IMU_TIMER2_CMP0_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER2_CMP1                                                                          0x40d7
+#define regGFX_IMU_TIMER2_CMP1_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER2_CMP3                                                                          0x40d9
+#define regGFX_IMU_TIMER2_CMP3_BASE_IDX                                                                 1
+#define regGFX_IMU_TIMER2_VALUE                                                                         0x40da
+#define regGFX_IMU_TIMER2_VALUE_BASE_IDX                                                                1
+#define regGFX_IMU_FUSE_CTRL                                                                            0x40e0
+#define regGFX_IMU_FUSE_CTRL_BASE_IDX                                                                   1
+#define regGFX_IMU_D_RAM_ADDR                                                                           0x40fc
+#define regGFX_IMU_D_RAM_ADDR_BASE_IDX                                                                  1
+#define regGFX_IMU_D_RAM_DATA                                                                           0x40fd
+#define regGFX_IMU_D_RAM_DATA_BASE_IDX                                                                  1
+#define regGFX_IMU_GFX_IH_GASKET_CTRL                                                                   0x40ff
+#define regGFX_IMU_GFX_IH_GASKET_CTRL_BASE_IDX                                                          1
+
+
+// addressBlock: gc_gfx_imu_gfx_imu_pspdec
+// base address: 0x3fe00
+#define regGFX_IMU_RLC_BOOTLOADER_ADDR_HI                                                               0x5f81
+#define regGFX_IMU_RLC_BOOTLOADER_ADDR_HI_BASE_IDX                                                      1
+#define regGFX_IMU_RLC_BOOTLOADER_ADDR_LO                                                               0x5f82
+#define regGFX_IMU_RLC_BOOTLOADER_ADDR_LO_BASE_IDX                                                      1
+#define regGFX_IMU_RLC_BOOTLOADER_SIZE                                                                  0x5f83
+#define regGFX_IMU_RLC_BOOTLOADER_SIZE_BASE_IDX                                                         1
+#define regGFX_IMU_I_RAM_ADDR                                                                           0x5f90
+#define regGFX_IMU_I_RAM_ADDR_BASE_IDX                                                                  1
+#define regGFX_IMU_I_RAM_DATA                                                                           0x5f91
+#define regGFX_IMU_I_RAM_DATA_BASE_IDX                                                                  1
+
+
+// addressBlock: gccacind
+// base address: 0x0
+#define ixGC_CAC_ID                                                                                    0x0000
+#define ixGC_CAC_CNTL                                                                                  0x0001
+#define ixGC_CAC_ACC_CP0                                                                               0x0010
+#define ixGC_CAC_ACC_CP1                                                                               0x0011
+#define ixGC_CAC_ACC_CP2                                                                               0x0012
+#define ixGC_CAC_ACC_EA0                                                                               0x0013
+#define ixGC_CAC_ACC_EA1                                                                               0x0014
+#define ixGC_CAC_ACC_EA2                                                                               0x0015
+#define ixGC_CAC_ACC_EA3                                                                               0x0016
+#define ixGC_CAC_ACC_EA4                                                                               0x0017
+#define ixGC_CAC_ACC_EA5                                                                               0x0018
+#define ixGC_CAC_ACC_UTCL2_ROUTER0                                                                     0x0019
+#define ixGC_CAC_ACC_UTCL2_ROUTER1                                                                     0x001a
+#define ixGC_CAC_ACC_UTCL2_ROUTER2                                                                     0x001b
+#define ixGC_CAC_ACC_UTCL2_ROUTER3                                                                     0x001c
+#define ixGC_CAC_ACC_UTCL2_ROUTER4                                                                     0x001d
+#define ixGC_CAC_ACC_UTCL2_ROUTER5                                                                     0x001e
+#define ixGC_CAC_ACC_UTCL2_ROUTER6                                                                     0x001f
+#define ixGC_CAC_ACC_UTCL2_ROUTER7                                                                     0x0020
+#define ixGC_CAC_ACC_UTCL2_ROUTER8                                                                     0x0021
+#define ixGC_CAC_ACC_UTCL2_ROUTER9                                                                     0x0022
+#define ixGC_CAC_ACC_UTCL2_VML20                                                                       0x0023
+#define ixGC_CAC_ACC_UTCL2_VML21                                                                       0x0024
+#define ixGC_CAC_ACC_UTCL2_VML22                                                                       0x0025
+#define ixGC_CAC_ACC_UTCL2_VML23                                                                       0x0026
+#define ixGC_CAC_ACC_UTCL2_VML24                                                                       0x0027
+#define ixGC_CAC_ACC_UTCL2_WALKER0                                                                     0x0028
+#define ixGC_CAC_ACC_UTCL2_WALKER1                                                                     0x0029
+#define ixGC_CAC_ACC_UTCL2_WALKER2                                                                     0x002a
+#define ixGC_CAC_ACC_UTCL2_WALKER3                                                                     0x002b
+#define ixGC_CAC_ACC_UTCL2_WALKER4                                                                     0x002c
+#define ixGC_CAC_ACC_GDS0                                                                              0x002d
+#define ixGC_CAC_ACC_GDS1                                                                              0x002e
+#define ixGC_CAC_ACC_GDS2                                                                              0x002f
+#define ixGC_CAC_ACC_GDS3                                                                              0x0030
+#define ixGC_CAC_ACC_GDS4                                                                              0x0031
+#define ixGC_CAC_ACC_GE0                                                                               0x0032
+#define ixGC_CAC_ACC_GE1                                                                               0x0033
+#define ixGC_CAC_ACC_GE2                                                                               0x0034
+#define ixGC_CAC_ACC_GE3                                                                               0x0035
+#define ixGC_CAC_ACC_GE4                                                                               0x0036
+#define ixGC_CAC_ACC_GE5                                                                               0x0037
+#define ixGC_CAC_ACC_GE6                                                                               0x0038
+#define ixGC_CAC_ACC_GE7                                                                               0x0039
+#define ixGC_CAC_ACC_GE8                                                                               0x003a
+#define ixGC_CAC_ACC_GE9                                                                               0x003b
+#define ixGC_CAC_ACC_GE10                                                                              0x003c
+#define ixGC_CAC_ACC_GE11                                                                              0x003d
+#define ixGC_CAC_ACC_GE12                                                                              0x003e
+#define ixGC_CAC_ACC_GE13                                                                              0x003f
+#define ixGC_CAC_ACC_GE14                                                                              0x0040
+#define ixGC_CAC_ACC_GE15                                                                              0x0041
+#define ixGC_CAC_ACC_GE16                                                                              0x0042
+#define ixGC_CAC_ACC_GE17                                                                              0x0043
+#define ixGC_CAC_ACC_GE18                                                                              0x0044
+#define ixGC_CAC_ACC_GE19                                                                              0x0045
+#define ixGC_CAC_ACC_GE20                                                                              0x0046
+#define ixGC_CAC_ACC_PMM0                                                                              0x0047
+#define ixGC_CAC_ACC_GL2C0                                                                             0x0048
+#define ixGC_CAC_ACC_GL2C1                                                                             0x0049
+#define ixGC_CAC_ACC_GL2C2                                                                             0x004a
+#define ixGC_CAC_ACC_GL2C3                                                                             0x004b
+#define ixGC_CAC_ACC_GL2C4                                                                             0x004c
+#define ixGC_CAC_ACC_PH0                                                                               0x004d
+#define ixGC_CAC_ACC_PH1                                                                               0x004e
+#define ixGC_CAC_ACC_PH2                                                                               0x004f
+#define ixGC_CAC_ACC_PH3                                                                               0x0050
+#define ixGC_CAC_ACC_PH4                                                                               0x0051
+#define ixGC_CAC_ACC_PH5                                                                               0x0052
+#define ixGC_CAC_ACC_PH6                                                                               0x0053
+#define ixGC_CAC_ACC_PH7                                                                               0x0054
+#define ixGC_CAC_ACC_SDMA0                                                                             0x0055
+#define ixGC_CAC_ACC_SDMA1                                                                             0x0056
+#define ixGC_CAC_ACC_SDMA2                                                                             0x0057
+#define ixGC_CAC_ACC_SDMA3                                                                             0x0058
+#define ixGC_CAC_ACC_SDMA4                                                                             0x0059
+#define ixGC_CAC_ACC_SDMA5                                                                             0x005a
+#define ixGC_CAC_ACC_SDMA6                                                                             0x005b
+#define ixGC_CAC_ACC_SDMA7                                                                             0x005c
+#define ixGC_CAC_ACC_SDMA8                                                                             0x005d
+#define ixGC_CAC_ACC_SDMA9                                                                             0x005e
+#define ixGC_CAC_ACC_SDMA10                                                                            0x005f
+#define ixGC_CAC_ACC_SDMA11                                                                            0x0060
+#define ixGC_CAC_ACC_CHC0                                                                              0x0061
+#define ixGC_CAC_ACC_CHC1                                                                              0x0062
+#define ixGC_CAC_ACC_CHC2                                                                              0x0063
+#define ixGC_CAC_ACC_GUS0                                                                              0x0064
+#define ixGC_CAC_ACC_GUS1                                                                              0x0065
+#define ixGC_CAC_ACC_GUS2                                                                              0x0066
+#define ixGC_CAC_ACC_RLC0                                                                              0x0067
+#define ixGC_CAC_ACC_UTCL2_ATCL20                                                                      0x0068
+#define ixGC_CAC_ACC_UTCL2_ATCL21                                                                      0x0069
+#define ixGC_CAC_ACC_UTCL2_ATCL22                                                                      0x006a
+#define ixGC_CAC_ACC_UTCL2_ATCL23                                                                      0x006b
+#define ixGC_CAC_ACC_UTCL2_ATCL24                                                                      0x006c
+#define ixRELEASE_TO_STALL_LUT_1_8                                                                     0x0100
+#define ixRELEASE_TO_STALL_LUT_9_16                                                                    0x0101
+#define ixRELEASE_TO_STALL_LUT_17_20                                                                   0x0102
+#define ixSTALL_TO_RELEASE_LUT_1_4                                                                     0x0103
+#define ixSTALL_TO_RELEASE_LUT_5_7                                                                     0x0104
+#define ixSTALL_TO_PWRBRK_LUT_1_4                                                                      0x0105
+#define ixSTALL_TO_PWRBRK_LUT_5_7                                                                      0x0106
+#define ixPWRBRK_STALL_TO_RELEASE_LUT_1_4                                                              0x0107
+#define ixPWRBRK_STALL_TO_RELEASE_LUT_5_7                                                              0x0108
+#define ixPWRBRK_RELEASE_TO_STALL_LUT_1_8                                                              0x0109
+#define ixPWRBRK_RELEASE_TO_STALL_LUT_9_16                                                             0x010a
+#define ixPWRBRK_RELEASE_TO_STALL_LUT_17_20                                                            0x010b
+#define ixFIXED_PATTERN_PERF_COUNTER_1                                                                 0x010c
+#define ixFIXED_PATTERN_PERF_COUNTER_2                                                                 0x010d
+#define ixFIXED_PATTERN_PERF_COUNTER_3                                                                 0x010e
+#define ixFIXED_PATTERN_PERF_COUNTER_4                                                                 0x010f
+#define ixFIXED_PATTERN_PERF_COUNTER_5                                                                 0x0110
+#define ixFIXED_PATTERN_PERF_COUNTER_6                                                                 0x0111
+#define ixFIXED_PATTERN_PERF_COUNTER_7                                                                 0x0112
+#define ixFIXED_PATTERN_PERF_COUNTER_8                                                                 0x0113
+#define ixFIXED_PATTERN_PERF_COUNTER_9                                                                 0x0114
+#define ixFIXED_PATTERN_PERF_COUNTER_10                                                                0x0115
+#define ixHW_LUT_UPDATE_STATUS                                                                         0x0116
+
+
+// addressBlock: secacind
+// base address: 0x0
+#define ixSE_CAC_ID                                                                                    0x0000
+#define ixSE_CAC_CNTL                                                                                  0x0001
+
+
+// addressBlock: grtavfsind
+// base address: 0x0
+#define ixRTAVFS_REG0                                                                                  0x0000
+#define ixRTAVFS_REG1                                                                                  0x0001
+#define ixRTAVFS_REG2                                                                                  0x0002
+#define ixRTAVFS_REG3                                                                                  0x0003
+#define ixRTAVFS_REG4                                                                                  0x0004
+#define ixRTAVFS_REG5                                                                                  0x0005
+#define ixRTAVFS_REG6                                                                                  0x0006
+#define ixRTAVFS_REG7                                                                                  0x0007
+#define ixRTAVFS_REG8                                                                                  0x0008
+#define ixRTAVFS_REG9                                                                                  0x0009
+#define ixRTAVFS_REG10                                                                                 0x000a
+#define ixRTAVFS_REG11                                                                                 0x000b
+#define ixRTAVFS_REG12                                                                                 0x000c
+#define ixRTAVFS_REG13                                                                                 0x000d
+#define ixRTAVFS_REG14                                                                                 0x000e
+#define ixRTAVFS_REG15                                                                                 0x000f
+#define ixRTAVFS_REG16                                                                                 0x0010
+#define ixRTAVFS_REG17                                                                                 0x0011
+#define ixRTAVFS_REG18                                                                                 0x0012
+#define ixRTAVFS_REG19                                                                                 0x0013
+#define ixRTAVFS_REG20                                                                                 0x0014
+#define ixRTAVFS_REG21                                                                                 0x0015
+#define ixRTAVFS_REG22                                                                                 0x0016
+#define ixRTAVFS_REG23                                                                                 0x0017
+#define ixRTAVFS_REG24                                                                                 0x0018
+#define ixRTAVFS_REG25                                                                                 0x0019
+#define ixRTAVFS_REG26                                                                                 0x001a
+#define ixRTAVFS_REG27                                                                                 0x001b
+#define ixRTAVFS_REG28                                                                                 0x001c
+#define ixRTAVFS_REG29                                                                                 0x001d
+#define ixRTAVFS_REG30                                                                                 0x001e
+#define ixRTAVFS_REG31                                                                                 0x001f
+#define ixRTAVFS_REG32                                                                                 0x0020
+#define ixRTAVFS_REG33                                                                                 0x0021
+#define ixRTAVFS_REG34                                                                                 0x0022
+#define ixRTAVFS_REG35                                                                                 0x0023
+#define ixRTAVFS_REG36                                                                                 0x0024
+#define ixRTAVFS_REG37                                                                                 0x0025
+#define ixRTAVFS_REG38                                                                                 0x0026
+#define ixRTAVFS_REG39                                                                                 0x0027
+#define ixRTAVFS_REG40                                                                                 0x0028
+#define ixRTAVFS_REG41                                                                                 0x0029
+#define ixRTAVFS_REG42                                                                                 0x002a
+#define ixRTAVFS_REG43                                                                                 0x002b
+#define ixRTAVFS_REG44                                                                                 0x002c
+#define ixRTAVFS_REG45                                                                                 0x002d
+#define ixRTAVFS_REG46                                                                                 0x002e
+#define ixRTAVFS_REG47                                                                                 0x002f
+#define ixRTAVFS_REG48                                                                                 0x0030
+#define ixRTAVFS_REG49                                                                                 0x0031
+#define ixRTAVFS_REG50                                                                                 0x0032
+#define ixRTAVFS_REG51                                                                                 0x0033
+#define ixRTAVFS_REG52                                                                                 0x0034
+#define ixRTAVFS_REG53                                                                                 0x0035
+#define ixRTAVFS_REG54                                                                                 0x0036
+#define ixRTAVFS_REG55                                                                                 0x0037
+#define ixRTAVFS_REG56                                                                                 0x0038
+#define ixRTAVFS_REG57                                                                                 0x0039
+#define ixRTAVFS_REG58                                                                                 0x003a
+#define ixRTAVFS_REG59                                                                                 0x003b
+#define ixRTAVFS_REG60                                                                                 0x003c
+#define ixRTAVFS_REG61                                                                                 0x003d
+#define ixRTAVFS_REG62                                                                                 0x003e
+#define ixRTAVFS_REG63                                                                                 0x003f
+#define ixRTAVFS_REG64                                                                                 0x0040
+#define ixRTAVFS_REG65                                                                                 0x0041
+#define ixRTAVFS_REG66                                                                                 0x0042
+#define ixRTAVFS_REG67                                                                                 0x0043
+#define ixRTAVFS_REG68                                                                                 0x0044
+#define ixRTAVFS_REG69                                                                                 0x0045
+#define ixRTAVFS_REG70                                                                                 0x0046
+#define ixRTAVFS_REG71                                                                                 0x0047
+#define ixRTAVFS_REG72                                                                                 0x0048
+#define ixRTAVFS_REG73                                                                                 0x0049
+#define ixRTAVFS_REG74                                                                                 0x004a
+#define ixRTAVFS_REG75                                                                                 0x004b
+#define ixRTAVFS_REG76                                                                                 0x004c
+#define ixRTAVFS_REG77                                                                                 0x004d
+#define ixRTAVFS_REG78                                                                                 0x004e
+#define ixRTAVFS_REG79                                                                                 0x004f
+#define ixRTAVFS_REG80                                                                                 0x0050
+#define ixRTAVFS_REG81                                                                                 0x0051
+#define ixRTAVFS_REG82                                                                                 0x0052
+#define ixRTAVFS_REG83                                                                                 0x0053
+#define ixRTAVFS_REG84                                                                                 0x0054
+#define ixRTAVFS_REG85                                                                                 0x0055
+#define ixRTAVFS_REG86                                                                                 0x0056
+#define ixRTAVFS_REG87                                                                                 0x0057
+#define ixRTAVFS_REG88                                                                                 0x0058
+#define ixRTAVFS_REG89                                                                                 0x0059
+#define ixRTAVFS_REG90                                                                                 0x005a
+#define ixRTAVFS_REG91                                                                                 0x005b
+#define ixRTAVFS_REG92                                                                                 0x005c
+#define ixRTAVFS_REG93                                                                                 0x005d
+#define ixRTAVFS_REG94                                                                                 0x005e
+#define ixRTAVFS_REG95                                                                                 0x005f
+#define ixRTAVFS_REG96                                                                                 0x0060
+#define ixRTAVFS_REG97                                                                                 0x0061
+#define ixRTAVFS_REG98                                                                                 0x0062
+#define ixRTAVFS_REG99                                                                                 0x0063
+#define ixRTAVFS_REG100                                                                                0x0064
+#define ixRTAVFS_REG101                                                                                0x0065
+#define ixRTAVFS_REG102                                                                                0x0066
+#define ixRTAVFS_REG103                                                                                0x0067
+#define ixRTAVFS_REG104                                                                                0x0068
+#define ixRTAVFS_REG105                                                                                0x0069
+#define ixRTAVFS_REG106                                                                                0x006a
+#define ixRTAVFS_REG107                                                                                0x006b
+#define ixRTAVFS_REG108                                                                                0x006c
+#define ixRTAVFS_REG109                                                                                0x006d
+#define ixRTAVFS_REG110                                                                                0x006e
+#define ixRTAVFS_REG111                                                                                0x006f
+#define ixRTAVFS_REG112                                                                                0x0070
+#define ixRTAVFS_REG113                                                                                0x0071
+#define ixRTAVFS_REG114                                                                                0x0072
+#define ixRTAVFS_REG115                                                                                0x0073
+#define ixRTAVFS_REG116                                                                                0x0074
+#define ixRTAVFS_REG117                                                                                0x0075
+#define ixRTAVFS_REG118                                                                                0x0076
+#define ixRTAVFS_REG119                                                                                0x0077
+#define ixRTAVFS_REG120                                                                                0x0078
+#define ixRTAVFS_REG121                                                                                0x0079
+#define ixRTAVFS_REG122                                                                                0x007a
+#define ixRTAVFS_REG123                                                                                0x007b
+#define ixRTAVFS_REG124                                                                                0x007c
+#define ixRTAVFS_REG125                                                                                0x007d
+#define ixRTAVFS_REG126                                                                                0x007e
+#define ixRTAVFS_REG127                                                                                0x007f
+#define ixRTAVFS_REG128                                                                                0x0080
+#define ixRTAVFS_REG129                                                                                0x0081
+#define ixRTAVFS_REG130                                                                                0x0082
+#define ixRTAVFS_REG131                                                                                0x0083
+#define ixRTAVFS_REG132                                                                                0x0084
+#define ixRTAVFS_REG133                                                                                0x0085
+#define ixRTAVFS_REG134                                                                                0x0086
+#define ixRTAVFS_REG135                                                                                0x0087
+#define ixRTAVFS_REG136                                                                                0x0088
+#define ixRTAVFS_REG137                                                                                0x0089
+#define ixRTAVFS_REG138                                                                                0x008a
+#define ixRTAVFS_REG139                                                                                0x008b
+#define ixRTAVFS_REG140                                                                                0x008c
+#define ixRTAVFS_REG141                                                                                0x008d
+#define ixRTAVFS_REG142                                                                                0x008e
+#define ixRTAVFS_REG143                                                                                0x008f
+#define ixRTAVFS_REG144                                                                                0x0090
+#define ixRTAVFS_REG145                                                                                0x0091
+#define ixRTAVFS_REG146                                                                                0x0092
+#define ixRTAVFS_REG147                                                                                0x0093
+#define ixRTAVFS_REG148                                                                                0x0094
+#define ixRTAVFS_REG149                                                                                0x0095
+#define ixRTAVFS_REG150                                                                                0x0096
+#define ixRTAVFS_REG151                                                                                0x0097
+#define ixRTAVFS_REG152                                                                                0x0098
+#define ixRTAVFS_REG153                                                                                0x0099
+#define ixRTAVFS_REG154                                                                                0x009a
+#define ixRTAVFS_REG155                                                                                0x009b
+#define ixRTAVFS_REG156                                                                                0x009c
+#define ixRTAVFS_REG157                                                                                0x009d
+#define ixRTAVFS_REG158                                                                                0x009e
+#define ixRTAVFS_REG159                                                                                0x009f
+#define ixRTAVFS_REG160                                                                                0x00a0
+#define ixRTAVFS_REG161                                                                                0x00a1
+#define ixRTAVFS_REG162                                                                                0x00a2
+#define ixRTAVFS_REG163                                                                                0x00a3
+#define ixRTAVFS_REG164                                                                                0x00a4
+#define ixRTAVFS_REG165                                                                                0x00a5
+#define ixRTAVFS_REG166                                                                                0x00a6
+#define ixRTAVFS_REG167                                                                                0x00a7
+#define ixRTAVFS_REG168                                                                                0x00a8
+#define ixRTAVFS_REG169                                                                                0x00a9
+#define ixRTAVFS_REG170                                                                                0x00aa
+#define ixRTAVFS_REG171                                                                                0x00ab
+#define ixRTAVFS_REG172                                                                                0x00ac
+#define ixRTAVFS_REG173                                                                                0x00ad
+#define ixRTAVFS_REG174                                                                                0x00ae
+#define ixRTAVFS_REG175                                                                                0x00af
+#define ixRTAVFS_REG176                                                                                0x00b0
+#define ixRTAVFS_REG177                                                                                0x00b1
+#define ixRTAVFS_REG178                                                                                0x00b2
+#define ixRTAVFS_REG179                                                                                0x00b3
+#define ixRTAVFS_REG180                                                                                0x00b4
+#define ixRTAVFS_REG181                                                                                0x00b5
+#define ixRTAVFS_REG182                                                                                0x00b6
+#define ixRTAVFS_REG183                                                                                0x00b7
+#define ixRTAVFS_REG184                                                                                0x00b8
+#define ixRTAVFS_REG185                                                                                0x00b9
+#define ixRTAVFS_REG186                                                                                0x00ba
+#define ixRTAVFS_REG187                                                                                0x00bb
+#define ixRTAVFS_REG189                                                                                0x00bd
+#define ixRTAVFS_REG190                                                                                0x00be
+#define ixRTAVFS_REG191                                                                                0x00bf
+#define ixRTAVFS_REG192                                                                                0x00c0
+#define ixRTAVFS_REG193                                                                                0x00c1
+#define ixRTAVFS_REG194                                                                                0x00c2
+
+
+// addressBlock: sqind
+// base address: 0x0
+#define ixSQ_DEBUG_STS_LOCAL                                                                           0x0008
+#define ixSQ_DEBUG_CTRL_LOCAL                                                                          0x0009
+#define ixSQ_WAVE_ACTIVE                                                                               0x000a
+#define ixSQ_WAVE_VALID_AND_IDLE                                                                       0x000b
+#define ixSQ_WAVE_MODE                                                                                 0x0101
+#define ixSQ_WAVE_STATUS                                                                               0x0102
+#define ixSQ_WAVE_TRAPSTS                                                                              0x0103
+#define ixSQ_WAVE_GPR_ALLOC                                                                            0x0105
+#define ixSQ_WAVE_LDS_ALLOC                                                                            0x0106
+#define ixSQ_WAVE_IB_STS                                                                               0x0107
+#define ixSQ_WAVE_PC_LO                                                                                0x0108
+#define ixSQ_WAVE_PC_HI                                                                                0x0109
+#define ixSQ_WAVE_IB_DBG1                                                                              0x010d
+#define ixSQ_WAVE_FLUSH_IB                                                                             0x010e
+#define ixSQ_WAVE_FLAT_SCRATCH_LO                                                                      0x0114
+#define ixSQ_WAVE_FLAT_SCRATCH_HI                                                                      0x0115
+#define ixSQ_WAVE_HW_ID1                                                                               0x0117
+#define ixSQ_WAVE_HW_ID2                                                                               0x0118
+#define ixSQ_WAVE_POPS_PACKER                                                                          0x0119
+#define ixSQ_WAVE_SCHED_MODE                                                                           0x011a
+#define ixSQ_WAVE_IB_STS2                                                                              0x011c
+#define ixSQ_WAVE_SHADER_CYCLES                                                                        0x011d
+#define ixSQ_WAVE_TTMP0                                                                                0x026c
+#define ixSQ_WAVE_TTMP1                                                                                0x026d
+#define ixSQ_WAVE_TTMP2                                                                                0x026e
+#define ixSQ_WAVE_TTMP3                                                                                0x026f
+#define ixSQ_WAVE_TTMP4                                                                                0x0270
+#define ixSQ_WAVE_TTMP5                                                                                0x0271
+#define ixSQ_WAVE_TTMP6                                                                                0x0272
+#define ixSQ_WAVE_TTMP7                                                                                0x0273
+#define ixSQ_WAVE_TTMP8                                                                                0x0274
+#define ixSQ_WAVE_TTMP9                                                                                0x0275
+#define ixSQ_WAVE_TTMP10                                                                               0x0276
+#define ixSQ_WAVE_TTMP11                                                                               0x0277
+#define ixSQ_WAVE_TTMP12                                                                               0x0278
+#define ixSQ_WAVE_TTMP13                                                                               0x0279
+#define ixSQ_WAVE_TTMP14                                                                               0x027a
+#define ixSQ_WAVE_TTMP15                                                                               0x027b
+#define ixSQ_WAVE_M0                                                                                   0x027d
+#define ixSQ_WAVE_EXEC_LO                                                                              0x027e
+#define ixSQ_WAVE_EXEC_HI                                                                              0x027f
+
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_3_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_3_sh_mask.h
new file mode 100644
index 000000000000..ae3ef8a9e702
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_11_0_3_sh_mask.h
@@ -0,0 +1,44640 @@
+/*
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _gc_11_0_3_SH_MASK_HEADER
+#define _gc_11_0_3_SH_MASK_HEADER
+
+
+// addressBlock: gc_sdma0_sdma0dec
+//SDMA0_DEC_START
+#define SDMA0_DEC_START__START__SHIFT                                                                         0x0
+#define SDMA0_DEC_START__START_MASK                                                                           0xFFFFFFFFL
+//SDMA0_F32_MISC_CNTL
+#define SDMA0_F32_MISC_CNTL__F32_WAKEUP__SHIFT                                                                0x0
+#define SDMA0_F32_MISC_CNTL__F32_WAKEUP_MASK                                                                  0x00000001L
+//SDMA0_GLOBAL_TIMESTAMP_LO
+#define SDMA0_GLOBAL_TIMESTAMP_LO__DATA__SHIFT                                                                0x0
+#define SDMA0_GLOBAL_TIMESTAMP_LO__DATA_MASK                                                                  0xFFFFFFFFL
+//SDMA0_GLOBAL_TIMESTAMP_HI
+#define SDMA0_GLOBAL_TIMESTAMP_HI__DATA__SHIFT                                                                0x0
+#define SDMA0_GLOBAL_TIMESTAMP_HI__DATA_MASK                                                                  0xFFFFFFFFL
+//SDMA0_POWER_CNTL
+#define SDMA0_POWER_CNTL__LS_ENABLE__SHIFT                                                                    0x8
+#define SDMA0_POWER_CNTL__LS_ENABLE_MASK                                                                      0x00000100L
+//SDMA0_CNTL
+#define SDMA0_CNTL__TRAP_ENABLE__SHIFT                                                                        0x0
+#define SDMA0_CNTL__SEM_WAIT_INT_ENABLE__SHIFT                                                                0x2
+#define SDMA0_CNTL__DATA_SWAP_ENABLE__SHIFT                                                                   0x3
+#define SDMA0_CNTL__FENCE_SWAP_ENABLE__SHIFT                                                                  0x4
+#define SDMA0_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                              0x5
+#define SDMA0_CNTL__PIO_DONE_ACK_ENABLE__SHIFT                                                                0x6
+#define SDMA0_CNTL__TMZ_MIDCMD_PREEMPT_ENABLE__SHIFT                                                          0x8
+#define SDMA0_CNTL__MIDCMD_EXPIRE_ENABLE__SHIFT                                                               0x9
+#define SDMA0_CNTL__CP_MES_INT_ENABLE__SHIFT                                                                  0xa
+#define SDMA0_CNTL__PAGE_RETRY_TIMEOUT_INT_ENABLE__SHIFT                                                      0xb
+#define SDMA0_CNTL__PAGE_NULL_INT_ENABLE__SHIFT                                                               0xc
+#define SDMA0_CNTL__PAGE_FAULT_INT_ENABLE__SHIFT                                                              0xd
+#define SDMA0_CNTL__CH_PERFCNT_ENABLE__SHIFT                                                                  0x10
+#define SDMA0_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT                                                          0x11
+#define SDMA0_CNTL__DRM_RESTORE_ENABLE__SHIFT                                                                 0x13
+#define SDMA0_CNTL__CTXEMPTY_INT_ENABLE__SHIFT                                                                0x1c
+#define SDMA0_CNTL__FROZEN_INT_ENABLE__SHIFT                                                                  0x1d
+#define SDMA0_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT                                                              0x1e
+#define SDMA0_CNTL__RB_PREEMPT_INT_ENABLE__SHIFT                                                              0x1f
+#define SDMA0_CNTL__TRAP_ENABLE_MASK                                                                          0x00000001L
+#define SDMA0_CNTL__SEM_WAIT_INT_ENABLE_MASK                                                                  0x00000004L
+#define SDMA0_CNTL__DATA_SWAP_ENABLE_MASK                                                                     0x00000008L
+#define SDMA0_CNTL__FENCE_SWAP_ENABLE_MASK                                                                    0x00000010L
+#define SDMA0_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                                0x00000020L
+#define SDMA0_CNTL__PIO_DONE_ACK_ENABLE_MASK                                                                  0x00000040L
+#define SDMA0_CNTL__TMZ_MIDCMD_PREEMPT_ENABLE_MASK                                                            0x00000100L
+#define SDMA0_CNTL__MIDCMD_EXPIRE_ENABLE_MASK                                                                 0x00000200L
+#define SDMA0_CNTL__CP_MES_INT_ENABLE_MASK                                                                    0x00000400L
+#define SDMA0_CNTL__PAGE_RETRY_TIMEOUT_INT_ENABLE_MASK                                                        0x00000800L
+#define SDMA0_CNTL__PAGE_NULL_INT_ENABLE_MASK                                                                 0x00001000L
+#define SDMA0_CNTL__PAGE_FAULT_INT_ENABLE_MASK                                                                0x00002000L
+#define SDMA0_CNTL__CH_PERFCNT_ENABLE_MASK                                                                    0x00010000L
+#define SDMA0_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK                                                            0x00020000L
+#define SDMA0_CNTL__DRM_RESTORE_ENABLE_MASK                                                                   0x00080000L
+#define SDMA0_CNTL__CTXEMPTY_INT_ENABLE_MASK                                                                  0x10000000L
+#define SDMA0_CNTL__FROZEN_INT_ENABLE_MASK                                                                    0x20000000L
+#define SDMA0_CNTL__IB_PREEMPT_INT_ENABLE_MASK                                                                0x40000000L
+#define SDMA0_CNTL__RB_PREEMPT_INT_ENABLE_MASK                                                                0x80000000L
+//SDMA0_CHICKEN_BITS
+#define SDMA0_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE__SHIFT                                                 0x1
+#define SDMA0_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE__SHIFT                                        0x2
+#define SDMA0_CHICKEN_BITS__BACK_COMPAT_ENABLE__SHIFT                                                         0x3
+#define SDMA0_CHICKEN_BITS__SRBM_POLL_RETRYING__SHIFT                                                         0x5
+#define SDMA0_CHICKEN_BITS__RD_BURST__SHIFT                                                                   0x6
+#define SDMA0_CHICKEN_BITS__WR_BURST__SHIFT                                                                   0x8
+#define SDMA0_CHICKEN_BITS__COMBINE_256B_WAIT_CYCLE__SHIFT                                                    0xa
+#define SDMA0_CHICKEN_BITS__WR_COMBINE_256B_ENABLE__SHIFT                                                     0xe
+#define SDMA0_CHICKEN_BITS__RD_COMBINE_256B_ENABLE__SHIFT                                                     0xf
+#define SDMA0_CHICKEN_BITS__COPY_OVERLAP_ENABLE__SHIFT                                                        0x10
+#define SDMA0_CHICKEN_BITS__RAW_CHECK_ENABLE__SHIFT                                                           0x11
+#define SDMA0_CHICKEN_BITS__T2L_256B_ENABLE__SHIFT                                                            0x12
+#define SDMA0_CHICKEN_BITS__SOFT_OVERRIDE_GCR_FGCG__SHIFT                                                     0x13
+#define SDMA0_CHICKEN_BITS__SOFT_OVERRIDE_GRBM_FGCG__SHIFT                                                    0x14
+#define SDMA0_CHICKEN_BITS__SOFT_OVERRIDE_CH_FGCG__SHIFT                                                      0x15
+#define SDMA0_CHICKEN_BITS__SOFT_OVERRIDE_UTCL2_INVREQ_FGCG__SHIFT                                            0x16
+#define SDMA0_CHICKEN_BITS__SOFT_OVERRIDE_UTCL1_FGCG__SHIFT                                                   0x17
+#define SDMA0_CHICKEN_BITS__CG_STATUS_OUTPUT__SHIFT                                                           0x18
+#define SDMA0_CHICKEN_BITS__SW_FREEZE_ENABLE__SHIFT                                                           0x19
+#define SDMA0_CHICKEN_BITS__DRAM_ECC_COPY_MODE_CNTL__SHIFT                                                    0x1a
+#define SDMA0_CHICKEN_BITS__RESERVED__SHIFT                                                                   0x1b
+#define SDMA0_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE_MASK                                                   0x00000002L
+#define SDMA0_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE_MASK                                          0x00000004L
+#define SDMA0_CHICKEN_BITS__BACK_COMPAT_ENABLE_MASK                                                           0x00000008L
+#define SDMA0_CHICKEN_BITS__SRBM_POLL_RETRYING_MASK                                                           0x00000020L
+#define SDMA0_CHICKEN_BITS__RD_BURST_MASK                                                                     0x000000C0L
+#define SDMA0_CHICKEN_BITS__WR_BURST_MASK                                                                     0x00000300L
+#define SDMA0_CHICKEN_BITS__COMBINE_256B_WAIT_CYCLE_MASK                                                      0x00003C00L
+#define SDMA0_CHICKEN_BITS__WR_COMBINE_256B_ENABLE_MASK                                                       0x00004000L
+#define SDMA0_CHICKEN_BITS__RD_COMBINE_256B_ENABLE_MASK                                                       0x00008000L
+#define SDMA0_CHICKEN_BITS__COPY_OVERLAP_ENABLE_MASK                                                          0x00010000L
+#define SDMA0_CHICKEN_BITS__RAW_CHECK_ENABLE_MASK                                                             0x00020000L
+#define SDMA0_CHICKEN_BITS__T2L_256B_ENABLE_MASK                                                              0x00040000L
+#define SDMA0_CHICKEN_BITS__SOFT_OVERRIDE_GCR_FGCG_MASK                                                       0x00080000L
+#define SDMA0_CHICKEN_BITS__SOFT_OVERRIDE_GRBM_FGCG_MASK                                                      0x00100000L
+#define SDMA0_CHICKEN_BITS__SOFT_OVERRIDE_CH_FGCG_MASK                                                        0x00200000L
+#define SDMA0_CHICKEN_BITS__SOFT_OVERRIDE_UTCL2_INVREQ_FGCG_MASK                                              0x00400000L
+#define SDMA0_CHICKEN_BITS__SOFT_OVERRIDE_UTCL1_FGCG_MASK                                                     0x00800000L
+#define SDMA0_CHICKEN_BITS__CG_STATUS_OUTPUT_MASK                                                             0x01000000L
+#define SDMA0_CHICKEN_BITS__SW_FREEZE_ENABLE_MASK                                                             0x02000000L
+#define SDMA0_CHICKEN_BITS__DRAM_ECC_COPY_MODE_CNTL_MASK                                                      0x04000000L
+#define SDMA0_CHICKEN_BITS__RESERVED_MASK                                                                     0xF8000000L
+//SDMA0_GB_ADDR_CONFIG
+#define SDMA0_GB_ADDR_CONFIG__NUM_PIPES__SHIFT                                                                0x0
+#define SDMA0_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                     0x3
+#define SDMA0_GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS__SHIFT                                                     0x6
+#define SDMA0_GB_ADDR_CONFIG__NUM_PKRS__SHIFT                                                                 0x8
+#define SDMA0_GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT                                                       0x13
+#define SDMA0_GB_ADDR_CONFIG__NUM_RB_PER_SE__SHIFT                                                            0x1a
+#define SDMA0_GB_ADDR_CONFIG__NUM_PIPES_MASK                                                                  0x00000007L
+#define SDMA0_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                       0x00000038L
+#define SDMA0_GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS_MASK                                                       0x000000C0L
+#define SDMA0_GB_ADDR_CONFIG__NUM_PKRS_MASK                                                                   0x00000700L
+#define SDMA0_GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK                                                         0x00180000L
+#define SDMA0_GB_ADDR_CONFIG__NUM_RB_PER_SE_MASK                                                              0x0C000000L
+//SDMA0_GB_ADDR_CONFIG_READ
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                           0x0
+#define SDMA0_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                0x3
+#define SDMA0_GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS__SHIFT                                                0x6
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_PKRS__SHIFT                                                            0x8
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                  0x13
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_RB_PER_SE__SHIFT                                                       0x1a
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                             0x00000007L
+#define SDMA0_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                  0x00000038L
+#define SDMA0_GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS_MASK                                                  0x000000C0L
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_PKRS_MASK                                                              0x00000700L
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                    0x00180000L
+#define SDMA0_GB_ADDR_CONFIG_READ__NUM_RB_PER_SE_MASK                                                         0x0C000000L
+//SDMA0_RB_RPTR_FETCH
+#define SDMA0_RB_RPTR_FETCH__OFFSET__SHIFT                                                                    0x2
+#define SDMA0_RB_RPTR_FETCH__OFFSET_MASK                                                                      0xFFFFFFFCL
+//SDMA0_RB_RPTR_FETCH_HI
+#define SDMA0_RB_RPTR_FETCH_HI__OFFSET__SHIFT                                                                 0x0
+#define SDMA0_RB_RPTR_FETCH_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//SDMA0_SEM_WAIT_FAIL_TIMER_CNTL
+#define SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__TIMER__SHIFT                                                          0x0
+#define SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__TIMER_MASK                                                            0xFFFFFFFFL
+//SDMA0_IB_OFFSET_FETCH
+#define SDMA0_IB_OFFSET_FETCH__OFFSET__SHIFT                                                                  0x2
+#define SDMA0_IB_OFFSET_FETCH__OFFSET_MASK                                                                    0x003FFFFCL
+//SDMA0_PROGRAM
+#define SDMA0_PROGRAM__STREAM__SHIFT                                                                          0x0
+#define SDMA0_PROGRAM__STREAM_MASK                                                                            0xFFFFFFFFL
+//SDMA0_STATUS_REG
+#define SDMA0_STATUS_REG__IDLE__SHIFT                                                                         0x0
+#define SDMA0_STATUS_REG__REG_IDLE__SHIFT                                                                     0x1
+#define SDMA0_STATUS_REG__RB_EMPTY__SHIFT                                                                     0x2
+#define SDMA0_STATUS_REG__RB_FULL__SHIFT                                                                      0x3
+#define SDMA0_STATUS_REG__RB_CMD_IDLE__SHIFT                                                                  0x4
+#define SDMA0_STATUS_REG__RB_CMD_FULL__SHIFT                                                                  0x5
+#define SDMA0_STATUS_REG__IB_CMD_IDLE__SHIFT                                                                  0x6
+#define SDMA0_STATUS_REG__IB_CMD_FULL__SHIFT                                                                  0x7
+#define SDMA0_STATUS_REG__BLOCK_IDLE__SHIFT                                                                   0x8
+#define SDMA0_STATUS_REG__INSIDE_IB__SHIFT                                                                    0x9
+#define SDMA0_STATUS_REG__EX_IDLE__SHIFT                                                                      0xa
+#define SDMA0_STATUS_REG__CGCG_FENCE__SHIFT                                                                   0xb
+#define SDMA0_STATUS_REG__PACKET_READY__SHIFT                                                                 0xc
+#define SDMA0_STATUS_REG__MC_WR_IDLE__SHIFT                                                                   0xd
+#define SDMA0_STATUS_REG__SRBM_IDLE__SHIFT                                                                    0xe
+#define SDMA0_STATUS_REG__CONTEXT_EMPTY__SHIFT                                                                0xf
+#define SDMA0_STATUS_REG__DELTA_RPTR_FULL__SHIFT                                                              0x10
+#define SDMA0_STATUS_REG__RB_MC_RREQ_IDLE__SHIFT                                                              0x11
+#define SDMA0_STATUS_REG__IB_MC_RREQ_IDLE__SHIFT                                                              0x12
+#define SDMA0_STATUS_REG__MC_RD_IDLE__SHIFT                                                                   0x13
+#define SDMA0_STATUS_REG__DELTA_RPTR_EMPTY__SHIFT                                                             0x14
+#define SDMA0_STATUS_REG__MC_RD_RET_STALL__SHIFT                                                              0x15
+#define SDMA0_STATUS_REG__MC_RD_NO_POLL_IDLE__SHIFT                                                           0x16
+#define SDMA0_STATUS_REG__DRM_IDLE__SHIFT                                                                     0x17
+#define SDMA0_STATUS_REG__DRM_MASK_FULL__SHIFT                                                                0x18
+#define SDMA0_STATUS_REG__PREV_CMD_IDLE__SHIFT                                                                0x19
+#define SDMA0_STATUS_REG__SEM_IDLE__SHIFT                                                                     0x1a
+#define SDMA0_STATUS_REG__SEM_REQ_STALL__SHIFT                                                                0x1b
+#define SDMA0_STATUS_REG__SEM_RESP_STATE__SHIFT                                                               0x1c
+#define SDMA0_STATUS_REG__INT_IDLE__SHIFT                                                                     0x1e
+#define SDMA0_STATUS_REG__INT_REQ_STALL__SHIFT                                                                0x1f
+#define SDMA0_STATUS_REG__IDLE_MASK                                                                           0x00000001L
+#define SDMA0_STATUS_REG__REG_IDLE_MASK                                                                       0x00000002L
+#define SDMA0_STATUS_REG__RB_EMPTY_MASK                                                                       0x00000004L
+#define SDMA0_STATUS_REG__RB_FULL_MASK                                                                        0x00000008L
+#define SDMA0_STATUS_REG__RB_CMD_IDLE_MASK                                                                    0x00000010L
+#define SDMA0_STATUS_REG__RB_CMD_FULL_MASK                                                                    0x00000020L
+#define SDMA0_STATUS_REG__IB_CMD_IDLE_MASK                                                                    0x00000040L
+#define SDMA0_STATUS_REG__IB_CMD_FULL_MASK                                                                    0x00000080L
+#define SDMA0_STATUS_REG__BLOCK_IDLE_MASK                                                                     0x00000100L
+#define SDMA0_STATUS_REG__INSIDE_IB_MASK                                                                      0x00000200L
+#define SDMA0_STATUS_REG__EX_IDLE_MASK                                                                        0x00000400L
+#define SDMA0_STATUS_REG__CGCG_FENCE_MASK                                                                     0x00000800L
+#define SDMA0_STATUS_REG__PACKET_READY_MASK                                                                   0x00001000L
+#define SDMA0_STATUS_REG__MC_WR_IDLE_MASK                                                                     0x00002000L
+#define SDMA0_STATUS_REG__SRBM_IDLE_MASK                                                                      0x00004000L
+#define SDMA0_STATUS_REG__CONTEXT_EMPTY_MASK                                                                  0x00008000L
+#define SDMA0_STATUS_REG__DELTA_RPTR_FULL_MASK                                                                0x00010000L
+#define SDMA0_STATUS_REG__RB_MC_RREQ_IDLE_MASK                                                                0x00020000L
+#define SDMA0_STATUS_REG__IB_MC_RREQ_IDLE_MASK                                                                0x00040000L
+#define SDMA0_STATUS_REG__MC_RD_IDLE_MASK                                                                     0x00080000L
+#define SDMA0_STATUS_REG__DELTA_RPTR_EMPTY_MASK                                                               0x00100000L
+#define SDMA0_STATUS_REG__MC_RD_RET_STALL_MASK                                                                0x00200000L
+#define SDMA0_STATUS_REG__MC_RD_NO_POLL_IDLE_MASK                                                             0x00400000L
+#define SDMA0_STATUS_REG__DRM_IDLE_MASK                                                                       0x00800000L
+#define SDMA0_STATUS_REG__DRM_MASK_FULL_MASK                                                                  0x01000000L
+#define SDMA0_STATUS_REG__PREV_CMD_IDLE_MASK                                                                  0x02000000L
+#define SDMA0_STATUS_REG__SEM_IDLE_MASK                                                                       0x04000000L
+#define SDMA0_STATUS_REG__SEM_REQ_STALL_MASK                                                                  0x08000000L
+#define SDMA0_STATUS_REG__SEM_RESP_STATE_MASK                                                                 0x30000000L
+#define SDMA0_STATUS_REG__INT_IDLE_MASK                                                                       0x40000000L
+#define SDMA0_STATUS_REG__INT_REQ_STALL_MASK                                                                  0x80000000L
+//SDMA0_STATUS1_REG
+#define SDMA0_STATUS1_REG__CE_WREQ_IDLE__SHIFT                                                                0x0
+#define SDMA0_STATUS1_REG__CE_WR_IDLE__SHIFT                                                                  0x1
+#define SDMA0_STATUS1_REG__CE_SPLIT_IDLE__SHIFT                                                               0x2
+#define SDMA0_STATUS1_REG__CE_RREQ_IDLE__SHIFT                                                                0x3
+#define SDMA0_STATUS1_REG__CE_OUT_IDLE__SHIFT                                                                 0x4
+#define SDMA0_STATUS1_REG__CE_IN_IDLE__SHIFT                                                                  0x5
+#define SDMA0_STATUS1_REG__CE_DST_IDLE__SHIFT                                                                 0x6
+#define SDMA0_STATUS1_REG__CE_DRM_IDLE__SHIFT                                                                 0x7
+#define SDMA0_STATUS1_REG__CE_DRM1_IDLE__SHIFT                                                                0x8
+#define SDMA0_STATUS1_REG__CE_CMD_IDLE__SHIFT                                                                 0x9
+#define SDMA0_STATUS1_REG__CE_AFIFO_FULL__SHIFT                                                               0xa
+#define SDMA0_STATUS1_REG__CE_INFO_FULL__SHIFT                                                                0xb
+#define SDMA0_STATUS1_REG__CE_INFO1_FULL__SHIFT                                                               0xc
+#define SDMA0_STATUS1_REG__EX_START__SHIFT                                                                    0xd
+#define SDMA0_STATUS1_REG__DRM_CTX_RESTORE__SHIFT                                                             0xe
+#define SDMA0_STATUS1_REG__CE_RD_STALL__SHIFT                                                                 0xf
+#define SDMA0_STATUS1_REG__CE_WR_STALL__SHIFT                                                                 0x10
+#define SDMA0_STATUS1_REG__SEC_INTR_STATUS__SHIFT                                                             0x11
+#define SDMA0_STATUS1_REG__WPTR_POLL_IDLE__SHIFT                                                              0x12
+#define SDMA0_STATUS1_REG__SDMA_IDLE__SHIFT                                                                   0x13
+#define SDMA0_STATUS1_REG__CE_WREQ_IDLE_MASK                                                                  0x00000001L
+#define SDMA0_STATUS1_REG__CE_WR_IDLE_MASK                                                                    0x00000002L
+#define SDMA0_STATUS1_REG__CE_SPLIT_IDLE_MASK                                                                 0x00000004L
+#define SDMA0_STATUS1_REG__CE_RREQ_IDLE_MASK                                                                  0x00000008L
+#define SDMA0_STATUS1_REG__CE_OUT_IDLE_MASK                                                                   0x00000010L
+#define SDMA0_STATUS1_REG__CE_IN_IDLE_MASK                                                                    0x00000020L
+#define SDMA0_STATUS1_REG__CE_DST_IDLE_MASK                                                                   0x00000040L
+#define SDMA0_STATUS1_REG__CE_DRM_IDLE_MASK                                                                   0x00000080L
+#define SDMA0_STATUS1_REG__CE_DRM1_IDLE_MASK                                                                  0x00000100L
+#define SDMA0_STATUS1_REG__CE_CMD_IDLE_MASK                                                                   0x00000200L
+#define SDMA0_STATUS1_REG__CE_AFIFO_FULL_MASK                                                                 0x00000400L
+#define SDMA0_STATUS1_REG__CE_INFO_FULL_MASK                                                                  0x00000800L
+#define SDMA0_STATUS1_REG__CE_INFO1_FULL_MASK                                                                 0x00001000L
+#define SDMA0_STATUS1_REG__EX_START_MASK                                                                      0x00002000L
+#define SDMA0_STATUS1_REG__DRM_CTX_RESTORE_MASK                                                               0x00004000L
+#define SDMA0_STATUS1_REG__CE_RD_STALL_MASK                                                                   0x00008000L
+#define SDMA0_STATUS1_REG__CE_WR_STALL_MASK                                                                   0x00010000L
+#define SDMA0_STATUS1_REG__SEC_INTR_STATUS_MASK                                                               0x00020000L
+#define SDMA0_STATUS1_REG__WPTR_POLL_IDLE_MASK                                                                0x00040000L
+#define SDMA0_STATUS1_REG__SDMA_IDLE_MASK                                                                     0x00080000L
+//SDMA0_CNTL1
+#define SDMA0_CNTL1__WPTR_POLL_FREQUENCY__SHIFT                                                               0x2
+#define SDMA0_CNTL1__WPTR_POLL_FREQUENCY_MASK                                                                 0x0000FFFCL
+//SDMA0_HBM_PAGE_CONFIG
+#define SDMA0_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT__SHIFT                                                      0x0
+#define SDMA0_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT_MASK                                                        0x00000003L
+//SDMA0_UCODE_CHECKSUM
+#define SDMA0_UCODE_CHECKSUM__DATA__SHIFT                                                                     0x0
+#define SDMA0_UCODE_CHECKSUM__DATA_MASK                                                                       0xFFFFFFFFL
+//SDMA0_FREEZE
+#define SDMA0_FREEZE__PREEMPT__SHIFT                                                                          0x0
+#define SDMA0_FREEZE__FREEZE__SHIFT                                                                           0x4
+#define SDMA0_FREEZE__FROZEN__SHIFT                                                                           0x5
+#define SDMA0_FREEZE__F32_FREEZE__SHIFT                                                                       0x6
+#define SDMA0_FREEZE__PREEMPT_MASK                                                                            0x00000001L
+#define SDMA0_FREEZE__FREEZE_MASK                                                                             0x00000010L
+#define SDMA0_FREEZE__FROZEN_MASK                                                                             0x00000020L
+#define SDMA0_FREEZE__F32_FREEZE_MASK                                                                         0x00000040L
+//SDMA0_PROCESS_QUANTUM0
+#define SDMA0_PROCESS_QUANTUM0__PROCESS0_QUANTUM__SHIFT                                                       0x0
+#define SDMA0_PROCESS_QUANTUM0__PROCESS1_QUANTUM__SHIFT                                                       0x8
+#define SDMA0_PROCESS_QUANTUM0__PROCESS2_QUANTUM__SHIFT                                                       0x10
+#define SDMA0_PROCESS_QUANTUM0__PROCESS3_QUANTUM__SHIFT                                                       0x18
+#define SDMA0_PROCESS_QUANTUM0__PROCESS0_QUANTUM_MASK                                                         0x000000FFL
+#define SDMA0_PROCESS_QUANTUM0__PROCESS1_QUANTUM_MASK                                                         0x0000FF00L
+#define SDMA0_PROCESS_QUANTUM0__PROCESS2_QUANTUM_MASK                                                         0x00FF0000L
+#define SDMA0_PROCESS_QUANTUM0__PROCESS3_QUANTUM_MASK                                                         0xFF000000L
+//SDMA0_PROCESS_QUANTUM1
+#define SDMA0_PROCESS_QUANTUM1__PROCESS4_QUANTUM__SHIFT                                                       0x0
+#define SDMA0_PROCESS_QUANTUM1__PROCESS5_QUANTUM__SHIFT                                                       0x8
+#define SDMA0_PROCESS_QUANTUM1__PROCESS6_QUANTUM__SHIFT                                                       0x10
+#define SDMA0_PROCESS_QUANTUM1__PROCESS7_QUANTUM__SHIFT                                                       0x18
+#define SDMA0_PROCESS_QUANTUM1__PROCESS4_QUANTUM_MASK                                                         0x000000FFL
+#define SDMA0_PROCESS_QUANTUM1__PROCESS5_QUANTUM_MASK                                                         0x0000FF00L
+#define SDMA0_PROCESS_QUANTUM1__PROCESS6_QUANTUM_MASK                                                         0x00FF0000L
+#define SDMA0_PROCESS_QUANTUM1__PROCESS7_QUANTUM_MASK                                                         0xFF000000L
+//SDMA0_WATCHDOG_CNTL
+#define SDMA0_WATCHDOG_CNTL__QUEUE_HANG_COUNT__SHIFT                                                          0x0
+#define SDMA0_WATCHDOG_CNTL__CMD_TIMEOUT_COUNT__SHIFT                                                         0x8
+#define SDMA0_WATCHDOG_CNTL__QUEUE_HANG_COUNT_MASK                                                            0x000000FFL
+#define SDMA0_WATCHDOG_CNTL__CMD_TIMEOUT_COUNT_MASK                                                           0x0000FF00L
+//SDMA0_QUEUE_STATUS0
+#define SDMA0_QUEUE_STATUS0__QUEUE0_STATUS__SHIFT                                                             0x0
+#define SDMA0_QUEUE_STATUS0__QUEUE1_STATUS__SHIFT                                                             0x4
+#define SDMA0_QUEUE_STATUS0__QUEUE2_STATUS__SHIFT                                                             0x8
+#define SDMA0_QUEUE_STATUS0__QUEUE3_STATUS__SHIFT                                                             0xc
+#define SDMA0_QUEUE_STATUS0__QUEUE4_STATUS__SHIFT                                                             0x10
+#define SDMA0_QUEUE_STATUS0__QUEUE5_STATUS__SHIFT                                                             0x14
+#define SDMA0_QUEUE_STATUS0__QUEUE6_STATUS__SHIFT                                                             0x18
+#define SDMA0_QUEUE_STATUS0__QUEUE7_STATUS__SHIFT                                                             0x1c
+#define SDMA0_QUEUE_STATUS0__QUEUE0_STATUS_MASK                                                               0x0000000FL
+#define SDMA0_QUEUE_STATUS0__QUEUE1_STATUS_MASK                                                               0x000000F0L
+#define SDMA0_QUEUE_STATUS0__QUEUE2_STATUS_MASK                                                               0x00000F00L
+#define SDMA0_QUEUE_STATUS0__QUEUE3_STATUS_MASK                                                               0x0000F000L
+#define SDMA0_QUEUE_STATUS0__QUEUE4_STATUS_MASK                                                               0x000F0000L
+#define SDMA0_QUEUE_STATUS0__QUEUE5_STATUS_MASK                                                               0x00F00000L
+#define SDMA0_QUEUE_STATUS0__QUEUE6_STATUS_MASK                                                               0x0F000000L
+#define SDMA0_QUEUE_STATUS0__QUEUE7_STATUS_MASK                                                               0xF0000000L
+//SDMA0_EDC_CONFIG
+#define SDMA0_EDC_CONFIG__WRITE_DIS__SHIFT                                                                    0x0
+#define SDMA0_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
+#define SDMA0_EDC_CONFIG__ECC_INT_ENABLE__SHIFT                                                               0x2
+#define SDMA0_EDC_CONFIG__WRITE_DIS_MASK                                                                      0x00000001L
+#define SDMA0_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
+#define SDMA0_EDC_CONFIG__ECC_INT_ENABLE_MASK                                                                 0x00000004L
+//SDMA0_BA_THRESHOLD
+#define SDMA0_BA_THRESHOLD__READ_THRES__SHIFT                                                                 0x0
+#define SDMA0_BA_THRESHOLD__WRITE_THRES__SHIFT                                                                0x10
+#define SDMA0_BA_THRESHOLD__READ_THRES_MASK                                                                   0x000003FFL
+#define SDMA0_BA_THRESHOLD__WRITE_THRES_MASK                                                                  0x03FF0000L
+//SDMA0_ID
+#define SDMA0_ID__DEVICE_ID__SHIFT                                                                            0x0
+#define SDMA0_ID__DEVICE_ID_MASK                                                                              0x000000FFL
+//SDMA0_VERSION
+#define SDMA0_VERSION__MINVER__SHIFT                                                                          0x0
+#define SDMA0_VERSION__MAJVER__SHIFT                                                                          0x8
+#define SDMA0_VERSION__REV__SHIFT                                                                             0x10
+#define SDMA0_VERSION__MINVER_MASK                                                                            0x0000007FL
+#define SDMA0_VERSION__MAJVER_MASK                                                                            0x00007F00L
+#define SDMA0_VERSION__REV_MASK                                                                               0x003F0000L
+//SDMA0_EDC_COUNTER
+#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_DED__SHIFT                                                          0x0
+#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_SEC__SHIFT                                                          0x1
+#define SDMA0_EDC_COUNTER__SDMA_RB_CMD_BUF_SED__SHIFT                                                         0x2
+#define SDMA0_EDC_COUNTER__SDMA_IB_CMD_BUF_SED__SHIFT                                                         0x3
+#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED__SHIFT                                                      0x4
+#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED__SHIFT                                                   0x5
+#define SDMA0_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED__SHIFT                                                      0x6
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED__SHIFT                                                    0x7
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED__SHIFT                                                    0x8
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED__SHIFT                                                    0x9
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED__SHIFT                                                    0xa
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED__SHIFT                                                    0xb
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED__SHIFT                                                    0xc
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED__SHIFT                                                    0xd
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED__SHIFT                                                    0xe
+#define SDMA0_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED__SHIFT                                                      0xf
+#define SDMA0_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED__SHIFT                                                    0x10
+#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_DED_MASK                                                            0x00000001L
+#define SDMA0_EDC_COUNTER__SDMA_UCODE_BUF_SEC_MASK                                                            0x00000002L
+#define SDMA0_EDC_COUNTER__SDMA_RB_CMD_BUF_SED_MASK                                                           0x00000004L
+#define SDMA0_EDC_COUNTER__SDMA_IB_CMD_BUF_SED_MASK                                                           0x00000008L
+#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED_MASK                                                        0x00000010L
+#define SDMA0_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED_MASK                                                     0x00000020L
+#define SDMA0_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED_MASK                                                        0x00000040L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED_MASK                                                      0x00000080L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED_MASK                                                      0x00000100L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED_MASK                                                      0x00000200L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED_MASK                                                      0x00000400L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED_MASK                                                      0x00000800L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED_MASK                                                      0x00001000L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED_MASK                                                      0x00002000L
+#define SDMA0_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED_MASK                                                      0x00004000L
+#define SDMA0_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED_MASK                                                        0x00008000L
+#define SDMA0_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED_MASK                                                      0x00010000L
+//SDMA0_EDC_COUNTER_CLEAR
+#define SDMA0_EDC_COUNTER_CLEAR__DUMMY__SHIFT                                                                 0x0
+#define SDMA0_EDC_COUNTER_CLEAR__DUMMY_MASK                                                                   0x00000001L
+//SDMA0_STATUS2_REG
+#define SDMA0_STATUS2_REG__ID__SHIFT                                                                          0x0
+#define SDMA0_STATUS2_REG__TH0F32_INSTR_PTR__SHIFT                                                            0x2
+#define SDMA0_STATUS2_REG__CMD_OP__SHIFT                                                                      0x10
+#define SDMA0_STATUS2_REG__ID_MASK                                                                            0x00000003L
+#define SDMA0_STATUS2_REG__TH0F32_INSTR_PTR_MASK                                                              0x0000FFFCL
+#define SDMA0_STATUS2_REG__CMD_OP_MASK                                                                        0xFFFF0000L
+//SDMA0_ATOMIC_CNTL
+#define SDMA0_ATOMIC_CNTL__LOOP_TIMER__SHIFT                                                                  0x0
+#define SDMA0_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT                                                       0x1f
+#define SDMA0_ATOMIC_CNTL__LOOP_TIMER_MASK                                                                    0x7FFFFFFFL
+#define SDMA0_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK                                                         0x80000000L
+//SDMA0_ATOMIC_PREOP_LO
+#define SDMA0_ATOMIC_PREOP_LO__DATA__SHIFT                                                                    0x0
+#define SDMA0_ATOMIC_PREOP_LO__DATA_MASK                                                                      0xFFFFFFFFL
+//SDMA0_ATOMIC_PREOP_HI
+#define SDMA0_ATOMIC_PREOP_HI__DATA__SHIFT                                                                    0x0
+#define SDMA0_ATOMIC_PREOP_HI__DATA_MASK                                                                      0xFFFFFFFFL
+//SDMA0_UTCL1_CNTL
+#define SDMA0_UTCL1_CNTL__REDO_DELAY__SHIFT                                                                   0x0
+#define SDMA0_UTCL1_CNTL__PAGE_WAIT_DELAY__SHIFT                                                              0x5
+#define SDMA0_UTCL1_CNTL__RESP_MODE__SHIFT                                                                    0x9
+#define SDMA0_UTCL1_CNTL__FORCE_INVALIDATION__SHIFT                                                           0xe
+#define SDMA0_UTCL1_CNTL__FORCE_INVREQ_HEAVY__SHIFT                                                           0xf
+#define SDMA0_UTCL1_CNTL__WR_EXE_PERMS_CTRL__SHIFT                                                            0x10
+#define SDMA0_UTCL1_CNTL__RD_EXE_PERMS_CTRL__SHIFT                                                            0x11
+#define SDMA0_UTCL1_CNTL__INVACK_DELAY__SHIFT                                                                 0x12
+#define SDMA0_UTCL1_CNTL__REQL2_CREDIT__SHIFT                                                                 0x18
+#define SDMA0_UTCL1_CNTL__REDO_DELAY_MASK                                                                     0x0000001FL
+#define SDMA0_UTCL1_CNTL__PAGE_WAIT_DELAY_MASK                                                                0x000001E0L
+#define SDMA0_UTCL1_CNTL__RESP_MODE_MASK                                                                      0x00000600L
+#define SDMA0_UTCL1_CNTL__FORCE_INVALIDATION_MASK                                                             0x00004000L
+#define SDMA0_UTCL1_CNTL__FORCE_INVREQ_HEAVY_MASK                                                             0x00008000L
+#define SDMA0_UTCL1_CNTL__WR_EXE_PERMS_CTRL_MASK                                                              0x00010000L
+#define SDMA0_UTCL1_CNTL__RD_EXE_PERMS_CTRL_MASK                                                              0x00020000L
+#define SDMA0_UTCL1_CNTL__INVACK_DELAY_MASK                                                                   0x003C0000L
+#define SDMA0_UTCL1_CNTL__REQL2_CREDIT_MASK                                                                   0x3F000000L
+//SDMA0_UTCL1_WATERMK
+#define SDMA0_UTCL1_WATERMK__WR_REQ_FIFO_WATERMK__SHIFT                                                       0x0
+#define SDMA0_UTCL1_WATERMK__WR_REQ_FIFO_DEPTH_STEP__SHIFT                                                    0x4
+#define SDMA0_UTCL1_WATERMK__RD_REQ_FIFO_WATERMK__SHIFT                                                       0x6
+#define SDMA0_UTCL1_WATERMK__RD_REQ_FIFO_DEPTH_STEP__SHIFT                                                    0xa
+#define SDMA0_UTCL1_WATERMK__WR_PAGE_FIFO_WATERMK__SHIFT                                                      0xc
+#define SDMA0_UTCL1_WATERMK__WR_PAGE_FIFO_DEPTH_STEP__SHIFT                                                   0x10
+#define SDMA0_UTCL1_WATERMK__RD_PAGE_FIFO_WATERMK__SHIFT                                                      0x12
+#define SDMA0_UTCL1_WATERMK__RD_PAGE_FIFO_DEPTH_STEP__SHIFT                                                   0x16
+#define SDMA0_UTCL1_WATERMK__WR_REQ_FIFO_WATERMK_MASK                                                         0x0000000FL
+#define SDMA0_UTCL1_WATERMK__WR_REQ_FIFO_DEPTH_STEP_MASK                                                      0x00000030L
+#define SDMA0_UTCL1_WATERMK__RD_REQ_FIFO_WATERMK_MASK                                                         0x000003C0L
+#define SDMA0_UTCL1_WATERMK__RD_REQ_FIFO_DEPTH_STEP_MASK                                                      0x00000C00L
+#define SDMA0_UTCL1_WATERMK__WR_PAGE_FIFO_WATERMK_MASK                                                        0x0000F000L
+#define SDMA0_UTCL1_WATERMK__WR_PAGE_FIFO_DEPTH_STEP_MASK                                                     0x00030000L
+#define SDMA0_UTCL1_WATERMK__RD_PAGE_FIFO_WATERMK_MASK                                                        0x003C0000L
+#define SDMA0_UTCL1_WATERMK__RD_PAGE_FIFO_DEPTH_STEP_MASK                                                     0x00C00000L
+//SDMA0_UTCL1_TIMEOUT
+#define SDMA0_UTCL1_TIMEOUT__XNACK_LIMIT__SHIFT                                                               0x0
+#define SDMA0_UTCL1_TIMEOUT__XNACK_LIMIT_MASK                                                                 0x0000FFFFL
+//SDMA0_UTCL1_PAGE
+#define SDMA0_UTCL1_PAGE__VM_HOLE__SHIFT                                                                      0x0
+#define SDMA0_UTCL1_PAGE__REQ_TYPE__SHIFT                                                                     0x1
+#define SDMA0_UTCL1_PAGE__TMZ_ENABLE__SHIFT                                                                   0x5
+#define SDMA0_UTCL1_PAGE__USE_MTYPE__SHIFT                                                                    0x6
+#define SDMA0_UTCL1_PAGE__USE_PT_SNOOP__SHIFT                                                                 0xa
+#define SDMA0_UTCL1_PAGE__USE_IO__SHIFT                                                                       0xb
+#define SDMA0_UTCL1_PAGE__RD_L2_POLICY__SHIFT                                                                 0xc
+#define SDMA0_UTCL1_PAGE__WR_L2_POLICY__SHIFT                                                                 0xe
+#define SDMA0_UTCL1_PAGE__DMA_PAGE_SIZE__SHIFT                                                                0x10
+#define SDMA0_UTCL1_PAGE__USE_BC__SHIFT                                                                       0x16
+#define SDMA0_UTCL1_PAGE__ADDR_IS_PA__SHIFT                                                                   0x17
+#define SDMA0_UTCL1_PAGE__LLC_NOALLOC__SHIFT                                                                  0x18
+#define SDMA0_UTCL1_PAGE__VM_HOLE_MASK                                                                        0x00000001L
+#define SDMA0_UTCL1_PAGE__REQ_TYPE_MASK                                                                       0x0000001EL
+#define SDMA0_UTCL1_PAGE__TMZ_ENABLE_MASK                                                                     0x00000020L
+#define SDMA0_UTCL1_PAGE__USE_MTYPE_MASK                                                                      0x000003C0L
+#define SDMA0_UTCL1_PAGE__USE_PT_SNOOP_MASK                                                                   0x00000400L
+#define SDMA0_UTCL1_PAGE__USE_IO_MASK                                                                         0x00000800L
+#define SDMA0_UTCL1_PAGE__RD_L2_POLICY_MASK                                                                   0x00003000L
+#define SDMA0_UTCL1_PAGE__WR_L2_POLICY_MASK                                                                   0x0000C000L
+#define SDMA0_UTCL1_PAGE__DMA_PAGE_SIZE_MASK                                                                  0x003F0000L
+#define SDMA0_UTCL1_PAGE__USE_BC_MASK                                                                         0x00400000L
+#define SDMA0_UTCL1_PAGE__ADDR_IS_PA_MASK                                                                     0x00800000L
+#define SDMA0_UTCL1_PAGE__LLC_NOALLOC_MASK                                                                    0x01000000L
+//SDMA0_UTCL1_RD_STATUS
+#define SDMA0_UTCL1_RD_STATUS__RD_VA_FIFO_EMPTY__SHIFT                                                        0x0
+#define SDMA0_UTCL1_RD_STATUS__RD_REG_ENTRY_EMPTY__SHIFT                                                      0x1
+#define SDMA0_UTCL1_RD_STATUS__RD_PAGE_FIFO_EMPTY__SHIFT                                                      0x2
+#define SDMA0_UTCL1_RD_STATUS__RD_REQ_FIFO_EMPTY__SHIFT                                                       0x3
+#define SDMA0_UTCL1_RD_STATUS__RD_VA_REQ_FIFO_EMPTY__SHIFT                                                    0x4
+#define SDMA0_UTCL1_RD_STATUS__RESERVED0__SHIFT                                                               0x5
+#define SDMA0_UTCL1_RD_STATUS__RESERVED1__SHIFT                                                               0x6
+#define SDMA0_UTCL1_RD_STATUS__META_Q_EMPTY__SHIFT                                                            0x7
+#define SDMA0_UTCL1_RD_STATUS__RD_VA_FIFO_FULL__SHIFT                                                         0x8
+#define SDMA0_UTCL1_RD_STATUS__RD_REG_ENTRY_FULL__SHIFT                                                       0x9
+#define SDMA0_UTCL1_RD_STATUS__RD_PAGE_FIFO_FULL__SHIFT                                                       0xa
+#define SDMA0_UTCL1_RD_STATUS__RD_REQ_FIFO_FULL__SHIFT                                                        0xb
+#define SDMA0_UTCL1_RD_STATUS__RD_VA_REQ_FIFO_FULL__SHIFT                                                     0xc
+#define SDMA0_UTCL1_RD_STATUS__RESERVED2__SHIFT                                                               0xd
+#define SDMA0_UTCL1_RD_STATUS__RESERVED3__SHIFT                                                               0xe
+#define SDMA0_UTCL1_RD_STATUS__META_Q_FULL__SHIFT                                                             0xf
+#define SDMA0_UTCL1_RD_STATUS__RD_L2_INTF_IDLE__SHIFT                                                         0x10
+#define SDMA0_UTCL1_RD_STATUS__RD_REQRET_IDLE__SHIFT                                                          0x11
+#define SDMA0_UTCL1_RD_STATUS__RD_REQ_IDLE__SHIFT                                                             0x12
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_TYPE__SHIFT                                                           0x13
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_DATA_PA_READY__SHIFT                                                  0x15
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_META_PA_READY__SHIFT                                                  0x16
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_REG_READY__SHIFT                                                      0x17
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_PAGE_FIFO_READY__SHIFT                                                0x18
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_REQ_FIFO_READY__SHIFT                                                 0x19
+#define SDMA0_UTCL1_RD_STATUS__RESERVED4__SHIFT                                                               0x1a
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_OUT_RTR__SHIFT                                                        0x1b
+#define SDMA0_UTCL1_RD_STATUS__RDREQ_IN_RTR__SHIFT                                                            0x1c
+#define SDMA0_UTCL1_RD_STATUS__RDREQ_OUT_RTR__SHIFT                                                           0x1d
+#define SDMA0_UTCL1_RD_STATUS__INV_BUSY__SHIFT                                                                0x1e
+#define SDMA0_UTCL1_RD_STATUS__DBIT_REQ_IDLE__SHIFT                                                           0x1f
+#define SDMA0_UTCL1_RD_STATUS__RD_VA_FIFO_EMPTY_MASK                                                          0x00000001L
+#define SDMA0_UTCL1_RD_STATUS__RD_REG_ENTRY_EMPTY_MASK                                                        0x00000002L
+#define SDMA0_UTCL1_RD_STATUS__RD_PAGE_FIFO_EMPTY_MASK                                                        0x00000004L
+#define SDMA0_UTCL1_RD_STATUS__RD_REQ_FIFO_EMPTY_MASK                                                         0x00000008L
+#define SDMA0_UTCL1_RD_STATUS__RD_VA_REQ_FIFO_EMPTY_MASK                                                      0x00000010L
+#define SDMA0_UTCL1_RD_STATUS__RESERVED0_MASK                                                                 0x00000020L
+#define SDMA0_UTCL1_RD_STATUS__RESERVED1_MASK                                                                 0x00000040L
+#define SDMA0_UTCL1_RD_STATUS__META_Q_EMPTY_MASK                                                              0x00000080L
+#define SDMA0_UTCL1_RD_STATUS__RD_VA_FIFO_FULL_MASK                                                           0x00000100L
+#define SDMA0_UTCL1_RD_STATUS__RD_REG_ENTRY_FULL_MASK                                                         0x00000200L
+#define SDMA0_UTCL1_RD_STATUS__RD_PAGE_FIFO_FULL_MASK                                                         0x00000400L
+#define SDMA0_UTCL1_RD_STATUS__RD_REQ_FIFO_FULL_MASK                                                          0x00000800L
+#define SDMA0_UTCL1_RD_STATUS__RD_VA_REQ_FIFO_FULL_MASK                                                       0x00001000L
+#define SDMA0_UTCL1_RD_STATUS__RESERVED2_MASK                                                                 0x00002000L
+#define SDMA0_UTCL1_RD_STATUS__RESERVED3_MASK                                                                 0x00004000L
+#define SDMA0_UTCL1_RD_STATUS__META_Q_FULL_MASK                                                               0x00008000L
+#define SDMA0_UTCL1_RD_STATUS__RD_L2_INTF_IDLE_MASK                                                           0x00010000L
+#define SDMA0_UTCL1_RD_STATUS__RD_REQRET_IDLE_MASK                                                            0x00020000L
+#define SDMA0_UTCL1_RD_STATUS__RD_REQ_IDLE_MASK                                                               0x00040000L
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_TYPE_MASK                                                             0x00180000L
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_DATA_PA_READY_MASK                                                    0x00200000L
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_META_PA_READY_MASK                                                    0x00400000L
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_REG_READY_MASK                                                        0x00800000L
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_PAGE_FIFO_READY_MASK                                                  0x01000000L
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_REQ_FIFO_READY_MASK                                                   0x02000000L
+#define SDMA0_UTCL1_RD_STATUS__RESERVED4_MASK                                                                 0x04000000L
+#define SDMA0_UTCL1_RD_STATUS__RD_MERGE_OUT_RTR_MASK                                                          0x08000000L
+#define SDMA0_UTCL1_RD_STATUS__RDREQ_IN_RTR_MASK                                                              0x10000000L
+#define SDMA0_UTCL1_RD_STATUS__RDREQ_OUT_RTR_MASK                                                             0x20000000L
+#define SDMA0_UTCL1_RD_STATUS__INV_BUSY_MASK                                                                  0x40000000L
+#define SDMA0_UTCL1_RD_STATUS__DBIT_REQ_IDLE_MASK                                                             0x80000000L
+//SDMA0_UTCL1_WR_STATUS
+#define SDMA0_UTCL1_WR_STATUS__WR_VA_FIFO_EMPTY__SHIFT                                                        0x0
+#define SDMA0_UTCL1_WR_STATUS__WR_REG_ENTRY_EMPTY__SHIFT                                                      0x1
+#define SDMA0_UTCL1_WR_STATUS__WR_PAGE_FIFO_EMPTY__SHIFT                                                      0x2
+#define SDMA0_UTCL1_WR_STATUS__WR_REQ_FIFO_EMPTY__SHIFT                                                       0x3
+#define SDMA0_UTCL1_WR_STATUS__WR_VA_REQ_FIFO_EMPTY__SHIFT                                                    0x4
+#define SDMA0_UTCL1_WR_STATUS__WR_DATA2_EMPTY__SHIFT                                                          0x5
+#define SDMA0_UTCL1_WR_STATUS__WR_DATA1_EMPTY__SHIFT                                                          0x6
+#define SDMA0_UTCL1_WR_STATUS__RESERVED0__SHIFT                                                               0x7
+#define SDMA0_UTCL1_WR_STATUS__WR_VA_FIFO_FULL__SHIFT                                                         0x8
+#define SDMA0_UTCL1_WR_STATUS__WR_REG_ENTRY_FULL__SHIFT                                                       0x9
+#define SDMA0_UTCL1_WR_STATUS__WR_PAGE_FIFO_FULL__SHIFT                                                       0xa
+#define SDMA0_UTCL1_WR_STATUS__WR_REQ_FIFO_FULL__SHIFT                                                        0xb
+#define SDMA0_UTCL1_WR_STATUS__WR_VA_REQ_FIFO_FULL__SHIFT                                                     0xc
+#define SDMA0_UTCL1_WR_STATUS__WR_DATA2_FULL__SHIFT                                                           0xd
+#define SDMA0_UTCL1_WR_STATUS__WR_DATA1_FULL__SHIFT                                                           0xe
+#define SDMA0_UTCL1_WR_STATUS__F32_WR_RTR__SHIFT                                                              0xf
+#define SDMA0_UTCL1_WR_STATUS__WR_L2_INTF_IDLE__SHIFT                                                         0x10
+#define SDMA0_UTCL1_WR_STATUS__WR_REQRET_IDLE__SHIFT                                                          0x11
+#define SDMA0_UTCL1_WR_STATUS__WR_REQ_IDLE__SHIFT                                                             0x12
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_TYPE__SHIFT                                                           0x13
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_DATA_PA_READY__SHIFT                                                  0x15
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_META_PA_READY__SHIFT                                                  0x16
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_REG_READY__SHIFT                                                      0x17
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_PAGE_FIFO_READY__SHIFT                                                0x18
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_REQ_FIFO_READY__SHIFT                                                 0x19
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_DATA_SEL__SHIFT                                                       0x1a
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_OUT_RTR__SHIFT                                                        0x1b
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_IN_RTR__SHIFT                                                            0x1c
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_OUT_RTR__SHIFT                                                           0x1d
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_IN_DATA1_RTR__SHIFT                                                      0x1e
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_IN_DATA2_RTR__SHIFT                                                      0x1f
+#define SDMA0_UTCL1_WR_STATUS__WR_VA_FIFO_EMPTY_MASK                                                          0x00000001L
+#define SDMA0_UTCL1_WR_STATUS__WR_REG_ENTRY_EMPTY_MASK                                                        0x00000002L
+#define SDMA0_UTCL1_WR_STATUS__WR_PAGE_FIFO_EMPTY_MASK                                                        0x00000004L
+#define SDMA0_UTCL1_WR_STATUS__WR_REQ_FIFO_EMPTY_MASK                                                         0x00000008L
+#define SDMA0_UTCL1_WR_STATUS__WR_VA_REQ_FIFO_EMPTY_MASK                                                      0x00000010L
+#define SDMA0_UTCL1_WR_STATUS__WR_DATA2_EMPTY_MASK                                                            0x00000020L
+#define SDMA0_UTCL1_WR_STATUS__WR_DATA1_EMPTY_MASK                                                            0x00000040L
+#define SDMA0_UTCL1_WR_STATUS__RESERVED0_MASK                                                                 0x00000080L
+#define SDMA0_UTCL1_WR_STATUS__WR_VA_FIFO_FULL_MASK                                                           0x00000100L
+#define SDMA0_UTCL1_WR_STATUS__WR_REG_ENTRY_FULL_MASK                                                         0x00000200L
+#define SDMA0_UTCL1_WR_STATUS__WR_PAGE_FIFO_FULL_MASK                                                         0x00000400L
+#define SDMA0_UTCL1_WR_STATUS__WR_REQ_FIFO_FULL_MASK                                                          0x00000800L
+#define SDMA0_UTCL1_WR_STATUS__WR_VA_REQ_FIFO_FULL_MASK                                                       0x00001000L
+#define SDMA0_UTCL1_WR_STATUS__WR_DATA2_FULL_MASK                                                             0x00002000L
+#define SDMA0_UTCL1_WR_STATUS__WR_DATA1_FULL_MASK                                                             0x00004000L
+#define SDMA0_UTCL1_WR_STATUS__F32_WR_RTR_MASK                                                                0x00008000L
+#define SDMA0_UTCL1_WR_STATUS__WR_L2_INTF_IDLE_MASK                                                           0x00010000L
+#define SDMA0_UTCL1_WR_STATUS__WR_REQRET_IDLE_MASK                                                            0x00020000L
+#define SDMA0_UTCL1_WR_STATUS__WR_REQ_IDLE_MASK                                                               0x00040000L
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_TYPE_MASK                                                             0x00180000L
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_DATA_PA_READY_MASK                                                    0x00200000L
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_META_PA_READY_MASK                                                    0x00400000L
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_REG_READY_MASK                                                        0x00800000L
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_PAGE_FIFO_READY_MASK                                                  0x01000000L
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_REQ_FIFO_READY_MASK                                                   0x02000000L
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_DATA_SEL_MASK                                                         0x04000000L
+#define SDMA0_UTCL1_WR_STATUS__WR_MERGE_OUT_RTR_MASK                                                          0x08000000L
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_IN_RTR_MASK                                                              0x10000000L
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_OUT_RTR_MASK                                                             0x20000000L
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_IN_DATA1_RTR_MASK                                                        0x40000000L
+#define SDMA0_UTCL1_WR_STATUS__WRREQ_IN_DATA2_RTR_MASK                                                        0x80000000L
+//SDMA0_UTCL1_INV0
+#define SDMA0_UTCL1_INV0__INV_PROC_BUSY__SHIFT                                                                0x0
+#define SDMA0_UTCL1_INV0__GPUVM_FRAG_SIZE__SHIFT                                                              0x1
+#define SDMA0_UTCL1_INV0__GPUVM_VMID__SHIFT                                                                   0x7
+#define SDMA0_UTCL1_INV0__GPUVM_MODE__SHIFT                                                                   0xb
+#define SDMA0_UTCL1_INV0__GPUVM_HIGH__SHIFT                                                                   0xd
+#define SDMA0_UTCL1_INV0__GPUVM_TAG__SHIFT                                                                    0xe
+#define SDMA0_UTCL1_INV0__GPUVM_VMID_HIGH__SHIFT                                                              0x12
+#define SDMA0_UTCL1_INV0__GPUVM_VMID_LOW__SHIFT                                                               0x16
+#define SDMA0_UTCL1_INV0__INV_TYPE__SHIFT                                                                     0x1a
+#define SDMA0_UTCL1_INV0__INV_PROC_BUSY_MASK                                                                  0x00000001L
+#define SDMA0_UTCL1_INV0__GPUVM_FRAG_SIZE_MASK                                                                0x0000007EL
+#define SDMA0_UTCL1_INV0__GPUVM_VMID_MASK                                                                     0x00000780L
+#define SDMA0_UTCL1_INV0__GPUVM_MODE_MASK                                                                     0x00001800L
+#define SDMA0_UTCL1_INV0__GPUVM_HIGH_MASK                                                                     0x00002000L
+#define SDMA0_UTCL1_INV0__GPUVM_TAG_MASK                                                                      0x0003C000L
+#define SDMA0_UTCL1_INV0__GPUVM_VMID_HIGH_MASK                                                                0x003C0000L
+#define SDMA0_UTCL1_INV0__GPUVM_VMID_LOW_MASK                                                                 0x03C00000L
+#define SDMA0_UTCL1_INV0__INV_TYPE_MASK                                                                       0x0C000000L
+//SDMA0_UTCL1_INV1
+#define SDMA0_UTCL1_INV1__INV_ADDR_LO__SHIFT                                                                  0x0
+#define SDMA0_UTCL1_INV1__INV_ADDR_LO_MASK                                                                    0xFFFFFFFFL
+//SDMA0_UTCL1_INV2
+#define SDMA0_UTCL1_INV2__CPF_VMID__SHIFT                                                                     0x0
+#define SDMA0_UTCL1_INV2__CPF_FLUSH_TYPE__SHIFT                                                               0x10
+#define SDMA0_UTCL1_INV2__CPF_FRAG_SIZE__SHIFT                                                                0x11
+#define SDMA0_UTCL1_INV2__CPF_VMID_MASK                                                                       0x0000FFFFL
+#define SDMA0_UTCL1_INV2__CPF_FLUSH_TYPE_MASK                                                                 0x00010000L
+#define SDMA0_UTCL1_INV2__CPF_FRAG_SIZE_MASK                                                                  0x007E0000L
+//SDMA0_UTCL1_RD_XNACK0
+#define SDMA0_UTCL1_RD_XNACK0__XNACK_FAULT_ADDR_LO__SHIFT                                                     0x0
+#define SDMA0_UTCL1_RD_XNACK0__XNACK_FAULT_ADDR_LO_MASK                                                       0xFFFFFFFFL
+//SDMA0_UTCL1_RD_XNACK1
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_FAULT_ADDR_HI__SHIFT                                                     0x0
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_FAULT_VMID__SHIFT                                                        0x4
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_FAULT_VECTOR__SHIFT                                                      0x8
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_NULL_VECTOR__SHIFT                                                       0xa
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_TIMEOUT_VECTOR__SHIFT                                                    0xc
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_FAULT_FLAG__SHIFT                                                        0xe
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_NULL_FLAG__SHIFT                                                         0xf
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_TIMEOUT_FLAG__SHIFT                                                      0x10
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_FAULT_ADDR_HI_MASK                                                       0x0000000FL
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_FAULT_VMID_MASK                                                          0x000000F0L
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_FAULT_VECTOR_MASK                                                        0x00000300L
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_NULL_VECTOR_MASK                                                         0x00000C00L
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_TIMEOUT_VECTOR_MASK                                                      0x00003000L
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_FAULT_FLAG_MASK                                                          0x00004000L
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_NULL_FLAG_MASK                                                           0x00008000L
+#define SDMA0_UTCL1_RD_XNACK1__XNACK_TIMEOUT_FLAG_MASK                                                        0x00010000L
+//SDMA0_UTCL1_WR_XNACK0
+#define SDMA0_UTCL1_WR_XNACK0__XNACK_FAULT_ADDR_LO__SHIFT                                                     0x0
+#define SDMA0_UTCL1_WR_XNACK0__XNACK_FAULT_ADDR_LO_MASK                                                       0xFFFFFFFFL
+//SDMA0_UTCL1_WR_XNACK1
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_FAULT_ADDR_HI__SHIFT                                                     0x0
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_FAULT_VMID__SHIFT                                                        0x4
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_FAULT_VECTOR__SHIFT                                                      0x8
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_NULL_VECTOR__SHIFT                                                       0xa
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_TIMEOUT_VECTOR__SHIFT                                                    0xc
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_FAULT_FLAG__SHIFT                                                        0xe
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_NULL_FLAG__SHIFT                                                         0xf
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_TIMEOUT_FLAG__SHIFT                                                      0x10
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_FAULT_ADDR_HI_MASK                                                       0x0000000FL
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_FAULT_VMID_MASK                                                          0x000000F0L
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_FAULT_VECTOR_MASK                                                        0x00000300L
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_NULL_VECTOR_MASK                                                         0x00000C00L
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_TIMEOUT_VECTOR_MASK                                                      0x00003000L
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_FAULT_FLAG_MASK                                                          0x00004000L
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_NULL_FLAG_MASK                                                           0x00008000L
+#define SDMA0_UTCL1_WR_XNACK1__XNACK_TIMEOUT_FLAG_MASK                                                        0x00010000L
+//SDMA0_RELAX_ORDERING_LUT
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED0__SHIFT                                                            0x0
+#define SDMA0_RELAX_ORDERING_LUT__COPY__SHIFT                                                                 0x1
+#define SDMA0_RELAX_ORDERING_LUT__WRITE__SHIFT                                                                0x2
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED3__SHIFT                                                            0x3
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED4__SHIFT                                                            0x4
+#define SDMA0_RELAX_ORDERING_LUT__FENCE__SHIFT                                                                0x5
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED76__SHIFT                                                           0x6
+#define SDMA0_RELAX_ORDERING_LUT__POLL_MEM__SHIFT                                                             0x8
+#define SDMA0_RELAX_ORDERING_LUT__COND_EXE__SHIFT                                                             0x9
+#define SDMA0_RELAX_ORDERING_LUT__ATOMIC__SHIFT                                                               0xa
+#define SDMA0_RELAX_ORDERING_LUT__CONST_FILL__SHIFT                                                           0xb
+#define SDMA0_RELAX_ORDERING_LUT__PTEPDE__SHIFT                                                               0xc
+#define SDMA0_RELAX_ORDERING_LUT__TIMESTAMP__SHIFT                                                            0xd
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED__SHIFT                                                             0xe
+#define SDMA0_RELAX_ORDERING_LUT__WORLD_SWITCH__SHIFT                                                         0x1b
+#define SDMA0_RELAX_ORDERING_LUT__RPTR_WRB__SHIFT                                                             0x1c
+#define SDMA0_RELAX_ORDERING_LUT__WPTR_POLL__SHIFT                                                            0x1d
+#define SDMA0_RELAX_ORDERING_LUT__IB_FETCH__SHIFT                                                             0x1e
+#define SDMA0_RELAX_ORDERING_LUT__RB_FETCH__SHIFT                                                             0x1f
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED0_MASK                                                              0x00000001L
+#define SDMA0_RELAX_ORDERING_LUT__COPY_MASK                                                                   0x00000002L
+#define SDMA0_RELAX_ORDERING_LUT__WRITE_MASK                                                                  0x00000004L
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED3_MASK                                                              0x00000008L
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED4_MASK                                                              0x00000010L
+#define SDMA0_RELAX_ORDERING_LUT__FENCE_MASK                                                                  0x00000020L
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED76_MASK                                                             0x000000C0L
+#define SDMA0_RELAX_ORDERING_LUT__POLL_MEM_MASK                                                               0x00000100L
+#define SDMA0_RELAX_ORDERING_LUT__COND_EXE_MASK                                                               0x00000200L
+#define SDMA0_RELAX_ORDERING_LUT__ATOMIC_MASK                                                                 0x00000400L
+#define SDMA0_RELAX_ORDERING_LUT__CONST_FILL_MASK                                                             0x00000800L
+#define SDMA0_RELAX_ORDERING_LUT__PTEPDE_MASK                                                                 0x00001000L
+#define SDMA0_RELAX_ORDERING_LUT__TIMESTAMP_MASK                                                              0x00002000L
+#define SDMA0_RELAX_ORDERING_LUT__RESERVED_MASK                                                               0x07FFC000L
+#define SDMA0_RELAX_ORDERING_LUT__WORLD_SWITCH_MASK                                                           0x08000000L
+#define SDMA0_RELAX_ORDERING_LUT__RPTR_WRB_MASK                                                               0x10000000L
+#define SDMA0_RELAX_ORDERING_LUT__WPTR_POLL_MASK                                                              0x20000000L
+#define SDMA0_RELAX_ORDERING_LUT__IB_FETCH_MASK                                                               0x40000000L
+#define SDMA0_RELAX_ORDERING_LUT__RB_FETCH_MASK                                                               0x80000000L
+//SDMA0_CHICKEN_BITS_2
+#define SDMA0_CHICKEN_BITS_2__F32_CMD_PROC_DELAY__SHIFT                                                       0x0
+#define SDMA0_CHICKEN_BITS_2__F32_SEND_POSTCODE_EN__SHIFT                                                     0x4
+#define SDMA0_CHICKEN_BITS_2__UCODE_BUF_DS_EN__SHIFT                                                          0x6
+#define SDMA0_CHICKEN_BITS_2__UCODE_SELFLOAD_THREAD_OVERLAP__SHIFT                                            0x7
+#define SDMA0_CHICKEN_BITS_2__WPTR_POLL_OUTSTANDING__SHIFT                                                    0x8
+#define SDMA0_CHICKEN_BITS_2__RESERVED_14_12__SHIFT                                                           0xc
+#define SDMA0_CHICKEN_BITS_2__RESERVED_15__SHIFT                                                              0xf
+#define SDMA0_CHICKEN_BITS_2__RB_FIFO_WATERMARK__SHIFT                                                        0x10
+#define SDMA0_CHICKEN_BITS_2__IB_FIFO_WATERMARK__SHIFT                                                        0x12
+#define SDMA0_CHICKEN_BITS_2__RESERVED_22_20__SHIFT                                                           0x14
+#define SDMA0_CHICKEN_BITS_2__CH_RD_WATERMARK__SHIFT                                                          0x17
+#define SDMA0_CHICKEN_BITS_2__CH_WR_WATERMARK__SHIFT                                                          0x19
+#define SDMA0_CHICKEN_BITS_2__CH_WR_WATERMARK_LSB__SHIFT                                                      0x1e
+#define SDMA0_CHICKEN_BITS_2__PIO_VFID_SOURCE__SHIFT                                                          0x1f
+#define SDMA0_CHICKEN_BITS_2__F32_CMD_PROC_DELAY_MASK                                                         0x0000000FL
+#define SDMA0_CHICKEN_BITS_2__F32_SEND_POSTCODE_EN_MASK                                                       0x00000010L
+#define SDMA0_CHICKEN_BITS_2__UCODE_BUF_DS_EN_MASK                                                            0x00000040L
+#define SDMA0_CHICKEN_BITS_2__UCODE_SELFLOAD_THREAD_OVERLAP_MASK                                              0x00000080L
+#define SDMA0_CHICKEN_BITS_2__WPTR_POLL_OUTSTANDING_MASK                                                      0x00000F00L
+#define SDMA0_CHICKEN_BITS_2__RESERVED_14_12_MASK                                                             0x00007000L
+#define SDMA0_CHICKEN_BITS_2__RESERVED_15_MASK                                                                0x00008000L
+#define SDMA0_CHICKEN_BITS_2__RB_FIFO_WATERMARK_MASK                                                          0x00030000L
+#define SDMA0_CHICKEN_BITS_2__IB_FIFO_WATERMARK_MASK                                                          0x000C0000L
+#define SDMA0_CHICKEN_BITS_2__RESERVED_22_20_MASK                                                             0x00700000L
+#define SDMA0_CHICKEN_BITS_2__CH_RD_WATERMARK_MASK                                                            0x01800000L
+#define SDMA0_CHICKEN_BITS_2__CH_WR_WATERMARK_MASK                                                            0x3E000000L
+#define SDMA0_CHICKEN_BITS_2__CH_WR_WATERMARK_LSB_MASK                                                        0x40000000L
+#define SDMA0_CHICKEN_BITS_2__PIO_VFID_SOURCE_MASK                                                            0x80000000L
+//SDMA0_STATUS3_REG
+#define SDMA0_STATUS3_REG__CMD_OP_STATUS__SHIFT                                                               0x0
+#define SDMA0_STATUS3_REG__PREV_VM_CMD__SHIFT                                                                 0x10
+#define SDMA0_STATUS3_REG__EXCEPTION_IDLE__SHIFT                                                              0x14
+#define SDMA0_STATUS3_REG__AQL_PREV_CMD_IDLE__SHIFT                                                           0x15
+#define SDMA0_STATUS3_REG__TLBI_IDLE__SHIFT                                                                   0x16
+#define SDMA0_STATUS3_REG__GCR_IDLE__SHIFT                                                                    0x17
+#define SDMA0_STATUS3_REG__INVREQ_IDLE__SHIFT                                                                 0x18
+#define SDMA0_STATUS3_REG__QUEUE_ID_MATCH__SHIFT                                                              0x19
+#define SDMA0_STATUS3_REG__INT_QUEUE_ID__SHIFT                                                                0x1a
+#define SDMA0_STATUS3_REG__TMZ_MTYPE_STATUS__SHIFT                                                            0x1e
+#define SDMA0_STATUS3_REG__CMD_OP_STATUS_MASK                                                                 0x0000FFFFL
+#define SDMA0_STATUS3_REG__PREV_VM_CMD_MASK                                                                   0x000F0000L
+#define SDMA0_STATUS3_REG__EXCEPTION_IDLE_MASK                                                                0x00100000L
+#define SDMA0_STATUS3_REG__AQL_PREV_CMD_IDLE_MASK                                                             0x00200000L
+#define SDMA0_STATUS3_REG__TLBI_IDLE_MASK                                                                     0x00400000L
+#define SDMA0_STATUS3_REG__GCR_IDLE_MASK                                                                      0x00800000L
+#define SDMA0_STATUS3_REG__INVREQ_IDLE_MASK                                                                   0x01000000L
+#define SDMA0_STATUS3_REG__QUEUE_ID_MATCH_MASK                                                                0x02000000L
+#define SDMA0_STATUS3_REG__INT_QUEUE_ID_MASK                                                                  0x3C000000L
+#define SDMA0_STATUS3_REG__TMZ_MTYPE_STATUS_MASK                                                              0xC0000000L
+//SDMA0_PHYSICAL_ADDR_LO
+#define SDMA0_PHYSICAL_ADDR_LO__D_VALID__SHIFT                                                                0x0
+#define SDMA0_PHYSICAL_ADDR_LO__DIRTY__SHIFT                                                                  0x1
+#define SDMA0_PHYSICAL_ADDR_LO__PHY_VALID__SHIFT                                                              0x2
+#define SDMA0_PHYSICAL_ADDR_LO__ADDR__SHIFT                                                                   0xc
+#define SDMA0_PHYSICAL_ADDR_LO__D_VALID_MASK                                                                  0x00000001L
+#define SDMA0_PHYSICAL_ADDR_LO__DIRTY_MASK                                                                    0x00000002L
+#define SDMA0_PHYSICAL_ADDR_LO__PHY_VALID_MASK                                                                0x00000004L
+#define SDMA0_PHYSICAL_ADDR_LO__ADDR_MASK                                                                     0xFFFFF000L
+//SDMA0_PHYSICAL_ADDR_HI
+#define SDMA0_PHYSICAL_ADDR_HI__ADDR__SHIFT                                                                   0x0
+#define SDMA0_PHYSICAL_ADDR_HI__ADDR_MASK                                                                     0x0000FFFFL
+//SDMA0_GLOBAL_QUANTUM
+#define SDMA0_GLOBAL_QUANTUM__GLOBAL_FOCUS_QUANTUM__SHIFT                                                     0x0
+#define SDMA0_GLOBAL_QUANTUM__GLOBAL_NORMAL_QUANTUM__SHIFT                                                    0x8
+#define SDMA0_GLOBAL_QUANTUM__GLOBAL_FOCUS_QUANTUM_MASK                                                       0x000000FFL
+#define SDMA0_GLOBAL_QUANTUM__GLOBAL_NORMAL_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA0_ERROR_LOG
+#define SDMA0_ERROR_LOG__OVERRIDE__SHIFT                                                                      0x0
+#define SDMA0_ERROR_LOG__STATUS__SHIFT                                                                        0x10
+#define SDMA0_ERROR_LOG__OVERRIDE_MASK                                                                        0x0000FFFFL
+#define SDMA0_ERROR_LOG__STATUS_MASK                                                                          0xFFFF0000L
+//SDMA0_PUB_DUMMY_REG0
+#define SDMA0_PUB_DUMMY_REG0__VALUE__SHIFT                                                                    0x0
+#define SDMA0_PUB_DUMMY_REG0__VALUE_MASK                                                                      0xFFFFFFFFL
+//SDMA0_PUB_DUMMY_REG1
+#define SDMA0_PUB_DUMMY_REG1__VALUE__SHIFT                                                                    0x0
+#define SDMA0_PUB_DUMMY_REG1__VALUE_MASK                                                                      0xFFFFFFFFL
+//SDMA0_PUB_DUMMY_REG2
+#define SDMA0_PUB_DUMMY_REG2__VALUE__SHIFT                                                                    0x0
+#define SDMA0_PUB_DUMMY_REG2__VALUE_MASK                                                                      0xFFFFFFFFL
+//SDMA0_PUB_DUMMY_REG3
+#define SDMA0_PUB_DUMMY_REG3__VALUE__SHIFT                                                                    0x0
+#define SDMA0_PUB_DUMMY_REG3__VALUE_MASK                                                                      0xFFFFFFFFL
+//SDMA0_F32_COUNTER
+#define SDMA0_F32_COUNTER__VALUE__SHIFT                                                                       0x0
+#define SDMA0_F32_COUNTER__VALUE_MASK                                                                         0xFFFFFFFFL
+//SDMA0_CRD_CNTL
+#define SDMA0_CRD_CNTL__DRM_CREDIT__SHIFT                                                                     0x0
+#define SDMA0_CRD_CNTL__MC_WRREQ_CREDIT__SHIFT                                                                0x7
+#define SDMA0_CRD_CNTL__MC_RDREQ_CREDIT__SHIFT                                                                0xd
+#define SDMA0_CRD_CNTL__CH_WRREQ_CREDIT__SHIFT                                                                0x13
+#define SDMA0_CRD_CNTL__CH_RDREQ_CREDIT__SHIFT                                                                0x19
+#define SDMA0_CRD_CNTL__DRM_CREDIT_MASK                                                                       0x0000007FL
+#define SDMA0_CRD_CNTL__MC_WRREQ_CREDIT_MASK                                                                  0x00001F80L
+#define SDMA0_CRD_CNTL__MC_RDREQ_CREDIT_MASK                                                                  0x0007E000L
+#define SDMA0_CRD_CNTL__CH_WRREQ_CREDIT_MASK                                                                  0x01F80000L
+#define SDMA0_CRD_CNTL__CH_RDREQ_CREDIT_MASK                                                                  0x7E000000L
+//SDMA0_RLC_CGCG_CTRL
+#define SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE__SHIFT                                                           0x1
+#define SDMA0_RLC_CGCG_CTRL__CGCG_IDLE_HYSTERESIS__SHIFT                                                      0x10
+#define SDMA0_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK                                                             0x00000002L
+#define SDMA0_RLC_CGCG_CTRL__CGCG_IDLE_HYSTERESIS_MASK                                                        0xFFFF0000L
+//SDMA0_GPU_IOV_VIOLATION_LOG
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                  0x0
+#define SDMA0_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                         0x1
+#define SDMA0_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                           0x2
+#define SDMA0_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION__SHIFT                                                   0x14
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                                0x15
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VFID__SHIFT                                                              0x16
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                    0x00000001L
+#define SDMA0_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                           0x00000002L
+#define SDMA0_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                             0x000FFFFCL
+#define SDMA0_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION_MASK                                                     0x00100000L
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VF_MASK                                                                  0x00200000L
+#define SDMA0_GPU_IOV_VIOLATION_LOG__VFID_MASK                                                                0x03C00000L
+//SDMA0_AQL_STATUS
+#define SDMA0_AQL_STATUS__COMPLETE_SIGNAL_EMPTY__SHIFT                                                        0x0
+#define SDMA0_AQL_STATUS__INVALID_CMD_EMPTY__SHIFT                                                            0x1
+#define SDMA0_AQL_STATUS__COMPLETE_SIGNAL_EMPTY_MASK                                                          0x00000001L
+#define SDMA0_AQL_STATUS__INVALID_CMD_EMPTY_MASK                                                              0x00000002L
+//SDMA0_EA_DBIT_ADDR_DATA
+#define SDMA0_EA_DBIT_ADDR_DATA__VALUE__SHIFT                                                                 0x0
+#define SDMA0_EA_DBIT_ADDR_DATA__VALUE_MASK                                                                   0xFFFFFFFFL
+//SDMA0_EA_DBIT_ADDR_INDEX
+#define SDMA0_EA_DBIT_ADDR_INDEX__VALUE__SHIFT                                                                0x0
+#define SDMA0_EA_DBIT_ADDR_INDEX__VALUE_MASK                                                                  0x00000007L
+//SDMA0_TLBI_GCR_CNTL
+#define SDMA0_TLBI_GCR_CNTL__TLBI_CMD_DW__SHIFT                                                               0x0
+#define SDMA0_TLBI_GCR_CNTL__GCR_CMD_DW__SHIFT                                                                0x4
+#define SDMA0_TLBI_GCR_CNTL__GCR_CLKEN_CYCLE__SHIFT                                                           0x8
+#define SDMA0_TLBI_GCR_CNTL__TLBI_CREDIT__SHIFT                                                               0x10
+#define SDMA0_TLBI_GCR_CNTL__GCR_CREDIT__SHIFT                                                                0x18
+#define SDMA0_TLBI_GCR_CNTL__TLBI_CMD_DW_MASK                                                                 0x0000000FL
+#define SDMA0_TLBI_GCR_CNTL__GCR_CMD_DW_MASK                                                                  0x000000F0L
+#define SDMA0_TLBI_GCR_CNTL__GCR_CLKEN_CYCLE_MASK                                                             0x00000F00L
+#define SDMA0_TLBI_GCR_CNTL__TLBI_CREDIT_MASK                                                                 0x00FF0000L
+#define SDMA0_TLBI_GCR_CNTL__GCR_CREDIT_MASK                                                                  0xFF000000L
+//SDMA0_TILING_CONFIG
+#define SDMA0_TILING_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                      0x4
+#define SDMA0_TILING_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                        0x00000070L
+//SDMA0_HASH
+#define SDMA0_HASH__CHANNEL_BITS__SHIFT                                                                       0x0
+#define SDMA0_HASH__BANK_BITS__SHIFT                                                                          0x4
+#define SDMA0_HASH__CHANNEL_XOR_COUNT__SHIFT                                                                  0x8
+#define SDMA0_HASH__BANK_XOR_COUNT__SHIFT                                                                     0xc
+#define SDMA0_HASH__CHANNEL_BITS_MASK                                                                         0x00000007L
+#define SDMA0_HASH__BANK_BITS_MASK                                                                            0x00000070L
+#define SDMA0_HASH__CHANNEL_XOR_COUNT_MASK                                                                    0x00000700L
+#define SDMA0_HASH__BANK_XOR_COUNT_MASK                                                                       0x00007000L
+//SDMA0_INT_STATUS
+#define SDMA0_INT_STATUS__DATA__SHIFT                                                                         0x0
+#define SDMA0_INT_STATUS__DATA_MASK                                                                           0xFFFFFFFFL
+//SDMA0_GPU_IOV_VIOLATION_LOG2
+#define SDMA0_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID__SHIFT                                                     0x0
+#define SDMA0_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID_MASK                                                       0x000003FFL
+//SDMA0_HOLE_ADDR_LO
+#define SDMA0_HOLE_ADDR_LO__VALUE__SHIFT                                                                      0x0
+#define SDMA0_HOLE_ADDR_LO__VALUE_MASK                                                                        0xFFFFFFFFL
+//SDMA0_HOLE_ADDR_HI
+#define SDMA0_HOLE_ADDR_HI__VALUE__SHIFT                                                                      0x0
+#define SDMA0_HOLE_ADDR_HI__VALUE_MASK                                                                        0xFFFFFFFFL
+//SDMA0_CLOCK_GATING_STATUS
+#define SDMA0_CLOCK_GATING_STATUS__DYN_CLK_GATE_STATUS__SHIFT                                                 0x0
+#define SDMA0_CLOCK_GATING_STATUS__CE_CLK_GATE_STATUS__SHIFT                                                  0x2
+#define SDMA0_CLOCK_GATING_STATUS__CE_BC_CLK_GATE_STATUS__SHIFT                                               0x3
+#define SDMA0_CLOCK_GATING_STATUS__CE_NBC_CLK_GATE_STATUS__SHIFT                                              0x4
+#define SDMA0_CLOCK_GATING_STATUS__REG_CLK_GATE_STATUS__SHIFT                                                 0x5
+#define SDMA0_CLOCK_GATING_STATUS__F32_CLK_GATE_STATUS__SHIFT                                                 0x6
+#define SDMA0_CLOCK_GATING_STATUS__DYN_CLK_GATE_STATUS_MASK                                                   0x00000001L
+#define SDMA0_CLOCK_GATING_STATUS__CE_CLK_GATE_STATUS_MASK                                                    0x00000004L
+#define SDMA0_CLOCK_GATING_STATUS__CE_BC_CLK_GATE_STATUS_MASK                                                 0x00000008L
+#define SDMA0_CLOCK_GATING_STATUS__CE_NBC_CLK_GATE_STATUS_MASK                                                0x00000010L
+#define SDMA0_CLOCK_GATING_STATUS__REG_CLK_GATE_STATUS_MASK                                                   0x00000020L
+#define SDMA0_CLOCK_GATING_STATUS__F32_CLK_GATE_STATUS_MASK                                                   0x00000040L
+//SDMA0_STATUS4_REG
+#define SDMA0_STATUS4_REG__IDLE__SHIFT                                                                        0x0
+#define SDMA0_STATUS4_REG__IH_OUTSTANDING__SHIFT                                                              0x2
+#define SDMA0_STATUS4_REG__SEM_OUTSTANDING__SHIFT                                                             0x3
+#define SDMA0_STATUS4_REG__CH_RD_OUTSTANDING__SHIFT                                                           0x4
+#define SDMA0_STATUS4_REG__CH_WR_OUTSTANDING__SHIFT                                                           0x5
+#define SDMA0_STATUS4_REG__GCR_OUTSTANDING__SHIFT                                                             0x6
+#define SDMA0_STATUS4_REG__TLBI_OUTSTANDING__SHIFT                                                            0x7
+#define SDMA0_STATUS4_REG__UTCL2_RD_OUTSTANDING__SHIFT                                                        0x8
+#define SDMA0_STATUS4_REG__UTCL2_WR_OUTSTANDING__SHIFT                                                        0x9
+#define SDMA0_STATUS4_REG__REG_POLLING__SHIFT                                                                 0xa
+#define SDMA0_STATUS4_REG__MEM_POLLING__SHIFT                                                                 0xb
+#define SDMA0_STATUS4_REG__RESERVED_13_12__SHIFT                                                              0xc
+#define SDMA0_STATUS4_REG__RESERVED_15_14__SHIFT                                                              0xe
+#define SDMA0_STATUS4_REG__ACTIVE_QUEUE_ID__SHIFT                                                             0x10
+#define SDMA0_STATUS4_REG__SRIOV_WATING_RLCV_CMD__SHIFT                                                       0x14
+#define SDMA0_STATUS4_REG__SRIOV_SDMA_EXECUTING_CMD__SHIFT                                                    0x15
+#define SDMA0_STATUS4_REG__UTCL2_RD_XNACK_FAULT__SHIFT                                                        0x16
+#define SDMA0_STATUS4_REG__UTCL2_RD_XNACK_NULL__SHIFT                                                         0x17
+#define SDMA0_STATUS4_REG__UTCL2_RD_XNACK_TIMEOUT__SHIFT                                                      0x18
+#define SDMA0_STATUS4_REG__UTCL2_WR_XNACK_FAULT__SHIFT                                                        0x19
+#define SDMA0_STATUS4_REG__UTCL2_WR_XNACK_NULL__SHIFT                                                         0x1a
+#define SDMA0_STATUS4_REG__UTCL2_WR_XNACK_TIMEOUT__SHIFT                                                      0x1b
+#define SDMA0_STATUS4_REG__IDLE_MASK                                                                          0x00000001L
+#define SDMA0_STATUS4_REG__IH_OUTSTANDING_MASK                                                                0x00000004L
+#define SDMA0_STATUS4_REG__SEM_OUTSTANDING_MASK                                                               0x00000008L
+#define SDMA0_STATUS4_REG__CH_RD_OUTSTANDING_MASK                                                             0x00000010L
+#define SDMA0_STATUS4_REG__CH_WR_OUTSTANDING_MASK                                                             0x00000020L
+#define SDMA0_STATUS4_REG__GCR_OUTSTANDING_MASK                                                               0x00000040L
+#define SDMA0_STATUS4_REG__TLBI_OUTSTANDING_MASK                                                              0x00000080L
+#define SDMA0_STATUS4_REG__UTCL2_RD_OUTSTANDING_MASK                                                          0x00000100L
+#define SDMA0_STATUS4_REG__UTCL2_WR_OUTSTANDING_MASK                                                          0x00000200L
+#define SDMA0_STATUS4_REG__REG_POLLING_MASK                                                                   0x00000400L
+#define SDMA0_STATUS4_REG__MEM_POLLING_MASK                                                                   0x00000800L
+#define SDMA0_STATUS4_REG__RESERVED_13_12_MASK                                                                0x00003000L
+#define SDMA0_STATUS4_REG__RESERVED_15_14_MASK                                                                0x0000C000L
+#define SDMA0_STATUS4_REG__ACTIVE_QUEUE_ID_MASK                                                               0x000F0000L
+#define SDMA0_STATUS4_REG__SRIOV_WATING_RLCV_CMD_MASK                                                         0x00100000L
+#define SDMA0_STATUS4_REG__SRIOV_SDMA_EXECUTING_CMD_MASK                                                      0x00200000L
+#define SDMA0_STATUS4_REG__UTCL2_RD_XNACK_FAULT_MASK                                                          0x00400000L
+#define SDMA0_STATUS4_REG__UTCL2_RD_XNACK_NULL_MASK                                                           0x00800000L
+#define SDMA0_STATUS4_REG__UTCL2_RD_XNACK_TIMEOUT_MASK                                                        0x01000000L
+#define SDMA0_STATUS4_REG__UTCL2_WR_XNACK_FAULT_MASK                                                          0x02000000L
+#define SDMA0_STATUS4_REG__UTCL2_WR_XNACK_NULL_MASK                                                           0x04000000L
+#define SDMA0_STATUS4_REG__UTCL2_WR_XNACK_TIMEOUT_MASK                                                        0x08000000L
+//SDMA0_SCRATCH_RAM_DATA
+#define SDMA0_SCRATCH_RAM_DATA__DATA__SHIFT                                                                   0x0
+#define SDMA0_SCRATCH_RAM_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//SDMA0_SCRATCH_RAM_ADDR
+#define SDMA0_SCRATCH_RAM_ADDR__ADDR__SHIFT                                                                   0x0
+#define SDMA0_SCRATCH_RAM_ADDR__ADDR_MASK                                                                     0x0000007FL
+//SDMA0_TIMESTAMP_CNTL
+#define SDMA0_TIMESTAMP_CNTL__CAPTURE__SHIFT                                                                  0x0
+#define SDMA0_TIMESTAMP_CNTL__CAPTURE_MASK                                                                    0x00000001L
+//SDMA0_STATUS5_REG
+#define SDMA0_STATUS5_REG__QUEUE0_RB_ENABLE_STATUS__SHIFT                                                     0x0
+#define SDMA0_STATUS5_REG__QUEUE1_RB_ENABLE_STATUS__SHIFT                                                     0x1
+#define SDMA0_STATUS5_REG__QUEUE2_RB_ENABLE_STATUS__SHIFT                                                     0x2
+#define SDMA0_STATUS5_REG__QUEUE3_RB_ENABLE_STATUS__SHIFT                                                     0x3
+#define SDMA0_STATUS5_REG__QUEUE4_RB_ENABLE_STATUS__SHIFT                                                     0x4
+#define SDMA0_STATUS5_REG__QUEUE5_RB_ENABLE_STATUS__SHIFT                                                     0x5
+#define SDMA0_STATUS5_REG__QUEUE6_RB_ENABLE_STATUS__SHIFT                                                     0x6
+#define SDMA0_STATUS5_REG__QUEUE7_RB_ENABLE_STATUS__SHIFT                                                     0x7
+#define SDMA0_STATUS5_REG__ACTIVE_QUEUE_ID__SHIFT                                                             0x10
+#define SDMA0_STATUS5_REG__QUEUE0_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x14
+#define SDMA0_STATUS5_REG__QUEUE1_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x15
+#define SDMA0_STATUS5_REG__QUEUE2_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x16
+#define SDMA0_STATUS5_REG__QUEUE3_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x17
+#define SDMA0_STATUS5_REG__QUEUE4_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x18
+#define SDMA0_STATUS5_REG__QUEUE5_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x19
+#define SDMA0_STATUS5_REG__QUEUE6_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x1a
+#define SDMA0_STATUS5_REG__QUEUE7_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x1b
+#define SDMA0_STATUS5_REG__QUEUE0_RB_ENABLE_STATUS_MASK                                                       0x00000001L
+#define SDMA0_STATUS5_REG__QUEUE1_RB_ENABLE_STATUS_MASK                                                       0x00000002L
+#define SDMA0_STATUS5_REG__QUEUE2_RB_ENABLE_STATUS_MASK                                                       0x00000004L
+#define SDMA0_STATUS5_REG__QUEUE3_RB_ENABLE_STATUS_MASK                                                       0x00000008L
+#define SDMA0_STATUS5_REG__QUEUE4_RB_ENABLE_STATUS_MASK                                                       0x00000010L
+#define SDMA0_STATUS5_REG__QUEUE5_RB_ENABLE_STATUS_MASK                                                       0x00000020L
+#define SDMA0_STATUS5_REG__QUEUE6_RB_ENABLE_STATUS_MASK                                                       0x00000040L
+#define SDMA0_STATUS5_REG__QUEUE7_RB_ENABLE_STATUS_MASK                                                       0x00000080L
+#define SDMA0_STATUS5_REG__ACTIVE_QUEUE_ID_MASK                                                               0x000F0000L
+#define SDMA0_STATUS5_REG__QUEUE0_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x00100000L
+#define SDMA0_STATUS5_REG__QUEUE1_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x00200000L
+#define SDMA0_STATUS5_REG__QUEUE2_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x00400000L
+#define SDMA0_STATUS5_REG__QUEUE3_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x00800000L
+#define SDMA0_STATUS5_REG__QUEUE4_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x01000000L
+#define SDMA0_STATUS5_REG__QUEUE5_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x02000000L
+#define SDMA0_STATUS5_REG__QUEUE6_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x04000000L
+#define SDMA0_STATUS5_REG__QUEUE7_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x08000000L
+//SDMA0_QUEUE_RESET_REQ
+#define SDMA0_QUEUE_RESET_REQ__QUEUE0_RESET__SHIFT                                                            0x0
+#define SDMA0_QUEUE_RESET_REQ__QUEUE1_RESET__SHIFT                                                            0x1
+#define SDMA0_QUEUE_RESET_REQ__QUEUE2_RESET__SHIFT                                                            0x2
+#define SDMA0_QUEUE_RESET_REQ__QUEUE3_RESET__SHIFT                                                            0x3
+#define SDMA0_QUEUE_RESET_REQ__QUEUE4_RESET__SHIFT                                                            0x4
+#define SDMA0_QUEUE_RESET_REQ__QUEUE5_RESET__SHIFT                                                            0x5
+#define SDMA0_QUEUE_RESET_REQ__QUEUE6_RESET__SHIFT                                                            0x6
+#define SDMA0_QUEUE_RESET_REQ__QUEUE7_RESET__SHIFT                                                            0x7
+#define SDMA0_QUEUE_RESET_REQ__RESERVED__SHIFT                                                                0x8
+#define SDMA0_QUEUE_RESET_REQ__QUEUE0_RESET_MASK                                                              0x00000001L
+#define SDMA0_QUEUE_RESET_REQ__QUEUE1_RESET_MASK                                                              0x00000002L
+#define SDMA0_QUEUE_RESET_REQ__QUEUE2_RESET_MASK                                                              0x00000004L
+#define SDMA0_QUEUE_RESET_REQ__QUEUE3_RESET_MASK                                                              0x00000008L
+#define SDMA0_QUEUE_RESET_REQ__QUEUE4_RESET_MASK                                                              0x00000010L
+#define SDMA0_QUEUE_RESET_REQ__QUEUE5_RESET_MASK                                                              0x00000020L
+#define SDMA0_QUEUE_RESET_REQ__QUEUE6_RESET_MASK                                                              0x00000040L
+#define SDMA0_QUEUE_RESET_REQ__QUEUE7_RESET_MASK                                                              0x00000080L
+#define SDMA0_QUEUE_RESET_REQ__RESERVED_MASK                                                                  0xFFFFFF00L
+//SDMA0_STATUS6_REG
+#define SDMA0_STATUS6_REG__ID__SHIFT                                                                          0x0
+#define SDMA0_STATUS6_REG__TH1F32_INSTR_PTR__SHIFT                                                            0x2
+#define SDMA0_STATUS6_REG__TH1_EXCEPTION__SHIFT                                                               0x10
+#define SDMA0_STATUS6_REG__ID_MASK                                                                            0x00000003L
+#define SDMA0_STATUS6_REG__TH1F32_INSTR_PTR_MASK                                                              0x0000FFFCL
+#define SDMA0_STATUS6_REG__TH1_EXCEPTION_MASK                                                                 0xFFFF0000L
+//SDMA0_UCODE1_CHECKSUM
+#define SDMA0_UCODE1_CHECKSUM__DATA__SHIFT                                                                    0x0
+#define SDMA0_UCODE1_CHECKSUM__DATA_MASK                                                                      0xFFFFFFFFL
+//SDMA0_CE_CTRL
+#define SDMA0_CE_CTRL__RD_LUT_WATERMARK__SHIFT                                                                0x0
+#define SDMA0_CE_CTRL__RD_LUT_DEPTH__SHIFT                                                                    0x3
+#define SDMA0_CE_CTRL__WR_AFIFO_WATERMARK__SHIFT                                                              0x5
+#define SDMA0_CE_CTRL__CE_DCC_READ_128B_ENABLE__SHIFT                                                         0x8
+#define SDMA0_CE_CTRL__RESERVED__SHIFT                                                                        0x9
+#define SDMA0_CE_CTRL__RD_LUT_WATERMARK_MASK                                                                  0x00000007L
+#define SDMA0_CE_CTRL__RD_LUT_DEPTH_MASK                                                                      0x00000018L
+#define SDMA0_CE_CTRL__WR_AFIFO_WATERMARK_MASK                                                                0x000000E0L
+#define SDMA0_CE_CTRL__CE_DCC_READ_128B_ENABLE_MASK                                                           0x00000100L
+#define SDMA0_CE_CTRL__RESERVED_MASK                                                                          0xFFFFFE00L
+//SDMA0_FED_STATUS
+#define SDMA0_FED_STATUS__RB_FETCH_ECC__SHIFT                                                                 0x0
+#define SDMA0_FED_STATUS__IB_FETCH_ECC__SHIFT                                                                 0x1
+#define SDMA0_FED_STATUS__F32_DATA_ECC__SHIFT                                                                 0x2
+#define SDMA0_FED_STATUS__WPTR_ATOMIC_ECC__SHIFT                                                              0x3
+#define SDMA0_FED_STATUS__COPY_DATA_ECC__SHIFT                                                                0x4
+#define SDMA0_FED_STATUS__COPY_METADATA_ECC__SHIFT                                                            0x5
+#define SDMA0_FED_STATUS__SELFLOAD_UCODE_ECC__SHIFT                                                           0x6
+#define SDMA0_FED_STATUS__RB_FETCH_ECC_MASK                                                                   0x00000001L
+#define SDMA0_FED_STATUS__IB_FETCH_ECC_MASK                                                                   0x00000002L
+#define SDMA0_FED_STATUS__F32_DATA_ECC_MASK                                                                   0x00000004L
+#define SDMA0_FED_STATUS__WPTR_ATOMIC_ECC_MASK                                                                0x00000008L
+#define SDMA0_FED_STATUS__COPY_DATA_ECC_MASK                                                                  0x00000010L
+#define SDMA0_FED_STATUS__COPY_METADATA_ECC_MASK                                                              0x00000020L
+#define SDMA0_FED_STATUS__SELFLOAD_UCODE_ECC_MASK                                                             0x00000040L
+//SDMA0_QUEUE0_RB_CNTL
+#define SDMA0_QUEUE0_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE0_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA0_QUEUE0_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA0_QUEUE0_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA0_QUEUE0_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA0_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA0_QUEUE0_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA0_QUEUE0_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA0_QUEUE0_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE0_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA0_QUEUE0_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA0_QUEUE0_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA0_QUEUE0_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA0_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA0_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA0_QUEUE0_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA0_QUEUE0_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA0_QUEUE0_RB_BASE
+#define SDMA0_QUEUE0_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA0_QUEUE0_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA0_QUEUE0_RB_BASE_HI
+#define SDMA0_QUEUE0_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE0_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA0_QUEUE0_RB_RPTR
+#define SDMA0_QUEUE0_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE0_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE0_RB_RPTR_HI
+#define SDMA0_QUEUE0_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE0_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE0_RB_WPTR
+#define SDMA0_QUEUE0_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE0_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE0_RB_WPTR_HI
+#define SDMA0_QUEUE0_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE0_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE0_RB_RPTR_ADDR_HI
+#define SDMA0_QUEUE0_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_QUEUE0_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE0_RB_RPTR_ADDR_LO
+#define SDMA0_QUEUE0_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_QUEUE0_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_QUEUE0_IB_CNTL
+#define SDMA0_QUEUE0_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE0_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA0_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA0_QUEUE0_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA0_QUEUE0_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA0_QUEUE0_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE0_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA0_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA0_QUEUE0_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA0_QUEUE0_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA0_QUEUE0_IB_RPTR
+#define SDMA0_QUEUE0_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA0_QUEUE0_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA0_QUEUE0_IB_OFFSET
+#define SDMA0_QUEUE0_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA0_QUEUE0_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA0_QUEUE0_IB_BASE_LO
+#define SDMA0_QUEUE0_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA0_QUEUE0_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA0_QUEUE0_IB_BASE_HI
+#define SDMA0_QUEUE0_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE0_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE0_IB_SIZE
+#define SDMA0_QUEUE0_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA0_QUEUE0_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA0_QUEUE0_SKIP_CNTL
+#define SDMA0_QUEUE0_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA0_QUEUE0_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA0_QUEUE0_CONTEXT_STATUS
+#define SDMA0_QUEUE0_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA0_QUEUE0_CONTEXT_STATUS__USE_IB__SHIFT                                                            0x1
+#define SDMA0_QUEUE0_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA0_QUEUE0_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA0_QUEUE0_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA0_QUEUE0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA0_QUEUE0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA0_QUEUE0_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA0_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA0_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA0_QUEUE0_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA0_QUEUE0_CONTEXT_STATUS__USE_IB_MASK                                                              0x00000002L
+#define SDMA0_QUEUE0_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA0_QUEUE0_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA0_QUEUE0_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA0_QUEUE0_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA0_QUEUE0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA0_QUEUE0_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA0_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA0_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA0_QUEUE0_DOORBELL
+#define SDMA0_QUEUE0_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA0_QUEUE0_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA0_QUEUE0_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA0_QUEUE0_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA0_QUEUE0_DOORBELL_LOG
+#define SDMA0_QUEUE0_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA0_QUEUE0_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA0_QUEUE0_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA0_QUEUE0_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA0_QUEUE0_DOORBELL_OFFSET
+#define SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA0_QUEUE0_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA0_QUEUE0_CSA_ADDR_LO
+#define SDMA0_QUEUE0_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA0_QUEUE0_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA0_QUEUE0_CSA_ADDR_HI
+#define SDMA0_QUEUE0_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA0_QUEUE0_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA0_QUEUE0_SCHEDULE_CNTL
+#define SDMA0_QUEUE0_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA0_QUEUE0_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA0_QUEUE0_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA0_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA0_QUEUE0_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA0_QUEUE0_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA0_QUEUE0_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA0_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA0_QUEUE0_IB_SUB_REMAIN
+#define SDMA0_QUEUE0_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA0_QUEUE0_PREEMPT
+#define SDMA0_QUEUE0_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA0_QUEUE0_DUMMY_REG
+#define SDMA0_QUEUE0_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA0_QUEUE0_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_QUEUE0_RB_AQL_CNTL
+#define SDMA0_QUEUE0_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA0_QUEUE0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA0_QUEUE0_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA0_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA0_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA0_QUEUE0_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA0_QUEUE0_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_QUEUE0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA0_QUEUE0_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA0_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA0_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA0_QUEUE0_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA0_QUEUE0_MINOR_PTR_UPDATE
+#define SDMA0_QUEUE0_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA0_QUEUE0_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA0_QUEUE0_RB_PREEMPT
+#define SDMA0_QUEUE0_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA0_QUEUE0_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA0_QUEUE0_MIDCMD_DATA0
+#define SDMA0_QUEUE0_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_DATA1
+#define SDMA0_QUEUE0_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_DATA2
+#define SDMA0_QUEUE0_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_DATA3
+#define SDMA0_QUEUE0_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_DATA4
+#define SDMA0_QUEUE0_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_DATA5
+#define SDMA0_QUEUE0_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_DATA6
+#define SDMA0_QUEUE0_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_DATA7
+#define SDMA0_QUEUE0_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_DATA8
+#define SDMA0_QUEUE0_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_DATA9
+#define SDMA0_QUEUE0_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_DATA10
+#define SDMA0_QUEUE0_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA0_QUEUE0_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE0_MIDCMD_CNTL
+#define SDMA0_QUEUE0_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA0_QUEUE0_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA0_QUEUE0_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA0_QUEUE0_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA0_QUEUE0_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA0_QUEUE0_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA0_QUEUE0_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA0_QUEUE0_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA0_QUEUE1_RB_CNTL
+#define SDMA0_QUEUE1_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE1_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA0_QUEUE1_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA0_QUEUE1_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA0_QUEUE1_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA0_QUEUE1_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA0_QUEUE1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA0_QUEUE1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA0_QUEUE1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA0_QUEUE1_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA0_QUEUE1_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA0_QUEUE1_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE1_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA0_QUEUE1_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA0_QUEUE1_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA0_QUEUE1_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA0_QUEUE1_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA0_QUEUE1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA0_QUEUE1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA0_QUEUE1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA0_QUEUE1_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA0_QUEUE1_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA0_QUEUE1_RB_BASE
+#define SDMA0_QUEUE1_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA0_QUEUE1_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA0_QUEUE1_RB_BASE_HI
+#define SDMA0_QUEUE1_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE1_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA0_QUEUE1_RB_RPTR
+#define SDMA0_QUEUE1_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE1_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE1_RB_RPTR_HI
+#define SDMA0_QUEUE1_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE1_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE1_RB_WPTR
+#define SDMA0_QUEUE1_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE1_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE1_RB_WPTR_HI
+#define SDMA0_QUEUE1_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE1_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE1_RB_RPTR_ADDR_HI
+#define SDMA0_QUEUE1_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_QUEUE1_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE1_RB_RPTR_ADDR_LO
+#define SDMA0_QUEUE1_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_QUEUE1_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_QUEUE1_IB_CNTL
+#define SDMA0_QUEUE1_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE1_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA0_QUEUE1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA0_QUEUE1_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA0_QUEUE1_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA0_QUEUE1_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE1_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA0_QUEUE1_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA0_QUEUE1_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA0_QUEUE1_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA0_QUEUE1_IB_RPTR
+#define SDMA0_QUEUE1_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA0_QUEUE1_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA0_QUEUE1_IB_OFFSET
+#define SDMA0_QUEUE1_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA0_QUEUE1_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA0_QUEUE1_IB_BASE_LO
+#define SDMA0_QUEUE1_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA0_QUEUE1_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA0_QUEUE1_IB_BASE_HI
+#define SDMA0_QUEUE1_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE1_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE1_IB_SIZE
+#define SDMA0_QUEUE1_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA0_QUEUE1_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA0_QUEUE1_SKIP_CNTL
+#define SDMA0_QUEUE1_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA0_QUEUE1_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA0_QUEUE1_CONTEXT_STATUS
+#define SDMA0_QUEUE1_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA0_QUEUE1_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA0_QUEUE1_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA0_QUEUE1_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA0_QUEUE1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA0_QUEUE1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA0_QUEUE1_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA0_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA0_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA0_QUEUE1_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA0_QUEUE1_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA0_QUEUE1_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA0_QUEUE1_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA0_QUEUE1_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA0_QUEUE1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA0_QUEUE1_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA0_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA0_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA0_QUEUE1_DOORBELL
+#define SDMA0_QUEUE1_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA0_QUEUE1_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA0_QUEUE1_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA0_QUEUE1_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA0_QUEUE1_DOORBELL_LOG
+#define SDMA0_QUEUE1_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA0_QUEUE1_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA0_QUEUE1_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA0_QUEUE1_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA0_QUEUE1_DOORBELL_OFFSET
+#define SDMA0_QUEUE1_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA0_QUEUE1_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA0_QUEUE1_CSA_ADDR_LO
+#define SDMA0_QUEUE1_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA0_QUEUE1_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA0_QUEUE1_CSA_ADDR_HI
+#define SDMA0_QUEUE1_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA0_QUEUE1_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA0_QUEUE1_SCHEDULE_CNTL
+#define SDMA0_QUEUE1_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA0_QUEUE1_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA0_QUEUE1_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA0_QUEUE1_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA0_QUEUE1_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA0_QUEUE1_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA0_QUEUE1_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA0_QUEUE1_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA0_QUEUE1_IB_SUB_REMAIN
+#define SDMA0_QUEUE1_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA0_QUEUE1_PREEMPT
+#define SDMA0_QUEUE1_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA0_QUEUE1_DUMMY_REG
+#define SDMA0_QUEUE1_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA0_QUEUE1_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE1_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_QUEUE1_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_QUEUE1_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_QUEUE1_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_QUEUE1_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_QUEUE1_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_QUEUE1_RB_AQL_CNTL
+#define SDMA0_QUEUE1_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA0_QUEUE1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA0_QUEUE1_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA0_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA0_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA0_QUEUE1_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA0_QUEUE1_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_QUEUE1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA0_QUEUE1_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA0_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA0_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA0_QUEUE1_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA0_QUEUE1_MINOR_PTR_UPDATE
+#define SDMA0_QUEUE1_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA0_QUEUE1_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA0_QUEUE1_RB_PREEMPT
+#define SDMA0_QUEUE1_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA0_QUEUE1_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA0_QUEUE1_MIDCMD_DATA0
+#define SDMA0_QUEUE1_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_DATA1
+#define SDMA0_QUEUE1_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_DATA2
+#define SDMA0_QUEUE1_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_DATA3
+#define SDMA0_QUEUE1_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_DATA4
+#define SDMA0_QUEUE1_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_DATA5
+#define SDMA0_QUEUE1_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_DATA6
+#define SDMA0_QUEUE1_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_DATA7
+#define SDMA0_QUEUE1_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_DATA8
+#define SDMA0_QUEUE1_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_DATA9
+#define SDMA0_QUEUE1_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_DATA10
+#define SDMA0_QUEUE1_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA0_QUEUE1_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE1_MIDCMD_CNTL
+#define SDMA0_QUEUE1_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA0_QUEUE1_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA0_QUEUE1_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA0_QUEUE1_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA0_QUEUE1_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA0_QUEUE1_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA0_QUEUE1_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA0_QUEUE1_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA0_QUEUE2_RB_CNTL
+#define SDMA0_QUEUE2_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE2_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA0_QUEUE2_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA0_QUEUE2_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA0_QUEUE2_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA0_QUEUE2_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA0_QUEUE2_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA0_QUEUE2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA0_QUEUE2_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA0_QUEUE2_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA0_QUEUE2_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA0_QUEUE2_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE2_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA0_QUEUE2_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA0_QUEUE2_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA0_QUEUE2_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA0_QUEUE2_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA0_QUEUE2_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA0_QUEUE2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA0_QUEUE2_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA0_QUEUE2_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA0_QUEUE2_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA0_QUEUE2_RB_BASE
+#define SDMA0_QUEUE2_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA0_QUEUE2_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA0_QUEUE2_RB_BASE_HI
+#define SDMA0_QUEUE2_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE2_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA0_QUEUE2_RB_RPTR
+#define SDMA0_QUEUE2_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE2_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE2_RB_RPTR_HI
+#define SDMA0_QUEUE2_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE2_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE2_RB_WPTR
+#define SDMA0_QUEUE2_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE2_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE2_RB_WPTR_HI
+#define SDMA0_QUEUE2_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE2_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE2_RB_RPTR_ADDR_HI
+#define SDMA0_QUEUE2_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_QUEUE2_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE2_RB_RPTR_ADDR_LO
+#define SDMA0_QUEUE2_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_QUEUE2_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_QUEUE2_IB_CNTL
+#define SDMA0_QUEUE2_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE2_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA0_QUEUE2_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA0_QUEUE2_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA0_QUEUE2_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA0_QUEUE2_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE2_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA0_QUEUE2_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA0_QUEUE2_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA0_QUEUE2_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA0_QUEUE2_IB_RPTR
+#define SDMA0_QUEUE2_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA0_QUEUE2_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA0_QUEUE2_IB_OFFSET
+#define SDMA0_QUEUE2_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA0_QUEUE2_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA0_QUEUE2_IB_BASE_LO
+#define SDMA0_QUEUE2_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA0_QUEUE2_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA0_QUEUE2_IB_BASE_HI
+#define SDMA0_QUEUE2_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE2_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE2_IB_SIZE
+#define SDMA0_QUEUE2_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA0_QUEUE2_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA0_QUEUE2_SKIP_CNTL
+#define SDMA0_QUEUE2_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA0_QUEUE2_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA0_QUEUE2_CONTEXT_STATUS
+#define SDMA0_QUEUE2_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA0_QUEUE2_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA0_QUEUE2_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA0_QUEUE2_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA0_QUEUE2_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA0_QUEUE2_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA0_QUEUE2_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA0_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA0_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA0_QUEUE2_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA0_QUEUE2_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA0_QUEUE2_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA0_QUEUE2_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA0_QUEUE2_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA0_QUEUE2_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA0_QUEUE2_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA0_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA0_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA0_QUEUE2_DOORBELL
+#define SDMA0_QUEUE2_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA0_QUEUE2_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA0_QUEUE2_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA0_QUEUE2_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA0_QUEUE2_DOORBELL_LOG
+#define SDMA0_QUEUE2_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA0_QUEUE2_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA0_QUEUE2_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA0_QUEUE2_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA0_QUEUE2_DOORBELL_OFFSET
+#define SDMA0_QUEUE2_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA0_QUEUE2_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA0_QUEUE2_CSA_ADDR_LO
+#define SDMA0_QUEUE2_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA0_QUEUE2_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA0_QUEUE2_CSA_ADDR_HI
+#define SDMA0_QUEUE2_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA0_QUEUE2_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA0_QUEUE2_SCHEDULE_CNTL
+#define SDMA0_QUEUE2_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA0_QUEUE2_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA0_QUEUE2_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA0_QUEUE2_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA0_QUEUE2_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA0_QUEUE2_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA0_QUEUE2_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA0_QUEUE2_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA0_QUEUE2_IB_SUB_REMAIN
+#define SDMA0_QUEUE2_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA0_QUEUE2_PREEMPT
+#define SDMA0_QUEUE2_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA0_QUEUE2_DUMMY_REG
+#define SDMA0_QUEUE2_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA0_QUEUE2_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE2_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_QUEUE2_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_QUEUE2_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_QUEUE2_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_QUEUE2_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_QUEUE2_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_QUEUE2_RB_AQL_CNTL
+#define SDMA0_QUEUE2_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA0_QUEUE2_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA0_QUEUE2_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA0_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA0_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA0_QUEUE2_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA0_QUEUE2_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_QUEUE2_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA0_QUEUE2_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA0_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA0_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA0_QUEUE2_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA0_QUEUE2_MINOR_PTR_UPDATE
+#define SDMA0_QUEUE2_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA0_QUEUE2_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA0_QUEUE2_RB_PREEMPT
+#define SDMA0_QUEUE2_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA0_QUEUE2_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA0_QUEUE2_MIDCMD_DATA0
+#define SDMA0_QUEUE2_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_DATA1
+#define SDMA0_QUEUE2_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_DATA2
+#define SDMA0_QUEUE2_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_DATA3
+#define SDMA0_QUEUE2_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_DATA4
+#define SDMA0_QUEUE2_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_DATA5
+#define SDMA0_QUEUE2_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_DATA6
+#define SDMA0_QUEUE2_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_DATA7
+#define SDMA0_QUEUE2_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_DATA8
+#define SDMA0_QUEUE2_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_DATA9
+#define SDMA0_QUEUE2_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_DATA10
+#define SDMA0_QUEUE2_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA0_QUEUE2_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE2_MIDCMD_CNTL
+#define SDMA0_QUEUE2_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA0_QUEUE2_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA0_QUEUE2_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA0_QUEUE2_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA0_QUEUE2_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA0_QUEUE2_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA0_QUEUE2_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA0_QUEUE2_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA0_QUEUE3_RB_CNTL
+#define SDMA0_QUEUE3_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE3_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA0_QUEUE3_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA0_QUEUE3_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA0_QUEUE3_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA0_QUEUE3_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA0_QUEUE3_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA0_QUEUE3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA0_QUEUE3_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA0_QUEUE3_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA0_QUEUE3_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA0_QUEUE3_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE3_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA0_QUEUE3_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA0_QUEUE3_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA0_QUEUE3_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA0_QUEUE3_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA0_QUEUE3_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA0_QUEUE3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA0_QUEUE3_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA0_QUEUE3_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA0_QUEUE3_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA0_QUEUE3_RB_BASE
+#define SDMA0_QUEUE3_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA0_QUEUE3_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA0_QUEUE3_RB_BASE_HI
+#define SDMA0_QUEUE3_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE3_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA0_QUEUE3_RB_RPTR
+#define SDMA0_QUEUE3_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE3_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE3_RB_RPTR_HI
+#define SDMA0_QUEUE3_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE3_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE3_RB_WPTR
+#define SDMA0_QUEUE3_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE3_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE3_RB_WPTR_HI
+#define SDMA0_QUEUE3_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE3_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE3_RB_RPTR_ADDR_HI
+#define SDMA0_QUEUE3_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_QUEUE3_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE3_RB_RPTR_ADDR_LO
+#define SDMA0_QUEUE3_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_QUEUE3_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_QUEUE3_IB_CNTL
+#define SDMA0_QUEUE3_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE3_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA0_QUEUE3_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA0_QUEUE3_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA0_QUEUE3_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA0_QUEUE3_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE3_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA0_QUEUE3_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA0_QUEUE3_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA0_QUEUE3_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA0_QUEUE3_IB_RPTR
+#define SDMA0_QUEUE3_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA0_QUEUE3_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA0_QUEUE3_IB_OFFSET
+#define SDMA0_QUEUE3_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA0_QUEUE3_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA0_QUEUE3_IB_BASE_LO
+#define SDMA0_QUEUE3_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA0_QUEUE3_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA0_QUEUE3_IB_BASE_HI
+#define SDMA0_QUEUE3_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE3_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE3_IB_SIZE
+#define SDMA0_QUEUE3_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA0_QUEUE3_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA0_QUEUE3_SKIP_CNTL
+#define SDMA0_QUEUE3_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA0_QUEUE3_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA0_QUEUE3_CONTEXT_STATUS
+#define SDMA0_QUEUE3_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA0_QUEUE3_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA0_QUEUE3_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA0_QUEUE3_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA0_QUEUE3_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA0_QUEUE3_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA0_QUEUE3_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA0_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA0_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA0_QUEUE3_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA0_QUEUE3_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA0_QUEUE3_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA0_QUEUE3_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA0_QUEUE3_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA0_QUEUE3_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA0_QUEUE3_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA0_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA0_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA0_QUEUE3_DOORBELL
+#define SDMA0_QUEUE3_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA0_QUEUE3_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA0_QUEUE3_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA0_QUEUE3_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA0_QUEUE3_DOORBELL_LOG
+#define SDMA0_QUEUE3_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA0_QUEUE3_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA0_QUEUE3_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA0_QUEUE3_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA0_QUEUE3_DOORBELL_OFFSET
+#define SDMA0_QUEUE3_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA0_QUEUE3_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA0_QUEUE3_CSA_ADDR_LO
+#define SDMA0_QUEUE3_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA0_QUEUE3_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA0_QUEUE3_CSA_ADDR_HI
+#define SDMA0_QUEUE3_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA0_QUEUE3_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA0_QUEUE3_SCHEDULE_CNTL
+#define SDMA0_QUEUE3_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA0_QUEUE3_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA0_QUEUE3_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA0_QUEUE3_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA0_QUEUE3_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA0_QUEUE3_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA0_QUEUE3_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA0_QUEUE3_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA0_QUEUE3_IB_SUB_REMAIN
+#define SDMA0_QUEUE3_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA0_QUEUE3_PREEMPT
+#define SDMA0_QUEUE3_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA0_QUEUE3_DUMMY_REG
+#define SDMA0_QUEUE3_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA0_QUEUE3_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE3_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_QUEUE3_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_QUEUE3_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_QUEUE3_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_QUEUE3_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_QUEUE3_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_QUEUE3_RB_AQL_CNTL
+#define SDMA0_QUEUE3_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA0_QUEUE3_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA0_QUEUE3_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA0_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA0_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA0_QUEUE3_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA0_QUEUE3_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_QUEUE3_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA0_QUEUE3_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA0_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA0_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA0_QUEUE3_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA0_QUEUE3_MINOR_PTR_UPDATE
+#define SDMA0_QUEUE3_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA0_QUEUE3_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA0_QUEUE3_RB_PREEMPT
+#define SDMA0_QUEUE3_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA0_QUEUE3_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA0_QUEUE3_MIDCMD_DATA0
+#define SDMA0_QUEUE3_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_DATA1
+#define SDMA0_QUEUE3_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_DATA2
+#define SDMA0_QUEUE3_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_DATA3
+#define SDMA0_QUEUE3_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_DATA4
+#define SDMA0_QUEUE3_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_DATA5
+#define SDMA0_QUEUE3_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_DATA6
+#define SDMA0_QUEUE3_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_DATA7
+#define SDMA0_QUEUE3_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_DATA8
+#define SDMA0_QUEUE3_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_DATA9
+#define SDMA0_QUEUE3_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_DATA10
+#define SDMA0_QUEUE3_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA0_QUEUE3_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE3_MIDCMD_CNTL
+#define SDMA0_QUEUE3_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA0_QUEUE3_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA0_QUEUE3_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA0_QUEUE3_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA0_QUEUE3_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA0_QUEUE3_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA0_QUEUE3_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA0_QUEUE3_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA0_QUEUE4_RB_CNTL
+#define SDMA0_QUEUE4_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE4_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA0_QUEUE4_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA0_QUEUE4_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA0_QUEUE4_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA0_QUEUE4_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA0_QUEUE4_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA0_QUEUE4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA0_QUEUE4_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA0_QUEUE4_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA0_QUEUE4_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA0_QUEUE4_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE4_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA0_QUEUE4_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA0_QUEUE4_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA0_QUEUE4_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA0_QUEUE4_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA0_QUEUE4_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA0_QUEUE4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA0_QUEUE4_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA0_QUEUE4_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA0_QUEUE4_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA0_QUEUE4_RB_BASE
+#define SDMA0_QUEUE4_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA0_QUEUE4_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA0_QUEUE4_RB_BASE_HI
+#define SDMA0_QUEUE4_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE4_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA0_QUEUE4_RB_RPTR
+#define SDMA0_QUEUE4_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE4_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE4_RB_RPTR_HI
+#define SDMA0_QUEUE4_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE4_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE4_RB_WPTR
+#define SDMA0_QUEUE4_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE4_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE4_RB_WPTR_HI
+#define SDMA0_QUEUE4_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE4_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE4_RB_RPTR_ADDR_HI
+#define SDMA0_QUEUE4_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_QUEUE4_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE4_RB_RPTR_ADDR_LO
+#define SDMA0_QUEUE4_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_QUEUE4_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_QUEUE4_IB_CNTL
+#define SDMA0_QUEUE4_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE4_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA0_QUEUE4_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA0_QUEUE4_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA0_QUEUE4_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA0_QUEUE4_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE4_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA0_QUEUE4_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA0_QUEUE4_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA0_QUEUE4_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA0_QUEUE4_IB_RPTR
+#define SDMA0_QUEUE4_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA0_QUEUE4_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA0_QUEUE4_IB_OFFSET
+#define SDMA0_QUEUE4_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA0_QUEUE4_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA0_QUEUE4_IB_BASE_LO
+#define SDMA0_QUEUE4_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA0_QUEUE4_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA0_QUEUE4_IB_BASE_HI
+#define SDMA0_QUEUE4_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE4_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE4_IB_SIZE
+#define SDMA0_QUEUE4_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA0_QUEUE4_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA0_QUEUE4_SKIP_CNTL
+#define SDMA0_QUEUE4_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA0_QUEUE4_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA0_QUEUE4_CONTEXT_STATUS
+#define SDMA0_QUEUE4_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA0_QUEUE4_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA0_QUEUE4_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA0_QUEUE4_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA0_QUEUE4_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA0_QUEUE4_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA0_QUEUE4_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA0_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA0_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA0_QUEUE4_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA0_QUEUE4_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA0_QUEUE4_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA0_QUEUE4_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA0_QUEUE4_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA0_QUEUE4_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA0_QUEUE4_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA0_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA0_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA0_QUEUE4_DOORBELL
+#define SDMA0_QUEUE4_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA0_QUEUE4_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA0_QUEUE4_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA0_QUEUE4_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA0_QUEUE4_DOORBELL_LOG
+#define SDMA0_QUEUE4_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA0_QUEUE4_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA0_QUEUE4_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA0_QUEUE4_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA0_QUEUE4_DOORBELL_OFFSET
+#define SDMA0_QUEUE4_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA0_QUEUE4_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA0_QUEUE4_CSA_ADDR_LO
+#define SDMA0_QUEUE4_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA0_QUEUE4_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA0_QUEUE4_CSA_ADDR_HI
+#define SDMA0_QUEUE4_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA0_QUEUE4_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA0_QUEUE4_SCHEDULE_CNTL
+#define SDMA0_QUEUE4_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA0_QUEUE4_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA0_QUEUE4_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA0_QUEUE4_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA0_QUEUE4_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA0_QUEUE4_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA0_QUEUE4_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA0_QUEUE4_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA0_QUEUE4_IB_SUB_REMAIN
+#define SDMA0_QUEUE4_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA0_QUEUE4_PREEMPT
+#define SDMA0_QUEUE4_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA0_QUEUE4_DUMMY_REG
+#define SDMA0_QUEUE4_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA0_QUEUE4_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE4_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_QUEUE4_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_QUEUE4_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_QUEUE4_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_QUEUE4_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_QUEUE4_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_QUEUE4_RB_AQL_CNTL
+#define SDMA0_QUEUE4_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA0_QUEUE4_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA0_QUEUE4_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA0_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA0_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA0_QUEUE4_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA0_QUEUE4_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_QUEUE4_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA0_QUEUE4_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA0_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA0_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA0_QUEUE4_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA0_QUEUE4_MINOR_PTR_UPDATE
+#define SDMA0_QUEUE4_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA0_QUEUE4_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA0_QUEUE4_RB_PREEMPT
+#define SDMA0_QUEUE4_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA0_QUEUE4_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA0_QUEUE4_MIDCMD_DATA0
+#define SDMA0_QUEUE4_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_DATA1
+#define SDMA0_QUEUE4_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_DATA2
+#define SDMA0_QUEUE4_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_DATA3
+#define SDMA0_QUEUE4_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_DATA4
+#define SDMA0_QUEUE4_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_DATA5
+#define SDMA0_QUEUE4_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_DATA6
+#define SDMA0_QUEUE4_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_DATA7
+#define SDMA0_QUEUE4_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_DATA8
+#define SDMA0_QUEUE4_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_DATA9
+#define SDMA0_QUEUE4_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_DATA10
+#define SDMA0_QUEUE4_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA0_QUEUE4_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE4_MIDCMD_CNTL
+#define SDMA0_QUEUE4_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA0_QUEUE4_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA0_QUEUE4_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA0_QUEUE4_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA0_QUEUE4_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA0_QUEUE4_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA0_QUEUE4_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA0_QUEUE4_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA0_QUEUE5_RB_CNTL
+#define SDMA0_QUEUE5_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE5_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA0_QUEUE5_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA0_QUEUE5_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA0_QUEUE5_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA0_QUEUE5_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA0_QUEUE5_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA0_QUEUE5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA0_QUEUE5_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA0_QUEUE5_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA0_QUEUE5_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA0_QUEUE5_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE5_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA0_QUEUE5_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA0_QUEUE5_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA0_QUEUE5_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA0_QUEUE5_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA0_QUEUE5_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA0_QUEUE5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA0_QUEUE5_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA0_QUEUE5_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA0_QUEUE5_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA0_QUEUE5_RB_BASE
+#define SDMA0_QUEUE5_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA0_QUEUE5_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA0_QUEUE5_RB_BASE_HI
+#define SDMA0_QUEUE5_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE5_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA0_QUEUE5_RB_RPTR
+#define SDMA0_QUEUE5_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE5_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE5_RB_RPTR_HI
+#define SDMA0_QUEUE5_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE5_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE5_RB_WPTR
+#define SDMA0_QUEUE5_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE5_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE5_RB_WPTR_HI
+#define SDMA0_QUEUE5_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE5_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE5_RB_RPTR_ADDR_HI
+#define SDMA0_QUEUE5_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_QUEUE5_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE5_RB_RPTR_ADDR_LO
+#define SDMA0_QUEUE5_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_QUEUE5_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_QUEUE5_IB_CNTL
+#define SDMA0_QUEUE5_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE5_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA0_QUEUE5_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA0_QUEUE5_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA0_QUEUE5_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA0_QUEUE5_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE5_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA0_QUEUE5_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA0_QUEUE5_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA0_QUEUE5_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA0_QUEUE5_IB_RPTR
+#define SDMA0_QUEUE5_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA0_QUEUE5_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA0_QUEUE5_IB_OFFSET
+#define SDMA0_QUEUE5_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA0_QUEUE5_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA0_QUEUE5_IB_BASE_LO
+#define SDMA0_QUEUE5_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA0_QUEUE5_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA0_QUEUE5_IB_BASE_HI
+#define SDMA0_QUEUE5_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE5_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE5_IB_SIZE
+#define SDMA0_QUEUE5_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA0_QUEUE5_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA0_QUEUE5_SKIP_CNTL
+#define SDMA0_QUEUE5_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA0_QUEUE5_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA0_QUEUE5_CONTEXT_STATUS
+#define SDMA0_QUEUE5_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA0_QUEUE5_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA0_QUEUE5_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA0_QUEUE5_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA0_QUEUE5_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA0_QUEUE5_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA0_QUEUE5_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA0_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA0_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA0_QUEUE5_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA0_QUEUE5_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA0_QUEUE5_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA0_QUEUE5_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA0_QUEUE5_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA0_QUEUE5_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA0_QUEUE5_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA0_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA0_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA0_QUEUE5_DOORBELL
+#define SDMA0_QUEUE5_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA0_QUEUE5_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA0_QUEUE5_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA0_QUEUE5_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA0_QUEUE5_DOORBELL_LOG
+#define SDMA0_QUEUE5_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA0_QUEUE5_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA0_QUEUE5_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA0_QUEUE5_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA0_QUEUE5_DOORBELL_OFFSET
+#define SDMA0_QUEUE5_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA0_QUEUE5_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA0_QUEUE5_CSA_ADDR_LO
+#define SDMA0_QUEUE5_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA0_QUEUE5_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA0_QUEUE5_CSA_ADDR_HI
+#define SDMA0_QUEUE5_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA0_QUEUE5_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA0_QUEUE5_SCHEDULE_CNTL
+#define SDMA0_QUEUE5_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA0_QUEUE5_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA0_QUEUE5_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA0_QUEUE5_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA0_QUEUE5_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA0_QUEUE5_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA0_QUEUE5_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA0_QUEUE5_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA0_QUEUE5_IB_SUB_REMAIN
+#define SDMA0_QUEUE5_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA0_QUEUE5_PREEMPT
+#define SDMA0_QUEUE5_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA0_QUEUE5_DUMMY_REG
+#define SDMA0_QUEUE5_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA0_QUEUE5_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE5_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_QUEUE5_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_QUEUE5_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_QUEUE5_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_QUEUE5_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_QUEUE5_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_QUEUE5_RB_AQL_CNTL
+#define SDMA0_QUEUE5_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA0_QUEUE5_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA0_QUEUE5_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA0_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA0_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA0_QUEUE5_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA0_QUEUE5_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_QUEUE5_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA0_QUEUE5_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA0_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA0_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA0_QUEUE5_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA0_QUEUE5_MINOR_PTR_UPDATE
+#define SDMA0_QUEUE5_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA0_QUEUE5_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA0_QUEUE5_RB_PREEMPT
+#define SDMA0_QUEUE5_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA0_QUEUE5_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA0_QUEUE5_MIDCMD_DATA0
+#define SDMA0_QUEUE5_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_DATA1
+#define SDMA0_QUEUE5_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_DATA2
+#define SDMA0_QUEUE5_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_DATA3
+#define SDMA0_QUEUE5_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_DATA4
+#define SDMA0_QUEUE5_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_DATA5
+#define SDMA0_QUEUE5_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_DATA6
+#define SDMA0_QUEUE5_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_DATA7
+#define SDMA0_QUEUE5_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_DATA8
+#define SDMA0_QUEUE5_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_DATA9
+#define SDMA0_QUEUE5_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_DATA10
+#define SDMA0_QUEUE5_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA0_QUEUE5_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE5_MIDCMD_CNTL
+#define SDMA0_QUEUE5_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA0_QUEUE5_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA0_QUEUE5_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA0_QUEUE5_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA0_QUEUE5_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA0_QUEUE5_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA0_QUEUE5_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA0_QUEUE5_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA0_QUEUE6_RB_CNTL
+#define SDMA0_QUEUE6_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE6_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA0_QUEUE6_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA0_QUEUE6_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA0_QUEUE6_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA0_QUEUE6_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA0_QUEUE6_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA0_QUEUE6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA0_QUEUE6_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA0_QUEUE6_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA0_QUEUE6_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA0_QUEUE6_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE6_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA0_QUEUE6_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA0_QUEUE6_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA0_QUEUE6_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA0_QUEUE6_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA0_QUEUE6_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA0_QUEUE6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA0_QUEUE6_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA0_QUEUE6_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA0_QUEUE6_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA0_QUEUE6_RB_BASE
+#define SDMA0_QUEUE6_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA0_QUEUE6_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA0_QUEUE6_RB_BASE_HI
+#define SDMA0_QUEUE6_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE6_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA0_QUEUE6_RB_RPTR
+#define SDMA0_QUEUE6_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE6_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE6_RB_RPTR_HI
+#define SDMA0_QUEUE6_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE6_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE6_RB_WPTR
+#define SDMA0_QUEUE6_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE6_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE6_RB_WPTR_HI
+#define SDMA0_QUEUE6_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE6_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE6_RB_RPTR_ADDR_HI
+#define SDMA0_QUEUE6_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_QUEUE6_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE6_RB_RPTR_ADDR_LO
+#define SDMA0_QUEUE6_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_QUEUE6_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_QUEUE6_IB_CNTL
+#define SDMA0_QUEUE6_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE6_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA0_QUEUE6_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA0_QUEUE6_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA0_QUEUE6_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA0_QUEUE6_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE6_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA0_QUEUE6_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA0_QUEUE6_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA0_QUEUE6_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA0_QUEUE6_IB_RPTR
+#define SDMA0_QUEUE6_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA0_QUEUE6_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA0_QUEUE6_IB_OFFSET
+#define SDMA0_QUEUE6_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA0_QUEUE6_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA0_QUEUE6_IB_BASE_LO
+#define SDMA0_QUEUE6_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA0_QUEUE6_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA0_QUEUE6_IB_BASE_HI
+#define SDMA0_QUEUE6_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE6_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE6_IB_SIZE
+#define SDMA0_QUEUE6_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA0_QUEUE6_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA0_QUEUE6_SKIP_CNTL
+#define SDMA0_QUEUE6_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA0_QUEUE6_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA0_QUEUE6_CONTEXT_STATUS
+#define SDMA0_QUEUE6_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA0_QUEUE6_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA0_QUEUE6_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA0_QUEUE6_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA0_QUEUE6_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA0_QUEUE6_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA0_QUEUE6_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA0_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA0_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA0_QUEUE6_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA0_QUEUE6_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA0_QUEUE6_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA0_QUEUE6_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA0_QUEUE6_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA0_QUEUE6_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA0_QUEUE6_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA0_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA0_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA0_QUEUE6_DOORBELL
+#define SDMA0_QUEUE6_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA0_QUEUE6_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA0_QUEUE6_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA0_QUEUE6_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA0_QUEUE6_DOORBELL_LOG
+#define SDMA0_QUEUE6_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA0_QUEUE6_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA0_QUEUE6_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA0_QUEUE6_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA0_QUEUE6_DOORBELL_OFFSET
+#define SDMA0_QUEUE6_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA0_QUEUE6_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA0_QUEUE6_CSA_ADDR_LO
+#define SDMA0_QUEUE6_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA0_QUEUE6_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA0_QUEUE6_CSA_ADDR_HI
+#define SDMA0_QUEUE6_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA0_QUEUE6_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA0_QUEUE6_SCHEDULE_CNTL
+#define SDMA0_QUEUE6_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA0_QUEUE6_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA0_QUEUE6_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA0_QUEUE6_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA0_QUEUE6_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA0_QUEUE6_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA0_QUEUE6_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA0_QUEUE6_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA0_QUEUE6_IB_SUB_REMAIN
+#define SDMA0_QUEUE6_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA0_QUEUE6_PREEMPT
+#define SDMA0_QUEUE6_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA0_QUEUE6_DUMMY_REG
+#define SDMA0_QUEUE6_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA0_QUEUE6_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE6_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_QUEUE6_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_QUEUE6_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_QUEUE6_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_QUEUE6_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_QUEUE6_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_QUEUE6_RB_AQL_CNTL
+#define SDMA0_QUEUE6_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA0_QUEUE6_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA0_QUEUE6_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA0_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA0_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA0_QUEUE6_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA0_QUEUE6_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_QUEUE6_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA0_QUEUE6_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA0_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA0_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA0_QUEUE6_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA0_QUEUE6_MINOR_PTR_UPDATE
+#define SDMA0_QUEUE6_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA0_QUEUE6_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA0_QUEUE6_RB_PREEMPT
+#define SDMA0_QUEUE6_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA0_QUEUE6_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA0_QUEUE6_MIDCMD_DATA0
+#define SDMA0_QUEUE6_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_DATA1
+#define SDMA0_QUEUE6_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_DATA2
+#define SDMA0_QUEUE6_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_DATA3
+#define SDMA0_QUEUE6_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_DATA4
+#define SDMA0_QUEUE6_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_DATA5
+#define SDMA0_QUEUE6_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_DATA6
+#define SDMA0_QUEUE6_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_DATA7
+#define SDMA0_QUEUE6_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_DATA8
+#define SDMA0_QUEUE6_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_DATA9
+#define SDMA0_QUEUE6_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_DATA10
+#define SDMA0_QUEUE6_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA0_QUEUE6_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE6_MIDCMD_CNTL
+#define SDMA0_QUEUE6_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA0_QUEUE6_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA0_QUEUE6_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA0_QUEUE6_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA0_QUEUE6_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA0_QUEUE6_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA0_QUEUE6_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA0_QUEUE6_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA0_QUEUE7_RB_CNTL
+#define SDMA0_QUEUE7_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE7_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA0_QUEUE7_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA0_QUEUE7_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA0_QUEUE7_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA0_QUEUE7_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA0_QUEUE7_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA0_QUEUE7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA0_QUEUE7_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA0_QUEUE7_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA0_QUEUE7_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA0_QUEUE7_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE7_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA0_QUEUE7_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA0_QUEUE7_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA0_QUEUE7_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA0_QUEUE7_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA0_QUEUE7_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA0_QUEUE7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA0_QUEUE7_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA0_QUEUE7_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA0_QUEUE7_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA0_QUEUE7_RB_BASE
+#define SDMA0_QUEUE7_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA0_QUEUE7_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA0_QUEUE7_RB_BASE_HI
+#define SDMA0_QUEUE7_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE7_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA0_QUEUE7_RB_RPTR
+#define SDMA0_QUEUE7_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE7_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE7_RB_RPTR_HI
+#define SDMA0_QUEUE7_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE7_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE7_RB_WPTR
+#define SDMA0_QUEUE7_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA0_QUEUE7_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA0_QUEUE7_RB_WPTR_HI
+#define SDMA0_QUEUE7_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA0_QUEUE7_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA0_QUEUE7_RB_RPTR_ADDR_HI
+#define SDMA0_QUEUE7_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA0_QUEUE7_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE7_RB_RPTR_ADDR_LO
+#define SDMA0_QUEUE7_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA0_QUEUE7_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA0_QUEUE7_IB_CNTL
+#define SDMA0_QUEUE7_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA0_QUEUE7_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA0_QUEUE7_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA0_QUEUE7_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA0_QUEUE7_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA0_QUEUE7_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA0_QUEUE7_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA0_QUEUE7_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA0_QUEUE7_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA0_QUEUE7_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA0_QUEUE7_IB_RPTR
+#define SDMA0_QUEUE7_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA0_QUEUE7_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA0_QUEUE7_IB_OFFSET
+#define SDMA0_QUEUE7_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA0_QUEUE7_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA0_QUEUE7_IB_BASE_LO
+#define SDMA0_QUEUE7_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA0_QUEUE7_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA0_QUEUE7_IB_BASE_HI
+#define SDMA0_QUEUE7_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA0_QUEUE7_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE7_IB_SIZE
+#define SDMA0_QUEUE7_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA0_QUEUE7_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA0_QUEUE7_SKIP_CNTL
+#define SDMA0_QUEUE7_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA0_QUEUE7_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA0_QUEUE7_CONTEXT_STATUS
+#define SDMA0_QUEUE7_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA0_QUEUE7_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA0_QUEUE7_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA0_QUEUE7_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA0_QUEUE7_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA0_QUEUE7_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA0_QUEUE7_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA0_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA0_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA0_QUEUE7_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA0_QUEUE7_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA0_QUEUE7_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA0_QUEUE7_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA0_QUEUE7_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA0_QUEUE7_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA0_QUEUE7_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA0_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA0_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA0_QUEUE7_DOORBELL
+#define SDMA0_QUEUE7_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA0_QUEUE7_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA0_QUEUE7_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA0_QUEUE7_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA0_QUEUE7_DOORBELL_LOG
+#define SDMA0_QUEUE7_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA0_QUEUE7_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA0_QUEUE7_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA0_QUEUE7_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA0_QUEUE7_DOORBELL_OFFSET
+#define SDMA0_QUEUE7_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA0_QUEUE7_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA0_QUEUE7_CSA_ADDR_LO
+#define SDMA0_QUEUE7_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA0_QUEUE7_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA0_QUEUE7_CSA_ADDR_HI
+#define SDMA0_QUEUE7_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA0_QUEUE7_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA0_QUEUE7_SCHEDULE_CNTL
+#define SDMA0_QUEUE7_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA0_QUEUE7_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA0_QUEUE7_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA0_QUEUE7_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA0_QUEUE7_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA0_QUEUE7_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA0_QUEUE7_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA0_QUEUE7_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA0_QUEUE7_IB_SUB_REMAIN
+#define SDMA0_QUEUE7_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA0_QUEUE7_PREEMPT
+#define SDMA0_QUEUE7_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA0_QUEUE7_DUMMY_REG
+#define SDMA0_QUEUE7_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA0_QUEUE7_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA0_QUEUE7_RB_WPTR_POLL_ADDR_HI
+#define SDMA0_QUEUE7_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA0_QUEUE7_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA0_QUEUE7_RB_WPTR_POLL_ADDR_LO
+#define SDMA0_QUEUE7_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA0_QUEUE7_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA0_QUEUE7_RB_AQL_CNTL
+#define SDMA0_QUEUE7_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA0_QUEUE7_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA0_QUEUE7_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA0_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA0_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA0_QUEUE7_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA0_QUEUE7_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA0_QUEUE7_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA0_QUEUE7_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA0_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA0_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA0_QUEUE7_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA0_QUEUE7_MINOR_PTR_UPDATE
+#define SDMA0_QUEUE7_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA0_QUEUE7_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA0_QUEUE7_RB_PREEMPT
+#define SDMA0_QUEUE7_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA0_QUEUE7_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA0_QUEUE7_MIDCMD_DATA0
+#define SDMA0_QUEUE7_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_DATA1
+#define SDMA0_QUEUE7_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_DATA2
+#define SDMA0_QUEUE7_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_DATA3
+#define SDMA0_QUEUE7_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_DATA4
+#define SDMA0_QUEUE7_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_DATA5
+#define SDMA0_QUEUE7_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_DATA6
+#define SDMA0_QUEUE7_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_DATA7
+#define SDMA0_QUEUE7_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_DATA8
+#define SDMA0_QUEUE7_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_DATA9
+#define SDMA0_QUEUE7_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_DATA10
+#define SDMA0_QUEUE7_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA0_QUEUE7_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA0_QUEUE7_MIDCMD_CNTL
+#define SDMA0_QUEUE7_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA0_QUEUE7_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA0_QUEUE7_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA0_QUEUE7_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA0_QUEUE7_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA0_QUEUE7_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA0_QUEUE7_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA0_QUEUE7_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+
+
+// addressBlock: gc_sdma0_sdma1dec
+//SDMA1_DEC_START
+#define SDMA1_DEC_START__START__SHIFT                                                                         0x0
+#define SDMA1_DEC_START__START_MASK                                                                           0xFFFFFFFFL
+//SDMA1_F32_MISC_CNTL
+#define SDMA1_F32_MISC_CNTL__F32_WAKEUP__SHIFT                                                                0x0
+#define SDMA1_F32_MISC_CNTL__F32_WAKEUP_MASK                                                                  0x00000001L
+//SDMA1_GLOBAL_TIMESTAMP_LO
+#define SDMA1_GLOBAL_TIMESTAMP_LO__DATA__SHIFT                                                                0x0
+#define SDMA1_GLOBAL_TIMESTAMP_LO__DATA_MASK                                                                  0xFFFFFFFFL
+//SDMA1_GLOBAL_TIMESTAMP_HI
+#define SDMA1_GLOBAL_TIMESTAMP_HI__DATA__SHIFT                                                                0x0
+#define SDMA1_GLOBAL_TIMESTAMP_HI__DATA_MASK                                                                  0xFFFFFFFFL
+//SDMA1_POWER_CNTL
+#define SDMA1_POWER_CNTL__LS_ENABLE__SHIFT                                                                    0x8
+#define SDMA1_POWER_CNTL__LS_ENABLE_MASK                                                                      0x00000100L
+//SDMA1_CNTL
+#define SDMA1_CNTL__TRAP_ENABLE__SHIFT                                                                        0x0
+#define SDMA1_CNTL__SEM_WAIT_INT_ENABLE__SHIFT                                                                0x2
+#define SDMA1_CNTL__DATA_SWAP_ENABLE__SHIFT                                                                   0x3
+#define SDMA1_CNTL__FENCE_SWAP_ENABLE__SHIFT                                                                  0x4
+#define SDMA1_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                              0x5
+#define SDMA1_CNTL__PIO_DONE_ACK_ENABLE__SHIFT                                                                0x6
+#define SDMA1_CNTL__TMZ_MIDCMD_PREEMPT_ENABLE__SHIFT                                                          0x8
+#define SDMA1_CNTL__MIDCMD_EXPIRE_ENABLE__SHIFT                                                               0x9
+#define SDMA1_CNTL__CP_MES_INT_ENABLE__SHIFT                                                                  0xa
+#define SDMA1_CNTL__PAGE_RETRY_TIMEOUT_INT_ENABLE__SHIFT                                                      0xb
+#define SDMA1_CNTL__PAGE_NULL_INT_ENABLE__SHIFT                                                               0xc
+#define SDMA1_CNTL__PAGE_FAULT_INT_ENABLE__SHIFT                                                              0xd
+#define SDMA1_CNTL__CH_PERFCNT_ENABLE__SHIFT                                                                  0x10
+#define SDMA1_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT                                                          0x11
+#define SDMA1_CNTL__DRM_RESTORE_ENABLE__SHIFT                                                                 0x13
+#define SDMA1_CNTL__CTXEMPTY_INT_ENABLE__SHIFT                                                                0x1c
+#define SDMA1_CNTL__FROZEN_INT_ENABLE__SHIFT                                                                  0x1d
+#define SDMA1_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT                                                              0x1e
+#define SDMA1_CNTL__RB_PREEMPT_INT_ENABLE__SHIFT                                                              0x1f
+#define SDMA1_CNTL__TRAP_ENABLE_MASK                                                                          0x00000001L
+#define SDMA1_CNTL__SEM_WAIT_INT_ENABLE_MASK                                                                  0x00000004L
+#define SDMA1_CNTL__DATA_SWAP_ENABLE_MASK                                                                     0x00000008L
+#define SDMA1_CNTL__FENCE_SWAP_ENABLE_MASK                                                                    0x00000010L
+#define SDMA1_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                                0x00000020L
+#define SDMA1_CNTL__PIO_DONE_ACK_ENABLE_MASK                                                                  0x00000040L
+#define SDMA1_CNTL__TMZ_MIDCMD_PREEMPT_ENABLE_MASK                                                            0x00000100L
+#define SDMA1_CNTL__MIDCMD_EXPIRE_ENABLE_MASK                                                                 0x00000200L
+#define SDMA1_CNTL__CP_MES_INT_ENABLE_MASK                                                                    0x00000400L
+#define SDMA1_CNTL__PAGE_RETRY_TIMEOUT_INT_ENABLE_MASK                                                        0x00000800L
+#define SDMA1_CNTL__PAGE_NULL_INT_ENABLE_MASK                                                                 0x00001000L
+#define SDMA1_CNTL__PAGE_FAULT_INT_ENABLE_MASK                                                                0x00002000L
+#define SDMA1_CNTL__CH_PERFCNT_ENABLE_MASK                                                                    0x00010000L
+#define SDMA1_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK                                                            0x00020000L
+#define SDMA1_CNTL__DRM_RESTORE_ENABLE_MASK                                                                   0x00080000L
+#define SDMA1_CNTL__CTXEMPTY_INT_ENABLE_MASK                                                                  0x10000000L
+#define SDMA1_CNTL__FROZEN_INT_ENABLE_MASK                                                                    0x20000000L
+#define SDMA1_CNTL__IB_PREEMPT_INT_ENABLE_MASK                                                                0x40000000L
+#define SDMA1_CNTL__RB_PREEMPT_INT_ENABLE_MASK                                                                0x80000000L
+//SDMA1_CHICKEN_BITS
+#define SDMA1_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE__SHIFT                                                 0x1
+#define SDMA1_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE__SHIFT                                        0x2
+#define SDMA1_CHICKEN_BITS__BACK_COMPAT_ENABLE__SHIFT                                                         0x3
+#define SDMA1_CHICKEN_BITS__SRBM_POLL_RETRYING__SHIFT                                                         0x5
+#define SDMA1_CHICKEN_BITS__RD_BURST__SHIFT                                                                   0x6
+#define SDMA1_CHICKEN_BITS__WR_BURST__SHIFT                                                                   0x8
+#define SDMA1_CHICKEN_BITS__COMBINE_256B_WAIT_CYCLE__SHIFT                                                    0xa
+#define SDMA1_CHICKEN_BITS__WR_COMBINE_256B_ENABLE__SHIFT                                                     0xe
+#define SDMA1_CHICKEN_BITS__RD_COMBINE_256B_ENABLE__SHIFT                                                     0xf
+#define SDMA1_CHICKEN_BITS__COPY_OVERLAP_ENABLE__SHIFT                                                        0x10
+#define SDMA1_CHICKEN_BITS__RAW_CHECK_ENABLE__SHIFT                                                           0x11
+#define SDMA1_CHICKEN_BITS__T2L_256B_ENABLE__SHIFT                                                            0x12
+#define SDMA1_CHICKEN_BITS__SOFT_OVERRIDE_GCR_FGCG__SHIFT                                                     0x13
+#define SDMA1_CHICKEN_BITS__SOFT_OVERRIDE_GRBM_FGCG__SHIFT                                                    0x14
+#define SDMA1_CHICKEN_BITS__SOFT_OVERRIDE_CH_FGCG__SHIFT                                                      0x15
+#define SDMA1_CHICKEN_BITS__SOFT_OVERRIDE_UTCL2_INVREQ_FGCG__SHIFT                                            0x16
+#define SDMA1_CHICKEN_BITS__SOFT_OVERRIDE_UTCL1_FGCG__SHIFT                                                   0x17
+#define SDMA1_CHICKEN_BITS__CG_STATUS_OUTPUT__SHIFT                                                           0x18
+#define SDMA1_CHICKEN_BITS__SW_FREEZE_ENABLE__SHIFT                                                           0x19
+#define SDMA1_CHICKEN_BITS__DRAM_ECC_COPY_MODE_CNTL__SHIFT                                                    0x1a
+#define SDMA1_CHICKEN_BITS__RESERVED__SHIFT                                                                   0x1b
+#define SDMA1_CHICKEN_BITS__STALL_ON_TRANS_FULL_ENABLE_MASK                                                   0x00000002L
+#define SDMA1_CHICKEN_BITS__STALL_ON_NO_FREE_DATA_BUFFER_ENABLE_MASK                                          0x00000004L
+#define SDMA1_CHICKEN_BITS__BACK_COMPAT_ENABLE_MASK                                                           0x00000008L
+#define SDMA1_CHICKEN_BITS__SRBM_POLL_RETRYING_MASK                                                           0x00000020L
+#define SDMA1_CHICKEN_BITS__RD_BURST_MASK                                                                     0x000000C0L
+#define SDMA1_CHICKEN_BITS__WR_BURST_MASK                                                                     0x00000300L
+#define SDMA1_CHICKEN_BITS__COMBINE_256B_WAIT_CYCLE_MASK                                                      0x00003C00L
+#define SDMA1_CHICKEN_BITS__WR_COMBINE_256B_ENABLE_MASK                                                       0x00004000L
+#define SDMA1_CHICKEN_BITS__RD_COMBINE_256B_ENABLE_MASK                                                       0x00008000L
+#define SDMA1_CHICKEN_BITS__COPY_OVERLAP_ENABLE_MASK                                                          0x00010000L
+#define SDMA1_CHICKEN_BITS__RAW_CHECK_ENABLE_MASK                                                             0x00020000L
+#define SDMA1_CHICKEN_BITS__T2L_256B_ENABLE_MASK                                                              0x00040000L
+#define SDMA1_CHICKEN_BITS__SOFT_OVERRIDE_GCR_FGCG_MASK                                                       0x00080000L
+#define SDMA1_CHICKEN_BITS__SOFT_OVERRIDE_GRBM_FGCG_MASK                                                      0x00100000L
+#define SDMA1_CHICKEN_BITS__SOFT_OVERRIDE_CH_FGCG_MASK                                                        0x00200000L
+#define SDMA1_CHICKEN_BITS__SOFT_OVERRIDE_UTCL2_INVREQ_FGCG_MASK                                              0x00400000L
+#define SDMA1_CHICKEN_BITS__SOFT_OVERRIDE_UTCL1_FGCG_MASK                                                     0x00800000L
+#define SDMA1_CHICKEN_BITS__CG_STATUS_OUTPUT_MASK                                                             0x01000000L
+#define SDMA1_CHICKEN_BITS__SW_FREEZE_ENABLE_MASK                                                             0x02000000L
+#define SDMA1_CHICKEN_BITS__DRAM_ECC_COPY_MODE_CNTL_MASK                                                      0x04000000L
+#define SDMA1_CHICKEN_BITS__RESERVED_MASK                                                                     0xF8000000L
+//SDMA1_GB_ADDR_CONFIG
+#define SDMA1_GB_ADDR_CONFIG__NUM_PIPES__SHIFT                                                                0x0
+#define SDMA1_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                     0x3
+#define SDMA1_GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS__SHIFT                                                     0x6
+#define SDMA1_GB_ADDR_CONFIG__NUM_PKRS__SHIFT                                                                 0x8
+#define SDMA1_GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT                                                       0x13
+#define SDMA1_GB_ADDR_CONFIG__NUM_RB_PER_SE__SHIFT                                                            0x1a
+#define SDMA1_GB_ADDR_CONFIG__NUM_PIPES_MASK                                                                  0x00000007L
+#define SDMA1_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                       0x00000038L
+#define SDMA1_GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS_MASK                                                       0x000000C0L
+#define SDMA1_GB_ADDR_CONFIG__NUM_PKRS_MASK                                                                   0x00000700L
+#define SDMA1_GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK                                                         0x00180000L
+#define SDMA1_GB_ADDR_CONFIG__NUM_RB_PER_SE_MASK                                                              0x0C000000L
+//SDMA1_GB_ADDR_CONFIG_READ
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                           0x0
+#define SDMA1_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                0x3
+#define SDMA1_GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS__SHIFT                                                0x6
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_PKRS__SHIFT                                                            0x8
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                  0x13
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_RB_PER_SE__SHIFT                                                       0x1a
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                             0x00000007L
+#define SDMA1_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                  0x00000038L
+#define SDMA1_GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS_MASK                                                  0x000000C0L
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_PKRS_MASK                                                              0x00000700L
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                    0x00180000L
+#define SDMA1_GB_ADDR_CONFIG_READ__NUM_RB_PER_SE_MASK                                                         0x0C000000L
+//SDMA1_RB_RPTR_FETCH
+#define SDMA1_RB_RPTR_FETCH__OFFSET__SHIFT                                                                    0x2
+#define SDMA1_RB_RPTR_FETCH__OFFSET_MASK                                                                      0xFFFFFFFCL
+//SDMA1_RB_RPTR_FETCH_HI
+#define SDMA1_RB_RPTR_FETCH_HI__OFFSET__SHIFT                                                                 0x0
+#define SDMA1_RB_RPTR_FETCH_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//SDMA1_SEM_WAIT_FAIL_TIMER_CNTL
+#define SDMA1_SEM_WAIT_FAIL_TIMER_CNTL__TIMER__SHIFT                                                          0x0
+#define SDMA1_SEM_WAIT_FAIL_TIMER_CNTL__TIMER_MASK                                                            0xFFFFFFFFL
+//SDMA1_IB_OFFSET_FETCH
+#define SDMA1_IB_OFFSET_FETCH__OFFSET__SHIFT                                                                  0x2
+#define SDMA1_IB_OFFSET_FETCH__OFFSET_MASK                                                                    0x003FFFFCL
+//SDMA1_PROGRAM
+#define SDMA1_PROGRAM__STREAM__SHIFT                                                                          0x0
+#define SDMA1_PROGRAM__STREAM_MASK                                                                            0xFFFFFFFFL
+//SDMA1_STATUS_REG
+#define SDMA1_STATUS_REG__IDLE__SHIFT                                                                         0x0
+#define SDMA1_STATUS_REG__REG_IDLE__SHIFT                                                                     0x1
+#define SDMA1_STATUS_REG__RB_EMPTY__SHIFT                                                                     0x2
+#define SDMA1_STATUS_REG__RB_FULL__SHIFT                                                                      0x3
+#define SDMA1_STATUS_REG__RB_CMD_IDLE__SHIFT                                                                  0x4
+#define SDMA1_STATUS_REG__RB_CMD_FULL__SHIFT                                                                  0x5
+#define SDMA1_STATUS_REG__IB_CMD_IDLE__SHIFT                                                                  0x6
+#define SDMA1_STATUS_REG__IB_CMD_FULL__SHIFT                                                                  0x7
+#define SDMA1_STATUS_REG__BLOCK_IDLE__SHIFT                                                                   0x8
+#define SDMA1_STATUS_REG__INSIDE_IB__SHIFT                                                                    0x9
+#define SDMA1_STATUS_REG__EX_IDLE__SHIFT                                                                      0xa
+#define SDMA1_STATUS_REG__CGCG_FENCE__SHIFT                                                                   0xb
+#define SDMA1_STATUS_REG__PACKET_READY__SHIFT                                                                 0xc
+#define SDMA1_STATUS_REG__MC_WR_IDLE__SHIFT                                                                   0xd
+#define SDMA1_STATUS_REG__SRBM_IDLE__SHIFT                                                                    0xe
+#define SDMA1_STATUS_REG__CONTEXT_EMPTY__SHIFT                                                                0xf
+#define SDMA1_STATUS_REG__DELTA_RPTR_FULL__SHIFT                                                              0x10
+#define SDMA1_STATUS_REG__RB_MC_RREQ_IDLE__SHIFT                                                              0x11
+#define SDMA1_STATUS_REG__IB_MC_RREQ_IDLE__SHIFT                                                              0x12
+#define SDMA1_STATUS_REG__MC_RD_IDLE__SHIFT                                                                   0x13
+#define SDMA1_STATUS_REG__DELTA_RPTR_EMPTY__SHIFT                                                             0x14
+#define SDMA1_STATUS_REG__MC_RD_RET_STALL__SHIFT                                                              0x15
+#define SDMA1_STATUS_REG__MC_RD_NO_POLL_IDLE__SHIFT                                                           0x16
+#define SDMA1_STATUS_REG__DRM_IDLE__SHIFT                                                                     0x17
+#define SDMA1_STATUS_REG__DRM_MASK_FULL__SHIFT                                                                0x18
+#define SDMA1_STATUS_REG__PREV_CMD_IDLE__SHIFT                                                                0x19
+#define SDMA1_STATUS_REG__SEM_IDLE__SHIFT                                                                     0x1a
+#define SDMA1_STATUS_REG__SEM_REQ_STALL__SHIFT                                                                0x1b
+#define SDMA1_STATUS_REG__SEM_RESP_STATE__SHIFT                                                               0x1c
+#define SDMA1_STATUS_REG__INT_IDLE__SHIFT                                                                     0x1e
+#define SDMA1_STATUS_REG__INT_REQ_STALL__SHIFT                                                                0x1f
+#define SDMA1_STATUS_REG__IDLE_MASK                                                                           0x00000001L
+#define SDMA1_STATUS_REG__REG_IDLE_MASK                                                                       0x00000002L
+#define SDMA1_STATUS_REG__RB_EMPTY_MASK                                                                       0x00000004L
+#define SDMA1_STATUS_REG__RB_FULL_MASK                                                                        0x00000008L
+#define SDMA1_STATUS_REG__RB_CMD_IDLE_MASK                                                                    0x00000010L
+#define SDMA1_STATUS_REG__RB_CMD_FULL_MASK                                                                    0x00000020L
+#define SDMA1_STATUS_REG__IB_CMD_IDLE_MASK                                                                    0x00000040L
+#define SDMA1_STATUS_REG__IB_CMD_FULL_MASK                                                                    0x00000080L
+#define SDMA1_STATUS_REG__BLOCK_IDLE_MASK                                                                     0x00000100L
+#define SDMA1_STATUS_REG__INSIDE_IB_MASK                                                                      0x00000200L
+#define SDMA1_STATUS_REG__EX_IDLE_MASK                                                                        0x00000400L
+#define SDMA1_STATUS_REG__CGCG_FENCE_MASK                                                                     0x00000800L
+#define SDMA1_STATUS_REG__PACKET_READY_MASK                                                                   0x00001000L
+#define SDMA1_STATUS_REG__MC_WR_IDLE_MASK                                                                     0x00002000L
+#define SDMA1_STATUS_REG__SRBM_IDLE_MASK                                                                      0x00004000L
+#define SDMA1_STATUS_REG__CONTEXT_EMPTY_MASK                                                                  0x00008000L
+#define SDMA1_STATUS_REG__DELTA_RPTR_FULL_MASK                                                                0x00010000L
+#define SDMA1_STATUS_REG__RB_MC_RREQ_IDLE_MASK                                                                0x00020000L
+#define SDMA1_STATUS_REG__IB_MC_RREQ_IDLE_MASK                                                                0x00040000L
+#define SDMA1_STATUS_REG__MC_RD_IDLE_MASK                                                                     0x00080000L
+#define SDMA1_STATUS_REG__DELTA_RPTR_EMPTY_MASK                                                               0x00100000L
+#define SDMA1_STATUS_REG__MC_RD_RET_STALL_MASK                                                                0x00200000L
+#define SDMA1_STATUS_REG__MC_RD_NO_POLL_IDLE_MASK                                                             0x00400000L
+#define SDMA1_STATUS_REG__DRM_IDLE_MASK                                                                       0x00800000L
+#define SDMA1_STATUS_REG__DRM_MASK_FULL_MASK                                                                  0x01000000L
+#define SDMA1_STATUS_REG__PREV_CMD_IDLE_MASK                                                                  0x02000000L
+#define SDMA1_STATUS_REG__SEM_IDLE_MASK                                                                       0x04000000L
+#define SDMA1_STATUS_REG__SEM_REQ_STALL_MASK                                                                  0x08000000L
+#define SDMA1_STATUS_REG__SEM_RESP_STATE_MASK                                                                 0x30000000L
+#define SDMA1_STATUS_REG__INT_IDLE_MASK                                                                       0x40000000L
+#define SDMA1_STATUS_REG__INT_REQ_STALL_MASK                                                                  0x80000000L
+//SDMA1_STATUS1_REG
+#define SDMA1_STATUS1_REG__CE_WREQ_IDLE__SHIFT                                                                0x0
+#define SDMA1_STATUS1_REG__CE_WR_IDLE__SHIFT                                                                  0x1
+#define SDMA1_STATUS1_REG__CE_SPLIT_IDLE__SHIFT                                                               0x2
+#define SDMA1_STATUS1_REG__CE_RREQ_IDLE__SHIFT                                                                0x3
+#define SDMA1_STATUS1_REG__CE_OUT_IDLE__SHIFT                                                                 0x4
+#define SDMA1_STATUS1_REG__CE_IN_IDLE__SHIFT                                                                  0x5
+#define SDMA1_STATUS1_REG__CE_DST_IDLE__SHIFT                                                                 0x6
+#define SDMA1_STATUS1_REG__CE_DRM_IDLE__SHIFT                                                                 0x7
+#define SDMA1_STATUS1_REG__CE_DRM1_IDLE__SHIFT                                                                0x8
+#define SDMA1_STATUS1_REG__CE_CMD_IDLE__SHIFT                                                                 0x9
+#define SDMA1_STATUS1_REG__CE_AFIFO_FULL__SHIFT                                                               0xa
+#define SDMA1_STATUS1_REG__CE_INFO_FULL__SHIFT                                                                0xb
+#define SDMA1_STATUS1_REG__CE_INFO1_FULL__SHIFT                                                               0xc
+#define SDMA1_STATUS1_REG__EX_START__SHIFT                                                                    0xd
+#define SDMA1_STATUS1_REG__DRM_CTX_RESTORE__SHIFT                                                             0xe
+#define SDMA1_STATUS1_REG__CE_RD_STALL__SHIFT                                                                 0xf
+#define SDMA1_STATUS1_REG__CE_WR_STALL__SHIFT                                                                 0x10
+#define SDMA1_STATUS1_REG__SEC_INTR_STATUS__SHIFT                                                             0x11
+#define SDMA1_STATUS1_REG__WPTR_POLL_IDLE__SHIFT                                                              0x12
+#define SDMA1_STATUS1_REG__SDMA_IDLE__SHIFT                                                                   0x13
+#define SDMA1_STATUS1_REG__CE_WREQ_IDLE_MASK                                                                  0x00000001L
+#define SDMA1_STATUS1_REG__CE_WR_IDLE_MASK                                                                    0x00000002L
+#define SDMA1_STATUS1_REG__CE_SPLIT_IDLE_MASK                                                                 0x00000004L
+#define SDMA1_STATUS1_REG__CE_RREQ_IDLE_MASK                                                                  0x00000008L
+#define SDMA1_STATUS1_REG__CE_OUT_IDLE_MASK                                                                   0x00000010L
+#define SDMA1_STATUS1_REG__CE_IN_IDLE_MASK                                                                    0x00000020L
+#define SDMA1_STATUS1_REG__CE_DST_IDLE_MASK                                                                   0x00000040L
+#define SDMA1_STATUS1_REG__CE_DRM_IDLE_MASK                                                                   0x00000080L
+#define SDMA1_STATUS1_REG__CE_DRM1_IDLE_MASK                                                                  0x00000100L
+#define SDMA1_STATUS1_REG__CE_CMD_IDLE_MASK                                                                   0x00000200L
+#define SDMA1_STATUS1_REG__CE_AFIFO_FULL_MASK                                                                 0x00000400L
+#define SDMA1_STATUS1_REG__CE_INFO_FULL_MASK                                                                  0x00000800L
+#define SDMA1_STATUS1_REG__CE_INFO1_FULL_MASK                                                                 0x00001000L
+#define SDMA1_STATUS1_REG__EX_START_MASK                                                                      0x00002000L
+#define SDMA1_STATUS1_REG__DRM_CTX_RESTORE_MASK                                                               0x00004000L
+#define SDMA1_STATUS1_REG__CE_RD_STALL_MASK                                                                   0x00008000L
+#define SDMA1_STATUS1_REG__CE_WR_STALL_MASK                                                                   0x00010000L
+#define SDMA1_STATUS1_REG__SEC_INTR_STATUS_MASK                                                               0x00020000L
+#define SDMA1_STATUS1_REG__WPTR_POLL_IDLE_MASK                                                                0x00040000L
+#define SDMA1_STATUS1_REG__SDMA_IDLE_MASK                                                                     0x00080000L
+//SDMA1_CNTL1
+#define SDMA1_CNTL1__WPTR_POLL_FREQUENCY__SHIFT                                                               0x2
+#define SDMA1_CNTL1__WPTR_POLL_FREQUENCY_MASK                                                                 0x0000FFFCL
+//SDMA1_HBM_PAGE_CONFIG
+#define SDMA1_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT__SHIFT                                                      0x0
+#define SDMA1_HBM_PAGE_CONFIG__PAGE_SIZE_EXPONENT_MASK                                                        0x00000003L
+//SDMA1_UCODE_CHECKSUM
+#define SDMA1_UCODE_CHECKSUM__DATA__SHIFT                                                                     0x0
+#define SDMA1_UCODE_CHECKSUM__DATA_MASK                                                                       0xFFFFFFFFL
+//SDMA1_FREEZE
+#define SDMA1_FREEZE__PREEMPT__SHIFT                                                                          0x0
+#define SDMA1_FREEZE__FREEZE__SHIFT                                                                           0x4
+#define SDMA1_FREEZE__FROZEN__SHIFT                                                                           0x5
+#define SDMA1_FREEZE__F32_FREEZE__SHIFT                                                                       0x6
+#define SDMA1_FREEZE__PREEMPT_MASK                                                                            0x00000001L
+#define SDMA1_FREEZE__FREEZE_MASK                                                                             0x00000010L
+#define SDMA1_FREEZE__FROZEN_MASK                                                                             0x00000020L
+#define SDMA1_FREEZE__F32_FREEZE_MASK                                                                         0x00000040L
+//SDMA1_PROCESS_QUANTUM0
+#define SDMA1_PROCESS_QUANTUM0__PROCESS0_QUANTUM__SHIFT                                                       0x0
+#define SDMA1_PROCESS_QUANTUM0__PROCESS1_QUANTUM__SHIFT                                                       0x8
+#define SDMA1_PROCESS_QUANTUM0__PROCESS2_QUANTUM__SHIFT                                                       0x10
+#define SDMA1_PROCESS_QUANTUM0__PROCESS3_QUANTUM__SHIFT                                                       0x18
+#define SDMA1_PROCESS_QUANTUM0__PROCESS0_QUANTUM_MASK                                                         0x000000FFL
+#define SDMA1_PROCESS_QUANTUM0__PROCESS1_QUANTUM_MASK                                                         0x0000FF00L
+#define SDMA1_PROCESS_QUANTUM0__PROCESS2_QUANTUM_MASK                                                         0x00FF0000L
+#define SDMA1_PROCESS_QUANTUM0__PROCESS3_QUANTUM_MASK                                                         0xFF000000L
+//SDMA1_PROCESS_QUANTUM1
+#define SDMA1_PROCESS_QUANTUM1__PROCESS4_QUANTUM__SHIFT                                                       0x0
+#define SDMA1_PROCESS_QUANTUM1__PROCESS5_QUANTUM__SHIFT                                                       0x8
+#define SDMA1_PROCESS_QUANTUM1__PROCESS6_QUANTUM__SHIFT                                                       0x10
+#define SDMA1_PROCESS_QUANTUM1__PROCESS7_QUANTUM__SHIFT                                                       0x18
+#define SDMA1_PROCESS_QUANTUM1__PROCESS4_QUANTUM_MASK                                                         0x000000FFL
+#define SDMA1_PROCESS_QUANTUM1__PROCESS5_QUANTUM_MASK                                                         0x0000FF00L
+#define SDMA1_PROCESS_QUANTUM1__PROCESS6_QUANTUM_MASK                                                         0x00FF0000L
+#define SDMA1_PROCESS_QUANTUM1__PROCESS7_QUANTUM_MASK                                                         0xFF000000L
+//SDMA1_WATCHDOG_CNTL
+#define SDMA1_WATCHDOG_CNTL__QUEUE_HANG_COUNT__SHIFT                                                          0x0
+#define SDMA1_WATCHDOG_CNTL__CMD_TIMEOUT_COUNT__SHIFT                                                         0x8
+#define SDMA1_WATCHDOG_CNTL__QUEUE_HANG_COUNT_MASK                                                            0x000000FFL
+#define SDMA1_WATCHDOG_CNTL__CMD_TIMEOUT_COUNT_MASK                                                           0x0000FF00L
+//SDMA1_QUEUE_STATUS0
+#define SDMA1_QUEUE_STATUS0__QUEUE0_STATUS__SHIFT                                                             0x0
+#define SDMA1_QUEUE_STATUS0__QUEUE1_STATUS__SHIFT                                                             0x4
+#define SDMA1_QUEUE_STATUS0__QUEUE2_STATUS__SHIFT                                                             0x8
+#define SDMA1_QUEUE_STATUS0__QUEUE3_STATUS__SHIFT                                                             0xc
+#define SDMA1_QUEUE_STATUS0__QUEUE4_STATUS__SHIFT                                                             0x10
+#define SDMA1_QUEUE_STATUS0__QUEUE5_STATUS__SHIFT                                                             0x14
+#define SDMA1_QUEUE_STATUS0__QUEUE6_STATUS__SHIFT                                                             0x18
+#define SDMA1_QUEUE_STATUS0__QUEUE7_STATUS__SHIFT                                                             0x1c
+#define SDMA1_QUEUE_STATUS0__QUEUE0_STATUS_MASK                                                               0x0000000FL
+#define SDMA1_QUEUE_STATUS0__QUEUE1_STATUS_MASK                                                               0x000000F0L
+#define SDMA1_QUEUE_STATUS0__QUEUE2_STATUS_MASK                                                               0x00000F00L
+#define SDMA1_QUEUE_STATUS0__QUEUE3_STATUS_MASK                                                               0x0000F000L
+#define SDMA1_QUEUE_STATUS0__QUEUE4_STATUS_MASK                                                               0x000F0000L
+#define SDMA1_QUEUE_STATUS0__QUEUE5_STATUS_MASK                                                               0x00F00000L
+#define SDMA1_QUEUE_STATUS0__QUEUE6_STATUS_MASK                                                               0x0F000000L
+#define SDMA1_QUEUE_STATUS0__QUEUE7_STATUS_MASK                                                               0xF0000000L
+//SDMA1_EDC_CONFIG
+#define SDMA1_EDC_CONFIG__WRITE_DIS__SHIFT                                                                    0x0
+#define SDMA1_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
+#define SDMA1_EDC_CONFIG__ECC_INT_ENABLE__SHIFT                                                               0x2
+#define SDMA1_EDC_CONFIG__WRITE_DIS_MASK                                                                      0x00000001L
+#define SDMA1_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
+#define SDMA1_EDC_CONFIG__ECC_INT_ENABLE_MASK                                                                 0x00000004L
+//SDMA1_BA_THRESHOLD
+#define SDMA1_BA_THRESHOLD__READ_THRES__SHIFT                                                                 0x0
+#define SDMA1_BA_THRESHOLD__WRITE_THRES__SHIFT                                                                0x10
+#define SDMA1_BA_THRESHOLD__READ_THRES_MASK                                                                   0x000003FFL
+#define SDMA1_BA_THRESHOLD__WRITE_THRES_MASK                                                                  0x03FF0000L
+//SDMA1_ID
+#define SDMA1_ID__DEVICE_ID__SHIFT                                                                            0x0
+#define SDMA1_ID__DEVICE_ID_MASK                                                                              0x000000FFL
+//SDMA1_VERSION
+#define SDMA1_VERSION__MINVER__SHIFT                                                                          0x0
+#define SDMA1_VERSION__MAJVER__SHIFT                                                                          0x8
+#define SDMA1_VERSION__REV__SHIFT                                                                             0x10
+#define SDMA1_VERSION__MINVER_MASK                                                                            0x0000007FL
+#define SDMA1_VERSION__MAJVER_MASK                                                                            0x00007F00L
+#define SDMA1_VERSION__REV_MASK                                                                               0x003F0000L
+//SDMA1_EDC_COUNTER
+#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_DED__SHIFT                                                          0x0
+#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_SEC__SHIFT                                                          0x1
+#define SDMA1_EDC_COUNTER__SDMA_RB_CMD_BUF_SED__SHIFT                                                         0x2
+#define SDMA1_EDC_COUNTER__SDMA_IB_CMD_BUF_SED__SHIFT                                                         0x3
+#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED__SHIFT                                                      0x4
+#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED__SHIFT                                                   0x5
+#define SDMA1_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED__SHIFT                                                      0x6
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED__SHIFT                                                    0x7
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED__SHIFT                                                    0x8
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED__SHIFT                                                    0x9
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED__SHIFT                                                    0xa
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED__SHIFT                                                    0xb
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED__SHIFT                                                    0xc
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED__SHIFT                                                    0xd
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED__SHIFT                                                    0xe
+#define SDMA1_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED__SHIFT                                                      0xf
+#define SDMA1_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED__SHIFT                                                    0x10
+#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_DED_MASK                                                            0x00000001L
+#define SDMA1_EDC_COUNTER__SDMA_UCODE_BUF_SEC_MASK                                                            0x00000002L
+#define SDMA1_EDC_COUNTER__SDMA_RB_CMD_BUF_SED_MASK                                                           0x00000004L
+#define SDMA1_EDC_COUNTER__SDMA_IB_CMD_BUF_SED_MASK                                                           0x00000008L
+#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RD_FIFO_SED_MASK                                                        0x00000010L
+#define SDMA1_EDC_COUNTER__SDMA_UTCL1_RDBST_FIFO_SED_MASK                                                     0x00000020L
+#define SDMA1_EDC_COUNTER__SDMA_DATA_LUT_FIFO_SED_MASK                                                        0x00000040L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF0_SED_MASK                                                      0x00000080L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF1_SED_MASK                                                      0x00000100L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF2_SED_MASK                                                      0x00000200L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF3_SED_MASK                                                      0x00000400L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF4_SED_MASK                                                      0x00000800L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF5_SED_MASK                                                      0x00001000L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF6_SED_MASK                                                      0x00002000L
+#define SDMA1_EDC_COUNTER__SDMA_MBANK_DATA_BUF7_SED_MASK                                                      0x00004000L
+#define SDMA1_EDC_COUNTER__SDMA_SPLIT_DAT_BUF_SED_MASK                                                        0x00008000L
+#define SDMA1_EDC_COUNTER__SDMA_MC_WR_ADDR_FIFO_SED_MASK                                                      0x00010000L
+//SDMA1_EDC_COUNTER_CLEAR
+#define SDMA1_EDC_COUNTER_CLEAR__DUMMY__SHIFT                                                                 0x0
+#define SDMA1_EDC_COUNTER_CLEAR__DUMMY_MASK                                                                   0x00000001L
+//SDMA1_STATUS2_REG
+#define SDMA1_STATUS2_REG__ID__SHIFT                                                                          0x0
+#define SDMA1_STATUS2_REG__TH0F32_INSTR_PTR__SHIFT                                                            0x2
+#define SDMA1_STATUS2_REG__CMD_OP__SHIFT                                                                      0x10
+#define SDMA1_STATUS2_REG__ID_MASK                                                                            0x00000003L
+#define SDMA1_STATUS2_REG__TH0F32_INSTR_PTR_MASK                                                              0x0000FFFCL
+#define SDMA1_STATUS2_REG__CMD_OP_MASK                                                                        0xFFFF0000L
+//SDMA1_ATOMIC_CNTL
+#define SDMA1_ATOMIC_CNTL__LOOP_TIMER__SHIFT                                                                  0x0
+#define SDMA1_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT                                                       0x1f
+#define SDMA1_ATOMIC_CNTL__LOOP_TIMER_MASK                                                                    0x7FFFFFFFL
+#define SDMA1_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK                                                         0x80000000L
+//SDMA1_ATOMIC_PREOP_LO
+#define SDMA1_ATOMIC_PREOP_LO__DATA__SHIFT                                                                    0x0
+#define SDMA1_ATOMIC_PREOP_LO__DATA_MASK                                                                      0xFFFFFFFFL
+//SDMA1_ATOMIC_PREOP_HI
+#define SDMA1_ATOMIC_PREOP_HI__DATA__SHIFT                                                                    0x0
+#define SDMA1_ATOMIC_PREOP_HI__DATA_MASK                                                                      0xFFFFFFFFL
+//SDMA1_UTCL1_CNTL
+#define SDMA1_UTCL1_CNTL__REDO_DELAY__SHIFT                                                                   0x0
+#define SDMA1_UTCL1_CNTL__PAGE_WAIT_DELAY__SHIFT                                                              0x5
+#define SDMA1_UTCL1_CNTL__RESP_MODE__SHIFT                                                                    0x9
+#define SDMA1_UTCL1_CNTL__FORCE_INVALIDATION__SHIFT                                                           0xe
+#define SDMA1_UTCL1_CNTL__FORCE_INVREQ_HEAVY__SHIFT                                                           0xf
+#define SDMA1_UTCL1_CNTL__WR_EXE_PERMS_CTRL__SHIFT                                                            0x10
+#define SDMA1_UTCL1_CNTL__RD_EXE_PERMS_CTRL__SHIFT                                                            0x11
+#define SDMA1_UTCL1_CNTL__INVACK_DELAY__SHIFT                                                                 0x12
+#define SDMA1_UTCL1_CNTL__REQL2_CREDIT__SHIFT                                                                 0x18
+#define SDMA1_UTCL1_CNTL__REDO_DELAY_MASK                                                                     0x0000001FL
+#define SDMA1_UTCL1_CNTL__PAGE_WAIT_DELAY_MASK                                                                0x000001E0L
+#define SDMA1_UTCL1_CNTL__RESP_MODE_MASK                                                                      0x00000600L
+#define SDMA1_UTCL1_CNTL__FORCE_INVALIDATION_MASK                                                             0x00004000L
+#define SDMA1_UTCL1_CNTL__FORCE_INVREQ_HEAVY_MASK                                                             0x00008000L
+#define SDMA1_UTCL1_CNTL__WR_EXE_PERMS_CTRL_MASK                                                              0x00010000L
+#define SDMA1_UTCL1_CNTL__RD_EXE_PERMS_CTRL_MASK                                                              0x00020000L
+#define SDMA1_UTCL1_CNTL__INVACK_DELAY_MASK                                                                   0x003C0000L
+#define SDMA1_UTCL1_CNTL__REQL2_CREDIT_MASK                                                                   0x3F000000L
+//SDMA1_UTCL1_WATERMK
+#define SDMA1_UTCL1_WATERMK__WR_REQ_FIFO_WATERMK__SHIFT                                                       0x0
+#define SDMA1_UTCL1_WATERMK__WR_REQ_FIFO_DEPTH_STEP__SHIFT                                                    0x4
+#define SDMA1_UTCL1_WATERMK__RD_REQ_FIFO_WATERMK__SHIFT                                                       0x6
+#define SDMA1_UTCL1_WATERMK__RD_REQ_FIFO_DEPTH_STEP__SHIFT                                                    0xa
+#define SDMA1_UTCL1_WATERMK__WR_PAGE_FIFO_WATERMK__SHIFT                                                      0xc
+#define SDMA1_UTCL1_WATERMK__WR_PAGE_FIFO_DEPTH_STEP__SHIFT                                                   0x10
+#define SDMA1_UTCL1_WATERMK__RD_PAGE_FIFO_WATERMK__SHIFT                                                      0x12
+#define SDMA1_UTCL1_WATERMK__RD_PAGE_FIFO_DEPTH_STEP__SHIFT                                                   0x16
+#define SDMA1_UTCL1_WATERMK__WR_REQ_FIFO_WATERMK_MASK                                                         0x0000000FL
+#define SDMA1_UTCL1_WATERMK__WR_REQ_FIFO_DEPTH_STEP_MASK                                                      0x00000030L
+#define SDMA1_UTCL1_WATERMK__RD_REQ_FIFO_WATERMK_MASK                                                         0x000003C0L
+#define SDMA1_UTCL1_WATERMK__RD_REQ_FIFO_DEPTH_STEP_MASK                                                      0x00000C00L
+#define SDMA1_UTCL1_WATERMK__WR_PAGE_FIFO_WATERMK_MASK                                                        0x0000F000L
+#define SDMA1_UTCL1_WATERMK__WR_PAGE_FIFO_DEPTH_STEP_MASK                                                     0x00030000L
+#define SDMA1_UTCL1_WATERMK__RD_PAGE_FIFO_WATERMK_MASK                                                        0x003C0000L
+#define SDMA1_UTCL1_WATERMK__RD_PAGE_FIFO_DEPTH_STEP_MASK                                                     0x00C00000L
+//SDMA1_UTCL1_TIMEOUT
+#define SDMA1_UTCL1_TIMEOUT__XNACK_LIMIT__SHIFT                                                               0x0
+#define SDMA1_UTCL1_TIMEOUT__XNACK_LIMIT_MASK                                                                 0x0000FFFFL
+//SDMA1_UTCL1_PAGE
+#define SDMA1_UTCL1_PAGE__VM_HOLE__SHIFT                                                                      0x0
+#define SDMA1_UTCL1_PAGE__REQ_TYPE__SHIFT                                                                     0x1
+#define SDMA1_UTCL1_PAGE__TMZ_ENABLE__SHIFT                                                                   0x5
+#define SDMA1_UTCL1_PAGE__USE_MTYPE__SHIFT                                                                    0x6
+#define SDMA1_UTCL1_PAGE__USE_PT_SNOOP__SHIFT                                                                 0xa
+#define SDMA1_UTCL1_PAGE__USE_IO__SHIFT                                                                       0xb
+#define SDMA1_UTCL1_PAGE__RD_L2_POLICY__SHIFT                                                                 0xc
+#define SDMA1_UTCL1_PAGE__WR_L2_POLICY__SHIFT                                                                 0xe
+#define SDMA1_UTCL1_PAGE__DMA_PAGE_SIZE__SHIFT                                                                0x10
+#define SDMA1_UTCL1_PAGE__USE_BC__SHIFT                                                                       0x16
+#define SDMA1_UTCL1_PAGE__ADDR_IS_PA__SHIFT                                                                   0x17
+#define SDMA1_UTCL1_PAGE__LLC_NOALLOC__SHIFT                                                                  0x18
+#define SDMA1_UTCL1_PAGE__VM_HOLE_MASK                                                                        0x00000001L
+#define SDMA1_UTCL1_PAGE__REQ_TYPE_MASK                                                                       0x0000001EL
+#define SDMA1_UTCL1_PAGE__TMZ_ENABLE_MASK                                                                     0x00000020L
+#define SDMA1_UTCL1_PAGE__USE_MTYPE_MASK                                                                      0x000003C0L
+#define SDMA1_UTCL1_PAGE__USE_PT_SNOOP_MASK                                                                   0x00000400L
+#define SDMA1_UTCL1_PAGE__USE_IO_MASK                                                                         0x00000800L
+#define SDMA1_UTCL1_PAGE__RD_L2_POLICY_MASK                                                                   0x00003000L
+#define SDMA1_UTCL1_PAGE__WR_L2_POLICY_MASK                                                                   0x0000C000L
+#define SDMA1_UTCL1_PAGE__DMA_PAGE_SIZE_MASK                                                                  0x003F0000L
+#define SDMA1_UTCL1_PAGE__USE_BC_MASK                                                                         0x00400000L
+#define SDMA1_UTCL1_PAGE__ADDR_IS_PA_MASK                                                                     0x00800000L
+#define SDMA1_UTCL1_PAGE__LLC_NOALLOC_MASK                                                                    0x01000000L
+//SDMA1_UTCL1_RD_STATUS
+#define SDMA1_UTCL1_RD_STATUS__RD_VA_FIFO_EMPTY__SHIFT                                                        0x0
+#define SDMA1_UTCL1_RD_STATUS__RD_REG_ENTRY_EMPTY__SHIFT                                                      0x1
+#define SDMA1_UTCL1_RD_STATUS__RD_PAGE_FIFO_EMPTY__SHIFT                                                      0x2
+#define SDMA1_UTCL1_RD_STATUS__RD_REQ_FIFO_EMPTY__SHIFT                                                       0x3
+#define SDMA1_UTCL1_RD_STATUS__RD_VA_REQ_FIFO_EMPTY__SHIFT                                                    0x4
+#define SDMA1_UTCL1_RD_STATUS__RESERVED0__SHIFT                                                               0x5
+#define SDMA1_UTCL1_RD_STATUS__RESERVED1__SHIFT                                                               0x6
+#define SDMA1_UTCL1_RD_STATUS__META_Q_EMPTY__SHIFT                                                            0x7
+#define SDMA1_UTCL1_RD_STATUS__RD_VA_FIFO_FULL__SHIFT                                                         0x8
+#define SDMA1_UTCL1_RD_STATUS__RD_REG_ENTRY_FULL__SHIFT                                                       0x9
+#define SDMA1_UTCL1_RD_STATUS__RD_PAGE_FIFO_FULL__SHIFT                                                       0xa
+#define SDMA1_UTCL1_RD_STATUS__RD_REQ_FIFO_FULL__SHIFT                                                        0xb
+#define SDMA1_UTCL1_RD_STATUS__RD_VA_REQ_FIFO_FULL__SHIFT                                                     0xc
+#define SDMA1_UTCL1_RD_STATUS__RESERVED2__SHIFT                                                               0xd
+#define SDMA1_UTCL1_RD_STATUS__RESERVED3__SHIFT                                                               0xe
+#define SDMA1_UTCL1_RD_STATUS__META_Q_FULL__SHIFT                                                             0xf
+#define SDMA1_UTCL1_RD_STATUS__RD_L2_INTF_IDLE__SHIFT                                                         0x10
+#define SDMA1_UTCL1_RD_STATUS__RD_REQRET_IDLE__SHIFT                                                          0x11
+#define SDMA1_UTCL1_RD_STATUS__RD_REQ_IDLE__SHIFT                                                             0x12
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_TYPE__SHIFT                                                           0x13
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_DATA_PA_READY__SHIFT                                                  0x15
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_META_PA_READY__SHIFT                                                  0x16
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_REG_READY__SHIFT                                                      0x17
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_PAGE_FIFO_READY__SHIFT                                                0x18
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_REQ_FIFO_READY__SHIFT                                                 0x19
+#define SDMA1_UTCL1_RD_STATUS__RESERVED4__SHIFT                                                               0x1a
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_OUT_RTR__SHIFT                                                        0x1b
+#define SDMA1_UTCL1_RD_STATUS__RDREQ_IN_RTR__SHIFT                                                            0x1c
+#define SDMA1_UTCL1_RD_STATUS__RDREQ_OUT_RTR__SHIFT                                                           0x1d
+#define SDMA1_UTCL1_RD_STATUS__INV_BUSY__SHIFT                                                                0x1e
+#define SDMA1_UTCL1_RD_STATUS__DBIT_REQ_IDLE__SHIFT                                                           0x1f
+#define SDMA1_UTCL1_RD_STATUS__RD_VA_FIFO_EMPTY_MASK                                                          0x00000001L
+#define SDMA1_UTCL1_RD_STATUS__RD_REG_ENTRY_EMPTY_MASK                                                        0x00000002L
+#define SDMA1_UTCL1_RD_STATUS__RD_PAGE_FIFO_EMPTY_MASK                                                        0x00000004L
+#define SDMA1_UTCL1_RD_STATUS__RD_REQ_FIFO_EMPTY_MASK                                                         0x00000008L
+#define SDMA1_UTCL1_RD_STATUS__RD_VA_REQ_FIFO_EMPTY_MASK                                                      0x00000010L
+#define SDMA1_UTCL1_RD_STATUS__RESERVED0_MASK                                                                 0x00000020L
+#define SDMA1_UTCL1_RD_STATUS__RESERVED1_MASK                                                                 0x00000040L
+#define SDMA1_UTCL1_RD_STATUS__META_Q_EMPTY_MASK                                                              0x00000080L
+#define SDMA1_UTCL1_RD_STATUS__RD_VA_FIFO_FULL_MASK                                                           0x00000100L
+#define SDMA1_UTCL1_RD_STATUS__RD_REG_ENTRY_FULL_MASK                                                         0x00000200L
+#define SDMA1_UTCL1_RD_STATUS__RD_PAGE_FIFO_FULL_MASK                                                         0x00000400L
+#define SDMA1_UTCL1_RD_STATUS__RD_REQ_FIFO_FULL_MASK                                                          0x00000800L
+#define SDMA1_UTCL1_RD_STATUS__RD_VA_REQ_FIFO_FULL_MASK                                                       0x00001000L
+#define SDMA1_UTCL1_RD_STATUS__RESERVED2_MASK                                                                 0x00002000L
+#define SDMA1_UTCL1_RD_STATUS__RESERVED3_MASK                                                                 0x00004000L
+#define SDMA1_UTCL1_RD_STATUS__META_Q_FULL_MASK                                                               0x00008000L
+#define SDMA1_UTCL1_RD_STATUS__RD_L2_INTF_IDLE_MASK                                                           0x00010000L
+#define SDMA1_UTCL1_RD_STATUS__RD_REQRET_IDLE_MASK                                                            0x00020000L
+#define SDMA1_UTCL1_RD_STATUS__RD_REQ_IDLE_MASK                                                               0x00040000L
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_TYPE_MASK                                                             0x00180000L
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_DATA_PA_READY_MASK                                                    0x00200000L
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_META_PA_READY_MASK                                                    0x00400000L
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_REG_READY_MASK                                                        0x00800000L
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_PAGE_FIFO_READY_MASK                                                  0x01000000L
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_REQ_FIFO_READY_MASK                                                   0x02000000L
+#define SDMA1_UTCL1_RD_STATUS__RESERVED4_MASK                                                                 0x04000000L
+#define SDMA1_UTCL1_RD_STATUS__RD_MERGE_OUT_RTR_MASK                                                          0x08000000L
+#define SDMA1_UTCL1_RD_STATUS__RDREQ_IN_RTR_MASK                                                              0x10000000L
+#define SDMA1_UTCL1_RD_STATUS__RDREQ_OUT_RTR_MASK                                                             0x20000000L
+#define SDMA1_UTCL1_RD_STATUS__INV_BUSY_MASK                                                                  0x40000000L
+#define SDMA1_UTCL1_RD_STATUS__DBIT_REQ_IDLE_MASK                                                             0x80000000L
+//SDMA1_UTCL1_WR_STATUS
+#define SDMA1_UTCL1_WR_STATUS__WR_VA_FIFO_EMPTY__SHIFT                                                        0x0
+#define SDMA1_UTCL1_WR_STATUS__WR_REG_ENTRY_EMPTY__SHIFT                                                      0x1
+#define SDMA1_UTCL1_WR_STATUS__WR_PAGE_FIFO_EMPTY__SHIFT                                                      0x2
+#define SDMA1_UTCL1_WR_STATUS__WR_REQ_FIFO_EMPTY__SHIFT                                                       0x3
+#define SDMA1_UTCL1_WR_STATUS__WR_VA_REQ_FIFO_EMPTY__SHIFT                                                    0x4
+#define SDMA1_UTCL1_WR_STATUS__WR_DATA2_EMPTY__SHIFT                                                          0x5
+#define SDMA1_UTCL1_WR_STATUS__WR_DATA1_EMPTY__SHIFT                                                          0x6
+#define SDMA1_UTCL1_WR_STATUS__RESERVED0__SHIFT                                                               0x7
+#define SDMA1_UTCL1_WR_STATUS__WR_VA_FIFO_FULL__SHIFT                                                         0x8
+#define SDMA1_UTCL1_WR_STATUS__WR_REG_ENTRY_FULL__SHIFT                                                       0x9
+#define SDMA1_UTCL1_WR_STATUS__WR_PAGE_FIFO_FULL__SHIFT                                                       0xa
+#define SDMA1_UTCL1_WR_STATUS__WR_REQ_FIFO_FULL__SHIFT                                                        0xb
+#define SDMA1_UTCL1_WR_STATUS__WR_VA_REQ_FIFO_FULL__SHIFT                                                     0xc
+#define SDMA1_UTCL1_WR_STATUS__WR_DATA2_FULL__SHIFT                                                           0xd
+#define SDMA1_UTCL1_WR_STATUS__WR_DATA1_FULL__SHIFT                                                           0xe
+#define SDMA1_UTCL1_WR_STATUS__F32_WR_RTR__SHIFT                                                              0xf
+#define SDMA1_UTCL1_WR_STATUS__WR_L2_INTF_IDLE__SHIFT                                                         0x10
+#define SDMA1_UTCL1_WR_STATUS__WR_REQRET_IDLE__SHIFT                                                          0x11
+#define SDMA1_UTCL1_WR_STATUS__WR_REQ_IDLE__SHIFT                                                             0x12
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_TYPE__SHIFT                                                           0x13
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_DATA_PA_READY__SHIFT                                                  0x15
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_META_PA_READY__SHIFT                                                  0x16
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_REG_READY__SHIFT                                                      0x17
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_PAGE_FIFO_READY__SHIFT                                                0x18
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_REQ_FIFO_READY__SHIFT                                                 0x19
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_DATA_SEL__SHIFT                                                       0x1a
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_OUT_RTR__SHIFT                                                        0x1b
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_IN_RTR__SHIFT                                                            0x1c
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_OUT_RTR__SHIFT                                                           0x1d
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_IN_DATA1_RTR__SHIFT                                                      0x1e
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_IN_DATA2_RTR__SHIFT                                                      0x1f
+#define SDMA1_UTCL1_WR_STATUS__WR_VA_FIFO_EMPTY_MASK                                                          0x00000001L
+#define SDMA1_UTCL1_WR_STATUS__WR_REG_ENTRY_EMPTY_MASK                                                        0x00000002L
+#define SDMA1_UTCL1_WR_STATUS__WR_PAGE_FIFO_EMPTY_MASK                                                        0x00000004L
+#define SDMA1_UTCL1_WR_STATUS__WR_REQ_FIFO_EMPTY_MASK                                                         0x00000008L
+#define SDMA1_UTCL1_WR_STATUS__WR_VA_REQ_FIFO_EMPTY_MASK                                                      0x00000010L
+#define SDMA1_UTCL1_WR_STATUS__WR_DATA2_EMPTY_MASK                                                            0x00000020L
+#define SDMA1_UTCL1_WR_STATUS__WR_DATA1_EMPTY_MASK                                                            0x00000040L
+#define SDMA1_UTCL1_WR_STATUS__RESERVED0_MASK                                                                 0x00000080L
+#define SDMA1_UTCL1_WR_STATUS__WR_VA_FIFO_FULL_MASK                                                           0x00000100L
+#define SDMA1_UTCL1_WR_STATUS__WR_REG_ENTRY_FULL_MASK                                                         0x00000200L
+#define SDMA1_UTCL1_WR_STATUS__WR_PAGE_FIFO_FULL_MASK                                                         0x00000400L
+#define SDMA1_UTCL1_WR_STATUS__WR_REQ_FIFO_FULL_MASK                                                          0x00000800L
+#define SDMA1_UTCL1_WR_STATUS__WR_VA_REQ_FIFO_FULL_MASK                                                       0x00001000L
+#define SDMA1_UTCL1_WR_STATUS__WR_DATA2_FULL_MASK                                                             0x00002000L
+#define SDMA1_UTCL1_WR_STATUS__WR_DATA1_FULL_MASK                                                             0x00004000L
+#define SDMA1_UTCL1_WR_STATUS__F32_WR_RTR_MASK                                                                0x00008000L
+#define SDMA1_UTCL1_WR_STATUS__WR_L2_INTF_IDLE_MASK                                                           0x00010000L
+#define SDMA1_UTCL1_WR_STATUS__WR_REQRET_IDLE_MASK                                                            0x00020000L
+#define SDMA1_UTCL1_WR_STATUS__WR_REQ_IDLE_MASK                                                               0x00040000L
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_TYPE_MASK                                                             0x00180000L
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_DATA_PA_READY_MASK                                                    0x00200000L
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_META_PA_READY_MASK                                                    0x00400000L
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_REG_READY_MASK                                                        0x00800000L
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_PAGE_FIFO_READY_MASK                                                  0x01000000L
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_REQ_FIFO_READY_MASK                                                   0x02000000L
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_DATA_SEL_MASK                                                         0x04000000L
+#define SDMA1_UTCL1_WR_STATUS__WR_MERGE_OUT_RTR_MASK                                                          0x08000000L
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_IN_RTR_MASK                                                              0x10000000L
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_OUT_RTR_MASK                                                             0x20000000L
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_IN_DATA1_RTR_MASK                                                        0x40000000L
+#define SDMA1_UTCL1_WR_STATUS__WRREQ_IN_DATA2_RTR_MASK                                                        0x80000000L
+//SDMA1_UTCL1_INV0
+#define SDMA1_UTCL1_INV0__INV_PROC_BUSY__SHIFT                                                                0x0
+#define SDMA1_UTCL1_INV0__GPUVM_FRAG_SIZE__SHIFT                                                              0x1
+#define SDMA1_UTCL1_INV0__GPUVM_VMID__SHIFT                                                                   0x7
+#define SDMA1_UTCL1_INV0__GPUVM_MODE__SHIFT                                                                   0xb
+#define SDMA1_UTCL1_INV0__GPUVM_HIGH__SHIFT                                                                   0xd
+#define SDMA1_UTCL1_INV0__GPUVM_TAG__SHIFT                                                                    0xe
+#define SDMA1_UTCL1_INV0__GPUVM_VMID_HIGH__SHIFT                                                              0x12
+#define SDMA1_UTCL1_INV0__GPUVM_VMID_LOW__SHIFT                                                               0x16
+#define SDMA1_UTCL1_INV0__INV_TYPE__SHIFT                                                                     0x1a
+#define SDMA1_UTCL1_INV0__INV_PROC_BUSY_MASK                                                                  0x00000001L
+#define SDMA1_UTCL1_INV0__GPUVM_FRAG_SIZE_MASK                                                                0x0000007EL
+#define SDMA1_UTCL1_INV0__GPUVM_VMID_MASK                                                                     0x00000780L
+#define SDMA1_UTCL1_INV0__GPUVM_MODE_MASK                                                                     0x00001800L
+#define SDMA1_UTCL1_INV0__GPUVM_HIGH_MASK                                                                     0x00002000L
+#define SDMA1_UTCL1_INV0__GPUVM_TAG_MASK                                                                      0x0003C000L
+#define SDMA1_UTCL1_INV0__GPUVM_VMID_HIGH_MASK                                                                0x003C0000L
+#define SDMA1_UTCL1_INV0__GPUVM_VMID_LOW_MASK                                                                 0x03C00000L
+#define SDMA1_UTCL1_INV0__INV_TYPE_MASK                                                                       0x0C000000L
+//SDMA1_UTCL1_INV1
+#define SDMA1_UTCL1_INV1__INV_ADDR_LO__SHIFT                                                                  0x0
+#define SDMA1_UTCL1_INV1__INV_ADDR_LO_MASK                                                                    0xFFFFFFFFL
+//SDMA1_UTCL1_INV2
+#define SDMA1_UTCL1_INV2__CPF_VMID__SHIFT                                                                     0x0
+#define SDMA1_UTCL1_INV2__CPF_FLUSH_TYPE__SHIFT                                                               0x10
+#define SDMA1_UTCL1_INV2__CPF_FRAG_SIZE__SHIFT                                                                0x11
+#define SDMA1_UTCL1_INV2__CPF_VMID_MASK                                                                       0x0000FFFFL
+#define SDMA1_UTCL1_INV2__CPF_FLUSH_TYPE_MASK                                                                 0x00010000L
+#define SDMA1_UTCL1_INV2__CPF_FRAG_SIZE_MASK                                                                  0x007E0000L
+//SDMA1_UTCL1_RD_XNACK0
+#define SDMA1_UTCL1_RD_XNACK0__XNACK_FAULT_ADDR_LO__SHIFT                                                     0x0
+#define SDMA1_UTCL1_RD_XNACK0__XNACK_FAULT_ADDR_LO_MASK                                                       0xFFFFFFFFL
+//SDMA1_UTCL1_RD_XNACK1
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_FAULT_ADDR_HI__SHIFT                                                     0x0
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_FAULT_VMID__SHIFT                                                        0x4
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_FAULT_VECTOR__SHIFT                                                      0x8
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_NULL_VECTOR__SHIFT                                                       0xa
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_TIMEOUT_VECTOR__SHIFT                                                    0xc
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_FAULT_FLAG__SHIFT                                                        0xe
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_NULL_FLAG__SHIFT                                                         0xf
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_TIMEOUT_FLAG__SHIFT                                                      0x10
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_FAULT_ADDR_HI_MASK                                                       0x0000000FL
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_FAULT_VMID_MASK                                                          0x000000F0L
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_FAULT_VECTOR_MASK                                                        0x00000300L
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_NULL_VECTOR_MASK                                                         0x00000C00L
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_TIMEOUT_VECTOR_MASK                                                      0x00003000L
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_FAULT_FLAG_MASK                                                          0x00004000L
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_NULL_FLAG_MASK                                                           0x00008000L
+#define SDMA1_UTCL1_RD_XNACK1__XNACK_TIMEOUT_FLAG_MASK                                                        0x00010000L
+//SDMA1_UTCL1_WR_XNACK0
+#define SDMA1_UTCL1_WR_XNACK0__XNACK_FAULT_ADDR_LO__SHIFT                                                     0x0
+#define SDMA1_UTCL1_WR_XNACK0__XNACK_FAULT_ADDR_LO_MASK                                                       0xFFFFFFFFL
+//SDMA1_UTCL1_WR_XNACK1
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_FAULT_ADDR_HI__SHIFT                                                     0x0
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_FAULT_VMID__SHIFT                                                        0x4
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_FAULT_VECTOR__SHIFT                                                      0x8
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_NULL_VECTOR__SHIFT                                                       0xa
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_TIMEOUT_VECTOR__SHIFT                                                    0xc
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_FAULT_FLAG__SHIFT                                                        0xe
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_NULL_FLAG__SHIFT                                                         0xf
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_TIMEOUT_FLAG__SHIFT                                                      0x10
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_FAULT_ADDR_HI_MASK                                                       0x0000000FL
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_FAULT_VMID_MASK                                                          0x000000F0L
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_FAULT_VECTOR_MASK                                                        0x00000300L
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_NULL_VECTOR_MASK                                                         0x00000C00L
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_TIMEOUT_VECTOR_MASK                                                      0x00003000L
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_FAULT_FLAG_MASK                                                          0x00004000L
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_NULL_FLAG_MASK                                                           0x00008000L
+#define SDMA1_UTCL1_WR_XNACK1__XNACK_TIMEOUT_FLAG_MASK                                                        0x00010000L
+//SDMA1_RELAX_ORDERING_LUT
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED0__SHIFT                                                            0x0
+#define SDMA1_RELAX_ORDERING_LUT__COPY__SHIFT                                                                 0x1
+#define SDMA1_RELAX_ORDERING_LUT__WRITE__SHIFT                                                                0x2
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED3__SHIFT                                                            0x3
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED4__SHIFT                                                            0x4
+#define SDMA1_RELAX_ORDERING_LUT__FENCE__SHIFT                                                                0x5
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED76__SHIFT                                                           0x6
+#define SDMA1_RELAX_ORDERING_LUT__POLL_MEM__SHIFT                                                             0x8
+#define SDMA1_RELAX_ORDERING_LUT__COND_EXE__SHIFT                                                             0x9
+#define SDMA1_RELAX_ORDERING_LUT__ATOMIC__SHIFT                                                               0xa
+#define SDMA1_RELAX_ORDERING_LUT__CONST_FILL__SHIFT                                                           0xb
+#define SDMA1_RELAX_ORDERING_LUT__PTEPDE__SHIFT                                                               0xc
+#define SDMA1_RELAX_ORDERING_LUT__TIMESTAMP__SHIFT                                                            0xd
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED__SHIFT                                                             0xe
+#define SDMA1_RELAX_ORDERING_LUT__WORLD_SWITCH__SHIFT                                                         0x1b
+#define SDMA1_RELAX_ORDERING_LUT__RPTR_WRB__SHIFT                                                             0x1c
+#define SDMA1_RELAX_ORDERING_LUT__WPTR_POLL__SHIFT                                                            0x1d
+#define SDMA1_RELAX_ORDERING_LUT__IB_FETCH__SHIFT                                                             0x1e
+#define SDMA1_RELAX_ORDERING_LUT__RB_FETCH__SHIFT                                                             0x1f
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED0_MASK                                                              0x00000001L
+#define SDMA1_RELAX_ORDERING_LUT__COPY_MASK                                                                   0x00000002L
+#define SDMA1_RELAX_ORDERING_LUT__WRITE_MASK                                                                  0x00000004L
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED3_MASK                                                              0x00000008L
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED4_MASK                                                              0x00000010L
+#define SDMA1_RELAX_ORDERING_LUT__FENCE_MASK                                                                  0x00000020L
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED76_MASK                                                             0x000000C0L
+#define SDMA1_RELAX_ORDERING_LUT__POLL_MEM_MASK                                                               0x00000100L
+#define SDMA1_RELAX_ORDERING_LUT__COND_EXE_MASK                                                               0x00000200L
+#define SDMA1_RELAX_ORDERING_LUT__ATOMIC_MASK                                                                 0x00000400L
+#define SDMA1_RELAX_ORDERING_LUT__CONST_FILL_MASK                                                             0x00000800L
+#define SDMA1_RELAX_ORDERING_LUT__PTEPDE_MASK                                                                 0x00001000L
+#define SDMA1_RELAX_ORDERING_LUT__TIMESTAMP_MASK                                                              0x00002000L
+#define SDMA1_RELAX_ORDERING_LUT__RESERVED_MASK                                                               0x07FFC000L
+#define SDMA1_RELAX_ORDERING_LUT__WORLD_SWITCH_MASK                                                           0x08000000L
+#define SDMA1_RELAX_ORDERING_LUT__RPTR_WRB_MASK                                                               0x10000000L
+#define SDMA1_RELAX_ORDERING_LUT__WPTR_POLL_MASK                                                              0x20000000L
+#define SDMA1_RELAX_ORDERING_LUT__IB_FETCH_MASK                                                               0x40000000L
+#define SDMA1_RELAX_ORDERING_LUT__RB_FETCH_MASK                                                               0x80000000L
+//SDMA1_CHICKEN_BITS_2
+#define SDMA1_CHICKEN_BITS_2__F32_CMD_PROC_DELAY__SHIFT                                                       0x0
+#define SDMA1_CHICKEN_BITS_2__F32_SEND_POSTCODE_EN__SHIFT                                                     0x4
+#define SDMA1_CHICKEN_BITS_2__UCODE_BUF_DS_EN__SHIFT                                                          0x6
+#define SDMA1_CHICKEN_BITS_2__UCODE_SELFLOAD_THREAD_OVERLAP__SHIFT                                            0x7
+#define SDMA1_CHICKEN_BITS_2__WPTR_POLL_OUTSTANDING__SHIFT                                                    0x8
+#define SDMA1_CHICKEN_BITS_2__RESERVED_14_12__SHIFT                                                           0xc
+#define SDMA1_CHICKEN_BITS_2__RESERVED_15__SHIFT                                                              0xf
+#define SDMA1_CHICKEN_BITS_2__RB_FIFO_WATERMARK__SHIFT                                                        0x10
+#define SDMA1_CHICKEN_BITS_2__IB_FIFO_WATERMARK__SHIFT                                                        0x12
+#define SDMA1_CHICKEN_BITS_2__RESERVED_22_20__SHIFT                                                           0x14
+#define SDMA1_CHICKEN_BITS_2__CH_RD_WATERMARK__SHIFT                                                          0x17
+#define SDMA1_CHICKEN_BITS_2__CH_WR_WATERMARK__SHIFT                                                          0x19
+#define SDMA1_CHICKEN_BITS_2__CH_WR_WATERMARK_LSB__SHIFT                                                      0x1e
+#define SDMA1_CHICKEN_BITS_2__PIO_VFID_SOURCE__SHIFT                                                          0x1f
+#define SDMA1_CHICKEN_BITS_2__F32_CMD_PROC_DELAY_MASK                                                         0x0000000FL
+#define SDMA1_CHICKEN_BITS_2__F32_SEND_POSTCODE_EN_MASK                                                       0x00000010L
+#define SDMA1_CHICKEN_BITS_2__UCODE_BUF_DS_EN_MASK                                                            0x00000040L
+#define SDMA1_CHICKEN_BITS_2__UCODE_SELFLOAD_THREAD_OVERLAP_MASK                                              0x00000080L
+#define SDMA1_CHICKEN_BITS_2__WPTR_POLL_OUTSTANDING_MASK                                                      0x00000F00L
+#define SDMA1_CHICKEN_BITS_2__RESERVED_14_12_MASK                                                             0x00007000L
+#define SDMA1_CHICKEN_BITS_2__RESERVED_15_MASK                                                                0x00008000L
+#define SDMA1_CHICKEN_BITS_2__RB_FIFO_WATERMARK_MASK                                                          0x00030000L
+#define SDMA1_CHICKEN_BITS_2__IB_FIFO_WATERMARK_MASK                                                          0x000C0000L
+#define SDMA1_CHICKEN_BITS_2__RESERVED_22_20_MASK                                                             0x00700000L
+#define SDMA1_CHICKEN_BITS_2__CH_RD_WATERMARK_MASK                                                            0x01800000L
+#define SDMA1_CHICKEN_BITS_2__CH_WR_WATERMARK_MASK                                                            0x3E000000L
+#define SDMA1_CHICKEN_BITS_2__CH_WR_WATERMARK_LSB_MASK                                                        0x40000000L
+#define SDMA1_CHICKEN_BITS_2__PIO_VFID_SOURCE_MASK                                                            0x80000000L
+//SDMA1_STATUS3_REG
+#define SDMA1_STATUS3_REG__CMD_OP_STATUS__SHIFT                                                               0x0
+#define SDMA1_STATUS3_REG__PREV_VM_CMD__SHIFT                                                                 0x10
+#define SDMA1_STATUS3_REG__EXCEPTION_IDLE__SHIFT                                                              0x14
+#define SDMA1_STATUS3_REG__AQL_PREV_CMD_IDLE__SHIFT                                                           0x15
+#define SDMA1_STATUS3_REG__TLBI_IDLE__SHIFT                                                                   0x16
+#define SDMA1_STATUS3_REG__GCR_IDLE__SHIFT                                                                    0x17
+#define SDMA1_STATUS3_REG__INVREQ_IDLE__SHIFT                                                                 0x18
+#define SDMA1_STATUS3_REG__QUEUE_ID_MATCH__SHIFT                                                              0x19
+#define SDMA1_STATUS3_REG__INT_QUEUE_ID__SHIFT                                                                0x1a
+#define SDMA1_STATUS3_REG__TMZ_MTYPE_STATUS__SHIFT                                                            0x1e
+#define SDMA1_STATUS3_REG__CMD_OP_STATUS_MASK                                                                 0x0000FFFFL
+#define SDMA1_STATUS3_REG__PREV_VM_CMD_MASK                                                                   0x000F0000L
+#define SDMA1_STATUS3_REG__EXCEPTION_IDLE_MASK                                                                0x00100000L
+#define SDMA1_STATUS3_REG__AQL_PREV_CMD_IDLE_MASK                                                             0x00200000L
+#define SDMA1_STATUS3_REG__TLBI_IDLE_MASK                                                                     0x00400000L
+#define SDMA1_STATUS3_REG__GCR_IDLE_MASK                                                                      0x00800000L
+#define SDMA1_STATUS3_REG__INVREQ_IDLE_MASK                                                                   0x01000000L
+#define SDMA1_STATUS3_REG__QUEUE_ID_MATCH_MASK                                                                0x02000000L
+#define SDMA1_STATUS3_REG__INT_QUEUE_ID_MASK                                                                  0x3C000000L
+#define SDMA1_STATUS3_REG__TMZ_MTYPE_STATUS_MASK                                                              0xC0000000L
+//SDMA1_PHYSICAL_ADDR_LO
+#define SDMA1_PHYSICAL_ADDR_LO__D_VALID__SHIFT                                                                0x0
+#define SDMA1_PHYSICAL_ADDR_LO__DIRTY__SHIFT                                                                  0x1
+#define SDMA1_PHYSICAL_ADDR_LO__PHY_VALID__SHIFT                                                              0x2
+#define SDMA1_PHYSICAL_ADDR_LO__ADDR__SHIFT                                                                   0xc
+#define SDMA1_PHYSICAL_ADDR_LO__D_VALID_MASK                                                                  0x00000001L
+#define SDMA1_PHYSICAL_ADDR_LO__DIRTY_MASK                                                                    0x00000002L
+#define SDMA1_PHYSICAL_ADDR_LO__PHY_VALID_MASK                                                                0x00000004L
+#define SDMA1_PHYSICAL_ADDR_LO__ADDR_MASK                                                                     0xFFFFF000L
+//SDMA1_PHYSICAL_ADDR_HI
+#define SDMA1_PHYSICAL_ADDR_HI__ADDR__SHIFT                                                                   0x0
+#define SDMA1_PHYSICAL_ADDR_HI__ADDR_MASK                                                                     0x0000FFFFL
+//SDMA1_GLOBAL_QUANTUM
+#define SDMA1_GLOBAL_QUANTUM__GLOBAL_FOCUS_QUANTUM__SHIFT                                                     0x0
+#define SDMA1_GLOBAL_QUANTUM__GLOBAL_NORMAL_QUANTUM__SHIFT                                                    0x8
+#define SDMA1_GLOBAL_QUANTUM__GLOBAL_FOCUS_QUANTUM_MASK                                                       0x000000FFL
+#define SDMA1_GLOBAL_QUANTUM__GLOBAL_NORMAL_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA1_ERROR_LOG
+#define SDMA1_ERROR_LOG__OVERRIDE__SHIFT                                                                      0x0
+#define SDMA1_ERROR_LOG__STATUS__SHIFT                                                                        0x10
+#define SDMA1_ERROR_LOG__OVERRIDE_MASK                                                                        0x0000FFFFL
+#define SDMA1_ERROR_LOG__STATUS_MASK                                                                          0xFFFF0000L
+//SDMA1_PUB_DUMMY_REG0
+#define SDMA1_PUB_DUMMY_REG0__VALUE__SHIFT                                                                    0x0
+#define SDMA1_PUB_DUMMY_REG0__VALUE_MASK                                                                      0xFFFFFFFFL
+//SDMA1_PUB_DUMMY_REG1
+#define SDMA1_PUB_DUMMY_REG1__VALUE__SHIFT                                                                    0x0
+#define SDMA1_PUB_DUMMY_REG1__VALUE_MASK                                                                      0xFFFFFFFFL
+//SDMA1_PUB_DUMMY_REG2
+#define SDMA1_PUB_DUMMY_REG2__VALUE__SHIFT                                                                    0x0
+#define SDMA1_PUB_DUMMY_REG2__VALUE_MASK                                                                      0xFFFFFFFFL
+//SDMA1_PUB_DUMMY_REG3
+#define SDMA1_PUB_DUMMY_REG3__VALUE__SHIFT                                                                    0x0
+#define SDMA1_PUB_DUMMY_REG3__VALUE_MASK                                                                      0xFFFFFFFFL
+//SDMA1_F32_COUNTER
+#define SDMA1_F32_COUNTER__VALUE__SHIFT                                                                       0x0
+#define SDMA1_F32_COUNTER__VALUE_MASK                                                                         0xFFFFFFFFL
+//SDMA1_CRD_CNTL
+#define SDMA1_CRD_CNTL__DRM_CREDIT__SHIFT                                                                     0x0
+#define SDMA1_CRD_CNTL__MC_WRREQ_CREDIT__SHIFT                                                                0x7
+#define SDMA1_CRD_CNTL__MC_RDREQ_CREDIT__SHIFT                                                                0xd
+#define SDMA1_CRD_CNTL__CH_WRREQ_CREDIT__SHIFT                                                                0x13
+#define SDMA1_CRD_CNTL__CH_RDREQ_CREDIT__SHIFT                                                                0x19
+#define SDMA1_CRD_CNTL__DRM_CREDIT_MASK                                                                       0x0000007FL
+#define SDMA1_CRD_CNTL__MC_WRREQ_CREDIT_MASK                                                                  0x00001F80L
+#define SDMA1_CRD_CNTL__MC_RDREQ_CREDIT_MASK                                                                  0x0007E000L
+#define SDMA1_CRD_CNTL__CH_WRREQ_CREDIT_MASK                                                                  0x01F80000L
+#define SDMA1_CRD_CNTL__CH_RDREQ_CREDIT_MASK                                                                  0x7E000000L
+//SDMA1_RLC_CGCG_CTRL
+#define SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE__SHIFT                                                           0x1
+#define SDMA1_RLC_CGCG_CTRL__CGCG_IDLE_HYSTERESIS__SHIFT                                                      0x10
+#define SDMA1_RLC_CGCG_CTRL__CGCG_INT_ENABLE_MASK                                                             0x00000002L
+#define SDMA1_RLC_CGCG_CTRL__CGCG_IDLE_HYSTERESIS_MASK                                                        0xFFFF0000L
+//SDMA1_GPU_IOV_VIOLATION_LOG
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS__SHIFT                                                  0x0
+#define SDMA1_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS__SHIFT                                         0x1
+#define SDMA1_GPU_IOV_VIOLATION_LOG__ADDRESS__SHIFT                                                           0x2
+#define SDMA1_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION__SHIFT                                                   0x14
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VF__SHIFT                                                                0x15
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VFID__SHIFT                                                              0x16
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VIOLATION_STATUS_MASK                                                    0x00000001L
+#define SDMA1_GPU_IOV_VIOLATION_LOG__MULTIPLE_VIOLATION_STATUS_MASK                                           0x00000002L
+#define SDMA1_GPU_IOV_VIOLATION_LOG__ADDRESS_MASK                                                             0x000FFFFCL
+#define SDMA1_GPU_IOV_VIOLATION_LOG__WRITE_OPERATION_MASK                                                     0x00100000L
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VF_MASK                                                                  0x00200000L
+#define SDMA1_GPU_IOV_VIOLATION_LOG__VFID_MASK                                                                0x03C00000L
+//SDMA1_AQL_STATUS
+#define SDMA1_AQL_STATUS__COMPLETE_SIGNAL_EMPTY__SHIFT                                                        0x0
+#define SDMA1_AQL_STATUS__INVALID_CMD_EMPTY__SHIFT                                                            0x1
+#define SDMA1_AQL_STATUS__COMPLETE_SIGNAL_EMPTY_MASK                                                          0x00000001L
+#define SDMA1_AQL_STATUS__INVALID_CMD_EMPTY_MASK                                                              0x00000002L
+//SDMA1_EA_DBIT_ADDR_DATA
+#define SDMA1_EA_DBIT_ADDR_DATA__VALUE__SHIFT                                                                 0x0
+#define SDMA1_EA_DBIT_ADDR_DATA__VALUE_MASK                                                                   0xFFFFFFFFL
+//SDMA1_EA_DBIT_ADDR_INDEX
+#define SDMA1_EA_DBIT_ADDR_INDEX__VALUE__SHIFT                                                                0x0
+#define SDMA1_EA_DBIT_ADDR_INDEX__VALUE_MASK                                                                  0x00000007L
+//SDMA1_TLBI_GCR_CNTL
+#define SDMA1_TLBI_GCR_CNTL__TLBI_CMD_DW__SHIFT                                                               0x0
+#define SDMA1_TLBI_GCR_CNTL__GCR_CMD_DW__SHIFT                                                                0x4
+#define SDMA1_TLBI_GCR_CNTL__GCR_CLKEN_CYCLE__SHIFT                                                           0x8
+#define SDMA1_TLBI_GCR_CNTL__TLBI_CREDIT__SHIFT                                                               0x10
+#define SDMA1_TLBI_GCR_CNTL__GCR_CREDIT__SHIFT                                                                0x18
+#define SDMA1_TLBI_GCR_CNTL__TLBI_CMD_DW_MASK                                                                 0x0000000FL
+#define SDMA1_TLBI_GCR_CNTL__GCR_CMD_DW_MASK                                                                  0x000000F0L
+#define SDMA1_TLBI_GCR_CNTL__GCR_CLKEN_CYCLE_MASK                                                             0x00000F00L
+#define SDMA1_TLBI_GCR_CNTL__TLBI_CREDIT_MASK                                                                 0x00FF0000L
+#define SDMA1_TLBI_GCR_CNTL__GCR_CREDIT_MASK                                                                  0xFF000000L
+//SDMA1_TILING_CONFIG
+#define SDMA1_TILING_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                      0x4
+#define SDMA1_TILING_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                        0x00000070L
+//SDMA1_HASH
+#define SDMA1_HASH__CHANNEL_BITS__SHIFT                                                                       0x0
+#define SDMA1_HASH__BANK_BITS__SHIFT                                                                          0x4
+#define SDMA1_HASH__CHANNEL_XOR_COUNT__SHIFT                                                                  0x8
+#define SDMA1_HASH__BANK_XOR_COUNT__SHIFT                                                                     0xc
+#define SDMA1_HASH__CHANNEL_BITS_MASK                                                                         0x00000007L
+#define SDMA1_HASH__BANK_BITS_MASK                                                                            0x00000070L
+#define SDMA1_HASH__CHANNEL_XOR_COUNT_MASK                                                                    0x00000700L
+#define SDMA1_HASH__BANK_XOR_COUNT_MASK                                                                       0x00007000L
+//SDMA1_INT_STATUS
+#define SDMA1_INT_STATUS__DATA__SHIFT                                                                         0x0
+#define SDMA1_INT_STATUS__DATA_MASK                                                                           0xFFFFFFFFL
+//SDMA1_GPU_IOV_VIOLATION_LOG2
+#define SDMA1_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID__SHIFT                                                     0x0
+#define SDMA1_GPU_IOV_VIOLATION_LOG2__INITIATOR_ID_MASK                                                       0x000003FFL
+//SDMA1_HOLE_ADDR_LO
+#define SDMA1_HOLE_ADDR_LO__VALUE__SHIFT                                                                      0x0
+#define SDMA1_HOLE_ADDR_LO__VALUE_MASK                                                                        0xFFFFFFFFL
+//SDMA1_HOLE_ADDR_HI
+#define SDMA1_HOLE_ADDR_HI__VALUE__SHIFT                                                                      0x0
+#define SDMA1_HOLE_ADDR_HI__VALUE_MASK                                                                        0xFFFFFFFFL
+//SDMA1_CLOCK_GATING_STATUS
+#define SDMA1_CLOCK_GATING_STATUS__DYN_CLK_GATE_STATUS__SHIFT                                                 0x0
+#define SDMA1_CLOCK_GATING_STATUS__CE_CLK_GATE_STATUS__SHIFT                                                  0x2
+#define SDMA1_CLOCK_GATING_STATUS__CE_BC_CLK_GATE_STATUS__SHIFT                                               0x3
+#define SDMA1_CLOCK_GATING_STATUS__CE_NBC_CLK_GATE_STATUS__SHIFT                                              0x4
+#define SDMA1_CLOCK_GATING_STATUS__REG_CLK_GATE_STATUS__SHIFT                                                 0x5
+#define SDMA1_CLOCK_GATING_STATUS__F32_CLK_GATE_STATUS__SHIFT                                                 0x6
+#define SDMA1_CLOCK_GATING_STATUS__DYN_CLK_GATE_STATUS_MASK                                                   0x00000001L
+#define SDMA1_CLOCK_GATING_STATUS__CE_CLK_GATE_STATUS_MASK                                                    0x00000004L
+#define SDMA1_CLOCK_GATING_STATUS__CE_BC_CLK_GATE_STATUS_MASK                                                 0x00000008L
+#define SDMA1_CLOCK_GATING_STATUS__CE_NBC_CLK_GATE_STATUS_MASK                                                0x00000010L
+#define SDMA1_CLOCK_GATING_STATUS__REG_CLK_GATE_STATUS_MASK                                                   0x00000020L
+#define SDMA1_CLOCK_GATING_STATUS__F32_CLK_GATE_STATUS_MASK                                                   0x00000040L
+//SDMA1_STATUS4_REG
+#define SDMA1_STATUS4_REG__IDLE__SHIFT                                                                        0x0
+#define SDMA1_STATUS4_REG__IH_OUTSTANDING__SHIFT                                                              0x2
+#define SDMA1_STATUS4_REG__SEM_OUTSTANDING__SHIFT                                                             0x3
+#define SDMA1_STATUS4_REG__CH_RD_OUTSTANDING__SHIFT                                                           0x4
+#define SDMA1_STATUS4_REG__CH_WR_OUTSTANDING__SHIFT                                                           0x5
+#define SDMA1_STATUS4_REG__GCR_OUTSTANDING__SHIFT                                                             0x6
+#define SDMA1_STATUS4_REG__TLBI_OUTSTANDING__SHIFT                                                            0x7
+#define SDMA1_STATUS4_REG__UTCL2_RD_OUTSTANDING__SHIFT                                                        0x8
+#define SDMA1_STATUS4_REG__UTCL2_WR_OUTSTANDING__SHIFT                                                        0x9
+#define SDMA1_STATUS4_REG__REG_POLLING__SHIFT                                                                 0xa
+#define SDMA1_STATUS4_REG__MEM_POLLING__SHIFT                                                                 0xb
+#define SDMA1_STATUS4_REG__RESERVED_13_12__SHIFT                                                              0xc
+#define SDMA1_STATUS4_REG__RESERVED_15_14__SHIFT                                                              0xe
+#define SDMA1_STATUS4_REG__ACTIVE_QUEUE_ID__SHIFT                                                             0x10
+#define SDMA1_STATUS4_REG__SRIOV_WATING_RLCV_CMD__SHIFT                                                       0x14
+#define SDMA1_STATUS4_REG__SRIOV_SDMA_EXECUTING_CMD__SHIFT                                                    0x15
+#define SDMA1_STATUS4_REG__UTCL2_RD_XNACK_FAULT__SHIFT                                                        0x16
+#define SDMA1_STATUS4_REG__UTCL2_RD_XNACK_NULL__SHIFT                                                         0x17
+#define SDMA1_STATUS4_REG__UTCL2_RD_XNACK_TIMEOUT__SHIFT                                                      0x18
+#define SDMA1_STATUS4_REG__UTCL2_WR_XNACK_FAULT__SHIFT                                                        0x19
+#define SDMA1_STATUS4_REG__UTCL2_WR_XNACK_NULL__SHIFT                                                         0x1a
+#define SDMA1_STATUS4_REG__UTCL2_WR_XNACK_TIMEOUT__SHIFT                                                      0x1b
+#define SDMA1_STATUS4_REG__IDLE_MASK                                                                          0x00000001L
+#define SDMA1_STATUS4_REG__IH_OUTSTANDING_MASK                                                                0x00000004L
+#define SDMA1_STATUS4_REG__SEM_OUTSTANDING_MASK                                                               0x00000008L
+#define SDMA1_STATUS4_REG__CH_RD_OUTSTANDING_MASK                                                             0x00000010L
+#define SDMA1_STATUS4_REG__CH_WR_OUTSTANDING_MASK                                                             0x00000020L
+#define SDMA1_STATUS4_REG__GCR_OUTSTANDING_MASK                                                               0x00000040L
+#define SDMA1_STATUS4_REG__TLBI_OUTSTANDING_MASK                                                              0x00000080L
+#define SDMA1_STATUS4_REG__UTCL2_RD_OUTSTANDING_MASK                                                          0x00000100L
+#define SDMA1_STATUS4_REG__UTCL2_WR_OUTSTANDING_MASK                                                          0x00000200L
+#define SDMA1_STATUS4_REG__REG_POLLING_MASK                                                                   0x00000400L
+#define SDMA1_STATUS4_REG__MEM_POLLING_MASK                                                                   0x00000800L
+#define SDMA1_STATUS4_REG__RESERVED_13_12_MASK                                                                0x00003000L
+#define SDMA1_STATUS4_REG__RESERVED_15_14_MASK                                                                0x0000C000L
+#define SDMA1_STATUS4_REG__ACTIVE_QUEUE_ID_MASK                                                               0x000F0000L
+#define SDMA1_STATUS4_REG__SRIOV_WATING_RLCV_CMD_MASK                                                         0x00100000L
+#define SDMA1_STATUS4_REG__SRIOV_SDMA_EXECUTING_CMD_MASK                                                      0x00200000L
+#define SDMA1_STATUS4_REG__UTCL2_RD_XNACK_FAULT_MASK                                                          0x00400000L
+#define SDMA1_STATUS4_REG__UTCL2_RD_XNACK_NULL_MASK                                                           0x00800000L
+#define SDMA1_STATUS4_REG__UTCL2_RD_XNACK_TIMEOUT_MASK                                                        0x01000000L
+#define SDMA1_STATUS4_REG__UTCL2_WR_XNACK_FAULT_MASK                                                          0x02000000L
+#define SDMA1_STATUS4_REG__UTCL2_WR_XNACK_NULL_MASK                                                           0x04000000L
+#define SDMA1_STATUS4_REG__UTCL2_WR_XNACK_TIMEOUT_MASK                                                        0x08000000L
+//SDMA1_SCRATCH_RAM_DATA
+#define SDMA1_SCRATCH_RAM_DATA__DATA__SHIFT                                                                   0x0
+#define SDMA1_SCRATCH_RAM_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//SDMA1_SCRATCH_RAM_ADDR
+#define SDMA1_SCRATCH_RAM_ADDR__ADDR__SHIFT                                                                   0x0
+#define SDMA1_SCRATCH_RAM_ADDR__ADDR_MASK                                                                     0x0000007FL
+//SDMA1_TIMESTAMP_CNTL
+#define SDMA1_TIMESTAMP_CNTL__CAPTURE__SHIFT                                                                  0x0
+#define SDMA1_TIMESTAMP_CNTL__CAPTURE_MASK                                                                    0x00000001L
+//SDMA1_STATUS5_REG
+#define SDMA1_STATUS5_REG__QUEUE0_RB_ENABLE_STATUS__SHIFT                                                     0x0
+#define SDMA1_STATUS5_REG__QUEUE1_RB_ENABLE_STATUS__SHIFT                                                     0x1
+#define SDMA1_STATUS5_REG__QUEUE2_RB_ENABLE_STATUS__SHIFT                                                     0x2
+#define SDMA1_STATUS5_REG__QUEUE3_RB_ENABLE_STATUS__SHIFT                                                     0x3
+#define SDMA1_STATUS5_REG__QUEUE4_RB_ENABLE_STATUS__SHIFT                                                     0x4
+#define SDMA1_STATUS5_REG__QUEUE5_RB_ENABLE_STATUS__SHIFT                                                     0x5
+#define SDMA1_STATUS5_REG__QUEUE6_RB_ENABLE_STATUS__SHIFT                                                     0x6
+#define SDMA1_STATUS5_REG__QUEUE7_RB_ENABLE_STATUS__SHIFT                                                     0x7
+#define SDMA1_STATUS5_REG__ACTIVE_QUEUE_ID__SHIFT                                                             0x10
+#define SDMA1_STATUS5_REG__QUEUE0_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x14
+#define SDMA1_STATUS5_REG__QUEUE1_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x15
+#define SDMA1_STATUS5_REG__QUEUE2_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x16
+#define SDMA1_STATUS5_REG__QUEUE3_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x17
+#define SDMA1_STATUS5_REG__QUEUE4_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x18
+#define SDMA1_STATUS5_REG__QUEUE5_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x19
+#define SDMA1_STATUS5_REG__QUEUE6_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x1a
+#define SDMA1_STATUS5_REG__QUEUE7_WPTR_POLL_PAGE_EXCEPTION__SHIFT                                             0x1b
+#define SDMA1_STATUS5_REG__QUEUE0_RB_ENABLE_STATUS_MASK                                                       0x00000001L
+#define SDMA1_STATUS5_REG__QUEUE1_RB_ENABLE_STATUS_MASK                                                       0x00000002L
+#define SDMA1_STATUS5_REG__QUEUE2_RB_ENABLE_STATUS_MASK                                                       0x00000004L
+#define SDMA1_STATUS5_REG__QUEUE3_RB_ENABLE_STATUS_MASK                                                       0x00000008L
+#define SDMA1_STATUS5_REG__QUEUE4_RB_ENABLE_STATUS_MASK                                                       0x00000010L
+#define SDMA1_STATUS5_REG__QUEUE5_RB_ENABLE_STATUS_MASK                                                       0x00000020L
+#define SDMA1_STATUS5_REG__QUEUE6_RB_ENABLE_STATUS_MASK                                                       0x00000040L
+#define SDMA1_STATUS5_REG__QUEUE7_RB_ENABLE_STATUS_MASK                                                       0x00000080L
+#define SDMA1_STATUS5_REG__ACTIVE_QUEUE_ID_MASK                                                               0x000F0000L
+#define SDMA1_STATUS5_REG__QUEUE0_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x00100000L
+#define SDMA1_STATUS5_REG__QUEUE1_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x00200000L
+#define SDMA1_STATUS5_REG__QUEUE2_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x00400000L
+#define SDMA1_STATUS5_REG__QUEUE3_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x00800000L
+#define SDMA1_STATUS5_REG__QUEUE4_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x01000000L
+#define SDMA1_STATUS5_REG__QUEUE5_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x02000000L
+#define SDMA1_STATUS5_REG__QUEUE6_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x04000000L
+#define SDMA1_STATUS5_REG__QUEUE7_WPTR_POLL_PAGE_EXCEPTION_MASK                                               0x08000000L
+//SDMA1_QUEUE_RESET_REQ
+#define SDMA1_QUEUE_RESET_REQ__QUEUE0_RESET__SHIFT                                                            0x0
+#define SDMA1_QUEUE_RESET_REQ__QUEUE1_RESET__SHIFT                                                            0x1
+#define SDMA1_QUEUE_RESET_REQ__QUEUE2_RESET__SHIFT                                                            0x2
+#define SDMA1_QUEUE_RESET_REQ__QUEUE3_RESET__SHIFT                                                            0x3
+#define SDMA1_QUEUE_RESET_REQ__QUEUE4_RESET__SHIFT                                                            0x4
+#define SDMA1_QUEUE_RESET_REQ__QUEUE5_RESET__SHIFT                                                            0x5
+#define SDMA1_QUEUE_RESET_REQ__QUEUE6_RESET__SHIFT                                                            0x6
+#define SDMA1_QUEUE_RESET_REQ__QUEUE7_RESET__SHIFT                                                            0x7
+#define SDMA1_QUEUE_RESET_REQ__RESERVED__SHIFT                                                                0x8
+#define SDMA1_QUEUE_RESET_REQ__QUEUE0_RESET_MASK                                                              0x00000001L
+#define SDMA1_QUEUE_RESET_REQ__QUEUE1_RESET_MASK                                                              0x00000002L
+#define SDMA1_QUEUE_RESET_REQ__QUEUE2_RESET_MASK                                                              0x00000004L
+#define SDMA1_QUEUE_RESET_REQ__QUEUE3_RESET_MASK                                                              0x00000008L
+#define SDMA1_QUEUE_RESET_REQ__QUEUE4_RESET_MASK                                                              0x00000010L
+#define SDMA1_QUEUE_RESET_REQ__QUEUE5_RESET_MASK                                                              0x00000020L
+#define SDMA1_QUEUE_RESET_REQ__QUEUE6_RESET_MASK                                                              0x00000040L
+#define SDMA1_QUEUE_RESET_REQ__QUEUE7_RESET_MASK                                                              0x00000080L
+#define SDMA1_QUEUE_RESET_REQ__RESERVED_MASK                                                                  0xFFFFFF00L
+//SDMA1_STATUS6_REG
+#define SDMA1_STATUS6_REG__ID__SHIFT                                                                          0x0
+#define SDMA1_STATUS6_REG__TH1F32_INSTR_PTR__SHIFT                                                            0x2
+#define SDMA1_STATUS6_REG__TH1_EXCEPTION__SHIFT                                                               0x10
+#define SDMA1_STATUS6_REG__ID_MASK                                                                            0x00000003L
+#define SDMA1_STATUS6_REG__TH1F32_INSTR_PTR_MASK                                                              0x0000FFFCL
+#define SDMA1_STATUS6_REG__TH1_EXCEPTION_MASK                                                                 0xFFFF0000L
+//SDMA1_UCODE1_CHECKSUM
+#define SDMA1_UCODE1_CHECKSUM__DATA__SHIFT                                                                    0x0
+#define SDMA1_UCODE1_CHECKSUM__DATA_MASK                                                                      0xFFFFFFFFL
+//SDMA1_CE_CTRL
+#define SDMA1_CE_CTRL__RD_LUT_WATERMARK__SHIFT                                                                0x0
+#define SDMA1_CE_CTRL__RD_LUT_DEPTH__SHIFT                                                                    0x3
+#define SDMA1_CE_CTRL__WR_AFIFO_WATERMARK__SHIFT                                                              0x5
+#define SDMA1_CE_CTRL__CE_DCC_READ_128B_ENABLE__SHIFT                                                         0x8
+#define SDMA1_CE_CTRL__RESERVED__SHIFT                                                                        0x9
+#define SDMA1_CE_CTRL__RD_LUT_WATERMARK_MASK                                                                  0x00000007L
+#define SDMA1_CE_CTRL__RD_LUT_DEPTH_MASK                                                                      0x00000018L
+#define SDMA1_CE_CTRL__WR_AFIFO_WATERMARK_MASK                                                                0x000000E0L
+#define SDMA1_CE_CTRL__CE_DCC_READ_128B_ENABLE_MASK                                                           0x00000100L
+#define SDMA1_CE_CTRL__RESERVED_MASK                                                                          0xFFFFFE00L
+//SDMA1_FED_STATUS
+#define SDMA1_FED_STATUS__RB_FETCH_ECC__SHIFT                                                                 0x0
+#define SDMA1_FED_STATUS__IB_FETCH_ECC__SHIFT                                                                 0x1
+#define SDMA1_FED_STATUS__F32_DATA_ECC__SHIFT                                                                 0x2
+#define SDMA1_FED_STATUS__WPTR_ATOMIC_ECC__SHIFT                                                              0x3
+#define SDMA1_FED_STATUS__COPY_DATA_ECC__SHIFT                                                                0x4
+#define SDMA1_FED_STATUS__COPY_METADATA_ECC__SHIFT                                                            0x5
+#define SDMA1_FED_STATUS__SELFLOAD_UCODE_ECC__SHIFT                                                           0x6
+#define SDMA1_FED_STATUS__RB_FETCH_ECC_MASK                                                                   0x00000001L
+#define SDMA1_FED_STATUS__IB_FETCH_ECC_MASK                                                                   0x00000002L
+#define SDMA1_FED_STATUS__F32_DATA_ECC_MASK                                                                   0x00000004L
+#define SDMA1_FED_STATUS__WPTR_ATOMIC_ECC_MASK                                                                0x00000008L
+#define SDMA1_FED_STATUS__COPY_DATA_ECC_MASK                                                                  0x00000010L
+#define SDMA1_FED_STATUS__COPY_METADATA_ECC_MASK                                                              0x00000020L
+#define SDMA1_FED_STATUS__SELFLOAD_UCODE_ECC_MASK                                                             0x00000040L
+//SDMA1_QUEUE0_RB_CNTL
+#define SDMA1_QUEUE0_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE0_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA1_QUEUE0_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA1_QUEUE0_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA1_QUEUE0_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA1_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA1_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA1_QUEUE0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA1_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA1_QUEUE0_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA1_QUEUE0_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA1_QUEUE0_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE0_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA1_QUEUE0_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA1_QUEUE0_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA1_QUEUE0_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA1_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA1_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA1_QUEUE0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA1_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA1_QUEUE0_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA1_QUEUE0_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA1_QUEUE0_RB_BASE
+#define SDMA1_QUEUE0_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA1_QUEUE0_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA1_QUEUE0_RB_BASE_HI
+#define SDMA1_QUEUE0_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE0_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA1_QUEUE0_RB_RPTR
+#define SDMA1_QUEUE0_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE0_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE0_RB_RPTR_HI
+#define SDMA1_QUEUE0_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE0_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE0_RB_WPTR
+#define SDMA1_QUEUE0_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE0_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE0_RB_WPTR_HI
+#define SDMA1_QUEUE0_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE0_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE0_RB_RPTR_ADDR_HI
+#define SDMA1_QUEUE0_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_QUEUE0_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE0_RB_RPTR_ADDR_LO
+#define SDMA1_QUEUE0_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_QUEUE0_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_QUEUE0_IB_CNTL
+#define SDMA1_QUEUE0_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE0_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA1_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA1_QUEUE0_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA1_QUEUE0_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA1_QUEUE0_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE0_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA1_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA1_QUEUE0_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA1_QUEUE0_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA1_QUEUE0_IB_RPTR
+#define SDMA1_QUEUE0_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA1_QUEUE0_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA1_QUEUE0_IB_OFFSET
+#define SDMA1_QUEUE0_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA1_QUEUE0_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA1_QUEUE0_IB_BASE_LO
+#define SDMA1_QUEUE0_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA1_QUEUE0_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA1_QUEUE0_IB_BASE_HI
+#define SDMA1_QUEUE0_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE0_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE0_IB_SIZE
+#define SDMA1_QUEUE0_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA1_QUEUE0_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA1_QUEUE0_SKIP_CNTL
+#define SDMA1_QUEUE0_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA1_QUEUE0_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA1_QUEUE0_CONTEXT_STATUS
+#define SDMA1_QUEUE0_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA1_QUEUE0_CONTEXT_STATUS__USE_IB__SHIFT                                                            0x1
+#define SDMA1_QUEUE0_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA1_QUEUE0_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA1_QUEUE0_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA1_QUEUE0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA1_QUEUE0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA1_QUEUE0_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA1_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA1_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA1_QUEUE0_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA1_QUEUE0_CONTEXT_STATUS__USE_IB_MASK                                                              0x00000002L
+#define SDMA1_QUEUE0_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA1_QUEUE0_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA1_QUEUE0_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA1_QUEUE0_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA1_QUEUE0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA1_QUEUE0_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA1_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA1_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA1_QUEUE0_DOORBELL
+#define SDMA1_QUEUE0_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA1_QUEUE0_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA1_QUEUE0_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA1_QUEUE0_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA1_QUEUE0_DOORBELL_LOG
+#define SDMA1_QUEUE0_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA1_QUEUE0_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA1_QUEUE0_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA1_QUEUE0_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA1_QUEUE0_DOORBELL_OFFSET
+#define SDMA1_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA1_QUEUE0_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA1_QUEUE0_CSA_ADDR_LO
+#define SDMA1_QUEUE0_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA1_QUEUE0_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA1_QUEUE0_CSA_ADDR_HI
+#define SDMA1_QUEUE0_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA1_QUEUE0_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA1_QUEUE0_SCHEDULE_CNTL
+#define SDMA1_QUEUE0_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA1_QUEUE0_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA1_QUEUE0_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA1_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA1_QUEUE0_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA1_QUEUE0_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA1_QUEUE0_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA1_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA1_QUEUE0_IB_SUB_REMAIN
+#define SDMA1_QUEUE0_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA1_QUEUE0_PREEMPT
+#define SDMA1_QUEUE0_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA1_QUEUE0_DUMMY_REG
+#define SDMA1_QUEUE0_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA1_QUEUE0_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE0_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_QUEUE0_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_QUEUE0_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_QUEUE0_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_QUEUE0_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_QUEUE0_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_QUEUE0_RB_AQL_CNTL
+#define SDMA1_QUEUE0_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA1_QUEUE0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA1_QUEUE0_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA1_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA1_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA1_QUEUE0_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA1_QUEUE0_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_QUEUE0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA1_QUEUE0_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA1_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA1_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA1_QUEUE0_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA1_QUEUE0_MINOR_PTR_UPDATE
+#define SDMA1_QUEUE0_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA1_QUEUE0_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA1_QUEUE0_RB_PREEMPT
+#define SDMA1_QUEUE0_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA1_QUEUE0_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA1_QUEUE0_MIDCMD_DATA0
+#define SDMA1_QUEUE0_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_DATA1
+#define SDMA1_QUEUE0_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_DATA2
+#define SDMA1_QUEUE0_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_DATA3
+#define SDMA1_QUEUE0_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_DATA4
+#define SDMA1_QUEUE0_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_DATA5
+#define SDMA1_QUEUE0_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_DATA6
+#define SDMA1_QUEUE0_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_DATA7
+#define SDMA1_QUEUE0_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_DATA8
+#define SDMA1_QUEUE0_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_DATA9
+#define SDMA1_QUEUE0_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_DATA10
+#define SDMA1_QUEUE0_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA1_QUEUE0_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE0_MIDCMD_CNTL
+#define SDMA1_QUEUE0_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA1_QUEUE0_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA1_QUEUE0_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA1_QUEUE0_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA1_QUEUE0_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA1_QUEUE0_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA1_QUEUE0_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA1_QUEUE0_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA1_QUEUE1_RB_CNTL
+#define SDMA1_QUEUE1_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE1_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA1_QUEUE1_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA1_QUEUE1_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA1_QUEUE1_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA1_QUEUE1_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA1_QUEUE1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA1_QUEUE1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA1_QUEUE1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA1_QUEUE1_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA1_QUEUE1_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA1_QUEUE1_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE1_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA1_QUEUE1_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA1_QUEUE1_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA1_QUEUE1_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA1_QUEUE1_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA1_QUEUE1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA1_QUEUE1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA1_QUEUE1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA1_QUEUE1_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA1_QUEUE1_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA1_QUEUE1_RB_BASE
+#define SDMA1_QUEUE1_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA1_QUEUE1_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA1_QUEUE1_RB_BASE_HI
+#define SDMA1_QUEUE1_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE1_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA1_QUEUE1_RB_RPTR
+#define SDMA1_QUEUE1_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE1_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE1_RB_RPTR_HI
+#define SDMA1_QUEUE1_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE1_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE1_RB_WPTR
+#define SDMA1_QUEUE1_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE1_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE1_RB_WPTR_HI
+#define SDMA1_QUEUE1_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE1_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE1_RB_RPTR_ADDR_HI
+#define SDMA1_QUEUE1_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_QUEUE1_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE1_RB_RPTR_ADDR_LO
+#define SDMA1_QUEUE1_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_QUEUE1_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_QUEUE1_IB_CNTL
+#define SDMA1_QUEUE1_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE1_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA1_QUEUE1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA1_QUEUE1_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA1_QUEUE1_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA1_QUEUE1_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE1_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA1_QUEUE1_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA1_QUEUE1_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA1_QUEUE1_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA1_QUEUE1_IB_RPTR
+#define SDMA1_QUEUE1_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA1_QUEUE1_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA1_QUEUE1_IB_OFFSET
+#define SDMA1_QUEUE1_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA1_QUEUE1_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA1_QUEUE1_IB_BASE_LO
+#define SDMA1_QUEUE1_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA1_QUEUE1_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA1_QUEUE1_IB_BASE_HI
+#define SDMA1_QUEUE1_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE1_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE1_IB_SIZE
+#define SDMA1_QUEUE1_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA1_QUEUE1_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA1_QUEUE1_SKIP_CNTL
+#define SDMA1_QUEUE1_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA1_QUEUE1_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA1_QUEUE1_CONTEXT_STATUS
+#define SDMA1_QUEUE1_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA1_QUEUE1_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA1_QUEUE1_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA1_QUEUE1_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA1_QUEUE1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA1_QUEUE1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA1_QUEUE1_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA1_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA1_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA1_QUEUE1_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA1_QUEUE1_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA1_QUEUE1_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA1_QUEUE1_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA1_QUEUE1_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA1_QUEUE1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA1_QUEUE1_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA1_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA1_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA1_QUEUE1_DOORBELL
+#define SDMA1_QUEUE1_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA1_QUEUE1_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA1_QUEUE1_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA1_QUEUE1_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA1_QUEUE1_DOORBELL_LOG
+#define SDMA1_QUEUE1_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA1_QUEUE1_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA1_QUEUE1_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA1_QUEUE1_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA1_QUEUE1_DOORBELL_OFFSET
+#define SDMA1_QUEUE1_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA1_QUEUE1_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA1_QUEUE1_CSA_ADDR_LO
+#define SDMA1_QUEUE1_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA1_QUEUE1_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA1_QUEUE1_CSA_ADDR_HI
+#define SDMA1_QUEUE1_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA1_QUEUE1_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA1_QUEUE1_SCHEDULE_CNTL
+#define SDMA1_QUEUE1_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA1_QUEUE1_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA1_QUEUE1_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA1_QUEUE1_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA1_QUEUE1_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA1_QUEUE1_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA1_QUEUE1_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA1_QUEUE1_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA1_QUEUE1_IB_SUB_REMAIN
+#define SDMA1_QUEUE1_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA1_QUEUE1_PREEMPT
+#define SDMA1_QUEUE1_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA1_QUEUE1_DUMMY_REG
+#define SDMA1_QUEUE1_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA1_QUEUE1_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE1_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_QUEUE1_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_QUEUE1_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_QUEUE1_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_QUEUE1_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_QUEUE1_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_QUEUE1_RB_AQL_CNTL
+#define SDMA1_QUEUE1_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA1_QUEUE1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA1_QUEUE1_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA1_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA1_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA1_QUEUE1_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA1_QUEUE1_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_QUEUE1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA1_QUEUE1_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA1_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA1_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA1_QUEUE1_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA1_QUEUE1_MINOR_PTR_UPDATE
+#define SDMA1_QUEUE1_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA1_QUEUE1_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA1_QUEUE1_RB_PREEMPT
+#define SDMA1_QUEUE1_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA1_QUEUE1_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA1_QUEUE1_MIDCMD_DATA0
+#define SDMA1_QUEUE1_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_DATA1
+#define SDMA1_QUEUE1_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_DATA2
+#define SDMA1_QUEUE1_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_DATA3
+#define SDMA1_QUEUE1_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_DATA4
+#define SDMA1_QUEUE1_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_DATA5
+#define SDMA1_QUEUE1_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_DATA6
+#define SDMA1_QUEUE1_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_DATA7
+#define SDMA1_QUEUE1_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_DATA8
+#define SDMA1_QUEUE1_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_DATA9
+#define SDMA1_QUEUE1_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_DATA10
+#define SDMA1_QUEUE1_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA1_QUEUE1_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE1_MIDCMD_CNTL
+#define SDMA1_QUEUE1_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA1_QUEUE1_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA1_QUEUE1_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA1_QUEUE1_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA1_QUEUE1_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA1_QUEUE1_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA1_QUEUE1_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA1_QUEUE1_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA1_QUEUE2_RB_CNTL
+#define SDMA1_QUEUE2_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE2_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA1_QUEUE2_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA1_QUEUE2_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA1_QUEUE2_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA1_QUEUE2_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA1_QUEUE2_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA1_QUEUE2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA1_QUEUE2_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA1_QUEUE2_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA1_QUEUE2_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA1_QUEUE2_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE2_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA1_QUEUE2_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA1_QUEUE2_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA1_QUEUE2_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA1_QUEUE2_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA1_QUEUE2_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA1_QUEUE2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA1_QUEUE2_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA1_QUEUE2_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA1_QUEUE2_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA1_QUEUE2_RB_BASE
+#define SDMA1_QUEUE2_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA1_QUEUE2_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA1_QUEUE2_RB_BASE_HI
+#define SDMA1_QUEUE2_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE2_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA1_QUEUE2_RB_RPTR
+#define SDMA1_QUEUE2_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE2_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE2_RB_RPTR_HI
+#define SDMA1_QUEUE2_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE2_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE2_RB_WPTR
+#define SDMA1_QUEUE2_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE2_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE2_RB_WPTR_HI
+#define SDMA1_QUEUE2_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE2_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE2_RB_RPTR_ADDR_HI
+#define SDMA1_QUEUE2_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_QUEUE2_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE2_RB_RPTR_ADDR_LO
+#define SDMA1_QUEUE2_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_QUEUE2_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_QUEUE2_IB_CNTL
+#define SDMA1_QUEUE2_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE2_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA1_QUEUE2_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA1_QUEUE2_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA1_QUEUE2_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA1_QUEUE2_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE2_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA1_QUEUE2_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA1_QUEUE2_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA1_QUEUE2_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA1_QUEUE2_IB_RPTR
+#define SDMA1_QUEUE2_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA1_QUEUE2_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA1_QUEUE2_IB_OFFSET
+#define SDMA1_QUEUE2_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA1_QUEUE2_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA1_QUEUE2_IB_BASE_LO
+#define SDMA1_QUEUE2_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA1_QUEUE2_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA1_QUEUE2_IB_BASE_HI
+#define SDMA1_QUEUE2_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE2_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE2_IB_SIZE
+#define SDMA1_QUEUE2_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA1_QUEUE2_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA1_QUEUE2_SKIP_CNTL
+#define SDMA1_QUEUE2_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA1_QUEUE2_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA1_QUEUE2_CONTEXT_STATUS
+#define SDMA1_QUEUE2_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA1_QUEUE2_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA1_QUEUE2_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA1_QUEUE2_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA1_QUEUE2_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA1_QUEUE2_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA1_QUEUE2_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA1_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA1_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA1_QUEUE2_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA1_QUEUE2_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA1_QUEUE2_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA1_QUEUE2_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA1_QUEUE2_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA1_QUEUE2_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA1_QUEUE2_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA1_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA1_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA1_QUEUE2_DOORBELL
+#define SDMA1_QUEUE2_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA1_QUEUE2_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA1_QUEUE2_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA1_QUEUE2_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA1_QUEUE2_DOORBELL_LOG
+#define SDMA1_QUEUE2_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA1_QUEUE2_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA1_QUEUE2_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA1_QUEUE2_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA1_QUEUE2_DOORBELL_OFFSET
+#define SDMA1_QUEUE2_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA1_QUEUE2_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA1_QUEUE2_CSA_ADDR_LO
+#define SDMA1_QUEUE2_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA1_QUEUE2_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA1_QUEUE2_CSA_ADDR_HI
+#define SDMA1_QUEUE2_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA1_QUEUE2_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA1_QUEUE2_SCHEDULE_CNTL
+#define SDMA1_QUEUE2_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA1_QUEUE2_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA1_QUEUE2_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA1_QUEUE2_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA1_QUEUE2_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA1_QUEUE2_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA1_QUEUE2_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA1_QUEUE2_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA1_QUEUE2_IB_SUB_REMAIN
+#define SDMA1_QUEUE2_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA1_QUEUE2_PREEMPT
+#define SDMA1_QUEUE2_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA1_QUEUE2_DUMMY_REG
+#define SDMA1_QUEUE2_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA1_QUEUE2_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE2_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_QUEUE2_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_QUEUE2_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_QUEUE2_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_QUEUE2_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_QUEUE2_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_QUEUE2_RB_AQL_CNTL
+#define SDMA1_QUEUE2_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA1_QUEUE2_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA1_QUEUE2_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA1_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA1_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA1_QUEUE2_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA1_QUEUE2_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_QUEUE2_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA1_QUEUE2_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA1_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA1_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA1_QUEUE2_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA1_QUEUE2_MINOR_PTR_UPDATE
+#define SDMA1_QUEUE2_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA1_QUEUE2_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA1_QUEUE2_RB_PREEMPT
+#define SDMA1_QUEUE2_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA1_QUEUE2_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA1_QUEUE2_MIDCMD_DATA0
+#define SDMA1_QUEUE2_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_DATA1
+#define SDMA1_QUEUE2_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_DATA2
+#define SDMA1_QUEUE2_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_DATA3
+#define SDMA1_QUEUE2_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_DATA4
+#define SDMA1_QUEUE2_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_DATA5
+#define SDMA1_QUEUE2_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_DATA6
+#define SDMA1_QUEUE2_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_DATA7
+#define SDMA1_QUEUE2_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_DATA8
+#define SDMA1_QUEUE2_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_DATA9
+#define SDMA1_QUEUE2_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_DATA10
+#define SDMA1_QUEUE2_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA1_QUEUE2_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE2_MIDCMD_CNTL
+#define SDMA1_QUEUE2_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA1_QUEUE2_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA1_QUEUE2_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA1_QUEUE2_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA1_QUEUE2_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA1_QUEUE2_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA1_QUEUE2_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA1_QUEUE2_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA1_QUEUE3_RB_CNTL
+#define SDMA1_QUEUE3_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE3_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA1_QUEUE3_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA1_QUEUE3_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA1_QUEUE3_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA1_QUEUE3_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA1_QUEUE3_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA1_QUEUE3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA1_QUEUE3_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA1_QUEUE3_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA1_QUEUE3_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA1_QUEUE3_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE3_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA1_QUEUE3_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA1_QUEUE3_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA1_QUEUE3_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA1_QUEUE3_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA1_QUEUE3_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA1_QUEUE3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA1_QUEUE3_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA1_QUEUE3_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA1_QUEUE3_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA1_QUEUE3_RB_BASE
+#define SDMA1_QUEUE3_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA1_QUEUE3_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA1_QUEUE3_RB_BASE_HI
+#define SDMA1_QUEUE3_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE3_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA1_QUEUE3_RB_RPTR
+#define SDMA1_QUEUE3_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE3_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE3_RB_RPTR_HI
+#define SDMA1_QUEUE3_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE3_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE3_RB_WPTR
+#define SDMA1_QUEUE3_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE3_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE3_RB_WPTR_HI
+#define SDMA1_QUEUE3_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE3_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE3_RB_RPTR_ADDR_HI
+#define SDMA1_QUEUE3_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_QUEUE3_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE3_RB_RPTR_ADDR_LO
+#define SDMA1_QUEUE3_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_QUEUE3_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_QUEUE3_IB_CNTL
+#define SDMA1_QUEUE3_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE3_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA1_QUEUE3_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA1_QUEUE3_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA1_QUEUE3_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA1_QUEUE3_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE3_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA1_QUEUE3_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA1_QUEUE3_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA1_QUEUE3_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA1_QUEUE3_IB_RPTR
+#define SDMA1_QUEUE3_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA1_QUEUE3_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA1_QUEUE3_IB_OFFSET
+#define SDMA1_QUEUE3_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA1_QUEUE3_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA1_QUEUE3_IB_BASE_LO
+#define SDMA1_QUEUE3_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA1_QUEUE3_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA1_QUEUE3_IB_BASE_HI
+#define SDMA1_QUEUE3_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE3_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE3_IB_SIZE
+#define SDMA1_QUEUE3_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA1_QUEUE3_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA1_QUEUE3_SKIP_CNTL
+#define SDMA1_QUEUE3_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA1_QUEUE3_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA1_QUEUE3_CONTEXT_STATUS
+#define SDMA1_QUEUE3_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA1_QUEUE3_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA1_QUEUE3_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA1_QUEUE3_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA1_QUEUE3_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA1_QUEUE3_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA1_QUEUE3_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA1_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA1_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA1_QUEUE3_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA1_QUEUE3_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA1_QUEUE3_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA1_QUEUE3_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA1_QUEUE3_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA1_QUEUE3_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA1_QUEUE3_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA1_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA1_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA1_QUEUE3_DOORBELL
+#define SDMA1_QUEUE3_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA1_QUEUE3_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA1_QUEUE3_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA1_QUEUE3_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA1_QUEUE3_DOORBELL_LOG
+#define SDMA1_QUEUE3_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA1_QUEUE3_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA1_QUEUE3_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA1_QUEUE3_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA1_QUEUE3_DOORBELL_OFFSET
+#define SDMA1_QUEUE3_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA1_QUEUE3_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA1_QUEUE3_CSA_ADDR_LO
+#define SDMA1_QUEUE3_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA1_QUEUE3_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA1_QUEUE3_CSA_ADDR_HI
+#define SDMA1_QUEUE3_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA1_QUEUE3_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA1_QUEUE3_SCHEDULE_CNTL
+#define SDMA1_QUEUE3_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA1_QUEUE3_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA1_QUEUE3_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA1_QUEUE3_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA1_QUEUE3_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA1_QUEUE3_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA1_QUEUE3_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA1_QUEUE3_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA1_QUEUE3_IB_SUB_REMAIN
+#define SDMA1_QUEUE3_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA1_QUEUE3_PREEMPT
+#define SDMA1_QUEUE3_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA1_QUEUE3_DUMMY_REG
+#define SDMA1_QUEUE3_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA1_QUEUE3_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE3_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_QUEUE3_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_QUEUE3_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_QUEUE3_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_QUEUE3_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_QUEUE3_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_QUEUE3_RB_AQL_CNTL
+#define SDMA1_QUEUE3_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA1_QUEUE3_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA1_QUEUE3_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA1_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA1_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA1_QUEUE3_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA1_QUEUE3_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_QUEUE3_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA1_QUEUE3_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA1_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA1_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA1_QUEUE3_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA1_QUEUE3_MINOR_PTR_UPDATE
+#define SDMA1_QUEUE3_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA1_QUEUE3_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA1_QUEUE3_RB_PREEMPT
+#define SDMA1_QUEUE3_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA1_QUEUE3_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA1_QUEUE3_MIDCMD_DATA0
+#define SDMA1_QUEUE3_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_DATA1
+#define SDMA1_QUEUE3_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_DATA2
+#define SDMA1_QUEUE3_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_DATA3
+#define SDMA1_QUEUE3_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_DATA4
+#define SDMA1_QUEUE3_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_DATA5
+#define SDMA1_QUEUE3_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_DATA6
+#define SDMA1_QUEUE3_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_DATA7
+#define SDMA1_QUEUE3_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_DATA8
+#define SDMA1_QUEUE3_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_DATA9
+#define SDMA1_QUEUE3_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_DATA10
+#define SDMA1_QUEUE3_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA1_QUEUE3_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE3_MIDCMD_CNTL
+#define SDMA1_QUEUE3_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA1_QUEUE3_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA1_QUEUE3_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA1_QUEUE3_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA1_QUEUE3_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA1_QUEUE3_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA1_QUEUE3_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA1_QUEUE3_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA1_QUEUE4_RB_CNTL
+#define SDMA1_QUEUE4_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE4_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA1_QUEUE4_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA1_QUEUE4_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA1_QUEUE4_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA1_QUEUE4_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA1_QUEUE4_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA1_QUEUE4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA1_QUEUE4_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA1_QUEUE4_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA1_QUEUE4_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA1_QUEUE4_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE4_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA1_QUEUE4_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA1_QUEUE4_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA1_QUEUE4_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA1_QUEUE4_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA1_QUEUE4_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA1_QUEUE4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA1_QUEUE4_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA1_QUEUE4_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA1_QUEUE4_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA1_QUEUE4_RB_BASE
+#define SDMA1_QUEUE4_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA1_QUEUE4_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA1_QUEUE4_RB_BASE_HI
+#define SDMA1_QUEUE4_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE4_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA1_QUEUE4_RB_RPTR
+#define SDMA1_QUEUE4_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE4_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE4_RB_RPTR_HI
+#define SDMA1_QUEUE4_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE4_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE4_RB_WPTR
+#define SDMA1_QUEUE4_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE4_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE4_RB_WPTR_HI
+#define SDMA1_QUEUE4_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE4_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE4_RB_RPTR_ADDR_HI
+#define SDMA1_QUEUE4_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_QUEUE4_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE4_RB_RPTR_ADDR_LO
+#define SDMA1_QUEUE4_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_QUEUE4_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_QUEUE4_IB_CNTL
+#define SDMA1_QUEUE4_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE4_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA1_QUEUE4_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA1_QUEUE4_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA1_QUEUE4_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA1_QUEUE4_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE4_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA1_QUEUE4_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA1_QUEUE4_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA1_QUEUE4_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA1_QUEUE4_IB_RPTR
+#define SDMA1_QUEUE4_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA1_QUEUE4_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA1_QUEUE4_IB_OFFSET
+#define SDMA1_QUEUE4_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA1_QUEUE4_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA1_QUEUE4_IB_BASE_LO
+#define SDMA1_QUEUE4_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA1_QUEUE4_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA1_QUEUE4_IB_BASE_HI
+#define SDMA1_QUEUE4_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE4_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE4_IB_SIZE
+#define SDMA1_QUEUE4_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA1_QUEUE4_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA1_QUEUE4_SKIP_CNTL
+#define SDMA1_QUEUE4_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA1_QUEUE4_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA1_QUEUE4_CONTEXT_STATUS
+#define SDMA1_QUEUE4_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA1_QUEUE4_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA1_QUEUE4_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA1_QUEUE4_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA1_QUEUE4_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA1_QUEUE4_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA1_QUEUE4_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA1_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA1_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA1_QUEUE4_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA1_QUEUE4_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA1_QUEUE4_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA1_QUEUE4_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA1_QUEUE4_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA1_QUEUE4_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA1_QUEUE4_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA1_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA1_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA1_QUEUE4_DOORBELL
+#define SDMA1_QUEUE4_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA1_QUEUE4_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA1_QUEUE4_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA1_QUEUE4_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA1_QUEUE4_DOORBELL_LOG
+#define SDMA1_QUEUE4_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA1_QUEUE4_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA1_QUEUE4_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA1_QUEUE4_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA1_QUEUE4_DOORBELL_OFFSET
+#define SDMA1_QUEUE4_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA1_QUEUE4_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA1_QUEUE4_CSA_ADDR_LO
+#define SDMA1_QUEUE4_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA1_QUEUE4_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA1_QUEUE4_CSA_ADDR_HI
+#define SDMA1_QUEUE4_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA1_QUEUE4_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA1_QUEUE4_SCHEDULE_CNTL
+#define SDMA1_QUEUE4_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA1_QUEUE4_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA1_QUEUE4_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA1_QUEUE4_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA1_QUEUE4_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA1_QUEUE4_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA1_QUEUE4_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA1_QUEUE4_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA1_QUEUE4_IB_SUB_REMAIN
+#define SDMA1_QUEUE4_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA1_QUEUE4_PREEMPT
+#define SDMA1_QUEUE4_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA1_QUEUE4_DUMMY_REG
+#define SDMA1_QUEUE4_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA1_QUEUE4_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE4_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_QUEUE4_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_QUEUE4_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_QUEUE4_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_QUEUE4_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_QUEUE4_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_QUEUE4_RB_AQL_CNTL
+#define SDMA1_QUEUE4_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA1_QUEUE4_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA1_QUEUE4_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA1_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA1_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA1_QUEUE4_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA1_QUEUE4_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_QUEUE4_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA1_QUEUE4_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA1_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA1_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA1_QUEUE4_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA1_QUEUE4_MINOR_PTR_UPDATE
+#define SDMA1_QUEUE4_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA1_QUEUE4_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA1_QUEUE4_RB_PREEMPT
+#define SDMA1_QUEUE4_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA1_QUEUE4_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA1_QUEUE4_MIDCMD_DATA0
+#define SDMA1_QUEUE4_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_DATA1
+#define SDMA1_QUEUE4_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_DATA2
+#define SDMA1_QUEUE4_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_DATA3
+#define SDMA1_QUEUE4_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_DATA4
+#define SDMA1_QUEUE4_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_DATA5
+#define SDMA1_QUEUE4_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_DATA6
+#define SDMA1_QUEUE4_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_DATA7
+#define SDMA1_QUEUE4_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_DATA8
+#define SDMA1_QUEUE4_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_DATA9
+#define SDMA1_QUEUE4_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_DATA10
+#define SDMA1_QUEUE4_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA1_QUEUE4_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE4_MIDCMD_CNTL
+#define SDMA1_QUEUE4_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA1_QUEUE4_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA1_QUEUE4_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA1_QUEUE4_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA1_QUEUE4_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA1_QUEUE4_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA1_QUEUE4_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA1_QUEUE4_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA1_QUEUE5_RB_CNTL
+#define SDMA1_QUEUE5_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE5_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA1_QUEUE5_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA1_QUEUE5_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA1_QUEUE5_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA1_QUEUE5_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA1_QUEUE5_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA1_QUEUE5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA1_QUEUE5_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA1_QUEUE5_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA1_QUEUE5_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA1_QUEUE5_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE5_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA1_QUEUE5_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA1_QUEUE5_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA1_QUEUE5_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA1_QUEUE5_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA1_QUEUE5_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA1_QUEUE5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA1_QUEUE5_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA1_QUEUE5_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA1_QUEUE5_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA1_QUEUE5_RB_BASE
+#define SDMA1_QUEUE5_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA1_QUEUE5_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA1_QUEUE5_RB_BASE_HI
+#define SDMA1_QUEUE5_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE5_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA1_QUEUE5_RB_RPTR
+#define SDMA1_QUEUE5_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE5_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE5_RB_RPTR_HI
+#define SDMA1_QUEUE5_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE5_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE5_RB_WPTR
+#define SDMA1_QUEUE5_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE5_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE5_RB_WPTR_HI
+#define SDMA1_QUEUE5_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE5_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE5_RB_RPTR_ADDR_HI
+#define SDMA1_QUEUE5_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_QUEUE5_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE5_RB_RPTR_ADDR_LO
+#define SDMA1_QUEUE5_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_QUEUE5_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_QUEUE5_IB_CNTL
+#define SDMA1_QUEUE5_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE5_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA1_QUEUE5_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA1_QUEUE5_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA1_QUEUE5_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA1_QUEUE5_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE5_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA1_QUEUE5_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA1_QUEUE5_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA1_QUEUE5_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA1_QUEUE5_IB_RPTR
+#define SDMA1_QUEUE5_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA1_QUEUE5_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA1_QUEUE5_IB_OFFSET
+#define SDMA1_QUEUE5_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA1_QUEUE5_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA1_QUEUE5_IB_BASE_LO
+#define SDMA1_QUEUE5_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA1_QUEUE5_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA1_QUEUE5_IB_BASE_HI
+#define SDMA1_QUEUE5_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE5_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE5_IB_SIZE
+#define SDMA1_QUEUE5_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA1_QUEUE5_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA1_QUEUE5_SKIP_CNTL
+#define SDMA1_QUEUE5_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA1_QUEUE5_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA1_QUEUE5_CONTEXT_STATUS
+#define SDMA1_QUEUE5_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA1_QUEUE5_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA1_QUEUE5_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA1_QUEUE5_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA1_QUEUE5_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA1_QUEUE5_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA1_QUEUE5_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA1_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA1_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA1_QUEUE5_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA1_QUEUE5_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA1_QUEUE5_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA1_QUEUE5_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA1_QUEUE5_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA1_QUEUE5_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA1_QUEUE5_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA1_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA1_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA1_QUEUE5_DOORBELL
+#define SDMA1_QUEUE5_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA1_QUEUE5_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA1_QUEUE5_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA1_QUEUE5_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA1_QUEUE5_DOORBELL_LOG
+#define SDMA1_QUEUE5_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA1_QUEUE5_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA1_QUEUE5_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA1_QUEUE5_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA1_QUEUE5_DOORBELL_OFFSET
+#define SDMA1_QUEUE5_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA1_QUEUE5_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA1_QUEUE5_CSA_ADDR_LO
+#define SDMA1_QUEUE5_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA1_QUEUE5_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA1_QUEUE5_CSA_ADDR_HI
+#define SDMA1_QUEUE5_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA1_QUEUE5_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA1_QUEUE5_SCHEDULE_CNTL
+#define SDMA1_QUEUE5_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA1_QUEUE5_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA1_QUEUE5_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA1_QUEUE5_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA1_QUEUE5_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA1_QUEUE5_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA1_QUEUE5_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA1_QUEUE5_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA1_QUEUE5_IB_SUB_REMAIN
+#define SDMA1_QUEUE5_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA1_QUEUE5_PREEMPT
+#define SDMA1_QUEUE5_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA1_QUEUE5_DUMMY_REG
+#define SDMA1_QUEUE5_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA1_QUEUE5_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE5_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_QUEUE5_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_QUEUE5_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_QUEUE5_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_QUEUE5_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_QUEUE5_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_QUEUE5_RB_AQL_CNTL
+#define SDMA1_QUEUE5_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA1_QUEUE5_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA1_QUEUE5_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA1_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA1_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA1_QUEUE5_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA1_QUEUE5_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_QUEUE5_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA1_QUEUE5_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA1_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA1_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA1_QUEUE5_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA1_QUEUE5_MINOR_PTR_UPDATE
+#define SDMA1_QUEUE5_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA1_QUEUE5_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA1_QUEUE5_RB_PREEMPT
+#define SDMA1_QUEUE5_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA1_QUEUE5_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA1_QUEUE5_MIDCMD_DATA0
+#define SDMA1_QUEUE5_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_DATA1
+#define SDMA1_QUEUE5_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_DATA2
+#define SDMA1_QUEUE5_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_DATA3
+#define SDMA1_QUEUE5_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_DATA4
+#define SDMA1_QUEUE5_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_DATA5
+#define SDMA1_QUEUE5_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_DATA6
+#define SDMA1_QUEUE5_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_DATA7
+#define SDMA1_QUEUE5_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_DATA8
+#define SDMA1_QUEUE5_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_DATA9
+#define SDMA1_QUEUE5_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_DATA10
+#define SDMA1_QUEUE5_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA1_QUEUE5_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE5_MIDCMD_CNTL
+#define SDMA1_QUEUE5_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA1_QUEUE5_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA1_QUEUE5_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA1_QUEUE5_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA1_QUEUE5_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA1_QUEUE5_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA1_QUEUE5_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA1_QUEUE5_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA1_QUEUE6_RB_CNTL
+#define SDMA1_QUEUE6_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE6_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA1_QUEUE6_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA1_QUEUE6_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA1_QUEUE6_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA1_QUEUE6_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA1_QUEUE6_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA1_QUEUE6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA1_QUEUE6_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA1_QUEUE6_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA1_QUEUE6_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA1_QUEUE6_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE6_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA1_QUEUE6_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA1_QUEUE6_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA1_QUEUE6_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA1_QUEUE6_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA1_QUEUE6_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA1_QUEUE6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA1_QUEUE6_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA1_QUEUE6_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA1_QUEUE6_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA1_QUEUE6_RB_BASE
+#define SDMA1_QUEUE6_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA1_QUEUE6_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA1_QUEUE6_RB_BASE_HI
+#define SDMA1_QUEUE6_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE6_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA1_QUEUE6_RB_RPTR
+#define SDMA1_QUEUE6_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE6_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE6_RB_RPTR_HI
+#define SDMA1_QUEUE6_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE6_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE6_RB_WPTR
+#define SDMA1_QUEUE6_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE6_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE6_RB_WPTR_HI
+#define SDMA1_QUEUE6_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE6_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE6_RB_RPTR_ADDR_HI
+#define SDMA1_QUEUE6_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_QUEUE6_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE6_RB_RPTR_ADDR_LO
+#define SDMA1_QUEUE6_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_QUEUE6_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_QUEUE6_IB_CNTL
+#define SDMA1_QUEUE6_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE6_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA1_QUEUE6_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA1_QUEUE6_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA1_QUEUE6_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA1_QUEUE6_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE6_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA1_QUEUE6_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA1_QUEUE6_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA1_QUEUE6_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA1_QUEUE6_IB_RPTR
+#define SDMA1_QUEUE6_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA1_QUEUE6_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA1_QUEUE6_IB_OFFSET
+#define SDMA1_QUEUE6_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA1_QUEUE6_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA1_QUEUE6_IB_BASE_LO
+#define SDMA1_QUEUE6_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA1_QUEUE6_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA1_QUEUE6_IB_BASE_HI
+#define SDMA1_QUEUE6_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE6_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE6_IB_SIZE
+#define SDMA1_QUEUE6_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA1_QUEUE6_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA1_QUEUE6_SKIP_CNTL
+#define SDMA1_QUEUE6_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA1_QUEUE6_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA1_QUEUE6_CONTEXT_STATUS
+#define SDMA1_QUEUE6_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA1_QUEUE6_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA1_QUEUE6_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA1_QUEUE6_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA1_QUEUE6_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA1_QUEUE6_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA1_QUEUE6_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA1_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA1_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA1_QUEUE6_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA1_QUEUE6_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA1_QUEUE6_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA1_QUEUE6_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA1_QUEUE6_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA1_QUEUE6_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA1_QUEUE6_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA1_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA1_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA1_QUEUE6_DOORBELL
+#define SDMA1_QUEUE6_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA1_QUEUE6_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA1_QUEUE6_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA1_QUEUE6_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA1_QUEUE6_DOORBELL_LOG
+#define SDMA1_QUEUE6_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA1_QUEUE6_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA1_QUEUE6_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA1_QUEUE6_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA1_QUEUE6_DOORBELL_OFFSET
+#define SDMA1_QUEUE6_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA1_QUEUE6_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA1_QUEUE6_CSA_ADDR_LO
+#define SDMA1_QUEUE6_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA1_QUEUE6_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA1_QUEUE6_CSA_ADDR_HI
+#define SDMA1_QUEUE6_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA1_QUEUE6_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA1_QUEUE6_SCHEDULE_CNTL
+#define SDMA1_QUEUE6_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA1_QUEUE6_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA1_QUEUE6_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA1_QUEUE6_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA1_QUEUE6_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA1_QUEUE6_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA1_QUEUE6_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA1_QUEUE6_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA1_QUEUE6_IB_SUB_REMAIN
+#define SDMA1_QUEUE6_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA1_QUEUE6_PREEMPT
+#define SDMA1_QUEUE6_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA1_QUEUE6_DUMMY_REG
+#define SDMA1_QUEUE6_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA1_QUEUE6_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE6_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_QUEUE6_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_QUEUE6_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_QUEUE6_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_QUEUE6_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_QUEUE6_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_QUEUE6_RB_AQL_CNTL
+#define SDMA1_QUEUE6_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA1_QUEUE6_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA1_QUEUE6_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA1_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA1_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA1_QUEUE6_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA1_QUEUE6_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_QUEUE6_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA1_QUEUE6_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA1_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA1_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA1_QUEUE6_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA1_QUEUE6_MINOR_PTR_UPDATE
+#define SDMA1_QUEUE6_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA1_QUEUE6_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA1_QUEUE6_RB_PREEMPT
+#define SDMA1_QUEUE6_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA1_QUEUE6_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA1_QUEUE6_MIDCMD_DATA0
+#define SDMA1_QUEUE6_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_DATA1
+#define SDMA1_QUEUE6_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_DATA2
+#define SDMA1_QUEUE6_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_DATA3
+#define SDMA1_QUEUE6_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_DATA4
+#define SDMA1_QUEUE6_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_DATA5
+#define SDMA1_QUEUE6_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_DATA6
+#define SDMA1_QUEUE6_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_DATA7
+#define SDMA1_QUEUE6_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_DATA8
+#define SDMA1_QUEUE6_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_DATA9
+#define SDMA1_QUEUE6_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_DATA10
+#define SDMA1_QUEUE6_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA1_QUEUE6_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE6_MIDCMD_CNTL
+#define SDMA1_QUEUE6_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA1_QUEUE6_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA1_QUEUE6_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA1_QUEUE6_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA1_QUEUE6_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA1_QUEUE6_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA1_QUEUE6_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA1_QUEUE6_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+//SDMA1_QUEUE7_RB_CNTL
+#define SDMA1_QUEUE7_RB_CNTL__RB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE7_RB_CNTL__RB_SIZE__SHIFT                                                                  0x1
+#define SDMA1_QUEUE7_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                         0x8
+#define SDMA1_QUEUE7_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                           0x9
+#define SDMA1_QUEUE7_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                    0xa
+#define SDMA1_QUEUE7_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                     0xb
+#define SDMA1_QUEUE7_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                    0xc
+#define SDMA1_QUEUE7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                               0xd
+#define SDMA1_QUEUE7_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                     0x10
+#define SDMA1_QUEUE7_RB_CNTL__RB_PRIV__SHIFT                                                                  0x17
+#define SDMA1_QUEUE7_RB_CNTL__RB_VMID__SHIFT                                                                  0x18
+#define SDMA1_QUEUE7_RB_CNTL__RB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE7_RB_CNTL__RB_SIZE_MASK                                                                    0x0000003EL
+#define SDMA1_QUEUE7_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                           0x00000100L
+#define SDMA1_QUEUE7_RB_CNTL__RB_SWAP_ENABLE_MASK                                                             0x00000200L
+#define SDMA1_QUEUE7_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                      0x00000400L
+#define SDMA1_QUEUE7_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                       0x00000800L
+#define SDMA1_QUEUE7_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                      0x00001000L
+#define SDMA1_QUEUE7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                 0x00002000L
+#define SDMA1_QUEUE7_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                       0x001F0000L
+#define SDMA1_QUEUE7_RB_CNTL__RB_PRIV_MASK                                                                    0x00800000L
+#define SDMA1_QUEUE7_RB_CNTL__RB_VMID_MASK                                                                    0x0F000000L
+//SDMA1_QUEUE7_RB_BASE
+#define SDMA1_QUEUE7_RB_BASE__ADDR__SHIFT                                                                     0x0
+#define SDMA1_QUEUE7_RB_BASE__ADDR_MASK                                                                       0xFFFFFFFFL
+//SDMA1_QUEUE7_RB_BASE_HI
+#define SDMA1_QUEUE7_RB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE7_RB_BASE_HI__ADDR_MASK                                                                    0x00FFFFFFL
+//SDMA1_QUEUE7_RB_RPTR
+#define SDMA1_QUEUE7_RB_RPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE7_RB_RPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE7_RB_RPTR_HI
+#define SDMA1_QUEUE7_RB_RPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE7_RB_RPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE7_RB_WPTR
+#define SDMA1_QUEUE7_RB_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SDMA1_QUEUE7_RB_WPTR__OFFSET_MASK                                                                     0xFFFFFFFFL
+//SDMA1_QUEUE7_RB_WPTR_HI
+#define SDMA1_QUEUE7_RB_WPTR_HI__OFFSET__SHIFT                                                                0x0
+#define SDMA1_QUEUE7_RB_WPTR_HI__OFFSET_MASK                                                                  0xFFFFFFFFL
+//SDMA1_QUEUE7_RB_RPTR_ADDR_HI
+#define SDMA1_QUEUE7_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define SDMA1_QUEUE7_RB_RPTR_ADDR_HI__ADDR_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE7_RB_RPTR_ADDR_LO
+#define SDMA1_QUEUE7_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                             0x2
+#define SDMA1_QUEUE7_RB_RPTR_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFCL
+//SDMA1_QUEUE7_IB_CNTL
+#define SDMA1_QUEUE7_IB_CNTL__IB_ENABLE__SHIFT                                                                0x0
+#define SDMA1_QUEUE7_IB_CNTL__IB_SWAP_ENABLE__SHIFT                                                           0x4
+#define SDMA1_QUEUE7_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                         0x8
+#define SDMA1_QUEUE7_IB_CNTL__CMD_VMID__SHIFT                                                                 0x10
+#define SDMA1_QUEUE7_IB_CNTL__IB_PRIV__SHIFT                                                                  0x1f
+#define SDMA1_QUEUE7_IB_CNTL__IB_ENABLE_MASK                                                                  0x00000001L
+#define SDMA1_QUEUE7_IB_CNTL__IB_SWAP_ENABLE_MASK                                                             0x00000010L
+#define SDMA1_QUEUE7_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                           0x00000100L
+#define SDMA1_QUEUE7_IB_CNTL__CMD_VMID_MASK                                                                   0x000F0000L
+#define SDMA1_QUEUE7_IB_CNTL__IB_PRIV_MASK                                                                    0x80000000L
+//SDMA1_QUEUE7_IB_RPTR
+#define SDMA1_QUEUE7_IB_RPTR__OFFSET__SHIFT                                                                   0x2
+#define SDMA1_QUEUE7_IB_RPTR__OFFSET_MASK                                                                     0x003FFFFCL
+//SDMA1_QUEUE7_IB_OFFSET
+#define SDMA1_QUEUE7_IB_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define SDMA1_QUEUE7_IB_OFFSET__OFFSET_MASK                                                                   0x003FFFFCL
+//SDMA1_QUEUE7_IB_BASE_LO
+#define SDMA1_QUEUE7_IB_BASE_LO__ADDR__SHIFT                                                                  0x5
+#define SDMA1_QUEUE7_IB_BASE_LO__ADDR_MASK                                                                    0xFFFFFFE0L
+//SDMA1_QUEUE7_IB_BASE_HI
+#define SDMA1_QUEUE7_IB_BASE_HI__ADDR__SHIFT                                                                  0x0
+#define SDMA1_QUEUE7_IB_BASE_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE7_IB_SIZE
+#define SDMA1_QUEUE7_IB_SIZE__SIZE__SHIFT                                                                     0x0
+#define SDMA1_QUEUE7_IB_SIZE__SIZE_MASK                                                                       0x000FFFFFL
+//SDMA1_QUEUE7_SKIP_CNTL
+#define SDMA1_QUEUE7_SKIP_CNTL__SKIP_COUNT__SHIFT                                                             0x0
+#define SDMA1_QUEUE7_SKIP_CNTL__SKIP_COUNT_MASK                                                               0x000FFFFFL
+//SDMA1_QUEUE7_CONTEXT_STATUS
+#define SDMA1_QUEUE7_CONTEXT_STATUS__SELECTED__SHIFT                                                          0x0
+#define SDMA1_QUEUE7_CONTEXT_STATUS__IDLE__SHIFT                                                              0x2
+#define SDMA1_QUEUE7_CONTEXT_STATUS__EXPIRED__SHIFT                                                           0x3
+#define SDMA1_QUEUE7_CONTEXT_STATUS__EXCEPTION__SHIFT                                                         0x4
+#define SDMA1_QUEUE7_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                        0x7
+#define SDMA1_QUEUE7_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                   0xa
+#define SDMA1_QUEUE7_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                      0xb
+#define SDMA1_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                               0xc
+#define SDMA1_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                            0x10
+#define SDMA1_QUEUE7_CONTEXT_STATUS__SELECTED_MASK                                                            0x00000001L
+#define SDMA1_QUEUE7_CONTEXT_STATUS__IDLE_MASK                                                                0x00000004L
+#define SDMA1_QUEUE7_CONTEXT_STATUS__EXPIRED_MASK                                                             0x00000008L
+#define SDMA1_QUEUE7_CONTEXT_STATUS__EXCEPTION_MASK                                                           0x00000070L
+#define SDMA1_QUEUE7_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                          0x00000080L
+#define SDMA1_QUEUE7_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                     0x00000400L
+#define SDMA1_QUEUE7_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                        0x00000800L
+#define SDMA1_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                 0x00001000L
+#define SDMA1_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                              0x00FF0000L
+//SDMA1_QUEUE7_DOORBELL
+#define SDMA1_QUEUE7_DOORBELL__ENABLE__SHIFT                                                                  0x1c
+#define SDMA1_QUEUE7_DOORBELL__CAPTURED__SHIFT                                                                0x1e
+#define SDMA1_QUEUE7_DOORBELL__ENABLE_MASK                                                                    0x10000000L
+#define SDMA1_QUEUE7_DOORBELL__CAPTURED_MASK                                                                  0x40000000L
+//SDMA1_QUEUE7_DOORBELL_LOG
+#define SDMA1_QUEUE7_DOORBELL_LOG__BE_ERROR__SHIFT                                                            0x0
+#define SDMA1_QUEUE7_DOORBELL_LOG__DATA__SHIFT                                                                0x2
+#define SDMA1_QUEUE7_DOORBELL_LOG__BE_ERROR_MASK                                                              0x00000001L
+#define SDMA1_QUEUE7_DOORBELL_LOG__DATA_MASK                                                                  0xFFFFFFFCL
+//SDMA1_QUEUE7_DOORBELL_OFFSET
+#define SDMA1_QUEUE7_DOORBELL_OFFSET__OFFSET__SHIFT                                                           0x2
+#define SDMA1_QUEUE7_DOORBELL_OFFSET__OFFSET_MASK                                                             0x0FFFFFFCL
+//SDMA1_QUEUE7_CSA_ADDR_LO
+#define SDMA1_QUEUE7_CSA_ADDR_LO__ADDR__SHIFT                                                                 0x2
+#define SDMA1_QUEUE7_CSA_ADDR_LO__ADDR_MASK                                                                   0xFFFFFFFCL
+//SDMA1_QUEUE7_CSA_ADDR_HI
+#define SDMA1_QUEUE7_CSA_ADDR_HI__ADDR__SHIFT                                                                 0x0
+#define SDMA1_QUEUE7_CSA_ADDR_HI__ADDR_MASK                                                                   0xFFFFFFFFL
+//SDMA1_QUEUE7_SCHEDULE_CNTL
+#define SDMA1_QUEUE7_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                          0x0
+#define SDMA1_QUEUE7_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                         0x2
+#define SDMA1_QUEUE7_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                           0x6
+#define SDMA1_QUEUE7_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                    0x8
+#define SDMA1_QUEUE7_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                            0x00000003L
+#define SDMA1_QUEUE7_SCHEDULE_CNTL__PROCESS_ID_MASK                                                           0x0000001CL
+#define SDMA1_QUEUE7_SCHEDULE_CNTL__LOCAL_ID_MASK                                                             0x000000C0L
+#define SDMA1_QUEUE7_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                      0x0000FF00L
+//SDMA1_QUEUE7_IB_SUB_REMAIN
+#define SDMA1_QUEUE7_IB_SUB_REMAIN__SIZE__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_IB_SUB_REMAIN__SIZE_MASK                                                                 0x00003FFFL
+//SDMA1_QUEUE7_PREEMPT
+#define SDMA1_QUEUE7_PREEMPT__IB_PREEMPT__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_PREEMPT__IB_PREEMPT_MASK                                                                 0x00000001L
+//SDMA1_QUEUE7_DUMMY_REG
+#define SDMA1_QUEUE7_DUMMY_REG__DUMMY__SHIFT                                                                  0x0
+#define SDMA1_QUEUE7_DUMMY_REG__DUMMY_MASK                                                                    0xFFFFFFFFL
+//SDMA1_QUEUE7_RB_WPTR_POLL_ADDR_HI
+#define SDMA1_QUEUE7_RB_WPTR_POLL_ADDR_HI__ADDR__SHIFT                                                        0x0
+#define SDMA1_QUEUE7_RB_WPTR_POLL_ADDR_HI__ADDR_MASK                                                          0xFFFFFFFFL
+//SDMA1_QUEUE7_RB_WPTR_POLL_ADDR_LO
+#define SDMA1_QUEUE7_RB_WPTR_POLL_ADDR_LO__ADDR__SHIFT                                                        0x2
+#define SDMA1_QUEUE7_RB_WPTR_POLL_ADDR_LO__ADDR_MASK                                                          0xFFFFFFFCL
+//SDMA1_QUEUE7_RB_AQL_CNTL
+#define SDMA1_QUEUE7_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                           0x0
+#define SDMA1_QUEUE7_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                      0x1
+#define SDMA1_QUEUE7_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                          0x8
+#define SDMA1_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                0x10
+#define SDMA1_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                          0x11
+#define SDMA1_QUEUE7_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                       0x12
+#define SDMA1_QUEUE7_RB_AQL_CNTL__AQL_ENABLE_MASK                                                             0x00000001L
+#define SDMA1_QUEUE7_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                        0x000000FEL
+#define SDMA1_QUEUE7_RB_AQL_CNTL__PACKET_STEP_MASK                                                            0x0000FF00L
+#define SDMA1_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                  0x00010000L
+#define SDMA1_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                            0x00020000L
+#define SDMA1_QUEUE7_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                         0x00040000L
+//SDMA1_QUEUE7_MINOR_PTR_UPDATE
+#define SDMA1_QUEUE7_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                          0x0
+#define SDMA1_QUEUE7_MINOR_PTR_UPDATE__ENABLE_MASK                                                            0x00000001L
+//SDMA1_QUEUE7_RB_PREEMPT
+#define SDMA1_QUEUE7_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                           0x0
+#define SDMA1_QUEUE7_RB_PREEMPT__PREEMPT_REQ_MASK                                                             0x00000001L
+//SDMA1_QUEUE7_MIDCMD_DATA0
+#define SDMA1_QUEUE7_MIDCMD_DATA0__DATA0__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA0__DATA0_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_DATA1
+#define SDMA1_QUEUE7_MIDCMD_DATA1__DATA1__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA1__DATA1_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_DATA2
+#define SDMA1_QUEUE7_MIDCMD_DATA2__DATA2__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA2__DATA2_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_DATA3
+#define SDMA1_QUEUE7_MIDCMD_DATA3__DATA3__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA3__DATA3_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_DATA4
+#define SDMA1_QUEUE7_MIDCMD_DATA4__DATA4__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA4__DATA4_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_DATA5
+#define SDMA1_QUEUE7_MIDCMD_DATA5__DATA5__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA5__DATA5_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_DATA6
+#define SDMA1_QUEUE7_MIDCMD_DATA6__DATA6__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA6__DATA6_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_DATA7
+#define SDMA1_QUEUE7_MIDCMD_DATA7__DATA7__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA7__DATA7_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_DATA8
+#define SDMA1_QUEUE7_MIDCMD_DATA8__DATA8__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA8__DATA8_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_DATA9
+#define SDMA1_QUEUE7_MIDCMD_DATA9__DATA9__SHIFT                                                               0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA9__DATA9_MASK                                                                 0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_DATA10
+#define SDMA1_QUEUE7_MIDCMD_DATA10__DATA10__SHIFT                                                             0x0
+#define SDMA1_QUEUE7_MIDCMD_DATA10__DATA10_MASK                                                               0xFFFFFFFFL
+//SDMA1_QUEUE7_MIDCMD_CNTL
+#define SDMA1_QUEUE7_MIDCMD_CNTL__DATA_VALID__SHIFT                                                           0x0
+#define SDMA1_QUEUE7_MIDCMD_CNTL__COPY_MODE__SHIFT                                                            0x1
+#define SDMA1_QUEUE7_MIDCMD_CNTL__SPLIT_STATE__SHIFT                                                          0x4
+#define SDMA1_QUEUE7_MIDCMD_CNTL__ALLOW_PREEMPT__SHIFT                                                        0x8
+#define SDMA1_QUEUE7_MIDCMD_CNTL__DATA_VALID_MASK                                                             0x00000001L
+#define SDMA1_QUEUE7_MIDCMD_CNTL__COPY_MODE_MASK                                                              0x00000002L
+#define SDMA1_QUEUE7_MIDCMD_CNTL__SPLIT_STATE_MASK                                                            0x000000F0L
+#define SDMA1_QUEUE7_MIDCMD_CNTL__ALLOW_PREEMPT_MASK                                                          0x00000100L
+
+
+// addressBlock: gc_sdma0_sdma0hypdec
+//SDMA0_UCODE_ADDR
+#define SDMA0_UCODE_ADDR__VALUE__SHIFT                                                                        0x0
+#define SDMA0_UCODE_ADDR__THID__SHIFT                                                                         0xf
+#define SDMA0_UCODE_ADDR__VALUE_MASK                                                                          0x00001FFFL
+#define SDMA0_UCODE_ADDR__THID_MASK                                                                           0x00008000L
+//SDMA0_UCODE_DATA
+#define SDMA0_UCODE_DATA__VALUE__SHIFT                                                                        0x0
+#define SDMA0_UCODE_DATA__VALUE_MASK                                                                          0xFFFFFFFFL
+//SDMA0_UCODE_SELFLOAD_CONTROL
+#define SDMA0_UCODE_SELFLOAD_CONTROL__GPA__SHIFT                                                              0x0
+#define SDMA0_UCODE_SELFLOAD_CONTROL__SYS__SHIFT                                                              0x1
+#define SDMA0_UCODE_SELFLOAD_CONTROL__CID__SHIFT                                                              0x4
+#define SDMA0_UCODE_SELFLOAD_CONTROL__CACHE_POLICY__SHIFT                                                     0x8
+#define SDMA0_UCODE_SELFLOAD_CONTROL__GPA_MASK                                                                0x00000001L
+#define SDMA0_UCODE_SELFLOAD_CONTROL__SYS_MASK                                                                0x00000002L
+#define SDMA0_UCODE_SELFLOAD_CONTROL__CID_MASK                                                                0x000000F0L
+#define SDMA0_UCODE_SELFLOAD_CONTROL__CACHE_POLICY_MASK                                                       0x00000300L
+//SDMA0_BROADCAST_UCODE_ADDR
+#define SDMA0_BROADCAST_UCODE_ADDR__VALUE__SHIFT                                                              0x0
+#define SDMA0_BROADCAST_UCODE_ADDR__THID__SHIFT                                                               0xf
+#define SDMA0_BROADCAST_UCODE_ADDR__VALUE_MASK                                                                0x00001FFFL
+#define SDMA0_BROADCAST_UCODE_ADDR__THID_MASK                                                                 0x00008000L
+//SDMA0_BROADCAST_UCODE_DATA
+#define SDMA0_BROADCAST_UCODE_DATA__VALUE__SHIFT                                                              0x0
+#define SDMA0_BROADCAST_UCODE_DATA__VALUE_MASK                                                                0xFFFFFFFFL
+//SDMA0_VM_CTX_LO
+#define SDMA0_VM_CTX_LO__ADDR__SHIFT                                                                          0x2
+#define SDMA0_VM_CTX_LO__ADDR_MASK                                                                            0xFFFFFFFCL
+//SDMA0_VM_CTX_HI
+#define SDMA0_VM_CTX_HI__ADDR__SHIFT                                                                          0x0
+#define SDMA0_VM_CTX_HI__ADDR_MASK                                                                            0xFFFFFFFFL
+//SDMA0_ACTIVE_FCN_ID
+#define SDMA0_ACTIVE_FCN_ID__VFID__SHIFT                                                                      0x0
+#define SDMA0_ACTIVE_FCN_ID__RESERVED__SHIFT                                                                  0x4
+#define SDMA0_ACTIVE_FCN_ID__VF__SHIFT                                                                        0x1f
+#define SDMA0_ACTIVE_FCN_ID__VFID_MASK                                                                        0x0000000FL
+#define SDMA0_ACTIVE_FCN_ID__RESERVED_MASK                                                                    0x7FFFFFF0L
+#define SDMA0_ACTIVE_FCN_ID__VF_MASK                                                                          0x80000000L
+//SDMA0_VM_CTX_CNTL
+#define SDMA0_VM_CTX_CNTL__PRIV__SHIFT                                                                        0x0
+#define SDMA0_VM_CTX_CNTL__VMID__SHIFT                                                                        0x4
+#define SDMA0_VM_CTX_CNTL__MEM_PHY__SHIFT                                                                     0x8
+#define SDMA0_VM_CTX_CNTL__BUSY_STATUS_REPORT_ENABLE__SHIFT                                                   0x10
+#define SDMA0_VM_CTX_CNTL__PRIV_MASK                                                                          0x00000001L
+#define SDMA0_VM_CTX_CNTL__VMID_MASK                                                                          0x000000F0L
+#define SDMA0_VM_CTX_CNTL__MEM_PHY_MASK                                                                       0x00000300L
+#define SDMA0_VM_CTX_CNTL__BUSY_STATUS_REPORT_ENABLE_MASK                                                     0x00010000L
+//SDMA0_VIRT_RESET_REQ
+#define SDMA0_VIRT_RESET_REQ__VF__SHIFT                                                                       0x0
+#define SDMA0_VIRT_RESET_REQ__PF__SHIFT                                                                       0x1f
+#define SDMA0_VIRT_RESET_REQ__VF_MASK                                                                         0x0000FFFFL
+#define SDMA0_VIRT_RESET_REQ__PF_MASK                                                                         0x80000000L
+//SDMA0_CONTEXT_REG_TYPE0
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_CNTL__SHIFT                                                  0x0
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_BASE__SHIFT                                                  0x1
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_BASE_HI__SHIFT                                               0x2
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_RPTR__SHIFT                                                  0x3
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_RPTR_HI__SHIFT                                               0x4
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_WPTR__SHIFT                                                  0x5
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_WPTR_HI__SHIFT                                               0x6
+#define SDMA0_CONTEXT_REG_TYPE0__RESERVED7__SHIFT                                                             0x7
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_RPTR_ADDR_HI__SHIFT                                          0x8
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_RPTR_ADDR_LO__SHIFT                                          0x9
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_CNTL__SHIFT                                                  0xa
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_RPTR__SHIFT                                                  0xb
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_OFFSET__SHIFT                                                0xc
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_BASE_LO__SHIFT                                               0xd
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_BASE_HI__SHIFT                                               0xe
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_SIZE__SHIFT                                                  0xf
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_SKIP_CNTL__SHIFT                                                0x10
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_CONTEXT_STATUS__SHIFT                                           0x11
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_DOORBELL__SHIFT                                                 0x12
+#define SDMA0_CONTEXT_REG_TYPE0__RESERVED31_19__SHIFT                                                         0x13
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_CNTL_MASK                                                    0x00000001L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_BASE_MASK                                                    0x00000002L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_BASE_HI_MASK                                                 0x00000004L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_RPTR_MASK                                                    0x00000008L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_RPTR_HI_MASK                                                 0x00000010L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_WPTR_MASK                                                    0x00000020L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_WPTR_HI_MASK                                                 0x00000040L
+#define SDMA0_CONTEXT_REG_TYPE0__RESERVED7_MASK                                                               0x00000080L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_RPTR_ADDR_HI_MASK                                            0x00000100L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_RB_RPTR_ADDR_LO_MASK                                            0x00000200L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_CNTL_MASK                                                    0x00000400L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_RPTR_MASK                                                    0x00000800L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_OFFSET_MASK                                                  0x00001000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_BASE_LO_MASK                                                 0x00002000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_BASE_HI_MASK                                                 0x00004000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_IB_SIZE_MASK                                                    0x00008000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_SKIP_CNTL_MASK                                                  0x00010000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_CONTEXT_STATUS_MASK                                             0x00020000L
+#define SDMA0_CONTEXT_REG_TYPE0__SDMA0_QUEUE0_DOORBELL_MASK                                                   0x00040000L
+#define SDMA0_CONTEXT_REG_TYPE0__RESERVED31_19_MASK                                                           0xFFF80000L
+//SDMA0_CONTEXT_REG_TYPE1
+#define SDMA0_CONTEXT_REG_TYPE1__RESERVED8_0__SHIFT                                                           0x0
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_DOORBELL_LOG__SHIFT                                             0x9
+#define SDMA0_CONTEXT_REG_TYPE1__RESERVED10__SHIFT                                                            0xa
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_DOORBELL_OFFSET__SHIFT                                          0xb
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_CSA_ADDR_LO__SHIFT                                              0xc
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_CSA_ADDR_HI__SHIFT                                              0xd
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_SCHEDULE_CNTL__SHIFT                                            0xe
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_IB_SUB_REMAIN__SHIFT                                            0xf
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_PREEMPT__SHIFT                                                  0x10
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_DUMMY_REG__SHIFT                                                0x11
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI__SHIFT                                     0x12
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO__SHIFT                                     0x13
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_RB_AQL_CNTL__SHIFT                                              0x14
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_MINOR_PTR_UPDATE__SHIFT                                         0x15
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_RB_PREEMPT__SHIFT                                               0x16
+#define SDMA0_CONTEXT_REG_TYPE1__RESERVED__SHIFT                                                              0x17
+#define SDMA0_CONTEXT_REG_TYPE1__RESERVED8_0_MASK                                                             0x000001FFL
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_DOORBELL_LOG_MASK                                               0x00000200L
+#define SDMA0_CONTEXT_REG_TYPE1__RESERVED10_MASK                                                              0x00000400L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_DOORBELL_OFFSET_MASK                                            0x00000800L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_CSA_ADDR_LO_MASK                                                0x00001000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_CSA_ADDR_HI_MASK                                                0x00002000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_SCHEDULE_CNTL_MASK                                              0x00004000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_IB_SUB_REMAIN_MASK                                              0x00008000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_PREEMPT_MASK                                                    0x00010000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_DUMMY_REG_MASK                                                  0x00020000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_RB_WPTR_POLL_ADDR_HI_MASK                                       0x00040000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_RB_WPTR_POLL_ADDR_LO_MASK                                       0x00080000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_RB_AQL_CNTL_MASK                                                0x00100000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_MINOR_PTR_UPDATE_MASK                                           0x00200000L
+#define SDMA0_CONTEXT_REG_TYPE1__SDMA0_QUEUE0_RB_PREEMPT_MASK                                                 0x00400000L
+#define SDMA0_CONTEXT_REG_TYPE1__RESERVED_MASK                                                                0xFF800000L
+//SDMA0_CONTEXT_REG_TYPE2
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA0__SHIFT                                             0x0
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA1__SHIFT                                             0x1
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA2__SHIFT                                             0x2
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA3__SHIFT                                             0x3
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA4__SHIFT                                             0x4
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA5__SHIFT                                             0x5
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA6__SHIFT                                             0x6
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA7__SHIFT                                             0x7
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA8__SHIFT                                             0x8
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA9__SHIFT                                             0x9
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA10__SHIFT                                            0xa
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_CNTL__SHIFT                                              0xb
+#define SDMA0_CONTEXT_REG_TYPE2__RESERVED__SHIFT                                                              0xe
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA0_MASK                                               0x00000001L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA1_MASK                                               0x00000002L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA2_MASK                                               0x00000004L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA3_MASK                                               0x00000008L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA4_MASK                                               0x00000010L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA5_MASK                                               0x00000020L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA6_MASK                                               0x00000040L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA7_MASK                                               0x00000080L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA8_MASK                                               0x00000100L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA9_MASK                                               0x00000200L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_DATA10_MASK                                              0x00000400L
+#define SDMA0_CONTEXT_REG_TYPE2__SDMA0_QUEUE0_MIDCMD_CNTL_MASK                                                0x00000800L
+#define SDMA0_CONTEXT_REG_TYPE2__RESERVED_MASK                                                                0xFFFFC000L
+//SDMA0_PUB_REG_TYPE0
+#define SDMA0_PUB_REG_TYPE0__SDMA0_DEC_START__SHIFT                                                           0x0
+#define SDMA0_PUB_REG_TYPE0__RESERVED_10_1__SHIFT                                                             0x1
+#define SDMA0_PUB_REG_TYPE0__SDMA0_F32_MISC_CNTL__SHIFT                                                       0xb
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GLOBAL_TIMESTAMP_LO__SHIFT                                                 0xf
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GLOBAL_TIMESTAMP_HI__SHIFT                                                 0x10
+#define SDMA0_PUB_REG_TYPE0__RESERVED22__SHIFT                                                                0x16
+#define SDMA0_PUB_REG_TYPE0__RESERVED23__SHIFT                                                                0x17
+#define SDMA0_PUB_REG_TYPE0__RESERVED24__SHIFT                                                                0x18
+#define SDMA0_PUB_REG_TYPE0__RESERVED25__SHIFT                                                                0x19
+#define SDMA0_PUB_REG_TYPE0__SDMA0_POWER_CNTL__SHIFT                                                          0x1a
+#define SDMA0_PUB_REG_TYPE0__RESERVED27__SHIFT                                                                0x1b
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CNTL__SHIFT                                                                0x1c
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CHICKEN_BITS__SHIFT                                                        0x1d
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG__SHIFT                                                      0x1e
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_READ__SHIFT                                                 0x1f
+#define SDMA0_PUB_REG_TYPE0__SDMA0_DEC_START_MASK                                                             0x00000001L
+#define SDMA0_PUB_REG_TYPE0__RESERVED_10_1_MASK                                                               0x000007FEL
+#define SDMA0_PUB_REG_TYPE0__SDMA0_F32_MISC_CNTL_MASK                                                         0x00000800L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GLOBAL_TIMESTAMP_LO_MASK                                                   0x00008000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GLOBAL_TIMESTAMP_HI_MASK                                                   0x00010000L
+#define SDMA0_PUB_REG_TYPE0__RESERVED22_MASK                                                                  0x00400000L
+#define SDMA0_PUB_REG_TYPE0__RESERVED23_MASK                                                                  0x00800000L
+#define SDMA0_PUB_REG_TYPE0__RESERVED24_MASK                                                                  0x01000000L
+#define SDMA0_PUB_REG_TYPE0__RESERVED25_MASK                                                                  0x02000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_POWER_CNTL_MASK                                                            0x04000000L
+#define SDMA0_PUB_REG_TYPE0__RESERVED27_MASK                                                                  0x08000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CNTL_MASK                                                                  0x10000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_CHICKEN_BITS_MASK                                                          0x20000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_MASK                                                        0x40000000L
+#define SDMA0_PUB_REG_TYPE0__SDMA0_GB_ADDR_CONFIG_READ_MASK                                                   0x80000000L
+//SDMA0_PUB_REG_TYPE1
+#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH__SHIFT                                                       0x0
+#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_HI__SHIFT                                                    0x1
+#define SDMA0_PUB_REG_TYPE1__SDMA0_SEM_WAIT_FAIL_TIMER_CNTL__SHIFT                                            0x2
+#define SDMA0_PUB_REG_TYPE1__SDMA0_IB_OFFSET_FETCH__SHIFT                                                     0x3
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PROGRAM__SHIFT                                                             0x4
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS_REG__SHIFT                                                          0x5
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS1_REG__SHIFT                                                         0x6
+#define SDMA0_PUB_REG_TYPE1__SDMA0_CNTL1__SHIFT                                                               0x7
+#define SDMA0_PUB_REG_TYPE1__SDMA0_HBM_PAGE_CONFIG__SHIFT                                                     0x8
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UCODE_CHECKSUM__SHIFT                                                      0x9
+#define SDMA0_PUB_REG_TYPE1__RESERVED10__SHIFT                                                                0xa
+#define SDMA0_PUB_REG_TYPE1__SDMA0_FREEZE__SHIFT                                                              0xb
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PROCESS_QUANTUM0__SHIFT                                                    0xc
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PROCESS_QUANTUM1__SHIFT                                                    0xd
+#define SDMA0_PUB_REG_TYPE1__SDMA0_WATCHDOG_CNTL__SHIFT                                                       0xe
+#define SDMA0_PUB_REG_TYPE1__RESERVED15__SHIFT                                                                0xf
+#define SDMA0_PUB_REG_TYPE1__RESERVED16__SHIFT                                                                0x10
+#define SDMA0_PUB_REG_TYPE1__RESERVED17__SHIFT                                                                0x11
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_CONFIG__SHIFT                                                          0x12
+#define SDMA0_PUB_REG_TYPE1__SDMA0_BA_THRESHOLD__SHIFT                                                        0x13
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ID__SHIFT                                                                  0x14
+#define SDMA0_PUB_REG_TYPE1__SDMA0_VERSION__SHIFT                                                             0x15
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER__SHIFT                                                         0x16
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_CLEAR__SHIFT                                                   0x17
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS2_REG__SHIFT                                                         0x18
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_CNTL__SHIFT                                                         0x19
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_LO__SHIFT                                                     0x1a
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_HI__SHIFT                                                     0x1b
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_CNTL__SHIFT                                                          0x1c
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WATERMK__SHIFT                                                       0x1d
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_TIMEOUT__SHIFT                                                       0x1e
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_PAGE__SHIFT                                                          0x1f
+#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_MASK                                                         0x00000001L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_RB_RPTR_FETCH_HI_MASK                                                      0x00000002L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_SEM_WAIT_FAIL_TIMER_CNTL_MASK                                              0x00000004L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_IB_OFFSET_FETCH_MASK                                                       0x00000008L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PROGRAM_MASK                                                               0x00000010L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS_REG_MASK                                                            0x00000020L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS1_REG_MASK                                                           0x00000040L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_CNTL1_MASK                                                                 0x00000080L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_HBM_PAGE_CONFIG_MASK                                                       0x00000100L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UCODE_CHECKSUM_MASK                                                        0x00000200L
+#define SDMA0_PUB_REG_TYPE1__RESERVED10_MASK                                                                  0x00000400L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_FREEZE_MASK                                                                0x00000800L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PROCESS_QUANTUM0_MASK                                                      0x00001000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_PROCESS_QUANTUM1_MASK                                                      0x00002000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_WATCHDOG_CNTL_MASK                                                         0x00004000L
+#define SDMA0_PUB_REG_TYPE1__RESERVED15_MASK                                                                  0x00008000L
+#define SDMA0_PUB_REG_TYPE1__RESERVED16_MASK                                                                  0x00010000L
+#define SDMA0_PUB_REG_TYPE1__RESERVED17_MASK                                                                  0x00020000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_CONFIG_MASK                                                            0x00040000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_BA_THRESHOLD_MASK                                                          0x00080000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ID_MASK                                                                    0x00100000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_VERSION_MASK                                                               0x00200000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_MASK                                                           0x00400000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_EDC_COUNTER_CLEAR_MASK                                                     0x00800000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_STATUS2_REG_MASK                                                           0x01000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_CNTL_MASK                                                           0x02000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_LO_MASK                                                       0x04000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_ATOMIC_PREOP_HI_MASK                                                       0x08000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_CNTL_MASK                                                            0x10000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_WATERMK_MASK                                                         0x20000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_TIMEOUT_MASK                                                         0x40000000L
+#define SDMA0_PUB_REG_TYPE1__SDMA0_UTCL1_PAGE_MASK                                                            0x80000000L
+//SDMA0_PUB_REG_TYPE2
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_STATUS__SHIFT                                                     0x0
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_STATUS__SHIFT                                                     0x1
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV0__SHIFT                                                          0x2
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV1__SHIFT                                                          0x3
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV2__SHIFT                                                          0x4
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK0__SHIFT                                                     0x5
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK1__SHIFT                                                     0x6
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK0__SHIFT                                                     0x7
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK1__SHIFT                                                     0x8
+#define SDMA0_PUB_REG_TYPE2__SDMA0_RELAX_ORDERING_LUT__SHIFT                                                  0xa
+#define SDMA0_PUB_REG_TYPE2__SDMA0_CHICKEN_BITS_2__SHIFT                                                      0xb
+#define SDMA0_PUB_REG_TYPE2__SDMA0_STATUS3_REG__SHIFT                                                         0xc
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_LO__SHIFT                                                    0xd
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_HI__SHIFT                                                    0xe
+#define SDMA0_PUB_REG_TYPE2__SDMA0_GLOBAL_QUANTUM__SHIFT                                                      0xf
+#define SDMA0_PUB_REG_TYPE2__SDMA0_ERROR_LOG__SHIFT                                                           0x10
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG0__SHIFT                                                      0x11
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG1__SHIFT                                                      0x12
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG2__SHIFT                                                      0x13
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG3__SHIFT                                                      0x14
+#define SDMA0_PUB_REG_TYPE2__SDMA0_F32_COUNTER__SHIFT                                                         0x15
+#define SDMA0_PUB_REG_TYPE2__RESERVE_22_22__SHIFT                                                             0x16
+#define SDMA0_PUB_REG_TYPE2__RESERVED23__SHIFT                                                                0x17
+#define SDMA0_PUB_REG_TYPE2__RESERVED24__SHIFT                                                                0x18
+#define SDMA0_PUB_REG_TYPE2__RESERVED25__SHIFT                                                                0x19
+#define SDMA0_PUB_REG_TYPE2__RESERVED26__SHIFT                                                                0x1a
+#define SDMA0_PUB_REG_TYPE2__SDMA0_CRD_CNTL__SHIFT                                                            0x1b
+#define SDMA0_PUB_REG_TYPE2__SDMA0_RLC_CGCG_CTRL__SHIFT                                                       0x1c
+#define SDMA0_PUB_REG_TYPE2__SDMA0_GPU_IOV_VIOLATION_LOG__SHIFT                                               0x1d
+#define SDMA0_PUB_REG_TYPE2__SDMA0_AQL_STATUS__SHIFT                                                          0x1f
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_STATUS_MASK                                                       0x00000001L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_STATUS_MASK                                                       0x00000002L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV0_MASK                                                            0x00000004L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV1_MASK                                                            0x00000008L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_INV2_MASK                                                            0x00000010L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK0_MASK                                                       0x00000020L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_RD_XNACK1_MASK                                                       0x00000040L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK0_MASK                                                       0x00000080L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_UTCL1_WR_XNACK1_MASK                                                       0x00000100L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_RELAX_ORDERING_LUT_MASK                                                    0x00000400L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_CHICKEN_BITS_2_MASK                                                        0x00000800L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_STATUS3_REG_MASK                                                           0x00001000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_LO_MASK                                                      0x00002000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PHYSICAL_ADDR_HI_MASK                                                      0x00004000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_GLOBAL_QUANTUM_MASK                                                        0x00008000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_ERROR_LOG_MASK                                                             0x00010000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG0_MASK                                                        0x00020000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG1_MASK                                                        0x00040000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG2_MASK                                                        0x00080000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_PUB_DUMMY_REG3_MASK                                                        0x00100000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_F32_COUNTER_MASK                                                           0x00200000L
+#define SDMA0_PUB_REG_TYPE2__RESERVE_22_22_MASK                                                               0x00400000L
+#define SDMA0_PUB_REG_TYPE2__RESERVED23_MASK                                                                  0x00800000L
+#define SDMA0_PUB_REG_TYPE2__RESERVED24_MASK                                                                  0x01000000L
+#define SDMA0_PUB_REG_TYPE2__RESERVED25_MASK                                                                  0x02000000L
+#define SDMA0_PUB_REG_TYPE2__RESERVED26_MASK                                                                  0x04000000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_CRD_CNTL_MASK                                                              0x08000000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_RLC_CGCG_CTRL_MASK                                                         0x10000000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_GPU_IOV_VIOLATION_LOG_MASK                                                 0x20000000L
+#define SDMA0_PUB_REG_TYPE2__SDMA0_AQL_STATUS_MASK                                                            0x80000000L
+//SDMA0_PUB_REG_TYPE3
+#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_DATA__SHIFT                                                   0x0
+#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_INDEX__SHIFT                                                  0x1
+#define SDMA0_PUB_REG_TYPE3__SDMA0_TLBI_GCR_CNTL__SHIFT                                                       0x2
+#define SDMA0_PUB_REG_TYPE3__SDMA0_TILING_CONFIG__SHIFT                                                       0x3
+#define SDMA0_PUB_REG_TYPE3__SDMA0_HASH__SHIFT                                                                0x4
+#define SDMA0_PUB_REG_TYPE3__RESERVED5__SHIFT                                                                 0x5
+#define SDMA0_PUB_REG_TYPE3__RESERVED__SHIFT                                                                  0x6
+#define SDMA0_PUB_REG_TYPE3__RESERVED7__SHIFT                                                                 0x7
+#define SDMA0_PUB_REG_TYPE3__SDMA0_CE_CTRL__SHIFT                                                             0x8
+#define SDMA0_PUB_REG_TYPE3__SDMA0_FED_STATUS__SHIFT                                                          0x9
+#define SDMA0_PUB_REG_TYPE3__RESERVED10__SHIFT                                                                0xa
+#define SDMA0_PUB_REG_TYPE3__RESERVED11__SHIFT                                                                0xb
+#define SDMA0_PUB_REG_TYPE3__RESERVED12__SHIFT                                                                0xc
+#define SDMA0_PUB_REG_TYPE3__RESERVED13__SHIFT                                                                0xd
+#define SDMA0_PUB_REG_TYPE3__RESERVED14__SHIFT                                                                0xe
+#define SDMA0_PUB_REG_TYPE3__RESERVED15__SHIFT                                                                0xf
+#define SDMA0_PUB_REG_TYPE3__SDMA0_INT_STATUS__SHIFT                                                          0x10
+#define SDMA0_PUB_REG_TYPE3__SDMA0_GPU_IOV_VIOLATION_LOG2__SHIFT                                              0x11
+#define SDMA0_PUB_REG_TYPE3__SDMA0_HOLE_ADDR_LO__SHIFT                                                        0x12
+#define SDMA0_PUB_REG_TYPE3__SDMA0_HOLE_ADDR_HI__SHIFT                                                        0x13
+#define SDMA0_PUB_REG_TYPE3__RESERVED20__SHIFT                                                                0x14
+#define SDMA0_PUB_REG_TYPE3__SDMA0_CLOCK_GATING_STATUS__SHIFT                                                 0x15
+#define SDMA0_PUB_REG_TYPE3__SDMA0_STATUS4_REG__SHIFT                                                         0x16
+#define SDMA0_PUB_REG_TYPE3__SDMA0_SCRATCH_RAM_DATA__SHIFT                                                    0x17
+#define SDMA0_PUB_REG_TYPE3__SDMA0_SCRATCH_RAM_ADDR__SHIFT                                                    0x18
+#define SDMA0_PUB_REG_TYPE3__SDMA0_TIMESTAMP_CNTL__SHIFT                                                      0x19
+#define SDMA0_PUB_REG_TYPE3__RESERVED26__SHIFT                                                                0x1a
+#define SDMA0_PUB_REG_TYPE3__RESERVED27__SHIFT                                                                0x1b
+#define SDMA0_PUB_REG_TYPE3__SDMA0_STATUS5_REG__SHIFT                                                         0x1c
+#define SDMA0_PUB_REG_TYPE3__SDMA0_QUEUE_RESET_REQ__SHIFT                                                     0x1d
+#define SDMA0_PUB_REG_TYPE3__SDMA0_STATUS6_REG__SHIFT                                                         0x1e
+#define SDMA0_PUB_REG_TYPE3__SDMA0_UCODE1_CHECKSUM__SHIFT                                                     0x1f
+#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_DATA_MASK                                                     0x00000001L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_EA_DBIT_ADDR_INDEX_MASK                                                    0x00000002L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_TLBI_GCR_CNTL_MASK                                                         0x00000004L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_TILING_CONFIG_MASK                                                         0x00000008L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_HASH_MASK                                                                  0x00000010L
+#define SDMA0_PUB_REG_TYPE3__RESERVED5_MASK                                                                   0x00000020L
+#define SDMA0_PUB_REG_TYPE3__RESERVED_MASK                                                                    0x00000040L
+#define SDMA0_PUB_REG_TYPE3__RESERVED7_MASK                                                                   0x00000080L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_CE_CTRL_MASK                                                               0x00000100L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_FED_STATUS_MASK                                                            0x00000200L
+#define SDMA0_PUB_REG_TYPE3__RESERVED10_MASK                                                                  0x00000400L
+#define SDMA0_PUB_REG_TYPE3__RESERVED11_MASK                                                                  0x00000800L
+#define SDMA0_PUB_REG_TYPE3__RESERVED12_MASK                                                                  0x00001000L
+#define SDMA0_PUB_REG_TYPE3__RESERVED13_MASK                                                                  0x00002000L
+#define SDMA0_PUB_REG_TYPE3__RESERVED14_MASK                                                                  0x00004000L
+#define SDMA0_PUB_REG_TYPE3__RESERVED15_MASK                                                                  0x00008000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_INT_STATUS_MASK                                                            0x00010000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_GPU_IOV_VIOLATION_LOG2_MASK                                                0x00020000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_HOLE_ADDR_LO_MASK                                                          0x00040000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_HOLE_ADDR_HI_MASK                                                          0x00080000L
+#define SDMA0_PUB_REG_TYPE3__RESERVED20_MASK                                                                  0x00100000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_CLOCK_GATING_STATUS_MASK                                                   0x00200000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_STATUS4_REG_MASK                                                           0x00400000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_SCRATCH_RAM_DATA_MASK                                                      0x00800000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_SCRATCH_RAM_ADDR_MASK                                                      0x01000000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_TIMESTAMP_CNTL_MASK                                                        0x02000000L
+#define SDMA0_PUB_REG_TYPE3__RESERVED26_MASK                                                                  0x04000000L
+#define SDMA0_PUB_REG_TYPE3__RESERVED27_MASK                                                                  0x08000000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_STATUS5_REG_MASK                                                           0x10000000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_QUEUE_RESET_REQ_MASK                                                       0x20000000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_STATUS6_REG_MASK                                                           0x40000000L
+#define SDMA0_PUB_REG_TYPE3__SDMA0_UCODE1_CHECKSUM_MASK                                                       0x80000000L
+//SDMA0_VM_CNTL
+#define SDMA0_VM_CNTL__CMD__SHIFT                                                                             0x0
+#define SDMA0_VM_CNTL__CMD_MASK                                                                               0x0000000FL
+//SDMA0_F32_CNTL
+#define SDMA0_F32_CNTL__HALT__SHIFT                                                                           0x0
+#define SDMA0_F32_CNTL__DBG_SELECT_BITS__SHIFT                                                                0x2
+#define SDMA0_F32_CNTL__TH0_CHECKSUM_CLR__SHIFT                                                               0x8
+#define SDMA0_F32_CNTL__TH0_RESET__SHIFT                                                                      0x9
+#define SDMA0_F32_CNTL__TH0_ENABLE__SHIFT                                                                     0xa
+#define SDMA0_F32_CNTL__TH1_CHECKSUM_CLR__SHIFT                                                               0xc
+#define SDMA0_F32_CNTL__TH1_RESET__SHIFT                                                                      0xd
+#define SDMA0_F32_CNTL__TH1_ENABLE__SHIFT                                                                     0xe
+#define SDMA0_F32_CNTL__TH0_PRIORITY__SHIFT                                                                   0x10
+#define SDMA0_F32_CNTL__TH1_PRIORITY__SHIFT                                                                   0x18
+#define SDMA0_F32_CNTL__HALT_MASK                                                                             0x00000001L
+#define SDMA0_F32_CNTL__DBG_SELECT_BITS_MASK                                                                  0x000000FCL
+#define SDMA0_F32_CNTL__TH0_CHECKSUM_CLR_MASK                                                                 0x00000100L
+#define SDMA0_F32_CNTL__TH0_RESET_MASK                                                                        0x00000200L
+#define SDMA0_F32_CNTL__TH0_ENABLE_MASK                                                                       0x00000400L
+#define SDMA0_F32_CNTL__TH1_CHECKSUM_CLR_MASK                                                                 0x00001000L
+#define SDMA0_F32_CNTL__TH1_RESET_MASK                                                                        0x00002000L
+#define SDMA0_F32_CNTL__TH1_ENABLE_MASK                                                                       0x00004000L
+#define SDMA0_F32_CNTL__TH0_PRIORITY_MASK                                                                     0x00FF0000L
+#define SDMA0_F32_CNTL__TH1_PRIORITY_MASK                                                                     0xFF000000L
+
+
+// addressBlock: gc_sdma0_sdma1hypdec
+//SDMA1_UCODE_ADDR
+#define SDMA1_UCODE_ADDR__VALUE__SHIFT                                                                        0x0
+#define SDMA1_UCODE_ADDR__THID__SHIFT                                                                         0xf
+#define SDMA1_UCODE_ADDR__VALUE_MASK                                                                          0x00001FFFL
+#define SDMA1_UCODE_ADDR__THID_MASK                                                                           0x00008000L
+//SDMA1_UCODE_DATA
+#define SDMA1_UCODE_DATA__VALUE__SHIFT                                                                        0x0
+#define SDMA1_UCODE_DATA__VALUE_MASK                                                                          0xFFFFFFFFL
+//SDMA1_UCODE_SELFLOAD_CONTROL
+#define SDMA1_UCODE_SELFLOAD_CONTROL__GPA__SHIFT                                                              0x0
+#define SDMA1_UCODE_SELFLOAD_CONTROL__SYS__SHIFT                                                              0x1
+#define SDMA1_UCODE_SELFLOAD_CONTROL__CID__SHIFT                                                              0x4
+#define SDMA1_UCODE_SELFLOAD_CONTROL__CACHE_POLICY__SHIFT                                                     0x8
+#define SDMA1_UCODE_SELFLOAD_CONTROL__GPA_MASK                                                                0x00000001L
+#define SDMA1_UCODE_SELFLOAD_CONTROL__SYS_MASK                                                                0x00000002L
+#define SDMA1_UCODE_SELFLOAD_CONTROL__CID_MASK                                                                0x000000F0L
+#define SDMA1_UCODE_SELFLOAD_CONTROL__CACHE_POLICY_MASK                                                       0x00000300L
+//SDMA1_BROADCAST_UCODE_ADDR
+#define SDMA1_BROADCAST_UCODE_ADDR__VALUE__SHIFT                                                              0x0
+#define SDMA1_BROADCAST_UCODE_ADDR__THID__SHIFT                                                               0xf
+#define SDMA1_BROADCAST_UCODE_ADDR__VALUE_MASK                                                                0x00001FFFL
+#define SDMA1_BROADCAST_UCODE_ADDR__THID_MASK                                                                 0x00008000L
+//SDMA1_BROADCAST_UCODE_DATA
+#define SDMA1_BROADCAST_UCODE_DATA__VALUE__SHIFT                                                              0x0
+#define SDMA1_BROADCAST_UCODE_DATA__VALUE_MASK                                                                0xFFFFFFFFL
+//SDMA1_VM_CTX_LO
+#define SDMA1_VM_CTX_LO__ADDR__SHIFT                                                                          0x2
+#define SDMA1_VM_CTX_LO__ADDR_MASK                                                                            0xFFFFFFFCL
+//SDMA1_VM_CTX_HI
+#define SDMA1_VM_CTX_HI__ADDR__SHIFT                                                                          0x0
+#define SDMA1_VM_CTX_HI__ADDR_MASK                                                                            0xFFFFFFFFL
+//SDMA1_ACTIVE_FCN_ID
+#define SDMA1_ACTIVE_FCN_ID__VFID__SHIFT                                                                      0x0
+#define SDMA1_ACTIVE_FCN_ID__RESERVED__SHIFT                                                                  0x4
+#define SDMA1_ACTIVE_FCN_ID__VF__SHIFT                                                                        0x1f
+#define SDMA1_ACTIVE_FCN_ID__VFID_MASK                                                                        0x0000000FL
+#define SDMA1_ACTIVE_FCN_ID__RESERVED_MASK                                                                    0x7FFFFFF0L
+#define SDMA1_ACTIVE_FCN_ID__VF_MASK                                                                          0x80000000L
+//SDMA1_VM_CTX_CNTL
+#define SDMA1_VM_CTX_CNTL__PRIV__SHIFT                                                                        0x0
+#define SDMA1_VM_CTX_CNTL__VMID__SHIFT                                                                        0x4
+#define SDMA1_VM_CTX_CNTL__MEM_PHY__SHIFT                                                                     0x8
+#define SDMA1_VM_CTX_CNTL__BUSY_STATUS_REPORT_ENABLE__SHIFT                                                   0x10
+#define SDMA1_VM_CTX_CNTL__PRIV_MASK                                                                          0x00000001L
+#define SDMA1_VM_CTX_CNTL__VMID_MASK                                                                          0x000000F0L
+#define SDMA1_VM_CTX_CNTL__MEM_PHY_MASK                                                                       0x00000300L
+#define SDMA1_VM_CTX_CNTL__BUSY_STATUS_REPORT_ENABLE_MASK                                                     0x00010000L
+//SDMA1_VIRT_RESET_REQ
+#define SDMA1_VIRT_RESET_REQ__VF__SHIFT                                                                       0x0
+#define SDMA1_VIRT_RESET_REQ__PF__SHIFT                                                                       0x1f
+#define SDMA1_VIRT_RESET_REQ__VF_MASK                                                                         0x0000FFFFL
+#define SDMA1_VIRT_RESET_REQ__PF_MASK                                                                         0x80000000L
+//SDMA1_CONTEXT_REG_TYPE0
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_CNTL__SHIFT                                                  0x0
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_BASE__SHIFT                                                  0x1
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_BASE_HI__SHIFT                                               0x2
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_RPTR__SHIFT                                                  0x3
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_RPTR_HI__SHIFT                                               0x4
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_WPTR__SHIFT                                                  0x5
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_WPTR_HI__SHIFT                                               0x6
+#define SDMA1_CONTEXT_REG_TYPE0__RESERVED7__SHIFT                                                             0x7
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_RPTR_ADDR_HI__SHIFT                                          0x8
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_RPTR_ADDR_LO__SHIFT                                          0x9
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_CNTL__SHIFT                                                  0xa
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_RPTR__SHIFT                                                  0xb
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_OFFSET__SHIFT                                                0xc
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_BASE_LO__SHIFT                                               0xd
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_BASE_HI__SHIFT                                               0xe
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_SIZE__SHIFT                                                  0xf
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_SKIP_CNTL__SHIFT                                                0x10
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_CONTEXT_STATUS__SHIFT                                           0x11
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_DOORBELL__SHIFT                                                 0x12
+#define SDMA1_CONTEXT_REG_TYPE0__RESERVED31_19__SHIFT                                                         0x13
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_CNTL_MASK                                                    0x00000001L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_BASE_MASK                                                    0x00000002L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_BASE_HI_MASK                                                 0x00000004L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_RPTR_MASK                                                    0x00000008L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_RPTR_HI_MASK                                                 0x00000010L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_WPTR_MASK                                                    0x00000020L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_WPTR_HI_MASK                                                 0x00000040L
+#define SDMA1_CONTEXT_REG_TYPE0__RESERVED7_MASK                                                               0x00000080L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_RPTR_ADDR_HI_MASK                                            0x00000100L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_RB_RPTR_ADDR_LO_MASK                                            0x00000200L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_CNTL_MASK                                                    0x00000400L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_RPTR_MASK                                                    0x00000800L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_OFFSET_MASK                                                  0x00001000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_BASE_LO_MASK                                                 0x00002000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_BASE_HI_MASK                                                 0x00004000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_IB_SIZE_MASK                                                    0x00008000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_SKIP_CNTL_MASK                                                  0x00010000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_CONTEXT_STATUS_MASK                                             0x00020000L
+#define SDMA1_CONTEXT_REG_TYPE0__SDMA1_QUEUE0_DOORBELL_MASK                                                   0x00040000L
+#define SDMA1_CONTEXT_REG_TYPE0__RESERVED31_19_MASK                                                           0xFFF80000L
+//SDMA1_CONTEXT_REG_TYPE1
+#define SDMA1_CONTEXT_REG_TYPE1__RESERVED8_0__SHIFT                                                           0x0
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_DOORBELL_LOG__SHIFT                                             0x9
+#define SDMA1_CONTEXT_REG_TYPE1__RESERVED10__SHIFT                                                            0xa
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_DOORBELL_OFFSET__SHIFT                                          0xb
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_CSA_ADDR_LO__SHIFT                                              0xc
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_CSA_ADDR_HI__SHIFT                                              0xd
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_SCHEDULE_CNTL__SHIFT                                            0xe
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_IB_SUB_REMAIN__SHIFT                                            0xf
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_PREEMPT__SHIFT                                                  0x10
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_DUMMY_REG__SHIFT                                                0x11
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_RB_WPTR_POLL_ADDR_HI__SHIFT                                     0x12
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_RB_WPTR_POLL_ADDR_LO__SHIFT                                     0x13
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_RB_AQL_CNTL__SHIFT                                              0x14
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_MINOR_PTR_UPDATE__SHIFT                                         0x15
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_RB_PREEMPT__SHIFT                                               0x16
+#define SDMA1_CONTEXT_REG_TYPE1__RESERVED__SHIFT                                                              0x17
+#define SDMA1_CONTEXT_REG_TYPE1__RESERVED8_0_MASK                                                             0x000001FFL
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_DOORBELL_LOG_MASK                                               0x00000200L
+#define SDMA1_CONTEXT_REG_TYPE1__RESERVED10_MASK                                                              0x00000400L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_DOORBELL_OFFSET_MASK                                            0x00000800L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_CSA_ADDR_LO_MASK                                                0x00001000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_CSA_ADDR_HI_MASK                                                0x00002000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_SCHEDULE_CNTL_MASK                                              0x00004000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_IB_SUB_REMAIN_MASK                                              0x00008000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_PREEMPT_MASK                                                    0x00010000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_DUMMY_REG_MASK                                                  0x00020000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_RB_WPTR_POLL_ADDR_HI_MASK                                       0x00040000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_RB_WPTR_POLL_ADDR_LO_MASK                                       0x00080000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_RB_AQL_CNTL_MASK                                                0x00100000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_MINOR_PTR_UPDATE_MASK                                           0x00200000L
+#define SDMA1_CONTEXT_REG_TYPE1__SDMA1_QUEUE0_RB_PREEMPT_MASK                                                 0x00400000L
+#define SDMA1_CONTEXT_REG_TYPE1__RESERVED_MASK                                                                0xFF800000L
+//SDMA1_CONTEXT_REG_TYPE2
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA0__SHIFT                                             0x0
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA1__SHIFT                                             0x1
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA2__SHIFT                                             0x2
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA3__SHIFT                                             0x3
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA4__SHIFT                                             0x4
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA5__SHIFT                                             0x5
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA6__SHIFT                                             0x6
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA7__SHIFT                                             0x7
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA8__SHIFT                                             0x8
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA9__SHIFT                                             0x9
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA10__SHIFT                                            0xa
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_CNTL__SHIFT                                              0xb
+#define SDMA1_CONTEXT_REG_TYPE2__RESERVED__SHIFT                                                              0xe
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA0_MASK                                               0x00000001L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA1_MASK                                               0x00000002L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA2_MASK                                               0x00000004L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA3_MASK                                               0x00000008L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA4_MASK                                               0x00000010L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA5_MASK                                               0x00000020L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA6_MASK                                               0x00000040L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA7_MASK                                               0x00000080L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA8_MASK                                               0x00000100L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA9_MASK                                               0x00000200L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_DATA10_MASK                                              0x00000400L
+#define SDMA1_CONTEXT_REG_TYPE2__SDMA1_QUEUE0_MIDCMD_CNTL_MASK                                                0x00000800L
+#define SDMA1_CONTEXT_REG_TYPE2__RESERVED_MASK                                                                0xFFFFC000L
+//SDMA1_PUB_REG_TYPE0
+#define SDMA1_PUB_REG_TYPE0__SDMA1_DEC_START__SHIFT                                                           0x0
+#define SDMA1_PUB_REG_TYPE0__RESERVED_10_1__SHIFT                                                             0x1
+#define SDMA1_PUB_REG_TYPE0__SDMA1_F32_MISC_CNTL__SHIFT                                                       0xb
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GLOBAL_TIMESTAMP_LO__SHIFT                                                 0xf
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GLOBAL_TIMESTAMP_HI__SHIFT                                                 0x10
+#define SDMA1_PUB_REG_TYPE0__RESERVED22__SHIFT                                                                0x16
+#define SDMA1_PUB_REG_TYPE0__RESERVED23__SHIFT                                                                0x17
+#define SDMA1_PUB_REG_TYPE0__RESERVED24__SHIFT                                                                0x18
+#define SDMA1_PUB_REG_TYPE0__RESERVED25__SHIFT                                                                0x19
+#define SDMA1_PUB_REG_TYPE0__SDMA1_POWER_CNTL__SHIFT                                                          0x1a
+#define SDMA1_PUB_REG_TYPE0__RESERVED27__SHIFT                                                                0x1b
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CNTL__SHIFT                                                                0x1c
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CHICKEN_BITS__SHIFT                                                        0x1d
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG__SHIFT                                                      0x1e
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG_READ__SHIFT                                                 0x1f
+#define SDMA1_PUB_REG_TYPE0__SDMA1_DEC_START_MASK                                                             0x00000001L
+#define SDMA1_PUB_REG_TYPE0__RESERVED_10_1_MASK                                                               0x000007FEL
+#define SDMA1_PUB_REG_TYPE0__SDMA1_F32_MISC_CNTL_MASK                                                         0x00000800L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GLOBAL_TIMESTAMP_LO_MASK                                                   0x00008000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GLOBAL_TIMESTAMP_HI_MASK                                                   0x00010000L
+#define SDMA1_PUB_REG_TYPE0__RESERVED22_MASK                                                                  0x00400000L
+#define SDMA1_PUB_REG_TYPE0__RESERVED23_MASK                                                                  0x00800000L
+#define SDMA1_PUB_REG_TYPE0__RESERVED24_MASK                                                                  0x01000000L
+#define SDMA1_PUB_REG_TYPE0__RESERVED25_MASK                                                                  0x02000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_POWER_CNTL_MASK                                                            0x04000000L
+#define SDMA1_PUB_REG_TYPE0__RESERVED27_MASK                                                                  0x08000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CNTL_MASK                                                                  0x10000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_CHICKEN_BITS_MASK                                                          0x20000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG_MASK                                                        0x40000000L
+#define SDMA1_PUB_REG_TYPE0__SDMA1_GB_ADDR_CONFIG_READ_MASK                                                   0x80000000L
+//SDMA1_PUB_REG_TYPE1
+#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH__SHIFT                                                       0x0
+#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH_HI__SHIFT                                                    0x1
+#define SDMA1_PUB_REG_TYPE1__SDMA1_SEM_WAIT_FAIL_TIMER_CNTL__SHIFT                                            0x2
+#define SDMA1_PUB_REG_TYPE1__SDMA1_IB_OFFSET_FETCH__SHIFT                                                     0x3
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PROGRAM__SHIFT                                                             0x4
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS_REG__SHIFT                                                          0x5
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS1_REG__SHIFT                                                         0x6
+#define SDMA1_PUB_REG_TYPE1__SDMA1_CNTL1__SHIFT                                                               0x7
+#define SDMA1_PUB_REG_TYPE1__SDMA1_HBM_PAGE_CONFIG__SHIFT                                                     0x8
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UCODE_CHECKSUM__SHIFT                                                      0x9
+#define SDMA1_PUB_REG_TYPE1__RESERVED10__SHIFT                                                                0xa
+#define SDMA1_PUB_REG_TYPE1__SDMA1_FREEZE__SHIFT                                                              0xb
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PROCESS_QUANTUM0__SHIFT                                                    0xc
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PROCESS_QUANTUM1__SHIFT                                                    0xd
+#define SDMA1_PUB_REG_TYPE1__SDMA1_WATCHDOG_CNTL__SHIFT                                                       0xe
+#define SDMA1_PUB_REG_TYPE1__RESERVED15__SHIFT                                                                0xf
+#define SDMA1_PUB_REG_TYPE1__RESERVED16__SHIFT                                                                0x10
+#define SDMA1_PUB_REG_TYPE1__RESERVED17__SHIFT                                                                0x11
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_CONFIG__SHIFT                                                          0x12
+#define SDMA1_PUB_REG_TYPE1__SDMA1_BA_THRESHOLD__SHIFT                                                        0x13
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ID__SHIFT                                                                  0x14
+#define SDMA1_PUB_REG_TYPE1__SDMA1_VERSION__SHIFT                                                             0x15
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER__SHIFT                                                         0x16
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER_CLEAR__SHIFT                                                   0x17
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS2_REG__SHIFT                                                         0x18
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_CNTL__SHIFT                                                         0x19
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_LO__SHIFT                                                     0x1a
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_HI__SHIFT                                                     0x1b
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_CNTL__SHIFT                                                          0x1c
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_WATERMK__SHIFT                                                       0x1d
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_TIMEOUT__SHIFT                                                       0x1e
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_PAGE__SHIFT                                                          0x1f
+#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH_MASK                                                         0x00000001L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_RB_RPTR_FETCH_HI_MASK                                                      0x00000002L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_SEM_WAIT_FAIL_TIMER_CNTL_MASK                                              0x00000004L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_IB_OFFSET_FETCH_MASK                                                       0x00000008L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PROGRAM_MASK                                                               0x00000010L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS_REG_MASK                                                            0x00000020L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS1_REG_MASK                                                           0x00000040L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_CNTL1_MASK                                                                 0x00000080L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_HBM_PAGE_CONFIG_MASK                                                       0x00000100L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UCODE_CHECKSUM_MASK                                                        0x00000200L
+#define SDMA1_PUB_REG_TYPE1__RESERVED10_MASK                                                                  0x00000400L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_FREEZE_MASK                                                                0x00000800L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PROCESS_QUANTUM0_MASK                                                      0x00001000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_PROCESS_QUANTUM1_MASK                                                      0x00002000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_WATCHDOG_CNTL_MASK                                                         0x00004000L
+#define SDMA1_PUB_REG_TYPE1__RESERVED15_MASK                                                                  0x00008000L
+#define SDMA1_PUB_REG_TYPE1__RESERVED16_MASK                                                                  0x00010000L
+#define SDMA1_PUB_REG_TYPE1__RESERVED17_MASK                                                                  0x00020000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_CONFIG_MASK                                                            0x00040000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_BA_THRESHOLD_MASK                                                          0x00080000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ID_MASK                                                                    0x00100000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_VERSION_MASK                                                               0x00200000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER_MASK                                                           0x00400000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_EDC_COUNTER_CLEAR_MASK                                                     0x00800000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_STATUS2_REG_MASK                                                           0x01000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_CNTL_MASK                                                           0x02000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_LO_MASK                                                       0x04000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_ATOMIC_PREOP_HI_MASK                                                       0x08000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_CNTL_MASK                                                            0x10000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_WATERMK_MASK                                                         0x20000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_TIMEOUT_MASK                                                         0x40000000L
+#define SDMA1_PUB_REG_TYPE1__SDMA1_UTCL1_PAGE_MASK                                                            0x80000000L
+//SDMA1_PUB_REG_TYPE2
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_STATUS__SHIFT                                                     0x0
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_STATUS__SHIFT                                                     0x1
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV0__SHIFT                                                          0x2
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV1__SHIFT                                                          0x3
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV2__SHIFT                                                          0x4
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK0__SHIFT                                                     0x5
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK1__SHIFT                                                     0x6
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK0__SHIFT                                                     0x7
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK1__SHIFT                                                     0x8
+#define SDMA1_PUB_REG_TYPE2__SDMA1_RELAX_ORDERING_LUT__SHIFT                                                  0xa
+#define SDMA1_PUB_REG_TYPE2__SDMA1_CHICKEN_BITS_2__SHIFT                                                      0xb
+#define SDMA1_PUB_REG_TYPE2__SDMA1_STATUS3_REG__SHIFT                                                         0xc
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_LO__SHIFT                                                    0xd
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_HI__SHIFT                                                    0xe
+#define SDMA1_PUB_REG_TYPE2__SDMA1_GLOBAL_QUANTUM__SHIFT                                                      0xf
+#define SDMA1_PUB_REG_TYPE2__SDMA1_ERROR_LOG__SHIFT                                                           0x10
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG0__SHIFT                                                      0x11
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG1__SHIFT                                                      0x12
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG2__SHIFT                                                      0x13
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG3__SHIFT                                                      0x14
+#define SDMA1_PUB_REG_TYPE2__SDMA1_F32_COUNTER__SHIFT                                                         0x15
+#define SDMA1_PUB_REG_TYPE2__RESERVE_22_22__SHIFT                                                             0x16
+#define SDMA1_PUB_REG_TYPE2__RESERVED23__SHIFT                                                                0x17
+#define SDMA1_PUB_REG_TYPE2__RESERVED24__SHIFT                                                                0x18
+#define SDMA1_PUB_REG_TYPE2__RESERVED25__SHIFT                                                                0x19
+#define SDMA1_PUB_REG_TYPE2__RESERVED26__SHIFT                                                                0x1a
+#define SDMA1_PUB_REG_TYPE2__SDMA1_CRD_CNTL__SHIFT                                                            0x1b
+#define SDMA1_PUB_REG_TYPE2__SDMA1_RLC_CGCG_CTRL__SHIFT                                                       0x1c
+#define SDMA1_PUB_REG_TYPE2__SDMA1_GPU_IOV_VIOLATION_LOG__SHIFT                                               0x1d
+#define SDMA1_PUB_REG_TYPE2__SDMA1_AQL_STATUS__SHIFT                                                          0x1f
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_STATUS_MASK                                                       0x00000001L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_STATUS_MASK                                                       0x00000002L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV0_MASK                                                            0x00000004L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV1_MASK                                                            0x00000008L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_INV2_MASK                                                            0x00000010L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK0_MASK                                                       0x00000020L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_RD_XNACK1_MASK                                                       0x00000040L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK0_MASK                                                       0x00000080L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_UTCL1_WR_XNACK1_MASK                                                       0x00000100L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_RELAX_ORDERING_LUT_MASK                                                    0x00000400L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_CHICKEN_BITS_2_MASK                                                        0x00000800L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_STATUS3_REG_MASK                                                           0x00001000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_LO_MASK                                                      0x00002000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PHYSICAL_ADDR_HI_MASK                                                      0x00004000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_GLOBAL_QUANTUM_MASK                                                        0x00008000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_ERROR_LOG_MASK                                                             0x00010000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG0_MASK                                                        0x00020000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG1_MASK                                                        0x00040000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG2_MASK                                                        0x00080000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_PUB_DUMMY_REG3_MASK                                                        0x00100000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_F32_COUNTER_MASK                                                           0x00200000L
+#define SDMA1_PUB_REG_TYPE2__RESERVE_22_22_MASK                                                               0x00400000L
+#define SDMA1_PUB_REG_TYPE2__RESERVED23_MASK                                                                  0x00800000L
+#define SDMA1_PUB_REG_TYPE2__RESERVED24_MASK                                                                  0x01000000L
+#define SDMA1_PUB_REG_TYPE2__RESERVED25_MASK                                                                  0x02000000L
+#define SDMA1_PUB_REG_TYPE2__RESERVED26_MASK                                                                  0x04000000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_CRD_CNTL_MASK                                                              0x08000000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_RLC_CGCG_CTRL_MASK                                                         0x10000000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_GPU_IOV_VIOLATION_LOG_MASK                                                 0x20000000L
+#define SDMA1_PUB_REG_TYPE2__SDMA1_AQL_STATUS_MASK                                                            0x80000000L
+//SDMA1_PUB_REG_TYPE3
+#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_DATA__SHIFT                                                   0x0
+#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_INDEX__SHIFT                                                  0x1
+#define SDMA1_PUB_REG_TYPE3__SDMA1_TLBI_GCR_CNTL__SHIFT                                                       0x2
+#define SDMA1_PUB_REG_TYPE3__SDMA1_TILING_CONFIG__SHIFT                                                       0x3
+#define SDMA1_PUB_REG_TYPE3__SDMA1_HASH__SHIFT                                                                0x4
+#define SDMA1_PUB_REG_TYPE3__RESERVED5__SHIFT                                                                 0x5
+#define SDMA1_PUB_REG_TYPE3__RESERVED__SHIFT                                                                  0x6
+#define SDMA1_PUB_REG_TYPE3__RESERVED7__SHIFT                                                                 0x7
+#define SDMA1_PUB_REG_TYPE3__SDMA1_CE_CTRL__SHIFT                                                             0x8
+#define SDMA1_PUB_REG_TYPE3__SDMA1_FED_STATUS__SHIFT                                                          0x9
+#define SDMA1_PUB_REG_TYPE3__RESERVED10__SHIFT                                                                0xa
+#define SDMA1_PUB_REG_TYPE3__RESERVED11__SHIFT                                                                0xb
+#define SDMA1_PUB_REG_TYPE3__RESERVED12__SHIFT                                                                0xc
+#define SDMA1_PUB_REG_TYPE3__RESERVED13__SHIFT                                                                0xd
+#define SDMA1_PUB_REG_TYPE3__RESERVED14__SHIFT                                                                0xe
+#define SDMA1_PUB_REG_TYPE3__RESERVED15__SHIFT                                                                0xf
+#define SDMA1_PUB_REG_TYPE3__SDMA1_INT_STATUS__SHIFT                                                          0x10
+#define SDMA1_PUB_REG_TYPE3__SDMA1_GPU_IOV_VIOLATION_LOG2__SHIFT                                              0x11
+#define SDMA1_PUB_REG_TYPE3__SDMA1_HOLE_ADDR_LO__SHIFT                                                        0x12
+#define SDMA1_PUB_REG_TYPE3__SDMA1_HOLE_ADDR_HI__SHIFT                                                        0x13
+#define SDMA1_PUB_REG_TYPE3__RESERVED20__SHIFT                                                                0x14
+#define SDMA1_PUB_REG_TYPE3__SDMA1_CLOCK_GATING_STATUS__SHIFT                                                 0x15
+#define SDMA1_PUB_REG_TYPE3__SDMA1_STATUS4_REG__SHIFT                                                         0x16
+#define SDMA1_PUB_REG_TYPE3__SDMA1_SCRATCH_RAM_DATA__SHIFT                                                    0x17
+#define SDMA1_PUB_REG_TYPE3__SDMA1_SCRATCH_RAM_ADDR__SHIFT                                                    0x18
+#define SDMA1_PUB_REG_TYPE3__SDMA1_TIMESTAMP_CNTL__SHIFT                                                      0x19
+#define SDMA1_PUB_REG_TYPE3__RESERVED26__SHIFT                                                                0x1a
+#define SDMA1_PUB_REG_TYPE3__RESERVED27__SHIFT                                                                0x1b
+#define SDMA1_PUB_REG_TYPE3__SDMA1_STATUS5_REG__SHIFT                                                         0x1c
+#define SDMA1_PUB_REG_TYPE3__SDMA1_QUEUE_RESET_REQ__SHIFT                                                     0x1d
+#define SDMA1_PUB_REG_TYPE3__SDMA1_STATUS6_REG__SHIFT                                                         0x1e
+#define SDMA1_PUB_REG_TYPE3__SDMA1_UCODE1_CHECKSUM__SHIFT                                                     0x1f
+#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_DATA_MASK                                                     0x00000001L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_EA_DBIT_ADDR_INDEX_MASK                                                    0x00000002L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_TLBI_GCR_CNTL_MASK                                                         0x00000004L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_TILING_CONFIG_MASK                                                         0x00000008L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_HASH_MASK                                                                  0x00000010L
+#define SDMA1_PUB_REG_TYPE3__RESERVED5_MASK                                                                   0x00000020L
+#define SDMA1_PUB_REG_TYPE3__RESERVED_MASK                                                                    0x00000040L
+#define SDMA1_PUB_REG_TYPE3__RESERVED7_MASK                                                                   0x00000080L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_CE_CTRL_MASK                                                               0x00000100L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_FED_STATUS_MASK                                                            0x00000200L
+#define SDMA1_PUB_REG_TYPE3__RESERVED10_MASK                                                                  0x00000400L
+#define SDMA1_PUB_REG_TYPE3__RESERVED11_MASK                                                                  0x00000800L
+#define SDMA1_PUB_REG_TYPE3__RESERVED12_MASK                                                                  0x00001000L
+#define SDMA1_PUB_REG_TYPE3__RESERVED13_MASK                                                                  0x00002000L
+#define SDMA1_PUB_REG_TYPE3__RESERVED14_MASK                                                                  0x00004000L
+#define SDMA1_PUB_REG_TYPE3__RESERVED15_MASK                                                                  0x00008000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_INT_STATUS_MASK                                                            0x00010000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_GPU_IOV_VIOLATION_LOG2_MASK                                                0x00020000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_HOLE_ADDR_LO_MASK                                                          0x00040000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_HOLE_ADDR_HI_MASK                                                          0x00080000L
+#define SDMA1_PUB_REG_TYPE3__RESERVED20_MASK                                                                  0x00100000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_CLOCK_GATING_STATUS_MASK                                                   0x00200000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_STATUS4_REG_MASK                                                           0x00400000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_SCRATCH_RAM_DATA_MASK                                                      0x00800000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_SCRATCH_RAM_ADDR_MASK                                                      0x01000000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_TIMESTAMP_CNTL_MASK                                                        0x02000000L
+#define SDMA1_PUB_REG_TYPE3__RESERVED26_MASK                                                                  0x04000000L
+#define SDMA1_PUB_REG_TYPE3__RESERVED27_MASK                                                                  0x08000000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_STATUS5_REG_MASK                                                           0x10000000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_QUEUE_RESET_REQ_MASK                                                       0x20000000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_STATUS6_REG_MASK                                                           0x40000000L
+#define SDMA1_PUB_REG_TYPE3__SDMA1_UCODE1_CHECKSUM_MASK                                                       0x80000000L
+//SDMA1_VM_CNTL
+#define SDMA1_VM_CNTL__CMD__SHIFT                                                                             0x0
+#define SDMA1_VM_CNTL__CMD_MASK                                                                               0x0000000FL
+//SDMA1_F32_CNTL
+#define SDMA1_F32_CNTL__HALT__SHIFT                                                                           0x0
+#define SDMA1_F32_CNTL__DBG_SELECT_BITS__SHIFT                                                                0x2
+#define SDMA1_F32_CNTL__TH0_CHECKSUM_CLR__SHIFT                                                               0x8
+#define SDMA1_F32_CNTL__TH0_RESET__SHIFT                                                                      0x9
+#define SDMA1_F32_CNTL__TH0_ENABLE__SHIFT                                                                     0xa
+#define SDMA1_F32_CNTL__TH1_CHECKSUM_CLR__SHIFT                                                               0xc
+#define SDMA1_F32_CNTL__TH1_RESET__SHIFT                                                                      0xd
+#define SDMA1_F32_CNTL__TH1_ENABLE__SHIFT                                                                     0xe
+#define SDMA1_F32_CNTL__TH0_PRIORITY__SHIFT                                                                   0x10
+#define SDMA1_F32_CNTL__TH1_PRIORITY__SHIFT                                                                   0x18
+#define SDMA1_F32_CNTL__HALT_MASK                                                                             0x00000001L
+#define SDMA1_F32_CNTL__DBG_SELECT_BITS_MASK                                                                  0x000000FCL
+#define SDMA1_F32_CNTL__TH0_CHECKSUM_CLR_MASK                                                                 0x00000100L
+#define SDMA1_F32_CNTL__TH0_RESET_MASK                                                                        0x00000200L
+#define SDMA1_F32_CNTL__TH0_ENABLE_MASK                                                                       0x00000400L
+#define SDMA1_F32_CNTL__TH1_CHECKSUM_CLR_MASK                                                                 0x00001000L
+#define SDMA1_F32_CNTL__TH1_RESET_MASK                                                                        0x00002000L
+#define SDMA1_F32_CNTL__TH1_ENABLE_MASK                                                                       0x00004000L
+#define SDMA1_F32_CNTL__TH0_PRIORITY_MASK                                                                     0x00FF0000L
+#define SDMA1_F32_CNTL__TH1_PRIORITY_MASK                                                                     0xFF000000L
+
+
+// addressBlock: gc_sdma0_sdma0perfsdec
+//SDMA0_PERFCNT_PERFCOUNTER0_CFG
+#define SDMA0_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                       0x0
+#define SDMA0_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                   0x8
+#define SDMA0_PERFCNT_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                      0x18
+#define SDMA0_PERFCNT_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                         0x1c
+#define SDMA0_PERFCNT_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                          0x1d
+#define SDMA0_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                         0x000000FFL
+#define SDMA0_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                     0x0000FF00L
+#define SDMA0_PERFCNT_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                        0x0F000000L
+#define SDMA0_PERFCNT_PERFCOUNTER0_CFG__ENABLE_MASK                                                           0x10000000L
+#define SDMA0_PERFCNT_PERFCOUNTER0_CFG__CLEAR_MASK                                                            0x20000000L
+//SDMA0_PERFCNT_PERFCOUNTER1_CFG
+#define SDMA0_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                       0x0
+#define SDMA0_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                   0x8
+#define SDMA0_PERFCNT_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                      0x18
+#define SDMA0_PERFCNT_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                         0x1c
+#define SDMA0_PERFCNT_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                          0x1d
+#define SDMA0_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                         0x000000FFL
+#define SDMA0_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                     0x0000FF00L
+#define SDMA0_PERFCNT_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                        0x0F000000L
+#define SDMA0_PERFCNT_PERFCOUNTER1_CFG__ENABLE_MASK                                                           0x10000000L
+#define SDMA0_PERFCNT_PERFCOUNTER1_CFG__CLEAR_MASK                                                            0x20000000L
+//SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                       0x0
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                             0x8
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                              0x10
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                0x18
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                 0x19
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                      0x1a
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                         0x0000000FL
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                               0x0000FF00L
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                0x00FF0000L
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                  0x01000000L
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                   0x02000000L
+#define SDMA0_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                        0x04000000L
+//SDMA0_PERFCNT_MISC_CNTL
+#define SDMA0_PERFCNT_MISC_CNTL__CMD_OP__SHIFT                                                                0x0
+#define SDMA0_PERFCNT_MISC_CNTL__CMD_OP_MASK                                                                  0x0000FFFFL
+//SDMA0_PERFCOUNTER0_SELECT
+#define SDMA0_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define SDMA0_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define SDMA0_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define SDMA0_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define SDMA0_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define SDMA0_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define SDMA0_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define SDMA0_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define SDMA0_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define SDMA0_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//SDMA0_PERFCOUNTER0_SELECT1
+#define SDMA0_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define SDMA0_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define SDMA0_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define SDMA0_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define SDMA0_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define SDMA0_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define SDMA0_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define SDMA0_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//SDMA0_PERFCOUNTER1_SELECT
+#define SDMA0_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define SDMA0_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define SDMA0_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define SDMA0_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define SDMA0_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define SDMA0_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define SDMA0_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define SDMA0_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define SDMA0_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define SDMA0_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//SDMA0_PERFCOUNTER1_SELECT1
+#define SDMA0_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define SDMA0_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define SDMA0_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define SDMA0_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define SDMA0_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define SDMA0_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define SDMA0_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define SDMA0_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+
+
+// addressBlock: gc_sdma0_sdma1perfsdec
+//SDMA1_PERFCNT_PERFCOUNTER0_CFG
+#define SDMA1_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                       0x0
+#define SDMA1_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                   0x8
+#define SDMA1_PERFCNT_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                      0x18
+#define SDMA1_PERFCNT_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                         0x1c
+#define SDMA1_PERFCNT_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                          0x1d
+#define SDMA1_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                         0x000000FFL
+#define SDMA1_PERFCNT_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                     0x0000FF00L
+#define SDMA1_PERFCNT_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                        0x0F000000L
+#define SDMA1_PERFCNT_PERFCOUNTER0_CFG__ENABLE_MASK                                                           0x10000000L
+#define SDMA1_PERFCNT_PERFCOUNTER0_CFG__CLEAR_MASK                                                            0x20000000L
+//SDMA1_PERFCNT_PERFCOUNTER1_CFG
+#define SDMA1_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                       0x0
+#define SDMA1_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                   0x8
+#define SDMA1_PERFCNT_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                      0x18
+#define SDMA1_PERFCNT_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                         0x1c
+#define SDMA1_PERFCNT_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                          0x1d
+#define SDMA1_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                         0x000000FFL
+#define SDMA1_PERFCNT_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                     0x0000FF00L
+#define SDMA1_PERFCNT_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                        0x0F000000L
+#define SDMA1_PERFCNT_PERFCOUNTER1_CFG__ENABLE_MASK                                                           0x10000000L
+#define SDMA1_PERFCNT_PERFCOUNTER1_CFG__CLEAR_MASK                                                            0x20000000L
+//SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                       0x0
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                             0x8
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                              0x10
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                0x18
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                 0x19
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                      0x1a
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                         0x0000000FL
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                               0x0000FF00L
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                0x00FF0000L
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                  0x01000000L
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                   0x02000000L
+#define SDMA1_PERFCNT_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                        0x04000000L
+//SDMA1_PERFCNT_MISC_CNTL
+#define SDMA1_PERFCNT_MISC_CNTL__CMD_OP__SHIFT                                                                0x0
+#define SDMA1_PERFCNT_MISC_CNTL__CMD_OP_MASK                                                                  0x0000FFFFL
+//SDMA1_PERFCOUNTER0_SELECT
+#define SDMA1_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define SDMA1_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define SDMA1_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define SDMA1_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define SDMA1_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define SDMA1_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define SDMA1_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define SDMA1_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define SDMA1_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define SDMA1_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//SDMA1_PERFCOUNTER0_SELECT1
+#define SDMA1_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define SDMA1_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define SDMA1_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define SDMA1_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define SDMA1_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define SDMA1_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define SDMA1_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define SDMA1_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//SDMA1_PERFCOUNTER1_SELECT
+#define SDMA1_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define SDMA1_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define SDMA1_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define SDMA1_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define SDMA1_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define SDMA1_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define SDMA1_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define SDMA1_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define SDMA1_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define SDMA1_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//SDMA1_PERFCOUNTER1_SELECT1
+#define SDMA1_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define SDMA1_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define SDMA1_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define SDMA1_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define SDMA1_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define SDMA1_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define SDMA1_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define SDMA1_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+
+
+// addressBlock: gc_sdma0_sdma0perfddec
+//SDMA0_PERFCNT_PERFCOUNTER_LO
+#define SDMA0_PERFCNT_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                       0x0
+#define SDMA0_PERFCNT_PERFCOUNTER_LO__COUNTER_LO_MASK                                                         0xFFFFFFFFL
+//SDMA0_PERFCNT_PERFCOUNTER_HI
+#define SDMA0_PERFCNT_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                       0x0
+#define SDMA0_PERFCNT_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                    0x10
+#define SDMA0_PERFCNT_PERFCOUNTER_HI__COUNTER_HI_MASK                                                         0x0000FFFFL
+#define SDMA0_PERFCNT_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                      0xFFFF0000L
+//SDMA0_PERFCOUNTER0_LO
+#define SDMA0_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define SDMA0_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//SDMA0_PERFCOUNTER0_HI
+#define SDMA0_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define SDMA0_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//SDMA0_PERFCOUNTER1_LO
+#define SDMA0_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define SDMA0_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//SDMA0_PERFCOUNTER1_HI
+#define SDMA0_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define SDMA0_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: gc_sdma0_sdma1perfddec
+//SDMA1_PERFCNT_PERFCOUNTER_LO
+#define SDMA1_PERFCNT_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                       0x0
+#define SDMA1_PERFCNT_PERFCOUNTER_LO__COUNTER_LO_MASK                                                         0xFFFFFFFFL
+//SDMA1_PERFCNT_PERFCOUNTER_HI
+#define SDMA1_PERFCNT_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                       0x0
+#define SDMA1_PERFCNT_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                    0x10
+#define SDMA1_PERFCNT_PERFCOUNTER_HI__COUNTER_HI_MASK                                                         0x0000FFFFL
+#define SDMA1_PERFCNT_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                      0xFFFF0000L
+//SDMA1_PERFCOUNTER0_LO
+#define SDMA1_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define SDMA1_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//SDMA1_PERFCOUNTER0_HI
+#define SDMA1_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define SDMA1_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//SDMA1_PERFCOUNTER1_LO
+#define SDMA1_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define SDMA1_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//SDMA1_PERFCOUNTER1_HI
+#define SDMA1_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define SDMA1_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: gc_grbmdec
+//GRBM_CNTL
+#define GRBM_CNTL__READ_TIMEOUT__SHIFT                                                                        0x0
+#define GRBM_CNTL__REPORT_LAST_RDERR__SHIFT                                                                   0x1f
+#define GRBM_CNTL__READ_TIMEOUT_MASK                                                                          0x000000FFL
+#define GRBM_CNTL__REPORT_LAST_RDERR_MASK                                                                     0x80000000L
+//GRBM_SKEW_CNTL
+#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD__SHIFT                                                             0x0
+#define GRBM_SKEW_CNTL__SKEW_COUNT__SHIFT                                                                     0x6
+#define GRBM_SKEW_CNTL__SKEW_TOP_THRESHOLD_MASK                                                               0x0000003FL
+#define GRBM_SKEW_CNTL__SKEW_COUNT_MASK                                                                       0x00000FC0L
+//GRBM_STATUS2
+#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL__SHIFT                                                           0x0
+#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING__SHIFT                                                           0x4
+#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING__SHIFT                                                           0x5
+#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING__SHIFT                                                              0x6
+#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING__SHIFT                                                              0x7
+#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING__SHIFT                                                              0x8
+#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING__SHIFT                                                              0x9
+#define GRBM_STATUS2__RLC_RQ_PENDING__SHIFT                                                                   0xe
+#define GRBM_STATUS2__UTCL2_BUSY__SHIFT                                                                       0xf
+#define GRBM_STATUS2__EA_BUSY__SHIFT                                                                          0x10
+#define GRBM_STATUS2__RMI_BUSY__SHIFT                                                                         0x11
+#define GRBM_STATUS2__UTCL2_RQ_PENDING__SHIFT                                                                 0x12
+#define GRBM_STATUS2__SDMA_SCH_RQ_PENDING__SHIFT                                                              0x13
+#define GRBM_STATUS2__EA_LINK_BUSY__SHIFT                                                                     0x14
+#define GRBM_STATUS2__SDMA_BUSY__SHIFT                                                                        0x15
+#define GRBM_STATUS2__SDMA0_RQ_PENDING__SHIFT                                                                 0x16
+#define GRBM_STATUS2__SDMA1_RQ_PENDING__SHIFT                                                                 0x17
+#define GRBM_STATUS2__RLC_BUSY__SHIFT                                                                         0x1a
+#define GRBM_STATUS2__TCP_BUSY__SHIFT                                                                         0x1b
+#define GRBM_STATUS2__CPF_BUSY__SHIFT                                                                         0x1c
+#define GRBM_STATUS2__CPC_BUSY__SHIFT                                                                         0x1d
+#define GRBM_STATUS2__CPG_BUSY__SHIFT                                                                         0x1e
+#define GRBM_STATUS2__CPAXI_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS2__ME0PIPE1_CMDFIFO_AVAIL_MASK                                                             0x0000000FL
+#define GRBM_STATUS2__ME0PIPE1_CF_RQ_PENDING_MASK                                                             0x00000010L
+#define GRBM_STATUS2__ME0PIPE1_PF_RQ_PENDING_MASK                                                             0x00000020L
+#define GRBM_STATUS2__ME1PIPE0_RQ_PENDING_MASK                                                                0x00000040L
+#define GRBM_STATUS2__ME1PIPE1_RQ_PENDING_MASK                                                                0x00000080L
+#define GRBM_STATUS2__ME1PIPE2_RQ_PENDING_MASK                                                                0x00000100L
+#define GRBM_STATUS2__ME1PIPE3_RQ_PENDING_MASK                                                                0x00000200L
+#define GRBM_STATUS2__RLC_RQ_PENDING_MASK                                                                     0x00004000L
+#define GRBM_STATUS2__UTCL2_BUSY_MASK                                                                         0x00008000L
+#define GRBM_STATUS2__EA_BUSY_MASK                                                                            0x00010000L
+#define GRBM_STATUS2__RMI_BUSY_MASK                                                                           0x00020000L
+#define GRBM_STATUS2__UTCL2_RQ_PENDING_MASK                                                                   0x00040000L
+#define GRBM_STATUS2__SDMA_SCH_RQ_PENDING_MASK                                                                0x00080000L
+#define GRBM_STATUS2__EA_LINK_BUSY_MASK                                                                       0x00100000L
+#define GRBM_STATUS2__SDMA_BUSY_MASK                                                                          0x00200000L
+#define GRBM_STATUS2__SDMA0_RQ_PENDING_MASK                                                                   0x00400000L
+#define GRBM_STATUS2__SDMA1_RQ_PENDING_MASK                                                                   0x00800000L
+#define GRBM_STATUS2__RLC_BUSY_MASK                                                                           0x04000000L
+#define GRBM_STATUS2__TCP_BUSY_MASK                                                                           0x08000000L
+#define GRBM_STATUS2__CPF_BUSY_MASK                                                                           0x10000000L
+#define GRBM_STATUS2__CPC_BUSY_MASK                                                                           0x20000000L
+#define GRBM_STATUS2__CPG_BUSY_MASK                                                                           0x40000000L
+#define GRBM_STATUS2__CPAXI_BUSY_MASK                                                                         0x80000000L
+//GRBM_PWR_CNTL
+#define GRBM_PWR_CNTL__ALL_REQ_TYPE__SHIFT                                                                    0x0
+#define GRBM_PWR_CNTL__GFX_REQ_TYPE__SHIFT                                                                    0x2
+#define GRBM_PWR_CNTL__ALL_RSP_TYPE__SHIFT                                                                    0x4
+#define GRBM_PWR_CNTL__GFX_RSP_TYPE__SHIFT                                                                    0x6
+#define GRBM_PWR_CNTL__GFX_REQ_EN__SHIFT                                                                      0xe
+#define GRBM_PWR_CNTL__ALL_REQ_EN__SHIFT                                                                      0xf
+#define GRBM_PWR_CNTL__ALL_REQ_TYPE_MASK                                                                      0x00000003L
+#define GRBM_PWR_CNTL__GFX_REQ_TYPE_MASK                                                                      0x0000000CL
+#define GRBM_PWR_CNTL__ALL_RSP_TYPE_MASK                                                                      0x00000030L
+#define GRBM_PWR_CNTL__GFX_RSP_TYPE_MASK                                                                      0x000000C0L
+#define GRBM_PWR_CNTL__GFX_REQ_EN_MASK                                                                        0x00004000L
+#define GRBM_PWR_CNTL__ALL_REQ_EN_MASK                                                                        0x00008000L
+//GRBM_STATUS
+#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL__SHIFT                                                            0x0
+#define GRBM_STATUS__RSMU_RQ_PENDING__SHIFT                                                                   0x5
+#define GRBM_STATUS__SDMA_RQ_PENDING__SHIFT                                                                   0x6
+#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING__SHIFT                                                            0x7
+#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING__SHIFT                                                            0x8
+#define GRBM_STATUS__GDS_DMA_RQ_PENDING__SHIFT                                                                0x9
+#define GRBM_STATUS__DB_CLEAN__SHIFT                                                                          0xc
+#define GRBM_STATUS__CB_CLEAN__SHIFT                                                                          0xd
+#define GRBM_STATUS__TA_BUSY__SHIFT                                                                           0xe
+#define GRBM_STATUS__GDS_BUSY__SHIFT                                                                          0xf
+#define GRBM_STATUS__GE_BUSY_NO_DMA__SHIFT                                                                    0x10
+#define GRBM_STATUS__SX_BUSY__SHIFT                                                                           0x14
+#define GRBM_STATUS__GE_BUSY__SHIFT                                                                           0x15
+#define GRBM_STATUS__SPI_BUSY__SHIFT                                                                          0x16
+#define GRBM_STATUS__BCI_BUSY__SHIFT                                                                          0x17
+#define GRBM_STATUS__SC_BUSY__SHIFT                                                                           0x18
+#define GRBM_STATUS__PA_BUSY__SHIFT                                                                           0x19
+#define GRBM_STATUS__DB_BUSY__SHIFT                                                                           0x1a
+#define GRBM_STATUS__ANY_ACTIVE__SHIFT                                                                        0x1b
+#define GRBM_STATUS__CP_COHERENCY_BUSY__SHIFT                                                                 0x1c
+#define GRBM_STATUS__CP_BUSY__SHIFT                                                                           0x1d
+#define GRBM_STATUS__CB_BUSY__SHIFT                                                                           0x1e
+#define GRBM_STATUS__GUI_ACTIVE__SHIFT                                                                        0x1f
+#define GRBM_STATUS__ME0PIPE0_CMDFIFO_AVAIL_MASK                                                              0x0000000FL
+#define GRBM_STATUS__RSMU_RQ_PENDING_MASK                                                                     0x00000020L
+#define GRBM_STATUS__SDMA_RQ_PENDING_MASK                                                                     0x00000040L
+#define GRBM_STATUS__ME0PIPE0_CF_RQ_PENDING_MASK                                                              0x00000080L
+#define GRBM_STATUS__ME0PIPE0_PF_RQ_PENDING_MASK                                                              0x00000100L
+#define GRBM_STATUS__GDS_DMA_RQ_PENDING_MASK                                                                  0x00000200L
+#define GRBM_STATUS__DB_CLEAN_MASK                                                                            0x00001000L
+#define GRBM_STATUS__CB_CLEAN_MASK                                                                            0x00002000L
+#define GRBM_STATUS__TA_BUSY_MASK                                                                             0x00004000L
+#define GRBM_STATUS__GDS_BUSY_MASK                                                                            0x00008000L
+#define GRBM_STATUS__GE_BUSY_NO_DMA_MASK                                                                      0x00010000L
+#define GRBM_STATUS__SX_BUSY_MASK                                                                             0x00100000L
+#define GRBM_STATUS__GE_BUSY_MASK                                                                             0x00200000L
+#define GRBM_STATUS__SPI_BUSY_MASK                                                                            0x00400000L
+#define GRBM_STATUS__BCI_BUSY_MASK                                                                            0x00800000L
+#define GRBM_STATUS__SC_BUSY_MASK                                                                             0x01000000L
+#define GRBM_STATUS__PA_BUSY_MASK                                                                             0x02000000L
+#define GRBM_STATUS__DB_BUSY_MASK                                                                             0x04000000L
+#define GRBM_STATUS__ANY_ACTIVE_MASK                                                                          0x08000000L
+#define GRBM_STATUS__CP_COHERENCY_BUSY_MASK                                                                   0x10000000L
+#define GRBM_STATUS__CP_BUSY_MASK                                                                             0x20000000L
+#define GRBM_STATUS__CB_BUSY_MASK                                                                             0x40000000L
+#define GRBM_STATUS__GUI_ACTIVE_MASK                                                                          0x80000000L
+//GRBM_STATUS_SE0
+#define GRBM_STATUS_SE0__DB_CLEAN__SHIFT                                                                      0x1
+#define GRBM_STATUS_SE0__CB_CLEAN__SHIFT                                                                      0x2
+#define GRBM_STATUS_SE0__UTCL1_BUSY__SHIFT                                                                    0x3
+#define GRBM_STATUS_SE0__TCP_BUSY__SHIFT                                                                      0x4
+#define GRBM_STATUS_SE0__GL1CC_BUSY__SHIFT                                                                    0x5
+#define GRBM_STATUS_SE0__GL1H_BUSY__SHIFT                                                                     0x6
+#define GRBM_STATUS_SE0__PC_BUSY__SHIFT                                                                       0x7
+#define GRBM_STATUS_SE0__SEDC_BUSY__SHIFT                                                                     0x8
+#define GRBM_STATUS_SE0__RMI_BUSY__SHIFT                                                                      0x15
+#define GRBM_STATUS_SE0__BCI_BUSY__SHIFT                                                                      0x16
+#define GRBM_STATUS_SE0__PA_BUSY__SHIFT                                                                       0x18
+#define GRBM_STATUS_SE0__TA_BUSY__SHIFT                                                                       0x19
+#define GRBM_STATUS_SE0__SX_BUSY__SHIFT                                                                       0x1a
+#define GRBM_STATUS_SE0__SPI_BUSY__SHIFT                                                                      0x1b
+#define GRBM_STATUS_SE0__SC_BUSY__SHIFT                                                                       0x1d
+#define GRBM_STATUS_SE0__DB_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS_SE0__CB_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS_SE0__DB_CLEAN_MASK                                                                        0x00000002L
+#define GRBM_STATUS_SE0__CB_CLEAN_MASK                                                                        0x00000004L
+#define GRBM_STATUS_SE0__UTCL1_BUSY_MASK                                                                      0x00000008L
+#define GRBM_STATUS_SE0__TCP_BUSY_MASK                                                                        0x00000010L
+#define GRBM_STATUS_SE0__GL1CC_BUSY_MASK                                                                      0x00000020L
+#define GRBM_STATUS_SE0__GL1H_BUSY_MASK                                                                       0x00000040L
+#define GRBM_STATUS_SE0__PC_BUSY_MASK                                                                         0x00000080L
+#define GRBM_STATUS_SE0__SEDC_BUSY_MASK                                                                       0x00000100L
+#define GRBM_STATUS_SE0__RMI_BUSY_MASK                                                                        0x00200000L
+#define GRBM_STATUS_SE0__BCI_BUSY_MASK                                                                        0x00400000L
+#define GRBM_STATUS_SE0__PA_BUSY_MASK                                                                         0x01000000L
+#define GRBM_STATUS_SE0__TA_BUSY_MASK                                                                         0x02000000L
+#define GRBM_STATUS_SE0__SX_BUSY_MASK                                                                         0x04000000L
+#define GRBM_STATUS_SE0__SPI_BUSY_MASK                                                                        0x08000000L
+#define GRBM_STATUS_SE0__SC_BUSY_MASK                                                                         0x20000000L
+#define GRBM_STATUS_SE0__DB_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS_SE0__CB_BUSY_MASK                                                                         0x80000000L
+//GRBM_STATUS_SE1
+#define GRBM_STATUS_SE1__DB_CLEAN__SHIFT                                                                      0x1
+#define GRBM_STATUS_SE1__CB_CLEAN__SHIFT                                                                      0x2
+#define GRBM_STATUS_SE1__UTCL1_BUSY__SHIFT                                                                    0x3
+#define GRBM_STATUS_SE1__TCP_BUSY__SHIFT                                                                      0x4
+#define GRBM_STATUS_SE1__GL1CC_BUSY__SHIFT                                                                    0x5
+#define GRBM_STATUS_SE1__GL1H_BUSY__SHIFT                                                                     0x6
+#define GRBM_STATUS_SE1__PC_BUSY__SHIFT                                                                       0x7
+#define GRBM_STATUS_SE1__SEDC_BUSY__SHIFT                                                                     0x8
+#define GRBM_STATUS_SE1__RMI_BUSY__SHIFT                                                                      0x15
+#define GRBM_STATUS_SE1__BCI_BUSY__SHIFT                                                                      0x16
+#define GRBM_STATUS_SE1__PA_BUSY__SHIFT                                                                       0x18
+#define GRBM_STATUS_SE1__TA_BUSY__SHIFT                                                                       0x19
+#define GRBM_STATUS_SE1__SX_BUSY__SHIFT                                                                       0x1a
+#define GRBM_STATUS_SE1__SPI_BUSY__SHIFT                                                                      0x1b
+#define GRBM_STATUS_SE1__SC_BUSY__SHIFT                                                                       0x1d
+#define GRBM_STATUS_SE1__DB_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS_SE1__CB_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS_SE1__DB_CLEAN_MASK                                                                        0x00000002L
+#define GRBM_STATUS_SE1__CB_CLEAN_MASK                                                                        0x00000004L
+#define GRBM_STATUS_SE1__UTCL1_BUSY_MASK                                                                      0x00000008L
+#define GRBM_STATUS_SE1__TCP_BUSY_MASK                                                                        0x00000010L
+#define GRBM_STATUS_SE1__GL1CC_BUSY_MASK                                                                      0x00000020L
+#define GRBM_STATUS_SE1__GL1H_BUSY_MASK                                                                       0x00000040L
+#define GRBM_STATUS_SE1__PC_BUSY_MASK                                                                         0x00000080L
+#define GRBM_STATUS_SE1__SEDC_BUSY_MASK                                                                       0x00000100L
+#define GRBM_STATUS_SE1__RMI_BUSY_MASK                                                                        0x00200000L
+#define GRBM_STATUS_SE1__BCI_BUSY_MASK                                                                        0x00400000L
+#define GRBM_STATUS_SE1__PA_BUSY_MASK                                                                         0x01000000L
+#define GRBM_STATUS_SE1__TA_BUSY_MASK                                                                         0x02000000L
+#define GRBM_STATUS_SE1__SX_BUSY_MASK                                                                         0x04000000L
+#define GRBM_STATUS_SE1__SPI_BUSY_MASK                                                                        0x08000000L
+#define GRBM_STATUS_SE1__SC_BUSY_MASK                                                                         0x20000000L
+#define GRBM_STATUS_SE1__DB_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS_SE1__CB_BUSY_MASK                                                                         0x80000000L
+//GRBM_STATUS3
+#define GRBM_STATUS3__GRBM_RLC_INTR_CREDIT_PENDING__SHIFT                                                     0x5
+#define GRBM_STATUS3__GRBM_CPF_INTR_CREDIT_PENDING__SHIFT                                                     0x7
+#define GRBM_STATUS3__MESPIPE0_RQ_PENDING__SHIFT                                                              0x8
+#define GRBM_STATUS3__MESPIPE1_RQ_PENDING__SHIFT                                                              0x9
+#define GRBM_STATUS3__PH_BUSY__SHIFT                                                                          0xd
+#define GRBM_STATUS3__CH_BUSY__SHIFT                                                                          0xe
+#define GRBM_STATUS3__GL2CC_BUSY__SHIFT                                                                       0xf
+#define GRBM_STATUS3__GL1CC_BUSY__SHIFT                                                                       0x10
+#define GRBM_STATUS3__SEDC_BUSY__SHIFT                                                                        0x19
+#define GRBM_STATUS3__PC_BUSY__SHIFT                                                                          0x1a
+#define GRBM_STATUS3__GL1H_BUSY__SHIFT                                                                        0x1b
+#define GRBM_STATUS3__GUS_LINK_BUSY__SHIFT                                                                    0x1c
+#define GRBM_STATUS3__GUS_BUSY__SHIFT                                                                         0x1d
+#define GRBM_STATUS3__UTCL1_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS3__PMM_BUSY__SHIFT                                                                         0x1f
+#define GRBM_STATUS3__GRBM_RLC_INTR_CREDIT_PENDING_MASK                                                       0x00000020L
+#define GRBM_STATUS3__GRBM_CPF_INTR_CREDIT_PENDING_MASK                                                       0x00000080L
+#define GRBM_STATUS3__MESPIPE0_RQ_PENDING_MASK                                                                0x00000100L
+#define GRBM_STATUS3__MESPIPE1_RQ_PENDING_MASK                                                                0x00000200L
+#define GRBM_STATUS3__PH_BUSY_MASK                                                                            0x00002000L
+#define GRBM_STATUS3__CH_BUSY_MASK                                                                            0x00004000L
+#define GRBM_STATUS3__GL2CC_BUSY_MASK                                                                         0x00008000L
+#define GRBM_STATUS3__GL1CC_BUSY_MASK                                                                         0x00010000L
+#define GRBM_STATUS3__SEDC_BUSY_MASK                                                                          0x02000000L
+#define GRBM_STATUS3__PC_BUSY_MASK                                                                            0x04000000L
+#define GRBM_STATUS3__GL1H_BUSY_MASK                                                                          0x08000000L
+#define GRBM_STATUS3__GUS_LINK_BUSY_MASK                                                                      0x10000000L
+#define GRBM_STATUS3__GUS_BUSY_MASK                                                                           0x20000000L
+#define GRBM_STATUS3__UTCL1_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS3__PMM_BUSY_MASK                                                                           0x80000000L
+//GRBM_SOFT_RESET
+#define GRBM_SOFT_RESET__SOFT_RESET_CP__SHIFT                                                                 0x0
+#define GRBM_SOFT_RESET__SOFT_RESET_RLC__SHIFT                                                                0x2
+#define GRBM_SOFT_RESET__SOFT_RESET_UTCL2__SHIFT                                                              0xf
+#define GRBM_SOFT_RESET__SOFT_RESET_GFX__SHIFT                                                                0x10
+#define GRBM_SOFT_RESET__SOFT_RESET_CPF__SHIFT                                                                0x11
+#define GRBM_SOFT_RESET__SOFT_RESET_CPC__SHIFT                                                                0x12
+#define GRBM_SOFT_RESET__SOFT_RESET_CPG__SHIFT                                                                0x13
+#define GRBM_SOFT_RESET__SOFT_RESET_CAC__SHIFT                                                                0x14
+#define GRBM_SOFT_RESET__SOFT_RESET_CPAXI__SHIFT                                                              0x15
+#define GRBM_SOFT_RESET__SOFT_RESET_EA__SHIFT                                                                 0x16
+#define GRBM_SOFT_RESET__SOFT_RESET_SDMA0__SHIFT                                                              0x17
+#define GRBM_SOFT_RESET__SOFT_RESET_SDMA1__SHIFT                                                              0x18
+#define GRBM_SOFT_RESET__SOFT_RESET_CP_MASK                                                                   0x00000001L
+#define GRBM_SOFT_RESET__SOFT_RESET_RLC_MASK                                                                  0x00000004L
+#define GRBM_SOFT_RESET__SOFT_RESET_UTCL2_MASK                                                                0x00008000L
+#define GRBM_SOFT_RESET__SOFT_RESET_GFX_MASK                                                                  0x00010000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPF_MASK                                                                  0x00020000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPC_MASK                                                                  0x00040000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPG_MASK                                                                  0x00080000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CAC_MASK                                                                  0x00100000L
+#define GRBM_SOFT_RESET__SOFT_RESET_CPAXI_MASK                                                                0x00200000L
+#define GRBM_SOFT_RESET__SOFT_RESET_EA_MASK                                                                   0x00400000L
+#define GRBM_SOFT_RESET__SOFT_RESET_SDMA0_MASK                                                                0x00800000L
+#define GRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK                                                                0x01000000L
+//GRBM_GFX_CLKEN_CNTL
+#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT__SHIFT                                                          0x0
+#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT__SHIFT                                                            0x8
+#define GRBM_GFX_CLKEN_CNTL__PREFIX_DELAY_CNT_MASK                                                            0x0000000FL
+#define GRBM_GFX_CLKEN_CNTL__POST_DELAY_CNT_MASK                                                              0x00001F00L
+//GRBM_WAIT_IDLE_CLOCKS
+#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS__SHIFT                                                        0x0
+#define GRBM_WAIT_IDLE_CLOCKS__WAIT_IDLE_CLOCKS_MASK                                                          0x000000FFL
+//GRBM_STATUS_SE2
+#define GRBM_STATUS_SE2__DB_CLEAN__SHIFT                                                                      0x1
+#define GRBM_STATUS_SE2__CB_CLEAN__SHIFT                                                                      0x2
+#define GRBM_STATUS_SE2__UTCL1_BUSY__SHIFT                                                                    0x3
+#define GRBM_STATUS_SE2__TCP_BUSY__SHIFT                                                                      0x4
+#define GRBM_STATUS_SE2__GL1CC_BUSY__SHIFT                                                                    0x5
+#define GRBM_STATUS_SE2__GL1H_BUSY__SHIFT                                                                     0x6
+#define GRBM_STATUS_SE2__PC_BUSY__SHIFT                                                                       0x7
+#define GRBM_STATUS_SE2__SEDC_BUSY__SHIFT                                                                     0x8
+#define GRBM_STATUS_SE2__RMI_BUSY__SHIFT                                                                      0x15
+#define GRBM_STATUS_SE2__BCI_BUSY__SHIFT                                                                      0x16
+#define GRBM_STATUS_SE2__PA_BUSY__SHIFT                                                                       0x18
+#define GRBM_STATUS_SE2__TA_BUSY__SHIFT                                                                       0x19
+#define GRBM_STATUS_SE2__SX_BUSY__SHIFT                                                                       0x1a
+#define GRBM_STATUS_SE2__SPI_BUSY__SHIFT                                                                      0x1b
+#define GRBM_STATUS_SE2__SC_BUSY__SHIFT                                                                       0x1d
+#define GRBM_STATUS_SE2__DB_BUSY__SHIFT                                                                       0x1e
+#define GRBM_STATUS_SE2__CB_BUSY__SHIFT                                                                       0x1f
+#define GRBM_STATUS_SE2__DB_CLEAN_MASK                                                                        0x00000002L
+#define GRBM_STATUS_SE2__CB_CLEAN_MASK                                                                        0x00000004L
+#define GRBM_STATUS_SE2__UTCL1_BUSY_MASK                                                                      0x00000008L
+#define GRBM_STATUS_SE2__TCP_BUSY_MASK                                                                        0x00000010L
+#define GRBM_STATUS_SE2__GL1CC_BUSY_MASK                                                                      0x00000020L
+#define GRBM_STATUS_SE2__GL1H_BUSY_MASK                                                                       0x00000040L
+#define GRBM_STATUS_SE2__PC_BUSY_MASK                                                                         0x00000080L
+#define GRBM_STATUS_SE2__SEDC_BUSY_MASK                                                                       0x00000100L
+#define GRBM_STATUS_SE2__RMI_BUSY_MASK                                                                        0x00200000L
+#define GRBM_STATUS_SE2__BCI_BUSY_MASK                                                                        0x00400000L
+#define GRBM_STATUS_SE2__PA_BUSY_MASK                                                                         0x01000000L
+#define GRBM_STATUS_SE2__TA_BUSY_MASK                                                                         0x02000000L
+#define GRBM_STATUS_SE2__SX_BUSY_MASK                                                                         0x04000000L
+#define GRBM_STATUS_SE2__SPI_BUSY_MASK                                                                        0x08000000L
+#define GRBM_STATUS_SE2__SC_BUSY_MASK                                                                         0x20000000L
+#define GRBM_STATUS_SE2__DB_BUSY_MASK                                                                         0x40000000L
+#define GRBM_STATUS_SE2__CB_BUSY_MASK                                                                         0x80000000L
+//GRBM_READ_ERROR
+#define GRBM_READ_ERROR__READ_ADDRESS__SHIFT                                                                  0x2
+#define GRBM_READ_ERROR__READ_PIPEID__SHIFT                                                                   0x14
+#define GRBM_READ_ERROR__READ_MEID__SHIFT                                                                     0x16
+#define GRBM_READ_ERROR__READ_ERROR__SHIFT                                                                    0x1f
+#define GRBM_READ_ERROR__READ_ADDRESS_MASK                                                                    0x000FFFFCL
+#define GRBM_READ_ERROR__READ_PIPEID_MASK                                                                     0x00300000L
+#define GRBM_READ_ERROR__READ_MEID_MASK                                                                       0x00C00000L
+#define GRBM_READ_ERROR__READ_ERROR_MASK                                                                      0x80000000L
+//GRBM_READ_ERROR2
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE0__SHIFT                                                      0x9
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE1__SHIFT                                                      0xa
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE2__SHIFT                                                      0xb
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE3__SHIFT                                                      0xc
+#define GRBM_READ_ERROR2__READ_REQUESTER_SDMA0__SHIFT                                                         0xd
+#define GRBM_READ_ERROR2__READ_REQUESTER_SDMA1__SHIFT                                                         0xe
+#define GRBM_READ_ERROR2__READ_REQUESTER_RSMU__SHIFT                                                          0x11
+#define GRBM_READ_ERROR2__READ_REQUESTER_RLC__SHIFT                                                           0x12
+#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA__SHIFT                                                       0x13
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF__SHIFT                                                   0x14
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF__SHIFT                                                   0x15
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF__SHIFT                                                   0x16
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF__SHIFT                                                   0x17
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0__SHIFT                                                      0x18
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1__SHIFT                                                      0x19
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2__SHIFT                                                      0x1a
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3__SHIFT                                                      0x1b
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0__SHIFT                                                      0x1c
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1__SHIFT                                                      0x1d
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2__SHIFT                                                      0x1e
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3__SHIFT                                                      0x1f
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE0_MASK                                                        0x00000200L
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE1_MASK                                                        0x00000400L
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE2_MASK                                                        0x00000800L
+#define GRBM_READ_ERROR2__READ_REQUESTER_MESPIPE3_MASK                                                        0x00001000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_SDMA0_MASK                                                           0x00002000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_SDMA1_MASK                                                           0x00004000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_RSMU_MASK                                                            0x00020000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_RLC_MASK                                                             0x00040000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_GDS_DMA_MASK                                                         0x00080000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_CF_MASK                                                     0x00100000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE0_PF_MASK                                                     0x00200000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_CF_MASK                                                     0x00400000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME0PIPE1_PF_MASK                                                     0x00800000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE0_MASK                                                        0x01000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE1_MASK                                                        0x02000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE2_MASK                                                        0x04000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME1PIPE3_MASK                                                        0x08000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE0_MASK                                                        0x10000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE1_MASK                                                        0x20000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE2_MASK                                                        0x40000000L
+#define GRBM_READ_ERROR2__READ_REQUESTER_ME2PIPE3_MASK                                                        0x80000000L
+//GRBM_INT_CNTL
+#define GRBM_INT_CNTL__RDERR_INT_ENABLE__SHIFT                                                                0x0
+#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE__SHIFT                                                             0x13
+#define GRBM_INT_CNTL__RDERR_INT_ENABLE_MASK                                                                  0x00000001L
+#define GRBM_INT_CNTL__GUI_IDLE_INT_ENABLE_MASK                                                               0x00080000L
+//GRBM_TRAP_OP
+#define GRBM_TRAP_OP__RW__SHIFT                                                                               0x0
+#define GRBM_TRAP_OP__RW_MASK                                                                                 0x00000001L
+//GRBM_TRAP_ADDR
+#define GRBM_TRAP_ADDR__DATA__SHIFT                                                                           0x0
+#define GRBM_TRAP_ADDR__DATA_MASK                                                                             0x0003FFFFL
+//GRBM_TRAP_ADDR_MSK
+#define GRBM_TRAP_ADDR_MSK__DATA__SHIFT                                                                       0x0
+#define GRBM_TRAP_ADDR_MSK__DATA_MASK                                                                         0x0003FFFFL
+//GRBM_TRAP_WD
+#define GRBM_TRAP_WD__DATA__SHIFT                                                                             0x0
+#define GRBM_TRAP_WD__DATA_MASK                                                                               0xFFFFFFFFL
+//GRBM_TRAP_WD_MSK
+#define GRBM_TRAP_WD_MSK__DATA__SHIFT                                                                         0x0
+#define GRBM_TRAP_WD_MSK__DATA_MASK                                                                           0xFFFFFFFFL
+//GRBM_DSM_BYPASS
+#define GRBM_DSM_BYPASS__BYPASS_BITS__SHIFT                                                                   0x0
+#define GRBM_DSM_BYPASS__BYPASS_EN__SHIFT                                                                     0x2
+#define GRBM_DSM_BYPASS__BYPASS_BITS_MASK                                                                     0x00000003L
+#define GRBM_DSM_BYPASS__BYPASS_EN_MASK                                                                       0x00000004L
+//GRBM_WRITE_ERROR
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RLC__SHIFT                                                          0x0
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RSMU__SHIFT                                                         0x1
+#define GRBM_WRITE_ERROR__WRITE_SSRCID__SHIFT                                                                 0x2
+#define GRBM_WRITE_ERROR__WRITE_VFID__SHIFT                                                                   0x8
+#define GRBM_WRITE_ERROR__WRITE_VF__SHIFT                                                                     0xc
+#define GRBM_WRITE_ERROR__WRITE_VMID__SHIFT                                                                   0xd
+#define GRBM_WRITE_ERROR__TMZ__SHIFT                                                                          0x11
+#define GRBM_WRITE_ERROR__CP_SECURE_WR_ILLEGAL__SHIFT                                                         0x12
+#define GRBM_WRITE_ERROR__WRITE_PIPEID__SHIFT                                                                 0x14
+#define GRBM_WRITE_ERROR__WRITE_MEID__SHIFT                                                                   0x16
+#define GRBM_WRITE_ERROR__WRITE_ERROR__SHIFT                                                                  0x1f
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RLC_MASK                                                            0x00000001L
+#define GRBM_WRITE_ERROR__WRITE_REQUESTER_RSMU_MASK                                                           0x00000002L
+#define GRBM_WRITE_ERROR__WRITE_SSRCID_MASK                                                                   0x0000003CL
+#define GRBM_WRITE_ERROR__WRITE_VFID_MASK                                                                     0x00000F00L
+#define GRBM_WRITE_ERROR__WRITE_VF_MASK                                                                       0x00001000L
+#define GRBM_WRITE_ERROR__WRITE_VMID_MASK                                                                     0x0001E000L
+#define GRBM_WRITE_ERROR__TMZ_MASK                                                                            0x00020000L
+#define GRBM_WRITE_ERROR__CP_SECURE_WR_ILLEGAL_MASK                                                           0x00040000L
+#define GRBM_WRITE_ERROR__WRITE_PIPEID_MASK                                                                   0x00300000L
+#define GRBM_WRITE_ERROR__WRITE_MEID_MASK                                                                     0x00C00000L
+#define GRBM_WRITE_ERROR__WRITE_ERROR_MASK                                                                    0x80000000L
+//GRBM_CHIP_REVISION
+#define GRBM_CHIP_REVISION__CHIP_REVISION__SHIFT                                                              0x0
+#define GRBM_CHIP_REVISION__CHIP_REVISION_MASK                                                                0x000000FFL
+//GRBM_RSMU_CFG
+#define GRBM_RSMU_CFG__APERTURE_ID__SHIFT                                                                     0x0
+#define GRBM_RSMU_CFG__QOS__SHIFT                                                                             0xc
+#define GRBM_RSMU_CFG__POSTED_WR__SHIFT                                                                       0x10
+#define GRBM_RSMU_CFG__DEBUG_MASK__SHIFT                                                                      0x11
+#define GRBM_RSMU_CFG__APERTURE_ID_MASK                                                                       0x00000FFFL
+#define GRBM_RSMU_CFG__QOS_MASK                                                                               0x0000F000L
+#define GRBM_RSMU_CFG__POSTED_WR_MASK                                                                         0x00010000L
+#define GRBM_RSMU_CFG__DEBUG_MASK_MASK                                                                        0x00020000L
+//GRBM_IH_CREDIT
+#define GRBM_IH_CREDIT__CREDIT_VALUE__SHIFT                                                                   0x0
+#define GRBM_IH_CREDIT__IH_CLIENT_ID__SHIFT                                                                   0x10
+#define GRBM_IH_CREDIT__CREDIT_VALUE_MASK                                                                     0x00000003L
+#define GRBM_IH_CREDIT__IH_CLIENT_ID_MASK                                                                     0x00FF0000L
+//GRBM_PWR_CNTL2
+#define GRBM_PWR_CNTL2__PWR_REQUEST_HALT__SHIFT                                                               0x10
+#define GRBM_PWR_CNTL2__PWR_GFX3D_REQUEST_HALT__SHIFT                                                         0x14
+#define GRBM_PWR_CNTL2__PWR_REQUEST_HALT_MASK                                                                 0x00010000L
+#define GRBM_PWR_CNTL2__PWR_GFX3D_REQUEST_HALT_MASK                                                           0x00100000L
+//GRBM_UTCL2_INVAL_RANGE_START
+#define GRBM_UTCL2_INVAL_RANGE_START__DATA__SHIFT                                                             0x0
+#define GRBM_UTCL2_INVAL_RANGE_START__DATA_MASK                                                               0x0003FFFFL
+//GRBM_UTCL2_INVAL_RANGE_END
+#define GRBM_UTCL2_INVAL_RANGE_END__DATA__SHIFT                                                               0x0
+#define GRBM_UTCL2_INVAL_RANGE_END__DATA_MASK                                                                 0x0003FFFFL
+//GRBM_RSMU_READ_ERROR
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ADDRESS__SHIFT                                                        0x2
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VF__SHIFT                                                             0x14
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VFID__SHIFT                                                           0x15
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_TYPE__SHIFT                                                     0x1b
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR__SHIFT                                                          0x1f
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ADDRESS_MASK                                                          0x000FFFFCL
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VF_MASK                                                               0x00100000L
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_VFID_MASK                                                             0x07E00000L
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_TYPE_MASK                                                       0x08000000L
+#define GRBM_RSMU_READ_ERROR__RSMU_READ_ERROR_MASK                                                            0x80000000L
+//GRBM_INVALID_PIPE
+#define GRBM_INVALID_PIPE__ADDR__SHIFT                                                                        0x2
+#define GRBM_INVALID_PIPE__PIPEID__SHIFT                                                                      0x14
+#define GRBM_INVALID_PIPE__MEID__SHIFT                                                                        0x16
+#define GRBM_INVALID_PIPE__QUEUEID__SHIFT                                                                     0x18
+#define GRBM_INVALID_PIPE__SSRCID__SHIFT                                                                      0x1b
+#define GRBM_INVALID_PIPE__INVALID_PIPE__SHIFT                                                                0x1f
+#define GRBM_INVALID_PIPE__ADDR_MASK                                                                          0x000FFFFCL
+#define GRBM_INVALID_PIPE__PIPEID_MASK                                                                        0x00300000L
+#define GRBM_INVALID_PIPE__MEID_MASK                                                                          0x00C00000L
+#define GRBM_INVALID_PIPE__QUEUEID_MASK                                                                       0x07000000L
+#define GRBM_INVALID_PIPE__SSRCID_MASK                                                                        0x78000000L
+#define GRBM_INVALID_PIPE__INVALID_PIPE_MASK                                                                  0x80000000L
+//GRBM_FENCE_RANGE0
+#define GRBM_FENCE_RANGE0__START__SHIFT                                                                       0x0
+#define GRBM_FENCE_RANGE0__END__SHIFT                                                                         0x10
+#define GRBM_FENCE_RANGE0__START_MASK                                                                         0x0000FFFFL
+#define GRBM_FENCE_RANGE0__END_MASK                                                                           0xFFFF0000L
+//GRBM_FENCE_RANGE1
+#define GRBM_FENCE_RANGE1__START__SHIFT                                                                       0x0
+#define GRBM_FENCE_RANGE1__END__SHIFT                                                                         0x10
+#define GRBM_FENCE_RANGE1__START_MASK                                                                         0x0000FFFFL
+#define GRBM_FENCE_RANGE1__END_MASK                                                                           0xFFFF0000L
+//GRBM_SCRATCH_REG0
+#define GRBM_SCRATCH_REG0__SCRATCH_REG0__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG0__SCRATCH_REG0_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG1
+#define GRBM_SCRATCH_REG1__SCRATCH_REG1__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG1__SCRATCH_REG1_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG2
+#define GRBM_SCRATCH_REG2__SCRATCH_REG2__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG2__SCRATCH_REG2_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG3
+#define GRBM_SCRATCH_REG3__SCRATCH_REG3__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG3__SCRATCH_REG3_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG4
+#define GRBM_SCRATCH_REG4__SCRATCH_REG4__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG4__SCRATCH_REG4_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG5
+#define GRBM_SCRATCH_REG5__SCRATCH_REG5__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG5__SCRATCH_REG5_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG6
+#define GRBM_SCRATCH_REG6__SCRATCH_REG6__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG6__SCRATCH_REG6_MASK                                                                  0xFFFFFFFFL
+//GRBM_SCRATCH_REG7
+#define GRBM_SCRATCH_REG7__SCRATCH_REG7__SHIFT                                                                0x0
+#define GRBM_SCRATCH_REG7__SCRATCH_REG7_MASK                                                                  0xFFFFFFFFL
+//VIOLATION_DATA_ASYNC_VF_PROG
+#define VIOLATION_DATA_ASYNC_VF_PROG__SSRCID__SHIFT                                                           0x0
+#define VIOLATION_DATA_ASYNC_VF_PROG__VFID__SHIFT                                                             0x4
+#define VIOLATION_DATA_ASYNC_VF_PROG__VIOLATION_ERROR__SHIFT                                                  0x1f
+#define VIOLATION_DATA_ASYNC_VF_PROG__SSRCID_MASK                                                             0x0000000FL
+#define VIOLATION_DATA_ASYNC_VF_PROG__VFID_MASK                                                               0x000003F0L
+#define VIOLATION_DATA_ASYNC_VF_PROG__VIOLATION_ERROR_MASK                                                    0x80000000L
+
+
+// addressBlock: gc_cpdec
+//CP_CPC_DEBUG_CNTL
+#define CP_CPC_DEBUG_CNTL__DEBUG_INDX__SHIFT                                                                  0x0
+#define CP_CPC_DEBUG_CNTL__DEBUG_INDX_MASK                                                                    0x0000007FL
+//CP_CPF_DEBUG_CNTL
+#define CP_CPF_DEBUG_CNTL__DEBUG_INDX__SHIFT                                                                  0x0
+#define CP_CPF_DEBUG_CNTL__DEBUG_INDX_MASK                                                                    0x0000007FL
+//CP_CPC_STATUS
+#define CP_CPC_STATUS__MEC1_BUSY__SHIFT                                                                       0x0
+#define CP_CPC_STATUS__MEC2_BUSY__SHIFT                                                                       0x1
+#define CP_CPC_STATUS__DC0_BUSY__SHIFT                                                                        0x2
+#define CP_CPC_STATUS__DC1_BUSY__SHIFT                                                                        0x3
+#define CP_CPC_STATUS__RCIU1_BUSY__SHIFT                                                                      0x4
+#define CP_CPC_STATUS__RCIU2_BUSY__SHIFT                                                                      0x5
+#define CP_CPC_STATUS__ROQ1_BUSY__SHIFT                                                                       0x6
+#define CP_CPC_STATUS__ROQ2_BUSY__SHIFT                                                                       0x7
+#define CP_CPC_STATUS__TCIU_BUSY__SHIFT                                                                       0xa
+#define CP_CPC_STATUS__SCRATCH_RAM_BUSY__SHIFT                                                                0xb
+#define CP_CPC_STATUS__QU_BUSY__SHIFT                                                                         0xc
+#define CP_CPC_STATUS__UTCL2IU_BUSY__SHIFT                                                                    0xd
+#define CP_CPC_STATUS__SAVE_RESTORE_BUSY__SHIFT                                                               0xe
+#define CP_CPC_STATUS__GCRIU_BUSY__SHIFT                                                                      0xf
+#define CP_CPC_STATUS__MES_BUSY__SHIFT                                                                        0x10
+#define CP_CPC_STATUS__MES_SCRATCH_RAM_BUSY__SHIFT                                                            0x11
+#define CP_CPC_STATUS__RCIU3_BUSY__SHIFT                                                                      0x12
+#define CP_CPC_STATUS__MES_INSTRUCTION_CACHE_BUSY__SHIFT                                                      0x13
+#define CP_CPC_STATUS__MES_DATA_CACHE_BUSY__SHIFT                                                             0x14
+#define CP_CPC_STATUS__MEC_DATA_CACHE_BUSY__SHIFT                                                             0x15
+#define CP_CPC_STATUS__CPG_CPC_BUSY__SHIFT                                                                    0x1d
+#define CP_CPC_STATUS__CPF_CPC_BUSY__SHIFT                                                                    0x1e
+#define CP_CPC_STATUS__CPC_BUSY__SHIFT                                                                        0x1f
+#define CP_CPC_STATUS__MEC1_BUSY_MASK                                                                         0x00000001L
+#define CP_CPC_STATUS__MEC2_BUSY_MASK                                                                         0x00000002L
+#define CP_CPC_STATUS__DC0_BUSY_MASK                                                                          0x00000004L
+#define CP_CPC_STATUS__DC1_BUSY_MASK                                                                          0x00000008L
+#define CP_CPC_STATUS__RCIU1_BUSY_MASK                                                                        0x00000010L
+#define CP_CPC_STATUS__RCIU2_BUSY_MASK                                                                        0x00000020L
+#define CP_CPC_STATUS__ROQ1_BUSY_MASK                                                                         0x00000040L
+#define CP_CPC_STATUS__ROQ2_BUSY_MASK                                                                         0x00000080L
+#define CP_CPC_STATUS__TCIU_BUSY_MASK                                                                         0x00000400L
+#define CP_CPC_STATUS__SCRATCH_RAM_BUSY_MASK                                                                  0x00000800L
+#define CP_CPC_STATUS__QU_BUSY_MASK                                                                           0x00001000L
+#define CP_CPC_STATUS__UTCL2IU_BUSY_MASK                                                                      0x00002000L
+#define CP_CPC_STATUS__SAVE_RESTORE_BUSY_MASK                                                                 0x00004000L
+#define CP_CPC_STATUS__GCRIU_BUSY_MASK                                                                        0x00008000L
+#define CP_CPC_STATUS__MES_BUSY_MASK                                                                          0x00010000L
+#define CP_CPC_STATUS__MES_SCRATCH_RAM_BUSY_MASK                                                              0x00020000L
+#define CP_CPC_STATUS__RCIU3_BUSY_MASK                                                                        0x00040000L
+#define CP_CPC_STATUS__MES_INSTRUCTION_CACHE_BUSY_MASK                                                        0x00080000L
+#define CP_CPC_STATUS__MES_DATA_CACHE_BUSY_MASK                                                               0x00100000L
+#define CP_CPC_STATUS__MEC_DATA_CACHE_BUSY_MASK                                                               0x00200000L
+#define CP_CPC_STATUS__CPG_CPC_BUSY_MASK                                                                      0x20000000L
+#define CP_CPC_STATUS__CPF_CPC_BUSY_MASK                                                                      0x40000000L
+#define CP_CPC_STATUS__CPC_BUSY_MASK                                                                          0x80000000L
+//CP_CPC_BUSY_STAT
+#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY__SHIFT                                                               0x0
+#define CP_CPC_BUSY_STAT__MEC1_SEMAPHORE_BUSY__SHIFT                                                          0x1
+#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY__SHIFT                                                              0x2
+#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY__SHIFT                                                            0x3
+#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY__SHIFT                                                          0x4
+#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY__SHIFT                                                           0x5
+#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY__SHIFT                                                           0x6
+#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY__SHIFT                                                                 0x7
+#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY__SHIFT                                                                0x8
+#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY__SHIFT                                                      0x9
+#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY__SHIFT                                                              0xa
+#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY__SHIFT                                                              0xb
+#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY__SHIFT                                                              0xc
+#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY__SHIFT                                                              0xd
+#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY__SHIFT                                                               0x10
+#define CP_CPC_BUSY_STAT__MEC2_SEMAPHORE_BUSY__SHIFT                                                          0x11
+#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY__SHIFT                                                              0x12
+#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY__SHIFT                                                            0x13
+#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY__SHIFT                                                          0x14
+#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY__SHIFT                                                           0x15
+#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY__SHIFT                                                           0x16
+#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY__SHIFT                                                                 0x17
+#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY__SHIFT                                                                0x18
+#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY__SHIFT                                                      0x19
+#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY__SHIFT                                                              0x1a
+#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY__SHIFT                                                              0x1b
+#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY__SHIFT                                                              0x1c
+#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY__SHIFT                                                              0x1d
+#define CP_CPC_BUSY_STAT__MEC1_LOAD_BUSY_MASK                                                                 0x00000001L
+#define CP_CPC_BUSY_STAT__MEC1_SEMAPHORE_BUSY_MASK                                                            0x00000002L
+#define CP_CPC_BUSY_STAT__MEC1_MUTEX_BUSY_MASK                                                                0x00000004L
+#define CP_CPC_BUSY_STAT__MEC1_MESSAGE_BUSY_MASK                                                              0x00000008L
+#define CP_CPC_BUSY_STAT__MEC1_EOP_QUEUE_BUSY_MASK                                                            0x00000010L
+#define CP_CPC_BUSY_STAT__MEC1_IQ_QUEUE_BUSY_MASK                                                             0x00000020L
+#define CP_CPC_BUSY_STAT__MEC1_IB_QUEUE_BUSY_MASK                                                             0x00000040L
+#define CP_CPC_BUSY_STAT__MEC1_TC_BUSY_MASK                                                                   0x00000080L
+#define CP_CPC_BUSY_STAT__MEC1_DMA_BUSY_MASK                                                                  0x00000100L
+#define CP_CPC_BUSY_STAT__MEC1_PARTIAL_FLUSH_BUSY_MASK                                                        0x00000200L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE0_BUSY_MASK                                                                0x00000400L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE1_BUSY_MASK                                                                0x00000800L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE2_BUSY_MASK                                                                0x00001000L
+#define CP_CPC_BUSY_STAT__MEC1_PIPE3_BUSY_MASK                                                                0x00002000L
+#define CP_CPC_BUSY_STAT__MEC2_LOAD_BUSY_MASK                                                                 0x00010000L
+#define CP_CPC_BUSY_STAT__MEC2_SEMAPHORE_BUSY_MASK                                                            0x00020000L
+#define CP_CPC_BUSY_STAT__MEC2_MUTEX_BUSY_MASK                                                                0x00040000L
+#define CP_CPC_BUSY_STAT__MEC2_MESSAGE_BUSY_MASK                                                              0x00080000L
+#define CP_CPC_BUSY_STAT__MEC2_EOP_QUEUE_BUSY_MASK                                                            0x00100000L
+#define CP_CPC_BUSY_STAT__MEC2_IQ_QUEUE_BUSY_MASK                                                             0x00200000L
+#define CP_CPC_BUSY_STAT__MEC2_IB_QUEUE_BUSY_MASK                                                             0x00400000L
+#define CP_CPC_BUSY_STAT__MEC2_TC_BUSY_MASK                                                                   0x00800000L
+#define CP_CPC_BUSY_STAT__MEC2_DMA_BUSY_MASK                                                                  0x01000000L
+#define CP_CPC_BUSY_STAT__MEC2_PARTIAL_FLUSH_BUSY_MASK                                                        0x02000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE0_BUSY_MASK                                                                0x04000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE1_BUSY_MASK                                                                0x08000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE2_BUSY_MASK                                                                0x10000000L
+#define CP_CPC_BUSY_STAT__MEC2_PIPE3_BUSY_MASK                                                                0x20000000L
+//CP_CPC_STALLED_STAT1
+#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL__SHIFT                                                       0x3
+#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION__SHIFT                                                      0x4
+#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL__SHIFT                                                       0x6
+#define CP_CPC_STALLED_STAT1__TCIU_WAITING_ON_TAGS__SHIFT                                                     0x7
+#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET__SHIFT                                                     0x8
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU__SHIFT                                                        0x9
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ__SHIFT                                                   0xa
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA__SHIFT                                                    0xd
+#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET__SHIFT                                                     0x10
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU__SHIFT                                                        0x11
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ__SHIFT                                                   0x12
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA__SHIFT                                                    0x15
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE__SHIFT                                                  0x16
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                  0x17
+#define CP_CPC_STALLED_STAT1__UTCL1_WAITING_ON_TRANS__SHIFT                                                   0x18
+#define CP_CPC_STALLED_STAT1__GCRIU_WAITING_ON_FREE__SHIFT                                                    0x19
+#define CP_CPC_STALLED_STAT1__RCIU_TX_FREE_STALL_MASK                                                         0x00000008L
+#define CP_CPC_STALLED_STAT1__RCIU_PRIV_VIOLATION_MASK                                                        0x00000010L
+#define CP_CPC_STALLED_STAT1__TCIU_TX_FREE_STALL_MASK                                                         0x00000040L
+#define CP_CPC_STALLED_STAT1__TCIU_WAITING_ON_TAGS_MASK                                                       0x00000080L
+#define CP_CPC_STALLED_STAT1__MEC1_DECODING_PACKET_MASK                                                       0x00000100L
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_MASK                                                          0x00000200L
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_RCIU_READ_MASK                                                     0x00000400L
+#define CP_CPC_STALLED_STAT1__MEC1_WAIT_ON_ROQ_DATA_MASK                                                      0x00002000L
+#define CP_CPC_STALLED_STAT1__MEC2_DECODING_PACKET_MASK                                                       0x00010000L
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_MASK                                                          0x00020000L
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_RCIU_READ_MASK                                                     0x00040000L
+#define CP_CPC_STALLED_STAT1__MEC2_WAIT_ON_ROQ_DATA_MASK                                                      0x00200000L
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE_MASK                                                    0x00400000L
+#define CP_CPC_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS_MASK                                                    0x00800000L
+#define CP_CPC_STALLED_STAT1__UTCL1_WAITING_ON_TRANS_MASK                                                     0x01000000L
+#define CP_CPC_STALLED_STAT1__GCRIU_WAITING_ON_FREE_MASK                                                      0x02000000L
+//CP_CPF_STATUS
+#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY__SHIFT                                                              0x0
+#define CP_CPF_STATUS__CSF_BUSY__SHIFT                                                                        0x1
+#define CP_CPF_STATUS__ROQ_ALIGN_BUSY__SHIFT                                                                  0x4
+#define CP_CPF_STATUS__ROQ_RING_BUSY__SHIFT                                                                   0x5
+#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY__SHIFT                                                              0x6
+#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY__SHIFT                                                              0x7
+#define CP_CPF_STATUS__ROQ_STATE_BUSY__SHIFT                                                                  0x8
+#define CP_CPF_STATUS__ROQ_CE_RING_BUSY__SHIFT                                                                0x9
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY__SHIFT                                                           0xa
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY__SHIFT                                                           0xb
+#define CP_CPF_STATUS__SEMAPHORE_BUSY__SHIFT                                                                  0xc
+#define CP_CPF_STATUS__INTERRUPT_BUSY__SHIFT                                                                  0xd
+#define CP_CPF_STATUS__TCIU_BUSY__SHIFT                                                                       0xe
+#define CP_CPF_STATUS__HQD_BUSY__SHIFT                                                                        0xf
+#define CP_CPF_STATUS__PRT_BUSY__SHIFT                                                                        0x10
+#define CP_CPF_STATUS__UTCL2IU_BUSY__SHIFT                                                                    0x11
+#define CP_CPF_STATUS__RCIU_BUSY__SHIFT                                                                       0x12
+#define CP_CPF_STATUS__RCIU_GFX_BUSY__SHIFT                                                                   0x13
+#define CP_CPF_STATUS__RCIU_CMP_BUSY__SHIFT                                                                   0x14
+#define CP_CPF_STATUS__ROQ_DATA_BUSY__SHIFT                                                                   0x15
+#define CP_CPF_STATUS__ROQ_CE_DATA_BUSY__SHIFT                                                                0x16
+#define CP_CPF_STATUS__GCRIU_BUSY__SHIFT                                                                      0x17
+#define CP_CPF_STATUS__MES_HQD_BUSY__SHIFT                                                                    0x18
+#define CP_CPF_STATUS__CPF_GFX_BUSY__SHIFT                                                                    0x1a
+#define CP_CPF_STATUS__CPF_CMP_BUSY__SHIFT                                                                    0x1b
+#define CP_CPF_STATUS__GRBM_CPF_STAT_BUSY__SHIFT                                                              0x1c
+#define CP_CPF_STATUS__CPC_CPF_BUSY__SHIFT                                                                    0x1e
+#define CP_CPF_STATUS__CPF_BUSY__SHIFT                                                                        0x1f
+#define CP_CPF_STATUS__POST_WPTR_GFX_BUSY_MASK                                                                0x00000001L
+#define CP_CPF_STATUS__CSF_BUSY_MASK                                                                          0x00000002L
+#define CP_CPF_STATUS__ROQ_ALIGN_BUSY_MASK                                                                    0x00000010L
+#define CP_CPF_STATUS__ROQ_RING_BUSY_MASK                                                                     0x00000020L
+#define CP_CPF_STATUS__ROQ_INDIRECT1_BUSY_MASK                                                                0x00000040L
+#define CP_CPF_STATUS__ROQ_INDIRECT2_BUSY_MASK                                                                0x00000080L
+#define CP_CPF_STATUS__ROQ_STATE_BUSY_MASK                                                                    0x00000100L
+#define CP_CPF_STATUS__ROQ_CE_RING_BUSY_MASK                                                                  0x00000200L
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT1_BUSY_MASK                                                             0x00000400L
+#define CP_CPF_STATUS__ROQ_CE_INDIRECT2_BUSY_MASK                                                             0x00000800L
+#define CP_CPF_STATUS__SEMAPHORE_BUSY_MASK                                                                    0x00001000L
+#define CP_CPF_STATUS__INTERRUPT_BUSY_MASK                                                                    0x00002000L
+#define CP_CPF_STATUS__TCIU_BUSY_MASK                                                                         0x00004000L
+#define CP_CPF_STATUS__HQD_BUSY_MASK                                                                          0x00008000L
+#define CP_CPF_STATUS__PRT_BUSY_MASK                                                                          0x00010000L
+#define CP_CPF_STATUS__UTCL2IU_BUSY_MASK                                                                      0x00020000L
+#define CP_CPF_STATUS__RCIU_BUSY_MASK                                                                         0x00040000L
+#define CP_CPF_STATUS__RCIU_GFX_BUSY_MASK                                                                     0x00080000L
+#define CP_CPF_STATUS__RCIU_CMP_BUSY_MASK                                                                     0x00100000L
+#define CP_CPF_STATUS__ROQ_DATA_BUSY_MASK                                                                     0x00200000L
+#define CP_CPF_STATUS__ROQ_CE_DATA_BUSY_MASK                                                                  0x00400000L
+#define CP_CPF_STATUS__GCRIU_BUSY_MASK                                                                        0x00800000L
+#define CP_CPF_STATUS__MES_HQD_BUSY_MASK                                                                      0x01000000L
+#define CP_CPF_STATUS__CPF_GFX_BUSY_MASK                                                                      0x04000000L
+#define CP_CPF_STATUS__CPF_CMP_BUSY_MASK                                                                      0x08000000L
+#define CP_CPF_STATUS__GRBM_CPF_STAT_BUSY_MASK                                                                0x30000000L
+#define CP_CPF_STATUS__CPC_CPF_BUSY_MASK                                                                      0x40000000L
+#define CP_CPF_STATUS__CPF_BUSY_MASK                                                                          0x80000000L
+//CP_CPF_BUSY_STAT
+#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT                                                            0x0
+#define CP_CPF_BUSY_STAT__CSF_RING_BUSY__SHIFT                                                                0x1
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY__SHIFT                                                           0x2
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY__SHIFT                                                           0x3
+#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY__SHIFT                                                               0x4
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY__SHIFT                                                            0x5
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY__SHIFT                                                            0x6
+#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY__SHIFT                                                             0x7
+#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY__SHIFT                                                               0x8
+#define CP_CPF_BUSY_STAT__CSF_DATA_BUSY__SHIFT                                                                0x9
+#define CP_CPF_BUSY_STAT__CSF_CE_DATA_BUSY__SHIFT                                                             0xa
+#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY__SHIFT                                                      0xb
+#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY__SHIFT                                                            0xc
+#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY__SHIFT                                                            0xd
+#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY__SHIFT                                                         0xe
+#define CP_CPF_BUSY_STAT__HQD_WAIT_SEMAPHORE_BUSY__SHIFT                                                      0xf
+#define CP_CPF_BUSY_STAT__HQD_SIGNAL_SEMAPHORE_BUSY__SHIFT                                                    0x10
+#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY__SHIFT                                                             0x11
+#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY__SHIFT                                                          0x12
+#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY__SHIFT                                                          0x13
+#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY__SHIFT                                                          0x14
+#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY__SHIFT                                                         0x15
+#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY__SHIFT                                                       0x16
+#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY__SHIFT                                                         0x17
+#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY__SHIFT                                                           0x18
+#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY__SHIFT                                                             0x19
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY__SHIFT                                                              0x1a
+#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY__SHIFT                                                              0x1b
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY__SHIFT                                                              0x1c
+#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY__SHIFT                                                           0x1d
+#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY__SHIFT                                                                  0x1e
+#define CP_CPF_BUSY_STAT__HQD_IB_BUSY__SHIFT                                                                  0x1f
+#define CP_CPF_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK                                                              0x00000001L
+#define CP_CPF_BUSY_STAT__CSF_RING_BUSY_MASK                                                                  0x00000002L
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT1_BUSY_MASK                                                             0x00000004L
+#define CP_CPF_BUSY_STAT__CSF_INDIRECT2_BUSY_MASK                                                             0x00000008L
+#define CP_CPF_BUSY_STAT__CSF_STATE_BUSY_MASK                                                                 0x00000010L
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR1_BUSY_MASK                                                              0x00000020L
+#define CP_CPF_BUSY_STAT__CSF_CE_INDR2_BUSY_MASK                                                              0x00000040L
+#define CP_CPF_BUSY_STAT__CSF_ARBITER_BUSY_MASK                                                               0x00000080L
+#define CP_CPF_BUSY_STAT__CSF_INPUT_BUSY_MASK                                                                 0x00000100L
+#define CP_CPF_BUSY_STAT__CSF_DATA_BUSY_MASK                                                                  0x00000200L
+#define CP_CPF_BUSY_STAT__CSF_CE_DATA_BUSY_MASK                                                               0x00000400L
+#define CP_CPF_BUSY_STAT__HPD_PROCESSING_EOP_BUSY_MASK                                                        0x00000800L
+#define CP_CPF_BUSY_STAT__HQD_DISPATCH_BUSY_MASK                                                              0x00001000L
+#define CP_CPF_BUSY_STAT__HQD_IQ_TIMER_BUSY_MASK                                                              0x00002000L
+#define CP_CPF_BUSY_STAT__HQD_DMA_OFFLOAD_BUSY_MASK                                                           0x00004000L
+#define CP_CPF_BUSY_STAT__HQD_WAIT_SEMAPHORE_BUSY_MASK                                                        0x00008000L
+#define CP_CPF_BUSY_STAT__HQD_SIGNAL_SEMAPHORE_BUSY_MASK                                                      0x00010000L
+#define CP_CPF_BUSY_STAT__HQD_MESSAGE_BUSY_MASK                                                               0x00020000L
+#define CP_CPF_BUSY_STAT__HQD_PQ_FETCHER_BUSY_MASK                                                            0x00040000L
+#define CP_CPF_BUSY_STAT__HQD_IB_FETCHER_BUSY_MASK                                                            0x00080000L
+#define CP_CPF_BUSY_STAT__HQD_IQ_FETCHER_BUSY_MASK                                                            0x00100000L
+#define CP_CPF_BUSY_STAT__HQD_EOP_FETCHER_BUSY_MASK                                                           0x00200000L
+#define CP_CPF_BUSY_STAT__HQD_CONSUMED_RPTR_BUSY_MASK                                                         0x00400000L
+#define CP_CPF_BUSY_STAT__HQD_FETCHER_ARB_BUSY_MASK                                                           0x00800000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_ALIGN_BUSY_MASK                                                             0x01000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_EOP_BUSY_MASK                                                               0x02000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IQ_BUSY_MASK                                                                0x04000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_PQ_BUSY_MASK                                                                0x08000000L
+#define CP_CPF_BUSY_STAT__HQD_ROQ_IB_BUSY_MASK                                                                0x10000000L
+#define CP_CPF_BUSY_STAT__HQD_WPTR_POLL_BUSY_MASK                                                             0x20000000L
+#define CP_CPF_BUSY_STAT__HQD_PQ_BUSY_MASK                                                                    0x40000000L
+#define CP_CPF_BUSY_STAT__HQD_IB_BUSY_MASK                                                                    0x80000000L
+//CP_CPF_STALLED_STAT1
+#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA__SHIFT                                                       0x0
+#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA__SHIFT                                                      0x1
+#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA__SHIFT                                                      0x2
+#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA__SHIFT                                                      0x3
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE__SHIFT                                                     0x5
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS__SHIFT                                                     0x6
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE__SHIFT                                                  0x7
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                  0x8
+#define CP_CPF_STALLED_STAT1__GFX_UTCL1_WAITING_ON_TRANS__SHIFT                                               0x9
+#define CP_CPF_STALLED_STAT1__CMP_UTCL1_WAITING_ON_TRANS__SHIFT                                               0xa
+#define CP_CPF_STALLED_STAT1__RCIU_WAITING_ON_FREE__SHIFT                                                     0xb
+#define CP_CPF_STALLED_STAT1__DATA_FETCHING_DATA__SHIFT                                                       0xc
+#define CP_CPF_STALLED_STAT1__GCRIU_WAIT_ON_FREE__SHIFT                                                       0xd
+#define CP_CPF_STALLED_STAT1__RING_FETCHING_DATA_MASK                                                         0x00000001L
+#define CP_CPF_STALLED_STAT1__INDR1_FETCHING_DATA_MASK                                                        0x00000002L
+#define CP_CPF_STALLED_STAT1__INDR2_FETCHING_DATA_MASK                                                        0x00000004L
+#define CP_CPF_STALLED_STAT1__STATE_FETCHING_DATA_MASK                                                        0x00000008L
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_FREE_MASK                                                       0x00000020L
+#define CP_CPF_STALLED_STAT1__TCIU_WAITING_ON_TAGS_MASK                                                       0x00000040L
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_FREE_MASK                                                    0x00000080L
+#define CP_CPF_STALLED_STAT1__UTCL2IU_WAITING_ON_TAGS_MASK                                                    0x00000100L
+#define CP_CPF_STALLED_STAT1__GFX_UTCL1_WAITING_ON_TRANS_MASK                                                 0x00000200L
+#define CP_CPF_STALLED_STAT1__CMP_UTCL1_WAITING_ON_TRANS_MASK                                                 0x00000400L
+#define CP_CPF_STALLED_STAT1__RCIU_WAITING_ON_FREE_MASK                                                       0x00000800L
+#define CP_CPF_STALLED_STAT1__DATA_FETCHING_DATA_MASK                                                         0x00001000L
+#define CP_CPF_STALLED_STAT1__GCRIU_WAIT_ON_FREE_MASK                                                         0x00002000L
+//CP_CPC_BUSY_STAT2
+#define CP_CPC_BUSY_STAT2__MES_LOAD_BUSY__SHIFT                                                               0x0
+#define CP_CPC_BUSY_STAT2__MES_MUTEX_BUSY__SHIFT                                                              0x2
+#define CP_CPC_BUSY_STAT2__MES_MESSAGE_BUSY__SHIFT                                                            0x3
+#define CP_CPC_BUSY_STAT2__MES_TC_BUSY__SHIFT                                                                 0x7
+#define CP_CPC_BUSY_STAT2__MES_DMA_BUSY__SHIFT                                                                0x8
+#define CP_CPC_BUSY_STAT2__MES_PIPE0_BUSY__SHIFT                                                              0xa
+#define CP_CPC_BUSY_STAT2__MES_PIPE1_BUSY__SHIFT                                                              0xb
+#define CP_CPC_BUSY_STAT2__MES_PIPE2_BUSY__SHIFT                                                              0xc
+#define CP_CPC_BUSY_STAT2__MES_PIPE3_BUSY__SHIFT                                                              0xd
+#define CP_CPC_BUSY_STAT2__MES_LOAD_BUSY_MASK                                                                 0x00000001L
+#define CP_CPC_BUSY_STAT2__MES_MUTEX_BUSY_MASK                                                                0x00000004L
+#define CP_CPC_BUSY_STAT2__MES_MESSAGE_BUSY_MASK                                                              0x00000008L
+#define CP_CPC_BUSY_STAT2__MES_TC_BUSY_MASK                                                                   0x00000080L
+#define CP_CPC_BUSY_STAT2__MES_DMA_BUSY_MASK                                                                  0x00000100L
+#define CP_CPC_BUSY_STAT2__MES_PIPE0_BUSY_MASK                                                                0x00000400L
+#define CP_CPC_BUSY_STAT2__MES_PIPE1_BUSY_MASK                                                                0x00000800L
+#define CP_CPC_BUSY_STAT2__MES_PIPE2_BUSY_MASK                                                                0x00001000L
+#define CP_CPC_BUSY_STAT2__MES_PIPE3_BUSY_MASK                                                                0x00002000L
+//CP_CPC_GRBM_FREE_COUNT
+#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                             0x0
+#define CP_CPC_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                               0x0000003FL
+//CP_CPC_PRIV_VIOLATION_ADDR
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR__SHIFT                                                0x0
+#define CP_CPC_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR_MASK                                                  0x0003FFFFL
+//CP_MEC_ME1_HEADER_DUMP
+#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP__SHIFT                                                            0x0
+#define CP_MEC_ME1_HEADER_DUMP__HEADER_DUMP_MASK                                                              0xFFFFFFFFL
+//CP_MEC_ME2_HEADER_DUMP
+#define CP_MEC_ME2_HEADER_DUMP__HEADER_DUMP__SHIFT                                                            0x0
+#define CP_MEC_ME2_HEADER_DUMP__HEADER_DUMP_MASK                                                              0xFFFFFFFFL
+//CP_CPC_SCRATCH_INDEX
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                            0x0
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE__SHIFT                                                 0x1f
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                              0x000001FFL
+#define CP_CPC_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE_MASK                                                   0x80000000L
+//CP_CPC_SCRATCH_DATA
+#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                              0x0
+#define CP_CPC_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                0xFFFFFFFFL
+//CP_CPF_GRBM_FREE_COUNT
+#define CP_CPF_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                             0x0
+#define CP_CPF_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                               0x00000007L
+//CP_CPF_BUSY_STAT2
+#define CP_CPF_BUSY_STAT2__CP_SDMA_CPG_BUSY__SHIFT                                                            0x0
+#define CP_CPF_BUSY_STAT2__CP_SDMA_CPC_BUSY__SHIFT                                                            0x1
+#define CP_CPF_BUSY_STAT2__MES_HQD_DISPATCH_BUSY__SHIFT                                                       0xc
+#define CP_CPF_BUSY_STAT2__MES_HQD_DMA_OFFLOAD_BUSY__SHIFT                                                    0xe
+#define CP_CPF_BUSY_STAT2__MES_HQD_MESSAGE_BUSY__SHIFT                                                        0x11
+#define CP_CPF_BUSY_STAT2__MES_HQD_PQ_FETCHER_BUSY__SHIFT                                                     0x12
+#define CP_CPF_BUSY_STAT2__MES_HQD_CONSUMED_RPTR_BUSY__SHIFT                                                  0x16
+#define CP_CPF_BUSY_STAT2__MES_HQD_FETCHER_ARB_BUSY__SHIFT                                                    0x17
+#define CP_CPF_BUSY_STAT2__MES_HQD_ROQ_ALIGN_BUSY__SHIFT                                                      0x18
+#define CP_CPF_BUSY_STAT2__MES_HQD_ROQ_PQ_BUSY__SHIFT                                                         0x1b
+#define CP_CPF_BUSY_STAT2__MES_HQD_PQ_BUSY__SHIFT                                                             0x1e
+#define CP_CPF_BUSY_STAT2__CP_SDMA_CPG_BUSY_MASK                                                              0x00000001L
+#define CP_CPF_BUSY_STAT2__CP_SDMA_CPC_BUSY_MASK                                                              0x00000002L
+#define CP_CPF_BUSY_STAT2__MES_HQD_DISPATCH_BUSY_MASK                                                         0x00001000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_DMA_OFFLOAD_BUSY_MASK                                                      0x00004000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_MESSAGE_BUSY_MASK                                                          0x00020000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_PQ_FETCHER_BUSY_MASK                                                       0x00040000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_CONSUMED_RPTR_BUSY_MASK                                                    0x00400000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_FETCHER_ARB_BUSY_MASK                                                      0x00800000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_ROQ_ALIGN_BUSY_MASK                                                        0x01000000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_ROQ_PQ_BUSY_MASK                                                           0x08000000L
+#define CP_CPF_BUSY_STAT2__MES_HQD_PQ_BUSY_MASK                                                               0x40000000L
+//CP_CPC_HALT_HYST_COUNT
+#define CP_CPC_HALT_HYST_COUNT__COUNT__SHIFT                                                                  0x0
+#define CP_CPC_HALT_HYST_COUNT__COUNT_MASK                                                                    0x0000000FL
+//CP_STALLED_STAT3
+#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV__SHIFT                                                     0x0
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV__SHIFT                                        0x1
+#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER__SHIFT                                     0x2
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY__SHIFT                                                       0x3
+#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY__SHIFT                                                       0x4
+#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY__SHIFT                                                      0x5
+#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV__SHIFT                                                0x6
+#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV__SHIFT                                                 0x7
+#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA__SHIFT                                                    0xa
+#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG__SHIFT                                                 0xb
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER__SHIFT                                                     0xc
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW__SHIFT                                           0xd
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE__SHIFT                                                         0xe
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS__SHIFT                                                         0xf
+#define CP_STALLED_STAT3__CE_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                  0x10
+#define CP_STALLED_STAT3__CE_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                                0x11
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_FREE__SHIFT                                                      0x12
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_TAGS__SHIFT                                                      0x13
+#define CP_STALLED_STAT3__UTCL1_WAITING_ON_TRANS__SHIFT                                                       0x14
+#define CP_STALLED_STAT3__GCRIU_WAITING_ON_FREE__SHIFT                                                        0x15
+#define CP_STALLED_STAT3__CE_TO_CSF_NOT_RDY_TO_RCV_MASK                                                       0x00000001L
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_FETCHER_NOT_RDY_TO_RCV_MASK                                          0x00000002L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DATA_FROM_RAM_INIT_FETCHER_MASK                                       0x00000004L
+#define CP_STALLED_STAT3__CE_TO_RAM_INIT_NOT_RDY_MASK                                                         0x00000008L
+#define CP_STALLED_STAT3__CE_TO_RAM_DUMP_NOT_RDY_MASK                                                         0x00000010L
+#define CP_STALLED_STAT3__CE_TO_RAM_WRITE_NOT_RDY_MASK                                                        0x00000020L
+#define CP_STALLED_STAT3__CE_TO_INC_FIFO_NOT_RDY_TO_RCV_MASK                                                  0x00000040L
+#define CP_STALLED_STAT3__CE_TO_WR_FIFO_NOT_RDY_TO_RCV_MASK                                                   0x00000080L
+#define CP_STALLED_STAT3__CE_WAITING_ON_BUFFER_DATA_MASK                                                      0x00000400L
+#define CP_STALLED_STAT3__CE_WAITING_ON_CE_BUFFER_FLAG_MASK                                                   0x00000800L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_MASK                                                       0x00001000L
+#define CP_STALLED_STAT3__CE_WAITING_ON_DE_COUNTER_UNDERFLOW_MASK                                             0x00002000L
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_FREE_MASK                                                           0x00004000L
+#define CP_STALLED_STAT3__TCIU_WAITING_ON_TAGS_MASK                                                           0x00008000L
+#define CP_STALLED_STAT3__CE_STALLED_ON_TC_WR_CONFIRM_MASK                                                    0x00010000L
+#define CP_STALLED_STAT3__CE_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                  0x00020000L
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_FREE_MASK                                                        0x00040000L
+#define CP_STALLED_STAT3__UTCL2IU_WAITING_ON_TAGS_MASK                                                        0x00080000L
+#define CP_STALLED_STAT3__UTCL1_WAITING_ON_TRANS_MASK                                                         0x00100000L
+#define CP_STALLED_STAT3__GCRIU_WAITING_ON_FREE_MASK                                                          0x00200000L
+//CP_STALLED_STAT1
+#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV__SHIFT                                                   0x0
+#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV_R0__SHIFT                                                0x2
+#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV_R1__SHIFT                                                0x3
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_R0__SHIFT                                              0x4
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_R1__SHIFT                                              0x5
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG__SHIFT                                                 0xa
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG__SHIFT                                                 0xb
+#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                  0xc
+#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                                0xd
+#define CP_STALLED_STAT1__ME_WAITING_ON_TC_READ_DATA__SHIFT                                                   0xe
+#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA__SHIFT                                                  0xf
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE__SHIFT                                                     0x17
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE__SHIFT                                                    0x18
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE__SHIFT                                                     0x19
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ__SHIFT                                                      0x1a
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ__SHIFT                                                     0x1b
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ__SHIFT                                                  0x1c
+#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION__SHIFT                                                 0x1d
+#define CP_STALLED_STAT1__RBIU_TO_DMA_NOT_RDY_TO_RCV_MASK                                                     0x00000001L
+#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV_R0_MASK                                                  0x00000004L
+#define CP_STALLED_STAT1__RBIU_TO_SEM_NOT_RDY_TO_RCV_R1_MASK                                                  0x00000008L
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_R0_MASK                                                0x00000010L
+#define CP_STALLED_STAT1__RBIU_TO_MEMWR_NOT_RDY_TO_RCV_R1_MASK                                                0x00000020L
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_CE_BUFFER_FLAG_MASK                                                   0x00000400L
+#define CP_STALLED_STAT1__ME_HAS_ACTIVE_DE_BUFFER_FLAG_MASK                                                   0x00000800L
+#define CP_STALLED_STAT1__ME_STALLED_ON_TC_WR_CONFIRM_MASK                                                    0x00001000L
+#define CP_STALLED_STAT1__ME_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                  0x00002000L
+#define CP_STALLED_STAT1__ME_WAITING_ON_TC_READ_DATA_MASK                                                     0x00004000L
+#define CP_STALLED_STAT1__ME_WAITING_ON_REG_READ_DATA_MASK                                                    0x00008000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GDS_FREE_MASK                                                       0x00800000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_GRBM_FREE_MASK                                                      0x01000000L
+#define CP_STALLED_STAT1__RCIU_WAITING_ON_VGT_FREE_MASK                                                       0x02000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_ME_READ_MASK                                                        0x04000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_DMA_READ_MASK                                                       0x08000000L
+#define CP_STALLED_STAT1__RCIU_STALLED_ON_APPEND_READ_MASK                                                    0x10000000L
+#define CP_STALLED_STAT1__RCIU_HALTED_BY_REG_VIOLATION_MASK                                                   0x20000000L
+//CP_STALLED_STAT2
+#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV__SHIFT                                                    0x0
+#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV__SHIFT                                                    0x1
+#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV__SHIFT                                                   0x2
+#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING__SHIFT                                                    0x4
+#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING__SHIFT                                                        0x5
+#define CP_STALLED_STAT2__PFP_TO_MEQ_DDID_NOT_RDY_TO_RCV__SHIFT                                               0x6
+#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA__SHIFT                                                   0x8
+#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER__SHIFT                                                        0x9
+#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER__SHIFT                                                      0xa
+#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME__SHIFT                                                     0xb
+#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV__SHIFT                                                       0xc
+#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV__SHIFT                                                   0xd
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP__SHIFT                                                     0xe
+#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH__SHIFT                                                  0xf
+#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV__SHIFT                                                     0x10
+#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV__SHIFT                                                     0x11
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ__SHIFT                                                     0x12
+#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM__SHIFT                                                 0x13
+#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA__SHIFT                                               0x14
+#define CP_STALLED_STAT2__QU_STALLED_ON_EOP_DONE_PULSE__SHIFT                                                 0x15
+#define CP_STALLED_STAT2__QU_STALLED_ON_EOP_DONE_WR_CONFIRM__SHIFT                                            0x16
+#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING__SHIFT                                                0x17
+#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING__SHIFT                                                   0x18
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE__SHIFT                                                   0x19
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE__SHIFT                                                   0x1a
+#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM__SHIFT                                                    0x1b
+#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION__SHIFT                                                      0x1c
+#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE__SHIFT                                              0x1d
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS__SHIFT                                                   0x1e
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN__SHIFT                                                    0x1f
+#define CP_STALLED_STAT2__PFP_TO_CSF_NOT_RDY_TO_RCV_MASK                                                      0x00000001L
+#define CP_STALLED_STAT2__PFP_TO_MEQ_NOT_RDY_TO_RCV_MASK                                                      0x00000002L
+#define CP_STALLED_STAT2__PFP_TO_RCIU_NOT_RDY_TO_RCV_MASK                                                     0x00000004L
+#define CP_STALLED_STAT2__PFP_TO_VGT_WRITES_PENDING_MASK                                                      0x00000010L
+#define CP_STALLED_STAT2__PFP_RCIU_READ_PENDING_MASK                                                          0x00000020L
+#define CP_STALLED_STAT2__PFP_TO_MEQ_DDID_NOT_RDY_TO_RCV_MASK                                                 0x00000040L
+#define CP_STALLED_STAT2__PFP_WAITING_ON_BUFFER_DATA_MASK                                                     0x00000100L
+#define CP_STALLED_STAT2__ME_WAIT_ON_CE_COUNTER_MASK                                                          0x00000200L
+#define CP_STALLED_STAT2__ME_WAIT_ON_AVAIL_BUFFER_MASK                                                        0x00000400L
+#define CP_STALLED_STAT2__GFX_CNTX_NOT_AVAIL_TO_ME_MASK                                                       0x00000800L
+#define CP_STALLED_STAT2__ME_RCIU_NOT_RDY_TO_RCV_MASK                                                         0x00001000L
+#define CP_STALLED_STAT2__ME_TO_CONST_NOT_RDY_TO_RCV_MASK                                                     0x00002000L
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_PFP_MASK                                                       0x00004000L
+#define CP_STALLED_STAT2__ME_WAITING_ON_PARTIAL_FLUSH_MASK                                                    0x00008000L
+#define CP_STALLED_STAT2__MEQ_TO_ME_NOT_RDY_TO_RCV_MASK                                                       0x00010000L
+#define CP_STALLED_STAT2__STQ_TO_ME_NOT_RDY_TO_RCV_MASK                                                       0x00020000L
+#define CP_STALLED_STAT2__ME_WAITING_DATA_FROM_STQ_MASK                                                       0x00040000L
+#define CP_STALLED_STAT2__PFP_STALLED_ON_TC_WR_CONFIRM_MASK                                                   0x00080000L
+#define CP_STALLED_STAT2__PFP_STALLED_ON_ATOMIC_RTN_DATA_MASK                                                 0x00100000L
+#define CP_STALLED_STAT2__QU_STALLED_ON_EOP_DONE_PULSE_MASK                                                   0x00200000L
+#define CP_STALLED_STAT2__QU_STALLED_ON_EOP_DONE_WR_CONFIRM_MASK                                              0x00400000L
+#define CP_STALLED_STAT2__STRMO_WR_OF_PRIM_DATA_PENDING_MASK                                                  0x00800000L
+#define CP_STALLED_STAT2__PIPE_STATS_WR_DATA_PENDING_MASK                                                     0x01000000L
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_CS_DONE_MASK                                                     0x02000000L
+#define CP_STALLED_STAT2__APPEND_RDY_WAIT_ON_PS_DONE_MASK                                                     0x04000000L
+#define CP_STALLED_STAT2__APPEND_WAIT_ON_WR_CONFIRM_MASK                                                      0x08000000L
+#define CP_STALLED_STAT2__APPEND_ACTIVE_PARTITION_MASK                                                        0x10000000L
+#define CP_STALLED_STAT2__APPEND_WAITING_TO_SEND_MEMWRITE_MASK                                                0x20000000L
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_IDLE_CNTXS_MASK                                                     0x40000000L
+#define CP_STALLED_STAT2__SURF_SYNC_NEEDS_ALL_CLEAN_MASK                                                      0x80000000L
+//CP_BUSY_STAT
+#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY__SHIFT                                                                0x0
+#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO__SHIFT                                                               0x6
+#define CP_BUSY_STAT__PFP_PARSING_PACKETS__SHIFT                                                              0x7
+#define CP_BUSY_STAT__ME_PARSING_PACKETS__SHIFT                                                               0x8
+#define CP_BUSY_STAT__RCIU_PFP_BUSY__SHIFT                                                                    0x9
+#define CP_BUSY_STAT__RCIU_ME_BUSY__SHIFT                                                                     0xa
+#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY__SHIFT                                                            0xc
+#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING__SHIFT                                                           0xd
+#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS__SHIFT                                                             0xe
+#define CP_BUSY_STAT__GFX_CONTEXT_BUSY__SHIFT                                                                 0xf
+#define CP_BUSY_STAT__ME_PARSER_BUSY__SHIFT                                                                   0x11
+#define CP_BUSY_STAT__EOP_DONE_BUSY__SHIFT                                                                    0x12
+#define CP_BUSY_STAT__STRM_OUT_BUSY__SHIFT                                                                    0x13
+#define CP_BUSY_STAT__PIPE_STATS_BUSY__SHIFT                                                                  0x14
+#define CP_BUSY_STAT__RCIU_CE_BUSY__SHIFT                                                                     0x15
+#define CP_BUSY_STAT__CE_PARSING_PACKETS__SHIFT                                                               0x16
+#define CP_BUSY_STAT__REG_BUS_FIFO_BUSY_MASK                                                                  0x00000001L
+#define CP_BUSY_STAT__COHER_CNT_NEQ_ZERO_MASK                                                                 0x00000040L
+#define CP_BUSY_STAT__PFP_PARSING_PACKETS_MASK                                                                0x00000080L
+#define CP_BUSY_STAT__ME_PARSING_PACKETS_MASK                                                                 0x00000100L
+#define CP_BUSY_STAT__RCIU_PFP_BUSY_MASK                                                                      0x00000200L
+#define CP_BUSY_STAT__RCIU_ME_BUSY_MASK                                                                       0x00000400L
+#define CP_BUSY_STAT__SEM_CMDFIFO_NOT_EMPTY_MASK                                                              0x00001000L
+#define CP_BUSY_STAT__SEM_FAILED_AND_HOLDING_MASK                                                             0x00002000L
+#define CP_BUSY_STAT__SEM_POLLING_FOR_PASS_MASK                                                               0x00004000L
+#define CP_BUSY_STAT__GFX_CONTEXT_BUSY_MASK                                                                   0x00008000L
+#define CP_BUSY_STAT__ME_PARSER_BUSY_MASK                                                                     0x00020000L
+#define CP_BUSY_STAT__EOP_DONE_BUSY_MASK                                                                      0x00040000L
+#define CP_BUSY_STAT__STRM_OUT_BUSY_MASK                                                                      0x00080000L
+#define CP_BUSY_STAT__PIPE_STATS_BUSY_MASK                                                                    0x00100000L
+#define CP_BUSY_STAT__RCIU_CE_BUSY_MASK                                                                       0x00200000L
+#define CP_BUSY_STAT__CE_PARSING_PACKETS_MASK                                                                 0x00400000L
+//CP_STAT
+#define CP_STAT__ROQ_DB_BUSY__SHIFT                                                                           0x5
+#define CP_STAT__ROQ_CE_DB_BUSY__SHIFT                                                                        0x6
+#define CP_STAT__ROQ_RING_BUSY__SHIFT                                                                         0x9
+#define CP_STAT__ROQ_INDIRECT1_BUSY__SHIFT                                                                    0xa
+#define CP_STAT__ROQ_INDIRECT2_BUSY__SHIFT                                                                    0xb
+#define CP_STAT__ROQ_STATE_BUSY__SHIFT                                                                        0xc
+#define CP_STAT__DC_BUSY__SHIFT                                                                               0xd
+#define CP_STAT__UTCL2IU_BUSY__SHIFT                                                                          0xe
+#define CP_STAT__PFP_BUSY__SHIFT                                                                              0xf
+#define CP_STAT__MEQ_BUSY__SHIFT                                                                              0x10
+#define CP_STAT__ME_BUSY__SHIFT                                                                               0x11
+#define CP_STAT__QUERY_BUSY__SHIFT                                                                            0x12
+#define CP_STAT__SEMAPHORE_BUSY__SHIFT                                                                        0x13
+#define CP_STAT__INTERRUPT_BUSY__SHIFT                                                                        0x14
+#define CP_STAT__SURFACE_SYNC_BUSY__SHIFT                                                                     0x15
+#define CP_STAT__DMA_BUSY__SHIFT                                                                              0x16
+#define CP_STAT__RCIU_BUSY__SHIFT                                                                             0x17
+#define CP_STAT__SCRATCH_RAM_BUSY__SHIFT                                                                      0x18
+#define CP_STAT__GCRIU_BUSY__SHIFT                                                                            0x19
+#define CP_STAT__CE_BUSY__SHIFT                                                                               0x1a
+#define CP_STAT__TCIU_BUSY__SHIFT                                                                             0x1b
+#define CP_STAT__ROQ_CE_RING_BUSY__SHIFT                                                                      0x1c
+#define CP_STAT__ROQ_CE_INDIRECT1_BUSY__SHIFT                                                                 0x1d
+#define CP_STAT__ROQ_CE_INDIRECT2_BUSY__SHIFT                                                                 0x1e
+#define CP_STAT__CP_BUSY__SHIFT                                                                               0x1f
+#define CP_STAT__ROQ_DB_BUSY_MASK                                                                             0x00000020L
+#define CP_STAT__ROQ_CE_DB_BUSY_MASK                                                                          0x00000040L
+#define CP_STAT__ROQ_RING_BUSY_MASK                                                                           0x00000200L
+#define CP_STAT__ROQ_INDIRECT1_BUSY_MASK                                                                      0x00000400L
+#define CP_STAT__ROQ_INDIRECT2_BUSY_MASK                                                                      0x00000800L
+#define CP_STAT__ROQ_STATE_BUSY_MASK                                                                          0x00001000L
+#define CP_STAT__DC_BUSY_MASK                                                                                 0x00002000L
+#define CP_STAT__UTCL2IU_BUSY_MASK                                                                            0x00004000L
+#define CP_STAT__PFP_BUSY_MASK                                                                                0x00008000L
+#define CP_STAT__MEQ_BUSY_MASK                                                                                0x00010000L
+#define CP_STAT__ME_BUSY_MASK                                                                                 0x00020000L
+#define CP_STAT__QUERY_BUSY_MASK                                                                              0x00040000L
+#define CP_STAT__SEMAPHORE_BUSY_MASK                                                                          0x00080000L
+#define CP_STAT__INTERRUPT_BUSY_MASK                                                                          0x00100000L
+#define CP_STAT__SURFACE_SYNC_BUSY_MASK                                                                       0x00200000L
+#define CP_STAT__DMA_BUSY_MASK                                                                                0x00400000L
+#define CP_STAT__RCIU_BUSY_MASK                                                                               0x00800000L
+#define CP_STAT__SCRATCH_RAM_BUSY_MASK                                                                        0x01000000L
+#define CP_STAT__GCRIU_BUSY_MASK                                                                              0x02000000L
+#define CP_STAT__CE_BUSY_MASK                                                                                 0x04000000L
+#define CP_STAT__TCIU_BUSY_MASK                                                                               0x08000000L
+#define CP_STAT__ROQ_CE_RING_BUSY_MASK                                                                        0x10000000L
+#define CP_STAT__ROQ_CE_INDIRECT1_BUSY_MASK                                                                   0x20000000L
+#define CP_STAT__ROQ_CE_INDIRECT2_BUSY_MASK                                                                   0x40000000L
+#define CP_STAT__CP_BUSY_MASK                                                                                 0x80000000L
+//CP_ME_HEADER_DUMP
+#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP__SHIFT                                                              0x0
+#define CP_ME_HEADER_DUMP__ME_HEADER_DUMP_MASK                                                                0xFFFFFFFFL
+//CP_PFP_HEADER_DUMP
+#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP__SHIFT                                                            0x0
+#define CP_PFP_HEADER_DUMP__PFP_HEADER_DUMP_MASK                                                              0xFFFFFFFFL
+//CP_GRBM_FREE_COUNT
+#define CP_GRBM_FREE_COUNT__FREE_COUNT__SHIFT                                                                 0x0
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS__SHIFT                                                             0x8
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP__SHIFT                                                             0x10
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_MASK                                                                   0x0000003FL
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_GDS_MASK                                                               0x00003F00L
+#define CP_GRBM_FREE_COUNT__FREE_COUNT_PFP_MASK                                                               0x003F0000L
+//CP_PFP_INSTR_PNTR
+#define CP_PFP_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                  0x0
+#define CP_PFP_INSTR_PNTR__INSTR_PNTR_MASK                                                                    0x0000FFFFL
+//CP_ME_INSTR_PNTR
+#define CP_ME_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                   0x0
+#define CP_ME_INSTR_PNTR__INSTR_PNTR_MASK                                                                     0x0000FFFFL
+//CP_MEC1_INSTR_PNTR
+#define CP_MEC1_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                 0x0
+#define CP_MEC1_INSTR_PNTR__INSTR_PNTR_MASK                                                                   0x0000FFFFL
+//CP_MEC2_INSTR_PNTR
+#define CP_MEC2_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                 0x0
+#define CP_MEC2_INSTR_PNTR__INSTR_PNTR_MASK                                                                   0x0000FFFFL
+//CP_CSF_STAT
+#define CP_CSF_STAT__BUFFER_REQUEST_COUNT__SHIFT                                                              0x8
+#define CP_CSF_STAT__BUFFER_REQUEST_COUNT_MASK                                                                0x0001FF00L
+//CP_CNTX_STAT
+#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS__SHIFT                                                             0x0
+#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT__SHIFT                                                             0x8
+#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS__SHIFT                                                              0x14
+#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT__SHIFT                                                              0x1c
+#define CP_CNTX_STAT__ACTIVE_HP3D_CONTEXTS_MASK                                                               0x000000FFL
+#define CP_CNTX_STAT__CURRENT_HP3D_CONTEXT_MASK                                                               0x00000700L
+#define CP_CNTX_STAT__ACTIVE_GFX_CONTEXTS_MASK                                                                0x0FF00000L
+#define CP_CNTX_STAT__CURRENT_GFX_CONTEXT_MASK                                                                0x70000000L
+//CP_ME_PREEMPTION
+#define CP_ME_PREEMPTION__OBSOLETE__SHIFT                                                                     0x0
+#define CP_ME_PREEMPTION__OBSOLETE_MASK                                                                       0x00000001L
+//CP_RB1_RPTR
+#define CP_RB1_RPTR__RB_RPTR__SHIFT                                                                           0x0
+#define CP_RB1_RPTR__RB_RPTR_MASK                                                                             0x000FFFFFL
+//CP_RB0_RPTR
+#define CP_RB0_RPTR__RB_RPTR__SHIFT                                                                           0x0
+#define CP_RB0_RPTR__RB_RPTR_MASK                                                                             0x000FFFFFL
+//CP_RB_RPTR
+#define CP_RB_RPTR__RB_RPTR__SHIFT                                                                            0x0
+#define CP_RB_RPTR__RB_RPTR_MASK                                                                              0x000FFFFFL
+//CP_RB_WPTR_DELAY
+#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER__SHIFT                                                              0x0
+#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT__SHIFT                                                              0x1c
+#define CP_RB_WPTR_DELAY__PRE_WRITE_TIMER_MASK                                                                0x0FFFFFFFL
+#define CP_RB_WPTR_DELAY__PRE_WRITE_LIMIT_MASK                                                                0xF0000000L
+//CP_RB_WPTR_POLL_CNTL
+#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT                                                           0x0
+#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT                                                          0x10
+#define CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY_MASK                                                             0x0000FFFFL
+#define CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK                                                            0xFFFF0000L
+//CP_ROQ1_THRESHOLDS
+#define CP_ROQ1_THRESHOLDS__RB1_START__SHIFT                                                                  0x0
+#define CP_ROQ1_THRESHOLDS__R0_IB1_START__SHIFT                                                               0xa
+#define CP_ROQ1_THRESHOLDS__R1_IB1_START__SHIFT                                                               0x14
+#define CP_ROQ1_THRESHOLDS__RB1_START_MASK                                                                    0x000003FFL
+#define CP_ROQ1_THRESHOLDS__R0_IB1_START_MASK                                                                 0x000FFC00L
+#define CP_ROQ1_THRESHOLDS__R1_IB1_START_MASK                                                                 0x3FF00000L
+//CP_ROQ2_THRESHOLDS
+#define CP_ROQ2_THRESHOLDS__R0_IB2_START__SHIFT                                                               0x0
+#define CP_ROQ2_THRESHOLDS__R1_IB2_START__SHIFT                                                               0xa
+#define CP_ROQ2_THRESHOLDS__R0_IB2_START_MASK                                                                 0x000003FFL
+#define CP_ROQ2_THRESHOLDS__R1_IB2_START_MASK                                                                 0x000FFC00L
+//CP_STQ_THRESHOLDS
+#define CP_STQ_THRESHOLDS__STQ0_START__SHIFT                                                                  0x0
+#define CP_STQ_THRESHOLDS__STQ1_START__SHIFT                                                                  0x8
+#define CP_STQ_THRESHOLDS__STQ2_START__SHIFT                                                                  0x10
+#define CP_STQ_THRESHOLDS__STQ0_START_MASK                                                                    0x000000FFL
+#define CP_STQ_THRESHOLDS__STQ1_START_MASK                                                                    0x0000FF00L
+#define CP_STQ_THRESHOLDS__STQ2_START_MASK                                                                    0x00FF0000L
+//CP_MEQ_THRESHOLDS
+#define CP_MEQ_THRESHOLDS__MEQ1_START__SHIFT                                                                  0x0
+#define CP_MEQ_THRESHOLDS__MEQ2_START__SHIFT                                                                  0x8
+#define CP_MEQ_THRESHOLDS__MEQ1_START_MASK                                                                    0x000000FFL
+#define CP_MEQ_THRESHOLDS__MEQ2_START_MASK                                                                    0x0000FF00L
+//CP_ROQ_AVAIL
+#define CP_ROQ_AVAIL__ROQ_CNT_RING__SHIFT                                                                     0x0
+#define CP_ROQ_AVAIL__ROQ_CNT_IB1__SHIFT                                                                      0x10
+#define CP_ROQ_AVAIL__ROQ_CNT_RING_MASK                                                                       0x00000FFFL
+#define CP_ROQ_AVAIL__ROQ_CNT_IB1_MASK                                                                        0x0FFF0000L
+//CP_STQ_AVAIL
+#define CP_STQ_AVAIL__STQ_CNT__SHIFT                                                                          0x0
+#define CP_STQ_AVAIL__STQ_CNT_MASK                                                                            0x000001FFL
+//CP_ROQ2_AVAIL
+#define CP_ROQ2_AVAIL__ROQ_CNT_IB2__SHIFT                                                                     0x0
+#define CP_ROQ2_AVAIL__ROQ_CNT_DB__SHIFT                                                                      0x10
+#define CP_ROQ2_AVAIL__ROQ_CNT_IB2_MASK                                                                       0x00000FFFL
+#define CP_ROQ2_AVAIL__ROQ_CNT_DB_MASK                                                                        0x0FFF0000L
+//CP_MEQ_AVAIL
+#define CP_MEQ_AVAIL__MEQ_CNT__SHIFT                                                                          0x0
+#define CP_MEQ_AVAIL__MEQ_CNT_MASK                                                                            0x000003FFL
+//CP_CMD_INDEX
+#define CP_CMD_INDEX__CMD_INDEX__SHIFT                                                                        0x0
+#define CP_CMD_INDEX__CMD_ME_SEL__SHIFT                                                                       0xc
+#define CP_CMD_INDEX__CMD_QUEUE_SEL__SHIFT                                                                    0x10
+#define CP_CMD_INDEX__CMD_INDEX_MASK                                                                          0x000007FFL
+#define CP_CMD_INDEX__CMD_ME_SEL_MASK                                                                         0x00003000L
+#define CP_CMD_INDEX__CMD_QUEUE_SEL_MASK                                                                      0x00070000L
+//CP_CMD_DATA
+#define CP_CMD_DATA__CMD_DATA__SHIFT                                                                          0x0
+#define CP_CMD_DATA__CMD_DATA_MASK                                                                            0xFFFFFFFFL
+//CP_ROQ_RB_STAT
+#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY__SHIFT                                                               0x0
+#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY__SHIFT                                                               0x10
+#define CP_ROQ_RB_STAT__ROQ_RPTR_PRIMARY_MASK                                                                 0x00000FFFL
+#define CP_ROQ_RB_STAT__ROQ_WPTR_PRIMARY_MASK                                                                 0x0FFF0000L
+//CP_ROQ_IB1_STAT
+#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1__SHIFT                                                            0x0
+#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1__SHIFT                                                            0x10
+#define CP_ROQ_IB1_STAT__ROQ_RPTR_INDIRECT1_MASK                                                              0x00000FFFL
+#define CP_ROQ_IB1_STAT__ROQ_WPTR_INDIRECT1_MASK                                                              0x0FFF0000L
+//CP_ROQ_IB2_STAT
+#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2__SHIFT                                                            0x0
+#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2__SHIFT                                                            0x10
+#define CP_ROQ_IB2_STAT__ROQ_RPTR_INDIRECT2_MASK                                                              0x00000FFFL
+#define CP_ROQ_IB2_STAT__ROQ_WPTR_INDIRECT2_MASK                                                              0x0FFF0000L
+//CP_STQ_STAT
+#define CP_STQ_STAT__STQ_RPTR__SHIFT                                                                          0x0
+#define CP_STQ_STAT__STQ_RPTR_MASK                                                                            0x000003FFL
+//CP_STQ_WR_STAT
+#define CP_STQ_WR_STAT__STQ_WPTR__SHIFT                                                                       0x0
+#define CP_STQ_WR_STAT__STQ_WPTR_MASK                                                                         0x000003FFL
+//CP_MEQ_STAT
+#define CP_MEQ_STAT__MEQ_RPTR__SHIFT                                                                          0x0
+#define CP_MEQ_STAT__MEQ_WPTR__SHIFT                                                                          0x10
+#define CP_MEQ_STAT__MEQ_RPTR_MASK                                                                            0x000003FFL
+#define CP_MEQ_STAT__MEQ_WPTR_MASK                                                                            0x03FF0000L
+//CP_ROQ3_THRESHOLDS
+#define CP_ROQ3_THRESHOLDS__R0_DB_START__SHIFT                                                                0x0
+#define CP_ROQ3_THRESHOLDS__R1_DB_START__SHIFT                                                                0xa
+#define CP_ROQ3_THRESHOLDS__R0_DB_START_MASK                                                                  0x000003FFL
+#define CP_ROQ3_THRESHOLDS__R1_DB_START_MASK                                                                  0x000FFC00L
+//CP_ROQ_DB_STAT
+#define CP_ROQ_DB_STAT__ROQ_RPTR_DB__SHIFT                                                                    0x0
+#define CP_ROQ_DB_STAT__ROQ_WPTR_DB__SHIFT                                                                    0x10
+#define CP_ROQ_DB_STAT__ROQ_RPTR_DB_MASK                                                                      0x00000FFFL
+#define CP_ROQ_DB_STAT__ROQ_WPTR_DB_MASK                                                                      0x0FFF0000L
+//CP_INT_STAT_DEBUG
+#define CP_INT_STAT_DEBUG__RESUME_INT_ASSERTED__SHIFT                                                         0x8
+#define CP_INT_STAT_DEBUG__SUSPEND_INT_ASSERTED__SHIFT                                                        0x9
+#define CP_INT_STAT_DEBUG__DMA_WATCH_INT_ASSERTED__SHIFT                                                      0xa
+#define CP_INT_STAT_DEBUG__CP_VM_DOORBELL_WR_INT_ASSERTED__SHIFT                                              0xb
+#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                                   0xe
+#define CP_INT_STAT_DEBUG__FUE_INT_STATUS_DEBUG__SHIFT                                                        0xf
+#define CP_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                            0x10
+#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                               0x11
+#define CP_INT_STAT_DEBUG__CMP_BUSY_INT_ASSERTED__SHIFT                                                       0x12
+#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED__SHIFT                                                      0x13
+#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED__SHIFT                                                     0x14
+#define CP_INT_STAT_DEBUG__GFX_IDLE_INT_ASSERTED__SHIFT                                                       0x15
+#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED__SHIFT                                                     0x16
+#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                       0x17
+#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                                   0x18
+#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                     0x1a
+#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                             0x1b
+#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                       0x1d
+#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                       0x1e
+#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                       0x1f
+#define CP_INT_STAT_DEBUG__RESUME_INT_ASSERTED_MASK                                                           0x00000100L
+#define CP_INT_STAT_DEBUG__SUSPEND_INT_ASSERTED_MASK                                                          0x00000200L
+#define CP_INT_STAT_DEBUG__DMA_WATCH_INT_ASSERTED_MASK                                                        0x00000400L
+#define CP_INT_STAT_DEBUG__CP_VM_DOORBELL_WR_INT_ASSERTED_MASK                                                0x00000800L
+#define CP_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                     0x00004000L
+#define CP_INT_STAT_DEBUG__FUE_INT_STATUS_DEBUG_MASK                                                          0x00008000L
+#define CP_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                              0x00010000L
+#define CP_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                                 0x00020000L
+#define CP_INT_STAT_DEBUG__CMP_BUSY_INT_ASSERTED_MASK                                                         0x00040000L
+#define CP_INT_STAT_DEBUG__CNTX_BUSY_INT_ASSERTED_MASK                                                        0x00080000L
+#define CP_INT_STAT_DEBUG__CNTX_EMPTY_INT_ASSERTED_MASK                                                       0x00100000L
+#define CP_INT_STAT_DEBUG__GFX_IDLE_INT_ASSERTED_MASK                                                         0x00200000L
+#define CP_INT_STAT_DEBUG__PRIV_INSTR_INT_ASSERTED_MASK                                                       0x00400000L
+#define CP_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                         0x00800000L
+#define CP_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                     0x01000000L
+#define CP_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                       0x04000000L
+#define CP_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                               0x08000000L
+#define CP_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                         0x20000000L
+#define CP_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                         0x40000000L
+#define CP_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                         0x80000000L
+//CP_DEBUG_CNTL
+#define CP_DEBUG_CNTL__DEBUG_INDX__SHIFT                                                                      0x0
+#define CP_DEBUG_CNTL__DEBUG_INDX_MASK                                                                        0x0000007FL
+//CP_PRIV_VIOLATION_ADDR
+#define CP_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR__SHIFT                                                    0x0
+#define CP_PRIV_VIOLATION_ADDR__PRIV_VIOLATION_ADDR_MASK                                                      0x0003FFFFL
+
+
+// addressBlock: gc_padec
+//VGT_DMA_DATA_FIFO_DEPTH
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH__SHIFT                                                   0x0
+#define VGT_DMA_DATA_FIFO_DEPTH__DMA_DATA_FIFO_DEPTH_MASK                                                     0x000003FFL
+//VGT_DMA_REQ_FIFO_DEPTH
+#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH__SHIFT                                                     0x0
+#define VGT_DMA_REQ_FIFO_DEPTH__DMA_REQ_FIFO_DEPTH_MASK                                                       0x0000003FL
+//VGT_DRAW_INIT_FIFO_DEPTH
+#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH__SHIFT                                                 0x0
+#define VGT_DRAW_INIT_FIFO_DEPTH__DRAW_INIT_FIFO_DEPTH_MASK                                                   0x0000003FL
+//VGT_MC_LAT_CNTL
+#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES__SHIFT                                                             0x0
+#define VGT_MC_LAT_CNTL__MC_TIME_STAMP_RES_MASK                                                               0x0000000FL
+//IA_UTCL1_STATUS_2
+#define IA_UTCL1_STATUS_2__IA_BUSY__SHIFT                                                                     0x0
+#define IA_UTCL1_STATUS_2__IA_DMA_BUSY__SHIFT                                                                 0x1
+#define IA_UTCL1_STATUS_2__IA_DMA_REQ_BUSY__SHIFT                                                             0x2
+#define IA_UTCL1_STATUS_2__IA_GRP_BUSY__SHIFT                                                                 0x3
+#define IA_UTCL1_STATUS_2__IA_ADC_BUSY__SHIFT                                                                 0x4
+#define IA_UTCL1_STATUS_2__FAULT_DETECTED__SHIFT                                                              0x5
+#define IA_UTCL1_STATUS_2__RETRY_DETECTED__SHIFT                                                              0x6
+#define IA_UTCL1_STATUS_2__PRT_DETECTED__SHIFT                                                                0x7
+#define IA_UTCL1_STATUS_2__FAULT_UTCL1ID__SHIFT                                                               0x8
+#define IA_UTCL1_STATUS_2__RETRY_UTCL1ID__SHIFT                                                               0x10
+#define IA_UTCL1_STATUS_2__PRT_UTCL1ID__SHIFT                                                                 0x18
+#define IA_UTCL1_STATUS_2__IA_BUSY_MASK                                                                       0x00000001L
+#define IA_UTCL1_STATUS_2__IA_DMA_BUSY_MASK                                                                   0x00000002L
+#define IA_UTCL1_STATUS_2__IA_DMA_REQ_BUSY_MASK                                                               0x00000004L
+#define IA_UTCL1_STATUS_2__IA_GRP_BUSY_MASK                                                                   0x00000008L
+#define IA_UTCL1_STATUS_2__IA_ADC_BUSY_MASK                                                                   0x00000010L
+#define IA_UTCL1_STATUS_2__FAULT_DETECTED_MASK                                                                0x00000020L
+#define IA_UTCL1_STATUS_2__RETRY_DETECTED_MASK                                                                0x00000040L
+#define IA_UTCL1_STATUS_2__PRT_DETECTED_MASK                                                                  0x00000080L
+#define IA_UTCL1_STATUS_2__FAULT_UTCL1ID_MASK                                                                 0x00003F00L
+#define IA_UTCL1_STATUS_2__RETRY_UTCL1ID_MASK                                                                 0x003F0000L
+#define IA_UTCL1_STATUS_2__PRT_UTCL1ID_MASK                                                                   0x3F000000L
+//WD_CNTL_STATUS
+#define WD_CNTL_STATUS__DIST_BUSY__SHIFT                                                                      0x0
+#define WD_CNTL_STATUS__DIST_BE_BUSY__SHIFT                                                                   0x1
+#define WD_CNTL_STATUS__GE_UTCL1_BUSY__SHIFT                                                                  0x2
+#define WD_CNTL_STATUS__WD_TE11_BUSY__SHIFT                                                                   0x3
+#define WD_CNTL_STATUS__PC_MANAGER_BUSY__SHIFT                                                                0x4
+#define WD_CNTL_STATUS__WLC_BUSY__SHIFT                                                                       0x5
+#define WD_CNTL_STATUS__DIST_BUSY_MASK                                                                        0x00000001L
+#define WD_CNTL_STATUS__DIST_BE_BUSY_MASK                                                                     0x00000002L
+#define WD_CNTL_STATUS__GE_UTCL1_BUSY_MASK                                                                    0x00000004L
+#define WD_CNTL_STATUS__WD_TE11_BUSY_MASK                                                                     0x00000008L
+#define WD_CNTL_STATUS__PC_MANAGER_BUSY_MASK                                                                  0x00000010L
+#define WD_CNTL_STATUS__WLC_BUSY_MASK                                                                         0x00000020L
+//CC_GC_PRIM_CONFIG
+#define CC_GC_PRIM_CONFIG__WRITE_DIS__SHIFT                                                                   0x0
+#define CC_GC_PRIM_CONFIG__INACTIVE_PA__SHIFT                                                                 0x4
+#define CC_GC_PRIM_CONFIG__WRITE_DIS_MASK                                                                     0x00000001L
+#define CC_GC_PRIM_CONFIG__INACTIVE_PA_MASK                                                                   0x000FFFF0L
+//WD_QOS
+#define WD_QOS__DRAW_STALL__SHIFT                                                                             0x0
+#define WD_QOS__DRAW_STALL_MASK                                                                               0x00000001L
+//WD_UTCL1_CNTL
+#define WD_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                            0x0
+#define WD_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                 0x17
+#define WD_UTCL1_CNTL__DROP_MODE__SHIFT                                                                       0x18
+#define WD_UTCL1_CNTL__BYPASS__SHIFT                                                                          0x19
+#define WD_UTCL1_CNTL__INVALIDATE__SHIFT                                                                      0x1a
+#define WD_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                 0x1b
+#define WD_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                     0x1c
+#define WD_UTCL1_CNTL__MTYPE_OVERRIDE__SHIFT                                                                  0x1d
+#define WD_UTCL1_CNTL__LLC_NOALLOC_OVERRIDE__SHIFT                                                            0x1e
+#define WD_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                              0x000FFFFFL
+#define WD_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                   0x00800000L
+#define WD_UTCL1_CNTL__DROP_MODE_MASK                                                                         0x01000000L
+#define WD_UTCL1_CNTL__BYPASS_MASK                                                                            0x02000000L
+#define WD_UTCL1_CNTL__INVALIDATE_MASK                                                                        0x04000000L
+#define WD_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                   0x08000000L
+#define WD_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                       0x10000000L
+#define WD_UTCL1_CNTL__MTYPE_OVERRIDE_MASK                                                                    0x20000000L
+#define WD_UTCL1_CNTL__LLC_NOALLOC_OVERRIDE_MASK                                                              0x40000000L
+//WD_UTCL1_STATUS
+#define WD_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                                0x0
+#define WD_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                                0x1
+#define WD_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                  0x2
+#define WD_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                 0x8
+#define WD_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                 0x10
+#define WD_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                   0x18
+#define WD_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                  0x00000001L
+#define WD_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                  0x00000002L
+#define WD_UTCL1_STATUS__PRT_DETECTED_MASK                                                                    0x00000004L
+#define WD_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                   0x00003F00L
+#define WD_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                   0x003F0000L
+#define WD_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                     0x3F000000L
+//IA_UTCL1_CNTL
+#define IA_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                            0x0
+#define IA_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                 0x17
+#define IA_UTCL1_CNTL__DROP_MODE__SHIFT                                                                       0x18
+#define IA_UTCL1_CNTL__BYPASS__SHIFT                                                                          0x19
+#define IA_UTCL1_CNTL__INVALIDATE__SHIFT                                                                      0x1a
+#define IA_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                 0x1b
+#define IA_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                     0x1c
+#define IA_UTCL1_CNTL__MTYPE_OVERRIDE__SHIFT                                                                  0x1d
+#define IA_UTCL1_CNTL__LLC_NOALLOC_OVERRIDE__SHIFT                                                            0x1e
+#define IA_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                              0x000FFFFFL
+#define IA_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                   0x00800000L
+#define IA_UTCL1_CNTL__DROP_MODE_MASK                                                                         0x01000000L
+#define IA_UTCL1_CNTL__BYPASS_MASK                                                                            0x02000000L
+#define IA_UTCL1_CNTL__INVALIDATE_MASK                                                                        0x04000000L
+#define IA_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                   0x08000000L
+#define IA_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                       0x10000000L
+#define IA_UTCL1_CNTL__MTYPE_OVERRIDE_MASK                                                                    0x20000000L
+#define IA_UTCL1_CNTL__LLC_NOALLOC_OVERRIDE_MASK                                                              0x40000000L
+//IA_UTCL1_STATUS
+#define IA_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                                0x0
+#define IA_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                                0x1
+#define IA_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                  0x2
+#define IA_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                 0x8
+#define IA_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                 0x10
+#define IA_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                   0x18
+#define IA_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                  0x00000001L
+#define IA_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                  0x00000002L
+#define IA_UTCL1_STATUS__PRT_DETECTED_MASK                                                                    0x00000004L
+#define IA_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                   0x00003F00L
+#define IA_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                   0x003F0000L
+#define IA_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                     0x3F000000L
+//CC_GC_SA_UNIT_DISABLE
+#define CC_GC_SA_UNIT_DISABLE__WRITE_DIS__SHIFT                                                               0x0
+#define CC_GC_SA_UNIT_DISABLE__SA_DISABLE__SHIFT                                                              0x8
+#define CC_GC_SA_UNIT_DISABLE__WRITE_DIS_MASK                                                                 0x00000001L
+#define CC_GC_SA_UNIT_DISABLE__SA_DISABLE_MASK                                                                0x00FFFF00L
+//GE_RATE_CNTL_1
+#define GE_RATE_CNTL_1__ADD_X_CLKS_LS_VERT__SHIFT                                                             0x0
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_LS_VERT__SHIFT                                                          0x4
+#define GE_RATE_CNTL_1__ADD_X_CLKS_HS_VERT__SHIFT                                                             0x8
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_HS_VERT__SHIFT                                                          0xc
+#define GE_RATE_CNTL_1__ADD_X_CLKS_ES_VERT__SHIFT                                                             0x10
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_ES_VERT__SHIFT                                                          0x14
+#define GE_RATE_CNTL_1__ADD_X_CLKS_GS_PRIM__SHIFT                                                             0x18
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_GS_PRIM__SHIFT                                                          0x1c
+#define GE_RATE_CNTL_1__ADD_X_CLKS_LS_VERT_MASK                                                               0x0000000FL
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_LS_VERT_MASK                                                            0x000000F0L
+#define GE_RATE_CNTL_1__ADD_X_CLKS_HS_VERT_MASK                                                               0x00000F00L
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_HS_VERT_MASK                                                            0x0000F000L
+#define GE_RATE_CNTL_1__ADD_X_CLKS_ES_VERT_MASK                                                               0x000F0000L
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_ES_VERT_MASK                                                            0x00F00000L
+#define GE_RATE_CNTL_1__ADD_X_CLKS_GS_PRIM_MASK                                                               0x0F000000L
+#define GE_RATE_CNTL_1__AFTER_Y_TRANS_GS_PRIM_MASK                                                            0xF0000000L
+//GE_RATE_CNTL_2
+#define GE_RATE_CNTL_2__ADD_X_CLKS_VS_VERT__SHIFT                                                             0x0
+#define GE_RATE_CNTL_2__AFTER_Y_TRANS_VS_VERT__SHIFT                                                          0x4
+#define GE_RATE_CNTL_2__ADD_X_CLKS_PA_PRIM__SHIFT                                                             0x8
+#define GE_RATE_CNTL_2__AFTER_Y_TRANS_PA_PRIM__SHIFT                                                          0xc
+#define GE_RATE_CNTL_2__ADD_X_CLKS_MERGED_HS_GS__SHIFT                                                        0x10
+#define GE_RATE_CNTL_2__ADD_X_CLKS_MERGED_LS_ES__SHIFT                                                        0x14
+#define GE_RATE_CNTL_2__MERGED_HS_GS_MODE__SHIFT                                                              0x18
+#define GE_RATE_CNTL_2__MERGED_LS_ES_MODE__SHIFT                                                              0x19
+#define GE_RATE_CNTL_2__ENABLE_RATE_CNTL__SHIFT                                                               0x1a
+#define GE_RATE_CNTL_2__SWAP_PRIORITY__SHIFT                                                                  0x1b
+#define GE_RATE_CNTL_2__ADD_X_CLKS_VS_VERT_MASK                                                               0x0000000FL
+#define GE_RATE_CNTL_2__AFTER_Y_TRANS_VS_VERT_MASK                                                            0x000000F0L
+#define GE_RATE_CNTL_2__ADD_X_CLKS_PA_PRIM_MASK                                                               0x00000F00L
+#define GE_RATE_CNTL_2__AFTER_Y_TRANS_PA_PRIM_MASK                                                            0x0000F000L
+#define GE_RATE_CNTL_2__ADD_X_CLKS_MERGED_HS_GS_MASK                                                          0x000F0000L
+#define GE_RATE_CNTL_2__ADD_X_CLKS_MERGED_LS_ES_MASK                                                          0x00F00000L
+#define GE_RATE_CNTL_2__MERGED_HS_GS_MODE_MASK                                                                0x01000000L
+#define GE_RATE_CNTL_2__MERGED_LS_ES_MODE_MASK                                                                0x02000000L
+#define GE_RATE_CNTL_2__ENABLE_RATE_CNTL_MASK                                                                 0x04000000L
+#define GE_RATE_CNTL_2__SWAP_PRIORITY_MASK                                                                    0x08000000L
+//VGT_SYS_CONFIG
+#define VGT_SYS_CONFIG__DUAL_CORE_EN__SHIFT                                                                   0x0
+#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP__SHIFT                                                               0x1
+#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE__SHIFT                                                       0x7
+#define VGT_SYS_CONFIG__NUM_SUBGROUPS_IN_FLIGHT__SHIFT                                                        0x8
+#define VGT_SYS_CONFIG__DUAL_CORE_EN_MASK                                                                     0x00000001L
+#define VGT_SYS_CONFIG__MAX_LS_HS_THDGRP_MASK                                                                 0x0000007EL
+#define VGT_SYS_CONFIG__ADC_EVENT_FILTER_DISABLE_MASK                                                         0x00000080L
+#define VGT_SYS_CONFIG__NUM_SUBGROUPS_IN_FLIGHT_MASK                                                          0x0007FF00L
+//GE_PRIV_CONTROL
+#define GE_PRIV_CONTROL__RESERVED__SHIFT                                                                      0x0
+#define GE_PRIV_CONTROL__CLAMP_PRIMGRP_SIZE__SHIFT                                                            0x1
+#define GE_PRIV_CONTROL__RESET_ON_PIPELINE_CHANGE__SHIFT                                                      0xa
+#define GE_PRIV_CONTROL__FGCG_OVERRIDE__SHIFT                                                                 0xf
+#define GE_PRIV_CONTROL__CLAMP_HS_OFFCHIP_PER_SE_OVERRIDE__SHIFT                                              0x10
+#define GE_PRIV_CONTROL__DISABLE_ACCUM_AGM__SHIFT                                                             0x11
+#define GE_PRIV_CONTROL__RESERVED_MASK                                                                        0x00000001L
+#define GE_PRIV_CONTROL__CLAMP_PRIMGRP_SIZE_MASK                                                              0x000003FEL
+#define GE_PRIV_CONTROL__RESET_ON_PIPELINE_CHANGE_MASK                                                        0x00000400L
+#define GE_PRIV_CONTROL__FGCG_OVERRIDE_MASK                                                                   0x00008000L
+#define GE_PRIV_CONTROL__CLAMP_HS_OFFCHIP_PER_SE_OVERRIDE_MASK                                                0x00010000L
+#define GE_PRIV_CONTROL__DISABLE_ACCUM_AGM_MASK                                                               0x00020000L
+//GE_STATUS
+#define GE_STATUS__PERFCOUNTER_STATUS__SHIFT                                                                  0x0
+#define GE_STATUS__THREAD_TRACE_STATUS__SHIFT                                                                 0x1
+#define GE_STATUS__PERFCOUNTER_STATUS_MASK                                                                    0x00000001L
+#define GE_STATUS__THREAD_TRACE_STATUS_MASK                                                                   0x00000002L
+//VGT_GS_MAX_WAVE_ID
+#define VGT_GS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                                0x0
+#define VGT_GS_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                                  0x00000FFFL
+//GFX_PIPE_CONTROL
+#define GFX_PIPE_CONTROL__HYSTERESIS_CNT__SHIFT                                                               0x0
+#define GFX_PIPE_CONTROL__RESERVED__SHIFT                                                                     0xd
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN__SHIFT                                                           0x10
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_STALL_EN__SHIFT                                                     0x11
+#define GFX_PIPE_CONTROL__HYSTERESIS_CNT_MASK                                                                 0x00001FFFL
+#define GFX_PIPE_CONTROL__RESERVED_MASK                                                                       0x0000E000L
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_EN_MASK                                                             0x00010000L
+#define GFX_PIPE_CONTROL__CONTEXT_SUSPEND_STALL_EN_MASK                                                       0x00020000L
+//CC_GC_SHADER_ARRAY_CONFIG
+#define CC_GC_SHADER_ARRAY_CONFIG__WRITE_DIS__SHIFT                                                           0x0
+#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT                                                       0x10
+#define CC_GC_SHADER_ARRAY_CONFIG__WRITE_DIS_MASK                                                             0x00000001L
+#define CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK                                                         0xFFFF0000L
+//GE2_SE_CNTL_STATUS
+#define GE2_SE_CNTL_STATUS__TE_BUSY__SHIFT                                                                    0x0
+#define GE2_SE_CNTL_STATUS__NGG_BUSY__SHIFT                                                                   0x1
+#define GE2_SE_CNTL_STATUS__HS_BUSY__SHIFT                                                                    0x2
+#define GE2_SE_CNTL_STATUS__TE_BUSY_MASK                                                                      0x00000001L
+#define GE2_SE_CNTL_STATUS__NGG_BUSY_MASK                                                                     0x00000002L
+#define GE2_SE_CNTL_STATUS__HS_BUSY_MASK                                                                      0x00000004L
+//VGT_RESET_DEBUG
+#define VGT_RESET_DEBUG__GS_DISABLE__SHIFT                                                                    0x0
+#define VGT_RESET_DEBUG__TESS_DISABLE__SHIFT                                                                  0x1
+#define VGT_RESET_DEBUG__WD_DISABLE__SHIFT                                                                    0x2
+#define VGT_RESET_DEBUG__DISABLE_TE11_DIST_PIPE0__SHIFT                                                       0x3
+#define VGT_RESET_DEBUG__DISABLE_TE11_DIST_PIPE1__SHIFT                                                       0x4
+#define VGT_RESET_DEBUG__ENABLE_VMID_RESET_UTCL1__SHIFT                                                       0x5
+#define VGT_RESET_DEBUG__DISABLE_PREFETCH__SHIFT                                                              0x6
+#define VGT_RESET_DEBUG__DISABLE_SWITCH_MODE_STALL_FIX__SHIFT                                                 0x7
+#define VGT_RESET_DEBUG__DISABLE_SENDING_MULTIPLE_SE_IN_PD__SHIFT                                             0x8
+#define VGT_RESET_DEBUG__ENABLE_DIST_STALL_TESS_ON_OFF__SHIFT                                                 0x9
+#define VGT_RESET_DEBUG__DISABLE_PATCH_OPTIMIZATION__SHIFT                                                    0xa
+#define VGT_RESET_DEBUG__ENABLE_DIST_STALL_TESS_OFF_ON__SHIFT                                                 0xb
+#define VGT_RESET_DEBUG__DISABLE_MERGE_GRP_PERF_FIX__SHIFT                                                    0xc
+#define VGT_RESET_DEBUG__DISABLE_MESH_SHADER_ATTR_PACKING__SHIFT                                              0xd
+#define VGT_RESET_DEBUG__ENABLE_SMALL_INST_PACK_ADJ_GS_OFF__SHIFT                                             0xe
+#define VGT_RESET_DEBUG__DISABLE_PATCH_DIST_LAST_DONUT_SE_SWITCH_LOGIC__SHIFT                                 0xf
+#define VGT_RESET_DEBUG__SPARE__SHIFT                                                                         0x10
+#define VGT_RESET_DEBUG__GS_DISABLE_MASK                                                                      0x00000001L
+#define VGT_RESET_DEBUG__TESS_DISABLE_MASK                                                                    0x00000002L
+#define VGT_RESET_DEBUG__WD_DISABLE_MASK                                                                      0x00000004L
+#define VGT_RESET_DEBUG__DISABLE_TE11_DIST_PIPE0_MASK                                                         0x00000008L
+#define VGT_RESET_DEBUG__DISABLE_TE11_DIST_PIPE1_MASK                                                         0x00000010L
+#define VGT_RESET_DEBUG__ENABLE_VMID_RESET_UTCL1_MASK                                                         0x00000020L
+#define VGT_RESET_DEBUG__DISABLE_PREFETCH_MASK                                                                0x00000040L
+#define VGT_RESET_DEBUG__DISABLE_SWITCH_MODE_STALL_FIX_MASK                                                   0x00000080L
+#define VGT_RESET_DEBUG__DISABLE_SENDING_MULTIPLE_SE_IN_PD_MASK                                               0x00000100L
+#define VGT_RESET_DEBUG__ENABLE_DIST_STALL_TESS_ON_OFF_MASK                                                   0x00000200L
+#define VGT_RESET_DEBUG__DISABLE_PATCH_OPTIMIZATION_MASK                                                      0x00000400L
+#define VGT_RESET_DEBUG__ENABLE_DIST_STALL_TESS_OFF_ON_MASK                                                   0x00000800L
+#define VGT_RESET_DEBUG__DISABLE_MERGE_GRP_PERF_FIX_MASK                                                      0x00001000L
+#define VGT_RESET_DEBUG__DISABLE_MESH_SHADER_ATTR_PACKING_MASK                                                0x00002000L
+#define VGT_RESET_DEBUG__ENABLE_SMALL_INST_PACK_ADJ_GS_OFF_MASK                                               0x00004000L
+#define VGT_RESET_DEBUG__DISABLE_PATCH_DIST_LAST_DONUT_SE_SWITCH_LOGIC_MASK                                   0x00008000L
+#define VGT_RESET_DEBUG__SPARE_MASK                                                                           0xFFFF0000L
+//GE_SPI_IF_SAFE_REG
+#define GE_SPI_IF_SAFE_REG__GE_SPI_LS_ES_DATA__SHIFT                                                          0x0
+#define GE_SPI_IF_SAFE_REG__GE_SPI_HS_GS_DATA__SHIFT                                                          0x6
+#define GE_SPI_IF_SAFE_REG__GE_SPI_GRP__SHIFT                                                                 0xc
+#define GE_SPI_IF_SAFE_REG__GE_SPI_LS_ES_DATA_MASK                                                            0x0000003FL
+#define GE_SPI_IF_SAFE_REG__GE_SPI_HS_GS_DATA_MASK                                                            0x00000FC0L
+#define GE_SPI_IF_SAFE_REG__GE_SPI_GRP_MASK                                                                   0x0003F000L
+//GE_PA_IF_SAFE_REG
+#define GE_PA_IF_SAFE_REG__GE_PA_CSB__SHIFT                                                                   0x0
+#define GE_PA_IF_SAFE_REG__GE_PA_PAYLOAD__SHIFT                                                               0xa
+#define GE_PA_IF_SAFE_REG__GE_PA_CSB_MASK                                                                     0x000003FFL
+#define GE_PA_IF_SAFE_REG__GE_PA_PAYLOAD_MASK                                                                 0x000FFC00L
+//PA_CL_CNTL_STATUS
+#define PA_CL_CNTL_STATUS__CL_BUSY__SHIFT                                                                     0x1f
+#define PA_CL_CNTL_STATUS__CL_BUSY_MASK                                                                       0x80000000L
+//PA_CL_ENHANCE
+#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA__SHIFT                                                            0x0
+#define PA_CL_ENHANCE__NUM_CLIP_SEQ__SHIFT                                                                    0x1
+#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL__SHIFT                                                          0x3
+#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE__SHIFT                                                             0x4
+#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL__SHIFT                                                              0x5
+#define PA_CL_ENHANCE__IGNORE_PIPELINE_RESET__SHIFT                                                           0x6
+#define PA_CL_ENHANCE__KILL_INNER_EDGE_FLAGS__SHIFT                                                           0x7
+#define PA_CL_ENHANCE__NGG_PA_TO_ALL_SC__SHIFT                                                                0x8
+#define PA_CL_ENHANCE__TC_LATENCY_TIME_STAMP_RESOLUTION__SHIFT                                                0x9
+#define PA_CL_ENHANCE__NGG_BYPASS_PRIM_FILTER__SHIFT                                                          0xb
+#define PA_CL_ENHANCE__NGG_SIDEBAND_MEMORY_DEPTH__SHIFT                                                       0xc
+#define PA_CL_ENHANCE__NGG_PRIM_INDICES_FIFO_DEPTH__SHIFT                                                     0xe
+#define PA_CL_ENHANCE__PROG_NEAR_CLIP_PLANE_ENABLE__SHIFT                                                     0x11
+#define PA_CL_ENHANCE__POLY_INNER_EDGE_FLAG_DISABLE__SHIFT                                                    0x12
+#define PA_CL_ENHANCE__TC_REQUEST_PERF_CNTR_ENABLE__SHIFT                                                     0x13
+#define PA_CL_ENHANCE__DISABLE_PA_PH_INTF_FINE_CLOCK_GATE__SHIFT                                              0x14
+#define PA_CL_ENHANCE__DISABLE_PA_SX_REQ_INTF_FINE_CLOCK_GATE__SHIFT                                          0x15
+#define PA_CL_ENHANCE__ENABLE_PA_RATE_CNTL__SHIFT                                                             0x16
+#define PA_CL_ENHANCE__CLAMP_NEGATIVE_BB_TO_ZERO__SHIFT                                                       0x17
+#define PA_CL_ENHANCE__ECO_SPARE3__SHIFT                                                                      0x1c
+#define PA_CL_ENHANCE__ECO_SPARE2__SHIFT                                                                      0x1d
+#define PA_CL_ENHANCE__ECO_SPARE1__SHIFT                                                                      0x1e
+#define PA_CL_ENHANCE__ECO_SPARE0__SHIFT                                                                      0x1f
+#define PA_CL_ENHANCE__CLIP_VTX_REORDER_ENA_MASK                                                              0x00000001L
+#define PA_CL_ENHANCE__NUM_CLIP_SEQ_MASK                                                                      0x00000006L
+#define PA_CL_ENHANCE__CLIPPED_PRIM_SEQ_STALL_MASK                                                            0x00000008L
+#define PA_CL_ENHANCE__VE_NAN_PROC_DISABLE_MASK                                                               0x00000010L
+#define PA_CL_ENHANCE__XTRA_DEBUG_REG_SEL_MASK                                                                0x00000020L
+#define PA_CL_ENHANCE__IGNORE_PIPELINE_RESET_MASK                                                             0x00000040L
+#define PA_CL_ENHANCE__KILL_INNER_EDGE_FLAGS_MASK                                                             0x00000080L
+#define PA_CL_ENHANCE__NGG_PA_TO_ALL_SC_MASK                                                                  0x00000100L
+#define PA_CL_ENHANCE__TC_LATENCY_TIME_STAMP_RESOLUTION_MASK                                                  0x00000600L
+#define PA_CL_ENHANCE__NGG_BYPASS_PRIM_FILTER_MASK                                                            0x00000800L
+#define PA_CL_ENHANCE__NGG_SIDEBAND_MEMORY_DEPTH_MASK                                                         0x00003000L
+#define PA_CL_ENHANCE__NGG_PRIM_INDICES_FIFO_DEPTH_MASK                                                       0x0001C000L
+#define PA_CL_ENHANCE__PROG_NEAR_CLIP_PLANE_ENABLE_MASK                                                       0x00020000L
+#define PA_CL_ENHANCE__POLY_INNER_EDGE_FLAG_DISABLE_MASK                                                      0x00040000L
+#define PA_CL_ENHANCE__TC_REQUEST_PERF_CNTR_ENABLE_MASK                                                       0x00080000L
+#define PA_CL_ENHANCE__DISABLE_PA_PH_INTF_FINE_CLOCK_GATE_MASK                                                0x00100000L
+#define PA_CL_ENHANCE__DISABLE_PA_SX_REQ_INTF_FINE_CLOCK_GATE_MASK                                            0x00200000L
+#define PA_CL_ENHANCE__ENABLE_PA_RATE_CNTL_MASK                                                               0x00400000L
+#define PA_CL_ENHANCE__CLAMP_NEGATIVE_BB_TO_ZERO_MASK                                                         0x00800000L
+#define PA_CL_ENHANCE__ECO_SPARE3_MASK                                                                        0x10000000L
+#define PA_CL_ENHANCE__ECO_SPARE2_MASK                                                                        0x20000000L
+#define PA_CL_ENHANCE__ECO_SPARE1_MASK                                                                        0x40000000L
+#define PA_CL_ENHANCE__ECO_SPARE0_MASK                                                                        0x80000000L
+//PA_CL_RESET_DEBUG
+#define PA_CL_RESET_DEBUG__CL_TRIV_DISC_DISABLE__SHIFT                                                        0x0
+#define PA_CL_RESET_DEBUG__CL_TRIV_DISC_DISABLE_MASK                                                          0x00000001L
+//PA_SU_CNTL_STATUS
+#define PA_SU_CNTL_STATUS__SU_BUSY__SHIFT                                                                     0x1f
+#define PA_SU_CNTL_STATUS__SU_BUSY_MASK                                                                       0x80000000L
+//PA_SC_FIFO_DEPTH_CNTL
+#define PA_SC_FIFO_DEPTH_CNTL__DEPTH__SHIFT                                                                   0x0
+#define PA_SC_FIFO_DEPTH_CNTL__DEPTH_MASK                                                                     0x000003FFL
+
+
+// addressBlock: gc_sqdec
+//SQ_CONFIG
+#define SQ_CONFIG__ECO_SPARE__SHIFT                                                                           0x0
+#define SQ_CONFIG__NEW_TRANS_ARB_SCHEME__SHIFT                                                                0x8
+#define SQ_CONFIG__DISABLE_VMEM_EXEC_ZERO_SKIP__SHIFT                                                         0x9
+#define SQ_CONFIG__DISABLE_SGPR_RD_KILL__SHIFT                                                                0xa
+#define SQ_CONFIG__ENABLE_HIPRIO_ON_EXP_RDY_GS__SHIFT                                                         0x12
+#define SQ_CONFIG__PRIO_VAL_ON_EXP_RDY_GS__SHIFT                                                              0x13
+#define SQ_CONFIG__WCLK_HYSTERESIS_CNT__SHIFT                                                                 0x15
+#define SQ_CONFIG__DISABLE_END_CLAUSE_TX__SHIFT                                                               0x1b
+#define SQ_CONFIG__ECO_SPARE_MASK                                                                             0x000000FFL
+#define SQ_CONFIG__NEW_TRANS_ARB_SCHEME_MASK                                                                  0x00000100L
+#define SQ_CONFIG__DISABLE_VMEM_EXEC_ZERO_SKIP_MASK                                                           0x00000200L
+#define SQ_CONFIG__DISABLE_SGPR_RD_KILL_MASK                                                                  0x00000400L
+#define SQ_CONFIG__ENABLE_HIPRIO_ON_EXP_RDY_GS_MASK                                                           0x00040000L
+#define SQ_CONFIG__PRIO_VAL_ON_EXP_RDY_GS_MASK                                                                0x00180000L
+#define SQ_CONFIG__WCLK_HYSTERESIS_CNT_MASK                                                                   0x00600000L
+#define SQ_CONFIG__DISABLE_END_CLAUSE_TX_MASK                                                                 0x08000000L
+//SQC_CONFIG
+#define SQC_CONFIG__INST_CACHE_SIZE__SHIFT                                                                    0x0
+#define SQC_CONFIG__DATA_CACHE_SIZE__SHIFT                                                                    0x2
+#define SQC_CONFIG__MISS_FIFO_DEPTH__SHIFT                                                                    0x4
+#define SQC_CONFIG__HIT_FIFO_DEPTH__SHIFT                                                                     0x6
+#define SQC_CONFIG__FORCE_ALWAYS_MISS__SHIFT                                                                  0x7
+#define SQC_CONFIG__FORCE_IN_ORDER__SHIFT                                                                     0x8
+#define SQC_CONFIG__PER_VMID_INV_DISABLE__SHIFT                                                               0x9
+#define SQC_CONFIG__EVICT_LRU__SHIFT                                                                          0xa
+#define SQC_CONFIG__FORCE_2_BANK__SHIFT                                                                       0xc
+#define SQC_CONFIG__FORCE_1_BANK__SHIFT                                                                       0xd
+#define SQC_CONFIG__LS_DISABLE_CLOCKS__SHIFT                                                                  0xe
+#define SQC_CONFIG__CACHE_CTRL_GCR_FIX_DISABLE__SHIFT                                                         0x16
+#define SQC_CONFIG__CACHE_CTRL_ALMOST_MAX_INFLIGHT_CONFIG__SHIFT                                              0x17
+#define SQC_CONFIG__SPARE__SHIFT                                                                              0x1a
+#define SQC_CONFIG__INST_CACHE_SIZE_MASK                                                                      0x00000003L
+#define SQC_CONFIG__DATA_CACHE_SIZE_MASK                                                                      0x0000000CL
+#define SQC_CONFIG__MISS_FIFO_DEPTH_MASK                                                                      0x00000030L
+#define SQC_CONFIG__HIT_FIFO_DEPTH_MASK                                                                       0x00000040L
+#define SQC_CONFIG__FORCE_ALWAYS_MISS_MASK                                                                    0x00000080L
+#define SQC_CONFIG__FORCE_IN_ORDER_MASK                                                                       0x00000100L
+#define SQC_CONFIG__PER_VMID_INV_DISABLE_MASK                                                                 0x00000200L
+#define SQC_CONFIG__EVICT_LRU_MASK                                                                            0x00000C00L
+#define SQC_CONFIG__FORCE_2_BANK_MASK                                                                         0x00001000L
+#define SQC_CONFIG__FORCE_1_BANK_MASK                                                                         0x00002000L
+#define SQC_CONFIG__LS_DISABLE_CLOCKS_MASK                                                                    0x003FC000L
+#define SQC_CONFIG__CACHE_CTRL_GCR_FIX_DISABLE_MASK                                                           0x00400000L
+#define SQC_CONFIG__CACHE_CTRL_ALMOST_MAX_INFLIGHT_CONFIG_MASK                                                0x03800000L
+#define SQC_CONFIG__SPARE_MASK                                                                                0xFC000000L
+//LDS_CONFIG
+#define LDS_CONFIG__ADDR_OUT_OF_RANGE_REPORTING__SHIFT                                                        0x0
+#define LDS_CONFIG__CONF_BIT_1__SHIFT                                                                         0x1
+#define LDS_CONFIG__WAVE32_INTERP_DUAL_ISSUE_DISABLE__SHIFT                                                   0x2
+#define LDS_CONFIG__SP_TDDATA_FGCG_OVERRIDE__SHIFT                                                            0x3
+#define LDS_CONFIG__SQC_PERF_FGCG_OVERRIDE__SHIFT                                                             0x4
+#define LDS_CONFIG__CONF_BIT_5__SHIFT                                                                         0x5
+#define LDS_CONFIG__CONF_BIT_6__SHIFT                                                                         0x6
+#define LDS_CONFIG__CONF_BIT_7__SHIFT                                                                         0x7
+#define LDS_CONFIG__CONF_BIT_8__SHIFT                                                                         0x8
+#define LDS_CONFIG__ADDR_OUT_OF_RANGE_REPORTING_MASK                                                          0x00000001L
+#define LDS_CONFIG__CONF_BIT_1_MASK                                                                           0x00000002L
+#define LDS_CONFIG__WAVE32_INTERP_DUAL_ISSUE_DISABLE_MASK                                                     0x00000004L
+#define LDS_CONFIG__SP_TDDATA_FGCG_OVERRIDE_MASK                                                              0x00000008L
+#define LDS_CONFIG__SQC_PERF_FGCG_OVERRIDE_MASK                                                               0x00000010L
+#define LDS_CONFIG__CONF_BIT_5_MASK                                                                           0x00000020L
+#define LDS_CONFIG__CONF_BIT_6_MASK                                                                           0x00000040L
+#define LDS_CONFIG__CONF_BIT_7_MASK                                                                           0x00000080L
+#define LDS_CONFIG__CONF_BIT_8_MASK                                                                           0x00000100L
+//SQ_RANDOM_WAVE_PRI
+#define SQ_RANDOM_WAVE_PRI__RET__SHIFT                                                                        0x0
+#define SQ_RANDOM_WAVE_PRI__RUI__SHIFT                                                                        0x7
+#define SQ_RANDOM_WAVE_PRI__RNG__SHIFT                                                                        0xa
+#define SQ_RANDOM_WAVE_PRI__FORCE_IB_ARB_PRIO_MSK_VALID__SHIFT                                                0x1f
+#define SQ_RANDOM_WAVE_PRI__RET_MASK                                                                          0x0000007FL
+#define SQ_RANDOM_WAVE_PRI__RUI_MASK                                                                          0x00000380L
+#define SQ_RANDOM_WAVE_PRI__RNG_MASK                                                                          0x00FFFC00L
+#define SQ_RANDOM_WAVE_PRI__FORCE_IB_ARB_PRIO_MSK_VALID_MASK                                                  0x80000000L
+//SQG_STATUS
+#define SQG_STATUS__REG_BUSY__SHIFT                                                                           0x0
+#define SQG_STATUS__REG_BUSY_MASK                                                                             0x00000001L
+//SQ_FIFO_SIZES
+#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE__SHIFT                                                             0x0
+#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE__SHIFT                                                                0x8
+#define SQ_FIFO_SIZES__EXPORT_BUF_GS_RESERVED__SHIFT                                                          0xc
+#define SQ_FIFO_SIZES__EXPORT_BUF_PS_RESERVED__SHIFT                                                          0xe
+#define SQ_FIFO_SIZES__EXPORT_BUF_REDUCE__SHIFT                                                               0x10
+#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE__SHIFT                                                             0x12
+#define SQ_FIFO_SIZES__EXPORT_BUF_PRIMPOS_LIMIT__SHIFT                                                        0x14
+#define SQ_FIFO_SIZES__INTERRUPT_FIFO_SIZE_MASK                                                               0x0000000FL
+#define SQ_FIFO_SIZES__TTRACE_FIFO_SIZE_MASK                                                                  0x00000300L
+#define SQ_FIFO_SIZES__EXPORT_BUF_GS_RESERVED_MASK                                                            0x00003000L
+#define SQ_FIFO_SIZES__EXPORT_BUF_PS_RESERVED_MASK                                                            0x0000C000L
+#define SQ_FIFO_SIZES__EXPORT_BUF_REDUCE_MASK                                                                 0x00030000L
+#define SQ_FIFO_SIZES__VMEM_DATA_FIFO_SIZE_MASK                                                               0x000C0000L
+#define SQ_FIFO_SIZES__EXPORT_BUF_PRIMPOS_LIMIT_MASK                                                          0x00300000L
+//SQ_DSM_CNTL
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_0__SHIFT                                                                 0x0
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_1__SHIFT                                                                 0x1
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_0__SHIFT                                                                0x2
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_1__SHIFT                                                                0x3
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA0__SHIFT                                                      0x8
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA1__SHIFT                                                      0x9
+#define SQ_DSM_CNTL__SGPR_ENABLE_SINGLE_WRITE__SHIFT                                                          0xa
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA0__SHIFT                                                       0x10
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA1__SHIFT                                                       0x11
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE01__SHIFT                                                         0x12
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA2__SHIFT                                                       0x13
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA3__SHIFT                                                       0x14
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE23__SHIFT                                                         0x15
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA0__SHIFT                                                        0x18
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA1__SHIFT                                                        0x19
+#define SQ_DSM_CNTL__SP_ENABLE_SINGLE_WRITE__SHIFT                                                            0x1a
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_0_MASK                                                                   0x00000001L
+#define SQ_DSM_CNTL__WAVEFRONT_STALL_1_MASK                                                                   0x00000002L
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_0_MASK                                                                  0x00000004L
+#define SQ_DSM_CNTL__SPI_BACKPRESSURE_1_MASK                                                                  0x00000008L
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA0_MASK                                                        0x00000100L
+#define SQ_DSM_CNTL__SEL_DSM_SGPR_IRRITATOR_DATA1_MASK                                                        0x00000200L
+#define SQ_DSM_CNTL__SGPR_ENABLE_SINGLE_WRITE_MASK                                                            0x00000400L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA0_MASK                                                         0x00010000L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA1_MASK                                                         0x00020000L
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE01_MASK                                                           0x00040000L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA2_MASK                                                         0x00080000L
+#define SQ_DSM_CNTL__SEL_DSM_LDS_IRRITATOR_DATA3_MASK                                                         0x00100000L
+#define SQ_DSM_CNTL__LDS_ENABLE_SINGLE_WRITE23_MASK                                                           0x00200000L
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA0_MASK                                                          0x01000000L
+#define SQ_DSM_CNTL__SEL_DSM_SP_IRRITATOR_DATA1_MASK                                                          0x02000000L
+#define SQ_DSM_CNTL__SP_ENABLE_SINGLE_WRITE_MASK                                                              0x04000000L
+//SQ_DSM_CNTL2
+#define SQ_DSM_CNTL2__SGPR_ENABLE_ERROR_INJECT__SHIFT                                                         0x0
+#define SQ_DSM_CNTL2__SGPR_SELECT_INJECT_DELAY__SHIFT                                                         0x2
+#define SQ_DSM_CNTL2__LDS_D_ENABLE_ERROR_INJECT__SHIFT                                                        0x3
+#define SQ_DSM_CNTL2__LDS_D_SELECT_INJECT_DELAY__SHIFT                                                        0x5
+#define SQ_DSM_CNTL2__LDS_I_ENABLE_ERROR_INJECT__SHIFT                                                        0x6
+#define SQ_DSM_CNTL2__LDS_I_SELECT_INJECT_DELAY__SHIFT                                                        0x8
+#define SQ_DSM_CNTL2__SP_ENABLE_ERROR_INJECT__SHIFT                                                           0x9
+#define SQ_DSM_CNTL2__SP_SELECT_INJECT_DELAY__SHIFT                                                           0xb
+#define SQ_DSM_CNTL2__LDS_INJECT_DELAY__SHIFT                                                                 0xe
+#define SQ_DSM_CNTL2__SP_INJECT_DELAY__SHIFT                                                                  0x14
+#define SQ_DSM_CNTL2__SQ_INJECT_DELAY__SHIFT                                                                  0x1a
+#define SQ_DSM_CNTL2__SGPR_ENABLE_ERROR_INJECT_MASK                                                           0x00000003L
+#define SQ_DSM_CNTL2__SGPR_SELECT_INJECT_DELAY_MASK                                                           0x00000004L
+#define SQ_DSM_CNTL2__LDS_D_ENABLE_ERROR_INJECT_MASK                                                          0x00000018L
+#define SQ_DSM_CNTL2__LDS_D_SELECT_INJECT_DELAY_MASK                                                          0x00000020L
+#define SQ_DSM_CNTL2__LDS_I_ENABLE_ERROR_INJECT_MASK                                                          0x000000C0L
+#define SQ_DSM_CNTL2__LDS_I_SELECT_INJECT_DELAY_MASK                                                          0x00000100L
+#define SQ_DSM_CNTL2__SP_ENABLE_ERROR_INJECT_MASK                                                             0x00000600L
+#define SQ_DSM_CNTL2__SP_SELECT_INJECT_DELAY_MASK                                                             0x00000800L
+#define SQ_DSM_CNTL2__LDS_INJECT_DELAY_MASK                                                                   0x000FC000L
+#define SQ_DSM_CNTL2__SP_INJECT_DELAY_MASK                                                                    0x03F00000L
+#define SQ_DSM_CNTL2__SQ_INJECT_DELAY_MASK                                                                    0xFC000000L
+//SP_CONFIG
+#define SP_CONFIG__DEST_CACHE_EVICT_COUNTER__SHIFT                                                            0x0
+#define SP_CONFIG__ALU_BUSY_MGCG_OVERRIDE__SHIFT                                                              0x2
+#define SP_CONFIG__DISABLE_TRANS_COEXEC__SHIFT                                                                0x3
+#define SP_CONFIG__CAC_COUNTER_OVERRIDE__SHIFT                                                                0x4
+#define SP_CONFIG__SP_SX_EXPVDATA_FGCG_OVERRIDE__SHIFT                                                        0x5
+#define SP_CONFIG__DEST_CACHE_EVICT_COUNTER_MASK                                                              0x00000003L
+#define SP_CONFIG__ALU_BUSY_MGCG_OVERRIDE_MASK                                                                0x00000004L
+#define SP_CONFIG__DISABLE_TRANS_COEXEC_MASK                                                                  0x00000008L
+#define SP_CONFIG__CAC_COUNTER_OVERRIDE_MASK                                                                  0x00000010L
+#define SP_CONFIG__SP_SX_EXPVDATA_FGCG_OVERRIDE_MASK                                                          0x00000020L
+//SQ_ARB_CONFIG
+#define SQ_ARB_CONFIG__WG_RR_INTERVAL__SHIFT                                                                  0x0
+#define SQ_ARB_CONFIG__FWD_PROG_INTERVAL__SHIFT                                                               0x4
+#define SQ_ARB_CONFIG__WG_RR_INTERVAL_MASK                                                                    0x00000003L
+#define SQ_ARB_CONFIG__FWD_PROG_INTERVAL_MASK                                                                 0x00000030L
+//SQ_DEBUG_HOST_TRAP_STATUS
+#define SQ_DEBUG_HOST_TRAP_STATUS__PENDING_COUNT__SHIFT                                                       0x0
+#define SQ_DEBUG_HOST_TRAP_STATUS__PENDING_COUNT_MASK                                                         0x0000007FL
+//SQG_GL1H_STATUS
+#define SQG_GL1H_STATUS__R0_ACK_ERR_DETECTED__SHIFT                                                           0x0
+#define SQG_GL1H_STATUS__R0_XNACK_ERR_DETECTED__SHIFT                                                         0x1
+#define SQG_GL1H_STATUS__R1_ACK_ERR_DETECTED__SHIFT                                                           0x2
+#define SQG_GL1H_STATUS__R1_XNACK_ERR_DETECTED__SHIFT                                                         0x3
+#define SQG_GL1H_STATUS__R0_ACK_ERR_DETECTED_MASK                                                             0x00000001L
+#define SQG_GL1H_STATUS__R0_XNACK_ERR_DETECTED_MASK                                                           0x00000002L
+#define SQG_GL1H_STATUS__R1_ACK_ERR_DETECTED_MASK                                                             0x00000004L
+#define SQG_GL1H_STATUS__R1_XNACK_ERR_DETECTED_MASK                                                           0x00000008L
+//SQG_CONFIG
+#define SQG_CONFIG__GL1H_PREFETCH_PAGE__SHIFT                                                                 0x0
+#define SQG_CONFIG__SQG_ICPFT_EN__SHIFT                                                                       0xd
+#define SQG_CONFIG__SQG_ICPFT_CLR__SHIFT                                                                      0xe
+#define SQG_CONFIG__XNACK_INTR_MASK__SHIFT                                                                    0x10
+#define SQG_CONFIG__GL1H_PREFETCH_PAGE_MASK                                                                   0x0000000FL
+#define SQG_CONFIG__SQG_ICPFT_EN_MASK                                                                         0x00002000L
+#define SQG_CONFIG__SQG_ICPFT_CLR_MASK                                                                        0x00004000L
+#define SQG_CONFIG__XNACK_INTR_MASK_MASK                                                                      0xFFFF0000L
+//SQ_PERF_SNAPSHOT_CTRL
+#define SQ_PERF_SNAPSHOT_CTRL__TIMER_ON_OFF__SHIFT                                                            0x0
+#define SQ_PERF_SNAPSHOT_CTRL__VMID_MASK__SHIFT                                                               0x1
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_SEL__SHIFT                                                               0x11
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_INTERVAL__SHIFT                                                          0x12
+#define SQ_PERF_SNAPSHOT_CTRL__TIMER_ON_OFF_MASK                                                              0x00000001L
+#define SQ_PERF_SNAPSHOT_CTRL__VMID_MASK_MASK                                                                 0x0001FFFEL
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_SEL_MASK                                                                 0x00020000L
+#define SQ_PERF_SNAPSHOT_CTRL__COUNT_INTERVAL_MASK                                                            0x003C0000L
+//CC_GC_SHADER_RATE_CONFIG
+#define CC_GC_SHADER_RATE_CONFIG__WRITE_DIS__SHIFT                                                            0x0
+#define CC_GC_SHADER_RATE_CONFIG__DPFP_RATE__SHIFT                                                            0x1
+#define CC_GC_SHADER_RATE_CONFIG__WRITE_DIS_MASK                                                              0x00000001L
+#define CC_GC_SHADER_RATE_CONFIG__DPFP_RATE_MASK                                                              0x00000006L
+//SQ_INTERRUPT_AUTO_MASK
+#define SQ_INTERRUPT_AUTO_MASK__MASK__SHIFT                                                                   0x0
+#define SQ_INTERRUPT_AUTO_MASK__MASK_MASK                                                                     0x00FFFFFFL
+//SQ_INTERRUPT_MSG_CTRL
+#define SQ_INTERRUPT_MSG_CTRL__STALL__SHIFT                                                                   0x0
+#define SQ_INTERRUPT_MSG_CTRL__STALL_MASK                                                                     0x00000001L
+//SQ_WATCH0_ADDR_H
+#define SQ_WATCH0_ADDR_H__ADDR__SHIFT                                                                         0x0
+#define SQ_WATCH0_ADDR_H__ADDR_MASK                                                                           0x0000FFFFL
+//SQ_WATCH0_ADDR_L
+#define SQ_WATCH0_ADDR_L__ADDR__SHIFT                                                                         0x6
+#define SQ_WATCH0_ADDR_L__ADDR_MASK                                                                           0xFFFFFFC0L
+//SQ_WATCH0_CNTL
+#define SQ_WATCH0_CNTL__MASK__SHIFT                                                                           0x0
+#define SQ_WATCH0_CNTL__VMID__SHIFT                                                                           0x18
+#define SQ_WATCH0_CNTL__VALID__SHIFT                                                                          0x1f
+#define SQ_WATCH0_CNTL__MASK_MASK                                                                             0x00FFFFFFL
+#define SQ_WATCH0_CNTL__VMID_MASK                                                                             0x0F000000L
+#define SQ_WATCH0_CNTL__VALID_MASK                                                                            0x80000000L
+//SQ_WATCH1_ADDR_H
+#define SQ_WATCH1_ADDR_H__ADDR__SHIFT                                                                         0x0
+#define SQ_WATCH1_ADDR_H__ADDR_MASK                                                                           0x0000FFFFL
+//SQ_WATCH1_ADDR_L
+#define SQ_WATCH1_ADDR_L__ADDR__SHIFT                                                                         0x6
+#define SQ_WATCH1_ADDR_L__ADDR_MASK                                                                           0xFFFFFFC0L
+//SQ_WATCH1_CNTL
+#define SQ_WATCH1_CNTL__MASK__SHIFT                                                                           0x0
+#define SQ_WATCH1_CNTL__VMID__SHIFT                                                                           0x18
+#define SQ_WATCH1_CNTL__VALID__SHIFT                                                                          0x1f
+#define SQ_WATCH1_CNTL__MASK_MASK                                                                             0x00FFFFFFL
+#define SQ_WATCH1_CNTL__VMID_MASK                                                                             0x0F000000L
+#define SQ_WATCH1_CNTL__VALID_MASK                                                                            0x80000000L
+//SQ_WATCH2_ADDR_H
+#define SQ_WATCH2_ADDR_H__ADDR__SHIFT                                                                         0x0
+#define SQ_WATCH2_ADDR_H__ADDR_MASK                                                                           0x0000FFFFL
+//SQ_WATCH2_ADDR_L
+#define SQ_WATCH2_ADDR_L__ADDR__SHIFT                                                                         0x6
+#define SQ_WATCH2_ADDR_L__ADDR_MASK                                                                           0xFFFFFFC0L
+//SQ_WATCH2_CNTL
+#define SQ_WATCH2_CNTL__MASK__SHIFT                                                                           0x0
+#define SQ_WATCH2_CNTL__VMID__SHIFT                                                                           0x18
+#define SQ_WATCH2_CNTL__VALID__SHIFT                                                                          0x1f
+#define SQ_WATCH2_CNTL__MASK_MASK                                                                             0x00FFFFFFL
+#define SQ_WATCH2_CNTL__VMID_MASK                                                                             0x0F000000L
+#define SQ_WATCH2_CNTL__VALID_MASK                                                                            0x80000000L
+//SQ_WATCH3_ADDR_H
+#define SQ_WATCH3_ADDR_H__ADDR__SHIFT                                                                         0x0
+#define SQ_WATCH3_ADDR_H__ADDR_MASK                                                                           0x0000FFFFL
+//SQ_WATCH3_ADDR_L
+#define SQ_WATCH3_ADDR_L__ADDR__SHIFT                                                                         0x6
+#define SQ_WATCH3_ADDR_L__ADDR_MASK                                                                           0xFFFFFFC0L
+//SQ_WATCH3_CNTL
+#define SQ_WATCH3_CNTL__MASK__SHIFT                                                                           0x0
+#define SQ_WATCH3_CNTL__VMID__SHIFT                                                                           0x18
+#define SQ_WATCH3_CNTL__VALID__SHIFT                                                                          0x1f
+#define SQ_WATCH3_CNTL__MASK_MASK                                                                             0x00FFFFFFL
+#define SQ_WATCH3_CNTL__VMID_MASK                                                                             0x0F000000L
+#define SQ_WATCH3_CNTL__VALID_MASK                                                                            0x80000000L
+//SQ_IND_INDEX
+#define SQ_IND_INDEX__WAVE_ID__SHIFT                                                                          0x0
+#define SQ_IND_INDEX__WORKITEM_ID__SHIFT                                                                      0x5
+#define SQ_IND_INDEX__AUTO_INCR__SHIFT                                                                        0xb
+#define SQ_IND_INDEX__INDEX__SHIFT                                                                            0x10
+#define SQ_IND_INDEX__WAVE_ID_MASK                                                                            0x0000001FL
+#define SQ_IND_INDEX__WORKITEM_ID_MASK                                                                        0x000007E0L
+#define SQ_IND_INDEX__AUTO_INCR_MASK                                                                          0x00000800L
+#define SQ_IND_INDEX__INDEX_MASK                                                                              0xFFFF0000L
+//SQ_IND_DATA
+#define SQ_IND_DATA__DATA__SHIFT                                                                              0x0
+#define SQ_IND_DATA__DATA_MASK                                                                                0xFFFFFFFFL
+//SQ_CMD
+#define SQ_CMD__CMD__SHIFT                                                                                    0x0
+#define SQ_CMD__MODE__SHIFT                                                                                   0x4
+#define SQ_CMD__CHECK_VMID__SHIFT                                                                             0x7
+#define SQ_CMD__DATA__SHIFT                                                                                   0x8
+#define SQ_CMD__WAVE_ID__SHIFT                                                                                0x10
+#define SQ_CMD__QUEUE_ID__SHIFT                                                                               0x18
+#define SQ_CMD__VM_ID__SHIFT                                                                                  0x1c
+#define SQ_CMD__CMD_MASK                                                                                      0x0000000FL
+#define SQ_CMD__MODE_MASK                                                                                     0x00000070L
+#define SQ_CMD__CHECK_VMID_MASK                                                                               0x00000080L
+#define SQ_CMD__DATA_MASK                                                                                     0x00000F00L
+#define SQ_CMD__WAVE_ID_MASK                                                                                  0x001F0000L
+#define SQ_CMD__QUEUE_ID_MASK                                                                                 0x07000000L
+#define SQ_CMD__VM_ID_MASK                                                                                    0xF0000000L
+//SQC_MISC_CONFIG
+#define SQC_MISC_CONFIG__UNUSED__SHIFT                                                                        0x0
+#define SQC_MISC_CONFIG__SQC_SPI_TTRACE_FGCG_OVERRIDE__SHIFT                                                  0x5
+#define SQC_MISC_CONFIG__SQ_SPI_MSG_FGCG_OVERRIDE__SHIFT                                                      0x6
+#define SQC_MISC_CONFIG__SPI_SQ_EXPALLOC_FGCG_OVERRIDE__SHIFT                                                 0x7
+#define SQC_MISC_CONFIG__SQC_SQ_DATA_RET_FGCG_OVERRIDE__SHIFT                                                 0x8
+#define SQC_MISC_CONFIG__SQC_SQ_INST_RET_FGCG_OVERRIDE__SHIFT                                                 0x9
+#define SQC_MISC_CONFIG__SQC_GCR_RSP_FGCG_OVERRIDE__SHIFT                                                     0xa
+#define SQC_MISC_CONFIG__ICLK_MGCG_DISABLE__SHIFT                                                             0xb
+#define SQC_MISC_CONFIG__ICLK_BANK_MGCG_DISABLE__SHIFT                                                        0xc
+#define SQC_MISC_CONFIG__DCLK_MGCG_DISABLE__SHIFT                                                             0xd
+#define SQC_MISC_CONFIG__GCLK_MGCG_DISABLE__SHIFT                                                             0xe
+#define SQC_MISC_CONFIG__MCLK_MGCG_DISABLE__SHIFT                                                             0xf
+#define SQC_MISC_CONFIG__PCLK_MGCG_DISABLE__SHIFT                                                             0x10
+#define SQC_MISC_CONFIG__BCLK_MGCG_DISABLE__SHIFT                                                             0x11
+#define SQC_MISC_CONFIG__SQC_TA_RESET_FGCG_OVERRIDE__SHIFT                                                    0x12
+#define SQC_MISC_CONFIG__SQC_LDS_CONFIG_FGCG_OVERRIDE__SHIFT                                                  0x13
+#define SQC_MISC_CONFIG__DCLK_BANK_MGCG_DISABLE__SHIFT                                                        0x14
+#define SQC_MISC_CONFIG__SQC_SQ_BARRIER_DONE_FGCG_OVERRIDE__SHIFT                                             0x15
+#define SQC_MISC_CONFIG__SQC_SQ_MSGDONE_FGCG_OVERRIDE__SHIFT                                                  0x16
+#define SQC_MISC_CONFIG__CMCLK_MGCG_DISABLE__SHIFT                                                            0x17
+#define SQC_MISC_CONFIG__SQC_GL1_CLKEN_OVERRIDE__SHIFT                                                        0x18
+#define SQC_MISC_CONFIG__SQC_CORE_OVERRIDE__SHIFT                                                             0x19
+#define SQC_MISC_CONFIG__ICLK_HMF_BS_MGCG_DISABLE__SHIFT                                                      0x1a
+#define SQC_MISC_CONFIG__ICLK_CC_MGCG_DISABLE__SHIFT                                                          0x1b
+#define SQC_MISC_CONFIG__DCLK_HMF_BS_MGCG_DISABLE__SHIFT                                                      0x1c
+#define SQC_MISC_CONFIG__DCLK_CC_MGCG_DISABLE__SHIFT                                                          0x1d
+#define SQC_MISC_CONFIG__UNUSED_MASK                                                                          0x0000001FL
+#define SQC_MISC_CONFIG__SQC_SPI_TTRACE_FGCG_OVERRIDE_MASK                                                    0x00000020L
+#define SQC_MISC_CONFIG__SQ_SPI_MSG_FGCG_OVERRIDE_MASK                                                        0x00000040L
+#define SQC_MISC_CONFIG__SPI_SQ_EXPALLOC_FGCG_OVERRIDE_MASK                                                   0x00000080L
+#define SQC_MISC_CONFIG__SQC_SQ_DATA_RET_FGCG_OVERRIDE_MASK                                                   0x00000100L
+#define SQC_MISC_CONFIG__SQC_SQ_INST_RET_FGCG_OVERRIDE_MASK                                                   0x00000200L
+#define SQC_MISC_CONFIG__SQC_GCR_RSP_FGCG_OVERRIDE_MASK                                                       0x00000400L
+#define SQC_MISC_CONFIG__ICLK_MGCG_DISABLE_MASK                                                               0x00000800L
+#define SQC_MISC_CONFIG__ICLK_BANK_MGCG_DISABLE_MASK                                                          0x00001000L
+#define SQC_MISC_CONFIG__DCLK_MGCG_DISABLE_MASK                                                               0x00002000L
+#define SQC_MISC_CONFIG__GCLK_MGCG_DISABLE_MASK                                                               0x00004000L
+#define SQC_MISC_CONFIG__MCLK_MGCG_DISABLE_MASK                                                               0x00008000L
+#define SQC_MISC_CONFIG__PCLK_MGCG_DISABLE_MASK                                                               0x00010000L
+#define SQC_MISC_CONFIG__BCLK_MGCG_DISABLE_MASK                                                               0x00020000L
+#define SQC_MISC_CONFIG__SQC_TA_RESET_FGCG_OVERRIDE_MASK                                                      0x00040000L
+#define SQC_MISC_CONFIG__SQC_LDS_CONFIG_FGCG_OVERRIDE_MASK                                                    0x00080000L
+#define SQC_MISC_CONFIG__DCLK_BANK_MGCG_DISABLE_MASK                                                          0x00100000L
+#define SQC_MISC_CONFIG__SQC_SQ_BARRIER_DONE_FGCG_OVERRIDE_MASK                                               0x00200000L
+#define SQC_MISC_CONFIG__SQC_SQ_MSGDONE_FGCG_OVERRIDE_MASK                                                    0x00400000L
+#define SQC_MISC_CONFIG__CMCLK_MGCG_DISABLE_MASK                                                              0x00800000L
+#define SQC_MISC_CONFIG__SQC_GL1_CLKEN_OVERRIDE_MASK                                                          0x01000000L
+#define SQC_MISC_CONFIG__SQC_CORE_OVERRIDE_MASK                                                               0x02000000L
+#define SQC_MISC_CONFIG__ICLK_HMF_BS_MGCG_DISABLE_MASK                                                        0x04000000L
+#define SQC_MISC_CONFIG__ICLK_CC_MGCG_DISABLE_MASK                                                            0x08000000L
+#define SQC_MISC_CONFIG__DCLK_HMF_BS_MGCG_DISABLE_MASK                                                        0x10000000L
+#define SQC_MISC_CONFIG__DCLK_CC_MGCG_DISABLE_MASK                                                            0x20000000L
+
+
+// addressBlock: gc_shsdec
+//SX_DEBUG_BUSY
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ3__SHIFT                                                             0x0
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ2__SHIFT                                                             0x1
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ1__SHIFT                                                             0x2
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ3__SHIFT                                                             0x3
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ2__SHIFT                                                             0x4
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ1__SHIFT                                                             0x5
+#define SX_DEBUG_BUSY__PCCMD_VALID__SHIFT                                                                     0x6
+#define SX_DEBUG_BUSY__VDATA1_VALID__SHIFT                                                                    0x7
+#define SX_DEBUG_BUSY__VDATA0_VALID__SHIFT                                                                    0x8
+#define SX_DEBUG_BUSY__CMD_BUSYORVAL__SHIFT                                                                   0x9
+#define SX_DEBUG_BUSY__ADDR_BUSYORVAL__SHIFT                                                                  0xa
+#define SX_DEBUG_BUSY__SX_SX_IN_VALID__SHIFT                                                                  0xb
+#define SX_DEBUG_BUSY__SX_SX_OUT_VALID__SHIFT                                                                 0xc
+#define SX_DEBUG_BUSY__RESERVED__SHIFT                                                                        0xd
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ3_MASK                                                               0x00000001L
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ2_MASK                                                               0x00000002L
+#define SX_DEBUG_BUSY__COL_WRCTRL1_VALIDQ1_MASK                                                               0x00000004L
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ3_MASK                                                               0x00000008L
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ2_MASK                                                               0x00000010L
+#define SX_DEBUG_BUSY__COL_WRCTRL0_VALIDQ1_MASK                                                               0x00000020L
+#define SX_DEBUG_BUSY__PCCMD_VALID_MASK                                                                       0x00000040L
+#define SX_DEBUG_BUSY__VDATA1_VALID_MASK                                                                      0x00000080L
+#define SX_DEBUG_BUSY__VDATA0_VALID_MASK                                                                      0x00000100L
+#define SX_DEBUG_BUSY__CMD_BUSYORVAL_MASK                                                                     0x00000200L
+#define SX_DEBUG_BUSY__ADDR_BUSYORVAL_MASK                                                                    0x00000400L
+#define SX_DEBUG_BUSY__SX_SX_IN_VALID_MASK                                                                    0x00000800L
+#define SX_DEBUG_BUSY__SX_SX_OUT_VALID_MASK                                                                   0x00001000L
+#define SX_DEBUG_BUSY__RESERVED_MASK                                                                          0xFFFFE000L
+//SX_DEBUG_BUSY_2
+#define SX_DEBUG_BUSY_2__COL_SCBD0_BUSY__SHIFT                                                                0x0
+#define SX_DEBUG_BUSY_2__COL_REQ3_FREECNT_NE0__SHIFT                                                          0x1
+#define SX_DEBUG_BUSY_2__COL_REQ3_IDLE__SHIFT                                                                 0x2
+#define SX_DEBUG_BUSY_2__COL_REQ3_BUSY__SHIFT                                                                 0x3
+#define SX_DEBUG_BUSY_2__COL_REQ2_FREECNT_NE0__SHIFT                                                          0x4
+#define SX_DEBUG_BUSY_2__COL_REQ2_IDLE__SHIFT                                                                 0x5
+#define SX_DEBUG_BUSY_2__COL_REQ2_BUSY__SHIFT                                                                 0x6
+#define SX_DEBUG_BUSY_2__COL_REQ1_FREECNT_NE0__SHIFT                                                          0x7
+#define SX_DEBUG_BUSY_2__COL_REQ1_IDLE__SHIFT                                                                 0x8
+#define SX_DEBUG_BUSY_2__COL_REQ1_BUSY__SHIFT                                                                 0x9
+#define SX_DEBUG_BUSY_2__COL_REQ0_FREECNT_NE0__SHIFT                                                          0xa
+#define SX_DEBUG_BUSY_2__COL_REQ0_IDLE__SHIFT                                                                 0xb
+#define SX_DEBUG_BUSY_2__COL_REQ0_BUSY__SHIFT                                                                 0xc
+#define SX_DEBUG_BUSY_2__COL_DBIF3_SENDFREE_BUSY__SHIFT                                                       0xd
+#define SX_DEBUG_BUSY_2__COL_DBIF3_FIFO_BUSY__SHIFT                                                           0xe
+#define SX_DEBUG_BUSY_2__COL_DBIF3_QUAD_FREE__SHIFT                                                           0xf
+#define SX_DEBUG_BUSY_2__COL_DBIF2_SENDFREE_BUSY__SHIFT                                                       0x10
+#define SX_DEBUG_BUSY_2__COL_DBIF2_FIFO_BUSY__SHIFT                                                           0x11
+#define SX_DEBUG_BUSY_2__COL_DBIF2_QUAD_FREE__SHIFT                                                           0x12
+#define SX_DEBUG_BUSY_2__COL_DBIF1_SENDFREE_BUSY__SHIFT                                                       0x13
+#define SX_DEBUG_BUSY_2__COL_DBIF1_FIFO_BUSY__SHIFT                                                           0x14
+#define SX_DEBUG_BUSY_2__COL_DBIF1_QUAD_FREE__SHIFT                                                           0x15
+#define SX_DEBUG_BUSY_2__COL_DBIF0_SENDFREE_BUSY__SHIFT                                                       0x16
+#define SX_DEBUG_BUSY_2__COL_DBIF0_FIFO_BUSY__SHIFT                                                           0x17
+#define SX_DEBUG_BUSY_2__COL_DBIF0_QUAD_FREE__SHIFT                                                           0x18
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL3_BUSY__SHIFT                                                     0x19
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL2_BUSY__SHIFT                                                     0x1a
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL1_BUSY__SHIFT                                                     0x1b
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL0_BUSY__SHIFT                                                     0x1c
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL3_BUSY__SHIFT                                                     0x1d
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL2_BUSY__SHIFT                                                     0x1e
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL1_BUSY__SHIFT                                                     0x1f
+#define SX_DEBUG_BUSY_2__COL_SCBD0_BUSY_MASK                                                                  0x00000001L
+#define SX_DEBUG_BUSY_2__COL_REQ3_FREECNT_NE0_MASK                                                            0x00000002L
+#define SX_DEBUG_BUSY_2__COL_REQ3_IDLE_MASK                                                                   0x00000004L
+#define SX_DEBUG_BUSY_2__COL_REQ3_BUSY_MASK                                                                   0x00000008L
+#define SX_DEBUG_BUSY_2__COL_REQ2_FREECNT_NE0_MASK                                                            0x00000010L
+#define SX_DEBUG_BUSY_2__COL_REQ2_IDLE_MASK                                                                   0x00000020L
+#define SX_DEBUG_BUSY_2__COL_REQ2_BUSY_MASK                                                                   0x00000040L
+#define SX_DEBUG_BUSY_2__COL_REQ1_FREECNT_NE0_MASK                                                            0x00000080L
+#define SX_DEBUG_BUSY_2__COL_REQ1_IDLE_MASK                                                                   0x00000100L
+#define SX_DEBUG_BUSY_2__COL_REQ1_BUSY_MASK                                                                   0x00000200L
+#define SX_DEBUG_BUSY_2__COL_REQ0_FREECNT_NE0_MASK                                                            0x00000400L
+#define SX_DEBUG_BUSY_2__COL_REQ0_IDLE_MASK                                                                   0x00000800L
+#define SX_DEBUG_BUSY_2__COL_REQ0_BUSY_MASK                                                                   0x00001000L
+#define SX_DEBUG_BUSY_2__COL_DBIF3_SENDFREE_BUSY_MASK                                                         0x00002000L
+#define SX_DEBUG_BUSY_2__COL_DBIF3_FIFO_BUSY_MASK                                                             0x00004000L
+#define SX_DEBUG_BUSY_2__COL_DBIF3_QUAD_FREE_MASK                                                             0x00008000L
+#define SX_DEBUG_BUSY_2__COL_DBIF2_SENDFREE_BUSY_MASK                                                         0x00010000L
+#define SX_DEBUG_BUSY_2__COL_DBIF2_FIFO_BUSY_MASK                                                             0x00020000L
+#define SX_DEBUG_BUSY_2__COL_DBIF2_QUAD_FREE_MASK                                                             0x00040000L
+#define SX_DEBUG_BUSY_2__COL_DBIF1_SENDFREE_BUSY_MASK                                                         0x00080000L
+#define SX_DEBUG_BUSY_2__COL_DBIF1_FIFO_BUSY_MASK                                                             0x00100000L
+#define SX_DEBUG_BUSY_2__COL_DBIF1_QUAD_FREE_MASK                                                             0x00200000L
+#define SX_DEBUG_BUSY_2__COL_DBIF0_SENDFREE_BUSY_MASK                                                         0x00400000L
+#define SX_DEBUG_BUSY_2__COL_DBIF0_FIFO_BUSY_MASK                                                             0x00800000L
+#define SX_DEBUG_BUSY_2__COL_DBIF0_QUAD_FREE_MASK                                                             0x01000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL3_BUSY_MASK                                                       0x02000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL2_BUSY_MASK                                                       0x04000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL1_BUSY_MASK                                                       0x08000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK3_VAL0_BUSY_MASK                                                       0x10000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL3_BUSY_MASK                                                       0x20000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL2_BUSY_MASK                                                       0x40000000L
+#define SX_DEBUG_BUSY_2__COL_BUFF3_BANK2_VAL1_BUSY_MASK                                                       0x80000000L
+//SX_DEBUG_BUSY_3
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK2_VAL0_BUSY__SHIFT                                                     0x0
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL3_BUSY__SHIFT                                                     0x1
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL2_BUSY__SHIFT                                                     0x2
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL1_BUSY__SHIFT                                                     0x3
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL0_BUSY__SHIFT                                                     0x4
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL3_BUSY__SHIFT                                                     0x5
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL2_BUSY__SHIFT                                                     0x6
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL1_BUSY__SHIFT                                                     0x7
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL0_BUSY__SHIFT                                                     0x8
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL3_BUSY__SHIFT                                                     0x9
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL2_BUSY__SHIFT                                                     0xa
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL1_BUSY__SHIFT                                                     0xb
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL0_BUSY__SHIFT                                                     0xc
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL3_BUSY__SHIFT                                                     0xd
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL2_BUSY__SHIFT                                                     0xe
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL1_BUSY__SHIFT                                                     0xf
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL0_BUSY__SHIFT                                                     0x10
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL3_BUSY__SHIFT                                                     0x11
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL2_BUSY__SHIFT                                                     0x12
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL1_BUSY__SHIFT                                                     0x13
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL0_BUSY__SHIFT                                                     0x14
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL3_BUSY__SHIFT                                                     0x15
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL2_BUSY__SHIFT                                                     0x16
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL1_BUSY__SHIFT                                                     0x17
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL0_BUSY__SHIFT                                                     0x18
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL3_BUSY__SHIFT                                                     0x19
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL2_BUSY__SHIFT                                                     0x1a
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL1_BUSY__SHIFT                                                     0x1b
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL0_BUSY__SHIFT                                                     0x1c
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL3_BUSY__SHIFT                                                     0x1d
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL2_BUSY__SHIFT                                                     0x1e
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL1_BUSY__SHIFT                                                     0x1f
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK2_VAL0_BUSY_MASK                                                       0x00000001L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL3_BUSY_MASK                                                       0x00000002L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL2_BUSY_MASK                                                       0x00000004L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL1_BUSY_MASK                                                       0x00000008L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK1_VAL0_BUSY_MASK                                                       0x00000010L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL3_BUSY_MASK                                                       0x00000020L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL2_BUSY_MASK                                                       0x00000040L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL1_BUSY_MASK                                                       0x00000080L
+#define SX_DEBUG_BUSY_3__COL_BUFF3_BANK0_VAL0_BUSY_MASK                                                       0x00000100L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL3_BUSY_MASK                                                       0x00000200L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL2_BUSY_MASK                                                       0x00000400L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL1_BUSY_MASK                                                       0x00000800L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK3_VAL0_BUSY_MASK                                                       0x00001000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL3_BUSY_MASK                                                       0x00002000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL2_BUSY_MASK                                                       0x00004000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL1_BUSY_MASK                                                       0x00008000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK2_VAL0_BUSY_MASK                                                       0x00010000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL3_BUSY_MASK                                                       0x00020000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL2_BUSY_MASK                                                       0x00040000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL1_BUSY_MASK                                                       0x00080000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK1_VAL0_BUSY_MASK                                                       0x00100000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL3_BUSY_MASK                                                       0x00200000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL2_BUSY_MASK                                                       0x00400000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL1_BUSY_MASK                                                       0x00800000L
+#define SX_DEBUG_BUSY_3__COL_BUFF2_BANK0_VAL0_BUSY_MASK                                                       0x01000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL3_BUSY_MASK                                                       0x02000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL2_BUSY_MASK                                                       0x04000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL1_BUSY_MASK                                                       0x08000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK3_VAL0_BUSY_MASK                                                       0x10000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL3_BUSY_MASK                                                       0x20000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL2_BUSY_MASK                                                       0x40000000L
+#define SX_DEBUG_BUSY_3__COL_BUFF1_BANK2_VAL1_BUSY_MASK                                                       0x80000000L
+//SX_DEBUG_BUSY_4
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK2_VAL0_BUSY__SHIFT                                                     0x0
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL3_BUSY__SHIFT                                                     0x1
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL2_BUSY__SHIFT                                                     0x2
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL1_BUSY__SHIFT                                                     0x3
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL0_BUSY__SHIFT                                                     0x4
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL3_BUSY__SHIFT                                                     0x5
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL2_BUSY__SHIFT                                                     0x6
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL1_BUSY__SHIFT                                                     0x7
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL0_BUSY__SHIFT                                                     0x8
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL3_BUSY__SHIFT                                                     0x9
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL2_BUSY__SHIFT                                                     0xa
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL1_BUSY__SHIFT                                                     0xb
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL0_BUSY__SHIFT                                                     0xc
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL3_BUSY__SHIFT                                                     0xd
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL2_BUSY__SHIFT                                                     0xe
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL1_BUSY__SHIFT                                                     0xf
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL0_BUSY__SHIFT                                                     0x10
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL3_BUSY__SHIFT                                                     0x11
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL2_BUSY__SHIFT                                                     0x12
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL1_BUSY__SHIFT                                                     0x13
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL0_BUSY__SHIFT                                                     0x14
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL3_BUSY__SHIFT                                                     0x15
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL2_BUSY__SHIFT                                                     0x16
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL1_BUSY__SHIFT                                                     0x17
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL0_BUSY__SHIFT                                                     0x18
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL3_BUSY__SHIFT                                                     0x19
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL2_BUSY__SHIFT                                                     0x1a
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL1_BUSY__SHIFT                                                     0x1b
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL0_BUSY__SHIFT                                                     0x1c
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL3_BUSY__SHIFT                                                     0x1d
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL2_BUSY__SHIFT                                                     0x1e
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL1_BUSY__SHIFT                                                     0x1f
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK2_VAL0_BUSY_MASK                                                       0x00000001L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL3_BUSY_MASK                                                       0x00000002L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL2_BUSY_MASK                                                       0x00000004L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL1_BUSY_MASK                                                       0x00000008L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK1_VAL0_BUSY_MASK                                                       0x00000010L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL3_BUSY_MASK                                                       0x00000020L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL2_BUSY_MASK                                                       0x00000040L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL1_BUSY_MASK                                                       0x00000080L
+#define SX_DEBUG_BUSY_4__COL_BUFF1_BANK0_VAL0_BUSY_MASK                                                       0x00000100L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL3_BUSY_MASK                                                       0x00000200L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL2_BUSY_MASK                                                       0x00000400L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL1_BUSY_MASK                                                       0x00000800L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK3_VAL0_BUSY_MASK                                                       0x00001000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL3_BUSY_MASK                                                       0x00002000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL2_BUSY_MASK                                                       0x00004000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL1_BUSY_MASK                                                       0x00008000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK2_VAL0_BUSY_MASK                                                       0x00010000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL3_BUSY_MASK                                                       0x00020000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL2_BUSY_MASK                                                       0x00040000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL1_BUSY_MASK                                                       0x00080000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK1_VAL0_BUSY_MASK                                                       0x00100000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL3_BUSY_MASK                                                       0x00200000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL2_BUSY_MASK                                                       0x00400000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL1_BUSY_MASK                                                       0x00800000L
+#define SX_DEBUG_BUSY_4__COL_BUFF0_BANK0_VAL0_BUSY_MASK                                                       0x01000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL3_BUSY_MASK                                                       0x02000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL2_BUSY_MASK                                                       0x04000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL1_BUSY_MASK                                                       0x08000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK7_VAL0_BUSY_MASK                                                       0x10000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL3_BUSY_MASK                                                       0x20000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL2_BUSY_MASK                                                       0x40000000L
+#define SX_DEBUG_BUSY_4__COL_BUFF3_BANK6_VAL1_BUSY_MASK                                                       0x80000000L
+//SX_DEBUG_1
+#define SX_DEBUG_1__SX_DB_QUAD_CREDIT__SHIFT                                                                  0x0
+#define SX_DEBUG_1__ENABLE_FIFO_DEBUG_WRITE__SHIFT                                                            0x7
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT                                                      0x8
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_BYPASS__SHIFT                                                           0x9
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT                                                    0xa
+#define SX_DEBUG_1__DISABLE_QUAD_PAIR_OPT__SHIFT                                                              0xb
+#define SX_DEBUG_1__DISABLE_PIX_EN_ZERO_OPT__SHIFT                                                            0xc
+#define SX_DEBUG_1__DISABLE_REP_FGCG__SHIFT                                                                   0xd
+#define SX_DEBUG_1__ENABLE_SAME_PC_GDS_CGTS__SHIFT                                                            0xe
+#define SX_DEBUG_1__DISABLE_RAM_FGCG__SHIFT                                                                   0xf
+#define SX_DEBUG_1__PC_DISABLE_SAME_ADDR_OPT__SHIFT                                                           0x10
+#define SX_DEBUG_1__DISABLE_COL_VAL_READ_OPT__SHIFT                                                           0x11
+#define SX_DEBUG_1__DISABLE_BC_RB_PLUS__SHIFT                                                                 0x12
+#define SX_DEBUG_1__DISABLE_NATIVE_DOWNCVT_FMT_MAPPING__SHIFT                                                 0x13
+#define SX_DEBUG_1__DISABLE_SCBD_READ_PWR_OPT__SHIFT                                                          0x14
+#define SX_DEBUG_1__DISABLE_GDS_CGTS_OPT__SHIFT                                                               0x15
+#define SX_DEBUG_1__DISABLE_DOWNCVT_PWR_OPT__SHIFT                                                            0x16
+#define SX_DEBUG_1__DISABLE_POS_BUFF_REUSE_OPT__SHIFT                                                         0x17
+#define SX_DEBUG_1__SX_DB_QUAD_CREDIT_MASK                                                                    0x0000007FL
+#define SX_DEBUG_1__ENABLE_FIFO_DEBUG_WRITE_MASK                                                              0x00000080L
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DONT_RD_DST_MASK                                                        0x00000100L
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_BYPASS_MASK                                                             0x00000200L
+#define SX_DEBUG_1__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK                                                      0x00000400L
+#define SX_DEBUG_1__DISABLE_QUAD_PAIR_OPT_MASK                                                                0x00000800L
+#define SX_DEBUG_1__DISABLE_PIX_EN_ZERO_OPT_MASK                                                              0x00001000L
+#define SX_DEBUG_1__DISABLE_REP_FGCG_MASK                                                                     0x00002000L
+#define SX_DEBUG_1__ENABLE_SAME_PC_GDS_CGTS_MASK                                                              0x00004000L
+#define SX_DEBUG_1__DISABLE_RAM_FGCG_MASK                                                                     0x00008000L
+#define SX_DEBUG_1__PC_DISABLE_SAME_ADDR_OPT_MASK                                                             0x00010000L
+#define SX_DEBUG_1__DISABLE_COL_VAL_READ_OPT_MASK                                                             0x00020000L
+#define SX_DEBUG_1__DISABLE_BC_RB_PLUS_MASK                                                                   0x00040000L
+#define SX_DEBUG_1__DISABLE_NATIVE_DOWNCVT_FMT_MAPPING_MASK                                                   0x00080000L
+#define SX_DEBUG_1__DISABLE_SCBD_READ_PWR_OPT_MASK                                                            0x00100000L
+#define SX_DEBUG_1__DISABLE_GDS_CGTS_OPT_MASK                                                                 0x00200000L
+#define SX_DEBUG_1__DISABLE_DOWNCVT_PWR_OPT_MASK                                                              0x00400000L
+#define SX_DEBUG_1__DISABLE_POS_BUFF_REUSE_OPT_MASK                                                           0x00800000L
+//SX_DEBUG_BUSY_5
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK6_VAL0_BUSY__SHIFT                                                     0x0
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL3_BUSY__SHIFT                                                     0x1
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL2_BUSY__SHIFT                                                     0x2
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL1_BUSY__SHIFT                                                     0x3
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL0_BUSY__SHIFT                                                     0x4
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL3_BUSY__SHIFT                                                     0x5
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL2_BUSY__SHIFT                                                     0x6
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL1_BUSY__SHIFT                                                     0x7
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL0_BUSY__SHIFT                                                     0x8
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL3_BUSY__SHIFT                                                     0x9
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL2_BUSY__SHIFT                                                     0xa
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL1_BUSY__SHIFT                                                     0xb
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL0_BUSY__SHIFT                                                     0xc
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL3_BUSY__SHIFT                                                     0xd
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL2_BUSY__SHIFT                                                     0xe
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL1_BUSY__SHIFT                                                     0xf
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL0_BUSY__SHIFT                                                     0x10
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL3_BUSY__SHIFT                                                     0x11
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL2_BUSY__SHIFT                                                     0x12
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL1_BUSY__SHIFT                                                     0x13
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL0_BUSY__SHIFT                                                     0x14
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL3_BUSY__SHIFT                                                     0x15
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL2_BUSY__SHIFT                                                     0x16
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL1_BUSY__SHIFT                                                     0x17
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL0_BUSY__SHIFT                                                     0x18
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL3_BUSY__SHIFT                                                     0x19
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL2_BUSY__SHIFT                                                     0x1a
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL1_BUSY__SHIFT                                                     0x1b
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL0_BUSY__SHIFT                                                     0x1c
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL3_BUSY__SHIFT                                                     0x1d
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL2_BUSY__SHIFT                                                     0x1e
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL1_BUSY__SHIFT                                                     0x1f
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK6_VAL0_BUSY_MASK                                                       0x00000001L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL3_BUSY_MASK                                                       0x00000002L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL2_BUSY_MASK                                                       0x00000004L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL1_BUSY_MASK                                                       0x00000008L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK5_VAL0_BUSY_MASK                                                       0x00000010L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL3_BUSY_MASK                                                       0x00000020L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL2_BUSY_MASK                                                       0x00000040L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL1_BUSY_MASK                                                       0x00000080L
+#define SX_DEBUG_BUSY_5__COL_BUFF3_BANK4_VAL0_BUSY_MASK                                                       0x00000100L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL3_BUSY_MASK                                                       0x00000200L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL2_BUSY_MASK                                                       0x00000400L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL1_BUSY_MASK                                                       0x00000800L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK7_VAL0_BUSY_MASK                                                       0x00001000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL3_BUSY_MASK                                                       0x00002000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL2_BUSY_MASK                                                       0x00004000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL1_BUSY_MASK                                                       0x00008000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK6_VAL0_BUSY_MASK                                                       0x00010000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL3_BUSY_MASK                                                       0x00020000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL2_BUSY_MASK                                                       0x00040000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL1_BUSY_MASK                                                       0x00080000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK5_VAL0_BUSY_MASK                                                       0x00100000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL3_BUSY_MASK                                                       0x00200000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL2_BUSY_MASK                                                       0x00400000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL1_BUSY_MASK                                                       0x00800000L
+#define SX_DEBUG_BUSY_5__COL_BUFF2_BANK4_VAL0_BUSY_MASK                                                       0x01000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL3_BUSY_MASK                                                       0x02000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL2_BUSY_MASK                                                       0x04000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL1_BUSY_MASK                                                       0x08000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK7_VAL0_BUSY_MASK                                                       0x10000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL3_BUSY_MASK                                                       0x20000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL2_BUSY_MASK                                                       0x40000000L
+#define SX_DEBUG_BUSY_5__COL_BUFF1_BANK6_VAL1_BUSY_MASK                                                       0x80000000L
+//SX_DEBUG_BUSY_6
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK6_VAL0_BUSY__SHIFT                                                     0x0
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL3_BUSY__SHIFT                                                     0x1
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL2_BUSY__SHIFT                                                     0x2
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL1_BUSY__SHIFT                                                     0x3
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL0_BUSY__SHIFT                                                     0x4
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL3_BUSY__SHIFT                                                     0x5
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL2_BUSY__SHIFT                                                     0x6
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL1_BUSY__SHIFT                                                     0x7
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL0_BUSY__SHIFT                                                     0x8
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL3_BUSY__SHIFT                                                     0x9
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL2_BUSY__SHIFT                                                     0xa
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL1_BUSY__SHIFT                                                     0xb
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL0_BUSY__SHIFT                                                     0xc
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL3_BUSY__SHIFT                                                     0xd
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL2_BUSY__SHIFT                                                     0xe
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL1_BUSY__SHIFT                                                     0xf
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL0_BUSY__SHIFT                                                     0x10
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL3_BUSY__SHIFT                                                     0x11
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL2_BUSY__SHIFT                                                     0x12
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL1_BUSY__SHIFT                                                     0x13
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL0_BUSY__SHIFT                                                     0x14
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL3_BUSY__SHIFT                                                     0x15
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL2_BUSY__SHIFT                                                     0x16
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL1_BUSY__SHIFT                                                     0x17
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL0_BUSY__SHIFT                                                     0x18
+#define SX_DEBUG_BUSY_6__COL_REQ3_CREDIT_BUSY__SHIFT                                                          0x19
+#define SX_DEBUG_BUSY_6__COL_REQ3_FLOP_BUSY__SHIFT                                                            0x1a
+#define SX_DEBUG_BUSY_6__COL_REQ2_CREDIT_BUSY__SHIFT                                                          0x1b
+#define SX_DEBUG_BUSY_6__COL_REQ2_FLOP_BUSY__SHIFT                                                            0x1c
+#define SX_DEBUG_BUSY_6__COL_REQ1_CREDIT_BUSY__SHIFT                                                          0x1d
+#define SX_DEBUG_BUSY_6__COL_REQ1_FLOP_BUSY__SHIFT                                                            0x1e
+#define SX_DEBUG_BUSY_6__COL_REQ0_CREDIT_BUSY__SHIFT                                                          0x1f
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK6_VAL0_BUSY_MASK                                                       0x00000001L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL3_BUSY_MASK                                                       0x00000002L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL2_BUSY_MASK                                                       0x00000004L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL1_BUSY_MASK                                                       0x00000008L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK5_VAL0_BUSY_MASK                                                       0x00000010L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL3_BUSY_MASK                                                       0x00000020L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL2_BUSY_MASK                                                       0x00000040L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL1_BUSY_MASK                                                       0x00000080L
+#define SX_DEBUG_BUSY_6__COL_BUFF1_BANK4_VAL0_BUSY_MASK                                                       0x00000100L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL3_BUSY_MASK                                                       0x00000200L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL2_BUSY_MASK                                                       0x00000400L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL1_BUSY_MASK                                                       0x00000800L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK7_VAL0_BUSY_MASK                                                       0x00001000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL3_BUSY_MASK                                                       0x00002000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL2_BUSY_MASK                                                       0x00004000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL1_BUSY_MASK                                                       0x00008000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK6_VAL0_BUSY_MASK                                                       0x00010000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL3_BUSY_MASK                                                       0x00020000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL2_BUSY_MASK                                                       0x00040000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL1_BUSY_MASK                                                       0x00080000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK5_VAL0_BUSY_MASK                                                       0x00100000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL3_BUSY_MASK                                                       0x00200000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL2_BUSY_MASK                                                       0x00400000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL1_BUSY_MASK                                                       0x00800000L
+#define SX_DEBUG_BUSY_6__COL_BUFF0_BANK4_VAL0_BUSY_MASK                                                       0x01000000L
+#define SX_DEBUG_BUSY_6__COL_REQ3_CREDIT_BUSY_MASK                                                            0x02000000L
+#define SX_DEBUG_BUSY_6__COL_REQ3_FLOP_BUSY_MASK                                                              0x04000000L
+#define SX_DEBUG_BUSY_6__COL_REQ2_CREDIT_BUSY_MASK                                                            0x08000000L
+#define SX_DEBUG_BUSY_6__COL_REQ2_FLOP_BUSY_MASK                                                              0x10000000L
+#define SX_DEBUG_BUSY_6__COL_REQ1_CREDIT_BUSY_MASK                                                            0x20000000L
+#define SX_DEBUG_BUSY_6__COL_REQ1_FLOP_BUSY_MASK                                                              0x40000000L
+#define SX_DEBUG_BUSY_6__COL_REQ0_CREDIT_BUSY_MASK                                                            0x80000000L
+//SX_DEBUG_BUSY_7
+#define SX_DEBUG_BUSY_7__COL_REQ0_FLOP_BUSY__SHIFT                                                            0x0
+#define SX_DEBUG_BUSY_7__COL_SCBD1_BUSY__SHIFT                                                                0x1
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ1__SHIFT                                                       0x2
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ1_ADJ__SHIFT                                                   0x3
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ2__SHIFT                                                       0x4
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ3__SHIFT                                                       0x5
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ4__SHIFT                                                       0x6
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ5__SHIFT                                                       0x7
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALID_OUT__SHIFT                                                     0x8
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ1__SHIFT                                                       0x9
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ1_ADJ__SHIFT                                                   0xa
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ2__SHIFT                                                       0xb
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ3__SHIFT                                                       0xc
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ4__SHIFT                                                       0xd
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ5__SHIFT                                                       0xe
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALID_OUT__SHIFT                                                     0xf
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ1__SHIFT                                                       0x10
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ1_ADJ__SHIFT                                                   0x11
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ2__SHIFT                                                       0x12
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ3__SHIFT                                                       0x13
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ4__SHIFT                                                       0x14
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ5__SHIFT                                                       0x15
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALID_OUT__SHIFT                                                     0x16
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ1__SHIFT                                                       0x17
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ1_ADJ__SHIFT                                                   0x18
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ2__SHIFT                                                       0x19
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ3__SHIFT                                                       0x1a
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ4__SHIFT                                                       0x1b
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ5__SHIFT                                                       0x1c
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALID_OUT__SHIFT                                                     0x1d
+#define SX_DEBUG_BUSY_7__RESERVED__SHIFT                                                                      0x1e
+#define SX_DEBUG_BUSY_7__COL_REQ0_FLOP_BUSY_MASK                                                              0x00000001L
+#define SX_DEBUG_BUSY_7__COL_SCBD1_BUSY_MASK                                                                  0x00000002L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ1_MASK                                                         0x00000004L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ1_ADJ_MASK                                                     0x00000008L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ2_MASK                                                         0x00000010L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ3_MASK                                                         0x00000020L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ4_MASK                                                         0x00000040L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALIDQ5_MASK                                                         0x00000080L
+#define SX_DEBUG_BUSY_7__COL_BLEND3_DATA_VALID_OUT_MASK                                                       0x00000100L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ1_MASK                                                         0x00000200L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ1_ADJ_MASK                                                     0x00000400L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ2_MASK                                                         0x00000800L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ3_MASK                                                         0x00001000L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ4_MASK                                                         0x00002000L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALIDQ5_MASK                                                         0x00004000L
+#define SX_DEBUG_BUSY_7__COL_BLEND2_DATA_VALID_OUT_MASK                                                       0x00008000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ1_MASK                                                         0x00010000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ1_ADJ_MASK                                                     0x00020000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ2_MASK                                                         0x00040000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ3_MASK                                                         0x00080000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ4_MASK                                                         0x00100000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALIDQ5_MASK                                                         0x00200000L
+#define SX_DEBUG_BUSY_7__COL_BLEND1_DATA_VALID_OUT_MASK                                                       0x00400000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ1_MASK                                                         0x00800000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ1_ADJ_MASK                                                     0x01000000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ2_MASK                                                         0x02000000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ3_MASK                                                         0x04000000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ4_MASK                                                         0x08000000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALIDQ5_MASK                                                         0x10000000L
+#define SX_DEBUG_BUSY_7__COL_BLEND0_DATA_VALID_OUT_MASK                                                       0x20000000L
+#define SX_DEBUG_BUSY_7__RESERVED_MASK                                                                        0xC0000000L
+//SX_DEBUG_BUSY_8
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL3_BUSY__SHIFT                                                            0x0
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL2_BUSY__SHIFT                                                            0x1
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL1_BUSY__SHIFT                                                            0x2
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL0_BUSY__SHIFT                                                            0x3
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL3_BUSY__SHIFT                                                            0x4
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL2_BUSY__SHIFT                                                            0x5
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL1_BUSY__SHIFT                                                            0x6
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL0_BUSY__SHIFT                                                            0x7
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL3_BUSY__SHIFT                                                            0x8
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL2_BUSY__SHIFT                                                            0x9
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL1_BUSY__SHIFT                                                            0xa
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL0_BUSY__SHIFT                                                            0xb
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL3_BUSY__SHIFT                                                            0xc
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL2_BUSY__SHIFT                                                            0xd
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL1_BUSY__SHIFT                                                            0xe
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL0_BUSY__SHIFT                                                            0xf
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL3_BUSY__SHIFT                                                            0x10
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL2_BUSY__SHIFT                                                            0x11
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL1_BUSY__SHIFT                                                            0x12
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL0_BUSY__SHIFT                                                            0x13
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL3_BUSY__SHIFT                                                            0x14
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL2_BUSY__SHIFT                                                            0x15
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL1_BUSY__SHIFT                                                            0x16
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL0_BUSY__SHIFT                                                            0x17
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL3_BUSY__SHIFT                                                            0x18
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL2_BUSY__SHIFT                                                            0x19
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL1_BUSY__SHIFT                                                            0x1a
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL0_BUSY__SHIFT                                                            0x1b
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL3_BUSY__SHIFT                                                            0x1c
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL2_BUSY__SHIFT                                                            0x1d
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL1_BUSY__SHIFT                                                            0x1e
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL0_BUSY__SHIFT                                                            0x1f
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL3_BUSY_MASK                                                              0x00000001L
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL2_BUSY_MASK                                                              0x00000002L
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL1_BUSY_MASK                                                              0x00000004L
+#define SX_DEBUG_BUSY_8__POS_BANK7VAL0_BUSY_MASK                                                              0x00000008L
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL3_BUSY_MASK                                                              0x00000010L
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL2_BUSY_MASK                                                              0x00000020L
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL1_BUSY_MASK                                                              0x00000040L
+#define SX_DEBUG_BUSY_8__POS_BANK6VAL0_BUSY_MASK                                                              0x00000080L
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL3_BUSY_MASK                                                              0x00000100L
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL2_BUSY_MASK                                                              0x00000200L
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL1_BUSY_MASK                                                              0x00000400L
+#define SX_DEBUG_BUSY_8__POS_BANK5VAL0_BUSY_MASK                                                              0x00000800L
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL3_BUSY_MASK                                                              0x00001000L
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL2_BUSY_MASK                                                              0x00002000L
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL1_BUSY_MASK                                                              0x00004000L
+#define SX_DEBUG_BUSY_8__POS_BANK4VAL0_BUSY_MASK                                                              0x00008000L
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL3_BUSY_MASK                                                              0x00010000L
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL2_BUSY_MASK                                                              0x00020000L
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL1_BUSY_MASK                                                              0x00040000L
+#define SX_DEBUG_BUSY_8__POS_BANK3VAL0_BUSY_MASK                                                              0x00080000L
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL3_BUSY_MASK                                                              0x00100000L
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL2_BUSY_MASK                                                              0x00200000L
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL1_BUSY_MASK                                                              0x00400000L
+#define SX_DEBUG_BUSY_8__POS_BANK2VAL0_BUSY_MASK                                                              0x00800000L
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL3_BUSY_MASK                                                              0x01000000L
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL2_BUSY_MASK                                                              0x02000000L
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL1_BUSY_MASK                                                              0x04000000L
+#define SX_DEBUG_BUSY_8__POS_BANK1VAL0_BUSY_MASK                                                              0x08000000L
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL3_BUSY_MASK                                                              0x10000000L
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL2_BUSY_MASK                                                              0x20000000L
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL1_BUSY_MASK                                                              0x40000000L
+#define SX_DEBUG_BUSY_8__POS_BANK0VAL0_BUSY_MASK                                                              0x80000000L
+//SX_DEBUG_BUSY_9
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL3_BUSY__SHIFT                                                            0x0
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL2_BUSY__SHIFT                                                            0x1
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL1_BUSY__SHIFT                                                            0x2
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL0_BUSY__SHIFT                                                            0x3
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL3_BUSY__SHIFT                                                            0x4
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL2_BUSY__SHIFT                                                            0x5
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL1_BUSY__SHIFT                                                            0x6
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL0_BUSY__SHIFT                                                            0x7
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL3_BUSY__SHIFT                                                            0x8
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL2_BUSY__SHIFT                                                            0x9
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL1_BUSY__SHIFT                                                            0xa
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL0_BUSY__SHIFT                                                            0xb
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL3_BUSY__SHIFT                                                            0xc
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL2_BUSY__SHIFT                                                            0xd
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL1_BUSY__SHIFT                                                            0xe
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL0_BUSY__SHIFT                                                            0xf
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL3_BUSY__SHIFT                                                            0x10
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL2_BUSY__SHIFT                                                            0x11
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL1_BUSY__SHIFT                                                            0x12
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL0_BUSY__SHIFT                                                            0x13
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL3_BUSY__SHIFT                                                            0x14
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL2_BUSY__SHIFT                                                            0x15
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL1_BUSY__SHIFT                                                            0x16
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL0_BUSY__SHIFT                                                            0x17
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL3_BUSY__SHIFT                                                            0x18
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL2_BUSY__SHIFT                                                            0x19
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL1_BUSY__SHIFT                                                            0x1a
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL0_BUSY__SHIFT                                                            0x1b
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL3_BUSY__SHIFT                                                            0x1c
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL2_BUSY__SHIFT                                                            0x1d
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL1_BUSY__SHIFT                                                            0x1e
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL0_BUSY__SHIFT                                                            0x1f
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL3_BUSY_MASK                                                              0x00000001L
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL2_BUSY_MASK                                                              0x00000002L
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL1_BUSY_MASK                                                              0x00000004L
+#define SX_DEBUG_BUSY_9__IDX_BANK7VAL0_BUSY_MASK                                                              0x00000008L
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL3_BUSY_MASK                                                              0x00000010L
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL2_BUSY_MASK                                                              0x00000020L
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL1_BUSY_MASK                                                              0x00000040L
+#define SX_DEBUG_BUSY_9__IDX_BANK6VAL0_BUSY_MASK                                                              0x00000080L
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL3_BUSY_MASK                                                              0x00000100L
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL2_BUSY_MASK                                                              0x00000200L
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL1_BUSY_MASK                                                              0x00000400L
+#define SX_DEBUG_BUSY_9__IDX_BANK5VAL0_BUSY_MASK                                                              0x00000800L
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL3_BUSY_MASK                                                              0x00001000L
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL2_BUSY_MASK                                                              0x00002000L
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL1_BUSY_MASK                                                              0x00004000L
+#define SX_DEBUG_BUSY_9__IDX_BANK4VAL0_BUSY_MASK                                                              0x00008000L
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL3_BUSY_MASK                                                              0x00010000L
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL2_BUSY_MASK                                                              0x00020000L
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL1_BUSY_MASK                                                              0x00040000L
+#define SX_DEBUG_BUSY_9__IDX_BANK3VAL0_BUSY_MASK                                                              0x00080000L
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL3_BUSY_MASK                                                              0x00100000L
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL2_BUSY_MASK                                                              0x00200000L
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL1_BUSY_MASK                                                              0x00400000L
+#define SX_DEBUG_BUSY_9__IDX_BANK2VAL0_BUSY_MASK                                                              0x00800000L
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL3_BUSY_MASK                                                              0x01000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL2_BUSY_MASK                                                              0x02000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL1_BUSY_MASK                                                              0x04000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK1VAL0_BUSY_MASK                                                              0x08000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL3_BUSY_MASK                                                              0x10000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL2_BUSY_MASK                                                              0x20000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL1_BUSY_MASK                                                              0x40000000L
+#define SX_DEBUG_BUSY_9__IDX_BANK0VAL0_BUSY_MASK                                                              0x80000000L
+//SX_DEBUG_BUSY_10
+#define SX_DEBUG_BUSY_10__POS_SCBD_BUSY__SHIFT                                                                0x0
+#define SX_DEBUG_BUSY_10__POS_FREE_OR_VALIDS__SHIFT                                                           0x1
+#define SX_DEBUG_BUSY_10__POS_REQUESTER_BUSY__SHIFT                                                           0x2
+#define SX_DEBUG_BUSY_10__PA_SX_BUSY__SHIFT                                                                   0x3
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ3__SHIFT                                                          0x4
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ2__SHIFT                                                          0x5
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ1__SHIFT                                                          0x6
+#define SX_DEBUG_BUSY_10__IDX_SCBD_BUSY__SHIFT                                                                0x7
+#define SX_DEBUG_BUSY_10__IDX_FREE_OR_VALIDS__SHIFT                                                           0x8
+#define SX_DEBUG_BUSY_10__IDX_REQUESTER_BUSY__SHIFT                                                           0x9
+#define SX_DEBUG_BUSY_10__PA_SX_IDX_BUSY__SHIFT                                                               0xa
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ3__SHIFT                                                          0xb
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ2__SHIFT                                                          0xc
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ1__SHIFT                                                          0xd
+#define SX_DEBUG_BUSY_10__RESERVED__SHIFT                                                                     0xe
+#define SX_DEBUG_BUSY_10__POS_SCBD_BUSY_MASK                                                                  0x00000001L
+#define SX_DEBUG_BUSY_10__POS_FREE_OR_VALIDS_MASK                                                             0x00000002L
+#define SX_DEBUG_BUSY_10__POS_REQUESTER_BUSY_MASK                                                             0x00000004L
+#define SX_DEBUG_BUSY_10__PA_SX_BUSY_MASK                                                                     0x00000008L
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ3_MASK                                                            0x00000010L
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ2_MASK                                                            0x00000020L
+#define SX_DEBUG_BUSY_10__POS_WRCTRL1_VALIDQ1_MASK                                                            0x00000040L
+#define SX_DEBUG_BUSY_10__IDX_SCBD_BUSY_MASK                                                                  0x00000080L
+#define SX_DEBUG_BUSY_10__IDX_FREE_OR_VALIDS_MASK                                                             0x00000100L
+#define SX_DEBUG_BUSY_10__IDX_REQUESTER_BUSY_MASK                                                             0x00000200L
+#define SX_DEBUG_BUSY_10__PA_SX_IDX_BUSY_MASK                                                                 0x00000400L
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ3_MASK                                                            0x00000800L
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ2_MASK                                                            0x00001000L
+#define SX_DEBUG_BUSY_10__IDX_WRCTRL1_VALIDQ1_MASK                                                            0x00002000L
+#define SX_DEBUG_BUSY_10__RESERVED_MASK                                                                       0xFFFFC000L
+//SPI_PS_MAX_WAVE_ID
+#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                                0x0
+#define SPI_PS_MAX_WAVE_ID__MAX_COLLISION_WAVE_ID__SHIFT                                                      0x10
+#define SPI_PS_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                                  0x00000FFFL
+#define SPI_PS_MAX_WAVE_ID__MAX_COLLISION_WAVE_ID_MASK                                                        0x03FF0000L
+//SPI_GFX_CNTL
+#define SPI_GFX_CNTL__RESET_COUNTS__SHIFT                                                                     0x0
+#define SPI_GFX_CNTL__RESET_COUNTS_MASK                                                                       0x00000001L
+//SPI_DEBUG_READ
+#define SPI_DEBUG_READ__DATA__SHIFT                                                                           0x0
+#define SPI_DEBUG_READ__DATA_MASK                                                                             0xFFFFFFFFL
+//SPI_DSM_CNTL
+#define SPI_DSM_CNTL__SPI_SR_MEM_DSM_IRRITATOR_DATA__SHIFT                                                    0x0
+#define SPI_DSM_CNTL__SPI_SR_MEM_ENABLE_SINGLE_WRITE__SHIFT                                                   0x2
+#define SPI_DSM_CNTL__SPI_SR_MEM_DSM_IRRITATOR_DATA_MASK                                                      0x00000003L
+#define SPI_DSM_CNTL__SPI_SR_MEM_ENABLE_SINGLE_WRITE_MASK                                                     0x00000004L
+//SPI_DSM_CNTL2
+#define SPI_DSM_CNTL2__SPI_SR_MEM_ENABLE_ERROR_INJECT__SHIFT                                                  0x0
+#define SPI_DSM_CNTL2__SPI_SR_MEM_SELECT_INJECT_DELAY__SHIFT                                                  0x2
+#define SPI_DSM_CNTL2__SPI_SR_MEM_INJECT_DELAY__SHIFT                                                         0x3
+#define SPI_DSM_CNTL2__SPI_SR_MEM_ENABLE_ERROR_INJECT_MASK                                                    0x00000003L
+#define SPI_DSM_CNTL2__SPI_SR_MEM_SELECT_INJECT_DELAY_MASK                                                    0x00000004L
+#define SPI_DSM_CNTL2__SPI_SR_MEM_INJECT_DELAY_MASK                                                           0x000001F8L
+//SPI_EDC_CNT
+#define SPI_EDC_CNT__SPI_SR_MEM_SED_COUNT__SHIFT                                                              0x0
+#define SPI_EDC_CNT__SPI_SR_MEM_SED_COUNT_MASK                                                                0x00000003L
+//SPI_DEBUG_BUSY
+#define SPI_DEBUG_BUSY__HS_BUSY__SHIFT                                                                        0x0
+#define SPI_DEBUG_BUSY__GS_BUSY__SHIFT                                                                        0x1
+#define SPI_DEBUG_BUSY__PS0_BUSY__SHIFT                                                                       0x2
+#define SPI_DEBUG_BUSY__PS1_BUSY__SHIFT                                                                       0x3
+#define SPI_DEBUG_BUSY__PS2_BUSY__SHIFT                                                                       0x4
+#define SPI_DEBUG_BUSY__PS3_BUSY__SHIFT                                                                       0x5
+#define SPI_DEBUG_BUSY__CSG0_BUSY__SHIFT                                                                      0x6
+#define SPI_DEBUG_BUSY__CSG1_BUSY__SHIFT                                                                      0x7
+#define SPI_DEBUG_BUSY__CS0_BUSY__SHIFT                                                                       0x8
+#define SPI_DEBUG_BUSY__CS1_BUSY__SHIFT                                                                       0x9
+#define SPI_DEBUG_BUSY__CS2_BUSY__SHIFT                                                                       0xa
+#define SPI_DEBUG_BUSY__CS3_BUSY__SHIFT                                                                       0xb
+#define SPI_DEBUG_BUSY__CS4_BUSY__SHIFT                                                                       0xc
+#define SPI_DEBUG_BUSY__CS5_BUSY__SHIFT                                                                       0xd
+#define SPI_DEBUG_BUSY__CS6_BUSY__SHIFT                                                                       0xe
+#define SPI_DEBUG_BUSY__CS7_BUSY__SHIFT                                                                       0xf
+#define SPI_DEBUG_BUSY__LDS_WR_CTL0_BUSY__SHIFT                                                               0x10
+#define SPI_DEBUG_BUSY__LDS_WR_CTL1_BUSY__SHIFT                                                               0x11
+#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY__SHIFT                                                                0x12
+#define SPI_DEBUG_BUSY__OFC_LDS_BUSY__SHIFT                                                                   0x13
+#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY__SHIFT                                                               0x14
+#define SPI_DEBUG_BUSY__GRBM_BUSY__SHIFT                                                                      0x15
+#define SPI_DEBUG_BUSY__SPIS_BUSY__SHIFT                                                                      0x16
+#define SPI_DEBUG_BUSY__RSRC_ALLOC_BUSY__SHIFT                                                                0x17
+#define SPI_DEBUG_BUSY__PWS_BUSY__SHIFT                                                                       0x18
+#define SPI_DEBUG_BUSY__HS_BUSY_MASK                                                                          0x00000001L
+#define SPI_DEBUG_BUSY__GS_BUSY_MASK                                                                          0x00000002L
+#define SPI_DEBUG_BUSY__PS0_BUSY_MASK                                                                         0x00000004L
+#define SPI_DEBUG_BUSY__PS1_BUSY_MASK                                                                         0x00000008L
+#define SPI_DEBUG_BUSY__PS2_BUSY_MASK                                                                         0x00000010L
+#define SPI_DEBUG_BUSY__PS3_BUSY_MASK                                                                         0x00000020L
+#define SPI_DEBUG_BUSY__CSG0_BUSY_MASK                                                                        0x00000040L
+#define SPI_DEBUG_BUSY__CSG1_BUSY_MASK                                                                        0x00000080L
+#define SPI_DEBUG_BUSY__CS0_BUSY_MASK                                                                         0x00000100L
+#define SPI_DEBUG_BUSY__CS1_BUSY_MASK                                                                         0x00000200L
+#define SPI_DEBUG_BUSY__CS2_BUSY_MASK                                                                         0x00000400L
+#define SPI_DEBUG_BUSY__CS3_BUSY_MASK                                                                         0x00000800L
+#define SPI_DEBUG_BUSY__CS4_BUSY_MASK                                                                         0x00001000L
+#define SPI_DEBUG_BUSY__CS5_BUSY_MASK                                                                         0x00002000L
+#define SPI_DEBUG_BUSY__CS6_BUSY_MASK                                                                         0x00004000L
+#define SPI_DEBUG_BUSY__CS7_BUSY_MASK                                                                         0x00008000L
+#define SPI_DEBUG_BUSY__LDS_WR_CTL0_BUSY_MASK                                                                 0x00010000L
+#define SPI_DEBUG_BUSY__LDS_WR_CTL1_BUSY_MASK                                                                 0x00020000L
+#define SPI_DEBUG_BUSY__PC_DEALLOC_BUSY_MASK                                                                  0x00040000L
+#define SPI_DEBUG_BUSY__OFC_LDS_BUSY_MASK                                                                     0x00080000L
+#define SPI_DEBUG_BUSY__EVENT_CLCTR_BUSY_MASK                                                                 0x00100000L
+#define SPI_DEBUG_BUSY__GRBM_BUSY_MASK                                                                        0x00200000L
+#define SPI_DEBUG_BUSY__SPIS_BUSY_MASK                                                                        0x00400000L
+#define SPI_DEBUG_BUSY__RSRC_ALLOC_BUSY_MASK                                                                  0x00800000L
+#define SPI_DEBUG_BUSY__PWS_BUSY_MASK                                                                         0x01000000L
+//SPI_CONFIG_PS_CU_EN
+#define SPI_CONFIG_PS_CU_EN__PKR_OFFSET__SHIFT                                                                0x0
+#define SPI_CONFIG_PS_CU_EN__PKR2_OFFSET__SHIFT                                                               0x4
+#define SPI_CONFIG_PS_CU_EN__PKR3_OFFSET__SHIFT                                                               0x8
+#define SPI_CONFIG_PS_CU_EN__PKR_OFFSET_MASK                                                                  0x0000000FL
+#define SPI_CONFIG_PS_CU_EN__PKR2_OFFSET_MASK                                                                 0x000000F0L
+#define SPI_CONFIG_PS_CU_EN__PKR3_OFFSET_MASK                                                                 0x00000F00L
+//SPI_WF_LIFETIME_CNTL
+#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD__SHIFT                                                            0x0
+#define SPI_WF_LIFETIME_CNTL__EN__SHIFT                                                                       0x4
+#define SPI_WF_LIFETIME_CNTL__SAMPLE_PERIOD_MASK                                                              0x0000000FL
+#define SPI_WF_LIFETIME_CNTL__EN_MASK                                                                         0x00000010L
+//SPI_WF_LIFETIME_LIMIT_0
+#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_0__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_0__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_1
+#define SPI_WF_LIFETIME_LIMIT_1__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_1__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_1__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_1__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_2
+#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_2__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_2__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_3
+#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_3__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_3__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_4
+#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_4__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_4__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_LIMIT_5
+#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT__SHIFT                                                               0x0
+#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN__SHIFT                                                               0x1f
+#define SPI_WF_LIFETIME_LIMIT_5__MAX_CNT_MASK                                                                 0x7FFFFFFFL
+#define SPI_WF_LIFETIME_LIMIT_5__EN_WARN_MASK                                                                 0x80000000L
+//SPI_WF_LIFETIME_STATUS_0
+#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_0__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_0__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_0__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_2
+#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_2__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_2__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_2__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_4
+#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_4__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_4__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_4__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_6
+#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_6__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_6__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_6__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_7
+#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_7__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_7__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_7__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_9
+#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT__SHIFT                                                              0x0
+#define SPI_WF_LIFETIME_STATUS_9__INT_SENT__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_STATUS_9__MAX_CNT_MASK                                                                0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_9__INT_SENT_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_11
+#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_11__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_11__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_11__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_13
+#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_13__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_13__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_13__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_14
+#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_14__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_14__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_14__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_15
+#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_15__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_15__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_15__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_16
+#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_16__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_16__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_16__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_17
+#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_17__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_17__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_17__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_18
+#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_18__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_18__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_18__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_19
+#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_19__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_19__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_19__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_STATUS_20
+#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_20__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_20__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_20__INT_SENT_MASK                                                              0x80000000L
+//SPI_WF_LIFETIME_DEBUG
+#define SPI_WF_LIFETIME_DEBUG__START_VALUE__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_DEBUG__OVERRIDE_EN__SHIFT                                                             0x1f
+#define SPI_WF_LIFETIME_DEBUG__START_VALUE_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_DEBUG__OVERRIDE_EN_MASK                                                               0x80000000L
+//SPI_WF_LIFETIME_STATUS_21
+#define SPI_WF_LIFETIME_STATUS_21__MAX_CNT__SHIFT                                                             0x0
+#define SPI_WF_LIFETIME_STATUS_21__INT_SENT__SHIFT                                                            0x1f
+#define SPI_WF_LIFETIME_STATUS_21__MAX_CNT_MASK                                                               0x7FFFFFFFL
+#define SPI_WF_LIFETIME_STATUS_21__INT_SENT_MASK                                                              0x80000000L
+//SPI_LB_CTR_CTRL
+#define SPI_LB_CTR_CTRL__LOAD__SHIFT                                                                          0x0
+#define SPI_LB_CTR_CTRL__WAVES_SELECT__SHIFT                                                                  0x1
+#define SPI_LB_CTR_CTRL__CLEAR_ON_READ__SHIFT                                                                 0x3
+#define SPI_LB_CTR_CTRL__RESET_COUNTS__SHIFT                                                                  0x4
+#define SPI_LB_CTR_CTRL__LOAD_MASK                                                                            0x00000001L
+#define SPI_LB_CTR_CTRL__WAVES_SELECT_MASK                                                                    0x00000006L
+#define SPI_LB_CTR_CTRL__CLEAR_ON_READ_MASK                                                                   0x00000008L
+#define SPI_LB_CTR_CTRL__RESET_COUNTS_MASK                                                                    0x00000010L
+//SPI_LB_WGP_MASK
+#define SPI_LB_WGP_MASK__WGP_MASK__SHIFT                                                                      0x0
+#define SPI_LB_WGP_MASK__WGP_MASK_MASK                                                                        0xFFFFL
+//SPI_LB_DATA_REG
+#define SPI_LB_DATA_REG__CNT_DATA__SHIFT                                                                      0x0
+#define SPI_LB_DATA_REG__CNT_DATA_MASK                                                                        0xFFFFFFFFL
+//SPI_PG_ENABLE_STATIC_WGP_MASK
+#define SPI_PG_ENABLE_STATIC_WGP_MASK__WGP_MASK__SHIFT                                                        0x0
+#define SPI_PG_ENABLE_STATIC_WGP_MASK__WGP_MASK_MASK                                                          0xFFFFL
+//SPI_GDS_CREDITS
+#define SPI_GDS_CREDITS__DS_DATA_CREDITS__SHIFT                                                               0x0
+#define SPI_GDS_CREDITS__DS_CMD_CREDITS__SHIFT                                                                0x8
+#define SPI_GDS_CREDITS__DS_DATA_CREDITS_MASK                                                                 0x000000FFL
+#define SPI_GDS_CREDITS__DS_CMD_CREDITS_MASK                                                                  0x0000FF00L
+//SPI_SX_EXPORT_BUFFER_SIZES
+#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE__SHIFT                                                  0x0
+#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE__SHIFT                                               0x10
+#define SPI_SX_EXPORT_BUFFER_SIZES__COLOR_BUFFER_SIZE_MASK                                                    0x0000FFFFL
+#define SPI_SX_EXPORT_BUFFER_SIZES__POSITION_BUFFER_SIZE_MASK                                                 0xFFFF0000L
+//SPI_SX_SCOREBOARD_BUFFER_SIZES
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE__SHIFT                                          0x0
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE__SHIFT                                       0x10
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__COLOR_SCOREBOARD_SIZE_MASK                                            0x0000FFFFL
+#define SPI_SX_SCOREBOARD_BUFFER_SIZES__POSITION_SCOREBOARD_SIZE_MASK                                         0xFFFF0000L
+//SPI_CSQ_WF_ACTIVE_STATUS
+#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_STATUS__ACTIVE_MASK                                                                 0xFFFFFFFFL
+//SPI_CSQ_WF_ACTIVE_COUNT_0
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_0__EVENTS_MASK                                                                0x07FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_1
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_1__EVENTS_MASK                                                                0x07FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_2
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_2__EVENTS_MASK                                                                0x07FF0000L
+//SPI_CSQ_WF_ACTIVE_COUNT_3
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT__SHIFT                                                               0x0
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__EVENTS__SHIFT                                                              0x10
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__COUNT_MASK                                                                 0x000007FFL
+#define SPI_CSQ_WF_ACTIVE_COUNT_3__EVENTS_MASK                                                                0x07FF0000L
+//SPI_LB_DATA_WAVES
+#define SPI_LB_DATA_WAVES__COUNT0__SHIFT                                                                      0x0
+#define SPI_LB_DATA_WAVES__COUNT1__SHIFT                                                                      0x10
+#define SPI_LB_DATA_WAVES__COUNT0_MASK                                                                        0x0000FFFFL
+#define SPI_LB_DATA_WAVES__COUNT1_MASK                                                                        0xFFFF0000L
+//SPI_LB_DATA_PERWGP_WAVE_HSGS
+#define SPI_LB_DATA_PERWGP_WAVE_HSGS__WGP_USED_HS__SHIFT                                                      0x0
+#define SPI_LB_DATA_PERWGP_WAVE_HSGS__WGP_USED_GS__SHIFT                                                      0x10
+#define SPI_LB_DATA_PERWGP_WAVE_HSGS__WGP_USED_HS_MASK                                                        0x0000FFFFL
+#define SPI_LB_DATA_PERWGP_WAVE_HSGS__WGP_USED_GS_MASK                                                        0xFFFF0000L
+//SPI_LB_DATA_PERWGP_WAVE_CS
+#define SPI_LB_DATA_PERWGP_WAVE_CS__ACTIVE__SHIFT                                                             0x0
+#define SPI_LB_DATA_PERWGP_WAVE_CS__ACTIVE_MASK                                                               0xFFFFL
+//SPIS_DEBUG_READ
+#define SPIS_DEBUG_READ__DATA__SHIFT                                                                          0x0
+#define SPIS_DEBUG_READ__DATA_MASK                                                                            0xFFFFFFFFL
+//BCI_DEBUG_READ
+#define BCI_DEBUG_READ__DATA__SHIFT                                                                           0x0
+#define BCI_DEBUG_READ__DATA_MASK                                                                             0xFFFFFFL
+//SPI_P0_TRAP_SCREEN_PSBA_LO
+#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P0_TRAP_SCREEN_PSBA_HI
+#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK                                                             0xFFL
+//SPI_P0_TRAP_SCREEN_PSMA_LO
+#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P0_TRAP_SCREEN_PSMA_HI
+#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK                                                             0xFFL
+//SPI_P0_TRAP_SCREEN_GPR_MIN
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT                                                           0x0
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT                                                           0x6
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK                                                             0x003FL
+#define SPI_P0_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK                                                             0x03C0L
+//SPI_P1_TRAP_SCREEN_PSBA_LO
+#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSBA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P1_TRAP_SCREEN_PSBA_HI
+#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSBA_HI__MEM_BASE_MASK                                                             0xFFL
+//SPI_P1_TRAP_SCREEN_PSMA_LO
+#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSMA_LO__MEM_BASE_MASK                                                             0xFFFFFFFFL
+//SPI_P1_TRAP_SCREEN_PSMA_HI
+#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_PSMA_HI__MEM_BASE_MASK                                                             0xFFL
+//SPI_P1_TRAP_SCREEN_GPR_MIN
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN__SHIFT                                                           0x0
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN__SHIFT                                                           0x6
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__VGPR_MIN_MASK                                                             0x003FL
+#define SPI_P1_TRAP_SCREEN_GPR_MIN__SGPR_MIN_MASK                                                             0x03C0L
+
+
+// addressBlock: gc_tpdec
+//TD_CNTL
+#define TD_CNTL__DISABLE_MEDIAN_CALC_FOR_CUBECORNER_PHANTOM_TEXELS__SHIFT                                     0x0
+#define TD_CNTL__FORCE_RESIDENCY_MAP_TO_BE_MAX_FILTER__SHIFT                                                  0x2
+#define TD_CNTL__FORCE_RESIDENCY_MAP_CC_MAX_OF_ALL_SAMPLES__SHIFT                                             0x7
+#define TD_CNTL__PRESERVE_VGPR_ON_UTC_ERROR__SHIFT                                                            0xd
+#define TD_CNTL__GATHER4_FLOAT_MODE__SHIFT                                                                    0x10
+#define TD_CNTL__FORCE_RT_BVH4_ARBITER_TO_PING_PONG__SHIFT                                                    0x11
+#define TD_CNTL__GATHER4_DX9_MODE__SHIFT                                                                      0x13
+#define TD_CNTL__DISABLE_POWER_THROTTLE__SHIFT                                                                0x14
+#define TD_CNTL__ENABLE_ROUND_TO_ZERO__SHIFT                                                                  0x15
+#define TD_CNTL__DISABLE_ROUND_TO_ZERO_FOR_LARGE_FLOAT_TO_SMALL_FLOAT__SHIFT                                  0x16
+#define TD_CNTL__DISABLE_2BIT_SIGNED_FORMAT__SHIFT                                                            0x17
+#define TD_CNTL__ARBITER_ROUND_ROBIN__SHIFT                                                                   0x18
+#define TD_CNTL__ARBITER_OLDEST_PRIORITY__SHIFT                                                               0x19
+#define TD_CNTL__DONE_SCOREBOARD_DEPTH__SHIFT                                                                 0x1a
+#define TD_CNTL__DISABLE_MEDIAN_CALC_FOR_CUBECORNER_PHANTOM_TEXELS_MASK                                       0x00000001L
+#define TD_CNTL__FORCE_RESIDENCY_MAP_TO_BE_MAX_FILTER_MASK                                                    0x00000004L
+#define TD_CNTL__FORCE_RESIDENCY_MAP_CC_MAX_OF_ALL_SAMPLES_MASK                                               0x00000080L
+#define TD_CNTL__PRESERVE_VGPR_ON_UTC_ERROR_MASK                                                              0x00002000L
+#define TD_CNTL__GATHER4_FLOAT_MODE_MASK                                                                      0x00010000L
+#define TD_CNTL__FORCE_RT_BVH4_ARBITER_TO_PING_PONG_MASK                                                      0x00020000L
+#define TD_CNTL__GATHER4_DX9_MODE_MASK                                                                        0x00080000L
+#define TD_CNTL__DISABLE_POWER_THROTTLE_MASK                                                                  0x00100000L
+#define TD_CNTL__ENABLE_ROUND_TO_ZERO_MASK                                                                    0x00200000L
+#define TD_CNTL__DISABLE_ROUND_TO_ZERO_FOR_LARGE_FLOAT_TO_SMALL_FLOAT_MASK                                    0x00400000L
+#define TD_CNTL__DISABLE_2BIT_SIGNED_FORMAT_MASK                                                              0x00800000L
+#define TD_CNTL__ARBITER_ROUND_ROBIN_MASK                                                                     0x01000000L
+#define TD_CNTL__ARBITER_OLDEST_PRIORITY_MASK                                                                 0x02000000L
+#define TD_CNTL__DONE_SCOREBOARD_DEPTH_MASK                                                                   0xFC000000L
+//TD_STATUS
+#define TD_STATUS__BUSY__SHIFT                                                                                0x1f
+#define TD_STATUS__BUSY_MASK                                                                                  0x80000000L
+//TD_POWER_CNTL
+#define TD_POWER_CNTL__DISABLE_NOFILTER_FORMATTER_POWER_OPT__SHIFT                                            0x6
+#define TD_POWER_CNTL__FORCE_NOFILTER_D16_FORMATTERS_ON__SHIFT                                                0x7
+#define TD_POWER_CNTL__ENABLE_DEBUG_REG__SHIFT                                                                0x8
+#define TD_POWER_CNTL__DISABLE_NOFILTER_FORMATTER_POWER_OPT_MASK                                              0x00000040L
+#define TD_POWER_CNTL__FORCE_NOFILTER_D16_FORMATTERS_ON_MASK                                                  0x00000080L
+#define TD_POWER_CNTL__ENABLE_DEBUG_REG_MASK                                                                  0x00000100L
+//TD_CNTL2
+#define TD_CNTL2__LDS_RETURN_FIFO_CREDIT__SHIFT                                                               0x0
+#define TD_CNTL2__MULTI_CYCLE_16FP__SHIFT                                                                     0x3
+#define TD_CNTL2__LDS_RETURN_FIFO_CREDIT_MASK                                                                 0x00000007L
+#define TD_CNTL2__MULTI_CYCLE_16FP_MASK                                                                       0x00000008L
+//TD_DSM_CNTL
+//TD_DSM_CNTL2
+//TD_SCRATCH
+#define TD_SCRATCH__SCRATCH__SHIFT                                                                            0x0
+#define TD_SCRATCH__SCRATCH_MASK                                                                              0xFFFFFFFFL
+//TA_CNTL
+#define TA_CNTL__TA_SQ_XNACK_FGCG_DISABLE__SHIFT                                                              0x0
+#define TA_CNTL__ALIGNER_CREDIT__SHIFT                                                                        0x10
+#define TA_CNTL__TD_FIFO_CREDIT__SHIFT                                                                        0x16
+#define TA_CNTL__TA_SQ_XNACK_FGCG_DISABLE_MASK                                                                0x00000001L
+#define TA_CNTL__ALIGNER_CREDIT_MASK                                                                          0x001F0000L
+#define TA_CNTL__TD_FIFO_CREDIT_MASK                                                                          0xFFC00000L
+//TA_CNTL_AUX
+#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N__SHIFT                                                                  0x0
+#define TA_CNTL_AUX__DEPTH_AS_PITCH_DIS__SHIFT                                                                0x1
+#define TA_CNTL_AUX__CORNER_SAMPLES_MIN_DIM__SHIFT                                                            0x2
+#define TA_CNTL_AUX__OVERRIDE_QUAD_MODE_DIS__SHIFT                                                            0x3
+#define TA_CNTL_AUX__DERIV_ADJUST_DIS__SHIFT                                                                  0x4
+#define TA_CNTL_AUX__TFAULT_EN_OVERRIDE__SHIFT                                                                0x5
+#define TA_CNTL_AUX__GATHERH_DST_SEL__SHIFT                                                                   0x6
+#define TA_CNTL_AUX__DISABLE_GATHER4_BC_SWIZZLE__SHIFT                                                        0x7
+#define TA_CNTL_AUX__ANISO_MAG_STEP_CLAMP__SHIFT                                                              0x8
+#define TA_CNTL_AUX__AUTO_ALIGN_FORMAT__SHIFT                                                                 0x9
+#define TA_CNTL_AUX__ANISO_HALF_THRESH__SHIFT                                                                 0xa
+#define TA_CNTL_AUX__ANISO_ERROR_FP_VBIAS__SHIFT                                                              0xc
+#define TA_CNTL_AUX__ANISO_STEP_ORDER__SHIFT                                                                  0xd
+#define TA_CNTL_AUX__ANISO_STEP__SHIFT                                                                        0xe
+#define TA_CNTL_AUX__MINMAG_UNNORM__SHIFT                                                                     0xf
+#define TA_CNTL_AUX__ANISO_WEIGHT_MODE__SHIFT                                                                 0x10
+#define TA_CNTL_AUX__ANISO_RATIO_LUT__SHIFT                                                                   0x11
+#define TA_CNTL_AUX__ANISO_TAP__SHIFT                                                                         0x12
+#define TA_CNTL_AUX__DETERMINISM_RESERVED_DISABLE__SHIFT                                                      0x14
+#define TA_CNTL_AUX__DETERMINISM_OPCODE_STRICT_DISABLE__SHIFT                                                 0x15
+#define TA_CNTL_AUX__DETERMINISM_MISC_DISABLE__SHIFT                                                          0x16
+#define TA_CNTL_AUX__DETERMINISM_SAMPLE_C_DFMT_DISABLE__SHIFT                                                 0x17
+#define TA_CNTL_AUX__DETERMINISM_SAMPLER_MSAA_DISABLE__SHIFT                                                  0x18
+#define TA_CNTL_AUX__DETERMINISM_WRITEOP_READFMT_DISABLE__SHIFT                                               0x19
+#define TA_CNTL_AUX__DETERMINISM_DFMT_NFMT_DISABLE__SHIFT                                                     0x1a
+#define TA_CNTL_AUX__CUBEMAP_SLICE_CLAMP__SHIFT                                                               0x1c
+#define TA_CNTL_AUX__TRUNC_SMALL_NEG__SHIFT                                                                   0x1d
+#define TA_CNTL_AUX__ARRAY_ROUND_MODE__SHIFT                                                                  0x1e
+#define TA_CNTL_AUX__SCOAL_DSWIZZLE_N_MASK                                                                    0x00000001L
+#define TA_CNTL_AUX__DEPTH_AS_PITCH_DIS_MASK                                                                  0x00000002L
+#define TA_CNTL_AUX__CORNER_SAMPLES_MIN_DIM_MASK                                                              0x00000004L
+#define TA_CNTL_AUX__OVERRIDE_QUAD_MODE_DIS_MASK                                                              0x00000008L
+#define TA_CNTL_AUX__DERIV_ADJUST_DIS_MASK                                                                    0x00000010L
+#define TA_CNTL_AUX__TFAULT_EN_OVERRIDE_MASK                                                                  0x00000020L
+#define TA_CNTL_AUX__GATHERH_DST_SEL_MASK                                                                     0x00000040L
+#define TA_CNTL_AUX__DISABLE_GATHER4_BC_SWIZZLE_MASK                                                          0x00000080L
+#define TA_CNTL_AUX__ANISO_MAG_STEP_CLAMP_MASK                                                                0x00000100L
+#define TA_CNTL_AUX__AUTO_ALIGN_FORMAT_MASK                                                                   0x00000200L
+#define TA_CNTL_AUX__ANISO_HALF_THRESH_MASK                                                                   0x00000C00L
+#define TA_CNTL_AUX__ANISO_ERROR_FP_VBIAS_MASK                                                                0x00001000L
+#define TA_CNTL_AUX__ANISO_STEP_ORDER_MASK                                                                    0x00002000L
+#define TA_CNTL_AUX__ANISO_STEP_MASK                                                                          0x00004000L
+#define TA_CNTL_AUX__MINMAG_UNNORM_MASK                                                                       0x00008000L
+#define TA_CNTL_AUX__ANISO_WEIGHT_MODE_MASK                                                                   0x00010000L
+#define TA_CNTL_AUX__ANISO_RATIO_LUT_MASK                                                                     0x00020000L
+#define TA_CNTL_AUX__ANISO_TAP_MASK                                                                           0x00040000L
+#define TA_CNTL_AUX__DETERMINISM_RESERVED_DISABLE_MASK                                                        0x00100000L
+#define TA_CNTL_AUX__DETERMINISM_OPCODE_STRICT_DISABLE_MASK                                                   0x00200000L
+#define TA_CNTL_AUX__DETERMINISM_MISC_DISABLE_MASK                                                            0x00400000L
+#define TA_CNTL_AUX__DETERMINISM_SAMPLE_C_DFMT_DISABLE_MASK                                                   0x00800000L
+#define TA_CNTL_AUX__DETERMINISM_SAMPLER_MSAA_DISABLE_MASK                                                    0x01000000L
+#define TA_CNTL_AUX__DETERMINISM_WRITEOP_READFMT_DISABLE_MASK                                                 0x02000000L
+#define TA_CNTL_AUX__DETERMINISM_DFMT_NFMT_DISABLE_MASK                                                       0x04000000L
+#define TA_CNTL_AUX__CUBEMAP_SLICE_CLAMP_MASK                                                                 0x10000000L
+#define TA_CNTL_AUX__TRUNC_SMALL_NEG_MASK                                                                     0x20000000L
+#define TA_CNTL_AUX__ARRAY_ROUND_MODE_MASK                                                                    0xC0000000L
+//TA_CNTL2
+#define TA_CNTL2__POINT_SAMPLE_ACCEL_DIS__SHIFT                                                               0x10
+#define TA_CNTL2__ELEMSIZE_HASH_DIS__SHIFT                                                                    0x11
+#define TA_CNTL2__TRUNCATE_COORD_MODE__SHIFT                                                                  0x12
+#define TA_CNTL2__ELIMINATE_UNLIT_QUAD_DIS__SHIFT                                                             0x13
+#define TA_CNTL2__POINT_SAMPLE_ACCEL_DIS_MASK                                                                 0x00010000L
+#define TA_CNTL2__ELEMSIZE_HASH_DIS_MASK                                                                      0x00020000L
+#define TA_CNTL2__TRUNCATE_COORD_MODE_MASK                                                                    0x00040000L
+#define TA_CNTL2__ELIMINATE_UNLIT_QUAD_DIS_MASK                                                               0x00080000L
+//TA_STATUS
+#define TA_STATUS__FG_PFIFO_EMPTYB__SHIFT                                                                     0xc
+#define TA_STATUS__FG_LFIFO_EMPTYB__SHIFT                                                                     0xd
+#define TA_STATUS__FG_SFIFO_EMPTYB__SHIFT                                                                     0xe
+#define TA_STATUS__FL_PFIFO_EMPTYB__SHIFT                                                                     0x10
+#define TA_STATUS__FL_LFIFO_EMPTYB__SHIFT                                                                     0x11
+#define TA_STATUS__FL_SFIFO_EMPTYB__SHIFT                                                                     0x12
+#define TA_STATUS__FA_PFIFO_EMPTYB__SHIFT                                                                     0x14
+#define TA_STATUS__FA_LFIFO_EMPTYB__SHIFT                                                                     0x15
+#define TA_STATUS__FA_SFIFO_EMPTYB__SHIFT                                                                     0x16
+#define TA_STATUS__IN_BUSY__SHIFT                                                                             0x18
+#define TA_STATUS__FG_BUSY__SHIFT                                                                             0x19
+#define TA_STATUS__LA_BUSY__SHIFT                                                                             0x1a
+#define TA_STATUS__FL_BUSY__SHIFT                                                                             0x1b
+#define TA_STATUS__TA_BUSY__SHIFT                                                                             0x1c
+#define TA_STATUS__FA_BUSY__SHIFT                                                                             0x1d
+#define TA_STATUS__AL_BUSY__SHIFT                                                                             0x1e
+#define TA_STATUS__BUSY__SHIFT                                                                                0x1f
+#define TA_STATUS__FG_PFIFO_EMPTYB_MASK                                                                       0x00001000L
+#define TA_STATUS__FG_LFIFO_EMPTYB_MASK                                                                       0x00002000L
+#define TA_STATUS__FG_SFIFO_EMPTYB_MASK                                                                       0x00004000L
+#define TA_STATUS__FL_PFIFO_EMPTYB_MASK                                                                       0x00010000L
+#define TA_STATUS__FL_LFIFO_EMPTYB_MASK                                                                       0x00020000L
+#define TA_STATUS__FL_SFIFO_EMPTYB_MASK                                                                       0x00040000L
+#define TA_STATUS__FA_PFIFO_EMPTYB_MASK                                                                       0x00100000L
+#define TA_STATUS__FA_LFIFO_EMPTYB_MASK                                                                       0x00200000L
+#define TA_STATUS__FA_SFIFO_EMPTYB_MASK                                                                       0x00400000L
+#define TA_STATUS__IN_BUSY_MASK                                                                               0x01000000L
+#define TA_STATUS__FG_BUSY_MASK                                                                               0x02000000L
+#define TA_STATUS__LA_BUSY_MASK                                                                               0x04000000L
+#define TA_STATUS__FL_BUSY_MASK                                                                               0x08000000L
+#define TA_STATUS__TA_BUSY_MASK                                                                               0x10000000L
+#define TA_STATUS__FA_BUSY_MASK                                                                               0x20000000L
+#define TA_STATUS__AL_BUSY_MASK                                                                               0x40000000L
+#define TA_STATUS__BUSY_MASK                                                                                  0x80000000L
+//TA_SCRATCH
+#define TA_SCRATCH__SCRATCH__SHIFT                                                                            0x0
+#define TA_SCRATCH__SCRATCH_MASK                                                                              0xFFFFFFFFL
+
+
+// addressBlock: gc_gdsdec
+//GDS_CONFIG
+#define GDS_CONFIG__WRITE_DIS__SHIFT                                                                          0x0
+#define GDS_CONFIG__UNUSED__SHIFT                                                                             0x1
+#define GDS_CONFIG__WRITE_DIS_MASK                                                                            0x00000001L
+#define GDS_CONFIG__UNUSED_MASK                                                                               0xFFFFFFFEL
+//GDS_CNTL_STATUS
+#define GDS_CNTL_STATUS__GDS_BUSY__SHIFT                                                                      0x0
+#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY__SHIFT                                                                0x1
+#define GDS_CNTL_STATUS__ORD_APP_BUSY__SHIFT                                                                  0x2
+#define GDS_CNTL_STATUS__DS_WR_CLAMP__SHIFT                                                                   0x3
+#define GDS_CNTL_STATUS__DS_RD_CLAMP__SHIFT                                                                   0x4
+#define GDS_CNTL_STATUS__GRBM_RBUF_BUSY__SHIFT                                                                0x5
+#define GDS_CNTL_STATUS__DS_BUSY__SHIFT                                                                       0x6
+#define GDS_CNTL_STATUS__GWS_BUSY__SHIFT                                                                      0x7
+#define GDS_CNTL_STATUS__ORD_FIFO_BUSY__SHIFT                                                                 0x8
+#define GDS_CNTL_STATUS__CREDIT_BUSY0__SHIFT                                                                  0x9
+#define GDS_CNTL_STATUS__CREDIT_BUSY1__SHIFT                                                                  0xa
+#define GDS_CNTL_STATUS__CREDIT_BUSY2__SHIFT                                                                  0xb
+#define GDS_CNTL_STATUS__CREDIT_BUSY3__SHIFT                                                                  0xc
+#define GDS_CNTL_STATUS__CREDIT_BUSY4__SHIFT                                                                  0xd
+#define GDS_CNTL_STATUS__CREDIT_BUSY5__SHIFT                                                                  0xe
+#define GDS_CNTL_STATUS__CREDIT_BUSY6__SHIFT                                                                  0xf
+#define GDS_CNTL_STATUS__CREDIT_BUSY7__SHIFT                                                                  0x10
+#define GDS_CNTL_STATUS__UNUSED__SHIFT                                                                        0x11
+#define GDS_CNTL_STATUS__GDS_BUSY_MASK                                                                        0x00000001L
+#define GDS_CNTL_STATUS__GRBM_WBUF_BUSY_MASK                                                                  0x00000002L
+#define GDS_CNTL_STATUS__ORD_APP_BUSY_MASK                                                                    0x00000004L
+#define GDS_CNTL_STATUS__DS_WR_CLAMP_MASK                                                                     0x00000008L
+#define GDS_CNTL_STATUS__DS_RD_CLAMP_MASK                                                                     0x00000010L
+#define GDS_CNTL_STATUS__GRBM_RBUF_BUSY_MASK                                                                  0x00000020L
+#define GDS_CNTL_STATUS__DS_BUSY_MASK                                                                         0x00000040L
+#define GDS_CNTL_STATUS__GWS_BUSY_MASK                                                                        0x00000080L
+#define GDS_CNTL_STATUS__ORD_FIFO_BUSY_MASK                                                                   0x00000100L
+#define GDS_CNTL_STATUS__CREDIT_BUSY0_MASK                                                                    0x00000200L
+#define GDS_CNTL_STATUS__CREDIT_BUSY1_MASK                                                                    0x00000400L
+#define GDS_CNTL_STATUS__CREDIT_BUSY2_MASK                                                                    0x00000800L
+#define GDS_CNTL_STATUS__CREDIT_BUSY3_MASK                                                                    0x00001000L
+#define GDS_CNTL_STATUS__CREDIT_BUSY4_MASK                                                                    0x00002000L
+#define GDS_CNTL_STATUS__CREDIT_BUSY5_MASK                                                                    0x00004000L
+#define GDS_CNTL_STATUS__CREDIT_BUSY6_MASK                                                                    0x00008000L
+#define GDS_CNTL_STATUS__CREDIT_BUSY7_MASK                                                                    0x00010000L
+#define GDS_CNTL_STATUS__UNUSED_MASK                                                                          0xFFFE0000L
+//GDS_ENHANCE
+#define GDS_ENHANCE__MISC__SHIFT                                                                              0x0
+#define GDS_ENHANCE__AUTO_INC_INDEX__SHIFT                                                                    0x10
+#define GDS_ENHANCE__CGPG_RESTORE__SHIFT                                                                      0x11
+#define GDS_ENHANCE__UNUSED__SHIFT                                                                            0x12
+#define GDS_ENHANCE__MISC_MASK                                                                                0x0000FFFFL
+#define GDS_ENHANCE__AUTO_INC_INDEX_MASK                                                                      0x00010000L
+#define GDS_ENHANCE__CGPG_RESTORE_MASK                                                                        0x00020000L
+#define GDS_ENHANCE__UNUSED_MASK                                                                              0xFFFC0000L
+//GDS_PROTECTION_FAULT
+#define GDS_PROTECTION_FAULT__WRITE_DIS__SHIFT                                                                0x0
+#define GDS_PROTECTION_FAULT__FAULT_DETECTED__SHIFT                                                           0x1
+#define GDS_PROTECTION_FAULT__GRBM__SHIFT                                                                     0x2
+#define GDS_PROTECTION_FAULT__SE_ID__SHIFT                                                                    0x3
+#define GDS_PROTECTION_FAULT__SA_ID__SHIFT                                                                    0x6
+#define GDS_PROTECTION_FAULT__WGP_ID__SHIFT                                                                   0x7
+#define GDS_PROTECTION_FAULT__SIMD_ID__SHIFT                                                                  0xb
+#define GDS_PROTECTION_FAULT__WAVE_ID__SHIFT                                                                  0xd
+#define GDS_PROTECTION_FAULT__ADDRESS__SHIFT                                                                  0x12
+#define GDS_PROTECTION_FAULT__WRITE_DIS_MASK                                                                  0x00000001L
+#define GDS_PROTECTION_FAULT__FAULT_DETECTED_MASK                                                             0x00000002L
+#define GDS_PROTECTION_FAULT__GRBM_MASK                                                                       0x00000004L
+#define GDS_PROTECTION_FAULT__SE_ID_MASK                                                                      0x00000038L
+#define GDS_PROTECTION_FAULT__SA_ID_MASK                                                                      0x00000040L
+#define GDS_PROTECTION_FAULT__WGP_ID_MASK                                                                     0x00000780L
+#define GDS_PROTECTION_FAULT__SIMD_ID_MASK                                                                    0x00001800L
+#define GDS_PROTECTION_FAULT__WAVE_ID_MASK                                                                    0x0003E000L
+#define GDS_PROTECTION_FAULT__ADDRESS_MASK                                                                    0xFFFC0000L
+//GDS_VM_PROTECTION_FAULT
+#define GDS_VM_PROTECTION_FAULT__WRITE_DIS__SHIFT                                                             0x0
+#define GDS_VM_PROTECTION_FAULT__FAULT_DETECTED__SHIFT                                                        0x1
+#define GDS_VM_PROTECTION_FAULT__GWS__SHIFT                                                                   0x2
+#define GDS_VM_PROTECTION_FAULT__OA__SHIFT                                                                    0x3
+#define GDS_VM_PROTECTION_FAULT__GRBM__SHIFT                                                                  0x4
+#define GDS_VM_PROTECTION_FAULT__TMZ__SHIFT                                                                   0x5
+#define GDS_VM_PROTECTION_FAULT__UNUSED1__SHIFT                                                               0x6
+#define GDS_VM_PROTECTION_FAULT__VMID__SHIFT                                                                  0x8
+#define GDS_VM_PROTECTION_FAULT__UNUSED2__SHIFT                                                               0xc
+#define GDS_VM_PROTECTION_FAULT__ADDRESS__SHIFT                                                               0x10
+#define GDS_VM_PROTECTION_FAULT__WRITE_DIS_MASK                                                               0x00000001L
+#define GDS_VM_PROTECTION_FAULT__FAULT_DETECTED_MASK                                                          0x00000002L
+#define GDS_VM_PROTECTION_FAULT__GWS_MASK                                                                     0x00000004L
+#define GDS_VM_PROTECTION_FAULT__OA_MASK                                                                      0x00000008L
+#define GDS_VM_PROTECTION_FAULT__GRBM_MASK                                                                    0x00000010L
+#define GDS_VM_PROTECTION_FAULT__TMZ_MASK                                                                     0x00000020L
+#define GDS_VM_PROTECTION_FAULT__UNUSED1_MASK                                                                 0x000000C0L
+#define GDS_VM_PROTECTION_FAULT__VMID_MASK                                                                    0x00000F00L
+#define GDS_VM_PROTECTION_FAULT__UNUSED2_MASK                                                                 0x0000F000L
+#define GDS_VM_PROTECTION_FAULT__ADDRESS_MASK                                                                 0xFFFF0000L
+//GDS_EDC_CNT
+#define GDS_EDC_CNT__GDS_MEM_DED__SHIFT                                                                       0x0
+#define GDS_EDC_CNT__GDS_INPUT_QUEUE_SED__SHIFT                                                               0x2
+#define GDS_EDC_CNT__GDS_MEM_SEC__SHIFT                                                                       0x4
+#define GDS_EDC_CNT__UNUSED__SHIFT                                                                            0x6
+#define GDS_EDC_CNT__GDS_MEM_DED_MASK                                                                         0x00000003L
+#define GDS_EDC_CNT__GDS_INPUT_QUEUE_SED_MASK                                                                 0x0000000CL
+#define GDS_EDC_CNT__GDS_MEM_SEC_MASK                                                                         0x00000030L
+#define GDS_EDC_CNT__UNUSED_MASK                                                                              0xFFFFFFC0L
+//GDS_EDC_GRBM_CNT
+#define GDS_EDC_GRBM_CNT__DED__SHIFT                                                                          0x0
+#define GDS_EDC_GRBM_CNT__SEC__SHIFT                                                                          0x2
+#define GDS_EDC_GRBM_CNT__UNUSED__SHIFT                                                                       0x4
+#define GDS_EDC_GRBM_CNT__DED_MASK                                                                            0x00000003L
+#define GDS_EDC_GRBM_CNT__SEC_MASK                                                                            0x0000000CL
+#define GDS_EDC_GRBM_CNT__UNUSED_MASK                                                                         0xFFFFFFF0L
+//GDS_EDC_OA_DED
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_PIX_DED__SHIFT                                                            0x0
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_VTX_DED__SHIFT                                                            0x1
+#define GDS_EDC_OA_DED__ME0_CS_DED__SHIFT                                                                     0x2
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_GS_DED__SHIFT                                                             0x3
+#define GDS_EDC_OA_DED__ME1_PIPE0_DED__SHIFT                                                                  0x4
+#define GDS_EDC_OA_DED__ME1_PIPE1_DED__SHIFT                                                                  0x5
+#define GDS_EDC_OA_DED__ME1_PIPE2_DED__SHIFT                                                                  0x6
+#define GDS_EDC_OA_DED__ME1_PIPE3_DED__SHIFT                                                                  0x7
+#define GDS_EDC_OA_DED__ME2_PIPE0_DED__SHIFT                                                                  0x8
+#define GDS_EDC_OA_DED__ME2_PIPE1_DED__SHIFT                                                                  0x9
+#define GDS_EDC_OA_DED__ME2_PIPE2_DED__SHIFT                                                                  0xa
+#define GDS_EDC_OA_DED__ME2_PIPE3_DED__SHIFT                                                                  0xb
+#define GDS_EDC_OA_DED__ME0_PIPE1_CS_DED__SHIFT                                                               0xc
+#define GDS_EDC_OA_DED__UNUSED1__SHIFT                                                                        0xd
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_PIX_DED_MASK                                                              0x00000001L
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_VTX_DED_MASK                                                              0x00000002L
+#define GDS_EDC_OA_DED__ME0_CS_DED_MASK                                                                       0x00000004L
+#define GDS_EDC_OA_DED__ME0_GFXHP3D_GS_DED_MASK                                                               0x00000008L
+#define GDS_EDC_OA_DED__ME1_PIPE0_DED_MASK                                                                    0x00000010L
+#define GDS_EDC_OA_DED__ME1_PIPE1_DED_MASK                                                                    0x00000020L
+#define GDS_EDC_OA_DED__ME1_PIPE2_DED_MASK                                                                    0x00000040L
+#define GDS_EDC_OA_DED__ME1_PIPE3_DED_MASK                                                                    0x00000080L
+#define GDS_EDC_OA_DED__ME2_PIPE0_DED_MASK                                                                    0x00000100L
+#define GDS_EDC_OA_DED__ME2_PIPE1_DED_MASK                                                                    0x00000200L
+#define GDS_EDC_OA_DED__ME2_PIPE2_DED_MASK                                                                    0x00000400L
+#define GDS_EDC_OA_DED__ME2_PIPE3_DED_MASK                                                                    0x00000800L
+#define GDS_EDC_OA_DED__ME0_PIPE1_CS_DED_MASK                                                                 0x00001000L
+#define GDS_EDC_OA_DED__UNUSED1_MASK                                                                          0xFFFFE000L
+//GDS_DSM_CNTL
+#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_0__SHIFT                                                 0x0
+#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_1__SHIFT                                                 0x1
+#define GDS_DSM_CNTL__GDS_MEM_ENABLE_SINGLE_WRITE__SHIFT                                                      0x2
+#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_0__SHIFT                                         0x3
+#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_1__SHIFT                                         0x4
+#define GDS_DSM_CNTL__GDS_INPUT_QUEUE_ENABLE_SINGLE_WRITE__SHIFT                                              0x5
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_0__SHIFT                                         0x6
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_1__SHIFT                                         0x7
+#define GDS_DSM_CNTL__GDS_PHY_CMD_RAM_ENABLE_SINGLE_WRITE__SHIFT                                              0x8
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_0__SHIFT                                        0x9
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_1__SHIFT                                        0xa
+#define GDS_DSM_CNTL__GDS_PHY_DATA_RAM_ENABLE_SINGLE_WRITE__SHIFT                                             0xb
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_0__SHIFT                                            0xc
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_1__SHIFT                                            0xd
+#define GDS_DSM_CNTL__GDS_PIPE_MEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xe
+#define GDS_DSM_CNTL__UNUSED__SHIFT                                                                           0xf
+#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_0_MASK                                                   0x00000001L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_MEM_IRRITATOR_DATA_1_MASK                                                   0x00000002L
+#define GDS_DSM_CNTL__GDS_MEM_ENABLE_SINGLE_WRITE_MASK                                                        0x00000004L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_0_MASK                                           0x00000008L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_INPUT_QUEUE_IRRITATOR_DATA_1_MASK                                           0x00000010L
+#define GDS_DSM_CNTL__GDS_INPUT_QUEUE_ENABLE_SINGLE_WRITE_MASK                                                0x00000020L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_0_MASK                                           0x00000040L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_CMD_RAM_IRRITATOR_DATA_1_MASK                                           0x00000080L
+#define GDS_DSM_CNTL__GDS_PHY_CMD_RAM_ENABLE_SINGLE_WRITE_MASK                                                0x00000100L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_0_MASK                                          0x00000200L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PHY_DATA_RAM_IRRITATOR_DATA_1_MASK                                          0x00000400L
+#define GDS_DSM_CNTL__GDS_PHY_DATA_RAM_ENABLE_SINGLE_WRITE_MASK                                               0x00000800L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_0_MASK                                              0x00001000L
+#define GDS_DSM_CNTL__SEL_DSM_GDS_PIPE_MEM_IRRITATOR_DATA_1_MASK                                              0x00002000L
+#define GDS_DSM_CNTL__GDS_PIPE_MEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00004000L
+#define GDS_DSM_CNTL__UNUSED_MASK                                                                             0xFFFF8000L
+//GDS_EDC_OA_PHY_CNT
+#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_SEC__SHIFT                                                        0x0
+#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_DED__SHIFT                                                        0x2
+#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_SEC__SHIFT                                                        0x4
+#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_DED__SHIFT                                                        0x6
+#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_SED__SHIFT                                                       0x8
+#define GDS_EDC_OA_PHY_CNT__UNUSED1__SHIFT                                                                    0xa
+#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_SEC_MASK                                                          0x00000003L
+#define GDS_EDC_OA_PHY_CNT__ME0_CS_PIPE_MEM_DED_MASK                                                          0x0000000CL
+#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_SEC_MASK                                                          0x00000030L
+#define GDS_EDC_OA_PHY_CNT__PHY_CMD_RAM_MEM_DED_MASK                                                          0x000000C0L
+#define GDS_EDC_OA_PHY_CNT__PHY_DATA_RAM_MEM_SED_MASK                                                         0x00000300L
+#define GDS_EDC_OA_PHY_CNT__UNUSED1_MASK                                                                      0xFFFFFC00L
+//GDS_EDC_OA_PIPE_CNT
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_SEC__SHIFT                                                    0x0
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_DED__SHIFT                                                    0x2
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_SEC__SHIFT                                                    0x4
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_DED__SHIFT                                                    0x6
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_SEC__SHIFT                                                    0x8
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_DED__SHIFT                                                    0xa
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_SEC__SHIFT                                                    0xc
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_DED__SHIFT                                                    0xe
+#define GDS_EDC_OA_PIPE_CNT__UNUSED__SHIFT                                                                    0x10
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_SEC_MASK                                                      0x00000003L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE0_PIPE_MEM_DED_MASK                                                      0x0000000CL
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_SEC_MASK                                                      0x00000030L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE1_PIPE_MEM_DED_MASK                                                      0x000000C0L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_SEC_MASK                                                      0x00000300L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE2_PIPE_MEM_DED_MASK                                                      0x00000C00L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_SEC_MASK                                                      0x00003000L
+#define GDS_EDC_OA_PIPE_CNT__ME1_PIPE3_PIPE_MEM_DED_MASK                                                      0x0000C000L
+#define GDS_EDC_OA_PIPE_CNT__UNUSED_MASK                                                                      0xFFFF0000L
+//GDS_DSM_CNTL2
+#define GDS_DSM_CNTL2__GDS_MEM_ENABLE_ERROR_INJECT__SHIFT                                                     0x0
+#define GDS_DSM_CNTL2__GDS_MEM_SELECT_INJECT_DELAY__SHIFT                                                     0x2
+#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_ENABLE_ERROR_INJECT__SHIFT                                             0x3
+#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_SELECT_INJECT_DELAY__SHIFT                                             0x5
+#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_ENABLE_ERROR_INJECT__SHIFT                                             0x6
+#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_SELECT_INJECT_DELAY__SHIFT                                             0x8
+#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_ENABLE_ERROR_INJECT__SHIFT                                            0x9
+#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_SELECT_INJECT_DELAY__SHIFT                                            0xb
+#define GDS_DSM_CNTL2__GDS_PIPE_MEM_ENABLE_ERROR_INJECT__SHIFT                                                0xc
+#define GDS_DSM_CNTL2__GDS_PIPE_MEM_SELECT_INJECT_DELAY__SHIFT                                                0xe
+#define GDS_DSM_CNTL2__UNUSED__SHIFT                                                                          0xf
+#define GDS_DSM_CNTL2__GDS_INJECT_DELAY__SHIFT                                                                0x1a
+#define GDS_DSM_CNTL2__GDS_MEM_ENABLE_ERROR_INJECT_MASK                                                       0x00000003L
+#define GDS_DSM_CNTL2__GDS_MEM_SELECT_INJECT_DELAY_MASK                                                       0x00000004L
+#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_ENABLE_ERROR_INJECT_MASK                                               0x00000018L
+#define GDS_DSM_CNTL2__GDS_INPUT_QUEUE_SELECT_INJECT_DELAY_MASK                                               0x00000020L
+#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_ENABLE_ERROR_INJECT_MASK                                               0x000000C0L
+#define GDS_DSM_CNTL2__GDS_PHY_CMD_RAM_SELECT_INJECT_DELAY_MASK                                               0x00000100L
+#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_ENABLE_ERROR_INJECT_MASK                                              0x00000600L
+#define GDS_DSM_CNTL2__GDS_PHY_DATA_RAM_SELECT_INJECT_DELAY_MASK                                              0x00000800L
+#define GDS_DSM_CNTL2__GDS_PIPE_MEM_ENABLE_ERROR_INJECT_MASK                                                  0x00003000L
+#define GDS_DSM_CNTL2__GDS_PIPE_MEM_SELECT_INJECT_DELAY_MASK                                                  0x00004000L
+#define GDS_DSM_CNTL2__UNUSED_MASK                                                                            0x03FF8000L
+#define GDS_DSM_CNTL2__GDS_INJECT_DELAY_MASK                                                                  0xFC000000L
+
+
+// addressBlock: gc_rbdec
+//DB_DEBUG
+#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE__SHIFT                                                       0x0
+#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE__SHIFT                                                         0x1
+#define DB_DEBUG__FETCH_FULL_Z_TILE__SHIFT                                                                    0x2
+#define DB_DEBUG__FETCH_FULL_STENCIL_TILE__SHIFT                                                              0x3
+#define DB_DEBUG__FORCE_Z_MODE__SHIFT                                                                         0x4
+#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ__SHIFT                                                               0x6
+#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ__SHIFT                                                             0x7
+#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE__SHIFT                                                               0x8
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0__SHIFT                                                              0xa
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1__SHIFT                                                              0xc
+#define DB_DEBUG__DEBUG_FAST_Z_DISABLE__SHIFT                                                                 0xe
+#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE__SHIFT                                                           0xf
+#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE__SHIFT                                                              0x10
+#define DB_DEBUG__DISABLE_SUMM_SQUADS__SHIFT                                                                  0x11
+#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS__SHIFT                                                               0x12
+#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE__SHIFT                                                             0x13
+#define DB_DEBUG__NEVER_FREE_Z_ONLY__SHIFT                                                                    0x15
+#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS__SHIFT                                                0x16
+#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION__SHIFT                                                    0x17
+#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES__SHIFT                                                           0x18
+#define DB_DEBUG__ONE_FREE_IN_FLIGHT__SHIFT                                                                   0x1c
+#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT__SHIFT                                                           0x1d
+#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC__SHIFT                                                           0x1e
+#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC__SHIFT                                                           0x1f
+#define DB_DEBUG__DEBUG_STENCIL_COMPRESS_DISABLE_MASK                                                         0x00000001L
+#define DB_DEBUG__DEBUG_DEPTH_COMPRESS_DISABLE_MASK                                                           0x00000002L
+#define DB_DEBUG__FETCH_FULL_Z_TILE_MASK                                                                      0x00000004L
+#define DB_DEBUG__FETCH_FULL_STENCIL_TILE_MASK                                                                0x00000008L
+#define DB_DEBUG__FORCE_Z_MODE_MASK                                                                           0x00000030L
+#define DB_DEBUG__DEBUG_FORCE_DEPTH_READ_MASK                                                                 0x00000040L
+#define DB_DEBUG__DEBUG_FORCE_STENCIL_READ_MASK                                                               0x00000080L
+#define DB_DEBUG__DEBUG_FORCE_HIZ_ENABLE_MASK                                                                 0x00000300L
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE0_MASK                                                                0x00000C00L
+#define DB_DEBUG__DEBUG_FORCE_HIS_ENABLE1_MASK                                                                0x00003000L
+#define DB_DEBUG__DEBUG_FAST_Z_DISABLE_MASK                                                                   0x00004000L
+#define DB_DEBUG__DEBUG_FAST_STENCIL_DISABLE_MASK                                                             0x00008000L
+#define DB_DEBUG__DEBUG_NOOP_CULL_DISABLE_MASK                                                                0x00010000L
+#define DB_DEBUG__DISABLE_SUMM_SQUADS_MASK                                                                    0x00020000L
+#define DB_DEBUG__DEPTH_CACHE_FORCE_MISS_MASK                                                                 0x00040000L
+#define DB_DEBUG__DEBUG_FORCE_FULL_Z_RANGE_MASK                                                               0x00180000L
+#define DB_DEBUG__NEVER_FREE_Z_ONLY_MASK                                                                      0x00200000L
+#define DB_DEBUG__ZPASS_COUNTS_LOOK_AT_PIPE_STAT_EVENTS_MASK                                                  0x00400000L
+#define DB_DEBUG__DISABLE_VPORT_ZPLANE_OPTIMIZATION_MASK                                                      0x00800000L
+#define DB_DEBUG__DECOMPRESS_AFTER_N_ZPLANES_MASK                                                             0x0F000000L
+#define DB_DEBUG__ONE_FREE_IN_FLIGHT_MASK                                                                     0x10000000L
+#define DB_DEBUG__FORCE_MISS_IF_NOT_INFLIGHT_MASK                                                             0x20000000L
+#define DB_DEBUG__DISABLE_DEPTH_SURFACE_SYNC_MASK                                                             0x40000000L
+#define DB_DEBUG__DISABLE_HTILE_SURFACE_SYNC_MASK                                                             0x80000000L
+//DB_DEBUG2
+#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING__SHIFT                                                            0x0
+#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE__SHIFT                                                          0x1
+#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE__SHIFT                                                            0x2
+#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB__SHIFT                                                                 0x3
+#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM__SHIFT                                                        0x4
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL__SHIFT                                                            0x5
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_REZ__SHIFT                                                        0x6
+#define DB_DEBUG2__ENABLE_VIEWPORT_STALL_ON_ALL__SHIFT                                                        0x7
+#define DB_DEBUG2__OPTIMIZE_HIZ_MATCHES_FB_DISABLE__SHIFT                                                     0x8
+#define DB_DEBUG2__CLK_OFF_DELAY__SHIFT                                                                       0x9
+#define DB_DEBUG2__FORCE_PERF_COUNTERS_ON__SHIFT                                                              0xe
+#define DB_DEBUG2__FULL_TILE_CACHE_EVICT_ON_HALF_FULL__SHIFT                                                  0xf
+#define DB_DEBUG2__DISABLE_HTILE_PAIRED_PIPES__SHIFT                                                          0x10
+#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING__SHIFT                                                         0x11
+#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING__SHIFT                                                         0x12
+#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL__SHIFT                                                        0x13
+#define DB_DEBUG2__DISABLE_FULL_TILE_WAVE_BREAK__SHIFT                                                        0x14
+#define DB_DEBUG2__ENABLE_FULL_TILE_WAVE_BREAK_FOR_ALL_TILES__SHIFT                                           0x15
+#define DB_DEBUG2__FORCE_ITERATE_256__SHIFT                                                                   0x18
+#define DB_DEBUG2__RESERVED1__SHIFT                                                                           0x1a
+#define DB_DEBUG2__DEBUG_BUS_FLOP_EN__SHIFT                                                                   0x1b
+#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM__SHIFT                                                             0x1c
+#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL__SHIFT                                                        0x1d
+#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM__SHIFT                                                    0x1e
+#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT__SHIFT                                                0x1f
+#define DB_DEBUG2__ALLOW_COMPZ_BYTE_MASKING_MASK                                                              0x00000001L
+#define DB_DEBUG2__DISABLE_TC_ZRANGE_L0_CACHE_MASK                                                            0x00000002L
+#define DB_DEBUG2__DISABLE_TC_MASK_L0_CACHE_MASK                                                              0x00000004L
+#define DB_DEBUG2__DTR_ROUND_ROBIN_ARB_MASK                                                                   0x00000008L
+#define DB_DEBUG2__DTR_PREZ_STALLS_FOR_ETF_ROOM_MASK                                                          0x00000010L
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_MASK                                                              0x00000020L
+#define DB_DEBUG2__DISABLE_PREZL_FIFO_STALL_REZ_MASK                                                          0x00000040L
+#define DB_DEBUG2__ENABLE_VIEWPORT_STALL_ON_ALL_MASK                                                          0x00000080L
+#define DB_DEBUG2__OPTIMIZE_HIZ_MATCHES_FB_DISABLE_MASK                                                       0x00000100L
+#define DB_DEBUG2__CLK_OFF_DELAY_MASK                                                                         0x00003E00L
+#define DB_DEBUG2__FORCE_PERF_COUNTERS_ON_MASK                                                                0x00004000L
+#define DB_DEBUG2__FULL_TILE_CACHE_EVICT_ON_HALF_FULL_MASK                                                    0x00008000L
+#define DB_DEBUG2__DISABLE_HTILE_PAIRED_PIPES_MASK                                                            0x00010000L
+#define DB_DEBUG2__DISABLE_NULL_EOT_FORWARDING_MASK                                                           0x00020000L
+#define DB_DEBUG2__DISABLE_DTT_DATA_FORWARDING_MASK                                                           0x00040000L
+#define DB_DEBUG2__DISABLE_QUAD_COHERENCY_STALL_MASK                                                          0x00080000L
+#define DB_DEBUG2__DISABLE_FULL_TILE_WAVE_BREAK_MASK                                                          0x00100000L
+#define DB_DEBUG2__ENABLE_FULL_TILE_WAVE_BREAK_FOR_ALL_TILES_MASK                                             0x00200000L
+#define DB_DEBUG2__FORCE_ITERATE_256_MASK                                                                     0x03000000L
+#define DB_DEBUG2__RESERVED1_MASK                                                                             0x04000000L
+#define DB_DEBUG2__DEBUG_BUS_FLOP_EN_MASK                                                                     0x08000000L
+#define DB_DEBUG2__ENABLE_PREZ_OF_REZ_SUMM_MASK                                                               0x10000000L
+#define DB_DEBUG2__DISABLE_PREZL_VIEWPORT_STALL_MASK                                                          0x20000000L
+#define DB_DEBUG2__DISABLE_SINGLE_STENCIL_QUAD_SUMM_MASK                                                      0x40000000L
+#define DB_DEBUG2__DISABLE_WRITE_STALL_ON_RDWR_CONFLICT_MASK                                                  0x80000000L
+//DB_DEBUG3
+#define DB_DEBUG3__DISABLE_CLEAR_ZRANGE_CORRECTION__SHIFT                                                     0x0
+#define DB_DEBUG3__DISABLE_RELOAD_CONTEXT_DRAW_DATA__SHIFT                                                    0x1
+#define DB_DEBUG3__FORCE_DB_IS_GOOD__SHIFT                                                                    0x2
+#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION__SHIFT                                                     0x3
+#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP__SHIFT                                                          0x4
+#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z__SHIFT                                                             0x5
+#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z__SHIFT                                                              0x6
+#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION__SHIFT                                                      0x8
+#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP__SHIFT                                            0xa
+#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS__SHIFT                                                        0xb
+#define DB_DEBUG3__DISABLE_OP_DF_BYPASS__SHIFT                                                                0xd
+#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE__SHIFT                                                         0xe
+#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK__SHIFT                                                       0xf
+#define DB_DEBUG3__DISABLE_SLOCS_PER_CTXT_MATCH__SHIFT                                                        0x10
+#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE__SHIFT                                                         0x11
+#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE__SHIFT                                                     0x13
+#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE__SHIFT                                                         0x14
+#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT__SHIFT                                                0x15
+#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB__SHIFT                                                        0x16
+#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD__SHIFT                                                  0x17
+#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT__SHIFT                                                           0x18
+#define DB_DEBUG3__DISABLE_DI_DT_STALL__SHIFT                                                                 0x19
+#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET__SHIFT                                                             0x1a
+#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX__SHIFT                                                       0x1b
+#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND__SHIFT                                                         0x1c
+#define DB_DEBUG3__DELETE_CONTEXT_SUSPEND__SHIFT                                                              0x1d
+#define DB_DEBUG3__DISABLE_TS_WRITE_L0__SHIFT                                                                 0x1e
+#define DB_DEBUG3__DISABLE_MULTIDTAG_FL_PANIC_REQUIREMENT__SHIFT                                              0x1f
+#define DB_DEBUG3__DISABLE_CLEAR_ZRANGE_CORRECTION_MASK                                                       0x00000001L
+#define DB_DEBUG3__DISABLE_RELOAD_CONTEXT_DRAW_DATA_MASK                                                      0x00000002L
+#define DB_DEBUG3__FORCE_DB_IS_GOOD_MASK                                                                      0x00000004L
+#define DB_DEBUG3__DISABLE_TL_SSO_NULL_SUPPRESSION_MASK                                                       0x00000008L
+#define DB_DEBUG3__DISABLE_HIZ_ON_VPORT_CLAMP_MASK                                                            0x00000010L
+#define DB_DEBUG3__EQAA_INTERPOLATE_COMP_Z_MASK                                                               0x00000020L
+#define DB_DEBUG3__EQAA_INTERPOLATE_SRC_Z_MASK                                                                0x00000040L
+#define DB_DEBUG3__DISABLE_ZCMP_DIRTY_SUPPRESSION_MASK                                                        0x00000100L
+#define DB_DEBUG3__DISABLE_RECOMP_TO_1ZPLANE_WITHOUT_FASTOP_MASK                                              0x00000400L
+#define DB_DEBUG3__ENABLE_INCOHERENT_EQAA_READS_MASK                                                          0x00000800L
+#define DB_DEBUG3__DISABLE_OP_DF_BYPASS_MASK                                                                  0x00002000L
+#define DB_DEBUG3__DISABLE_OP_DF_WRITE_COMBINE_MASK                                                           0x00004000L
+#define DB_DEBUG3__DISABLE_OP_DF_DIRECT_FEEDBACK_MASK                                                         0x00008000L
+#define DB_DEBUG3__DISABLE_SLOCS_PER_CTXT_MATCH_MASK                                                          0x00010000L
+#define DB_DEBUG3__SLOW_PREZ_TO_A2M_OMASK_RATE_MASK                                                           0x00020000L
+#define DB_DEBUG3__DISABLE_TC_UPDATE_WRITE_COMBINE_MASK                                                       0x00080000L
+#define DB_DEBUG3__DISABLE_HZ_TC_WRITE_COMBINE_MASK                                                           0x00100000L
+#define DB_DEBUG3__ENABLE_RECOMP_ZDIRTY_SUPPRESSION_OPT_MASK                                                  0x00200000L
+#define DB_DEBUG3__ENABLE_TC_MA_ROUND_ROBIN_ARB_MASK                                                          0x00400000L
+#define DB_DEBUG3__DISABLE_RAM_READ_SUPPRESION_ON_FWD_MASK                                                    0x00800000L
+#define DB_DEBUG3__DISABLE_EQAA_A2M_PERF_OPT_MASK                                                             0x01000000L
+#define DB_DEBUG3__DISABLE_DI_DT_STALL_MASK                                                                   0x02000000L
+#define DB_DEBUG3__ENABLE_DB_PROCESS_RESET_MASK                                                               0x04000000L
+#define DB_DEBUG3__DISABLE_OVERRASTERIZATION_FIX_MASK                                                         0x08000000L
+#define DB_DEBUG3__DONT_INSERT_CONTEXT_SUSPEND_MASK                                                           0x10000000L
+#define DB_DEBUG3__DELETE_CONTEXT_SUSPEND_MASK                                                                0x20000000L
+#define DB_DEBUG3__DISABLE_TS_WRITE_L0_MASK                                                                   0x40000000L
+#define DB_DEBUG3__DISABLE_MULTIDTAG_FL_PANIC_REQUIREMENT_MASK                                                0x80000000L
+//DB_DEBUG4
+#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION__SHIFT                                                         0x0
+#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION__SHIFT                                                   0x1
+#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL__SHIFT                                                    0x2
+#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL__SHIFT                                             0x3
+#define DB_DEBUG4__DISABLE_SEPARATE_OP_PIPE_CLK__SHIFT                                                        0x4
+#define DB_DEBUG4__DISABLE_SEPARATE_SX_CLK__SHIFT                                                             0x5
+#define DB_DEBUG4__ALWAYS_ON_RMI_CLK_EN__SHIFT                                                                0x6
+#define DB_DEBUG4__ENABLE_DBCB_SLOW_FORMAT_COLLAPSE__SHIFT                                                    0x7
+#define DB_DEBUG4__DISABLE_SEPARATE_DBG_CLK__SHIFT                                                            0x8
+#define DB_DEBUG4__DISABLE_UNMAPPED_Z_INDICATOR__SHIFT                                                        0x9
+#define DB_DEBUG4__DISABLE_UNMAPPED_S_INDICATOR__SHIFT                                                        0xa
+#define DB_DEBUG4__DISABLE_UNMAPPED_H_INDICATOR__SHIFT                                                        0xb
+#define DB_DEBUG4__ENABLE_A2M_DQUAD_OPTIMIZATION__SHIFT                                                       0xc
+#define DB_DEBUG4__DISABLE_DTT_FAST_HTILENACK_LOOKUP__SHIFT                                                   0xd
+#define DB_DEBUG4__DISABLE_RESCHECK_MEMCOHER_OPTIMIZATION__SHIFT                                              0xe
+#define DB_DEBUG4__DISABLE_DYNAMIC_RAM_LIGHT_SLEEP_MODE__SHIFT                                                0xf
+#define DB_DEBUG4__DISABLE_HIZ_TS_COLLISION_DETECT__SHIFT                                                     0x10
+#define DB_DEBUG4__DISABLE_LAST_OF_BURST_ON_FLUSH_CHUNK0_ALL_DONE__SHIFT                                      0x12
+#define DB_DEBUG4__ENABLE_CZ_OVERFLOW_TESTMODE__SHIFT                                                         0x13
+#define DB_DEBUG4__DISABLE_MCC_BURST_FIFO__SHIFT                                                              0x15
+#define DB_DEBUG4__DISABLE_MCC_BURST_FIFO_CONFLICT__SHIFT                                                     0x16
+#define DB_DEBUG4__WR_MEM_BURST_CTL__SHIFT                                                                    0x18
+#define DB_DEBUG4__DISABLE_WR_MEM_BURST_POOLING__SHIFT                                                        0x1b
+#define DB_DEBUG4__DISABLE_RD_MEM_BURST__SHIFT                                                                0x1c
+#define DB_DEBUG4__LATE_ACK_SCOREBOARD_MULTIPLE_SLOT__SHIFT                                                   0x1e
+#define DB_DEBUG4__LATE_ACK_PSD_EOP_OLD_METHOD__SHIFT                                                         0x1f
+#define DB_DEBUG4__DISABLE_QC_Z_MASK_SUMMATION_MASK                                                           0x00000001L
+#define DB_DEBUG4__DISABLE_QC_STENCIL_MASK_SUMMATION_MASK                                                     0x00000002L
+#define DB_DEBUG4__DISABLE_RESUMM_TO_SINGLE_STENCIL_MASK                                                      0x00000004L
+#define DB_DEBUG4__DISABLE_PREZ_POSTZ_DTILE_CONFLICT_STALL_MASK                                               0x00000008L
+#define DB_DEBUG4__DISABLE_SEPARATE_OP_PIPE_CLK_MASK                                                          0x00000010L
+#define DB_DEBUG4__DISABLE_SEPARATE_SX_CLK_MASK                                                               0x00000020L
+#define DB_DEBUG4__ALWAYS_ON_RMI_CLK_EN_MASK                                                                  0x00000040L
+#define DB_DEBUG4__ENABLE_DBCB_SLOW_FORMAT_COLLAPSE_MASK                                                      0x00000080L
+#define DB_DEBUG4__DISABLE_SEPARATE_DBG_CLK_MASK                                                              0x00000100L
+#define DB_DEBUG4__DISABLE_UNMAPPED_Z_INDICATOR_MASK                                                          0x00000200L
+#define DB_DEBUG4__DISABLE_UNMAPPED_S_INDICATOR_MASK                                                          0x00000400L
+#define DB_DEBUG4__DISABLE_UNMAPPED_H_INDICATOR_MASK                                                          0x00000800L
+#define DB_DEBUG4__ENABLE_A2M_DQUAD_OPTIMIZATION_MASK                                                         0x00001000L
+#define DB_DEBUG4__DISABLE_DTT_FAST_HTILENACK_LOOKUP_MASK                                                     0x00002000L
+#define DB_DEBUG4__DISABLE_RESCHECK_MEMCOHER_OPTIMIZATION_MASK                                                0x00004000L
+#define DB_DEBUG4__DISABLE_DYNAMIC_RAM_LIGHT_SLEEP_MODE_MASK                                                  0x00008000L
+#define DB_DEBUG4__DISABLE_HIZ_TS_COLLISION_DETECT_MASK                                                       0x00010000L
+#define DB_DEBUG4__DISABLE_LAST_OF_BURST_ON_FLUSH_CHUNK0_ALL_DONE_MASK                                        0x00040000L
+#define DB_DEBUG4__ENABLE_CZ_OVERFLOW_TESTMODE_MASK                                                           0x00080000L
+#define DB_DEBUG4__DISABLE_MCC_BURST_FIFO_MASK                                                                0x00200000L
+#define DB_DEBUG4__DISABLE_MCC_BURST_FIFO_CONFLICT_MASK                                                       0x00400000L
+#define DB_DEBUG4__WR_MEM_BURST_CTL_MASK                                                                      0x07000000L
+#define DB_DEBUG4__DISABLE_WR_MEM_BURST_POOLING_MASK                                                          0x08000000L
+#define DB_DEBUG4__DISABLE_RD_MEM_BURST_MASK                                                                  0x10000000L
+#define DB_DEBUG4__LATE_ACK_SCOREBOARD_MULTIPLE_SLOT_MASK                                                     0x40000000L
+#define DB_DEBUG4__LATE_ACK_PSD_EOP_OLD_METHOD_MASK                                                           0x80000000L
+//DB_ETILE_STUTTER_CONTROL
+#define DB_ETILE_STUTTER_CONTROL__THRESHOLD__SHIFT                                                            0x0
+#define DB_ETILE_STUTTER_CONTROL__TIMEOUT__SHIFT                                                              0x10
+#define DB_ETILE_STUTTER_CONTROL__THRESHOLD_MASK                                                              0x000000FFL
+#define DB_ETILE_STUTTER_CONTROL__TIMEOUT_MASK                                                                0x00FF0000L
+//DB_LTILE_STUTTER_CONTROL
+#define DB_LTILE_STUTTER_CONTROL__THRESHOLD__SHIFT                                                            0x0
+#define DB_LTILE_STUTTER_CONTROL__TIMEOUT__SHIFT                                                              0x10
+#define DB_LTILE_STUTTER_CONTROL__THRESHOLD_MASK                                                              0x000000FFL
+#define DB_LTILE_STUTTER_CONTROL__TIMEOUT_MASK                                                                0x00FF0000L
+//DB_EQUAD_STUTTER_CONTROL
+#define DB_EQUAD_STUTTER_CONTROL__THRESHOLD__SHIFT                                                            0x0
+#define DB_EQUAD_STUTTER_CONTROL__TIMEOUT__SHIFT                                                              0x10
+#define DB_EQUAD_STUTTER_CONTROL__THRESHOLD_MASK                                                              0x000000FFL
+#define DB_EQUAD_STUTTER_CONTROL__TIMEOUT_MASK                                                                0x00FF0000L
+//DB_LQUAD_STUTTER_CONTROL
+#define DB_LQUAD_STUTTER_CONTROL__THRESHOLD__SHIFT                                                            0x0
+#define DB_LQUAD_STUTTER_CONTROL__TIMEOUT__SHIFT                                                              0x10
+#define DB_LQUAD_STUTTER_CONTROL__THRESHOLD_MASK                                                              0x000000FFL
+#define DB_LQUAD_STUTTER_CONTROL__TIMEOUT_MASK                                                                0x00FF0000L
+//DB_CREDIT_LIMIT
+#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS__SHIFT                                                            0x0
+#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS__SHIFT                                                            0x5
+#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS__SHIFT                                                           0xa
+#define DB_CREDIT_LIMIT__DB_SC_WAVE_CREDITS__SHIFT                                                            0xd
+#define DB_CREDIT_LIMIT__DB_SC_FREE_WAVE_CREDITS__SHIFT                                                       0x12
+#define DB_CREDIT_LIMIT__DB_SC_TILE_CREDITS_MASK                                                              0x0000001FL
+#define DB_CREDIT_LIMIT__DB_SC_QUAD_CREDITS_MASK                                                              0x000003E0L
+#define DB_CREDIT_LIMIT__DB_CB_LQUAD_CREDITS_MASK                                                             0x00001C00L
+#define DB_CREDIT_LIMIT__DB_SC_WAVE_CREDITS_MASK                                                              0x0003E000L
+#define DB_CREDIT_LIMIT__DB_SC_FREE_WAVE_CREDITS_MASK                                                         0x007C0000L
+//DB_WATERMARKS
+#define DB_WATERMARKS__DEPTH_FREE__SHIFT                                                                      0x0
+#define DB_WATERMARKS__DEPTH_FLUSH__SHIFT                                                                     0x8
+#define DB_WATERMARKS__DEPTH_PENDING_FREE__SHIFT                                                              0x10
+#define DB_WATERMARKS__DEPTH_CACHELINE_FREE__SHIFT                                                            0x18
+#define DB_WATERMARKS__DEPTH_FREE_MASK                                                                        0x000000FFL
+#define DB_WATERMARKS__DEPTH_FLUSH_MASK                                                                       0x0000FF00L
+#define DB_WATERMARKS__DEPTH_PENDING_FREE_MASK                                                                0x00FF0000L
+#define DB_WATERMARKS__DEPTH_CACHELINE_FREE_MASK                                                              0xFF000000L
+//DB_SUBTILE_CONTROL
+#define DB_SUBTILE_CONTROL__MSAA1_X__SHIFT                                                                    0x0
+#define DB_SUBTILE_CONTROL__MSAA1_Y__SHIFT                                                                    0x2
+#define DB_SUBTILE_CONTROL__MSAA2_X__SHIFT                                                                    0x4
+#define DB_SUBTILE_CONTROL__MSAA2_Y__SHIFT                                                                    0x6
+#define DB_SUBTILE_CONTROL__MSAA4_X__SHIFT                                                                    0x8
+#define DB_SUBTILE_CONTROL__MSAA4_Y__SHIFT                                                                    0xa
+#define DB_SUBTILE_CONTROL__MSAA8_X__SHIFT                                                                    0xc
+#define DB_SUBTILE_CONTROL__MSAA8_Y__SHIFT                                                                    0xe
+#define DB_SUBTILE_CONTROL__MSAA16_X__SHIFT                                                                   0x10
+#define DB_SUBTILE_CONTROL__MSAA16_Y__SHIFT                                                                   0x12
+#define DB_SUBTILE_CONTROL__MSAA1_X_MASK                                                                      0x00000003L
+#define DB_SUBTILE_CONTROL__MSAA1_Y_MASK                                                                      0x0000000CL
+#define DB_SUBTILE_CONTROL__MSAA2_X_MASK                                                                      0x00000030L
+#define DB_SUBTILE_CONTROL__MSAA2_Y_MASK                                                                      0x000000C0L
+#define DB_SUBTILE_CONTROL__MSAA4_X_MASK                                                                      0x00000300L
+#define DB_SUBTILE_CONTROL__MSAA4_Y_MASK                                                                      0x00000C00L
+#define DB_SUBTILE_CONTROL__MSAA8_X_MASK                                                                      0x00003000L
+#define DB_SUBTILE_CONTROL__MSAA8_Y_MASK                                                                      0x0000C000L
+#define DB_SUBTILE_CONTROL__MSAA16_X_MASK                                                                     0x00030000L
+#define DB_SUBTILE_CONTROL__MSAA16_Y_MASK                                                                     0x000C0000L
+//DB_FREE_CACHELINES
+#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH__SHIFT                                                           0x0
+#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH__SHIFT                                                           0x8
+#define DB_FREE_CACHELINES__FREE_Z_DEPTH__SHIFT                                                               0x10
+#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH__SHIFT                                                           0x18
+#define DB_FREE_CACHELINES__FREE_DTILE_DEPTH_MASK                                                             0x000000FFL
+#define DB_FREE_CACHELINES__FREE_PLANE_DEPTH_MASK                                                             0x0000FF00L
+#define DB_FREE_CACHELINES__FREE_Z_DEPTH_MASK                                                                 0x00FF0000L
+#define DB_FREE_CACHELINES__FREE_HTILE_DEPTH_MASK                                                             0xFF000000L
+//DB_FIFO_DEPTH1
+#define DB_FIFO_DEPTH1__MI_RDREQ_FIFO_DEPTH__SHIFT                                                            0x0
+#define DB_FIFO_DEPTH1__MI_WRREQ_FIFO_DEPTH__SHIFT                                                            0x8
+#define DB_FIFO_DEPTH1__MCC_DEPTH__SHIFT                                                                      0x10
+#define DB_FIFO_DEPTH1__QC_DEPTH__SHIFT                                                                       0x18
+#define DB_FIFO_DEPTH1__MI_RDREQ_FIFO_DEPTH_MASK                                                              0x000000FFL
+#define DB_FIFO_DEPTH1__MI_WRREQ_FIFO_DEPTH_MASK                                                              0x0000FF00L
+#define DB_FIFO_DEPTH1__MCC_DEPTH_MASK                                                                        0x00FF0000L
+#define DB_FIFO_DEPTH1__QC_DEPTH_MASK                                                                         0xFF000000L
+//DB_FIFO_DEPTH2
+#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH__SHIFT                                                               0x0
+#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH__SHIFT                                                            0x8
+#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH__SHIFT                                                               0x10
+#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH__SHIFT                                                            0x19
+#define DB_FIFO_DEPTH2__EQUAD_FIFO_DEPTH_MASK                                                                 0x000000FFL
+#define DB_FIFO_DEPTH2__ETILE_OP_FIFO_DEPTH_MASK                                                              0x0000FF00L
+#define DB_FIFO_DEPTH2__LQUAD_FIFO_DEPTH_MASK                                                                 0x01FF0000L
+#define DB_FIFO_DEPTH2__LTILE_OP_FIFO_DEPTH_MASK                                                              0xFE000000L
+//DB_LAST_OF_BURST_CONFIG
+#define DB_LAST_OF_BURST_CONFIG__MAXBURST__SHIFT                                                              0x0
+#define DB_LAST_OF_BURST_CONFIG__TIMEOUT__SHIFT                                                               0x8
+#define DB_LAST_OF_BURST_CONFIG__DBCB_LOB_SWITCH_TIMEOUT__SHIFT                                               0xb
+#define DB_LAST_OF_BURST_CONFIG__ENABLE_FG_DEFAULT_TIMEOUT__SHIFT                                             0x11
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_MCC_BURST_COUNT_RESET_ON_LOB__SHIFT                                  0x12
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_FLQ_LOB_EVERY_256B__SHIFT                                            0x13
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_ZCACHE_FL_OP_EVEN_ARB__SHIFT                                         0x14
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_MCC_BURST_FORCE_FLUSH_BEFORE_FIFO__SHIFT                             0x15
+#define DB_LAST_OF_BURST_CONFIG__ENABLE_TIMEOUT_DKG_LOB_GEN__SHIFT                                            0x16
+#define DB_LAST_OF_BURST_CONFIG__ENABLE_TIMEOUT_LPF_LOB_GEN__SHIFT                                            0x17
+#define DB_LAST_OF_BURST_CONFIG__ENABLE_TIMEOUT_FL_BURST__SHIFT                                               0x19
+#define DB_LAST_OF_BURST_CONFIG__ENABLE_TIMEOUT_FG_LOB_FWDR__SHIFT                                            0x1a
+#define DB_LAST_OF_BURST_CONFIG__BYPASS_SORT_RD_BA__SHIFT                                                     0x1c
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_256B_COALESCE__SHIFT                                                 0x1d
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_RD_BURST__SHIFT                                                      0x1e
+#define DB_LAST_OF_BURST_CONFIG__LEGACY_LOB_INSERT_EN__SHIFT                                                  0x1f
+#define DB_LAST_OF_BURST_CONFIG__MAXBURST_MASK                                                                0x000000FFL
+#define DB_LAST_OF_BURST_CONFIG__TIMEOUT_MASK                                                                 0x00000700L
+#define DB_LAST_OF_BURST_CONFIG__DBCB_LOB_SWITCH_TIMEOUT_MASK                                                 0x0000F800L
+#define DB_LAST_OF_BURST_CONFIG__ENABLE_FG_DEFAULT_TIMEOUT_MASK                                               0x00020000L
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_MCC_BURST_COUNT_RESET_ON_LOB_MASK                                    0x00040000L
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_FLQ_LOB_EVERY_256B_MASK                                              0x00080000L
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_ZCACHE_FL_OP_EVEN_ARB_MASK                                           0x00100000L
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_MCC_BURST_FORCE_FLUSH_BEFORE_FIFO_MASK                               0x00200000L
+#define DB_LAST_OF_BURST_CONFIG__ENABLE_TIMEOUT_DKG_LOB_GEN_MASK                                              0x00400000L
+#define DB_LAST_OF_BURST_CONFIG__ENABLE_TIMEOUT_LPF_LOB_GEN_MASK                                              0x00800000L
+#define DB_LAST_OF_BURST_CONFIG__ENABLE_TIMEOUT_FL_BURST_MASK                                                 0x02000000L
+#define DB_LAST_OF_BURST_CONFIG__ENABLE_TIMEOUT_FG_LOB_FWDR_MASK                                              0x04000000L
+#define DB_LAST_OF_BURST_CONFIG__BYPASS_SORT_RD_BA_MASK                                                       0x10000000L
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_256B_COALESCE_MASK                                                   0x20000000L
+#define DB_LAST_OF_BURST_CONFIG__DISABLE_RD_BURST_MASK                                                        0x40000000L
+#define DB_LAST_OF_BURST_CONFIG__LEGACY_LOB_INSERT_EN_MASK                                                    0x80000000L
+//DB_RING_CONTROL
+#define DB_RING_CONTROL__COUNTER_CONTROL__SHIFT                                                               0x0
+#define DB_RING_CONTROL__COUNTER_CONTROL_MASK                                                                 0x00000003L
+//DB_MEM_ARB_WATERMARKS
+#define DB_MEM_ARB_WATERMARKS__CLIENT0_WATERMARK__SHIFT                                                       0x0
+#define DB_MEM_ARB_WATERMARKS__CLIENT1_WATERMARK__SHIFT                                                       0x8
+#define DB_MEM_ARB_WATERMARKS__CLIENT2_WATERMARK__SHIFT                                                       0x10
+#define DB_MEM_ARB_WATERMARKS__CLIENT3_WATERMARK__SHIFT                                                       0x18
+#define DB_MEM_ARB_WATERMARKS__CLIENT0_WATERMARK_MASK                                                         0x00000007L
+#define DB_MEM_ARB_WATERMARKS__CLIENT1_WATERMARK_MASK                                                         0x00000700L
+#define DB_MEM_ARB_WATERMARKS__CLIENT2_WATERMARK_MASK                                                         0x00070000L
+#define DB_MEM_ARB_WATERMARKS__CLIENT3_WATERMARK_MASK                                                         0x07000000L
+//DB_FIFO_DEPTH3
+#define DB_FIFO_DEPTH3__LTILE_PROBE_FIFO_DEPTH__SHIFT                                                         0x0
+#define DB_FIFO_DEPTH3__OSB_WAVE_TABLE_DEPTH__SHIFT                                                           0x8
+#define DB_FIFO_DEPTH3__OREO_WAVE_HIDE_DEPTH__SHIFT                                                           0x10
+#define DB_FIFO_DEPTH3__QUAD_READ_REQS__SHIFT                                                                 0x18
+#define DB_FIFO_DEPTH3__LTILE_PROBE_FIFO_DEPTH_MASK                                                           0x000000FFL
+#define DB_FIFO_DEPTH3__OSB_WAVE_TABLE_DEPTH_MASK                                                             0x0000FF00L
+#define DB_FIFO_DEPTH3__OREO_WAVE_HIDE_DEPTH_MASK                                                             0x00FF0000L
+#define DB_FIFO_DEPTH3__QUAD_READ_REQS_MASK                                                                   0xFF000000L
+//DB_DEBUG6
+#define DB_DEBUG6__FORCE_DB_SC_WAVE_CONFLICT__SHIFT                                                           0x0
+#define DB_DEBUG6__FORCE_DB_SC_WAVE_HARD_CONFLICT__SHIFT                                                      0x1
+#define DB_DEBUG6__FORCE_DB_SC_QUAD_CONFLICT__SHIFT                                                           0x2
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_ALL__SHIFT                                                           0x3
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_ID__SHIFT                                                            0x4
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_EN__SHIFT                                                            0xa
+#define DB_DEBUG6__DISABLE_PWS_PLUS_TCP_CM_LIVENESS_STALL__SHIFT                                              0xb
+#define DB_DEBUG6__DISABLE_PWS_PLUS_DTT_TAG_LIVENESS_STALL__SHIFT                                             0xc
+#define DB_DEBUG6__SET_DB_PERFMON_PWS_PIPE_ID__SHIFT                                                          0xd
+#define DB_DEBUG6__FTWB_MAX_TIMEOUT_VAL__SHIFT                                                                0x10
+#define DB_DEBUG6__DISABLE_LQO_SMT_RAM_OPT__SHIFT                                                             0x18
+#define DB_DEBUG6__FORCE_MAX_TILES_IN_WAVE_CHECK__SHIFT                                                       0x19
+#define DB_DEBUG6__DISABLE_OSB_DEADLOCK_FIX__SHIFT                                                            0x1a
+#define DB_DEBUG6__DISABLE_OSB_DEADLOCK_WAIT_PANIC__SHIFT                                                     0x1b
+#define DB_DEBUG6__FORCE_DB_SC_WAVE_CONFLICT_MASK                                                             0x00000001L
+#define DB_DEBUG6__FORCE_DB_SC_WAVE_HARD_CONFLICT_MASK                                                        0x00000002L
+#define DB_DEBUG6__FORCE_DB_SC_QUAD_CONFLICT_MASK                                                             0x00000004L
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_ALL_MASK                                                             0x00000008L
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_ID_MASK                                                              0x000003F0L
+#define DB_DEBUG6__OREO_TRANSITION_EVENT_EN_MASK                                                              0x00000400L
+#define DB_DEBUG6__DISABLE_PWS_PLUS_TCP_CM_LIVENESS_STALL_MASK                                                0x00000800L
+#define DB_DEBUG6__DISABLE_PWS_PLUS_DTT_TAG_LIVENESS_STALL_MASK                                               0x00001000L
+#define DB_DEBUG6__SET_DB_PERFMON_PWS_PIPE_ID_MASK                                                            0x00006000L
+#define DB_DEBUG6__FTWB_MAX_TIMEOUT_VAL_MASK                                                                  0x00FF0000L
+#define DB_DEBUG6__DISABLE_LQO_SMT_RAM_OPT_MASK                                                               0x01000000L
+#define DB_DEBUG6__FORCE_MAX_TILES_IN_WAVE_CHECK_MASK                                                         0x02000000L
+#define DB_DEBUG6__DISABLE_OSB_DEADLOCK_FIX_MASK                                                              0x04000000L
+#define DB_DEBUG6__DISABLE_OSB_DEADLOCK_WAIT_PANIC_MASK                                                       0x08000000L
+//DB_EXCEPTION_CONTROL
+#define DB_EXCEPTION_CONTROL__EARLY_Z_PANIC_DISABLE__SHIFT                                                    0x0
+#define DB_EXCEPTION_CONTROL__LATE_Z_PANIC_DISABLE__SHIFT                                                     0x1
+#define DB_EXCEPTION_CONTROL__RE_Z_PANIC_DISABLE__SHIFT                                                       0x2
+#define DB_EXCEPTION_CONTROL__AUTO_FLUSH_HTILE__SHIFT                                                         0x3
+#define DB_EXCEPTION_CONTROL__AUTO_FLUSH_QUAD__SHIFT                                                          0x4
+#define DB_EXCEPTION_CONTROL__FORCE_SUMMARIZE__SHIFT                                                          0x8
+#define DB_EXCEPTION_CONTROL__DTAG_WATERMARK__SHIFT                                                           0x18
+#define DB_EXCEPTION_CONTROL__EARLY_Z_PANIC_DISABLE_MASK                                                      0x00000001L
+#define DB_EXCEPTION_CONTROL__LATE_Z_PANIC_DISABLE_MASK                                                       0x00000002L
+#define DB_EXCEPTION_CONTROL__RE_Z_PANIC_DISABLE_MASK                                                         0x00000004L
+#define DB_EXCEPTION_CONTROL__AUTO_FLUSH_HTILE_MASK                                                           0x00000008L
+#define DB_EXCEPTION_CONTROL__AUTO_FLUSH_QUAD_MASK                                                            0x00000010L
+#define DB_EXCEPTION_CONTROL__FORCE_SUMMARIZE_MASK                                                            0x00000F00L
+#define DB_EXCEPTION_CONTROL__DTAG_WATERMARK_MASK                                                             0x7F000000L
+//DB_DEBUG7
+#define DB_DEBUG7__SPARE_BITS__SHIFT                                                                          0x0
+#define DB_DEBUG7__SPARE_BITS_MASK                                                                            0xFFFFFFFFL
+//DB_DEBUG5
+#define DB_DEBUG5__DISABLE_TILE_CACHE_PRELOAD__SHIFT                                                          0x0
+#define DB_DEBUG5__ENABLE_SECONDARY_MIPS_TAILS_COMPRESSION__SHIFT                                             0x1
+#define DB_DEBUG5__DISABLE_CLEAR_VALUE_UPDATE_ON_TILE_CACHE_HIT__SHIFT                                        0x2
+#define DB_DEBUG5__DISABLE_2SRC_VRS_HARD_CONFLICT__SHIFT                                                      0x3
+#define DB_DEBUG5__DISABLE_FLQ_MCC_DTILEID_CHECK__SHIFT                                                       0x4
+#define DB_DEBUG5__DISABLE_NOZ_POWER_SAVINGS__SHIFT                                                           0x5
+#define DB_DEBUG5__DISABLE_TILE_INFLIGHT_DEC_POSTZ_FIX__SHIFT                                                 0x6
+#define DB_DEBUG5__DISABLE_MGCG_GATING_ON_SHADER_WAIT__SHIFT                                                  0x7
+#define DB_DEBUG5__DISABLE_VRS_1X2_2XAA__SHIFT                                                                0x8
+#define DB_DEBUG5__ENABLE_FULL_TILE_WAVE_BREAK_ON_COARSE__SHIFT                                               0x9
+#define DB_DEBUG5__DISABLE_HTILE_HARVESTING__SHIFT                                                            0xa
+#define DB_DEBUG5__DISABLE_SEPARATE_TILE_CLK__SHIFT                                                           0xb
+#define DB_DEBUG5__DISABLE_TILE_CACHE_PREFETCH__SHIFT                                                         0xc
+#define DB_DEBUG5__DISABLE_PSL_AUTO_MODE_FIX__SHIFT                                                           0xd
+#define DB_DEBUG5__DISABLE_FORCE_ZMASK_EXPANDED__SHIFT                                                        0xe
+#define DB_DEBUG5__DISABLE_SEPARATE_LQO_CLK__SHIFT                                                            0xf
+#define DB_DEBUG5__DISABLE_Z_WITHOUT_PLANES_FLQ__SHIFT                                                        0x10
+#define DB_DEBUG5__PRESERVE_QMASK_FOR_POSTZ_OP_PIPE__SHIFT                                                    0x11
+#define DB_DEBUG5__Z_NACK_BEHAVIOR_ONLY_WHEN_Z_IS_PRT__SHIFT                                                  0x12
+#define DB_DEBUG5__S_NACK_BEHAVIOR_ONLY_WHEN_S_IS_PRT__SHIFT                                                  0x13
+#define DB_DEBUG5__DISABLE_RESIDENCY_CHECK_Z__SHIFT                                                           0x14
+#define DB_DEBUG5__DISABLE_RESIDENCY_CHECK_STENCIL__SHIFT                                                     0x15
+#define DB_DEBUG5__DISABLE_LQO_FTCQ_DUAL_QUAD_REGION_CHECK__SHIFT                                             0x16
+#define DB_DEBUG5__DISABLE_EVENT_INSERTION_AFTER_ZPC_BEFORE_CONTEXT_DONE__SHIFT                               0x17
+#define DB_DEBUG5__SPARE_BITS__SHIFT                                                                          0x18
+#define DB_DEBUG5__DISABLE_TILE_CACHE_PRELOAD_MASK                                                            0x00000001L
+#define DB_DEBUG5__ENABLE_SECONDARY_MIPS_TAILS_COMPRESSION_MASK                                               0x00000002L
+#define DB_DEBUG5__DISABLE_CLEAR_VALUE_UPDATE_ON_TILE_CACHE_HIT_MASK                                          0x00000004L
+#define DB_DEBUG5__DISABLE_2SRC_VRS_HARD_CONFLICT_MASK                                                        0x00000008L
+#define DB_DEBUG5__DISABLE_FLQ_MCC_DTILEID_CHECK_MASK                                                         0x00000010L
+#define DB_DEBUG5__DISABLE_NOZ_POWER_SAVINGS_MASK                                                             0x00000020L
+#define DB_DEBUG5__DISABLE_TILE_INFLIGHT_DEC_POSTZ_FIX_MASK                                                   0x00000040L
+#define DB_DEBUG5__DISABLE_MGCG_GATING_ON_SHADER_WAIT_MASK                                                    0x00000080L
+#define DB_DEBUG5__DISABLE_VRS_1X2_2XAA_MASK                                                                  0x00000100L
+#define DB_DEBUG5__ENABLE_FULL_TILE_WAVE_BREAK_ON_COARSE_MASK                                                 0x00000200L
+#define DB_DEBUG5__DISABLE_HTILE_HARVESTING_MASK                                                              0x00000400L
+#define DB_DEBUG5__DISABLE_SEPARATE_TILE_CLK_MASK                                                             0x00000800L
+#define DB_DEBUG5__DISABLE_TILE_CACHE_PREFETCH_MASK                                                           0x00001000L
+#define DB_DEBUG5__DISABLE_PSL_AUTO_MODE_FIX_MASK                                                             0x00002000L
+#define DB_DEBUG5__DISABLE_FORCE_ZMASK_EXPANDED_MASK                                                          0x00004000L
+#define DB_DEBUG5__DISABLE_SEPARATE_LQO_CLK_MASK                                                              0x00008000L
+#define DB_DEBUG5__DISABLE_Z_WITHOUT_PLANES_FLQ_MASK                                                          0x00010000L
+#define DB_DEBUG5__PRESERVE_QMASK_FOR_POSTZ_OP_PIPE_MASK                                                      0x00020000L
+#define DB_DEBUG5__Z_NACK_BEHAVIOR_ONLY_WHEN_Z_IS_PRT_MASK                                                    0x00040000L
+#define DB_DEBUG5__S_NACK_BEHAVIOR_ONLY_WHEN_S_IS_PRT_MASK                                                    0x00080000L
+#define DB_DEBUG5__DISABLE_RESIDENCY_CHECK_Z_MASK                                                             0x00100000L
+#define DB_DEBUG5__DISABLE_RESIDENCY_CHECK_STENCIL_MASK                                                       0x00200000L
+#define DB_DEBUG5__DISABLE_LQO_FTCQ_DUAL_QUAD_REGION_CHECK_MASK                                               0x00400000L
+#define DB_DEBUG5__DISABLE_EVENT_INSERTION_AFTER_ZPC_BEFORE_CONTEXT_DONE_MASK                                 0x00800000L
+#define DB_DEBUG5__SPARE_BITS_MASK                                                                            0xFF000000L
+//DB_FGCG_SRAMS_CLK_CTRL
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE0__SHIFT                                                              0x0
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE1__SHIFT                                                              0x1
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE2__SHIFT                                                              0x2
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE3__SHIFT                                                              0x3
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE4__SHIFT                                                              0x4
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE5__SHIFT                                                              0x5
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE6__SHIFT                                                              0x6
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE7__SHIFT                                                              0x7
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE8__SHIFT                                                              0x8
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE9__SHIFT                                                              0x9
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE10__SHIFT                                                             0xa
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE11__SHIFT                                                             0xb
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE12__SHIFT                                                             0xc
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE13__SHIFT                                                             0xd
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE14__SHIFT                                                             0xe
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE15__SHIFT                                                             0xf
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE16__SHIFT                                                             0x10
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE17__SHIFT                                                             0x11
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE18__SHIFT                                                             0x12
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE19__SHIFT                                                             0x13
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE20__SHIFT                                                             0x14
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE21__SHIFT                                                             0x15
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE22__SHIFT                                                             0x16
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE23__SHIFT                                                             0x17
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE24__SHIFT                                                             0x18
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE25__SHIFT                                                             0x19
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE26__SHIFT                                                             0x1a
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE27__SHIFT                                                             0x1b
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE28__SHIFT                                                             0x1c
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE29__SHIFT                                                             0x1d
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE30__SHIFT                                                             0x1e
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE31__SHIFT                                                             0x1f
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE0_MASK                                                                0x00000001L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE1_MASK                                                                0x00000002L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE2_MASK                                                                0x00000004L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE3_MASK                                                                0x00000008L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE4_MASK                                                                0x00000010L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE5_MASK                                                                0x00000020L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE6_MASK                                                                0x00000040L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE7_MASK                                                                0x00000080L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE8_MASK                                                                0x00000100L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE9_MASK                                                                0x00000200L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE10_MASK                                                               0x00000400L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE11_MASK                                                               0x00000800L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE12_MASK                                                               0x00001000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE13_MASK                                                               0x00002000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE14_MASK                                                               0x00004000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE15_MASK                                                               0x00008000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE16_MASK                                                               0x00010000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE17_MASK                                                               0x00020000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE18_MASK                                                               0x00040000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE19_MASK                                                               0x00080000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE20_MASK                                                               0x00100000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE21_MASK                                                               0x00200000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE22_MASK                                                               0x00400000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE23_MASK                                                               0x00800000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE24_MASK                                                               0x01000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE25_MASK                                                               0x02000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE26_MASK                                                               0x04000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE27_MASK                                                               0x08000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE28_MASK                                                               0x10000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE29_MASK                                                               0x20000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE30_MASK                                                               0x40000000L
+#define DB_FGCG_SRAMS_CLK_CTRL__OVERRIDE31_MASK                                                               0x80000000L
+//DB_FGCG_INTERFACES_CLK_CTRL
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_QUAD_OVERRIDE__SHIFT                                               0x0
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_CB_EXPORT_OVERRIDE__SHIFT                                             0x2
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_RMI_RDREQ_OVERRIDE__SHIFT                                             0x3
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_RMI_WRREQ_OVERRIDE__SHIFT                                             0x4
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_TILE_OVERRIDE__SHIFT                                               0x5
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_CB_RMIRET_OVERRIDE__SHIFT                                             0x6
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_WAVE_OVERRIDE__SHIFT                                               0x7
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_FREE_WAVE_OVERRIDE__SHIFT                                          0x8
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_QUAD_OVERRIDE_MASK                                                 0x00000001L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_CB_EXPORT_OVERRIDE_MASK                                               0x00000004L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_RMI_RDREQ_OVERRIDE_MASK                                               0x00000008L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_RMI_WRREQ_OVERRIDE_MASK                                               0x00000010L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_TILE_OVERRIDE_MASK                                                 0x00000020L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_CB_RMIRET_OVERRIDE_MASK                                               0x00000040L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_WAVE_OVERRIDE_MASK                                                 0x00000080L
+#define DB_FGCG_INTERFACES_CLK_CTRL__DB_SC_FREE_WAVE_OVERRIDE_MASK                                            0x00000100L
+//DB_FIFO_DEPTH4
+#define DB_FIFO_DEPTH4__OSB_SQUAD_TABLE_DEPTH__SHIFT                                                          0x0
+#define DB_FIFO_DEPTH4__OSB_TILE_TABLE_DEPTH__SHIFT                                                           0x8
+#define DB_FIFO_DEPTH4__OSB_SCORE_BOARD_DEPTH__SHIFT                                                          0x10
+#define DB_FIFO_DEPTH4__OSB_EVENT_FIFO_DEPTH__SHIFT                                                           0x18
+#define DB_FIFO_DEPTH4__OSB_SQUAD_TABLE_DEPTH_MASK                                                            0x000000FFL
+#define DB_FIFO_DEPTH4__OSB_TILE_TABLE_DEPTH_MASK                                                             0x0000FF00L
+#define DB_FIFO_DEPTH4__OSB_SCORE_BOARD_DEPTH_MASK                                                            0x00FF0000L
+#define DB_FIFO_DEPTH4__OSB_EVENT_FIFO_DEPTH_MASK                                                             0xFF000000L
+//CC_RB_REDUNDANCY
+#define CC_RB_REDUNDANCY__WRITE_DIS__SHIFT                                                                    0x0
+#define CC_RB_REDUNDANCY__FAILED_RB0__SHIFT                                                                   0x8
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT                                                               0xc
+#define CC_RB_REDUNDANCY__FAILED_RB1__SHIFT                                                                   0x10
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT                                                               0x14
+#define CC_RB_REDUNDANCY__WRITE_DIS_MASK                                                                      0x00000001L
+#define CC_RB_REDUNDANCY__FAILED_RB0_MASK                                                                     0x00000F00L
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY0_MASK                                                                 0x00001000L
+#define CC_RB_REDUNDANCY__FAILED_RB1_MASK                                                                     0x000F0000L
+#define CC_RB_REDUNDANCY__EN_REDUNDANCY1_MASK                                                                 0x00100000L
+//CC_RB_BACKEND_DISABLE
+#define CC_RB_BACKEND_DISABLE__WRITE_DIS__SHIFT                                                               0x0
+#define CC_RB_BACKEND_DISABLE__RESERVED__SHIFT                                                                0x2
+#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT                                                         0x4
+#define CC_RB_BACKEND_DISABLE__WRITE_DIS_MASK                                                                 0x00000001L
+#define CC_RB_BACKEND_DISABLE__RESERVED_MASK                                                                  0x0000000CL
+#define CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK                                                           0xFFFFFFF0L
+//GB_ADDR_CONFIG
+#define GB_ADDR_CONFIG__NUM_PIPES__SHIFT                                                                      0x0
+#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                           0x3
+#define GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS__SHIFT                                                           0x6
+#define GB_ADDR_CONFIG__NUM_PKRS__SHIFT                                                                       0x8
+#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT                                                             0x13
+#define GB_ADDR_CONFIG__NUM_RB_PER_SE__SHIFT                                                                  0x1a
+#define GB_ADDR_CONFIG__NUM_PIPES_MASK                                                                        0x00000007L
+#define GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                             0x00000038L
+#define GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS_MASK                                                             0x000000C0L
+#define GB_ADDR_CONFIG__NUM_PKRS_MASK                                                                         0x00000700L
+#define GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK                                                               0x00180000L
+#define GB_ADDR_CONFIG__NUM_RB_PER_SE_MASK                                                                    0x0C000000L
+//GB_BACKEND_MAP
+#define GB_BACKEND_MAP__BACKEND_MAP__SHIFT                                                                    0x0
+#define GB_BACKEND_MAP__BACKEND_MAP_MASK                                                                      0xFFFFFFFFL
+//GB_GPU_ID
+#define GB_GPU_ID__GPU_ID__SHIFT                                                                              0x0
+#define GB_GPU_ID__GPU_ID_MASK                                                                                0x0000000FL
+//CC_RB_DAISY_CHAIN
+#define CC_RB_DAISY_CHAIN__RB_0__SHIFT                                                                        0x0
+#define CC_RB_DAISY_CHAIN__RB_1__SHIFT                                                                        0x4
+#define CC_RB_DAISY_CHAIN__RB_2__SHIFT                                                                        0x8
+#define CC_RB_DAISY_CHAIN__RB_3__SHIFT                                                                        0xc
+#define CC_RB_DAISY_CHAIN__RB_4__SHIFT                                                                        0x10
+#define CC_RB_DAISY_CHAIN__RB_5__SHIFT                                                                        0x14
+#define CC_RB_DAISY_CHAIN__RB_6__SHIFT                                                                        0x18
+#define CC_RB_DAISY_CHAIN__RB_7__SHIFT                                                                        0x1c
+#define CC_RB_DAISY_CHAIN__RB_0_MASK                                                                          0x0000000FL
+#define CC_RB_DAISY_CHAIN__RB_1_MASK                                                                          0x000000F0L
+#define CC_RB_DAISY_CHAIN__RB_2_MASK                                                                          0x00000F00L
+#define CC_RB_DAISY_CHAIN__RB_3_MASK                                                                          0x0000F000L
+#define CC_RB_DAISY_CHAIN__RB_4_MASK                                                                          0x000F0000L
+#define CC_RB_DAISY_CHAIN__RB_5_MASK                                                                          0x00F00000L
+#define CC_RB_DAISY_CHAIN__RB_6_MASK                                                                          0x0F000000L
+#define CC_RB_DAISY_CHAIN__RB_7_MASK                                                                          0xF0000000L
+//GB_ADDR_CONFIG_READ
+#define GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                                 0x0
+#define GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                      0x3
+#define GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS__SHIFT                                                      0x6
+#define GB_ADDR_CONFIG_READ__NUM_PKRS__SHIFT                                                                  0x8
+#define GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                        0x13
+#define GB_ADDR_CONFIG_READ__NUM_RB_PER_SE__SHIFT                                                             0x1a
+#define GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                                   0x00000007L
+#define GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                        0x00000038L
+#define GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS_MASK                                                        0x000000C0L
+#define GB_ADDR_CONFIG_READ__NUM_PKRS_MASK                                                                    0x00000700L
+#define GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                          0x00180000L
+#define GB_ADDR_CONFIG_READ__NUM_RB_PER_SE_MASK                                                               0x0C000000L
+//CB_HW_CONTROL_4
+#define CB_HW_CONTROL_4__COLOR_CACHE_FETCH_NUM_QB_LOG2__SHIFT                                                 0x0
+#define CB_HW_CONTROL_4__COLOR_CACHE_FETCH_ALGORITHM__SHIFT                                                   0x3
+#define CB_HW_CONTROL_4__DISABLE_USE_OF_SMT_SCORE__SHIFT                                                      0x5
+#define CB_HW_CONTROL_4__SPARE_10__SHIFT                                                                      0x6
+#define CB_HW_CONTROL_4__SPARE_11__SHIFT                                                                      0x7
+#define CB_HW_CONTROL_4__SPARE_12__SHIFT                                                                      0x8
+#define CB_HW_CONTROL_4__DISABLE_MA_WAIT_FOR_LAST__SHIFT                                                      0x9
+#define CB_HW_CONTROL_4__SMT_TIMEOUT_THRESHOLD__SHIFT                                                         0xa
+#define CB_HW_CONTROL_4__SMT_QPFIFO_THRESHOLD__SHIFT                                                          0xd
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_RAW_HAZARD__SHIFT                                          0x10
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_COARSE_RAW_HAZARD__SHIFT                                   0x11
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_DS_RAW_HAZARD__SHIFT                                       0x12
+#define CB_HW_CONTROL_4__COLOR_CACHE_FETCH_NUM_QB_LOG2_MASK                                                   0x00000007L
+#define CB_HW_CONTROL_4__COLOR_CACHE_FETCH_ALGORITHM_MASK                                                     0x00000018L
+#define CB_HW_CONTROL_4__DISABLE_USE_OF_SMT_SCORE_MASK                                                        0x00000020L
+#define CB_HW_CONTROL_4__SPARE_10_MASK                                                                        0x00000040L
+#define CB_HW_CONTROL_4__SPARE_11_MASK                                                                        0x00000080L
+#define CB_HW_CONTROL_4__SPARE_12_MASK                                                                        0x00000100L
+#define CB_HW_CONTROL_4__DISABLE_MA_WAIT_FOR_LAST_MASK                                                        0x00000200L
+#define CB_HW_CONTROL_4__SMT_TIMEOUT_THRESHOLD_MASK                                                           0x00001C00L
+#define CB_HW_CONTROL_4__SMT_QPFIFO_THRESHOLD_MASK                                                            0x0000E000L
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_RAW_HAZARD_MASK                                            0x00010000L
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_COARSE_RAW_HAZARD_MASK                                     0x00020000L
+#define CB_HW_CONTROL_4__ENABLE_FRAGOP_STALLING_ON_DS_RAW_HAZARD_MASK                                         0x00040000L
+//CB_HW_CONTROL_3
+#define CB_HW_CONTROL_3__SPARE_5__SHIFT                                                                       0x0
+#define CB_HW_CONTROL_3__RAM_ADDRESS_CONFLICTS_DISALLOWED__SHIFT                                              0x1
+#define CB_HW_CONTROL_3__SPARE_6__SHIFT                                                                       0x2
+#define CB_HW_CONTROL_3__SPARE_7__SHIFT                                                                       0x3
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_STATUS_ACCUM__SHIFT                                            0x4
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_PANIC_GATING__SHIFT                                                 0x5
+#define CB_HW_CONTROL_3__SPLIT_ALL_FAST_MODE_TRANSFERS__SHIFT                                                 0x6
+#define CB_HW_CONTROL_3__DISABLE_SHADER_BLEND_OPTS__SHIFT                                                     0x7
+#define CB_HW_CONTROL_3__FORCE_RMI_LAST_HIGH__SHIFT                                                           0xb
+#define CB_HW_CONTROL_3__FORCE_RMI_CLKEN_HIGH__SHIFT                                                          0xc
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_CC__SHIFT                                                       0xd
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_DC__SHIFT                                                       0xe
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CC__SHIFT                                                    0xf
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_DC__SHIFT                                                    0x10
+#define CB_HW_CONTROL_3__SPARE_8__SHIFT                                                                       0x11
+#define CB_HW_CONTROL_3__SPARE_9__SHIFT                                                                       0x12
+#define CB_HW_CONTROL_3__DISABLE_DCC_VRS_OPT__SHIFT                                                           0x14
+#define CB_HW_CONTROL_3__DISABLE_FMASK_NOALLOC_OPT__SHIFT                                                     0x15
+#define CB_HW_CONTROL_3__SPARE_5_MASK                                                                         0x00000001L
+#define CB_HW_CONTROL_3__RAM_ADDRESS_CONFLICTS_DISALLOWED_MASK                                                0x00000002L
+#define CB_HW_CONTROL_3__SPARE_6_MASK                                                                         0x00000004L
+#define CB_HW_CONTROL_3__SPARE_7_MASK                                                                         0x00000008L
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_OVWR_STATUS_ACCUM_MASK                                              0x00000010L
+#define CB_HW_CONTROL_3__DISABLE_CC_CACHE_PANIC_GATING_MASK                                                   0x00000020L
+#define CB_HW_CONTROL_3__SPLIT_ALL_FAST_MODE_TRANSFERS_MASK                                                   0x00000040L
+#define CB_HW_CONTROL_3__DISABLE_SHADER_BLEND_OPTS_MASK                                                       0x00000080L
+#define CB_HW_CONTROL_3__FORCE_RMI_LAST_HIGH_MASK                                                             0x00000800L
+#define CB_HW_CONTROL_3__FORCE_RMI_CLKEN_HIGH_MASK                                                            0x00001000L
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_CC_MASK                                                         0x00002000L
+#define CB_HW_CONTROL_3__DISABLE_EARLY_WRACKS_DC_MASK                                                         0x00004000L
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_CC_MASK                                                      0x00008000L
+#define CB_HW_CONTROL_3__DISABLE_NACK_PROCESSING_DC_MASK                                                      0x00010000L
+#define CB_HW_CONTROL_3__SPARE_8_MASK                                                                         0x00020000L
+#define CB_HW_CONTROL_3__SPARE_9_MASK                                                                         0x00040000L
+#define CB_HW_CONTROL_3__DISABLE_DCC_VRS_OPT_MASK                                                             0x00100000L
+#define CB_HW_CONTROL_3__DISABLE_FMASK_NOALLOC_OPT_MASK                                                       0x00200000L
+//CB_HW_CONTROL
+#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE__SHIFT                                                      0x0
+#define CB_HW_CONTROL__DISABLE_VRS_FILLRATE_OPTIMIZATION__SHIFT                                               0x1
+#define CB_HW_CONTROL__DISABLE_SMT_WHEN_NO_FDCC_FIX__SHIFT                                                    0x2
+#define CB_HW_CONTROL__RMI_CREDITS__SHIFT                                                                     0x6
+#define CB_HW_CONTROL__NUM_CCC_SKID_FIFO_ENTRIES__SHIFT                                                       0xc
+#define CB_HW_CONTROL__FORCE_FEA_HIGH__SHIFT                                                                  0xf
+#define CB_HW_CONTROL__FORCE_EVICT_ALL_VALID__SHIFT                                                           0x10
+#define CB_HW_CONTROL__DISABLE_DCC_CACHE_BYTEMASKING__SHIFT                                                   0x11
+#define CB_HW_CONTROL__FORCE_NEEDS_DST__SHIFT                                                                 0x13
+#define CB_HW_CONTROL__DISABLE_USE_OF_SET_HASH__SHIFT                                                         0x14
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST__SHIFT                                                0x15
+#define CB_HW_CONTROL__SPARE_2__SHIFT                                                                         0x16
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST__SHIFT                                                   0x18
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS__SHIFT                                                        0x19
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL__SHIFT                                                 0x1a
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED__SHIFT                                0x1b
+#define CB_HW_CONTROL__SPARE_3__SHIFT                                                                         0x1d
+#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT__SHIFT                                              0x1e
+#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE__SHIFT                                    0x1f
+#define CB_HW_CONTROL__ALLOW_MRT_WITH_DUAL_SOURCE_MASK                                                        0x00000001L
+#define CB_HW_CONTROL__DISABLE_VRS_FILLRATE_OPTIMIZATION_MASK                                                 0x00000002L
+#define CB_HW_CONTROL__DISABLE_SMT_WHEN_NO_FDCC_FIX_MASK                                                      0x00000004L
+#define CB_HW_CONTROL__RMI_CREDITS_MASK                                                                       0x00000FC0L
+#define CB_HW_CONTROL__NUM_CCC_SKID_FIFO_ENTRIES_MASK                                                         0x00007000L
+#define CB_HW_CONTROL__FORCE_FEA_HIGH_MASK                                                                    0x00008000L
+#define CB_HW_CONTROL__FORCE_EVICT_ALL_VALID_MASK                                                             0x00010000L
+#define CB_HW_CONTROL__DISABLE_DCC_CACHE_BYTEMASKING_MASK                                                     0x00020000L
+#define CB_HW_CONTROL__FORCE_NEEDS_DST_MASK                                                                   0x00080000L
+#define CB_HW_CONTROL__DISABLE_USE_OF_SET_HASH_MASK                                                           0x00100000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_RESULT_EQ_DEST_MASK                                                  0x00200000L
+#define CB_HW_CONTROL__SPARE_2_MASK                                                                           0x00400000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DONT_RD_DST_MASK                                                     0x01000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_BYPASS_MASK                                                          0x02000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_DISCARD_PIXEL_MASK                                                   0x04000000L
+#define CB_HW_CONTROL__DISABLE_BLEND_OPT_WHEN_DISABLED_SRCALPHA_IS_USED_MASK                                  0x08000000L
+#define CB_HW_CONTROL__SPARE_3_MASK                                                                           0x20000000L
+#define CB_HW_CONTROL__DISABLE_CC_IB_SERIALIZER_STATE_OPT_MASK                                                0x40000000L
+#define CB_HW_CONTROL__DISABLE_PIXEL_IN_QUAD_FIX_FOR_LINEAR_SURFACE_MASK                                      0x80000000L
+//CB_HW_CONTROL_1
+#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS__SHIFT                                                             0x0
+#define CB_HW_CONTROL_1__CC_CACHE_NUM_TAGS_MASK                                                               0x0000003FL
+//CB_HW_CONTROL_2
+#define CB_HW_CONTROL_2__SPARE_4__SHIFT                                                                       0x0
+#define CB_HW_CONTROL_2__DRR_ASSUMED_FIFO_DEPTH_DIV8__SHIFT                                                   0x8
+#define CB_HW_CONTROL_2__SPARE__SHIFT                                                                         0xe
+#define CB_HW_CONTROL_2__SPARE_4_MASK                                                                         0x000000FFL
+#define CB_HW_CONTROL_2__DRR_ASSUMED_FIFO_DEPTH_DIV8_MASK                                                     0x00003F00L
+#define CB_HW_CONTROL_2__SPARE_MASK                                                                           0xFFFFC000L
+//CB_DCC_CONFIG
+#define CB_DCC_CONFIG__SAMPLE_MASK_TRACKER_DEPTH__SHIFT                                                       0x0
+#define CB_DCC_CONFIG__SAMPLE_MASK_TRACKER_DISABLE__SHIFT                                                     0x5
+#define CB_DCC_CONFIG__SPARE_13__SHIFT                                                                        0x6
+#define CB_DCC_CONFIG__DISABLE_CONSTANT_ENCODE__SHIFT                                                         0x7
+#define CB_DCC_CONFIG__SPARE_14__SHIFT                                                                        0x8
+#define CB_DCC_CONFIG__READ_RETURN_SKID_FIFO_DEPTH__SHIFT                                                     0x10
+#define CB_DCC_CONFIG__DCC_CACHE_NUM_TAGS__SHIFT                                                              0x19
+#define CB_DCC_CONFIG__SAMPLE_MASK_TRACKER_DEPTH_MASK                                                         0x0000001FL
+#define CB_DCC_CONFIG__SAMPLE_MASK_TRACKER_DISABLE_MASK                                                       0x00000020L
+#define CB_DCC_CONFIG__SPARE_13_MASK                                                                          0x00000040L
+#define CB_DCC_CONFIG__DISABLE_CONSTANT_ENCODE_MASK                                                           0x00000080L
+#define CB_DCC_CONFIG__SPARE_14_MASK                                                                          0x0000FF00L
+#define CB_DCC_CONFIG__READ_RETURN_SKID_FIFO_DEPTH_MASK                                                       0x01FF0000L
+#define CB_DCC_CONFIG__DCC_CACHE_NUM_TAGS_MASK                                                                0xFE000000L
+//CB_HW_MEM_ARBITER_RD
+#define CB_HW_MEM_ARBITER_RD__MODE__SHIFT                                                                     0x0
+#define CB_HW_MEM_ARBITER_RD__IGNORE_URGENT_AGE__SHIFT                                                        0x2
+#define CB_HW_MEM_ARBITER_RD__BREAK_GROUP_AGE__SHIFT                                                          0x6
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_CC__SHIFT                                                                0xa
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DC__SHIFT                                                                0xc
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_REQS__SHIFT                                                        0xe
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_NOREQS__SHIFT                                                      0x10
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_IGNORE_NUM_TIDS__SHIFT                                                   0x12
+#define CB_HW_MEM_ARBITER_RD__SCALE_AGE__SHIFT                                                                0x13
+#define CB_HW_MEM_ARBITER_RD__SCALE_WEIGHT__SHIFT                                                             0x16
+#define CB_HW_MEM_ARBITER_RD__SEND_LASTS_WITHIN_GROUPS__SHIFT                                                 0x19
+#define CB_HW_MEM_ARBITER_RD__MODE_MASK                                                                       0x00000003L
+#define CB_HW_MEM_ARBITER_RD__IGNORE_URGENT_AGE_MASK                                                          0x0000003CL
+#define CB_HW_MEM_ARBITER_RD__BREAK_GROUP_AGE_MASK                                                            0x000003C0L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_CC_MASK                                                                  0x00000C00L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DC_MASK                                                                  0x00003000L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_REQS_MASK                                                          0x0000C000L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_DECAY_NOREQS_MASK                                                        0x00030000L
+#define CB_HW_MEM_ARBITER_RD__WEIGHT_IGNORE_NUM_TIDS_MASK                                                     0x00040000L
+#define CB_HW_MEM_ARBITER_RD__SCALE_AGE_MASK                                                                  0x00380000L
+#define CB_HW_MEM_ARBITER_RD__SCALE_WEIGHT_MASK                                                               0x01C00000L
+#define CB_HW_MEM_ARBITER_RD__SEND_LASTS_WITHIN_GROUPS_MASK                                                   0x02000000L
+//CB_HW_MEM_ARBITER_WR
+#define CB_HW_MEM_ARBITER_WR__MODE__SHIFT                                                                     0x0
+#define CB_HW_MEM_ARBITER_WR__IGNORE_URGENT_AGE__SHIFT                                                        0x2
+#define CB_HW_MEM_ARBITER_WR__BREAK_GROUP_AGE__SHIFT                                                          0x6
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_CC__SHIFT                                                                0xa
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DC__SHIFT                                                                0xc
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_REQS__SHIFT                                                        0xe
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_NOREQS__SHIFT                                                      0x10
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_IGNORE_BYTE_MASK__SHIFT                                                  0x12
+#define CB_HW_MEM_ARBITER_WR__SCALE_AGE__SHIFT                                                                0x13
+#define CB_HW_MEM_ARBITER_WR__SCALE_WEIGHT__SHIFT                                                             0x16
+#define CB_HW_MEM_ARBITER_WR__SEND_LASTS_WITHIN_GROUPS__SHIFT                                                 0x19
+#define CB_HW_MEM_ARBITER_WR__MODE_MASK                                                                       0x00000003L
+#define CB_HW_MEM_ARBITER_WR__IGNORE_URGENT_AGE_MASK                                                          0x0000003CL
+#define CB_HW_MEM_ARBITER_WR__BREAK_GROUP_AGE_MASK                                                            0x000003C0L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_CC_MASK                                                                  0x00000C00L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DC_MASK                                                                  0x00003000L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_REQS_MASK                                                          0x0000C000L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_DECAY_NOREQS_MASK                                                        0x00030000L
+#define CB_HW_MEM_ARBITER_WR__WEIGHT_IGNORE_BYTE_MASK_MASK                                                    0x00040000L
+#define CB_HW_MEM_ARBITER_WR__SCALE_AGE_MASK                                                                  0x00380000L
+#define CB_HW_MEM_ARBITER_WR__SCALE_WEIGHT_MASK                                                               0x01C00000L
+#define CB_HW_MEM_ARBITER_WR__SEND_LASTS_WITHIN_GROUPS_MASK                                                   0x02000000L
+//CB_FGCG_SRAM_OVERRIDE
+#define CB_FGCG_SRAM_OVERRIDE__DISABLE_FGCG__SHIFT                                                            0x0
+#define CB_FGCG_SRAM_OVERRIDE__DISABLE_FGCG_MASK                                                              0x000FFFFFL
+//CB_DCC_CONFIG2
+#define CB_DCC_CONFIG2__INVALID_KEY_ERROR_CODE__SHIFT                                                         0x0
+#define CB_DCC_CONFIG2__CLEAR_FRAG2DCC_KEY_ERROR_CODE__SHIFT                                                  0x8
+#define CB_DCC_CONFIG2__ENABLE_COMP_KEY_ERROR_DETECTION__SHIFT                                                0x9
+#define CB_DCC_CONFIG2__INVALID_KEY_ERROR_CODE_MASK                                                           0x000000FFL
+#define CB_DCC_CONFIG2__CLEAR_FRAG2DCC_KEY_ERROR_CODE_MASK                                                    0x00000100L
+#define CB_DCC_CONFIG2__ENABLE_COMP_KEY_ERROR_DETECTION_MASK                                                  0x00000200L
+//CHICKEN_BITS
+#define CHICKEN_BITS__SPARE__SHIFT                                                                            0x0
+#define CHICKEN_BITS__SPARE_MASK                                                                              0xFFFFFFFFL
+//CB_CACHE_EVICT_POINTS
+#define CB_CACHE_EVICT_POINTS__CC_COLOR_EVICT_POINT__SHIFT                                                    0x0
+#define CB_CACHE_EVICT_POINTS__CC_FMASK_EVICT_POINT__SHIFT                                                    0x8
+#define CB_CACHE_EVICT_POINTS__DCC_CACHE_EVICT_POINT__SHIFT                                                   0x10
+#define CB_CACHE_EVICT_POINTS__CC_CACHE_EVICT_POINT__SHIFT                                                    0x18
+#define CB_CACHE_EVICT_POINTS__CC_COLOR_EVICT_POINT_MASK                                                      0x000000FFL
+#define CB_CACHE_EVICT_POINTS__CC_FMASK_EVICT_POINT_MASK                                                      0x0000FF00L
+#define CB_CACHE_EVICT_POINTS__DCC_CACHE_EVICT_POINT_MASK                                                     0x00FF0000L
+#define CB_CACHE_EVICT_POINTS__CC_CACHE_EVICT_POINT_MASK                                                      0xFF000000L
+
+
+// addressBlock: gc_gceadec
+//GCEA_DRAM_RD_CLI2GRP_MAP0
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//GCEA_DRAM_RD_CLI2GRP_MAP1
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define GCEA_DRAM_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//GCEA_DRAM_WR_CLI2GRP_MAP0
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                          0x0
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                          0x2
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                          0x4
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                          0x6
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                          0x8
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                          0xa
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                          0xc
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                          0xe
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                          0x10
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                          0x12
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                         0x14
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                         0x16
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                         0x18
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                         0x1a
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                         0x1c
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                         0x1e
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                            0x00000003L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                            0x0000000CL
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                            0x00000030L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                            0x000000C0L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                            0x00000300L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                            0x00000C00L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                            0x00003000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                            0x0000C000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                            0x00030000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                            0x000C0000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                           0x00300000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                           0x00C00000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                           0x03000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                           0x0C000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                           0x30000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                           0xC0000000L
+//GCEA_DRAM_WR_CLI2GRP_MAP1
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                         0x0
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                         0x2
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                         0x4
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                         0x6
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                         0x8
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                         0xa
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                         0xc
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                         0xe
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                         0x10
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                         0x12
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                         0x14
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                         0x16
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                         0x18
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                         0x1a
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                         0x1c
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                         0x1e
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                           0x00000003L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                           0x0000000CL
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                           0x00000030L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                           0x000000C0L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                           0x00000300L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                           0x00000C00L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                           0x00003000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                           0x0000C000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                           0x00030000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                           0x000C0000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                           0x00300000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                           0x00C00000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                           0x03000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                           0x0C000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                           0x30000000L
+#define GCEA_DRAM_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                           0xC0000000L
+//GCEA_DRAM_RD_GRP2VC_MAP
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define GCEA_DRAM_RD_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//GCEA_DRAM_WR_GRP2VC_MAP
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP0_VC__SHIFT                                                             0x0
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP1_VC__SHIFT                                                             0x3
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP2_VC__SHIFT                                                             0x6
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP3_VC__SHIFT                                                             0x9
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP0_VC_MASK                                                               0x00000007L
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP1_VC_MASK                                                               0x00000038L
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP2_VC_MASK                                                               0x000001C0L
+#define GCEA_DRAM_WR_GRP2VC_MAP__GROUP3_VC_MASK                                                               0x00000E00L
+//GCEA_DRAM_RD_LAZY
+#define GCEA_DRAM_RD_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define GCEA_DRAM_RD_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define GCEA_DRAM_RD_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define GCEA_DRAM_RD_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define GCEA_DRAM_RD_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define GCEA_DRAM_RD_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define GCEA_DRAM_RD_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define GCEA_DRAM_RD_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define GCEA_DRAM_RD_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//GCEA_DRAM_WR_LAZY
+#define GCEA_DRAM_WR_LAZY__GROUP0_DELAY__SHIFT                                                                0x0
+#define GCEA_DRAM_WR_LAZY__GROUP1_DELAY__SHIFT                                                                0x3
+#define GCEA_DRAM_WR_LAZY__GROUP2_DELAY__SHIFT                                                                0x6
+#define GCEA_DRAM_WR_LAZY__GROUP3_DELAY__SHIFT                                                                0x9
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_THRESH__SHIFT                                                            0xc
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT__SHIFT                                                           0x14
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX__SHIFT                                                           0x1b
+#define GCEA_DRAM_WR_LAZY__GROUP0_DELAY_MASK                                                                  0x00000007L
+#define GCEA_DRAM_WR_LAZY__GROUP1_DELAY_MASK                                                                  0x00000038L
+#define GCEA_DRAM_WR_LAZY__GROUP2_DELAY_MASK                                                                  0x000001C0L
+#define GCEA_DRAM_WR_LAZY__GROUP3_DELAY_MASK                                                                  0x00000E00L
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_THRESH_MASK                                                              0x0003F000L
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_TIMEOUT_MASK                                                             0x07F00000L
+#define GCEA_DRAM_WR_LAZY__REQ_ACCUM_IDLEMAX_MASK                                                             0x78000000L
+//GCEA_DRAM_RD_CAM_CNTL
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define GCEA_DRAM_RD_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define GCEA_DRAM_RD_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define GCEA_DRAM_RD_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define GCEA_DRAM_RD_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+//GCEA_DRAM_WR_CAM_CNTL
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP0__SHIFT                                                            0x0
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP1__SHIFT                                                            0x4
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP2__SHIFT                                                            0x8
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP3__SHIFT                                                            0xc
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0__SHIFT                                                    0x10
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1__SHIFT                                                    0x13
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2__SHIFT                                                    0x16
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3__SHIFT                                                    0x19
+#define GCEA_DRAM_WR_CAM_CNTL__REFILL_CHAIN__SHIFT                                                            0x1c
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP0_MASK                                                              0x0000000FL
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP1_MASK                                                              0x000000F0L
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP2_MASK                                                              0x00000F00L
+#define GCEA_DRAM_WR_CAM_CNTL__DEPTH_GROUP3_MASK                                                              0x0000F000L
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP0_MASK                                                      0x00070000L
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP1_MASK                                                      0x00380000L
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP2_MASK                                                      0x01C00000L
+#define GCEA_DRAM_WR_CAM_CNTL__REORDER_LIMIT_GROUP3_MASK                                                      0x0E000000L
+#define GCEA_DRAM_WR_CAM_CNTL__REFILL_CHAIN_MASK                                                              0x10000000L
+//GCEA_DRAM_PAGE_BURST
+#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_LO__SHIFT                                                              0x0
+#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_HI__SHIFT                                                              0x8
+#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_LO__SHIFT                                                              0x10
+#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_HI__SHIFT                                                              0x18
+#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_LO_MASK                                                                0x000000FFL
+#define GCEA_DRAM_PAGE_BURST__RD_LIMIT_HI_MASK                                                                0x0000FF00L
+#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_LO_MASK                                                                0x00FF0000L
+#define GCEA_DRAM_PAGE_BURST__WR_LIMIT_HI_MASK                                                                0xFF000000L
+//GCEA_DRAM_RD_PRI_AGE
+#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define GCEA_DRAM_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//GCEA_DRAM_WR_PRI_AGE
+#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                        0x0
+#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                        0x3
+#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                        0x6
+#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                        0x9
+#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                   0xc
+#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                   0xf
+#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                   0x12
+#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                   0x15
+#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                          0x00000007L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                          0x00000038L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                          0x000001C0L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                          0x00000E00L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                     0x00007000L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                     0x00038000L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                     0x001C0000L
+#define GCEA_DRAM_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                     0x00E00000L
+//GCEA_DRAM_RD_PRI_QUEUING
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define GCEA_DRAM_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//GCEA_DRAM_WR_PRI_QUEUING
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                           0x0
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                           0x3
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                           0x6
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                           0x9
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                             0x00000007L
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                             0x00000038L
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                             0x000001C0L
+#define GCEA_DRAM_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                             0x00000E00L
+//GCEA_DRAM_RD_PRI_FIXED
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define GCEA_DRAM_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//GCEA_DRAM_WR_PRI_FIXED
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                               0x0
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                               0x3
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                               0x6
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                               0x9
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                 0x00000007L
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                 0x00000038L
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                 0x000001C0L
+#define GCEA_DRAM_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                 0x00000E00L
+//GCEA_DRAM_RD_PRI_URGENCY
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define GCEA_DRAM_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//GCEA_DRAM_WR_PRI_URGENCY
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                           0x0
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                           0x3
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                           0x6
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                           0x9
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                  0xc
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                  0xd
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                  0xe
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                  0xf
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                             0x00000007L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                             0x00000038L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                             0x000001C0L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                             0x00000E00L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                    0x00001000L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                    0x00002000L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                    0x00004000L
+#define GCEA_DRAM_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                    0x00008000L
+//GCEA_DRAM_RD_PRI_QUANT_PRI1
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_DRAM_RD_PRI_QUANT_PRI2
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_DRAM_RD_PRI_QUANT_PRI3
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_DRAM_WR_PRI_QUANT_PRI1
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_DRAM_WR_PRI_QUANT_PRI2
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_DRAM_WR_PRI_QUANT_PRI3
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                  0x0
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                  0x8
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                  0x10
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                  0x18
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                    0x000000FFL
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                    0x0000FF00L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                    0x00FF0000L
+#define GCEA_DRAM_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                    0xFF000000L
+//GCEA_IO_RD_CLI2GRP_MAP0
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                            0x0
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                            0x2
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                            0x4
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                            0x6
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                            0x8
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                            0xa
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                            0xc
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                            0xe
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                            0x10
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                            0x12
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                           0x14
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                           0x16
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                           0x18
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                           0x1a
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                           0x1c
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                           0x1e
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID0_GROUP_MASK                                                              0x00000003L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID1_GROUP_MASK                                                              0x0000000CL
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID2_GROUP_MASK                                                              0x00000030L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID3_GROUP_MASK                                                              0x000000C0L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID4_GROUP_MASK                                                              0x00000300L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID5_GROUP_MASK                                                              0x00000C00L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID6_GROUP_MASK                                                              0x00003000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID7_GROUP_MASK                                                              0x0000C000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID8_GROUP_MASK                                                              0x00030000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID9_GROUP_MASK                                                              0x000C0000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID10_GROUP_MASK                                                             0x00300000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID11_GROUP_MASK                                                             0x00C00000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID12_GROUP_MASK                                                             0x03000000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID13_GROUP_MASK                                                             0x0C000000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID14_GROUP_MASK                                                             0x30000000L
+#define GCEA_IO_RD_CLI2GRP_MAP0__CID15_GROUP_MASK                                                             0xC0000000L
+//GCEA_IO_RD_CLI2GRP_MAP1
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                           0x0
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                           0x2
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                           0x4
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                           0x6
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                           0x8
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                           0xa
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                           0xc
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                           0xe
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                           0x10
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                           0x12
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                           0x14
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                           0x16
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                           0x18
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                           0x1a
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                           0x1c
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                           0x1e
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID16_GROUP_MASK                                                             0x00000003L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID17_GROUP_MASK                                                             0x0000000CL
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID18_GROUP_MASK                                                             0x00000030L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID19_GROUP_MASK                                                             0x000000C0L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID20_GROUP_MASK                                                             0x00000300L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID21_GROUP_MASK                                                             0x00000C00L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID22_GROUP_MASK                                                             0x00003000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID23_GROUP_MASK                                                             0x0000C000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID24_GROUP_MASK                                                             0x00030000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID25_GROUP_MASK                                                             0x000C0000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID26_GROUP_MASK                                                             0x00300000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID27_GROUP_MASK                                                             0x00C00000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID28_GROUP_MASK                                                             0x03000000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID29_GROUP_MASK                                                             0x0C000000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID30_GROUP_MASK                                                             0x30000000L
+#define GCEA_IO_RD_CLI2GRP_MAP1__CID31_GROUP_MASK                                                             0xC0000000L
+//GCEA_IO_WR_CLI2GRP_MAP0
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID0_GROUP__SHIFT                                                            0x0
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID1_GROUP__SHIFT                                                            0x2
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID2_GROUP__SHIFT                                                            0x4
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID3_GROUP__SHIFT                                                            0x6
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID4_GROUP__SHIFT                                                            0x8
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID5_GROUP__SHIFT                                                            0xa
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID6_GROUP__SHIFT                                                            0xc
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID7_GROUP__SHIFT                                                            0xe
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID8_GROUP__SHIFT                                                            0x10
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID9_GROUP__SHIFT                                                            0x12
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID10_GROUP__SHIFT                                                           0x14
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID11_GROUP__SHIFT                                                           0x16
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID12_GROUP__SHIFT                                                           0x18
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID13_GROUP__SHIFT                                                           0x1a
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID14_GROUP__SHIFT                                                           0x1c
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID15_GROUP__SHIFT                                                           0x1e
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID0_GROUP_MASK                                                              0x00000003L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID1_GROUP_MASK                                                              0x0000000CL
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID2_GROUP_MASK                                                              0x00000030L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID3_GROUP_MASK                                                              0x000000C0L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID4_GROUP_MASK                                                              0x00000300L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID5_GROUP_MASK                                                              0x00000C00L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID6_GROUP_MASK                                                              0x00003000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID7_GROUP_MASK                                                              0x0000C000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID8_GROUP_MASK                                                              0x00030000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID9_GROUP_MASK                                                              0x000C0000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID10_GROUP_MASK                                                             0x00300000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID11_GROUP_MASK                                                             0x00C00000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID12_GROUP_MASK                                                             0x03000000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID13_GROUP_MASK                                                             0x0C000000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID14_GROUP_MASK                                                             0x30000000L
+#define GCEA_IO_WR_CLI2GRP_MAP0__CID15_GROUP_MASK                                                             0xC0000000L
+//GCEA_IO_WR_CLI2GRP_MAP1
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID16_GROUP__SHIFT                                                           0x0
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID17_GROUP__SHIFT                                                           0x2
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID18_GROUP__SHIFT                                                           0x4
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID19_GROUP__SHIFT                                                           0x6
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID20_GROUP__SHIFT                                                           0x8
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID21_GROUP__SHIFT                                                           0xa
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID22_GROUP__SHIFT                                                           0xc
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID23_GROUP__SHIFT                                                           0xe
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID24_GROUP__SHIFT                                                           0x10
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID25_GROUP__SHIFT                                                           0x12
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID26_GROUP__SHIFT                                                           0x14
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID27_GROUP__SHIFT                                                           0x16
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID28_GROUP__SHIFT                                                           0x18
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID29_GROUP__SHIFT                                                           0x1a
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID30_GROUP__SHIFT                                                           0x1c
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID31_GROUP__SHIFT                                                           0x1e
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID16_GROUP_MASK                                                             0x00000003L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID17_GROUP_MASK                                                             0x0000000CL
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID18_GROUP_MASK                                                             0x00000030L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID19_GROUP_MASK                                                             0x000000C0L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID20_GROUP_MASK                                                             0x00000300L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID21_GROUP_MASK                                                             0x00000C00L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID22_GROUP_MASK                                                             0x00003000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID23_GROUP_MASK                                                             0x0000C000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID24_GROUP_MASK                                                             0x00030000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID25_GROUP_MASK                                                             0x000C0000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID26_GROUP_MASK                                                             0x00300000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID27_GROUP_MASK                                                             0x00C00000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID28_GROUP_MASK                                                             0x03000000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID29_GROUP_MASK                                                             0x0C000000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID30_GROUP_MASK                                                             0x30000000L
+#define GCEA_IO_WR_CLI2GRP_MAP1__CID31_GROUP_MASK                                                             0xC0000000L
+//GCEA_IO_RD_COMBINE_FLUSH
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                         0x0
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                         0x4
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                         0x8
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                         0xc
+#define GCEA_IO_RD_COMBINE_FLUSH__COMB_MODE__SHIFT                                                            0x10
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                           0x0000000FL
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                           0x000000F0L
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                           0x00000F00L
+#define GCEA_IO_RD_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                           0x0000F000L
+#define GCEA_IO_RD_COMBINE_FLUSH__COMB_MODE_MASK                                                              0x00030000L
+//GCEA_IO_WR_COMBINE_FLUSH
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                         0x0
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                         0x4
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                         0x8
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                         0xc
+#define GCEA_IO_WR_COMBINE_FLUSH__COMB_MODE__SHIFT                                                            0x10
+#define GCEA_IO_WR_COMBINE_FLUSH__DISABLE_MAM_CHAINING__SHIFT                                                 0x12
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                           0x0000000FL
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                           0x000000F0L
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                           0x00000F00L
+#define GCEA_IO_WR_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                           0x0000F000L
+#define GCEA_IO_WR_COMBINE_FLUSH__COMB_MODE_MASK                                                              0x00030000L
+#define GCEA_IO_WR_COMBINE_FLUSH__DISABLE_MAM_CHAINING_MASK                                                   0x00040000L
+//GCEA_IO_GROUP_BURST
+#define GCEA_IO_GROUP_BURST__RD_LIMIT_LO__SHIFT                                                               0x0
+#define GCEA_IO_GROUP_BURST__RD_LIMIT_HI__SHIFT                                                               0x8
+#define GCEA_IO_GROUP_BURST__WR_LIMIT_LO__SHIFT                                                               0x10
+#define GCEA_IO_GROUP_BURST__WR_LIMIT_HI__SHIFT                                                               0x18
+#define GCEA_IO_GROUP_BURST__RD_LIMIT_LO_MASK                                                                 0x000000FFL
+#define GCEA_IO_GROUP_BURST__RD_LIMIT_HI_MASK                                                                 0x0000FF00L
+#define GCEA_IO_GROUP_BURST__WR_LIMIT_LO_MASK                                                                 0x00FF0000L
+#define GCEA_IO_GROUP_BURST__WR_LIMIT_HI_MASK                                                                 0xFF000000L
+//GCEA_IO_RD_PRI_AGE
+#define GCEA_IO_RD_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                          0x0
+#define GCEA_IO_RD_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                          0x3
+#define GCEA_IO_RD_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                          0x6
+#define GCEA_IO_RD_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                          0x9
+#define GCEA_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                     0xc
+#define GCEA_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                     0xf
+#define GCEA_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                     0x12
+#define GCEA_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                     0x15
+#define GCEA_IO_RD_PRI_AGE__GROUP0_AGING_RATE_MASK                                                            0x00000007L
+#define GCEA_IO_RD_PRI_AGE__GROUP1_AGING_RATE_MASK                                                            0x00000038L
+#define GCEA_IO_RD_PRI_AGE__GROUP2_AGING_RATE_MASK                                                            0x000001C0L
+#define GCEA_IO_RD_PRI_AGE__GROUP3_AGING_RATE_MASK                                                            0x00000E00L
+#define GCEA_IO_RD_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                       0x00007000L
+#define GCEA_IO_RD_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                       0x00038000L
+#define GCEA_IO_RD_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                       0x001C0000L
+#define GCEA_IO_RD_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                       0x00E00000L
+//GCEA_IO_WR_PRI_AGE
+#define GCEA_IO_WR_PRI_AGE__GROUP0_AGING_RATE__SHIFT                                                          0x0
+#define GCEA_IO_WR_PRI_AGE__GROUP1_AGING_RATE__SHIFT                                                          0x3
+#define GCEA_IO_WR_PRI_AGE__GROUP2_AGING_RATE__SHIFT                                                          0x6
+#define GCEA_IO_WR_PRI_AGE__GROUP3_AGING_RATE__SHIFT                                                          0x9
+#define GCEA_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT__SHIFT                                                     0xc
+#define GCEA_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT__SHIFT                                                     0xf
+#define GCEA_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT__SHIFT                                                     0x12
+#define GCEA_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT__SHIFT                                                     0x15
+#define GCEA_IO_WR_PRI_AGE__GROUP0_AGING_RATE_MASK                                                            0x00000007L
+#define GCEA_IO_WR_PRI_AGE__GROUP1_AGING_RATE_MASK                                                            0x00000038L
+#define GCEA_IO_WR_PRI_AGE__GROUP2_AGING_RATE_MASK                                                            0x000001C0L
+#define GCEA_IO_WR_PRI_AGE__GROUP3_AGING_RATE_MASK                                                            0x00000E00L
+#define GCEA_IO_WR_PRI_AGE__GROUP0_AGE_COEFFICIENT_MASK                                                       0x00007000L
+#define GCEA_IO_WR_PRI_AGE__GROUP1_AGE_COEFFICIENT_MASK                                                       0x00038000L
+#define GCEA_IO_WR_PRI_AGE__GROUP2_AGE_COEFFICIENT_MASK                                                       0x001C0000L
+#define GCEA_IO_WR_PRI_AGE__GROUP3_AGE_COEFFICIENT_MASK                                                       0x00E00000L
+//GCEA_IO_RD_PRI_QUEUING
+#define GCEA_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                             0x0
+#define GCEA_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                             0x3
+#define GCEA_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                             0x6
+#define GCEA_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                             0x9
+#define GCEA_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                               0x00000007L
+#define GCEA_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                               0x00000038L
+#define GCEA_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                               0x000001C0L
+#define GCEA_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                               0x00000E00L
+//GCEA_IO_WR_PRI_QUEUING
+#define GCEA_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                             0x0
+#define GCEA_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                             0x3
+#define GCEA_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                             0x6
+#define GCEA_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                             0x9
+#define GCEA_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                               0x00000007L
+#define GCEA_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                               0x00000038L
+#define GCEA_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                               0x000001C0L
+#define GCEA_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                               0x00000E00L
+//GCEA_IO_RD_PRI_FIXED
+#define GCEA_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                 0x0
+#define GCEA_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                 0x3
+#define GCEA_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                 0x6
+#define GCEA_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                 0x9
+#define GCEA_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                   0x00000007L
+#define GCEA_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                   0x00000038L
+#define GCEA_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                   0x000001C0L
+#define GCEA_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                   0x00000E00L
+//GCEA_IO_WR_PRI_FIXED
+#define GCEA_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                 0x0
+#define GCEA_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                 0x3
+#define GCEA_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                 0x6
+#define GCEA_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                 0x9
+#define GCEA_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                   0x00000007L
+#define GCEA_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                   0x00000038L
+#define GCEA_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                   0x000001C0L
+#define GCEA_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                   0x00000E00L
+//GCEA_IO_RD_PRI_URGENCY
+#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                             0x0
+#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                             0x3
+#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                             0x6
+#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                             0x9
+#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                    0xc
+#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                    0xd
+#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                    0xe
+#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                    0xf
+#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                               0x00000007L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                               0x00000038L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                               0x000001C0L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                               0x00000E00L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                      0x00001000L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                      0x00002000L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                      0x00004000L
+#define GCEA_IO_RD_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                      0x00008000L
+//GCEA_IO_WR_PRI_URGENCY
+#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT__SHIFT                                             0x0
+#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT__SHIFT                                             0x3
+#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT__SHIFT                                             0x6
+#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT__SHIFT                                             0x9
+#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE__SHIFT                                                    0xc
+#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE__SHIFT                                                    0xd
+#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE__SHIFT                                                    0xe
+#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE__SHIFT                                                    0xf
+#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_COEFFICIENT_MASK                                               0x00000007L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_COEFFICIENT_MASK                                               0x00000038L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_COEFFICIENT_MASK                                               0x000001C0L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_COEFFICIENT_MASK                                               0x00000E00L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP0_URGENCY_MODE_MASK                                                      0x00001000L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP1_URGENCY_MODE_MASK                                                      0x00002000L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP2_URGENCY_MODE_MASK                                                      0x00004000L
+#define GCEA_IO_WR_PRI_URGENCY__GROUP3_URGENCY_MODE_MASK                                                      0x00008000L
+//GCEA_IO_RD_PRI_URGENCY_MASKING
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                      0x0
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                      0x1
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                      0x2
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                      0x3
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                      0x4
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                      0x5
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                      0x6
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                      0x7
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                      0x8
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                      0x9
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                     0xa
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                     0xb
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                     0xc
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                     0xd
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                     0xe
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                     0xf
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                     0x10
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                     0x11
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                     0x12
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                     0x13
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                     0x14
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                     0x15
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                     0x16
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                     0x17
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                     0x18
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                     0x19
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                     0x1a
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                     0x1b
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                     0x1c
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                     0x1d
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                     0x1e
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                     0x1f
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                        0x00000001L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                        0x00000002L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                        0x00000004L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                        0x00000008L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                        0x00000010L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                        0x00000020L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                        0x00000040L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                        0x00000080L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                        0x00000100L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                        0x00000200L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                       0x00000400L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                       0x00000800L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                       0x00001000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                       0x00002000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                       0x00004000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                       0x00008000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                       0x00010000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                       0x00020000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                       0x00040000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                       0x00080000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                       0x00100000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                       0x00200000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                       0x00400000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                       0x00800000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                       0x01000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                       0x02000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                       0x04000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                       0x08000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                       0x10000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                       0x20000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                       0x40000000L
+#define GCEA_IO_RD_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                       0x80000000L
+//GCEA_IO_WR_PRI_URGENCY_MASKING
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID0_MASK__SHIFT                                                      0x0
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID1_MASK__SHIFT                                                      0x1
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID2_MASK__SHIFT                                                      0x2
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID3_MASK__SHIFT                                                      0x3
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID4_MASK__SHIFT                                                      0x4
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID5_MASK__SHIFT                                                      0x5
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID6_MASK__SHIFT                                                      0x6
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID7_MASK__SHIFT                                                      0x7
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID8_MASK__SHIFT                                                      0x8
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID9_MASK__SHIFT                                                      0x9
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID10_MASK__SHIFT                                                     0xa
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID11_MASK__SHIFT                                                     0xb
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID12_MASK__SHIFT                                                     0xc
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID13_MASK__SHIFT                                                     0xd
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID14_MASK__SHIFT                                                     0xe
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID15_MASK__SHIFT                                                     0xf
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID16_MASK__SHIFT                                                     0x10
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID17_MASK__SHIFT                                                     0x11
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID18_MASK__SHIFT                                                     0x12
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID19_MASK__SHIFT                                                     0x13
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID20_MASK__SHIFT                                                     0x14
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID21_MASK__SHIFT                                                     0x15
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID22_MASK__SHIFT                                                     0x16
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID23_MASK__SHIFT                                                     0x17
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID24_MASK__SHIFT                                                     0x18
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID25_MASK__SHIFT                                                     0x19
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID26_MASK__SHIFT                                                     0x1a
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID27_MASK__SHIFT                                                     0x1b
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID28_MASK__SHIFT                                                     0x1c
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID29_MASK__SHIFT                                                     0x1d
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID30_MASK__SHIFT                                                     0x1e
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID31_MASK__SHIFT                                                     0x1f
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID0_MASK_MASK                                                        0x00000001L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID1_MASK_MASK                                                        0x00000002L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID2_MASK_MASK                                                        0x00000004L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID3_MASK_MASK                                                        0x00000008L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID4_MASK_MASK                                                        0x00000010L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID5_MASK_MASK                                                        0x00000020L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID6_MASK_MASK                                                        0x00000040L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID7_MASK_MASK                                                        0x00000080L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID8_MASK_MASK                                                        0x00000100L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID9_MASK_MASK                                                        0x00000200L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID10_MASK_MASK                                                       0x00000400L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID11_MASK_MASK                                                       0x00000800L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID12_MASK_MASK                                                       0x00001000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID13_MASK_MASK                                                       0x00002000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID14_MASK_MASK                                                       0x00004000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID15_MASK_MASK                                                       0x00008000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID16_MASK_MASK                                                       0x00010000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID17_MASK_MASK                                                       0x00020000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID18_MASK_MASK                                                       0x00040000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID19_MASK_MASK                                                       0x00080000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID20_MASK_MASK                                                       0x00100000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID21_MASK_MASK                                                       0x00200000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID22_MASK_MASK                                                       0x00400000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID23_MASK_MASK                                                       0x00800000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID24_MASK_MASK                                                       0x01000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID25_MASK_MASK                                                       0x02000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID26_MASK_MASK                                                       0x04000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID27_MASK_MASK                                                       0x08000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID28_MASK_MASK                                                       0x10000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID29_MASK_MASK                                                       0x20000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID30_MASK_MASK                                                       0x40000000L
+#define GCEA_IO_WR_PRI_URGENCY_MASKING__CID31_MASK_MASK                                                       0x80000000L
+//GCEA_IO_RD_PRI_QUANT_PRI1
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_IO_RD_PRI_QUANT_PRI2
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_IO_RD_PRI_QUANT_PRI3
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_IO_WR_PRI_QUANT_PRI1
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_IO_WR_PRI_QUANT_PRI2
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_IO_WR_PRI_QUANT_PRI3
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                    0x0
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                    0x8
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                    0x10
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                    0x18
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                      0x000000FFL
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                      0x0000FF00L
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                      0x00FF0000L
+#define GCEA_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                      0xFF000000L
+//GCEA_SDP_ARB_DRAM
+#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL__SHIFT                                                       0x0
+#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA__SHIFT                                                       0x8
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI__SHIFT                                                          0x10
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI__SHIFT                                                          0x11
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES__SHIFT                                                          0x12
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES__SHIFT                                                          0x13
+#define GCEA_SDP_ARB_DRAM__EOB_ON_EXPIRE__SHIFT                                                               0x14
+#define GCEA_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE__SHIFT                                                      0x15
+#define GCEA_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING__SHIFT                                                        0x16
+#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_CYCL_MASK                                                         0x0000007FL
+#define GCEA_SDP_ARB_DRAM__RDWR_BURST_LIMIT_DATA_MASK                                                         0x00007F00L
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_PRI_MASK                                                            0x00010000L
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_PRI_MASK                                                            0x00020000L
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2RD_ON_RES_MASK                                                            0x00040000L
+#define GCEA_SDP_ARB_DRAM__EARLY_SW2WR_ON_RES_MASK                                                            0x00080000L
+#define GCEA_SDP_ARB_DRAM__EOB_ON_EXPIRE_MASK                                                                 0x00100000L
+#define GCEA_SDP_ARB_DRAM__DECOUPLE_RDWR_BNKSTATE_MASK                                                        0x00200000L
+#define GCEA_SDP_ARB_DRAM__ALLOW_CHAIN_BREAKING_MASK                                                          0x00400000L
+//GCEA_SDP_ARB_FINAL
+#define GCEA_SDP_ARB_FINAL__DRAM_BURST_LIMIT__SHIFT                                                           0x0
+#define GCEA_SDP_ARB_FINAL__GMI_BURST_LIMIT__SHIFT                                                            0x5
+#define GCEA_SDP_ARB_FINAL__IO_BURST_LIMIT__SHIFT                                                             0xa
+#define GCEA_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER__SHIFT                                                     0xf
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC0__SHIFT                                                                 0x11
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC1__SHIFT                                                                 0x12
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC2__SHIFT                                                                 0x13
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC3__SHIFT                                                                 0x14
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC4__SHIFT                                                                 0x15
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC5__SHIFT                                                                 0x16
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC6__SHIFT                                                                 0x17
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC7__SHIFT                                                                 0x18
+#define GCEA_SDP_ARB_FINAL__ERREVENT_ON_ERROR__SHIFT                                                          0x19
+#define GCEA_SDP_ARB_FINAL__HALTREQ_ON_ERROR__SHIFT                                                           0x1a
+#define GCEA_SDP_ARB_FINAL__GMI_BURST_STRETCH__SHIFT                                                          0x1b
+#define GCEA_SDP_ARB_FINAL__DRAM_RD_THROTTLE__SHIFT                                                           0x1c
+#define GCEA_SDP_ARB_FINAL__DRAM_WR_THROTTLE__SHIFT                                                           0x1d
+#define GCEA_SDP_ARB_FINAL__GMI_RD_THROTTLE__SHIFT                                                            0x1e
+#define GCEA_SDP_ARB_FINAL__GMI_WR_THROTTLE__SHIFT                                                            0x1f
+#define GCEA_SDP_ARB_FINAL__DRAM_BURST_LIMIT_MASK                                                             0x0000001FL
+#define GCEA_SDP_ARB_FINAL__GMI_BURST_LIMIT_MASK                                                              0x000003E0L
+#define GCEA_SDP_ARB_FINAL__IO_BURST_LIMIT_MASK                                                               0x00007C00L
+#define GCEA_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER_MASK                                                       0x00018000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC0_MASK                                                                   0x00020000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC1_MASK                                                                   0x00040000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC2_MASK                                                                   0x00080000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC3_MASK                                                                   0x00100000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC4_MASK                                                                   0x00200000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC5_MASK                                                                   0x00400000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC6_MASK                                                                   0x00800000L
+#define GCEA_SDP_ARB_FINAL__RDONLY_VC7_MASK                                                                   0x01000000L
+#define GCEA_SDP_ARB_FINAL__ERREVENT_ON_ERROR_MASK                                                            0x02000000L
+#define GCEA_SDP_ARB_FINAL__HALTREQ_ON_ERROR_MASK                                                             0x04000000L
+#define GCEA_SDP_ARB_FINAL__GMI_BURST_STRETCH_MASK                                                            0x08000000L
+#define GCEA_SDP_ARB_FINAL__DRAM_RD_THROTTLE_MASK                                                             0x10000000L
+#define GCEA_SDP_ARB_FINAL__DRAM_WR_THROTTLE_MASK                                                             0x20000000L
+#define GCEA_SDP_ARB_FINAL__GMI_RD_THROTTLE_MASK                                                              0x40000000L
+#define GCEA_SDP_ARB_FINAL__GMI_WR_THROTTLE_MASK                                                              0x80000000L
+//GCEA_SDP_DRAM_PRIORITY
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                     0x0
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                     0x4
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                     0x8
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                     0xc
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                     0x10
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                     0x14
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                     0x18
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                     0x1c
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                       0x0000000FL
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                       0x000000F0L
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                       0x00000F00L
+#define GCEA_SDP_DRAM_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                       0x0000F000L
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                       0x000F0000L
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                       0x00F00000L
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                       0x0F000000L
+#define GCEA_SDP_DRAM_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                       0xF0000000L
+//GCEA_SDP_IO_PRIORITY
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY__SHIFT                                                       0x0
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY__SHIFT                                                       0x4
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY__SHIFT                                                       0x8
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY__SHIFT                                                       0xc
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY__SHIFT                                                       0x10
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY__SHIFT                                                       0x14
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY__SHIFT                                                       0x18
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY__SHIFT                                                       0x1c
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP0_PRIORITY_MASK                                                         0x0000000FL
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP1_PRIORITY_MASK                                                         0x000000F0L
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP2_PRIORITY_MASK                                                         0x00000F00L
+#define GCEA_SDP_IO_PRIORITY__RD_GROUP3_PRIORITY_MASK                                                         0x0000F000L
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP0_PRIORITY_MASK                                                         0x000F0000L
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP1_PRIORITY_MASK                                                         0x00F00000L
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP2_PRIORITY_MASK                                                         0x0F000000L
+#define GCEA_SDP_IO_PRIORITY__WR_GROUP3_PRIORITY_MASK                                                         0xF0000000L
+//GCEA_SDP_CREDITS
+#define GCEA_SDP_CREDITS__TAG_LIMIT__SHIFT                                                                    0x0
+#define GCEA_SDP_CREDITS__WR_RESP_CREDITS__SHIFT                                                              0x8
+#define GCEA_SDP_CREDITS__RD_RESP_CREDITS__SHIFT                                                              0x10
+#define GCEA_SDP_CREDITS__PRB_REQ_CREDITS__SHIFT                                                              0x18
+#define GCEA_SDP_CREDITS__TAG_LIMIT_MASK                                                                      0x000000FFL
+#define GCEA_SDP_CREDITS__WR_RESP_CREDITS_MASK                                                                0x00007F00L
+#define GCEA_SDP_CREDITS__RD_RESP_CREDITS_MASK                                                                0x007F0000L
+#define GCEA_SDP_CREDITS__PRB_REQ_CREDITS_MASK                                                                0x3F000000L
+//GCEA_SDP_TAG_RESERVE0
+#define GCEA_SDP_TAG_RESERVE0__VC0__SHIFT                                                                     0x0
+#define GCEA_SDP_TAG_RESERVE0__VC1__SHIFT                                                                     0x8
+#define GCEA_SDP_TAG_RESERVE0__VC2__SHIFT                                                                     0x10
+#define GCEA_SDP_TAG_RESERVE0__VC3__SHIFT                                                                     0x18
+#define GCEA_SDP_TAG_RESERVE0__VC0_MASK                                                                       0x000000FFL
+#define GCEA_SDP_TAG_RESERVE0__VC1_MASK                                                                       0x0000FF00L
+#define GCEA_SDP_TAG_RESERVE0__VC2_MASK                                                                       0x00FF0000L
+#define GCEA_SDP_TAG_RESERVE0__VC3_MASK                                                                       0xFF000000L
+//GCEA_SDP_TAG_RESERVE1
+#define GCEA_SDP_TAG_RESERVE1__VC4__SHIFT                                                                     0x0
+#define GCEA_SDP_TAG_RESERVE1__VC5__SHIFT                                                                     0x8
+#define GCEA_SDP_TAG_RESERVE1__VC6__SHIFT                                                                     0x10
+#define GCEA_SDP_TAG_RESERVE1__VC7__SHIFT                                                                     0x18
+#define GCEA_SDP_TAG_RESERVE1__VC4_MASK                                                                       0x000000FFL
+#define GCEA_SDP_TAG_RESERVE1__VC5_MASK                                                                       0x0000FF00L
+#define GCEA_SDP_TAG_RESERVE1__VC6_MASK                                                                       0x00FF0000L
+#define GCEA_SDP_TAG_RESERVE1__VC7_MASK                                                                       0xFF000000L
+//GCEA_SDP_VCC_RESERVE0
+#define GCEA_SDP_VCC_RESERVE0__VC0_CREDITS__SHIFT                                                             0x0
+#define GCEA_SDP_VCC_RESERVE0__VC1_CREDITS__SHIFT                                                             0x6
+#define GCEA_SDP_VCC_RESERVE0__VC2_CREDITS__SHIFT                                                             0xc
+#define GCEA_SDP_VCC_RESERVE0__VC3_CREDITS__SHIFT                                                             0x12
+#define GCEA_SDP_VCC_RESERVE0__VC4_CREDITS__SHIFT                                                             0x18
+#define GCEA_SDP_VCC_RESERVE0__VC0_CREDITS_MASK                                                               0x0000003FL
+#define GCEA_SDP_VCC_RESERVE0__VC1_CREDITS_MASK                                                               0x00000FC0L
+#define GCEA_SDP_VCC_RESERVE0__VC2_CREDITS_MASK                                                               0x0003F000L
+#define GCEA_SDP_VCC_RESERVE0__VC3_CREDITS_MASK                                                               0x00FC0000L
+#define GCEA_SDP_VCC_RESERVE0__VC4_CREDITS_MASK                                                               0x3F000000L
+//GCEA_SDP_VCC_RESERVE1
+#define GCEA_SDP_VCC_RESERVE1__VC5_CREDITS__SHIFT                                                             0x0
+#define GCEA_SDP_VCC_RESERVE1__VC6_CREDITS__SHIFT                                                             0x6
+#define GCEA_SDP_VCC_RESERVE1__VC7_CREDITS__SHIFT                                                             0xc
+#define GCEA_SDP_VCC_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                         0x1f
+#define GCEA_SDP_VCC_RESERVE1__VC5_CREDITS_MASK                                                               0x0000003FL
+#define GCEA_SDP_VCC_RESERVE1__VC6_CREDITS_MASK                                                               0x00000FC0L
+#define GCEA_SDP_VCC_RESERVE1__VC7_CREDITS_MASK                                                               0x0003F000L
+#define GCEA_SDP_VCC_RESERVE1__DISTRIBUTE_POOL_MASK                                                           0x80000000L
+//GCEA_SDP_VCD_RESERVE0
+#define GCEA_SDP_VCD_RESERVE0__VC0_CREDITS__SHIFT                                                             0x0
+#define GCEA_SDP_VCD_RESERVE0__VC1_CREDITS__SHIFT                                                             0x6
+#define GCEA_SDP_VCD_RESERVE0__VC2_CREDITS__SHIFT                                                             0xc
+#define GCEA_SDP_VCD_RESERVE0__VC3_CREDITS__SHIFT                                                             0x12
+#define GCEA_SDP_VCD_RESERVE0__VC4_CREDITS__SHIFT                                                             0x18
+#define GCEA_SDP_VCD_RESERVE0__VC0_CREDITS_MASK                                                               0x0000003FL
+#define GCEA_SDP_VCD_RESERVE0__VC1_CREDITS_MASK                                                               0x00000FC0L
+#define GCEA_SDP_VCD_RESERVE0__VC2_CREDITS_MASK                                                               0x0003F000L
+#define GCEA_SDP_VCD_RESERVE0__VC3_CREDITS_MASK                                                               0x00FC0000L
+#define GCEA_SDP_VCD_RESERVE0__VC4_CREDITS_MASK                                                               0x3F000000L
+
+
+// addressBlock: gc_gceadec2
+//GCEA_SDP_VCD_RESERVE1
+#define GCEA_SDP_VCD_RESERVE1__VC5_CREDITS__SHIFT                                                             0x0
+#define GCEA_SDP_VCD_RESERVE1__VC6_CREDITS__SHIFT                                                             0x6
+#define GCEA_SDP_VCD_RESERVE1__VC7_CREDITS__SHIFT                                                             0xc
+#define GCEA_SDP_VCD_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                         0x1f
+#define GCEA_SDP_VCD_RESERVE1__VC5_CREDITS_MASK                                                               0x0000003FL
+#define GCEA_SDP_VCD_RESERVE1__VC6_CREDITS_MASK                                                               0x00000FC0L
+#define GCEA_SDP_VCD_RESERVE1__VC7_CREDITS_MASK                                                               0x0003F000L
+#define GCEA_SDP_VCD_RESERVE1__DISTRIBUTE_POOL_MASK                                                           0x80000000L
+//GCEA_SDP_REQ_CNTL
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ__SHIFT                                                   0x0
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE__SHIFT                                                  0x1
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC__SHIFT                                                 0x2
+#define GCEA_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM__SHIFT                                                     0x3
+#define GCEA_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI__SHIFT                                                      0x4
+#define GCEA_SDP_REQ_CNTL__INNER_DOMAIN_MODE__SHIFT                                                           0x5
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ__SHIFT                                                        0x6
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE__SHIFT                                                       0x8
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC__SHIFT                                                      0xa
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ_MASK                                                     0x00000001L
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE_MASK                                                    0x00000002L
+#define GCEA_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC_MASK                                                   0x00000004L
+#define GCEA_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM_MASK                                                       0x00000008L
+#define GCEA_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_GMI_MASK                                                        0x00000010L
+#define GCEA_SDP_REQ_CNTL__INNER_DOMAIN_MODE_MASK                                                             0x00000020L
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_READ_MASK                                                          0x000000C0L
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_WRITE_MASK                                                         0x00000300L
+#define GCEA_SDP_REQ_CNTL__REQ_BLOCK_LEVEL_ATOMIC_MASK                                                        0x00000C00L
+//GCEA_MISC
+#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB__SHIFT                                                         0x0
+#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB__SHIFT                                                         0x1
+#define GCEA_MISC__RELATIVE_PRI_IN_GMI_RD_ARB__SHIFT                                                          0x2
+#define GCEA_MISC__RELATIVE_PRI_IN_GMI_WR_ARB__SHIFT                                                          0x3
+#define GCEA_MISC__RELATIVE_PRI_IN_IO_RD_ARB__SHIFT                                                           0x4
+#define GCEA_MISC__RELATIVE_PRI_IN_IO_WR_ARB__SHIFT                                                           0x5
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC0__SHIFT                                                               0x6
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC1__SHIFT                                                               0x7
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC2__SHIFT                                                               0x8
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC3__SHIFT                                                               0x9
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC4__SHIFT                                                               0xa
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC5__SHIFT                                                               0xb
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC6__SHIFT                                                               0xc
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC7__SHIFT                                                               0xd
+#define GCEA_MISC__EARLY_SDP_ORIGDATA__SHIFT                                                                  0xe
+#define GCEA_MISC__LINKMGR_DYNAMIC_MODE__SHIFT                                                                0xf
+#define GCEA_MISC__LINKMGR_HALT_THRESHOLD__SHIFT                                                              0x11
+#define GCEA_MISC__LINKMGR_RECONNECT_DELAY__SHIFT                                                             0x13
+#define GCEA_MISC__LINKMGR_IDLE_THRESHOLD__SHIFT                                                              0x15
+#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB__SHIFT                                                      0x1a
+#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB__SHIFT                                                       0x1b
+#define GCEA_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB__SHIFT                                                          0x1c
+#define GCEA_MISC__FAVOUR_LAST_CS_IN_GMI_ARB__SHIFT                                                           0x1d
+#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB__SHIFT                                                        0x1e
+#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB__SHIFT                                                         0x1f
+#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_RD_ARB_MASK                                                           0x00000001L
+#define GCEA_MISC__RELATIVE_PRI_IN_DRAM_WR_ARB_MASK                                                           0x00000002L
+#define GCEA_MISC__RELATIVE_PRI_IN_GMI_RD_ARB_MASK                                                            0x00000004L
+#define GCEA_MISC__RELATIVE_PRI_IN_GMI_WR_ARB_MASK                                                            0x00000008L
+#define GCEA_MISC__RELATIVE_PRI_IN_IO_RD_ARB_MASK                                                             0x00000010L
+#define GCEA_MISC__RELATIVE_PRI_IN_IO_WR_ARB_MASK                                                             0x00000020L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC0_MASK                                                                 0x00000040L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC1_MASK                                                                 0x00000080L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC2_MASK                                                                 0x00000100L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC3_MASK                                                                 0x00000200L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC4_MASK                                                                 0x00000400L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC5_MASK                                                                 0x00000800L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC6_MASK                                                                 0x00001000L
+#define GCEA_MISC__EARLYWRRET_ENABLE_VC7_MASK                                                                 0x00002000L
+#define GCEA_MISC__EARLY_SDP_ORIGDATA_MASK                                                                    0x00004000L
+#define GCEA_MISC__LINKMGR_DYNAMIC_MODE_MASK                                                                  0x00018000L
+#define GCEA_MISC__LINKMGR_HALT_THRESHOLD_MASK                                                                0x00060000L
+#define GCEA_MISC__LINKMGR_RECONNECT_DELAY_MASK                                                               0x00180000L
+#define GCEA_MISC__LINKMGR_IDLE_THRESHOLD_MASK                                                                0x03E00000L
+#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_DRAM_ARB_MASK                                                        0x04000000L
+#define GCEA_MISC__FAVOUR_MIDCHAIN_CS_IN_GMI_ARB_MASK                                                         0x08000000L
+#define GCEA_MISC__FAVOUR_LAST_CS_IN_DRAM_ARB_MASK                                                            0x10000000L
+#define GCEA_MISC__FAVOUR_LAST_CS_IN_GMI_ARB_MASK                                                             0x20000000L
+#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_DRAM_ARB_MASK                                                          0x40000000L
+#define GCEA_MISC__SWITCH_CS_ON_W2R_IN_GMI_ARB_MASK                                                           0x80000000L
+//GCEA_LATENCY_SAMPLING
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_DRAM__SHIFT                                                           0x0
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_DRAM__SHIFT                                                           0x1
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_GMI__SHIFT                                                            0x2
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_GMI__SHIFT                                                            0x3
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_IO__SHIFT                                                             0x4
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_IO__SHIFT                                                             0x5
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_READ__SHIFT                                                           0x6
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_READ__SHIFT                                                           0x7
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_WRITE__SHIFT                                                          0x8
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_WRITE__SHIFT                                                          0x9
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET__SHIFT                                                     0xa
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET__SHIFT                                                     0xb
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET__SHIFT                                                   0xc
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET__SHIFT                                                   0xd
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_VC__SHIFT                                                             0xe
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_VC__SHIFT                                                             0x16
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_DRAM_MASK                                                             0x00000001L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_DRAM_MASK                                                             0x00000002L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_GMI_MASK                                                              0x00000004L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_GMI_MASK                                                              0x00000008L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_IO_MASK                                                               0x00000010L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_IO_MASK                                                               0x00000020L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_READ_MASK                                                             0x00000040L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_READ_MASK                                                             0x00000080L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_WRITE_MASK                                                            0x00000100L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_WRITE_MASK                                                            0x00000200L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET_MASK                                                       0x00000400L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET_MASK                                                       0x00000800L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET_MASK                                                     0x00001000L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET_MASK                                                     0x00002000L
+#define GCEA_LATENCY_SAMPLING__SAMPLER0_VC_MASK                                                               0x003FC000L
+#define GCEA_LATENCY_SAMPLING__SAMPLER1_VC_MASK                                                               0x3FC00000L
+//GCEA_MAM_CTRL2
+#define GCEA_MAM_CTRL2__ARAM_FLUSH_DISABLE__SHIFT                                                             0x0
+#define GCEA_MAM_CTRL2__DBIT_PF_CLR_ONLY__SHIFT                                                               0x1
+#define GCEA_MAM_CTRL2__DBIT_PF_RD_ONLY__SHIFT                                                                0x2
+#define GCEA_MAM_CTRL2__DBIT_TRACK_SEGMENT__SHIFT                                                             0x3
+#define GCEA_MAM_CTRL2__ARAM_TRACK_SEGMENT__SHIFT                                                             0x6
+#define GCEA_MAM_CTRL2__ARAM_FB_TRACK_SIZE__SHIFT                                                             0x9
+#define GCEA_MAM_CTRL2__ARAM_RB_ENTRY_SIZE__SHIFT                                                             0xf
+#define GCEA_MAM_CTRL2__ARAM_OVERRIDE_EA_STRAP__SHIFT                                                         0x12
+#define GCEA_MAM_CTRL2__ABIT_FLUSH_SPACE_OVERRIDE_ENABLE__SHIFT                                               0x13
+#define GCEA_MAM_CTRL2__ABIT_FLUSH_SPACE_OVERRIDE_VALUE__SHIFT                                                0x14
+#define GCEA_MAM_CTRL2__ARAM_REMOVE_TRACKER__SHIFT                                                            0x15
+#define GCEA_MAM_CTRL2__FORCE_DBIT_QUERY_DIRTY_ENABLE__SHIFT                                                  0x16
+#define GCEA_MAM_CTRL2__FORCE_DBIT_QUERY_DIRTY_VALUE__SHIFT                                                   0x17
+#define GCEA_MAM_CTRL2__RESERVED_FIELD__SHIFT                                                                 0x18
+#define GCEA_MAM_CTRL2__ARAM_FLUSH_DISABLE_MASK                                                               0x00000001L
+#define GCEA_MAM_CTRL2__DBIT_PF_CLR_ONLY_MASK                                                                 0x00000002L
+#define GCEA_MAM_CTRL2__DBIT_PF_RD_ONLY_MASK                                                                  0x00000004L
+#define GCEA_MAM_CTRL2__DBIT_TRACK_SEGMENT_MASK                                                               0x00000038L
+#define GCEA_MAM_CTRL2__ARAM_TRACK_SEGMENT_MASK                                                               0x000001C0L
+#define GCEA_MAM_CTRL2__ARAM_FB_TRACK_SIZE_MASK                                                               0x00007E00L
+#define GCEA_MAM_CTRL2__ARAM_RB_ENTRY_SIZE_MASK                                                               0x00038000L
+#define GCEA_MAM_CTRL2__ARAM_OVERRIDE_EA_STRAP_MASK                                                           0x00040000L
+#define GCEA_MAM_CTRL2__ABIT_FLUSH_SPACE_OVERRIDE_ENABLE_MASK                                                 0x00080000L
+#define GCEA_MAM_CTRL2__ABIT_FLUSH_SPACE_OVERRIDE_VALUE_MASK                                                  0x00100000L
+#define GCEA_MAM_CTRL2__ARAM_REMOVE_TRACKER_MASK                                                              0x00200000L
+#define GCEA_MAM_CTRL2__FORCE_DBIT_QUERY_DIRTY_ENABLE_MASK                                                    0x00400000L
+#define GCEA_MAM_CTRL2__FORCE_DBIT_QUERY_DIRTY_VALUE_MASK                                                     0x00800000L
+#define GCEA_MAM_CTRL2__RESERVED_FIELD_MASK                                                                   0xFF000000L
+//GCEA_MAM_CTRL
+#define GCEA_MAM_CTRL__MAM_DISABLE__SHIFT                                                                     0x0
+#define GCEA_MAM_CTRL__DBIT_COALESCE_DISABLE__SHIFT                                                           0x1
+#define GCEA_MAM_CTRL__ARAM_COALESCE_DISABLE__SHIFT                                                           0x2
+#define GCEA_MAM_CTRL__ARAM_FLUSH_SNOOP_EN__SHIFT                                                             0x3
+#define GCEA_MAM_CTRL__SDMA_UPDT_ARAM__SHIFT                                                                  0x4
+#define GCEA_MAM_CTRL__ARAM_FLUSH_NOALLOC__SHIFT                                                              0x5
+#define GCEA_MAM_CTRL__FLUSH_TRACKER__SHIFT                                                                   0x6
+#define GCEA_MAM_CTRL__CLEAR_TRACKER__SHIFT                                                                   0x7
+#define GCEA_MAM_CTRL__SDP_PRIORITY__SHIFT                                                                    0x8
+#define GCEA_MAM_CTRL__FORCE_FLUSH_UPDT_TRACKER__SHIFT                                                        0xc
+#define GCEA_MAM_CTRL__FORCE_FLUSH_GEN_INTERRUPT__SHIFT                                                       0xd
+#define GCEA_MAM_CTRL__TIMER_FLUSH_UPDT_TRACKER__SHIFT                                                        0xe
+#define GCEA_MAM_CTRL__TIMER_FLUSH_GEN_INTERRUPT__SHIFT                                                       0xf
+#define GCEA_MAM_CTRL__RESERVED_FIELD__SHIFT                                                                  0x10
+#define GCEA_MAM_CTRL__ARAM_NUM_RB_ENTRIES__SHIFT                                                             0x17
+#define GCEA_MAM_CTRL__ARAM_RB_ADDR_HI__SHIFT                                                                 0x1c
+#define GCEA_MAM_CTRL__MAM_DISABLE_MASK                                                                       0x00000001L
+#define GCEA_MAM_CTRL__DBIT_COALESCE_DISABLE_MASK                                                             0x00000002L
+#define GCEA_MAM_CTRL__ARAM_COALESCE_DISABLE_MASK                                                             0x00000004L
+#define GCEA_MAM_CTRL__ARAM_FLUSH_SNOOP_EN_MASK                                                               0x00000008L
+#define GCEA_MAM_CTRL__SDMA_UPDT_ARAM_MASK                                                                    0x00000010L
+#define GCEA_MAM_CTRL__ARAM_FLUSH_NOALLOC_MASK                                                                0x00000020L
+#define GCEA_MAM_CTRL__FLUSH_TRACKER_MASK                                                                     0x00000040L
+#define GCEA_MAM_CTRL__CLEAR_TRACKER_MASK                                                                     0x00000080L
+#define GCEA_MAM_CTRL__SDP_PRIORITY_MASK                                                                      0x00000F00L
+#define GCEA_MAM_CTRL__FORCE_FLUSH_UPDT_TRACKER_MASK                                                          0x00001000L
+#define GCEA_MAM_CTRL__FORCE_FLUSH_GEN_INTERRUPT_MASK                                                         0x00002000L
+#define GCEA_MAM_CTRL__TIMER_FLUSH_UPDT_TRACKER_MASK                                                          0x00004000L
+#define GCEA_MAM_CTRL__TIMER_FLUSH_GEN_INTERRUPT_MASK                                                         0x00008000L
+#define GCEA_MAM_CTRL__RESERVED_FIELD_MASK                                                                    0x007F0000L
+#define GCEA_MAM_CTRL__ARAM_NUM_RB_ENTRIES_MASK                                                               0x0F800000L
+#define GCEA_MAM_CTRL__ARAM_RB_ADDR_HI_MASK                                                                   0xF0000000L
+//GCEA_EDC_CNT
+#define GCEA_EDC_CNT__DRAMRD_CMDMEM_SEC_COUNT__SHIFT                                                          0x0
+#define GCEA_EDC_CNT__DRAMRD_CMDMEM_DED_COUNT__SHIFT                                                          0x2
+#define GCEA_EDC_CNT__DRAMWR_CMDMEM_SEC_COUNT__SHIFT                                                          0x4
+#define GCEA_EDC_CNT__DRAMWR_CMDMEM_DED_COUNT__SHIFT                                                          0x6
+#define GCEA_EDC_CNT__DRAMWR_DATAMEM_SEC_COUNT__SHIFT                                                         0x8
+#define GCEA_EDC_CNT__DRAMWR_DATAMEM_DED_COUNT__SHIFT                                                         0xa
+#define GCEA_EDC_CNT__RRET_TAGMEM_SEC_COUNT__SHIFT                                                            0xc
+#define GCEA_EDC_CNT__RRET_TAGMEM_DED_COUNT__SHIFT                                                            0xe
+#define GCEA_EDC_CNT__WRET_TAGMEM_SEC_COUNT__SHIFT                                                            0x10
+#define GCEA_EDC_CNT__WRET_TAGMEM_DED_COUNT__SHIFT                                                            0x12
+#define GCEA_EDC_CNT__IOWR_DATAMEM_SEC_COUNT__SHIFT                                                           0x14
+#define GCEA_EDC_CNT__IOWR_DATAMEM_DED_COUNT__SHIFT                                                           0x16
+#define GCEA_EDC_CNT__DRAMRD_PAGEMEM_SED_COUNT__SHIFT                                                         0x18
+#define GCEA_EDC_CNT__DRAMWR_PAGEMEM_SED_COUNT__SHIFT                                                         0x1a
+#define GCEA_EDC_CNT__IORD_CMDMEM_SED_COUNT__SHIFT                                                            0x1c
+#define GCEA_EDC_CNT__IOWR_CMDMEM_SED_COUNT__SHIFT                                                            0x1e
+#define GCEA_EDC_CNT__DRAMRD_CMDMEM_SEC_COUNT_MASK                                                            0x00000003L
+#define GCEA_EDC_CNT__DRAMRD_CMDMEM_DED_COUNT_MASK                                                            0x0000000CL
+#define GCEA_EDC_CNT__DRAMWR_CMDMEM_SEC_COUNT_MASK                                                            0x00000030L
+#define GCEA_EDC_CNT__DRAMWR_CMDMEM_DED_COUNT_MASK                                                            0x000000C0L
+#define GCEA_EDC_CNT__DRAMWR_DATAMEM_SEC_COUNT_MASK                                                           0x00000300L
+#define GCEA_EDC_CNT__DRAMWR_DATAMEM_DED_COUNT_MASK                                                           0x00000C00L
+#define GCEA_EDC_CNT__RRET_TAGMEM_SEC_COUNT_MASK                                                              0x00003000L
+#define GCEA_EDC_CNT__RRET_TAGMEM_DED_COUNT_MASK                                                              0x0000C000L
+#define GCEA_EDC_CNT__WRET_TAGMEM_SEC_COUNT_MASK                                                              0x00030000L
+#define GCEA_EDC_CNT__WRET_TAGMEM_DED_COUNT_MASK                                                              0x000C0000L
+#define GCEA_EDC_CNT__IOWR_DATAMEM_SEC_COUNT_MASK                                                             0x00300000L
+#define GCEA_EDC_CNT__IOWR_DATAMEM_DED_COUNT_MASK                                                             0x00C00000L
+#define GCEA_EDC_CNT__DRAMRD_PAGEMEM_SED_COUNT_MASK                                                           0x03000000L
+#define GCEA_EDC_CNT__DRAMWR_PAGEMEM_SED_COUNT_MASK                                                           0x0C000000L
+#define GCEA_EDC_CNT__IORD_CMDMEM_SED_COUNT_MASK                                                              0x30000000L
+#define GCEA_EDC_CNT__IOWR_CMDMEM_SED_COUNT_MASK                                                              0xC0000000L
+//GCEA_EDC_CNT2
+#define GCEA_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT__SHIFT                                                          0x0
+#define GCEA_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT__SHIFT                                                          0x2
+#define GCEA_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT__SHIFT                                                          0x4
+#define GCEA_EDC_CNT2__GMIWR_CMDMEM_DED_COUNT__SHIFT                                                          0x6
+#define GCEA_EDC_CNT2__GMIWR_DATAMEM_SEC_COUNT__SHIFT                                                         0x8
+#define GCEA_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT__SHIFT                                                         0xa
+#define GCEA_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT__SHIFT                                                         0xc
+#define GCEA_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT__SHIFT                                                         0xe
+#define GCEA_EDC_CNT2__MAM_D0MEM_SED_COUNT__SHIFT                                                             0x10
+#define GCEA_EDC_CNT2__MAM_D1MEM_SED_COUNT__SHIFT                                                             0x12
+#define GCEA_EDC_CNT2__MAM_D2MEM_SED_COUNT__SHIFT                                                             0x14
+#define GCEA_EDC_CNT2__MAM_D3MEM_SED_COUNT__SHIFT                                                             0x16
+#define GCEA_EDC_CNT2__MAM_D0MEM_DED_COUNT__SHIFT                                                             0x18
+#define GCEA_EDC_CNT2__MAM_D1MEM_DED_COUNT__SHIFT                                                             0x1a
+#define GCEA_EDC_CNT2__MAM_D2MEM_DED_COUNT__SHIFT                                                             0x1c
+#define GCEA_EDC_CNT2__MAM_D3MEM_DED_COUNT__SHIFT                                                             0x1e
+#define GCEA_EDC_CNT2__GMIRD_CMDMEM_SEC_COUNT_MASK                                                            0x00000003L
+#define GCEA_EDC_CNT2__GMIRD_CMDMEM_DED_COUNT_MASK                                                            0x0000000CL
+#define GCEA_EDC_CNT2__GMIWR_CMDMEM_SEC_COUNT_MASK                                                            0x00000030L
+#define GCEA_EDC_CNT2__GMIWR_CMDMEM_DED_COUNT_MASK                                                            0x000000C0L
+#define GCEA_EDC_CNT2__GMIWR_DATAMEM_SEC_COUNT_MASK                                                           0x00000300L
+#define GCEA_EDC_CNT2__GMIWR_DATAMEM_DED_COUNT_MASK                                                           0x00000C00L
+#define GCEA_EDC_CNT2__GMIRD_PAGEMEM_SED_COUNT_MASK                                                           0x00003000L
+#define GCEA_EDC_CNT2__GMIWR_PAGEMEM_SED_COUNT_MASK                                                           0x0000C000L
+#define GCEA_EDC_CNT2__MAM_D0MEM_SED_COUNT_MASK                                                               0x00030000L
+#define GCEA_EDC_CNT2__MAM_D1MEM_SED_COUNT_MASK                                                               0x000C0000L
+#define GCEA_EDC_CNT2__MAM_D2MEM_SED_COUNT_MASK                                                               0x00300000L
+#define GCEA_EDC_CNT2__MAM_D3MEM_SED_COUNT_MASK                                                               0x00C00000L
+#define GCEA_EDC_CNT2__MAM_D0MEM_DED_COUNT_MASK                                                               0x03000000L
+#define GCEA_EDC_CNT2__MAM_D1MEM_DED_COUNT_MASK                                                               0x0C000000L
+#define GCEA_EDC_CNT2__MAM_D2MEM_DED_COUNT_MASK                                                               0x30000000L
+#define GCEA_EDC_CNT2__MAM_D3MEM_DED_COUNT_MASK                                                               0xC0000000L
+//GCEA_DSM_CNTL
+#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x0
+#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x2
+#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x3
+#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x5
+#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x6
+#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x8
+#define GCEA_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                  0x9
+#define GCEA_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xb
+#define GCEA_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA__SHIFT                                                  0xc
+#define GCEA_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE__SHIFT                                                 0xe
+#define GCEA_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0xf
+#define GCEA_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0x11
+#define GCEA_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x12
+#define GCEA_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0x14
+#define GCEA_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                                0x15
+#define GCEA_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0x17
+#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000003L
+#define GCEA_DSM_CNTL__DRAMRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000004L
+#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00000018L
+#define GCEA_DSM_CNTL__DRAMWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00000020L
+#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                 0x000000C0L
+#define GCEA_DSM_CNTL__DRAMWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00000100L
+#define GCEA_DSM_CNTL__RRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                    0x00000600L
+#define GCEA_DSM_CNTL__RRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00000800L
+#define GCEA_DSM_CNTL__WRET_TAGMEM_DSM_IRRITATOR_DATA_MASK                                                    0x00003000L
+#define GCEA_DSM_CNTL__WRET_TAGMEM_ENABLE_SINGLE_WRITE_MASK                                                   0x00004000L
+#define GCEA_DSM_CNTL__GMIRD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00018000L
+#define GCEA_DSM_CNTL__GMIRD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00020000L
+#define GCEA_DSM_CNTL__GMIWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x000C0000L
+#define GCEA_DSM_CNTL__GMIWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00100000L
+#define GCEA_DSM_CNTL__GMIWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00600000L
+#define GCEA_DSM_CNTL__GMIWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00800000L
+//GCEA_DSM_CNTLA
+#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x0
+#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x2
+#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                              0x3
+#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                             0x5
+#define GCEA_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x6
+#define GCEA_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0x8
+#define GCEA_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA__SHIFT                                                 0x9
+#define GCEA_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE__SHIFT                                                0xb
+#define GCEA_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA__SHIFT                                                0xc
+#define GCEA_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE__SHIFT                                               0xe
+#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                               0xf
+#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x11
+#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA__SHIFT                                               0x12
+#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE__SHIFT                                              0x14
+#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00000003L
+#define GCEA_DSM_CNTLA__DRAMRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000004L
+#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                0x00000018L
+#define GCEA_DSM_CNTLA__DRAMWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                               0x00000020L
+#define GCEA_DSM_CNTLA__IORD_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x000000C0L
+#define GCEA_DSM_CNTLA__IORD_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000100L
+#define GCEA_DSM_CNTLA__IOWR_CMDMEM_DSM_IRRITATOR_DATA_MASK                                                   0x00000600L
+#define GCEA_DSM_CNTLA__IOWR_CMDMEM_ENABLE_SINGLE_WRITE_MASK                                                  0x00000800L
+#define GCEA_DSM_CNTLA__IOWR_DATAMEM_DSM_IRRITATOR_DATA_MASK                                                  0x00003000L
+#define GCEA_DSM_CNTLA__IOWR_DATAMEM_ENABLE_SINGLE_WRITE_MASK                                                 0x00004000L
+#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                 0x00018000L
+#define GCEA_DSM_CNTLA__GMIRD_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00020000L
+#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_DSM_IRRITATOR_DATA_MASK                                                 0x000C0000L
+#define GCEA_DSM_CNTLA__GMIWR_PAGEMEM_ENABLE_SINGLE_WRITE_MASK                                                0x00100000L
+//GCEA_DSM_CNTLB
+#define GCEA_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x0
+#define GCEA_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x2
+#define GCEA_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x3
+#define GCEA_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x5
+#define GCEA_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x6
+#define GCEA_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x8
+#define GCEA_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x9
+#define GCEA_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0xb
+#define GCEA_DSM_CNTLB__MAM_A0MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0xc
+#define GCEA_DSM_CNTLB__MAM_A0MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0xe
+#define GCEA_DSM_CNTLB__MAM_A1MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0xf
+#define GCEA_DSM_CNTLB__MAM_A1MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x11
+#define GCEA_DSM_CNTLB__MAM_A2MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x12
+#define GCEA_DSM_CNTLB__MAM_A2MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x14
+#define GCEA_DSM_CNTLB__MAM_A3MEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x15
+#define GCEA_DSM_CNTLB__MAM_A3MEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x17
+#define GCEA_DSM_CNTLB__MAM_AFMEM_DSM_IRRITATOR_DATA__SHIFT                                                   0x18
+#define GCEA_DSM_CNTLB__MAM_AFMEM_ENABLE_SINGLE_WRITE__SHIFT                                                  0x1a
+#define GCEA_DSM_CNTLB__MAM_D0MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00000003L
+#define GCEA_DSM_CNTLB__MAM_D0MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00000004L
+#define GCEA_DSM_CNTLB__MAM_D1MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00000018L
+#define GCEA_DSM_CNTLB__MAM_D1MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00000020L
+#define GCEA_DSM_CNTLB__MAM_D2MEM_DSM_IRRITATOR_DATA_MASK                                                     0x000000C0L
+#define GCEA_DSM_CNTLB__MAM_D2MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00000100L
+#define GCEA_DSM_CNTLB__MAM_D3MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00000600L
+#define GCEA_DSM_CNTLB__MAM_D3MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00000800L
+#define GCEA_DSM_CNTLB__MAM_A0MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00003000L
+#define GCEA_DSM_CNTLB__MAM_A0MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00004000L
+#define GCEA_DSM_CNTLB__MAM_A1MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00018000L
+#define GCEA_DSM_CNTLB__MAM_A1MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00020000L
+#define GCEA_DSM_CNTLB__MAM_A2MEM_DSM_IRRITATOR_DATA_MASK                                                     0x000C0000L
+#define GCEA_DSM_CNTLB__MAM_A2MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00100000L
+#define GCEA_DSM_CNTLB__MAM_A3MEM_DSM_IRRITATOR_DATA_MASK                                                     0x00600000L
+#define GCEA_DSM_CNTLB__MAM_A3MEM_ENABLE_SINGLE_WRITE_MASK                                                    0x00800000L
+#define GCEA_DSM_CNTLB__MAM_AFMEM_DSM_IRRITATOR_DATA_MASK                                                     0x03000000L
+#define GCEA_DSM_CNTLB__MAM_AFMEM_ENABLE_SINGLE_WRITE_MASK                                                    0x04000000L
+//GCEA_DSM_CNTL2
+#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x0
+#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x2
+#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x3
+#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                              0x5
+#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x6
+#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                             0x8
+#define GCEA_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                                0x9
+#define GCEA_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                                0xb
+#define GCEA_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT__SHIFT                                                0xc
+#define GCEA_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY__SHIFT                                                0xe
+#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0xf
+#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0x11
+#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x12
+#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0x14
+#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                              0x15
+#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                              0x17
+#define GCEA_DSM_CNTL2__INJECT_DELAY__SHIFT                                                                   0x1a
+#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000003L
+#define GCEA_DSM_CNTL2__DRAMRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000004L
+#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                0x00000018L
+#define GCEA_DSM_CNTL2__DRAMWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                0x00000020L
+#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                               0x000000C0L
+#define GCEA_DSM_CNTL2__DRAMWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                               0x00000100L
+#define GCEA_DSM_CNTL2__RRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                  0x00000600L
+#define GCEA_DSM_CNTL2__RRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                  0x00000800L
+#define GCEA_DSM_CNTL2__WRET_TAGMEM_ENABLE_ERROR_INJECT_MASK                                                  0x00003000L
+#define GCEA_DSM_CNTL2__WRET_TAGMEM_SELECT_INJECT_DELAY_MASK                                                  0x00004000L
+#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00018000L
+#define GCEA_DSM_CNTL2__GMIRD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00020000L
+#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x000C0000L
+#define GCEA_DSM_CNTL2__GMIWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00100000L
+#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                                0x00600000L
+#define GCEA_DSM_CNTL2__GMIWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                                0x00800000L
+#define GCEA_DSM_CNTL2__INJECT_DELAY_MASK                                                                     0xFC000000L
+//GCEA_DSM_CNTL2A
+#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x0
+#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x2
+#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                            0x3
+#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                            0x5
+#define GCEA_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x6
+#define GCEA_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0x8
+#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT__SHIFT                                               0x9
+#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY__SHIFT                                               0xb
+#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT__SHIFT                                              0xc
+#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY__SHIFT                                              0xe
+#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                             0xf
+#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                             0x11
+#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT__SHIFT                                             0x12
+#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY__SHIFT                                             0x14
+#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00000003L
+#define GCEA_DSM_CNTL2A__DRAMRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00000004L
+#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                              0x00000018L
+#define GCEA_DSM_CNTL2A__DRAMWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                              0x00000020L
+#define GCEA_DSM_CNTL2A__IORD_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x000000C0L
+#define GCEA_DSM_CNTL2A__IORD_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000100L
+#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_ENABLE_ERROR_INJECT_MASK                                                 0x00000600L
+#define GCEA_DSM_CNTL2A__IOWR_CMDMEM_SELECT_INJECT_DELAY_MASK                                                 0x00000800L
+#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_ENABLE_ERROR_INJECT_MASK                                                0x00003000L
+#define GCEA_DSM_CNTL2A__IOWR_DATAMEM_SELECT_INJECT_DELAY_MASK                                                0x00004000L
+#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                               0x00018000L
+#define GCEA_DSM_CNTL2A__GMIRD_PAGEMEM_SELECT_INJECT_DELAY_MASK                                               0x00020000L
+#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_ENABLE_ERROR_INJECT_MASK                                               0x000C0000L
+#define GCEA_DSM_CNTL2A__GMIWR_PAGEMEM_SELECT_INJECT_DELAY_MASK                                               0x00100000L
+//GCEA_DSM_CNTL2B
+#define GCEA_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x0
+#define GCEA_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x2
+#define GCEA_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x3
+#define GCEA_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x5
+#define GCEA_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x6
+#define GCEA_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x8
+#define GCEA_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x9
+#define GCEA_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY__SHIFT                                                 0xb
+#define GCEA_DSM_CNTL2B__MAM_A0MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0xc
+#define GCEA_DSM_CNTL2B__MAM_A0MEM_SELECT_INJECT_DELAY__SHIFT                                                 0xe
+#define GCEA_DSM_CNTL2B__MAM_A1MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0xf
+#define GCEA_DSM_CNTL2B__MAM_A1MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x11
+#define GCEA_DSM_CNTL2B__MAM_A2MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x12
+#define GCEA_DSM_CNTL2B__MAM_A2MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x14
+#define GCEA_DSM_CNTL2B__MAM_A3MEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x15
+#define GCEA_DSM_CNTL2B__MAM_A3MEM_SELECT_INJECT_DELAY__SHIFT                                                 0x17
+#define GCEA_DSM_CNTL2B__MAM_AFMEM_ENABLE_ERROR_INJECT__SHIFT                                                 0x18
+#define GCEA_DSM_CNTL2B__MAM_AFMEM_SELECT_INJECT_DELAY__SHIFT                                                 0x1a
+#define GCEA_DSM_CNTL2B__MAM_D0MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00000003L
+#define GCEA_DSM_CNTL2B__MAM_D0MEM_SELECT_INJECT_DELAY_MASK                                                   0x00000004L
+#define GCEA_DSM_CNTL2B__MAM_D1MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00000018L
+#define GCEA_DSM_CNTL2B__MAM_D1MEM_SELECT_INJECT_DELAY_MASK                                                   0x00000020L
+#define GCEA_DSM_CNTL2B__MAM_D2MEM_ENABLE_ERROR_INJECT_MASK                                                   0x000000C0L
+#define GCEA_DSM_CNTL2B__MAM_D2MEM_SELECT_INJECT_DELAY_MASK                                                   0x00000100L
+#define GCEA_DSM_CNTL2B__MAM_D3MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00000600L
+#define GCEA_DSM_CNTL2B__MAM_D3MEM_SELECT_INJECT_DELAY_MASK                                                   0x00000800L
+#define GCEA_DSM_CNTL2B__MAM_A0MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00003000L
+#define GCEA_DSM_CNTL2B__MAM_A0MEM_SELECT_INJECT_DELAY_MASK                                                   0x00004000L
+#define GCEA_DSM_CNTL2B__MAM_A1MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00018000L
+#define GCEA_DSM_CNTL2B__MAM_A1MEM_SELECT_INJECT_DELAY_MASK                                                   0x00020000L
+#define GCEA_DSM_CNTL2B__MAM_A2MEM_ENABLE_ERROR_INJECT_MASK                                                   0x000C0000L
+#define GCEA_DSM_CNTL2B__MAM_A2MEM_SELECT_INJECT_DELAY_MASK                                                   0x00100000L
+#define GCEA_DSM_CNTL2B__MAM_A3MEM_ENABLE_ERROR_INJECT_MASK                                                   0x00600000L
+#define GCEA_DSM_CNTL2B__MAM_A3MEM_SELECT_INJECT_DELAY_MASK                                                   0x00800000L
+#define GCEA_DSM_CNTL2B__MAM_AFMEM_ENABLE_ERROR_INJECT_MASK                                                   0x03000000L
+#define GCEA_DSM_CNTL2B__MAM_AFMEM_SELECT_INJECT_DELAY_MASK                                                   0x04000000L
+//GCEA_GL2C_XBR_CREDITS
+#define GCEA_GL2C_XBR_CREDITS__DRAM_RD_LIMIT__SHIFT                                                           0x0
+#define GCEA_GL2C_XBR_CREDITS__DRAM_RD_RESERVE__SHIFT                                                         0x6
+#define GCEA_GL2C_XBR_CREDITS__IO_RD_LIMIT__SHIFT                                                             0x8
+#define GCEA_GL2C_XBR_CREDITS__IO_RD_RESERVE__SHIFT                                                           0xe
+#define GCEA_GL2C_XBR_CREDITS__DRAM_WR_LIMIT__SHIFT                                                           0x10
+#define GCEA_GL2C_XBR_CREDITS__DRAM_WR_RESERVE__SHIFT                                                         0x16
+#define GCEA_GL2C_XBR_CREDITS__IO_WR_LIMIT__SHIFT                                                             0x18
+#define GCEA_GL2C_XBR_CREDITS__IO_WR_RESERVE__SHIFT                                                           0x1e
+#define GCEA_GL2C_XBR_CREDITS__DRAM_RD_LIMIT_MASK                                                             0x0000003FL
+#define GCEA_GL2C_XBR_CREDITS__DRAM_RD_RESERVE_MASK                                                           0x000000C0L
+#define GCEA_GL2C_XBR_CREDITS__IO_RD_LIMIT_MASK                                                               0x00003F00L
+#define GCEA_GL2C_XBR_CREDITS__IO_RD_RESERVE_MASK                                                             0x0000C000L
+#define GCEA_GL2C_XBR_CREDITS__DRAM_WR_LIMIT_MASK                                                             0x003F0000L
+#define GCEA_GL2C_XBR_CREDITS__DRAM_WR_RESERVE_MASK                                                           0x00C00000L
+#define GCEA_GL2C_XBR_CREDITS__IO_WR_LIMIT_MASK                                                               0x3F000000L
+#define GCEA_GL2C_XBR_CREDITS__IO_WR_RESERVE_MASK                                                             0xC0000000L
+//GCEA_GL2C_XBR_MAXBURST
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_RD__SHIFT                                                                0x0
+#define GCEA_GL2C_XBR_MAXBURST__IO_RD__SHIFT                                                                  0x4
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_WR__SHIFT                                                                0x8
+#define GCEA_GL2C_XBR_MAXBURST__IO_WR__SHIFT                                                                  0xc
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_RD_COMB_FLUSH_TIMER__SHIFT                                               0x10
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_RD_COMB_SAME64B_ONLY__SHIFT                                              0x13
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_WR_COMB_FLUSH_TIMER__SHIFT                                               0x14
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_WR_COMB_SAME64B_ONLY__SHIFT                                              0x17
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_RD_MASK                                                                  0x0000000FL
+#define GCEA_GL2C_XBR_MAXBURST__IO_RD_MASK                                                                    0x000000F0L
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_WR_MASK                                                                  0x00000F00L
+#define GCEA_GL2C_XBR_MAXBURST__IO_WR_MASK                                                                    0x0000F000L
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_RD_COMB_FLUSH_TIMER_MASK                                                 0x00070000L
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_RD_COMB_SAME64B_ONLY_MASK                                                0x00080000L
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_WR_COMB_FLUSH_TIMER_MASK                                                 0x00700000L
+#define GCEA_GL2C_XBR_MAXBURST__DRAM_WR_COMB_SAME64B_ONLY_MASK                                                0x00800000L
+//GCEA_PROBE_CNTL
+#define GCEA_PROBE_CNTL__REQ2RSP_DELAY__SHIFT                                                                 0x0
+#define GCEA_PROBE_CNTL__PRB_FILTER_DISABLE__SHIFT                                                            0x5
+#define GCEA_PROBE_CNTL__REQ2RSP_DELAY_MASK                                                                   0x0000001FL
+#define GCEA_PROBE_CNTL__PRB_FILTER_DISABLE_MASK                                                              0x00000020L
+//GCEA_PROBE_MAP
+#define GCEA_PROBE_MAP__CHADDR0_TO_RIGHTGL2C__SHIFT                                                           0x0
+#define GCEA_PROBE_MAP__CHADDR1_TO_RIGHTGL2C__SHIFT                                                           0x1
+#define GCEA_PROBE_MAP__CHADDR2_TO_RIGHTGL2C__SHIFT                                                           0x2
+#define GCEA_PROBE_MAP__CHADDR3_TO_RIGHTGL2C__SHIFT                                                           0x3
+#define GCEA_PROBE_MAP__CHADDR4_TO_RIGHTGL2C__SHIFT                                                           0x4
+#define GCEA_PROBE_MAP__CHADDR5_TO_RIGHTGL2C__SHIFT                                                           0x5
+#define GCEA_PROBE_MAP__CHADDR6_TO_RIGHTGL2C__SHIFT                                                           0x6
+#define GCEA_PROBE_MAP__CHADDR7_TO_RIGHTGL2C__SHIFT                                                           0x7
+#define GCEA_PROBE_MAP__CHADDR8_TO_RIGHTGL2C__SHIFT                                                           0x8
+#define GCEA_PROBE_MAP__CHADDR9_TO_RIGHTGL2C__SHIFT                                                           0x9
+#define GCEA_PROBE_MAP__CHADDR10_TO_RIGHTGL2C__SHIFT                                                          0xa
+#define GCEA_PROBE_MAP__CHADDR11_TO_RIGHTGL2C__SHIFT                                                          0xb
+#define GCEA_PROBE_MAP__CHADDR12_TO_RIGHTGL2C__SHIFT                                                          0xc
+#define GCEA_PROBE_MAP__CHADDR13_TO_RIGHTGL2C__SHIFT                                                          0xd
+#define GCEA_PROBE_MAP__CHADDR14_TO_RIGHTGL2C__SHIFT                                                          0xe
+#define GCEA_PROBE_MAP__CHADDR15_TO_RIGHTGL2C__SHIFT                                                          0xf
+#define GCEA_PROBE_MAP__INTLV_SIZE__SHIFT                                                                     0x10
+#define GCEA_PROBE_MAP__CHADDR0_TO_RIGHTGL2C_MASK                                                             0x00000001L
+#define GCEA_PROBE_MAP__CHADDR1_TO_RIGHTGL2C_MASK                                                             0x00000002L
+#define GCEA_PROBE_MAP__CHADDR2_TO_RIGHTGL2C_MASK                                                             0x00000004L
+#define GCEA_PROBE_MAP__CHADDR3_TO_RIGHTGL2C_MASK                                                             0x00000008L
+#define GCEA_PROBE_MAP__CHADDR4_TO_RIGHTGL2C_MASK                                                             0x00000010L
+#define GCEA_PROBE_MAP__CHADDR5_TO_RIGHTGL2C_MASK                                                             0x00000020L
+#define GCEA_PROBE_MAP__CHADDR6_TO_RIGHTGL2C_MASK                                                             0x00000040L
+#define GCEA_PROBE_MAP__CHADDR7_TO_RIGHTGL2C_MASK                                                             0x00000080L
+#define GCEA_PROBE_MAP__CHADDR8_TO_RIGHTGL2C_MASK                                                             0x00000100L
+#define GCEA_PROBE_MAP__CHADDR9_TO_RIGHTGL2C_MASK                                                             0x00000200L
+#define GCEA_PROBE_MAP__CHADDR10_TO_RIGHTGL2C_MASK                                                            0x00000400L
+#define GCEA_PROBE_MAP__CHADDR11_TO_RIGHTGL2C_MASK                                                            0x00000800L
+#define GCEA_PROBE_MAP__CHADDR12_TO_RIGHTGL2C_MASK                                                            0x00001000L
+#define GCEA_PROBE_MAP__CHADDR13_TO_RIGHTGL2C_MASK                                                            0x00002000L
+#define GCEA_PROBE_MAP__CHADDR14_TO_RIGHTGL2C_MASK                                                            0x00004000L
+#define GCEA_PROBE_MAP__CHADDR15_TO_RIGHTGL2C_MASK                                                            0x00008000L
+#define GCEA_PROBE_MAP__INTLV_SIZE_MASK                                                                       0x00030000L
+//GCEA_ERR_STATUS
+#define GCEA_ERR_STATUS__SDP_RDRSP_STATUS__SHIFT                                                              0x0
+#define GCEA_ERR_STATUS__SDP_WRRSP_STATUS__SHIFT                                                              0x4
+#define GCEA_ERR_STATUS__SDP_RDRSP_DATASTATUS__SHIFT                                                          0x8
+#define GCEA_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR__SHIFT                                                    0xa
+#define GCEA_ERR_STATUS__CLEAR_ERROR_STATUS__SHIFT                                                            0xb
+#define GCEA_ERR_STATUS__BUSY_ON_ERROR__SHIFT                                                                 0xc
+#define GCEA_ERR_STATUS__FUE_FLAG__SHIFT                                                                      0xd
+#define GCEA_ERR_STATUS__IGNORE_RDRSP_FED__SHIFT                                                              0xe
+#define GCEA_ERR_STATUS__INTERRUPT_ON_FATAL__SHIFT                                                            0xf
+#define GCEA_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL__SHIFT                                                    0x10
+#define GCEA_ERR_STATUS__LEVEL_INTERRUPT__SHIFT                                                               0x11
+#define GCEA_ERR_STATUS__SDP_RDRSP_STATUS_MASK                                                                0x0000000FL
+#define GCEA_ERR_STATUS__SDP_WRRSP_STATUS_MASK                                                                0x000000F0L
+#define GCEA_ERR_STATUS__SDP_RDRSP_DATASTATUS_MASK                                                            0x00000300L
+#define GCEA_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR_MASK                                                      0x00000400L
+#define GCEA_ERR_STATUS__CLEAR_ERROR_STATUS_MASK                                                              0x00000800L
+#define GCEA_ERR_STATUS__BUSY_ON_ERROR_MASK                                                                   0x00001000L
+#define GCEA_ERR_STATUS__FUE_FLAG_MASK                                                                        0x00002000L
+#define GCEA_ERR_STATUS__IGNORE_RDRSP_FED_MASK                                                                0x00004000L
+#define GCEA_ERR_STATUS__INTERRUPT_ON_FATAL_MASK                                                              0x00008000L
+#define GCEA_ERR_STATUS__INTERRUPT_IGNORE_CLI_FATAL_MASK                                                      0x00010000L
+#define GCEA_ERR_STATUS__LEVEL_INTERRUPT_MASK                                                                 0x00020000L
+//GCEA_MISC2
+#define GCEA_MISC2__CSGROUP_SWAP_IN_DRAM_ARB__SHIFT                                                           0x0
+#define GCEA_MISC2__CSGROUP_SWAP_IN_GMI_ARB__SHIFT                                                            0x1
+#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM__SHIFT                                                        0x2
+#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_GMI__SHIFT                                                         0x7
+#define GCEA_MISC2__IO_RDWR_PRIORITY_ENABLE__SHIFT                                                            0xc
+#define GCEA_MISC2__BLOCK_REQUESTS__SHIFT                                                                     0xd
+#define GCEA_MISC2__REQUESTS_BLOCKED__SHIFT                                                                   0xe
+#define GCEA_MISC2__FGCLKEN_OVERRIDE__SHIFT                                                                   0xf
+#define GCEA_MISC2__LINKMGR_CRBUSY_MASK__SHIFT                                                                0x10
+#define GCEA_MISC2__RDRET_FED_MASK__SHIFT                                                                     0x11
+#define GCEA_MISC2__CSGROUP_SWAP_IN_DRAM_ARB_MASK                                                             0x00000001L
+#define GCEA_MISC2__CSGROUP_SWAP_IN_GMI_ARB_MASK                                                              0x00000002L
+#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_DRAM_MASK                                                          0x0000007CL
+#define GCEA_MISC2__CSGRP_BURST_LIMIT_DATA_GMI_MASK                                                           0x00000F80L
+#define GCEA_MISC2__IO_RDWR_PRIORITY_ENABLE_MASK                                                              0x00001000L
+#define GCEA_MISC2__BLOCK_REQUESTS_MASK                                                                       0x00002000L
+#define GCEA_MISC2__REQUESTS_BLOCKED_MASK                                                                     0x00004000L
+#define GCEA_MISC2__FGCLKEN_OVERRIDE_MASK                                                                     0x00008000L
+#define GCEA_MISC2__LINKMGR_CRBUSY_MASK_MASK                                                                  0x00010000L
+#define GCEA_MISC2__RDRET_FED_MASK_MASK                                                                       0x00020000L
+
+
+// addressBlock: gc_gceadec3
+//GCEA_SDP_BACKDOOR_CMDCREDITS0
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC0_CREDITS_RECEIVED__SHIFT                                            0x0
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC1_CREDITS_RECEIVED__SHIFT                                            0x7
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC2_CREDITS_RECEIVED__SHIFT                                            0xe
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC3_CREDITS_RECEIVED__SHIFT                                            0x15
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC4_CREDITS_RECEIVED__SHIFT                                            0x1c
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC0_CREDITS_RECEIVED_MASK                                              0x0000007FL
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC1_CREDITS_RECEIVED_MASK                                              0x00003F80L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC2_CREDITS_RECEIVED_MASK                                              0x001FC000L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC3_CREDITS_RECEIVED_MASK                                              0x0FE00000L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS0__VC4_CREDITS_RECEIVED_MASK                                              0xF0000000L
+//GCEA_SDP_BACKDOOR_CMDCREDITS1
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC4_CREDITS_RECEIVED__SHIFT                                            0x0
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC5_CREDITS_RECEIVED__SHIFT                                            0x3
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC6_CREDITS_RECEIVED__SHIFT                                            0xa
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC7_CREDITS_RECEIVED__SHIFT                                            0x11
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__POOL_CREDITS_RECEIVED__SHIFT                                           0x18
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC4_CREDITS_RECEIVED_MASK                                              0x00000007L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC5_CREDITS_RECEIVED_MASK                                              0x000003F8L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC6_CREDITS_RECEIVED_MASK                                              0x0001FC00L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__VC7_CREDITS_RECEIVED_MASK                                              0x00FE0000L
+#define GCEA_SDP_BACKDOOR_CMDCREDITS1__POOL_CREDITS_RECEIVED_MASK                                             0x7F000000L
+//GCEA_SDP_BACKDOOR_DATACREDITS0
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC0_CREDITS_RECEIVED__SHIFT                                           0x0
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC1_CREDITS_RECEIVED__SHIFT                                           0x7
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC2_CREDITS_RECEIVED__SHIFT                                           0xe
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC3_CREDITS_RECEIVED__SHIFT                                           0x15
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC4_CREDITS_RECEIVED__SHIFT                                           0x1c
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC0_CREDITS_RECEIVED_MASK                                             0x0000007FL
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC1_CREDITS_RECEIVED_MASK                                             0x00003F80L
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC2_CREDITS_RECEIVED_MASK                                             0x001FC000L
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC3_CREDITS_RECEIVED_MASK                                             0x0FE00000L
+#define GCEA_SDP_BACKDOOR_DATACREDITS0__VC4_CREDITS_RECEIVED_MASK                                             0xF0000000L
+//GCEA_SDP_BACKDOOR_DATACREDITS1
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC4_CREDITS_RECEIVED__SHIFT                                           0x0
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC5_CREDITS_RECEIVED__SHIFT                                           0x3
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC6_CREDITS_RECEIVED__SHIFT                                           0xa
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC7_CREDITS_RECEIVED__SHIFT                                           0x11
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__POOL_CREDITS_RECEIVED__SHIFT                                          0x18
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC4_CREDITS_RECEIVED_MASK                                             0x00000007L
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC5_CREDITS_RECEIVED_MASK                                             0x000003F8L
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC6_CREDITS_RECEIVED_MASK                                             0x0001FC00L
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__VC7_CREDITS_RECEIVED_MASK                                             0x00FE0000L
+#define GCEA_SDP_BACKDOOR_DATACREDITS1__POOL_CREDITS_RECEIVED_MASK                                            0x7F000000L
+//GCEA_SDP_BACKDOOR_MISCCREDITS
+#define GCEA_SDP_BACKDOOR_MISCCREDITS__PRB_RSP_CREDITS_RECEIVED__SHIFT                                        0x0
+#define GCEA_SDP_BACKDOOR_MISCCREDITS__PRB_RSP_CREDITS_RECEIVED_MASK                                          0x0000007FL
+//GCEA_RRET_MEM_RESERVE
+#define GCEA_RRET_MEM_RESERVE__VC0__SHIFT                                                                     0x0
+#define GCEA_RRET_MEM_RESERVE__VC1__SHIFT                                                                     0x4
+#define GCEA_RRET_MEM_RESERVE__VC2__SHIFT                                                                     0x8
+#define GCEA_RRET_MEM_RESERVE__VC3__SHIFT                                                                     0xc
+#define GCEA_RRET_MEM_RESERVE__VC4__SHIFT                                                                     0x10
+#define GCEA_RRET_MEM_RESERVE__VC5__SHIFT                                                                     0x14
+#define GCEA_RRET_MEM_RESERVE__VC6__SHIFT                                                                     0x18
+#define GCEA_RRET_MEM_RESERVE__VC7__SHIFT                                                                     0x1c
+#define GCEA_RRET_MEM_RESERVE__VC0_MASK                                                                       0x0000000FL
+#define GCEA_RRET_MEM_RESERVE__VC1_MASK                                                                       0x000000F0L
+#define GCEA_RRET_MEM_RESERVE__VC2_MASK                                                                       0x00000F00L
+#define GCEA_RRET_MEM_RESERVE__VC3_MASK                                                                       0x0000F000L
+#define GCEA_RRET_MEM_RESERVE__VC4_MASK                                                                       0x000F0000L
+#define GCEA_RRET_MEM_RESERVE__VC5_MASK                                                                       0x00F00000L
+#define GCEA_RRET_MEM_RESERVE__VC6_MASK                                                                       0x0F000000L
+#define GCEA_RRET_MEM_RESERVE__VC7_MASK                                                                       0xF0000000L
+//GCEA_EDC_CNT3
+#define GCEA_EDC_CNT3__DRAMRD_PAGEMEM_DED_COUNT__SHIFT                                                        0x0
+#define GCEA_EDC_CNT3__DRAMWR_PAGEMEM_DED_COUNT__SHIFT                                                        0x2
+#define GCEA_EDC_CNT3__IORD_CMDMEM_DED_COUNT__SHIFT                                                           0x4
+#define GCEA_EDC_CNT3__IOWR_CMDMEM_DED_COUNT__SHIFT                                                           0x6
+#define GCEA_EDC_CNT3__GMIRD_PAGEMEM_DED_COUNT__SHIFT                                                         0x8
+#define GCEA_EDC_CNT3__GMIWR_PAGEMEM_DED_COUNT__SHIFT                                                         0xa
+#define GCEA_EDC_CNT3__MAM_A0MEM_SEC_COUNT__SHIFT                                                             0xc
+#define GCEA_EDC_CNT3__MAM_A0MEM_DED_COUNT__SHIFT                                                             0xe
+#define GCEA_EDC_CNT3__MAM_A1MEM_SEC_COUNT__SHIFT                                                             0x10
+#define GCEA_EDC_CNT3__MAM_A1MEM_DED_COUNT__SHIFT                                                             0x12
+#define GCEA_EDC_CNT3__MAM_A2MEM_SEC_COUNT__SHIFT                                                             0x14
+#define GCEA_EDC_CNT3__MAM_A2MEM_DED_COUNT__SHIFT                                                             0x16
+#define GCEA_EDC_CNT3__MAM_A3MEM_SEC_COUNT__SHIFT                                                             0x18
+#define GCEA_EDC_CNT3__MAM_A3MEM_DED_COUNT__SHIFT                                                             0x1a
+#define GCEA_EDC_CNT3__MAM_AFMEM_SEC_COUNT__SHIFT                                                             0x1c
+#define GCEA_EDC_CNT3__MAM_AFMEM_DED_COUNT__SHIFT                                                             0x1e
+#define GCEA_EDC_CNT3__DRAMRD_PAGEMEM_DED_COUNT_MASK                                                          0x00000003L
+#define GCEA_EDC_CNT3__DRAMWR_PAGEMEM_DED_COUNT_MASK                                                          0x0000000CL
+#define GCEA_EDC_CNT3__IORD_CMDMEM_DED_COUNT_MASK                                                             0x00000030L
+#define GCEA_EDC_CNT3__IOWR_CMDMEM_DED_COUNT_MASK                                                             0x000000C0L
+#define GCEA_EDC_CNT3__GMIRD_PAGEMEM_DED_COUNT_MASK                                                           0x00000300L
+#define GCEA_EDC_CNT3__GMIWR_PAGEMEM_DED_COUNT_MASK                                                           0x00000C00L
+#define GCEA_EDC_CNT3__MAM_A0MEM_SEC_COUNT_MASK                                                               0x00003000L
+#define GCEA_EDC_CNT3__MAM_A0MEM_DED_COUNT_MASK                                                               0x0000C000L
+#define GCEA_EDC_CNT3__MAM_A1MEM_SEC_COUNT_MASK                                                               0x00030000L
+#define GCEA_EDC_CNT3__MAM_A1MEM_DED_COUNT_MASK                                                               0x000C0000L
+#define GCEA_EDC_CNT3__MAM_A2MEM_SEC_COUNT_MASK                                                               0x00300000L
+#define GCEA_EDC_CNT3__MAM_A2MEM_DED_COUNT_MASK                                                               0x00C00000L
+#define GCEA_EDC_CNT3__MAM_A3MEM_SEC_COUNT_MASK                                                               0x03000000L
+#define GCEA_EDC_CNT3__MAM_A3MEM_DED_COUNT_MASK                                                               0x0C000000L
+#define GCEA_EDC_CNT3__MAM_AFMEM_SEC_COUNT_MASK                                                               0x30000000L
+#define GCEA_EDC_CNT3__MAM_AFMEM_DED_COUNT_MASK                                                               0xC0000000L
+//GCEA_SDP_ENABLE
+#define GCEA_SDP_ENABLE__ENABLE__SHIFT                                                                        0x0
+#define GCEA_SDP_ENABLE__EARLY_CREDIT_REQUEST__SHIFT                                                          0x1
+#define GCEA_SDP_ENABLE__ENABLE_MASK                                                                          0x00000001L
+#define GCEA_SDP_ENABLE__EARLY_CREDIT_REQUEST_MASK                                                            0x00000002L
+
+
+// addressBlock: gc_spipdec2
+//SPI_PQEV_CTRL
+#define SPI_PQEV_CTRL__SCAN_PERIOD__SHIFT                                                                     0x0
+#define SPI_PQEV_CTRL__QUEUE_DURATION__SHIFT                                                                  0xa
+#define SPI_PQEV_CTRL__COMPUTE_PIPE_EN__SHIFT                                                                 0x10
+#define SPI_PQEV_CTRL__SCAN_PERIOD_MASK                                                                       0x000003FFL
+#define SPI_PQEV_CTRL__QUEUE_DURATION_MASK                                                                    0x0000FC00L
+#define SPI_PQEV_CTRL__COMPUTE_PIPE_EN_MASK                                                                   0x00FF0000L
+//SPI_EXP_THROTTLE_CTRL
+#define SPI_EXP_THROTTLE_CTRL__ENABLE__SHIFT                                                                  0x0
+#define SPI_EXP_THROTTLE_CTRL__PERIOD__SHIFT                                                                  0x1
+#define SPI_EXP_THROTTLE_CTRL__UPSTEP__SHIFT                                                                  0x5
+#define SPI_EXP_THROTTLE_CTRL__DOWNSTEP__SHIFT                                                                0x9
+#define SPI_EXP_THROTTLE_CTRL__LOW_STALL_MON_HIST_COUNT__SHIFT                                                0xd
+#define SPI_EXP_THROTTLE_CTRL__HIGH_STALL_MON_HIST_COUNT__SHIFT                                               0x10
+#define SPI_EXP_THROTTLE_CTRL__EXP_STALL_THRESHOLD__SHIFT                                                     0x13
+#define SPI_EXP_THROTTLE_CTRL__SKEW_COUNT__SHIFT                                                              0x1a
+#define SPI_EXP_THROTTLE_CTRL__THROTTLE_RESET__SHIFT                                                          0x1d
+#define SPI_EXP_THROTTLE_CTRL__ENABLE_MASK                                                                    0x00000001L
+#define SPI_EXP_THROTTLE_CTRL__PERIOD_MASK                                                                    0x0000001EL
+#define SPI_EXP_THROTTLE_CTRL__UPSTEP_MASK                                                                    0x000001E0L
+#define SPI_EXP_THROTTLE_CTRL__DOWNSTEP_MASK                                                                  0x00001E00L
+#define SPI_EXP_THROTTLE_CTRL__LOW_STALL_MON_HIST_COUNT_MASK                                                  0x0000E000L
+#define SPI_EXP_THROTTLE_CTRL__HIGH_STALL_MON_HIST_COUNT_MASK                                                 0x00070000L
+#define SPI_EXP_THROTTLE_CTRL__EXP_STALL_THRESHOLD_MASK                                                       0x03F80000L
+#define SPI_EXP_THROTTLE_CTRL__SKEW_COUNT_MASK                                                                0x1C000000L
+#define SPI_EXP_THROTTLE_CTRL__THROTTLE_RESET_MASK                                                            0x20000000L
+
+
+// addressBlock: gc_rmi_rmidec
+//RMI_GENERAL_CNTL
+#define RMI_GENERAL_CNTL__BURST_DISABLE__SHIFT                                                                0x0
+#define RMI_GENERAL_CNTL__VMID_BYPASS_ENABLE__SHIFT                                                           0x1
+#define RMI_GENERAL_CNTL__RB0_HARVEST_EN__SHIFT                                                               0x13
+#define RMI_GENERAL_CNTL__LOOPBACK_DIS_BY_REQ_TYPE__SHIFT                                                     0x15
+#define RMI_GENERAL_CNTL__BURST_DISABLE_MASK                                                                  0x00000001L
+#define RMI_GENERAL_CNTL__VMID_BYPASS_ENABLE_MASK                                                             0x0001FFFEL
+#define RMI_GENERAL_CNTL__RB0_HARVEST_EN_MASK                                                                 0x00080000L
+#define RMI_GENERAL_CNTL__LOOPBACK_DIS_BY_REQ_TYPE_MASK                                                       0x01E00000L
+//RMI_GENERAL_CNTL1
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_ENABLE_PER_MTYPE__SHIFT                                                0x0
+#define RMI_GENERAL_CNTL1__TCIW0_64B_RD_STALL_MODE__SHIFT                                                     0x4
+#define RMI_GENERAL_CNTL1__TCIW1_64B_RD_STALL_MODE__SHIFT                                                     0x6
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_DISABLE_FOR_LOOPBACK__SHIFT                                            0x8
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_VALUE__SHIFT                                                       0x9
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE__SHIFT                                                             0xb
+#define RMI_GENERAL_CNTL1__ARBITER_ADDRESS_CHANGE_ENABLE__SHIFT                                               0xe
+#define RMI_GENERAL_CNTL1__LAST_OF_BURST_INSERTION_DISABLE__SHIFT                                             0xf
+#define RMI_GENERAL_CNTL1__TCIW0_PRODUCER_CREDITS__SHIFT                                                      0x10
+#define RMI_GENERAL_CNTL1__TCIW1_PRODUCER_CREDITS__SHIFT                                                      0x16
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_ENABLE_PER_MTYPE_MASK                                                  0x0000000FL
+#define RMI_GENERAL_CNTL1__TCIW0_64B_RD_STALL_MODE_MASK                                                       0x00000030L
+#define RMI_GENERAL_CNTL1__TCIW1_64B_RD_STALL_MODE_MASK                                                       0x000000C0L
+#define RMI_GENERAL_CNTL1__EARLY_WRACK_DISABLE_FOR_LOOPBACK_MASK                                              0x00000100L
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_VALUE_MASK                                                         0x00000600L
+#define RMI_GENERAL_CNTL1__POLICY_OVERRIDE_MASK                                                               0x00000800L
+#define RMI_GENERAL_CNTL1__ARBITER_ADDRESS_CHANGE_ENABLE_MASK                                                 0x00004000L
+#define RMI_GENERAL_CNTL1__LAST_OF_BURST_INSERTION_DISABLE_MASK                                               0x00008000L
+#define RMI_GENERAL_CNTL1__TCIW0_PRODUCER_CREDITS_MASK                                                        0x003F0000L
+#define RMI_GENERAL_CNTL1__TCIW1_PRODUCER_CREDITS_MASK                                                        0x0FC00000L
+//RMI_GENERAL_STATUS
+#define RMI_GENERAL_STATUS__GENERAL_RMI_ERRORS_COMBINED__SHIFT                                                0x0
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_OVERFLOW_ERROR__SHIFT                                                 0x1
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_UNDERFLOW_ERROR__SHIFT                                                0x2
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_OVERFLOW_ERROR__SHIFT                                                 0x3
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_UNDERFLOW_ERROR__SHIFT                                                0x4
+#define RMI_GENERAL_STATUS__RMI_XBAR_BUSY__SHIFT                                                              0x5
+#define RMI_GENERAL_STATUS__RESERVED_BIT_6__SHIFT                                                             0x6
+#define RMI_GENERAL_STATUS__RMI_SCOREBOARD_BUSY__SHIFT                                                        0x7
+#define RMI_GENERAL_STATUS__TCIW0_PRT_FIFO_BUSY__SHIFT                                                        0x8
+#define RMI_GENERAL_STATUS__TCIW_FRMTR0_BUSY__SHIFT                                                           0x9
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR0_BUSY__SHIFT                                                       0xa
+#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_0_BUSY__SHIFT                                                 0xb
+#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_0_BUSY__SHIFT                                                 0xc
+#define RMI_GENERAL_STATUS__TCIW1_PRT_FIFO_BUSY__SHIFT                                                        0xd
+#define RMI_GENERAL_STATUS__TCIW_FRMTR1_BUSY__SHIFT                                                           0xe
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR1_BUSY__SHIFT                                                       0xf
+#define RMI_GENERAL_STATUS__RESERVED_BIT_18__SHIFT                                                            0x12
+#define RMI_GENERAL_STATUS__RESERVED_BIT_19__SHIFT                                                            0x13
+#define RMI_GENERAL_STATUS__RESERVED_BIT_20__SHIFT                                                            0x14
+#define RMI_GENERAL_STATUS__RESERVED_BITS_28_21__SHIFT                                                        0x15
+#define RMI_GENERAL_STATUS__RESERVED_BIT_29__SHIFT                                                            0x1d
+#define RMI_GENERAL_STATUS__RESERVED_BIT_30__SHIFT                                                            0x1e
+#define RMI_GENERAL_STATUS__SKID_FIFO_FREESPACE_IS_ZERO_ERROR__SHIFT                                          0x1f
+#define RMI_GENERAL_STATUS__GENERAL_RMI_ERRORS_COMBINED_MASK                                                  0x00000001L
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_OVERFLOW_ERROR_MASK                                                   0x00000002L
+#define RMI_GENERAL_STATUS__SKID_FIFO_0_UNDERFLOW_ERROR_MASK                                                  0x00000004L
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_OVERFLOW_ERROR_MASK                                                   0x00000008L
+#define RMI_GENERAL_STATUS__SKID_FIFO_1_UNDERFLOW_ERROR_MASK                                                  0x00000010L
+#define RMI_GENERAL_STATUS__RMI_XBAR_BUSY_MASK                                                                0x00000020L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_6_MASK                                                               0x00000040L
+#define RMI_GENERAL_STATUS__RMI_SCOREBOARD_BUSY_MASK                                                          0x00000080L
+#define RMI_GENERAL_STATUS__TCIW0_PRT_FIFO_BUSY_MASK                                                          0x00000100L
+#define RMI_GENERAL_STATUS__TCIW_FRMTR0_BUSY_MASK                                                             0x00000200L
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR0_BUSY_MASK                                                         0x00000400L
+#define RMI_GENERAL_STATUS__WRREQ_CONSUMER_FIFO_0_BUSY_MASK                                                   0x00000800L
+#define RMI_GENERAL_STATUS__RDREQ_CONSUMER_FIFO_0_BUSY_MASK                                                   0x00001000L
+#define RMI_GENERAL_STATUS__TCIW1_PRT_FIFO_BUSY_MASK                                                          0x00002000L
+#define RMI_GENERAL_STATUS__TCIW_FRMTR1_BUSY_MASK                                                             0x00004000L
+#define RMI_GENERAL_STATUS__TCIW_RTN_FRMTR1_BUSY_MASK                                                         0x00008000L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_18_MASK                                                              0x00040000L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_19_MASK                                                              0x00080000L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_20_MASK                                                              0x00100000L
+#define RMI_GENERAL_STATUS__RESERVED_BITS_28_21_MASK                                                          0x1FE00000L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_29_MASK                                                              0x20000000L
+#define RMI_GENERAL_STATUS__RESERVED_BIT_30_MASK                                                              0x40000000L
+#define RMI_GENERAL_STATUS__SKID_FIFO_FREESPACE_IS_ZERO_ERROR_MASK                                            0x80000000L
+//RMI_SUBBLOCK_STATUS0
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE0__SHIFT                                     0x0
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE0__SHIFT                                         0x7
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE0__SHIFT                                        0x8
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE1__SHIFT                                     0x9
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE1__SHIFT                                         0x10
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE1__SHIFT                                        0x11
+#define RMI_SUBBLOCK_STATUS0__TCIW0_INFLIGHT_CNT__SHIFT                                                       0x12
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE0_MASK                                       0x0000007FL
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE0_MASK                                           0x00000080L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE0_MASK                                          0x00000100L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_NUM_USED_PROBE1_MASK                                       0x0000FE00L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_FULL_PROBE1_MASK                                           0x00010000L
+#define RMI_SUBBLOCK_STATUS0__UTC_EXT_LAT_HID_FIFO_EMPTY_PROBE1_MASK                                          0x00020000L
+#define RMI_SUBBLOCK_STATUS0__TCIW0_INFLIGHT_CNT_MASK                                                         0x0FFC0000L
+//RMI_SUBBLOCK_STATUS1
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_0_FREE_SPACE__SHIFT                                                   0x0
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_1_FREE_SPACE__SHIFT                                                   0xa
+#define RMI_SUBBLOCK_STATUS1__TCIW1_INFLIGHT_CNT__SHIFT                                                       0x14
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_0_FREE_SPACE_MASK                                                     0x000003FFL
+#define RMI_SUBBLOCK_STATUS1__SKID_FIFO_1_FREE_SPACE_MASK                                                     0x000FFC00L
+#define RMI_SUBBLOCK_STATUS1__TCIW1_INFLIGHT_CNT_MASK                                                         0x3FF00000L
+//RMI_SUBBLOCK_STATUS2
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_0_NUM_USED__SHIFT                                                      0x0
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_1_NUM_USED__SHIFT                                                      0x9
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_0_NUM_USED_MASK                                                        0x000001FFL
+#define RMI_SUBBLOCK_STATUS2__PRT_FIFO_1_NUM_USED_MASK                                                        0x0003FE00L
+//RMI_SUBBLOCK_STATUS3
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_0_FREE_SPACE_TOTAL__SHIFT                                             0x0
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_1_FREE_SPACE_TOTAL__SHIFT                                             0xa
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_0_FREE_SPACE_TOTAL_MASK                                               0x000003FFL
+#define RMI_SUBBLOCK_STATUS3__SKID_FIFO_1_FREE_SPACE_TOTAL_MASK                                               0x000FFC00L
+//RMI_XBAR_CONFIG
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_OVERRIDE__SHIFT                                                      0x0
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_REQ_TYPE_OVERRIDE__SHIFT                                             0x2
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_CB_DB_OVERRIDE__SHIFT                                                0x6
+#define RMI_XBAR_CONFIG__ARBITER_DIS__SHIFT                                                                   0x7
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ__SHIFT                                                                0x8
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_OVERRIDE__SHIFT                                                       0xc
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB0__SHIFT                                                                0xd
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_OVERRIDE_MASK                                                        0x00000003L
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_REQ_TYPE_OVERRIDE_MASK                                               0x0000003CL
+#define RMI_XBAR_CONFIG__XBAR_MUX_CONFIG_CB_DB_OVERRIDE_MASK                                                  0x00000040L
+#define RMI_XBAR_CONFIG__ARBITER_DIS_MASK                                                                     0x00000080L
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_MASK                                                                  0x00000F00L
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_REQ_OVERRIDE_MASK                                                         0x00001000L
+#define RMI_XBAR_CONFIG__XBAR_EN_IN_RB0_MASK                                                                  0x00002000L
+//RMI_PROBE_POP_LOGIC_CNTL
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_0_MAX_DEPTH__SHIFT                                             0x0
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE0_DIS__SHIFT                                                    0x7
+#define RMI_PROBE_POP_LOGIC_CNTL__REDUCE_MAX_XLAT_CHAIN_SIZE_BY_2__SHIFT                                      0x8
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_1_MAX_DEPTH__SHIFT                                             0xa
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE1_DIS__SHIFT                                                    0x11
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_0_MAX_DEPTH_MASK                                               0x0000007FL
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE0_DIS_MASK                                                      0x00000080L
+#define RMI_PROBE_POP_LOGIC_CNTL__REDUCE_MAX_XLAT_CHAIN_SIZE_BY_2_MASK                                        0x00000300L
+#define RMI_PROBE_POP_LOGIC_CNTL__EXT_LAT_FIFO_1_MAX_DEPTH_MASK                                               0x0001FC00L
+#define RMI_PROBE_POP_LOGIC_CNTL__XLAT_COMBINE1_DIS_MASK                                                      0x00020000L
+//RMI_UTC_XNACK_N_MISC_CNTL
+#define RMI_UTC_XNACK_N_MISC_CNTL__MASTER_XNACK_TIMER_INC__SHIFT                                              0x0
+#define RMI_UTC_XNACK_N_MISC_CNTL__IND_XNACK_TIMER_START_VALUE__SHIFT                                         0x8
+#define RMI_UTC_XNACK_N_MISC_CNTL__UTCL1_PERM_MODE__SHIFT                                                     0xc
+#define RMI_UTC_XNACK_N_MISC_CNTL__CP_VMID_RESET_REQUEST_DISABLE__SHIFT                                       0xd
+#define RMI_UTC_XNACK_N_MISC_CNTL__MASTER_XNACK_TIMER_INC_MASK                                                0x000000FFL
+#define RMI_UTC_XNACK_N_MISC_CNTL__IND_XNACK_TIMER_START_VALUE_MASK                                           0x00000F00L
+#define RMI_UTC_XNACK_N_MISC_CNTL__UTCL1_PERM_MODE_MASK                                                       0x00001000L
+#define RMI_UTC_XNACK_N_MISC_CNTL__CP_VMID_RESET_REQUEST_DISABLE_MASK                                         0x00002000L
+//RMI_DEMUX_CNTL
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE_OVERRIDE_EN__SHIFT                                                    0x2
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_START_VALUE__SHIFT                                             0x6
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE__SHIFT                                                                0xe
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE_OVERRIDE_EN__SHIFT                                                    0x12
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_START_VALUE__SHIFT                                             0x16
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE__SHIFT                                                                0x1e
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE_OVERRIDE_EN_MASK                                                      0x00000004L
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_STALL_TIMER_START_VALUE_MASK                                               0x00003FC0L
+#define RMI_DEMUX_CNTL__DEMUX_ARB0_MODE_MASK                                                                  0x0000C000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE_OVERRIDE_EN_MASK                                                      0x00040000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_STALL_TIMER_START_VALUE_MASK                                               0x3FC00000L
+#define RMI_DEMUX_CNTL__DEMUX_ARB1_MODE_MASK                                                                  0xC0000000L
+//RMI_UTCL1_CNTL1
+#define RMI_UTCL1_CNTL1__FORCE_4K_L2_RESP__SHIFT                                                              0x0
+#define RMI_UTCL1_CNTL1__GPUVM_64K_DEF__SHIFT                                                                 0x1
+#define RMI_UTCL1_CNTL1__GPUVM_PERM_MODE__SHIFT                                                               0x2
+#define RMI_UTCL1_CNTL1__RESP_MODE__SHIFT                                                                     0x3
+#define RMI_UTCL1_CNTL1__RESP_FAULT_MODE__SHIFT                                                               0x5
+#define RMI_UTCL1_CNTL1__CLIENTID__SHIFT                                                                      0x7
+#define RMI_UTCL1_CNTL1__USERVM_DIS__SHIFT                                                                    0x10
+#define RMI_UTCL1_CNTL1__ENABLE_PUSH_LFIFO__SHIFT                                                             0x11
+#define RMI_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB__SHIFT                                                          0x12
+#define RMI_UTCL1_CNTL1__REG_INV_VMID__SHIFT                                                                  0x13
+#define RMI_UTCL1_CNTL1__REG_INV_ALL_VMID__SHIFT                                                              0x17
+#define RMI_UTCL1_CNTL1__REG_INV_TOGGLE__SHIFT                                                                0x18
+#define RMI_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID__SHIFT                                                    0x19
+#define RMI_UTCL1_CNTL1__FORCE_MISS__SHIFT                                                                    0x1a
+#define RMI_UTCL1_CNTL1__FORCE_IN_ORDER__SHIFT                                                                0x1b
+#define RMI_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2__SHIFT                                                        0x1c
+#define RMI_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2__SHIFT                                                        0x1e
+#define RMI_UTCL1_CNTL1__FORCE_4K_L2_RESP_MASK                                                                0x00000001L
+#define RMI_UTCL1_CNTL1__GPUVM_64K_DEF_MASK                                                                   0x00000002L
+#define RMI_UTCL1_CNTL1__GPUVM_PERM_MODE_MASK                                                                 0x00000004L
+#define RMI_UTCL1_CNTL1__RESP_MODE_MASK                                                                       0x00000018L
+#define RMI_UTCL1_CNTL1__RESP_FAULT_MODE_MASK                                                                 0x00000060L
+#define RMI_UTCL1_CNTL1__CLIENTID_MASK                                                                        0x0000FF80L
+#define RMI_UTCL1_CNTL1__USERVM_DIS_MASK                                                                      0x00010000L
+#define RMI_UTCL1_CNTL1__ENABLE_PUSH_LFIFO_MASK                                                               0x00020000L
+#define RMI_UTCL1_CNTL1__ENABLE_LFIFO_PRI_ARB_MASK                                                            0x00040000L
+#define RMI_UTCL1_CNTL1__REG_INV_VMID_MASK                                                                    0x00780000L
+#define RMI_UTCL1_CNTL1__REG_INV_ALL_VMID_MASK                                                                0x00800000L
+#define RMI_UTCL1_CNTL1__REG_INV_TOGGLE_MASK                                                                  0x01000000L
+#define RMI_UTCL1_CNTL1__CLIENT_INVALIDATE_ALL_VMID_MASK                                                      0x02000000L
+#define RMI_UTCL1_CNTL1__FORCE_MISS_MASK                                                                      0x04000000L
+#define RMI_UTCL1_CNTL1__FORCE_IN_ORDER_MASK                                                                  0x08000000L
+#define RMI_UTCL1_CNTL1__REDUCE_FIFO_DEPTH_BY_2_MASK                                                          0x30000000L
+#define RMI_UTCL1_CNTL1__REDUCE_CACHE_SIZE_BY_2_MASK                                                          0xC0000000L
+//RMI_UTCL1_CNTL2
+#define RMI_UTCL1_CNTL2__UTC_SPARE__SHIFT                                                                     0x0
+#define RMI_UTCL1_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                                0x9
+#define RMI_UTCL1_CNTL2__LINE_VALID__SHIFT                                                                    0xa
+#define RMI_UTCL1_CNTL2__DIS_EDC__SHIFT                                                                       0xb
+#define RMI_UTCL1_CNTL2__GPUVM_INV_MODE__SHIFT                                                                0xc
+#define RMI_UTCL1_CNTL2__SHOOTDOWN_OPT__SHIFT                                                                 0xd
+#define RMI_UTCL1_CNTL2__FORCE_SNOOP__SHIFT                                                                   0xe
+#define RMI_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK__SHIFT                                                           0xf
+#define RMI_UTCL1_CNTL2__UTCL1_ARB_BURST_MODE__SHIFT                                                          0x10
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_RD_WR__SHIFT                                                 0x12
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_RD_WR__SHIFT                                                        0x13
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_VMID__SHIFT                                                  0x14
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_VMID__SHIFT                                                         0x15
+#define RMI_UTCL1_CNTL2__UTCL1_DIS_DUAL_L2_REQ__SHIFT                                                         0x19
+#define RMI_UTCL1_CNTL2__UTCL1_FORCE_FRAG_2M_TO_64K__SHIFT                                                    0x1a
+#define RMI_UTCL1_CNTL2__PERM_MODE_OVRD__SHIFT                                                                0x1b
+#define RMI_UTCL1_CNTL2__LINE_INVALIDATE_OPT__SHIFT                                                           0x1c
+#define RMI_UTCL1_CNTL2__GPUVM_16K_DEFAULT__SHIFT                                                             0x1d
+#define RMI_UTCL1_CNTL2__FGCG_DISABLE__SHIFT                                                                  0x1e
+#define RMI_UTCL1_CNTL2__RESERVED__SHIFT                                                                      0x1f
+#define RMI_UTCL1_CNTL2__UTC_SPARE_MASK                                                                       0x000000FFL
+#define RMI_UTCL1_CNTL2__MTYPE_OVRD_DIS_MASK                                                                  0x00000200L
+#define RMI_UTCL1_CNTL2__LINE_VALID_MASK                                                                      0x00000400L
+#define RMI_UTCL1_CNTL2__DIS_EDC_MASK                                                                         0x00000800L
+#define RMI_UTCL1_CNTL2__GPUVM_INV_MODE_MASK                                                                  0x00001000L
+#define RMI_UTCL1_CNTL2__SHOOTDOWN_OPT_MASK                                                                   0x00002000L
+#define RMI_UTCL1_CNTL2__FORCE_SNOOP_MASK                                                                     0x00004000L
+#define RMI_UTCL1_CNTL2__FORCE_GPUVM_INV_ACK_MASK                                                             0x00008000L
+#define RMI_UTCL1_CNTL2__UTCL1_ARB_BURST_MODE_MASK                                                            0x00030000L
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_RD_WR_MASK                                                   0x00040000L
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_RD_WR_MASK                                                          0x00080000L
+#define RMI_UTCL1_CNTL2__UTCL1_ENABLE_PERF_EVENT_VMID_MASK                                                    0x00100000L
+#define RMI_UTCL1_CNTL2__UTCL1_PERF_EVENT_VMID_MASK                                                           0x01E00000L
+#define RMI_UTCL1_CNTL2__UTCL1_DIS_DUAL_L2_REQ_MASK                                                           0x02000000L
+#define RMI_UTCL1_CNTL2__UTCL1_FORCE_FRAG_2M_TO_64K_MASK                                                      0x04000000L
+#define RMI_UTCL1_CNTL2__PERM_MODE_OVRD_MASK                                                                  0x08000000L
+#define RMI_UTCL1_CNTL2__LINE_INVALIDATE_OPT_MASK                                                             0x10000000L
+#define RMI_UTCL1_CNTL2__GPUVM_16K_DEFAULT_MASK                                                               0x20000000L
+#define RMI_UTCL1_CNTL2__FGCG_DISABLE_MASK                                                                    0x40000000L
+#define RMI_UTCL1_CNTL2__RESERVED_MASK                                                                        0x80000000L
+//RMI_UTC_UNIT_CONFIG
+#define RMI_UTC_UNIT_CONFIG__TMZ_REQ_EN__SHIFT                                                                0x0
+#define RMI_UTC_UNIT_CONFIG__TMZ_REQ_EN_MASK                                                                  0x0000FFFFL
+//RMI_TCIW_FORMATTER0_CNTL
+#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_MAX_ALLOWED_INFLIGHT_REQ__SHIFT                                       0x9
+#define RMI_TCIW_FORMATTER0_CNTL__RMI_IN0_REORDER_DIS__SHIFT                                                  0x1d
+#define RMI_TCIW_FORMATTER0_CNTL__ALL_FAULT_RET0_DATA__SHIFT                                                  0x1f
+#define RMI_TCIW_FORMATTER0_CNTL__TCIW0_MAX_ALLOWED_INFLIGHT_REQ_MASK                                         0x0007FE00L
+#define RMI_TCIW_FORMATTER0_CNTL__RMI_IN0_REORDER_DIS_MASK                                                    0x20000000L
+#define RMI_TCIW_FORMATTER0_CNTL__ALL_FAULT_RET0_DATA_MASK                                                    0x80000000L
+//RMI_TCIW_FORMATTER1_CNTL
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_OVERRIDE__SHIFT                                             0x0
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_TIME_OUT_WINDOW__SHIFT                                          0x1
+#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_MAX_ALLOWED_INFLIGHT_REQ__SHIFT                                       0x9
+#define RMI_TCIW_FORMATTER1_CNTL__RMI_IN1_REORDER_DIS__SHIFT                                                  0x1d
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_AT_LAST_OF_BURST__SHIFT                                     0x1e
+#define RMI_TCIW_FORMATTER1_CNTL__ALL_FAULT_RET1_DATA__SHIFT                                                  0x1f
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_OVERRIDE_MASK                                               0x00000001L
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_TIME_OUT_WINDOW_MASK                                            0x000001FEL
+#define RMI_TCIW_FORMATTER1_CNTL__TCIW1_MAX_ALLOWED_INFLIGHT_REQ_MASK                                         0x0007FE00L
+#define RMI_TCIW_FORMATTER1_CNTL__RMI_IN1_REORDER_DIS_MASK                                                    0x20000000L
+#define RMI_TCIW_FORMATTER1_CNTL__WR_COMBINE1_DIS_AT_LAST_OF_BURST_MASK                                       0x40000000L
+#define RMI_TCIW_FORMATTER1_CNTL__ALL_FAULT_RET1_DATA_MASK                                                    0x80000000L
+//RMI_SCOREBOARD_CNTL
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB0_FLUSH__SHIFT                                                        0x0
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB0__SHIFT                                              0x1
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB1_FLUSH__SHIFT                                                        0x2
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB1__SHIFT                                              0x3
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_EN__SHIFT                                         0x5
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_VALUE__SHIFT                                      0x6
+#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_TIMER_START_VALUE__SHIFT                                   0x9
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB0_FLUSH_MASK                                                          0x00000001L
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB0_MASK                                                0x00000002L
+#define RMI_SCOREBOARD_CNTL__COMPLETE_RB1_FLUSH_MASK                                                          0x00000004L
+#define RMI_SCOREBOARD_CNTL__REQ_IN_RE_EN_AFTER_FLUSH_RB1_MASK                                                0x00000008L
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_EN_MASK                                           0x00000020L
+#define RMI_SCOREBOARD_CNTL__VMID_INVAL_FLUSH_TYPE_OVERRIDE_VALUE_MASK                                        0x00000040L
+#define RMI_SCOREBOARD_CNTL__FORCE_VMID_INVAL_DONE_TIMER_START_VALUE_MASK                                     0x001FFE00L
+//RMI_SCOREBOARD_STATUS0
+#define RMI_SCOREBOARD_STATUS0__CURRENT_SESSION_ID__SHIFT                                                     0x0
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_IN_PROG__SHIFT                                                    0x1
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_REQ_VMID__SHIFT                                                   0x2
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_UTC_DONE__SHIFT                                                   0x12
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_DONE__SHIFT                                                       0x13
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_FLUSH_TYPE__SHIFT                                                 0x14
+#define RMI_SCOREBOARD_STATUS0__FORCE_VMID_INV_DONE__SHIFT                                                    0x15
+#define RMI_SCOREBOARD_STATUS0__COUNTER_SELECT__SHIFT                                                         0x16
+#define RMI_SCOREBOARD_STATUS0__CURRENT_SESSION_ID_MASK                                                       0x00000001L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_IN_PROG_MASK                                                      0x00000002L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_REQ_VMID_MASK                                                     0x0003FFFCL
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_UTC_DONE_MASK                                                     0x00040000L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_DONE_MASK                                                         0x00080000L
+#define RMI_SCOREBOARD_STATUS0__CP_VMID_INV_FLUSH_TYPE_MASK                                                   0x00100000L
+#define RMI_SCOREBOARD_STATUS0__FORCE_VMID_INV_DONE_MASK                                                      0x00200000L
+#define RMI_SCOREBOARD_STATUS0__COUNTER_SELECT_MASK                                                           0x07C00000L
+//RMI_SCOREBOARD_STATUS1
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB0__SHIFT                                                        0x0
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB0__SHIFT                                              0xc
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB0__SHIFT                                               0xd
+#define RMI_SCOREBOARD_STATUS1__MULTI_VMID_INVAL_FROM_CP_DETECTED__SHIFT                                      0xe
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB1__SHIFT                                                        0xf
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB1__SHIFT                                              0x1b
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB1__SHIFT                                               0x1c
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB1__SHIFT                                                  0x1d
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB0__SHIFT                                                  0x1e
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB0_MASK                                                          0x00000FFFL
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB0_MASK                                                0x00001000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB0_MASK                                                 0x00002000L
+#define RMI_SCOREBOARD_STATUS1__MULTI_VMID_INVAL_FROM_CP_DETECTED_MASK                                        0x00004000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_RB1_MASK                                                          0x07FF8000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_UNDERFLOW_RB1_MASK                                                0x08000000L
+#define RMI_SCOREBOARD_STATUS1__RUNNING_CNT_OVERFLOW_RB1_MASK                                                 0x10000000L
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB1_MASK                                                    0x20000000L
+#define RMI_SCOREBOARD_STATUS1__COM_FLUSH_IN_PROG_RB0_MASK                                                    0x40000000L
+//RMI_SCOREBOARD_STATUS2
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB0__SHIFT                                                       0x0
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB0__SHIFT                                             0xc
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB1__SHIFT                                                       0xd
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB1__SHIFT                                             0x19
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB1__SHIFT                                                     0x1a
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB0__SHIFT                                                     0x1b
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB0__SHIFT                                           0x1c
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB1__SHIFT                                           0x1d
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB0__SHIFT                                              0x1e
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB1__SHIFT                                              0x1f
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB0_MASK                                                         0x00000FFFL
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB0_MASK                                               0x00001000L
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_RB1_MASK                                                         0x01FFE000L
+#define RMI_SCOREBOARD_STATUS2__SNAPSHOT_CNT_UNDERFLOW_RB1_MASK                                               0x02000000L
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB1_MASK                                                       0x04000000L
+#define RMI_SCOREBOARD_STATUS2__COM_FLUSH_DONE_RB0_MASK                                                       0x08000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB0_MASK                                             0x10000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_IN_PROG_RB1_MASK                                             0x20000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB0_MASK                                                0x40000000L
+#define RMI_SCOREBOARD_STATUS2__TIME_STAMP_FLUSH_DONE_RB1_MASK                                                0x80000000L
+//RMI_XBAR_ARBITER_CONFIG
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE__SHIFT                                                        0x0
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_WEIGHTEDRR__SHIFT                                     0x2
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL__SHIFT                                                       0x3
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_IDLEIN__SHIFT                                         0x4
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE_OVERRIDE_EN__SHIFT                                            0x5
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_OVERRIDE__SHIFT                                        0x6
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_START_VALUE__SHIFT                                     0x8
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE__SHIFT                                                        0x10
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_WEIGHTEDRR__SHIFT                                     0x12
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL__SHIFT                                                       0x13
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_IDLEIN__SHIFT                                         0x14
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE_OVERRIDE_EN__SHIFT                                            0x15
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_OVERRIDE__SHIFT                                        0x16
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_START_VALUE__SHIFT                                     0x18
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE_MASK                                                          0x00000003L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_WEIGHTEDRR_MASK                                       0x00000004L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_MASK                                                         0x00000008L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_BREAK_LOB_ON_IDLEIN_MASK                                           0x00000010L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_MODE_OVERRIDE_EN_MASK                                              0x00000020L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_OVERRIDE_MASK                                          0x000000C0L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB0_STALL_TIMER_START_VALUE_MASK                                       0x0000FF00L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE_MASK                                                          0x00030000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_WEIGHTEDRR_MASK                                       0x00040000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_MASK                                                         0x00080000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_BREAK_LOB_ON_IDLEIN_MASK                                           0x00100000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_MODE_OVERRIDE_EN_MASK                                              0x00200000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_OVERRIDE_MASK                                          0x00C00000L
+#define RMI_XBAR_ARBITER_CONFIG__XBAR_ARB1_STALL_TIMER_START_VALUE_MASK                                       0xFF000000L
+//RMI_XBAR_ARBITER_CONFIG_1
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_RD__SHIFT                                  0x0
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_WR__SHIFT                                  0x8
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_RD_MASK                                    0x000000FFL
+#define RMI_XBAR_ARBITER_CONFIG_1__XBAR_ARB_ROUND_ROBIN_WEIGHT_RB0_WR_MASK                                    0x0000FF00L
+//RMI_CLOCK_CNTRL
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_BUSY_MASK__SHIFT                                                         0x0
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_BUSY_MASK__SHIFT                                                         0x5
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_WAKEUP_MASK__SHIFT                                                       0xa
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_WAKEUP_MASK__SHIFT                                                       0xf
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_BUSY_MASK_MASK                                                           0x0000001FL
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_BUSY_MASK_MASK                                                           0x000003E0L
+#define RMI_CLOCK_CNTRL__DYN_CLK_RB0_WAKEUP_MASK_MASK                                                         0x00007C00L
+#define RMI_CLOCK_CNTRL__DYN_CLK_CMN_WAKEUP_MASK_MASK                                                         0x000F8000L
+//RMI_UTCL1_STATUS
+#define RMI_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define RMI_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define RMI_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define RMI_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define RMI_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define RMI_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+//RMI_RB_GLX_CID_MAP
+#define RMI_RB_GLX_CID_MAP__CB_COLOR_MAP__SHIFT                                                               0x0
+#define RMI_RB_GLX_CID_MAP__CB_FMASK_MAP__SHIFT                                                               0x4
+#define RMI_RB_GLX_CID_MAP__CB_CMASK_MAP__SHIFT                                                               0x8
+#define RMI_RB_GLX_CID_MAP__CB_DCC_MAP__SHIFT                                                                 0xc
+#define RMI_RB_GLX_CID_MAP__DB_Z_MAP__SHIFT                                                                   0x10
+#define RMI_RB_GLX_CID_MAP__DB_S_MAP__SHIFT                                                                   0x14
+#define RMI_RB_GLX_CID_MAP__DB_TILE_MAP__SHIFT                                                                0x18
+#define RMI_RB_GLX_CID_MAP__DB_ZPCPSD_MAP__SHIFT                                                              0x1c
+#define RMI_RB_GLX_CID_MAP__CB_COLOR_MAP_MASK                                                                 0x0000000FL
+#define RMI_RB_GLX_CID_MAP__CB_FMASK_MAP_MASK                                                                 0x000000F0L
+#define RMI_RB_GLX_CID_MAP__CB_CMASK_MAP_MASK                                                                 0x00000F00L
+#define RMI_RB_GLX_CID_MAP__CB_DCC_MAP_MASK                                                                   0x0000F000L
+#define RMI_RB_GLX_CID_MAP__DB_Z_MAP_MASK                                                                     0x000F0000L
+#define RMI_RB_GLX_CID_MAP__DB_S_MAP_MASK                                                                     0x00F00000L
+#define RMI_RB_GLX_CID_MAP__DB_TILE_MAP_MASK                                                                  0x0F000000L
+#define RMI_RB_GLX_CID_MAP__DB_ZPCPSD_MAP_MASK                                                                0xF0000000L
+//RMI_XNACK_DEBUG
+#define RMI_XNACK_DEBUG__XNACK_PER_VMID__SHIFT                                                                0x0
+#define RMI_XNACK_DEBUG__XNACK_PER_VMID_MASK                                                                  0x0000FFFFL
+//RMI_SPARE
+#define RMI_SPARE__RMI_2_GL1_128B_READ_DISABLE__SHIFT                                                         0x1
+#define RMI_SPARE__RMI_2_GL1_REPEATER_FGCG_DISABLE__SHIFT                                                     0x2
+#define RMI_SPARE__RMI_2_RB_REPEATER_FGCG_DISABLE__SHIFT                                                      0x3
+#define RMI_SPARE__EARLY_WRITE_ACK_ENABLE_C_RW_NOA_RESOLVE_DIS__SHIFT                                         0x4
+#define RMI_SPARE__RMI_REORDER_BYPASS_CHANNEL_DIS__SHIFT                                                      0x5
+#define RMI_SPARE__XNACK_RETURN_DATA_OVERRIDE__SHIFT                                                          0x6
+#define RMI_SPARE__SPARE_BIT_7__SHIFT                                                                         0x7
+#define RMI_SPARE__NOFILL_RMI_CID_CC__SHIFT                                                                   0x8
+#define RMI_SPARE__NOFILL_RMI_CID_FC__SHIFT                                                                   0x9
+#define RMI_SPARE__NOFILL_RMI_CID_CM__SHIFT                                                                   0xa
+#define RMI_SPARE__NOFILL_RMI_CID_DC__SHIFT                                                                   0xb
+#define RMI_SPARE__NOFILL_RMI_CID_Z__SHIFT                                                                    0xc
+#define RMI_SPARE__NOFILL_RMI_CID_S__SHIFT                                                                    0xd
+#define RMI_SPARE__NOFILL_RMI_CID_TILE__SHIFT                                                                 0xe
+#define RMI_SPARE__SPARE_BIT_15_0__SHIFT                                                                      0xf
+#define RMI_SPARE__ARBITER_ADDRESS_MASK__SHIFT                                                                0x10
+#define RMI_SPARE__RMI_2_GL1_128B_READ_DISABLE_MASK                                                           0x00000002L
+#define RMI_SPARE__RMI_2_GL1_REPEATER_FGCG_DISABLE_MASK                                                       0x00000004L
+#define RMI_SPARE__RMI_2_RB_REPEATER_FGCG_DISABLE_MASK                                                        0x00000008L
+#define RMI_SPARE__EARLY_WRITE_ACK_ENABLE_C_RW_NOA_RESOLVE_DIS_MASK                                           0x00000010L
+#define RMI_SPARE__RMI_REORDER_BYPASS_CHANNEL_DIS_MASK                                                        0x00000020L
+#define RMI_SPARE__XNACK_RETURN_DATA_OVERRIDE_MASK                                                            0x00000040L
+#define RMI_SPARE__SPARE_BIT_7_MASK                                                                           0x00000080L
+#define RMI_SPARE__NOFILL_RMI_CID_CC_MASK                                                                     0x00000100L
+#define RMI_SPARE__NOFILL_RMI_CID_FC_MASK                                                                     0x00000200L
+#define RMI_SPARE__NOFILL_RMI_CID_CM_MASK                                                                     0x00000400L
+#define RMI_SPARE__NOFILL_RMI_CID_DC_MASK                                                                     0x00000800L
+#define RMI_SPARE__NOFILL_RMI_CID_Z_MASK                                                                      0x00001000L
+#define RMI_SPARE__NOFILL_RMI_CID_S_MASK                                                                      0x00002000L
+#define RMI_SPARE__NOFILL_RMI_CID_TILE_MASK                                                                   0x00004000L
+#define RMI_SPARE__SPARE_BIT_15_0_MASK                                                                        0x00008000L
+#define RMI_SPARE__ARBITER_ADDRESS_MASK_MASK                                                                  0xFFFF0000L
+//RMI_SPARE_1
+#define RMI_SPARE_1__EARLY_WRACK_FIFO_DISABLE__SHIFT                                                          0x0
+#define RMI_SPARE_1__SPARE_BIT_9__SHIFT                                                                       0x1
+#define RMI_SPARE_1__SPARE_BIT_10__SHIFT                                                                      0x2
+#define RMI_SPARE_1__SPARE_BIT_11__SHIFT                                                                      0x3
+#define RMI_SPARE_1__SPARE_BIT_12__SHIFT                                                                      0x4
+#define RMI_SPARE_1__SPARE_BIT_13__SHIFT                                                                      0x5
+#define RMI_SPARE_1__SPARE_BIT_14__SHIFT                                                                      0x6
+#define RMI_SPARE_1__SPARE_BIT_15__SHIFT                                                                      0x7
+#define RMI_SPARE_1__RMI_REORDER_DIS_BY_CID__SHIFT                                                            0x8
+#define RMI_SPARE_1__SPARE_BIT_16_1__SHIFT                                                                    0x10
+#define RMI_SPARE_1__EARLY_WRACK_FIFO_DISABLE_MASK                                                            0x00000001L
+#define RMI_SPARE_1__SPARE_BIT_9_MASK                                                                         0x00000002L
+#define RMI_SPARE_1__SPARE_BIT_10_MASK                                                                        0x00000004L
+#define RMI_SPARE_1__SPARE_BIT_11_MASK                                                                        0x00000008L
+#define RMI_SPARE_1__SPARE_BIT_12_MASK                                                                        0x00000010L
+#define RMI_SPARE_1__SPARE_BIT_13_MASK                                                                        0x00000020L
+#define RMI_SPARE_1__SPARE_BIT_14_MASK                                                                        0x00000040L
+#define RMI_SPARE_1__SPARE_BIT_15_MASK                                                                        0x00000080L
+#define RMI_SPARE_1__RMI_REORDER_DIS_BY_CID_MASK                                                              0x0000FF00L
+#define RMI_SPARE_1__SPARE_BIT_16_1_MASK                                                                      0xFFFF0000L
+//RMI_SPARE_2
+#define RMI_SPARE_2__ERROR_ZERO_BYTE_MASK_CID__SHIFT                                                          0x0
+#define RMI_SPARE_2__SPARE_BIT_8_2__SHIFT                                                                     0x10
+#define RMI_SPARE_2__SPARE_BIT_8_3__SHIFT                                                                     0x18
+#define RMI_SPARE_2__ERROR_ZERO_BYTE_MASK_CID_MASK                                                            0x0000FFFFL
+#define RMI_SPARE_2__SPARE_BIT_8_2_MASK                                                                       0x00FF0000L
+#define RMI_SPARE_2__SPARE_BIT_8_3_MASK                                                                       0xFF000000L
+//CC_RMI_REDUNDANCY
+#define CC_RMI_REDUNDANCY__WRITE_DIS__SHIFT                                                                   0x0
+#define CC_RMI_REDUNDANCY__REPAIR_EN_IN_0__SHIFT                                                              0x1
+#define CC_RMI_REDUNDANCY__REPAIR_EN_IN_1__SHIFT                                                              0x2
+#define CC_RMI_REDUNDANCY__REPAIR_RMI_OVERRIDE__SHIFT                                                         0x3
+#define CC_RMI_REDUNDANCY__REPAIR_ID_SWAP__SHIFT                                                              0x4
+#define CC_RMI_REDUNDANCY__WRITE_DIS_MASK                                                                     0x00000001L
+#define CC_RMI_REDUNDANCY__REPAIR_EN_IN_0_MASK                                                                0x00000002L
+#define CC_RMI_REDUNDANCY__REPAIR_EN_IN_1_MASK                                                                0x00000004L
+#define CC_RMI_REDUNDANCY__REPAIR_RMI_OVERRIDE_MASK                                                           0x00000008L
+#define CC_RMI_REDUNDANCY__REPAIR_ID_SWAP_MASK                                                                0x00000010L
+
+
+// addressBlock: gc_pmmdec
+//GCR_PIO_CNTL
+#define GCR_PIO_CNTL__GCR_DATA_INDEX__SHIFT                                                                   0x0
+#define GCR_PIO_CNTL__GCR_REG_DONE__SHIFT                                                                     0x2
+#define GCR_PIO_CNTL__GCR_REG_RESET__SHIFT                                                                    0x3
+#define GCR_PIO_CNTL__GCR_PIO_RSP_TAG__SHIFT                                                                  0x10
+#define GCR_PIO_CNTL__GCR_PIO_RSP_DONE__SHIFT                                                                 0x1e
+#define GCR_PIO_CNTL__GCR_READY__SHIFT                                                                        0x1f
+#define GCR_PIO_CNTL__GCR_DATA_INDEX_MASK                                                                     0x00000003L
+#define GCR_PIO_CNTL__GCR_REG_DONE_MASK                                                                       0x00000004L
+#define GCR_PIO_CNTL__GCR_REG_RESET_MASK                                                                      0x00000008L
+#define GCR_PIO_CNTL__GCR_PIO_RSP_TAG_MASK                                                                    0x00FF0000L
+#define GCR_PIO_CNTL__GCR_PIO_RSP_DONE_MASK                                                                   0x40000000L
+#define GCR_PIO_CNTL__GCR_READY_MASK                                                                          0x80000000L
+//GCR_PIO_DATA
+#define GCR_PIO_DATA__GCR_DATA__SHIFT                                                                         0x0
+#define GCR_PIO_DATA__GCR_DATA_MASK                                                                           0xFFFFFFFFL
+//PMM_CNTL
+#define PMM_CNTL__PMM_DISABLE__SHIFT                                                                          0x0
+#define PMM_CNTL__ABIT_FORCE_FLUSH__SHIFT                                                                     0x1
+#define PMM_CNTL__ABIT_TIMER_THRESHOLD__SHIFT                                                                 0x2
+#define PMM_CNTL__ABIT_TIMER_DISABLE__SHIFT                                                                   0x6
+#define PMM_CNTL__ABIT_TIMER_RESET__SHIFT                                                                     0x7
+#define PMM_CNTL__INTERRUPT_PRIORITY__SHIFT                                                                   0x8
+#define PMM_CNTL__PMM_INTERRUPTS_DISABLE__SHIFT                                                               0xa
+#define PMM_CNTL__RESERVED__SHIFT                                                                             0xb
+#define PMM_CNTL__PMM_DISABLE_MASK                                                                            0x00000001L
+#define PMM_CNTL__ABIT_FORCE_FLUSH_MASK                                                                       0x00000002L
+#define PMM_CNTL__ABIT_TIMER_THRESHOLD_MASK                                                                   0x0000003CL
+#define PMM_CNTL__ABIT_TIMER_DISABLE_MASK                                                                     0x00000040L
+#define PMM_CNTL__ABIT_TIMER_RESET_MASK                                                                       0x00000080L
+#define PMM_CNTL__INTERRUPT_PRIORITY_MASK                                                                     0x00000300L
+#define PMM_CNTL__PMM_INTERRUPTS_DISABLE_MASK                                                                 0x00000400L
+#define PMM_CNTL__RESERVED_MASK                                                                               0xFFFFF800L
+//PMM_STATUS
+#define PMM_STATUS__PMM_IDLE__SHIFT                                                                           0x0
+#define PMM_STATUS__ABIT_FORCE_FLUSH_IN_PROGRESS__SHIFT                                                       0x1
+#define PMM_STATUS__ABIT_FORCE_FLUSH_DONE__SHIFT                                                              0x2
+#define PMM_STATUS__ABIT_TIMER_FLUSH_IN_PROGRESS__SHIFT                                                       0x3
+#define PMM_STATUS__ABIT_TIMER_FLUSH_DONE__SHIFT                                                              0x4
+#define PMM_STATUS__ABIT_TIMER_RUNNING__SHIFT                                                                 0x5
+#define PMM_STATUS__PMM_INTERRUPTS_PENDING__SHIFT                                                             0x6
+#define PMM_STATUS__ABIT_FLUSH_ERROR__SHIFT                                                                   0x7
+#define PMM_STATUS__ABIT_TIMER_RESET_CDC_IN_PROGRESS__SHIFT                                                   0x8
+#define PMM_STATUS__ABIT_TIMER_ENABLE_CDC_IN_PROGRESS__SHIFT                                                  0x9
+#define PMM_STATUS__ABIT_TIMER_THRESHOLD_CDC_IN_PROGRESS__SHIFT                                               0xa
+#define PMM_STATUS__RESERVED__SHIFT                                                                           0xb
+#define PMM_STATUS__PMM_IDLE_MASK                                                                             0x00000001L
+#define PMM_STATUS__ABIT_FORCE_FLUSH_IN_PROGRESS_MASK                                                         0x00000002L
+#define PMM_STATUS__ABIT_FORCE_FLUSH_DONE_MASK                                                                0x00000004L
+#define PMM_STATUS__ABIT_TIMER_FLUSH_IN_PROGRESS_MASK                                                         0x00000008L
+#define PMM_STATUS__ABIT_TIMER_FLUSH_DONE_MASK                                                                0x00000010L
+#define PMM_STATUS__ABIT_TIMER_RUNNING_MASK                                                                   0x00000020L
+#define PMM_STATUS__PMM_INTERRUPTS_PENDING_MASK                                                               0x00000040L
+#define PMM_STATUS__ABIT_FLUSH_ERROR_MASK                                                                     0x00000080L
+#define PMM_STATUS__ABIT_TIMER_RESET_CDC_IN_PROGRESS_MASK                                                     0x00000100L
+#define PMM_STATUS__ABIT_TIMER_ENABLE_CDC_IN_PROGRESS_MASK                                                    0x00000200L
+#define PMM_STATUS__ABIT_TIMER_THRESHOLD_CDC_IN_PROGRESS_MASK                                                 0x00000400L
+#define PMM_STATUS__RESERVED_MASK                                                                             0xFFFFF800L
+
+
+// addressBlock: gc_utcl1dec
+//UTCL1_CTRL_1
+#define UTCL1_CTRL_1__UTCL1_CACHE_CORE_BYPASS__SHIFT                                                          0x0
+#define UTCL1_CTRL_1__UTCL1_TCP_BYPASS__SHIFT                                                                 0x1
+#define UTCL1_CTRL_1__UTCL1_SQCI_BYPASS__SHIFT                                                                0x2
+#define UTCL1_CTRL_1__UTCL1_SQCD_BYPASS__SHIFT                                                                0x3
+#define UTCL1_CTRL_1__UTCL1_RMI_BYPASS__SHIFT                                                                 0x4
+#define UTCL1_CTRL_1__UTCL1_SQG_BYPASS__SHIFT                                                                 0x5
+#define UTCL1_CTRL_1__UTCL1_FORCE_RANGE_INV_TO_VMID__SHIFT                                                    0x6
+#define UTCL1_CTRL_1__UTCL1_FORCE_INV_ALL__SHIFT                                                              0x7
+#define UTCL1_CTRL_1__UTCL1_FORCE_INV_ALL_DONE__SHIFT                                                         0x8
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_1__SHIFT                                                                0x9
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_2__SHIFT                                                                0xb
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_3__SHIFT                                                                0xd
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_4__SHIFT                                                                0xf
+#define UTCL1_CTRL_1__RESERVED__SHIFT                                                                         0x11
+#define UTCL1_CTRL_1__UTCL1_CACHE_CORE_BYPASS_MASK                                                            0x00000001L
+#define UTCL1_CTRL_1__UTCL1_TCP_BYPASS_MASK                                                                   0x00000002L
+#define UTCL1_CTRL_1__UTCL1_SQCI_BYPASS_MASK                                                                  0x00000004L
+#define UTCL1_CTRL_1__UTCL1_SQCD_BYPASS_MASK                                                                  0x00000008L
+#define UTCL1_CTRL_1__UTCL1_RMI_BYPASS_MASK                                                                   0x00000010L
+#define UTCL1_CTRL_1__UTCL1_SQG_BYPASS_MASK                                                                   0x00000020L
+#define UTCL1_CTRL_1__UTCL1_FORCE_RANGE_INV_TO_VMID_MASK                                                      0x00000040L
+#define UTCL1_CTRL_1__UTCL1_FORCE_INV_ALL_MASK                                                                0x00000080L
+#define UTCL1_CTRL_1__UTCL1_FORCE_INV_ALL_DONE_MASK                                                           0x00000100L
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_1_MASK                                                                  0x00000600L
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_2_MASK                                                                  0x00001800L
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_3_MASK                                                                  0x00006000L
+#define UTCL1_CTRL_1__UTCL1_PAGE_SIZE_4_MASK                                                                  0x00018000L
+#define UTCL1_CTRL_1__RESERVED_MASK                                                                           0xFFFE0000L
+//UTCL1_ALOG
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER1_THRESHOLD__SHIFT                                                 0x0
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER2_BYPASS__SHIFT                                                    0x3
+#define UTCL1_ALOG__UTCL1_ALOG_ACTIVE__SHIFT                                                                  0x4
+#define UTCL1_ALOG__UTCL1_ALOG_MODE__SHIFT                                                                    0x5
+#define UTCL1_ALOG__UTCL1_ALOG_MODE2_LOCK_WINDOW__SHIFT                                                       0x6
+#define UTCL1_ALOG__UTCL1_ALOG_ONLY_MISS__SHIFT                                                               0x9
+#define UTCL1_ALOG__UTCL1_ALOG_MODE2_INTR_THRESHOLD__SHIFT                                                    0xa
+#define UTCL1_ALOG__UTCL1_ALOG_SPACE_EN__SHIFT                                                                0xc
+#define UTCL1_ALOG__UTCL1_ALOG_CLEAN__SHIFT                                                                   0xf
+#define UTCL1_ALOG__UTCL1_ALOG_IDLE__SHIFT                                                                    0x10
+#define UTCL1_ALOG__UTCL1_ALOG_TRACK_SEGMENT_SIZE__SHIFT                                                      0x11
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER1_BYPASS__SHIFT                                                    0x17
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_INTR_ON_ALLOC__SHIFT                                                     0x18
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER1_THRESHOLD_MASK                                                   0x00000007L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER2_BYPASS_MASK                                                      0x00000008L
+#define UTCL1_ALOG__UTCL1_ALOG_ACTIVE_MASK                                                                    0x00000010L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE_MASK                                                                      0x00000020L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE2_LOCK_WINDOW_MASK                                                         0x000001C0L
+#define UTCL1_ALOG__UTCL1_ALOG_ONLY_MISS_MASK                                                                 0x00000200L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE2_INTR_THRESHOLD_MASK                                                      0x00000C00L
+#define UTCL1_ALOG__UTCL1_ALOG_SPACE_EN_MASK                                                                  0x00007000L
+#define UTCL1_ALOG__UTCL1_ALOG_CLEAN_MASK                                                                     0x00008000L
+#define UTCL1_ALOG__UTCL1_ALOG_IDLE_MASK                                                                      0x00010000L
+#define UTCL1_ALOG__UTCL1_ALOG_TRACK_SEGMENT_SIZE_MASK                                                        0x007E0000L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_FILTER1_BYPASS_MASK                                                      0x00800000L
+#define UTCL1_ALOG__UTCL1_ALOG_MODE1_INTR_ON_ALLOC_MASK                                                       0x01000000L
+//UTCL1_STATUS
+#define UTCL1_STATUS__UTCL1_HIT_PATH_BUSY__SHIFT                                                              0x0
+#define UTCL1_STATUS__UTCL1_MH_BUSY__SHIFT                                                                    0x1
+#define UTCL1_STATUS__UTCL1_INV_BUSY__SHIFT                                                                   0x2
+#define UTCL1_STATUS__UTCL1_PENDING_UTCL2_REQ__SHIFT                                                          0x3
+#define UTCL1_STATUS__UTCL1_PENDING_UTCL2_RET__SHIFT                                                          0x4
+#define UTCL1_STATUS__UTCL1_LAST_UTCL2_RET_XNACK__SHIFT                                                       0x5
+#define UTCL1_STATUS__UTCL1_RANGE_INV_IN_PROGRESS__SHIFT                                                      0x7
+#define UTCL1_STATUS__RESERVED__SHIFT                                                                         0x8
+#define UTCL1_STATUS__UTCL1_HIT_PATH_BUSY_MASK                                                                0x00000001L
+#define UTCL1_STATUS__UTCL1_MH_BUSY_MASK                                                                      0x00000002L
+#define UTCL1_STATUS__UTCL1_INV_BUSY_MASK                                                                     0x00000004L
+#define UTCL1_STATUS__UTCL1_PENDING_UTCL2_REQ_MASK                                                            0x00000008L
+#define UTCL1_STATUS__UTCL1_PENDING_UTCL2_RET_MASK                                                            0x00000010L
+#define UTCL1_STATUS__UTCL1_LAST_UTCL2_RET_XNACK_MASK                                                         0x00000060L
+#define UTCL1_STATUS__UTCL1_RANGE_INV_IN_PROGRESS_MASK                                                        0x00000080L
+#define UTCL1_STATUS__RESERVED_MASK                                                                           0x00000100L
+
+
+// addressBlock: gc_gcvmsharedpfdec
+//GCMC_VM_NB_MMIOBASE
+#define GCMC_VM_NB_MMIOBASE__MMIOBASE__SHIFT                                                                  0x0
+#define GCMC_VM_NB_MMIOBASE__MMIOBASE_MASK                                                                    0xFFFFFFFFL
+//GCMC_VM_NB_MMIOLIMIT
+#define GCMC_VM_NB_MMIOLIMIT__MMIOLIMIT__SHIFT                                                                0x0
+#define GCMC_VM_NB_MMIOLIMIT__MMIOLIMIT_MASK                                                                  0xFFFFFFFFL
+//GCMC_VM_NB_PCI_CTRL
+#define GCMC_VM_NB_PCI_CTRL__MMIOENABLE__SHIFT                                                                0x17
+#define GCMC_VM_NB_PCI_CTRL__MMIOENABLE_MASK                                                                  0x00800000L
+//GCMC_VM_NB_PCI_ARB
+#define GCMC_VM_NB_PCI_ARB__VGA_HOLE__SHIFT                                                                   0x3
+#define GCMC_VM_NB_PCI_ARB__VGA_HOLE_MASK                                                                     0x00000008L
+//GCMC_VM_NB_TOP_OF_DRAM_SLOT1
+#define GCMC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM__SHIFT                                                      0x17
+#define GCMC_VM_NB_TOP_OF_DRAM_SLOT1__TOP_OF_DRAM_MASK                                                        0xFF800000L
+//GCMC_VM_NB_LOWER_TOP_OF_DRAM2
+#define GCMC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE__SHIFT                                                          0x0
+#define GCMC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2__SHIFT                                                      0x17
+#define GCMC_VM_NB_LOWER_TOP_OF_DRAM2__ENABLE_MASK                                                            0x00000001L
+#define GCMC_VM_NB_LOWER_TOP_OF_DRAM2__LOWER_TOM2_MASK                                                        0xFF800000L
+//GCMC_VM_NB_UPPER_TOP_OF_DRAM2
+#define GCMC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2__SHIFT                                                      0x0
+#define GCMC_VM_NB_UPPER_TOP_OF_DRAM2__UPPER_TOM2_MASK                                                        0x00000FFFL
+//GCMC_VM_FB_OFFSET
+#define GCMC_VM_FB_OFFSET__FB_OFFSET__SHIFT                                                                   0x0
+#define GCMC_VM_FB_OFFSET__FB_OFFSET_MASK                                                                     0x00FFFFFFL
+//GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB
+#define GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB__SHIFT                             0x0
+#define GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB__PHYSICAL_PAGE_NUMBER_LSB_MASK                               0xFFFFFFFFL
+//GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB
+#define GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB__SHIFT                             0x0
+#define GCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB__PHYSICAL_PAGE_NUMBER_MSB_MASK                               0x0000000FL
+//GCMC_VM_STEERING
+#define GCMC_VM_STEERING__DEFAULT_STEERING__SHIFT                                                             0x0
+#define GCMC_VM_STEERING__DEFAULT_STEERING_MASK                                                               0x00000003L
+//GCMC_SHARED_VIRT_RESET_REQ
+#define GCMC_SHARED_VIRT_RESET_REQ__VF__SHIFT                                                                 0x0
+#define GCMC_SHARED_VIRT_RESET_REQ__PF__SHIFT                                                                 0x1f
+#define GCMC_SHARED_VIRT_RESET_REQ__VF_MASK                                                                   0x0000FFFFL
+#define GCMC_SHARED_VIRT_RESET_REQ__PF_MASK                                                                   0x80000000L
+//GCMC_MEM_POWER_LS
+#define GCMC_MEM_POWER_LS__LS_SETUP__SHIFT                                                                    0x0
+#define GCMC_MEM_POWER_LS__LS_HOLD__SHIFT                                                                     0x6
+#define GCMC_MEM_POWER_LS__LS_SETUP_MASK                                                                      0x0000003FL
+#define GCMC_MEM_POWER_LS__LS_HOLD_MASK                                                                       0x00000FC0L
+//GCMC_VM_CACHEABLE_DRAM_ADDRESS_START
+#define GCMC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS__SHIFT                                                  0x0
+#define GCMC_VM_CACHEABLE_DRAM_ADDRESS_START__ADDRESS_MASK                                                    0x000FFFFFL
+//GCMC_VM_CACHEABLE_DRAM_ADDRESS_END
+#define GCMC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS__SHIFT                                                    0x0
+#define GCMC_VM_CACHEABLE_DRAM_ADDRESS_END__ADDRESS_MASK                                                      0x000FFFFFL
+//GCMC_VM_LOCAL_SYSMEM_ADDRESS_START
+#define GCMC_VM_LOCAL_SYSMEM_ADDRESS_START__ADDRESS__SHIFT                                                    0x0
+#define GCMC_VM_LOCAL_SYSMEM_ADDRESS_START__ADDRESS_MASK                                                      0x000FFFFFL
+//GCMC_VM_LOCAL_SYSMEM_ADDRESS_END
+#define GCMC_VM_LOCAL_SYSMEM_ADDRESS_END__ADDRESS__SHIFT                                                      0x0
+#define GCMC_VM_LOCAL_SYSMEM_ADDRESS_END__ADDRESS_MASK                                                        0x000FFFFFL
+//GCMC_VM_APT_CNTL
+#define GCMC_VM_APT_CNTL__FORCE_MTYPE_UC__SHIFT                                                               0x0
+#define GCMC_VM_APT_CNTL__DIRECT_SYSTEM_EN__SHIFT                                                             0x1
+#define GCMC_VM_APT_CNTL__FRAG_APT_INTXN_MODE__SHIFT                                                          0x2
+#define GCMC_VM_APT_CNTL__CHECK_IS_LOCAL__SHIFT                                                               0x4
+#define GCMC_VM_APT_CNTL__CAP_FRAG_SIZE_2M__SHIFT                                                             0x5
+#define GCMC_VM_APT_CNTL__LOCAL_SYSMEM_APERTURE_CNTL__SHIFT                                                   0x6
+#define GCMC_VM_APT_CNTL__FORCE_MTYPE_UC_MASK                                                                 0x00000001L
+#define GCMC_VM_APT_CNTL__DIRECT_SYSTEM_EN_MASK                                                               0x00000002L
+#define GCMC_VM_APT_CNTL__FRAG_APT_INTXN_MODE_MASK                                                            0x0000000CL
+#define GCMC_VM_APT_CNTL__CHECK_IS_LOCAL_MASK                                                                 0x00000010L
+#define GCMC_VM_APT_CNTL__CAP_FRAG_SIZE_2M_MASK                                                               0x00000020L
+#define GCMC_VM_APT_CNTL__LOCAL_SYSMEM_APERTURE_CNTL_MASK                                                     0x000000C0L
+//GCMC_VM_LOCAL_FB_ADDRESS_START
+#define GCMC_VM_LOCAL_FB_ADDRESS_START__ADDRESS__SHIFT                                                        0x0
+#define GCMC_VM_LOCAL_FB_ADDRESS_START__ADDRESS_MASK                                                          0x000FFFFFL
+//GCMC_VM_LOCAL_FB_ADDRESS_END
+#define GCMC_VM_LOCAL_FB_ADDRESS_END__ADDRESS__SHIFT                                                          0x0
+#define GCMC_VM_LOCAL_FB_ADDRESS_END__ADDRESS_MASK                                                            0x000FFFFFL
+//GCMC_VM_LOCAL_FB_ADDRESS_LOCK_CNTL
+#define GCMC_VM_LOCAL_FB_ADDRESS_LOCK_CNTL__LOCK__SHIFT                                                       0x0
+#define GCMC_VM_LOCAL_FB_ADDRESS_LOCK_CNTL__LOCK_MASK                                                         0x00000001L
+//GCUTCL2_ICG_CTRL
+#define GCUTCL2_ICG_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x0
+#define GCUTCL2_ICG_CTRL__DYNAMIC_CLOCK_OVERRIDE__SHIFT                                                       0x4
+#define GCUTCL2_ICG_CTRL__STATIC_CLOCK_OVERRIDE__SHIFT                                                        0x5
+#define GCUTCL2_ICG_CTRL__AON_CLOCK_OVERRIDE__SHIFT                                                           0x6
+#define GCUTCL2_ICG_CTRL__PERFMON_CLOCK_OVERRIDE__SHIFT                                                       0x7
+#define GCUTCL2_ICG_CTRL__OFF_HYSTERESIS_MASK                                                                 0x0000000FL
+#define GCUTCL2_ICG_CTRL__DYNAMIC_CLOCK_OVERRIDE_MASK                                                         0x00000010L
+#define GCUTCL2_ICG_CTRL__STATIC_CLOCK_OVERRIDE_MASK                                                          0x00000020L
+#define GCUTCL2_ICG_CTRL__AON_CLOCK_OVERRIDE_MASK                                                             0x00000040L
+#define GCUTCL2_ICG_CTRL__PERFMON_CLOCK_OVERRIDE_MASK                                                         0x00000080L
+//GCMC_SHARED_ACTIVE_FCN_ID
+#define GCMC_SHARED_ACTIVE_FCN_ID__VFID__SHIFT                                                                0x0
+#define GCMC_SHARED_ACTIVE_FCN_ID__VF__SHIFT                                                                  0x1e
+#define GCMC_SHARED_ACTIVE_FCN_ID__VFID_MASK                                                                  0x0000000FL
+#define GCMC_SHARED_ACTIVE_FCN_ID__VF_MASK                                                                    0x40000000L
+//GCUTCL2_CGTT_BUSY_CTRL
+#define GCUTCL2_CGTT_BUSY_CTRL__READ_DELAY__SHIFT                                                             0x0
+#define GCUTCL2_CGTT_BUSY_CTRL__ALWAYS_BUSY__SHIFT                                                            0x5
+#define GCUTCL2_CGTT_BUSY_CTRL__READ_DELAY_MASK                                                               0x0000001FL
+#define GCUTCL2_CGTT_BUSY_CTRL__ALWAYS_BUSY_MASK                                                              0x00000020L
+//GCMC_VM_FB_NOALLOC_CNTL
+#define GCMC_VM_FB_NOALLOC_CNTL__LOCAL_FB_NOALLOC_NOPTE__SHIFT                                                0x0
+#define GCMC_VM_FB_NOALLOC_CNTL__REMOTE_FB_NOALLOC_NOPTE__SHIFT                                               0x1
+#define GCMC_VM_FB_NOALLOC_CNTL__FB_NOALLOC_WALKER_FETCH__SHIFT                                               0x2
+#define GCMC_VM_FB_NOALLOC_CNTL__ROUTER_ATCL2_NOALLOC__SHIFT                                                  0x3
+#define GCMC_VM_FB_NOALLOC_CNTL__ROUTER_GPA_MODE2_NOALLOC__SHIFT                                              0x4
+#define GCMC_VM_FB_NOALLOC_CNTL__ROUTER_GPA_MODE3_NOALLOC__SHIFT                                              0x5
+#define GCMC_VM_FB_NOALLOC_CNTL__LOCAL_FB_NOALLOC_NOPTE_MASK                                                  0x00000001L
+#define GCMC_VM_FB_NOALLOC_CNTL__REMOTE_FB_NOALLOC_NOPTE_MASK                                                 0x00000002L
+#define GCMC_VM_FB_NOALLOC_CNTL__FB_NOALLOC_WALKER_FETCH_MASK                                                 0x00000004L
+#define GCMC_VM_FB_NOALLOC_CNTL__ROUTER_ATCL2_NOALLOC_MASK                                                    0x00000008L
+#define GCMC_VM_FB_NOALLOC_CNTL__ROUTER_GPA_MODE2_NOALLOC_MASK                                                0x00000010L
+#define GCMC_VM_FB_NOALLOC_CNTL__ROUTER_GPA_MODE3_NOALLOC_MASK                                                0x00000020L
+//GCUTCL2_HARVEST_BYPASS_GROUPS
+#define GCUTCL2_HARVEST_BYPASS_GROUPS__BYPASS_GROUPS__SHIFT                                                   0x0
+#define GCUTCL2_HARVEST_BYPASS_GROUPS__BYPASS_GROUPS_MASK                                                     0xFFFFFFFFL
+//GCUTCL2_GROUP_RET_FAULT_STATUS
+#define GCUTCL2_GROUP_RET_FAULT_STATUS__FAULT_GROUPS__SHIFT                                                   0x0
+#define GCUTCL2_GROUP_RET_FAULT_STATUS__FAULT_GROUPS_MASK                                                     0xFFFFFFFFL
+
+
+// addressBlock: gc_gcvml2pfdec
+//GCVM_L2_CNTL
+#define GCVM_L2_CNTL__ENABLE_L2_CACHE__SHIFT                                                                  0x0
+#define GCVM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING__SHIFT                                                    0x1
+#define GCVM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE__SHIFT                                                    0x2
+#define GCVM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE__SHIFT                                                    0x4
+#define GCVM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE__SHIFT                                                0x8
+#define GCVM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                          0x9
+#define GCVM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                         0xa
+#define GCVM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY__SHIFT                                         0xb
+#define GCVM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE__SHIFT                                                         0xc
+#define GCVM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT                                                          0xf
+#define GCVM_L2_CNTL__PDE_FAULT_CLASSIFICATION__SHIFT                                                         0x12
+#define GCVM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT                                                    0x13
+#define GCVM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE__SHIFT                                                      0x15
+#define GCVM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE__SHIFT                                                           0x1a
+#define GCVM_L2_CNTL__ENABLE_L2_CACHE_MASK                                                                    0x00000001L
+#define GCVM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK                                                      0x00000002L
+#define GCVM_L2_CNTL__L2_CACHE_PTE_ENDIAN_SWAP_MODE_MASK                                                      0x0000000CL
+#define GCVM_L2_CNTL__L2_CACHE_PDE_ENDIAN_SWAP_MODE_MASK                                                      0x00000030L
+#define GCVM_L2_CNTL__L2_PDE0_CACHE_TAG_GENERATION_MODE_MASK                                                  0x00000100L
+#define GCVM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK                                            0x00000200L
+#define GCVM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK                                           0x00000400L
+#define GCVM_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY_MASK                                           0x00000800L
+#define GCVM_L2_CNTL__L2_PDE0_CACHE_SPLIT_MODE_MASK                                                           0x00007000L
+#define GCVM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE_MASK                                                            0x00038000L
+#define GCVM_L2_CNTL__PDE_FAULT_CLASSIFICATION_MASK                                                           0x00040000L
+#define GCVM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE_MASK                                                      0x00180000L
+#define GCVM_L2_CNTL__IDENTITY_MODE_FRAGMENT_SIZE_MASK                                                        0x03E00000L
+#define GCVM_L2_CNTL__L2_PTE_CACHE_ADDR_MODE_MASK                                                             0x0C000000L
+//GCVM_L2_CNTL2
+#define GCVM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS__SHIFT                                                          0x0
+#define GCVM_L2_CNTL2__INVALIDATE_L2_CACHE__SHIFT                                                             0x1
+#define GCVM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN__SHIFT                                                   0x15
+#define GCVM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION__SHIFT                                                 0x16
+#define GCVM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE__SHIFT                                                          0x17
+#define GCVM_L2_CNTL2__INVALIDATE_CACHE_MODE__SHIFT                                                           0x1a
+#define GCVM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE__SHIFT                                                        0x1c
+#define GCVM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK                                                            0x00000001L
+#define GCVM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK                                                               0x00000002L
+#define GCVM_L2_CNTL2__DISABLE_INVALIDATE_PER_DOMAIN_MASK                                                     0x00200000L
+#define GCVM_L2_CNTL2__DISABLE_BIGK_CACHE_OPTIMIZATION_MASK                                                   0x00400000L
+#define GCVM_L2_CNTL2__L2_PTE_CACHE_VMID_MODE_MASK                                                            0x03800000L
+#define GCVM_L2_CNTL2__INVALIDATE_CACHE_MODE_MASK                                                             0x0C000000L
+#define GCVM_L2_CNTL2__PDE_CACHE_EFFECTIVE_SIZE_MASK                                                          0x70000000L
+//GCVM_L2_CNTL3
+#define GCVM_L2_CNTL3__BANK_SELECT__SHIFT                                                                     0x0
+#define GCVM_L2_CNTL3__L2_CACHE_UPDATE_MODE__SHIFT                                                            0x6
+#define GCVM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                        0x8
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                                                     0xf
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY__SHIFT                                                     0x14
+#define GCVM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE__SHIFT                                                      0x15
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE__SHIFT                                                    0x18
+#define GCVM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS__SHIFT                                                          0x1c
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS__SHIFT                                                        0x1d
+#define GCVM_L2_CNTL3__PDE_CACHE_FORCE_MISS__SHIFT                                                            0x1e
+#define GCVM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY__SHIFT                                                       0x1f
+#define GCVM_L2_CNTL3__BANK_SELECT_MASK                                                                       0x0000003FL
+#define GCVM_L2_CNTL3__L2_CACHE_UPDATE_MODE_MASK                                                              0x000000C0L
+#define GCVM_L2_CNTL3__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                          0x00001F00L
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                                                       0x000F8000L
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK                                                       0x00100000L
+#define GCVM_L2_CNTL3__L2_CACHE_4K_EFFECTIVE_SIZE_MASK                                                        0x00E00000L
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_EFFECTIVE_SIZE_MASK                                                      0x0F000000L
+#define GCVM_L2_CNTL3__L2_CACHE_4K_FORCE_MISS_MASK                                                            0x10000000L
+#define GCVM_L2_CNTL3__L2_CACHE_BIGK_FORCE_MISS_MASK                                                          0x20000000L
+#define GCVM_L2_CNTL3__PDE_CACHE_FORCE_MISS_MASK                                                              0x40000000L
+#define GCVM_L2_CNTL3__L2_CACHE_4K_ASSOCIATIVITY_MASK                                                         0x80000000L
+//GCVM_L2_STATUS
+#define GCVM_L2_STATUS__L2_BUSY__SHIFT                                                                        0x0
+#define GCVM_L2_STATUS__CONTEXT_DOMAIN_BUSY__SHIFT                                                            0x1
+#define GCVM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS__SHIFT                                               0x11
+#define GCVM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS__SHIFT                                             0x12
+#define GCVM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS__SHIFT                                                 0x13
+#define GCVM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS__SHIFT                                                 0x14
+#define GCVM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS__SHIFT                                                 0x15
+#define GCVM_L2_STATUS__L2_BUSY_MASK                                                                          0x00000001L
+#define GCVM_L2_STATUS__CONTEXT_DOMAIN_BUSY_MASK                                                              0x0001FFFEL
+#define GCVM_L2_STATUS__FOUND_4K_PTE_CACHE_PARITY_ERRORS_MASK                                                 0x00020000L
+#define GCVM_L2_STATUS__FOUND_BIGK_PTE_CACHE_PARITY_ERRORS_MASK                                               0x00040000L
+#define GCVM_L2_STATUS__FOUND_PDE0_CACHE_PARITY_ERRORS_MASK                                                   0x00080000L
+#define GCVM_L2_STATUS__FOUND_PDE1_CACHE_PARITY_ERRORS_MASK                                                   0x00100000L
+#define GCVM_L2_STATUS__FOUND_PDE2_CACHE_PARITY_ERRORS_MASK                                                   0x00200000L
+//GCVM_DUMMY_PAGE_FAULT_CNTL
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE__SHIFT                                            0x0
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL__SHIFT                                         0x1
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS__SHIFT                                            0x2
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_FAULT_ENABLE_MASK                                              0x00000001L
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_ADDRESS_LOGICAL_MASK                                           0x00000002L
+#define GCVM_DUMMY_PAGE_FAULT_CNTL__DUMMY_PAGE_COMPARE_MSBS_MASK                                              0x000000FCL
+//GCVM_DUMMY_PAGE_FAULT_ADDR_LO32
+#define GCVM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32__SHIFT                                          0x0
+#define GCVM_DUMMY_PAGE_FAULT_ADDR_LO32__DUMMY_PAGE_ADDR_LO32_MASK                                            0xFFFFFFFFL
+//GCVM_DUMMY_PAGE_FAULT_ADDR_HI32
+#define GCVM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI4__SHIFT                                           0x0
+#define GCVM_DUMMY_PAGE_FAULT_ADDR_HI32__DUMMY_PAGE_ADDR_HI4_MASK                                             0x0000000FL
+//GCVM_INVALIDATE_CNTL
+#define GCVM_INVALIDATE_CNTL__PRI_REG_ALTERNATING__SHIFT                                                      0x0
+#define GCVM_INVALIDATE_CNTL__MAX_REG_OUTSTANDING__SHIFT                                                      0x8
+#define GCVM_INVALIDATE_CNTL__PRI_REG_ALTERNATING_MASK                                                        0x000000FFL
+#define GCVM_INVALIDATE_CNTL__MAX_REG_OUTSTANDING_MASK                                                        0x0000FF00L
+//GCVM_L2_PROTECTION_FAULT_CNTL
+#define GCVM_L2_PROTECTION_FAULT_CNTL__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                              0x0
+#define GCVM_L2_PROTECTION_FAULT_CNTL__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES__SHIFT           0x1
+#define GCVM_L2_PROTECTION_FAULT_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                           0x2
+#define GCVM_L2_PROTECTION_FAULT_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                            0x3
+#define GCVM_L2_PROTECTION_FAULT_CNTL__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                            0x4
+#define GCVM_L2_PROTECTION_FAULT_CNTL__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                            0x5
+#define GCVM_L2_PROTECTION_FAULT_CNTL__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT               0x6
+#define GCVM_L2_PROTECTION_FAULT_CNTL__NACK_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                            0x7
+#define GCVM_L2_PROTECTION_FAULT_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                      0x8
+#define GCVM_L2_PROTECTION_FAULT_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                           0x9
+#define GCVM_L2_PROTECTION_FAULT_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                            0xa
+#define GCVM_L2_PROTECTION_FAULT_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                           0xb
+#define GCVM_L2_PROTECTION_FAULT_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                         0xc
+#define GCVM_L2_PROTECTION_FAULT_CNTL__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                              0xd
+#define GCVM_L2_PROTECTION_FAULT_CNTL__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                        0x1d
+#define GCVM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_NO_RETRY_FAULT__SHIFT                                         0x1e
+#define GCVM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_RETRY_FAULT__SHIFT                                            0x1f
+#define GCVM_L2_PROTECTION_FAULT_CNTL__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                0x00000001L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__ALLOW_SUBSEQUENT_PROTECTION_FAULT_STATUS_ADDR_UPDATES_MASK             0x00000002L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                             0x00000004L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                              0x00000008L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__PDE1_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                              0x00000010L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__PDE2_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                              0x00000020L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                 0x00000040L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__NACK_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                              0x00000080L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                        0x00000100L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                             0x00000200L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                              0x00000400L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                             0x00000800L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                           0x00001000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                                0x1FFFE000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__OTHER_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                          0x20000000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_NO_RETRY_FAULT_MASK                                           0x40000000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL__CRASH_ON_RETRY_FAULT_MASK                                              0x80000000L
+//GCVM_L2_PROTECTION_FAULT_CNTL2
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                                  0x0
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT__SHIFT                            0x10
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE__SHIFT                                      0x11
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY__SHIFT                           0x12
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT__SHIFT                                   0x13
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                                    0x0000FFFFL
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__OTHER_CLIENT_ID_PRT_FAULT_INTERRUPT_MASK                              0x00010000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_MASK                                        0x00020000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY_MASK                             0x00040000L
+#define GCVM_L2_PROTECTION_FAULT_CNTL2__ENABLE_RETRY_FAULT_INTERRUPT_MASK                                     0x00080000L
+//GCVM_L2_PROTECTION_FAULT_MM_CNTL3
+#define GCVM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT                0x0
+#define GCVM_L2_PROTECTION_FAULT_MM_CNTL3__VML1_READ_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                  0xFFFFFFFFL
+//GCVM_L2_PROTECTION_FAULT_MM_CNTL4
+#define GCVM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT__SHIFT               0x0
+#define GCVM_L2_PROTECTION_FAULT_MM_CNTL4__VML1_WRITE_CLIENT_ID_NO_RETRY_FAULT_INTERRUPT_MASK                 0xFFFFFFFFL
+//GCVM_L2_PROTECTION_FAULT_STATUS
+#define GCVM_L2_PROTECTION_FAULT_STATUS__MORE_FAULTS__SHIFT                                                   0x0
+#define GCVM_L2_PROTECTION_FAULT_STATUS__WALKER_ERROR__SHIFT                                                  0x1
+#define GCVM_L2_PROTECTION_FAULT_STATUS__PERMISSION_FAULTS__SHIFT                                             0x4
+#define GCVM_L2_PROTECTION_FAULT_STATUS__MAPPING_ERROR__SHIFT                                                 0x8
+#define GCVM_L2_PROTECTION_FAULT_STATUS__CID__SHIFT                                                           0x9
+#define GCVM_L2_PROTECTION_FAULT_STATUS__RW__SHIFT                                                            0x12
+#define GCVM_L2_PROTECTION_FAULT_STATUS__ATOMIC__SHIFT                                                        0x13
+#define GCVM_L2_PROTECTION_FAULT_STATUS__VMID__SHIFT                                                          0x14
+#define GCVM_L2_PROTECTION_FAULT_STATUS__VF__SHIFT                                                            0x18
+#define GCVM_L2_PROTECTION_FAULT_STATUS__VFID__SHIFT                                                          0x19
+#define GCVM_L2_PROTECTION_FAULT_STATUS__PRT__SHIFT                                                           0x1d
+#define GCVM_L2_PROTECTION_FAULT_STATUS__FED__SHIFT                                                           0x1e
+#define GCVM_L2_PROTECTION_FAULT_STATUS__MORE_FAULTS_MASK                                                     0x00000001L
+#define GCVM_L2_PROTECTION_FAULT_STATUS__WALKER_ERROR_MASK                                                    0x0000000EL
+#define GCVM_L2_PROTECTION_FAULT_STATUS__PERMISSION_FAULTS_MASK                                               0x000000F0L
+#define GCVM_L2_PROTECTION_FAULT_STATUS__MAPPING_ERROR_MASK                                                   0x00000100L
+#define GCVM_L2_PROTECTION_FAULT_STATUS__CID_MASK                                                             0x0003FE00L
+#define GCVM_L2_PROTECTION_FAULT_STATUS__RW_MASK                                                              0x00040000L
+#define GCVM_L2_PROTECTION_FAULT_STATUS__ATOMIC_MASK                                                          0x00080000L
+#define GCVM_L2_PROTECTION_FAULT_STATUS__VMID_MASK                                                            0x00F00000L
+#define GCVM_L2_PROTECTION_FAULT_STATUS__VF_MASK                                                              0x01000000L
+#define GCVM_L2_PROTECTION_FAULT_STATUS__VFID_MASK                                                            0x1E000000L
+#define GCVM_L2_PROTECTION_FAULT_STATUS__PRT_MASK                                                             0x20000000L
+#define GCVM_L2_PROTECTION_FAULT_STATUS__FED_MASK                                                             0x40000000L
+//GCVM_L2_PROTECTION_FAULT_ADDR_LO32
+#define GCVM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32__SHIFT                                     0x0
+#define GCVM_L2_PROTECTION_FAULT_ADDR_LO32__LOGICAL_PAGE_ADDR_LO32_MASK                                       0xFFFFFFFFL
+//GCVM_L2_PROTECTION_FAULT_ADDR_HI32
+#define GCVM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI4__SHIFT                                      0x0
+#define GCVM_L2_PROTECTION_FAULT_ADDR_HI32__LOGICAL_PAGE_ADDR_HI4_MASK                                        0x0000000FL
+//GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32
+#define GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32__SHIFT                            0x0
+#define GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32__PHYSICAL_PAGE_ADDR_LO32_MASK                              0xFFFFFFFFL
+//GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32
+#define GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI4__SHIFT                             0x0
+#define GCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32__PHYSICAL_PAGE_ADDR_HI4_MASK                               0x0000000FL
+//GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                     0x0
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                       0xFFFFFFFFL
+//GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                      0x0
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                        0x0000000FL
+//GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                    0x0
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                      0xFFFFFFFFL
+//GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                     0x0
+#define GCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                       0x0000000FL
+//GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32
+#define GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32__SHIFT                       0x0
+#define GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32__PHYSICAL_PAGE_OFFSET_LO32_MASK                         0xFFFFFFFFL
+//GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32
+#define GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI4__SHIFT                        0x0
+#define GCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32__PHYSICAL_PAGE_OFFSET_HI4_MASK                          0x0000000FL
+//GCVM_L2_CNTL4
+#define GCVM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT__SHIFT                                                     0x0
+#define GCVM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL__SHIFT                                                    0x6
+#define GCVM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL__SHIFT                                                    0x7
+#define GCVM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                         0x8
+#define GCVM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT__SHIFT                                        0x12
+#define GCVM_L2_CNTL4__BPM_CGCGLS_OVERRIDE__SHIFT                                                             0x1c
+#define GCVM_L2_CNTL4__GC_CH_FGCG_OFF__SHIFT                                                                  0x1d
+#define GCVM_L2_CNTL4__VFIFO_HEAD_OF_QUEUE__SHIFT                                                             0x1e
+#define GCVM_L2_CNTL4__VFIFO_VISIBLE_BANK_SILOS__SHIFT                                                        0x1f
+#define GCVM_L2_CNTL4__L2_CACHE_4K_PARTITION_COUNT_MASK                                                       0x0000003FL
+#define GCVM_L2_CNTL4__VMC_TAP_PDE_REQUEST_PHYSICAL_MASK                                                      0x00000040L
+#define GCVM_L2_CNTL4__VMC_TAP_PTE_REQUEST_PHYSICAL_MASK                                                      0x00000080L
+#define GCVM_L2_CNTL4__MM_NONRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                           0x0003FF00L
+#define GCVM_L2_CNTL4__MM_SOFTRT_IFIFO_ACTIVE_TRANSACTION_LIMIT_MASK                                          0x0FFC0000L
+#define GCVM_L2_CNTL4__BPM_CGCGLS_OVERRIDE_MASK                                                               0x10000000L
+#define GCVM_L2_CNTL4__GC_CH_FGCG_OFF_MASK                                                                    0x20000000L
+#define GCVM_L2_CNTL4__VFIFO_HEAD_OF_QUEUE_MASK                                                               0x40000000L
+#define GCVM_L2_CNTL4__VFIFO_VISIBLE_BANK_SILOS_MASK                                                          0x80000000L
+//GCVM_L2_MM_GROUP_RT_CLASSES
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS__SHIFT                                                  0x0
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS__SHIFT                                                  0x1
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS__SHIFT                                                  0x2
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS__SHIFT                                                  0x3
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS__SHIFT                                                  0x4
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS__SHIFT                                                  0x5
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS__SHIFT                                                  0x6
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS__SHIFT                                                  0x7
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS__SHIFT                                                  0x8
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS__SHIFT                                                  0x9
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS__SHIFT                                                 0xa
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS__SHIFT                                                 0xb
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS__SHIFT                                                 0xc
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS__SHIFT                                                 0xd
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS__SHIFT                                                 0xe
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS__SHIFT                                                 0xf
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS__SHIFT                                                 0x10
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS__SHIFT                                                 0x11
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS__SHIFT                                                 0x12
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS__SHIFT                                                 0x13
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS__SHIFT                                                 0x14
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS__SHIFT                                                 0x15
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS__SHIFT                                                 0x16
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS__SHIFT                                                 0x17
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS__SHIFT                                                 0x18
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS__SHIFT                                                 0x19
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS__SHIFT                                                 0x1a
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS__SHIFT                                                 0x1b
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS__SHIFT                                                 0x1c
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS__SHIFT                                                 0x1d
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS__SHIFT                                                 0x1e
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS__SHIFT                                                 0x1f
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_0_RT_CLASS_MASK                                                    0x00000001L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_1_RT_CLASS_MASK                                                    0x00000002L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_2_RT_CLASS_MASK                                                    0x00000004L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_3_RT_CLASS_MASK                                                    0x00000008L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_4_RT_CLASS_MASK                                                    0x00000010L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_5_RT_CLASS_MASK                                                    0x00000020L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_6_RT_CLASS_MASK                                                    0x00000040L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_7_RT_CLASS_MASK                                                    0x00000080L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_8_RT_CLASS_MASK                                                    0x00000100L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_9_RT_CLASS_MASK                                                    0x00000200L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_10_RT_CLASS_MASK                                                   0x00000400L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_11_RT_CLASS_MASK                                                   0x00000800L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_12_RT_CLASS_MASK                                                   0x00001000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_13_RT_CLASS_MASK                                                   0x00002000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_14_RT_CLASS_MASK                                                   0x00004000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_15_RT_CLASS_MASK                                                   0x00008000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_16_RT_CLASS_MASK                                                   0x00010000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_17_RT_CLASS_MASK                                                   0x00020000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_18_RT_CLASS_MASK                                                   0x00040000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_19_RT_CLASS_MASK                                                   0x00080000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_20_RT_CLASS_MASK                                                   0x00100000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_21_RT_CLASS_MASK                                                   0x00200000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_22_RT_CLASS_MASK                                                   0x00400000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_23_RT_CLASS_MASK                                                   0x00800000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_24_RT_CLASS_MASK                                                   0x01000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_25_RT_CLASS_MASK                                                   0x02000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_26_RT_CLASS_MASK                                                   0x04000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_27_RT_CLASS_MASK                                                   0x08000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_28_RT_CLASS_MASK                                                   0x10000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_29_RT_CLASS_MASK                                                   0x20000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_30_RT_CLASS_MASK                                                   0x40000000L
+#define GCVM_L2_MM_GROUP_RT_CLASSES__GROUP_31_RT_CLASS_MASK                                                   0x80000000L
+//GCVM_L2_BANK_SELECT_RESERVED_CID
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID__SHIFT                                      0x0
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID__SHIFT                                     0xa
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__ENABLE__SHIFT                                                       0x14
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                             0x18
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                          0x19
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_FRAGMENT_SIZE__SHIFT                                 0x1a
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_READ_CLIENT_ID_MASK                                        0x000001FFL
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_WRITE_CLIENT_ID_MASK                                       0x0007FC00L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__ENABLE_MASK                                                         0x00100000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_INVALIDATION_MODE_MASK                               0x01000000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                            0x02000000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID__RESERVED_CACHE_FRAGMENT_SIZE_MASK                                   0x7C000000L
+//GCVM_L2_BANK_SELECT_RESERVED_CID2
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID__SHIFT                                     0x0
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID__SHIFT                                    0xa
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__ENABLE__SHIFT                                                      0x14
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE__SHIFT                            0x18
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION__SHIFT                         0x19
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_FRAGMENT_SIZE__SHIFT                                0x1a
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_READ_CLIENT_ID_MASK                                       0x000001FFL
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_WRITE_CLIENT_ID_MASK                                      0x0007FC00L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__ENABLE_MASK                                                        0x00100000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_INVALIDATION_MODE_MASK                              0x01000000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_PRIVATE_INVALIDATION_MASK                           0x02000000L
+#define GCVM_L2_BANK_SELECT_RESERVED_CID2__RESERVED_CACHE_FRAGMENT_SIZE_MASK                                  0x7C000000L
+//GCVM_L2_CACHE_PARITY_CNTL
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES__SHIFT                               0x0
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES__SHIFT                             0x1
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES__SHIFT                                  0x2
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE__SHIFT                               0x3
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE__SHIFT                             0x4
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE__SHIFT                                  0x5
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK__SHIFT                                                    0x6
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER__SHIFT                                                  0x9
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC__SHIFT                                                   0xc
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_4K_PTE_CACHES_MASK                                 0x00000001L
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_BIGK_PTE_CACHES_MASK                               0x00000002L
+#define GCVM_L2_CACHE_PARITY_CNTL__ENABLE_PARITY_CHECKS_IN_PDE_CACHES_MASK                                    0x00000004L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_4K_PTE_CACHE_MASK                                 0x00000008L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_BIGK_PTE_CACHE_MASK                               0x00000010L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_PARITY_MISMATCH_IN_PDE_CACHE_MASK                                    0x00000020L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_BANK_MASK                                                      0x000001C0L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_NUMBER_MASK                                                    0x00000E00L
+#define GCVM_L2_CACHE_PARITY_CNTL__FORCE_CACHE_ASSOC_MASK                                                     0x0000F000L
+//GCVM_L2_ICG_CTRL
+#define GCVM_L2_ICG_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x0
+#define GCVM_L2_ICG_CTRL__DYNAMIC_CLOCK_OVERRIDE__SHIFT                                                       0x4
+#define GCVM_L2_ICG_CTRL__STATIC_CLOCK_OVERRIDE__SHIFT                                                        0x5
+#define GCVM_L2_ICG_CTRL__AON_CLOCK_OVERRIDE__SHIFT                                                           0x6
+#define GCVM_L2_ICG_CTRL__PERFMON_CLOCK_OVERRIDE__SHIFT                                                       0x7
+#define GCVM_L2_ICG_CTRL__OFF_HYSTERESIS_MASK                                                                 0x0000000FL
+#define GCVM_L2_ICG_CTRL__DYNAMIC_CLOCK_OVERRIDE_MASK                                                         0x00000010L
+#define GCVM_L2_ICG_CTRL__STATIC_CLOCK_OVERRIDE_MASK                                                          0x00000020L
+#define GCVM_L2_ICG_CTRL__AON_CLOCK_OVERRIDE_MASK                                                             0x00000040L
+#define GCVM_L2_ICG_CTRL__PERFMON_CLOCK_OVERRIDE_MASK                                                         0x00000080L
+//GCVM_L2_CNTL5
+#define GCVM_L2_CNTL5__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT                                                   0x0
+#define GCVM_L2_CNTL5__WALKER_PRIORITY_CLIENT_ID__SHIFT                                                       0x5
+#define GCVM_L2_CNTL5__WALKER_FETCH_PDE_NOALLOC_ENABLE__SHIFT                                                 0xe
+#define GCVM_L2_CNTL5__WALKER_FETCH_PDE_MTYPE_ENABLE__SHIFT                                                   0xf
+#define GCVM_L2_CNTL5__UTCL2_ATC_REQ_FGCG_OFF__SHIFT                                                          0x10
+#define GCVM_L2_CNTL5__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                                                     0x0000001FL
+#define GCVM_L2_CNTL5__WALKER_PRIORITY_CLIENT_ID_MASK                                                         0x00003FE0L
+#define GCVM_L2_CNTL5__WALKER_FETCH_PDE_NOALLOC_ENABLE_MASK                                                   0x00004000L
+#define GCVM_L2_CNTL5__WALKER_FETCH_PDE_MTYPE_ENABLE_MASK                                                     0x00008000L
+#define GCVM_L2_CNTL5__UTCL2_ATC_REQ_FGCG_OFF_MASK                                                            0x00010000L
+//GCVM_L2_GCR_CNTL
+#define GCVM_L2_GCR_CNTL__GCR_ENABLE__SHIFT                                                                   0x0
+#define GCVM_L2_GCR_CNTL__GCR_CLIENT_ID__SHIFT                                                                0x1
+#define GCVM_L2_GCR_CNTL__GCR_ENABLE_MASK                                                                     0x00000001L
+#define GCVM_L2_GCR_CNTL__GCR_CLIENT_ID_MASK                                                                  0x000003FEL
+//GCVML2_WALKER_MACRO_THROTTLE_TIME
+#define GCVML2_WALKER_MACRO_THROTTLE_TIME__TIME__SHIFT                                                        0x0
+#define GCVML2_WALKER_MACRO_THROTTLE_TIME__TIME_MASK                                                          0x00FFFFFFL
+//GCVML2_WALKER_MACRO_THROTTLE_FETCH_LIMIT
+#define GCVML2_WALKER_MACRO_THROTTLE_FETCH_LIMIT__LIMIT__SHIFT                                                0x1
+#define GCVML2_WALKER_MACRO_THROTTLE_FETCH_LIMIT__LIMIT_MASK                                                  0x0000FFFEL
+//GCVML2_WALKER_MICRO_THROTTLE_TIME
+#define GCVML2_WALKER_MICRO_THROTTLE_TIME__TIME__SHIFT                                                        0x0
+#define GCVML2_WALKER_MICRO_THROTTLE_TIME__TIME_MASK                                                          0x00FFFFFFL
+//GCVML2_WALKER_MICRO_THROTTLE_FETCH_LIMIT
+#define GCVML2_WALKER_MICRO_THROTTLE_FETCH_LIMIT__LIMIT__SHIFT                                                0x1
+#define GCVML2_WALKER_MICRO_THROTTLE_FETCH_LIMIT__LIMIT_MASK                                                  0x0000FFFEL
+//GCVM_L2_CGTT_BUSY_CTRL
+#define GCVM_L2_CGTT_BUSY_CTRL__READ_DELAY__SHIFT                                                             0x0
+#define GCVM_L2_CGTT_BUSY_CTRL__ALWAYS_BUSY__SHIFT                                                            0x5
+#define GCVM_L2_CGTT_BUSY_CTRL__READ_DELAY_MASK                                                               0x0000001FL
+#define GCVM_L2_CGTT_BUSY_CTRL__ALWAYS_BUSY_MASK                                                              0x00000020L
+//GCVM_L2_PTE_CACHE_DUMP_CNTL
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__ENABLE__SHIFT                                                            0x0
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__READY__SHIFT                                                             0x1
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__BANK__SHIFT                                                              0x4
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__CACHE__SHIFT                                                             0x8
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__ASSOC__SHIFT                                                             0xc
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__INDEX__SHIFT                                                             0x10
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__ENABLE_MASK                                                              0x00000001L
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__READY_MASK                                                               0x00000002L
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__BANK_MASK                                                                0x000000F0L
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__CACHE_MASK                                                               0x00000F00L
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__ASSOC_MASK                                                               0x0000F000L
+#define GCVM_L2_PTE_CACHE_DUMP_CNTL__INDEX_MASK                                                               0xFFFF0000L
+//GCVM_L2_PTE_CACHE_DUMP_READ
+#define GCVM_L2_PTE_CACHE_DUMP_READ__DATA__SHIFT                                                              0x0
+#define GCVM_L2_PTE_CACHE_DUMP_READ__DATA_MASK                                                                0xFFFFFFFFL
+//GCVM_L2_BANK_SELECT_MASKS
+#define GCVM_L2_BANK_SELECT_MASKS__MASK0__SHIFT                                                               0x0
+#define GCVM_L2_BANK_SELECT_MASKS__MASK1__SHIFT                                                               0x4
+#define GCVM_L2_BANK_SELECT_MASKS__MASK2__SHIFT                                                               0x8
+#define GCVM_L2_BANK_SELECT_MASKS__MASK3__SHIFT                                                               0xc
+#define GCVM_L2_BANK_SELECT_MASKS__MASK0_MASK                                                                 0x0000000FL
+#define GCVM_L2_BANK_SELECT_MASKS__MASK1_MASK                                                                 0x000000F0L
+#define GCVM_L2_BANK_SELECT_MASKS__MASK2_MASK                                                                 0x00000F00L
+#define GCVM_L2_BANK_SELECT_MASKS__MASK3_MASK                                                                 0x0000F000L
+//GCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC
+#define GCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__CREDITS__SHIFT                                                   0x0
+#define GCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__UPDATE__SHIFT                                                    0xa
+#define GCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__CREDITS_MASK                                                     0x000003FFL
+#define GCUTCL2_CREDIT_SAFETY_GROUP_RET_CDC__UPDATE_MASK                                                      0x00000400L
+//GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__CREDITS__SHIFT                                        0x0
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__UPDATE__SHIFT                                         0xa
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__CREDITS_MASK                                          0x000003FFL
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_CDC__UPDATE_MASK                                           0x00000400L
+//GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__CREDITS__SHIFT                                      0x0
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__UPDATE__SHIFT                                       0xa
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__CREDITS_MASK                                        0x000003FFL
+#define GCUTCL2_CREDIT_SAFETY_GROUP_CLIENTS_INVREQ_NOCDC__UPDATE_MASK                                         0x00000400L
+//GCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT
+#define GCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__CREDITS__SHIFT                                               0x0
+#define GCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__UPDATE__SHIFT                                                0xa
+#define GCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__CREDITS_MASK                                                 0x000003FFL
+#define GCVML2_CREDIT_SAFETY_IH_FAULT_INTERRUPT__UPDATE_MASK                                                  0x00000400L
+//GCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ
+#define GCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__CREDITS__SHIFT                                               0x0
+#define GCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__UPDATE__SHIFT                                                0xa
+#define GCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__CREDITS_MASK                                                 0x000003FFL
+#define GCVML2_WALKER_CREDIT_SAFETY_FETCH_RDREQ__UPDATE_MASK                                                  0x00000400L
+
+
+// addressBlock: gc_gcatcl2dec
+//GC_ATC_L2_CNTL
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS__SHIFT                                            0x0
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS__SHIFT                                           0x3
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                                0x6
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                               0x7
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READ_REQUESTS__SHIFT                                       0x8
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITE_REQUESTS__SHIFT                                      0xb
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD__SHIFT                           0xe
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD__SHIFT                          0xf
+#define GC_ATC_L2_CNTL__CACHE_INVALIDATE_MODE__SHIFT                                                          0x10
+#define GC_ATC_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY__SHIFT                                       0x13
+#define GC_ATC_L2_CNTL__FRAG_APT_INTXN_MODE__SHIFT                                                            0x14
+#define GC_ATC_L2_CNTL__CLI_GPA_REQ_FRAG_SIZE__SHIFT                                                          0x16
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READ_REQUESTS_MASK                                              0x00000003L
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITE_REQUESTS_MASK                                             0x00000018L
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                                  0x00000040L
+#define GC_ATC_L2_CNTL__NUMBER_OF_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                                 0x00000080L
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READ_REQUESTS_MASK                                         0x00000300L
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITE_REQUESTS_MASK                                        0x00001800L
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_READS_DEPENDS_ON_ADDR_MOD_MASK                             0x00004000L
+#define GC_ATC_L2_CNTL__NUMBER_OF_HOST_TRANSLATION_WRITES_DEPENDS_ON_ADDR_MOD_MASK                            0x00008000L
+#define GC_ATC_L2_CNTL__CACHE_INVALIDATE_MODE_MASK                                                            0x00070000L
+#define GC_ATC_L2_CNTL__ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY_MASK                                         0x00080000L
+#define GC_ATC_L2_CNTL__FRAG_APT_INTXN_MODE_MASK                                                              0x00300000L
+#define GC_ATC_L2_CNTL__CLI_GPA_REQ_FRAG_SIZE_MASK                                                            0x0FC00000L
+//GC_ATC_L2_CNTL2
+#define GC_ATC_L2_CNTL2__BANK_SELECT__SHIFT                                                                   0x0
+#define GC_ATC_L2_CNTL2__NUM_BANKS_LOG2__SHIFT                                                                0x6
+#define GC_ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE__SHIFT                                                          0x9
+#define GC_ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE__SHIFT                                           0xb
+#define GC_ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS__SHIFT                                                  0xc
+#define GC_ATC_L2_CNTL2__L2_CACHE_VMID_MODE__SHIFT                                                            0xf
+#define GC_ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE__SHIFT                                      0x12
+#define GC_ATC_L2_CNTL2__BANK_SELECT_MASK                                                                     0x0000003FL
+#define GC_ATC_L2_CNTL2__NUM_BANKS_LOG2_MASK                                                                  0x000001C0L
+#define GC_ATC_L2_CNTL2__L2_CACHE_UPDATE_MODE_MASK                                                            0x00000600L
+#define GC_ATC_L2_CNTL2__ENABLE_L2_CACHE_LRU_UPDATE_BY_WRITE_MASK                                             0x00000800L
+#define GC_ATC_L2_CNTL2__L2_CACHE_SWAP_TAG_INDEX_LSBS_MASK                                                    0x00007000L
+#define GC_ATC_L2_CNTL2__L2_CACHE_VMID_MODE_MASK                                                              0x00038000L
+#define GC_ATC_L2_CNTL2__L2_CACHE_UPDATE_WILDCARD_REFERENCE_VALUE_MASK                                        0x00FC0000L
+//GC_ATC_L2_CACHE_DATA0
+#define GC_ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID__SHIFT                                                     0x0
+#define GC_ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID__SHIFT                                                       0x1
+#define GC_ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES__SHIFT                                                       0x2
+#define GC_ATC_L2_CACHE_DATA0__VIRTUAL_PAGE_ADDRESS_HIGH__SHIFT                                               0x18
+#define GC_ATC_L2_CACHE_DATA0__DATA_REGISTER_VALID_MASK                                                       0x00000001L
+#define GC_ATC_L2_CACHE_DATA0__CACHE_ENTRY_VALID_MASK                                                         0x00000002L
+#define GC_ATC_L2_CACHE_DATA0__CACHED_ATTRIBUTES_MASK                                                         0x00FFFFFCL
+#define GC_ATC_L2_CACHE_DATA0__VIRTUAL_PAGE_ADDRESS_HIGH_MASK                                                 0x0F000000L
+//GC_ATC_L2_CACHE_DATA1
+#define GC_ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW__SHIFT                                                0x0
+#define GC_ATC_L2_CACHE_DATA1__VIRTUAL_PAGE_ADDRESS_LOW_MASK                                                  0xFFFFFFFFL
+//GC_ATC_L2_CACHE_DATA2
+#define GC_ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS__SHIFT                                                   0x0
+#define GC_ATC_L2_CACHE_DATA2__PHYSICAL_PAGE_ADDRESS_MASK                                                     0xFFFFFFFFL
+//GC_ATC_L2_CNTL3
+#define GC_ATC_L2_CNTL3__L2_SMALLK_CACHE_FRAGMENT_SIZE__SHIFT                                                 0x0
+#define GC_ATC_L2_CNTL3__L2_MIDK_CACHE_FRAGMENT_SIZE__SHIFT                                                   0x6
+#define GC_ATC_L2_CNTL3__L2_BIGK_CACHE_FRAGMENT_SIZE__SHIFT                                                   0xc
+#define GC_ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST__SHIFT                                               0x12
+#define GC_ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1__SHIFT                                                     0x15
+#define GC_ATC_L2_CNTL3__COMPCLKREQ_OFF_HYSTERESIS__SHIFT                                                     0x1b
+#define GC_ATC_L2_CNTL3__REPEATER_FGCG_OFF__SHIFT                                                             0x1e
+#define GC_ATC_L2_CNTL3__L2_SMALLK_CACHE_FRAGMENT_SIZE_MASK                                                   0x0000003FL
+#define GC_ATC_L2_CNTL3__L2_MIDK_CACHE_FRAGMENT_SIZE_MASK                                                     0x00000FC0L
+#define GC_ATC_L2_CNTL3__L2_BIGK_CACHE_FRAGMENT_SIZE_MASK                                                     0x0003F000L
+#define GC_ATC_L2_CNTL3__DELAY_SEND_INVALIDATION_REQUEST_MASK                                                 0x001C0000L
+#define GC_ATC_L2_CNTL3__ATS_REQUEST_CREDIT_MINUS1_MASK                                                       0x07E00000L
+#define GC_ATC_L2_CNTL3__COMPCLKREQ_OFF_HYSTERESIS_MASK                                                       0x38000000L
+#define GC_ATC_L2_CNTL3__REPEATER_FGCG_OFF_MASK                                                               0x40000000L
+//GC_ATC_L2_STATUS
+#define GC_ATC_L2_STATUS__BUSY__SHIFT                                                                         0x0
+#define GC_ATC_L2_STATUS__NO_OUTSTANDING_AT_REQUESTS__SHIFT                                                   0x1
+#define GC_ATC_L2_STATUS__BUSY_MASK                                                                           0x00000001L
+#define GC_ATC_L2_STATUS__NO_OUTSTANDING_AT_REQUESTS_MASK                                                     0x00000002L
+//GC_ATC_L2_STATUS2
+#define GC_ATC_L2_STATUS2__IFIFO_NON_FATAL_PARITY_ERROR_INFO__SHIFT                                           0x0
+#define GC_ATC_L2_STATUS2__IFIFO_FATAL_PARITY_ERROR_INFO__SHIFT                                               0x8
+#define GC_ATC_L2_STATUS2__IFIFO_NON_FATAL_PARITY_ERROR_INFO_MASK                                             0x000000FFL
+#define GC_ATC_L2_STATUS2__IFIFO_FATAL_PARITY_ERROR_INFO_MASK                                                 0x0000FF00L
+//GC_ATC_L2_MISC_CG
+#define GC_ATC_L2_MISC_CG__OFFDLY__SHIFT                                                                      0x6
+#define GC_ATC_L2_MISC_CG__ENABLE__SHIFT                                                                      0x12
+#define GC_ATC_L2_MISC_CG__MEM_LS_ENABLE__SHIFT                                                               0x13
+#define GC_ATC_L2_MISC_CG__OFFDLY_MASK                                                                        0x00000FC0L
+#define GC_ATC_L2_MISC_CG__ENABLE_MASK                                                                        0x00040000L
+#define GC_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK                                                                 0x00080000L
+//GC_ATC_L2_MEM_POWER_LS
+#define GC_ATC_L2_MEM_POWER_LS__LS_SETUP__SHIFT                                                               0x0
+#define GC_ATC_L2_MEM_POWER_LS__LS_HOLD__SHIFT                                                                0x6
+#define GC_ATC_L2_MEM_POWER_LS__LS_SETUP_MASK                                                                 0x0000003FL
+#define GC_ATC_L2_MEM_POWER_LS__LS_HOLD_MASK                                                                  0x00000FC0L
+//GC_ATC_L2_SDPPORT_CTRL
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKEN__SHIFT                                                      0x0
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKENRCV__SHIFT                                                   0x1
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKEN__SHIFT                                                  0x2
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKENRCV__SHIFT                                               0x3
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKEN__SHIFT                                                      0x4
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKENRCV__SHIFT                                                   0x5
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKEN__SHIFT                                                        0x6
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKENRCV__SHIFT                                                     0x7
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKEN__SHIFT                                                   0x8
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKENRCV__SHIFT                                                0x9
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKEN_MASK                                                        0x00000001L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPCKENRCV_MASK                                                     0x00000002L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKEN_MASK                                                    0x00000004L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_RDRSPDATACKENRCV_MASK                                                 0x00000008L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKEN_MASK                                                        0x00000010L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_WRRSPCKENRCV_MASK                                                     0x00000020L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKEN_MASK                                                          0x00000040L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_REQCKENRCV_MASK                                                       0x00000080L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKEN_MASK                                                     0x00000100L
+#define GC_ATC_L2_SDPPORT_CTRL__SDPVDCI_ORIGDATACKENRCV_MASK                                                  0x00000200L
+
+
+// addressBlock: gc_gcl2tlbpfdec
+//GCL2TLB_TLB0_STATUS
+#define GCL2TLB_TLB0_STATUS__BUSY__SHIFT                                                                      0x0
+#define GCL2TLB_TLB0_STATUS__FOUND_PARITY_ERRORS__SHIFT                                                       0x1
+#define GCL2TLB_TLB0_STATUS__FOUND_APERTURE_FAULTS__SHIFT                                                     0x2
+#define GCL2TLB_TLB0_STATUS__BUSY_MASK                                                                        0x00000001L
+#define GCL2TLB_TLB0_STATUS__FOUND_PARITY_ERRORS_MASK                                                         0x00000002L
+#define GCL2TLB_TLB0_STATUS__FOUND_APERTURE_FAULTS_MASK                                                       0x00000004L
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO__ADDR__SHIFT                                           0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_LO__ADDR_MASK                                             0xFFFFFFFFL
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__ADDR__SHIFT                                           0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VMID__SHIFT                                           0x4
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VFID__SHIFT                                           0x8
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VF__SHIFT                                             0xc
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__GPA__SHIFT                                            0xd
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__RD_PERM__SHIFT                                        0xf
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__WR_PERM__SHIFT                                        0x10
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__EX_PERM__SHIFT                                        0x11
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__CLIENT_ID__SHIFT                                      0x12
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__REQ__SHIFT                                            0x1e
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__ADDR_MASK                                             0x0000000FL
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VMID_MASK                                             0x000000F0L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VFID_MASK                                             0x00000F00L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__VF_MASK                                               0x00001000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__GPA_MASK                                              0x00006000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__RD_PERM_MASK                                          0x00008000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__WR_PERM_MASK                                          0x00010000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__EX_PERM_MASK                                          0x00020000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__CLIENT_ID_MASK                                        0x07FC0000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_REQUEST_HI__REQ_MASK                                              0x40000000L
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO__ADDR__SHIFT                                          0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_LO__ADDR_MASK                                            0xFFFFFFFFL
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ADDR__SHIFT                                          0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PERMS__SHIFT                                         0x4
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__FRAGMENT_SIZE__SHIFT                                 0x7
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SNOOP__SHIFT                                         0xd
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SPA__SHIFT                                           0xe
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__IO__SHIFT                                            0xf
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PTE_TMZ__SHIFT                                       0x10
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NO_PTE__SHIFT                                        0x11
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MTYPE__SHIFT                                         0x12
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MEMLOG__SHIFT                                        0x15
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NACK__SHIFT                                          0x16
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__LLC_NOALLOC__SHIFT                                   0x18
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ACK__SHIFT                                           0x1f
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ADDR_MASK                                            0x0000000FL
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PERMS_MASK                                           0x00000070L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__FRAGMENT_SIZE_MASK                                   0x00001F80L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SNOOP_MASK                                           0x00002000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__SPA_MASK                                             0x00004000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__IO_MASK                                              0x00008000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__PTE_TMZ_MASK                                         0x00010000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NO_PTE_MASK                                          0x00020000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MTYPE_MASK                                           0x001C0000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__MEMLOG_MASK                                          0x00200000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__NACK_MASK                                            0x00C00000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__LLC_NOALLOC_MASK                                     0x01000000L
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_RESPONSE_HI__ACK_MASK                                             0x80000000L
+
+
+// addressBlock: gc_gcvmsharedvcdec
+//GCMC_VM_FB_LOCATION_BASE
+#define GCMC_VM_FB_LOCATION_BASE__FB_BASE__SHIFT                                                              0x0
+#define GCMC_VM_FB_LOCATION_BASE__FB_BASE_MASK                                                                0x00FFFFFFL
+//GCMC_VM_FB_LOCATION_TOP
+#define GCMC_VM_FB_LOCATION_TOP__FB_TOP__SHIFT                                                                0x0
+#define GCMC_VM_FB_LOCATION_TOP__FB_TOP_MASK                                                                  0x00FFFFFFL
+//GCMC_VM_AGP_TOP
+#define GCMC_VM_AGP_TOP__AGP_TOP__SHIFT                                                                       0x0
+#define GCMC_VM_AGP_TOP__AGP_TOP_MASK                                                                         0x00FFFFFFL
+//GCMC_VM_AGP_BOT
+#define GCMC_VM_AGP_BOT__AGP_BOT__SHIFT                                                                       0x0
+#define GCMC_VM_AGP_BOT__AGP_BOT_MASK                                                                         0x00FFFFFFL
+//GCMC_VM_AGP_BASE
+#define GCMC_VM_AGP_BASE__AGP_BASE__SHIFT                                                                     0x0
+#define GCMC_VM_AGP_BASE__AGP_BASE_MASK                                                                       0x00FFFFFFL
+//GCMC_VM_SYSTEM_APERTURE_LOW_ADDR
+#define GCMC_VM_SYSTEM_APERTURE_LOW_ADDR__LOGICAL_ADDR__SHIFT                                                 0x0
+#define GCMC_VM_SYSTEM_APERTURE_LOW_ADDR__LOGICAL_ADDR_MASK                                                   0x3FFFFFFFL
+//GCMC_VM_SYSTEM_APERTURE_HIGH_ADDR
+#define GCMC_VM_SYSTEM_APERTURE_HIGH_ADDR__LOGICAL_ADDR__SHIFT                                                0x0
+#define GCMC_VM_SYSTEM_APERTURE_HIGH_ADDR__LOGICAL_ADDR_MASK                                                  0x3FFFFFFFL
+//GCMC_VM_MX_L1_TLB_CNTL
+#define GCMC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB__SHIFT                                                          0x0
+#define GCMC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE__SHIFT                                                     0x3
+#define GCMC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT                                        0x5
+#define GCMC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL__SHIFT                                           0x6
+#define GCMC_VM_MX_L1_TLB_CNTL__ECO_BITS__SHIFT                                                               0x7
+#define GCMC_VM_MX_L1_TLB_CNTL__MTYPE__SHIFT                                                                  0xb
+#define GCMC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK                                                            0x00000001L
+#define GCMC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK                                                       0x00000018L
+#define GCMC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS_MASK                                          0x00000020L
+#define GCMC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK                                             0x00000040L
+#define GCMC_VM_MX_L1_TLB_CNTL__ECO_BITS_MASK                                                                 0x00000780L
+#define GCMC_VM_MX_L1_TLB_CNTL__MTYPE_MASK                                                                    0x00003800L
+
+
+// addressBlock: gc_gcvml2vcdec
+//GCVM_CONTEXT0_CNTL
+#define GCVM_CONTEXT0_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x3
+#define GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x7
+#define GCVM_CONTEXT0_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x8
+#define GCVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x9
+#define GCVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xa
+#define GCVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xb
+#define GCVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xc
+#define GCVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xd
+#define GCVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xe
+#define GCVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xf
+#define GCVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x10
+#define GCVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x11
+#define GCVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x12
+#define GCVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x13
+#define GCVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x14
+#define GCVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x15
+#define GCVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x16
+#define GCVM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x17
+#define GCVM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x18
+#define GCVM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x00000006L
+#define GCVM_CONTEXT0_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x00000078L
+#define GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000080L
+#define GCVM_CONTEXT0_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000100L
+#define GCVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000200L
+#define GCVM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000400L
+#define GCVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00000800L
+#define GCVM_CONTEXT0_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00001000L
+#define GCVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00002000L
+#define GCVM_CONTEXT0_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00004000L
+#define GCVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00008000L
+#define GCVM_CONTEXT0_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00010000L
+#define GCVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00020000L
+#define GCVM_CONTEXT0_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00040000L
+#define GCVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00080000L
+#define GCVM_CONTEXT0_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00100000L
+#define GCVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00200000L
+#define GCVM_CONTEXT0_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00400000L
+#define GCVM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00800000L
+#define GCVM_CONTEXT0_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x01000000L
+//GCVM_CONTEXT1_CNTL
+#define GCVM_CONTEXT1_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x3
+#define GCVM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x7
+#define GCVM_CONTEXT1_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x8
+#define GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x9
+#define GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xa
+#define GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xb
+#define GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xc
+#define GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xd
+#define GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xe
+#define GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xf
+#define GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x10
+#define GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x11
+#define GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x12
+#define GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x13
+#define GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x14
+#define GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x15
+#define GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x16
+#define GCVM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x17
+#define GCVM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x18
+#define GCVM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x00000006L
+#define GCVM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x00000078L
+#define GCVM_CONTEXT1_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000080L
+#define GCVM_CONTEXT1_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000100L
+#define GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000200L
+#define GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000400L
+#define GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00000800L
+#define GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00001000L
+#define GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00002000L
+#define GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00004000L
+#define GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00008000L
+#define GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00010000L
+#define GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00020000L
+#define GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00040000L
+#define GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00080000L
+#define GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00100000L
+#define GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00200000L
+#define GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00400000L
+#define GCVM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00800000L
+#define GCVM_CONTEXT1_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x01000000L
+//GCVM_CONTEXT2_CNTL
+#define GCVM_CONTEXT2_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x3
+#define GCVM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x7
+#define GCVM_CONTEXT2_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x8
+#define GCVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x9
+#define GCVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xa
+#define GCVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xb
+#define GCVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xc
+#define GCVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xd
+#define GCVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xe
+#define GCVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xf
+#define GCVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x10
+#define GCVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x11
+#define GCVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x12
+#define GCVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x13
+#define GCVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x14
+#define GCVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x15
+#define GCVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x16
+#define GCVM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x17
+#define GCVM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x18
+#define GCVM_CONTEXT2_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT2_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x00000006L
+#define GCVM_CONTEXT2_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x00000078L
+#define GCVM_CONTEXT2_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000080L
+#define GCVM_CONTEXT2_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000100L
+#define GCVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000200L
+#define GCVM_CONTEXT2_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000400L
+#define GCVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00000800L
+#define GCVM_CONTEXT2_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00001000L
+#define GCVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00002000L
+#define GCVM_CONTEXT2_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00004000L
+#define GCVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00008000L
+#define GCVM_CONTEXT2_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00010000L
+#define GCVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00020000L
+#define GCVM_CONTEXT2_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00040000L
+#define GCVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00080000L
+#define GCVM_CONTEXT2_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00100000L
+#define GCVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00200000L
+#define GCVM_CONTEXT2_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00400000L
+#define GCVM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00800000L
+#define GCVM_CONTEXT2_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x01000000L
+//GCVM_CONTEXT3_CNTL
+#define GCVM_CONTEXT3_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x3
+#define GCVM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x7
+#define GCVM_CONTEXT3_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x8
+#define GCVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x9
+#define GCVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xa
+#define GCVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xb
+#define GCVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xc
+#define GCVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xd
+#define GCVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xe
+#define GCVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xf
+#define GCVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x10
+#define GCVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x11
+#define GCVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x12
+#define GCVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x13
+#define GCVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x14
+#define GCVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x15
+#define GCVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x16
+#define GCVM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x17
+#define GCVM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x18
+#define GCVM_CONTEXT3_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT3_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x00000006L
+#define GCVM_CONTEXT3_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x00000078L
+#define GCVM_CONTEXT3_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000080L
+#define GCVM_CONTEXT3_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000100L
+#define GCVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000200L
+#define GCVM_CONTEXT3_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000400L
+#define GCVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00000800L
+#define GCVM_CONTEXT3_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00001000L
+#define GCVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00002000L
+#define GCVM_CONTEXT3_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00004000L
+#define GCVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00008000L
+#define GCVM_CONTEXT3_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00010000L
+#define GCVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00020000L
+#define GCVM_CONTEXT3_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00040000L
+#define GCVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00080000L
+#define GCVM_CONTEXT3_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00100000L
+#define GCVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00200000L
+#define GCVM_CONTEXT3_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00400000L
+#define GCVM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00800000L
+#define GCVM_CONTEXT3_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x01000000L
+//GCVM_CONTEXT4_CNTL
+#define GCVM_CONTEXT4_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x3
+#define GCVM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x7
+#define GCVM_CONTEXT4_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x8
+#define GCVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x9
+#define GCVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xa
+#define GCVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xb
+#define GCVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xc
+#define GCVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xd
+#define GCVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xe
+#define GCVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xf
+#define GCVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x10
+#define GCVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x11
+#define GCVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x12
+#define GCVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x13
+#define GCVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x14
+#define GCVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x15
+#define GCVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x16
+#define GCVM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x17
+#define GCVM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x18
+#define GCVM_CONTEXT4_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT4_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x00000006L
+#define GCVM_CONTEXT4_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x00000078L
+#define GCVM_CONTEXT4_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000080L
+#define GCVM_CONTEXT4_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000100L
+#define GCVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000200L
+#define GCVM_CONTEXT4_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000400L
+#define GCVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00000800L
+#define GCVM_CONTEXT4_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00001000L
+#define GCVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00002000L
+#define GCVM_CONTEXT4_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00004000L
+#define GCVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00008000L
+#define GCVM_CONTEXT4_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00010000L
+#define GCVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00020000L
+#define GCVM_CONTEXT4_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00040000L
+#define GCVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00080000L
+#define GCVM_CONTEXT4_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00100000L
+#define GCVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00200000L
+#define GCVM_CONTEXT4_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00400000L
+#define GCVM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00800000L
+#define GCVM_CONTEXT4_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x01000000L
+//GCVM_CONTEXT5_CNTL
+#define GCVM_CONTEXT5_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x3
+#define GCVM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x7
+#define GCVM_CONTEXT5_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x8
+#define GCVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x9
+#define GCVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xa
+#define GCVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xb
+#define GCVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xc
+#define GCVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xd
+#define GCVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xe
+#define GCVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xf
+#define GCVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x10
+#define GCVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x11
+#define GCVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x12
+#define GCVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x13
+#define GCVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x14
+#define GCVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x15
+#define GCVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x16
+#define GCVM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x17
+#define GCVM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x18
+#define GCVM_CONTEXT5_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT5_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x00000006L
+#define GCVM_CONTEXT5_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x00000078L
+#define GCVM_CONTEXT5_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000080L
+#define GCVM_CONTEXT5_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000100L
+#define GCVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000200L
+#define GCVM_CONTEXT5_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000400L
+#define GCVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00000800L
+#define GCVM_CONTEXT5_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00001000L
+#define GCVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00002000L
+#define GCVM_CONTEXT5_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00004000L
+#define GCVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00008000L
+#define GCVM_CONTEXT5_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00010000L
+#define GCVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00020000L
+#define GCVM_CONTEXT5_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00040000L
+#define GCVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00080000L
+#define GCVM_CONTEXT5_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00100000L
+#define GCVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00200000L
+#define GCVM_CONTEXT5_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00400000L
+#define GCVM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00800000L
+#define GCVM_CONTEXT5_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x01000000L
+//GCVM_CONTEXT6_CNTL
+#define GCVM_CONTEXT6_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x3
+#define GCVM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x7
+#define GCVM_CONTEXT6_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x8
+#define GCVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x9
+#define GCVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xa
+#define GCVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xb
+#define GCVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xc
+#define GCVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xd
+#define GCVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xe
+#define GCVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xf
+#define GCVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x10
+#define GCVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x11
+#define GCVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x12
+#define GCVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x13
+#define GCVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x14
+#define GCVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x15
+#define GCVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x16
+#define GCVM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x17
+#define GCVM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x18
+#define GCVM_CONTEXT6_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT6_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x00000006L
+#define GCVM_CONTEXT6_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x00000078L
+#define GCVM_CONTEXT6_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000080L
+#define GCVM_CONTEXT6_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000100L
+#define GCVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000200L
+#define GCVM_CONTEXT6_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000400L
+#define GCVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00000800L
+#define GCVM_CONTEXT6_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00001000L
+#define GCVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00002000L
+#define GCVM_CONTEXT6_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00004000L
+#define GCVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00008000L
+#define GCVM_CONTEXT6_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00010000L
+#define GCVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00020000L
+#define GCVM_CONTEXT6_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00040000L
+#define GCVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00080000L
+#define GCVM_CONTEXT6_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00100000L
+#define GCVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00200000L
+#define GCVM_CONTEXT6_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00400000L
+#define GCVM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00800000L
+#define GCVM_CONTEXT6_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x01000000L
+//GCVM_CONTEXT7_CNTL
+#define GCVM_CONTEXT7_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x3
+#define GCVM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x7
+#define GCVM_CONTEXT7_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x8
+#define GCVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x9
+#define GCVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xa
+#define GCVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xb
+#define GCVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xc
+#define GCVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xd
+#define GCVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xe
+#define GCVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xf
+#define GCVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x10
+#define GCVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x11
+#define GCVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x12
+#define GCVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x13
+#define GCVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x14
+#define GCVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x15
+#define GCVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x16
+#define GCVM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x17
+#define GCVM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x18
+#define GCVM_CONTEXT7_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT7_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x00000006L
+#define GCVM_CONTEXT7_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x00000078L
+#define GCVM_CONTEXT7_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000080L
+#define GCVM_CONTEXT7_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000100L
+#define GCVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000200L
+#define GCVM_CONTEXT7_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000400L
+#define GCVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00000800L
+#define GCVM_CONTEXT7_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00001000L
+#define GCVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00002000L
+#define GCVM_CONTEXT7_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00004000L
+#define GCVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00008000L
+#define GCVM_CONTEXT7_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00010000L
+#define GCVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00020000L
+#define GCVM_CONTEXT7_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00040000L
+#define GCVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00080000L
+#define GCVM_CONTEXT7_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00100000L
+#define GCVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00200000L
+#define GCVM_CONTEXT7_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00400000L
+#define GCVM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00800000L
+#define GCVM_CONTEXT7_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x01000000L
+//GCVM_CONTEXT8_CNTL
+#define GCVM_CONTEXT8_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x3
+#define GCVM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x7
+#define GCVM_CONTEXT8_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x8
+#define GCVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x9
+#define GCVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xa
+#define GCVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xb
+#define GCVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xc
+#define GCVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xd
+#define GCVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xe
+#define GCVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xf
+#define GCVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x10
+#define GCVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x11
+#define GCVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x12
+#define GCVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x13
+#define GCVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x14
+#define GCVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x15
+#define GCVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x16
+#define GCVM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x17
+#define GCVM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x18
+#define GCVM_CONTEXT8_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT8_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x00000006L
+#define GCVM_CONTEXT8_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x00000078L
+#define GCVM_CONTEXT8_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000080L
+#define GCVM_CONTEXT8_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000100L
+#define GCVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000200L
+#define GCVM_CONTEXT8_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000400L
+#define GCVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00000800L
+#define GCVM_CONTEXT8_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00001000L
+#define GCVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00002000L
+#define GCVM_CONTEXT8_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00004000L
+#define GCVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00008000L
+#define GCVM_CONTEXT8_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00010000L
+#define GCVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00020000L
+#define GCVM_CONTEXT8_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00040000L
+#define GCVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00080000L
+#define GCVM_CONTEXT8_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00100000L
+#define GCVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00200000L
+#define GCVM_CONTEXT8_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00400000L
+#define GCVM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00800000L
+#define GCVM_CONTEXT8_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x01000000L
+//GCVM_CONTEXT9_CNTL
+#define GCVM_CONTEXT9_CNTL__ENABLE_CONTEXT__SHIFT                                                             0x0
+#define GCVM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                           0x1
+#define GCVM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                      0x3
+#define GCVM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                     0x7
+#define GCVM_CONTEXT9_CNTL__RETRY_OTHER_FAULT__SHIFT                                                          0x8
+#define GCVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x9
+#define GCVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xa
+#define GCVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                               0xb
+#define GCVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                 0xc
+#define GCVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0xd
+#define GCVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0xe
+#define GCVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xf
+#define GCVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x10
+#define GCVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                     0x11
+#define GCVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                       0x12
+#define GCVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x13
+#define GCVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x14
+#define GCVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x15
+#define GCVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x16
+#define GCVM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x17
+#define GCVM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x18
+#define GCVM_CONTEXT9_CNTL__ENABLE_CONTEXT_MASK                                                               0x00000001L
+#define GCVM_CONTEXT9_CNTL__PAGE_TABLE_DEPTH_MASK                                                             0x00000006L
+#define GCVM_CONTEXT9_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                        0x00000078L
+#define GCVM_CONTEXT9_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                       0x00000080L
+#define GCVM_CONTEXT9_CNTL__RETRY_OTHER_FAULT_MASK                                                            0x00000100L
+#define GCVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00000200L
+#define GCVM_CONTEXT9_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00000400L
+#define GCVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                 0x00000800L
+#define GCVM_CONTEXT9_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                   0x00001000L
+#define GCVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00002000L
+#define GCVM_CONTEXT9_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00004000L
+#define GCVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00008000L
+#define GCVM_CONTEXT9_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00010000L
+#define GCVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                       0x00020000L
+#define GCVM_CONTEXT9_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                         0x00040000L
+#define GCVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00080000L
+#define GCVM_CONTEXT9_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00100000L
+#define GCVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00200000L
+#define GCVM_CONTEXT9_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x00400000L
+#define GCVM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00800000L
+#define GCVM_CONTEXT9_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x01000000L
+//GCVM_CONTEXT10_CNTL
+#define GCVM_CONTEXT10_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x3
+#define GCVM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x7
+#define GCVM_CONTEXT10_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x8
+#define GCVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x9
+#define GCVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xa
+#define GCVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xb
+#define GCVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xc
+#define GCVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xd
+#define GCVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xe
+#define GCVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xf
+#define GCVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x10
+#define GCVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x11
+#define GCVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x12
+#define GCVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x13
+#define GCVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x14
+#define GCVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x15
+#define GCVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x16
+#define GCVM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x17
+#define GCVM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x18
+#define GCVM_CONTEXT10_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT10_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x00000006L
+#define GCVM_CONTEXT10_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x00000078L
+#define GCVM_CONTEXT10_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000080L
+#define GCVM_CONTEXT10_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000100L
+#define GCVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000200L
+#define GCVM_CONTEXT10_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000400L
+#define GCVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00000800L
+#define GCVM_CONTEXT10_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00001000L
+#define GCVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00002000L
+#define GCVM_CONTEXT10_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00004000L
+#define GCVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00008000L
+#define GCVM_CONTEXT10_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00010000L
+#define GCVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00020000L
+#define GCVM_CONTEXT10_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00040000L
+#define GCVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00080000L
+#define GCVM_CONTEXT10_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00100000L
+#define GCVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00200000L
+#define GCVM_CONTEXT10_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00400000L
+#define GCVM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00800000L
+#define GCVM_CONTEXT10_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x01000000L
+//GCVM_CONTEXT11_CNTL
+#define GCVM_CONTEXT11_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x3
+#define GCVM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x7
+#define GCVM_CONTEXT11_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x8
+#define GCVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x9
+#define GCVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xa
+#define GCVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xb
+#define GCVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xc
+#define GCVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xd
+#define GCVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xe
+#define GCVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xf
+#define GCVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x10
+#define GCVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x11
+#define GCVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x12
+#define GCVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x13
+#define GCVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x14
+#define GCVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x15
+#define GCVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x16
+#define GCVM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x17
+#define GCVM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x18
+#define GCVM_CONTEXT11_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT11_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x00000006L
+#define GCVM_CONTEXT11_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x00000078L
+#define GCVM_CONTEXT11_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000080L
+#define GCVM_CONTEXT11_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000100L
+#define GCVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000200L
+#define GCVM_CONTEXT11_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000400L
+#define GCVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00000800L
+#define GCVM_CONTEXT11_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00001000L
+#define GCVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00002000L
+#define GCVM_CONTEXT11_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00004000L
+#define GCVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00008000L
+#define GCVM_CONTEXT11_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00010000L
+#define GCVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00020000L
+#define GCVM_CONTEXT11_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00040000L
+#define GCVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00080000L
+#define GCVM_CONTEXT11_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00100000L
+#define GCVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00200000L
+#define GCVM_CONTEXT11_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00400000L
+#define GCVM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00800000L
+#define GCVM_CONTEXT11_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x01000000L
+//GCVM_CONTEXT12_CNTL
+#define GCVM_CONTEXT12_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x3
+#define GCVM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x7
+#define GCVM_CONTEXT12_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x8
+#define GCVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x9
+#define GCVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xa
+#define GCVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xb
+#define GCVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xc
+#define GCVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xd
+#define GCVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xe
+#define GCVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xf
+#define GCVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x10
+#define GCVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x11
+#define GCVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x12
+#define GCVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x13
+#define GCVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x14
+#define GCVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x15
+#define GCVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x16
+#define GCVM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x17
+#define GCVM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x18
+#define GCVM_CONTEXT12_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT12_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x00000006L
+#define GCVM_CONTEXT12_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x00000078L
+#define GCVM_CONTEXT12_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000080L
+#define GCVM_CONTEXT12_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000100L
+#define GCVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000200L
+#define GCVM_CONTEXT12_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000400L
+#define GCVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00000800L
+#define GCVM_CONTEXT12_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00001000L
+#define GCVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00002000L
+#define GCVM_CONTEXT12_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00004000L
+#define GCVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00008000L
+#define GCVM_CONTEXT12_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00010000L
+#define GCVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00020000L
+#define GCVM_CONTEXT12_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00040000L
+#define GCVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00080000L
+#define GCVM_CONTEXT12_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00100000L
+#define GCVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00200000L
+#define GCVM_CONTEXT12_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00400000L
+#define GCVM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00800000L
+#define GCVM_CONTEXT12_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x01000000L
+//GCVM_CONTEXT13_CNTL
+#define GCVM_CONTEXT13_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x3
+#define GCVM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x7
+#define GCVM_CONTEXT13_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x8
+#define GCVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x9
+#define GCVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xa
+#define GCVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xb
+#define GCVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xc
+#define GCVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xd
+#define GCVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xe
+#define GCVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xf
+#define GCVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x10
+#define GCVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x11
+#define GCVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x12
+#define GCVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x13
+#define GCVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x14
+#define GCVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x15
+#define GCVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x16
+#define GCVM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x17
+#define GCVM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x18
+#define GCVM_CONTEXT13_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT13_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x00000006L
+#define GCVM_CONTEXT13_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x00000078L
+#define GCVM_CONTEXT13_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000080L
+#define GCVM_CONTEXT13_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000100L
+#define GCVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000200L
+#define GCVM_CONTEXT13_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000400L
+#define GCVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00000800L
+#define GCVM_CONTEXT13_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00001000L
+#define GCVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00002000L
+#define GCVM_CONTEXT13_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00004000L
+#define GCVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00008000L
+#define GCVM_CONTEXT13_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00010000L
+#define GCVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00020000L
+#define GCVM_CONTEXT13_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00040000L
+#define GCVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00080000L
+#define GCVM_CONTEXT13_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00100000L
+#define GCVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00200000L
+#define GCVM_CONTEXT13_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00400000L
+#define GCVM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00800000L
+#define GCVM_CONTEXT13_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x01000000L
+//GCVM_CONTEXT14_CNTL
+#define GCVM_CONTEXT14_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x3
+#define GCVM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x7
+#define GCVM_CONTEXT14_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x8
+#define GCVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x9
+#define GCVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xa
+#define GCVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xb
+#define GCVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xc
+#define GCVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xd
+#define GCVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xe
+#define GCVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xf
+#define GCVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x10
+#define GCVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x11
+#define GCVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x12
+#define GCVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x13
+#define GCVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x14
+#define GCVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x15
+#define GCVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x16
+#define GCVM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x17
+#define GCVM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x18
+#define GCVM_CONTEXT14_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT14_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x00000006L
+#define GCVM_CONTEXT14_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x00000078L
+#define GCVM_CONTEXT14_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000080L
+#define GCVM_CONTEXT14_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000100L
+#define GCVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000200L
+#define GCVM_CONTEXT14_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000400L
+#define GCVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00000800L
+#define GCVM_CONTEXT14_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00001000L
+#define GCVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00002000L
+#define GCVM_CONTEXT14_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00004000L
+#define GCVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00008000L
+#define GCVM_CONTEXT14_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00010000L
+#define GCVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00020000L
+#define GCVM_CONTEXT14_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00040000L
+#define GCVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00080000L
+#define GCVM_CONTEXT14_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00100000L
+#define GCVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00200000L
+#define GCVM_CONTEXT14_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00400000L
+#define GCVM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00800000L
+#define GCVM_CONTEXT14_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x01000000L
+//GCVM_CONTEXT15_CNTL
+#define GCVM_CONTEXT15_CNTL__ENABLE_CONTEXT__SHIFT                                                            0x0
+#define GCVM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH__SHIFT                                                          0x1
+#define GCVM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT                                                     0x3
+#define GCVM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT                                    0x7
+#define GCVM_CONTEXT15_CNTL__RETRY_OTHER_FAULT__SHIFT                                                         0x8
+#define GCVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x9
+#define GCVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0xa
+#define GCVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                              0xb
+#define GCVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                0xc
+#define GCVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0xd
+#define GCVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0xe
+#define GCVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0xf
+#define GCVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x10
+#define GCVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                    0x11
+#define GCVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                      0x12
+#define GCVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                   0x13
+#define GCVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                     0x14
+#define GCVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                 0x15
+#define GCVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                   0x16
+#define GCVM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT__SHIFT                                  0x17
+#define GCVM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT__SHIFT                                    0x18
+#define GCVM_CONTEXT15_CNTL__ENABLE_CONTEXT_MASK                                                              0x00000001L
+#define GCVM_CONTEXT15_CNTL__PAGE_TABLE_DEPTH_MASK                                                            0x00000006L
+#define GCVM_CONTEXT15_CNTL__PAGE_TABLE_BLOCK_SIZE_MASK                                                       0x00000078L
+#define GCVM_CONTEXT15_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT_MASK                                      0x00000080L
+#define GCVM_CONTEXT15_CNTL__RETRY_OTHER_FAULT_MASK                                                           0x00000100L
+#define GCVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00000200L
+#define GCVM_CONTEXT15_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00000400L
+#define GCVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                0x00000800L
+#define GCVM_CONTEXT15_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                  0x00001000L
+#define GCVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00002000L
+#define GCVM_CONTEXT15_CNTL__PDE0_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00004000L
+#define GCVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00008000L
+#define GCVM_CONTEXT15_CNTL__VALID_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00010000L
+#define GCVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                      0x00020000L
+#define GCVM_CONTEXT15_CNTL__READ_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                        0x00040000L
+#define GCVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                     0x00080000L
+#define GCVM_CONTEXT15_CNTL__WRITE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                       0x00100000L
+#define GCVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                   0x00200000L
+#define GCVM_CONTEXT15_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                     0x00400000L
+#define GCVM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK                                    0x00800000L
+#define GCVM_CONTEXT15_CNTL__SECURE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK                                      0x01000000L
+//GCVM_CONTEXTS_DISABLE
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0__SHIFT                                                       0x0
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1__SHIFT                                                       0x1
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2__SHIFT                                                       0x2
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3__SHIFT                                                       0x3
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4__SHIFT                                                       0x4
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5__SHIFT                                                       0x5
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6__SHIFT                                                       0x6
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7__SHIFT                                                       0x7
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8__SHIFT                                                       0x8
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9__SHIFT                                                       0x9
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10__SHIFT                                                      0xa
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11__SHIFT                                                      0xb
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12__SHIFT                                                      0xc
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13__SHIFT                                                      0xd
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14__SHIFT                                                      0xe
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15__SHIFT                                                      0xf
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_0_MASK                                                         0x00000001L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_1_MASK                                                         0x00000002L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_2_MASK                                                         0x00000004L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_3_MASK                                                         0x00000008L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_4_MASK                                                         0x00000010L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_5_MASK                                                         0x00000020L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_6_MASK                                                         0x00000040L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_7_MASK                                                         0x00000080L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_8_MASK                                                         0x00000100L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_9_MASK                                                         0x00000200L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_10_MASK                                                        0x00000400L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_11_MASK                                                        0x00000800L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_12_MASK                                                        0x00001000L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_13_MASK                                                        0x00002000L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_14_MASK                                                        0x00004000L
+#define GCVM_CONTEXTS_DISABLE__DISABLE_CONTEXT_15_MASK                                                        0x00008000L
+//GCVM_INVALIDATE_ENG0_SEM
+#define GCVM_INVALIDATE_ENG0_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG0_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG1_SEM
+#define GCVM_INVALIDATE_ENG1_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG1_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG2_SEM
+#define GCVM_INVALIDATE_ENG2_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG2_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG3_SEM
+#define GCVM_INVALIDATE_ENG3_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG3_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG4_SEM
+#define GCVM_INVALIDATE_ENG4_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG4_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG5_SEM
+#define GCVM_INVALIDATE_ENG5_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG5_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG6_SEM
+#define GCVM_INVALIDATE_ENG6_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG6_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG7_SEM
+#define GCVM_INVALIDATE_ENG7_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG7_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG8_SEM
+#define GCVM_INVALIDATE_ENG8_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG8_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG9_SEM
+#define GCVM_INVALIDATE_ENG9_SEM__SEMAPHORE__SHIFT                                                            0x0
+#define GCVM_INVALIDATE_ENG9_SEM__SEMAPHORE_MASK                                                              0x00000001L
+//GCVM_INVALIDATE_ENG10_SEM
+#define GCVM_INVALIDATE_ENG10_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG10_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG11_SEM
+#define GCVM_INVALIDATE_ENG11_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG11_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG12_SEM
+#define GCVM_INVALIDATE_ENG12_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG12_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG13_SEM
+#define GCVM_INVALIDATE_ENG13_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG13_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG14_SEM
+#define GCVM_INVALIDATE_ENG14_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG14_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG15_SEM
+#define GCVM_INVALIDATE_ENG15_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG15_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG16_SEM
+#define GCVM_INVALIDATE_ENG16_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG16_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG17_SEM
+#define GCVM_INVALIDATE_ENG17_SEM__SEMAPHORE__SHIFT                                                           0x0
+#define GCVM_INVALIDATE_ENG17_SEM__SEMAPHORE_MASK                                                             0x00000001L
+//GCVM_INVALIDATE_ENG0_REQ
+#define GCVM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG0_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG0_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG0_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG0_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG0_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG0_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG0_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+//GCVM_INVALIDATE_ENG1_REQ
+#define GCVM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG1_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG1_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG1_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG1_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG1_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG1_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG1_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+//GCVM_INVALIDATE_ENG2_REQ
+#define GCVM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG2_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG2_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG2_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG2_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG2_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG2_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG2_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+//GCVM_INVALIDATE_ENG3_REQ
+#define GCVM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG3_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG3_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG3_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG3_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG3_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG3_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG3_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+//GCVM_INVALIDATE_ENG4_REQ
+#define GCVM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG4_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG4_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG4_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG4_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG4_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG4_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG4_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+//GCVM_INVALIDATE_ENG5_REQ
+#define GCVM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG5_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG5_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG5_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG5_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG5_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG5_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG5_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+//GCVM_INVALIDATE_ENG6_REQ
+#define GCVM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG6_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG6_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG6_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG6_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG6_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG6_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG6_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+//GCVM_INVALIDATE_ENG7_REQ
+#define GCVM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG7_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG7_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG7_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG7_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG7_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG7_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG7_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+//GCVM_INVALIDATE_ENG8_REQ
+#define GCVM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG8_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG8_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG8_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG8_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG8_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG8_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG8_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+//GCVM_INVALIDATE_ENG9_REQ
+#define GCVM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG9_REQ__FLUSH_TYPE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES__SHIFT                                                   0x13
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0__SHIFT                                                   0x14
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1__SHIFT                                                   0x15
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2__SHIFT                                                   0x16
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES__SHIFT                                                   0x17
+#define GCVM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                   0x18
+#define GCVM_INVALIDATE_ENG9_REQ__LOG_REQUEST__SHIFT                                                          0x19
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                             0x1a
+#define GCVM_INVALIDATE_ENG9_REQ__PER_VMID_INVALIDATE_REQ_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG9_REQ__FLUSH_TYPE_MASK                                                             0x00070000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PTES_MASK                                                     0x00080000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE0_MASK                                                     0x00100000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE1_MASK                                                     0x00200000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L2_PDE2_MASK                                                     0x00400000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_L1_PTES_MASK                                                     0x00800000L
+#define GCVM_INVALIDATE_ENG9_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                     0x01000000L
+#define GCVM_INVALIDATE_ENG9_REQ__LOG_REQUEST_MASK                                                            0x02000000L
+#define GCVM_INVALIDATE_ENG9_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                               0x04000000L
+//GCVM_INVALIDATE_ENG10_REQ
+#define GCVM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG10_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG10_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG10_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG10_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG10_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG10_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG10_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+//GCVM_INVALIDATE_ENG11_REQ
+#define GCVM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG11_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG11_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG11_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG11_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG11_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG11_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG11_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+//GCVM_INVALIDATE_ENG12_REQ
+#define GCVM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG12_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG12_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG12_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG12_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG12_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG12_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG12_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+//GCVM_INVALIDATE_ENG13_REQ
+#define GCVM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG13_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG13_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG13_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG13_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG13_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG13_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG13_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+//GCVM_INVALIDATE_ENG14_REQ
+#define GCVM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG14_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG14_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG14_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG14_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG14_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG14_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG14_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+//GCVM_INVALIDATE_ENG15_REQ
+#define GCVM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG15_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG15_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG15_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG15_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG15_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG15_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG15_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+//GCVM_INVALIDATE_ENG16_REQ
+#define GCVM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG16_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG16_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG16_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG16_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG16_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG16_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG16_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+//GCVM_INVALIDATE_ENG17_REQ
+#define GCVM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG17_REQ__FLUSH_TYPE__SHIFT                                                          0x10
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES__SHIFT                                                  0x13
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0__SHIFT                                                  0x14
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1__SHIFT                                                  0x15
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2__SHIFT                                                  0x16
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES__SHIFT                                                  0x17
+#define GCVM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR__SHIFT                                  0x18
+#define GCVM_INVALIDATE_ENG17_REQ__LOG_REQUEST__SHIFT                                                         0x19
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_4K_PAGES_ONLY__SHIFT                                            0x1a
+#define GCVM_INVALIDATE_ENG17_REQ__PER_VMID_INVALIDATE_REQ_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG17_REQ__FLUSH_TYPE_MASK                                                            0x00070000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PTES_MASK                                                    0x00080000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE0_MASK                                                    0x00100000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE1_MASK                                                    0x00200000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L2_PDE2_MASK                                                    0x00400000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_L1_PTES_MASK                                                    0x00800000L
+#define GCVM_INVALIDATE_ENG17_REQ__CLEAR_PROTECTION_FAULT_STATUS_ADDR_MASK                                    0x01000000L
+#define GCVM_INVALIDATE_ENG17_REQ__LOG_REQUEST_MASK                                                           0x02000000L
+#define GCVM_INVALIDATE_ENG17_REQ__INVALIDATE_4K_PAGES_ONLY_MASK                                              0x04000000L
+//GCVM_INVALIDATE_ENG0_ACK
+#define GCVM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG0_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG0_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG0_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG1_ACK
+#define GCVM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG1_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG1_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG1_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG2_ACK
+#define GCVM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG2_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG2_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG2_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG3_ACK
+#define GCVM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG3_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG3_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG3_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG4_ACK
+#define GCVM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG4_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG4_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG4_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG5_ACK
+#define GCVM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG5_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG5_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG5_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG6_ACK
+#define GCVM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG6_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG6_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG6_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG7_ACK
+#define GCVM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG7_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG7_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG7_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG8_ACK
+#define GCVM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG8_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG8_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG8_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG9_ACK
+#define GCVM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                              0x0
+#define GCVM_INVALIDATE_ENG9_ACK__SEMAPHORE__SHIFT                                                            0x10
+#define GCVM_INVALIDATE_ENG9_ACK__PER_VMID_INVALIDATE_ACK_MASK                                                0x0000FFFFL
+#define GCVM_INVALIDATE_ENG9_ACK__SEMAPHORE_MASK                                                              0x00010000L
+//GCVM_INVALIDATE_ENG10_ACK
+#define GCVM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG10_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG10_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG10_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG11_ACK
+#define GCVM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG11_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG11_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG11_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG12_ACK
+#define GCVM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG12_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG12_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG12_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG13_ACK
+#define GCVM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG13_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG13_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG13_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG14_ACK
+#define GCVM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG14_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG14_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG14_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG15_ACK
+#define GCVM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG15_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG15_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG15_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG16_ACK
+#define GCVM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG16_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG16_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG16_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG17_ACK
+#define GCVM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK__SHIFT                                             0x0
+#define GCVM_INVALIDATE_ENG17_ACK__SEMAPHORE__SHIFT                                                           0x10
+#define GCVM_INVALIDATE_ENG17_ACK__PER_VMID_INVALIDATE_ACK_MASK                                               0x0000FFFFL
+#define GCVM_INVALIDATE_ENG17_ACK__SEMAPHORE_MASK                                                             0x00010000L
+//GCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                 0x0
+#define GCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                   0x0000001FL
+//GCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG1_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                 0x0
+#define GCVM_INVALIDATE_ENG1_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                   0x0000001FL
+//GCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG2_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                 0x0
+#define GCVM_INVALIDATE_ENG2_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                   0x0000001FL
+//GCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG3_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                 0x0
+#define GCVM_INVALIDATE_ENG3_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                   0x0000001FL
+//GCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG4_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                 0x0
+#define GCVM_INVALIDATE_ENG4_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                   0x0000001FL
+//GCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG5_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                 0x0
+#define GCVM_INVALIDATE_ENG5_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                   0x0000001FL
+//GCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG6_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                 0x0
+#define GCVM_INVALIDATE_ENG6_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                   0x0000001FL
+//GCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG7_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                 0x0
+#define GCVM_INVALIDATE_ENG7_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                   0x0000001FL
+//GCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG8_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                 0x0
+#define GCVM_INVALIDATE_ENG8_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                   0x0000001FL
+//GCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT__SHIFT                                                    0x0
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                                0x1
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__S_BIT_MASK                                                      0x00000001L
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                  0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG9_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                 0x0
+#define GCVM_INVALIDATE_ENG9_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                   0x0000001FL
+//GCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG10_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG10_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                  0x0000001FL
+//GCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG11_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG11_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                  0x0000001FL
+//GCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG12_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG12_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                  0x0000001FL
+//GCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG13_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG13_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                  0x0000001FL
+//GCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG14_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG14_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                  0x0000001FL
+//GCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG15_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG15_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                  0x0000001FL
+//GCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG16_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG16_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                  0x0000001FL
+//GCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT__SHIFT                                                   0x0
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31__SHIFT                               0x1
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__S_BIT_MASK                                                     0x00000001L
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_LO32__LOGI_PAGE_ADDR_RANGE_LO31_MASK                                 0xFFFFFFFEL
+//GCVM_INVALIDATE_ENG17_ADDR_RANGE_HI32
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5__SHIFT                                0x0
+#define GCVM_INVALIDATE_ENG17_ADDR_RANGE_HI32__LOGI_PAGE_ADDR_RANGE_HI5_MASK                                  0x0000001FL
+//GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                             0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32
+#define GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_LO32__PAGE_DIRECTORY_ENTRY_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32
+#define GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32__SHIFT                            0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_BASE_ADDR_HI32__PAGE_DIRECTORY_ENTRY_HI32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                              0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                0x0000000FL
+//GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                              0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                0x0000000FL
+//GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                              0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                0x0000000FL
+//GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                              0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                0x0000000FL
+//GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                              0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                0x0000000FL
+//GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                              0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                0x0000000FL
+//GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                              0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                0x0000000FL
+//GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                              0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                0x0000000FL
+//GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                              0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                0x0000000FL
+//GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                             0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                               0xFFFFFFFFL
+//GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                              0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                0x0000000FL
+//GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                             0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                               0x0000000FL
+//GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                             0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                               0x0000000FL
+//GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                             0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                               0x0000000FL
+//GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                             0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                               0x0000000FL
+//GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                             0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                               0x0000000FL
+//GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32
+#define GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                            0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                              0xFFFFFFFFL
+//GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32
+#define GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                             0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_START_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                               0x0000000FL
+//GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define GCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define GCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define GCVM_CONTEXT2_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define GCVM_CONTEXT3_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define GCVM_CONTEXT4_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define GCVM_CONTEXT5_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define GCVM_CONTEXT6_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define GCVM_CONTEXT7_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define GCVM_CONTEXT8_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                               0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                 0xFFFFFFFFL
+//GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                                0x0
+#define GCVM_CONTEXT9_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                  0x0000000FL
+//GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define GCVM_CONTEXT10_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define GCVM_CONTEXT11_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define GCVM_CONTEXT12_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define GCVM_CONTEXT13_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define GCVM_CONTEXT14_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32
+#define GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32__SHIFT                              0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_LO32__LOGICAL_PAGE_NUMBER_LO32_MASK                                0xFFFFFFFFL
+//GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32
+#define GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4__SHIFT                               0x0
+#define GCVM_CONTEXT15_PAGE_TABLE_END_ADDR_HI32__LOGICAL_PAGE_NUMBER_HI4_MASK                                 0x0000000FL
+//GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT                       0x0
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                         0x5
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                         0xa
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                         0x0000001FL
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                           0x000003E0L
+#define GCVM_L2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                           0x0000FC00L
+//GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT0_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT1_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT2_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT3_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT4_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT5_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT6_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT7_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT8_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT              0x0
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT                0x5
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                                0xa
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK                0x0000001FL
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                  0x000003E0L
+#define GCVM_L2_CONTEXT9_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                  0x0000FC00L
+//GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT10_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT11_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT12_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT13_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT14_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+//GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE__SHIFT             0x0
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT               0x5
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT__SHIFT                               0xa
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_SMALLK_FRAGMENT_SIZE_MASK               0x0000001FL
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__L2_CACHE_BIGK_FRAGMENT_SIZE_MASK                 0x000003E0L
+#define GCVM_L2_CONTEXT15_PER_PFVF_PTE_CACHE_FRAGMENT_SIZES__BANK_SELECT_MASK                                 0x0000FC00L
+
+
+// addressBlock: gc_gcvml2perfddec
+//GCVML2_PERFCOUNTER2_0_LO
+#define GCVML2_PERFCOUNTER2_0_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GCVML2_PERFCOUNTER2_0_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GCVML2_PERFCOUNTER2_1_LO
+#define GCVML2_PERFCOUNTER2_1_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GCVML2_PERFCOUNTER2_1_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GCVML2_PERFCOUNTER2_0_HI
+#define GCVML2_PERFCOUNTER2_0_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GCVML2_PERFCOUNTER2_0_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+//GCVML2_PERFCOUNTER2_1_HI
+#define GCVML2_PERFCOUNTER2_1_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GCVML2_PERFCOUNTER2_1_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+
+
+// addressBlock: gc_gcvml2prdec
+//GCMC_VM_L2_PERFCOUNTER_LO
+#define GCMC_VM_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GCMC_VM_L2_PERFCOUNTER_HI
+#define GCMC_VM_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                       0x10
+#define GCMC_VM_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                            0x0000FFFFL
+#define GCMC_VM_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                         0xFFFF0000L
+//GCUTCL2_PERFCOUNTER_LO
+#define GCUTCL2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                               0xFFFFFFFFL
+//GCUTCL2_PERFCOUNTER_HI
+#define GCUTCL2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                          0x10
+#define GCUTCL2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                               0x0000FFFFL
+#define GCUTCL2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                            0xFFFF0000L
+
+
+// addressBlock: gc_gcatcl2perfddec
+//GC_ATC_L2_PERFCOUNTER2_LO
+#define GC_ATC_L2_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                      0x0
+#define GC_ATC_L2_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                        0xFFFFFFFFL
+//GC_ATC_L2_PERFCOUNTER2_HI
+#define GC_ATC_L2_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                      0x0
+#define GC_ATC_L2_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                        0xFFFFFFFFL
+
+
+// addressBlock: gc_gcatcl2pfcntrdec
+//GC_ATC_L2_PERFCOUNTER_LO
+#define GC_ATC_L2_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER_LO__COUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GC_ATC_L2_PERFCOUNTER_HI
+#define GC_ATC_L2_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                        0x10
+#define GC_ATC_L2_PERFCOUNTER_HI__COUNTER_HI_MASK                                                             0x0000FFFFL
+#define GC_ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                          0xFFFF0000L
+
+
+// addressBlock: gc_gcl2tlbprdec
+//GCL2TLB_PERFCOUNTER_LO
+#define GCL2TLB_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                             0x0
+#define GCL2TLB_PERFCOUNTER_LO__COUNTER_LO_MASK                                                               0xFFFFFFFFL
+//GCL2TLB_PERFCOUNTER_HI
+#define GCL2TLB_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                             0x0
+#define GCL2TLB_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                          0x10
+#define GCL2TLB_PERFCOUNTER_HI__COUNTER_HI_MASK                                                               0x0000FFFFL
+#define GCL2TLB_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                            0xFFFF0000L
+
+
+// addressBlock: gc_gcvml2perfsdec
+//GCVML2_PERFCOUNTER2_0_SELECT
+#define GCVML2_PERFCOUNTER2_0_SELECT__PERF_SEL__SHIFT                                                         0x0
+#define GCVML2_PERFCOUNTER2_0_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GCVML2_PERFCOUNTER2_0_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GCVML2_PERFCOUNTER2_0_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GCVML2_PERFCOUNTER2_0_SELECT__PERF_MODE__SHIFT                                                        0x1c
+#define GCVML2_PERFCOUNTER2_0_SELECT__PERF_SEL_MASK                                                           0x000003FFL
+#define GCVML2_PERFCOUNTER2_0_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GCVML2_PERFCOUNTER2_0_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GCVML2_PERFCOUNTER2_0_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GCVML2_PERFCOUNTER2_0_SELECT__PERF_MODE_MASK                                                          0xF0000000L
+//GCVML2_PERFCOUNTER2_1_SELECT
+#define GCVML2_PERFCOUNTER2_1_SELECT__PERF_SEL__SHIFT                                                         0x0
+#define GCVML2_PERFCOUNTER2_1_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GCVML2_PERFCOUNTER2_1_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GCVML2_PERFCOUNTER2_1_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GCVML2_PERFCOUNTER2_1_SELECT__PERF_MODE__SHIFT                                                        0x1c
+#define GCVML2_PERFCOUNTER2_1_SELECT__PERF_SEL_MASK                                                           0x000003FFL
+#define GCVML2_PERFCOUNTER2_1_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GCVML2_PERFCOUNTER2_1_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GCVML2_PERFCOUNTER2_1_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GCVML2_PERFCOUNTER2_1_SELECT__PERF_MODE_MASK                                                          0xF0000000L
+//GCVML2_PERFCOUNTER2_0_SELECT1
+#define GCVML2_PERFCOUNTER2_0_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GCVML2_PERFCOUNTER2_0_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GCVML2_PERFCOUNTER2_0_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GCVML2_PERFCOUNTER2_0_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GCVML2_PERFCOUNTER2_0_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GCVML2_PERFCOUNTER2_0_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GCVML2_PERFCOUNTER2_0_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GCVML2_PERFCOUNTER2_0_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GCVML2_PERFCOUNTER2_1_SELECT1
+#define GCVML2_PERFCOUNTER2_1_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GCVML2_PERFCOUNTER2_1_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GCVML2_PERFCOUNTER2_1_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GCVML2_PERFCOUNTER2_1_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GCVML2_PERFCOUNTER2_1_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GCVML2_PERFCOUNTER2_1_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GCVML2_PERFCOUNTER2_1_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GCVML2_PERFCOUNTER2_1_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GCVML2_PERFCOUNTER2_0_MODE
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_MODE0__SHIFT                                                      0x0
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_MODE1__SHIFT                                                      0x2
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_MODE2__SHIFT                                                      0x4
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_MODE3__SHIFT                                                      0x6
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_VALUE0__SHIFT                                                     0x8
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_VALUE1__SHIFT                                                     0xc
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_VALUE2__SHIFT                                                     0x10
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_VALUE3__SHIFT                                                     0x14
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_MODE0_MASK                                                        0x00000003L
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_MODE1_MASK                                                        0x0000000CL
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_MODE2_MASK                                                        0x00000030L
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_MODE3_MASK                                                        0x000000C0L
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_VALUE0_MASK                                                       0x00000F00L
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_VALUE1_MASK                                                       0x0000F000L
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_VALUE2_MASK                                                       0x000F0000L
+#define GCVML2_PERFCOUNTER2_0_MODE__COMPARE_VALUE3_MASK                                                       0x00F00000L
+//GCVML2_PERFCOUNTER2_1_MODE
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_MODE0__SHIFT                                                      0x0
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_MODE1__SHIFT                                                      0x2
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_MODE2__SHIFT                                                      0x4
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_MODE3__SHIFT                                                      0x6
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_VALUE0__SHIFT                                                     0x8
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_VALUE1__SHIFT                                                     0xc
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_VALUE2__SHIFT                                                     0x10
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_VALUE3__SHIFT                                                     0x14
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_MODE0_MASK                                                        0x00000003L
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_MODE1_MASK                                                        0x0000000CL
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_MODE2_MASK                                                        0x00000030L
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_MODE3_MASK                                                        0x000000C0L
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_VALUE0_MASK                                                       0x00000F00L
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_VALUE1_MASK                                                       0x0000F000L
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_VALUE2_MASK                                                       0x000F0000L
+#define GCVML2_PERFCOUNTER2_1_MODE__COMPARE_VALUE3_MASK                                                       0x00F00000L
+
+
+// addressBlock: gc_gcvml2pldec
+//GCMC_VM_L2_PERFCOUNTER0_CFG
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER1_CFG
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER2_CFG
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER2_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER3_CFG
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER3_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER4_CFG
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER4_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER5_CFG
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER5_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER6_CFG
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER6_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER7_CFG
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL__SHIFT                                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END__SHIFT                                                      0x8
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE__SHIFT                                                         0x18
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__ENABLE__SHIFT                                                            0x1c
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__CLEAR__SHIFT                                                             0x1d
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_MASK                                                            0x000000FFL
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_SEL_END_MASK                                                        0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__PERF_MODE_MASK                                                           0x0F000000L
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__ENABLE_MASK                                                              0x10000000L
+#define GCMC_VM_L2_PERFCOUNTER7_CFG__CLEAR_MASK                                                               0x20000000L
+//GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                          0x0
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                0x8
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                 0x10
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                   0x18
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                    0x19
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                         0x1a
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                            0x0000000FL
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                  0x0000FF00L
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                   0x00FF0000L
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                     0x01000000L
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                      0x02000000L
+#define GCMC_VM_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                           0x04000000L
+//GCUTCL2_PERFCOUNTER0_CFG
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCUTCL2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCUTCL2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCUTCL2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCUTCL2_PERFCOUNTER0_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCUTCL2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCUTCL2_PERFCOUNTER1_CFG
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCUTCL2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCUTCL2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCUTCL2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCUTCL2_PERFCOUNTER1_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCUTCL2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCUTCL2_PERFCOUNTER2_CFG
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCUTCL2_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCUTCL2_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCUTCL2_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCUTCL2_PERFCOUNTER2_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCUTCL2_PERFCOUNTER2_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCUTCL2_PERFCOUNTER3_CFG
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCUTCL2_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCUTCL2_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCUTCL2_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCUTCL2_PERFCOUNTER3_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCUTCL2_PERFCOUNTER3_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCUTCL2_PERFCOUNTER_RSLT_CNTL
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                             0x0
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                   0x8
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                    0x10
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                      0x18
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                       0x19
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                            0x1a
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                               0x0000000FL
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                     0x0000FF00L
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                      0x00FF0000L
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                        0x01000000L
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                         0x02000000L
+#define GCUTCL2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                              0x04000000L
+
+
+// addressBlock: gc_gcatcl2perfsdec
+//GC_ATC_L2_PERFCOUNTER2_SELECT
+#define GC_ATC_L2_PERFCOUNTER2_SELECT__PERF_SEL0__SHIFT                                                       0x0
+#define GC_ATC_L2_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                       0xa
+#define GC_ATC_L2_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                       0x14
+#define GC_ATC_L2_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                      0x18
+#define GC_ATC_L2_PERFCOUNTER2_SELECT__PERF_MODE0__SHIFT                                                      0x1c
+#define GC_ATC_L2_PERFCOUNTER2_SELECT__PERF_SEL0_MASK                                                         0x000003FFL
+#define GC_ATC_L2_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                         0x000FFC00L
+#define GC_ATC_L2_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                         0x00F00000L
+#define GC_ATC_L2_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                        0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER2_SELECT__PERF_MODE0_MASK                                                        0xF0000000L
+//GC_ATC_L2_PERFCOUNTER2_SELECT1
+#define GC_ATC_L2_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                      0x0
+#define GC_ATC_L2_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                      0xa
+#define GC_ATC_L2_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                     0x18
+#define GC_ATC_L2_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                     0x1c
+#define GC_ATC_L2_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                        0x000003FFL
+#define GC_ATC_L2_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                        0x000FFC00L
+#define GC_ATC_L2_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                       0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                       0xF0000000L
+//GC_ATC_L2_PERFCOUNTER2_MODE
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_MODE0__SHIFT                                                     0x0
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_MODE1__SHIFT                                                     0x2
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_MODE2__SHIFT                                                     0x4
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_MODE3__SHIFT                                                     0x6
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_VALUE0__SHIFT                                                    0x8
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_VALUE1__SHIFT                                                    0xc
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_VALUE2__SHIFT                                                    0x10
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_VALUE3__SHIFT                                                    0x14
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_MODE0_MASK                                                       0x00000003L
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_MODE1_MASK                                                       0x0000000CL
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_MODE2_MASK                                                       0x00000030L
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_MODE3_MASK                                                       0x000000C0L
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_VALUE0_MASK                                                      0x00000F00L
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_VALUE1_MASK                                                      0x0000F000L
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_VALUE2_MASK                                                      0x000F0000L
+#define GC_ATC_L2_PERFCOUNTER2_MODE__COMPARE_VALUE3_MASK                                                      0x00F00000L
+
+
+// addressBlock: gc_gcatcl2pfcntldec
+//GC_ATC_L2_PERFCOUNTER0_CFG
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER0_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER0_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER1_CFG
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                           0x0
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                       0x8
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                          0x18
+#define GC_ATC_L2_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                             0x1c
+#define GC_ATC_L2_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                              0x1d
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                             0x000000FFL
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                         0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                            0x0F000000L
+#define GC_ATC_L2_PERFCOUNTER1_CFG__ENABLE_MASK                                                               0x10000000L
+#define GC_ATC_L2_PERFCOUNTER1_CFG__CLEAR_MASK                                                                0x20000000L
+//GC_ATC_L2_PERFCOUNTER_RSLT_CNTL
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                           0x0
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                 0x8
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                  0x10
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                    0x18
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                     0x19
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                          0x1a
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                             0x0000000FL
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                   0x0000FF00L
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                    0x00FF0000L
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                      0x01000000L
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                       0x02000000L
+#define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                            0x04000000L
+
+
+// addressBlock: gc_gcl2tlbpldec
+//GCL2TLB_PERFCOUNTER0_CFG
+#define GCL2TLB_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCL2TLB_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCL2TLB_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCL2TLB_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCL2TLB_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCL2TLB_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCL2TLB_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCL2TLB_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCL2TLB_PERFCOUNTER0_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCL2TLB_PERFCOUNTER0_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCL2TLB_PERFCOUNTER1_CFG
+#define GCL2TLB_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCL2TLB_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCL2TLB_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCL2TLB_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCL2TLB_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCL2TLB_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCL2TLB_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCL2TLB_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCL2TLB_PERFCOUNTER1_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCL2TLB_PERFCOUNTER1_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCL2TLB_PERFCOUNTER2_CFG
+#define GCL2TLB_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCL2TLB_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCL2TLB_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCL2TLB_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCL2TLB_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCL2TLB_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCL2TLB_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCL2TLB_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCL2TLB_PERFCOUNTER2_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCL2TLB_PERFCOUNTER2_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCL2TLB_PERFCOUNTER3_CFG
+#define GCL2TLB_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                             0x0
+#define GCL2TLB_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                         0x8
+#define GCL2TLB_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                            0x18
+#define GCL2TLB_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                               0x1c
+#define GCL2TLB_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                                0x1d
+#define GCL2TLB_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                               0x000000FFL
+#define GCL2TLB_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                           0x0000FF00L
+#define GCL2TLB_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                              0x0F000000L
+#define GCL2TLB_PERFCOUNTER3_CFG__ENABLE_MASK                                                                 0x10000000L
+#define GCL2TLB_PERFCOUNTER3_CFG__CLEAR_MASK                                                                  0x20000000L
+//GCL2TLB_PERFCOUNTER_RSLT_CNTL
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                             0x0
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                   0x8
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                    0x10
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                      0x18
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                       0x19
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                            0x1a
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                               0x0000000FL
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                     0x0000FF00L
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                      0x00FF0000L
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                        0x01000000L
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                         0x02000000L
+#define GCL2TLB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                              0x04000000L
+
+
+// addressBlock: gc_gcvml2pspdec
+//GCUTCL2_TRANSLATION_BYPASS_BY_VMID
+#define GCUTCL2_TRANSLATION_BYPASS_BY_VMID__TRANS_BYPASS_VMIDS__SHIFT                                         0x0
+#define GCUTCL2_TRANSLATION_BYPASS_BY_VMID__GPA_MODE_VMIDS__SHIFT                                             0x10
+#define GCUTCL2_TRANSLATION_BYPASS_BY_VMID__TRANS_BYPASS_VMIDS_MASK                                           0x0000FFFFL
+#define GCUTCL2_TRANSLATION_BYPASS_BY_VMID__GPA_MODE_VMIDS_MASK                                               0xFFFF0000L
+//GCVM_IOMMU_GPU_HOST_TRANSLATION_ENABLE
+#define GCVM_IOMMU_GPU_HOST_TRANSLATION_ENABLE__GPU_HOST_TRANSLATION_ENABLE__SHIFT                            0x0
+#define GCVM_IOMMU_GPU_HOST_TRANSLATION_ENABLE__GPU_HOST_TRANSLATION_ENABLE_MASK                              0x00000001L
+//GCVM_IOMMU_CONTROL_REGISTER
+#define GCVM_IOMMU_CONTROL_REGISTER__IOMMUEN__SHIFT                                                           0x0
+#define GCVM_IOMMU_CONTROL_REGISTER__IOMMUEN_MASK                                                             0x00000001L
+//GCVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER
+#define GCVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN__SHIFT                                0xd
+#define GCVM_IOMMU_PERFORMANCE_OPTIMIZATION_CONTROL_REGISTER__PERFOPTEN_MASK                                  0x00002000L
+//GCVM_IOMMU_MMIO_CNTRL_1
+#define GCVM_IOMMU_MMIO_CNTRL_1__MARC_EN__SHIFT                                                               0x8
+#define GCVM_IOMMU_MMIO_CNTRL_1__MARC_EN_MASK                                                                 0x00000100L
+//GCMC_VM_MARC_BASE_LO_0
+#define GCMC_VM_MARC_BASE_LO_0__MARC_BASE_LO_0__SHIFT                                                         0xc
+#define GCMC_VM_MARC_BASE_LO_0__MARC_BASE_LO_0_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_1
+#define GCMC_VM_MARC_BASE_LO_1__MARC_BASE_LO_1__SHIFT                                                         0xc
+#define GCMC_VM_MARC_BASE_LO_1__MARC_BASE_LO_1_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_2
+#define GCMC_VM_MARC_BASE_LO_2__MARC_BASE_LO_2__SHIFT                                                         0xc
+#define GCMC_VM_MARC_BASE_LO_2__MARC_BASE_LO_2_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_3
+#define GCMC_VM_MARC_BASE_LO_3__MARC_BASE_LO_3__SHIFT                                                         0xc
+#define GCMC_VM_MARC_BASE_LO_3__MARC_BASE_LO_3_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_4
+#define GCMC_VM_MARC_BASE_LO_4__MARC_BASE_LO_4__SHIFT                                                         0xc
+#define GCMC_VM_MARC_BASE_LO_4__MARC_BASE_LO_4_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_5
+#define GCMC_VM_MARC_BASE_LO_5__MARC_BASE_LO_5__SHIFT                                                         0xc
+#define GCMC_VM_MARC_BASE_LO_5__MARC_BASE_LO_5_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_6
+#define GCMC_VM_MARC_BASE_LO_6__MARC_BASE_LO_6__SHIFT                                                         0xc
+#define GCMC_VM_MARC_BASE_LO_6__MARC_BASE_LO_6_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_7
+#define GCMC_VM_MARC_BASE_LO_7__MARC_BASE_LO_7__SHIFT                                                         0xc
+#define GCMC_VM_MARC_BASE_LO_7__MARC_BASE_LO_7_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_8
+#define GCMC_VM_MARC_BASE_LO_8__MARC_BASE_LO_8__SHIFT                                                         0xc
+#define GCMC_VM_MARC_BASE_LO_8__MARC_BASE_LO_8_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_9
+#define GCMC_VM_MARC_BASE_LO_9__MARC_BASE_LO_9__SHIFT                                                         0xc
+#define GCMC_VM_MARC_BASE_LO_9__MARC_BASE_LO_9_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_10
+#define GCMC_VM_MARC_BASE_LO_10__MARC_BASE_LO_10__SHIFT                                                       0xc
+#define GCMC_VM_MARC_BASE_LO_10__MARC_BASE_LO_10_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_11
+#define GCMC_VM_MARC_BASE_LO_11__MARC_BASE_LO_11__SHIFT                                                       0xc
+#define GCMC_VM_MARC_BASE_LO_11__MARC_BASE_LO_11_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_12
+#define GCMC_VM_MARC_BASE_LO_12__MARC_BASE_LO_12__SHIFT                                                       0xc
+#define GCMC_VM_MARC_BASE_LO_12__MARC_BASE_LO_12_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_13
+#define GCMC_VM_MARC_BASE_LO_13__MARC_BASE_LO_13__SHIFT                                                       0xc
+#define GCMC_VM_MARC_BASE_LO_13__MARC_BASE_LO_13_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_14
+#define GCMC_VM_MARC_BASE_LO_14__MARC_BASE_LO_14__SHIFT                                                       0xc
+#define GCMC_VM_MARC_BASE_LO_14__MARC_BASE_LO_14_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_BASE_LO_15
+#define GCMC_VM_MARC_BASE_LO_15__MARC_BASE_LO_15__SHIFT                                                       0xc
+#define GCMC_VM_MARC_BASE_LO_15__MARC_BASE_LO_15_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_BASE_HI_0
+#define GCMC_VM_MARC_BASE_HI_0__MARC_BASE_HI_0__SHIFT                                                         0x0
+#define GCMC_VM_MARC_BASE_HI_0__MARC_BASE_HI_0_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_1
+#define GCMC_VM_MARC_BASE_HI_1__MARC_BASE_HI_1__SHIFT                                                         0x0
+#define GCMC_VM_MARC_BASE_HI_1__MARC_BASE_HI_1_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_2
+#define GCMC_VM_MARC_BASE_HI_2__MARC_BASE_HI_2__SHIFT                                                         0x0
+#define GCMC_VM_MARC_BASE_HI_2__MARC_BASE_HI_2_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_3
+#define GCMC_VM_MARC_BASE_HI_3__MARC_BASE_HI_3__SHIFT                                                         0x0
+#define GCMC_VM_MARC_BASE_HI_3__MARC_BASE_HI_3_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_4
+#define GCMC_VM_MARC_BASE_HI_4__MARC_BASE_HI_4__SHIFT                                                         0x0
+#define GCMC_VM_MARC_BASE_HI_4__MARC_BASE_HI_4_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_5
+#define GCMC_VM_MARC_BASE_HI_5__MARC_BASE_HI_5__SHIFT                                                         0x0
+#define GCMC_VM_MARC_BASE_HI_5__MARC_BASE_HI_5_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_6
+#define GCMC_VM_MARC_BASE_HI_6__MARC_BASE_HI_6__SHIFT                                                         0x0
+#define GCMC_VM_MARC_BASE_HI_6__MARC_BASE_HI_6_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_7
+#define GCMC_VM_MARC_BASE_HI_7__MARC_BASE_HI_7__SHIFT                                                         0x0
+#define GCMC_VM_MARC_BASE_HI_7__MARC_BASE_HI_7_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_8
+#define GCMC_VM_MARC_BASE_HI_8__MARC_BASE_HI_8__SHIFT                                                         0x0
+#define GCMC_VM_MARC_BASE_HI_8__MARC_BASE_HI_8_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_9
+#define GCMC_VM_MARC_BASE_HI_9__MARC_BASE_HI_9__SHIFT                                                         0x0
+#define GCMC_VM_MARC_BASE_HI_9__MARC_BASE_HI_9_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_10
+#define GCMC_VM_MARC_BASE_HI_10__MARC_BASE_HI_10__SHIFT                                                       0x0
+#define GCMC_VM_MARC_BASE_HI_10__MARC_BASE_HI_10_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_11
+#define GCMC_VM_MARC_BASE_HI_11__MARC_BASE_HI_11__SHIFT                                                       0x0
+#define GCMC_VM_MARC_BASE_HI_11__MARC_BASE_HI_11_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_12
+#define GCMC_VM_MARC_BASE_HI_12__MARC_BASE_HI_12__SHIFT                                                       0x0
+#define GCMC_VM_MARC_BASE_HI_12__MARC_BASE_HI_12_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_13
+#define GCMC_VM_MARC_BASE_HI_13__MARC_BASE_HI_13__SHIFT                                                       0x0
+#define GCMC_VM_MARC_BASE_HI_13__MARC_BASE_HI_13_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_14
+#define GCMC_VM_MARC_BASE_HI_14__MARC_BASE_HI_14__SHIFT                                                       0x0
+#define GCMC_VM_MARC_BASE_HI_14__MARC_BASE_HI_14_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_BASE_HI_15
+#define GCMC_VM_MARC_BASE_HI_15__MARC_BASE_HI_15__SHIFT                                                       0x0
+#define GCMC_VM_MARC_BASE_HI_15__MARC_BASE_HI_15_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_LO_0
+#define GCMC_VM_MARC_RELOC_LO_0__MARC_ENABLE_0__SHIFT                                                         0x0
+#define GCMC_VM_MARC_RELOC_LO_0__MARC_READONLY_0__SHIFT                                                       0x1
+#define GCMC_VM_MARC_RELOC_LO_0__MARC_RELOC_LO_0__SHIFT                                                       0xc
+#define GCMC_VM_MARC_RELOC_LO_0__MARC_ENABLE_0_MASK                                                           0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_0__MARC_READONLY_0_MASK                                                         0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_0__MARC_RELOC_LO_0_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_1
+#define GCMC_VM_MARC_RELOC_LO_1__MARC_ENABLE_1__SHIFT                                                         0x0
+#define GCMC_VM_MARC_RELOC_LO_1__MARC_READONLY_1__SHIFT                                                       0x1
+#define GCMC_VM_MARC_RELOC_LO_1__MARC_RELOC_LO_1__SHIFT                                                       0xc
+#define GCMC_VM_MARC_RELOC_LO_1__MARC_ENABLE_1_MASK                                                           0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_1__MARC_READONLY_1_MASK                                                         0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_1__MARC_RELOC_LO_1_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_2
+#define GCMC_VM_MARC_RELOC_LO_2__MARC_ENABLE_2__SHIFT                                                         0x0
+#define GCMC_VM_MARC_RELOC_LO_2__MARC_READONLY_2__SHIFT                                                       0x1
+#define GCMC_VM_MARC_RELOC_LO_2__MARC_RELOC_LO_2__SHIFT                                                       0xc
+#define GCMC_VM_MARC_RELOC_LO_2__MARC_ENABLE_2_MASK                                                           0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_2__MARC_READONLY_2_MASK                                                         0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_2__MARC_RELOC_LO_2_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_3
+#define GCMC_VM_MARC_RELOC_LO_3__MARC_ENABLE_3__SHIFT                                                         0x0
+#define GCMC_VM_MARC_RELOC_LO_3__MARC_READONLY_3__SHIFT                                                       0x1
+#define GCMC_VM_MARC_RELOC_LO_3__MARC_RELOC_LO_3__SHIFT                                                       0xc
+#define GCMC_VM_MARC_RELOC_LO_3__MARC_ENABLE_3_MASK                                                           0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_3__MARC_READONLY_3_MASK                                                         0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_3__MARC_RELOC_LO_3_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_4
+#define GCMC_VM_MARC_RELOC_LO_4__MARC_ENABLE_4__SHIFT                                                         0x0
+#define GCMC_VM_MARC_RELOC_LO_4__MARC_READONLY_4__SHIFT                                                       0x1
+#define GCMC_VM_MARC_RELOC_LO_4__MARC_RELOC_LO_4__SHIFT                                                       0xc
+#define GCMC_VM_MARC_RELOC_LO_4__MARC_ENABLE_4_MASK                                                           0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_4__MARC_READONLY_4_MASK                                                         0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_4__MARC_RELOC_LO_4_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_5
+#define GCMC_VM_MARC_RELOC_LO_5__MARC_ENABLE_5__SHIFT                                                         0x0
+#define GCMC_VM_MARC_RELOC_LO_5__MARC_READONLY_5__SHIFT                                                       0x1
+#define GCMC_VM_MARC_RELOC_LO_5__MARC_RELOC_LO_5__SHIFT                                                       0xc
+#define GCMC_VM_MARC_RELOC_LO_5__MARC_ENABLE_5_MASK                                                           0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_5__MARC_READONLY_5_MASK                                                         0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_5__MARC_RELOC_LO_5_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_6
+#define GCMC_VM_MARC_RELOC_LO_6__MARC_ENABLE_6__SHIFT                                                         0x0
+#define GCMC_VM_MARC_RELOC_LO_6__MARC_READONLY_6__SHIFT                                                       0x1
+#define GCMC_VM_MARC_RELOC_LO_6__MARC_RELOC_LO_6__SHIFT                                                       0xc
+#define GCMC_VM_MARC_RELOC_LO_6__MARC_ENABLE_6_MASK                                                           0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_6__MARC_READONLY_6_MASK                                                         0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_6__MARC_RELOC_LO_6_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_7
+#define GCMC_VM_MARC_RELOC_LO_7__MARC_ENABLE_7__SHIFT                                                         0x0
+#define GCMC_VM_MARC_RELOC_LO_7__MARC_READONLY_7__SHIFT                                                       0x1
+#define GCMC_VM_MARC_RELOC_LO_7__MARC_RELOC_LO_7__SHIFT                                                       0xc
+#define GCMC_VM_MARC_RELOC_LO_7__MARC_ENABLE_7_MASK                                                           0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_7__MARC_READONLY_7_MASK                                                         0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_7__MARC_RELOC_LO_7_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_8
+#define GCMC_VM_MARC_RELOC_LO_8__MARC_ENABLE_8__SHIFT                                                         0x0
+#define GCMC_VM_MARC_RELOC_LO_8__MARC_READONLY_8__SHIFT                                                       0x1
+#define GCMC_VM_MARC_RELOC_LO_8__MARC_RELOC_LO_8__SHIFT                                                       0xc
+#define GCMC_VM_MARC_RELOC_LO_8__MARC_ENABLE_8_MASK                                                           0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_8__MARC_READONLY_8_MASK                                                         0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_8__MARC_RELOC_LO_8_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_9
+#define GCMC_VM_MARC_RELOC_LO_9__MARC_ENABLE_9__SHIFT                                                         0x0
+#define GCMC_VM_MARC_RELOC_LO_9__MARC_READONLY_9__SHIFT                                                       0x1
+#define GCMC_VM_MARC_RELOC_LO_9__MARC_RELOC_LO_9__SHIFT                                                       0xc
+#define GCMC_VM_MARC_RELOC_LO_9__MARC_ENABLE_9_MASK                                                           0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_9__MARC_READONLY_9_MASK                                                         0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_9__MARC_RELOC_LO_9_MASK                                                         0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_10
+#define GCMC_VM_MARC_RELOC_LO_10__MARC_ENABLE_10__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_LO_10__MARC_READONLY_10__SHIFT                                                     0x1
+#define GCMC_VM_MARC_RELOC_LO_10__MARC_RELOC_LO_10__SHIFT                                                     0xc
+#define GCMC_VM_MARC_RELOC_LO_10__MARC_ENABLE_10_MASK                                                         0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_10__MARC_READONLY_10_MASK                                                       0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_10__MARC_RELOC_LO_10_MASK                                                       0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_11
+#define GCMC_VM_MARC_RELOC_LO_11__MARC_ENABLE_11__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_LO_11__MARC_READONLY_11__SHIFT                                                     0x1
+#define GCMC_VM_MARC_RELOC_LO_11__MARC_RELOC_LO_11__SHIFT                                                     0xc
+#define GCMC_VM_MARC_RELOC_LO_11__MARC_ENABLE_11_MASK                                                         0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_11__MARC_READONLY_11_MASK                                                       0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_11__MARC_RELOC_LO_11_MASK                                                       0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_12
+#define GCMC_VM_MARC_RELOC_LO_12__MARC_ENABLE_12__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_LO_12__MARC_READONLY_12__SHIFT                                                     0x1
+#define GCMC_VM_MARC_RELOC_LO_12__MARC_RELOC_LO_12__SHIFT                                                     0xc
+#define GCMC_VM_MARC_RELOC_LO_12__MARC_ENABLE_12_MASK                                                         0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_12__MARC_READONLY_12_MASK                                                       0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_12__MARC_RELOC_LO_12_MASK                                                       0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_13
+#define GCMC_VM_MARC_RELOC_LO_13__MARC_ENABLE_13__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_LO_13__MARC_READONLY_13__SHIFT                                                     0x1
+#define GCMC_VM_MARC_RELOC_LO_13__MARC_RELOC_LO_13__SHIFT                                                     0xc
+#define GCMC_VM_MARC_RELOC_LO_13__MARC_ENABLE_13_MASK                                                         0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_13__MARC_READONLY_13_MASK                                                       0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_13__MARC_RELOC_LO_13_MASK                                                       0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_14
+#define GCMC_VM_MARC_RELOC_LO_14__MARC_ENABLE_14__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_LO_14__MARC_READONLY_14__SHIFT                                                     0x1
+#define GCMC_VM_MARC_RELOC_LO_14__MARC_RELOC_LO_14__SHIFT                                                     0xc
+#define GCMC_VM_MARC_RELOC_LO_14__MARC_ENABLE_14_MASK                                                         0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_14__MARC_READONLY_14_MASK                                                       0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_14__MARC_RELOC_LO_14_MASK                                                       0xFFFFF000L
+//GCMC_VM_MARC_RELOC_LO_15
+#define GCMC_VM_MARC_RELOC_LO_15__MARC_ENABLE_15__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_LO_15__MARC_READONLY_15__SHIFT                                                     0x1
+#define GCMC_VM_MARC_RELOC_LO_15__MARC_RELOC_LO_15__SHIFT                                                     0xc
+#define GCMC_VM_MARC_RELOC_LO_15__MARC_ENABLE_15_MASK                                                         0x00000001L
+#define GCMC_VM_MARC_RELOC_LO_15__MARC_READONLY_15_MASK                                                       0x00000002L
+#define GCMC_VM_MARC_RELOC_LO_15__MARC_RELOC_LO_15_MASK                                                       0xFFFFF000L
+//GCMC_VM_MARC_RELOC_HI_0
+#define GCMC_VM_MARC_RELOC_HI_0__MARC_RELOC_HI_0__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_HI_0__MARC_RELOC_HI_0_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_1
+#define GCMC_VM_MARC_RELOC_HI_1__MARC_RELOC_HI_1__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_HI_1__MARC_RELOC_HI_1_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_2
+#define GCMC_VM_MARC_RELOC_HI_2__MARC_RELOC_HI_2__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_HI_2__MARC_RELOC_HI_2_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_3
+#define GCMC_VM_MARC_RELOC_HI_3__MARC_RELOC_HI_3__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_HI_3__MARC_RELOC_HI_3_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_4
+#define GCMC_VM_MARC_RELOC_HI_4__MARC_RELOC_HI_4__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_HI_4__MARC_RELOC_HI_4_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_5
+#define GCMC_VM_MARC_RELOC_HI_5__MARC_RELOC_HI_5__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_HI_5__MARC_RELOC_HI_5_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_6
+#define GCMC_VM_MARC_RELOC_HI_6__MARC_RELOC_HI_6__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_HI_6__MARC_RELOC_HI_6_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_7
+#define GCMC_VM_MARC_RELOC_HI_7__MARC_RELOC_HI_7__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_HI_7__MARC_RELOC_HI_7_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_8
+#define GCMC_VM_MARC_RELOC_HI_8__MARC_RELOC_HI_8__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_HI_8__MARC_RELOC_HI_8_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_9
+#define GCMC_VM_MARC_RELOC_HI_9__MARC_RELOC_HI_9__SHIFT                                                       0x0
+#define GCMC_VM_MARC_RELOC_HI_9__MARC_RELOC_HI_9_MASK                                                         0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_10
+#define GCMC_VM_MARC_RELOC_HI_10__MARC_RELOC_HI_10__SHIFT                                                     0x0
+#define GCMC_VM_MARC_RELOC_HI_10__MARC_RELOC_HI_10_MASK                                                       0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_11
+#define GCMC_VM_MARC_RELOC_HI_11__MARC_RELOC_HI_11__SHIFT                                                     0x0
+#define GCMC_VM_MARC_RELOC_HI_11__MARC_RELOC_HI_11_MASK                                                       0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_12
+#define GCMC_VM_MARC_RELOC_HI_12__MARC_RELOC_HI_12__SHIFT                                                     0x0
+#define GCMC_VM_MARC_RELOC_HI_12__MARC_RELOC_HI_12_MASK                                                       0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_13
+#define GCMC_VM_MARC_RELOC_HI_13__MARC_RELOC_HI_13__SHIFT                                                     0x0
+#define GCMC_VM_MARC_RELOC_HI_13__MARC_RELOC_HI_13_MASK                                                       0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_14
+#define GCMC_VM_MARC_RELOC_HI_14__MARC_RELOC_HI_14__SHIFT                                                     0x0
+#define GCMC_VM_MARC_RELOC_HI_14__MARC_RELOC_HI_14_MASK                                                       0x000FFFFFL
+//GCMC_VM_MARC_RELOC_HI_15
+#define GCMC_VM_MARC_RELOC_HI_15__MARC_RELOC_HI_15__SHIFT                                                     0x0
+#define GCMC_VM_MARC_RELOC_HI_15__MARC_RELOC_HI_15_MASK                                                       0x000FFFFFL
+//GCMC_VM_MARC_LEN_LO_0
+#define GCMC_VM_MARC_LEN_LO_0__MARC_LEN_LO_0__SHIFT                                                           0xc
+#define GCMC_VM_MARC_LEN_LO_0__MARC_LEN_LO_0_MASK                                                             0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_1
+#define GCMC_VM_MARC_LEN_LO_1__MARC_LEN_LO_1__SHIFT                                                           0xc
+#define GCMC_VM_MARC_LEN_LO_1__MARC_LEN_LO_1_MASK                                                             0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_2
+#define GCMC_VM_MARC_LEN_LO_2__MARC_LEN_LO_2__SHIFT                                                           0xc
+#define GCMC_VM_MARC_LEN_LO_2__MARC_LEN_LO_2_MASK                                                             0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_3
+#define GCMC_VM_MARC_LEN_LO_3__MARC_LEN_LO_3__SHIFT                                                           0xc
+#define GCMC_VM_MARC_LEN_LO_3__MARC_LEN_LO_3_MASK                                                             0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_4
+#define GCMC_VM_MARC_LEN_LO_4__MARC_LEN_LO_4__SHIFT                                                           0xc
+#define GCMC_VM_MARC_LEN_LO_4__MARC_LEN_LO_4_MASK                                                             0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_5
+#define GCMC_VM_MARC_LEN_LO_5__MARC_LEN_LO_5__SHIFT                                                           0xc
+#define GCMC_VM_MARC_LEN_LO_5__MARC_LEN_LO_5_MASK                                                             0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_6
+#define GCMC_VM_MARC_LEN_LO_6__MARC_LEN_LO_6__SHIFT                                                           0xc
+#define GCMC_VM_MARC_LEN_LO_6__MARC_LEN_LO_6_MASK                                                             0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_7
+#define GCMC_VM_MARC_LEN_LO_7__MARC_LEN_LO_7__SHIFT                                                           0xc
+#define GCMC_VM_MARC_LEN_LO_7__MARC_LEN_LO_7_MASK                                                             0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_8
+#define GCMC_VM_MARC_LEN_LO_8__MARC_LEN_LO_8__SHIFT                                                           0xc
+#define GCMC_VM_MARC_LEN_LO_8__MARC_LEN_LO_8_MASK                                                             0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_9
+#define GCMC_VM_MARC_LEN_LO_9__MARC_LEN_LO_9__SHIFT                                                           0xc
+#define GCMC_VM_MARC_LEN_LO_9__MARC_LEN_LO_9_MASK                                                             0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_10
+#define GCMC_VM_MARC_LEN_LO_10__MARC_LEN_LO_10__SHIFT                                                         0xc
+#define GCMC_VM_MARC_LEN_LO_10__MARC_LEN_LO_10_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_11
+#define GCMC_VM_MARC_LEN_LO_11__MARC_LEN_LO_11__SHIFT                                                         0xc
+#define GCMC_VM_MARC_LEN_LO_11__MARC_LEN_LO_11_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_12
+#define GCMC_VM_MARC_LEN_LO_12__MARC_LEN_LO_12__SHIFT                                                         0xc
+#define GCMC_VM_MARC_LEN_LO_12__MARC_LEN_LO_12_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_13
+#define GCMC_VM_MARC_LEN_LO_13__MARC_LEN_LO_13__SHIFT                                                         0xc
+#define GCMC_VM_MARC_LEN_LO_13__MARC_LEN_LO_13_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_14
+#define GCMC_VM_MARC_LEN_LO_14__MARC_LEN_LO_14__SHIFT                                                         0xc
+#define GCMC_VM_MARC_LEN_LO_14__MARC_LEN_LO_14_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_LEN_LO_15
+#define GCMC_VM_MARC_LEN_LO_15__MARC_LEN_LO_15__SHIFT                                                         0xc
+#define GCMC_VM_MARC_LEN_LO_15__MARC_LEN_LO_15_MASK                                                           0xFFFFF000L
+//GCMC_VM_MARC_LEN_HI_0
+#define GCMC_VM_MARC_LEN_HI_0__MARC_LEN_HI_0__SHIFT                                                           0x0
+#define GCMC_VM_MARC_LEN_HI_0__MARC_LEN_HI_0_MASK                                                             0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_1
+#define GCMC_VM_MARC_LEN_HI_1__MARC_LEN_HI_1__SHIFT                                                           0x0
+#define GCMC_VM_MARC_LEN_HI_1__MARC_LEN_HI_1_MASK                                                             0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_2
+#define GCMC_VM_MARC_LEN_HI_2__MARC_LEN_HI_2__SHIFT                                                           0x0
+#define GCMC_VM_MARC_LEN_HI_2__MARC_LEN_HI_2_MASK                                                             0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_3
+#define GCMC_VM_MARC_LEN_HI_3__MARC_LEN_HI_3__SHIFT                                                           0x0
+#define GCMC_VM_MARC_LEN_HI_3__MARC_LEN_HI_3_MASK                                                             0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_4
+#define GCMC_VM_MARC_LEN_HI_4__MARC_LEN_HI_4__SHIFT                                                           0x0
+#define GCMC_VM_MARC_LEN_HI_4__MARC_LEN_HI_4_MASK                                                             0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_5
+#define GCMC_VM_MARC_LEN_HI_5__MARC_LEN_HI_5__SHIFT                                                           0x0
+#define GCMC_VM_MARC_LEN_HI_5__MARC_LEN_HI_5_MASK                                                             0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_6
+#define GCMC_VM_MARC_LEN_HI_6__MARC_LEN_HI_6__SHIFT                                                           0x0
+#define GCMC_VM_MARC_LEN_HI_6__MARC_LEN_HI_6_MASK                                                             0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_7
+#define GCMC_VM_MARC_LEN_HI_7__MARC_LEN_HI_7__SHIFT                                                           0x0
+#define GCMC_VM_MARC_LEN_HI_7__MARC_LEN_HI_7_MASK                                                             0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_8
+#define GCMC_VM_MARC_LEN_HI_8__MARC_LEN_HI_8__SHIFT                                                           0x0
+#define GCMC_VM_MARC_LEN_HI_8__MARC_LEN_HI_8_MASK                                                             0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_9
+#define GCMC_VM_MARC_LEN_HI_9__MARC_LEN_HI_9__SHIFT                                                           0x0
+#define GCMC_VM_MARC_LEN_HI_9__MARC_LEN_HI_9_MASK                                                             0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_10
+#define GCMC_VM_MARC_LEN_HI_10__MARC_LEN_HI_10__SHIFT                                                         0x0
+#define GCMC_VM_MARC_LEN_HI_10__MARC_LEN_HI_10_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_11
+#define GCMC_VM_MARC_LEN_HI_11__MARC_LEN_HI_11__SHIFT                                                         0x0
+#define GCMC_VM_MARC_LEN_HI_11__MARC_LEN_HI_11_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_12
+#define GCMC_VM_MARC_LEN_HI_12__MARC_LEN_HI_12__SHIFT                                                         0x0
+#define GCMC_VM_MARC_LEN_HI_12__MARC_LEN_HI_12_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_13
+#define GCMC_VM_MARC_LEN_HI_13__MARC_LEN_HI_13__SHIFT                                                         0x0
+#define GCMC_VM_MARC_LEN_HI_13__MARC_LEN_HI_13_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_14
+#define GCMC_VM_MARC_LEN_HI_14__MARC_LEN_HI_14__SHIFT                                                         0x0
+#define GCMC_VM_MARC_LEN_HI_14__MARC_LEN_HI_14_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_LEN_HI_15
+#define GCMC_VM_MARC_LEN_HI_15__MARC_LEN_HI_15__SHIFT                                                         0x0
+#define GCMC_VM_MARC_LEN_HI_15__MARC_LEN_HI_15_MASK                                                           0x000FFFFFL
+//GCMC_VM_MARC_PFVF_MAPPING_0
+#define GCMC_VM_MARC_PFVF_MAPPING_0__ENABLE_VFS__SHIFT                                                        0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_0__ENABLE_PF__SHIFT                                                         0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_0__ENABLE_VFS_MASK                                                          0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_0__ENABLE_PF_MASK                                                           0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_1
+#define GCMC_VM_MARC_PFVF_MAPPING_1__ENABLE_VFS__SHIFT                                                        0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_1__ENABLE_PF__SHIFT                                                         0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_1__ENABLE_VFS_MASK                                                          0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_1__ENABLE_PF_MASK                                                           0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_2
+#define GCMC_VM_MARC_PFVF_MAPPING_2__ENABLE_VFS__SHIFT                                                        0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_2__ENABLE_PF__SHIFT                                                         0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_2__ENABLE_VFS_MASK                                                          0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_2__ENABLE_PF_MASK                                                           0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_3
+#define GCMC_VM_MARC_PFVF_MAPPING_3__ENABLE_VFS__SHIFT                                                        0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_3__ENABLE_PF__SHIFT                                                         0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_3__ENABLE_VFS_MASK                                                          0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_3__ENABLE_PF_MASK                                                           0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_4
+#define GCMC_VM_MARC_PFVF_MAPPING_4__ENABLE_VFS__SHIFT                                                        0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_4__ENABLE_PF__SHIFT                                                         0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_4__ENABLE_VFS_MASK                                                          0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_4__ENABLE_PF_MASK                                                           0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_5
+#define GCMC_VM_MARC_PFVF_MAPPING_5__ENABLE_VFS__SHIFT                                                        0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_5__ENABLE_PF__SHIFT                                                         0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_5__ENABLE_VFS_MASK                                                          0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_5__ENABLE_PF_MASK                                                           0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_6
+#define GCMC_VM_MARC_PFVF_MAPPING_6__ENABLE_VFS__SHIFT                                                        0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_6__ENABLE_PF__SHIFT                                                         0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_6__ENABLE_VFS_MASK                                                          0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_6__ENABLE_PF_MASK                                                           0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_7
+#define GCMC_VM_MARC_PFVF_MAPPING_7__ENABLE_VFS__SHIFT                                                        0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_7__ENABLE_PF__SHIFT                                                         0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_7__ENABLE_VFS_MASK                                                          0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_7__ENABLE_PF_MASK                                                           0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_8
+#define GCMC_VM_MARC_PFVF_MAPPING_8__ENABLE_VFS__SHIFT                                                        0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_8__ENABLE_PF__SHIFT                                                         0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_8__ENABLE_VFS_MASK                                                          0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_8__ENABLE_PF_MASK                                                           0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_9
+#define GCMC_VM_MARC_PFVF_MAPPING_9__ENABLE_VFS__SHIFT                                                        0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_9__ENABLE_PF__SHIFT                                                         0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_9__ENABLE_VFS_MASK                                                          0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_9__ENABLE_PF_MASK                                                           0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_10
+#define GCMC_VM_MARC_PFVF_MAPPING_10__ENABLE_VFS__SHIFT                                                       0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_10__ENABLE_PF__SHIFT                                                        0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_10__ENABLE_VFS_MASK                                                         0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_10__ENABLE_PF_MASK                                                          0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_11
+#define GCMC_VM_MARC_PFVF_MAPPING_11__ENABLE_VFS__SHIFT                                                       0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_11__ENABLE_PF__SHIFT                                                        0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_11__ENABLE_VFS_MASK                                                         0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_11__ENABLE_PF_MASK                                                          0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_12
+#define GCMC_VM_MARC_PFVF_MAPPING_12__ENABLE_VFS__SHIFT                                                       0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_12__ENABLE_PF__SHIFT                                                        0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_12__ENABLE_VFS_MASK                                                         0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_12__ENABLE_PF_MASK                                                          0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_13
+#define GCMC_VM_MARC_PFVF_MAPPING_13__ENABLE_VFS__SHIFT                                                       0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_13__ENABLE_PF__SHIFT                                                        0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_13__ENABLE_VFS_MASK                                                         0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_13__ENABLE_PF_MASK                                                          0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_14
+#define GCMC_VM_MARC_PFVF_MAPPING_14__ENABLE_VFS__SHIFT                                                       0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_14__ENABLE_PF__SHIFT                                                        0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_14__ENABLE_VFS_MASK                                                         0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_14__ENABLE_PF_MASK                                                          0x00010000L
+//GCMC_VM_MARC_PFVF_MAPPING_15
+#define GCMC_VM_MARC_PFVF_MAPPING_15__ENABLE_VFS__SHIFT                                                       0x0
+#define GCMC_VM_MARC_PFVF_MAPPING_15__ENABLE_PF__SHIFT                                                        0x10
+#define GCMC_VM_MARC_PFVF_MAPPING_15__ENABLE_VFS_MASK                                                         0x0000FFFFL
+#define GCMC_VM_MARC_PFVF_MAPPING_15__ENABLE_PF_MASK                                                          0x00010000L
+//GCUTC_TRANSLATION_FAULT_CNTL0
+#define GCUTC_TRANSLATION_FAULT_CNTL0__DEFAULT_PHYSICAL_PAGE_ADDRESS_LSB__SHIFT                               0x0
+#define GCUTC_TRANSLATION_FAULT_CNTL0__DEFAULT_PHYSICAL_PAGE_ADDRESS_LSB_MASK                                 0xFFFFFFFFL
+//GCUTC_TRANSLATION_FAULT_CNTL1
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_PHYSICAL_PAGE_ADDRESS_MSB__SHIFT                               0x0
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_IO__SHIFT                                                      0x4
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_SPA__SHIFT                                                     0x5
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_SNOOP__SHIFT                                                   0x6
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_PHYSICAL_PAGE_ADDRESS_MSB_MASK                                 0x0000000FL
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_IO_MASK                                                        0x00000010L
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_SPA_MASK                                                       0x00000020L
+#define GCUTC_TRANSLATION_FAULT_CNTL1__DEFAULT_SNOOP_MASK                                                     0x00000040L
+
+
+// addressBlock: gc_gcl2tlbpspdec
+//GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_CNTL
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_CNTL__ENABLE__SHIFT                                               0x0
+#define GCUTC_GPUVA_VMID_TRANSLATION_ASSIST_CNTL__ENABLE_MASK                                                 0x00000001L
+
+
+// addressBlock: gc_shdec
+//SPI_SHADER_PGM_RSRC4_PS
+#define SPI_SHADER_PGM_RSRC4_PS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC4_PS__INST_PREF_SIZE__SHIFT                                                        0x10
+#define SPI_SHADER_PGM_RSRC4_PS__TRAP_ON_START__SHIFT                                                         0x1d
+#define SPI_SHADER_PGM_RSRC4_PS__TRAP_ON_END__SHIFT                                                           0x1e
+#define SPI_SHADER_PGM_RSRC4_PS__IMAGE_OP__SHIFT                                                              0x1f
+#define SPI_SHADER_PGM_RSRC4_PS__CU_EN_MASK                                                                   0x0000FFFFL
+#define SPI_SHADER_PGM_RSRC4_PS__INST_PREF_SIZE_MASK                                                          0x003F0000L
+#define SPI_SHADER_PGM_RSRC4_PS__TRAP_ON_START_MASK                                                           0x20000000L
+#define SPI_SHADER_PGM_RSRC4_PS__TRAP_ON_END_MASK                                                             0x40000000L
+#define SPI_SHADER_PGM_RSRC4_PS__IMAGE_OP_MASK                                                                0x80000000L
+//SPI_SHADER_PGM_CHKSUM_PS
+#define SPI_SHADER_PGM_CHKSUM_PS__CHECKSUM__SHIFT                                                             0x0
+#define SPI_SHADER_PGM_CHKSUM_PS__CHECKSUM_MASK                                                               0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC3_PS
+#define SPI_SHADER_PGM_RSRC3_PS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC3_PS__WAVE_LIMIT__SHIFT                                                            0x10
+#define SPI_SHADER_PGM_RSRC3_PS__LDS_GROUP_SIZE__SHIFT                                                        0x16
+#define SPI_SHADER_PGM_RSRC3_PS__CU_EN_MASK                                                                   0x0000FFFFL
+#define SPI_SHADER_PGM_RSRC3_PS__WAVE_LIMIT_MASK                                                              0x003F0000L
+#define SPI_SHADER_PGM_RSRC3_PS__LDS_GROUP_SIZE_MASK                                                          0x00C00000L
+//SPI_SHADER_PGM_LO_PS
+#define SPI_SHADER_PGM_LO_PS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_PS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_PS
+#define SPI_SHADER_PGM_HI_PS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_PS__MEM_BASE_MASK                                                                   0xFFL
+//SPI_SHADER_PGM_RSRC1_PS
+#define SPI_SHADER_PGM_RSRC1_PS__VGPRS__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC1_PS__SGPRS__SHIFT                                                                 0x6
+#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY__SHIFT                                                              0xa
+#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE__SHIFT                                                            0xc
+#define SPI_SHADER_PGM_RSRC1_PS__PRIV__SHIFT                                                                  0x14
+#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP__SHIFT                                                            0x15
+#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE__SHIFT                                                             0x17
+#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE__SHIFT                                                      0x18
+#define SPI_SHADER_PGM_RSRC1_PS__MEM_ORDERED__SHIFT                                                           0x19
+#define SPI_SHADER_PGM_RSRC1_PS__FWD_PROGRESS__SHIFT                                                          0x1a
+#define SPI_SHADER_PGM_RSRC1_PS__LOAD_PROVOKING_VTX__SHIFT                                                    0x1b
+#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER__SHIFT                                                             0x1c
+#define SPI_SHADER_PGM_RSRC1_PS__FP16_OVFL__SHIFT                                                             0x1d
+#define SPI_SHADER_PGM_RSRC1_PS__VGPRS_MASK                                                                   0x0000003FL
+#define SPI_SHADER_PGM_RSRC1_PS__SGPRS_MASK                                                                   0x000003C0L
+#define SPI_SHADER_PGM_RSRC1_PS__PRIORITY_MASK                                                                0x00000C00L
+#define SPI_SHADER_PGM_RSRC1_PS__FLOAT_MODE_MASK                                                              0x000FF000L
+#define SPI_SHADER_PGM_RSRC1_PS__PRIV_MASK                                                                    0x00100000L
+#define SPI_SHADER_PGM_RSRC1_PS__DX10_CLAMP_MASK                                                              0x00200000L
+#define SPI_SHADER_PGM_RSRC1_PS__DEBUG_MODE_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC1_PS__IEEE_MODE_MASK                                                               0x00800000L
+#define SPI_SHADER_PGM_RSRC1_PS__CU_GROUP_DISABLE_MASK                                                        0x01000000L
+#define SPI_SHADER_PGM_RSRC1_PS__MEM_ORDERED_MASK                                                             0x02000000L
+#define SPI_SHADER_PGM_RSRC1_PS__FWD_PROGRESS_MASK                                                            0x04000000L
+#define SPI_SHADER_PGM_RSRC1_PS__LOAD_PROVOKING_VTX_MASK                                                      0x08000000L
+#define SPI_SHADER_PGM_RSRC1_PS__CDBG_USER_MASK                                                               0x10000000L
+#define SPI_SHADER_PGM_RSRC1_PS__FP16_OVFL_MASK                                                               0x20000000L
+//SPI_SHADER_PGM_RSRC2_PS
+#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR__SHIFT                                                             0x1
+#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT__SHIFT                                                          0x6
+#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN__SHIFT                                                           0x7
+#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE__SHIFT                                                        0x8
+#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN__SHIFT                                                               0x10
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_COLLISION_WAVEID__SHIFT                                                 0x19
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_INTRAWAVE_COLLISION__SHIFT                                              0x1a
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MSB__SHIFT                                                         0x1b
+#define SPI_SHADER_PGM_RSRC2_PS__SHARED_VGPR_CNT__SHIFT                                                       0x1c
+#define SPI_SHADER_PGM_RSRC2_PS__SCRATCH_EN_MASK                                                              0x00000001L
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MASK                                                               0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_PS__TRAP_PRESENT_MASK                                                            0x00000040L
+#define SPI_SHADER_PGM_RSRC2_PS__WAVE_CNT_EN_MASK                                                             0x00000080L
+#define SPI_SHADER_PGM_RSRC2_PS__EXTRA_LDS_SIZE_MASK                                                          0x0000FF00L
+#define SPI_SHADER_PGM_RSRC2_PS__EXCP_EN_MASK                                                                 0x01FF0000L
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_COLLISION_WAVEID_MASK                                                   0x02000000L
+#define SPI_SHADER_PGM_RSRC2_PS__LOAD_INTRAWAVE_COLLISION_MASK                                                0x04000000L
+#define SPI_SHADER_PGM_RSRC2_PS__USER_SGPR_MSB_MASK                                                           0x08000000L
+#define SPI_SHADER_PGM_RSRC2_PS__SHARED_VGPR_CNT_MASK                                                         0xF0000000L
+//SPI_SHADER_USER_DATA_PS_0
+#define SPI_SHADER_USER_DATA_PS_0__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_0__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_1
+#define SPI_SHADER_USER_DATA_PS_1__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_1__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_2
+#define SPI_SHADER_USER_DATA_PS_2__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_2__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_3
+#define SPI_SHADER_USER_DATA_PS_3__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_3__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_4
+#define SPI_SHADER_USER_DATA_PS_4__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_4__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_5
+#define SPI_SHADER_USER_DATA_PS_5__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_5__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_6
+#define SPI_SHADER_USER_DATA_PS_6__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_6__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_7
+#define SPI_SHADER_USER_DATA_PS_7__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_7__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_8
+#define SPI_SHADER_USER_DATA_PS_8__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_8__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_9
+#define SPI_SHADER_USER_DATA_PS_9__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_PS_9__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_10
+#define SPI_SHADER_USER_DATA_PS_10__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_10__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_11
+#define SPI_SHADER_USER_DATA_PS_11__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_11__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_12
+#define SPI_SHADER_USER_DATA_PS_12__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_12__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_13
+#define SPI_SHADER_USER_DATA_PS_13__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_13__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_14
+#define SPI_SHADER_USER_DATA_PS_14__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_14__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_15
+#define SPI_SHADER_USER_DATA_PS_15__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_15__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_16
+#define SPI_SHADER_USER_DATA_PS_16__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_16__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_17
+#define SPI_SHADER_USER_DATA_PS_17__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_17__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_18
+#define SPI_SHADER_USER_DATA_PS_18__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_18__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_19
+#define SPI_SHADER_USER_DATA_PS_19__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_19__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_20
+#define SPI_SHADER_USER_DATA_PS_20__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_20__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_21
+#define SPI_SHADER_USER_DATA_PS_21__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_21__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_22
+#define SPI_SHADER_USER_DATA_PS_22__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_22__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_23
+#define SPI_SHADER_USER_DATA_PS_23__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_23__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_24
+#define SPI_SHADER_USER_DATA_PS_24__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_24__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_25
+#define SPI_SHADER_USER_DATA_PS_25__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_25__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_26
+#define SPI_SHADER_USER_DATA_PS_26__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_26__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_27
+#define SPI_SHADER_USER_DATA_PS_27__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_27__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_28
+#define SPI_SHADER_USER_DATA_PS_28__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_28__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_29
+#define SPI_SHADER_USER_DATA_PS_29__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_29__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_30
+#define SPI_SHADER_USER_DATA_PS_30__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_30__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_PS_31
+#define SPI_SHADER_USER_DATA_PS_31__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_PS_31__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_REQ_CTRL_PS
+#define SPI_SHADER_REQ_CTRL_PS__SOFT_GROUPING_EN__SHIFT                                                       0x0
+#define SPI_SHADER_REQ_CTRL_PS__NUMBER_OF_REQUESTS_PER_CU__SHIFT                                              0x1
+#define SPI_SHADER_REQ_CTRL_PS__SOFT_GROUPING_ALLOCATION_TIMEOUT__SHIFT                                       0x5
+#define SPI_SHADER_REQ_CTRL_PS__HARD_LOCK_HYSTERESIS__SHIFT                                                   0x9
+#define SPI_SHADER_REQ_CTRL_PS__HARD_LOCK_LOW_THRESHOLD__SHIFT                                                0xa
+#define SPI_SHADER_REQ_CTRL_PS__PRODUCER_REQUEST_LOCKOUT__SHIFT                                               0xf
+#define SPI_SHADER_REQ_CTRL_PS__GLOBAL_SCANNING_EN__SHIFT                                                     0x10
+#define SPI_SHADER_REQ_CTRL_PS__ALLOCATION_RATE_THROTTLING_THRESHOLD__SHIFT                                   0x11
+#define SPI_SHADER_REQ_CTRL_PS__SOFT_GROUPING_EN_MASK                                                         0x00000001L
+#define SPI_SHADER_REQ_CTRL_PS__NUMBER_OF_REQUESTS_PER_CU_MASK                                                0x0000001EL
+#define SPI_SHADER_REQ_CTRL_PS__SOFT_GROUPING_ALLOCATION_TIMEOUT_MASK                                         0x000001E0L
+#define SPI_SHADER_REQ_CTRL_PS__HARD_LOCK_HYSTERESIS_MASK                                                     0x00000200L
+#define SPI_SHADER_REQ_CTRL_PS__HARD_LOCK_LOW_THRESHOLD_MASK                                                  0x00007C00L
+#define SPI_SHADER_REQ_CTRL_PS__PRODUCER_REQUEST_LOCKOUT_MASK                                                 0x00008000L
+#define SPI_SHADER_REQ_CTRL_PS__GLOBAL_SCANNING_EN_MASK                                                       0x00010000L
+#define SPI_SHADER_REQ_CTRL_PS__ALLOCATION_RATE_THROTTLING_THRESHOLD_MASK                                     0x000E0000L
+//SPI_SHADER_USER_ACCUM_PS_0
+#define SPI_SHADER_USER_ACCUM_PS_0__CONTRIBUTION__SHIFT                                                       0x0
+#define SPI_SHADER_USER_ACCUM_PS_0__CONTRIBUTION_MASK                                                         0x0000007FL
+//SPI_SHADER_USER_ACCUM_PS_1
+#define SPI_SHADER_USER_ACCUM_PS_1__CONTRIBUTION__SHIFT                                                       0x0
+#define SPI_SHADER_USER_ACCUM_PS_1__CONTRIBUTION_MASK                                                         0x0000007FL
+//SPI_SHADER_USER_ACCUM_PS_2
+#define SPI_SHADER_USER_ACCUM_PS_2__CONTRIBUTION__SHIFT                                                       0x0
+#define SPI_SHADER_USER_ACCUM_PS_2__CONTRIBUTION_MASK                                                         0x0000007FL
+//SPI_SHADER_USER_ACCUM_PS_3
+#define SPI_SHADER_USER_ACCUM_PS_3__CONTRIBUTION__SHIFT                                                       0x0
+#define SPI_SHADER_USER_ACCUM_PS_3__CONTRIBUTION_MASK                                                         0x0000007FL
+//SPI_SHADER_PGM_CHKSUM_GS
+#define SPI_SHADER_PGM_CHKSUM_GS__CHECKSUM__SHIFT                                                             0x0
+#define SPI_SHADER_PGM_CHKSUM_GS__CHECKSUM_MASK                                                               0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC4_GS
+#define SPI_SHADER_PGM_RSRC4_GS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC4_GS__RESERVED__SHIFT                                                              0x1
+#define SPI_SHADER_PGM_RSRC4_GS__PH_THROTTLE_EN__SHIFT                                                        0xe
+#define SPI_SHADER_PGM_RSRC4_GS__SPI_THROTTLE_EN__SHIFT                                                       0xf
+#define SPI_SHADER_PGM_RSRC4_GS__SPI_SHADER_LATE_ALLOC_GS__SHIFT                                              0x10
+#define SPI_SHADER_PGM_RSRC4_GS__INST_PREF_SIZE__SHIFT                                                        0x17
+#define SPI_SHADER_PGM_RSRC4_GS__TRAP_ON_START__SHIFT                                                         0x1d
+#define SPI_SHADER_PGM_RSRC4_GS__TRAP_ON_END__SHIFT                                                           0x1e
+#define SPI_SHADER_PGM_RSRC4_GS__IMAGE_OP__SHIFT                                                              0x1f
+#define SPI_SHADER_PGM_RSRC4_GS__CU_EN_MASK                                                                   0x00000001L
+#define SPI_SHADER_PGM_RSRC4_GS__RESERVED_MASK                                                                0x00003FFEL
+#define SPI_SHADER_PGM_RSRC4_GS__PH_THROTTLE_EN_MASK                                                          0x00004000L
+#define SPI_SHADER_PGM_RSRC4_GS__SPI_THROTTLE_EN_MASK                                                         0x00008000L
+#define SPI_SHADER_PGM_RSRC4_GS__SPI_SHADER_LATE_ALLOC_GS_MASK                                                0x007F0000L
+#define SPI_SHADER_PGM_RSRC4_GS__INST_PREF_SIZE_MASK                                                          0x1F800000L
+#define SPI_SHADER_PGM_RSRC4_GS__TRAP_ON_START_MASK                                                           0x20000000L
+#define SPI_SHADER_PGM_RSRC4_GS__TRAP_ON_END_MASK                                                             0x40000000L
+#define SPI_SHADER_PGM_RSRC4_GS__IMAGE_OP_MASK                                                                0x80000000L
+//SPI_SHADER_USER_DATA_ADDR_LO_GS
+#define SPI_SHADER_USER_DATA_ADDR_LO_GS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_LO_GS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ADDR_HI_GS
+#define SPI_SHADER_USER_DATA_ADDR_HI_GS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_HI_GS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_PGM_LO_ES_GS
+#define SPI_SHADER_PGM_LO_ES_GS__MEM_BASE__SHIFT                                                              0x0
+#define SPI_SHADER_PGM_LO_ES_GS__MEM_BASE_MASK                                                                0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_ES_GS
+#define SPI_SHADER_PGM_HI_ES_GS__MEM_BASE__SHIFT                                                              0x0
+#define SPI_SHADER_PGM_HI_ES_GS__MEM_BASE_MASK                                                                0xFFL
+//SPI_SHADER_PGM_RSRC3_GS
+#define SPI_SHADER_PGM_RSRC3_GS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC3_GS__WAVE_LIMIT__SHIFT                                                            0x10
+#define SPI_SHADER_PGM_RSRC3_GS__LOCK_LOW_THRESHOLD__SHIFT                                                    0x16
+#define SPI_SHADER_PGM_RSRC3_GS__GROUP_FIFO_DEPTH__SHIFT                                                      0x1a
+#define SPI_SHADER_PGM_RSRC3_GS__CU_EN_MASK                                                                   0x0000FFFFL
+#define SPI_SHADER_PGM_RSRC3_GS__WAVE_LIMIT_MASK                                                              0x003F0000L
+#define SPI_SHADER_PGM_RSRC3_GS__LOCK_LOW_THRESHOLD_MASK                                                      0x03C00000L
+#define SPI_SHADER_PGM_RSRC3_GS__GROUP_FIFO_DEPTH_MASK                                                        0xFC000000L
+//SPI_SHADER_PGM_LO_GS
+#define SPI_SHADER_PGM_LO_GS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_GS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_GS
+#define SPI_SHADER_PGM_HI_GS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_GS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC1_GS
+#define SPI_SHADER_PGM_RSRC1_GS__VGPRS__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC1_GS__SGPRS__SHIFT                                                                 0x6
+#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY__SHIFT                                                              0xa
+#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE__SHIFT                                                            0xc
+#define SPI_SHADER_PGM_RSRC1_GS__PRIV__SHIFT                                                                  0x14
+#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP__SHIFT                                                            0x15
+#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE__SHIFT                                                             0x17
+#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE__SHIFT                                                       0x18
+#define SPI_SHADER_PGM_RSRC1_GS__MEM_ORDERED__SHIFT                                                           0x19
+#define SPI_SHADER_PGM_RSRC1_GS__FWD_PROGRESS__SHIFT                                                          0x1a
+#define SPI_SHADER_PGM_RSRC1_GS__WGP_MODE__SHIFT                                                              0x1b
+#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER__SHIFT                                                             0x1c
+#define SPI_SHADER_PGM_RSRC1_GS__GS_VGPR_COMP_CNT__SHIFT                                                      0x1d
+#define SPI_SHADER_PGM_RSRC1_GS__FP16_OVFL__SHIFT                                                             0x1f
+#define SPI_SHADER_PGM_RSRC1_GS__VGPRS_MASK                                                                   0x0000003FL
+#define SPI_SHADER_PGM_RSRC1_GS__SGPRS_MASK                                                                   0x000003C0L
+#define SPI_SHADER_PGM_RSRC1_GS__PRIORITY_MASK                                                                0x00000C00L
+#define SPI_SHADER_PGM_RSRC1_GS__FLOAT_MODE_MASK                                                              0x000FF000L
+#define SPI_SHADER_PGM_RSRC1_GS__PRIV_MASK                                                                    0x00100000L
+#define SPI_SHADER_PGM_RSRC1_GS__DX10_CLAMP_MASK                                                              0x00200000L
+#define SPI_SHADER_PGM_RSRC1_GS__DEBUG_MODE_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC1_GS__IEEE_MODE_MASK                                                               0x00800000L
+#define SPI_SHADER_PGM_RSRC1_GS__CU_GROUP_ENABLE_MASK                                                         0x01000000L
+#define SPI_SHADER_PGM_RSRC1_GS__MEM_ORDERED_MASK                                                             0x02000000L
+#define SPI_SHADER_PGM_RSRC1_GS__FWD_PROGRESS_MASK                                                            0x04000000L
+#define SPI_SHADER_PGM_RSRC1_GS__WGP_MODE_MASK                                                                0x08000000L
+#define SPI_SHADER_PGM_RSRC1_GS__CDBG_USER_MASK                                                               0x10000000L
+#define SPI_SHADER_PGM_RSRC1_GS__GS_VGPR_COMP_CNT_MASK                                                        0x60000000L
+#define SPI_SHADER_PGM_RSRC1_GS__FP16_OVFL_MASK                                                               0x80000000L
+//SPI_SHADER_PGM_RSRC2_GS
+#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR__SHIFT                                                             0x1
+#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT__SHIFT                                                          0x6
+#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN__SHIFT                                                               0x7
+#define SPI_SHADER_PGM_RSRC2_GS__ES_VGPR_COMP_CNT__SHIFT                                                      0x10
+#define SPI_SHADER_PGM_RSRC2_GS__OC_LDS_EN__SHIFT                                                             0x12
+#define SPI_SHADER_PGM_RSRC2_GS__LDS_SIZE__SHIFT                                                              0x13
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MSB__SHIFT                                                         0x1b
+#define SPI_SHADER_PGM_RSRC2_GS__SHARED_VGPR_CNT__SHIFT                                                       0x1c
+#define SPI_SHADER_PGM_RSRC2_GS__SCRATCH_EN_MASK                                                              0x00000001L
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MASK                                                               0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_GS__TRAP_PRESENT_MASK                                                            0x00000040L
+#define SPI_SHADER_PGM_RSRC2_GS__EXCP_EN_MASK                                                                 0x0000FF80L
+#define SPI_SHADER_PGM_RSRC2_GS__ES_VGPR_COMP_CNT_MASK                                                        0x00030000L
+#define SPI_SHADER_PGM_RSRC2_GS__OC_LDS_EN_MASK                                                               0x00040000L
+#define SPI_SHADER_PGM_RSRC2_GS__LDS_SIZE_MASK                                                                0x07F80000L
+#define SPI_SHADER_PGM_RSRC2_GS__USER_SGPR_MSB_MASK                                                           0x08000000L
+#define SPI_SHADER_PGM_RSRC2_GS__SHARED_VGPR_CNT_MASK                                                         0xF0000000L
+//SPI_SHADER_USER_DATA_GS_0
+#define SPI_SHADER_USER_DATA_GS_0__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_0__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_1
+#define SPI_SHADER_USER_DATA_GS_1__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_1__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_2
+#define SPI_SHADER_USER_DATA_GS_2__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_2__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_3
+#define SPI_SHADER_USER_DATA_GS_3__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_3__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_4
+#define SPI_SHADER_USER_DATA_GS_4__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_4__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_5
+#define SPI_SHADER_USER_DATA_GS_5__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_5__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_6
+#define SPI_SHADER_USER_DATA_GS_6__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_6__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_7
+#define SPI_SHADER_USER_DATA_GS_7__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_7__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_8
+#define SPI_SHADER_USER_DATA_GS_8__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_8__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_9
+#define SPI_SHADER_USER_DATA_GS_9__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_GS_9__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_10
+#define SPI_SHADER_USER_DATA_GS_10__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_10__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_11
+#define SPI_SHADER_USER_DATA_GS_11__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_11__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_12
+#define SPI_SHADER_USER_DATA_GS_12__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_12__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_13
+#define SPI_SHADER_USER_DATA_GS_13__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_13__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_14
+#define SPI_SHADER_USER_DATA_GS_14__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_14__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_15
+#define SPI_SHADER_USER_DATA_GS_15__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_15__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_16
+#define SPI_SHADER_USER_DATA_GS_16__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_16__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_17
+#define SPI_SHADER_USER_DATA_GS_17__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_17__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_18
+#define SPI_SHADER_USER_DATA_GS_18__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_18__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_19
+#define SPI_SHADER_USER_DATA_GS_19__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_19__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_20
+#define SPI_SHADER_USER_DATA_GS_20__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_20__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_21
+#define SPI_SHADER_USER_DATA_GS_21__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_21__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_22
+#define SPI_SHADER_USER_DATA_GS_22__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_22__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_23
+#define SPI_SHADER_USER_DATA_GS_23__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_23__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_24
+#define SPI_SHADER_USER_DATA_GS_24__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_24__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_25
+#define SPI_SHADER_USER_DATA_GS_25__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_25__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_26
+#define SPI_SHADER_USER_DATA_GS_26__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_26__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_27
+#define SPI_SHADER_USER_DATA_GS_27__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_27__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_28
+#define SPI_SHADER_USER_DATA_GS_28__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_28__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_29
+#define SPI_SHADER_USER_DATA_GS_29__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_29__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_30
+#define SPI_SHADER_USER_DATA_GS_30__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_30__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_GS_31
+#define SPI_SHADER_USER_DATA_GS_31__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_GS_31__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_GS_MESHLET_DIM
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_X__SHIFT                                                0x0
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_Y__SHIFT                                                0x8
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_Z__SHIFT                                                0x10
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_THREADGROUP_SIZE__SHIFT                                            0x18
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_X_MASK                                                  0x000000FFL
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_Y_MASK                                                  0x0000FF00L
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_NUM_THREAD_Z_MASK                                                  0x00FF0000L
+#define SPI_SHADER_GS_MESHLET_DIM__MESHLET_THREADGROUP_SIZE_MASK                                              0xFF000000L
+//SPI_SHADER_GS_MESHLET_EXP_ALLOC
+#define SPI_SHADER_GS_MESHLET_EXP_ALLOC__MAX_EXP_VERTS__SHIFT                                                 0x0
+#define SPI_SHADER_GS_MESHLET_EXP_ALLOC__MAX_EXP_PRIMS__SHIFT                                                 0x9
+#define SPI_SHADER_GS_MESHLET_EXP_ALLOC__MAX_EXP_VERTS_MASK                                                   0x000001FFL
+#define SPI_SHADER_GS_MESHLET_EXP_ALLOC__MAX_EXP_PRIMS_MASK                                                   0x0003FE00L
+//SPI_SHADER_REQ_CTRL_ESGS
+#define SPI_SHADER_REQ_CTRL_ESGS__SOFT_GROUPING_EN__SHIFT                                                     0x0
+#define SPI_SHADER_REQ_CTRL_ESGS__NUMBER_OF_REQUESTS_PER_CU__SHIFT                                            0x1
+#define SPI_SHADER_REQ_CTRL_ESGS__SOFT_GROUPING_ALLOCATION_TIMEOUT__SHIFT                                     0x5
+#define SPI_SHADER_REQ_CTRL_ESGS__HARD_LOCK_HYSTERESIS__SHIFT                                                 0x9
+#define SPI_SHADER_REQ_CTRL_ESGS__HARD_LOCK_LOW_THRESHOLD__SHIFT                                              0xa
+#define SPI_SHADER_REQ_CTRL_ESGS__PRODUCER_REQUEST_LOCKOUT__SHIFT                                             0xf
+#define SPI_SHADER_REQ_CTRL_ESGS__GLOBAL_SCANNING_EN__SHIFT                                                   0x10
+#define SPI_SHADER_REQ_CTRL_ESGS__ALLOCATION_RATE_THROTTLING_THRESHOLD__SHIFT                                 0x11
+#define SPI_SHADER_REQ_CTRL_ESGS__SOFT_GROUPING_EN_MASK                                                       0x00000001L
+#define SPI_SHADER_REQ_CTRL_ESGS__NUMBER_OF_REQUESTS_PER_CU_MASK                                              0x0000001EL
+#define SPI_SHADER_REQ_CTRL_ESGS__SOFT_GROUPING_ALLOCATION_TIMEOUT_MASK                                       0x000001E0L
+#define SPI_SHADER_REQ_CTRL_ESGS__HARD_LOCK_HYSTERESIS_MASK                                                   0x00000200L
+#define SPI_SHADER_REQ_CTRL_ESGS__HARD_LOCK_LOW_THRESHOLD_MASK                                                0x00007C00L
+#define SPI_SHADER_REQ_CTRL_ESGS__PRODUCER_REQUEST_LOCKOUT_MASK                                               0x00008000L
+#define SPI_SHADER_REQ_CTRL_ESGS__GLOBAL_SCANNING_EN_MASK                                                     0x00010000L
+#define SPI_SHADER_REQ_CTRL_ESGS__ALLOCATION_RATE_THROTTLING_THRESHOLD_MASK                                   0x000E0000L
+//SPI_SHADER_USER_ACCUM_ESGS_0
+#define SPI_SHADER_USER_ACCUM_ESGS_0__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_ESGS_0__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_ESGS_1
+#define SPI_SHADER_USER_ACCUM_ESGS_1__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_ESGS_1__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_ESGS_2
+#define SPI_SHADER_USER_ACCUM_ESGS_2__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_ESGS_2__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_ESGS_3
+#define SPI_SHADER_USER_ACCUM_ESGS_3__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_ESGS_3__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_PGM_LO_ES
+#define SPI_SHADER_PGM_LO_ES__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_ES__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_ES
+#define SPI_SHADER_PGM_HI_ES__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_ES__MEM_BASE_MASK                                                                   0xFFL
+//SPI_SHADER_PGM_CHKSUM_HS
+#define SPI_SHADER_PGM_CHKSUM_HS__CHECKSUM__SHIFT                                                             0x0
+#define SPI_SHADER_PGM_CHKSUM_HS__CHECKSUM_MASK                                                               0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC4_HS
+#define SPI_SHADER_PGM_RSRC4_HS__CU_EN__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC4_HS__INST_PREF_SIZE__SHIFT                                                        0x10
+#define SPI_SHADER_PGM_RSRC4_HS__TRAP_ON_START__SHIFT                                                         0x1d
+#define SPI_SHADER_PGM_RSRC4_HS__TRAP_ON_END__SHIFT                                                           0x1e
+#define SPI_SHADER_PGM_RSRC4_HS__IMAGE_OP__SHIFT                                                              0x1f
+#define SPI_SHADER_PGM_RSRC4_HS__CU_EN_MASK                                                                   0x0000FFFFL
+#define SPI_SHADER_PGM_RSRC4_HS__INST_PREF_SIZE_MASK                                                          0x003F0000L
+#define SPI_SHADER_PGM_RSRC4_HS__TRAP_ON_START_MASK                                                           0x20000000L
+#define SPI_SHADER_PGM_RSRC4_HS__TRAP_ON_END_MASK                                                             0x40000000L
+#define SPI_SHADER_PGM_RSRC4_HS__IMAGE_OP_MASK                                                                0x80000000L
+//SPI_SHADER_USER_DATA_ADDR_LO_HS
+#define SPI_SHADER_USER_DATA_ADDR_LO_HS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_LO_HS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_ADDR_HI_HS
+#define SPI_SHADER_USER_DATA_ADDR_HI_HS__MEM_BASE__SHIFT                                                      0x0
+#define SPI_SHADER_USER_DATA_ADDR_HI_HS__MEM_BASE_MASK                                                        0xFFFFFFFFL
+//SPI_SHADER_PGM_LO_LS_HS
+#define SPI_SHADER_PGM_LO_LS_HS__MEM_BASE__SHIFT                                                              0x0
+#define SPI_SHADER_PGM_LO_LS_HS__MEM_BASE_MASK                                                                0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_LS_HS
+#define SPI_SHADER_PGM_HI_LS_HS__MEM_BASE__SHIFT                                                              0x0
+#define SPI_SHADER_PGM_HI_LS_HS__MEM_BASE_MASK                                                                0xFFL
+//SPI_SHADER_PGM_RSRC3_HS
+#define SPI_SHADER_PGM_RSRC3_HS__WAVE_LIMIT__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC3_HS__LOCK_LOW_THRESHOLD__SHIFT                                                    0x6
+#define SPI_SHADER_PGM_RSRC3_HS__GROUP_FIFO_DEPTH__SHIFT                                                      0xa
+#define SPI_SHADER_PGM_RSRC3_HS__CU_EN__SHIFT                                                                 0x10
+#define SPI_SHADER_PGM_RSRC3_HS__WAVE_LIMIT_MASK                                                              0x0000003FL
+#define SPI_SHADER_PGM_RSRC3_HS__LOCK_LOW_THRESHOLD_MASK                                                      0x000003C0L
+#define SPI_SHADER_PGM_RSRC3_HS__GROUP_FIFO_DEPTH_MASK                                                        0x0000FC00L
+#define SPI_SHADER_PGM_RSRC3_HS__CU_EN_MASK                                                                   0xFFFF0000L
+//SPI_SHADER_PGM_LO_HS
+#define SPI_SHADER_PGM_LO_HS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_HS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_HS
+#define SPI_SHADER_PGM_HI_HS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_HS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_RSRC1_HS
+#define SPI_SHADER_PGM_RSRC1_HS__VGPRS__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_RSRC1_HS__SGPRS__SHIFT                                                                 0x6
+#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY__SHIFT                                                              0xa
+#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE__SHIFT                                                            0xc
+#define SPI_SHADER_PGM_RSRC1_HS__PRIV__SHIFT                                                                  0x14
+#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP__SHIFT                                                            0x15
+#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE__SHIFT                                                            0x16
+#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE__SHIFT                                                             0x17
+#define SPI_SHADER_PGM_RSRC1_HS__MEM_ORDERED__SHIFT                                                           0x18
+#define SPI_SHADER_PGM_RSRC1_HS__FWD_PROGRESS__SHIFT                                                          0x19
+#define SPI_SHADER_PGM_RSRC1_HS__WGP_MODE__SHIFT                                                              0x1a
+#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER__SHIFT                                                             0x1b
+#define SPI_SHADER_PGM_RSRC1_HS__LS_VGPR_COMP_CNT__SHIFT                                                      0x1c
+#define SPI_SHADER_PGM_RSRC1_HS__FP16_OVFL__SHIFT                                                             0x1e
+#define SPI_SHADER_PGM_RSRC1_HS__VGPRS_MASK                                                                   0x0000003FL
+#define SPI_SHADER_PGM_RSRC1_HS__SGPRS_MASK                                                                   0x000003C0L
+#define SPI_SHADER_PGM_RSRC1_HS__PRIORITY_MASK                                                                0x00000C00L
+#define SPI_SHADER_PGM_RSRC1_HS__FLOAT_MODE_MASK                                                              0x000FF000L
+#define SPI_SHADER_PGM_RSRC1_HS__PRIV_MASK                                                                    0x00100000L
+#define SPI_SHADER_PGM_RSRC1_HS__DX10_CLAMP_MASK                                                              0x00200000L
+#define SPI_SHADER_PGM_RSRC1_HS__DEBUG_MODE_MASK                                                              0x00400000L
+#define SPI_SHADER_PGM_RSRC1_HS__IEEE_MODE_MASK                                                               0x00800000L
+#define SPI_SHADER_PGM_RSRC1_HS__MEM_ORDERED_MASK                                                             0x01000000L
+#define SPI_SHADER_PGM_RSRC1_HS__FWD_PROGRESS_MASK                                                            0x02000000L
+#define SPI_SHADER_PGM_RSRC1_HS__WGP_MODE_MASK                                                                0x04000000L
+#define SPI_SHADER_PGM_RSRC1_HS__CDBG_USER_MASK                                                               0x08000000L
+#define SPI_SHADER_PGM_RSRC1_HS__LS_VGPR_COMP_CNT_MASK                                                        0x30000000L
+#define SPI_SHADER_PGM_RSRC1_HS__FP16_OVFL_MASK                                                               0x40000000L
+//SPI_SHADER_PGM_RSRC2_HS
+#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN__SHIFT                                                            0x0
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR__SHIFT                                                             0x1
+#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT__SHIFT                                                          0x6
+#define SPI_SHADER_PGM_RSRC2_HS__OC_LDS_EN__SHIFT                                                             0x7
+#define SPI_SHADER_PGM_RSRC2_HS__TG_SIZE_EN__SHIFT                                                            0x8
+#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN__SHIFT                                                               0x9
+#define SPI_SHADER_PGM_RSRC2_HS__LDS_SIZE__SHIFT                                                              0x12
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MSB__SHIFT                                                         0x1b
+#define SPI_SHADER_PGM_RSRC2_HS__SHARED_VGPR_CNT__SHIFT                                                       0x1c
+#define SPI_SHADER_PGM_RSRC2_HS__SCRATCH_EN_MASK                                                              0x00000001L
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MASK                                                               0x0000003EL
+#define SPI_SHADER_PGM_RSRC2_HS__TRAP_PRESENT_MASK                                                            0x00000040L
+#define SPI_SHADER_PGM_RSRC2_HS__OC_LDS_EN_MASK                                                               0x00000080L
+#define SPI_SHADER_PGM_RSRC2_HS__TG_SIZE_EN_MASK                                                              0x00000100L
+#define SPI_SHADER_PGM_RSRC2_HS__EXCP_EN_MASK                                                                 0x0003FE00L
+#define SPI_SHADER_PGM_RSRC2_HS__LDS_SIZE_MASK                                                                0x07FC0000L
+#define SPI_SHADER_PGM_RSRC2_HS__USER_SGPR_MSB_MASK                                                           0x08000000L
+#define SPI_SHADER_PGM_RSRC2_HS__SHARED_VGPR_CNT_MASK                                                         0xF0000000L
+//SPI_SHADER_USER_DATA_HS_0
+#define SPI_SHADER_USER_DATA_HS_0__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_0__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_1
+#define SPI_SHADER_USER_DATA_HS_1__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_1__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_2
+#define SPI_SHADER_USER_DATA_HS_2__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_2__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_3
+#define SPI_SHADER_USER_DATA_HS_3__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_3__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_4
+#define SPI_SHADER_USER_DATA_HS_4__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_4__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_5
+#define SPI_SHADER_USER_DATA_HS_5__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_5__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_6
+#define SPI_SHADER_USER_DATA_HS_6__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_6__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_7
+#define SPI_SHADER_USER_DATA_HS_7__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_7__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_8
+#define SPI_SHADER_USER_DATA_HS_8__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_8__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_9
+#define SPI_SHADER_USER_DATA_HS_9__DATA__SHIFT                                                                0x0
+#define SPI_SHADER_USER_DATA_HS_9__DATA_MASK                                                                  0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_10
+#define SPI_SHADER_USER_DATA_HS_10__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_10__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_11
+#define SPI_SHADER_USER_DATA_HS_11__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_11__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_12
+#define SPI_SHADER_USER_DATA_HS_12__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_12__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_13
+#define SPI_SHADER_USER_DATA_HS_13__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_13__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_14
+#define SPI_SHADER_USER_DATA_HS_14__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_14__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_15
+#define SPI_SHADER_USER_DATA_HS_15__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_15__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_16
+#define SPI_SHADER_USER_DATA_HS_16__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_16__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_17
+#define SPI_SHADER_USER_DATA_HS_17__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_17__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_18
+#define SPI_SHADER_USER_DATA_HS_18__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_18__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_19
+#define SPI_SHADER_USER_DATA_HS_19__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_19__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_20
+#define SPI_SHADER_USER_DATA_HS_20__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_20__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_21
+#define SPI_SHADER_USER_DATA_HS_21__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_21__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_22
+#define SPI_SHADER_USER_DATA_HS_22__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_22__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_23
+#define SPI_SHADER_USER_DATA_HS_23__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_23__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_24
+#define SPI_SHADER_USER_DATA_HS_24__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_24__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_25
+#define SPI_SHADER_USER_DATA_HS_25__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_25__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_26
+#define SPI_SHADER_USER_DATA_HS_26__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_26__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_27
+#define SPI_SHADER_USER_DATA_HS_27__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_27__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_28
+#define SPI_SHADER_USER_DATA_HS_28__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_28__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_29
+#define SPI_SHADER_USER_DATA_HS_29__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_29__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_30
+#define SPI_SHADER_USER_DATA_HS_30__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_30__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_USER_DATA_HS_31
+#define SPI_SHADER_USER_DATA_HS_31__DATA__SHIFT                                                               0x0
+#define SPI_SHADER_USER_DATA_HS_31__DATA_MASK                                                                 0xFFFFFFFFL
+//SPI_SHADER_REQ_CTRL_LSHS
+#define SPI_SHADER_REQ_CTRL_LSHS__SOFT_GROUPING_EN__SHIFT                                                     0x0
+#define SPI_SHADER_REQ_CTRL_LSHS__NUMBER_OF_REQUESTS_PER_CU__SHIFT                                            0x1
+#define SPI_SHADER_REQ_CTRL_LSHS__SOFT_GROUPING_ALLOCATION_TIMEOUT__SHIFT                                     0x5
+#define SPI_SHADER_REQ_CTRL_LSHS__HARD_LOCK_HYSTERESIS__SHIFT                                                 0x9
+#define SPI_SHADER_REQ_CTRL_LSHS__HARD_LOCK_LOW_THRESHOLD__SHIFT                                              0xa
+#define SPI_SHADER_REQ_CTRL_LSHS__PRODUCER_REQUEST_LOCKOUT__SHIFT                                             0xf
+#define SPI_SHADER_REQ_CTRL_LSHS__GLOBAL_SCANNING_EN__SHIFT                                                   0x10
+#define SPI_SHADER_REQ_CTRL_LSHS__ALLOCATION_RATE_THROTTLING_THRESHOLD__SHIFT                                 0x11
+#define SPI_SHADER_REQ_CTRL_LSHS__SOFT_GROUPING_EN_MASK                                                       0x00000001L
+#define SPI_SHADER_REQ_CTRL_LSHS__NUMBER_OF_REQUESTS_PER_CU_MASK                                              0x0000001EL
+#define SPI_SHADER_REQ_CTRL_LSHS__SOFT_GROUPING_ALLOCATION_TIMEOUT_MASK                                       0x000001E0L
+#define SPI_SHADER_REQ_CTRL_LSHS__HARD_LOCK_HYSTERESIS_MASK                                                   0x00000200L
+#define SPI_SHADER_REQ_CTRL_LSHS__HARD_LOCK_LOW_THRESHOLD_MASK                                                0x00007C00L
+#define SPI_SHADER_REQ_CTRL_LSHS__PRODUCER_REQUEST_LOCKOUT_MASK                                               0x00008000L
+#define SPI_SHADER_REQ_CTRL_LSHS__GLOBAL_SCANNING_EN_MASK                                                     0x00010000L
+#define SPI_SHADER_REQ_CTRL_LSHS__ALLOCATION_RATE_THROTTLING_THRESHOLD_MASK                                   0x000E0000L
+//SPI_SHADER_USER_ACCUM_LSHS_0
+#define SPI_SHADER_USER_ACCUM_LSHS_0__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_LSHS_0__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_LSHS_1
+#define SPI_SHADER_USER_ACCUM_LSHS_1__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_LSHS_1__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_LSHS_2
+#define SPI_SHADER_USER_ACCUM_LSHS_2__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_LSHS_2__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_USER_ACCUM_LSHS_3
+#define SPI_SHADER_USER_ACCUM_LSHS_3__CONTRIBUTION__SHIFT                                                     0x0
+#define SPI_SHADER_USER_ACCUM_LSHS_3__CONTRIBUTION_MASK                                                       0x0000007FL
+//SPI_SHADER_PGM_LO_LS
+#define SPI_SHADER_PGM_LO_LS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_LO_LS__MEM_BASE_MASK                                                                   0xFFFFFFFFL
+//SPI_SHADER_PGM_HI_LS
+#define SPI_SHADER_PGM_HI_LS__MEM_BASE__SHIFT                                                                 0x0
+#define SPI_SHADER_PGM_HI_LS__MEM_BASE_MASK                                                                   0xFFL
+//COMPUTE_DISPATCH_INITIATOR
+#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN__SHIFT                                                  0x0
+#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN__SHIFT                                                      0x1
+#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000__SHIFT                                                 0x2
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL__SHIFT                                                0x3
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE__SHIFT                                                0x4
+#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS__SHIFT                                              0x5
+#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE__SHIFT                                                         0x6
+#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL__SHIFT                                                  0xa
+#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL__SHIFT                                                  0xb
+#define COMPUTE_DISPATCH_INITIATOR__RESERVED__SHIFT                                                           0xc
+#define COMPUTE_DISPATCH_INITIATOR__TUNNEL_ENABLE__SHIFT                                                      0xd
+#define COMPUTE_DISPATCH_INITIATOR__RESTORE__SHIFT                                                            0xe
+#define COMPUTE_DISPATCH_INITIATOR__CS_W32_EN__SHIFT                                                          0xf
+#define COMPUTE_DISPATCH_INITIATOR__AMP_SHADER_EN__SHIFT                                                      0x10
+#define COMPUTE_DISPATCH_INITIATOR__DISABLE_DISP_PREMPT_EN__SHIFT                                             0x11
+#define COMPUTE_DISPATCH_INITIATOR__COMPUTE_SHADER_EN_MASK                                                    0x00000001L
+#define COMPUTE_DISPATCH_INITIATOR__PARTIAL_TG_EN_MASK                                                        0x00000002L
+#define COMPUTE_DISPATCH_INITIATOR__FORCE_START_AT_000_MASK                                                   0x00000004L
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_ENBL_MASK                                                  0x00000008L
+#define COMPUTE_DISPATCH_INITIATOR__ORDERED_APPEND_MODE_MASK                                                  0x00000010L
+#define COMPUTE_DISPATCH_INITIATOR__USE_THREAD_DIMENSIONS_MASK                                                0x00000020L
+#define COMPUTE_DISPATCH_INITIATOR__ORDER_MODE_MASK                                                           0x00000040L
+#define COMPUTE_DISPATCH_INITIATOR__SCALAR_L1_INV_VOL_MASK                                                    0x00000400L
+#define COMPUTE_DISPATCH_INITIATOR__VECTOR_L1_INV_VOL_MASK                                                    0x00000800L
+#define COMPUTE_DISPATCH_INITIATOR__RESERVED_MASK                                                             0x00001000L
+#define COMPUTE_DISPATCH_INITIATOR__TUNNEL_ENABLE_MASK                                                        0x00002000L
+#define COMPUTE_DISPATCH_INITIATOR__RESTORE_MASK                                                              0x00004000L
+#define COMPUTE_DISPATCH_INITIATOR__CS_W32_EN_MASK                                                            0x00008000L
+#define COMPUTE_DISPATCH_INITIATOR__AMP_SHADER_EN_MASK                                                        0x00010000L
+#define COMPUTE_DISPATCH_INITIATOR__DISABLE_DISP_PREMPT_EN_MASK                                               0x00020000L
+//COMPUTE_DIM_X
+#define COMPUTE_DIM_X__SIZE__SHIFT                                                                            0x0
+#define COMPUTE_DIM_X__SIZE_MASK                                                                              0xFFFFFFFFL
+//COMPUTE_DIM_Y
+#define COMPUTE_DIM_Y__SIZE__SHIFT                                                                            0x0
+#define COMPUTE_DIM_Y__SIZE_MASK                                                                              0xFFFFFFFFL
+//COMPUTE_DIM_Z
+#define COMPUTE_DIM_Z__SIZE__SHIFT                                                                            0x0
+#define COMPUTE_DIM_Z__SIZE_MASK                                                                              0xFFFFFFFFL
+//COMPUTE_START_X
+#define COMPUTE_START_X__START__SHIFT                                                                         0x0
+#define COMPUTE_START_X__START_MASK                                                                           0xFFFFFFFFL
+//COMPUTE_START_Y
+#define COMPUTE_START_Y__START__SHIFT                                                                         0x0
+#define COMPUTE_START_Y__START_MASK                                                                           0xFFFFFFFFL
+//COMPUTE_START_Z
+#define COMPUTE_START_Z__START__SHIFT                                                                         0x0
+#define COMPUTE_START_Z__START_MASK                                                                           0xFFFFFFFFL
+//COMPUTE_NUM_THREAD_X
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL__SHIFT                                                          0x0
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_FULL_MASK                                                            0x0000FFFFL
+#define COMPUTE_NUM_THREAD_X__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
+//COMPUTE_NUM_THREAD_Y
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL__SHIFT                                                          0x0
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_FULL_MASK                                                            0x0000FFFFL
+#define COMPUTE_NUM_THREAD_Y__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
+//COMPUTE_NUM_THREAD_Z
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL__SHIFT                                                          0x0
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL__SHIFT                                                       0x10
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_FULL_MASK                                                            0x0000FFFFL
+#define COMPUTE_NUM_THREAD_Z__NUM_THREAD_PARTIAL_MASK                                                         0xFFFF0000L
+//COMPUTE_PIPELINESTAT_ENABLE
+#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE__SHIFT                                               0x0
+#define COMPUTE_PIPELINESTAT_ENABLE__PIPELINESTAT_ENABLE_MASK                                                 0x00000001L
+//COMPUTE_PERFCOUNT_ENABLE
+#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE__SHIFT                                                     0x0
+#define COMPUTE_PERFCOUNT_ENABLE__PERFCOUNT_ENABLE_MASK                                                       0x00000001L
+//COMPUTE_PGM_LO
+#define COMPUTE_PGM_LO__DATA__SHIFT                                                                           0x0
+#define COMPUTE_PGM_LO__DATA_MASK                                                                             0xFFFFFFFFL
+//COMPUTE_PGM_HI
+#define COMPUTE_PGM_HI__DATA__SHIFT                                                                           0x0
+#define COMPUTE_PGM_HI__DATA_MASK                                                                             0x000000FFL
+//COMPUTE_DISPATCH_PKT_ADDR_LO
+#define COMPUTE_DISPATCH_PKT_ADDR_LO__DATA__SHIFT                                                             0x0
+#define COMPUTE_DISPATCH_PKT_ADDR_LO__DATA_MASK                                                               0xFFFFFFFFL
+//COMPUTE_DISPATCH_PKT_ADDR_HI
+#define COMPUTE_DISPATCH_PKT_ADDR_HI__DATA__SHIFT                                                             0x0
+#define COMPUTE_DISPATCH_PKT_ADDR_HI__DATA_MASK                                                               0x000000FFL
+//COMPUTE_DISPATCH_SCRATCH_BASE_LO
+#define COMPUTE_DISPATCH_SCRATCH_BASE_LO__DATA__SHIFT                                                         0x0
+#define COMPUTE_DISPATCH_SCRATCH_BASE_LO__DATA_MASK                                                           0xFFFFFFFFL
+//COMPUTE_DISPATCH_SCRATCH_BASE_HI
+#define COMPUTE_DISPATCH_SCRATCH_BASE_HI__DATA__SHIFT                                                         0x0
+#define COMPUTE_DISPATCH_SCRATCH_BASE_HI__DATA_MASK                                                           0x000000FFL
+//COMPUTE_PGM_RSRC1
+#define COMPUTE_PGM_RSRC1__VGPRS__SHIFT                                                                       0x0
+#define COMPUTE_PGM_RSRC1__SGPRS__SHIFT                                                                       0x6
+#define COMPUTE_PGM_RSRC1__PRIORITY__SHIFT                                                                    0xa
+#define COMPUTE_PGM_RSRC1__FLOAT_MODE__SHIFT                                                                  0xc
+#define COMPUTE_PGM_RSRC1__PRIV__SHIFT                                                                        0x14
+#define COMPUTE_PGM_RSRC1__DX10_CLAMP__SHIFT                                                                  0x15
+#define COMPUTE_PGM_RSRC1__DEBUG_MODE__SHIFT                                                                  0x16
+#define COMPUTE_PGM_RSRC1__IEEE_MODE__SHIFT                                                                   0x17
+#define COMPUTE_PGM_RSRC1__BULKY__SHIFT                                                                       0x18
+#define COMPUTE_PGM_RSRC1__CDBG_USER__SHIFT                                                                   0x19
+#define COMPUTE_PGM_RSRC1__FP16_OVFL__SHIFT                                                                   0x1a
+#define COMPUTE_PGM_RSRC1__WGP_MODE__SHIFT                                                                    0x1d
+#define COMPUTE_PGM_RSRC1__MEM_ORDERED__SHIFT                                                                 0x1e
+#define COMPUTE_PGM_RSRC1__FWD_PROGRESS__SHIFT                                                                0x1f
+#define COMPUTE_PGM_RSRC1__VGPRS_MASK                                                                         0x0000003FL
+#define COMPUTE_PGM_RSRC1__SGPRS_MASK                                                                         0x000003C0L
+#define COMPUTE_PGM_RSRC1__PRIORITY_MASK                                                                      0x00000C00L
+#define COMPUTE_PGM_RSRC1__FLOAT_MODE_MASK                                                                    0x000FF000L
+#define COMPUTE_PGM_RSRC1__PRIV_MASK                                                                          0x00100000L
+#define COMPUTE_PGM_RSRC1__DX10_CLAMP_MASK                                                                    0x00200000L
+#define COMPUTE_PGM_RSRC1__DEBUG_MODE_MASK                                                                    0x00400000L
+#define COMPUTE_PGM_RSRC1__IEEE_MODE_MASK                                                                     0x00800000L
+#define COMPUTE_PGM_RSRC1__BULKY_MASK                                                                         0x01000000L
+#define COMPUTE_PGM_RSRC1__CDBG_USER_MASK                                                                     0x02000000L
+#define COMPUTE_PGM_RSRC1__FP16_OVFL_MASK                                                                     0x04000000L
+#define COMPUTE_PGM_RSRC1__WGP_MODE_MASK                                                                      0x20000000L
+#define COMPUTE_PGM_RSRC1__MEM_ORDERED_MASK                                                                   0x40000000L
+#define COMPUTE_PGM_RSRC1__FWD_PROGRESS_MASK                                                                  0x80000000L
+//COMPUTE_PGM_RSRC2
+#define COMPUTE_PGM_RSRC2__SCRATCH_EN__SHIFT                                                                  0x0
+#define COMPUTE_PGM_RSRC2__USER_SGPR__SHIFT                                                                   0x1
+#define COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT                                                                0x6
+#define COMPUTE_PGM_RSRC2__TGID_X_EN__SHIFT                                                                   0x7
+#define COMPUTE_PGM_RSRC2__TGID_Y_EN__SHIFT                                                                   0x8
+#define COMPUTE_PGM_RSRC2__TGID_Z_EN__SHIFT                                                                   0x9
+#define COMPUTE_PGM_RSRC2__TG_SIZE_EN__SHIFT                                                                  0xa
+#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT__SHIFT                                                              0xb
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB__SHIFT                                                                 0xd
+#define COMPUTE_PGM_RSRC2__LDS_SIZE__SHIFT                                                                    0xf
+#define COMPUTE_PGM_RSRC2__EXCP_EN__SHIFT                                                                     0x18
+#define COMPUTE_PGM_RSRC2__SCRATCH_EN_MASK                                                                    0x00000001L
+#define COMPUTE_PGM_RSRC2__USER_SGPR_MASK                                                                     0x0000003EL
+#define COMPUTE_PGM_RSRC2__TRAP_PRESENT_MASK                                                                  0x00000040L
+#define COMPUTE_PGM_RSRC2__TGID_X_EN_MASK                                                                     0x00000080L
+#define COMPUTE_PGM_RSRC2__TGID_Y_EN_MASK                                                                     0x00000100L
+#define COMPUTE_PGM_RSRC2__TGID_Z_EN_MASK                                                                     0x00000200L
+#define COMPUTE_PGM_RSRC2__TG_SIZE_EN_MASK                                                                    0x00000400L
+#define COMPUTE_PGM_RSRC2__TIDIG_COMP_CNT_MASK                                                                0x00001800L
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MSB_MASK                                                                   0x00006000L
+#define COMPUTE_PGM_RSRC2__LDS_SIZE_MASK                                                                      0x00FF8000L
+#define COMPUTE_PGM_RSRC2__EXCP_EN_MASK                                                                       0x7F000000L
+//COMPUTE_VMID
+#define COMPUTE_VMID__DATA__SHIFT                                                                             0x0
+#define COMPUTE_VMID__DATA_MASK                                                                               0x0000000FL
+//COMPUTE_RESOURCE_LIMITS
+#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH__SHIFT                                                          0x0
+#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU__SHIFT                                                             0xc
+#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD__SHIFT                                                        0x10
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL__SHIFT                                                        0x16
+#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST__SHIFT                                                       0x17
+#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT__SHIFT                                                        0x18
+#define COMPUTE_RESOURCE_LIMITS__WAVES_PER_SH_MASK                                                            0x000003FFL
+#define COMPUTE_RESOURCE_LIMITS__TG_PER_CU_MASK                                                               0x0000F000L
+#define COMPUTE_RESOURCE_LIMITS__LOCK_THRESHOLD_MASK                                                          0x003F0000L
+#define COMPUTE_RESOURCE_LIMITS__SIMD_DEST_CNTL_MASK                                                          0x00400000L
+#define COMPUTE_RESOURCE_LIMITS__FORCE_SIMD_DIST_MASK                                                         0x00800000L
+#define COMPUTE_RESOURCE_LIMITS__CU_GROUP_COUNT_MASK                                                          0x07000000L
+//COMPUTE_DESTINATION_EN_SE0
+#define COMPUTE_DESTINATION_EN_SE0__CU_EN__SHIFT                                                              0x0
+#define COMPUTE_DESTINATION_EN_SE0__CU_EN_MASK                                                                0xFFFFFFFFL
+//COMPUTE_STATIC_THREAD_MGMT_SE0
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE0__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_DESTINATION_EN_SE1
+#define COMPUTE_DESTINATION_EN_SE1__CU_EN__SHIFT                                                              0x0
+#define COMPUTE_DESTINATION_EN_SE1__CU_EN_MASK                                                                0xFFFFFFFFL
+//COMPUTE_STATIC_THREAD_MGMT_SE1
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE1__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_TMPRING_SIZE
+#define COMPUTE_TMPRING_SIZE__WAVES__SHIFT                                                                    0x0
+#define COMPUTE_TMPRING_SIZE__WAVESIZE__SHIFT                                                                 0xc
+#define COMPUTE_TMPRING_SIZE__WAVES_MASK                                                                      0x00000FFFL
+#define COMPUTE_TMPRING_SIZE__WAVESIZE_MASK                                                                   0x07FFF000L
+//COMPUTE_DESTINATION_EN_SE2
+#define COMPUTE_DESTINATION_EN_SE2__CU_EN__SHIFT                                                              0x0
+#define COMPUTE_DESTINATION_EN_SE2__CU_EN_MASK                                                                0xFFFFFFFFL
+//COMPUTE_STATIC_THREAD_MGMT_SE2
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE2__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_DESTINATION_EN_SE3
+#define COMPUTE_DESTINATION_EN_SE3__CU_EN__SHIFT                                                              0x0
+#define COMPUTE_DESTINATION_EN_SE3__CU_EN_MASK                                                                0xFFFFFFFFL
+//COMPUTE_STATIC_THREAD_MGMT_SE3
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE3__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_RESTART_X
+#define COMPUTE_RESTART_X__RESTART__SHIFT                                                                     0x0
+#define COMPUTE_RESTART_X__RESTART_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_RESTART_Y
+#define COMPUTE_RESTART_Y__RESTART__SHIFT                                                                     0x0
+#define COMPUTE_RESTART_Y__RESTART_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_RESTART_Z
+#define COMPUTE_RESTART_Z__RESTART__SHIFT                                                                     0x0
+#define COMPUTE_RESTART_Z__RESTART_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_THREAD_TRACE_ENABLE
+#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE__SHIFT                                               0x0
+#define COMPUTE_THREAD_TRACE_ENABLE__THREAD_TRACE_ENABLE_MASK                                                 0x00000001L
+//COMPUTE_MISC_RESERVED
+#define COMPUTE_MISC_RESERVED__SEND_SEID__SHIFT                                                               0x0
+#define COMPUTE_MISC_RESERVED__RESERVED3__SHIFT                                                               0x3
+#define COMPUTE_MISC_RESERVED__RESERVED4__SHIFT                                                               0x4
+#define COMPUTE_MISC_RESERVED__WAVE_ID_BASE__SHIFT                                                            0x5
+#define COMPUTE_MISC_RESERVED__SEND_SEID_MASK                                                                 0x00000007L
+#define COMPUTE_MISC_RESERVED__RESERVED3_MASK                                                                 0x00000008L
+#define COMPUTE_MISC_RESERVED__RESERVED4_MASK                                                                 0x00000010L
+#define COMPUTE_MISC_RESERVED__WAVE_ID_BASE_MASK                                                              0x0001FFE0L
+//COMPUTE_DISPATCH_ID
+#define COMPUTE_DISPATCH_ID__DISPATCH_ID__SHIFT                                                               0x0
+#define COMPUTE_DISPATCH_ID__DISPATCH_ID_MASK                                                                 0xFFFFFFFFL
+//COMPUTE_THREADGROUP_ID
+#define COMPUTE_THREADGROUP_ID__THREADGROUP_ID__SHIFT                                                         0x0
+#define COMPUTE_THREADGROUP_ID__THREADGROUP_ID_MASK                                                           0xFFFFFFFFL
+//COMPUTE_REQ_CTRL
+#define COMPUTE_REQ_CTRL__SOFT_GROUPING_EN__SHIFT                                                             0x0
+#define COMPUTE_REQ_CTRL__NUMBER_OF_REQUESTS_PER_CU__SHIFT                                                    0x1
+#define COMPUTE_REQ_CTRL__SOFT_GROUPING_ALLOCATION_TIMEOUT__SHIFT                                             0x5
+#define COMPUTE_REQ_CTRL__HARD_LOCK_HYSTERESIS__SHIFT                                                         0x9
+#define COMPUTE_REQ_CTRL__HARD_LOCK_LOW_THRESHOLD__SHIFT                                                      0xa
+#define COMPUTE_REQ_CTRL__PRODUCER_REQUEST_LOCKOUT__SHIFT                                                     0xf
+#define COMPUTE_REQ_CTRL__GLOBAL_SCANNING_EN__SHIFT                                                           0x10
+#define COMPUTE_REQ_CTRL__ALLOCATION_RATE_THROTTLING_THRESHOLD__SHIFT                                         0x11
+#define COMPUTE_REQ_CTRL__DEDICATED_PREALLOCATION_BUFFER_LIMIT__SHIFT                                         0x14
+#define COMPUTE_REQ_CTRL__SOFT_GROUPING_EN_MASK                                                               0x00000001L
+#define COMPUTE_REQ_CTRL__NUMBER_OF_REQUESTS_PER_CU_MASK                                                      0x0000001EL
+#define COMPUTE_REQ_CTRL__SOFT_GROUPING_ALLOCATION_TIMEOUT_MASK                                               0x000001E0L
+#define COMPUTE_REQ_CTRL__HARD_LOCK_HYSTERESIS_MASK                                                           0x00000200L
+#define COMPUTE_REQ_CTRL__HARD_LOCK_LOW_THRESHOLD_MASK                                                        0x00007C00L
+#define COMPUTE_REQ_CTRL__PRODUCER_REQUEST_LOCKOUT_MASK                                                       0x00008000L
+#define COMPUTE_REQ_CTRL__GLOBAL_SCANNING_EN_MASK                                                             0x00010000L
+#define COMPUTE_REQ_CTRL__ALLOCATION_RATE_THROTTLING_THRESHOLD_MASK                                           0x000E0000L
+#define COMPUTE_REQ_CTRL__DEDICATED_PREALLOCATION_BUFFER_LIMIT_MASK                                           0x07F00000L
+//COMPUTE_USER_ACCUM_0
+#define COMPUTE_USER_ACCUM_0__CONTRIBUTION__SHIFT                                                             0x0
+#define COMPUTE_USER_ACCUM_0__CONTRIBUTION_MASK                                                               0x0000007FL
+//COMPUTE_USER_ACCUM_1
+#define COMPUTE_USER_ACCUM_1__CONTRIBUTION__SHIFT                                                             0x0
+#define COMPUTE_USER_ACCUM_1__CONTRIBUTION_MASK                                                               0x0000007FL
+//COMPUTE_USER_ACCUM_2
+#define COMPUTE_USER_ACCUM_2__CONTRIBUTION__SHIFT                                                             0x0
+#define COMPUTE_USER_ACCUM_2__CONTRIBUTION_MASK                                                               0x0000007FL
+//COMPUTE_USER_ACCUM_3
+#define COMPUTE_USER_ACCUM_3__CONTRIBUTION__SHIFT                                                             0x0
+#define COMPUTE_USER_ACCUM_3__CONTRIBUTION_MASK                                                               0x0000007FL
+//COMPUTE_PGM_RSRC3
+#define COMPUTE_PGM_RSRC3__SHARED_VGPR_CNT__SHIFT                                                             0x0
+#define COMPUTE_PGM_RSRC3__INST_PREF_SIZE__SHIFT                                                              0x4
+#define COMPUTE_PGM_RSRC3__TRAP_ON_START__SHIFT                                                               0xa
+#define COMPUTE_PGM_RSRC3__TRAP_ON_END__SHIFT                                                                 0xb
+#define COMPUTE_PGM_RSRC3__IMAGE_OP__SHIFT                                                                    0x1f
+#define COMPUTE_PGM_RSRC3__SHARED_VGPR_CNT_MASK                                                               0x0000000FL
+#define COMPUTE_PGM_RSRC3__INST_PREF_SIZE_MASK                                                                0x000003F0L
+#define COMPUTE_PGM_RSRC3__TRAP_ON_START_MASK                                                                 0x00000400L
+#define COMPUTE_PGM_RSRC3__TRAP_ON_END_MASK                                                                   0x00000800L
+#define COMPUTE_PGM_RSRC3__IMAGE_OP_MASK                                                                      0x80000000L
+//COMPUTE_DDID_INDEX
+#define COMPUTE_DDID_INDEX__INDEX__SHIFT                                                                      0x0
+#define COMPUTE_DDID_INDEX__INDEX_MASK                                                                        0x000007FFL
+//COMPUTE_SHADER_CHKSUM
+#define COMPUTE_SHADER_CHKSUM__CHECKSUM__SHIFT                                                                0x0
+#define COMPUTE_SHADER_CHKSUM__CHECKSUM_MASK                                                                  0xFFFFFFFFL
+//COMPUTE_STATIC_THREAD_MGMT_SE4
+#define COMPUTE_STATIC_THREAD_MGMT_SE4__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE4__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE4__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE4__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_STATIC_THREAD_MGMT_SE5
+#define COMPUTE_STATIC_THREAD_MGMT_SE5__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE5__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE5__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE5__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_STATIC_THREAD_MGMT_SE6
+#define COMPUTE_STATIC_THREAD_MGMT_SE6__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE6__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE6__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE6__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_STATIC_THREAD_MGMT_SE7
+#define COMPUTE_STATIC_THREAD_MGMT_SE7__SA0_CU_EN__SHIFT                                                      0x0
+#define COMPUTE_STATIC_THREAD_MGMT_SE7__SA1_CU_EN__SHIFT                                                      0x10
+#define COMPUTE_STATIC_THREAD_MGMT_SE7__SA0_CU_EN_MASK                                                        0x0000FFFFL
+#define COMPUTE_STATIC_THREAD_MGMT_SE7__SA1_CU_EN_MASK                                                        0xFFFF0000L
+//COMPUTE_DISPATCH_INTERLEAVE
+#define COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE__SHIFT                                                        0x0
+#define COMPUTE_DISPATCH_INTERLEAVE__INTERLEAVE_MASK                                                          0x000003FFL
+//COMPUTE_RELAUNCH
+#define COMPUTE_RELAUNCH__PAYLOAD__SHIFT                                                                      0x0
+#define COMPUTE_RELAUNCH__IS_EVENT__SHIFT                                                                     0x1e
+#define COMPUTE_RELAUNCH__IS_STATE__SHIFT                                                                     0x1f
+#define COMPUTE_RELAUNCH__PAYLOAD_MASK                                                                        0x3FFFFFFFL
+#define COMPUTE_RELAUNCH__IS_EVENT_MASK                                                                       0x40000000L
+#define COMPUTE_RELAUNCH__IS_STATE_MASK                                                                       0x80000000L
+//COMPUTE_WAVE_RESTORE_ADDR_LO
+#define COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR__SHIFT                                                             0x0
+#define COMPUTE_WAVE_RESTORE_ADDR_LO__ADDR_MASK                                                               0xFFFFFFFFL
+//COMPUTE_WAVE_RESTORE_ADDR_HI
+#define COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR__SHIFT                                                             0x0
+#define COMPUTE_WAVE_RESTORE_ADDR_HI__ADDR_MASK                                                               0xFFFFL
+//COMPUTE_RELAUNCH2
+#define COMPUTE_RELAUNCH2__PAYLOAD__SHIFT                                                                     0x0
+#define COMPUTE_RELAUNCH2__IS_EVENT__SHIFT                                                                    0x1e
+#define COMPUTE_RELAUNCH2__IS_STATE__SHIFT                                                                    0x1f
+#define COMPUTE_RELAUNCH2__PAYLOAD_MASK                                                                       0x3FFFFFFFL
+#define COMPUTE_RELAUNCH2__IS_EVENT_MASK                                                                      0x40000000L
+#define COMPUTE_RELAUNCH2__IS_STATE_MASK                                                                      0x80000000L
+//COMPUTE_USER_DATA_0
+#define COMPUTE_USER_DATA_0__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_0__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_1
+#define COMPUTE_USER_DATA_1__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_1__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_2
+#define COMPUTE_USER_DATA_2__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_2__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_3
+#define COMPUTE_USER_DATA_3__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_3__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_4
+#define COMPUTE_USER_DATA_4__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_4__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_5
+#define COMPUTE_USER_DATA_5__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_5__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_6
+#define COMPUTE_USER_DATA_6__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_6__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_7
+#define COMPUTE_USER_DATA_7__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_7__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_8
+#define COMPUTE_USER_DATA_8__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_8__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_9
+#define COMPUTE_USER_DATA_9__DATA__SHIFT                                                                      0x0
+#define COMPUTE_USER_DATA_9__DATA_MASK                                                                        0xFFFFFFFFL
+//COMPUTE_USER_DATA_10
+#define COMPUTE_USER_DATA_10__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_10__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_11
+#define COMPUTE_USER_DATA_11__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_11__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_12
+#define COMPUTE_USER_DATA_12__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_12__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_13
+#define COMPUTE_USER_DATA_13__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_13__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_14
+#define COMPUTE_USER_DATA_14__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_14__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_USER_DATA_15
+#define COMPUTE_USER_DATA_15__DATA__SHIFT                                                                     0x0
+#define COMPUTE_USER_DATA_15__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_DISPATCH_TUNNEL
+#define COMPUTE_DISPATCH_TUNNEL__OFF_DELAY__SHIFT                                                             0x0
+#define COMPUTE_DISPATCH_TUNNEL__IMMEDIATE__SHIFT                                                             0xa
+#define COMPUTE_DISPATCH_TUNNEL__OFF_DELAY_MASK                                                               0x000003FFL
+#define COMPUTE_DISPATCH_TUNNEL__IMMEDIATE_MASK                                                               0x00000400L
+//COMPUTE_DISPATCH_END
+#define COMPUTE_DISPATCH_END__DATA__SHIFT                                                                     0x0
+#define COMPUTE_DISPATCH_END__DATA_MASK                                                                       0xFFFFFFFFL
+//COMPUTE_NOWHERE
+#define COMPUTE_NOWHERE__DATA__SHIFT                                                                          0x0
+#define COMPUTE_NOWHERE__DATA_MASK                                                                            0xFFFFFFFFL
+//SH_RESERVED_REG0
+#define SH_RESERVED_REG0__DATA__SHIFT                                                                         0x0
+#define SH_RESERVED_REG0__DATA_MASK                                                                           0xFFFFFFFFL
+//SH_RESERVED_REG1
+#define SH_RESERVED_REG1__DATA__SHIFT                                                                         0x0
+#define SH_RESERVED_REG1__DATA_MASK                                                                           0xFFFFFFFFL
+
+
+// addressBlock: gc_cppdec
+//CP_CU_MASK_ADDR_LO
+#define CP_CU_MASK_ADDR_LO__ADDR_LO__SHIFT                                                                    0x2
+#define CP_CU_MASK_ADDR_LO__ADDR_LO_MASK                                                                      0xFFFFFFFCL
+//CP_CU_MASK_ADDR_HI
+#define CP_CU_MASK_ADDR_HI__ADDR_HI__SHIFT                                                                    0x0
+#define CP_CU_MASK_ADDR_HI__ADDR_HI_MASK                                                                      0xFFFFFFFFL
+//CP_CU_MASK_CNTL
+#define CP_CU_MASK_CNTL__POLICY__SHIFT                                                                        0x0
+#define CP_CU_MASK_CNTL__POLICY_MASK                                                                          0x00000001L
+//CP_EOPQ_WAIT_TIME
+#define CP_EOPQ_WAIT_TIME__WAIT_TIME__SHIFT                                                                   0x0
+#define CP_EOPQ_WAIT_TIME__SCALE_COUNT__SHIFT                                                                 0xa
+#define CP_EOPQ_WAIT_TIME__WAIT_TIME_MASK                                                                     0x000003FFL
+#define CP_EOPQ_WAIT_TIME__SCALE_COUNT_MASK                                                                   0x0003FC00L
+//CP_CPC_MGCG_SYNC_CNTL
+#define CP_CPC_MGCG_SYNC_CNTL__COOLDOWN_PERIOD__SHIFT                                                         0x0
+#define CP_CPC_MGCG_SYNC_CNTL__WARMUP_PERIOD__SHIFT                                                           0x8
+#define CP_CPC_MGCG_SYNC_CNTL__COOLDOWN_PERIOD_MASK                                                           0x000000FFL
+#define CP_CPC_MGCG_SYNC_CNTL__WARMUP_PERIOD_MASK                                                             0x0000FF00L
+//CPC_INT_INFO
+#define CPC_INT_INFO__ADDR_HI__SHIFT                                                                          0x0
+#define CPC_INT_INFO__TYPE__SHIFT                                                                             0x10
+#define CPC_INT_INFO__VMID__SHIFT                                                                             0x14
+#define CPC_INT_INFO__QUEUE_ID__SHIFT                                                                         0x1c
+#define CPC_INT_INFO__ADDR_HI_MASK                                                                            0x0000FFFFL
+#define CPC_INT_INFO__TYPE_MASK                                                                               0x00010000L
+#define CPC_INT_INFO__VMID_MASK                                                                               0x00F00000L
+#define CPC_INT_INFO__QUEUE_ID_MASK                                                                           0x70000000L
+//CP_VIRT_STATUS
+#define CP_VIRT_STATUS__VIRT_STATUS__SHIFT                                                                    0x0
+#define CP_VIRT_STATUS__VIRT_STATUS_MASK                                                                      0xFFFFFFFFL
+//CPC_INT_ADDR
+#define CPC_INT_ADDR__ADDR__SHIFT                                                                             0x0
+#define CPC_INT_ADDR__ADDR_MASK                                                                               0xFFFFFFFFL
+//CPC_INT_PASID
+#define CPC_INT_PASID__PASID__SHIFT                                                                           0x0
+#define CPC_INT_PASID__BYPASS_PASID__SHIFT                                                                    0x10
+#define CPC_INT_PASID__PASID_MASK                                                                             0x0000FFFFL
+#define CPC_INT_PASID__BYPASS_PASID_MASK                                                                      0x00010000L
+//CP_GFX_ERROR
+#define CP_GFX_ERROR__ME_INSTR_CACHE_UTCL1_ERROR__SHIFT                                                       0x0
+#define CP_GFX_ERROR__PFP_INSTR_CACHE_UTCL1_ERROR__SHIFT                                                      0x1
+#define CP_GFX_ERROR__DDID_DRAW_UTCL1_ERROR__SHIFT                                                            0x2
+#define CP_GFX_ERROR__DDID_DISPATCH_UTCL1_ERROR__SHIFT                                                        0x3
+#define CP_GFX_ERROR__SUA_ERROR__SHIFT                                                                        0x4
+#define CP_GFX_ERROR__DATA_FETCHER_UTCL1_ERROR__SHIFT                                                         0x6
+#define CP_GFX_ERROR__SEM_UTCL1_ERROR__SHIFT                                                                  0x7
+#define CP_GFX_ERROR__QU_EOP_UTCL1_ERROR__SHIFT                                                               0x9
+#define CP_GFX_ERROR__QU_PIPE_UTCL1_ERROR__SHIFT                                                              0xa
+#define CP_GFX_ERROR__QU_READ_UTCL1_ERROR__SHIFT                                                              0xb
+#define CP_GFX_ERROR__SYNC_MEMRD_UTCL1_ERROR__SHIFT                                                           0xc
+#define CP_GFX_ERROR__SYNC_MEMWR_UTCL1_ERROR__SHIFT                                                           0xd
+#define CP_GFX_ERROR__SHADOW_UTCL1_ERROR__SHIFT                                                               0xe
+#define CP_GFX_ERROR__APPEND_UTCL1_ERROR__SHIFT                                                               0xf
+#define CP_GFX_ERROR__DMA_SRC_UTCL1_ERROR__SHIFT                                                              0x12
+#define CP_GFX_ERROR__DMA_DST_UTCL1_ERROR__SHIFT                                                              0x13
+#define CP_GFX_ERROR__PFP_TC_UTCL1_ERROR__SHIFT                                                               0x14
+#define CP_GFX_ERROR__ME_TC_UTCL1_ERROR__SHIFT                                                                0x15
+#define CP_GFX_ERROR__PRT_LOD_UTCL1_ERROR__SHIFT                                                              0x17
+#define CP_GFX_ERROR__RDPTR_RPT_UTCL1_ERROR__SHIFT                                                            0x18
+#define CP_GFX_ERROR__RB_FETCHER_UTCL1_ERROR__SHIFT                                                           0x19
+#define CP_GFX_ERROR__I1_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1a
+#define CP_GFX_ERROR__I2_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1b
+#define CP_GFX_ERROR__ST_FETCHER_UTCL1_ERROR__SHIFT                                                           0x1e
+#define CP_GFX_ERROR__RESERVED__SHIFT                                                                         0x1f
+#define CP_GFX_ERROR__ME_INSTR_CACHE_UTCL1_ERROR_MASK                                                         0x00000001L
+#define CP_GFX_ERROR__PFP_INSTR_CACHE_UTCL1_ERROR_MASK                                                        0x00000002L
+#define CP_GFX_ERROR__DDID_DRAW_UTCL1_ERROR_MASK                                                              0x00000004L
+#define CP_GFX_ERROR__DDID_DISPATCH_UTCL1_ERROR_MASK                                                          0x00000008L
+#define CP_GFX_ERROR__SUA_ERROR_MASK                                                                          0x00000010L
+#define CP_GFX_ERROR__DATA_FETCHER_UTCL1_ERROR_MASK                                                           0x00000040L
+#define CP_GFX_ERROR__SEM_UTCL1_ERROR_MASK                                                                    0x00000080L
+#define CP_GFX_ERROR__QU_EOP_UTCL1_ERROR_MASK                                                                 0x00000200L
+#define CP_GFX_ERROR__QU_PIPE_UTCL1_ERROR_MASK                                                                0x00000400L
+#define CP_GFX_ERROR__QU_READ_UTCL1_ERROR_MASK                                                                0x00000800L
+#define CP_GFX_ERROR__SYNC_MEMRD_UTCL1_ERROR_MASK                                                             0x00001000L
+#define CP_GFX_ERROR__SYNC_MEMWR_UTCL1_ERROR_MASK                                                             0x00002000L
+#define CP_GFX_ERROR__SHADOW_UTCL1_ERROR_MASK                                                                 0x00004000L
+#define CP_GFX_ERROR__APPEND_UTCL1_ERROR_MASK                                                                 0x00008000L
+#define CP_GFX_ERROR__DMA_SRC_UTCL1_ERROR_MASK                                                                0x00040000L
+#define CP_GFX_ERROR__DMA_DST_UTCL1_ERROR_MASK                                                                0x00080000L
+#define CP_GFX_ERROR__PFP_TC_UTCL1_ERROR_MASK                                                                 0x00100000L
+#define CP_GFX_ERROR__ME_TC_UTCL1_ERROR_MASK                                                                  0x00200000L
+#define CP_GFX_ERROR__PRT_LOD_UTCL1_ERROR_MASK                                                                0x00800000L
+#define CP_GFX_ERROR__RDPTR_RPT_UTCL1_ERROR_MASK                                                              0x01000000L
+#define CP_GFX_ERROR__RB_FETCHER_UTCL1_ERROR_MASK                                                             0x02000000L
+#define CP_GFX_ERROR__I1_FETCHER_UTCL1_ERROR_MASK                                                             0x04000000L
+#define CP_GFX_ERROR__I2_FETCHER_UTCL1_ERROR_MASK                                                             0x08000000L
+#define CP_GFX_ERROR__ST_FETCHER_UTCL1_ERROR_MASK                                                             0x40000000L
+#define CP_GFX_ERROR__RESERVED_MASK                                                                           0x80000000L
+//CPG_UTCL1_CNTL
+#define CPG_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
+#define CPG_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                0x17
+#define CPG_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
+#define CPG_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
+#define CPG_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
+#define CPG_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
+#define CPG_UTCL1_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                          0x1d
+#define CPG_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
+#define CPG_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
+#define CPG_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                  0x00800000L
+#define CPG_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
+#define CPG_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
+#define CPG_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
+#define CPG_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
+#define CPG_UTCL1_CNTL__IGNORE_PTE_PERMISSION_MASK                                                            0x20000000L
+#define CPG_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
+//CPC_UTCL1_CNTL
+#define CPC_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
+#define CPC_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
+#define CPC_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
+#define CPC_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
+#define CPC_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
+#define CPC_UTCL1_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                          0x1d
+#define CPC_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
+#define CPC_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
+#define CPC_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
+#define CPC_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
+#define CPC_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
+#define CPC_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
+#define CPC_UTCL1_CNTL__IGNORE_PTE_PERMISSION_MASK                                                            0x20000000L
+#define CPC_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
+//CPF_UTCL1_CNTL
+#define CPF_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                           0x0
+#define CPF_UTCL1_CNTL__VMID_RESET_MODE__SHIFT                                                                0x17
+#define CPF_UTCL1_CNTL__DROP_MODE__SHIFT                                                                      0x18
+#define CPF_UTCL1_CNTL__INVALIDATE__SHIFT                                                                     0x1a
+#define CPF_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                                0x1b
+#define CPF_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                    0x1c
+#define CPF_UTCL1_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                          0x1d
+#define CPF_UTCL1_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                              0x1e
+#define CPF_UTCL1_CNTL__FORCE_NO_EXE__SHIFT                                                                   0x1f
+#define CPF_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                             0x000FFFFFL
+#define CPF_UTCL1_CNTL__VMID_RESET_MODE_MASK                                                                  0x00800000L
+#define CPF_UTCL1_CNTL__DROP_MODE_MASK                                                                        0x01000000L
+#define CPF_UTCL1_CNTL__INVALIDATE_MASK                                                                       0x04000000L
+#define CPF_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                                  0x08000000L
+#define CPF_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                      0x10000000L
+#define CPF_UTCL1_CNTL__IGNORE_PTE_PERMISSION_MASK                                                            0x20000000L
+#define CPF_UTCL1_CNTL__MTYPE_NO_PTE_MODE_MASK                                                                0x40000000L
+#define CPF_UTCL1_CNTL__FORCE_NO_EXE_MASK                                                                     0x80000000L
+//CP_AQL_SMM_STATUS
+#define CP_AQL_SMM_STATUS__AQL_QUEUE_SMM__SHIFT                                                               0x0
+#define CP_AQL_SMM_STATUS__AQL_QUEUE_SMM_MASK                                                                 0xFFFFFFFFL
+//CP_RB0_BASE
+#define CP_RB0_BASE__RB_BASE__SHIFT                                                                           0x0
+#define CP_RB0_BASE__RB_BASE_MASK                                                                             0xFFFFFFFFL
+//CP_RB_BASE
+#define CP_RB_BASE__RB_BASE__SHIFT                                                                            0x0
+#define CP_RB_BASE__RB_BASE_MASK                                                                              0xFFFFFFFFL
+//CP_RB0_CNTL
+#define CP_RB0_CNTL__RB_BUFSZ__SHIFT                                                                          0x0
+#define CP_RB0_CNTL__TMZ_STATE__SHIFT                                                                         0x6
+#define CP_RB0_CNTL__TMZ_MATCH__SHIFT                                                                         0x7
+#define CP_RB0_CNTL__RB_BLKSZ__SHIFT                                                                          0x8
+#define CP_RB0_CNTL__RB_NON_PRIV__SHIFT                                                                       0xf
+#define CP_RB0_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x14
+#define CP_RB0_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                    0x16
+#define CP_RB0_CNTL__CACHE_POLICY__SHIFT                                                                      0x18
+#define CP_RB0_CNTL__RB_VOLATILE__SHIFT                                                                       0x1a
+#define CP_RB0_CNTL__RB_NO_UPDATE__SHIFT                                                                      0x1b
+#define CP_RB0_CNTL__RB_EXE__SHIFT                                                                            0x1c
+#define CP_RB0_CNTL__KMD_QUEUE__SHIFT                                                                         0x1d
+#define CP_RB0_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                    0x1f
+#define CP_RB0_CNTL__RB_BUFSZ_MASK                                                                            0x0000003FL
+#define CP_RB0_CNTL__TMZ_STATE_MASK                                                                           0x00000040L
+#define CP_RB0_CNTL__TMZ_MATCH_MASK                                                                           0x00000080L
+#define CP_RB0_CNTL__RB_BLKSZ_MASK                                                                            0x00003F00L
+#define CP_RB0_CNTL__RB_NON_PRIV_MASK                                                                         0x00008000L
+#define CP_RB0_CNTL__MIN_AVAILSZ_MASK                                                                         0x00300000L
+#define CP_RB0_CNTL__MIN_IB_AVAILSZ_MASK                                                                      0x00C00000L
+#define CP_RB0_CNTL__CACHE_POLICY_MASK                                                                        0x03000000L
+#define CP_RB0_CNTL__RB_VOLATILE_MASK                                                                         0x04000000L
+#define CP_RB0_CNTL__RB_NO_UPDATE_MASK                                                                        0x08000000L
+#define CP_RB0_CNTL__RB_EXE_MASK                                                                              0x10000000L
+#define CP_RB0_CNTL__KMD_QUEUE_MASK                                                                           0x20000000L
+#define CP_RB0_CNTL__RB_RPTR_WR_ENA_MASK                                                                      0x80000000L
+//CP_RB_CNTL
+#define CP_RB_CNTL__RB_BUFSZ__SHIFT                                                                           0x0
+#define CP_RB_CNTL__TMZ_STATE__SHIFT                                                                          0x6
+#define CP_RB_CNTL__TMZ_MATCH__SHIFT                                                                          0x7
+#define CP_RB_CNTL__RB_BLKSZ__SHIFT                                                                           0x8
+#define CP_RB_CNTL__RB_NON_PRIV__SHIFT                                                                        0xf
+#define CP_RB_CNTL__MIN_AVAILSZ__SHIFT                                                                        0x14
+#define CP_RB_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                     0x16
+#define CP_RB_CNTL__CACHE_POLICY__SHIFT                                                                       0x18
+#define CP_RB_CNTL__RB_VOLATILE__SHIFT                                                                        0x1a
+#define CP_RB_CNTL__RB_NO_UPDATE__SHIFT                                                                       0x1b
+#define CP_RB_CNTL__RB_EXE__SHIFT                                                                             0x1c
+#define CP_RB_CNTL__KMD_QUEUE__SHIFT                                                                          0x1d
+#define CP_RB_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                     0x1f
+#define CP_RB_CNTL__RB_BUFSZ_MASK                                                                             0x0000003FL
+#define CP_RB_CNTL__TMZ_STATE_MASK                                                                            0x00000040L
+#define CP_RB_CNTL__TMZ_MATCH_MASK                                                                            0x00000080L
+#define CP_RB_CNTL__RB_BLKSZ_MASK                                                                             0x00003F00L
+#define CP_RB_CNTL__RB_NON_PRIV_MASK                                                                          0x00008000L
+#define CP_RB_CNTL__MIN_AVAILSZ_MASK                                                                          0x00300000L
+#define CP_RB_CNTL__MIN_IB_AVAILSZ_MASK                                                                       0x00C00000L
+#define CP_RB_CNTL__CACHE_POLICY_MASK                                                                         0x03000000L
+#define CP_RB_CNTL__RB_VOLATILE_MASK                                                                          0x04000000L
+#define CP_RB_CNTL__RB_NO_UPDATE_MASK                                                                         0x08000000L
+#define CP_RB_CNTL__RB_EXE_MASK                                                                               0x10000000L
+#define CP_RB_CNTL__KMD_QUEUE_MASK                                                                            0x20000000L
+#define CP_RB_CNTL__RB_RPTR_WR_ENA_MASK                                                                       0x80000000L
+//CP_RB_RPTR_WR
+#define CP_RB_RPTR_WR__RB_RPTR_WR__SHIFT                                                                      0x0
+#define CP_RB_RPTR_WR__RB_RPTR_WR_MASK                                                                        0x000FFFFFL
+//CP_RB0_RPTR_ADDR
+#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                 0x2
+#define CP_RB0_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                   0xFFFFFFFCL
+//CP_RB_RPTR_ADDR
+#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                  0x2
+#define CP_RB_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                    0xFFFFFFFCL
+//CP_RB0_RPTR_ADDR_HI
+#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                           0x0
+#define CP_RB0_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                             0x0000FFFFL
+//CP_RB_RPTR_ADDR_HI
+#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                            0x0
+#define CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                              0x0000FFFFL
+//CP_RB0_BUFSZ_MASK
+#define CP_RB0_BUFSZ_MASK__DATA__SHIFT                                                                        0x0
+#define CP_RB0_BUFSZ_MASK__DATA_MASK                                                                          0x000FFFFFL
+//CP_RB_BUFSZ_MASK
+#define CP_RB_BUFSZ_MASK__DATA__SHIFT                                                                         0x0
+#define CP_RB_BUFSZ_MASK__DATA_MASK                                                                           0x000FFFFFL
+//GC_PRIV_MODE
+#define GC_PRIV_MODE__MC_PRIV_MODE__SHIFT                                                                     0x0
+#define GC_PRIV_MODE__MC_PRIV_MODE_MASK                                                                       0x00000001L
+//CP_INT_CNTL
+#define CP_INT_CNTL__RESUME_INT_ENABLE__SHIFT                                                                 0x8
+#define CP_INT_CNTL__SUSPEND_INT_ENABLE__SHIFT                                                                0x9
+#define CP_INT_CNTL__DMA_WATCH_INT_ENABLE__SHIFT                                                              0xa
+#define CP_INT_CNTL__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                      0xb
+#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                           0xe
+#define CP_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                                    0x10
+#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                       0x11
+#define CP_INT_CNTL__CMP_BUSY_INT_ENABLE__SHIFT                                                               0x12
+#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE__SHIFT                                                              0x13
+#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE__SHIFT                                                             0x14
+#define CP_INT_CNTL__GFX_IDLE_INT_ENABLE__SHIFT                                                               0x15
+#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE__SHIFT                                                             0x16
+#define CP_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                               0x17
+#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                           0x18
+#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                             0x1a
+#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                                     0x1b
+#define CP_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                               0x1d
+#define CP_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                               0x1e
+#define CP_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                               0x1f
+#define CP_INT_CNTL__RESUME_INT_ENABLE_MASK                                                                   0x00000100L
+#define CP_INT_CNTL__SUSPEND_INT_ENABLE_MASK                                                                  0x00000200L
+#define CP_INT_CNTL__DMA_WATCH_INT_ENABLE_MASK                                                                0x00000400L
+#define CP_INT_CNTL__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                        0x00000800L
+#define CP_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                             0x00004000L
+#define CP_INT_CNTL__GPF_INT_ENABLE_MASK                                                                      0x00010000L
+#define CP_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                         0x00020000L
+#define CP_INT_CNTL__CMP_BUSY_INT_ENABLE_MASK                                                                 0x00040000L
+#define CP_INT_CNTL__CNTX_BUSY_INT_ENABLE_MASK                                                                0x00080000L
+#define CP_INT_CNTL__CNTX_EMPTY_INT_ENABLE_MASK                                                               0x00100000L
+#define CP_INT_CNTL__GFX_IDLE_INT_ENABLE_MASK                                                                 0x00200000L
+#define CP_INT_CNTL__PRIV_INSTR_INT_ENABLE_MASK                                                               0x00400000L
+#define CP_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                                 0x00800000L
+#define CP_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                             0x01000000L
+#define CP_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                               0x04000000L
+#define CP_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                       0x08000000L
+#define CP_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                                 0x20000000L
+#define CP_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                                 0x40000000L
+#define CP_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                                 0x80000000L
+//CP_INT_STATUS
+#define CP_INT_STATUS__RESUME_INT_STAT__SHIFT                                                                 0x8
+#define CP_INT_STATUS__SUSPEND_INT_STAT__SHIFT                                                                0x9
+#define CP_INT_STATUS__DMA_WATCH_INT_STAT__SHIFT                                                              0xa
+#define CP_INT_STATUS__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                      0xb
+#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT__SHIFT                                                           0xe
+#define CP_INT_STATUS__GPF_INT_STAT__SHIFT                                                                    0x10
+#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                       0x11
+#define CP_INT_STATUS__CMP_BUSY_INT_STAT__SHIFT                                                               0x12
+#define CP_INT_STATUS__CNTX_BUSY_INT_STAT__SHIFT                                                              0x13
+#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT__SHIFT                                                             0x14
+#define CP_INT_STATUS__GFX_IDLE_INT_STAT__SHIFT                                                               0x15
+#define CP_INT_STATUS__PRIV_INSTR_INT_STAT__SHIFT                                                             0x16
+#define CP_INT_STATUS__PRIV_REG_INT_STAT__SHIFT                                                               0x17
+#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT__SHIFT                                                           0x18
+#define CP_INT_STATUS__TIME_STAMP_INT_STAT__SHIFT                                                             0x1a
+#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                                     0x1b
+#define CP_INT_STATUS__GENERIC2_INT_STAT__SHIFT                                                               0x1d
+#define CP_INT_STATUS__GENERIC1_INT_STAT__SHIFT                                                               0x1e
+#define CP_INT_STATUS__GENERIC0_INT_STAT__SHIFT                                                               0x1f
+#define CP_INT_STATUS__RESUME_INT_STAT_MASK                                                                   0x00000100L
+#define CP_INT_STATUS__SUSPEND_INT_STAT_MASK                                                                  0x00000200L
+#define CP_INT_STATUS__DMA_WATCH_INT_STAT_MASK                                                                0x00000400L
+#define CP_INT_STATUS__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                        0x00000800L
+#define CP_INT_STATUS__CP_ECC_ERROR_INT_STAT_MASK                                                             0x00004000L
+#define CP_INT_STATUS__GPF_INT_STAT_MASK                                                                      0x00010000L
+#define CP_INT_STATUS__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                         0x00020000L
+#define CP_INT_STATUS__CMP_BUSY_INT_STAT_MASK                                                                 0x00040000L
+#define CP_INT_STATUS__CNTX_BUSY_INT_STAT_MASK                                                                0x00080000L
+#define CP_INT_STATUS__CNTX_EMPTY_INT_STAT_MASK                                                               0x00100000L
+#define CP_INT_STATUS__GFX_IDLE_INT_STAT_MASK                                                                 0x00200000L
+#define CP_INT_STATUS__PRIV_INSTR_INT_STAT_MASK                                                               0x00400000L
+#define CP_INT_STATUS__PRIV_REG_INT_STAT_MASK                                                                 0x00800000L
+#define CP_INT_STATUS__OPCODE_ERROR_INT_STAT_MASK                                                             0x01000000L
+#define CP_INT_STATUS__TIME_STAMP_INT_STAT_MASK                                                               0x04000000L
+#define CP_INT_STATUS__RESERVED_BIT_ERROR_INT_STAT_MASK                                                       0x08000000L
+#define CP_INT_STATUS__GENERIC2_INT_STAT_MASK                                                                 0x20000000L
+#define CP_INT_STATUS__GENERIC1_INT_STAT_MASK                                                                 0x40000000L
+#define CP_INT_STATUS__GENERIC0_INT_STAT_MASK                                                                 0x80000000L
+//CP_DEVICE_ID
+#define CP_DEVICE_ID__DEVICE_ID__SHIFT                                                                        0x0
+#define CP_DEVICE_ID__DEVICE_ID_MASK                                                                          0x000000FFL
+//CP_ME0_PIPE_PRIORITY_CNTS
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
+#define CP_ME0_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
+//CP_RING_PRIORITY_CNTS
+#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                           0x0
+#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                          0x8
+#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                          0x10
+#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                           0x18
+#define CP_RING_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                             0x000000FFL
+#define CP_RING_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                            0x0000FF00L
+#define CP_RING_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                            0x00FF0000L
+#define CP_RING_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                             0xFF000000L
+//CP_ME0_PIPE0_PRIORITY
+#define CP_ME0_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME0_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_RING0_PRIORITY
+#define CP_RING0_PRIORITY__PRIORITY__SHIFT                                                                    0x0
+#define CP_RING0_PRIORITY__PRIORITY_MASK                                                                      0x00000003L
+//CP_ME0_PIPE1_PRIORITY
+#define CP_ME0_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME0_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_RING1_PRIORITY
+#define CP_RING1_PRIORITY__PRIORITY__SHIFT                                                                    0x0
+#define CP_RING1_PRIORITY__PRIORITY_MASK                                                                      0x00000003L
+//CP_FATAL_ERROR
+#define CP_FATAL_ERROR__CPF_FATAL_ERROR__SHIFT                                                                0x0
+#define CP_FATAL_ERROR__CPG_FATAL_ERROR__SHIFT                                                                0x1
+#define CP_FATAL_ERROR__GFX_HALT_PROC__SHIFT                                                                  0x2
+#define CP_FATAL_ERROR__DIS_CPG_FATAL_ERROR__SHIFT                                                            0x3
+#define CP_FATAL_ERROR__CPG_TAG_FATAL_ERROR_EN__SHIFT                                                         0x4
+#define CP_FATAL_ERROR__CPF_FATAL_ERROR_MASK                                                                  0x00000001L
+#define CP_FATAL_ERROR__CPG_FATAL_ERROR_MASK                                                                  0x00000002L
+#define CP_FATAL_ERROR__GFX_HALT_PROC_MASK                                                                    0x00000004L
+#define CP_FATAL_ERROR__DIS_CPG_FATAL_ERROR_MASK                                                              0x00000008L
+#define CP_FATAL_ERROR__CPG_TAG_FATAL_ERROR_EN_MASK                                                           0x00000010L
+//CP_RB_VMID
+#define CP_RB_VMID__RB0_VMID__SHIFT                                                                           0x0
+#define CP_RB_VMID__RB1_VMID__SHIFT                                                                           0x8
+#define CP_RB_VMID__RB2_VMID__SHIFT                                                                           0x10
+#define CP_RB_VMID__RB0_VMID_MASK                                                                             0x0000000FL
+#define CP_RB_VMID__RB1_VMID_MASK                                                                             0x00000F00L
+#define CP_RB_VMID__RB2_VMID_MASK                                                                             0x000F0000L
+//CP_ME0_PIPE0_VMID
+#define CP_ME0_PIPE0_VMID__VMID__SHIFT                                                                        0x0
+#define CP_ME0_PIPE0_VMID__VMID_MASK                                                                          0x0000000FL
+//CP_ME0_PIPE1_VMID
+#define CP_ME0_PIPE1_VMID__VMID__SHIFT                                                                        0x0
+#define CP_ME0_PIPE1_VMID__VMID_MASK                                                                          0x0000000FL
+//CP_RB0_WPTR
+#define CP_RB0_WPTR__RB_WPTR__SHIFT                                                                           0x0
+#define CP_RB0_WPTR__RB_WPTR_MASK                                                                             0xFFFFFFFFL
+//CP_RB_WPTR
+#define CP_RB_WPTR__RB_WPTR__SHIFT                                                                            0x0
+#define CP_RB_WPTR__RB_WPTR_MASK                                                                              0xFFFFFFFFL
+//CP_RB0_WPTR_HI
+#define CP_RB0_WPTR_HI__RB_WPTR__SHIFT                                                                        0x0
+#define CP_RB0_WPTR_HI__RB_WPTR_MASK                                                                          0xFFFFFFFFL
+//CP_RB_WPTR_HI
+#define CP_RB_WPTR_HI__RB_WPTR__SHIFT                                                                         0x0
+#define CP_RB_WPTR_HI__RB_WPTR_MASK                                                                           0xFFFFFFFFL
+//CP_RB1_WPTR
+#define CP_RB1_WPTR__RB_WPTR__SHIFT                                                                           0x0
+#define CP_RB1_WPTR__RB_WPTR_MASK                                                                             0xFFFFFFFFL
+//CP_RB1_WPTR_HI
+#define CP_RB1_WPTR_HI__RB_WPTR__SHIFT                                                                        0x0
+#define CP_RB1_WPTR_HI__RB_WPTR_MASK                                                                          0xFFFFFFFFL
+//CP_PROCESS_QUANTUM
+#define CP_PROCESS_QUANTUM__QUANTUM_DURATION__SHIFT                                                           0x0
+#define CP_PROCESS_QUANTUM__TIMER_EXPIRED__SHIFT                                                              0x1c
+#define CP_PROCESS_QUANTUM__QUANTUM_SCALE__SHIFT                                                              0x1d
+#define CP_PROCESS_QUANTUM__QUANTUM_EN__SHIFT                                                                 0x1f
+#define CP_PROCESS_QUANTUM__QUANTUM_DURATION_MASK                                                             0x0FFFFFFFL
+#define CP_PROCESS_QUANTUM__TIMER_EXPIRED_MASK                                                                0x10000000L
+#define CP_PROCESS_QUANTUM__QUANTUM_SCALE_MASK                                                                0x60000000L
+#define CP_PROCESS_QUANTUM__QUANTUM_EN_MASK                                                                   0x80000000L
+//CP_RB_DOORBELL_RANGE_LOWER
+#define CP_RB_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER__SHIFT                                               0x2
+#define CP_RB_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER_MASK                                                 0x00000FFCL
+//CP_RB_DOORBELL_RANGE_UPPER
+#define CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER__SHIFT                                               0x2
+#define CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK                                                 0x00000FFCL
+//CP_MEC_DOORBELL_RANGE_LOWER
+#define CP_MEC_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER__SHIFT                                              0x2
+#define CP_MEC_DOORBELL_RANGE_LOWER__DOORBELL_RANGE_LOWER_MASK                                                0x00000FFCL
+//CP_MEC_DOORBELL_RANGE_UPPER
+#define CP_MEC_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER__SHIFT                                              0x2
+#define CP_MEC_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK                                                0x00000FFCL
+//CPG_UTCL1_ERROR
+#define CPG_UTCL1_ERROR__ERROR_DETECTED_HALT__SHIFT                                                           0x0
+#define CPG_UTCL1_ERROR__ERROR_DETECTED_HALT_MASK                                                             0x00000001L
+//CPC_UTCL1_ERROR
+#define CPC_UTCL1_ERROR__ERROR_DETECTED_HALT__SHIFT                                                           0x0
+#define CPC_UTCL1_ERROR__ERROR_DETECTED_HALT_MASK                                                             0x00000001L
+//CP_RB1_BASE
+#define CP_RB1_BASE__RB_BASE__SHIFT                                                                           0x0
+#define CP_RB1_BASE__RB_BASE_MASK                                                                             0xFFFFFFFFL
+//CP_RB1_CNTL
+#define CP_RB1_CNTL__RB_BUFSZ__SHIFT                                                                          0x0
+#define CP_RB1_CNTL__TMZ_STATE__SHIFT                                                                         0x6
+#define CP_RB1_CNTL__TMZ_MATCH__SHIFT                                                                         0x7
+#define CP_RB1_CNTL__RB_BLKSZ__SHIFT                                                                          0x8
+#define CP_RB1_CNTL__RB_NON_PRIV__SHIFT                                                                       0xf
+#define CP_RB1_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x14
+#define CP_RB1_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                    0x16
+#define CP_RB1_CNTL__CACHE_POLICY__SHIFT                                                                      0x18
+#define CP_RB1_CNTL__RB_VOLATILE__SHIFT                                                                       0x1a
+#define CP_RB1_CNTL__RB_NO_UPDATE__SHIFT                                                                      0x1b
+#define CP_RB1_CNTL__RB_EXE__SHIFT                                                                            0x1c
+#define CP_RB1_CNTL__KMD_QUEUE__SHIFT                                                                         0x1d
+#define CP_RB1_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                    0x1f
+#define CP_RB1_CNTL__RB_BUFSZ_MASK                                                                            0x0000003FL
+#define CP_RB1_CNTL__TMZ_STATE_MASK                                                                           0x00000040L
+#define CP_RB1_CNTL__TMZ_MATCH_MASK                                                                           0x00000080L
+#define CP_RB1_CNTL__RB_BLKSZ_MASK                                                                            0x00003F00L
+#define CP_RB1_CNTL__RB_NON_PRIV_MASK                                                                         0x00008000L
+#define CP_RB1_CNTL__MIN_AVAILSZ_MASK                                                                         0x00300000L
+#define CP_RB1_CNTL__MIN_IB_AVAILSZ_MASK                                                                      0x00C00000L
+#define CP_RB1_CNTL__CACHE_POLICY_MASK                                                                        0x03000000L
+#define CP_RB1_CNTL__RB_VOLATILE_MASK                                                                         0x04000000L
+#define CP_RB1_CNTL__RB_NO_UPDATE_MASK                                                                        0x08000000L
+#define CP_RB1_CNTL__RB_EXE_MASK                                                                              0x10000000L
+#define CP_RB1_CNTL__KMD_QUEUE_MASK                                                                           0x20000000L
+#define CP_RB1_CNTL__RB_RPTR_WR_ENA_MASK                                                                      0x80000000L
+//CP_RB1_RPTR_ADDR
+#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                                 0x2
+#define CP_RB1_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                                   0xFFFFFFFCL
+//CP_RB1_RPTR_ADDR_HI
+#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                           0x0
+#define CP_RB1_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                             0x0000FFFFL
+//CP_RB1_BUFSZ_MASK
+#define CP_RB1_BUFSZ_MASK__DATA__SHIFT                                                                        0x0
+#define CP_RB1_BUFSZ_MASK__DATA_MASK                                                                          0x000FFFFFL
+//CP_INT_CNTL_RING0
+#define CP_INT_CNTL_RING0__RESUME_INT_ENABLE__SHIFT                                                           0x8
+#define CP_INT_CNTL_RING0__SUSPEND_INT_ENABLE__SHIFT                                                          0x9
+#define CP_INT_CNTL_RING0__DMA_WATCH_INT_ENABLE__SHIFT                                                        0xa
+#define CP_INT_CNTL_RING0__CP_VM_DOORBELL_WR_INT_ENABLE__SHIFT                                                0xb
+#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                     0xe
+#define CP_INT_CNTL_RING0__GPF_INT_ENABLE__SHIFT                                                              0x10
+#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                 0x11
+#define CP_INT_CNTL_RING0__CMP_BUSY_INT_ENABLE__SHIFT                                                         0x12
+#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE__SHIFT                                                        0x13
+#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE__SHIFT                                                       0x14
+#define CP_INT_CNTL_RING0__GFX_IDLE_INT_ENABLE__SHIFT                                                         0x15
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE__SHIFT                                                       0x16
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE__SHIFT                                                         0x17
+#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE__SHIFT                                                     0x18
+#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE__SHIFT                                                       0x1a
+#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                               0x1b
+#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE__SHIFT                                                         0x1d
+#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE__SHIFT                                                         0x1e
+#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE__SHIFT                                                         0x1f
+#define CP_INT_CNTL_RING0__RESUME_INT_ENABLE_MASK                                                             0x00000100L
+#define CP_INT_CNTL_RING0__SUSPEND_INT_ENABLE_MASK                                                            0x00000200L
+#define CP_INT_CNTL_RING0__DMA_WATCH_INT_ENABLE_MASK                                                          0x00000400L
+#define CP_INT_CNTL_RING0__CP_VM_DOORBELL_WR_INT_ENABLE_MASK                                                  0x00000800L
+#define CP_INT_CNTL_RING0__CP_ECC_ERROR_INT_ENABLE_MASK                                                       0x00004000L
+#define CP_INT_CNTL_RING0__GPF_INT_ENABLE_MASK                                                                0x00010000L
+#define CP_INT_CNTL_RING0__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                   0x00020000L
+#define CP_INT_CNTL_RING0__CMP_BUSY_INT_ENABLE_MASK                                                           0x00040000L
+#define CP_INT_CNTL_RING0__CNTX_BUSY_INT_ENABLE_MASK                                                          0x00080000L
+#define CP_INT_CNTL_RING0__CNTX_EMPTY_INT_ENABLE_MASK                                                         0x00100000L
+#define CP_INT_CNTL_RING0__GFX_IDLE_INT_ENABLE_MASK                                                           0x00200000L
+#define CP_INT_CNTL_RING0__PRIV_INSTR_INT_ENABLE_MASK                                                         0x00400000L
+#define CP_INT_CNTL_RING0__PRIV_REG_INT_ENABLE_MASK                                                           0x00800000L
+#define CP_INT_CNTL_RING0__OPCODE_ERROR_INT_ENABLE_MASK                                                       0x01000000L
+#define CP_INT_CNTL_RING0__TIME_STAMP_INT_ENABLE_MASK                                                         0x04000000L
+#define CP_INT_CNTL_RING0__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                 0x08000000L
+#define CP_INT_CNTL_RING0__GENERIC2_INT_ENABLE_MASK                                                           0x20000000L
+#define CP_INT_CNTL_RING0__GENERIC1_INT_ENABLE_MASK                                                           0x40000000L
+#define CP_INT_CNTL_RING0__GENERIC0_INT_ENABLE_MASK                                                           0x80000000L
+//CP_INT_CNTL_RING1
+#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                     0xe
+#define CP_INT_CNTL_RING1__GPF_INT_ENABLE__SHIFT                                                              0x10
+#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                 0x11
+#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE__SHIFT                                                       0x16
+#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE__SHIFT                                                         0x17
+#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE__SHIFT                                                     0x18
+#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE__SHIFT                                                       0x1a
+#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                               0x1b
+#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE__SHIFT                                                         0x1d
+#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE__SHIFT                                                         0x1e
+#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE__SHIFT                                                         0x1f
+#define CP_INT_CNTL_RING1__CP_ECC_ERROR_INT_ENABLE_MASK                                                       0x00004000L
+#define CP_INT_CNTL_RING1__GPF_INT_ENABLE_MASK                                                                0x00010000L
+#define CP_INT_CNTL_RING1__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                   0x00020000L
+#define CP_INT_CNTL_RING1__PRIV_INSTR_INT_ENABLE_MASK                                                         0x00400000L
+#define CP_INT_CNTL_RING1__PRIV_REG_INT_ENABLE_MASK                                                           0x00800000L
+#define CP_INT_CNTL_RING1__OPCODE_ERROR_INT_ENABLE_MASK                                                       0x01000000L
+#define CP_INT_CNTL_RING1__TIME_STAMP_INT_ENABLE_MASK                                                         0x04000000L
+#define CP_INT_CNTL_RING1__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                 0x08000000L
+#define CP_INT_CNTL_RING1__GENERIC2_INT_ENABLE_MASK                                                           0x20000000L
+#define CP_INT_CNTL_RING1__GENERIC1_INT_ENABLE_MASK                                                           0x40000000L
+#define CP_INT_CNTL_RING1__GENERIC0_INT_ENABLE_MASK                                                           0x80000000L
+//CP_INT_STATUS_RING0
+#define CP_INT_STATUS_RING0__RESUME_INT_STAT__SHIFT                                                           0x8
+#define CP_INT_STATUS_RING0__SUSPEND_INT_STAT__SHIFT                                                          0x9
+#define CP_INT_STATUS_RING0__DMA_WATCH_INT_STAT__SHIFT                                                        0xa
+#define CP_INT_STATUS_RING0__CP_VM_DOORBELL_WR_INT_STAT__SHIFT                                                0xb
+#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT__SHIFT                                                     0xe
+#define CP_INT_STATUS_RING0__GPF_INT_STAT__SHIFT                                                              0x10
+#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                 0x11
+#define CP_INT_STATUS_RING0__CMP_BUSY_INT_STAT__SHIFT                                                         0x12
+#define CP_INT_STATUS_RING0__GCNTX_BUSY_INT_STAT__SHIFT                                                       0x13
+#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT__SHIFT                                                       0x14
+#define CP_INT_STATUS_RING0__GFX_IDLE_INT_STAT__SHIFT                                                         0x15
+#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT__SHIFT                                                       0x16
+#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT__SHIFT                                                         0x17
+#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT__SHIFT                                                     0x18
+#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT__SHIFT                                                       0x1a
+#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                               0x1b
+#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT__SHIFT                                                         0x1d
+#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT__SHIFT                                                         0x1e
+#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT__SHIFT                                                         0x1f
+#define CP_INT_STATUS_RING0__RESUME_INT_STAT_MASK                                                             0x00000100L
+#define CP_INT_STATUS_RING0__SUSPEND_INT_STAT_MASK                                                            0x00000200L
+#define CP_INT_STATUS_RING0__DMA_WATCH_INT_STAT_MASK                                                          0x00000400L
+#define CP_INT_STATUS_RING0__CP_VM_DOORBELL_WR_INT_STAT_MASK                                                  0x00000800L
+#define CP_INT_STATUS_RING0__CP_ECC_ERROR_INT_STAT_MASK                                                       0x00004000L
+#define CP_INT_STATUS_RING0__GPF_INT_STAT_MASK                                                                0x00010000L
+#define CP_INT_STATUS_RING0__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                   0x00020000L
+#define CP_INT_STATUS_RING0__CMP_BUSY_INT_STAT_MASK                                                           0x00040000L
+#define CP_INT_STATUS_RING0__GCNTX_BUSY_INT_STAT_MASK                                                         0x00080000L
+#define CP_INT_STATUS_RING0__CNTX_EMPTY_INT_STAT_MASK                                                         0x00100000L
+#define CP_INT_STATUS_RING0__GFX_IDLE_INT_STAT_MASK                                                           0x00200000L
+#define CP_INT_STATUS_RING0__PRIV_INSTR_INT_STAT_MASK                                                         0x00400000L
+#define CP_INT_STATUS_RING0__PRIV_REG_INT_STAT_MASK                                                           0x00800000L
+#define CP_INT_STATUS_RING0__OPCODE_ERROR_INT_STAT_MASK                                                       0x01000000L
+#define CP_INT_STATUS_RING0__TIME_STAMP_INT_STAT_MASK                                                         0x04000000L
+#define CP_INT_STATUS_RING0__RESERVED_BIT_ERROR_INT_STAT_MASK                                                 0x08000000L
+#define CP_INT_STATUS_RING0__GENERIC2_INT_STAT_MASK                                                           0x20000000L
+#define CP_INT_STATUS_RING0__GENERIC1_INT_STAT_MASK                                                           0x40000000L
+#define CP_INT_STATUS_RING0__GENERIC0_INT_STAT_MASK                                                           0x80000000L
+//CP_INT_STATUS_RING1
+#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT__SHIFT                                                     0xe
+#define CP_INT_STATUS_RING1__GPF_INT_STAT__SHIFT                                                              0x10
+#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT__SHIFT                                                 0x11
+#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT__SHIFT                                                       0x16
+#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT__SHIFT                                                         0x17
+#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT__SHIFT                                                     0x18
+#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT__SHIFT                                                       0x1a
+#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT__SHIFT                                               0x1b
+#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT__SHIFT                                                         0x1d
+#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT__SHIFT                                                         0x1e
+#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT__SHIFT                                                         0x1f
+#define CP_INT_STATUS_RING1__CP_ECC_ERROR_INT_STAT_MASK                                                       0x00004000L
+#define CP_INT_STATUS_RING1__GPF_INT_STAT_MASK                                                                0x00010000L
+#define CP_INT_STATUS_RING1__WRM_POLL_TIMEOUT_INT_STAT_MASK                                                   0x00020000L
+#define CP_INT_STATUS_RING1__PRIV_INSTR_INT_STAT_MASK                                                         0x00400000L
+#define CP_INT_STATUS_RING1__PRIV_REG_INT_STAT_MASK                                                           0x00800000L
+#define CP_INT_STATUS_RING1__OPCODE_ERROR_INT_STAT_MASK                                                       0x01000000L
+#define CP_INT_STATUS_RING1__TIME_STAMP_INT_STAT_MASK                                                         0x04000000L
+#define CP_INT_STATUS_RING1__RESERVED_BIT_ERROR_INT_STAT_MASK                                                 0x08000000L
+#define CP_INT_STATUS_RING1__GENERIC2_INT_STAT_MASK                                                           0x20000000L
+#define CP_INT_STATUS_RING1__GENERIC1_INT_STAT_MASK                                                           0x40000000L
+#define CP_INT_STATUS_RING1__GENERIC0_INT_STAT_MASK                                                           0x80000000L
+//CP_ME_F32_INTERRUPT
+#define CP_ME_F32_INTERRUPT__ECC_ERROR_INT__SHIFT                                                             0x0
+#define CP_ME_F32_INTERRUPT__TIME_STAMP_INT__SHIFT                                                            0x1
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_2__SHIFT                                                              0x2
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_3__SHIFT                                                              0x3
+#define CP_ME_F32_INTERRUPT__ECC_ERROR_INT_MASK                                                               0x00000001L
+#define CP_ME_F32_INTERRUPT__TIME_STAMP_INT_MASK                                                              0x00000002L
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_2_MASK                                                                0x00000004L
+#define CP_ME_F32_INTERRUPT__ME_F32_INT_3_MASK                                                                0x00000008L
+//CP_PFP_F32_INTERRUPT
+#define CP_PFP_F32_INTERRUPT__ECC_ERROR_INT__SHIFT                                                            0x0
+#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                             0x1
+#define CP_PFP_F32_INTERRUPT__RESERVED_BIT_ERR_INT__SHIFT                                                     0x2
+#define CP_PFP_F32_INTERRUPT__PFP_F32_INT_3__SHIFT                                                            0x3
+#define CP_PFP_F32_INTERRUPT__ECC_ERROR_INT_MASK                                                              0x00000001L
+#define CP_PFP_F32_INTERRUPT__PRIV_REG_INT_MASK                                                               0x00000002L
+#define CP_PFP_F32_INTERRUPT__RESERVED_BIT_ERR_INT_MASK                                                       0x00000004L
+#define CP_PFP_F32_INTERRUPT__PFP_F32_INT_3_MASK                                                              0x00000008L
+//CP_MEC1_F32_INTERRUPT
+#define CP_MEC1_F32_INTERRUPT__EDC_ROQ_FED_INT__SHIFT                                                         0x0
+#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                            0x1
+#define CP_MEC1_F32_INTERRUPT__RESERVED_BIT_ERR_INT__SHIFT                                                    0x2
+#define CP_MEC1_F32_INTERRUPT__EDC_TC_FED_INT__SHIFT                                                          0x3
+#define CP_MEC1_F32_INTERRUPT__EDC_GDS_FED_INT__SHIFT                                                         0x4
+#define CP_MEC1_F32_INTERRUPT__EDC_SCRATCH_FED_INT__SHIFT                                                     0x5
+#define CP_MEC1_F32_INTERRUPT__WAVE_RESTORE_INT__SHIFT                                                        0x6
+#define CP_MEC1_F32_INTERRUPT__SUA_VIOLATION_INT__SHIFT                                                       0x7
+#define CP_MEC1_F32_INTERRUPT__EDC_DMA_FED_INT__SHIFT                                                         0x8
+#define CP_MEC1_F32_INTERRUPT__IQ_TIMER_INT__SHIFT                                                            0x9
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPF__SHIFT                                                             0xa
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_DMA__SHIFT                                                             0xb
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPC__SHIFT                                                             0xc
+#define CP_MEC1_F32_INTERRUPT__EDC_SR_MEM_FED_INT__SHIFT                                                      0xd
+#define CP_MEC1_F32_INTERRUPT__QUEUE_MESSAGE_INT__SHIFT                                                       0xe
+#define CP_MEC1_F32_INTERRUPT__FATAL_EDC_ERROR_INT__SHIFT                                                     0xf
+#define CP_MEC1_F32_INTERRUPT__EDC_ROQ_FED_INT_MASK                                                           0x00000001L
+#define CP_MEC1_F32_INTERRUPT__PRIV_REG_INT_MASK                                                              0x00000002L
+#define CP_MEC1_F32_INTERRUPT__RESERVED_BIT_ERR_INT_MASK                                                      0x00000004L
+#define CP_MEC1_F32_INTERRUPT__EDC_TC_FED_INT_MASK                                                            0x00000008L
+#define CP_MEC1_F32_INTERRUPT__EDC_GDS_FED_INT_MASK                                                           0x00000010L
+#define CP_MEC1_F32_INTERRUPT__EDC_SCRATCH_FED_INT_MASK                                                       0x00000020L
+#define CP_MEC1_F32_INTERRUPT__WAVE_RESTORE_INT_MASK                                                          0x00000040L
+#define CP_MEC1_F32_INTERRUPT__SUA_VIOLATION_INT_MASK                                                         0x00000080L
+#define CP_MEC1_F32_INTERRUPT__EDC_DMA_FED_INT_MASK                                                           0x00000100L
+#define CP_MEC1_F32_INTERRUPT__IQ_TIMER_INT_MASK                                                              0x00000200L
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPF_MASK                                                               0x00000400L
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_DMA_MASK                                                               0x00000800L
+#define CP_MEC1_F32_INTERRUPT__GPF_INT_CPC_MASK                                                               0x00001000L
+#define CP_MEC1_F32_INTERRUPT__EDC_SR_MEM_FED_INT_MASK                                                        0x00002000L
+#define CP_MEC1_F32_INTERRUPT__QUEUE_MESSAGE_INT_MASK                                                         0x00004000L
+#define CP_MEC1_F32_INTERRUPT__FATAL_EDC_ERROR_INT_MASK                                                       0x00008000L
+//CP_MEC2_F32_INTERRUPT
+#define CP_MEC2_F32_INTERRUPT__EDC_ROQ_FED_INT__SHIFT                                                         0x0
+#define CP_MEC2_F32_INTERRUPT__PRIV_REG_INT__SHIFT                                                            0x1
+#define CP_MEC2_F32_INTERRUPT__RESERVED_BIT_ERR_INT__SHIFT                                                    0x2
+#define CP_MEC2_F32_INTERRUPT__EDC_TC_FED_INT__SHIFT                                                          0x3
+#define CP_MEC2_F32_INTERRUPT__EDC_GDS_FED_INT__SHIFT                                                         0x4
+#define CP_MEC2_F32_INTERRUPT__EDC_SCRATCH_FED_INT__SHIFT                                                     0x5
+#define CP_MEC2_F32_INTERRUPT__WAVE_RESTORE_INT__SHIFT                                                        0x6
+#define CP_MEC2_F32_INTERRUPT__SUA_VIOLATION_INT__SHIFT                                                       0x7
+#define CP_MEC2_F32_INTERRUPT__EDC_DMA_FED_INT__SHIFT                                                         0x8
+#define CP_MEC2_F32_INTERRUPT__IQ_TIMER_INT__SHIFT                                                            0x9
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_CPF__SHIFT                                                             0xa
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_DMA__SHIFT                                                             0xb
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_CPC__SHIFT                                                             0xc
+#define CP_MEC2_F32_INTERRUPT__EDC_SR_MEM_FED_INT__SHIFT                                                      0xd
+#define CP_MEC2_F32_INTERRUPT__QUEUE_MESSAGE_INT__SHIFT                                                       0xe
+#define CP_MEC2_F32_INTERRUPT__FATAL_EDC_ERROR_INT__SHIFT                                                     0xf
+#define CP_MEC2_F32_INTERRUPT__EDC_ROQ_FED_INT_MASK                                                           0x00000001L
+#define CP_MEC2_F32_INTERRUPT__PRIV_REG_INT_MASK                                                              0x00000002L
+#define CP_MEC2_F32_INTERRUPT__RESERVED_BIT_ERR_INT_MASK                                                      0x00000004L
+#define CP_MEC2_F32_INTERRUPT__EDC_TC_FED_INT_MASK                                                            0x00000008L
+#define CP_MEC2_F32_INTERRUPT__EDC_GDS_FED_INT_MASK                                                           0x00000010L
+#define CP_MEC2_F32_INTERRUPT__EDC_SCRATCH_FED_INT_MASK                                                       0x00000020L
+#define CP_MEC2_F32_INTERRUPT__WAVE_RESTORE_INT_MASK                                                          0x00000040L
+#define CP_MEC2_F32_INTERRUPT__SUA_VIOLATION_INT_MASK                                                         0x00000080L
+#define CP_MEC2_F32_INTERRUPT__EDC_DMA_FED_INT_MASK                                                           0x00000100L
+#define CP_MEC2_F32_INTERRUPT__IQ_TIMER_INT_MASK                                                              0x00000200L
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_CPF_MASK                                                               0x00000400L
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_DMA_MASK                                                               0x00000800L
+#define CP_MEC2_F32_INTERRUPT__GPF_INT_CPC_MASK                                                               0x00001000L
+#define CP_MEC2_F32_INTERRUPT__EDC_SR_MEM_FED_INT_MASK                                                        0x00002000L
+#define CP_MEC2_F32_INTERRUPT__QUEUE_MESSAGE_INT_MASK                                                         0x00004000L
+#define CP_MEC2_F32_INTERRUPT__FATAL_EDC_ERROR_INT_MASK                                                       0x00008000L
+//CP_PWR_CNTL
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE0__SHIFT                                                            0x0
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE1__SHIFT                                                            0x1
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE0__SHIFT                                                            0x8
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE1__SHIFT                                                            0x9
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE2__SHIFT                                                            0xa
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE3__SHIFT                                                            0xb
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE0__SHIFT                                                            0x10
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE1__SHIFT                                                            0x11
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE2__SHIFT                                                            0x12
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE3__SHIFT                                                            0x13
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE0__SHIFT                                                            0x14
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE1__SHIFT                                                            0x15
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE2__SHIFT                                                            0x16
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE3__SHIFT                                                            0x17
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE0_MASK                                                              0x00000001L
+#define CP_PWR_CNTL__GFX_CLK_HALT_ME0_PIPE1_MASK                                                              0x00000002L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE0_MASK                                                              0x00000100L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE1_MASK                                                              0x00000200L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE2_MASK                                                              0x00000400L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME1_PIPE3_MASK                                                              0x00000800L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE0_MASK                                                              0x00010000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE1_MASK                                                              0x00020000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE2_MASK                                                              0x00040000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME2_PIPE3_MASK                                                              0x00080000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE0_MASK                                                              0x00100000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE1_MASK                                                              0x00200000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE2_MASK                                                              0x00400000L
+#define CP_PWR_CNTL__CMP_CLK_HALT_ME3_PIPE3_MASK                                                              0x00800000L
+//CP_ECC_FIRSTOCCURRENCE
+#define CP_ECC_FIRSTOCCURRENCE__INTERFACE__SHIFT                                                              0x0
+#define CP_ECC_FIRSTOCCURRENCE__CLIENT__SHIFT                                                                 0x4
+#define CP_ECC_FIRSTOCCURRENCE__ME__SHIFT                                                                     0x8
+#define CP_ECC_FIRSTOCCURRENCE__PIPE__SHIFT                                                                   0xa
+#define CP_ECC_FIRSTOCCURRENCE__VMID__SHIFT                                                                   0x10
+#define CP_ECC_FIRSTOCCURRENCE__INTERFACE_MASK                                                                0x00000003L
+#define CP_ECC_FIRSTOCCURRENCE__CLIENT_MASK                                                                   0x000000F0L
+#define CP_ECC_FIRSTOCCURRENCE__ME_MASK                                                                       0x00000300L
+#define CP_ECC_FIRSTOCCURRENCE__PIPE_MASK                                                                     0x00000C00L
+#define CP_ECC_FIRSTOCCURRENCE__VMID_MASK                                                                     0x000F0000L
+//CP_ECC_FIRSTOCCURRENCE_RING0
+#define CP_ECC_FIRSTOCCURRENCE_RING0__OBSOLETE__SHIFT                                                         0x0
+#define CP_ECC_FIRSTOCCURRENCE_RING0__OBSOLETE_MASK                                                           0xFFFFFFFFL
+//CP_ECC_FIRSTOCCURRENCE_RING1
+#define CP_ECC_FIRSTOCCURRENCE_RING1__OBSOLETE__SHIFT                                                         0x0
+#define CP_ECC_FIRSTOCCURRENCE_RING1__OBSOLETE_MASK                                                           0xFFFFFFFFL
+//GB_EDC_MODE
+#define GB_EDC_MODE__FORCE_SEC_ON_DED__SHIFT                                                                  0xf
+#define GB_EDC_MODE__COUNT_FED_OUT__SHIFT                                                                     0x10
+#define GB_EDC_MODE__GATE_FUE__SHIFT                                                                          0x11
+#define GB_EDC_MODE__DED_MODE__SHIFT                                                                          0x14
+#define GB_EDC_MODE__PROP_FED__SHIFT                                                                          0x1d
+#define GB_EDC_MODE__BYPASS__SHIFT                                                                            0x1f
+#define GB_EDC_MODE__FORCE_SEC_ON_DED_MASK                                                                    0x00008000L
+#define GB_EDC_MODE__COUNT_FED_OUT_MASK                                                                       0x00010000L
+#define GB_EDC_MODE__GATE_FUE_MASK                                                                            0x00020000L
+#define GB_EDC_MODE__DED_MODE_MASK                                                                            0x00300000L
+#define GB_EDC_MODE__PROP_FED_MASK                                                                            0x20000000L
+#define GB_EDC_MODE__BYPASS_MASK                                                                              0x80000000L
+//CP_DEBUG
+#define CP_DEBUG__PERFMON_RING_SEL__SHIFT                                                                     0x0
+#define CP_DEBUG__DEBUG_BUS_SELECT_BITS__SHIFT                                                                0x2
+#define CP_DEBUG__DEBUG_BUS_FLOP_EN__SHIFT                                                                    0x8
+#define CP_DEBUG__CPG_REPEATER_FGCG_OVERRIDE__SHIFT                                                           0x9
+#define CP_DEBUG__PACKET_FILTER_DISABLE__SHIFT                                                                0xa
+#define CP_DEBUG__NOT_EOP_PREEMPT_DISABLE__SHIFT                                                              0xb
+#define CP_DEBUG__CPG_CHIU_RO_DISABLE__SHIFT                                                                  0xc
+#define CP_DEBUG__CPG_GCR_CNTL_BYPASS__SHIFT                                                                  0xd
+#define CP_DEBUG__CPG_RAM_CLK_GATING_DISABLE__SHIFT                                                           0xe
+#define CP_DEBUG__CPG_UTCL1_ERROR_HALT_DISABLE__SHIFT                                                         0xf
+#define CP_DEBUG__SURFSYNC_CNTX_RDADDR__SHIFT                                                                 0x10
+#define CP_DEBUG__CPG_DATA_POISONING_INT_DISABLE__SHIFT                                                       0x13
+#define CP_DEBUG__PRIV_VIOLATION_WRITE_DISABLE__SHIFT                                                         0x14
+#define CP_DEBUG__CPG_CHIU_GUS_DISABLE__SHIFT                                                                 0x15
+#define CP_DEBUG__INTERRUPT_DISABLE__SHIFT                                                                    0x16
+#define CP_DEBUG__PREDICATE_DISABLE__SHIFT                                                                    0x17
+#define CP_DEBUG__UNDERFLOW_BUSY_DISABLE__SHIFT                                                               0x18
+#define CP_DEBUG__OVERFLOW_BUSY_DISABLE__SHIFT                                                                0x19
+#define CP_DEBUG__EVENT_FILT_DISABLE__SHIFT                                                                   0x1a
+#define CP_DEBUG__CPG_CHIU_MTYPE_OVERRIDE__SHIFT                                                              0x1b
+#define CP_DEBUG__CPG_TC_ONE_CYCLE_WRITE_DISABLE__SHIFT                                                       0x1c
+#define CP_DEBUG__CS_STATE_FILT_DISABLE__SHIFT                                                                0x1d
+#define CP_DEBUG__CS_PIPELINE_RESET_DISABLE__SHIFT                                                            0x1e
+#define CP_DEBUG__IB_PACKET_INJECTOR_DISABLE__SHIFT                                                           0x1f
+#define CP_DEBUG__PERFMON_RING_SEL_MASK                                                                       0x00000003L
+#define CP_DEBUG__DEBUG_BUS_SELECT_BITS_MASK                                                                  0x000000FCL
+#define CP_DEBUG__DEBUG_BUS_FLOP_EN_MASK                                                                      0x00000100L
+#define CP_DEBUG__CPG_REPEATER_FGCG_OVERRIDE_MASK                                                             0x00000200L
+#define CP_DEBUG__PACKET_FILTER_DISABLE_MASK                                                                  0x00000400L
+#define CP_DEBUG__NOT_EOP_PREEMPT_DISABLE_MASK                                                                0x00000800L
+#define CP_DEBUG__CPG_CHIU_RO_DISABLE_MASK                                                                    0x00001000L
+#define CP_DEBUG__CPG_GCR_CNTL_BYPASS_MASK                                                                    0x00002000L
+#define CP_DEBUG__CPG_RAM_CLK_GATING_DISABLE_MASK                                                             0x00004000L
+#define CP_DEBUG__CPG_UTCL1_ERROR_HALT_DISABLE_MASK                                                           0x00008000L
+#define CP_DEBUG__SURFSYNC_CNTX_RDADDR_MASK                                                                   0x00070000L
+#define CP_DEBUG__CPG_DATA_POISONING_INT_DISABLE_MASK                                                         0x00080000L
+#define CP_DEBUG__PRIV_VIOLATION_WRITE_DISABLE_MASK                                                           0x00100000L
+#define CP_DEBUG__CPG_CHIU_GUS_DISABLE_MASK                                                                   0x00200000L
+#define CP_DEBUG__INTERRUPT_DISABLE_MASK                                                                      0x00400000L
+#define CP_DEBUG__PREDICATE_DISABLE_MASK                                                                      0x00800000L
+#define CP_DEBUG__UNDERFLOW_BUSY_DISABLE_MASK                                                                 0x01000000L
+#define CP_DEBUG__OVERFLOW_BUSY_DISABLE_MASK                                                                  0x02000000L
+#define CP_DEBUG__EVENT_FILT_DISABLE_MASK                                                                     0x04000000L
+#define CP_DEBUG__CPG_CHIU_MTYPE_OVERRIDE_MASK                                                                0x08000000L
+#define CP_DEBUG__CPG_TC_ONE_CYCLE_WRITE_DISABLE_MASK                                                         0x10000000L
+#define CP_DEBUG__CS_STATE_FILT_DISABLE_MASK                                                                  0x20000000L
+#define CP_DEBUG__CS_PIPELINE_RESET_DISABLE_MASK                                                              0x40000000L
+#define CP_DEBUG__IB_PACKET_INJECTOR_DISABLE_MASK                                                             0x80000000L
+//CP_CPF_DEBUG
+#define CP_CPF_DEBUG__DEBUG_BUS_FLOP_EN__SHIFT                                                                0xe
+#define CP_CPF_DEBUG__CPF_REPEATER_FGCG_OVERRIDE__SHIFT                                                       0x10
+#define CP_CPF_DEBUG__CPF_GCR_CNTL_BYPASS__SHIFT                                                              0x11
+#define CP_CPF_DEBUG__CPF_RAM_CLK_GATING_DISABLE__SHIFT                                                       0x12
+#define CP_CPF_DEBUG__CPF_DATA_POISONING_INT_DISABLE__SHIFT                                                   0x13
+#define CP_CPF_DEBUG__CLOCK_ACTIVE_DELAY_OVERRIDE__SHIFT                                                      0x16
+#define CP_CPF_DEBUG__CLOCK_ACTIVE_OVERRIDE__SHIFT                                                            0x17
+#define CP_CPF_DEBUG__UNDERFLOW_BUSY_DISABLE__SHIFT                                                           0x18
+#define CP_CPF_DEBUG__OVERFLOW_BUSY_DISABLE__SHIFT                                                            0x19
+#define CP_CPF_DEBUG__CPF_CHIU_NOALLOC_OVERRIDE__SHIFT                                                        0x1a
+#define CP_CPF_DEBUG__CE_FETCHER_DISABLE__SHIFT                                                               0x1b
+#define CP_CPF_DEBUG__CPF_CHIU_GUS_DISABLE__SHIFT                                                             0x1c
+#define CP_CPF_DEBUG__CPF_PRIORITY_YIELD_ACTIVE_DIS__SHIFT                                                    0x1d
+#define CP_CPF_DEBUG__CPF_CHIU_MTYPE_OVERRIDE__SHIFT                                                          0x1e
+#define CP_CPF_DEBUG__DBGU_TRIGGER__SHIFT                                                                     0x1f
+#define CP_CPF_DEBUG__DEBUG_BUS_FLOP_EN_MASK                                                                  0x00004000L
+#define CP_CPF_DEBUG__CPF_REPEATER_FGCG_OVERRIDE_MASK                                                         0x00010000L
+#define CP_CPF_DEBUG__CPF_GCR_CNTL_BYPASS_MASK                                                                0x00020000L
+#define CP_CPF_DEBUG__CPF_RAM_CLK_GATING_DISABLE_MASK                                                         0x00040000L
+#define CP_CPF_DEBUG__CPF_DATA_POISONING_INT_DISABLE_MASK                                                     0x00080000L
+#define CP_CPF_DEBUG__CLOCK_ACTIVE_DELAY_OVERRIDE_MASK                                                        0x00400000L
+#define CP_CPF_DEBUG__CLOCK_ACTIVE_OVERRIDE_MASK                                                              0x00800000L
+#define CP_CPF_DEBUG__UNDERFLOW_BUSY_DISABLE_MASK                                                             0x01000000L
+#define CP_CPF_DEBUG__OVERFLOW_BUSY_DISABLE_MASK                                                              0x02000000L
+#define CP_CPF_DEBUG__CPF_CHIU_NOALLOC_OVERRIDE_MASK                                                          0x04000000L
+#define CP_CPF_DEBUG__CE_FETCHER_DISABLE_MASK                                                                 0x08000000L
+#define CP_CPF_DEBUG__CPF_CHIU_GUS_DISABLE_MASK                                                               0x10000000L
+#define CP_CPF_DEBUG__CPF_PRIORITY_YIELD_ACTIVE_DIS_MASK                                                      0x20000000L
+#define CP_CPF_DEBUG__CPF_CHIU_MTYPE_OVERRIDE_MASK                                                            0x40000000L
+#define CP_CPF_DEBUG__DBGU_TRIGGER_MASK                                                                       0x80000000L
+//CP_CPC_DEBUG
+#define CP_CPC_DEBUG__PIPE_SELECT__SHIFT                                                                      0x0
+#define CP_CPC_DEBUG__ME_SELECT__SHIFT                                                                        0x2
+#define CP_CPC_DEBUG__ADC_INTERLEAVE_DISABLE__SHIFT                                                           0x4
+#define CP_CPC_DEBUG__DEBUG_BUS_FLOP_EN__SHIFT                                                                0xe
+#define CP_CPC_DEBUG__CPC_REPEATER_FGCG_OVERRIDE__SHIFT                                                       0xf
+#define CP_CPC_DEBUG__CPC_CHIU_NOALLOC_OVERRIDE__SHIFT                                                        0x10
+#define CP_CPC_DEBUG__CPC_GCR_CNTL_BYPASS__SHIFT                                                              0x11
+#define CP_CPC_DEBUG__CPC_RAM_CLK_GATING_DISABLE__SHIFT                                                       0x12
+#define CP_CPC_DEBUG__CPC_DATA_POISONING_INT_DISABLE__SHIFT                                                   0x13
+#define CP_CPC_DEBUG__PRIV_VIOLATION_WRITE_DISABLE__SHIFT                                                     0x14
+#define CP_CPC_DEBUG__UCODE_ECC_ERROR_DISABLE__SHIFT                                                          0x15
+#define CP_CPC_DEBUG__INTERRUPT_DISABLE__SHIFT                                                                0x16
+#define CP_CPC_DEBUG__CPC_CHIU_RO_DISABLE__SHIFT                                                              0x17
+#define CP_CPC_DEBUG__UNDERFLOW_BUSY_DISABLE__SHIFT                                                           0x18
+#define CP_CPC_DEBUG__OVERFLOW_BUSY_DISABLE__SHIFT                                                            0x19
+#define CP_CPC_DEBUG__EVENT_FILT_DISABLE__SHIFT                                                               0x1a
+#define CP_CPC_DEBUG__CPC_CHIU_GUS_DISABLE__SHIFT                                                             0x1b
+#define CP_CPC_DEBUG__CPC_TC_ONE_CYCLE_WRITE_DISABLE__SHIFT                                                   0x1c
+#define CP_CPC_DEBUG__CS_STATE_FILT_DISABLE__SHIFT                                                            0x1d
+#define CP_CPC_DEBUG__CPC_CHIU_MTYPE_OVERRIDE__SHIFT                                                          0x1e
+#define CP_CPC_DEBUG__ME2_UCODE_RAM_ENABLE__SHIFT                                                             0x1f
+#define CP_CPC_DEBUG__PIPE_SELECT_MASK                                                                        0x00000003L
+#define CP_CPC_DEBUG__ME_SELECT_MASK                                                                          0x00000004L
+#define CP_CPC_DEBUG__ADC_INTERLEAVE_DISABLE_MASK                                                             0x00000010L
+#define CP_CPC_DEBUG__DEBUG_BUS_FLOP_EN_MASK                                                                  0x00004000L
+#define CP_CPC_DEBUG__CPC_REPEATER_FGCG_OVERRIDE_MASK                                                         0x00008000L
+#define CP_CPC_DEBUG__CPC_CHIU_NOALLOC_OVERRIDE_MASK                                                          0x00010000L
+#define CP_CPC_DEBUG__CPC_GCR_CNTL_BYPASS_MASK                                                                0x00020000L
+#define CP_CPC_DEBUG__CPC_RAM_CLK_GATING_DISABLE_MASK                                                         0x00040000L
+#define CP_CPC_DEBUG__CPC_DATA_POISONING_INT_DISABLE_MASK                                                     0x00080000L
+#define CP_CPC_DEBUG__PRIV_VIOLATION_WRITE_DISABLE_MASK                                                       0x00100000L
+#define CP_CPC_DEBUG__UCODE_ECC_ERROR_DISABLE_MASK                                                            0x00200000L
+#define CP_CPC_DEBUG__INTERRUPT_DISABLE_MASK                                                                  0x00400000L
+#define CP_CPC_DEBUG__CPC_CHIU_RO_DISABLE_MASK                                                                0x00800000L
+#define CP_CPC_DEBUG__UNDERFLOW_BUSY_DISABLE_MASK                                                             0x01000000L
+#define CP_CPC_DEBUG__OVERFLOW_BUSY_DISABLE_MASK                                                              0x02000000L
+#define CP_CPC_DEBUG__EVENT_FILT_DISABLE_MASK                                                                 0x04000000L
+#define CP_CPC_DEBUG__CPC_CHIU_GUS_DISABLE_MASK                                                               0x08000000L
+#define CP_CPC_DEBUG__CPC_TC_ONE_CYCLE_WRITE_DISABLE_MASK                                                     0x10000000L
+#define CP_CPC_DEBUG__CS_STATE_FILT_DISABLE_MASK                                                              0x20000000L
+#define CP_CPC_DEBUG__CPC_CHIU_MTYPE_OVERRIDE_MASK                                                            0x40000000L
+#define CP_CPC_DEBUG__ME2_UCODE_RAM_ENABLE_MASK                                                               0x80000000L
+//CP_PQ_WPTR_POLL_CNTL
+#define CP_PQ_WPTR_POLL_CNTL__PERIOD__SHIFT                                                                   0x0
+#define CP_PQ_WPTR_POLL_CNTL__DISABLE_PEND_REQ_ONE_SHOT__SHIFT                                                0x1d
+#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE__SHIFT                                                              0x1e
+#define CP_PQ_WPTR_POLL_CNTL__EN__SHIFT                                                                       0x1f
+#define CP_PQ_WPTR_POLL_CNTL__PERIOD_MASK                                                                     0x000000FFL
+#define CP_PQ_WPTR_POLL_CNTL__DISABLE_PEND_REQ_ONE_SHOT_MASK                                                  0x20000000L
+#define CP_PQ_WPTR_POLL_CNTL__POLL_ACTIVE_MASK                                                                0x40000000L
+#define CP_PQ_WPTR_POLL_CNTL__EN_MASK                                                                         0x80000000L
+//CP_PQ_WPTR_POLL_CNTL1
+#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK__SHIFT                                                              0x0
+#define CP_PQ_WPTR_POLL_CNTL1__QUEUE_MASK_MASK                                                                0xFFFFFFFFL
+//CP_ME1_PIPE0_INT_CNTL
+#define CP_ME1_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE0_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE0_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE1_INT_CNTL
+#define CP_ME1_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE1_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE1_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE2_INT_CNTL
+#define CP_ME1_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE2_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE2_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE3_INT_CNTL
+#define CP_ME1_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME1_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME1_PIPE3_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME1_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME1_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME1_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME1_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME1_PIPE3_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME1_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME1_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME1_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME1_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME1_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME1_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME2_PIPE0_INT_CNTL
+#define CP_ME2_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME2_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME2_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME2_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME2_PIPE0_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME2_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME2_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME2_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME2_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME2_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME2_PIPE0_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME2_PIPE0_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME2_PIPE0_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME2_PIPE0_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME2_PIPE0_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME2_PIPE0_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME2_PIPE0_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME2_PIPE0_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME2_PIPE0_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME2_PIPE0_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME2_PIPE0_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME2_PIPE1_INT_CNTL
+#define CP_ME2_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME2_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME2_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME2_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME2_PIPE1_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME2_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME2_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME2_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME2_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME2_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME2_PIPE1_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME2_PIPE1_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME2_PIPE1_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME2_PIPE1_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME2_PIPE1_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME2_PIPE1_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME2_PIPE1_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME2_PIPE1_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME2_PIPE1_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME2_PIPE1_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME2_PIPE1_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME2_PIPE2_INT_CNTL
+#define CP_ME2_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME2_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME2_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME2_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME2_PIPE2_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME2_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME2_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME2_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME2_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME2_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME2_PIPE2_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME2_PIPE2_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME2_PIPE2_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME2_PIPE2_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME2_PIPE2_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME2_PIPE2_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME2_PIPE2_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME2_PIPE2_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME2_PIPE2_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME2_PIPE2_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME2_PIPE2_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME2_PIPE3_INT_CNTL
+#define CP_ME2_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                             0xc
+#define CP_ME2_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                              0xd
+#define CP_ME2_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                 0xe
+#define CP_ME2_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                0xf
+#define CP_ME2_PIPE3_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                          0x10
+#define CP_ME2_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                             0x11
+#define CP_ME2_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                     0x17
+#define CP_ME2_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                 0x18
+#define CP_ME2_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                   0x1a
+#define CP_ME2_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                           0x1b
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                     0x1d
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                     0x1e
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                     0x1f
+#define CP_ME2_PIPE3_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                               0x00001000L
+#define CP_ME2_PIPE3_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                0x00002000L
+#define CP_ME2_PIPE3_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                   0x00004000L
+#define CP_ME2_PIPE3_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                  0x00008000L
+#define CP_ME2_PIPE3_INT_CNTL__GPF_INT_ENABLE_MASK                                                            0x00010000L
+#define CP_ME2_PIPE3_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                               0x00020000L
+#define CP_ME2_PIPE3_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                       0x00800000L
+#define CP_ME2_PIPE3_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                   0x01000000L
+#define CP_ME2_PIPE3_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                     0x04000000L
+#define CP_ME2_PIPE3_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                             0x08000000L
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                       0x20000000L
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                       0x40000000L
+#define CP_ME2_PIPE3_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                       0x80000000L
+//CP_ME1_PIPE0_INT_STATUS
+#define CP_ME1_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE0_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE0_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_PIPE1_INT_STATUS
+#define CP_ME1_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE1_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE1_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_PIPE2_INT_STATUS
+#define CP_ME1_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE2_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE2_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_PIPE3_INT_STATUS
+#define CP_ME1_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME1_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME1_PIPE3_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME1_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME1_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME1_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME1_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME1_PIPE3_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME1_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME1_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME1_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME1_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME1_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME1_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME2_PIPE0_INT_STATUS
+#define CP_ME2_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME2_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME2_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME2_PIPE0_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME2_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME2_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME2_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME2_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME2_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME2_PIPE0_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME2_PIPE0_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME2_PIPE0_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME2_PIPE0_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME2_PIPE0_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME2_PIPE0_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME2_PIPE0_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME2_PIPE0_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME2_PIPE0_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME2_PIPE0_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME2_PIPE0_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME2_PIPE1_INT_STATUS
+#define CP_ME2_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME2_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME2_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME2_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME2_PIPE1_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME2_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME2_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME2_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME2_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME2_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME2_PIPE1_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME2_PIPE1_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME2_PIPE1_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME2_PIPE1_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME2_PIPE1_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME2_PIPE1_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME2_PIPE1_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME2_PIPE1_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME2_PIPE1_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME2_PIPE1_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME2_PIPE1_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME2_PIPE2_INT_STATUS
+#define CP_ME2_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME2_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME2_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME2_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME2_PIPE2_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME2_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME2_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME2_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME2_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME2_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME2_PIPE2_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME2_PIPE2_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME2_PIPE2_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME2_PIPE2_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME2_PIPE2_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME2_PIPE2_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME2_PIPE2_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME2_PIPE2_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME2_PIPE2_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME2_PIPE2_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME2_PIPE2_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME2_PIPE3_INT_STATUS
+#define CP_ME2_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                           0xc
+#define CP_ME2_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                            0xd
+#define CP_ME2_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                               0xe
+#define CP_ME2_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                              0xf
+#define CP_ME2_PIPE3_INT_STATUS__GPF_INT_STATUS__SHIFT                                                        0x10
+#define CP_ME2_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                           0x11
+#define CP_ME2_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                   0x17
+#define CP_ME2_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                               0x18
+#define CP_ME2_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                 0x1a
+#define CP_ME2_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                         0x1b
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                   0x1d
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                   0x1e
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                   0x1f
+#define CP_ME2_PIPE3_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                             0x00001000L
+#define CP_ME2_PIPE3_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                              0x00002000L
+#define CP_ME2_PIPE3_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                 0x00004000L
+#define CP_ME2_PIPE3_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                0x00008000L
+#define CP_ME2_PIPE3_INT_STATUS__GPF_INT_STATUS_MASK                                                          0x00010000L
+#define CP_ME2_PIPE3_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                             0x00020000L
+#define CP_ME2_PIPE3_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                     0x00800000L
+#define CP_ME2_PIPE3_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                 0x01000000L
+#define CP_ME2_PIPE3_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                   0x04000000L
+#define CP_ME2_PIPE3_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                           0x08000000L
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                     0x20000000L
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                     0x40000000L
+#define CP_ME2_PIPE3_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                     0x80000000L
+//CP_ME1_INT_STAT_DEBUG
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME1_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME1_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME1_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME1_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME1_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME1_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME1_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME1_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME1_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME1_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME1_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
+//CP_ME2_INT_STAT_DEBUG
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED__SHIFT                                           0xc
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED__SHIFT                                            0xd
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED__SHIFT                                               0xe
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS__SHIFT                                                0xf
+#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED__SHIFT                                                        0x10
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED__SHIFT                                           0x11
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED__SHIFT                                                   0x17
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED__SHIFT                                               0x18
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED__SHIFT                                                 0x1a
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED__SHIFT                                         0x1b
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED__SHIFT                                                   0x1d
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED__SHIFT                                                   0x1e
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED__SHIFT                                                   0x1f
+#define CP_ME2_INT_STAT_DEBUG__CMP_QUERY_STATUS_INT_ASSERTED_MASK                                             0x00001000L
+#define CP_ME2_INT_STAT_DEBUG__DEQUEUE_REQUEST_INT_ASSERTED_MASK                                              0x00002000L
+#define CP_ME2_INT_STAT_DEBUG__CP_ECC_ERROR_INT_ASSERTED_MASK                                                 0x00004000L
+#define CP_ME2_INT_STAT_DEBUG__SUA_VIOLATION_INT_STATUS_MASK                                                  0x00008000L
+#define CP_ME2_INT_STAT_DEBUG__GPF_INT_ASSERTED_MASK                                                          0x00010000L
+#define CP_ME2_INT_STAT_DEBUG__WRM_POLL_TIMEOUT_INT_ASSERTED_MASK                                             0x00020000L
+#define CP_ME2_INT_STAT_DEBUG__PRIV_REG_INT_ASSERTED_MASK                                                     0x00800000L
+#define CP_ME2_INT_STAT_DEBUG__OPCODE_ERROR_INT_ASSERTED_MASK                                                 0x01000000L
+#define CP_ME2_INT_STAT_DEBUG__TIME_STAMP_INT_ASSERTED_MASK                                                   0x04000000L
+#define CP_ME2_INT_STAT_DEBUG__RESERVED_BIT_ERROR_INT_ASSERTED_MASK                                           0x08000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC2_INT_ASSERTED_MASK                                                     0x20000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC1_INT_ASSERTED_MASK                                                     0x40000000L
+#define CP_ME2_INT_STAT_DEBUG__GENERIC0_INT_ASSERTED_MASK                                                     0x80000000L
+//CP_GFX_QUEUE_INDEX
+#define CP_GFX_QUEUE_INDEX__QUEUE_ACCESS__SHIFT                                                               0x0
+#define CP_GFX_QUEUE_INDEX__PIPE_ID__SHIFT                                                                    0x4
+#define CP_GFX_QUEUE_INDEX__QUEUE_ID__SHIFT                                                                   0x8
+#define CP_GFX_QUEUE_INDEX__QUEUE_ACCESS_MASK                                                                 0x00000001L
+#define CP_GFX_QUEUE_INDEX__PIPE_ID_MASK                                                                      0x00000030L
+#define CP_GFX_QUEUE_INDEX__QUEUE_ID_MASK                                                                     0x00000700L
+//CC_GC_EDC_CONFIG
+#define CC_GC_EDC_CONFIG__WRITE_DIS__SHIFT                                                                    0x0
+#define CC_GC_EDC_CONFIG__DIS_EDC__SHIFT                                                                      0x1
+#define CC_GC_EDC_CONFIG__WRITE_DIS_MASK                                                                      0x00000001L
+#define CC_GC_EDC_CONFIG__DIS_EDC_MASK                                                                        0x00000002L
+//CP_ME1_PIPE_PRIORITY_CNTS
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
+#define CP_ME1_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
+//CP_ME1_PIPE0_PRIORITY
+#define CP_ME1_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME1_PIPE1_PRIORITY
+#define CP_ME1_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME1_PIPE2_PRIORITY
+#define CP_ME1_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME1_PIPE3_PRIORITY
+#define CP_ME1_PIPE3_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME1_PIPE3_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME2_PIPE_PRIORITY_CNTS
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
+#define CP_ME2_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
+//CP_ME2_PIPE0_PRIORITY
+#define CP_ME2_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME2_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME2_PIPE1_PRIORITY
+#define CP_ME2_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME2_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME2_PIPE2_PRIORITY
+#define CP_ME2_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME2_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_ME2_PIPE3_PRIORITY
+#define CP_ME2_PIPE3_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_ME2_PIPE3_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_PFP_PRGRM_CNTR_START
+#define CP_PFP_PRGRM_CNTR_START__IP_START__SHIFT                                                              0x0
+#define CP_PFP_PRGRM_CNTR_START__IP_START_MASK                                                                0xFFFFFFFFL
+//CP_ME_PRGRM_CNTR_START
+#define CP_ME_PRGRM_CNTR_START__IP_START__SHIFT                                                               0x0
+#define CP_ME_PRGRM_CNTR_START__IP_START_MASK                                                                 0xFFFFFFFFL
+//CP_MEC1_PRGRM_CNTR_START
+#define CP_MEC1_PRGRM_CNTR_START__IP_START__SHIFT                                                             0x0
+#define CP_MEC1_PRGRM_CNTR_START__IP_START_MASK                                                               0x000FFFFFL
+//CP_MEC2_PRGRM_CNTR_START
+#define CP_MEC2_PRGRM_CNTR_START__IP_START__SHIFT                                                             0x0
+#define CP_MEC2_PRGRM_CNTR_START__IP_START_MASK                                                               0x000FFFFFL
+//CP_PFP_INTR_ROUTINE_START
+#define CP_PFP_INTR_ROUTINE_START__IR_START__SHIFT                                                            0x0
+#define CP_PFP_INTR_ROUTINE_START__IR_START_MASK                                                              0xFFFFFFFFL
+//CP_ME_INTR_ROUTINE_START
+#define CP_ME_INTR_ROUTINE_START__IR_START__SHIFT                                                             0x0
+#define CP_ME_INTR_ROUTINE_START__IR_START_MASK                                                               0xFFFFFFFFL
+//CP_MEC1_INTR_ROUTINE_START
+#define CP_MEC1_INTR_ROUTINE_START__IR_START__SHIFT                                                           0x0
+#define CP_MEC1_INTR_ROUTINE_START__IR_START_MASK                                                             0x000FFFFFL
+//CP_MEC2_INTR_ROUTINE_START
+#define CP_MEC2_INTR_ROUTINE_START__IR_START__SHIFT                                                           0x0
+#define CP_MEC2_INTR_ROUTINE_START__IR_START_MASK                                                             0x000FFFFFL
+//CP_CONTEXT_CNTL
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_GE_CNTX__SHIFT                                                          0x0
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX__SHIFT                                                        0x4
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_GE_CNTX__SHIFT                                                          0x10
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX__SHIFT                                                        0x14
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_GE_CNTX_MASK                                                            0x00000007L
+#define CP_CONTEXT_CNTL__ME0PIPE0_MAX_PIPE_CNTX_MASK                                                          0x00000070L
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_GE_CNTX_MASK                                                            0x00070000L
+#define CP_CONTEXT_CNTL__ME0PIPE1_MAX_PIPE_CNTX_MASK                                                          0x00700000L
+//CP_MAX_CONTEXT
+#define CP_MAX_CONTEXT__MAX_CONTEXT__SHIFT                                                                    0x0
+#define CP_MAX_CONTEXT__MAX_CONTEXT_MASK                                                                      0x00000007L
+//CP_IQ_WAIT_TIME1
+#define CP_IQ_WAIT_TIME1__IB_OFFLOAD__SHIFT                                                                   0x0
+#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD__SHIFT                                                               0x8
+#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD__SHIFT                                                                  0x10
+#define CP_IQ_WAIT_TIME1__GWS__SHIFT                                                                          0x18
+#define CP_IQ_WAIT_TIME1__IB_OFFLOAD_MASK                                                                     0x000000FFL
+#define CP_IQ_WAIT_TIME1__ATOMIC_OFFLOAD_MASK                                                                 0x0000FF00L
+#define CP_IQ_WAIT_TIME1__WRM_OFFLOAD_MASK                                                                    0x00FF0000L
+#define CP_IQ_WAIT_TIME1__GWS_MASK                                                                            0xFF000000L
+//CP_IQ_WAIT_TIME2
+#define CP_IQ_WAIT_TIME2__QUE_SLEEP__SHIFT                                                                    0x0
+#define CP_IQ_WAIT_TIME2__SCH_WAVE__SHIFT                                                                     0x8
+#define CP_IQ_WAIT_TIME2__SEM_REARM__SHIFT                                                                    0x10
+#define CP_IQ_WAIT_TIME2__DEQ_RETRY__SHIFT                                                                    0x18
+#define CP_IQ_WAIT_TIME2__QUE_SLEEP_MASK                                                                      0x000000FFL
+#define CP_IQ_WAIT_TIME2__SCH_WAVE_MASK                                                                       0x0000FF00L
+#define CP_IQ_WAIT_TIME2__SEM_REARM_MASK                                                                      0x00FF0000L
+#define CP_IQ_WAIT_TIME2__DEQ_RETRY_MASK                                                                      0xFF000000L
+//CP_RB0_BASE_HI
+#define CP_RB0_BASE_HI__RB_BASE_HI__SHIFT                                                                     0x0
+#define CP_RB0_BASE_HI__RB_BASE_HI_MASK                                                                       0x000000FFL
+//CP_RB1_BASE_HI
+#define CP_RB1_BASE_HI__RB_BASE_HI__SHIFT                                                                     0x0
+#define CP_RB1_BASE_HI__RB_BASE_HI_MASK                                                                       0x000000FFL
+//CP_VMID_RESET
+#define CP_VMID_RESET__RESET_REQUEST__SHIFT                                                                   0x0
+#define CP_VMID_RESET__PIPE0_QUEUES__SHIFT                                                                    0x10
+#define CP_VMID_RESET__PIPE1_QUEUES__SHIFT                                                                    0x18
+#define CP_VMID_RESET__RESET_REQUEST_MASK                                                                     0x0000FFFFL
+#define CP_VMID_RESET__PIPE0_QUEUES_MASK                                                                      0x00FF0000L
+#define CP_VMID_RESET__PIPE1_QUEUES_MASK                                                                      0xFF000000L
+//CPC_INT_CNTL
+#define CPC_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE__SHIFT                                                      0xc
+#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE__SHIFT                                                       0xd
+#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE__SHIFT                                                          0xe
+#define CPC_INT_CNTL__SUA_VIOLATION_INT_ENABLE__SHIFT                                                         0xf
+#define CPC_INT_CNTL__GPF_INT_ENABLE__SHIFT                                                                   0x10
+#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE__SHIFT                                                      0x11
+#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE__SHIFT                                                              0x17
+#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE__SHIFT                                                          0x18
+#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE__SHIFT                                                            0x1a
+#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE__SHIFT                                                    0x1b
+#define CPC_INT_CNTL__GENERIC2_INT_ENABLE__SHIFT                                                              0x1d
+#define CPC_INT_CNTL__GENERIC1_INT_ENABLE__SHIFT                                                              0x1e
+#define CPC_INT_CNTL__GENERIC0_INT_ENABLE__SHIFT                                                              0x1f
+#define CPC_INT_CNTL__CMP_QUERY_STATUS_INT_ENABLE_MASK                                                        0x00001000L
+#define CPC_INT_CNTL__DEQUEUE_REQUEST_INT_ENABLE_MASK                                                         0x00002000L
+#define CPC_INT_CNTL__CP_ECC_ERROR_INT_ENABLE_MASK                                                            0x00004000L
+#define CPC_INT_CNTL__SUA_VIOLATION_INT_ENABLE_MASK                                                           0x00008000L
+#define CPC_INT_CNTL__GPF_INT_ENABLE_MASK                                                                     0x00010000L
+#define CPC_INT_CNTL__WRM_POLL_TIMEOUT_INT_ENABLE_MASK                                                        0x00020000L
+#define CPC_INT_CNTL__PRIV_REG_INT_ENABLE_MASK                                                                0x00800000L
+#define CPC_INT_CNTL__OPCODE_ERROR_INT_ENABLE_MASK                                                            0x01000000L
+#define CPC_INT_CNTL__TIME_STAMP_INT_ENABLE_MASK                                                              0x04000000L
+#define CPC_INT_CNTL__RESERVED_BIT_ERROR_INT_ENABLE_MASK                                                      0x08000000L
+#define CPC_INT_CNTL__GENERIC2_INT_ENABLE_MASK                                                                0x20000000L
+#define CPC_INT_CNTL__GENERIC1_INT_ENABLE_MASK                                                                0x40000000L
+#define CPC_INT_CNTL__GENERIC0_INT_ENABLE_MASK                                                                0x80000000L
+//CPC_INT_STATUS
+#define CPC_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS__SHIFT                                                    0xc
+#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS__SHIFT                                                     0xd
+#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS__SHIFT                                                        0xe
+#define CPC_INT_STATUS__SUA_VIOLATION_INT_STATUS__SHIFT                                                       0xf
+#define CPC_INT_STATUS__GPF_INT_STATUS__SHIFT                                                                 0x10
+#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS__SHIFT                                                    0x11
+#define CPC_INT_STATUS__PRIV_REG_INT_STATUS__SHIFT                                                            0x17
+#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS__SHIFT                                                        0x18
+#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS__SHIFT                                                          0x1a
+#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS__SHIFT                                                  0x1b
+#define CPC_INT_STATUS__GENERIC2_INT_STATUS__SHIFT                                                            0x1d
+#define CPC_INT_STATUS__GENERIC1_INT_STATUS__SHIFT                                                            0x1e
+#define CPC_INT_STATUS__GENERIC0_INT_STATUS__SHIFT                                                            0x1f
+#define CPC_INT_STATUS__CMP_QUERY_STATUS_INT_STATUS_MASK                                                      0x00001000L
+#define CPC_INT_STATUS__DEQUEUE_REQUEST_INT_STATUS_MASK                                                       0x00002000L
+#define CPC_INT_STATUS__CP_ECC_ERROR_INT_STATUS_MASK                                                          0x00004000L
+#define CPC_INT_STATUS__SUA_VIOLATION_INT_STATUS_MASK                                                         0x00008000L
+#define CPC_INT_STATUS__GPF_INT_STATUS_MASK                                                                   0x00010000L
+#define CPC_INT_STATUS__WRM_POLL_TIMEOUT_INT_STATUS_MASK                                                      0x00020000L
+#define CPC_INT_STATUS__PRIV_REG_INT_STATUS_MASK                                                              0x00800000L
+#define CPC_INT_STATUS__OPCODE_ERROR_INT_STATUS_MASK                                                          0x01000000L
+#define CPC_INT_STATUS__TIME_STAMP_INT_STATUS_MASK                                                            0x04000000L
+#define CPC_INT_STATUS__RESERVED_BIT_ERROR_INT_STATUS_MASK                                                    0x08000000L
+#define CPC_INT_STATUS__GENERIC2_INT_STATUS_MASK                                                              0x20000000L
+#define CPC_INT_STATUS__GENERIC1_INT_STATUS_MASK                                                              0x40000000L
+#define CPC_INT_STATUS__GENERIC0_INT_STATUS_MASK                                                              0x80000000L
+//CP_VMID_PREEMPT
+#define CP_VMID_PREEMPT__PREEMPT_REQUEST__SHIFT                                                               0x0
+#define CP_VMID_PREEMPT__VIRT_COMMAND__SHIFT                                                                  0x10
+#define CP_VMID_PREEMPT__PREEMPT_REQUEST_MASK                                                                 0x0000FFFFL
+#define CP_VMID_PREEMPT__VIRT_COMMAND_MASK                                                                    0x000F0000L
+//CPC_INT_CNTX_ID
+#define CPC_INT_CNTX_ID__CNTX_ID__SHIFT                                                                       0x0
+#define CPC_INT_CNTX_ID__CNTX_ID_MASK                                                                         0xFFFFFFFFL
+//CP_PQ_STATUS
+#define CP_PQ_STATUS__DOORBELL_UPDATED__SHIFT                                                                 0x0
+#define CP_PQ_STATUS__DOORBELL_ENABLE__SHIFT                                                                  0x1
+#define CP_PQ_STATUS__DOORBELL_UPDATED_EN__SHIFT                                                              0x2
+#define CP_PQ_STATUS__DOORBELL_UPDATED_MODE__SHIFT                                                            0x3
+#define CP_PQ_STATUS__DOORBELL_UPDATED_MASK                                                                   0x00000001L
+#define CP_PQ_STATUS__DOORBELL_ENABLE_MASK                                                                    0x00000002L
+#define CP_PQ_STATUS__DOORBELL_UPDATED_EN_MASK                                                                0x00000004L
+#define CP_PQ_STATUS__DOORBELL_UPDATED_MODE_MASK                                                              0x00000008L
+//CP_PFP_PRGRM_CNTR_START_HI
+#define CP_PFP_PRGRM_CNTR_START_HI__IP_START__SHIFT                                                           0x0
+#define CP_PFP_PRGRM_CNTR_START_HI__IP_START_MASK                                                             0x3FFFFFFFL
+//CP_MAX_DRAW_COUNT
+#define CP_MAX_DRAW_COUNT__MAX_DRAW_COUNT__SHIFT                                                              0x0
+#define CP_MAX_DRAW_COUNT__MAX_DRAW_COUNT_MASK                                                                0xFFFFFFFFL
+//CP_MEC1_F32_INT_DIS
+#define CP_MEC1_F32_INT_DIS__EDC_ROQ_FED_INT__SHIFT                                                           0x0
+#define CP_MEC1_F32_INT_DIS__PRIV_REG_INT__SHIFT                                                              0x1
+#define CP_MEC1_F32_INT_DIS__RESERVED_BIT_ERR_INT__SHIFT                                                      0x2
+#define CP_MEC1_F32_INT_DIS__EDC_TC_FED_INT__SHIFT                                                            0x3
+#define CP_MEC1_F32_INT_DIS__EDC_GDS_FED_INT__SHIFT                                                           0x4
+#define CP_MEC1_F32_INT_DIS__EDC_SCRATCH_FED_INT__SHIFT                                                       0x5
+#define CP_MEC1_F32_INT_DIS__WAVE_RESTORE_INT__SHIFT                                                          0x6
+#define CP_MEC1_F32_INT_DIS__SUA_VIOLATION_INT__SHIFT                                                         0x7
+#define CP_MEC1_F32_INT_DIS__EDC_DMA_FED_INT__SHIFT                                                           0x8
+#define CP_MEC1_F32_INT_DIS__IQ_TIMER_INT__SHIFT                                                              0x9
+#define CP_MEC1_F32_INT_DIS__GPF_INT_CPF__SHIFT                                                               0xa
+#define CP_MEC1_F32_INT_DIS__GPF_INT_DMA__SHIFT                                                               0xb
+#define CP_MEC1_F32_INT_DIS__GPF_INT_CPC__SHIFT                                                               0xc
+#define CP_MEC1_F32_INT_DIS__EDC_SR_MEM_FED_INT__SHIFT                                                        0xd
+#define CP_MEC1_F32_INT_DIS__QUEUE_MESSAGE_INT__SHIFT                                                         0xe
+#define CP_MEC1_F32_INT_DIS__FATAL_EDC_ERROR_INT__SHIFT                                                       0xf
+#define CP_MEC1_F32_INT_DIS__EDC_ROQ_FED_INT_MASK                                                             0x00000001L
+#define CP_MEC1_F32_INT_DIS__PRIV_REG_INT_MASK                                                                0x00000002L
+#define CP_MEC1_F32_INT_DIS__RESERVED_BIT_ERR_INT_MASK                                                        0x00000004L
+#define CP_MEC1_F32_INT_DIS__EDC_TC_FED_INT_MASK                                                              0x00000008L
+#define CP_MEC1_F32_INT_DIS__EDC_GDS_FED_INT_MASK                                                             0x00000010L
+#define CP_MEC1_F32_INT_DIS__EDC_SCRATCH_FED_INT_MASK                                                         0x00000020L
+#define CP_MEC1_F32_INT_DIS__WAVE_RESTORE_INT_MASK                                                            0x00000040L
+#define CP_MEC1_F32_INT_DIS__SUA_VIOLATION_INT_MASK                                                           0x00000080L
+#define CP_MEC1_F32_INT_DIS__EDC_DMA_FED_INT_MASK                                                             0x00000100L
+#define CP_MEC1_F32_INT_DIS__IQ_TIMER_INT_MASK                                                                0x00000200L
+#define CP_MEC1_F32_INT_DIS__GPF_INT_CPF_MASK                                                                 0x00000400L
+#define CP_MEC1_F32_INT_DIS__GPF_INT_DMA_MASK                                                                 0x00000800L
+#define CP_MEC1_F32_INT_DIS__GPF_INT_CPC_MASK                                                                 0x00001000L
+#define CP_MEC1_F32_INT_DIS__EDC_SR_MEM_FED_INT_MASK                                                          0x00002000L
+#define CP_MEC1_F32_INT_DIS__QUEUE_MESSAGE_INT_MASK                                                           0x00004000L
+#define CP_MEC1_F32_INT_DIS__FATAL_EDC_ERROR_INT_MASK                                                         0x00008000L
+//CP_MEC2_F32_INT_DIS
+#define CP_MEC2_F32_INT_DIS__EDC_ROQ_FED_INT__SHIFT                                                           0x0
+#define CP_MEC2_F32_INT_DIS__PRIV_REG_INT__SHIFT                                                              0x1
+#define CP_MEC2_F32_INT_DIS__RESERVED_BIT_ERR_INT__SHIFT                                                      0x2
+#define CP_MEC2_F32_INT_DIS__EDC_TC_FED_INT__SHIFT                                                            0x3
+#define CP_MEC2_F32_INT_DIS__EDC_GDS_FED_INT__SHIFT                                                           0x4
+#define CP_MEC2_F32_INT_DIS__EDC_SCRATCH_FED_INT__SHIFT                                                       0x5
+#define CP_MEC2_F32_INT_DIS__WAVE_RESTORE_INT__SHIFT                                                          0x6
+#define CP_MEC2_F32_INT_DIS__SUA_VIOLATION_INT__SHIFT                                                         0x7
+#define CP_MEC2_F32_INT_DIS__EDC_DMA_FED_INT__SHIFT                                                           0x8
+#define CP_MEC2_F32_INT_DIS__IQ_TIMER_INT__SHIFT                                                              0x9
+#define CP_MEC2_F32_INT_DIS__GPF_INT_CPF__SHIFT                                                               0xa
+#define CP_MEC2_F32_INT_DIS__GPF_INT_DMA__SHIFT                                                               0xb
+#define CP_MEC2_F32_INT_DIS__GPF_INT_CPC__SHIFT                                                               0xc
+#define CP_MEC2_F32_INT_DIS__EDC_SR_MEM_FED_INT__SHIFT                                                        0xd
+#define CP_MEC2_F32_INT_DIS__QUEUE_MESSAGE_INT__SHIFT                                                         0xe
+#define CP_MEC2_F32_INT_DIS__FATAL_EDC_ERROR_INT__SHIFT                                                       0xf
+#define CP_MEC2_F32_INT_DIS__EDC_ROQ_FED_INT_MASK                                                             0x00000001L
+#define CP_MEC2_F32_INT_DIS__PRIV_REG_INT_MASK                                                                0x00000002L
+#define CP_MEC2_F32_INT_DIS__RESERVED_BIT_ERR_INT_MASK                                                        0x00000004L
+#define CP_MEC2_F32_INT_DIS__EDC_TC_FED_INT_MASK                                                              0x00000008L
+#define CP_MEC2_F32_INT_DIS__EDC_GDS_FED_INT_MASK                                                             0x00000010L
+#define CP_MEC2_F32_INT_DIS__EDC_SCRATCH_FED_INT_MASK                                                         0x00000020L
+#define CP_MEC2_F32_INT_DIS__WAVE_RESTORE_INT_MASK                                                            0x00000040L
+#define CP_MEC2_F32_INT_DIS__SUA_VIOLATION_INT_MASK                                                           0x00000080L
+#define CP_MEC2_F32_INT_DIS__EDC_DMA_FED_INT_MASK                                                             0x00000100L
+#define CP_MEC2_F32_INT_DIS__IQ_TIMER_INT_MASK                                                                0x00000200L
+#define CP_MEC2_F32_INT_DIS__GPF_INT_CPF_MASK                                                                 0x00000400L
+#define CP_MEC2_F32_INT_DIS__GPF_INT_DMA_MASK                                                                 0x00000800L
+#define CP_MEC2_F32_INT_DIS__GPF_INT_CPC_MASK                                                                 0x00001000L
+#define CP_MEC2_F32_INT_DIS__EDC_SR_MEM_FED_INT_MASK                                                          0x00002000L
+#define CP_MEC2_F32_INT_DIS__QUEUE_MESSAGE_INT_MASK                                                           0x00004000L
+#define CP_MEC2_F32_INT_DIS__FATAL_EDC_ERROR_INT_MASK                                                         0x00008000L
+//CP_VMID_STATUS
+#define CP_VMID_STATUS__PREEMPT_DE_STATUS__SHIFT                                                              0x0
+#define CP_VMID_STATUS__PREEMPT_CE_STATUS__SHIFT                                                              0x10
+#define CP_VMID_STATUS__PREEMPT_DE_STATUS_MASK                                                                0x0000FFFFL
+#define CP_VMID_STATUS__PREEMPT_CE_STATUS_MASK                                                                0xFFFF0000L
+//CPC_SUSPEND_CTX_SAVE_BASE_ADDR_LO
+#define CPC_SUSPEND_CTX_SAVE_BASE_ADDR_LO__ADDR__SHIFT                                                        0xc
+#define CPC_SUSPEND_CTX_SAVE_BASE_ADDR_LO__ADDR_MASK                                                          0xFFFFF000L
+//CPC_SUSPEND_CTX_SAVE_BASE_ADDR_HI
+#define CPC_SUSPEND_CTX_SAVE_BASE_ADDR_HI__ADDR_HI__SHIFT                                                     0x0
+#define CPC_SUSPEND_CTX_SAVE_BASE_ADDR_HI__ADDR_HI_MASK                                                       0x0000FFFFL
+//CPC_SUSPEND_CTX_SAVE_CONTROL
+#define CPC_SUSPEND_CTX_SAVE_CONTROL__POLICY__SHIFT                                                           0x3
+#define CPC_SUSPEND_CTX_SAVE_CONTROL__EXE_DISABLE__SHIFT                                                      0x17
+#define CPC_SUSPEND_CTX_SAVE_CONTROL__POLICY_MASK                                                             0x00000018L
+#define CPC_SUSPEND_CTX_SAVE_CONTROL__EXE_DISABLE_MASK                                                        0x00800000L
+//CPC_SUSPEND_CNTL_STACK_OFFSET
+#define CPC_SUSPEND_CNTL_STACK_OFFSET__OFFSET__SHIFT                                                          0x2
+#define CPC_SUSPEND_CNTL_STACK_OFFSET__OFFSET_MASK                                                            0x0000FFFCL
+//CPC_SUSPEND_CNTL_STACK_SIZE
+#define CPC_SUSPEND_CNTL_STACK_SIZE__SIZE__SHIFT                                                              0xc
+#define CPC_SUSPEND_CNTL_STACK_SIZE__SIZE_MASK                                                                0x0000F000L
+//CPC_SUSPEND_WG_STATE_OFFSET
+#define CPC_SUSPEND_WG_STATE_OFFSET__OFFSET__SHIFT                                                            0x2
+#define CPC_SUSPEND_WG_STATE_OFFSET__OFFSET_MASK                                                              0x03FFFFFCL
+//CPC_SUSPEND_CTX_SAVE_SIZE
+#define CPC_SUSPEND_CTX_SAVE_SIZE__SIZE__SHIFT                                                                0xc
+#define CPC_SUSPEND_CTX_SAVE_SIZE__SIZE_MASK                                                                  0x03FFF000L
+//CPC_OS_PIPES
+#define CPC_OS_PIPES__OS_PIPES__SHIFT                                                                         0x0
+#define CPC_OS_PIPES__OS_PIPES_MASK                                                                           0x000000FFL
+//CP_SUSPEND_RESUME_REQ
+#define CP_SUSPEND_RESUME_REQ__SUSPEND_REQ__SHIFT                                                             0x0
+#define CP_SUSPEND_RESUME_REQ__RESUME_REQ__SHIFT                                                              0x1
+#define CP_SUSPEND_RESUME_REQ__SUSPEND_REQ_MASK                                                               0x00000001L
+#define CP_SUSPEND_RESUME_REQ__RESUME_REQ_MASK                                                                0x00000002L
+//CP_SUSPEND_CNTL
+#define CP_SUSPEND_CNTL__SUSPEND_MODE__SHIFT                                                                  0x0
+#define CP_SUSPEND_CNTL__SUSPEND_ENABLE__SHIFT                                                                0x1
+#define CP_SUSPEND_CNTL__RESUME_LOCK__SHIFT                                                                   0x2
+#define CP_SUSPEND_CNTL__ACE_SUSPEND_ACTIVE__SHIFT                                                            0x3
+#define CP_SUSPEND_CNTL__SUSPEND_MODE_MASK                                                                    0x00000001L
+#define CP_SUSPEND_CNTL__SUSPEND_ENABLE_MASK                                                                  0x00000002L
+#define CP_SUSPEND_CNTL__RESUME_LOCK_MASK                                                                     0x00000004L
+#define CP_SUSPEND_CNTL__ACE_SUSPEND_ACTIVE_MASK                                                              0x00000008L
+//CP_IQ_WAIT_TIME3
+#define CP_IQ_WAIT_TIME3__SUSPEND_QUE__SHIFT                                                                  0x0
+#define CP_IQ_WAIT_TIME3__SUSPEND_QUE_MASK                                                                    0x000000FFL
+//CPC_DDID_BASE_ADDR_LO
+#define CPC_DDID_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                                            0x6
+#define CPC_DDID_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                              0xFFFFFFC0L
+//CP_DDID_BASE_ADDR_LO
+#define CP_DDID_BASE_ADDR_LO__BASE_ADDR_LO__SHIFT                                                             0x6
+#define CP_DDID_BASE_ADDR_LO__BASE_ADDR_LO_MASK                                                               0xFFFFFFC0L
+//CPC_DDID_BASE_ADDR_HI
+#define CPC_DDID_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                            0x0
+#define CPC_DDID_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                              0x0000FFFFL
+//CP_DDID_BASE_ADDR_HI
+#define CP_DDID_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                             0x0
+#define CP_DDID_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                               0x0000FFFFL
+//CPC_DDID_CNTL
+#define CPC_DDID_CNTL__THRESHOLD__SHIFT                                                                       0x0
+#define CPC_DDID_CNTL__SIZE__SHIFT                                                                            0x10
+#define CPC_DDID_CNTL__NO_RING_MEMORY__SHIFT                                                                  0x13
+#define CPC_DDID_CNTL__POLICY__SHIFT                                                                          0x1c
+#define CPC_DDID_CNTL__MODE__SHIFT                                                                            0x1e
+#define CPC_DDID_CNTL__ENABLE__SHIFT                                                                          0x1f
+#define CPC_DDID_CNTL__THRESHOLD_MASK                                                                         0x000000FFL
+#define CPC_DDID_CNTL__SIZE_MASK                                                                              0x00010000L
+#define CPC_DDID_CNTL__NO_RING_MEMORY_MASK                                                                    0x00080000L
+#define CPC_DDID_CNTL__POLICY_MASK                                                                            0x30000000L
+#define CPC_DDID_CNTL__MODE_MASK                                                                              0x40000000L
+#define CPC_DDID_CNTL__ENABLE_MASK                                                                            0x80000000L
+//CP_DDID_CNTL
+#define CP_DDID_CNTL__THRESHOLD__SHIFT                                                                        0x0
+#define CP_DDID_CNTL__SIZE__SHIFT                                                                             0x10
+#define CP_DDID_CNTL__NO_RING_MEMORY__SHIFT                                                                   0x13
+#define CP_DDID_CNTL__VMID__SHIFT                                                                             0x14
+#define CP_DDID_CNTL__VMID_SEL__SHIFT                                                                         0x18
+#define CP_DDID_CNTL__POLICY__SHIFT                                                                           0x1c
+#define CP_DDID_CNTL__MODE__SHIFT                                                                             0x1e
+#define CP_DDID_CNTL__ENABLE__SHIFT                                                                           0x1f
+#define CP_DDID_CNTL__THRESHOLD_MASK                                                                          0x000000FFL
+#define CP_DDID_CNTL__SIZE_MASK                                                                               0x00010000L
+#define CP_DDID_CNTL__NO_RING_MEMORY_MASK                                                                     0x00080000L
+#define CP_DDID_CNTL__VMID_MASK                                                                               0x00F00000L
+#define CP_DDID_CNTL__VMID_SEL_MASK                                                                           0x01000000L
+#define CP_DDID_CNTL__POLICY_MASK                                                                             0x30000000L
+#define CP_DDID_CNTL__MODE_MASK                                                                               0x40000000L
+#define CP_DDID_CNTL__ENABLE_MASK                                                                             0x80000000L
+//CP_GFX_DDID_INFLIGHT_COUNT
+#define CP_GFX_DDID_INFLIGHT_COUNT__COUNT__SHIFT                                                              0x0
+#define CP_GFX_DDID_INFLIGHT_COUNT__COUNT_MASK                                                                0x0000FFFFL
+//CP_GFX_DDID_WPTR
+#define CP_GFX_DDID_WPTR__COUNT__SHIFT                                                                        0x0
+#define CP_GFX_DDID_WPTR__COUNT_MASK                                                                          0x0000FFFFL
+//CP_GFX_DDID_RPTR
+#define CP_GFX_DDID_RPTR__COUNT__SHIFT                                                                        0x0
+#define CP_GFX_DDID_RPTR__COUNT_MASK                                                                          0x0000FFFFL
+//CP_GFX_DDID_DELTA_RPT_COUNT
+#define CP_GFX_DDID_DELTA_RPT_COUNT__COUNT__SHIFT                                                             0x0
+#define CP_GFX_DDID_DELTA_RPT_COUNT__COUNT_MASK                                                               0x000000FFL
+//CP_GFX_HPD_STATUS0
+#define CP_GFX_HPD_STATUS0__QUEUE_STATE__SHIFT                                                                0x0
+#define CP_GFX_HPD_STATUS0__MAPPED_QUEUE__SHIFT                                                               0x5
+#define CP_GFX_HPD_STATUS0__QUEUE_AVAILABLE__SHIFT                                                            0x8
+#define CP_GFX_HPD_STATUS0__FORCE_MAPPED_QUEUE__SHIFT                                                         0x10
+#define CP_GFX_HPD_STATUS0__FORCE_QUEUE_STATE__SHIFT                                                          0x14
+#define CP_GFX_HPD_STATUS0__SUSPEND_REQ__SHIFT                                                                0x1c
+#define CP_GFX_HPD_STATUS0__ENABLE_OVERIDE_QUEUEID__SHIFT                                                     0x1d
+#define CP_GFX_HPD_STATUS0__OVERIDE_QUEUEID__SHIFT                                                            0x1e
+#define CP_GFX_HPD_STATUS0__FORCE_QUEUE__SHIFT                                                                0x1f
+#define CP_GFX_HPD_STATUS0__QUEUE_STATE_MASK                                                                  0x0000001FL
+#define CP_GFX_HPD_STATUS0__MAPPED_QUEUE_MASK                                                                 0x000000E0L
+#define CP_GFX_HPD_STATUS0__QUEUE_AVAILABLE_MASK                                                              0x0000FF00L
+#define CP_GFX_HPD_STATUS0__FORCE_MAPPED_QUEUE_MASK                                                           0x00070000L
+#define CP_GFX_HPD_STATUS0__FORCE_QUEUE_STATE_MASK                                                            0x01F00000L
+#define CP_GFX_HPD_STATUS0__SUSPEND_REQ_MASK                                                                  0x10000000L
+#define CP_GFX_HPD_STATUS0__ENABLE_OVERIDE_QUEUEID_MASK                                                       0x20000000L
+#define CP_GFX_HPD_STATUS0__OVERIDE_QUEUEID_MASK                                                              0x40000000L
+#define CP_GFX_HPD_STATUS0__FORCE_QUEUE_MASK                                                                  0x80000000L
+//CP_GFX_HPD_CONTROL0
+#define CP_GFX_HPD_CONTROL0__SUSPEND_ENABLE__SHIFT                                                            0x0
+#define CP_GFX_HPD_CONTROL0__PIPE_HOLDING__SHIFT                                                              0x4
+#define CP_GFX_HPD_CONTROL0__RB_CE_ROQ_CNTL__SHIFT                                                            0x8
+#define CP_GFX_HPD_CONTROL0__SUSPEND_ENABLE_MASK                                                              0x00000001L
+#define CP_GFX_HPD_CONTROL0__PIPE_HOLDING_MASK                                                                0x00000010L
+#define CP_GFX_HPD_CONTROL0__RB_CE_ROQ_CNTL_MASK                                                              0x00000100L
+//CP_GFX_HPD_OSPRE_FENCE_ADDR_LO
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_LO__ADDR_LO__SHIFT                                                        0x2
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_LO__ADDR_LO_MASK                                                          0xFFFFFFFCL
+//CP_GFX_HPD_OSPRE_FENCE_ADDR_HI
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_HI__ADDR_HI__SHIFT                                                        0x0
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_HI__RSVD__SHIFT                                                           0x10
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_HI__ADDR_HI_MASK                                                          0x0000FFFFL
+#define CP_GFX_HPD_OSPRE_FENCE_ADDR_HI__RSVD_MASK                                                             0xFFFF0000L
+//CP_GFX_HPD_OSPRE_FENCE_DATA_LO
+#define CP_GFX_HPD_OSPRE_FENCE_DATA_LO__DATA_LO__SHIFT                                                        0x0
+#define CP_GFX_HPD_OSPRE_FENCE_DATA_LO__DATA_LO_MASK                                                          0xFFFFFFFFL
+//CP_GFX_HPD_OSPRE_FENCE_DATA_HI
+#define CP_GFX_HPD_OSPRE_FENCE_DATA_HI__DATA_HI__SHIFT                                                        0x0
+#define CP_GFX_HPD_OSPRE_FENCE_DATA_HI__DATA_HI_MASK                                                          0xFFFFFFFFL
+//CP_GFX_INDEX_MUTEX
+#define CP_GFX_INDEX_MUTEX__REQUEST__SHIFT                                                                    0x0
+#define CP_GFX_INDEX_MUTEX__CLIENTID__SHIFT                                                                   0x1
+#define CP_GFX_INDEX_MUTEX__REQUEST_MASK                                                                      0x00000001L
+#define CP_GFX_INDEX_MUTEX__CLIENTID_MASK                                                                     0x0000000EL
+//CP_ME_PRGRM_CNTR_START_HI
+#define CP_ME_PRGRM_CNTR_START_HI__IP_START__SHIFT                                                            0x0
+#define CP_ME_PRGRM_CNTR_START_HI__IP_START_MASK                                                              0x3FFFFFFFL
+//CP_PFP_INTR_ROUTINE_START_HI
+#define CP_PFP_INTR_ROUTINE_START_HI__IR_START__SHIFT                                                         0x0
+#define CP_PFP_INTR_ROUTINE_START_HI__IR_START_MASK                                                           0x3FFFFFFFL
+//CP_ME_INTR_ROUTINE_START_HI
+#define CP_ME_INTR_ROUTINE_START_HI__IR_START__SHIFT                                                          0x0
+#define CP_ME_INTR_ROUTINE_START_HI__IR_START_MASK                                                            0x3FFFFFFFL
+//CP_GFX_MQD_BASE_ADDR
+#define CP_GFX_MQD_BASE_ADDR__BASE_ADDR__SHIFT                                                                0x2
+#define CP_GFX_MQD_BASE_ADDR__BASE_ADDR_MASK                                                                  0xFFFFFFFCL
+//CP_GFX_MQD_BASE_ADDR_HI
+#define CP_GFX_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                          0x0
+#define CP_GFX_MQD_BASE_ADDR_HI__APP_VMID__SHIFT                                                              0x1c
+#define CP_GFX_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                            0x0000FFFFL
+#define CP_GFX_MQD_BASE_ADDR_HI__APP_VMID_MASK                                                                0xF0000000L
+//CP_GFX_HQD_ACTIVE
+#define CP_GFX_HQD_ACTIVE__ACTIVE__SHIFT                                                                      0x0
+#define CP_GFX_HQD_ACTIVE__ACTIVE_MASK                                                                        0x00000001L
+//CP_GFX_HQD_VMID
+#define CP_GFX_HQD_VMID__VMID__SHIFT                                                                          0x0
+#define CP_GFX_HQD_VMID__VMID_MASK                                                                            0x0000000FL
+//CP_GFX_HQD_QUEUE_PRIORITY
+#define CP_GFX_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL__SHIFT                                                      0x0
+#define CP_GFX_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL_MASK                                                        0x0000000FL
+//CP_GFX_HQD_QUANTUM
+#define CP_GFX_HQD_QUANTUM__QUANTUM_EN__SHIFT                                                                 0x0
+#define CP_GFX_HQD_QUANTUM__QUANTUM_SCALE__SHIFT                                                              0x3
+#define CP_GFX_HQD_QUANTUM__QUANTUM_DURATION__SHIFT                                                           0x8
+#define CP_GFX_HQD_QUANTUM__QUANTUM_ACTIVE__SHIFT                                                             0x1f
+#define CP_GFX_HQD_QUANTUM__QUANTUM_EN_MASK                                                                   0x00000001L
+#define CP_GFX_HQD_QUANTUM__QUANTUM_SCALE_MASK                                                                0x00000018L
+#define CP_GFX_HQD_QUANTUM__QUANTUM_DURATION_MASK                                                             0x0000FF00L
+#define CP_GFX_HQD_QUANTUM__QUANTUM_ACTIVE_MASK                                                               0x80000000L
+//CP_GFX_HQD_BASE
+#define CP_GFX_HQD_BASE__RB_BASE__SHIFT                                                                       0x0
+#define CP_GFX_HQD_BASE__RB_BASE_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_HQD_BASE_HI
+#define CP_GFX_HQD_BASE_HI__RB_BASE_HI__SHIFT                                                                 0x0
+#define CP_GFX_HQD_BASE_HI__RB_BASE_HI_MASK                                                                   0x000000FFL
+//CP_GFX_HQD_RPTR
+#define CP_GFX_HQD_RPTR__RB_RPTR__SHIFT                                                                       0x0
+#define CP_GFX_HQD_RPTR__RB_RPTR_MASK                                                                         0x000FFFFFL
+//CP_GFX_HQD_RPTR_ADDR
+#define CP_GFX_HQD_RPTR_ADDR__RB_RPTR_ADDR__SHIFT                                                             0x2
+#define CP_GFX_HQD_RPTR_ADDR__RB_RPTR_ADDR_MASK                                                               0xFFFFFFFCL
+//CP_GFX_HQD_RPTR_ADDR_HI
+#define CP_GFX_HQD_RPTR_ADDR_HI__RB_RPTR_ADDR_HI__SHIFT                                                       0x0
+#define CP_GFX_HQD_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK                                                         0x0000FFFFL
+//CP_RB_WPTR_POLL_ADDR_LO
+#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO__SHIFT                                                  0x2
+#define CP_RB_WPTR_POLL_ADDR_LO__RB_WPTR_POLL_ADDR_LO_MASK                                                    0xFFFFFFFCL
+//CP_RB_WPTR_POLL_ADDR_HI
+#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI__SHIFT                                                  0x0
+#define CP_RB_WPTR_POLL_ADDR_HI__RB_WPTR_POLL_ADDR_HI_MASK                                                    0x0000FFFFL
+//CP_RB_DOORBELL_CONTROL
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT                                                      0x1
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT                                                        0x2
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_EN__SHIFT                                                            0x1e
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT                                                           0x1f
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_BIF_DROP_MASK                                                        0x00000002L
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK                                                          0x0FFFFFFCL
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_EN_MASK                                                              0x40000000L
+#define CP_RB_DOORBELL_CONTROL__DOORBELL_HIT_MASK                                                             0x80000000L
+//CP_GFX_HQD_OFFSET
+#define CP_GFX_HQD_OFFSET__RB_OFFSET__SHIFT                                                                   0x0
+#define CP_GFX_HQD_OFFSET__DISABLE_RB_OFFSET__SHIFT                                                           0x1f
+#define CP_GFX_HQD_OFFSET__RB_OFFSET_MASK                                                                     0x000FFFFFL
+#define CP_GFX_HQD_OFFSET__DISABLE_RB_OFFSET_MASK                                                             0x80000000L
+//CP_GFX_HQD_CNTL
+#define CP_GFX_HQD_CNTL__RB_BUFSZ__SHIFT                                                                      0x0
+#define CP_GFX_HQD_CNTL__TMZ_STATE__SHIFT                                                                     0x6
+#define CP_GFX_HQD_CNTL__TMZ_MATCH__SHIFT                                                                     0x7
+#define CP_GFX_HQD_CNTL__RB_BLKSZ__SHIFT                                                                      0x8
+#define CP_GFX_HQD_CNTL__RB_NON_PRIV__SHIFT                                                                   0xf
+#define CP_GFX_HQD_CNTL__BUF_SWAP__SHIFT                                                                      0x10
+#define CP_GFX_HQD_CNTL__MIN_AVAILSZ__SHIFT                                                                   0x14
+#define CP_GFX_HQD_CNTL__MIN_IB_AVAILSZ__SHIFT                                                                0x16
+#define CP_GFX_HQD_CNTL__CACHE_POLICY__SHIFT                                                                  0x18
+#define CP_GFX_HQD_CNTL__RB_VOLATILE__SHIFT                                                                   0x1a
+#define CP_GFX_HQD_CNTL__RB_NO_UPDATE__SHIFT                                                                  0x1b
+#define CP_GFX_HQD_CNTL__RB_EXE__SHIFT                                                                        0x1c
+#define CP_GFX_HQD_CNTL__KMD_QUEUE__SHIFT                                                                     0x1d
+#define CP_GFX_HQD_CNTL__RB_RPTR_WR_ENA__SHIFT                                                                0x1f
+#define CP_GFX_HQD_CNTL__RB_BUFSZ_MASK                                                                        0x0000003FL
+#define CP_GFX_HQD_CNTL__TMZ_STATE_MASK                                                                       0x00000040L
+#define CP_GFX_HQD_CNTL__TMZ_MATCH_MASK                                                                       0x00000080L
+#define CP_GFX_HQD_CNTL__RB_BLKSZ_MASK                                                                        0x00003F00L
+#define CP_GFX_HQD_CNTL__RB_NON_PRIV_MASK                                                                     0x00008000L
+#define CP_GFX_HQD_CNTL__BUF_SWAP_MASK                                                                        0x00030000L
+#define CP_GFX_HQD_CNTL__MIN_AVAILSZ_MASK                                                                     0x00300000L
+#define CP_GFX_HQD_CNTL__MIN_IB_AVAILSZ_MASK                                                                  0x00C00000L
+#define CP_GFX_HQD_CNTL__CACHE_POLICY_MASK                                                                    0x03000000L
+#define CP_GFX_HQD_CNTL__RB_VOLATILE_MASK                                                                     0x04000000L
+#define CP_GFX_HQD_CNTL__RB_NO_UPDATE_MASK                                                                    0x08000000L
+#define CP_GFX_HQD_CNTL__RB_EXE_MASK                                                                          0x10000000L
+#define CP_GFX_HQD_CNTL__KMD_QUEUE_MASK                                                                       0x20000000L
+#define CP_GFX_HQD_CNTL__RB_RPTR_WR_ENA_MASK                                                                  0x80000000L
+//CP_GFX_HQD_CSMD_RPTR
+#define CP_GFX_HQD_CSMD_RPTR__RB_RPTR__SHIFT                                                                  0x0
+#define CP_GFX_HQD_CSMD_RPTR__RB_RPTR_MASK                                                                    0x000FFFFFL
+//CP_GFX_HQD_WPTR
+#define CP_GFX_HQD_WPTR__RB_WPTR__SHIFT                                                                       0x0
+#define CP_GFX_HQD_WPTR__RB_WPTR_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_HQD_WPTR_HI
+#define CP_GFX_HQD_WPTR_HI__RB_WPTR__SHIFT                                                                    0x0
+#define CP_GFX_HQD_WPTR_HI__RB_WPTR_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_HQD_DEQUEUE_REQUEST
+#define CP_GFX_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ__SHIFT                                                        0x0
+#define CP_GFX_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND__SHIFT                                                        0x4
+#define CP_GFX_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN__SHIFT                                                     0x9
+#define CP_GFX_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN__SHIFT                                                     0xa
+#define CP_GFX_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_MASK                                                          0x00000001L
+#define CP_GFX_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK                                                          0x00000010L
+#define CP_GFX_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN_MASK                                                       0x00000200L
+#define CP_GFX_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN_MASK                                                       0x00000400L
+//CP_GFX_HQD_MAPPED
+#define CP_GFX_HQD_MAPPED__MAPPED__SHIFT                                                                      0x0
+#define CP_GFX_HQD_MAPPED__MAPPED_MASK                                                                        0x00000001L
+//CP_GFX_HQD_QUE_MGR_CONTROL
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_IDLE_QUEUE_DISCONNECT__SHIFT                                      0x0
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_CONNECT_HANDSHAKE__SHIFT                                          0x4
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_FETCHER_DISCONNECT__SHIFT                                         0x5
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_QUEUE_ACTIVE_EN__SHIFT                                              0x6
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_ALLOW_DB_UPDATE_EN__SHIFT                                           0x7
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_QUEUE__SHIFT                                                        0x8
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_OFFSET_UPDATE__SHIFT                                              0xb
+#define CP_GFX_HQD_QUE_MGR_CONTROL__PRIORITY_PREEMPT_DISABLE__SHIFT                                           0xd
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_QUEUE_MGR__SHIFT                                                  0xf
+#define CP_GFX_HQD_QUE_MGR_CONTROL__ENABLE_IDLE_MESSAGE__SHIFT                                                0x10
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_SWITCH_MESSAGE_IDLE__SHIFT                                        0x11
+#define CP_GFX_HQD_QUE_MGR_CONTROL__ENABLE_SWITCH_MSG_PREEMPT__SHIFT                                          0x12
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_MAPPED_QUEUE_IDLE_MSG__SHIFT                                      0x17
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_IDLE_QUEUE_DISCONNECT_MASK                                        0x00000001L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_CONNECT_HANDSHAKE_MASK                                            0x00000010L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_FETCHER_DISCONNECT_MASK                                           0x00000020L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_QUEUE_ACTIVE_EN_MASK                                                0x00000040L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_ALLOW_DB_UPDATE_EN_MASK                                             0x00000080L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__FORCE_QUEUE_MASK                                                          0x00000700L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_OFFSET_UPDATE_MASK                                                0x00000800L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__PRIORITY_PREEMPT_DISABLE_MASK                                             0x00002000L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_QUEUE_MGR_MASK                                                    0x00008000L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__ENABLE_IDLE_MESSAGE_MASK                                                  0x00010000L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_SWITCH_MESSAGE_IDLE_MASK                                          0x00020000L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__ENABLE_SWITCH_MSG_PREEMPT_MASK                                            0x00040000L
+#define CP_GFX_HQD_QUE_MGR_CONTROL__DISABLE_MAPPED_QUEUE_IDLE_MSG_MASK                                        0x00800000L
+//CP_GFX_HQD_IQ_TIMER
+#define CP_GFX_HQD_IQ_TIMER__WAIT_TIME__SHIFT                                                                 0x0
+#define CP_GFX_HQD_IQ_TIMER__RETRY_TYPE__SHIFT                                                                0x8
+#define CP_GFX_HQD_IQ_TIMER__IMMEDIATE_EXPIRE__SHIFT                                                          0xb
+#define CP_GFX_HQD_IQ_TIMER__INTERRUPT_TYPE__SHIFT                                                            0xc
+#define CP_GFX_HQD_IQ_TIMER__CLOCK_COUNT__SHIFT                                                               0xe
+#define CP_GFX_HQD_IQ_TIMER__QUANTUM_TIMER__SHIFT                                                             0x16
+#define CP_GFX_HQD_IQ_TIMER__QUEUE_TYPE__SHIFT                                                                0x1b
+#define CP_GFX_HQD_IQ_TIMER__REARM_TIMER__SHIFT                                                               0x1c
+#define CP_GFX_HQD_IQ_TIMER__ACTIVE__SHIFT                                                                    0x1f
+#define CP_GFX_HQD_IQ_TIMER__WAIT_TIME_MASK                                                                   0x000000FFL
+#define CP_GFX_HQD_IQ_TIMER__RETRY_TYPE_MASK                                                                  0x00000700L
+#define CP_GFX_HQD_IQ_TIMER__IMMEDIATE_EXPIRE_MASK                                                            0x00000800L
+#define CP_GFX_HQD_IQ_TIMER__INTERRUPT_TYPE_MASK                                                              0x00003000L
+#define CP_GFX_HQD_IQ_TIMER__CLOCK_COUNT_MASK                                                                 0x0000C000L
+#define CP_GFX_HQD_IQ_TIMER__QUANTUM_TIMER_MASK                                                               0x00400000L
+#define CP_GFX_HQD_IQ_TIMER__QUEUE_TYPE_MASK                                                                  0x08000000L
+#define CP_GFX_HQD_IQ_TIMER__REARM_TIMER_MASK                                                                 0x10000000L
+#define CP_GFX_HQD_IQ_TIMER__ACTIVE_MASK                                                                      0x80000000L
+//CP_GFX_HQD_HQ_STATUS0
+#define CP_GFX_HQD_HQ_STATUS0__DEQUEUE_STATUS__SHIFT                                                          0x0
+#define CP_GFX_HQD_HQ_STATUS0__OS_PREEMPT_STATUS__SHIFT                                                       0x4
+#define CP_GFX_HQD_HQ_STATUS0__PREEMPT_ACK__SHIFT                                                             0x6
+#define CP_GFX_HQD_HQ_STATUS0__QUEUE_IDLE__SHIFT                                                              0x1e
+#define CP_GFX_HQD_HQ_STATUS0__DEQUEUE_STATUS_MASK                                                            0x00000001L
+#define CP_GFX_HQD_HQ_STATUS0__OS_PREEMPT_STATUS_MASK                                                         0x00000030L
+#define CP_GFX_HQD_HQ_STATUS0__PREEMPT_ACK_MASK                                                               0x00000040L
+#define CP_GFX_HQD_HQ_STATUS0__QUEUE_IDLE_MASK                                                                0x40000000L
+//CP_GFX_HQD_HQ_CONTROL0
+#define CP_GFX_HQD_HQ_CONTROL0__COMMAND__SHIFT                                                                0x0
+#define CP_GFX_HQD_HQ_CONTROL0__SPARES__SHIFT                                                                 0x4
+#define CP_GFX_HQD_HQ_CONTROL0__COMMAND_MASK                                                                  0x0000000FL
+#define CP_GFX_HQD_HQ_CONTROL0__SPARES_MASK                                                                   0x000000F0L
+//CP_GFX_MQD_CONTROL
+#define CP_GFX_MQD_CONTROL__VMID__SHIFT                                                                       0x0
+#define CP_GFX_MQD_CONTROL__PRIV_STATE__SHIFT                                                                 0x8
+#define CP_GFX_MQD_CONTROL__PROCESSING_MQD__SHIFT                                                             0xc
+#define CP_GFX_MQD_CONTROL__PROCESSING_MQD_EN__SHIFT                                                          0xd
+#define CP_GFX_MQD_CONTROL__EXE_DISABLE__SHIFT                                                                0x17
+#define CP_GFX_MQD_CONTROL__CACHE_POLICY__SHIFT                                                               0x18
+#define CP_GFX_MQD_CONTROL__VMID_MASK                                                                         0x0000000FL
+#define CP_GFX_MQD_CONTROL__PRIV_STATE_MASK                                                                   0x00000100L
+#define CP_GFX_MQD_CONTROL__PROCESSING_MQD_MASK                                                               0x00001000L
+#define CP_GFX_MQD_CONTROL__PROCESSING_MQD_EN_MASK                                                            0x00002000L
+#define CP_GFX_MQD_CONTROL__EXE_DISABLE_MASK                                                                  0x00800000L
+#define CP_GFX_MQD_CONTROL__CACHE_POLICY_MASK                                                                 0x03000000L
+//CP_HQD_GFX_CONTROL
+#define CP_HQD_GFX_CONTROL__MESSAGE__SHIFT                                                                    0x0
+#define CP_HQD_GFX_CONTROL__MISC__SHIFT                                                                       0x4
+#define CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN__SHIFT                                                          0xf
+#define CP_HQD_GFX_CONTROL__MESSAGE_MASK                                                                      0x0000000FL
+#define CP_HQD_GFX_CONTROL__MISC_MASK                                                                         0x00007FF0L
+#define CP_HQD_GFX_CONTROL__DB_UPDATED_MSG_EN_MASK                                                            0x00008000L
+//CP_HQD_GFX_STATUS
+#define CP_HQD_GFX_STATUS__STATUS__SHIFT                                                                      0x0
+#define CP_HQD_GFX_STATUS__STATUS_MASK                                                                        0x0000FFFFL
+//CP_DMA_WATCH0_ADDR_LO
+#define CP_DMA_WATCH0_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_DMA_WATCH0_ADDR_LO__ADDR_LO__SHIFT                                                                 0x7
+#define CP_DMA_WATCH0_ADDR_LO__RSVD_MASK                                                                      0x0000007FL
+#define CP_DMA_WATCH0_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFF80L
+//CP_DMA_WATCH0_ADDR_HI
+#define CP_DMA_WATCH0_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_DMA_WATCH0_ADDR_HI__RSVD__SHIFT                                                                    0x10
+#define CP_DMA_WATCH0_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
+#define CP_DMA_WATCH0_ADDR_HI__RSVD_MASK                                                                      0xFFFF0000L
+//CP_DMA_WATCH0_MASK
+#define CP_DMA_WATCH0_MASK__RSVD__SHIFT                                                                       0x0
+#define CP_DMA_WATCH0_MASK__MASK__SHIFT                                                                       0x7
+#define CP_DMA_WATCH0_MASK__RSVD_MASK                                                                         0x0000007FL
+#define CP_DMA_WATCH0_MASK__MASK_MASK                                                                         0xFFFFFF80L
+//CP_DMA_WATCH0_CNTL
+#define CP_DMA_WATCH0_CNTL__VMID__SHIFT                                                                       0x0
+#define CP_DMA_WATCH0_CNTL__RSVD1__SHIFT                                                                      0x4
+#define CP_DMA_WATCH0_CNTL__WATCH_READS__SHIFT                                                                0x8
+#define CP_DMA_WATCH0_CNTL__WATCH_WRITES__SHIFT                                                               0x9
+#define CP_DMA_WATCH0_CNTL__ANY_VMID__SHIFT                                                                   0xa
+#define CP_DMA_WATCH0_CNTL__RSVD2__SHIFT                                                                      0xb
+#define CP_DMA_WATCH0_CNTL__VMID_MASK                                                                         0x0000000FL
+#define CP_DMA_WATCH0_CNTL__RSVD1_MASK                                                                        0x000000F0L
+#define CP_DMA_WATCH0_CNTL__WATCH_READS_MASK                                                                  0x00000100L
+#define CP_DMA_WATCH0_CNTL__WATCH_WRITES_MASK                                                                 0x00000200L
+#define CP_DMA_WATCH0_CNTL__ANY_VMID_MASK                                                                     0x00000400L
+#define CP_DMA_WATCH0_CNTL__RSVD2_MASK                                                                        0xFFFFF800L
+//CP_DMA_WATCH1_ADDR_LO
+#define CP_DMA_WATCH1_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_DMA_WATCH1_ADDR_LO__ADDR_LO__SHIFT                                                                 0x7
+#define CP_DMA_WATCH1_ADDR_LO__RSVD_MASK                                                                      0x0000007FL
+#define CP_DMA_WATCH1_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFF80L
+//CP_DMA_WATCH1_ADDR_HI
+#define CP_DMA_WATCH1_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_DMA_WATCH1_ADDR_HI__RSVD__SHIFT                                                                    0x10
+#define CP_DMA_WATCH1_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
+#define CP_DMA_WATCH1_ADDR_HI__RSVD_MASK                                                                      0xFFFF0000L
+//CP_DMA_WATCH1_MASK
+#define CP_DMA_WATCH1_MASK__RSVD__SHIFT                                                                       0x0
+#define CP_DMA_WATCH1_MASK__MASK__SHIFT                                                                       0x7
+#define CP_DMA_WATCH1_MASK__RSVD_MASK                                                                         0x0000007FL
+#define CP_DMA_WATCH1_MASK__MASK_MASK                                                                         0xFFFFFF80L
+//CP_DMA_WATCH1_CNTL
+#define CP_DMA_WATCH1_CNTL__VMID__SHIFT                                                                       0x0
+#define CP_DMA_WATCH1_CNTL__RSVD1__SHIFT                                                                      0x4
+#define CP_DMA_WATCH1_CNTL__WATCH_READS__SHIFT                                                                0x8
+#define CP_DMA_WATCH1_CNTL__WATCH_WRITES__SHIFT                                                               0x9
+#define CP_DMA_WATCH1_CNTL__ANY_VMID__SHIFT                                                                   0xa
+#define CP_DMA_WATCH1_CNTL__RSVD2__SHIFT                                                                      0xb
+#define CP_DMA_WATCH1_CNTL__VMID_MASK                                                                         0x0000000FL
+#define CP_DMA_WATCH1_CNTL__RSVD1_MASK                                                                        0x000000F0L
+#define CP_DMA_WATCH1_CNTL__WATCH_READS_MASK                                                                  0x00000100L
+#define CP_DMA_WATCH1_CNTL__WATCH_WRITES_MASK                                                                 0x00000200L
+#define CP_DMA_WATCH1_CNTL__ANY_VMID_MASK                                                                     0x00000400L
+#define CP_DMA_WATCH1_CNTL__RSVD2_MASK                                                                        0xFFFFF800L
+//CP_DMA_WATCH2_ADDR_LO
+#define CP_DMA_WATCH2_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_DMA_WATCH2_ADDR_LO__ADDR_LO__SHIFT                                                                 0x7
+#define CP_DMA_WATCH2_ADDR_LO__RSVD_MASK                                                                      0x0000007FL
+#define CP_DMA_WATCH2_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFF80L
+//CP_DMA_WATCH2_ADDR_HI
+#define CP_DMA_WATCH2_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_DMA_WATCH2_ADDR_HI__RSVD__SHIFT                                                                    0x10
+#define CP_DMA_WATCH2_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
+#define CP_DMA_WATCH2_ADDR_HI__RSVD_MASK                                                                      0xFFFF0000L
+//CP_DMA_WATCH2_MASK
+#define CP_DMA_WATCH2_MASK__RSVD__SHIFT                                                                       0x0
+#define CP_DMA_WATCH2_MASK__MASK__SHIFT                                                                       0x7
+#define CP_DMA_WATCH2_MASK__RSVD_MASK                                                                         0x0000007FL
+#define CP_DMA_WATCH2_MASK__MASK_MASK                                                                         0xFFFFFF80L
+//CP_DMA_WATCH2_CNTL
+#define CP_DMA_WATCH2_CNTL__VMID__SHIFT                                                                       0x0
+#define CP_DMA_WATCH2_CNTL__RSVD1__SHIFT                                                                      0x4
+#define CP_DMA_WATCH2_CNTL__WATCH_READS__SHIFT                                                                0x8
+#define CP_DMA_WATCH2_CNTL__WATCH_WRITES__SHIFT                                                               0x9
+#define CP_DMA_WATCH2_CNTL__ANY_VMID__SHIFT                                                                   0xa
+#define CP_DMA_WATCH2_CNTL__RSVD2__SHIFT                                                                      0xb
+#define CP_DMA_WATCH2_CNTL__VMID_MASK                                                                         0x0000000FL
+#define CP_DMA_WATCH2_CNTL__RSVD1_MASK                                                                        0x000000F0L
+#define CP_DMA_WATCH2_CNTL__WATCH_READS_MASK                                                                  0x00000100L
+#define CP_DMA_WATCH2_CNTL__WATCH_WRITES_MASK                                                                 0x00000200L
+#define CP_DMA_WATCH2_CNTL__ANY_VMID_MASK                                                                     0x00000400L
+#define CP_DMA_WATCH2_CNTL__RSVD2_MASK                                                                        0xFFFFF800L
+//CP_DMA_WATCH3_ADDR_LO
+#define CP_DMA_WATCH3_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_DMA_WATCH3_ADDR_LO__ADDR_LO__SHIFT                                                                 0x7
+#define CP_DMA_WATCH3_ADDR_LO__RSVD_MASK                                                                      0x0000007FL
+#define CP_DMA_WATCH3_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFF80L
+//CP_DMA_WATCH3_ADDR_HI
+#define CP_DMA_WATCH3_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_DMA_WATCH3_ADDR_HI__RSVD__SHIFT                                                                    0x10
+#define CP_DMA_WATCH3_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
+#define CP_DMA_WATCH3_ADDR_HI__RSVD_MASK                                                                      0xFFFF0000L
+//CP_DMA_WATCH3_MASK
+#define CP_DMA_WATCH3_MASK__RSVD__SHIFT                                                                       0x0
+#define CP_DMA_WATCH3_MASK__MASK__SHIFT                                                                       0x7
+#define CP_DMA_WATCH3_MASK__RSVD_MASK                                                                         0x0000007FL
+#define CP_DMA_WATCH3_MASK__MASK_MASK                                                                         0xFFFFFF80L
+//CP_DMA_WATCH3_CNTL
+#define CP_DMA_WATCH3_CNTL__VMID__SHIFT                                                                       0x0
+#define CP_DMA_WATCH3_CNTL__RSVD1__SHIFT                                                                      0x4
+#define CP_DMA_WATCH3_CNTL__WATCH_READS__SHIFT                                                                0x8
+#define CP_DMA_WATCH3_CNTL__WATCH_WRITES__SHIFT                                                               0x9
+#define CP_DMA_WATCH3_CNTL__ANY_VMID__SHIFT                                                                   0xa
+#define CP_DMA_WATCH3_CNTL__RSVD2__SHIFT                                                                      0xb
+#define CP_DMA_WATCH3_CNTL__VMID_MASK                                                                         0x0000000FL
+#define CP_DMA_WATCH3_CNTL__RSVD1_MASK                                                                        0x000000F0L
+#define CP_DMA_WATCH3_CNTL__WATCH_READS_MASK                                                                  0x00000100L
+#define CP_DMA_WATCH3_CNTL__WATCH_WRITES_MASK                                                                 0x00000200L
+#define CP_DMA_WATCH3_CNTL__ANY_VMID_MASK                                                                     0x00000400L
+#define CP_DMA_WATCH3_CNTL__RSVD2_MASK                                                                        0xFFFFF800L
+//CP_DMA_WATCH_STAT_ADDR_LO
+#define CP_DMA_WATCH_STAT_ADDR_LO__ADDR_LO__SHIFT                                                             0x2
+#define CP_DMA_WATCH_STAT_ADDR_LO__ADDR_LO_MASK                                                               0xFFFFFFFCL
+//CP_DMA_WATCH_STAT_ADDR_HI
+#define CP_DMA_WATCH_STAT_ADDR_HI__ADDR_HI__SHIFT                                                             0x0
+#define CP_DMA_WATCH_STAT_ADDR_HI__ADDR_HI_MASK                                                               0x0000FFFFL
+//CP_DMA_WATCH_STAT
+#define CP_DMA_WATCH_STAT__VMID__SHIFT                                                                        0x0
+#define CP_DMA_WATCH_STAT__QUEUE_ID__SHIFT                                                                    0x4
+#define CP_DMA_WATCH_STAT__CLIENT_ID__SHIFT                                                                   0x8
+#define CP_DMA_WATCH_STAT__PIPE__SHIFT                                                                        0xc
+#define CP_DMA_WATCH_STAT__WATCH_ID__SHIFT                                                                    0x10
+#define CP_DMA_WATCH_STAT__RD_WR__SHIFT                                                                       0x14
+#define CP_DMA_WATCH_STAT__TRAP_FLAG__SHIFT                                                                   0x1f
+#define CP_DMA_WATCH_STAT__VMID_MASK                                                                          0x0000000FL
+#define CP_DMA_WATCH_STAT__QUEUE_ID_MASK                                                                      0x00000070L
+#define CP_DMA_WATCH_STAT__CLIENT_ID_MASK                                                                     0x00000700L
+#define CP_DMA_WATCH_STAT__PIPE_MASK                                                                          0x00003000L
+#define CP_DMA_WATCH_STAT__WATCH_ID_MASK                                                                      0x00030000L
+#define CP_DMA_WATCH_STAT__RD_WR_MASK                                                                         0x00100000L
+#define CP_DMA_WATCH_STAT__TRAP_FLAG_MASK                                                                     0x80000000L
+//CP_PFP_JT_STAT
+#define CP_PFP_JT_STAT__JT_LOADED__SHIFT                                                                      0x0
+#define CP_PFP_JT_STAT__WR_MASK__SHIFT                                                                        0x10
+#define CP_PFP_JT_STAT__JT_LOADED_MASK                                                                        0x00000003L
+#define CP_PFP_JT_STAT__WR_MASK_MASK                                                                          0x00030000L
+//CP_MEC_JT_STAT
+#define CP_MEC_JT_STAT__JT_LOADED__SHIFT                                                                      0x0
+#define CP_MEC_JT_STAT__WR_MASK__SHIFT                                                                        0x10
+#define CP_MEC_JT_STAT__JT_LOADED_MASK                                                                        0x000000FFL
+#define CP_MEC_JT_STAT__WR_MASK_MASK                                                                          0x00FF0000L
+//CP_CPC_BUSY_HYSTERESIS
+#define CP_CPC_BUSY_HYSTERESIS__CAC_ACTIVE__SHIFT                                                             0x0
+#define CP_CPC_BUSY_HYSTERESIS__CPC_BUSY__SHIFT                                                               0x8
+#define CP_CPC_BUSY_HYSTERESIS__CAC_ACTIVE_MASK                                                               0x000000FFL
+#define CP_CPC_BUSY_HYSTERESIS__CPC_BUSY_MASK                                                                 0x0000FF00L
+//CP_CPF_BUSY_HYSTERESIS1
+#define CP_CPF_BUSY_HYSTERESIS1__CAC_ACTIVE__SHIFT                                                            0x0
+#define CP_CPF_BUSY_HYSTERESIS1__CPF_BUSY__SHIFT                                                              0x8
+#define CP_CPF_BUSY_HYSTERESIS1__CORE_BUSY__SHIFT                                                             0x10
+#define CP_CPF_BUSY_HYSTERESIS1__GFX_BUSY__SHIFT                                                              0x18
+#define CP_CPF_BUSY_HYSTERESIS1__CAC_ACTIVE_MASK                                                              0x000000FFL
+#define CP_CPF_BUSY_HYSTERESIS1__CPF_BUSY_MASK                                                                0x0000FF00L
+#define CP_CPF_BUSY_HYSTERESIS1__CORE_BUSY_MASK                                                               0x00FF0000L
+#define CP_CPF_BUSY_HYSTERESIS1__GFX_BUSY_MASK                                                                0xFF000000L
+//CP_CPF_BUSY_HYSTERESIS2
+#define CP_CPF_BUSY_HYSTERESIS2__CMP_BUSY__SHIFT                                                              0x0
+#define CP_CPF_BUSY_HYSTERESIS2__CMP_BUSY_MASK                                                                0x000000FFL
+//CP_CPG_BUSY_HYSTERESIS1
+#define CP_CPG_BUSY_HYSTERESIS1__CAC_ACTIVE__SHIFT                                                            0x0
+#define CP_CPG_BUSY_HYSTERESIS1__CP_BUSY__SHIFT                                                               0x8
+#define CP_CPG_BUSY_HYSTERESIS1__DMA_BUSY__SHIFT                                                              0x10
+#define CP_CPG_BUSY_HYSTERESIS1__GFX_BUSY__SHIFT                                                              0x18
+#define CP_CPG_BUSY_HYSTERESIS1__CAC_ACTIVE_MASK                                                              0x000000FFL
+#define CP_CPG_BUSY_HYSTERESIS1__CP_BUSY_MASK                                                                 0x0000FF00L
+#define CP_CPG_BUSY_HYSTERESIS1__DMA_BUSY_MASK                                                                0x00FF0000L
+#define CP_CPG_BUSY_HYSTERESIS1__GFX_BUSY_MASK                                                                0xFF000000L
+//CP_CPG_BUSY_HYSTERESIS2
+#define CP_CPG_BUSY_HYSTERESIS2__CMP_BUSY__SHIFT                                                              0x0
+#define CP_CPG_BUSY_HYSTERESIS2__SPI_CLOCK_0__SHIFT                                                           0x8
+#define CP_CPG_BUSY_HYSTERESIS2__SPI_CLOCK_1__SHIFT                                                           0x10
+#define CP_CPG_BUSY_HYSTERESIS2__CMP_BUSY_MASK                                                                0x000000FFL
+#define CP_CPG_BUSY_HYSTERESIS2__SPI_CLOCK_0_MASK                                                             0x0000FF00L
+#define CP_CPG_BUSY_HYSTERESIS2__SPI_CLOCK_1_MASK                                                             0x00FF0000L
+//CP_RB_DOORBELL_CLEAR
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUEUE__SHIFT                                                             0x0
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_EN_CLEAR__SHIFT                                             0x8
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_HIT_CLEAR__SHIFT                                            0x9
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_EN_CLEAR__SHIFT                                                 0xa
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_HIT_CLEAR__SHIFT                                                0xb
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_EN_CLEAR__SHIFT                                                 0xc
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_HIT_CLEAR__SHIFT                                                0xd
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUEUE_MASK                                                               0x00000007L
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_EN_CLEAR_MASK                                               0x00000100L
+#define CP_RB_DOORBELL_CLEAR__MAPPED_QUE_DOORBELL_HIT_CLEAR_MASK                                              0x00000200L
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_EN_CLEAR_MASK                                                   0x00000400L
+#define CP_RB_DOORBELL_CLEAR__MASTER_DOORBELL_HIT_CLEAR_MASK                                                  0x00000800L
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_EN_CLEAR_MASK                                                   0x00001000L
+#define CP_RB_DOORBELL_CLEAR__QUEUES_DOORBELL_HIT_CLEAR_MASK                                                  0x00002000L
+//CP_RB0_ACTIVE
+#define CP_RB0_ACTIVE__ACTIVE__SHIFT                                                                          0x0
+#define CP_RB0_ACTIVE__ACTIVE_MASK                                                                            0x00000001L
+//CP_RB_ACTIVE
+#define CP_RB_ACTIVE__ACTIVE__SHIFT                                                                           0x0
+#define CP_RB_ACTIVE__ACTIVE_MASK                                                                             0x00000001L
+//CP_RB1_ACTIVE
+#define CP_RB1_ACTIVE__ACTIVE__SHIFT                                                                          0x0
+#define CP_RB1_ACTIVE__ACTIVE_MASK                                                                            0x00000001L
+//CP_RB_STATUS
+#define CP_RB_STATUS__DOORBELL_UPDATED__SHIFT                                                                 0x0
+#define CP_RB_STATUS__DOORBELL_ENABLE__SHIFT                                                                  0x1
+#define CP_RB_STATUS__DOORBELL_UPDATED_MASK                                                                   0x00000001L
+#define CP_RB_STATUS__DOORBELL_ENABLE_MASK                                                                    0x00000002L
+//CPG_RCIU_CAM_INDEX
+#define CPG_RCIU_CAM_INDEX__INDEX__SHIFT                                                                      0x0
+#define CPG_RCIU_CAM_INDEX__INDEX_MASK                                                                        0x0000001FL
+//CPG_RCIU_CAM_DATA
+#define CPG_RCIU_CAM_DATA__DATA__SHIFT                                                                        0x0
+#define CPG_RCIU_CAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//CPG_RCIU_CAM_DATA_PHASE0
+#define CPG_RCIU_CAM_DATA_PHASE0__ADDR__SHIFT                                                                 0x0
+#define CPG_RCIU_CAM_DATA_PHASE0__PIPE0_EN__SHIFT                                                             0x18
+#define CPG_RCIU_CAM_DATA_PHASE0__PIPE1_EN__SHIFT                                                             0x19
+#define CPG_RCIU_CAM_DATA_PHASE0__SKIP_WR__SHIFT                                                              0x1f
+#define CPG_RCIU_CAM_DATA_PHASE0__ADDR_MASK                                                                   0x0003FFFFL
+#define CPG_RCIU_CAM_DATA_PHASE0__PIPE0_EN_MASK                                                               0x01000000L
+#define CPG_RCIU_CAM_DATA_PHASE0__PIPE1_EN_MASK                                                               0x02000000L
+#define CPG_RCIU_CAM_DATA_PHASE0__SKIP_WR_MASK                                                                0x80000000L
+//CPG_RCIU_CAM_DATA_PHASE1
+#define CPG_RCIU_CAM_DATA_PHASE1__MASK__SHIFT                                                                 0x0
+#define CPG_RCIU_CAM_DATA_PHASE1__MASK_MASK                                                                   0xFFFFFFFFL
+//CPG_RCIU_CAM_DATA_PHASE2
+#define CPG_RCIU_CAM_DATA_PHASE2__VALUE__SHIFT                                                                0x0
+#define CPG_RCIU_CAM_DATA_PHASE2__VALUE_MASK                                                                  0xFFFFFFFFL
+//CP_GPU_TIMESTAMP_OFFSET_LO
+#define CP_GPU_TIMESTAMP_OFFSET_LO__OFFSET_LO__SHIFT                                                          0x0
+#define CP_GPU_TIMESTAMP_OFFSET_LO__OFFSET_LO_MASK                                                            0xFFFFFFFFL
+//CP_GPU_TIMESTAMP_OFFSET_HI
+#define CP_GPU_TIMESTAMP_OFFSET_HI__OFFSET_HI__SHIFT                                                          0x0
+#define CP_GPU_TIMESTAMP_OFFSET_HI__OFFSET_HI_MASK                                                            0xFFFFFFFFL
+//CP_SDMA_DMA_DONE
+#define CP_SDMA_DMA_DONE__SDMA_ID__SHIFT                                                                      0x0
+#define CP_SDMA_DMA_DONE__SDMA_ID_MASK                                                                        0x0000000FL
+//CP_PFP_SDMA_CS
+#define CP_PFP_SDMA_CS__REQUEST_GRANT__SHIFT                                                                  0x0
+#define CP_PFP_SDMA_CS__SDMA_ID__SHIFT                                                                        0x4
+#define CP_PFP_SDMA_CS__REQUEST_POSITION__SHIFT                                                               0x8
+#define CP_PFP_SDMA_CS__SDMA_COUNT__SHIFT                                                                     0xc
+#define CP_PFP_SDMA_CS__REQUEST_GRANT_MASK                                                                    0x00000001L
+#define CP_PFP_SDMA_CS__SDMA_ID_MASK                                                                          0x000000F0L
+#define CP_PFP_SDMA_CS__REQUEST_POSITION_MASK                                                                 0x00000F00L
+#define CP_PFP_SDMA_CS__SDMA_COUNT_MASK                                                                       0x00003000L
+//CP_ME_SDMA_CS
+#define CP_ME_SDMA_CS__REQUEST_GRANT__SHIFT                                                                   0x0
+#define CP_ME_SDMA_CS__SDMA_ID__SHIFT                                                                         0x4
+#define CP_ME_SDMA_CS__REQUEST_POSITION__SHIFT                                                                0x8
+#define CP_ME_SDMA_CS__SDMA_COUNT__SHIFT                                                                      0xc
+#define CP_ME_SDMA_CS__REQUEST_GRANT_MASK                                                                     0x00000001L
+#define CP_ME_SDMA_CS__SDMA_ID_MASK                                                                           0x000000F0L
+#define CP_ME_SDMA_CS__REQUEST_POSITION_MASK                                                                  0x00000F00L
+#define CP_ME_SDMA_CS__SDMA_COUNT_MASK                                                                        0x00003000L
+//CPF_GCR_CNTL
+#define CPF_GCR_CNTL__GCR_GL_CMD__SHIFT                                                                       0x0
+#define CPF_GCR_CNTL__GCR_GL_CMD_MASK                                                                         0x0007FFFFL
+//CPG_UTCL1_STATUS
+#define CPG_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define CPG_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define CPG_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define CPG_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define CPG_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define CPG_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define CPG_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define CPG_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define CPG_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define CPG_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define CPG_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define CPG_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+//CPC_UTCL1_STATUS
+#define CPC_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define CPC_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define CPC_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define CPC_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define CPC_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define CPC_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define CPC_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define CPC_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define CPC_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define CPC_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define CPC_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define CPC_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+//CPF_UTCL1_STATUS
+#define CPF_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define CPF_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define CPF_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define CPF_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define CPF_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define CPF_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define CPF_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define CPF_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define CPF_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define CPF_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define CPF_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define CPF_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+//CP_SD_CNTL
+#define CP_SD_CNTL__CPF_EN__SHIFT                                                                             0x0
+#define CP_SD_CNTL__CPG_EN__SHIFT                                                                             0x1
+#define CP_SD_CNTL__CPC_EN__SHIFT                                                                             0x2
+#define CP_SD_CNTL__RLC_EN__SHIFT                                                                             0x3
+#define CP_SD_CNTL__GE_EN__SHIFT                                                                              0x5
+#define CP_SD_CNTL__UTCL1_EN__SHIFT                                                                           0x6
+#define CP_SD_CNTL__EA_EN__SHIFT                                                                              0x9
+#define CP_SD_CNTL__SDMA_EN__SHIFT                                                                            0xa
+#define CP_SD_CNTL__SD_VMIDVEC_OVERRIDE__SHIFT                                                                0x1f
+#define CP_SD_CNTL__CPF_EN_MASK                                                                               0x00000001L
+#define CP_SD_CNTL__CPG_EN_MASK                                                                               0x00000002L
+#define CP_SD_CNTL__CPC_EN_MASK                                                                               0x00000004L
+#define CP_SD_CNTL__RLC_EN_MASK                                                                               0x00000008L
+#define CP_SD_CNTL__GE_EN_MASK                                                                                0x00000020L
+#define CP_SD_CNTL__UTCL1_EN_MASK                                                                             0x00000040L
+#define CP_SD_CNTL__EA_EN_MASK                                                                                0x00000200L
+#define CP_SD_CNTL__SDMA_EN_MASK                                                                              0x00000400L
+#define CP_SD_CNTL__SD_VMIDVEC_OVERRIDE_MASK                                                                  0x80000000L
+//CP_SOFT_RESET_CNTL
+#define CP_SOFT_RESET_CNTL__CMP_ONLY_SOFT_RESET__SHIFT                                                        0x0
+#define CP_SOFT_RESET_CNTL__GFX_ONLY_SOFT_RESET__SHIFT                                                        0x1
+#define CP_SOFT_RESET_CNTL__CMP_HQD_REG_RESET__SHIFT                                                          0x2
+#define CP_SOFT_RESET_CNTL__CMP_INTR_REG_RESET__SHIFT                                                         0x3
+#define CP_SOFT_RESET_CNTL__CMP_HQD_QUEUE_DOORBELL_RESET__SHIFT                                               0x4
+#define CP_SOFT_RESET_CNTL__GFX_RB_DOORBELL_RESET__SHIFT                                                      0x5
+#define CP_SOFT_RESET_CNTL__GFX_INTR_REG_RESET__SHIFT                                                         0x6
+#define CP_SOFT_RESET_CNTL__GFX_HQD_REG_RESET__SHIFT                                                          0x7
+#define CP_SOFT_RESET_CNTL__CMP_ONLY_SOFT_RESET_MASK                                                          0x00000001L
+#define CP_SOFT_RESET_CNTL__GFX_ONLY_SOFT_RESET_MASK                                                          0x00000002L
+#define CP_SOFT_RESET_CNTL__CMP_HQD_REG_RESET_MASK                                                            0x00000004L
+#define CP_SOFT_RESET_CNTL__CMP_INTR_REG_RESET_MASK                                                           0x00000008L
+#define CP_SOFT_RESET_CNTL__CMP_HQD_QUEUE_DOORBELL_RESET_MASK                                                 0x00000010L
+#define CP_SOFT_RESET_CNTL__GFX_RB_DOORBELL_RESET_MASK                                                        0x00000020L
+#define CP_SOFT_RESET_CNTL__GFX_INTR_REG_RESET_MASK                                                           0x00000040L
+#define CP_SOFT_RESET_CNTL__GFX_HQD_REG_RESET_MASK                                                            0x00000080L
+//CP_CPC_GFX_CNTL
+#define CP_CPC_GFX_CNTL__QUEUEID__SHIFT                                                                       0x0
+#define CP_CPC_GFX_CNTL__PIPEID__SHIFT                                                                        0x3
+#define CP_CPC_GFX_CNTL__MEID__SHIFT                                                                          0x5
+#define CP_CPC_GFX_CNTL__VALID__SHIFT                                                                         0x7
+#define CP_CPC_GFX_CNTL__QUEUEID_MASK                                                                         0x00000007L
+#define CP_CPC_GFX_CNTL__PIPEID_MASK                                                                          0x00000018L
+#define CP_CPC_GFX_CNTL__MEID_MASK                                                                            0x00000060L
+#define CP_CPC_GFX_CNTL__VALID_MASK                                                                           0x00000080L
+
+
+// addressBlock: gc_spipdec
+//SPI_ARB_PRIORITY
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0__SHIFT                                                               0x0
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1__SHIFT                                                               0x3
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2__SHIFT                                                               0x6
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3__SHIFT                                                               0x9
+#define SPI_ARB_PRIORITY__TS0_DUR_MULT__SHIFT                                                                 0xc
+#define SPI_ARB_PRIORITY__TS1_DUR_MULT__SHIFT                                                                 0xe
+#define SPI_ARB_PRIORITY__TS2_DUR_MULT__SHIFT                                                                 0x10
+#define SPI_ARB_PRIORITY__TS3_DUR_MULT__SHIFT                                                                 0x12
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS0_MASK                                                                 0x00000007L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS1_MASK                                                                 0x00000038L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS2_MASK                                                                 0x000001C0L
+#define SPI_ARB_PRIORITY__PIPE_ORDER_TS3_MASK                                                                 0x00000E00L
+#define SPI_ARB_PRIORITY__TS0_DUR_MULT_MASK                                                                   0x00003000L
+#define SPI_ARB_PRIORITY__TS1_DUR_MULT_MASK                                                                   0x0000C000L
+#define SPI_ARB_PRIORITY__TS2_DUR_MULT_MASK                                                                   0x00030000L
+#define SPI_ARB_PRIORITY__TS3_DUR_MULT_MASK                                                                   0x000C0000L
+//SPI_ARB_CYCLES_0
+#define SPI_ARB_CYCLES_0__TS0_DURATION__SHIFT                                                                 0x0
+#define SPI_ARB_CYCLES_0__TS1_DURATION__SHIFT                                                                 0x10
+#define SPI_ARB_CYCLES_0__TS0_DURATION_MASK                                                                   0x0000FFFFL
+#define SPI_ARB_CYCLES_0__TS1_DURATION_MASK                                                                   0xFFFF0000L
+//SPI_ARB_CYCLES_1
+#define SPI_ARB_CYCLES_1__TS2_DURATION__SHIFT                                                                 0x0
+#define SPI_ARB_CYCLES_1__TS3_DURATION__SHIFT                                                                 0x10
+#define SPI_ARB_CYCLES_1__TS2_DURATION_MASK                                                                   0x0000FFFFL
+#define SPI_ARB_CYCLES_1__TS3_DURATION_MASK                                                                   0xFFFF0000L
+//SPI_WCL_PIPE_PERCENT_GFX
+#define SPI_WCL_PIPE_PERCENT_GFX__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_GFX__HS_GRP_VALUE__SHIFT                                                         0xc
+#define SPI_WCL_PIPE_PERCENT_GFX__GS_GRP_VALUE__SHIFT                                                         0x16
+#define SPI_WCL_PIPE_PERCENT_GFX__VALUE_MASK                                                                  0x0000007FL
+#define SPI_WCL_PIPE_PERCENT_GFX__HS_GRP_VALUE_MASK                                                           0x0001F000L
+#define SPI_WCL_PIPE_PERCENT_GFX__GS_GRP_VALUE_MASK                                                           0x07C00000L
+//SPI_WCL_PIPE_PERCENT_HP3D
+#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE__SHIFT                                                               0x0
+#define SPI_WCL_PIPE_PERCENT_HP3D__HS_GRP_VALUE__SHIFT                                                        0xc
+#define SPI_WCL_PIPE_PERCENT_HP3D__GS_GRP_VALUE__SHIFT                                                        0x16
+#define SPI_WCL_PIPE_PERCENT_HP3D__VALUE_MASK                                                                 0x0000007FL
+#define SPI_WCL_PIPE_PERCENT_HP3D__HS_GRP_VALUE_MASK                                                          0x0001F000L
+#define SPI_WCL_PIPE_PERCENT_HP3D__GS_GRP_VALUE_MASK                                                          0x07C00000L
+//SPI_WCL_PIPE_PERCENT_CS0
+#define SPI_WCL_PIPE_PERCENT_CS0__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS0__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS1
+#define SPI_WCL_PIPE_PERCENT_CS1__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS1__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS2
+#define SPI_WCL_PIPE_PERCENT_CS2__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS2__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS3
+#define SPI_WCL_PIPE_PERCENT_CS3__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS3__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS4
+#define SPI_WCL_PIPE_PERCENT_CS4__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS4__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS5
+#define SPI_WCL_PIPE_PERCENT_CS5__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS5__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS6
+#define SPI_WCL_PIPE_PERCENT_CS6__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS6__VALUE_MASK                                                                  0x7FL
+//SPI_WCL_PIPE_PERCENT_CS7
+#define SPI_WCL_PIPE_PERCENT_CS7__VALUE__SHIFT                                                                0x0
+#define SPI_WCL_PIPE_PERCENT_CS7__VALUE_MASK                                                                  0x7FL
+//SPI_USER_ACCUM_VMID_CNTL
+#define SPI_USER_ACCUM_VMID_CNTL__EN_USER_ACCUM__SHIFT                                                        0x0
+#define SPI_USER_ACCUM_VMID_CNTL__EN_USER_ACCUM_MASK                                                          0x0000000FL
+//SPI_GDBG_PER_VMID_CNTL
+#define SPI_GDBG_PER_VMID_CNTL__STALL_VMID__SHIFT                                                             0x0
+#define SPI_GDBG_PER_VMID_CNTL__LAUNCH_MODE__SHIFT                                                            0x1
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_EN__SHIFT                                                                0x3
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_EN__SHIFT                                                                0x4
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_REPLACE__SHIFT                                                           0xd
+#define SPI_GDBG_PER_VMID_CNTL__STALL_VMID_MASK                                                               0x00000001L
+#define SPI_GDBG_PER_VMID_CNTL__LAUNCH_MODE_MASK                                                              0x00000006L
+#define SPI_GDBG_PER_VMID_CNTL__TRAP_EN_MASK                                                                  0x00000008L
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_EN_MASK                                                                  0x00001FF0L
+#define SPI_GDBG_PER_VMID_CNTL__EXCP_REPLACE_MASK                                                             0x00002000L
+//SPI_COMPUTE_QUEUE_RESET
+#define SPI_COMPUTE_QUEUE_RESET__RESET__SHIFT                                                                 0x0
+#define SPI_COMPUTE_QUEUE_RESET__RESET_MASK                                                                   0x01L
+//SPI_COMPUTE_WF_CTX_SAVE
+#define SPI_COMPUTE_WF_CTX_SAVE__INITIATE__SHIFT                                                              0x0
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_INTERRUPT_EN__SHIFT                                                      0x1
+#define SPI_COMPUTE_WF_CTX_SAVE__DONE_INTERRUPT_EN__SHIFT                                                     0x2
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_REQ_BUSY__SHIFT                                                          0x1e
+#define SPI_COMPUTE_WF_CTX_SAVE__SAVE_BUSY__SHIFT                                                             0x1f
+#define SPI_COMPUTE_WF_CTX_SAVE__INITIATE_MASK                                                                0x00000001L
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_INTERRUPT_EN_MASK                                                        0x00000002L
+#define SPI_COMPUTE_WF_CTX_SAVE__DONE_INTERRUPT_EN_MASK                                                       0x00000004L
+#define SPI_COMPUTE_WF_CTX_SAVE__GDS_REQ_BUSY_MASK                                                            0x40000000L
+#define SPI_COMPUTE_WF_CTX_SAVE__SAVE_BUSY_MASK                                                               0x80000000L
+
+
+// addressBlock: gc_cpphqddec
+//CP_HPD_UTCL1_CNTL
+#define CP_HPD_UTCL1_CNTL__SELECT__SHIFT                                                                      0x0
+#define CP_HPD_UTCL1_CNTL__DISABLE_ERROR_REPORT__SHIFT                                                        0xa
+#define CP_HPD_UTCL1_CNTL__SELECT_MASK                                                                        0x0000000FL
+#define CP_HPD_UTCL1_CNTL__DISABLE_ERROR_REPORT_MASK                                                          0x00000400L
+//CP_HPD_UTCL1_ERROR
+#define CP_HPD_UTCL1_ERROR__ADDR_HI__SHIFT                                                                    0x0
+#define CP_HPD_UTCL1_ERROR__TYPE__SHIFT                                                                       0x10
+#define CP_HPD_UTCL1_ERROR__VMID__SHIFT                                                                       0x14
+#define CP_HPD_UTCL1_ERROR__ADDR_HI_MASK                                                                      0x0000FFFFL
+#define CP_HPD_UTCL1_ERROR__TYPE_MASK                                                                         0x00010000L
+#define CP_HPD_UTCL1_ERROR__VMID_MASK                                                                         0x00F00000L
+//CP_HPD_UTCL1_ERROR_ADDR
+#define CP_HPD_UTCL1_ERROR_ADDR__ADDR__SHIFT                                                                  0xc
+#define CP_HPD_UTCL1_ERROR_ADDR__ADDR_MASK                                                                    0xFFFFF000L
+//CP_MQD_BASE_ADDR
+#define CP_MQD_BASE_ADDR__BASE_ADDR__SHIFT                                                                    0x2
+#define CP_MQD_BASE_ADDR__BASE_ADDR_MASK                                                                      0xFFFFFFFCL
+//CP_MQD_BASE_ADDR_HI
+#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                              0x0
+#define CP_MQD_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                                0x0000FFFFL
+//CP_HQD_ACTIVE
+#define CP_HQD_ACTIVE__ACTIVE__SHIFT                                                                          0x0
+#define CP_HQD_ACTIVE__BUSY_GATE__SHIFT                                                                       0x1
+#define CP_HQD_ACTIVE__ACTIVE_MASK                                                                            0x00000001L
+#define CP_HQD_ACTIVE__BUSY_GATE_MASK                                                                         0x00000002L
+//CP_HQD_VMID
+#define CP_HQD_VMID__VMID__SHIFT                                                                              0x0
+#define CP_HQD_VMID__IB_VMID__SHIFT                                                                           0x8
+#define CP_HQD_VMID__VQID__SHIFT                                                                              0x10
+#define CP_HQD_VMID__VMID_MASK                                                                                0x0000000FL
+#define CP_HQD_VMID__IB_VMID_MASK                                                                             0x00000F00L
+#define CP_HQD_VMID__VQID_MASK                                                                                0x03FF0000L
+//CP_HQD_PERSISTENT_STATE
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ__SHIFT                                                           0x0
+#define CP_HQD_PERSISTENT_STATE__TMZ_CONNECT_OVERRIDE__SHIFT                                                  0x1
+#define CP_HQD_PERSISTENT_STATE__SUSPEND_STATUS__SHIFT                                                        0x7
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE__SHIFT                                                          0x8
+#define CP_HQD_PERSISTENT_STATE__TMZ_SWITCH_EXEMPT__SHIFT                                                     0x12
+#define CP_HQD_PERSISTENT_STATE__TMZ_MATCH_DIS__SHIFT                                                         0x13
+#define CP_HQD_PERSISTENT_STATE__WPP_CLAMP_EN__SHIFT                                                          0x14
+#define CP_HQD_PERSISTENT_STATE__WPP_SWITCH_QOS_EN__SHIFT                                                     0x15
+#define CP_HQD_PERSISTENT_STATE__IQ_SWITCH_QOS_EN__SHIFT                                                      0x16
+#define CP_HQD_PERSISTENT_STATE__IB_SWITCH_QOS_EN__SHIFT                                                      0x17
+#define CP_HQD_PERSISTENT_STATE__EOP_SWITCH_QOS_EN__SHIFT                                                     0x18
+#define CP_HQD_PERSISTENT_STATE__PQ_SWITCH_QOS_EN__SHIFT                                                      0x19
+#define CP_HQD_PERSISTENT_STATE__TC_OFFLOAD_QOS_EN__SHIFT                                                     0x1a
+#define CP_HQD_PERSISTENT_STATE__CACHE_FULL_PACKET_EN__SHIFT                                                  0x1b
+#define CP_HQD_PERSISTENT_STATE__RESTORE_ACTIVE__SHIFT                                                        0x1c
+#define CP_HQD_PERSISTENT_STATE__RELAUNCH_WAVES__SHIFT                                                        0x1d
+#define CP_HQD_PERSISTENT_STATE__QSWITCH_MODE__SHIFT                                                          0x1e
+#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE__SHIFT                                                           0x1f
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_REQ_MASK                                                             0x00000001L
+#define CP_HQD_PERSISTENT_STATE__TMZ_CONNECT_OVERRIDE_MASK                                                    0x00000002L
+#define CP_HQD_PERSISTENT_STATE__SUSPEND_STATUS_MASK                                                          0x00000080L
+#define CP_HQD_PERSISTENT_STATE__PRELOAD_SIZE_MASK                                                            0x0003FF00L
+#define CP_HQD_PERSISTENT_STATE__TMZ_SWITCH_EXEMPT_MASK                                                       0x00040000L
+#define CP_HQD_PERSISTENT_STATE__TMZ_MATCH_DIS_MASK                                                           0x00080000L
+#define CP_HQD_PERSISTENT_STATE__WPP_CLAMP_EN_MASK                                                            0x00100000L
+#define CP_HQD_PERSISTENT_STATE__WPP_SWITCH_QOS_EN_MASK                                                       0x00200000L
+#define CP_HQD_PERSISTENT_STATE__IQ_SWITCH_QOS_EN_MASK                                                        0x00400000L
+#define CP_HQD_PERSISTENT_STATE__IB_SWITCH_QOS_EN_MASK                                                        0x00800000L
+#define CP_HQD_PERSISTENT_STATE__EOP_SWITCH_QOS_EN_MASK                                                       0x01000000L
+#define CP_HQD_PERSISTENT_STATE__PQ_SWITCH_QOS_EN_MASK                                                        0x02000000L
+#define CP_HQD_PERSISTENT_STATE__TC_OFFLOAD_QOS_EN_MASK                                                       0x04000000L
+#define CP_HQD_PERSISTENT_STATE__CACHE_FULL_PACKET_EN_MASK                                                    0x08000000L
+#define CP_HQD_PERSISTENT_STATE__RESTORE_ACTIVE_MASK                                                          0x10000000L
+#define CP_HQD_PERSISTENT_STATE__RELAUNCH_WAVES_MASK                                                          0x20000000L
+#define CP_HQD_PERSISTENT_STATE__QSWITCH_MODE_MASK                                                            0x40000000L
+#define CP_HQD_PERSISTENT_STATE__DISP_ACTIVE_MASK                                                             0x80000000L
+//CP_HQD_PIPE_PRIORITY
+#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY__SHIFT                                                            0x0
+#define CP_HQD_PIPE_PRIORITY__PIPE_PRIORITY_MASK                                                              0x00000003L
+//CP_HQD_QUEUE_PRIORITY
+#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL__SHIFT                                                          0x0
+#define CP_HQD_QUEUE_PRIORITY__PRIORITY_LEVEL_MASK                                                            0x0000000FL
+//CP_HQD_QUANTUM
+#define CP_HQD_QUANTUM__QUANTUM_EN__SHIFT                                                                     0x0
+#define CP_HQD_QUANTUM__QUANTUM_SCALE__SHIFT                                                                  0x4
+#define CP_HQD_QUANTUM__QUANTUM_DURATION__SHIFT                                                               0x8
+#define CP_HQD_QUANTUM__QUANTUM_ACTIVE__SHIFT                                                                 0x1f
+#define CP_HQD_QUANTUM__QUANTUM_EN_MASK                                                                       0x00000001L
+#define CP_HQD_QUANTUM__QUANTUM_SCALE_MASK                                                                    0x00000010L
+#define CP_HQD_QUANTUM__QUANTUM_DURATION_MASK                                                                 0x00003F00L
+#define CP_HQD_QUANTUM__QUANTUM_ACTIVE_MASK                                                                   0x80000000L
+//CP_HQD_PQ_BASE
+#define CP_HQD_PQ_BASE__ADDR__SHIFT                                                                           0x0
+#define CP_HQD_PQ_BASE__ADDR_MASK                                                                             0xFFFFFFFFL
+//CP_HQD_PQ_BASE_HI
+#define CP_HQD_PQ_BASE_HI__ADDR_HI__SHIFT                                                                     0x0
+#define CP_HQD_PQ_BASE_HI__ADDR_HI_MASK                                                                       0x000000FFL
+//CP_HQD_PQ_RPTR
+#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET__SHIFT                                                                0x0
+#define CP_HQD_PQ_RPTR__CONSUMED_OFFSET_MASK                                                                  0xFFFFFFFFL
+//CP_HQD_PQ_RPTR_REPORT_ADDR
+#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR__SHIFT                                                   0x2
+#define CP_HQD_PQ_RPTR_REPORT_ADDR__RPTR_REPORT_ADDR_MASK                                                     0xFFFFFFFCL
+//CP_HQD_PQ_RPTR_REPORT_ADDR_HI
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI__SHIFT                                             0x0
+#define CP_HQD_PQ_RPTR_REPORT_ADDR_HI__RPTR_REPORT_ADDR_HI_MASK                                               0x0000FFFFL
+//CP_HQD_PQ_WPTR_POLL_ADDR
+#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR__SHIFT                                                            0x3
+#define CP_HQD_PQ_WPTR_POLL_ADDR__WPTR_ADDR_MASK                                                              0xFFFFFFF8L
+//CP_HQD_PQ_WPTR_POLL_ADDR_HI
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI__SHIFT                                                      0x0
+#define CP_HQD_PQ_WPTR_POLL_ADDR_HI__WPTR_ADDR_HI_MASK                                                        0x0000FFFFL
+//CP_HQD_PQ_DOORBELL_CONTROL
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE__SHIFT                                                      0x0
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP__SHIFT                                                  0x1
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET__SHIFT                                                    0x2
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE__SHIFT                                                    0x1c
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT__SHIFT                                                  0x1d
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN__SHIFT                                                        0x1e
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT__SHIFT                                                       0x1f
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE_MASK                                                        0x00000001L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_BIF_DROP_MASK                                                    0x00000002L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_OFFSET_MASK                                                      0x0FFFFFFCL
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SOURCE_MASK                                                      0x10000000L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_SCHD_HIT_MASK                                                    0x20000000L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_EN_MASK                                                          0x40000000L
+#define CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_HIT_MASK                                                         0x80000000L
+//CP_HQD_PQ_CONTROL
+#define CP_HQD_PQ_CONTROL__QUEUE_SIZE__SHIFT                                                                  0x0
+#define CP_HQD_PQ_CONTROL__WPTR_CARRY__SHIFT                                                                  0x6
+#define CP_HQD_PQ_CONTROL__RPTR_CARRY__SHIFT                                                                  0x7
+#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE__SHIFT                                                             0x8
+#define CP_HQD_PQ_CONTROL__QUEUE_FULL_EN__SHIFT                                                               0xe
+#define CP_HQD_PQ_CONTROL__PQ_EMPTY__SHIFT                                                                    0xf
+#define CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR__SHIFT                                                             0x12
+#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE__SHIFT                                                              0x14
+#define CP_HQD_PQ_CONTROL__TMZ__SHIFT                                                                         0x16
+#define CP_HQD_PQ_CONTROL__EXE_DISABLE__SHIFT                                                                 0x17
+#define CP_HQD_PQ_CONTROL__CACHE_POLICY__SHIFT                                                                0x18
+#define CP_HQD_PQ_CONTROL__PQ_VOLATILE__SHIFT                                                                 0x1a
+#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR__SHIFT                                                              0x1b
+#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH__SHIFT                                                              0x1c
+#define CP_HQD_PQ_CONTROL__TUNNEL_DISPATCH__SHIFT                                                             0x1d
+#define CP_HQD_PQ_CONTROL__PRIV_STATE__SHIFT                                                                  0x1e
+#define CP_HQD_PQ_CONTROL__KMD_QUEUE__SHIFT                                                                   0x1f
+#define CP_HQD_PQ_CONTROL__QUEUE_SIZE_MASK                                                                    0x0000003FL
+#define CP_HQD_PQ_CONTROL__WPTR_CARRY_MASK                                                                    0x00000040L
+#define CP_HQD_PQ_CONTROL__RPTR_CARRY_MASK                                                                    0x00000080L
+#define CP_HQD_PQ_CONTROL__RPTR_BLOCK_SIZE_MASK                                                               0x00003F00L
+#define CP_HQD_PQ_CONTROL__QUEUE_FULL_EN_MASK                                                                 0x00004000L
+#define CP_HQD_PQ_CONTROL__PQ_EMPTY_MASK                                                                      0x00008000L
+#define CP_HQD_PQ_CONTROL__SLOT_BASED_WPTR_MASK                                                               0x000C0000L
+#define CP_HQD_PQ_CONTROL__MIN_AVAIL_SIZE_MASK                                                                0x00300000L
+#define CP_HQD_PQ_CONTROL__TMZ_MASK                                                                           0x00400000L
+#define CP_HQD_PQ_CONTROL__EXE_DISABLE_MASK                                                                   0x00800000L
+#define CP_HQD_PQ_CONTROL__CACHE_POLICY_MASK                                                                  0x03000000L
+#define CP_HQD_PQ_CONTROL__PQ_VOLATILE_MASK                                                                   0x04000000L
+#define CP_HQD_PQ_CONTROL__NO_UPDATE_RPTR_MASK                                                                0x08000000L
+#define CP_HQD_PQ_CONTROL__UNORD_DISPATCH_MASK                                                                0x10000000L
+#define CP_HQD_PQ_CONTROL__TUNNEL_DISPATCH_MASK                                                               0x20000000L
+#define CP_HQD_PQ_CONTROL__PRIV_STATE_MASK                                                                    0x40000000L
+#define CP_HQD_PQ_CONTROL__KMD_QUEUE_MASK                                                                     0x80000000L
+//CP_HQD_IB_BASE_ADDR
+#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR__SHIFT                                                              0x2
+#define CP_HQD_IB_BASE_ADDR__IB_BASE_ADDR_MASK                                                                0xFFFFFFFCL
+//CP_HQD_IB_BASE_ADDR_HI
+#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI__SHIFT                                                        0x0
+#define CP_HQD_IB_BASE_ADDR_HI__IB_BASE_ADDR_HI_MASK                                                          0x0000FFFFL
+//CP_HQD_IB_RPTR
+#define CP_HQD_IB_RPTR__CONSUMED_OFFSET__SHIFT                                                                0x0
+#define CP_HQD_IB_RPTR__CONSUMED_OFFSET_MASK                                                                  0x000FFFFFL
+//CP_HQD_IB_CONTROL
+#define CP_HQD_IB_CONTROL__IB_SIZE__SHIFT                                                                     0x0
+#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE__SHIFT                                                           0x14
+#define CP_HQD_IB_CONTROL__IB_EXE_DISABLE__SHIFT                                                              0x17
+#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY__SHIFT                                                             0x18
+#define CP_HQD_IB_CONTROL__IB_VOLATILE__SHIFT                                                                 0x1a
+#define CP_HQD_IB_CONTROL__IB_PRIV_STATE__SHIFT                                                               0x1e
+#define CP_HQD_IB_CONTROL__PROCESSING_IB__SHIFT                                                               0x1f
+#define CP_HQD_IB_CONTROL__IB_SIZE_MASK                                                                       0x000FFFFFL
+#define CP_HQD_IB_CONTROL__MIN_IB_AVAIL_SIZE_MASK                                                             0x00300000L
+#define CP_HQD_IB_CONTROL__IB_EXE_DISABLE_MASK                                                                0x00800000L
+#define CP_HQD_IB_CONTROL__IB_CACHE_POLICY_MASK                                                               0x03000000L
+#define CP_HQD_IB_CONTROL__IB_VOLATILE_MASK                                                                   0x04000000L
+#define CP_HQD_IB_CONTROL__IB_PRIV_STATE_MASK                                                                 0x40000000L
+#define CP_HQD_IB_CONTROL__PROCESSING_IB_MASK                                                                 0x80000000L
+//CP_HQD_IQ_TIMER
+#define CP_HQD_IQ_TIMER__WAIT_TIME__SHIFT                                                                     0x0
+#define CP_HQD_IQ_TIMER__RETRY_TYPE__SHIFT                                                                    0x8
+#define CP_HQD_IQ_TIMER__IMMEDIATE_EXPIRE__SHIFT                                                              0xb
+#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE__SHIFT                                                                0xc
+#define CP_HQD_IQ_TIMER__CLOCK_COUNT__SHIFT                                                                   0xe
+#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE__SHIFT                                                                0x10
+#define CP_HQD_IQ_TIMER__QUANTUM_TIMER__SHIFT                                                                 0x16
+#define CP_HQD_IQ_TIMER__EXE_DISABLE__SHIFT                                                                   0x17
+#define CP_HQD_IQ_TIMER__CACHE_POLICY__SHIFT                                                                  0x18
+#define CP_HQD_IQ_TIMER__IQ_VOLATILE__SHIFT                                                                   0x1a
+#define CP_HQD_IQ_TIMER__QUEUE_TYPE__SHIFT                                                                    0x1b
+#define CP_HQD_IQ_TIMER__REARM_TIMER__SHIFT                                                                   0x1c
+#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN__SHIFT                                                                 0x1d
+#define CP_HQD_IQ_TIMER__PROCESSING_IQ__SHIFT                                                                 0x1e
+#define CP_HQD_IQ_TIMER__ACTIVE__SHIFT                                                                        0x1f
+#define CP_HQD_IQ_TIMER__WAIT_TIME_MASK                                                                       0x000000FFL
+#define CP_HQD_IQ_TIMER__RETRY_TYPE_MASK                                                                      0x00000700L
+#define CP_HQD_IQ_TIMER__IMMEDIATE_EXPIRE_MASK                                                                0x00000800L
+#define CP_HQD_IQ_TIMER__INTERRUPT_TYPE_MASK                                                                  0x00003000L
+#define CP_HQD_IQ_TIMER__CLOCK_COUNT_MASK                                                                     0x0000C000L
+#define CP_HQD_IQ_TIMER__INTERRUPT_SIZE_MASK                                                                  0x003F0000L
+#define CP_HQD_IQ_TIMER__QUANTUM_TIMER_MASK                                                                   0x00400000L
+#define CP_HQD_IQ_TIMER__EXE_DISABLE_MASK                                                                     0x00800000L
+#define CP_HQD_IQ_TIMER__CACHE_POLICY_MASK                                                                    0x03000000L
+#define CP_HQD_IQ_TIMER__IQ_VOLATILE_MASK                                                                     0x04000000L
+#define CP_HQD_IQ_TIMER__QUEUE_TYPE_MASK                                                                      0x08000000L
+#define CP_HQD_IQ_TIMER__REARM_TIMER_MASK                                                                     0x10000000L
+#define CP_HQD_IQ_TIMER__PROCESS_IQ_EN_MASK                                                                   0x20000000L
+#define CP_HQD_IQ_TIMER__PROCESSING_IQ_MASK                                                                   0x40000000L
+#define CP_HQD_IQ_TIMER__ACTIVE_MASK                                                                          0x80000000L
+//CP_HQD_IQ_RPTR
+#define CP_HQD_IQ_RPTR__OFFSET__SHIFT                                                                         0x0
+#define CP_HQD_IQ_RPTR__OFFSET_MASK                                                                           0x0000003FL
+//CP_HQD_DEQUEUE_REQUEST
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ__SHIFT                                                            0x0
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND__SHIFT                                                            0x4
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT__SHIFT                                                            0x8
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN__SHIFT                                                         0x9
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN__SHIFT                                                         0xa
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_MASK                                                              0x0000000FL
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_MASK                                                              0x00000010L
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_INT_MASK                                                              0x00000100L
+#define CP_HQD_DEQUEUE_REQUEST__IQ_REQ_PEND_EN_MASK                                                           0x00000200L
+#define CP_HQD_DEQUEUE_REQUEST__DEQUEUE_REQ_EN_MASK                                                           0x00000400L
+//CP_HQD_DMA_OFFLOAD
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD__SHIFT                                                                0x0
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD_EN__SHIFT                                                             0x1
+#define CP_HQD_DMA_OFFLOAD__AQL_OFFLOAD__SHIFT                                                                0x2
+#define CP_HQD_DMA_OFFLOAD__AQL_OFFLOAD_EN__SHIFT                                                             0x3
+#define CP_HQD_DMA_OFFLOAD__EOP_OFFLOAD__SHIFT                                                                0x4
+#define CP_HQD_DMA_OFFLOAD__EOP_OFFLOAD_EN__SHIFT                                                             0x5
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD_MASK                                                                  0x00000001L
+#define CP_HQD_DMA_OFFLOAD__DMA_OFFLOAD_EN_MASK                                                               0x00000002L
+#define CP_HQD_DMA_OFFLOAD__AQL_OFFLOAD_MASK                                                                  0x00000004L
+#define CP_HQD_DMA_OFFLOAD__AQL_OFFLOAD_EN_MASK                                                               0x00000008L
+#define CP_HQD_DMA_OFFLOAD__EOP_OFFLOAD_MASK                                                                  0x00000010L
+#define CP_HQD_DMA_OFFLOAD__EOP_OFFLOAD_EN_MASK                                                               0x00000020L
+//CP_HQD_OFFLOAD
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD__SHIFT                                                                    0x0
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_EN__SHIFT                                                                 0x1
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD__SHIFT                                                                    0x2
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD_EN__SHIFT                                                                 0x3
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD__SHIFT                                                                    0x4
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_EN__SHIFT                                                                 0x5
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_MASK                                                                      0x00000001L
+#define CP_HQD_OFFLOAD__DMA_OFFLOAD_EN_MASK                                                                   0x00000002L
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD_MASK                                                                      0x00000004L
+#define CP_HQD_OFFLOAD__AQL_OFFLOAD_EN_MASK                                                                   0x00000008L
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_MASK                                                                      0x00000010L
+#define CP_HQD_OFFLOAD__EOP_OFFLOAD_EN_MASK                                                                   0x00000020L
+//CP_HQD_SEMA_CMD
+#define CP_HQD_SEMA_CMD__RETRY__SHIFT                                                                         0x0
+#define CP_HQD_SEMA_CMD__RESULT__SHIFT                                                                        0x1
+#define CP_HQD_SEMA_CMD__POLLING_DIS__SHIFT                                                                   0x8
+#define CP_HQD_SEMA_CMD__MESSAGE_EN__SHIFT                                                                    0x9
+#define CP_HQD_SEMA_CMD__RETRY_MASK                                                                           0x00000001L
+#define CP_HQD_SEMA_CMD__RESULT_MASK                                                                          0x00000006L
+#define CP_HQD_SEMA_CMD__POLLING_DIS_MASK                                                                     0x00000100L
+#define CP_HQD_SEMA_CMD__MESSAGE_EN_MASK                                                                      0x00000200L
+//CP_HQD_MSG_TYPE
+#define CP_HQD_MSG_TYPE__ACTION__SHIFT                                                                        0x0
+#define CP_HQD_MSG_TYPE__SAVE_STATE__SHIFT                                                                    0x4
+#define CP_HQD_MSG_TYPE__ACTION_MASK                                                                          0x00000007L
+#define CP_HQD_MSG_TYPE__SAVE_STATE_MASK                                                                      0x00000070L
+//CP_HQD_ATOMIC0_PREOP_LO
+#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC0_PREOP_LO__ATOMIC0_PREOP_LO_MASK                                                        0xFFFFFFFFL
+//CP_HQD_ATOMIC0_PREOP_HI
+#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC0_PREOP_HI__ATOMIC0_PREOP_HI_MASK                                                        0xFFFFFFFFL
+//CP_HQD_ATOMIC1_PREOP_LO
+#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC1_PREOP_LO__ATOMIC1_PREOP_LO_MASK                                                        0xFFFFFFFFL
+//CP_HQD_ATOMIC1_PREOP_HI
+#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI__SHIFT                                                      0x0
+#define CP_HQD_ATOMIC1_PREOP_HI__ATOMIC1_PREOP_HI_MASK                                                        0xFFFFFFFFL
+//CP_HQD_HQ_SCHEDULER0
+#define CP_HQD_HQ_SCHEDULER0__CWSR__SHIFT                                                                     0x0
+#define CP_HQD_HQ_SCHEDULER0__SAVE_STATUS__SHIFT                                                              0x1
+#define CP_HQD_HQ_SCHEDULER0__RSRV__SHIFT                                                                     0x2
+#define CP_HQD_HQ_SCHEDULER0__STATIC_QUEUE__SHIFT                                                             0x3
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_RUN_ONCE__SHIFT                                                           0x6
+#define CP_HQD_HQ_SCHEDULER0__SCRATCH_RAM_INIT__SHIFT                                                         0x7
+#define CP_HQD_HQ_SCHEDULER0__TCL2_DIRTY__SHIFT                                                               0x8
+#define CP_HQD_HQ_SCHEDULER0__C_INHERIT_VMID__SHIFT                                                           0x9
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_SCHEDULER_TYPE__SHIFT                                                     0xa
+#define CP_HQD_HQ_SCHEDULER0__C_QUEUE_USE_GWS__SHIFT                                                          0xd
+#define CP_HQD_HQ_SCHEDULER0__C_QUEUE_DEBUG_EN__SHIFT                                                         0xe
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_SLOT_CONNECTED__SHIFT                                                     0xf
+#define CP_HQD_HQ_SCHEDULER0__MES_INTERRUPT_ENABLED__SHIFT                                                    0x14
+#define CP_HQD_HQ_SCHEDULER0__MES_INTERRUPT_PIPE__SHIFT                                                       0x15
+#define CP_HQD_HQ_SCHEDULER0__CONCURRENT_PROCESS_COUNT__SHIFT                                                 0x18
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_IDLE__SHIFT                                                               0x1e
+#define CP_HQD_HQ_SCHEDULER0__DB_UPDATED_MSG_EN__SHIFT                                                        0x1f
+#define CP_HQD_HQ_SCHEDULER0__CWSR_MASK                                                                       0x00000001L
+#define CP_HQD_HQ_SCHEDULER0__SAVE_STATUS_MASK                                                                0x00000002L
+#define CP_HQD_HQ_SCHEDULER0__RSRV_MASK                                                                       0x00000004L
+#define CP_HQD_HQ_SCHEDULER0__STATIC_QUEUE_MASK                                                               0x00000038L
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_RUN_ONCE_MASK                                                             0x00000040L
+#define CP_HQD_HQ_SCHEDULER0__SCRATCH_RAM_INIT_MASK                                                           0x00000080L
+#define CP_HQD_HQ_SCHEDULER0__TCL2_DIRTY_MASK                                                                 0x00000100L
+#define CP_HQD_HQ_SCHEDULER0__C_INHERIT_VMID_MASK                                                             0x00000200L
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_SCHEDULER_TYPE_MASK                                                       0x00001C00L
+#define CP_HQD_HQ_SCHEDULER0__C_QUEUE_USE_GWS_MASK                                                            0x00002000L
+#define CP_HQD_HQ_SCHEDULER0__C_QUEUE_DEBUG_EN_MASK                                                           0x00004000L
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_SLOT_CONNECTED_MASK                                                       0x00008000L
+#define CP_HQD_HQ_SCHEDULER0__MES_INTERRUPT_ENABLED_MASK                                                      0x00100000L
+#define CP_HQD_HQ_SCHEDULER0__MES_INTERRUPT_PIPE_MASK                                                         0x00600000L
+#define CP_HQD_HQ_SCHEDULER0__CONCURRENT_PROCESS_COUNT_MASK                                                   0x0F000000L
+#define CP_HQD_HQ_SCHEDULER0__QUEUE_IDLE_MASK                                                                 0x40000000L
+#define CP_HQD_HQ_SCHEDULER0__DB_UPDATED_MSG_EN_MASK                                                          0x80000000L
+//CP_HQD_HQ_STATUS0
+#define CP_HQD_HQ_STATUS0__CWSR__SHIFT                                                                        0x0
+#define CP_HQD_HQ_STATUS0__SAVE_STATUS__SHIFT                                                                 0x1
+#define CP_HQD_HQ_STATUS0__RSRV__SHIFT                                                                        0x2
+#define CP_HQD_HQ_STATUS0__STATIC_QUEUE__SHIFT                                                                0x3
+#define CP_HQD_HQ_STATUS0__QUEUE_RUN_ONCE__SHIFT                                                              0x6
+#define CP_HQD_HQ_STATUS0__SCRATCH_RAM_INIT__SHIFT                                                            0x7
+#define CP_HQD_HQ_STATUS0__TCL2_DIRTY__SHIFT                                                                  0x8
+#define CP_HQD_HQ_STATUS0__C_INHERIT_VMID__SHIFT                                                              0x9
+#define CP_HQD_HQ_STATUS0__QUEUE_SCHEDULER_TYPE__SHIFT                                                        0xa
+#define CP_HQD_HQ_STATUS0__C_QUEUE_USE_GWS__SHIFT                                                             0xd
+#define CP_HQD_HQ_STATUS0__C_QUEUE_DEBUG_EN__SHIFT                                                            0xe
+#define CP_HQD_HQ_STATUS0__QUEUE_SLOT_CONNECTED__SHIFT                                                        0xf
+#define CP_HQD_HQ_STATUS0__MES_INTERRUPT_ENABLED__SHIFT                                                       0x14
+#define CP_HQD_HQ_STATUS0__MES_INTERRUPT_PIPE__SHIFT                                                          0x15
+#define CP_HQD_HQ_STATUS0__CONCURRENT_PROCESS_COUNT__SHIFT                                                    0x18
+#define CP_HQD_HQ_STATUS0__QUEUE_IDLE__SHIFT                                                                  0x1e
+#define CP_HQD_HQ_STATUS0__DB_UPDATED_MSG_EN__SHIFT                                                           0x1f
+#define CP_HQD_HQ_STATUS0__CWSR_MASK                                                                          0x00000001L
+#define CP_HQD_HQ_STATUS0__SAVE_STATUS_MASK                                                                   0x00000002L
+#define CP_HQD_HQ_STATUS0__RSRV_MASK                                                                          0x00000004L
+#define CP_HQD_HQ_STATUS0__STATIC_QUEUE_MASK                                                                  0x00000038L
+#define CP_HQD_HQ_STATUS0__QUEUE_RUN_ONCE_MASK                                                                0x00000040L
+#define CP_HQD_HQ_STATUS0__SCRATCH_RAM_INIT_MASK                                                              0x00000080L
+#define CP_HQD_HQ_STATUS0__TCL2_DIRTY_MASK                                                                    0x00000100L
+#define CP_HQD_HQ_STATUS0__C_INHERIT_VMID_MASK                                                                0x00000200L
+#define CP_HQD_HQ_STATUS0__QUEUE_SCHEDULER_TYPE_MASK                                                          0x00001C00L
+#define CP_HQD_HQ_STATUS0__C_QUEUE_USE_GWS_MASK                                                               0x00002000L
+#define CP_HQD_HQ_STATUS0__C_QUEUE_DEBUG_EN_MASK                                                              0x00004000L
+#define CP_HQD_HQ_STATUS0__QUEUE_SLOT_CONNECTED_MASK                                                          0x00008000L
+#define CP_HQD_HQ_STATUS0__MES_INTERRUPT_ENABLED_MASK                                                         0x00100000L
+#define CP_HQD_HQ_STATUS0__MES_INTERRUPT_PIPE_MASK                                                            0x00600000L
+#define CP_HQD_HQ_STATUS0__CONCURRENT_PROCESS_COUNT_MASK                                                      0x0F000000L
+#define CP_HQD_HQ_STATUS0__QUEUE_IDLE_MASK                                                                    0x40000000L
+#define CP_HQD_HQ_STATUS0__DB_UPDATED_MSG_EN_MASK                                                             0x80000000L
+//CP_HQD_HQ_CONTROL0
+#define CP_HQD_HQ_CONTROL0__CONTROL__SHIFT                                                                    0x0
+#define CP_HQD_HQ_CONTROL0__CONTROL_MASK                                                                      0xFFFFFFFFL
+//CP_HQD_HQ_SCHEDULER1
+#define CP_HQD_HQ_SCHEDULER1__SCHEDULER__SHIFT                                                                0x0
+#define CP_HQD_HQ_SCHEDULER1__SCHEDULER_MASK                                                                  0xFFFFFFFFL
+//CP_MQD_CONTROL
+#define CP_MQD_CONTROL__VMID__SHIFT                                                                           0x0
+#define CP_MQD_CONTROL__PRIV_STATE__SHIFT                                                                     0x8
+#define CP_MQD_CONTROL__PROCESSING_MQD__SHIFT                                                                 0xc
+#define CP_MQD_CONTROL__PROCESSING_MQD_EN__SHIFT                                                              0xd
+#define CP_MQD_CONTROL__EXE_DISABLE__SHIFT                                                                    0x17
+#define CP_MQD_CONTROL__CACHE_POLICY__SHIFT                                                                   0x18
+#define CP_MQD_CONTROL__MQD_VOLATILE__SHIFT                                                                   0x1a
+#define CP_MQD_CONTROL__VMID_MASK                                                                             0x0000000FL
+#define CP_MQD_CONTROL__PRIV_STATE_MASK                                                                       0x00000100L
+#define CP_MQD_CONTROL__PROCESSING_MQD_MASK                                                                   0x00001000L
+#define CP_MQD_CONTROL__PROCESSING_MQD_EN_MASK                                                                0x00002000L
+#define CP_MQD_CONTROL__EXE_DISABLE_MASK                                                                      0x00800000L
+#define CP_MQD_CONTROL__CACHE_POLICY_MASK                                                                     0x03000000L
+#define CP_MQD_CONTROL__MQD_VOLATILE_MASK                                                                     0x04000000L
+//CP_HQD_HQ_STATUS1
+#define CP_HQD_HQ_STATUS1__STATUS__SHIFT                                                                      0x0
+#define CP_HQD_HQ_STATUS1__STATUS_MASK                                                                        0xFFFFFFFFL
+//CP_HQD_HQ_CONTROL1
+#define CP_HQD_HQ_CONTROL1__CONTROL__SHIFT                                                                    0x0
+#define CP_HQD_HQ_CONTROL1__CONTROL_MASK                                                                      0xFFFFFFFFL
+//CP_HQD_EOP_BASE_ADDR
+#define CP_HQD_EOP_BASE_ADDR__BASE_ADDR__SHIFT                                                                0x0
+#define CP_HQD_EOP_BASE_ADDR__BASE_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_HQD_EOP_BASE_ADDR_HI
+#define CP_HQD_EOP_BASE_ADDR_HI__BASE_ADDR_HI__SHIFT                                                          0x0
+#define CP_HQD_EOP_BASE_ADDR_HI__BASE_ADDR_HI_MASK                                                            0x000000FFL
+//CP_HQD_EOP_CONTROL
+#define CP_HQD_EOP_CONTROL__EOP_SIZE__SHIFT                                                                   0x0
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOP__SHIFT                                                             0x8
+#define CP_HQD_EOP_CONTROL__PROCESS_EOP_EN__SHIFT                                                             0xc
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOPIB__SHIFT                                                           0xd
+#define CP_HQD_EOP_CONTROL__PROCESS_EOPIB_EN__SHIFT                                                           0xe
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER__SHIFT                                                               0x15
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER_EN__SHIFT                                                            0x16
+#define CP_HQD_EOP_CONTROL__EXE_DISABLE__SHIFT                                                                0x17
+#define CP_HQD_EOP_CONTROL__CACHE_POLICY__SHIFT                                                               0x18
+#define CP_HQD_EOP_CONTROL__EOP_VOLATILE__SHIFT                                                               0x1a
+#define CP_HQD_EOP_CONTROL__SIG_SEM_RESULT__SHIFT                                                             0x1d
+#define CP_HQD_EOP_CONTROL__PEND_SIG_SEM__SHIFT                                                               0x1f
+#define CP_HQD_EOP_CONTROL__EOP_SIZE_MASK                                                                     0x0000003FL
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOP_MASK                                                               0x00000100L
+#define CP_HQD_EOP_CONTROL__PROCESS_EOP_EN_MASK                                                               0x00001000L
+#define CP_HQD_EOP_CONTROL__PROCESSING_EOPIB_MASK                                                             0x00002000L
+#define CP_HQD_EOP_CONTROL__PROCESS_EOPIB_EN_MASK                                                             0x00004000L
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER_MASK                                                                 0x00200000L
+#define CP_HQD_EOP_CONTROL__HALT_FETCHER_EN_MASK                                                              0x00400000L
+#define CP_HQD_EOP_CONTROL__EXE_DISABLE_MASK                                                                  0x00800000L
+#define CP_HQD_EOP_CONTROL__CACHE_POLICY_MASK                                                                 0x03000000L
+#define CP_HQD_EOP_CONTROL__EOP_VOLATILE_MASK                                                                 0x04000000L
+#define CP_HQD_EOP_CONTROL__SIG_SEM_RESULT_MASK                                                               0x60000000L
+#define CP_HQD_EOP_CONTROL__PEND_SIG_SEM_MASK                                                                 0x80000000L
+//CP_HQD_EOP_RPTR
+#define CP_HQD_EOP_RPTR__RPTR__SHIFT                                                                          0x0
+#define CP_HQD_EOP_RPTR__RESET_FETCHER__SHIFT                                                                 0x1c
+#define CP_HQD_EOP_RPTR__DEQUEUE_PEND__SHIFT                                                                  0x1d
+#define CP_HQD_EOP_RPTR__RPTR_EQ_CSMD_WPTR__SHIFT                                                             0x1e
+#define CP_HQD_EOP_RPTR__INIT_FETCHER__SHIFT                                                                  0x1f
+#define CP_HQD_EOP_RPTR__RPTR_MASK                                                                            0x00001FFFL
+#define CP_HQD_EOP_RPTR__RESET_FETCHER_MASK                                                                   0x10000000L
+#define CP_HQD_EOP_RPTR__DEQUEUE_PEND_MASK                                                                    0x20000000L
+#define CP_HQD_EOP_RPTR__RPTR_EQ_CSMD_WPTR_MASK                                                               0x40000000L
+#define CP_HQD_EOP_RPTR__INIT_FETCHER_MASK                                                                    0x80000000L
+//CP_HQD_EOP_WPTR
+#define CP_HQD_EOP_WPTR__WPTR__SHIFT                                                                          0x0
+#define CP_HQD_EOP_WPTR__EOP_EMPTY__SHIFT                                                                     0xf
+#define CP_HQD_EOP_WPTR__EOP_AVAIL__SHIFT                                                                     0x10
+#define CP_HQD_EOP_WPTR__WPTR_MASK                                                                            0x00001FFFL
+#define CP_HQD_EOP_WPTR__EOP_EMPTY_MASK                                                                       0x00008000L
+#define CP_HQD_EOP_WPTR__EOP_AVAIL_MASK                                                                       0x1FFF0000L
+//CP_HQD_EOP_EVENTS
+#define CP_HQD_EOP_EVENTS__EVENT_COUNT__SHIFT                                                                 0x0
+#define CP_HQD_EOP_EVENTS__CS_PARTIAL_FLUSH_PEND__SHIFT                                                       0x10
+#define CP_HQD_EOP_EVENTS__EVENT_COUNT_MASK                                                                   0x00000FFFL
+#define CP_HQD_EOP_EVENTS__CS_PARTIAL_FLUSH_PEND_MASK                                                         0x00010000L
+//CP_HQD_CTX_SAVE_BASE_ADDR_LO
+#define CP_HQD_CTX_SAVE_BASE_ADDR_LO__ADDR__SHIFT                                                             0xc
+#define CP_HQD_CTX_SAVE_BASE_ADDR_LO__ADDR_MASK                                                               0xFFFFF000L
+//CP_HQD_CTX_SAVE_BASE_ADDR_HI
+#define CP_HQD_CTX_SAVE_BASE_ADDR_HI__ADDR_HI__SHIFT                                                          0x0
+#define CP_HQD_CTX_SAVE_BASE_ADDR_HI__ADDR_HI_MASK                                                            0x0000FFFFL
+//CP_HQD_CTX_SAVE_CONTROL
+#define CP_HQD_CTX_SAVE_CONTROL__POLICY__SHIFT                                                                0x3
+#define CP_HQD_CTX_SAVE_CONTROL__EXE_DISABLE__SHIFT                                                           0x17
+#define CP_HQD_CTX_SAVE_CONTROL__POLICY_MASK                                                                  0x00000018L
+#define CP_HQD_CTX_SAVE_CONTROL__EXE_DISABLE_MASK                                                             0x00800000L
+//CP_HQD_CNTL_STACK_OFFSET
+#define CP_HQD_CNTL_STACK_OFFSET__OFFSET__SHIFT                                                               0x2
+#define CP_HQD_CNTL_STACK_OFFSET__OFFSET_MASK                                                                 0x0000FFFCL
+//CP_HQD_CNTL_STACK_SIZE
+#define CP_HQD_CNTL_STACK_SIZE__SIZE__SHIFT                                                                   0xc
+#define CP_HQD_CNTL_STACK_SIZE__SIZE_MASK                                                                     0x0000F000L
+//CP_HQD_WG_STATE_OFFSET
+#define CP_HQD_WG_STATE_OFFSET__OFFSET__SHIFT                                                                 0x2
+#define CP_HQD_WG_STATE_OFFSET__OFFSET_MASK                                                                   0x03FFFFFCL
+//CP_HQD_CTX_SAVE_SIZE
+#define CP_HQD_CTX_SAVE_SIZE__SIZE__SHIFT                                                                     0xc
+#define CP_HQD_CTX_SAVE_SIZE__SIZE_MASK                                                                       0x03FFF000L
+//CP_HQD_GDS_RESOURCE_STATE
+#define CP_HQD_GDS_RESOURCE_STATE__OA_REQUIRED__SHIFT                                                         0x0
+#define CP_HQD_GDS_RESOURCE_STATE__OA_ACQUIRED__SHIFT                                                         0x1
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_SIZE__SHIFT                                                            0x4
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_PNTR__SHIFT                                                            0xc
+#define CP_HQD_GDS_RESOURCE_STATE__OA_REQUIRED_MASK                                                           0x00000001L
+#define CP_HQD_GDS_RESOURCE_STATE__OA_ACQUIRED_MASK                                                           0x00000002L
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_SIZE_MASK                                                              0x000003F0L
+#define CP_HQD_GDS_RESOURCE_STATE__GWS_PNTR_MASK                                                              0x0003F000L
+//CP_HQD_ERROR
+#define CP_HQD_ERROR__EDC_ERROR_ID__SHIFT                                                                     0x0
+#define CP_HQD_ERROR__SUA_ERROR__SHIFT                                                                        0x4
+#define CP_HQD_ERROR__AQL_ERROR__SHIFT                                                                        0x5
+#define CP_HQD_ERROR__PQ_UTCL1_ERROR__SHIFT                                                                   0x8
+#define CP_HQD_ERROR__IB_UTCL1_ERROR__SHIFT                                                                   0x9
+#define CP_HQD_ERROR__EOP_UTCL1_ERROR__SHIFT                                                                  0xa
+#define CP_HQD_ERROR__IQ_UTCL1_ERROR__SHIFT                                                                   0xb
+#define CP_HQD_ERROR__RRPT_UTCL1_ERROR__SHIFT                                                                 0xc
+#define CP_HQD_ERROR__WPP_UTCL1_ERROR__SHIFT                                                                  0xd
+#define CP_HQD_ERROR__SEM_UTCL1_ERROR__SHIFT                                                                  0xe
+#define CP_HQD_ERROR__DMA_SRC_UTCL1_ERROR__SHIFT                                                              0xf
+#define CP_HQD_ERROR__DMA_DST_UTCL1_ERROR__SHIFT                                                              0x10
+#define CP_HQD_ERROR__SR_UTCL1_ERROR__SHIFT                                                                   0x11
+#define CP_HQD_ERROR__QU_UTCL1_ERROR__SHIFT                                                                   0x12
+#define CP_HQD_ERROR__TC_UTCL1_ERROR__SHIFT                                                                   0x13
+#define CP_HQD_ERROR__EDC_ERROR_ID_MASK                                                                       0x0000000FL
+#define CP_HQD_ERROR__SUA_ERROR_MASK                                                                          0x00000010L
+#define CP_HQD_ERROR__AQL_ERROR_MASK                                                                          0x00000020L
+#define CP_HQD_ERROR__PQ_UTCL1_ERROR_MASK                                                                     0x00000100L
+#define CP_HQD_ERROR__IB_UTCL1_ERROR_MASK                                                                     0x00000200L
+#define CP_HQD_ERROR__EOP_UTCL1_ERROR_MASK                                                                    0x00000400L
+#define CP_HQD_ERROR__IQ_UTCL1_ERROR_MASK                                                                     0x00000800L
+#define CP_HQD_ERROR__RRPT_UTCL1_ERROR_MASK                                                                   0x00001000L
+#define CP_HQD_ERROR__WPP_UTCL1_ERROR_MASK                                                                    0x00002000L
+#define CP_HQD_ERROR__SEM_UTCL1_ERROR_MASK                                                                    0x00004000L
+#define CP_HQD_ERROR__DMA_SRC_UTCL1_ERROR_MASK                                                                0x00008000L
+#define CP_HQD_ERROR__DMA_DST_UTCL1_ERROR_MASK                                                                0x00010000L
+#define CP_HQD_ERROR__SR_UTCL1_ERROR_MASK                                                                     0x00020000L
+#define CP_HQD_ERROR__QU_UTCL1_ERROR_MASK                                                                     0x00040000L
+#define CP_HQD_ERROR__TC_UTCL1_ERROR_MASK                                                                     0x00080000L
+//CP_HQD_EOP_WPTR_MEM
+#define CP_HQD_EOP_WPTR_MEM__WPTR__SHIFT                                                                      0x0
+#define CP_HQD_EOP_WPTR_MEM__WPTR_MASK                                                                        0x00001FFFL
+//CP_HQD_AQL_CONTROL
+#define CP_HQD_AQL_CONTROL__CONTROL0__SHIFT                                                                   0x0
+#define CP_HQD_AQL_CONTROL__CONTROL0_EN__SHIFT                                                                0xf
+#define CP_HQD_AQL_CONTROL__CONTROL1__SHIFT                                                                   0x10
+#define CP_HQD_AQL_CONTROL__CONTROL1_EN__SHIFT                                                                0x1f
+#define CP_HQD_AQL_CONTROL__CONTROL0_MASK                                                                     0x00007FFFL
+#define CP_HQD_AQL_CONTROL__CONTROL0_EN_MASK                                                                  0x00008000L
+#define CP_HQD_AQL_CONTROL__CONTROL1_MASK                                                                     0x7FFF0000L
+#define CP_HQD_AQL_CONTROL__CONTROL1_EN_MASK                                                                  0x80000000L
+//CP_HQD_PQ_WPTR_LO
+#define CP_HQD_PQ_WPTR_LO__OFFSET__SHIFT                                                                      0x0
+#define CP_HQD_PQ_WPTR_LO__OFFSET_MASK                                                                        0xFFFFFFFFL
+//CP_HQD_PQ_WPTR_HI
+#define CP_HQD_PQ_WPTR_HI__DATA__SHIFT                                                                        0x0
+#define CP_HQD_PQ_WPTR_HI__DATA_MASK                                                                          0xFFFFFFFFL
+//CP_HQD_SUSPEND_CNTL_STACK_OFFSET
+#define CP_HQD_SUSPEND_CNTL_STACK_OFFSET__OFFSET__SHIFT                                                       0x2
+#define CP_HQD_SUSPEND_CNTL_STACK_OFFSET__OFFSET_MASK                                                         0x0000FFFCL
+//CP_HQD_SUSPEND_CNTL_STACK_DW_CNT
+#define CP_HQD_SUSPEND_CNTL_STACK_DW_CNT__CNT__SHIFT                                                          0x0
+#define CP_HQD_SUSPEND_CNTL_STACK_DW_CNT__CNT_MASK                                                            0x00003FFFL
+//CP_HQD_SUSPEND_WG_STATE_OFFSET
+#define CP_HQD_SUSPEND_WG_STATE_OFFSET__OFFSET__SHIFT                                                         0x2
+#define CP_HQD_SUSPEND_WG_STATE_OFFSET__OFFSET_MASK                                                           0x03FFFFFCL
+//CP_HQD_DDID_RPTR
+#define CP_HQD_DDID_RPTR__RPTR__SHIFT                                                                         0x0
+#define CP_HQD_DDID_RPTR__RPTR_MASK                                                                           0x000007FFL
+//CP_HQD_DDID_WPTR
+#define CP_HQD_DDID_WPTR__WPTR__SHIFT                                                                         0x0
+#define CP_HQD_DDID_WPTR__WPTR_MASK                                                                           0x000007FFL
+//CP_HQD_DDID_INFLIGHT_COUNT
+#define CP_HQD_DDID_INFLIGHT_COUNT__COUNT__SHIFT                                                              0x0
+#define CP_HQD_DDID_INFLIGHT_COUNT__COUNT_MASK                                                                0x0000FFFFL
+//CP_HQD_DDID_DELTA_RPT_COUNT
+#define CP_HQD_DDID_DELTA_RPT_COUNT__COUNT__SHIFT                                                             0x0
+#define CP_HQD_DDID_DELTA_RPT_COUNT__COUNT_MASK                                                               0x000000FFL
+//CP_HQD_DEQUEUE_STATUS
+#define CP_HQD_DEQUEUE_STATUS__DEQUEUE_STAT__SHIFT                                                            0x0
+#define CP_HQD_DEQUEUE_STATUS__SUSPEND_REQ_PEND__SHIFT                                                        0x4
+#define CP_HQD_DEQUEUE_STATUS__SUSPEND_REQ_PEND_EN__SHIFT                                                     0x9
+#define CP_HQD_DEQUEUE_STATUS__DEQUEUE_STAT_EN__SHIFT                                                         0xa
+#define CP_HQD_DEQUEUE_STATUS__DEQUEUE_STAT_MASK                                                              0x0000000FL
+#define CP_HQD_DEQUEUE_STATUS__SUSPEND_REQ_PEND_MASK                                                          0x00000010L
+#define CP_HQD_DEQUEUE_STATUS__SUSPEND_REQ_PEND_EN_MASK                                                       0x00000200L
+#define CP_HQD_DEQUEUE_STATUS__DEQUEUE_STAT_EN_MASK                                                           0x00000400L
+
+
+// addressBlock: gc_tcpdec
+//TCP_WATCH0_ADDR_H
+#define TCP_WATCH0_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH0_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH0_ADDR_L
+#define TCP_WATCH0_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH0_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH0_CNTL
+#define TCP_WATCH0_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH0_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH0_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH0_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH0_CNTL__MASK_MASK                                                                            0x007FFFFFL
+#define TCP_WATCH0_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH0_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH0_CNTL__VALID_MASK                                                                           0x80000000L
+//TCP_WATCH1_ADDR_H
+#define TCP_WATCH1_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH1_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH1_ADDR_L
+#define TCP_WATCH1_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH1_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH1_CNTL
+#define TCP_WATCH1_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH1_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH1_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH1_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH1_CNTL__MASK_MASK                                                                            0x007FFFFFL
+#define TCP_WATCH1_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH1_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH1_CNTL__VALID_MASK                                                                           0x80000000L
+//TCP_WATCH2_ADDR_H
+#define TCP_WATCH2_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH2_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH2_ADDR_L
+#define TCP_WATCH2_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH2_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH2_CNTL
+#define TCP_WATCH2_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH2_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH2_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH2_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH2_CNTL__MASK_MASK                                                                            0x007FFFFFL
+#define TCP_WATCH2_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH2_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH2_CNTL__VALID_MASK                                                                           0x80000000L
+//TCP_WATCH3_ADDR_H
+#define TCP_WATCH3_ADDR_H__ADDR__SHIFT                                                                        0x0
+#define TCP_WATCH3_ADDR_H__ADDR_MASK                                                                          0x0000FFFFL
+//TCP_WATCH3_ADDR_L
+#define TCP_WATCH3_ADDR_L__ADDR__SHIFT                                                                        0x7
+#define TCP_WATCH3_ADDR_L__ADDR_MASK                                                                          0xFFFFFF80L
+//TCP_WATCH3_CNTL
+#define TCP_WATCH3_CNTL__MASK__SHIFT                                                                          0x0
+#define TCP_WATCH3_CNTL__VMID__SHIFT                                                                          0x18
+#define TCP_WATCH3_CNTL__MODE__SHIFT                                                                          0x1d
+#define TCP_WATCH3_CNTL__VALID__SHIFT                                                                         0x1f
+#define TCP_WATCH3_CNTL__MASK_MASK                                                                            0x007FFFFFL
+#define TCP_WATCH3_CNTL__VMID_MASK                                                                            0x0F000000L
+#define TCP_WATCH3_CNTL__MODE_MASK                                                                            0x60000000L
+#define TCP_WATCH3_CNTL__VALID_MASK                                                                           0x80000000L
+
+
+// addressBlock: gc_gdspdec
+//GDS_VMID0_BASE
+#define GDS_VMID0_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID0_BASE__UNUSED__SHIFT                                                                         0x10
+#define GDS_VMID0_BASE__BASE_MASK                                                                             0x0000FFFFL
+#define GDS_VMID0_BASE__UNUSED_MASK                                                                           0xFFFF0000L
+//GDS_VMID0_SIZE
+#define GDS_VMID0_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID0_SIZE__UNUSED__SHIFT                                                                         0x11
+#define GDS_VMID0_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+#define GDS_VMID0_SIZE__UNUSED_MASK                                                                           0xFFFE0000L
+//GDS_VMID1_BASE
+#define GDS_VMID1_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID1_BASE__UNUSED__SHIFT                                                                         0x10
+#define GDS_VMID1_BASE__BASE_MASK                                                                             0x0000FFFFL
+#define GDS_VMID1_BASE__UNUSED_MASK                                                                           0xFFFF0000L
+//GDS_VMID1_SIZE
+#define GDS_VMID1_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID1_SIZE__UNUSED__SHIFT                                                                         0x11
+#define GDS_VMID1_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+#define GDS_VMID1_SIZE__UNUSED_MASK                                                                           0xFFFE0000L
+//GDS_VMID2_BASE
+#define GDS_VMID2_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID2_BASE__UNUSED__SHIFT                                                                         0x10
+#define GDS_VMID2_BASE__BASE_MASK                                                                             0x0000FFFFL
+#define GDS_VMID2_BASE__UNUSED_MASK                                                                           0xFFFF0000L
+//GDS_VMID2_SIZE
+#define GDS_VMID2_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID2_SIZE__UNUSED__SHIFT                                                                         0x11
+#define GDS_VMID2_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+#define GDS_VMID2_SIZE__UNUSED_MASK                                                                           0xFFFE0000L
+//GDS_VMID3_BASE
+#define GDS_VMID3_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID3_BASE__UNUSED__SHIFT                                                                         0x10
+#define GDS_VMID3_BASE__BASE_MASK                                                                             0x0000FFFFL
+#define GDS_VMID3_BASE__UNUSED_MASK                                                                           0xFFFF0000L
+//GDS_VMID3_SIZE
+#define GDS_VMID3_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID3_SIZE__UNUSED__SHIFT                                                                         0x11
+#define GDS_VMID3_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+#define GDS_VMID3_SIZE__UNUSED_MASK                                                                           0xFFFE0000L
+//GDS_VMID4_BASE
+#define GDS_VMID4_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID4_BASE__UNUSED__SHIFT                                                                         0x10
+#define GDS_VMID4_BASE__BASE_MASK                                                                             0x0000FFFFL
+#define GDS_VMID4_BASE__UNUSED_MASK                                                                           0xFFFF0000L
+//GDS_VMID4_SIZE
+#define GDS_VMID4_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID4_SIZE__UNUSED__SHIFT                                                                         0x11
+#define GDS_VMID4_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+#define GDS_VMID4_SIZE__UNUSED_MASK                                                                           0xFFFE0000L
+//GDS_VMID5_BASE
+#define GDS_VMID5_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID5_BASE__UNUSED__SHIFT                                                                         0x10
+#define GDS_VMID5_BASE__BASE_MASK                                                                             0x0000FFFFL
+#define GDS_VMID5_BASE__UNUSED_MASK                                                                           0xFFFF0000L
+//GDS_VMID5_SIZE
+#define GDS_VMID5_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID5_SIZE__UNUSED__SHIFT                                                                         0x11
+#define GDS_VMID5_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+#define GDS_VMID5_SIZE__UNUSED_MASK                                                                           0xFFFE0000L
+//GDS_VMID6_BASE
+#define GDS_VMID6_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID6_BASE__UNUSED__SHIFT                                                                         0x10
+#define GDS_VMID6_BASE__BASE_MASK                                                                             0x0000FFFFL
+#define GDS_VMID6_BASE__UNUSED_MASK                                                                           0xFFFF0000L
+//GDS_VMID6_SIZE
+#define GDS_VMID6_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID6_SIZE__UNUSED__SHIFT                                                                         0x11
+#define GDS_VMID6_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+#define GDS_VMID6_SIZE__UNUSED_MASK                                                                           0xFFFE0000L
+//GDS_VMID7_BASE
+#define GDS_VMID7_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID7_BASE__UNUSED__SHIFT                                                                         0x10
+#define GDS_VMID7_BASE__BASE_MASK                                                                             0x0000FFFFL
+#define GDS_VMID7_BASE__UNUSED_MASK                                                                           0xFFFF0000L
+//GDS_VMID7_SIZE
+#define GDS_VMID7_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID7_SIZE__UNUSED__SHIFT                                                                         0x11
+#define GDS_VMID7_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+#define GDS_VMID7_SIZE__UNUSED_MASK                                                                           0xFFFE0000L
+//GDS_VMID8_BASE
+#define GDS_VMID8_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID8_BASE__UNUSED__SHIFT                                                                         0x10
+#define GDS_VMID8_BASE__BASE_MASK                                                                             0x0000FFFFL
+#define GDS_VMID8_BASE__UNUSED_MASK                                                                           0xFFFF0000L
+//GDS_VMID8_SIZE
+#define GDS_VMID8_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID8_SIZE__UNUSED__SHIFT                                                                         0x11
+#define GDS_VMID8_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+#define GDS_VMID8_SIZE__UNUSED_MASK                                                                           0xFFFE0000L
+//GDS_VMID9_BASE
+#define GDS_VMID9_BASE__BASE__SHIFT                                                                           0x0
+#define GDS_VMID9_BASE__UNUSED__SHIFT                                                                         0x10
+#define GDS_VMID9_BASE__BASE_MASK                                                                             0x0000FFFFL
+#define GDS_VMID9_BASE__UNUSED_MASK                                                                           0xFFFF0000L
+//GDS_VMID9_SIZE
+#define GDS_VMID9_SIZE__SIZE__SHIFT                                                                           0x0
+#define GDS_VMID9_SIZE__UNUSED__SHIFT                                                                         0x11
+#define GDS_VMID9_SIZE__SIZE_MASK                                                                             0x0001FFFFL
+#define GDS_VMID9_SIZE__UNUSED_MASK                                                                           0xFFFE0000L
+//GDS_VMID10_BASE
+#define GDS_VMID10_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID10_BASE__UNUSED__SHIFT                                                                        0x10
+#define GDS_VMID10_BASE__BASE_MASK                                                                            0x0000FFFFL
+#define GDS_VMID10_BASE__UNUSED_MASK                                                                          0xFFFF0000L
+//GDS_VMID10_SIZE
+#define GDS_VMID10_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID10_SIZE__UNUSED__SHIFT                                                                        0x11
+#define GDS_VMID10_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+#define GDS_VMID10_SIZE__UNUSED_MASK                                                                          0xFFFE0000L
+//GDS_VMID11_BASE
+#define GDS_VMID11_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID11_BASE__UNUSED__SHIFT                                                                        0x10
+#define GDS_VMID11_BASE__BASE_MASK                                                                            0x0000FFFFL
+#define GDS_VMID11_BASE__UNUSED_MASK                                                                          0xFFFF0000L
+//GDS_VMID11_SIZE
+#define GDS_VMID11_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID11_SIZE__UNUSED__SHIFT                                                                        0x11
+#define GDS_VMID11_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+#define GDS_VMID11_SIZE__UNUSED_MASK                                                                          0xFFFE0000L
+//GDS_VMID12_BASE
+#define GDS_VMID12_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID12_BASE__UNUSED__SHIFT                                                                        0x10
+#define GDS_VMID12_BASE__BASE_MASK                                                                            0x0000FFFFL
+#define GDS_VMID12_BASE__UNUSED_MASK                                                                          0xFFFF0000L
+//GDS_VMID12_SIZE
+#define GDS_VMID12_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID12_SIZE__UNUSED__SHIFT                                                                        0x11
+#define GDS_VMID12_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+#define GDS_VMID12_SIZE__UNUSED_MASK                                                                          0xFFFE0000L
+//GDS_VMID13_BASE
+#define GDS_VMID13_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID13_BASE__UNUSED__SHIFT                                                                        0x10
+#define GDS_VMID13_BASE__BASE_MASK                                                                            0x0000FFFFL
+#define GDS_VMID13_BASE__UNUSED_MASK                                                                          0xFFFF0000L
+//GDS_VMID13_SIZE
+#define GDS_VMID13_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID13_SIZE__UNUSED__SHIFT                                                                        0x11
+#define GDS_VMID13_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+#define GDS_VMID13_SIZE__UNUSED_MASK                                                                          0xFFFE0000L
+//GDS_VMID14_BASE
+#define GDS_VMID14_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID14_BASE__UNUSED__SHIFT                                                                        0x10
+#define GDS_VMID14_BASE__BASE_MASK                                                                            0x0000FFFFL
+#define GDS_VMID14_BASE__UNUSED_MASK                                                                          0xFFFF0000L
+//GDS_VMID14_SIZE
+#define GDS_VMID14_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID14_SIZE__UNUSED__SHIFT                                                                        0x11
+#define GDS_VMID14_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+#define GDS_VMID14_SIZE__UNUSED_MASK                                                                          0xFFFE0000L
+//GDS_VMID15_BASE
+#define GDS_VMID15_BASE__BASE__SHIFT                                                                          0x0
+#define GDS_VMID15_BASE__UNUSED__SHIFT                                                                        0x10
+#define GDS_VMID15_BASE__BASE_MASK                                                                            0x0000FFFFL
+#define GDS_VMID15_BASE__UNUSED_MASK                                                                          0xFFFF0000L
+//GDS_VMID15_SIZE
+#define GDS_VMID15_SIZE__SIZE__SHIFT                                                                          0x0
+#define GDS_VMID15_SIZE__UNUSED__SHIFT                                                                        0x11
+#define GDS_VMID15_SIZE__SIZE_MASK                                                                            0x0001FFFFL
+#define GDS_VMID15_SIZE__UNUSED_MASK                                                                          0xFFFE0000L
+//GDS_GWS_VMID0
+#define GDS_GWS_VMID0__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID0__UNUSED1__SHIFT                                                                         0x6
+#define GDS_GWS_VMID0__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID0__UNUSED2__SHIFT                                                                         0x17
+#define GDS_GWS_VMID0__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID0__UNUSED1_MASK                                                                           0x0000FFC0L
+#define GDS_GWS_VMID0__SIZE_MASK                                                                              0x007F0000L
+#define GDS_GWS_VMID0__UNUSED2_MASK                                                                           0xFF800000L
+//GDS_GWS_VMID1
+#define GDS_GWS_VMID1__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID1__UNUSED1__SHIFT                                                                         0x6
+#define GDS_GWS_VMID1__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID1__UNUSED2__SHIFT                                                                         0x17
+#define GDS_GWS_VMID1__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID1__UNUSED1_MASK                                                                           0x0000FFC0L
+#define GDS_GWS_VMID1__SIZE_MASK                                                                              0x007F0000L
+#define GDS_GWS_VMID1__UNUSED2_MASK                                                                           0xFF800000L
+//GDS_GWS_VMID2
+#define GDS_GWS_VMID2__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID2__UNUSED1__SHIFT                                                                         0x6
+#define GDS_GWS_VMID2__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID2__UNUSED2__SHIFT                                                                         0x17
+#define GDS_GWS_VMID2__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID2__UNUSED1_MASK                                                                           0x0000FFC0L
+#define GDS_GWS_VMID2__SIZE_MASK                                                                              0x007F0000L
+#define GDS_GWS_VMID2__UNUSED2_MASK                                                                           0xFF800000L
+//GDS_GWS_VMID3
+#define GDS_GWS_VMID3__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID3__UNUSED1__SHIFT                                                                         0x6
+#define GDS_GWS_VMID3__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID3__UNUSED2__SHIFT                                                                         0x17
+#define GDS_GWS_VMID3__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID3__UNUSED1_MASK                                                                           0x0000FFC0L
+#define GDS_GWS_VMID3__SIZE_MASK                                                                              0x007F0000L
+#define GDS_GWS_VMID3__UNUSED2_MASK                                                                           0xFF800000L
+//GDS_GWS_VMID4
+#define GDS_GWS_VMID4__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID4__UNUSED1__SHIFT                                                                         0x6
+#define GDS_GWS_VMID4__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID4__UNUSED2__SHIFT                                                                         0x17
+#define GDS_GWS_VMID4__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID4__UNUSED1_MASK                                                                           0x0000FFC0L
+#define GDS_GWS_VMID4__SIZE_MASK                                                                              0x007F0000L
+#define GDS_GWS_VMID4__UNUSED2_MASK                                                                           0xFF800000L
+//GDS_GWS_VMID5
+#define GDS_GWS_VMID5__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID5__UNUSED1__SHIFT                                                                         0x6
+#define GDS_GWS_VMID5__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID5__UNUSED2__SHIFT                                                                         0x17
+#define GDS_GWS_VMID5__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID5__UNUSED1_MASK                                                                           0x0000FFC0L
+#define GDS_GWS_VMID5__SIZE_MASK                                                                              0x007F0000L
+#define GDS_GWS_VMID5__UNUSED2_MASK                                                                           0xFF800000L
+//GDS_GWS_VMID6
+#define GDS_GWS_VMID6__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID6__UNUSED1__SHIFT                                                                         0x6
+#define GDS_GWS_VMID6__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID6__UNUSED2__SHIFT                                                                         0x17
+#define GDS_GWS_VMID6__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID6__UNUSED1_MASK                                                                           0x0000FFC0L
+#define GDS_GWS_VMID6__SIZE_MASK                                                                              0x007F0000L
+#define GDS_GWS_VMID6__UNUSED2_MASK                                                                           0xFF800000L
+//GDS_GWS_VMID7
+#define GDS_GWS_VMID7__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID7__UNUSED1__SHIFT                                                                         0x6
+#define GDS_GWS_VMID7__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID7__UNUSED2__SHIFT                                                                         0x17
+#define GDS_GWS_VMID7__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID7__UNUSED1_MASK                                                                           0x0000FFC0L
+#define GDS_GWS_VMID7__SIZE_MASK                                                                              0x007F0000L
+#define GDS_GWS_VMID7__UNUSED2_MASK                                                                           0xFF800000L
+//GDS_GWS_VMID8
+#define GDS_GWS_VMID8__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID8__UNUSED1__SHIFT                                                                         0x6
+#define GDS_GWS_VMID8__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID8__UNUSED2__SHIFT                                                                         0x17
+#define GDS_GWS_VMID8__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID8__UNUSED1_MASK                                                                           0x0000FFC0L
+#define GDS_GWS_VMID8__SIZE_MASK                                                                              0x007F0000L
+#define GDS_GWS_VMID8__UNUSED2_MASK                                                                           0xFF800000L
+//GDS_GWS_VMID9
+#define GDS_GWS_VMID9__BASE__SHIFT                                                                            0x0
+#define GDS_GWS_VMID9__UNUSED1__SHIFT                                                                         0x6
+#define GDS_GWS_VMID9__SIZE__SHIFT                                                                            0x10
+#define GDS_GWS_VMID9__UNUSED2__SHIFT                                                                         0x17
+#define GDS_GWS_VMID9__BASE_MASK                                                                              0x0000003FL
+#define GDS_GWS_VMID9__UNUSED1_MASK                                                                           0x0000FFC0L
+#define GDS_GWS_VMID9__SIZE_MASK                                                                              0x007F0000L
+#define GDS_GWS_VMID9__UNUSED2_MASK                                                                           0xFF800000L
+//GDS_GWS_VMID10
+#define GDS_GWS_VMID10__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID10__UNUSED1__SHIFT                                                                        0x6
+#define GDS_GWS_VMID10__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID10__UNUSED2__SHIFT                                                                        0x17
+#define GDS_GWS_VMID10__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID10__UNUSED1_MASK                                                                          0x0000FFC0L
+#define GDS_GWS_VMID10__SIZE_MASK                                                                             0x007F0000L
+#define GDS_GWS_VMID10__UNUSED2_MASK                                                                          0xFF800000L
+//GDS_GWS_VMID11
+#define GDS_GWS_VMID11__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID11__UNUSED1__SHIFT                                                                        0x6
+#define GDS_GWS_VMID11__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID11__UNUSED2__SHIFT                                                                        0x17
+#define GDS_GWS_VMID11__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID11__UNUSED1_MASK                                                                          0x0000FFC0L
+#define GDS_GWS_VMID11__SIZE_MASK                                                                             0x007F0000L
+#define GDS_GWS_VMID11__UNUSED2_MASK                                                                          0xFF800000L
+//GDS_GWS_VMID12
+#define GDS_GWS_VMID12__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID12__UNUSED1__SHIFT                                                                        0x6
+#define GDS_GWS_VMID12__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID12__UNUSED2__SHIFT                                                                        0x17
+#define GDS_GWS_VMID12__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID12__UNUSED1_MASK                                                                          0x0000FFC0L
+#define GDS_GWS_VMID12__SIZE_MASK                                                                             0x007F0000L
+#define GDS_GWS_VMID12__UNUSED2_MASK                                                                          0xFF800000L
+//GDS_GWS_VMID13
+#define GDS_GWS_VMID13__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID13__UNUSED1__SHIFT                                                                        0x6
+#define GDS_GWS_VMID13__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID13__UNUSED2__SHIFT                                                                        0x17
+#define GDS_GWS_VMID13__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID13__UNUSED1_MASK                                                                          0x0000FFC0L
+#define GDS_GWS_VMID13__SIZE_MASK                                                                             0x007F0000L
+#define GDS_GWS_VMID13__UNUSED2_MASK                                                                          0xFF800000L
+//GDS_GWS_VMID14
+#define GDS_GWS_VMID14__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID14__UNUSED1__SHIFT                                                                        0x6
+#define GDS_GWS_VMID14__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID14__UNUSED2__SHIFT                                                                        0x17
+#define GDS_GWS_VMID14__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID14__UNUSED1_MASK                                                                          0x0000FFC0L
+#define GDS_GWS_VMID14__SIZE_MASK                                                                             0x007F0000L
+#define GDS_GWS_VMID14__UNUSED2_MASK                                                                          0xFF800000L
+//GDS_GWS_VMID15
+#define GDS_GWS_VMID15__BASE__SHIFT                                                                           0x0
+#define GDS_GWS_VMID15__UNUSED1__SHIFT                                                                        0x6
+#define GDS_GWS_VMID15__SIZE__SHIFT                                                                           0x10
+#define GDS_GWS_VMID15__UNUSED2__SHIFT                                                                        0x17
+#define GDS_GWS_VMID15__BASE_MASK                                                                             0x0000003FL
+#define GDS_GWS_VMID15__UNUSED1_MASK                                                                          0x0000FFC0L
+#define GDS_GWS_VMID15__SIZE_MASK                                                                             0x007F0000L
+#define GDS_GWS_VMID15__UNUSED2_MASK                                                                          0xFF800000L
+//GDS_OA_VMID0
+#define GDS_OA_VMID0__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID0__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID0__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID0__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID1
+#define GDS_OA_VMID1__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID1__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID1__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID1__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID2
+#define GDS_OA_VMID2__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID2__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID2__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID2__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID3
+#define GDS_OA_VMID3__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID3__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID3__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID3__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID4
+#define GDS_OA_VMID4__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID4__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID4__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID4__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID5
+#define GDS_OA_VMID5__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID5__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID5__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID5__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID6
+#define GDS_OA_VMID6__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID6__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID6__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID6__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID7
+#define GDS_OA_VMID7__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID7__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID7__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID7__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID8
+#define GDS_OA_VMID8__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID8__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID8__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID8__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID9
+#define GDS_OA_VMID9__MASK__SHIFT                                                                             0x0
+#define GDS_OA_VMID9__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_VMID9__MASK_MASK                                                                               0x0000FFFFL
+#define GDS_OA_VMID9__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_OA_VMID10
+#define GDS_OA_VMID10__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID10__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID10__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID10__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_OA_VMID11
+#define GDS_OA_VMID11__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID11__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID11__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID11__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_OA_VMID12
+#define GDS_OA_VMID12__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID12__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID12__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID12__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_OA_VMID13
+#define GDS_OA_VMID13__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID13__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID13__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID13__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_OA_VMID14
+#define GDS_OA_VMID14__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID14__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID14__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID14__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_OA_VMID15
+#define GDS_OA_VMID15__MASK__SHIFT                                                                            0x0
+#define GDS_OA_VMID15__UNUSED__SHIFT                                                                          0x10
+#define GDS_OA_VMID15__MASK_MASK                                                                              0x0000FFFFL
+#define GDS_OA_VMID15__UNUSED_MASK                                                                            0xFFFF0000L
+//GDS_GWS_RESET0
+#define GDS_GWS_RESET0__RESOURCE0_RESET__SHIFT                                                                0x0
+#define GDS_GWS_RESET0__RESOURCE1_RESET__SHIFT                                                                0x1
+#define GDS_GWS_RESET0__RESOURCE2_RESET__SHIFT                                                                0x2
+#define GDS_GWS_RESET0__RESOURCE3_RESET__SHIFT                                                                0x3
+#define GDS_GWS_RESET0__RESOURCE4_RESET__SHIFT                                                                0x4
+#define GDS_GWS_RESET0__RESOURCE5_RESET__SHIFT                                                                0x5
+#define GDS_GWS_RESET0__RESOURCE6_RESET__SHIFT                                                                0x6
+#define GDS_GWS_RESET0__RESOURCE7_RESET__SHIFT                                                                0x7
+#define GDS_GWS_RESET0__RESOURCE8_RESET__SHIFT                                                                0x8
+#define GDS_GWS_RESET0__RESOURCE9_RESET__SHIFT                                                                0x9
+#define GDS_GWS_RESET0__RESOURCE10_RESET__SHIFT                                                               0xa
+#define GDS_GWS_RESET0__RESOURCE11_RESET__SHIFT                                                               0xb
+#define GDS_GWS_RESET0__RESOURCE12_RESET__SHIFT                                                               0xc
+#define GDS_GWS_RESET0__RESOURCE13_RESET__SHIFT                                                               0xd
+#define GDS_GWS_RESET0__RESOURCE14_RESET__SHIFT                                                               0xe
+#define GDS_GWS_RESET0__RESOURCE15_RESET__SHIFT                                                               0xf
+#define GDS_GWS_RESET0__RESOURCE16_RESET__SHIFT                                                               0x10
+#define GDS_GWS_RESET0__RESOURCE17_RESET__SHIFT                                                               0x11
+#define GDS_GWS_RESET0__RESOURCE18_RESET__SHIFT                                                               0x12
+#define GDS_GWS_RESET0__RESOURCE19_RESET__SHIFT                                                               0x13
+#define GDS_GWS_RESET0__RESOURCE20_RESET__SHIFT                                                               0x14
+#define GDS_GWS_RESET0__RESOURCE21_RESET__SHIFT                                                               0x15
+#define GDS_GWS_RESET0__RESOURCE22_RESET__SHIFT                                                               0x16
+#define GDS_GWS_RESET0__RESOURCE23_RESET__SHIFT                                                               0x17
+#define GDS_GWS_RESET0__RESOURCE24_RESET__SHIFT                                                               0x18
+#define GDS_GWS_RESET0__RESOURCE25_RESET__SHIFT                                                               0x19
+#define GDS_GWS_RESET0__RESOURCE26_RESET__SHIFT                                                               0x1a
+#define GDS_GWS_RESET0__RESOURCE27_RESET__SHIFT                                                               0x1b
+#define GDS_GWS_RESET0__RESOURCE28_RESET__SHIFT                                                               0x1c
+#define GDS_GWS_RESET0__RESOURCE29_RESET__SHIFT                                                               0x1d
+#define GDS_GWS_RESET0__RESOURCE30_RESET__SHIFT                                                               0x1e
+#define GDS_GWS_RESET0__RESOURCE31_RESET__SHIFT                                                               0x1f
+#define GDS_GWS_RESET0__RESOURCE0_RESET_MASK                                                                  0x00000001L
+#define GDS_GWS_RESET0__RESOURCE1_RESET_MASK                                                                  0x00000002L
+#define GDS_GWS_RESET0__RESOURCE2_RESET_MASK                                                                  0x00000004L
+#define GDS_GWS_RESET0__RESOURCE3_RESET_MASK                                                                  0x00000008L
+#define GDS_GWS_RESET0__RESOURCE4_RESET_MASK                                                                  0x00000010L
+#define GDS_GWS_RESET0__RESOURCE5_RESET_MASK                                                                  0x00000020L
+#define GDS_GWS_RESET0__RESOURCE6_RESET_MASK                                                                  0x00000040L
+#define GDS_GWS_RESET0__RESOURCE7_RESET_MASK                                                                  0x00000080L
+#define GDS_GWS_RESET0__RESOURCE8_RESET_MASK                                                                  0x00000100L
+#define GDS_GWS_RESET0__RESOURCE9_RESET_MASK                                                                  0x00000200L
+#define GDS_GWS_RESET0__RESOURCE10_RESET_MASK                                                                 0x00000400L
+#define GDS_GWS_RESET0__RESOURCE11_RESET_MASK                                                                 0x00000800L
+#define GDS_GWS_RESET0__RESOURCE12_RESET_MASK                                                                 0x00001000L
+#define GDS_GWS_RESET0__RESOURCE13_RESET_MASK                                                                 0x00002000L
+#define GDS_GWS_RESET0__RESOURCE14_RESET_MASK                                                                 0x00004000L
+#define GDS_GWS_RESET0__RESOURCE15_RESET_MASK                                                                 0x00008000L
+#define GDS_GWS_RESET0__RESOURCE16_RESET_MASK                                                                 0x00010000L
+#define GDS_GWS_RESET0__RESOURCE17_RESET_MASK                                                                 0x00020000L
+#define GDS_GWS_RESET0__RESOURCE18_RESET_MASK                                                                 0x00040000L
+#define GDS_GWS_RESET0__RESOURCE19_RESET_MASK                                                                 0x00080000L
+#define GDS_GWS_RESET0__RESOURCE20_RESET_MASK                                                                 0x00100000L
+#define GDS_GWS_RESET0__RESOURCE21_RESET_MASK                                                                 0x00200000L
+#define GDS_GWS_RESET0__RESOURCE22_RESET_MASK                                                                 0x00400000L
+#define GDS_GWS_RESET0__RESOURCE23_RESET_MASK                                                                 0x00800000L
+#define GDS_GWS_RESET0__RESOURCE24_RESET_MASK                                                                 0x01000000L
+#define GDS_GWS_RESET0__RESOURCE25_RESET_MASK                                                                 0x02000000L
+#define GDS_GWS_RESET0__RESOURCE26_RESET_MASK                                                                 0x04000000L
+#define GDS_GWS_RESET0__RESOURCE27_RESET_MASK                                                                 0x08000000L
+#define GDS_GWS_RESET0__RESOURCE28_RESET_MASK                                                                 0x10000000L
+#define GDS_GWS_RESET0__RESOURCE29_RESET_MASK                                                                 0x20000000L
+#define GDS_GWS_RESET0__RESOURCE30_RESET_MASK                                                                 0x40000000L
+#define GDS_GWS_RESET0__RESOURCE31_RESET_MASK                                                                 0x80000000L
+//GDS_GWS_RESET1
+#define GDS_GWS_RESET1__RESOURCE32_RESET__SHIFT                                                               0x0
+#define GDS_GWS_RESET1__RESOURCE33_RESET__SHIFT                                                               0x1
+#define GDS_GWS_RESET1__RESOURCE34_RESET__SHIFT                                                               0x2
+#define GDS_GWS_RESET1__RESOURCE35_RESET__SHIFT                                                               0x3
+#define GDS_GWS_RESET1__RESOURCE36_RESET__SHIFT                                                               0x4
+#define GDS_GWS_RESET1__RESOURCE37_RESET__SHIFT                                                               0x5
+#define GDS_GWS_RESET1__RESOURCE38_RESET__SHIFT                                                               0x6
+#define GDS_GWS_RESET1__RESOURCE39_RESET__SHIFT                                                               0x7
+#define GDS_GWS_RESET1__RESOURCE40_RESET__SHIFT                                                               0x8
+#define GDS_GWS_RESET1__RESOURCE41_RESET__SHIFT                                                               0x9
+#define GDS_GWS_RESET1__RESOURCE42_RESET__SHIFT                                                               0xa
+#define GDS_GWS_RESET1__RESOURCE43_RESET__SHIFT                                                               0xb
+#define GDS_GWS_RESET1__RESOURCE44_RESET__SHIFT                                                               0xc
+#define GDS_GWS_RESET1__RESOURCE45_RESET__SHIFT                                                               0xd
+#define GDS_GWS_RESET1__RESOURCE46_RESET__SHIFT                                                               0xe
+#define GDS_GWS_RESET1__RESOURCE47_RESET__SHIFT                                                               0xf
+#define GDS_GWS_RESET1__RESOURCE48_RESET__SHIFT                                                               0x10
+#define GDS_GWS_RESET1__RESOURCE49_RESET__SHIFT                                                               0x11
+#define GDS_GWS_RESET1__RESOURCE50_RESET__SHIFT                                                               0x12
+#define GDS_GWS_RESET1__RESOURCE51_RESET__SHIFT                                                               0x13
+#define GDS_GWS_RESET1__RESOURCE52_RESET__SHIFT                                                               0x14
+#define GDS_GWS_RESET1__RESOURCE53_RESET__SHIFT                                                               0x15
+#define GDS_GWS_RESET1__RESOURCE54_RESET__SHIFT                                                               0x16
+#define GDS_GWS_RESET1__RESOURCE55_RESET__SHIFT                                                               0x17
+#define GDS_GWS_RESET1__RESOURCE56_RESET__SHIFT                                                               0x18
+#define GDS_GWS_RESET1__RESOURCE57_RESET__SHIFT                                                               0x19
+#define GDS_GWS_RESET1__RESOURCE58_RESET__SHIFT                                                               0x1a
+#define GDS_GWS_RESET1__RESOURCE59_RESET__SHIFT                                                               0x1b
+#define GDS_GWS_RESET1__RESOURCE60_RESET__SHIFT                                                               0x1c
+#define GDS_GWS_RESET1__RESOURCE61_RESET__SHIFT                                                               0x1d
+#define GDS_GWS_RESET1__RESOURCE62_RESET__SHIFT                                                               0x1e
+#define GDS_GWS_RESET1__RESOURCE63_RESET__SHIFT                                                               0x1f
+#define GDS_GWS_RESET1__RESOURCE32_RESET_MASK                                                                 0x00000001L
+#define GDS_GWS_RESET1__RESOURCE33_RESET_MASK                                                                 0x00000002L
+#define GDS_GWS_RESET1__RESOURCE34_RESET_MASK                                                                 0x00000004L
+#define GDS_GWS_RESET1__RESOURCE35_RESET_MASK                                                                 0x00000008L
+#define GDS_GWS_RESET1__RESOURCE36_RESET_MASK                                                                 0x00000010L
+#define GDS_GWS_RESET1__RESOURCE37_RESET_MASK                                                                 0x00000020L
+#define GDS_GWS_RESET1__RESOURCE38_RESET_MASK                                                                 0x00000040L
+#define GDS_GWS_RESET1__RESOURCE39_RESET_MASK                                                                 0x00000080L
+#define GDS_GWS_RESET1__RESOURCE40_RESET_MASK                                                                 0x00000100L
+#define GDS_GWS_RESET1__RESOURCE41_RESET_MASK                                                                 0x00000200L
+#define GDS_GWS_RESET1__RESOURCE42_RESET_MASK                                                                 0x00000400L
+#define GDS_GWS_RESET1__RESOURCE43_RESET_MASK                                                                 0x00000800L
+#define GDS_GWS_RESET1__RESOURCE44_RESET_MASK                                                                 0x00001000L
+#define GDS_GWS_RESET1__RESOURCE45_RESET_MASK                                                                 0x00002000L
+#define GDS_GWS_RESET1__RESOURCE46_RESET_MASK                                                                 0x00004000L
+#define GDS_GWS_RESET1__RESOURCE47_RESET_MASK                                                                 0x00008000L
+#define GDS_GWS_RESET1__RESOURCE48_RESET_MASK                                                                 0x00010000L
+#define GDS_GWS_RESET1__RESOURCE49_RESET_MASK                                                                 0x00020000L
+#define GDS_GWS_RESET1__RESOURCE50_RESET_MASK                                                                 0x00040000L
+#define GDS_GWS_RESET1__RESOURCE51_RESET_MASK                                                                 0x00080000L
+#define GDS_GWS_RESET1__RESOURCE52_RESET_MASK                                                                 0x00100000L
+#define GDS_GWS_RESET1__RESOURCE53_RESET_MASK                                                                 0x00200000L
+#define GDS_GWS_RESET1__RESOURCE54_RESET_MASK                                                                 0x00400000L
+#define GDS_GWS_RESET1__RESOURCE55_RESET_MASK                                                                 0x00800000L
+#define GDS_GWS_RESET1__RESOURCE56_RESET_MASK                                                                 0x01000000L
+#define GDS_GWS_RESET1__RESOURCE57_RESET_MASK                                                                 0x02000000L
+#define GDS_GWS_RESET1__RESOURCE58_RESET_MASK                                                                 0x04000000L
+#define GDS_GWS_RESET1__RESOURCE59_RESET_MASK                                                                 0x08000000L
+#define GDS_GWS_RESET1__RESOURCE60_RESET_MASK                                                                 0x10000000L
+#define GDS_GWS_RESET1__RESOURCE61_RESET_MASK                                                                 0x20000000L
+#define GDS_GWS_RESET1__RESOURCE62_RESET_MASK                                                                 0x40000000L
+#define GDS_GWS_RESET1__RESOURCE63_RESET_MASK                                                                 0x80000000L
+//GDS_GWS_RESOURCE_RESET
+#define GDS_GWS_RESOURCE_RESET__RESET__SHIFT                                                                  0x0
+#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID__SHIFT                                                            0x8
+#define GDS_GWS_RESOURCE_RESET__UNUSED__SHIFT                                                                 0x10
+#define GDS_GWS_RESOURCE_RESET__RESET_MASK                                                                    0x00000001L
+#define GDS_GWS_RESOURCE_RESET__RESOURCE_ID_MASK                                                              0x0000FF00L
+#define GDS_GWS_RESOURCE_RESET__UNUSED_MASK                                                                   0xFFFF0000L
+//GDS_COMPUTE_MAX_WAVE_ID
+#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID__SHIFT                                                           0x0
+#define GDS_COMPUTE_MAX_WAVE_ID__UNUSED__SHIFT                                                                0xc
+#define GDS_COMPUTE_MAX_WAVE_ID__MAX_WAVE_ID_MASK                                                             0x00000FFFL
+#define GDS_COMPUTE_MAX_WAVE_ID__UNUSED_MASK                                                                  0xFFFFF000L
+//GDS_OA_RESET_MASK
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_PIX_RESET__SHIFT                                                       0x0
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_VTX_RESET__SHIFT                                                       0x1
+#define GDS_OA_RESET_MASK__ME0_CS_RESET__SHIFT                                                                0x2
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_GS_RESET__SHIFT                                                        0x3
+#define GDS_OA_RESET_MASK__ME1_PIPE0_RESET__SHIFT                                                             0x4
+#define GDS_OA_RESET_MASK__ME1_PIPE1_RESET__SHIFT                                                             0x5
+#define GDS_OA_RESET_MASK__ME1_PIPE2_RESET__SHIFT                                                             0x6
+#define GDS_OA_RESET_MASK__ME1_PIPE3_RESET__SHIFT                                                             0x7
+#define GDS_OA_RESET_MASK__ME2_PIPE0_RESET__SHIFT                                                             0x8
+#define GDS_OA_RESET_MASK__ME2_PIPE1_RESET__SHIFT                                                             0x9
+#define GDS_OA_RESET_MASK__ME2_PIPE2_RESET__SHIFT                                                             0xa
+#define GDS_OA_RESET_MASK__ME2_PIPE3_RESET__SHIFT                                                             0xb
+#define GDS_OA_RESET_MASK__ME0_PIPE1_CS_RESET__SHIFT                                                          0xc
+#define GDS_OA_RESET_MASK__UNUSED1__SHIFT                                                                     0xd
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_PIX_RESET_MASK                                                         0x00000001L
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_VTX_RESET_MASK                                                         0x00000002L
+#define GDS_OA_RESET_MASK__ME0_CS_RESET_MASK                                                                  0x00000004L
+#define GDS_OA_RESET_MASK__ME0_GFXHP3D_GS_RESET_MASK                                                          0x00000008L
+#define GDS_OA_RESET_MASK__ME1_PIPE0_RESET_MASK                                                               0x00000010L
+#define GDS_OA_RESET_MASK__ME1_PIPE1_RESET_MASK                                                               0x00000020L
+#define GDS_OA_RESET_MASK__ME1_PIPE2_RESET_MASK                                                               0x00000040L
+#define GDS_OA_RESET_MASK__ME1_PIPE3_RESET_MASK                                                               0x00000080L
+#define GDS_OA_RESET_MASK__ME2_PIPE0_RESET_MASK                                                               0x00000100L
+#define GDS_OA_RESET_MASK__ME2_PIPE1_RESET_MASK                                                               0x00000200L
+#define GDS_OA_RESET_MASK__ME2_PIPE2_RESET_MASK                                                               0x00000400L
+#define GDS_OA_RESET_MASK__ME2_PIPE3_RESET_MASK                                                               0x00000800L
+#define GDS_OA_RESET_MASK__ME0_PIPE1_CS_RESET_MASK                                                            0x00001000L
+#define GDS_OA_RESET_MASK__UNUSED1_MASK                                                                       0xFFFFE000L
+//GDS_OA_RESET
+#define GDS_OA_RESET__RESET__SHIFT                                                                            0x0
+#define GDS_OA_RESET__PIPE_ID__SHIFT                                                                          0x8
+#define GDS_OA_RESET__UNUSED__SHIFT                                                                           0x10
+#define GDS_OA_RESET__RESET_MASK                                                                              0x00000001L
+#define GDS_OA_RESET__PIPE_ID_MASK                                                                            0x0000FF00L
+#define GDS_OA_RESET__UNUSED_MASK                                                                             0xFFFF0000L
+//GDS_CS_CTXSW_STATUS
+#define GDS_CS_CTXSW_STATUS__R__SHIFT                                                                         0x0
+#define GDS_CS_CTXSW_STATUS__W__SHIFT                                                                         0x1
+#define GDS_CS_CTXSW_STATUS__UNUSED__SHIFT                                                                    0x2
+#define GDS_CS_CTXSW_STATUS__R_MASK                                                                           0x00000001L
+#define GDS_CS_CTXSW_STATUS__W_MASK                                                                           0x00000002L
+#define GDS_CS_CTXSW_STATUS__UNUSED_MASK                                                                      0xFFFFFFFCL
+//GDS_CS_CTXSW_CNT0
+#define GDS_CS_CTXSW_CNT0__UPDN__SHIFT                                                                        0x0
+#define GDS_CS_CTXSW_CNT0__PTR__SHIFT                                                                         0x10
+#define GDS_CS_CTXSW_CNT0__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_CS_CTXSW_CNT0__PTR_MASK                                                                           0xFFFF0000L
+//GDS_CS_CTXSW_CNT1
+#define GDS_CS_CTXSW_CNT1__UPDN__SHIFT                                                                        0x0
+#define GDS_CS_CTXSW_CNT1__PTR__SHIFT                                                                         0x10
+#define GDS_CS_CTXSW_CNT1__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_CS_CTXSW_CNT1__PTR_MASK                                                                           0xFFFF0000L
+//GDS_CS_CTXSW_CNT2
+#define GDS_CS_CTXSW_CNT2__UPDN__SHIFT                                                                        0x0
+#define GDS_CS_CTXSW_CNT2__PTR__SHIFT                                                                         0x10
+#define GDS_CS_CTXSW_CNT2__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_CS_CTXSW_CNT2__PTR_MASK                                                                           0xFFFF0000L
+//GDS_CS_CTXSW_CNT3
+#define GDS_CS_CTXSW_CNT3__UPDN__SHIFT                                                                        0x0
+#define GDS_CS_CTXSW_CNT3__PTR__SHIFT                                                                         0x10
+#define GDS_CS_CTXSW_CNT3__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_CS_CTXSW_CNT3__PTR_MASK                                                                           0xFFFF0000L
+//GDS_GFX_CTXSW_STATUS
+#define GDS_GFX_CTXSW_STATUS__R__SHIFT                                                                        0x0
+#define GDS_GFX_CTXSW_STATUS__W__SHIFT                                                                        0x1
+#define GDS_GFX_CTXSW_STATUS__UNUSED__SHIFT                                                                   0x2
+#define GDS_GFX_CTXSW_STATUS__R_MASK                                                                          0x00000001L
+#define GDS_GFX_CTXSW_STATUS__W_MASK                                                                          0x00000002L
+#define GDS_GFX_CTXSW_STATUS__UNUSED_MASK                                                                     0xFFFFFFFCL
+//GDS_PS_CTXSW_CNT0
+#define GDS_PS_CTXSW_CNT0__UPDN__SHIFT                                                                        0x0
+#define GDS_PS_CTXSW_CNT0__PTR__SHIFT                                                                         0x10
+#define GDS_PS_CTXSW_CNT0__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_PS_CTXSW_CNT0__PTR_MASK                                                                           0xFFFF0000L
+//GDS_PS_CTXSW_CNT1
+#define GDS_PS_CTXSW_CNT1__UPDN__SHIFT                                                                        0x0
+#define GDS_PS_CTXSW_CNT1__PTR__SHIFT                                                                         0x10
+#define GDS_PS_CTXSW_CNT1__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_PS_CTXSW_CNT1__PTR_MASK                                                                           0xFFFF0000L
+//GDS_PS_CTXSW_CNT2
+#define GDS_PS_CTXSW_CNT2__UPDN__SHIFT                                                                        0x0
+#define GDS_PS_CTXSW_CNT2__PTR__SHIFT                                                                         0x10
+#define GDS_PS_CTXSW_CNT2__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_PS_CTXSW_CNT2__PTR_MASK                                                                           0xFFFF0000L
+//GDS_PS_CTXSW_CNT3
+#define GDS_PS_CTXSW_CNT3__UPDN__SHIFT                                                                        0x0
+#define GDS_PS_CTXSW_CNT3__PTR__SHIFT                                                                         0x10
+#define GDS_PS_CTXSW_CNT3__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_PS_CTXSW_CNT3__PTR_MASK                                                                           0xFFFF0000L
+//GDS_PS_CTXSW_IDX
+#define GDS_PS_CTXSW_IDX__PACKER_ID__SHIFT                                                                    0x0
+#define GDS_PS_CTXSW_IDX__UNUSED__SHIFT                                                                       0x6
+#define GDS_PS_CTXSW_IDX__PACKER_ID_MASK                                                                      0x0000003FL
+#define GDS_PS_CTXSW_IDX__UNUSED_MASK                                                                         0xFFFFFFC0L
+//GDS_GS_CTXSW_CNT0
+#define GDS_GS_CTXSW_CNT0__UPDN__SHIFT                                                                        0x0
+#define GDS_GS_CTXSW_CNT0__PTR__SHIFT                                                                         0x10
+#define GDS_GS_CTXSW_CNT0__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_GS_CTXSW_CNT0__PTR_MASK                                                                           0xFFFF0000L
+//GDS_GS_CTXSW_CNT1
+#define GDS_GS_CTXSW_CNT1__UPDN__SHIFT                                                                        0x0
+#define GDS_GS_CTXSW_CNT1__PTR__SHIFT                                                                         0x10
+#define GDS_GS_CTXSW_CNT1__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_GS_CTXSW_CNT1__PTR_MASK                                                                           0xFFFF0000L
+//GDS_GS_CTXSW_CNT2
+#define GDS_GS_CTXSW_CNT2__UPDN__SHIFT                                                                        0x0
+#define GDS_GS_CTXSW_CNT2__PTR__SHIFT                                                                         0x10
+#define GDS_GS_CTXSW_CNT2__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_GS_CTXSW_CNT2__PTR_MASK                                                                           0xFFFF0000L
+//GDS_GS_CTXSW_CNT3
+#define GDS_GS_CTXSW_CNT3__UPDN__SHIFT                                                                        0x0
+#define GDS_GS_CTXSW_CNT3__PTR__SHIFT                                                                         0x10
+#define GDS_GS_CTXSW_CNT3__UPDN_MASK                                                                          0x0000FFFFL
+#define GDS_GS_CTXSW_CNT3__PTR_MASK                                                                           0xFFFF0000L
+//GDS_MEMORY_CLEAN
+#define GDS_MEMORY_CLEAN__START__SHIFT                                                                        0x0
+#define GDS_MEMORY_CLEAN__FINISH__SHIFT                                                                       0x1
+#define GDS_MEMORY_CLEAN__UNUSED__SHIFT                                                                       0x2
+#define GDS_MEMORY_CLEAN__START_MASK                                                                          0x00000001L
+#define GDS_MEMORY_CLEAN__FINISH_MASK                                                                         0x00000002L
+#define GDS_MEMORY_CLEAN__UNUSED_MASK                                                                         0xFFFFFFFCL
+
+
+// addressBlock: gc_rasdec
+//RAS_SIGNATURE_CONTROL
+#define RAS_SIGNATURE_CONTROL__ENABLE__SHIFT                                                                  0x0
+#define RAS_SIGNATURE_CONTROL__ENABLE_MASK                                                                    0x00000001L
+//RAS_SIGNATURE_MASK
+#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK__SHIFT                                                             0x0
+#define RAS_SIGNATURE_MASK__INPUT_BUS_MASK_MASK                                                               0xFFFFFFFFL
+//RAS_SX_SIGNATURE0
+#define RAS_SX_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SX_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SX_SIGNATURE1
+#define RAS_SX_SIGNATURE1__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SX_SIGNATURE1__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SX_SIGNATURE2
+#define RAS_SX_SIGNATURE2__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SX_SIGNATURE2__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SX_SIGNATURE3
+#define RAS_SX_SIGNATURE3__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SX_SIGNATURE3__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_DB_SIGNATURE0
+#define RAS_DB_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_DB_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_PA_SIGNATURE0
+#define RAS_PA_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_PA_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE0
+#define RAS_SC_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE1
+#define RAS_SC_SIGNATURE1__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE1__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE2
+#define RAS_SC_SIGNATURE2__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE2__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE3
+#define RAS_SC_SIGNATURE3__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE3__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE4
+#define RAS_SC_SIGNATURE4__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE4__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE5
+#define RAS_SC_SIGNATURE5__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE5__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE6
+#define RAS_SC_SIGNATURE6__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE6__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SC_SIGNATURE7
+#define RAS_SC_SIGNATURE7__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_SC_SIGNATURE7__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_SPI_SIGNATURE0
+#define RAS_SPI_SIGNATURE0__SIGNATURE__SHIFT                                                                  0x0
+#define RAS_SPI_SIGNATURE0__SIGNATURE_MASK                                                                    0xFFFFFFFFL
+//RAS_SPI_SIGNATURE1
+#define RAS_SPI_SIGNATURE1__SIGNATURE__SHIFT                                                                  0x0
+#define RAS_SPI_SIGNATURE1__SIGNATURE_MASK                                                                    0xFFFFFFFFL
+//RAS_CB_SIGNATURE0
+#define RAS_CB_SIGNATURE0__SIGNATURE__SHIFT                                                                   0x0
+#define RAS_CB_SIGNATURE0__SIGNATURE_MASK                                                                     0xFFFFFFFFL
+//RAS_BCI_SIGNATURE0
+#define RAS_BCI_SIGNATURE0__SIGNATURE__SHIFT                                                                  0x0
+#define RAS_BCI_SIGNATURE0__SIGNATURE_MASK                                                                    0xFFFFFFFFL
+//RAS_BCI_SIGNATURE1
+#define RAS_BCI_SIGNATURE1__SIGNATURE__SHIFT                                                                  0x0
+#define RAS_BCI_SIGNATURE1__SIGNATURE_MASK                                                                    0xFFFFFFFFL
+
+
+// addressBlock: gc_gusdec
+//GUS_IO_RD_COMBINE_FLUSH
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                          0x0
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                          0x4
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                          0x8
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                          0xc
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP4_TIMER__SHIFT                                                          0x10
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP5_TIMER__SHIFT                                                          0x14
+#define GUS_IO_RD_COMBINE_FLUSH__COMB_MODE__SHIFT                                                             0x18
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                            0x0000000FL
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                            0x000000F0L
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                            0x00000F00L
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                            0x0000F000L
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP4_TIMER_MASK                                                            0x000F0000L
+#define GUS_IO_RD_COMBINE_FLUSH__GROUP5_TIMER_MASK                                                            0x00F00000L
+#define GUS_IO_RD_COMBINE_FLUSH__COMB_MODE_MASK                                                               0x03000000L
+//GUS_IO_WR_COMBINE_FLUSH
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                          0x0
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                          0x4
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                          0x8
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                          0xc
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP4_TIMER__SHIFT                                                          0x10
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP5_TIMER__SHIFT                                                          0x14
+#define GUS_IO_WR_COMBINE_FLUSH__COMB_MODE__SHIFT                                                             0x18
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                            0x0000000FL
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                            0x000000F0L
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                            0x00000F00L
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                            0x0000F000L
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP4_TIMER_MASK                                                            0x000F0000L
+#define GUS_IO_WR_COMBINE_FLUSH__GROUP5_TIMER_MASK                                                            0x00F00000L
+#define GUS_IO_WR_COMBINE_FLUSH__COMB_MODE_MASK                                                               0x03000000L
+//GUS_IO_RD_PRI_AGE_RATE
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP0_AGING_RATE__SHIFT                                                      0x0
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP1_AGING_RATE__SHIFT                                                      0x3
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP2_AGING_RATE__SHIFT                                                      0x6
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP3_AGING_RATE__SHIFT                                                      0x9
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP4_AGING_RATE__SHIFT                                                      0xc
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP5_AGING_RATE__SHIFT                                                      0xf
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP0_AGING_RATE_MASK                                                        0x00000007L
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP1_AGING_RATE_MASK                                                        0x00000038L
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP2_AGING_RATE_MASK                                                        0x000001C0L
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP3_AGING_RATE_MASK                                                        0x00000E00L
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP4_AGING_RATE_MASK                                                        0x00007000L
+#define GUS_IO_RD_PRI_AGE_RATE__GROUP5_AGING_RATE_MASK                                                        0x00038000L
+//GUS_IO_WR_PRI_AGE_RATE
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP0_AGING_RATE__SHIFT                                                      0x0
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP1_AGING_RATE__SHIFT                                                      0x3
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP2_AGING_RATE__SHIFT                                                      0x6
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP3_AGING_RATE__SHIFT                                                      0x9
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP4_AGING_RATE__SHIFT                                                      0xc
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP5_AGING_RATE__SHIFT                                                      0xf
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP0_AGING_RATE_MASK                                                        0x00000007L
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP1_AGING_RATE_MASK                                                        0x00000038L
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP2_AGING_RATE_MASK                                                        0x000001C0L
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP3_AGING_RATE_MASK                                                        0x00000E00L
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP4_AGING_RATE_MASK                                                        0x00007000L
+#define GUS_IO_WR_PRI_AGE_RATE__GROUP5_AGING_RATE_MASK                                                        0x00038000L
+//GUS_IO_RD_PRI_AGE_COEFF
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP0_AGE_COEFFICIENT__SHIFT                                                0x0
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP1_AGE_COEFFICIENT__SHIFT                                                0x3
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP2_AGE_COEFFICIENT__SHIFT                                                0x6
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP3_AGE_COEFFICIENT__SHIFT                                                0x9
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP4_AGE_COEFFICIENT__SHIFT                                                0xc
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP5_AGE_COEFFICIENT__SHIFT                                                0xf
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP0_AGE_COEFFICIENT_MASK                                                  0x00000007L
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP1_AGE_COEFFICIENT_MASK                                                  0x00000038L
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP2_AGE_COEFFICIENT_MASK                                                  0x000001C0L
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP3_AGE_COEFFICIENT_MASK                                                  0x00000E00L
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP4_AGE_COEFFICIENT_MASK                                                  0x00007000L
+#define GUS_IO_RD_PRI_AGE_COEFF__GROUP5_AGE_COEFFICIENT_MASK                                                  0x00038000L
+//GUS_IO_WR_PRI_AGE_COEFF
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP0_AGE_COEFFICIENT__SHIFT                                                0x0
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP1_AGE_COEFFICIENT__SHIFT                                                0x3
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP2_AGE_COEFFICIENT__SHIFT                                                0x6
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP3_AGE_COEFFICIENT__SHIFT                                                0x9
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP4_AGE_COEFFICIENT__SHIFT                                                0xc
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP5_AGE_COEFFICIENT__SHIFT                                                0xf
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP0_AGE_COEFFICIENT_MASK                                                  0x00000007L
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP1_AGE_COEFFICIENT_MASK                                                  0x00000038L
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP2_AGE_COEFFICIENT_MASK                                                  0x000001C0L
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP3_AGE_COEFFICIENT_MASK                                                  0x00000E00L
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP4_AGE_COEFFICIENT_MASK                                                  0x00007000L
+#define GUS_IO_WR_PRI_AGE_COEFF__GROUP5_AGE_COEFFICIENT_MASK                                                  0x00038000L
+//GUS_IO_RD_PRI_QUEUING
+#define GUS_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                              0x0
+#define GUS_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                              0x3
+#define GUS_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                              0x6
+#define GUS_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                              0x9
+#define GUS_IO_RD_PRI_QUEUING__GROUP4_QUEUING_COEFFICIENT__SHIFT                                              0xc
+#define GUS_IO_RD_PRI_QUEUING__GROUP5_QUEUING_COEFFICIENT__SHIFT                                              0xf
+#define GUS_IO_RD_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                                0x00000007L
+#define GUS_IO_RD_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                                0x00000038L
+#define GUS_IO_RD_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                                0x000001C0L
+#define GUS_IO_RD_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                                0x00000E00L
+#define GUS_IO_RD_PRI_QUEUING__GROUP4_QUEUING_COEFFICIENT_MASK                                                0x00007000L
+#define GUS_IO_RD_PRI_QUEUING__GROUP5_QUEUING_COEFFICIENT_MASK                                                0x00038000L
+//GUS_IO_WR_PRI_QUEUING
+#define GUS_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                              0x0
+#define GUS_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                              0x3
+#define GUS_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                              0x6
+#define GUS_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                              0x9
+#define GUS_IO_WR_PRI_QUEUING__GROUP4_QUEUING_COEFFICIENT__SHIFT                                              0xc
+#define GUS_IO_WR_PRI_QUEUING__GROUP5_QUEUING_COEFFICIENT__SHIFT                                              0xf
+#define GUS_IO_WR_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                                0x00000007L
+#define GUS_IO_WR_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                                0x00000038L
+#define GUS_IO_WR_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                                0x000001C0L
+#define GUS_IO_WR_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                                0x00000E00L
+#define GUS_IO_WR_PRI_QUEUING__GROUP4_QUEUING_COEFFICIENT_MASK                                                0x00007000L
+#define GUS_IO_WR_PRI_QUEUING__GROUP5_QUEUING_COEFFICIENT_MASK                                                0x00038000L
+//GUS_IO_RD_PRI_FIXED
+#define GUS_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                  0x0
+#define GUS_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                  0x3
+#define GUS_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                  0x6
+#define GUS_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                  0x9
+#define GUS_IO_RD_PRI_FIXED__GROUP4_FIXED_COEFFICIENT__SHIFT                                                  0xc
+#define GUS_IO_RD_PRI_FIXED__GROUP5_FIXED_COEFFICIENT__SHIFT                                                  0xf
+#define GUS_IO_RD_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                    0x00000007L
+#define GUS_IO_RD_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                    0x00000038L
+#define GUS_IO_RD_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                    0x000001C0L
+#define GUS_IO_RD_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                    0x00000E00L
+#define GUS_IO_RD_PRI_FIXED__GROUP4_FIXED_COEFFICIENT_MASK                                                    0x00007000L
+#define GUS_IO_RD_PRI_FIXED__GROUP5_FIXED_COEFFICIENT_MASK                                                    0x00038000L
+//GUS_IO_WR_PRI_FIXED
+#define GUS_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                  0x0
+#define GUS_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                  0x3
+#define GUS_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                  0x6
+#define GUS_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                  0x9
+#define GUS_IO_WR_PRI_FIXED__GROUP4_FIXED_COEFFICIENT__SHIFT                                                  0xc
+#define GUS_IO_WR_PRI_FIXED__GROUP5_FIXED_COEFFICIENT__SHIFT                                                  0xf
+#define GUS_IO_WR_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                    0x00000007L
+#define GUS_IO_WR_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                    0x00000038L
+#define GUS_IO_WR_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                    0x000001C0L
+#define GUS_IO_WR_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                    0x00000E00L
+#define GUS_IO_WR_PRI_FIXED__GROUP4_FIXED_COEFFICIENT_MASK                                                    0x00007000L
+#define GUS_IO_WR_PRI_FIXED__GROUP5_FIXED_COEFFICIENT_MASK                                                    0x00038000L
+//GUS_IO_RD_PRI_URGENCY_COEFF
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP0_URGENCY_COEFFICIENT__SHIFT                                        0x0
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP1_URGENCY_COEFFICIENT__SHIFT                                        0x3
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP2_URGENCY_COEFFICIENT__SHIFT                                        0x6
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP3_URGENCY_COEFFICIENT__SHIFT                                        0x9
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP4_URGENCY_COEFFICIENT__SHIFT                                        0xc
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP5_URGENCY_COEFFICIENT__SHIFT                                        0xf
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP0_URGENCY_COEFFICIENT_MASK                                          0x00000007L
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP1_URGENCY_COEFFICIENT_MASK                                          0x00000038L
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP2_URGENCY_COEFFICIENT_MASK                                          0x000001C0L
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP3_URGENCY_COEFFICIENT_MASK                                          0x00000E00L
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP4_URGENCY_COEFFICIENT_MASK                                          0x00007000L
+#define GUS_IO_RD_PRI_URGENCY_COEFF__GROUP5_URGENCY_COEFFICIENT_MASK                                          0x00038000L
+//GUS_IO_WR_PRI_URGENCY_COEFF
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP0_URGENCY_COEFFICIENT__SHIFT                                        0x0
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP1_URGENCY_COEFFICIENT__SHIFT                                        0x3
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP2_URGENCY_COEFFICIENT__SHIFT                                        0x6
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP3_URGENCY_COEFFICIENT__SHIFT                                        0x9
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP4_URGENCY_COEFFICIENT__SHIFT                                        0xc
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP5_URGENCY_COEFFICIENT__SHIFT                                        0xf
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP0_URGENCY_COEFFICIENT_MASK                                          0x00000007L
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP1_URGENCY_COEFFICIENT_MASK                                          0x00000038L
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP2_URGENCY_COEFFICIENT_MASK                                          0x000001C0L
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP3_URGENCY_COEFFICIENT_MASK                                          0x00000E00L
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP4_URGENCY_COEFFICIENT_MASK                                          0x00007000L
+#define GUS_IO_WR_PRI_URGENCY_COEFF__GROUP5_URGENCY_COEFFICIENT_MASK                                          0x00038000L
+//GUS_IO_RD_PRI_URGENCY_MODE
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP0_URGENCY_MODE__SHIFT                                                0x0
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP1_URGENCY_MODE__SHIFT                                                0x1
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP2_URGENCY_MODE__SHIFT                                                0x2
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP3_URGENCY_MODE__SHIFT                                                0x3
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP4_URGENCY_MODE__SHIFT                                                0x4
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP5_URGENCY_MODE__SHIFT                                                0x5
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP0_URGENCY_MODE_MASK                                                  0x00000001L
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP1_URGENCY_MODE_MASK                                                  0x00000002L
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP2_URGENCY_MODE_MASK                                                  0x00000004L
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP3_URGENCY_MODE_MASK                                                  0x00000008L
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP4_URGENCY_MODE_MASK                                                  0x00000010L
+#define GUS_IO_RD_PRI_URGENCY_MODE__GROUP5_URGENCY_MODE_MASK                                                  0x00000020L
+//GUS_IO_WR_PRI_URGENCY_MODE
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP0_URGENCY_MODE__SHIFT                                                0x0
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP1_URGENCY_MODE__SHIFT                                                0x1
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP2_URGENCY_MODE__SHIFT                                                0x2
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP3_URGENCY_MODE__SHIFT                                                0x3
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP4_URGENCY_MODE__SHIFT                                                0x4
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP5_URGENCY_MODE__SHIFT                                                0x5
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP0_URGENCY_MODE_MASK                                                  0x00000001L
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP1_URGENCY_MODE_MASK                                                  0x00000002L
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP2_URGENCY_MODE_MASK                                                  0x00000004L
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP3_URGENCY_MODE_MASK                                                  0x00000008L
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP4_URGENCY_MODE_MASK                                                  0x00000010L
+#define GUS_IO_WR_PRI_URGENCY_MODE__GROUP5_URGENCY_MODE_MASK                                                  0x00000020L
+//GUS_IO_RD_PRI_QUANT_PRI1
+#define GUS_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                     0x0
+#define GUS_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                     0x8
+#define GUS_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                     0x10
+#define GUS_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                     0x18
+#define GUS_IO_RD_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_IO_RD_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                       0x0000FF00L
+#define GUS_IO_RD_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                       0x00FF0000L
+#define GUS_IO_RD_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                       0xFF000000L
+//GUS_IO_RD_PRI_QUANT_PRI2
+#define GUS_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                     0x0
+#define GUS_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                     0x8
+#define GUS_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                     0x10
+#define GUS_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                     0x18
+#define GUS_IO_RD_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_IO_RD_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                       0x0000FF00L
+#define GUS_IO_RD_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                       0x00FF0000L
+#define GUS_IO_RD_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                       0xFF000000L
+//GUS_IO_RD_PRI_QUANT_PRI3
+#define GUS_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                     0x0
+#define GUS_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                     0x8
+#define GUS_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                     0x10
+#define GUS_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                     0x18
+#define GUS_IO_RD_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_IO_RD_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                       0x0000FF00L
+#define GUS_IO_RD_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                       0x00FF0000L
+#define GUS_IO_RD_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                       0xFF000000L
+//GUS_IO_RD_PRI_QUANT_PRI4
+#define GUS_IO_RD_PRI_QUANT_PRI4__GROUP0_THRESHOLD__SHIFT                                                     0x0
+#define GUS_IO_RD_PRI_QUANT_PRI4__GROUP1_THRESHOLD__SHIFT                                                     0x8
+#define GUS_IO_RD_PRI_QUANT_PRI4__GROUP2_THRESHOLD__SHIFT                                                     0x10
+#define GUS_IO_RD_PRI_QUANT_PRI4__GROUP3_THRESHOLD__SHIFT                                                     0x18
+#define GUS_IO_RD_PRI_QUANT_PRI4__GROUP0_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_IO_RD_PRI_QUANT_PRI4__GROUP1_THRESHOLD_MASK                                                       0x0000FF00L
+#define GUS_IO_RD_PRI_QUANT_PRI4__GROUP2_THRESHOLD_MASK                                                       0x00FF0000L
+#define GUS_IO_RD_PRI_QUANT_PRI4__GROUP3_THRESHOLD_MASK                                                       0xFF000000L
+//GUS_IO_WR_PRI_QUANT_PRI1
+#define GUS_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                     0x0
+#define GUS_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                     0x8
+#define GUS_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                     0x10
+#define GUS_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                     0x18
+#define GUS_IO_WR_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_IO_WR_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                       0x0000FF00L
+#define GUS_IO_WR_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                       0x00FF0000L
+#define GUS_IO_WR_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                       0xFF000000L
+//GUS_IO_WR_PRI_QUANT_PRI2
+#define GUS_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                     0x0
+#define GUS_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                     0x8
+#define GUS_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                     0x10
+#define GUS_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                     0x18
+#define GUS_IO_WR_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_IO_WR_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                       0x0000FF00L
+#define GUS_IO_WR_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                       0x00FF0000L
+#define GUS_IO_WR_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                       0xFF000000L
+//GUS_IO_WR_PRI_QUANT_PRI3
+#define GUS_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                     0x0
+#define GUS_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                     0x8
+#define GUS_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                     0x10
+#define GUS_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                     0x18
+#define GUS_IO_WR_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_IO_WR_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                       0x0000FF00L
+#define GUS_IO_WR_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                       0x00FF0000L
+#define GUS_IO_WR_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                       0xFF000000L
+//GUS_IO_WR_PRI_QUANT_PRI4
+#define GUS_IO_WR_PRI_QUANT_PRI4__GROUP0_THRESHOLD__SHIFT                                                     0x0
+#define GUS_IO_WR_PRI_QUANT_PRI4__GROUP1_THRESHOLD__SHIFT                                                     0x8
+#define GUS_IO_WR_PRI_QUANT_PRI4__GROUP2_THRESHOLD__SHIFT                                                     0x10
+#define GUS_IO_WR_PRI_QUANT_PRI4__GROUP3_THRESHOLD__SHIFT                                                     0x18
+#define GUS_IO_WR_PRI_QUANT_PRI4__GROUP0_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_IO_WR_PRI_QUANT_PRI4__GROUP1_THRESHOLD_MASK                                                       0x0000FF00L
+#define GUS_IO_WR_PRI_QUANT_PRI4__GROUP2_THRESHOLD_MASK                                                       0x00FF0000L
+#define GUS_IO_WR_PRI_QUANT_PRI4__GROUP3_THRESHOLD_MASK                                                       0xFF000000L
+//GUS_IO_RD_PRI_QUANT1_PRI1
+#define GUS_IO_RD_PRI_QUANT1_PRI1__GROUP4_THRESHOLD__SHIFT                                                    0x0
+#define GUS_IO_RD_PRI_QUANT1_PRI1__GROUP5_THRESHOLD__SHIFT                                                    0x8
+#define GUS_IO_RD_PRI_QUANT1_PRI1__GROUP4_THRESHOLD_MASK                                                      0x000000FFL
+#define GUS_IO_RD_PRI_QUANT1_PRI1__GROUP5_THRESHOLD_MASK                                                      0x0000FF00L
+//GUS_IO_RD_PRI_QUANT1_PRI2
+#define GUS_IO_RD_PRI_QUANT1_PRI2__GROUP4_THRESHOLD__SHIFT                                                    0x0
+#define GUS_IO_RD_PRI_QUANT1_PRI2__GROUP5_THRESHOLD__SHIFT                                                    0x8
+#define GUS_IO_RD_PRI_QUANT1_PRI2__GROUP4_THRESHOLD_MASK                                                      0x000000FFL
+#define GUS_IO_RD_PRI_QUANT1_PRI2__GROUP5_THRESHOLD_MASK                                                      0x0000FF00L
+//GUS_IO_RD_PRI_QUANT1_PRI3
+#define GUS_IO_RD_PRI_QUANT1_PRI3__GROUP4_THRESHOLD__SHIFT                                                    0x0
+#define GUS_IO_RD_PRI_QUANT1_PRI3__GROUP5_THRESHOLD__SHIFT                                                    0x8
+#define GUS_IO_RD_PRI_QUANT1_PRI3__GROUP4_THRESHOLD_MASK                                                      0x000000FFL
+#define GUS_IO_RD_PRI_QUANT1_PRI3__GROUP5_THRESHOLD_MASK                                                      0x0000FF00L
+//GUS_IO_RD_PRI_QUANT1_PRI4
+#define GUS_IO_RD_PRI_QUANT1_PRI4__GROUP4_THRESHOLD__SHIFT                                                    0x0
+#define GUS_IO_RD_PRI_QUANT1_PRI4__GROUP5_THRESHOLD__SHIFT                                                    0x8
+#define GUS_IO_RD_PRI_QUANT1_PRI4__GROUP4_THRESHOLD_MASK                                                      0x000000FFL
+#define GUS_IO_RD_PRI_QUANT1_PRI4__GROUP5_THRESHOLD_MASK                                                      0x0000FF00L
+//GUS_IO_WR_PRI_QUANT1_PRI1
+#define GUS_IO_WR_PRI_QUANT1_PRI1__GROUP4_THRESHOLD__SHIFT                                                    0x0
+#define GUS_IO_WR_PRI_QUANT1_PRI1__GROUP5_THRESHOLD__SHIFT                                                    0x8
+#define GUS_IO_WR_PRI_QUANT1_PRI1__GROUP4_THRESHOLD_MASK                                                      0x000000FFL
+#define GUS_IO_WR_PRI_QUANT1_PRI1__GROUP5_THRESHOLD_MASK                                                      0x0000FF00L
+//GUS_IO_WR_PRI_QUANT1_PRI2
+#define GUS_IO_WR_PRI_QUANT1_PRI2__GROUP4_THRESHOLD__SHIFT                                                    0x0
+#define GUS_IO_WR_PRI_QUANT1_PRI2__GROUP5_THRESHOLD__SHIFT                                                    0x8
+#define GUS_IO_WR_PRI_QUANT1_PRI2__GROUP4_THRESHOLD_MASK                                                      0x000000FFL
+#define GUS_IO_WR_PRI_QUANT1_PRI2__GROUP5_THRESHOLD_MASK                                                      0x0000FF00L
+//GUS_IO_WR_PRI_QUANT1_PRI3
+#define GUS_IO_WR_PRI_QUANT1_PRI3__GROUP4_THRESHOLD__SHIFT                                                    0x0
+#define GUS_IO_WR_PRI_QUANT1_PRI3__GROUP5_THRESHOLD__SHIFT                                                    0x8
+#define GUS_IO_WR_PRI_QUANT1_PRI3__GROUP4_THRESHOLD_MASK                                                      0x000000FFL
+#define GUS_IO_WR_PRI_QUANT1_PRI3__GROUP5_THRESHOLD_MASK                                                      0x0000FF00L
+//GUS_IO_WR_PRI_QUANT1_PRI4
+#define GUS_IO_WR_PRI_QUANT1_PRI4__GROUP4_THRESHOLD__SHIFT                                                    0x0
+#define GUS_IO_WR_PRI_QUANT1_PRI4__GROUP5_THRESHOLD__SHIFT                                                    0x8
+#define GUS_IO_WR_PRI_QUANT1_PRI4__GROUP4_THRESHOLD_MASK                                                      0x000000FFL
+#define GUS_IO_WR_PRI_QUANT1_PRI4__GROUP5_THRESHOLD_MASK                                                      0x0000FF00L
+//GUS_DRAM_COMBINE_FLUSH
+#define GUS_DRAM_COMBINE_FLUSH__GROUP0_TIMER__SHIFT                                                           0x0
+#define GUS_DRAM_COMBINE_FLUSH__GROUP1_TIMER__SHIFT                                                           0x4
+#define GUS_DRAM_COMBINE_FLUSH__GROUP2_TIMER__SHIFT                                                           0x8
+#define GUS_DRAM_COMBINE_FLUSH__GROUP3_TIMER__SHIFT                                                           0xc
+#define GUS_DRAM_COMBINE_FLUSH__GROUP4_TIMER__SHIFT                                                           0x10
+#define GUS_DRAM_COMBINE_FLUSH__GROUP5_TIMER__SHIFT                                                           0x14
+#define GUS_DRAM_COMBINE_FLUSH__GROUP0_TIMER_MASK                                                             0x0000000FL
+#define GUS_DRAM_COMBINE_FLUSH__GROUP1_TIMER_MASK                                                             0x000000F0L
+#define GUS_DRAM_COMBINE_FLUSH__GROUP2_TIMER_MASK                                                             0x00000F00L
+#define GUS_DRAM_COMBINE_FLUSH__GROUP3_TIMER_MASK                                                             0x0000F000L
+#define GUS_DRAM_COMBINE_FLUSH__GROUP4_TIMER_MASK                                                             0x000F0000L
+#define GUS_DRAM_COMBINE_FLUSH__GROUP5_TIMER_MASK                                                             0x00F00000L
+//GUS_DRAM_COMBINE_RD_WR_EN
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP0_TIMER__SHIFT                                                        0x0
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP1_TIMER__SHIFT                                                        0x2
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP2_TIMER__SHIFT                                                        0x4
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP3_TIMER__SHIFT                                                        0x6
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP4_TIMER__SHIFT                                                        0x8
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP5_TIMER__SHIFT                                                        0xa
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP0_TIMER_MASK                                                          0x00000003L
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP1_TIMER_MASK                                                          0x0000000CL
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP2_TIMER_MASK                                                          0x00000030L
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP3_TIMER_MASK                                                          0x000000C0L
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP4_TIMER_MASK                                                          0x00000300L
+#define GUS_DRAM_COMBINE_RD_WR_EN__GROUP5_TIMER_MASK                                                          0x00000C00L
+//GUS_DRAM_PRI_AGE_RATE
+#define GUS_DRAM_PRI_AGE_RATE__GROUP0_AGING_RATE__SHIFT                                                       0x0
+#define GUS_DRAM_PRI_AGE_RATE__GROUP1_AGING_RATE__SHIFT                                                       0x3
+#define GUS_DRAM_PRI_AGE_RATE__GROUP2_AGING_RATE__SHIFT                                                       0x6
+#define GUS_DRAM_PRI_AGE_RATE__GROUP3_AGING_RATE__SHIFT                                                       0x9
+#define GUS_DRAM_PRI_AGE_RATE__GROUP4_AGING_RATE__SHIFT                                                       0xc
+#define GUS_DRAM_PRI_AGE_RATE__GROUP5_AGING_RATE__SHIFT                                                       0xf
+#define GUS_DRAM_PRI_AGE_RATE__GROUP0_AGING_RATE_MASK                                                         0x00000007L
+#define GUS_DRAM_PRI_AGE_RATE__GROUP1_AGING_RATE_MASK                                                         0x00000038L
+#define GUS_DRAM_PRI_AGE_RATE__GROUP2_AGING_RATE_MASK                                                         0x000001C0L
+#define GUS_DRAM_PRI_AGE_RATE__GROUP3_AGING_RATE_MASK                                                         0x00000E00L
+#define GUS_DRAM_PRI_AGE_RATE__GROUP4_AGING_RATE_MASK                                                         0x00007000L
+#define GUS_DRAM_PRI_AGE_RATE__GROUP5_AGING_RATE_MASK                                                         0x00038000L
+//GUS_DRAM_PRI_AGE_COEFF
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP0_AGE_COEFFICIENT__SHIFT                                                 0x0
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP1_AGE_COEFFICIENT__SHIFT                                                 0x3
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP2_AGE_COEFFICIENT__SHIFT                                                 0x6
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP3_AGE_COEFFICIENT__SHIFT                                                 0x9
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP4_AGE_COEFFICIENT__SHIFT                                                 0xc
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP5_AGE_COEFFICIENT__SHIFT                                                 0xf
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP0_AGE_COEFFICIENT_MASK                                                   0x00000007L
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP1_AGE_COEFFICIENT_MASK                                                   0x00000038L
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP2_AGE_COEFFICIENT_MASK                                                   0x000001C0L
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP3_AGE_COEFFICIENT_MASK                                                   0x00000E00L
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP4_AGE_COEFFICIENT_MASK                                                   0x00007000L
+#define GUS_DRAM_PRI_AGE_COEFF__GROUP5_AGE_COEFFICIENT_MASK                                                   0x00038000L
+//GUS_DRAM_PRI_QUEUING
+#define GUS_DRAM_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT__SHIFT                                               0x0
+#define GUS_DRAM_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT__SHIFT                                               0x3
+#define GUS_DRAM_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT__SHIFT                                               0x6
+#define GUS_DRAM_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT__SHIFT                                               0x9
+#define GUS_DRAM_PRI_QUEUING__GROUP4_QUEUING_COEFFICIENT__SHIFT                                               0xc
+#define GUS_DRAM_PRI_QUEUING__GROUP5_QUEUING_COEFFICIENT__SHIFT                                               0xf
+#define GUS_DRAM_PRI_QUEUING__GROUP0_QUEUING_COEFFICIENT_MASK                                                 0x00000007L
+#define GUS_DRAM_PRI_QUEUING__GROUP1_QUEUING_COEFFICIENT_MASK                                                 0x00000038L
+#define GUS_DRAM_PRI_QUEUING__GROUP2_QUEUING_COEFFICIENT_MASK                                                 0x000001C0L
+#define GUS_DRAM_PRI_QUEUING__GROUP3_QUEUING_COEFFICIENT_MASK                                                 0x00000E00L
+#define GUS_DRAM_PRI_QUEUING__GROUP4_QUEUING_COEFFICIENT_MASK                                                 0x00007000L
+#define GUS_DRAM_PRI_QUEUING__GROUP5_QUEUING_COEFFICIENT_MASK                                                 0x00038000L
+//GUS_DRAM_PRI_FIXED
+#define GUS_DRAM_PRI_FIXED__GROUP0_FIXED_COEFFICIENT__SHIFT                                                   0x0
+#define GUS_DRAM_PRI_FIXED__GROUP1_FIXED_COEFFICIENT__SHIFT                                                   0x3
+#define GUS_DRAM_PRI_FIXED__GROUP2_FIXED_COEFFICIENT__SHIFT                                                   0x6
+#define GUS_DRAM_PRI_FIXED__GROUP3_FIXED_COEFFICIENT__SHIFT                                                   0x9
+#define GUS_DRAM_PRI_FIXED__GROUP4_FIXED_COEFFICIENT__SHIFT                                                   0xc
+#define GUS_DRAM_PRI_FIXED__GROUP5_FIXED_COEFFICIENT__SHIFT                                                   0xf
+#define GUS_DRAM_PRI_FIXED__GROUP0_FIXED_COEFFICIENT_MASK                                                     0x00000007L
+#define GUS_DRAM_PRI_FIXED__GROUP1_FIXED_COEFFICIENT_MASK                                                     0x00000038L
+#define GUS_DRAM_PRI_FIXED__GROUP2_FIXED_COEFFICIENT_MASK                                                     0x000001C0L
+#define GUS_DRAM_PRI_FIXED__GROUP3_FIXED_COEFFICIENT_MASK                                                     0x00000E00L
+#define GUS_DRAM_PRI_FIXED__GROUP4_FIXED_COEFFICIENT_MASK                                                     0x00007000L
+#define GUS_DRAM_PRI_FIXED__GROUP5_FIXED_COEFFICIENT_MASK                                                     0x00038000L
+//GUS_DRAM_PRI_URGENCY_COEFF
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP0_URGENCY_COEFFICIENT__SHIFT                                         0x0
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP1_URGENCY_COEFFICIENT__SHIFT                                         0x3
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP2_URGENCY_COEFFICIENT__SHIFT                                         0x6
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP3_URGENCY_COEFFICIENT__SHIFT                                         0x9
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP4_URGENCY_COEFFICIENT__SHIFT                                         0xc
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP5_URGENCY_COEFFICIENT__SHIFT                                         0xf
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP0_URGENCY_COEFFICIENT_MASK                                           0x00000007L
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP1_URGENCY_COEFFICIENT_MASK                                           0x00000038L
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP2_URGENCY_COEFFICIENT_MASK                                           0x000001C0L
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP3_URGENCY_COEFFICIENT_MASK                                           0x00000E00L
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP4_URGENCY_COEFFICIENT_MASK                                           0x00007000L
+#define GUS_DRAM_PRI_URGENCY_COEFF__GROUP5_URGENCY_COEFFICIENT_MASK                                           0x00038000L
+//GUS_DRAM_PRI_URGENCY_MODE
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP0_URGENCY_MODE__SHIFT                                                 0x0
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP1_URGENCY_MODE__SHIFT                                                 0x1
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP2_URGENCY_MODE__SHIFT                                                 0x2
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP3_URGENCY_MODE__SHIFT                                                 0x3
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP4_URGENCY_MODE__SHIFT                                                 0x4
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP5_URGENCY_MODE__SHIFT                                                 0x5
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP0_URGENCY_MODE_MASK                                                   0x00000001L
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP1_URGENCY_MODE_MASK                                                   0x00000002L
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP2_URGENCY_MODE_MASK                                                   0x00000004L
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP3_URGENCY_MODE_MASK                                                   0x00000008L
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP4_URGENCY_MODE_MASK                                                   0x00000010L
+#define GUS_DRAM_PRI_URGENCY_MODE__GROUP5_URGENCY_MODE_MASK                                                   0x00000020L
+//GUS_DRAM_PRI_QUANT_PRI1
+#define GUS_DRAM_PRI_QUANT_PRI1__GROUP0_THRESHOLD__SHIFT                                                      0x0
+#define GUS_DRAM_PRI_QUANT_PRI1__GROUP1_THRESHOLD__SHIFT                                                      0x8
+#define GUS_DRAM_PRI_QUANT_PRI1__GROUP2_THRESHOLD__SHIFT                                                      0x10
+#define GUS_DRAM_PRI_QUANT_PRI1__GROUP3_THRESHOLD__SHIFT                                                      0x18
+#define GUS_DRAM_PRI_QUANT_PRI1__GROUP0_THRESHOLD_MASK                                                        0x000000FFL
+#define GUS_DRAM_PRI_QUANT_PRI1__GROUP1_THRESHOLD_MASK                                                        0x0000FF00L
+#define GUS_DRAM_PRI_QUANT_PRI1__GROUP2_THRESHOLD_MASK                                                        0x00FF0000L
+#define GUS_DRAM_PRI_QUANT_PRI1__GROUP3_THRESHOLD_MASK                                                        0xFF000000L
+//GUS_DRAM_PRI_QUANT_PRI2
+#define GUS_DRAM_PRI_QUANT_PRI2__GROUP0_THRESHOLD__SHIFT                                                      0x0
+#define GUS_DRAM_PRI_QUANT_PRI2__GROUP1_THRESHOLD__SHIFT                                                      0x8
+#define GUS_DRAM_PRI_QUANT_PRI2__GROUP2_THRESHOLD__SHIFT                                                      0x10
+#define GUS_DRAM_PRI_QUANT_PRI2__GROUP3_THRESHOLD__SHIFT                                                      0x18
+#define GUS_DRAM_PRI_QUANT_PRI2__GROUP0_THRESHOLD_MASK                                                        0x000000FFL
+#define GUS_DRAM_PRI_QUANT_PRI2__GROUP1_THRESHOLD_MASK                                                        0x0000FF00L
+#define GUS_DRAM_PRI_QUANT_PRI2__GROUP2_THRESHOLD_MASK                                                        0x00FF0000L
+#define GUS_DRAM_PRI_QUANT_PRI2__GROUP3_THRESHOLD_MASK                                                        0xFF000000L
+//GUS_DRAM_PRI_QUANT_PRI3
+#define GUS_DRAM_PRI_QUANT_PRI3__GROUP0_THRESHOLD__SHIFT                                                      0x0
+#define GUS_DRAM_PRI_QUANT_PRI3__GROUP1_THRESHOLD__SHIFT                                                      0x8
+#define GUS_DRAM_PRI_QUANT_PRI3__GROUP2_THRESHOLD__SHIFT                                                      0x10
+#define GUS_DRAM_PRI_QUANT_PRI3__GROUP3_THRESHOLD__SHIFT                                                      0x18
+#define GUS_DRAM_PRI_QUANT_PRI3__GROUP0_THRESHOLD_MASK                                                        0x000000FFL
+#define GUS_DRAM_PRI_QUANT_PRI3__GROUP1_THRESHOLD_MASK                                                        0x0000FF00L
+#define GUS_DRAM_PRI_QUANT_PRI3__GROUP2_THRESHOLD_MASK                                                        0x00FF0000L
+#define GUS_DRAM_PRI_QUANT_PRI3__GROUP3_THRESHOLD_MASK                                                        0xFF000000L
+//GUS_DRAM_PRI_QUANT_PRI4
+#define GUS_DRAM_PRI_QUANT_PRI4__GROUP0_THRESHOLD__SHIFT                                                      0x0
+#define GUS_DRAM_PRI_QUANT_PRI4__GROUP1_THRESHOLD__SHIFT                                                      0x8
+#define GUS_DRAM_PRI_QUANT_PRI4__GROUP2_THRESHOLD__SHIFT                                                      0x10
+#define GUS_DRAM_PRI_QUANT_PRI4__GROUP3_THRESHOLD__SHIFT                                                      0x18
+#define GUS_DRAM_PRI_QUANT_PRI4__GROUP0_THRESHOLD_MASK                                                        0x000000FFL
+#define GUS_DRAM_PRI_QUANT_PRI4__GROUP1_THRESHOLD_MASK                                                        0x0000FF00L
+#define GUS_DRAM_PRI_QUANT_PRI4__GROUP2_THRESHOLD_MASK                                                        0x00FF0000L
+#define GUS_DRAM_PRI_QUANT_PRI4__GROUP3_THRESHOLD_MASK                                                        0xFF000000L
+//GUS_DRAM_PRI_QUANT_PRI5
+#define GUS_DRAM_PRI_QUANT_PRI5__GROUP0_THRESHOLD__SHIFT                                                      0x0
+#define GUS_DRAM_PRI_QUANT_PRI5__GROUP1_THRESHOLD__SHIFT                                                      0x8
+#define GUS_DRAM_PRI_QUANT_PRI5__GROUP2_THRESHOLD__SHIFT                                                      0x10
+#define GUS_DRAM_PRI_QUANT_PRI5__GROUP3_THRESHOLD__SHIFT                                                      0x18
+#define GUS_DRAM_PRI_QUANT_PRI5__GROUP0_THRESHOLD_MASK                                                        0x000000FFL
+#define GUS_DRAM_PRI_QUANT_PRI5__GROUP1_THRESHOLD_MASK                                                        0x0000FF00L
+#define GUS_DRAM_PRI_QUANT_PRI5__GROUP2_THRESHOLD_MASK                                                        0x00FF0000L
+#define GUS_DRAM_PRI_QUANT_PRI5__GROUP3_THRESHOLD_MASK                                                        0xFF000000L
+//GUS_DRAM_PRI_QUANT1_PRI1
+#define GUS_DRAM_PRI_QUANT1_PRI1__GROUP4_THRESHOLD__SHIFT                                                     0x0
+#define GUS_DRAM_PRI_QUANT1_PRI1__GROUP5_THRESHOLD__SHIFT                                                     0x8
+#define GUS_DRAM_PRI_QUANT1_PRI1__GROUP4_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_DRAM_PRI_QUANT1_PRI1__GROUP5_THRESHOLD_MASK                                                       0x0000FF00L
+//GUS_DRAM_PRI_QUANT1_PRI2
+#define GUS_DRAM_PRI_QUANT1_PRI2__GROUP4_THRESHOLD__SHIFT                                                     0x0
+#define GUS_DRAM_PRI_QUANT1_PRI2__GROUP5_THRESHOLD__SHIFT                                                     0x8
+#define GUS_DRAM_PRI_QUANT1_PRI2__GROUP4_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_DRAM_PRI_QUANT1_PRI2__GROUP5_THRESHOLD_MASK                                                       0x0000FF00L
+//GUS_DRAM_PRI_QUANT1_PRI3
+#define GUS_DRAM_PRI_QUANT1_PRI3__GROUP4_THRESHOLD__SHIFT                                                     0x0
+#define GUS_DRAM_PRI_QUANT1_PRI3__GROUP5_THRESHOLD__SHIFT                                                     0x8
+#define GUS_DRAM_PRI_QUANT1_PRI3__GROUP4_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_DRAM_PRI_QUANT1_PRI3__GROUP5_THRESHOLD_MASK                                                       0x0000FF00L
+//GUS_DRAM_PRI_QUANT1_PRI4
+#define GUS_DRAM_PRI_QUANT1_PRI4__GROUP4_THRESHOLD__SHIFT                                                     0x0
+#define GUS_DRAM_PRI_QUANT1_PRI4__GROUP5_THRESHOLD__SHIFT                                                     0x8
+#define GUS_DRAM_PRI_QUANT1_PRI4__GROUP4_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_DRAM_PRI_QUANT1_PRI4__GROUP5_THRESHOLD_MASK                                                       0x0000FF00L
+//GUS_DRAM_PRI_QUANT1_PRI5
+#define GUS_DRAM_PRI_QUANT1_PRI5__GROUP4_THRESHOLD__SHIFT                                                     0x0
+#define GUS_DRAM_PRI_QUANT1_PRI5__GROUP5_THRESHOLD__SHIFT                                                     0x8
+#define GUS_DRAM_PRI_QUANT1_PRI5__GROUP4_THRESHOLD_MASK                                                       0x000000FFL
+#define GUS_DRAM_PRI_QUANT1_PRI5__GROUP5_THRESHOLD_MASK                                                       0x0000FF00L
+//GUS_IO_GROUP_BURST
+#define GUS_IO_GROUP_BURST__RD_LIMIT_LO__SHIFT                                                                0x0
+#define GUS_IO_GROUP_BURST__RD_LIMIT_HI__SHIFT                                                                0x8
+#define GUS_IO_GROUP_BURST__WR_LIMIT_LO__SHIFT                                                                0x10
+#define GUS_IO_GROUP_BURST__WR_LIMIT_HI__SHIFT                                                                0x18
+#define GUS_IO_GROUP_BURST__RD_LIMIT_LO_MASK                                                                  0x000000FFL
+#define GUS_IO_GROUP_BURST__RD_LIMIT_HI_MASK                                                                  0x0000FF00L
+#define GUS_IO_GROUP_BURST__WR_LIMIT_LO_MASK                                                                  0x00FF0000L
+#define GUS_IO_GROUP_BURST__WR_LIMIT_HI_MASK                                                                  0xFF000000L
+//GUS_DRAM_GROUP_BURST
+#define GUS_DRAM_GROUP_BURST__DRAM_LIMIT_LO__SHIFT                                                            0x0
+#define GUS_DRAM_GROUP_BURST__DRAM_LIMIT_HI__SHIFT                                                            0x8
+#define GUS_DRAM_GROUP_BURST__DRAM_LIMIT_LO_MASK                                                              0x000000FFL
+#define GUS_DRAM_GROUP_BURST__DRAM_LIMIT_HI_MASK                                                              0x0000FF00L
+//GUS_SDP_ARB_FINAL
+#define GUS_SDP_ARB_FINAL__HI_DRAM_BURST_LIMIT__SHIFT                                                         0x0
+#define GUS_SDP_ARB_FINAL__DRAM_BURST_LIMIT__SHIFT                                                            0x5
+#define GUS_SDP_ARB_FINAL__IO_BURST_LIMIT__SHIFT                                                              0xa
+#define GUS_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER__SHIFT                                                      0xf
+#define GUS_SDP_ARB_FINAL__ERREVENT_ON_ERROR__SHIFT                                                           0x11
+#define GUS_SDP_ARB_FINAL__HALTREQ_ON_ERROR__SHIFT                                                            0x12
+#define GUS_SDP_ARB_FINAL__HI_DRAM_BURST_LIMIT_MASK                                                           0x0000001FL
+#define GUS_SDP_ARB_FINAL__DRAM_BURST_LIMIT_MASK                                                              0x000003E0L
+#define GUS_SDP_ARB_FINAL__IO_BURST_LIMIT_MASK                                                                0x00007C00L
+#define GUS_SDP_ARB_FINAL__BURST_LIMIT_MULTIPLIER_MASK                                                        0x00018000L
+#define GUS_SDP_ARB_FINAL__ERREVENT_ON_ERROR_MASK                                                             0x00020000L
+#define GUS_SDP_ARB_FINAL__HALTREQ_ON_ERROR_MASK                                                              0x00040000L
+//GUS_SDP_QOS_VC_PRIORITY
+#define GUS_SDP_QOS_VC_PRIORITY__VC2_IORD__SHIFT                                                              0x0
+#define GUS_SDP_QOS_VC_PRIORITY__VC3_IOWR__SHIFT                                                              0x4
+#define GUS_SDP_QOS_VC_PRIORITY__VC4_DRAM__SHIFT                                                              0x8
+#define GUS_SDP_QOS_VC_PRIORITY__VC4_HI_DRAM__SHIFT                                                           0xc
+#define GUS_SDP_QOS_VC_PRIORITY__VC2_IORD_MASK                                                                0x0000000FL
+#define GUS_SDP_QOS_VC_PRIORITY__VC3_IOWR_MASK                                                                0x000000F0L
+#define GUS_SDP_QOS_VC_PRIORITY__VC4_DRAM_MASK                                                                0x00000F00L
+#define GUS_SDP_QOS_VC_PRIORITY__VC4_HI_DRAM_MASK                                                             0x0000F000L
+//GUS_SDP_CREDITS
+#define GUS_SDP_CREDITS__TAG_LIMIT__SHIFT                                                                     0x0
+#define GUS_SDP_CREDITS__WR_RESP_CREDITS__SHIFT                                                               0x8
+#define GUS_SDP_CREDITS__RD_RESP_CREDITS__SHIFT                                                               0x10
+#define GUS_SDP_CREDITS__TAG_LIMIT_MASK                                                                       0x000000FFL
+#define GUS_SDP_CREDITS__WR_RESP_CREDITS_MASK                                                                 0x00007F00L
+#define GUS_SDP_CREDITS__RD_RESP_CREDITS_MASK                                                                 0x007F0000L
+//GUS_SDP_TAG_RESERVE0
+#define GUS_SDP_TAG_RESERVE0__VC0__SHIFT                                                                      0x0
+#define GUS_SDP_TAG_RESERVE0__VC1__SHIFT                                                                      0x8
+#define GUS_SDP_TAG_RESERVE0__VC2__SHIFT                                                                      0x10
+#define GUS_SDP_TAG_RESERVE0__VC3__SHIFT                                                                      0x18
+#define GUS_SDP_TAG_RESERVE0__VC0_MASK                                                                        0x000000FFL
+#define GUS_SDP_TAG_RESERVE0__VC1_MASK                                                                        0x0000FF00L
+#define GUS_SDP_TAG_RESERVE0__VC2_MASK                                                                        0x00FF0000L
+#define GUS_SDP_TAG_RESERVE0__VC3_MASK                                                                        0xFF000000L
+//GUS_SDP_TAG_RESERVE1
+#define GUS_SDP_TAG_RESERVE1__VC4__SHIFT                                                                      0x0
+#define GUS_SDP_TAG_RESERVE1__VC5__SHIFT                                                                      0x8
+#define GUS_SDP_TAG_RESERVE1__VC6__SHIFT                                                                      0x10
+#define GUS_SDP_TAG_RESERVE1__VC7__SHIFT                                                                      0x18
+#define GUS_SDP_TAG_RESERVE1__VC4_MASK                                                                        0x000000FFL
+#define GUS_SDP_TAG_RESERVE1__VC5_MASK                                                                        0x0000FF00L
+#define GUS_SDP_TAG_RESERVE1__VC6_MASK                                                                        0x00FF0000L
+#define GUS_SDP_TAG_RESERVE1__VC7_MASK                                                                        0xFF000000L
+//GUS_SDP_VCC_RESERVE0
+#define GUS_SDP_VCC_RESERVE0__VC0_CREDITS__SHIFT                                                              0x0
+#define GUS_SDP_VCC_RESERVE0__VC1_CREDITS__SHIFT                                                              0x6
+#define GUS_SDP_VCC_RESERVE0__VC2_CREDITS__SHIFT                                                              0xc
+#define GUS_SDP_VCC_RESERVE0__VC3_CREDITS__SHIFT                                                              0x12
+#define GUS_SDP_VCC_RESERVE0__VC4_CREDITS__SHIFT                                                              0x18
+#define GUS_SDP_VCC_RESERVE0__VC0_CREDITS_MASK                                                                0x0000003FL
+#define GUS_SDP_VCC_RESERVE0__VC1_CREDITS_MASK                                                                0x00000FC0L
+#define GUS_SDP_VCC_RESERVE0__VC2_CREDITS_MASK                                                                0x0003F000L
+#define GUS_SDP_VCC_RESERVE0__VC3_CREDITS_MASK                                                                0x00FC0000L
+#define GUS_SDP_VCC_RESERVE0__VC4_CREDITS_MASK                                                                0x3F000000L
+//GUS_SDP_VCC_RESERVE1
+#define GUS_SDP_VCC_RESERVE1__VC5_CREDITS__SHIFT                                                              0x0
+#define GUS_SDP_VCC_RESERVE1__VC6_CREDITS__SHIFT                                                              0x6
+#define GUS_SDP_VCC_RESERVE1__VC7_CREDITS__SHIFT                                                              0xc
+#define GUS_SDP_VCC_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                          0x1f
+#define GUS_SDP_VCC_RESERVE1__VC5_CREDITS_MASK                                                                0x0000003FL
+#define GUS_SDP_VCC_RESERVE1__VC6_CREDITS_MASK                                                                0x00000FC0L
+#define GUS_SDP_VCC_RESERVE1__VC7_CREDITS_MASK                                                                0x0003F000L
+#define GUS_SDP_VCC_RESERVE1__DISTRIBUTE_POOL_MASK                                                            0x80000000L
+//GUS_SDP_VCD_RESERVE0
+#define GUS_SDP_VCD_RESERVE0__VC0_CREDITS__SHIFT                                                              0x0
+#define GUS_SDP_VCD_RESERVE0__VC1_CREDITS__SHIFT                                                              0x6
+#define GUS_SDP_VCD_RESERVE0__VC2_CREDITS__SHIFT                                                              0xc
+#define GUS_SDP_VCD_RESERVE0__VC3_CREDITS__SHIFT                                                              0x12
+#define GUS_SDP_VCD_RESERVE0__VC4_CREDITS__SHIFT                                                              0x18
+#define GUS_SDP_VCD_RESERVE0__VC0_CREDITS_MASK                                                                0x0000003FL
+#define GUS_SDP_VCD_RESERVE0__VC1_CREDITS_MASK                                                                0x00000FC0L
+#define GUS_SDP_VCD_RESERVE0__VC2_CREDITS_MASK                                                                0x0003F000L
+#define GUS_SDP_VCD_RESERVE0__VC3_CREDITS_MASK                                                                0x00FC0000L
+#define GUS_SDP_VCD_RESERVE0__VC4_CREDITS_MASK                                                                0x3F000000L
+//GUS_SDP_VCD_RESERVE1
+#define GUS_SDP_VCD_RESERVE1__VC5_CREDITS__SHIFT                                                              0x0
+#define GUS_SDP_VCD_RESERVE1__VC6_CREDITS__SHIFT                                                              0x6
+#define GUS_SDP_VCD_RESERVE1__VC7_CREDITS__SHIFT                                                              0xc
+#define GUS_SDP_VCD_RESERVE1__DISTRIBUTE_POOL__SHIFT                                                          0x1f
+#define GUS_SDP_VCD_RESERVE1__VC5_CREDITS_MASK                                                                0x0000003FL
+#define GUS_SDP_VCD_RESERVE1__VC6_CREDITS_MASK                                                                0x00000FC0L
+#define GUS_SDP_VCD_RESERVE1__VC7_CREDITS_MASK                                                                0x0003F000L
+#define GUS_SDP_VCD_RESERVE1__DISTRIBUTE_POOL_MASK                                                            0x80000000L
+//GUS_SDP_REQ_CNTL
+#define GUS_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ__SHIFT                                                    0x0
+#define GUS_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE__SHIFT                                                   0x1
+#define GUS_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC__SHIFT                                                  0x2
+#define GUS_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM__SHIFT                                                      0x3
+#define GUS_SDP_REQ_CNTL__INNER_DOMAIN_MODE__SHIFT                                                            0x4
+#define GUS_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_READ_MASK                                                      0x00000001L
+#define GUS_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_WRITE_MASK                                                     0x00000002L
+#define GUS_SDP_REQ_CNTL__REQ_PASS_PW_OVERRIDE_ATOMIC_MASK                                                    0x00000004L
+#define GUS_SDP_REQ_CNTL__REQ_CHAIN_OVERRIDE_DRAM_MASK                                                        0x00000008L
+#define GUS_SDP_REQ_CNTL__INNER_DOMAIN_MODE_MASK                                                              0x00000010L
+//GUS_MISC
+#define GUS_MISC__RELATIVE_PRI_IN_DRAM_ARB__SHIFT                                                             0x0
+#define GUS_MISC__RELATIVE_PRI_IN_IO_RD_ARB__SHIFT                                                            0x1
+#define GUS_MISC__RELATIVE_PRI_IN_IO_WR_ARB__SHIFT                                                            0x2
+#define GUS_MISC__EARLY_SDP_ORIGDATA__SHIFT                                                                   0x3
+#define GUS_MISC__LINKMGR_DYNAMIC_MODE__SHIFT                                                                 0x4
+#define GUS_MISC__LINKMGR_HALT_THRESHOLD__SHIFT                                                               0x6
+#define GUS_MISC__LINKMGR_RECONNECT_DELAY__SHIFT                                                              0x8
+#define GUS_MISC__LINKMGR_IDLE_THRESHOLD__SHIFT                                                               0xa
+#define GUS_MISC__SEND0_IOWR_ONLY__SHIFT                                                                      0xf
+#define GUS_MISC__RELATIVE_PRI_IN_DRAM_ARB_MASK                                                               0x00000001L
+#define GUS_MISC__RELATIVE_PRI_IN_IO_RD_ARB_MASK                                                              0x00000002L
+#define GUS_MISC__RELATIVE_PRI_IN_IO_WR_ARB_MASK                                                              0x00000004L
+#define GUS_MISC__EARLY_SDP_ORIGDATA_MASK                                                                     0x00000008L
+#define GUS_MISC__LINKMGR_DYNAMIC_MODE_MASK                                                                   0x00000030L
+#define GUS_MISC__LINKMGR_HALT_THRESHOLD_MASK                                                                 0x000000C0L
+#define GUS_MISC__LINKMGR_RECONNECT_DELAY_MASK                                                                0x00000300L
+#define GUS_MISC__LINKMGR_IDLE_THRESHOLD_MASK                                                                 0x00007C00L
+#define GUS_MISC__SEND0_IOWR_ONLY_MASK                                                                        0x00008000L
+//GUS_LATENCY_SAMPLING
+#define GUS_LATENCY_SAMPLING__SAMPLER0_DRAM__SHIFT                                                            0x0
+#define GUS_LATENCY_SAMPLING__SAMPLER1_DRAM__SHIFT                                                            0x1
+#define GUS_LATENCY_SAMPLING__SAMPLER0_IO__SHIFT                                                              0x2
+#define GUS_LATENCY_SAMPLING__SAMPLER1_IO__SHIFT                                                              0x3
+#define GUS_LATENCY_SAMPLING__SAMPLER0_READ__SHIFT                                                            0x4
+#define GUS_LATENCY_SAMPLING__SAMPLER1_READ__SHIFT                                                            0x5
+#define GUS_LATENCY_SAMPLING__SAMPLER0_WRITE__SHIFT                                                           0x6
+#define GUS_LATENCY_SAMPLING__SAMPLER1_WRITE__SHIFT                                                           0x7
+#define GUS_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET__SHIFT                                                      0x8
+#define GUS_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET__SHIFT                                                      0x9
+#define GUS_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET__SHIFT                                                    0xa
+#define GUS_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET__SHIFT                                                    0xb
+#define GUS_LATENCY_SAMPLING__SAMPLER0_VC__SHIFT                                                              0xc
+#define GUS_LATENCY_SAMPLING__SAMPLER1_VC__SHIFT                                                              0x14
+#define GUS_LATENCY_SAMPLING__SAMPLER0_DRAM_MASK                                                              0x00000001L
+#define GUS_LATENCY_SAMPLING__SAMPLER1_DRAM_MASK                                                              0x00000002L
+#define GUS_LATENCY_SAMPLING__SAMPLER0_IO_MASK                                                                0x00000004L
+#define GUS_LATENCY_SAMPLING__SAMPLER1_IO_MASK                                                                0x00000008L
+#define GUS_LATENCY_SAMPLING__SAMPLER0_READ_MASK                                                              0x00000010L
+#define GUS_LATENCY_SAMPLING__SAMPLER1_READ_MASK                                                              0x00000020L
+#define GUS_LATENCY_SAMPLING__SAMPLER0_WRITE_MASK                                                             0x00000040L
+#define GUS_LATENCY_SAMPLING__SAMPLER1_WRITE_MASK                                                             0x00000080L
+#define GUS_LATENCY_SAMPLING__SAMPLER0_ATOMIC_RET_MASK                                                        0x00000100L
+#define GUS_LATENCY_SAMPLING__SAMPLER1_ATOMIC_RET_MASK                                                        0x00000200L
+#define GUS_LATENCY_SAMPLING__SAMPLER0_ATOMIC_NORET_MASK                                                      0x00000400L
+#define GUS_LATENCY_SAMPLING__SAMPLER1_ATOMIC_NORET_MASK                                                      0x00000800L
+#define GUS_LATENCY_SAMPLING__SAMPLER0_VC_MASK                                                                0x000FF000L
+#define GUS_LATENCY_SAMPLING__SAMPLER1_VC_MASK                                                                0x0FF00000L
+//GUS_ERR_STATUS
+#define GUS_ERR_STATUS__SDP_RDRSP_STATUS__SHIFT                                                               0x0
+#define GUS_ERR_STATUS__SDP_WRRSP_STATUS__SHIFT                                                               0x4
+#define GUS_ERR_STATUS__SDP_RDRSP_DATASTATUS__SHIFT                                                           0x8
+#define GUS_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR__SHIFT                                                     0xa
+#define GUS_ERR_STATUS__CLEAR_ERROR_STATUS__SHIFT                                                             0xb
+#define GUS_ERR_STATUS__BUSY_ON_ERROR__SHIFT                                                                  0xc
+#define GUS_ERR_STATUS__FUE_FLAG__SHIFT                                                                       0xd
+#define GUS_ERR_STATUS__SDP_RDRSP_STATUS_MASK                                                                 0x0000000FL
+#define GUS_ERR_STATUS__SDP_WRRSP_STATUS_MASK                                                                 0x000000F0L
+#define GUS_ERR_STATUS__SDP_RDRSP_DATASTATUS_MASK                                                             0x00000300L
+#define GUS_ERR_STATUS__SDP_RDRSP_DATAPARITY_ERROR_MASK                                                       0x00000400L
+#define GUS_ERR_STATUS__CLEAR_ERROR_STATUS_MASK                                                               0x00000800L
+#define GUS_ERR_STATUS__BUSY_ON_ERROR_MASK                                                                    0x00001000L
+#define GUS_ERR_STATUS__FUE_FLAG_MASK                                                                         0x00002000L
+//GUS_MISC2
+#define GUS_MISC2__IO_RDWR_PRIORITY_ENABLE__SHIFT                                                             0x0
+#define GUS_MISC2__CH_L1_RO_MASK__SHIFT                                                                       0x1
+#define GUS_MISC2__SA0_L1_RO_MASK__SHIFT                                                                      0x2
+#define GUS_MISC2__SA1_L1_RO_MASK__SHIFT                                                                      0x3
+#define GUS_MISC2__SA2_L1_RO_MASK__SHIFT                                                                      0x4
+#define GUS_MISC2__SA3_L1_RO_MASK__SHIFT                                                                      0x5
+#define GUS_MISC2__CH_L1_PERF_MASK__SHIFT                                                                     0x6
+#define GUS_MISC2__SA0_L1_PERF_MASK__SHIFT                                                                    0x7
+#define GUS_MISC2__SA1_L1_PERF_MASK__SHIFT                                                                    0x8
+#define GUS_MISC2__SA2_L1_PERF_MASK__SHIFT                                                                    0x9
+#define GUS_MISC2__SA3_L1_PERF_MASK__SHIFT                                                                    0xa
+#define GUS_MISC2__FP_ATOMICS_ENABLE__SHIFT                                                                   0xb
+#define GUS_MISC2__L1_RET_CLKEN__SHIFT                                                                        0xc
+#define GUS_MISC2__FGCLKEN_HIGH__SHIFT                                                                        0xd
+#define GUS_MISC2__BLOCK_REQUESTS__SHIFT                                                                      0xe
+#define GUS_MISC2__REQUESTS_BLOCKED__SHIFT                                                                    0xf
+#define GUS_MISC2__RIO_ICG_L1_ROUTER_BUSY_MASK__SHIFT                                                         0x10
+#define GUS_MISC2__WIO_ICG_L1_ROUTER_BUSY_MASK__SHIFT                                                         0x11
+#define GUS_MISC2__DRAM_ICG_L1_ROUTER_BUSY_MASK__SHIFT                                                        0x12
+#define GUS_MISC2__RDRET_FED_MASK__SHIFT                                                                      0x13
+#define GUS_MISC2__IO_RDWR_PRIORITY_ENABLE_MASK                                                               0x00000001L
+#define GUS_MISC2__CH_L1_RO_MASK_MASK                                                                         0x00000002L
+#define GUS_MISC2__SA0_L1_RO_MASK_MASK                                                                        0x00000004L
+#define GUS_MISC2__SA1_L1_RO_MASK_MASK                                                                        0x00000008L
+#define GUS_MISC2__SA2_L1_RO_MASK_MASK                                                                        0x00000010L
+#define GUS_MISC2__SA3_L1_RO_MASK_MASK                                                                        0x00000020L
+#define GUS_MISC2__CH_L1_PERF_MASK_MASK                                                                       0x00000040L
+#define GUS_MISC2__SA0_L1_PERF_MASK_MASK                                                                      0x00000080L
+#define GUS_MISC2__SA1_L1_PERF_MASK_MASK                                                                      0x00000100L
+#define GUS_MISC2__SA2_L1_PERF_MASK_MASK                                                                      0x00000200L
+#define GUS_MISC2__SA3_L1_PERF_MASK_MASK                                                                      0x00000400L
+#define GUS_MISC2__FP_ATOMICS_ENABLE_MASK                                                                     0x00000800L
+#define GUS_MISC2__L1_RET_CLKEN_MASK                                                                          0x00001000L
+#define GUS_MISC2__FGCLKEN_HIGH_MASK                                                                          0x00002000L
+#define GUS_MISC2__BLOCK_REQUESTS_MASK                                                                        0x00004000L
+#define GUS_MISC2__REQUESTS_BLOCKED_MASK                                                                      0x00008000L
+#define GUS_MISC2__RIO_ICG_L1_ROUTER_BUSY_MASK_MASK                                                           0x00010000L
+#define GUS_MISC2__WIO_ICG_L1_ROUTER_BUSY_MASK_MASK                                                           0x00020000L
+#define GUS_MISC2__DRAM_ICG_L1_ROUTER_BUSY_MASK_MASK                                                          0x00040000L
+#define GUS_MISC2__RDRET_FED_MASK_MASK                                                                        0x00080000L
+//GUS_SDP_BACKDOOR_CMDCREDITS0
+#define GUS_SDP_BACKDOOR_CMDCREDITS0__CREDITS_RECEIVED__SHIFT                                                 0x0
+#define GUS_SDP_BACKDOOR_CMDCREDITS0__CREDITS_RECEIVED_MASK                                                   0xFFFFFFFFL
+//GUS_SDP_BACKDOOR_CMDCREDITS1
+#define GUS_SDP_BACKDOOR_CMDCREDITS1__CREDITS_RECEIVED__SHIFT                                                 0x0
+#define GUS_SDP_BACKDOOR_CMDCREDITS1__CREDITS_RECEIVED_MASK                                                   0x7FFFFFFFL
+//GUS_SDP_BACKDOOR_DATACREDITS0
+#define GUS_SDP_BACKDOOR_DATACREDITS0__CREDITS_RECEIVED__SHIFT                                                0x0
+#define GUS_SDP_BACKDOOR_DATACREDITS0__CREDITS_RECEIVED_MASK                                                  0xFFFFFFFFL
+//GUS_SDP_BACKDOOR_DATACREDITS1
+#define GUS_SDP_BACKDOOR_DATACREDITS1__CREDITS_RECEIVED__SHIFT                                                0x0
+#define GUS_SDP_BACKDOOR_DATACREDITS1__CREDITS_RECEIVED_MASK                                                  0x7FFFFFFFL
+//GUS_SDP_BACKDOOR_MISCCREDITS
+#define GUS_SDP_BACKDOOR_MISCCREDITS__RDRSP_CREDITS_RELEASED__SHIFT                                           0x0
+#define GUS_SDP_BACKDOOR_MISCCREDITS__WRRSP_CREDITS_RELEASED__SHIFT                                           0x8
+#define GUS_SDP_BACKDOOR_MISCCREDITS__RDRSP_CREDITS_RELEASED_MASK                                             0x000000FFL
+#define GUS_SDP_BACKDOOR_MISCCREDITS__WRRSP_CREDITS_RELEASED_MASK                                             0x0000FF00L
+//GUS_SDP_ENABLE
+#define GUS_SDP_ENABLE__ENABLE__SHIFT                                                                         0x0
+#define GUS_SDP_ENABLE__ENABLE_MASK                                                                           0x00000001L
+//GUS_L1_CH0_CMD_IN
+#define GUS_L1_CH0_CMD_IN__COUNT__SHIFT                                                                       0x0
+#define GUS_L1_CH0_CMD_IN__COUNT_MASK                                                                         0xFFFFFFFFL
+//GUS_L1_CH0_CMD_OUT
+#define GUS_L1_CH0_CMD_OUT__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_CH0_CMD_OUT__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_CH0_DATA_IN
+#define GUS_L1_CH0_DATA_IN__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_CH0_DATA_IN__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_CH0_DATA_OUT
+#define GUS_L1_CH0_DATA_OUT__COUNT__SHIFT                                                                     0x0
+#define GUS_L1_CH0_DATA_OUT__COUNT_MASK                                                                       0xFFFFFFFFL
+//GUS_L1_CH0_DATA_U_IN
+#define GUS_L1_CH0_DATA_U_IN__COUNT__SHIFT                                                                    0x0
+#define GUS_L1_CH0_DATA_U_IN__COUNT_MASK                                                                      0xFFFFFFFFL
+//GUS_L1_CH0_DATA_U_OUT
+#define GUS_L1_CH0_DATA_U_OUT__COUNT__SHIFT                                                                   0x0
+#define GUS_L1_CH0_DATA_U_OUT__COUNT_MASK                                                                     0xFFFFFFFFL
+//GUS_L1_CH1_CMD_IN
+#define GUS_L1_CH1_CMD_IN__COUNT__SHIFT                                                                       0x0
+#define GUS_L1_CH1_CMD_IN__COUNT_MASK                                                                         0xFFFFFFFFL
+//GUS_L1_CH1_CMD_OUT
+#define GUS_L1_CH1_CMD_OUT__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_CH1_CMD_OUT__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_CH1_DATA_IN
+#define GUS_L1_CH1_DATA_IN__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_CH1_DATA_IN__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_CH1_DATA_OUT
+#define GUS_L1_CH1_DATA_OUT__COUNT__SHIFT                                                                     0x0
+#define GUS_L1_CH1_DATA_OUT__COUNT_MASK                                                                       0xFFFFFFFFL
+//GUS_L1_CH1_DATA_U_IN
+#define GUS_L1_CH1_DATA_U_IN__COUNT__SHIFT                                                                    0x0
+#define GUS_L1_CH1_DATA_U_IN__COUNT_MASK                                                                      0xFFFFFFFFL
+//GUS_L1_CH1_DATA_U_OUT
+#define GUS_L1_CH1_DATA_U_OUT__COUNT__SHIFT                                                                   0x0
+#define GUS_L1_CH1_DATA_U_OUT__COUNT_MASK                                                                     0xFFFFFFFFL
+//GUS_L1_SA0_CMD_IN
+#define GUS_L1_SA0_CMD_IN__COUNT__SHIFT                                                                       0x0
+#define GUS_L1_SA0_CMD_IN__COUNT_MASK                                                                         0xFFFFFFFFL
+//GUS_L1_SA0_CMD_OUT
+#define GUS_L1_SA0_CMD_OUT__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_SA0_CMD_OUT__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_SA0_DATA_IN
+#define GUS_L1_SA0_DATA_IN__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_SA0_DATA_IN__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_SA0_DATA_OUT
+#define GUS_L1_SA0_DATA_OUT__COUNT__SHIFT                                                                     0x0
+#define GUS_L1_SA0_DATA_OUT__COUNT_MASK                                                                       0xFFFFFFFFL
+//GUS_L1_SA0_DATA_U_IN
+#define GUS_L1_SA0_DATA_U_IN__COUNT__SHIFT                                                                    0x0
+#define GUS_L1_SA0_DATA_U_IN__COUNT_MASK                                                                      0xFFFFFFFFL
+//GUS_L1_SA0_DATA_U_OUT
+#define GUS_L1_SA0_DATA_U_OUT__COUNT__SHIFT                                                                   0x0
+#define GUS_L1_SA0_DATA_U_OUT__COUNT_MASK                                                                     0xFFFFFFFFL
+//GUS_L1_SA1_CMD_IN
+#define GUS_L1_SA1_CMD_IN__COUNT__SHIFT                                                                       0x0
+#define GUS_L1_SA1_CMD_IN__COUNT_MASK                                                                         0xFFFFFFFFL
+//GUS_L1_SA1_CMD_OUT
+#define GUS_L1_SA1_CMD_OUT__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_SA1_CMD_OUT__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_SA1_DATA_IN
+#define GUS_L1_SA1_DATA_IN__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_SA1_DATA_IN__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_SA1_DATA_OUT
+#define GUS_L1_SA1_DATA_OUT__COUNT__SHIFT                                                                     0x0
+#define GUS_L1_SA1_DATA_OUT__COUNT_MASK                                                                       0xFFFFFFFFL
+//GUS_L1_SA1_DATA_U_IN
+#define GUS_L1_SA1_DATA_U_IN__COUNT__SHIFT                                                                    0x0
+#define GUS_L1_SA1_DATA_U_IN__COUNT_MASK                                                                      0xFFFFFFFFL
+//GUS_L1_SA1_DATA_U_OUT
+#define GUS_L1_SA1_DATA_U_OUT__COUNT__SHIFT                                                                   0x0
+#define GUS_L1_SA1_DATA_U_OUT__COUNT_MASK                                                                     0xFFFFFFFFL
+//GUS_L1_SA2_CMD_IN
+#define GUS_L1_SA2_CMD_IN__COUNT__SHIFT                                                                       0x0
+#define GUS_L1_SA2_CMD_IN__COUNT_MASK                                                                         0xFFFFFFFFL
+//GUS_L1_SA2_CMD_OUT
+#define GUS_L1_SA2_CMD_OUT__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_SA2_CMD_OUT__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_SA2_DATA_IN
+#define GUS_L1_SA2_DATA_IN__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_SA2_DATA_IN__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_SA2_DATA_OUT
+#define GUS_L1_SA2_DATA_OUT__COUNT__SHIFT                                                                     0x0
+#define GUS_L1_SA2_DATA_OUT__COUNT_MASK                                                                       0xFFFFFFFFL
+//GUS_L1_SA2_DATA_U_IN
+#define GUS_L1_SA2_DATA_U_IN__COUNT__SHIFT                                                                    0x0
+#define GUS_L1_SA2_DATA_U_IN__COUNT_MASK                                                                      0xFFFFFFFFL
+//GUS_L1_SA2_DATA_U_OUT
+#define GUS_L1_SA2_DATA_U_OUT__COUNT__SHIFT                                                                   0x0
+#define GUS_L1_SA2_DATA_U_OUT__COUNT_MASK                                                                     0xFFFFFFFFL
+//GUS_L1_SA3_CMD_IN
+#define GUS_L1_SA3_CMD_IN__COUNT__SHIFT                                                                       0x0
+#define GUS_L1_SA3_CMD_IN__COUNT_MASK                                                                         0xFFFFFFFFL
+//GUS_L1_SA3_CMD_OUT
+#define GUS_L1_SA3_CMD_OUT__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_SA3_CMD_OUT__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_SA3_DATA_IN
+#define GUS_L1_SA3_DATA_IN__COUNT__SHIFT                                                                      0x0
+#define GUS_L1_SA3_DATA_IN__COUNT_MASK                                                                        0xFFFFFFFFL
+//GUS_L1_SA3_DATA_OUT
+#define GUS_L1_SA3_DATA_OUT__COUNT__SHIFT                                                                     0x0
+#define GUS_L1_SA3_DATA_OUT__COUNT_MASK                                                                       0xFFFFFFFFL
+//GUS_L1_SA3_DATA_U_IN
+#define GUS_L1_SA3_DATA_U_IN__COUNT__SHIFT                                                                    0x0
+#define GUS_L1_SA3_DATA_U_IN__COUNT_MASK                                                                      0xFFFFFFFFL
+//GUS_L1_SA3_DATA_U_OUT
+#define GUS_L1_SA3_DATA_U_OUT__COUNT__SHIFT                                                                   0x0
+#define GUS_L1_SA3_DATA_U_OUT__COUNT_MASK                                                                     0xFFFFFFFFL
+//GUS_MISC3
+#define GUS_MISC3__FP_ATOMICS_LOG__SHIFT                                                                      0x0
+#define GUS_MISC3__CLEAR_LOG__SHIFT                                                                           0x1
+#define GUS_MISC3__FP_ATOMICS_LOG_MASK                                                                        0x00000001L
+#define GUS_MISC3__CLEAR_LOG_MASK                                                                             0x00000002L
+//GUS_WRRSP_FIFO_CNTL
+#define GUS_WRRSP_FIFO_CNTL__THRESHOLD__SHIFT                                                                 0x0
+#define GUS_WRRSP_FIFO_CNTL__THRESHOLD_MASK                                                                   0x0000003FL
+
+
+// addressBlock: gc_gfxdec0
+//DB_RENDER_CONTROL
+#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE__SHIFT                                                          0x0
+#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE__SHIFT                                                        0x1
+#define DB_RENDER_CONTROL__DEPTH_COPY__SHIFT                                                                  0x2
+#define DB_RENDER_CONTROL__STENCIL_COPY__SHIFT                                                                0x3
+#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE__SHIFT                                                          0x4
+#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE__SHIFT                                                    0x5
+#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE__SHIFT                                                      0x6
+#define DB_RENDER_CONTROL__COPY_CENTROID__SHIFT                                                               0x7
+#define DB_RENDER_CONTROL__COPY_SAMPLE__SHIFT                                                                 0x8
+#define DB_RENDER_CONTROL__DECOMPRESS_ENABLE__SHIFT                                                           0xc
+#define DB_RENDER_CONTROL__PS_INVOKE_DISABLE__SHIFT                                                           0xe
+#define DB_RENDER_CONTROL__OREO_MODE__SHIFT                                                                   0x10
+#define DB_RENDER_CONTROL__FORCE_OREO_MODE__SHIFT                                                             0x12
+#define DB_RENDER_CONTROL__FORCE_EXPORT_ORDER__SHIFT                                                          0x13
+#define DB_RENDER_CONTROL__MAX_ALLOWED_TILES_IN_WAVE__SHIFT                                                   0x14
+#define DB_RENDER_CONTROL__DEPTH_CLEAR_ENABLE_MASK                                                            0x00000001L
+#define DB_RENDER_CONTROL__STENCIL_CLEAR_ENABLE_MASK                                                          0x00000002L
+#define DB_RENDER_CONTROL__DEPTH_COPY_MASK                                                                    0x00000004L
+#define DB_RENDER_CONTROL__STENCIL_COPY_MASK                                                                  0x00000008L
+#define DB_RENDER_CONTROL__RESUMMARIZE_ENABLE_MASK                                                            0x00000010L
+#define DB_RENDER_CONTROL__STENCIL_COMPRESS_DISABLE_MASK                                                      0x00000020L
+#define DB_RENDER_CONTROL__DEPTH_COMPRESS_DISABLE_MASK                                                        0x00000040L
+#define DB_RENDER_CONTROL__COPY_CENTROID_MASK                                                                 0x00000080L
+#define DB_RENDER_CONTROL__COPY_SAMPLE_MASK                                                                   0x00000F00L
+#define DB_RENDER_CONTROL__DECOMPRESS_ENABLE_MASK                                                             0x00001000L
+#define DB_RENDER_CONTROL__PS_INVOKE_DISABLE_MASK                                                             0x00004000L
+#define DB_RENDER_CONTROL__OREO_MODE_MASK                                                                     0x00030000L
+#define DB_RENDER_CONTROL__FORCE_OREO_MODE_MASK                                                               0x00040000L
+#define DB_RENDER_CONTROL__FORCE_EXPORT_ORDER_MASK                                                            0x00080000L
+#define DB_RENDER_CONTROL__MAX_ALLOWED_TILES_IN_WAVE_MASK                                                     0x00F00000L
+//DB_COUNT_CONTROL
+#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS__SHIFT                                                         0x1
+#define DB_COUNT_CONTROL__DISABLE_CONSERVATIVE_ZPASS_COUNTS__SHIFT                                            0x2
+#define DB_COUNT_CONTROL__ENHANCED_CONSERVATIVE_ZPASS_COUNTS__SHIFT                                           0x3
+#define DB_COUNT_CONTROL__SAMPLE_RATE__SHIFT                                                                  0x4
+#define DB_COUNT_CONTROL__ZPASS_ENABLE__SHIFT                                                                 0x8
+#define DB_COUNT_CONTROL__ZFAIL_ENABLE__SHIFT                                                                 0xc
+#define DB_COUNT_CONTROL__SFAIL_ENABLE__SHIFT                                                                 0x10
+#define DB_COUNT_CONTROL__DBFAIL_ENABLE__SHIFT                                                                0x14
+#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE__SHIFT                                                            0x18
+#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE__SHIFT                                                             0x1c
+#define DB_COUNT_CONTROL__PERFECT_ZPASS_COUNTS_MASK                                                           0x00000002L
+#define DB_COUNT_CONTROL__DISABLE_CONSERVATIVE_ZPASS_COUNTS_MASK                                              0x00000004L
+#define DB_COUNT_CONTROL__ENHANCED_CONSERVATIVE_ZPASS_COUNTS_MASK                                             0x00000008L
+#define DB_COUNT_CONTROL__SAMPLE_RATE_MASK                                                                    0x00000070L
+#define DB_COUNT_CONTROL__ZPASS_ENABLE_MASK                                                                   0x00000F00L
+#define DB_COUNT_CONTROL__ZFAIL_ENABLE_MASK                                                                   0x0000F000L
+#define DB_COUNT_CONTROL__SFAIL_ENABLE_MASK                                                                   0x000F0000L
+#define DB_COUNT_CONTROL__DBFAIL_ENABLE_MASK                                                                  0x00F00000L
+#define DB_COUNT_CONTROL__SLICE_EVEN_ENABLE_MASK                                                              0x0F000000L
+#define DB_COUNT_CONTROL__SLICE_ODD_ENABLE_MASK                                                               0xF0000000L
+//DB_DEPTH_VIEW
+#define DB_DEPTH_VIEW__SLICE_START__SHIFT                                                                     0x0
+#define DB_DEPTH_VIEW__SLICE_START_HI__SHIFT                                                                  0xb
+#define DB_DEPTH_VIEW__SLICE_MAX__SHIFT                                                                       0xd
+#define DB_DEPTH_VIEW__Z_READ_ONLY__SHIFT                                                                     0x18
+#define DB_DEPTH_VIEW__STENCIL_READ_ONLY__SHIFT                                                               0x19
+#define DB_DEPTH_VIEW__MIPID__SHIFT                                                                           0x1a
+#define DB_DEPTH_VIEW__SLICE_MAX_HI__SHIFT                                                                    0x1e
+#define DB_DEPTH_VIEW__SLICE_START_MASK                                                                       0x000007FFL
+#define DB_DEPTH_VIEW__SLICE_START_HI_MASK                                                                    0x00001800L
+#define DB_DEPTH_VIEW__SLICE_MAX_MASK                                                                         0x00FFE000L
+#define DB_DEPTH_VIEW__Z_READ_ONLY_MASK                                                                       0x01000000L
+#define DB_DEPTH_VIEW__STENCIL_READ_ONLY_MASK                                                                 0x02000000L
+#define DB_DEPTH_VIEW__MIPID_MASK                                                                             0x3C000000L
+#define DB_DEPTH_VIEW__SLICE_MAX_HI_MASK                                                                      0xC0000000L
+//DB_RENDER_OVERRIDE
+#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE__SHIFT                                                           0x0
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0__SHIFT                                                          0x2
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1__SHIFT                                                          0x4
+#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER__SHIFT                                                       0x6
+#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE__SHIFT                                                             0x7
+#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE__SHIFT                                                       0x8
+#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE__SHIFT                                                          0x9
+#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL__SHIFT                                                           0xa
+#define DB_RENDER_OVERRIDE__FORCE_Z_READ__SHIFT                                                               0xb
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ__SHIFT                                                         0xc
+#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE__SHIFT                                                         0xd
+#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP__SHIFT                                                     0x10
+#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE__SHIFT                                                           0x11
+#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED__SHIFT                                                      0x12
+#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM__SHIFT                                                         0x13
+#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT__SHIFT                                                           0x15
+#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES__SHIFT                                                    0x1a
+#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY__SHIFT                                                              0x1b
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY__SHIFT                                                        0x1c
+#define DB_RENDER_OVERRIDE__FORCE_Z_VALID__SHIFT                                                              0x1d
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID__SHIFT                                                        0x1e
+#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION__SHIFT                                                       0x1f
+#define DB_RENDER_OVERRIDE__FORCE_HIZ_ENABLE_MASK                                                             0x00000003L
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE0_MASK                                                            0x0000000CL
+#define DB_RENDER_OVERRIDE__FORCE_HIS_ENABLE1_MASK                                                            0x00000030L
+#define DB_RENDER_OVERRIDE__FORCE_SHADER_Z_ORDER_MASK                                                         0x00000040L
+#define DB_RENDER_OVERRIDE__FAST_Z_DISABLE_MASK                                                               0x00000080L
+#define DB_RENDER_OVERRIDE__FAST_STENCIL_DISABLE_MASK                                                         0x00000100L
+#define DB_RENDER_OVERRIDE__NOOP_CULL_DISABLE_MASK                                                            0x00000200L
+#define DB_RENDER_OVERRIDE__FORCE_COLOR_KILL_MASK                                                             0x00000400L
+#define DB_RENDER_OVERRIDE__FORCE_Z_READ_MASK                                                                 0x00000800L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_READ_MASK                                                           0x00001000L
+#define DB_RENDER_OVERRIDE__FORCE_FULL_Z_RANGE_MASK                                                           0x00006000L
+#define DB_RENDER_OVERRIDE__DISABLE_VIEWPORT_CLAMP_MASK                                                       0x00010000L
+#define DB_RENDER_OVERRIDE__IGNORE_SC_ZRANGE_MASK                                                             0x00020000L
+#define DB_RENDER_OVERRIDE__DISABLE_FULLY_COVERED_MASK                                                        0x00040000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_LIMIT_SUMM_MASK                                                           0x00180000L
+#define DB_RENDER_OVERRIDE__MAX_TILES_IN_DTT_MASK                                                             0x03E00000L
+#define DB_RENDER_OVERRIDE__DISABLE_TILE_RATE_TILES_MASK                                                      0x04000000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_DIRTY_MASK                                                                0x08000000L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_DIRTY_MASK                                                          0x10000000L
+#define DB_RENDER_OVERRIDE__FORCE_Z_VALID_MASK                                                                0x20000000L
+#define DB_RENDER_OVERRIDE__FORCE_STENCIL_VALID_MASK                                                          0x40000000L
+#define DB_RENDER_OVERRIDE__PRESERVE_COMPRESSION_MASK                                                         0x80000000L
+//DB_RENDER_OVERRIDE2
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL__SHIFT                                              0x0
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN__SHIFT                                            0x2
+#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION__SHIFT                                       0x5
+#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION__SHIFT                                        0x6
+#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION__SHIFT                                               0x7
+#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH__SHIFT                                                     0x8
+#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP__SHIFT                                                         0x9
+#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE__SHIFT                                           0xa
+#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE__SHIFT                                                 0xb
+#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC__SHIFT                                                                 0xc
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF__SHIFT                                                              0xf
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF__SHIFT                                                              0x12
+#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE__SHIFT                                                           0x15
+#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS__SHIFT                                                         0x16
+#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS__SHIFT                                                         0x17
+#define DB_RENDER_OVERRIDE2__ALLOW_PARTIAL_RES_HIER_KILL__SHIFT                                               0x19
+#define DB_RENDER_OVERRIDE2__CENTROID_COMPUTATION_MODE__SHIFT                                                 0x1b
+#define DB_RENDER_OVERRIDE2__DISABLE_NOZ__SHIFT                                                               0x1d
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_CONTROL_MASK                                                0x00000003L
+#define DB_RENDER_OVERRIDE2__PARTIAL_SQUAD_LAUNCH_COUNTDOWN_MASK                                              0x0000001CL
+#define DB_RENDER_OVERRIDE2__DISABLE_ZMASK_EXPCLEAR_OPTIMIZATION_MASK                                         0x00000020L
+#define DB_RENDER_OVERRIDE2__DISABLE_SMEM_EXPCLEAR_OPTIMIZATION_MASK                                          0x00000040L
+#define DB_RENDER_OVERRIDE2__DISABLE_COLOR_ON_VALIDATION_MASK                                                 0x00000080L
+#define DB_RENDER_OVERRIDE2__DECOMPRESS_Z_ON_FLUSH_MASK                                                       0x00000100L
+#define DB_RENDER_OVERRIDE2__DISABLE_REG_SNOOP_MASK                                                           0x00000200L
+#define DB_RENDER_OVERRIDE2__DEPTH_BOUNDS_HIER_DEPTH_DISABLE_MASK                                             0x00000400L
+#define DB_RENDER_OVERRIDE2__SEPARATE_HIZS_FUNC_ENABLE_MASK                                                   0x00000800L
+#define DB_RENDER_OVERRIDE2__HIZ_ZFUNC_MASK                                                                   0x00007000L
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_FF_MASK                                                                0x00038000L
+#define DB_RENDER_OVERRIDE2__HIS_SFUNC_BF_MASK                                                                0x001C0000L
+#define DB_RENDER_OVERRIDE2__PRESERVE_ZRANGE_MASK                                                             0x00200000L
+#define DB_RENDER_OVERRIDE2__PRESERVE_SRESULTS_MASK                                                           0x00400000L
+#define DB_RENDER_OVERRIDE2__DISABLE_FAST_PASS_MASK                                                           0x00800000L
+#define DB_RENDER_OVERRIDE2__ALLOW_PARTIAL_RES_HIER_KILL_MASK                                                 0x02000000L
+#define DB_RENDER_OVERRIDE2__CENTROID_COMPUTATION_MODE_MASK                                                   0x18000000L
+#define DB_RENDER_OVERRIDE2__DISABLE_NOZ_MASK                                                                 0x20000000L
+//DB_HTILE_DATA_BASE
+#define DB_HTILE_DATA_BASE__BASE_256B__SHIFT                                                                  0x0
+#define DB_HTILE_DATA_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//DB_DEPTH_SIZE_XY
+#define DB_DEPTH_SIZE_XY__X_MAX__SHIFT                                                                        0x0
+#define DB_DEPTH_SIZE_XY__Y_MAX__SHIFT                                                                        0x10
+#define DB_DEPTH_SIZE_XY__X_MAX_MASK                                                                          0x00003FFFL
+#define DB_DEPTH_SIZE_XY__Y_MAX_MASK                                                                          0x3FFF0000L
+//DB_DEPTH_BOUNDS_MIN
+#define DB_DEPTH_BOUNDS_MIN__MIN__SHIFT                                                                       0x0
+#define DB_DEPTH_BOUNDS_MIN__MIN_MASK                                                                         0xFFFFFFFFL
+//DB_DEPTH_BOUNDS_MAX
+#define DB_DEPTH_BOUNDS_MAX__MAX__SHIFT                                                                       0x0
+#define DB_DEPTH_BOUNDS_MAX__MAX_MASK                                                                         0xFFFFFFFFL
+//DB_STENCIL_CLEAR
+#define DB_STENCIL_CLEAR__CLEAR__SHIFT                                                                        0x0
+#define DB_STENCIL_CLEAR__CLEAR_MASK                                                                          0x000000FFL
+//DB_DEPTH_CLEAR
+#define DB_DEPTH_CLEAR__DEPTH_CLEAR__SHIFT                                                                    0x0
+#define DB_DEPTH_CLEAR__DEPTH_CLEAR_MASK                                                                      0xFFFFFFFFL
+//PA_SC_SCREEN_SCISSOR_TL
+#define PA_SC_SCREEN_SCISSOR_TL__TL_X__SHIFT                                                                  0x0
+#define PA_SC_SCREEN_SCISSOR_TL__TL_Y__SHIFT                                                                  0x10
+#define PA_SC_SCREEN_SCISSOR_TL__TL_X_MASK                                                                    0x0000FFFFL
+#define PA_SC_SCREEN_SCISSOR_TL__TL_Y_MASK                                                                    0xFFFF0000L
+//PA_SC_SCREEN_SCISSOR_BR
+#define PA_SC_SCREEN_SCISSOR_BR__BR_X__SHIFT                                                                  0x0
+#define PA_SC_SCREEN_SCISSOR_BR__BR_Y__SHIFT                                                                  0x10
+#define PA_SC_SCREEN_SCISSOR_BR__BR_X_MASK                                                                    0x0000FFFFL
+#define PA_SC_SCREEN_SCISSOR_BR__BR_Y_MASK                                                                    0xFFFF0000L
+//DB_RESERVED_REG_2
+#define DB_RESERVED_REG_2__FIELD_1__SHIFT                                                                     0x0
+#define DB_RESERVED_REG_2__FIELD_2__SHIFT                                                                     0x4
+#define DB_RESERVED_REG_2__FIELD_3__SHIFT                                                                     0x8
+#define DB_RESERVED_REG_2__FIELD_4__SHIFT                                                                     0xd
+#define DB_RESERVED_REG_2__FIELD_5__SHIFT                                                                     0xf
+#define DB_RESERVED_REG_2__FIELD_6__SHIFT                                                                     0x11
+#define DB_RESERVED_REG_2__FIELD_7__SHIFT                                                                     0x13
+#define DB_RESERVED_REG_2__FIELD_8__SHIFT                                                                     0x1c
+#define DB_RESERVED_REG_2__FIELD_1_MASK                                                                       0x0000000FL
+#define DB_RESERVED_REG_2__FIELD_2_MASK                                                                       0x000000F0L
+#define DB_RESERVED_REG_2__FIELD_3_MASK                                                                       0x00001F00L
+#define DB_RESERVED_REG_2__FIELD_4_MASK                                                                       0x00006000L
+#define DB_RESERVED_REG_2__FIELD_5_MASK                                                                       0x00018000L
+#define DB_RESERVED_REG_2__FIELD_6_MASK                                                                       0x00060000L
+#define DB_RESERVED_REG_2__FIELD_7_MASK                                                                       0x00180000L
+#define DB_RESERVED_REG_2__FIELD_8_MASK                                                                       0xF0000000L
+//DB_Z_INFO
+#define DB_Z_INFO__FORMAT__SHIFT                                                                              0x0
+#define DB_Z_INFO__NUM_SAMPLES__SHIFT                                                                         0x2
+#define DB_Z_INFO__SW_MODE__SHIFT                                                                             0x4
+#define DB_Z_INFO__FAULT_BEHAVIOR__SHIFT                                                                      0x9
+#define DB_Z_INFO__ITERATE_FLUSH__SHIFT                                                                       0xb
+#define DB_Z_INFO__PARTIALLY_RESIDENT__SHIFT                                                                  0xc
+#define DB_Z_INFO__RESERVED_FIELD_1__SHIFT                                                                    0xd
+#define DB_Z_INFO__MAXMIP__SHIFT                                                                              0x10
+#define DB_Z_INFO__ITERATE_256__SHIFT                                                                         0x14
+#define DB_Z_INFO__DECOMPRESS_ON_N_ZPLANES__SHIFT                                                             0x17
+#define DB_Z_INFO__ALLOW_EXPCLEAR__SHIFT                                                                      0x1b
+#define DB_Z_INFO__READ_SIZE__SHIFT                                                                           0x1c
+#define DB_Z_INFO__TILE_SURFACE_ENABLE__SHIFT                                                                 0x1d
+#define DB_Z_INFO__ZRANGE_PRECISION__SHIFT                                                                    0x1f
+#define DB_Z_INFO__FORMAT_MASK                                                                                0x00000003L
+#define DB_Z_INFO__NUM_SAMPLES_MASK                                                                           0x0000000CL
+#define DB_Z_INFO__SW_MODE_MASK                                                                               0x000001F0L
+#define DB_Z_INFO__FAULT_BEHAVIOR_MASK                                                                        0x00000600L
+#define DB_Z_INFO__ITERATE_FLUSH_MASK                                                                         0x00000800L
+#define DB_Z_INFO__PARTIALLY_RESIDENT_MASK                                                                    0x00001000L
+#define DB_Z_INFO__RESERVED_FIELD_1_MASK                                                                      0x0000E000L
+#define DB_Z_INFO__MAXMIP_MASK                                                                                0x000F0000L
+#define DB_Z_INFO__ITERATE_256_MASK                                                                           0x00100000L
+#define DB_Z_INFO__DECOMPRESS_ON_N_ZPLANES_MASK                                                               0x07800000L
+#define DB_Z_INFO__ALLOW_EXPCLEAR_MASK                                                                        0x08000000L
+#define DB_Z_INFO__READ_SIZE_MASK                                                                             0x10000000L
+#define DB_Z_INFO__TILE_SURFACE_ENABLE_MASK                                                                   0x20000000L
+#define DB_Z_INFO__ZRANGE_PRECISION_MASK                                                                      0x80000000L
+//DB_STENCIL_INFO
+#define DB_STENCIL_INFO__FORMAT__SHIFT                                                                        0x0
+#define DB_STENCIL_INFO__SW_MODE__SHIFT                                                                       0x4
+#define DB_STENCIL_INFO__FAULT_BEHAVIOR__SHIFT                                                                0x9
+#define DB_STENCIL_INFO__ITERATE_FLUSH__SHIFT                                                                 0xb
+#define DB_STENCIL_INFO__PARTIALLY_RESIDENT__SHIFT                                                            0xc
+#define DB_STENCIL_INFO__RESERVED_FIELD_1__SHIFT                                                              0xd
+#define DB_STENCIL_INFO__ITERATE_256__SHIFT                                                                   0x14
+#define DB_STENCIL_INFO__ALLOW_EXPCLEAR__SHIFT                                                                0x1b
+#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE__SHIFT                                                          0x1d
+#define DB_STENCIL_INFO__FORMAT_MASK                                                                          0x00000001L
+#define DB_STENCIL_INFO__SW_MODE_MASK                                                                         0x000001F0L
+#define DB_STENCIL_INFO__FAULT_BEHAVIOR_MASK                                                                  0x00000600L
+#define DB_STENCIL_INFO__ITERATE_FLUSH_MASK                                                                   0x00000800L
+#define DB_STENCIL_INFO__PARTIALLY_RESIDENT_MASK                                                              0x00001000L
+#define DB_STENCIL_INFO__RESERVED_FIELD_1_MASK                                                                0x0000E000L
+#define DB_STENCIL_INFO__ITERATE_256_MASK                                                                     0x00100000L
+#define DB_STENCIL_INFO__ALLOW_EXPCLEAR_MASK                                                                  0x08000000L
+#define DB_STENCIL_INFO__TILE_STENCIL_DISABLE_MASK                                                            0x20000000L
+//DB_Z_READ_BASE
+#define DB_Z_READ_BASE__BASE_256B__SHIFT                                                                      0x0
+#define DB_Z_READ_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//DB_STENCIL_READ_BASE
+#define DB_STENCIL_READ_BASE__BASE_256B__SHIFT                                                                0x0
+#define DB_STENCIL_READ_BASE__BASE_256B_MASK                                                                  0xFFFFFFFFL
+//DB_Z_WRITE_BASE
+#define DB_Z_WRITE_BASE__BASE_256B__SHIFT                                                                     0x0
+#define DB_Z_WRITE_BASE__BASE_256B_MASK                                                                       0xFFFFFFFFL
+//DB_STENCIL_WRITE_BASE
+#define DB_STENCIL_WRITE_BASE__BASE_256B__SHIFT                                                               0x0
+#define DB_STENCIL_WRITE_BASE__BASE_256B_MASK                                                                 0xFFFFFFFFL
+//DB_RESERVED_REG_1
+#define DB_RESERVED_REG_1__FIELD_1__SHIFT                                                                     0x0
+#define DB_RESERVED_REG_1__FIELD_2__SHIFT                                                                     0xb
+#define DB_RESERVED_REG_1__FIELD_1_MASK                                                                       0x000007FFL
+#define DB_RESERVED_REG_1__FIELD_2_MASK                                                                       0x003FF800L
+//DB_RESERVED_REG_3
+#define DB_RESERVED_REG_3__FIELD_1__SHIFT                                                                     0x0
+#define DB_RESERVED_REG_3__FIELD_1_MASK                                                                       0x003FFFFFL
+//DB_Z_READ_BASE_HI
+#define DB_Z_READ_BASE_HI__BASE_HI__SHIFT                                                                     0x0
+#define DB_Z_READ_BASE_HI__BASE_HI_MASK                                                                       0x000000FFL
+//DB_STENCIL_READ_BASE_HI
+#define DB_STENCIL_READ_BASE_HI__BASE_HI__SHIFT                                                               0x0
+#define DB_STENCIL_READ_BASE_HI__BASE_HI_MASK                                                                 0x000000FFL
+//DB_Z_WRITE_BASE_HI
+#define DB_Z_WRITE_BASE_HI__BASE_HI__SHIFT                                                                    0x0
+#define DB_Z_WRITE_BASE_HI__BASE_HI_MASK                                                                      0x000000FFL
+//DB_STENCIL_WRITE_BASE_HI
+#define DB_STENCIL_WRITE_BASE_HI__BASE_HI__SHIFT                                                              0x0
+#define DB_STENCIL_WRITE_BASE_HI__BASE_HI_MASK                                                                0x000000FFL
+//DB_HTILE_DATA_BASE_HI
+#define DB_HTILE_DATA_BASE_HI__BASE_HI__SHIFT                                                                 0x0
+#define DB_HTILE_DATA_BASE_HI__BASE_HI_MASK                                                                   0x000000FFL
+//DB_RMI_L2_CACHE_CONTROL
+#define DB_RMI_L2_CACHE_CONTROL__Z_WR_POLICY__SHIFT                                                           0x0
+#define DB_RMI_L2_CACHE_CONTROL__S_WR_POLICY__SHIFT                                                           0x2
+#define DB_RMI_L2_CACHE_CONTROL__HTILE_WR_POLICY__SHIFT                                                       0x4
+#define DB_RMI_L2_CACHE_CONTROL__ZPCPSD_WR_POLICY__SHIFT                                                      0x6
+#define DB_RMI_L2_CACHE_CONTROL__Z_RD_POLICY__SHIFT                                                           0x10
+#define DB_RMI_L2_CACHE_CONTROL__S_RD_POLICY__SHIFT                                                           0x12
+#define DB_RMI_L2_CACHE_CONTROL__HTILE_RD_POLICY__SHIFT                                                       0x14
+#define DB_RMI_L2_CACHE_CONTROL__Z_BIG_PAGE__SHIFT                                                            0x18
+#define DB_RMI_L2_CACHE_CONTROL__S_BIG_PAGE__SHIFT                                                            0x19
+#define DB_RMI_L2_CACHE_CONTROL__Z_NOALLOC__SHIFT                                                             0x1a
+#define DB_RMI_L2_CACHE_CONTROL__S_NOALLOC__SHIFT                                                             0x1b
+#define DB_RMI_L2_CACHE_CONTROL__HTILE_NOALLOC__SHIFT                                                         0x1c
+#define DB_RMI_L2_CACHE_CONTROL__ZPCPSD_NOALLOC__SHIFT                                                        0x1d
+#define DB_RMI_L2_CACHE_CONTROL__Z_WR_POLICY_MASK                                                             0x00000003L
+#define DB_RMI_L2_CACHE_CONTROL__S_WR_POLICY_MASK                                                             0x0000000CL
+#define DB_RMI_L2_CACHE_CONTROL__HTILE_WR_POLICY_MASK                                                         0x00000030L
+#define DB_RMI_L2_CACHE_CONTROL__ZPCPSD_WR_POLICY_MASK                                                        0x000000C0L
+#define DB_RMI_L2_CACHE_CONTROL__Z_RD_POLICY_MASK                                                             0x00030000L
+#define DB_RMI_L2_CACHE_CONTROL__S_RD_POLICY_MASK                                                             0x000C0000L
+#define DB_RMI_L2_CACHE_CONTROL__HTILE_RD_POLICY_MASK                                                         0x00300000L
+#define DB_RMI_L2_CACHE_CONTROL__Z_BIG_PAGE_MASK                                                              0x01000000L
+#define DB_RMI_L2_CACHE_CONTROL__S_BIG_PAGE_MASK                                                              0x02000000L
+#define DB_RMI_L2_CACHE_CONTROL__Z_NOALLOC_MASK                                                               0x04000000L
+#define DB_RMI_L2_CACHE_CONTROL__S_NOALLOC_MASK                                                               0x08000000L
+#define DB_RMI_L2_CACHE_CONTROL__HTILE_NOALLOC_MASK                                                           0x10000000L
+#define DB_RMI_L2_CACHE_CONTROL__ZPCPSD_NOALLOC_MASK                                                          0x20000000L
+//TA_BC_BASE_ADDR
+#define TA_BC_BASE_ADDR__ADDRESS__SHIFT                                                                       0x0
+#define TA_BC_BASE_ADDR__ADDRESS_MASK                                                                         0xFFFFFFFFL
+//TA_BC_BASE_ADDR_HI
+#define TA_BC_BASE_ADDR_HI__ADDRESS__SHIFT                                                                    0x0
+#define TA_BC_BASE_ADDR_HI__ADDRESS_MASK                                                                      0x000000FFL
+//COHER_DEST_BASE_HI_0
+#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_0__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_HI_1
+#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_1__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_HI_2
+#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_2__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_HI_3
+#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B__SHIFT                                                        0x0
+#define COHER_DEST_BASE_HI_3__DEST_BASE_HI_256B_MASK                                                          0x000000FFL
+//COHER_DEST_BASE_2
+#define COHER_DEST_BASE_2__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_2__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//COHER_DEST_BASE_3
+#define COHER_DEST_BASE_3__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_3__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//PA_SC_WINDOW_OFFSET
+#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET__SHIFT                                                           0x0
+#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET__SHIFT                                                           0x10
+#define PA_SC_WINDOW_OFFSET__WINDOW_X_OFFSET_MASK                                                             0x0000FFFFL
+#define PA_SC_WINDOW_OFFSET__WINDOW_Y_OFFSET_MASK                                                             0xFFFF0000L
+//PA_SC_WINDOW_SCISSOR_TL
+#define PA_SC_WINDOW_SCISSOR_TL__TL_X__SHIFT                                                                  0x0
+#define PA_SC_WINDOW_SCISSOR_TL__TL_Y__SHIFT                                                                  0x10
+#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                 0x1f
+#define PA_SC_WINDOW_SCISSOR_TL__TL_X_MASK                                                                    0x00007FFFL
+#define PA_SC_WINDOW_SCISSOR_TL__TL_Y_MASK                                                                    0x7FFF0000L
+#define PA_SC_WINDOW_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK                                                   0x80000000L
+//PA_SC_WINDOW_SCISSOR_BR
+#define PA_SC_WINDOW_SCISSOR_BR__BR_X__SHIFT                                                                  0x0
+#define PA_SC_WINDOW_SCISSOR_BR__BR_Y__SHIFT                                                                  0x10
+#define PA_SC_WINDOW_SCISSOR_BR__BR_X_MASK                                                                    0x00007FFFL
+#define PA_SC_WINDOW_SCISSOR_BR__BR_Y_MASK                                                                    0x7FFF0000L
+//PA_SC_CLIPRECT_RULE
+#define PA_SC_CLIPRECT_RULE__CLIP_RULE__SHIFT                                                                 0x0
+#define PA_SC_CLIPRECT_RULE__CLIP_RULE_MASK                                                                   0x0000FFFFL
+//PA_SC_CLIPRECT_0_TL
+#define PA_SC_CLIPRECT_0_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_0_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_0_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_0_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_0_BR
+#define PA_SC_CLIPRECT_0_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_0_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_0_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_0_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_1_TL
+#define PA_SC_CLIPRECT_1_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_1_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_1_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_1_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_1_BR
+#define PA_SC_CLIPRECT_1_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_1_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_1_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_1_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_2_TL
+#define PA_SC_CLIPRECT_2_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_2_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_2_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_2_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_2_BR
+#define PA_SC_CLIPRECT_2_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_2_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_2_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_2_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_3_TL
+#define PA_SC_CLIPRECT_3_TL__TL_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_3_TL__TL_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_3_TL__TL_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_3_TL__TL_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_CLIPRECT_3_BR
+#define PA_SC_CLIPRECT_3_BR__BR_X__SHIFT                                                                      0x0
+#define PA_SC_CLIPRECT_3_BR__BR_Y__SHIFT                                                                      0x10
+#define PA_SC_CLIPRECT_3_BR__BR_X_MASK                                                                        0x00007FFFL
+#define PA_SC_CLIPRECT_3_BR__BR_Y_MASK                                                                        0x7FFF0000L
+//PA_SC_EDGERULE
+#define PA_SC_EDGERULE__ER_TRI__SHIFT                                                                         0x0
+#define PA_SC_EDGERULE__ER_POINT__SHIFT                                                                       0x4
+#define PA_SC_EDGERULE__ER_RECT__SHIFT                                                                        0x8
+#define PA_SC_EDGERULE__ER_LINE_LR__SHIFT                                                                     0xc
+#define PA_SC_EDGERULE__ER_LINE_RL__SHIFT                                                                     0x12
+#define PA_SC_EDGERULE__ER_LINE_TB__SHIFT                                                                     0x18
+#define PA_SC_EDGERULE__ER_LINE_BT__SHIFT                                                                     0x1c
+#define PA_SC_EDGERULE__ER_TRI_MASK                                                                           0x0000000FL
+#define PA_SC_EDGERULE__ER_POINT_MASK                                                                         0x000000F0L
+#define PA_SC_EDGERULE__ER_RECT_MASK                                                                          0x00000F00L
+#define PA_SC_EDGERULE__ER_LINE_LR_MASK                                                                       0x0003F000L
+#define PA_SC_EDGERULE__ER_LINE_RL_MASK                                                                       0x00FC0000L
+#define PA_SC_EDGERULE__ER_LINE_TB_MASK                                                                       0x0F000000L
+#define PA_SC_EDGERULE__ER_LINE_BT_MASK                                                                       0xF0000000L
+//PA_SU_HARDWARE_SCREEN_OFFSET
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X__SHIFT                                               0x0
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y__SHIFT                                               0x10
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_X_MASK                                                 0x000001FFL
+#define PA_SU_HARDWARE_SCREEN_OFFSET__HW_SCREEN_OFFSET_Y_MASK                                                 0x01FF0000L
+//CB_TARGET_MASK
+#define CB_TARGET_MASK__TARGET0_ENABLE__SHIFT                                                                 0x0
+#define CB_TARGET_MASK__TARGET1_ENABLE__SHIFT                                                                 0x4
+#define CB_TARGET_MASK__TARGET2_ENABLE__SHIFT                                                                 0x8
+#define CB_TARGET_MASK__TARGET3_ENABLE__SHIFT                                                                 0xc
+#define CB_TARGET_MASK__TARGET4_ENABLE__SHIFT                                                                 0x10
+#define CB_TARGET_MASK__TARGET5_ENABLE__SHIFT                                                                 0x14
+#define CB_TARGET_MASK__TARGET6_ENABLE__SHIFT                                                                 0x18
+#define CB_TARGET_MASK__TARGET7_ENABLE__SHIFT                                                                 0x1c
+#define CB_TARGET_MASK__TARGET0_ENABLE_MASK                                                                   0x0000000FL
+#define CB_TARGET_MASK__TARGET1_ENABLE_MASK                                                                   0x000000F0L
+#define CB_TARGET_MASK__TARGET2_ENABLE_MASK                                                                   0x00000F00L
+#define CB_TARGET_MASK__TARGET3_ENABLE_MASK                                                                   0x0000F000L
+#define CB_TARGET_MASK__TARGET4_ENABLE_MASK                                                                   0x000F0000L
+#define CB_TARGET_MASK__TARGET5_ENABLE_MASK                                                                   0x00F00000L
+#define CB_TARGET_MASK__TARGET6_ENABLE_MASK                                                                   0x0F000000L
+#define CB_TARGET_MASK__TARGET7_ENABLE_MASK                                                                   0xF0000000L
+//CB_SHADER_MASK
+#define CB_SHADER_MASK__OUTPUT0_ENABLE__SHIFT                                                                 0x0
+#define CB_SHADER_MASK__OUTPUT1_ENABLE__SHIFT                                                                 0x4
+#define CB_SHADER_MASK__OUTPUT2_ENABLE__SHIFT                                                                 0x8
+#define CB_SHADER_MASK__OUTPUT3_ENABLE__SHIFT                                                                 0xc
+#define CB_SHADER_MASK__OUTPUT4_ENABLE__SHIFT                                                                 0x10
+#define CB_SHADER_MASK__OUTPUT5_ENABLE__SHIFT                                                                 0x14
+#define CB_SHADER_MASK__OUTPUT6_ENABLE__SHIFT                                                                 0x18
+#define CB_SHADER_MASK__OUTPUT7_ENABLE__SHIFT                                                                 0x1c
+#define CB_SHADER_MASK__OUTPUT0_ENABLE_MASK                                                                   0x0000000FL
+#define CB_SHADER_MASK__OUTPUT1_ENABLE_MASK                                                                   0x000000F0L
+#define CB_SHADER_MASK__OUTPUT2_ENABLE_MASK                                                                   0x00000F00L
+#define CB_SHADER_MASK__OUTPUT3_ENABLE_MASK                                                                   0x0000F000L
+#define CB_SHADER_MASK__OUTPUT4_ENABLE_MASK                                                                   0x000F0000L
+#define CB_SHADER_MASK__OUTPUT5_ENABLE_MASK                                                                   0x00F00000L
+#define CB_SHADER_MASK__OUTPUT6_ENABLE_MASK                                                                   0x0F000000L
+#define CB_SHADER_MASK__OUTPUT7_ENABLE_MASK                                                                   0xF0000000L
+//PA_SC_GENERIC_SCISSOR_TL
+#define PA_SC_GENERIC_SCISSOR_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_GENERIC_SCISSOR_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_GENERIC_SCISSOR_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_GENERIC_SCISSOR_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_GENERIC_SCISSOR_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_GENERIC_SCISSOR_BR
+#define PA_SC_GENERIC_SCISSOR_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_GENERIC_SCISSOR_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_GENERIC_SCISSOR_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_GENERIC_SCISSOR_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//COHER_DEST_BASE_0
+#define COHER_DEST_BASE_0__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_0__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//COHER_DEST_BASE_1
+#define COHER_DEST_BASE_1__DEST_BASE_256B__SHIFT                                                              0x0
+#define COHER_DEST_BASE_1__DEST_BASE_256B_MASK                                                                0xFFFFFFFFL
+//PA_SC_VPORT_SCISSOR_0_TL
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_0_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_0_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_0_BR
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_0_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_1_TL
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_1_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_1_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_1_BR
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_1_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_2_TL
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_2_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_2_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_2_BR
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_2_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_3_TL
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_3_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_3_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_3_BR
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_3_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_4_TL
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_4_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_4_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_4_BR
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_4_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_5_TL
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_5_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_5_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_5_BR
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_5_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_6_TL
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_6_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_6_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_6_BR
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_6_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_7_TL
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_7_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_7_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_7_BR
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_7_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_8_TL
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_8_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_8_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_8_BR
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_8_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_9_TL
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE__SHIFT                                                0x1f
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_9_TL__TL_Y_MASK                                                                   0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_9_TL__WINDOW_OFFSET_DISABLE_MASK                                                  0x80000000L
+//PA_SC_VPORT_SCISSOR_9_BR
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_X__SHIFT                                                                 0x0
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y__SHIFT                                                                 0x10
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_X_MASK                                                                   0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_9_BR__BR_Y_MASK                                                                   0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_10_TL
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_10_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_10_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_10_BR
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_10_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_11_TL
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_11_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_11_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_11_BR
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_11_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_12_TL
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_12_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_12_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_12_BR
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_12_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_13_TL
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_13_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_13_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_13_BR
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_13_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_14_TL
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_14_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_14_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_14_BR
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_14_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_SCISSOR_15_TL
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE__SHIFT                                               0x1f
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_15_TL__TL_Y_MASK                                                                  0x7FFF0000L
+#define PA_SC_VPORT_SCISSOR_15_TL__WINDOW_OFFSET_DISABLE_MASK                                                 0x80000000L
+//PA_SC_VPORT_SCISSOR_15_BR
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_X__SHIFT                                                                0x0
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y__SHIFT                                                                0x10
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_X_MASK                                                                  0x00007FFFL
+#define PA_SC_VPORT_SCISSOR_15_BR__BR_Y_MASK                                                                  0x7FFF0000L
+//PA_SC_VPORT_ZMIN_0
+#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_0__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_0
+#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_0__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_1
+#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_1__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_1
+#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_1__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_2
+#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_2__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_2
+#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_2__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_3
+#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_3__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_3
+#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_3__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_4
+#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_4__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_4
+#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_4__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_5
+#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_5__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_5
+#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_5__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_6
+#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_6__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_6
+#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_6__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_7
+#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_7__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_7
+#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_7__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_8
+#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_8__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_8
+#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_8__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_9
+#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMIN_9__VPORT_ZMIN_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_9
+#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX__SHIFT                                                                 0x0
+#define PA_SC_VPORT_ZMAX_9__VPORT_ZMAX_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_10
+#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_10__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_10
+#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_10__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_11
+#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_11__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_11
+#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_11__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_12
+#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_12__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_12
+#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_12__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_13
+#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_13__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_13
+#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_13__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_14
+#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_14__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_14
+#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_14__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMIN_15
+#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMIN_15__VPORT_ZMIN_MASK                                                                  0xFFFFFFFFL
+//PA_SC_VPORT_ZMAX_15
+#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX__SHIFT                                                                0x0
+#define PA_SC_VPORT_ZMAX_15__VPORT_ZMAX_MASK                                                                  0xFFFFFFFFL
+//PA_SC_RASTER_CONFIG
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0__SHIFT                                                               0x0
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1__SHIFT                                                               0x2
+#define PA_SC_RASTER_CONFIG__RB_XSEL2__SHIFT                                                                  0x4
+#define PA_SC_RASTER_CONFIG__RB_XSEL__SHIFT                                                                   0x6
+#define PA_SC_RASTER_CONFIG__RB_YSEL__SHIFT                                                                   0x7
+#define PA_SC_RASTER_CONFIG__PKR_MAP__SHIFT                                                                   0x8
+#define PA_SC_RASTER_CONFIG__PKR_XSEL__SHIFT                                                                  0xa
+#define PA_SC_RASTER_CONFIG__PKR_YSEL__SHIFT                                                                  0xc
+#define PA_SC_RASTER_CONFIG__PKR_XSEL2__SHIFT                                                                 0xe
+#define PA_SC_RASTER_CONFIG__SC_MAP__SHIFT                                                                    0x10
+#define PA_SC_RASTER_CONFIG__SC_XSEL__SHIFT                                                                   0x12
+#define PA_SC_RASTER_CONFIG__SC_YSEL__SHIFT                                                                   0x14
+#define PA_SC_RASTER_CONFIG__SE_MAP__SHIFT                                                                    0x18
+#define PA_SC_RASTER_CONFIG__SE_XSEL__SHIFT                                                                   0x1a
+#define PA_SC_RASTER_CONFIG__SE_YSEL__SHIFT                                                                   0x1c
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR0_MASK                                                                 0x00000003L
+#define PA_SC_RASTER_CONFIG__RB_MAP_PKR1_MASK                                                                 0x0000000CL
+#define PA_SC_RASTER_CONFIG__RB_XSEL2_MASK                                                                    0x00000030L
+#define PA_SC_RASTER_CONFIG__RB_XSEL_MASK                                                                     0x00000040L
+#define PA_SC_RASTER_CONFIG__RB_YSEL_MASK                                                                     0x00000080L
+#define PA_SC_RASTER_CONFIG__PKR_MAP_MASK                                                                     0x00000300L
+#define PA_SC_RASTER_CONFIG__PKR_XSEL_MASK                                                                    0x00000C00L
+#define PA_SC_RASTER_CONFIG__PKR_YSEL_MASK                                                                    0x00003000L
+#define PA_SC_RASTER_CONFIG__PKR_XSEL2_MASK                                                                   0x0000C000L
+#define PA_SC_RASTER_CONFIG__SC_MAP_MASK                                                                      0x00030000L
+#define PA_SC_RASTER_CONFIG__SC_XSEL_MASK                                                                     0x000C0000L
+#define PA_SC_RASTER_CONFIG__SC_YSEL_MASK                                                                     0x00300000L
+#define PA_SC_RASTER_CONFIG__SE_MAP_MASK                                                                      0x03000000L
+#define PA_SC_RASTER_CONFIG__SE_XSEL_MASK                                                                     0x0C000000L
+#define PA_SC_RASTER_CONFIG__SE_YSEL_MASK                                                                     0x30000000L
+//PA_SC_RASTER_CONFIG_1
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP__SHIFT                                                             0x0
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL__SHIFT                                                            0x2
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL__SHIFT                                                            0x4
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_MAP_MASK                                                               0x00000003L
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_XSEL_MASK                                                              0x0000000CL
+#define PA_SC_RASTER_CONFIG_1__SE_PAIR_YSEL_MASK                                                              0x00000030L
+//PA_SC_SCREEN_EXTENT_CONTROL
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE__SHIFT                                                 0x0
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE__SHIFT                                                  0x2
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_EVEN_ENABLE_MASK                                                   0x00000003L
+#define PA_SC_SCREEN_EXTENT_CONTROL__SLICE_ODD_ENABLE_MASK                                                    0x0000000CL
+//PA_SC_TILE_STEERING_OVERRIDE
+#define PA_SC_TILE_STEERING_OVERRIDE__ENABLE__SHIFT                                                           0x0
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_SC__SHIFT                                                           0xc
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SC__SHIFT                                                    0x10
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_PACKER_PER_SC__SHIFT                                                0x14
+#define PA_SC_TILE_STEERING_OVERRIDE__ENABLE_MASK                                                             0x00000001L
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_SC_MASK                                                             0x00003000L
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_RB_PER_SC_MASK                                                      0x00030000L
+#define PA_SC_TILE_STEERING_OVERRIDE__NUM_PACKER_PER_SC_MASK                                                  0x00300000L
+//CP_PERFMON_CNTX_CNTL
+#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE__SHIFT                                                           0x1f
+#define CP_PERFMON_CNTX_CNTL__PERFMON_ENABLE_MASK                                                             0x80000000L
+//CP_PIPEID
+#define CP_PIPEID__PIPE_ID__SHIFT                                                                             0x0
+#define CP_PIPEID__PIPE_ID_MASK                                                                               0x00000003L
+//CP_RINGID
+#define CP_RINGID__RINGID__SHIFT                                                                              0x0
+#define CP_RINGID__RINGID_MASK                                                                                0x00000003L
+//CP_VMID
+#define CP_VMID__VMID__SHIFT                                                                                  0x0
+#define CP_VMID__VMID_MASK                                                                                    0x0000000FL
+//CONTEXT_RESERVED_REG0
+#define CONTEXT_RESERVED_REG0__DATA__SHIFT                                                                    0x0
+#define CONTEXT_RESERVED_REG0__DATA_MASK                                                                      0xFFFFFFFFL
+//CONTEXT_RESERVED_REG1
+#define CONTEXT_RESERVED_REG1__DATA__SHIFT                                                                    0x0
+#define CONTEXT_RESERVED_REG1__DATA_MASK                                                                      0xFFFFFFFFL
+//PA_SC_VRS_OVERRIDE_CNTL
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_OVERRIDE_RATE_COMBINER_MODE__SHIFT                                       0x0
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_RATE__SHIFT                                                              0x4
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_SURFACE_ENABLE__SHIFT                                                    0xc
+#define PA_SC_VRS_OVERRIDE_CNTL__RATE_HINT_WRITE_BACK_ENABLE__SHIFT                                           0xd
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_FEEDBACK_RATE_OVERRIDE__SHIFT                                            0xe
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_OVERRIDE_RATE_COMBINER_MODE_MASK                                         0x00000007L
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_RATE_MASK                                                                0x000000F0L
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_SURFACE_ENABLE_MASK                                                      0x00001000L
+#define PA_SC_VRS_OVERRIDE_CNTL__RATE_HINT_WRITE_BACK_ENABLE_MASK                                             0x00002000L
+#define PA_SC_VRS_OVERRIDE_CNTL__VRS_FEEDBACK_RATE_OVERRIDE_MASK                                              0x00004000L
+//PA_SC_VRS_RATE_FEEDBACK_BASE
+#define PA_SC_VRS_RATE_FEEDBACK_BASE__BASE_256B__SHIFT                                                        0x0
+#define PA_SC_VRS_RATE_FEEDBACK_BASE__BASE_256B_MASK                                                          0xFFFFFFFFL
+//PA_SC_VRS_RATE_FEEDBACK_BASE_EXT
+#define PA_SC_VRS_RATE_FEEDBACK_BASE_EXT__BASE_256B__SHIFT                                                    0x0
+#define PA_SC_VRS_RATE_FEEDBACK_BASE_EXT__BASE_256B_MASK                                                      0x000000FFL
+//PA_SC_VRS_RATE_FEEDBACK_SIZE_XY
+#define PA_SC_VRS_RATE_FEEDBACK_SIZE_XY__X_MAX__SHIFT                                                         0x0
+#define PA_SC_VRS_RATE_FEEDBACK_SIZE_XY__Y_MAX__SHIFT                                                         0x10
+#define PA_SC_VRS_RATE_FEEDBACK_SIZE_XY__X_MAX_MASK                                                           0x000007FFL
+#define PA_SC_VRS_RATE_FEEDBACK_SIZE_XY__Y_MAX_MASK                                                           0x07FF0000L
+//PA_SC_VRS_RATE_CACHE_CNTL
+#define PA_SC_VRS_RATE_CACHE_CNTL__BIG_PAGE_RD__SHIFT                                                         0x0
+#define PA_SC_VRS_RATE_CACHE_CNTL__BIG_PAGE_WR__SHIFT                                                         0x1
+#define PA_SC_VRS_RATE_CACHE_CNTL__L1_RD_POLICY__SHIFT                                                        0x2
+#define PA_SC_VRS_RATE_CACHE_CNTL__L2_RD_POLICY__SHIFT                                                        0x4
+#define PA_SC_VRS_RATE_CACHE_CNTL__L2_WR_POLICY__SHIFT                                                        0x6
+#define PA_SC_VRS_RATE_CACHE_CNTL__LLC_RD_NOALLOC__SHIFT                                                      0x8
+#define PA_SC_VRS_RATE_CACHE_CNTL__LLC_WR_NOALLOC__SHIFT                                                      0x9
+#define PA_SC_VRS_RATE_CACHE_CNTL__NOFILL_RD__SHIFT                                                           0xa
+#define PA_SC_VRS_RATE_CACHE_CNTL__NOFILL_WR__SHIFT                                                           0xb
+#define PA_SC_VRS_RATE_CACHE_CNTL__PERF_CNTR_EN_RD__SHIFT                                                     0xc
+#define PA_SC_VRS_RATE_CACHE_CNTL__PERF_CNTR_EN_WR__SHIFT                                                     0xd
+#define PA_SC_VRS_RATE_CACHE_CNTL__BIG_PAGE_RD_MASK                                                           0x00000001L
+#define PA_SC_VRS_RATE_CACHE_CNTL__BIG_PAGE_WR_MASK                                                           0x00000002L
+#define PA_SC_VRS_RATE_CACHE_CNTL__L1_RD_POLICY_MASK                                                          0x0000000CL
+#define PA_SC_VRS_RATE_CACHE_CNTL__L2_RD_POLICY_MASK                                                          0x00000030L
+#define PA_SC_VRS_RATE_CACHE_CNTL__L2_WR_POLICY_MASK                                                          0x000000C0L
+#define PA_SC_VRS_RATE_CACHE_CNTL__LLC_RD_NOALLOC_MASK                                                        0x00000100L
+#define PA_SC_VRS_RATE_CACHE_CNTL__LLC_WR_NOALLOC_MASK                                                        0x00000200L
+#define PA_SC_VRS_RATE_CACHE_CNTL__NOFILL_RD_MASK                                                             0x00000400L
+#define PA_SC_VRS_RATE_CACHE_CNTL__NOFILL_WR_MASK                                                             0x00000800L
+#define PA_SC_VRS_RATE_CACHE_CNTL__PERF_CNTR_EN_RD_MASK                                                       0x00001000L
+#define PA_SC_VRS_RATE_CACHE_CNTL__PERF_CNTR_EN_WR_MASK                                                       0x00002000L
+//PA_SC_VRS_RATE_BASE
+#define PA_SC_VRS_RATE_BASE__BASE_256B__SHIFT                                                                 0x0
+#define PA_SC_VRS_RATE_BASE__BASE_256B_MASK                                                                   0xFFFFFFFFL
+//PA_SC_VRS_RATE_BASE_EXT
+#define PA_SC_VRS_RATE_BASE_EXT__BASE_256B__SHIFT                                                             0x0
+#define PA_SC_VRS_RATE_BASE_EXT__TB_SYNC_SIM_ID__SHIFT                                                        0x1c
+#define PA_SC_VRS_RATE_BASE_EXT__BASE_256B_MASK                                                               0x000000FFL
+#define PA_SC_VRS_RATE_BASE_EXT__TB_SYNC_SIM_ID_MASK                                                          0xF0000000L
+//PA_SC_VRS_RATE_SIZE_XY
+#define PA_SC_VRS_RATE_SIZE_XY__X_MAX__SHIFT                                                                  0x0
+#define PA_SC_VRS_RATE_SIZE_XY__Y_MAX__SHIFT                                                                  0x10
+#define PA_SC_VRS_RATE_SIZE_XY__X_MAX_MASK                                                                    0x000007FFL
+#define PA_SC_VRS_RATE_SIZE_XY__Y_MAX_MASK                                                                    0x07FF0000L
+//VGT_MULTI_PRIM_IB_RESET_INDX
+#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX__SHIFT                                                       0x0
+#define VGT_MULTI_PRIM_IB_RESET_INDX__RESET_INDX_MASK                                                         0xFFFFFFFFL
+//CB_RMI_GL2_CACHE_CONTROL
+#define CB_RMI_GL2_CACHE_CONTROL__DCC_WR_POLICY__SHIFT                                                        0x0
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_WR_POLICY__SHIFT                                                      0x2
+#define CB_RMI_GL2_CACHE_CONTROL__DCC_RD_POLICY__SHIFT                                                        0x14
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_RD_POLICY__SHIFT                                                      0x16
+#define CB_RMI_GL2_CACHE_CONTROL__DCC_L3_BYPASS__SHIFT                                                        0x1a
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_L3_BYPASS__SHIFT                                                      0x1b
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_BIG_PAGE__SHIFT                                                       0x1f
+#define CB_RMI_GL2_CACHE_CONTROL__DCC_WR_POLICY_MASK                                                          0x00000003L
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_WR_POLICY_MASK                                                        0x0000000CL
+#define CB_RMI_GL2_CACHE_CONTROL__DCC_RD_POLICY_MASK                                                          0x00300000L
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_RD_POLICY_MASK                                                        0x00C00000L
+#define CB_RMI_GL2_CACHE_CONTROL__DCC_L3_BYPASS_MASK                                                          0x04000000L
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_L3_BYPASS_MASK                                                        0x08000000L
+#define CB_RMI_GL2_CACHE_CONTROL__COLOR_BIG_PAGE_MASK                                                         0x80000000L
+//CB_BLEND_RED
+#define CB_BLEND_RED__BLEND_RED__SHIFT                                                                        0x0
+#define CB_BLEND_RED__BLEND_RED_MASK                                                                          0xFFFFFFFFL
+//CB_BLEND_GREEN
+#define CB_BLEND_GREEN__BLEND_GREEN__SHIFT                                                                    0x0
+#define CB_BLEND_GREEN__BLEND_GREEN_MASK                                                                      0xFFFFFFFFL
+//CB_BLEND_BLUE
+#define CB_BLEND_BLUE__BLEND_BLUE__SHIFT                                                                      0x0
+#define CB_BLEND_BLUE__BLEND_BLUE_MASK                                                                        0xFFFFFFFFL
+//CB_BLEND_ALPHA
+#define CB_BLEND_ALPHA__BLEND_ALPHA__SHIFT                                                                    0x0
+#define CB_BLEND_ALPHA__BLEND_ALPHA_MASK                                                                      0xFFFFFFFFL
+//CB_FDCC_CONTROL
+#define CB_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE__SHIFT                                                   0x0
+#define CB_FDCC_CONTROL__SAMPLE_MASK_TRACKER_WATERMARK__SHIFT                                                 0x2
+#define CB_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_AC01__SHIFT                                                  0x8
+#define CB_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_SINGLE__SHIFT                                                0x9
+#define CB_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                                   0xa
+#define CB_FDCC_CONTROL__DISABLE_ELIMFC_SKIP_OF_AC01__SHIFT                                                   0xc
+#define CB_FDCC_CONTROL__DISABLE_ELIMFC_SKIP_OF_SINGLE__SHIFT                                                 0xd
+#define CB_FDCC_CONTROL__ENABLE_ELIMFC_SKIP_OF_REG__SHIFT                                                     0xe
+#define CB_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE_MASK                                                     0x00000001L
+#define CB_FDCC_CONTROL__SAMPLE_MASK_TRACKER_WATERMARK_MASK                                                   0x0000007CL
+#define CB_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_AC01_MASK                                                    0x00000100L
+#define CB_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_SINGLE_MASK                                                  0x00000200L
+#define CB_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                                     0x00000400L
+#define CB_FDCC_CONTROL__DISABLE_ELIMFC_SKIP_OF_AC01_MASK                                                     0x00001000L
+#define CB_FDCC_CONTROL__DISABLE_ELIMFC_SKIP_OF_SINGLE_MASK                                                   0x00002000L
+#define CB_FDCC_CONTROL__ENABLE_ELIMFC_SKIP_OF_REG_MASK                                                       0x00004000L
+//CB_COVERAGE_OUT_CONTROL
+#define CB_COVERAGE_OUT_CONTROL__COVERAGE_OUT_ENABLE__SHIFT                                                   0x0
+#define CB_COVERAGE_OUT_CONTROL__COVERAGE_OUT_MRT__SHIFT                                                      0x1
+#define CB_COVERAGE_OUT_CONTROL__COVERAGE_OUT_CHANNEL__SHIFT                                                  0x4
+#define CB_COVERAGE_OUT_CONTROL__COVERAGE_OUT_SAMPLES__SHIFT                                                  0x8
+#define CB_COVERAGE_OUT_CONTROL__COVERAGE_OUT_ENABLE_MASK                                                     0x00000001L
+#define CB_COVERAGE_OUT_CONTROL__COVERAGE_OUT_MRT_MASK                                                        0x0000000EL
+#define CB_COVERAGE_OUT_CONTROL__COVERAGE_OUT_CHANNEL_MASK                                                    0x00000030L
+#define CB_COVERAGE_OUT_CONTROL__COVERAGE_OUT_SAMPLES_MASK                                                    0x00000F00L
+//DB_STENCIL_CONTROL
+#define DB_STENCIL_CONTROL__STENCILFAIL__SHIFT                                                                0x0
+#define DB_STENCIL_CONTROL__STENCILZPASS__SHIFT                                                               0x4
+#define DB_STENCIL_CONTROL__STENCILZFAIL__SHIFT                                                               0x8
+#define DB_STENCIL_CONTROL__STENCILFAIL_BF__SHIFT                                                             0xc
+#define DB_STENCIL_CONTROL__STENCILZPASS_BF__SHIFT                                                            0x10
+#define DB_STENCIL_CONTROL__STENCILZFAIL_BF__SHIFT                                                            0x14
+#define DB_STENCIL_CONTROL__STENCILFAIL_MASK                                                                  0x0000000FL
+#define DB_STENCIL_CONTROL__STENCILZPASS_MASK                                                                 0x000000F0L
+#define DB_STENCIL_CONTROL__STENCILZFAIL_MASK                                                                 0x00000F00L
+#define DB_STENCIL_CONTROL__STENCILFAIL_BF_MASK                                                               0x0000F000L
+#define DB_STENCIL_CONTROL__STENCILZPASS_BF_MASK                                                              0x000F0000L
+#define DB_STENCIL_CONTROL__STENCILZFAIL_BF_MASK                                                              0x00F00000L
+//DB_STENCILREFMASK
+#define DB_STENCILREFMASK__STENCILTESTVAL__SHIFT                                                              0x0
+#define DB_STENCILREFMASK__STENCILMASK__SHIFT                                                                 0x8
+#define DB_STENCILREFMASK__STENCILWRITEMASK__SHIFT                                                            0x10
+#define DB_STENCILREFMASK__STENCILOPVAL__SHIFT                                                                0x18
+#define DB_STENCILREFMASK__STENCILTESTVAL_MASK                                                                0x000000FFL
+#define DB_STENCILREFMASK__STENCILMASK_MASK                                                                   0x0000FF00L
+#define DB_STENCILREFMASK__STENCILWRITEMASK_MASK                                                              0x00FF0000L
+#define DB_STENCILREFMASK__STENCILOPVAL_MASK                                                                  0xFF000000L
+//DB_STENCILREFMASK_BF
+#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF__SHIFT                                                        0x0
+#define DB_STENCILREFMASK_BF__STENCILMASK_BF__SHIFT                                                           0x8
+#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF__SHIFT                                                      0x10
+#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF__SHIFT                                                          0x18
+#define DB_STENCILREFMASK_BF__STENCILTESTVAL_BF_MASK                                                          0x000000FFL
+#define DB_STENCILREFMASK_BF__STENCILMASK_BF_MASK                                                             0x0000FF00L
+#define DB_STENCILREFMASK_BF__STENCILWRITEMASK_BF_MASK                                                        0x00FF0000L
+#define DB_STENCILREFMASK_BF__STENCILOPVAL_BF_MASK                                                            0xFF000000L
+//PA_CL_VPORT_XSCALE
+#define PA_CL_VPORT_XSCALE__VPORT_XSCALE__SHIFT                                                               0x0
+#define PA_CL_VPORT_XSCALE__VPORT_XSCALE_MASK                                                                 0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET
+#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET__SHIFT                                                             0x0
+#define PA_CL_VPORT_XOFFSET__VPORT_XOFFSET_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE
+#define PA_CL_VPORT_YSCALE__VPORT_YSCALE__SHIFT                                                               0x0
+#define PA_CL_VPORT_YSCALE__VPORT_YSCALE_MASK                                                                 0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET
+#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET__SHIFT                                                             0x0
+#define PA_CL_VPORT_YOFFSET__VPORT_YOFFSET_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE
+#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE__SHIFT                                                               0x0
+#define PA_CL_VPORT_ZSCALE__VPORT_ZSCALE_MASK                                                                 0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET
+#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZOFFSET__VPORT_ZOFFSET_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_1
+#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_1__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_1
+#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_1__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_1
+#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_1__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_1
+#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_1__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_1
+#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_1__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_1
+#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_1__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_2
+#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_2__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_2
+#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_2__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_2
+#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_2__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_2
+#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_2__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_2
+#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_2__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_2
+#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_2__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_3
+#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_3__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_3
+#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_3__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_3
+#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_3__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_3
+#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_3__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_3
+#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_3__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_3
+#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_3__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_4
+#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_4__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_4
+#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_4__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_4
+#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_4__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_4
+#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_4__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_4
+#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_4__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_4
+#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_4__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_5
+#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_5__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_5
+#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_5__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_5
+#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_5__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_5
+#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_5__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_5
+#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_5__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_5
+#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_5__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_6
+#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_6__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_6
+#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_6__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_6
+#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_6__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_6
+#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_6__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_6
+#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_6__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_6
+#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_6__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_7
+#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_7__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_7
+#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_7__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_7
+#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_7__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_7
+#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_7__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_7
+#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_7__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_7
+#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_7__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_8
+#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_8__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_8
+#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_8__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_8
+#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_8__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_8
+#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_8__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_8
+#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_8__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_8
+#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_8__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_9
+#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_XSCALE_9__VPORT_XSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_9
+#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_XOFFSET_9__VPORT_XOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_9
+#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_YSCALE_9__VPORT_YSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_9
+#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_YOFFSET_9__VPORT_YOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_9
+#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE__SHIFT                                                             0x0
+#define PA_CL_VPORT_ZSCALE_9__VPORT_ZSCALE_MASK                                                               0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_9
+#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET__SHIFT                                                           0x0
+#define PA_CL_VPORT_ZOFFSET_9__VPORT_ZOFFSET_MASK                                                             0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_10
+#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_10__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_10
+#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_10__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_10
+#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_10__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_10
+#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_10__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_10
+#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_10__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_10
+#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_10__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_11
+#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_11__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_11
+#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_11__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_11
+#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_11__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_11
+#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_11__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_11
+#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_11__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_11
+#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_11__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_12
+#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_12__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_12
+#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_12__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_12
+#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_12__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_12
+#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_12__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_12
+#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_12__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_12
+#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_12__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_13
+#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_13__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_13
+#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_13__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_13
+#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_13__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_13
+#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_13__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_13
+#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_13__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_13
+#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_13__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_14
+#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_14__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_14
+#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_14__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_14
+#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_14__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_14
+#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_14__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_14
+#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_14__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_14
+#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_14__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_XSCALE_15
+#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_XSCALE_15__VPORT_XSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_XOFFSET_15
+#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_XOFFSET_15__VPORT_XOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_YSCALE_15
+#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_YSCALE_15__VPORT_YSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_YOFFSET_15
+#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_YOFFSET_15__VPORT_YOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_VPORT_ZSCALE_15
+#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE__SHIFT                                                            0x0
+#define PA_CL_VPORT_ZSCALE_15__VPORT_ZSCALE_MASK                                                              0xFFFFFFFFL
+//PA_CL_VPORT_ZOFFSET_15
+#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET__SHIFT                                                          0x0
+#define PA_CL_VPORT_ZOFFSET_15__VPORT_ZOFFSET_MASK                                                            0xFFFFFFFFL
+//PA_CL_UCP_0_X
+#define PA_CL_UCP_0_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_0_Y
+#define PA_CL_UCP_0_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_0_Z
+#define PA_CL_UCP_0_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_0_W
+#define PA_CL_UCP_0_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_0_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_X
+#define PA_CL_UCP_1_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_Y
+#define PA_CL_UCP_1_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_Z
+#define PA_CL_UCP_1_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_1_W
+#define PA_CL_UCP_1_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_1_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_X
+#define PA_CL_UCP_2_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_Y
+#define PA_CL_UCP_2_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_Z
+#define PA_CL_UCP_2_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_2_W
+#define PA_CL_UCP_2_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_2_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_X
+#define PA_CL_UCP_3_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_Y
+#define PA_CL_UCP_3_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_Z
+#define PA_CL_UCP_3_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_3_W
+#define PA_CL_UCP_3_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_3_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_X
+#define PA_CL_UCP_4_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_Y
+#define PA_CL_UCP_4_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_Z
+#define PA_CL_UCP_4_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_4_W
+#define PA_CL_UCP_4_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_4_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_X
+#define PA_CL_UCP_5_X__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_X__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_Y
+#define PA_CL_UCP_5_Y__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_Y__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_Z
+#define PA_CL_UCP_5_Z__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_Z__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_UCP_5_W
+#define PA_CL_UCP_5_W__DATA_REGISTER__SHIFT                                                                   0x0
+#define PA_CL_UCP_5_W__DATA_REGISTER_MASK                                                                     0xFFFFFFFFL
+//PA_CL_PROG_NEAR_CLIP_Z
+#define PA_CL_PROG_NEAR_CLIP_Z__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_PROG_NEAR_CLIP_Z__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_RATE_CNTL
+#define PA_RATE_CNTL__VERTEX_RATE__SHIFT                                                                      0x0
+#define PA_RATE_CNTL__PRIM_RATE__SHIFT                                                                        0x4
+#define PA_RATE_CNTL__VERTEX_RATE_MASK                                                                        0x0000000FL
+#define PA_RATE_CNTL__PRIM_RATE_MASK                                                                          0x000000F0L
+//SPI_PS_INPUT_CNTL_0
+#define SPI_PS_INPUT_CNTL_0__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_0__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_0__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_0__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_0__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_0__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_0__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_0__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_0__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_0__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_0__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_0__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_0__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_0__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_0__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_0__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_0__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_0__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_0__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_1
+#define SPI_PS_INPUT_CNTL_1__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_1__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_1__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_1__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_1__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_1__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_1__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_1__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_1__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_1__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_1__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_1__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_1__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_1__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_1__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_1__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_1__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_1__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_1__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_2
+#define SPI_PS_INPUT_CNTL_2__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_2__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_2__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_2__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_2__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_2__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_2__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_2__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_2__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_2__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_2__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_2__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_2__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_2__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_2__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_2__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_2__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_2__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_2__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_3
+#define SPI_PS_INPUT_CNTL_3__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_3__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_3__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_3__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_3__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_3__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_3__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_3__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_3__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_3__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_3__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_3__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_3__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_3__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_3__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_3__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_3__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_3__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_3__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_4
+#define SPI_PS_INPUT_CNTL_4__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_4__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_4__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_4__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_4__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_4__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_4__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_4__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_4__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_4__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_4__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_4__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_4__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_4__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_4__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_4__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_4__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_4__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_4__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_5
+#define SPI_PS_INPUT_CNTL_5__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_5__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_5__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_5__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_5__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_5__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_5__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_5__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_5__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_5__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_5__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_5__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_5__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_5__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_5__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_5__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_5__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_5__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_5__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_6
+#define SPI_PS_INPUT_CNTL_6__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_6__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_6__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_6__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_6__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_6__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_6__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_6__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_6__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_6__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_6__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_6__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_6__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_6__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_6__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_6__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_6__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_6__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_6__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_7
+#define SPI_PS_INPUT_CNTL_7__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_7__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_7__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_7__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_7__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_7__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_7__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_7__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_7__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_7__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_7__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_7__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_7__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_7__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_7__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_7__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_7__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_7__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_7__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_8
+#define SPI_PS_INPUT_CNTL_8__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_8__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_8__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_8__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_8__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_8__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_8__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_8__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_8__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_8__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_8__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_8__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_8__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_8__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_8__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_8__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_8__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_8__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_8__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_9
+#define SPI_PS_INPUT_CNTL_9__OFFSET__SHIFT                                                                    0x0
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL__SHIFT                                                               0x8
+#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE__SHIFT                                                                0xa
+#define SPI_PS_INPUT_CNTL_9__ROTATE_PC_PTR__SHIFT                                                             0xb
+#define SPI_PS_INPUT_CNTL_9__PRIM_ATTR__SHIFT                                                                 0xc
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX__SHIFT                                                             0x11
+#define SPI_PS_INPUT_CNTL_9__DUP__SHIFT                                                                       0x12
+#define SPI_PS_INPUT_CNTL_9__FP16_INTERP_MODE__SHIFT                                                          0x13
+#define SPI_PS_INPUT_CNTL_9__USE_DEFAULT_ATTR1__SHIFT                                                         0x14
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_ATTR1__SHIFT                                                         0x15
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_ATTR1__SHIFT                                                       0x17
+#define SPI_PS_INPUT_CNTL_9__ATTR0_VALID__SHIFT                                                               0x18
+#define SPI_PS_INPUT_CNTL_9__ATTR1_VALID__SHIFT                                                               0x19
+#define SPI_PS_INPUT_CNTL_9__OFFSET_MASK                                                                      0x0000003FL
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_MASK                                                                 0x00000300L
+#define SPI_PS_INPUT_CNTL_9__FLAT_SHADE_MASK                                                                  0x00000400L
+#define SPI_PS_INPUT_CNTL_9__ROTATE_PC_PTR_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_CNTL_9__PRIM_ATTR_MASK                                                                   0x00001000L
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_MASK                                                               0x00020000L
+#define SPI_PS_INPUT_CNTL_9__DUP_MASK                                                                         0x00040000L
+#define SPI_PS_INPUT_CNTL_9__FP16_INTERP_MODE_MASK                                                            0x00080000L
+#define SPI_PS_INPUT_CNTL_9__USE_DEFAULT_ATTR1_MASK                                                           0x00100000L
+#define SPI_PS_INPUT_CNTL_9__DEFAULT_VAL_ATTR1_MASK                                                           0x00600000L
+#define SPI_PS_INPUT_CNTL_9__PT_SPRITE_TEX_ATTR1_MASK                                                         0x00800000L
+#define SPI_PS_INPUT_CNTL_9__ATTR0_VALID_MASK                                                                 0x01000000L
+#define SPI_PS_INPUT_CNTL_9__ATTR1_VALID_MASK                                                                 0x02000000L
+//SPI_PS_INPUT_CNTL_10
+#define SPI_PS_INPUT_CNTL_10__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_10__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_10__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_10__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_10__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_10__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_10__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_10__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_10__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_10__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_10__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_10__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_10__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_10__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_10__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_10__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_10__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_10__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_10__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_11
+#define SPI_PS_INPUT_CNTL_11__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_11__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_11__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_11__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_11__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_11__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_11__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_11__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_11__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_11__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_11__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_11__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_11__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_11__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_11__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_11__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_11__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_11__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_11__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_12
+#define SPI_PS_INPUT_CNTL_12__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_12__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_12__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_12__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_12__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_12__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_12__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_12__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_12__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_12__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_12__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_12__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_12__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_12__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_12__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_12__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_12__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_12__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_12__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_13
+#define SPI_PS_INPUT_CNTL_13__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_13__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_13__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_13__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_13__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_13__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_13__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_13__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_13__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_13__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_13__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_13__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_13__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_13__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_13__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_13__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_13__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_13__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_13__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_14
+#define SPI_PS_INPUT_CNTL_14__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_14__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_14__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_14__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_14__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_14__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_14__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_14__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_14__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_14__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_14__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_14__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_14__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_14__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_14__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_14__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_14__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_14__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_14__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_15
+#define SPI_PS_INPUT_CNTL_15__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_15__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_15__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_15__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_15__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_15__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_15__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_15__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_15__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_15__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_15__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_15__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_15__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_15__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_15__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_15__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_15__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_15__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_15__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_16
+#define SPI_PS_INPUT_CNTL_16__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_16__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_16__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_16__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_16__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_16__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_16__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_16__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_16__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_16__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_16__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_16__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_16__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_16__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_16__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_16__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_16__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_16__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_16__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_17
+#define SPI_PS_INPUT_CNTL_17__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_17__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_17__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_17__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_17__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_17__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_17__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_17__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_17__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_17__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_17__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_17__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_17__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_17__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_17__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_17__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_17__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_17__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_17__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_18
+#define SPI_PS_INPUT_CNTL_18__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_18__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_18__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_18__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_18__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_18__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_18__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_18__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_18__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_18__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_18__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_18__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_18__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_18__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_18__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_18__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_18__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_18__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_18__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_19
+#define SPI_PS_INPUT_CNTL_19__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_19__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_19__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX__SHIFT                                                            0x11
+#define SPI_PS_INPUT_CNTL_19__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_19__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_19__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_ATTR1__SHIFT                                                      0x17
+#define SPI_PS_INPUT_CNTL_19__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_19__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_19__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_19__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_19__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_19__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_MASK                                                              0x00020000L
+#define SPI_PS_INPUT_CNTL_19__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_19__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_19__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_19__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_19__PT_SPRITE_TEX_ATTR1_MASK                                                        0x00800000L
+#define SPI_PS_INPUT_CNTL_19__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_19__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_20
+#define SPI_PS_INPUT_CNTL_20__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_20__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_20__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_20__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_20__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_20__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_20__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_20__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_20__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_20__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_20__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_20__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_20__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_20__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_20__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_20__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_20__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_20__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_21
+#define SPI_PS_INPUT_CNTL_21__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_21__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_21__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_21__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_21__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_21__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_21__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_21__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_21__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_21__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_21__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_21__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_21__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_21__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_21__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_21__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_21__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_21__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_22
+#define SPI_PS_INPUT_CNTL_22__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_22__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_22__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_22__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_22__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_22__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_22__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_22__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_22__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_22__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_22__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_22__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_22__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_22__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_22__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_22__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_22__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_22__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_23
+#define SPI_PS_INPUT_CNTL_23__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_23__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_23__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_23__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_23__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_23__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_23__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_23__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_23__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_23__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_23__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_23__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_23__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_23__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_23__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_23__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_23__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_23__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_24
+#define SPI_PS_INPUT_CNTL_24__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_24__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_24__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_24__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_24__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_24__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_24__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_24__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_24__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_24__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_24__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_24__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_24__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_24__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_24__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_24__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_24__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_24__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_25
+#define SPI_PS_INPUT_CNTL_25__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_25__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_25__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_25__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_25__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_25__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_25__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_25__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_25__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_25__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_25__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_25__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_25__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_25__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_25__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_25__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_25__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_25__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_26
+#define SPI_PS_INPUT_CNTL_26__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_26__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_26__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_26__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_26__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_26__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_26__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_26__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_26__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_26__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_26__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_26__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_26__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_26__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_26__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_26__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_26__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_26__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_27
+#define SPI_PS_INPUT_CNTL_27__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_27__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_27__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_27__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_27__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_27__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_27__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_27__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_27__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_27__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_27__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_27__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_27__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_27__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_27__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_27__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_27__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_27__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_28
+#define SPI_PS_INPUT_CNTL_28__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_28__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_28__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_28__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_28__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_28__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_28__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_28__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_28__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_28__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_28__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_28__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_28__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_28__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_28__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_28__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_28__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_28__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_29
+#define SPI_PS_INPUT_CNTL_29__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_29__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_29__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_29__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_29__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_29__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_29__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_29__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_29__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_29__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_29__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_29__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_29__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_29__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_29__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_29__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_29__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_29__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_30
+#define SPI_PS_INPUT_CNTL_30__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_30__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_30__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_30__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_30__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_30__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_30__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_30__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_30__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_30__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_30__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_30__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_30__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_30__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_30__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_30__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_30__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_30__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_PS_INPUT_CNTL_31
+#define SPI_PS_INPUT_CNTL_31__OFFSET__SHIFT                                                                   0x0
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL__SHIFT                                                              0x8
+#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE__SHIFT                                                               0xa
+#define SPI_PS_INPUT_CNTL_31__ROTATE_PC_PTR__SHIFT                                                            0xb
+#define SPI_PS_INPUT_CNTL_31__PRIM_ATTR__SHIFT                                                                0xc
+#define SPI_PS_INPUT_CNTL_31__DUP__SHIFT                                                                      0x12
+#define SPI_PS_INPUT_CNTL_31__FP16_INTERP_MODE__SHIFT                                                         0x13
+#define SPI_PS_INPUT_CNTL_31__USE_DEFAULT_ATTR1__SHIFT                                                        0x14
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_ATTR1__SHIFT                                                        0x15
+#define SPI_PS_INPUT_CNTL_31__ATTR0_VALID__SHIFT                                                              0x18
+#define SPI_PS_INPUT_CNTL_31__ATTR1_VALID__SHIFT                                                              0x19
+#define SPI_PS_INPUT_CNTL_31__OFFSET_MASK                                                                     0x0000003FL
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_MASK                                                                0x00000300L
+#define SPI_PS_INPUT_CNTL_31__FLAT_SHADE_MASK                                                                 0x00000400L
+#define SPI_PS_INPUT_CNTL_31__ROTATE_PC_PTR_MASK                                                              0x00000800L
+#define SPI_PS_INPUT_CNTL_31__PRIM_ATTR_MASK                                                                  0x00001000L
+#define SPI_PS_INPUT_CNTL_31__DUP_MASK                                                                        0x00040000L
+#define SPI_PS_INPUT_CNTL_31__FP16_INTERP_MODE_MASK                                                           0x00080000L
+#define SPI_PS_INPUT_CNTL_31__USE_DEFAULT_ATTR1_MASK                                                          0x00100000L
+#define SPI_PS_INPUT_CNTL_31__DEFAULT_VAL_ATTR1_MASK                                                          0x00600000L
+#define SPI_PS_INPUT_CNTL_31__ATTR0_VALID_MASK                                                                0x01000000L
+#define SPI_PS_INPUT_CNTL_31__ATTR1_VALID_MASK                                                                0x02000000L
+//SPI_VS_OUT_CONFIG
+#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT__SHIFT                                                             0x1
+#define SPI_VS_OUT_CONFIG__NO_PC_EXPORT__SHIFT                                                                0x7
+#define SPI_VS_OUT_CONFIG__PRIM_EXPORT_COUNT__SHIFT                                                           0x8
+#define SPI_VS_OUT_CONFIG__VS_EXPORT_COUNT_MASK                                                               0x0000003EL
+#define SPI_VS_OUT_CONFIG__NO_PC_EXPORT_MASK                                                                  0x00000080L
+#define SPI_VS_OUT_CONFIG__PRIM_EXPORT_COUNT_MASK                                                             0x00001F00L
+//SPI_PS_INPUT_ENA
+#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA__SHIFT                                                             0x0
+#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA__SHIFT                                                             0x1
+#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA__SHIFT                                                           0x2
+#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA__SHIFT                                                         0x3
+#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA__SHIFT                                                            0x4
+#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA__SHIFT                                                            0x5
+#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA__SHIFT                                                          0x6
+#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA__SHIFT                                                         0x7
+#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA__SHIFT                                                              0x8
+#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA__SHIFT                                                              0x9
+#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA__SHIFT                                                              0xa
+#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA__SHIFT                                                              0xb
+#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA__SHIFT                                                               0xc
+#define SPI_PS_INPUT_ENA__ANCILLARY_ENA__SHIFT                                                                0xd
+#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA__SHIFT                                                          0xe
+#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA__SHIFT                                                             0xf
+#define SPI_PS_INPUT_ENA__PERSP_SAMPLE_ENA_MASK                                                               0x00000001L
+#define SPI_PS_INPUT_ENA__PERSP_CENTER_ENA_MASK                                                               0x00000002L
+#define SPI_PS_INPUT_ENA__PERSP_CENTROID_ENA_MASK                                                             0x00000004L
+#define SPI_PS_INPUT_ENA__PERSP_PULL_MODEL_ENA_MASK                                                           0x00000008L
+#define SPI_PS_INPUT_ENA__LINEAR_SAMPLE_ENA_MASK                                                              0x00000010L
+#define SPI_PS_INPUT_ENA__LINEAR_CENTER_ENA_MASK                                                              0x00000020L
+#define SPI_PS_INPUT_ENA__LINEAR_CENTROID_ENA_MASK                                                            0x00000040L
+#define SPI_PS_INPUT_ENA__LINE_STIPPLE_TEX_ENA_MASK                                                           0x00000080L
+#define SPI_PS_INPUT_ENA__POS_X_FLOAT_ENA_MASK                                                                0x00000100L
+#define SPI_PS_INPUT_ENA__POS_Y_FLOAT_ENA_MASK                                                                0x00000200L
+#define SPI_PS_INPUT_ENA__POS_Z_FLOAT_ENA_MASK                                                                0x00000400L
+#define SPI_PS_INPUT_ENA__POS_W_FLOAT_ENA_MASK                                                                0x00000800L
+#define SPI_PS_INPUT_ENA__FRONT_FACE_ENA_MASK                                                                 0x00001000L
+#define SPI_PS_INPUT_ENA__ANCILLARY_ENA_MASK                                                                  0x00002000L
+#define SPI_PS_INPUT_ENA__SAMPLE_COVERAGE_ENA_MASK                                                            0x00004000L
+#define SPI_PS_INPUT_ENA__POS_FIXED_PT_ENA_MASK                                                               0x00008000L
+//SPI_PS_INPUT_ADDR
+#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA__SHIFT                                                            0x0
+#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA__SHIFT                                                            0x1
+#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA__SHIFT                                                          0x2
+#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA__SHIFT                                                        0x3
+#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA__SHIFT                                                           0x4
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA__SHIFT                                                           0x5
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA__SHIFT                                                         0x6
+#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA__SHIFT                                                        0x7
+#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA__SHIFT                                                             0x8
+#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA__SHIFT                                                             0x9
+#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA__SHIFT                                                             0xa
+#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA__SHIFT                                                             0xb
+#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA__SHIFT                                                              0xc
+#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA__SHIFT                                                               0xd
+#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA__SHIFT                                                         0xe
+#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA__SHIFT                                                            0xf
+#define SPI_PS_INPUT_ADDR__PERSP_SAMPLE_ENA_MASK                                                              0x00000001L
+#define SPI_PS_INPUT_ADDR__PERSP_CENTER_ENA_MASK                                                              0x00000002L
+#define SPI_PS_INPUT_ADDR__PERSP_CENTROID_ENA_MASK                                                            0x00000004L
+#define SPI_PS_INPUT_ADDR__PERSP_PULL_MODEL_ENA_MASK                                                          0x00000008L
+#define SPI_PS_INPUT_ADDR__LINEAR_SAMPLE_ENA_MASK                                                             0x00000010L
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTER_ENA_MASK                                                             0x00000020L
+#define SPI_PS_INPUT_ADDR__LINEAR_CENTROID_ENA_MASK                                                           0x00000040L
+#define SPI_PS_INPUT_ADDR__LINE_STIPPLE_TEX_ENA_MASK                                                          0x00000080L
+#define SPI_PS_INPUT_ADDR__POS_X_FLOAT_ENA_MASK                                                               0x00000100L
+#define SPI_PS_INPUT_ADDR__POS_Y_FLOAT_ENA_MASK                                                               0x00000200L
+#define SPI_PS_INPUT_ADDR__POS_Z_FLOAT_ENA_MASK                                                               0x00000400L
+#define SPI_PS_INPUT_ADDR__POS_W_FLOAT_ENA_MASK                                                               0x00000800L
+#define SPI_PS_INPUT_ADDR__FRONT_FACE_ENA_MASK                                                                0x00001000L
+#define SPI_PS_INPUT_ADDR__ANCILLARY_ENA_MASK                                                                 0x00002000L
+#define SPI_PS_INPUT_ADDR__SAMPLE_COVERAGE_ENA_MASK                                                           0x00004000L
+#define SPI_PS_INPUT_ADDR__POS_FIXED_PT_ENA_MASK                                                              0x00008000L
+//SPI_INTERP_CONTROL_0
+#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA__SHIFT                                                           0x0
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA__SHIFT                                                           0x1
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X__SHIFT                                                        0x2
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y__SHIFT                                                        0x5
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z__SHIFT                                                        0x8
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W__SHIFT                                                        0xb
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1__SHIFT                                                         0xe
+#define SPI_INTERP_CONTROL_0__FLAT_SHADE_ENA_MASK                                                             0x00000001L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_ENA_MASK                                                             0x00000002L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_X_MASK                                                          0x0000001CL
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Y_MASK                                                          0x000000E0L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_Z_MASK                                                          0x00000700L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_OVRD_W_MASK                                                          0x00003800L
+#define SPI_INTERP_CONTROL_0__PNT_SPRITE_TOP_1_MASK                                                           0x00004000L
+//SPI_PS_IN_CONTROL
+#define SPI_PS_IN_CONTROL__NUM_INTERP__SHIFT                                                                  0x0
+#define SPI_PS_IN_CONTROL__PARAM_GEN__SHIFT                                                                   0x6
+#define SPI_PS_IN_CONTROL__OFFCHIP_PARAM_EN__SHIFT                                                            0x7
+#define SPI_PS_IN_CONTROL__LATE_PC_DEALLOC__SHIFT                                                             0x8
+#define SPI_PS_IN_CONTROL__NUM_PRIM_INTERP__SHIFT                                                             0x9
+#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE__SHIFT                                                         0xe
+#define SPI_PS_IN_CONTROL__PS_W32_EN__SHIFT                                                                   0xf
+#define SPI_PS_IN_CONTROL__NUM_INTERP_MASK                                                                    0x0000003FL
+#define SPI_PS_IN_CONTROL__PARAM_GEN_MASK                                                                     0x00000040L
+#define SPI_PS_IN_CONTROL__OFFCHIP_PARAM_EN_MASK                                                              0x00000080L
+#define SPI_PS_IN_CONTROL__LATE_PC_DEALLOC_MASK                                                               0x00000100L
+#define SPI_PS_IN_CONTROL__NUM_PRIM_INTERP_MASK                                                               0x00003E00L
+#define SPI_PS_IN_CONTROL__BC_OPTIMIZE_DISABLE_MASK                                                           0x00004000L
+#define SPI_PS_IN_CONTROL__PS_W32_EN_MASK                                                                     0x00008000L
+//SPI_BARYC_CNTL
+#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL__SHIFT                                                              0x0
+#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL__SHIFT                                                            0x4
+#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL__SHIFT                                                             0x8
+#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL__SHIFT                                                           0xc
+#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION__SHIFT                                                             0x10
+#define SPI_BARYC_CNTL__POS_FLOAT_ULC__SHIFT                                                                  0x14
+#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS__SHIFT                                                            0x18
+#define SPI_BARYC_CNTL__PERSP_CENTER_CNTL_MASK                                                                0x00000001L
+#define SPI_BARYC_CNTL__PERSP_CENTROID_CNTL_MASK                                                              0x00000010L
+#define SPI_BARYC_CNTL__LINEAR_CENTER_CNTL_MASK                                                               0x00000100L
+#define SPI_BARYC_CNTL__LINEAR_CENTROID_CNTL_MASK                                                             0x00001000L
+#define SPI_BARYC_CNTL__POS_FLOAT_LOCATION_MASK                                                               0x00030000L
+#define SPI_BARYC_CNTL__POS_FLOAT_ULC_MASK                                                                    0x00100000L
+#define SPI_BARYC_CNTL__FRONT_FACE_ALL_BITS_MASK                                                              0x01000000L
+//SPI_TMPRING_SIZE
+#define SPI_TMPRING_SIZE__WAVES__SHIFT                                                                        0x0
+#define SPI_TMPRING_SIZE__WAVESIZE__SHIFT                                                                     0xc
+#define SPI_TMPRING_SIZE__WAVES_MASK                                                                          0x00000FFFL
+#define SPI_TMPRING_SIZE__WAVESIZE_MASK                                                                       0x07FFF000L
+//SPI_GFX_SCRATCH_BASE_LO
+#define SPI_GFX_SCRATCH_BASE_LO__DATA__SHIFT                                                                  0x0
+#define SPI_GFX_SCRATCH_BASE_LO__DATA_MASK                                                                    0xFFFFFFFFL
+//SPI_GFX_SCRATCH_BASE_HI
+#define SPI_GFX_SCRATCH_BASE_HI__DATA__SHIFT                                                                  0x0
+#define SPI_GFX_SCRATCH_BASE_HI__DATA_MASK                                                                    0x000000FFL
+//SPI_SHADER_IDX_FORMAT
+#define SPI_SHADER_IDX_FORMAT__IDX0_EXPORT_FORMAT__SHIFT                                                      0x0
+#define SPI_SHADER_IDX_FORMAT__IDX0_EXPORT_FORMAT_MASK                                                        0x0000000FL
+//SPI_SHADER_POS_FORMAT
+#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT__SHIFT                                                      0x0
+#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT__SHIFT                                                      0x4
+#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT__SHIFT                                                      0x8
+#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT__SHIFT                                                      0xc
+#define SPI_SHADER_POS_FORMAT__POS4_EXPORT_FORMAT__SHIFT                                                      0x10
+#define SPI_SHADER_POS_FORMAT__POS0_EXPORT_FORMAT_MASK                                                        0x0000000FL
+#define SPI_SHADER_POS_FORMAT__POS1_EXPORT_FORMAT_MASK                                                        0x000000F0L
+#define SPI_SHADER_POS_FORMAT__POS2_EXPORT_FORMAT_MASK                                                        0x00000F00L
+#define SPI_SHADER_POS_FORMAT__POS3_EXPORT_FORMAT_MASK                                                        0x0000F000L
+#define SPI_SHADER_POS_FORMAT__POS4_EXPORT_FORMAT_MASK                                                        0x000F0000L
+//SPI_SHADER_Z_FORMAT
+#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT__SHIFT                                                           0x0
+#define SPI_SHADER_Z_FORMAT__Z_EXPORT_FORMAT_MASK                                                             0x0000000FL
+//SPI_SHADER_COL_FORMAT
+#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT__SHIFT                                                      0x0
+#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT__SHIFT                                                      0x4
+#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT__SHIFT                                                      0x8
+#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT__SHIFT                                                      0xc
+#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT__SHIFT                                                      0x10
+#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT__SHIFT                                                      0x14
+#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT__SHIFT                                                      0x18
+#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT__SHIFT                                                      0x1c
+#define SPI_SHADER_COL_FORMAT__COL0_EXPORT_FORMAT_MASK                                                        0x0000000FL
+#define SPI_SHADER_COL_FORMAT__COL1_EXPORT_FORMAT_MASK                                                        0x000000F0L
+#define SPI_SHADER_COL_FORMAT__COL2_EXPORT_FORMAT_MASK                                                        0x00000F00L
+#define SPI_SHADER_COL_FORMAT__COL3_EXPORT_FORMAT_MASK                                                        0x0000F000L
+#define SPI_SHADER_COL_FORMAT__COL4_EXPORT_FORMAT_MASK                                                        0x000F0000L
+#define SPI_SHADER_COL_FORMAT__COL5_EXPORT_FORMAT_MASK                                                        0x00F00000L
+#define SPI_SHADER_COL_FORMAT__COL6_EXPORT_FORMAT_MASK                                                        0x0F000000L
+#define SPI_SHADER_COL_FORMAT__COL7_EXPORT_FORMAT_MASK                                                        0xF0000000L
+//SX_PS_DOWNCONVERT_CONTROL
+#define SX_PS_DOWNCONVERT_CONTROL__MRT0_FMT_MAPPING_DISABLE__SHIFT                                            0x0
+#define SX_PS_DOWNCONVERT_CONTROL__MRT1_FMT_MAPPING_DISABLE__SHIFT                                            0x1
+#define SX_PS_DOWNCONVERT_CONTROL__MRT2_FMT_MAPPING_DISABLE__SHIFT                                            0x2
+#define SX_PS_DOWNCONVERT_CONTROL__MRT3_FMT_MAPPING_DISABLE__SHIFT                                            0x3
+#define SX_PS_DOWNCONVERT_CONTROL__MRT4_FMT_MAPPING_DISABLE__SHIFT                                            0x4
+#define SX_PS_DOWNCONVERT_CONTROL__MRT5_FMT_MAPPING_DISABLE__SHIFT                                            0x5
+#define SX_PS_DOWNCONVERT_CONTROL__MRT6_FMT_MAPPING_DISABLE__SHIFT                                            0x6
+#define SX_PS_DOWNCONVERT_CONTROL__MRT7_FMT_MAPPING_DISABLE__SHIFT                                            0x7
+#define SX_PS_DOWNCONVERT_CONTROL__MRT0_FMT_MAPPING_DISABLE_MASK                                              0x00000001L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT1_FMT_MAPPING_DISABLE_MASK                                              0x00000002L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT2_FMT_MAPPING_DISABLE_MASK                                              0x00000004L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT3_FMT_MAPPING_DISABLE_MASK                                              0x00000008L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT4_FMT_MAPPING_DISABLE_MASK                                              0x00000010L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT5_FMT_MAPPING_DISABLE_MASK                                              0x00000020L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT6_FMT_MAPPING_DISABLE_MASK                                              0x00000040L
+#define SX_PS_DOWNCONVERT_CONTROL__MRT7_FMT_MAPPING_DISABLE_MASK                                              0x00000080L
+//SX_PS_DOWNCONVERT
+#define SX_PS_DOWNCONVERT__MRT0__SHIFT                                                                        0x0
+#define SX_PS_DOWNCONVERT__MRT1__SHIFT                                                                        0x4
+#define SX_PS_DOWNCONVERT__MRT2__SHIFT                                                                        0x8
+#define SX_PS_DOWNCONVERT__MRT3__SHIFT                                                                        0xc
+#define SX_PS_DOWNCONVERT__MRT4__SHIFT                                                                        0x10
+#define SX_PS_DOWNCONVERT__MRT5__SHIFT                                                                        0x14
+#define SX_PS_DOWNCONVERT__MRT6__SHIFT                                                                        0x18
+#define SX_PS_DOWNCONVERT__MRT7__SHIFT                                                                        0x1c
+#define SX_PS_DOWNCONVERT__MRT0_MASK                                                                          0x0000000FL
+#define SX_PS_DOWNCONVERT__MRT1_MASK                                                                          0x000000F0L
+#define SX_PS_DOWNCONVERT__MRT2_MASK                                                                          0x00000F00L
+#define SX_PS_DOWNCONVERT__MRT3_MASK                                                                          0x0000F000L
+#define SX_PS_DOWNCONVERT__MRT4_MASK                                                                          0x000F0000L
+#define SX_PS_DOWNCONVERT__MRT5_MASK                                                                          0x00F00000L
+#define SX_PS_DOWNCONVERT__MRT6_MASK                                                                          0x0F000000L
+#define SX_PS_DOWNCONVERT__MRT7_MASK                                                                          0xF0000000L
+//SX_BLEND_OPT_EPSILON
+#define SX_BLEND_OPT_EPSILON__MRT0_EPSILON__SHIFT                                                             0x0
+#define SX_BLEND_OPT_EPSILON__MRT1_EPSILON__SHIFT                                                             0x4
+#define SX_BLEND_OPT_EPSILON__MRT2_EPSILON__SHIFT                                                             0x8
+#define SX_BLEND_OPT_EPSILON__MRT3_EPSILON__SHIFT                                                             0xc
+#define SX_BLEND_OPT_EPSILON__MRT4_EPSILON__SHIFT                                                             0x10
+#define SX_BLEND_OPT_EPSILON__MRT5_EPSILON__SHIFT                                                             0x14
+#define SX_BLEND_OPT_EPSILON__MRT6_EPSILON__SHIFT                                                             0x18
+#define SX_BLEND_OPT_EPSILON__MRT7_EPSILON__SHIFT                                                             0x1c
+#define SX_BLEND_OPT_EPSILON__MRT0_EPSILON_MASK                                                               0x0000000FL
+#define SX_BLEND_OPT_EPSILON__MRT1_EPSILON_MASK                                                               0x000000F0L
+#define SX_BLEND_OPT_EPSILON__MRT2_EPSILON_MASK                                                               0x00000F00L
+#define SX_BLEND_OPT_EPSILON__MRT3_EPSILON_MASK                                                               0x0000F000L
+#define SX_BLEND_OPT_EPSILON__MRT4_EPSILON_MASK                                                               0x000F0000L
+#define SX_BLEND_OPT_EPSILON__MRT5_EPSILON_MASK                                                               0x00F00000L
+#define SX_BLEND_OPT_EPSILON__MRT6_EPSILON_MASK                                                               0x0F000000L
+#define SX_BLEND_OPT_EPSILON__MRT7_EPSILON_MASK                                                               0xF0000000L
+//SX_BLEND_OPT_CONTROL
+#define SX_BLEND_OPT_CONTROL__MRT0_COLOR_OPT_DISABLE__SHIFT                                                   0x0
+#define SX_BLEND_OPT_CONTROL__MRT0_ALPHA_OPT_DISABLE__SHIFT                                                   0x1
+#define SX_BLEND_OPT_CONTROL__MRT1_COLOR_OPT_DISABLE__SHIFT                                                   0x4
+#define SX_BLEND_OPT_CONTROL__MRT1_ALPHA_OPT_DISABLE__SHIFT                                                   0x5
+#define SX_BLEND_OPT_CONTROL__MRT2_COLOR_OPT_DISABLE__SHIFT                                                   0x8
+#define SX_BLEND_OPT_CONTROL__MRT2_ALPHA_OPT_DISABLE__SHIFT                                                   0x9
+#define SX_BLEND_OPT_CONTROL__MRT3_COLOR_OPT_DISABLE__SHIFT                                                   0xc
+#define SX_BLEND_OPT_CONTROL__MRT3_ALPHA_OPT_DISABLE__SHIFT                                                   0xd
+#define SX_BLEND_OPT_CONTROL__MRT4_COLOR_OPT_DISABLE__SHIFT                                                   0x10
+#define SX_BLEND_OPT_CONTROL__MRT4_ALPHA_OPT_DISABLE__SHIFT                                                   0x11
+#define SX_BLEND_OPT_CONTROL__MRT5_COLOR_OPT_DISABLE__SHIFT                                                   0x14
+#define SX_BLEND_OPT_CONTROL__MRT5_ALPHA_OPT_DISABLE__SHIFT                                                   0x15
+#define SX_BLEND_OPT_CONTROL__MRT6_COLOR_OPT_DISABLE__SHIFT                                                   0x18
+#define SX_BLEND_OPT_CONTROL__MRT6_ALPHA_OPT_DISABLE__SHIFT                                                   0x19
+#define SX_BLEND_OPT_CONTROL__MRT7_COLOR_OPT_DISABLE__SHIFT                                                   0x1c
+#define SX_BLEND_OPT_CONTROL__MRT7_ALPHA_OPT_DISABLE__SHIFT                                                   0x1d
+#define SX_BLEND_OPT_CONTROL__PIXEN_ZERO_OPT_DISABLE__SHIFT                                                   0x1f
+#define SX_BLEND_OPT_CONTROL__MRT0_COLOR_OPT_DISABLE_MASK                                                     0x00000001L
+#define SX_BLEND_OPT_CONTROL__MRT0_ALPHA_OPT_DISABLE_MASK                                                     0x00000002L
+#define SX_BLEND_OPT_CONTROL__MRT1_COLOR_OPT_DISABLE_MASK                                                     0x00000010L
+#define SX_BLEND_OPT_CONTROL__MRT1_ALPHA_OPT_DISABLE_MASK                                                     0x00000020L
+#define SX_BLEND_OPT_CONTROL__MRT2_COLOR_OPT_DISABLE_MASK                                                     0x00000100L
+#define SX_BLEND_OPT_CONTROL__MRT2_ALPHA_OPT_DISABLE_MASK                                                     0x00000200L
+#define SX_BLEND_OPT_CONTROL__MRT3_COLOR_OPT_DISABLE_MASK                                                     0x00001000L
+#define SX_BLEND_OPT_CONTROL__MRT3_ALPHA_OPT_DISABLE_MASK                                                     0x00002000L
+#define SX_BLEND_OPT_CONTROL__MRT4_COLOR_OPT_DISABLE_MASK                                                     0x00010000L
+#define SX_BLEND_OPT_CONTROL__MRT4_ALPHA_OPT_DISABLE_MASK                                                     0x00020000L
+#define SX_BLEND_OPT_CONTROL__MRT5_COLOR_OPT_DISABLE_MASK                                                     0x00100000L
+#define SX_BLEND_OPT_CONTROL__MRT5_ALPHA_OPT_DISABLE_MASK                                                     0x00200000L
+#define SX_BLEND_OPT_CONTROL__MRT6_COLOR_OPT_DISABLE_MASK                                                     0x01000000L
+#define SX_BLEND_OPT_CONTROL__MRT6_ALPHA_OPT_DISABLE_MASK                                                     0x02000000L
+#define SX_BLEND_OPT_CONTROL__MRT7_COLOR_OPT_DISABLE_MASK                                                     0x10000000L
+#define SX_BLEND_OPT_CONTROL__MRT7_ALPHA_OPT_DISABLE_MASK                                                     0x20000000L
+#define SX_BLEND_OPT_CONTROL__PIXEN_ZERO_OPT_DISABLE_MASK                                                     0x80000000L
+//SX_MRT0_BLEND_OPT
+#define SX_MRT0_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT0_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT0_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT0_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT0_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT0_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT0_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT0_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT0_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT0_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT0_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT0_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT1_BLEND_OPT
+#define SX_MRT1_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT1_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT1_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT1_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT1_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT1_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT1_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT1_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT1_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT1_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT1_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT1_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT2_BLEND_OPT
+#define SX_MRT2_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT2_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT2_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT2_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT2_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT2_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT2_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT2_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT2_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT2_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT2_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT2_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT3_BLEND_OPT
+#define SX_MRT3_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT3_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT3_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT3_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT3_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT3_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT3_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT3_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT3_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT3_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT3_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT3_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT4_BLEND_OPT
+#define SX_MRT4_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT4_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT4_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT4_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT4_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT4_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT4_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT4_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT4_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT4_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT4_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT4_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT5_BLEND_OPT
+#define SX_MRT5_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT5_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT5_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT5_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT5_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT5_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT5_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT5_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT5_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT5_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT5_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT5_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT6_BLEND_OPT
+#define SX_MRT6_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT6_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT6_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT6_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT6_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT6_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT6_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT6_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT6_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT6_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT6_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT6_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//SX_MRT7_BLEND_OPT
+#define SX_MRT7_BLEND_OPT__COLOR_SRC_OPT__SHIFT                                                               0x0
+#define SX_MRT7_BLEND_OPT__COLOR_DST_OPT__SHIFT                                                               0x4
+#define SX_MRT7_BLEND_OPT__COLOR_COMB_FCN__SHIFT                                                              0x8
+#define SX_MRT7_BLEND_OPT__ALPHA_SRC_OPT__SHIFT                                                               0x10
+#define SX_MRT7_BLEND_OPT__ALPHA_DST_OPT__SHIFT                                                               0x14
+#define SX_MRT7_BLEND_OPT__ALPHA_COMB_FCN__SHIFT                                                              0x18
+#define SX_MRT7_BLEND_OPT__COLOR_SRC_OPT_MASK                                                                 0x00000007L
+#define SX_MRT7_BLEND_OPT__COLOR_DST_OPT_MASK                                                                 0x00000070L
+#define SX_MRT7_BLEND_OPT__COLOR_COMB_FCN_MASK                                                                0x00000700L
+#define SX_MRT7_BLEND_OPT__ALPHA_SRC_OPT_MASK                                                                 0x00070000L
+#define SX_MRT7_BLEND_OPT__ALPHA_DST_OPT_MASK                                                                 0x00700000L
+#define SX_MRT7_BLEND_OPT__ALPHA_COMB_FCN_MASK                                                                0x07000000L
+//CB_BLEND0_CONTROL
+#define CB_BLEND0_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND0_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND0_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND0_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND0_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND0_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND0_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND0_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND0_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND0_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND0_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND0_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND0_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND0_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND1_CONTROL
+#define CB_BLEND1_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND1_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND1_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND1_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND1_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND1_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND1_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND1_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND1_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND1_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND1_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND1_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND1_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND1_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND2_CONTROL
+#define CB_BLEND2_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND2_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND2_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND2_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND2_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND2_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND2_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND2_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND2_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND2_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND2_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND2_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND2_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND2_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND3_CONTROL
+#define CB_BLEND3_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND3_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND3_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND3_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND3_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND3_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND3_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND3_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND3_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND3_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND3_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND3_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND3_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND3_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND4_CONTROL
+#define CB_BLEND4_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND4_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND4_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND4_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND4_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND4_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND4_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND4_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND4_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND4_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND4_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND4_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND4_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND4_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND5_CONTROL
+#define CB_BLEND5_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND5_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND5_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND5_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND5_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND5_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND5_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND5_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND5_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND5_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND5_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND5_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND5_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND5_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND6_CONTROL
+#define CB_BLEND6_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND6_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND6_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND6_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND6_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND6_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND6_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND6_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND6_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND6_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND6_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND6_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND6_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND6_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//CB_BLEND7_CONTROL
+#define CB_BLEND7_CONTROL__COLOR_SRCBLEND__SHIFT                                                              0x0
+#define CB_BLEND7_CONTROL__COLOR_COMB_FCN__SHIFT                                                              0x5
+#define CB_BLEND7_CONTROL__COLOR_DESTBLEND__SHIFT                                                             0x8
+#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND__SHIFT                                                              0x10
+#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN__SHIFT                                                              0x15
+#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND__SHIFT                                                             0x18
+#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND__SHIFT                                                        0x1d
+#define CB_BLEND7_CONTROL__ENABLE__SHIFT                                                                      0x1e
+#define CB_BLEND7_CONTROL__DISABLE_ROP3__SHIFT                                                                0x1f
+#define CB_BLEND7_CONTROL__COLOR_SRCBLEND_MASK                                                                0x0000001FL
+#define CB_BLEND7_CONTROL__COLOR_COMB_FCN_MASK                                                                0x000000E0L
+#define CB_BLEND7_CONTROL__COLOR_DESTBLEND_MASK                                                               0x00001F00L
+#define CB_BLEND7_CONTROL__ALPHA_SRCBLEND_MASK                                                                0x001F0000L
+#define CB_BLEND7_CONTROL__ALPHA_COMB_FCN_MASK                                                                0x00E00000L
+#define CB_BLEND7_CONTROL__ALPHA_DESTBLEND_MASK                                                               0x1F000000L
+#define CB_BLEND7_CONTROL__SEPARATE_ALPHA_BLEND_MASK                                                          0x20000000L
+#define CB_BLEND7_CONTROL__ENABLE_MASK                                                                        0x40000000L
+#define CB_BLEND7_CONTROL__DISABLE_ROP3_MASK                                                                  0x80000000L
+//GFX_COPY_STATE
+#define GFX_COPY_STATE__SRC_STATE_ID__SHIFT                                                                   0x0
+#define GFX_COPY_STATE__SRC_STATE_ID_MASK                                                                     0x00000007L
+//PA_CL_POINT_X_RAD
+#define PA_CL_POINT_X_RAD__DATA_REGISTER__SHIFT                                                               0x0
+#define PA_CL_POINT_X_RAD__DATA_REGISTER_MASK                                                                 0xFFFFFFFFL
+//PA_CL_POINT_Y_RAD
+#define PA_CL_POINT_Y_RAD__DATA_REGISTER__SHIFT                                                               0x0
+#define PA_CL_POINT_Y_RAD__DATA_REGISTER_MASK                                                                 0xFFFFFFFFL
+//PA_CL_POINT_SIZE
+#define PA_CL_POINT_SIZE__DATA_REGISTER__SHIFT                                                                0x0
+#define PA_CL_POINT_SIZE__DATA_REGISTER_MASK                                                                  0xFFFFFFFFL
+//PA_CL_POINT_CULL_RAD
+#define PA_CL_POINT_CULL_RAD__DATA_REGISTER__SHIFT                                                            0x0
+#define PA_CL_POINT_CULL_RAD__DATA_REGISTER_MASK                                                              0xFFFFFFFFL
+//VGT_DMA_BASE_HI
+#define VGT_DMA_BASE_HI__BASE_ADDR__SHIFT                                                                     0x0
+#define VGT_DMA_BASE_HI__BASE_ADDR_MASK                                                                       0x0000FFFFL
+//VGT_DMA_BASE
+#define VGT_DMA_BASE__BASE_ADDR__SHIFT                                                                        0x0
+#define VGT_DMA_BASE__BASE_ADDR_MASK                                                                          0xFFFFFFFFL
+//VGT_DRAW_INITIATOR
+#define VGT_DRAW_INITIATOR__SOURCE_SELECT__SHIFT                                                              0x0
+#define VGT_DRAW_INITIATOR__MAJOR_MODE__SHIFT                                                                 0x2
+#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX__SHIFT                                                             0x4
+#define VGT_DRAW_INITIATOR__NOT_EOP__SHIFT                                                                    0x5
+#define VGT_DRAW_INITIATOR__USE_OPAQUE__SHIFT                                                                 0x6
+#define VGT_DRAW_INITIATOR__REG_RT_INDEX__SHIFT                                                               0x1d
+#define VGT_DRAW_INITIATOR__SOURCE_SELECT_MASK                                                                0x00000003L
+#define VGT_DRAW_INITIATOR__MAJOR_MODE_MASK                                                                   0x0000000CL
+#define VGT_DRAW_INITIATOR__SPRITE_EN_R6XX_MASK                                                               0x00000010L
+#define VGT_DRAW_INITIATOR__NOT_EOP_MASK                                                                      0x00000020L
+#define VGT_DRAW_INITIATOR__USE_OPAQUE_MASK                                                                   0x00000040L
+#define VGT_DRAW_INITIATOR__REG_RT_INDEX_MASK                                                                 0xE0000000L
+//VGT_EVENT_ADDRESS_REG
+#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW__SHIFT                                                             0x0
+#define VGT_EVENT_ADDRESS_REG__ADDRESS_LOW_MASK                                                               0x0FFFFFFFL
+//GE_MAX_OUTPUT_PER_SUBGROUP
+#define GE_MAX_OUTPUT_PER_SUBGROUP__MAX_VERTS_PER_SUBGROUP__SHIFT                                             0x0
+#define GE_MAX_OUTPUT_PER_SUBGROUP__MAX_VERTS_PER_SUBGROUP_MASK                                               0x000003FFL
+//DB_DEPTH_CONTROL
+#define DB_DEPTH_CONTROL__STENCIL_ENABLE__SHIFT                                                               0x0
+#define DB_DEPTH_CONTROL__Z_ENABLE__SHIFT                                                                     0x1
+#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE__SHIFT                                                               0x2
+#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE__SHIFT                                                          0x3
+#define DB_DEPTH_CONTROL__ZFUNC__SHIFT                                                                        0x4
+#define DB_DEPTH_CONTROL__BACKFACE_ENABLE__SHIFT                                                              0x7
+#define DB_DEPTH_CONTROL__STENCILFUNC__SHIFT                                                                  0x8
+#define DB_DEPTH_CONTROL__STENCILFUNC_BF__SHIFT                                                               0x14
+#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL__SHIFT                                            0x1e
+#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS__SHIFT                                           0x1f
+#define DB_DEPTH_CONTROL__STENCIL_ENABLE_MASK                                                                 0x00000001L
+#define DB_DEPTH_CONTROL__Z_ENABLE_MASK                                                                       0x00000002L
+#define DB_DEPTH_CONTROL__Z_WRITE_ENABLE_MASK                                                                 0x00000004L
+#define DB_DEPTH_CONTROL__DEPTH_BOUNDS_ENABLE_MASK                                                            0x00000008L
+#define DB_DEPTH_CONTROL__ZFUNC_MASK                                                                          0x00000070L
+#define DB_DEPTH_CONTROL__BACKFACE_ENABLE_MASK                                                                0x00000080L
+#define DB_DEPTH_CONTROL__STENCILFUNC_MASK                                                                    0x00000700L
+#define DB_DEPTH_CONTROL__STENCILFUNC_BF_MASK                                                                 0x00700000L
+#define DB_DEPTH_CONTROL__ENABLE_COLOR_WRITES_ON_DEPTH_FAIL_MASK                                              0x40000000L
+#define DB_DEPTH_CONTROL__DISABLE_COLOR_WRITES_ON_DEPTH_PASS_MASK                                             0x80000000L
+//DB_EQAA
+#define DB_EQAA__MAX_ANCHOR_SAMPLES__SHIFT                                                                    0x0
+#define DB_EQAA__PS_ITER_SAMPLES__SHIFT                                                                       0x4
+#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES__SHIFT                                                               0x8
+#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES__SHIFT                                                             0xc
+#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS__SHIFT                                                            0x10
+#define DB_EQAA__INCOHERENT_EQAA_READS__SHIFT                                                                 0x11
+#define DB_EQAA__INTERPOLATE_COMP_Z__SHIFT                                                                    0x12
+#define DB_EQAA__INTERPOLATE_SRC_Z__SHIFT                                                                     0x13
+#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS__SHIFT                                                            0x14
+#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE__SHIFT                                                            0x15
+#define DB_EQAA__OVERRASTERIZATION_AMOUNT__SHIFT                                                              0x18
+#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION__SHIFT                                                        0x1b
+#define DB_EQAA__MAX_ANCHOR_SAMPLES_MASK                                                                      0x00000007L
+#define DB_EQAA__PS_ITER_SAMPLES_MASK                                                                         0x00000070L
+#define DB_EQAA__MASK_EXPORT_NUM_SAMPLES_MASK                                                                 0x00000700L
+#define DB_EQAA__ALPHA_TO_MASK_NUM_SAMPLES_MASK                                                               0x00007000L
+#define DB_EQAA__HIGH_QUALITY_INTERSECTIONS_MASK                                                              0x00010000L
+#define DB_EQAA__INCOHERENT_EQAA_READS_MASK                                                                   0x00020000L
+#define DB_EQAA__INTERPOLATE_COMP_Z_MASK                                                                      0x00040000L
+#define DB_EQAA__INTERPOLATE_SRC_Z_MASK                                                                       0x00080000L
+#define DB_EQAA__STATIC_ANCHOR_ASSOCIATIONS_MASK                                                              0x00100000L
+#define DB_EQAA__ALPHA_TO_MASK_EQAA_DISABLE_MASK                                                              0x00200000L
+#define DB_EQAA__OVERRASTERIZATION_AMOUNT_MASK                                                                0x07000000L
+#define DB_EQAA__ENABLE_POSTZ_OVERRASTERIZATION_MASK                                                          0x08000000L
+//CB_COLOR_CONTROL
+#define CB_COLOR_CONTROL__DISABLE_DUAL_QUAD__SHIFT                                                            0x0
+#define CB_COLOR_CONTROL__ENABLE_1FRAG_PS_INVOKE__SHIFT                                                       0x1
+#define CB_COLOR_CONTROL__DEGAMMA_ENABLE__SHIFT                                                               0x3
+#define CB_COLOR_CONTROL__MODE__SHIFT                                                                         0x4
+#define CB_COLOR_CONTROL__ROP3__SHIFT                                                                         0x10
+#define CB_COLOR_CONTROL__DISABLE_DUAL_QUAD_MASK                                                              0x00000001L
+#define CB_COLOR_CONTROL__ENABLE_1FRAG_PS_INVOKE_MASK                                                         0x00000002L
+#define CB_COLOR_CONTROL__DEGAMMA_ENABLE_MASK                                                                 0x00000008L
+#define CB_COLOR_CONTROL__MODE_MASK                                                                           0x00000070L
+#define CB_COLOR_CONTROL__ROP3_MASK                                                                           0x00FF0000L
+//DB_SHADER_CONTROL
+#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE__SHIFT                                                             0x0
+#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE__SHIFT                                              0x1
+#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE__SHIFT                                                0x2
+#define DB_SHADER_CONTROL__Z_ORDER__SHIFT                                                                     0x4
+#define DB_SHADER_CONTROL__KILL_ENABLE__SHIFT                                                                 0x6
+#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE__SHIFT                                                     0x7
+#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE__SHIFT                                                          0x8
+#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL__SHIFT                                                           0x9
+#define DB_SHADER_CONTROL__EXEC_ON_NOOP__SHIFT                                                                0xa
+#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE__SHIFT                                                       0xb
+#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER__SHIFT                                                         0xc
+#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT__SHIFT                                                       0xd
+#define DB_SHADER_CONTROL__DUAL_QUAD_DISABLE__SHIFT                                                           0xf
+#define DB_SHADER_CONTROL__PRIMITIVE_ORDERED_PIXEL_SHADER__SHIFT                                              0x10
+#define DB_SHADER_CONTROL__PRE_SHADER_DEPTH_COVERAGE_ENABLE__SHIFT                                            0x17
+#define DB_SHADER_CONTROL__OREO_BLEND_ENABLE__SHIFT                                                           0x18
+#define DB_SHADER_CONTROL__OVERRIDE_INTRINSIC_RATE_ENABLE__SHIFT                                              0x19
+#define DB_SHADER_CONTROL__OVERRIDE_INTRINSIC_RATE__SHIFT                                                     0x1a
+#define DB_SHADER_CONTROL__Z_EXPORT_ENABLE_MASK                                                               0x00000001L
+#define DB_SHADER_CONTROL__STENCIL_TEST_VAL_EXPORT_ENABLE_MASK                                                0x00000002L
+#define DB_SHADER_CONTROL__STENCIL_OP_VAL_EXPORT_ENABLE_MASK                                                  0x00000004L
+#define DB_SHADER_CONTROL__Z_ORDER_MASK                                                                       0x00000030L
+#define DB_SHADER_CONTROL__KILL_ENABLE_MASK                                                                   0x00000040L
+#define DB_SHADER_CONTROL__COVERAGE_TO_MASK_ENABLE_MASK                                                       0x00000080L
+#define DB_SHADER_CONTROL__MASK_EXPORT_ENABLE_MASK                                                            0x00000100L
+#define DB_SHADER_CONTROL__EXEC_ON_HIER_FAIL_MASK                                                             0x00000200L
+#define DB_SHADER_CONTROL__EXEC_ON_NOOP_MASK                                                                  0x00000400L
+#define DB_SHADER_CONTROL__ALPHA_TO_MASK_DISABLE_MASK                                                         0x00000800L
+#define DB_SHADER_CONTROL__DEPTH_BEFORE_SHADER_MASK                                                           0x00001000L
+#define DB_SHADER_CONTROL__CONSERVATIVE_Z_EXPORT_MASK                                                         0x00006000L
+#define DB_SHADER_CONTROL__DUAL_QUAD_DISABLE_MASK                                                             0x00008000L
+#define DB_SHADER_CONTROL__PRIMITIVE_ORDERED_PIXEL_SHADER_MASK                                                0x00010000L
+#define DB_SHADER_CONTROL__PRE_SHADER_DEPTH_COVERAGE_ENABLE_MASK                                              0x00800000L
+#define DB_SHADER_CONTROL__OREO_BLEND_ENABLE_MASK                                                             0x01000000L
+#define DB_SHADER_CONTROL__OVERRIDE_INTRINSIC_RATE_ENABLE_MASK                                                0x02000000L
+#define DB_SHADER_CONTROL__OVERRIDE_INTRINSIC_RATE_MASK                                                       0x1C000000L
+//PA_CL_CLIP_CNTL
+#define PA_CL_CLIP_CNTL__UCP_ENA_0__SHIFT                                                                     0x0
+#define PA_CL_CLIP_CNTL__UCP_ENA_1__SHIFT                                                                     0x1
+#define PA_CL_CLIP_CNTL__UCP_ENA_2__SHIFT                                                                     0x2
+#define PA_CL_CLIP_CNTL__UCP_ENA_3__SHIFT                                                                     0x3
+#define PA_CL_CLIP_CNTL__UCP_ENA_4__SHIFT                                                                     0x4
+#define PA_CL_CLIP_CNTL__UCP_ENA_5__SHIFT                                                                     0x5
+#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG__SHIFT                                                            0xd
+#define PA_CL_CLIP_CNTL__PS_UCP_MODE__SHIFT                                                                   0xe
+#define PA_CL_CLIP_CNTL__CLIP_DISABLE__SHIFT                                                                  0x10
+#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA__SHIFT                                                             0x11
+#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA__SHIFT                                                        0x12
+#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF__SHIFT                                                             0x13
+#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT__SHIFT                                                           0x14
+#define PA_CL_CLIP_CNTL__VTX_KILL_OR__SHIFT                                                                   0x15
+#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL__SHIFT                                                         0x16
+#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA__SHIFT                                                       0x18
+#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE__SHIFT                                                     0x19
+#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE__SHIFT                                                            0x1a
+#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE__SHIFT                                                             0x1b
+#define PA_CL_CLIP_CNTL__ZCLIP_PROG_NEAR_ENA__SHIFT                                                           0x1c
+#define PA_CL_CLIP_CNTL__UCP_ENA_0_MASK                                                                       0x00000001L
+#define PA_CL_CLIP_CNTL__UCP_ENA_1_MASK                                                                       0x00000002L
+#define PA_CL_CLIP_CNTL__UCP_ENA_2_MASK                                                                       0x00000004L
+#define PA_CL_CLIP_CNTL__UCP_ENA_3_MASK                                                                       0x00000008L
+#define PA_CL_CLIP_CNTL__UCP_ENA_4_MASK                                                                       0x00000010L
+#define PA_CL_CLIP_CNTL__UCP_ENA_5_MASK                                                                       0x00000020L
+#define PA_CL_CLIP_CNTL__PS_UCP_Y_SCALE_NEG_MASK                                                              0x00002000L
+#define PA_CL_CLIP_CNTL__PS_UCP_MODE_MASK                                                                     0x0000C000L
+#define PA_CL_CLIP_CNTL__CLIP_DISABLE_MASK                                                                    0x00010000L
+#define PA_CL_CLIP_CNTL__UCP_CULL_ONLY_ENA_MASK                                                               0x00020000L
+#define PA_CL_CLIP_CNTL__BOUNDARY_EDGE_FLAG_ENA_MASK                                                          0x00040000L
+#define PA_CL_CLIP_CNTL__DX_CLIP_SPACE_DEF_MASK                                                               0x00080000L
+#define PA_CL_CLIP_CNTL__DIS_CLIP_ERR_DETECT_MASK                                                             0x00100000L
+#define PA_CL_CLIP_CNTL__VTX_KILL_OR_MASK                                                                     0x00200000L
+#define PA_CL_CLIP_CNTL__DX_RASTERIZATION_KILL_MASK                                                           0x00400000L
+#define PA_CL_CLIP_CNTL__DX_LINEAR_ATTR_CLIP_ENA_MASK                                                         0x01000000L
+#define PA_CL_CLIP_CNTL__VTE_VPORT_PROVOKE_DISABLE_MASK                                                       0x02000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_NEAR_DISABLE_MASK                                                              0x04000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_FAR_DISABLE_MASK                                                               0x08000000L
+#define PA_CL_CLIP_CNTL__ZCLIP_PROG_NEAR_ENA_MASK                                                             0x10000000L
+//PA_SU_SC_MODE_CNTL
+#define PA_SU_SC_MODE_CNTL__CULL_FRONT__SHIFT                                                                 0x0
+#define PA_SU_SC_MODE_CNTL__CULL_BACK__SHIFT                                                                  0x1
+#define PA_SU_SC_MODE_CNTL__FACE__SHIFT                                                                       0x2
+#define PA_SU_SC_MODE_CNTL__POLY_MODE__SHIFT                                                                  0x3
+#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE__SHIFT                                                       0x5
+#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE__SHIFT                                                        0x8
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE__SHIFT                                                   0xb
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE__SHIFT                                                    0xc
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE__SHIFT                                                    0xd
+#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE__SHIFT                                                   0x10
+#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST__SHIFT                                                         0x13
+#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS__SHIFT                                                             0x14
+#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA__SHIFT                                                          0x15
+#define PA_SU_SC_MODE_CNTL__RIGHT_TRIANGLE_ALTERNATE_GRADIENT_REF__SHIFT                                      0x16
+#define PA_SU_SC_MODE_CNTL__NEW_QUAD_DECOMPOSITION__SHIFT                                                     0x17
+#define PA_SU_SC_MODE_CNTL__KEEP_TOGETHER_ENABLE__SHIFT                                                       0x18
+#define PA_SU_SC_MODE_CNTL__CULL_FRONT_MASK                                                                   0x00000001L
+#define PA_SU_SC_MODE_CNTL__CULL_BACK_MASK                                                                    0x00000002L
+#define PA_SU_SC_MODE_CNTL__FACE_MASK                                                                         0x00000004L
+#define PA_SU_SC_MODE_CNTL__POLY_MODE_MASK                                                                    0x00000018L
+#define PA_SU_SC_MODE_CNTL__POLYMODE_FRONT_PTYPE_MASK                                                         0x000000E0L
+#define PA_SU_SC_MODE_CNTL__POLYMODE_BACK_PTYPE_MASK                                                          0x00000700L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_FRONT_ENABLE_MASK                                                     0x00000800L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_BACK_ENABLE_MASK                                                      0x00001000L
+#define PA_SU_SC_MODE_CNTL__POLY_OFFSET_PARA_ENABLE_MASK                                                      0x00002000L
+#define PA_SU_SC_MODE_CNTL__VTX_WINDOW_OFFSET_ENABLE_MASK                                                     0x00010000L
+#define PA_SU_SC_MODE_CNTL__PROVOKING_VTX_LAST_MASK                                                           0x00080000L
+#define PA_SU_SC_MODE_CNTL__PERSP_CORR_DIS_MASK                                                               0x00100000L
+#define PA_SU_SC_MODE_CNTL__MULTI_PRIM_IB_ENA_MASK                                                            0x00200000L
+#define PA_SU_SC_MODE_CNTL__RIGHT_TRIANGLE_ALTERNATE_GRADIENT_REF_MASK                                        0x00400000L
+#define PA_SU_SC_MODE_CNTL__NEW_QUAD_DECOMPOSITION_MASK                                                       0x00800000L
+#define PA_SU_SC_MODE_CNTL__KEEP_TOGETHER_ENABLE_MASK                                                         0x01000000L
+//PA_CL_VTE_CNTL
+#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA__SHIFT                                                              0x0
+#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA__SHIFT                                                             0x1
+#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA__SHIFT                                                              0x2
+#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA__SHIFT                                                             0x3
+#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA__SHIFT                                                              0x4
+#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA__SHIFT                                                             0x5
+#define PA_CL_VTE_CNTL__VTX_XY_FMT__SHIFT                                                                     0x8
+#define PA_CL_VTE_CNTL__VTX_Z_FMT__SHIFT                                                                      0x9
+#define PA_CL_VTE_CNTL__VTX_W0_FMT__SHIFT                                                                     0xa
+#define PA_CL_VTE_CNTL__PERFCOUNTER_REF__SHIFT                                                                0xb
+#define PA_CL_VTE_CNTL__VPORT_X_SCALE_ENA_MASK                                                                0x00000001L
+#define PA_CL_VTE_CNTL__VPORT_X_OFFSET_ENA_MASK                                                               0x00000002L
+#define PA_CL_VTE_CNTL__VPORT_Y_SCALE_ENA_MASK                                                                0x00000004L
+#define PA_CL_VTE_CNTL__VPORT_Y_OFFSET_ENA_MASK                                                               0x00000008L
+#define PA_CL_VTE_CNTL__VPORT_Z_SCALE_ENA_MASK                                                                0x00000010L
+#define PA_CL_VTE_CNTL__VPORT_Z_OFFSET_ENA_MASK                                                               0x00000020L
+#define PA_CL_VTE_CNTL__VTX_XY_FMT_MASK                                                                       0x00000100L
+#define PA_CL_VTE_CNTL__VTX_Z_FMT_MASK                                                                        0x00000200L
+#define PA_CL_VTE_CNTL__VTX_W0_FMT_MASK                                                                       0x00000400L
+#define PA_CL_VTE_CNTL__PERFCOUNTER_REF_MASK                                                                  0x00000800L
+//PA_CL_VS_OUT_CNTL
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0__SHIFT                                                             0x0
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1__SHIFT                                                             0x1
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2__SHIFT                                                             0x2
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3__SHIFT                                                             0x3
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4__SHIFT                                                             0x4
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5__SHIFT                                                             0x5
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6__SHIFT                                                             0x6
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7__SHIFT                                                             0x7
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0__SHIFT                                                             0x8
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1__SHIFT                                                             0x9
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2__SHIFT                                                             0xa
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3__SHIFT                                                             0xb
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4__SHIFT                                                             0xc
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5__SHIFT                                                             0xd
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6__SHIFT                                                             0xe
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7__SHIFT                                                             0xf
+#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE__SHIFT                                                          0x10
+#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG__SHIFT                                                           0x11
+#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX__SHIFT                                                  0x12
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX__SHIFT                                                       0x13
+#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG__SHIFT                                                           0x14
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA__SHIFT                                                         0x15
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA__SHIFT                                                      0x16
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA__SHIFT                                                      0x17
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA__SHIFT                                                    0x18
+#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH__SHIFT                                                          0x1b
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VRS_RATE__SHIFT                                                            0x1c
+#define PA_CL_VS_OUT_CNTL__BYPASS_VTX_RATE_COMBINER__SHIFT                                                    0x1d
+#define PA_CL_VS_OUT_CNTL__BYPASS_PRIM_RATE_COMBINER__SHIFT                                                   0x1e
+#define PA_CL_VS_OUT_CNTL__USE_VTX_FSR_SELECT__SHIFT                                                          0x1f
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_0_MASK                                                               0x00000001L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_1_MASK                                                               0x00000002L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_2_MASK                                                               0x00000004L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_3_MASK                                                               0x00000008L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_4_MASK                                                               0x00000010L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_5_MASK                                                               0x00000020L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_6_MASK                                                               0x00000040L
+#define PA_CL_VS_OUT_CNTL__CLIP_DIST_ENA_7_MASK                                                               0x00000080L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_0_MASK                                                               0x00000100L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_1_MASK                                                               0x00000200L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_2_MASK                                                               0x00000400L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_3_MASK                                                               0x00000800L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_4_MASK                                                               0x00001000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_5_MASK                                                               0x00002000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_6_MASK                                                               0x00004000L
+#define PA_CL_VS_OUT_CNTL__CULL_DIST_ENA_7_MASK                                                               0x00008000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_POINT_SIZE_MASK                                                            0x00010000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_EDGE_FLAG_MASK                                                             0x00020000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_RENDER_TARGET_INDX_MASK                                                    0x00040000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VIEWPORT_INDX_MASK                                                         0x00080000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_KILL_FLAG_MASK                                                             0x00100000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_VEC_ENA_MASK                                                           0x00200000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST0_VEC_ENA_MASK                                                        0x00400000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_CCDIST1_VEC_ENA_MASK                                                        0x00800000L
+#define PA_CL_VS_OUT_CNTL__VS_OUT_MISC_SIDE_BUS_ENA_MASK                                                      0x01000000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_LINE_WIDTH_MASK                                                            0x08000000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_VRS_RATE_MASK                                                              0x10000000L
+#define PA_CL_VS_OUT_CNTL__BYPASS_VTX_RATE_COMBINER_MASK                                                      0x20000000L
+#define PA_CL_VS_OUT_CNTL__BYPASS_PRIM_RATE_COMBINER_MASK                                                     0x40000000L
+#define PA_CL_VS_OUT_CNTL__USE_VTX_FSR_SELECT_MASK                                                            0x80000000L
+//PA_CL_NANINF_CNTL
+#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD__SHIFT                                                          0x0
+#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD__SHIFT                                                           0x1
+#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD__SHIFT                                                           0x2
+#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0__SHIFT                                                           0x3
+#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN__SHIFT                                                           0x4
+#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN__SHIFT                                                            0x5
+#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN__SHIFT                                                            0x6
+#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0__SHIFT                                                        0x7
+#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF__SHIFT                                                            0x8
+#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN__SHIFT                                                            0x9
+#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF__SHIFT                                                             0xa
+#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN__SHIFT                                                             0xb
+#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF__SHIFT                                                             0xc
+#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN__SHIFT                                                             0xd
+#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD__SHIFT                                                    0xe
+#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0__SHIFT                                                         0x14
+#define PA_CL_NANINF_CNTL__VTE_XY_INF_DISCARD_MASK                                                            0x00000001L
+#define PA_CL_NANINF_CNTL__VTE_Z_INF_DISCARD_MASK                                                             0x00000002L
+#define PA_CL_NANINF_CNTL__VTE_W_INF_DISCARD_MASK                                                             0x00000004L
+#define PA_CL_NANINF_CNTL__VTE_0XNANINF_IS_0_MASK                                                             0x00000008L
+#define PA_CL_NANINF_CNTL__VTE_XY_NAN_RETAIN_MASK                                                             0x00000010L
+#define PA_CL_NANINF_CNTL__VTE_Z_NAN_RETAIN_MASK                                                              0x00000020L
+#define PA_CL_NANINF_CNTL__VTE_W_NAN_RETAIN_MASK                                                              0x00000040L
+#define PA_CL_NANINF_CNTL__VTE_W_RECIP_NAN_IS_0_MASK                                                          0x00000080L
+#define PA_CL_NANINF_CNTL__VS_XY_NAN_TO_INF_MASK                                                              0x00000100L
+#define PA_CL_NANINF_CNTL__VS_XY_INF_RETAIN_MASK                                                              0x00000200L
+#define PA_CL_NANINF_CNTL__VS_Z_NAN_TO_INF_MASK                                                               0x00000400L
+#define PA_CL_NANINF_CNTL__VS_Z_INF_RETAIN_MASK                                                               0x00000800L
+#define PA_CL_NANINF_CNTL__VS_W_NAN_TO_INF_MASK                                                               0x00001000L
+#define PA_CL_NANINF_CNTL__VS_W_INF_RETAIN_MASK                                                               0x00002000L
+#define PA_CL_NANINF_CNTL__VS_CLIP_DIST_INF_DISCARD_MASK                                                      0x00004000L
+#define PA_CL_NANINF_CNTL__VTE_NO_OUTPUT_NEG_0_MASK                                                           0x00100000L
+//PA_SU_LINE_STIPPLE_CNTL
+#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET__SHIFT                                                    0x0
+#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH__SHIFT                                                    0x2
+#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM__SHIFT                                                      0x3
+#define PA_SU_LINE_STIPPLE_CNTL__LINE_STIPPLE_RESET_MASK                                                      0x00000003L
+#define PA_SU_LINE_STIPPLE_CNTL__EXPAND_FULL_LENGTH_MASK                                                      0x00000004L
+#define PA_SU_LINE_STIPPLE_CNTL__FRACTIONAL_ACCUM_MASK                                                        0x00000008L
+//PA_SU_LINE_STIPPLE_SCALE
+#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE__SHIFT                                                   0x0
+#define PA_SU_LINE_STIPPLE_SCALE__LINE_STIPPLE_SCALE_MASK                                                     0xFFFFFFFFL
+//PA_SU_PRIM_FILTER_CNTL
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT                                                0x0
+#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT                                                    0x1
+#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT                                                   0x2
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT                                               0x3
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA__SHIFT                                                    0x4
+#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA__SHIFT                                                        0x5
+#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA__SHIFT                                                       0x6
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA__SHIFT                                                   0x7
+#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT__SHIFT                                                   0x8
+#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION__SHIFT                                                   0x1e
+#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION__SHIFT                                                  0x1f
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK                                                  0x00000001L
+#define PA_SU_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK                                                      0x00000002L
+#define PA_SU_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK                                                     0x00000004L
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK                                                 0x00000008L
+#define PA_SU_PRIM_FILTER_CNTL__TRIANGLE_EXPAND_ENA_MASK                                                      0x00000010L
+#define PA_SU_PRIM_FILTER_CNTL__LINE_EXPAND_ENA_MASK                                                          0x00000020L
+#define PA_SU_PRIM_FILTER_CNTL__POINT_EXPAND_ENA_MASK                                                         0x00000040L
+#define PA_SU_PRIM_FILTER_CNTL__RECTANGLE_EXPAND_ENA_MASK                                                     0x00000080L
+#define PA_SU_PRIM_FILTER_CNTL__PRIM_EXPAND_CONSTANT_MASK                                                     0x0000FF00L
+#define PA_SU_PRIM_FILTER_CNTL__XMAX_RIGHT_EXCLUSION_MASK                                                     0x40000000L
+#define PA_SU_PRIM_FILTER_CNTL__YMAX_BOTTOM_EXCLUSION_MASK                                                    0x80000000L
+//PA_SU_SMALL_PRIM_FILTER_CNTL
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SMALL_PRIM_FILTER_ENABLE__SHIFT                                         0x0
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE__SHIFT                                          0x1
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE__SHIFT                                              0x2
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE__SHIFT                                             0x3
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE__SHIFT                                         0x4
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SC_1XMSAA_COMPATIBLE_DISABLE__SHIFT                                     0x6
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SMALL_PRIM_FILTER_ENABLE_MASK                                           0x00000001L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__TRIANGLE_FILTER_DISABLE_MASK                                            0x00000002L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__LINE_FILTER_DISABLE_MASK                                                0x00000004L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__POINT_FILTER_DISABLE_MASK                                               0x00000008L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__RECTANGLE_FILTER_DISABLE_MASK                                           0x00000010L
+#define PA_SU_SMALL_PRIM_FILTER_CNTL__SC_1XMSAA_COMPATIBLE_DISABLE_MASK                                       0x00000040L
+//PA_CL_NGG_CNTL
+#define PA_CL_NGG_CNTL__VERTEX_REUSE_OFF__SHIFT                                                               0x0
+#define PA_CL_NGG_CNTL__INDEX_BUF_EDGE_FLAG_ENA__SHIFT                                                        0x1
+#define PA_CL_NGG_CNTL__VERTEX_REUSE_DEPTH__SHIFT                                                             0x2
+#define PA_CL_NGG_CNTL__VERTEX_REUSE_OFF_MASK                                                                 0x00000001L
+#define PA_CL_NGG_CNTL__INDEX_BUF_EDGE_FLAG_ENA_MASK                                                          0x00000002L
+#define PA_CL_NGG_CNTL__VERTEX_REUSE_DEPTH_MASK                                                               0x000003FCL
+//PA_SU_OVER_RASTERIZATION_CNTL
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_TRIANGLES__SHIFT                                        0x0
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_LINES__SHIFT                                            0x1
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_POINTS__SHIFT                                           0x2
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_RECTANGLES__SHIFT                                       0x3
+#define PA_SU_OVER_RASTERIZATION_CNTL__USE_PROVOKING_ZW__SHIFT                                                0x4
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_TRIANGLES_MASK                                          0x00000001L
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_LINES_MASK                                              0x00000002L
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_POINTS_MASK                                             0x00000004L
+#define PA_SU_OVER_RASTERIZATION_CNTL__DISCARD_0_AREA_RECTANGLES_MASK                                         0x00000008L
+#define PA_SU_OVER_RASTERIZATION_CNTL__USE_PROVOKING_ZW_MASK                                                  0x00000010L
+//PA_STEREO_CNTL
+#define PA_STEREO_CNTL__STEREO_MODE__SHIFT                                                                    0x1
+#define PA_STEREO_CNTL__RT_SLICE_MODE__SHIFT                                                                  0x5
+#define PA_STEREO_CNTL__RT_SLICE_OFFSET__SHIFT                                                                0x8
+#define PA_STEREO_CNTL__VP_ID_MODE__SHIFT                                                                     0x10
+#define PA_STEREO_CNTL__VP_ID_OFFSET__SHIFT                                                                   0x13
+#define PA_STEREO_CNTL__FSR_MODE__SHIFT                                                                       0x18
+#define PA_STEREO_CNTL__FSR_OFFSET__SHIFT                                                                     0x1a
+#define PA_STEREO_CNTL__STEREO_MODE_MASK                                                                      0x0000001EL
+#define PA_STEREO_CNTL__RT_SLICE_MODE_MASK                                                                    0x000000E0L
+#define PA_STEREO_CNTL__RT_SLICE_OFFSET_MASK                                                                  0x00000F00L
+#define PA_STEREO_CNTL__VP_ID_MODE_MASK                                                                       0x00070000L
+#define PA_STEREO_CNTL__VP_ID_OFFSET_MASK                                                                     0x00780000L
+#define PA_STEREO_CNTL__FSR_MODE_MASK                                                                         0x03000000L
+#define PA_STEREO_CNTL__FSR_OFFSET_MASK                                                                       0x0C000000L
+//PA_STATE_STEREO_X
+#define PA_STATE_STEREO_X__STEREO_X_OFFSET__SHIFT                                                             0x0
+#define PA_STATE_STEREO_X__STEREO_X_OFFSET_MASK                                                               0xFFFFFFFFL
+//PA_CL_VRS_CNTL
+#define PA_CL_VRS_CNTL__VERTEX_RATE_COMBINER_MODE__SHIFT                                                      0x0
+#define PA_CL_VRS_CNTL__PRIMITIVE_RATE_COMBINER_MODE__SHIFT                                                   0x3
+#define PA_CL_VRS_CNTL__HTILE_RATE_COMBINER_MODE__SHIFT                                                       0x6
+#define PA_CL_VRS_CNTL__SAMPLE_ITER_COMBINER_MODE__SHIFT                                                      0x9
+#define PA_CL_VRS_CNTL__EXPOSE_VRS_PIXELS_MASK__SHIFT                                                         0xd
+#define PA_CL_VRS_CNTL__CMASK_RATE_HINT_FORCE_ZERO__SHIFT                                                     0xe
+#define PA_CL_VRS_CNTL__VERTEX_RATE_COMBINER_MODE_MASK                                                        0x00000007L
+#define PA_CL_VRS_CNTL__PRIMITIVE_RATE_COMBINER_MODE_MASK                                                     0x00000038L
+#define PA_CL_VRS_CNTL__HTILE_RATE_COMBINER_MODE_MASK                                                         0x000001C0L
+#define PA_CL_VRS_CNTL__SAMPLE_ITER_COMBINER_MODE_MASK                                                        0x00000E00L
+#define PA_CL_VRS_CNTL__EXPOSE_VRS_PIXELS_MASK_MASK                                                           0x00002000L
+#define PA_CL_VRS_CNTL__CMASK_RATE_HINT_FORCE_ZERO_MASK                                                       0x00004000L
+//PA_SU_POINT_SIZE
+#define PA_SU_POINT_SIZE__HEIGHT__SHIFT                                                                       0x0
+#define PA_SU_POINT_SIZE__WIDTH__SHIFT                                                                        0x10
+#define PA_SU_POINT_SIZE__HEIGHT_MASK                                                                         0x0000FFFFL
+#define PA_SU_POINT_SIZE__WIDTH_MASK                                                                          0xFFFF0000L
+//PA_SU_POINT_MINMAX
+#define PA_SU_POINT_MINMAX__MIN_SIZE__SHIFT                                                                   0x0
+#define PA_SU_POINT_MINMAX__MAX_SIZE__SHIFT                                                                   0x10
+#define PA_SU_POINT_MINMAX__MIN_SIZE_MASK                                                                     0x0000FFFFL
+#define PA_SU_POINT_MINMAX__MAX_SIZE_MASK                                                                     0xFFFF0000L
+//PA_SU_LINE_CNTL
+#define PA_SU_LINE_CNTL__WIDTH__SHIFT                                                                         0x0
+#define PA_SU_LINE_CNTL__WIDTH_MASK                                                                           0x0000FFFFL
+//PA_SC_LINE_STIPPLE
+#define PA_SC_LINE_STIPPLE__LINE_PATTERN__SHIFT                                                               0x0
+#define PA_SC_LINE_STIPPLE__REPEAT_COUNT__SHIFT                                                               0x10
+#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER__SHIFT                                                          0x1c
+#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL__SHIFT                                                            0x1d
+#define PA_SC_LINE_STIPPLE__LINE_PATTERN_MASK                                                                 0x0000FFFFL
+#define PA_SC_LINE_STIPPLE__REPEAT_COUNT_MASK                                                                 0x00FF0000L
+#define PA_SC_LINE_STIPPLE__PATTERN_BIT_ORDER_MASK                                                            0x10000000L
+#define PA_SC_LINE_STIPPLE__AUTO_RESET_CNTL_MASK                                                              0x60000000L
+//VGT_HOS_MAX_TESS_LEVEL
+#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS__SHIFT                                                               0x0
+#define VGT_HOS_MAX_TESS_LEVEL__MAX_TESS_MASK                                                                 0xFFFFFFFFL
+//VGT_HOS_MIN_TESS_LEVEL
+#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS__SHIFT                                                               0x0
+#define VGT_HOS_MIN_TESS_LEVEL__MIN_TESS_MASK                                                                 0xFFFFFFFFL
+//PA_SC_MODE_CNTL_0
+#define PA_SC_MODE_CNTL_0__MSAA_ENABLE__SHIFT                                                                 0x0
+#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE__SHIFT                                                        0x1
+#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE__SHIFT                                                         0x2
+#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR__SHIFT                                                    0x3
+#define PA_SC_MODE_CNTL_0__ALTERNATE_RBS_PER_TILE__SHIFT                                                      0x5
+#define PA_SC_MODE_CNTL_0__COARSE_TILE_STARTS_ON_EVEN_RB__SHIFT                                               0x6
+#define PA_SC_MODE_CNTL_0__MSAA_ENABLE_MASK                                                                   0x00000001L
+#define PA_SC_MODE_CNTL_0__VPORT_SCISSOR_ENABLE_MASK                                                          0x00000002L
+#define PA_SC_MODE_CNTL_0__LINE_STIPPLE_ENABLE_MASK                                                           0x00000004L
+#define PA_SC_MODE_CNTL_0__SEND_UNLIT_STILES_TO_PKR_MASK                                                      0x00000008L
+#define PA_SC_MODE_CNTL_0__ALTERNATE_RBS_PER_TILE_MASK                                                        0x00000020L
+#define PA_SC_MODE_CNTL_0__COARSE_TILE_STARTS_ON_EVEN_RB_MASK                                                 0x00000040L
+//PA_SC_MODE_CNTL_1
+#define PA_SC_MODE_CNTL_1__WALK_SIZE__SHIFT                                                                   0x0
+#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT__SHIFT                                                              0x1
+#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST__SHIFT                                                    0x2
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE__SHIFT                                                           0x3
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE__SHIFT                                                             0x4
+#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE__SHIFT                                                 0x7
+#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE__SHIFT                                                      0x8
+#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE__SHIFT                                                          0x9
+#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR__SHIFT                                                       0xa
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT__SHIFT                                                             0xb
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET__SHIFT                                                             0xc
+#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT__SHIFT                                                             0xd
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z__SHIFT                                                          0xe
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK__SHIFT                                                   0xf
+#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE__SHIFT                                                              0x10
+#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE__SHIFT                                     0x11
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE__SHIFT                                                  0x12
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE__SHIFT                                                      0x13
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE__SHIFT                                                             0x14
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE__SHIFT                                               0x18
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE__SHIFT                                                     0x19
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE__SHIFT                                                        0x1a
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE__SHIFT                                               0x1b
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK__SHIFT                                                     0x1c
+#define PA_SC_MODE_CNTL_1__WALK_SIZE_MASK                                                                     0x00000001L
+#define PA_SC_MODE_CNTL_1__WALK_ALIGNMENT_MASK                                                                0x00000002L
+#define PA_SC_MODE_CNTL_1__WALK_ALIGN8_PRIM_FITS_ST_MASK                                                      0x00000004L
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_ENABLE_MASK                                                             0x00000008L
+#define PA_SC_MODE_CNTL_1__WALK_FENCE_SIZE_MASK                                                               0x00000070L
+#define PA_SC_MODE_CNTL_1__SUPERTILE_WALK_ORDER_ENABLE_MASK                                                   0x00000080L
+#define PA_SC_MODE_CNTL_1__TILE_WALK_ORDER_ENABLE_MASK                                                        0x00000100L
+#define PA_SC_MODE_CNTL_1__TILE_COVER_DISABLE_MASK                                                            0x00000200L
+#define PA_SC_MODE_CNTL_1__TILE_COVER_NO_SCISSOR_MASK                                                         0x00000400L
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_EXTENT_MASK                                                               0x00000800L
+#define PA_SC_MODE_CNTL_1__ZMM_LINE_OFFSET_MASK                                                               0x00001000L
+#define PA_SC_MODE_CNTL_1__ZMM_RECT_EXTENT_MASK                                                               0x00002000L
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_HI_Z_MASK                                                            0x00004000L
+#define PA_SC_MODE_CNTL_1__KILL_PIX_POST_DETAIL_MASK_MASK                                                     0x00008000L
+#define PA_SC_MODE_CNTL_1__PS_ITER_SAMPLE_MASK                                                                0x00010000L
+#define PA_SC_MODE_CNTL_1__MULTI_SHADER_ENGINE_PRIM_DISCARD_ENABLE_MASK                                       0x00020000L
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_SUPERTILE_ENABLE_MASK                                                    0x00040000L
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_ENABLE_MASK                                                        0x00080000L
+#define PA_SC_MODE_CNTL_1__GPU_ID_OVERRIDE_MASK                                                               0x00F00000L
+#define PA_SC_MODE_CNTL_1__MULTI_GPU_PRIM_DISCARD_ENABLE_MASK                                                 0x01000000L
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_CNTDWN_ENABLE_MASK                                                       0x02000000L
+#define PA_SC_MODE_CNTL_1__FORCE_EOV_REZ_ENABLE_MASK                                                          0x04000000L
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_PRIMITIVE_ENABLE_MASK                                                 0x08000000L
+#define PA_SC_MODE_CNTL_1__OUT_OF_ORDER_WATER_MARK_MASK                                                       0x70000000L
+//VGT_ENHANCE
+#define VGT_ENHANCE__MISC__SHIFT                                                                              0x0
+#define VGT_ENHANCE__MISC_MASK                                                                                0xFFFFFFFFL
+//IA_ENHANCE
+#define IA_ENHANCE__MISC__SHIFT                                                                               0x0
+#define IA_ENHANCE__MISC_MASK                                                                                 0xFFFFFFFFL
+//VGT_DMA_SIZE
+#define VGT_DMA_SIZE__NUM_INDICES__SHIFT                                                                      0x0
+#define VGT_DMA_SIZE__NUM_INDICES_MASK                                                                        0xFFFFFFFFL
+//VGT_DMA_MAX_SIZE
+#define VGT_DMA_MAX_SIZE__MAX_SIZE__SHIFT                                                                     0x0
+#define VGT_DMA_MAX_SIZE__MAX_SIZE_MASK                                                                       0xFFFFFFFFL
+//VGT_DMA_INDEX_TYPE
+#define VGT_DMA_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                 0x0
+#define VGT_DMA_INDEX_TYPE__SWAP_MODE__SHIFT                                                                  0x2
+#define VGT_DMA_INDEX_TYPE__BUF_TYPE__SHIFT                                                                   0x4
+#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY__SHIFT                                                               0x6
+#define VGT_DMA_INDEX_TYPE__ATC__SHIFT                                                                        0x8
+#define VGT_DMA_INDEX_TYPE__NOT_EOP__SHIFT                                                                    0x9
+#define VGT_DMA_INDEX_TYPE__REQ_PATH__SHIFT                                                                   0xa
+#define VGT_DMA_INDEX_TYPE__MTYPE__SHIFT                                                                      0xb
+#define VGT_DMA_INDEX_TYPE__DISABLE_INSTANCE_PACKING__SHIFT                                                   0xe
+#define VGT_DMA_INDEX_TYPE__INDEX_TYPE_MASK                                                                   0x00000003L
+#define VGT_DMA_INDEX_TYPE__SWAP_MODE_MASK                                                                    0x0000000CL
+#define VGT_DMA_INDEX_TYPE__BUF_TYPE_MASK                                                                     0x00000030L
+#define VGT_DMA_INDEX_TYPE__RDREQ_POLICY_MASK                                                                 0x000000C0L
+#define VGT_DMA_INDEX_TYPE__ATC_MASK                                                                          0x00000100L
+#define VGT_DMA_INDEX_TYPE__NOT_EOP_MASK                                                                      0x00000200L
+#define VGT_DMA_INDEX_TYPE__REQ_PATH_MASK                                                                     0x00000400L
+#define VGT_DMA_INDEX_TYPE__MTYPE_MASK                                                                        0x00003800L
+#define VGT_DMA_INDEX_TYPE__DISABLE_INSTANCE_PACKING_MASK                                                     0x00004000L
+//WD_ENHANCE
+#define WD_ENHANCE__MISC__SHIFT                                                                               0x0
+#define WD_ENHANCE__MISC_MASK                                                                                 0xFFFFFFFFL
+//VGT_PRIMITIVEID_EN
+#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN__SHIFT                                                             0x0
+#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI__SHIFT                                                       0x1
+#define VGT_PRIMITIVEID_EN__NGG_DISABLE_PROVOK_REUSE__SHIFT                                                   0x2
+#define VGT_PRIMITIVEID_EN__PRIMITIVEID_EN_MASK                                                               0x00000001L
+#define VGT_PRIMITIVEID_EN__DISABLE_RESET_ON_EOI_MASK                                                         0x00000002L
+#define VGT_PRIMITIVEID_EN__NGG_DISABLE_PROVOK_REUSE_MASK                                                     0x00000004L
+//VGT_DMA_NUM_INSTANCES
+#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES__SHIFT                                                           0x0
+#define VGT_DMA_NUM_INSTANCES__NUM_INSTANCES_MASK                                                             0xFFFFFFFFL
+//VGT_PRIMITIVEID_RESET
+#define VGT_PRIMITIVEID_RESET__VALUE__SHIFT                                                                   0x0
+#define VGT_PRIMITIVEID_RESET__VALUE_MASK                                                                     0xFFFFFFFFL
+//VGT_EVENT_INITIATOR
+#define VGT_EVENT_INITIATOR__EVENT_TYPE__SHIFT                                                                0x0
+#define VGT_EVENT_INITIATOR__ADDRESS_HI__SHIFT                                                                0xa
+#define VGT_EVENT_INITIATOR__EXTENDED_EVENT__SHIFT                                                            0x1b
+#define VGT_EVENT_INITIATOR__EVENT_TYPE_MASK                                                                  0x0000003FL
+#define VGT_EVENT_INITIATOR__ADDRESS_HI_MASK                                                                  0x07FFFC00L
+#define VGT_EVENT_INITIATOR__EXTENDED_EVENT_MASK                                                              0x08000000L
+//VGT_DRAW_PAYLOAD_CNTL
+#define VGT_DRAW_PAYLOAD_CNTL__EN_REG_RT_INDEX__SHIFT                                                         0x1
+#define VGT_DRAW_PAYLOAD_CNTL__EN_PRIM_PAYLOAD__SHIFT                                                         0x3
+#define VGT_DRAW_PAYLOAD_CNTL__EN_DRAW_VP__SHIFT                                                              0x4
+#define VGT_DRAW_PAYLOAD_CNTL__EN_FSR__SHIFT                                                                  0x5
+#define VGT_DRAW_PAYLOAD_CNTL__EN_VRS_RATE__SHIFT                                                             0x6
+#define VGT_DRAW_PAYLOAD_CNTL__EN_REG_RT_INDEX_MASK                                                           0x00000002L
+#define VGT_DRAW_PAYLOAD_CNTL__EN_PRIM_PAYLOAD_MASK                                                           0x00000008L
+#define VGT_DRAW_PAYLOAD_CNTL__EN_DRAW_VP_MASK                                                                0x00000010L
+#define VGT_DRAW_PAYLOAD_CNTL__EN_FSR_MASK                                                                    0x00000020L
+#define VGT_DRAW_PAYLOAD_CNTL__EN_VRS_RATE_MASK                                                               0x00000040L
+//VGT_ESGS_RING_ITEMSIZE
+#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE__SHIFT                                                               0x0
+#define VGT_ESGS_RING_ITEMSIZE__ITEMSIZE_MASK                                                                 0x00007FFFL
+//VGT_REUSE_OFF
+#define VGT_REUSE_OFF__REUSE_OFF__SHIFT                                                                       0x0
+#define VGT_REUSE_OFF__REUSE_OFF_MASK                                                                         0x00000001L
+//DB_HTILE_SURFACE
+#define DB_HTILE_SURFACE__RESERVED_FIELD_1__SHIFT                                                             0x0
+#define DB_HTILE_SURFACE__FULL_CACHE__SHIFT                                                                   0x1
+#define DB_HTILE_SURFACE__RESERVED_FIELD_2__SHIFT                                                             0x2
+#define DB_HTILE_SURFACE__RESERVED_FIELD_3__SHIFT                                                             0x3
+#define DB_HTILE_SURFACE__RESERVED_FIELD_4__SHIFT                                                             0x4
+#define DB_HTILE_SURFACE__RESERVED_FIELD_5__SHIFT                                                             0xa
+#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE__SHIFT                                                      0x10
+#define DB_HTILE_SURFACE__RESERVED_FIELD_6__SHIFT                                                             0x11
+#define DB_HTILE_SURFACE__PIPE_ALIGNED__SHIFT                                                                 0x12
+#define DB_HTILE_SURFACE__RESERVED_FIELD_1_MASK                                                               0x00000001L
+#define DB_HTILE_SURFACE__FULL_CACHE_MASK                                                                     0x00000002L
+#define DB_HTILE_SURFACE__RESERVED_FIELD_2_MASK                                                               0x00000004L
+#define DB_HTILE_SURFACE__RESERVED_FIELD_3_MASK                                                               0x00000008L
+#define DB_HTILE_SURFACE__RESERVED_FIELD_4_MASK                                                               0x000003F0L
+#define DB_HTILE_SURFACE__RESERVED_FIELD_5_MASK                                                               0x0000FC00L
+#define DB_HTILE_SURFACE__DST_OUTSIDE_ZERO_TO_ONE_MASK                                                        0x00010000L
+#define DB_HTILE_SURFACE__RESERVED_FIELD_6_MASK                                                               0x00020000L
+#define DB_HTILE_SURFACE__PIPE_ALIGNED_MASK                                                                   0x00040000L
+//DB_SRESULTS_COMPARE_STATE0
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0__SHIFT                                                       0x0
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0__SHIFT                                                      0x4
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0__SHIFT                                                       0xc
+#define DB_SRESULTS_COMPARE_STATE0__ENABLE0__SHIFT                                                            0x18
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREFUNC0_MASK                                                         0x00000007L
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREVALUE0_MASK                                                        0x00000FF0L
+#define DB_SRESULTS_COMPARE_STATE0__COMPAREMASK0_MASK                                                         0x000FF000L
+#define DB_SRESULTS_COMPARE_STATE0__ENABLE0_MASK                                                              0x01000000L
+//DB_SRESULTS_COMPARE_STATE1
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1__SHIFT                                                       0x0
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1__SHIFT                                                      0x4
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1__SHIFT                                                       0xc
+#define DB_SRESULTS_COMPARE_STATE1__ENABLE1__SHIFT                                                            0x18
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREFUNC1_MASK                                                         0x00000007L
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREVALUE1_MASK                                                        0x00000FF0L
+#define DB_SRESULTS_COMPARE_STATE1__COMPAREMASK1_MASK                                                         0x000FF000L
+#define DB_SRESULTS_COMPARE_STATE1__ENABLE1_MASK                                                              0x01000000L
+//DB_PRELOAD_CONTROL
+#define DB_PRELOAD_CONTROL__START_X__SHIFT                                                                    0x0
+#define DB_PRELOAD_CONTROL__START_Y__SHIFT                                                                    0x8
+#define DB_PRELOAD_CONTROL__MAX_X__SHIFT                                                                      0x10
+#define DB_PRELOAD_CONTROL__MAX_Y__SHIFT                                                                      0x18
+#define DB_PRELOAD_CONTROL__START_X_MASK                                                                      0x000000FFL
+#define DB_PRELOAD_CONTROL__START_Y_MASK                                                                      0x0000FF00L
+#define DB_PRELOAD_CONTROL__MAX_X_MASK                                                                        0x00FF0000L
+#define DB_PRELOAD_CONTROL__MAX_Y_MASK                                                                        0xFF000000L
+//VGT_STRMOUT_DRAW_OPAQUE_OFFSET
+#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET__SHIFT                                                         0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_OFFSET__OFFSET_MASK                                                           0xFFFFFFFFL
+//VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE
+#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE__SHIFT                                               0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_BUFFER_FILLED_SIZE__SIZE_MASK                                                 0xFFFFFFFFL
+//VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE
+#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE__SHIFT                                           0x0
+#define VGT_STRMOUT_DRAW_OPAQUE_VERTEX_STRIDE__VERTEX_STRIDE_MASK                                             0x000001FFL
+//VGT_GS_MAX_VERT_OUT
+#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT__SHIFT                                                              0x0
+#define VGT_GS_MAX_VERT_OUT__MAX_VERT_OUT_MASK                                                                0x000007FFL
+//GE_NGG_SUBGRP_CNTL
+#define GE_NGG_SUBGRP_CNTL__PRIM_AMP_FACTOR__SHIFT                                                            0x0
+#define GE_NGG_SUBGRP_CNTL__THDS_PER_SUBGRP__SHIFT                                                            0x9
+#define GE_NGG_SUBGRP_CNTL__PRIM_AMP_FACTOR_MASK                                                              0x000001FFL
+#define GE_NGG_SUBGRP_CNTL__THDS_PER_SUBGRP_MASK                                                              0x0003FE00L
+//VGT_TESS_DISTRIBUTION
+#define VGT_TESS_DISTRIBUTION__ACCUM_ISOLINE__SHIFT                                                           0x0
+#define VGT_TESS_DISTRIBUTION__ACCUM_TRI__SHIFT                                                               0x8
+#define VGT_TESS_DISTRIBUTION__ACCUM_QUAD__SHIFT                                                              0x10
+#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT__SHIFT                                                             0x18
+#define VGT_TESS_DISTRIBUTION__TRAP_SPLIT__SHIFT                                                              0x1d
+#define VGT_TESS_DISTRIBUTION__ACCUM_ISOLINE_MASK                                                             0x000000FFL
+#define VGT_TESS_DISTRIBUTION__ACCUM_TRI_MASK                                                                 0x0000FF00L
+#define VGT_TESS_DISTRIBUTION__ACCUM_QUAD_MASK                                                                0x00FF0000L
+#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT_MASK                                                               0x1F000000L
+#define VGT_TESS_DISTRIBUTION__TRAP_SPLIT_MASK                                                                0xE0000000L
+//VGT_SHADER_STAGES_EN
+#define VGT_SHADER_STAGES_EN__LS_EN__SHIFT                                                                    0x0
+#define VGT_SHADER_STAGES_EN__HS_EN__SHIFT                                                                    0x2
+#define VGT_SHADER_STAGES_EN__ES_EN__SHIFT                                                                    0x3
+#define VGT_SHADER_STAGES_EN__GS_EN__SHIFT                                                                    0x5
+#define VGT_SHADER_STAGES_EN__VS_EN__SHIFT                                                                    0x6
+#define VGT_SHADER_STAGES_EN__DYNAMIC_HS__SHIFT                                                               0x8
+#define VGT_SHADER_STAGES_EN__VS_WAVE_ID_EN__SHIFT                                                            0xc
+#define VGT_SHADER_STAGES_EN__PRIMGEN_EN__SHIFT                                                               0xd
+#define VGT_SHADER_STAGES_EN__ORDERED_ID_MODE__SHIFT                                                          0xe
+#define VGT_SHADER_STAGES_EN__MAX_PRIMGRP_IN_WAVE__SHIFT                                                      0xf
+#define VGT_SHADER_STAGES_EN__GS_FAST_LAUNCH__SHIFT                                                           0x13
+#define VGT_SHADER_STAGES_EN__HS_W32_EN__SHIFT                                                                0x15
+#define VGT_SHADER_STAGES_EN__GS_W32_EN__SHIFT                                                                0x16
+#define VGT_SHADER_STAGES_EN__VS_W32_EN__SHIFT                                                                0x17
+#define VGT_SHADER_STAGES_EN__NGG_WAVE_ID_EN__SHIFT                                                           0x18
+#define VGT_SHADER_STAGES_EN__PRIMGEN_PASSTHRU_EN__SHIFT                                                      0x19
+#define VGT_SHADER_STAGES_EN__PRIMGEN_PASSTHRU_NO_MSG__SHIFT                                                  0x1a
+#define VGT_SHADER_STAGES_EN__LS_EN_MASK                                                                      0x00000003L
+#define VGT_SHADER_STAGES_EN__HS_EN_MASK                                                                      0x00000004L
+#define VGT_SHADER_STAGES_EN__ES_EN_MASK                                                                      0x00000018L
+#define VGT_SHADER_STAGES_EN__GS_EN_MASK                                                                      0x00000020L
+#define VGT_SHADER_STAGES_EN__VS_EN_MASK                                                                      0x000000C0L
+#define VGT_SHADER_STAGES_EN__DYNAMIC_HS_MASK                                                                 0x00000100L
+#define VGT_SHADER_STAGES_EN__VS_WAVE_ID_EN_MASK                                                              0x00001000L
+#define VGT_SHADER_STAGES_EN__PRIMGEN_EN_MASK                                                                 0x00002000L
+#define VGT_SHADER_STAGES_EN__ORDERED_ID_MODE_MASK                                                            0x00004000L
+#define VGT_SHADER_STAGES_EN__MAX_PRIMGRP_IN_WAVE_MASK                                                        0x00078000L
+#define VGT_SHADER_STAGES_EN__GS_FAST_LAUNCH_MASK                                                             0x00180000L
+#define VGT_SHADER_STAGES_EN__HS_W32_EN_MASK                                                                  0x00200000L
+#define VGT_SHADER_STAGES_EN__GS_W32_EN_MASK                                                                  0x00400000L
+#define VGT_SHADER_STAGES_EN__VS_W32_EN_MASK                                                                  0x00800000L
+#define VGT_SHADER_STAGES_EN__NGG_WAVE_ID_EN_MASK                                                             0x01000000L
+#define VGT_SHADER_STAGES_EN__PRIMGEN_PASSTHRU_EN_MASK                                                        0x02000000L
+#define VGT_SHADER_STAGES_EN__PRIMGEN_PASSTHRU_NO_MSG_MASK                                                    0x04000000L
+//VGT_LS_HS_CONFIG
+#define VGT_LS_HS_CONFIG__NUM_PATCHES__SHIFT                                                                  0x0
+#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP__SHIFT                                                              0x8
+#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP__SHIFT                                                             0xe
+#define VGT_LS_HS_CONFIG__NUM_PATCHES_MASK                                                                    0x000000FFL
+#define VGT_LS_HS_CONFIG__HS_NUM_INPUT_CP_MASK                                                                0x00003F00L
+#define VGT_LS_HS_CONFIG__HS_NUM_OUTPUT_CP_MASK                                                               0x000FC000L
+//VGT_TF_PARAM
+#define VGT_TF_PARAM__TYPE__SHIFT                                                                             0x0
+#define VGT_TF_PARAM__PARTITIONING__SHIFT                                                                     0x2
+#define VGT_TF_PARAM__TOPOLOGY__SHIFT                                                                         0x5
+#define VGT_TF_PARAM__RESERVED_REDUC_AXIS__SHIFT                                                              0x8
+#define VGT_TF_PARAM__NOT_USED__SHIFT                                                                         0x9
+#define VGT_TF_PARAM__NUM_DS_WAVES_PER_SIMD__SHIFT                                                            0xa
+#define VGT_TF_PARAM__DISABLE_DONUTS__SHIFT                                                                   0xe
+#define VGT_TF_PARAM__RDREQ_POLICY__SHIFT                                                                     0xf
+#define VGT_TF_PARAM__DISTRIBUTION_MODE__SHIFT                                                                0x11
+#define VGT_TF_PARAM__DETECT_ONE__SHIFT                                                                       0x13
+#define VGT_TF_PARAM__DETECT_ZERO__SHIFT                                                                      0x14
+#define VGT_TF_PARAM__MTYPE__SHIFT                                                                            0x17
+#define VGT_TF_PARAM__TYPE_MASK                                                                               0x00000003L
+#define VGT_TF_PARAM__PARTITIONING_MASK                                                                       0x0000001CL
+#define VGT_TF_PARAM__TOPOLOGY_MASK                                                                           0x000000E0L
+#define VGT_TF_PARAM__RESERVED_REDUC_AXIS_MASK                                                                0x00000100L
+#define VGT_TF_PARAM__NOT_USED_MASK                                                                           0x00000200L
+#define VGT_TF_PARAM__NUM_DS_WAVES_PER_SIMD_MASK                                                              0x00003C00L
+#define VGT_TF_PARAM__DISABLE_DONUTS_MASK                                                                     0x00004000L
+#define VGT_TF_PARAM__RDREQ_POLICY_MASK                                                                       0x00018000L
+#define VGT_TF_PARAM__DISTRIBUTION_MODE_MASK                                                                  0x00060000L
+#define VGT_TF_PARAM__DETECT_ONE_MASK                                                                         0x00080000L
+#define VGT_TF_PARAM__DETECT_ZERO_MASK                                                                        0x00100000L
+#define VGT_TF_PARAM__MTYPE_MASK                                                                              0x03800000L
+//DB_ALPHA_TO_MASK
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE__SHIFT                                                         0x0
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0__SHIFT                                                        0x8
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1__SHIFT                                                        0xa
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2__SHIFT                                                        0xc
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3__SHIFT                                                        0xe
+#define DB_ALPHA_TO_MASK__OFFSET_ROUND__SHIFT                                                                 0x10
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_ENABLE_MASK                                                           0x00000001L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET0_MASK                                                          0x00000300L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET1_MASK                                                          0x00000C00L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET2_MASK                                                          0x00003000L
+#define DB_ALPHA_TO_MASK__ALPHA_TO_MASK_OFFSET3_MASK                                                          0x0000C000L
+#define DB_ALPHA_TO_MASK__OFFSET_ROUND_MASK                                                                   0x00010000L
+//PA_SU_POLY_OFFSET_DB_FMT_CNTL
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS__SHIFT                                     0x0
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT__SHIFT                                     0x8
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_NEG_NUM_DB_BITS_MASK                                       0x000000FFL
+#define PA_SU_POLY_OFFSET_DB_FMT_CNTL__POLY_OFFSET_DB_IS_FLOAT_FMT_MASK                                       0x00000100L
+//PA_SU_POLY_OFFSET_CLAMP
+#define PA_SU_POLY_OFFSET_CLAMP__CLAMP__SHIFT                                                                 0x0
+#define PA_SU_POLY_OFFSET_CLAMP__CLAMP_MASK                                                                   0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_FRONT_SCALE
+#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE__SHIFT                                                           0x0
+#define PA_SU_POLY_OFFSET_FRONT_SCALE__SCALE_MASK                                                             0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_FRONT_OFFSET
+#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET__SHIFT                                                         0x0
+#define PA_SU_POLY_OFFSET_FRONT_OFFSET__OFFSET_MASK                                                           0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_BACK_SCALE
+#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE__SHIFT                                                            0x0
+#define PA_SU_POLY_OFFSET_BACK_SCALE__SCALE_MASK                                                              0xFFFFFFFFL
+//PA_SU_POLY_OFFSET_BACK_OFFSET
+#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET__SHIFT                                                          0x0
+#define PA_SU_POLY_OFFSET_BACK_OFFSET__OFFSET_MASK                                                            0xFFFFFFFFL
+//VGT_GS_INSTANCE_CNT
+#define VGT_GS_INSTANCE_CNT__ENABLE__SHIFT                                                                    0x0
+#define VGT_GS_INSTANCE_CNT__CNT__SHIFT                                                                       0x2
+#define VGT_GS_INSTANCE_CNT__EN_MAX_VERT_OUT_PER_GS_INSTANCE__SHIFT                                           0x1f
+#define VGT_GS_INSTANCE_CNT__ENABLE_MASK                                                                      0x00000001L
+#define VGT_GS_INSTANCE_CNT__CNT_MASK                                                                         0x000001FCL
+#define VGT_GS_INSTANCE_CNT__EN_MAX_VERT_OUT_PER_GS_INSTANCE_MASK                                             0x80000000L
+//PA_SC_CENTROID_PRIORITY_0
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0__SHIFT                                                          0x0
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1__SHIFT                                                          0x4
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2__SHIFT                                                          0x8
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3__SHIFT                                                          0xc
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4__SHIFT                                                          0x10
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5__SHIFT                                                          0x14
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6__SHIFT                                                          0x18
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7__SHIFT                                                          0x1c
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_0_MASK                                                            0x0000000FL
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_1_MASK                                                            0x000000F0L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_2_MASK                                                            0x00000F00L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_3_MASK                                                            0x0000F000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_4_MASK                                                            0x000F0000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_5_MASK                                                            0x00F00000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_6_MASK                                                            0x0F000000L
+#define PA_SC_CENTROID_PRIORITY_0__DISTANCE_7_MASK                                                            0xF0000000L
+//PA_SC_CENTROID_PRIORITY_1
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8__SHIFT                                                          0x0
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9__SHIFT                                                          0x4
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10__SHIFT                                                         0x8
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11__SHIFT                                                         0xc
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12__SHIFT                                                         0x10
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13__SHIFT                                                         0x14
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14__SHIFT                                                         0x18
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15__SHIFT                                                         0x1c
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_8_MASK                                                            0x0000000FL
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_9_MASK                                                            0x000000F0L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_10_MASK                                                           0x00000F00L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_11_MASK                                                           0x0000F000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_12_MASK                                                           0x000F0000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_13_MASK                                                           0x00F00000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_14_MASK                                                           0x0F000000L
+#define PA_SC_CENTROID_PRIORITY_1__DISTANCE_15_MASK                                                           0xF0000000L
+//PA_SC_LINE_CNTL
+#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH__SHIFT                                                             0x9
+#define PA_SC_LINE_CNTL__LAST_PIXEL__SHIFT                                                                    0xa
+#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA__SHIFT                                                      0xb
+#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA__SHIFT                                                         0xc
+#define PA_SC_LINE_CNTL__EXTRA_DX_DY_PRECISION__SHIFT                                                         0xd
+#define PA_SC_LINE_CNTL__EXPAND_LINE_WIDTH_MASK                                                               0x00000200L
+#define PA_SC_LINE_CNTL__LAST_PIXEL_MASK                                                                      0x00000400L
+#define PA_SC_LINE_CNTL__PERPENDICULAR_ENDCAP_ENA_MASK                                                        0x00000800L
+#define PA_SC_LINE_CNTL__DX10_DIAMOND_TEST_ENA_MASK                                                           0x00001000L
+#define PA_SC_LINE_CNTL__EXTRA_DX_DY_PRECISION_MASK                                                           0x00002000L
+//PA_SC_AA_CONFIG
+#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES__SHIFT                                                              0x0
+#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN__SHIFT                                                         0x4
+#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST__SHIFT                                                               0xd
+#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES__SHIFT                                                          0x14
+#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE__SHIFT                                                        0x18
+#define PA_SC_AA_CONFIG__COVERAGE_TO_SHADER_SELECT__SHIFT                                                     0x1a
+#define PA_SC_AA_CONFIG__SAMPLE_COVERAGE_ENCODING__SHIFT                                                      0x1c
+#define PA_SC_AA_CONFIG__COVERED_CENTROID_IS_CENTER__SHIFT                                                    0x1d
+#define PA_SC_AA_CONFIG__MSAA_NUM_SAMPLES_MASK                                                                0x00000007L
+#define PA_SC_AA_CONFIG__AA_MASK_CENTROID_DTMN_MASK                                                           0x00000010L
+#define PA_SC_AA_CONFIG__MAX_SAMPLE_DIST_MASK                                                                 0x0001E000L
+#define PA_SC_AA_CONFIG__MSAA_EXPOSED_SAMPLES_MASK                                                            0x00700000L
+#define PA_SC_AA_CONFIG__DETAIL_TO_EXPOSED_MODE_MASK                                                          0x03000000L
+#define PA_SC_AA_CONFIG__COVERAGE_TO_SHADER_SELECT_MASK                                                       0x0C000000L
+#define PA_SC_AA_CONFIG__SAMPLE_COVERAGE_ENCODING_MASK                                                        0x10000000L
+#define PA_SC_AA_CONFIG__COVERED_CENTROID_IS_CENTER_MASK                                                      0x20000000L
+//PA_SU_VTX_CNTL
+#define PA_SU_VTX_CNTL__PIX_CENTER__SHIFT                                                                     0x0
+#define PA_SU_VTX_CNTL__ROUND_MODE__SHIFT                                                                     0x1
+#define PA_SU_VTX_CNTL__QUANT_MODE__SHIFT                                                                     0x3
+#define PA_SU_VTX_CNTL__PIX_CENTER_MASK                                                                       0x00000001L
+#define PA_SU_VTX_CNTL__ROUND_MODE_MASK                                                                       0x00000006L
+#define PA_SU_VTX_CNTL__QUANT_MODE_MASK                                                                       0x00000038L
+//PA_CL_GB_VERT_CLIP_ADJ
+#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_VERT_CLIP_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_CL_GB_VERT_DISC_ADJ
+#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_VERT_DISC_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_CL_GB_HORZ_CLIP_ADJ
+#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_HORZ_CLIP_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_CL_GB_HORZ_DISC_ADJ
+#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER__SHIFT                                                          0x0
+#define PA_CL_GB_HORZ_DISC_ADJ__DATA_REGISTER_MASK                                                            0xFFFFFFFFL
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y0_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y0_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X0Y1_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S0_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S1_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S2_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_0__S3_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X__SHIFT                                                        0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y__SHIFT                                                        0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X__SHIFT                                                        0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y__SHIFT                                                        0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S4_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S5_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_X_MASK                                                          0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S6_Y_MASK                                                          0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_X_MASK                                                          0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_1__S7_Y_MASK                                                          0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X__SHIFT                                                        0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y__SHIFT                                                        0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X__SHIFT                                                        0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y__SHIFT                                                        0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_X_MASK                                                          0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S8_Y_MASK                                                          0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_X_MASK                                                          0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S9_Y_MASK                                                          0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S10_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_2__S11_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X__SHIFT                                                       0x0
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y__SHIFT                                                       0x4
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X__SHIFT                                                       0x8
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y__SHIFT                                                       0xc
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X__SHIFT                                                       0x10
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y__SHIFT                                                       0x14
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X__SHIFT                                                       0x18
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y__SHIFT                                                       0x1c
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_X_MASK                                                         0x0000000FL
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S12_Y_MASK                                                         0x000000F0L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_X_MASK                                                         0x00000F00L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S13_Y_MASK                                                         0x0000F000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_X_MASK                                                         0x000F0000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S14_Y_MASK                                                         0x00F00000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_X_MASK                                                         0x0F000000L
+#define PA_SC_AA_SAMPLE_LOCS_PIXEL_X1Y1_3__S15_Y_MASK                                                         0xF0000000L
+//PA_SC_AA_MASK_X0Y0_X1Y0
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0__SHIFT                                                          0x0
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0__SHIFT                                                          0x10
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X0Y0_MASK                                                            0x0000FFFFL
+#define PA_SC_AA_MASK_X0Y0_X1Y0__AA_MASK_X1Y0_MASK                                                            0xFFFF0000L
+//PA_SC_AA_MASK_X0Y1_X1Y1
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1__SHIFT                                                          0x0
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1__SHIFT                                                          0x10
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X0Y1_MASK                                                            0x0000FFFFL
+#define PA_SC_AA_MASK_X0Y1_X1Y1__AA_MASK_X1Y1_MASK                                                            0xFFFF0000L
+//PA_SC_SHADER_CONTROL
+#define PA_SC_SHADER_CONTROL__REALIGN_DQUADS_AFTER_N_WAVES__SHIFT                                             0x0
+#define PA_SC_SHADER_CONTROL__LOAD_COLLISION_WAVEID__SHIFT                                                    0x2
+#define PA_SC_SHADER_CONTROL__LOAD_INTRAWAVE_COLLISION__SHIFT                                                 0x3
+#define PA_SC_SHADER_CONTROL__WAVE_BREAK_REGION_SIZE__SHIFT                                                   0x5
+#define PA_SC_SHADER_CONTROL__DISABLE_OREO_CONFLICT_QUAD__SHIFT                                               0x7
+#define PA_SC_SHADER_CONTROL__REALIGN_DQUADS_AFTER_N_WAVES_MASK                                               0x00000003L
+#define PA_SC_SHADER_CONTROL__LOAD_COLLISION_WAVEID_MASK                                                      0x00000004L
+#define PA_SC_SHADER_CONTROL__LOAD_INTRAWAVE_COLLISION_MASK                                                   0x00000008L
+#define PA_SC_SHADER_CONTROL__WAVE_BREAK_REGION_SIZE_MASK                                                     0x00000060L
+#define PA_SC_SHADER_CONTROL__DISABLE_OREO_CONFLICT_QUAD_MASK                                                 0x00000080L
+//PA_SC_BINNER_CNTL_0
+#define PA_SC_BINNER_CNTL_0__BINNING_MODE__SHIFT                                                              0x0
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X__SHIFT                                                                0x2
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y__SHIFT                                                                0x3
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_EXTEND__SHIFT                                                         0x4
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_EXTEND__SHIFT                                                         0x7
+#define PA_SC_BINNER_CNTL_0__CONTEXT_STATES_PER_BIN__SHIFT                                                    0xa
+#define PA_SC_BINNER_CNTL_0__PERSISTENT_STATES_PER_BIN__SHIFT                                                 0xd
+#define PA_SC_BINNER_CNTL_0__DISABLE_START_OF_PRIM__SHIFT                                                     0x12
+#define PA_SC_BINNER_CNTL_0__FPOVS_PER_BATCH__SHIFT                                                           0x13
+#define PA_SC_BINNER_CNTL_0__OPTIMAL_BIN_SELECTION__SHIFT                                                     0x1b
+#define PA_SC_BINNER_CNTL_0__FLUSH_ON_BINNING_TRANSITION__SHIFT                                               0x1c
+#define PA_SC_BINNER_CNTL_0__BIN_MAPPING_MODE__SHIFT                                                          0x1d
+#define PA_SC_BINNER_CNTL_0__FSR_EXPANSION_ENABLE__SHIFT                                                      0x1f
+#define PA_SC_BINNER_CNTL_0__BINNING_MODE_MASK                                                                0x00000003L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_MASK                                                                  0x00000004L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_MASK                                                                  0x00000008L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_X_EXTEND_MASK                                                           0x00000070L
+#define PA_SC_BINNER_CNTL_0__BIN_SIZE_Y_EXTEND_MASK                                                           0x00000380L
+#define PA_SC_BINNER_CNTL_0__CONTEXT_STATES_PER_BIN_MASK                                                      0x00001C00L
+#define PA_SC_BINNER_CNTL_0__PERSISTENT_STATES_PER_BIN_MASK                                                   0x0003E000L
+#define PA_SC_BINNER_CNTL_0__DISABLE_START_OF_PRIM_MASK                                                       0x00040000L
+#define PA_SC_BINNER_CNTL_0__FPOVS_PER_BATCH_MASK                                                             0x07F80000L
+#define PA_SC_BINNER_CNTL_0__OPTIMAL_BIN_SELECTION_MASK                                                       0x08000000L
+#define PA_SC_BINNER_CNTL_0__FLUSH_ON_BINNING_TRANSITION_MASK                                                 0x10000000L
+#define PA_SC_BINNER_CNTL_0__BIN_MAPPING_MODE_MASK                                                            0x60000000L
+#define PA_SC_BINNER_CNTL_0__FSR_EXPANSION_ENABLE_MASK                                                        0x80000000L
+//PA_SC_BINNER_CNTL_1
+#define PA_SC_BINNER_CNTL_1__MAX_ALLOC_COUNT__SHIFT                                                           0x0
+#define PA_SC_BINNER_CNTL_1__MAX_PRIM_PER_BATCH__SHIFT                                                        0x10
+#define PA_SC_BINNER_CNTL_1__MAX_ALLOC_COUNT_MASK                                                             0x0000FFFFL
+#define PA_SC_BINNER_CNTL_1__MAX_PRIM_PER_BATCH_MASK                                                          0xFFFF0000L
+//PA_SC_CONSERVATIVE_RASTERIZATION_CNTL
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_ENABLE__SHIFT                                        0x0
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_SAMPLE_SELECT__SHIFT                                 0x1
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_ENABLE__SHIFT                                       0x5
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_SAMPLE_SELECT__SHIFT                                0x6
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PBB_UNCERTAINTY_REGION_ENABLE__SHIFT                           0xa
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_EXTENT__SHIFT                                          0xb
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_OFFSET__SHIFT                                          0xc
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_OVER_RAST_INNER_TO_NORMAL__SHIFT                      0xd
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_UNDER_RAST_INNER_TO_NORMAL__SHIFT                     0xe
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__DEGENERATE_OVERRIDE_INNER_TO_NORMAL_DISABLE__SHIFT             0xf
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MODE__SHIFT                                 0x10
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OUTER_UNCERTAINTY_EDGERULE_OVERRIDE__SHIFT                     0x12
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__INNER_UNCERTAINTY_EDGERULE_OVERRIDE__SHIFT                     0x13
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__NULL_SQUAD_AA_MASK_ENABLE__SHIFT                               0x14
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__COVERAGE_AA_MASK_ENABLE__SHIFT                                 0x15
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PREZ_AA_MASK_ENABLE__SHIFT                                     0x16
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__POSTZ_AA_MASK_ENABLE__SHIFT                                    0x17
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__CENTROID_SAMPLE_OVERRIDE__SHIFT                                0x18
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MULT__SHIFT                                 0x19
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_PBB_MULT__SHIFT                             0x1b
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_ENABLE_MASK                                          0x00000001L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVER_RAST_SAMPLE_SELECT_MASK                                   0x0000001EL
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_ENABLE_MASK                                         0x00000020L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNDER_RAST_SAMPLE_SELECT_MASK                                  0x000003C0L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PBB_UNCERTAINTY_REGION_ENABLE_MASK                             0x00000400L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_EXTENT_MASK                                            0x00000800L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__ZMM_TRI_OFFSET_MASK                                            0x00001000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_OVER_RAST_INNER_TO_NORMAL_MASK                        0x00002000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OVERRIDE_UNDER_RAST_INNER_TO_NORMAL_MASK                       0x00004000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__DEGENERATE_OVERRIDE_INNER_TO_NORMAL_DISABLE_MASK               0x00008000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MODE_MASK                                   0x00030000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__OUTER_UNCERTAINTY_EDGERULE_OVERRIDE_MASK                       0x00040000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__INNER_UNCERTAINTY_EDGERULE_OVERRIDE_MASK                       0x00080000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__NULL_SQUAD_AA_MASK_ENABLE_MASK                                 0x00100000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__COVERAGE_AA_MASK_ENABLE_MASK                                   0x00200000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__PREZ_AA_MASK_ENABLE_MASK                                       0x00400000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__POSTZ_AA_MASK_ENABLE_MASK                                      0x00800000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__CENTROID_SAMPLE_OVERRIDE_MASK                                  0x01000000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_MULT_MASK                                   0x06000000L
+#define PA_SC_CONSERVATIVE_RASTERIZATION_CNTL__UNCERTAINTY_REGION_PBB_MULT_MASK                               0x18000000L
+//PA_SC_NGG_MODE_CNTL
+#define PA_SC_NGG_MODE_CNTL__MAX_DEALLOCS_IN_WAVE__SHIFT                                                      0x0
+#define PA_SC_NGG_MODE_CNTL__DISABLE_FPOG_AND_DEALLOC_CONFLICT__SHIFT                                         0xc
+#define PA_SC_NGG_MODE_CNTL__DISABLE_MAX_DEALLOC__SHIFT                                                       0xd
+#define PA_SC_NGG_MODE_CNTL__DISABLE_MAX_ATTRIBUTES__SHIFT                                                    0xe
+#define PA_SC_NGG_MODE_CNTL__MAX_FPOVS_IN_WAVE__SHIFT                                                         0x10
+#define PA_SC_NGG_MODE_CNTL__MAX_ATTRIBUTES_IN_WAVE__SHIFT                                                    0x18
+#define PA_SC_NGG_MODE_CNTL__MAX_DEALLOCS_IN_WAVE_MASK                                                        0x000007FFL
+#define PA_SC_NGG_MODE_CNTL__DISABLE_FPOG_AND_DEALLOC_CONFLICT_MASK                                           0x00001000L
+#define PA_SC_NGG_MODE_CNTL__DISABLE_MAX_DEALLOC_MASK                                                         0x00002000L
+#define PA_SC_NGG_MODE_CNTL__DISABLE_MAX_ATTRIBUTES_MASK                                                      0x00004000L
+#define PA_SC_NGG_MODE_CNTL__MAX_FPOVS_IN_WAVE_MASK                                                           0x00FF0000L
+#define PA_SC_NGG_MODE_CNTL__MAX_ATTRIBUTES_IN_WAVE_MASK                                                      0xFF000000L
+//PA_SC_BINNER_CNTL_2
+#define PA_SC_BINNER_CNTL_2__BIN_SIZE_X_MULT_BY_1P5X__SHIFT                                                   0x0
+#define PA_SC_BINNER_CNTL_2__BIN_SIZE_Y_MULT_BY_1P5X__SHIFT                                                   0x1
+#define PA_SC_BINNER_CNTL_2__ENABLE_LIGHT_VOLUME_RENDERING_OPTIMIZATION__SHIFT                                0x2
+#define PA_SC_BINNER_CNTL_2__DUAL_LIGHT_SHAFT_IN_DRAW__SHIFT                                                  0x3
+#define PA_SC_BINNER_CNTL_2__LIGHT_SHAFT_DRAW_CALL_LIMIT__SHIFT                                               0x4
+#define PA_SC_BINNER_CNTL_2__CONTEXT_DONE_EVENTS_PER_BIN__SHIFT                                               0x7
+#define PA_SC_BINNER_CNTL_2__ZPP_ENABLED__SHIFT                                                               0xb
+#define PA_SC_BINNER_CNTL_2__ZPP_OPTIMIZATION_ENABLED__SHIFT                                                  0xc
+#define PA_SC_BINNER_CNTL_2__ZPP_AREA_THRESHOLD__SHIFT                                                        0xd
+#define PA_SC_BINNER_CNTL_2__DISABLE_NOPCEXPORT_BREAKBATCH_CONDITION__SHIFT                                   0x15
+#define PA_SC_BINNER_CNTL_2__BIN_SIZE_X_MULT_BY_1P5X_MASK                                                     0x00000001L
+#define PA_SC_BINNER_CNTL_2__BIN_SIZE_Y_MULT_BY_1P5X_MASK                                                     0x00000002L
+#define PA_SC_BINNER_CNTL_2__ENABLE_LIGHT_VOLUME_RENDERING_OPTIMIZATION_MASK                                  0x00000004L
+#define PA_SC_BINNER_CNTL_2__DUAL_LIGHT_SHAFT_IN_DRAW_MASK                                                    0x00000008L
+#define PA_SC_BINNER_CNTL_2__LIGHT_SHAFT_DRAW_CALL_LIMIT_MASK                                                 0x00000070L
+#define PA_SC_BINNER_CNTL_2__CONTEXT_DONE_EVENTS_PER_BIN_MASK                                                 0x00000780L
+#define PA_SC_BINNER_CNTL_2__ZPP_ENABLED_MASK                                                                 0x00000800L
+#define PA_SC_BINNER_CNTL_2__ZPP_OPTIMIZATION_ENABLED_MASK                                                    0x00001000L
+#define PA_SC_BINNER_CNTL_2__ZPP_AREA_THRESHOLD_MASK                                                          0x001FE000L
+#define PA_SC_BINNER_CNTL_2__DISABLE_NOPCEXPORT_BREAKBATCH_CONDITION_MASK                                     0x00200000L
+//CB_COLOR0_BASE
+#define CB_COLOR0_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR0_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR0_VIEW
+#define CB_COLOR0_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR0_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR0_VIEW__MIP_LEVEL__SHIFT                                                                      0x1a
+#define CB_COLOR0_VIEW__SLICE_START_MASK                                                                      0x00001FFFL
+#define CB_COLOR0_VIEW__SLICE_MAX_MASK                                                                        0x03FFE000L
+#define CB_COLOR0_VIEW__MIP_LEVEL_MASK                                                                        0x3C000000L
+//CB_COLOR0_INFO
+#define CB_COLOR0_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR0_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR0_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR0_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR0_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR0_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR0_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR0_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR0_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR0_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR0_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR0_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR0_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR0_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR0_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR0_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR0_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR0_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+//CB_COLOR0_ATTRIB
+#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR0_ATTRIB__DISABLE_FMASK_NOALLOC_OPT__SHIFT                                                    0x3
+#define CB_COLOR0_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX__SHIFT                                                0x4
+#define CB_COLOR0_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x5
+#define CB_COLOR0_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR0_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR0_ATTRIB__DISABLE_FMASK_NOALLOC_OPT_MASK                                                      0x00000008L
+#define CB_COLOR0_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX_MASK                                                  0x00000010L
+#define CB_COLOR0_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000020L
+//CB_COLOR0_FDCC_CONTROL
+#define CB_COLOR0_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE__SHIFT                                            0x0
+#define CB_COLOR0_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE__SHIFT                                          0x1
+#define CB_COLOR0_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR0_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x4
+#define CB_COLOR0_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR0_FDCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                        0x7
+#define CB_COLOR0_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                 0x9
+#define CB_COLOR0_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS__SHIFT                                                0xa
+#define CB_COLOR0_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                            0x12
+#define CB_COLOR0_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                       0x13
+#define CB_COLOR0_FDCC_CONTROL__SKIP_LOW_COMP_RATIO__SHIFT                                                    0x15
+#define CB_COLOR0_FDCC_CONTROL__FDCC_ENABLE__SHIFT                                                            0x16
+#define CB_COLOR0_FDCC_CONTROL__DCC_COMPRESS_DISABLE__SHIFT                                                   0x17
+#define CB_COLOR0_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR0_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE_MASK                                              0x00000001L
+#define CB_COLOR0_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE_MASK                                            0x00000002L
+#define CB_COLOR0_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x0000000CL
+#define CB_COLOR0_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000010L
+#define CB_COLOR0_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR0_FDCC_CONTROL__COLOR_TRANSFORM_MASK                                                          0x00000180L
+#define CB_COLOR0_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                   0x00000200L
+#define CB_COLOR0_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS_MASK                                                  0x00000400L
+#define CB_COLOR0_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                              0x00040000L
+#define CB_COLOR0_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                         0x00080000L
+#define CB_COLOR0_FDCC_CONTROL__SKIP_LOW_COMP_RATIO_MASK                                                      0x00200000L
+#define CB_COLOR0_FDCC_CONTROL__FDCC_ENABLE_MASK                                                              0x00400000L
+#define CB_COLOR0_FDCC_CONTROL__DCC_COMPRESS_DISABLE_MASK                                                     0x00800000L
+#define CB_COLOR0_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+//CB_COLOR0_DCC_BASE
+#define CB_COLOR0_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR0_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR1_BASE
+#define CB_COLOR1_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR1_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR1_VIEW
+#define CB_COLOR1_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR1_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR1_VIEW__MIP_LEVEL__SHIFT                                                                      0x1a
+#define CB_COLOR1_VIEW__SLICE_START_MASK                                                                      0x00001FFFL
+#define CB_COLOR1_VIEW__SLICE_MAX_MASK                                                                        0x03FFE000L
+#define CB_COLOR1_VIEW__MIP_LEVEL_MASK                                                                        0x3C000000L
+//CB_COLOR1_INFO
+#define CB_COLOR1_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR1_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR1_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR1_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR1_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR1_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR1_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR1_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR1_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR1_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR1_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR1_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR1_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR1_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR1_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR1_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR1_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR1_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+//CB_COLOR1_ATTRIB
+#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR1_ATTRIB__DISABLE_FMASK_NOALLOC_OPT__SHIFT                                                    0x3
+#define CB_COLOR1_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX__SHIFT                                                0x4
+#define CB_COLOR1_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x5
+#define CB_COLOR1_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR1_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR1_ATTRIB__DISABLE_FMASK_NOALLOC_OPT_MASK                                                      0x00000008L
+#define CB_COLOR1_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX_MASK                                                  0x00000010L
+#define CB_COLOR1_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000020L
+//CB_COLOR1_FDCC_CONTROL
+#define CB_COLOR1_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE__SHIFT                                            0x0
+#define CB_COLOR1_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE__SHIFT                                          0x1
+#define CB_COLOR1_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR1_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x4
+#define CB_COLOR1_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR1_FDCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                        0x7
+#define CB_COLOR1_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                 0x9
+#define CB_COLOR1_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS__SHIFT                                                0xa
+#define CB_COLOR1_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                            0x12
+#define CB_COLOR1_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                       0x13
+#define CB_COLOR1_FDCC_CONTROL__SKIP_LOW_COMP_RATIO__SHIFT                                                    0x15
+#define CB_COLOR1_FDCC_CONTROL__FDCC_ENABLE__SHIFT                                                            0x16
+#define CB_COLOR1_FDCC_CONTROL__DCC_COMPRESS_DISABLE__SHIFT                                                   0x17
+#define CB_COLOR1_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR1_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE_MASK                                              0x00000001L
+#define CB_COLOR1_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE_MASK                                            0x00000002L
+#define CB_COLOR1_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x0000000CL
+#define CB_COLOR1_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000010L
+#define CB_COLOR1_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR1_FDCC_CONTROL__COLOR_TRANSFORM_MASK                                                          0x00000180L
+#define CB_COLOR1_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                   0x00000200L
+#define CB_COLOR1_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS_MASK                                                  0x00000400L
+#define CB_COLOR1_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                              0x00040000L
+#define CB_COLOR1_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                         0x00080000L
+#define CB_COLOR1_FDCC_CONTROL__SKIP_LOW_COMP_RATIO_MASK                                                      0x00200000L
+#define CB_COLOR1_FDCC_CONTROL__FDCC_ENABLE_MASK                                                              0x00400000L
+#define CB_COLOR1_FDCC_CONTROL__DCC_COMPRESS_DISABLE_MASK                                                     0x00800000L
+#define CB_COLOR1_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+//CB_COLOR1_DCC_BASE
+#define CB_COLOR1_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR1_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR2_BASE
+#define CB_COLOR2_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR2_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR2_VIEW
+#define CB_COLOR2_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR2_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR2_VIEW__MIP_LEVEL__SHIFT                                                                      0x1a
+#define CB_COLOR2_VIEW__SLICE_START_MASK                                                                      0x00001FFFL
+#define CB_COLOR2_VIEW__SLICE_MAX_MASK                                                                        0x03FFE000L
+#define CB_COLOR2_VIEW__MIP_LEVEL_MASK                                                                        0x3C000000L
+//CB_COLOR2_INFO
+#define CB_COLOR2_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR2_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR2_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR2_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR2_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR2_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR2_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR2_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR2_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR2_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR2_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR2_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR2_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR2_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR2_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR2_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR2_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR2_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+//CB_COLOR2_ATTRIB
+#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR2_ATTRIB__DISABLE_FMASK_NOALLOC_OPT__SHIFT                                                    0x3
+#define CB_COLOR2_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX__SHIFT                                                0x4
+#define CB_COLOR2_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x5
+#define CB_COLOR2_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR2_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR2_ATTRIB__DISABLE_FMASK_NOALLOC_OPT_MASK                                                      0x00000008L
+#define CB_COLOR2_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX_MASK                                                  0x00000010L
+#define CB_COLOR2_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000020L
+//CB_COLOR2_FDCC_CONTROL
+#define CB_COLOR2_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE__SHIFT                                            0x0
+#define CB_COLOR2_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE__SHIFT                                          0x1
+#define CB_COLOR2_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR2_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x4
+#define CB_COLOR2_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR2_FDCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                        0x7
+#define CB_COLOR2_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                 0x9
+#define CB_COLOR2_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS__SHIFT                                                0xa
+#define CB_COLOR2_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                            0x12
+#define CB_COLOR2_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                       0x13
+#define CB_COLOR2_FDCC_CONTROL__SKIP_LOW_COMP_RATIO__SHIFT                                                    0x15
+#define CB_COLOR2_FDCC_CONTROL__FDCC_ENABLE__SHIFT                                                            0x16
+#define CB_COLOR2_FDCC_CONTROL__DCC_COMPRESS_DISABLE__SHIFT                                                   0x17
+#define CB_COLOR2_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR2_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE_MASK                                              0x00000001L
+#define CB_COLOR2_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE_MASK                                            0x00000002L
+#define CB_COLOR2_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x0000000CL
+#define CB_COLOR2_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000010L
+#define CB_COLOR2_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR2_FDCC_CONTROL__COLOR_TRANSFORM_MASK                                                          0x00000180L
+#define CB_COLOR2_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                   0x00000200L
+#define CB_COLOR2_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS_MASK                                                  0x00000400L
+#define CB_COLOR2_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                              0x00040000L
+#define CB_COLOR2_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                         0x00080000L
+#define CB_COLOR2_FDCC_CONTROL__SKIP_LOW_COMP_RATIO_MASK                                                      0x00200000L
+#define CB_COLOR2_FDCC_CONTROL__FDCC_ENABLE_MASK                                                              0x00400000L
+#define CB_COLOR2_FDCC_CONTROL__DCC_COMPRESS_DISABLE_MASK                                                     0x00800000L
+#define CB_COLOR2_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+//CB_COLOR2_DCC_BASE
+#define CB_COLOR2_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR2_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR3_BASE
+#define CB_COLOR3_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR3_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR3_VIEW
+#define CB_COLOR3_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR3_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR3_VIEW__MIP_LEVEL__SHIFT                                                                      0x1a
+#define CB_COLOR3_VIEW__SLICE_START_MASK                                                                      0x00001FFFL
+#define CB_COLOR3_VIEW__SLICE_MAX_MASK                                                                        0x03FFE000L
+#define CB_COLOR3_VIEW__MIP_LEVEL_MASK                                                                        0x3C000000L
+//CB_COLOR3_INFO
+#define CB_COLOR3_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR3_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR3_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR3_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR3_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR3_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR3_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR3_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR3_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR3_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR3_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR3_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR3_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR3_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR3_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR3_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR3_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR3_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+//CB_COLOR3_ATTRIB
+#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR3_ATTRIB__DISABLE_FMASK_NOALLOC_OPT__SHIFT                                                    0x3
+#define CB_COLOR3_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX__SHIFT                                                0x4
+#define CB_COLOR3_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x5
+#define CB_COLOR3_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR3_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR3_ATTRIB__DISABLE_FMASK_NOALLOC_OPT_MASK                                                      0x00000008L
+#define CB_COLOR3_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX_MASK                                                  0x00000010L
+#define CB_COLOR3_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000020L
+//CB_COLOR3_FDCC_CONTROL
+#define CB_COLOR3_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE__SHIFT                                            0x0
+#define CB_COLOR3_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE__SHIFT                                          0x1
+#define CB_COLOR3_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR3_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x4
+#define CB_COLOR3_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR3_FDCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                        0x7
+#define CB_COLOR3_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                 0x9
+#define CB_COLOR3_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS__SHIFT                                                0xa
+#define CB_COLOR3_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                            0x12
+#define CB_COLOR3_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                       0x13
+#define CB_COLOR3_FDCC_CONTROL__SKIP_LOW_COMP_RATIO__SHIFT                                                    0x15
+#define CB_COLOR3_FDCC_CONTROL__FDCC_ENABLE__SHIFT                                                            0x16
+#define CB_COLOR3_FDCC_CONTROL__DCC_COMPRESS_DISABLE__SHIFT                                                   0x17
+#define CB_COLOR3_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR3_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE_MASK                                              0x00000001L
+#define CB_COLOR3_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE_MASK                                            0x00000002L
+#define CB_COLOR3_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x0000000CL
+#define CB_COLOR3_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000010L
+#define CB_COLOR3_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR3_FDCC_CONTROL__COLOR_TRANSFORM_MASK                                                          0x00000180L
+#define CB_COLOR3_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                   0x00000200L
+#define CB_COLOR3_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS_MASK                                                  0x00000400L
+#define CB_COLOR3_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                              0x00040000L
+#define CB_COLOR3_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                         0x00080000L
+#define CB_COLOR3_FDCC_CONTROL__SKIP_LOW_COMP_RATIO_MASK                                                      0x00200000L
+#define CB_COLOR3_FDCC_CONTROL__FDCC_ENABLE_MASK                                                              0x00400000L
+#define CB_COLOR3_FDCC_CONTROL__DCC_COMPRESS_DISABLE_MASK                                                     0x00800000L
+#define CB_COLOR3_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+//CB_COLOR3_DCC_BASE
+#define CB_COLOR3_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR3_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR4_BASE
+#define CB_COLOR4_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR4_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR4_VIEW
+#define CB_COLOR4_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR4_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR4_VIEW__MIP_LEVEL__SHIFT                                                                      0x1a
+#define CB_COLOR4_VIEW__SLICE_START_MASK                                                                      0x00001FFFL
+#define CB_COLOR4_VIEW__SLICE_MAX_MASK                                                                        0x03FFE000L
+#define CB_COLOR4_VIEW__MIP_LEVEL_MASK                                                                        0x3C000000L
+//CB_COLOR4_INFO
+#define CB_COLOR4_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR4_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR4_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR4_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR4_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR4_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR4_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR4_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR4_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR4_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR4_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR4_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR4_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR4_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR4_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR4_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR4_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR4_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+//CB_COLOR4_ATTRIB
+#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR4_ATTRIB__DISABLE_FMASK_NOALLOC_OPT__SHIFT                                                    0x3
+#define CB_COLOR4_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX__SHIFT                                                0x4
+#define CB_COLOR4_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x5
+#define CB_COLOR4_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR4_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR4_ATTRIB__DISABLE_FMASK_NOALLOC_OPT_MASK                                                      0x00000008L
+#define CB_COLOR4_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX_MASK                                                  0x00000010L
+#define CB_COLOR4_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000020L
+//CB_COLOR4_FDCC_CONTROL
+#define CB_COLOR4_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE__SHIFT                                            0x0
+#define CB_COLOR4_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE__SHIFT                                          0x1
+#define CB_COLOR4_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR4_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x4
+#define CB_COLOR4_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR4_FDCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                        0x7
+#define CB_COLOR4_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                 0x9
+#define CB_COLOR4_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS__SHIFT                                                0xa
+#define CB_COLOR4_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                            0x12
+#define CB_COLOR4_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                       0x13
+#define CB_COLOR4_FDCC_CONTROL__SKIP_LOW_COMP_RATIO__SHIFT                                                    0x15
+#define CB_COLOR4_FDCC_CONTROL__FDCC_ENABLE__SHIFT                                                            0x16
+#define CB_COLOR4_FDCC_CONTROL__DCC_COMPRESS_DISABLE__SHIFT                                                   0x17
+#define CB_COLOR4_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR4_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE_MASK                                              0x00000001L
+#define CB_COLOR4_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE_MASK                                            0x00000002L
+#define CB_COLOR4_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x0000000CL
+#define CB_COLOR4_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000010L
+#define CB_COLOR4_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR4_FDCC_CONTROL__COLOR_TRANSFORM_MASK                                                          0x00000180L
+#define CB_COLOR4_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                   0x00000200L
+#define CB_COLOR4_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS_MASK                                                  0x00000400L
+#define CB_COLOR4_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                              0x00040000L
+#define CB_COLOR4_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                         0x00080000L
+#define CB_COLOR4_FDCC_CONTROL__SKIP_LOW_COMP_RATIO_MASK                                                      0x00200000L
+#define CB_COLOR4_FDCC_CONTROL__FDCC_ENABLE_MASK                                                              0x00400000L
+#define CB_COLOR4_FDCC_CONTROL__DCC_COMPRESS_DISABLE_MASK                                                     0x00800000L
+#define CB_COLOR4_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+//CB_COLOR4_DCC_BASE
+#define CB_COLOR4_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR4_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR5_BASE
+#define CB_COLOR5_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR5_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR5_VIEW
+#define CB_COLOR5_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR5_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR5_VIEW__MIP_LEVEL__SHIFT                                                                      0x1a
+#define CB_COLOR5_VIEW__SLICE_START_MASK                                                                      0x00001FFFL
+#define CB_COLOR5_VIEW__SLICE_MAX_MASK                                                                        0x03FFE000L
+#define CB_COLOR5_VIEW__MIP_LEVEL_MASK                                                                        0x3C000000L
+//CB_COLOR5_INFO
+#define CB_COLOR5_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR5_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR5_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR5_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR5_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR5_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR5_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR5_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR5_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR5_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR5_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR5_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR5_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR5_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR5_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR5_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR5_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR5_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+//CB_COLOR5_ATTRIB
+#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR5_ATTRIB__DISABLE_FMASK_NOALLOC_OPT__SHIFT                                                    0x3
+#define CB_COLOR5_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX__SHIFT                                                0x4
+#define CB_COLOR5_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x5
+#define CB_COLOR5_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR5_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR5_ATTRIB__DISABLE_FMASK_NOALLOC_OPT_MASK                                                      0x00000008L
+#define CB_COLOR5_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX_MASK                                                  0x00000010L
+#define CB_COLOR5_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000020L
+//CB_COLOR5_FDCC_CONTROL
+#define CB_COLOR5_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE__SHIFT                                            0x0
+#define CB_COLOR5_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE__SHIFT                                          0x1
+#define CB_COLOR5_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR5_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x4
+#define CB_COLOR5_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR5_FDCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                        0x7
+#define CB_COLOR5_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                 0x9
+#define CB_COLOR5_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS__SHIFT                                                0xa
+#define CB_COLOR5_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                            0x12
+#define CB_COLOR5_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                       0x13
+#define CB_COLOR5_FDCC_CONTROL__SKIP_LOW_COMP_RATIO__SHIFT                                                    0x15
+#define CB_COLOR5_FDCC_CONTROL__FDCC_ENABLE__SHIFT                                                            0x16
+#define CB_COLOR5_FDCC_CONTROL__DCC_COMPRESS_DISABLE__SHIFT                                                   0x17
+#define CB_COLOR5_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR5_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE_MASK                                              0x00000001L
+#define CB_COLOR5_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE_MASK                                            0x00000002L
+#define CB_COLOR5_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x0000000CL
+#define CB_COLOR5_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000010L
+#define CB_COLOR5_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR5_FDCC_CONTROL__COLOR_TRANSFORM_MASK                                                          0x00000180L
+#define CB_COLOR5_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                   0x00000200L
+#define CB_COLOR5_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS_MASK                                                  0x00000400L
+#define CB_COLOR5_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                              0x00040000L
+#define CB_COLOR5_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                         0x00080000L
+#define CB_COLOR5_FDCC_CONTROL__SKIP_LOW_COMP_RATIO_MASK                                                      0x00200000L
+#define CB_COLOR5_FDCC_CONTROL__FDCC_ENABLE_MASK                                                              0x00400000L
+#define CB_COLOR5_FDCC_CONTROL__DCC_COMPRESS_DISABLE_MASK                                                     0x00800000L
+#define CB_COLOR5_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+//CB_COLOR5_DCC_BASE
+#define CB_COLOR5_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR5_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR6_BASE
+#define CB_COLOR6_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR6_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR6_VIEW
+#define CB_COLOR6_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR6_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR6_VIEW__MIP_LEVEL__SHIFT                                                                      0x1a
+#define CB_COLOR6_VIEW__SLICE_START_MASK                                                                      0x00001FFFL
+#define CB_COLOR6_VIEW__SLICE_MAX_MASK                                                                        0x03FFE000L
+#define CB_COLOR6_VIEW__MIP_LEVEL_MASK                                                                        0x3C000000L
+//CB_COLOR6_INFO
+#define CB_COLOR6_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR6_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR6_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR6_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR6_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR6_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR6_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR6_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR6_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR6_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR6_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR6_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR6_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR6_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR6_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR6_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR6_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR6_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+//CB_COLOR6_ATTRIB
+#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR6_ATTRIB__DISABLE_FMASK_NOALLOC_OPT__SHIFT                                                    0x3
+#define CB_COLOR6_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX__SHIFT                                                0x4
+#define CB_COLOR6_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x5
+#define CB_COLOR6_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR6_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR6_ATTRIB__DISABLE_FMASK_NOALLOC_OPT_MASK                                                      0x00000008L
+#define CB_COLOR6_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX_MASK                                                  0x00000010L
+#define CB_COLOR6_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000020L
+//CB_COLOR6_FDCC_CONTROL
+#define CB_COLOR6_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE__SHIFT                                            0x0
+#define CB_COLOR6_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE__SHIFT                                          0x1
+#define CB_COLOR6_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR6_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x4
+#define CB_COLOR6_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR6_FDCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                        0x7
+#define CB_COLOR6_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                 0x9
+#define CB_COLOR6_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS__SHIFT                                                0xa
+#define CB_COLOR6_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                            0x12
+#define CB_COLOR6_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                       0x13
+#define CB_COLOR6_FDCC_CONTROL__SKIP_LOW_COMP_RATIO__SHIFT                                                    0x15
+#define CB_COLOR6_FDCC_CONTROL__FDCC_ENABLE__SHIFT                                                            0x16
+#define CB_COLOR6_FDCC_CONTROL__DCC_COMPRESS_DISABLE__SHIFT                                                   0x17
+#define CB_COLOR6_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR6_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE_MASK                                              0x00000001L
+#define CB_COLOR6_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE_MASK                                            0x00000002L
+#define CB_COLOR6_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x0000000CL
+#define CB_COLOR6_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000010L
+#define CB_COLOR6_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR6_FDCC_CONTROL__COLOR_TRANSFORM_MASK                                                          0x00000180L
+#define CB_COLOR6_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                   0x00000200L
+#define CB_COLOR6_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS_MASK                                                  0x00000400L
+#define CB_COLOR6_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                              0x00040000L
+#define CB_COLOR6_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                         0x00080000L
+#define CB_COLOR6_FDCC_CONTROL__SKIP_LOW_COMP_RATIO_MASK                                                      0x00200000L
+#define CB_COLOR6_FDCC_CONTROL__FDCC_ENABLE_MASK                                                              0x00400000L
+#define CB_COLOR6_FDCC_CONTROL__DCC_COMPRESS_DISABLE_MASK                                                     0x00800000L
+#define CB_COLOR6_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+//CB_COLOR6_DCC_BASE
+#define CB_COLOR6_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR6_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR7_BASE
+#define CB_COLOR7_BASE__BASE_256B__SHIFT                                                                      0x0
+#define CB_COLOR7_BASE__BASE_256B_MASK                                                                        0xFFFFFFFFL
+//CB_COLOR7_VIEW
+#define CB_COLOR7_VIEW__SLICE_START__SHIFT                                                                    0x0
+#define CB_COLOR7_VIEW__SLICE_MAX__SHIFT                                                                      0xd
+#define CB_COLOR7_VIEW__MIP_LEVEL__SHIFT                                                                      0x1a
+#define CB_COLOR7_VIEW__SLICE_START_MASK                                                                      0x00001FFFL
+#define CB_COLOR7_VIEW__SLICE_MAX_MASK                                                                        0x03FFE000L
+#define CB_COLOR7_VIEW__MIP_LEVEL_MASK                                                                        0x3C000000L
+//CB_COLOR7_INFO
+#define CB_COLOR7_INFO__FORMAT__SHIFT                                                                         0x0
+#define CB_COLOR7_INFO__LINEAR_GENERAL__SHIFT                                                                 0x7
+#define CB_COLOR7_INFO__NUMBER_TYPE__SHIFT                                                                    0x8
+#define CB_COLOR7_INFO__COMP_SWAP__SHIFT                                                                      0xb
+#define CB_COLOR7_INFO__BLEND_CLAMP__SHIFT                                                                    0xf
+#define CB_COLOR7_INFO__BLEND_BYPASS__SHIFT                                                                   0x10
+#define CB_COLOR7_INFO__SIMPLE_FLOAT__SHIFT                                                                   0x11
+#define CB_COLOR7_INFO__ROUND_MODE__SHIFT                                                                     0x12
+#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST__SHIFT                                                          0x14
+#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL__SHIFT                                                        0x17
+#define CB_COLOR7_INFO__FORMAT_MASK                                                                           0x0000001FL
+#define CB_COLOR7_INFO__LINEAR_GENERAL_MASK                                                                   0x00000080L
+#define CB_COLOR7_INFO__NUMBER_TYPE_MASK                                                                      0x00000700L
+#define CB_COLOR7_INFO__COMP_SWAP_MASK                                                                        0x00001800L
+#define CB_COLOR7_INFO__BLEND_CLAMP_MASK                                                                      0x00008000L
+#define CB_COLOR7_INFO__BLEND_BYPASS_MASK                                                                     0x00010000L
+#define CB_COLOR7_INFO__SIMPLE_FLOAT_MASK                                                                     0x00020000L
+#define CB_COLOR7_INFO__ROUND_MODE_MASK                                                                       0x00040000L
+#define CB_COLOR7_INFO__BLEND_OPT_DONT_RD_DST_MASK                                                            0x00700000L
+#define CB_COLOR7_INFO__BLEND_OPT_DISCARD_PIXEL_MASK                                                          0x03800000L
+//CB_COLOR7_ATTRIB
+#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS__SHIFT                                                                0x0
+#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1__SHIFT                                                            0x2
+#define CB_COLOR7_ATTRIB__DISABLE_FMASK_NOALLOC_OPT__SHIFT                                                    0x3
+#define CB_COLOR7_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX__SHIFT                                                0x4
+#define CB_COLOR7_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX__SHIFT                                         0x5
+#define CB_COLOR7_ATTRIB__NUM_FRAGMENTS_MASK                                                                  0x00000003L
+#define CB_COLOR7_ATTRIB__FORCE_DST_ALPHA_1_MASK                                                              0x00000004L
+#define CB_COLOR7_ATTRIB__DISABLE_FMASK_NOALLOC_OPT_MASK                                                      0x00000008L
+#define CB_COLOR7_ATTRIB__LIMIT_COLOR_FETCH_TO_256B_MAX_MASK                                                  0x00000010L
+#define CB_COLOR7_ATTRIB__FORCE_LIMIT_COLOR_SECTOR_TO_256B_MAX_MASK                                           0x00000020L
+//CB_COLOR7_FDCC_CONTROL
+#define CB_COLOR7_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE__SHIFT                                            0x0
+#define CB_COLOR7_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE__SHIFT                                          0x1
+#define CB_COLOR7_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE__SHIFT                                            0x2
+#define CB_COLOR7_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x4
+#define CB_COLOR7_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE__SHIFT                                              0x5
+#define CB_COLOR7_FDCC_CONTROL__COLOR_TRANSFORM__SHIFT                                                        0x7
+#define CB_COLOR7_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS__SHIFT                                                 0x9
+#define CB_COLOR7_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS__SHIFT                                                0xa
+#define CB_COLOR7_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG__SHIFT                                            0x12
+#define CB_COLOR7_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE__SHIFT                                       0x13
+#define CB_COLOR7_FDCC_CONTROL__SKIP_LOW_COMP_RATIO__SHIFT                                                    0x15
+#define CB_COLOR7_FDCC_CONTROL__FDCC_ENABLE__SHIFT                                                            0x16
+#define CB_COLOR7_FDCC_CONTROL__DCC_COMPRESS_DISABLE__SHIFT                                                   0x17
+#define CB_COLOR7_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE__SHIFT                                              0x18
+#define CB_COLOR7_FDCC_CONTROL__SAMPLE_MASK_TRACKER_DISABLE_MASK                                              0x00000001L
+#define CB_COLOR7_FDCC_CONTROL__SAMPLE_MASK_TRACKER_FEA_FORCE_MASK                                            0x00000002L
+#define CB_COLOR7_FDCC_CONTROL__MAX_UNCOMPRESSED_BLOCK_SIZE_MASK                                              0x0000000CL
+#define CB_COLOR7_FDCC_CONTROL__MIN_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000010L
+#define CB_COLOR7_FDCC_CONTROL__MAX_COMPRESSED_BLOCK_SIZE_MASK                                                0x00000060L
+#define CB_COLOR7_FDCC_CONTROL__COLOR_TRANSFORM_MASK                                                          0x00000180L
+#define CB_COLOR7_FDCC_CONTROL__INDEPENDENT_64B_BLOCKS_MASK                                                   0x00000200L
+#define CB_COLOR7_FDCC_CONTROL__INDEPENDENT_128B_BLOCKS_MASK                                                  0x00000400L
+#define CB_COLOR7_FDCC_CONTROL__DISABLE_CONSTANT_ENCODE_REG_MASK                                              0x00040000L
+#define CB_COLOR7_FDCC_CONTROL__ENABLE_CONSTANT_ENCODE_REG_WRITE_MASK                                         0x00080000L
+#define CB_COLOR7_FDCC_CONTROL__SKIP_LOW_COMP_RATIO_MASK                                                      0x00200000L
+#define CB_COLOR7_FDCC_CONTROL__FDCC_ENABLE_MASK                                                              0x00400000L
+#define CB_COLOR7_FDCC_CONTROL__DCC_COMPRESS_DISABLE_MASK                                                     0x00800000L
+#define CB_COLOR7_FDCC_CONTROL__FRAGMENT_COMPRESS_DISABLE_MASK                                                0x01000000L
+//CB_COLOR7_DCC_BASE
+#define CB_COLOR7_DCC_BASE__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR7_DCC_BASE__BASE_256B_MASK                                                                    0xFFFFFFFFL
+//CB_COLOR0_BASE_EXT
+#define CB_COLOR0_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR0_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR1_BASE_EXT
+#define CB_COLOR1_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR1_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR2_BASE_EXT
+#define CB_COLOR2_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR2_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR3_BASE_EXT
+#define CB_COLOR3_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR3_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR4_BASE_EXT
+#define CB_COLOR4_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR4_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR5_BASE_EXT
+#define CB_COLOR5_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR5_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR6_BASE_EXT
+#define CB_COLOR6_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR6_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR7_BASE_EXT
+#define CB_COLOR7_BASE_EXT__BASE_256B__SHIFT                                                                  0x0
+#define CB_COLOR7_BASE_EXT__BASE_256B_MASK                                                                    0x000000FFL
+//CB_COLOR0_DCC_BASE_EXT
+#define CB_COLOR0_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR0_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR1_DCC_BASE_EXT
+#define CB_COLOR1_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR1_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR2_DCC_BASE_EXT
+#define CB_COLOR2_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR2_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR3_DCC_BASE_EXT
+#define CB_COLOR3_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR3_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR4_DCC_BASE_EXT
+#define CB_COLOR4_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR4_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR5_DCC_BASE_EXT
+#define CB_COLOR5_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR5_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR6_DCC_BASE_EXT
+#define CB_COLOR6_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR6_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR7_DCC_BASE_EXT
+#define CB_COLOR7_DCC_BASE_EXT__BASE_256B__SHIFT                                                              0x0
+#define CB_COLOR7_DCC_BASE_EXT__BASE_256B_MASK                                                                0x000000FFL
+//CB_COLOR0_ATTRIB2
+#define CB_COLOR0_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR0_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR0_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR0_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR0_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR0_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR1_ATTRIB2
+#define CB_COLOR1_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR1_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR1_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR1_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR1_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR1_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR2_ATTRIB2
+#define CB_COLOR2_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR2_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR2_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR2_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR2_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR2_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR3_ATTRIB2
+#define CB_COLOR3_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR3_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR3_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR3_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR3_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR3_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR4_ATTRIB2
+#define CB_COLOR4_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR4_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR4_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR4_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR4_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR4_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR5_ATTRIB2
+#define CB_COLOR5_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR5_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR5_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR5_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR5_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR5_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR6_ATTRIB2
+#define CB_COLOR6_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR6_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR6_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR6_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR6_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR6_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR7_ATTRIB2
+#define CB_COLOR7_ATTRIB2__MIP0_HEIGHT__SHIFT                                                                 0x0
+#define CB_COLOR7_ATTRIB2__MIP0_WIDTH__SHIFT                                                                  0xe
+#define CB_COLOR7_ATTRIB2__MAX_MIP__SHIFT                                                                     0x1c
+#define CB_COLOR7_ATTRIB2__MIP0_HEIGHT_MASK                                                                   0x00003FFFL
+#define CB_COLOR7_ATTRIB2__MIP0_WIDTH_MASK                                                                    0x0FFFC000L
+#define CB_COLOR7_ATTRIB2__MAX_MIP_MASK                                                                       0xF0000000L
+//CB_COLOR0_ATTRIB3
+#define CB_COLOR0_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR0_ATTRIB3__META_LINEAR__SHIFT                                                                 0xd
+#define CB_COLOR0_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xe
+#define CB_COLOR0_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR0_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT                                                            0x1e
+#define CB_COLOR0_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00001FFFL
+#define CB_COLOR0_ATTRIB3__META_LINEAR_MASK                                                                   0x00002000L
+#define CB_COLOR0_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x0007C000L
+#define CB_COLOR0_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR0_ATTRIB3__DCC_PIPE_ALIGNED_MASK                                                              0x40000000L
+//CB_COLOR1_ATTRIB3
+#define CB_COLOR1_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR1_ATTRIB3__META_LINEAR__SHIFT                                                                 0xd
+#define CB_COLOR1_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xe
+#define CB_COLOR1_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR1_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT                                                            0x1e
+#define CB_COLOR1_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00001FFFL
+#define CB_COLOR1_ATTRIB3__META_LINEAR_MASK                                                                   0x00002000L
+#define CB_COLOR1_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x0007C000L
+#define CB_COLOR1_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR1_ATTRIB3__DCC_PIPE_ALIGNED_MASK                                                              0x40000000L
+//CB_COLOR2_ATTRIB3
+#define CB_COLOR2_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR2_ATTRIB3__META_LINEAR__SHIFT                                                                 0xd
+#define CB_COLOR2_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xe
+#define CB_COLOR2_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR2_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT                                                            0x1e
+#define CB_COLOR2_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00001FFFL
+#define CB_COLOR2_ATTRIB3__META_LINEAR_MASK                                                                   0x00002000L
+#define CB_COLOR2_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x0007C000L
+#define CB_COLOR2_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR2_ATTRIB3__DCC_PIPE_ALIGNED_MASK                                                              0x40000000L
+//CB_COLOR3_ATTRIB3
+#define CB_COLOR3_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR3_ATTRIB3__META_LINEAR__SHIFT                                                                 0xd
+#define CB_COLOR3_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xe
+#define CB_COLOR3_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR3_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT                                                            0x1e
+#define CB_COLOR3_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00001FFFL
+#define CB_COLOR3_ATTRIB3__META_LINEAR_MASK                                                                   0x00002000L
+#define CB_COLOR3_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x0007C000L
+#define CB_COLOR3_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR3_ATTRIB3__DCC_PIPE_ALIGNED_MASK                                                              0x40000000L
+//CB_COLOR4_ATTRIB3
+#define CB_COLOR4_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR4_ATTRIB3__META_LINEAR__SHIFT                                                                 0xd
+#define CB_COLOR4_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xe
+#define CB_COLOR4_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR4_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT                                                            0x1e
+#define CB_COLOR4_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00001FFFL
+#define CB_COLOR4_ATTRIB3__META_LINEAR_MASK                                                                   0x00002000L
+#define CB_COLOR4_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x0007C000L
+#define CB_COLOR4_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR4_ATTRIB3__DCC_PIPE_ALIGNED_MASK                                                              0x40000000L
+//CB_COLOR5_ATTRIB3
+#define CB_COLOR5_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR5_ATTRIB3__META_LINEAR__SHIFT                                                                 0xd
+#define CB_COLOR5_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xe
+#define CB_COLOR5_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR5_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT                                                            0x1e
+#define CB_COLOR5_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00001FFFL
+#define CB_COLOR5_ATTRIB3__META_LINEAR_MASK                                                                   0x00002000L
+#define CB_COLOR5_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x0007C000L
+#define CB_COLOR5_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR5_ATTRIB3__DCC_PIPE_ALIGNED_MASK                                                              0x40000000L
+//CB_COLOR6_ATTRIB3
+#define CB_COLOR6_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR6_ATTRIB3__META_LINEAR__SHIFT                                                                 0xd
+#define CB_COLOR6_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xe
+#define CB_COLOR6_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR6_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT                                                            0x1e
+#define CB_COLOR6_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00001FFFL
+#define CB_COLOR6_ATTRIB3__META_LINEAR_MASK                                                                   0x00002000L
+#define CB_COLOR6_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x0007C000L
+#define CB_COLOR6_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR6_ATTRIB3__DCC_PIPE_ALIGNED_MASK                                                              0x40000000L
+//CB_COLOR7_ATTRIB3
+#define CB_COLOR7_ATTRIB3__MIP0_DEPTH__SHIFT                                                                  0x0
+#define CB_COLOR7_ATTRIB3__META_LINEAR__SHIFT                                                                 0xd
+#define CB_COLOR7_ATTRIB3__COLOR_SW_MODE__SHIFT                                                               0xe
+#define CB_COLOR7_ATTRIB3__RESOURCE_TYPE__SHIFT                                                               0x18
+#define CB_COLOR7_ATTRIB3__DCC_PIPE_ALIGNED__SHIFT                                                            0x1e
+#define CB_COLOR7_ATTRIB3__MIP0_DEPTH_MASK                                                                    0x00001FFFL
+#define CB_COLOR7_ATTRIB3__META_LINEAR_MASK                                                                   0x00002000L
+#define CB_COLOR7_ATTRIB3__COLOR_SW_MODE_MASK                                                                 0x0007C000L
+#define CB_COLOR7_ATTRIB3__RESOURCE_TYPE_MASK                                                                 0x03000000L
+#define CB_COLOR7_ATTRIB3__DCC_PIPE_ALIGNED_MASK                                                              0x40000000L
+
+
+// addressBlock: gc_pfvf_cpdec
+//CONFIG_RESERVED_REG0
+#define CONFIG_RESERVED_REG0__DATA__SHIFT                                                                     0x0
+#define CONFIG_RESERVED_REG0__DATA_MASK                                                                       0xFFFFFFFFL
+//CONFIG_RESERVED_REG1
+#define CONFIG_RESERVED_REG1__DATA__SHIFT                                                                     0x0
+#define CONFIG_RESERVED_REG1__DATA_MASK                                                                       0xFFFFFFFFL
+//CP_MEC_CNTL
+#define CP_MEC_CNTL__MEC_ME1_PIPE0_RESET__SHIFT                                                               0x10
+#define CP_MEC_CNTL__MEC_ME1_PIPE1_RESET__SHIFT                                                               0x11
+#define CP_MEC_CNTL__MEC_ME1_PIPE2_RESET__SHIFT                                                               0x12
+#define CP_MEC_CNTL__MEC_ME1_PIPE3_RESET__SHIFT                                                               0x13
+#define CP_MEC_CNTL__MEC_ME2_PIPE0_RESET__SHIFT                                                               0x14
+#define CP_MEC_CNTL__MEC_ME2_PIPE1_RESET__SHIFT                                                               0x15
+#define CP_MEC_CNTL__MEC_ME2_PIPE2_RESET__SHIFT                                                               0x16
+#define CP_MEC_CNTL__MEC_ME2_PIPE3_RESET__SHIFT                                                               0x17
+#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE__SHIFT                                                             0x1b
+#define CP_MEC_CNTL__MEC_ME2_HALT__SHIFT                                                                      0x1c
+#define CP_MEC_CNTL__MEC_ME2_STEP__SHIFT                                                                      0x1d
+#define CP_MEC_CNTL__MEC_ME1_HALT__SHIFT                                                                      0x1e
+#define CP_MEC_CNTL__MEC_ME1_STEP__SHIFT                                                                      0x1f
+#define CP_MEC_CNTL__MEC_ME1_PIPE0_RESET_MASK                                                                 0x00010000L
+#define CP_MEC_CNTL__MEC_ME1_PIPE1_RESET_MASK                                                                 0x00020000L
+#define CP_MEC_CNTL__MEC_ME1_PIPE2_RESET_MASK                                                                 0x00040000L
+#define CP_MEC_CNTL__MEC_ME1_PIPE3_RESET_MASK                                                                 0x00080000L
+#define CP_MEC_CNTL__MEC_ME2_PIPE0_RESET_MASK                                                                 0x00100000L
+#define CP_MEC_CNTL__MEC_ME2_PIPE1_RESET_MASK                                                                 0x00200000L
+#define CP_MEC_CNTL__MEC_ME2_PIPE2_RESET_MASK                                                                 0x00400000L
+#define CP_MEC_CNTL__MEC_ME2_PIPE3_RESET_MASK                                                                 0x00800000L
+#define CP_MEC_CNTL__MEC_INVALIDATE_ICACHE_MASK                                                               0x08000000L
+#define CP_MEC_CNTL__MEC_ME2_HALT_MASK                                                                        0x10000000L
+#define CP_MEC_CNTL__MEC_ME2_STEP_MASK                                                                        0x20000000L
+#define CP_MEC_CNTL__MEC_ME1_HALT_MASK                                                                        0x40000000L
+#define CP_MEC_CNTL__MEC_ME1_STEP_MASK                                                                        0x80000000L
+//CP_ME_CNTL
+#define CP_ME_CNTL__CE_INVALIDATE_ICACHE__SHIFT                                                               0x4
+#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE__SHIFT                                                              0x6
+#define CP_ME_CNTL__ME_INVALIDATE_ICACHE__SHIFT                                                               0x8
+#define CP_ME_CNTL__PFP_PIPE0_DISABLE__SHIFT                                                                  0xc
+#define CP_ME_CNTL__PFP_PIPE1_DISABLE__SHIFT                                                                  0xd
+#define CP_ME_CNTL__ME_PIPE0_DISABLE__SHIFT                                                                   0xe
+#define CP_ME_CNTL__ME_PIPE1_DISABLE__SHIFT                                                                   0xf
+#define CP_ME_CNTL__CE_PIPE0_RESET__SHIFT                                                                     0x10
+#define CP_ME_CNTL__CE_PIPE1_RESET__SHIFT                                                                     0x11
+#define CP_ME_CNTL__PFP_PIPE0_RESET__SHIFT                                                                    0x12
+#define CP_ME_CNTL__PFP_PIPE1_RESET__SHIFT                                                                    0x13
+#define CP_ME_CNTL__ME_PIPE0_RESET__SHIFT                                                                     0x14
+#define CP_ME_CNTL__ME_PIPE1_RESET__SHIFT                                                                     0x15
+#define CP_ME_CNTL__CE_HALT__SHIFT                                                                            0x18
+#define CP_ME_CNTL__CE_STEP__SHIFT                                                                            0x19
+#define CP_ME_CNTL__PFP_HALT__SHIFT                                                                           0x1a
+#define CP_ME_CNTL__PFP_STEP__SHIFT                                                                           0x1b
+#define CP_ME_CNTL__ME_HALT__SHIFT                                                                            0x1c
+#define CP_ME_CNTL__ME_STEP__SHIFT                                                                            0x1d
+#define CP_ME_CNTL__CE_INVALIDATE_ICACHE_MASK                                                                 0x00000010L
+#define CP_ME_CNTL__PFP_INVALIDATE_ICACHE_MASK                                                                0x00000040L
+#define CP_ME_CNTL__ME_INVALIDATE_ICACHE_MASK                                                                 0x00000100L
+#define CP_ME_CNTL__PFP_PIPE0_DISABLE_MASK                                                                    0x00001000L
+#define CP_ME_CNTL__PFP_PIPE1_DISABLE_MASK                                                                    0x00002000L
+#define CP_ME_CNTL__ME_PIPE0_DISABLE_MASK                                                                     0x00004000L
+#define CP_ME_CNTL__ME_PIPE1_DISABLE_MASK                                                                     0x00008000L
+#define CP_ME_CNTL__CE_PIPE0_RESET_MASK                                                                       0x00010000L
+#define CP_ME_CNTL__CE_PIPE1_RESET_MASK                                                                       0x00020000L
+#define CP_ME_CNTL__PFP_PIPE0_RESET_MASK                                                                      0x00040000L
+#define CP_ME_CNTL__PFP_PIPE1_RESET_MASK                                                                      0x00080000L
+#define CP_ME_CNTL__ME_PIPE0_RESET_MASK                                                                       0x00100000L
+#define CP_ME_CNTL__ME_PIPE1_RESET_MASK                                                                       0x00200000L
+#define CP_ME_CNTL__CE_HALT_MASK                                                                              0x01000000L
+#define CP_ME_CNTL__CE_STEP_MASK                                                                              0x02000000L
+#define CP_ME_CNTL__PFP_HALT_MASK                                                                             0x04000000L
+#define CP_ME_CNTL__PFP_STEP_MASK                                                                             0x08000000L
+#define CP_ME_CNTL__ME_HALT_MASK                                                                              0x10000000L
+#define CP_ME_CNTL__ME_STEP_MASK                                                                              0x20000000L
+
+
+// addressBlock: gc_pfvf_grbmdec
+//GRBM_GFX_CNTL
+#define GRBM_GFX_CNTL__PIPEID__SHIFT                                                                          0x0
+#define GRBM_GFX_CNTL__MEID__SHIFT                                                                            0x2
+#define GRBM_GFX_CNTL__VMID__SHIFT                                                                            0x4
+#define GRBM_GFX_CNTL__QUEUEID__SHIFT                                                                         0x8
+#define GRBM_GFX_CNTL__CTXID__SHIFT                                                                           0xb
+#define GRBM_GFX_CNTL__PIPEID_MASK                                                                            0x00000003L
+#define GRBM_GFX_CNTL__MEID_MASK                                                                              0x0000000CL
+#define GRBM_GFX_CNTL__VMID_MASK                                                                              0x000000F0L
+#define GRBM_GFX_CNTL__QUEUEID_MASK                                                                           0x00000700L
+#define GRBM_GFX_CNTL__CTXID_MASK                                                                             0x00003800L
+//GRBM_NOWHERE
+#define GRBM_NOWHERE__DATA__SHIFT                                                                             0x0
+#define GRBM_NOWHERE__DATA_MASK                                                                               0xFFFFFFFFL
+
+
+// addressBlock: gc_pfvf_padec
+//PA_SC_VRS_SURFACE_CNTL
+#define PA_SC_VRS_SURFACE_CNTL__VRC_CONTEXT_DONE_SYNC_DISABLE__SHIFT                                          0x6
+#define PA_SC_VRS_SURFACE_CNTL__VRS_FEEDBACK_RATE_OVERRIDE__SHIFT                                             0x7
+#define PA_SC_VRS_SURFACE_CNTL__VRC_FLUSH_EVENT_MASK_DISABLE__SHIFT                                           0x8
+#define PA_SC_VRS_SURFACE_CNTL__VRC_PREFETCH_DISABLE__SHIFT                                                   0xd
+#define PA_SC_VRS_SURFACE_CNTL__VRC_FLUSH_NO_INV_DISABLE__SHIFT                                               0xe
+#define PA_SC_VRS_SURFACE_CNTL__VRC_NONSTALLING_FLUSH_DISABLE__SHIFT                                          0xf
+#define PA_SC_VRS_SURFACE_CNTL__VRC_PARTIAL_FLUSH_DISABLE__SHIFT                                              0x10
+#define PA_SC_VRS_SURFACE_CNTL__VRC_AUTO_FLUSH__SHIFT                                                         0x11
+#define PA_SC_VRS_SURFACE_CNTL__VRC_EOP_SYNC_DISABLE__SHIFT                                                   0x12
+#define PA_SC_VRS_SURFACE_CNTL__VRC_MAX_TAGS__SHIFT                                                           0x13
+#define PA_SC_VRS_SURFACE_CNTL__VRC_EVICT_POINT__SHIFT                                                        0x1a
+#define PA_SC_VRS_SURFACE_CNTL__VRC_CONTEXT_DONE_SYNC_DISABLE_MASK                                            0x00000040L
+#define PA_SC_VRS_SURFACE_CNTL__VRS_FEEDBACK_RATE_OVERRIDE_MASK                                               0x00000080L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_FLUSH_EVENT_MASK_DISABLE_MASK                                             0x00001F00L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_PREFETCH_DISABLE_MASK                                                     0x00002000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_FLUSH_NO_INV_DISABLE_MASK                                                 0x00004000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_NONSTALLING_FLUSH_DISABLE_MASK                                            0x00008000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_PARTIAL_FLUSH_DISABLE_MASK                                                0x00010000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_AUTO_FLUSH_MASK                                                           0x00020000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_EOP_SYNC_DISABLE_MASK                                                     0x00040000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_MAX_TAGS_MASK                                                             0x03F80000L
+#define PA_SC_VRS_SURFACE_CNTL__VRC_EVICT_POINT_MASK                                                          0xFC000000L
+//PA_SC_ENHANCE
+#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER__SHIFT                                                       0x0
+#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX__SHIFT                                                          0x1
+#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX__SHIFT                                                        0x2
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS__SHIFT                                                  0x3
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID__SHIFT                                               0x4
+#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX__SHIFT                                                             0x5
+#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER__SHIFT                                                     0x6
+#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION__SHIFT                                              0x7
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM__SHIFT                                                   0x8
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE__SHIFT                                              0x9
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE__SHIFT                                                   0xa
+#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE__SHIFT                                                          0xb
+#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS__SHIFT                                          0xc
+#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE__SHIFT                                                 0xd
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE__SHIFT                                             0xe
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE__SHIFT                                                   0xf
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST__SHIFT                                   0x10
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING__SHIFT                                        0x11
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY__SHIFT                               0x12
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING__SHIFT                               0x13
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING__SHIFT                              0x14
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS__SHIFT                                 0x15
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID__SHIFT                                   0x16
+#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO__SHIFT                           0x17
+#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT__SHIFT                                          0x18
+#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING__SHIFT                                       0x19
+#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET__SHIFT                                                  0x1a
+#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET__SHIFT                                              0x1b
+#define PA_SC_ENHANCE__IOO_DISABLE_SCAN_UNSELECTED_FIFOS_FOR_DUAL_GFX_RING_CHANGE__SHIFT                      0x1c
+#define PA_SC_ENHANCE__OOO_USE_ABSOLUTE_FIFO_COUNT_IN_THRESHOLD_SWITCHING__SHIFT                              0x1d
+#define PA_SC_ENHANCE__ENABLE_PA_SC_OUT_OF_ORDER_MASK                                                         0x00000001L
+#define PA_SC_ENHANCE__DISABLE_SC_DB_TILE_FIX_MASK                                                            0x00000002L
+#define PA_SC_ENHANCE__DISABLE_AA_MASK_FULL_FIX_MASK                                                          0x00000004L
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOCATIONS_MASK                                                    0x00000008L
+#define PA_SC_ENHANCE__ENABLE_1XMSAA_SAMPLE_LOC_CENTROID_MASK                                                 0x00000010L
+#define PA_SC_ENHANCE__DISABLE_SCISSOR_FIX_MASK                                                               0x00000020L
+#define PA_SC_ENHANCE__SEND_UNLIT_STILES_TO_PACKER_MASK                                                       0x00000040L
+#define PA_SC_ENHANCE__DISABLE_DUALGRAD_PERF_OPTIMIZATION_MASK                                                0x00000080L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_PRIM_MASK                                                     0x00000100L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_SUPERTILE_MASK                                                0x00000200L
+#define PA_SC_ENHANCE__DISABLE_SC_PROCESS_RESET_TILE_MASK                                                     0x00000400L
+#define PA_SC_ENHANCE__DISABLE_PA_SC_GUIDANCE_MASK                                                            0x00000800L
+#define PA_SC_ENHANCE__DISABLE_EOV_ALL_CTRL_ONLY_COMBINATIONS_MASK                                            0x00001000L
+#define PA_SC_ENHANCE__ENABLE_MULTICYCLE_BUBBLE_FREEZE_MASK                                                   0x00002000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_PA_SC_GUIDANCE_MASK                                               0x00004000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_POLY_MODE_MASK                                                     0x00008000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EOP_SYNC_NULL_PRIMS_LAST_MASK                                     0x00010000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_THRESHOLD_SWITCHING_MASK                                          0x00020000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_THRESHOLD_SWITCH_AT_EOPG_ONLY_MASK                                 0x00040000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_DESIRED_FIFO_EMPTY_SWITCHING_MASK                                 0x00080000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_SELECTED_FIFO_EMPTY_SWITCHING_MASK                                0x00100000L
+#define PA_SC_ENHANCE__DISABLE_OUT_OF_ORDER_EMPTY_SWITCHING_HYSTERYSIS_MASK                                   0x00200000L
+#define PA_SC_ENHANCE__ENABLE_OUT_OF_ORDER_DESIRED_FIFO_IS_NEXT_FEID_MASK                                     0x00400000L
+#define PA_SC_ENHANCE__DISABLE_OOO_NO_EOPG_SKEW_DESIRED_FIFO_IS_CURRENT_FIFO_MASK                             0x00800000L
+#define PA_SC_ENHANCE__OOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT_MASK                                            0x01000000L
+#define PA_SC_ENHANCE__OOO_DISABLE_EOPG_SKEW_THRESHOLD_SWITCHING_MASK                                         0x02000000L
+#define PA_SC_ENHANCE__DISABLE_EOP_LINE_STIPPLE_RESET_MASK                                                    0x04000000L
+#define PA_SC_ENHANCE__DISABLE_VPZ_EOP_LINE_STIPPLE_RESET_MASK                                                0x08000000L
+#define PA_SC_ENHANCE__IOO_DISABLE_SCAN_UNSELECTED_FIFOS_FOR_DUAL_GFX_RING_CHANGE_MASK                        0x10000000L
+#define PA_SC_ENHANCE__OOO_USE_ABSOLUTE_FIFO_COUNT_IN_THRESHOLD_SWITCHING_MASK                                0x20000000L
+//PA_SC_ENHANCE_1
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_ENABLE__SHIFT                                                0x0
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE__SHIFT                                                       0x1
+#define PA_SC_ENHANCE_1__DISABLE_SC_BINNING__SHIFT                                                            0x3
+#define PA_SC_ENHANCE_1__BYPASS_PBB__SHIFT                                                                    0x4
+#define PA_SC_ENHANCE_1__DISABLE_NONBINNED_LIVE_PRIM_DG1_LS0_CL0_EOPKT_POKE__SHIFT                            0x5
+#define PA_SC_ENHANCE_1__ECO_SPARE1__SHIFT                                                                    0x6
+#define PA_SC_ENHANCE_1__ECO_SPARE2__SHIFT                                                                    0x7
+#define PA_SC_ENHANCE_1__ECO_SPARE3__SHIFT                                                                    0x8
+#define PA_SC_ENHANCE_1__DISABLE_SC_PROCESS_RESET_PBB__SHIFT                                                  0x9
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_OPT__SHIFT                                                       0xa
+#define PA_SC_ENHANCE_1__ENABLE_DFSM_FLUSH_EVENT_TO_FLUSH_POPS_CAM__SHIFT                                     0xb
+#define PA_SC_ENHANCE_1__DEBUG_PIXEL_PICKER_COUNT_PIXELS__SHIFT                                               0xd
+#define PA_SC_ENHANCE_1__DISABLE_SC_DB_TILE_INTF_FINE_CLOCK_GATE__SHIFT                                       0xe
+#define PA_SC_ENHANCE_1__DISABLE_PACKER_ODC_ENHANCE__SHIFT                                                    0x10
+#define PA_SC_ENHANCE_1__OPTIMAL_BIN_SELECTION__SHIFT                                                         0x12
+#define PA_SC_ENHANCE_1__DISABLE_FORCE_SOP_ALL_EVENTS__SHIFT                                                  0x13
+#define PA_SC_ENHANCE_1__DISABLE_PBB_CLK_OPTIMIZATION__SHIFT                                                  0x14
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_CLK_OPTIMIZATION__SHIFT                                          0x15
+#define PA_SC_ENHANCE_1__DISABLE_PBB_BINNING_CLK_OPTIMIZATION__SHIFT                                          0x16
+#define PA_SC_ENHANCE_1__DISABLE_INTF_CG__SHIFT                                                               0x17
+#define PA_SC_ENHANCE_1__IOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT__SHIFT                                        0x18
+#define PA_SC_ENHANCE_1__DISABLE_SHADER_PROFILING_FOR_POWER__SHIFT                                            0x19
+#define PA_SC_ENHANCE_1__FLUSH_ON_BINNING_TRANSITION__SHIFT                                                   0x1a
+#define PA_SC_ENHANCE_1__DISABLE_QUAD_PROC_FDCE_ENHANCE__SHIFT                                                0x1b
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX__SHIFT                                                  0x1c
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX_1__SHIFT                                                0x1d
+#define PA_SC_ENHANCE_1__PASS_VPZ_EVENT_TO_SPI__SHIFT                                                         0x1e
+#define PA_SC_ENHANCE_1__DISABLE_FSR_NEAR_AXIS_LINE_VERT_ORDER_SORT_FIX__SHIFT                                0x1f
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_ENABLE_MASK                                                  0x00000001L
+#define PA_SC_ENHANCE_1__REALIGN_DQUADS_OVERRIDE_MASK                                                         0x00000006L
+#define PA_SC_ENHANCE_1__DISABLE_SC_BINNING_MASK                                                              0x00000008L
+#define PA_SC_ENHANCE_1__BYPASS_PBB_MASK                                                                      0x00000010L
+#define PA_SC_ENHANCE_1__DISABLE_NONBINNED_LIVE_PRIM_DG1_LS0_CL0_EOPKT_POKE_MASK                              0x00000020L
+#define PA_SC_ENHANCE_1__ECO_SPARE1_MASK                                                                      0x00000040L
+#define PA_SC_ENHANCE_1__ECO_SPARE2_MASK                                                                      0x00000080L
+#define PA_SC_ENHANCE_1__ECO_SPARE3_MASK                                                                      0x00000100L
+#define PA_SC_ENHANCE_1__DISABLE_SC_PROCESS_RESET_PBB_MASK                                                    0x00000200L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_OPT_MASK                                                         0x00000400L
+#define PA_SC_ENHANCE_1__ENABLE_DFSM_FLUSH_EVENT_TO_FLUSH_POPS_CAM_MASK                                       0x00000800L
+#define PA_SC_ENHANCE_1__DEBUG_PIXEL_PICKER_COUNT_PIXELS_MASK                                                 0x00002000L
+#define PA_SC_ENHANCE_1__DISABLE_SC_DB_TILE_INTF_FINE_CLOCK_GATE_MASK                                         0x00004000L
+#define PA_SC_ENHANCE_1__DISABLE_PACKER_ODC_ENHANCE_MASK                                                      0x00010000L
+#define PA_SC_ENHANCE_1__OPTIMAL_BIN_SELECTION_MASK                                                           0x00040000L
+#define PA_SC_ENHANCE_1__DISABLE_FORCE_SOP_ALL_EVENTS_MASK                                                    0x00080000L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_CLK_OPTIMIZATION_MASK                                                    0x00100000L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_SCISSOR_CLK_OPTIMIZATION_MASK                                            0x00200000L
+#define PA_SC_ENHANCE_1__DISABLE_PBB_BINNING_CLK_OPTIMIZATION_MASK                                            0x00400000L
+#define PA_SC_ENHANCE_1__DISABLE_INTF_CG_MASK                                                                 0x00800000L
+#define PA_SC_ENHANCE_1__IOO_DISABLE_EOP_ON_FIRST_LIVE_PRIM_HIT_MASK                                          0x01000000L
+#define PA_SC_ENHANCE_1__DISABLE_SHADER_PROFILING_FOR_POWER_MASK                                              0x02000000L
+#define PA_SC_ENHANCE_1__FLUSH_ON_BINNING_TRANSITION_MASK                                                     0x04000000L
+#define PA_SC_ENHANCE_1__DISABLE_QUAD_PROC_FDCE_ENHANCE_MASK                                                  0x08000000L
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX_MASK                                                    0x10000000L
+#define PA_SC_ENHANCE_1__DISABLE_SC_PS_PA_ARBITER_FIX_1_MASK                                                  0x20000000L
+#define PA_SC_ENHANCE_1__PASS_VPZ_EVENT_TO_SPI_MASK                                                           0x40000000L
+#define PA_SC_ENHANCE_1__DISABLE_FSR_NEAR_AXIS_LINE_VERT_ORDER_SORT_FIX_MASK                                  0x80000000L
+//PA_SC_ENHANCE_2
+#define PA_SC_ENHANCE_2__DISABLE_SC_MEM_MACRO_FINE_CLOCK_GATE__SHIFT                                          0x0
+#define PA_SC_ENHANCE_2__DISABLE_SC_DB_QUAD_INTF_FINE_CLOCK_GATE__SHIFT                                       0x1
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_QUAD_INTF_FINE_CLOCK_GATE__SHIFT                                      0x2
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_PRIM_INTF_FINE_CLOCK_GATE__SHIFT                                      0x3
+#define PA_SC_ENHANCE_2__ENABLE_LPOV_WAVE_BREAK__SHIFT                                                        0x4
+#define PA_SC_ENHANCE_2__ENABLE_FPOV_WAVE_BREAK__SHIFT                                                        0x5
+#define PA_SC_ENHANCE_2__ENABLE_SC_SEND_DB_VPZ_FOR_EN_PRIM_PAYLOAD__SHIFT                                     0x7
+#define PA_SC_ENHANCE_2__DISABLE_BREAK_BATCH_ON_GFX_PIPE_SWITCH__SHIFT                                        0x8
+#define PA_SC_ENHANCE_2__DISABLE_FULL_TILE_WAVE_BREAK__SHIFT                                                  0x9
+#define PA_SC_ENHANCE_2__ENABLE_VPZ_INJECTION_BEFORE_NULL_PRIMS__SHIFT                                        0xa
+#define PA_SC_ENHANCE_2__PBB_TIMEOUT_THRESHOLD_MODE__SHIFT                                                    0xb
+#define PA_SC_ENHANCE_2__DISABLE_PACKER_GRAD_FDCE_ENHANCE__SHIFT                                              0xc
+#define PA_SC_ENHANCE_2__DISABLE_SC_SPI_INTF_EARLY_WAKEUP__SHIFT                                              0xd
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_INTF_EARLY_WAKEUP__SHIFT                                              0xe
+#define PA_SC_ENHANCE_2__DISABLE_EXPOSED_GT_DETAIL_RATE_TILE_COV_ADJ__SHIFT                                   0xf
+#define PA_SC_ENHANCE_2__PBB_WARP_CLK_MAIN_CLK_WAKEUP__SHIFT                                                  0x10
+#define PA_SC_ENHANCE_2__PBB_MAIN_CLK_REG_BUSY_WAKEUP__SHIFT                                                  0x11
+#define PA_SC_ENHANCE_2__DISABLE_BREAK_BATCH_ON_GFX_PIPELINE_RESET__SHIFT                                     0x12
+#define PA_SC_ENHANCE_2__DISABLE_SC_DBR_DATAPATH_FGCG__SHIFT                                                  0x15
+#define PA_SC_ENHANCE_2__FSR_BB_OPTIMIZATION_DISABLE_OVERRIDE__SHIFT                                          0x16
+#define PA_SC_ENHANCE_2__PROCESS_RESET_FORCE_STILE_MASK_TO_ZERO__SHIFT                                        0x17
+#define PA_SC_ENHANCE_2__BREAK_WHEN_ONE_NULL_PRIM_BATCH__SHIFT                                                0x1a
+#define PA_SC_ENHANCE_2__NULL_PRIM_BREAK_BATCH_LIMIT__SHIFT                                                   0x1b
+#define PA_SC_ENHANCE_2__DISABLE_MAX_DEALLOC_FORCE_EOV_RESET_N_WAVES_COUNT__SHIFT                             0x1e
+#define PA_SC_ENHANCE_2__RSVD__SHIFT                                                                          0x1f
+#define PA_SC_ENHANCE_2__DISABLE_SC_MEM_MACRO_FINE_CLOCK_GATE_MASK                                            0x00000001L
+#define PA_SC_ENHANCE_2__DISABLE_SC_DB_QUAD_INTF_FINE_CLOCK_GATE_MASK                                         0x00000002L
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_QUAD_INTF_FINE_CLOCK_GATE_MASK                                        0x00000004L
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_PRIM_INTF_FINE_CLOCK_GATE_MASK                                        0x00000008L
+#define PA_SC_ENHANCE_2__ENABLE_LPOV_WAVE_BREAK_MASK                                                          0x00000010L
+#define PA_SC_ENHANCE_2__ENABLE_FPOV_WAVE_BREAK_MASK                                                          0x00000020L
+#define PA_SC_ENHANCE_2__ENABLE_SC_SEND_DB_VPZ_FOR_EN_PRIM_PAYLOAD_MASK                                       0x00000080L
+#define PA_SC_ENHANCE_2__DISABLE_BREAK_BATCH_ON_GFX_PIPE_SWITCH_MASK                                          0x00000100L
+#define PA_SC_ENHANCE_2__DISABLE_FULL_TILE_WAVE_BREAK_MASK                                                    0x00000200L
+#define PA_SC_ENHANCE_2__ENABLE_VPZ_INJECTION_BEFORE_NULL_PRIMS_MASK                                          0x00000400L
+#define PA_SC_ENHANCE_2__PBB_TIMEOUT_THRESHOLD_MODE_MASK                                                      0x00000800L
+#define PA_SC_ENHANCE_2__DISABLE_PACKER_GRAD_FDCE_ENHANCE_MASK                                                0x00001000L
+#define PA_SC_ENHANCE_2__DISABLE_SC_SPI_INTF_EARLY_WAKEUP_MASK                                                0x00002000L
+#define PA_SC_ENHANCE_2__DISABLE_SC_BCI_INTF_EARLY_WAKEUP_MASK                                                0x00004000L
+#define PA_SC_ENHANCE_2__DISABLE_EXPOSED_GT_DETAIL_RATE_TILE_COV_ADJ_MASK                                     0x00008000L
+#define PA_SC_ENHANCE_2__PBB_WARP_CLK_MAIN_CLK_WAKEUP_MASK                                                    0x00010000L
+#define PA_SC_ENHANCE_2__PBB_MAIN_CLK_REG_BUSY_WAKEUP_MASK                                                    0x00020000L
+#define PA_SC_ENHANCE_2__DISABLE_BREAK_BATCH_ON_GFX_PIPELINE_RESET_MASK                                       0x00040000L
+#define PA_SC_ENHANCE_2__DISABLE_SC_DBR_DATAPATH_FGCG_MASK                                                    0x00200000L
+#define PA_SC_ENHANCE_2__FSR_BB_OPTIMIZATION_DISABLE_OVERRIDE_MASK                                            0x00400000L
+#define PA_SC_ENHANCE_2__PROCESS_RESET_FORCE_STILE_MASK_TO_ZERO_MASK                                          0x00800000L
+#define PA_SC_ENHANCE_2__BREAK_WHEN_ONE_NULL_PRIM_BATCH_MASK                                                  0x04000000L
+#define PA_SC_ENHANCE_2__NULL_PRIM_BREAK_BATCH_LIMIT_MASK                                                     0x38000000L
+#define PA_SC_ENHANCE_2__DISABLE_MAX_DEALLOC_FORCE_EOV_RESET_N_WAVES_COUNT_MASK                               0x40000000L
+#define PA_SC_ENHANCE_2__RSVD_MASK                                                                            0x80000000L
+//PA_SC_ENHANCE_3
+#define PA_SC_ENHANCE_3__FORCE_USE_OF_SC_CENTROID_DATA__SHIFT                                                 0x0
+#define PA_SC_ENHANCE_3__DISABLE_RB_MASK_COPY_FOR_NONP2_SA_PAIR_HARVEST__SHIFT                                0x2
+#define PA_SC_ENHANCE_3__FORCE_PBB_WORKLOAD_MODE_TO_ZERO__SHIFT                                               0x3
+#define PA_SC_ENHANCE_3__DISABLE_PKR_BCI_QUAD_NEW_PRIM_DATA_LOAD_OPTIMIZATION__SHIFT                          0x4
+#define PA_SC_ENHANCE_3__DISABLE_CP_CONTEXT_DONE_PERFCOUNT_SAMPLE_EN__SHIFT                                   0x5
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_FIRST_PHASE_FILTER__SHIFT                                     0x6
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_LAST_PHASE_FILTER__SHIFT                                      0x7
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_LAST_PHASE_FILTER_FOR_PBB_BINNED_PRIMS__SHIFT                 0x8
+#define PA_SC_ENHANCE_3__DISABLE_SET_VPZ_DIRTY_EOPKT_LAST_PHASE_ONLY__SHIFT                                   0x9
+#define PA_SC_ENHANCE_3__DISABLE_PBB_EOP_OPTIMIZATION_WITH_SAME_CONTEXT_BATCHES__SHIFT                        0xa
+#define PA_SC_ENHANCE_3__DISABLE_FAST_NULL_PRIM_OPTIMIZATION__SHIFT                                           0xb
+#define PA_SC_ENHANCE_3__USE_PBB_PRIM_STORAGE_WHEN_STALLED__SHIFT                                             0xc
+#define PA_SC_ENHANCE_3__DISABLE_LIGHT_VOLUME_RENDERING_OPTIMIZATION__SHIFT                                   0xd
+#define PA_SC_ENHANCE_3__DISABLE_ZPRE_PASS_OPTIMIZATION__SHIFT                                                0xe
+#define PA_SC_ENHANCE_3__DISABLE_EVENT_INCLUSION_IN_CONTEXT_STATES_PER_BIN__SHIFT                             0xf
+#define PA_SC_ENHANCE_3__DISABLE_PIXEL_WAIT_SYNC_COUNTERS__SHIFT                                              0x10
+#define PA_SC_ENHANCE_3__DISABLE_SC_CPG_PSINVOC_SEDC_ISOLATION_ACCUM__SHIFT                                   0x11
+#define PA_SC_ENHANCE_3__DISABLE_SC_QP_VRS_RATE_FB_FINE_CLOCK_GATE__SHIFT                                     0x12
+#define PA_SC_ENHANCE_3__DISABLE_SC_QP_VRS_RATE_CACHE_RD_FINE_CLOCK_GATE__SHIFT                               0x13
+#define PA_SC_ENHANCE_3__DISABLE_PKR_FORCE_EOV_MAX_REZ_CNT_FOR_SPI_BACKPRESSURE_ONLY__SHIFT                   0x14
+#define PA_SC_ENHANCE_3__DISABLE_PKR_FORCE_EOV_MAX_CLK_CNT_FOR_SPI_BACKPRESSURE_ONLY__SHIFT                   0x15
+#define PA_SC_ENHANCE_3__DO_NOT_INCLUDE_OREO_WAVEID_IN_FORCE_EOV_MAX_CNT_DISABLE__SHIFT                       0x16
+#define PA_SC_ENHANCE_3__DISABLE_PWS_PRE_DEPTH_WAIT_SYNC_VPZ_INSERTION__SHIFT                                 0x17
+#define PA_SC_ENHANCE_3__PKR_CNT_FORCE_EOV_AT_QS_EMPTY_ONLY__SHIFT                                            0x18
+#define PA_SC_ENHANCE_3__PKR_S0_FORCE_EOV_STALL__SHIFT                                                        0x19
+#define PA_SC_ENHANCE_3__PKR_S1_FORCE_EOV_STALL__SHIFT                                                        0x1a
+#define PA_SC_ENHANCE_3__PKR_S2_FORCE_EOV_STALL__SHIFT                                                        0x1b
+#define PA_SC_ENHANCE_3__ECO_SPARE0__SHIFT                                                                    0x1c
+#define PA_SC_ENHANCE_3__ECO_SPARE1__SHIFT                                                                    0x1d
+#define PA_SC_ENHANCE_3__ECO_SPARE2__SHIFT                                                                    0x1e
+#define PA_SC_ENHANCE_3__ECO_SPARE3__SHIFT                                                                    0x1f
+#define PA_SC_ENHANCE_3__FORCE_USE_OF_SC_CENTROID_DATA_MASK                                                   0x00000001L
+#define PA_SC_ENHANCE_3__DISABLE_RB_MASK_COPY_FOR_NONP2_SA_PAIR_HARVEST_MASK                                  0x00000004L
+#define PA_SC_ENHANCE_3__FORCE_PBB_WORKLOAD_MODE_TO_ZERO_MASK                                                 0x00000008L
+#define PA_SC_ENHANCE_3__DISABLE_PKR_BCI_QUAD_NEW_PRIM_DATA_LOAD_OPTIMIZATION_MASK                            0x00000010L
+#define PA_SC_ENHANCE_3__DISABLE_CP_CONTEXT_DONE_PERFCOUNT_SAMPLE_EN_MASK                                     0x00000020L
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_FIRST_PHASE_FILTER_MASK                                       0x00000040L
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_LAST_PHASE_FILTER_MASK                                        0x00000080L
+#define PA_SC_ENHANCE_3__ENABLE_SINGLE_PA_EOPKT_LAST_PHASE_FILTER_FOR_PBB_BINNED_PRIMS_MASK                   0x00000100L
+#define PA_SC_ENHANCE_3__DISABLE_SET_VPZ_DIRTY_EOPKT_LAST_PHASE_ONLY_MASK                                     0x00000200L
+#define PA_SC_ENHANCE_3__DISABLE_PBB_EOP_OPTIMIZATION_WITH_SAME_CONTEXT_BATCHES_MASK                          0x00000400L
+#define PA_SC_ENHANCE_3__DISABLE_FAST_NULL_PRIM_OPTIMIZATION_MASK                                             0x00000800L
+#define PA_SC_ENHANCE_3__USE_PBB_PRIM_STORAGE_WHEN_STALLED_MASK                                               0x00001000L
+#define PA_SC_ENHANCE_3__DISABLE_LIGHT_VOLUME_RENDERING_OPTIMIZATION_MASK                                     0x00002000L
+#define PA_SC_ENHANCE_3__DISABLE_ZPRE_PASS_OPTIMIZATION_MASK                                                  0x00004000L
+#define PA_SC_ENHANCE_3__DISABLE_EVENT_INCLUSION_IN_CONTEXT_STATES_PER_BIN_MASK                               0x00008000L
+#define PA_SC_ENHANCE_3__DISABLE_PIXEL_WAIT_SYNC_COUNTERS_MASK                                                0x00010000L
+#define PA_SC_ENHANCE_3__DISABLE_SC_CPG_PSINVOC_SEDC_ISOLATION_ACCUM_MASK                                     0x00020000L
+#define PA_SC_ENHANCE_3__DISABLE_SC_QP_VRS_RATE_FB_FINE_CLOCK_GATE_MASK                                       0x00040000L
+#define PA_SC_ENHANCE_3__DISABLE_SC_QP_VRS_RATE_CACHE_RD_FINE_CLOCK_GATE_MASK                                 0x00080000L
+#define PA_SC_ENHANCE_3__DISABLE_PKR_FORCE_EOV_MAX_REZ_CNT_FOR_SPI_BACKPRESSURE_ONLY_MASK                     0x00100000L
+#define PA_SC_ENHANCE_3__DISABLE_PKR_FORCE_EOV_MAX_CLK_CNT_FOR_SPI_BACKPRESSURE_ONLY_MASK                     0x00200000L
+#define PA_SC_ENHANCE_3__DO_NOT_INCLUDE_OREO_WAVEID_IN_FORCE_EOV_MAX_CNT_DISABLE_MASK                         0x00400000L
+#define PA_SC_ENHANCE_3__DISABLE_PWS_PRE_DEPTH_WAIT_SYNC_VPZ_INSERTION_MASK                                   0x00800000L
+#define PA_SC_ENHANCE_3__PKR_CNT_FORCE_EOV_AT_QS_EMPTY_ONLY_MASK                                              0x01000000L
+#define PA_SC_ENHANCE_3__PKR_S0_FORCE_EOV_STALL_MASK                                                          0x02000000L
+#define PA_SC_ENHANCE_3__PKR_S1_FORCE_EOV_STALL_MASK                                                          0x04000000L
+#define PA_SC_ENHANCE_3__PKR_S2_FORCE_EOV_STALL_MASK                                                          0x08000000L
+#define PA_SC_ENHANCE_3__ECO_SPARE0_MASK                                                                      0x10000000L
+#define PA_SC_ENHANCE_3__ECO_SPARE1_MASK                                                                      0x20000000L
+#define PA_SC_ENHANCE_3__ECO_SPARE2_MASK                                                                      0x40000000L
+#define PA_SC_ENHANCE_3__ECO_SPARE3_MASK                                                                      0x80000000L
+//PA_SC_BINNER_CNTL_OVERRIDE
+#define PA_SC_BINNER_CNTL_OVERRIDE__BINNING_MODE__SHIFT                                                       0x0
+#define PA_SC_BINNER_CNTL_OVERRIDE__CONTEXT_STATES_PER_BIN__SHIFT                                             0xa
+#define PA_SC_BINNER_CNTL_OVERRIDE__PERSISTENT_STATES_PER_BIN__SHIFT                                          0xd
+#define PA_SC_BINNER_CNTL_OVERRIDE__FPOVS_PER_BATCH__SHIFT                                                    0x13
+#define PA_SC_BINNER_CNTL_OVERRIDE__DIRECT_OVERRIDE_MODE__SHIFT                                               0x1b
+#define PA_SC_BINNER_CNTL_OVERRIDE__OVERRIDE__SHIFT                                                           0x1c
+#define PA_SC_BINNER_CNTL_OVERRIDE__BINNING_MODE_MASK                                                         0x00000003L
+#define PA_SC_BINNER_CNTL_OVERRIDE__CONTEXT_STATES_PER_BIN_MASK                                               0x00001C00L
+#define PA_SC_BINNER_CNTL_OVERRIDE__PERSISTENT_STATES_PER_BIN_MASK                                            0x0003E000L
+#define PA_SC_BINNER_CNTL_OVERRIDE__FPOVS_PER_BATCH_MASK                                                      0x07F80000L
+#define PA_SC_BINNER_CNTL_OVERRIDE__DIRECT_OVERRIDE_MODE_MASK                                                 0x08000000L
+#define PA_SC_BINNER_CNTL_OVERRIDE__OVERRIDE_MASK                                                             0xF0000000L
+//PA_SC_PBB_OVERRIDE_FLAG
+#define PA_SC_PBB_OVERRIDE_FLAG__OVERRIDE__SHIFT                                                              0x0
+#define PA_SC_PBB_OVERRIDE_FLAG__PIPE_ID__SHIFT                                                               0x1
+#define PA_SC_PBB_OVERRIDE_FLAG__OVERRIDE_MASK                                                                0x00000001L
+#define PA_SC_PBB_OVERRIDE_FLAG__PIPE_ID_MASK                                                                 0x00000002L
+//PA_SC_DSM_CNTL
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_0__SHIFT                                                                0x0
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_1__SHIFT                                                                0x1
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_0_MASK                                                                  0x00000001L
+#define PA_SC_DSM_CNTL__FORCE_EOV_REZ_1_MASK                                                                  0x00000002L
+//PA_SC_TILE_STEERING_CREST_OVERRIDE
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__ONE_RB_MODE_ENABLE__SHIFT                                         0x0
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SE_SELECT__SHIFT                                                  0x1
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__RB_SELECT__SHIFT                                                  0x5
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SA_SELECT__SHIFT                                                  0x8
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__FORCE_TILE_STEERING_OVERRIDE_USE__SHIFT                           0x1f
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__ONE_RB_MODE_ENABLE_MASK                                           0x00000001L
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SE_SELECT_MASK                                                    0x00000006L
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__RB_SELECT_MASK                                                    0x00000060L
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__SA_SELECT_MASK                                                    0x00000700L
+#define PA_SC_TILE_STEERING_CREST_OVERRIDE__FORCE_TILE_STEERING_OVERRIDE_USE_MASK                             0x80000000L
+//PA_SC_FIFO_SIZE
+#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE__SHIFT                                                    0x0
+#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE__SHIFT                                                     0x6
+#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE__SHIFT                                                         0xf
+#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE__SHIFT                                                      0x15
+#define PA_SC_FIFO_SIZE__SC_FRONTEND_PRIM_FIFO_SIZE_MASK                                                      0x0000003FL
+#define PA_SC_FIFO_SIZE__SC_BACKEND_PRIM_FIFO_SIZE_MASK                                                       0x00007FC0L
+#define PA_SC_FIFO_SIZE__SC_HIZ_TILE_FIFO_SIZE_MASK                                                           0x001F8000L
+#define PA_SC_FIFO_SIZE__SC_EARLYZ_TILE_FIFO_SIZE_MASK                                                        0xFFE00000L
+//PA_SC_IF_FIFO_SIZE
+#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE__SHIFT                                                    0x0
+#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE__SHIFT                                                    0x6
+#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE__SHIFT                                                        0xc
+#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE__SHIFT                                                        0x12
+#define PA_SC_IF_FIFO_SIZE__SC_DB_TILE_IF_FIFO_SIZE_MASK                                                      0x0000003FL
+#define PA_SC_IF_FIFO_SIZE__SC_DB_QUAD_IF_FIFO_SIZE_MASK                                                      0x00000FC0L
+#define PA_SC_IF_FIFO_SIZE__SC_SPI_IF_FIFO_SIZE_MASK                                                          0x0003F000L
+#define PA_SC_IF_FIFO_SIZE__SC_BCI_IF_FIFO_SIZE_MASK                                                          0x00FC0000L
+//PA_SC_PACKER_WAVE_ID_CNTL
+#define PA_SC_PACKER_WAVE_ID_CNTL__WAVE_TABLE_SIZE__SHIFT                                                     0x0
+#define PA_SC_PACKER_WAVE_ID_CNTL__SC_DB_WAVE_IF_FIFO_SIZE__SHIFT                                             0xa
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_SC_DB_WAVE_IF_FGCG_EN__SHIFT                                       0x10
+#define PA_SC_PACKER_WAVE_ID_CNTL__SC_SPI_WAVE_IF_FIFO_SIZE__SHIFT                                            0x11
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_SC_SPI_WAVE_IF_FGCG_EN__SHIFT                                      0x17
+#define PA_SC_PACKER_WAVE_ID_CNTL__DEBUG_CONFLICT_QUAD__SHIFT                                                 0x18
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_OREO_CONFLICT_QUAD__SHIFT                                          0x1f
+#define PA_SC_PACKER_WAVE_ID_CNTL__WAVE_TABLE_SIZE_MASK                                                       0x000003FFL
+#define PA_SC_PACKER_WAVE_ID_CNTL__SC_DB_WAVE_IF_FIFO_SIZE_MASK                                               0x0000FC00L
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_SC_DB_WAVE_IF_FGCG_EN_MASK                                         0x00010000L
+#define PA_SC_PACKER_WAVE_ID_CNTL__SC_SPI_WAVE_IF_FIFO_SIZE_MASK                                              0x007E0000L
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_SC_SPI_WAVE_IF_FGCG_EN_MASK                                        0x00800000L
+#define PA_SC_PACKER_WAVE_ID_CNTL__DEBUG_CONFLICT_QUAD_MASK                                                   0x0F000000L
+#define PA_SC_PACKER_WAVE_ID_CNTL__DISABLE_OREO_CONFLICT_QUAD_MASK                                            0x80000000L
+//PA_SC_ATM_CNTL
+#define PA_SC_ATM_CNTL__SC_PC_IF_SIZE__SHIFT                                                                  0x0
+#define PA_SC_ATM_CNTL__DISABLE_SC_PC_IF_FGCG_EN__SHIFT                                                       0x7
+#define PA_SC_ATM_CNTL__MAX_ATTRIBUTES_IN_WAVE__SHIFT                                                         0x8
+#define PA_SC_ATM_CNTL__DISABLE_MAX_ATTRIBUTES__SHIFT                                                         0x10
+#define PA_SC_ATM_CNTL__SELECT_MAX_ATTRIBUTES__SHIFT                                                          0x11
+#define PA_SC_ATM_CNTL__SC_PC_IF_SIZE_MASK                                                                    0x0000003FL
+#define PA_SC_ATM_CNTL__DISABLE_SC_PC_IF_FGCG_EN_MASK                                                         0x00000080L
+#define PA_SC_ATM_CNTL__MAX_ATTRIBUTES_IN_WAVE_MASK                                                           0x0000FF00L
+#define PA_SC_ATM_CNTL__DISABLE_MAX_ATTRIBUTES_MASK                                                           0x00010000L
+#define PA_SC_ATM_CNTL__SELECT_MAX_ATTRIBUTES_MASK                                                            0x00020000L
+//PA_SC_PKR_WAVE_TABLE_CNTL
+#define PA_SC_PKR_WAVE_TABLE_CNTL__SIZE__SHIFT                                                                0x0
+#define PA_SC_PKR_WAVE_TABLE_CNTL__SIZE_MASK                                                                  0x0000003FL
+//PA_SC_FORCE_EOV_MAX_CNTS
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT__SHIFT                                                0x0
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT__SHIFT                                                0x10
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_CLK_CNT_MASK                                                  0x0000FFFFL
+#define PA_SC_FORCE_EOV_MAX_CNTS__FORCE_EOV_MAX_REZ_CNT_MASK                                                  0xFFFF0000L
+//PA_SC_BINNER_EVENT_CNTL_0
+#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_0__SHIFT                                                          0x0
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS1__SHIFT                                              0x2
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS2__SHIFT                                              0x4
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS3__SHIFT                                              0x6
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_TS__SHIFT                                                      0x8
+#define PA_SC_BINNER_EVENT_CNTL_0__CONTEXT_DONE__SHIFT                                                        0xa
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH__SHIFT                                                         0xc
+#define PA_SC_BINNER_EVENT_CNTL_0__CS_PARTIAL_FLUSH__SHIFT                                                    0xe
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_SYNC__SHIFT                                                  0x10
+#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_9__SHIFT                                                          0x12
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_RESET__SHIFT                                                 0x14
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_INCR_DE__SHIFT                                                 0x16
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_IB_END__SHIFT                                                  0x18
+#define PA_SC_BINNER_EVENT_CNTL_0__RST_PIX_CNT__SHIFT                                                         0x1a
+#define PA_SC_BINNER_EVENT_CNTL_0__BREAK_BATCH__SHIFT                                                         0x1c
+#define PA_SC_BINNER_EVENT_CNTL_0__VS_PARTIAL_FLUSH__SHIFT                                                    0x1e
+#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_0_MASK                                                            0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS1_MASK                                                0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS2_MASK                                                0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_0__SAMPLE_STREAMOUTSTATS3_MASK                                                0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_TS_MASK                                                        0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_0__CONTEXT_DONE_MASK                                                          0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_0__CACHE_FLUSH_MASK                                                           0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_0__CS_PARTIAL_FLUSH_MASK                                                      0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_SYNC_MASK                                                    0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_0__RESERVED_9_MASK                                                            0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_0__VGT_STREAMOUT_RESET_MASK                                                   0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_INCR_DE_MASK                                                   0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_0__END_OF_PIPE_IB_END_MASK                                                    0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_0__RST_PIX_CNT_MASK                                                           0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_0__BREAK_BATCH_MASK                                                           0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_0__VS_PARTIAL_FLUSH_MASK                                                      0xC0000000L
+//PA_SC_BINNER_EVENT_CNTL_1
+#define PA_SC_BINNER_EVENT_CNTL_1__PS_PARTIAL_FLUSH__SHIFT                                                    0x0
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_HS_OUTPUT__SHIFT                                                     0x2
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_DFSM__SHIFT                                                          0x4
+#define PA_SC_BINNER_EVENT_CNTL_1__RESET_TO_LOWEST_VGT__SHIFT                                                 0x6
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_TS_EVENT__SHIFT                                        0x8
+#define PA_SC_BINNER_EVENT_CNTL_1__WAIT_SYNC__SHIFT                                                           0xa
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_EVENT__SHIFT                                           0xc
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_START__SHIFT                                                   0xe
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_STOP__SHIFT                                                    0x10
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_START__SHIFT                                                  0x12
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_STOP__SHIFT                                                   0x14
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_SAMPLE__SHIFT                                                  0x16
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_ES_OUTPUT__SHIFT                                                     0x18
+#define PA_SC_BINNER_EVENT_CNTL_1__BIN_CONF_OVERRIDE_CHECK__SHIFT                                             0x1a
+#define PA_SC_BINNER_EVENT_CNTL_1__SAMPLE_PIPELINESTAT__SHIFT                                                 0x1c
+#define PA_SC_BINNER_EVENT_CNTL_1__SO_VGTSTREAMOUT_FLUSH__SHIFT                                               0x1e
+#define PA_SC_BINNER_EVENT_CNTL_1__PS_PARTIAL_FLUSH_MASK                                                      0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_HS_OUTPUT_MASK                                                       0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_DFSM_MASK                                                            0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_1__RESET_TO_LOWEST_VGT_MASK                                                   0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_TS_EVENT_MASK                                          0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_1__WAIT_SYNC_MASK                                                             0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_1__CACHE_FLUSH_AND_INV_EVENT_MASK                                             0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_START_MASK                                                     0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_STOP_MASK                                                      0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_START_MASK                                                    0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PIPELINESTAT_STOP_MASK                                                     0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_1__PERFCOUNTER_SAMPLE_MASK                                                    0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_1__FLUSH_ES_OUTPUT_MASK                                                       0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_1__BIN_CONF_OVERRIDE_CHECK_MASK                                               0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_1__SAMPLE_PIPELINESTAT_MASK                                                   0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_1__SO_VGTSTREAMOUT_FLUSH_MASK                                                 0xC0000000L
+//PA_SC_BINNER_EVENT_CNTL_2
+#define PA_SC_BINNER_EVENT_CNTL_2__SAMPLE_STREAMOUTSTATS__SHIFT                                               0x0
+#define PA_SC_BINNER_EVENT_CNTL_2__RESET_VTX_CNT__SHIFT                                                       0x2
+#define PA_SC_BINNER_EVENT_CNTL_2__BLOCK_CONTEXT_DONE__SHIFT                                                  0x4
+#define PA_SC_BINNER_EVENT_CNTL_2__RESERVED_35__SHIFT                                                         0x6
+#define PA_SC_BINNER_EVENT_CNTL_2__VGT_FLUSH__SHIFT                                                           0x8
+#define PA_SC_BINNER_EVENT_CNTL_2__TGID_ROLLOVER__SHIFT                                                       0xa
+#define PA_SC_BINNER_EVENT_CNTL_2__SQ_NON_EVENT__SHIFT                                                        0xc
+#define PA_SC_BINNER_EVENT_CNTL_2__SC_SEND_DB_VPZ__SHIFT                                                      0xe
+#define PA_SC_BINNER_EVENT_CNTL_2__BOTTOM_OF_PIPE_TS__SHIFT                                                   0x10
+#define PA_SC_BINNER_EVENT_CNTL_2__RESERVED_41__SHIFT                                                         0x12
+#define PA_SC_BINNER_EVENT_CNTL_2__DB_CACHE_FLUSH_AND_INV__SHIFT                                              0x14
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_DATA_TS__SHIFT                                            0x16
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_META__SHIFT                                               0x18
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_DATA_TS__SHIFT                                            0x1a
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_META__SHIFT                                               0x1c
+#define PA_SC_BINNER_EVENT_CNTL_2__CS_DONE__SHIFT                                                             0x1e
+#define PA_SC_BINNER_EVENT_CNTL_2__SAMPLE_STREAMOUTSTATS_MASK                                                 0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_2__RESET_VTX_CNT_MASK                                                         0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_2__BLOCK_CONTEXT_DONE_MASK                                                    0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_2__RESERVED_35_MASK                                                           0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_2__VGT_FLUSH_MASK                                                             0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_2__TGID_ROLLOVER_MASK                                                         0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_2__SQ_NON_EVENT_MASK                                                          0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_2__SC_SEND_DB_VPZ_MASK                                                        0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_2__BOTTOM_OF_PIPE_TS_MASK                                                     0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_2__RESERVED_41_MASK                                                           0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_2__DB_CACHE_FLUSH_AND_INV_MASK                                                0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_DATA_TS_MASK                                              0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_DB_META_MASK                                                 0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_DATA_TS_MASK                                              0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_2__FLUSH_AND_INV_CB_META_MASK                                                 0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_2__CS_DONE_MASK                                                               0xC0000000L
+//PA_SC_BINNER_EVENT_CNTL_3
+#define PA_SC_BINNER_EVENT_CNTL_3__PS_DONE__SHIFT                                                             0x0
+#define PA_SC_BINNER_EVENT_CNTL_3__FLUSH_AND_INV_CB_PIXEL_DATA__SHIFT                                         0x2
+#define PA_SC_BINNER_EVENT_CNTL_3__RESERVED_50__SHIFT                                                         0x4
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_START__SHIFT                                                  0x6
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_STOP__SHIFT                                                   0x8
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_MARKER__SHIFT                                                 0xa
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_DRAW__SHIFT                                                   0xc
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FINISH__SHIFT                                                 0xe
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_CONTROL__SHIFT                                             0x10
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_DUMP__SHIFT                                                0x12
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_RESET__SHIFT                                               0x14
+#define PA_SC_BINNER_EVENT_CNTL_3__CONTEXT_SUSPEND__SHIFT                                                     0x16
+#define PA_SC_BINNER_EVENT_CNTL_3__OFFCHIP_HS_DEALLOC__SHIFT                                                  0x18
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_NGG_PIPELINE__SHIFT                                                 0x1a
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_PIPELINE_NOT_USED__SHIFT                                            0x1c
+#define PA_SC_BINNER_EVENT_CNTL_3__DRAW_DONE__SHIFT                                                           0x1e
+#define PA_SC_BINNER_EVENT_CNTL_3__PS_DONE_MASK                                                               0x00000003L
+#define PA_SC_BINNER_EVENT_CNTL_3__FLUSH_AND_INV_CB_PIXEL_DATA_MASK                                           0x0000000CL
+#define PA_SC_BINNER_EVENT_CNTL_3__RESERVED_50_MASK                                                           0x00000030L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_START_MASK                                                    0x000000C0L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_STOP_MASK                                                     0x00000300L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_MARKER_MASK                                                   0x00000C00L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_DRAW_MASK                                                     0x00003000L
+#define PA_SC_BINNER_EVENT_CNTL_3__THREAD_TRACE_FINISH_MASK                                                   0x0000C000L
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_CONTROL_MASK                                               0x00030000L
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_DUMP_MASK                                                  0x000C0000L
+#define PA_SC_BINNER_EVENT_CNTL_3__PIXEL_PIPE_STAT_RESET_MASK                                                 0x00300000L
+#define PA_SC_BINNER_EVENT_CNTL_3__CONTEXT_SUSPEND_MASK                                                       0x00C00000L
+#define PA_SC_BINNER_EVENT_CNTL_3__OFFCHIP_HS_DEALLOC_MASK                                                    0x03000000L
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_NGG_PIPELINE_MASK                                                   0x0C000000L
+#define PA_SC_BINNER_EVENT_CNTL_3__ENABLE_PIPELINE_NOT_USED_MASK                                              0x30000000L
+#define PA_SC_BINNER_EVENT_CNTL_3__DRAW_DONE_MASK                                                             0xC0000000L
+//PA_SC_BINNER_TIMEOUT_COUNTER
+#define PA_SC_BINNER_TIMEOUT_COUNTER__THRESHOLD__SHIFT                                                        0x0
+#define PA_SC_BINNER_TIMEOUT_COUNTER__THRESHOLD_MASK                                                          0xFFFFFFFFL
+//PA_SC_BINNER_PERF_CNTL_0
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_PRIMS_THRESHOLD__SHIFT                                         0x0
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_PRIMS_THRESHOLD__SHIFT                                       0xa
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_CONTEXT_THRESHOLD__SHIFT                                       0x14
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_CONTEXT_THRESHOLD__SHIFT                                     0x17
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_PRIMS_THRESHOLD_MASK                                           0x000003FFL
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_PRIMS_THRESHOLD_MASK                                         0x000FFC00L
+#define PA_SC_BINNER_PERF_CNTL_0__BIN_HIST_NUM_CONTEXT_THRESHOLD_MASK                                         0x00700000L
+#define PA_SC_BINNER_PERF_CNTL_0__BATCH_HIST_NUM_CONTEXT_THRESHOLD_MASK                                       0x03800000L
+//PA_SC_BINNER_PERF_CNTL_1
+#define PA_SC_BINNER_PERF_CNTL_1__BIN_HIST_NUM_PERSISTENT_STATE_THRESHOLD__SHIFT                              0x0
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_PERSISTENT_STATE_THRESHOLD__SHIFT                            0x5
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_TRIV_REJECTED_PRIMS_THRESHOLD__SHIFT                         0xa
+#define PA_SC_BINNER_PERF_CNTL_1__BIN_HIST_NUM_PERSISTENT_STATE_THRESHOLD_MASK                                0x0000001FL
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_PERSISTENT_STATE_THRESHOLD_MASK                              0x000003E0L
+#define PA_SC_BINNER_PERF_CNTL_1__BATCH_HIST_NUM_TRIV_REJECTED_PRIMS_THRESHOLD_MASK                           0x03FFFC00L
+//PA_SC_BINNER_PERF_CNTL_2
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_ROWS_PER_PRIM_THRESHOLD__SHIFT                               0x0
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_COLUMNS_PER_ROW_THRESHOLD__SHIFT                             0xb
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_ROWS_PER_PRIM_THRESHOLD_MASK                                 0x000007FFL
+#define PA_SC_BINNER_PERF_CNTL_2__BATCH_HIST_NUM_COLUMNS_PER_ROW_THRESHOLD_MASK                               0x003FF800L
+//PA_SC_BINNER_PERF_CNTL_3
+#define PA_SC_BINNER_PERF_CNTL_3__BATCH_HIST_NUM_PS_WAVE_BREAKS_THRESHOLD__SHIFT                              0x0
+#define PA_SC_BINNER_PERF_CNTL_3__BATCH_HIST_NUM_PS_WAVE_BREAKS_THRESHOLD_MASK                                0xFFFFFFFFL
+//PA_SC_P3D_TRAP_SCREEN_HV_LOCK
+#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                         0x0
+#define PA_SC_P3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                           0x00000001L
+//PA_SC_HP3D_TRAP_SCREEN_HV_LOCK
+#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                        0x0
+#define PA_SC_HP3D_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                          0x00000001L
+//PA_SC_TRAP_SCREEN_HV_LOCK
+#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES__SHIFT                                             0x0
+#define PA_SC_TRAP_SCREEN_HV_LOCK__DISABLE_NON_PRIV_WRITES_MASK                                               0x00000001L
+//PA_PH_INTERFACE_FIFO_SIZE
+#define PA_PH_INTERFACE_FIFO_SIZE__PA_PH_IF_FIFO_SIZE__SHIFT                                                  0x0
+#define PA_PH_INTERFACE_FIFO_SIZE__PH_SC_IF_FIFO_SIZE__SHIFT                                                  0x10
+#define PA_PH_INTERFACE_FIFO_SIZE__PA_PH_IF_FIFO_SIZE_MASK                                                    0x000003FFL
+#define PA_PH_INTERFACE_FIFO_SIZE__PH_SC_IF_FIFO_SIZE_MASK                                                    0x003F0000L
+//PA_PH_ENHANCE
+#define PA_PH_ENHANCE__ECO_SPARE0__SHIFT                                                                      0x0
+#define PA_PH_ENHANCE__ECO_SPARE1__SHIFT                                                                      0x1
+#define PA_PH_ENHANCE__ECO_SPARE2__SHIFT                                                                      0x2
+#define PA_PH_ENHANCE__ECO_SPARE3__SHIFT                                                                      0x3
+#define PA_PH_ENHANCE__DISABLE_PH_SC_INTF_FINE_CLOCK_GATE__SHIFT                                              0x4
+#define PA_PH_ENHANCE__DISABLE_FOPKT__SHIFT                                                                   0x5
+#define PA_PH_ENHANCE__DISABLE_FOPKT_SCAN_POST_RESET__SHIFT                                                   0x6
+#define PA_PH_ENHANCE__DISABLE_PH_SC_INTF_CLKEN_CLOCK_GATE__SHIFT                                             0x7
+#define PA_PH_ENHANCE__DISABLE_PH_DEBUG_REG_FGCG__SHIFT                                                       0x8
+#define PA_PH_ENHANCE__DISABLE_PH_PERF_REG_FGCG__SHIFT                                                        0x9
+#define PA_PH_ENHANCE__ENABLE_PH_INTF_CLKEN_STRETCH__SHIFT                                                    0xa
+#define PA_PH_ENHANCE__DISABLE_USE_LAST_PH_ARBITER_PERFCOUNTER_SAMPLE_EVENT__SHIFT                            0xd
+#define PA_PH_ENHANCE__USE_PERFCOUNTER_START_STOP_EVENTS__SHIFT                                               0xe
+#define PA_PH_ENHANCE__FORCE_PH_PERFCOUNTER_SAMPLE_ENABLE_ON__SHIFT                                           0xf
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_MODE__SHIFT                                                         0x10
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_MODE_DISABLE__SHIFT                                                 0x11
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_PERFCOUNTER_COUNT_MODE__SHIFT                                       0x12
+#define PA_PH_ENHANCE__ECO_SPARE0_MASK                                                                        0x00000001L
+#define PA_PH_ENHANCE__ECO_SPARE1_MASK                                                                        0x00000002L
+#define PA_PH_ENHANCE__ECO_SPARE2_MASK                                                                        0x00000004L
+#define PA_PH_ENHANCE__ECO_SPARE3_MASK                                                                        0x00000008L
+#define PA_PH_ENHANCE__DISABLE_PH_SC_INTF_FINE_CLOCK_GATE_MASK                                                0x00000010L
+#define PA_PH_ENHANCE__DISABLE_FOPKT_MASK                                                                     0x00000020L
+#define PA_PH_ENHANCE__DISABLE_FOPKT_SCAN_POST_RESET_MASK                                                     0x00000040L
+#define PA_PH_ENHANCE__DISABLE_PH_SC_INTF_CLKEN_CLOCK_GATE_MASK                                               0x00000080L
+#define PA_PH_ENHANCE__DISABLE_PH_DEBUG_REG_FGCG_MASK                                                         0x00000100L
+#define PA_PH_ENHANCE__DISABLE_PH_PERF_REG_FGCG_MASK                                                          0x00000200L
+#define PA_PH_ENHANCE__ENABLE_PH_INTF_CLKEN_STRETCH_MASK                                                      0x00001C00L
+#define PA_PH_ENHANCE__DISABLE_USE_LAST_PH_ARBITER_PERFCOUNTER_SAMPLE_EVENT_MASK                              0x00002000L
+#define PA_PH_ENHANCE__USE_PERFCOUNTER_START_STOP_EVENTS_MASK                                                 0x00004000L
+#define PA_PH_ENHANCE__FORCE_PH_PERFCOUNTER_SAMPLE_ENABLE_ON_MASK                                             0x00008000L
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_MODE_MASK                                                           0x00010000L
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_MODE_DISABLE_MASK                                                   0x00020000L
+#define PA_PH_ENHANCE__PH_SPI_GE_THROTTLE_PERFCOUNTER_COUNT_MODE_MASK                                         0x00040000L
+//PA_SC_VRS_SURFACE_CNTL_1
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE__SHIFT                                               0x0
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_SHADER_KILL_ENABLE__SHIFT                            0x1
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_MASK_OPS_ENABLE__SHIFT                               0x2
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_RATE_16XAA__SHIFT                                    0x3
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_Z_OR_STENCIL__SHIFT                                  0x4
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_PRE_SHADER_DEPTH_COVERAGE_ENABLED__SHIFT             0x5
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_POST_DEPTH_IMPORT__SHIFT                             0x6
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_POPS__SHIFT                                          0x7
+#define PA_SC_VRS_SURFACE_CNTL_1__USE_ONLY_VRS_RATE_FINE_CFG__SHIFT                                           0x8
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_SSAA_VRS_RATE_NORMALIZATION__SHIFT                                  0xc
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_PS_ITER_RATE_COMBINER_PASSTHRU_OVERRIDE__SHIFT                      0xf
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_CMASK_RATE_HINT_FORCE_ZERO_OVERRIDE__SHIFT                          0x13
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_SSAA_DETAIL_TO_EXPOSED_RATE_CLAMPING__SHIFT                         0x14
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_0__SHIFT                                                      0x15
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_1__SHIFT                                                      0x16
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_2__SHIFT                                                      0x17
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_3__SHIFT                                                      0x18
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_4__SHIFT                                                      0x19
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_5__SHIFT                                                      0x1a
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_6__SHIFT                                                      0x1b
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_7__SHIFT                                                      0x1c
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_8__SHIFT                                                      0x1d
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_9__SHIFT                                                      0x1e
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_10__SHIFT                                                     0x1f
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_MASK                                                 0x00000001L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_SHADER_KILL_ENABLE_MASK                              0x00000002L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_MASK_OPS_ENABLE_MASK                                 0x00000004L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_RATE_16XAA_MASK                                      0x00000008L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_Z_OR_STENCIL_MASK                                    0x00000010L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_PRE_SHADER_DEPTH_COVERAGE_ENABLED_MASK               0x00000020L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_POST_DEPTH_IMPORT_MASK                               0x00000040L
+#define PA_SC_VRS_SURFACE_CNTL_1__FORCE_SC_VRS_RATE_FINE_POPS_MASK                                            0x00000080L
+#define PA_SC_VRS_SURFACE_CNTL_1__USE_ONLY_VRS_RATE_FINE_CFG_MASK                                             0x00000100L
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_SSAA_VRS_RATE_NORMALIZATION_MASK                                    0x00001000L
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_PS_ITER_RATE_COMBINER_PASSTHRU_OVERRIDE_MASK                        0x00008000L
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_CMASK_RATE_HINT_FORCE_ZERO_OVERRIDE_MASK                            0x00080000L
+#define PA_SC_VRS_SURFACE_CNTL_1__DISABLE_SSAA_DETAIL_TO_EXPOSED_RATE_CLAMPING_MASK                           0x00100000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_0_MASK                                                        0x00200000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_1_MASK                                                        0x00400000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_2_MASK                                                        0x00800000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_3_MASK                                                        0x01000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_4_MASK                                                        0x02000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_5_MASK                                                        0x04000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_6_MASK                                                        0x08000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_7_MASK                                                        0x10000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_8_MASK                                                        0x20000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_9_MASK                                                        0x40000000L
+#define PA_SC_VRS_SURFACE_CNTL_1__VRS_ECO_SPARE_10_MASK                                                       0x80000000L
+
+
+// addressBlock: gc_pfvf_sqdec
+//SQ_RUNTIME_CONFIG
+#define SQ_RUNTIME_CONFIG__UNUSED_REGISTER__SHIFT                                                             0x0
+#define SQ_RUNTIME_CONFIG__UNUSED_REGISTER_MASK                                                               0x00000001L
+//SQ_DEBUG_STS_GLOBAL
+#define SQ_DEBUG_STS_GLOBAL__BUSY__SHIFT                                                                      0x0
+#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_BUSY__SHIFT                                                            0x1
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SA0__SHIFT                                                            0x4
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SA1__SHIFT                                                            0x10
+#define SQ_DEBUG_STS_GLOBAL__BUSY_MASK                                                                        0x00000001L
+#define SQ_DEBUG_STS_GLOBAL__INTERRUPT_BUSY_MASK                                                              0x00000002L
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SA0_MASK                                                              0x0000FFF0L
+#define SQ_DEBUG_STS_GLOBAL__WAVE_LEVEL_SA1_MASK                                                              0x0FFF0000L
+//SQ_DEBUG_STS_GLOBAL2
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_GFX0__SHIFT                                                      0x0
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_GFX1__SHIFT                                                      0x8
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_COMPUTE__SHIFT                                                   0x10
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_GFX0_MASK                                                        0x000000FFL
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_GFX1_MASK                                                        0x0000FF00L
+#define SQ_DEBUG_STS_GLOBAL2__REG_FIFO_LEVEL_COMPUTE_MASK                                                     0x00FF0000L
+//SH_MEM_BASES
+#define SH_MEM_BASES__PRIVATE_BASE__SHIFT                                                                     0x0
+#define SH_MEM_BASES__SHARED_BASE__SHIFT                                                                      0x10
+#define SH_MEM_BASES__PRIVATE_BASE_MASK                                                                       0x0000FFFFL
+#define SH_MEM_BASES__SHARED_BASE_MASK                                                                        0xFFFF0000L
+//SH_MEM_CONFIG
+#define SH_MEM_CONFIG__ADDRESS_MODE__SHIFT                                                                    0x0
+#define SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT                                                                  0x2
+#define SH_MEM_CONFIG__INITIAL_INST_PREFETCH__SHIFT                                                           0xe
+#define SH_MEM_CONFIG__ICACHE_USE_GL1__SHIFT                                                                  0x12
+#define SH_MEM_CONFIG__ADDRESS_MODE_MASK                                                                      0x00000001L
+#define SH_MEM_CONFIG__ALIGNMENT_MODE_MASK                                                                    0x0000000CL
+#define SH_MEM_CONFIG__INITIAL_INST_PREFETCH_MASK                                                             0x0000C000L
+#define SH_MEM_CONFIG__ICACHE_USE_GL1_MASK                                                                    0x00040000L
+//SQ_DEBUG
+#define SQ_DEBUG__SINGLE_MEMOP__SHIFT                                                                         0x0
+#define SQ_DEBUG__SINGLE_ALU_OP__SHIFT                                                                        0x1
+#define SQ_DEBUG__WAIT_DEP_CTR_ZERO__SHIFT                                                                    0x2
+#define SQ_DEBUG__SINGLE_MEMOP_MASK                                                                           0x00000001L
+#define SQ_DEBUG__SINGLE_ALU_OP_MASK                                                                          0x00000002L
+#define SQ_DEBUG__WAIT_DEP_CTR_ZERO_MASK                                                                      0x00000004L
+//SQ_SHADER_TBA_LO
+#define SQ_SHADER_TBA_LO__ADDR_LO__SHIFT                                                                      0x0
+#define SQ_SHADER_TBA_LO__ADDR_LO_MASK                                                                        0xFFFFFFFFL
+//SQ_SHADER_TBA_HI
+#define SQ_SHADER_TBA_HI__ADDR_HI__SHIFT                                                                      0x0
+#define SQ_SHADER_TBA_HI__TRAP_EN__SHIFT                                                                      0x1f
+#define SQ_SHADER_TBA_HI__ADDR_HI_MASK                                                                        0x000000FFL
+#define SQ_SHADER_TBA_HI__TRAP_EN_MASK                                                                        0x80000000L
+//SQ_SHADER_TMA_LO
+#define SQ_SHADER_TMA_LO__ADDR_LO__SHIFT                                                                      0x0
+#define SQ_SHADER_TMA_LO__ADDR_LO_MASK                                                                        0xFFFFFFFFL
+//SQ_SHADER_TMA_HI
+#define SQ_SHADER_TMA_HI__ADDR_HI__SHIFT                                                                      0x0
+#define SQ_SHADER_TMA_HI__ADDR_HI_MASK                                                                        0x000000FFL
+
+
+// addressBlock: gc_pfonly_cpdec
+//CP_DEBUG_2
+#define CP_DEBUG_2__CHIU_NOALLOC_OVERRIDE__SHIFT                                                              0xc
+#define CP_DEBUG_2__RCIU_SECURE_CHECK_DISABLE__SHIFT                                                          0xd
+#define CP_DEBUG_2__RB_PACKET_INJECTOR_DISABLE__SHIFT                                                         0xe
+#define CP_DEBUG_2__CNTX_DONE_COPY_STATE_DISABLE__SHIFT                                                       0xf
+#define CP_DEBUG_2__NOP_DISCARD_DISABLE__SHIFT                                                                0x10
+#define CP_DEBUG_2__DC_INTERLEAVE_DISABLE__SHIFT                                                              0x11
+#define CP_DEBUG_2__BC_LOOKUP_CB_DB_FLUSH_DISABLE__SHIFT                                                      0x1b
+#define CP_DEBUG_2__DC_FORCE_CLK_EN__SHIFT                                                                    0x1c
+#define CP_DEBUG_2__DC_DISABLE_BROADCAST__SHIFT                                                               0x1d
+#define CP_DEBUG_2__NOT_EOP_HW_DETECT_DISABLE__SHIFT                                                          0x1e
+#define CP_DEBUG_2__PFP_DDID_HW_DETECT_DISABLE__SHIFT                                                         0x1f
+#define CP_DEBUG_2__CHIU_NOALLOC_OVERRIDE_MASK                                                                0x00001000L
+#define CP_DEBUG_2__RCIU_SECURE_CHECK_DISABLE_MASK                                                            0x00002000L
+#define CP_DEBUG_2__RB_PACKET_INJECTOR_DISABLE_MASK                                                           0x00004000L
+#define CP_DEBUG_2__CNTX_DONE_COPY_STATE_DISABLE_MASK                                                         0x00008000L
+#define CP_DEBUG_2__NOP_DISCARD_DISABLE_MASK                                                                  0x00010000L
+#define CP_DEBUG_2__DC_INTERLEAVE_DISABLE_MASK                                                                0x00020000L
+#define CP_DEBUG_2__BC_LOOKUP_CB_DB_FLUSH_DISABLE_MASK                                                        0x08000000L
+#define CP_DEBUG_2__DC_FORCE_CLK_EN_MASK                                                                      0x10000000L
+#define CP_DEBUG_2__DC_DISABLE_BROADCAST_MASK                                                                 0x20000000L
+#define CP_DEBUG_2__NOT_EOP_HW_DETECT_DISABLE_MASK                                                            0x40000000L
+#define CP_DEBUG_2__PFP_DDID_HW_DETECT_DISABLE_MASK                                                           0x80000000L
+//CP_FETCHER_SOURCE
+#define CP_FETCHER_SOURCE__ME_SRC__SHIFT                                                                      0x0
+#define CP_FETCHER_SOURCE__ME_SRC_MASK                                                                        0x00000001L
+//CP_DFY_CNTL
+#define CP_DFY_CNTL__POLICY__SHIFT                                                                            0x8
+#define CP_DFY_CNTL__VOL__SHIFT                                                                               0xa
+#define CP_DFY_CNTL__MTYPE__SHIFT                                                                             0xc
+#define CP_DFY_CNTL__REPEATER_FGCG_DISABLE__SHIFT                                                             0x19
+#define CP_DFY_CNTL__TPI_SDP_SEL__SHIFT                                                                       0x1a
+#define CP_DFY_CNTL__WRITE_DIS__SHIFT                                                                         0x1b
+#define CP_DFY_CNTL__LFSR_RESET__SHIFT                                                                        0x1c
+#define CP_DFY_CNTL__MODE__SHIFT                                                                              0x1d
+#define CP_DFY_CNTL__ENABLE__SHIFT                                                                            0x1f
+#define CP_DFY_CNTL__POLICY_MASK                                                                              0x00000300L
+#define CP_DFY_CNTL__VOL_MASK                                                                                 0x00000400L
+#define CP_DFY_CNTL__MTYPE_MASK                                                                               0x00007000L
+#define CP_DFY_CNTL__REPEATER_FGCG_DISABLE_MASK                                                               0x02000000L
+#define CP_DFY_CNTL__TPI_SDP_SEL_MASK                                                                         0x04000000L
+#define CP_DFY_CNTL__WRITE_DIS_MASK                                                                           0x08000000L
+#define CP_DFY_CNTL__LFSR_RESET_MASK                                                                          0x10000000L
+#define CP_DFY_CNTL__MODE_MASK                                                                                0x60000000L
+#define CP_DFY_CNTL__ENABLE_MASK                                                                              0x80000000L
+//CP_DFY_STAT
+#define CP_DFY_STAT__BURST_COUNT__SHIFT                                                                       0x0
+#define CP_DFY_STAT__TAGS_PENDING__SHIFT                                                                      0x10
+#define CP_DFY_STAT__BUSY__SHIFT                                                                              0x1f
+#define CP_DFY_STAT__BURST_COUNT_MASK                                                                         0x0000FFFFL
+#define CP_DFY_STAT__TAGS_PENDING_MASK                                                                        0x07FF0000L
+#define CP_DFY_STAT__BUSY_MASK                                                                                0x80000000L
+//CP_DFY_ADDR_HI
+#define CP_DFY_ADDR_HI__ADDR_HI__SHIFT                                                                        0x0
+#define CP_DFY_ADDR_HI__ADDR_HI_MASK                                                                          0x0000FFFFL
+//CP_DFY_ADDR_LO
+#define CP_DFY_ADDR_LO__ADDR_LO__SHIFT                                                                        0x5
+#define CP_DFY_ADDR_LO__ADDR_LO_MASK                                                                          0xFFFFFFE0L
+//CP_DFY_DATA_0
+#define CP_DFY_DATA_0__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_0__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_1
+#define CP_DFY_DATA_1__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_1__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_2
+#define CP_DFY_DATA_2__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_2__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_3
+#define CP_DFY_DATA_3__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_3__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_4
+#define CP_DFY_DATA_4__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_4__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_5
+#define CP_DFY_DATA_5__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_5__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_6
+#define CP_DFY_DATA_6__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_6__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_7
+#define CP_DFY_DATA_7__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_7__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_8
+#define CP_DFY_DATA_8__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_8__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_9
+#define CP_DFY_DATA_9__DATA__SHIFT                                                                            0x0
+#define CP_DFY_DATA_9__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_DFY_DATA_10
+#define CP_DFY_DATA_10__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_10__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_11
+#define CP_DFY_DATA_11__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_11__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_12
+#define CP_DFY_DATA_12__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_12__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_13
+#define CP_DFY_DATA_13__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_13__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_14
+#define CP_DFY_DATA_14__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_14__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_DATA_15
+#define CP_DFY_DATA_15__DATA__SHIFT                                                                           0x0
+#define CP_DFY_DATA_15__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_DFY_CMD
+#define CP_DFY_CMD__SIZE__SHIFT                                                                               0x10
+#define CP_DFY_CMD__SIZE_MASK                                                                                 0xFFFF0000L
+
+
+// addressBlock: gc_pfonly_cpphqddec
+//CP_HPD_MES_ROQ_OFFSETS
+#define CP_HPD_MES_ROQ_OFFSETS__IQ_OFFSET__SHIFT                                                              0x0
+#define CP_HPD_MES_ROQ_OFFSETS__PQ_OFFSET__SHIFT                                                              0x8
+#define CP_HPD_MES_ROQ_OFFSETS__IB_OFFSET__SHIFT                                                              0x10
+#define CP_HPD_MES_ROQ_OFFSETS__IQ_OFFSET_MASK                                                                0x00000007L
+#define CP_HPD_MES_ROQ_OFFSETS__PQ_OFFSET_MASK                                                                0x00003F00L
+#define CP_HPD_MES_ROQ_OFFSETS__IB_OFFSET_MASK                                                                0x007F0000L
+//CP_HPD_ROQ_OFFSETS
+#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET__SHIFT                                                                  0x0
+#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET__SHIFT                                                                  0x8
+#define CP_HPD_ROQ_OFFSETS__IB_OFFSET__SHIFT                                                                  0x10
+#define CP_HPD_ROQ_OFFSETS__IQ_OFFSET_MASK                                                                    0x00000007L
+#define CP_HPD_ROQ_OFFSETS__PQ_OFFSET_MASK                                                                    0x00003F00L
+#define CP_HPD_ROQ_OFFSETS__IB_OFFSET_MASK                                                                    0x007F0000L
+//CP_HPD_STATUS0
+#define CP_HPD_STATUS0__QUEUE_STATE__SHIFT                                                                    0x0
+#define CP_HPD_STATUS0__MAPPED_QUEUE__SHIFT                                                                   0x5
+#define CP_HPD_STATUS0__QUEUE_AVAILABLE__SHIFT                                                                0x8
+#define CP_HPD_STATUS0__FETCHING_MQD__SHIFT                                                                   0x10
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_PQIB__SHIFT                                                           0x11
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_IQ__SHIFT                                                             0x12
+#define CP_HPD_STATUS0__FORCE_QUEUE_STATE__SHIFT                                                              0x14
+#define CP_HPD_STATUS0__MASTER_QUEUE_IDLE_DIS__SHIFT                                                          0x1b
+#define CP_HPD_STATUS0__ENABLE_OFFLOAD_CHECK__SHIFT                                                           0x1c
+#define CP_HPD_STATUS0__FREEZE_QUEUE_STATE__SHIFT                                                             0x1e
+#define CP_HPD_STATUS0__FORCE_QUEUE__SHIFT                                                                    0x1f
+#define CP_HPD_STATUS0__QUEUE_STATE_MASK                                                                      0x0000001FL
+#define CP_HPD_STATUS0__MAPPED_QUEUE_MASK                                                                     0x000000E0L
+#define CP_HPD_STATUS0__QUEUE_AVAILABLE_MASK                                                                  0x0000FF00L
+#define CP_HPD_STATUS0__FETCHING_MQD_MASK                                                                     0x00010000L
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_PQIB_MASK                                                             0x00020000L
+#define CP_HPD_STATUS0__PEND_TXFER_SIZE_IQ_MASK                                                               0x00040000L
+#define CP_HPD_STATUS0__FORCE_QUEUE_STATE_MASK                                                                0x01F00000L
+#define CP_HPD_STATUS0__MASTER_QUEUE_IDLE_DIS_MASK                                                            0x08000000L
+#define CP_HPD_STATUS0__ENABLE_OFFLOAD_CHECK_MASK                                                             0x30000000L
+#define CP_HPD_STATUS0__FREEZE_QUEUE_STATE_MASK                                                               0x40000000L
+#define CP_HPD_STATUS0__FORCE_QUEUE_MASK                                                                      0x80000000L
+
+
+// addressBlock: gc_pfonly_didtdec
+//DIDT_INDEX_AUTO_INCR_EN
+#define DIDT_INDEX_AUTO_INCR_EN__DIDT_INDEX_AUTO_INCR_EN__SHIFT                                               0x0
+#define DIDT_INDEX_AUTO_INCR_EN__DIDT_INDEX_AUTO_INCR_EN_MASK                                                 0x00000001L
+//DIDT_EDC_CTRL
+#define DIDT_EDC_CTRL__EDC_EN__SHIFT                                                                          0x0
+#define DIDT_EDC_CTRL__EDC_SW_RST__SHIFT                                                                      0x1
+#define DIDT_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT                                                             0x2
+#define DIDT_EDC_CTRL__EDC_FORCE_STALL__SHIFT                                                                 0x3
+#define DIDT_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                     0x4
+#define DIDT_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS__SHIFT                                                      0xa
+#define DIDT_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT                                                        0xe
+#define DIDT_EDC_CTRL__EDC_ALGORITHM_MODE__SHIFT                                                              0xf
+#define DIDT_EDC_CTRL__EDC_AVGDIV__SHIFT                                                                      0x10
+#define DIDT_EDC_CTRL__EDC_THRESHOLD_RSHIFT_SEL__SHIFT                                                        0x14
+#define DIDT_EDC_CTRL__EDC_THRESHOLD_RSHIFT_BIT_NUMS__SHIFT                                                   0x15
+#define DIDT_EDC_CTRL__RLC_FORCE_STALL_EN__SHIFT                                                              0x18
+#define DIDT_EDC_CTRL__RLC_STALL_LEVEL_SEL__SHIFT                                                             0x19
+#define DIDT_EDC_CTRL__EDC_EN_MASK                                                                            0x00000001L
+#define DIDT_EDC_CTRL__EDC_SW_RST_MASK                                                                        0x00000002L
+#define DIDT_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK                                                               0x00000004L
+#define DIDT_EDC_CTRL__EDC_FORCE_STALL_MASK                                                                   0x00000008L
+#define DIDT_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK                                                       0x000003F0L
+#define DIDT_EDC_CTRL__EDC_STALL_PATTERN_BIT_NUMS_MASK                                                        0x00003C00L
+#define DIDT_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK                                                          0x00004000L
+#define DIDT_EDC_CTRL__EDC_ALGORITHM_MODE_MASK                                                                0x00008000L
+#define DIDT_EDC_CTRL__EDC_AVGDIV_MASK                                                                        0x000F0000L
+#define DIDT_EDC_CTRL__EDC_THRESHOLD_RSHIFT_SEL_MASK                                                          0x00100000L
+#define DIDT_EDC_CTRL__EDC_THRESHOLD_RSHIFT_BIT_NUMS_MASK                                                     0x00E00000L
+#define DIDT_EDC_CTRL__RLC_FORCE_STALL_EN_MASK                                                                0x01000000L
+#define DIDT_EDC_CTRL__RLC_STALL_LEVEL_SEL_MASK                                                               0x02000000L
+//DIDT_EDC_THROTTLE_CTRL
+#define DIDT_EDC_THROTTLE_CTRL__SQ_STALL_EN__SHIFT                                                            0x0
+#define DIDT_EDC_THROTTLE_CTRL__DB_STALL_EN__SHIFT                                                            0x1
+#define DIDT_EDC_THROTTLE_CTRL__TCP_STALL_EN__SHIFT                                                           0x2
+#define DIDT_EDC_THROTTLE_CTRL__TD_STALL_EN__SHIFT                                                            0x3
+#define DIDT_EDC_THROTTLE_CTRL__PATTERN_EXTEND_EN__SHIFT                                                      0x4
+#define DIDT_EDC_THROTTLE_CTRL__PATTERN_EXTEND_MODE__SHIFT                                                    0x5
+#define DIDT_EDC_THROTTLE_CTRL__SQ_STALL_EN_MASK                                                              0x00000001L
+#define DIDT_EDC_THROTTLE_CTRL__DB_STALL_EN_MASK                                                              0x00000002L
+#define DIDT_EDC_THROTTLE_CTRL__TCP_STALL_EN_MASK                                                             0x00000004L
+#define DIDT_EDC_THROTTLE_CTRL__TD_STALL_EN_MASK                                                              0x00000008L
+#define DIDT_EDC_THROTTLE_CTRL__PATTERN_EXTEND_EN_MASK                                                        0x00000010L
+#define DIDT_EDC_THROTTLE_CTRL__PATTERN_EXTEND_MODE_MASK                                                      0x000000E0L
+//DIDT_EDC_THRESHOLD
+#define DIDT_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT                                                              0x0
+#define DIDT_EDC_THRESHOLD__EDC_THRESHOLD_MASK                                                                0xFFFFFFFFL
+//DIDT_EDC_STALL_PATTERN_1_2
+#define DIDT_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1__SHIFT                                                0x0
+#define DIDT_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2__SHIFT                                                0x10
+#define DIDT_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_1_MASK                                                  0x00007FFFL
+#define DIDT_EDC_STALL_PATTERN_1_2__EDC_STALL_PATTERN_2_MASK                                                  0x7FFF0000L
+//DIDT_EDC_STALL_PATTERN_3_4
+#define DIDT_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3__SHIFT                                                0x0
+#define DIDT_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4__SHIFT                                                0x10
+#define DIDT_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_3_MASK                                                  0x00007FFFL
+#define DIDT_EDC_STALL_PATTERN_3_4__EDC_STALL_PATTERN_4_MASK                                                  0x7FFF0000L
+//DIDT_EDC_STALL_PATTERN_5_6
+#define DIDT_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5__SHIFT                                                0x0
+#define DIDT_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6__SHIFT                                                0x10
+#define DIDT_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_5_MASK                                                  0x00007FFFL
+#define DIDT_EDC_STALL_PATTERN_5_6__EDC_STALL_PATTERN_6_MASK                                                  0x7FFF0000L
+//DIDT_EDC_STALL_PATTERN_7
+#define DIDT_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7__SHIFT                                                  0x0
+#define DIDT_EDC_STALL_PATTERN_7__EDC_STALL_PATTERN_7_MASK                                                    0x00007FFFL
+//DIDT_EDC_STATUS
+#define DIDT_EDC_STATUS__EDC_FSM_STATE__SHIFT                                                                 0x0
+#define DIDT_EDC_STATUS__EDC_THROTTLE_LEVEL__SHIFT                                                            0x1
+#define DIDT_EDC_STATUS__EDC_FSM_STATE_MASK                                                                   0x00000001L
+#define DIDT_EDC_STATUS__EDC_THROTTLE_LEVEL_MASK                                                              0x0000000EL
+//DIDT_EDC_DYNAMIC_THRESHOLD_RO
+#define DIDT_EDC_DYNAMIC_THRESHOLD_RO__EDC_DYNAMIC_THRESHOLD_RO__SHIFT                                        0x0
+#define DIDT_EDC_DYNAMIC_THRESHOLD_RO__EDC_DYNAMIC_THRESHOLD_RO_MASK                                          0x00000001L
+//DIDT_EDC_OVERFLOW
+#define DIDT_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW__SHIFT                                            0x0
+#define DIDT_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER__SHIFT                                         0x1
+#define DIDT_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW_MASK                                              0x00000001L
+#define DIDT_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER_MASK                                           0x0001FFFEL
+//DIDT_EDC_ROLLING_POWER_DELTA
+#define DIDT_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA__SHIFT                                          0x0
+#define DIDT_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA_MASK                                            0xFFFFFFFFL
+//DIDT_IND_INDEX
+#define DIDT_IND_INDEX__DIDT_IND_INDEX__SHIFT                                                                 0x0
+#define DIDT_IND_INDEX__DIDT_IND_INDEX_MASK                                                                   0xFFFFFFFFL
+//DIDT_IND_DATA
+#define DIDT_IND_DATA__DIDT_IND_DATA__SHIFT                                                                   0x0
+#define DIDT_IND_DATA__DIDT_IND_DATA_MASK                                                                     0xFFFFFFFFL
+
+
+// addressBlock: gc_pfonly_spidec
+//SPI_CDBG_SYS_GFX
+#define SPI_CDBG_SYS_GFX__PS_EN__SHIFT                                                                        0x0
+#define SPI_CDBG_SYS_GFX__GS_EN__SHIFT                                                                        0x2
+#define SPI_CDBG_SYS_GFX__HS_EN__SHIFT                                                                        0x4
+#define SPI_CDBG_SYS_GFX__CS_EN__SHIFT                                                                        0x6
+#define SPI_CDBG_SYS_GFX__PS_EN_MASK                                                                          0x0001L
+#define SPI_CDBG_SYS_GFX__GS_EN_MASK                                                                          0x0004L
+#define SPI_CDBG_SYS_GFX__HS_EN_MASK                                                                          0x0010L
+#define SPI_CDBG_SYS_GFX__CS_EN_MASK                                                                          0x0040L
+//SPI_CDBG_SYS_HP3D
+#define SPI_CDBG_SYS_HP3D__PS_EN__SHIFT                                                                       0x0
+#define SPI_CDBG_SYS_HP3D__GS_EN__SHIFT                                                                       0x2
+#define SPI_CDBG_SYS_HP3D__HS_EN__SHIFT                                                                       0x4
+#define SPI_CDBG_SYS_HP3D__CS_EN__SHIFT                                                                       0x6
+#define SPI_CDBG_SYS_HP3D__PS_EN_MASK                                                                         0x0001L
+#define SPI_CDBG_SYS_HP3D__GS_EN_MASK                                                                         0x0004L
+#define SPI_CDBG_SYS_HP3D__HS_EN_MASK                                                                         0x0010L
+#define SPI_CDBG_SYS_HP3D__CS_EN_MASK                                                                         0x0040L
+//SPI_CDBG_SYS_CS0
+#define SPI_CDBG_SYS_CS0__PIPE0__SHIFT                                                                        0x0
+#define SPI_CDBG_SYS_CS0__PIPE1__SHIFT                                                                        0x8
+#define SPI_CDBG_SYS_CS0__PIPE2__SHIFT                                                                        0x10
+#define SPI_CDBG_SYS_CS0__PIPE3__SHIFT                                                                        0x18
+#define SPI_CDBG_SYS_CS0__PIPE0_MASK                                                                          0x000000FFL
+#define SPI_CDBG_SYS_CS0__PIPE1_MASK                                                                          0x0000FF00L
+#define SPI_CDBG_SYS_CS0__PIPE2_MASK                                                                          0x00FF0000L
+#define SPI_CDBG_SYS_CS0__PIPE3_MASK                                                                          0xFF000000L
+//SPI_GDBG_WAVE_CNTL
+#define SPI_GDBG_WAVE_CNTL__STALL_RA__SHIFT                                                                   0x0
+#define SPI_GDBG_WAVE_CNTL__STALL_LAUNCH__SHIFT                                                               0x1
+#define SPI_GDBG_WAVE_CNTL__STALL_RA_MASK                                                                     0x00000001L
+#define SPI_GDBG_WAVE_CNTL__STALL_LAUNCH_MASK                                                                 0x00000002L
+//SPI_GDBG_TRAP_CONFIG
+#define SPI_GDBG_TRAP_CONFIG__PIPE0_EN__SHIFT                                                                 0x0
+#define SPI_GDBG_TRAP_CONFIG__PIPE1_EN__SHIFT                                                                 0x8
+#define SPI_GDBG_TRAP_CONFIG__PIPE2_EN__SHIFT                                                                 0x10
+#define SPI_GDBG_TRAP_CONFIG__PIPE3_EN__SHIFT                                                                 0x18
+#define SPI_GDBG_TRAP_CONFIG__PIPE0_EN_MASK                                                                   0x000000FFL
+#define SPI_GDBG_TRAP_CONFIG__PIPE1_EN_MASK                                                                   0x0000FF00L
+#define SPI_GDBG_TRAP_CONFIG__PIPE2_EN_MASK                                                                   0x00FF0000L
+#define SPI_GDBG_TRAP_CONFIG__PIPE3_EN_MASK                                                                   0xFF000000L
+//SPI_GDBG_WAVE_CNTL3
+#define SPI_GDBG_WAVE_CNTL3__STALL_PS__SHIFT                                                                  0x0
+#define SPI_GDBG_WAVE_CNTL3__STALL_GS__SHIFT                                                                  0x2
+#define SPI_GDBG_WAVE_CNTL3__STALL_HS__SHIFT                                                                  0x3
+#define SPI_GDBG_WAVE_CNTL3__STALL_CSG__SHIFT                                                                 0x4
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS0__SHIFT                                                                 0x5
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS1__SHIFT                                                                 0x6
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS2__SHIFT                                                                 0x7
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS3__SHIFT                                                                 0x8
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS4__SHIFT                                                                 0x9
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS5__SHIFT                                                                 0xa
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS6__SHIFT                                                                 0xb
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS7__SHIFT                                                                 0xc
+#define SPI_GDBG_WAVE_CNTL3__STALL_DURATION__SHIFT                                                            0xd
+#define SPI_GDBG_WAVE_CNTL3__STALL_MULT__SHIFT                                                                0x1c
+#define SPI_GDBG_WAVE_CNTL3__STALL_PS_MASK                                                                    0x00000001L
+#define SPI_GDBG_WAVE_CNTL3__STALL_GS_MASK                                                                    0x00000004L
+#define SPI_GDBG_WAVE_CNTL3__STALL_HS_MASK                                                                    0x00000008L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CSG_MASK                                                                   0x00000010L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS0_MASK                                                                   0x00000020L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS1_MASK                                                                   0x00000040L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS2_MASK                                                                   0x00000080L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS3_MASK                                                                   0x00000100L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS4_MASK                                                                   0x00000200L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS5_MASK                                                                   0x00000400L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS6_MASK                                                                   0x00000800L
+#define SPI_GDBG_WAVE_CNTL3__STALL_CS7_MASK                                                                   0x00001000L
+#define SPI_GDBG_WAVE_CNTL3__STALL_DURATION_MASK                                                              0x0FFFE000L
+#define SPI_GDBG_WAVE_CNTL3__STALL_MULT_MASK                                                                  0x10000000L
+//SPI_RESET_DEBUG
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET__SHIFT                                                             0x0
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PER_VMID__SHIFT                                                    0x1
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_ALL_VMID__SHIFT                                                    0x2
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_RESOURCE__SHIFT                                                    0x3
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PRIORITY__SHIFT                                                    0x4
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_MASK                                                               0x01L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PER_VMID_MASK                                                      0x02L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_ALL_VMID_MASK                                                      0x04L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_RESOURCE_MASK                                                      0x08L
+#define SPI_RESET_DEBUG__DISABLE_GFX_RESET_PRIORITY_MASK                                                      0x10L
+//SPI_ARB_CNTL_0
+#define SPI_ARB_CNTL_0__EXP_ARB_COL_WT__SHIFT                                                                 0x0
+#define SPI_ARB_CNTL_0__EXP_ARB_POS_WT__SHIFT                                                                 0x4
+#define SPI_ARB_CNTL_0__EXP_ARB_GDS_WT__SHIFT                                                                 0x8
+#define SPI_ARB_CNTL_0__EXP_ARB_COL_WT_MASK                                                                   0x0000000FL
+#define SPI_ARB_CNTL_0__EXP_ARB_POS_WT_MASK                                                                   0x000000F0L
+#define SPI_ARB_CNTL_0__EXP_ARB_GDS_WT_MASK                                                                   0x00000F00L
+//SPI_FEATURE_CTRL
+#define SPI_FEATURE_CTRL__TUNNELING_WAVE_LIMIT__SHIFT                                                         0x0
+#define SPI_FEATURE_CTRL__RA_PROBE_IGNORE__SHIFT                                                              0x4
+#define SPI_FEATURE_CTRL__PS_THROTTLE_MAX_WAVE_LIMIT__SHIFT                                                   0x5
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_WIF_CTRL__SHIFT                                                       0xb
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_OOO_CTRL__SHIFT                                                       0xd
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_DISABLE__SHIFT                                                        0xe
+#define SPI_FEATURE_CTRL__TUNNELING_WAVE_LIMIT_MASK                                                           0x0000000FL
+#define SPI_FEATURE_CTRL__RA_PROBE_IGNORE_MASK                                                                0x00000010L
+#define SPI_FEATURE_CTRL__PS_THROTTLE_MAX_WAVE_LIMIT_MASK                                                     0x000007E0L
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_WIF_CTRL_MASK                                                         0x00001800L
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_OOO_CTRL_MASK                                                         0x00002000L
+#define SPI_FEATURE_CTRL__RA_PROBE_SKEW_DISABLE_MASK                                                          0x00004000L
+//SPI_SHADER_RSRC_LIMIT_CTRL
+#define SPI_SHADER_RSRC_LIMIT_CTRL__WAVES_PER_SIMD32__SHIFT                                                   0x0
+#define SPI_SHADER_RSRC_LIMIT_CTRL__VGPR_PER_SIMD32__SHIFT                                                    0x5
+#define SPI_SHADER_RSRC_LIMIT_CTRL__VGPR_WRAP_DISABLE__SHIFT                                                  0xc
+#define SPI_SHADER_RSRC_LIMIT_CTRL__BARRIER_LIMIT__SHIFT                                                      0xd
+#define SPI_SHADER_RSRC_LIMIT_CTRL__BARRIER_LIMIT_HIERARCHY_LEVEL__SHIFT                                      0x13
+#define SPI_SHADER_RSRC_LIMIT_CTRL__LDS_LIMIT__SHIFT                                                          0x14
+#define SPI_SHADER_RSRC_LIMIT_CTRL__LDS_LIMIT_HIERARCHY_LEVEL__SHIFT                                          0x1c
+#define SPI_SHADER_RSRC_LIMIT_CTRL__PERFORMANCE_LIMIT_ENABLE__SHIFT                                           0x1f
+#define SPI_SHADER_RSRC_LIMIT_CTRL__WAVES_PER_SIMD32_MASK                                                     0x0000001FL
+#define SPI_SHADER_RSRC_LIMIT_CTRL__VGPR_PER_SIMD32_MASK                                                      0x00000FE0L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__VGPR_WRAP_DISABLE_MASK                                                    0x00001000L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__BARRIER_LIMIT_MASK                                                        0x0007E000L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__BARRIER_LIMIT_HIERARCHY_LEVEL_MASK                                        0x00080000L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__LDS_LIMIT_MASK                                                            0x0FF00000L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__LDS_LIMIT_HIERARCHY_LEVEL_MASK                                            0x10000000L
+#define SPI_SHADER_RSRC_LIMIT_CTRL__PERFORMANCE_LIMIT_ENABLE_MASK                                             0x80000000L
+//SPI_COMPUTE_WF_CTX_SAVE_STATUS
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE0_SAVE_BUSY__SHIFT                                         0x0
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE1_SAVE_BUSY__SHIFT                                         0x1
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE2_SAVE_BUSY__SHIFT                                         0x2
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE3_SAVE_BUSY__SHIFT                                         0x3
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE4_SAVE_BUSY__SHIFT                                         0x4
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE5_SAVE_BUSY__SHIFT                                         0x5
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE6_SAVE_BUSY__SHIFT                                         0x6
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE7_SAVE_BUSY__SHIFT                                         0x7
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE0_SAVE_BUSY__SHIFT                                         0x8
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE1_SAVE_BUSY__SHIFT                                         0x9
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE2_SAVE_BUSY__SHIFT                                         0xa
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE3_SAVE_BUSY__SHIFT                                         0xb
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE4_SAVE_BUSY__SHIFT                                         0xc
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE5_SAVE_BUSY__SHIFT                                         0xd
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE6_SAVE_BUSY__SHIFT                                         0xe
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE7_SAVE_BUSY__SHIFT                                         0xf
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE0_SAVE_BUSY__SHIFT                                         0x10
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE1_SAVE_BUSY__SHIFT                                         0x11
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE2_SAVE_BUSY__SHIFT                                         0x12
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE3_SAVE_BUSY__SHIFT                                         0x13
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE4_SAVE_BUSY__SHIFT                                         0x14
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE5_SAVE_BUSY__SHIFT                                         0x15
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE6_SAVE_BUSY__SHIFT                                         0x16
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE7_SAVE_BUSY__SHIFT                                         0x17
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE0_SAVE_BUSY__SHIFT                                         0x18
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE1_SAVE_BUSY__SHIFT                                         0x19
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE2_SAVE_BUSY__SHIFT                                         0x1a
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE3_SAVE_BUSY__SHIFT                                         0x1b
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE4_SAVE_BUSY__SHIFT                                         0x1c
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE5_SAVE_BUSY__SHIFT                                         0x1d
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE6_SAVE_BUSY__SHIFT                                         0x1e
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE7_SAVE_BUSY__SHIFT                                         0x1f
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE0_SAVE_BUSY_MASK                                           0x00000001L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE1_SAVE_BUSY_MASK                                           0x00000002L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE2_SAVE_BUSY_MASK                                           0x00000004L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE3_SAVE_BUSY_MASK                                           0x00000008L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE4_SAVE_BUSY_MASK                                           0x00000010L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE5_SAVE_BUSY_MASK                                           0x00000020L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE6_SAVE_BUSY_MASK                                           0x00000040L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE0_QUEUE7_SAVE_BUSY_MASK                                           0x00000080L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE0_SAVE_BUSY_MASK                                           0x00000100L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE1_SAVE_BUSY_MASK                                           0x00000200L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE2_SAVE_BUSY_MASK                                           0x00000400L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE3_SAVE_BUSY_MASK                                           0x00000800L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE4_SAVE_BUSY_MASK                                           0x00001000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE5_SAVE_BUSY_MASK                                           0x00002000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE6_SAVE_BUSY_MASK                                           0x00004000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE1_QUEUE7_SAVE_BUSY_MASK                                           0x00008000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE0_SAVE_BUSY_MASK                                           0x00010000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE1_SAVE_BUSY_MASK                                           0x00020000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE2_SAVE_BUSY_MASK                                           0x00040000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE3_SAVE_BUSY_MASK                                           0x00080000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE4_SAVE_BUSY_MASK                                           0x00100000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE5_SAVE_BUSY_MASK                                           0x00200000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE6_SAVE_BUSY_MASK                                           0x00400000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE2_QUEUE7_SAVE_BUSY_MASK                                           0x00800000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE0_SAVE_BUSY_MASK                                           0x01000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE1_SAVE_BUSY_MASK                                           0x02000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE2_SAVE_BUSY_MASK                                           0x04000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE3_SAVE_BUSY_MASK                                           0x08000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE4_SAVE_BUSY_MASK                                           0x10000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE5_SAVE_BUSY_MASK                                           0x20000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE6_SAVE_BUSY_MASK                                           0x40000000L
+#define SPI_COMPUTE_WF_CTX_SAVE_STATUS__PIPE3_QUEUE7_SAVE_BUSY_MASK                                           0x80000000L
+
+
+// addressBlock: gc_pfonly_tcpdec
+//TCP_INVALIDATE
+#define TCP_INVALIDATE__START__SHIFT                                                                          0x0
+#define TCP_INVALIDATE__START_MASK                                                                            0x00000001L
+//TCP_STATUS
+#define TCP_STATUS__TCP_BUSY__SHIFT                                                                           0x0
+#define TCP_STATUS__INPUT_BUSY__SHIFT                                                                         0x1
+#define TCP_STATUS__ADRS_BUSY__SHIFT                                                                          0x2
+#define TCP_STATUS__TAGRAMS_BUSY__SHIFT                                                                       0x3
+#define TCP_STATUS__CNTRL_BUSY__SHIFT                                                                         0x4
+#define TCP_STATUS__LFIFO_BUSY__SHIFT                                                                         0x5
+#define TCP_STATUS__READ_BUSY__SHIFT                                                                          0x6
+#define TCP_STATUS__FORMAT_BUSY__SHIFT                                                                        0x7
+#define TCP_STATUS__VM_BUSY__SHIFT                                                                            0x8
+#define TCP_STATUS__MEMIF_BUSY__SHIFT                                                                         0x9
+#define TCP_STATUS__GCR_BUSY__SHIFT                                                                           0xa
+#define TCP_STATUS__OFIFO_BUSY__SHIFT                                                                         0xb
+#define TCP_STATUS__OFIFO_QUEUE_BUSY__SHIFT                                                                   0xc
+#define TCP_STATUS__XNACK_PRT__SHIFT                                                                          0xf
+#define TCP_STATUS__TCP_BUSY_MASK                                                                             0x00000001L
+#define TCP_STATUS__INPUT_BUSY_MASK                                                                           0x00000002L
+#define TCP_STATUS__ADRS_BUSY_MASK                                                                            0x00000004L
+#define TCP_STATUS__TAGRAMS_BUSY_MASK                                                                         0x00000008L
+#define TCP_STATUS__CNTRL_BUSY_MASK                                                                           0x00000010L
+#define TCP_STATUS__LFIFO_BUSY_MASK                                                                           0x00000020L
+#define TCP_STATUS__READ_BUSY_MASK                                                                            0x00000040L
+#define TCP_STATUS__FORMAT_BUSY_MASK                                                                          0x00000080L
+#define TCP_STATUS__VM_BUSY_MASK                                                                              0x00000100L
+#define TCP_STATUS__MEMIF_BUSY_MASK                                                                           0x00000200L
+#define TCP_STATUS__GCR_BUSY_MASK                                                                             0x00000400L
+#define TCP_STATUS__OFIFO_BUSY_MASK                                                                           0x00000800L
+#define TCP_STATUS__OFIFO_QUEUE_BUSY_MASK                                                                     0x00003000L
+#define TCP_STATUS__XNACK_PRT_MASK                                                                            0x00008000L
+//TCP_CNTL
+#define TCP_CNTL__FORCE_HIT__SHIFT                                                                            0x0
+#define TCP_CNTL__FORCE_MISS__SHIFT                                                                           0x1
+#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE__SHIFT                                                               0x5
+#define TCP_CNTL__TD_DATA_EN_OVERRIDE__SHIFT                                                                  0x6
+#define TCP_CNTL__ENABLE_128B_DCC_COMP_READ_FOR_INDEP64__SHIFT                                                0x7
+#define TCP_CNTL__DISABLE_WRITE_COMBINING__SHIFT                                                              0x9
+#define TCP_CNTL__FORCE_EOW_TOTAL_CNT__SHIFT                                                                  0xf
+#define TCP_CNTL__FORCE_EOW_SET_CNT__SHIFT                                                                    0x16
+#define TCP_CNTL__DISABLE_Z_MAP__SHIFT                                                                        0x1c
+#define TCP_CNTL__FORCE_ORDER_BETWEEN_READ_WRITE_TO_SAME_ADDRESS__SHIFT                                       0x1d
+#define TCP_CNTL__ASTC_VE_MSB_TOLERANT__SHIFT                                                                 0x1f
+#define TCP_CNTL__FORCE_HIT_MASK                                                                              0x00000001L
+#define TCP_CNTL__FORCE_MISS_MASK                                                                             0x00000002L
+#define TCP_CNTL__FLAT_BUF_CACHE_SWIZZLE_MASK                                                                 0x00000020L
+#define TCP_CNTL__TD_DATA_EN_OVERRIDE_MASK                                                                    0x00000040L
+#define TCP_CNTL__ENABLE_128B_DCC_COMP_READ_FOR_INDEP64_MASK                                                  0x00000080L
+#define TCP_CNTL__DISABLE_WRITE_COMBINING_MASK                                                                0x00000200L
+#define TCP_CNTL__FORCE_EOW_TOTAL_CNT_MASK                                                                    0x001F8000L
+#define TCP_CNTL__FORCE_EOW_SET_CNT_MASK                                                                      0x07C00000L
+#define TCP_CNTL__DISABLE_Z_MAP_MASK                                                                          0x10000000L
+#define TCP_CNTL__FORCE_ORDER_BETWEEN_READ_WRITE_TO_SAME_ADDRESS_MASK                                         0x20000000L
+#define TCP_CNTL__ASTC_VE_MSB_TOLERANT_MASK                                                                   0x80000000L
+//TCP_CNTL2
+#define TCP_CNTL2__LS_DISABLE_CLOCKS__SHIFT                                                                   0x0
+#define TCP_CNTL2__TCP_FMT_MGCG_DISABLE__SHIFT                                                                0x8
+#define TCP_CNTL2__TCPF_LATENCY_BYPASS_DISABLE__SHIFT                                                         0x9
+#define TCP_CNTL2__TCP_WRITE_DATA_MGCG_DISABLE__SHIFT                                                         0xa
+#define TCP_CNTL2__TCP_INNER_BLOCK_MGCG_DISABLE__SHIFT                                                        0xb
+#define TCP_CNTL2__TCP_ADRS_IMG_CALC_MGCG_DISABLE__SHIFT                                                      0xc
+#define TCP_CNTL2__V64_COMBINE_ENABLE__SHIFT                                                                  0xd
+#define TCP_CNTL2__TAGRAM_ADDR_SWIZZLE_DISABLE__SHIFT                                                         0xe
+#define TCP_CNTL2__RETURN_ORDER_OVERRIDE__SHIFT                                                               0xf
+#define TCP_CNTL2__POWER_OPT_DISABLE__SHIFT                                                                   0x10
+#define TCP_CNTL2__GCR_RSP_FGCG_DISABLE__SHIFT                                                                0x11
+#define TCP_CNTL2__PERF_EN_OVERRIDE__SHIFT                                                                    0x12
+#define TCP_CNTL2__TC_TD_RAM_CLKEN_DISABLE__SHIFT                                                             0x14
+#define TCP_CNTL2__TC_TD_DATA_CLKEN_DISABLE__SHIFT                                                            0x15
+#define TCP_CNTL2__TCP_GL1_REQ_CLKEN_DISABLE__SHIFT                                                           0x16
+#define TCP_CNTL2__TCP_GL1R_SRC_CLKEN_DISABLE__SHIFT                                                          0x17
+#define TCP_CNTL2__SPARE_BIT__SHIFT                                                                           0x1a
+#define TCP_CNTL2__TAGRAM_XY_BIAS_OVERRIDE__SHIFT                                                             0x1b
+#define TCP_CNTL2__TCP_REQ_MGCG_DISABLE__SHIFT                                                                0x1d
+#define TCP_CNTL2__TCP_MISS_MGCG_DISABLE__SHIFT                                                               0x1e
+#define TCP_CNTL2__DISABLE_MIPMAP_PARAM_CALC_SELF_GATING__SHIFT                                               0x1f
+#define TCP_CNTL2__LS_DISABLE_CLOCKS_MASK                                                                     0x000000FFL
+#define TCP_CNTL2__TCP_FMT_MGCG_DISABLE_MASK                                                                  0x00000100L
+#define TCP_CNTL2__TCPF_LATENCY_BYPASS_DISABLE_MASK                                                           0x00000200L
+#define TCP_CNTL2__TCP_WRITE_DATA_MGCG_DISABLE_MASK                                                           0x00000400L
+#define TCP_CNTL2__TCP_INNER_BLOCK_MGCG_DISABLE_MASK                                                          0x00000800L
+#define TCP_CNTL2__TCP_ADRS_IMG_CALC_MGCG_DISABLE_MASK                                                        0x00001000L
+#define TCP_CNTL2__V64_COMBINE_ENABLE_MASK                                                                    0x00002000L
+#define TCP_CNTL2__TAGRAM_ADDR_SWIZZLE_DISABLE_MASK                                                           0x00004000L
+#define TCP_CNTL2__RETURN_ORDER_OVERRIDE_MASK                                                                 0x00008000L
+#define TCP_CNTL2__POWER_OPT_DISABLE_MASK                                                                     0x00010000L
+#define TCP_CNTL2__GCR_RSP_FGCG_DISABLE_MASK                                                                  0x00020000L
+#define TCP_CNTL2__PERF_EN_OVERRIDE_MASK                                                                      0x000C0000L
+#define TCP_CNTL2__TC_TD_RAM_CLKEN_DISABLE_MASK                                                               0x00100000L
+#define TCP_CNTL2__TC_TD_DATA_CLKEN_DISABLE_MASK                                                              0x00200000L
+#define TCP_CNTL2__TCP_GL1_REQ_CLKEN_DISABLE_MASK                                                             0x00400000L
+#define TCP_CNTL2__TCP_GL1R_SRC_CLKEN_DISABLE_MASK                                                            0x00800000L
+#define TCP_CNTL2__SPARE_BIT_MASK                                                                             0x04000000L
+#define TCP_CNTL2__TAGRAM_XY_BIAS_OVERRIDE_MASK                                                               0x18000000L
+#define TCP_CNTL2__TCP_REQ_MGCG_DISABLE_MASK                                                                  0x20000000L
+#define TCP_CNTL2__TCP_MISS_MGCG_DISABLE_MASK                                                                 0x40000000L
+#define TCP_CNTL2__DISABLE_MIPMAP_PARAM_CALC_SELF_GATING_MASK                                                 0x80000000L
+//TCP_CREDIT
+#define TCP_CREDIT__LFIFO_RAM_DEPTH__SHIFT                                                                    0x0
+#define TCP_CREDIT__GL1_REQ_CREDIT__SHIFT                                                                     0xa
+#define TCP_CREDIT__REQ_FIFO_CREDIT__SHIFT                                                                    0x10
+#define TCP_CREDIT__TD_RAM_CREDIT__SHIFT                                                                      0x17
+#define TCP_CREDIT__TD_DATA_CREDIT__SHIFT                                                                     0x1d
+#define TCP_CREDIT__LFIFO_RAM_DEPTH_MASK                                                                      0x000003FFL
+#define TCP_CREDIT__GL1_REQ_CREDIT_MASK                                                                       0x0000FC00L
+#define TCP_CREDIT__REQ_FIFO_CREDIT_MASK                                                                      0x007F0000L
+#define TCP_CREDIT__TD_RAM_CREDIT_MASK                                                                        0x0F800000L
+#define TCP_CREDIT__TD_DATA_CREDIT_MASK                                                                       0xE0000000L
+
+
+// addressBlock: gc_pfonly_gdsdec
+//GDS_ENHANCE2
+#define GDS_ENHANCE2__DISABLE_MEMORY_VIOLATION_REPORT__SHIFT                                                  0x0
+#define GDS_ENHANCE2__GDS_INTERFACES_FGCG_OVERRIDE__SHIFT                                                     0x1
+#define GDS_ENHANCE2__DISABLE_PIPE_MEMORY_RD_OPT__SHIFT                                                       0x2
+#define GDS_ENHANCE2__UNUSED__SHIFT                                                                           0x3
+#define GDS_ENHANCE2__DISABLE_MEMORY_VIOLATION_REPORT_MASK                                                    0x00000001L
+#define GDS_ENHANCE2__GDS_INTERFACES_FGCG_OVERRIDE_MASK                                                       0x00000002L
+#define GDS_ENHANCE2__DISABLE_PIPE_MEMORY_RD_OPT_MASK                                                         0x00000004L
+#define GDS_ENHANCE2__UNUSED_MASK                                                                             0xFFFFFFF8L
+//GDS_OA_CGPG_RESTORE
+#define GDS_OA_CGPG_RESTORE__VMID__SHIFT                                                                      0x0
+#define GDS_OA_CGPG_RESTORE__MEID__SHIFT                                                                      0x8
+#define GDS_OA_CGPG_RESTORE__PIPEID__SHIFT                                                                    0xc
+#define GDS_OA_CGPG_RESTORE__QUEUEID__SHIFT                                                                   0x10
+#define GDS_OA_CGPG_RESTORE__UNUSED__SHIFT                                                                    0x14
+#define GDS_OA_CGPG_RESTORE__VMID_MASK                                                                        0x000000FFL
+#define GDS_OA_CGPG_RESTORE__MEID_MASK                                                                        0x00000F00L
+#define GDS_OA_CGPG_RESTORE__PIPEID_MASK                                                                      0x0000F000L
+#define GDS_OA_CGPG_RESTORE__QUEUEID_MASK                                                                     0x000F0000L
+#define GDS_OA_CGPG_RESTORE__UNUSED_MASK                                                                      0xFFF00000L
+
+
+// addressBlock: gc_pfonly_utcl1dec
+//UTCL1_CTRL_0
+#define UTCL1_CTRL_0__UTCL1_L0_REQ_VFIFO_DISABLE__SHIFT                                                       0x0
+#define UTCL1_CTRL_0__UTCL1_UTCL2_INVACK_CDC_FIFO_DISABLE__SHIFT                                              0x1
+#define UTCL1_CTRL_0__RESERVED_0__SHIFT                                                                       0x2
+#define UTCL1_CTRL_0__UTCL1_UTCL2_REQ_CREDITS__SHIFT                                                          0x3
+#define UTCL1_CTRL_0__UTCL1_UTCL0_INVREQ_CREDITS__SHIFT                                                       0x9
+#define UTCL1_CTRL_0__UTCL1_LIMIT_INV_TO_ONE__SHIFT                                                           0xd
+#define UTCL1_CTRL_0__UTCL1_LIMIT_XLAT_TO_ONE__SHIFT                                                          0xe
+#define UTCL1_CTRL_0__UTCL1_UTCL2_FGCG_REPEATERS_OVERRIDE__SHIFT                                              0xf
+#define UTCL1_CTRL_0__UTCL1_INV_FILTER_VMID__SHIFT                                                            0x10
+#define UTCL1_CTRL_0__UTCL1_RANGE_INV_FORCE_CHK_ALL__SHIFT                                                    0x11
+#define UTCL1_CTRL_0__UTCL1_UTCL0_RET_FGCG_REPEATERS_OVERRIDE__SHIFT                                          0x12
+#define UTCL1_CTRL_0__UTCL1_UTCL0_INVREQ_FGCG_REPEATERS_OVERRIDE__SHIFT                                       0x13
+#define UTCL1_CTRL_0__GCRD_FGCG_DISABLE__SHIFT                                                                0x14
+#define UTCL1_CTRL_0__UTCL1_MH_RANGE_INV_TO_VMID_OVERRIDE__SHIFT                                              0x15
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_DUPLICATES__SHIFT                                                      0x16
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_REQUEST_SQUASHING__SHIFT                                               0x17
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_RECENT_BUFFER__SHIFT                                                   0x18
+#define UTCL1_CTRL_0__UTCL1_XLAT_FAULT_LOCK_CTRL__SHIFT                                                       0x19
+#define UTCL1_CTRL_0__UTCL1_REDUCE_CC_SIZE__SHIFT                                                             0x1b
+#define UTCL1_CTRL_0__RESERVED_1__SHIFT                                                                       0x1d
+#define UTCL1_CTRL_0__MH_SPARE0__SHIFT                                                                        0x1e
+#define UTCL1_CTRL_0__RESERVED_2__SHIFT                                                                       0x1f
+#define UTCL1_CTRL_0__UTCL1_L0_REQ_VFIFO_DISABLE_MASK                                                         0x00000001L
+#define UTCL1_CTRL_0__UTCL1_UTCL2_INVACK_CDC_FIFO_DISABLE_MASK                                                0x00000002L
+#define UTCL1_CTRL_0__RESERVED_0_MASK                                                                         0x00000004L
+#define UTCL1_CTRL_0__UTCL1_UTCL2_REQ_CREDITS_MASK                                                            0x000001F8L
+#define UTCL1_CTRL_0__UTCL1_UTCL0_INVREQ_CREDITS_MASK                                                         0x00001E00L
+#define UTCL1_CTRL_0__UTCL1_LIMIT_INV_TO_ONE_MASK                                                             0x00002000L
+#define UTCL1_CTRL_0__UTCL1_LIMIT_XLAT_TO_ONE_MASK                                                            0x00004000L
+#define UTCL1_CTRL_0__UTCL1_UTCL2_FGCG_REPEATERS_OVERRIDE_MASK                                                0x00008000L
+#define UTCL1_CTRL_0__UTCL1_INV_FILTER_VMID_MASK                                                              0x00010000L
+#define UTCL1_CTRL_0__UTCL1_RANGE_INV_FORCE_CHK_ALL_MASK                                                      0x00020000L
+#define UTCL1_CTRL_0__UTCL1_UTCL0_RET_FGCG_REPEATERS_OVERRIDE_MASK                                            0x00040000L
+#define UTCL1_CTRL_0__UTCL1_UTCL0_INVREQ_FGCG_REPEATERS_OVERRIDE_MASK                                         0x00080000L
+#define UTCL1_CTRL_0__GCRD_FGCG_DISABLE_MASK                                                                  0x00100000L
+#define UTCL1_CTRL_0__UTCL1_MH_RANGE_INV_TO_VMID_OVERRIDE_MASK                                                0x00200000L
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_DUPLICATES_MASK                                                        0x00400000L
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_REQUEST_SQUASHING_MASK                                                 0x00800000L
+#define UTCL1_CTRL_0__UTCL1_MH_DISABLE_RECENT_BUFFER_MASK                                                     0x01000000L
+#define UTCL1_CTRL_0__UTCL1_XLAT_FAULT_LOCK_CTRL_MASK                                                         0x06000000L
+#define UTCL1_CTRL_0__UTCL1_REDUCE_CC_SIZE_MASK                                                               0x18000000L
+#define UTCL1_CTRL_0__RESERVED_1_MASK                                                                         0x20000000L
+#define UTCL1_CTRL_0__MH_SPARE0_MASK                                                                          0x40000000L
+#define UTCL1_CTRL_0__RESERVED_2_MASK                                                                         0x80000000L
+//UTCL1_UTCL0_INVREQ_DISABLE
+#define UTCL1_UTCL0_INVREQ_DISABLE__UTCL1_UTCL0_INVREQ_DISABLE__SHIFT                                         0x0
+#define UTCL1_UTCL0_INVREQ_DISABLE__UTCL1_UTCL0_INVREQ_DISABLE_MASK                                           0xFFFFFFFFL
+//UTCL1_CTRL_2
+#define UTCL1_CTRL_2__UTCL1_RNG_TO_VMID_INV_OVRD__SHIFT                                                       0x0
+#define UTCL1_CTRL_2__UTCL1_PMM_INTERRUPT_CREDITS_OVERRIDE__SHIFT                                             0x4
+#define UTCL1_CTRL_2__UTCL1_CACHE_WRITE_PERM__SHIFT                                                           0xa
+#define UTCL1_CTRL_2__UTCL1_PAGE_OVRD_DISABLE__SHIFT                                                          0xb
+#define UTCL1_CTRL_2__UTCL1_SPARE0__SHIFT                                                                     0xc
+#define UTCL1_CTRL_2__UTCL1_SPARE1__SHIFT                                                                     0xd
+#define UTCL1_CTRL_2__RESERVED__SHIFT                                                                         0xe
+#define UTCL1_CTRL_2__UTCL1_RNG_TO_VMID_INV_OVRD_MASK                                                         0x0000000FL
+#define UTCL1_CTRL_2__UTCL1_PMM_INTERRUPT_CREDITS_OVERRIDE_MASK                                               0x000003F0L
+#define UTCL1_CTRL_2__UTCL1_CACHE_WRITE_PERM_MASK                                                             0x00000400L
+#define UTCL1_CTRL_2__UTCL1_PAGE_OVRD_DISABLE_MASK                                                            0x00000800L
+#define UTCL1_CTRL_2__UTCL1_SPARE0_MASK                                                                       0x00001000L
+#define UTCL1_CTRL_2__UTCL1_SPARE1_MASK                                                                       0x00002000L
+#define UTCL1_CTRL_2__RESERVED_MASK                                                                           0xFFFFC000L
+//UTCL1_FIFO_SIZING
+#define UTCL1_FIFO_SIZING__UTCL1_UTCL2_INVACK_CDC_FIFO_THRESH__SHIFT                                          0x0
+#define UTCL1_FIFO_SIZING__UTCL1_GENERAL_SIZING_CTRL_LOW__SHIFT                                               0x3
+#define UTCL1_FIFO_SIZING__UTCL1_GENERAL_SIZING_CTRL_HIGH__SHIFT                                              0x10
+#define UTCL1_FIFO_SIZING__UTCL1_UTCL2_INVACK_CDC_FIFO_THRESH_MASK                                            0x00000007L
+#define UTCL1_FIFO_SIZING__UTCL1_GENERAL_SIZING_CTRL_LOW_MASK                                                 0x0000FFF8L
+#define UTCL1_FIFO_SIZING__UTCL1_GENERAL_SIZING_CTRL_HIGH_MASK                                                0xFFFF0000L
+//GCRD_SA0_TARGETS_DISABLE
+#define GCRD_SA0_TARGETS_DISABLE__GCRD_SA0_TARGETS_DISABLE__SHIFT                                             0x0
+#define GCRD_SA0_TARGETS_DISABLE__GCRD_SA0_TARGETS_DISABLE_MASK                                               0x0007FFFFL
+//GCRD_SA1_TARGETS_DISABLE
+#define GCRD_SA1_TARGETS_DISABLE__GCRD_SA1_TARGETS_DISABLE__SHIFT                                             0x0
+#define GCRD_SA1_TARGETS_DISABLE__GCRD_SA1_TARGETS_DISABLE_MASK                                               0x0007FFFFL
+//GCRD_CREDIT_SAFE
+#define GCRD_CREDIT_SAFE__GCRD_CHAIN_CREDIT_SAFE_REG__SHIFT                                                   0x0
+#define GCRD_CREDIT_SAFE__GCRD_TARGET_CREDIT_SAFE_REG__SHIFT                                                  0x4
+#define GCRD_CREDIT_SAFE__GCRD_CHAIN_CREDIT_SAFE_REG_MASK                                                     0x00000007L
+#define GCRD_CREDIT_SAFE__GCRD_TARGET_CREDIT_SAFE_REG_MASK                                                    0x00000070L
+
+
+// addressBlock: gc_pfonly_pmmdec
+//GCR_GENERAL_CNTL
+#define GCR_GENERAL_CNTL__FORCE_4K_L2_RESP__SHIFT                                                             0x0
+#define GCR_GENERAL_CNTL__REDUCE_HALF_MAIN_WQ__SHIFT                                                          0x1
+#define GCR_GENERAL_CNTL__REDUCE_HALF_PHY_WQ__SHIFT                                                           0x2
+#define GCR_GENERAL_CNTL__FORCE_INV_ALL__SHIFT                                                                0x3
+#define GCR_GENERAL_CNTL__HI_PRIORITY_CNTL__SHIFT                                                             0x4
+#define GCR_GENERAL_CNTL__HI_PRIORITY_DISABLE__SHIFT                                                          0x6
+#define GCR_GENERAL_CNTL__BIG_PAGE_FILTER_DISABLE__SHIFT                                                      0x7
+#define GCR_GENERAL_CNTL__PERF_CNTR_ENABLE__SHIFT                                                             0x8
+#define GCR_GENERAL_CNTL__FORCE_SINGLE_WQ__SHIFT                                                              0x9
+#define GCR_GENERAL_CNTL__UTCL2_REQ_PERM__SHIFT                                                               0xa
+#define GCR_GENERAL_CNTL__TARGET_MGCG_CLKEN_DIS__SHIFT                                                        0xd
+#define GCR_GENERAL_CNTL__MIXED_RANGE_MODE_DIS__SHIFT                                                         0xe
+#define GCR_GENERAL_CNTL__ENABLE_16K_UTCL2_REQ__SHIFT                                                         0xf
+#define GCR_GENERAL_CNTL__DISABLE_FGCG__SHIFT                                                                 0x10
+#define GCR_GENERAL_CNTL__CLIENT_ID__SHIFT                                                                    0x14
+#define GCR_GENERAL_CNTL__FORCE_4K_L2_RESP_MASK                                                               0x00000001L
+#define GCR_GENERAL_CNTL__REDUCE_HALF_MAIN_WQ_MASK                                                            0x00000002L
+#define GCR_GENERAL_CNTL__REDUCE_HALF_PHY_WQ_MASK                                                             0x00000004L
+#define GCR_GENERAL_CNTL__FORCE_INV_ALL_MASK                                                                  0x00000008L
+#define GCR_GENERAL_CNTL__HI_PRIORITY_CNTL_MASK                                                               0x00000030L
+#define GCR_GENERAL_CNTL__HI_PRIORITY_DISABLE_MASK                                                            0x00000040L
+#define GCR_GENERAL_CNTL__BIG_PAGE_FILTER_DISABLE_MASK                                                        0x00000080L
+#define GCR_GENERAL_CNTL__PERF_CNTR_ENABLE_MASK                                                               0x00000100L
+#define GCR_GENERAL_CNTL__FORCE_SINGLE_WQ_MASK                                                                0x00000200L
+#define GCR_GENERAL_CNTL__UTCL2_REQ_PERM_MASK                                                                 0x00001C00L
+#define GCR_GENERAL_CNTL__TARGET_MGCG_CLKEN_DIS_MASK                                                          0x00002000L
+#define GCR_GENERAL_CNTL__MIXED_RANGE_MODE_DIS_MASK                                                           0x00004000L
+#define GCR_GENERAL_CNTL__ENABLE_16K_UTCL2_REQ_MASK                                                           0x00008000L
+#define GCR_GENERAL_CNTL__DISABLE_FGCG_MASK                                                                   0x00010000L
+#define GCR_GENERAL_CNTL__CLIENT_ID_MASK                                                                      0x1FF00000L
+//GCR_TARGET_DISABLE
+#define GCR_TARGET_DISABLE__DISABLE_SE0_PHY__SHIFT                                                            0x0
+#define GCR_TARGET_DISABLE__DISABLE_SE0_VIRT__SHIFT                                                           0x1
+#define GCR_TARGET_DISABLE__DISABLE_SE1_PHY__SHIFT                                                            0x2
+#define GCR_TARGET_DISABLE__DISABLE_SE1_VIRT__SHIFT                                                           0x3
+#define GCR_TARGET_DISABLE__DISABLE_SE2_PHY__SHIFT                                                            0x4
+#define GCR_TARGET_DISABLE__DISABLE_SE2_VIRT__SHIFT                                                           0x5
+#define GCR_TARGET_DISABLE__DISABLE_GL2A0_PHY__SHIFT                                                          0x6
+#define GCR_TARGET_DISABLE__DISABLE_GL2A1_PHY__SHIFT                                                          0x7
+#define GCR_TARGET_DISABLE__DISABLE_GL2A2_PHY__SHIFT                                                          0x8
+#define GCR_TARGET_DISABLE__DISABLE_GL2A3_PHY__SHIFT                                                          0x9
+#define GCR_TARGET_DISABLE__DISABLE_SE3_PHY__SHIFT                                                            0xa
+#define GCR_TARGET_DISABLE__DISABLE_SE3_VIRT__SHIFT                                                           0xb
+#define GCR_TARGET_DISABLE__DISABLE_SE4_PHY__SHIFT                                                            0xc
+#define GCR_TARGET_DISABLE__DISABLE_SE4_VIRT__SHIFT                                                           0xd
+#define GCR_TARGET_DISABLE__DISABLE_SE5_PHY__SHIFT                                                            0xe
+#define GCR_TARGET_DISABLE__DISABLE_SE5_VIRT__SHIFT                                                           0xf
+#define GCR_TARGET_DISABLE__SE0_INACTIVE_STATUS__SHIFT                                                        0x10
+#define GCR_TARGET_DISABLE__SE1_INACTIVE_STATUS__SHIFT                                                        0x11
+#define GCR_TARGET_DISABLE__SE2_INACTIVE_STATUS__SHIFT                                                        0x12
+#define GCR_TARGET_DISABLE__SE3_INACTIVE_STATUS__SHIFT                                                        0x13
+#define GCR_TARGET_DISABLE__SE4_INACTIVE_STATUS__SHIFT                                                        0x14
+#define GCR_TARGET_DISABLE__SE5_INACTIVE_STATUS__SHIFT                                                        0x15
+#define GCR_TARGET_DISABLE__DISABLE_SE0_PHY_MASK                                                              0x00000001L
+#define GCR_TARGET_DISABLE__DISABLE_SE0_VIRT_MASK                                                             0x00000002L
+#define GCR_TARGET_DISABLE__DISABLE_SE1_PHY_MASK                                                              0x00000004L
+#define GCR_TARGET_DISABLE__DISABLE_SE1_VIRT_MASK                                                             0x00000008L
+#define GCR_TARGET_DISABLE__DISABLE_SE2_PHY_MASK                                                              0x00000010L
+#define GCR_TARGET_DISABLE__DISABLE_SE2_VIRT_MASK                                                             0x00000020L
+#define GCR_TARGET_DISABLE__DISABLE_GL2A0_PHY_MASK                                                            0x00000040L
+#define GCR_TARGET_DISABLE__DISABLE_GL2A1_PHY_MASK                                                            0x00000080L
+#define GCR_TARGET_DISABLE__DISABLE_GL2A2_PHY_MASK                                                            0x00000100L
+#define GCR_TARGET_DISABLE__DISABLE_GL2A3_PHY_MASK                                                            0x00000200L
+#define GCR_TARGET_DISABLE__DISABLE_SE3_PHY_MASK                                                              0x00000400L
+#define GCR_TARGET_DISABLE__DISABLE_SE3_VIRT_MASK                                                             0x00000800L
+#define GCR_TARGET_DISABLE__DISABLE_SE4_PHY_MASK                                                              0x00001000L
+#define GCR_TARGET_DISABLE__DISABLE_SE4_VIRT_MASK                                                             0x00002000L
+#define GCR_TARGET_DISABLE__DISABLE_SE5_PHY_MASK                                                              0x00004000L
+#define GCR_TARGET_DISABLE__DISABLE_SE5_VIRT_MASK                                                             0x00008000L
+#define GCR_TARGET_DISABLE__SE0_INACTIVE_STATUS_MASK                                                          0x00010000L
+#define GCR_TARGET_DISABLE__SE1_INACTIVE_STATUS_MASK                                                          0x00020000L
+#define GCR_TARGET_DISABLE__SE2_INACTIVE_STATUS_MASK                                                          0x00040000L
+#define GCR_TARGET_DISABLE__SE3_INACTIVE_STATUS_MASK                                                          0x00080000L
+#define GCR_TARGET_DISABLE__SE4_INACTIVE_STATUS_MASK                                                          0x00100000L
+#define GCR_TARGET_DISABLE__SE5_INACTIVE_STATUS_MASK                                                          0x00200000L
+//GCR_CMD_STATUS
+#define GCR_CMD_STATUS__GCR_CONTROL__SHIFT                                                                    0x0
+#define GCR_CMD_STATUS__GCR_SRC__SHIFT                                                                        0x13
+#define GCR_CMD_STATUS__GCR_TLB_SHOOTDOWN__SHIFT                                                              0x17
+#define GCR_CMD_STATUS__GCR_TLB_SHOOTDOWN_VMID__SHIFT                                                         0x18
+#define GCR_CMD_STATUS__UTCL2_NACK_STATUS__SHIFT                                                              0x1c
+#define GCR_CMD_STATUS__GCR_SEQ_OP_ERROR__SHIFT                                                               0x1e
+#define GCR_CMD_STATUS__UTCL2_NACK_ERROR__SHIFT                                                               0x1f
+#define GCR_CMD_STATUS__GCR_CONTROL_MASK                                                                      0x0007FFFFL
+#define GCR_CMD_STATUS__GCR_SRC_MASK                                                                          0x00380000L
+#define GCR_CMD_STATUS__GCR_TLB_SHOOTDOWN_MASK                                                                0x00800000L
+#define GCR_CMD_STATUS__GCR_TLB_SHOOTDOWN_VMID_MASK                                                           0x0F000000L
+#define GCR_CMD_STATUS__UTCL2_NACK_STATUS_MASK                                                                0x30000000L
+#define GCR_CMD_STATUS__GCR_SEQ_OP_ERROR_MASK                                                                 0x40000000L
+#define GCR_CMD_STATUS__UTCL2_NACK_ERROR_MASK                                                                 0x80000000L
+//GCR_SPARE
+#define GCR_SPARE__SPARE_BIT_1__SHIFT                                                                         0x1
+#define GCR_SPARE__SPARE_BIT_2__SHIFT                                                                         0x2
+#define GCR_SPARE__SPARE_BIT_3__SHIFT                                                                         0x3
+#define GCR_SPARE__SPARE_BIT_4__SHIFT                                                                         0x4
+#define GCR_SPARE__SPARE_BIT_5__SHIFT                                                                         0x5
+#define GCR_SPARE__SPARE_BIT_6__SHIFT                                                                         0x6
+#define GCR_SPARE__SPARE_BIT_7__SHIFT                                                                         0x7
+#define GCR_SPARE__UTCL2_REQ_CREDIT__SHIFT                                                                    0x8
+#define GCR_SPARE__GCRD_GL2A_REQ_CREDIT__SHIFT                                                                0x10
+#define GCR_SPARE__GCRD_SE_REQ_CREDIT__SHIFT                                                                  0x14
+#define GCR_SPARE__SPARE_BIT_31_24__SHIFT                                                                     0x18
+#define GCR_SPARE__SPARE_BIT_1_MASK                                                                           0x00000002L
+#define GCR_SPARE__SPARE_BIT_2_MASK                                                                           0x00000004L
+#define GCR_SPARE__SPARE_BIT_3_MASK                                                                           0x00000008L
+#define GCR_SPARE__SPARE_BIT_4_MASK                                                                           0x00000010L
+#define GCR_SPARE__SPARE_BIT_5_MASK                                                                           0x00000020L
+#define GCR_SPARE__SPARE_BIT_6_MASK                                                                           0x00000040L
+#define GCR_SPARE__SPARE_BIT_7_MASK                                                                           0x00000080L
+#define GCR_SPARE__UTCL2_REQ_CREDIT_MASK                                                                      0x0000FF00L
+#define GCR_SPARE__GCRD_GL2A_REQ_CREDIT_MASK                                                                  0x000F0000L
+#define GCR_SPARE__GCRD_SE_REQ_CREDIT_MASK                                                                    0x00F00000L
+#define GCR_SPARE__SPARE_BIT_31_24_MASK                                                                       0xFF000000L
+//PMM_CNTL2
+#define PMM_CNTL2__GCEA_MAM_DISABLE__SHIFT                                                                    0x0
+#define PMM_CNTL2__ABIT_FORCE_FLUSH_OVERRIDE__SHIFT                                                           0x18
+#define PMM_CNTL2__ABIT_TIMER_FLUSH_OVERRIDE__SHIFT                                                           0x19
+#define PMM_CNTL2__PMM_IH_INTERRUPT_CREDITS_OVERRIDE__SHIFT                                                   0x1a
+#define PMM_CNTL2__ABIT_INTR_ON_FLUSH_DONE__SHIFT                                                             0x1e
+#define PMM_CNTL2__RESERVED__SHIFT                                                                            0x1f
+#define PMM_CNTL2__GCEA_MAM_DISABLE_MASK                                                                      0x00FFFFFFL
+#define PMM_CNTL2__ABIT_FORCE_FLUSH_OVERRIDE_MASK                                                             0x01000000L
+#define PMM_CNTL2__ABIT_TIMER_FLUSH_OVERRIDE_MASK                                                             0x02000000L
+#define PMM_CNTL2__PMM_IH_INTERRUPT_CREDITS_OVERRIDE_MASK                                                     0x3C000000L
+#define PMM_CNTL2__ABIT_INTR_ON_FLUSH_DONE_MASK                                                               0x40000000L
+#define PMM_CNTL2__RESERVED_MASK                                                                              0x80000000L
+
+
+// addressBlock: gc_pfonly_gccacdec
+//GC_CAC_CTRL_1
+#define GC_CAC_CTRL_1__CAC_WINDOW__SHIFT                                                                      0x0
+#define GC_CAC_CTRL_1__TDP_WINDOW__SHIFT                                                                      0x8
+#define GC_CAC_CTRL_1__CAC_WINDOW_MASK                                                                        0x000000FFL
+#define GC_CAC_CTRL_1__TDP_WINDOW_MASK                                                                        0xFFFFFF00L
+//GC_CAC_CTRL_2
+#define GC_CAC_CTRL_2__CAC_ENABLE__SHIFT                                                                      0x0
+#define GC_CAC_CTRL_2__GC_LCAC_ENABLE__SHIFT                                                                  0x1
+#define GC_CAC_CTRL_2__GC_CAC_INDEX_AUTO_INCR_EN__SHIFT                                                       0x2
+#define GC_CAC_CTRL_2__TOGGLE_EN__SHIFT                                                                       0x3
+#define GC_CAC_CTRL_2__INTR_EN__SHIFT                                                                         0x4
+#define GC_CAC_CTRL_2__CAC_COUNTER_SNAP_SEL__SHIFT                                                            0x5
+#define GC_CAC_CTRL_2__SE_AGGR_ACC_EN__SHIFT                                                                  0x6
+#define GC_CAC_CTRL_2__GC_AGGR_ACC_EN__SHIFT                                                                  0xe
+#define GC_CAC_CTRL_2__CAC_ENABLE_MASK                                                                        0x00000001L
+#define GC_CAC_CTRL_2__GC_LCAC_ENABLE_MASK                                                                    0x00000002L
+#define GC_CAC_CTRL_2__GC_CAC_INDEX_AUTO_INCR_EN_MASK                                                         0x00000004L
+#define GC_CAC_CTRL_2__TOGGLE_EN_MASK                                                                         0x00000008L
+#define GC_CAC_CTRL_2__INTR_EN_MASK                                                                           0x00000010L
+#define GC_CAC_CTRL_2__CAC_COUNTER_SNAP_SEL_MASK                                                              0x00000020L
+#define GC_CAC_CTRL_2__SE_AGGR_ACC_EN_MASK                                                                    0x00003FC0L
+#define GC_CAC_CTRL_2__GC_AGGR_ACC_EN_MASK                                                                    0x00004000L
+//GC_CAC_AGGR_LOWER
+#define GC_CAC_AGGR_LOWER__GC_AGGR_31_0__SHIFT                                                                0x0
+#define GC_CAC_AGGR_LOWER__GC_AGGR_31_0_MASK                                                                  0xFFFFFFFFL
+//GC_CAC_AGGR_UPPER
+#define GC_CAC_AGGR_UPPER__GC_AGGR_63_32__SHIFT                                                               0x0
+#define GC_CAC_AGGR_UPPER__GC_AGGR_63_32_MASK                                                                 0xFFFFFFFFL
+//SE0_CAC_AGGR_LOWER
+#define SE0_CAC_AGGR_LOWER__SE0_AGGR_31_0__SHIFT                                                              0x0
+#define SE0_CAC_AGGR_LOWER__SE0_AGGR_31_0_MASK                                                                0xFFFFFFFFL
+//SE0_CAC_AGGR_UPPER
+#define SE0_CAC_AGGR_UPPER__SE0_AGGR_63_32__SHIFT                                                             0x0
+#define SE0_CAC_AGGR_UPPER__SE0_AGGR_63_32_MASK                                                               0xFFFFFFFFL
+//SE1_CAC_AGGR_LOWER
+#define SE1_CAC_AGGR_LOWER__SE1_AGGR_31_0__SHIFT                                                              0x0
+#define SE1_CAC_AGGR_LOWER__SE1_AGGR_31_0_MASK                                                                0xFFFFFFFFL
+//SE1_CAC_AGGR_UPPER
+#define SE1_CAC_AGGR_UPPER__SE1_AGGR_63_32__SHIFT                                                             0x0
+#define SE1_CAC_AGGR_UPPER__SE1_AGGR_63_32_MASK                                                               0xFFFFFFFFL
+//SE2_CAC_AGGR_LOWER
+#define SE2_CAC_AGGR_LOWER__SE2_AGGR_31_0__SHIFT                                                              0x0
+#define SE2_CAC_AGGR_LOWER__SE2_AGGR_31_0_MASK                                                                0xFFFFFFFFL
+//SE2_CAC_AGGR_UPPER
+#define SE2_CAC_AGGR_UPPER__SE2_AGGR_63_32__SHIFT                                                             0x0
+#define SE2_CAC_AGGR_UPPER__SE2_AGGR_63_32_MASK                                                               0xFFFFFFFFL
+//GC_CAC_AGGR_GFXCLK_CYCLE
+#define GC_CAC_AGGR_GFXCLK_CYCLE__GC_AGGR_GFXCLK_CYCLE__SHIFT                                                 0x0
+#define GC_CAC_AGGR_GFXCLK_CYCLE__GC_AGGR_GFXCLK_CYCLE_MASK                                                   0xFFFFFFFFL
+//SE0_CAC_AGGR_GFXCLK_CYCLE
+#define SE0_CAC_AGGR_GFXCLK_CYCLE__SE0_AGGR_GFXCLK_CYCLE__SHIFT                                               0x0
+#define SE0_CAC_AGGR_GFXCLK_CYCLE__SE0_AGGR_GFXCLK_CYCLE_MASK                                                 0xFFFFFFFFL
+//SE1_CAC_AGGR_GFXCLK_CYCLE
+#define SE1_CAC_AGGR_GFXCLK_CYCLE__SE1_AGGR_GFXCLK_CYCLE__SHIFT                                               0x0
+#define SE1_CAC_AGGR_GFXCLK_CYCLE__SE1_AGGR_GFXCLK_CYCLE_MASK                                                 0xFFFFFFFFL
+//SE2_CAC_AGGR_GFXCLK_CYCLE
+#define SE2_CAC_AGGR_GFXCLK_CYCLE__SE2_AGGR_GFXCLK_CYCLE__SHIFT                                               0x0
+#define SE2_CAC_AGGR_GFXCLK_CYCLE__SE2_AGGR_GFXCLK_CYCLE_MASK                                                 0xFFFFFFFFL
+//GC_EDC_CTRL
+#define GC_EDC_CTRL__EDC_EN__SHIFT                                                                            0x0
+#define GC_EDC_CTRL__EDC_SW_RST__SHIFT                                                                        0x1
+#define GC_EDC_CTRL__EDC_CLK_EN_OVERRIDE__SHIFT                                                               0x2
+#define GC_EDC_CTRL__EDC_FORCE_STALL__SHIFT                                                                   0x3
+#define GC_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT__SHIFT                                                       0x4
+#define GC_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA__SHIFT                                                          0xa
+#define GC_EDC_CTRL__EDC_THROTTLE_PATTERN_BIT_NUMS__SHIFT                                                     0xb
+#define GC_EDC_CTRL__EDC_LEVEL_SEL__SHIFT                                                                     0xf
+#define GC_EDC_CTRL__EDC_ALGORITHM_MODE__SHIFT                                                                0x10
+#define GC_EDC_CTRL__EDC_AVGDIV__SHIFT                                                                        0x11
+#define GC_EDC_CTRL__PSM_THROTTLE_SRC_SEL__SHIFT                                                              0x15
+#define GC_EDC_CTRL__THROTTLE_SRC0_MASK__SHIFT                                                                0x18
+#define GC_EDC_CTRL__THROTTLE_SRC1_MASK__SHIFT                                                                0x19
+#define GC_EDC_CTRL__THROTTLE_SRC2_MASK__SHIFT                                                                0x1a
+#define GC_EDC_CTRL__THROTTLE_SRC3_MASK__SHIFT                                                                0x1b
+#define GC_EDC_CTRL__EDC_CREDIT_SHIFT_BIT_NUMS__SHIFT                                                         0x1c
+#define GC_EDC_CTRL__EDC_EN_MASK                                                                              0x00000001L
+#define GC_EDC_CTRL__EDC_SW_RST_MASK                                                                          0x00000002L
+#define GC_EDC_CTRL__EDC_CLK_EN_OVERRIDE_MASK                                                                 0x00000004L
+#define GC_EDC_CTRL__EDC_FORCE_STALL_MASK                                                                     0x00000008L
+#define GC_EDC_CTRL__EDC_TRIGGER_THROTTLE_LOWBIT_MASK                                                         0x000003F0L
+#define GC_EDC_CTRL__EDC_ALLOW_WRITE_PWRDELTA_MASK                                                            0x00000400L
+#define GC_EDC_CTRL__EDC_THROTTLE_PATTERN_BIT_NUMS_MASK                                                       0x00007800L
+#define GC_EDC_CTRL__EDC_LEVEL_SEL_MASK                                                                       0x00008000L
+#define GC_EDC_CTRL__EDC_ALGORITHM_MODE_MASK                                                                  0x00010000L
+#define GC_EDC_CTRL__EDC_AVGDIV_MASK                                                                          0x001E0000L
+#define GC_EDC_CTRL__PSM_THROTTLE_SRC_SEL_MASK                                                                0x00E00000L
+#define GC_EDC_CTRL__THROTTLE_SRC0_MASK_MASK                                                                  0x01000000L
+#define GC_EDC_CTRL__THROTTLE_SRC1_MASK_MASK                                                                  0x02000000L
+#define GC_EDC_CTRL__THROTTLE_SRC2_MASK_MASK                                                                  0x04000000L
+#define GC_EDC_CTRL__THROTTLE_SRC3_MASK_MASK                                                                  0x08000000L
+#define GC_EDC_CTRL__EDC_CREDIT_SHIFT_BIT_NUMS_MASK                                                           0xF0000000L
+//GC_EDC_THRESHOLD
+#define GC_EDC_THRESHOLD__EDC_THRESHOLD__SHIFT                                                                0x0
+#define GC_EDC_THRESHOLD__EDC_THRESHOLD_MASK                                                                  0xFFFFFFFFL
+//GC_EDC_STRETCH_CTRL
+#define GC_EDC_STRETCH_CTRL__EDC_STRETCH_EN__SHIFT                                                            0x0
+#define GC_EDC_STRETCH_CTRL__EDC_STRETCH_DELAY__SHIFT                                                         0x1
+#define GC_EDC_STRETCH_CTRL__EDC_UNSTRETCH_DELAY__SHIFT                                                       0xa
+#define GC_EDC_STRETCH_CTRL__EDC_STRETCH_EN_MASK                                                              0x00000001L
+#define GC_EDC_STRETCH_CTRL__EDC_STRETCH_DELAY_MASK                                                           0x000003FEL
+#define GC_EDC_STRETCH_CTRL__EDC_UNSTRETCH_DELAY_MASK                                                         0x0007FC00L
+//GC_EDC_STRETCH_THRESHOLD
+#define GC_EDC_STRETCH_THRESHOLD__EDC_STRETCH_THRESHOLD__SHIFT                                                0x0
+#define GC_EDC_STRETCH_THRESHOLD__EDC_STRETCH_THRESHOLD_MASK                                                  0xFFFFFFFFL
+//EDC_HYSTERESIS_CNTL
+#define EDC_HYSTERESIS_CNTL__MAX_HYSTERESIS__SHIFT                                                            0x0
+#define EDC_HYSTERESIS_CNTL__EDC_AGGR_TIMER__SHIFT                                                            0x8
+#define EDC_HYSTERESIS_CNTL__PATTERN_EXTEND_EN__SHIFT                                                         0x10
+#define EDC_HYSTERESIS_CNTL__PATTERN_EXTEND_MODE__SHIFT                                                       0x11
+#define EDC_HYSTERESIS_CNTL__EDC_AGGR_MODE__SHIFT                                                             0x14
+#define EDC_HYSTERESIS_CNTL__MAX_HYSTERESIS_MASK                                                              0x000000FFL
+#define EDC_HYSTERESIS_CNTL__EDC_AGGR_TIMER_MASK                                                              0x0000FF00L
+#define EDC_HYSTERESIS_CNTL__PATTERN_EXTEND_EN_MASK                                                           0x00010000L
+#define EDC_HYSTERESIS_CNTL__PATTERN_EXTEND_MODE_MASK                                                         0x000E0000L
+#define EDC_HYSTERESIS_CNTL__EDC_AGGR_MODE_MASK                                                               0x00100000L
+//GC_THROTTLE_CTRL
+#define GC_THROTTLE_CTRL__THROTTLE_CTRL_SW_RST__SHIFT                                                         0x0
+#define GC_THROTTLE_CTRL__GC_EDC_STALL_EN__SHIFT                                                              0x1
+#define GC_THROTTLE_CTRL__PWRBRK_STALL_EN__SHIFT                                                              0x2
+#define GC_THROTTLE_CTRL__PWRBRK_POLARITY_CNTL__SHIFT                                                         0x3
+#define GC_THROTTLE_CTRL__PCC_STALL_EN__SHIFT                                                                 0x4
+#define GC_THROTTLE_CTRL__PATTERN_MODE__SHIFT                                                                 0x5
+#define GC_THROTTLE_CTRL__GC_EDC_ONLY_MODE__SHIFT                                                             0x6
+#define GC_THROTTLE_CTRL__GC_EDC_OVERRIDE__SHIFT                                                              0x7
+#define GC_THROTTLE_CTRL__PCC_OVERRIDE__SHIFT                                                                 0x8
+#define GC_THROTTLE_CTRL__PWRBRK_OVERRIDE__SHIFT                                                              0x9
+#define GC_THROTTLE_CTRL__GC_EDC_PERF_COUNTER_EN__SHIFT                                                       0xa
+#define GC_THROTTLE_CTRL__PCC_PERF_COUNTER_EN__SHIFT                                                          0xb
+#define GC_THROTTLE_CTRL__PWRBRK_PERF_COUNTER_EN__SHIFT                                                       0xc
+#define GC_THROTTLE_CTRL__RELEASE_STEP_INTERVAL__SHIFT                                                        0xd
+#define GC_THROTTLE_CTRL__FIXED_PATTERN_PERF_COUNTER_EN__SHIFT                                                0x17
+#define GC_THROTTLE_CTRL__FIXED_PATTERN_LOG_INDEX__SHIFT                                                      0x18
+#define GC_THROTTLE_CTRL__LUT_HW_UPDATE__SHIFT                                                                0x1d
+#define GC_THROTTLE_CTRL__THROTTLE_CTRL_CLK_EN_OVERRIDE__SHIFT                                                0x1e
+#define GC_THROTTLE_CTRL__PCC_POLARITY_CNTL__SHIFT                                                            0x1f
+#define GC_THROTTLE_CTRL__THROTTLE_CTRL_SW_RST_MASK                                                           0x00000001L
+#define GC_THROTTLE_CTRL__GC_EDC_STALL_EN_MASK                                                                0x00000002L
+#define GC_THROTTLE_CTRL__PWRBRK_STALL_EN_MASK                                                                0x00000004L
+#define GC_THROTTLE_CTRL__PWRBRK_POLARITY_CNTL_MASK                                                           0x00000008L
+#define GC_THROTTLE_CTRL__PCC_STALL_EN_MASK                                                                   0x00000010L
+#define GC_THROTTLE_CTRL__PATTERN_MODE_MASK                                                                   0x00000020L
+#define GC_THROTTLE_CTRL__GC_EDC_ONLY_MODE_MASK                                                               0x00000040L
+#define GC_THROTTLE_CTRL__GC_EDC_OVERRIDE_MASK                                                                0x00000080L
+#define GC_THROTTLE_CTRL__PCC_OVERRIDE_MASK                                                                   0x00000100L
+#define GC_THROTTLE_CTRL__PWRBRK_OVERRIDE_MASK                                                                0x00000200L
+#define GC_THROTTLE_CTRL__GC_EDC_PERF_COUNTER_EN_MASK                                                         0x00000400L
+#define GC_THROTTLE_CTRL__PCC_PERF_COUNTER_EN_MASK                                                            0x00000800L
+#define GC_THROTTLE_CTRL__PWRBRK_PERF_COUNTER_EN_MASK                                                         0x00001000L
+#define GC_THROTTLE_CTRL__RELEASE_STEP_INTERVAL_MASK                                                          0x007FE000L
+#define GC_THROTTLE_CTRL__FIXED_PATTERN_PERF_COUNTER_EN_MASK                                                  0x00800000L
+#define GC_THROTTLE_CTRL__FIXED_PATTERN_LOG_INDEX_MASK                                                        0x1F000000L
+#define GC_THROTTLE_CTRL__LUT_HW_UPDATE_MASK                                                                  0x20000000L
+#define GC_THROTTLE_CTRL__THROTTLE_CTRL_CLK_EN_OVERRIDE_MASK                                                  0x40000000L
+#define GC_THROTTLE_CTRL__PCC_POLARITY_CNTL_MASK                                                              0x80000000L
+//GC_THROTTLE_CTRL1
+#define GC_THROTTLE_CTRL1__PCC_FP_PROGRAM_STEP_EN__SHIFT                                                      0x0
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_MIN_STEP__SHIFT                                                        0x1
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_MAX_STEP__SHIFT                                                        0x5
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_UPWARDS_STEP_SIZE__SHIFT                                               0xa
+#define GC_THROTTLE_CTRL1__PWRBRK_FP_PROGRAM_STEP_EN__SHIFT                                                   0xd
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_MIN_STEP__SHIFT                                                     0xe
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_MAX_STEP__SHIFT                                                     0x12
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_UPWARDS_STEP_SIZE__SHIFT                                            0x17
+#define GC_THROTTLE_CTRL1__FIXED_PATTERN_SELECT__SHIFT                                                        0x1a
+#define GC_THROTTLE_CTRL1__GC_EDC_STRETCH_PERF_COUNTER_EN__SHIFT                                              0x1e
+#define GC_THROTTLE_CTRL1__GC_EDC_UNSTRETCH_PERF_COUNTER_EN__SHIFT                                            0x1f
+#define GC_THROTTLE_CTRL1__PCC_FP_PROGRAM_STEP_EN_MASK                                                        0x00000001L
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_MIN_STEP_MASK                                                          0x0000001EL
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_MAX_STEP_MASK                                                          0x000003E0L
+#define GC_THROTTLE_CTRL1__PCC_PROGRAM_UPWARDS_STEP_SIZE_MASK                                                 0x00001C00L
+#define GC_THROTTLE_CTRL1__PWRBRK_FP_PROGRAM_STEP_EN_MASK                                                     0x00002000L
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_MIN_STEP_MASK                                                       0x0003C000L
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_MAX_STEP_MASK                                                       0x007C0000L
+#define GC_THROTTLE_CTRL1__PWRBRK_PROGRAM_UPWARDS_STEP_SIZE_MASK                                              0x03800000L
+#define GC_THROTTLE_CTRL1__FIXED_PATTERN_SELECT_MASK                                                          0x0C000000L
+#define GC_THROTTLE_CTRL1__GC_EDC_STRETCH_PERF_COUNTER_EN_MASK                                                0x40000000L
+#define GC_THROTTLE_CTRL1__GC_EDC_UNSTRETCH_PERF_COUNTER_EN_MASK                                              0x80000000L
+//PCC_STALL_PATTERN_CTRL
+#define PCC_STALL_PATTERN_CTRL__PCC_STEP_INTERVAL__SHIFT                                                      0x0
+#define PCC_STALL_PATTERN_CTRL__PCC_BEGIN_STEP__SHIFT                                                         0xa
+#define PCC_STALL_PATTERN_CTRL__PCC_END_STEP__SHIFT                                                           0xf
+#define PCC_STALL_PATTERN_CTRL__PCC_THROTTLE_PATTERN_BIT_NUMS__SHIFT                                          0x14
+#define PCC_STALL_PATTERN_CTRL__PCC_INST_THROT_INCR__SHIFT                                                    0x18
+#define PCC_STALL_PATTERN_CTRL__PCC_INST_THROT_DECR__SHIFT                                                    0x19
+#define PCC_STALL_PATTERN_CTRL__PCC_DITHER_MODE__SHIFT                                                        0x1a
+#define PCC_STALL_PATTERN_CTRL__PCC_STEP_INTERVAL_MASK                                                        0x000003FFL
+#define PCC_STALL_PATTERN_CTRL__PCC_BEGIN_STEP_MASK                                                           0x00007C00L
+#define PCC_STALL_PATTERN_CTRL__PCC_END_STEP_MASK                                                             0x000F8000L
+#define PCC_STALL_PATTERN_CTRL__PCC_THROTTLE_PATTERN_BIT_NUMS_MASK                                            0x00F00000L
+#define PCC_STALL_PATTERN_CTRL__PCC_INST_THROT_INCR_MASK                                                      0x01000000L
+#define PCC_STALL_PATTERN_CTRL__PCC_INST_THROT_DECR_MASK                                                      0x02000000L
+#define PCC_STALL_PATTERN_CTRL__PCC_DITHER_MODE_MASK                                                          0x04000000L
+//PWRBRK_STALL_PATTERN_CTRL
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_STEP_INTERVAL__SHIFT                                                0x0
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_BEGIN_STEP__SHIFT                                                   0xa
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_END_STEP__SHIFT                                                     0xf
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_THROTTLE_PATTERN_BIT_NUMS__SHIFT                                    0x14
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_STEP_INTERVAL_MASK                                                  0x000003FFL
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_BEGIN_STEP_MASK                                                     0x00007C00L
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_END_STEP_MASK                                                       0x000F8000L
+#define PWRBRK_STALL_PATTERN_CTRL__PWRBRK_THROTTLE_PATTERN_BIT_NUMS_MASK                                      0x00F00000L
+//PCC_STALL_PATTERN_1_2
+#define PCC_STALL_PATTERN_1_2__PCC_STALL_PATTERN_1__SHIFT                                                     0x0
+#define PCC_STALL_PATTERN_1_2__PCC_STALL_PATTERN_2__SHIFT                                                     0x10
+#define PCC_STALL_PATTERN_1_2__PCC_STALL_PATTERN_1_MASK                                                       0x00007FFFL
+#define PCC_STALL_PATTERN_1_2__PCC_STALL_PATTERN_2_MASK                                                       0x7FFF0000L
+//PCC_STALL_PATTERN_3_4
+#define PCC_STALL_PATTERN_3_4__PCC_STALL_PATTERN_3__SHIFT                                                     0x0
+#define PCC_STALL_PATTERN_3_4__PCC_STALL_PATTERN_4__SHIFT                                                     0x10
+#define PCC_STALL_PATTERN_3_4__PCC_STALL_PATTERN_3_MASK                                                       0x00007FFFL
+#define PCC_STALL_PATTERN_3_4__PCC_STALL_PATTERN_4_MASK                                                       0x7FFF0000L
+//PCC_STALL_PATTERN_5_6
+#define PCC_STALL_PATTERN_5_6__PCC_STALL_PATTERN_5__SHIFT                                                     0x0
+#define PCC_STALL_PATTERN_5_6__PCC_STALL_PATTERN_6__SHIFT                                                     0x10
+#define PCC_STALL_PATTERN_5_6__PCC_STALL_PATTERN_5_MASK                                                       0x00007FFFL
+#define PCC_STALL_PATTERN_5_6__PCC_STALL_PATTERN_6_MASK                                                       0x7FFF0000L
+//PCC_STALL_PATTERN_7
+#define PCC_STALL_PATTERN_7__PCC_STALL_PATTERN_7__SHIFT                                                       0x0
+#define PCC_STALL_PATTERN_7__PCC_STALL_PATTERN_7_MASK                                                         0x00007FFFL
+//PWRBRK_STALL_PATTERN_1_2
+#define PWRBRK_STALL_PATTERN_1_2__PWRBRK_STALL_PATTERN_1__SHIFT                                               0x0
+#define PWRBRK_STALL_PATTERN_1_2__PWRBRK_STALL_PATTERN_2__SHIFT                                               0x10
+#define PWRBRK_STALL_PATTERN_1_2__PWRBRK_STALL_PATTERN_1_MASK                                                 0x00007FFFL
+#define PWRBRK_STALL_PATTERN_1_2__PWRBRK_STALL_PATTERN_2_MASK                                                 0x7FFF0000L
+//PWRBRK_STALL_PATTERN_3_4
+#define PWRBRK_STALL_PATTERN_3_4__PWRBRK_STALL_PATTERN_3__SHIFT                                               0x0
+#define PWRBRK_STALL_PATTERN_3_4__PWRBRK_STALL_PATTERN_4__SHIFT                                               0x10
+#define PWRBRK_STALL_PATTERN_3_4__PWRBRK_STALL_PATTERN_3_MASK                                                 0x00007FFFL
+#define PWRBRK_STALL_PATTERN_3_4__PWRBRK_STALL_PATTERN_4_MASK                                                 0x7FFF0000L
+//PWRBRK_STALL_PATTERN_5_6
+#define PWRBRK_STALL_PATTERN_5_6__PWRBRK_STALL_PATTERN_5__SHIFT                                               0x0
+#define PWRBRK_STALL_PATTERN_5_6__PWRBRK_STALL_PATTERN_6__SHIFT                                               0x10
+#define PWRBRK_STALL_PATTERN_5_6__PWRBRK_STALL_PATTERN_5_MASK                                                 0x00007FFFL
+#define PWRBRK_STALL_PATTERN_5_6__PWRBRK_STALL_PATTERN_6_MASK                                                 0x7FFF0000L
+//PWRBRK_STALL_PATTERN_7
+#define PWRBRK_STALL_PATTERN_7__PWRBRK_STALL_PATTERN_7__SHIFT                                                 0x0
+#define PWRBRK_STALL_PATTERN_7__PWRBRK_STALL_PATTERN_7_MASK                                                   0x00007FFFL
+//DIDT_STALL_PATTERN_CTRL
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_CTRL_EN__SHIFT                                                    0x0
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_SW_RST__SHIFT                                                     0x1
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_CLK_EN_OVERRIDE__SHIFT                                            0x2
+#define DIDT_STALL_PATTERN_CTRL__DIDT_STALL_PATTERN_BIT_NUMS__SHIFT                                           0x3
+#define DIDT_STALL_PATTERN_CTRL__DIDT_PATTERN_EXTEND_EN__SHIFT                                                0x7
+#define DIDT_STALL_PATTERN_CTRL__DIDT_PATTERN_EXTEND_MODE__SHIFT                                              0x8
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_CTRL_EN_MASK                                                      0x00000001L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_SW_RST_MASK                                                       0x00000002L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_DROOP_CLK_EN_OVERRIDE_MASK                                              0x00000004L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_STALL_PATTERN_BIT_NUMS_MASK                                             0x00000078L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_PATTERN_EXTEND_EN_MASK                                                  0x00000080L
+#define DIDT_STALL_PATTERN_CTRL__DIDT_PATTERN_EXTEND_MODE_MASK                                                0x00000700L
+//DIDT_STALL_PATTERN_1_2
+#define DIDT_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1__SHIFT                                                   0x0
+#define DIDT_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2__SHIFT                                                   0x10
+#define DIDT_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_1_MASK                                                     0x00007FFFL
+#define DIDT_STALL_PATTERN_1_2__DIDT_STALL_PATTERN_2_MASK                                                     0x7FFF0000L
+//DIDT_STALL_PATTERN_3_4
+#define DIDT_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3__SHIFT                                                   0x0
+#define DIDT_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4__SHIFT                                                   0x10
+#define DIDT_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_3_MASK                                                     0x00007FFFL
+#define DIDT_STALL_PATTERN_3_4__DIDT_STALL_PATTERN_4_MASK                                                     0x7FFF0000L
+//DIDT_STALL_PATTERN_5_6
+#define DIDT_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5__SHIFT                                                   0x0
+#define DIDT_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6__SHIFT                                                   0x10
+#define DIDT_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_5_MASK                                                     0x00007FFFL
+#define DIDT_STALL_PATTERN_5_6__DIDT_STALL_PATTERN_6_MASK                                                     0x7FFF0000L
+//DIDT_STALL_PATTERN_7
+#define DIDT_STALL_PATTERN_7__DIDT_STALL_PATTERN_7__SHIFT                                                     0x0
+#define DIDT_STALL_PATTERN_7__DIDT_STALL_PATTERN_7_MASK                                                       0x00007FFFL
+//PCC_PWRBRK_HYSTERESIS_CTRL
+#define PCC_PWRBRK_HYSTERESIS_CTRL__PCC_MAX_HYSTERESIS__SHIFT                                                 0x0
+#define PCC_PWRBRK_HYSTERESIS_CTRL__PWRBRK_MAX_HYSTERESIS__SHIFT                                              0x8
+#define PCC_PWRBRK_HYSTERESIS_CTRL__PCC_MAX_HYSTERESIS_MASK                                                   0x000000FFL
+#define PCC_PWRBRK_HYSTERESIS_CTRL__PWRBRK_MAX_HYSTERESIS_MASK                                                0x0000FF00L
+//EDC_STRETCH_PERF_COUNTER
+#define EDC_STRETCH_PERF_COUNTER__STRETCH_PERF_COUNTER__SHIFT                                                 0x0
+#define EDC_STRETCH_PERF_COUNTER__STRETCH_PERF_COUNTER_MASK                                                   0xFFFFFFFFL
+//EDC_UNSTRETCH_PERF_COUNTER
+#define EDC_UNSTRETCH_PERF_COUNTER__UNSTRETCH_PERF_COUNTER__SHIFT                                             0x0
+#define EDC_UNSTRETCH_PERF_COUNTER__UNSTRETCH_PERF_COUNTER_MASK                                               0xFFFFFFFFL
+//EDC_STRETCH_NUM_PERF_COUNTER
+#define EDC_STRETCH_NUM_PERF_COUNTER__STRETCH_NUM_PERF_COUNTER__SHIFT                                         0x0
+#define EDC_STRETCH_NUM_PERF_COUNTER__STRETCH_NUM_PERF_COUNTER_MASK                                           0xFFFFFFFFL
+//GC_EDC_STATUS
+#define GC_EDC_STATUS__EDC_THROTTLE_LEVEL__SHIFT                                                              0x0
+#define GC_EDC_STATUS__GPIO_IN_0__SHIFT                                                                       0x3
+#define GC_EDC_STATUS__GPIO_IN_1__SHIFT                                                                       0x4
+#define GC_EDC_STATUS__EDC_THROTTLE_LEVEL_MASK                                                                0x00000007L
+#define GC_EDC_STATUS__GPIO_IN_0_MASK                                                                         0x00000008L
+#define GC_EDC_STATUS__GPIO_IN_1_MASK                                                                         0x00000010L
+//GC_EDC_OVERFLOW
+#define GC_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW__SHIFT                                              0x0
+#define GC_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER__SHIFT                                           0x1
+#define GC_EDC_OVERFLOW__EDC_ROLLING_POWER_DELTA_OVERFLOW_MASK                                                0x00000001L
+#define GC_EDC_OVERFLOW__EDC_THROTTLE_LEVEL_OVERFLOW_COUNTER_MASK                                             0x0001FFFEL
+//GC_EDC_ROLLING_POWER_DELTA
+#define GC_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA__SHIFT                                            0x0
+#define GC_EDC_ROLLING_POWER_DELTA__EDC_ROLLING_POWER_DELTA_MASK                                              0xFFFFFFFFL
+//GC_THROTTLE_STATUS
+#define GC_THROTTLE_STATUS__FSM_STATE__SHIFT                                                                  0x0
+#define GC_THROTTLE_STATUS__PATTERN_INDEX__SHIFT                                                              0x4
+#define GC_THROTTLE_STATUS__FSM_STATE_MASK                                                                    0x0000000FL
+#define GC_THROTTLE_STATUS__PATTERN_INDEX_MASK                                                                0x000001F0L
+//EDC_PERF_COUNTER
+#define EDC_PERF_COUNTER__EDC_PERF_COUNTER__SHIFT                                                             0x0
+#define EDC_PERF_COUNTER__EDC_PERF_COUNTER_MASK                                                               0xFFFFFFFFL
+//PCC_PERF_COUNTER
+#define PCC_PERF_COUNTER__PCC_PERF_COUNTER__SHIFT                                                             0x0
+#define PCC_PERF_COUNTER__PCC_PERF_COUNTER_MASK                                                               0xFFFFFFFFL
+//PWRBRK_PERF_COUNTER
+#define PWRBRK_PERF_COUNTER__PWRBRK_PERF_COUNTER__SHIFT                                                       0x0
+#define PWRBRK_PERF_COUNTER__PWRBRK_PERF_COUNTER_MASK                                                         0xFFFFFFFFL
+//EDC_HYSTERESIS_STAT
+#define EDC_HYSTERESIS_STAT__HYSTERESIS_CNT__SHIFT                                                            0x0
+#define EDC_HYSTERESIS_STAT__EDC_STATUS__SHIFT                                                                0x8
+#define EDC_HYSTERESIS_STAT__EDC_CREDIT_INCR_OVERFLOW__SHIFT                                                  0x9
+#define EDC_HYSTERESIS_STAT__EDC_THRESHOLD_SEL__SHIFT                                                         0xa
+#define EDC_HYSTERESIS_STAT__HYSTERESIS_CNT_MASK                                                              0x000000FFL
+#define EDC_HYSTERESIS_STAT__EDC_STATUS_MASK                                                                  0x00000100L
+#define EDC_HYSTERESIS_STAT__EDC_CREDIT_INCR_OVERFLOW_MASK                                                    0x00000200L
+#define EDC_HYSTERESIS_STAT__EDC_THRESHOLD_SEL_MASK                                                           0x00000400L
+//GC_CAC_WEIGHT_CP_0
+#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG0__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG1__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG0_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_CP_0__WEIGHT_CP_SIG1_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_CP_1
+#define GC_CAC_WEIGHT_CP_1__WEIGHT_CP_SIG2__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_CP_1__WEIGHT_CP_SIG2_MASK                                                               0x0000FFFFL
+//GC_CAC_WEIGHT_EA_0
+#define GC_CAC_WEIGHT_EA_0__WEIGHT_EA_SIG0__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_EA_0__WEIGHT_EA_SIG1__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_EA_0__WEIGHT_EA_SIG0_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_EA_0__WEIGHT_EA_SIG1_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_EA_1
+#define GC_CAC_WEIGHT_EA_1__WEIGHT_EA_SIG2__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_EA_1__WEIGHT_EA_SIG3__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_EA_1__WEIGHT_EA_SIG2_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_EA_1__WEIGHT_EA_SIG3_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_EA_2
+#define GC_CAC_WEIGHT_EA_2__WEIGHT_EA_SIG4__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_EA_2__WEIGHT_EA_SIG5__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_EA_2__WEIGHT_EA_SIG4_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_EA_2__WEIGHT_EA_SIG5_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_UTCL2_ROUTER_0
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_0__WEIGHT_UTCL2_ROUTER_SIG0__SHIFT                                         0x0
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_0__WEIGHT_UTCL2_ROUTER_SIG1__SHIFT                                         0x10
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_0__WEIGHT_UTCL2_ROUTER_SIG0_MASK                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_0__WEIGHT_UTCL2_ROUTER_SIG1_MASK                                           0xFFFF0000L
+//GC_CAC_WEIGHT_UTCL2_ROUTER_1
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_1__WEIGHT_UTCL2_ROUTER_SIG2__SHIFT                                         0x0
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_1__WEIGHT_UTCL2_ROUTER_SIG3__SHIFT                                         0x10
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_1__WEIGHT_UTCL2_ROUTER_SIG2_MASK                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_1__WEIGHT_UTCL2_ROUTER_SIG3_MASK                                           0xFFFF0000L
+//GC_CAC_WEIGHT_UTCL2_ROUTER_2
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_2__WEIGHT_UTCL2_ROUTER_SIG4__SHIFT                                         0x0
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_2__WEIGHT_UTCL2_ROUTER_SIG5__SHIFT                                         0x10
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_2__WEIGHT_UTCL2_ROUTER_SIG4_MASK                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_2__WEIGHT_UTCL2_ROUTER_SIG5_MASK                                           0xFFFF0000L
+//GC_CAC_WEIGHT_UTCL2_ROUTER_3
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_3__WEIGHT_UTCL2_ROUTER_SIG6__SHIFT                                         0x0
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_3__WEIGHT_UTCL2_ROUTER_SIG7__SHIFT                                         0x10
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_3__WEIGHT_UTCL2_ROUTER_SIG6_MASK                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_3__WEIGHT_UTCL2_ROUTER_SIG7_MASK                                           0xFFFF0000L
+//GC_CAC_WEIGHT_UTCL2_ROUTER_4
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_4__WEIGHT_UTCL2_ROUTER_SIG8__SHIFT                                         0x0
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_4__WEIGHT_UTCL2_ROUTER_SIG9__SHIFT                                         0x10
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_4__WEIGHT_UTCL2_ROUTER_SIG8_MASK                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_UTCL2_ROUTER_4__WEIGHT_UTCL2_ROUTER_SIG9_MASK                                           0xFFFF0000L
+//GC_CAC_WEIGHT_UTCL2_VML2_0
+#define GC_CAC_WEIGHT_UTCL2_VML2_0__WEIGHT_UTCL2_VML2_SIG0__SHIFT                                             0x0
+#define GC_CAC_WEIGHT_UTCL2_VML2_0__WEIGHT_UTCL2_VML2_SIG1__SHIFT                                             0x10
+#define GC_CAC_WEIGHT_UTCL2_VML2_0__WEIGHT_UTCL2_VML2_SIG0_MASK                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_UTCL2_VML2_0__WEIGHT_UTCL2_VML2_SIG1_MASK                                               0xFFFF0000L
+//GC_CAC_WEIGHT_UTCL2_VML2_1
+#define GC_CAC_WEIGHT_UTCL2_VML2_1__WEIGHT_UTCL2_VML2_SIG2__SHIFT                                             0x0
+#define GC_CAC_WEIGHT_UTCL2_VML2_1__WEIGHT_UTCL2_VML2_SIG3__SHIFT                                             0x10
+#define GC_CAC_WEIGHT_UTCL2_VML2_1__WEIGHT_UTCL2_VML2_SIG2_MASK                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_UTCL2_VML2_1__WEIGHT_UTCL2_VML2_SIG3_MASK                                               0xFFFF0000L
+//GC_CAC_WEIGHT_UTCL2_VML2_2
+#define GC_CAC_WEIGHT_UTCL2_VML2_2__WEIGHT_UTCL2_VML2_SIG4__SHIFT                                             0x0
+#define GC_CAC_WEIGHT_UTCL2_VML2_2__WEIGHT_UTCL2_VML2_SIG4_MASK                                               0x0000FFFFL
+//GC_CAC_WEIGHT_UTCL2_WALKER_0
+#define GC_CAC_WEIGHT_UTCL2_WALKER_0__WEIGHT_UTCL2_WALKER_SIG0__SHIFT                                         0x0
+#define GC_CAC_WEIGHT_UTCL2_WALKER_0__WEIGHT_UTCL2_WALKER_SIG1__SHIFT                                         0x10
+#define GC_CAC_WEIGHT_UTCL2_WALKER_0__WEIGHT_UTCL2_WALKER_SIG0_MASK                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_UTCL2_WALKER_0__WEIGHT_UTCL2_WALKER_SIG1_MASK                                           0xFFFF0000L
+//GC_CAC_WEIGHT_UTCL2_WALKER_1
+#define GC_CAC_WEIGHT_UTCL2_WALKER_1__WEIGHT_UTCL2_WALKER_SIG2__SHIFT                                         0x0
+#define GC_CAC_WEIGHT_UTCL2_WALKER_1__WEIGHT_UTCL2_WALKER_SIG3__SHIFT                                         0x10
+#define GC_CAC_WEIGHT_UTCL2_WALKER_1__WEIGHT_UTCL2_WALKER_SIG2_MASK                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_UTCL2_WALKER_1__WEIGHT_UTCL2_WALKER_SIG3_MASK                                           0xFFFF0000L
+//GC_CAC_WEIGHT_UTCL2_WALKER_2
+#define GC_CAC_WEIGHT_UTCL2_WALKER_2__WEIGHT_UTCL2_WALKER_SIG4__SHIFT                                         0x0
+#define GC_CAC_WEIGHT_UTCL2_WALKER_2__WEIGHT_UTCL2_WALKER_SIG4_MASK                                           0x0000FFFFL
+//GC_CAC_WEIGHT_GDS_0
+#define GC_CAC_WEIGHT_GDS_0__WEIGHT_GDS_SIG0__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_GDS_0__WEIGHT_GDS_SIG1__SHIFT                                                           0x10
+#define GC_CAC_WEIGHT_GDS_0__WEIGHT_GDS_SIG0_MASK                                                             0x0000FFFFL
+#define GC_CAC_WEIGHT_GDS_0__WEIGHT_GDS_SIG1_MASK                                                             0xFFFF0000L
+//GC_CAC_WEIGHT_GDS_1
+#define GC_CAC_WEIGHT_GDS_1__WEIGHT_GDS_SIG2__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_GDS_1__WEIGHT_GDS_SIG3__SHIFT                                                           0x10
+#define GC_CAC_WEIGHT_GDS_1__WEIGHT_GDS_SIG2_MASK                                                             0x0000FFFFL
+#define GC_CAC_WEIGHT_GDS_1__WEIGHT_GDS_SIG3_MASK                                                             0xFFFF0000L
+//GC_CAC_WEIGHT_GDS_2
+#define GC_CAC_WEIGHT_GDS_2__WEIGHT_GDS_SIG4__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_GDS_2__WEIGHT_GDS_SIG4_MASK                                                             0x0000FFFFL
+//GC_CAC_WEIGHT_GE_0
+#define GC_CAC_WEIGHT_GE_0__WEIGHT_GE_SIG0__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_GE_0__WEIGHT_GE_SIG1__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_GE_0__WEIGHT_GE_SIG0_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_GE_0__WEIGHT_GE_SIG1_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_GE_1
+#define GC_CAC_WEIGHT_GE_1__WEIGHT_GE_SIG2__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_GE_1__WEIGHT_GE_SIG3__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_GE_1__WEIGHT_GE_SIG2_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_GE_1__WEIGHT_GE_SIG3_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_GE_2
+#define GC_CAC_WEIGHT_GE_2__WEIGHT_GE_SIG4__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_GE_2__WEIGHT_GE_SIG5__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_GE_2__WEIGHT_GE_SIG4_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_GE_2__WEIGHT_GE_SIG5_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_GE_3
+#define GC_CAC_WEIGHT_GE_3__WEIGHT_GE_SIG6__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_GE_3__WEIGHT_GE_SIG6_MASK                                                               0x0000FFFFL
+//GC_CAC_WEIGHT_PMM_0
+#define GC_CAC_WEIGHT_PMM_0__WEIGHT_PMM_SIG0__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_PMM_0__WEIGHT_PMM_SIG0_MASK                                                             0x0000FFFFL
+//GC_CAC_WEIGHT_GL2C_0
+#define GC_CAC_WEIGHT_GL2C_0__WEIGHT_GL2C_SIG0__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_GL2C_0__WEIGHT_GL2C_SIG1__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_GL2C_0__WEIGHT_GL2C_SIG0_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_GL2C_0__WEIGHT_GL2C_SIG1_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_GL2C_1
+#define GC_CAC_WEIGHT_GL2C_1__WEIGHT_GL2C_SIG2__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_GL2C_1__WEIGHT_GL2C_SIG3__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_GL2C_1__WEIGHT_GL2C_SIG2_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_GL2C_1__WEIGHT_GL2C_SIG3_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_GL2C_2
+#define GC_CAC_WEIGHT_GL2C_2__WEIGHT_GL2C_SIG4__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_GL2C_2__WEIGHT_GL2C_SIG4_MASK                                                           0x0000FFFFL
+//GC_CAC_WEIGHT_PH_0
+#define GC_CAC_WEIGHT_PH_0__WEIGHT_PH_SIG0__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_PH_0__WEIGHT_PH_SIG1__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_PH_0__WEIGHT_PH_SIG0_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_PH_0__WEIGHT_PH_SIG1_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_PH_1
+#define GC_CAC_WEIGHT_PH_1__WEIGHT_PH_SIG2__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_PH_1__WEIGHT_PH_SIG3__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_PH_1__WEIGHT_PH_SIG2_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_PH_1__WEIGHT_PH_SIG3_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_PH_2
+#define GC_CAC_WEIGHT_PH_2__WEIGHT_PH_SIG4__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_PH_2__WEIGHT_PH_SIG5__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_PH_2__WEIGHT_PH_SIG4_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_PH_2__WEIGHT_PH_SIG5_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_PH_3
+#define GC_CAC_WEIGHT_PH_3__WEIGHT_PH_SIG6__SHIFT                                                             0x0
+#define GC_CAC_WEIGHT_PH_3__WEIGHT_PH_SIG7__SHIFT                                                             0x10
+#define GC_CAC_WEIGHT_PH_3__WEIGHT_PH_SIG6_MASK                                                               0x0000FFFFL
+#define GC_CAC_WEIGHT_PH_3__WEIGHT_PH_SIG7_MASK                                                               0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_0
+#define GC_CAC_WEIGHT_SDMA_0__WEIGHT_SDMA_SIG0__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_SDMA_0__WEIGHT_SDMA_SIG1__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_SDMA_0__WEIGHT_SDMA_SIG0_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_0__WEIGHT_SDMA_SIG1_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_1
+#define GC_CAC_WEIGHT_SDMA_1__WEIGHT_SDMA_SIG2__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_SDMA_1__WEIGHT_SDMA_SIG3__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_SDMA_1__WEIGHT_SDMA_SIG2_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_1__WEIGHT_SDMA_SIG3_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_2
+#define GC_CAC_WEIGHT_SDMA_2__WEIGHT_SDMA_SIG4__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_SDMA_2__WEIGHT_SDMA_SIG5__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_SDMA_2__WEIGHT_SDMA_SIG4_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_2__WEIGHT_SDMA_SIG5_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_3
+#define GC_CAC_WEIGHT_SDMA_3__WEIGHT_SDMA_SIG6__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_SDMA_3__WEIGHT_SDMA_SIG7__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_SDMA_3__WEIGHT_SDMA_SIG6_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_3__WEIGHT_SDMA_SIG7_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_4
+#define GC_CAC_WEIGHT_SDMA_4__WEIGHT_SDMA_SIG8__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_SDMA_4__WEIGHT_SDMA_SIG9__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_SDMA_4__WEIGHT_SDMA_SIG8_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_4__WEIGHT_SDMA_SIG9_MASK                                                           0xFFFF0000L
+//GC_CAC_WEIGHT_SDMA_5
+#define GC_CAC_WEIGHT_SDMA_5__WEIGHT_SDMA_SIG10__SHIFT                                                        0x0
+#define GC_CAC_WEIGHT_SDMA_5__WEIGHT_SDMA_SIG11__SHIFT                                                        0x10
+#define GC_CAC_WEIGHT_SDMA_5__WEIGHT_SDMA_SIG10_MASK                                                          0x0000FFFFL
+#define GC_CAC_WEIGHT_SDMA_5__WEIGHT_SDMA_SIG11_MASK                                                          0xFFFF0000L
+//GC_CAC_WEIGHT_CHC_0
+#define GC_CAC_WEIGHT_CHC_0__WEIGHT_CHC_SIG0__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_CHC_0__WEIGHT_CHC_SIG1__SHIFT                                                           0x10
+#define GC_CAC_WEIGHT_CHC_0__WEIGHT_CHC_SIG0_MASK                                                             0x0000FFFFL
+#define GC_CAC_WEIGHT_CHC_0__WEIGHT_CHC_SIG1_MASK                                                             0xFFFF0000L
+//GC_CAC_WEIGHT_CHC_1
+#define GC_CAC_WEIGHT_CHC_1__WEIGHT_CHC_SIG2__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_CHC_1__WEIGHT_CHC_SIG2_MASK                                                             0x0000FFFFL
+//GC_CAC_WEIGHT_GUS_0
+#define GC_CAC_WEIGHT_GUS_0__WEIGHT_GUS_SIG0__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_GUS_0__WEIGHT_GUS_SIG1__SHIFT                                                           0x10
+#define GC_CAC_WEIGHT_GUS_0__WEIGHT_GUS_SIG0_MASK                                                             0x0000FFFFL
+#define GC_CAC_WEIGHT_GUS_0__WEIGHT_GUS_SIG1_MASK                                                             0xFFFF0000L
+//GC_CAC_WEIGHT_GUS_1
+#define GC_CAC_WEIGHT_GUS_1__WEIGHT_GUS_SIG2__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_GUS_1__WEIGHT_GUS_SIG2_MASK                                                             0x0000FFFFL
+//GC_CAC_WEIGHT_RLC_0
+#define GC_CAC_WEIGHT_RLC_0__WEIGHT_RLC_SIG0__SHIFT                                                           0x0
+#define GC_CAC_WEIGHT_RLC_0__WEIGHT_RLC_SIG0_MASK                                                             0x0000FFFFL
+//GC_CAC_WEIGHT_GRBM_0
+#define GC_CAC_WEIGHT_GRBM_0__WEIGHT_GRBM_SIG0__SHIFT                                                         0x0
+#define GC_CAC_WEIGHT_GRBM_0__WEIGHT_GRBM_SIG1__SHIFT                                                         0x10
+#define GC_CAC_WEIGHT_GRBM_0__WEIGHT_GRBM_SIG0_MASK                                                           0x0000FFFFL
+#define GC_CAC_WEIGHT_GRBM_0__WEIGHT_GRBM_SIG1_MASK                                                           0xFFFF0000L
+//GC_EDC_CLK_MONITOR_CTRL
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_EN__SHIFT                                                    0x0
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_INTERVAL__SHIFT                                              0x1
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_THRESHOLD__SHIFT                                             0x5
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_EN_MASK                                                      0x00000001L
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_INTERVAL_MASK                                                0x0000001EL
+#define GC_EDC_CLK_MONITOR_CTRL__EDC_CLK_MONITOR_THRESHOLD_MASK                                               0x0001FFE0L
+//GC_CAC_IND_INDEX
+#define GC_CAC_IND_INDEX__GC_CAC_IND_ADDR__SHIFT                                                              0x0
+#define GC_CAC_IND_INDEX__GC_CAC_IND_ADDR_MASK                                                                0xFFFFFFFFL
+//GC_CAC_IND_DATA
+#define GC_CAC_IND_DATA__GC_CAC_IND_DATA__SHIFT                                                               0x0
+#define GC_CAC_IND_DATA__GC_CAC_IND_DATA_MASK                                                                 0xFFFFFFFFL
+//SE_CAC_CTRL_1
+#define SE_CAC_CTRL_1__CAC_WINDOW__SHIFT                                                                      0x0
+#define SE_CAC_CTRL_1__TDP_WINDOW__SHIFT                                                                      0x8
+#define SE_CAC_CTRL_1__CAC_WINDOW_MASK                                                                        0x000000FFL
+#define SE_CAC_CTRL_1__TDP_WINDOW_MASK                                                                        0xFFFFFF00L
+//SE_CAC_CTRL_2
+#define SE_CAC_CTRL_2__CAC_ENABLE__SHIFT                                                                      0x0
+#define SE_CAC_CTRL_2__SE_LCAC_ENABLE__SHIFT                                                                  0x1
+#define SE_CAC_CTRL_2__WGP_CAC_CLK_OVERRIDE__SHIFT                                                            0x2
+#define SE_CAC_CTRL_2__SE_CAC_INDEX_AUTO_INCR_EN__SHIFT                                                       0x3
+#define SE_CAC_CTRL_2__CAC_ENABLE_MASK                                                                        0x00000001L
+#define SE_CAC_CTRL_2__SE_LCAC_ENABLE_MASK                                                                    0x00000002L
+#define SE_CAC_CTRL_2__WGP_CAC_CLK_OVERRIDE_MASK                                                              0x00000004L
+#define SE_CAC_CTRL_2__SE_CAC_INDEX_AUTO_INCR_EN_MASK                                                         0x00000008L
+//SE_CAC_WEIGHT_TA_0
+#define SE_CAC_WEIGHT_TA_0__WEIGHT_TA_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_TA_0__WEIGHT_TA_SIG0_MASK                                                               0x0000FFFFL
+//SE_CAC_WEIGHT_TD_0
+#define SE_CAC_WEIGHT_TD_0__WEIGHT_TD_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_TD_0__WEIGHT_TD_SIG1__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_TD_0__WEIGHT_TD_SIG0_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_TD_0__WEIGHT_TD_SIG1_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_TD_1
+#define SE_CAC_WEIGHT_TD_1__WEIGHT_TD_SIG2__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_TD_1__WEIGHT_TD_SIG3__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_TD_1__WEIGHT_TD_SIG2_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_TD_1__WEIGHT_TD_SIG3_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_TD_2
+#define SE_CAC_WEIGHT_TD_2__WEIGHT_TD_SIG4__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_TD_2__WEIGHT_TD_SIG5__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_TD_2__WEIGHT_TD_SIG4_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_TD_2__WEIGHT_TD_SIG5_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_TD_3
+#define SE_CAC_WEIGHT_TD_3__WEIGHT_TD_SIG6__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_TD_3__WEIGHT_TD_SIG7__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_TD_3__WEIGHT_TD_SIG6_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_TD_3__WEIGHT_TD_SIG7_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_TD_4
+#define SE_CAC_WEIGHT_TD_4__WEIGHT_TD_SIG8__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_TD_4__WEIGHT_TD_SIG9__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_TD_4__WEIGHT_TD_SIG8_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_TD_4__WEIGHT_TD_SIG9_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_TD_5
+#define SE_CAC_WEIGHT_TD_5__WEIGHT_TD_SIG10__SHIFT                                                            0x0
+#define SE_CAC_WEIGHT_TD_5__WEIGHT_TD_SIG10_MASK                                                              0x0000FFFFL
+//SE_CAC_WEIGHT_TCP_0
+#define SE_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG0__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG1__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG0_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_0__WEIGHT_TCP_SIG1_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_1
+#define SE_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG2__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG3__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG2_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_1__WEIGHT_TCP_SIG3_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_2
+#define SE_CAC_WEIGHT_TCP_2__WEIGHT_TCP_SIG4__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_TCP_2__WEIGHT_TCP_SIG5__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_TCP_2__WEIGHT_TCP_SIG4_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_2__WEIGHT_TCP_SIG5_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_TCP_3
+#define SE_CAC_WEIGHT_TCP_3__WEIGHT_TCP_SIG6__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_TCP_3__WEIGHT_TCP_SIG7__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_TCP_3__WEIGHT_TCP_SIG6_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_TCP_3__WEIGHT_TCP_SIG7_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_SQ_0
+#define SE_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG1__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG0_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SQ_0__WEIGHT_SQ_SIG1_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SQ_1
+#define SE_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG2__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG3__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG2_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SQ_1__WEIGHT_SQ_SIG3_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SQ_2
+#define SE_CAC_WEIGHT_SQ_2__WEIGHT_SQ_SIG4__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SQ_2__WEIGHT_SQ_SIG4_MASK                                                               0x0000FFFFL
+//SE_CAC_WEIGHT_SP_0
+#define SE_CAC_WEIGHT_SP_0__WEIGHT_SP_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SP_0__WEIGHT_SP_SIG1__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SP_0__WEIGHT_SP_SIG0_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SP_0__WEIGHT_SP_SIG1_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SP_1
+#define SE_CAC_WEIGHT_SP_1__WEIGHT_SP_SIG2__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SP_1__WEIGHT_SP_SIG2_MASK                                                               0x0000FFFFL
+//SE_CAC_WEIGHT_LDS_0
+#define SE_CAC_WEIGHT_LDS_0__WEIGHT_LDS_SIG0__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_LDS_0__WEIGHT_LDS_SIG1__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_LDS_0__WEIGHT_LDS_SIG0_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS_0__WEIGHT_LDS_SIG1_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_LDS_1
+#define SE_CAC_WEIGHT_LDS_1__WEIGHT_LDS_SIG2__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_LDS_1__WEIGHT_LDS_SIG3__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_LDS_1__WEIGHT_LDS_SIG2_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS_1__WEIGHT_LDS_SIG3_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_LDS_2
+#define SE_CAC_WEIGHT_LDS_2__WEIGHT_LDS_SIG4__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_LDS_2__WEIGHT_LDS_SIG5__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_LDS_2__WEIGHT_LDS_SIG4_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS_2__WEIGHT_LDS_SIG5_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_LDS_3
+#define SE_CAC_WEIGHT_LDS_3__WEIGHT_LDS_SIG6__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_LDS_3__WEIGHT_LDS_SIG7__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_LDS_3__WEIGHT_LDS_SIG6_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_LDS_3__WEIGHT_LDS_SIG7_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_SQC_0
+#define SE_CAC_WEIGHT_SQC_0__WEIGHT_SQC_SIG0__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_SQC_0__WEIGHT_SQC_SIG1__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_SQC_0__WEIGHT_SQC_SIG0_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_SQC_0__WEIGHT_SQC_SIG1_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_SQC_1
+#define SE_CAC_WEIGHT_SQC_1__WEIGHT_SQC_SIG2__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_SQC_1__WEIGHT_SQC_SIG2_MASK                                                             0x0000FFFFL
+//SE_CAC_WEIGHT_CU_0
+#define SE_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_CU_0__WEIGHT_CU_SIG0_MASK                                                               0x0000FFFFL
+//SE_CAC_WEIGHT_BCI_0
+#define SE_CAC_WEIGHT_BCI_0__WEIGHT_BCI_SIG0__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_BCI_0__WEIGHT_BCI_SIG1__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_BCI_0__WEIGHT_BCI_SIG0_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_BCI_0__WEIGHT_BCI_SIG1_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_CB_0
+#define SE_CAC_WEIGHT_CB_0__WEIGHT_CB_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_CB_0__WEIGHT_CB_SIG1__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_CB_0__WEIGHT_CB_SIG0_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_0__WEIGHT_CB_SIG1_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_CB_1
+#define SE_CAC_WEIGHT_CB_1__WEIGHT_CB_SIG2__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_CB_1__WEIGHT_CB_SIG3__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_CB_1__WEIGHT_CB_SIG2_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_1__WEIGHT_CB_SIG3_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_CB_2
+#define SE_CAC_WEIGHT_CB_2__WEIGHT_CB_SIG4__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_CB_2__WEIGHT_CB_SIG5__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_CB_2__WEIGHT_CB_SIG4_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_2__WEIGHT_CB_SIG5_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_CB_3
+#define SE_CAC_WEIGHT_CB_3__WEIGHT_CB_SIG6__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_CB_3__WEIGHT_CB_SIG7__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_CB_3__WEIGHT_CB_SIG6_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_3__WEIGHT_CB_SIG7_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_CB_4
+#define SE_CAC_WEIGHT_CB_4__WEIGHT_CB_SIG8__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_CB_4__WEIGHT_CB_SIG9__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_CB_4__WEIGHT_CB_SIG8_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_4__WEIGHT_CB_SIG9_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_CB_5
+#define SE_CAC_WEIGHT_CB_5__WEIGHT_CB_SIG10__SHIFT                                                            0x0
+#define SE_CAC_WEIGHT_CB_5__WEIGHT_CB_SIG11__SHIFT                                                            0x10
+#define SE_CAC_WEIGHT_CB_5__WEIGHT_CB_SIG10_MASK                                                              0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_5__WEIGHT_CB_SIG11_MASK                                                              0xFFFF0000L
+//SE_CAC_WEIGHT_CB_6
+#define SE_CAC_WEIGHT_CB_6__WEIGHT_CB_SIG12__SHIFT                                                            0x0
+#define SE_CAC_WEIGHT_CB_6__WEIGHT_CB_SIG13__SHIFT                                                            0x10
+#define SE_CAC_WEIGHT_CB_6__WEIGHT_CB_SIG12_MASK                                                              0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_6__WEIGHT_CB_SIG13_MASK                                                              0xFFFF0000L
+//SE_CAC_WEIGHT_CB_7
+#define SE_CAC_WEIGHT_CB_7__WEIGHT_CB_SIG14__SHIFT                                                            0x0
+#define SE_CAC_WEIGHT_CB_7__WEIGHT_CB_SIG15__SHIFT                                                            0x10
+#define SE_CAC_WEIGHT_CB_7__WEIGHT_CB_SIG14_MASK                                                              0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_7__WEIGHT_CB_SIG15_MASK                                                              0xFFFF0000L
+//SE_CAC_WEIGHT_CB_8
+#define SE_CAC_WEIGHT_CB_8__WEIGHT_CB_SIG16__SHIFT                                                            0x0
+#define SE_CAC_WEIGHT_CB_8__WEIGHT_CB_SIG17__SHIFT                                                            0x10
+#define SE_CAC_WEIGHT_CB_8__WEIGHT_CB_SIG16_MASK                                                              0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_8__WEIGHT_CB_SIG17_MASK                                                              0xFFFF0000L
+//SE_CAC_WEIGHT_CB_9
+#define SE_CAC_WEIGHT_CB_9__WEIGHT_CB_SIG18__SHIFT                                                            0x0
+#define SE_CAC_WEIGHT_CB_9__WEIGHT_CB_SIG19__SHIFT                                                            0x10
+#define SE_CAC_WEIGHT_CB_9__WEIGHT_CB_SIG18_MASK                                                              0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_9__WEIGHT_CB_SIG19_MASK                                                              0xFFFF0000L
+//SE_CAC_WEIGHT_CB_10
+#define SE_CAC_WEIGHT_CB_10__WEIGHT_CB_SIG20__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_CB_10__WEIGHT_CB_SIG21__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_CB_10__WEIGHT_CB_SIG20_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_10__WEIGHT_CB_SIG21_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_CB_11
+#define SE_CAC_WEIGHT_CB_11__WEIGHT_CB_SIG22__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_CB_11__WEIGHT_CB_SIG23__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_CB_11__WEIGHT_CB_SIG22_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_CB_11__WEIGHT_CB_SIG23_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_DB_0
+#define SE_CAC_WEIGHT_DB_0__WEIGHT_DB_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_DB_0__WEIGHT_DB_SIG1__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_DB_0__WEIGHT_DB_SIG0_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_DB_0__WEIGHT_DB_SIG1_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_DB_1
+#define SE_CAC_WEIGHT_DB_1__WEIGHT_DB_SIG2__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_DB_1__WEIGHT_DB_SIG3__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_DB_1__WEIGHT_DB_SIG2_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_DB_1__WEIGHT_DB_SIG3_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_DB_2
+#define SE_CAC_WEIGHT_DB_2__WEIGHT_DB_SIG4__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_DB_2__WEIGHT_DB_SIG5__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_DB_2__WEIGHT_DB_SIG4_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_DB_2__WEIGHT_DB_SIG5_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_DB_3
+#define SE_CAC_WEIGHT_DB_3__WEIGHT_DB_SIG6__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_DB_3__WEIGHT_DB_SIG7__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_DB_3__WEIGHT_DB_SIG6_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_DB_3__WEIGHT_DB_SIG7_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_DB_4
+#define SE_CAC_WEIGHT_DB_4__WEIGHT_DB_SIG8__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_DB_4__WEIGHT_DB_SIG9__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_DB_4__WEIGHT_DB_SIG8_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_DB_4__WEIGHT_DB_SIG9_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_RMI_0
+#define SE_CAC_WEIGHT_RMI_0__WEIGHT_RMI_SIG0__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_RMI_0__WEIGHT_RMI_SIG1__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_RMI_0__WEIGHT_RMI_SIG0_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_RMI_0__WEIGHT_RMI_SIG1_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_RMI_1
+#define SE_CAC_WEIGHT_RMI_1__WEIGHT_RMI_SIG2__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_RMI_1__WEIGHT_RMI_SIG3__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_RMI_1__WEIGHT_RMI_SIG2_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_RMI_1__WEIGHT_RMI_SIG3_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_SX_0
+#define SE_CAC_WEIGHT_SX_0__WEIGHT_SX_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SX_0__WEIGHT_SX_SIG0_MASK                                                               0x0000FFFFL
+//SE_CAC_WEIGHT_SXRB_0
+#define SE_CAC_WEIGHT_SXRB_0__WEIGHT_SXRB_SIG0__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_SXRB_0__WEIGHT_SXRB_SIG0_MASK                                                           0x0000FFFFL
+//SE_CAC_WEIGHT_UTCL1_0
+#define SE_CAC_WEIGHT_UTCL1_0__WEIGHT_UTCL1_SIG0__SHIFT                                                       0x0
+#define SE_CAC_WEIGHT_UTCL1_0__WEIGHT_UTCL1_SIG0_MASK                                                         0x0000FFFFL
+//SE_CAC_WEIGHT_GL1C_0
+#define SE_CAC_WEIGHT_GL1C_0__WEIGHT_GL1C_SIG0__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_GL1C_0__WEIGHT_GL1C_SIG1__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_GL1C_0__WEIGHT_GL1C_SIG0_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_GL1C_0__WEIGHT_GL1C_SIG1_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_GL1C_1
+#define SE_CAC_WEIGHT_GL1C_1__WEIGHT_GL1C_SIG2__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_GL1C_1__WEIGHT_GL1C_SIG3__SHIFT                                                         0x10
+#define SE_CAC_WEIGHT_GL1C_1__WEIGHT_GL1C_SIG2_MASK                                                           0x0000FFFFL
+#define SE_CAC_WEIGHT_GL1C_1__WEIGHT_GL1C_SIG3_MASK                                                           0xFFFF0000L
+//SE_CAC_WEIGHT_GL1C_2
+#define SE_CAC_WEIGHT_GL1C_2__WEIGHT_GL1C_SIG4__SHIFT                                                         0x0
+#define SE_CAC_WEIGHT_GL1C_2__WEIGHT_GL1C_SIG4_MASK                                                           0x0000FFFFL
+//SE_CAC_WEIGHT_SPI_0
+#define SE_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG0__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG1__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG0_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_SPI_0__WEIGHT_SPI_SIG1_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_SPI_1
+#define SE_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG2__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG3__SHIFT                                                           0x10
+#define SE_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG2_MASK                                                             0x0000FFFFL
+#define SE_CAC_WEIGHT_SPI_1__WEIGHT_SPI_SIG3_MASK                                                             0xFFFF0000L
+//SE_CAC_WEIGHT_SPI_2
+#define SE_CAC_WEIGHT_SPI_2__WEIGHT_SPI_SIG4__SHIFT                                                           0x0
+#define SE_CAC_WEIGHT_SPI_2__WEIGHT_SPI_SIG4_MASK                                                             0x0000FFFFL
+//SE_CAC_WEIGHT_PC_0
+#define SE_CAC_WEIGHT_PC_0__WEIGHT_PC_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_PC_0__WEIGHT_PC_SIG0_MASK                                                               0x0000FFFFL
+//SE_CAC_WEIGHT_PA_0
+#define SE_CAC_WEIGHT_PA_0__WEIGHT_PA_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_PA_0__WEIGHT_PA_SIG1__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_PA_0__WEIGHT_PA_SIG0_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_PA_0__WEIGHT_PA_SIG1_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_PA_1
+#define SE_CAC_WEIGHT_PA_1__WEIGHT_PA_SIG2__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_PA_1__WEIGHT_PA_SIG3__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_PA_1__WEIGHT_PA_SIG2_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_PA_1__WEIGHT_PA_SIG3_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_PA_2
+#define SE_CAC_WEIGHT_PA_2__WEIGHT_PA_SIG4__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_PA_2__WEIGHT_PA_SIG5__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_PA_2__WEIGHT_PA_SIG4_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_PA_2__WEIGHT_PA_SIG5_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_PA_3
+#define SE_CAC_WEIGHT_PA_3__WEIGHT_PA_SIG6__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_PA_3__WEIGHT_PA_SIG7__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_PA_3__WEIGHT_PA_SIG6_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_PA_3__WEIGHT_PA_SIG7_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SC_0
+#define SE_CAC_WEIGHT_SC_0__WEIGHT_SC_SIG0__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SC_0__WEIGHT_SC_SIG1__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SC_0__WEIGHT_SC_SIG0_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SC_0__WEIGHT_SC_SIG1_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SC_1
+#define SE_CAC_WEIGHT_SC_1__WEIGHT_SC_SIG2__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SC_1__WEIGHT_SC_SIG3__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SC_1__WEIGHT_SC_SIG2_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SC_1__WEIGHT_SC_SIG3_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SC_2
+#define SE_CAC_WEIGHT_SC_2__WEIGHT_SC_SIG4__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SC_2__WEIGHT_SC_SIG5__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SC_2__WEIGHT_SC_SIG4_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SC_2__WEIGHT_SC_SIG5_MASK                                                               0xFFFF0000L
+//SE_CAC_WEIGHT_SC_3
+#define SE_CAC_WEIGHT_SC_3__WEIGHT_SC_SIG6__SHIFT                                                             0x0
+#define SE_CAC_WEIGHT_SC_3__WEIGHT_SC_SIG7__SHIFT                                                             0x10
+#define SE_CAC_WEIGHT_SC_3__WEIGHT_SC_SIG6_MASK                                                               0x0000FFFFL
+#define SE_CAC_WEIGHT_SC_3__WEIGHT_SC_SIG7_MASK                                                               0xFFFF0000L
+//SE_CAC_WINDOW_AGGR_VALUE
+#define SE_CAC_WINDOW_AGGR_VALUE__SE_CAC_WINDOW_AGGR_VALUE__SHIFT                                             0x0
+#define SE_CAC_WINDOW_AGGR_VALUE__SE_CAC_WINDOW_AGGR_VALUE_MASK                                               0xFFFFFFFFL
+//SE_CAC_WINDOW_GFXCLK_CYCLE
+#define SE_CAC_WINDOW_GFXCLK_CYCLE__SE_CAC_WINDOW_GFXCLK_CYCLE__SHIFT                                         0x0
+#define SE_CAC_WINDOW_GFXCLK_CYCLE__SE_CAC_WINDOW_GFXCLK_CYCLE_MASK                                           0x000003FFL
+//SE_CAC_IND_INDEX
+#define SE_CAC_IND_INDEX__SE_CAC_IND_ADDR__SHIFT                                                              0x0
+#define SE_CAC_IND_INDEX__SE_CAC_IND_ADDR_MASK                                                                0xFFFFFFFFL
+//SE_CAC_IND_DATA
+#define SE_CAC_IND_DATA__SE_CAC_IND_DATA__SHIFT                                                               0x0
+#define SE_CAC_IND_DATA__SE_CAC_IND_DATA_MASK                                                                 0xFFFFFFFFL
+
+
+// addressBlock: gc_pfonly2_spidec
+//SPI_RESOURCE_RESERVE_CU_0
+#define SPI_RESOURCE_RESERVE_CU_0__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_0__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_0__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_0__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_0__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_0__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_0__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_0__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_0__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_1
+#define SPI_RESOURCE_RESERVE_CU_1__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_1__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_1__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_1__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_1__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_1__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_1__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_1__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_1__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_2
+#define SPI_RESOURCE_RESERVE_CU_2__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_2__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_2__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_2__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_2__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_2__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_2__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_2__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_2__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_3
+#define SPI_RESOURCE_RESERVE_CU_3__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_3__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_3__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_3__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_3__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_3__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_3__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_3__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_3__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_4
+#define SPI_RESOURCE_RESERVE_CU_4__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_4__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_4__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_4__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_4__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_4__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_4__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_4__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_4__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_5
+#define SPI_RESOURCE_RESERVE_CU_5__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_5__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_5__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_5__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_5__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_5__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_5__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_5__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_5__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_6
+#define SPI_RESOURCE_RESERVE_CU_6__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_6__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_6__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_6__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_6__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_6__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_6__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_6__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_6__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_7
+#define SPI_RESOURCE_RESERVE_CU_7__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_7__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_7__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_7__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_7__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_7__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_7__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_7__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_7__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_8
+#define SPI_RESOURCE_RESERVE_CU_8__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_8__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_8__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_8__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_8__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_8__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_8__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_8__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_8__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_9
+#define SPI_RESOURCE_RESERVE_CU_9__VGPR__SHIFT                                                                0x0
+#define SPI_RESOURCE_RESERVE_CU_9__SGPR__SHIFT                                                                0x4
+#define SPI_RESOURCE_RESERVE_CU_9__LDS__SHIFT                                                                 0x8
+#define SPI_RESOURCE_RESERVE_CU_9__WAVES__SHIFT                                                               0xc
+#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS__SHIFT                                                            0xf
+#define SPI_RESOURCE_RESERVE_CU_9__VGPR_MASK                                                                  0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_9__SGPR_MASK                                                                  0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_9__LDS_MASK                                                                   0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_9__WAVES_MASK                                                                 0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_9__BARRIERS_MASK                                                              0x00078000L
+//SPI_RESOURCE_RESERVE_CU_10
+#define SPI_RESOURCE_RESERVE_CU_10__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_10__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_10__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_10__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_10__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_10__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_10__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_10__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_10__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_11
+#define SPI_RESOURCE_RESERVE_CU_11__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_11__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_11__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_11__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_11__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_11__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_11__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_11__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_11__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_12
+#define SPI_RESOURCE_RESERVE_CU_12__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_12__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_12__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_12__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_12__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_12__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_12__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_12__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_12__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_12__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_13
+#define SPI_RESOURCE_RESERVE_CU_13__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_13__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_13__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_13__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_13__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_13__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_13__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_13__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_13__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_13__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_14
+#define SPI_RESOURCE_RESERVE_CU_14__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_14__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_14__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_14__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_14__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_14__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_14__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_14__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_14__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_14__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_CU_15
+#define SPI_RESOURCE_RESERVE_CU_15__VGPR__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_CU_15__SGPR__SHIFT                                                               0x4
+#define SPI_RESOURCE_RESERVE_CU_15__LDS__SHIFT                                                                0x8
+#define SPI_RESOURCE_RESERVE_CU_15__WAVES__SHIFT                                                              0xc
+#define SPI_RESOURCE_RESERVE_CU_15__BARRIERS__SHIFT                                                           0xf
+#define SPI_RESOURCE_RESERVE_CU_15__VGPR_MASK                                                                 0x0000000FL
+#define SPI_RESOURCE_RESERVE_CU_15__SGPR_MASK                                                                 0x000000F0L
+#define SPI_RESOURCE_RESERVE_CU_15__LDS_MASK                                                                  0x00000F00L
+#define SPI_RESOURCE_RESERVE_CU_15__WAVES_MASK                                                                0x00007000L
+#define SPI_RESOURCE_RESERVE_CU_15__BARRIERS_MASK                                                             0x00078000L
+//SPI_RESOURCE_RESERVE_EN_CU_0
+#define SPI_RESOURCE_RESERVE_EN_CU_0__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_0__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_0__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_0__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_1
+#define SPI_RESOURCE_RESERVE_EN_CU_1__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_1__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_1__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_1__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_2
+#define SPI_RESOURCE_RESERVE_EN_CU_2__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_2__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_2__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_2__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_3
+#define SPI_RESOURCE_RESERVE_EN_CU_3__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_3__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_3__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_3__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_4
+#define SPI_RESOURCE_RESERVE_EN_CU_4__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_4__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_4__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_4__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_5
+#define SPI_RESOURCE_RESERVE_EN_CU_5__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_5__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_5__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_5__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_6
+#define SPI_RESOURCE_RESERVE_EN_CU_6__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_6__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_6__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_6__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_7
+#define SPI_RESOURCE_RESERVE_EN_CU_7__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_7__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_7__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_7__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_8
+#define SPI_RESOURCE_RESERVE_EN_CU_8__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_8__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_8__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_8__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_9
+#define SPI_RESOURCE_RESERVE_EN_CU_9__EN__SHIFT                                                               0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK__SHIFT                                                        0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK__SHIFT                                                       0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_9__EN_MASK                                                                 0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_9__TYPE_MASK_MASK                                                          0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_9__QUEUE_MASK_MASK                                                         0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_10
+#define SPI_RESOURCE_RESERVE_EN_CU_10__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_10__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_10__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_10__QUEUE_MASK_MASK                                                        0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_11
+#define SPI_RESOURCE_RESERVE_EN_CU_11__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_11__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_11__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_11__QUEUE_MASK_MASK                                                        0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_12
+#define SPI_RESOURCE_RESERVE_EN_CU_12__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_12__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_12__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_12__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_12__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_12__QUEUE_MASK_MASK                                                        0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_13
+#define SPI_RESOURCE_RESERVE_EN_CU_13__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_13__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_13__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_13__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_13__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_13__QUEUE_MASK_MASK                                                        0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_14
+#define SPI_RESOURCE_RESERVE_EN_CU_14__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_14__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_14__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_14__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_14__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_14__QUEUE_MASK_MASK                                                        0x00FF0000L
+//SPI_RESOURCE_RESERVE_EN_CU_15
+#define SPI_RESOURCE_RESERVE_EN_CU_15__EN__SHIFT                                                              0x0
+#define SPI_RESOURCE_RESERVE_EN_CU_15__TYPE_MASK__SHIFT                                                       0x1
+#define SPI_RESOURCE_RESERVE_EN_CU_15__QUEUE_MASK__SHIFT                                                      0x10
+#define SPI_RESOURCE_RESERVE_EN_CU_15__EN_MASK                                                                0x00000001L
+#define SPI_RESOURCE_RESERVE_EN_CU_15__TYPE_MASK_MASK                                                         0x0000FFFEL
+#define SPI_RESOURCE_RESERVE_EN_CU_15__QUEUE_MASK_MASK                                                        0x00FF0000L
+
+
+// addressBlock: gc_gfxudec
+//CP_EOP_DONE_ADDR_LO
+#define CP_EOP_DONE_ADDR_LO__ADDR_LO__SHIFT                                                                   0x2
+#define CP_EOP_DONE_ADDR_LO__ADDR_LO_MASK                                                                     0xFFFFFFFCL
+//CP_EOP_DONE_ADDR_HI
+#define CP_EOP_DONE_ADDR_HI__ADDR_HI__SHIFT                                                                   0x0
+#define CP_EOP_DONE_ADDR_HI__ADDR_HI_MASK                                                                     0x0000FFFFL
+//CP_EOP_DONE_DATA_LO
+#define CP_EOP_DONE_DATA_LO__DATA_LO__SHIFT                                                                   0x0
+#define CP_EOP_DONE_DATA_LO__DATA_LO_MASK                                                                     0xFFFFFFFFL
+//CP_EOP_DONE_DATA_HI
+#define CP_EOP_DONE_DATA_HI__DATA_HI__SHIFT                                                                   0x0
+#define CP_EOP_DONE_DATA_HI__DATA_HI_MASK                                                                     0xFFFFFFFFL
+//CP_EOP_LAST_FENCE_LO
+#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO__SHIFT                                                            0x0
+#define CP_EOP_LAST_FENCE_LO__LAST_FENCE_LO_MASK                                                              0xFFFFFFFFL
+//CP_EOP_LAST_FENCE_HI
+#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI__SHIFT                                                            0x0
+#define CP_EOP_LAST_FENCE_HI__LAST_FENCE_HI_MASK                                                              0xFFFFFFFFL
+//CP_PIPE_STATS_ADDR_LO
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO__SHIFT                                                      0x2
+#define CP_PIPE_STATS_ADDR_LO__PIPE_STATS_ADDR_LO_MASK                                                        0xFFFFFFFCL
+//CP_PIPE_STATS_ADDR_HI
+#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI__SHIFT                                                      0x0
+#define CP_PIPE_STATS_ADDR_HI__PIPE_STATS_ADDR_HI_MASK                                                        0x0000FFFFL
+//CP_VGT_IAVERT_COUNT_LO
+#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO__SHIFT                                                        0x0
+#define CP_VGT_IAVERT_COUNT_LO__IAVERT_COUNT_LO_MASK                                                          0xFFFFFFFFL
+//CP_VGT_IAVERT_COUNT_HI
+#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI__SHIFT                                                        0x0
+#define CP_VGT_IAVERT_COUNT_HI__IAVERT_COUNT_HI_MASK                                                          0xFFFFFFFFL
+//CP_VGT_IAPRIM_COUNT_LO
+#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO__SHIFT                                                        0x0
+#define CP_VGT_IAPRIM_COUNT_LO__IAPRIM_COUNT_LO_MASK                                                          0xFFFFFFFFL
+//CP_VGT_IAPRIM_COUNT_HI
+#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI__SHIFT                                                        0x0
+#define CP_VGT_IAPRIM_COUNT_HI__IAPRIM_COUNT_HI_MASK                                                          0xFFFFFFFFL
+//CP_VGT_GSPRIM_COUNT_LO
+#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO__SHIFT                                                        0x0
+#define CP_VGT_GSPRIM_COUNT_LO__GSPRIM_COUNT_LO_MASK                                                          0xFFFFFFFFL
+//CP_VGT_GSPRIM_COUNT_HI
+#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI__SHIFT                                                        0x0
+#define CP_VGT_GSPRIM_COUNT_HI__GSPRIM_COUNT_HI_MASK                                                          0xFFFFFFFFL
+//CP_VGT_VSINVOC_COUNT_LO
+#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_VSINVOC_COUNT_LO__VSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_VSINVOC_COUNT_HI
+#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_VSINVOC_COUNT_HI__VSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_GSINVOC_COUNT_LO
+#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_GSINVOC_COUNT_LO__GSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_GSINVOC_COUNT_HI
+#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_GSINVOC_COUNT_HI__GSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_HSINVOC_COUNT_LO
+#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_HSINVOC_COUNT_LO__HSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_HSINVOC_COUNT_HI
+#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_HSINVOC_COUNT_HI__HSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_DSINVOC_COUNT_LO
+#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_DSINVOC_COUNT_LO__DSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_DSINVOC_COUNT_HI
+#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_DSINVOC_COUNT_HI__DSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_PA_CINVOC_COUNT_LO
+#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO__SHIFT                                                         0x0
+#define CP_PA_CINVOC_COUNT_LO__CINVOC_COUNT_LO_MASK                                                           0xFFFFFFFFL
+//CP_PA_CINVOC_COUNT_HI
+#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI__SHIFT                                                         0x0
+#define CP_PA_CINVOC_COUNT_HI__CINVOC_COUNT_HI_MASK                                                           0xFFFFFFFFL
+//CP_PA_CPRIM_COUNT_LO
+#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO__SHIFT                                                           0x0
+#define CP_PA_CPRIM_COUNT_LO__CPRIM_COUNT_LO_MASK                                                             0xFFFFFFFFL
+//CP_PA_CPRIM_COUNT_HI
+#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI__SHIFT                                                           0x0
+#define CP_PA_CPRIM_COUNT_HI__CPRIM_COUNT_HI_MASK                                                             0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT0_LO
+#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO__SHIFT                                                     0x0
+#define CP_SC_PSINVOC_COUNT0_LO__PSINVOC_COUNT0_LO_MASK                                                       0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT0_HI
+#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI__SHIFT                                                     0x0
+#define CP_SC_PSINVOC_COUNT0_HI__PSINVOC_COUNT0_HI_MASK                                                       0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT1_LO
+#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE__SHIFT                                                              0x0
+#define CP_SC_PSINVOC_COUNT1_LO__OBSOLETE_MASK                                                                0xFFFFFFFFL
+//CP_SC_PSINVOC_COUNT1_HI
+#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE__SHIFT                                                              0x0
+#define CP_SC_PSINVOC_COUNT1_HI__OBSOLETE_MASK                                                                0xFFFFFFFFL
+//CP_VGT_CSINVOC_COUNT_LO
+#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_CSINVOC_COUNT_LO__CSINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_CSINVOC_COUNT_HI
+#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_CSINVOC_COUNT_HI__CSINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_VGT_ASINVOC_COUNT_LO
+#define CP_VGT_ASINVOC_COUNT_LO__ASINVOC_COUNT_LO__SHIFT                                                      0x0
+#define CP_VGT_ASINVOC_COUNT_LO__ASINVOC_COUNT_LO_MASK                                                        0xFFFFFFFFL
+//CP_VGT_ASINVOC_COUNT_HI
+#define CP_VGT_ASINVOC_COUNT_HI__ASINVOC_COUNT_HI__SHIFT                                                      0x0
+#define CP_VGT_ASINVOC_COUNT_HI__ASINVOC_COUNT_HI_MASK                                                        0xFFFFFFFFL
+//CP_PIPE_STATS_CONTROL
+#define CP_PIPE_STATS_CONTROL__CACHE_POLICY__SHIFT                                                            0x19
+#define CP_PIPE_STATS_CONTROL__CACHE_POLICY_MASK                                                              0x06000000L
+//SCRATCH_REG0
+#define SCRATCH_REG0__SCRATCH_REG0__SHIFT                                                                     0x0
+#define SCRATCH_REG0__SCRATCH_REG0_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG1
+#define SCRATCH_REG1__SCRATCH_REG1__SHIFT                                                                     0x0
+#define SCRATCH_REG1__SCRATCH_REG1_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG2
+#define SCRATCH_REG2__SCRATCH_REG2__SHIFT                                                                     0x0
+#define SCRATCH_REG2__SCRATCH_REG2_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG3
+#define SCRATCH_REG3__SCRATCH_REG3__SHIFT                                                                     0x0
+#define SCRATCH_REG3__SCRATCH_REG3_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG4
+#define SCRATCH_REG4__SCRATCH_REG4__SHIFT                                                                     0x0
+#define SCRATCH_REG4__SCRATCH_REG4_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG5
+#define SCRATCH_REG5__SCRATCH_REG5__SHIFT                                                                     0x0
+#define SCRATCH_REG5__SCRATCH_REG5_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG6
+#define SCRATCH_REG6__SCRATCH_REG6__SHIFT                                                                     0x0
+#define SCRATCH_REG6__SCRATCH_REG6_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG7
+#define SCRATCH_REG7__SCRATCH_REG7__SHIFT                                                                     0x0
+#define SCRATCH_REG7__SCRATCH_REG7_MASK                                                                       0xFFFFFFFFL
+//SCRATCH_REG_ATOMIC
+#define SCRATCH_REG_ATOMIC__IMMED__SHIFT                                                                      0x0
+#define SCRATCH_REG_ATOMIC__ID__SHIFT                                                                         0x18
+#define SCRATCH_REG_ATOMIC__reserved27__SHIFT                                                                 0x1b
+#define SCRATCH_REG_ATOMIC__OP__SHIFT                                                                         0x1c
+#define SCRATCH_REG_ATOMIC__reserved31__SHIFT                                                                 0x1f
+#define SCRATCH_REG_ATOMIC__IMMED_MASK                                                                        0x00FFFFFFL
+#define SCRATCH_REG_ATOMIC__ID_MASK                                                                           0x07000000L
+#define SCRATCH_REG_ATOMIC__reserved27_MASK                                                                   0x08000000L
+#define SCRATCH_REG_ATOMIC__OP_MASK                                                                           0x70000000L
+#define SCRATCH_REG_ATOMIC__reserved31_MASK                                                                   0x80000000L
+//SCRATCH_REG_CMPSWAP_ATOMIC
+#define SCRATCH_REG_CMPSWAP_ATOMIC__IMMED_COMPARE__SHIFT                                                      0x0
+#define SCRATCH_REG_CMPSWAP_ATOMIC__IMMED_REPLACE__SHIFT                                                      0xc
+#define SCRATCH_REG_CMPSWAP_ATOMIC__ID__SHIFT                                                                 0x18
+#define SCRATCH_REG_CMPSWAP_ATOMIC__reserved27__SHIFT                                                         0x1b
+#define SCRATCH_REG_CMPSWAP_ATOMIC__OP__SHIFT                                                                 0x1c
+#define SCRATCH_REG_CMPSWAP_ATOMIC__reserved31__SHIFT                                                         0x1f
+#define SCRATCH_REG_CMPSWAP_ATOMIC__IMMED_COMPARE_MASK                                                        0x00000FFFL
+#define SCRATCH_REG_CMPSWAP_ATOMIC__IMMED_REPLACE_MASK                                                        0x00FFF000L
+#define SCRATCH_REG_CMPSWAP_ATOMIC__ID_MASK                                                                   0x07000000L
+#define SCRATCH_REG_CMPSWAP_ATOMIC__reserved27_MASK                                                           0x08000000L
+#define SCRATCH_REG_CMPSWAP_ATOMIC__OP_MASK                                                                   0x70000000L
+#define SCRATCH_REG_CMPSWAP_ATOMIC__reserved31_MASK                                                           0x80000000L
+//CP_APPEND_DDID_CNT
+#define CP_APPEND_DDID_CNT__DATA__SHIFT                                                                       0x0
+#define CP_APPEND_DDID_CNT__DATA_MASK                                                                         0x000000FFL
+//CP_APPEND_DATA_HI
+#define CP_APPEND_DATA_HI__DATA__SHIFT                                                                        0x0
+#define CP_APPEND_DATA_HI__DATA_MASK                                                                          0xFFFFFFFFL
+//CP_APPEND_LAST_CS_FENCE_HI
+#define CP_APPEND_LAST_CS_FENCE_HI__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_CS_FENCE_HI__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_APPEND_LAST_PS_FENCE_HI
+#define CP_APPEND_LAST_PS_FENCE_HI__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_PS_FENCE_HI__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_PFP_ATOMIC_PREOP_LO
+#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                        0x0
+#define CP_PFP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                          0xFFFFFFFFL
+//CP_PFP_ATOMIC_PREOP_HI
+#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                        0x0
+#define CP_PFP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                          0xFFFFFFFFL
+//CP_PFP_GDS_ATOMIC0_PREOP_LO
+#define CP_PFP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT                                              0x0
+#define CP_PFP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK                                                0xFFFFFFFFL
+//CP_PFP_GDS_ATOMIC0_PREOP_HI
+#define CP_PFP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT                                              0x0
+#define CP_PFP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK                                                0xFFFFFFFFL
+//CP_PFP_GDS_ATOMIC1_PREOP_LO
+#define CP_PFP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT                                              0x0
+#define CP_PFP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK                                                0xFFFFFFFFL
+//CP_PFP_GDS_ATOMIC1_PREOP_HI
+#define CP_PFP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT                                              0x0
+#define CP_PFP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK                                                0xFFFFFFFFL
+//CP_APPEND_ADDR_LO
+#define CP_APPEND_ADDR_LO__MEM_ADDR_LO__SHIFT                                                                 0x2
+#define CP_APPEND_ADDR_LO__MEM_ADDR_LO_MASK                                                                   0xFFFFFFFCL
+//CP_APPEND_ADDR_HI
+#define CP_APPEND_ADDR_HI__MEM_ADDR_HI__SHIFT                                                                 0x0
+#define CP_APPEND_ADDR_HI__CS_PS_SEL__SHIFT                                                                   0x10
+#define CP_APPEND_ADDR_HI__FENCE_SIZE__SHIFT                                                                  0x12
+#define CP_APPEND_ADDR_HI__PWS_ENABLE__SHIFT                                                                  0x13
+#define CP_APPEND_ADDR_HI__CACHE_POLICY__SHIFT                                                                0x19
+#define CP_APPEND_ADDR_HI__COMMAND__SHIFT                                                                     0x1d
+#define CP_APPEND_ADDR_HI__MEM_ADDR_HI_MASK                                                                   0x0000FFFFL
+#define CP_APPEND_ADDR_HI__CS_PS_SEL_MASK                                                                     0x00030000L
+#define CP_APPEND_ADDR_HI__FENCE_SIZE_MASK                                                                    0x00040000L
+#define CP_APPEND_ADDR_HI__PWS_ENABLE_MASK                                                                    0x00080000L
+#define CP_APPEND_ADDR_HI__CACHE_POLICY_MASK                                                                  0x06000000L
+#define CP_APPEND_ADDR_HI__COMMAND_MASK                                                                       0xE0000000L
+//CP_APPEND_DATA
+#define CP_APPEND_DATA__DATA__SHIFT                                                                           0x0
+#define CP_APPEND_DATA__DATA_MASK                                                                             0xFFFFFFFFL
+//CP_APPEND_DATA_LO
+#define CP_APPEND_DATA_LO__DATA__SHIFT                                                                        0x0
+#define CP_APPEND_DATA_LO__DATA_MASK                                                                          0xFFFFFFFFL
+//CP_APPEND_LAST_CS_FENCE
+#define CP_APPEND_LAST_CS_FENCE__LAST_FENCE__SHIFT                                                            0x0
+#define CP_APPEND_LAST_CS_FENCE__LAST_FENCE_MASK                                                              0xFFFFFFFFL
+//CP_APPEND_LAST_CS_FENCE_LO
+#define CP_APPEND_LAST_CS_FENCE_LO__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_CS_FENCE_LO__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_APPEND_LAST_PS_FENCE
+#define CP_APPEND_LAST_PS_FENCE__LAST_FENCE__SHIFT                                                            0x0
+#define CP_APPEND_LAST_PS_FENCE__LAST_FENCE_MASK                                                              0xFFFFFFFFL
+//CP_APPEND_LAST_PS_FENCE_LO
+#define CP_APPEND_LAST_PS_FENCE_LO__LAST_FENCE__SHIFT                                                         0x0
+#define CP_APPEND_LAST_PS_FENCE_LO__LAST_FENCE_MASK                                                           0xFFFFFFFFL
+//CP_ATOMIC_PREOP_LO
+#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                            0x0
+#define CP_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                              0xFFFFFFFFL
+//CP_ME_ATOMIC_PREOP_LO
+#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO__SHIFT                                                         0x0
+#define CP_ME_ATOMIC_PREOP_LO__ATOMIC_PREOP_LO_MASK                                                           0xFFFFFFFFL
+//CP_ATOMIC_PREOP_HI
+#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                            0x0
+#define CP_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                              0xFFFFFFFFL
+//CP_ME_ATOMIC_PREOP_HI
+#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI__SHIFT                                                         0x0
+#define CP_ME_ATOMIC_PREOP_HI__ATOMIC_PREOP_HI_MASK                                                           0xFFFFFFFFL
+//CP_GDS_ATOMIC0_PREOP_LO
+#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT                                                  0x0
+#define CP_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK                                                    0xFFFFFFFFL
+//CP_ME_GDS_ATOMIC0_PREOP_LO
+#define CP_ME_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO__SHIFT                                               0x0
+#define CP_ME_GDS_ATOMIC0_PREOP_LO__GDS_ATOMIC0_PREOP_LO_MASK                                                 0xFFFFFFFFL
+//CP_GDS_ATOMIC0_PREOP_HI
+#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT                                                  0x0
+#define CP_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK                                                    0xFFFFFFFFL
+//CP_ME_GDS_ATOMIC0_PREOP_HI
+#define CP_ME_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI__SHIFT                                               0x0
+#define CP_ME_GDS_ATOMIC0_PREOP_HI__GDS_ATOMIC0_PREOP_HI_MASK                                                 0xFFFFFFFFL
+//CP_GDS_ATOMIC1_PREOP_LO
+#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT                                                  0x0
+#define CP_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK                                                    0xFFFFFFFFL
+//CP_ME_GDS_ATOMIC1_PREOP_LO
+#define CP_ME_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO__SHIFT                                               0x0
+#define CP_ME_GDS_ATOMIC1_PREOP_LO__GDS_ATOMIC1_PREOP_LO_MASK                                                 0xFFFFFFFFL
+//CP_GDS_ATOMIC1_PREOP_HI
+#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT                                                  0x0
+#define CP_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK                                                    0xFFFFFFFFL
+//CP_ME_GDS_ATOMIC1_PREOP_HI
+#define CP_ME_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI__SHIFT                                               0x0
+#define CP_ME_GDS_ATOMIC1_PREOP_HI__GDS_ATOMIC1_PREOP_HI_MASK                                                 0xFFFFFFFFL
+//CP_ME_MC_WADDR_LO
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO__SHIFT                                                              0x2
+#define CP_ME_MC_WADDR_LO__ME_MC_WADDR_LO_MASK                                                                0xFFFFFFFCL
+//CP_ME_MC_WADDR_HI
+#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI__SHIFT                                                              0x0
+#define CP_ME_MC_WADDR_HI__WRITE_CONFIRM__SHIFT                                                               0x11
+#define CP_ME_MC_WADDR_HI__WRITE64__SHIFT                                                                     0x12
+#define CP_ME_MC_WADDR_HI__CACHE_POLICY__SHIFT                                                                0x16
+#define CP_ME_MC_WADDR_HI__VMID__SHIFT                                                                        0x18
+#define CP_ME_MC_WADDR_HI__RINGID__SHIFT                                                                      0x1c
+#define CP_ME_MC_WADDR_HI__PRIVILEGE__SHIFT                                                                   0x1f
+#define CP_ME_MC_WADDR_HI__ME_MC_WADDR_HI_MASK                                                                0x0000FFFFL
+#define CP_ME_MC_WADDR_HI__WRITE_CONFIRM_MASK                                                                 0x00020000L
+#define CP_ME_MC_WADDR_HI__WRITE64_MASK                                                                       0x00040000L
+#define CP_ME_MC_WADDR_HI__CACHE_POLICY_MASK                                                                  0x00C00000L
+#define CP_ME_MC_WADDR_HI__VMID_MASK                                                                          0x0F000000L
+#define CP_ME_MC_WADDR_HI__RINGID_MASK                                                                        0x30000000L
+#define CP_ME_MC_WADDR_HI__PRIVILEGE_MASK                                                                     0x80000000L
+//CP_ME_MC_WDATA_LO
+#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO__SHIFT                                                              0x0
+#define CP_ME_MC_WDATA_LO__ME_MC_WDATA_LO_MASK                                                                0xFFFFFFFFL
+//CP_ME_MC_WDATA_HI
+#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI__SHIFT                                                              0x0
+#define CP_ME_MC_WDATA_HI__ME_MC_WDATA_HI_MASK                                                                0xFFFFFFFFL
+//CP_ME_MC_RADDR_LO
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO__SHIFT                                                              0x2
+#define CP_ME_MC_RADDR_LO__ME_MC_RADDR_LO_MASK                                                                0xFFFFFFFCL
+//CP_ME_MC_RADDR_HI
+#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI__SHIFT                                                              0x0
+#define CP_ME_MC_RADDR_HI__SIZE__SHIFT                                                                        0x10
+#define CP_ME_MC_RADDR_HI__CACHE_POLICY__SHIFT                                                                0x16
+#define CP_ME_MC_RADDR_HI__VMID__SHIFT                                                                        0x18
+#define CP_ME_MC_RADDR_HI__PRIVILEGE__SHIFT                                                                   0x1f
+#define CP_ME_MC_RADDR_HI__ME_MC_RADDR_HI_MASK                                                                0x0000FFFFL
+#define CP_ME_MC_RADDR_HI__SIZE_MASK                                                                          0x000F0000L
+#define CP_ME_MC_RADDR_HI__CACHE_POLICY_MASK                                                                  0x00C00000L
+#define CP_ME_MC_RADDR_HI__VMID_MASK                                                                          0x0F000000L
+#define CP_ME_MC_RADDR_HI__PRIVILEGE_MASK                                                                     0x80000000L
+//CP_SEM_WAIT_TIMER
+#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER__SHIFT                                                              0x0
+#define CP_SEM_WAIT_TIMER__SEM_WAIT_TIMER_MASK                                                                0xFFFFFFFFL
+//CP_SIG_SEM_ADDR_LO
+#define CP_SIG_SEM_ADDR_LO__SEM_PRIV__SHIFT                                                                   0x0
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT                                                                0x3
+#define CP_SIG_SEM_ADDR_LO__SEM_PRIV_MASK                                                                     0x00000001L
+#define CP_SIG_SEM_ADDR_LO__SEM_ADDR_LO_MASK                                                                  0xFFFFFFF8L
+//CP_SIG_SEM_ADDR_HI
+#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT                                                                0x0
+#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT                                                            0x10
+#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT                                                            0x14
+#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT                                                            0x18
+#define CP_SIG_SEM_ADDR_HI__SEM_SELECT__SHIFT                                                                 0x1d
+#define CP_SIG_SEM_ADDR_HI__SEM_ADDR_HI_MASK                                                                  0x0000FFFFL
+#define CP_SIG_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK                                                              0x00010000L
+#define CP_SIG_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK                                                              0x00100000L
+#define CP_SIG_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK                                                              0x03000000L
+#define CP_SIG_SEM_ADDR_HI__SEM_SELECT_MASK                                                                   0xE0000000L
+//CP_WAIT_REG_MEM_TIMEOUT
+#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT__SHIFT                                                  0x0
+#define CP_WAIT_REG_MEM_TIMEOUT__WAIT_REG_MEM_TIMEOUT_MASK                                                    0xFFFFFFFFL
+//CP_WAIT_SEM_ADDR_LO
+#define CP_WAIT_SEM_ADDR_LO__SEM_PRIV__SHIFT                                                                  0x0
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO__SHIFT                                                               0x3
+#define CP_WAIT_SEM_ADDR_LO__SEM_PRIV_MASK                                                                    0x00000001L
+#define CP_WAIT_SEM_ADDR_LO__SEM_ADDR_LO_MASK                                                                 0xFFFFFFF8L
+//CP_WAIT_SEM_ADDR_HI
+#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI__SHIFT                                                               0x0
+#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX__SHIFT                                                           0x10
+#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE__SHIFT                                                           0x14
+#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE__SHIFT                                                           0x18
+#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT__SHIFT                                                                0x1d
+#define CP_WAIT_SEM_ADDR_HI__SEM_ADDR_HI_MASK                                                                 0x0000FFFFL
+#define CP_WAIT_SEM_ADDR_HI__SEM_USE_MAILBOX_MASK                                                             0x00010000L
+#define CP_WAIT_SEM_ADDR_HI__SEM_SIGNAL_TYPE_MASK                                                             0x00100000L
+#define CP_WAIT_SEM_ADDR_HI__SEM_CLIENT_CODE_MASK                                                             0x03000000L
+#define CP_WAIT_SEM_ADDR_HI__SEM_SELECT_MASK                                                                  0xE0000000L
+//CP_DMA_PFP_CONTROL
+#define CP_DMA_PFP_CONTROL__VMID__SHIFT                                                                       0x0
+#define CP_DMA_PFP_CONTROL__TMZ__SHIFT                                                                        0x4
+#define CP_DMA_PFP_CONTROL__MEMLOG_CLEAR__SHIFT                                                               0xa
+#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY__SHIFT                                                           0xd
+#define CP_DMA_PFP_CONTROL__SRC_VOLATLE__SHIFT                                                                0xf
+#define CP_DMA_PFP_CONTROL__DST_SELECT__SHIFT                                                                 0x14
+#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY__SHIFT                                                           0x19
+#define CP_DMA_PFP_CONTROL__DST_VOLATLE__SHIFT                                                                0x1b
+#define CP_DMA_PFP_CONTROL__SRC_SELECT__SHIFT                                                                 0x1d
+#define CP_DMA_PFP_CONTROL__VMID_MASK                                                                         0x0000000FL
+#define CP_DMA_PFP_CONTROL__TMZ_MASK                                                                          0x00000010L
+#define CP_DMA_PFP_CONTROL__MEMLOG_CLEAR_MASK                                                                 0x00000400L
+#define CP_DMA_PFP_CONTROL__SRC_CACHE_POLICY_MASK                                                             0x00006000L
+#define CP_DMA_PFP_CONTROL__SRC_VOLATLE_MASK                                                                  0x00008000L
+#define CP_DMA_PFP_CONTROL__DST_SELECT_MASK                                                                   0x00300000L
+#define CP_DMA_PFP_CONTROL__DST_CACHE_POLICY_MASK                                                             0x06000000L
+#define CP_DMA_PFP_CONTROL__DST_VOLATLE_MASK                                                                  0x08000000L
+#define CP_DMA_PFP_CONTROL__SRC_SELECT_MASK                                                                   0x60000000L
+//CP_DMA_ME_CONTROL
+#define CP_DMA_ME_CONTROL__VMID__SHIFT                                                                        0x0
+#define CP_DMA_ME_CONTROL__TMZ__SHIFT                                                                         0x4
+#define CP_DMA_ME_CONTROL__MEMLOG_CLEAR__SHIFT                                                                0xa
+#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY__SHIFT                                                            0xd
+#define CP_DMA_ME_CONTROL__SRC_VOLATLE__SHIFT                                                                 0xf
+#define CP_DMA_ME_CONTROL__DST_SELECT__SHIFT                                                                  0x14
+#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY__SHIFT                                                            0x19
+#define CP_DMA_ME_CONTROL__DST_VOLATLE__SHIFT                                                                 0x1b
+#define CP_DMA_ME_CONTROL__SRC_SELECT__SHIFT                                                                  0x1d
+#define CP_DMA_ME_CONTROL__VMID_MASK                                                                          0x0000000FL
+#define CP_DMA_ME_CONTROL__TMZ_MASK                                                                           0x00000010L
+#define CP_DMA_ME_CONTROL__MEMLOG_CLEAR_MASK                                                                  0x00000400L
+#define CP_DMA_ME_CONTROL__SRC_CACHE_POLICY_MASK                                                              0x00006000L
+#define CP_DMA_ME_CONTROL__SRC_VOLATLE_MASK                                                                   0x00008000L
+#define CP_DMA_ME_CONTROL__DST_SELECT_MASK                                                                    0x00300000L
+#define CP_DMA_ME_CONTROL__DST_CACHE_POLICY_MASK                                                              0x06000000L
+#define CP_DMA_ME_CONTROL__DST_VOLATLE_MASK                                                                   0x08000000L
+#define CP_DMA_ME_CONTROL__SRC_SELECT_MASK                                                                    0x60000000L
+//CP_DMA_ME_SRC_ADDR
+#define CP_DMA_ME_SRC_ADDR__SRC_ADDR__SHIFT                                                                   0x0
+#define CP_DMA_ME_SRC_ADDR__SRC_ADDR_MASK                                                                     0xFFFFFFFFL
+//CP_DMA_ME_SRC_ADDR_HI
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT                                                             0x0
+#define CP_DMA_ME_SRC_ADDR_HI__SRC_ADDR_HI_MASK                                                               0x0000FFFFL
+//CP_DMA_ME_DST_ADDR
+#define CP_DMA_ME_DST_ADDR__DST_ADDR__SHIFT                                                                   0x0
+#define CP_DMA_ME_DST_ADDR__DST_ADDR_MASK                                                                     0xFFFFFFFFL
+//CP_DMA_ME_DST_ADDR_HI
+#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI__SHIFT                                                             0x0
+#define CP_DMA_ME_DST_ADDR_HI__DST_ADDR_HI_MASK                                                               0x0000FFFFL
+//CP_DMA_ME_COMMAND
+#define CP_DMA_ME_COMMAND__BYTE_COUNT__SHIFT                                                                  0x0
+#define CP_DMA_ME_COMMAND__SAS__SHIFT                                                                         0x1a
+#define CP_DMA_ME_COMMAND__DAS__SHIFT                                                                         0x1b
+#define CP_DMA_ME_COMMAND__SAIC__SHIFT                                                                        0x1c
+#define CP_DMA_ME_COMMAND__DAIC__SHIFT                                                                        0x1d
+#define CP_DMA_ME_COMMAND__RAW_WAIT__SHIFT                                                                    0x1e
+#define CP_DMA_ME_COMMAND__DIS_WC__SHIFT                                                                      0x1f
+#define CP_DMA_ME_COMMAND__BYTE_COUNT_MASK                                                                    0x03FFFFFFL
+#define CP_DMA_ME_COMMAND__SAS_MASK                                                                           0x04000000L
+#define CP_DMA_ME_COMMAND__DAS_MASK                                                                           0x08000000L
+#define CP_DMA_ME_COMMAND__SAIC_MASK                                                                          0x10000000L
+#define CP_DMA_ME_COMMAND__DAIC_MASK                                                                          0x20000000L
+#define CP_DMA_ME_COMMAND__RAW_WAIT_MASK                                                                      0x40000000L
+#define CP_DMA_ME_COMMAND__DIS_WC_MASK                                                                        0x80000000L
+//CP_DMA_PFP_SRC_ADDR
+#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR__SHIFT                                                                  0x0
+#define CP_DMA_PFP_SRC_ADDR__SRC_ADDR_MASK                                                                    0xFFFFFFFFL
+//CP_DMA_PFP_SRC_ADDR_HI
+#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI__SHIFT                                                            0x0
+#define CP_DMA_PFP_SRC_ADDR_HI__SRC_ADDR_HI_MASK                                                              0x0000FFFFL
+//CP_DMA_PFP_DST_ADDR
+#define CP_DMA_PFP_DST_ADDR__DST_ADDR__SHIFT                                                                  0x0
+#define CP_DMA_PFP_DST_ADDR__DST_ADDR_MASK                                                                    0xFFFFFFFFL
+//CP_DMA_PFP_DST_ADDR_HI
+#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI__SHIFT                                                            0x0
+#define CP_DMA_PFP_DST_ADDR_HI__DST_ADDR_HI_MASK                                                              0x0000FFFFL
+//CP_DMA_PFP_COMMAND
+#define CP_DMA_PFP_COMMAND__BYTE_COUNT__SHIFT                                                                 0x0
+#define CP_DMA_PFP_COMMAND__SAS__SHIFT                                                                        0x1a
+#define CP_DMA_PFP_COMMAND__DAS__SHIFT                                                                        0x1b
+#define CP_DMA_PFP_COMMAND__SAIC__SHIFT                                                                       0x1c
+#define CP_DMA_PFP_COMMAND__DAIC__SHIFT                                                                       0x1d
+#define CP_DMA_PFP_COMMAND__RAW_WAIT__SHIFT                                                                   0x1e
+#define CP_DMA_PFP_COMMAND__DIS_WC__SHIFT                                                                     0x1f
+#define CP_DMA_PFP_COMMAND__BYTE_COUNT_MASK                                                                   0x03FFFFFFL
+#define CP_DMA_PFP_COMMAND__SAS_MASK                                                                          0x04000000L
+#define CP_DMA_PFP_COMMAND__DAS_MASK                                                                          0x08000000L
+#define CP_DMA_PFP_COMMAND__SAIC_MASK                                                                         0x10000000L
+#define CP_DMA_PFP_COMMAND__DAIC_MASK                                                                         0x20000000L
+#define CP_DMA_PFP_COMMAND__RAW_WAIT_MASK                                                                     0x40000000L
+#define CP_DMA_PFP_COMMAND__DIS_WC_MASK                                                                       0x80000000L
+//CP_DMA_CNTL
+#define CP_DMA_CNTL__UTCL1_FAULT_CONTROL__SHIFT                                                               0x0
+#define CP_DMA_CNTL__WATCH_CONTROL__SHIFT                                                                     0x1
+#define CP_DMA_CNTL__MIN_AVAILSZ__SHIFT                                                                       0x4
+#define CP_DMA_CNTL__BUFFER_DEPTH__SHIFT                                                                      0x10
+#define CP_DMA_CNTL__PIO_FIFO_EMPTY__SHIFT                                                                    0x1c
+#define CP_DMA_CNTL__PIO_FIFO_FULL__SHIFT                                                                     0x1d
+#define CP_DMA_CNTL__PIO_COUNT__SHIFT                                                                         0x1e
+#define CP_DMA_CNTL__UTCL1_FAULT_CONTROL_MASK                                                                 0x00000001L
+#define CP_DMA_CNTL__WATCH_CONTROL_MASK                                                                       0x00000002L
+#define CP_DMA_CNTL__MIN_AVAILSZ_MASK                                                                         0x00000030L
+#define CP_DMA_CNTL__BUFFER_DEPTH_MASK                                                                        0x01FF0000L
+#define CP_DMA_CNTL__PIO_FIFO_EMPTY_MASK                                                                      0x10000000L
+#define CP_DMA_CNTL__PIO_FIFO_FULL_MASK                                                                       0x20000000L
+#define CP_DMA_CNTL__PIO_COUNT_MASK                                                                           0xC0000000L
+//CP_DMA_READ_TAGS
+#define CP_DMA_READ_TAGS__DMA_READ_TAG__SHIFT                                                                 0x0
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID__SHIFT                                                           0x1c
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_MASK                                                                   0x03FFFFFFL
+#define CP_DMA_READ_TAGS__DMA_READ_TAG_VALID_MASK                                                             0x10000000L
+//CP_PFP_IB_CONTROL
+#define CP_PFP_IB_CONTROL__IB_EN__SHIFT                                                                       0x0
+#define CP_PFP_IB_CONTROL__IB_EN_MASK                                                                         0x000000FFL
+//CP_PFP_LOAD_CONTROL
+#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN__SHIFT                                                             0x0
+#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN__SHIFT                                                               0x1
+#define CP_PFP_LOAD_CONTROL__UCONFIG_REG_EN__SHIFT                                                            0xf
+#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN__SHIFT                                                             0x10
+#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN__SHIFT                                                              0x18
+#define CP_PFP_LOAD_CONTROL__LOAD_ORDINAL__SHIFT                                                              0x1f
+#define CP_PFP_LOAD_CONTROL__CONFIG_REG_EN_MASK                                                               0x00000001L
+#define CP_PFP_LOAD_CONTROL__CNTX_REG_EN_MASK                                                                 0x00000002L
+#define CP_PFP_LOAD_CONTROL__UCONFIG_REG_EN_MASK                                                              0x00008000L
+#define CP_PFP_LOAD_CONTROL__SH_GFX_REG_EN_MASK                                                               0x00010000L
+#define CP_PFP_LOAD_CONTROL__SH_CS_REG_EN_MASK                                                                0x01000000L
+#define CP_PFP_LOAD_CONTROL__LOAD_ORDINAL_MASK                                                                0x80000000L
+//CP_SCRATCH_INDEX
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                                0x0
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE__SHIFT                                                     0x1f
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                                  0x000001FFL
+#define CP_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE_MASK                                                       0x80000000L
+//CP_SCRATCH_DATA
+#define CP_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                                  0x0
+#define CP_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                    0xFFFFFFFFL
+//CP_RB_OFFSET
+#define CP_RB_OFFSET__RB_OFFSET__SHIFT                                                                        0x0
+#define CP_RB_OFFSET__RB_OFFSET_MASK                                                                          0x000FFFFFL
+//CP_IB1_OFFSET
+#define CP_IB1_OFFSET__IB1_OFFSET__SHIFT                                                                      0x0
+#define CP_IB1_OFFSET__IB1_OFFSET_MASK                                                                        0x000FFFFFL
+//CP_IB2_OFFSET
+#define CP_IB2_OFFSET__IB2_OFFSET__SHIFT                                                                      0x0
+#define CP_IB2_OFFSET__IB2_OFFSET_MASK                                                                        0x000FFFFFL
+//CP_IB1_PREAMBLE_BEGIN
+#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN__SHIFT                                                      0x0
+#define CP_IB1_PREAMBLE_BEGIN__IB1_PREAMBLE_BEGIN_MASK                                                        0x000FFFFFL
+//CP_IB1_PREAMBLE_END
+#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END__SHIFT                                                          0x0
+#define CP_IB1_PREAMBLE_END__IB1_PREAMBLE_END_MASK                                                            0x000FFFFFL
+//CP_IB2_PREAMBLE_BEGIN
+#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN__SHIFT                                                      0x0
+#define CP_IB2_PREAMBLE_BEGIN__IB2_PREAMBLE_BEGIN_MASK                                                        0x000FFFFFL
+//CP_IB2_PREAMBLE_END
+#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END__SHIFT                                                          0x0
+#define CP_IB2_PREAMBLE_END__IB2_PREAMBLE_END_MASK                                                            0x000FFFFFL
+//CP_DMA_ME_CMD_ADDR_LO
+#define CP_DMA_ME_CMD_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_DMA_ME_CMD_ADDR_LO__ADDR_LO__SHIFT                                                                 0x2
+#define CP_DMA_ME_CMD_ADDR_LO__RSVD_MASK                                                                      0x00000003L
+#define CP_DMA_ME_CMD_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFFFCL
+//CP_DMA_ME_CMD_ADDR_HI
+#define CP_DMA_ME_CMD_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_DMA_ME_CMD_ADDR_HI__RSVD__SHIFT                                                                    0x10
+#define CP_DMA_ME_CMD_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
+#define CP_DMA_ME_CMD_ADDR_HI__RSVD_MASK                                                                      0xFFFF0000L
+//CP_DMA_PFP_CMD_ADDR_LO
+#define CP_DMA_PFP_CMD_ADDR_LO__RSVD__SHIFT                                                                   0x0
+#define CP_DMA_PFP_CMD_ADDR_LO__ADDR_LO__SHIFT                                                                0x2
+#define CP_DMA_PFP_CMD_ADDR_LO__RSVD_MASK                                                                     0x00000003L
+#define CP_DMA_PFP_CMD_ADDR_LO__ADDR_LO_MASK                                                                  0xFFFFFFFCL
+//CP_DMA_PFP_CMD_ADDR_HI
+#define CP_DMA_PFP_CMD_ADDR_HI__ADDR_HI__SHIFT                                                                0x0
+#define CP_DMA_PFP_CMD_ADDR_HI__RSVD__SHIFT                                                                   0x10
+#define CP_DMA_PFP_CMD_ADDR_HI__ADDR_HI_MASK                                                                  0x0000FFFFL
+#define CP_DMA_PFP_CMD_ADDR_HI__RSVD_MASK                                                                     0xFFFF0000L
+//CP_APPEND_CMD_ADDR_LO
+#define CP_APPEND_CMD_ADDR_LO__RSVD__SHIFT                                                                    0x0
+#define CP_APPEND_CMD_ADDR_LO__ADDR_LO__SHIFT                                                                 0x2
+#define CP_APPEND_CMD_ADDR_LO__RSVD_MASK                                                                      0x00000003L
+#define CP_APPEND_CMD_ADDR_LO__ADDR_LO_MASK                                                                   0xFFFFFFFCL
+//CP_APPEND_CMD_ADDR_HI
+#define CP_APPEND_CMD_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_APPEND_CMD_ADDR_HI__RSVD__SHIFT                                                                    0x10
+#define CP_APPEND_CMD_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
+#define CP_APPEND_CMD_ADDR_HI__RSVD_MASK                                                                      0xFFFF0000L
+//UCONFIG_RESERVED_REG0
+#define UCONFIG_RESERVED_REG0__DATA__SHIFT                                                                    0x0
+#define UCONFIG_RESERVED_REG0__DATA_MASK                                                                      0xFFFFFFFFL
+//UCONFIG_RESERVED_REG1
+#define UCONFIG_RESERVED_REG1__DATA__SHIFT                                                                    0x0
+#define UCONFIG_RESERVED_REG1__DATA_MASK                                                                      0xFFFFFFFFL
+//CP_PA_MSPRIM_COUNT_LO
+#define CP_PA_MSPRIM_COUNT_LO__MSPRIM_COUNT_LO__SHIFT                                                         0x0
+#define CP_PA_MSPRIM_COUNT_LO__MSPRIM_COUNT_LO_MASK                                                           0xFFFFFFFFL
+//CP_PA_MSPRIM_COUNT_HI
+#define CP_PA_MSPRIM_COUNT_HI__MSPRIM_COUNT_HI__SHIFT                                                         0x0
+#define CP_PA_MSPRIM_COUNT_HI__MSPRIM_COUNT_HI_MASK                                                           0xFFFFFFFFL
+//CP_GE_MSINVOC_COUNT_LO
+#define CP_GE_MSINVOC_COUNT_LO__MSINVOC_COUNT_LO__SHIFT                                                       0x0
+#define CP_GE_MSINVOC_COUNT_LO__MSINVOC_COUNT_LO_MASK                                                         0xFFFFFFFFL
+//CP_GE_MSINVOC_COUNT_HI
+#define CP_GE_MSINVOC_COUNT_HI__MSINVOC_COUNT_HI__SHIFT                                                       0x0
+#define CP_GE_MSINVOC_COUNT_HI__MSINVOC_COUNT_HI_MASK                                                         0xFFFFFFFFL
+//CP_IB1_CMD_BUFSZ
+#define CP_IB1_CMD_BUFSZ__IB1_CMD_REQSZ__SHIFT                                                                0x0
+#define CP_IB1_CMD_BUFSZ__IB1_CMD_REQSZ_MASK                                                                  0x000FFFFFL
+//CP_IB2_CMD_BUFSZ
+#define CP_IB2_CMD_BUFSZ__IB2_CMD_REQSZ__SHIFT                                                                0x0
+#define CP_IB2_CMD_BUFSZ__IB2_CMD_REQSZ_MASK                                                                  0x000FFFFFL
+//CP_ST_CMD_BUFSZ
+#define CP_ST_CMD_BUFSZ__ST_CMD_REQSZ__SHIFT                                                                  0x0
+#define CP_ST_CMD_BUFSZ__ST_CMD_REQSZ_MASK                                                                    0x000FFFFFL
+//CP_IB1_BASE_LO
+#define CP_IB1_BASE_LO__IB1_BASE_LO__SHIFT                                                                    0x2
+#define CP_IB1_BASE_LO__IB1_BASE_LO_MASK                                                                      0xFFFFFFFCL
+//CP_IB1_BASE_HI
+#define CP_IB1_BASE_HI__IB1_BASE_HI__SHIFT                                                                    0x0
+#define CP_IB1_BASE_HI__IB1_BASE_HI_MASK                                                                      0x0000FFFFL
+//CP_IB1_BUFSZ
+#define CP_IB1_BUFSZ__IB1_BUFSZ__SHIFT                                                                        0x0
+#define CP_IB1_BUFSZ__IB1_BUFSZ_MASK                                                                          0x000FFFFFL
+//CP_IB2_BASE_LO
+#define CP_IB2_BASE_LO__IB2_BASE_LO__SHIFT                                                                    0x2
+#define CP_IB2_BASE_LO__IB2_BASE_LO_MASK                                                                      0xFFFFFFFCL
+//CP_IB2_BASE_HI
+#define CP_IB2_BASE_HI__IB2_BASE_HI__SHIFT                                                                    0x0
+#define CP_IB2_BASE_HI__IB2_BASE_HI_MASK                                                                      0x0000FFFFL
+//CP_IB2_BUFSZ
+#define CP_IB2_BUFSZ__IB2_BUFSZ__SHIFT                                                                        0x0
+#define CP_IB2_BUFSZ__IB2_BUFSZ_MASK                                                                          0x000FFFFFL
+//CP_ST_BASE_LO
+#define CP_ST_BASE_LO__ST_BASE_LO__SHIFT                                                                      0x2
+#define CP_ST_BASE_LO__ST_BASE_LO_MASK                                                                        0xFFFFFFFCL
+//CP_ST_BASE_HI
+#define CP_ST_BASE_HI__ST_BASE_HI__SHIFT                                                                      0x0
+#define CP_ST_BASE_HI__ST_BASE_HI_MASK                                                                        0x0000FFFFL
+//CP_ST_BUFSZ
+#define CP_ST_BUFSZ__ST_BUFSZ__SHIFT                                                                          0x0
+#define CP_ST_BUFSZ__ST_BUFSZ_MASK                                                                            0x000FFFFFL
+//CP_EOP_DONE_EVENT_CNTL
+#define CP_EOP_DONE_EVENT_CNTL__GCR_CNTL__SHIFT                                                               0xc
+#define CP_EOP_DONE_EVENT_CNTL__CACHE_POLICY__SHIFT                                                           0x19
+#define CP_EOP_DONE_EVENT_CNTL__EOP_VOLATILE__SHIFT                                                           0x1b
+#define CP_EOP_DONE_EVENT_CNTL__EXECUTE__SHIFT                                                                0x1c
+#define CP_EOP_DONE_EVENT_CNTL__GLK_INV__SHIFT                                                                0x1e
+#define CP_EOP_DONE_EVENT_CNTL__PWS_ENABLE__SHIFT                                                             0x1f
+#define CP_EOP_DONE_EVENT_CNTL__GCR_CNTL_MASK                                                                 0x01FFF000L
+#define CP_EOP_DONE_EVENT_CNTL__CACHE_POLICY_MASK                                                             0x06000000L
+#define CP_EOP_DONE_EVENT_CNTL__EOP_VOLATILE_MASK                                                             0x08000000L
+#define CP_EOP_DONE_EVENT_CNTL__EXECUTE_MASK                                                                  0x10000000L
+#define CP_EOP_DONE_EVENT_CNTL__GLK_INV_MASK                                                                  0x40000000L
+#define CP_EOP_DONE_EVENT_CNTL__PWS_ENABLE_MASK                                                               0x80000000L
+//CP_EOP_DONE_DATA_CNTL
+#define CP_EOP_DONE_DATA_CNTL__DST_SEL__SHIFT                                                                 0x10
+#define CP_EOP_DONE_DATA_CNTL__SEMAPHORE_SIGNAL_TYPE__SHIFT                                                   0x13
+#define CP_EOP_DONE_DATA_CNTL__ACTION_PIPE_ID__SHIFT                                                          0x14
+#define CP_EOP_DONE_DATA_CNTL__ACTION_ID__SHIFT                                                               0x16
+#define CP_EOP_DONE_DATA_CNTL__INT_SEL__SHIFT                                                                 0x18
+#define CP_EOP_DONE_DATA_CNTL__DATA_SEL__SHIFT                                                                0x1d
+#define CP_EOP_DONE_DATA_CNTL__DST_SEL_MASK                                                                   0x00030000L
+#define CP_EOP_DONE_DATA_CNTL__SEMAPHORE_SIGNAL_TYPE_MASK                                                     0x00080000L
+#define CP_EOP_DONE_DATA_CNTL__ACTION_PIPE_ID_MASK                                                            0x00300000L
+#define CP_EOP_DONE_DATA_CNTL__ACTION_ID_MASK                                                                 0x00C00000L
+#define CP_EOP_DONE_DATA_CNTL__INT_SEL_MASK                                                                   0x07000000L
+#define CP_EOP_DONE_DATA_CNTL__DATA_SEL_MASK                                                                  0xE0000000L
+//CP_EOP_DONE_CNTX_ID
+#define CP_EOP_DONE_CNTX_ID__CNTX_ID__SHIFT                                                                   0x0
+#define CP_EOP_DONE_CNTX_ID__CNTX_ID_MASK                                                                     0xFFFFFFFFL
+//CP_DB_BASE_LO
+#define CP_DB_BASE_LO__DB_BASE_LO__SHIFT                                                                      0x2
+#define CP_DB_BASE_LO__DB_BASE_LO_MASK                                                                        0xFFFFFFFCL
+//CP_DB_BASE_HI
+#define CP_DB_BASE_HI__DB_BASE_HI__SHIFT                                                                      0x0
+#define CP_DB_BASE_HI__DB_BASE_HI_MASK                                                                        0x0000FFFFL
+//CP_DB_BUFSZ
+#define CP_DB_BUFSZ__DB_BUFSZ__SHIFT                                                                          0x0
+#define CP_DB_BUFSZ__DB_BUFSZ_MASK                                                                            0x000FFFFFL
+//CP_DB_CMD_BUFSZ
+#define CP_DB_CMD_BUFSZ__DB_CMD_REQSZ__SHIFT                                                                  0x0
+#define CP_DB_CMD_BUFSZ__DB_CMD_REQSZ_MASK                                                                    0x000FFFFFL
+//CP_PFP_COMPLETION_STATUS
+#define CP_PFP_COMPLETION_STATUS__STATUS__SHIFT                                                               0x0
+#define CP_PFP_COMPLETION_STATUS__STATUS_MASK                                                                 0x00000003L
+//CP_PRED_NOT_VISIBLE
+#define CP_PRED_NOT_VISIBLE__NOT_VISIBLE__SHIFT                                                               0x0
+#define CP_PRED_NOT_VISIBLE__NOT_VISIBLE_MASK                                                                 0x00000001L
+//CP_PFP_METADATA_BASE_ADDR
+#define CP_PFP_METADATA_BASE_ADDR__ADDR_LO__SHIFT                                                             0x0
+#define CP_PFP_METADATA_BASE_ADDR__ADDR_LO_MASK                                                               0xFFFFFFFFL
+//CP_PFP_METADATA_BASE_ADDR_HI
+#define CP_PFP_METADATA_BASE_ADDR_HI__ADDR_HI__SHIFT                                                          0x0
+#define CP_PFP_METADATA_BASE_ADDR_HI__ADDR_HI_MASK                                                            0x0000FFFFL
+//CP_DRAW_INDX_INDR_ADDR
+#define CP_DRAW_INDX_INDR_ADDR__ADDR_LO__SHIFT                                                                0x0
+#define CP_DRAW_INDX_INDR_ADDR__ADDR_LO_MASK                                                                  0xFFFFFFFFL
+//CP_DRAW_INDX_INDR_ADDR_HI
+#define CP_DRAW_INDX_INDR_ADDR_HI__ADDR_HI__SHIFT                                                             0x0
+#define CP_DRAW_INDX_INDR_ADDR_HI__ADDR_HI_MASK                                                               0x0000FFFFL
+//CP_DISPATCH_INDR_ADDR
+#define CP_DISPATCH_INDR_ADDR__ADDR_LO__SHIFT                                                                 0x0
+#define CP_DISPATCH_INDR_ADDR__ADDR_LO_MASK                                                                   0xFFFFFFFFL
+//CP_DISPATCH_INDR_ADDR_HI
+#define CP_DISPATCH_INDR_ADDR_HI__ADDR_HI__SHIFT                                                              0x0
+#define CP_DISPATCH_INDR_ADDR_HI__ADDR_HI_MASK                                                                0x0000FFFFL
+//CP_INDEX_BASE_ADDR
+#define CP_INDEX_BASE_ADDR__ADDR_LO__SHIFT                                                                    0x0
+#define CP_INDEX_BASE_ADDR__ADDR_LO_MASK                                                                      0xFFFFFFFFL
+//CP_INDEX_BASE_ADDR_HI
+#define CP_INDEX_BASE_ADDR_HI__ADDR_HI__SHIFT                                                                 0x0
+#define CP_INDEX_BASE_ADDR_HI__ADDR_HI_MASK                                                                   0x0000FFFFL
+//CP_INDEX_TYPE
+#define CP_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                      0x0
+#define CP_INDEX_TYPE__INDEX_TYPE_MASK                                                                        0x00000003L
+//CP_GDS_BKUP_ADDR
+#define CP_GDS_BKUP_ADDR__ADDR_LO__SHIFT                                                                      0x0
+#define CP_GDS_BKUP_ADDR__ADDR_LO_MASK                                                                        0xFFFFFFFFL
+//CP_GDS_BKUP_ADDR_HI
+#define CP_GDS_BKUP_ADDR_HI__ADDR_HI__SHIFT                                                                   0x0
+#define CP_GDS_BKUP_ADDR_HI__ADDR_HI_MASK                                                                     0x0000FFFFL
+//CP_SAMPLE_STATUS
+#define CP_SAMPLE_STATUS__Z_PASS_ACITVE__SHIFT                                                                0x0
+#define CP_SAMPLE_STATUS__STREAMOUT_ACTIVE__SHIFT                                                             0x1
+#define CP_SAMPLE_STATUS__PIPELINE_ACTIVE__SHIFT                                                              0x2
+#define CP_SAMPLE_STATUS__STIPPLE_ACTIVE__SHIFT                                                               0x3
+#define CP_SAMPLE_STATUS__VGT_BUFFERS_ACTIVE__SHIFT                                                           0x4
+#define CP_SAMPLE_STATUS__SCREEN_EXT_ACTIVE__SHIFT                                                            0x5
+#define CP_SAMPLE_STATUS__DRAW_INDIRECT_ACTIVE__SHIFT                                                         0x6
+#define CP_SAMPLE_STATUS__DISP_INDIRECT_ACTIVE__SHIFT                                                         0x7
+#define CP_SAMPLE_STATUS__Z_PASS_ACITVE_MASK                                                                  0x00000001L
+#define CP_SAMPLE_STATUS__STREAMOUT_ACTIVE_MASK                                                               0x00000002L
+#define CP_SAMPLE_STATUS__PIPELINE_ACTIVE_MASK                                                                0x00000004L
+#define CP_SAMPLE_STATUS__STIPPLE_ACTIVE_MASK                                                                 0x00000008L
+#define CP_SAMPLE_STATUS__VGT_BUFFERS_ACTIVE_MASK                                                             0x00000010L
+#define CP_SAMPLE_STATUS__SCREEN_EXT_ACTIVE_MASK                                                              0x00000020L
+#define CP_SAMPLE_STATUS__DRAW_INDIRECT_ACTIVE_MASK                                                           0x00000040L
+#define CP_SAMPLE_STATUS__DISP_INDIRECT_ACTIVE_MASK                                                           0x00000080L
+//CP_ME_COHER_CNTL
+#define CP_ME_COHER_CNTL__DEST_BASE_0_ENA__SHIFT                                                              0x0
+#define CP_ME_COHER_CNTL__DEST_BASE_1_ENA__SHIFT                                                              0x1
+#define CP_ME_COHER_CNTL__CB0_DEST_BASE_ENA__SHIFT                                                            0x6
+#define CP_ME_COHER_CNTL__CB1_DEST_BASE_ENA__SHIFT                                                            0x7
+#define CP_ME_COHER_CNTL__CB2_DEST_BASE_ENA__SHIFT                                                            0x8
+#define CP_ME_COHER_CNTL__CB3_DEST_BASE_ENA__SHIFT                                                            0x9
+#define CP_ME_COHER_CNTL__CB4_DEST_BASE_ENA__SHIFT                                                            0xa
+#define CP_ME_COHER_CNTL__CB5_DEST_BASE_ENA__SHIFT                                                            0xb
+#define CP_ME_COHER_CNTL__CB6_DEST_BASE_ENA__SHIFT                                                            0xc
+#define CP_ME_COHER_CNTL__CB7_DEST_BASE_ENA__SHIFT                                                            0xd
+#define CP_ME_COHER_CNTL__DB_DEST_BASE_ENA__SHIFT                                                             0xe
+#define CP_ME_COHER_CNTL__DEST_BASE_2_ENA__SHIFT                                                              0x13
+#define CP_ME_COHER_CNTL__DEST_BASE_3_ENA__SHIFT                                                              0x15
+#define CP_ME_COHER_CNTL__DEST_BASE_0_ENA_MASK                                                                0x00000001L
+#define CP_ME_COHER_CNTL__DEST_BASE_1_ENA_MASK                                                                0x00000002L
+#define CP_ME_COHER_CNTL__CB0_DEST_BASE_ENA_MASK                                                              0x00000040L
+#define CP_ME_COHER_CNTL__CB1_DEST_BASE_ENA_MASK                                                              0x00000080L
+#define CP_ME_COHER_CNTL__CB2_DEST_BASE_ENA_MASK                                                              0x00000100L
+#define CP_ME_COHER_CNTL__CB3_DEST_BASE_ENA_MASK                                                              0x00000200L
+#define CP_ME_COHER_CNTL__CB4_DEST_BASE_ENA_MASK                                                              0x00000400L
+#define CP_ME_COHER_CNTL__CB5_DEST_BASE_ENA_MASK                                                              0x00000800L
+#define CP_ME_COHER_CNTL__CB6_DEST_BASE_ENA_MASK                                                              0x00001000L
+#define CP_ME_COHER_CNTL__CB7_DEST_BASE_ENA_MASK                                                              0x00002000L
+#define CP_ME_COHER_CNTL__DB_DEST_BASE_ENA_MASK                                                               0x00004000L
+#define CP_ME_COHER_CNTL__DEST_BASE_2_ENA_MASK                                                                0x00080000L
+#define CP_ME_COHER_CNTL__DEST_BASE_3_ENA_MASK                                                                0x00200000L
+//CP_ME_COHER_SIZE
+#define CP_ME_COHER_SIZE__COHER_SIZE_256B__SHIFT                                                              0x0
+#define CP_ME_COHER_SIZE__COHER_SIZE_256B_MASK                                                                0xFFFFFFFFL
+//CP_ME_COHER_SIZE_HI
+#define CP_ME_COHER_SIZE_HI__COHER_SIZE_HI_256B__SHIFT                                                        0x0
+#define CP_ME_COHER_SIZE_HI__COHER_SIZE_HI_256B_MASK                                                          0x000000FFL
+//CP_ME_COHER_BASE
+#define CP_ME_COHER_BASE__COHER_BASE_256B__SHIFT                                                              0x0
+#define CP_ME_COHER_BASE__COHER_BASE_256B_MASK                                                                0xFFFFFFFFL
+//CP_ME_COHER_BASE_HI
+#define CP_ME_COHER_BASE_HI__COHER_BASE_HI_256B__SHIFT                                                        0x0
+#define CP_ME_COHER_BASE_HI__COHER_BASE_HI_256B_MASK                                                          0x000000FFL
+//CP_ME_COHER_STATUS
+#define CP_ME_COHER_STATUS__MATCHING_GFX_CNTX__SHIFT                                                          0x0
+#define CP_ME_COHER_STATUS__STATUS__SHIFT                                                                     0x1f
+#define CP_ME_COHER_STATUS__MATCHING_GFX_CNTX_MASK                                                            0x000000FFL
+#define CP_ME_COHER_STATUS__STATUS_MASK                                                                       0x80000000L
+//RLC_GPM_PERF_COUNT_0
+#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL__SHIFT                                                              0x0
+#define RLC_GPM_PERF_COUNT_0__SE_INDEX__SHIFT                                                                 0x4
+#define RLC_GPM_PERF_COUNT_0__SA_INDEX__SHIFT                                                                 0x8
+#define RLC_GPM_PERF_COUNT_0__WGP_INDEX__SHIFT                                                                0xc
+#define RLC_GPM_PERF_COUNT_0__EVENT_SEL__SHIFT                                                                0x10
+#define RLC_GPM_PERF_COUNT_0__UNUSED__SHIFT                                                                   0x12
+#define RLC_GPM_PERF_COUNT_0__ENABLE__SHIFT                                                                   0x14
+#define RLC_GPM_PERF_COUNT_0__RESERVED__SHIFT                                                                 0x15
+#define RLC_GPM_PERF_COUNT_0__FEATURE_SEL_MASK                                                                0x0000000FL
+#define RLC_GPM_PERF_COUNT_0__SE_INDEX_MASK                                                                   0x000000F0L
+#define RLC_GPM_PERF_COUNT_0__SA_INDEX_MASK                                                                   0x00000F00L
+#define RLC_GPM_PERF_COUNT_0__WGP_INDEX_MASK                                                                  0x0000F000L
+#define RLC_GPM_PERF_COUNT_0__EVENT_SEL_MASK                                                                  0x00030000L
+#define RLC_GPM_PERF_COUNT_0__UNUSED_MASK                                                                     0x000C0000L
+#define RLC_GPM_PERF_COUNT_0__ENABLE_MASK                                                                     0x00100000L
+#define RLC_GPM_PERF_COUNT_0__RESERVED_MASK                                                                   0xFFE00000L
+//RLC_GPM_PERF_COUNT_1
+#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL__SHIFT                                                              0x0
+#define RLC_GPM_PERF_COUNT_1__SE_INDEX__SHIFT                                                                 0x4
+#define RLC_GPM_PERF_COUNT_1__SA_INDEX__SHIFT                                                                 0x8
+#define RLC_GPM_PERF_COUNT_1__WGP_INDEX__SHIFT                                                                0xc
+#define RLC_GPM_PERF_COUNT_1__EVENT_SEL__SHIFT                                                                0x10
+#define RLC_GPM_PERF_COUNT_1__UNUSED__SHIFT                                                                   0x12
+#define RLC_GPM_PERF_COUNT_1__ENABLE__SHIFT                                                                   0x14
+#define RLC_GPM_PERF_COUNT_1__RESERVED__SHIFT                                                                 0x15
+#define RLC_GPM_PERF_COUNT_1__FEATURE_SEL_MASK                                                                0x0000000FL
+#define RLC_GPM_PERF_COUNT_1__SE_INDEX_MASK                                                                   0x000000F0L
+#define RLC_GPM_PERF_COUNT_1__SA_INDEX_MASK                                                                   0x00000F00L
+#define RLC_GPM_PERF_COUNT_1__WGP_INDEX_MASK                                                                  0x0000F000L
+#define RLC_GPM_PERF_COUNT_1__EVENT_SEL_MASK                                                                  0x00030000L
+#define RLC_GPM_PERF_COUNT_1__UNUSED_MASK                                                                     0x000C0000L
+#define RLC_GPM_PERF_COUNT_1__ENABLE_MASK                                                                     0x00100000L
+#define RLC_GPM_PERF_COUNT_1__RESERVED_MASK                                                                   0xFFE00000L
+//GRBM_GFX_INDEX
+#define GRBM_GFX_INDEX__INSTANCE_INDEX__SHIFT                                                                 0x0
+#define GRBM_GFX_INDEX__SA_INDEX__SHIFT                                                                       0x8
+#define GRBM_GFX_INDEX__SE_INDEX__SHIFT                                                                       0x10
+#define GRBM_GFX_INDEX__SA_BROADCAST_WRITES__SHIFT                                                            0x1d
+#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES__SHIFT                                                      0x1e
+#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT                                                            0x1f
+#define GRBM_GFX_INDEX__INSTANCE_INDEX_MASK                                                                   0x000000FFL
+#define GRBM_GFX_INDEX__SA_INDEX_MASK                                                                         0x0000FF00L
+#define GRBM_GFX_INDEX__SE_INDEX_MASK                                                                         0x00FF0000L
+#define GRBM_GFX_INDEX__SA_BROADCAST_WRITES_MASK                                                              0x20000000L
+#define GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK                                                        0x40000000L
+#define GRBM_GFX_INDEX__SE_BROADCAST_WRITES_MASK                                                              0x80000000L
+//VGT_PRIMITIVE_TYPE
+#define VGT_PRIMITIVE_TYPE__PRIM_TYPE__SHIFT                                                                  0x0
+#define VGT_PRIMITIVE_TYPE__PRIM_TYPE_MASK                                                                    0x0000003FL
+//VGT_INDEX_TYPE
+#define VGT_INDEX_TYPE__INDEX_TYPE__SHIFT                                                                     0x0
+#define VGT_INDEX_TYPE__DISABLE_INSTANCE_PACKING__SHIFT                                                       0xe
+#define VGT_INDEX_TYPE__INDEX_TYPE_MASK                                                                       0x00000003L
+#define VGT_INDEX_TYPE__DISABLE_INSTANCE_PACKING_MASK                                                         0x00004000L
+//GE_MIN_VTX_INDX
+#define GE_MIN_VTX_INDX__MIN_INDX__SHIFT                                                                      0x0
+#define GE_MIN_VTX_INDX__MIN_INDX_MASK                                                                        0xFFFFFFFFL
+//GE_INDX_OFFSET
+#define GE_INDX_OFFSET__INDX_OFFSET__SHIFT                                                                    0x0
+#define GE_INDX_OFFSET__INDX_OFFSET_MASK                                                                      0xFFFFFFFFL
+//GE_MULTI_PRIM_IB_RESET_EN
+#define GE_MULTI_PRIM_IB_RESET_EN__RESET_EN__SHIFT                                                            0x0
+#define GE_MULTI_PRIM_IB_RESET_EN__MATCH_ALL_BITS__SHIFT                                                      0x1
+#define GE_MULTI_PRIM_IB_RESET_EN__DISABLE_FOR_AUTO_INDEX__SHIFT                                              0x2
+#define GE_MULTI_PRIM_IB_RESET_EN__RESET_EN_MASK                                                              0x00000001L
+#define GE_MULTI_PRIM_IB_RESET_EN__MATCH_ALL_BITS_MASK                                                        0x00000002L
+#define GE_MULTI_PRIM_IB_RESET_EN__DISABLE_FOR_AUTO_INDEX_MASK                                                0x00000004L
+//VGT_NUM_INDICES
+#define VGT_NUM_INDICES__NUM_INDICES__SHIFT                                                                   0x0
+#define VGT_NUM_INDICES__NUM_INDICES_MASK                                                                     0xFFFFFFFFL
+//VGT_NUM_INSTANCES
+#define VGT_NUM_INSTANCES__NUM_INSTANCES__SHIFT                                                               0x0
+#define VGT_NUM_INSTANCES__NUM_INSTANCES_MASK                                                                 0xFFFFFFFFL
+//VGT_TF_RING_SIZE
+#define VGT_TF_RING_SIZE__SIZE__SHIFT                                                                         0x0
+#define VGT_TF_RING_SIZE__SIZE_MASK                                                                           0x0001FFFFL
+//VGT_HS_OFFCHIP_PARAM
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING__SHIFT                                                        0x0
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY__SHIFT                                                      0xa
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_BUFFERING_MASK                                                          0x000003FFL
+#define VGT_HS_OFFCHIP_PARAM__OFFCHIP_GRANULARITY_MASK                                                        0x00000C00L
+//VGT_TF_MEMORY_BASE
+#define VGT_TF_MEMORY_BASE__BASE__SHIFT                                                                       0x0
+#define VGT_TF_MEMORY_BASE__BASE_MASK                                                                         0xFFFFFFFFL
+//GE_MAX_VTX_INDX
+#define GE_MAX_VTX_INDX__MAX_INDX__SHIFT                                                                      0x0
+#define GE_MAX_VTX_INDX__MAX_INDX_MASK                                                                        0xFFFFFFFFL
+//VGT_INSTANCE_BASE_ID
+#define VGT_INSTANCE_BASE_ID__INSTANCE_BASE_ID__SHIFT                                                         0x0
+#define VGT_INSTANCE_BASE_ID__INSTANCE_BASE_ID_MASK                                                           0xFFFFFFFFL
+//GE_CNTL
+#define GE_CNTL__PRIMS_PER_SUBGRP__SHIFT                                                                      0x0
+#define GE_CNTL__VERTS_PER_SUBGRP__SHIFT                                                                      0x9
+#define GE_CNTL__BREAK_SUBGRP_AT_EOI__SHIFT                                                                   0x12
+#define GE_CNTL__PACKET_TO_ONE_PA__SHIFT                                                                      0x13
+#define GE_CNTL__BREAK_PRIMGRP_AT_EOI__SHIFT                                                                  0x14
+#define GE_CNTL__PRIM_GRP_SIZE__SHIFT                                                                         0x15
+#define GE_CNTL__GCR_DISABLE__SHIFT                                                                           0x1e
+#define GE_CNTL__DIS_PG_SIZE_ADJUST_FOR_STRIP__SHIFT                                                          0x1f
+#define GE_CNTL__PRIMS_PER_SUBGRP_MASK                                                                        0x000001FFL
+#define GE_CNTL__VERTS_PER_SUBGRP_MASK                                                                        0x0003FE00L
+#define GE_CNTL__BREAK_SUBGRP_AT_EOI_MASK                                                                     0x00040000L
+#define GE_CNTL__PACKET_TO_ONE_PA_MASK                                                                        0x00080000L
+#define GE_CNTL__BREAK_PRIMGRP_AT_EOI_MASK                                                                    0x00100000L
+#define GE_CNTL__PRIM_GRP_SIZE_MASK                                                                           0x3FE00000L
+#define GE_CNTL__GCR_DISABLE_MASK                                                                             0x40000000L
+#define GE_CNTL__DIS_PG_SIZE_ADJUST_FOR_STRIP_MASK                                                            0x80000000L
+//GE_USER_VGPR1
+#define GE_USER_VGPR1__DATA__SHIFT                                                                            0x0
+#define GE_USER_VGPR1__DATA_MASK                                                                              0xFFFFFFFFL
+//GE_USER_VGPR2
+#define GE_USER_VGPR2__DATA__SHIFT                                                                            0x0
+#define GE_USER_VGPR2__DATA_MASK                                                                              0xFFFFFFFFL
+//GE_USER_VGPR3
+#define GE_USER_VGPR3__DATA__SHIFT                                                                            0x0
+#define GE_USER_VGPR3__DATA_MASK                                                                              0xFFFFFFFFL
+//GE_STEREO_CNTL
+#define GE_STEREO_CNTL__RT_SLICE__SHIFT                                                                       0x0
+#define GE_STEREO_CNTL__VIEWPORT__SHIFT                                                                       0x3
+#define GE_STEREO_CNTL__FSR_SELECT__SHIFT                                                                     0x7
+#define GE_STEREO_CNTL__EN_STEREO__SHIFT                                                                      0x8
+#define GE_STEREO_CNTL__RT_SLICE_MASK                                                                         0x00000007L
+#define GE_STEREO_CNTL__VIEWPORT_MASK                                                                         0x00000078L
+#define GE_STEREO_CNTL__FSR_SELECT_MASK                                                                       0x00000080L
+#define GE_STEREO_CNTL__EN_STEREO_MASK                                                                        0x00000100L
+//GE_PC_ALLOC
+#define GE_PC_ALLOC__OVERSUB_EN__SHIFT                                                                        0x0
+#define GE_PC_ALLOC__NUM_PC_LINES__SHIFT                                                                      0x1
+#define GE_PC_ALLOC__OVERSUB_EN_MASK                                                                          0x00000001L
+#define GE_PC_ALLOC__NUM_PC_LINES_MASK                                                                        0x000007FEL
+//VGT_TF_MEMORY_BASE_HI
+#define VGT_TF_MEMORY_BASE_HI__BASE_HI__SHIFT                                                                 0x0
+#define VGT_TF_MEMORY_BASE_HI__BASE_HI_MASK                                                                   0x000000FFL
+//GE_USER_VGPR_EN
+#define GE_USER_VGPR_EN__EN_USER_VGPR1__SHIFT                                                                 0x0
+#define GE_USER_VGPR_EN__EN_USER_VGPR2__SHIFT                                                                 0x1
+#define GE_USER_VGPR_EN__EN_USER_VGPR3__SHIFT                                                                 0x2
+#define GE_USER_VGPR_EN__EN_USER_VGPR1_MASK                                                                   0x00000001L
+#define GE_USER_VGPR_EN__EN_USER_VGPR2_MASK                                                                   0x00000002L
+#define GE_USER_VGPR_EN__EN_USER_VGPR3_MASK                                                                   0x00000004L
+//GE_GS_FAST_LAUNCH_WG_DIM
+#define GE_GS_FAST_LAUNCH_WG_DIM__GS_FL_DIM_X__SHIFT                                                          0x0
+#define GE_GS_FAST_LAUNCH_WG_DIM__GS_FL_DIM_Y__SHIFT                                                          0x10
+#define GE_GS_FAST_LAUNCH_WG_DIM__GS_FL_DIM_X_MASK                                                            0x0000FFFFL
+#define GE_GS_FAST_LAUNCH_WG_DIM__GS_FL_DIM_Y_MASK                                                            0xFFFF0000L
+//GE_GS_FAST_LAUNCH_WG_DIM_1
+#define GE_GS_FAST_LAUNCH_WG_DIM_1__GS_FL_DIM_Z__SHIFT                                                        0x0
+#define GE_GS_FAST_LAUNCH_WG_DIM_1__GS_FL_DIM_Z_MASK                                                          0x0000FFFFL
+//VGT_GS_OUT_PRIM_TYPE
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE__SHIFT                                                             0x0
+#define VGT_GS_OUT_PRIM_TYPE__OUTPRIM_TYPE_MASK                                                               0x0000003FL
+//PA_SU_LINE_STIPPLE_VALUE
+#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE__SHIFT                                                   0x0
+#define PA_SU_LINE_STIPPLE_VALUE__LINE_STIPPLE_VALUE_MASK                                                     0x00FFFFFFL
+//PA_SC_LINE_STIPPLE_STATE
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR__SHIFT                                                          0x0
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT__SHIFT                                                        0x8
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_PTR_MASK                                                            0x0000000FL
+#define PA_SC_LINE_STIPPLE_STATE__CURRENT_COUNT_MASK                                                          0x0000FF00L
+//PA_SC_SCREEN_EXTENT_MIN_0
+#define PA_SC_SCREEN_EXTENT_MIN_0__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MIN_0__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MIN_0__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MIN_0__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_SCREEN_EXTENT_MAX_0
+#define PA_SC_SCREEN_EXTENT_MAX_0__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MAX_0__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MAX_0__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MAX_0__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_SCREEN_EXTENT_MIN_1
+#define PA_SC_SCREEN_EXTENT_MIN_1__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MIN_1__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MIN_1__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MIN_1__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_SCREEN_EXTENT_MAX_1
+#define PA_SC_SCREEN_EXTENT_MAX_1__X__SHIFT                                                                   0x0
+#define PA_SC_SCREEN_EXTENT_MAX_1__Y__SHIFT                                                                   0x10
+#define PA_SC_SCREEN_EXTENT_MAX_1__X_MASK                                                                     0x0000FFFFL
+#define PA_SC_SCREEN_EXTENT_MAX_1__Y_MASK                                                                     0xFFFF0000L
+//PA_SC_P3D_TRAP_SCREEN_HV_EN
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                              0x0
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                       0x1
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                                0x00000001L
+#define PA_SC_P3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                         0x00000002L
+//PA_SC_P3D_TRAP_SCREEN_H
+#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD__SHIFT                                                               0x0
+#define PA_SC_P3D_TRAP_SCREEN_H__X_COORD_MASK                                                                 0x00003FFFL
+//PA_SC_P3D_TRAP_SCREEN_V
+#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD__SHIFT                                                               0x0
+#define PA_SC_P3D_TRAP_SCREEN_V__Y_COORD_MASK                                                                 0x00003FFFL
+//PA_SC_P3D_TRAP_SCREEN_OCCURRENCE
+#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                        0x0
+#define PA_SC_P3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                          0x0000FFFFL
+//PA_SC_P3D_TRAP_SCREEN_COUNT
+#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                             0x0
+#define PA_SC_P3D_TRAP_SCREEN_COUNT__COUNT_MASK                                                               0x0000FFFFL
+//PA_SC_HP3D_TRAP_SCREEN_HV_EN
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                             0x0
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                      0x1
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                               0x00000001L
+#define PA_SC_HP3D_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                        0x00000002L
+//PA_SC_HP3D_TRAP_SCREEN_H
+#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD__SHIFT                                                              0x0
+#define PA_SC_HP3D_TRAP_SCREEN_H__X_COORD_MASK                                                                0x00003FFFL
+//PA_SC_HP3D_TRAP_SCREEN_V
+#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD__SHIFT                                                              0x0
+#define PA_SC_HP3D_TRAP_SCREEN_V__Y_COORD_MASK                                                                0x00003FFFL
+//PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE
+#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                       0x0
+#define PA_SC_HP3D_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                         0x0000FFFFL
+//PA_SC_HP3D_TRAP_SCREEN_COUNT
+#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                            0x0
+#define PA_SC_HP3D_TRAP_SCREEN_COUNT__COUNT_MASK                                                              0x0000FFFFL
+//PA_SC_TRAP_SCREEN_HV_EN
+#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER__SHIFT                                                  0x0
+#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS__SHIFT                                           0x1
+#define PA_SC_TRAP_SCREEN_HV_EN__ENABLE_HV_PRE_SHADER_MASK                                                    0x00000001L
+#define PA_SC_TRAP_SCREEN_HV_EN__FORCE_PRE_SHADER_ALL_PIXELS_MASK                                             0x00000002L
+//PA_SC_TRAP_SCREEN_H
+#define PA_SC_TRAP_SCREEN_H__X_COORD__SHIFT                                                                   0x0
+#define PA_SC_TRAP_SCREEN_H__X_COORD_MASK                                                                     0x00003FFFL
+//PA_SC_TRAP_SCREEN_V
+#define PA_SC_TRAP_SCREEN_V__Y_COORD__SHIFT                                                                   0x0
+#define PA_SC_TRAP_SCREEN_V__Y_COORD_MASK                                                                     0x00003FFFL
+//PA_SC_TRAP_SCREEN_OCCURRENCE
+#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT__SHIFT                                                            0x0
+#define PA_SC_TRAP_SCREEN_OCCURRENCE__COUNT_MASK                                                              0x0000FFFFL
+//PA_SC_TRAP_SCREEN_COUNT
+#define PA_SC_TRAP_SCREEN_COUNT__COUNT__SHIFT                                                                 0x0
+#define PA_SC_TRAP_SCREEN_COUNT__COUNT_MASK                                                                   0x0000FFFFL
+//SQ_THREAD_TRACE_USERDATA_0
+#define SQ_THREAD_TRACE_USERDATA_0__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_0__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_1
+#define SQ_THREAD_TRACE_USERDATA_1__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_1__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_2
+#define SQ_THREAD_TRACE_USERDATA_2__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_2__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_3
+#define SQ_THREAD_TRACE_USERDATA_3__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_3__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_4
+#define SQ_THREAD_TRACE_USERDATA_4__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_4__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_5
+#define SQ_THREAD_TRACE_USERDATA_5__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_5__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_6
+#define SQ_THREAD_TRACE_USERDATA_6__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_6__DATA_MASK                                                                 0xFFFFFFFFL
+//SQ_THREAD_TRACE_USERDATA_7
+#define SQ_THREAD_TRACE_USERDATA_7__DATA__SHIFT                                                               0x0
+#define SQ_THREAD_TRACE_USERDATA_7__DATA_MASK                                                                 0xFFFFFFFFL
+//SQC_CACHES
+#define SQC_CACHES__TARGET_INST__SHIFT                                                                        0x0
+#define SQC_CACHES__TARGET_DATA__SHIFT                                                                        0x1
+#define SQC_CACHES__INVALIDATE__SHIFT                                                                         0x2
+#define SQC_CACHES__COMPLETE__SHIFT                                                                           0x10
+#define SQC_CACHES__TARGET_INST_MASK                                                                          0x00000001L
+#define SQC_CACHES__TARGET_DATA_MASK                                                                          0x00000002L
+#define SQC_CACHES__INVALIDATE_MASK                                                                           0x00000004L
+#define SQC_CACHES__COMPLETE_MASK                                                                             0x00010000L
+//TA_CS_BC_BASE_ADDR
+#define TA_CS_BC_BASE_ADDR__ADDRESS__SHIFT                                                                    0x0
+#define TA_CS_BC_BASE_ADDR__ADDRESS_MASK                                                                      0xFFFFFFFFL
+//TA_CS_BC_BASE_ADDR_HI
+#define TA_CS_BC_BASE_ADDR_HI__ADDRESS__SHIFT                                                                 0x0
+#define TA_CS_BC_BASE_ADDR_HI__ADDRESS_MASK                                                                   0x000000FFL
+//DB_OCCLUSION_COUNT0_LOW
+#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT0_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT0_HI
+#define DB_OCCLUSION_COUNT0_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT0_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//DB_OCCLUSION_COUNT1_LOW
+#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT1_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT1_HI
+#define DB_OCCLUSION_COUNT1_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT1_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//DB_OCCLUSION_COUNT2_LOW
+#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT2_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT2_HI
+#define DB_OCCLUSION_COUNT2_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT2_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//DB_OCCLUSION_COUNT3_LOW
+#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW__SHIFT                                                             0x0
+#define DB_OCCLUSION_COUNT3_LOW__COUNT_LOW_MASK                                                               0xFFFFFFFFL
+//DB_OCCLUSION_COUNT3_HI
+#define DB_OCCLUSION_COUNT3_HI__COUNT_HI__SHIFT                                                               0x0
+#define DB_OCCLUSION_COUNT3_HI__COUNT_HI_MASK                                                                 0x7FFFFFFFL
+//GDS_RD_ADDR
+#define GDS_RD_ADDR__READ_ADDR__SHIFT                                                                         0x0
+#define GDS_RD_ADDR__READ_ADDR_MASK                                                                           0xFFFFFFFFL
+//GDS_RD_DATA
+#define GDS_RD_DATA__READ_DATA__SHIFT                                                                         0x0
+#define GDS_RD_DATA__READ_DATA_MASK                                                                           0xFFFFFFFFL
+//GDS_RD_BURST_ADDR
+#define GDS_RD_BURST_ADDR__BURST_ADDR__SHIFT                                                                  0x0
+#define GDS_RD_BURST_ADDR__BURST_ADDR_MASK                                                                    0xFFFFFFFFL
+//GDS_RD_BURST_COUNT
+#define GDS_RD_BURST_COUNT__BURST_COUNT__SHIFT                                                                0x0
+#define GDS_RD_BURST_COUNT__BURST_COUNT_MASK                                                                  0xFFFFFFFFL
+//GDS_RD_BURST_DATA
+#define GDS_RD_BURST_DATA__BURST_DATA__SHIFT                                                                  0x0
+#define GDS_RD_BURST_DATA__BURST_DATA_MASK                                                                    0xFFFFFFFFL
+//GDS_WR_ADDR
+#define GDS_WR_ADDR__WRITE_ADDR__SHIFT                                                                        0x0
+#define GDS_WR_ADDR__WRITE_ADDR_MASK                                                                          0xFFFFFFFFL
+//GDS_WR_DATA
+#define GDS_WR_DATA__WRITE_DATA__SHIFT                                                                        0x0
+#define GDS_WR_DATA__WRITE_DATA_MASK                                                                          0xFFFFFFFFL
+//GDS_WR_BURST_ADDR
+#define GDS_WR_BURST_ADDR__WRITE_ADDR__SHIFT                                                                  0x0
+#define GDS_WR_BURST_ADDR__WRITE_ADDR_MASK                                                                    0xFFFFFFFFL
+//GDS_WR_BURST_DATA
+#define GDS_WR_BURST_DATA__WRITE_DATA__SHIFT                                                                  0x0
+#define GDS_WR_BURST_DATA__WRITE_DATA_MASK                                                                    0xFFFFFFFFL
+//GDS_WRITE_COMPLETE
+#define GDS_WRITE_COMPLETE__WRITE_COMPLETE__SHIFT                                                             0x0
+#define GDS_WRITE_COMPLETE__WRITE_COMPLETE_MASK                                                               0xFFFFFFFFL
+//GDS_ATOM_CNTL
+#define GDS_ATOM_CNTL__AINC__SHIFT                                                                            0x0
+#define GDS_ATOM_CNTL__UNUSED1__SHIFT                                                                         0x6
+#define GDS_ATOM_CNTL__DMODE__SHIFT                                                                           0x8
+#define GDS_ATOM_CNTL__UNUSED2__SHIFT                                                                         0xa
+#define GDS_ATOM_CNTL__AINC_MASK                                                                              0x0000003FL
+#define GDS_ATOM_CNTL__UNUSED1_MASK                                                                           0x000000C0L
+#define GDS_ATOM_CNTL__DMODE_MASK                                                                             0x00000300L
+#define GDS_ATOM_CNTL__UNUSED2_MASK                                                                           0xFFFFFC00L
+//GDS_ATOM_COMPLETE
+#define GDS_ATOM_COMPLETE__COMPLETE__SHIFT                                                                    0x0
+#define GDS_ATOM_COMPLETE__UNUSED__SHIFT                                                                      0x1
+#define GDS_ATOM_COMPLETE__COMPLETE_MASK                                                                      0x00000001L
+#define GDS_ATOM_COMPLETE__UNUSED_MASK                                                                        0xFFFFFFFEL
+//GDS_ATOM_BASE
+#define GDS_ATOM_BASE__BASE__SHIFT                                                                            0x0
+#define GDS_ATOM_BASE__UNUSED__SHIFT                                                                          0xc
+#define GDS_ATOM_BASE__BASE_MASK                                                                              0x00000FFFL
+#define GDS_ATOM_BASE__UNUSED_MASK                                                                            0xFFFFF000L
+//GDS_ATOM_SIZE
+#define GDS_ATOM_SIZE__SIZE__SHIFT                                                                            0x0
+#define GDS_ATOM_SIZE__UNUSED__SHIFT                                                                          0xd
+#define GDS_ATOM_SIZE__SIZE_MASK                                                                              0x00001FFFL
+#define GDS_ATOM_SIZE__UNUSED_MASK                                                                            0xFFFFE000L
+//GDS_ATOM_OFFSET0
+#define GDS_ATOM_OFFSET0__OFFSET0__SHIFT                                                                      0x0
+#define GDS_ATOM_OFFSET0__UNUSED__SHIFT                                                                       0x8
+#define GDS_ATOM_OFFSET0__OFFSET0_MASK                                                                        0x000000FFL
+#define GDS_ATOM_OFFSET0__UNUSED_MASK                                                                         0xFFFFFF00L
+//GDS_ATOM_OFFSET1
+#define GDS_ATOM_OFFSET1__OFFSET1__SHIFT                                                                      0x0
+#define GDS_ATOM_OFFSET1__UNUSED__SHIFT                                                                       0x8
+#define GDS_ATOM_OFFSET1__OFFSET1_MASK                                                                        0x000000FFL
+#define GDS_ATOM_OFFSET1__UNUSED_MASK                                                                         0xFFFFFF00L
+//GDS_ATOM_DST
+#define GDS_ATOM_DST__DST__SHIFT                                                                              0x0
+#define GDS_ATOM_DST__DST_MASK                                                                                0xFFFFFFFFL
+//GDS_ATOM_OP
+#define GDS_ATOM_OP__OP__SHIFT                                                                                0x0
+#define GDS_ATOM_OP__UNUSED__SHIFT                                                                            0x8
+#define GDS_ATOM_OP__OP_MASK                                                                                  0x000000FFL
+#define GDS_ATOM_OP__UNUSED_MASK                                                                              0xFFFFFF00L
+//GDS_ATOM_SRC0
+#define GDS_ATOM_SRC0__DATA__SHIFT                                                                            0x0
+#define GDS_ATOM_SRC0__DATA_MASK                                                                              0xFFFFFFFFL
+//GDS_ATOM_SRC0_U
+#define GDS_ATOM_SRC0_U__DATA__SHIFT                                                                          0x0
+#define GDS_ATOM_SRC0_U__DATA_MASK                                                                            0xFFFFFFFFL
+//GDS_ATOM_SRC1
+#define GDS_ATOM_SRC1__DATA__SHIFT                                                                            0x0
+#define GDS_ATOM_SRC1__DATA_MASK                                                                              0xFFFFFFFFL
+//GDS_ATOM_SRC1_U
+#define GDS_ATOM_SRC1_U__DATA__SHIFT                                                                          0x0
+#define GDS_ATOM_SRC1_U__DATA_MASK                                                                            0xFFFFFFFFL
+//GDS_ATOM_READ0
+#define GDS_ATOM_READ0__DATA__SHIFT                                                                           0x0
+#define GDS_ATOM_READ0__DATA_MASK                                                                             0xFFFFFFFFL
+//GDS_ATOM_READ0_U
+#define GDS_ATOM_READ0_U__DATA__SHIFT                                                                         0x0
+#define GDS_ATOM_READ0_U__DATA_MASK                                                                           0xFFFFFFFFL
+//GDS_ATOM_READ1
+#define GDS_ATOM_READ1__DATA__SHIFT                                                                           0x0
+#define GDS_ATOM_READ1__DATA_MASK                                                                             0xFFFFFFFFL
+//GDS_ATOM_READ1_U
+#define GDS_ATOM_READ1_U__DATA__SHIFT                                                                         0x0
+#define GDS_ATOM_READ1_U__DATA_MASK                                                                           0xFFFFFFFFL
+//GDS_GWS_RESOURCE_CNTL
+#define GDS_GWS_RESOURCE_CNTL__INDEX__SHIFT                                                                   0x0
+#define GDS_GWS_RESOURCE_CNTL__UNUSED__SHIFT                                                                  0x6
+#define GDS_GWS_RESOURCE_CNTL__INDEX_MASK                                                                     0x0000003FL
+#define GDS_GWS_RESOURCE_CNTL__UNUSED_MASK                                                                    0xFFFFFFC0L
+//GDS_GWS_RESOURCE
+#define GDS_GWS_RESOURCE__FLAG__SHIFT                                                                         0x0
+#define GDS_GWS_RESOURCE__COUNTER__SHIFT                                                                      0x1
+#define GDS_GWS_RESOURCE__TYPE__SHIFT                                                                         0xd
+#define GDS_GWS_RESOURCE__DED__SHIFT                                                                          0xe
+#define GDS_GWS_RESOURCE__RELEASE_ALL__SHIFT                                                                  0xf
+#define GDS_GWS_RESOURCE__HEAD_QUEUE__SHIFT                                                                   0x10
+#define GDS_GWS_RESOURCE__HEAD_VALID__SHIFT                                                                   0x1d
+#define GDS_GWS_RESOURCE__HEAD_FLAG__SHIFT                                                                    0x1e
+#define GDS_GWS_RESOURCE__HALTED__SHIFT                                                                       0x1f
+#define GDS_GWS_RESOURCE__FLAG_MASK                                                                           0x00000001L
+#define GDS_GWS_RESOURCE__COUNTER_MASK                                                                        0x00001FFEL
+#define GDS_GWS_RESOURCE__TYPE_MASK                                                                           0x00002000L
+#define GDS_GWS_RESOURCE__DED_MASK                                                                            0x00004000L
+#define GDS_GWS_RESOURCE__RELEASE_ALL_MASK                                                                    0x00008000L
+#define GDS_GWS_RESOURCE__HEAD_QUEUE_MASK                                                                     0x1FFF0000L
+#define GDS_GWS_RESOURCE__HEAD_VALID_MASK                                                                     0x20000000L
+#define GDS_GWS_RESOURCE__HEAD_FLAG_MASK                                                                      0x40000000L
+#define GDS_GWS_RESOURCE__HALTED_MASK                                                                         0x80000000L
+//GDS_GWS_RESOURCE_CNT
+#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT__SHIFT                                                             0x0
+#define GDS_GWS_RESOURCE_CNT__UNUSED__SHIFT                                                                   0x10
+#define GDS_GWS_RESOURCE_CNT__RESOURCE_CNT_MASK                                                               0x0000FFFFL
+#define GDS_GWS_RESOURCE_CNT__UNUSED_MASK                                                                     0xFFFF0000L
+//GDS_OA_CNTL
+#define GDS_OA_CNTL__INDEX__SHIFT                                                                             0x0
+#define GDS_OA_CNTL__UNUSED__SHIFT                                                                            0x4
+#define GDS_OA_CNTL__INDEX_MASK                                                                               0x0000000FL
+#define GDS_OA_CNTL__UNUSED_MASK                                                                              0xFFFFFFF0L
+//GDS_OA_COUNTER
+#define GDS_OA_COUNTER__SPACE_AVAILABLE__SHIFT                                                                0x0
+#define GDS_OA_COUNTER__SPACE_AVAILABLE_MASK                                                                  0xFFFFFFFFL
+//GDS_OA_ADDRESS
+#define GDS_OA_ADDRESS__DS_ADDRESS__SHIFT                                                                     0x0
+#define GDS_OA_ADDRESS__CRAWLER_TYPE__SHIFT                                                                   0x10
+#define GDS_OA_ADDRESS__CRAWLER__SHIFT                                                                        0x14
+#define GDS_OA_ADDRESS__UNUSED__SHIFT                                                                         0x18
+#define GDS_OA_ADDRESS__NO_ALLOC__SHIFT                                                                       0x1e
+#define GDS_OA_ADDRESS__ENABLE__SHIFT                                                                         0x1f
+#define GDS_OA_ADDRESS__DS_ADDRESS_MASK                                                                       0x0000FFFFL
+#define GDS_OA_ADDRESS__CRAWLER_TYPE_MASK                                                                     0x000F0000L
+#define GDS_OA_ADDRESS__CRAWLER_MASK                                                                          0x00F00000L
+#define GDS_OA_ADDRESS__UNUSED_MASK                                                                           0x3F000000L
+#define GDS_OA_ADDRESS__NO_ALLOC_MASK                                                                         0x40000000L
+#define GDS_OA_ADDRESS__ENABLE_MASK                                                                           0x80000000L
+//GDS_OA_INCDEC
+#define GDS_OA_INCDEC__VALUE__SHIFT                                                                           0x0
+#define GDS_OA_INCDEC__INCDEC__SHIFT                                                                          0x1f
+#define GDS_OA_INCDEC__VALUE_MASK                                                                             0x7FFFFFFFL
+#define GDS_OA_INCDEC__INCDEC_MASK                                                                            0x80000000L
+//GDS_OA_RING_SIZE
+#define GDS_OA_RING_SIZE__RING_SIZE__SHIFT                                                                    0x0
+#define GDS_OA_RING_SIZE__RING_SIZE_MASK                                                                      0xFFFFFFFFL
+//GDS_STRMOUT_DWORDS_WRITTEN_0
+#define GDS_STRMOUT_DWORDS_WRITTEN_0__DATA__SHIFT                                                             0x0
+#define GDS_STRMOUT_DWORDS_WRITTEN_0__DATA_MASK                                                               0xFFFFFFFFL
+//GDS_STRMOUT_DWORDS_WRITTEN_1
+#define GDS_STRMOUT_DWORDS_WRITTEN_1__DATA__SHIFT                                                             0x0
+#define GDS_STRMOUT_DWORDS_WRITTEN_1__DATA_MASK                                                               0xFFFFFFFFL
+//GDS_STRMOUT_DWORDS_WRITTEN_2
+#define GDS_STRMOUT_DWORDS_WRITTEN_2__DATA__SHIFT                                                             0x0
+#define GDS_STRMOUT_DWORDS_WRITTEN_2__DATA_MASK                                                               0xFFFFFFFFL
+//GDS_STRMOUT_DWORDS_WRITTEN_3
+#define GDS_STRMOUT_DWORDS_WRITTEN_3__DATA__SHIFT                                                             0x0
+#define GDS_STRMOUT_DWORDS_WRITTEN_3__DATA_MASK                                                               0xFFFFFFFFL
+//GDS_GS_0
+#define GDS_GS_0__DATA__SHIFT                                                                                 0x0
+#define GDS_GS_0__DATA_MASK                                                                                   0xFFFFFFFFL
+//GDS_GS_1
+#define GDS_GS_1__DATA__SHIFT                                                                                 0x0
+#define GDS_GS_1__DATA_MASK                                                                                   0xFFFFFFFFL
+//GDS_GS_2
+#define GDS_GS_2__DATA__SHIFT                                                                                 0x0
+#define GDS_GS_2__DATA_MASK                                                                                   0xFFFFFFFFL
+//GDS_GS_3
+#define GDS_GS_3__DATA__SHIFT                                                                                 0x0
+#define GDS_GS_3__DATA_MASK                                                                                   0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_NEEDED_0_LO
+#define GDS_STRMOUT_PRIMS_NEEDED_0_LO__DATA__SHIFT                                                            0x0
+#define GDS_STRMOUT_PRIMS_NEEDED_0_LO__DATA_MASK                                                              0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_NEEDED_0_HI
+#define GDS_STRMOUT_PRIMS_NEEDED_0_HI__DATA__SHIFT                                                            0x0
+#define GDS_STRMOUT_PRIMS_NEEDED_0_HI__DATA_MASK                                                              0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_WRITTEN_0_LO
+#define GDS_STRMOUT_PRIMS_WRITTEN_0_LO__DATA__SHIFT                                                           0x0
+#define GDS_STRMOUT_PRIMS_WRITTEN_0_LO__DATA_MASK                                                             0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_WRITTEN_0_HI
+#define GDS_STRMOUT_PRIMS_WRITTEN_0_HI__DATA__SHIFT                                                           0x0
+#define GDS_STRMOUT_PRIMS_WRITTEN_0_HI__DATA_MASK                                                             0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_NEEDED_1_LO
+#define GDS_STRMOUT_PRIMS_NEEDED_1_LO__DATA__SHIFT                                                            0x0
+#define GDS_STRMOUT_PRIMS_NEEDED_1_LO__DATA_MASK                                                              0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_NEEDED_1_HI
+#define GDS_STRMOUT_PRIMS_NEEDED_1_HI__DATA__SHIFT                                                            0x0
+#define GDS_STRMOUT_PRIMS_NEEDED_1_HI__DATA_MASK                                                              0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_WRITTEN_1_LO
+#define GDS_STRMOUT_PRIMS_WRITTEN_1_LO__DATA__SHIFT                                                           0x0
+#define GDS_STRMOUT_PRIMS_WRITTEN_1_LO__DATA_MASK                                                             0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_WRITTEN_1_HI
+#define GDS_STRMOUT_PRIMS_WRITTEN_1_HI__DATA__SHIFT                                                           0x0
+#define GDS_STRMOUT_PRIMS_WRITTEN_1_HI__DATA_MASK                                                             0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_NEEDED_2_LO
+#define GDS_STRMOUT_PRIMS_NEEDED_2_LO__DATA__SHIFT                                                            0x0
+#define GDS_STRMOUT_PRIMS_NEEDED_2_LO__DATA_MASK                                                              0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_NEEDED_2_HI
+#define GDS_STRMOUT_PRIMS_NEEDED_2_HI__DATA__SHIFT                                                            0x0
+#define GDS_STRMOUT_PRIMS_NEEDED_2_HI__DATA_MASK                                                              0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_WRITTEN_2_LO
+#define GDS_STRMOUT_PRIMS_WRITTEN_2_LO__DATA__SHIFT                                                           0x0
+#define GDS_STRMOUT_PRIMS_WRITTEN_2_LO__DATA_MASK                                                             0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_WRITTEN_2_HI
+#define GDS_STRMOUT_PRIMS_WRITTEN_2_HI__DATA__SHIFT                                                           0x0
+#define GDS_STRMOUT_PRIMS_WRITTEN_2_HI__DATA_MASK                                                             0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_NEEDED_3_LO
+#define GDS_STRMOUT_PRIMS_NEEDED_3_LO__DATA__SHIFT                                                            0x0
+#define GDS_STRMOUT_PRIMS_NEEDED_3_LO__DATA_MASK                                                              0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_NEEDED_3_HI
+#define GDS_STRMOUT_PRIMS_NEEDED_3_HI__DATA__SHIFT                                                            0x0
+#define GDS_STRMOUT_PRIMS_NEEDED_3_HI__DATA_MASK                                                              0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_WRITTEN_3_LO
+#define GDS_STRMOUT_PRIMS_WRITTEN_3_LO__DATA__SHIFT                                                           0x0
+#define GDS_STRMOUT_PRIMS_WRITTEN_3_LO__DATA_MASK                                                             0xFFFFFFFFL
+//GDS_STRMOUT_PRIMS_WRITTEN_3_HI
+#define GDS_STRMOUT_PRIMS_WRITTEN_3_HI__DATA__SHIFT                                                           0x0
+#define GDS_STRMOUT_PRIMS_WRITTEN_3_HI__DATA_MASK                                                             0xFFFFFFFFL
+//SPI_CONFIG_CNTL
+#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY__SHIFT                                                            0x0
+#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER__SHIFT                                                            0x15
+#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS__SHIFT                                                         0x18
+#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS__SHIFT                                                         0x19
+#define SPI_CONFIG_CNTL__ALLOC_ARB_LRU_ENA__SHIFT                                                             0x1c
+#define SPI_CONFIG_CNTL__EXP_ARB_LRU_ENA__SHIFT                                                               0x1d
+#define SPI_CONFIG_CNTL__PS_PKR_PRIORITY_CNTL__SHIFT                                                          0x1e
+#define SPI_CONFIG_CNTL__GPR_WRITE_PRIORITY_MASK                                                              0x001FFFFFL
+#define SPI_CONFIG_CNTL__EXP_PRIORITY_ORDER_MASK                                                              0x00E00000L
+#define SPI_CONFIG_CNTL__ENABLE_SQG_TOP_EVENTS_MASK                                                           0x01000000L
+#define SPI_CONFIG_CNTL__ENABLE_SQG_BOP_EVENTS_MASK                                                           0x02000000L
+#define SPI_CONFIG_CNTL__ALLOC_ARB_LRU_ENA_MASK                                                               0x10000000L
+#define SPI_CONFIG_CNTL__EXP_ARB_LRU_ENA_MASK                                                                 0x20000000L
+#define SPI_CONFIG_CNTL__PS_PKR_PRIORITY_CNTL_MASK                                                            0xC0000000L
+//SPI_CONFIG_CNTL_1
+#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY__SHIFT                                                              0x0
+#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW__SHIFT                                                     0x4
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE__SHIFT                                                             0x5
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT__SHIFT                                                             0x7
+#define SPI_CONFIG_CNTL_1__PS_GROUP_TIMEOUT_MODE__SHIFT                                                       0x8
+#define SPI_CONFIG_CNTL_1__OREO_EXPALLOC_STALL__SHIFT                                                         0x9
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT__SHIFT                                                             0xa
+#define SPI_CONFIG_CNTL_1__CSC_PWR_SAVE_DISABLE__SHIFT                                                        0xe
+#define SPI_CONFIG_CNTL_1__CSG_PWR_SAVE_DISABLE__SHIFT                                                        0xf
+#define SPI_CONFIG_CNTL_1__MAX_VTX_SYNC_CNT__SHIFT                                                            0x10
+#define SPI_CONFIG_CNTL_1__EN_USER_ACCUM__SHIFT                                                               0x15
+#define SPI_CONFIG_CNTL_1__SA_SCREEN_MAP__SHIFT                                                               0x16
+#define SPI_CONFIG_CNTL_1__PS_GROUP_TIMEOUT__SHIFT                                                            0x17
+#define SPI_CONFIG_CNTL_1__VTX_DONE_DELAY_MASK                                                                0x0000000FL
+#define SPI_CONFIG_CNTL_1__INTERP_ONE_PRIM_PER_ROW_MASK                                                       0x00000010L
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_ENABLE_MASK                                                               0x00000060L
+#define SPI_CONFIG_CNTL_1__PC_LIMIT_STRICT_MASK                                                               0x00000080L
+#define SPI_CONFIG_CNTL_1__PS_GROUP_TIMEOUT_MODE_MASK                                                         0x00000100L
+#define SPI_CONFIG_CNTL_1__OREO_EXPALLOC_STALL_MASK                                                           0x00000200L
+#define SPI_CONFIG_CNTL_1__LBPW_CU_CHK_CNT_MASK                                                               0x00003C00L
+#define SPI_CONFIG_CNTL_1__CSC_PWR_SAVE_DISABLE_MASK                                                          0x00004000L
+#define SPI_CONFIG_CNTL_1__CSG_PWR_SAVE_DISABLE_MASK                                                          0x00008000L
+#define SPI_CONFIG_CNTL_1__MAX_VTX_SYNC_CNT_MASK                                                              0x001F0000L
+#define SPI_CONFIG_CNTL_1__EN_USER_ACCUM_MASK                                                                 0x00200000L
+#define SPI_CONFIG_CNTL_1__SA_SCREEN_MAP_MASK                                                                 0x00400000L
+#define SPI_CONFIG_CNTL_1__PS_GROUP_TIMEOUT_MASK                                                              0xFF800000L
+//SPI_CONFIG_CNTL_2
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_REQUEST_CYCLE_OVHD__SHIFT                                    0x0
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_GRANT_CYCLE_OVHD__SHIFT                                      0x4
+#define SPI_CONFIG_CNTL_2__PWS_CSG_WAIT_DISABLE__SHIFT                                                        0x8
+#define SPI_CONFIG_CNTL_2__PWS_HS_WAIT_DISABLE__SHIFT                                                         0x9
+#define SPI_CONFIG_CNTL_2__PWS_GS_WAIT_DISABLE__SHIFT                                                         0xa
+#define SPI_CONFIG_CNTL_2__PWS_PS_WAIT_DISABLE__SHIFT                                                         0xb
+#define SPI_CONFIG_CNTL_2__CSC_HALT_ACK_DELAY__SHIFT                                                          0xc
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_REQUEST_CYCLE_OVHD_MASK                                      0x0000000FL
+#define SPI_CONFIG_CNTL_2__CONTEXT_SAVE_WAIT_GDS_GRANT_CYCLE_OVHD_MASK                                        0x000000F0L
+#define SPI_CONFIG_CNTL_2__PWS_CSG_WAIT_DISABLE_MASK                                                          0x00000100L
+#define SPI_CONFIG_CNTL_2__PWS_HS_WAIT_DISABLE_MASK                                                           0x00000200L
+#define SPI_CONFIG_CNTL_2__PWS_GS_WAIT_DISABLE_MASK                                                           0x00000400L
+#define SPI_CONFIG_CNTL_2__PWS_PS_WAIT_DISABLE_MASK                                                           0x00000800L
+#define SPI_CONFIG_CNTL_2__CSC_HALT_ACK_DELAY_MASK                                                            0x0001F000L
+//SPI_WAVE_LIMIT_CNTL
+#define SPI_WAVE_LIMIT_CNTL__PS_WAVE_GRAN__SHIFT                                                              0x0
+#define SPI_WAVE_LIMIT_CNTL__GS_WAVE_GRAN__SHIFT                                                              0x4
+#define SPI_WAVE_LIMIT_CNTL__HS_WAVE_GRAN__SHIFT                                                              0x6
+#define SPI_WAVE_LIMIT_CNTL__PS_WAVE_GRAN_MASK                                                                0x00000003L
+#define SPI_WAVE_LIMIT_CNTL__GS_WAVE_GRAN_MASK                                                                0x00000030L
+#define SPI_WAVE_LIMIT_CNTL__HS_WAVE_GRAN_MASK                                                                0x000000C0L
+//SPI_GS_THROTTLE_CNTL1
+#define SPI_GS_THROTTLE_CNTL1__PH_POLL_INTERVAL__SHIFT                                                        0x0
+#define SPI_GS_THROTTLE_CNTL1__PH_THROTTLE_BASE__SHIFT                                                        0x4
+#define SPI_GS_THROTTLE_CNTL1__PH_THROTTLE_STEP_SIZE__SHIFT                                                   0x8
+#define SPI_GS_THROTTLE_CNTL1__SPI_VGPR_THRESHOLD__SHIFT                                                      0xc
+#define SPI_GS_THROTTLE_CNTL1__SPI_LDS_THRESHOLD__SHIFT                                                       0x10
+#define SPI_GS_THROTTLE_CNTL1__SPI_POLL_INTERVAL__SHIFT                                                       0x14
+#define SPI_GS_THROTTLE_CNTL1__SPI_THROTTLE_BASE__SHIFT                                                       0x18
+#define SPI_GS_THROTTLE_CNTL1__SPI_THROTTLE_STEP_SIZE__SHIFT                                                  0x1c
+#define SPI_GS_THROTTLE_CNTL1__PH_POLL_INTERVAL_MASK                                                          0x0000000FL
+#define SPI_GS_THROTTLE_CNTL1__PH_THROTTLE_BASE_MASK                                                          0x000000F0L
+#define SPI_GS_THROTTLE_CNTL1__PH_THROTTLE_STEP_SIZE_MASK                                                     0x00000F00L
+#define SPI_GS_THROTTLE_CNTL1__SPI_VGPR_THRESHOLD_MASK                                                        0x0000F000L
+#define SPI_GS_THROTTLE_CNTL1__SPI_LDS_THRESHOLD_MASK                                                         0x000F0000L
+#define SPI_GS_THROTTLE_CNTL1__SPI_POLL_INTERVAL_MASK                                                         0x00F00000L
+#define SPI_GS_THROTTLE_CNTL1__SPI_THROTTLE_BASE_MASK                                                         0x0F000000L
+#define SPI_GS_THROTTLE_CNTL1__SPI_THROTTLE_STEP_SIZE_MASK                                                    0xF0000000L
+//SPI_GS_THROTTLE_CNTL2
+#define SPI_GS_THROTTLE_CNTL2__SPI_THROTTLE_MODE__SHIFT                                                       0x0
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_THRESHOLD__SHIFT                                                  0x2
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_THRESHOLD_FACTOR__SHIFT                                           0x6
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_PENALTY1__SHIFT                                                   0x8
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_PENALTY2__SHIFT                                                   0xb
+#define SPI_GS_THROTTLE_CNTL2__PS_STALL_THRESHOLD__SHIFT                                                      0xe
+#define SPI_GS_THROTTLE_CNTL2__PH_MODE__SHIFT                                                                 0x10
+#define SPI_GS_THROTTLE_CNTL2__RESERVED__SHIFT                                                                0x11
+#define SPI_GS_THROTTLE_CNTL2__SPI_THROTTLE_MODE_MASK                                                         0x00000003L
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_THRESHOLD_MASK                                                    0x0000003CL
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_THRESHOLD_FACTOR_MASK                                             0x000000C0L
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_PENALTY1_MASK                                                     0x00000700L
+#define SPI_GS_THROTTLE_CNTL2__GRP_LIFETIME_PENALTY2_MASK                                                     0x00003800L
+#define SPI_GS_THROTTLE_CNTL2__PS_STALL_THRESHOLD_MASK                                                        0x0000C000L
+#define SPI_GS_THROTTLE_CNTL2__PH_MODE_MASK                                                                   0x00010000L
+#define SPI_GS_THROTTLE_CNTL2__RESERVED_MASK                                                                  0xFFFE0000L
+//SPI_ATTRIBUTE_RING_BASE
+#define SPI_ATTRIBUTE_RING_BASE__BASE__SHIFT                                                                  0x0
+#define SPI_ATTRIBUTE_RING_BASE__BASE_MASK                                                                    0xFFFFFFFFL
+//SPI_ATTRIBUTE_RING_SIZE
+#define SPI_ATTRIBUTE_RING_SIZE__MEM_SIZE__SHIFT                                                              0x0
+#define SPI_ATTRIBUTE_RING_SIZE__BIG_PAGE__SHIFT                                                              0x10
+#define SPI_ATTRIBUTE_RING_SIZE__L1_POLICY__SHIFT                                                             0x11
+#define SPI_ATTRIBUTE_RING_SIZE__L2_POLICY__SHIFT                                                             0x13
+#define SPI_ATTRIBUTE_RING_SIZE__LLC_NOALLOC__SHIFT                                                           0x15
+#define SPI_ATTRIBUTE_RING_SIZE__GL1_PERF_COUNTER_DISABLE__SHIFT                                              0x16
+#define SPI_ATTRIBUTE_RING_SIZE__MEM_SIZE_MASK                                                                0x000000FFL
+#define SPI_ATTRIBUTE_RING_SIZE__BIG_PAGE_MASK                                                                0x00010000L
+#define SPI_ATTRIBUTE_RING_SIZE__L1_POLICY_MASK                                                               0x00060000L
+#define SPI_ATTRIBUTE_RING_SIZE__L2_POLICY_MASK                                                               0x00180000L
+#define SPI_ATTRIBUTE_RING_SIZE__LLC_NOALLOC_MASK                                                             0x00200000L
+#define SPI_ATTRIBUTE_RING_SIZE__GL1_PERF_COUNTER_DISABLE_MASK                                                0x00400000L
+
+
+// addressBlock: gc_cprs64dec
+//CP_MES_PRGRM_CNTR_START
+#define CP_MES_PRGRM_CNTR_START__IP_START__SHIFT                                                              0x0
+#define CP_MES_PRGRM_CNTR_START__IP_START_MASK                                                                0xFFFFFFFFL
+//CP_MES_INTR_ROUTINE_START
+#define CP_MES_INTR_ROUTINE_START__IR_START__SHIFT                                                            0x0
+#define CP_MES_INTR_ROUTINE_START__IR_START_MASK                                                              0xFFFFFFFFL
+//CP_MES_MTVEC_LO
+#define CP_MES_MTVEC_LO__ADDR_LO__SHIFT                                                                       0x0
+#define CP_MES_MTVEC_LO__ADDR_LO_MASK                                                                         0xFFFFFFFFL
+//CP_MES_INTR_ROUTINE_START_HI
+#define CP_MES_INTR_ROUTINE_START_HI__IR_START__SHIFT                                                         0x0
+#define CP_MES_INTR_ROUTINE_START_HI__IR_START_MASK                                                           0xFFFFFFFFL
+//CP_MES_MTVEC_HI
+#define CP_MES_MTVEC_HI__ADDR_LO__SHIFT                                                                       0x0
+#define CP_MES_MTVEC_HI__ADDR_LO_MASK                                                                         0xFFFFFFFFL
+//CP_MES_CNTL
+#define CP_MES_CNTL__MES_INVALIDATE_ICACHE__SHIFT                                                             0x4
+#define CP_MES_CNTL__MES_PIPE0_RESET__SHIFT                                                                   0x10
+#define CP_MES_CNTL__MES_PIPE1_RESET__SHIFT                                                                   0x11
+#define CP_MES_CNTL__MES_PIPE2_RESET__SHIFT                                                                   0x12
+#define CP_MES_CNTL__MES_PIPE3_RESET__SHIFT                                                                   0x13
+#define CP_MES_CNTL__MES_PIPE0_ACTIVE__SHIFT                                                                  0x1a
+#define CP_MES_CNTL__MES_PIPE1_ACTIVE__SHIFT                                                                  0x1b
+#define CP_MES_CNTL__MES_PIPE2_ACTIVE__SHIFT                                                                  0x1c
+#define CP_MES_CNTL__MES_PIPE3_ACTIVE__SHIFT                                                                  0x1d
+#define CP_MES_CNTL__MES_HALT__SHIFT                                                                          0x1e
+#define CP_MES_CNTL__MES_STEP__SHIFT                                                                          0x1f
+#define CP_MES_CNTL__MES_INVALIDATE_ICACHE_MASK                                                               0x00000010L
+#define CP_MES_CNTL__MES_PIPE0_RESET_MASK                                                                     0x00010000L
+#define CP_MES_CNTL__MES_PIPE1_RESET_MASK                                                                     0x00020000L
+#define CP_MES_CNTL__MES_PIPE2_RESET_MASK                                                                     0x00040000L
+#define CP_MES_CNTL__MES_PIPE3_RESET_MASK                                                                     0x00080000L
+#define CP_MES_CNTL__MES_PIPE0_ACTIVE_MASK                                                                    0x04000000L
+#define CP_MES_CNTL__MES_PIPE1_ACTIVE_MASK                                                                    0x08000000L
+#define CP_MES_CNTL__MES_PIPE2_ACTIVE_MASK                                                                    0x10000000L
+#define CP_MES_CNTL__MES_PIPE3_ACTIVE_MASK                                                                    0x20000000L
+#define CP_MES_CNTL__MES_HALT_MASK                                                                            0x40000000L
+#define CP_MES_CNTL__MES_STEP_MASK                                                                            0x80000000L
+//CP_MES_PIPE_PRIORITY_CNTS
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY1_CNT__SHIFT                                                       0x0
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT__SHIFT                                                      0x8
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT__SHIFT                                                      0x10
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY3_CNT__SHIFT                                                       0x18
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY1_CNT_MASK                                                         0x000000FFL
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY2A_CNT_MASK                                                        0x0000FF00L
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY2B_CNT_MASK                                                        0x00FF0000L
+#define CP_MES_PIPE_PRIORITY_CNTS__PRIORITY3_CNT_MASK                                                         0xFF000000L
+//CP_MES_PIPE0_PRIORITY
+#define CP_MES_PIPE0_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_MES_PIPE0_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_MES_PIPE1_PRIORITY
+#define CP_MES_PIPE1_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_MES_PIPE1_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_MES_PIPE2_PRIORITY
+#define CP_MES_PIPE2_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_MES_PIPE2_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_MES_PIPE3_PRIORITY
+#define CP_MES_PIPE3_PRIORITY__PRIORITY__SHIFT                                                                0x0
+#define CP_MES_PIPE3_PRIORITY__PRIORITY_MASK                                                                  0x00000003L
+//CP_MES_HEADER_DUMP
+#define CP_MES_HEADER_DUMP__HEADER_DUMP__SHIFT                                                                0x0
+#define CP_MES_HEADER_DUMP__HEADER_DUMP_MASK                                                                  0xFFFFFFFFL
+//CP_MES_MIE_LO
+#define CP_MES_MIE_LO__MES_INT__SHIFT                                                                         0x0
+#define CP_MES_MIE_LO__MES_INT_MASK                                                                           0xFFFFFFFFL
+//CP_MES_MIE_HI
+#define CP_MES_MIE_HI__MES_INT__SHIFT                                                                         0x0
+#define CP_MES_MIE_HI__MES_INT_MASK                                                                           0xFFFFFFFFL
+//CP_MES_INTERRUPT
+#define CP_MES_INTERRUPT__MES_INT__SHIFT                                                                      0x0
+#define CP_MES_INTERRUPT__MES_INT_MASK                                                                        0xFFFFFFFFL
+//CP_MES_SCRATCH_INDEX
+#define CP_MES_SCRATCH_INDEX__SCRATCH_INDEX__SHIFT                                                            0x0
+#define CP_MES_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE__SHIFT                                                 0x1f
+#define CP_MES_SCRATCH_INDEX__SCRATCH_INDEX_MASK                                                              0x000001FFL
+#define CP_MES_SCRATCH_INDEX__SCRATCH_INDEX_64BIT_MODE_MASK                                                   0x80000000L
+//CP_MES_SCRATCH_DATA
+#define CP_MES_SCRATCH_DATA__SCRATCH_DATA__SHIFT                                                              0x0
+#define CP_MES_SCRATCH_DATA__SCRATCH_DATA_MASK                                                                0xFFFFFFFFL
+//CP_MES_INSTR_PNTR
+#define CP_MES_INSTR_PNTR__INSTR_PNTR__SHIFT                                                                  0x0
+#define CP_MES_INSTR_PNTR__INSTR_PNTR_MASK                                                                    0x000FFFFFL
+//CP_MES_MSCRATCH_HI
+#define CP_MES_MSCRATCH_HI__DATA__SHIFT                                                                       0x0
+#define CP_MES_MSCRATCH_HI__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_MES_MSCRATCH_LO
+#define CP_MES_MSCRATCH_LO__DATA__SHIFT                                                                       0x0
+#define CP_MES_MSCRATCH_LO__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_MES_MSTATUS_LO
+#define CP_MES_MSTATUS_LO__STATUS_LO__SHIFT                                                                   0x0
+#define CP_MES_MSTATUS_LO__STATUS_LO_MASK                                                                     0xFFFFFFFFL
+//CP_MES_MSTATUS_HI
+#define CP_MES_MSTATUS_HI__STATUS_HI__SHIFT                                                                   0x0
+#define CP_MES_MSTATUS_HI__STATUS_HI_MASK                                                                     0xFFFFFFFFL
+//CP_MES_MEPC_LO
+#define CP_MES_MEPC_LO__MEPC_LO__SHIFT                                                                        0x0
+#define CP_MES_MEPC_LO__MEPC_LO_MASK                                                                          0xFFFFFFFFL
+//CP_MES_MEPC_HI
+#define CP_MES_MEPC_HI__MEPC_HI__SHIFT                                                                        0x0
+#define CP_MES_MEPC_HI__MEPC_HI_MASK                                                                          0xFFFFFFFFL
+//CP_MES_MCAUSE_LO
+#define CP_MES_MCAUSE_LO__CAUSE_LO__SHIFT                                                                     0x0
+#define CP_MES_MCAUSE_LO__CAUSE_LO_MASK                                                                       0xFFFFFFFFL
+//CP_MES_MCAUSE_HI
+#define CP_MES_MCAUSE_HI__CAUSE_HI__SHIFT                                                                     0x0
+#define CP_MES_MCAUSE_HI__CAUSE_HI_MASK                                                                       0xFFFFFFFFL
+//CP_MES_MBADADDR_LO
+#define CP_MES_MBADADDR_LO__ADDR_LO__SHIFT                                                                    0x0
+#define CP_MES_MBADADDR_LO__ADDR_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MBADADDR_HI
+#define CP_MES_MBADADDR_HI__ADDR_HI__SHIFT                                                                    0x0
+#define CP_MES_MBADADDR_HI__ADDR_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MIP_LO
+#define CP_MES_MIP_LO__MIP_LO__SHIFT                                                                          0x0
+#define CP_MES_MIP_LO__MIP_LO_MASK                                                                            0xFFFFFFFFL
+//CP_MES_MIP_HI
+#define CP_MES_MIP_HI__MIP_HI__SHIFT                                                                          0x0
+#define CP_MES_MIP_HI__MIP_HI_MASK                                                                            0xFFFFFFFFL
+//CP_MES_IC_OP_CNTL
+#define CP_MES_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                            0x0
+#define CP_MES_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                0x4
+#define CP_MES_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                               0x5
+#define CP_MES_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                              0x00000001L
+#define CP_MES_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                  0x00000010L
+#define CP_MES_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                 0x00000020L
+//CP_MES_MCYCLE_LO
+#define CP_MES_MCYCLE_LO__CYCLE_LO__SHIFT                                                                     0x0
+#define CP_MES_MCYCLE_LO__CYCLE_LO_MASK                                                                       0xFFFFFFFFL
+//CP_MES_MCYCLE_HI
+#define CP_MES_MCYCLE_HI__CYCLE_HI__SHIFT                                                                     0x0
+#define CP_MES_MCYCLE_HI__CYCLE_HI_MASK                                                                       0xFFFFFFFFL
+//CP_MES_MTIME_LO
+#define CP_MES_MTIME_LO__TIME_LO__SHIFT                                                                       0x0
+#define CP_MES_MTIME_LO__TIME_LO_MASK                                                                         0xFFFFFFFFL
+//CP_MES_MTIME_HI
+#define CP_MES_MTIME_HI__TIME_HI__SHIFT                                                                       0x0
+#define CP_MES_MTIME_HI__TIME_HI_MASK                                                                         0xFFFFFFFFL
+//CP_MES_MINSTRET_LO
+#define CP_MES_MINSTRET_LO__INSTRET_LO__SHIFT                                                                 0x0
+#define CP_MES_MINSTRET_LO__INSTRET_LO_MASK                                                                   0xFFFFFFFFL
+//CP_MES_MINSTRET_HI
+#define CP_MES_MINSTRET_HI__INSTRET_HI__SHIFT                                                                 0x0
+#define CP_MES_MINSTRET_HI__INSTRET_HI_MASK                                                                   0xFFFFFFFFL
+//CP_MES_MISA_LO
+#define CP_MES_MISA_LO__MISA_LO__SHIFT                                                                        0x0
+#define CP_MES_MISA_LO__MISA_LO_MASK                                                                          0xFFFFFFFFL
+//CP_MES_MISA_HI
+#define CP_MES_MISA_HI__MISA_HI__SHIFT                                                                        0x0
+#define CP_MES_MISA_HI__MISA_HI_MASK                                                                          0xFFFFFFFFL
+//CP_MES_MVENDORID_LO
+#define CP_MES_MVENDORID_LO__MVENDORID_LO__SHIFT                                                              0x0
+#define CP_MES_MVENDORID_LO__MVENDORID_LO_MASK                                                                0xFFFFFFFFL
+//CP_MES_MVENDORID_HI
+#define CP_MES_MVENDORID_HI__MVENDORID_HI__SHIFT                                                              0x0
+#define CP_MES_MVENDORID_HI__MVENDORID_HI_MASK                                                                0xFFFFFFFFL
+//CP_MES_MARCHID_LO
+#define CP_MES_MARCHID_LO__MARCHID_LO__SHIFT                                                                  0x0
+#define CP_MES_MARCHID_LO__MARCHID_LO_MASK                                                                    0xFFFFFFFFL
+//CP_MES_MARCHID_HI
+#define CP_MES_MARCHID_HI__MARCHID_HI__SHIFT                                                                  0x0
+#define CP_MES_MARCHID_HI__MARCHID_HI_MASK                                                                    0xFFFFFFFFL
+//CP_MES_MIMPID_LO
+#define CP_MES_MIMPID_LO__MIMPID_LO__SHIFT                                                                    0x0
+#define CP_MES_MIMPID_LO__MIMPID_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MIMPID_HI
+#define CP_MES_MIMPID_HI__MIMPID_HI__SHIFT                                                                    0x0
+#define CP_MES_MIMPID_HI__MIMPID_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MHARTID_LO
+#define CP_MES_MHARTID_LO__MHARTID_LO__SHIFT                                                                  0x0
+#define CP_MES_MHARTID_LO__MHARTID_LO_MASK                                                                    0xFFFFFFFFL
+//CP_MES_MHARTID_HI
+#define CP_MES_MHARTID_HI__MHARTID_HI__SHIFT                                                                  0x0
+#define CP_MES_MHARTID_HI__MHARTID_HI_MASK                                                                    0xFFFFFFFFL
+//CP_MES_DC_BASE_CNTL
+#define CP_MES_DC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_MES_DC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_MES_DC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_MES_DC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x03000000L
+//CP_MES_DC_OP_CNTL
+#define CP_MES_DC_OP_CNTL__INVALIDATE_DCACHE__SHIFT                                                           0x0
+#define CP_MES_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE__SHIFT                                                  0x1
+#define CP_MES_DC_OP_CNTL__BYPASS_ALL__SHIFT                                                                  0x2
+#define CP_MES_DC_OP_CNTL__INVALIDATE_DCACHE_MASK                                                             0x00000001L
+#define CP_MES_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE_MASK                                                    0x00000002L
+#define CP_MES_DC_OP_CNTL__BYPASS_ALL_MASK                                                                    0x00000004L
+//CP_MES_MTIMECMP_LO
+#define CP_MES_MTIMECMP_LO__TIME_LO__SHIFT                                                                    0x0
+#define CP_MES_MTIMECMP_LO__TIME_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MTIMECMP_HI
+#define CP_MES_MTIMECMP_HI__TIME_HI__SHIFT                                                                    0x0
+#define CP_MES_MTIMECMP_HI__TIME_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MES_PROCESS_QUANTUM_PIPE0
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_DURATION__SHIFT                                                 0x0
+#define CP_MES_PROCESS_QUANTUM_PIPE0__TIMER_EXPIRED__SHIFT                                                    0x1c
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_SCALE__SHIFT                                                    0x1d
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_EN__SHIFT                                                       0x1f
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_DURATION_MASK                                                   0x0FFFFFFFL
+#define CP_MES_PROCESS_QUANTUM_PIPE0__TIMER_EXPIRED_MASK                                                      0x10000000L
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_SCALE_MASK                                                      0x60000000L
+#define CP_MES_PROCESS_QUANTUM_PIPE0__QUANTUM_EN_MASK                                                         0x80000000L
+//CP_MES_PROCESS_QUANTUM_PIPE1
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_DURATION__SHIFT                                                 0x0
+#define CP_MES_PROCESS_QUANTUM_PIPE1__TIMER_EXPIRED__SHIFT                                                    0x1c
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_SCALE__SHIFT                                                    0x1d
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_EN__SHIFT                                                       0x1f
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_DURATION_MASK                                                   0x0FFFFFFFL
+#define CP_MES_PROCESS_QUANTUM_PIPE1__TIMER_EXPIRED_MASK                                                      0x10000000L
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_SCALE_MASK                                                      0x60000000L
+#define CP_MES_PROCESS_QUANTUM_PIPE1__QUANTUM_EN_MASK                                                         0x80000000L
+//CP_MES_DOORBELL_CONTROL1
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL1__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_DOORBELL_CONTROL2
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL2__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_DOORBELL_CONTROL3
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL3__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_DOORBELL_CONTROL4
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL4__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_DOORBELL_CONTROL5
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL5__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_DOORBELL_CONTROL6
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_OFFSET__SHIFT                                                      0x2
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_EN__SHIFT                                                          0x1e
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_HIT__SHIFT                                                         0x1f
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_OFFSET_MASK                                                        0x0FFFFFFCL
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_EN_MASK                                                            0x40000000L
+#define CP_MES_DOORBELL_CONTROL6__DOORBELL_HIT_MASK                                                           0x80000000L
+//CP_MES_GP0_LO
+#define CP_MES_GP0_LO__PG_VIRT_HALTED__SHIFT                                                                  0x0
+#define CP_MES_GP0_LO__DATA__SHIFT                                                                            0x1
+#define CP_MES_GP0_LO__PG_VIRT_HALTED_MASK                                                                    0x00000001L
+#define CP_MES_GP0_LO__DATA_MASK                                                                              0xFFFFFFFEL
+//CP_MES_GP0_HI
+#define CP_MES_GP0_HI__M_RET_ADDR__SHIFT                                                                      0x0
+#define CP_MES_GP0_HI__M_RET_ADDR_MASK                                                                        0xFFFFFFFFL
+//CP_MES_GP1_LO
+#define CP_MES_GP1_LO__RD_WR_SELECT_LO__SHIFT                                                                 0x0
+#define CP_MES_GP1_LO__RD_WR_SELECT_LO_MASK                                                                   0xFFFFFFFFL
+//CP_MES_GP1_HI
+#define CP_MES_GP1_HI__RD_WR_SELECT_HI__SHIFT                                                                 0x0
+#define CP_MES_GP1_HI__RD_WR_SELECT_HI_MASK                                                                   0xFFFFFFFFL
+//CP_MES_GP2_LO
+#define CP_MES_GP2_LO__STACK_PNTR_LO__SHIFT                                                                   0x0
+#define CP_MES_GP2_LO__STACK_PNTR_LO_MASK                                                                     0xFFFFFFFFL
+//CP_MES_GP2_HI
+#define CP_MES_GP2_HI__STACK_PNTR_HI__SHIFT                                                                   0x0
+#define CP_MES_GP2_HI__STACK_PNTR_HI_MASK                                                                     0xFFFFFFFFL
+//CP_MES_GP3_LO
+#define CP_MES_GP3_LO__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP3_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP3_HI
+#define CP_MES_GP3_HI__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP3_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP4_LO
+#define CP_MES_GP4_LO__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP4_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP4_HI
+#define CP_MES_GP4_HI__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP4_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP5_LO
+#define CP_MES_GP5_LO__PG_VIRT_HALTED__SHIFT                                                                  0x0
+#define CP_MES_GP5_LO__DATA__SHIFT                                                                            0x1
+#define CP_MES_GP5_LO__PG_VIRT_HALTED_MASK                                                                    0x00000001L
+#define CP_MES_GP5_LO__DATA_MASK                                                                              0xFFFFFFFEL
+//CP_MES_GP5_HI
+#define CP_MES_GP5_HI__M_RET_ADDR__SHIFT                                                                      0x0
+#define CP_MES_GP5_HI__M_RET_ADDR_MASK                                                                        0xFFFFFFFFL
+//CP_MES_GP6_LO
+#define CP_MES_GP6_LO__RD_WR_SELECT_LO__SHIFT                                                                 0x0
+#define CP_MES_GP6_LO__RD_WR_SELECT_LO_MASK                                                                   0xFFFFFFFFL
+//CP_MES_GP6_HI
+#define CP_MES_GP6_HI__RD_WR_SELECT_HI__SHIFT                                                                 0x0
+#define CP_MES_GP6_HI__RD_WR_SELECT_HI_MASK                                                                   0xFFFFFFFFL
+//CP_MES_GP7_LO
+#define CP_MES_GP7_LO__STACK_PNTR_LO__SHIFT                                                                   0x0
+#define CP_MES_GP7_LO__STACK_PNTR_LO_MASK                                                                     0xFFFFFFFFL
+//CP_MES_GP7_HI
+#define CP_MES_GP7_HI__STACK_PNTR_HI__SHIFT                                                                   0x0
+#define CP_MES_GP7_HI__STACK_PNTR_HI_MASK                                                                     0xFFFFFFFFL
+//CP_MES_GP8_LO
+#define CP_MES_GP8_LO__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP8_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP8_HI
+#define CP_MES_GP8_HI__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP8_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP9_LO
+#define CP_MES_GP9_LO__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP9_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_GP9_HI
+#define CP_MES_GP9_HI__DATA__SHIFT                                                                            0x0
+#define CP_MES_GP9_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MES_LOCAL_BASE0_LO
+#define CP_MES_LOCAL_BASE0_LO__BASE0_LO__SHIFT                                                                0x10
+#define CP_MES_LOCAL_BASE0_LO__BASE0_LO_MASK                                                                  0xFFFF0000L
+//CP_MES_LOCAL_BASE0_HI
+#define CP_MES_LOCAL_BASE0_HI__BASE0_HI__SHIFT                                                                0x0
+#define CP_MES_LOCAL_BASE0_HI__BASE0_HI_MASK                                                                  0x0000FFFFL
+//CP_MES_LOCAL_MASK0_LO
+#define CP_MES_LOCAL_MASK0_LO__MASK0_LO__SHIFT                                                                0x10
+#define CP_MES_LOCAL_MASK0_LO__MASK0_LO_MASK                                                                  0xFFFF0000L
+//CP_MES_LOCAL_MASK0_HI
+#define CP_MES_LOCAL_MASK0_HI__MASK0_HI__SHIFT                                                                0x0
+#define CP_MES_LOCAL_MASK0_HI__MASK0_HI_MASK                                                                  0x0000FFFFL
+//CP_MES_LOCAL_APERTURE
+#define CP_MES_LOCAL_APERTURE__APERTURE__SHIFT                                                                0x0
+#define CP_MES_LOCAL_APERTURE__APERTURE_MASK                                                                  0x00000007L
+//CP_MES_LOCAL_INSTR_BASE_LO
+#define CP_MES_LOCAL_INSTR_BASE_LO__BASE_LO__SHIFT                                                            0x10
+#define CP_MES_LOCAL_INSTR_BASE_LO__BASE_LO_MASK                                                              0xFFFF0000L
+//CP_MES_LOCAL_INSTR_BASE_HI
+#define CP_MES_LOCAL_INSTR_BASE_HI__BASE_HI__SHIFT                                                            0x0
+#define CP_MES_LOCAL_INSTR_BASE_HI__BASE_HI_MASK                                                              0x0000FFFFL
+//CP_MES_LOCAL_INSTR_MASK_LO
+#define CP_MES_LOCAL_INSTR_MASK_LO__MASK_LO__SHIFT                                                            0x10
+#define CP_MES_LOCAL_INSTR_MASK_LO__MASK_LO_MASK                                                              0xFFFF0000L
+//CP_MES_LOCAL_INSTR_MASK_HI
+#define CP_MES_LOCAL_INSTR_MASK_HI__MASK_HI__SHIFT                                                            0x0
+#define CP_MES_LOCAL_INSTR_MASK_HI__MASK_HI_MASK                                                              0x0000FFFFL
+//CP_MES_LOCAL_INSTR_APERTURE
+#define CP_MES_LOCAL_INSTR_APERTURE__APERTURE__SHIFT                                                          0x0
+#define CP_MES_LOCAL_INSTR_APERTURE__APERTURE_MASK                                                            0x00000007L
+//CP_MES_LOCAL_SCRATCH_APERTURE
+#define CP_MES_LOCAL_SCRATCH_APERTURE__APERTURE__SHIFT                                                        0x0
+#define CP_MES_LOCAL_SCRATCH_APERTURE__APERTURE_MASK                                                          0x00000007L
+//CP_MES_LOCAL_SCRATCH_BASE_LO
+#define CP_MES_LOCAL_SCRATCH_BASE_LO__BASE_LO__SHIFT                                                          0x10
+#define CP_MES_LOCAL_SCRATCH_BASE_LO__BASE_LO_MASK                                                            0xFFFF0000L
+//CP_MES_LOCAL_SCRATCH_BASE_HI
+#define CP_MES_LOCAL_SCRATCH_BASE_HI__BASE_HI__SHIFT                                                          0x0
+#define CP_MES_LOCAL_SCRATCH_BASE_HI__BASE_HI_MASK                                                            0x0000FFFFL
+//CP_MES_PERFCOUNT_CNTL
+#define CP_MES_PERFCOUNT_CNTL__EVENT_SEL__SHIFT                                                               0x0
+#define CP_MES_PERFCOUNT_CNTL__EVENT_SEL_MASK                                                                 0x0000001FL
+//CP_MES_PENDING_INTERRUPT
+#define CP_MES_PENDING_INTERRUPT__PENDING_INTERRUPT__SHIFT                                                    0x0
+#define CP_MES_PENDING_INTERRUPT__PENDING_INTERRUPT_MASK                                                      0xFFFFFFFFL
+//CP_MES_PRGRM_CNTR_START_HI
+#define CP_MES_PRGRM_CNTR_START_HI__IP_START__SHIFT                                                           0x0
+#define CP_MES_PRGRM_CNTR_START_HI__IP_START_MASK                                                             0x3FFFFFFFL
+//CP_MES_INTERRUPT_DATA_16
+#define CP_MES_INTERRUPT_DATA_16__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_16__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_17
+#define CP_MES_INTERRUPT_DATA_17__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_17__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_18
+#define CP_MES_INTERRUPT_DATA_18__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_18__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_19
+#define CP_MES_INTERRUPT_DATA_19__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_19__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_20
+#define CP_MES_INTERRUPT_DATA_20__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_20__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_21
+#define CP_MES_INTERRUPT_DATA_21__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_21__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_22
+#define CP_MES_INTERRUPT_DATA_22__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_22__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_23
+#define CP_MES_INTERRUPT_DATA_23__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_23__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_24
+#define CP_MES_INTERRUPT_DATA_24__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_24__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_25
+#define CP_MES_INTERRUPT_DATA_25__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_25__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_26
+#define CP_MES_INTERRUPT_DATA_26__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_26__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_27
+#define CP_MES_INTERRUPT_DATA_27__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_27__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_28
+#define CP_MES_INTERRUPT_DATA_28__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_28__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_29
+#define CP_MES_INTERRUPT_DATA_29__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_29__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_30
+#define CP_MES_INTERRUPT_DATA_30__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_30__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_INTERRUPT_DATA_31
+#define CP_MES_INTERRUPT_DATA_31__DATA__SHIFT                                                                 0x0
+#define CP_MES_INTERRUPT_DATA_31__DATA_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE0_BASE
+#define CP_MES_DC_APERTURE0_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE0_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE0_MASK
+#define CP_MES_DC_APERTURE0_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE0_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE0_CNTL
+#define CP_MES_DC_APERTURE0_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE0_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MES_DC_APERTURE0_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MES_DC_APERTURE0_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MES_DC_APERTURE1_BASE
+#define CP_MES_DC_APERTURE1_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE1_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE1_MASK
+#define CP_MES_DC_APERTURE1_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE1_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE1_CNTL
+#define CP_MES_DC_APERTURE1_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE1_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MES_DC_APERTURE1_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MES_DC_APERTURE1_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MES_DC_APERTURE2_BASE
+#define CP_MES_DC_APERTURE2_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE2_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE2_MASK
+#define CP_MES_DC_APERTURE2_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE2_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE2_CNTL
+#define CP_MES_DC_APERTURE2_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE2_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MES_DC_APERTURE2_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MES_DC_APERTURE2_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MES_DC_APERTURE3_BASE
+#define CP_MES_DC_APERTURE3_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE3_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE3_MASK
+#define CP_MES_DC_APERTURE3_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE3_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE3_CNTL
+#define CP_MES_DC_APERTURE3_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE3_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MES_DC_APERTURE3_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MES_DC_APERTURE3_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MES_DC_APERTURE4_BASE
+#define CP_MES_DC_APERTURE4_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE4_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE4_MASK
+#define CP_MES_DC_APERTURE4_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE4_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE4_CNTL
+#define CP_MES_DC_APERTURE4_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE4_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MES_DC_APERTURE4_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MES_DC_APERTURE4_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MES_DC_APERTURE5_BASE
+#define CP_MES_DC_APERTURE5_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE5_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE5_MASK
+#define CP_MES_DC_APERTURE5_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE5_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE5_CNTL
+#define CP_MES_DC_APERTURE5_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE5_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MES_DC_APERTURE5_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MES_DC_APERTURE5_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MES_DC_APERTURE6_BASE
+#define CP_MES_DC_APERTURE6_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE6_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE6_MASK
+#define CP_MES_DC_APERTURE6_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE6_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE6_CNTL
+#define CP_MES_DC_APERTURE6_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE6_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MES_DC_APERTURE6_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MES_DC_APERTURE6_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MES_DC_APERTURE7_BASE
+#define CP_MES_DC_APERTURE7_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE7_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE7_MASK
+#define CP_MES_DC_APERTURE7_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE7_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE7_CNTL
+#define CP_MES_DC_APERTURE7_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE7_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MES_DC_APERTURE7_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MES_DC_APERTURE7_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MES_DC_APERTURE8_BASE
+#define CP_MES_DC_APERTURE8_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE8_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE8_MASK
+#define CP_MES_DC_APERTURE8_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE8_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE8_CNTL
+#define CP_MES_DC_APERTURE8_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE8_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MES_DC_APERTURE8_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MES_DC_APERTURE8_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MES_DC_APERTURE9_BASE
+#define CP_MES_DC_APERTURE9_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE9_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE9_MASK
+#define CP_MES_DC_APERTURE9_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE9_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MES_DC_APERTURE9_CNTL
+#define CP_MES_DC_APERTURE9_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MES_DC_APERTURE9_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MES_DC_APERTURE9_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MES_DC_APERTURE9_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MES_DC_APERTURE10_BASE
+#define CP_MES_DC_APERTURE10_BASE__BASE__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE10_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE10_MASK
+#define CP_MES_DC_APERTURE10_MASK__MASK__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE10_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE10_CNTL
+#define CP_MES_DC_APERTURE10_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE10_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MES_DC_APERTURE10_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MES_DC_APERTURE10_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MES_DC_APERTURE11_BASE
+#define CP_MES_DC_APERTURE11_BASE__BASE__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE11_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE11_MASK
+#define CP_MES_DC_APERTURE11_MASK__MASK__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE11_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE11_CNTL
+#define CP_MES_DC_APERTURE11_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE11_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MES_DC_APERTURE11_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MES_DC_APERTURE11_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MES_DC_APERTURE12_BASE
+#define CP_MES_DC_APERTURE12_BASE__BASE__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE12_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE12_MASK
+#define CP_MES_DC_APERTURE12_MASK__MASK__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE12_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE12_CNTL
+#define CP_MES_DC_APERTURE12_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE12_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MES_DC_APERTURE12_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MES_DC_APERTURE12_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MES_DC_APERTURE13_BASE
+#define CP_MES_DC_APERTURE13_BASE__BASE__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE13_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE13_MASK
+#define CP_MES_DC_APERTURE13_MASK__MASK__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE13_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE13_CNTL
+#define CP_MES_DC_APERTURE13_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE13_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MES_DC_APERTURE13_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MES_DC_APERTURE13_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MES_DC_APERTURE14_BASE
+#define CP_MES_DC_APERTURE14_BASE__BASE__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE14_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE14_MASK
+#define CP_MES_DC_APERTURE14_MASK__MASK__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE14_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE14_CNTL
+#define CP_MES_DC_APERTURE14_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE14_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MES_DC_APERTURE14_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MES_DC_APERTURE14_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MES_DC_APERTURE15_BASE
+#define CP_MES_DC_APERTURE15_BASE__BASE__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE15_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE15_MASK
+#define CP_MES_DC_APERTURE15_MASK__MASK__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE15_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MES_DC_APERTURE15_CNTL
+#define CP_MES_DC_APERTURE15_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MES_DC_APERTURE15_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MES_DC_APERTURE15_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MES_DC_APERTURE15_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MEC_RS64_PRGRM_CNTR_START
+#define CP_MEC_RS64_PRGRM_CNTR_START__IP_START__SHIFT                                                         0x0
+#define CP_MEC_RS64_PRGRM_CNTR_START__IP_START_MASK                                                           0xFFFFFFFFL
+//CP_MEC_MTVEC_LO
+#define CP_MEC_MTVEC_LO__ADDR_LO__SHIFT                                                                       0x0
+#define CP_MEC_MTVEC_LO__ADDR_LO_MASK                                                                         0xFFFFFFFFL
+//CP_MEC_MTVEC_HI
+#define CP_MEC_MTVEC_HI__ADDR_LO__SHIFT                                                                       0x0
+#define CP_MEC_MTVEC_HI__ADDR_LO_MASK                                                                         0xFFFFFFFFL
+//CP_MEC_ISA_CNTL
+#define CP_MEC_ISA_CNTL__ISA_MODE__SHIFT                                                                      0x0
+#define CP_MEC_ISA_CNTL__ISA_MODE_MASK                                                                        0x00000001L
+//CP_MEC_RS64_CNTL
+#define CP_MEC_RS64_CNTL__MEC_INVALIDATE_ICACHE__SHIFT                                                        0x4
+#define CP_MEC_RS64_CNTL__MEC_PIPE0_RESET__SHIFT                                                              0x10
+#define CP_MEC_RS64_CNTL__MEC_PIPE1_RESET__SHIFT                                                              0x11
+#define CP_MEC_RS64_CNTL__MEC_PIPE2_RESET__SHIFT                                                              0x12
+#define CP_MEC_RS64_CNTL__MEC_PIPE3_RESET__SHIFT                                                              0x13
+#define CP_MEC_RS64_CNTL__MEC_PIPE0_ACTIVE__SHIFT                                                             0x1a
+#define CP_MEC_RS64_CNTL__MEC_PIPE1_ACTIVE__SHIFT                                                             0x1b
+#define CP_MEC_RS64_CNTL__MEC_PIPE2_ACTIVE__SHIFT                                                             0x1c
+#define CP_MEC_RS64_CNTL__MEC_PIPE3_ACTIVE__SHIFT                                                             0x1d
+#define CP_MEC_RS64_CNTL__MEC_HALT__SHIFT                                                                     0x1e
+#define CP_MEC_RS64_CNTL__MEC_STEP__SHIFT                                                                     0x1f
+#define CP_MEC_RS64_CNTL__MEC_INVALIDATE_ICACHE_MASK                                                          0x00000010L
+#define CP_MEC_RS64_CNTL__MEC_PIPE0_RESET_MASK                                                                0x00010000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE1_RESET_MASK                                                                0x00020000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE2_RESET_MASK                                                                0x00040000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE3_RESET_MASK                                                                0x00080000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE0_ACTIVE_MASK                                                               0x04000000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE1_ACTIVE_MASK                                                               0x08000000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE2_ACTIVE_MASK                                                               0x10000000L
+#define CP_MEC_RS64_CNTL__MEC_PIPE3_ACTIVE_MASK                                                               0x20000000L
+#define CP_MEC_RS64_CNTL__MEC_HALT_MASK                                                                       0x40000000L
+#define CP_MEC_RS64_CNTL__MEC_STEP_MASK                                                                       0x80000000L
+//CP_MEC_MIE_LO
+#define CP_MEC_MIE_LO__MEC_INT__SHIFT                                                                         0x0
+#define CP_MEC_MIE_LO__MEC_INT_MASK                                                                           0xFFFFFFFFL
+//CP_MEC_MIE_HI
+#define CP_MEC_MIE_HI__MEC_INT__SHIFT                                                                         0x0
+#define CP_MEC_MIE_HI__MEC_INT_MASK                                                                           0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT
+#define CP_MEC_RS64_INTERRUPT__MEC_INT__SHIFT                                                                 0x0
+#define CP_MEC_RS64_INTERRUPT__MEC_INT_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_RS64_INSTR_PNTR
+#define CP_MEC_RS64_INSTR_PNTR__INSTR_PNTR__SHIFT                                                             0x0
+#define CP_MEC_RS64_INSTR_PNTR__INSTR_PNTR_MASK                                                               0x000FFFFFL
+//CP_MEC_MIP_LO
+#define CP_MEC_MIP_LO__MIP_LO__SHIFT                                                                          0x0
+#define CP_MEC_MIP_LO__MIP_LO_MASK                                                                            0xFFFFFFFFL
+//CP_MEC_MIP_HI
+#define CP_MEC_MIP_HI__MIP_HI__SHIFT                                                                          0x0
+#define CP_MEC_MIP_HI__MIP_HI_MASK                                                                            0xFFFFFFFFL
+//CP_MEC_DC_BASE_CNTL
+#define CP_MEC_DC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_MEC_DC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_MEC_DC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_MEC_DC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x03000000L
+//CP_MEC_DC_OP_CNTL
+#define CP_MEC_DC_OP_CNTL__INVALIDATE_DCACHE__SHIFT                                                           0x0
+#define CP_MEC_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE__SHIFT                                                  0x1
+#define CP_MEC_DC_OP_CNTL__BYPASS_ALL__SHIFT                                                                  0x2
+#define CP_MEC_DC_OP_CNTL__INVALIDATE_DCACHE_MASK                                                             0x00000001L
+#define CP_MEC_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE_MASK                                                    0x00000002L
+#define CP_MEC_DC_OP_CNTL__BYPASS_ALL_MASK                                                                    0x00000004L
+//CP_MEC_MTIMECMP_LO
+#define CP_MEC_MTIMECMP_LO__TIME_LO__SHIFT                                                                    0x0
+#define CP_MEC_MTIMECMP_LO__TIME_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MEC_MTIMECMP_HI
+#define CP_MEC_MTIMECMP_HI__TIME_HI__SHIFT                                                                    0x0
+#define CP_MEC_MTIMECMP_HI__TIME_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MEC_GP0_LO
+#define CP_MEC_GP0_LO__PG_VIRT_HALTED__SHIFT                                                                  0x0
+#define CP_MEC_GP0_LO__DATA__SHIFT                                                                            0x1
+#define CP_MEC_GP0_LO__PG_VIRT_HALTED_MASK                                                                    0x00000001L
+#define CP_MEC_GP0_LO__DATA_MASK                                                                              0xFFFFFFFEL
+//CP_MEC_GP0_HI
+#define CP_MEC_GP0_HI__M_RET_ADDR__SHIFT                                                                      0x0
+#define CP_MEC_GP0_HI__M_RET_ADDR_MASK                                                                        0xFFFFFFFFL
+//CP_MEC_GP1_LO
+#define CP_MEC_GP1_LO__RD_WR_SELECT_LO__SHIFT                                                                 0x0
+#define CP_MEC_GP1_LO__RD_WR_SELECT_LO_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_GP1_HI
+#define CP_MEC_GP1_HI__RD_WR_SELECT_HI__SHIFT                                                                 0x0
+#define CP_MEC_GP1_HI__RD_WR_SELECT_HI_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_GP2_LO
+#define CP_MEC_GP2_LO__STACK_PNTR_LO__SHIFT                                                                   0x0
+#define CP_MEC_GP2_LO__STACK_PNTR_LO_MASK                                                                     0xFFFFFFFFL
+//CP_MEC_GP2_HI
+#define CP_MEC_GP2_HI__STACK_PNTR_HI__SHIFT                                                                   0x0
+#define CP_MEC_GP2_HI__STACK_PNTR_HI_MASK                                                                     0xFFFFFFFFL
+//CP_MEC_GP3_LO
+#define CP_MEC_GP3_LO__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP3_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP3_HI
+#define CP_MEC_GP3_HI__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP3_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP4_LO
+#define CP_MEC_GP4_LO__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP4_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP4_HI
+#define CP_MEC_GP4_HI__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP4_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP5_LO
+#define CP_MEC_GP5_LO__PG_VIRT_HALTED__SHIFT                                                                  0x0
+#define CP_MEC_GP5_LO__DATA__SHIFT                                                                            0x1
+#define CP_MEC_GP5_LO__PG_VIRT_HALTED_MASK                                                                    0x00000001L
+#define CP_MEC_GP5_LO__DATA_MASK                                                                              0xFFFFFFFEL
+//CP_MEC_GP5_HI
+#define CP_MEC_GP5_HI__M_RET_ADDR__SHIFT                                                                      0x0
+#define CP_MEC_GP5_HI__M_RET_ADDR_MASK                                                                        0xFFFFFFFFL
+//CP_MEC_GP6_LO
+#define CP_MEC_GP6_LO__RD_WR_SELECT_LO__SHIFT                                                                 0x0
+#define CP_MEC_GP6_LO__RD_WR_SELECT_LO_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_GP6_HI
+#define CP_MEC_GP6_HI__RD_WR_SELECT_HI__SHIFT                                                                 0x0
+#define CP_MEC_GP6_HI__RD_WR_SELECT_HI_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_GP7_LO
+#define CP_MEC_GP7_LO__STACK_PNTR_LO__SHIFT                                                                   0x0
+#define CP_MEC_GP7_LO__STACK_PNTR_LO_MASK                                                                     0xFFFFFFFFL
+//CP_MEC_GP7_HI
+#define CP_MEC_GP7_HI__STACK_PNTR_HI__SHIFT                                                                   0x0
+#define CP_MEC_GP7_HI__STACK_PNTR_HI_MASK                                                                     0xFFFFFFFFL
+//CP_MEC_GP8_LO
+#define CP_MEC_GP8_LO__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP8_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP8_HI
+#define CP_MEC_GP8_HI__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP8_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP9_LO
+#define CP_MEC_GP9_LO__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP9_LO__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_GP9_HI
+#define CP_MEC_GP9_HI__DATA__SHIFT                                                                            0x0
+#define CP_MEC_GP9_HI__DATA_MASK                                                                              0xFFFFFFFFL
+//CP_MEC_LOCAL_BASE0_LO
+#define CP_MEC_LOCAL_BASE0_LO__BASE0_LO__SHIFT                                                                0x10
+#define CP_MEC_LOCAL_BASE0_LO__BASE0_LO_MASK                                                                  0xFFFF0000L
+//CP_MEC_LOCAL_BASE0_HI
+#define CP_MEC_LOCAL_BASE0_HI__BASE0_HI__SHIFT                                                                0x0
+#define CP_MEC_LOCAL_BASE0_HI__BASE0_HI_MASK                                                                  0x0000FFFFL
+//CP_MEC_LOCAL_MASK0_LO
+#define CP_MEC_LOCAL_MASK0_LO__MASK0_LO__SHIFT                                                                0x10
+#define CP_MEC_LOCAL_MASK0_LO__MASK0_LO_MASK                                                                  0xFFFF0000L
+//CP_MEC_LOCAL_MASK0_HI
+#define CP_MEC_LOCAL_MASK0_HI__MASK0_HI__SHIFT                                                                0x0
+#define CP_MEC_LOCAL_MASK0_HI__MASK0_HI_MASK                                                                  0x0000FFFFL
+//CP_MEC_LOCAL_APERTURE
+#define CP_MEC_LOCAL_APERTURE__APERTURE__SHIFT                                                                0x0
+#define CP_MEC_LOCAL_APERTURE__APERTURE_MASK                                                                  0x00000007L
+//CP_MEC_LOCAL_INSTR_BASE_LO
+#define CP_MEC_LOCAL_INSTR_BASE_LO__BASE_LO__SHIFT                                                            0x10
+#define CP_MEC_LOCAL_INSTR_BASE_LO__BASE_LO_MASK                                                              0xFFFF0000L
+//CP_MEC_LOCAL_INSTR_BASE_HI
+#define CP_MEC_LOCAL_INSTR_BASE_HI__BASE_HI__SHIFT                                                            0x0
+#define CP_MEC_LOCAL_INSTR_BASE_HI__BASE_HI_MASK                                                              0x0000FFFFL
+//CP_MEC_LOCAL_INSTR_MASK_LO
+#define CP_MEC_LOCAL_INSTR_MASK_LO__MASK_LO__SHIFT                                                            0x10
+#define CP_MEC_LOCAL_INSTR_MASK_LO__MASK_LO_MASK                                                              0xFFFF0000L
+//CP_MEC_LOCAL_INSTR_MASK_HI
+#define CP_MEC_LOCAL_INSTR_MASK_HI__MASK_HI__SHIFT                                                            0x0
+#define CP_MEC_LOCAL_INSTR_MASK_HI__MASK_HI_MASK                                                              0x0000FFFFL
+//CP_MEC_LOCAL_INSTR_APERTURE
+#define CP_MEC_LOCAL_INSTR_APERTURE__APERTURE__SHIFT                                                          0x0
+#define CP_MEC_LOCAL_INSTR_APERTURE__APERTURE_MASK                                                            0x00000007L
+//CP_MEC_LOCAL_SCRATCH_APERTURE
+#define CP_MEC_LOCAL_SCRATCH_APERTURE__APERTURE__SHIFT                                                        0x0
+#define CP_MEC_LOCAL_SCRATCH_APERTURE__APERTURE_MASK                                                          0x00000007L
+//CP_MEC_LOCAL_SCRATCH_BASE_LO
+#define CP_MEC_LOCAL_SCRATCH_BASE_LO__BASE_LO__SHIFT                                                          0x10
+#define CP_MEC_LOCAL_SCRATCH_BASE_LO__BASE_LO_MASK                                                            0xFFFF0000L
+//CP_MEC_LOCAL_SCRATCH_BASE_HI
+#define CP_MEC_LOCAL_SCRATCH_BASE_HI__BASE_HI__SHIFT                                                          0x0
+#define CP_MEC_LOCAL_SCRATCH_BASE_HI__BASE_HI_MASK                                                            0x0000FFFFL
+//CP_MEC_RS64_PERFCOUNT_CNTL
+#define CP_MEC_RS64_PERFCOUNT_CNTL__EVENT_SEL__SHIFT                                                          0x0
+#define CP_MEC_RS64_PERFCOUNT_CNTL__EVENT_SEL_MASK                                                            0x0000001FL
+//CP_MEC_RS64_PENDING_INTERRUPT
+#define CP_MEC_RS64_PENDING_INTERRUPT__PENDING_INTERRUPT__SHIFT                                               0x0
+#define CP_MEC_RS64_PENDING_INTERRUPT__PENDING_INTERRUPT_MASK                                                 0xFFFFFFFFL
+//CP_MEC_RS64_PRGRM_CNTR_START_HI
+#define CP_MEC_RS64_PRGRM_CNTR_START_HI__IP_START__SHIFT                                                      0x0
+#define CP_MEC_RS64_PRGRM_CNTR_START_HI__IP_START_MASK                                                        0x3FFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_16
+#define CP_MEC_RS64_INTERRUPT_DATA_16__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_16__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_17
+#define CP_MEC_RS64_INTERRUPT_DATA_17__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_17__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_18
+#define CP_MEC_RS64_INTERRUPT_DATA_18__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_18__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_19
+#define CP_MEC_RS64_INTERRUPT_DATA_19__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_19__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_20
+#define CP_MEC_RS64_INTERRUPT_DATA_20__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_20__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_21
+#define CP_MEC_RS64_INTERRUPT_DATA_21__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_21__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_22
+#define CP_MEC_RS64_INTERRUPT_DATA_22__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_22__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_23
+#define CP_MEC_RS64_INTERRUPT_DATA_23__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_23__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_24
+#define CP_MEC_RS64_INTERRUPT_DATA_24__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_24__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_25
+#define CP_MEC_RS64_INTERRUPT_DATA_25__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_25__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_26
+#define CP_MEC_RS64_INTERRUPT_DATA_26__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_26__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_27
+#define CP_MEC_RS64_INTERRUPT_DATA_27__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_27__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_28
+#define CP_MEC_RS64_INTERRUPT_DATA_28__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_28__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_29
+#define CP_MEC_RS64_INTERRUPT_DATA_29__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_29__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_30
+#define CP_MEC_RS64_INTERRUPT_DATA_30__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_30__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_RS64_INTERRUPT_DATA_31
+#define CP_MEC_RS64_INTERRUPT_DATA_31__DATA__SHIFT                                                            0x0
+#define CP_MEC_RS64_INTERRUPT_DATA_31__DATA_MASK                                                              0xFFFFFFFFL
+//CP_MEC_DC_APERTURE0_BASE
+#define CP_MEC_DC_APERTURE0_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE0_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE0_MASK
+#define CP_MEC_DC_APERTURE0_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE0_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE0_CNTL
+#define CP_MEC_DC_APERTURE0_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE0_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MEC_DC_APERTURE0_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MEC_DC_APERTURE0_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MEC_DC_APERTURE1_BASE
+#define CP_MEC_DC_APERTURE1_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE1_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE1_MASK
+#define CP_MEC_DC_APERTURE1_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE1_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE1_CNTL
+#define CP_MEC_DC_APERTURE1_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE1_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MEC_DC_APERTURE1_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MEC_DC_APERTURE1_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MEC_DC_APERTURE2_BASE
+#define CP_MEC_DC_APERTURE2_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE2_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE2_MASK
+#define CP_MEC_DC_APERTURE2_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE2_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE2_CNTL
+#define CP_MEC_DC_APERTURE2_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE2_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MEC_DC_APERTURE2_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MEC_DC_APERTURE2_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MEC_DC_APERTURE3_BASE
+#define CP_MEC_DC_APERTURE3_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE3_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE3_MASK
+#define CP_MEC_DC_APERTURE3_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE3_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE3_CNTL
+#define CP_MEC_DC_APERTURE3_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE3_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MEC_DC_APERTURE3_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MEC_DC_APERTURE3_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MEC_DC_APERTURE4_BASE
+#define CP_MEC_DC_APERTURE4_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE4_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE4_MASK
+#define CP_MEC_DC_APERTURE4_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE4_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE4_CNTL
+#define CP_MEC_DC_APERTURE4_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE4_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MEC_DC_APERTURE4_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MEC_DC_APERTURE4_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MEC_DC_APERTURE5_BASE
+#define CP_MEC_DC_APERTURE5_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE5_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE5_MASK
+#define CP_MEC_DC_APERTURE5_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE5_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE5_CNTL
+#define CP_MEC_DC_APERTURE5_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE5_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MEC_DC_APERTURE5_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MEC_DC_APERTURE5_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MEC_DC_APERTURE6_BASE
+#define CP_MEC_DC_APERTURE6_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE6_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE6_MASK
+#define CP_MEC_DC_APERTURE6_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE6_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE6_CNTL
+#define CP_MEC_DC_APERTURE6_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE6_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MEC_DC_APERTURE6_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MEC_DC_APERTURE6_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MEC_DC_APERTURE7_BASE
+#define CP_MEC_DC_APERTURE7_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE7_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE7_MASK
+#define CP_MEC_DC_APERTURE7_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE7_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE7_CNTL
+#define CP_MEC_DC_APERTURE7_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE7_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MEC_DC_APERTURE7_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MEC_DC_APERTURE7_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MEC_DC_APERTURE8_BASE
+#define CP_MEC_DC_APERTURE8_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE8_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE8_MASK
+#define CP_MEC_DC_APERTURE8_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE8_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE8_CNTL
+#define CP_MEC_DC_APERTURE8_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE8_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MEC_DC_APERTURE8_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MEC_DC_APERTURE8_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MEC_DC_APERTURE9_BASE
+#define CP_MEC_DC_APERTURE9_BASE__BASE__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE9_BASE__BASE_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE9_MASK
+#define CP_MEC_DC_APERTURE9_MASK__MASK__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE9_MASK__MASK_MASK                                                                   0xFFFFFFFFL
+//CP_MEC_DC_APERTURE9_CNTL
+#define CP_MEC_DC_APERTURE9_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_MEC_DC_APERTURE9_CNTL__BYPASS_MODE__SHIFT                                                          0x4
+#define CP_MEC_DC_APERTURE9_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_MEC_DC_APERTURE9_CNTL__BYPASS_MODE_MASK                                                            0x00000010L
+//CP_MEC_DC_APERTURE10_BASE
+#define CP_MEC_DC_APERTURE10_BASE__BASE__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE10_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE10_MASK
+#define CP_MEC_DC_APERTURE10_MASK__MASK__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE10_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE10_CNTL
+#define CP_MEC_DC_APERTURE10_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE10_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MEC_DC_APERTURE10_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MEC_DC_APERTURE10_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MEC_DC_APERTURE11_BASE
+#define CP_MEC_DC_APERTURE11_BASE__BASE__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE11_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE11_MASK
+#define CP_MEC_DC_APERTURE11_MASK__MASK__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE11_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE11_CNTL
+#define CP_MEC_DC_APERTURE11_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE11_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MEC_DC_APERTURE11_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MEC_DC_APERTURE11_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MEC_DC_APERTURE12_BASE
+#define CP_MEC_DC_APERTURE12_BASE__BASE__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE12_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE12_MASK
+#define CP_MEC_DC_APERTURE12_MASK__MASK__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE12_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE12_CNTL
+#define CP_MEC_DC_APERTURE12_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE12_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MEC_DC_APERTURE12_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MEC_DC_APERTURE12_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MEC_DC_APERTURE13_BASE
+#define CP_MEC_DC_APERTURE13_BASE__BASE__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE13_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE13_MASK
+#define CP_MEC_DC_APERTURE13_MASK__MASK__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE13_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE13_CNTL
+#define CP_MEC_DC_APERTURE13_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE13_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MEC_DC_APERTURE13_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MEC_DC_APERTURE13_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MEC_DC_APERTURE14_BASE
+#define CP_MEC_DC_APERTURE14_BASE__BASE__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE14_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE14_MASK
+#define CP_MEC_DC_APERTURE14_MASK__MASK__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE14_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE14_CNTL
+#define CP_MEC_DC_APERTURE14_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE14_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MEC_DC_APERTURE14_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MEC_DC_APERTURE14_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_MEC_DC_APERTURE15_BASE
+#define CP_MEC_DC_APERTURE15_BASE__BASE__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE15_BASE__BASE_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE15_MASK
+#define CP_MEC_DC_APERTURE15_MASK__MASK__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE15_MASK__MASK_MASK                                                                  0xFFFFFFFFL
+//CP_MEC_DC_APERTURE15_CNTL
+#define CP_MEC_DC_APERTURE15_CNTL__VMID__SHIFT                                                                0x0
+#define CP_MEC_DC_APERTURE15_CNTL__BYPASS_MODE__SHIFT                                                         0x4
+#define CP_MEC_DC_APERTURE15_CNTL__VMID_MASK                                                                  0x0000000FL
+#define CP_MEC_DC_APERTURE15_CNTL__BYPASS_MODE_MASK                                                           0x00000010L
+//CP_CPC_IC_OP_CNTL
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                            0x0
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE__SHIFT                                                   0x1
+#define CP_CPC_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                0x4
+#define CP_CPC_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                               0x5
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                              0x00000001L
+#define CP_CPC_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE_MASK                                                     0x00000002L
+#define CP_CPC_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                  0x00000010L
+#define CP_CPC_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                 0x00000020L
+//CP_GFX_CNTL
+#define CP_GFX_CNTL__ENGINE_SEL__SHIFT                                                                        0x0
+#define CP_GFX_CNTL__CONFIG__SHIFT                                                                            0x1
+#define CP_GFX_CNTL__ENGINE_SEL_MASK                                                                          0x00000001L
+#define CP_GFX_CNTL__CONFIG_MASK                                                                              0x00000006L
+//CP_GFX_RS64_INTERRUPT0
+#define CP_GFX_RS64_INTERRUPT0__ME_INT__SHIFT                                                                 0x0
+#define CP_GFX_RS64_INTERRUPT0__ME_INT_MASK                                                                   0xFFFFFFFFL
+//CP_GFX_RS64_INTR_EN0
+#define CP_GFX_RS64_INTR_EN0__ME_INT__SHIFT                                                                   0x0
+#define CP_GFX_RS64_INTR_EN0__ME_INT_MASK                                                                     0xFFFFFFFFL
+//CP_GFX_RS64_INTR_EN1
+#define CP_GFX_RS64_INTR_EN1__ME_INT__SHIFT                                                                   0x0
+#define CP_GFX_RS64_INTR_EN1__ME_INT_MASK                                                                     0xFFFFFFFFL
+//CP_GFX_RS64_DC_BASE_CNTL
+#define CP_GFX_RS64_DC_BASE_CNTL__VMID__SHIFT                                                                 0x0
+#define CP_GFX_RS64_DC_BASE_CNTL__CACHE_POLICY__SHIFT                                                         0x18
+#define CP_GFX_RS64_DC_BASE_CNTL__VMID_MASK                                                                   0x0000000FL
+#define CP_GFX_RS64_DC_BASE_CNTL__CACHE_POLICY_MASK                                                           0x03000000L
+//CP_GFX_RS64_DC_OP_CNTL
+#define CP_GFX_RS64_DC_OP_CNTL__INVALIDATE_DCACHE__SHIFT                                                      0x0
+#define CP_GFX_RS64_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE__SHIFT                                             0x1
+#define CP_GFX_RS64_DC_OP_CNTL__BYPASS_ALL__SHIFT                                                             0x2
+#define CP_GFX_RS64_DC_OP_CNTL__RESERVED__SHIFT                                                               0x3
+#define CP_GFX_RS64_DC_OP_CNTL__PRIME_DCACHE__SHIFT                                                           0x4
+#define CP_GFX_RS64_DC_OP_CNTL__DCACHE_PRIMED__SHIFT                                                          0x5
+#define CP_GFX_RS64_DC_OP_CNTL__INVALIDATE_DCACHE_MASK                                                        0x00000001L
+#define CP_GFX_RS64_DC_OP_CNTL__INVALIDATE_DCACHE_COMPLETE_MASK                                               0x00000002L
+#define CP_GFX_RS64_DC_OP_CNTL__BYPASS_ALL_MASK                                                               0x00000004L
+#define CP_GFX_RS64_DC_OP_CNTL__RESERVED_MASK                                                                 0x00000008L
+#define CP_GFX_RS64_DC_OP_CNTL__PRIME_DCACHE_MASK                                                             0x00000010L
+#define CP_GFX_RS64_DC_OP_CNTL__DCACHE_PRIMED_MASK                                                            0x00000020L
+//CP_GFX_RS64_LOCAL_BASE0_LO
+#define CP_GFX_RS64_LOCAL_BASE0_LO__BASE0_LO__SHIFT                                                           0x10
+#define CP_GFX_RS64_LOCAL_BASE0_LO__BASE0_LO_MASK                                                             0xFFFF0000L
+//CP_GFX_RS64_LOCAL_BASE0_HI
+#define CP_GFX_RS64_LOCAL_BASE0_HI__BASE0_HI__SHIFT                                                           0x0
+#define CP_GFX_RS64_LOCAL_BASE0_HI__BASE0_HI_MASK                                                             0x0000FFFFL
+//CP_GFX_RS64_LOCAL_MASK0_LO
+#define CP_GFX_RS64_LOCAL_MASK0_LO__MASK0_LO__SHIFT                                                           0x10
+#define CP_GFX_RS64_LOCAL_MASK0_LO__MASK0_LO_MASK                                                             0xFFFF0000L
+//CP_GFX_RS64_LOCAL_MASK0_HI
+#define CP_GFX_RS64_LOCAL_MASK0_HI__MASK0_HI__SHIFT                                                           0x0
+#define CP_GFX_RS64_LOCAL_MASK0_HI__MASK0_HI_MASK                                                             0x0000FFFFL
+//CP_GFX_RS64_LOCAL_APERTURE
+#define CP_GFX_RS64_LOCAL_APERTURE__APERTURE__SHIFT                                                           0x0
+#define CP_GFX_RS64_LOCAL_APERTURE__APERTURE_MASK                                                             0x00000007L
+//CP_GFX_RS64_LOCAL_INSTR_BASE_LO
+#define CP_GFX_RS64_LOCAL_INSTR_BASE_LO__BASE_LO__SHIFT                                                       0x10
+#define CP_GFX_RS64_LOCAL_INSTR_BASE_LO__BASE_LO_MASK                                                         0xFFFF0000L
+//CP_GFX_RS64_LOCAL_INSTR_BASE_HI
+#define CP_GFX_RS64_LOCAL_INSTR_BASE_HI__BASE_HI__SHIFT                                                       0x0
+#define CP_GFX_RS64_LOCAL_INSTR_BASE_HI__BASE_HI_MASK                                                         0x0000FFFFL
+//CP_GFX_RS64_LOCAL_INSTR_MASK_LO
+#define CP_GFX_RS64_LOCAL_INSTR_MASK_LO__MASK_LO__SHIFT                                                       0x10
+#define CP_GFX_RS64_LOCAL_INSTR_MASK_LO__MASK_LO_MASK                                                         0xFFFF0000L
+//CP_GFX_RS64_LOCAL_INSTR_MASK_HI
+#define CP_GFX_RS64_LOCAL_INSTR_MASK_HI__MASK_HI__SHIFT                                                       0x0
+#define CP_GFX_RS64_LOCAL_INSTR_MASK_HI__MASK_HI_MASK                                                         0x0000FFFFL
+//CP_GFX_RS64_LOCAL_INSTR_APERTURE
+#define CP_GFX_RS64_LOCAL_INSTR_APERTURE__APERTURE__SHIFT                                                     0x0
+#define CP_GFX_RS64_LOCAL_INSTR_APERTURE__APERTURE_MASK                                                       0x00000007L
+//CP_GFX_RS64_LOCAL_SCRATCH_APERTURE
+#define CP_GFX_RS64_LOCAL_SCRATCH_APERTURE__APERTURE__SHIFT                                                   0x0
+#define CP_GFX_RS64_LOCAL_SCRATCH_APERTURE__APERTURE_MASK                                                     0x00000007L
+//CP_GFX_RS64_LOCAL_SCRATCH_BASE_LO
+#define CP_GFX_RS64_LOCAL_SCRATCH_BASE_LO__BASE_LO__SHIFT                                                     0x10
+#define CP_GFX_RS64_LOCAL_SCRATCH_BASE_LO__BASE_LO_MASK                                                       0xFFFF0000L
+//CP_GFX_RS64_LOCAL_SCRATCH_BASE_HI
+#define CP_GFX_RS64_LOCAL_SCRATCH_BASE_HI__BASE_HI__SHIFT                                                     0x0
+#define CP_GFX_RS64_LOCAL_SCRATCH_BASE_HI__BASE_HI_MASK                                                       0x0000FFFFL
+//CP_GFX_RS64_PERFCOUNT_CNTL0
+#define CP_GFX_RS64_PERFCOUNT_CNTL0__EVENT_SEL__SHIFT                                                         0x0
+#define CP_GFX_RS64_PERFCOUNT_CNTL0__EVENT_SEL_MASK                                                           0x0000001FL
+//CP_GFX_RS64_PERFCOUNT_CNTL1
+#define CP_GFX_RS64_PERFCOUNT_CNTL1__EVENT_SEL__SHIFT                                                         0x0
+#define CP_GFX_RS64_PERFCOUNT_CNTL1__EVENT_SEL_MASK                                                           0x0000001FL
+//CP_GFX_RS64_MIP_LO0
+#define CP_GFX_RS64_MIP_LO0__MIP_LO__SHIFT                                                                    0x0
+#define CP_GFX_RS64_MIP_LO0__MIP_LO_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_RS64_MIP_LO1
+#define CP_GFX_RS64_MIP_LO1__MIP_LO__SHIFT                                                                    0x0
+#define CP_GFX_RS64_MIP_LO1__MIP_LO_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_RS64_MIP_HI0
+#define CP_GFX_RS64_MIP_HI0__MIP_HI__SHIFT                                                                    0x0
+#define CP_GFX_RS64_MIP_HI0__MIP_HI_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_RS64_MIP_HI1
+#define CP_GFX_RS64_MIP_HI1__MIP_HI__SHIFT                                                                    0x0
+#define CP_GFX_RS64_MIP_HI1__MIP_HI_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_RS64_MTIMECMP_LO0
+#define CP_GFX_RS64_MTIMECMP_LO0__TIME_LO__SHIFT                                                              0x0
+#define CP_GFX_RS64_MTIMECMP_LO0__TIME_LO_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_MTIMECMP_LO1
+#define CP_GFX_RS64_MTIMECMP_LO1__TIME_LO__SHIFT                                                              0x0
+#define CP_GFX_RS64_MTIMECMP_LO1__TIME_LO_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_MTIMECMP_HI0
+#define CP_GFX_RS64_MTIMECMP_HI0__TIME_HI__SHIFT                                                              0x0
+#define CP_GFX_RS64_MTIMECMP_HI0__TIME_HI_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_MTIMECMP_HI1
+#define CP_GFX_RS64_MTIMECMP_HI1__TIME_HI__SHIFT                                                              0x0
+#define CP_GFX_RS64_MTIMECMP_HI1__TIME_HI_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_GP0_LO0
+#define CP_GFX_RS64_GP0_LO0__PG_VIRT_HALTED__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP0_LO0__DATA__SHIFT                                                                      0x1
+#define CP_GFX_RS64_GP0_LO0__PG_VIRT_HALTED_MASK                                                              0x00000001L
+#define CP_GFX_RS64_GP0_LO0__DATA_MASK                                                                        0xFFFFFFFEL
+//CP_GFX_RS64_GP0_LO1
+#define CP_GFX_RS64_GP0_LO1__PG_VIRT_HALTED__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP0_LO1__DATA__SHIFT                                                                      0x1
+#define CP_GFX_RS64_GP0_LO1__PG_VIRT_HALTED_MASK                                                              0x00000001L
+#define CP_GFX_RS64_GP0_LO1__DATA_MASK                                                                        0xFFFFFFFEL
+//CP_GFX_RS64_GP0_HI0
+#define CP_GFX_RS64_GP0_HI0__M_RET_ADDR__SHIFT                                                                0x0
+#define CP_GFX_RS64_GP0_HI0__M_RET_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_GFX_RS64_GP0_HI1
+#define CP_GFX_RS64_GP0_HI1__M_RET_ADDR__SHIFT                                                                0x0
+#define CP_GFX_RS64_GP0_HI1__M_RET_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_GFX_RS64_GP1_LO0
+#define CP_GFX_RS64_GP1_LO0__RD_WR_SELECT_LO__SHIFT                                                           0x0
+#define CP_GFX_RS64_GP1_LO0__RD_WR_SELECT_LO_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_GP1_LO1
+#define CP_GFX_RS64_GP1_LO1__RD_WR_SELECT_LO__SHIFT                                                           0x0
+#define CP_GFX_RS64_GP1_LO1__RD_WR_SELECT_LO_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_GP1_HI0
+#define CP_GFX_RS64_GP1_HI0__RD_WR_SELECT_HI__SHIFT                                                           0x0
+#define CP_GFX_RS64_GP1_HI0__RD_WR_SELECT_HI_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_GP1_HI1
+#define CP_GFX_RS64_GP1_HI1__RD_WR_SELECT_HI__SHIFT                                                           0x0
+#define CP_GFX_RS64_GP1_HI1__RD_WR_SELECT_HI_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_GP2_LO0
+#define CP_GFX_RS64_GP2_LO0__STACK_PNTR_LO__SHIFT                                                             0x0
+#define CP_GFX_RS64_GP2_LO0__STACK_PNTR_LO_MASK                                                               0xFFFFFFFFL
+//CP_GFX_RS64_GP2_LO1
+#define CP_GFX_RS64_GP2_LO1__STACK_PNTR_LO__SHIFT                                                             0x0
+#define CP_GFX_RS64_GP2_LO1__STACK_PNTR_LO_MASK                                                               0xFFFFFFFFL
+//CP_GFX_RS64_GP2_HI0
+#define CP_GFX_RS64_GP2_HI0__STACK_PNTR_HI__SHIFT                                                             0x0
+#define CP_GFX_RS64_GP2_HI0__STACK_PNTR_HI_MASK                                                               0xFFFFFFFFL
+//CP_GFX_RS64_GP2_HI1
+#define CP_GFX_RS64_GP2_HI1__STACK_PNTR_HI__SHIFT                                                             0x0
+#define CP_GFX_RS64_GP2_HI1__STACK_PNTR_HI_MASK                                                               0xFFFFFFFFL
+//CP_GFX_RS64_GP3_LO0
+#define CP_GFX_RS64_GP3_LO0__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP3_LO0__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP3_LO1
+#define CP_GFX_RS64_GP3_LO1__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP3_LO1__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP3_HI0
+#define CP_GFX_RS64_GP3_HI0__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP3_HI0__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP3_HI1
+#define CP_GFX_RS64_GP3_HI1__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP3_HI1__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP4_LO0
+#define CP_GFX_RS64_GP4_LO0__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP4_LO0__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP4_LO1
+#define CP_GFX_RS64_GP4_LO1__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP4_LO1__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP4_HI0
+#define CP_GFX_RS64_GP4_HI0__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP4_HI0__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP4_HI1
+#define CP_GFX_RS64_GP4_HI1__DATA__SHIFT                                                                      0x0
+#define CP_GFX_RS64_GP4_HI1__DATA_MASK                                                                        0xFFFFFFFFL
+//CP_GFX_RS64_GP5_LO0
+#define CP_GFX_RS64_GP5_LO0__PG_VIRT_HALTED__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP5_LO0__DATA__SHIFT                                                                      0x1
+#define CP_GFX_RS64_GP5_LO0__PG_VIRT_HALTED_MASK                                                              0x00000001L
+#define CP_GFX_RS64_GP5_LO0__DATA_MASK                                                                        0xFFFFFFFEL
+//CP_GFX_RS64_GP5_LO1
+#define CP_GFX_RS64_GP5_LO1__PG_VIRT_HALTED__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP5_LO1__DATA__SHIFT                                                                      0x1
+#define CP_GFX_RS64_GP5_LO1__PG_VIRT_HALTED_MASK                                                              0x00000001L
+#define CP_GFX_RS64_GP5_LO1__DATA_MASK                                                                        0xFFFFFFFEL
+//CP_GFX_RS64_GP5_HI0
+#define CP_GFX_RS64_GP5_HI0__M_RET_ADDR__SHIFT                                                                0x0
+#define CP_GFX_RS64_GP5_HI0__M_RET_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_GFX_RS64_GP5_HI1
+#define CP_GFX_RS64_GP5_HI1__M_RET_ADDR__SHIFT                                                                0x0
+#define CP_GFX_RS64_GP5_HI1__M_RET_ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_GFX_RS64_GP6_LO
+#define CP_GFX_RS64_GP6_LO__RD_WR_SELECT_LO__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP6_LO__RD_WR_SELECT_LO_MASK                                                              0xFFFFFFFFL
+//CP_GFX_RS64_GP6_HI
+#define CP_GFX_RS64_GP6_HI__RD_WR_SELECT_HI__SHIFT                                                            0x0
+#define CP_GFX_RS64_GP6_HI__RD_WR_SELECT_HI_MASK                                                              0xFFFFFFFFL
+//CP_GFX_RS64_GP7_LO
+#define CP_GFX_RS64_GP7_LO__STACK_PNTR_LO__SHIFT                                                              0x0
+#define CP_GFX_RS64_GP7_LO__STACK_PNTR_LO_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_GP7_HI
+#define CP_GFX_RS64_GP7_HI__STACK_PNTR_HI__SHIFT                                                              0x0
+#define CP_GFX_RS64_GP7_HI__STACK_PNTR_HI_MASK                                                                0xFFFFFFFFL
+//CP_GFX_RS64_GP8_LO
+#define CP_GFX_RS64_GP8_LO__DATA__SHIFT                                                                       0x0
+#define CP_GFX_RS64_GP8_LO__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_RS64_GP8_HI
+#define CP_GFX_RS64_GP8_HI__DATA__SHIFT                                                                       0x0
+#define CP_GFX_RS64_GP8_HI__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_RS64_GP9_LO
+#define CP_GFX_RS64_GP9_LO__DATA__SHIFT                                                                       0x0
+#define CP_GFX_RS64_GP9_LO__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_RS64_GP9_HI
+#define CP_GFX_RS64_GP9_HI__DATA__SHIFT                                                                       0x0
+#define CP_GFX_RS64_GP9_HI__DATA_MASK                                                                         0xFFFFFFFFL
+//CP_GFX_RS64_INSTR_PNTR0
+#define CP_GFX_RS64_INSTR_PNTR0__INSTR_PNTR__SHIFT                                                            0x0
+#define CP_GFX_RS64_INSTR_PNTR0__INSTR_PNTR_MASK                                                              0x000FFFFFL
+//CP_GFX_RS64_INSTR_PNTR1
+#define CP_GFX_RS64_INSTR_PNTR1__INSTR_PNTR__SHIFT                                                            0x0
+#define CP_GFX_RS64_INSTR_PNTR1__INSTR_PNTR_MASK                                                              0x000FFFFFL
+//CP_GFX_RS64_PENDING_INTERRUPT0
+#define CP_GFX_RS64_PENDING_INTERRUPT0__PENDING_INTERRUPT__SHIFT                                              0x0
+#define CP_GFX_RS64_PENDING_INTERRUPT0__PENDING_INTERRUPT_MASK                                                0xFFFFFFFFL
+//CP_GFX_RS64_PENDING_INTERRUPT1
+#define CP_GFX_RS64_PENDING_INTERRUPT1__PENDING_INTERRUPT__SHIFT                                              0x0
+#define CP_GFX_RS64_PENDING_INTERRUPT1__PENDING_INTERRUPT_MASK                                                0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE0_BASE0
+#define CP_GFX_RS64_DC_APERTURE0_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE0_MASK0
+#define CP_GFX_RS64_DC_APERTURE0_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE0_CNTL0
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE0_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE1_BASE0
+#define CP_GFX_RS64_DC_APERTURE1_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE1_MASK0
+#define CP_GFX_RS64_DC_APERTURE1_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE1_CNTL0
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE1_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE2_BASE0
+#define CP_GFX_RS64_DC_APERTURE2_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE2_MASK0
+#define CP_GFX_RS64_DC_APERTURE2_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE2_CNTL0
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE2_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE3_BASE0
+#define CP_GFX_RS64_DC_APERTURE3_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE3_MASK0
+#define CP_GFX_RS64_DC_APERTURE3_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE3_CNTL0
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE3_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE4_BASE0
+#define CP_GFX_RS64_DC_APERTURE4_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE4_MASK0
+#define CP_GFX_RS64_DC_APERTURE4_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE4_CNTL0
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE4_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE5_BASE0
+#define CP_GFX_RS64_DC_APERTURE5_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE5_MASK0
+#define CP_GFX_RS64_DC_APERTURE5_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE5_CNTL0
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE5_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE6_BASE0
+#define CP_GFX_RS64_DC_APERTURE6_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE6_MASK0
+#define CP_GFX_RS64_DC_APERTURE6_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE6_CNTL0
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE6_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE7_BASE0
+#define CP_GFX_RS64_DC_APERTURE7_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE7_MASK0
+#define CP_GFX_RS64_DC_APERTURE7_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE7_CNTL0
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE7_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE8_BASE0
+#define CP_GFX_RS64_DC_APERTURE8_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE8_MASK0
+#define CP_GFX_RS64_DC_APERTURE8_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE8_CNTL0
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE8_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE9_BASE0
+#define CP_GFX_RS64_DC_APERTURE9_BASE0__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_BASE0__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE9_MASK0
+#define CP_GFX_RS64_DC_APERTURE9_MASK0__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_MASK0__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE9_CNTL0
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE9_CNTL0__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE10_BASE0
+#define CP_GFX_RS64_DC_APERTURE10_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE10_MASK0
+#define CP_GFX_RS64_DC_APERTURE10_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE10_CNTL0
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE10_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE11_BASE0
+#define CP_GFX_RS64_DC_APERTURE11_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE11_MASK0
+#define CP_GFX_RS64_DC_APERTURE11_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE11_CNTL0
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE11_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE12_BASE0
+#define CP_GFX_RS64_DC_APERTURE12_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE12_MASK0
+#define CP_GFX_RS64_DC_APERTURE12_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE12_CNTL0
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE12_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE13_BASE0
+#define CP_GFX_RS64_DC_APERTURE13_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE13_MASK0
+#define CP_GFX_RS64_DC_APERTURE13_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE13_CNTL0
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE13_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE14_BASE0
+#define CP_GFX_RS64_DC_APERTURE14_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE14_MASK0
+#define CP_GFX_RS64_DC_APERTURE14_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE14_CNTL0
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE14_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE15_BASE0
+#define CP_GFX_RS64_DC_APERTURE15_BASE0__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_BASE0__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE15_MASK0
+#define CP_GFX_RS64_DC_APERTURE15_MASK0__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_MASK0__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE15_CNTL0
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE15_CNTL0__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE0_BASE1
+#define CP_GFX_RS64_DC_APERTURE0_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE0_MASK1
+#define CP_GFX_RS64_DC_APERTURE0_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE0_CNTL1
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE0_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE1_BASE1
+#define CP_GFX_RS64_DC_APERTURE1_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE1_MASK1
+#define CP_GFX_RS64_DC_APERTURE1_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE1_CNTL1
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE1_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE2_BASE1
+#define CP_GFX_RS64_DC_APERTURE2_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE2_MASK1
+#define CP_GFX_RS64_DC_APERTURE2_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE2_CNTL1
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE2_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE3_BASE1
+#define CP_GFX_RS64_DC_APERTURE3_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE3_MASK1
+#define CP_GFX_RS64_DC_APERTURE3_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE3_CNTL1
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE3_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE4_BASE1
+#define CP_GFX_RS64_DC_APERTURE4_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE4_MASK1
+#define CP_GFX_RS64_DC_APERTURE4_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE4_CNTL1
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE4_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE5_BASE1
+#define CP_GFX_RS64_DC_APERTURE5_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE5_MASK1
+#define CP_GFX_RS64_DC_APERTURE5_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE5_CNTL1
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE5_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE6_BASE1
+#define CP_GFX_RS64_DC_APERTURE6_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE6_MASK1
+#define CP_GFX_RS64_DC_APERTURE6_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE6_CNTL1
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE6_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE7_BASE1
+#define CP_GFX_RS64_DC_APERTURE7_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE7_MASK1
+#define CP_GFX_RS64_DC_APERTURE7_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE7_CNTL1
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE7_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE8_BASE1
+#define CP_GFX_RS64_DC_APERTURE8_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE8_MASK1
+#define CP_GFX_RS64_DC_APERTURE8_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE8_CNTL1
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE8_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE9_BASE1
+#define CP_GFX_RS64_DC_APERTURE9_BASE1__BASE__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_BASE1__BASE_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE9_MASK1
+#define CP_GFX_RS64_DC_APERTURE9_MASK1__MASK__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_MASK1__MASK_MASK                                                             0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE9_CNTL1
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__VMID__SHIFT                                                           0x0
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__BYPASS_MODE__SHIFT                                                    0x4
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__VMID_MASK                                                             0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE9_CNTL1__BYPASS_MODE_MASK                                                      0x00000010L
+//CP_GFX_RS64_DC_APERTURE10_BASE1
+#define CP_GFX_RS64_DC_APERTURE10_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE10_MASK1
+#define CP_GFX_RS64_DC_APERTURE10_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE10_CNTL1
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE10_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE11_BASE1
+#define CP_GFX_RS64_DC_APERTURE11_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE11_MASK1
+#define CP_GFX_RS64_DC_APERTURE11_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE11_CNTL1
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE11_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE12_BASE1
+#define CP_GFX_RS64_DC_APERTURE12_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE12_MASK1
+#define CP_GFX_RS64_DC_APERTURE12_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE12_CNTL1
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE12_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE13_BASE1
+#define CP_GFX_RS64_DC_APERTURE13_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE13_MASK1
+#define CP_GFX_RS64_DC_APERTURE13_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE13_CNTL1
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE13_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE14_BASE1
+#define CP_GFX_RS64_DC_APERTURE14_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE14_MASK1
+#define CP_GFX_RS64_DC_APERTURE14_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE14_CNTL1
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE14_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_DC_APERTURE15_BASE1
+#define CP_GFX_RS64_DC_APERTURE15_BASE1__BASE__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_BASE1__BASE_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE15_MASK1
+#define CP_GFX_RS64_DC_APERTURE15_MASK1__MASK__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_MASK1__MASK_MASK                                                            0xFFFFFFFFL
+//CP_GFX_RS64_DC_APERTURE15_CNTL1
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__VMID__SHIFT                                                          0x0
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__BYPASS_MODE__SHIFT                                                   0x4
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__VMID_MASK                                                            0x0000000FL
+#define CP_GFX_RS64_DC_APERTURE15_CNTL1__BYPASS_MODE_MASK                                                     0x00000010L
+//CP_GFX_RS64_INTERRUPT1
+#define CP_GFX_RS64_INTERRUPT1__ME_INT__SHIFT                                                                 0x0
+#define CP_GFX_RS64_INTERRUPT1__ME_INT_MASK                                                                   0xFFFFFFFFL
+
+
+// addressBlock: gc_gl1dec
+//GL1_ARB_CTRL
+#define GL1_ARB_CTRL__NUM_MEM_PIPES__SHIFT                                                                    0x0
+#define GL1_ARB_CTRL__FGCG_DISABLE__SHIFT                                                                     0x2
+#define GL1_ARB_CTRL__PERF_CNTR_EN_OVERRIDE__SHIFT                                                            0x3
+#define GL1_ARB_CTRL__CHICKEN_BITS__SHIFT                                                                     0x4
+#define GL1_ARB_CTRL__NUM_MEM_PIPES_MASK                                                                      0x00000003L
+#define GL1_ARB_CTRL__FGCG_DISABLE_MASK                                                                       0x00000004L
+#define GL1_ARB_CTRL__PERF_CNTR_EN_OVERRIDE_MASK                                                              0x00000008L
+#define GL1_ARB_CTRL__CHICKEN_BITS_MASK                                                                       0x00000FF0L
+//GL1_DRAM_BURST_MASK
+#define GL1_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK__SHIFT                                                      0x0
+#define GL1_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK_MASK                                                        0x000000FFL
+//GL1_ARB_STATUS
+#define GL1_ARB_STATUS__REQ_ARB_BUSY__SHIFT                                                                   0x0
+#define GL1_ARB_STATUS__RET_ARB_BUSY__SHIFT                                                                   0x1
+#define GL1_ARB_STATUS__REQ_ARB_BUSY_MASK                                                                     0x00000001L
+#define GL1_ARB_STATUS__RET_ARB_BUSY_MASK                                                                     0x00000002L
+//GL1_DRAM_BURST_CTRL
+#define GL1_DRAM_BURST_CTRL__MAX_DRAM_BURST__SHIFT                                                            0x0
+#define GL1_DRAM_BURST_CTRL__BURST_DISABLE__SHIFT                                                             0x3
+#define GL1_DRAM_BURST_CTRL__GATHER_64B_BURST_DISABLE__SHIFT                                                  0x4
+#define GL1_DRAM_BURST_CTRL__GATHER_32B_BURST_DISABLE__SHIFT                                                  0x5
+#define GL1_DRAM_BURST_CTRL__WRITE_BURSTABLE_STALL_DISABLE__SHIFT                                             0x8
+#define GL1_DRAM_BURST_CTRL__MAX_DRAM_BURST_MASK                                                              0x00000007L
+#define GL1_DRAM_BURST_CTRL__BURST_DISABLE_MASK                                                               0x00000008L
+#define GL1_DRAM_BURST_CTRL__GATHER_64B_BURST_DISABLE_MASK                                                    0x00000010L
+#define GL1_DRAM_BURST_CTRL__GATHER_32B_BURST_DISABLE_MASK                                                    0x00000020L
+#define GL1_DRAM_BURST_CTRL__WRITE_BURSTABLE_STALL_DISABLE_MASK                                               0x00000100L
+//GL1I_GL1R_REP_FGCG_OVERRIDE
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1IR_REP_FGCG_OVERRIDE__SHIFT                                      0x0
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1IW_REP_FGCG_OVERRIDE__SHIFT                                      0x1
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1R_SRC_REP_FGCG_OVERRIDE__SHIFT                                   0x2
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1R_RET_REP_FGCG_OVERRIDE__SHIFT                                   0x3
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1IR_REP_FGCG_OVERRIDE_MASK                                        0x00000001L
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1IW_REP_FGCG_OVERRIDE_MASK                                        0x00000002L
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1R_SRC_REP_FGCG_OVERRIDE_MASK                                     0x00000004L
+#define GL1I_GL1R_REP_FGCG_OVERRIDE__GL1A_GL1R_RET_REP_FGCG_OVERRIDE_MASK                                     0x00000008L
+//GL1C_CTRL
+#define GL1C_CTRL__FORCE_MISS__SHIFT                                                                          0x0
+#define GL1C_CTRL__FORCE_HIT__SHIFT                                                                           0x1
+#define GL1C_CTRL__NOFILL_32B__SHIFT                                                                          0x2
+#define GL1C_CTRL__NOFILL_64B__SHIFT                                                                          0x3
+#define GL1C_CTRL__LATENCY_FIFO_SIZE__SHIFT                                                                   0x4
+#define GL1C_CTRL__ACK_QUEUE_DISABLE__SHIFT                                                                   0x8
+#define GL1C_CTRL__RMI_META_READ_MISS_QUEUE_DISABLE__SHIFT                                                    0x9
+#define GL1C_CTRL__HIT_QUEUE_DISABLE__SHIFT                                                                   0xa
+#define GL1C_CTRL__GL2_REQ_CREDITS__SHIFT                                                                     0xb
+#define GL1C_CTRL__GL2_DATA_CREDITS__SHIFT                                                                    0x12
+#define GL1C_CTRL__TO_L1_REPEATER_FGCG_DISABLE__SHIFT                                                         0x19
+#define GL1C_CTRL__TO_L2_REPEATER_FGCG_DISABLE__SHIFT                                                         0x1a
+#define GL1C_CTRL__GCR_RSP_FGCG_DISABLE__SHIFT                                                                0x1b
+#define GL1C_CTRL__DISABLE_HASH_TO_UPPER_16_SETS__SHIFT                                                       0x1c
+#define GL1C_CTRL__DISABLE_PERF_WR_DATA_ALLOC_COUNT__SHIFT                                                    0x1d
+#define GL1C_CTRL__DISABLE_PERF_SPLIT_EVICT_WRITE__SHIFT                                                      0x1e
+#define GL1C_CTRL__FORCE_MISS_MASK                                                                            0x00000001L
+#define GL1C_CTRL__FORCE_HIT_MASK                                                                             0x00000002L
+#define GL1C_CTRL__NOFILL_32B_MASK                                                                            0x00000004L
+#define GL1C_CTRL__NOFILL_64B_MASK                                                                            0x00000008L
+#define GL1C_CTRL__LATENCY_FIFO_SIZE_MASK                                                                     0x000000F0L
+#define GL1C_CTRL__ACK_QUEUE_DISABLE_MASK                                                                     0x00000100L
+#define GL1C_CTRL__RMI_META_READ_MISS_QUEUE_DISABLE_MASK                                                      0x00000200L
+#define GL1C_CTRL__HIT_QUEUE_DISABLE_MASK                                                                     0x00000400L
+#define GL1C_CTRL__GL2_REQ_CREDITS_MASK                                                                       0x0003F800L
+#define GL1C_CTRL__GL2_DATA_CREDITS_MASK                                                                      0x01FC0000L
+#define GL1C_CTRL__TO_L1_REPEATER_FGCG_DISABLE_MASK                                                           0x02000000L
+#define GL1C_CTRL__TO_L2_REPEATER_FGCG_DISABLE_MASK                                                           0x04000000L
+#define GL1C_CTRL__GCR_RSP_FGCG_DISABLE_MASK                                                                  0x08000000L
+#define GL1C_CTRL__DISABLE_HASH_TO_UPPER_16_SETS_MASK                                                         0x10000000L
+#define GL1C_CTRL__DISABLE_PERF_WR_DATA_ALLOC_COUNT_MASK                                                      0x20000000L
+#define GL1C_CTRL__DISABLE_PERF_SPLIT_EVICT_WRITE_MASK                                                        0x40000000L
+//GL1C_STATUS
+#define GL1C_STATUS__INPUT_BUFFER_VC0_FIFO_FULL__SHIFT                                                        0x0
+#define GL1C_STATUS__OUTPUT_FIFOS_BUSY__SHIFT                                                                 0x1
+#define GL1C_STATUS__SRC_DATA_FIFO_VC0_FULL__SHIFT                                                            0x2
+#define GL1C_STATUS__GL2_REQ_VC0_STALL__SHIFT                                                                 0x3
+#define GL1C_STATUS__GL2_DATA_VC0_STALL__SHIFT                                                                0x4
+#define GL1C_STATUS__GL2_REQ_VC1_STALL__SHIFT                                                                 0x5
+#define GL1C_STATUS__GL2_DATA_VC1_STALL__SHIFT                                                                0x6
+#define GL1C_STATUS__INPUT_BUFFER_VC0_BUSY__SHIFT                                                             0x7
+#define GL1C_STATUS__SRC_DATA_FIFO_VC0_BUSY__SHIFT                                                            0x8
+#define GL1C_STATUS__GL2_RH_BUSY__SHIFT                                                                       0x9
+#define GL1C_STATUS__NUM_REQ_PENDING_FROM_L2__SHIFT                                                           0xa
+#define GL1C_STATUS__LATENCY_FIFO_FULL_STALL__SHIFT                                                           0x14
+#define GL1C_STATUS__TAG_STALL__SHIFT                                                                         0x15
+#define GL1C_STATUS__TAG_BUSY__SHIFT                                                                          0x16
+#define GL1C_STATUS__TAG_ACK_STALL__SHIFT                                                                     0x17
+#define GL1C_STATUS__TAG_GCR_INV_STALL__SHIFT                                                                 0x18
+#define GL1C_STATUS__TAG_NO_AVAILABLE_LINE_TO_EVICT_STALL__SHIFT                                              0x19
+#define GL1C_STATUS__TAG_EVICT__SHIFT                                                                         0x1a
+#define GL1C_STATUS__TAG_REQUEST_STATE_OPERATION__SHIFT                                                       0x1b
+#define GL1C_STATUS__TRACKER_LAST_SET_MATCHES_CURRENT_SET__SHIFT                                              0x1f
+#define GL1C_STATUS__INPUT_BUFFER_VC0_FIFO_FULL_MASK                                                          0x00000001L
+#define GL1C_STATUS__OUTPUT_FIFOS_BUSY_MASK                                                                   0x00000002L
+#define GL1C_STATUS__SRC_DATA_FIFO_VC0_FULL_MASK                                                              0x00000004L
+#define GL1C_STATUS__GL2_REQ_VC0_STALL_MASK                                                                   0x00000008L
+#define GL1C_STATUS__GL2_DATA_VC0_STALL_MASK                                                                  0x00000010L
+#define GL1C_STATUS__GL2_REQ_VC1_STALL_MASK                                                                   0x00000020L
+#define GL1C_STATUS__GL2_DATA_VC1_STALL_MASK                                                                  0x00000040L
+#define GL1C_STATUS__INPUT_BUFFER_VC0_BUSY_MASK                                                               0x00000080L
+#define GL1C_STATUS__SRC_DATA_FIFO_VC0_BUSY_MASK                                                              0x00000100L
+#define GL1C_STATUS__GL2_RH_BUSY_MASK                                                                         0x00000200L
+#define GL1C_STATUS__NUM_REQ_PENDING_FROM_L2_MASK                                                             0x000FFC00L
+#define GL1C_STATUS__LATENCY_FIFO_FULL_STALL_MASK                                                             0x00100000L
+#define GL1C_STATUS__TAG_STALL_MASK                                                                           0x00200000L
+#define GL1C_STATUS__TAG_BUSY_MASK                                                                            0x00400000L
+#define GL1C_STATUS__TAG_ACK_STALL_MASK                                                                       0x00800000L
+#define GL1C_STATUS__TAG_GCR_INV_STALL_MASK                                                                   0x01000000L
+#define GL1C_STATUS__TAG_NO_AVAILABLE_LINE_TO_EVICT_STALL_MASK                                                0x02000000L
+#define GL1C_STATUS__TAG_EVICT_MASK                                                                           0x04000000L
+#define GL1C_STATUS__TAG_REQUEST_STATE_OPERATION_MASK                                                         0x78000000L
+#define GL1C_STATUS__TRACKER_LAST_SET_MATCHES_CURRENT_SET_MASK                                                0x80000000L
+//GL1C_UTCL0_CNTL2
+#define GL1C_UTCL0_CNTL2__SPARE__SHIFT                                                                        0x0
+#define GL1C_UTCL0_CNTL2__COMP_SYNC_DISABLE__SHIFT                                                            0x8
+#define GL1C_UTCL0_CNTL2__MTYPE_OVRD_DIS__SHIFT                                                               0x9
+#define GL1C_UTCL0_CNTL2__ANY_LINE_VALID__SHIFT                                                               0xa
+#define GL1C_UTCL0_CNTL2__FORCE_SNOOP__SHIFT                                                                  0xe
+#define GL1C_UTCL0_CNTL2__DISABLE_BURST__SHIFT                                                                0x11
+#define GL1C_UTCL0_CNTL2__FORCE_FRAG_2M_TO_64K__SHIFT                                                         0x1a
+#define GL1C_UTCL0_CNTL2__FGCG_DISABLE__SHIFT                                                                 0x1e
+#define GL1C_UTCL0_CNTL2__BIG_PAGE_DISABLE__SHIFT                                                             0x1f
+#define GL1C_UTCL0_CNTL2__SPARE_MASK                                                                          0x000000FFL
+#define GL1C_UTCL0_CNTL2__COMP_SYNC_DISABLE_MASK                                                              0x00000100L
+#define GL1C_UTCL0_CNTL2__MTYPE_OVRD_DIS_MASK                                                                 0x00000200L
+#define GL1C_UTCL0_CNTL2__ANY_LINE_VALID_MASK                                                                 0x00000400L
+#define GL1C_UTCL0_CNTL2__FORCE_SNOOP_MASK                                                                    0x00004000L
+#define GL1C_UTCL0_CNTL2__DISABLE_BURST_MASK                                                                  0x00020000L
+#define GL1C_UTCL0_CNTL2__FORCE_FRAG_2M_TO_64K_MASK                                                           0x04000000L
+#define GL1C_UTCL0_CNTL2__FGCG_DISABLE_MASK                                                                   0x40000000L
+#define GL1C_UTCL0_CNTL2__BIG_PAGE_DISABLE_MASK                                                               0x80000000L
+//GL1C_UTCL0_STATUS
+#define GL1C_UTCL0_STATUS__FAULT_DETECTED__SHIFT                                                              0x0
+#define GL1C_UTCL0_STATUS__RETRY_DETECTED__SHIFT                                                              0x1
+#define GL1C_UTCL0_STATUS__PRT_DETECTED__SHIFT                                                                0x2
+#define GL1C_UTCL0_STATUS__FAULT_DETECTED_MASK                                                                0x00000001L
+#define GL1C_UTCL0_STATUS__RETRY_DETECTED_MASK                                                                0x00000002L
+#define GL1C_UTCL0_STATUS__PRT_DETECTED_MASK                                                                  0x00000004L
+//GL1C_UTCL0_RETRY
+#define GL1C_UTCL0_RETRY__INCR__SHIFT                                                                         0x0
+#define GL1C_UTCL0_RETRY__COUNT__SHIFT                                                                        0x8
+#define GL1C_UTCL0_RETRY__INCR_MASK                                                                           0x000000FFL
+#define GL1C_UTCL0_RETRY__COUNT_MASK                                                                          0x00000F00L
+//GL1C_CTRL2
+#define GL1C_CTRL2__UTCL0_INFLIGHT_MAX__SHIFT                                                                 0x0
+#define GL1C_CTRL2__UTCL0_SD_SIDEBAND_IF_DISABLE__SHIFT                                                       0x8
+#define GL1C_CTRL2__REDUCE_REQ_PROTECTION_LINE_LEVEL__SHIFT                                                   0x9
+#define GL1C_CTRL2__UTCL0_INFLIGHT_MAX_MASK                                                                   0x000000FFL
+#define GL1C_CTRL2__UTCL0_SD_SIDEBAND_IF_DISABLE_MASK                                                         0x00000100L
+#define GL1C_CTRL2__REDUCE_REQ_PROTECTION_LINE_LEVEL_MASK                                                     0x00003E00L
+
+
+// addressBlock: gc_chdec
+//CH_ARB_CTRL
+#define CH_ARB_CTRL__NUM_MEM_PIPES__SHIFT                                                                     0x0
+#define CH_ARB_CTRL__UC_IO_WR_PATH__SHIFT                                                                     0x2
+#define CH_ARB_CTRL__FGCG_DISABLE__SHIFT                                                                      0x3
+#define CH_ARB_CTRL__PERF_CNTR_EN_OVERRIDE__SHIFT                                                             0x4
+#define CH_ARB_CTRL__CHICKEN_BITS__SHIFT                                                                      0x5
+#define CH_ARB_CTRL__NUM_MEM_PIPES_MASK                                                                       0x00000003L
+#define CH_ARB_CTRL__UC_IO_WR_PATH_MASK                                                                       0x00000004L
+#define CH_ARB_CTRL__FGCG_DISABLE_MASK                                                                        0x00000008L
+#define CH_ARB_CTRL__PERF_CNTR_EN_OVERRIDE_MASK                                                               0x00000010L
+#define CH_ARB_CTRL__CHICKEN_BITS_MASK                                                                        0x00001FE0L
+//CH_DRAM_BURST_MASK
+#define CH_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK__SHIFT                                                       0x0
+#define CH_DRAM_BURST_MASK__DRAM_BURST_ADDR_MASK_MASK                                                         0x000000FFL
+//CH_ARB_STATUS
+#define CH_ARB_STATUS__REQ_ARB_BUSY__SHIFT                                                                    0x0
+#define CH_ARB_STATUS__RET_ARB_BUSY__SHIFT                                                                    0x1
+#define CH_ARB_STATUS__REQ_ARB_BUSY_MASK                                                                      0x00000001L
+#define CH_ARB_STATUS__RET_ARB_BUSY_MASK                                                                      0x00000002L
+//CH_DRAM_BURST_CTRL
+#define CH_DRAM_BURST_CTRL__MAX_DRAM_BURST__SHIFT                                                             0x0
+#define CH_DRAM_BURST_CTRL__BURST_DISABLE__SHIFT                                                              0x3
+#define CH_DRAM_BURST_CTRL__GATHER_64B_MEMORY_BURST_DISABLE__SHIFT                                            0x4
+#define CH_DRAM_BURST_CTRL__GATHER_64B_IO_BURST_DISABLE__SHIFT                                                0x5
+#define CH_DRAM_BURST_CTRL__GATHER_32B_MEMORY_BURST_DISABLE__SHIFT                                            0x6
+#define CH_DRAM_BURST_CTRL__GATHER_32B_IO_BURST_DISABLE__SHIFT                                                0x7
+#define CH_DRAM_BURST_CTRL__WRITE_BURSTABLE_STALL_DISABLE__SHIFT                                              0x8
+#define CH_DRAM_BURST_CTRL__MAX_DRAM_BURST_MASK                                                               0x00000007L
+#define CH_DRAM_BURST_CTRL__BURST_DISABLE_MASK                                                                0x00000008L
+#define CH_DRAM_BURST_CTRL__GATHER_64B_MEMORY_BURST_DISABLE_MASK                                              0x00000010L
+#define CH_DRAM_BURST_CTRL__GATHER_64B_IO_BURST_DISABLE_MASK                                                  0x00000020L
+#define CH_DRAM_BURST_CTRL__GATHER_32B_MEMORY_BURST_DISABLE_MASK                                              0x00000040L
+#define CH_DRAM_BURST_CTRL__GATHER_32B_IO_BURST_DISABLE_MASK                                                  0x00000080L
+#define CH_DRAM_BURST_CTRL__WRITE_BURSTABLE_STALL_DISABLE_MASK                                                0x00000100L
+//CHA_CHC_CREDITS
+#define CHA_CHC_CREDITS__CHC_REQ_CREDITS__SHIFT                                                               0x0
+#define CHA_CHC_CREDITS__CHCG_REQ_CREDITS__SHIFT                                                              0x8
+#define CHA_CHC_CREDITS__CHC_REQ_CREDITS_MASK                                                                 0x000000FFL
+#define CHA_CHC_CREDITS__CHCG_REQ_CREDITS_MASK                                                                0x0000FF00L
+//CHA_CLIENT_FREE_DELAY
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_0_FREE_DELAY__SHIFT                                                0x0
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_1_FREE_DELAY__SHIFT                                                0x3
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_2_FREE_DELAY__SHIFT                                                0x6
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_3_FREE_DELAY__SHIFT                                                0x9
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_4_FREE_DELAY__SHIFT                                                0xc
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_0_FREE_DELAY_MASK                                                  0x00000007L
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_1_FREE_DELAY_MASK                                                  0x00000038L
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_2_FREE_DELAY_MASK                                                  0x000001C0L
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_3_FREE_DELAY_MASK                                                  0x00000E00L
+#define CHA_CLIENT_FREE_DELAY__CLIENT_TYPE_4_FREE_DELAY_MASK                                                  0x00007000L
+//CHI_CHR_REP_FGCG_OVERRIDE
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHIW_REP_FGCG_OVERRIDE__SHIFT                                          0x0
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHIR_REP_FGCG_OVERRIDE__SHIFT                                          0x1
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHR_SRC_REP_FGCG_OVERRIDE__SHIFT                                       0x2
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHR_RET_REP_FGCG_OVERRIDE__SHIFT                                       0x3
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHIW_REP_FGCG_OVERRIDE_MASK                                            0x00000001L
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHIR_REP_FGCG_OVERRIDE_MASK                                            0x00000002L
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHR_SRC_REP_FGCG_OVERRIDE_MASK                                         0x00000004L
+#define CHI_CHR_REP_FGCG_OVERRIDE__CHA_CHR_RET_REP_FGCG_OVERRIDE_MASK                                         0x00000008L
+//CH_VC5_ENABLE
+#define CH_VC5_ENABLE__UTCL2_VC5_ENABLE__SHIFT                                                                0x1
+#define CH_VC5_ENABLE__UTCL2_VC5_ENABLE_MASK                                                                  0x00000002L
+//CHC_CTRL
+#define CHC_CTRL__BUFFER_DEPTH_MAX__SHIFT                                                                     0x0
+#define CHC_CTRL__GL2_REQ_CREDITS__SHIFT                                                                      0x4
+#define CHC_CTRL__GL2_DATA_CREDITS__SHIFT                                                                     0xb
+#define CHC_CTRL__TO_L1_REPEATER_FGCG_DISABLE__SHIFT                                                          0x12
+#define CHC_CTRL__TO_L2_REPEATER_FGCG_DISABLE__SHIFT                                                          0x13
+#define CHC_CTRL__DISABLE_PERF_WR_DATA_ALLOC_COUNT__SHIFT                                                     0x1d
+#define CHC_CTRL__BUFFER_DEPTH_MAX_MASK                                                                       0x0000000FL
+#define CHC_CTRL__GL2_REQ_CREDITS_MASK                                                                        0x000007F0L
+#define CHC_CTRL__GL2_DATA_CREDITS_MASK                                                                       0x0003F800L
+#define CHC_CTRL__TO_L1_REPEATER_FGCG_DISABLE_MASK                                                            0x00040000L
+#define CHC_CTRL__TO_L2_REPEATER_FGCG_DISABLE_MASK                                                            0x00080000L
+#define CHC_CTRL__DISABLE_PERF_WR_DATA_ALLOC_COUNT_MASK                                                       0x20000000L
+//CHC_STATUS
+#define CHC_STATUS__INPUT_BUFFER_VC0_FIFO_FULL__SHIFT                                                         0x0
+#define CHC_STATUS__OUTPUT_FIFOS_BUSY__SHIFT                                                                  0x1
+#define CHC_STATUS__SRC_DATA_FIFO_VC0_FULL__SHIFT                                                             0x2
+#define CHC_STATUS__GL2_REQ_VC0_STALL__SHIFT                                                                  0x3
+#define CHC_STATUS__GL2_DATA_VC0_STALL__SHIFT                                                                 0x4
+#define CHC_STATUS__GL2_REQ_VC1_STALL__SHIFT                                                                  0x5
+#define CHC_STATUS__GL2_DATA_VC1_STALL__SHIFT                                                                 0x6
+#define CHC_STATUS__INPUT_BUFFER_VC0_BUSY__SHIFT                                                              0x7
+#define CHC_STATUS__SRC_DATA_FIFO_VC0_BUSY__SHIFT                                                             0x8
+#define CHC_STATUS__GL2_RH_BUSY__SHIFT                                                                        0x9
+#define CHC_STATUS__NUM_REQ_PENDING_FROM_L2__SHIFT                                                            0xa
+#define CHC_STATUS__VIRTUAL_FIFO_FULL_STALL__SHIFT                                                            0x14
+#define CHC_STATUS__REQUEST_TRACKER_BUFFER_STALL__SHIFT                                                       0x15
+#define CHC_STATUS__REQUEST_TRACKER_BUSY__SHIFT                                                               0x16
+#define CHC_STATUS__BUFFER_FULL__SHIFT                                                                        0x17
+#define CHC_STATUS__INPUT_BUFFER_VC0_FIFO_FULL_MASK                                                           0x00000001L
+#define CHC_STATUS__OUTPUT_FIFOS_BUSY_MASK                                                                    0x00000002L
+#define CHC_STATUS__SRC_DATA_FIFO_VC0_FULL_MASK                                                               0x00000004L
+#define CHC_STATUS__GL2_REQ_VC0_STALL_MASK                                                                    0x00000008L
+#define CHC_STATUS__GL2_DATA_VC0_STALL_MASK                                                                   0x00000010L
+#define CHC_STATUS__GL2_REQ_VC1_STALL_MASK                                                                    0x00000020L
+#define CHC_STATUS__GL2_DATA_VC1_STALL_MASK                                                                   0x00000040L
+#define CHC_STATUS__INPUT_BUFFER_VC0_BUSY_MASK                                                                0x00000080L
+#define CHC_STATUS__SRC_DATA_FIFO_VC0_BUSY_MASK                                                               0x00000100L
+#define CHC_STATUS__GL2_RH_BUSY_MASK                                                                          0x00000200L
+#define CHC_STATUS__NUM_REQ_PENDING_FROM_L2_MASK                                                              0x000FFC00L
+#define CHC_STATUS__VIRTUAL_FIFO_FULL_STALL_MASK                                                              0x00100000L
+#define CHC_STATUS__REQUEST_TRACKER_BUFFER_STALL_MASK                                                         0x00200000L
+#define CHC_STATUS__REQUEST_TRACKER_BUSY_MASK                                                                 0x00400000L
+#define CHC_STATUS__BUFFER_FULL_MASK                                                                          0x00800000L
+//CHCG_CTRL
+#define CHCG_CTRL__BUFFER_DEPTH_MAX__SHIFT                                                                    0x0
+#define CHCG_CTRL__VC0_BUFFER_DEPTH_MAX__SHIFT                                                                0x4
+#define CHCG_CTRL__GL2_REQ_CREDITS__SHIFT                                                                     0x8
+#define CHCG_CTRL__GL2_DATA_CREDITS__SHIFT                                                                    0xf
+#define CHCG_CTRL__TO_L1_REPEATER_FGCG_DISABLE__SHIFT                                                         0x16
+#define CHCG_CTRL__TO_L2_REPEATER_FGCG_DISABLE__SHIFT                                                         0x17
+#define CHCG_CTRL__BUFFER_DEPTH_MAX_MASK                                                                      0x0000000FL
+#define CHCG_CTRL__VC0_BUFFER_DEPTH_MAX_MASK                                                                  0x000000F0L
+#define CHCG_CTRL__GL2_REQ_CREDITS_MASK                                                                       0x00007F00L
+#define CHCG_CTRL__GL2_DATA_CREDITS_MASK                                                                      0x003F8000L
+#define CHCG_CTRL__TO_L1_REPEATER_FGCG_DISABLE_MASK                                                           0x00400000L
+#define CHCG_CTRL__TO_L2_REPEATER_FGCG_DISABLE_MASK                                                           0x00800000L
+//CHCG_STATUS
+#define CHCG_STATUS__INPUT_BUFFER_VC0_FIFO_FULL__SHIFT                                                        0x0
+#define CHCG_STATUS__OUTPUT_FIFOS_BUSY__SHIFT                                                                 0x1
+#define CHCG_STATUS__SRC_DATA_FIFO_VC0_FULL__SHIFT                                                            0x2
+#define CHCG_STATUS__GL2_REQ_VC0_STALL__SHIFT                                                                 0x3
+#define CHCG_STATUS__GL2_DATA_VC0_STALL__SHIFT                                                                0x4
+#define CHCG_STATUS__GL2_REQ_VC1_STALL__SHIFT                                                                 0x5
+#define CHCG_STATUS__GL2_DATA_VC1_STALL__SHIFT                                                                0x6
+#define CHCG_STATUS__INPUT_BUFFER_VC0_BUSY__SHIFT                                                             0x7
+#define CHCG_STATUS__SRC_DATA_FIFO_VC0_BUSY__SHIFT                                                            0x8
+#define CHCG_STATUS__GL2_RH_BUSY__SHIFT                                                                       0x9
+#define CHCG_STATUS__NUM_REQ_PENDING_FROM_L2__SHIFT                                                           0xa
+#define CHCG_STATUS__VIRTUAL_FIFO_FULL_STALL__SHIFT                                                           0x14
+#define CHCG_STATUS__REQUEST_TRACKER_BUFFER_STALL__SHIFT                                                      0x15
+#define CHCG_STATUS__REQUEST_TRACKER_BUSY__SHIFT                                                              0x16
+#define CHCG_STATUS__BUFFER_FULL__SHIFT                                                                       0x17
+#define CHCG_STATUS__INPUT_BUFFER_VC1_BUSY__SHIFT                                                             0x18
+#define CHCG_STATUS__SRC_DATA_FIFO_VC1_BUSY__SHIFT                                                            0x19
+#define CHCG_STATUS__INPUT_BUFFER_VC1_FIFO_FULL__SHIFT                                                        0x1a
+#define CHCG_STATUS__SRC_DATA_FIFO_VC1_FULL__SHIFT                                                            0x1b
+#define CHCG_STATUS__INPUT_BUFFER_VC0_FIFO_FULL_MASK                                                          0x00000001L
+#define CHCG_STATUS__OUTPUT_FIFOS_BUSY_MASK                                                                   0x00000002L
+#define CHCG_STATUS__SRC_DATA_FIFO_VC0_FULL_MASK                                                              0x00000004L
+#define CHCG_STATUS__GL2_REQ_VC0_STALL_MASK                                                                   0x00000008L
+#define CHCG_STATUS__GL2_DATA_VC0_STALL_MASK                                                                  0x00000010L
+#define CHCG_STATUS__GL2_REQ_VC1_STALL_MASK                                                                   0x00000020L
+#define CHCG_STATUS__GL2_DATA_VC1_STALL_MASK                                                                  0x00000040L
+#define CHCG_STATUS__INPUT_BUFFER_VC0_BUSY_MASK                                                               0x00000080L
+#define CHCG_STATUS__SRC_DATA_FIFO_VC0_BUSY_MASK                                                              0x00000100L
+#define CHCG_STATUS__GL2_RH_BUSY_MASK                                                                         0x00000200L
+#define CHCG_STATUS__NUM_REQ_PENDING_FROM_L2_MASK                                                             0x000FFC00L
+#define CHCG_STATUS__VIRTUAL_FIFO_FULL_STALL_MASK                                                             0x00100000L
+#define CHCG_STATUS__REQUEST_TRACKER_BUFFER_STALL_MASK                                                        0x00200000L
+#define CHCG_STATUS__REQUEST_TRACKER_BUSY_MASK                                                                0x00400000L
+#define CHCG_STATUS__BUFFER_FULL_MASK                                                                         0x00800000L
+#define CHCG_STATUS__INPUT_BUFFER_VC1_BUSY_MASK                                                               0x01000000L
+#define CHCG_STATUS__SRC_DATA_FIFO_VC1_BUSY_MASK                                                              0x02000000L
+#define CHCG_STATUS__INPUT_BUFFER_VC1_FIFO_FULL_MASK                                                          0x04000000L
+#define CHCG_STATUS__SRC_DATA_FIFO_VC1_FULL_MASK                                                              0x08000000L
+
+
+// addressBlock: gc_gl2dec
+//GL2C_CTRL
+#define GL2C_CTRL__CACHE_SIZE__SHIFT                                                                          0x0
+#define GL2C_CTRL__RATE__SHIFT                                                                                0x2
+#define GL2C_CTRL__WRITEBACK_MARGIN__SHIFT                                                                    0x4
+#define GL2C_CTRL__METADATA_LATENCY_FIFO_SIZE__SHIFT                                                          0x8
+#define GL2C_CTRL__SRC_FIFO_SIZE__SHIFT                                                                       0xc
+#define GL2C_CTRL__LATENCY_FIFO_SIZE__SHIFT                                                                   0x10
+#define GL2C_CTRL__METADATA_TO_HI_PRIORITY__SHIFT                                                             0x14
+#define GL2C_CTRL__LINEAR_SET_HASH__SHIFT                                                                     0x15
+#define GL2C_CTRL__FORCE_HIT_QUEUE_POP__SHIFT                                                                 0x16
+#define GL2C_CTRL__MDC_SIZE__SHIFT                                                                            0x18
+#define GL2C_CTRL__METADATA_TO_HIT_QUEUE__SHIFT                                                               0x1a
+#define GL2C_CTRL__IGNORE_FULLY_WRITTEN__SHIFT                                                                0x1b
+#define GL2C_CTRL__MDC_SIDEBAND_FIFO_SIZE__SHIFT                                                              0x1c
+#define GL2C_CTRL__CACHE_SIZE_MASK                                                                            0x00000003L
+#define GL2C_CTRL__RATE_MASK                                                                                  0x0000000CL
+#define GL2C_CTRL__WRITEBACK_MARGIN_MASK                                                                      0x000000F0L
+#define GL2C_CTRL__METADATA_LATENCY_FIFO_SIZE_MASK                                                            0x00000F00L
+#define GL2C_CTRL__SRC_FIFO_SIZE_MASK                                                                         0x0000F000L
+#define GL2C_CTRL__LATENCY_FIFO_SIZE_MASK                                                                     0x000F0000L
+#define GL2C_CTRL__METADATA_TO_HI_PRIORITY_MASK                                                               0x00100000L
+#define GL2C_CTRL__LINEAR_SET_HASH_MASK                                                                       0x00200000L
+#define GL2C_CTRL__FORCE_HIT_QUEUE_POP_MASK                                                                   0x00C00000L
+#define GL2C_CTRL__MDC_SIZE_MASK                                                                              0x03000000L
+#define GL2C_CTRL__METADATA_TO_HIT_QUEUE_MASK                                                                 0x04000000L
+#define GL2C_CTRL__IGNORE_FULLY_WRITTEN_MASK                                                                  0x08000000L
+#define GL2C_CTRL__MDC_SIDEBAND_FIFO_SIZE_MASK                                                                0xF0000000L
+//GL2C_CTRL2
+#define GL2C_CTRL2__PROBE_FIFO_SIZE__SHIFT                                                                    0x0
+#define GL2C_CTRL2__ADDR_MATCH_DISABLE__SHIFT                                                                 0x4
+#define GL2C_CTRL2__FILL_SIZE_32__SHIFT                                                                       0x5
+#define GL2C_CTRL2__RB_TO_HI_PRIORITY__SHIFT                                                                  0x6
+#define GL2C_CTRL2__HIT_UNDER_MISS_DISABLE__SHIFT                                                             0x7
+#define GL2C_CTRL2__RO_DISABLE__SHIFT                                                                         0x8
+#define GL2C_CTRL2__FORCE_MDC_INV__SHIFT                                                                      0x9
+#define GL2C_CTRL2__GCR_ARB_CTRL__SHIFT                                                                       0xa
+#define GL2C_CTRL2__GCR_ALL_SET__SHIFT                                                                        0xd
+#define GL2C_CTRL2__FILL_SIZE_64__SHIFT                                                                       0x11
+#define GL2C_CTRL2__USE_EA_EARLYWRRET_ON_WRITEBACK__SHIFT                                                     0x12
+#define GL2C_CTRL2__WRITEBACK_ALL_WAIT_FOR_ALL_EA_WRITE_COMPLETE__SHIFT                                       0x13
+#define GL2C_CTRL2__METADATA_VOLATILE_EN__SHIFT                                                               0x14
+#define GL2C_CTRL2__RB_VOLATILE_EN__SHIFT                                                                     0x15
+#define GL2C_CTRL2__PROBE_UNSHARED_EN__SHIFT                                                                  0x16
+#define GL2C_CTRL2__MAX_MIN_CTRL__SHIFT                                                                       0x17
+#define GL2C_CTRL2__MDC_UC_TO_C_RO_EN__SHIFT                                                                  0x1a
+#define GL2C_CTRL2__PROBE_FIFO_SIZE_MASK                                                                      0x0000000FL
+#define GL2C_CTRL2__ADDR_MATCH_DISABLE_MASK                                                                   0x00000010L
+#define GL2C_CTRL2__FILL_SIZE_32_MASK                                                                         0x00000020L
+#define GL2C_CTRL2__RB_TO_HI_PRIORITY_MASK                                                                    0x00000040L
+#define GL2C_CTRL2__HIT_UNDER_MISS_DISABLE_MASK                                                               0x00000080L
+#define GL2C_CTRL2__RO_DISABLE_MASK                                                                           0x00000100L
+#define GL2C_CTRL2__FORCE_MDC_INV_MASK                                                                        0x00000200L
+#define GL2C_CTRL2__GCR_ARB_CTRL_MASK                                                                         0x00001C00L
+#define GL2C_CTRL2__GCR_ALL_SET_MASK                                                                          0x00002000L
+#define GL2C_CTRL2__FILL_SIZE_64_MASK                                                                         0x00020000L
+#define GL2C_CTRL2__USE_EA_EARLYWRRET_ON_WRITEBACK_MASK                                                       0x00040000L
+#define GL2C_CTRL2__WRITEBACK_ALL_WAIT_FOR_ALL_EA_WRITE_COMPLETE_MASK                                         0x00080000L
+#define GL2C_CTRL2__METADATA_VOLATILE_EN_MASK                                                                 0x00100000L
+#define GL2C_CTRL2__RB_VOLATILE_EN_MASK                                                                       0x00200000L
+#define GL2C_CTRL2__PROBE_UNSHARED_EN_MASK                                                                    0x00400000L
+#define GL2C_CTRL2__MAX_MIN_CTRL_MASK                                                                         0x01800000L
+#define GL2C_CTRL2__MDC_UC_TO_C_RO_EN_MASK                                                                    0x04000000L
+//GL2C_STATUS
+#define GL2C_STATUS__NONCACHEABLE_FLOAT_ATOMIC__SHIFT                                                         0x0
+#define GL2C_STATUS__NONCACHEABLE_U8_ATOMIC__SHIFT                                                            0x4
+#define GL2C_STATUS__NONCACHEABLE_CLAMP_SUB_ATOMIC__SHIFT                                                     0x5
+#define GL2C_STATUS__WRRET_NACK_FAULT__SHIFT                                                                  0x6
+#define GL2C_STATUS__RDRET_NACK_FAULT__SHIFT                                                                  0x7
+#define GL2C_STATUS__METADATA_FED__SHIFT                                                                      0x8
+#define GL2C_STATUS__FED_FSM_STATE__SHIFT                                                                     0x9
+#define GL2C_STATUS__SAFE_MODE_FED__SHIFT                                                                     0xb
+#define GL2C_STATUS__DCC_OUT_INVALID_KEY_ERROR_CODE__SHIFT                                                    0x12
+#define GL2C_STATUS__NONCACHEABLE_FLOAT_ATOMIC_MASK                                                           0x00000001L
+#define GL2C_STATUS__NONCACHEABLE_U8_ATOMIC_MASK                                                              0x00000010L
+#define GL2C_STATUS__NONCACHEABLE_CLAMP_SUB_ATOMIC_MASK                                                       0x00000020L
+#define GL2C_STATUS__WRRET_NACK_FAULT_MASK                                                                    0x00000040L
+#define GL2C_STATUS__RDRET_NACK_FAULT_MASK                                                                    0x00000080L
+#define GL2C_STATUS__METADATA_FED_MASK                                                                        0x00000100L
+#define GL2C_STATUS__FED_FSM_STATE_MASK                                                                       0x00000600L
+#define GL2C_STATUS__SAFE_MODE_FED_MASK                                                                       0x00000800L
+#define GL2C_STATUS__DCC_OUT_INVALID_KEY_ERROR_CODE_MASK                                                      0x007C0000L
+//GL2C_ADDR_MATCH_MASK
+#define GL2C_ADDR_MATCH_MASK__ADDR_MASK__SHIFT                                                                0x0
+#define GL2C_ADDR_MATCH_MASK__ADDR_MASK_MASK                                                                  0xFFFFFFFFL
+//GL2C_ADDR_MATCH_SIZE
+#define GL2C_ADDR_MATCH_SIZE__MAX_COUNT__SHIFT                                                                0x0
+#define GL2C_ADDR_MATCH_SIZE__MAX_COUNT_MASK                                                                  0x00000007L
+//GL2C_WBINVL2
+#define GL2C_WBINVL2__DONE__SHIFT                                                                             0x4
+#define GL2C_WBINVL2__DONE_MASK                                                                               0x00000010L
+//GL2C_SOFT_RESET
+#define GL2C_SOFT_RESET__HALT_FOR_RESET__SHIFT                                                                0x0
+#define GL2C_SOFT_RESET__HALT_FOR_RESET_MASK                                                                  0x00000001L
+//GL2C_CM_CTRL0
+#define GL2C_CM_CTRL0__HASH_MASK__SHIFT                                                                       0x0
+#define GL2C_CM_CTRL0__HASH_MASK_MASK                                                                         0xFFFFFFFFL
+//GL2C_CM_CTRL1
+#define GL2C_CM_CTRL1__HASH_MASK__SHIFT                                                                       0x0
+#define GL2C_CM_CTRL1__BURST_TIMER__SHIFT                                                                     0x8
+#define GL2C_CM_CTRL1__RVF_SIZE__SHIFT                                                                        0x10
+#define GL2C_CM_CTRL1__WRITE_COH_MODE__SHIFT                                                                  0x17
+#define GL2C_CM_CTRL1__MDC_ARB_MODE__SHIFT                                                                    0x19
+#define GL2C_CM_CTRL1__READ_REQ_ONLY__SHIFT                                                                   0x1a
+#define GL2C_CM_CTRL1__COMP_TO_CONSTANT_EN__SHIFT                                                             0x1b
+#define GL2C_CM_CTRL1__COMP_TO_SINGLE_EN__SHIFT                                                               0x1c
+#define GL2C_CM_CTRL1__BURST_MODE__SHIFT                                                                      0x1d
+#define GL2C_CM_CTRL1__UNCOMP_READBACK_FILTER__SHIFT                                                          0x1e
+#define GL2C_CM_CTRL1__WAIT_ATOMIC_RECOMP_WRITE__SHIFT                                                        0x1f
+#define GL2C_CM_CTRL1__HASH_MASK_MASK                                                                         0x0000000FL
+#define GL2C_CM_CTRL1__BURST_TIMER_MASK                                                                       0x0000FF00L
+#define GL2C_CM_CTRL1__RVF_SIZE_MASK                                                                          0x000F0000L
+#define GL2C_CM_CTRL1__WRITE_COH_MODE_MASK                                                                    0x01800000L
+#define GL2C_CM_CTRL1__MDC_ARB_MODE_MASK                                                                      0x02000000L
+#define GL2C_CM_CTRL1__READ_REQ_ONLY_MASK                                                                     0x04000000L
+#define GL2C_CM_CTRL1__COMP_TO_CONSTANT_EN_MASK                                                               0x08000000L
+#define GL2C_CM_CTRL1__COMP_TO_SINGLE_EN_MASK                                                                 0x10000000L
+#define GL2C_CM_CTRL1__BURST_MODE_MASK                                                                        0x20000000L
+#define GL2C_CM_CTRL1__UNCOMP_READBACK_FILTER_MASK                                                            0x40000000L
+#define GL2C_CM_CTRL1__WAIT_ATOMIC_RECOMP_WRITE_MASK                                                          0x80000000L
+//GL2C_CM_STALL
+#define GL2C_CM_STALL__QUEUE__SHIFT                                                                           0x0
+#define GL2C_CM_STALL__QUEUE_MASK                                                                             0xFFFFFFFFL
+//GL2C_CM_CTRL2
+#define GL2C_CM_CTRL2__READ_BURST_TIMER__SHIFT                                                                0x0
+#define GL2C_CM_CTRL2__VRS_DISABLE__SHIFT                                                                     0x8
+#define GL2C_CM_CTRL2__SKIP_LOW_COMP_RATIO__SHIFT                                                             0x9
+#define GL2C_CM_CTRL2__CM_NBC_IND64_DISABLE__SHIFT                                                            0xa
+#define GL2C_CM_CTRL2__PARTIAL_WR_OPT_MODE__SHIFT                                                             0xb
+#define GL2C_CM_CTRL2__PARTIAL_WR_OPT_METADATA_WR_MODE__SHIFT                                                 0xc
+#define GL2C_CM_CTRL2__PARTIAL_WR_OPT_MAX_UNCOMP_BLK_SZ_MODE__SHIFT                                           0xd
+#define GL2C_CM_CTRL2__PARTIAL_WR_OPT_SECTOR_READBACK_MODE__SHIFT                                             0xf
+#define GL2C_CM_CTRL2__RECOMP_DISABLE__SHIFT                                                                  0x10
+#define GL2C_CM_CTRL2__DCC_COMP_KEY_ERROR_DETECTION_EN__SHIFT                                                 0x11
+#define GL2C_CM_CTRL2__DCC_CLEAR_FRAG2DCC_KEY_ERROR_CODE__SHIFT                                               0x12
+#define GL2C_CM_CTRL2__READ_BURST_TIMER_MASK                                                                  0x000000FFL
+#define GL2C_CM_CTRL2__VRS_DISABLE_MASK                                                                       0x00000100L
+#define GL2C_CM_CTRL2__SKIP_LOW_COMP_RATIO_MASK                                                               0x00000200L
+#define GL2C_CM_CTRL2__CM_NBC_IND64_DISABLE_MASK                                                              0x00000400L
+#define GL2C_CM_CTRL2__PARTIAL_WR_OPT_MODE_MASK                                                               0x00000800L
+#define GL2C_CM_CTRL2__PARTIAL_WR_OPT_METADATA_WR_MODE_MASK                                                   0x00001000L
+#define GL2C_CM_CTRL2__PARTIAL_WR_OPT_MAX_UNCOMP_BLK_SZ_MODE_MASK                                             0x00006000L
+#define GL2C_CM_CTRL2__PARTIAL_WR_OPT_SECTOR_READBACK_MODE_MASK                                               0x00008000L
+#define GL2C_CM_CTRL2__RECOMP_DISABLE_MASK                                                                    0x00010000L
+#define GL2C_CM_CTRL2__DCC_COMP_KEY_ERROR_DETECTION_EN_MASK                                                   0x00020000L
+#define GL2C_CM_CTRL2__DCC_CLEAR_FRAG2DCC_KEY_ERROR_CODE_MASK                                                 0x00040000L
+//GL2C_CTRL3
+#define GL2C_CTRL3__METADATA_MTYPE_COHERENCY__SHIFT                                                           0x0
+#define GL2C_CTRL3__METADATA_NOFILL__SHIFT                                                                    0x3
+#define GL2C_CTRL3__METADATA_NEXT_CL_PREFETCH__SHIFT                                                          0x4
+#define GL2C_CTRL3__BANK_LINEAR_HASH_MODE__SHIFT                                                              0x5
+#define GL2C_CTRL3__HTILE_TO_HI_PRIORITY__SHIFT                                                               0x6
+#define GL2C_CTRL3__UNCACHED_WRITE_ATOMIC_TO_UC_WRITE__SHIFT                                                  0x7
+#define GL2C_CTRL3__IO_CHANNEL_ENABLE__SHIFT                                                                  0x8
+#define GL2C_CTRL3__FMASK_TO_HI_PRIORITY__SHIFT                                                               0x9
+#define GL2C_CTRL3__DCC_CMASK_TO_HI_PRIORITY__SHIFT                                                           0xa
+#define GL2C_CTRL3__BANK_LINEAR_HASH_ENABLE__SHIFT                                                            0xb
+#define GL2C_CTRL3__HASH_256B_ENABLE__SHIFT                                                                   0xc
+#define GL2C_CTRL3__DECOMP_NBC_IND64_DISABLE__SHIFT                                                           0xd
+#define GL2C_CTRL3__FORCE_READ_ON_WRITE_OP__SHIFT                                                             0xe
+#define GL2C_CTRL3__FGCG_OVERRIDE__SHIFT                                                                      0xf
+#define GL2C_CTRL3__FORCE_MTYPE_UC__SHIFT                                                                     0x10
+#define GL2C_CTRL3__DGPU_SHARED_MODE__SHIFT                                                                   0x11
+#define GL2C_CTRL3__WRITE_SET_SECTOR_FULLY_WRITTEN__SHIFT                                                     0x12
+#define GL2C_CTRL3__EA_READ_SIZE_LIMIT__SHIFT                                                                 0x13
+#define GL2C_CTRL3__READ_BYPASS_AS_UC__SHIFT                                                                  0x14
+#define GL2C_CTRL3__WB_OPT_ENABLE__SHIFT                                                                      0x15
+#define GL2C_CTRL3__WB_OPT_BURST_MAX_COUNT__SHIFT                                                             0x16
+#define GL2C_CTRL3__SET_GROUP_LINEAR_HASH_ENABLE__SHIFT                                                       0x18
+#define GL2C_CTRL3__EA_GMI_DISABLE__SHIFT                                                                     0x19
+#define GL2C_CTRL3__SQC_TO_HI_PRIORITY__SHIFT                                                                 0x1a
+#define GL2C_CTRL3__INF_NAN_CLAMP__SHIFT                                                                      0x1b
+#define GL2C_CTRL3__SCRATCH__SHIFT                                                                            0x1c
+#define GL2C_CTRL3__METADATA_MTYPE_COHERENCY_MASK                                                             0x00000003L
+#define GL2C_CTRL3__METADATA_NOFILL_MASK                                                                      0x00000008L
+#define GL2C_CTRL3__METADATA_NEXT_CL_PREFETCH_MASK                                                            0x00000010L
+#define GL2C_CTRL3__BANK_LINEAR_HASH_MODE_MASK                                                                0x00000020L
+#define GL2C_CTRL3__HTILE_TO_HI_PRIORITY_MASK                                                                 0x00000040L
+#define GL2C_CTRL3__UNCACHED_WRITE_ATOMIC_TO_UC_WRITE_MASK                                                    0x00000080L
+#define GL2C_CTRL3__IO_CHANNEL_ENABLE_MASK                                                                    0x00000100L
+#define GL2C_CTRL3__FMASK_TO_HI_PRIORITY_MASK                                                                 0x00000200L
+#define GL2C_CTRL3__DCC_CMASK_TO_HI_PRIORITY_MASK                                                             0x00000400L
+#define GL2C_CTRL3__BANK_LINEAR_HASH_ENABLE_MASK                                                              0x00000800L
+#define GL2C_CTRL3__HASH_256B_ENABLE_MASK                                                                     0x00001000L
+#define GL2C_CTRL3__DECOMP_NBC_IND64_DISABLE_MASK                                                             0x00002000L
+#define GL2C_CTRL3__FORCE_READ_ON_WRITE_OP_MASK                                                               0x00004000L
+#define GL2C_CTRL3__FGCG_OVERRIDE_MASK                                                                        0x00008000L
+#define GL2C_CTRL3__FORCE_MTYPE_UC_MASK                                                                       0x00010000L
+#define GL2C_CTRL3__DGPU_SHARED_MODE_MASK                                                                     0x00020000L
+#define GL2C_CTRL3__WRITE_SET_SECTOR_FULLY_WRITTEN_MASK                                                       0x00040000L
+#define GL2C_CTRL3__EA_READ_SIZE_LIMIT_MASK                                                                   0x00080000L
+#define GL2C_CTRL3__READ_BYPASS_AS_UC_MASK                                                                    0x00100000L
+#define GL2C_CTRL3__WB_OPT_ENABLE_MASK                                                                        0x00200000L
+#define GL2C_CTRL3__WB_OPT_BURST_MAX_COUNT_MASK                                                               0x00C00000L
+#define GL2C_CTRL3__SET_GROUP_LINEAR_HASH_ENABLE_MASK                                                         0x01000000L
+#define GL2C_CTRL3__EA_GMI_DISABLE_MASK                                                                       0x02000000L
+#define GL2C_CTRL3__SQC_TO_HI_PRIORITY_MASK                                                                   0x04000000L
+#define GL2C_CTRL3__INF_NAN_CLAMP_MASK                                                                        0x08000000L
+#define GL2C_CTRL3__SCRATCH_MASK                                                                              0xF0000000L
+//GL2C_LB_CTR_CTRL
+#define GL2C_LB_CTR_CTRL__START__SHIFT                                                                        0x0
+#define GL2C_LB_CTR_CTRL__LOAD__SHIFT                                                                         0x1
+#define GL2C_LB_CTR_CTRL__CLEAR__SHIFT                                                                        0x2
+#define GL2C_LB_CTR_CTRL__PERF_CNTR_EN_OVERRIDE__SHIFT                                                        0x1f
+#define GL2C_LB_CTR_CTRL__START_MASK                                                                          0x00000001L
+#define GL2C_LB_CTR_CTRL__LOAD_MASK                                                                           0x00000002L
+#define GL2C_LB_CTR_CTRL__CLEAR_MASK                                                                          0x00000004L
+#define GL2C_LB_CTR_CTRL__PERF_CNTR_EN_OVERRIDE_MASK                                                          0x80000000L
+//GL2C_LB_DATA0
+#define GL2C_LB_DATA0__DATA__SHIFT                                                                            0x0
+#define GL2C_LB_DATA0__DATA_MASK                                                                              0xFFFFFFFFL
+//GL2C_LB_DATA1
+#define GL2C_LB_DATA1__DATA__SHIFT                                                                            0x0
+#define GL2C_LB_DATA1__DATA_MASK                                                                              0xFFFFFFFFL
+//GL2C_LB_DATA2
+#define GL2C_LB_DATA2__DATA__SHIFT                                                                            0x0
+#define GL2C_LB_DATA2__DATA_MASK                                                                              0xFFFFFFFFL
+//GL2C_LB_DATA3
+#define GL2C_LB_DATA3__DATA__SHIFT                                                                            0x0
+#define GL2C_LB_DATA3__DATA_MASK                                                                              0xFFFFFFFFL
+//GL2C_LB_CTR_SEL0
+#define GL2C_LB_CTR_SEL0__SEL0__SHIFT                                                                         0x0
+#define GL2C_LB_CTR_SEL0__DIV0__SHIFT                                                                         0xf
+#define GL2C_LB_CTR_SEL0__SEL1__SHIFT                                                                         0x10
+#define GL2C_LB_CTR_SEL0__DIV1__SHIFT                                                                         0x1f
+#define GL2C_LB_CTR_SEL0__SEL0_MASK                                                                           0x000000FFL
+#define GL2C_LB_CTR_SEL0__DIV0_MASK                                                                           0x00008000L
+#define GL2C_LB_CTR_SEL0__SEL1_MASK                                                                           0x00FF0000L
+#define GL2C_LB_CTR_SEL0__DIV1_MASK                                                                           0x80000000L
+//GL2C_LB_CTR_SEL1
+#define GL2C_LB_CTR_SEL1__SEL2__SHIFT                                                                         0x0
+#define GL2C_LB_CTR_SEL1__DIV2__SHIFT                                                                         0xf
+#define GL2C_LB_CTR_SEL1__SEL3__SHIFT                                                                         0x10
+#define GL2C_LB_CTR_SEL1__DIV3__SHIFT                                                                         0x1f
+#define GL2C_LB_CTR_SEL1__SEL2_MASK                                                                           0x000000FFL
+#define GL2C_LB_CTR_SEL1__DIV2_MASK                                                                           0x00008000L
+#define GL2C_LB_CTR_SEL1__SEL3_MASK                                                                           0x00FF0000L
+#define GL2C_LB_CTR_SEL1__DIV3_MASK                                                                           0x80000000L
+//GL2C_CTRL4
+#define GL2C_CTRL4__METADATA_WR_OP_CID__SHIFT                                                                 0x0
+#define GL2C_CTRL4__SPA_CHANNEL_ENABLE__SHIFT                                                                 0x1
+#define GL2C_CTRL4__SRC_FIFO_MDC_LOW_PRIORITY__SHIFT                                                          0x2
+#define GL2C_CTRL4__WRITEBACK_FIFO_STALL_ENABLE__SHIFT                                                        0x3
+#define GL2C_CTRL4__CM_MGCG_MODE__SHIFT                                                                       0x4
+#define GL2C_CTRL4__MDC_MGCG_MODE__SHIFT                                                                      0x5
+#define GL2C_CTRL4__TAG_MGCG_MODE__SHIFT                                                                      0x6
+#define GL2C_CTRL4__CORE_MGCG_MODE__SHIFT                                                                     0x7
+#define GL2C_CTRL4__EXECUTE_MGCG_MODE__SHIFT                                                                  0x8
+#define GL2C_CTRL4__EA_NACK_DISABLE__SHIFT                                                                    0x9
+#define GL2C_CTRL4__FED_SAFE_MODE__SHIFT                                                                      0xa
+#define GL2C_CTRL4__FLUSH_SET_COUNTER_MASK_DISABLE__SHIFT                                                     0xb
+#define GL2C_CTRL4__NO_WRITE_ACK_TO_HIT_QUEUE__SHIFT                                                          0x1a
+#define GL2C_CTRL4__METADATA_WR_OP_CID_MASK                                                                   0x00000001L
+#define GL2C_CTRL4__SPA_CHANNEL_ENABLE_MASK                                                                   0x00000002L
+#define GL2C_CTRL4__SRC_FIFO_MDC_LOW_PRIORITY_MASK                                                            0x00000004L
+#define GL2C_CTRL4__WRITEBACK_FIFO_STALL_ENABLE_MASK                                                          0x00000008L
+#define GL2C_CTRL4__CM_MGCG_MODE_MASK                                                                         0x00000010L
+#define GL2C_CTRL4__MDC_MGCG_MODE_MASK                                                                        0x00000020L
+#define GL2C_CTRL4__TAG_MGCG_MODE_MASK                                                                        0x00000040L
+#define GL2C_CTRL4__CORE_MGCG_MODE_MASK                                                                       0x00000080L
+#define GL2C_CTRL4__EXECUTE_MGCG_MODE_MASK                                                                    0x00000100L
+#define GL2C_CTRL4__EA_NACK_DISABLE_MASK                                                                      0x00000200L
+#define GL2C_CTRL4__FED_SAFE_MODE_MASK                                                                        0x00000400L
+#define GL2C_CTRL4__FLUSH_SET_COUNTER_MASK_DISABLE_MASK                                                       0x00000800L
+#define GL2C_CTRL4__NO_WRITE_ACK_TO_HIT_QUEUE_MASK                                                            0x04000000L
+//GL2C_DISCARD_STALL_CTRL
+#define GL2C_DISCARD_STALL_CTRL__LIMIT__SHIFT                                                                 0x0
+#define GL2C_DISCARD_STALL_CTRL__WINDOW__SHIFT                                                                0xf
+#define GL2C_DISCARD_STALL_CTRL__DROP_NEXT__SHIFT                                                             0x1e
+#define GL2C_DISCARD_STALL_CTRL__ENABLE__SHIFT                                                                0x1f
+#define GL2C_DISCARD_STALL_CTRL__LIMIT_MASK                                                                   0x00007FFFL
+#define GL2C_DISCARD_STALL_CTRL__WINDOW_MASK                                                                  0x3FFF8000L
+#define GL2C_DISCARD_STALL_CTRL__DROP_NEXT_MASK                                                               0x40000000L
+#define GL2C_DISCARD_STALL_CTRL__ENABLE_MASK                                                                  0x80000000L
+//GL2A_ADDR_MATCH_CTRL
+#define GL2A_ADDR_MATCH_CTRL__DISABLE__SHIFT                                                                  0x0
+#define GL2A_ADDR_MATCH_CTRL__DISABLE_MASK                                                                    0xFFFFFFFFL
+//GL2A_ADDR_MATCH_MASK
+#define GL2A_ADDR_MATCH_MASK__ADDR_MASK__SHIFT                                                                0x0
+#define GL2A_ADDR_MATCH_MASK__ADDR_MASK_MASK                                                                  0xFFFFFFFFL
+//GL2A_ADDR_MATCH_SIZE
+#define GL2A_ADDR_MATCH_SIZE__MAX_COUNT__SHIFT                                                                0x0
+#define GL2A_ADDR_MATCH_SIZE__MAX_COUNT_MASK                                                                  0x00000007L
+//GL2A_PRIORITY_CTRL
+#define GL2A_PRIORITY_CTRL__DISABLE__SHIFT                                                                    0x0
+#define GL2A_PRIORITY_CTRL__DISABLE_MASK                                                                      0xFFFFFFFFL
+//GL2A_CTRL
+#define GL2A_CTRL__RTN_ARB_TIMER_RESET_VALUE__SHIFT                                                           0x0
+#define GL2A_CTRL__STAY_ON_BURST__SHIFT                                                                       0x1
+#define GL2A_CTRL__FGCG_OVERRIDE__SHIFT                                                                       0x2
+#define GL2A_CTRL__CLIENT_ARB_PRIO_STAY__SHIFT                                                                0x3
+#define GL2A_CTRL__GCRD_CREDIT_SAFE_REG__SHIFT                                                                0x4
+#define GL2A_CTRL__REQ_CREDIT_SAFE_REG__SHIFT                                                                 0x8
+#define GL2A_CTRL__WRITE_COMBINE_TIMEOUT_COUNT__SHIFT                                                         0xc
+#define GL2A_CTRL__INTERNAL_RETURN_BYPASS_ENABLE__SHIFT                                                       0x11
+#define GL2A_CTRL__ADDR_REMOVE_COLBITS__SHIFT                                                                 0x12
+#define GL2A_CTRL__RTN_ARB_TIMER_RESET_VALUE_MASK                                                             0x00000001L
+#define GL2A_CTRL__STAY_ON_BURST_MASK                                                                         0x00000002L
+#define GL2A_CTRL__FGCG_OVERRIDE_MASK                                                                         0x00000004L
+#define GL2A_CTRL__CLIENT_ARB_PRIO_STAY_MASK                                                                  0x00000008L
+#define GL2A_CTRL__GCRD_CREDIT_SAFE_REG_MASK                                                                  0x000000F0L
+#define GL2A_CTRL__REQ_CREDIT_SAFE_REG_MASK                                                                   0x00000F00L
+#define GL2A_CTRL__WRITE_COMBINE_TIMEOUT_COUNT_MASK                                                           0x0001F000L
+#define GL2A_CTRL__INTERNAL_RETURN_BYPASS_ENABLE_MASK                                                         0x00020000L
+#define GL2A_CTRL__ADDR_REMOVE_COLBITS_MASK                                                                   0x00040000L
+//GL2A_RESP_THROTTLE_CTRL
+#define GL2A_RESP_THROTTLE_CTRL__DISABLE__SHIFT                                                               0x0
+#define GL2A_RESP_THROTTLE_CTRL__CREDIT_GL1__SHIFT                                                            0x10
+#define GL2A_RESP_THROTTLE_CTRL__CREDIT_CH__SHIFT                                                             0x18
+#define GL2A_RESP_THROTTLE_CTRL__DISABLE_MASK                                                                 0x0000FFFFL
+#define GL2A_RESP_THROTTLE_CTRL__CREDIT_GL1_MASK                                                              0x00FF0000L
+#define GL2A_RESP_THROTTLE_CTRL__CREDIT_CH_MASK                                                               0xFF000000L
+
+
+// addressBlock: gc_gl1hdec
+//GL1H_ARB_CTRL
+#define GL1H_ARB_CTRL__REQ_FGCG_DISABLE__SHIFT                                                                0x0
+#define GL1H_ARB_CTRL__SRC_FGCG_DISABLE__SHIFT                                                                0x1
+#define GL1H_ARB_CTRL__RET_FGCG_DISABLE__SHIFT                                                                0x2
+#define GL1H_ARB_CTRL__CHICKEN_BITS__SHIFT                                                                    0x3
+#define GL1H_ARB_CTRL__PERF_CNTR_EN_OVERRIDE__SHIFT                                                           0xb
+#define GL1H_ARB_CTRL__REQ_FGCG_DISABLE_MASK                                                                  0x00000001L
+#define GL1H_ARB_CTRL__SRC_FGCG_DISABLE_MASK                                                                  0x00000002L
+#define GL1H_ARB_CTRL__RET_FGCG_DISABLE_MASK                                                                  0x00000004L
+#define GL1H_ARB_CTRL__CHICKEN_BITS_MASK                                                                      0x000007F8L
+#define GL1H_ARB_CTRL__PERF_CNTR_EN_OVERRIDE_MASK                                                             0x00000800L
+//GL1H_GL1_CREDITS
+#define GL1H_GL1_CREDITS__GL1_REQ_CREDITS__SHIFT                                                              0x0
+#define GL1H_GL1_CREDITS__GL1_REQ_CREDITS_MASK                                                                0x000000FFL
+//GL1H_BURST_MASK
+#define GL1H_BURST_MASK__BURST_ADDR_MASK__SHIFT                                                               0x0
+#define GL1H_BURST_MASK__BURST_ADDR_MASK_MASK                                                                 0x000000FFL
+//GL1H_BURST_CTRL
+#define GL1H_BURST_CTRL__MAX_BURST_SIZE__SHIFT                                                                0x0
+#define GL1H_BURST_CTRL__BURST_DISABLE__SHIFT                                                                 0x3
+#define GL1H_BURST_CTRL__SPARE_BURST_CTRL_BITS__SHIFT                                                         0x4
+#define GL1H_BURST_CTRL__MAX_BURST_SIZE_MASK                                                                  0x00000007L
+#define GL1H_BURST_CTRL__BURST_DISABLE_MASK                                                                   0x00000008L
+#define GL1H_BURST_CTRL__SPARE_BURST_CTRL_BITS_MASK                                                           0x00000030L
+//GL1H_ARB_STATUS
+#define GL1H_ARB_STATUS__REQ_ARB_BUSY__SHIFT                                                                  0x0
+#define GL1H_ARB_STATUS__CLIENT1_ILLEGAL_REQ__SHIFT                                                           0x1
+#define GL1H_ARB_STATUS__REQ_ARB_BUSY_MASK                                                                    0x00000001L
+#define GL1H_ARB_STATUS__CLIENT1_ILLEGAL_REQ_MASK                                                             0x00000002L
+
+
+// addressBlock: gc_perfddec
+//CPG_PERFCOUNTER1_LO
+#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPG_PERFCOUNTER1_HI
+#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPG_PERFCOUNTER0_LO
+#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPG_PERFCOUNTER0_HI
+#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER1_LO
+#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER1_HI
+#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER0_LO
+#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPC_PERFCOUNTER0_HI
+#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER1_LO
+#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER1_HI
+#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER0_LO
+#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CPF_PERFCOUNTER0_HI
+#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CPF_LATENCY_STATS_DATA
+#define CPF_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
+#define CPF_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//CPG_LATENCY_STATS_DATA
+#define CPG_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
+#define CPG_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//CPC_LATENCY_STATS_DATA
+#define CPC_LATENCY_STATS_DATA__DATA__SHIFT                                                                   0x0
+#define CPC_LATENCY_STATS_DATA__DATA_MASK                                                                     0xFFFFFFFFL
+//GRBM_PERFCOUNTER0_LO
+#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GRBM_PERFCOUNTER0_HI
+#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GRBM_PERFCOUNTER1_LO
+#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GRBM_PERFCOUNTER1_HI
+#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GRBM_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GRBM_SE0_PERFCOUNTER_LO
+#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
+#define GRBM_SE0_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
+//GRBM_SE0_PERFCOUNTER_HI
+#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
+#define GRBM_SE0_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
+//GRBM_SE1_PERFCOUNTER_LO
+#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
+#define GRBM_SE1_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
+//GRBM_SE1_PERFCOUNTER_HI
+#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
+#define GRBM_SE1_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
+//GRBM_SE2_PERFCOUNTER_LO
+#define GRBM_SE2_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
+#define GRBM_SE2_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
+//GRBM_SE2_PERFCOUNTER_HI
+#define GRBM_SE2_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
+#define GRBM_SE2_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
+//GRBM_SE3_PERFCOUNTER_LO
+#define GRBM_SE3_PERFCOUNTER_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
+#define GRBM_SE3_PERFCOUNTER_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
+//GRBM_SE3_PERFCOUNTER_HI
+#define GRBM_SE3_PERFCOUNTER_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
+#define GRBM_SE3_PERFCOUNTER_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
+//GE1_PERFCOUNTER0_LO
+#define GE1_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER0_HI
+#define GE1_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER1_LO
+#define GE1_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER1_HI
+#define GE1_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER2_LO
+#define GE1_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER2_HI
+#define GE1_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER3_LO
+#define GE1_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GE1_PERFCOUNTER3_HI
+#define GE1_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER0_LO
+#define GE2_DIST_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER0_HI
+#define GE2_DIST_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER1_LO
+#define GE2_DIST_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER1_HI
+#define GE2_DIST_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER2_LO
+#define GE2_DIST_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER2_HI
+#define GE2_DIST_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER3_LO
+#define GE2_DIST_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GE2_DIST_PERFCOUNTER3_HI
+#define GE2_DIST_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER0_LO
+#define GE2_SE_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER0_HI
+#define GE2_SE_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER1_LO
+#define GE2_SE_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER1_HI
+#define GE2_SE_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER2_LO
+#define GE2_SE_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER2_HI
+#define GE2_SE_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER3_LO
+#define GE2_SE_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                           0xFFFFFFFFL
+//GE2_SE_PERFCOUNTER3_HI
+#define GE2_SE_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                           0xFFFFFFFFL
+//PA_SU_PERFCOUNTER0_LO
+#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER0_HI
+#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER1_LO
+#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER1_HI
+#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER2_LO
+#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER2_HI
+#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER3_LO
+#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SU_PERFCOUNTER3_HI
+#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER0_LO
+#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER0_HI
+#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER1_LO
+#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER1_HI
+#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER2_LO
+#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER2_HI
+#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER3_LO
+#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER3_HI
+#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER4_LO
+#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER4_HI
+#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER5_LO
+#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER5_HI
+#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER6_LO
+#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER6_HI
+#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER7_LO
+#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_SC_PERFCOUNTER7_HI
+#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//SPI_PERFCOUNTER0_HI
+#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER0_LO
+#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER1_HI
+#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER1_LO
+#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER2_HI
+#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER2_LO
+#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER3_HI
+#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER3_LO
+#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER4_HI
+#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER4_LO
+#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER5_HI
+#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SPI_PERFCOUNTER5_LO
+#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//PC_PERFCOUNTER0_HI
+#define PC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define PC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER0_LO
+#define PC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define PC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER1_HI
+#define PC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define PC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER1_LO
+#define PC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define PC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER2_HI
+#define PC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define PC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER2_LO
+#define PC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define PC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER3_HI
+#define PC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define PC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//PC_PERFCOUNTER3_LO
+#define PC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define PC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER0_LO
+#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER1_LO
+#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER2_LO
+#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER3_LO
+#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER4_LO
+#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER5_LO
+#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER6_LO
+#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQ_PERFCOUNTER7_LO
+#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SQ_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SQG_PERFCOUNTER0_LO
+#define SQG_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER0_HI
+#define SQG_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER1_LO
+#define SQG_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER1_HI
+#define SQG_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER2_LO
+#define SQG_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER2_HI
+#define SQG_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER3_LO
+#define SQG_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER3_HI
+#define SQG_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER4_LO
+#define SQG_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER4_HI
+#define SQG_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER5_LO
+#define SQG_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER5_HI
+#define SQG_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER6_LO
+#define SQG_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER6_HI
+#define SQG_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER7_LO
+#define SQG_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//SQG_PERFCOUNTER7_HI
+#define SQG_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define SQG_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//SX_PERFCOUNTER0_LO
+#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER0_HI
+#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER1_LO
+#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER1_HI
+#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER2_LO
+#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER2_HI
+#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER3_LO
+#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define SX_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//SX_PERFCOUNTER3_HI
+#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define SX_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//GCEA_PERFCOUNTER2_LO
+#define GCEA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GCEA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GCEA_PERFCOUNTER2_HI
+#define GCEA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GCEA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GCEA_PERFCOUNTER_LO
+#define GCEA_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                                0x0
+#define GCEA_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                  0xFFFFFFFFL
+//GCEA_PERFCOUNTER_HI
+#define GCEA_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                                0x0
+#define GCEA_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                             0x10
+#define GCEA_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                  0x0000FFFFL
+#define GCEA_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                               0xFFFF0000L
+//GDS_PERFCOUNTER0_LO
+#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER0_HI
+#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER1_LO
+#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER1_HI
+#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER2_LO
+#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER2_HI
+#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER3_LO
+#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GDS_PERFCOUNTER3_HI
+#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TA_PERFCOUNTER0_LO
+#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TA_PERFCOUNTER0_HI
+#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TA_PERFCOUNTER1_LO
+#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TA_PERFCOUNTER1_HI
+#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER0_LO
+#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TD_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER0_HI
+#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TD_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER1_LO
+#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define TD_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//TD_PERFCOUNTER1_HI
+#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define TD_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//TCP_PERFCOUNTER0_LO
+#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER0_HI
+#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER1_LO
+#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER1_HI
+#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER2_LO
+#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER2_HI
+#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER3_LO
+#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER3_HI
+#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//TCP_PERFCOUNTER_FILTER
+#define TCP_PERFCOUNTER_FILTER__BUFFER__SHIFT                                                                 0x0
+#define TCP_PERFCOUNTER_FILTER__FLAT__SHIFT                                                                   0x1
+#define TCP_PERFCOUNTER_FILTER__DIM__SHIFT                                                                    0x2
+#define TCP_PERFCOUNTER_FILTER__DATA_FORMAT__SHIFT                                                            0x5
+#define TCP_PERFCOUNTER_FILTER__NUM_FORMAT__SHIFT                                                             0xd
+#define TCP_PERFCOUNTER_FILTER__SW_MODE__SHIFT                                                                0x11
+#define TCP_PERFCOUNTER_FILTER__NUM_SAMPLES__SHIFT                                                            0x16
+#define TCP_PERFCOUNTER_FILTER__OPCODE_TYPE__SHIFT                                                            0x18
+#define TCP_PERFCOUNTER_FILTER__SLC__SHIFT                                                                    0x1b
+#define TCP_PERFCOUNTER_FILTER__DLC__SHIFT                                                                    0x1c
+#define TCP_PERFCOUNTER_FILTER__GLC__SHIFT                                                                    0x1d
+#define TCP_PERFCOUNTER_FILTER__COMPRESSION_ENABLE__SHIFT                                                     0x1e
+#define TCP_PERFCOUNTER_FILTER__BUFFER_MASK                                                                   0x00000001L
+#define TCP_PERFCOUNTER_FILTER__FLAT_MASK                                                                     0x00000002L
+#define TCP_PERFCOUNTER_FILTER__DIM_MASK                                                                      0x0000001CL
+#define TCP_PERFCOUNTER_FILTER__DATA_FORMAT_MASK                                                              0x00000FE0L
+#define TCP_PERFCOUNTER_FILTER__NUM_FORMAT_MASK                                                               0x0001E000L
+#define TCP_PERFCOUNTER_FILTER__SW_MODE_MASK                                                                  0x003E0000L
+#define TCP_PERFCOUNTER_FILTER__NUM_SAMPLES_MASK                                                              0x00C00000L
+#define TCP_PERFCOUNTER_FILTER__OPCODE_TYPE_MASK                                                              0x07000000L
+#define TCP_PERFCOUNTER_FILTER__SLC_MASK                                                                      0x08000000L
+#define TCP_PERFCOUNTER_FILTER__DLC_MASK                                                                      0x10000000L
+#define TCP_PERFCOUNTER_FILTER__GLC_MASK                                                                      0x20000000L
+#define TCP_PERFCOUNTER_FILTER__COMPRESSION_ENABLE_MASK                                                       0x40000000L
+//TCP_PERFCOUNTER_FILTER2
+#define TCP_PERFCOUNTER_FILTER2__REQ_MODE__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER_FILTER2__REQ_MODE_MASK                                                                0x00000007L
+//TCP_PERFCOUNTER_FILTER_EN
+#define TCP_PERFCOUNTER_FILTER_EN__BUFFER__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER_FILTER_EN__FLAT__SHIFT                                                                0x1
+#define TCP_PERFCOUNTER_FILTER_EN__DIM__SHIFT                                                                 0x2
+#define TCP_PERFCOUNTER_FILTER_EN__DATA_FORMAT__SHIFT                                                         0x3
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_FORMAT__SHIFT                                                          0x4
+#define TCP_PERFCOUNTER_FILTER_EN__SW_MODE__SHIFT                                                             0x5
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_SAMPLES__SHIFT                                                         0x6
+#define TCP_PERFCOUNTER_FILTER_EN__OPCODE_TYPE__SHIFT                                                         0x7
+#define TCP_PERFCOUNTER_FILTER_EN__SLC__SHIFT                                                                 0x8
+#define TCP_PERFCOUNTER_FILTER_EN__DLC__SHIFT                                                                 0x9
+#define TCP_PERFCOUNTER_FILTER_EN__GLC__SHIFT                                                                 0xa
+#define TCP_PERFCOUNTER_FILTER_EN__COMPRESSION_ENABLE__SHIFT                                                  0xb
+#define TCP_PERFCOUNTER_FILTER_EN__REQ_MODE__SHIFT                                                            0xc
+#define TCP_PERFCOUNTER_FILTER_EN__BUFFER_MASK                                                                0x00000001L
+#define TCP_PERFCOUNTER_FILTER_EN__FLAT_MASK                                                                  0x00000002L
+#define TCP_PERFCOUNTER_FILTER_EN__DIM_MASK                                                                   0x00000004L
+#define TCP_PERFCOUNTER_FILTER_EN__DATA_FORMAT_MASK                                                           0x00000008L
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_FORMAT_MASK                                                            0x00000010L
+#define TCP_PERFCOUNTER_FILTER_EN__SW_MODE_MASK                                                               0x00000020L
+#define TCP_PERFCOUNTER_FILTER_EN__NUM_SAMPLES_MASK                                                           0x00000040L
+#define TCP_PERFCOUNTER_FILTER_EN__OPCODE_TYPE_MASK                                                           0x00000080L
+#define TCP_PERFCOUNTER_FILTER_EN__SLC_MASK                                                                   0x00000100L
+#define TCP_PERFCOUNTER_FILTER_EN__DLC_MASK                                                                   0x00000200L
+#define TCP_PERFCOUNTER_FILTER_EN__GLC_MASK                                                                   0x00000400L
+#define TCP_PERFCOUNTER_FILTER_EN__COMPRESSION_ENABLE_MASK                                                    0x00000800L
+#define TCP_PERFCOUNTER_FILTER_EN__REQ_MODE_MASK                                                              0x00001000L
+//GL2C_PERFCOUNTER0_LO
+#define GL2C_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL2C_PERFCOUNTER0_HI
+#define GL2C_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL2C_PERFCOUNTER1_LO
+#define GL2C_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL2C_PERFCOUNTER1_HI
+#define GL2C_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL2C_PERFCOUNTER2_LO
+#define GL2C_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL2C_PERFCOUNTER2_HI
+#define GL2C_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL2C_PERFCOUNTER3_LO
+#define GL2C_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL2C_PERFCOUNTER3_HI
+#define GL2C_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL2A_PERFCOUNTER0_LO
+#define GL2A_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL2A_PERFCOUNTER0_HI
+#define GL2A_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL2A_PERFCOUNTER1_LO
+#define GL2A_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL2A_PERFCOUNTER1_HI
+#define GL2A_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL2A_PERFCOUNTER2_LO
+#define GL2A_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL2A_PERFCOUNTER2_HI
+#define GL2A_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL2A_PERFCOUNTER3_LO
+#define GL2A_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL2A_PERFCOUNTER3_HI
+#define GL2A_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER0_LO
+#define GL1C_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER0_HI
+#define GL1C_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER1_LO
+#define GL1C_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER1_HI
+#define GL1C_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER2_LO
+#define GL1C_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER2_HI
+#define GL1C_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER3_LO
+#define GL1C_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1C_PERFCOUNTER3_HI
+#define GL1C_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//CHC_PERFCOUNTER0_LO
+#define CHC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER0_HI
+#define CHC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER1_LO
+#define CHC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER1_HI
+#define CHC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER2_LO
+#define CHC_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER2_HI
+#define CHC_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER3_LO
+#define CHC_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHC_PERFCOUNTER3_HI
+#define CHC_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHCG_PERFCOUNTER0_LO
+#define CHCG_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define CHCG_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//CHCG_PERFCOUNTER0_HI
+#define CHCG_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define CHCG_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//CHCG_PERFCOUNTER1_LO
+#define CHCG_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define CHCG_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//CHCG_PERFCOUNTER1_HI
+#define CHCG_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define CHCG_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//CHCG_PERFCOUNTER2_LO
+#define CHCG_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define CHCG_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//CHCG_PERFCOUNTER2_HI
+#define CHCG_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define CHCG_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//CHCG_PERFCOUNTER3_LO
+#define CHCG_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define CHCG_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//CHCG_PERFCOUNTER3_HI
+#define CHCG_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define CHCG_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//CB_PERFCOUNTER0_LO
+#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER0_HI
+#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER1_LO
+#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER1_HI
+#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER2_LO
+#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER2_HI
+#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER3_LO
+#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define CB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//CB_PERFCOUNTER3_HI
+#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define CB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER0_LO
+#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER0_HI
+#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER1_LO
+#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER1_HI
+#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER2_LO
+#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER2_HI
+#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER3_LO
+#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                             0x0
+#define DB_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                               0xFFFFFFFFL
+//DB_PERFCOUNTER3_HI
+#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                             0x0
+#define DB_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                               0xFFFFFFFFL
+//RLC_PERFCOUNTER0_LO
+#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RLC_PERFCOUNTER0_HI
+#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RLC_PERFCOUNTER1_LO
+#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RLC_PERFCOUNTER1_HI
+#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RLC_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER0_LO
+#define RMI_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER0_HI
+#define RMI_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER1_LO
+#define RMI_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER1_HI
+#define RMI_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER2_LO
+#define RMI_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER2_HI
+#define RMI_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER3_LO
+#define RMI_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//RMI_PERFCOUNTER3_HI
+#define RMI_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GCR_PERFCOUNTER0_LO
+#define GCR_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GCR_PERFCOUNTER0_HI
+#define GCR_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GCR_PERFCOUNTER1_LO
+#define GCR_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GCR_PERFCOUNTER1_HI
+#define GCR_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//PA_PH_PERFCOUNTER0_LO
+#define PA_PH_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER0_HI
+#define PA_PH_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER1_LO
+#define PA_PH_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER1_HI
+#define PA_PH_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER2_LO
+#define PA_PH_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER2_HI
+#define PA_PH_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER3_LO
+#define PA_PH_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER3_HI
+#define PA_PH_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER4_LO
+#define PA_PH_PERFCOUNTER4_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER4_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER4_HI
+#define PA_PH_PERFCOUNTER4_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER4_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER5_LO
+#define PA_PH_PERFCOUNTER5_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER5_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER5_HI
+#define PA_PH_PERFCOUNTER5_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER5_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER6_LO
+#define PA_PH_PERFCOUNTER6_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER6_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER6_HI
+#define PA_PH_PERFCOUNTER6_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER6_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER7_LO
+#define PA_PH_PERFCOUNTER7_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER7_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//PA_PH_PERFCOUNTER7_HI
+#define PA_PH_PERFCOUNTER7_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER7_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER0_LO
+#define UTCL1_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER0_HI
+#define UTCL1_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER1_LO
+#define UTCL1_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER1_HI
+#define UTCL1_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER2_LO
+#define UTCL1_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER2_HI
+#define UTCL1_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER3_LO
+#define UTCL1_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//UTCL1_PERFCOUNTER3_HI
+#define UTCL1_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define UTCL1_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//GL1A_PERFCOUNTER0_LO
+#define GL1A_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER0_HI
+#define GL1A_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER1_LO
+#define GL1A_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER1_HI
+#define GL1A_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER2_LO
+#define GL1A_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER2_HI
+#define GL1A_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER3_LO
+#define GL1A_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1A_PERFCOUNTER3_HI
+#define GL1A_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1H_PERFCOUNTER0_LO
+#define GL1H_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1H_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1H_PERFCOUNTER0_HI
+#define GL1H_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1H_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1H_PERFCOUNTER1_LO
+#define GL1H_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1H_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1H_PERFCOUNTER1_HI
+#define GL1H_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1H_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1H_PERFCOUNTER2_LO
+#define GL1H_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1H_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1H_PERFCOUNTER2_HI
+#define GL1H_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1H_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//GL1H_PERFCOUNTER3_LO
+#define GL1H_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                           0x0
+#define GL1H_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                             0xFFFFFFFFL
+//GL1H_PERFCOUNTER3_HI
+#define GL1H_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                           0x0
+#define GL1H_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                             0xFFFFFFFFL
+//CHA_PERFCOUNTER0_LO
+#define CHA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER0_HI
+#define CHA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER1_LO
+#define CHA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER1_HI
+#define CHA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER2_LO
+#define CHA_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER2_HI
+#define CHA_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER3_LO
+#define CHA_PERFCOUNTER3_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER3_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//CHA_PERFCOUNTER3_HI
+#define CHA_PERFCOUNTER3_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GUS_PERFCOUNTER2_LO
+#define GUS_PERFCOUNTER2_LO__PERFCOUNTER_LO__SHIFT                                                            0x0
+#define GUS_PERFCOUNTER2_LO__PERFCOUNTER_LO_MASK                                                              0xFFFFFFFFL
+//GUS_PERFCOUNTER2_HI
+#define GUS_PERFCOUNTER2_HI__PERFCOUNTER_HI__SHIFT                                                            0x0
+#define GUS_PERFCOUNTER2_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
+//GUS_PERFCOUNTER_LO
+#define GUS_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                                 0x0
+#define GUS_PERFCOUNTER_LO__COUNTER_LO_MASK                                                                   0xFFFFFFFFL
+//GUS_PERFCOUNTER_HI
+#define GUS_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                                 0x0
+#define GUS_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                              0x10
+#define GUS_PERFCOUNTER_HI__COUNTER_HI_MASK                                                                   0x0000FFFFL
+#define GUS_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                                0xFFFF0000L
+
+
+// addressBlock: gc_perfsdec
+//CPG_PERFCOUNTER1_SELECT
+#define CPG_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPG_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x1c
+#define CPG_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPG_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPG_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0xF0000000L
+//CPG_PERFCOUNTER0_SELECT1
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CPG_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
+#define CPG_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
+//CPG_PERFCOUNTER0_SELECT
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CPG_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPG_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CPG_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPG_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CPC_PERFCOUNTER1_SELECT
+#define CPC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPC_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x1c
+#define CPC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPC_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0xF0000000L
+//CPC_PERFCOUNTER0_SELECT1
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CPC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
+#define CPC_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
+//CPF_PERFCOUNTER1_SELECT
+#define CPF_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPF_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x1c
+#define CPF_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPF_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPF_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0xF0000000L
+//CPF_PERFCOUNTER0_SELECT1
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE3__SHIFT                                                           0x18
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE2__SHIFT                                                           0x1c
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CPF_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE3_MASK                                                             0x0F000000L
+#define CPF_PERFCOUNTER0_SELECT1__CNTR_MODE2_MASK                                                             0xF0000000L
+//CPF_PERFCOUNTER0_SELECT
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CPF_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPF_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CPF_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPF_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CP_PERFMON_CNTL
+#define CP_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                                 0x0
+#define CP_PERFMON_CNTL__SPM_PERFMON_STATE__SHIFT                                                             0x4
+#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE__SHIFT                                                           0x8
+#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT                                                         0xa
+#define CP_PERFMON_CNTL__PERFMON_STATE_MASK                                                                   0x0000000FL
+#define CP_PERFMON_CNTL__SPM_PERFMON_STATE_MASK                                                               0x000000F0L
+#define CP_PERFMON_CNTL__PERFMON_ENABLE_MODE_MASK                                                             0x00000300L
+#define CP_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK                                                           0x00000400L
+//CPC_PERFCOUNTER0_SELECT
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CPC_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE1__SHIFT                                                            0x18
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE0__SHIFT                                                            0x1c
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CPC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CPC_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE1_MASK                                                              0x0F000000L
+#define CPC_PERFCOUNTER0_SELECT__CNTR_MODE0_MASK                                                              0xF0000000L
+//CPF_TC_PERF_COUNTER_WINDOW_SELECT
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x00000007L
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
+#define CPF_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
+//CPG_TC_PERF_COUNTER_WINDOW_SELECT
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x0000001FL
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
+#define CPG_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
+//CPF_LATENCY_STATS_SELECT
+#define CPF_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
+#define CPF_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
+#define CPF_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
+#define CPF_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000000FL
+#define CPF_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
+#define CPF_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
+//CPG_LATENCY_STATS_SELECT
+#define CPG_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
+#define CPG_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
+#define CPG_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
+#define CPG_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000001FL
+#define CPG_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
+#define CPG_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
+//CPC_LATENCY_STATS_SELECT
+#define CPC_LATENCY_STATS_SELECT__INDEX__SHIFT                                                                0x0
+#define CPC_LATENCY_STATS_SELECT__CLEAR__SHIFT                                                                0x1e
+#define CPC_LATENCY_STATS_SELECT__ENABLE__SHIFT                                                               0x1f
+#define CPC_LATENCY_STATS_SELECT__INDEX_MASK                                                                  0x0000001FL
+#define CPC_LATENCY_STATS_SELECT__CLEAR_MASK                                                                  0x40000000L
+#define CPC_LATENCY_STATS_SELECT__ENABLE_MASK                                                                 0x80000000L
+//CPC_TC_PERF_COUNTER_WINDOW_SELECT
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__INDEX__SHIFT                                                       0x0
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS__SHIFT                                                      0x1e
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE__SHIFT                                                      0x1f
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__INDEX_MASK                                                         0x0000001FL
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__ALWAYS_MASK                                                        0x40000000L
+#define CPC_TC_PERF_COUNTER_WINDOW_SELECT__ENABLE_MASK                                                        0x80000000L
+//CP_DRAW_OBJECT
+#define CP_DRAW_OBJECT__OBJECT__SHIFT                                                                         0x0
+#define CP_DRAW_OBJECT__OBJECT_MASK                                                                           0xFFFFFFFFL
+//CP_DRAW_OBJECT_COUNTER
+#define CP_DRAW_OBJECT_COUNTER__COUNT__SHIFT                                                                  0x0
+#define CP_DRAW_OBJECT_COUNTER__COUNT_MASK                                                                    0x0000FFFFL
+//CP_DRAW_WINDOW_MASK_HI
+#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI__SHIFT                                                         0x0
+#define CP_DRAW_WINDOW_MASK_HI__WINDOW_MASK_HI_MASK                                                           0xFFFFFFFFL
+//CP_DRAW_WINDOW_HI
+#define CP_DRAW_WINDOW_HI__WINDOW_HI__SHIFT                                                                   0x0
+#define CP_DRAW_WINDOW_HI__WINDOW_HI_MASK                                                                     0xFFFFFFFFL
+//CP_DRAW_WINDOW_LO
+#define CP_DRAW_WINDOW_LO__MIN__SHIFT                                                                         0x0
+#define CP_DRAW_WINDOW_LO__MAX__SHIFT                                                                         0x10
+#define CP_DRAW_WINDOW_LO__MIN_MASK                                                                           0x0000FFFFL
+#define CP_DRAW_WINDOW_LO__MAX_MASK                                                                           0xFFFF0000L
+//CP_DRAW_WINDOW_CNTL
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX__SHIFT                                                0x0
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN__SHIFT                                                0x1
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI__SHIFT                                                    0x2
+#define CP_DRAW_WINDOW_CNTL__MODE__SHIFT                                                                      0x8
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MAX_MASK                                                  0x00000001L
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_LO_MIN_MASK                                                  0x00000002L
+#define CP_DRAW_WINDOW_CNTL__DISABLE_DRAW_WINDOW_HI_MASK                                                      0x00000004L
+#define CP_DRAW_WINDOW_CNTL__MODE_MASK                                                                        0x00000100L
+//GRBM_PERFCOUNTER0_SELECT
+#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xa
+#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xb
+#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                            0xd
+#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                            0xe
+#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x10
+#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                            0x11
+#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x12
+#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT                                          0x13
+#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x14
+#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x15
+#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT                                            0x16
+#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT                                           0x18
+#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x19
+#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1a
+#define GRBM_PERFCOUNTER0_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1b
+#define GRBM_PERFCOUNTER0_SELECT__GE_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1c
+#define GRBM_PERFCOUNTER0_SELECT__UTCL2_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1d
+#define GRBM_PERFCOUNTER0_SELECT__EA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1e
+#define GRBM_PERFCOUNTER0_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1f
+#define GRBM_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x0000003FL
+#define GRBM_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000400L
+#define GRBM_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000800L
+#define GRBM_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                              0x00002000L
+#define GRBM_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                              0x00004000L
+#define GRBM_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                             0x00010000L
+#define GRBM_PERFCOUNTER0_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                              0x00020000L
+#define GRBM_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                              0x00040000L
+#define GRBM_PERFCOUNTER0_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK                                            0x00080000L
+#define GRBM_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                              0x00100000L
+#define GRBM_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                              0x00200000L
+#define GRBM_PERFCOUNTER0_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK                                              0x00400000L
+#define GRBM_PERFCOUNTER0_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK                                             0x01000000L
+#define GRBM_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                             0x02000000L
+#define GRBM_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                             0x04000000L
+#define GRBM_PERFCOUNTER0_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                             0x08000000L
+#define GRBM_PERFCOUNTER0_SELECT__GE_BUSY_USER_DEFINED_MASK_MASK                                              0x10000000L
+#define GRBM_PERFCOUNTER0_SELECT__UTCL2_BUSY_USER_DEFINED_MASK_MASK                                           0x20000000L
+#define GRBM_PERFCOUNTER0_SELECT__EA_BUSY_USER_DEFINED_MASK_MASK                                              0x40000000L
+#define GRBM_PERFCOUNTER0_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                             0x80000000L
+//GRBM_PERFCOUNTER1_SELECT
+#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xa
+#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                           0xb
+#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                            0xd
+#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                            0xe
+#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x10
+#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                            0x11
+#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x12
+#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK__SHIFT                                          0x13
+#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x14
+#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                            0x15
+#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK__SHIFT                                            0x16
+#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK__SHIFT                                           0x18
+#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x19
+#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1a
+#define GRBM_PERFCOUNTER1_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1b
+#define GRBM_PERFCOUNTER1_SELECT__GE_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1c
+#define GRBM_PERFCOUNTER1_SELECT__UTCL2_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1d
+#define GRBM_PERFCOUNTER1_SELECT__EA_BUSY_USER_DEFINED_MASK__SHIFT                                            0x1e
+#define GRBM_PERFCOUNTER1_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1f
+#define GRBM_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x0000003FL
+#define GRBM_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000400L
+#define GRBM_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                             0x00000800L
+#define GRBM_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                              0x00002000L
+#define GRBM_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                              0x00004000L
+#define GRBM_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                             0x00010000L
+#define GRBM_PERFCOUNTER1_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                              0x00020000L
+#define GRBM_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                              0x00040000L
+#define GRBM_PERFCOUNTER1_SELECT__GRBM_BUSY_USER_DEFINED_MASK_MASK                                            0x00080000L
+#define GRBM_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                              0x00100000L
+#define GRBM_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                              0x00200000L
+#define GRBM_PERFCOUNTER1_SELECT__CP_BUSY_USER_DEFINED_MASK_MASK                                              0x00400000L
+#define GRBM_PERFCOUNTER1_SELECT__GDS_BUSY_USER_DEFINED_MASK_MASK                                             0x01000000L
+#define GRBM_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                             0x02000000L
+#define GRBM_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                             0x04000000L
+#define GRBM_PERFCOUNTER1_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                             0x08000000L
+#define GRBM_PERFCOUNTER1_SELECT__GE_BUSY_USER_DEFINED_MASK_MASK                                              0x10000000L
+#define GRBM_PERFCOUNTER1_SELECT__UTCL2_BUSY_USER_DEFINED_MASK_MASK                                           0x20000000L
+#define GRBM_PERFCOUNTER1_SELECT__EA_BUSY_USER_DEFINED_MASK_MASK                                              0x40000000L
+#define GRBM_PERFCOUNTER1_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                             0x80000000L
+//GRBM_SE0_PERFCOUNTER_SELECT
+#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
+#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
+#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
+#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
+#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
+#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
+#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
+#define GRBM_SE0_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
+#define GRBM_SE0_PERFCOUNTER_SELECT__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                      0x17
+#define GRBM_SE0_PERFCOUNTER_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                        0x18
+#define GRBM_SE0_PERFCOUNTER_SELECT__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                      0x19
+#define GRBM_SE0_PERFCOUNTER_SELECT__GL1H_BUSY_USER_DEFINED_MASK__SHIFT                                       0x1a
+#define GRBM_SE0_PERFCOUNTER_SELECT__PC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1b
+#define GRBM_SE0_PERFCOUNTER_SELECT__SEDC_BUSY_USER_DEFINED_MASK__SHIFT                                       0x1c
+#define GRBM_SE0_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
+#define GRBM_SE0_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                        0x00800000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                          0x01000000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                        0x02000000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__GL1H_BUSY_USER_DEFINED_MASK_MASK                                         0x04000000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__PC_BUSY_USER_DEFINED_MASK_MASK                                           0x08000000L
+#define GRBM_SE0_PERFCOUNTER_SELECT__SEDC_BUSY_USER_DEFINED_MASK_MASK                                         0x10000000L
+//GRBM_SE1_PERFCOUNTER_SELECT
+#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
+#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
+#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
+#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
+#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
+#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
+#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
+#define GRBM_SE1_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
+#define GRBM_SE1_PERFCOUNTER_SELECT__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                      0x17
+#define GRBM_SE1_PERFCOUNTER_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                        0x18
+#define GRBM_SE1_PERFCOUNTER_SELECT__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                      0x19
+#define GRBM_SE1_PERFCOUNTER_SELECT__GL1H_BUSY_USER_DEFINED_MASK__SHIFT                                       0x1a
+#define GRBM_SE1_PERFCOUNTER_SELECT__PC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1b
+#define GRBM_SE1_PERFCOUNTER_SELECT__SEDC_BUSY_USER_DEFINED_MASK__SHIFT                                       0x1c
+#define GRBM_SE1_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
+#define GRBM_SE1_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                        0x00800000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                          0x01000000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                        0x02000000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__GL1H_BUSY_USER_DEFINED_MASK_MASK                                         0x04000000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__PC_BUSY_USER_DEFINED_MASK_MASK                                           0x08000000L
+#define GRBM_SE1_PERFCOUNTER_SELECT__SEDC_BUSY_USER_DEFINED_MASK_MASK                                         0x10000000L
+//GRBM_SE2_PERFCOUNTER_SELECT
+#define GRBM_SE2_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
+#define GRBM_SE2_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
+#define GRBM_SE2_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
+#define GRBM_SE2_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
+#define GRBM_SE2_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
+#define GRBM_SE2_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
+#define GRBM_SE2_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
+#define GRBM_SE2_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
+#define GRBM_SE2_PERFCOUNTER_SELECT__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                      0x17
+#define GRBM_SE2_PERFCOUNTER_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                        0x18
+#define GRBM_SE2_PERFCOUNTER_SELECT__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                      0x19
+#define GRBM_SE2_PERFCOUNTER_SELECT__GL1H_BUSY_USER_DEFINED_MASK__SHIFT                                       0x1a
+#define GRBM_SE2_PERFCOUNTER_SELECT__PC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1b
+#define GRBM_SE2_PERFCOUNTER_SELECT__SEDC_BUSY_USER_DEFINED_MASK__SHIFT                                       0x1c
+#define GRBM_SE2_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
+#define GRBM_SE2_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                        0x00800000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                          0x01000000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                        0x02000000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__GL1H_BUSY_USER_DEFINED_MASK_MASK                                         0x04000000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__PC_BUSY_USER_DEFINED_MASK_MASK                                           0x08000000L
+#define GRBM_SE2_PERFCOUNTER_SELECT__SEDC_BUSY_USER_DEFINED_MASK_MASK                                         0x10000000L
+//GRBM_SE3_PERFCOUNTER_SELECT
+#define GRBM_SE3_PERFCOUNTER_SELECT__PERF_SEL__SHIFT                                                          0x0
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xa
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                        0xb
+#define GRBM_SE3_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                         0xc
+#define GRBM_SE3_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                         0xd
+#define GRBM_SE3_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                        0xf
+#define GRBM_SE3_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x10
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                         0x12
+#define GRBM_SE3_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                         0x14
+#define GRBM_SE3_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x15
+#define GRBM_SE3_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK__SHIFT                                        0x16
+#define GRBM_SE3_PERFCOUNTER_SELECT__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                      0x17
+#define GRBM_SE3_PERFCOUNTER_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                        0x18
+#define GRBM_SE3_PERFCOUNTER_SELECT__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                      0x19
+#define GRBM_SE3_PERFCOUNTER_SELECT__GL1H_BUSY_USER_DEFINED_MASK__SHIFT                                       0x1a
+#define GRBM_SE3_PERFCOUNTER_SELECT__PC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x1b
+#define GRBM_SE3_PERFCOUNTER_SELECT__SEDC_BUSY_USER_DEFINED_MASK__SHIFT                                       0x1c
+#define GRBM_SE3_PERFCOUNTER_SELECT__PERF_SEL_MASK                                                            0x0000003FL
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000400L
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                          0x00000800L
+#define GRBM_SE3_PERFCOUNTER_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                           0x00001000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                           0x00002000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                          0x00008000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__SC_BUSY_USER_DEFINED_MASK_MASK                                           0x00010000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                           0x00040000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                           0x00100000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                          0x00200000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__RMI_BUSY_USER_DEFINED_MASK_MASK                                          0x00400000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                        0x00800000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                          0x01000000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                        0x02000000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__GL1H_BUSY_USER_DEFINED_MASK_MASK                                         0x04000000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__PC_BUSY_USER_DEFINED_MASK_MASK                                           0x08000000L
+#define GRBM_SE3_PERFCOUNTER_SELECT__SEDC_BUSY_USER_DEFINED_MASK_MASK                                         0x10000000L
+//GRBM_PERFCOUNTER0_SELECT_HI
+#define GRBM_PERFCOUNTER0_SELECT_HI__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                      0x1
+#define GRBM_PERFCOUNTER0_SELECT_HI__GL2CC_BUSY_USER_DEFINED_MASK__SHIFT                                      0x2
+#define GRBM_PERFCOUNTER0_SELECT_HI__SDMA_BUSY_USER_DEFINED_MASK__SHIFT                                       0x3
+#define GRBM_PERFCOUNTER0_SELECT_HI__CH_BUSY_USER_DEFINED_MASK__SHIFT                                         0x4
+#define GRBM_PERFCOUNTER0_SELECT_HI__PH_BUSY_USER_DEFINED_MASK__SHIFT                                         0x5
+#define GRBM_PERFCOUNTER0_SELECT_HI__PMM_BUSY_USER_DEFINED_MASK__SHIFT                                        0x6
+#define GRBM_PERFCOUNTER0_SELECT_HI__GUS_BUSY_USER_DEFINED_MASK__SHIFT                                        0x7
+#define GRBM_PERFCOUNTER0_SELECT_HI__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                      0x8
+#define GRBM_PERFCOUNTER0_SELECT_HI__GL1H_BUSY_USER_DEFINED_MASK__SHIFT                                       0x9
+#define GRBM_PERFCOUNTER0_SELECT_HI__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                        0x00000002L
+#define GRBM_PERFCOUNTER0_SELECT_HI__GL2CC_BUSY_USER_DEFINED_MASK_MASK                                        0x00000004L
+#define GRBM_PERFCOUNTER0_SELECT_HI__SDMA_BUSY_USER_DEFINED_MASK_MASK                                         0x00000008L
+#define GRBM_PERFCOUNTER0_SELECT_HI__CH_BUSY_USER_DEFINED_MASK_MASK                                           0x00000010L
+#define GRBM_PERFCOUNTER0_SELECT_HI__PH_BUSY_USER_DEFINED_MASK_MASK                                           0x00000020L
+#define GRBM_PERFCOUNTER0_SELECT_HI__PMM_BUSY_USER_DEFINED_MASK_MASK                                          0x00000040L
+#define GRBM_PERFCOUNTER0_SELECT_HI__GUS_BUSY_USER_DEFINED_MASK_MASK                                          0x00000080L
+#define GRBM_PERFCOUNTER0_SELECT_HI__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                        0x00000100L
+#define GRBM_PERFCOUNTER0_SELECT_HI__GL1H_BUSY_USER_DEFINED_MASK_MASK                                         0x00000200L
+//GRBM_PERFCOUNTER1_SELECT_HI
+#define GRBM_PERFCOUNTER1_SELECT_HI__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                      0x1
+#define GRBM_PERFCOUNTER1_SELECT_HI__GL2CC_BUSY_USER_DEFINED_MASK__SHIFT                                      0x2
+#define GRBM_PERFCOUNTER1_SELECT_HI__SDMA_BUSY_USER_DEFINED_MASK__SHIFT                                       0x3
+#define GRBM_PERFCOUNTER1_SELECT_HI__CH_BUSY_USER_DEFINED_MASK__SHIFT                                         0x4
+#define GRBM_PERFCOUNTER1_SELECT_HI__PH_BUSY_USER_DEFINED_MASK__SHIFT                                         0x5
+#define GRBM_PERFCOUNTER1_SELECT_HI__PMM_BUSY_USER_DEFINED_MASK__SHIFT                                        0x6
+#define GRBM_PERFCOUNTER1_SELECT_HI__GUS_BUSY_USER_DEFINED_MASK__SHIFT                                        0x7
+#define GRBM_PERFCOUNTER1_SELECT_HI__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                      0x8
+#define GRBM_PERFCOUNTER1_SELECT_HI__GL1H_BUSY_USER_DEFINED_MASK__SHIFT                                       0x9
+#define GRBM_PERFCOUNTER1_SELECT_HI__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                        0x00000002L
+#define GRBM_PERFCOUNTER1_SELECT_HI__GL2CC_BUSY_USER_DEFINED_MASK_MASK                                        0x00000004L
+#define GRBM_PERFCOUNTER1_SELECT_HI__SDMA_BUSY_USER_DEFINED_MASK_MASK                                         0x00000008L
+#define GRBM_PERFCOUNTER1_SELECT_HI__CH_BUSY_USER_DEFINED_MASK_MASK                                           0x00000010L
+#define GRBM_PERFCOUNTER1_SELECT_HI__PH_BUSY_USER_DEFINED_MASK_MASK                                           0x00000020L
+#define GRBM_PERFCOUNTER1_SELECT_HI__PMM_BUSY_USER_DEFINED_MASK_MASK                                          0x00000040L
+#define GRBM_PERFCOUNTER1_SELECT_HI__GUS_BUSY_USER_DEFINED_MASK_MASK                                          0x00000080L
+#define GRBM_PERFCOUNTER1_SELECT_HI__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                        0x00000100L
+#define GRBM_PERFCOUNTER1_SELECT_HI__GL1H_BUSY_USER_DEFINED_MASK_MASK                                         0x00000200L
+//GE1_PERFCOUNTER0_SELECT
+#define GE1_PERFCOUNTER0_SELECT__PERF_SEL0__SHIFT                                                             0x0
+#define GE1_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GE1_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GE1_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GE1_PERFCOUNTER0_SELECT__PERF_MODE0__SHIFT                                                            0x1c
+#define GE1_PERFCOUNTER0_SELECT__PERF_SEL0_MASK                                                               0x000003FFL
+#define GE1_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GE1_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GE1_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GE1_PERFCOUNTER0_SELECT__PERF_MODE0_MASK                                                              0xF0000000L
+//GE1_PERFCOUNTER0_SELECT1
+#define GE1_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GE1_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GE1_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GE1_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GE1_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GE1_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GE1_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GE1_PERFCOUNTER1_SELECT
+#define GE1_PERFCOUNTER1_SELECT__PERF_SEL0__SHIFT                                                             0x0
+#define GE1_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GE1_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GE1_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GE1_PERFCOUNTER1_SELECT__PERF_MODE0__SHIFT                                                            0x1c
+#define GE1_PERFCOUNTER1_SELECT__PERF_SEL0_MASK                                                               0x000003FFL
+#define GE1_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GE1_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GE1_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GE1_PERFCOUNTER1_SELECT__PERF_MODE0_MASK                                                              0xF0000000L
+//GE1_PERFCOUNTER1_SELECT1
+#define GE1_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GE1_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GE1_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GE1_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GE1_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GE1_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GE1_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GE1_PERFCOUNTER2_SELECT
+#define GE1_PERFCOUNTER2_SELECT__PERF_SEL0__SHIFT                                                             0x0
+#define GE1_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GE1_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GE1_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GE1_PERFCOUNTER2_SELECT__PERF_MODE0__SHIFT                                                            0x1c
+#define GE1_PERFCOUNTER2_SELECT__PERF_SEL0_MASK                                                               0x000003FFL
+#define GE1_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GE1_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GE1_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GE1_PERFCOUNTER2_SELECT__PERF_MODE0_MASK                                                              0xF0000000L
+//GE1_PERFCOUNTER2_SELECT1
+#define GE1_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GE1_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GE1_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GE1_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GE1_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GE1_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GE1_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GE1_PERFCOUNTER3_SELECT
+#define GE1_PERFCOUNTER3_SELECT__PERF_SEL0__SHIFT                                                             0x0
+#define GE1_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GE1_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GE1_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GE1_PERFCOUNTER3_SELECT__PERF_MODE0__SHIFT                                                            0x1c
+#define GE1_PERFCOUNTER3_SELECT__PERF_SEL0_MASK                                                               0x000003FFL
+#define GE1_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GE1_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GE1_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GE1_PERFCOUNTER3_SELECT__PERF_MODE0_MASK                                                              0xF0000000L
+//GE1_PERFCOUNTER3_SELECT1
+#define GE1_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GE1_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GE1_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GE1_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GE1_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GE1_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GE1_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GE1_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GE2_DIST_PERFCOUNTER0_SELECT
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_SEL0__SHIFT                                                        0x0
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GE2_DIST_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_MODE0__SHIFT                                                       0x1c
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_SEL0_MASK                                                          0x000003FFL
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GE2_DIST_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GE2_DIST_PERFCOUNTER0_SELECT__PERF_MODE0_MASK                                                         0xF0000000L
+//GE2_DIST_PERFCOUNTER0_SELECT1
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GE2_DIST_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GE2_DIST_PERFCOUNTER1_SELECT
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_SEL0__SHIFT                                                        0x0
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GE2_DIST_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_MODE0__SHIFT                                                       0x1c
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_SEL0_MASK                                                          0x000003FFL
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GE2_DIST_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GE2_DIST_PERFCOUNTER1_SELECT__PERF_MODE0_MASK                                                         0xF0000000L
+//GE2_DIST_PERFCOUNTER1_SELECT1
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GE2_DIST_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GE2_DIST_PERFCOUNTER2_SELECT
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_SEL0__SHIFT                                                        0x0
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GE2_DIST_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_MODE0__SHIFT                                                       0x1c
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_SEL0_MASK                                                          0x000003FFL
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GE2_DIST_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GE2_DIST_PERFCOUNTER2_SELECT__PERF_MODE0_MASK                                                         0xF0000000L
+//GE2_DIST_PERFCOUNTER2_SELECT1
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GE2_DIST_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GE2_DIST_PERFCOUNTER3_SELECT
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_SEL0__SHIFT                                                        0x0
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GE2_DIST_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_MODE0__SHIFT                                                       0x1c
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_SEL0_MASK                                                          0x000003FFL
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GE2_DIST_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GE2_DIST_PERFCOUNTER3_SELECT__PERF_MODE0_MASK                                                         0xF0000000L
+//GE2_DIST_PERFCOUNTER3_SELECT1
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GE2_DIST_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GE2_SE_PERFCOUNTER0_SELECT
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_SEL0__SHIFT                                                          0x0
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GE2_SE_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_MODE0__SHIFT                                                         0x1c
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_SEL0_MASK                                                            0x000003FFL
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GE2_SE_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GE2_SE_PERFCOUNTER0_SELECT__PERF_MODE0_MASK                                                           0xF0000000L
+//GE2_SE_PERFCOUNTER0_SELECT1
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GE2_SE_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GE2_SE_PERFCOUNTER1_SELECT
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_SEL0__SHIFT                                                          0x0
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GE2_SE_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_MODE0__SHIFT                                                         0x1c
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_SEL0_MASK                                                            0x000003FFL
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GE2_SE_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GE2_SE_PERFCOUNTER1_SELECT__PERF_MODE0_MASK                                                           0xF0000000L
+//GE2_SE_PERFCOUNTER1_SELECT1
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GE2_SE_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GE2_SE_PERFCOUNTER2_SELECT
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_SEL0__SHIFT                                                          0x0
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GE2_SE_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_MODE0__SHIFT                                                         0x1c
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_SEL0_MASK                                                            0x000003FFL
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GE2_SE_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GE2_SE_PERFCOUNTER2_SELECT__PERF_MODE0_MASK                                                           0xF0000000L
+//GE2_SE_PERFCOUNTER2_SELECT1
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GE2_SE_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//GE2_SE_PERFCOUNTER3_SELECT
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_SEL0__SHIFT                                                          0x0
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                          0xa
+#define GE2_SE_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                          0x14
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                         0x18
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_MODE0__SHIFT                                                         0x1c
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_SEL0_MASK                                                            0x000003FFL
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                            0x000FFC00L
+#define GE2_SE_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                            0x00F00000L
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                           0x0F000000L
+#define GE2_SE_PERFCOUNTER3_SELECT__PERF_MODE0_MASK                                                           0xF0000000L
+//GE2_SE_PERFCOUNTER3_SELECT1
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                         0x0
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                         0xa
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                        0x18
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                        0x1c
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                           0x000003FFL
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                           0x000FFC00L
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                          0x0F000000L
+#define GE2_SE_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                          0xF0000000L
+//PA_SU_PERFCOUNTER0_SELECT
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SU_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SU_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER0_SELECT1
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SU_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SU_PERFCOUNTER1_SELECT
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SU_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SU_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER1_SELECT1
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SU_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SU_PERFCOUNTER2_SELECT
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SU_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SU_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER2_SELECT1
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SU_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SU_PERFCOUNTER3_SELECT
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SU_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SU_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SU_PERFCOUNTER3_SELECT1
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SU_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SC_PERFCOUNTER0_SELECT
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_SC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_SC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_SC_PERFCOUNTER0_SELECT1
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_SC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_SC_PERFCOUNTER1_SELECT
+#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER2_SELECT
+#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER3_SELECT
+#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER4_SELECT
+#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER5_SELECT
+#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER6_SELECT
+#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_SC_PERFCOUNTER7_SELECT
+#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_SC_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//SPI_PERFCOUNTER0_SELECT
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER1_SELECT
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER2_SELECT
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER3_SELECT
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define SPI_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define SPI_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define SPI_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SPI_PERFCOUNTER0_SELECT1
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER1_SELECT1
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER2_SELECT1
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER3_SELECT1
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define SPI_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define SPI_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//SPI_PERFCOUNTER4_SELECT
+#define SPI_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+//SPI_PERFCOUNTER5_SELECT
+#define SPI_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SPI_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+//SPI_PERFCOUNTER_BINS
+#define SPI_PERFCOUNTER_BINS__BIN0_MIN__SHIFT                                                                 0x0
+#define SPI_PERFCOUNTER_BINS__BIN0_MAX__SHIFT                                                                 0x4
+#define SPI_PERFCOUNTER_BINS__BIN1_MIN__SHIFT                                                                 0x8
+#define SPI_PERFCOUNTER_BINS__BIN1_MAX__SHIFT                                                                 0xc
+#define SPI_PERFCOUNTER_BINS__BIN2_MIN__SHIFT                                                                 0x10
+#define SPI_PERFCOUNTER_BINS__BIN2_MAX__SHIFT                                                                 0x14
+#define SPI_PERFCOUNTER_BINS__BIN3_MIN__SHIFT                                                                 0x18
+#define SPI_PERFCOUNTER_BINS__BIN3_MAX__SHIFT                                                                 0x1c
+#define SPI_PERFCOUNTER_BINS__BIN0_MIN_MASK                                                                   0x0000000FL
+#define SPI_PERFCOUNTER_BINS__BIN0_MAX_MASK                                                                   0x000000F0L
+#define SPI_PERFCOUNTER_BINS__BIN1_MIN_MASK                                                                   0x00000F00L
+#define SPI_PERFCOUNTER_BINS__BIN1_MAX_MASK                                                                   0x0000F000L
+#define SPI_PERFCOUNTER_BINS__BIN2_MIN_MASK                                                                   0x000F0000L
+#define SPI_PERFCOUNTER_BINS__BIN2_MAX_MASK                                                                   0x00F00000L
+#define SPI_PERFCOUNTER_BINS__BIN3_MIN_MASK                                                                   0x0F000000L
+#define SPI_PERFCOUNTER_BINS__BIN3_MAX_MASK                                                                   0xF0000000L
+//PC_PERFCOUNTER0_SELECT
+#define PC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define PC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define PC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define PC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define PC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define PC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define PC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define PC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define PC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define PC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//PC_PERFCOUNTER1_SELECT
+#define PC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define PC_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define PC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define PC_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define PC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define PC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define PC_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define PC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define PC_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define PC_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//PC_PERFCOUNTER2_SELECT
+#define PC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define PC_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define PC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define PC_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define PC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define PC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define PC_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define PC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define PC_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define PC_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//PC_PERFCOUNTER3_SELECT
+#define PC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define PC_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define PC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define PC_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define PC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define PC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define PC_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define PC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define PC_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define PC_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//PC_PERFCOUNTER0_SELECT1
+#define PC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define PC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define PC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define PC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define PC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define PC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define PC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define PC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//PC_PERFCOUNTER1_SELECT1
+#define PC_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define PC_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define PC_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define PC_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define PC_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define PC_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define PC_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define PC_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//PC_PERFCOUNTER2_SELECT1
+#define PC_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define PC_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define PC_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define PC_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define PC_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define PC_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define PC_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define PC_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//PC_PERFCOUNTER3_SELECT1
+#define PC_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define PC_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define PC_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define PC_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define PC_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define PC_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define PC_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define PC_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//SQ_PERFCOUNTER0_SELECT
+#define SQ_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER1_SELECT
+#define SQ_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER2_SELECT
+#define SQ_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER2_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER3_SELECT
+#define SQ_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER3_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER4_SELECT
+#define SQ_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER4_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER4_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER5_SELECT
+#define SQ_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER5_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER5_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER6_SELECT
+#define SQ_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER6_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER6_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER6_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER6_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER7_SELECT
+#define SQ_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER7_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER7_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER7_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER7_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER8_SELECT
+#define SQ_PERFCOUNTER8_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER8_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER8_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER8_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER8_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER8_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER9_SELECT
+#define SQ_PERFCOUNTER9_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SQ_PERFCOUNTER9_SELECT__SPM_MODE__SHIFT                                                               0x14
+#define SQ_PERFCOUNTER9_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SQ_PERFCOUNTER9_SELECT__PERF_SEL_MASK                                                                 0x000001FFL
+#define SQ_PERFCOUNTER9_SELECT__SPM_MODE_MASK                                                                 0x00F00000L
+#define SQ_PERFCOUNTER9_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SQ_PERFCOUNTER10_SELECT
+#define SQ_PERFCOUNTER10_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER10_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER10_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER10_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER10_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER10_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER11_SELECT
+#define SQ_PERFCOUNTER11_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER11_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER11_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER11_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER11_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER11_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER12_SELECT
+#define SQ_PERFCOUNTER12_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER12_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER12_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER12_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER12_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER12_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER13_SELECT
+#define SQ_PERFCOUNTER13_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER13_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER13_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER13_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER13_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER13_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER14_SELECT
+#define SQ_PERFCOUNTER14_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER14_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER14_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER14_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER14_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER14_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQ_PERFCOUNTER15_SELECT
+#define SQ_PERFCOUNTER15_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQ_PERFCOUNTER15_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQ_PERFCOUNTER15_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQ_PERFCOUNTER15_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQ_PERFCOUNTER15_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQ_PERFCOUNTER15_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER0_SELECT
+#define SQG_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER0_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER0_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER1_SELECT
+#define SQG_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER1_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER1_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER2_SELECT
+#define SQG_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER2_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER2_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER3_SELECT
+#define SQG_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER3_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER3_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER4_SELECT
+#define SQG_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER4_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER4_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER4_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER4_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER5_SELECT
+#define SQG_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER5_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER5_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER5_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER5_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER6_SELECT
+#define SQG_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER6_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER6_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER6_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER6_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER7_SELECT
+#define SQG_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define SQG_PERFCOUNTER7_SELECT__SPM_MODE__SHIFT                                                              0x14
+#define SQG_PERFCOUNTER7_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define SQG_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                                0x000001FFL
+#define SQG_PERFCOUNTER7_SELECT__SPM_MODE_MASK                                                                0x00F00000L
+#define SQG_PERFCOUNTER7_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//SQG_PERFCOUNTER_CTRL
+#define SQG_PERFCOUNTER_CTRL__PS_EN__SHIFT                                                                    0x0
+#define SQG_PERFCOUNTER_CTRL__GS_EN__SHIFT                                                                    0x2
+#define SQG_PERFCOUNTER_CTRL__HS_EN__SHIFT                                                                    0x4
+#define SQG_PERFCOUNTER_CTRL__CS_EN__SHIFT                                                                    0x6
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME0PIPE0_PERF__SHIFT                                                    0xe
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME0PIPE1_PERF__SHIFT                                                    0xf
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE0_PERF__SHIFT                                                    0x10
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE1_PERF__SHIFT                                                    0x11
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE2_PERF__SHIFT                                                    0x12
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE3_PERF__SHIFT                                                    0x13
+#define SQG_PERFCOUNTER_CTRL__PS_EN_MASK                                                                      0x00000001L
+#define SQG_PERFCOUNTER_CTRL__GS_EN_MASK                                                                      0x00000004L
+#define SQG_PERFCOUNTER_CTRL__HS_EN_MASK                                                                      0x00000010L
+#define SQG_PERFCOUNTER_CTRL__CS_EN_MASK                                                                      0x00000040L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME0PIPE0_PERF_MASK                                                      0x00004000L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME0PIPE1_PERF_MASK                                                      0x00008000L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE0_PERF_MASK                                                      0x00010000L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE1_PERF_MASK                                                      0x00020000L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE2_PERF_MASK                                                      0x00040000L
+#define SQG_PERFCOUNTER_CTRL__DISABLE_ME1PIPE3_PERF_MASK                                                      0x00080000L
+//SQG_PERFCOUNTER_CTRL2
+#define SQG_PERFCOUNTER_CTRL2__FORCE_EN__SHIFT                                                                0x0
+#define SQG_PERFCOUNTER_CTRL2__VMID_EN__SHIFT                                                                 0x1
+#define SQG_PERFCOUNTER_CTRL2__FORCE_EN_MASK                                                                  0x00000001L
+#define SQG_PERFCOUNTER_CTRL2__VMID_EN_MASK                                                                   0x0001FFFEL
+//SQG_PERF_SAMPLE_FINISH
+#define SQG_PERF_SAMPLE_FINISH__STATUS__SHIFT                                                                 0x0
+#define SQG_PERF_SAMPLE_FINISH__STATUS_MASK                                                                   0x0000007FL
+//SQ_PERFCOUNTER_CTRL
+#define SQ_PERFCOUNTER_CTRL__PS_EN__SHIFT                                                                     0x0
+#define SQ_PERFCOUNTER_CTRL__GS_EN__SHIFT                                                                     0x2
+#define SQ_PERFCOUNTER_CTRL__HS_EN__SHIFT                                                                     0x4
+#define SQ_PERFCOUNTER_CTRL__CS_EN__SHIFT                                                                     0x6
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME0PIPE0_PERF__SHIFT                                                     0xe
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME0PIPE1_PERF__SHIFT                                                     0xf
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE0_PERF__SHIFT                                                     0x10
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE1_PERF__SHIFT                                                     0x11
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE2_PERF__SHIFT                                                     0x12
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE3_PERF__SHIFT                                                     0x13
+#define SQ_PERFCOUNTER_CTRL__PS_EN_MASK                                                                       0x00000001L
+#define SQ_PERFCOUNTER_CTRL__GS_EN_MASK                                                                       0x00000004L
+#define SQ_PERFCOUNTER_CTRL__HS_EN_MASK                                                                       0x00000010L
+#define SQ_PERFCOUNTER_CTRL__CS_EN_MASK                                                                       0x00000040L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME0PIPE0_PERF_MASK                                                       0x00004000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME0PIPE1_PERF_MASK                                                       0x00008000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE0_PERF_MASK                                                       0x00010000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE1_PERF_MASK                                                       0x00020000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE2_PERF_MASK                                                       0x00040000L
+#define SQ_PERFCOUNTER_CTRL__DISABLE_ME1PIPE3_PERF_MASK                                                       0x00080000L
+//SQ_PERFCOUNTER_CTRL2
+#define SQ_PERFCOUNTER_CTRL2__FORCE_EN__SHIFT                                                                 0x0
+#define SQ_PERFCOUNTER_CTRL2__VMID_EN__SHIFT                                                                  0x1
+#define SQ_PERFCOUNTER_CTRL2__FORCE_EN_MASK                                                                   0x00000001L
+#define SQ_PERFCOUNTER_CTRL2__VMID_EN_MASK                                                                    0x0001FFFEL
+//SQ_THREAD_TRACE_BUF0_BASE
+#define SQ_THREAD_TRACE_BUF0_BASE__BASE_LO__SHIFT                                                             0x0
+#define SQ_THREAD_TRACE_BUF0_BASE__BASE_LO_MASK                                                               0xFFFFFFFFL
+//SQ_THREAD_TRACE_BUF0_SIZE
+#define SQ_THREAD_TRACE_BUF0_SIZE__BASE_HI__SHIFT                                                             0x0
+#define SQ_THREAD_TRACE_BUF0_SIZE__SIZE__SHIFT                                                                0x8
+#define SQ_THREAD_TRACE_BUF0_SIZE__BASE_HI_MASK                                                               0x0000000FL
+#define SQ_THREAD_TRACE_BUF0_SIZE__SIZE_MASK                                                                  0x3FFFFF00L
+//SQ_THREAD_TRACE_BUF1_BASE
+#define SQ_THREAD_TRACE_BUF1_BASE__BASE_LO__SHIFT                                                             0x0
+#define SQ_THREAD_TRACE_BUF1_BASE__BASE_LO_MASK                                                               0xFFFFFFFFL
+//SQ_THREAD_TRACE_BUF1_SIZE
+#define SQ_THREAD_TRACE_BUF1_SIZE__BASE_HI__SHIFT                                                             0x0
+#define SQ_THREAD_TRACE_BUF1_SIZE__SIZE__SHIFT                                                                0x8
+#define SQ_THREAD_TRACE_BUF1_SIZE__BASE_HI_MASK                                                               0x0000000FL
+#define SQ_THREAD_TRACE_BUF1_SIZE__SIZE_MASK                                                                  0x3FFFFF00L
+//SQ_THREAD_TRACE_CTRL
+#define SQ_THREAD_TRACE_CTRL__MODE__SHIFT                                                                     0x0
+#define SQ_THREAD_TRACE_CTRL__ALL_VMID__SHIFT                                                                 0x2
+#define SQ_THREAD_TRACE_CTRL__GL1_PERF_EN__SHIFT                                                              0x3
+#define SQ_THREAD_TRACE_CTRL__INTERRUPT_EN__SHIFT                                                             0x4
+#define SQ_THREAD_TRACE_CTRL__DOUBLE_BUFFER__SHIFT                                                            0x5
+#define SQ_THREAD_TRACE_CTRL__HIWATER__SHIFT                                                                  0x6
+#define SQ_THREAD_TRACE_CTRL__REG_AT_HWM__SHIFT                                                               0x9
+#define SQ_THREAD_TRACE_CTRL__SPI_STALL_EN__SHIFT                                                             0xb
+#define SQ_THREAD_TRACE_CTRL__SQ_STALL_EN__SHIFT                                                              0xc
+#define SQ_THREAD_TRACE_CTRL__UTIL_TIMER__SHIFT                                                               0xd
+#define SQ_THREAD_TRACE_CTRL__WAVESTART_MODE__SHIFT                                                           0xe
+#define SQ_THREAD_TRACE_CTRL__RT_FREQ__SHIFT                                                                  0x10
+#define SQ_THREAD_TRACE_CTRL__SYNC_COUNT_MARKERS__SHIFT                                                       0x12
+#define SQ_THREAD_TRACE_CTRL__SYNC_COUNT_DRAWS__SHIFT                                                         0x13
+#define SQ_THREAD_TRACE_CTRL__LOWATER_OFFSET__SHIFT                                                           0x14
+#define SQ_THREAD_TRACE_CTRL__AUTO_FLUSH_PADDING_DIS__SHIFT                                                   0x1c
+#define SQ_THREAD_TRACE_CTRL__AUTO_FLUSH_MODE__SHIFT                                                          0x1d
+#define SQ_THREAD_TRACE_CTRL__DRAW_EVENT_EN__SHIFT                                                            0x1f
+#define SQ_THREAD_TRACE_CTRL__MODE_MASK                                                                       0x00000003L
+#define SQ_THREAD_TRACE_CTRL__ALL_VMID_MASK                                                                   0x00000004L
+#define SQ_THREAD_TRACE_CTRL__GL1_PERF_EN_MASK                                                                0x00000008L
+#define SQ_THREAD_TRACE_CTRL__INTERRUPT_EN_MASK                                                               0x00000010L
+#define SQ_THREAD_TRACE_CTRL__DOUBLE_BUFFER_MASK                                                              0x00000020L
+#define SQ_THREAD_TRACE_CTRL__HIWATER_MASK                                                                    0x000001C0L
+#define SQ_THREAD_TRACE_CTRL__REG_AT_HWM_MASK                                                                 0x00000600L
+#define SQ_THREAD_TRACE_CTRL__SPI_STALL_EN_MASK                                                               0x00000800L
+#define SQ_THREAD_TRACE_CTRL__SQ_STALL_EN_MASK                                                                0x00001000L
+#define SQ_THREAD_TRACE_CTRL__UTIL_TIMER_MASK                                                                 0x00002000L
+#define SQ_THREAD_TRACE_CTRL__WAVESTART_MODE_MASK                                                             0x0000C000L
+#define SQ_THREAD_TRACE_CTRL__RT_FREQ_MASK                                                                    0x00030000L
+#define SQ_THREAD_TRACE_CTRL__SYNC_COUNT_MARKERS_MASK                                                         0x00040000L
+#define SQ_THREAD_TRACE_CTRL__SYNC_COUNT_DRAWS_MASK                                                           0x00080000L
+#define SQ_THREAD_TRACE_CTRL__LOWATER_OFFSET_MASK                                                             0x00700000L
+#define SQ_THREAD_TRACE_CTRL__AUTO_FLUSH_PADDING_DIS_MASK                                                     0x10000000L
+#define SQ_THREAD_TRACE_CTRL__AUTO_FLUSH_MODE_MASK                                                            0x20000000L
+#define SQ_THREAD_TRACE_CTRL__DRAW_EVENT_EN_MASK                                                              0x80000000L
+//SQ_THREAD_TRACE_MASK
+#define SQ_THREAD_TRACE_MASK__SIMD_SEL__SHIFT                                                                 0x0
+#define SQ_THREAD_TRACE_MASK__WGP_SEL__SHIFT                                                                  0x4
+#define SQ_THREAD_TRACE_MASK__SA_SEL__SHIFT                                                                   0x9
+#define SQ_THREAD_TRACE_MASK__WTYPE_INCLUDE__SHIFT                                                            0xa
+#define SQ_THREAD_TRACE_MASK__EXCLUDE_NONDETAIL_SHADERDATA__SHIFT                                             0x11
+#define SQ_THREAD_TRACE_MASK__SIMD_SEL_MASK                                                                   0x00000003L
+#define SQ_THREAD_TRACE_MASK__WGP_SEL_MASK                                                                    0x000000F0L
+#define SQ_THREAD_TRACE_MASK__SA_SEL_MASK                                                                     0x00000200L
+#define SQ_THREAD_TRACE_MASK__WTYPE_INCLUDE_MASK                                                              0x0001FC00L
+#define SQ_THREAD_TRACE_MASK__EXCLUDE_NONDETAIL_SHADERDATA_MASK                                               0x00020000L
+//SQ_THREAD_TRACE_TOKEN_MASK
+#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_EXCLUDE__SHIFT                                                      0x0
+#define SQ_THREAD_TRACE_TOKEN_MASK__TTRACE_EXEC__SHIFT                                                        0xb
+#define SQ_THREAD_TRACE_TOKEN_MASK__BOP_EVENTS_TOKEN_INCLUDE__SHIFT                                           0xc
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_INCLUDE__SHIFT                                                        0x10
+#define SQ_THREAD_TRACE_TOKEN_MASK__INST_EXCLUDE__SHIFT                                                       0x18
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_EXCLUDE__SHIFT                                                        0x1a
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DETAIL_ALL__SHIFT                                                     0x1f
+#define SQ_THREAD_TRACE_TOKEN_MASK__TOKEN_EXCLUDE_MASK                                                        0x000007FFL
+#define SQ_THREAD_TRACE_TOKEN_MASK__TTRACE_EXEC_MASK                                                          0x00000800L
+#define SQ_THREAD_TRACE_TOKEN_MASK__BOP_EVENTS_TOKEN_INCLUDE_MASK                                             0x00001000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_INCLUDE_MASK                                                          0x00FF0000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__INST_EXCLUDE_MASK                                                         0x03000000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_EXCLUDE_MASK                                                          0x1C000000L
+#define SQ_THREAD_TRACE_TOKEN_MASK__REG_DETAIL_ALL_MASK                                                       0x80000000L
+//SQ_THREAD_TRACE_WPTR
+#define SQ_THREAD_TRACE_WPTR__OFFSET__SHIFT                                                                   0x0
+#define SQ_THREAD_TRACE_WPTR__BUFFER_ID__SHIFT                                                                0x1f
+#define SQ_THREAD_TRACE_WPTR__OFFSET_MASK                                                                     0x1FFFFFFFL
+#define SQ_THREAD_TRACE_WPTR__BUFFER_ID_MASK                                                                  0x80000000L
+//SQ_THREAD_TRACE_STATUS
+#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING__SHIFT                                                         0x0
+#define SQ_THREAD_TRACE_STATUS__FINISH_DONE__SHIFT                                                            0xc
+#define SQ_THREAD_TRACE_STATUS__WRITE_ERROR__SHIFT                                                            0x18
+#define SQ_THREAD_TRACE_STATUS__BUSY__SHIFT                                                                   0x19
+#define SQ_THREAD_TRACE_STATUS__OWNER_VMID__SHIFT                                                             0x1c
+#define SQ_THREAD_TRACE_STATUS__FINISH_PENDING_MASK                                                           0x00000FFFL
+#define SQ_THREAD_TRACE_STATUS__FINISH_DONE_MASK                                                              0x00FFF000L
+#define SQ_THREAD_TRACE_STATUS__WRITE_ERROR_MASK                                                              0x01000000L
+#define SQ_THREAD_TRACE_STATUS__BUSY_MASK                                                                     0x02000000L
+#define SQ_THREAD_TRACE_STATUS__OWNER_VMID_MASK                                                               0xF0000000L
+//SQ_THREAD_TRACE_STATUS2
+#define SQ_THREAD_TRACE_STATUS2__BUF0_FULL__SHIFT                                                             0x0
+#define SQ_THREAD_TRACE_STATUS2__BUF1_FULL__SHIFT                                                             0x1
+#define SQ_THREAD_TRACE_STATUS2__PACKET_LOST_BUF_NO_LOCKDOWN__SHIFT                                           0x4
+#define SQ_THREAD_TRACE_STATUS2__BUF_ISSUE_STATUS__SHIFT                                                      0x8
+#define SQ_THREAD_TRACE_STATUS2__BUF_ISSUE__SHIFT                                                             0xd
+#define SQ_THREAD_TRACE_STATUS2__WRITE_BUF_FULL__SHIFT                                                        0xe
+#define SQ_THREAD_TRACE_STATUS2__BUF0_FULL_MASK                                                               0x00000001L
+#define SQ_THREAD_TRACE_STATUS2__BUF1_FULL_MASK                                                               0x00000002L
+#define SQ_THREAD_TRACE_STATUS2__PACKET_LOST_BUF_NO_LOCKDOWN_MASK                                             0x00000010L
+#define SQ_THREAD_TRACE_STATUS2__BUF_ISSUE_STATUS_MASK                                                        0x00001F00L
+#define SQ_THREAD_TRACE_STATUS2__BUF_ISSUE_MASK                                                               0x00002000L
+#define SQ_THREAD_TRACE_STATUS2__WRITE_BUF_FULL_MASK                                                          0x00004000L
+//SQ_THREAD_TRACE_GFX_DRAW_CNTR
+#define SQ_THREAD_TRACE_GFX_DRAW_CNTR__CNTR__SHIFT                                                            0x0
+#define SQ_THREAD_TRACE_GFX_DRAW_CNTR__CNTR_MASK                                                              0xFFFFFFFFL
+//SQ_THREAD_TRACE_GFX_MARKER_CNTR
+#define SQ_THREAD_TRACE_GFX_MARKER_CNTR__CNTR__SHIFT                                                          0x0
+#define SQ_THREAD_TRACE_GFX_MARKER_CNTR__CNTR_MASK                                                            0xFFFFFFFFL
+//SQ_THREAD_TRACE_HP3D_DRAW_CNTR
+#define SQ_THREAD_TRACE_HP3D_DRAW_CNTR__CNTR__SHIFT                                                           0x0
+#define SQ_THREAD_TRACE_HP3D_DRAW_CNTR__CNTR_MASK                                                             0xFFFFFFFFL
+//SQ_THREAD_TRACE_HP3D_MARKER_CNTR
+#define SQ_THREAD_TRACE_HP3D_MARKER_CNTR__CNTR__SHIFT                                                         0x0
+#define SQ_THREAD_TRACE_HP3D_MARKER_CNTR__CNTR_MASK                                                           0xFFFFFFFFL
+//SQ_THREAD_TRACE_DROPPED_CNTR
+#define SQ_THREAD_TRACE_DROPPED_CNTR__CNTR__SHIFT                                                             0x0
+#define SQ_THREAD_TRACE_DROPPED_CNTR__CNTR_MASK                                                               0xFFFFFFFFL
+//GCEA_PERFCOUNTER2_SELECT
+#define GCEA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GCEA_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GCEA_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GCEA_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GCEA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GCEA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GCEA_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GCEA_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GCEA_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GCEA_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GCEA_PERFCOUNTER2_SELECT1
+#define GCEA_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GCEA_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GCEA_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GCEA_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GCEA_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GCEA_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GCEA_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GCEA_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GCEA_PERFCOUNTER2_MODE
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_MODE0__SHIFT                                                          0x0
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_MODE1__SHIFT                                                          0x2
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_MODE2__SHIFT                                                          0x4
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_MODE3__SHIFT                                                          0x6
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_VALUE0__SHIFT                                                         0x8
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_VALUE1__SHIFT                                                         0xc
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_VALUE2__SHIFT                                                         0x10
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_VALUE3__SHIFT                                                         0x14
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_MODE0_MASK                                                            0x00000003L
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_MODE1_MASK                                                            0x0000000CL
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_MODE2_MASK                                                            0x00000030L
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_MODE3_MASK                                                            0x000000C0L
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_VALUE0_MASK                                                           0x00000F00L
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_VALUE1_MASK                                                           0x0000F000L
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_VALUE2_MASK                                                           0x000F0000L
+#define GCEA_PERFCOUNTER2_MODE__COMPARE_VALUE3_MASK                                                           0x00F00000L
+//GCEA_PERFCOUNTER0_CFG
+#define GCEA_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                                0x0
+#define GCEA_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                            0x8
+#define GCEA_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                               0x18
+#define GCEA_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                  0x1c
+#define GCEA_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                   0x1d
+#define GCEA_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                  0x000000FFL
+#define GCEA_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                              0x0000FF00L
+#define GCEA_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                 0x0F000000L
+#define GCEA_PERFCOUNTER0_CFG__ENABLE_MASK                                                                    0x10000000L
+#define GCEA_PERFCOUNTER0_CFG__CLEAR_MASK                                                                     0x20000000L
+//GCEA_PERFCOUNTER1_CFG
+#define GCEA_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                                0x0
+#define GCEA_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                            0x8
+#define GCEA_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                               0x18
+#define GCEA_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                  0x1c
+#define GCEA_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                   0x1d
+#define GCEA_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                  0x000000FFL
+#define GCEA_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                              0x0000FF00L
+#define GCEA_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                 0x0F000000L
+#define GCEA_PERFCOUNTER1_CFG__ENABLE_MASK                                                                    0x10000000L
+#define GCEA_PERFCOUNTER1_CFG__CLEAR_MASK                                                                     0x20000000L
+//GCEA_PERFCOUNTER_RSLT_CNTL
+#define GCEA_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                                0x0
+#define GCEA_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                      0x8
+#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                       0x10
+#define GCEA_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                         0x18
+#define GCEA_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                          0x19
+#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                               0x1a
+#define GCEA_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                  0x0000000FL
+#define GCEA_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                        0x0000FF00L
+#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                         0x00FF0000L
+#define GCEA_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                           0x01000000L
+#define GCEA_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                            0x02000000L
+#define GCEA_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                 0x04000000L
+//SX_PERFCOUNTER0_SELECT
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define SX_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define SX_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define SX_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER1_SELECT
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define SX_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define SX_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define SX_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER2_SELECT
+#define SX_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER3_SELECT
+#define SX_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define SX_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define SX_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define SX_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define SX_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define SX_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//SX_PERFCOUNTER0_SELECT1
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define SX_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define SX_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//SX_PERFCOUNTER1_SELECT1
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define SX_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define SX_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//GDS_PERFCOUNTER0_SELECT
+#define GDS_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GDS_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GDS_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GDS_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GDS_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GDS_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GDS_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GDS_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GDS_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GDS_PERFCOUNTER1_SELECT
+#define GDS_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GDS_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GDS_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GDS_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GDS_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GDS_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GDS_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GDS_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GDS_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GDS_PERFCOUNTER2_SELECT
+#define GDS_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GDS_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GDS_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GDS_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GDS_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GDS_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GDS_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GDS_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GDS_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GDS_PERFCOUNTER3_SELECT
+#define GDS_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GDS_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GDS_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GDS_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GDS_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GDS_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GDS_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GDS_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GDS_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GDS_PERFCOUNTER0_SELECT1
+#define GDS_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GDS_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GDS_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GDS_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GDS_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GDS_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GDS_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GDS_PERFCOUNTER1_SELECT1
+#define GDS_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GDS_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GDS_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GDS_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GDS_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GDS_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GDS_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GDS_PERFCOUNTER2_SELECT1
+#define GDS_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GDS_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GDS_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GDS_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GDS_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GDS_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GDS_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GDS_PERFCOUNTER3_SELECT1
+#define GDS_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GDS_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GDS_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GDS_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GDS_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GDS_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GDS_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GDS_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//TA_PERFCOUNTER0_SELECT
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define TA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define TA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define TA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define TA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TA_PERFCOUNTER0_SELECT1
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define TA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define TA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//TA_PERFCOUNTER1_SELECT
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define TA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TD_PERFCOUNTER0_SELECT
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define TD_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define TD_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define TD_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define TD_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TD_PERFCOUNTER0_SELECT1
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define TD_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define TD_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//TD_PERFCOUNTER1_SELECT
+#define TD_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define TD_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define TD_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define TD_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define TD_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define TD_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//TCP_PERFCOUNTER0_SELECT
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define TCP_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCP_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER0_SELECT1
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define TCP_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define TCP_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//TCP_PERFCOUNTER1_SELECT
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define TCP_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define TCP_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER1_SELECT1
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define TCP_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define TCP_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//TCP_PERFCOUNTER2_SELECT
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//TCP_PERFCOUNTER3_SELECT
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define TCP_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define TCP_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define TCP_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GL2C_PERFCOUNTER0_SELECT
+#define GL2C_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2C_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2C_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2C_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2C_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2C_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2C_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2C_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2C_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2C_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2C_PERFCOUNTER0_SELECT1
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                          0x18
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                          0x1c
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                            0x0F000000L
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                            0xF0000000L
+//GL2C_PERFCOUNTER1_SELECT
+#define GL2C_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2C_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2C_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2C_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2C_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2C_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2C_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2C_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2C_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2C_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2C_PERFCOUNTER1_SELECT1
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                          0x18
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                          0x1c
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                            0x0F000000L
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                            0xF0000000L
+//GL2C_PERFCOUNTER2_SELECT
+#define GL2C_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2C_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2C_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2C_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2C_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2C_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2C_PERFCOUNTER3_SELECT
+#define GL2C_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2C_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2C_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2C_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2C_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2C_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2A_PERFCOUNTER0_SELECT
+#define GL2A_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2A_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2A_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2A_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2A_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2A_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2A_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2A_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2A_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2A_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2A_PERFCOUNTER0_SELECT1
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                          0x18
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                          0x1c
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                            0x0F000000L
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                            0xF0000000L
+//GL2A_PERFCOUNTER1_SELECT
+#define GL2A_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2A_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2A_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2A_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2A_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2A_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2A_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2A_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2A_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2A_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2A_PERFCOUNTER1_SELECT1
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                          0x18
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                          0x1c
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                            0x0F000000L
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                            0xF0000000L
+//GL2A_PERFCOUNTER2_SELECT
+#define GL2A_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2A_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2A_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2A_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2A_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2A_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2A_PERFCOUNTER3_SELECT
+#define GL2A_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2A_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2A_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2A_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2A_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2A_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1C_PERFCOUNTER0_SELECT
+#define GL1C_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1C_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1C_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1C_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1C_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1C_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1C_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1C_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1C_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1C_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1C_PERFCOUNTER0_SELECT1
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                          0x18
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                          0x1c
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                            0x0F000000L
+#define GL1C_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                            0xF0000000L
+//GL1C_PERFCOUNTER1_SELECT
+#define GL1C_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1C_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1C_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1C_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1C_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1C_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1C_PERFCOUNTER2_SELECT
+#define GL1C_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1C_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1C_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1C_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1C_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1C_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1C_PERFCOUNTER3_SELECT
+#define GL1C_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1C_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1C_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1C_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1C_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1C_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//CHC_PERFCOUNTER0_SELECT
+#define CHC_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHC_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CHC_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHC_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define CHC_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHC_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHC_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CHC_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHC_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define CHC_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHC_PERFCOUNTER0_SELECT1
+#define CHC_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CHC_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CHC_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x18
+#define CHC_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x1c
+#define CHC_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CHC_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CHC_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0x0F000000L
+#define CHC_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0xF0000000L
+//CHC_PERFCOUNTER1_SELECT
+#define CHC_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHC_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHC_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHC_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHC_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHC_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHC_PERFCOUNTER2_SELECT
+#define CHC_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHC_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHC_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHC_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHC_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHC_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHC_PERFCOUNTER3_SELECT
+#define CHC_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHC_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHC_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHC_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHC_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHC_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHCG_PERFCOUNTER0_SELECT
+#define CHCG_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define CHCG_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define CHCG_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define CHCG_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define CHCG_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define CHCG_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define CHCG_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define CHCG_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define CHCG_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define CHCG_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//CHCG_PERFCOUNTER0_SELECT1
+#define CHCG_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define CHCG_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define CHCG_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                          0x18
+#define CHCG_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                          0x1c
+#define CHCG_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define CHCG_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define CHCG_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                            0x0F000000L
+#define CHCG_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                            0xF0000000L
+//CHCG_PERFCOUNTER1_SELECT
+#define CHCG_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define CHCG_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define CHCG_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define CHCG_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define CHCG_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define CHCG_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//CHCG_PERFCOUNTER2_SELECT
+#define CHCG_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define CHCG_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define CHCG_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define CHCG_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define CHCG_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define CHCG_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//CHCG_PERFCOUNTER3_SELECT
+#define CHCG_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define CHCG_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define CHCG_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define CHCG_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define CHCG_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define CHCG_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//CB_PERFCOUNTER_FILTER
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE__SHIFT                                                        0x0
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL__SHIFT                                                           0x1
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE__SHIFT                                                    0x4
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL__SHIFT                                                       0x5
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE__SHIFT                                                     0xa
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL__SHIFT                                                        0xb
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE__SHIFT                                                       0xc
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL__SHIFT                                                          0xd
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE__SHIFT                                               0x11
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL__SHIFT                                                  0x12
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE__SHIFT                                             0x15
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL__SHIFT                                                0x16
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_ENABLE_MASK                                                          0x00000001L
+#define CB_PERFCOUNTER_FILTER__OP_FILTER_SEL_MASK                                                             0x0000000EL
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_ENABLE_MASK                                                      0x00000010L
+#define CB_PERFCOUNTER_FILTER__FORMAT_FILTER_SEL_MASK                                                         0x000003E0L
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_ENABLE_MASK                                                       0x00000400L
+#define CB_PERFCOUNTER_FILTER__CLEAR_FILTER_SEL_MASK                                                          0x00000800L
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_ENABLE_MASK                                                         0x00001000L
+#define CB_PERFCOUNTER_FILTER__MRT_FILTER_SEL_MASK                                                            0x0000E000L
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_ENABLE_MASK                                                 0x00020000L
+#define CB_PERFCOUNTER_FILTER__NUM_SAMPLES_FILTER_SEL_MASK                                                    0x001C0000L
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_ENABLE_MASK                                               0x00200000L
+#define CB_PERFCOUNTER_FILTER__NUM_FRAGMENTS_FILTER_SEL_MASK                                                  0x00C00000L
+//CB_PERFCOUNTER0_SELECT
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define CB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define CB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define CB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define CB_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//CB_PERFCOUNTER0_SELECT1
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define CB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define CB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//CB_PERFCOUNTER1_SELECT
+#define CB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define CB_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//CB_PERFCOUNTER2_SELECT
+#define CB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define CB_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//CB_PERFCOUNTER3_SELECT
+#define CB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define CB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define CB_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define CB_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER0_SELECT
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER0_SELECT1
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define DB_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define DB_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//DB_PERFCOUNTER1_SELECT
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER1_SELECT1
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                             0x0
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                             0xa
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                            0x18
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                            0x1c
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                               0x000003FFL
+#define DB_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                               0x000FFC00L
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                              0x0F000000L
+#define DB_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                              0xF0000000L
+//DB_PERFCOUNTER2_SELECT
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//DB_PERFCOUNTER3_SELECT
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                               0x0
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                              0xa
+#define DB_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                              0x14
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                             0x18
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                              0x1c
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                 0x000003FFL
+#define DB_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                                0x000FFC00L
+#define DB_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                                0x00F00000L
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                               0x0F000000L
+#define DB_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                                0xF0000000L
+//RLC_SPM_PERFMON_CNTL
+#define RLC_SPM_PERFMON_CNTL__RESERVED1__SHIFT                                                                0x0
+#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE__SHIFT                                                        0xc
+#define RLC_SPM_PERFMON_CNTL__DISABLE_GFXCLOCK_COUNT__SHIFT                                                   0xe
+#define RLC_SPM_PERFMON_CNTL__RESERVED__SHIFT                                                                 0xf
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL__SHIFT                                                  0x10
+#define RLC_SPM_PERFMON_CNTL__RESERVED1_MASK                                                                  0x00000FFFL
+#define RLC_SPM_PERFMON_CNTL__PERFMON_RING_MODE_MASK                                                          0x00003000L
+#define RLC_SPM_PERFMON_CNTL__DISABLE_GFXCLOCK_COUNT_MASK                                                     0x00004000L
+#define RLC_SPM_PERFMON_CNTL__RESERVED_MASK                                                                   0x00008000L
+#define RLC_SPM_PERFMON_CNTL__PERFMON_SAMPLE_INTERVAL_MASK                                                    0xFFFF0000L
+//RLC_SPM_PERFMON_RING_BASE_LO
+#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO__SHIFT                                                     0x0
+#define RLC_SPM_PERFMON_RING_BASE_LO__RING_BASE_LO_MASK                                                       0xFFFFFFFFL
+//RLC_SPM_PERFMON_RING_BASE_HI
+#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI__SHIFT                                                     0x0
+#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED__SHIFT                                                         0x10
+#define RLC_SPM_PERFMON_RING_BASE_HI__RING_BASE_HI_MASK                                                       0x0000FFFFL
+#define RLC_SPM_PERFMON_RING_BASE_HI__RESERVED_MASK                                                           0xFFFF0000L
+//RLC_SPM_PERFMON_RING_SIZE
+#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE__SHIFT                                                      0x0
+#define RLC_SPM_PERFMON_RING_SIZE__RING_BASE_SIZE_MASK                                                        0xFFFFFFFFL
+//RLC_SPM_RING_WRPTR
+#define RLC_SPM_RING_WRPTR__RESERVED__SHIFT                                                                   0x0
+#define RLC_SPM_RING_WRPTR__PERFMON_RING_WRPTR__SHIFT                                                         0x5
+#define RLC_SPM_RING_WRPTR__RESERVED_MASK                                                                     0x0000001FL
+#define RLC_SPM_RING_WRPTR__PERFMON_RING_WRPTR_MASK                                                           0xFFFFFFE0L
+//RLC_SPM_RING_RDPTR
+#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR__SHIFT                                                         0x0
+#define RLC_SPM_RING_RDPTR__PERFMON_RING_RDPTR_MASK                                                           0xFFFFFFFFL
+//RLC_SPM_SEGMENT_THRESHOLD
+#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD__SHIFT                                               0x0
+#define RLC_SPM_SEGMENT_THRESHOLD__RESERVED__SHIFT                                                            0x8
+#define RLC_SPM_SEGMENT_THRESHOLD__NUM_SEGMENT_THRESHOLD_MASK                                                 0x000000FFL
+#define RLC_SPM_SEGMENT_THRESHOLD__RESERVED_MASK                                                              0xFFFFFF00L
+//RLC_SPM_PERFMON_SEGMENT_SIZE
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__TOTAL_NUM_SEGMENT__SHIFT                                                0x0
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_SEGMENT__SHIFT                                               0x10
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE_NUM_SEGMENT__SHIFT                                                   0x18
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__TOTAL_NUM_SEGMENT_MASK                                                  0x0000FFFFL
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__GLOBAL_NUM_SEGMENT_MASK                                                 0x00FF0000L
+#define RLC_SPM_PERFMON_SEGMENT_SIZE__SE_NUM_SEGMENT_MASK                                                     0xFF000000L
+//RLC_SPM_GLOBAL_MUXSEL_ADDR
+#define RLC_SPM_GLOBAL_MUXSEL_ADDR__ADDR__SHIFT                                                               0x0
+#define RLC_SPM_GLOBAL_MUXSEL_ADDR__ADDR_MASK                                                                 0x00000FFFL
+//RLC_SPM_GLOBAL_MUXSEL_DATA
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__SEL0__SHIFT                                                               0x0
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__SEL1__SHIFT                                                               0x10
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__SEL0_MASK                                                                 0x0000FFFFL
+#define RLC_SPM_GLOBAL_MUXSEL_DATA__SEL1_MASK                                                                 0xFFFF0000L
+//RLC_SPM_SE_MUXSEL_ADDR
+#define RLC_SPM_SE_MUXSEL_ADDR__ADDR__SHIFT                                                                   0x0
+#define RLC_SPM_SE_MUXSEL_ADDR__ADDR_MASK                                                                     0x00000FFFL
+//RLC_SPM_SE_MUXSEL_DATA
+#define RLC_SPM_SE_MUXSEL_DATA__SEL0__SHIFT                                                                   0x0
+#define RLC_SPM_SE_MUXSEL_DATA__SEL1__SHIFT                                                                   0x10
+#define RLC_SPM_SE_MUXSEL_DATA__SEL0_MASK                                                                     0x0000FFFFL
+#define RLC_SPM_SE_MUXSEL_DATA__SEL1_MASK                                                                     0xFFFF0000L
+//RLC_SPM_ACCUM_DATARAM_ADDR
+#define RLC_SPM_ACCUM_DATARAM_ADDR__addr__SHIFT                                                               0x0
+#define RLC_SPM_ACCUM_DATARAM_ADDR__RESERVED__SHIFT                                                           0x7
+#define RLC_SPM_ACCUM_DATARAM_ADDR__addr_MASK                                                                 0x0000007FL
+#define RLC_SPM_ACCUM_DATARAM_ADDR__RESERVED_MASK                                                             0xFFFFFF80L
+//RLC_SPM_ACCUM_DATARAM_DATA
+#define RLC_SPM_ACCUM_DATARAM_DATA__data__SHIFT                                                               0x0
+#define RLC_SPM_ACCUM_DATARAM_DATA__data_MASK                                                                 0xFFFFFFFFL
+//RLC_SPM_ACCUM_SWA_DATARAM_ADDR
+#define RLC_SPM_ACCUM_SWA_DATARAM_ADDR__addr__SHIFT                                                           0x0
+#define RLC_SPM_ACCUM_SWA_DATARAM_ADDR__RESERVED__SHIFT                                                       0x7
+#define RLC_SPM_ACCUM_SWA_DATARAM_ADDR__addr_MASK                                                             0x0000007FL
+#define RLC_SPM_ACCUM_SWA_DATARAM_ADDR__RESERVED_MASK                                                         0xFFFFFF80L
+//RLC_SPM_ACCUM_SWA_DATARAM_DATA
+#define RLC_SPM_ACCUM_SWA_DATARAM_DATA__data__SHIFT                                                           0x0
+#define RLC_SPM_ACCUM_SWA_DATARAM_DATA__data_MASK                                                             0xFFFFFFFFL
+//RLC_SPM_ACCUM_CTRLRAM_ADDR
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR__addr__SHIFT                                                               0x0
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR__RESERVED__SHIFT                                                           0xb
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR__addr_MASK                                                                 0x000007FFL
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR__RESERVED_MASK                                                             0xFFFFF800L
+//RLC_SPM_ACCUM_CTRLRAM_DATA
+#define RLC_SPM_ACCUM_CTRLRAM_DATA__data__SHIFT                                                               0x0
+#define RLC_SPM_ACCUM_CTRLRAM_DATA__RESERVED__SHIFT                                                           0x8
+#define RLC_SPM_ACCUM_CTRLRAM_DATA__data_MASK                                                                 0x000000FFL
+#define RLC_SPM_ACCUM_CTRLRAM_DATA__RESERVED_MASK                                                             0xFFFFFF00L
+//RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__global_offset__SHIFT                                               0x0
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__spmwithaccum_se_offset__SHIFT                                      0x8
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__spmwithaccum_global_offset__SHIFT                                  0x10
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__RESERVED__SHIFT                                                    0x18
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__global_offset_MASK                                                 0x000000FFL
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__spmwithaccum_se_offset_MASK                                        0x0000FF00L
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__spmwithaccum_global_offset_MASK                                    0x00FF0000L
+#define RLC_SPM_ACCUM_CTRLRAM_ADDR_OFFSET__RESERVED_MASK                                                      0xFF000000L
+//RLC_SPM_ACCUM_STATUS
+#define RLC_SPM_ACCUM_STATUS__NumbSamplesCompleted__SHIFT                                                     0x0
+#define RLC_SPM_ACCUM_STATUS__AccumDone__SHIFT                                                                0x8
+#define RLC_SPM_ACCUM_STATUS__SpmDone__SHIFT                                                                  0x9
+#define RLC_SPM_ACCUM_STATUS__AccumOverflow__SHIFT                                                            0xa
+#define RLC_SPM_ACCUM_STATUS__AccumArmed__SHIFT                                                               0xb
+#define RLC_SPM_ACCUM_STATUS__SequenceInProgress__SHIFT                                                       0xc
+#define RLC_SPM_ACCUM_STATUS__FinalSequenceInProgress__SHIFT                                                  0xd
+#define RLC_SPM_ACCUM_STATUS__AllFifosEmpty__SHIFT                                                            0xe
+#define RLC_SPM_ACCUM_STATUS__FSMIsIdle__SHIFT                                                                0xf
+#define RLC_SPM_ACCUM_STATUS__SwaAccumDone__SHIFT                                                             0x10
+#define RLC_SPM_ACCUM_STATUS__SwaSpmDone__SHIFT                                                               0x11
+#define RLC_SPM_ACCUM_STATUS__SwaAccumOverflow__SHIFT                                                         0x12
+#define RLC_SPM_ACCUM_STATUS__SwaAccumArmed__SHIFT                                                            0x13
+#define RLC_SPM_ACCUM_STATUS__AllSegsDone__SHIFT                                                              0x14
+#define RLC_SPM_ACCUM_STATUS__RearmSwaPending__SHIFT                                                          0x15
+#define RLC_SPM_ACCUM_STATUS__RearmSppPending__SHIFT                                                          0x16
+#define RLC_SPM_ACCUM_STATUS__MultiSampleAborted__SHIFT                                                       0x17
+#define RLC_SPM_ACCUM_STATUS__RESERVED__SHIFT                                                                 0x18
+#define RLC_SPM_ACCUM_STATUS__NumbSamplesCompleted_MASK                                                       0x000000FFL
+#define RLC_SPM_ACCUM_STATUS__AccumDone_MASK                                                                  0x00000100L
+#define RLC_SPM_ACCUM_STATUS__SpmDone_MASK                                                                    0x00000200L
+#define RLC_SPM_ACCUM_STATUS__AccumOverflow_MASK                                                              0x00000400L
+#define RLC_SPM_ACCUM_STATUS__AccumArmed_MASK                                                                 0x00000800L
+#define RLC_SPM_ACCUM_STATUS__SequenceInProgress_MASK                                                         0x00001000L
+#define RLC_SPM_ACCUM_STATUS__FinalSequenceInProgress_MASK                                                    0x00002000L
+#define RLC_SPM_ACCUM_STATUS__AllFifosEmpty_MASK                                                              0x00004000L
+#define RLC_SPM_ACCUM_STATUS__FSMIsIdle_MASK                                                                  0x00008000L
+#define RLC_SPM_ACCUM_STATUS__SwaAccumDone_MASK                                                               0x00010000L
+#define RLC_SPM_ACCUM_STATUS__SwaSpmDone_MASK                                                                 0x00020000L
+#define RLC_SPM_ACCUM_STATUS__SwaAccumOverflow_MASK                                                           0x00040000L
+#define RLC_SPM_ACCUM_STATUS__SwaAccumArmed_MASK                                                              0x00080000L
+#define RLC_SPM_ACCUM_STATUS__AllSegsDone_MASK                                                                0x00100000L
+#define RLC_SPM_ACCUM_STATUS__RearmSwaPending_MASK                                                            0x00200000L
+#define RLC_SPM_ACCUM_STATUS__RearmSppPending_MASK                                                            0x00400000L
+#define RLC_SPM_ACCUM_STATUS__MultiSampleAborted_MASK                                                         0x00800000L
+#define RLC_SPM_ACCUM_STATUS__RESERVED_MASK                                                                   0xFF000000L
+//RLC_SPM_ACCUM_CTRL
+#define RLC_SPM_ACCUM_CTRL__StrobeResetPerfMonitors__SHIFT                                                    0x0
+#define RLC_SPM_ACCUM_CTRL__StrobeStartAccumulation__SHIFT                                                    0x1
+#define RLC_SPM_ACCUM_CTRL__StrobeRearmAccum__SHIFT                                                           0x2
+#define RLC_SPM_ACCUM_CTRL__StrobeResetSpmBlock__SHIFT                                                        0x3
+#define RLC_SPM_ACCUM_CTRL__StrobeStartSpm__SHIFT                                                             0x4
+#define RLC_SPM_ACCUM_CTRL__StrobeRearmSwaAccum__SHIFT                                                        0x8
+#define RLC_SPM_ACCUM_CTRL__StrobeStartSwa__SHIFT                                                             0x9
+#define RLC_SPM_ACCUM_CTRL__StrobePerfmonSampleWires__SHIFT                                                   0xa
+#define RLC_SPM_ACCUM_CTRL__RESERVED__SHIFT                                                                   0xb
+#define RLC_SPM_ACCUM_CTRL__StrobeResetPerfMonitors_MASK                                                      0x00000001L
+#define RLC_SPM_ACCUM_CTRL__StrobeStartAccumulation_MASK                                                      0x00000002L
+#define RLC_SPM_ACCUM_CTRL__StrobeRearmAccum_MASK                                                             0x00000004L
+#define RLC_SPM_ACCUM_CTRL__StrobeResetSpmBlock_MASK                                                          0x00000008L
+#define RLC_SPM_ACCUM_CTRL__StrobeStartSpm_MASK                                                               0x000000F0L
+#define RLC_SPM_ACCUM_CTRL__StrobeRearmSwaAccum_MASK                                                          0x00000100L
+#define RLC_SPM_ACCUM_CTRL__StrobeStartSwa_MASK                                                               0x00000200L
+#define RLC_SPM_ACCUM_CTRL__StrobePerfmonSampleWires_MASK                                                     0x00000400L
+#define RLC_SPM_ACCUM_CTRL__RESERVED_MASK                                                                     0xFFFFF800L
+//RLC_SPM_ACCUM_MODE
+#define RLC_SPM_ACCUM_MODE__EnableAccum__SHIFT                                                                0x0
+#define RLC_SPM_ACCUM_MODE__EnableSpmWithAccumMode__SHIFT                                                     0x1
+#define RLC_SPM_ACCUM_MODE__EnableSPPMode__SHIFT                                                              0x2
+#define RLC_SPM_ACCUM_MODE__AutoResetPerfmonDisable__SHIFT                                                    0x3
+#define RLC_SPM_ACCUM_MODE__AutoAccumEn__SHIFT                                                                0x5
+#define RLC_SPM_ACCUM_MODE__SwaAutoAccumEn__SHIFT                                                             0x6
+#define RLC_SPM_ACCUM_MODE__AutoSpmEn__SHIFT                                                                  0x7
+#define RLC_SPM_ACCUM_MODE__SwaAutoSpmEn__SHIFT                                                               0x8
+#define RLC_SPM_ACCUM_MODE__Globals_LoadOverride__SHIFT                                                       0x9
+#define RLC_SPM_ACCUM_MODE__Globals_SwaLoadOverride__SHIFT                                                    0xa
+#define RLC_SPM_ACCUM_MODE__SE0_LoadOverride__SHIFT                                                           0xb
+#define RLC_SPM_ACCUM_MODE__SE0_SwaLoadOverride__SHIFT                                                        0xc
+#define RLC_SPM_ACCUM_MODE__SE1_LoadOverride__SHIFT                                                           0xd
+#define RLC_SPM_ACCUM_MODE__SE1_SwaLoadOverride__SHIFT                                                        0xe
+#define RLC_SPM_ACCUM_MODE__SE2_LoadOverride__SHIFT                                                           0xf
+#define RLC_SPM_ACCUM_MODE__SE2_SwaLoadOverride__SHIFT                                                        0x10
+#define RLC_SPM_ACCUM_MODE__EnableAccum_MASK                                                                  0x00000001L
+#define RLC_SPM_ACCUM_MODE__EnableSpmWithAccumMode_MASK                                                       0x00000002L
+#define RLC_SPM_ACCUM_MODE__EnableSPPMode_MASK                                                                0x00000004L
+#define RLC_SPM_ACCUM_MODE__AutoResetPerfmonDisable_MASK                                                      0x00000008L
+#define RLC_SPM_ACCUM_MODE__AutoAccumEn_MASK                                                                  0x00000020L
+#define RLC_SPM_ACCUM_MODE__SwaAutoAccumEn_MASK                                                               0x00000040L
+#define RLC_SPM_ACCUM_MODE__AutoSpmEn_MASK                                                                    0x00000080L
+#define RLC_SPM_ACCUM_MODE__SwaAutoSpmEn_MASK                                                                 0x00000100L
+#define RLC_SPM_ACCUM_MODE__Globals_LoadOverride_MASK                                                         0x00000200L
+#define RLC_SPM_ACCUM_MODE__Globals_SwaLoadOverride_MASK                                                      0x00000400L
+#define RLC_SPM_ACCUM_MODE__SE0_LoadOverride_MASK                                                             0x00000800L
+#define RLC_SPM_ACCUM_MODE__SE0_SwaLoadOverride_MASK                                                          0x00001000L
+#define RLC_SPM_ACCUM_MODE__SE1_LoadOverride_MASK                                                             0x00002000L
+#define RLC_SPM_ACCUM_MODE__SE1_SwaLoadOverride_MASK                                                          0x00004000L
+#define RLC_SPM_ACCUM_MODE__SE2_LoadOverride_MASK                                                             0x00008000L
+#define RLC_SPM_ACCUM_MODE__SE2_SwaLoadOverride_MASK                                                          0x00010000L
+//RLC_SPM_ACCUM_THRESHOLD
+#define RLC_SPM_ACCUM_THRESHOLD__Threshold__SHIFT                                                             0x0
+#define RLC_SPM_ACCUM_THRESHOLD__Threshold_MASK                                                               0x0000FFFFL
+//RLC_SPM_ACCUM_SAMPLES_REQUESTED
+#define RLC_SPM_ACCUM_SAMPLES_REQUESTED__SamplesRequested__SHIFT                                              0x0
+#define RLC_SPM_ACCUM_SAMPLES_REQUESTED__SamplesRequested_MASK                                                0x000000FFL
+//RLC_SPM_ACCUM_DATARAM_WRCOUNT
+#define RLC_SPM_ACCUM_DATARAM_WRCOUNT__DataRamWrCount__SHIFT                                                  0x0
+#define RLC_SPM_ACCUM_DATARAM_WRCOUNT__RESERVED__SHIFT                                                        0x13
+#define RLC_SPM_ACCUM_DATARAM_WRCOUNT__DataRamWrCount_MASK                                                    0x0007FFFFL
+#define RLC_SPM_ACCUM_DATARAM_WRCOUNT__RESERVED_MASK                                                          0xFFF80000L
+//RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS
+#define RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS__spp_addr_region__SHIFT                                      0x0
+#define RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS__swa_addr_region__SHIFT                                      0x8
+#define RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS__RESERVED__SHIFT                                             0x10
+#define RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS__spp_addr_region_MASK                                        0x000000FFL
+#define RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS__swa_addr_region_MASK                                        0x0000FF00L
+#define RLC_SPM_ACCUM_DATARAM_32BITCNTRS_REGIONS__RESERVED_MASK                                               0xFFFF0000L
+//RLC_SPM_PAUSE
+#define RLC_SPM_PAUSE__PAUSE__SHIFT                                                                           0x0
+#define RLC_SPM_PAUSE__PAUSED__SHIFT                                                                          0x1
+#define RLC_SPM_PAUSE__PAUSE_MASK                                                                             0x00000001L
+#define RLC_SPM_PAUSE__PAUSED_MASK                                                                            0x00000002L
+//RLC_SPM_STATUS
+#define RLC_SPM_STATUS__CTL_BUSY__SHIFT                                                                       0x0
+#define RLC_SPM_STATUS__RSPM_REG_BUSY__SHIFT                                                                  0x1
+#define RLC_SPM_STATUS__SPM_RSPM_BUSY__SHIFT                                                                  0x2
+#define RLC_SPM_STATUS__SPM_RSPM_IO_BUSY__SHIFT                                                               0x3
+#define RLC_SPM_STATUS__SE_RSPM_IO_BUSY__SHIFT                                                                0x4
+#define RLC_SPM_STATUS__ACCUM_BUSY__SHIFT                                                                     0xf
+#define RLC_SPM_STATUS__FSM_MASTER_STATE__SHIFT                                                               0x10
+#define RLC_SPM_STATUS__FSM_MEMORY_STATE__SHIFT                                                               0x14
+#define RLC_SPM_STATUS__CTL_REQ_STATE__SHIFT                                                                  0x18
+#define RLC_SPM_STATUS__CTL_RET_STATE__SHIFT                                                                  0x1a
+#define RLC_SPM_STATUS__CTL_BUSY_MASK                                                                         0x00000001L
+#define RLC_SPM_STATUS__RSPM_REG_BUSY_MASK                                                                    0x00000002L
+#define RLC_SPM_STATUS__SPM_RSPM_BUSY_MASK                                                                    0x00000004L
+#define RLC_SPM_STATUS__SPM_RSPM_IO_BUSY_MASK                                                                 0x00000008L
+#define RLC_SPM_STATUS__SE_RSPM_IO_BUSY_MASK                                                                  0x00000FF0L
+#define RLC_SPM_STATUS__ACCUM_BUSY_MASK                                                                       0x00008000L
+#define RLC_SPM_STATUS__FSM_MASTER_STATE_MASK                                                                 0x000F0000L
+#define RLC_SPM_STATUS__FSM_MEMORY_STATE_MASK                                                                 0x00F00000L
+#define RLC_SPM_STATUS__CTL_REQ_STATE_MASK                                                                    0x03000000L
+#define RLC_SPM_STATUS__CTL_RET_STATE_MASK                                                                    0x04000000L
+//RLC_SPM_GFXCLOCK_LOWCOUNT
+#define RLC_SPM_GFXCLOCK_LOWCOUNT__GFXCLOCK_LOWCOUNT__SHIFT                                                   0x0
+#define RLC_SPM_GFXCLOCK_LOWCOUNT__GFXCLOCK_LOWCOUNT_MASK                                                     0xFFFFFFFFL
+//RLC_SPM_GFXCLOCK_HIGHCOUNT
+#define RLC_SPM_GFXCLOCK_HIGHCOUNT__GFXCLOCK_HIGHCOUNT__SHIFT                                                 0x0
+#define RLC_SPM_GFXCLOCK_HIGHCOUNT__GFXCLOCK_HIGHCOUNT_MASK                                                   0xFFFFFFFFL
+//RLC_SPM_MODE
+#define RLC_SPM_MODE__MODE__SHIFT                                                                             0x0
+#define RLC_SPM_MODE__MODE_MASK                                                                               0x00000001L
+//RLC_SPM_RSPM_REQ_DATA_LO
+#define RLC_SPM_RSPM_REQ_DATA_LO__DATA__SHIFT                                                                 0x0
+#define RLC_SPM_RSPM_REQ_DATA_LO__DATA_MASK                                                                   0xFFFFFFFFL
+//RLC_SPM_RSPM_REQ_DATA_HI
+#define RLC_SPM_RSPM_REQ_DATA_HI__DATA__SHIFT                                                                 0x0
+#define RLC_SPM_RSPM_REQ_DATA_HI__DATA_MASK                                                                   0x00000FFFL
+//RLC_SPM_RSPM_REQ_OP
+#define RLC_SPM_RSPM_REQ_OP__OP__SHIFT                                                                        0x0
+#define RLC_SPM_RSPM_REQ_OP__OP_MASK                                                                          0x0000000FL
+//RLC_SPM_RSPM_RET_DATA
+#define RLC_SPM_RSPM_RET_DATA__DATA__SHIFT                                                                    0x0
+#define RLC_SPM_RSPM_RET_DATA__DATA_MASK                                                                      0xFFFFFFFFL
+//RLC_SPM_RSPM_RET_OP
+#define RLC_SPM_RSPM_RET_OP__OP__SHIFT                                                                        0x0
+#define RLC_SPM_RSPM_RET_OP__VALID__SHIFT                                                                     0x8
+#define RLC_SPM_RSPM_RET_OP__OP_MASK                                                                          0x0000000FL
+#define RLC_SPM_RSPM_RET_OP__VALID_MASK                                                                       0x00000100L
+//RLC_SPM_SE_RSPM_REQ_DATA_LO
+#define RLC_SPM_SE_RSPM_REQ_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_SPM_SE_RSPM_REQ_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_SPM_SE_RSPM_REQ_DATA_HI
+#define RLC_SPM_SE_RSPM_REQ_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_SPM_SE_RSPM_REQ_DATA_HI__DATA_MASK                                                                0x00000FFFL
+//RLC_SPM_SE_RSPM_REQ_OP
+#define RLC_SPM_SE_RSPM_REQ_OP__OP__SHIFT                                                                     0x0
+#define RLC_SPM_SE_RSPM_REQ_OP__OP_MASK                                                                       0x0000000FL
+//RLC_SPM_SE_RSPM_RET_DATA
+#define RLC_SPM_SE_RSPM_RET_DATA__DATA__SHIFT                                                                 0x0
+#define RLC_SPM_SE_RSPM_RET_DATA__DATA_MASK                                                                   0xFFFFFFFFL
+//RLC_SPM_SE_RSPM_RET_OP
+#define RLC_SPM_SE_RSPM_RET_OP__OP__SHIFT                                                                     0x0
+#define RLC_SPM_SE_RSPM_RET_OP__VALID__SHIFT                                                                  0x8
+#define RLC_SPM_SE_RSPM_RET_OP__OP_MASK                                                                       0x0000000FL
+#define RLC_SPM_SE_RSPM_RET_OP__VALID_MASK                                                                    0x00000100L
+//RLC_SPM_RSPM_CMD
+#define RLC_SPM_RSPM_CMD__CMD__SHIFT                                                                          0x0
+#define RLC_SPM_RSPM_CMD__CMD_MASK                                                                            0x0000000FL
+//RLC_SPM_RSPM_CMD_ACK
+#define RLC_SPM_RSPM_CMD_ACK__SE0_ACK__SHIFT                                                                  0x0
+#define RLC_SPM_RSPM_CMD_ACK__SE1_ACK__SHIFT                                                                  0x1
+#define RLC_SPM_RSPM_CMD_ACK__SE2_ACK__SHIFT                                                                  0x2
+#define RLC_SPM_RSPM_CMD_ACK__SE3_ACK__SHIFT                                                                  0x3
+#define RLC_SPM_RSPM_CMD_ACK__SE4_ACK__SHIFT                                                                  0x4
+#define RLC_SPM_RSPM_CMD_ACK__SE5_ACK__SHIFT                                                                  0x5
+#define RLC_SPM_RSPM_CMD_ACK__SE6_ACK__SHIFT                                                                  0x6
+#define RLC_SPM_RSPM_CMD_ACK__SE7_ACK__SHIFT                                                                  0x7
+#define RLC_SPM_RSPM_CMD_ACK__SPM_ACK__SHIFT                                                                  0x8
+#define RLC_SPM_RSPM_CMD_ACK__SE0_ACK_MASK                                                                    0x00000001L
+#define RLC_SPM_RSPM_CMD_ACK__SE1_ACK_MASK                                                                    0x00000002L
+#define RLC_SPM_RSPM_CMD_ACK__SE2_ACK_MASK                                                                    0x00000004L
+#define RLC_SPM_RSPM_CMD_ACK__SE3_ACK_MASK                                                                    0x00000008L
+#define RLC_SPM_RSPM_CMD_ACK__SE4_ACK_MASK                                                                    0x00000010L
+#define RLC_SPM_RSPM_CMD_ACK__SE5_ACK_MASK                                                                    0x00000020L
+#define RLC_SPM_RSPM_CMD_ACK__SE6_ACK_MASK                                                                    0x00000040L
+#define RLC_SPM_RSPM_CMD_ACK__SE7_ACK_MASK                                                                    0x00000080L
+#define RLC_SPM_RSPM_CMD_ACK__SPM_ACK_MASK                                                                    0x00000100L
+//RLC_SPM_SPARE
+#define RLC_SPM_SPARE__SPARE__SHIFT                                                                           0x0
+#define RLC_SPM_SPARE__SPARE_MASK                                                                             0xFFFFFFFFL
+//RLC_PERFMON_CNTL
+#define RLC_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                                0x0
+#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT                                                        0xa
+#define RLC_PERFMON_CNTL__PERFMON_STATE_MASK                                                                  0x00000007L
+#define RLC_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK                                                          0x00000400L
+//RLC_PERFCOUNTER0_SELECT
+#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
+#define RLC_PERFCOUNTER0_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x000000FFL
+//RLC_PERFCOUNTER1_SELECT
+#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT__SHIFT                                                    0x0
+#define RLC_PERFCOUNTER1_SELECT__PERFCOUNTER_SELECT_MASK                                                      0x000000FFL
+//RLC_GPU_IOV_PERF_CNT_CNTL
+#define RLC_GPU_IOV_PERF_CNT_CNTL__ENABLE__SHIFT                                                              0x0
+#define RLC_GPU_IOV_PERF_CNT_CNTL__MODE_SELECT__SHIFT                                                         0x1
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESET__SHIFT                                                               0x2
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESERVED__SHIFT                                                            0x3
+#define RLC_GPU_IOV_PERF_CNT_CNTL__ENABLE_MASK                                                                0x00000001L
+#define RLC_GPU_IOV_PERF_CNT_CNTL__MODE_SELECT_MASK                                                           0x00000002L
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESET_MASK                                                                 0x00000004L
+#define RLC_GPU_IOV_PERF_CNT_CNTL__RESERVED_MASK                                                              0xFFFFFFF8L
+//RLC_GPU_IOV_PERF_CNT_WR_ADDR
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__VFID__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__CNT_ID__SHIFT                                                           0x4
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__RESERVED__SHIFT                                                         0x6
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__VFID_MASK                                                               0x0000000FL
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__CNT_ID_MASK                                                             0x00000030L
+#define RLC_GPU_IOV_PERF_CNT_WR_ADDR__RESERVED_MASK                                                           0xFFFFFFC0L
+//RLC_GPU_IOV_PERF_CNT_WR_DATA
+#define RLC_GPU_IOV_PERF_CNT_WR_DATA__DATA__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_WR_DATA__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_GPU_IOV_PERF_CNT_RD_ADDR
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__VFID__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__CNT_ID__SHIFT                                                           0x4
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__RESERVED__SHIFT                                                         0x6
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__VFID_MASK                                                               0x0000000FL
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__CNT_ID_MASK                                                             0x00000030L
+#define RLC_GPU_IOV_PERF_CNT_RD_ADDR__RESERVED_MASK                                                           0xFFFFFFC0L
+//RLC_GPU_IOV_PERF_CNT_RD_DATA
+#define RLC_GPU_IOV_PERF_CNT_RD_DATA__DATA__SHIFT                                                             0x0
+#define RLC_GPU_IOV_PERF_CNT_RD_DATA__DATA_MASK                                                               0xFFFFFFFFL
+//RMI_PERFCOUNTER0_SELECT
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define RMI_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define RMI_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define RMI_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define RMI_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERFCOUNTER0_SELECT1
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define RMI_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define RMI_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//RMI_PERFCOUNTER1_SELECT
+#define RMI_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define RMI_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERFCOUNTER2_SELECT
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define RMI_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define RMI_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define RMI_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define RMI_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERFCOUNTER2_SELECT1
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define RMI_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define RMI_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//RMI_PERFCOUNTER3_SELECT
+#define RMI_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define RMI_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define RMI_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define RMI_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//RMI_PERF_COUNTER_CNTL
+#define RMI_PERF_COUNTER_CNTL__TRANS_BASED_PERF_EN_SEL__SHIFT                                                 0x0
+#define RMI_PERF_COUNTER_CNTL__EVENT_BASED_PERF_EN_SEL__SHIFT                                                 0x2
+#define RMI_PERF_COUNTER_CNTL__TC_PERF_EN_SEL__SHIFT                                                          0x4
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK0__SHIFT                                                 0x6
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK1__SHIFT                                                 0x8
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_CID__SHIFT                                                        0xa
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_VMID__SHIFT                                                       0xe
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_BURST_LENGTH_THRESHOLD__SHIFT                                     0x13
+#define RMI_PERF_COUNTER_CNTL__PERF_SOFT_RESET__SHIFT                                                         0x19
+#define RMI_PERF_COUNTER_CNTL__PERF_CNTR_SPM_SEL__SHIFT                                                       0x1a
+#define RMI_PERF_COUNTER_CNTL__TRANS_BASED_PERF_EN_SEL_MASK                                                   0x00000003L
+#define RMI_PERF_COUNTER_CNTL__EVENT_BASED_PERF_EN_SEL_MASK                                                   0x0000000CL
+#define RMI_PERF_COUNTER_CNTL__TC_PERF_EN_SEL_MASK                                                            0x00000030L
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK0_MASK                                                   0x000000C0L
+#define RMI_PERF_COUNTER_CNTL__PERF_EVENT_WINDOW_MASK1_MASK                                                   0x00000300L
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_CID_MASK                                                          0x00003C00L
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_VMID_MASK                                                         0x0007C000L
+#define RMI_PERF_COUNTER_CNTL__PERF_COUNTER_BURST_LENGTH_THRESHOLD_MASK                                       0x01F80000L
+#define RMI_PERF_COUNTER_CNTL__PERF_SOFT_RESET_MASK                                                           0x02000000L
+#define RMI_PERF_COUNTER_CNTL__PERF_CNTR_SPM_SEL_MASK                                                         0x04000000L
+//GCR_PERFCOUNTER0_SELECT
+#define GCR_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GCR_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GCR_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GCR_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GCR_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GCR_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GCR_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GCR_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GCR_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GCR_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GCR_PERFCOUNTER0_SELECT1
+#define GCR_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GCR_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GCR_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GCR_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GCR_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GCR_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GCR_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GCR_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GCR_PERFCOUNTER1_SELECT
+#define GCR_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GCR_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GCR_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GCR_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GCR_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GCR_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//PA_PH_PERFCOUNTER0_SELECT
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_PH_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_PH_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_PH_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_PH_PERFCOUNTER0_SELECT1
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_PH_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_PH_PERFCOUNTER1_SELECT
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_PH_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_PH_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_PH_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_PH_PERFCOUNTER2_SELECT
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_PH_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_PH_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_PH_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_PH_PERFCOUNTER3_SELECT
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                           0xa
+#define PA_PH_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                           0x14
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                          0x18
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                           0x1c
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                             0x000FFC00L
+#define PA_PH_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                             0x00F00000L
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                            0x0F000000L
+#define PA_PH_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                             0xF0000000L
+//PA_PH_PERFCOUNTER4_SELECT
+#define PA_PH_PERFCOUNTER4_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER4_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_PH_PERFCOUNTER5_SELECT
+#define PA_PH_PERFCOUNTER5_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER5_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_PH_PERFCOUNTER6_SELECT
+#define PA_PH_PERFCOUNTER6_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER6_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_PH_PERFCOUNTER7_SELECT
+#define PA_PH_PERFCOUNTER7_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define PA_PH_PERFCOUNTER7_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+//PA_PH_PERFCOUNTER1_SELECT1
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_PH_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_PH_PERFCOUNTER2_SELECT1
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_PH_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//PA_PH_PERFCOUNTER3_SELECT1
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                          0x0
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                          0xa
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                         0x18
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                         0x1c
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                            0x000003FFL
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                            0x000FFC00L
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                           0x0F000000L
+#define PA_PH_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                           0xF0000000L
+//UTCL1_PERFCOUNTER0_SELECT
+#define UTCL1_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define UTCL1_PERFCOUNTER0_SELECT__COUNTER_MODE__SHIFT                                                        0x1c
+#define UTCL1_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define UTCL1_PERFCOUNTER0_SELECT__COUNTER_MODE_MASK                                                          0xF0000000L
+//UTCL1_PERFCOUNTER1_SELECT
+#define UTCL1_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define UTCL1_PERFCOUNTER1_SELECT__COUNTER_MODE__SHIFT                                                        0x1c
+#define UTCL1_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define UTCL1_PERFCOUNTER1_SELECT__COUNTER_MODE_MASK                                                          0xF0000000L
+//UTCL1_PERFCOUNTER2_SELECT
+#define UTCL1_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define UTCL1_PERFCOUNTER2_SELECT__COUNTER_MODE__SHIFT                                                        0x1c
+#define UTCL1_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define UTCL1_PERFCOUNTER2_SELECT__COUNTER_MODE_MASK                                                          0xF0000000L
+//UTCL1_PERFCOUNTER3_SELECT
+#define UTCL1_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define UTCL1_PERFCOUNTER3_SELECT__COUNTER_MODE__SHIFT                                                        0x1c
+#define UTCL1_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                              0x000003FFL
+#define UTCL1_PERFCOUNTER3_SELECT__COUNTER_MODE_MASK                                                          0xF0000000L
+//GL1A_PERFCOUNTER0_SELECT
+#define GL1A_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1A_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1A_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1A_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1A_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1A_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1A_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1A_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1A_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1A_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1A_PERFCOUNTER0_SELECT1
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                          0x18
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                          0x1c
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                            0x0F000000L
+#define GL1A_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                            0xF0000000L
+//GL1A_PERFCOUNTER1_SELECT
+#define GL1A_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1A_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1A_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1A_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1A_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1A_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1A_PERFCOUNTER2_SELECT
+#define GL1A_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1A_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1A_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1A_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1A_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1A_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1A_PERFCOUNTER3_SELECT
+#define GL1A_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1A_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1A_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1A_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1A_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1A_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1H_PERFCOUNTER0_SELECT
+#define GL1H_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1H_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL1H_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1H_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL1H_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1H_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1H_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL1H_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1H_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL1H_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1H_PERFCOUNTER0_SELECT1
+#define GL1H_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL1H_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL1H_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                          0x18
+#define GL1H_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                          0x1c
+#define GL1H_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL1H_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL1H_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                            0x0F000000L
+#define GL1H_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                            0xF0000000L
+//GL1H_PERFCOUNTER1_SELECT
+#define GL1H_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1H_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1H_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1H_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1H_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1H_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1H_PERFCOUNTER2_SELECT
+#define GL1H_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1H_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1H_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1H_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1H_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1H_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL1H_PERFCOUNTER3_SELECT
+#define GL1H_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL1H_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL1H_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL1H_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL1H_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL1H_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//CHA_PERFCOUNTER0_SELECT
+#define CHA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHA_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define CHA_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHA_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define CHA_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHA_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define CHA_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHA_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define CHA_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHA_PERFCOUNTER0_SELECT1
+#define CHA_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define CHA_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define CHA_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                           0x18
+#define CHA_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                           0x1c
+#define CHA_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define CHA_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define CHA_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                             0x0F000000L
+#define CHA_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                             0xF0000000L
+//CHA_PERFCOUNTER1_SELECT
+#define CHA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHA_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHA_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHA_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHA_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHA_PERFCOUNTER2_SELECT
+#define CHA_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHA_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHA_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHA_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHA_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHA_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//CHA_PERFCOUNTER3_SELECT
+#define CHA_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define CHA_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define CHA_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define CHA_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define CHA_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define CHA_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GUS_PERFCOUNTER2_SELECT
+#define GUS_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                              0x0
+#define GUS_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                             0xa
+#define GUS_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                             0x14
+#define GUS_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                            0x18
+#define GUS_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                             0x1c
+#define GUS_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                                0x000003FFL
+#define GUS_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                               0x000FFC00L
+#define GUS_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                               0x00F00000L
+#define GUS_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                              0x0F000000L
+#define GUS_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                               0xF0000000L
+//GUS_PERFCOUNTER2_SELECT1
+#define GUS_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                            0x0
+#define GUS_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                            0xa
+#define GUS_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                           0x18
+#define GUS_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                           0x1c
+#define GUS_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                              0x000003FFL
+#define GUS_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                              0x000FFC00L
+#define GUS_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                             0x0F000000L
+#define GUS_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
+//GUS_PERFCOUNTER2_MODE
+#define GUS_PERFCOUNTER2_MODE__COMPARE_MODE0__SHIFT                                                           0x0
+#define GUS_PERFCOUNTER2_MODE__COMPARE_MODE1__SHIFT                                                           0x2
+#define GUS_PERFCOUNTER2_MODE__COMPARE_MODE2__SHIFT                                                           0x4
+#define GUS_PERFCOUNTER2_MODE__COMPARE_MODE3__SHIFT                                                           0x6
+#define GUS_PERFCOUNTER2_MODE__COMPARE_VALUE0__SHIFT                                                          0x8
+#define GUS_PERFCOUNTER2_MODE__COMPARE_VALUE1__SHIFT                                                          0xc
+#define GUS_PERFCOUNTER2_MODE__COMPARE_VALUE2__SHIFT                                                          0x10
+#define GUS_PERFCOUNTER2_MODE__COMPARE_VALUE3__SHIFT                                                          0x14
+#define GUS_PERFCOUNTER2_MODE__COMPARE_MODE0_MASK                                                             0x00000003L
+#define GUS_PERFCOUNTER2_MODE__COMPARE_MODE1_MASK                                                             0x0000000CL
+#define GUS_PERFCOUNTER2_MODE__COMPARE_MODE2_MASK                                                             0x00000030L
+#define GUS_PERFCOUNTER2_MODE__COMPARE_MODE3_MASK                                                             0x000000C0L
+#define GUS_PERFCOUNTER2_MODE__COMPARE_VALUE0_MASK                                                            0x00000F00L
+#define GUS_PERFCOUNTER2_MODE__COMPARE_VALUE1_MASK                                                            0x0000F000L
+#define GUS_PERFCOUNTER2_MODE__COMPARE_VALUE2_MASK                                                            0x000F0000L
+#define GUS_PERFCOUNTER2_MODE__COMPARE_VALUE3_MASK                                                            0x00F00000L
+//GUS_PERFCOUNTER0_CFG
+#define GUS_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                                 0x0
+#define GUS_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                             0x8
+#define GUS_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                                0x18
+#define GUS_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                                   0x1c
+#define GUS_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                                    0x1d
+#define GUS_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                                   0x000000FFL
+#define GUS_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
+#define GUS_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                                  0x0F000000L
+#define GUS_PERFCOUNTER0_CFG__ENABLE_MASK                                                                     0x10000000L
+#define GUS_PERFCOUNTER0_CFG__CLEAR_MASK                                                                      0x20000000L
+//GUS_PERFCOUNTER1_CFG
+#define GUS_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                                 0x0
+#define GUS_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                             0x8
+#define GUS_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                                0x18
+#define GUS_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                                   0x1c
+#define GUS_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                                    0x1d
+#define GUS_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                                   0x000000FFL
+#define GUS_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                               0x0000FF00L
+#define GUS_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                                  0x0F000000L
+#define GUS_PERFCOUNTER1_CFG__ENABLE_MASK                                                                     0x10000000L
+#define GUS_PERFCOUNTER1_CFG__CLEAR_MASK                                                                      0x20000000L
+//GUS_PERFCOUNTER_RSLT_CNTL
+#define GUS_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                                 0x0
+#define GUS_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                                       0x8
+#define GUS_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                                        0x10
+#define GUS_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                          0x18
+#define GUS_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                           0x19
+#define GUS_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                                0x1a
+#define GUS_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                                   0x0000000FL
+#define GUS_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                         0x0000FF00L
+#define GUS_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                          0x00FF0000L
+#define GUS_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                            0x01000000L
+#define GUS_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                             0x02000000L
+#define GUS_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                                  0x04000000L
+
+
+// addressBlock: gc_gdfll_gdfll_dec
+//GDFLL_EDC_HYSTERESIS_CNTL
+#define GDFLL_EDC_HYSTERESIS_CNTL__MAX_HYSTERESIS__SHIFT                                                      0x0
+#define GDFLL_EDC_HYSTERESIS_CNTL__MAX_HYSTERESIS_MASK                                                        0x000000FFL
+//GDFLL_EDC_HYSTERESIS_STAT
+#define GDFLL_EDC_HYSTERESIS_STAT__HYSTERESIS_CNT__SHIFT                                                      0x0
+#define GDFLL_EDC_HYSTERESIS_STAT__EDC__SHIFT                                                                 0x8
+#define GDFLL_EDC_HYSTERESIS_STAT__HYSTERESIS_CNT_MASK                                                        0x000000FFL
+#define GDFLL_EDC_HYSTERESIS_STAT__EDC_MASK                                                                   0x00000100L
+
+
+// addressBlock: gc_gdfll_se_gdfll_dec
+//GDFLL_SE_EDC_HYSTERESIS_CNTL
+#define GDFLL_SE_EDC_HYSTERESIS_CNTL__MAX_HYSTERESIS__SHIFT                                                   0x0
+#define GDFLL_SE_EDC_HYSTERESIS_CNTL__MAX_HYSTERESIS_MASK                                                     0x000000FFL
+//GDFLL_SE_EDC_HYSTERESIS_STAT
+#define GDFLL_SE_EDC_HYSTERESIS_STAT__HYSTERESIS_CNT__SHIFT                                                   0x0
+#define GDFLL_SE_EDC_HYSTERESIS_STAT__EDC__SHIFT                                                              0x8
+#define GDFLL_SE_EDC_HYSTERESIS_STAT__HYSTERESIS_CNT_MASK                                                     0x000000FFL
+#define GDFLL_SE_EDC_HYSTERESIS_STAT__EDC_MASK                                                                0x00000100L
+
+
+// addressBlock: gc_grtavfs_grtavfs_dec
+//GRTAVFS_RTAVFS_REG_ADDR
+#define GRTAVFS_RTAVFS_REG_ADDR__RTAVFSADDR__SHIFT                                                            0x0
+#define GRTAVFS_RTAVFS_REG_ADDR__RTAVFSADDR_MASK                                                              0x000003FFL
+//GRTAVFS_RTAVFS_WR_DATA
+#define GRTAVFS_RTAVFS_WR_DATA__RTAVFSDATA__SHIFT                                                             0x0
+#define GRTAVFS_RTAVFS_WR_DATA__RTAVFSDATA_MASK                                                               0xFFFFFFFFL
+//GRTAVFS_GENERAL_0
+#define GRTAVFS_GENERAL_0__DATA__SHIFT                                                                        0x0
+#define GRTAVFS_GENERAL_0__DATA_MASK                                                                          0xFFFFFFFFL
+//GRTAVFS_RTAVFS_RD_DATA
+#define GRTAVFS_RTAVFS_RD_DATA__RTAVFSDATA__SHIFT                                                             0x0
+#define GRTAVFS_RTAVFS_RD_DATA__RTAVFSDATA_MASK                                                               0xFFFFFFFFL
+//GRTAVFS_RTAVFS_REG_CTRL
+#define GRTAVFS_RTAVFS_REG_CTRL__SET_WR_EN__SHIFT                                                             0x0
+#define GRTAVFS_RTAVFS_REG_CTRL__SET_RD_EN__SHIFT                                                             0x1
+#define GRTAVFS_RTAVFS_REG_CTRL__SET_WR_EN_MASK                                                               0x00000001L
+#define GRTAVFS_RTAVFS_REG_CTRL__SET_RD_EN_MASK                                                               0x00000002L
+//GRTAVFS_RTAVFS_REG_STATUS
+#define GRTAVFS_RTAVFS_REG_STATUS__RTAVFS_WR_ACK__SHIFT                                                       0x0
+#define GRTAVFS_RTAVFS_REG_STATUS__RTAVFS_RD_DATA_VALID__SHIFT                                                0x1
+#define GRTAVFS_RTAVFS_REG_STATUS__RTAVFS_WR_ACK_MASK                                                         0x00000001L
+#define GRTAVFS_RTAVFS_REG_STATUS__RTAVFS_RD_DATA_VALID_MASK                                                  0x00000002L
+//GRTAVFS_TARG_FREQ
+#define GRTAVFS_TARG_FREQ__TARGET_FREQUENCY__SHIFT                                                            0x0
+#define GRTAVFS_TARG_FREQ__REQUEST__SHIFT                                                                     0x10
+#define GRTAVFS_TARG_FREQ__RESERVED__SHIFT                                                                    0x11
+#define GRTAVFS_TARG_FREQ__TARGET_FREQUENCY_MASK                                                              0x0000FFFFL
+#define GRTAVFS_TARG_FREQ__REQUEST_MASK                                                                       0x00010000L
+#define GRTAVFS_TARG_FREQ__RESERVED_MASK                                                                      0xFFFE0000L
+//GRTAVFS_TARG_VOLT
+#define GRTAVFS_TARG_VOLT__TARGET_VOLTAGE__SHIFT                                                              0x0
+#define GRTAVFS_TARG_VOLT__VALID__SHIFT                                                                       0xa
+#define GRTAVFS_TARG_VOLT__RESERVED__SHIFT                                                                    0xb
+#define GRTAVFS_TARG_VOLT__TARGET_VOLTAGE_MASK                                                                0x000003FFL
+#define GRTAVFS_TARG_VOLT__VALID_MASK                                                                         0x00000400L
+#define GRTAVFS_TARG_VOLT__RESERVED_MASK                                                                      0xFFFFF800L
+//GRTAVFS_SOFT_RESET
+#define GRTAVFS_SOFT_RESET__RESETN_OVERRIDE__SHIFT                                                            0x0
+#define GRTAVFS_SOFT_RESET__RESERVED__SHIFT                                                                   0x1
+#define GRTAVFS_SOFT_RESET__RESETN_OVERRIDE_MASK                                                              0x00000001L
+#define GRTAVFS_SOFT_RESET__RESERVED_MASK                                                                     0xFFFFFFFEL
+//GRTAVFS_PSM_CNTL
+#define GRTAVFS_PSM_CNTL__PSM_COUNT__SHIFT                                                                    0x0
+#define GRTAVFS_PSM_CNTL__PSM_SAMPLE_EN__SHIFT                                                                0xe
+#define GRTAVFS_PSM_CNTL__RESERVED__SHIFT                                                                     0xf
+#define GRTAVFS_PSM_CNTL__PSM_COUNT_MASK                                                                      0x00003FFFL
+#define GRTAVFS_PSM_CNTL__PSM_SAMPLE_EN_MASK                                                                  0x00004000L
+#define GRTAVFS_PSM_CNTL__RESERVED_MASK                                                                       0xFFFF8000L
+//GRTAVFS_CLK_CNTL
+#define GRTAVFS_CLK_CNTL__GRTAVFS_MUX_CLK_SEL__SHIFT                                                          0x0
+#define GRTAVFS_CLK_CNTL__FORCE_GRTAVFS_CLK_SEL__SHIFT                                                        0x1
+#define GRTAVFS_CLK_CNTL__RESERVED__SHIFT                                                                     0x2
+#define GRTAVFS_CLK_CNTL__GRTAVFS_MUX_CLK_SEL_MASK                                                            0x00000001L
+#define GRTAVFS_CLK_CNTL__FORCE_GRTAVFS_CLK_SEL_MASK                                                          0x00000002L
+#define GRTAVFS_CLK_CNTL__RESERVED_MASK                                                                       0xFFFFFFFCL
+
+
+// addressBlock: gc_grtavfs_se_grtavfs_dec
+//GRTAVFS_SE_RTAVFS_REG_ADDR
+#define GRTAVFS_SE_RTAVFS_REG_ADDR__RTAVFSADDR__SHIFT                                                         0x0
+#define GRTAVFS_SE_RTAVFS_REG_ADDR__RTAVFSADDR_MASK                                                           0x000003FFL
+//GRTAVFS_SE_RTAVFS_WR_DATA
+#define GRTAVFS_SE_RTAVFS_WR_DATA__RTAVFSDATA__SHIFT                                                          0x0
+#define GRTAVFS_SE_RTAVFS_WR_DATA__RTAVFSDATA_MASK                                                            0xFFFFFFFFL
+//GRTAVFS_SE_GENERAL_0
+#define GRTAVFS_SE_GENERAL_0__DATA__SHIFT                                                                     0x0
+#define GRTAVFS_SE_GENERAL_0__DATA_MASK                                                                       0xFFFFFFFFL
+//GRTAVFS_SE_RTAVFS_RD_DATA
+#define GRTAVFS_SE_RTAVFS_RD_DATA__RTAVFSDATA__SHIFT                                                          0x0
+#define GRTAVFS_SE_RTAVFS_RD_DATA__RTAVFSDATA_MASK                                                            0xFFFFFFFFL
+//GRTAVFS_SE_RTAVFS_REG_CTRL
+#define GRTAVFS_SE_RTAVFS_REG_CTRL__SET_WR_EN__SHIFT                                                          0x0
+#define GRTAVFS_SE_RTAVFS_REG_CTRL__SET_RD_EN__SHIFT                                                          0x1
+#define GRTAVFS_SE_RTAVFS_REG_CTRL__SET_WR_EN_MASK                                                            0x00000001L
+#define GRTAVFS_SE_RTAVFS_REG_CTRL__SET_RD_EN_MASK                                                            0x00000002L
+//GRTAVFS_SE_RTAVFS_REG_STATUS
+#define GRTAVFS_SE_RTAVFS_REG_STATUS__RTAVFS_WR_ACK__SHIFT                                                    0x0
+#define GRTAVFS_SE_RTAVFS_REG_STATUS__RTAVFS_RD_DATA_VALID__SHIFT                                             0x1
+#define GRTAVFS_SE_RTAVFS_REG_STATUS__RTAVFS_WR_ACK_MASK                                                      0x00000001L
+#define GRTAVFS_SE_RTAVFS_REG_STATUS__RTAVFS_RD_DATA_VALID_MASK                                               0x00000002L
+//GRTAVFS_SE_TARG_FREQ
+#define GRTAVFS_SE_TARG_FREQ__TARGET_FREQUENCY__SHIFT                                                         0x0
+#define GRTAVFS_SE_TARG_FREQ__REQUEST__SHIFT                                                                  0x10
+#define GRTAVFS_SE_TARG_FREQ__RESERVED__SHIFT                                                                 0x11
+#define GRTAVFS_SE_TARG_FREQ__TARGET_FREQUENCY_MASK                                                           0x0000FFFFL
+#define GRTAVFS_SE_TARG_FREQ__REQUEST_MASK                                                                    0x00010000L
+#define GRTAVFS_SE_TARG_FREQ__RESERVED_MASK                                                                   0xFFFE0000L
+//GRTAVFS_SE_TARG_VOLT
+#define GRTAVFS_SE_TARG_VOLT__TARGET_VOLTAGE__SHIFT                                                           0x0
+#define GRTAVFS_SE_TARG_VOLT__VALID__SHIFT                                                                    0xa
+#define GRTAVFS_SE_TARG_VOLT__RESERVED__SHIFT                                                                 0xb
+#define GRTAVFS_SE_TARG_VOLT__TARGET_VOLTAGE_MASK                                                             0x000003FFL
+#define GRTAVFS_SE_TARG_VOLT__VALID_MASK                                                                      0x00000400L
+#define GRTAVFS_SE_TARG_VOLT__RESERVED_MASK                                                                   0xFFFFF800L
+//GRTAVFS_SE_SOFT_RESET
+#define GRTAVFS_SE_SOFT_RESET__RESETN_OVERRIDE__SHIFT                                                         0x0
+#define GRTAVFS_SE_SOFT_RESET__RESERVED__SHIFT                                                                0x1
+#define GRTAVFS_SE_SOFT_RESET__RESETN_OVERRIDE_MASK                                                           0x00000001L
+#define GRTAVFS_SE_SOFT_RESET__RESERVED_MASK                                                                  0xFFFFFFFEL
+//GRTAVFS_SE_PSM_CNTL
+#define GRTAVFS_SE_PSM_CNTL__PSM_COUNT__SHIFT                                                                 0x0
+#define GRTAVFS_SE_PSM_CNTL__PSM_SAMPLE_EN__SHIFT                                                             0xe
+#define GRTAVFS_SE_PSM_CNTL__RESERVED__SHIFT                                                                  0xf
+#define GRTAVFS_SE_PSM_CNTL__PSM_COUNT_MASK                                                                   0x00003FFFL
+#define GRTAVFS_SE_PSM_CNTL__PSM_SAMPLE_EN_MASK                                                               0x00004000L
+#define GRTAVFS_SE_PSM_CNTL__RESERVED_MASK                                                                    0xFFFF8000L
+//GRTAVFS_SE_CLK_CNTL
+#define GRTAVFS_SE_CLK_CNTL__GRTAVFS_MUX_CLK_SEL__SHIFT                                                       0x0
+#define GRTAVFS_SE_CLK_CNTL__FORCE_GRTAVFS_CLK_SEL__SHIFT                                                     0x1
+#define GRTAVFS_SE_CLK_CNTL__RESERVED__SHIFT                                                                  0x2
+#define GRTAVFS_SE_CLK_CNTL__GRTAVFS_MUX_CLK_SEL_MASK                                                         0x00000001L
+#define GRTAVFS_SE_CLK_CNTL__FORCE_GRTAVFS_CLK_SEL_MASK                                                       0x00000002L
+#define GRTAVFS_SE_CLK_CNTL__RESERVED_MASK                                                                    0xFFFFFFFCL
+
+
+// addressBlock: gc_grtavfsdec
+//RTAVFS_RTAVFS_REG_ADDR
+#define RTAVFS_RTAVFS_REG_ADDR__RTAVFSADDR__SHIFT                                                             0x0
+#define RTAVFS_RTAVFS_REG_ADDR__RTAVFSADDR_MASK                                                               0x000003FFL
+//RTAVFS_RTAVFS_WR_DATA
+#define RTAVFS_RTAVFS_WR_DATA__RTAVFSDATA__SHIFT                                                              0x0
+#define RTAVFS_RTAVFS_WR_DATA__RTAVFSDATA_MASK                                                                0xFFFFFFFFL
+
+
+// addressBlock: gc_hypdec
+//GFX_PIPE_PRIORITY
+#define GFX_PIPE_PRIORITY__HP_PIPE_SELECT__SHIFT                                                              0x0
+#define GFX_PIPE_PRIORITY__HP_PIPE_SELECT_MASK                                                                0x00000001L
+//RLC_GPU_IOV_VF_ENABLE
+#define RLC_GPU_IOV_VF_ENABLE__VF_ENABLE__SHIFT                                                               0x0
+#define RLC_GPU_IOV_VF_ENABLE__RESERVED__SHIFT                                                                0x1
+#define RLC_GPU_IOV_VF_ENABLE__VF_NUM__SHIFT                                                                  0x10
+#define RLC_GPU_IOV_VF_ENABLE__VF_ENABLE_MASK                                                                 0x00000001L
+#define RLC_GPU_IOV_VF_ENABLE__RESERVED_MASK                                                                  0x0000FFFEL
+#define RLC_GPU_IOV_VF_ENABLE__VF_NUM_MASK                                                                    0xFFFF0000L
+//RLC_GPU_IOV_CFG_REG6
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_SIZE__SHIFT                                                               0x0
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_LOCATION__SHIFT                                                           0x7
+#define RLC_GPU_IOV_CFG_REG6__RESERVED__SHIFT                                                                 0x8
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_OFFSET__SHIFT                                                             0xa
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_SIZE_MASK                                                                 0x0000007FL
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_LOCATION_MASK                                                             0x00000080L
+#define RLC_GPU_IOV_CFG_REG6__RESERVED_MASK                                                                   0x00000300L
+#define RLC_GPU_IOV_CFG_REG6__CNTXT_OFFSET_MASK                                                               0xFFFFFC00L
+//RLC_SDMA0_STATUS
+#define RLC_SDMA0_STATUS__STATUS__SHIFT                                                                       0x0
+#define RLC_SDMA0_STATUS__STATUS_MASK                                                                         0xFFFFFFFFL
+//RLC_SDMA1_STATUS
+#define RLC_SDMA1_STATUS__STATUS__SHIFT                                                                       0x0
+#define RLC_SDMA1_STATUS__STATUS_MASK                                                                         0xFFFFFFFFL
+//RLC_SDMA2_STATUS
+#define RLC_SDMA2_STATUS__STATUS__SHIFT                                                                       0x0
+#define RLC_SDMA2_STATUS__STATUS_MASK                                                                         0xFFFFFFFFL
+//RLC_SDMA3_STATUS
+#define RLC_SDMA3_STATUS__STATUS__SHIFT                                                                       0x0
+#define RLC_SDMA3_STATUS__STATUS_MASK                                                                         0xFFFFFFFFL
+//RLC_SDMA0_BUSY_STATUS
+#define RLC_SDMA0_BUSY_STATUS__BUSY_STATUS__SHIFT                                                             0x0
+#define RLC_SDMA0_BUSY_STATUS__BUSY_STATUS_MASK                                                               0xFFFFFFFFL
+//RLC_SDMA1_BUSY_STATUS
+#define RLC_SDMA1_BUSY_STATUS__BUSY_STATUS__SHIFT                                                             0x0
+#define RLC_SDMA1_BUSY_STATUS__BUSY_STATUS_MASK                                                               0xFFFFFFFFL
+//RLC_SDMA2_BUSY_STATUS
+#define RLC_SDMA2_BUSY_STATUS__BUSY_STATUS__SHIFT                                                             0x0
+#define RLC_SDMA2_BUSY_STATUS__BUSY_STATUS_MASK                                                               0xFFFFFFFFL
+//RLC_SDMA3_BUSY_STATUS
+#define RLC_SDMA3_BUSY_STATUS__BUSY_STATUS__SHIFT                                                             0x0
+#define RLC_SDMA3_BUSY_STATUS__BUSY_STATUS_MASK                                                               0xFFFFFFFFL
+//RLC_GPU_IOV_CFG_REG8
+#define RLC_GPU_IOV_CFG_REG8__VM_BUSY_STATUS__SHIFT                                                           0x0
+#define RLC_GPU_IOV_CFG_REG8__VM_BUSY_STATUS_MASK                                                             0xFFFFFFFFL
+//RLC_RLCV_TIMER_INT_0
+#define RLC_RLCV_TIMER_INT_0__TIMER__SHIFT                                                                    0x0
+#define RLC_RLCV_TIMER_INT_0__TIMER_MASK                                                                      0xFFFFFFFFL
+//RLC_RLCV_TIMER_INT_1
+#define RLC_RLCV_TIMER_INT_1__TIMER__SHIFT                                                                    0x0
+#define RLC_RLCV_TIMER_INT_1__TIMER_MASK                                                                      0xFFFFFFFFL
+//RLC_RLCV_TIMER_CTRL
+#define RLC_RLCV_TIMER_CTRL__TIMER_0_EN__SHIFT                                                                0x0
+#define RLC_RLCV_TIMER_CTRL__TIMER_1_EN__SHIFT                                                                0x1
+#define RLC_RLCV_TIMER_CTRL__TIMER_0_AUTO_REARM__SHIFT                                                        0x2
+#define RLC_RLCV_TIMER_CTRL__TIMER_1_AUTO_REARM__SHIFT                                                        0x3
+#define RLC_RLCV_TIMER_CTRL__TIMER_0_INT_CLEAR__SHIFT                                                         0x4
+#define RLC_RLCV_TIMER_CTRL__TIMER_1_INT_CLEAR__SHIFT                                                         0x5
+#define RLC_RLCV_TIMER_CTRL__RESERVED__SHIFT                                                                  0x6
+#define RLC_RLCV_TIMER_CTRL__TIMER_0_EN_MASK                                                                  0x00000001L
+#define RLC_RLCV_TIMER_CTRL__TIMER_1_EN_MASK                                                                  0x00000002L
+#define RLC_RLCV_TIMER_CTRL__TIMER_0_AUTO_REARM_MASK                                                          0x00000004L
+#define RLC_RLCV_TIMER_CTRL__TIMER_1_AUTO_REARM_MASK                                                          0x00000008L
+#define RLC_RLCV_TIMER_CTRL__TIMER_0_INT_CLEAR_MASK                                                           0x00000010L
+#define RLC_RLCV_TIMER_CTRL__TIMER_1_INT_CLEAR_MASK                                                           0x00000020L
+#define RLC_RLCV_TIMER_CTRL__RESERVED_MASK                                                                    0xFFFFFFC0L
+//RLC_RLCV_TIMER_STAT
+#define RLC_RLCV_TIMER_STAT__TIMER_0_STAT__SHIFT                                                              0x0
+#define RLC_RLCV_TIMER_STAT__TIMER_1_STAT__SHIFT                                                              0x1
+#define RLC_RLCV_TIMER_STAT__RESERVED__SHIFT                                                                  0x2
+#define RLC_RLCV_TIMER_STAT__TIMER_0_ENABLE_SYNC__SHIFT                                                       0x8
+#define RLC_RLCV_TIMER_STAT__TIMER_1_ENABLE_SYNC__SHIFT                                                       0x9
+#define RLC_RLCV_TIMER_STAT__TIMER_0_AUTO_REARM_SYNC__SHIFT                                                   0xa
+#define RLC_RLCV_TIMER_STAT__TIMER_1_AUTO_REARM_SYNC__SHIFT                                                   0xb
+#define RLC_RLCV_TIMER_STAT__TIMER_0_STAT_MASK                                                                0x00000001L
+#define RLC_RLCV_TIMER_STAT__TIMER_1_STAT_MASK                                                                0x00000002L
+#define RLC_RLCV_TIMER_STAT__RESERVED_MASK                                                                    0x000000FCL
+#define RLC_RLCV_TIMER_STAT__TIMER_0_ENABLE_SYNC_MASK                                                         0x00000100L
+#define RLC_RLCV_TIMER_STAT__TIMER_1_ENABLE_SYNC_MASK                                                         0x00000200L
+#define RLC_RLCV_TIMER_STAT__TIMER_0_AUTO_REARM_SYNC_MASK                                                     0x00000400L
+#define RLC_RLCV_TIMER_STAT__TIMER_1_AUTO_REARM_SYNC_MASK                                                     0x00000800L
+//RLC_GPU_IOV_VF_DOORBELL_STATUS
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS__SHIFT                                             0x0
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__PF_DOORBELL_STATUS__SHIFT                                             0x1f
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_MASK                                               0x7FFFFFFFL
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS__PF_DOORBELL_STATUS_MASK                                               0x80000000L
+//RLC_GPU_IOV_VF_DOORBELL_STATUS_SET
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__VF_DOORBELL_STATUS_SET__SHIFT                                     0x0
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__PF_DOORBELL_STATUS_SET__SHIFT                                     0x1f
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__VF_DOORBELL_STATUS_SET_MASK                                       0x7FFFFFFFL
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_SET__PF_DOORBELL_STATUS_SET_MASK                                       0x80000000L
+//RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__VF_DOORBELL_STATUS_CLR__SHIFT                                     0x0
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__PF_DOORBELL_STATUS_CLR__SHIFT                                     0x1f
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__VF_DOORBELL_STATUS_CLR_MASK                                       0x7FFFFFFFL
+#define RLC_GPU_IOV_VF_DOORBELL_STATUS_CLR__PF_DOORBELL_STATUS_CLR_MASK                                       0x80000000L
+//RLC_GPU_IOV_VF_MASK
+#define RLC_GPU_IOV_VF_MASK__VF_MASK__SHIFT                                                                   0x0
+#define RLC_GPU_IOV_VF_MASK__VF_MASK_MASK                                                                     0x7FFFFFFFL
+//RLC_HYP_SEMAPHORE_0
+#define RLC_HYP_SEMAPHORE_0__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_0__RESERVED__SHIFT                                                                  0x5
+#define RLC_HYP_SEMAPHORE_0__CLIENT_ID_MASK                                                                   0x0000001FL
+#define RLC_HYP_SEMAPHORE_0__RESERVED_MASK                                                                    0xFFFFFFE0L
+//RLC_HYP_SEMAPHORE_1
+#define RLC_HYP_SEMAPHORE_1__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_1__RESERVED__SHIFT                                                                  0x5
+#define RLC_HYP_SEMAPHORE_1__CLIENT_ID_MASK                                                                   0x0000001FL
+#define RLC_HYP_SEMAPHORE_1__RESERVED_MASK                                                                    0xFFFFFFE0L
+//RLC_BUSY_CLK_CNTL
+#define RLC_BUSY_CLK_CNTL__BUSY_OFF_LATENCY__SHIFT                                                            0x0
+#define RLC_BUSY_CLK_CNTL__GRBM_BUSY_OFF_LATENCY__SHIFT                                                       0x8
+#define RLC_BUSY_CLK_CNTL__BUSY_OFF_LATENCY_MASK                                                              0x0000003FL
+#define RLC_BUSY_CLK_CNTL__GRBM_BUSY_OFF_LATENCY_MASK                                                         0x00003F00L
+//RLC_CLK_CNTL
+#define RLC_CLK_CNTL__RLC_SRM_ICG_OVERRIDE__SHIFT                                                             0x0
+#define RLC_CLK_CNTL__RLC_IMU_ICG_OVERRIDE__SHIFT                                                             0x1
+#define RLC_CLK_CNTL__RLC_SPM_ICG_OVERRIDE__SHIFT                                                             0x2
+#define RLC_CLK_CNTL__RLC_SPM_RSPM_ICG_OVERRIDE__SHIFT                                                        0x3
+#define RLC_CLK_CNTL__RLC_GPM_ICG_OVERRIDE__SHIFT                                                             0x4
+#define RLC_CLK_CNTL__RLC_CMN_ICG_OVERRIDE__SHIFT                                                             0x5
+#define RLC_CLK_CNTL__RLC_TC_ICG_OVERRIDE__SHIFT                                                              0x6
+#define RLC_CLK_CNTL__RLC_REG_ICG_OVERRIDE__SHIFT                                                             0x7
+#define RLC_CLK_CNTL__RLC_SRAM_CLK_GATER_OVERRIDE__SHIFT                                                      0x8
+#define RLC_CLK_CNTL__RESERVED_9__SHIFT                                                                       0x9
+#define RLC_CLK_CNTL__RLC_SPP_ICG_OVERRIDE__SHIFT                                                             0xa
+#define RLC_CLK_CNTL__RESERVED_11__SHIFT                                                                      0xb
+#define RLC_CLK_CNTL__RLC_TC_FGCG_REP_OVERRIDE__SHIFT                                                         0xc
+#define RLC_CLK_CNTL__RLC_DFLL_ICG_OVERRIDE__SHIFT                                                            0xd
+#define RLC_CLK_CNTL__RESERVED_15__SHIFT                                                                      0xf
+#define RLC_CLK_CNTL__RLC_LX6_CORE_ICG_OVERRIDE__SHIFT                                                        0x10
+#define RLC_CLK_CNTL__RLC_LX6_ICG_OVERRIDE__SHIFT                                                             0x11
+#define RLC_CLK_CNTL__RLC_UTCL2_FGCG_OVERRIDE__SHIFT                                                          0x12
+#define RLC_CLK_CNTL__RLC_IH_GASKET_ICG_OVERRIDE__SHIFT                                                       0x13
+#define RLC_CLK_CNTL__RESERVED__SHIFT                                                                         0x14
+#define RLC_CLK_CNTL__RLC_SRM_ICG_OVERRIDE_MASK                                                               0x00000001L
+#define RLC_CLK_CNTL__RLC_IMU_ICG_OVERRIDE_MASK                                                               0x00000002L
+#define RLC_CLK_CNTL__RLC_SPM_ICG_OVERRIDE_MASK                                                               0x00000004L
+#define RLC_CLK_CNTL__RLC_SPM_RSPM_ICG_OVERRIDE_MASK                                                          0x00000008L
+#define RLC_CLK_CNTL__RLC_GPM_ICG_OVERRIDE_MASK                                                               0x00000010L
+#define RLC_CLK_CNTL__RLC_CMN_ICG_OVERRIDE_MASK                                                               0x00000020L
+#define RLC_CLK_CNTL__RLC_TC_ICG_OVERRIDE_MASK                                                                0x00000040L
+#define RLC_CLK_CNTL__RLC_REG_ICG_OVERRIDE_MASK                                                               0x00000080L
+#define RLC_CLK_CNTL__RLC_SRAM_CLK_GATER_OVERRIDE_MASK                                                        0x00000100L
+#define RLC_CLK_CNTL__RESERVED_9_MASK                                                                         0x00000200L
+#define RLC_CLK_CNTL__RLC_SPP_ICG_OVERRIDE_MASK                                                               0x00000400L
+#define RLC_CLK_CNTL__RESERVED_11_MASK                                                                        0x00000800L
+#define RLC_CLK_CNTL__RLC_TC_FGCG_REP_OVERRIDE_MASK                                                           0x00001000L
+#define RLC_CLK_CNTL__RLC_DFLL_ICG_OVERRIDE_MASK                                                              0x00002000L
+#define RLC_CLK_CNTL__RESERVED_15_MASK                                                                        0x00008000L
+#define RLC_CLK_CNTL__RLC_LX6_CORE_ICG_OVERRIDE_MASK                                                          0x00010000L
+#define RLC_CLK_CNTL__RLC_LX6_ICG_OVERRIDE_MASK                                                               0x00020000L
+#define RLC_CLK_CNTL__RLC_UTCL2_FGCG_OVERRIDE_MASK                                                            0x00040000L
+#define RLC_CLK_CNTL__RLC_IH_GASKET_ICG_OVERRIDE_MASK                                                         0x00080000L
+#define RLC_CLK_CNTL__RESERVED_MASK                                                                           0xFFF00000L
+//RLC_PACE_TIMER_STAT
+#define RLC_PACE_TIMER_STAT__TIMER_0_STAT__SHIFT                                                              0x0
+#define RLC_PACE_TIMER_STAT__TIMER_1_STAT__SHIFT                                                              0x1
+#define RLC_PACE_TIMER_STAT__RESERVED__SHIFT                                                                  0x2
+#define RLC_PACE_TIMER_STAT__TIMER_0_ENABLE_SYNC__SHIFT                                                       0x8
+#define RLC_PACE_TIMER_STAT__TIMER_1_ENABLE_SYNC__SHIFT                                                       0x9
+#define RLC_PACE_TIMER_STAT__TIMER_0_AUTO_REARM_SYNC__SHIFT                                                   0xa
+#define RLC_PACE_TIMER_STAT__TIMER_1_AUTO_REARM_SYNC__SHIFT                                                   0xb
+#define RLC_PACE_TIMER_STAT__TIMER_0_STAT_MASK                                                                0x00000001L
+#define RLC_PACE_TIMER_STAT__TIMER_1_STAT_MASK                                                                0x00000002L
+#define RLC_PACE_TIMER_STAT__RESERVED_MASK                                                                    0x000000FCL
+#define RLC_PACE_TIMER_STAT__TIMER_0_ENABLE_SYNC_MASK                                                         0x00000100L
+#define RLC_PACE_TIMER_STAT__TIMER_1_ENABLE_SYNC_MASK                                                         0x00000200L
+#define RLC_PACE_TIMER_STAT__TIMER_0_AUTO_REARM_SYNC_MASK                                                     0x00000400L
+#define RLC_PACE_TIMER_STAT__TIMER_1_AUTO_REARM_SYNC_MASK                                                     0x00000800L
+//RLC_GPU_IOV_SCH_BLOCK
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_ID__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Ver__SHIFT                                                           0x4
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Size__SHIFT                                                          0x8
+#define RLC_GPU_IOV_SCH_BLOCK__RESERVED__SHIFT                                                                0x10
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_ID_MASK                                                              0x0000000FL
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Ver_MASK                                                             0x000000F0L
+#define RLC_GPU_IOV_SCH_BLOCK__Sch_Block_Size_MASK                                                            0x0000FF00L
+#define RLC_GPU_IOV_SCH_BLOCK__RESERVED_MASK                                                                  0xFFFF0000L
+//RLC_GPU_IOV_CFG_REG1
+#define RLC_GPU_IOV_CFG_REG1__CMD_TYPE__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE__SHIFT                                                              0x4
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_INTR_EN__SHIFT                                                      0x5
+#define RLC_GPU_IOV_CFG_REG1__RESERVED__SHIFT                                                                 0x6
+#define RLC_GPU_IOV_CFG_REG1__FCN_ID__SHIFT                                                                   0x8
+#define RLC_GPU_IOV_CFG_REG1__NEXT_FCN_ID__SHIFT                                                              0x10
+#define RLC_GPU_IOV_CFG_REG1__RESERVED1__SHIFT                                                                0x18
+#define RLC_GPU_IOV_CFG_REG1__CMD_TYPE_MASK                                                                   0x0000000FL
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_MASK                                                                0x00000010L
+#define RLC_GPU_IOV_CFG_REG1__CMD_EXECUTE_INTR_EN_MASK                                                        0x00000020L
+#define RLC_GPU_IOV_CFG_REG1__RESERVED_MASK                                                                   0x000000C0L
+#define RLC_GPU_IOV_CFG_REG1__FCN_ID_MASK                                                                     0x0000FF00L
+#define RLC_GPU_IOV_CFG_REG1__NEXT_FCN_ID_MASK                                                                0x00FF0000L
+#define RLC_GPU_IOV_CFG_REG1__RESERVED1_MASK                                                                  0xFF000000L
+//RLC_GPU_IOV_CFG_REG2
+#define RLC_GPU_IOV_CFG_REG2__CMD_STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_CFG_REG2__RESERVED__SHIFT                                                                 0x4
+#define RLC_GPU_IOV_CFG_REG2__CMD_STATUS_MASK                                                                 0x0000000FL
+#define RLC_GPU_IOV_CFG_REG2__RESERVED_MASK                                                                   0xFFFFFFF0L
+//RLC_GPU_IOV_VM_BUSY_STATUS
+#define RLC_GPU_IOV_VM_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                     0x0
+#define RLC_GPU_IOV_VM_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                       0xFFFFFFFFL
+//RLC_GPU_IOV_SCH_0
+#define RLC_GPU_IOV_SCH_0__ACTIVE_FUNCTIONS__SHIFT                                                            0x0
+#define RLC_GPU_IOV_SCH_0__ACTIVE_FUNCTIONS_MASK                                                              0xFFFFFFFFL
+//RLC_GPU_IOV_ACTIVE_FCN_ID
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__VF_ID__SHIFT                                                               0x0
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_7_4__SHIFT                                                        0x4
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__ACTIVE_FCN_ID_STATUS__SHIFT                                                0x8
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_30_12__SHIFT                                                      0xc
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__PF_VF__SHIFT                                                               0x1f
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__VF_ID_MASK                                                                 0x0000000FL
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_7_4_MASK                                                          0x000000F0L
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__ACTIVE_FCN_ID_STATUS_MASK                                                  0x00000F00L
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__RESERVED_30_12_MASK                                                        0x7FFFF000L
+#define RLC_GPU_IOV_ACTIVE_FCN_ID__PF_VF_MASK                                                                 0x80000000L
+//RLC_GPU_IOV_SCH_3
+#define RLC_GPU_IOV_SCH_3__Time_Quanta_Def__SHIFT                                                             0x0
+#define RLC_GPU_IOV_SCH_3__Time_Quanta_Def_MASK                                                               0xFFFFFFFFL
+//RLC_GPU_IOV_SCH_1
+#define RLC_GPU_IOV_SCH_1__DATA__SHIFT                                                                        0x0
+#define RLC_GPU_IOV_SCH_1__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPU_IOV_SCH_2
+#define RLC_GPU_IOV_SCH_2__DATA__SHIFT                                                                        0x0
+#define RLC_GPU_IOV_SCH_2__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_PACE_INT_FORCE
+#define RLC_PACE_INT_FORCE__FORCE_INT__SHIFT                                                                  0x0
+#define RLC_PACE_INT_FORCE__FORCE_INT_MASK                                                                    0xFFFFFFFFL
+//RLC_PACE_INT_CLEAR
+#define RLC_PACE_INT_CLEAR__SMU_STRETCH_PCC_CLEAR__SHIFT                                                      0x0
+#define RLC_PACE_INT_CLEAR__SMU_PCC_CLEAR__SHIFT                                                              0x1
+#define RLC_PACE_INT_CLEAR__SMU_STRETCH_PCC_CLEAR_MASK                                                        0x00000001L
+#define RLC_PACE_INT_CLEAR__SMU_PCC_CLEAR_MASK                                                                0x00000002L
+//RLC_GPU_IOV_INT_STAT
+#define RLC_GPU_IOV_INT_STAT__STATUS__SHIFT                                                                   0x0
+#define RLC_GPU_IOV_INT_STAT__STATUS_MASK                                                                     0xFFFFFFFFL
+//RLC_IH_COOKIE
+#define RLC_IH_COOKIE__DATA__SHIFT                                                                            0x0
+#define RLC_IH_COOKIE__DATA_MASK                                                                              0xFFFFFFFFL
+//RLC_IH_COOKIE_CNTL
+#define RLC_IH_COOKIE_CNTL__CREDIT__SHIFT                                                                     0x0
+#define RLC_IH_COOKIE_CNTL__RESET_COUNTER__SHIFT                                                              0x2
+#define RLC_IH_COOKIE_CNTL__CREDIT_MASK                                                                       0x00000003L
+#define RLC_IH_COOKIE_CNTL__RESET_COUNTER_MASK                                                                0x00000004L
+//RLC_HYP_RLCG_UCODE_CHKSUM
+#define RLC_HYP_RLCG_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                        0x0
+#define RLC_HYP_RLCG_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                          0xFFFFFFFFL
+//RLC_HYP_RLCP_UCODE_CHKSUM
+#define RLC_HYP_RLCP_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                        0x0
+#define RLC_HYP_RLCP_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                          0xFFFFFFFFL
+//RLC_HYP_RLCV_UCODE_CHKSUM
+#define RLC_HYP_RLCV_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                        0x0
+#define RLC_HYP_RLCV_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                          0xFFFFFFFFL
+//RLC_GPU_IOV_F32_CNTL
+#define RLC_GPU_IOV_F32_CNTL__ENABLE__SHIFT                                                                   0x0
+#define RLC_GPU_IOV_F32_CNTL__ENABLE_MASK                                                                     0x00000001L
+//RLC_GPU_IOV_F32_RESET
+#define RLC_GPU_IOV_F32_RESET__RESET__SHIFT                                                                   0x0
+#define RLC_GPU_IOV_F32_RESET__RESET_MASK                                                                     0x00000001L
+//RLC_GPU_IOV_UCODE_ADDR
+#define RLC_GPU_IOV_UCODE_ADDR__UCODE_ADDR__SHIFT                                                             0x0
+#define RLC_GPU_IOV_UCODE_ADDR__RESERVED__SHIFT                                                               0xc
+#define RLC_GPU_IOV_UCODE_ADDR__UCODE_ADDR_MASK                                                               0x00000FFFL
+#define RLC_GPU_IOV_UCODE_ADDR__RESERVED_MASK                                                                 0xFFFFF000L
+//RLC_GPU_IOV_UCODE_DATA
+#define RLC_GPU_IOV_UCODE_DATA__UCODE_DATA__SHIFT                                                             0x0
+#define RLC_GPU_IOV_UCODE_DATA__UCODE_DATA_MASK                                                               0xFFFFFFFFL
+//RLC_GPU_IOV_SMU_RESPONSE
+#define RLC_GPU_IOV_SMU_RESPONSE__RESP__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_SMU_RESPONSE__RESP_MASK                                                                   0xFFFFFFFFL
+//RLC_GPU_IOV_F32_INVALIDATE_CACHE
+#define RLC_GPU_IOV_F32_INVALIDATE_CACHE__INVALIDATE_CACHE__SHIFT                                             0x0
+#define RLC_GPU_IOV_F32_INVALIDATE_CACHE__INVALIDATE_CACHE_MASK                                               0x00000001L
+//RLC_GPU_IOV_VIRT_RESET_REQ
+#define RLC_GPU_IOV_VIRT_RESET_REQ__VF_FLR__SHIFT                                                             0x0
+#define RLC_GPU_IOV_VIRT_RESET_REQ__SOFT_PF_FLR__SHIFT                                                        0x1f
+#define RLC_GPU_IOV_VIRT_RESET_REQ__VF_FLR_MASK                                                               0x7FFFFFFFL
+#define RLC_GPU_IOV_VIRT_RESET_REQ__SOFT_PF_FLR_MASK                                                          0x80000000L
+//RLC_GPU_IOV_RLC_RESPONSE
+#define RLC_GPU_IOV_RLC_RESPONSE__RESP__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_RLC_RESPONSE__RESP_MASK                                                                   0xFFFFFFFFL
+//RLC_GPU_IOV_INT_DISABLE
+#define RLC_GPU_IOV_INT_DISABLE__DISABLE_INT__SHIFT                                                           0x0
+#define RLC_GPU_IOV_INT_DISABLE__DISABLE_INT_MASK                                                             0xFFFFFFFFL
+//RLC_GPU_IOV_INT_FORCE
+#define RLC_GPU_IOV_INT_FORCE__FORCE_INT__SHIFT                                                               0x0
+#define RLC_GPU_IOV_INT_FORCE__FORCE_INT_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SCRATCH_ADDR
+#define RLC_GPU_IOV_SCRATCH_ADDR__ADDR__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_SCRATCH_ADDR__ADDR_MASK                                                                   0x0000FFFFL
+//RLC_GPU_IOV_SCRATCH_DATA
+#define RLC_GPU_IOV_SCRATCH_DATA__DATA__SHIFT                                                                 0x0
+#define RLC_GPU_IOV_SCRATCH_DATA__DATA_MASK                                                                   0xFFFFFFFFL
+//RLC_HYP_SEMAPHORE_2
+#define RLC_HYP_SEMAPHORE_2__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_2__RESERVED__SHIFT                                                                  0x5
+#define RLC_HYP_SEMAPHORE_2__CLIENT_ID_MASK                                                                   0x0000001FL
+#define RLC_HYP_SEMAPHORE_2__RESERVED_MASK                                                                    0xFFFFFFE0L
+//RLC_HYP_SEMAPHORE_3
+#define RLC_HYP_SEMAPHORE_3__CLIENT_ID__SHIFT                                                                 0x0
+#define RLC_HYP_SEMAPHORE_3__RESERVED__SHIFT                                                                  0x5
+#define RLC_HYP_SEMAPHORE_3__CLIENT_ID_MASK                                                                   0x0000001FL
+#define RLC_HYP_SEMAPHORE_3__RESERVED_MASK                                                                    0xFFFFFFE0L
+//RLC_LX6_SCRATCH_ADDR
+#define RLC_LX6_SCRATCH_ADDR__ADDR__SHIFT                                                                     0x0
+#define RLC_LX6_SCRATCH_ADDR__ADDR_MASK                                                                       0x000000FFL
+//RLC_LX6_CORE1_SCRATCH_ADDR
+#define RLC_LX6_CORE1_SCRATCH_ADDR__ADDR__SHIFT                                                               0x0
+#define RLC_LX6_CORE1_SCRATCH_ADDR__ADDR_MASK                                                                 0x000000FFL
+//RLC_GPM_UCODE_ADDR
+#define RLC_GPM_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                 0x0
+#define RLC_GPM_UCODE_ADDR__RESERVED__SHIFT                                                                   0xe
+#define RLC_GPM_UCODE_ADDR__UCODE_ADDR_MASK                                                                   0x00003FFFL
+#define RLC_GPM_UCODE_ADDR__RESERVED_MASK                                                                     0xFFFFC000L
+//RLC_GPM_UCODE_DATA
+#define RLC_GPM_UCODE_DATA__UCODE_DATA__SHIFT                                                                 0x0
+#define RLC_GPM_UCODE_DATA__UCODE_DATA_MASK                                                                   0xFFFFFFFFL
+//RLC_GPM_IRAM_ADDR
+#define RLC_GPM_IRAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_GPM_IRAM_ADDR__ADDR_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_IRAM_DATA
+#define RLC_GPM_IRAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_IRAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_RLCP_IRAM_ADDR
+#define RLC_RLCP_IRAM_ADDR__ADDR__SHIFT                                                                       0x0
+#define RLC_RLCP_IRAM_ADDR__ADDR_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCP_IRAM_DATA
+#define RLC_RLCP_IRAM_DATA__DATA__SHIFT                                                                       0x0
+#define RLC_RLCP_IRAM_DATA__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCV_IRAM_ADDR
+#define RLC_RLCV_IRAM_ADDR__ADDR__SHIFT                                                                       0x0
+#define RLC_RLCV_IRAM_ADDR__ADDR_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCV_IRAM_DATA
+#define RLC_RLCV_IRAM_DATA__DATA__SHIFT                                                                       0x0
+#define RLC_RLCV_IRAM_DATA__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_LX6_DRAM_ADDR
+#define RLC_LX6_DRAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_LX6_DRAM_ADDR__ADDR_MASK                                                                          0x000007FFL
+//RLC_LX6_DRAM_DATA
+#define RLC_LX6_DRAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_LX6_DRAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_LX6_IRAM_ADDR
+#define RLC_LX6_IRAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_LX6_IRAM_ADDR__ADDR_MASK                                                                          0x00000FFFL
+//RLC_LX6_IRAM_DATA
+#define RLC_LX6_IRAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_LX6_IRAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_PACE_UCODE_ADDR
+#define RLC_PACE_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                0x0
+#define RLC_PACE_UCODE_ADDR__RESERVED__SHIFT                                                                  0xc
+#define RLC_PACE_UCODE_ADDR__UCODE_ADDR_MASK                                                                  0x00000FFFL
+#define RLC_PACE_UCODE_ADDR__RESERVED_MASK                                                                    0xFFFFF000L
+//RLC_PACE_UCODE_DATA
+#define RLC_PACE_UCODE_DATA__UCODE_DATA__SHIFT                                                                0x0
+#define RLC_PACE_UCODE_DATA__UCODE_DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_GPM_SCRATCH_ADDR
+#define RLC_GPM_SCRATCH_ADDR__ADDR__SHIFT                                                                     0x0
+#define RLC_GPM_SCRATCH_ADDR__ADDR_MASK                                                                       0x0000FFFFL
+//RLC_GPM_SCRATCH_DATA
+#define RLC_GPM_SCRATCH_DATA__DATA__SHIFT                                                                     0x0
+#define RLC_GPM_SCRATCH_DATA__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SRM_DRAM_ADDR
+#define RLC_SRM_DRAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_SRM_DRAM_ADDR__RESERVED__SHIFT                                                                    0xd
+#define RLC_SRM_DRAM_ADDR__ADDR_MASK                                                                          0x00001FFFL
+#define RLC_SRM_DRAM_ADDR__RESERVED_MASK                                                                      0xFFFFE000L
+//RLC_SRM_DRAM_DATA
+#define RLC_SRM_DRAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_SRM_DRAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_SRM_ARAM_ADDR
+#define RLC_SRM_ARAM_ADDR__ADDR__SHIFT                                                                        0x0
+#define RLC_SRM_ARAM_ADDR__RESERVED__SHIFT                                                                    0xd
+#define RLC_SRM_ARAM_ADDR__ADDR_MASK                                                                          0x00001FFFL
+#define RLC_SRM_ARAM_ADDR__RESERVED_MASK                                                                      0xFFFFE000L
+//RLC_SRM_ARAM_DATA
+#define RLC_SRM_ARAM_DATA__DATA__SHIFT                                                                        0x0
+#define RLC_SRM_ARAM_DATA__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_PACE_SCRATCH_ADDR
+#define RLC_PACE_SCRATCH_ADDR__ADDR__SHIFT                                                                    0x0
+#define RLC_PACE_SCRATCH_ADDR__ADDR_MASK                                                                      0x0000FFFFL
+//RLC_PACE_SCRATCH_DATA
+#define RLC_PACE_SCRATCH_DATA__DATA__SHIFT                                                                    0x0
+#define RLC_PACE_SCRATCH_DATA__DATA_MASK                                                                      0xFFFFFFFFL
+//RLC_GTS_OFFSET_LSB
+#define RLC_GTS_OFFSET_LSB__DATA__SHIFT                                                                       0x0
+#define RLC_GTS_OFFSET_LSB__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GTS_OFFSET_MSB
+#define RLC_GTS_OFFSET_MSB__DATA__SHIFT                                                                       0x0
+#define RLC_GTS_OFFSET_MSB__DATA_MASK                                                                         0xFFFFFFFFL
+//GL2_PIPE_STEER_0
+#define GL2_PIPE_STEER_0__PIPE_0_TO_CHAN_IN_Q0__SHIFT                                                         0x0
+#define GL2_PIPE_STEER_0__PIPE_1_TO_CHAN_IN_Q0__SHIFT                                                         0x4
+#define GL2_PIPE_STEER_0__PIPE_2_TO_CHAN_IN_Q0__SHIFT                                                         0x8
+#define GL2_PIPE_STEER_0__PIPE_3_TO_CHAN_IN_Q0__SHIFT                                                         0xc
+#define GL2_PIPE_STEER_0__PIPE_0_TO_CHAN_IN_Q1__SHIFT                                                         0x10
+#define GL2_PIPE_STEER_0__PIPE_1_TO_CHAN_IN_Q1__SHIFT                                                         0x14
+#define GL2_PIPE_STEER_0__PIPE_2_TO_CHAN_IN_Q1__SHIFT                                                         0x18
+#define GL2_PIPE_STEER_0__PIPE_3_TO_CHAN_IN_Q1__SHIFT                                                         0x1c
+#define GL2_PIPE_STEER_0__PIPE_0_TO_CHAN_IN_Q0_MASK                                                           0x00000007L
+#define GL2_PIPE_STEER_0__PIPE_1_TO_CHAN_IN_Q0_MASK                                                           0x00000070L
+#define GL2_PIPE_STEER_0__PIPE_2_TO_CHAN_IN_Q0_MASK                                                           0x00000700L
+#define GL2_PIPE_STEER_0__PIPE_3_TO_CHAN_IN_Q0_MASK                                                           0x00007000L
+#define GL2_PIPE_STEER_0__PIPE_0_TO_CHAN_IN_Q1_MASK                                                           0x00070000L
+#define GL2_PIPE_STEER_0__PIPE_1_TO_CHAN_IN_Q1_MASK                                                           0x00700000L
+#define GL2_PIPE_STEER_0__PIPE_2_TO_CHAN_IN_Q1_MASK                                                           0x07000000L
+#define GL2_PIPE_STEER_0__PIPE_3_TO_CHAN_IN_Q1_MASK                                                           0x70000000L
+//GL2_PIPE_STEER_1
+#define GL2_PIPE_STEER_1__PIPE_0_TO_CHAN_IN_Q2__SHIFT                                                         0x0
+#define GL2_PIPE_STEER_1__PIPE_1_TO_CHAN_IN_Q2__SHIFT                                                         0x4
+#define GL2_PIPE_STEER_1__PIPE_2_TO_CHAN_IN_Q2__SHIFT                                                         0x8
+#define GL2_PIPE_STEER_1__PIPE_3_TO_CHAN_IN_Q2__SHIFT                                                         0xc
+#define GL2_PIPE_STEER_1__PIPE_0_TO_CHAN_IN_Q3__SHIFT                                                         0x10
+#define GL2_PIPE_STEER_1__PIPE_1_TO_CHAN_IN_Q3__SHIFT                                                         0x14
+#define GL2_PIPE_STEER_1__PIPE_2_TO_CHAN_IN_Q3__SHIFT                                                         0x18
+#define GL2_PIPE_STEER_1__PIPE_3_TO_CHAN_IN_Q3__SHIFT                                                         0x1c
+#define GL2_PIPE_STEER_1__PIPE_0_TO_CHAN_IN_Q2_MASK                                                           0x00000007L
+#define GL2_PIPE_STEER_1__PIPE_1_TO_CHAN_IN_Q2_MASK                                                           0x00000070L
+#define GL2_PIPE_STEER_1__PIPE_2_TO_CHAN_IN_Q2_MASK                                                           0x00000700L
+#define GL2_PIPE_STEER_1__PIPE_3_TO_CHAN_IN_Q2_MASK                                                           0x00007000L
+#define GL2_PIPE_STEER_1__PIPE_0_TO_CHAN_IN_Q3_MASK                                                           0x00070000L
+#define GL2_PIPE_STEER_1__PIPE_1_TO_CHAN_IN_Q3_MASK                                                           0x00700000L
+#define GL2_PIPE_STEER_1__PIPE_2_TO_CHAN_IN_Q3_MASK                                                           0x07000000L
+#define GL2_PIPE_STEER_1__PIPE_3_TO_CHAN_IN_Q3_MASK                                                           0x70000000L
+//GL2_PIPE_STEER_2
+#define GL2_PIPE_STEER_2__PIPE_4_TO_CHAN_IN_Q0__SHIFT                                                         0x0
+#define GL2_PIPE_STEER_2__PIPE_5_TO_CHAN_IN_Q0__SHIFT                                                         0x4
+#define GL2_PIPE_STEER_2__PIPE_6_TO_CHAN_IN_Q0__SHIFT                                                         0x8
+#define GL2_PIPE_STEER_2__PIPE_7_TO_CHAN_IN_Q0__SHIFT                                                         0xc
+#define GL2_PIPE_STEER_2__PIPE_4_TO_CHAN_IN_Q1__SHIFT                                                         0x10
+#define GL2_PIPE_STEER_2__PIPE_5_TO_CHAN_IN_Q1__SHIFT                                                         0x14
+#define GL2_PIPE_STEER_2__PIPE_6_TO_CHAN_IN_Q1__SHIFT                                                         0x18
+#define GL2_PIPE_STEER_2__PIPE_7_TO_CHAN_IN_Q1__SHIFT                                                         0x1c
+#define GL2_PIPE_STEER_2__PIPE_4_TO_CHAN_IN_Q0_MASK                                                           0x00000007L
+#define GL2_PIPE_STEER_2__PIPE_5_TO_CHAN_IN_Q0_MASK                                                           0x00000070L
+#define GL2_PIPE_STEER_2__PIPE_6_TO_CHAN_IN_Q0_MASK                                                           0x00000700L
+#define GL2_PIPE_STEER_2__PIPE_7_TO_CHAN_IN_Q0_MASK                                                           0x00007000L
+#define GL2_PIPE_STEER_2__PIPE_4_TO_CHAN_IN_Q1_MASK                                                           0x00070000L
+#define GL2_PIPE_STEER_2__PIPE_5_TO_CHAN_IN_Q1_MASK                                                           0x00700000L
+#define GL2_PIPE_STEER_2__PIPE_6_TO_CHAN_IN_Q1_MASK                                                           0x07000000L
+#define GL2_PIPE_STEER_2__PIPE_7_TO_CHAN_IN_Q1_MASK                                                           0x70000000L
+//GL2_PIPE_STEER_3
+#define GL2_PIPE_STEER_3__PIPE_4_TO_CHAN_IN_Q2__SHIFT                                                         0x0
+#define GL2_PIPE_STEER_3__PIPE_5_TO_CHAN_IN_Q2__SHIFT                                                         0x4
+#define GL2_PIPE_STEER_3__PIPE_6_TO_CHAN_IN_Q2__SHIFT                                                         0x8
+#define GL2_PIPE_STEER_3__PIPE_7_TO_CHAN_IN_Q2__SHIFT                                                         0xc
+#define GL2_PIPE_STEER_3__PIPE_4_TO_CHAN_IN_Q3__SHIFT                                                         0x10
+#define GL2_PIPE_STEER_3__PIPE_5_TO_CHAN_IN_Q3__SHIFT                                                         0x14
+#define GL2_PIPE_STEER_3__PIPE_6_TO_CHAN_IN_Q3__SHIFT                                                         0x18
+#define GL2_PIPE_STEER_3__PIPE_7_TO_CHAN_IN_Q3__SHIFT                                                         0x1c
+#define GL2_PIPE_STEER_3__PIPE_4_TO_CHAN_IN_Q2_MASK                                                           0x00000007L
+#define GL2_PIPE_STEER_3__PIPE_5_TO_CHAN_IN_Q2_MASK                                                           0x00000070L
+#define GL2_PIPE_STEER_3__PIPE_6_TO_CHAN_IN_Q2_MASK                                                           0x00000700L
+#define GL2_PIPE_STEER_3__PIPE_7_TO_CHAN_IN_Q2_MASK                                                           0x00007000L
+#define GL2_PIPE_STEER_3__PIPE_4_TO_CHAN_IN_Q3_MASK                                                           0x00070000L
+#define GL2_PIPE_STEER_3__PIPE_5_TO_CHAN_IN_Q3_MASK                                                           0x00700000L
+#define GL2_PIPE_STEER_3__PIPE_6_TO_CHAN_IN_Q3_MASK                                                           0x07000000L
+#define GL2_PIPE_STEER_3__PIPE_7_TO_CHAN_IN_Q3_MASK                                                           0x70000000L
+//GL1_PIPE_STEER
+#define GL1_PIPE_STEER__PIPE0__SHIFT                                                                          0x0
+#define GL1_PIPE_STEER__PIPE1__SHIFT                                                                          0x2
+#define GL1_PIPE_STEER__PIPE2__SHIFT                                                                          0x4
+#define GL1_PIPE_STEER__PIPE3__SHIFT                                                                          0x6
+#define GL1_PIPE_STEER__PIPE0_MASK                                                                            0x00000003L
+#define GL1_PIPE_STEER__PIPE1_MASK                                                                            0x0000000CL
+#define GL1_PIPE_STEER__PIPE2_MASK                                                                            0x00000030L
+#define GL1_PIPE_STEER__PIPE3_MASK                                                                            0x000000C0L
+//CH_PIPE_STEER
+#define CH_PIPE_STEER__PIPE0__SHIFT                                                                           0x0
+#define CH_PIPE_STEER__PIPE1__SHIFT                                                                           0x2
+#define CH_PIPE_STEER__PIPE2__SHIFT                                                                           0x4
+#define CH_PIPE_STEER__PIPE3__SHIFT                                                                           0x6
+#define CH_PIPE_STEER__PIPE0_MASK                                                                             0x00000003L
+#define CH_PIPE_STEER__PIPE1_MASK                                                                             0x0000000CL
+#define CH_PIPE_STEER__PIPE2_MASK                                                                             0x00000030L
+#define CH_PIPE_STEER__PIPE3_MASK                                                                             0x000000C0L
+//GC_USER_SHADER_ARRAY_CONFIG
+#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS__SHIFT                                                     0x10
+#define GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_WGPS_MASK                                                       0xFFFF0000L
+//GC_USER_PRIM_CONFIG
+#define GC_USER_PRIM_CONFIG__INACTIVE_PA__SHIFT                                                               0x4
+#define GC_USER_PRIM_CONFIG__INACTIVE_PA_MASK                                                                 0x000FFFF0L
+//GC_USER_SA_UNIT_DISABLE
+#define GC_USER_SA_UNIT_DISABLE__SA_DISABLE__SHIFT                                                            0x8
+#define GC_USER_SA_UNIT_DISABLE__SA_DISABLE_MASK                                                              0x00FFFF00L
+//GC_USER_RB_REDUNDANCY
+#define GC_USER_RB_REDUNDANCY__FAILED_RB0__SHIFT                                                              0x8
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY0__SHIFT                                                          0xc
+#define GC_USER_RB_REDUNDANCY__FAILED_RB1__SHIFT                                                              0x10
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY1__SHIFT                                                          0x14
+#define GC_USER_RB_REDUNDANCY__FAILED_RB0_MASK                                                                0x00000F00L
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY0_MASK                                                            0x00001000L
+#define GC_USER_RB_REDUNDANCY__FAILED_RB1_MASK                                                                0x000F0000L
+#define GC_USER_RB_REDUNDANCY__EN_REDUNDANCY1_MASK                                                            0x00100000L
+//GC_USER_RB_BACKEND_DISABLE
+#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT                                                    0x4
+#define GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK                                                      0xFFFFFFF0L
+//GC_USER_RMI_REDUNDANCY
+#define GC_USER_RMI_REDUNDANCY__REPAIR_EN_IN_0__SHIFT                                                         0x1
+#define GC_USER_RMI_REDUNDANCY__REPAIR_EN_IN_1__SHIFT                                                         0x2
+#define GC_USER_RMI_REDUNDANCY__REPAIR_RMI_OVERRIDE__SHIFT                                                    0x3
+#define GC_USER_RMI_REDUNDANCY__REPAIR_ID_SWAP__SHIFT                                                         0x4
+#define GC_USER_RMI_REDUNDANCY__REPAIR_EN_IN_0_MASK                                                           0x00000002L
+#define GC_USER_RMI_REDUNDANCY__REPAIR_EN_IN_1_MASK                                                           0x00000004L
+#define GC_USER_RMI_REDUNDANCY__REPAIR_RMI_OVERRIDE_MASK                                                      0x00000008L
+#define GC_USER_RMI_REDUNDANCY__REPAIR_ID_SWAP_MASK                                                           0x00000010L
+//CGTS_USER_TCC_DISABLE
+#define CGTS_USER_TCC_DISABLE__HI_TCC_DISABLE__SHIFT                                                          0x8
+#define CGTS_USER_TCC_DISABLE__TCC_DISABLE__SHIFT                                                             0x10
+#define CGTS_USER_TCC_DISABLE__HI_TCC_DISABLE_MASK                                                            0x0000FF00L
+#define CGTS_USER_TCC_DISABLE__TCC_DISABLE_MASK                                                               0xFFFF0000L
+//GC_USER_SHADER_RATE_CONFIG
+#define GC_USER_SHADER_RATE_CONFIG__DPFP_RATE__SHIFT                                                          0x1
+#define GC_USER_SHADER_RATE_CONFIG__DPFP_RATE_MASK                                                            0x00000006L
+//RLC_GPU_IOV_SDMA0_STATUS
+#define RLC_GPU_IOV_SDMA0_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA0_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA1_STATUS
+#define RLC_GPU_IOV_SDMA1_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA1_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA2_STATUS
+#define RLC_GPU_IOV_SDMA2_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA2_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA3_STATUS
+#define RLC_GPU_IOV_SDMA3_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA3_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA4_STATUS
+#define RLC_GPU_IOV_SDMA4_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA4_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA5_STATUS
+#define RLC_GPU_IOV_SDMA5_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA5_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA6_STATUS
+#define RLC_GPU_IOV_SDMA6_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA6_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA7_STATUS
+#define RLC_GPU_IOV_SDMA7_STATUS__STATUS__SHIFT                                                               0x0
+#define RLC_GPU_IOV_SDMA7_STATUS__STATUS_MASK                                                                 0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA0_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA0_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA0_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA1_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA1_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA1_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA2_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA2_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA2_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA3_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA3_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA3_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA4_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA4_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA4_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA5_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA5_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA5_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA6_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA6_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA6_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+//RLC_GPU_IOV_SDMA7_BUSY_STATUS
+#define RLC_GPU_IOV_SDMA7_BUSY_STATUS__VM_BUSY_STATUS__SHIFT                                                  0x0
+#define RLC_GPU_IOV_SDMA7_BUSY_STATUS__VM_BUSY_STATUS_MASK                                                    0xFFFFFFFFL
+
+
+// addressBlock: gc_cphypdec
+//CP_HYP_PFP_UCODE_ADDR
+#define CP_HYP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
+#define CP_HYP_PFP_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x000FFFFFL
+//CP_PFP_UCODE_ADDR
+#define CP_PFP_UCODE_ADDR__UCODE_ADDR__SHIFT                                                                  0x0
+#define CP_PFP_UCODE_ADDR__UCODE_ADDR_MASK                                                                    0x000FFFFFL
+//CP_HYP_PFP_UCODE_DATA
+#define CP_HYP_PFP_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
+#define CP_HYP_PFP_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
+//CP_PFP_UCODE_DATA
+#define CP_PFP_UCODE_DATA__UCODE_DATA__SHIFT                                                                  0x0
+#define CP_PFP_UCODE_DATA__UCODE_DATA_MASK                                                                    0xFFFFFFFFL
+//CP_HYP_ME_UCODE_ADDR
+#define CP_HYP_ME_UCODE_ADDR__UCODE_ADDR__SHIFT                                                               0x0
+#define CP_HYP_ME_UCODE_ADDR__UCODE_ADDR_MASK                                                                 0x000FFFFFL
+//CP_ME_RAM_RADDR
+#define CP_ME_RAM_RADDR__ME_RAM_RADDR__SHIFT                                                                  0x0
+#define CP_ME_RAM_RADDR__ME_RAM_RADDR_MASK                                                                    0x000FFFFFL
+//CP_ME_RAM_WADDR
+#define CP_ME_RAM_WADDR__ME_RAM_WADDR__SHIFT                                                                  0x0
+#define CP_ME_RAM_WADDR__ME_RAM_WADDR_MASK                                                                    0x001FFFFFL
+//CP_HYP_ME_UCODE_DATA
+#define CP_HYP_ME_UCODE_DATA__UCODE_DATA__SHIFT                                                               0x0
+#define CP_HYP_ME_UCODE_DATA__UCODE_DATA_MASK                                                                 0xFFFFFFFFL
+//CP_ME_RAM_DATA
+#define CP_ME_RAM_DATA__ME_RAM_DATA__SHIFT                                                                    0x0
+#define CP_ME_RAM_DATA__ME_RAM_DATA_MASK                                                                      0xFFFFFFFFL
+//CP_HYP_MEC1_UCODE_ADDR
+#define CP_HYP_MEC1_UCODE_ADDR__UCODE_ADDR__SHIFT                                                             0x0
+#define CP_HYP_MEC1_UCODE_ADDR__UCODE_ADDR_MASK                                                               0x000FFFFFL
+//CP_MEC_ME1_UCODE_ADDR
+#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
+#define CP_MEC_ME1_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x000FFFFFL
+//CP_HYP_MEC1_UCODE_DATA
+#define CP_HYP_MEC1_UCODE_DATA__UCODE_DATA__SHIFT                                                             0x0
+#define CP_HYP_MEC1_UCODE_DATA__UCODE_DATA_MASK                                                               0xFFFFFFFFL
+//CP_MEC_ME1_UCODE_DATA
+#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
+#define CP_MEC_ME1_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
+//CP_HYP_MEC2_UCODE_ADDR
+#define CP_HYP_MEC2_UCODE_ADDR__UCODE_ADDR__SHIFT                                                             0x0
+#define CP_HYP_MEC2_UCODE_ADDR__UCODE_ADDR_MASK                                                               0x000FFFFFL
+//CP_MEC_ME2_UCODE_ADDR
+#define CP_MEC_ME2_UCODE_ADDR__UCODE_ADDR__SHIFT                                                              0x0
+#define CP_MEC_ME2_UCODE_ADDR__UCODE_ADDR_MASK                                                                0x000FFFFFL
+//CP_HYP_MEC2_UCODE_DATA
+#define CP_HYP_MEC2_UCODE_DATA__UCODE_DATA__SHIFT                                                             0x0
+#define CP_HYP_MEC2_UCODE_DATA__UCODE_DATA_MASK                                                               0xFFFFFFFFL
+//CP_MEC_ME2_UCODE_DATA
+#define CP_MEC_ME2_UCODE_DATA__UCODE_DATA__SHIFT                                                              0x0
+#define CP_MEC_ME2_UCODE_DATA__UCODE_DATA_MASK                                                                0xFFFFFFFFL
+//CP_HYP_PFP_UCODE_CHKSUM
+#define CP_HYP_PFP_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                          0x0
+#define CP_HYP_PFP_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                            0xFFFFFFFFL
+//CP_HYP_ME_UCODE_CHKSUM
+#define CP_HYP_ME_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                           0x0
+#define CP_HYP_ME_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                             0xFFFFFFFFL
+//CP_HYP_MEC_ME1_UCODE_CHKSUM
+#define CP_HYP_MEC_ME1_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                      0x0
+#define CP_HYP_MEC_ME1_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                        0xFFFFFFFFL
+//CP_HYP_MEC_ME2_UCODE_CHKSUM
+#define CP_HYP_MEC_ME2_UCODE_CHKSUM__UCODE_CHKSUM__SHIFT                                                      0x0
+#define CP_HYP_MEC_ME2_UCODE_CHKSUM__UCODE_CHKSUM_MASK                                                        0xFFFFFFFFL
+//CP_PFP_IC_BASE_LO
+#define CP_PFP_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                  0xc
+#define CP_PFP_IC_BASE_LO__IC_BASE_LO_MASK                                                                    0xFFFFF000L
+//CP_PFP_IC_BASE_HI
+#define CP_PFP_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                  0x0
+#define CP_PFP_IC_BASE_HI__IC_BASE_HI_MASK                                                                    0x0000FFFFL
+//CP_PFP_IC_BASE_CNTL
+#define CP_PFP_IC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_PFP_IC_BASE_CNTL__ADDRESS_CLAMP__SHIFT                                                             0x4
+#define CP_PFP_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                               0x17
+#define CP_PFP_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_PFP_IC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_PFP_IC_BASE_CNTL__ADDRESS_CLAMP_MASK                                                               0x00000010L
+#define CP_PFP_IC_BASE_CNTL__EXE_DISABLE_MASK                                                                 0x00800000L
+#define CP_PFP_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x03000000L
+//CP_PFP_IC_OP_CNTL
+#define CP_PFP_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                            0x0
+#define CP_PFP_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE__SHIFT                                                   0x1
+#define CP_PFP_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                0x4
+#define CP_PFP_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                               0x5
+#define CP_PFP_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                              0x00000001L
+#define CP_PFP_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE_MASK                                                     0x00000002L
+#define CP_PFP_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                  0x00000010L
+#define CP_PFP_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                 0x00000020L
+//CP_ME_IC_BASE_LO
+#define CP_ME_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                   0xc
+#define CP_ME_IC_BASE_LO__IC_BASE_LO_MASK                                                                     0xFFFFF000L
+//CP_ME_IC_BASE_HI
+#define CP_ME_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                   0x0
+#define CP_ME_IC_BASE_HI__IC_BASE_HI_MASK                                                                     0x0000FFFFL
+//CP_ME_IC_BASE_CNTL
+#define CP_ME_IC_BASE_CNTL__VMID__SHIFT                                                                       0x0
+#define CP_ME_IC_BASE_CNTL__ADDRESS_CLAMP__SHIFT                                                              0x4
+#define CP_ME_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                                0x17
+#define CP_ME_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                               0x18
+#define CP_ME_IC_BASE_CNTL__VMID_MASK                                                                         0x0000000FL
+#define CP_ME_IC_BASE_CNTL__ADDRESS_CLAMP_MASK                                                                0x00000010L
+#define CP_ME_IC_BASE_CNTL__EXE_DISABLE_MASK                                                                  0x00800000L
+#define CP_ME_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                 0x03000000L
+//CP_ME_IC_OP_CNTL
+#define CP_ME_IC_OP_CNTL__INVALIDATE_CACHE__SHIFT                                                             0x0
+#define CP_ME_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE__SHIFT                                                    0x1
+#define CP_ME_IC_OP_CNTL__PRIME_ICACHE__SHIFT                                                                 0x4
+#define CP_ME_IC_OP_CNTL__ICACHE_PRIMED__SHIFT                                                                0x5
+#define CP_ME_IC_OP_CNTL__INVALIDATE_CACHE_MASK                                                               0x00000001L
+#define CP_ME_IC_OP_CNTL__INVALIDATE_CACHE_COMPLETE_MASK                                                      0x00000002L
+#define CP_ME_IC_OP_CNTL__PRIME_ICACHE_MASK                                                                   0x00000010L
+#define CP_ME_IC_OP_CNTL__ICACHE_PRIMED_MASK                                                                  0x00000020L
+//CP_CPC_IC_BASE_LO
+#define CP_CPC_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                  0xc
+#define CP_CPC_IC_BASE_LO__IC_BASE_LO_MASK                                                                    0xFFFFF000L
+//CP_CPC_IC_BASE_HI
+#define CP_CPC_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                  0x0
+#define CP_CPC_IC_BASE_HI__IC_BASE_HI_MASK                                                                    0x0000FFFFL
+//CP_CPC_IC_BASE_CNTL
+#define CP_CPC_IC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_CPC_IC_BASE_CNTL__ADDRESS_CLAMP__SHIFT                                                             0x4
+#define CP_CPC_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                               0x17
+#define CP_CPC_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_CPC_IC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_CPC_IC_BASE_CNTL__ADDRESS_CLAMP_MASK                                                               0x00000010L
+#define CP_CPC_IC_BASE_CNTL__EXE_DISABLE_MASK                                                                 0x00800000L
+#define CP_CPC_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x03000000L
+//CP_MES_IC_BASE_LO
+#define CP_MES_IC_BASE_LO__IC_BASE_LO__SHIFT                                                                  0xc
+#define CP_MES_IC_BASE_LO__IC_BASE_LO_MASK                                                                    0xFFFFF000L
+//CP_MES_MIBASE_LO
+#define CP_MES_MIBASE_LO__IC_BASE_LO__SHIFT                                                                   0xc
+#define CP_MES_MIBASE_LO__IC_BASE_LO_MASK                                                                     0xFFFFF000L
+//CP_MES_IC_BASE_HI
+#define CP_MES_IC_BASE_HI__IC_BASE_HI__SHIFT                                                                  0x0
+#define CP_MES_IC_BASE_HI__IC_BASE_HI_MASK                                                                    0x0000FFFFL
+//CP_MES_MIBASE_HI
+#define CP_MES_MIBASE_HI__IC_BASE_HI__SHIFT                                                                   0x0
+#define CP_MES_MIBASE_HI__IC_BASE_HI_MASK                                                                     0x0000FFFFL
+//CP_MES_IC_BASE_CNTL
+#define CP_MES_IC_BASE_CNTL__VMID__SHIFT                                                                      0x0
+#define CP_MES_IC_BASE_CNTL__EXE_DISABLE__SHIFT                                                               0x17
+#define CP_MES_IC_BASE_CNTL__CACHE_POLICY__SHIFT                                                              0x18
+#define CP_MES_IC_BASE_CNTL__VMID_MASK                                                                        0x0000000FL
+#define CP_MES_IC_BASE_CNTL__EXE_DISABLE_MASK                                                                 0x00800000L
+#define CP_MES_IC_BASE_CNTL__CACHE_POLICY_MASK                                                                0x03000000L
+//CP_MES_DC_BASE_LO
+#define CP_MES_DC_BASE_LO__DC_BASE_LO__SHIFT                                                                  0x10
+#define CP_MES_DC_BASE_LO__DC_BASE_LO_MASK                                                                    0xFFFF0000L
+//CP_MES_MDBASE_LO
+#define CP_MES_MDBASE_LO__BASE_LO__SHIFT                                                                      0x10
+#define CP_MES_MDBASE_LO__BASE_LO_MASK                                                                        0xFFFF0000L
+//CP_MES_DC_BASE_HI
+#define CP_MES_DC_BASE_HI__DC_BASE_HI__SHIFT                                                                  0x0
+#define CP_MES_DC_BASE_HI__DC_BASE_HI_MASK                                                                    0x0000FFFFL
+//CP_MES_MDBASE_HI
+#define CP_MES_MDBASE_HI__BASE_HI__SHIFT                                                                      0x0
+#define CP_MES_MDBASE_HI__BASE_HI_MASK                                                                        0x0000FFFFL
+//CP_MES_MIBOUND_LO
+#define CP_MES_MIBOUND_LO__BOUND_LO__SHIFT                                                                    0x0
+#define CP_MES_MIBOUND_LO__BOUND_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MIBOUND_HI
+#define CP_MES_MIBOUND_HI__BOUND_HI__SHIFT                                                                    0x0
+#define CP_MES_MIBOUND_HI__BOUND_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MDBOUND_LO
+#define CP_MES_MDBOUND_LO__BOUND_LO__SHIFT                                                                    0x0
+#define CP_MES_MDBOUND_LO__BOUND_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MES_MDBOUND_HI
+#define CP_MES_MDBOUND_HI__BOUND_HI__SHIFT                                                                    0x0
+#define CP_MES_MDBOUND_HI__BOUND_HI_MASK                                                                      0xFFFFFFFFL
+//CP_GFX_RS64_DC_BASE0_LO
+#define CP_GFX_RS64_DC_BASE0_LO__DC_BASE_LO__SHIFT                                                            0x10
+#define CP_GFX_RS64_DC_BASE0_LO__DC_BASE_LO_MASK                                                              0xFFFF0000L
+//CP_GFX_RS64_DC_BASE1_LO
+#define CP_GFX_RS64_DC_BASE1_LO__DC_BASE_LO__SHIFT                                                            0x10
+#define CP_GFX_RS64_DC_BASE1_LO__DC_BASE_LO_MASK                                                              0xFFFF0000L
+//CP_GFX_RS64_DC_BASE0_HI
+#define CP_GFX_RS64_DC_BASE0_HI__DC_BASE_HI__SHIFT                                                            0x0
+#define CP_GFX_RS64_DC_BASE0_HI__DC_BASE_HI_MASK                                                              0x0000FFFFL
+//CP_GFX_RS64_DC_BASE1_HI
+#define CP_GFX_RS64_DC_BASE1_HI__DC_BASE_HI__SHIFT                                                            0x0
+#define CP_GFX_RS64_DC_BASE1_HI__DC_BASE_HI_MASK                                                              0x0000FFFFL
+//CP_GFX_RS64_MIBOUND_LO
+#define CP_GFX_RS64_MIBOUND_LO__BOUND__SHIFT                                                                  0x0
+#define CP_GFX_RS64_MIBOUND_LO__BOUND_MASK                                                                    0xFFFFFFFFL
+//CP_GFX_RS64_MIBOUND_HI
+#define CP_GFX_RS64_MIBOUND_HI__BOUND__SHIFT                                                                  0x0
+#define CP_GFX_RS64_MIBOUND_HI__BOUND_MASK                                                                    0xFFFFFFFFL
+//CP_MEC_DC_BASE_LO
+#define CP_MEC_DC_BASE_LO__DC_BASE_LO__SHIFT                                                                  0x10
+#define CP_MEC_DC_BASE_LO__DC_BASE_LO_MASK                                                                    0xFFFF0000L
+//CP_MEC_MDBASE_LO
+#define CP_MEC_MDBASE_LO__BASE_LO__SHIFT                                                                      0x10
+#define CP_MEC_MDBASE_LO__BASE_LO_MASK                                                                        0xFFFF0000L
+//CP_MEC_DC_BASE_HI
+#define CP_MEC_DC_BASE_HI__DC_BASE_HI__SHIFT                                                                  0x0
+#define CP_MEC_DC_BASE_HI__DC_BASE_HI_MASK                                                                    0x0000FFFFL
+//CP_MEC_MDBASE_HI
+#define CP_MEC_MDBASE_HI__BASE_HI__SHIFT                                                                      0x0
+#define CP_MEC_MDBASE_HI__BASE_HI_MASK                                                                        0x0000FFFFL
+//CP_MEC_MIBOUND_LO
+#define CP_MEC_MIBOUND_LO__BOUND_LO__SHIFT                                                                    0x0
+#define CP_MEC_MIBOUND_LO__BOUND_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MEC_MIBOUND_HI
+#define CP_MEC_MIBOUND_HI__BOUND_HI__SHIFT                                                                    0x0
+#define CP_MEC_MIBOUND_HI__BOUND_HI_MASK                                                                      0xFFFFFFFFL
+//CP_MEC_MDBOUND_LO
+#define CP_MEC_MDBOUND_LO__BOUND_LO__SHIFT                                                                    0x0
+#define CP_MEC_MDBOUND_LO__BOUND_LO_MASK                                                                      0xFFFFFFFFL
+//CP_MEC_MDBOUND_HI
+#define CP_MEC_MDBOUND_HI__BOUND_HI__SHIFT                                                                    0x0
+#define CP_MEC_MDBOUND_HI__BOUND_HI_MASK                                                                      0xFFFFFFFFL
+
+
+// addressBlock: gc_grbm_hypdec
+//GRBM_GFX_INDEX_SR_SELECT
+#define GRBM_GFX_INDEX_SR_SELECT__INDEX__SHIFT                                                                0x0
+#define GRBM_GFX_INDEX_SR_SELECT__VF_PF__SHIFT                                                                0x1f
+#define GRBM_GFX_INDEX_SR_SELECT__INDEX_MASK                                                                  0x00000007L
+#define GRBM_GFX_INDEX_SR_SELECT__VF_PF_MASK                                                                  0x80000000L
+//GRBM_GFX_INDEX_SR_DATA
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_INDEX__SHIFT                                                         0x0
+#define GRBM_GFX_INDEX_SR_DATA__SA_INDEX__SHIFT                                                               0x8
+#define GRBM_GFX_INDEX_SR_DATA__SE_INDEX__SHIFT                                                               0x10
+#define GRBM_GFX_INDEX_SR_DATA__SA_BROADCAST_WRITES__SHIFT                                                    0x1d
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_BROADCAST_WRITES__SHIFT                                              0x1e
+#define GRBM_GFX_INDEX_SR_DATA__SE_BROADCAST_WRITES__SHIFT                                                    0x1f
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_INDEX_MASK                                                           0x000000FFL
+#define GRBM_GFX_INDEX_SR_DATA__SA_INDEX_MASK                                                                 0x0000FF00L
+#define GRBM_GFX_INDEX_SR_DATA__SE_INDEX_MASK                                                                 0x00FF0000L
+#define GRBM_GFX_INDEX_SR_DATA__SA_BROADCAST_WRITES_MASK                                                      0x20000000L
+#define GRBM_GFX_INDEX_SR_DATA__INSTANCE_BROADCAST_WRITES_MASK                                                0x40000000L
+#define GRBM_GFX_INDEX_SR_DATA__SE_BROADCAST_WRITES_MASK                                                      0x80000000L
+//GRBM_GFX_CNTL_SR_SELECT
+#define GRBM_GFX_CNTL_SR_SELECT__INDEX__SHIFT                                                                 0x0
+#define GRBM_GFX_CNTL_SR_SELECT__VF_PF__SHIFT                                                                 0x1f
+#define GRBM_GFX_CNTL_SR_SELECT__INDEX_MASK                                                                   0x00000007L
+#define GRBM_GFX_CNTL_SR_SELECT__VF_PF_MASK                                                                   0x80000000L
+//GRBM_GFX_CNTL_SR_DATA
+#define GRBM_GFX_CNTL_SR_DATA__PIPEID__SHIFT                                                                  0x0
+#define GRBM_GFX_CNTL_SR_DATA__MEID__SHIFT                                                                    0x2
+#define GRBM_GFX_CNTL_SR_DATA__VMID__SHIFT                                                                    0x4
+#define GRBM_GFX_CNTL_SR_DATA__QUEUEID__SHIFT                                                                 0x8
+#define GRBM_GFX_CNTL_SR_DATA__PIPEID_MASK                                                                    0x00000003L
+#define GRBM_GFX_CNTL_SR_DATA__MEID_MASK                                                                      0x0000000CL
+#define GRBM_GFX_CNTL_SR_DATA__VMID_MASK                                                                      0x000000F0L
+#define GRBM_GFX_CNTL_SR_DATA__QUEUEID_MASK                                                                   0x00000700L
+//GC_IH_COOKIE_0_PTR
+#define GC_IH_COOKIE_0_PTR__ADDR__SHIFT                                                                       0x0
+#define GC_IH_COOKIE_0_PTR__ADDR_MASK                                                                         0x000FFFFFL
+//GRBM_SE_REMAP_CNTL
+#define GRBM_SE_REMAP_CNTL__SE0_REMAP_EN__SHIFT                                                               0x0
+#define GRBM_SE_REMAP_CNTL__SE0_REMAP__SHIFT                                                                  0x1
+#define GRBM_SE_REMAP_CNTL__SE1_REMAP_EN__SHIFT                                                               0x4
+#define GRBM_SE_REMAP_CNTL__SE1_REMAP__SHIFT                                                                  0x5
+#define GRBM_SE_REMAP_CNTL__SE2_REMAP_EN__SHIFT                                                               0x8
+#define GRBM_SE_REMAP_CNTL__SE2_REMAP__SHIFT                                                                  0x9
+#define GRBM_SE_REMAP_CNTL__SE3_REMAP_EN__SHIFT                                                               0xc
+#define GRBM_SE_REMAP_CNTL__SE3_REMAP__SHIFT                                                                  0xd
+#define GRBM_SE_REMAP_CNTL__SE4_REMAP_EN__SHIFT                                                               0x10
+#define GRBM_SE_REMAP_CNTL__SE4_REMAP__SHIFT                                                                  0x11
+#define GRBM_SE_REMAP_CNTL__SE5_REMAP_EN__SHIFT                                                               0x14
+#define GRBM_SE_REMAP_CNTL__SE5_REMAP__SHIFT                                                                  0x15
+#define GRBM_SE_REMAP_CNTL__SE6_REMAP_EN__SHIFT                                                               0x18
+#define GRBM_SE_REMAP_CNTL__SE6_REMAP__SHIFT                                                                  0x19
+#define GRBM_SE_REMAP_CNTL__SE7_REMAP_EN__SHIFT                                                               0x1c
+#define GRBM_SE_REMAP_CNTL__SE7_REMAP__SHIFT                                                                  0x1d
+#define GRBM_SE_REMAP_CNTL__SE0_REMAP_EN_MASK                                                                 0x00000001L
+#define GRBM_SE_REMAP_CNTL__SE0_REMAP_MASK                                                                    0x0000000EL
+#define GRBM_SE_REMAP_CNTL__SE1_REMAP_EN_MASK                                                                 0x00000010L
+#define GRBM_SE_REMAP_CNTL__SE1_REMAP_MASK                                                                    0x000000E0L
+#define GRBM_SE_REMAP_CNTL__SE2_REMAP_EN_MASK                                                                 0x00000100L
+#define GRBM_SE_REMAP_CNTL__SE2_REMAP_MASK                                                                    0x00000E00L
+#define GRBM_SE_REMAP_CNTL__SE3_REMAP_EN_MASK                                                                 0x00001000L
+#define GRBM_SE_REMAP_CNTL__SE3_REMAP_MASK                                                                    0x0000E000L
+#define GRBM_SE_REMAP_CNTL__SE4_REMAP_EN_MASK                                                                 0x00010000L
+#define GRBM_SE_REMAP_CNTL__SE4_REMAP_MASK                                                                    0x000E0000L
+#define GRBM_SE_REMAP_CNTL__SE5_REMAP_EN_MASK                                                                 0x00100000L
+#define GRBM_SE_REMAP_CNTL__SE5_REMAP_MASK                                                                    0x00E00000L
+#define GRBM_SE_REMAP_CNTL__SE6_REMAP_EN_MASK                                                                 0x01000000L
+#define GRBM_SE_REMAP_CNTL__SE6_REMAP_MASK                                                                    0x0E000000L
+#define GRBM_SE_REMAP_CNTL__SE7_REMAP_EN_MASK                                                                 0x10000000L
+#define GRBM_SE_REMAP_CNTL__SE7_REMAP_MASK                                                                    0xE0000000L
+
+
+// addressBlock: gc_gcvmsharedhvdec
+//GCMC_VM_FB_SIZE_OFFSET_VF0
+#define GCMC_VM_FB_SIZE_OFFSET_VF0__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF0__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF0__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF0__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF1
+#define GCMC_VM_FB_SIZE_OFFSET_VF1__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF1__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF1__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF1__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF2
+#define GCMC_VM_FB_SIZE_OFFSET_VF2__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF2__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF2__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF2__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF3
+#define GCMC_VM_FB_SIZE_OFFSET_VF3__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF3__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF3__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF3__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF4
+#define GCMC_VM_FB_SIZE_OFFSET_VF4__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF4__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF4__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF4__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF5
+#define GCMC_VM_FB_SIZE_OFFSET_VF5__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF5__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF5__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF5__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF6
+#define GCMC_VM_FB_SIZE_OFFSET_VF6__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF6__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF6__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF6__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF7
+#define GCMC_VM_FB_SIZE_OFFSET_VF7__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF7__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF7__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF7__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF8
+#define GCMC_VM_FB_SIZE_OFFSET_VF8__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF8__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF8__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF8__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF9
+#define GCMC_VM_FB_SIZE_OFFSET_VF9__VF_FB_SIZE__SHIFT                                                         0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF9__VF_FB_OFFSET__SHIFT                                                       0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF9__VF_FB_SIZE_MASK                                                           0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF9__VF_FB_OFFSET_MASK                                                         0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF10
+#define GCMC_VM_FB_SIZE_OFFSET_VF10__VF_FB_SIZE__SHIFT                                                        0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF10__VF_FB_OFFSET__SHIFT                                                      0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF10__VF_FB_SIZE_MASK                                                          0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF10__VF_FB_OFFSET_MASK                                                        0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF11
+#define GCMC_VM_FB_SIZE_OFFSET_VF11__VF_FB_SIZE__SHIFT                                                        0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF11__VF_FB_OFFSET__SHIFT                                                      0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF11__VF_FB_SIZE_MASK                                                          0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF11__VF_FB_OFFSET_MASK                                                        0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF12
+#define GCMC_VM_FB_SIZE_OFFSET_VF12__VF_FB_SIZE__SHIFT                                                        0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF12__VF_FB_OFFSET__SHIFT                                                      0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF12__VF_FB_SIZE_MASK                                                          0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF12__VF_FB_OFFSET_MASK                                                        0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF13
+#define GCMC_VM_FB_SIZE_OFFSET_VF13__VF_FB_SIZE__SHIFT                                                        0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF13__VF_FB_OFFSET__SHIFT                                                      0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF13__VF_FB_SIZE_MASK                                                          0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF13__VF_FB_OFFSET_MASK                                                        0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF14
+#define GCMC_VM_FB_SIZE_OFFSET_VF14__VF_FB_SIZE__SHIFT                                                        0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF14__VF_FB_OFFSET__SHIFT                                                      0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF14__VF_FB_SIZE_MASK                                                          0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF14__VF_FB_OFFSET_MASK                                                        0xFFFF0000L
+//GCMC_VM_FB_SIZE_OFFSET_VF15
+#define GCMC_VM_FB_SIZE_OFFSET_VF15__VF_FB_SIZE__SHIFT                                                        0x0
+#define GCMC_VM_FB_SIZE_OFFSET_VF15__VF_FB_OFFSET__SHIFT                                                      0x10
+#define GCMC_VM_FB_SIZE_OFFSET_VF15__VF_FB_SIZE_MASK                                                          0x0000FFFFL
+#define GCMC_VM_FB_SIZE_OFFSET_VF15__VF_FB_OFFSET_MASK                                                        0xFFFF0000L
+
+
+// addressBlock: gc_rlcdec
+//RLC_CNTL
+#define RLC_CNTL__RLC_ENABLE_F32__SHIFT                                                                       0x0
+#define RLC_CNTL__FORCE_RETRY__SHIFT                                                                          0x1
+#define RLC_CNTL__READ_CACHE_DISABLE__SHIFT                                                                   0x2
+#define RLC_CNTL__RLC_STEP_F32__SHIFT                                                                         0x3
+#define RLC_CNTL__RESERVED__SHIFT                                                                             0x4
+#define RLC_CNTL__RLC_ENABLE_F32_MASK                                                                         0x00000001L
+#define RLC_CNTL__FORCE_RETRY_MASK                                                                            0x00000002L
+#define RLC_CNTL__READ_CACHE_DISABLE_MASK                                                                     0x00000004L
+#define RLC_CNTL__RLC_STEP_F32_MASK                                                                           0x00000008L
+#define RLC_CNTL__RESERVED_MASK                                                                               0xFFFFFFF0L
+//RLC_F32_UCODE_VERSION
+#define RLC_F32_UCODE_VERSION__THREAD0_VERSION__SHIFT                                                         0x0
+#define RLC_F32_UCODE_VERSION__THREAD1_VERSION__SHIFT                                                         0xa
+#define RLC_F32_UCODE_VERSION__THREAD2_VERSION__SHIFT                                                         0x14
+#define RLC_F32_UCODE_VERSION__THREAD0_VERSION_MASK                                                           0x000003FFL
+#define RLC_F32_UCODE_VERSION__THREAD1_VERSION_MASK                                                           0x000FFC00L
+#define RLC_F32_UCODE_VERSION__THREAD2_VERSION_MASK                                                           0x3FF00000L
+//RLC_STAT
+#define RLC_STAT__RLC_BUSY__SHIFT                                                                             0x0
+#define RLC_STAT__RLC_SRM_BUSY__SHIFT                                                                         0x1
+#define RLC_STAT__RLC_GPM_BUSY__SHIFT                                                                         0x2
+#define RLC_STAT__RLC_SPM_BUSY__SHIFT                                                                         0x3
+#define RLC_STAT__MC_BUSY__SHIFT                                                                              0x4
+#define RLC_STAT__RLC_THREAD_0_BUSY__SHIFT                                                                    0x5
+#define RLC_STAT__RLC_THREAD_1_BUSY__SHIFT                                                                    0x6
+#define RLC_STAT__RLC_THREAD_2_BUSY__SHIFT                                                                    0x7
+#define RLC_STAT__RESERVED__SHIFT                                                                             0x8
+#define RLC_STAT__RLC_BUSY_MASK                                                                               0x00000001L
+#define RLC_STAT__RLC_SRM_BUSY_MASK                                                                           0x00000002L
+#define RLC_STAT__RLC_GPM_BUSY_MASK                                                                           0x00000004L
+#define RLC_STAT__RLC_SPM_BUSY_MASK                                                                           0x00000008L
+#define RLC_STAT__MC_BUSY_MASK                                                                                0x00000010L
+#define RLC_STAT__RLC_THREAD_0_BUSY_MASK                                                                      0x00000020L
+#define RLC_STAT__RLC_THREAD_1_BUSY_MASK                                                                      0x00000040L
+#define RLC_STAT__RLC_THREAD_2_BUSY_MASK                                                                      0x00000080L
+#define RLC_STAT__RESERVED_MASK                                                                               0xFFFFFF00L
+//RLC_REFCLOCK_TIMESTAMP_LSB
+#define RLC_REFCLOCK_TIMESTAMP_LSB__TIMESTAMP_LSB__SHIFT                                                      0x0
+#define RLC_REFCLOCK_TIMESTAMP_LSB__TIMESTAMP_LSB_MASK                                                        0xFFFFFFFFL
+//RLC_REFCLOCK_TIMESTAMP_MSB
+#define RLC_REFCLOCK_TIMESTAMP_MSB__TIMESTAMP_MSB__SHIFT                                                      0x0
+#define RLC_REFCLOCK_TIMESTAMP_MSB__TIMESTAMP_MSB_MASK                                                        0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_0
+#define RLC_GPM_TIMER_INT_0__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_0__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_1
+#define RLC_GPM_TIMER_INT_1__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_1__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_2
+#define RLC_GPM_TIMER_INT_2__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_2__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_3
+#define RLC_GPM_TIMER_INT_3__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_3__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_INT_4
+#define RLC_GPM_TIMER_INT_4__TIMER__SHIFT                                                                     0x0
+#define RLC_GPM_TIMER_INT_4__TIMER_MASK                                                                       0xFFFFFFFFL
+//RLC_GPM_TIMER_CTRL
+#define RLC_GPM_TIMER_CTRL__TIMER_0_EN__SHIFT                                                                 0x0
+#define RLC_GPM_TIMER_CTRL__TIMER_1_EN__SHIFT                                                                 0x1
+#define RLC_GPM_TIMER_CTRL__TIMER_2_EN__SHIFT                                                                 0x2
+#define RLC_GPM_TIMER_CTRL__TIMER_3_EN__SHIFT                                                                 0x3
+#define RLC_GPM_TIMER_CTRL__TIMER_4_EN__SHIFT                                                                 0x4
+#define RLC_GPM_TIMER_CTRL__RESERVED_1__SHIFT                                                                 0x5
+#define RLC_GPM_TIMER_CTRL__TIMER_0_AUTO_REARM__SHIFT                                                         0x8
+#define RLC_GPM_TIMER_CTRL__TIMER_1_AUTO_REARM__SHIFT                                                         0x9
+#define RLC_GPM_TIMER_CTRL__TIMER_2_AUTO_REARM__SHIFT                                                         0xa
+#define RLC_GPM_TIMER_CTRL__TIMER_3_AUTO_REARM__SHIFT                                                         0xb
+#define RLC_GPM_TIMER_CTRL__TIMER_4_AUTO_REARM__SHIFT                                                         0xc
+#define RLC_GPM_TIMER_CTRL__RESERVED_2__SHIFT                                                                 0xd
+#define RLC_GPM_TIMER_CTRL__TIMER_0_INT_CLEAR__SHIFT                                                          0x10
+#define RLC_GPM_TIMER_CTRL__TIMER_1_INT_CLEAR__SHIFT                                                          0x11
+#define RLC_GPM_TIMER_CTRL__TIMER_2_INT_CLEAR__SHIFT                                                          0x12
+#define RLC_GPM_TIMER_CTRL__TIMER_3_INT_CLEAR__SHIFT                                                          0x13
+#define RLC_GPM_TIMER_CTRL__TIMER_4_INT_CLEAR__SHIFT                                                          0x14
+#define RLC_GPM_TIMER_CTRL__RESERVED__SHIFT                                                                   0x15
+#define RLC_GPM_TIMER_CTRL__TIMER_0_EN_MASK                                                                   0x00000001L
+#define RLC_GPM_TIMER_CTRL__TIMER_1_EN_MASK                                                                   0x00000002L
+#define RLC_GPM_TIMER_CTRL__TIMER_2_EN_MASK                                                                   0x00000004L
+#define RLC_GPM_TIMER_CTRL__TIMER_3_EN_MASK                                                                   0x00000008L
+#define RLC_GPM_TIMER_CTRL__TIMER_4_EN_MASK                                                                   0x00000010L
+#define RLC_GPM_TIMER_CTRL__RESERVED_1_MASK                                                                   0x000000E0L
+#define RLC_GPM_TIMER_CTRL__TIMER_0_AUTO_REARM_MASK                                                           0x00000100L
+#define RLC_GPM_TIMER_CTRL__TIMER_1_AUTO_REARM_MASK                                                           0x00000200L
+#define RLC_GPM_TIMER_CTRL__TIMER_2_AUTO_REARM_MASK                                                           0x00000400L
+#define RLC_GPM_TIMER_CTRL__TIMER_3_AUTO_REARM_MASK                                                           0x00000800L
+#define RLC_GPM_TIMER_CTRL__TIMER_4_AUTO_REARM_MASK                                                           0x00001000L
+#define RLC_GPM_TIMER_CTRL__RESERVED_2_MASK                                                                   0x0000E000L
+#define RLC_GPM_TIMER_CTRL__TIMER_0_INT_CLEAR_MASK                                                            0x00010000L
+#define RLC_GPM_TIMER_CTRL__TIMER_1_INT_CLEAR_MASK                                                            0x00020000L
+#define RLC_GPM_TIMER_CTRL__TIMER_2_INT_CLEAR_MASK                                                            0x00040000L
+#define RLC_GPM_TIMER_CTRL__TIMER_3_INT_CLEAR_MASK                                                            0x00080000L
+#define RLC_GPM_TIMER_CTRL__TIMER_4_INT_CLEAR_MASK                                                            0x00100000L
+#define RLC_GPM_TIMER_CTRL__RESERVED_MASK                                                                     0xFFE00000L
+//RLC_GPM_TIMER_STAT
+#define RLC_GPM_TIMER_STAT__TIMER_0_STAT__SHIFT                                                               0x0
+#define RLC_GPM_TIMER_STAT__TIMER_1_STAT__SHIFT                                                               0x1
+#define RLC_GPM_TIMER_STAT__TIMER_2_STAT__SHIFT                                                               0x2
+#define RLC_GPM_TIMER_STAT__TIMER_3_STAT__SHIFT                                                               0x3
+#define RLC_GPM_TIMER_STAT__TIMER_4_STAT__SHIFT                                                               0x4
+#define RLC_GPM_TIMER_STAT__RESERVED_1__SHIFT                                                                 0x5
+#define RLC_GPM_TIMER_STAT__TIMER_0_ENABLE_SYNC__SHIFT                                                        0x8
+#define RLC_GPM_TIMER_STAT__TIMER_1_ENABLE_SYNC__SHIFT                                                        0x9
+#define RLC_GPM_TIMER_STAT__TIMER_2_ENABLE_SYNC__SHIFT                                                        0xa
+#define RLC_GPM_TIMER_STAT__TIMER_3_ENABLE_SYNC__SHIFT                                                        0xb
+#define RLC_GPM_TIMER_STAT__TIMER_4_ENABLE_SYNC__SHIFT                                                        0xc
+#define RLC_GPM_TIMER_STAT__RESERVED_2__SHIFT                                                                 0xd
+#define RLC_GPM_TIMER_STAT__TIMER_0_AUTO_REARM_SYNC__SHIFT                                                    0x10
+#define RLC_GPM_TIMER_STAT__TIMER_1_AUTO_REARM_SYNC__SHIFT                                                    0x11
+#define RLC_GPM_TIMER_STAT__TIMER_2_AUTO_REARM_SYNC__SHIFT                                                    0x12
+#define RLC_GPM_TIMER_STAT__TIMER_3_AUTO_REARM_SYNC__SHIFT                                                    0x13
+#define RLC_GPM_TIMER_STAT__TIMER_4_AUTO_REARM_SYNC__SHIFT                                                    0x14
+#define RLC_GPM_TIMER_STAT__RESERVED__SHIFT                                                                   0x15
+#define RLC_GPM_TIMER_STAT__TIMER_0_STAT_MASK                                                                 0x00000001L
+#define RLC_GPM_TIMER_STAT__TIMER_1_STAT_MASK                                                                 0x00000002L
+#define RLC_GPM_TIMER_STAT__TIMER_2_STAT_MASK                                                                 0x00000004L
+#define RLC_GPM_TIMER_STAT__TIMER_3_STAT_MASK                                                                 0x00000008L
+#define RLC_GPM_TIMER_STAT__TIMER_4_STAT_MASK                                                                 0x00000010L
+#define RLC_GPM_TIMER_STAT__RESERVED_1_MASK                                                                   0x000000E0L
+#define RLC_GPM_TIMER_STAT__TIMER_0_ENABLE_SYNC_MASK                                                          0x00000100L
+#define RLC_GPM_TIMER_STAT__TIMER_1_ENABLE_SYNC_MASK                                                          0x00000200L
+#define RLC_GPM_TIMER_STAT__TIMER_2_ENABLE_SYNC_MASK                                                          0x00000400L
+#define RLC_GPM_TIMER_STAT__TIMER_3_ENABLE_SYNC_MASK                                                          0x00000800L
+#define RLC_GPM_TIMER_STAT__TIMER_4_ENABLE_SYNC_MASK                                                          0x00001000L
+#define RLC_GPM_TIMER_STAT__RESERVED_2_MASK                                                                   0x0000E000L
+#define RLC_GPM_TIMER_STAT__TIMER_0_AUTO_REARM_SYNC_MASK                                                      0x00010000L
+#define RLC_GPM_TIMER_STAT__TIMER_1_AUTO_REARM_SYNC_MASK                                                      0x00020000L
+#define RLC_GPM_TIMER_STAT__TIMER_2_AUTO_REARM_SYNC_MASK                                                      0x00040000L
+#define RLC_GPM_TIMER_STAT__TIMER_3_AUTO_REARM_SYNC_MASK                                                      0x00080000L
+#define RLC_GPM_TIMER_STAT__TIMER_4_AUTO_REARM_SYNC_MASK                                                      0x00100000L
+#define RLC_GPM_TIMER_STAT__RESERVED_MASK                                                                     0xFFE00000L
+//RLC_GPM_LEGACY_INT_STAT
+#define RLC_GPM_LEGACY_INT_STAT__SPP_PVT_INT_CHANGED__SHIFT                                                   0x0
+#define RLC_GPM_LEGACY_INT_STAT__CP_RLC_STAT_INVAL_PEND_CHANGED__SHIFT                                        0x1
+#define RLC_GPM_LEGACY_INT_STAT__RLC_EOF_INT_CHANGED__SHIFT                                                   0x2
+#define RLC_GPM_LEGACY_INT_STAT__RLC_PG_CNTL_CHANGED__SHIFT                                                   0x3
+#define RLC_GPM_LEGACY_INT_STAT__STORE_LOAD_TIMER3_EXPIRED_T0__SHIFT                                          0x4
+#define RLC_GPM_LEGACY_INT_STAT__SPP_PVT_INT_CHANGED_MASK                                                     0x00000001L
+#define RLC_GPM_LEGACY_INT_STAT__CP_RLC_STAT_INVAL_PEND_CHANGED_MASK                                          0x00000002L
+#define RLC_GPM_LEGACY_INT_STAT__RLC_EOF_INT_CHANGED_MASK                                                     0x00000004L
+#define RLC_GPM_LEGACY_INT_STAT__RLC_PG_CNTL_CHANGED_MASK                                                     0x00000008L
+#define RLC_GPM_LEGACY_INT_STAT__STORE_LOAD_TIMER3_EXPIRED_T0_MASK                                            0x00000010L
+//RLC_GPM_LEGACY_INT_CLEAR
+#define RLC_GPM_LEGACY_INT_CLEAR__SPP_PVT_INT_CHANGED__SHIFT                                                  0x0
+#define RLC_GPM_LEGACY_INT_CLEAR__CP_RLC_STAT_INVAL_PEND_CHANGED__SHIFT                                       0x1
+#define RLC_GPM_LEGACY_INT_CLEAR__RLC_EOF_INT_CHANGED__SHIFT                                                  0x2
+#define RLC_GPM_LEGACY_INT_CLEAR__RLC_PG_CNTL_CHANGED__SHIFT                                                  0x3
+#define RLC_GPM_LEGACY_INT_CLEAR__RESERVED_4__SHIFT                                                           0x4
+#define RLC_GPM_LEGACY_INT_CLEAR__SPP_PVT_INT_CHANGED_MASK                                                    0x00000001L
+#define RLC_GPM_LEGACY_INT_CLEAR__CP_RLC_STAT_INVAL_PEND_CHANGED_MASK                                         0x00000002L
+#define RLC_GPM_LEGACY_INT_CLEAR__RLC_EOF_INT_CHANGED_MASK                                                    0x00000004L
+#define RLC_GPM_LEGACY_INT_CLEAR__RLC_PG_CNTL_CHANGED_MASK                                                    0x00000008L
+#define RLC_GPM_LEGACY_INT_CLEAR__RESERVED_4_MASK                                                             0x00000010L
+//RLC_INT_STAT
+#define RLC_INT_STAT__LAST_CP_RLC_INT_ID__SHIFT                                                               0x0
+#define RLC_INT_STAT__CP_RLC_INT_PENDING__SHIFT                                                               0x8
+#define RLC_INT_STAT__RESERVED__SHIFT                                                                         0x9
+#define RLC_INT_STAT__LAST_CP_RLC_INT_ID_MASK                                                                 0x000000FFL
+#define RLC_INT_STAT__CP_RLC_INT_PENDING_MASK                                                                 0x00000100L
+#define RLC_INT_STAT__RESERVED_MASK                                                                           0xFFFFFE00L
+//RLC_MGCG_CTRL
+#define RLC_MGCG_CTRL__MGCG_EN__SHIFT                                                                         0x0
+#define RLC_MGCG_CTRL__SILICON_EN__SHIFT                                                                      0x1
+#define RLC_MGCG_CTRL__SIMULATION_EN__SHIFT                                                                   0x2
+#define RLC_MGCG_CTRL__ON_DELAY__SHIFT                                                                        0x3
+#define RLC_MGCG_CTRL__OFF_HYSTERESIS__SHIFT                                                                  0x7
+#define RLC_MGCG_CTRL__SPARE__SHIFT                                                                           0xf
+#define RLC_MGCG_CTRL__MGCG_EN_MASK                                                                           0x00000001L
+#define RLC_MGCG_CTRL__SILICON_EN_MASK                                                                        0x00000002L
+#define RLC_MGCG_CTRL__SIMULATION_EN_MASK                                                                     0x00000004L
+#define RLC_MGCG_CTRL__ON_DELAY_MASK                                                                          0x00000078L
+#define RLC_MGCG_CTRL__OFF_HYSTERESIS_MASK                                                                    0x00007F80L
+#define RLC_MGCG_CTRL__SPARE_MASK                                                                             0xFFFF8000L
+//RLC_JUMP_TABLE_RESTORE
+#define RLC_JUMP_TABLE_RESTORE__ADDR__SHIFT                                                                   0x0
+#define RLC_JUMP_TABLE_RESTORE__ADDR_MASK                                                                     0xFFFFFFFFL
+//RLC_PG_DELAY_2
+#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE__SHIFT                                                           0x0
+#define RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT                                                               0x8
+#define RLC_PG_DELAY_2__PERWGP_TIMEOUT_VALUE__SHIFT                                                           0x10
+#define RLC_PG_DELAY_2__SERDES_TIMEOUT_VALUE_MASK                                                             0x000000FFL
+#define RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK                                                                 0x0000FF00L
+#define RLC_PG_DELAY_2__PERWGP_TIMEOUT_VALUE_MASK                                                             0xFFFF0000L
+//RLC_GPU_CLOCK_COUNT_LSB
+#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB__SHIFT                                                        0x0
+#define RLC_GPU_CLOCK_COUNT_LSB__GPU_CLOCKS_LSB_MASK                                                          0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_MSB
+#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB__SHIFT                                                        0x0
+#define RLC_GPU_CLOCK_COUNT_MSB__GPU_CLOCKS_MSB_MASK                                                          0xFFFFFFFFL
+//RLC_CAPTURE_GPU_CLOCK_COUNT
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE__SHIFT                                                           0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED__SHIFT                                                          0x1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__CAPTURE_MASK                                                             0x00000001L
+#define RLC_CAPTURE_GPU_CLOCK_COUNT__RESERVED_MASK                                                            0xFFFFFFFEL
+//RLC_UCODE_CNTL
+#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS__SHIFT                                                                0x0
+#define RLC_UCODE_CNTL__RLC_UCODE_FLAGS_MASK                                                                  0xFFFFFFFFL
+//RLC_GPM_THREAD_RESET
+#define RLC_GPM_THREAD_RESET__THREAD0_RESET__SHIFT                                                            0x0
+#define RLC_GPM_THREAD_RESET__THREAD1_RESET__SHIFT                                                            0x1
+#define RLC_GPM_THREAD_RESET__THREAD2_RESET__SHIFT                                                            0x2
+#define RLC_GPM_THREAD_RESET__THREAD3_RESET__SHIFT                                                            0x3
+#define RLC_GPM_THREAD_RESET__RESERVED__SHIFT                                                                 0x4
+#define RLC_GPM_THREAD_RESET__THREAD0_RESET_MASK                                                              0x00000001L
+#define RLC_GPM_THREAD_RESET__THREAD1_RESET_MASK                                                              0x00000002L
+#define RLC_GPM_THREAD_RESET__THREAD2_RESET_MASK                                                              0x00000004L
+#define RLC_GPM_THREAD_RESET__THREAD3_RESET_MASK                                                              0x00000008L
+#define RLC_GPM_THREAD_RESET__RESERVED_MASK                                                                   0xFFFFFFF0L
+//RLC_GPM_CP_DMA_COMPLETE_T0
+#define RLC_GPM_CP_DMA_COMPLETE_T0__DATA__SHIFT                                                               0x0
+#define RLC_GPM_CP_DMA_COMPLETE_T0__RESERVED__SHIFT                                                           0x1
+#define RLC_GPM_CP_DMA_COMPLETE_T0__DATA_MASK                                                                 0x00000001L
+#define RLC_GPM_CP_DMA_COMPLETE_T0__RESERVED_MASK                                                             0xFFFFFFFEL
+//RLC_GPM_CP_DMA_COMPLETE_T1
+#define RLC_GPM_CP_DMA_COMPLETE_T1__DATA__SHIFT                                                               0x0
+#define RLC_GPM_CP_DMA_COMPLETE_T1__RESERVED__SHIFT                                                           0x1
+#define RLC_GPM_CP_DMA_COMPLETE_T1__DATA_MASK                                                                 0x00000001L
+#define RLC_GPM_CP_DMA_COMPLETE_T1__RESERVED_MASK                                                             0xFFFFFFFEL
+//RLC_GPM_THREAD_INVALIDATE_CACHE
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD0_INVALIDATE_CACHE__SHIFT                                      0x0
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD1_INVALIDATE_CACHE__SHIFT                                      0x1
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD2_INVALIDATE_CACHE__SHIFT                                      0x2
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD3_INVALIDATE_CACHE__SHIFT                                      0x3
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__RESERVED__SHIFT                                                      0x4
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD0_INVALIDATE_CACHE_MASK                                        0x00000001L
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD1_INVALIDATE_CACHE_MASK                                        0x00000002L
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD2_INVALIDATE_CACHE_MASK                                        0x00000004L
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__THREAD3_INVALIDATE_CACHE_MASK                                        0x00000008L
+#define RLC_GPM_THREAD_INVALIDATE_CACHE__RESERVED_MASK                                                        0xFFFFFFF0L
+//RLC_CLK_COUNT_GFXCLK_LSB
+#define RLC_CLK_COUNT_GFXCLK_LSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_GFXCLK_LSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_GFXCLK_MSB
+#define RLC_CLK_COUNT_GFXCLK_MSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_GFXCLK_MSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_REFCLK_LSB
+#define RLC_CLK_COUNT_REFCLK_LSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_REFCLK_LSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_REFCLK_MSB
+#define RLC_CLK_COUNT_REFCLK_MSB__COUNTER__SHIFT                                                              0x0
+#define RLC_CLK_COUNT_REFCLK_MSB__COUNTER_MASK                                                                0xFFFFFFFFL
+//RLC_CLK_COUNT_CTRL
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RUN__SHIFT                                                                 0x0
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RESET__SHIFT                                                               0x1
+#define RLC_CLK_COUNT_CTRL__GFXCLK_SAMPLE__SHIFT                                                              0x2
+#define RLC_CLK_COUNT_CTRL__REFCLK_RUN__SHIFT                                                                 0x3
+#define RLC_CLK_COUNT_CTRL__REFCLK_RESET__SHIFT                                                               0x4
+#define RLC_CLK_COUNT_CTRL__REFCLK_SAMPLE__SHIFT                                                              0x5
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RUN_MASK                                                                   0x00000001L
+#define RLC_CLK_COUNT_CTRL__GFXCLK_RESET_MASK                                                                 0x00000002L
+#define RLC_CLK_COUNT_CTRL__GFXCLK_SAMPLE_MASK                                                                0x00000004L
+#define RLC_CLK_COUNT_CTRL__REFCLK_RUN_MASK                                                                   0x00000008L
+#define RLC_CLK_COUNT_CTRL__REFCLK_RESET_MASK                                                                 0x00000010L
+#define RLC_CLK_COUNT_CTRL__REFCLK_SAMPLE_MASK                                                                0x00000020L
+//RLC_CLK_COUNT_STAT
+#define RLC_CLK_COUNT_STAT__GFXCLK_VALID__SHIFT                                                               0x0
+#define RLC_CLK_COUNT_STAT__REFCLK_VALID__SHIFT                                                               0x1
+#define RLC_CLK_COUNT_STAT__REFCLK_RUN_RESYNC__SHIFT                                                          0x2
+#define RLC_CLK_COUNT_STAT__REFCLK_RESET_RESYNC__SHIFT                                                        0x3
+#define RLC_CLK_COUNT_STAT__REFCLK_SAMPLE_RESYNC__SHIFT                                                       0x4
+#define RLC_CLK_COUNT_STAT__RESERVED__SHIFT                                                                   0x5
+#define RLC_CLK_COUNT_STAT__GFXCLK_VALID_MASK                                                                 0x00000001L
+#define RLC_CLK_COUNT_STAT__REFCLK_VALID_MASK                                                                 0x00000002L
+#define RLC_CLK_COUNT_STAT__REFCLK_RUN_RESYNC_MASK                                                            0x00000004L
+#define RLC_CLK_COUNT_STAT__REFCLK_RESET_RESYNC_MASK                                                          0x00000008L
+#define RLC_CLK_COUNT_STAT__REFCLK_SAMPLE_RESYNC_MASK                                                         0x00000010L
+#define RLC_CLK_COUNT_STAT__RESERVED_MASK                                                                     0xFFFFFFE0L
+//RLC_RLCG_DOORBELL_CNTL
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_0_MODE__SHIFT                                                        0x0
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_1_MODE__SHIFT                                                        0x2
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_2_MODE__SHIFT                                                        0x4
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_3_MODE__SHIFT                                                        0x6
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_ID__SHIFT                                                            0x10
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_ID_EN__SHIFT                                                         0x15
+#define RLC_RLCG_DOORBELL_CNTL__RESERVED__SHIFT                                                               0x16
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_0_MODE_MASK                                                          0x00000003L
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_1_MODE_MASK                                                          0x0000000CL
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_2_MODE_MASK                                                          0x00000030L
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_3_MODE_MASK                                                          0x000000C0L
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_ID_MASK                                                              0x001F0000L
+#define RLC_RLCG_DOORBELL_CNTL__DOORBELL_ID_EN_MASK                                                           0x00200000L
+#define RLC_RLCG_DOORBELL_CNTL__RESERVED_MASK                                                                 0xFFC00000L
+//RLC_RLCG_DOORBELL_STAT
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_0_VALID__SHIFT                                                       0x0
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_1_VALID__SHIFT                                                       0x1
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_2_VALID__SHIFT                                                       0x2
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_3_VALID__SHIFT                                                       0x3
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_0_VALID_MASK                                                         0x00000001L
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_1_VALID_MASK                                                         0x00000002L
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_2_VALID_MASK                                                         0x00000004L
+#define RLC_RLCG_DOORBELL_STAT__DOORBELL_3_VALID_MASK                                                         0x00000008L
+//RLC_RLCG_DOORBELL_0_DATA_LO
+#define RLC_RLCG_DOORBELL_0_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_0_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_0_DATA_HI
+#define RLC_RLCG_DOORBELL_0_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_0_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_1_DATA_LO
+#define RLC_RLCG_DOORBELL_1_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_1_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_1_DATA_HI
+#define RLC_RLCG_DOORBELL_1_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_1_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_2_DATA_LO
+#define RLC_RLCG_DOORBELL_2_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_2_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_2_DATA_HI
+#define RLC_RLCG_DOORBELL_2_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_2_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_3_DATA_LO
+#define RLC_RLCG_DOORBELL_3_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_3_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCG_DOORBELL_3_DATA_HI
+#define RLC_RLCG_DOORBELL_3_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCG_DOORBELL_3_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_GPU_CLOCK_32_RES_SEL
+#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL__SHIFT                                                              0x0
+#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED__SHIFT                                                             0x6
+#define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL_MASK                                                                0x0000003FL
+#define RLC_GPU_CLOCK_32_RES_SEL__RESERVED_MASK                                                               0xFFFFFFC0L
+//RLC_GPU_CLOCK_32
+#define RLC_GPU_CLOCK_32__GPU_CLOCK_32__SHIFT                                                                 0x0
+#define RLC_GPU_CLOCK_32__GPU_CLOCK_32_MASK                                                                   0xFFFFFFFFL
+//RLC_PG_CNTL
+#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE__SHIFT                                                           0x0
+#define RLC_PG_CNTL__GFX_POWER_GATING_SRC__SHIFT                                                              0x1
+#define RLC_PG_CNTL__DYN_PER_WGP_PG_ENABLE__SHIFT                                                             0x2
+#define RLC_PG_CNTL__STATIC_PER_WGP_PG_ENABLE__SHIFT                                                          0x3
+#define RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE__SHIFT                                                            0x4
+#define RLC_PG_CNTL__RESERVED__SHIFT                                                                          0x5
+#define RLC_PG_CNTL__MEM_DS_DISABLE__SHIFT                                                                    0xd
+#define RLC_PG_CNTL__PG_OVERRIDE__SHIFT                                                                       0xe
+#define RLC_PG_CNTL__CP_PG_DISABLE__SHIFT                                                                     0xf
+#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE__SHIFT                                                             0x10
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE__SHIFT                                                     0x11
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE__SHIFT                                                     0x12
+#define RLC_PG_CNTL__RESERVED1__SHIFT                                                                         0x13
+#define RLC_PG_CNTL__Ultra_Low_Voltage_Enable__SHIFT                                                          0x15
+#define RLC_PG_CNTL__RESERVED2__SHIFT                                                                         0x16
+#define RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE__SHIFT                                                             0x17
+#define RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK                                                             0x00000001L
+#define RLC_PG_CNTL__GFX_POWER_GATING_SRC_MASK                                                                0x00000002L
+#define RLC_PG_CNTL__DYN_PER_WGP_PG_ENABLE_MASK                                                               0x00000004L
+#define RLC_PG_CNTL__STATIC_PER_WGP_PG_ENABLE_MASK                                                            0x00000008L
+#define RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK                                                              0x00000010L
+#define RLC_PG_CNTL__RESERVED_MASK                                                                            0x00001FE0L
+#define RLC_PG_CNTL__MEM_DS_DISABLE_MASK                                                                      0x00002000L
+#define RLC_PG_CNTL__PG_OVERRIDE_MASK                                                                         0x00004000L
+#define RLC_PG_CNTL__CP_PG_DISABLE_MASK                                                                       0x00008000L
+#define RLC_PG_CNTL__CHUB_HANDSHAKE_ENABLE_MASK                                                               0x00010000L
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK                                                       0x00020000L
+#define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK                                                       0x00040000L
+#define RLC_PG_CNTL__RESERVED1_MASK                                                                           0x00180000L
+#define RLC_PG_CNTL__Ultra_Low_Voltage_Enable_MASK                                                            0x00200000L
+#define RLC_PG_CNTL__RESERVED2_MASK                                                                           0x00400000L
+#define RLC_PG_CNTL__SMU_HANDSHAKE_DISABLE_MASK                                                               0x00800000L
+//RLC_GPM_THREAD_PRIORITY
+#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY__SHIFT                                                      0x0
+#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY__SHIFT                                                      0x8
+#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY__SHIFT                                                      0x10
+#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY__SHIFT                                                      0x18
+#define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY_MASK                                                        0x000000FFL
+#define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY_MASK                                                        0x0000FF00L
+#define RLC_GPM_THREAD_PRIORITY__THREAD2_PRIORITY_MASK                                                        0x00FF0000L
+#define RLC_GPM_THREAD_PRIORITY__THREAD3_PRIORITY_MASK                                                        0xFF000000L
+//RLC_GPM_THREAD_ENABLE
+#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE__SHIFT                                                          0x0
+#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE__SHIFT                                                          0x1
+#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE__SHIFT                                                          0x2
+#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE__SHIFT                                                          0x3
+#define RLC_GPM_THREAD_ENABLE__RESERVED__SHIFT                                                                0x4
+#define RLC_GPM_THREAD_ENABLE__THREAD0_ENABLE_MASK                                                            0x00000001L
+#define RLC_GPM_THREAD_ENABLE__THREAD1_ENABLE_MASK                                                            0x00000002L
+#define RLC_GPM_THREAD_ENABLE__THREAD2_ENABLE_MASK                                                            0x00000004L
+#define RLC_GPM_THREAD_ENABLE__THREAD3_ENABLE_MASK                                                            0x00000008L
+#define RLC_GPM_THREAD_ENABLE__RESERVED_MASK                                                                  0xFFFFFFF0L
+//RLC_RLCG_DOORBELL_RANGE
+#define RLC_RLCG_DOORBELL_RANGE__LOWER_ADDR_RESERVED__SHIFT                                                   0x0
+#define RLC_RLCG_DOORBELL_RANGE__LOWER_ADDR__SHIFT                                                            0x2
+#define RLC_RLCG_DOORBELL_RANGE__UPPER_ADDR_RESERVED__SHIFT                                                   0x10
+#define RLC_RLCG_DOORBELL_RANGE__UPPER_ADDR__SHIFT                                                            0x12
+#define RLC_RLCG_DOORBELL_RANGE__LOWER_ADDR_RESERVED_MASK                                                     0x00000003L
+#define RLC_RLCG_DOORBELL_RANGE__LOWER_ADDR_MASK                                                              0x00000FFCL
+#define RLC_RLCG_DOORBELL_RANGE__UPPER_ADDR_RESERVED_MASK                                                     0x00030000L
+#define RLC_RLCG_DOORBELL_RANGE__UPPER_ADDR_MASK                                                              0x0FFC0000L
+//RLC_CGTT_MGCG_OVERRIDE
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE__SHIFT                                             0x0
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE__SHIFT                                                 0x1
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE__SHIFT                                                    0x2
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE__SHIFT                                                    0x3
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE__SHIFT                                                    0x4
+#define RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE__SHIFT                                                0x5
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE__SHIFT                                                    0x6
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE__SHIFT                                                0x7
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE__SHIFT                                                    0x8
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE__SHIFT                                           0x9
+#define RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE__SHIFT                                                    0xa
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_16_11__SHIFT                                                         0xb
+#define RLC_CGTT_MGCG_OVERRIDE__GC_CAC_MGCG_CLK_CNTL__SHIFT                                                   0x11
+#define RLC_CGTT_MGCG_OVERRIDE__SE_CAC_MGCG_CLK_CNTL__SHIFT                                                   0x12
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_31_19__SHIFT                                                         0x13
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_REPEATER_FGCG_OVERRIDE_MASK                                               0x00000001L
+#define RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK                                                   0x00000002L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK                                                      0x00000004L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK                                                      0x00000008L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK                                                      0x00000010L
+#define RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK                                                  0x00000020L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK                                                      0x00000040L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK                                                  0x00000080L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_FGCG_OVERRIDE_MASK                                                      0x00000100L
+#define RLC_CGTT_MGCG_OVERRIDE__GFXIP_REPEATER_FGCG_OVERRIDE_MASK                                             0x00000200L
+#define RLC_CGTT_MGCG_OVERRIDE__PERFMON_CLOCK_STATE_MASK                                                      0x00000400L
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_16_11_MASK                                                           0x0001F800L
+#define RLC_CGTT_MGCG_OVERRIDE__GC_CAC_MGCG_CLK_CNTL_MASK                                                     0x00020000L
+#define RLC_CGTT_MGCG_OVERRIDE__SE_CAC_MGCG_CLK_CNTL_MASK                                                     0x00040000L
+#define RLC_CGTT_MGCG_OVERRIDE__RESERVED_31_19_MASK                                                           0xFFF80000L
+//RLC_CGCG_CGLS_CTRL
+#define RLC_CGCG_CGLS_CTRL__CGCG_EN__SHIFT                                                                    0x0
+#define RLC_CGCG_CGLS_CTRL__CGLS_EN__SHIFT                                                                    0x1
+#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT                                                   0x2
+#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT                                                    0x8
+#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER__SHIFT                                                            0x1b
+#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL__SHIFT                                                              0x1c
+#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE__SHIFT                                                                 0x1d
+#define RLC_CGCG_CGLS_CTRL__SIM_SILICON_EN__SHIFT                                                             0x1f
+#define RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK                                                                      0x00000001L
+#define RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK                                                                      0x00000002L
+#define RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY_MASK                                                     0x000000FCL
+#define RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD_MASK                                                      0x07FFFF00L
+#define RLC_CGCG_CGLS_CTRL__CGCG_CONTROLLER_MASK                                                              0x08000000L
+#define RLC_CGCG_CGLS_CTRL__CGCG_REG_CTRL_MASK                                                                0x10000000L
+#define RLC_CGCG_CGLS_CTRL__SLEEP_MODE_MASK                                                                   0x60000000L
+#define RLC_CGCG_CGLS_CTRL__SIM_SILICON_EN_MASK                                                               0x80000000L
+//RLC_CGCG_RAMP_CTRL
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT__SHIFT                                                        0x0
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT__SHIFT                                                         0x4
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT__SHIFT                                                          0x8
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT__SHIFT                                                           0xc
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT__SHIFT                                                             0x10
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT__SHIFT                                                            0x1c
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_START_UNIT_MASK                                                          0x0000000FL
+#define RLC_CGCG_RAMP_CTRL__DOWN_DIV_STEP_UNIT_MASK                                                           0x000000F0L
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_START_UNIT_MASK                                                            0x00000F00L
+#define RLC_CGCG_RAMP_CTRL__UP_DIV_STEP_UNIT_MASK                                                             0x0000F000L
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_CNT_MASK                                                               0x0FFF0000L
+#define RLC_CGCG_RAMP_CTRL__STEP_DELAY_UNIT_MASK                                                              0xF0000000L
+//RLC_DYN_PG_STATUS
+#define RLC_DYN_PG_STATUS__PG_STATUS_WGP_MASK__SHIFT                                                          0x0
+#define RLC_DYN_PG_STATUS__PG_STATUS_WGP_MASK_MASK                                                            0xFFFFFFFFL
+//RLC_DYN_PG_REQUEST
+#define RLC_DYN_PG_REQUEST__PG_REQUEST_WGP_MASK__SHIFT                                                        0x0
+#define RLC_DYN_PG_REQUEST__PG_REQUEST_WGP_MASK_MASK                                                          0xFFFFFFFFL
+//RLC_PG_DELAY
+#define RLC_PG_DELAY__POWER_UP_DELAY__SHIFT                                                                   0x0
+#define RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT                                                                 0x8
+#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT                                                              0x10
+#define RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT                                                                  0x18
+#define RLC_PG_DELAY__POWER_UP_DELAY_MASK                                                                     0x000000FFL
+#define RLC_PG_DELAY__POWER_DOWN_DELAY_MASK                                                                   0x0000FF00L
+#define RLC_PG_DELAY__CMD_PROPAGATE_DELAY_MASK                                                                0x00FF0000L
+#define RLC_PG_DELAY__MEM_SLEEP_DELAY_MASK                                                                    0xFF000000L
+//RLC_WGP_STATUS
+#define RLC_WGP_STATUS__WORK_PENDING__SHIFT                                                                   0x0
+#define RLC_WGP_STATUS__WORK_PENDING_MASK                                                                     0xFFFFFFFFL
+//RLC_PG_ALWAYS_ON_WGP_MASK
+#define RLC_PG_ALWAYS_ON_WGP_MASK__AON_WGP_MASK__SHIFT                                                        0x0
+#define RLC_PG_ALWAYS_ON_WGP_MASK__AON_WGP_MASK_MASK                                                          0xFFFFFFFFL
+//RLC_MAX_PG_WGP
+#define RLC_MAX_PG_WGP__MAX_POWERED_UP_WGP__SHIFT                                                             0x0
+#define RLC_MAX_PG_WGP__SPARE__SHIFT                                                                          0x8
+#define RLC_MAX_PG_WGP__MAX_POWERED_UP_WGP_MASK                                                               0x000000FFL
+#define RLC_MAX_PG_WGP__SPARE_MASK                                                                            0xFFFFFF00L
+//RLC_AUTO_PG_CTRL
+#define RLC_AUTO_PG_CTRL__AUTO_PG_EN__SHIFT                                                                   0x0
+#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN__SHIFT                                                0x1
+#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN__SHIFT                                                              0x2
+#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT                                             0x3
+#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD__SHIFT                                             0x13
+#define RLC_AUTO_PG_CTRL__AUTO_PG_EN_MASK                                                                     0x00000001L
+#define RLC_AUTO_PG_CTRL__AUTO_GRBM_REG_SAVE_ON_IDLE_EN_MASK                                                  0x00000002L
+#define RLC_AUTO_PG_CTRL__AUTO_WAKE_UP_EN_MASK                                                                0x00000004L
+#define RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK                                               0x0007FFF8L
+#define RLC_AUTO_PG_CTRL__PG_AFTER_GRBM_REG_SAVE_THRESHOLD_MASK                                               0xFFF80000L
+//RLC_SERDES_RD_INDEX
+#define RLC_SERDES_RD_INDEX__DATA_REG_ID__SHIFT                                                               0x0
+#define RLC_SERDES_RD_INDEX__SPARE__SHIFT                                                                     0x2
+#define RLC_SERDES_RD_INDEX__DATA_REG_ID_MASK                                                                 0x00000003L
+#define RLC_SERDES_RD_INDEX__SPARE_MASK                                                                       0xFFFFFFFCL
+//RLC_SERDES_RD_DATA_0
+#define RLC_SERDES_RD_DATA_0__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_0__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_RD_DATA_1
+#define RLC_SERDES_RD_DATA_1__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_1__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_RD_DATA_2
+#define RLC_SERDES_RD_DATA_2__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_2__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_RD_DATA_3
+#define RLC_SERDES_RD_DATA_3__DATA__SHIFT                                                                     0x0
+#define RLC_SERDES_RD_DATA_3__DATA_MASK                                                                       0xFFFFFFFFL
+//RLC_SERDES_MASK
+#define RLC_SERDES_MASK__GC_CENTER_HUB_0__SHIFT                                                               0x0
+#define RLC_SERDES_MASK__GC_CENTER_HUB_1__SHIFT                                                               0x1
+#define RLC_SERDES_MASK__RESERVED__SHIFT                                                                      0x2
+#define RLC_SERDES_MASK__GC_SE_0__SHIFT                                                                       0x10
+#define RLC_SERDES_MASK__GC_SE_1__SHIFT                                                                       0x11
+#define RLC_SERDES_MASK__GC_SE_2__SHIFT                                                                       0x12
+#define RLC_SERDES_MASK__GC_SE_3__SHIFT                                                                       0x13
+#define RLC_SERDES_MASK__GC_SE_4__SHIFT                                                                       0x14
+#define RLC_SERDES_MASK__GC_SE_5__SHIFT                                                                       0x15
+#define RLC_SERDES_MASK__GC_SE_6__SHIFT                                                                       0x16
+#define RLC_SERDES_MASK__GC_SE_7__SHIFT                                                                       0x17
+#define RLC_SERDES_MASK__RESERVED_31_24__SHIFT                                                                0x18
+#define RLC_SERDES_MASK__GC_CENTER_HUB_0_MASK                                                                 0x00000001L
+#define RLC_SERDES_MASK__GC_CENTER_HUB_1_MASK                                                                 0x00000002L
+#define RLC_SERDES_MASK__RESERVED_MASK                                                                        0x0000FFFCL
+#define RLC_SERDES_MASK__GC_SE_0_MASK                                                                         0x00010000L
+#define RLC_SERDES_MASK__GC_SE_1_MASK                                                                         0x00020000L
+#define RLC_SERDES_MASK__GC_SE_2_MASK                                                                         0x00040000L
+#define RLC_SERDES_MASK__GC_SE_3_MASK                                                                         0x00080000L
+#define RLC_SERDES_MASK__GC_SE_4_MASK                                                                         0x00100000L
+#define RLC_SERDES_MASK__GC_SE_5_MASK                                                                         0x00200000L
+#define RLC_SERDES_MASK__GC_SE_6_MASK                                                                         0x00400000L
+#define RLC_SERDES_MASK__GC_SE_7_MASK                                                                         0x00800000L
+#define RLC_SERDES_MASK__RESERVED_31_24_MASK                                                                  0xFF000000L
+//RLC_SERDES_CTRL
+#define RLC_SERDES_CTRL__BPM_BROADCAST__SHIFT                                                                 0x0
+#define RLC_SERDES_CTRL__BPM_REG_WRITE__SHIFT                                                                 0x1
+#define RLC_SERDES_CTRL__BPM_LONG_CMD__SHIFT                                                                  0x2
+#define RLC_SERDES_CTRL__BPM_ADDR__SHIFT                                                                      0x3
+#define RLC_SERDES_CTRL__REG_ADDR__SHIFT                                                                      0x10
+#define RLC_SERDES_CTRL__BPM_BROADCAST_MASK                                                                   0x000001L
+#define RLC_SERDES_CTRL__BPM_REG_WRITE_MASK                                                                   0x000002L
+#define RLC_SERDES_CTRL__BPM_LONG_CMD_MASK                                                                    0x000004L
+#define RLC_SERDES_CTRL__BPM_ADDR_MASK                                                                        0x00FFF8L
+#define RLC_SERDES_CTRL__REG_ADDR_MASK                                                                        0xFF0000L
+//RLC_SERDES_DATA
+#define RLC_SERDES_DATA__DATA__SHIFT                                                                          0x0
+#define RLC_SERDES_DATA__DATA_MASK                                                                            0xFFFFFFFFL
+//RLC_SERDES_BUSY
+#define RLC_SERDES_BUSY__GC_CENTER_HUB_0__SHIFT                                                               0x0
+#define RLC_SERDES_BUSY__GC_CENTER_HUB_1__SHIFT                                                               0x1
+#define RLC_SERDES_BUSY__RESERVED__SHIFT                                                                      0x2
+#define RLC_SERDES_BUSY__GC_SE_0__SHIFT                                                                       0x10
+#define RLC_SERDES_BUSY__GC_SE_1__SHIFT                                                                       0x11
+#define RLC_SERDES_BUSY__GC_SE_2__SHIFT                                                                       0x12
+#define RLC_SERDES_BUSY__GC_SE_3__SHIFT                                                                       0x13
+#define RLC_SERDES_BUSY__GC_SE_4__SHIFT                                                                       0x14
+#define RLC_SERDES_BUSY__GC_SE_5__SHIFT                                                                       0x15
+#define RLC_SERDES_BUSY__GC_SE_6__SHIFT                                                                       0x16
+#define RLC_SERDES_BUSY__GC_SE_7__SHIFT                                                                       0x17
+#define RLC_SERDES_BUSY__RESERVED_29_24__SHIFT                                                                0x18
+#define RLC_SERDES_BUSY__RD_FIFO_NOT_EMPTY__SHIFT                                                             0x1e
+#define RLC_SERDES_BUSY__RD_PENDING__SHIFT                                                                    0x1f
+#define RLC_SERDES_BUSY__GC_CENTER_HUB_0_MASK                                                                 0x00000001L
+#define RLC_SERDES_BUSY__GC_CENTER_HUB_1_MASK                                                                 0x00000002L
+#define RLC_SERDES_BUSY__RESERVED_MASK                                                                        0x0000FFFCL
+#define RLC_SERDES_BUSY__GC_SE_0_MASK                                                                         0x00010000L
+#define RLC_SERDES_BUSY__GC_SE_1_MASK                                                                         0x00020000L
+#define RLC_SERDES_BUSY__GC_SE_2_MASK                                                                         0x00040000L
+#define RLC_SERDES_BUSY__GC_SE_3_MASK                                                                         0x00080000L
+#define RLC_SERDES_BUSY__GC_SE_4_MASK                                                                         0x00100000L
+#define RLC_SERDES_BUSY__GC_SE_5_MASK                                                                         0x00200000L
+#define RLC_SERDES_BUSY__GC_SE_6_MASK                                                                         0x00400000L
+#define RLC_SERDES_BUSY__GC_SE_7_MASK                                                                         0x00800000L
+#define RLC_SERDES_BUSY__RESERVED_29_24_MASK                                                                  0x3F000000L
+#define RLC_SERDES_BUSY__RD_FIFO_NOT_EMPTY_MASK                                                               0x40000000L
+#define RLC_SERDES_BUSY__RD_PENDING_MASK                                                                      0x80000000L
+//RLC_GPM_GENERAL_0
+#define RLC_GPM_GENERAL_0__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_0__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_1
+#define RLC_GPM_GENERAL_1__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_1__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_2
+#define RLC_GPM_GENERAL_2__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_2__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_3
+#define RLC_GPM_GENERAL_3__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_3__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_4
+#define RLC_GPM_GENERAL_4__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_4__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_5
+#define RLC_GPM_GENERAL_5__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_5__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_6
+#define RLC_GPM_GENERAL_6__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_6__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_7
+#define RLC_GPM_GENERAL_7__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_7__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_STATIC_PG_STATUS
+#define RLC_STATIC_PG_STATUS__PG_STATUS_WGP_MASK__SHIFT                                                       0x0
+#define RLC_STATIC_PG_STATUS__PG_STATUS_WGP_MASK_MASK                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_16
+#define RLC_GPM_GENERAL_16__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_16__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_PG_DELAY_3
+#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT                                                        0x0
+#define RLC_PG_DELAY_3__RESERVED__SHIFT                                                                       0x8
+#define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK                                                          0x000000FFL
+#define RLC_PG_DELAY_3__RESERVED_MASK                                                                         0xFFFFFF00L
+//RLC_GPR_REG1
+#define RLC_GPR_REG1__DATA__SHIFT                                                                             0x0
+#define RLC_GPR_REG1__DATA_MASK                                                                               0xFFFFFFFFL
+//RLC_GPR_REG2
+#define RLC_GPR_REG2__DATA__SHIFT                                                                             0x0
+#define RLC_GPR_REG2__DATA_MASK                                                                               0xFFFFFFFFL
+//RLC_GPM_INT_DISABLE_TH0
+#define RLC_GPM_INT_DISABLE_TH0__DISABLE_INT__SHIFT                                                           0x0
+#define RLC_GPM_INT_DISABLE_TH0__DISABLE_INT_MASK                                                             0xFFFFFFFFL
+//RLC_GPM_LEGACY_INT_DISABLE
+#define RLC_GPM_LEGACY_INT_DISABLE__SPP_PVT_INT_CHANGED__SHIFT                                                0x0
+#define RLC_GPM_LEGACY_INT_DISABLE__CP_RLC_STAT_INVAL_PEND_CHANGED__SHIFT                                     0x1
+#define RLC_GPM_LEGACY_INT_DISABLE__RLC_EOF_INT_CHANGED__SHIFT                                                0x2
+#define RLC_GPM_LEGACY_INT_DISABLE__RLC_PG_CNTL_CHANGED__SHIFT                                                0x3
+#define RLC_GPM_LEGACY_INT_DISABLE__STORE_LOAD_TIMER3_EXPIRED_T0__SHIFT                                       0x4
+#define RLC_GPM_LEGACY_INT_DISABLE__SPP_PVT_INT_CHANGED_MASK                                                  0x00000001L
+#define RLC_GPM_LEGACY_INT_DISABLE__CP_RLC_STAT_INVAL_PEND_CHANGED_MASK                                       0x00000002L
+#define RLC_GPM_LEGACY_INT_DISABLE__RLC_EOF_INT_CHANGED_MASK                                                  0x00000004L
+#define RLC_GPM_LEGACY_INT_DISABLE__RLC_PG_CNTL_CHANGED_MASK                                                  0x00000008L
+#define RLC_GPM_LEGACY_INT_DISABLE__STORE_LOAD_TIMER3_EXPIRED_T0_MASK                                         0x00000010L
+//RLC_GPM_INT_FORCE_TH0
+#define RLC_GPM_INT_FORCE_TH0__FORCE_INT__SHIFT                                                               0x0
+#define RLC_GPM_INT_FORCE_TH0__FORCE_INT_MASK                                                                 0xFFFFFFFFL
+//RLC_SRM_CNTL
+#define RLC_SRM_CNTL__SRM_ENABLE__SHIFT                                                                       0x0
+#define RLC_SRM_CNTL__AUTO_INCR_ADDR__SHIFT                                                                   0x1
+#define RLC_SRM_CNTL__RESERVED__SHIFT                                                                         0x2
+#define RLC_SRM_CNTL__SRM_ENABLE_MASK                                                                         0x00000001L
+#define RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK                                                                     0x00000002L
+#define RLC_SRM_CNTL__RESERVED_MASK                                                                           0xFFFFFFFCL
+//RLC_SRM_GPM_COMMAND_STATUS
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_EMPTY__SHIFT                                                         0x0
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_FULL__SHIFT                                                          0x1
+#define RLC_SRM_GPM_COMMAND_STATUS__RESERVED__SHIFT                                                           0x2
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_EMPTY_MASK                                                           0x00000001L
+#define RLC_SRM_GPM_COMMAND_STATUS__FIFO_FULL_MASK                                                            0x00000002L
+#define RLC_SRM_GPM_COMMAND_STATUS__RESERVED_MASK                                                             0xFFFFFFFCL
+//RLC_SRM_INDEX_CNTL_ADDR_0
+#define RLC_SRM_INDEX_CNTL_ADDR_0__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_0__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_1
+#define RLC_SRM_INDEX_CNTL_ADDR_1__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_1__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_2
+#define RLC_SRM_INDEX_CNTL_ADDR_2__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_2__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_3
+#define RLC_SRM_INDEX_CNTL_ADDR_3__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_3__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_4
+#define RLC_SRM_INDEX_CNTL_ADDR_4__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_4__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_5
+#define RLC_SRM_INDEX_CNTL_ADDR_5__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_5__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_6
+#define RLC_SRM_INDEX_CNTL_ADDR_6__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_6__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_ADDR_7
+#define RLC_SRM_INDEX_CNTL_ADDR_7__ADDRESS__SHIFT                                                             0x0
+#define RLC_SRM_INDEX_CNTL_ADDR_7__ADDRESS_MASK                                                               0x0003FFFFL
+//RLC_SRM_INDEX_CNTL_DATA_0
+#define RLC_SRM_INDEX_CNTL_DATA_0__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_0__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_1
+#define RLC_SRM_INDEX_CNTL_DATA_1__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_1__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_2
+#define RLC_SRM_INDEX_CNTL_DATA_2__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_2__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_3
+#define RLC_SRM_INDEX_CNTL_DATA_3__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_3__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_4
+#define RLC_SRM_INDEX_CNTL_DATA_4__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_4__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_5
+#define RLC_SRM_INDEX_CNTL_DATA_5__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_5__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_6
+#define RLC_SRM_INDEX_CNTL_DATA_6__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_6__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_INDEX_CNTL_DATA_7
+#define RLC_SRM_INDEX_CNTL_DATA_7__DATA__SHIFT                                                                0x0
+#define RLC_SRM_INDEX_CNTL_DATA_7__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_SRM_STAT
+#define RLC_SRM_STAT__SRM_BUSY__SHIFT                                                                         0x0
+#define RLC_SRM_STAT__SRM_BUSY_DELAY__SHIFT                                                                   0x1
+#define RLC_SRM_STAT__RESERVED__SHIFT                                                                         0x2
+#define RLC_SRM_STAT__SRM_BUSY_MASK                                                                           0x00000001L
+#define RLC_SRM_STAT__SRM_BUSY_DELAY_MASK                                                                     0x00000002L
+#define RLC_SRM_STAT__RESERVED_MASK                                                                           0xFFFFFFFCL
+//RLC_GPM_GENERAL_8
+#define RLC_GPM_GENERAL_8__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_8__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_9
+#define RLC_GPM_GENERAL_9__DATA__SHIFT                                                                        0x0
+#define RLC_GPM_GENERAL_9__DATA_MASK                                                                          0xFFFFFFFFL
+//RLC_GPM_GENERAL_10
+#define RLC_GPM_GENERAL_10__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_10__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_11
+#define RLC_GPM_GENERAL_11__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_11__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_12
+#define RLC_GPM_GENERAL_12__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_12__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_UTCL1_CNTL_0
+#define RLC_GPM_UTCL1_CNTL_0__XNACK_REDO_TIMER_CNT__SHIFT                                                     0x0
+#define RLC_GPM_UTCL1_CNTL_0__DROP_MODE__SHIFT                                                                0x18
+#define RLC_GPM_UTCL1_CNTL_0__BYPASS__SHIFT                                                                   0x19
+#define RLC_GPM_UTCL1_CNTL_0__INVALIDATE__SHIFT                                                               0x1a
+#define RLC_GPM_UTCL1_CNTL_0__FRAG_LIMIT_MODE__SHIFT                                                          0x1b
+#define RLC_GPM_UTCL1_CNTL_0__FORCE_SNOOP__SHIFT                                                              0x1c
+#define RLC_GPM_UTCL1_CNTL_0__RESERVED__SHIFT                                                                 0x1e
+#define RLC_GPM_UTCL1_CNTL_0__XNACK_REDO_TIMER_CNT_MASK                                                       0x000FFFFFL
+#define RLC_GPM_UTCL1_CNTL_0__DROP_MODE_MASK                                                                  0x01000000L
+#define RLC_GPM_UTCL1_CNTL_0__BYPASS_MASK                                                                     0x02000000L
+#define RLC_GPM_UTCL1_CNTL_0__INVALIDATE_MASK                                                                 0x04000000L
+#define RLC_GPM_UTCL1_CNTL_0__FRAG_LIMIT_MODE_MASK                                                            0x08000000L
+#define RLC_GPM_UTCL1_CNTL_0__FORCE_SNOOP_MASK                                                                0x10000000L
+#define RLC_GPM_UTCL1_CNTL_0__RESERVED_MASK                                                                   0xC0000000L
+//RLC_GPM_UTCL1_CNTL_1
+#define RLC_GPM_UTCL1_CNTL_1__XNACK_REDO_TIMER_CNT__SHIFT                                                     0x0
+#define RLC_GPM_UTCL1_CNTL_1__DROP_MODE__SHIFT                                                                0x18
+#define RLC_GPM_UTCL1_CNTL_1__BYPASS__SHIFT                                                                   0x19
+#define RLC_GPM_UTCL1_CNTL_1__INVALIDATE__SHIFT                                                               0x1a
+#define RLC_GPM_UTCL1_CNTL_1__FRAG_LIMIT_MODE__SHIFT                                                          0x1b
+#define RLC_GPM_UTCL1_CNTL_1__FORCE_SNOOP__SHIFT                                                              0x1c
+#define RLC_GPM_UTCL1_CNTL_1__RESERVED__SHIFT                                                                 0x1e
+#define RLC_GPM_UTCL1_CNTL_1__XNACK_REDO_TIMER_CNT_MASK                                                       0x000FFFFFL
+#define RLC_GPM_UTCL1_CNTL_1__DROP_MODE_MASK                                                                  0x01000000L
+#define RLC_GPM_UTCL1_CNTL_1__BYPASS_MASK                                                                     0x02000000L
+#define RLC_GPM_UTCL1_CNTL_1__INVALIDATE_MASK                                                                 0x04000000L
+#define RLC_GPM_UTCL1_CNTL_1__FRAG_LIMIT_MODE_MASK                                                            0x08000000L
+#define RLC_GPM_UTCL1_CNTL_1__FORCE_SNOOP_MASK                                                                0x10000000L
+#define RLC_GPM_UTCL1_CNTL_1__RESERVED_MASK                                                                   0xC0000000L
+//RLC_GPM_UTCL1_CNTL_2
+#define RLC_GPM_UTCL1_CNTL_2__XNACK_REDO_TIMER_CNT__SHIFT                                                     0x0
+#define RLC_GPM_UTCL1_CNTL_2__DROP_MODE__SHIFT                                                                0x18
+#define RLC_GPM_UTCL1_CNTL_2__BYPASS__SHIFT                                                                   0x19
+#define RLC_GPM_UTCL1_CNTL_2__INVALIDATE__SHIFT                                                               0x1a
+#define RLC_GPM_UTCL1_CNTL_2__FRAG_LIMIT_MODE__SHIFT                                                          0x1b
+#define RLC_GPM_UTCL1_CNTL_2__FORCE_SNOOP__SHIFT                                                              0x1c
+#define RLC_GPM_UTCL1_CNTL_2__RESERVED__SHIFT                                                                 0x1e
+#define RLC_GPM_UTCL1_CNTL_2__XNACK_REDO_TIMER_CNT_MASK                                                       0x000FFFFFL
+#define RLC_GPM_UTCL1_CNTL_2__DROP_MODE_MASK                                                                  0x01000000L
+#define RLC_GPM_UTCL1_CNTL_2__BYPASS_MASK                                                                     0x02000000L
+#define RLC_GPM_UTCL1_CNTL_2__INVALIDATE_MASK                                                                 0x04000000L
+#define RLC_GPM_UTCL1_CNTL_2__FRAG_LIMIT_MODE_MASK                                                            0x08000000L
+#define RLC_GPM_UTCL1_CNTL_2__FORCE_SNOOP_MASK                                                                0x10000000L
+#define RLC_GPM_UTCL1_CNTL_2__RESERVED_MASK                                                                   0xC0000000L
+//RLC_SPM_UTCL1_CNTL
+#define RLC_SPM_UTCL1_CNTL__XNACK_REDO_TIMER_CNT__SHIFT                                                       0x0
+#define RLC_SPM_UTCL1_CNTL__DROP_MODE__SHIFT                                                                  0x18
+#define RLC_SPM_UTCL1_CNTL__BYPASS__SHIFT                                                                     0x19
+#define RLC_SPM_UTCL1_CNTL__INVALIDATE__SHIFT                                                                 0x1a
+#define RLC_SPM_UTCL1_CNTL__FRAG_LIMIT_MODE__SHIFT                                                            0x1b
+#define RLC_SPM_UTCL1_CNTL__FORCE_SNOOP__SHIFT                                                                0x1c
+#define RLC_SPM_UTCL1_CNTL__RESERVED__SHIFT                                                                   0x1e
+#define RLC_SPM_UTCL1_CNTL__XNACK_REDO_TIMER_CNT_MASK                                                         0x000FFFFFL
+#define RLC_SPM_UTCL1_CNTL__DROP_MODE_MASK                                                                    0x01000000L
+#define RLC_SPM_UTCL1_CNTL__BYPASS_MASK                                                                       0x02000000L
+#define RLC_SPM_UTCL1_CNTL__INVALIDATE_MASK                                                                   0x04000000L
+#define RLC_SPM_UTCL1_CNTL__FRAG_LIMIT_MODE_MASK                                                              0x08000000L
+#define RLC_SPM_UTCL1_CNTL__FORCE_SNOOP_MASK                                                                  0x10000000L
+#define RLC_SPM_UTCL1_CNTL__RESERVED_MASK                                                                     0xC0000000L
+//RLC_UTCL1_STATUS_2
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_BUSY__SHIFT                                                         0x0
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_BUSY__SHIFT                                                         0x1
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_BUSY__SHIFT                                                         0x2
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_BUSY__SHIFT                                                             0x3
+#define RLC_UTCL1_STATUS_2__RESERVED_1__SHIFT                                                                 0x4
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_StallOnTrans__SHIFT                                                 0x5
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_StallOnTrans__SHIFT                                                 0x6
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_StallOnTrans__SHIFT                                                 0x7
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_StallOnTrans__SHIFT                                                     0x8
+#define RLC_UTCL1_STATUS_2__RESERVED__SHIFT                                                                   0x9
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_BUSY_MASK                                                           0x00000001L
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_BUSY_MASK                                                           0x00000002L
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_BUSY_MASK                                                           0x00000004L
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_BUSY_MASK                                                               0x00000008L
+#define RLC_UTCL1_STATUS_2__RESERVED_1_MASK                                                                   0x00000010L
+#define RLC_UTCL1_STATUS_2__GPM_TH0_UTCL1_StallOnTrans_MASK                                                   0x00000020L
+#define RLC_UTCL1_STATUS_2__GPM_TH1_UTCL1_StallOnTrans_MASK                                                   0x00000040L
+#define RLC_UTCL1_STATUS_2__GPM_TH2_UTCL1_StallOnTrans_MASK                                                   0x00000080L
+#define RLC_UTCL1_STATUS_2__SPM_UTCL1_StallOnTrans_MASK                                                       0x00000100L
+#define RLC_UTCL1_STATUS_2__RESERVED_MASK                                                                     0xFFFFFE00L
+//RLC_SPM_UTCL1_ERROR_1
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqError__SHIFT                                                     0x0
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorVmid__SHIFT                                                 0x2
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                             0x6
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqError_MASK                                                       0x00000003L
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorVmid_MASK                                                   0x0000003CL
+#define RLC_SPM_UTCL1_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                               0x000003C0L
+//RLC_SPM_UTCL1_ERROR_2
+#define RLC_SPM_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                             0x0
+#define RLC_SPM_UTCL1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                               0xFFFFFFFFL
+//RLC_GPM_UTCL1_TH0_ERROR_1
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqError__SHIFT                                                 0x0
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorVmid__SHIFT                                             0x2
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                         0x6
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqError_MASK                                                   0x00000003L
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorVmid_MASK                                               0x0000003CL
+#define RLC_GPM_UTCL1_TH0_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                           0x000003C0L
+//RLC_GPM_UTCL1_TH0_ERROR_2
+#define RLC_GPM_UTCL1_TH0_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                         0x0
+#define RLC_GPM_UTCL1_TH0_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                           0xFFFFFFFFL
+//RLC_GPM_UTCL1_TH1_ERROR_1
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqError__SHIFT                                                 0x0
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorVmid__SHIFT                                             0x2
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                         0x6
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqError_MASK                                                   0x00000003L
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorVmid_MASK                                               0x0000003CL
+#define RLC_GPM_UTCL1_TH1_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                           0x000003C0L
+//RLC_GPM_UTCL1_TH1_ERROR_2
+#define RLC_GPM_UTCL1_TH1_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                         0x0
+#define RLC_GPM_UTCL1_TH1_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                           0xFFFFFFFFL
+//RLC_GPM_UTCL1_TH2_ERROR_1
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqError__SHIFT                                                 0x0
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorVmid__SHIFT                                             0x2
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorAddr_MSB__SHIFT                                         0x6
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqError_MASK                                                   0x00000003L
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorVmid_MASK                                               0x0000003CL
+#define RLC_GPM_UTCL1_TH2_ERROR_1__Translated_ReqErrorAddr_MSB_MASK                                           0x000003C0L
+//RLC_GPM_UTCL1_TH2_ERROR_2
+#define RLC_GPM_UTCL1_TH2_ERROR_2__Translated_ReqErrorAddr_LSB__SHIFT                                         0x0
+#define RLC_GPM_UTCL1_TH2_ERROR_2__Translated_ReqErrorAddr_LSB_MASK                                           0xFFFFFFFFL
+//RLC_CGCG_CGLS_CTRL_3D
+#define RLC_CGCG_CGLS_CTRL_3D__CGCG_EN__SHIFT                                                                 0x0
+#define RLC_CGCG_CGLS_CTRL_3D__CGLS_EN__SHIFT                                                                 0x1
+#define RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT                                                0x2
+#define RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT                                                 0x8
+#define RLC_CGCG_CGLS_CTRL_3D__CGCG_CONTROLLER__SHIFT                                                         0x1b
+#define RLC_CGCG_CGLS_CTRL_3D__CGCG_REG_CTRL__SHIFT                                                           0x1c
+#define RLC_CGCG_CGLS_CTRL_3D__SLEEP_MODE__SHIFT                                                              0x1d
+#define RLC_CGCG_CGLS_CTRL_3D__SIM_SILICON_EN__SHIFT                                                          0x1f
+#define RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK                                                                   0x00000001L
+#define RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK                                                                   0x00000002L
+#define RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY_MASK                                                  0x000000FCL
+#define RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD_MASK                                                   0x07FFFF00L
+#define RLC_CGCG_CGLS_CTRL_3D__CGCG_CONTROLLER_MASK                                                           0x08000000L
+#define RLC_CGCG_CGLS_CTRL_3D__CGCG_REG_CTRL_MASK                                                             0x10000000L
+#define RLC_CGCG_CGLS_CTRL_3D__SLEEP_MODE_MASK                                                                0x60000000L
+#define RLC_CGCG_CGLS_CTRL_3D__SIM_SILICON_EN_MASK                                                            0x80000000L
+//RLC_CGCG_RAMP_CTRL_3D
+#define RLC_CGCG_RAMP_CTRL_3D__DOWN_DIV_START_UNIT__SHIFT                                                     0x0
+#define RLC_CGCG_RAMP_CTRL_3D__DOWN_DIV_STEP_UNIT__SHIFT                                                      0x4
+#define RLC_CGCG_RAMP_CTRL_3D__UP_DIV_START_UNIT__SHIFT                                                       0x8
+#define RLC_CGCG_RAMP_CTRL_3D__UP_DIV_STEP_UNIT__SHIFT                                                        0xc
+#define RLC_CGCG_RAMP_CTRL_3D__STEP_DELAY_CNT__SHIFT                                                          0x10
+#define RLC_CGCG_RAMP_CTRL_3D__STEP_DELAY_UNIT__SHIFT                                                         0x1c
+#define RLC_CGCG_RAMP_CTRL_3D__DOWN_DIV_START_UNIT_MASK                                                       0x0000000FL
+#define RLC_CGCG_RAMP_CTRL_3D__DOWN_DIV_STEP_UNIT_MASK                                                        0x000000F0L
+#define RLC_CGCG_RAMP_CTRL_3D__UP_DIV_START_UNIT_MASK                                                         0x00000F00L
+#define RLC_CGCG_RAMP_CTRL_3D__UP_DIV_STEP_UNIT_MASK                                                          0x0000F000L
+#define RLC_CGCG_RAMP_CTRL_3D__STEP_DELAY_CNT_MASK                                                            0x0FFF0000L
+#define RLC_CGCG_RAMP_CTRL_3D__STEP_DELAY_UNIT_MASK                                                           0xF0000000L
+//RLC_SEMAPHORE_0
+#define RLC_SEMAPHORE_0__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_0__RESERVED__SHIFT                                                                      0x5
+#define RLC_SEMAPHORE_0__CLIENT_ID_MASK                                                                       0x0000001FL
+#define RLC_SEMAPHORE_0__RESERVED_MASK                                                                        0xFFFFFFE0L
+//RLC_SEMAPHORE_1
+#define RLC_SEMAPHORE_1__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_1__RESERVED__SHIFT                                                                      0x5
+#define RLC_SEMAPHORE_1__CLIENT_ID_MASK                                                                       0x0000001FL
+#define RLC_SEMAPHORE_1__RESERVED_MASK                                                                        0xFFFFFFE0L
+//RLC_SEMAPHORE_2
+#define RLC_SEMAPHORE_2__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_2__RESERVED__SHIFT                                                                      0x5
+#define RLC_SEMAPHORE_2__CLIENT_ID_MASK                                                                       0x0000001FL
+#define RLC_SEMAPHORE_2__RESERVED_MASK                                                                        0xFFFFFFE0L
+//RLC_SEMAPHORE_3
+#define RLC_SEMAPHORE_3__CLIENT_ID__SHIFT                                                                     0x0
+#define RLC_SEMAPHORE_3__RESERVED__SHIFT                                                                      0x5
+#define RLC_SEMAPHORE_3__CLIENT_ID_MASK                                                                       0x0000001FL
+#define RLC_SEMAPHORE_3__RESERVED_MASK                                                                        0xFFFFFFE0L
+//RLC_PACE_INT_STAT
+#define RLC_PACE_INT_STAT__STATUS__SHIFT                                                                      0x0
+#define RLC_PACE_INT_STAT__STATUS_MASK                                                                        0xFFFFFFFFL
+//RLC_UTCL1_STATUS
+#define RLC_UTCL1_STATUS__FAULT_DETECTED__SHIFT                                                               0x0
+#define RLC_UTCL1_STATUS__RETRY_DETECTED__SHIFT                                                               0x1
+#define RLC_UTCL1_STATUS__PRT_DETECTED__SHIFT                                                                 0x2
+#define RLC_UTCL1_STATUS__RESERVED__SHIFT                                                                     0x3
+#define RLC_UTCL1_STATUS__FAULT_UTCL1ID__SHIFT                                                                0x8
+#define RLC_UTCL1_STATUS__RESERVED_1__SHIFT                                                                   0xe
+#define RLC_UTCL1_STATUS__RETRY_UTCL1ID__SHIFT                                                                0x10
+#define RLC_UTCL1_STATUS__RESERVED_2__SHIFT                                                                   0x16
+#define RLC_UTCL1_STATUS__PRT_UTCL1ID__SHIFT                                                                  0x18
+#define RLC_UTCL1_STATUS__RESERVED_3__SHIFT                                                                   0x1e
+#define RLC_UTCL1_STATUS__FAULT_DETECTED_MASK                                                                 0x00000001L
+#define RLC_UTCL1_STATUS__RETRY_DETECTED_MASK                                                                 0x00000002L
+#define RLC_UTCL1_STATUS__PRT_DETECTED_MASK                                                                   0x00000004L
+#define RLC_UTCL1_STATUS__RESERVED_MASK                                                                       0x000000F8L
+#define RLC_UTCL1_STATUS__FAULT_UTCL1ID_MASK                                                                  0x00003F00L
+#define RLC_UTCL1_STATUS__RESERVED_1_MASK                                                                     0x0000C000L
+#define RLC_UTCL1_STATUS__RETRY_UTCL1ID_MASK                                                                  0x003F0000L
+#define RLC_UTCL1_STATUS__RESERVED_2_MASK                                                                     0x00C00000L
+#define RLC_UTCL1_STATUS__PRT_UTCL1ID_MASK                                                                    0x3F000000L
+#define RLC_UTCL1_STATUS__RESERVED_3_MASK                                                                     0xC0000000L
+//RLC_R2I_CNTL_0
+#define RLC_R2I_CNTL_0__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_0__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_R2I_CNTL_1
+#define RLC_R2I_CNTL_1__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_1__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_R2I_CNTL_2
+#define RLC_R2I_CNTL_2__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_2__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_R2I_CNTL_3
+#define RLC_R2I_CNTL_3__Data__SHIFT                                                                           0x0
+#define RLC_R2I_CNTL_3__Data_MASK                                                                             0xFFFFFFFFL
+//RLC_GPM_INT_STAT_TH0
+#define RLC_GPM_INT_STAT_TH0__STATUS__SHIFT                                                                   0x0
+#define RLC_GPM_INT_STAT_TH0__STATUS_MASK                                                                     0xFFFFFFFFL
+//RLC_GPM_GENERAL_13
+#define RLC_GPM_GENERAL_13__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_13__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_14
+#define RLC_GPM_GENERAL_14__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_14__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GPM_GENERAL_15
+#define RLC_GPM_GENERAL_15__DATA__SHIFT                                                                       0x0
+#define RLC_GPM_GENERAL_15__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_CAPTURE_GPU_CLOCK_COUNT_1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__CAPTURE__SHIFT                                                         0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__RESERVED__SHIFT                                                        0x1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__CAPTURE_MASK                                                           0x00000001L
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_1__RESERVED_MASK                                                          0xFFFFFFFEL
+//RLC_GPU_CLOCK_COUNT_LSB_2
+#define RLC_GPU_CLOCK_COUNT_LSB_2__GPU_CLOCKS_LSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_LSB_2__GPU_CLOCKS_LSB_MASK                                                        0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_MSB_2
+#define RLC_GPU_CLOCK_COUNT_MSB_2__GPU_CLOCKS_MSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_MSB_2__GPU_CLOCKS_MSB_MASK                                                        0xFFFFFFFFL
+//RLC_PACE_INT_DISABLE
+#define RLC_PACE_INT_DISABLE__DISABLE_INT__SHIFT                                                              0x0
+#define RLC_PACE_INT_DISABLE__DISABLE_INT_MASK                                                                0xFFFFFFFFL
+//RLC_CAPTURE_GPU_CLOCK_COUNT_2
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__CAPTURE__SHIFT                                                         0x0
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__RESERVED__SHIFT                                                        0x1
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__CAPTURE_MASK                                                           0x00000001L
+#define RLC_CAPTURE_GPU_CLOCK_COUNT_2__RESERVED_MASK                                                          0xFFFFFFFEL
+//RLC_RLCV_DOORBELL_RANGE
+#define RLC_RLCV_DOORBELL_RANGE__LOWER_ADDR_RESERVED__SHIFT                                                   0x0
+#define RLC_RLCV_DOORBELL_RANGE__LOWER_ADDR__SHIFT                                                            0x2
+#define RLC_RLCV_DOORBELL_RANGE__UPPER_ADDR_RESERVED__SHIFT                                                   0x10
+#define RLC_RLCV_DOORBELL_RANGE__UPPER_ADDR__SHIFT                                                            0x12
+#define RLC_RLCV_DOORBELL_RANGE__LOWER_ADDR_RESERVED_MASK                                                     0x00000003L
+#define RLC_RLCV_DOORBELL_RANGE__LOWER_ADDR_MASK                                                              0x00000FFCL
+#define RLC_RLCV_DOORBELL_RANGE__UPPER_ADDR_RESERVED_MASK                                                     0x00030000L
+#define RLC_RLCV_DOORBELL_RANGE__UPPER_ADDR_MASK                                                              0x0FFC0000L
+//RLC_RLCV_DOORBELL_CNTL
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_0_MODE__SHIFT                                                        0x0
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_1_MODE__SHIFT                                                        0x2
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_2_MODE__SHIFT                                                        0x4
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_3_MODE__SHIFT                                                        0x6
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_ID__SHIFT                                                            0x10
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_ID_EN__SHIFT                                                         0x15
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_0_MODE_MASK                                                          0x00000003L
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_1_MODE_MASK                                                          0x0000000CL
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_2_MODE_MASK                                                          0x00000030L
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_3_MODE_MASK                                                          0x000000C0L
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_ID_MASK                                                              0x001F0000L
+#define RLC_RLCV_DOORBELL_CNTL__DOORBELL_ID_EN_MASK                                                           0x00200000L
+//RLC_RLCV_DOORBELL_STAT
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_0_VALID__SHIFT                                                       0x0
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_1_VALID__SHIFT                                                       0x1
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_2_VALID__SHIFT                                                       0x2
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_3_VALID__SHIFT                                                       0x3
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_0_VALID_MASK                                                         0x00000001L
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_1_VALID_MASK                                                         0x00000002L
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_2_VALID_MASK                                                         0x00000004L
+#define RLC_RLCV_DOORBELL_STAT__DOORBELL_3_VALID_MASK                                                         0x00000008L
+//RLC_RLCV_DOORBELL_0_DATA_LO
+#define RLC_RLCV_DOORBELL_0_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_0_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_0_DATA_HI
+#define RLC_RLCV_DOORBELL_0_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_0_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_1_DATA_LO
+#define RLC_RLCV_DOORBELL_1_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_1_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_1_DATA_HI
+#define RLC_RLCV_DOORBELL_1_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_1_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_2_DATA_LO
+#define RLC_RLCV_DOORBELL_2_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_2_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_2_DATA_HI
+#define RLC_RLCV_DOORBELL_2_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_2_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_3_DATA_LO
+#define RLC_RLCV_DOORBELL_3_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_3_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCV_DOORBELL_3_DATA_HI
+#define RLC_RLCV_DOORBELL_3_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCV_DOORBELL_3_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_LSB_1
+#define RLC_GPU_CLOCK_COUNT_LSB_1__GPU_CLOCKS_LSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_LSB_1__GPU_CLOCKS_LSB_MASK                                                        0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_MSB_1
+#define RLC_GPU_CLOCK_COUNT_MSB_1__GPU_CLOCKS_MSB__SHIFT                                                      0x0
+#define RLC_GPU_CLOCK_COUNT_MSB_1__GPU_CLOCKS_MSB_MASK                                                        0xFFFFFFFFL
+//RLC_RLCV_SPARE_INT
+#define RLC_RLCV_SPARE_INT__INTERRUPT__SHIFT                                                                  0x0
+#define RLC_RLCV_SPARE_INT__RESERVED__SHIFT                                                                   0x1
+#define RLC_RLCV_SPARE_INT__INTERRUPT_MASK                                                                    0x00000001L
+#define RLC_RLCV_SPARE_INT__RESERVED_MASK                                                                     0xFFFFFFFEL
+//RLC_FIREWALL_VIOLATION
+#define RLC_FIREWALL_VIOLATION__ADDR__SHIFT                                                                   0x0
+#define RLC_FIREWALL_VIOLATION__ADDR_MASK                                                                     0xFFFFFFFFL
+//RLC_PACE_TIMER_INT_0
+#define RLC_PACE_TIMER_INT_0__TIMER__SHIFT                                                                    0x0
+#define RLC_PACE_TIMER_INT_0__TIMER_MASK                                                                      0xFFFFFFFFL
+//RLC_PACE_TIMER_INT_1
+#define RLC_PACE_TIMER_INT_1__TIMER__SHIFT                                                                    0x0
+#define RLC_PACE_TIMER_INT_1__TIMER_MASK                                                                      0xFFFFFFFFL
+//RLC_PACE_TIMER_CTRL
+#define RLC_PACE_TIMER_CTRL__TIMER_0_EN__SHIFT                                                                0x0
+#define RLC_PACE_TIMER_CTRL__TIMER_1_EN__SHIFT                                                                0x1
+#define RLC_PACE_TIMER_CTRL__TIMER_0_AUTO_REARM__SHIFT                                                        0x2
+#define RLC_PACE_TIMER_CTRL__TIMER_1_AUTO_REARM__SHIFT                                                        0x3
+#define RLC_PACE_TIMER_CTRL__TIMER_0_INT_CLEAR__SHIFT                                                         0x4
+#define RLC_PACE_TIMER_CTRL__TIMER_1_INT_CLEAR__SHIFT                                                         0x5
+#define RLC_PACE_TIMER_CTRL__RESERVED__SHIFT                                                                  0x6
+#define RLC_PACE_TIMER_CTRL__TIMER_0_EN_MASK                                                                  0x00000001L
+#define RLC_PACE_TIMER_CTRL__TIMER_1_EN_MASK                                                                  0x00000002L
+#define RLC_PACE_TIMER_CTRL__TIMER_0_AUTO_REARM_MASK                                                          0x00000004L
+#define RLC_PACE_TIMER_CTRL__TIMER_1_AUTO_REARM_MASK                                                          0x00000008L
+#define RLC_PACE_TIMER_CTRL__TIMER_0_INT_CLEAR_MASK                                                           0x00000010L
+#define RLC_PACE_TIMER_CTRL__TIMER_1_INT_CLEAR_MASK                                                           0x00000020L
+#define RLC_PACE_TIMER_CTRL__RESERVED_MASK                                                                    0xFFFFFFC0L
+//RLC_SMU_CLK_REQ
+#define RLC_SMU_CLK_REQ__VALID__SHIFT                                                                         0x0
+#define RLC_SMU_CLK_REQ__VALID_MASK                                                                           0x00000001L
+//RLC_CP_STAT_INVAL_STAT
+#define RLC_CP_STAT_INVAL_STAT__CPG_STAT_INVAL_PEND__SHIFT                                                    0x0
+#define RLC_CP_STAT_INVAL_STAT__CPC_STAT_INVAL_PEND__SHIFT                                                    0x1
+#define RLC_CP_STAT_INVAL_STAT__CPF_STAT_INVAL_PEND__SHIFT                                                    0x2
+#define RLC_CP_STAT_INVAL_STAT__CPG_STAT_INVAL_PEND_CHANGED__SHIFT                                            0x3
+#define RLC_CP_STAT_INVAL_STAT__CPC_STAT_INVAL_PEND_CHANGED__SHIFT                                            0x4
+#define RLC_CP_STAT_INVAL_STAT__CPF_STAT_INVAL_PEND_CHANGED__SHIFT                                            0x5
+#define RLC_CP_STAT_INVAL_STAT__CPG_STAT_INVAL_PEND_MASK                                                      0x00000001L
+#define RLC_CP_STAT_INVAL_STAT__CPC_STAT_INVAL_PEND_MASK                                                      0x00000002L
+#define RLC_CP_STAT_INVAL_STAT__CPF_STAT_INVAL_PEND_MASK                                                      0x00000004L
+#define RLC_CP_STAT_INVAL_STAT__CPG_STAT_INVAL_PEND_CHANGED_MASK                                              0x00000008L
+#define RLC_CP_STAT_INVAL_STAT__CPC_STAT_INVAL_PEND_CHANGED_MASK                                              0x00000010L
+#define RLC_CP_STAT_INVAL_STAT__CPF_STAT_INVAL_PEND_CHANGED_MASK                                              0x00000020L
+//RLC_CP_STAT_INVAL_CTRL
+#define RLC_CP_STAT_INVAL_CTRL__CPG_STAT_INVAL_PEND_EN__SHIFT                                                 0x0
+#define RLC_CP_STAT_INVAL_CTRL__CPC_STAT_INVAL_PEND_EN__SHIFT                                                 0x1
+#define RLC_CP_STAT_INVAL_CTRL__CPF_STAT_INVAL_PEND_EN__SHIFT                                                 0x2
+#define RLC_CP_STAT_INVAL_CTRL__CPG_STAT_INVAL_PEND_EN_MASK                                                   0x00000001L
+#define RLC_CP_STAT_INVAL_CTRL__CPC_STAT_INVAL_PEND_EN_MASK                                                   0x00000002L
+#define RLC_CP_STAT_INVAL_CTRL__CPF_STAT_INVAL_PEND_EN_MASK                                                   0x00000004L
+//RLC_SPARE
+#define RLC_SPARE__SPARE__SHIFT                                                                               0x0
+#define RLC_SPARE__SPARE_MASK                                                                                 0xFFFFFFFFL
+//RLC_SPP_CTRL
+#define RLC_SPP_CTRL__ENABLE__SHIFT                                                                           0x0
+#define RLC_SPP_CTRL__ENABLE_PPROF__SHIFT                                                                     0x1
+#define RLC_SPP_CTRL__ENABLE_PWR_OPT__SHIFT                                                                   0x2
+#define RLC_SPP_CTRL__PAUSE__SHIFT                                                                            0x3
+#define RLC_SPP_CTRL__ENABLE_MASK                                                                             0x00000001L
+#define RLC_SPP_CTRL__ENABLE_PPROF_MASK                                                                       0x00000002L
+#define RLC_SPP_CTRL__ENABLE_PWR_OPT_MASK                                                                     0x00000004L
+#define RLC_SPP_CTRL__PAUSE_MASK                                                                              0x00000008L
+//RLC_SPP_SHADER_PROFILE_EN
+#define RLC_SPP_SHADER_PROFILE_EN__PS_ENABLE__SHIFT                                                           0x0
+#define RLC_SPP_SHADER_PROFILE_EN__RESERVED_1__SHIFT                                                          0x1
+#define RLC_SPP_SHADER_PROFILE_EN__GS_ENABLE__SHIFT                                                           0x2
+#define RLC_SPP_SHADER_PROFILE_EN__HS_ENABLE__SHIFT                                                           0x3
+#define RLC_SPP_SHADER_PROFILE_EN__CSG_ENABLE__SHIFT                                                          0x4
+#define RLC_SPP_SHADER_PROFILE_EN__CS_ENABLE__SHIFT                                                           0x5
+#define RLC_SPP_SHADER_PROFILE_EN__PS_STOP_CONDITION__SHIFT                                                   0x6
+#define RLC_SPP_SHADER_PROFILE_EN__RESERVED_7__SHIFT                                                          0x7
+#define RLC_SPP_SHADER_PROFILE_EN__GS_STOP_CONDITION__SHIFT                                                   0x8
+#define RLC_SPP_SHADER_PROFILE_EN__HS_STOP_CONDITION__SHIFT                                                   0x9
+#define RLC_SPP_SHADER_PROFILE_EN__CSG_STOP_CONDITION__SHIFT                                                  0xa
+#define RLC_SPP_SHADER_PROFILE_EN__CS_STOP_CONDITION__SHIFT                                                   0xb
+#define RLC_SPP_SHADER_PROFILE_EN__PS_START_CONDITION__SHIFT                                                  0xc
+#define RLC_SPP_SHADER_PROFILE_EN__CS_START_CONDITION__SHIFT                                                  0xd
+#define RLC_SPP_SHADER_PROFILE_EN__FORCE_MISS__SHIFT                                                          0xe
+#define RLC_SPP_SHADER_PROFILE_EN__FORCE_UNLOCKED__SHIFT                                                      0xf
+#define RLC_SPP_SHADER_PROFILE_EN__ENABLE_PROF_INFO_LOCK__SHIFT                                               0x10
+#define RLC_SPP_SHADER_PROFILE_EN__PS_ENABLE_MASK                                                             0x00000001L
+#define RLC_SPP_SHADER_PROFILE_EN__RESERVED_1_MASK                                                            0x00000002L
+#define RLC_SPP_SHADER_PROFILE_EN__GS_ENABLE_MASK                                                             0x00000004L
+#define RLC_SPP_SHADER_PROFILE_EN__HS_ENABLE_MASK                                                             0x00000008L
+#define RLC_SPP_SHADER_PROFILE_EN__CSG_ENABLE_MASK                                                            0x00000010L
+#define RLC_SPP_SHADER_PROFILE_EN__CS_ENABLE_MASK                                                             0x00000020L
+#define RLC_SPP_SHADER_PROFILE_EN__PS_STOP_CONDITION_MASK                                                     0x00000040L
+#define RLC_SPP_SHADER_PROFILE_EN__RESERVED_7_MASK                                                            0x00000080L
+#define RLC_SPP_SHADER_PROFILE_EN__GS_STOP_CONDITION_MASK                                                     0x00000100L
+#define RLC_SPP_SHADER_PROFILE_EN__HS_STOP_CONDITION_MASK                                                     0x00000200L
+#define RLC_SPP_SHADER_PROFILE_EN__CSG_STOP_CONDITION_MASK                                                    0x00000400L
+#define RLC_SPP_SHADER_PROFILE_EN__CS_STOP_CONDITION_MASK                                                     0x00000800L
+#define RLC_SPP_SHADER_PROFILE_EN__PS_START_CONDITION_MASK                                                    0x00001000L
+#define RLC_SPP_SHADER_PROFILE_EN__CS_START_CONDITION_MASK                                                    0x00002000L
+#define RLC_SPP_SHADER_PROFILE_EN__FORCE_MISS_MASK                                                            0x00004000L
+#define RLC_SPP_SHADER_PROFILE_EN__FORCE_UNLOCKED_MASK                                                        0x00008000L
+#define RLC_SPP_SHADER_PROFILE_EN__ENABLE_PROF_INFO_LOCK_MASK                                                 0x00010000L
+//RLC_SPP_SSF_CAPTURE_EN
+#define RLC_SPP_SSF_CAPTURE_EN__PS_ENABLE__SHIFT                                                              0x0
+#define RLC_SPP_SSF_CAPTURE_EN__RESERVED_1__SHIFT                                                             0x1
+#define RLC_SPP_SSF_CAPTURE_EN__GS_ENABLE__SHIFT                                                              0x2
+#define RLC_SPP_SSF_CAPTURE_EN__HS_ENABLE__SHIFT                                                              0x3
+#define RLC_SPP_SSF_CAPTURE_EN__CSG_ENABLE__SHIFT                                                             0x4
+#define RLC_SPP_SSF_CAPTURE_EN__CS_ENABLE__SHIFT                                                              0x5
+#define RLC_SPP_SSF_CAPTURE_EN__PS_ENABLE_MASK                                                                0x00000001L
+#define RLC_SPP_SSF_CAPTURE_EN__RESERVED_1_MASK                                                               0x00000002L
+#define RLC_SPP_SSF_CAPTURE_EN__GS_ENABLE_MASK                                                                0x00000004L
+#define RLC_SPP_SSF_CAPTURE_EN__HS_ENABLE_MASK                                                                0x00000008L
+#define RLC_SPP_SSF_CAPTURE_EN__CSG_ENABLE_MASK                                                               0x00000010L
+#define RLC_SPP_SSF_CAPTURE_EN__CS_ENABLE_MASK                                                                0x00000020L
+//RLC_SPP_SSF_THRESHOLD_0
+#define RLC_SPP_SSF_THRESHOLD_0__PS_THRESHOLD__SHIFT                                                          0x0
+#define RLC_SPP_SSF_THRESHOLD_0__RESERVED__SHIFT                                                              0x10
+#define RLC_SPP_SSF_THRESHOLD_0__PS_THRESHOLD_MASK                                                            0x0000FFFFL
+#define RLC_SPP_SSF_THRESHOLD_0__RESERVED_MASK                                                                0xFFFF0000L
+//RLC_SPP_SSF_THRESHOLD_1
+#define RLC_SPP_SSF_THRESHOLD_1__GS_THRESHOLD__SHIFT                                                          0x0
+#define RLC_SPP_SSF_THRESHOLD_1__HS_THRESHOLD__SHIFT                                                          0x10
+#define RLC_SPP_SSF_THRESHOLD_1__GS_THRESHOLD_MASK                                                            0x0000FFFFL
+#define RLC_SPP_SSF_THRESHOLD_1__HS_THRESHOLD_MASK                                                            0xFFFF0000L
+//RLC_SPP_SSF_THRESHOLD_2
+#define RLC_SPP_SSF_THRESHOLD_2__CSG_THRESHOLD__SHIFT                                                         0x0
+#define RLC_SPP_SSF_THRESHOLD_2__CS_THRESHOLD__SHIFT                                                          0x10
+#define RLC_SPP_SSF_THRESHOLD_2__CSG_THRESHOLD_MASK                                                           0x0000FFFFL
+#define RLC_SPP_SSF_THRESHOLD_2__CS_THRESHOLD_MASK                                                            0xFFFF0000L
+//RLC_SPP_INFLIGHT_RD_ADDR
+#define RLC_SPP_INFLIGHT_RD_ADDR__ADDR__SHIFT                                                                 0x0
+#define RLC_SPP_INFLIGHT_RD_ADDR__ADDR_MASK                                                                   0x0000001FL
+//RLC_SPP_INFLIGHT_RD_DATA
+#define RLC_SPP_INFLIGHT_RD_DATA__DATA__SHIFT                                                                 0x0
+#define RLC_SPP_INFLIGHT_RD_DATA__DATA_MASK                                                                   0xFFFFFFFFL
+//RLC_SPP_PROF_INFO_1
+#define RLC_SPP_PROF_INFO_1__SH_ID__SHIFT                                                                     0x0
+#define RLC_SPP_PROF_INFO_1__SH_ID_MASK                                                                       0xFFFFFFFFL
+//RLC_SPP_PROF_INFO_2
+#define RLC_SPP_PROF_INFO_2__SH_TYPE__SHIFT                                                                   0x0
+#define RLC_SPP_PROF_INFO_2__CAM_HIT__SHIFT                                                                   0x4
+#define RLC_SPP_PROF_INFO_2__CAM_LOCK__SHIFT                                                                  0x5
+#define RLC_SPP_PROF_INFO_2__CAM_CONFLICT__SHIFT                                                              0x6
+#define RLC_SPP_PROF_INFO_2__SH_TYPE_MASK                                                                     0x0000000FL
+#define RLC_SPP_PROF_INFO_2__CAM_HIT_MASK                                                                     0x00000010L
+#define RLC_SPP_PROF_INFO_2__CAM_LOCK_MASK                                                                    0x00000020L
+#define RLC_SPP_PROF_INFO_2__CAM_CONFLICT_MASK                                                                0x00000040L
+//RLC_SPP_GLOBAL_SH_ID
+#define RLC_SPP_GLOBAL_SH_ID__SH_ID__SHIFT                                                                    0x0
+#define RLC_SPP_GLOBAL_SH_ID__SH_ID_MASK                                                                      0xFFFFFFFFL
+//RLC_SPP_GLOBAL_SH_ID_VALID
+#define RLC_SPP_GLOBAL_SH_ID_VALID__VALID__SHIFT                                                              0x0
+#define RLC_SPP_GLOBAL_SH_ID_VALID__VALID_MASK                                                                0x00000001L
+//RLC_SPP_STATUS
+#define RLC_SPP_STATUS__RESERVED_0__SHIFT                                                                     0x0
+#define RLC_SPP_STATUS__SSF_BUSY__SHIFT                                                                       0x1
+#define RLC_SPP_STATUS__EVENT_ARB_BUSY__SHIFT                                                                 0x2
+#define RLC_SPP_STATUS__SPP_BUSY__SHIFT                                                                       0x1f
+#define RLC_SPP_STATUS__RESERVED_0_MASK                                                                       0x00000001L
+#define RLC_SPP_STATUS__SSF_BUSY_MASK                                                                         0x00000002L
+#define RLC_SPP_STATUS__EVENT_ARB_BUSY_MASK                                                                   0x00000004L
+#define RLC_SPP_STATUS__SPP_BUSY_MASK                                                                         0x80000000L
+//RLC_SPP_PVT_STAT_0
+#define RLC_SPP_PVT_STAT_0__LEVEL_0_COUNTER__SHIFT                                                            0x0
+#define RLC_SPP_PVT_STAT_0__LEVEL_1_COUNTER__SHIFT                                                            0x8
+#define RLC_SPP_PVT_STAT_0__LEVEL_2_COUNTER__SHIFT                                                            0x10
+#define RLC_SPP_PVT_STAT_0__LEVEL_3_COUNTER__SHIFT                                                            0x18
+#define RLC_SPP_PVT_STAT_0__LEVEL_0_COUNTER_MASK                                                              0x000000FFL
+#define RLC_SPP_PVT_STAT_0__LEVEL_1_COUNTER_MASK                                                              0x0000FF00L
+#define RLC_SPP_PVT_STAT_0__LEVEL_2_COUNTER_MASK                                                              0x00FF0000L
+#define RLC_SPP_PVT_STAT_0__LEVEL_3_COUNTER_MASK                                                              0xFF000000L
+//RLC_SPP_PVT_STAT_1
+#define RLC_SPP_PVT_STAT_1__LEVEL_4_COUNTER__SHIFT                                                            0x0
+#define RLC_SPP_PVT_STAT_1__LEVEL_5_COUNTER__SHIFT                                                            0x8
+#define RLC_SPP_PVT_STAT_1__LEVEL_6_COUNTER__SHIFT                                                            0x10
+#define RLC_SPP_PVT_STAT_1__LEVEL_7_COUNTER__SHIFT                                                            0x18
+#define RLC_SPP_PVT_STAT_1__LEVEL_4_COUNTER_MASK                                                              0x000000FFL
+#define RLC_SPP_PVT_STAT_1__LEVEL_5_COUNTER_MASK                                                              0x0000FF00L
+#define RLC_SPP_PVT_STAT_1__LEVEL_6_COUNTER_MASK                                                              0x00FF0000L
+#define RLC_SPP_PVT_STAT_1__LEVEL_7_COUNTER_MASK                                                              0xFF000000L
+//RLC_SPP_PVT_STAT_2
+#define RLC_SPP_PVT_STAT_2__LEVEL_8_COUNTER__SHIFT                                                            0x0
+#define RLC_SPP_PVT_STAT_2__LEVEL_9_COUNTER__SHIFT                                                            0x8
+#define RLC_SPP_PVT_STAT_2__LEVEL_10_COUNTER__SHIFT                                                           0x10
+#define RLC_SPP_PVT_STAT_2__LEVEL_11_COUNTER__SHIFT                                                           0x18
+#define RLC_SPP_PVT_STAT_2__LEVEL_8_COUNTER_MASK                                                              0x000000FFL
+#define RLC_SPP_PVT_STAT_2__LEVEL_9_COUNTER_MASK                                                              0x0000FF00L
+#define RLC_SPP_PVT_STAT_2__LEVEL_10_COUNTER_MASK                                                             0x00FF0000L
+#define RLC_SPP_PVT_STAT_2__LEVEL_11_COUNTER_MASK                                                             0xFF000000L
+//RLC_SPP_PVT_STAT_3
+#define RLC_SPP_PVT_STAT_3__LEVEL_12_COUNTER__SHIFT                                                           0x0
+#define RLC_SPP_PVT_STAT_3__LEVEL_13_COUNTER__SHIFT                                                           0x8
+#define RLC_SPP_PVT_STAT_3__LEVEL_14_COUNTER__SHIFT                                                           0x10
+#define RLC_SPP_PVT_STAT_3__LEVEL_15_COUNTER__SHIFT                                                           0x18
+#define RLC_SPP_PVT_STAT_3__LEVEL_12_COUNTER_MASK                                                             0x000000FFL
+#define RLC_SPP_PVT_STAT_3__LEVEL_13_COUNTER_MASK                                                             0x0000FF00L
+#define RLC_SPP_PVT_STAT_3__LEVEL_14_COUNTER_MASK                                                             0x00FF0000L
+#define RLC_SPP_PVT_STAT_3__LEVEL_15_COUNTER_MASK                                                             0xFF000000L
+//RLC_SPP_PVT_LEVEL_MAX
+#define RLC_SPP_PVT_LEVEL_MAX__LEVEL__SHIFT                                                                   0x0
+#define RLC_SPP_PVT_LEVEL_MAX__LEVEL_MASK                                                                     0x0000000FL
+//RLC_SPP_STALL_STATE_UPDATE
+#define RLC_SPP_STALL_STATE_UPDATE__STALL__SHIFT                                                              0x0
+#define RLC_SPP_STALL_STATE_UPDATE__ENABLE__SHIFT                                                             0x1
+#define RLC_SPP_STALL_STATE_UPDATE__STALL_MASK                                                                0x00000001L
+#define RLC_SPP_STALL_STATE_UPDATE__ENABLE_MASK                                                               0x00000002L
+//RLC_SPP_PBB_INFO
+#define RLC_SPP_PBB_INFO__PIPE0_OVERRIDE__SHIFT                                                               0x0
+#define RLC_SPP_PBB_INFO__PIPE0_OVERRIDE_VALID__SHIFT                                                         0x1
+#define RLC_SPP_PBB_INFO__PIPE1_OVERRIDE__SHIFT                                                               0x2
+#define RLC_SPP_PBB_INFO__PIPE1_OVERRIDE_VALID__SHIFT                                                         0x3
+#define RLC_SPP_PBB_INFO__PIPE0_OVERRIDE_MASK                                                                 0x00000001L
+#define RLC_SPP_PBB_INFO__PIPE0_OVERRIDE_VALID_MASK                                                           0x00000002L
+#define RLC_SPP_PBB_INFO__PIPE1_OVERRIDE_MASK                                                                 0x00000004L
+#define RLC_SPP_PBB_INFO__PIPE1_OVERRIDE_VALID_MASK                                                           0x00000008L
+//RLC_SPP_RESET
+#define RLC_SPP_RESET__SSF_RESET__SHIFT                                                                       0x0
+#define RLC_SPP_RESET__EVENT_ARB_RESET__SHIFT                                                                 0x1
+#define RLC_SPP_RESET__CAM_RESET__SHIFT                                                                       0x2
+#define RLC_SPP_RESET__PVT_RESET__SHIFT                                                                       0x3
+#define RLC_SPP_RESET__SSF_RESET_MASK                                                                         0x00000001L
+#define RLC_SPP_RESET__EVENT_ARB_RESET_MASK                                                                   0x00000002L
+#define RLC_SPP_RESET__CAM_RESET_MASK                                                                         0x00000004L
+#define RLC_SPP_RESET__PVT_RESET_MASK                                                                         0x00000008L
+//RLC_RLCP_DOORBELL_RANGE
+#define RLC_RLCP_DOORBELL_RANGE__LOWER_ADDR_RESERVED__SHIFT                                                   0x0
+#define RLC_RLCP_DOORBELL_RANGE__LOWER_ADDR__SHIFT                                                            0x2
+#define RLC_RLCP_DOORBELL_RANGE__UPPER_ADDR_RESERVED__SHIFT                                                   0x10
+#define RLC_RLCP_DOORBELL_RANGE__UPPER_ADDR__SHIFT                                                            0x12
+#define RLC_RLCP_DOORBELL_RANGE__LOWER_ADDR_RESERVED_MASK                                                     0x00000003L
+#define RLC_RLCP_DOORBELL_RANGE__LOWER_ADDR_MASK                                                              0x00000FFCL
+#define RLC_RLCP_DOORBELL_RANGE__UPPER_ADDR_RESERVED_MASK                                                     0x00030000L
+#define RLC_RLCP_DOORBELL_RANGE__UPPER_ADDR_MASK                                                              0x0FFC0000L
+//RLC_RLCP_DOORBELL_CNTL
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_0_MODE__SHIFT                                                        0x0
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_1_MODE__SHIFT                                                        0x2
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_2_MODE__SHIFT                                                        0x4
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_3_MODE__SHIFT                                                        0x6
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_ID__SHIFT                                                            0x10
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_ID_EN__SHIFT                                                         0x15
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_0_MODE_MASK                                                          0x00000003L
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_1_MODE_MASK                                                          0x0000000CL
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_2_MODE_MASK                                                          0x00000030L
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_3_MODE_MASK                                                          0x000000C0L
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_ID_MASK                                                              0x001F0000L
+#define RLC_RLCP_DOORBELL_CNTL__DOORBELL_ID_EN_MASK                                                           0x00200000L
+//RLC_RLCP_DOORBELL_STAT
+#define RLC_RLCP_DOORBELL_STAT__DOORBELL_0_VALID__SHIFT                                                       0x0
+#define RLC_RLCP_DOORBELL_STAT__DOORBELL_1_VALID__SHIFT                                                       0x1
+#define RLC_RLCP_DOORBELL_STAT__DOORBELL_2_VALID__SHIFT                                                       0x2
+#define RLC_RLCP_DOORBELL_STAT__DOORBELL_3_VALID__SHIFT                                                       0x3
+#define RLC_RLCP_DOORBELL_STAT__DOORBELL_0_VALID_MASK                                                         0x00000001L
+#define RLC_RLCP_DOORBELL_STAT__DOORBELL_1_VALID_MASK                                                         0x00000002L
+#define RLC_RLCP_DOORBELL_STAT__DOORBELL_2_VALID_MASK                                                         0x00000004L
+#define RLC_RLCP_DOORBELL_STAT__DOORBELL_3_VALID_MASK                                                         0x00000008L
+//RLC_RLCP_DOORBELL_0_DATA_LO
+#define RLC_RLCP_DOORBELL_0_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCP_DOORBELL_0_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCP_DOORBELL_0_DATA_HI
+#define RLC_RLCP_DOORBELL_0_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCP_DOORBELL_0_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCP_DOORBELL_1_DATA_LO
+#define RLC_RLCP_DOORBELL_1_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCP_DOORBELL_1_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCP_DOORBELL_1_DATA_HI
+#define RLC_RLCP_DOORBELL_1_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCP_DOORBELL_1_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCP_DOORBELL_2_DATA_LO
+#define RLC_RLCP_DOORBELL_2_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCP_DOORBELL_2_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCP_DOORBELL_2_DATA_HI
+#define RLC_RLCP_DOORBELL_2_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCP_DOORBELL_2_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCP_DOORBELL_3_DATA_LO
+#define RLC_RLCP_DOORBELL_3_DATA_LO__DATA__SHIFT                                                              0x0
+#define RLC_RLCP_DOORBELL_3_DATA_LO__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCP_DOORBELL_3_DATA_HI
+#define RLC_RLCP_DOORBELL_3_DATA_HI__DATA__SHIFT                                                              0x0
+#define RLC_RLCP_DOORBELL_3_DATA_HI__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_CAC_MASK_CNTL
+#define RLC_CAC_MASK_CNTL__RLC_CAC_MASK__SHIFT                                                                0x0
+#define RLC_CAC_MASK_CNTL__RLC_CAC_MASK_MASK                                                                  0xFFFFFFFFL
+//RLC_POWER_RESIDENCY_CNTR_CTRL
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                           0x0
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                          0x1
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                       0x2
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                      0x3
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                                0x4
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                        0x5
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESET_MASK                                                             0x00000001L
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                            0x00000002L
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                         0x00000004L
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                        0x00000008L
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                  0x00000010L
+#define RLC_POWER_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                          0xFFFFFFE0L
+//RLC_CLK_RESIDENCY_CNTR_CTRL
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                             0x0
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                            0x1
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                         0x2
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                        0x3
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                                  0x4
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                          0x5
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESET_MASK                                                               0x00000001L
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                              0x00000002L
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                           0x00000004L
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                          0x00000008L
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                    0x00000010L
+#define RLC_CLK_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                            0xFFFFFFE0L
+//RLC_DS_RESIDENCY_CNTR_CTRL
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                              0x0
+#define RLC_DS_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                             0x1
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                          0x2
+#define RLC_DS_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                         0x3
+#define RLC_DS_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                                   0x4
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                           0x5
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESET_MASK                                                                0x00000001L
+#define RLC_DS_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                               0x00000002L
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                            0x00000004L
+#define RLC_DS_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                           0x00000008L
+#define RLC_DS_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                     0x00000010L
+#define RLC_DS_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                             0xFFFFFFE0L
+//RLC_ULV_RESIDENCY_CNTR_CTRL
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                             0x0
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                            0x1
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                         0x2
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                        0x3
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                                  0x4
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                          0x5
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESET_MASK                                                               0x00000001L
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                              0x00000002L
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                           0x00000004L
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                          0x00000008L
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                    0x00000010L
+#define RLC_ULV_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                            0xFFFFFFE0L
+//RLC_PCC_RESIDENCY_CNTR_CTRL
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                             0x0
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                            0x1
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                         0x2
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                        0x3
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                                  0x4
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__EVENT_SEL__SHIFT                                                         0x5
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                          0x9
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESET_MASK                                                               0x00000001L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                              0x00000002L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                           0x00000004L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                          0x00000008L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                    0x00000010L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__EVENT_SEL_MASK                                                           0x000001E0L
+#define RLC_PCC_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                            0xFFFFFE00L
+//RLC_GENERAL_RESIDENCY_CNTR_CTRL
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESET__SHIFT                                                         0x0
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__ENABLE__SHIFT                                                        0x1
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESET_ACK__SHIFT                                                     0x2
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__ENABLE_ACK__SHIFT                                                    0x3
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW__SHIFT                                              0x4
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESERVED__SHIFT                                                      0x5
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESET_MASK                                                           0x00000001L
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__ENABLE_MASK                                                          0x00000002L
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESET_ACK_MASK                                                       0x00000004L
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__ENABLE_ACK_MASK                                                      0x00000008L
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__COUNTER_OVERFLOW_MASK                                                0x00000010L
+#define RLC_GENERAL_RESIDENCY_CNTR_CTRL__RESERVED_MASK                                                        0xFFFFFFE0L
+//RLC_POWER_RESIDENCY_EVENT_CNTR
+#define RLC_POWER_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                           0x0
+#define RLC_POWER_RESIDENCY_EVENT_CNTR__DATA_MASK                                                             0xFFFFFFFFL
+//RLC_CLK_RESIDENCY_EVENT_CNTR
+#define RLC_CLK_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                             0x0
+#define RLC_CLK_RESIDENCY_EVENT_CNTR__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_DS_RESIDENCY_EVENT_CNTR
+#define RLC_DS_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                              0x0
+#define RLC_DS_RESIDENCY_EVENT_CNTR__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_ULV_RESIDENCY_EVENT_CNTR
+#define RLC_ULV_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                             0x0
+#define RLC_ULV_RESIDENCY_EVENT_CNTR__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_PCC_RESIDENCY_EVENT_CNTR
+#define RLC_PCC_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                             0x0
+#define RLC_PCC_RESIDENCY_EVENT_CNTR__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_GENERAL_RESIDENCY_EVENT_CNTR
+#define RLC_GENERAL_RESIDENCY_EVENT_CNTR__DATA__SHIFT                                                         0x0
+#define RLC_GENERAL_RESIDENCY_EVENT_CNTR__DATA_MASK                                                           0xFFFFFFFFL
+//RLC_POWER_RESIDENCY_REF_CNTR
+#define RLC_POWER_RESIDENCY_REF_CNTR__DATA__SHIFT                                                             0x0
+#define RLC_POWER_RESIDENCY_REF_CNTR__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_CLK_RESIDENCY_REF_CNTR
+#define RLC_CLK_RESIDENCY_REF_CNTR__DATA__SHIFT                                                               0x0
+#define RLC_CLK_RESIDENCY_REF_CNTR__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_DS_RESIDENCY_REF_CNTR
+#define RLC_DS_RESIDENCY_REF_CNTR__DATA__SHIFT                                                                0x0
+#define RLC_DS_RESIDENCY_REF_CNTR__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_ULV_RESIDENCY_REF_CNTR
+#define RLC_ULV_RESIDENCY_REF_CNTR__DATA__SHIFT                                                               0x0
+#define RLC_ULV_RESIDENCY_REF_CNTR__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_PCC_RESIDENCY_REF_CNTR
+#define RLC_PCC_RESIDENCY_REF_CNTR__DATA__SHIFT                                                               0x0
+#define RLC_PCC_RESIDENCY_REF_CNTR__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_GENERAL_RESIDENCY_REF_CNTR
+#define RLC_GENERAL_RESIDENCY_REF_CNTR__DATA__SHIFT                                                           0x0
+#define RLC_GENERAL_RESIDENCY_REF_CNTR__DATA_MASK                                                             0xFFFFFFFFL
+//RLC_GFX_IH_CLIENT_CTRL
+#define RLC_GFX_IH_CLIENT_CTRL__SE_INTERRUPT_MASK__SHIFT                                                      0x0
+#define RLC_GFX_IH_CLIENT_CTRL__SDMA_INTERRUPT_MASK__SHIFT                                                    0x8
+#define RLC_GFX_IH_CLIENT_CTRL__UTCL2_INTERRUPT_MASK__SHIFT                                                   0xc
+#define RLC_GFX_IH_CLIENT_CTRL__PMM_INTERRUPT_MASK__SHIFT                                                     0xd
+#define RLC_GFX_IH_CLIENT_CTRL__FED_INTERRUPT_MASK__SHIFT                                                     0xe
+#define RLC_GFX_IH_CLIENT_CTRL__RESERVED_15__SHIFT                                                            0xf
+#define RLC_GFX_IH_CLIENT_CTRL__SE_INTERRUPT_ERROR_CLEAR__SHIFT                                               0x10
+#define RLC_GFX_IH_CLIENT_CTRL__SDMA_INTERRUPT_ERROR_CLEAR__SHIFT                                             0x18
+#define RLC_GFX_IH_CLIENT_CTRL__UTCL2_INTERRUPT_ERROR_CLEAR__SHIFT                                            0x1c
+#define RLC_GFX_IH_CLIENT_CTRL__PMM_INTERRUPT_ERROR_CLEAR__SHIFT                                              0x1d
+#define RLC_GFX_IH_CLIENT_CTRL__FED_INTERRUPT_ERROR_CLEAR__SHIFT                                              0x1e
+#define RLC_GFX_IH_CLIENT_CTRL__RESERVED_31__SHIFT                                                            0x1f
+#define RLC_GFX_IH_CLIENT_CTRL__SE_INTERRUPT_MASK_MASK                                                        0x000000FFL
+#define RLC_GFX_IH_CLIENT_CTRL__SDMA_INTERRUPT_MASK_MASK                                                      0x00000F00L
+#define RLC_GFX_IH_CLIENT_CTRL__UTCL2_INTERRUPT_MASK_MASK                                                     0x00001000L
+#define RLC_GFX_IH_CLIENT_CTRL__PMM_INTERRUPT_MASK_MASK                                                       0x00002000L
+#define RLC_GFX_IH_CLIENT_CTRL__FED_INTERRUPT_MASK_MASK                                                       0x00004000L
+#define RLC_GFX_IH_CLIENT_CTRL__RESERVED_15_MASK                                                              0x00008000L
+#define RLC_GFX_IH_CLIENT_CTRL__SE_INTERRUPT_ERROR_CLEAR_MASK                                                 0x00FF0000L
+#define RLC_GFX_IH_CLIENT_CTRL__SDMA_INTERRUPT_ERROR_CLEAR_MASK                                               0x0F000000L
+#define RLC_GFX_IH_CLIENT_CTRL__UTCL2_INTERRUPT_ERROR_CLEAR_MASK                                              0x10000000L
+#define RLC_GFX_IH_CLIENT_CTRL__PMM_INTERRUPT_ERROR_CLEAR_MASK                                                0x20000000L
+#define RLC_GFX_IH_CLIENT_CTRL__FED_INTERRUPT_ERROR_CLEAR_MASK                                                0x40000000L
+#define RLC_GFX_IH_CLIENT_CTRL__RESERVED_31_MASK                                                              0x80000000L
+//RLC_GFX_IH_ARBITER_STAT
+#define RLC_GFX_IH_ARBITER_STAT__CLIENT_GRANTED__SHIFT                                                        0x0
+#define RLC_GFX_IH_ARBITER_STAT__RESERVED__SHIFT                                                              0x10
+#define RLC_GFX_IH_ARBITER_STAT__LAST_CLIENT_GRANTED__SHIFT                                                   0x1c
+#define RLC_GFX_IH_ARBITER_STAT__CLIENT_GRANTED_MASK                                                          0x0000FFFFL
+#define RLC_GFX_IH_ARBITER_STAT__RESERVED_MASK                                                                0x0FFF0000L
+#define RLC_GFX_IH_ARBITER_STAT__LAST_CLIENT_GRANTED_MASK                                                     0xF0000000L
+//RLC_GFX_IH_CLIENT_SE_STAT_L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_LEVEL__SHIFT                                                  0x0
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_LOADING__SHIFT                                                0x4
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_OVERFLOW__SHIFT                                               0x5
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_PROTOCOL_ERROR__SHIFT                                                0x6
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_RESERVED__SHIFT                                                      0x7
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_LEVEL__SHIFT                                                  0x8
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_LOADING__SHIFT                                                0xc
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_OVERFLOW__SHIFT                                               0xd
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_PROTOCOL_ERROR__SHIFT                                                0xe
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_RESERVED__SHIFT                                                      0xf
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_LEVEL__SHIFT                                                  0x10
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_LOADING__SHIFT                                                0x14
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_OVERFLOW__SHIFT                                               0x15
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_PROTOCOL_ERROR__SHIFT                                                0x16
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_RESERVED__SHIFT                                                      0x17
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_LEVEL__SHIFT                                                  0x18
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_LOADING__SHIFT                                                0x1c
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_OVERFLOW__SHIFT                                               0x1d
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_PROTOCOL_ERROR__SHIFT                                                0x1e
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_RESERVED__SHIFT                                                      0x1f
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_LEVEL_MASK                                                    0x0000000FL
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_LOADING_MASK                                                  0x00000010L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_BUFFER_OVERFLOW_MASK                                                 0x00000020L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_PROTOCOL_ERROR_MASK                                                  0x00000040L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE0_RESERVED_MASK                                                        0x00000080L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_LEVEL_MASK                                                    0x00000F00L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_LOADING_MASK                                                  0x00001000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_BUFFER_OVERFLOW_MASK                                                 0x00002000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_PROTOCOL_ERROR_MASK                                                  0x00004000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE1_RESERVED_MASK                                                        0x00008000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_LEVEL_MASK                                                    0x000F0000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_LOADING_MASK                                                  0x00100000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_BUFFER_OVERFLOW_MASK                                                 0x00200000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_PROTOCOL_ERROR_MASK                                                  0x00400000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE2_RESERVED_MASK                                                        0x00800000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_LEVEL_MASK                                                    0x0F000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_LOADING_MASK                                                  0x10000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_BUFFER_OVERFLOW_MASK                                                 0x20000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_PROTOCOL_ERROR_MASK                                                  0x40000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_L__SE3_RESERVED_MASK                                                        0x80000000L
+//RLC_GFX_IH_CLIENT_SE_STAT_H
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_LEVEL__SHIFT                                                  0x0
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_LOADING__SHIFT                                                0x4
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_OVERFLOW__SHIFT                                               0x5
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_PROTOCOL_ERROR__SHIFT                                                0x6
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_RESERVED__SHIFT                                                      0x7
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_LEVEL__SHIFT                                                  0x8
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_LOADING__SHIFT                                                0xc
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_OVERFLOW__SHIFT                                               0xd
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_PROTOCOL_ERROR__SHIFT                                                0xe
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_RESERVED__SHIFT                                                      0xf
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_LEVEL__SHIFT                                                  0x10
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_LOADING__SHIFT                                                0x14
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_OVERFLOW__SHIFT                                               0x15
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_PROTOCOL_ERROR__SHIFT                                                0x16
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_RESERVED__SHIFT                                                      0x17
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_LEVEL__SHIFT                                                  0x18
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_LOADING__SHIFT                                                0x1c
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_OVERFLOW__SHIFT                                               0x1d
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_PROTOCOL_ERROR__SHIFT                                                0x1e
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_RESERVED__SHIFT                                                      0x1f
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_LEVEL_MASK                                                    0x0000000FL
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_LOADING_MASK                                                  0x00000010L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_BUFFER_OVERFLOW_MASK                                                 0x00000020L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_PROTOCOL_ERROR_MASK                                                  0x00000040L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE4_RESERVED_MASK                                                        0x00000080L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_LEVEL_MASK                                                    0x00000F00L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_LOADING_MASK                                                  0x00001000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_BUFFER_OVERFLOW_MASK                                                 0x00002000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_PROTOCOL_ERROR_MASK                                                  0x00004000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE5_RESERVED_MASK                                                        0x00008000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_LEVEL_MASK                                                    0x000F0000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_LOADING_MASK                                                  0x00100000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_BUFFER_OVERFLOW_MASK                                                 0x00200000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_PROTOCOL_ERROR_MASK                                                  0x00400000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE6_RESERVED_MASK                                                        0x00800000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_LEVEL_MASK                                                    0x0F000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_LOADING_MASK                                                  0x10000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_BUFFER_OVERFLOW_MASK                                                 0x20000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_PROTOCOL_ERROR_MASK                                                  0x40000000L
+#define RLC_GFX_IH_CLIENT_SE_STAT_H__SE7_RESERVED_MASK                                                        0x80000000L
+//RLC_GFX_IH_CLIENT_SDMA_STAT
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_LEVEL__SHIFT                                                0x0
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_LOADING__SHIFT                                              0x4
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_OVERFLOW__SHIFT                                             0x5
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_PROTOCOL_ERROR__SHIFT                                              0x6
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_RESERVED__SHIFT                                                    0x7
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_LEVEL__SHIFT                                                0x8
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_LOADING__SHIFT                                              0xc
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_OVERFLOW__SHIFT                                             0xd
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_PROTOCOL_ERROR__SHIFT                                              0xe
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_RESERVED__SHIFT                                                    0xf
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_LEVEL__SHIFT                                                0x10
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_LOADING__SHIFT                                              0x14
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_OVERFLOW__SHIFT                                             0x15
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_PROTOCOL_ERROR__SHIFT                                              0x16
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_RESERVED__SHIFT                                                    0x17
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_LEVEL__SHIFT                                                0x18
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_LOADING__SHIFT                                              0x1c
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_OVERFLOW__SHIFT                                             0x1d
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_PROTOCOL_ERROR__SHIFT                                              0x1e
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_RESERVED__SHIFT                                                    0x1f
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_LEVEL_MASK                                                  0x0000000FL
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_LOADING_MASK                                                0x00000010L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_BUFFER_OVERFLOW_MASK                                               0x00000020L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_PROTOCOL_ERROR_MASK                                                0x00000040L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA0_RESERVED_MASK                                                      0x00000080L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_LEVEL_MASK                                                  0x00000F00L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_LOADING_MASK                                                0x00001000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_BUFFER_OVERFLOW_MASK                                               0x00002000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_PROTOCOL_ERROR_MASK                                                0x00004000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA1_RESERVED_MASK                                                      0x00008000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_LEVEL_MASK                                                  0x000F0000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_LOADING_MASK                                                0x00100000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_BUFFER_OVERFLOW_MASK                                               0x00200000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_PROTOCOL_ERROR_MASK                                                0x00400000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA2_RESERVED_MASK                                                      0x00800000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_LEVEL_MASK                                                  0x0F000000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_LOADING_MASK                                                0x10000000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_BUFFER_OVERFLOW_MASK                                               0x20000000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_PROTOCOL_ERROR_MASK                                                0x40000000L
+#define RLC_GFX_IH_CLIENT_SDMA_STAT__SDMA3_RESERVED_MASK                                                      0x80000000L
+//RLC_GFX_IH_CLIENT_OTHER_STAT
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_BUFFER_LEVEL__SHIFT                                               0x0
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_BUFFER_LOADING__SHIFT                                             0x4
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_BUFFER_OVERFLOW__SHIFT                                            0x5
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_PROTOCOL_ERROR__SHIFT                                             0x6
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_RESERVED__SHIFT                                                   0x7
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_BUFFER_LEVEL__SHIFT                                                 0x8
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_BUFFER_LOADING__SHIFT                                               0xc
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_BUFFER_OVERFLOW__SHIFT                                              0xd
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_PROTOCOL_ERROR__SHIFT                                               0xe
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_RESERVED__SHIFT                                                     0xf
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_LEVEL__SHIFT                                                 0x10
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_LOADING__SHIFT                                               0x14
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_OVERFLOW__SHIFT                                              0x15
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_PROTOCOL_ERROR__SHIFT                                               0x16
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_RESERVED__SHIFT                                                     0x17
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__RESERVED_31_24__SHIFT                                                   0x18
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_BUFFER_LEVEL_MASK                                                 0x0000000FL
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_BUFFER_LOADING_MASK                                               0x00000010L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_BUFFER_OVERFLOW_MASK                                              0x00000020L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_PROTOCOL_ERROR_MASK                                               0x00000040L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__UTCL2_RESERVED_MASK                                                     0x00000080L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_BUFFER_LEVEL_MASK                                                   0x00000F00L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_BUFFER_LOADING_MASK                                                 0x00001000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_BUFFER_OVERFLOW_MASK                                                0x00002000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_PROTOCOL_ERROR_MASK                                                 0x00004000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__PMM_RESERVED_MASK                                                       0x00008000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_LEVEL_MASK                                                   0x000F0000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_LOADING_MASK                                                 0x00100000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_BUFFER_OVERFLOW_MASK                                                0x00200000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_PROTOCOL_ERROR_MASK                                                 0x00400000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__FED_RESERVED_MASK                                                       0x00800000L
+#define RLC_GFX_IH_CLIENT_OTHER_STAT__RESERVED_31_24_MASK                                                     0xFF000000L
+//RLC_SPM_GLOBAL_DELAY_IND_ADDR
+#define RLC_SPM_GLOBAL_DELAY_IND_ADDR__ADDR__SHIFT                                                            0x0
+#define RLC_SPM_GLOBAL_DELAY_IND_ADDR__ADDR_MASK                                                              0x00000FFFL
+//RLC_SPM_GLOBAL_DELAY_IND_DATA
+#define RLC_SPM_GLOBAL_DELAY_IND_DATA__DATA__SHIFT                                                            0x0
+#define RLC_SPM_GLOBAL_DELAY_IND_DATA__DATA_MASK                                                              0x0000003FL
+//RLC_SPM_SE_DELAY_IND_ADDR
+#define RLC_SPM_SE_DELAY_IND_ADDR__ADDR__SHIFT                                                                0x0
+#define RLC_SPM_SE_DELAY_IND_ADDR__ADDR_MASK                                                                  0x00000FFFL
+//RLC_SPM_SE_DELAY_IND_DATA
+#define RLC_SPM_SE_DELAY_IND_DATA__DATA__SHIFT                                                                0x0
+#define RLC_SPM_SE_DELAY_IND_DATA__DATA_MASK                                                                  0x0000003FL
+//RLC_LX6_CNTL
+#define RLC_LX6_CNTL__BRESET__SHIFT                                                                           0x0
+#define RLC_LX6_CNTL__RUNSTALL__SHIFT                                                                         0x1
+#define RLC_LX6_CNTL__PDEBUG_ENABLE__SHIFT                                                                    0x2
+#define RLC_LX6_CNTL__STAT_VECTOR_SEL__SHIFT                                                                  0x3
+#define RLC_LX6_CNTL__BRESET_MASK                                                                             0x00000001L
+#define RLC_LX6_CNTL__RUNSTALL_MASK                                                                           0x00000002L
+#define RLC_LX6_CNTL__PDEBUG_ENABLE_MASK                                                                      0x00000004L
+#define RLC_LX6_CNTL__STAT_VECTOR_SEL_MASK                                                                    0x00000008L
+//RLC_XT_CORE_STATUS
+#define RLC_XT_CORE_STATUS__P_WAIT_MODE__SHIFT                                                                0x0
+#define RLC_XT_CORE_STATUS__P_FATAL_ERROR__SHIFT                                                              0x1
+#define RLC_XT_CORE_STATUS__DOUBLE_EXCEPTION_ERROR__SHIFT                                                     0x2
+#define RLC_XT_CORE_STATUS__P_WAIT_MODE_MASK                                                                  0x00000001L
+#define RLC_XT_CORE_STATUS__P_FATAL_ERROR_MASK                                                                0x00000002L
+#define RLC_XT_CORE_STATUS__DOUBLE_EXCEPTION_ERROR_MASK                                                       0x00000004L
+//RLC_XT_CORE_INTERRUPT
+#define RLC_XT_CORE_INTERRUPT__EXTINT1__SHIFT                                                                 0x0
+#define RLC_XT_CORE_INTERRUPT__EXTINT2__SHIFT                                                                 0x1a
+#define RLC_XT_CORE_INTERRUPT__NMI__SHIFT                                                                     0x1b
+#define RLC_XT_CORE_INTERRUPT__EXTINT1_MASK                                                                   0x03FFFFFFL
+#define RLC_XT_CORE_INTERRUPT__EXTINT2_MASK                                                                   0x04000000L
+#define RLC_XT_CORE_INTERRUPT__NMI_MASK                                                                       0x08000000L
+//RLC_XT_CORE_FAULT_INFO
+#define RLC_XT_CORE_FAULT_INFO__FAULT_INFO__SHIFT                                                             0x0
+#define RLC_XT_CORE_FAULT_INFO__FAULT_INFO_MASK                                                               0xFFFFFFFFL
+//RLC_XT_CORE_ALT_RESET_VEC
+#define RLC_XT_CORE_ALT_RESET_VEC__ALT_RESET_VEC__SHIFT                                                       0x0
+#define RLC_XT_CORE_ALT_RESET_VEC__ALT_RESET_VEC_MASK                                                         0xFFFFFFFFL
+//RLC_XT_CORE_RESERVED
+#define RLC_XT_CORE_RESERVED__RESERVED__SHIFT                                                                 0x0
+#define RLC_XT_CORE_RESERVED__RESERVED_MASK                                                                   0xFFFFFFFFL
+//RLC_XT_INT_VEC_FORCE
+#define RLC_XT_INT_VEC_FORCE__NUM_0__SHIFT                                                                    0x0
+#define RLC_XT_INT_VEC_FORCE__NUM_1__SHIFT                                                                    0x1
+#define RLC_XT_INT_VEC_FORCE__NUM_2__SHIFT                                                                    0x2
+#define RLC_XT_INT_VEC_FORCE__NUM_3__SHIFT                                                                    0x3
+#define RLC_XT_INT_VEC_FORCE__NUM_4__SHIFT                                                                    0x4
+#define RLC_XT_INT_VEC_FORCE__NUM_5__SHIFT                                                                    0x5
+#define RLC_XT_INT_VEC_FORCE__NUM_6__SHIFT                                                                    0x6
+#define RLC_XT_INT_VEC_FORCE__NUM_7__SHIFT                                                                    0x7
+#define RLC_XT_INT_VEC_FORCE__NUM_8__SHIFT                                                                    0x8
+#define RLC_XT_INT_VEC_FORCE__NUM_9__SHIFT                                                                    0x9
+#define RLC_XT_INT_VEC_FORCE__NUM_10__SHIFT                                                                   0xa
+#define RLC_XT_INT_VEC_FORCE__NUM_11__SHIFT                                                                   0xb
+#define RLC_XT_INT_VEC_FORCE__NUM_12__SHIFT                                                                   0xc
+#define RLC_XT_INT_VEC_FORCE__NUM_13__SHIFT                                                                   0xd
+#define RLC_XT_INT_VEC_FORCE__NUM_14__SHIFT                                                                   0xe
+#define RLC_XT_INT_VEC_FORCE__NUM_15__SHIFT                                                                   0xf
+#define RLC_XT_INT_VEC_FORCE__NUM_16__SHIFT                                                                   0x10
+#define RLC_XT_INT_VEC_FORCE__NUM_17__SHIFT                                                                   0x11
+#define RLC_XT_INT_VEC_FORCE__NUM_18__SHIFT                                                                   0x12
+#define RLC_XT_INT_VEC_FORCE__NUM_19__SHIFT                                                                   0x13
+#define RLC_XT_INT_VEC_FORCE__NUM_20__SHIFT                                                                   0x14
+#define RLC_XT_INT_VEC_FORCE__NUM_21__SHIFT                                                                   0x15
+#define RLC_XT_INT_VEC_FORCE__NUM_22__SHIFT                                                                   0x16
+#define RLC_XT_INT_VEC_FORCE__NUM_23__SHIFT                                                                   0x17
+#define RLC_XT_INT_VEC_FORCE__NUM_24__SHIFT                                                                   0x18
+#define RLC_XT_INT_VEC_FORCE__NUM_25__SHIFT                                                                   0x19
+#define RLC_XT_INT_VEC_FORCE__NUM_0_MASK                                                                      0x00000001L
+#define RLC_XT_INT_VEC_FORCE__NUM_1_MASK                                                                      0x00000002L
+#define RLC_XT_INT_VEC_FORCE__NUM_2_MASK                                                                      0x00000004L
+#define RLC_XT_INT_VEC_FORCE__NUM_3_MASK                                                                      0x00000008L
+#define RLC_XT_INT_VEC_FORCE__NUM_4_MASK                                                                      0x00000010L
+#define RLC_XT_INT_VEC_FORCE__NUM_5_MASK                                                                      0x00000020L
+#define RLC_XT_INT_VEC_FORCE__NUM_6_MASK                                                                      0x00000040L
+#define RLC_XT_INT_VEC_FORCE__NUM_7_MASK                                                                      0x00000080L
+#define RLC_XT_INT_VEC_FORCE__NUM_8_MASK                                                                      0x00000100L
+#define RLC_XT_INT_VEC_FORCE__NUM_9_MASK                                                                      0x00000200L
+#define RLC_XT_INT_VEC_FORCE__NUM_10_MASK                                                                     0x00000400L
+#define RLC_XT_INT_VEC_FORCE__NUM_11_MASK                                                                     0x00000800L
+#define RLC_XT_INT_VEC_FORCE__NUM_12_MASK                                                                     0x00001000L
+#define RLC_XT_INT_VEC_FORCE__NUM_13_MASK                                                                     0x00002000L
+#define RLC_XT_INT_VEC_FORCE__NUM_14_MASK                                                                     0x00004000L
+#define RLC_XT_INT_VEC_FORCE__NUM_15_MASK                                                                     0x00008000L
+#define RLC_XT_INT_VEC_FORCE__NUM_16_MASK                                                                     0x00010000L
+#define RLC_XT_INT_VEC_FORCE__NUM_17_MASK                                                                     0x00020000L
+#define RLC_XT_INT_VEC_FORCE__NUM_18_MASK                                                                     0x00040000L
+#define RLC_XT_INT_VEC_FORCE__NUM_19_MASK                                                                     0x00080000L
+#define RLC_XT_INT_VEC_FORCE__NUM_20_MASK                                                                     0x00100000L
+#define RLC_XT_INT_VEC_FORCE__NUM_21_MASK                                                                     0x00200000L
+#define RLC_XT_INT_VEC_FORCE__NUM_22_MASK                                                                     0x00400000L
+#define RLC_XT_INT_VEC_FORCE__NUM_23_MASK                                                                     0x00800000L
+#define RLC_XT_INT_VEC_FORCE__NUM_24_MASK                                                                     0x01000000L
+#define RLC_XT_INT_VEC_FORCE__NUM_25_MASK                                                                     0x02000000L
+//RLC_XT_INT_VEC_CLEAR
+#define RLC_XT_INT_VEC_CLEAR__NUM_0__SHIFT                                                                    0x0
+#define RLC_XT_INT_VEC_CLEAR__NUM_1__SHIFT                                                                    0x1
+#define RLC_XT_INT_VEC_CLEAR__NUM_2__SHIFT                                                                    0x2
+#define RLC_XT_INT_VEC_CLEAR__NUM_3__SHIFT                                                                    0x3
+#define RLC_XT_INT_VEC_CLEAR__NUM_4__SHIFT                                                                    0x4
+#define RLC_XT_INT_VEC_CLEAR__NUM_5__SHIFT                                                                    0x5
+#define RLC_XT_INT_VEC_CLEAR__NUM_6__SHIFT                                                                    0x6
+#define RLC_XT_INT_VEC_CLEAR__NUM_7__SHIFT                                                                    0x7
+#define RLC_XT_INT_VEC_CLEAR__NUM_8__SHIFT                                                                    0x8
+#define RLC_XT_INT_VEC_CLEAR__NUM_9__SHIFT                                                                    0x9
+#define RLC_XT_INT_VEC_CLEAR__NUM_10__SHIFT                                                                   0xa
+#define RLC_XT_INT_VEC_CLEAR__NUM_11__SHIFT                                                                   0xb
+#define RLC_XT_INT_VEC_CLEAR__NUM_12__SHIFT                                                                   0xc
+#define RLC_XT_INT_VEC_CLEAR__NUM_13__SHIFT                                                                   0xd
+#define RLC_XT_INT_VEC_CLEAR__NUM_14__SHIFT                                                                   0xe
+#define RLC_XT_INT_VEC_CLEAR__NUM_15__SHIFT                                                                   0xf
+#define RLC_XT_INT_VEC_CLEAR__NUM_16__SHIFT                                                                   0x10
+#define RLC_XT_INT_VEC_CLEAR__NUM_17__SHIFT                                                                   0x11
+#define RLC_XT_INT_VEC_CLEAR__NUM_18__SHIFT                                                                   0x12
+#define RLC_XT_INT_VEC_CLEAR__NUM_19__SHIFT                                                                   0x13
+#define RLC_XT_INT_VEC_CLEAR__NUM_20__SHIFT                                                                   0x14
+#define RLC_XT_INT_VEC_CLEAR__NUM_21__SHIFT                                                                   0x15
+#define RLC_XT_INT_VEC_CLEAR__NUM_22__SHIFT                                                                   0x16
+#define RLC_XT_INT_VEC_CLEAR__NUM_23__SHIFT                                                                   0x17
+#define RLC_XT_INT_VEC_CLEAR__NUM_24__SHIFT                                                                   0x18
+#define RLC_XT_INT_VEC_CLEAR__NUM_25__SHIFT                                                                   0x19
+#define RLC_XT_INT_VEC_CLEAR__NUM_0_MASK                                                                      0x00000001L
+#define RLC_XT_INT_VEC_CLEAR__NUM_1_MASK                                                                      0x00000002L
+#define RLC_XT_INT_VEC_CLEAR__NUM_2_MASK                                                                      0x00000004L
+#define RLC_XT_INT_VEC_CLEAR__NUM_3_MASK                                                                      0x00000008L
+#define RLC_XT_INT_VEC_CLEAR__NUM_4_MASK                                                                      0x00000010L
+#define RLC_XT_INT_VEC_CLEAR__NUM_5_MASK                                                                      0x00000020L
+#define RLC_XT_INT_VEC_CLEAR__NUM_6_MASK                                                                      0x00000040L
+#define RLC_XT_INT_VEC_CLEAR__NUM_7_MASK                                                                      0x00000080L
+#define RLC_XT_INT_VEC_CLEAR__NUM_8_MASK                                                                      0x00000100L
+#define RLC_XT_INT_VEC_CLEAR__NUM_9_MASK                                                                      0x00000200L
+#define RLC_XT_INT_VEC_CLEAR__NUM_10_MASK                                                                     0x00000400L
+#define RLC_XT_INT_VEC_CLEAR__NUM_11_MASK                                                                     0x00000800L
+#define RLC_XT_INT_VEC_CLEAR__NUM_12_MASK                                                                     0x00001000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_13_MASK                                                                     0x00002000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_14_MASK                                                                     0x00004000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_15_MASK                                                                     0x00008000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_16_MASK                                                                     0x00010000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_17_MASK                                                                     0x00020000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_18_MASK                                                                     0x00040000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_19_MASK                                                                     0x00080000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_20_MASK                                                                     0x00100000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_21_MASK                                                                     0x00200000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_22_MASK                                                                     0x00400000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_23_MASK                                                                     0x00800000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_24_MASK                                                                     0x01000000L
+#define RLC_XT_INT_VEC_CLEAR__NUM_25_MASK                                                                     0x02000000L
+//RLC_XT_INT_VEC_MUX_SEL
+#define RLC_XT_INT_VEC_MUX_SEL__MUX_SEL__SHIFT                                                                0x0
+#define RLC_XT_INT_VEC_MUX_SEL__MUX_SEL_MASK                                                                  0x0000001FL
+//RLC_XT_INT_VEC_MUX_INT_SEL
+#define RLC_XT_INT_VEC_MUX_INT_SEL__INT_SEL__SHIFT                                                            0x0
+#define RLC_XT_INT_VEC_MUX_INT_SEL__INT_SEL_MASK                                                              0x0000003FL
+//RLC_GPU_CLOCK_COUNT_SPM_LSB
+#define RLC_GPU_CLOCK_COUNT_SPM_LSB__GPU_CLOCKS_LSB__SHIFT                                                    0x0
+#define RLC_GPU_CLOCK_COUNT_SPM_LSB__GPU_CLOCKS_LSB_MASK                                                      0xFFFFFFFFL
+//RLC_GPU_CLOCK_COUNT_SPM_MSB
+#define RLC_GPU_CLOCK_COUNT_SPM_MSB__GPU_CLOCKS_MSB__SHIFT                                                    0x0
+#define RLC_GPU_CLOCK_COUNT_SPM_MSB__GPU_CLOCKS_MSB_MASK                                                      0xFFFFFFFFL
+//RLC_SPM_THREAD_TRACE_CTRL
+#define RLC_SPM_THREAD_TRACE_CTRL__THREAD_TRACE_INT_EN__SHIFT                                                 0x0
+#define RLC_SPM_THREAD_TRACE_CTRL__THREAD_TRACE_INT_EN_MASK                                                   0x00000001L
+//RLC_SPP_CAM_ADDR
+#define RLC_SPP_CAM_ADDR__ADDR__SHIFT                                                                         0x0
+#define RLC_SPP_CAM_ADDR__ADDR_MASK                                                                           0x000000FFL
+//RLC_SPP_CAM_DATA
+#define RLC_SPP_CAM_DATA__DATA__SHIFT                                                                         0x0
+#define RLC_SPP_CAM_DATA__TAG__SHIFT                                                                          0x8
+#define RLC_SPP_CAM_DATA__DATA_MASK                                                                           0x000000FFL
+#define RLC_SPP_CAM_DATA__TAG_MASK                                                                            0xFFFFFF00L
+//RLC_SPP_CAM_EXT_ADDR
+#define RLC_SPP_CAM_EXT_ADDR__ADDR__SHIFT                                                                     0x0
+#define RLC_SPP_CAM_EXT_ADDR__ADDR_MASK                                                                       0x000000FFL
+//RLC_SPP_CAM_EXT_DATA
+#define RLC_SPP_CAM_EXT_DATA__VALID__SHIFT                                                                    0x0
+#define RLC_SPP_CAM_EXT_DATA__LOCK__SHIFT                                                                     0x1
+#define RLC_SPP_CAM_EXT_DATA__VALID_MASK                                                                      0x00000001L
+#define RLC_SPP_CAM_EXT_DATA__LOCK_MASK                                                                       0x00000002L
+//RLC_CPAXI_DOORBELL_MON_CTRL
+#define RLC_CPAXI_DOORBELL_MON_CTRL__EN__SHIFT                                                                0x0
+#define RLC_CPAXI_DOORBELL_MON_CTRL__ID__SHIFT                                                                0x1
+#define RLC_CPAXI_DOORBELL_MON_CTRL__EN_MASK                                                                  0x00000001L
+#define RLC_CPAXI_DOORBELL_MON_CTRL__ID_MASK                                                                  0x0000003EL
+//RLC_CPAXI_DOORBELL_MON_STAT
+#define RLC_CPAXI_DOORBELL_MON_STAT__ID_MATCH__SHIFT                                                          0x0
+#define RLC_CPAXI_DOORBELL_MON_STAT__MATCH_CLEAR__SHIFT                                                       0x1
+#define RLC_CPAXI_DOORBELL_MON_STAT__ADDR__SHIFT                                                              0x2
+#define RLC_CPAXI_DOORBELL_MON_STAT__ID_MATCH_MASK                                                            0x00000001L
+#define RLC_CPAXI_DOORBELL_MON_STAT__MATCH_CLEAR_MASK                                                         0x00000002L
+#define RLC_CPAXI_DOORBELL_MON_STAT__ADDR_MASK                                                                0x0FFFFFFCL
+//RLC_CPAXI_DOORBELL_MON_DATA_LSB
+#define RLC_CPAXI_DOORBELL_MON_DATA_LSB__DATA__SHIFT                                                          0x0
+#define RLC_CPAXI_DOORBELL_MON_DATA_LSB__DATA_MASK                                                            0xFFFFFFFFL
+//RLC_CPAXI_DOORBELL_MON_DATA_MSB
+#define RLC_CPAXI_DOORBELL_MON_DATA_MSB__DATA__SHIFT                                                          0x0
+#define RLC_CPAXI_DOORBELL_MON_DATA_MSB__DATA_MASK                                                            0xFFFFFFFFL
+//RLC_XT_DOORBELL_RANGE
+#define RLC_XT_DOORBELL_RANGE__LOWER_ADDR_RESERVED__SHIFT                                                     0x0
+#define RLC_XT_DOORBELL_RANGE__LOWER_ADDR__SHIFT                                                              0x2
+#define RLC_XT_DOORBELL_RANGE__UPPER_ADDR_RESERVED__SHIFT                                                     0x10
+#define RLC_XT_DOORBELL_RANGE__UPPER_ADDR__SHIFT                                                              0x12
+#define RLC_XT_DOORBELL_RANGE__LOWER_ADDR_RESERVED_MASK                                                       0x00000003L
+#define RLC_XT_DOORBELL_RANGE__LOWER_ADDR_MASK                                                                0x00000FFCL
+#define RLC_XT_DOORBELL_RANGE__UPPER_ADDR_RESERVED_MASK                                                       0x00030000L
+#define RLC_XT_DOORBELL_RANGE__UPPER_ADDR_MASK                                                                0x0FFC0000L
+//RLC_XT_DOORBELL_CNTL
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_0_MODE__SHIFT                                                          0x0
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_1_MODE__SHIFT                                                          0x2
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_2_MODE__SHIFT                                                          0x4
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_3_MODE__SHIFT                                                          0x6
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_ID__SHIFT                                                              0x10
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_ID_EN__SHIFT                                                           0x15
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_0_MODE_MASK                                                            0x00000003L
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_1_MODE_MASK                                                            0x0000000CL
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_2_MODE_MASK                                                            0x00000030L
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_3_MODE_MASK                                                            0x000000C0L
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_ID_MASK                                                                0x001F0000L
+#define RLC_XT_DOORBELL_CNTL__DOORBELL_ID_EN_MASK                                                             0x00200000L
+//RLC_XT_DOORBELL_STAT
+#define RLC_XT_DOORBELL_STAT__DOORBELL_0_VALID__SHIFT                                                         0x0
+#define RLC_XT_DOORBELL_STAT__DOORBELL_1_VALID__SHIFT                                                         0x1
+#define RLC_XT_DOORBELL_STAT__DOORBELL_2_VALID__SHIFT                                                         0x2
+#define RLC_XT_DOORBELL_STAT__DOORBELL_3_VALID__SHIFT                                                         0x3
+#define RLC_XT_DOORBELL_STAT__DOORBELL_0_VALID_MASK                                                           0x00000001L
+#define RLC_XT_DOORBELL_STAT__DOORBELL_1_VALID_MASK                                                           0x00000002L
+#define RLC_XT_DOORBELL_STAT__DOORBELL_2_VALID_MASK                                                           0x00000004L
+#define RLC_XT_DOORBELL_STAT__DOORBELL_3_VALID_MASK                                                           0x00000008L
+//RLC_XT_DOORBELL_0_DATA_LO
+#define RLC_XT_DOORBELL_0_DATA_LO__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_0_DATA_LO__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_0_DATA_HI
+#define RLC_XT_DOORBELL_0_DATA_HI__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_0_DATA_HI__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_1_DATA_LO
+#define RLC_XT_DOORBELL_1_DATA_LO__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_1_DATA_LO__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_1_DATA_HI
+#define RLC_XT_DOORBELL_1_DATA_HI__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_1_DATA_HI__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_2_DATA_LO
+#define RLC_XT_DOORBELL_2_DATA_LO__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_2_DATA_LO__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_2_DATA_HI
+#define RLC_XT_DOORBELL_2_DATA_HI__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_2_DATA_HI__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_3_DATA_LO
+#define RLC_XT_DOORBELL_3_DATA_LO__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_3_DATA_LO__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_XT_DOORBELL_3_DATA_HI
+#define RLC_XT_DOORBELL_3_DATA_HI__DATA__SHIFT                                                                0x0
+#define RLC_XT_DOORBELL_3_DATA_HI__DATA_MASK                                                                  0xFFFFFFFFL
+//RLC_MEM_SLP_CNTL
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN__SHIFT                                                                0x0
+#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN__SHIFT                                                                0x1
+#define RLC_MEM_SLP_CNTL__RLC_SRM_MEM_LS_OVERRIDE__SHIFT                                                      0x2
+#define RLC_MEM_SLP_CNTL__RLC_SRM_MEM_DS_OVERRIDE__SHIFT                                                      0x3
+#define RLC_MEM_SLP_CNTL__RLC_SPM_MEM_LS_OVERRIDE__SHIFT                                                      0x4
+#define RLC_MEM_SLP_CNTL__RLC_SPM_MEM_DS_OVERRIDE__SHIFT                                                      0x5
+#define RLC_MEM_SLP_CNTL__RESERVED__SHIFT                                                                     0x6
+#define RLC_MEM_SLP_CNTL__RLC_LS_DS_BUSY_OVERRIDE__SHIFT                                                      0x7
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY__SHIFT                                                          0x8
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY__SHIFT                                                         0x10
+#define RLC_MEM_SLP_CNTL__RLC_SPP_MEM_LS_OVERRIDE__SHIFT                                                      0x18
+#define RLC_MEM_SLP_CNTL__RLC_SPP_MEM_DS_OVERRIDE__SHIFT                                                      0x19
+#define RLC_MEM_SLP_CNTL__RESERVED1__SHIFT                                                                    0x1a
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK                                                                  0x00000001L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_DS_EN_MASK                                                                  0x00000002L
+#define RLC_MEM_SLP_CNTL__RLC_SRM_MEM_LS_OVERRIDE_MASK                                                        0x00000004L
+#define RLC_MEM_SLP_CNTL__RLC_SRM_MEM_DS_OVERRIDE_MASK                                                        0x00000008L
+#define RLC_MEM_SLP_CNTL__RLC_SPM_MEM_LS_OVERRIDE_MASK                                                        0x00000010L
+#define RLC_MEM_SLP_CNTL__RLC_SPM_MEM_DS_OVERRIDE_MASK                                                        0x00000020L
+#define RLC_MEM_SLP_CNTL__RESERVED_MASK                                                                       0x00000040L
+#define RLC_MEM_SLP_CNTL__RLC_LS_DS_BUSY_OVERRIDE_MASK                                                        0x00000080L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_ON_DELAY_MASK                                                            0x0000FF00L
+#define RLC_MEM_SLP_CNTL__RLC_MEM_LS_OFF_DELAY_MASK                                                           0x00FF0000L
+#define RLC_MEM_SLP_CNTL__RLC_SPP_MEM_LS_OVERRIDE_MASK                                                        0x01000000L
+#define RLC_MEM_SLP_CNTL__RLC_SPP_MEM_DS_OVERRIDE_MASK                                                        0x02000000L
+#define RLC_MEM_SLP_CNTL__RESERVED1_MASK                                                                      0xFC000000L
+//SMU_RLC_RESPONSE
+#define SMU_RLC_RESPONSE__RESP__SHIFT                                                                         0x0
+#define SMU_RLC_RESPONSE__RESP_MASK                                                                           0xFFFFFFFFL
+//RLC_RLCV_SAFE_MODE
+#define RLC_RLCV_SAFE_MODE__CMD__SHIFT                                                                        0x0
+#define RLC_RLCV_SAFE_MODE__MESSAGE__SHIFT                                                                    0x1
+#define RLC_RLCV_SAFE_MODE__RESERVED1__SHIFT                                                                  0x5
+#define RLC_RLCV_SAFE_MODE__RESPONSE__SHIFT                                                                   0x8
+#define RLC_RLCV_SAFE_MODE__RESERVED__SHIFT                                                                   0xc
+#define RLC_RLCV_SAFE_MODE__CMD_MASK                                                                          0x00000001L
+#define RLC_RLCV_SAFE_MODE__MESSAGE_MASK                                                                      0x0000001EL
+#define RLC_RLCV_SAFE_MODE__RESERVED1_MASK                                                                    0x000000E0L
+#define RLC_RLCV_SAFE_MODE__RESPONSE_MASK                                                                     0x00000F00L
+#define RLC_RLCV_SAFE_MODE__RESERVED_MASK                                                                     0xFFFFF000L
+//RLC_SMU_SAFE_MODE
+#define RLC_SMU_SAFE_MODE__CMD__SHIFT                                                                         0x0
+#define RLC_SMU_SAFE_MODE__MESSAGE__SHIFT                                                                     0x1
+#define RLC_SMU_SAFE_MODE__RESERVED1__SHIFT                                                                   0x5
+#define RLC_SMU_SAFE_MODE__RESPONSE__SHIFT                                                                    0x8
+#define RLC_SMU_SAFE_MODE__RESERVED__SHIFT                                                                    0xc
+#define RLC_SMU_SAFE_MODE__CMD_MASK                                                                           0x00000001L
+#define RLC_SMU_SAFE_MODE__MESSAGE_MASK                                                                       0x0000001EL
+#define RLC_SMU_SAFE_MODE__RESERVED1_MASK                                                                     0x000000E0L
+#define RLC_SMU_SAFE_MODE__RESPONSE_MASK                                                                      0x00000F00L
+#define RLC_SMU_SAFE_MODE__RESERVED_MASK                                                                      0xFFFFF000L
+//RLC_RLCV_COMMAND
+#define RLC_RLCV_COMMAND__CMD__SHIFT                                                                          0x0
+#define RLC_RLCV_COMMAND__RESERVED__SHIFT                                                                     0x4
+#define RLC_RLCV_COMMAND__CMD_MASK                                                                            0x0000000FL
+#define RLC_RLCV_COMMAND__RESERVED_MASK                                                                       0xFFFFFFF0L
+//RLC_SMU_MESSAGE
+#define RLC_SMU_MESSAGE__CMD__SHIFT                                                                           0x0
+#define RLC_SMU_MESSAGE__CMD_MASK                                                                             0xFFFFFFFFL
+//RLC_SMU_MESSAGE_1
+#define RLC_SMU_MESSAGE_1__CMD__SHIFT                                                                         0x0
+#define RLC_SMU_MESSAGE_1__CMD_MASK                                                                           0xFFFFFFFFL
+//RLC_SMU_MESSAGE_2
+#define RLC_SMU_MESSAGE_2__CMD__SHIFT                                                                         0x0
+#define RLC_SMU_MESSAGE_2__CMD_MASK                                                                           0xFFFFFFFFL
+//RLC_SRM_GPM_COMMAND
+#define RLC_SRM_GPM_COMMAND__OP__SHIFT                                                                        0x0
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL__SHIFT                                                                0x1
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_NUM__SHIFT                                                            0x2
+#define RLC_SRM_GPM_COMMAND__SIZE__SHIFT                                                                      0x5
+#define RLC_SRM_GPM_COMMAND__START_OFFSET__SHIFT                                                              0x12
+#define RLC_SRM_GPM_COMMAND__DEST_MEMORY__SHIFT                                                               0x1f
+#define RLC_SRM_GPM_COMMAND__OP_MASK                                                                          0x00000001L
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_MASK                                                                  0x00000002L
+#define RLC_SRM_GPM_COMMAND__INDEX_CNTL_NUM_MASK                                                              0x0000001CL
+#define RLC_SRM_GPM_COMMAND__SIZE_MASK                                                                        0x0003FFE0L
+#define RLC_SRM_GPM_COMMAND__START_OFFSET_MASK                                                                0x7FFC0000L
+#define RLC_SRM_GPM_COMMAND__DEST_MEMORY_MASK                                                                 0x80000000L
+//RLC_SRM_GPM_ABORT
+#define RLC_SRM_GPM_ABORT__ABORT__SHIFT                                                                       0x0
+#define RLC_SRM_GPM_ABORT__RESERVED__SHIFT                                                                    0x1
+#define RLC_SRM_GPM_ABORT__ABORT_MASK                                                                         0x00000001L
+#define RLC_SRM_GPM_ABORT__RESERVED_MASK                                                                      0xFFFFFFFEL
+//RLC_SMU_COMMAND
+#define RLC_SMU_COMMAND__CMD__SHIFT                                                                           0x0
+#define RLC_SMU_COMMAND__CMD_MASK                                                                             0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_1
+#define RLC_SMU_ARGUMENT_1__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_1__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_2
+#define RLC_SMU_ARGUMENT_2__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_2__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_3
+#define RLC_SMU_ARGUMENT_3__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_3__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_4
+#define RLC_SMU_ARGUMENT_4__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_4__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_SMU_ARGUMENT_5
+#define RLC_SMU_ARGUMENT_5__ARG__SHIFT                                                                        0x0
+#define RLC_SMU_ARGUMENT_5__ARG_MASK                                                                          0xFFFFFFFFL
+//RLC_IMU_BOOTLOAD_ADDR_HI
+#define RLC_IMU_BOOTLOAD_ADDR_HI__ADDR_HI__SHIFT                                                              0x0
+#define RLC_IMU_BOOTLOAD_ADDR_HI__ADDR_HI_MASK                                                                0xFFFFFFFFL
+//RLC_IMU_BOOTLOAD_ADDR_LO
+#define RLC_IMU_BOOTLOAD_ADDR_LO__ADDR_LO__SHIFT                                                              0x0
+#define RLC_IMU_BOOTLOAD_ADDR_LO__ADDR_LO_MASK                                                                0xFFFFFFFFL
+//RLC_IMU_BOOTLOAD_SIZE
+#define RLC_IMU_BOOTLOAD_SIZE__SIZE__SHIFT                                                                    0x0
+#define RLC_IMU_BOOTLOAD_SIZE__RESERVED__SHIFT                                                                0x1a
+#define RLC_IMU_BOOTLOAD_SIZE__SIZE_MASK                                                                      0x03FFFFFFL
+#define RLC_IMU_BOOTLOAD_SIZE__RESERVED_MASK                                                                  0xFC000000L
+//RLC_IMU_MISC
+#define RLC_IMU_MISC__THROTTLE_GFX__SHIFT                                                                     0x0
+#define RLC_IMU_MISC__EARLY_MGCG__SHIFT                                                                       0x1
+#define RLC_IMU_MISC__RESERVED__SHIFT                                                                         0x2
+#define RLC_IMU_MISC__THROTTLE_GFX_MASK                                                                       0x00000001L
+#define RLC_IMU_MISC__EARLY_MGCG_MASK                                                                         0x00000002L
+#define RLC_IMU_MISC__RESERVED_MASK                                                                           0xFFFFFFFCL
+//RLC_IMU_RESET_VECTOR
+#define RLC_IMU_RESET_VECTOR__COLD_BOOT_EXIT__SHIFT                                                           0x0
+#define RLC_IMU_RESET_VECTOR__VDDGFX_EXIT__SHIFT                                                              0x1
+#define RLC_IMU_RESET_VECTOR__VECTOR__SHIFT                                                                   0x2
+#define RLC_IMU_RESET_VECTOR__RESERVED__SHIFT                                                                 0x8
+#define RLC_IMU_RESET_VECTOR__COLD_BOOT_EXIT_MASK                                                             0x00000001L
+#define RLC_IMU_RESET_VECTOR__VDDGFX_EXIT_MASK                                                                0x00000002L
+#define RLC_IMU_RESET_VECTOR__VECTOR_MASK                                                                     0x000000FCL
+#define RLC_IMU_RESET_VECTOR__RESERVED_MASK                                                                   0xFFFFFF00L
+
+
+// addressBlock: gc_rlcsdec
+//RLC_RLCS_DEC_START
+//RLC_RLCS_DEC_DUMP_ADDR
+//RLC_RLCS_EXCEPTION_REG_1
+#define RLC_RLCS_EXCEPTION_REG_1__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_EXCEPTION_REG_1__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_EXCEPTION_REG_1__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_EXCEPTION_REG_1__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_EXCEPTION_REG_2
+#define RLC_RLCS_EXCEPTION_REG_2__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_EXCEPTION_REG_2__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_EXCEPTION_REG_2__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_EXCEPTION_REG_2__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_EXCEPTION_REG_3
+#define RLC_RLCS_EXCEPTION_REG_3__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_EXCEPTION_REG_3__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_EXCEPTION_REG_3__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_EXCEPTION_REG_3__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_EXCEPTION_REG_4
+#define RLC_RLCS_EXCEPTION_REG_4__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_EXCEPTION_REG_4__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_EXCEPTION_REG_4__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_EXCEPTION_REG_4__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_CGCG_REQUEST
+#define RLC_RLCS_CGCG_REQUEST__CGCG_REQUEST__SHIFT                                                            0x0
+#define RLC_RLCS_CGCG_REQUEST__CGCG_REQUEST_3D__SHIFT                                                         0x1
+#define RLC_RLCS_CGCG_REQUEST__RESERVED__SHIFT                                                                0x2
+#define RLC_RLCS_CGCG_REQUEST__CGCG_REQUEST_MASK                                                              0x00000001L
+#define RLC_RLCS_CGCG_REQUEST__CGCG_REQUEST_3D_MASK                                                           0x00000002L
+#define RLC_RLCS_CGCG_REQUEST__RESERVED_MASK                                                                  0xFFFFFFFCL
+//RLC_RLCS_CGCG_STATUS
+#define RLC_RLCS_CGCG_STATUS__CGCG_RAMP_STATUS__SHIFT                                                         0x0
+#define RLC_RLCS_CGCG_STATUS__GFX_CLK_STATUS__SHIFT                                                           0x2
+#define RLC_RLCS_CGCG_STATUS__CGCG_RAMP_STATUS_3D__SHIFT                                                      0x3
+#define RLC_RLCS_CGCG_STATUS__GFX_CLK_STATUS_3D__SHIFT                                                        0x5
+#define RLC_RLCS_CGCG_STATUS__RESERVED__SHIFT                                                                 0x6
+#define RLC_RLCS_CGCG_STATUS__CGCG_RAMP_STATUS_MASK                                                           0x00000003L
+#define RLC_RLCS_CGCG_STATUS__GFX_CLK_STATUS_MASK                                                             0x00000004L
+#define RLC_RLCS_CGCG_STATUS__CGCG_RAMP_STATUS_3D_MASK                                                        0x00000018L
+#define RLC_RLCS_CGCG_STATUS__GFX_CLK_STATUS_3D_MASK                                                          0x00000020L
+#define RLC_RLCS_CGCG_STATUS__RESERVED_MASK                                                                   0xFFFFFFC0L
+//RLC_RLCS_SOC_DS_CNTL
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_ALLOW__SHIFT                                                         0x0
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_RLC_BUSY_MASK__SHIFT                                                 0x1
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_CP_BUSY_MASK__SHIFT                                                  0x2
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_GFX_PWR_STALLED_MASK__SHIFT                                          0x6
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_NON3D_PWR_STALLED_MASK__SHIFT                                        0x7
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_0_BUSY_MASK__SHIFT                                              0x10
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_1_BUSY_MASK__SHIFT                                              0x11
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_2_BUSY_MASK__SHIFT                                              0x12
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_3_BUSY_MASK__SHIFT                                              0x13
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_4_BUSY_MASK__SHIFT                                              0x14
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_5_BUSY_MASK__SHIFT                                              0x15
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_6_BUSY_MASK__SHIFT                                              0x16
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_7_BUSY_MASK__SHIFT                                              0x17
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_ALLOW_MASK                                                           0x00000001L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_RLC_BUSY_MASK_MASK                                                   0x00000002L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_CP_BUSY_MASK_MASK                                                    0x00000004L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_GFX_PWR_STALLED_MASK_MASK                                            0x00000040L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_NON3D_PWR_STALLED_MASK_MASK                                          0x00000080L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_0_BUSY_MASK_MASK                                                0x00010000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_1_BUSY_MASK_MASK                                                0x00020000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_2_BUSY_MASK_MASK                                                0x00040000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_3_BUSY_MASK_MASK                                                0x00080000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_4_BUSY_MASK_MASK                                                0x00100000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_5_BUSY_MASK_MASK                                                0x00200000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_6_BUSY_MASK_MASK                                                0x00400000L
+#define RLC_RLCS_SOC_DS_CNTL__SOC_CLK_DS_SDMA_7_BUSY_MASK_MASK                                                0x00800000L
+//RLC_RLCS_GFX_DS_CNTL
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_ALLOW__SHIFT                                                         0x0
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_RLC_BUSY_MASK__SHIFT                                                 0x1
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_CP_BUSY_MASK__SHIFT                                                  0x2
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_GFX_PWR_STALLED_MASK__SHIFT                                          0x6
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_NON3D_PWR_STALLED_MASK__SHIFT                                        0x7
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_IMU_DISABLE_MASK__SHIFT                                              0x8
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_0_BUSY_MASK__SHIFT                                              0x10
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_1_BUSY_MASK__SHIFT                                              0x11
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_2_BUSY_MASK__SHIFT                                              0x12
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_3_BUSY_MASK__SHIFT                                              0x13
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_4_BUSY_MASK__SHIFT                                              0x14
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_5_BUSY_MASK__SHIFT                                              0x15
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_6_BUSY_MASK__SHIFT                                              0x16
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_7_BUSY_MASK__SHIFT                                              0x17
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_ALLOW_MASK                                                           0x00000001L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_RLC_BUSY_MASK_MASK                                                   0x00000002L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_CP_BUSY_MASK_MASK                                                    0x00000004L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_GFX_PWR_STALLED_MASK_MASK                                            0x00000040L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_NON3D_PWR_STALLED_MASK_MASK                                          0x00000080L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_IMU_DISABLE_MASK_MASK                                                0x00000100L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_0_BUSY_MASK_MASK                                                0x00010000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_1_BUSY_MASK_MASK                                                0x00020000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_2_BUSY_MASK_MASK                                                0x00040000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_3_BUSY_MASK_MASK                                                0x00080000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_4_BUSY_MASK_MASK                                                0x00100000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_5_BUSY_MASK_MASK                                                0x00200000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_6_BUSY_MASK_MASK                                                0x00400000L
+#define RLC_RLCS_GFX_DS_CNTL__GFX_CLK_DS_SDMA_7_BUSY_MASK_MASK                                                0x00800000L
+//RLC_RLCS_GFX_DS_ALLOW_MASK_CNTL
+#define RLC_RLCS_GFX_DS_ALLOW_MASK_CNTL__GFX_CLK_DS_ALLOW_MASK_GDFLL__SHIFT                                   0x0
+#define RLC_RLCS_GFX_DS_ALLOW_MASK_CNTL__GFX_CLK_DS_ALLOW_MASK_GDFLL_SE0__SHIFT                               0x1
+#define RLC_RLCS_GFX_DS_ALLOW_MASK_CNTL__GFX_CLK_DS_ALLOW_MASK_GDFLL_SE1__SHIFT                               0x2
+#define RLC_RLCS_GFX_DS_ALLOW_MASK_CNTL__GFX_CLK_DS_ALLOW_MASK_GDFLL_SE2__SHIFT                               0x3
+#define RLC_RLCS_GFX_DS_ALLOW_MASK_CNTL__GFX_CLK_DS_ALLOW_MASK_GDFLL_MASK                                     0x00000001L
+#define RLC_RLCS_GFX_DS_ALLOW_MASK_CNTL__GFX_CLK_DS_ALLOW_MASK_GDFLL_SE0_MASK                                 0x00000002L
+#define RLC_RLCS_GFX_DS_ALLOW_MASK_CNTL__GFX_CLK_DS_ALLOW_MASK_GDFLL_SE1_MASK                                 0x00000004L
+#define RLC_RLCS_GFX_DS_ALLOW_MASK_CNTL__GFX_CLK_DS_ALLOW_MASK_GDFLL_SE2_MASK                                 0x00000008L
+//RLC_GPM_STAT
+#define RLC_GPM_STAT__RLC_BUSY__SHIFT                                                                         0x0
+#define RLC_GPM_STAT__GFX_POWER_STATUS__SHIFT                                                                 0x1
+#define RLC_GPM_STAT__GFX_CLOCK_STATUS__SHIFT                                                                 0x2
+#define RLC_GPM_STAT__GFX_LS_STATUS__SHIFT                                                                    0x3
+#define RLC_GPM_STAT__GFX_PIPELINE_POWER_STATUS__SHIFT                                                        0x4
+#define RLC_GPM_STAT__CNTX_IDLE_BEING_PROCESSED__SHIFT                                                        0x5
+#define RLC_GPM_STAT__CNTX_BUSY_BEING_PROCESSED__SHIFT                                                        0x6
+#define RLC_GPM_STAT__GFX_IDLE_BEING_PROCESSED__SHIFT                                                         0x7
+#define RLC_GPM_STAT__CMP_BUSY_BEING_PROCESSED__SHIFT                                                         0x8
+#define RLC_GPM_STAT__SAVING_REGISTERS__SHIFT                                                                 0x9
+#define RLC_GPM_STAT__RESTORING_REGISTERS__SHIFT                                                              0xa
+#define RLC_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE__SHIFT                                                0xb
+#define RLC_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE__SHIFT                                                  0xc
+#define RLC_GPM_STAT__STATIC_WGP_POWERING_UP__SHIFT                                                           0xd
+#define RLC_GPM_STAT__STATIC_WGP_POWERING_DOWN__SHIFT                                                         0xe
+#define RLC_GPM_STAT__DYN_WGP_POWERING_UP__SHIFT                                                              0xf
+#define RLC_GPM_STAT__DYN_WGP_POWERING_DOWN__SHIFT                                                            0x10
+#define RLC_GPM_STAT__ABORTED_PD_SEQUENCE__SHIFT                                                              0x11
+#define RLC_GPM_STAT__CMP_power_status__SHIFT                                                                 0x12
+#define RLC_GPM_STAT__GFX_LS_STATUS_3D__SHIFT                                                                 0x13
+#define RLC_GPM_STAT__GFX_CLOCK_STATUS_3D__SHIFT                                                              0x14
+#define RLC_GPM_STAT__MGCG_OVERRIDE_STATUS__SHIFT                                                             0x15
+#define RLC_GPM_STAT__RLC_EXEC_ROM_CODE__SHIFT                                                                0x16
+#define RLC_GPM_STAT__FGCG_OVERRIDE_STATUS__SHIFT                                                             0x17
+#define RLC_GPM_STAT__PG_ERROR_STATUS__SHIFT                                                                  0x18
+#define RLC_GPM_STAT__RLC_BUSY_MASK                                                                           0x00000001L
+#define RLC_GPM_STAT__GFX_POWER_STATUS_MASK                                                                   0x00000002L
+#define RLC_GPM_STAT__GFX_CLOCK_STATUS_MASK                                                                   0x00000004L
+#define RLC_GPM_STAT__GFX_LS_STATUS_MASK                                                                      0x00000008L
+#define RLC_GPM_STAT__GFX_PIPELINE_POWER_STATUS_MASK                                                          0x00000010L
+#define RLC_GPM_STAT__CNTX_IDLE_BEING_PROCESSED_MASK                                                          0x00000020L
+#define RLC_GPM_STAT__CNTX_BUSY_BEING_PROCESSED_MASK                                                          0x00000040L
+#define RLC_GPM_STAT__GFX_IDLE_BEING_PROCESSED_MASK                                                           0x00000080L
+#define RLC_GPM_STAT__CMP_BUSY_BEING_PROCESSED_MASK                                                           0x00000100L
+#define RLC_GPM_STAT__SAVING_REGISTERS_MASK                                                                   0x00000200L
+#define RLC_GPM_STAT__RESTORING_REGISTERS_MASK                                                                0x00000400L
+#define RLC_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE_MASK                                                  0x00000800L
+#define RLC_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE_MASK                                                    0x00001000L
+#define RLC_GPM_STAT__STATIC_WGP_POWERING_UP_MASK                                                             0x00002000L
+#define RLC_GPM_STAT__STATIC_WGP_POWERING_DOWN_MASK                                                           0x00004000L
+#define RLC_GPM_STAT__DYN_WGP_POWERING_UP_MASK                                                                0x00008000L
+#define RLC_GPM_STAT__DYN_WGP_POWERING_DOWN_MASK                                                              0x00010000L
+#define RLC_GPM_STAT__ABORTED_PD_SEQUENCE_MASK                                                                0x00020000L
+#define RLC_GPM_STAT__CMP_power_status_MASK                                                                   0x00040000L
+#define RLC_GPM_STAT__GFX_LS_STATUS_3D_MASK                                                                   0x00080000L
+#define RLC_GPM_STAT__GFX_CLOCK_STATUS_3D_MASK                                                                0x00100000L
+#define RLC_GPM_STAT__MGCG_OVERRIDE_STATUS_MASK                                                               0x00200000L
+#define RLC_GPM_STAT__RLC_EXEC_ROM_CODE_MASK                                                                  0x00400000L
+#define RLC_GPM_STAT__FGCG_OVERRIDE_STATUS_MASK                                                               0x00800000L
+#define RLC_GPM_STAT__PG_ERROR_STATUS_MASK                                                                    0xFF000000L
+//RLC_RLCS_GPM_STAT
+#define RLC_RLCS_GPM_STAT__RLC_BUSY__SHIFT                                                                    0x0
+#define RLC_RLCS_GPM_STAT__GFX_POWER_STATUS__SHIFT                                                            0x1
+#define RLC_RLCS_GPM_STAT__GFX_CLOCK_STATUS__SHIFT                                                            0x2
+#define RLC_RLCS_GPM_STAT__GFX_LS_STATUS__SHIFT                                                               0x3
+#define RLC_RLCS_GPM_STAT__GFX_PIPELINE_POWER_STATUS__SHIFT                                                   0x4
+#define RLC_RLCS_GPM_STAT__CNTX_IDLE_BEING_PROCESSED__SHIFT                                                   0x5
+#define RLC_RLCS_GPM_STAT__CNTX_BUSY_BEING_PROCESSED__SHIFT                                                   0x6
+#define RLC_RLCS_GPM_STAT__GFX_IDLE_BEING_PROCESSED__SHIFT                                                    0x7
+#define RLC_RLCS_GPM_STAT__CMP_BUSY_BEING_PROCESSED__SHIFT                                                    0x8
+#define RLC_RLCS_GPM_STAT__SAVING_REGISTERS__SHIFT                                                            0x9
+#define RLC_RLCS_GPM_STAT__RESTORING_REGISTERS__SHIFT                                                         0xa
+#define RLC_RLCS_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE__SHIFT                                           0xb
+#define RLC_RLCS_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE__SHIFT                                             0xc
+#define RLC_RLCS_GPM_STAT__STATIC_WGP_POWERING_UP__SHIFT                                                      0xd
+#define RLC_RLCS_GPM_STAT__STATIC_WGP_POWERING_DOWN__SHIFT                                                    0xe
+#define RLC_RLCS_GPM_STAT__DYN_WGP_POWERING_UP__SHIFT                                                         0xf
+#define RLC_RLCS_GPM_STAT__DYN_WGP_POWERING_DOWN__SHIFT                                                       0x10
+#define RLC_RLCS_GPM_STAT__ABORTED_PD_SEQUENCE__SHIFT                                                         0x11
+#define RLC_RLCS_GPM_STAT__CMP_POWER_STATUS__SHIFT                                                            0x12
+#define RLC_RLCS_GPM_STAT__GFX_LS_STATUS_3D__SHIFT                                                            0x13
+#define RLC_RLCS_GPM_STAT__GFX_CLOCK_STATUS_3D__SHIFT                                                         0x14
+#define RLC_RLCS_GPM_STAT__MGCG_OVERRIDE_STATUS__SHIFT                                                        0x15
+#define RLC_RLCS_GPM_STAT__RLC_EXEC_ROM_CODE__SHIFT                                                           0x16
+#define RLC_RLCS_GPM_STAT__FGCG_OVERRIDE_STATUS__SHIFT                                                        0x17
+#define RLC_RLCS_GPM_STAT__PG_ERROR_STATUS__SHIFT                                                             0x18
+#define RLC_RLCS_GPM_STAT__RLC_BUSY_MASK                                                                      0x00000001L
+#define RLC_RLCS_GPM_STAT__GFX_POWER_STATUS_MASK                                                              0x00000002L
+#define RLC_RLCS_GPM_STAT__GFX_CLOCK_STATUS_MASK                                                              0x00000004L
+#define RLC_RLCS_GPM_STAT__GFX_LS_STATUS_MASK                                                                 0x00000008L
+#define RLC_RLCS_GPM_STAT__GFX_PIPELINE_POWER_STATUS_MASK                                                     0x00000010L
+#define RLC_RLCS_GPM_STAT__CNTX_IDLE_BEING_PROCESSED_MASK                                                     0x00000020L
+#define RLC_RLCS_GPM_STAT__CNTX_BUSY_BEING_PROCESSED_MASK                                                     0x00000040L
+#define RLC_RLCS_GPM_STAT__GFX_IDLE_BEING_PROCESSED_MASK                                                      0x00000080L
+#define RLC_RLCS_GPM_STAT__CMP_BUSY_BEING_PROCESSED_MASK                                                      0x00000100L
+#define RLC_RLCS_GPM_STAT__SAVING_REGISTERS_MASK                                                              0x00000200L
+#define RLC_RLCS_GPM_STAT__RESTORING_REGISTERS_MASK                                                           0x00000400L
+#define RLC_RLCS_GPM_STAT__GFX3D_BLOCKS_CHANGING_POWER_STATE_MASK                                             0x00000800L
+#define RLC_RLCS_GPM_STAT__CMP_BLOCKS_CHANGING_POWER_STATE_MASK                                               0x00001000L
+#define RLC_RLCS_GPM_STAT__STATIC_WGP_POWERING_UP_MASK                                                        0x00002000L
+#define RLC_RLCS_GPM_STAT__STATIC_WGP_POWERING_DOWN_MASK                                                      0x00004000L
+#define RLC_RLCS_GPM_STAT__DYN_WGP_POWERING_UP_MASK                                                           0x00008000L
+#define RLC_RLCS_GPM_STAT__DYN_WGP_POWERING_DOWN_MASK                                                         0x00010000L
+#define RLC_RLCS_GPM_STAT__ABORTED_PD_SEQUENCE_MASK                                                           0x00020000L
+#define RLC_RLCS_GPM_STAT__CMP_POWER_STATUS_MASK                                                              0x00040000L
+#define RLC_RLCS_GPM_STAT__GFX_LS_STATUS_3D_MASK                                                              0x00080000L
+#define RLC_RLCS_GPM_STAT__GFX_CLOCK_STATUS_3D_MASK                                                           0x00100000L
+#define RLC_RLCS_GPM_STAT__MGCG_OVERRIDE_STATUS_MASK                                                          0x00200000L
+#define RLC_RLCS_GPM_STAT__RLC_EXEC_ROM_CODE_MASK                                                             0x00400000L
+#define RLC_RLCS_GPM_STAT__FGCG_OVERRIDE_STATUS_MASK                                                          0x00800000L
+#define RLC_RLCS_GPM_STAT__PG_ERROR_STATUS_MASK                                                               0xFF000000L
+//RLC_RLCS_ABORTED_PD_SEQUENCE
+#define RLC_RLCS_ABORTED_PD_SEQUENCE__APS__SHIFT                                                              0x0
+#define RLC_RLCS_ABORTED_PD_SEQUENCE__RESERVED__SHIFT                                                         0x10
+#define RLC_RLCS_ABORTED_PD_SEQUENCE__APS_MASK                                                                0x0000FFFFL
+#define RLC_RLCS_ABORTED_PD_SEQUENCE__RESERVED_MASK                                                           0xFFFF0000L
+//RLC_RLCS_DIDT_FORCE_STALL
+#define RLC_RLCS_DIDT_FORCE_STALL__DFS__SHIFT                                                                 0x0
+#define RLC_RLCS_DIDT_FORCE_STALL__VALID__SHIFT                                                               0x3
+#define RLC_RLCS_DIDT_FORCE_STALL__RESERVED__SHIFT                                                            0x4
+#define RLC_RLCS_DIDT_FORCE_STALL__DFS_MASK                                                                   0x00000007L
+#define RLC_RLCS_DIDT_FORCE_STALL__VALID_MASK                                                                 0x00000008L
+#define RLC_RLCS_DIDT_FORCE_STALL__RESERVED_MASK                                                              0xFFFFFFF0L
+//RLC_RLCS_IOV_CMD_STATUS
+#define RLC_RLCS_IOV_CMD_STATUS__DATA__SHIFT                                                                  0x0
+#define RLC_RLCS_IOV_CMD_STATUS__DATA_MASK                                                                    0xFFFFFFFFL
+//RLC_RLCS_IOV_CNTX_LOC_SIZE
+#define RLC_RLCS_IOV_CNTX_LOC_SIZE__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IOV_CNTX_LOC_SIZE__RESERVED__SHIFT                                                           0x8
+#define RLC_RLCS_IOV_CNTX_LOC_SIZE__DATA_MASK                                                                 0x000000FFL
+#define RLC_RLCS_IOV_CNTX_LOC_SIZE__RESERVED_MASK                                                             0xFFFFFF00L
+//RLC_RLCS_IOV_SCH_BLOCK
+#define RLC_RLCS_IOV_SCH_BLOCK__DATA__SHIFT                                                                   0x0
+#define RLC_RLCS_IOV_SCH_BLOCK__DATA_MASK                                                                     0xFFFFFFFFL
+//RLC_RLCS_IOV_VM_BUSY_STATUS
+#define RLC_RLCS_IOV_VM_BUSY_STATUS__DATA__SHIFT                                                              0x0
+#define RLC_RLCS_IOV_VM_BUSY_STATUS__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCS_GPM_STAT_2
+#define RLC_RLCS_GPM_STAT_2__TC_TRANS_ERROR__SHIFT                                                            0x0
+#define RLC_RLCS_GPM_STAT_2__RLC_PWR_NON3D_STALLED__SHIFT                                                     0x1
+#define RLC_RLCS_GPM_STAT_2__GFX_PWR_STALLED_STATUS__SHIFT                                                    0x2
+#define RLC_RLCS_GPM_STAT_2__GFX_ULV_STATUS__SHIFT                                                            0x3
+#define RLC_RLCS_GPM_STAT_2__GFX_GENERAL_STATUS__SHIFT                                                        0x4
+#define RLC_RLCS_GPM_STAT_2__RESERVED__SHIFT                                                                  0x5
+#define RLC_RLCS_GPM_STAT_2__TC_TRANS_ERROR_MASK                                                              0x00000001L
+#define RLC_RLCS_GPM_STAT_2__RLC_PWR_NON3D_STALLED_MASK                                                       0x00000002L
+#define RLC_RLCS_GPM_STAT_2__GFX_PWR_STALLED_STATUS_MASK                                                      0x00000004L
+#define RLC_RLCS_GPM_STAT_2__GFX_ULV_STATUS_MASK                                                              0x00000008L
+#define RLC_RLCS_GPM_STAT_2__GFX_GENERAL_STATUS_MASK                                                          0x00000010L
+#define RLC_RLCS_GPM_STAT_2__RESERVED_MASK                                                                    0xFFFFFFE0L
+//RLC_RLCS_GRBM_SOFT_RESET
+#define RLC_RLCS_GRBM_SOFT_RESET__RESET__SHIFT                                                                0x0
+#define RLC_RLCS_GRBM_SOFT_RESET__RESERVED__SHIFT                                                             0x1
+#define RLC_RLCS_GRBM_SOFT_RESET__RESET_MASK                                                                  0x00000001L
+#define RLC_RLCS_GRBM_SOFT_RESET__RESERVED_MASK                                                               0xFFFFFFFEL
+//RLC_RLCS_PG_CHANGE_STATUS
+#define RLC_RLCS_PG_CHANGE_STATUS__PG_CNTL_CHANGED__SHIFT                                                     0x0
+#define RLC_RLCS_PG_CHANGE_STATUS__PG_REG_CHANGED__SHIFT                                                      0x1
+#define RLC_RLCS_PG_CHANGE_STATUS__DYN_PG_STATUS_CHANGED__SHIFT                                               0x2
+#define RLC_RLCS_PG_CHANGE_STATUS__DYN_PG_REQ_CHANGED__SHIFT                                                  0x3
+#define RLC_RLCS_PG_CHANGE_STATUS__RESERVED__SHIFT                                                            0x4
+#define RLC_RLCS_PG_CHANGE_STATUS__PG_CNTL_CHANGED_MASK                                                       0x00000001L
+#define RLC_RLCS_PG_CHANGE_STATUS__PG_REG_CHANGED_MASK                                                        0x00000002L
+#define RLC_RLCS_PG_CHANGE_STATUS__DYN_PG_STATUS_CHANGED_MASK                                                 0x00000004L
+#define RLC_RLCS_PG_CHANGE_STATUS__DYN_PG_REQ_CHANGED_MASK                                                    0x00000008L
+#define RLC_RLCS_PG_CHANGE_STATUS__RESERVED_MASK                                                              0xFFFFFFF0L
+//RLC_RLCS_PG_CHANGE_READ
+#define RLC_RLCS_PG_CHANGE_READ__RESERVED__SHIFT                                                              0x0
+#define RLC_RLCS_PG_CHANGE_READ__PG_REG_CHANGED__SHIFT                                                        0x1
+#define RLC_RLCS_PG_CHANGE_READ__DYN_PG_STATUS_CHANGED__SHIFT                                                 0x2
+#define RLC_RLCS_PG_CHANGE_READ__DYN_PG_REQ_CHANGED__SHIFT                                                    0x3
+#define RLC_RLCS_PG_CHANGE_READ__RESERVED_MASK                                                                0x00000001L
+#define RLC_RLCS_PG_CHANGE_READ__PG_REG_CHANGED_MASK                                                          0x00000002L
+#define RLC_RLCS_PG_CHANGE_READ__DYN_PG_STATUS_CHANGED_MASK                                                   0x00000004L
+#define RLC_RLCS_PG_CHANGE_READ__DYN_PG_REQ_CHANGED_MASK                                                      0x00000008L
+//RLC_RLCS_IH_SEMAPHORE
+#define RLC_RLCS_IH_SEMAPHORE__CLIENT_ID__SHIFT                                                               0x0
+#define RLC_RLCS_IH_SEMAPHORE__RESERVED__SHIFT                                                                0x5
+#define RLC_RLCS_IH_SEMAPHORE__CLIENT_ID_MASK                                                                 0x0000001FL
+#define RLC_RLCS_IH_SEMAPHORE__RESERVED_MASK                                                                  0xFFFFFFE0L
+//RLC_RLCS_IH_COOKIE_SEMAPHORE
+#define RLC_RLCS_IH_COOKIE_SEMAPHORE__CLIENT_ID__SHIFT                                                        0x0
+#define RLC_RLCS_IH_COOKIE_SEMAPHORE__RESERVED__SHIFT                                                         0x5
+#define RLC_RLCS_IH_COOKIE_SEMAPHORE__CLIENT_ID_MASK                                                          0x0000001FL
+#define RLC_RLCS_IH_COOKIE_SEMAPHORE__RESERVED_MASK                                                           0xFFFFFFE0L
+//RLC_RLCS_WGP_STATUS
+#define RLC_RLCS_WGP_STATUS__CS_WORK_ACTIVE__SHIFT                                                            0x0
+#define RLC_RLCS_WGP_STATUS__STATIC_WGP_STATUS_CHANGED__SHIFT                                                 0x1
+#define RLC_RLCS_WGP_STATUS__DYMANIC_WGP_STATUS_CHANGED__SHIFT                                                0x2
+#define RLC_RLCS_WGP_STATUS__STATIC_PERWGP_PD_INCOMPLETE__SHIFT                                               0x3
+#define RLC_RLCS_WGP_STATUS__RESERVED__SHIFT                                                                  0x4
+#define RLC_RLCS_WGP_STATUS__CS_WORK_ACTIVE_MASK                                                              0x00000001L
+#define RLC_RLCS_WGP_STATUS__STATIC_WGP_STATUS_CHANGED_MASK                                                   0x00000002L
+#define RLC_RLCS_WGP_STATUS__DYMANIC_WGP_STATUS_CHANGED_MASK                                                  0x00000004L
+#define RLC_RLCS_WGP_STATUS__STATIC_PERWGP_PD_INCOMPLETE_MASK                                                 0x00000008L
+#define RLC_RLCS_WGP_STATUS__RESERVED_MASK                                                                    0xFFFFFFF0L
+//RLC_RLCS_WGP_READ
+#define RLC_RLCS_WGP_READ__CS_WORK_ACTIVE__SHIFT                                                              0x0
+#define RLC_RLCS_WGP_READ__STATIC_WGP_STATUS_CHANGED__SHIFT                                                   0x1
+#define RLC_RLCS_WGP_READ__DYMANIC_WGP_STATUS_CHANGED__SHIFT                                                  0x2
+#define RLC_RLCS_WGP_READ__RESERVED__SHIFT                                                                    0x3
+#define RLC_RLCS_WGP_READ__CS_WORK_ACTIVE_MASK                                                                0x00000001L
+#define RLC_RLCS_WGP_READ__STATIC_WGP_STATUS_CHANGED_MASK                                                     0x00000002L
+#define RLC_RLCS_WGP_READ__DYMANIC_WGP_STATUS_CHANGED_MASK                                                    0x00000004L
+#define RLC_RLCS_WGP_READ__RESERVED_MASK                                                                      0xFFFFFFF8L
+//RLC_RLCS_CP_INT_CTRL_1
+#define RLC_RLCS_CP_INT_CTRL_1__INTERRUPT_ACK__SHIFT                                                          0x0
+#define RLC_RLCS_CP_INT_CTRL_1__RESERVED__SHIFT                                                               0x1
+#define RLC_RLCS_CP_INT_CTRL_1__INTERRUPT_ACK_MASK                                                            0x00000001L
+#define RLC_RLCS_CP_INT_CTRL_1__RESERVED_MASK                                                                 0xFFFFFFFEL
+//RLC_RLCS_CP_INT_CTRL_2
+#define RLC_RLCS_CP_INT_CTRL_2__IDLE_AUTO_ACK_EN__SHIFT                                                       0x0
+#define RLC_RLCS_CP_INT_CTRL_2__BUSY_AUTO_ACK_EN__SHIFT                                                       0x1
+#define RLC_RLCS_CP_INT_CTRL_2__IDLE_AUTO_ACK_ACTIVE__SHIFT                                                   0x2
+#define RLC_RLCS_CP_INT_CTRL_2__BUSY_AUTO_ACK_ACTIVE__SHIFT                                                   0x3
+#define RLC_RLCS_CP_INT_CTRL_2__INTERRUPT_PENDING__SHIFT                                                      0x4
+#define RLC_RLCS_CP_INT_CTRL_2__RESERVED__SHIFT                                                               0x5
+#define RLC_RLCS_CP_INT_CTRL_2__IDLE_AUTO_ACK_EN_MASK                                                         0x00000001L
+#define RLC_RLCS_CP_INT_CTRL_2__BUSY_AUTO_ACK_EN_MASK                                                         0x00000002L
+#define RLC_RLCS_CP_INT_CTRL_2__IDLE_AUTO_ACK_ACTIVE_MASK                                                     0x00000004L
+#define RLC_RLCS_CP_INT_CTRL_2__BUSY_AUTO_ACK_ACTIVE_MASK                                                     0x00000008L
+#define RLC_RLCS_CP_INT_CTRL_2__INTERRUPT_PENDING_MASK                                                        0x00000010L
+#define RLC_RLCS_CP_INT_CTRL_2__RESERVED_MASK                                                                 0xFFFFFFE0L
+//RLC_RLCS_CP_INT_INFO_1
+#define RLC_RLCS_CP_INT_INFO_1__INTERRUPT_INFO_1__SHIFT                                                       0x0
+#define RLC_RLCS_CP_INT_INFO_1__INTERRUPT_INFO_1_MASK                                                         0xFFFFFFFFL
+//RLC_RLCS_CP_INT_INFO_2
+#define RLC_RLCS_CP_INT_INFO_2__INTERRUPT_INFO_2__SHIFT                                                       0x0
+#define RLC_RLCS_CP_INT_INFO_2__INTERRUPT_ID__SHIFT                                                           0x10
+#define RLC_RLCS_CP_INT_INFO_2__RESERVED__SHIFT                                                               0x19
+#define RLC_RLCS_CP_INT_INFO_2__INTERRUPT_INFO_2_MASK                                                         0x0000FFFFL
+#define RLC_RLCS_CP_INT_INFO_2__INTERRUPT_ID_MASK                                                             0x01FF0000L
+#define RLC_RLCS_CP_INT_INFO_2__RESERVED_MASK                                                                 0xFE000000L
+//RLC_RLCS_SPM_INT_CTRL
+#define RLC_RLCS_SPM_INT_CTRL__INTERRUPT_ACK__SHIFT                                                           0x0
+#define RLC_RLCS_SPM_INT_CTRL__RESERVED__SHIFT                                                                0x1
+#define RLC_RLCS_SPM_INT_CTRL__INTERRUPT_ACK_MASK                                                             0x00000001L
+#define RLC_RLCS_SPM_INT_CTRL__RESERVED_MASK                                                                  0xFFFFFFFEL
+//RLC_RLCS_SPM_INT_INFO_1
+#define RLC_RLCS_SPM_INT_INFO_1__INTERRUPT_INFO_1__SHIFT                                                      0x0
+#define RLC_RLCS_SPM_INT_INFO_1__INTERRUPT_INFO_1_MASK                                                        0xFFFFFFFFL
+//RLC_RLCS_SPM_INT_INFO_2
+#define RLC_RLCS_SPM_INT_INFO_2__INTERRUPT_INFO_2__SHIFT                                                      0x0
+#define RLC_RLCS_SPM_INT_INFO_2__INTERRUPT_ID__SHIFT                                                          0x10
+#define RLC_RLCS_SPM_INT_INFO_2__RESERVED__SHIFT                                                              0x19
+#define RLC_RLCS_SPM_INT_INFO_2__INTERRUPT_INFO_2_MASK                                                        0x0000FFFFL
+#define RLC_RLCS_SPM_INT_INFO_2__INTERRUPT_ID_MASK                                                            0x01FF0000L
+#define RLC_RLCS_SPM_INT_INFO_2__RESERVED_MASK                                                                0xFE000000L
+//RLC_RLCS_DSM_TRIG
+#define RLC_RLCS_DSM_TRIG__START__SHIFT                                                                       0x0
+#define RLC_RLCS_DSM_TRIG__RESERVED__SHIFT                                                                    0x1
+#define RLC_RLCS_DSM_TRIG__START_MASK                                                                         0x00000001L
+#define RLC_RLCS_DSM_TRIG__RESERVED_MASK                                                                      0xFFFFFFFEL
+//RLC_RLCS_BOOTLOAD_STATUS
+#define RLC_RLCS_BOOTLOAD_STATUS__GFX_INIT_DONE__SHIFT                                                        0x0
+#define RLC_RLCS_BOOTLOAD_STATUS__GFX_SECURITY_POLICY_LOADED__SHIFT                                           0x1
+#define RLC_RLCS_BOOTLOAD_STATUS__GFX_SECURITY_POLICY_DONE__SHIFT                                             0x2
+#define RLC_RLCS_BOOTLOAD_STATUS__RLC_GPM_IRAM_LOADED__SHIFT                                                  0x3
+#define RLC_RLCS_BOOTLOAD_STATUS__RLC_GPM_IRAM_DONE__SHIFT                                                    0x4
+#define RLC_RLCS_BOOTLOAD_STATUS__RESERVED__SHIFT                                                             0x5
+#define RLC_RLCS_BOOTLOAD_STATUS__BOOTLOAD_COMPLETE__SHIFT                                                    0x1f
+#define RLC_RLCS_BOOTLOAD_STATUS__GFX_INIT_DONE_MASK                                                          0x00000001L
+#define RLC_RLCS_BOOTLOAD_STATUS__GFX_SECURITY_POLICY_LOADED_MASK                                             0x00000002L
+#define RLC_RLCS_BOOTLOAD_STATUS__GFX_SECURITY_POLICY_DONE_MASK                                               0x00000004L
+#define RLC_RLCS_BOOTLOAD_STATUS__RLC_GPM_IRAM_LOADED_MASK                                                    0x00000008L
+#define RLC_RLCS_BOOTLOAD_STATUS__RLC_GPM_IRAM_DONE_MASK                                                      0x00000010L
+#define RLC_RLCS_BOOTLOAD_STATUS__RESERVED_MASK                                                               0x7FFFFFE0L
+#define RLC_RLCS_BOOTLOAD_STATUS__BOOTLOAD_COMPLETE_MASK                                                      0x80000000L
+//RLC_RLCS_POWER_BRAKE_CNTL
+#define RLC_RLCS_POWER_BRAKE_CNTL__POWER_BRAKE__SHIFT                                                         0x0
+#define RLC_RLCS_POWER_BRAKE_CNTL__INT_CLEAR__SHIFT                                                           0x1
+#define RLC_RLCS_POWER_BRAKE_CNTL__MAX_HYSTERESIS__SHIFT                                                      0x2
+#define RLC_RLCS_POWER_BRAKE_CNTL__HYSTERESIS_CNT__SHIFT                                                      0xa
+#define RLC_RLCS_POWER_BRAKE_CNTL__RESERVED__SHIFT                                                            0x12
+#define RLC_RLCS_POWER_BRAKE_CNTL__POWER_BRAKE_MASK                                                           0x00000001L
+#define RLC_RLCS_POWER_BRAKE_CNTL__INT_CLEAR_MASK                                                             0x00000002L
+#define RLC_RLCS_POWER_BRAKE_CNTL__MAX_HYSTERESIS_MASK                                                        0x000003FCL
+#define RLC_RLCS_POWER_BRAKE_CNTL__HYSTERESIS_CNT_MASK                                                        0x0003FC00L
+#define RLC_RLCS_POWER_BRAKE_CNTL__RESERVED_MASK                                                              0xFFFC0000L
+//RLC_RLCS_POWER_BRAKE_CNTL_TH1
+#define RLC_RLCS_POWER_BRAKE_CNTL_TH1__POWER_BRAKE__SHIFT                                                     0x0
+#define RLC_RLCS_POWER_BRAKE_CNTL_TH1__INT_CLEAR__SHIFT                                                       0x1
+#define RLC_RLCS_POWER_BRAKE_CNTL_TH1__MAX_HYSTERESIS__SHIFT                                                  0x2
+#define RLC_RLCS_POWER_BRAKE_CNTL_TH1__HYSTERESIS_CNT__SHIFT                                                  0xa
+#define RLC_RLCS_POWER_BRAKE_CNTL_TH1__RESERVED__SHIFT                                                        0x12
+#define RLC_RLCS_POWER_BRAKE_CNTL_TH1__POWER_BRAKE_MASK                                                       0x00000001L
+#define RLC_RLCS_POWER_BRAKE_CNTL_TH1__INT_CLEAR_MASK                                                         0x00000002L
+#define RLC_RLCS_POWER_BRAKE_CNTL_TH1__MAX_HYSTERESIS_MASK                                                    0x000003FCL
+#define RLC_RLCS_POWER_BRAKE_CNTL_TH1__HYSTERESIS_CNT_MASK                                                    0x0003FC00L
+#define RLC_RLCS_POWER_BRAKE_CNTL_TH1__RESERVED_MASK                                                          0xFFFC0000L
+//RLC_RLCS_GRBM_IDLE_BUSY_STAT
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__GRBM_RLC_GC_STAT_IDLE__SHIFT                                            0x0
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_0_BUSY__SHIFT                                                      0x10
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_1_BUSY__SHIFT                                                      0x11
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_2_BUSY__SHIFT                                                      0x12
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_3_BUSY__SHIFT                                                      0x13
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_4_BUSY__SHIFT                                                      0x14
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_5_BUSY__SHIFT                                                      0x15
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_6_BUSY__SHIFT                                                      0x16
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_7_BUSY__SHIFT                                                      0x17
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_0_BUSY_CHANGED__SHIFT                                              0x18
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_1_BUSY_CHANGED__SHIFT                                              0x19
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_2_BUSY_CHANGED__SHIFT                                              0x1a
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_3_BUSY_CHANGED__SHIFT                                              0x1b
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_4_BUSY_CHANGED__SHIFT                                              0x1c
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_5_BUSY_CHANGED__SHIFT                                              0x1d
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_6_BUSY_CHANGED__SHIFT                                              0x1e
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_7_BUSY_CHANGED__SHIFT                                              0x1f
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__GRBM_RLC_GC_STAT_IDLE_MASK                                              0x00000003L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_0_BUSY_MASK                                                        0x00010000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_1_BUSY_MASK                                                        0x00020000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_2_BUSY_MASK                                                        0x00040000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_3_BUSY_MASK                                                        0x00080000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_4_BUSY_MASK                                                        0x00100000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_5_BUSY_MASK                                                        0x00200000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_6_BUSY_MASK                                                        0x00400000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_7_BUSY_MASK                                                        0x00800000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_0_BUSY_CHANGED_MASK                                                0x01000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_1_BUSY_CHANGED_MASK                                                0x02000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_2_BUSY_CHANGED_MASK                                                0x04000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_3_BUSY_CHANGED_MASK                                                0x08000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_4_BUSY_CHANGED_MASK                                                0x10000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_5_BUSY_CHANGED_MASK                                                0x20000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_6_BUSY_CHANGED_MASK                                                0x40000000L
+#define RLC_RLCS_GRBM_IDLE_BUSY_STAT__SDMA_7_BUSY_CHANGED_MASK                                                0x80000000L
+//RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA0_BUSY_INT_CLEAR__SHIFT                                         0x0
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA1_BUSY_INT_CLEAR__SHIFT                                         0x1
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA2_BUSY_INT_CLEAR__SHIFT                                         0x2
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA3_BUSY_INT_CLEAR__SHIFT                                         0x3
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA4_BUSY_INT_CLEAR__SHIFT                                         0x4
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA5_BUSY_INT_CLEAR__SHIFT                                         0x5
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA6_BUSY_INT_CLEAR__SHIFT                                         0x6
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA7_BUSY_INT_CLEAR__SHIFT                                         0x7
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA0_BUSY_INT_CLEAR_MASK                                           0x00000001L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA1_BUSY_INT_CLEAR_MASK                                           0x00000002L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA2_BUSY_INT_CLEAR_MASK                                           0x00000004L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA3_BUSY_INT_CLEAR_MASK                                           0x00000008L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA4_BUSY_INT_CLEAR_MASK                                           0x00000010L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA5_BUSY_INT_CLEAR_MASK                                           0x00000020L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA6_BUSY_INT_CLEAR_MASK                                           0x00000040L
+#define RLC_RLCS_GRBM_IDLE_BUSY_INT_CNTL__SDMA7_BUSY_INT_CLEAR_MASK                                           0x00000080L
+//RLC_RLCS_CMP_IDLE_CNTL
+#define RLC_RLCS_CMP_IDLE_CNTL__INT_CLEAR__SHIFT                                                              0x0
+#define RLC_RLCS_CMP_IDLE_CNTL__CMP_IDLE_HYST__SHIFT                                                          0x1
+#define RLC_RLCS_CMP_IDLE_CNTL__CMP_IDLE__SHIFT                                                               0x2
+#define RLC_RLCS_CMP_IDLE_CNTL__MAX_HYSTERESIS__SHIFT                                                         0x3
+#define RLC_RLCS_CMP_IDLE_CNTL__HYSTERESIS_CNT__SHIFT                                                         0xb
+#define RLC_RLCS_CMP_IDLE_CNTL__RESERVED__SHIFT                                                               0x13
+#define RLC_RLCS_CMP_IDLE_CNTL__INT_CLEAR_MASK                                                                0x00000001L
+#define RLC_RLCS_CMP_IDLE_CNTL__CMP_IDLE_HYST_MASK                                                            0x00000002L
+#define RLC_RLCS_CMP_IDLE_CNTL__CMP_IDLE_MASK                                                                 0x00000004L
+#define RLC_RLCS_CMP_IDLE_CNTL__MAX_HYSTERESIS_MASK                                                           0x000007F8L
+#define RLC_RLCS_CMP_IDLE_CNTL__HYSTERESIS_CNT_MASK                                                           0x0007F800L
+#define RLC_RLCS_CMP_IDLE_CNTL__RESERVED_MASK                                                                 0xFFF80000L
+//RLC_RLCS_GENERAL_0
+#define RLC_RLCS_GENERAL_0__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_0__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_1
+#define RLC_RLCS_GENERAL_1__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_1__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_2
+#define RLC_RLCS_GENERAL_2__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_2__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_3
+#define RLC_RLCS_GENERAL_3__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_3__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_4
+#define RLC_RLCS_GENERAL_4__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_4__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_5
+#define RLC_RLCS_GENERAL_5__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_5__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_6
+#define RLC_RLCS_GENERAL_6__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_6__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_7
+#define RLC_RLCS_GENERAL_7__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_7__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_8
+#define RLC_RLCS_GENERAL_8__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_8__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_9
+#define RLC_RLCS_GENERAL_9__DATA__SHIFT                                                                       0x0
+#define RLC_RLCS_GENERAL_9__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_RLCS_GENERAL_10
+#define RLC_RLCS_GENERAL_10__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_10__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_11
+#define RLC_RLCS_GENERAL_11__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_11__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_12
+#define RLC_RLCS_GENERAL_12__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_12__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_13
+#define RLC_RLCS_GENERAL_13__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_13__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_14
+#define RLC_RLCS_GENERAL_14__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_14__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_15
+#define RLC_RLCS_GENERAL_15__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_15__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_GENERAL_16
+#define RLC_RLCS_GENERAL_16__DATA__SHIFT                                                                      0x0
+#define RLC_RLCS_GENERAL_16__DATA_MASK                                                                        0xFFFFFFFFL
+//RLC_RLCS_AUXILIARY_REG_1
+#define RLC_RLCS_AUXILIARY_REG_1__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_AUXILIARY_REG_1__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_AUXILIARY_REG_1__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_AUXILIARY_REG_1__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_AUXILIARY_REG_2
+#define RLC_RLCS_AUXILIARY_REG_2__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_AUXILIARY_REG_2__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_AUXILIARY_REG_2__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_AUXILIARY_REG_2__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_AUXILIARY_REG_3
+#define RLC_RLCS_AUXILIARY_REG_3__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_AUXILIARY_REG_3__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_AUXILIARY_REG_3__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_AUXILIARY_REG_3__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_AUXILIARY_REG_4
+#define RLC_RLCS_AUXILIARY_REG_4__ADDR__SHIFT                                                                 0x0
+#define RLC_RLCS_AUXILIARY_REG_4__RESERVED__SHIFT                                                             0x12
+#define RLC_RLCS_AUXILIARY_REG_4__ADDR_MASK                                                                   0x0003FFFFL
+#define RLC_RLCS_AUXILIARY_REG_4__RESERVED_MASK                                                               0xFFFC0000L
+//RLC_RLCS_SPM_SQTT_MODE
+#define RLC_RLCS_SPM_SQTT_MODE__MODE__SHIFT                                                                   0x0
+#define RLC_RLCS_SPM_SQTT_MODE__MODE_MASK                                                                     0x00000001L
+//RLC_RLCS_CP_DMA_SRCID_OVER
+#define RLC_RLCS_CP_DMA_SRCID_OVER__SRCID_OVERRIDE__SHIFT                                                     0x0
+#define RLC_RLCS_CP_DMA_SRCID_OVER__SRCID_OVERRIDE_MASK                                                       0x00000001L
+//RLC_RLCS_BOOTLOAD_ID_STATUS1
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_0_LOADED__SHIFT                                                      0x0
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_1_LOADED__SHIFT                                                      0x1
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_2_LOADED__SHIFT                                                      0x2
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_3_LOADED__SHIFT                                                      0x3
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_4_LOADED__SHIFT                                                      0x4
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_5_LOADED__SHIFT                                                      0x5
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_6_LOADED__SHIFT                                                      0x6
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_7_LOADED__SHIFT                                                      0x7
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_8_LOADED__SHIFT                                                      0x8
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_9_LOADED__SHIFT                                                      0x9
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_10_LOADED__SHIFT                                                     0xa
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_11_LOADED__SHIFT                                                     0xb
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_12_LOADED__SHIFT                                                     0xc
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_13_LOADED__SHIFT                                                     0xd
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_14_LOADED__SHIFT                                                     0xe
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_15_LOADED__SHIFT                                                     0xf
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_16_LOADED__SHIFT                                                     0x10
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_17_LOADED__SHIFT                                                     0x11
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_18_LOADED__SHIFT                                                     0x12
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_19_LOADED__SHIFT                                                     0x13
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_20_LOADED__SHIFT                                                     0x14
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_21_LOADED__SHIFT                                                     0x15
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_22_LOADED__SHIFT                                                     0x16
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_23_LOADED__SHIFT                                                     0x17
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_24_LOADED__SHIFT                                                     0x18
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_25_LOADED__SHIFT                                                     0x19
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_26_LOADED__SHIFT                                                     0x1a
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_27_LOADED__SHIFT                                                     0x1b
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_28_LOADED__SHIFT                                                     0x1c
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_29_LOADED__SHIFT                                                     0x1d
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_30_LOADED__SHIFT                                                     0x1e
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_31_LOADED__SHIFT                                                     0x1f
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_0_LOADED_MASK                                                        0x00000001L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_1_LOADED_MASK                                                        0x00000002L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_2_LOADED_MASK                                                        0x00000004L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_3_LOADED_MASK                                                        0x00000008L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_4_LOADED_MASK                                                        0x00000010L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_5_LOADED_MASK                                                        0x00000020L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_6_LOADED_MASK                                                        0x00000040L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_7_LOADED_MASK                                                        0x00000080L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_8_LOADED_MASK                                                        0x00000100L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_9_LOADED_MASK                                                        0x00000200L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_10_LOADED_MASK                                                       0x00000400L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_11_LOADED_MASK                                                       0x00000800L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_12_LOADED_MASK                                                       0x00001000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_13_LOADED_MASK                                                       0x00002000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_14_LOADED_MASK                                                       0x00004000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_15_LOADED_MASK                                                       0x00008000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_16_LOADED_MASK                                                       0x00010000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_17_LOADED_MASK                                                       0x00020000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_18_LOADED_MASK                                                       0x00040000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_19_LOADED_MASK                                                       0x00080000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_20_LOADED_MASK                                                       0x00100000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_21_LOADED_MASK                                                       0x00200000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_22_LOADED_MASK                                                       0x00400000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_23_LOADED_MASK                                                       0x00800000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_24_LOADED_MASK                                                       0x01000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_25_LOADED_MASK                                                       0x02000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_26_LOADED_MASK                                                       0x04000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_27_LOADED_MASK                                                       0x08000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_28_LOADED_MASK                                                       0x10000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_29_LOADED_MASK                                                       0x20000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_30_LOADED_MASK                                                       0x40000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS1__ID_31_LOADED_MASK                                                       0x80000000L
+//RLC_RLCS_BOOTLOAD_ID_STATUS2
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_32_LOADED__SHIFT                                                     0x0
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_33_LOADED__SHIFT                                                     0x1
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_34_LOADED__SHIFT                                                     0x2
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_35_LOADED__SHIFT                                                     0x3
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_36_LOADED__SHIFT                                                     0x4
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_37_LOADED__SHIFT                                                     0x5
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_38_LOADED__SHIFT                                                     0x6
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_39_LOADED__SHIFT                                                     0x7
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_40_LOADED__SHIFT                                                     0x8
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_41_LOADED__SHIFT                                                     0x9
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_42_LOADED__SHIFT                                                     0xa
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_43_LOADED__SHIFT                                                     0xb
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_44_LOADED__SHIFT                                                     0xc
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_45_LOADED__SHIFT                                                     0xd
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_46_LOADED__SHIFT                                                     0xe
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_47_LOADED__SHIFT                                                     0xf
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_48_LOADED__SHIFT                                                     0x10
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_49_LOADED__SHIFT                                                     0x11
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_50_LOADED__SHIFT                                                     0x12
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_51_LOADED__SHIFT                                                     0x13
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_52_LOADED__SHIFT                                                     0x14
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_53_LOADED__SHIFT                                                     0x15
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_54_LOADED__SHIFT                                                     0x16
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_55_LOADED__SHIFT                                                     0x17
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_56_LOADED__SHIFT                                                     0x18
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_57_LOADED__SHIFT                                                     0x19
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_58_LOADED__SHIFT                                                     0x1a
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_59_LOADED__SHIFT                                                     0x1b
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_60_LOADED__SHIFT                                                     0x1c
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_61_LOADED__SHIFT                                                     0x1d
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_62_LOADED__SHIFT                                                     0x1e
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_63_LOADED__SHIFT                                                     0x1f
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_32_LOADED_MASK                                                       0x00000001L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_33_LOADED_MASK                                                       0x00000002L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_34_LOADED_MASK                                                       0x00000004L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_35_LOADED_MASK                                                       0x00000008L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_36_LOADED_MASK                                                       0x00000010L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_37_LOADED_MASK                                                       0x00000020L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_38_LOADED_MASK                                                       0x00000040L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_39_LOADED_MASK                                                       0x00000080L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_40_LOADED_MASK                                                       0x00000100L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_41_LOADED_MASK                                                       0x00000200L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_42_LOADED_MASK                                                       0x00000400L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_43_LOADED_MASK                                                       0x00000800L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_44_LOADED_MASK                                                       0x00001000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_45_LOADED_MASK                                                       0x00002000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_46_LOADED_MASK                                                       0x00004000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_47_LOADED_MASK                                                       0x00008000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_48_LOADED_MASK                                                       0x00010000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_49_LOADED_MASK                                                       0x00020000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_50_LOADED_MASK                                                       0x00040000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_51_LOADED_MASK                                                       0x00080000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_52_LOADED_MASK                                                       0x00100000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_53_LOADED_MASK                                                       0x00200000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_54_LOADED_MASK                                                       0x00400000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_55_LOADED_MASK                                                       0x00800000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_56_LOADED_MASK                                                       0x01000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_57_LOADED_MASK                                                       0x02000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_58_LOADED_MASK                                                       0x04000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_59_LOADED_MASK                                                       0x08000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_60_LOADED_MASK                                                       0x10000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_61_LOADED_MASK                                                       0x20000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_62_LOADED_MASK                                                       0x40000000L
+#define RLC_RLCS_BOOTLOAD_ID_STATUS2__ID_63_LOADED_MASK                                                       0x80000000L
+//RLC_RLCS_IMU_VIDCHG_CNTL
+#define RLC_RLCS_IMU_VIDCHG_CNTL__REQ__SHIFT                                                                  0x0
+#define RLC_RLCS_IMU_VIDCHG_CNTL__DATA__SHIFT                                                                 0x1
+#define RLC_RLCS_IMU_VIDCHG_CNTL__PSIEN__SHIFT                                                                0xa
+#define RLC_RLCS_IMU_VIDCHG_CNTL__ACK__SHIFT                                                                  0xb
+#define RLC_RLCS_IMU_VIDCHG_CNTL__RESERVED__SHIFT                                                             0xc
+#define RLC_RLCS_IMU_VIDCHG_CNTL__REQ_MASK                                                                    0x00000001L
+#define RLC_RLCS_IMU_VIDCHG_CNTL__DATA_MASK                                                                   0x000003FEL
+#define RLC_RLCS_IMU_VIDCHG_CNTL__PSIEN_MASK                                                                  0x00000400L
+#define RLC_RLCS_IMU_VIDCHG_CNTL__ACK_MASK                                                                    0x00000800L
+#define RLC_RLCS_IMU_VIDCHG_CNTL__RESERVED_MASK                                                               0xFFFFF000L
+//RLC_RLCS_EDC_INT_CNTL
+#define RLC_RLCS_EDC_INT_CNTL__EDC_EVENT_INT_CLEAR__SHIFT                                                     0x0
+#define RLC_RLCS_EDC_INT_CNTL__EDC_EVENT_INT_CLEAR_MASK                                                       0x00000001L
+//RLC_RLCS_KMD_LOG_CNTL1
+#define RLC_RLCS_KMD_LOG_CNTL1__DATA__SHIFT                                                                   0x0
+#define RLC_RLCS_KMD_LOG_CNTL1__DATA_MASK                                                                     0xFFFFFFFFL
+//RLC_RLCS_KMD_LOG_CNTL2
+#define RLC_RLCS_KMD_LOG_CNTL2__DATA__SHIFT                                                                   0x0
+#define RLC_RLCS_KMD_LOG_CNTL2__DATA_MASK                                                                     0xFFFFFFFFL
+//RLC_RLCS_GPM_LEGACY_INT_STAT
+#define RLC_RLCS_GPM_LEGACY_INT_STAT__GC_CAC_EDC_EVENT_CHANGED__SHIFT                                         0x0
+#define RLC_RLCS_GPM_LEGACY_INT_STAT__GFX_POWER_BRAKE_CHANGED__SHIFT                                          0x1
+#define RLC_RLCS_GPM_LEGACY_INT_STAT__GC_CAC_EDC_EVENT_CHANGED_MASK                                           0x00000001L
+#define RLC_RLCS_GPM_LEGACY_INT_STAT__GFX_POWER_BRAKE_CHANGED_MASK                                            0x00000002L
+//RLC_RLCS_GPM_LEGACY_INT_DISABLE
+#define RLC_RLCS_GPM_LEGACY_INT_DISABLE__GC_CAC_EDC_EVENT_CHANGED__SHIFT                                      0x0
+#define RLC_RLCS_GPM_LEGACY_INT_DISABLE__GFX_POWER_BRAKE_CHANGED__SHIFT                                       0x1
+#define RLC_RLCS_GPM_LEGACY_INT_DISABLE__GC_CAC_EDC_EVENT_CHANGED_MASK                                        0x00000001L
+#define RLC_RLCS_GPM_LEGACY_INT_DISABLE__GFX_POWER_BRAKE_CHANGED_MASK                                         0x00000002L
+//RLC_RLCS_SRM_SRCID_CNTL
+#define RLC_RLCS_SRM_SRCID_CNTL__SRCID__SHIFT                                                                 0x0
+#define RLC_RLCS_SRM_SRCID_CNTL__SRCID_MASK                                                                   0x00000007L
+//RLC_RLCS_GCR_DATA_0
+#define RLC_RLCS_GCR_DATA_0__PHASE_0__SHIFT                                                                   0x0
+#define RLC_RLCS_GCR_DATA_0__PHASE_1__SHIFT                                                                   0x10
+#define RLC_RLCS_GCR_DATA_0__PHASE_0_MASK                                                                     0x0000FFFFL
+#define RLC_RLCS_GCR_DATA_0__PHASE_1_MASK                                                                     0xFFFF0000L
+//RLC_RLCS_GCR_DATA_1
+#define RLC_RLCS_GCR_DATA_1__PHASE_2__SHIFT                                                                   0x0
+#define RLC_RLCS_GCR_DATA_1__PHASE_3__SHIFT                                                                   0x10
+#define RLC_RLCS_GCR_DATA_1__PHASE_2_MASK                                                                     0x0000FFFFL
+#define RLC_RLCS_GCR_DATA_1__PHASE_3_MASK                                                                     0xFFFF0000L
+//RLC_RLCS_GCR_DATA_2
+#define RLC_RLCS_GCR_DATA_2__PHASE_4__SHIFT                                                                   0x0
+#define RLC_RLCS_GCR_DATA_2__PHASE_5__SHIFT                                                                   0x10
+#define RLC_RLCS_GCR_DATA_2__PHASE_4_MASK                                                                     0x0000FFFFL
+#define RLC_RLCS_GCR_DATA_2__PHASE_5_MASK                                                                     0xFFFF0000L
+//RLC_RLCS_GCR_DATA_3
+#define RLC_RLCS_GCR_DATA_3__PHASE_6__SHIFT                                                                   0x0
+#define RLC_RLCS_GCR_DATA_3__PHASE_7__SHIFT                                                                   0x10
+#define RLC_RLCS_GCR_DATA_3__PHASE_6_MASK                                                                     0x0000FFFFL
+#define RLC_RLCS_GCR_DATA_3__PHASE_7_MASK                                                                     0xFFFF0000L
+//RLC_RLCS_GCR_STATUS
+#define RLC_RLCS_GCR_STATUS__GCR_BUSY__SHIFT                                                                  0x0
+#define RLC_RLCS_GCR_STATUS__GCR_OUT_COUNT__SHIFT                                                             0x1
+#define RLC_RLCS_GCR_STATUS__RESERVED_2__SHIFT                                                                0x5
+#define RLC_RLCS_GCR_STATUS__GCRIU_CLI_RSP_TAG__SHIFT                                                         0x8
+#define RLC_RLCS_GCR_STATUS__RESERVED__SHIFT                                                                  0x10
+#define RLC_RLCS_GCR_STATUS__GCR_BUSY_MASK                                                                    0x00000001L
+#define RLC_RLCS_GCR_STATUS__GCR_OUT_COUNT_MASK                                                               0x0000001EL
+#define RLC_RLCS_GCR_STATUS__RESERVED_2_MASK                                                                  0x000000E0L
+#define RLC_RLCS_GCR_STATUS__GCRIU_CLI_RSP_TAG_MASK                                                           0x0000FF00L
+#define RLC_RLCS_GCR_STATUS__RESERVED_MASK                                                                    0xFFFF0000L
+//RLC_RLCS_PERFMON_CLK_CNTL_UCODE
+#define RLC_RLCS_PERFMON_CLK_CNTL_UCODE__PERFMON_CLOCK_STATE__SHIFT                                           0x0
+#define RLC_RLCS_PERFMON_CLK_CNTL_UCODE__PERFMON_CLOCK_STATE_MASK                                             0x00000001L
+//RLC_RLCS_UTCL2_CNTL
+#define RLC_RLCS_UTCL2_CNTL__MTYPE_NO_PTE_MODE__SHIFT                                                         0x0
+#define RLC_RLCS_UTCL2_CNTL__GPA_OVERRIDE__SHIFT                                                              0x1
+#define RLC_RLCS_UTCL2_CNTL__VF_OVERRIDE__SHIFT                                                               0x2
+#define RLC_RLCS_UTCL2_CNTL__GPA_OVERRIDE_VALUE__SHIFT                                                        0x3
+#define RLC_RLCS_UTCL2_CNTL__VF_OVERRIDE_VALUE__SHIFT                                                         0x5
+#define RLC_RLCS_UTCL2_CNTL__IGNORE_PTE_PERMISSION__SHIFT                                                     0x6
+#define RLC_RLCS_UTCL2_CNTL__RESERVED__SHIFT                                                                  0x7
+#define RLC_RLCS_UTCL2_CNTL__MTYPE_NO_PTE_MODE_MASK                                                           0x00000001L
+#define RLC_RLCS_UTCL2_CNTL__GPA_OVERRIDE_MASK                                                                0x00000002L
+#define RLC_RLCS_UTCL2_CNTL__VF_OVERRIDE_MASK                                                                 0x00000004L
+#define RLC_RLCS_UTCL2_CNTL__GPA_OVERRIDE_VALUE_MASK                                                          0x00000018L
+#define RLC_RLCS_UTCL2_CNTL__VF_OVERRIDE_VALUE_MASK                                                           0x00000020L
+#define RLC_RLCS_UTCL2_CNTL__IGNORE_PTE_PERMISSION_MASK                                                       0x00000040L
+#define RLC_RLCS_UTCL2_CNTL__RESERVED_MASK                                                                    0xFFFFFF80L
+//RLC_RLCS_IMU_RLC_MSG_DATA0
+#define RLC_RLCS_IMU_RLC_MSG_DATA0__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MSG_DATA0__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_DATA1
+#define RLC_RLCS_IMU_RLC_MSG_DATA1__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MSG_DATA1__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_DATA2
+#define RLC_RLCS_IMU_RLC_MSG_DATA2__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MSG_DATA2__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_DATA3
+#define RLC_RLCS_IMU_RLC_MSG_DATA3__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MSG_DATA3__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_DATA4
+#define RLC_RLCS_IMU_RLC_MSG_DATA4__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MSG_DATA4__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_CONTROL
+#define RLC_RLCS_IMU_RLC_MSG_CONTROL__DATA__SHIFT                                                             0x0
+#define RLC_RLCS_IMU_RLC_MSG_CONTROL__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_RLCS_IMU_RLC_MSG_CNTL
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__DONETOG__SHIFT                                                             0x0
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__CHGTOG__SHIFT                                                              0x1
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__RESERVED__SHIFT                                                            0x2
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__DONETOG_MASK                                                               0x00000001L
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__CHGTOG_MASK                                                                0x00000002L
+#define RLC_RLCS_IMU_RLC_MSG_CNTL__RESERVED_MASK                                                              0xFFFFFFFCL
+//RLC_RLCS_RLC_IMU_MSG_DATA0
+#define RLC_RLCS_RLC_IMU_MSG_DATA0__DATA__SHIFT                                                               0x0
+#define RLC_RLCS_RLC_IMU_MSG_DATA0__DATA_MASK                                                                 0xFFFFFFFFL
+//RLC_RLCS_RLC_IMU_MSG_CONTROL
+#define RLC_RLCS_RLC_IMU_MSG_CONTROL__DATA__SHIFT                                                             0x0
+#define RLC_RLCS_RLC_IMU_MSG_CONTROL__DATA_MASK                                                               0xFFFFFFFFL
+//RLC_RLCS_RLC_IMU_MSG_CNTL
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__CHGTOG__SHIFT                                                              0x0
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__DONETOG__SHIFT                                                             0x1
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__RESERVED__SHIFT                                                            0x2
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__CHGTOG_MASK                                                                0x00000001L
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__DONETOG_MASK                                                               0x00000002L
+#define RLC_RLCS_RLC_IMU_MSG_CNTL__RESERVED_MASK                                                              0xFFFFFFFCL
+//RLC_RLCS_IMU_RLC_TELEMETRY_DATA_0
+#define RLC_RLCS_IMU_RLC_TELEMETRY_DATA_0__CURRENT__SHIFT                                                     0x0
+#define RLC_RLCS_IMU_RLC_TELEMETRY_DATA_0__VOLTAGE__SHIFT                                                     0x10
+#define RLC_RLCS_IMU_RLC_TELEMETRY_DATA_0__CURRENT_MASK                                                       0x0000FFFFL
+#define RLC_RLCS_IMU_RLC_TELEMETRY_DATA_0__VOLTAGE_MASK                                                       0xFFFF0000L
+//RLC_RLCS_IMU_RLC_TELEMETRY_DATA_1
+#define RLC_RLCS_IMU_RLC_TELEMETRY_DATA_1__TEMPERATURE1__SHIFT                                                0x0
+#define RLC_RLCS_IMU_RLC_TELEMETRY_DATA_1__RESERVED__SHIFT                                                    0x10
+#define RLC_RLCS_IMU_RLC_TELEMETRY_DATA_1__TEMPERATURE1_MASK                                                  0x0000FFFFL
+#define RLC_RLCS_IMU_RLC_TELEMETRY_DATA_1__RESERVED_MASK                                                      0xFFFF0000L
+//RLC_RLCS_IMU_RLC_MUTEX_CNTL
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__REQ__SHIFT                                                               0x0
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__ACQUIRE__SHIFT                                                           0x1
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__RESERVED__SHIFT                                                          0x2
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__REQ_MASK                                                                 0x00000001L
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__ACQUIRE_MASK                                                             0x00000002L
+#define RLC_RLCS_IMU_RLC_MUTEX_CNTL__RESERVED_MASK                                                            0xFFFFFFFCL
+//RLC_RLCS_IMU_RLC_STATUS
+#define RLC_RLCS_IMU_RLC_STATUS__ALLOW_GFXOFF__SHIFT                                                          0x0
+#define RLC_RLCS_IMU_RLC_STATUS__ALLOW_FA_DCS__SHIFT                                                          0x1
+#define RLC_RLCS_IMU_RLC_STATUS__RESERVED_14_2__SHIFT                                                         0x2
+#define RLC_RLCS_IMU_RLC_STATUS__DISABLE_GFXCLK_DS__SHIFT                                                     0xf
+#define RLC_RLCS_IMU_RLC_STATUS__RESERVED__SHIFT                                                              0x10
+#define RLC_RLCS_IMU_RLC_STATUS__ALLOW_GFXOFF_MASK                                                            0x00000001L
+#define RLC_RLCS_IMU_RLC_STATUS__ALLOW_FA_DCS_MASK                                                            0x00000002L
+#define RLC_RLCS_IMU_RLC_STATUS__RESERVED_14_2_MASK                                                           0x00007FFCL
+#define RLC_RLCS_IMU_RLC_STATUS__DISABLE_GFXCLK_DS_MASK                                                       0x00008000L
+#define RLC_RLCS_IMU_RLC_STATUS__RESERVED_MASK                                                                0xFFFF0000L
+//RLC_RLCS_RLC_IMU_STATUS
+#define RLC_RLCS_RLC_IMU_STATUS__PWR_DOWN_ACTIVE__SHIFT                                                       0x0
+#define RLC_RLCS_RLC_IMU_STATUS__RLC_ALIVE__SHIFT                                                             0x1
+#define RLC_RLCS_RLC_IMU_STATUS__RESERVED_3_2__SHIFT                                                          0x2
+#define RLC_RLCS_RLC_IMU_STATUS__RESERVED__SHIFT                                                              0x4
+#define RLC_RLCS_RLC_IMU_STATUS__PWR_DOWN_ACTIVE_MASK                                                         0x00000001L
+#define RLC_RLCS_RLC_IMU_STATUS__RLC_ALIVE_MASK                                                               0x00000002L
+#define RLC_RLCS_RLC_IMU_STATUS__RESERVED_3_2_MASK                                                            0x0000000CL
+#define RLC_RLCS_RLC_IMU_STATUS__RESERVED_MASK                                                                0xFFFFFFF0L
+//RLC_RLCS_IMU_RAM_DATA_1
+#define RLC_RLCS_IMU_RAM_DATA_1__DATA__SHIFT                                                                  0x0
+#define RLC_RLCS_IMU_RAM_DATA_1__DATA_MASK                                                                    0xFFFFFFFFL
+//RLC_RLCS_IMU_RAM_ADDR_1_LSB
+#define RLC_RLCS_IMU_RAM_ADDR_1_LSB__DATA__SHIFT                                                              0x0
+#define RLC_RLCS_IMU_RAM_ADDR_1_LSB__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCS_IMU_RAM_ADDR_1_MSB
+#define RLC_RLCS_IMU_RAM_ADDR_1_MSB__DATA__SHIFT                                                              0x0
+#define RLC_RLCS_IMU_RAM_ADDR_1_MSB__RESERVED__SHIFT                                                          0x10
+#define RLC_RLCS_IMU_RAM_ADDR_1_MSB__DATA_MASK                                                                0x0000FFFFL
+#define RLC_RLCS_IMU_RAM_ADDR_1_MSB__RESERVED_MASK                                                            0xFFFF0000L
+//RLC_RLCS_IMU_RAM_DATA_0
+#define RLC_RLCS_IMU_RAM_DATA_0__DATA__SHIFT                                                                  0x0
+#define RLC_RLCS_IMU_RAM_DATA_0__DATA_MASK                                                                    0xFFFFFFFFL
+//RLC_RLCS_IMU_RAM_ADDR_0_LSB
+#define RLC_RLCS_IMU_RAM_ADDR_0_LSB__DATA__SHIFT                                                              0x0
+#define RLC_RLCS_IMU_RAM_ADDR_0_LSB__DATA_MASK                                                                0xFFFFFFFFL
+//RLC_RLCS_IMU_RAM_ADDR_0_MSB
+#define RLC_RLCS_IMU_RAM_ADDR_0_MSB__DATA__SHIFT                                                              0x0
+#define RLC_RLCS_IMU_RAM_ADDR_0_MSB__RESERVED__SHIFT                                                          0x10
+#define RLC_RLCS_IMU_RAM_ADDR_0_MSB__DATA_MASK                                                                0x0000FFFFL
+#define RLC_RLCS_IMU_RAM_ADDR_0_MSB__RESERVED_MASK                                                            0xFFFF0000L
+//RLC_RLCS_IMU_RAM_CNTL
+#define RLC_RLCS_IMU_RAM_CNTL__REQTOG__SHIFT                                                                  0x0
+#define RLC_RLCS_IMU_RAM_CNTL__ACKTOG__SHIFT                                                                  0x1
+#define RLC_RLCS_IMU_RAM_CNTL__RESERVED__SHIFT                                                                0x2
+#define RLC_RLCS_IMU_RAM_CNTL__REQTOG_MASK                                                                    0x00000001L
+#define RLC_RLCS_IMU_RAM_CNTL__ACKTOG_MASK                                                                    0x00000002L
+#define RLC_RLCS_IMU_RAM_CNTL__RESERVED_MASK                                                                  0xFFFFFFFCL
+//RLC_RLCS_IMU_GFX_DOORBELL_FENCE
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__ENABLE__SHIFT                                                        0x0
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__ACK__SHIFT                                                           0x1
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__RESERVED__SHIFT                                                      0x2
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__ENABLE_MASK                                                          0x00000001L
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__ACK_MASK                                                             0x00000002L
+#define RLC_RLCS_IMU_GFX_DOORBELL_FENCE__RESERVED_MASK                                                        0xFFFFFFFCL
+//RLC_RLCS_SDMA_INT_CNTL_1
+#define RLC_RLCS_SDMA_INT_CNTL_1__INTERRUPT_ACK__SHIFT                                                        0x0
+#define RLC_RLCS_SDMA_INT_CNTL_1__RESP_ID__SHIFT                                                              0x1
+#define RLC_RLCS_SDMA_INT_CNTL_1__RESERVED__SHIFT                                                             0x2
+#define RLC_RLCS_SDMA_INT_CNTL_1__INTERRUPT_ACK_MASK                                                          0x00000001L
+#define RLC_RLCS_SDMA_INT_CNTL_1__RESP_ID_MASK                                                                0x00000002L
+#define RLC_RLCS_SDMA_INT_CNTL_1__RESERVED_MASK                                                               0xFFFFFFFCL
+//RLC_RLCS_SDMA_INT_CNTL_2
+#define RLC_RLCS_SDMA_INT_CNTL_2__AUTO_ACK_EN__SHIFT                                                          0x0
+#define RLC_RLCS_SDMA_INT_CNTL_2__AUTO_ACK_ACTIVE__SHIFT                                                      0x1
+#define RLC_RLCS_SDMA_INT_CNTL_2__RESERVED__SHIFT                                                             0x2
+#define RLC_RLCS_SDMA_INT_CNTL_2__AUTO_ACK_EN_MASK                                                            0x00000001L
+#define RLC_RLCS_SDMA_INT_CNTL_2__AUTO_ACK_ACTIVE_MASK                                                        0x00000002L
+#define RLC_RLCS_SDMA_INT_CNTL_2__RESERVED_MASK                                                               0xFFFFFFFCL
+//RLC_RLCS_SDMA_INT_STAT
+#define RLC_RLCS_SDMA_INT_STAT__REQ_IDLE_HIST__SHIFT                                                          0x0
+#define RLC_RLCS_SDMA_INT_STAT__REQ_BUSY_HIST__SHIFT                                                          0x8
+#define RLC_RLCS_SDMA_INT_STAT__LAST_SDMA_RLC_INT_ID__SHIFT                                                   0x10
+#define RLC_RLCS_SDMA_INT_STAT__SDMA_RLC_INT_PENDING__SHIFT                                                   0x11
+#define RLC_RLCS_SDMA_INT_STAT__RESERVED__SHIFT                                                               0x12
+#define RLC_RLCS_SDMA_INT_STAT__REQ_IDLE_HIST_MASK                                                            0x000000FFL
+#define RLC_RLCS_SDMA_INT_STAT__REQ_BUSY_HIST_MASK                                                            0x0000FF00L
+#define RLC_RLCS_SDMA_INT_STAT__LAST_SDMA_RLC_INT_ID_MASK                                                     0x00010000L
+#define RLC_RLCS_SDMA_INT_STAT__SDMA_RLC_INT_PENDING_MASK                                                     0x00020000L
+#define RLC_RLCS_SDMA_INT_STAT__RESERVED_MASK                                                                 0xFFFC0000L
+//RLC_RLCS_SDMA_INT_INFO
+#define RLC_RLCS_SDMA_INT_INFO__REQ_IDLE_TO_FW__SHIFT                                                         0x0
+#define RLC_RLCS_SDMA_INT_INFO__REQ_BUSY_TO_FW__SHIFT                                                         0x8
+#define RLC_RLCS_SDMA_INT_INFO__INTERRUPT_ID__SHIFT                                                           0x10
+#define RLC_RLCS_SDMA_INT_INFO__RESERVED__SHIFT                                                               0x11
+#define RLC_RLCS_SDMA_INT_INFO__REQ_IDLE_TO_FW_MASK                                                           0x000000FFL
+#define RLC_RLCS_SDMA_INT_INFO__REQ_BUSY_TO_FW_MASK                                                           0x0000FF00L
+#define RLC_RLCS_SDMA_INT_INFO__INTERRUPT_ID_MASK                                                             0x00010000L
+#define RLC_RLCS_SDMA_INT_INFO__RESERVED_MASK                                                                 0xFFFE0000L
+//RLC_RLCS_PMM_CGCG_CNTL
+#define RLC_RLCS_PMM_CGCG_CNTL__VALID__SHIFT                                                                  0x0
+#define RLC_RLCS_PMM_CGCG_CNTL__CLEAN__SHIFT                                                                  0x1
+#define RLC_RLCS_PMM_CGCG_CNTL__RESERVED__SHIFT                                                               0x2
+#define RLC_RLCS_PMM_CGCG_CNTL__VALID_MASK                                                                    0x00000001L
+#define RLC_RLCS_PMM_CGCG_CNTL__CLEAN_MASK                                                                    0x00000002L
+#define RLC_RLCS_PMM_CGCG_CNTL__RESERVED_MASK                                                                 0xFFFFFFFCL
+//RLC_RLCS_GFX_MEM_POWER_CTRL_LO
+#define RLC_RLCS_GFX_MEM_POWER_CTRL_LO__DATA__SHIFT                                                           0x0
+#define RLC_RLCS_GFX_MEM_POWER_CTRL_LO__DATA_MASK                                                             0xFFFFFFFFL
+//RLC_RLCS_GFX_RM_CNTL
+#define RLC_RLCS_GFX_RM_CNTL__RLC_GFX_RM_VALID__SHIFT                                                         0x0
+#define RLC_RLCS_GFX_RM_CNTL__RESERVED__SHIFT                                                                 0x1
+#define RLC_RLCS_GFX_RM_CNTL__RLC_GFX_RM_VALID_MASK                                                           0x00000001L
+#define RLC_RLCS_GFX_RM_CNTL__RESERVED_MASK                                                                   0xFFFFFFFEL
+//RLC_RLCS_IH_CTRL_1
+#define RLC_RLCS_IH_CTRL_1__IH_CONTEXT_ID_1__SHIFT                                                            0x0
+#define RLC_RLCS_IH_CTRL_1__IH_CONTEXT_ID_1_MASK                                                              0xFFFFFFFFL
+//RLC_RLCS_IH_CTRL_2
+#define RLC_RLCS_IH_CTRL_2__IH_CONTEXT_ID_2__SHIFT                                                            0x0
+#define RLC_RLCS_IH_CTRL_2__IH_RING_ID__SHIFT                                                                 0x8
+#define RLC_RLCS_IH_CTRL_2__IH_VM_ID__SHIFT                                                                   0x10
+#define RLC_RLCS_IH_CTRL_2__RESERVED__SHIFT                                                                   0x14
+#define RLC_RLCS_IH_CTRL_2__IH_CONTEXT_ID_2_MASK                                                              0x000000FFL
+#define RLC_RLCS_IH_CTRL_2__IH_RING_ID_MASK                                                                   0x0000FF00L
+#define RLC_RLCS_IH_CTRL_2__IH_VM_ID_MASK                                                                     0x000F0000L
+#define RLC_RLCS_IH_CTRL_2__RESERVED_MASK                                                                     0xFFF00000L
+//RLC_RLCS_IH_CTRL_3
+#define RLC_RLCS_IH_CTRL_3__IH_SOURCE_ID__SHIFT                                                               0x0
+#define RLC_RLCS_IH_CTRL_3__IH_VF_ID__SHIFT                                                                   0x8
+#define RLC_RLCS_IH_CTRL_3__IH_VF__SHIFT                                                                      0xd
+#define RLC_RLCS_IH_CTRL_3__RESERVED__SHIFT                                                                   0xe
+#define RLC_RLCS_IH_CTRL_3__IH_SOURCE_ID_MASK                                                                 0x000000FFL
+#define RLC_RLCS_IH_CTRL_3__IH_VF_ID_MASK                                                                     0x00001F00L
+#define RLC_RLCS_IH_CTRL_3__IH_VF_MASK                                                                        0x00002000L
+#define RLC_RLCS_IH_CTRL_3__RESERVED_MASK                                                                     0xFFFFC000L
+//RLC_RLCS_IH_STATUS
+#define RLC_RLCS_IH_STATUS__IH_CREDIT_COUNT__SHIFT                                                            0x0
+#define RLC_RLCS_IH_STATUS__IH_BUSY__SHIFT                                                                    0x6
+#define RLC_RLCS_IH_STATUS__IH_WRITE_DONE__SHIFT                                                              0x7
+#define RLC_RLCS_IH_STATUS__RESERVED__SHIFT                                                                   0x8
+#define RLC_RLCS_IH_STATUS__IH_CREDIT_COUNT_MASK                                                              0x0000003FL
+#define RLC_RLCS_IH_STATUS__IH_BUSY_MASK                                                                      0x00000040L
+#define RLC_RLCS_IH_STATUS__IH_WRITE_DONE_MASK                                                                0x00000080L
+#define RLC_RLCS_IH_STATUS__RESERVED_MASK                                                                     0xFFFFFF00L
+//RLC_RLCS_DEC_END
+
+
+// addressBlock: gc_pfvfdec_rlc
+//RLC_SAFE_MODE
+#define RLC_SAFE_MODE__CMD__SHIFT                                                                             0x0
+#define RLC_SAFE_MODE__MESSAGE__SHIFT                                                                         0x1
+#define RLC_SAFE_MODE__RESERVED1__SHIFT                                                                       0x5
+#define RLC_SAFE_MODE__RESPONSE__SHIFT                                                                        0x8
+#define RLC_SAFE_MODE__RESERVED__SHIFT                                                                        0xc
+#define RLC_SAFE_MODE__CMD_MASK                                                                               0x00000001L
+#define RLC_SAFE_MODE__MESSAGE_MASK                                                                           0x0000001EL
+#define RLC_SAFE_MODE__RESERVED1_MASK                                                                         0x000000E0L
+#define RLC_SAFE_MODE__RESPONSE_MASK                                                                          0x00000F00L
+#define RLC_SAFE_MODE__RESERVED_MASK                                                                          0xFFFFF000L
+//RLC_SPM_SAMPLE_CNT
+#define RLC_SPM_SAMPLE_CNT__COUNT__SHIFT                                                                      0x0
+#define RLC_SPM_SAMPLE_CNT__COUNT_MASK                                                                        0xFFFFFFFFL
+//RLC_SPM_MC_CNTL
+#define RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT                                                                  0x0
+#define RLC_SPM_MC_CNTL__RLC_SPM_POLICY__SHIFT                                                                0x4
+#define RLC_SPM_MC_CNTL__RLC_SPM_PERF_CNTR__SHIFT                                                             0x6
+#define RLC_SPM_MC_CNTL__RLC_SPM_FED__SHIFT                                                                   0x7
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_OVER__SHIFT                                                            0x8
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE__SHIFT                                                                 0x9
+#define RLC_SPM_MC_CNTL__RLC_SPM_BC__SHIFT                                                                    0xc
+#define RLC_SPM_MC_CNTL__RLC_SPM_RO__SHIFT                                                                    0xd
+#define RLC_SPM_MC_CNTL__RLC_SPM_VOL__SHIFT                                                                   0xe
+#define RLC_SPM_MC_CNTL__RLC_SPM_NOFILL__SHIFT                                                                0xf
+#define RLC_SPM_MC_CNTL__RESERVED_3__SHIFT                                                                    0x10
+#define RLC_SPM_MC_CNTL__RLC_SPM_LLC_NOALLOC__SHIFT                                                           0x12
+#define RLC_SPM_MC_CNTL__RLC_SPM_LLC_NOALLOC_OVER__SHIFT                                                      0x13
+#define RLC_SPM_MC_CNTL__RESERVED__SHIFT                                                                      0x14
+#define RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK                                                                    0x0000000FL
+#define RLC_SPM_MC_CNTL__RLC_SPM_POLICY_MASK                                                                  0x00000030L
+#define RLC_SPM_MC_CNTL__RLC_SPM_PERF_CNTR_MASK                                                               0x00000040L
+#define RLC_SPM_MC_CNTL__RLC_SPM_FED_MASK                                                                     0x00000080L
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_OVER_MASK                                                              0x00000100L
+#define RLC_SPM_MC_CNTL__RLC_SPM_MTYPE_MASK                                                                   0x00000E00L
+#define RLC_SPM_MC_CNTL__RLC_SPM_BC_MASK                                                                      0x00001000L
+#define RLC_SPM_MC_CNTL__RLC_SPM_RO_MASK                                                                      0x00002000L
+#define RLC_SPM_MC_CNTL__RLC_SPM_VOL_MASK                                                                     0x00004000L
+#define RLC_SPM_MC_CNTL__RLC_SPM_NOFILL_MASK                                                                  0x00008000L
+#define RLC_SPM_MC_CNTL__RESERVED_3_MASK                                                                      0x00030000L
+#define RLC_SPM_MC_CNTL__RLC_SPM_LLC_NOALLOC_MASK                                                             0x00040000L
+#define RLC_SPM_MC_CNTL__RLC_SPM_LLC_NOALLOC_OVER_MASK                                                        0x00080000L
+#define RLC_SPM_MC_CNTL__RESERVED_MASK                                                                        0xFFF00000L
+//RLC_SPM_INT_CNTL
+#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL__SHIFT                                                             0x0
+#define RLC_SPM_INT_CNTL__RESERVED__SHIFT                                                                     0x1
+#define RLC_SPM_INT_CNTL__RLC_SPM_INT_CNTL_MASK                                                               0x00000001L
+#define RLC_SPM_INT_CNTL__RESERVED_MASK                                                                       0xFFFFFFFEL
+//RLC_SPM_INT_STATUS
+#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS__SHIFT                                                         0x0
+#define RLC_SPM_INT_STATUS__RESERVED__SHIFT                                                                   0x1
+#define RLC_SPM_INT_STATUS__RLC_SPM_INT_STATUS_MASK                                                           0x00000001L
+#define RLC_SPM_INT_STATUS__RESERVED_MASK                                                                     0xFFFFFFFEL
+//RLC_SPM_INT_INFO_1
+#define RLC_SPM_INT_INFO_1__INTERRUPT_INFO_1__SHIFT                                                           0x0
+#define RLC_SPM_INT_INFO_1__INTERRUPT_INFO_1_MASK                                                             0xFFFFFFFFL
+//RLC_SPM_INT_INFO_2
+#define RLC_SPM_INT_INFO_2__INTERRUPT_INFO_2__SHIFT                                                           0x0
+#define RLC_SPM_INT_INFO_2__INTERRUPT_ID__SHIFT                                                               0x10
+#define RLC_SPM_INT_INFO_2__RESERVED__SHIFT                                                                   0x18
+#define RLC_SPM_INT_INFO_2__INTERRUPT_INFO_2_MASK                                                             0x0000FFFFL
+#define RLC_SPM_INT_INFO_2__INTERRUPT_ID_MASK                                                                 0x00FF0000L
+#define RLC_SPM_INT_INFO_2__RESERVED_MASK                                                                     0xFF000000L
+//RLC_CSIB_ADDR_LO
+#define RLC_CSIB_ADDR_LO__ADDRESS__SHIFT                                                                      0x0
+#define RLC_CSIB_ADDR_LO__ADDRESS_MASK                                                                        0xFFFFFFFFL
+//RLC_CSIB_ADDR_HI
+#define RLC_CSIB_ADDR_HI__ADDRESS__SHIFT                                                                      0x0
+#define RLC_CSIB_ADDR_HI__ADDRESS_MASK                                                                        0x0000FFFFL
+//RLC_CSIB_LENGTH
+#define RLC_CSIB_LENGTH__LENGTH__SHIFT                                                                        0x0
+#define RLC_CSIB_LENGTH__LENGTH_MASK                                                                          0xFFFFFFFFL
+//RLC_CP_SCHEDULERS
+#define RLC_CP_SCHEDULERS__scheduler0__SHIFT                                                                  0x0
+#define RLC_CP_SCHEDULERS__scheduler1__SHIFT                                                                  0x8
+#define RLC_CP_SCHEDULERS__scheduler0_MASK                                                                    0x000000FFL
+#define RLC_CP_SCHEDULERS__scheduler1_MASK                                                                    0x0000FF00L
+//RLC_CP_EOF_INT
+#define RLC_CP_EOF_INT__INTERRUPT__SHIFT                                                                      0x0
+#define RLC_CP_EOF_INT__RESERVED__SHIFT                                                                       0x1
+#define RLC_CP_EOF_INT__INTERRUPT_MASK                                                                        0x00000001L
+#define RLC_CP_EOF_INT__RESERVED_MASK                                                                         0xFFFFFFFEL
+//RLC_CP_EOF_INT_CNT
+#define RLC_CP_EOF_INT_CNT__CNT__SHIFT                                                                        0x0
+#define RLC_CP_EOF_INT_CNT__CNT_MASK                                                                          0xFFFFFFFFL
+//RLC_SPARE_INT_0
+#define RLC_SPARE_INT_0__DATA__SHIFT                                                                          0x0
+#define RLC_SPARE_INT_0__PROCESSING__SHIFT                                                                    0x1e
+#define RLC_SPARE_INT_0__COMPLETE__SHIFT                                                                      0x1f
+#define RLC_SPARE_INT_0__DATA_MASK                                                                            0x3FFFFFFFL
+#define RLC_SPARE_INT_0__PROCESSING_MASK                                                                      0x40000000L
+#define RLC_SPARE_INT_0__COMPLETE_MASK                                                                        0x80000000L
+//RLC_SPARE_INT_1
+#define RLC_SPARE_INT_1__DATA__SHIFT                                                                          0x0
+#define RLC_SPARE_INT_1__PROCESSING__SHIFT                                                                    0x1e
+#define RLC_SPARE_INT_1__COMPLETE__SHIFT                                                                      0x1f
+#define RLC_SPARE_INT_1__DATA_MASK                                                                            0x3FFFFFFFL
+#define RLC_SPARE_INT_1__PROCESSING_MASK                                                                      0x40000000L
+#define RLC_SPARE_INT_1__COMPLETE_MASK                                                                        0x80000000L
+//RLC_SPARE_INT_2
+#define RLC_SPARE_INT_2__DATA__SHIFT                                                                          0x0
+#define RLC_SPARE_INT_2__PROCESSING__SHIFT                                                                    0x1e
+#define RLC_SPARE_INT_2__COMPLETE__SHIFT                                                                      0x1f
+#define RLC_SPARE_INT_2__DATA_MASK                                                                            0x3FFFFFFFL
+#define RLC_SPARE_INT_2__PROCESSING_MASK                                                                      0x40000000L
+#define RLC_SPARE_INT_2__COMPLETE_MASK                                                                        0x80000000L
+//RLC_PACE_SPARE_INT
+#define RLC_PACE_SPARE_INT__INTERRUPT__SHIFT                                                                  0x0
+#define RLC_PACE_SPARE_INT__RESERVED__SHIFT                                                                   0x1
+#define RLC_PACE_SPARE_INT__INTERRUPT_MASK                                                                    0x00000001L
+#define RLC_PACE_SPARE_INT__RESERVED_MASK                                                                     0xFFFFFFFEL
+//RLC_PACE_SPARE_INT_1
+#define RLC_PACE_SPARE_INT_1__INTERRUPT__SHIFT                                                                0x0
+#define RLC_PACE_SPARE_INT_1__RESERVED__SHIFT                                                                 0x1
+#define RLC_PACE_SPARE_INT_1__INTERRUPT_MASK                                                                  0x00000001L
+#define RLC_PACE_SPARE_INT_1__RESERVED_MASK                                                                   0xFFFFFFFEL
+//RLC_RLCV_SPARE_INT_1
+#define RLC_RLCV_SPARE_INT_1__INTERRUPT__SHIFT                                                                0x0
+#define RLC_RLCV_SPARE_INT_1__RESERVED__SHIFT                                                                 0x1
+#define RLC_RLCV_SPARE_INT_1__INTERRUPT_MASK                                                                  0x00000001L
+#define RLC_RLCV_SPARE_INT_1__RESERVED_MASK                                                                   0xFFFFFFFEL
+
+
+// addressBlock: gc_pwrdec
+//CGTS_TCC_DISABLE
+#define CGTS_TCC_DISABLE__WRITE_DIS__SHIFT                                                                    0x0
+#define CGTS_TCC_DISABLE__HI_TCC_DISABLE__SHIFT                                                               0x8
+#define CGTS_TCC_DISABLE__TCC_DISABLE__SHIFT                                                                  0x10
+#define CGTS_TCC_DISABLE__WRITE_DIS_MASK                                                                      0x00000001L
+#define CGTS_TCC_DISABLE__HI_TCC_DISABLE_MASK                                                                 0x0000FF00L
+#define CGTS_TCC_DISABLE__TCC_DISABLE_MASK                                                                    0xFFFF0000L
+//CGTX_SPI_DEBUG_CLK_CTRL
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OFF_HYST__SHIFT                                                      0x0
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OVERRIDE__SHIFT                                                      0x6
+#define CGTX_SPI_DEBUG_CLK_CTRL__ALL_CLK_ON_OVERRIDE__SHIFT                                                   0x7
+#define CGTX_SPI_DEBUG_CLK_CTRL__SPI_SH_CLK_CONTROL__SHIFT                                                    0x8
+#define CGTX_SPI_DEBUG_CLK_CTRL__SPI_REPEATER_FGCG_OVERRIDE__SHIFT                                            0x9
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OFF_HYST_MASK                                                        0x0000003FL
+#define CGTX_SPI_DEBUG_CLK_CTRL__GRP5_CG_OVERRIDE_MASK                                                        0x00000040L
+#define CGTX_SPI_DEBUG_CLK_CTRL__ALL_CLK_ON_OVERRIDE_MASK                                                     0x00000080L
+#define CGTX_SPI_DEBUG_CLK_CTRL__SPI_SH_CLK_CONTROL_MASK                                                      0x00000100L
+#define CGTX_SPI_DEBUG_CLK_CTRL__SPI_REPEATER_FGCG_OVERRIDE_MASK                                              0x00000200L
+//CGTT_VGT_CLK_CTRL
+#define CGTT_VGT_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE__SHIFT                                                                 0xf
+#define CGTT_VGT_CLK_CTRL__DBG_ENABLE__SHIFT                                                                  0x10
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_VGT_CLK_CTRL__PI1_OVERRIDE__SHIFT                                                                0x17
+#define CGTT_VGT_CLK_CTRL__PI0_OVERRIDE__SHIFT                                                                0x18
+#define CGTT_VGT_CLK_CTRL__HS_OVERRIDE__SHIFT                                                                 0x19
+#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE__SHIFT                                                               0x1c
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                               0x1d
+#define CGTT_VGT_CLK_CTRL__RBIU_INPUT_OVERRIDE__SHIFT                                                         0x1e
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_VGT_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_VGT_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_VGT_CLK_CTRL__PERF_ENABLE_MASK                                                                   0x00008000L
+#define CGTT_VGT_CLK_CTRL__DBG_ENABLE_MASK                                                                    0x00010000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_VGT_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_VGT_CLK_CTRL__PI1_OVERRIDE_MASK                                                                  0x00800000L
+#define CGTT_VGT_CLK_CTRL__PI0_OVERRIDE_MASK                                                                  0x01000000L
+#define CGTT_VGT_CLK_CTRL__HS_OVERRIDE_MASK                                                                   0x02000000L
+#define CGTT_VGT_CLK_CTRL__TESS_OVERRIDE_MASK                                                                 0x10000000L
+#define CGTT_VGT_CLK_CTRL__CORE_OVERRIDE_MASK                                                                 0x20000000L
+#define CGTT_VGT_CLK_CTRL__RBIU_INPUT_OVERRIDE_MASK                                                           0x40000000L
+#define CGTT_VGT_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//CGTT_IA_CLK_CTRL
+#define CGTT_IA_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE__SHIFT                                                                  0xf
+#define CGTT_IA_CLK_CTRL__DBG_ENABLE__SHIFT                                                                   0x10
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                               0x18
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                               0x19
+#define CGTT_IA_CLK_CTRL__DIST_OVERRIDE__SHIFT                                                                0x1a
+#define CGTT_IA_CLK_CTRL__PERF_OVERRIDE__SHIFT                                                                0x1b
+#define CGTT_IA_CLK_CTRL__PCM_OVERRIDE__SHIFT                                                                 0x1c
+#define CGTT_IA_CLK_CTRL__TESS_DIST_OVERRIDE__SHIFT                                                           0x1d
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1e
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_IA_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_IA_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_IA_CLK_CTRL__PERF_ENABLE_MASK                                                                    0x00008000L
+#define CGTT_IA_CLK_CTRL__DBG_ENABLE_MASK                                                                     0x00010000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_IA_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                                 0x01000000L
+#define CGTT_IA_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                                 0x02000000L
+#define CGTT_IA_CLK_CTRL__DIST_OVERRIDE_MASK                                                                  0x04000000L
+#define CGTT_IA_CLK_CTRL__PERF_OVERRIDE_MASK                                                                  0x08000000L
+#define CGTT_IA_CLK_CTRL__PCM_OVERRIDE_MASK                                                                   0x10000000L
+#define CGTT_IA_CLK_CTRL__TESS_DIST_OVERRIDE_MASK                                                             0x20000000L
+#define CGTT_IA_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x40000000L
+#define CGTT_IA_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_WD_CLK_CTRL
+#define CGTT_WD_CLK_CTRL__ON_DELAY__SHIFT                                                                     0x0
+#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_WD_CLK_CTRL__PERF_ENABLE__SHIFT                                                                  0xf
+#define CGTT_WD_CLK_CTRL__DBG_ENABLE__SHIFT                                                                   0x10
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_WD_CLK_CTRL__FE_OUT_OVERRIDE__SHIFT                                                              0x17
+#define CGTT_WD_CLK_CTRL__ASSEMBLER_OVERRIDE__SHIFT                                                           0x18
+#define CGTT_WD_CLK_CTRL__DMA_PROC0_OVERRIDE__SHIFT                                                           0x19
+#define CGTT_WD_CLK_CTRL__DMA_PROC1_OVERRIDE__SHIFT                                                           0x1a
+#define CGTT_WD_CLK_CTRL__PERF_OVERRIDE__SHIFT                                                                0x1b
+#define CGTT_WD_CLK_CTRL__DMA_OVERRIDE__SHIFT                                                                 0x1c
+#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                                0x1d
+#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE__SHIFT                                                          0x1e
+#define CGTT_WD_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                 0x1f
+#define CGTT_WD_CLK_CTRL__ON_DELAY_MASK                                                                       0x0000000FL
+#define CGTT_WD_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_WD_CLK_CTRL__PERF_ENABLE_MASK                                                                    0x00008000L
+#define CGTT_WD_CLK_CTRL__DBG_ENABLE_MASK                                                                     0x00010000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_WD_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_WD_CLK_CTRL__FE_OUT_OVERRIDE_MASK                                                                0x00800000L
+#define CGTT_WD_CLK_CTRL__ASSEMBLER_OVERRIDE_MASK                                                             0x01000000L
+#define CGTT_WD_CLK_CTRL__DMA_PROC0_OVERRIDE_MASK                                                             0x02000000L
+#define CGTT_WD_CLK_CTRL__DMA_PROC1_OVERRIDE_MASK                                                             0x04000000L
+#define CGTT_WD_CLK_CTRL__PERF_OVERRIDE_MASK                                                                  0x08000000L
+#define CGTT_WD_CLK_CTRL__DMA_OVERRIDE_MASK                                                                   0x10000000L
+#define CGTT_WD_CLK_CTRL__CORE_OVERRIDE_MASK                                                                  0x20000000L
+#define CGTT_WD_CLK_CTRL__RBIU_INPUT_OVERRIDE_MASK                                                            0x40000000L
+#define CGTT_WD_CLK_CTRL__REG_OVERRIDE_MASK                                                                   0x80000000L
+//CGTT_GS_NGG_CLK_CTRL
+#define CGTT_GS_NGG_CLK_CTRL__ON_DELAY__SHIFT                                                                 0x0
+#define CGTT_GS_NGG_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                           0x4
+#define CGTT_GS_NGG_CLK_CTRL__PERF_ENABLE__SHIFT                                                              0xf
+#define CGTT_GS_NGG_CLK_CTRL__DBG_ENABLE__SHIFT                                                               0x10
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                     0x11
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                     0x12
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                     0x13
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                     0x14
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                     0x15
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                     0x16
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                     0x17
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                           0x18
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                           0x19
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                           0x1a
+#define CGTT_GS_NGG_CLK_CTRL__PERF_OVERRIDE__SHIFT                                                            0x1b
+#define CGTT_GS_NGG_CLK_CTRL__PRIMGEN_OVERRIDE__SHIFT                                                         0x1c
+#define CGTT_GS_NGG_CLK_CTRL__REG_OVERRIDE__SHIFT                                                             0x1f
+#define CGTT_GS_NGG_CLK_CTRL__ON_DELAY_MASK                                                                   0x0000000FL
+#define CGTT_GS_NGG_CLK_CTRL__OFF_HYSTERESIS_MASK                                                             0x00000FF0L
+#define CGTT_GS_NGG_CLK_CTRL__PERF_ENABLE_MASK                                                                0x00008000L
+#define CGTT_GS_NGG_CLK_CTRL__DBG_ENABLE_MASK                                                                 0x00010000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                       0x00020000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                       0x00040000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                       0x00080000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                       0x00100000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                       0x00200000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                       0x00400000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                       0x00800000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE7_MASK                                                             0x01000000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE6_MASK                                                             0x02000000L
+#define CGTT_GS_NGG_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                             0x04000000L
+#define CGTT_GS_NGG_CLK_CTRL__PERF_OVERRIDE_MASK                                                              0x08000000L
+#define CGTT_GS_NGG_CLK_CTRL__PRIMGEN_OVERRIDE_MASK                                                           0x10000000L
+#define CGTT_GS_NGG_CLK_CTRL__REG_OVERRIDE_MASK                                                               0x80000000L
+//CGTT_PA_CLK_CTRL
+#define CGTT_PA_CLK_CTRL__CLIP_SU_PRIM_FIFO_CLK_OVERRIDE__SHIFT                                               0xc
+#define CGTT_PA_CLK_CTRL__SXIFCCG_CLK_OVERRIDE__SHIFT                                                         0xd
+#define CGTT_PA_CLK_CTRL__AG_CLK_OVERRIDE__SHIFT                                                              0xe
+#define CGTT_PA_CLK_CTRL__VE_VTE_REC_CLK_OVERRIDE__SHIFT                                                      0xf
+#define CGTT_PA_CLK_CTRL__ENGG_CLK_OVERRIDE__SHIFT                                                            0x10
+#define CGTT_PA_CLK_CTRL__CL_VTE_CLK_OVERRIDE__SHIFT                                                          0x11
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_PA_CLK_CTRL__AG_REG_CLK_OVERRIDE__SHIFT                                                          0x14
+#define CGTT_PA_CLK_CTRL__CL_VTE_REG_CLK_OVERRIDE__SHIFT                                                      0x15
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_PA_CLK_CTRL__DEBUG_BUS_EN__SHIFT                                                                 0x17
+#define CGTT_PA_CLK_CTRL__VTE_REG_CLK_OVERRIDE__SHIFT                                                         0x18
+#define CGTT_PA_CLK_CTRL__PERFMON_CLK_OVERRIDE__SHIFT                                                         0x19
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                               0x1a
+#define CGTT_PA_CLK_CTRL__NGG_INDEX_CLK_OVERRIDE__SHIFT                                                       0x1b
+#define CGTT_PA_CLK_CTRL__NGG_CSB_CLK_OVERRIDE__SHIFT                                                         0x1c
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE__SHIFT                                                              0x1d
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE__SHIFT                                                              0x1e
+#define CGTT_PA_CLK_CTRL__SU_CL_REG_CLK_OVERRIDE__SHIFT                                                       0x1f
+#define CGTT_PA_CLK_CTRL__CLIP_SU_PRIM_FIFO_CLK_OVERRIDE_MASK                                                 0x00001000L
+#define CGTT_PA_CLK_CTRL__SXIFCCG_CLK_OVERRIDE_MASK                                                           0x00002000L
+#define CGTT_PA_CLK_CTRL__AG_CLK_OVERRIDE_MASK                                                                0x00004000L
+#define CGTT_PA_CLK_CTRL__VE_VTE_REC_CLK_OVERRIDE_MASK                                                        0x00008000L
+#define CGTT_PA_CLK_CTRL__ENGG_CLK_OVERRIDE_MASK                                                              0x00010000L
+#define CGTT_PA_CLK_CTRL__CL_VTE_CLK_OVERRIDE_MASK                                                            0x00020000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_PA_CLK_CTRL__AG_REG_CLK_OVERRIDE_MASK                                                            0x00100000L
+#define CGTT_PA_CLK_CTRL__CL_VTE_REG_CLK_OVERRIDE_MASK                                                        0x00200000L
+#define CGTT_PA_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_PA_CLK_CTRL__DEBUG_BUS_EN_MASK                                                                   0x00800000L
+#define CGTT_PA_CLK_CTRL__VTE_REG_CLK_OVERRIDE_MASK                                                           0x01000000L
+#define CGTT_PA_CLK_CTRL__PERFMON_CLK_OVERRIDE_MASK                                                           0x02000000L
+#define CGTT_PA_CLK_CTRL__SOFT_OVERRIDE5_MASK                                                                 0x04000000L
+#define CGTT_PA_CLK_CTRL__NGG_INDEX_CLK_OVERRIDE_MASK                                                         0x08000000L
+#define CGTT_PA_CLK_CTRL__NGG_CSB_CLK_OVERRIDE_MASK                                                           0x10000000L
+#define CGTT_PA_CLK_CTRL__SU_CLK_OVERRIDE_MASK                                                                0x20000000L
+#define CGTT_PA_CLK_CTRL__CL_CLK_OVERRIDE_MASK                                                                0x40000000L
+#define CGTT_PA_CLK_CTRL__SU_CL_REG_CLK_OVERRIDE_MASK                                                         0x80000000L
+//CGTT_SC_CLK_CTRL0
+#define CGTT_SC_CLK_CTRL0__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_STALL_OVERRIDE__SHIFT                                              0x10
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x11
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x12
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x13
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x14
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x15
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x16
+#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE__SHIFT                                                      0x17
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_OVERRIDE__SHIFT                                                    0x18
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT                                                              0x19
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT                                                              0x1a
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT                                                              0x1b
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT                                                              0x1c
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE1__SHIFT                                                              0x1d
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE0__SHIFT                                                              0x1e
+#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE__SHIFT                                                            0x1f
+#define CGTT_SC_CLK_CTRL0__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SC_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_STALL_OVERRIDE_MASK                                                0x00010000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE5_MASK                                                          0x00020000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE4_MASK                                                          0x00040000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE3_MASK                                                          0x00080000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE2_MASK                                                          0x00100000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE1_MASK                                                          0x00200000L
+#define CGTT_SC_CLK_CTRL0__SOFT_STALL_OVERRIDE0_MASK                                                          0x00400000L
+#define CGTT_SC_CLK_CTRL0__REG_CLK_STALL_OVERRIDE_MASK                                                        0x00800000L
+#define CGTT_SC_CLK_CTRL0__PFF_ZFF_MEM_CLK_OVERRIDE_MASK                                                      0x01000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE5_MASK                                                                0x02000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE4_MASK                                                                0x04000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE3_MASK                                                                0x08000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE2_MASK                                                                0x10000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE1_MASK                                                                0x20000000L
+#define CGTT_SC_CLK_CTRL0__SOFT_OVERRIDE0_MASK                                                                0x40000000L
+#define CGTT_SC_CLK_CTRL0__REG_CLK_OVERRIDE_MASK                                                              0x80000000L
+//CGTT_SC_CLK_CTRL1
+#define CGTT_SC_CLK_CTRL1__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SC_CLK_CTRL1__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE0__SHIFT                                             0x10
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE__SHIFT                                              0x11
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE__SHIFT                                              0x12
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_STALL_OVERRIDE__SHIFT                                     0x13
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_STALL_OVERRIDE__SHIFT                                           0x14
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE__SHIFT                                            0x15
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE__SHIFT                                                      0x16
+#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_STALL_OVERRIDE__SHIFT                                                 0x17
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE0__SHIFT                                                   0x18
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE__SHIFT                                                    0x19
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE__SHIFT                                                    0x1a
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_OVERRIDE__SHIFT                                           0x1b
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE__SHIFT                                                 0x1c
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE__SHIFT                                                  0x1d
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_OVERRIDE__SHIFT                                                            0x1e
+#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_OVERRIDE__SHIFT                                                       0x1f
+#define CGTT_SC_CLK_CTRL1__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SC_CLK_CTRL1__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE0_MASK                                               0x00010000L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_STALL_OVERRIDE_MASK                                                0x00020000L
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_STALL_OVERRIDE_MASK                                                0x00040000L
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_STALL_OVERRIDE_MASK                                       0x00080000L
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_STALL_OVERRIDE_MASK                                             0x00100000L
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_STALL_OVERRIDE_MASK                                              0x00200000L
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_STALL_OVERRIDE_MASK                                                        0x00400000L
+#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_STALL_OVERRIDE_MASK                                                   0x00800000L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE0_MASK                                                     0x01000000L
+#define CGTT_SC_CLK_CTRL1__PBB_BINNING_CLK_OVERRIDE_MASK                                                      0x02000000L
+#define CGTT_SC_CLK_CTRL1__PBB_SCISSOR_CLK_OVERRIDE_MASK                                                      0x04000000L
+#define CGTT_SC_CLK_CTRL1__OTHER_SPECIAL_SC_REG_CLK_OVERRIDE_MASK                                             0x08000000L
+#define CGTT_SC_CLK_CTRL1__SCREEN_EXT_REG_CLK_OVERRIDE_MASK                                                   0x10000000L
+#define CGTT_SC_CLK_CTRL1__VPORT_REG_MEM_CLK_OVERRIDE_MASK                                                    0x20000000L
+#define CGTT_SC_CLK_CTRL1__PBB_CLK_OVERRIDE_MASK                                                              0x40000000L
+#define CGTT_SC_CLK_CTRL1__PBB_WARP_CLK_OVERRIDE_MASK                                                         0x80000000L
+//CGTT_SC_CLK_CTRL2
+#define CGTT_SC_CLK_CTRL2__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SC_CLK_CTRL2__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SC_CLK_CTRL2__DISABLE_DEBUG_BUS_FLOP_EN_ON_PERFMON__SHIFT                                        0xf
+#define CGTT_SC_CLK_CTRL2__SCF_SCB_VRS_INTF_CLK_OVERRIDE__SHIFT                                               0x10
+#define CGTT_SC_CLK_CTRL2__SC_DB_COURSE_MGCG_BUSY_ENABLE__SHIFT                                               0x11
+#define CGTT_SC_CLK_CTRL2__SC_DB_STAGE_IN_TP_PFFB_WR_OVERRIDE__SHIFT                                          0x12
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUADMASK_OVERRIDE__SHIFT                                                     0x13
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUADMASK_Z_OVERRIDE__SHIFT                                                   0x14
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUAD_PROC_OVERRIDE__SHIFT                                                    0x15
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUAD_ACCUM_OVERRIDE__SHIFT                                                   0x16
+#define CGTT_SC_CLK_CTRL2__SC_DB_PFFB_RP_OVERRIDE__SHIFT                                                      0x17
+#define CGTT_SC_CLK_CTRL2__SC_DB_PKR_OVERRIDE__SHIFT                                                          0x18
+#define CGTT_SC_CLK_CTRL2__SC_DB_SC_FREE_WAVE_CLK_OVERRIDE__SHIFT                                             0x19
+#define CGTT_SC_CLK_CTRL2__SC_DB_SC_WAVE_2_SC_SPI_WAVE_CLK_OVERRIDE__SHIFT                                    0x1a
+#define CGTT_SC_CLK_CTRL2__SCF_SCB_INTF_CLK_OVERRIDE__SHIFT                                                   0x1b
+#define CGTT_SC_CLK_CTRL2__SC_PKR_INTF_CLK_OVERRIDE__SHIFT                                                    0x1c
+#define CGTT_SC_CLK_CTRL2__SC_DB_INTF_CLK_OVERRIDE__SHIFT                                                     0x1d
+#define CGTT_SC_CLK_CTRL2__PA_SC_INTF_CLK_OVERRIDE__SHIFT                                                     0x1e
+#define CGTT_SC_CLK_CTRL2__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SC_CLK_CTRL2__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SC_CLK_CTRL2__DISABLE_DEBUG_BUS_FLOP_EN_ON_PERFMON_MASK                                          0x00008000L
+#define CGTT_SC_CLK_CTRL2__SCF_SCB_VRS_INTF_CLK_OVERRIDE_MASK                                                 0x00010000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_COURSE_MGCG_BUSY_ENABLE_MASK                                                 0x00020000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_STAGE_IN_TP_PFFB_WR_OVERRIDE_MASK                                            0x00040000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUADMASK_OVERRIDE_MASK                                                       0x00080000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUADMASK_Z_OVERRIDE_MASK                                                     0x00100000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUAD_PROC_OVERRIDE_MASK                                                      0x00200000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_QUAD_ACCUM_OVERRIDE_MASK                                                     0x00400000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_PFFB_RP_OVERRIDE_MASK                                                        0x00800000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_PKR_OVERRIDE_MASK                                                            0x01000000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_SC_FREE_WAVE_CLK_OVERRIDE_MASK                                               0x02000000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_SC_WAVE_2_SC_SPI_WAVE_CLK_OVERRIDE_MASK                                      0x04000000L
+#define CGTT_SC_CLK_CTRL2__SCF_SCB_INTF_CLK_OVERRIDE_MASK                                                     0x08000000L
+#define CGTT_SC_CLK_CTRL2__SC_PKR_INTF_CLK_OVERRIDE_MASK                                                      0x10000000L
+#define CGTT_SC_CLK_CTRL2__SC_DB_INTF_CLK_OVERRIDE_MASK                                                       0x20000000L
+#define CGTT_SC_CLK_CTRL2__PA_SC_INTF_CLK_OVERRIDE_MASK                                                       0x40000000L
+//CGTT_SQG_CLK_CTRL
+#define CGTT_SQG_CLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_SQG_CLK_CTRL__FORCE_GL1H_CLKEN__SHIFT                                                            0x17
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPALLOC_FGCG__SHIFT                                                         0x18
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPGRANT_FGCG__SHIFT                                                         0x19
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPREQ_FGCG__SHIFT                                                           0x1a
+#define CGTT_SQG_CLK_CTRL__FORCE_CMD_FGCG__SHIFT                                                              0x1b
+#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE__SHIFT                                                             0x1c
+#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE__SHIFT                                                            0x1d
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE__SHIFT                                                               0x1e
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE__SHIFT                                                                0x1f
+#define CGTT_SQG_CLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_SQG_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_SQG_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_SQG_CLK_CTRL__FORCE_GL1H_CLKEN_MASK                                                              0x00800000L
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPALLOC_FGCG_MASK                                                           0x01000000L
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPGRANT_FGCG_MASK                                                           0x02000000L
+#define CGTT_SQG_CLK_CTRL__FORCE_EXPREQ_FGCG_MASK                                                             0x04000000L
+#define CGTT_SQG_CLK_CTRL__FORCE_CMD_FGCG_MASK                                                                0x08000000L
+#define CGTT_SQG_CLK_CTRL__TTRACE_OVERRIDE_MASK                                                               0x10000000L
+#define CGTT_SQG_CLK_CTRL__PERFMON_OVERRIDE_MASK                                                              0x20000000L
+#define CGTT_SQG_CLK_CTRL__CORE_OVERRIDE_MASK                                                                 0x40000000L
+#define CGTT_SQG_CLK_CTRL__REG_OVERRIDE_MASK                                                                  0x80000000L
+//SQ_ALU_CLK_CTRL
+#define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA0__SHIFT                                                              0x0
+#define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA1__SHIFT                                                              0x10
+#define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA0_MASK                                                                0x0000FFFFL
+#define SQ_ALU_CLK_CTRL__FORCE_WGP_ON_SA1_MASK                                                                0xFFFF0000L
+//SQ_TEX_CLK_CTRL
+#define SQ_TEX_CLK_CTRL__FORCE_WGP_ON_SA0__SHIFT                                                              0x0
+#define SQ_TEX_CLK_CTRL__FORCE_WGP_ON_SA1__SHIFT                                                              0x10
+#define SQ_TEX_CLK_CTRL__FORCE_WGP_ON_SA0_MASK                                                                0x0000FFFFL
+#define SQ_TEX_CLK_CTRL__FORCE_WGP_ON_SA1_MASK                                                                0xFFFF0000L
+//SQ_LDS_CLK_CTRL
+#define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA0__SHIFT                                                              0x0
+#define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA1__SHIFT                                                              0x10
+#define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA0_MASK                                                                0x0000FFFFL
+#define SQ_LDS_CLK_CTRL__FORCE_WGP_ON_SA1_MASK                                                                0xFFFF0000L
+//ICG_SP_CLK_CTRL
+#define ICG_SP_CLK_CTRL__CLK_OVERRIDE__SHIFT                                                                  0x0
+#define ICG_SP_CLK_CTRL__CLK_OVERRIDE_MASK                                                                    0xFFFFFFFFL
+//TA_CGTT_CTRL
+#define TA_CGTT_CTRL__ON_DELAY__SHIFT                                                                         0x0
+#define TA_CGTT_CTRL__OFF_HYSTERESIS__SHIFT                                                                   0x4
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                             0x10
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                             0x11
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                             0x12
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                             0x13
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                             0x14
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                             0x15
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                             0x16
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                             0x17
+#define TA_CGTT_CTRL__SOFT_OVERRIDE7__SHIFT                                                                   0x18
+#define TA_CGTT_CTRL__SOFT_OVERRIDE6__SHIFT                                                                   0x19
+#define TA_CGTT_CTRL__SOFT_OVERRIDE5__SHIFT                                                                   0x1a
+#define TA_CGTT_CTRL__SOFT_OVERRIDE4__SHIFT                                                                   0x1b
+#define TA_CGTT_CTRL__SOFT_OVERRIDE3__SHIFT                                                                   0x1c
+#define TA_CGTT_CTRL__SOFT_OVERRIDE2__SHIFT                                                                   0x1d
+#define TA_CGTT_CTRL__SOFT_OVERRIDE1__SHIFT                                                                   0x1e
+#define TA_CGTT_CTRL__SOFT_OVERRIDE0__SHIFT                                                                   0x1f
+#define TA_CGTT_CTRL__ON_DELAY_MASK                                                                           0x0000000FL
+#define TA_CGTT_CTRL__OFF_HYSTERESIS_MASK                                                                     0x00000FF0L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                               0x00010000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                               0x00020000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                               0x00040000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                               0x00080000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                               0x00100000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                               0x00200000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                               0x00400000L
+#define TA_CGTT_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                               0x00800000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE7_MASK                                                                     0x01000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE6_MASK                                                                     0x02000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE5_MASK                                                                     0x04000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE4_MASK                                                                     0x08000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE3_MASK                                                                     0x10000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE2_MASK                                                                     0x20000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE1_MASK                                                                     0x40000000L
+#define TA_CGTT_CTRL__SOFT_OVERRIDE0_MASK                                                                     0x80000000L
+//DB_CGTT_CLK_CTRL_0
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0__SHIFT                                                             0x0
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1__SHIFT                                                             0x1
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2__SHIFT                                                             0x2
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3__SHIFT                                                             0x3
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4__SHIFT                                                             0x4
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5__SHIFT                                                             0x5
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6__SHIFT                                                             0x6
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7__SHIFT                                                             0x7
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE8__SHIFT                                                             0x8
+#define DB_CGTT_CLK_CTRL_0__RESERVED__SHIFT                                                                   0x9
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE0_MASK                                                               0x00000001L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE1_MASK                                                               0x00000002L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE2_MASK                                                               0x00000004L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE3_MASK                                                               0x00000008L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE4_MASK                                                               0x00000010L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE5_MASK                                                               0x00000020L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE6_MASK                                                               0x00000040L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE7_MASK                                                               0x00000080L
+#define DB_CGTT_CLK_CTRL_0__SOFT_OVERRIDE8_MASK                                                               0x00000100L
+#define DB_CGTT_CLK_CTRL_0__RESERVED_MASK                                                                     0xFFFFFE00L
+//CB_CGTT_SCLK_CTRL
+#define CB_CGTT_SCLK_CTRL__ON_DELAY__SHIFT                                                                    0x0
+#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0__SHIFT                                                              0x1f
+#define CB_CGTT_SCLK_CTRL__ON_DELAY_MASK                                                                      0x0000000FL
+#define CB_CGTT_SCLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CB_CGTT_SCLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+#define CB_CGTT_SCLK_CTRL__SOFT_OVERRIDE0_MASK                                                                0x80000000L
+//GFX_ICG_GL2A_CTRL
+#define GFX_ICG_GL2A_CTRL__REG_OVERRIDE__SHIFT                                                                0x0
+#define GFX_ICG_GL2A_CTRL__PERFMON_OVERRIDE__SHIFT                                                            0x1
+#define GFX_ICG_GL2A_CTRL__CROSSBAR_OVERRIDE__SHIFT                                                           0x2
+#define GFX_ICG_GL2A_CTRL__RTN_ARB_OVERRIDE__SHIFT                                                            0x3
+#define GFX_ICG_GL2A_CTRL__GCRD_OVERRIDE__SHIFT                                                               0x4
+#define GFX_ICG_GL2A_CTRL__CLIENT0_OVERRIDE__SHIFT                                                            0x8
+#define GFX_ICG_GL2A_CTRL__CLIENT1_OVERRIDE__SHIFT                                                            0x9
+#define GFX_ICG_GL2A_CTRL__CLIENT2_OVERRIDE__SHIFT                                                            0xa
+#define GFX_ICG_GL2A_CTRL__CLIENT3_OVERRIDE__SHIFT                                                            0xb
+#define GFX_ICG_GL2A_CTRL__CLIENT4_OVERRIDE__SHIFT                                                            0xc
+#define GFX_ICG_GL2A_CTRL__CLIENT5_OVERRIDE__SHIFT                                                            0xd
+#define GFX_ICG_GL2A_CTRL__CLIENT6_OVERRIDE__SHIFT                                                            0xe
+#define GFX_ICG_GL2A_CTRL__CLIENT7_OVERRIDE__SHIFT                                                            0xf
+#define GFX_ICG_GL2A_CTRL__CLIENT8_OVERRIDE__SHIFT                                                            0x10
+#define GFX_ICG_GL2A_CTRL__CLIENT9_OVERRIDE__SHIFT                                                            0x11
+#define GFX_ICG_GL2A_CTRL__CLIENT10_OVERRIDE__SHIFT                                                           0x12
+#define GFX_ICG_GL2A_CTRL__CLIENT11_OVERRIDE__SHIFT                                                           0x13
+#define GFX_ICG_GL2A_CTRL__CLIENT12_OVERRIDE__SHIFT                                                           0x14
+#define GFX_ICG_GL2A_CTRL__CLIENT13_OVERRIDE__SHIFT                                                           0x15
+#define GFX_ICG_GL2A_CTRL__CLIENT14_OVERRIDE__SHIFT                                                           0x16
+#define GFX_ICG_GL2A_CTRL__CLIENT15_OVERRIDE__SHIFT                                                           0x17
+#define GFX_ICG_GL2A_CTRL__REG_OVERRIDE_MASK                                                                  0x00000001L
+#define GFX_ICG_GL2A_CTRL__PERFMON_OVERRIDE_MASK                                                              0x00000002L
+#define GFX_ICG_GL2A_CTRL__CROSSBAR_OVERRIDE_MASK                                                             0x00000004L
+#define GFX_ICG_GL2A_CTRL__RTN_ARB_OVERRIDE_MASK                                                              0x00000008L
+#define GFX_ICG_GL2A_CTRL__GCRD_OVERRIDE_MASK                                                                 0x00000010L
+#define GFX_ICG_GL2A_CTRL__CLIENT0_OVERRIDE_MASK                                                              0x00000100L
+#define GFX_ICG_GL2A_CTRL__CLIENT1_OVERRIDE_MASK                                                              0x00000200L
+#define GFX_ICG_GL2A_CTRL__CLIENT2_OVERRIDE_MASK                                                              0x00000400L
+#define GFX_ICG_GL2A_CTRL__CLIENT3_OVERRIDE_MASK                                                              0x00000800L
+#define GFX_ICG_GL2A_CTRL__CLIENT4_OVERRIDE_MASK                                                              0x00001000L
+#define GFX_ICG_GL2A_CTRL__CLIENT5_OVERRIDE_MASK                                                              0x00002000L
+#define GFX_ICG_GL2A_CTRL__CLIENT6_OVERRIDE_MASK                                                              0x00004000L
+#define GFX_ICG_GL2A_CTRL__CLIENT7_OVERRIDE_MASK                                                              0x00008000L
+#define GFX_ICG_GL2A_CTRL__CLIENT8_OVERRIDE_MASK                                                              0x00010000L
+#define GFX_ICG_GL2A_CTRL__CLIENT9_OVERRIDE_MASK                                                              0x00020000L
+#define GFX_ICG_GL2A_CTRL__CLIENT10_OVERRIDE_MASK                                                             0x00040000L
+#define GFX_ICG_GL2A_CTRL__CLIENT11_OVERRIDE_MASK                                                             0x00080000L
+#define GFX_ICG_GL2A_CTRL__CLIENT12_OVERRIDE_MASK                                                             0x00100000L
+#define GFX_ICG_GL2A_CTRL__CLIENT13_OVERRIDE_MASK                                                             0x00200000L
+#define GFX_ICG_GL2A_CTRL__CLIENT14_OVERRIDE_MASK                                                             0x00400000L
+#define GFX_ICG_GL2A_CTRL__CLIENT15_OVERRIDE_MASK                                                             0x00800000L
+//CGTT_CP_CLK_CTRL
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                               0x4
+#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                                0xf
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                         0x10
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                         0x11
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                         0x12
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                         0x13
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                         0x14
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                         0x15
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                         0x16
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                         0x17
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                        0x1d
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                            0x1e
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                            0x1f
+#define CGTT_CP_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                 0x00000FF0L
+#define CGTT_CP_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                  0x00008000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                           0x00010000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                           0x00020000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                           0x00040000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                           0x00080000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                           0x00100000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                           0x00200000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                           0x00400000L
+#define CGTT_CP_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                           0x00800000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                          0x20000000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                              0x40000000L
+#define CGTT_CP_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                              0x80000000L
+//CGTT_CPF_CLK_CTRL
+#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_CPF_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                               0xf
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1a
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PRT__SHIFT                                                           0x1b
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_CMP__SHIFT                                                           0x1c
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_GFX__SHIFT                                                           0x1d
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
+#define CGTT_CPF_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_CPF_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                 0x00008000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_CPF_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x04000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_PRT_MASK                                                             0x08000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_CMP_MASK                                                             0x10000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_GFX_MASK                                                             0x20000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+#define CGTT_CPF_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
+//CGTT_CPC_CLK_CTRL
+#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE__SHIFT                                                               0xf
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7__SHIFT                                                        0x10
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6__SHIFT                                                        0x11
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5__SHIFT                                                        0x12
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4__SHIFT                                                        0x13
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3__SHIFT                                                        0x14
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2__SHIFT                                                        0x15
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1__SHIFT                                                        0x16
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0__SHIFT                                                        0x17
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                       0x1d
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN__SHIFT                                                           0x1e
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG__SHIFT                                                           0x1f
+#define CGTT_CPC_CLK_CTRL__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_CPC_CLK_CTRL__MGLS_OVERRIDE_MASK                                                                 0x00008000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE7_MASK                                                          0x00010000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE6_MASK                                                          0x00020000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE5_MASK                                                          0x00040000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE4_MASK                                                          0x00080000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE3_MASK                                                          0x00100000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE2_MASK                                                          0x00200000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE1_MASK                                                          0x00400000L
+#define CGTT_CPC_CLK_CTRL__SOFT_STALL_OVERRIDE0_MASK                                                          0x00800000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                         0x20000000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_DYN_MASK                                                             0x40000000L
+#define CGTT_CPC_CLK_CTRL__SOFT_OVERRIDE_REG_MASK                                                             0x80000000L
+//CGTT_RLC_CLK_CTRL
+#define CGTT_RLC_CLK_CTRL__RESERVED__SHIFT                                                                    0x0
+#define CGTT_RLC_CLK_CTRL__RESERVED_MASK                                                                      0xFFFFFFFFL
+//CGTT_SC_CLK_CTRL3
+#define CGTT_SC_CLK_CTRL3__PBB_WARPBINROWWARP_CLK_STALL_OVERRIDE__SHIFT                                       0x0
+#define CGTT_SC_CLK_CTRL3__PBB_WARPBINWARP_CLK_STALL_OVERRIDE__SHIFT                                          0x1
+#define CGTT_SC_CLK_CTRL3__PBB_WARPFBWBINWARP_CLK_STALL_OVERRIDE__SHIFT                                       0x2
+#define CGTT_SC_CLK_CTRL3__PBB_WARPHSPANUNWARP_CLK_STALL_OVERRIDE__SHIFT                                      0x3
+#define CGTT_SC_CLK_CTRL3__PBB_WARPSCISSORUNWARP_CLK_STALL_OVERRIDE__SHIFT                                    0x4
+#define CGTT_SC_CLK_CTRL3__PBB_FBWBACK_CLK_STALL_OVERRIDE__SHIFT                                              0x5
+#define CGTT_SC_CLK_CTRL3__PBB_FBWBACKREPEATER_CLK_STALL_OVERRIDE__SHIFT                                      0x6
+#define CGTT_SC_CLK_CTRL3__PBB_FBWFRONT_CLK_STALL_OVERRIDE__SHIFT                                             0x7
+#define CGTT_SC_CLK_CTRL3__PBB_FBWFRONTREPEATER_CLK_STALL_OVERRIDE__SHIFT                                     0x8
+#define CGTT_SC_CLK_CTRL3__PBB_FBWSCALER_CLK_STALL_OVERRIDE__SHIFT                                            0x9
+#define CGTT_SC_CLK_CTRL3__PBB_FRONT_CLK_STALL_OVERRIDE__SHIFT                                                0xa
+#define CGTT_SC_CLK_CTRL3__PBB_BATCHIN_CLK_STALL_OVERRIDE__SHIFT                                              0xb
+#define CGTT_SC_CLK_CTRL3__PBB_VRASTER_CLK_STALL_OVERRIDE__SHIFT                                              0xc
+#define CGTT_SC_CLK_CTRL3__PBB_VGATHER_CLK_STALL_OVERRIDE__SHIFT                                              0xd
+#define CGTT_SC_CLK_CTRL3__PBB_WARPBINROWWARP_CLK_OVERRIDE__SHIFT                                             0x12
+#define CGTT_SC_CLK_CTRL3__PBB_WARPBINWARP_CLK_OVERRIDE__SHIFT                                                0x13
+#define CGTT_SC_CLK_CTRL3__PBB_WARPFBWBINWARP_CLK_OVERRIDE__SHIFT                                             0x14
+#define CGTT_SC_CLK_CTRL3__PBB_WARPHSPANUNWARP_CLK_OVERRIDE__SHIFT                                            0x15
+#define CGTT_SC_CLK_CTRL3__PBB_WARPSCISSORUNWARP_CLK_OVERRIDE__SHIFT                                          0x16
+#define CGTT_SC_CLK_CTRL3__PBB_FBWBACK_CLK_OVERRIDE__SHIFT                                                    0x17
+#define CGTT_SC_CLK_CTRL3__PBB_FBWBACKREPEATER_CLK_OVERRIDE__SHIFT                                            0x18
+#define CGTT_SC_CLK_CTRL3__PBB_FBWFRONT_CLK_OVERRIDE__SHIFT                                                   0x19
+#define CGTT_SC_CLK_CTRL3__PBB_FBWFRONTREPEATER_CLK_OVERRIDE__SHIFT                                           0x1a
+#define CGTT_SC_CLK_CTRL3__PBB_FBWSCALER_CLK_OVERRIDE__SHIFT                                                  0x1b
+#define CGTT_SC_CLK_CTRL3__PBB_FRONT_CLK_OVERRIDE__SHIFT                                                      0x1c
+#define CGTT_SC_CLK_CTRL3__PBB_BATCHIN_CLK_OVERRIDE__SHIFT                                                    0x1d
+#define CGTT_SC_CLK_CTRL3__PBB_VRASTER_CLK_OVERRIDE__SHIFT                                                    0x1e
+#define CGTT_SC_CLK_CTRL3__PBB_VGATHER_CLK_OVERRIDE__SHIFT                                                    0x1f
+#define CGTT_SC_CLK_CTRL3__PBB_WARPBINROWWARP_CLK_STALL_OVERRIDE_MASK                                         0x00000001L
+#define CGTT_SC_CLK_CTRL3__PBB_WARPBINWARP_CLK_STALL_OVERRIDE_MASK                                            0x00000002L
+#define CGTT_SC_CLK_CTRL3__PBB_WARPFBWBINWARP_CLK_STALL_OVERRIDE_MASK                                         0x00000004L
+#define CGTT_SC_CLK_CTRL3__PBB_WARPHSPANUNWARP_CLK_STALL_OVERRIDE_MASK                                        0x00000008L
+#define CGTT_SC_CLK_CTRL3__PBB_WARPSCISSORUNWARP_CLK_STALL_OVERRIDE_MASK                                      0x00000010L
+#define CGTT_SC_CLK_CTRL3__PBB_FBWBACK_CLK_STALL_OVERRIDE_MASK                                                0x00000020L
+#define CGTT_SC_CLK_CTRL3__PBB_FBWBACKREPEATER_CLK_STALL_OVERRIDE_MASK                                        0x00000040L
+#define CGTT_SC_CLK_CTRL3__PBB_FBWFRONT_CLK_STALL_OVERRIDE_MASK                                               0x00000080L
+#define CGTT_SC_CLK_CTRL3__PBB_FBWFRONTREPEATER_CLK_STALL_OVERRIDE_MASK                                       0x00000100L
+#define CGTT_SC_CLK_CTRL3__PBB_FBWSCALER_CLK_STALL_OVERRIDE_MASK                                              0x00000200L
+#define CGTT_SC_CLK_CTRL3__PBB_FRONT_CLK_STALL_OVERRIDE_MASK                                                  0x00000400L
+#define CGTT_SC_CLK_CTRL3__PBB_BATCHIN_CLK_STALL_OVERRIDE_MASK                                                0x00000800L
+#define CGTT_SC_CLK_CTRL3__PBB_VRASTER_CLK_STALL_OVERRIDE_MASK                                                0x00001000L
+#define CGTT_SC_CLK_CTRL3__PBB_VGATHER_CLK_STALL_OVERRIDE_MASK                                                0x00002000L
+#define CGTT_SC_CLK_CTRL3__PBB_WARPBINROWWARP_CLK_OVERRIDE_MASK                                               0x00040000L
+#define CGTT_SC_CLK_CTRL3__PBB_WARPBINWARP_CLK_OVERRIDE_MASK                                                  0x00080000L
+#define CGTT_SC_CLK_CTRL3__PBB_WARPFBWBINWARP_CLK_OVERRIDE_MASK                                               0x00100000L
+#define CGTT_SC_CLK_CTRL3__PBB_WARPHSPANUNWARP_CLK_OVERRIDE_MASK                                              0x00200000L
+#define CGTT_SC_CLK_CTRL3__PBB_WARPSCISSORUNWARP_CLK_OVERRIDE_MASK                                            0x00400000L
+#define CGTT_SC_CLK_CTRL3__PBB_FBWBACK_CLK_OVERRIDE_MASK                                                      0x00800000L
+#define CGTT_SC_CLK_CTRL3__PBB_FBWBACKREPEATER_CLK_OVERRIDE_MASK                                              0x01000000L
+#define CGTT_SC_CLK_CTRL3__PBB_FBWFRONT_CLK_OVERRIDE_MASK                                                     0x02000000L
+#define CGTT_SC_CLK_CTRL3__PBB_FBWFRONTREPEATER_CLK_OVERRIDE_MASK                                             0x04000000L
+#define CGTT_SC_CLK_CTRL3__PBB_FBWSCALER_CLK_OVERRIDE_MASK                                                    0x08000000L
+#define CGTT_SC_CLK_CTRL3__PBB_FRONT_CLK_OVERRIDE_MASK                                                        0x10000000L
+#define CGTT_SC_CLK_CTRL3__PBB_BATCHIN_CLK_OVERRIDE_MASK                                                      0x20000000L
+#define CGTT_SC_CLK_CTRL3__PBB_VRASTER_CLK_OVERRIDE_MASK                                                      0x40000000L
+#define CGTT_SC_CLK_CTRL3__PBB_VGATHER_CLK_OVERRIDE_MASK                                                      0x80000000L
+//CGTT_SC_CLK_CTRL4
+#define CGTT_SC_CLK_CTRL4__PBB_VCOARSE_CLK_STALL_OVERRIDE__SHIFT                                              0x0
+#define CGTT_SC_CLK_CTRL4__PBB_VDETAIL_CLK_STALL_OVERRIDE__SHIFT                                              0x1
+#define CGTT_SC_CLK_CTRL4__PBB_HRASTER_CLK_STALL_OVERRIDE__SHIFT                                              0x2
+#define CGTT_SC_CLK_CTRL4__PBB_HCONFIG_CLK_STALL_OVERRIDE__SHIFT                                              0x3
+#define CGTT_SC_CLK_CTRL4__PBB_HGATHER_CLK_STALL_OVERRIDE__SHIFT                                              0x4
+#define CGTT_SC_CLK_CTRL4__PBB_HCOARSE_CLK_STALL_OVERRIDE__SHIFT                                              0x5
+#define CGTT_SC_CLK_CTRL4__PBB_HDETAIL_CLK_STALL_OVERRIDE__SHIFT                                              0x6
+#define CGTT_SC_CLK_CTRL4__PBB_HREPEAT_CLK_STALL_OVERRIDE__SHIFT                                              0x7
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHOUT_CLK_STALL_OVERRIDE__SHIFT                                             0x8
+#define CGTT_SC_CLK_CTRL4__PBB_OUTPUT_CLK_STALL_OVERRIDE__SHIFT                                               0x9
+#define CGTT_SC_CLK_CTRL4__PBB_OUTMUX_CLK_STALL_OVERRIDE__SHIFT                                               0xa
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHINFO_CLK_STALL_OVERRIDE__SHIFT                                            0xb
+#define CGTT_SC_CLK_CTRL4__PBB_EVENTINFO_CLK_STALL_OVERRIDE__SHIFT                                            0xc
+#define CGTT_SC_CLK_CTRL4__PBB_VCOARSE_CLK_OVERRIDE__SHIFT                                                    0x13
+#define CGTT_SC_CLK_CTRL4__PBB_VDETAIL_CLK_OVERRIDE__SHIFT                                                    0x14
+#define CGTT_SC_CLK_CTRL4__PBB_HRASTER_CLK_OVERRIDE__SHIFT                                                    0x15
+#define CGTT_SC_CLK_CTRL4__PBB_HCONFIG_CLK_OVERRIDE__SHIFT                                                    0x16
+#define CGTT_SC_CLK_CTRL4__PBB_HGATHER_CLK_OVERRIDE__SHIFT                                                    0x17
+#define CGTT_SC_CLK_CTRL4__PBB_HCOARSE_CLK_OVERRIDE__SHIFT                                                    0x18
+#define CGTT_SC_CLK_CTRL4__PBB_HDETAIL_CLK_OVERRIDE__SHIFT                                                    0x19
+#define CGTT_SC_CLK_CTRL4__PBB_HREPEAT_CLK_OVERRIDE__SHIFT                                                    0x1a
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHOUT_CLK_OVERRIDE__SHIFT                                                   0x1b
+#define CGTT_SC_CLK_CTRL4__PBB_OUTPUT_CLK_OVERRIDE__SHIFT                                                     0x1c
+#define CGTT_SC_CLK_CTRL4__PBB_OUTMUX_CLK_OVERRIDE__SHIFT                                                     0x1d
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHINFO_CLK_OVERRIDE__SHIFT                                                  0x1e
+#define CGTT_SC_CLK_CTRL4__PBB_EVENTINFO_CLK_OVERRIDE__SHIFT                                                  0x1f
+#define CGTT_SC_CLK_CTRL4__PBB_VCOARSE_CLK_STALL_OVERRIDE_MASK                                                0x00000001L
+#define CGTT_SC_CLK_CTRL4__PBB_VDETAIL_CLK_STALL_OVERRIDE_MASK                                                0x00000002L
+#define CGTT_SC_CLK_CTRL4__PBB_HRASTER_CLK_STALL_OVERRIDE_MASK                                                0x00000004L
+#define CGTT_SC_CLK_CTRL4__PBB_HCONFIG_CLK_STALL_OVERRIDE_MASK                                                0x00000008L
+#define CGTT_SC_CLK_CTRL4__PBB_HGATHER_CLK_STALL_OVERRIDE_MASK                                                0x00000010L
+#define CGTT_SC_CLK_CTRL4__PBB_HCOARSE_CLK_STALL_OVERRIDE_MASK                                                0x00000020L
+#define CGTT_SC_CLK_CTRL4__PBB_HDETAIL_CLK_STALL_OVERRIDE_MASK                                                0x00000040L
+#define CGTT_SC_CLK_CTRL4__PBB_HREPEAT_CLK_STALL_OVERRIDE_MASK                                                0x00000080L
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHOUT_CLK_STALL_OVERRIDE_MASK                                               0x00000100L
+#define CGTT_SC_CLK_CTRL4__PBB_OUTPUT_CLK_STALL_OVERRIDE_MASK                                                 0x00000200L
+#define CGTT_SC_CLK_CTRL4__PBB_OUTMUX_CLK_STALL_OVERRIDE_MASK                                                 0x00000400L
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHINFO_CLK_STALL_OVERRIDE_MASK                                              0x00000800L
+#define CGTT_SC_CLK_CTRL4__PBB_EVENTINFO_CLK_STALL_OVERRIDE_MASK                                              0x00001000L
+#define CGTT_SC_CLK_CTRL4__PBB_VCOARSE_CLK_OVERRIDE_MASK                                                      0x00080000L
+#define CGTT_SC_CLK_CTRL4__PBB_VDETAIL_CLK_OVERRIDE_MASK                                                      0x00100000L
+#define CGTT_SC_CLK_CTRL4__PBB_HRASTER_CLK_OVERRIDE_MASK                                                      0x00200000L
+#define CGTT_SC_CLK_CTRL4__PBB_HCONFIG_CLK_OVERRIDE_MASK                                                      0x00400000L
+#define CGTT_SC_CLK_CTRL4__PBB_HGATHER_CLK_OVERRIDE_MASK                                                      0x00800000L
+#define CGTT_SC_CLK_CTRL4__PBB_HCOARSE_CLK_OVERRIDE_MASK                                                      0x01000000L
+#define CGTT_SC_CLK_CTRL4__PBB_HDETAIL_CLK_OVERRIDE_MASK                                                      0x02000000L
+#define CGTT_SC_CLK_CTRL4__PBB_HREPEAT_CLK_OVERRIDE_MASK                                                      0x04000000L
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHOUT_CLK_OVERRIDE_MASK                                                     0x08000000L
+#define CGTT_SC_CLK_CTRL4__PBB_OUTPUT_CLK_OVERRIDE_MASK                                                       0x10000000L
+#define CGTT_SC_CLK_CTRL4__PBB_OUTMUX_CLK_OVERRIDE_MASK                                                       0x20000000L
+#define CGTT_SC_CLK_CTRL4__PBB_BATCHINFO_CLK_OVERRIDE_MASK                                                    0x40000000L
+#define CGTT_SC_CLK_CTRL4__PBB_EVENTINFO_CLK_OVERRIDE_MASK                                                    0x80000000L
+//GCEA_ICG_CTRL
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_RETURN__SHIFT                                                            0x0
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_READ__SHIFT                                                              0x1
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_WRITE__SHIFT                                                             0x2
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                          0x3
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                           0x4
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_MAM__SHIFT                                                               0x5
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_RETURN_MASK                                                              0x00000001L
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_READ_MASK                                                                0x00000002L
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_WRITE_MASK                                                               0x00000004L
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                            0x00000008L
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                             0x00000010L
+#define GCEA_ICG_CTRL__SOFT_OVERRIDE_MAM_MASK                                                                 0x00000020L
+//GL1I_GL1R_MGCG_OVERRIDE
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IR_MGCG_SCLK_OVERRIDE__SHIFT                                         0x0
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IR_MGCG_RET_DCLK_OVERRIDE__SHIFT                                     0x1
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IW_MGCG_SCLK_OVERRIDE__SHIFT                                         0x2
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IW_MGCG_RET_DCLK_OVERRIDE__SHIFT                                     0x3
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IW_MGCG_SRC_DCLK_OVERRIDE__SHIFT                                     0x4
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1R_SRC_MGCG_SCLK_OVERRIDE__SHIFT                                      0x5
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1R_RET_MGCG_SCLK_OVERRIDE__SHIFT                                      0x6
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IR_MGCG_SCLK_OVERRIDE_MASK                                           0x00000001L
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IR_MGCG_RET_DCLK_OVERRIDE_MASK                                       0x00000002L
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IW_MGCG_SCLK_OVERRIDE_MASK                                           0x00000004L
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IW_MGCG_RET_DCLK_OVERRIDE_MASK                                       0x00000008L
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1IW_MGCG_SRC_DCLK_OVERRIDE_MASK                                       0x00000010L
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1R_SRC_MGCG_SCLK_OVERRIDE_MASK                                        0x00000020L
+#define GL1I_GL1R_MGCG_OVERRIDE__GL1A_GL1R_RET_MGCG_SCLK_OVERRIDE_MASK                                        0x00000040L
+//GL1H_ICG_CTRL
+#define GL1H_ICG_CTRL__REG_DCLK_OVERRIDE__SHIFT                                                               0x0
+#define GL1H_ICG_CTRL__REQ_ARB_DCLK_OVERRIDE__SHIFT                                                           0x1
+#define GL1H_ICG_CTRL__PERFMON_DCLK_OVERRIDE__SHIFT                                                           0x2
+#define GL1H_ICG_CTRL__REQ_ARB_CLI0_DCLK_OVERRIDE__SHIFT                                                      0x3
+#define GL1H_ICG_CTRL__REQ_ARB_CLI1_DCLK_OVERRIDE__SHIFT                                                      0x4
+#define GL1H_ICG_CTRL__REQ_ARB_CLI2_DCLK_OVERRIDE__SHIFT                                                      0x5
+#define GL1H_ICG_CTRL__REQ_ARB_CLI3_DCLK_OVERRIDE__SHIFT                                                      0x6
+#define GL1H_ICG_CTRL__SRC_DCLK_OVERRIDE__SHIFT                                                               0x7
+#define GL1H_ICG_CTRL__RET_DCLK_OVERRIDE__SHIFT                                                               0x8
+#define GL1H_ICG_CTRL__REG_DCLK_OVERRIDE_MASK                                                                 0x00000001L
+#define GL1H_ICG_CTRL__REQ_ARB_DCLK_OVERRIDE_MASK                                                             0x00000002L
+#define GL1H_ICG_CTRL__PERFMON_DCLK_OVERRIDE_MASK                                                             0x00000004L
+#define GL1H_ICG_CTRL__REQ_ARB_CLI0_DCLK_OVERRIDE_MASK                                                        0x00000008L
+#define GL1H_ICG_CTRL__REQ_ARB_CLI1_DCLK_OVERRIDE_MASK                                                        0x00000010L
+#define GL1H_ICG_CTRL__REQ_ARB_CLI2_DCLK_OVERRIDE_MASK                                                        0x00000020L
+#define GL1H_ICG_CTRL__REQ_ARB_CLI3_DCLK_OVERRIDE_MASK                                                        0x00000040L
+#define GL1H_ICG_CTRL__SRC_DCLK_OVERRIDE_MASK                                                                 0x00000080L
+#define GL1H_ICG_CTRL__RET_DCLK_OVERRIDE_MASK                                                                 0x00000100L
+//CHI_CHR_MGCG_OVERRIDE
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIR_MGCG_SCLK_OVERRIDE__SHIFT                                             0x0
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIR_MGCG_RET_DCLK_OVERRIDE__SHIFT                                         0x1
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIW_MGCG_SCLK_OVERRIDE__SHIFT                                             0x2
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIW_MGCG_RET_DCLK_OVERRIDE__SHIFT                                         0x3
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIW_MGCG_SRC_DCLK_OVERRIDE__SHIFT                                         0x4
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHR_RET_MGCG_SCLK_OVERRIDE__SHIFT                                          0x5
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHR_SRC_MGCG_SCLK_OVERRIDE__SHIFT                                          0x6
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIR_MGCG_SCLK_OVERRIDE_MASK                                               0x00000001L
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIR_MGCG_RET_DCLK_OVERRIDE_MASK                                           0x00000002L
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIW_MGCG_SCLK_OVERRIDE_MASK                                               0x00000004L
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIW_MGCG_RET_DCLK_OVERRIDE_MASK                                           0x00000008L
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHIW_MGCG_SRC_DCLK_OVERRIDE_MASK                                           0x00000010L
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHR_RET_MGCG_SCLK_OVERRIDE_MASK                                            0x00000020L
+#define CHI_CHR_MGCG_OVERRIDE__CHA_CHR_SRC_MGCG_SCLK_OVERRIDE_MASK                                            0x00000040L
+//ICG_GL1C_CLK_CTRL
+#define ICG_GL1C_CLK_CTRL__GLOBAL_CLK_OVERRIDE__SHIFT                                                         0x0
+#define ICG_GL1C_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE__SHIFT                                          0x1
+#define ICG_GL1C_CLK_CTRL__REQUEST_CLK_OVERRIDE__SHIFT                                                        0x2
+#define ICG_GL1C_CLK_CTRL__VM_CLK_OVERRIDE__SHIFT                                                             0x3
+#define ICG_GL1C_CLK_CTRL__TAG_CLK_OVERRIDE__SHIFT                                                            0x4
+#define ICG_GL1C_CLK_CTRL__GCR_CLK_OVERRIDE__SHIFT                                                            0x5
+#define ICG_GL1C_CLK_CTRL__SRC_DATA_CLK_OVERRIDE__SHIFT                                                       0x6
+#define ICG_GL1C_CLK_CTRL__RETURN_CLK_OVERRIDE__SHIFT                                                         0x7
+#define ICG_GL1C_CLK_CTRL__GRBM_CLK_OVERRIDE__SHIFT                                                           0x8
+#define ICG_GL1C_CLK_CTRL__PERF_CLK_OVERRIDE__SHIFT                                                           0x9
+#define ICG_GL1C_CLK_CTRL__LATENCY_FIFO_CLK_OVERRIDE__SHIFT                                                   0xa
+#define ICG_GL1C_CLK_CTRL__GLOBAL_CLK_OVERRIDE_MASK                                                           0x00000001L
+#define ICG_GL1C_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE_MASK                                            0x00000002L
+#define ICG_GL1C_CLK_CTRL__REQUEST_CLK_OVERRIDE_MASK                                                          0x00000004L
+#define ICG_GL1C_CLK_CTRL__VM_CLK_OVERRIDE_MASK                                                               0x00000008L
+#define ICG_GL1C_CLK_CTRL__TAG_CLK_OVERRIDE_MASK                                                              0x00000010L
+#define ICG_GL1C_CLK_CTRL__GCR_CLK_OVERRIDE_MASK                                                              0x00000020L
+#define ICG_GL1C_CLK_CTRL__SRC_DATA_CLK_OVERRIDE_MASK                                                         0x00000040L
+#define ICG_GL1C_CLK_CTRL__RETURN_CLK_OVERRIDE_MASK                                                           0x00000080L
+#define ICG_GL1C_CLK_CTRL__GRBM_CLK_OVERRIDE_MASK                                                             0x00000100L
+#define ICG_GL1C_CLK_CTRL__PERF_CLK_OVERRIDE_MASK                                                             0x00000200L
+#define ICG_GL1C_CLK_CTRL__LATENCY_FIFO_CLK_OVERRIDE_MASK                                                     0x00000400L
+//ICG_GL1A_CTRL
+#define ICG_GL1A_CTRL__REG_CLK_OVERRIDE__SHIFT                                                                0x0
+#define ICG_GL1A_CTRL__REQ_CLI_CLK_OVERRIDE__SHIFT                                                            0x1
+#define ICG_GL1A_CTRL__REQ_ARB_CLK_OVERRIDE__SHIFT                                                            0x2
+#define ICG_GL1A_CTRL__RET_CLK_OVERRIDE__SHIFT                                                                0x3
+#define ICG_GL1A_CTRL__REQ_CREDIT_CLK_OVERRIDE__SHIFT                                                         0x4
+#define ICG_GL1A_CTRL__PERFMON_CLK_OVERRIDE__SHIFT                                                            0x5
+#define ICG_GL1A_CTRL__REG_CLK_OVERRIDE_MASK                                                                  0x00000001L
+#define ICG_GL1A_CTRL__REQ_CLI_CLK_OVERRIDE_MASK                                                              0x00000002L
+#define ICG_GL1A_CTRL__REQ_ARB_CLK_OVERRIDE_MASK                                                              0x00000004L
+#define ICG_GL1A_CTRL__RET_CLK_OVERRIDE_MASK                                                                  0x00000008L
+#define ICG_GL1A_CTRL__REQ_CREDIT_CLK_OVERRIDE_MASK                                                           0x00000010L
+#define ICG_GL1A_CTRL__PERFMON_CLK_OVERRIDE_MASK                                                              0x00000020L
+//ICG_CHA_CTRL
+#define ICG_CHA_CTRL__REG_CLK_OVERRIDE__SHIFT                                                                 0x0
+#define ICG_CHA_CTRL__REQ_CLI_CLK_OVERRIDE__SHIFT                                                             0x1
+#define ICG_CHA_CTRL__REQ_ARB_CLK_OVERRIDE__SHIFT                                                             0x2
+#define ICG_CHA_CTRL__RET_CLK_OVERRIDE__SHIFT                                                                 0x3
+#define ICG_CHA_CTRL__REQ_CREDIT_CLK_OVERRIDE__SHIFT                                                          0x4
+#define ICG_CHA_CTRL__PERFMON_CLK_OVERRIDE__SHIFT                                                             0x5
+#define ICG_CHA_CTRL__REG_CLK_OVERRIDE_MASK                                                                   0x00000001L
+#define ICG_CHA_CTRL__REQ_CLI_CLK_OVERRIDE_MASK                                                               0x00000002L
+#define ICG_CHA_CTRL__REQ_ARB_CLK_OVERRIDE_MASK                                                               0x00000004L
+#define ICG_CHA_CTRL__RET_CLK_OVERRIDE_MASK                                                                   0x00000008L
+#define ICG_CHA_CTRL__REQ_CREDIT_CLK_OVERRIDE_MASK                                                            0x00000010L
+#define ICG_CHA_CTRL__PERFMON_CLK_OVERRIDE_MASK                                                               0x00000020L
+//GUS_ICG_CTRL
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_DRAM__SHIFT                                                               0x0
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_WRITE__SHIFT                                                              0x1
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_READ__SHIFT                                                               0x2
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_RETURN_DEMUX__SHIFT                                                       0x3
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_RETURN_WRITE__SHIFT                                                       0x4
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_RETURN_READ__SHIFT                                                        0x5
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_REGISTER__SHIFT                                                           0x6
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_PERFMON__SHIFT                                                            0x7
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_STATIC__SHIFT                                                             0x8
+#define GUS_ICG_CTRL__SPARE1__SHIFT                                                                           0x9
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_DRAM_MASK                                                                 0x00000001L
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_WRITE_MASK                                                                0x00000002L
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_READ_MASK                                                                 0x00000004L
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_RETURN_DEMUX_MASK                                                         0x00000008L
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_RETURN_WRITE_MASK                                                         0x00000010L
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_RETURN_READ_MASK                                                          0x00000020L
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_REGISTER_MASK                                                             0x00000040L
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_PERFMON_MASK                                                              0x00000080L
+#define GUS_ICG_CTRL__SOFT_OVERRIDE_STATIC_MASK                                                               0x00000100L
+#define GUS_ICG_CTRL__SPARE1_MASK                                                                             0x0003FE00L
+//CGTT_PH_CLK_CTRL0
+#define CGTT_PH_CLK_CTRL0__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL0__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL0__DEBUG_BUS_EN__SHIFT                                                                0x17
+#define CGTT_PH_CLK_CTRL0__DISABLE_DEBUG_BUS_FLOP_EN_ON_PERFMON__SHIFT                                        0x18
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL0__PERFMON_CLK_OVERRIDE__SHIFT                                                        0x1e
+#define CGTT_PH_CLK_CTRL0__REG_CLK_OVERRIDE__SHIFT                                                            0x1f
+#define CGTT_PH_CLK_CTRL0__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL0__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL0__DEBUG_BUS_EN_MASK                                                                  0x00800000L
+#define CGTT_PH_CLK_CTRL0__DISABLE_DEBUG_BUS_FLOP_EN_ON_PERFMON_MASK                                          0x01000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL0__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL0__PERFMON_CLK_OVERRIDE_MASK                                                          0x40000000L
+#define CGTT_PH_CLK_CTRL0__REG_CLK_OVERRIDE_MASK                                                              0x80000000L
+//CGTT_PH_CLK_CTRL1
+#define CGTT_PH_CLK_CTRL1__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL1__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_PH_CLK_CTRL1__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL1__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL1__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+//CGTT_PH_CLK_CTRL2
+#define CGTT_PH_CLK_CTRL2__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL2__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_PH_CLK_CTRL2__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL2__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL2__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+//CGTT_PH_CLK_CTRL3
+#define CGTT_PH_CLK_CTRL3__ON_DELAY__SHIFT                                                                    0x0
+#define CGTT_PH_CLK_CTRL3__OFF_HYSTERESIS__SHIFT                                                              0x4
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE7__SHIFT                                                              0x18
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE6__SHIFT                                                              0x19
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE5__SHIFT                                                              0x1a
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE4__SHIFT                                                              0x1b
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE3__SHIFT                                                              0x1c
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE2__SHIFT                                                              0x1d
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE1__SHIFT                                                              0x1e
+#define CGTT_PH_CLK_CTRL3__ON_DELAY_MASK                                                                      0x0000000FL
+#define CGTT_PH_CLK_CTRL3__OFF_HYSTERESIS_MASK                                                                0x00000FF0L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE7_MASK                                                                0x01000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE6_MASK                                                                0x02000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE5_MASK                                                                0x04000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE4_MASK                                                                0x08000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE3_MASK                                                                0x10000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE2_MASK                                                                0x20000000L
+#define CGTT_PH_CLK_CTRL3__SOFT_OVERRIDE1_MASK                                                                0x40000000L
+//GFX_ICG_GL2C_CTRL
+#define GFX_ICG_GL2C_CTRL__REG_OVERRIDE__SHIFT                                                                0x0
+#define GFX_ICG_GL2C_CTRL__PERFMON_OVERRIDE__SHIFT                                                            0x1
+#define GFX_ICG_GL2C_CTRL__IB_OVERRIDE__SHIFT                                                                 0x2
+#define GFX_ICG_GL2C_CTRL__TAG_OVERRIDE__SHIFT                                                                0x3
+#define GFX_ICG_GL2C_CTRL__CM_CORE_OVERRIDE__SHIFT                                                            0x4
+#define GFX_ICG_GL2C_CTRL__CORE_OVERRIDE__SHIFT                                                               0x5
+#define GFX_ICG_GL2C_CTRL__CACHE_RAM_OVERRIDE__SHIFT                                                          0x6
+#define GFX_ICG_GL2C_CTRL__GCR_OVERRIDE__SHIFT                                                                0x7
+#define GFX_ICG_GL2C_CTRL__EXECUTE_OVERRIDE__SHIFT                                                            0x8
+#define GFX_ICG_GL2C_CTRL__RETURN_BUFFER_OVERRIDE__SHIFT                                                      0x9
+#define GFX_ICG_GL2C_CTRL__LATENCY_FIFO_OVERRIDE__SHIFT                                                       0xa
+#define GFX_ICG_GL2C_CTRL__OUTPUT_FIFOS_OVERRIDE__SHIFT                                                       0xb
+#define GFX_ICG_GL2C_CTRL__MC_WRITE_OVERRIDE__SHIFT                                                           0xc
+#define GFX_ICG_GL2C_CTRL__EXECUTE_DECOMP_OVERRIDE__SHIFT                                                     0xd
+#define GFX_ICG_GL2C_CTRL__EXECUTE_WRITE_OVERRIDE__SHIFT                                                      0xe
+#define GFX_ICG_GL2C_CTRL__TAG_FLOPSET_GROUP0_OVERRIDE__SHIFT                                                 0xf
+#define GFX_ICG_GL2C_CTRL__TAG_FLOPSET_GROUP1_OVERRIDE__SHIFT                                                 0x10
+#define GFX_ICG_GL2C_CTRL__TAG_FLOPSET_GROUP2_OVERRIDE__SHIFT                                                 0x11
+#define GFX_ICG_GL2C_CTRL__TAG_FLOPSET_GROUP3_OVERRIDE__SHIFT                                                 0x12
+#define GFX_ICG_GL2C_CTRL__CM_RVF_OVERRIDE__SHIFT                                                             0x14
+#define GFX_ICG_GL2C_CTRL__CM_SDR_OVERRIDE__SHIFT                                                             0x15
+#define GFX_ICG_GL2C_CTRL__CM_RPF_OVERRIDE__SHIFT                                                             0x16
+#define GFX_ICG_GL2C_CTRL__CM_STS_OVERRIDE__SHIFT                                                             0x17
+#define GFX_ICG_GL2C_CTRL__CM_READ_OVERRIDE__SHIFT                                                            0x18
+#define GFX_ICG_GL2C_CTRL__CM_MERGE_OVERRIDE__SHIFT                                                           0x19
+#define GFX_ICG_GL2C_CTRL__CM_COMP_OVERRIDE__SHIFT                                                            0x1a
+#define GFX_ICG_GL2C_CTRL__CM_DCC_OVERRIDE__SHIFT                                                             0x1b
+#define GFX_ICG_GL2C_CTRL__CM_WRITE_OVERRIDE__SHIFT                                                           0x1c
+#define GFX_ICG_GL2C_CTRL__CM_NOOP_OVERRIDE__SHIFT                                                            0x1d
+#define GFX_ICG_GL2C_CTRL__MDC_TAG_OVERRIDE__SHIFT                                                            0x1e
+#define GFX_ICG_GL2C_CTRL__MDC_DATA_OVERRIDE__SHIFT                                                           0x1f
+#define GFX_ICG_GL2C_CTRL__REG_OVERRIDE_MASK                                                                  0x00000001L
+#define GFX_ICG_GL2C_CTRL__PERFMON_OVERRIDE_MASK                                                              0x00000002L
+#define GFX_ICG_GL2C_CTRL__IB_OVERRIDE_MASK                                                                   0x00000004L
+#define GFX_ICG_GL2C_CTRL__TAG_OVERRIDE_MASK                                                                  0x00000008L
+#define GFX_ICG_GL2C_CTRL__CM_CORE_OVERRIDE_MASK                                                              0x00000010L
+#define GFX_ICG_GL2C_CTRL__CORE_OVERRIDE_MASK                                                                 0x00000020L
+#define GFX_ICG_GL2C_CTRL__CACHE_RAM_OVERRIDE_MASK                                                            0x00000040L
+#define GFX_ICG_GL2C_CTRL__GCR_OVERRIDE_MASK                                                                  0x00000080L
+#define GFX_ICG_GL2C_CTRL__EXECUTE_OVERRIDE_MASK                                                              0x00000100L
+#define GFX_ICG_GL2C_CTRL__RETURN_BUFFER_OVERRIDE_MASK                                                        0x00000200L
+#define GFX_ICG_GL2C_CTRL__LATENCY_FIFO_OVERRIDE_MASK                                                         0x00000400L
+#define GFX_ICG_GL2C_CTRL__OUTPUT_FIFOS_OVERRIDE_MASK                                                         0x00000800L
+#define GFX_ICG_GL2C_CTRL__MC_WRITE_OVERRIDE_MASK                                                             0x00001000L
+#define GFX_ICG_GL2C_CTRL__EXECUTE_DECOMP_OVERRIDE_MASK                                                       0x00002000L
+#define GFX_ICG_GL2C_CTRL__EXECUTE_WRITE_OVERRIDE_MASK                                                        0x00004000L
+#define GFX_ICG_GL2C_CTRL__TAG_FLOPSET_GROUP0_OVERRIDE_MASK                                                   0x00008000L
+#define GFX_ICG_GL2C_CTRL__TAG_FLOPSET_GROUP1_OVERRIDE_MASK                                                   0x00010000L
+#define GFX_ICG_GL2C_CTRL__TAG_FLOPSET_GROUP2_OVERRIDE_MASK                                                   0x00020000L
+#define GFX_ICG_GL2C_CTRL__TAG_FLOPSET_GROUP3_OVERRIDE_MASK                                                   0x00040000L
+#define GFX_ICG_GL2C_CTRL__CM_RVF_OVERRIDE_MASK                                                               0x00100000L
+#define GFX_ICG_GL2C_CTRL__CM_SDR_OVERRIDE_MASK                                                               0x00200000L
+#define GFX_ICG_GL2C_CTRL__CM_RPF_OVERRIDE_MASK                                                               0x00400000L
+#define GFX_ICG_GL2C_CTRL__CM_STS_OVERRIDE_MASK                                                               0x00800000L
+#define GFX_ICG_GL2C_CTRL__CM_READ_OVERRIDE_MASK                                                              0x01000000L
+#define GFX_ICG_GL2C_CTRL__CM_MERGE_OVERRIDE_MASK                                                             0x02000000L
+#define GFX_ICG_GL2C_CTRL__CM_COMP_OVERRIDE_MASK                                                              0x04000000L
+#define GFX_ICG_GL2C_CTRL__CM_DCC_OVERRIDE_MASK                                                               0x08000000L
+#define GFX_ICG_GL2C_CTRL__CM_WRITE_OVERRIDE_MASK                                                             0x10000000L
+#define GFX_ICG_GL2C_CTRL__CM_NOOP_OVERRIDE_MASK                                                              0x20000000L
+#define GFX_ICG_GL2C_CTRL__MDC_TAG_OVERRIDE_MASK                                                              0x40000000L
+#define GFX_ICG_GL2C_CTRL__MDC_DATA_OVERRIDE_MASK                                                             0x80000000L
+//GFX_ICG_GL2C_CTRL1
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT0_OVERRIDE__SHIFT                                     0x0
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT1_OVERRIDE__SHIFT                                     0x1
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT2_OVERRIDE__SHIFT                                     0x2
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT3_OVERRIDE__SHIFT                                     0x3
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT4_OVERRIDE__SHIFT                                     0x4
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT5_OVERRIDE__SHIFT                                     0x5
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT6_OVERRIDE__SHIFT                                     0x6
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT7_OVERRIDE__SHIFT                                     0x7
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT8_OVERRIDE__SHIFT                                     0x8
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT9_OVERRIDE__SHIFT                                     0x9
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT10_OVERRIDE__SHIFT                                    0xa
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT11_OVERRIDE__SHIFT                                    0xb
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT12_OVERRIDE__SHIFT                                    0xc
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT13_OVERRIDE__SHIFT                                    0xd
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT14_OVERRIDE__SHIFT                                    0xe
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT15_OVERRIDE__SHIFT                                    0xf
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT16_OVERRIDE__SHIFT                                    0x10
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT17_OVERRIDE__SHIFT                                    0x11
+#define GFX_ICG_GL2C_CTRL1__TAG_PROBE_OVERRIDE__SHIFT                                                         0x18
+#define GFX_ICG_GL2C_CTRL1__DCC_UPPER_OVERRIDE__SHIFT                                                         0x19
+#define GFX_ICG_GL2C_CTRL1__DCC_LOWER_OVERRIDE__SHIFT                                                         0x1a
+#define GFX_ICG_GL2C_CTRL1__ZD_UPPER_OVERRIDE__SHIFT                                                          0x1b
+#define GFX_ICG_GL2C_CTRL1__ZD_LOWER_OVERRIDE__SHIFT                                                          0x1c
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT0_OVERRIDE_MASK                                       0x00000001L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT1_OVERRIDE_MASK                                       0x00000002L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT2_OVERRIDE_MASK                                       0x00000004L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT3_OVERRIDE_MASK                                       0x00000008L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT4_OVERRIDE_MASK                                       0x00000010L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT5_OVERRIDE_MASK                                       0x00000020L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT6_OVERRIDE_MASK                                       0x00000040L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT7_OVERRIDE_MASK                                       0x00000080L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT8_OVERRIDE_MASK                                       0x00000100L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT9_OVERRIDE_MASK                                       0x00000200L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT10_OVERRIDE_MASK                                      0x00000400L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT11_OVERRIDE_MASK                                      0x00000800L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT12_OVERRIDE_MASK                                      0x00001000L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT13_OVERRIDE_MASK                                      0x00002000L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT14_OVERRIDE_MASK                                      0x00004000L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT15_OVERRIDE_MASK                                      0x00008000L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT16_OVERRIDE_MASK                                      0x00010000L
+#define GFX_ICG_GL2C_CTRL1__OUTPUT_FIFOS_INTERNAL_CLIENT17_OVERRIDE_MASK                                      0x00020000L
+#define GFX_ICG_GL2C_CTRL1__TAG_PROBE_OVERRIDE_MASK                                                           0x01000000L
+#define GFX_ICG_GL2C_CTRL1__DCC_UPPER_OVERRIDE_MASK                                                           0x02000000L
+#define GFX_ICG_GL2C_CTRL1__DCC_LOWER_OVERRIDE_MASK                                                           0x04000000L
+#define GFX_ICG_GL2C_CTRL1__ZD_UPPER_OVERRIDE_MASK                                                            0x08000000L
+#define GFX_ICG_GL2C_CTRL1__ZD_LOWER_OVERRIDE_MASK                                                            0x10000000L
+//ICG_LDS_CLK_CTRL
+#define ICG_LDS_CLK_CTRL__LDS_DLOAD0_OVERRIDE__SHIFT                                                          0x0
+#define ICG_LDS_CLK_CTRL__LDS_DLOAD1_OVERRIDE__SHIFT                                                          0x1
+#define ICG_LDS_CLK_CTRL__LDS_WGP_ARB_OVERRIDE__SHIFT                                                         0x2
+#define ICG_LDS_CLK_CTRL__LDS_TD_OVERRIDE__SHIFT                                                              0x3
+#define ICG_LDS_CLK_CTRL__LDS_ATTR_WR_OVERRIDE__SHIFT                                                         0x4
+#define ICG_LDS_CLK_CTRL__LDS_CONFIG_REG_OVERRIDE__SHIFT                                                      0x5
+#define ICG_LDS_CLK_CTRL__LDS_IDX_PIPE_OVERRIDE__SHIFT                                                        0x6
+#define ICG_LDS_CLK_CTRL__LDS_IDX_DIR_OVERRIDE__SHIFT                                                         0x7
+#define ICG_LDS_CLK_CTRL__LDS_IDX_WR_OVERRIDE__SHIFT                                                          0x8
+#define ICG_LDS_CLK_CTRL__LDS_IDX_INPUT_QUEUE_OVERRIDE__SHIFT                                                 0x9
+#define ICG_LDS_CLK_CTRL__LDS_MEM_OVERRIDE__SHIFT                                                             0xa
+#define ICG_LDS_CLK_CTRL__LDS_IDX_OUTPUT_ALIGNER_OVERRIDE__SHIFT                                              0xb
+#define ICG_LDS_CLK_CTRL__LDS_DIR_OUTPUT_ALIGNER_OVERRIDE__SHIFT                                              0xc
+#define ICG_LDS_CLK_CTRL__LDS_IDX_BANK_CONFLICT_OVERRIDE__SHIFT                                               0xd
+#define ICG_LDS_CLK_CTRL__LDS_IDX_SCHED_INPUT_OVERRIDE__SHIFT                                                 0xe
+#define ICG_LDS_CLK_CTRL__LDS_IDX_SCHED_OUTPUT_OVERRIDE__SHIFT                                                0xf
+#define ICG_LDS_CLK_CTRL__LDS_IDX_SCHED_PIPE_OVERRIDE__SHIFT                                                  0x10
+#define ICG_LDS_CLK_CTRL__LDS_IDX_SCHEDULER_OVERRIDE__SHIFT                                                   0x11
+#define ICG_LDS_CLK_CTRL__LDS_IDX_RDRTN_OVERRIDE__SHIFT                                                       0x12
+#define ICG_LDS_CLK_CTRL__LDS_SP_DONE_OVERRIDE__SHIFT                                                         0x13
+#define ICG_LDS_CLK_CTRL__LDS_SQC_PERF_OVERRIDE__SHIFT                                                        0x14
+#define ICG_LDS_CLK_CTRL__LDS_SP_READ_OVERRIDE__SHIFT                                                         0x15
+#define ICG_LDS_CLK_CTRL__SQ_LDS_VMEMCMD_OVERRIDE__SHIFT                                                      0x16
+#define ICG_LDS_CLK_CTRL__SP_LDS_VMEMREQ_OVERRIDE__SHIFT                                                      0x17
+#define ICG_LDS_CLK_CTRL__SPI_LDS_STALL_OVERRIDE__SHIFT                                                       0x18
+#define ICG_LDS_CLK_CTRL__MEM_WR_OVERRIDE__SHIFT                                                              0x19
+#define ICG_LDS_CLK_CTRL__LDS_CLK_OVERRIDE_UNUSED__SHIFT                                                      0x1a
+#define ICG_LDS_CLK_CTRL__LDS_DLOAD0_OVERRIDE_MASK                                                            0x00000001L
+#define ICG_LDS_CLK_CTRL__LDS_DLOAD1_OVERRIDE_MASK                                                            0x00000002L
+#define ICG_LDS_CLK_CTRL__LDS_WGP_ARB_OVERRIDE_MASK                                                           0x00000004L
+#define ICG_LDS_CLK_CTRL__LDS_TD_OVERRIDE_MASK                                                                0x00000008L
+#define ICG_LDS_CLK_CTRL__LDS_ATTR_WR_OVERRIDE_MASK                                                           0x00000010L
+#define ICG_LDS_CLK_CTRL__LDS_CONFIG_REG_OVERRIDE_MASK                                                        0x00000020L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_PIPE_OVERRIDE_MASK                                                          0x00000040L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_DIR_OVERRIDE_MASK                                                           0x00000080L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_WR_OVERRIDE_MASK                                                            0x00000100L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_INPUT_QUEUE_OVERRIDE_MASK                                                   0x00000200L
+#define ICG_LDS_CLK_CTRL__LDS_MEM_OVERRIDE_MASK                                                               0x00000400L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_OUTPUT_ALIGNER_OVERRIDE_MASK                                                0x00000800L
+#define ICG_LDS_CLK_CTRL__LDS_DIR_OUTPUT_ALIGNER_OVERRIDE_MASK                                                0x00001000L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_BANK_CONFLICT_OVERRIDE_MASK                                                 0x00002000L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_SCHED_INPUT_OVERRIDE_MASK                                                   0x00004000L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_SCHED_OUTPUT_OVERRIDE_MASK                                                  0x00008000L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_SCHED_PIPE_OVERRIDE_MASK                                                    0x00010000L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_SCHEDULER_OVERRIDE_MASK                                                     0x00020000L
+#define ICG_LDS_CLK_CTRL__LDS_IDX_RDRTN_OVERRIDE_MASK                                                         0x00040000L
+#define ICG_LDS_CLK_CTRL__LDS_SP_DONE_OVERRIDE_MASK                                                           0x00080000L
+#define ICG_LDS_CLK_CTRL__LDS_SQC_PERF_OVERRIDE_MASK                                                          0x00100000L
+#define ICG_LDS_CLK_CTRL__LDS_SP_READ_OVERRIDE_MASK                                                           0x00200000L
+#define ICG_LDS_CLK_CTRL__SQ_LDS_VMEMCMD_OVERRIDE_MASK                                                        0x00400000L
+#define ICG_LDS_CLK_CTRL__SP_LDS_VMEMREQ_OVERRIDE_MASK                                                        0x00800000L
+#define ICG_LDS_CLK_CTRL__SPI_LDS_STALL_OVERRIDE_MASK                                                         0x01000000L
+#define ICG_LDS_CLK_CTRL__MEM_WR_OVERRIDE_MASK                                                                0x02000000L
+#define ICG_LDS_CLK_CTRL__LDS_CLK_OVERRIDE_UNUSED_MASK                                                        0xFC000000L
+//GFX_ICG_UTCL1_CTRL
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE0__SHIFT                                                             0x0
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE1__SHIFT                                                             0x1
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE2__SHIFT                                                             0x2
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE3__SHIFT                                                             0x3
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE4__SHIFT                                                             0x4
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE5__SHIFT                                                             0x5
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE6__SHIFT                                                             0x6
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE7__SHIFT                                                             0x7
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE8__SHIFT                                                             0x8
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE9__SHIFT                                                             0x9
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE10__SHIFT                                                            0xa
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE11__SHIFT                                                            0xb
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE12__SHIFT                                                            0xc
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE13__SHIFT                                                            0xd
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE14__SHIFT                                                            0xe
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE15_31__SHIFT                                                         0xf
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE0_MASK                                                               0x00000001L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE1_MASK                                                               0x00000002L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE2_MASK                                                               0x00000004L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE3_MASK                                                               0x00000008L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE4_MASK                                                               0x00000010L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE5_MASK                                                               0x00000020L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE6_MASK                                                               0x00000040L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE7_MASK                                                               0x00000080L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE8_MASK                                                               0x00000100L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE9_MASK                                                               0x00000200L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE10_MASK                                                              0x00000400L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE11_MASK                                                              0x00000800L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE12_MASK                                                              0x00001000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE13_MASK                                                              0x00002000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE14_MASK                                                              0x00004000L
+#define GFX_ICG_UTCL1_CTRL__SOFT_OVERRIDE15_31_MASK                                                           0xFFFF8000L
+//ICG_CHC_CLK_CTRL
+#define ICG_CHC_CLK_CTRL__GLOBAL_CLK_OVERRIDE__SHIFT                                                          0x0
+#define ICG_CHC_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE__SHIFT                                           0x1
+#define ICG_CHC_CLK_CTRL__REQUEST_CLK_OVERRIDE__SHIFT                                                         0x2
+#define ICG_CHC_CLK_CTRL__SRC_DATA_CLK_OVERRIDE__SHIFT                                                        0x3
+#define ICG_CHC_CLK_CTRL__RETURN_CLK_OVERRIDE__SHIFT                                                          0x4
+#define ICG_CHC_CLK_CTRL__GRBM_CLK_OVERRIDE__SHIFT                                                            0x5
+#define ICG_CHC_CLK_CTRL__PERF_CLK_OVERRIDE__SHIFT                                                            0x6
+#define ICG_CHC_CLK_CTRL__GLOBAL_CLK_OVERRIDE_MASK                                                            0x00000001L
+#define ICG_CHC_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE_MASK                                             0x00000002L
+#define ICG_CHC_CLK_CTRL__REQUEST_CLK_OVERRIDE_MASK                                                           0x00000004L
+#define ICG_CHC_CLK_CTRL__SRC_DATA_CLK_OVERRIDE_MASK                                                          0x00000008L
+#define ICG_CHC_CLK_CTRL__RETURN_CLK_OVERRIDE_MASK                                                            0x00000010L
+#define ICG_CHC_CLK_CTRL__GRBM_CLK_OVERRIDE_MASK                                                              0x00000020L
+#define ICG_CHC_CLK_CTRL__PERF_CLK_OVERRIDE_MASK                                                              0x00000040L
+//ICG_CHCG_CLK_CTRL
+#define ICG_CHCG_CLK_CTRL__GLOBAL_CLK_OVERRIDE__SHIFT                                                         0x0
+#define ICG_CHCG_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE__SHIFT                                          0x1
+#define ICG_CHCG_CLK_CTRL__REQUEST_CLK_OVERRIDE__SHIFT                                                        0x2
+#define ICG_CHCG_CLK_CTRL__SRC_DATA_CLK_OVERRIDE__SHIFT                                                       0x3
+#define ICG_CHCG_CLK_CTRL__RETURN_CLK_OVERRIDE__SHIFT                                                         0x4
+#define ICG_CHCG_CLK_CTRL__GRBM_CLK_OVERRIDE__SHIFT                                                           0x5
+#define ICG_CHCG_CLK_CTRL__PERF_CLK_OVERRIDE__SHIFT                                                           0x6
+#define ICG_CHCG_CLK_CTRL__GLOBAL_CLK_OVERRIDE_MASK                                                           0x00000001L
+#define ICG_CHCG_CLK_CTRL__GLOBAL_NONHARVESTABLE_CLK_OVERRIDE_MASK                                            0x00000002L
+#define ICG_CHCG_CLK_CTRL__REQUEST_CLK_OVERRIDE_MASK                                                          0x00000004L
+#define ICG_CHCG_CLK_CTRL__SRC_DATA_CLK_OVERRIDE_MASK                                                         0x00000008L
+#define ICG_CHCG_CLK_CTRL__RETURN_CLK_OVERRIDE_MASK                                                           0x00000010L
+#define ICG_CHCG_CLK_CTRL__GRBM_CLK_OVERRIDE_MASK                                                             0x00000020L
+#define ICG_CHCG_CLK_CTRL__PERF_CLK_OVERRIDE_MASK                                                             0x00000040L
+
+
+// addressBlock: gc_pspdec
+//CP_MES_DM_INDEX_ADDR
+#define CP_MES_DM_INDEX_ADDR__ADDR__SHIFT                                                                     0x0
+#define CP_MES_DM_INDEX_ADDR__ADDR_MASK                                                                       0xFFFFFFFFL
+//CP_MES_DM_INDEX_DATA
+#define CP_MES_DM_INDEX_DATA__DATA__SHIFT                                                                     0x0
+#define CP_MES_DM_INDEX_DATA__DATA_MASK                                                                       0xFFFFFFFFL
+//CP_MEC_DM_INDEX_ADDR
+#define CP_MEC_DM_INDEX_ADDR__ADDR__SHIFT                                                                     0x0
+#define CP_MEC_DM_INDEX_ADDR__ADDR_MASK                                                                       0xFFFFFFFFL
+//CP_MEC_DM_INDEX_DATA
+#define CP_MEC_DM_INDEX_DATA__DATA__SHIFT                                                                     0x0
+#define CP_MEC_DM_INDEX_DATA__DATA_MASK                                                                       0xFFFFFFFFL
+//CP_GFX_RS64_DM_INDEX_ADDR
+#define CP_GFX_RS64_DM_INDEX_ADDR__ADDR__SHIFT                                                                0x0
+#define CP_GFX_RS64_DM_INDEX_ADDR__ADDR_MASK                                                                  0xFFFFFFFFL
+//CP_GFX_RS64_DM_INDEX_DATA
+#define CP_GFX_RS64_DM_INDEX_DATA__DATA__SHIFT                                                                0x0
+#define CP_GFX_RS64_DM_INDEX_DATA__DATA_MASK                                                                  0xFFFFFFFFL
+//CPG_PSP_DEBUG
+#define CPG_PSP_DEBUG__PRIV_VIOLATION_CNTL__SHIFT                                                             0x0
+#define CPG_PSP_DEBUG__VMID_VIOLATION_CNTL__SHIFT                                                             0x2
+#define CPG_PSP_DEBUG__GPA_OVERRIDE__SHIFT                                                                    0x3
+#define CPG_PSP_DEBUG__UCODE_VF_OVERRIDE__SHIFT                                                               0x4
+#define CPG_PSP_DEBUG__MTYPE_TMZ_OVERRIDE__SHIFT                                                              0x5
+#define CPG_PSP_DEBUG__SECURE_REG_OVERRIDE__SHIFT                                                             0x6
+#define CPG_PSP_DEBUG__PRIV_VIOLATION_CNTL_MASK                                                               0x00000003L
+#define CPG_PSP_DEBUG__VMID_VIOLATION_CNTL_MASK                                                               0x00000004L
+#define CPG_PSP_DEBUG__GPA_OVERRIDE_MASK                                                                      0x00000008L
+#define CPG_PSP_DEBUG__UCODE_VF_OVERRIDE_MASK                                                                 0x00000010L
+#define CPG_PSP_DEBUG__MTYPE_TMZ_OVERRIDE_MASK                                                                0x00000020L
+#define CPG_PSP_DEBUG__SECURE_REG_OVERRIDE_MASK                                                               0x00000040L
+//CPC_PSP_DEBUG
+#define CPC_PSP_DEBUG__PRIV_VIOLATION_CNTL__SHIFT                                                             0x0
+#define CPC_PSP_DEBUG__GPA_OVERRIDE__SHIFT                                                                    0x3
+#define CPC_PSP_DEBUG__UCODE_VF_OVERRIDE__SHIFT                                                               0x4
+#define CPC_PSP_DEBUG__MTYPE_TMZ_OVERRIDE__SHIFT                                                              0x5
+#define CPC_PSP_DEBUG__SECURE_REG_OVERRIDE__SHIFT                                                             0x6
+#define CPC_PSP_DEBUG__PRIV_VIOLATION_CNTL_MASK                                                               0x00000003L
+#define CPC_PSP_DEBUG__GPA_OVERRIDE_MASK                                                                      0x00000008L
+#define CPC_PSP_DEBUG__UCODE_VF_OVERRIDE_MASK                                                                 0x00000010L
+#define CPC_PSP_DEBUG__MTYPE_TMZ_OVERRIDE_MASK                                                                0x00000020L
+#define CPC_PSP_DEBUG__SECURE_REG_OVERRIDE_MASK                                                               0x00000040L
+//GRBM_IOV_ERROR_FIFO
+#define GRBM_IOV_ERROR_FIFO__IOV_ADDR__SHIFT                                                                  0x0
+#define GRBM_IOV_ERROR_FIFO__IOV_VFID__SHIFT                                                                  0x12
+#define GRBM_IOV_ERROR_FIFO__IOV_SSRCID__SHIFT                                                                0x18
+#define GRBM_IOV_ERROR_FIFO__IOV_OP__SHIFT                                                                    0x1c
+#define GRBM_IOV_ERROR_FIFO__IOV_VF__SHIFT                                                                    0x1d
+#define GRBM_IOV_ERROR_FIFO__FIFO_OVERFLOW__SHIFT                                                             0x1e
+#define GRBM_IOV_ERROR_FIFO__READ_VALID__SHIFT                                                                0x1f
+#define GRBM_IOV_ERROR_FIFO__IOV_ADDR_MASK                                                                    0x0003FFFFL
+#define GRBM_IOV_ERROR_FIFO__IOV_VFID_MASK                                                                    0x00FC0000L
+#define GRBM_IOV_ERROR_FIFO__IOV_SSRCID_MASK                                                                  0x0F000000L
+#define GRBM_IOV_ERROR_FIFO__IOV_OP_MASK                                                                      0x10000000L
+#define GRBM_IOV_ERROR_FIFO__IOV_VF_MASK                                                                      0x20000000L
+#define GRBM_IOV_ERROR_FIFO__FIFO_OVERFLOW_MASK                                                               0x40000000L
+#define GRBM_IOV_ERROR_FIFO__READ_VALID_MASK                                                                  0x80000000L
+//GRBM_SEC_CNTL
+#define GRBM_SEC_CNTL__DEBUG_ENABLE__SHIFT                                                                    0x0
+#define GRBM_SEC_CNTL__DEBUG_ENABLE_MASK                                                                      0x00000001L
+//GRBM_CAM_INDEX
+#define GRBM_CAM_INDEX__CAM_INDEX__SHIFT                                                                      0x0
+#define GRBM_CAM_INDEX__CAM_INDEX_MASK                                                                        0x0000000FL
+//GRBM_HYP_CAM_INDEX
+#define GRBM_HYP_CAM_INDEX__CAM_INDEX__SHIFT                                                                  0x0
+#define GRBM_HYP_CAM_INDEX__CAM_INDEX_MASK                                                                    0x0000000FL
+//GRBM_CAM_DATA
+#define GRBM_CAM_DATA__CAM_ADDR__SHIFT                                                                        0x0
+#define GRBM_CAM_DATA__CAM_REMAPADDR__SHIFT                                                                   0x10
+#define GRBM_CAM_DATA__CAM_ADDR_MASK                                                                          0x0000FFFFL
+#define GRBM_CAM_DATA__CAM_REMAPADDR_MASK                                                                     0xFFFF0000L
+//GRBM_HYP_CAM_DATA
+#define GRBM_HYP_CAM_DATA__CAM_ADDR__SHIFT                                                                    0x0
+#define GRBM_HYP_CAM_DATA__CAM_REMAPADDR__SHIFT                                                               0x10
+#define GRBM_HYP_CAM_DATA__CAM_ADDR_MASK                                                                      0x0000FFFFL
+#define GRBM_HYP_CAM_DATA__CAM_REMAPADDR_MASK                                                                 0xFFFF0000L
+//GRBM_CAM_DATA_UPPER
+#define GRBM_CAM_DATA_UPPER__CAM_ADDR__SHIFT                                                                  0x0
+#define GRBM_CAM_DATA_UPPER__CAM_REMAPADDR__SHIFT                                                             0x10
+#define GRBM_CAM_DATA_UPPER__CAM_ADDR_MASK                                                                    0x00000003L
+#define GRBM_CAM_DATA_UPPER__CAM_REMAPADDR_MASK                                                               0x00030000L
+//GRBM_HYP_CAM_DATA_UPPER
+#define GRBM_HYP_CAM_DATA_UPPER__CAM_ADDR__SHIFT                                                              0x0
+#define GRBM_HYP_CAM_DATA_UPPER__CAM_REMAPADDR__SHIFT                                                         0x10
+#define GRBM_HYP_CAM_DATA_UPPER__CAM_ADDR_MASK                                                                0x00000003L
+#define GRBM_HYP_CAM_DATA_UPPER__CAM_REMAPADDR_MASK                                                           0x00030000L
+//RLC_FWL_FIRST_VIOL_ADDR
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_ADDR__SHIFT                                                             0x0
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_APERTURE_ID__SHIFT                                                      0x12
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_OP__SHIFT                                                               0x1e
+#define RLC_FWL_FIRST_VIOL_ADDR__RESERVED__SHIFT                                                              0x1f
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_ADDR_MASK                                                               0x0003FFFFL
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_APERTURE_ID_MASK                                                        0x3FFC0000L
+#define RLC_FWL_FIRST_VIOL_ADDR__VIOL_OP_MASK                                                                 0x40000000L
+#define RLC_FWL_FIRST_VIOL_ADDR__RESERVED_MASK                                                                0x80000000L
+
+
+// addressBlock: gc_gfx_imu_gfx_imudec
+//GFX_IMU_C2PMSG_0
+#define GFX_IMU_C2PMSG_0__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_0__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_1
+#define GFX_IMU_C2PMSG_1__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_1__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_2
+#define GFX_IMU_C2PMSG_2__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_2__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_3
+#define GFX_IMU_C2PMSG_3__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_3__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_4
+#define GFX_IMU_C2PMSG_4__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_4__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_5
+#define GFX_IMU_C2PMSG_5__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_5__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_6
+#define GFX_IMU_C2PMSG_6__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_6__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_7
+#define GFX_IMU_C2PMSG_7__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_7__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_8
+#define GFX_IMU_C2PMSG_8__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_8__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_9
+#define GFX_IMU_C2PMSG_9__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_C2PMSG_9__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_C2PMSG_10
+#define GFX_IMU_C2PMSG_10__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_10__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_11
+#define GFX_IMU_C2PMSG_11__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_11__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_12
+#define GFX_IMU_C2PMSG_12__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_12__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_13
+#define GFX_IMU_C2PMSG_13__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_13__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_14
+#define GFX_IMU_C2PMSG_14__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_14__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_15
+#define GFX_IMU_C2PMSG_15__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_15__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_16
+#define GFX_IMU_C2PMSG_16__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_16__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_17
+#define GFX_IMU_C2PMSG_17__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_17__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_18
+#define GFX_IMU_C2PMSG_18__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_18__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_19
+#define GFX_IMU_C2PMSG_19__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_19__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_20
+#define GFX_IMU_C2PMSG_20__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_20__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_21
+#define GFX_IMU_C2PMSG_21__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_21__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_22
+#define GFX_IMU_C2PMSG_22__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_22__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_23
+#define GFX_IMU_C2PMSG_23__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_23__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_24
+#define GFX_IMU_C2PMSG_24__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_24__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_25
+#define GFX_IMU_C2PMSG_25__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_25__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_26
+#define GFX_IMU_C2PMSG_26__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_26__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_27
+#define GFX_IMU_C2PMSG_27__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_27__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_28
+#define GFX_IMU_C2PMSG_28__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_28__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_29
+#define GFX_IMU_C2PMSG_29__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_29__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_30
+#define GFX_IMU_C2PMSG_30__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_30__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_31
+#define GFX_IMU_C2PMSG_31__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_31__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_32
+#define GFX_IMU_C2PMSG_32__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_32__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_33
+#define GFX_IMU_C2PMSG_33__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_33__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_34
+#define GFX_IMU_C2PMSG_34__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_34__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_35
+#define GFX_IMU_C2PMSG_35__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_35__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_36
+#define GFX_IMU_C2PMSG_36__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_36__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_37
+#define GFX_IMU_C2PMSG_37__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_37__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_38
+#define GFX_IMU_C2PMSG_38__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_38__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_39
+#define GFX_IMU_C2PMSG_39__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_39__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_40
+#define GFX_IMU_C2PMSG_40__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_40__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_41
+#define GFX_IMU_C2PMSG_41__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_41__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_42
+#define GFX_IMU_C2PMSG_42__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_42__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_43
+#define GFX_IMU_C2PMSG_43__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_43__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_44
+#define GFX_IMU_C2PMSG_44__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_44__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_45
+#define GFX_IMU_C2PMSG_45__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_45__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_46
+#define GFX_IMU_C2PMSG_46__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_46__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_C2PMSG_47
+#define GFX_IMU_C2PMSG_47__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_C2PMSG_47__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_MSG_FLAGS
+#define GFX_IMU_MSG_FLAGS__STATUS__SHIFT                                                                      0x0
+#define GFX_IMU_MSG_FLAGS__STATUS_MASK                                                                        0xFFFFFFFFL
+//GFX_IMU_C2PMSG_ACCESS_CTRL0
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC0__SHIFT                                                              0x0
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC1__SHIFT                                                              0x3
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC2__SHIFT                                                              0x6
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC3__SHIFT                                                              0x9
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC4__SHIFT                                                              0xc
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC5__SHIFT                                                              0xf
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC6__SHIFT                                                              0x12
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC7__SHIFT                                                              0x15
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC0_MASK                                                                0x00000007L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC1_MASK                                                                0x00000038L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC2_MASK                                                                0x000001C0L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC3_MASK                                                                0x00000E00L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC4_MASK                                                                0x00007000L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC5_MASK                                                                0x00038000L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC6_MASK                                                                0x001C0000L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL0__ACC7_MASK                                                                0x00E00000L
+//GFX_IMU_C2PMSG_ACCESS_CTRL1
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC8_15__SHIFT                                                           0x0
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC16_23__SHIFT                                                          0x3
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC24_31__SHIFT                                                          0x6
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC32_39__SHIFT                                                          0x9
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC40_47__SHIFT                                                          0xc
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC8_15_MASK                                                             0x00000007L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC16_23_MASK                                                            0x00000038L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC24_31_MASK                                                            0x000001C0L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC32_39_MASK                                                            0x00000E00L
+#define GFX_IMU_C2PMSG_ACCESS_CTRL1__ACC40_47_MASK                                                            0x00007000L
+//GFX_IMU_PWRMGT_IRQ_CTRL
+#define GFX_IMU_PWRMGT_IRQ_CTRL__REQ__SHIFT                                                                   0x0
+#define GFX_IMU_PWRMGT_IRQ_CTRL__REQ_MASK                                                                     0x00000001L
+//GFX_IMU_MP1_MUTEX
+#define GFX_IMU_MP1_MUTEX__MUTEX__SHIFT                                                                       0x0
+#define GFX_IMU_MP1_MUTEX__MUTEX_MASK                                                                         0x00000003L
+//GFX_IMU_RLC_DATA_4
+#define GFX_IMU_RLC_DATA_4__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_DATA_4__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_DATA_3
+#define GFX_IMU_RLC_DATA_3__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_DATA_3__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_DATA_2
+#define GFX_IMU_RLC_DATA_2__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_DATA_2__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_DATA_1
+#define GFX_IMU_RLC_DATA_1__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_DATA_1__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_DATA_0
+#define GFX_IMU_RLC_DATA_0__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_DATA_0__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_CMD
+#define GFX_IMU_RLC_CMD__CMD__SHIFT                                                                           0x0
+#define GFX_IMU_RLC_CMD__CMD_MASK                                                                             0xFFFFFFFFL
+//GFX_IMU_RLC_MUTEX
+#define GFX_IMU_RLC_MUTEX__MUTEX__SHIFT                                                                       0x0
+#define GFX_IMU_RLC_MUTEX__MUTEX_MASK                                                                         0x00000003L
+//GFX_IMU_RLC_MSG_STATUS
+#define GFX_IMU_RLC_MSG_STATUS__IMU2RLC_BUSY__SHIFT                                                           0x0
+#define GFX_IMU_RLC_MSG_STATUS__IMU2RLC_MSG_ERROR__SHIFT                                                      0x1
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_MSGDONE__SHIFT                                                        0x10
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_CHGTOG__SHIFT                                                         0x1e
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_DONETOG__SHIFT                                                        0x1f
+#define GFX_IMU_RLC_MSG_STATUS__IMU2RLC_BUSY_MASK                                                             0x00000001L
+#define GFX_IMU_RLC_MSG_STATUS__IMU2RLC_MSG_ERROR_MASK                                                        0x00000002L
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_MSGDONE_MASK                                                          0x00010000L
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_CHGTOG_MASK                                                           0x40000000L
+#define GFX_IMU_RLC_MSG_STATUS__RLC2IMU_DONETOG_MASK                                                          0x80000000L
+//RLC_GFX_IMU_DATA_0
+#define RLC_GFX_IMU_DATA_0__DATA__SHIFT                                                                       0x0
+#define RLC_GFX_IMU_DATA_0__DATA_MASK                                                                         0xFFFFFFFFL
+//RLC_GFX_IMU_CMD
+#define RLC_GFX_IMU_CMD__CMD__SHIFT                                                                           0x0
+#define RLC_GFX_IMU_CMD__CMD_MASK                                                                             0xFFFFFFFFL
+//GFX_IMU_RLC_STATUS
+#define GFX_IMU_RLC_STATUS__PD_ACTIVE__SHIFT                                                                  0x0
+#define GFX_IMU_RLC_STATUS__RLC_ALIVE__SHIFT                                                                  0x1
+#define GFX_IMU_RLC_STATUS__TBD2__SHIFT                                                                       0x2
+#define GFX_IMU_RLC_STATUS__TBD3__SHIFT                                                                       0x3
+#define GFX_IMU_RLC_STATUS__PD_ACTIVE_MASK                                                                    0x00000001L
+#define GFX_IMU_RLC_STATUS__RLC_ALIVE_MASK                                                                    0x00000002L
+#define GFX_IMU_RLC_STATUS__TBD2_MASK                                                                         0x00000004L
+#define GFX_IMU_RLC_STATUS__TBD3_MASK                                                                         0x00000008L
+//GFX_IMU_STATUS
+#define GFX_IMU_STATUS__ALLOW_GFXOFF__SHIFT                                                                   0x0
+#define GFX_IMU_STATUS__ALLOW_FA_DCS__SHIFT                                                                   0x1
+#define GFX_IMU_STATUS__TBD2__SHIFT                                                                           0x2
+#define GFX_IMU_STATUS__TBD3__SHIFT                                                                           0x3
+#define GFX_IMU_STATUS__TBD4__SHIFT                                                                           0x4
+#define GFX_IMU_STATUS__TBD5__SHIFT                                                                           0x5
+#define GFX_IMU_STATUS__TBD6__SHIFT                                                                           0x6
+#define GFX_IMU_STATUS__TBD7__SHIFT                                                                           0x7
+#define GFX_IMU_STATUS__TBD8__SHIFT                                                                           0x8
+#define GFX_IMU_STATUS__TBD9__SHIFT                                                                           0x9
+#define GFX_IMU_STATUS__TBD10__SHIFT                                                                          0xa
+#define GFX_IMU_STATUS__TBD11__SHIFT                                                                          0xb
+#define GFX_IMU_STATUS__TBD12__SHIFT                                                                          0xc
+#define GFX_IMU_STATUS__TBD13__SHIFT                                                                          0xd
+#define GFX_IMU_STATUS__TBD14__SHIFT                                                                          0xe
+#define GFX_IMU_STATUS__DISABLE_GFXCLK_DS__SHIFT                                                              0xf
+#define GFX_IMU_STATUS__ALLOW_GFXOFF_MASK                                                                     0x00000001L
+#define GFX_IMU_STATUS__ALLOW_FA_DCS_MASK                                                                     0x00000002L
+#define GFX_IMU_STATUS__TBD2_MASK                                                                             0x00000004L
+#define GFX_IMU_STATUS__TBD3_MASK                                                                             0x00000008L
+#define GFX_IMU_STATUS__TBD4_MASK                                                                             0x00000010L
+#define GFX_IMU_STATUS__TBD5_MASK                                                                             0x00000020L
+#define GFX_IMU_STATUS__TBD6_MASK                                                                             0x00000040L
+#define GFX_IMU_STATUS__TBD7_MASK                                                                             0x00000080L
+#define GFX_IMU_STATUS__TBD8_MASK                                                                             0x00000100L
+#define GFX_IMU_STATUS__TBD9_MASK                                                                             0x00000200L
+#define GFX_IMU_STATUS__TBD10_MASK                                                                            0x00000400L
+#define GFX_IMU_STATUS__TBD11_MASK                                                                            0x00000800L
+#define GFX_IMU_STATUS__TBD12_MASK                                                                            0x00001000L
+#define GFX_IMU_STATUS__TBD13_MASK                                                                            0x00002000L
+#define GFX_IMU_STATUS__TBD14_MASK                                                                            0x00004000L
+#define GFX_IMU_STATUS__DISABLE_GFXCLK_DS_MASK                                                                0x00008000L
+//GFX_IMU_SOC_DATA
+#define GFX_IMU_SOC_DATA__DATA__SHIFT                                                                         0x0
+#define GFX_IMU_SOC_DATA__DATA_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_SOC_ADDR
+#define GFX_IMU_SOC_ADDR__ADDR__SHIFT                                                                         0x0
+#define GFX_IMU_SOC_ADDR__ADDR_MASK                                                                           0xFFFFFFFFL
+//GFX_IMU_SOC_REQ
+#define GFX_IMU_SOC_REQ__REQ_BUSY__SHIFT                                                                      0x0
+#define GFX_IMU_SOC_REQ__R_W__SHIFT                                                                           0x1
+#define GFX_IMU_SOC_REQ__ERR__SHIFT                                                                           0x1f
+#define GFX_IMU_SOC_REQ__REQ_BUSY_MASK                                                                        0x00000001L
+#define GFX_IMU_SOC_REQ__R_W_MASK                                                                             0x00000002L
+#define GFX_IMU_SOC_REQ__ERR_MASK                                                                             0x80000000L
+//GFX_IMU_VF_CTRL
+#define GFX_IMU_VF_CTRL__VF__SHIFT                                                                            0x0
+#define GFX_IMU_VF_CTRL__VFID__SHIFT                                                                          0x1
+#define GFX_IMU_VF_CTRL__QOS__SHIFT                                                                           0x7
+#define GFX_IMU_VF_CTRL__VF_MASK                                                                              0x00000001L
+#define GFX_IMU_VF_CTRL__VFID_MASK                                                                            0x0000007EL
+#define GFX_IMU_VF_CTRL__QOS_MASK                                                                             0x00000780L
+//GFX_IMU_TELEMETRY
+#define GFX_IMU_TELEMETRY__TELEMETRY_ENTRIES__SHIFT                                                           0x0
+#define GFX_IMU_TELEMETRY__TELEMETRY_DATA_SAMPLE_SIZE__SHIFT                                                  0x5
+#define GFX_IMU_TELEMETRY__FIFO_OVERFLOW__SHIFT                                                               0x6
+#define GFX_IMU_TELEMETRY__FIFO_UNDERFLOW__SHIFT                                                              0x7
+#define GFX_IMU_TELEMETRY__FSM_STATE__SHIFT                                                                   0x8
+#define GFX_IMU_TELEMETRY__SVI_TYPE__SHIFT                                                                    0xc
+#define GFX_IMU_TELEMETRY__ENABLE_FIFO__SHIFT                                                                 0x1e
+#define GFX_IMU_TELEMETRY__ENABLE_IMU_RLC_TELEMETRY__SHIFT                                                    0x1f
+#define GFX_IMU_TELEMETRY__TELEMETRY_ENTRIES_MASK                                                             0x0000001FL
+#define GFX_IMU_TELEMETRY__TELEMETRY_DATA_SAMPLE_SIZE_MASK                                                    0x00000020L
+#define GFX_IMU_TELEMETRY__FIFO_OVERFLOW_MASK                                                                 0x00000040L
+#define GFX_IMU_TELEMETRY__FIFO_UNDERFLOW_MASK                                                                0x00000080L
+#define GFX_IMU_TELEMETRY__FSM_STATE_MASK                                                                     0x00000700L
+#define GFX_IMU_TELEMETRY__SVI_TYPE_MASK                                                                      0x00003000L
+#define GFX_IMU_TELEMETRY__ENABLE_FIFO_MASK                                                                   0x40000000L
+#define GFX_IMU_TELEMETRY__ENABLE_IMU_RLC_TELEMETRY_MASK                                                      0x80000000L
+//GFX_IMU_TELEMETRY_DATA
+#define GFX_IMU_TELEMETRY_DATA__CURRENT__SHIFT                                                                0x0
+#define GFX_IMU_TELEMETRY_DATA__VOLTAGE__SHIFT                                                                0x10
+#define GFX_IMU_TELEMETRY_DATA__CURRENT_MASK                                                                  0x0000FFFFL
+#define GFX_IMU_TELEMETRY_DATA__VOLTAGE_MASK                                                                  0xFFFF0000L
+//GFX_IMU_TELEMETRY_TEMPERATURE
+#define GFX_IMU_TELEMETRY_TEMPERATURE__TEMPERATURE__SHIFT                                                     0x0
+#define GFX_IMU_TELEMETRY_TEMPERATURE__TEMPERATURE_MASK                                                       0x0000FFFFL
+//GFX_IMU_SCRATCH_0
+#define GFX_IMU_SCRATCH_0__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_0__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_1
+#define GFX_IMU_SCRATCH_1__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_1__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_2
+#define GFX_IMU_SCRATCH_2__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_2__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_3
+#define GFX_IMU_SCRATCH_3__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_3__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_4
+#define GFX_IMU_SCRATCH_4__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_4__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_5
+#define GFX_IMU_SCRATCH_5__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_5__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_6
+#define GFX_IMU_SCRATCH_6__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_6__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_7
+#define GFX_IMU_SCRATCH_7__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_7__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_8
+#define GFX_IMU_SCRATCH_8__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_8__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_9
+#define GFX_IMU_SCRATCH_9__DATA__SHIFT                                                                        0x0
+#define GFX_IMU_SCRATCH_9__DATA_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_SCRATCH_10
+#define GFX_IMU_SCRATCH_10__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_10__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_SCRATCH_11
+#define GFX_IMU_SCRATCH_11__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_11__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_SCRATCH_12
+#define GFX_IMU_SCRATCH_12__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_12__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_SCRATCH_13
+#define GFX_IMU_SCRATCH_13__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_13__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_SCRATCH_14
+#define GFX_IMU_SCRATCH_14__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_14__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_SCRATCH_15
+#define GFX_IMU_SCRATCH_15__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_SCRATCH_15__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_FW_GTS_LO
+#define GFX_IMU_FW_GTS_LO__TSTAMP_LO__SHIFT                                                                   0x0
+#define GFX_IMU_FW_GTS_LO__TSTAMP_LO_MASK                                                                     0xFFFFFFFFL
+//GFX_IMU_FW_GTS_HI
+#define GFX_IMU_FW_GTS_HI__TSTAMP_HI__SHIFT                                                                   0x0
+#define GFX_IMU_FW_GTS_HI__TSTAMP_HI_MASK                                                                     0x00FFFFFFL
+//GFX_IMU_GTS_OFFSET_LO
+#define GFX_IMU_GTS_OFFSET_LO__GTS_OFFSET_LO__SHIFT                                                           0x0
+#define GFX_IMU_GTS_OFFSET_LO__GTS_OFFSET_LO_MASK                                                             0xFFFFFFFFL
+//GFX_IMU_GTS_OFFSET_HI
+#define GFX_IMU_GTS_OFFSET_HI__GTS_OFFSET_HI__SHIFT                                                           0x0
+#define GFX_IMU_GTS_OFFSET_HI__GTS_OFFSET_HI_MASK                                                             0x00FFFFFFL
+//GFX_IMU_RLC_GTS_OFFSET_LO
+#define GFX_IMU_RLC_GTS_OFFSET_LO__GTS_OFFSET_LO__SHIFT                                                       0x0
+#define GFX_IMU_RLC_GTS_OFFSET_LO__GTS_OFFSET_LO_MASK                                                         0xFFFFFFFFL
+//GFX_IMU_RLC_GTS_OFFSET_HI
+#define GFX_IMU_RLC_GTS_OFFSET_HI__GTS_OFFSET_HI__SHIFT                                                       0x0
+#define GFX_IMU_RLC_GTS_OFFSET_HI__GTS_OFFSET_HI_MASK                                                         0x00FFFFFFL
+//GFX_IMU_CORE_INT_STATUS
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_24__SHIFT                                                          0x18
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_25__SHIFT                                                          0x19
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_29__SHIFT                                                          0x1d
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_24_MASK                                                            0x01000000L
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_25_MASK                                                            0x02000000L
+#define GFX_IMU_CORE_INT_STATUS__INTERRUPT_29_MASK                                                            0x20000000L
+//GFX_IMU_PIC_INT_MASK
+#define GFX_IMU_PIC_INT_MASK__MASK_0__SHIFT                                                                   0x0
+#define GFX_IMU_PIC_INT_MASK__MASK_1__SHIFT                                                                   0x1
+#define GFX_IMU_PIC_INT_MASK__MASK_2__SHIFT                                                                   0x2
+#define GFX_IMU_PIC_INT_MASK__MASK_3__SHIFT                                                                   0x3
+#define GFX_IMU_PIC_INT_MASK__MASK_4__SHIFT                                                                   0x4
+#define GFX_IMU_PIC_INT_MASK__MASK_5__SHIFT                                                                   0x5
+#define GFX_IMU_PIC_INT_MASK__MASK_6__SHIFT                                                                   0x6
+#define GFX_IMU_PIC_INT_MASK__MASK_7__SHIFT                                                                   0x7
+#define GFX_IMU_PIC_INT_MASK__MASK_8__SHIFT                                                                   0x8
+#define GFX_IMU_PIC_INT_MASK__MASK_9__SHIFT                                                                   0x9
+#define GFX_IMU_PIC_INT_MASK__MASK_10__SHIFT                                                                  0xa
+#define GFX_IMU_PIC_INT_MASK__MASK_11__SHIFT                                                                  0xb
+#define GFX_IMU_PIC_INT_MASK__MASK_12__SHIFT                                                                  0xc
+#define GFX_IMU_PIC_INT_MASK__MASK_13__SHIFT                                                                  0xd
+#define GFX_IMU_PIC_INT_MASK__MASK_14__SHIFT                                                                  0xe
+#define GFX_IMU_PIC_INT_MASK__MASK_15__SHIFT                                                                  0xf
+#define GFX_IMU_PIC_INT_MASK__MASK_16__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_MASK__MASK_17__SHIFT                                                                  0x11
+#define GFX_IMU_PIC_INT_MASK__MASK_18__SHIFT                                                                  0x12
+#define GFX_IMU_PIC_INT_MASK__MASK_19__SHIFT                                                                  0x13
+#define GFX_IMU_PIC_INT_MASK__MASK_20__SHIFT                                                                  0x14
+#define GFX_IMU_PIC_INT_MASK__MASK_21__SHIFT                                                                  0x15
+#define GFX_IMU_PIC_INT_MASK__MASK_22__SHIFT                                                                  0x16
+#define GFX_IMU_PIC_INT_MASK__MASK_23__SHIFT                                                                  0x17
+#define GFX_IMU_PIC_INT_MASK__MASK_24__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_MASK__MASK_25__SHIFT                                                                  0x19
+#define GFX_IMU_PIC_INT_MASK__MASK_26__SHIFT                                                                  0x1a
+#define GFX_IMU_PIC_INT_MASK__MASK_27__SHIFT                                                                  0x1b
+#define GFX_IMU_PIC_INT_MASK__MASK_28__SHIFT                                                                  0x1c
+#define GFX_IMU_PIC_INT_MASK__MASK_29__SHIFT                                                                  0x1d
+#define GFX_IMU_PIC_INT_MASK__MASK_30__SHIFT                                                                  0x1e
+#define GFX_IMU_PIC_INT_MASK__MASK_31__SHIFT                                                                  0x1f
+#define GFX_IMU_PIC_INT_MASK__MASK_0_MASK                                                                     0x00000001L
+#define GFX_IMU_PIC_INT_MASK__MASK_1_MASK                                                                     0x00000002L
+#define GFX_IMU_PIC_INT_MASK__MASK_2_MASK                                                                     0x00000004L
+#define GFX_IMU_PIC_INT_MASK__MASK_3_MASK                                                                     0x00000008L
+#define GFX_IMU_PIC_INT_MASK__MASK_4_MASK                                                                     0x00000010L
+#define GFX_IMU_PIC_INT_MASK__MASK_5_MASK                                                                     0x00000020L
+#define GFX_IMU_PIC_INT_MASK__MASK_6_MASK                                                                     0x00000040L
+#define GFX_IMU_PIC_INT_MASK__MASK_7_MASK                                                                     0x00000080L
+#define GFX_IMU_PIC_INT_MASK__MASK_8_MASK                                                                     0x00000100L
+#define GFX_IMU_PIC_INT_MASK__MASK_9_MASK                                                                     0x00000200L
+#define GFX_IMU_PIC_INT_MASK__MASK_10_MASK                                                                    0x00000400L
+#define GFX_IMU_PIC_INT_MASK__MASK_11_MASK                                                                    0x00000800L
+#define GFX_IMU_PIC_INT_MASK__MASK_12_MASK                                                                    0x00001000L
+#define GFX_IMU_PIC_INT_MASK__MASK_13_MASK                                                                    0x00002000L
+#define GFX_IMU_PIC_INT_MASK__MASK_14_MASK                                                                    0x00004000L
+#define GFX_IMU_PIC_INT_MASK__MASK_15_MASK                                                                    0x00008000L
+#define GFX_IMU_PIC_INT_MASK__MASK_16_MASK                                                                    0x00010000L
+#define GFX_IMU_PIC_INT_MASK__MASK_17_MASK                                                                    0x00020000L
+#define GFX_IMU_PIC_INT_MASK__MASK_18_MASK                                                                    0x00040000L
+#define GFX_IMU_PIC_INT_MASK__MASK_19_MASK                                                                    0x00080000L
+#define GFX_IMU_PIC_INT_MASK__MASK_20_MASK                                                                    0x00100000L
+#define GFX_IMU_PIC_INT_MASK__MASK_21_MASK                                                                    0x00200000L
+#define GFX_IMU_PIC_INT_MASK__MASK_22_MASK                                                                    0x00400000L
+#define GFX_IMU_PIC_INT_MASK__MASK_23_MASK                                                                    0x00800000L
+#define GFX_IMU_PIC_INT_MASK__MASK_24_MASK                                                                    0x01000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_25_MASK                                                                    0x02000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_26_MASK                                                                    0x04000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_27_MASK                                                                    0x08000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_28_MASK                                                                    0x10000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_29_MASK                                                                    0x20000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_30_MASK                                                                    0x40000000L
+#define GFX_IMU_PIC_INT_MASK__MASK_31_MASK                                                                    0x80000000L
+//GFX_IMU_PIC_INT_LVL
+#define GFX_IMU_PIC_INT_LVL__LVL_0__SHIFT                                                                     0x0
+#define GFX_IMU_PIC_INT_LVL__LVL_1__SHIFT                                                                     0x1
+#define GFX_IMU_PIC_INT_LVL__LVL_2__SHIFT                                                                     0x2
+#define GFX_IMU_PIC_INT_LVL__LVL_3__SHIFT                                                                     0x3
+#define GFX_IMU_PIC_INT_LVL__LVL_4__SHIFT                                                                     0x4
+#define GFX_IMU_PIC_INT_LVL__LVL_5__SHIFT                                                                     0x5
+#define GFX_IMU_PIC_INT_LVL__LVL_6__SHIFT                                                                     0x6
+#define GFX_IMU_PIC_INT_LVL__LVL_7__SHIFT                                                                     0x7
+#define GFX_IMU_PIC_INT_LVL__LVL_8__SHIFT                                                                     0x8
+#define GFX_IMU_PIC_INT_LVL__LVL_9__SHIFT                                                                     0x9
+#define GFX_IMU_PIC_INT_LVL__LVL_10__SHIFT                                                                    0xa
+#define GFX_IMU_PIC_INT_LVL__LVL_11__SHIFT                                                                    0xb
+#define GFX_IMU_PIC_INT_LVL__LVL_12__SHIFT                                                                    0xc
+#define GFX_IMU_PIC_INT_LVL__LVL_13__SHIFT                                                                    0xd
+#define GFX_IMU_PIC_INT_LVL__LVL_14__SHIFT                                                                    0xe
+#define GFX_IMU_PIC_INT_LVL__LVL_15__SHIFT                                                                    0xf
+#define GFX_IMU_PIC_INT_LVL__LVL_16__SHIFT                                                                    0x10
+#define GFX_IMU_PIC_INT_LVL__LVL_17__SHIFT                                                                    0x11
+#define GFX_IMU_PIC_INT_LVL__LVL_18__SHIFT                                                                    0x12
+#define GFX_IMU_PIC_INT_LVL__LVL_19__SHIFT                                                                    0x13
+#define GFX_IMU_PIC_INT_LVL__LVL_20__SHIFT                                                                    0x14
+#define GFX_IMU_PIC_INT_LVL__LVL_21__SHIFT                                                                    0x15
+#define GFX_IMU_PIC_INT_LVL__LVL_22__SHIFT                                                                    0x16
+#define GFX_IMU_PIC_INT_LVL__LVL_23__SHIFT                                                                    0x17
+#define GFX_IMU_PIC_INT_LVL__LVL_24__SHIFT                                                                    0x18
+#define GFX_IMU_PIC_INT_LVL__LVL_25__SHIFT                                                                    0x19
+#define GFX_IMU_PIC_INT_LVL__LVL_26__SHIFT                                                                    0x1a
+#define GFX_IMU_PIC_INT_LVL__LVL_27__SHIFT                                                                    0x1b
+#define GFX_IMU_PIC_INT_LVL__LVL_28__SHIFT                                                                    0x1c
+#define GFX_IMU_PIC_INT_LVL__LVL_29__SHIFT                                                                    0x1d
+#define GFX_IMU_PIC_INT_LVL__LVL_30__SHIFT                                                                    0x1e
+#define GFX_IMU_PIC_INT_LVL__LVL_31__SHIFT                                                                    0x1f
+#define GFX_IMU_PIC_INT_LVL__LVL_0_MASK                                                                       0x00000001L
+#define GFX_IMU_PIC_INT_LVL__LVL_1_MASK                                                                       0x00000002L
+#define GFX_IMU_PIC_INT_LVL__LVL_2_MASK                                                                       0x00000004L
+#define GFX_IMU_PIC_INT_LVL__LVL_3_MASK                                                                       0x00000008L
+#define GFX_IMU_PIC_INT_LVL__LVL_4_MASK                                                                       0x00000010L
+#define GFX_IMU_PIC_INT_LVL__LVL_5_MASK                                                                       0x00000020L
+#define GFX_IMU_PIC_INT_LVL__LVL_6_MASK                                                                       0x00000040L
+#define GFX_IMU_PIC_INT_LVL__LVL_7_MASK                                                                       0x00000080L
+#define GFX_IMU_PIC_INT_LVL__LVL_8_MASK                                                                       0x00000100L
+#define GFX_IMU_PIC_INT_LVL__LVL_9_MASK                                                                       0x00000200L
+#define GFX_IMU_PIC_INT_LVL__LVL_10_MASK                                                                      0x00000400L
+#define GFX_IMU_PIC_INT_LVL__LVL_11_MASK                                                                      0x00000800L
+#define GFX_IMU_PIC_INT_LVL__LVL_12_MASK                                                                      0x00001000L
+#define GFX_IMU_PIC_INT_LVL__LVL_13_MASK                                                                      0x00002000L
+#define GFX_IMU_PIC_INT_LVL__LVL_14_MASK                                                                      0x00004000L
+#define GFX_IMU_PIC_INT_LVL__LVL_15_MASK                                                                      0x00008000L
+#define GFX_IMU_PIC_INT_LVL__LVL_16_MASK                                                                      0x00010000L
+#define GFX_IMU_PIC_INT_LVL__LVL_17_MASK                                                                      0x00020000L
+#define GFX_IMU_PIC_INT_LVL__LVL_18_MASK                                                                      0x00040000L
+#define GFX_IMU_PIC_INT_LVL__LVL_19_MASK                                                                      0x00080000L
+#define GFX_IMU_PIC_INT_LVL__LVL_20_MASK                                                                      0x00100000L
+#define GFX_IMU_PIC_INT_LVL__LVL_21_MASK                                                                      0x00200000L
+#define GFX_IMU_PIC_INT_LVL__LVL_22_MASK                                                                      0x00400000L
+#define GFX_IMU_PIC_INT_LVL__LVL_23_MASK                                                                      0x00800000L
+#define GFX_IMU_PIC_INT_LVL__LVL_24_MASK                                                                      0x01000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_25_MASK                                                                      0x02000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_26_MASK                                                                      0x04000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_27_MASK                                                                      0x08000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_28_MASK                                                                      0x10000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_29_MASK                                                                      0x20000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_30_MASK                                                                      0x40000000L
+#define GFX_IMU_PIC_INT_LVL__LVL_31_MASK                                                                      0x80000000L
+//GFX_IMU_PIC_INT_EDGE
+#define GFX_IMU_PIC_INT_EDGE__EDGE_0__SHIFT                                                                   0x0
+#define GFX_IMU_PIC_INT_EDGE__EDGE_1__SHIFT                                                                   0x1
+#define GFX_IMU_PIC_INT_EDGE__EDGE_2__SHIFT                                                                   0x2
+#define GFX_IMU_PIC_INT_EDGE__EDGE_3__SHIFT                                                                   0x3
+#define GFX_IMU_PIC_INT_EDGE__EDGE_4__SHIFT                                                                   0x4
+#define GFX_IMU_PIC_INT_EDGE__EDGE_5__SHIFT                                                                   0x5
+#define GFX_IMU_PIC_INT_EDGE__EDGE_6__SHIFT                                                                   0x6
+#define GFX_IMU_PIC_INT_EDGE__EDGE_7__SHIFT                                                                   0x7
+#define GFX_IMU_PIC_INT_EDGE__EDGE_8__SHIFT                                                                   0x8
+#define GFX_IMU_PIC_INT_EDGE__EDGE_9__SHIFT                                                                   0x9
+#define GFX_IMU_PIC_INT_EDGE__EDGE_10__SHIFT                                                                  0xa
+#define GFX_IMU_PIC_INT_EDGE__EDGE_11__SHIFT                                                                  0xb
+#define GFX_IMU_PIC_INT_EDGE__EDGE_12__SHIFT                                                                  0xc
+#define GFX_IMU_PIC_INT_EDGE__EDGE_13__SHIFT                                                                  0xd
+#define GFX_IMU_PIC_INT_EDGE__EDGE_14__SHIFT                                                                  0xe
+#define GFX_IMU_PIC_INT_EDGE__EDGE_15__SHIFT                                                                  0xf
+#define GFX_IMU_PIC_INT_EDGE__EDGE_16__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_EDGE__EDGE_17__SHIFT                                                                  0x11
+#define GFX_IMU_PIC_INT_EDGE__EDGE_18__SHIFT                                                                  0x12
+#define GFX_IMU_PIC_INT_EDGE__EDGE_19__SHIFT                                                                  0x13
+#define GFX_IMU_PIC_INT_EDGE__EDGE_20__SHIFT                                                                  0x14
+#define GFX_IMU_PIC_INT_EDGE__EDGE_21__SHIFT                                                                  0x15
+#define GFX_IMU_PIC_INT_EDGE__EDGE_22__SHIFT                                                                  0x16
+#define GFX_IMU_PIC_INT_EDGE__EDGE_23__SHIFT                                                                  0x17
+#define GFX_IMU_PIC_INT_EDGE__EDGE_24__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_EDGE__EDGE_25__SHIFT                                                                  0x19
+#define GFX_IMU_PIC_INT_EDGE__EDGE_26__SHIFT                                                                  0x1a
+#define GFX_IMU_PIC_INT_EDGE__EDGE_27__SHIFT                                                                  0x1b
+#define GFX_IMU_PIC_INT_EDGE__EDGE_28__SHIFT                                                                  0x1c
+#define GFX_IMU_PIC_INT_EDGE__EDGE_29__SHIFT                                                                  0x1d
+#define GFX_IMU_PIC_INT_EDGE__EDGE_30__SHIFT                                                                  0x1e
+#define GFX_IMU_PIC_INT_EDGE__EDGE_31__SHIFT                                                                  0x1f
+#define GFX_IMU_PIC_INT_EDGE__EDGE_0_MASK                                                                     0x00000001L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_1_MASK                                                                     0x00000002L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_2_MASK                                                                     0x00000004L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_3_MASK                                                                     0x00000008L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_4_MASK                                                                     0x00000010L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_5_MASK                                                                     0x00000020L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_6_MASK                                                                     0x00000040L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_7_MASK                                                                     0x00000080L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_8_MASK                                                                     0x00000100L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_9_MASK                                                                     0x00000200L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_10_MASK                                                                    0x00000400L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_11_MASK                                                                    0x00000800L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_12_MASK                                                                    0x00001000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_13_MASK                                                                    0x00002000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_14_MASK                                                                    0x00004000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_15_MASK                                                                    0x00008000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_16_MASK                                                                    0x00010000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_17_MASK                                                                    0x00020000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_18_MASK                                                                    0x00040000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_19_MASK                                                                    0x00080000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_20_MASK                                                                    0x00100000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_21_MASK                                                                    0x00200000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_22_MASK                                                                    0x00400000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_23_MASK                                                                    0x00800000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_24_MASK                                                                    0x01000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_25_MASK                                                                    0x02000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_26_MASK                                                                    0x04000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_27_MASK                                                                    0x08000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_28_MASK                                                                    0x10000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_29_MASK                                                                    0x20000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_30_MASK                                                                    0x40000000L
+#define GFX_IMU_PIC_INT_EDGE__EDGE_31_MASK                                                                    0x80000000L
+//GFX_IMU_PIC_INT_PRI_0
+#define GFX_IMU_PIC_INT_PRI_0__PRI_0__SHIFT                                                                   0x0
+#define GFX_IMU_PIC_INT_PRI_0__PRI_1__SHIFT                                                                   0x8
+#define GFX_IMU_PIC_INT_PRI_0__PRI_2__SHIFT                                                                   0x10
+#define GFX_IMU_PIC_INT_PRI_0__PRI_3__SHIFT                                                                   0x18
+#define GFX_IMU_PIC_INT_PRI_0__PRI_0_MASK                                                                     0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_0__PRI_1_MASK                                                                     0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_0__PRI_2_MASK                                                                     0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_0__PRI_3_MASK                                                                     0xFF000000L
+//GFX_IMU_PIC_INT_PRI_1
+#define GFX_IMU_PIC_INT_PRI_1__PRI_4__SHIFT                                                                   0x0
+#define GFX_IMU_PIC_INT_PRI_1__PRI_5__SHIFT                                                                   0x8
+#define GFX_IMU_PIC_INT_PRI_1__PRI_6__SHIFT                                                                   0x10
+#define GFX_IMU_PIC_INT_PRI_1__PRI_7__SHIFT                                                                   0x18
+#define GFX_IMU_PIC_INT_PRI_1__PRI_4_MASK                                                                     0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_1__PRI_5_MASK                                                                     0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_1__PRI_6_MASK                                                                     0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_1__PRI_7_MASK                                                                     0xFF000000L
+//GFX_IMU_PIC_INT_PRI_2
+#define GFX_IMU_PIC_INT_PRI_2__PRI_8__SHIFT                                                                   0x0
+#define GFX_IMU_PIC_INT_PRI_2__PRI_9__SHIFT                                                                   0x8
+#define GFX_IMU_PIC_INT_PRI_2__PRI_10__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_2__PRI_11__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_2__PRI_8_MASK                                                                     0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_2__PRI_9_MASK                                                                     0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_2__PRI_10_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_2__PRI_11_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_PRI_3
+#define GFX_IMU_PIC_INT_PRI_3__PRI_12__SHIFT                                                                  0x0
+#define GFX_IMU_PIC_INT_PRI_3__PRI_13__SHIFT                                                                  0x8
+#define GFX_IMU_PIC_INT_PRI_3__PRI_14__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_3__PRI_15__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_3__PRI_12_MASK                                                                    0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_3__PRI_13_MASK                                                                    0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_3__PRI_14_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_3__PRI_15_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_PRI_4
+#define GFX_IMU_PIC_INT_PRI_4__PRI_16__SHIFT                                                                  0x0
+#define GFX_IMU_PIC_INT_PRI_4__PRI_17__SHIFT                                                                  0x8
+#define GFX_IMU_PIC_INT_PRI_4__PRI_18__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_4__PRI_19__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_4__PRI_16_MASK                                                                    0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_4__PRI_17_MASK                                                                    0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_4__PRI_18_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_4__PRI_19_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_PRI_5
+#define GFX_IMU_PIC_INT_PRI_5__PRI_20__SHIFT                                                                  0x0
+#define GFX_IMU_PIC_INT_PRI_5__PRI_21__SHIFT                                                                  0x8
+#define GFX_IMU_PIC_INT_PRI_5__PRI_22__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_5__PRI_23__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_5__PRI_20_MASK                                                                    0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_5__PRI_21_MASK                                                                    0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_5__PRI_22_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_5__PRI_23_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_PRI_6
+#define GFX_IMU_PIC_INT_PRI_6__PRI_24__SHIFT                                                                  0x0
+#define GFX_IMU_PIC_INT_PRI_6__PRI_25__SHIFT                                                                  0x8
+#define GFX_IMU_PIC_INT_PRI_6__PRI_26__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_6__PRI_27__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_6__PRI_24_MASK                                                                    0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_6__PRI_25_MASK                                                                    0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_6__PRI_26_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_6__PRI_27_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_PRI_7
+#define GFX_IMU_PIC_INT_PRI_7__PRI_28__SHIFT                                                                  0x0
+#define GFX_IMU_PIC_INT_PRI_7__PRI_29__SHIFT                                                                  0x8
+#define GFX_IMU_PIC_INT_PRI_7__PRI_30__SHIFT                                                                  0x10
+#define GFX_IMU_PIC_INT_PRI_7__PRI_31__SHIFT                                                                  0x18
+#define GFX_IMU_PIC_INT_PRI_7__PRI_28_MASK                                                                    0x000000FFL
+#define GFX_IMU_PIC_INT_PRI_7__PRI_29_MASK                                                                    0x0000FF00L
+#define GFX_IMU_PIC_INT_PRI_7__PRI_30_MASK                                                                    0x00FF0000L
+#define GFX_IMU_PIC_INT_PRI_7__PRI_31_MASK                                                                    0xFF000000L
+//GFX_IMU_PIC_INT_STATUS
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS0__SHIFT                                                            0x0
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS1__SHIFT                                                            0x1
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS2__SHIFT                                                            0x2
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS3__SHIFT                                                            0x3
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS4__SHIFT                                                            0x4
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS5__SHIFT                                                            0x5
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS6__SHIFT                                                            0x6
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS7__SHIFT                                                            0x7
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS8__SHIFT                                                            0x8
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS9__SHIFT                                                            0x9
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS10__SHIFT                                                           0xa
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS11__SHIFT                                                           0xb
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS12__SHIFT                                                           0xc
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS13__SHIFT                                                           0xd
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS14__SHIFT                                                           0xe
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS15__SHIFT                                                           0xf
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS16__SHIFT                                                           0x10
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS17__SHIFT                                                           0x11
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS18__SHIFT                                                           0x12
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS19__SHIFT                                                           0x13
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS20__SHIFT                                                           0x14
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS21__SHIFT                                                           0x15
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS22__SHIFT                                                           0x16
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS23__SHIFT                                                           0x17
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS24__SHIFT                                                           0x18
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS25__SHIFT                                                           0x19
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS26__SHIFT                                                           0x1a
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS27__SHIFT                                                           0x1b
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS28__SHIFT                                                           0x1c
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS29__SHIFT                                                           0x1d
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS30__SHIFT                                                           0x1e
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS31__SHIFT                                                           0x1f
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS0_MASK                                                              0x00000001L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS1_MASK                                                              0x00000002L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS2_MASK                                                              0x00000004L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS3_MASK                                                              0x00000008L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS4_MASK                                                              0x00000010L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS5_MASK                                                              0x00000020L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS6_MASK                                                              0x00000040L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS7_MASK                                                              0x00000080L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS8_MASK                                                              0x00000100L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS9_MASK                                                              0x00000200L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS10_MASK                                                             0x00000400L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS11_MASK                                                             0x00000800L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS12_MASK                                                             0x00001000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS13_MASK                                                             0x00002000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS14_MASK                                                             0x00004000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS15_MASK                                                             0x00008000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS16_MASK                                                             0x00010000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS17_MASK                                                             0x00020000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS18_MASK                                                             0x00040000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS19_MASK                                                             0x00080000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS20_MASK                                                             0x00100000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS21_MASK                                                             0x00200000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS22_MASK                                                             0x00400000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS23_MASK                                                             0x00800000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS24_MASK                                                             0x01000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS25_MASK                                                             0x02000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS26_MASK                                                             0x04000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS27_MASK                                                             0x08000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS28_MASK                                                             0x10000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS29_MASK                                                             0x20000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS30_MASK                                                             0x40000000L
+#define GFX_IMU_PIC_INT_STATUS__INT_STATUS31_MASK                                                             0x80000000L
+//GFX_IMU_PIC_INTR
+#define GFX_IMU_PIC_INTR__INTR_n__SHIFT                                                                       0x0
+#define GFX_IMU_PIC_INTR__INTR_n_MASK                                                                         0x00000001L
+//GFX_IMU_PIC_INTR_ID
+#define GFX_IMU_PIC_INTR_ID__INTR_n__SHIFT                                                                    0x0
+#define GFX_IMU_PIC_INTR_ID__INTR_n_MASK                                                                      0x000000FFL
+//GFX_IMU_IH_CTRL_1
+#define GFX_IMU_IH_CTRL_1__CONTEXT_ID__SHIFT                                                                  0x0
+#define GFX_IMU_IH_CTRL_1__CONTEXT_ID_MASK                                                                    0xFFFFFFFFL
+//GFX_IMU_IH_CTRL_2
+#define GFX_IMU_IH_CTRL_2__CONTEXT_ID__SHIFT                                                                  0x0
+#define GFX_IMU_IH_CTRL_2__RING_ID__SHIFT                                                                     0x8
+#define GFX_IMU_IH_CTRL_2__VM_ID__SHIFT                                                                       0x10
+#define GFX_IMU_IH_CTRL_2__SRSTB__SHIFT                                                                       0x1f
+#define GFX_IMU_IH_CTRL_2__CONTEXT_ID_MASK                                                                    0x000000FFL
+#define GFX_IMU_IH_CTRL_2__RING_ID_MASK                                                                       0x0000FF00L
+#define GFX_IMU_IH_CTRL_2__VM_ID_MASK                                                                         0x000F0000L
+#define GFX_IMU_IH_CTRL_2__SRSTB_MASK                                                                         0x80000000L
+//GFX_IMU_IH_CTRL_3
+#define GFX_IMU_IH_CTRL_3__SOURCE_ID__SHIFT                                                                   0x0
+#define GFX_IMU_IH_CTRL_3__VF_ID__SHIFT                                                                       0x8
+#define GFX_IMU_IH_CTRL_3__VF__SHIFT                                                                          0xd
+#define GFX_IMU_IH_CTRL_3__SOURCE_ID_MASK                                                                     0x000000FFL
+#define GFX_IMU_IH_CTRL_3__VF_ID_MASK                                                                         0x00001F00L
+#define GFX_IMU_IH_CTRL_3__VF_MASK                                                                            0x00002000L
+//GFX_IMU_IH_STATUS
+#define GFX_IMU_IH_STATUS__IH_BUSY__SHIFT                                                                     0x0
+#define GFX_IMU_IH_STATUS__IH_BUSY_MASK                                                                       0x00000001L
+//GFX_IMU_FUSESTRAP
+#define GFX_IMU_FUSESTRAP__BOOT_VID__SHIFT                                                                    0x0
+#define GFX_IMU_FUSESTRAP__BOOT_VID_MASK                                                                      0x000001FFL
+//GFX_IMU_SMUIO_VIDCHG_CTRL
+#define GFX_IMU_SMUIO_VIDCHG_CTRL__REQ__SHIFT                                                                 0x0
+#define GFX_IMU_SMUIO_VIDCHG_CTRL__DATA__SHIFT                                                                0x1
+#define GFX_IMU_SMUIO_VIDCHG_CTRL__PSIEN__SHIFT                                                               0xa
+#define GFX_IMU_SMUIO_VIDCHG_CTRL__ACK__SHIFT                                                                 0xb
+#define GFX_IMU_SMUIO_VIDCHG_CTRL__SRC_SEL__SHIFT                                                             0x1f
+#define GFX_IMU_SMUIO_VIDCHG_CTRL__REQ_MASK                                                                   0x00000001L
+#define GFX_IMU_SMUIO_VIDCHG_CTRL__DATA_MASK                                                                  0x000003FEL
+#define GFX_IMU_SMUIO_VIDCHG_CTRL__PSIEN_MASK                                                                 0x00000400L
+#define GFX_IMU_SMUIO_VIDCHG_CTRL__ACK_MASK                                                                   0x00000800L
+#define GFX_IMU_SMUIO_VIDCHG_CTRL__SRC_SEL_MASK                                                               0x80000000L
+//GFX_IMU_GFXCLK_BYPASS_CTRL
+#define GFX_IMU_GFXCLK_BYPASS_CTRL__BYPASS_SEL__SHIFT                                                         0x0
+#define GFX_IMU_GFXCLK_BYPASS_CTRL__BYPASS_SEL_MASK                                                           0x00000001L
+//GFX_IMU_CLK_CTRL
+#define GFX_IMU_CLK_CTRL__CG_OVR__SHIFT                                                                       0x0
+#define GFX_IMU_CLK_CTRL__CG_OVR_CORE__SHIFT                                                                  0x1
+#define GFX_IMU_CLK_CTRL__CLKDIV__SHIFT                                                                       0x4
+#define GFX_IMU_CLK_CTRL__GFXBYPASSCLK_CHGTOG__SHIFT                                                          0x8
+#define GFX_IMU_CLK_CTRL__GFXBYPASSCLK_DONETOG__SHIFT                                                         0x9
+#define GFX_IMU_CLK_CTRL__GFXBYPASSCLK_DIV__SHIFT                                                             0x10
+#define GFX_IMU_CLK_CTRL__COOLDOWN_PERIOD__SHIFT                                                              0x1c
+#define GFX_IMU_CLK_CTRL__CG_OVR_MASK                                                                         0x00000001L
+#define GFX_IMU_CLK_CTRL__CG_OVR_CORE_MASK                                                                    0x00000002L
+#define GFX_IMU_CLK_CTRL__CLKDIV_MASK                                                                         0x00000010L
+#define GFX_IMU_CLK_CTRL__GFXBYPASSCLK_CHGTOG_MASK                                                            0x00000100L
+#define GFX_IMU_CLK_CTRL__GFXBYPASSCLK_DONETOG_MASK                                                           0x00000200L
+#define GFX_IMU_CLK_CTRL__GFXBYPASSCLK_DIV_MASK                                                               0x007F0000L
+#define GFX_IMU_CLK_CTRL__COOLDOWN_PERIOD_MASK                                                                0xF0000000L
+//GFX_IMU_DOORBELL_CONTROL
+#define GFX_IMU_DOORBELL_CONTROL__OVR_EN__SHIFT                                                               0x0
+#define GFX_IMU_DOORBELL_CONTROL__FENCE_EN_OVR__SHIFT                                                         0x1
+#define GFX_IMU_DOORBELL_CONTROL__CP_DB_RESP_PEND_COUNT__SHIFT                                                0x18
+#define GFX_IMU_DOORBELL_CONTROL__FENCE_EN_STATUS__SHIFT                                                      0x1f
+#define GFX_IMU_DOORBELL_CONTROL__OVR_EN_MASK                                                                 0x00000001L
+#define GFX_IMU_DOORBELL_CONTROL__FENCE_EN_OVR_MASK                                                           0x00000002L
+#define GFX_IMU_DOORBELL_CONTROL__CP_DB_RESP_PEND_COUNT_MASK                                                  0x7F000000L
+#define GFX_IMU_DOORBELL_CONTROL__FENCE_EN_STATUS_MASK                                                        0x80000000L
+//GFX_IMU_RLC_CG_CTRL
+#define GFX_IMU_RLC_CG_CTRL__FORCE_CGCG__SHIFT                                                                0x0
+#define GFX_IMU_RLC_CG_CTRL__MGCG_EARLY_EN__SHIFT                                                             0x1
+#define GFX_IMU_RLC_CG_CTRL__FORCE_CGCG_MASK                                                                  0x00000001L
+#define GFX_IMU_RLC_CG_CTRL__MGCG_EARLY_EN_MASK                                                               0x00000002L
+//GFX_IMU_RLC_THROTTLE_GFX
+#define GFX_IMU_RLC_THROTTLE_GFX__THROTTLE_EN__SHIFT                                                          0x0
+#define GFX_IMU_RLC_THROTTLE_GFX__THROTTLE_EN_MASK                                                            0x00000001L
+//GFX_IMU_RLC_RESET_VECTOR
+#define GFX_IMU_RLC_RESET_VECTOR__COLD_VS_GFXOFF__SHIFT                                                       0x0
+#define GFX_IMU_RLC_RESET_VECTOR__WARM_RESET_EXIT__SHIFT                                                      0x2
+#define GFX_IMU_RLC_RESET_VECTOR__VF_FLR_EXIT__SHIFT                                                          0x3
+#define GFX_IMU_RLC_RESET_VECTOR__VECTOR__SHIFT                                                               0x4
+#define GFX_IMU_RLC_RESET_VECTOR__COLD_VS_GFXOFF_MASK                                                         0x00000001L
+#define GFX_IMU_RLC_RESET_VECTOR__WARM_RESET_EXIT_MASK                                                        0x00000004L
+#define GFX_IMU_RLC_RESET_VECTOR__VF_FLR_EXIT_MASK                                                            0x00000008L
+#define GFX_IMU_RLC_RESET_VECTOR__VECTOR_MASK                                                                 0x000000F0L
+//GFX_IMU_RLC_OVERRIDE
+#define GFX_IMU_RLC_OVERRIDE__DS_ALLOW__SHIFT                                                                 0x0
+#define GFX_IMU_RLC_OVERRIDE__DS_ALLOW_MASK                                                                   0x00000001L
+//GFX_IMU_DPM_CONTROL
+#define GFX_IMU_DPM_CONTROL__ACC_RESET__SHIFT                                                                 0x0
+#define GFX_IMU_DPM_CONTROL__ACC_START__SHIFT                                                                 0x1
+#define GFX_IMU_DPM_CONTROL__BUSY_MASK__SHIFT                                                                 0x2
+#define GFX_IMU_DPM_CONTROL__ACC_RESET_MASK                                                                   0x00000001L
+#define GFX_IMU_DPM_CONTROL__ACC_START_MASK                                                                   0x00000002L
+#define GFX_IMU_DPM_CONTROL__BUSY_MASK_MASK                                                                   0x0003FFFCL
+//GFX_IMU_DPM_ACC
+#define GFX_IMU_DPM_ACC__COUNT__SHIFT                                                                         0x0
+#define GFX_IMU_DPM_ACC__COUNT_MASK                                                                           0x00FFFFFFL
+//GFX_IMU_DPM_REF_COUNTER
+#define GFX_IMU_DPM_REF_COUNTER__COUNT__SHIFT                                                                 0x0
+#define GFX_IMU_DPM_REF_COUNTER__COUNT_MASK                                                                   0x00FFFFFFL
+//GFX_IMU_RLC_RAM_INDEX
+#define GFX_IMU_RLC_RAM_INDEX__INDEX__SHIFT                                                                   0x0
+#define GFX_IMU_RLC_RAM_INDEX__RLC_INDEX__SHIFT                                                               0x10
+#define GFX_IMU_RLC_RAM_INDEX__RAM_VALID__SHIFT                                                               0x1f
+#define GFX_IMU_RLC_RAM_INDEX__INDEX_MASK                                                                     0x000000FFL
+#define GFX_IMU_RLC_RAM_INDEX__RLC_INDEX_MASK                                                                 0x00FF0000L
+#define GFX_IMU_RLC_RAM_INDEX__RAM_VALID_MASK                                                                 0x80000000L
+//GFX_IMU_RLC_RAM_ADDR_HIGH
+#define GFX_IMU_RLC_RAM_ADDR_HIGH__ADDR_MSB__SHIFT                                                            0x0
+#define GFX_IMU_RLC_RAM_ADDR_HIGH__ADDR_MSB_MASK                                                              0x0000FFFFL
+//GFX_IMU_RLC_RAM_ADDR_LOW
+#define GFX_IMU_RLC_RAM_ADDR_LOW__ADDR_LSB__SHIFT                                                             0x0
+#define GFX_IMU_RLC_RAM_ADDR_LOW__ADDR_LSB_MASK                                                               0xFFFFFFFFL
+//GFX_IMU_RLC_RAM_DATA
+#define GFX_IMU_RLC_RAM_DATA__DATA__SHIFT                                                                     0x0
+#define GFX_IMU_RLC_RAM_DATA__DATA_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_FENCE_CTRL
+#define GFX_IMU_FENCE_CTRL__ENABLED__SHIFT                                                                    0x0
+#define GFX_IMU_FENCE_CTRL__ARM_LOG__SHIFT                                                                    0x1
+#define GFX_IMU_FENCE_CTRL__GRBM_RSMU_FENCE_ENABLE__SHIFT                                                     0x2
+#define GFX_IMU_FENCE_CTRL__FLUSH_ARBITER_CREDITS__SHIFT                                                      0x3
+#define GFX_IMU_FENCE_CTRL__GFX_REG_FENCE_OVR_EN__SHIFT                                                       0x8
+#define GFX_IMU_FENCE_CTRL__GFX_REG_FENCE_OVR__SHIFT                                                          0x9
+#define GFX_IMU_FENCE_CTRL__ENABLED_MASK                                                                      0x00000001L
+#define GFX_IMU_FENCE_CTRL__ARM_LOG_MASK                                                                      0x00000002L
+#define GFX_IMU_FENCE_CTRL__GRBM_RSMU_FENCE_ENABLE_MASK                                                       0x00000004L
+#define GFX_IMU_FENCE_CTRL__FLUSH_ARBITER_CREDITS_MASK                                                        0x00000008L
+#define GFX_IMU_FENCE_CTRL__GFX_REG_FENCE_OVR_EN_MASK                                                         0x00000100L
+#define GFX_IMU_FENCE_CTRL__GFX_REG_FENCE_OVR_MASK                                                            0x00000200L
+//GFX_IMU_FENCE_LOG_INIT
+#define GFX_IMU_FENCE_LOG_INIT__UNIT_ID__SHIFT                                                                0x0
+#define GFX_IMU_FENCE_LOG_INIT__INITIATOR_ID__SHIFT                                                           0x7
+#define GFX_IMU_FENCE_LOG_INIT__UNIT_ID_MASK                                                                  0x0000007FL
+#define GFX_IMU_FENCE_LOG_INIT__INITIATOR_ID_MASK                                                             0x0001FF80L
+//GFX_IMU_FENCE_LOG_ADDR
+#define GFX_IMU_FENCE_LOG_ADDR__ADDR__SHIFT                                                                   0x2
+#define GFX_IMU_FENCE_LOG_ADDR__ADDR_MASK                                                                     0x000FFFFCL
+//GFX_IMU_PROGRAM_CTR
+#define GFX_IMU_PROGRAM_CTR__PC__SHIFT                                                                        0x0
+#define GFX_IMU_PROGRAM_CTR__PC_MASK                                                                          0xFFFFFFFFL
+//GFX_IMU_CORE_CTRL
+#define GFX_IMU_CORE_CTRL__CRESET__SHIFT                                                                      0x0
+#define GFX_IMU_CORE_CTRL__CSTALL__SHIFT                                                                      0x1
+#define GFX_IMU_CORE_CTRL__CDBGENABLE__SHIFT                                                                  0x2
+#define GFX_IMU_CORE_CTRL__DRESET__SHIFT                                                                      0x3
+#define GFX_IMU_CORE_CTRL__HALT_ON_RESET__SHIFT                                                               0x4
+#define GFX_IMU_CORE_CTRL__BREAK_IN__SHIFT                                                                    0x8
+#define GFX_IMU_CORE_CTRL__BREAK_OUT_ACK__SHIFT                                                               0x9
+#define GFX_IMU_CORE_CTRL__CRESET_MASK                                                                        0x00000001L
+#define GFX_IMU_CORE_CTRL__CSTALL_MASK                                                                        0x00000002L
+#define GFX_IMU_CORE_CTRL__CDBGENABLE_MASK                                                                    0x00000004L
+#define GFX_IMU_CORE_CTRL__DRESET_MASK                                                                        0x00000008L
+#define GFX_IMU_CORE_CTRL__HALT_ON_RESET_MASK                                                                 0x00000010L
+#define GFX_IMU_CORE_CTRL__BREAK_IN_MASK                                                                      0x00000100L
+#define GFX_IMU_CORE_CTRL__BREAK_OUT_ACK_MASK                                                                 0x00000200L
+//GFX_IMU_CORE_STATUS
+#define GFX_IMU_CORE_STATUS__CBUSY__SHIFT                                                                     0x0
+#define GFX_IMU_CORE_STATUS__PWAIT_MODE__SHIFT                                                                0x1
+#define GFX_IMU_CORE_STATUS__PSP_ACC_ERR__SHIFT                                                               0x2
+#define GFX_IMU_CORE_STATUS__CINTLEVEL__SHIFT                                                                 0x4
+#define GFX_IMU_CORE_STATUS__BREAK_IN_ACK__SHIFT                                                              0x8
+#define GFX_IMU_CORE_STATUS__BREAK_OUT__SHIFT                                                                 0x9
+#define GFX_IMU_CORE_STATUS__DEBUG_MODE__SHIFT                                                                0xa
+#define GFX_IMU_CORE_STATUS__P_FATAL_ERROR__SHIFT                                                             0xb
+#define GFX_IMU_CORE_STATUS__FAULT_SEVERITY_LEVEL__SHIFT                                                      0x18
+#define GFX_IMU_CORE_STATUS__FAULT_TYPE__SHIFT                                                                0x1c
+#define GFX_IMU_CORE_STATUS__CBUSY_MASK                                                                       0x00000001L
+#define GFX_IMU_CORE_STATUS__PWAIT_MODE_MASK                                                                  0x00000002L
+#define GFX_IMU_CORE_STATUS__PSP_ACC_ERR_MASK                                                                 0x00000004L
+#define GFX_IMU_CORE_STATUS__CINTLEVEL_MASK                                                                   0x000000F0L
+#define GFX_IMU_CORE_STATUS__BREAK_IN_ACK_MASK                                                                0x00000100L
+#define GFX_IMU_CORE_STATUS__BREAK_OUT_MASK                                                                   0x00000200L
+#define GFX_IMU_CORE_STATUS__DEBUG_MODE_MASK                                                                  0x00000400L
+#define GFX_IMU_CORE_STATUS__P_FATAL_ERROR_MASK                                                               0x00000800L
+#define GFX_IMU_CORE_STATUS__FAULT_SEVERITY_LEVEL_MASK                                                        0x0F000000L
+#define GFX_IMU_CORE_STATUS__FAULT_TYPE_MASK                                                                  0xF0000000L
+//GFX_IMU_PWROKRAW
+#define GFX_IMU_PWROKRAW__PWROKRAW__SHIFT                                                                     0x0
+#define GFX_IMU_PWROKRAW__PWROKRAW_MASK                                                                       0x00000001L
+//GFX_IMU_PWROK
+#define GFX_IMU_PWROK__PWROK__SHIFT                                                                           0x0
+#define GFX_IMU_PWROK__PWROK_MASK                                                                             0x00000001L
+//GFX_IMU_GAP_PWROK
+#define GFX_IMU_GAP_PWROK__GAP_PWROK__SHIFT                                                                   0x0
+#define GFX_IMU_GAP_PWROK__GAP_PWROK_MASK                                                                     0x00000001L
+//GFX_IMU_RESETn
+#define GFX_IMU_RESETn__Cpl_RESETn__SHIFT                                                                     0x0
+#define GFX_IMU_RESETn__Cpl_RESETn_MASK                                                                       0x00000001L
+//GFX_IMU_GFX_RESET_CTRL
+#define GFX_IMU_GFX_RESET_CTRL__HARD_RESETB__SHIFT                                                            0x0
+#define GFX_IMU_GFX_RESET_CTRL__EA_RESETB__SHIFT                                                              0x1
+#define GFX_IMU_GFX_RESET_CTRL__UTCL2_RESETB__SHIFT                                                           0x2
+#define GFX_IMU_GFX_RESET_CTRL__SDMA_RESETB__SHIFT                                                            0x3
+#define GFX_IMU_GFX_RESET_CTRL__GRBM_RESETB__SHIFT                                                            0x4
+#define GFX_IMU_GFX_RESET_CTRL__HARD_RESETB_MASK                                                              0x00000001L
+#define GFX_IMU_GFX_RESET_CTRL__EA_RESETB_MASK                                                                0x00000002L
+#define GFX_IMU_GFX_RESET_CTRL__UTCL2_RESETB_MASK                                                             0x00000004L
+#define GFX_IMU_GFX_RESET_CTRL__SDMA_RESETB_MASK                                                              0x00000008L
+#define GFX_IMU_GFX_RESET_CTRL__GRBM_RESETB_MASK                                                              0x00000010L
+//GFX_IMU_AEB_OVERRIDE
+#define GFX_IMU_AEB_OVERRIDE__AEB_OVERRIDE_CTRL__SHIFT                                                        0x0
+#define GFX_IMU_AEB_OVERRIDE__AEB_RESET_VALUE__SHIFT                                                          0x1
+#define GFX_IMU_AEB_OVERRIDE__AEB_VALID_VALUE__SHIFT                                                          0x2
+#define GFX_IMU_AEB_OVERRIDE__AEB_OVERRIDE_CTRL_MASK                                                          0x00000001L
+#define GFX_IMU_AEB_OVERRIDE__AEB_RESET_VALUE_MASK                                                            0x00000002L
+#define GFX_IMU_AEB_OVERRIDE__AEB_VALID_VALUE_MASK                                                            0x00000004L
+//GFX_IMU_VDCI_RESET_CTRL
+#define GFX_IMU_VDCI_RESET_CTRL__SOC2GFX_VDCI_RESETn__SHIFT                                                   0x0
+#define GFX_IMU_VDCI_RESET_CTRL__SOC_EA_SDF_VDCI_RESET__SHIFT                                                 0x1
+#define GFX_IMU_VDCI_RESET_CTRL__SOC_UTCL2_ATHUB_VDCI_RESET__SHIFT                                            0x2
+#define GFX_IMU_VDCI_RESET_CTRL__SOC_IMUAXI_SYSHUB_VDCI_RESET__SHIFT                                          0x3
+#define GFX_IMU_VDCI_RESET_CTRL__IMU2GFX_VDCI_RESETn__SHIFT                                                   0x4
+#define GFX_IMU_VDCI_RESET_CTRL__SOC2GFX_VDCI_RESETn_MASK                                                     0x00000001L
+#define GFX_IMU_VDCI_RESET_CTRL__SOC_EA_SDF_VDCI_RESET_MASK                                                   0x00000002L
+#define GFX_IMU_VDCI_RESET_CTRL__SOC_UTCL2_ATHUB_VDCI_RESET_MASK                                              0x00000004L
+#define GFX_IMU_VDCI_RESET_CTRL__SOC_IMUAXI_SYSHUB_VDCI_RESET_MASK                                            0x00000008L
+#define GFX_IMU_VDCI_RESET_CTRL__IMU2GFX_VDCI_RESETn_MASK                                                     0x00000010L
+//GFX_IMU_GFX_ISO_CTRL
+#define GFX_IMU_GFX_ISO_CTRL__GFX2IMU_ISOn__SHIFT                                                             0x0
+#define GFX_IMU_GFX_ISO_CTRL__SOC_EA_SDF_VDCI_ISOn_EN__SHIFT                                                  0x1
+#define GFX_IMU_GFX_ISO_CTRL__SOC_UTCL2_ATHUB_VDCI_ISOn_EN__SHIFT                                             0x2
+#define GFX_IMU_GFX_ISO_CTRL__GFX2SOC_ISOn__SHIFT                                                             0x3
+#define GFX_IMU_GFX_ISO_CTRL__GFX2SOC_CLK_ISOn__SHIFT                                                         0x4
+#define GFX_IMU_GFX_ISO_CTRL__GFX2IMU_ISOn_MASK                                                               0x00000001L
+#define GFX_IMU_GFX_ISO_CTRL__SOC_EA_SDF_VDCI_ISOn_EN_MASK                                                    0x00000002L
+#define GFX_IMU_GFX_ISO_CTRL__SOC_UTCL2_ATHUB_VDCI_ISOn_EN_MASK                                               0x00000004L
+#define GFX_IMU_GFX_ISO_CTRL__GFX2SOC_ISOn_MASK                                                               0x00000008L
+#define GFX_IMU_GFX_ISO_CTRL__GFX2SOC_CLK_ISOn_MASK                                                           0x00000010L
+//GFX_IMU_TIMER0_CTRL0
+#define GFX_IMU_TIMER0_CTRL0__START_STOP__SHIFT                                                               0x0
+#define GFX_IMU_TIMER0_CTRL0__CLEAR__SHIFT                                                                    0x8
+#define GFX_IMU_TIMER0_CTRL0__UP_DOWN__SHIFT                                                                  0x10
+#define GFX_IMU_TIMER0_CTRL0__PULSE_EN__SHIFT                                                                 0x18
+#define GFX_IMU_TIMER0_CTRL0__START_STOP_MASK                                                                 0x00000001L
+#define GFX_IMU_TIMER0_CTRL0__CLEAR_MASK                                                                      0x00000100L
+#define GFX_IMU_TIMER0_CTRL0__UP_DOWN_MASK                                                                    0x00010000L
+#define GFX_IMU_TIMER0_CTRL0__PULSE_EN_MASK                                                                   0x01000000L
+//GFX_IMU_TIMER0_CTRL1
+#define GFX_IMU_TIMER0_CTRL1__PWM_EN__SHIFT                                                                   0x0
+#define GFX_IMU_TIMER0_CTRL1__TS_MODE__SHIFT                                                                  0x8
+#define GFX_IMU_TIMER0_CTRL1__SAT_EN__SHIFT                                                                   0x10
+#define GFX_IMU_TIMER0_CTRL1__PWM_EN_MASK                                                                     0x00000001L
+#define GFX_IMU_TIMER0_CTRL1__TS_MODE_MASK                                                                    0x00000100L
+#define GFX_IMU_TIMER0_CTRL1__SAT_EN_MASK                                                                     0x00010000L
+//GFX_IMU_TIMER0_CMP_AUTOINC
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN0__SHIFT                                                        0x0
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN1__SHIFT                                                        0x1
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN2__SHIFT                                                        0x2
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN3__SHIFT                                                        0x3
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN0_MASK                                                          0x00000001L
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN1_MASK                                                          0x00000002L
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN2_MASK                                                          0x00000004L
+#define GFX_IMU_TIMER0_CMP_AUTOINC__AUTOINC_EN3_MASK                                                          0x00000008L
+//GFX_IMU_TIMER0_CMP_INTEN
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN0__SHIFT                                                              0x0
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN1__SHIFT                                                              0x1
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN2__SHIFT                                                              0x2
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN3__SHIFT                                                              0x3
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN0_MASK                                                                0x00000001L
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN1_MASK                                                                0x00000002L
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN2_MASK                                                                0x00000004L
+#define GFX_IMU_TIMER0_CMP_INTEN__INT_EN3_MASK                                                                0x00000008L
+//GFX_IMU_TIMER0_CMP0
+#define GFX_IMU_TIMER0_CMP0__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER0_CMP0__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER0_CMP1
+#define GFX_IMU_TIMER0_CMP1__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER0_CMP1__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER0_CMP3
+#define GFX_IMU_TIMER0_CMP3__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER0_CMP3__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER0_VALUE
+#define GFX_IMU_TIMER0_VALUE__VALUE__SHIFT                                                                    0x0
+#define GFX_IMU_TIMER0_VALUE__VALUE_MASK                                                                      0xFFFFFFFFL
+//GFX_IMU_TIMER1_CTRL0
+#define GFX_IMU_TIMER1_CTRL0__START_STOP__SHIFT                                                               0x0
+#define GFX_IMU_TIMER1_CTRL0__CLEAR__SHIFT                                                                    0x8
+#define GFX_IMU_TIMER1_CTRL0__UP_DOWN__SHIFT                                                                  0x10
+#define GFX_IMU_TIMER1_CTRL0__PULSE_EN__SHIFT                                                                 0x18
+#define GFX_IMU_TIMER1_CTRL0__START_STOP_MASK                                                                 0x00000001L
+#define GFX_IMU_TIMER1_CTRL0__CLEAR_MASK                                                                      0x00000100L
+#define GFX_IMU_TIMER1_CTRL0__UP_DOWN_MASK                                                                    0x00010000L
+#define GFX_IMU_TIMER1_CTRL0__PULSE_EN_MASK                                                                   0x01000000L
+//GFX_IMU_TIMER1_CTRL1
+#define GFX_IMU_TIMER1_CTRL1__PWM_EN__SHIFT                                                                   0x0
+#define GFX_IMU_TIMER1_CTRL1__TS_MODE__SHIFT                                                                  0x8
+#define GFX_IMU_TIMER1_CTRL1__SAT_EN__SHIFT                                                                   0x10
+#define GFX_IMU_TIMER1_CTRL1__PWM_EN_MASK                                                                     0x00000001L
+#define GFX_IMU_TIMER1_CTRL1__TS_MODE_MASK                                                                    0x00000100L
+#define GFX_IMU_TIMER1_CTRL1__SAT_EN_MASK                                                                     0x00010000L
+//GFX_IMU_TIMER1_CMP_AUTOINC
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN0__SHIFT                                                        0x0
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN1__SHIFT                                                        0x1
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN2__SHIFT                                                        0x2
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN3__SHIFT                                                        0x3
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN0_MASK                                                          0x00000001L
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN1_MASK                                                          0x00000002L
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN2_MASK                                                          0x00000004L
+#define GFX_IMU_TIMER1_CMP_AUTOINC__AUTOINC_EN3_MASK                                                          0x00000008L
+//GFX_IMU_TIMER1_CMP_INTEN
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN0__SHIFT                                                              0x0
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN1__SHIFT                                                              0x1
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN2__SHIFT                                                              0x2
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN3__SHIFT                                                              0x3
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN0_MASK                                                                0x00000001L
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN1_MASK                                                                0x00000002L
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN2_MASK                                                                0x00000004L
+#define GFX_IMU_TIMER1_CMP_INTEN__INT_EN3_MASK                                                                0x00000008L
+//GFX_IMU_TIMER1_CMP0
+#define GFX_IMU_TIMER1_CMP0__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER1_CMP0__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER1_CMP1
+#define GFX_IMU_TIMER1_CMP1__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER1_CMP1__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER1_CMP3
+#define GFX_IMU_TIMER1_CMP3__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER1_CMP3__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER1_VALUE
+#define GFX_IMU_TIMER1_VALUE__VALUE__SHIFT                                                                    0x0
+#define GFX_IMU_TIMER1_VALUE__VALUE_MASK                                                                      0xFFFFFFFFL
+//GFX_IMU_TIMER2_CTRL0
+#define GFX_IMU_TIMER2_CTRL0__START_STOP__SHIFT                                                               0x0
+#define GFX_IMU_TIMER2_CTRL0__CLEAR__SHIFT                                                                    0x8
+#define GFX_IMU_TIMER2_CTRL0__UP_DOWN__SHIFT                                                                  0x10
+#define GFX_IMU_TIMER2_CTRL0__PULSE_EN__SHIFT                                                                 0x18
+#define GFX_IMU_TIMER2_CTRL0__START_STOP_MASK                                                                 0x00000001L
+#define GFX_IMU_TIMER2_CTRL0__CLEAR_MASK                                                                      0x00000100L
+#define GFX_IMU_TIMER2_CTRL0__UP_DOWN_MASK                                                                    0x00010000L
+#define GFX_IMU_TIMER2_CTRL0__PULSE_EN_MASK                                                                   0x01000000L
+//GFX_IMU_TIMER2_CTRL1
+#define GFX_IMU_TIMER2_CTRL1__PWM_EN__SHIFT                                                                   0x0
+#define GFX_IMU_TIMER2_CTRL1__TS_MODE__SHIFT                                                                  0x8
+#define GFX_IMU_TIMER2_CTRL1__SAT_EN__SHIFT                                                                   0x10
+#define GFX_IMU_TIMER2_CTRL1__PWM_EN_MASK                                                                     0x00000001L
+#define GFX_IMU_TIMER2_CTRL1__TS_MODE_MASK                                                                    0x00000100L
+#define GFX_IMU_TIMER2_CTRL1__SAT_EN_MASK                                                                     0x00010000L
+//GFX_IMU_TIMER2_CMP_AUTOINC
+#define GFX_IMU_TIMER2_CMP_AUTOINC__AUTOINC_EN0__SHIFT                                                        0x0
+#define GFX_IMU_TIMER2_CMP_AUTOINC__AUTOINC_EN1__SHIFT                                                        0x1
+#define GFX_IMU_TIMER2_CMP_AUTOINC__AUTOINC_EN2__SHIFT                                                        0x2
+#define GFX_IMU_TIMER2_CMP_AUTOINC__AUTOINC_EN3__SHIFT                                                        0x3
+#define GFX_IMU_TIMER2_CMP_AUTOINC__AUTOINC_EN0_MASK                                                          0x00000001L
+#define GFX_IMU_TIMER2_CMP_AUTOINC__AUTOINC_EN1_MASK                                                          0x00000002L
+#define GFX_IMU_TIMER2_CMP_AUTOINC__AUTOINC_EN2_MASK                                                          0x00000004L
+#define GFX_IMU_TIMER2_CMP_AUTOINC__AUTOINC_EN3_MASK                                                          0x00000008L
+//GFX_IMU_TIMER2_CMP_INTEN
+#define GFX_IMU_TIMER2_CMP_INTEN__INT_EN0__SHIFT                                                              0x0
+#define GFX_IMU_TIMER2_CMP_INTEN__INT_EN1__SHIFT                                                              0x1
+#define GFX_IMU_TIMER2_CMP_INTEN__INT_EN2__SHIFT                                                              0x2
+#define GFX_IMU_TIMER2_CMP_INTEN__INT_EN3__SHIFT                                                              0x3
+#define GFX_IMU_TIMER2_CMP_INTEN__INT_EN0_MASK                                                                0x00000001L
+#define GFX_IMU_TIMER2_CMP_INTEN__INT_EN1_MASK                                                                0x00000002L
+#define GFX_IMU_TIMER2_CMP_INTEN__INT_EN2_MASK                                                                0x00000004L
+#define GFX_IMU_TIMER2_CMP_INTEN__INT_EN3_MASK                                                                0x00000008L
+//GFX_IMU_TIMER2_CMP0
+#define GFX_IMU_TIMER2_CMP0__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER2_CMP0__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER2_CMP1
+#define GFX_IMU_TIMER2_CMP1__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER2_CMP1__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER2_CMP3
+#define GFX_IMU_TIMER2_CMP3__VALUE__SHIFT                                                                     0x0
+#define GFX_IMU_TIMER2_CMP3__VALUE_MASK                                                                       0xFFFFFFFFL
+//GFX_IMU_TIMER2_VALUE
+#define GFX_IMU_TIMER2_VALUE__VALUE__SHIFT                                                                    0x0
+#define GFX_IMU_TIMER2_VALUE__VALUE_MASK                                                                      0xFFFFFFFFL
+//GFX_IMU_FUSE_CTRL
+#define GFX_IMU_FUSE_CTRL__DIV_OVR__SHIFT                                                                     0x0
+#define GFX_IMU_FUSE_CTRL__DIV_OVR_EN__SHIFT                                                                  0x5
+#define GFX_IMU_FUSE_CTRL__FORCE_DONE__SHIFT                                                                  0x6
+#define GFX_IMU_FUSE_CTRL__DIV_OVR_MASK                                                                       0x0000001FL
+#define GFX_IMU_FUSE_CTRL__DIV_OVR_EN_MASK                                                                    0x00000020L
+#define GFX_IMU_FUSE_CTRL__FORCE_DONE_MASK                                                                    0x00000040L
+//GFX_IMU_D_RAM_ADDR
+#define GFX_IMU_D_RAM_ADDR__ADDR__SHIFT                                                                       0x2
+#define GFX_IMU_D_RAM_ADDR__ADDR_MASK                                                                         0x0000FFFCL
+//GFX_IMU_D_RAM_DATA
+#define GFX_IMU_D_RAM_DATA__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_D_RAM_DATA__DATA_MASK                                                                         0xFFFFFFFFL
+//GFX_IMU_GFX_IH_GASKET_CTRL
+#define GFX_IMU_GFX_IH_GASKET_CTRL__SRSTB__SHIFT                                                              0x0
+#define GFX_IMU_GFX_IH_GASKET_CTRL__BUFFER_LEVEL__SHIFT                                                       0x10
+#define GFX_IMU_GFX_IH_GASKET_CTRL__BUFFER_OVERFLOW__SHIFT                                                    0x14
+#define GFX_IMU_GFX_IH_GASKET_CTRL__SRSTB_MASK                                                                0x00000001L
+#define GFX_IMU_GFX_IH_GASKET_CTRL__BUFFER_LEVEL_MASK                                                         0x000F0000L
+#define GFX_IMU_GFX_IH_GASKET_CTRL__BUFFER_OVERFLOW_MASK                                                      0x00100000L
+
+
+// addressBlock: gc_gfx_imu_gfx_imu_pspdec
+//GFX_IMU_RLC_BOOTLOADER_ADDR_HI
+#define GFX_IMU_RLC_BOOTLOADER_ADDR_HI__ADDR_HI__SHIFT                                                        0x0
+#define GFX_IMU_RLC_BOOTLOADER_ADDR_HI__ADDR_HI_MASK                                                          0xFFFFFFFFL
+//GFX_IMU_RLC_BOOTLOADER_ADDR_LO
+#define GFX_IMU_RLC_BOOTLOADER_ADDR_LO__ADDR_LO__SHIFT                                                        0x0
+#define GFX_IMU_RLC_BOOTLOADER_ADDR_LO__ADDR_LO_MASK                                                          0xFFFFFFFFL
+//GFX_IMU_RLC_BOOTLOADER_SIZE
+#define GFX_IMU_RLC_BOOTLOADER_SIZE__SIZE__SHIFT                                                              0x0
+#define GFX_IMU_RLC_BOOTLOADER_SIZE__SIZE_MASK                                                                0x03FFFFFFL
+//GFX_IMU_I_RAM_ADDR
+#define GFX_IMU_I_RAM_ADDR__ADDR__SHIFT                                                                       0x2
+#define GFX_IMU_I_RAM_ADDR__ADDR_MASK                                                                         0x0000FFFCL
+//GFX_IMU_I_RAM_DATA
+#define GFX_IMU_I_RAM_DATA__DATA__SHIFT                                                                       0x0
+#define GFX_IMU_I_RAM_DATA__DATA_MASK                                                                         0xFFFFFFFFL
+
+
+// addressBlock: gccacind
+//GC_CAC_ID
+#define GC_CAC_ID__CAC_BLOCK_ID__SHIFT                                                                        0x0
+#define GC_CAC_ID__CAC_SIGNAL_ID__SHIFT                                                                       0x6
+#define GC_CAC_ID__CAC_BLOCK_ID_MASK                                                                          0x0000003FL
+#define GC_CAC_ID__CAC_SIGNAL_ID_MASK                                                                         0x00003FC0L
+//GC_CAC_CNTL
+#define GC_CAC_CNTL__CAC_THRESHOLD__SHIFT                                                                     0x0
+#define GC_CAC_CNTL__CAC_THRESHOLD_MASK                                                                       0x0000FFFFL
+//GC_CAC_ACC_CP0
+#define GC_CAC_ACC_CP0__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_CP0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_CP1
+#define GC_CAC_ACC_CP1__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_CP1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_CP2
+#define GC_CAC_ACC_CP2__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_CP2__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_EA0
+#define GC_CAC_ACC_EA0__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_EA0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_EA1
+#define GC_CAC_ACC_EA1__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_EA1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_EA2
+#define GC_CAC_ACC_EA2__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_EA2__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_EA3
+#define GC_CAC_ACC_EA3__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_EA3__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_EA4
+#define GC_CAC_ACC_EA4__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_EA4__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_EA5
+#define GC_CAC_ACC_EA5__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_EA5__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ROUTER0
+#define GC_CAC_ACC_UTCL2_ROUTER0__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_ROUTER0__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ROUTER1
+#define GC_CAC_ACC_UTCL2_ROUTER1__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_ROUTER1__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ROUTER2
+#define GC_CAC_ACC_UTCL2_ROUTER2__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_ROUTER2__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ROUTER3
+#define GC_CAC_ACC_UTCL2_ROUTER3__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_ROUTER3__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ROUTER4
+#define GC_CAC_ACC_UTCL2_ROUTER4__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_ROUTER4__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ROUTER5
+#define GC_CAC_ACC_UTCL2_ROUTER5__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_ROUTER5__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ROUTER6
+#define GC_CAC_ACC_UTCL2_ROUTER6__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_ROUTER6__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ROUTER7
+#define GC_CAC_ACC_UTCL2_ROUTER7__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_ROUTER7__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ROUTER8
+#define GC_CAC_ACC_UTCL2_ROUTER8__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_ROUTER8__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ROUTER9
+#define GC_CAC_ACC_UTCL2_ROUTER9__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_ROUTER9__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_VML20
+#define GC_CAC_ACC_UTCL2_VML20__ACCUMULATOR_31_0__SHIFT                                                       0x0
+#define GC_CAC_ACC_UTCL2_VML20__ACCUMULATOR_31_0_MASK                                                         0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_VML21
+#define GC_CAC_ACC_UTCL2_VML21__ACCUMULATOR_31_0__SHIFT                                                       0x0
+#define GC_CAC_ACC_UTCL2_VML21__ACCUMULATOR_31_0_MASK                                                         0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_VML22
+#define GC_CAC_ACC_UTCL2_VML22__ACCUMULATOR_31_0__SHIFT                                                       0x0
+#define GC_CAC_ACC_UTCL2_VML22__ACCUMULATOR_31_0_MASK                                                         0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_VML23
+#define GC_CAC_ACC_UTCL2_VML23__ACCUMULATOR_31_0__SHIFT                                                       0x0
+#define GC_CAC_ACC_UTCL2_VML23__ACCUMULATOR_31_0_MASK                                                         0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_VML24
+#define GC_CAC_ACC_UTCL2_VML24__ACCUMULATOR_31_0__SHIFT                                                       0x0
+#define GC_CAC_ACC_UTCL2_VML24__ACCUMULATOR_31_0_MASK                                                         0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_WALKER0
+#define GC_CAC_ACC_UTCL2_WALKER0__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_WALKER0__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_WALKER1
+#define GC_CAC_ACC_UTCL2_WALKER1__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_WALKER1__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_WALKER2
+#define GC_CAC_ACC_UTCL2_WALKER2__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_WALKER2__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_WALKER3
+#define GC_CAC_ACC_UTCL2_WALKER3__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_WALKER3__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_WALKER4
+#define GC_CAC_ACC_UTCL2_WALKER4__ACCUMULATOR_31_0__SHIFT                                                     0x0
+#define GC_CAC_ACC_UTCL2_WALKER4__ACCUMULATOR_31_0_MASK                                                       0xFFFFFFFFL
+//GC_CAC_ACC_GDS0
+#define GC_CAC_ACC_GDS0__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GDS0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GDS1
+#define GC_CAC_ACC_GDS1__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GDS1__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GDS2
+#define GC_CAC_ACC_GDS2__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GDS2__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GDS3
+#define GC_CAC_ACC_GDS3__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GDS3__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GDS4
+#define GC_CAC_ACC_GDS4__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GDS4__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE0
+#define GC_CAC_ACC_GE0__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_GE0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_GE1
+#define GC_CAC_ACC_GE1__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_GE1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_GE2
+#define GC_CAC_ACC_GE2__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_GE2__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_GE3
+#define GC_CAC_ACC_GE3__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_GE3__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_GE4
+#define GC_CAC_ACC_GE4__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_GE4__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_GE5
+#define GC_CAC_ACC_GE5__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_GE5__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_GE6
+#define GC_CAC_ACC_GE6__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_GE6__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_GE7
+#define GC_CAC_ACC_GE7__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_GE7__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_GE8
+#define GC_CAC_ACC_GE8__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_GE8__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_GE9
+#define GC_CAC_ACC_GE9__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_GE9__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_GE10
+#define GC_CAC_ACC_GE10__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE10__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE11
+#define GC_CAC_ACC_GE11__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE11__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE12
+#define GC_CAC_ACC_GE12__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE12__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE13
+#define GC_CAC_ACC_GE13__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE13__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE14
+#define GC_CAC_ACC_GE14__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE14__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE15
+#define GC_CAC_ACC_GE15__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE15__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE16
+#define GC_CAC_ACC_GE16__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE16__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE17
+#define GC_CAC_ACC_GE17__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE17__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE18
+#define GC_CAC_ACC_GE18__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE18__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE19
+#define GC_CAC_ACC_GE19__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE19__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GE20
+#define GC_CAC_ACC_GE20__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GE20__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_PMM0
+#define GC_CAC_ACC_PMM0__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_PMM0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GL2C0
+#define GC_CAC_ACC_GL2C0__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_GL2C0__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_GL2C1
+#define GC_CAC_ACC_GL2C1__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_GL2C1__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_GL2C2
+#define GC_CAC_ACC_GL2C2__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_GL2C2__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_GL2C3
+#define GC_CAC_ACC_GL2C3__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_GL2C3__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_GL2C4
+#define GC_CAC_ACC_GL2C4__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_GL2C4__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_PH0
+#define GC_CAC_ACC_PH0__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_PH0__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_PH1
+#define GC_CAC_ACC_PH1__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_PH1__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_PH2
+#define GC_CAC_ACC_PH2__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_PH2__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_PH3
+#define GC_CAC_ACC_PH3__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_PH3__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_PH4
+#define GC_CAC_ACC_PH4__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_PH4__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_PH5
+#define GC_CAC_ACC_PH5__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_PH5__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_PH6
+#define GC_CAC_ACC_PH6__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_PH6__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_PH7
+#define GC_CAC_ACC_PH7__ACCUMULATOR_31_0__SHIFT                                                               0x0
+#define GC_CAC_ACC_PH7__ACCUMULATOR_31_0_MASK                                                                 0xFFFFFFFFL
+//GC_CAC_ACC_SDMA0
+#define GC_CAC_ACC_SDMA0__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_SDMA0__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_SDMA1
+#define GC_CAC_ACC_SDMA1__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_SDMA1__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_SDMA2
+#define GC_CAC_ACC_SDMA2__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_SDMA2__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_SDMA3
+#define GC_CAC_ACC_SDMA3__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_SDMA3__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_SDMA4
+#define GC_CAC_ACC_SDMA4__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_SDMA4__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_SDMA5
+#define GC_CAC_ACC_SDMA5__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_SDMA5__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_SDMA6
+#define GC_CAC_ACC_SDMA6__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_SDMA6__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_SDMA7
+#define GC_CAC_ACC_SDMA7__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_SDMA7__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_SDMA8
+#define GC_CAC_ACC_SDMA8__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_SDMA8__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_SDMA9
+#define GC_CAC_ACC_SDMA9__ACCUMULATOR_31_0__SHIFT                                                             0x0
+#define GC_CAC_ACC_SDMA9__ACCUMULATOR_31_0_MASK                                                               0xFFFFFFFFL
+//GC_CAC_ACC_SDMA10
+#define GC_CAC_ACC_SDMA10__ACCUMULATOR_31_0__SHIFT                                                            0x0
+#define GC_CAC_ACC_SDMA10__ACCUMULATOR_31_0_MASK                                                              0xFFFFFFFFL
+//GC_CAC_ACC_SDMA11
+#define GC_CAC_ACC_SDMA11__ACCUMULATOR_31_0__SHIFT                                                            0x0
+#define GC_CAC_ACC_SDMA11__ACCUMULATOR_31_0_MASK                                                              0xFFFFFFFFL
+//GC_CAC_ACC_CHC0
+#define GC_CAC_ACC_CHC0__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_CHC0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_CHC1
+#define GC_CAC_ACC_CHC1__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_CHC1__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_CHC2
+#define GC_CAC_ACC_CHC2__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_CHC2__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GUS0
+#define GC_CAC_ACC_GUS0__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GUS0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GUS1
+#define GC_CAC_ACC_GUS1__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GUS1__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_GUS2
+#define GC_CAC_ACC_GUS2__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_GUS2__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_RLC0
+#define GC_CAC_ACC_RLC0__ACCUMULATOR_31_0__SHIFT                                                              0x0
+#define GC_CAC_ACC_RLC0__ACCUMULATOR_31_0_MASK                                                                0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ATCL20
+#define GC_CAC_ACC_UTCL2_ATCL20__ACCUMULATOR_31_0__SHIFT                                                      0x0
+#define GC_CAC_ACC_UTCL2_ATCL20__ACCUMULATOR_31_0_MASK                                                        0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ATCL21
+#define GC_CAC_ACC_UTCL2_ATCL21__ACCUMULATOR_31_0__SHIFT                                                      0x0
+#define GC_CAC_ACC_UTCL2_ATCL21__ACCUMULATOR_31_0_MASK                                                        0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ATCL22
+#define GC_CAC_ACC_UTCL2_ATCL22__ACCUMULATOR_31_0__SHIFT                                                      0x0
+#define GC_CAC_ACC_UTCL2_ATCL22__ACCUMULATOR_31_0_MASK                                                        0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ATCL23
+#define GC_CAC_ACC_UTCL2_ATCL23__ACCUMULATOR_31_0__SHIFT                                                      0x0
+#define GC_CAC_ACC_UTCL2_ATCL23__ACCUMULATOR_31_0_MASK                                                        0xFFFFFFFFL
+//GC_CAC_ACC_UTCL2_ATCL24
+#define GC_CAC_ACC_UTCL2_ATCL24__ACCUMULATOR_31_0__SHIFT                                                      0x0
+#define GC_CAC_ACC_UTCL2_ATCL24__ACCUMULATOR_31_0_MASK                                                        0xFFFFFFFFL
+//RELEASE_TO_STALL_LUT_1_8
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_1__SHIFT                                                      0x0
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_2__SHIFT                                                      0x4
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_3__SHIFT                                                      0x8
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_4__SHIFT                                                      0xc
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_5__SHIFT                                                      0x10
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_6__SHIFT                                                      0x14
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_7__SHIFT                                                      0x18
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_8__SHIFT                                                      0x1c
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_1_MASK                                                        0x00000007L
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_2_MASK                                                        0x00000070L
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_3_MASK                                                        0x00000700L
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_4_MASK                                                        0x00007000L
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_5_MASK                                                        0x00070000L
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_6_MASK                                                        0x00700000L
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_7_MASK                                                        0x07000000L
+#define RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_8_MASK                                                        0x70000000L
+//RELEASE_TO_STALL_LUT_9_16
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_9__SHIFT                                                     0x0
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_10__SHIFT                                                    0x4
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_11__SHIFT                                                    0x8
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_12__SHIFT                                                    0xc
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_13__SHIFT                                                    0x10
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_14__SHIFT                                                    0x14
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_15__SHIFT                                                    0x18
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_16__SHIFT                                                    0x1c
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_9_MASK                                                       0x00000007L
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_10_MASK                                                      0x00000070L
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_11_MASK                                                      0x00000700L
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_12_MASK                                                      0x00007000L
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_13_MASK                                                      0x00070000L
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_14_MASK                                                      0x00700000L
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_15_MASK                                                      0x07000000L
+#define RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_16_MASK                                                      0x70000000L
+//RELEASE_TO_STALL_LUT_17_20
+#define RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_17__SHIFT                                                   0x0
+#define RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_18__SHIFT                                                   0x4
+#define RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_19__SHIFT                                                   0x8
+#define RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_20__SHIFT                                                   0xc
+#define RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_17_MASK                                                     0x00000007L
+#define RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_18_MASK                                                     0x00000070L
+#define RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_19_MASK                                                     0x00000700L
+#define RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_20_MASK                                                     0x00007000L
+//STALL_TO_RELEASE_LUT_1_4
+#define STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_1__SHIFT                                                      0x0
+#define STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_2__SHIFT                                                      0x8
+#define STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_3__SHIFT                                                      0x10
+#define STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_4__SHIFT                                                      0x18
+#define STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_1_MASK                                                        0x0000001FL
+#define STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_2_MASK                                                        0x00001F00L
+#define STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_3_MASK                                                        0x001F0000L
+#define STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_4_MASK                                                        0x1F000000L
+//STALL_TO_RELEASE_LUT_5_7
+#define STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_5__SHIFT                                                      0x0
+#define STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_6__SHIFT                                                      0x8
+#define STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_7__SHIFT                                                      0x10
+#define STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_5_MASK                                                        0x0000001FL
+#define STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_6_MASK                                                        0x00001F00L
+#define STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_7_MASK                                                        0x001F0000L
+//STALL_TO_PWRBRK_LUT_1_4
+#define STALL_TO_PWRBRK_LUT_1_4__FIRST_PATTERN_1__SHIFT                                                       0x0
+#define STALL_TO_PWRBRK_LUT_1_4__FIRST_PATTERN_2__SHIFT                                                       0x8
+#define STALL_TO_PWRBRK_LUT_1_4__FIRST_PATTERN_3__SHIFT                                                       0x10
+#define STALL_TO_PWRBRK_LUT_1_4__FIRST_PATTERN_4__SHIFT                                                       0x18
+#define STALL_TO_PWRBRK_LUT_1_4__FIRST_PATTERN_1_MASK                                                         0x00000007L
+#define STALL_TO_PWRBRK_LUT_1_4__FIRST_PATTERN_2_MASK                                                         0x00000700L
+#define STALL_TO_PWRBRK_LUT_1_4__FIRST_PATTERN_3_MASK                                                         0x00070000L
+#define STALL_TO_PWRBRK_LUT_1_4__FIRST_PATTERN_4_MASK                                                         0x07000000L
+//STALL_TO_PWRBRK_LUT_5_7
+#define STALL_TO_PWRBRK_LUT_5_7__FIRST_PATTERN_5__SHIFT                                                       0x0
+#define STALL_TO_PWRBRK_LUT_5_7__FIRST_PATTERN_6__SHIFT                                                       0x8
+#define STALL_TO_PWRBRK_LUT_5_7__FIRST_PATTERN_7__SHIFT                                                       0x10
+#define STALL_TO_PWRBRK_LUT_5_7__FIRST_PATTERN_5_MASK                                                         0x00000007L
+#define STALL_TO_PWRBRK_LUT_5_7__FIRST_PATTERN_6_MASK                                                         0x00000700L
+#define STALL_TO_PWRBRK_LUT_5_7__FIRST_PATTERN_7_MASK                                                         0x00070000L
+//PWRBRK_STALL_TO_RELEASE_LUT_1_4
+#define PWRBRK_STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_1__SHIFT                                               0x0
+#define PWRBRK_STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_2__SHIFT                                               0x8
+#define PWRBRK_STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_3__SHIFT                                               0x10
+#define PWRBRK_STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_4__SHIFT                                               0x18
+#define PWRBRK_STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_1_MASK                                                 0x0000001FL
+#define PWRBRK_STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_2_MASK                                                 0x00001F00L
+#define PWRBRK_STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_3_MASK                                                 0x001F0000L
+#define PWRBRK_STALL_TO_RELEASE_LUT_1_4__FIRST_PATTERN_4_MASK                                                 0x1F000000L
+//PWRBRK_STALL_TO_RELEASE_LUT_5_7
+#define PWRBRK_STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_5__SHIFT                                               0x0
+#define PWRBRK_STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_6__SHIFT                                               0x8
+#define PWRBRK_STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_7__SHIFT                                               0x10
+#define PWRBRK_STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_5_MASK                                                 0x0000001FL
+#define PWRBRK_STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_6_MASK                                                 0x00001F00L
+#define PWRBRK_STALL_TO_RELEASE_LUT_5_7__FIRST_PATTERN_7_MASK                                                 0x001F0000L
+//PWRBRK_RELEASE_TO_STALL_LUT_1_8
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_1__SHIFT                                               0x0
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_2__SHIFT                                               0x4
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_3__SHIFT                                               0x8
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_4__SHIFT                                               0xc
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_5__SHIFT                                               0x10
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_6__SHIFT                                               0x14
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_7__SHIFT                                               0x18
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_8__SHIFT                                               0x1c
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_1_MASK                                                 0x00000007L
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_2_MASK                                                 0x00000070L
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_3_MASK                                                 0x00000700L
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_4_MASK                                                 0x00007000L
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_5_MASK                                                 0x00070000L
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_6_MASK                                                 0x00700000L
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_7_MASK                                                 0x07000000L
+#define PWRBRK_RELEASE_TO_STALL_LUT_1_8__FIRST_PATTERN_8_MASK                                                 0x70000000L
+//PWRBRK_RELEASE_TO_STALL_LUT_9_16
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_9__SHIFT                                              0x0
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_10__SHIFT                                             0x4
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_11__SHIFT                                             0x8
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_12__SHIFT                                             0xc
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_13__SHIFT                                             0x10
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_14__SHIFT                                             0x14
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_15__SHIFT                                             0x18
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_16__SHIFT                                             0x1c
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_9_MASK                                                0x00000007L
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_10_MASK                                               0x00000070L
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_11_MASK                                               0x00000700L
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_12_MASK                                               0x00007000L
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_13_MASK                                               0x00070000L
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_14_MASK                                               0x00700000L
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_15_MASK                                               0x07000000L
+#define PWRBRK_RELEASE_TO_STALL_LUT_9_16__FIRST_PATTERN_16_MASK                                               0x70000000L
+//PWRBRK_RELEASE_TO_STALL_LUT_17_20
+#define PWRBRK_RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_17__SHIFT                                            0x0
+#define PWRBRK_RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_18__SHIFT                                            0x4
+#define PWRBRK_RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_19__SHIFT                                            0x8
+#define PWRBRK_RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_20__SHIFT                                            0xc
+#define PWRBRK_RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_17_MASK                                              0x00000007L
+#define PWRBRK_RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_18_MASK                                              0x00000070L
+#define PWRBRK_RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_19_MASK                                              0x00000700L
+#define PWRBRK_RELEASE_TO_STALL_LUT_17_20__FIRST_PATTERN_20_MASK                                              0x00007000L
+//FIXED_PATTERN_PERF_COUNTER_1
+#define FIXED_PATTERN_PERF_COUNTER_1__PERF_COUNTER__SHIFT                                                     0x0
+#define FIXED_PATTERN_PERF_COUNTER_1__PERF_COUNTER_MASK                                                       0x0001FFFFL
+//FIXED_PATTERN_PERF_COUNTER_2
+#define FIXED_PATTERN_PERF_COUNTER_2__PERF_COUNTER__SHIFT                                                     0x0
+#define FIXED_PATTERN_PERF_COUNTER_2__PERF_COUNTER_MASK                                                       0x0001FFFFL
+//FIXED_PATTERN_PERF_COUNTER_3
+#define FIXED_PATTERN_PERF_COUNTER_3__PERF_COUNTER__SHIFT                                                     0x0
+#define FIXED_PATTERN_PERF_COUNTER_3__PERF_COUNTER_MASK                                                       0x0001FFFFL
+//FIXED_PATTERN_PERF_COUNTER_4
+#define FIXED_PATTERN_PERF_COUNTER_4__PERF_COUNTER__SHIFT                                                     0x0
+#define FIXED_PATTERN_PERF_COUNTER_4__PERF_COUNTER_MASK                                                       0x0001FFFFL
+//FIXED_PATTERN_PERF_COUNTER_5
+#define FIXED_PATTERN_PERF_COUNTER_5__PERF_COUNTER__SHIFT                                                     0x0
+#define FIXED_PATTERN_PERF_COUNTER_5__PERF_COUNTER_MASK                                                       0x0001FFFFL
+//FIXED_PATTERN_PERF_COUNTER_6
+#define FIXED_PATTERN_PERF_COUNTER_6__PERF_COUNTER__SHIFT                                                     0x0
+#define FIXED_PATTERN_PERF_COUNTER_6__PERF_COUNTER_MASK                                                       0x0001FFFFL
+//FIXED_PATTERN_PERF_COUNTER_7
+#define FIXED_PATTERN_PERF_COUNTER_7__PERF_COUNTER__SHIFT                                                     0x0
+#define FIXED_PATTERN_PERF_COUNTER_7__PERF_COUNTER_MASK                                                       0x0001FFFFL
+//FIXED_PATTERN_PERF_COUNTER_8
+#define FIXED_PATTERN_PERF_COUNTER_8__PERF_COUNTER__SHIFT                                                     0x0
+#define FIXED_PATTERN_PERF_COUNTER_8__PERF_COUNTER_MASK                                                       0x0001FFFFL
+//FIXED_PATTERN_PERF_COUNTER_9
+#define FIXED_PATTERN_PERF_COUNTER_9__PERF_COUNTER__SHIFT                                                     0x0
+#define FIXED_PATTERN_PERF_COUNTER_9__PERF_COUNTER_MASK                                                       0x0001FFFFL
+//FIXED_PATTERN_PERF_COUNTER_10
+#define FIXED_PATTERN_PERF_COUNTER_10__PERF_COUNTER__SHIFT                                                    0x0
+#define FIXED_PATTERN_PERF_COUNTER_10__PERF_COUNTER_MASK                                                      0x0001FFFFL
+//HW_LUT_UPDATE_STATUS
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_1_DONE__SHIFT                                                      0x0
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_1_ERROR__SHIFT                                                     0x1
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_1_ERROR_STEP__SHIFT                                                0x2
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_2_DONE__SHIFT                                                      0x5
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_2_ERROR__SHIFT                                                     0x6
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_2_ERROR_STEP__SHIFT                                                0x7
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_3_DONE__SHIFT                                                      0xa
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_3_ERROR__SHIFT                                                     0xb
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_3_ERROR_STEP__SHIFT                                                0xc
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_4_DONE__SHIFT                                                      0x11
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_4_ERROR__SHIFT                                                     0x12
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_4_ERROR_STEP__SHIFT                                                0x13
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_5_DONE__SHIFT                                                      0x16
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_5_ERROR__SHIFT                                                     0x17
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_5_ERROR_STEP__SHIFT                                                0x18
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_1_DONE_MASK                                                        0x00000001L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_1_ERROR_MASK                                                       0x00000002L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_1_ERROR_STEP_MASK                                                  0x0000001CL
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_2_DONE_MASK                                                        0x00000020L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_2_ERROR_MASK                                                       0x00000040L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_2_ERROR_STEP_MASK                                                  0x00000380L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_3_DONE_MASK                                                        0x00000400L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_3_ERROR_MASK                                                       0x00000800L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_3_ERROR_STEP_MASK                                                  0x0001F000L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_4_DONE_MASK                                                        0x00020000L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_4_ERROR_MASK                                                       0x00040000L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_4_ERROR_STEP_MASK                                                  0x00380000L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_5_DONE_MASK                                                        0x00400000L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_5_ERROR_MASK                                                       0x00800000L
+#define HW_LUT_UPDATE_STATUS__UPDATE_TABLE_5_ERROR_STEP_MASK                                                  0x1F000000L
+
+
+// addressBlock: secacind
+//SE_CAC_ID
+#define SE_CAC_ID__CAC_BLOCK_ID__SHIFT                                                                        0x0
+#define SE_CAC_ID__CAC_SIGNAL_ID__SHIFT                                                                       0x6
+#define SE_CAC_ID__CAC_BLOCK_ID_MASK                                                                          0x0000003FL
+#define SE_CAC_ID__CAC_SIGNAL_ID_MASK                                                                         0x00003FC0L
+//SE_CAC_CNTL
+#define SE_CAC_CNTL__CAC_THRESHOLD__SHIFT                                                                     0x0
+#define SE_CAC_CNTL__CAC_THRESHOLD_MASK                                                                       0x0000FFFFL
+
+
+// addressBlock: grtavfsind
+//RTAVFS_REG0
+#define RTAVFS_REG0__RTAVFSZONE0STARTCNT__SHIFT                                                               0x0
+#define RTAVFS_REG0__RTAVFSZONE0STOPCNT__SHIFT                                                                0x10
+#define RTAVFS_REG0__RTAVFSZONE0STARTCNT_MASK                                                                 0x0000FFFFL
+#define RTAVFS_REG0__RTAVFSZONE0STOPCNT_MASK                                                                  0xFFFF0000L
+//RTAVFS_REG1
+#define RTAVFS_REG1__RTAVFSZONE1STARTCNT__SHIFT                                                               0x0
+#define RTAVFS_REG1__RTAVFSZONE1STOPCNT__SHIFT                                                                0x10
+#define RTAVFS_REG1__RTAVFSZONE1STARTCNT_MASK                                                                 0x0000FFFFL
+#define RTAVFS_REG1__RTAVFSZONE1STOPCNT_MASK                                                                  0xFFFF0000L
+//RTAVFS_REG2
+#define RTAVFS_REG2__RTAVFSZONE2STARTCNT__SHIFT                                                               0x0
+#define RTAVFS_REG2__RTAVFSZONE2STOPCNT__SHIFT                                                                0x10
+#define RTAVFS_REG2__RTAVFSZONE2STARTCNT_MASK                                                                 0x0000FFFFL
+#define RTAVFS_REG2__RTAVFSZONE2STOPCNT_MASK                                                                  0xFFFF0000L
+//RTAVFS_REG3
+#define RTAVFS_REG3__RTAVFSZONE3STARTCNT__SHIFT                                                               0x0
+#define RTAVFS_REG3__RTAVFSZONE3STOPCNT__SHIFT                                                                0x10
+#define RTAVFS_REG3__RTAVFSZONE3STARTCNT_MASK                                                                 0x0000FFFFL
+#define RTAVFS_REG3__RTAVFSZONE3STOPCNT_MASK                                                                  0xFFFF0000L
+//RTAVFS_REG4
+#define RTAVFS_REG4__RTAVFSZONE4STARTCNT__SHIFT                                                               0x0
+#define RTAVFS_REG4__RTAVFSZONE4STOPCNT__SHIFT                                                                0x10
+#define RTAVFS_REG4__RTAVFSZONE4STARTCNT_MASK                                                                 0x0000FFFFL
+#define RTAVFS_REG4__RTAVFSZONE4STOPCNT_MASK                                                                  0xFFFF0000L
+//RTAVFS_REG5
+#define RTAVFS_REG5__RTAVFSZONE0EN0__SHIFT                                                                    0x0
+#define RTAVFS_REG5__RTAVFSZONE0EN0_MASK                                                                      0xFFFFFFFFL
+//RTAVFS_REG6
+#define RTAVFS_REG6__RTAVFSZONE0EN1__SHIFT                                                                    0x0
+#define RTAVFS_REG6__RTAVFSZONE0EN1_MASK                                                                      0xFFFFFFFFL
+//RTAVFS_REG7
+#define RTAVFS_REG7__RTAVFSZONE1EN0__SHIFT                                                                    0x0
+#define RTAVFS_REG7__RTAVFSZONE1EN0_MASK                                                                      0xFFFFFFFFL
+//RTAVFS_REG8
+#define RTAVFS_REG8__RTAVFSZONE1EN1__SHIFT                                                                    0x0
+#define RTAVFS_REG8__RTAVFSZONE1EN1_MASK                                                                      0xFFFFFFFFL
+//RTAVFS_REG9
+#define RTAVFS_REG9__RTAVFSZONE2EN0__SHIFT                                                                    0x0
+#define RTAVFS_REG9__RTAVFSZONE2EN0_MASK                                                                      0xFFFFFFFFL
+//RTAVFS_REG10
+#define RTAVFS_REG10__RTAVFSZONE2EN1__SHIFT                                                                   0x0
+#define RTAVFS_REG10__RTAVFSZONE2EN1_MASK                                                                     0xFFFFFFFFL
+//RTAVFS_REG11
+#define RTAVFS_REG11__RTAVFSZONE3EN0__SHIFT                                                                   0x0
+#define RTAVFS_REG11__RTAVFSZONE3EN0_MASK                                                                     0xFFFFFFFFL
+//RTAVFS_REG12
+#define RTAVFS_REG12__RTAVFSZONE3EN1__SHIFT                                                                   0x0
+#define RTAVFS_REG12__RTAVFSZONE3EN1_MASK                                                                     0xFFFFFFFFL
+//RTAVFS_REG13
+#define RTAVFS_REG13__RTAVFSZONE4EN0__SHIFT                                                                   0x0
+#define RTAVFS_REG13__RTAVFSZONE4EN0_MASK                                                                     0xFFFFFFFFL
+//RTAVFS_REG14
+#define RTAVFS_REG14__RTAVFSZONE4EN1__SHIFT                                                                   0x0
+#define RTAVFS_REG14__RTAVFSZONE4EN1_MASK                                                                     0xFFFFFFFFL
+//RTAVFS_REG15
+#define RTAVFS_REG15__RTAVFSVF0FREQCOUNT__SHIFT                                                               0x0
+#define RTAVFS_REG15__RTAVFSVF0VOLTCODE__SHIFT                                                                0x10
+#define RTAVFS_REG15__RTAVFSVF0FREQCOUNT_MASK                                                                 0x0000FFFFL
+#define RTAVFS_REG15__RTAVFSVF0VOLTCODE_MASK                                                                  0xFFFF0000L
+//RTAVFS_REG16
+#define RTAVFS_REG16__RTAVFSVF1FREQCOUNT__SHIFT                                                               0x0
+#define RTAVFS_REG16__RTAVFSVF1VOLTCODE__SHIFT                                                                0x10
+#define RTAVFS_REG16__RTAVFSVF1FREQCOUNT_MASK                                                                 0x0000FFFFL
+#define RTAVFS_REG16__RTAVFSVF1VOLTCODE_MASK                                                                  0xFFFF0000L
+//RTAVFS_REG17
+#define RTAVFS_REG17__RTAVFSVF2FREQCOUNT__SHIFT                                                               0x0
+#define RTAVFS_REG17__RTAVFSVF2VOLTCODE__SHIFT                                                                0x10
+#define RTAVFS_REG17__RTAVFSVF2FREQCOUNT_MASK                                                                 0x0000FFFFL
+#define RTAVFS_REG17__RTAVFSVF2VOLTCODE_MASK                                                                  0xFFFF0000L
+//RTAVFS_REG18
+#define RTAVFS_REG18__RTAVFSVF3FREQCOUNT__SHIFT                                                               0x0
+#define RTAVFS_REG18__RTAVFSVF3VOLTCODE__SHIFT                                                                0x10
+#define RTAVFS_REG18__RTAVFSVF3FREQCOUNT_MASK                                                                 0x0000FFFFL
+#define RTAVFS_REG18__RTAVFSVF3VOLTCODE_MASK                                                                  0xFFFF0000L
+//RTAVFS_REG19
+#define RTAVFS_REG19__RTAVFSGB_ZONE0__SHIFT                                                                   0x0
+#define RTAVFS_REG19__RTAVFSGB_ZONE1__SHIFT                                                                   0x6
+#define RTAVFS_REG19__RTAVFSGB_ZONE2__SHIFT                                                                   0xc
+#define RTAVFS_REG19__RTAVFSGB_ZONE3__SHIFT                                                                   0x12
+#define RTAVFS_REG19__RTAVFSGB_ZONE4__SHIFT                                                                   0x19
+#define RTAVFS_REG19__RTAVFSGB_ZONE0_MASK                                                                     0x0000003FL
+#define RTAVFS_REG19__RTAVFSGB_ZONE1_MASK                                                                     0x00000FC0L
+#define RTAVFS_REG19__RTAVFSGB_ZONE2_MASK                                                                     0x0003F000L
+#define RTAVFS_REG19__RTAVFSGB_ZONE3_MASK                                                                     0x01FC0000L
+#define RTAVFS_REG19__RTAVFSGB_ZONE4_MASK                                                                     0xFE000000L
+//RTAVFS_REG20
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV0__SHIFT                                                            0x0
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV1__SHIFT                                                            0x2
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV2__SHIFT                                                            0x4
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV3__SHIFT                                                            0x6
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV4__SHIFT                                                            0x8
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV5__SHIFT                                                            0xa
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV6__SHIFT                                                            0xc
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV7__SHIFT                                                            0xe
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIVFINAL__SHIFT                                                        0x10
+#define RTAVFS_REG20__RTAVFSZONE0RESERVED__SHIFT                                                              0x12
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV0_MASK                                                              0x00000003L
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV1_MASK                                                              0x0000000CL
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV2_MASK                                                              0x00000030L
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV3_MASK                                                              0x000000C0L
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV4_MASK                                                              0x00000300L
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV5_MASK                                                              0x00000C00L
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV6_MASK                                                              0x00003000L
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIV7_MASK                                                              0x0000C000L
+#define RTAVFS_REG20__RTAVFSZONE0CPOAVGDIVFINAL_MASK                                                          0x00030000L
+#define RTAVFS_REG20__RTAVFSZONE0RESERVED_MASK                                                                0xFFFC0000L
+//RTAVFS_REG21
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV0__SHIFT                                                            0x0
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV1__SHIFT                                                            0x2
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV2__SHIFT                                                            0x4
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV3__SHIFT                                                            0x6
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV4__SHIFT                                                            0x8
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV5__SHIFT                                                            0xa
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV6__SHIFT                                                            0xc
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV7__SHIFT                                                            0xe
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIVFINAL__SHIFT                                                        0x10
+#define RTAVFS_REG21__RTAVFSZONE1RESERVED__SHIFT                                                              0x12
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV0_MASK                                                              0x00000003L
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV1_MASK                                                              0x0000000CL
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV2_MASK                                                              0x00000030L
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV3_MASK                                                              0x000000C0L
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV4_MASK                                                              0x00000300L
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV5_MASK                                                              0x00000C00L
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV6_MASK                                                              0x00003000L
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIV7_MASK                                                              0x0000C000L
+#define RTAVFS_REG21__RTAVFSZONE1CPOAVGDIVFINAL_MASK                                                          0x00030000L
+#define RTAVFS_REG21__RTAVFSZONE1RESERVED_MASK                                                                0xFFFC0000L
+//RTAVFS_REG22
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV0__SHIFT                                                            0x0
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV1__SHIFT                                                            0x2
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV2__SHIFT                                                            0x4
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV3__SHIFT                                                            0x6
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV4__SHIFT                                                            0x8
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV5__SHIFT                                                            0xa
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV6__SHIFT                                                            0xc
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV7__SHIFT                                                            0xe
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIVFINAL__SHIFT                                                        0x10
+#define RTAVFS_REG22__RTAVFSZONE2RESERVED__SHIFT                                                              0x12
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV0_MASK                                                              0x00000003L
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV1_MASK                                                              0x0000000CL
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV2_MASK                                                              0x00000030L
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV3_MASK                                                              0x000000C0L
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV4_MASK                                                              0x00000300L
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV5_MASK                                                              0x00000C00L
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV6_MASK                                                              0x00003000L
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIV7_MASK                                                              0x0000C000L
+#define RTAVFS_REG22__RTAVFSZONE2CPOAVGDIVFINAL_MASK                                                          0x00030000L
+#define RTAVFS_REG22__RTAVFSZONE2RESERVED_MASK                                                                0xFFFC0000L
+//RTAVFS_REG23
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV0__SHIFT                                                            0x0
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV1__SHIFT                                                            0x2
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV2__SHIFT                                                            0x4
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV3__SHIFT                                                            0x6
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV4__SHIFT                                                            0x8
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV5__SHIFT                                                            0xa
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV6__SHIFT                                                            0xc
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV7__SHIFT                                                            0xe
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIVFINAL__SHIFT                                                        0x10
+#define RTAVFS_REG23__RTAVFSZONE3RESERVED__SHIFT                                                              0x12
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV0_MASK                                                              0x00000003L
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV1_MASK                                                              0x0000000CL
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV2_MASK                                                              0x00000030L
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV3_MASK                                                              0x000000C0L
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV4_MASK                                                              0x00000300L
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV5_MASK                                                              0x00000C00L
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV6_MASK                                                              0x00003000L
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIV7_MASK                                                              0x0000C000L
+#define RTAVFS_REG23__RTAVFSZONE3CPOAVGDIVFINAL_MASK                                                          0x00030000L
+#define RTAVFS_REG23__RTAVFSZONE3RESERVED_MASK                                                                0xFFFC0000L
+//RTAVFS_REG24
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV0__SHIFT                                                            0x0
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV1__SHIFT                                                            0x2
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV2__SHIFT                                                            0x4
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV3__SHIFT                                                            0x6
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV4__SHIFT                                                            0x8
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV5__SHIFT                                                            0xa
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV6__SHIFT                                                            0xc
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV7__SHIFT                                                            0xe
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIVFINAL__SHIFT                                                        0x10
+#define RTAVFS_REG24__RTAVFSZONE4RESERVED__SHIFT                                                              0x12
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV0_MASK                                                              0x00000003L
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV1_MASK                                                              0x0000000CL
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV2_MASK                                                              0x00000030L
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV3_MASK                                                              0x000000C0L
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV4_MASK                                                              0x00000300L
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV5_MASK                                                              0x00000C00L
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV6_MASK                                                              0x00003000L
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIV7_MASK                                                              0x0000C000L
+#define RTAVFS_REG24__RTAVFSZONE4CPOAVGDIVFINAL_MASK                                                          0x00030000L
+#define RTAVFS_REG24__RTAVFSZONE4RESERVED_MASK                                                                0xFFFC0000L
+//RTAVFS_REG25
+#define RTAVFS_REG25__RTAVFSRESERVED0__SHIFT                                                                  0x0
+#define RTAVFS_REG25__RTAVFSRESERVED0_MASK                                                                    0xFFFFFFFFL
+//RTAVFS_REG26
+#define RTAVFS_REG26__RTAVFSRESERVED1__SHIFT                                                                  0x0
+#define RTAVFS_REG26__RTAVFSRESERVED1_MASK                                                                    0xFFFFFFFFL
+//RTAVFS_REG27
+#define RTAVFS_REG27__RTAVFSRESERVED2__SHIFT                                                                  0x0
+#define RTAVFS_REG27__RTAVFSRESERVED2_MASK                                                                    0xFFFFFFFFL
+//RTAVFS_REG28
+#define RTAVFS_REG28__RTAVFSZONE0INTERCEPT__SHIFT                                                             0x0
+#define RTAVFS_REG28__RTAVFSZONE1INTERCEPT__SHIFT                                                             0x10
+#define RTAVFS_REG28__RTAVFSZONE0INTERCEPT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG28__RTAVFSZONE1INTERCEPT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG29
+#define RTAVFS_REG29__RTAVFSZONE2INTERCEPT__SHIFT                                                             0x0
+#define RTAVFS_REG29__RTAVFSZONE3INTERCEPT__SHIFT                                                             0x10
+#define RTAVFS_REG29__RTAVFSZONE2INTERCEPT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG29__RTAVFSZONE3INTERCEPT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG30
+#define RTAVFS_REG30__RTAVFSZONE4INTERCEPT__SHIFT                                                             0x0
+#define RTAVFS_REG30__RTAVFSRESERVEDINTERCEPT__SHIFT                                                          0x10
+#define RTAVFS_REG30__RTAVFSZONE4INTERCEPT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG30__RTAVFSRESERVEDINTERCEPT_MASK                                                            0xFFFF0000L
+//RTAVFS_REG31
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV0__SHIFT                                                                 0x0
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV1__SHIFT                                                                 0x2
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV2__SHIFT                                                                 0x4
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV3__SHIFT                                                                 0x6
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV4__SHIFT                                                                 0x8
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV5__SHIFT                                                                 0xa
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV6__SHIFT                                                                 0xc
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV7__SHIFT                                                                 0xe
+#define RTAVFS_REG31__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV0_MASK                                                                   0x00000003L
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV1_MASK                                                                   0x0000000CL
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV2_MASK                                                                   0x00000030L
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV3_MASK                                                                   0x000000C0L
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV4_MASK                                                                   0x00000300L
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV5_MASK                                                                   0x00000C00L
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV6_MASK                                                                   0x00003000L
+#define RTAVFS_REG31__RTAVFSCPOCLKDIV7_MASK                                                                   0x0000C000L
+#define RTAVFS_REG31__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG32
+#define RTAVFS_REG32__RTAVFSFSMSTARTUPCNT__SHIFT                                                              0x0
+#define RTAVFS_REG32__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG32__RTAVFSFSMSTARTUPCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG32__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG33
+#define RTAVFS_REG33__RTAVFSFSMIDLECNT__SHIFT                                                                 0x0
+#define RTAVFS_REG33__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG33__RTAVFSFSMIDLECNT_MASK                                                                   0x0000FFFFL
+#define RTAVFS_REG33__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG34
+#define RTAVFS_REG34__RTAVFSFSMRESETCPORIPPLECOUNTERSCNT__SHIFT                                               0x0
+#define RTAVFS_REG34__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG34__RTAVFSFSMRESETCPORIPPLECOUNTERSCNT_MASK                                                 0x0000FFFFL
+#define RTAVFS_REG34__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG35
+#define RTAVFS_REG35__RTAVFSFSMSTARTCPOSCNT__SHIFT                                                            0x0
+#define RTAVFS_REG35__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG35__RTAVFSFSMSTARTCPOSCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG35__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG36
+#define RTAVFS_REG36__RTAVFSFSMSTARTRIPPLECOUNTERSCNT__SHIFT                                                  0x0
+#define RTAVFS_REG36__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG36__RTAVFSFSMSTARTRIPPLECOUNTERSCNT_MASK                                                    0x0000FFFFL
+#define RTAVFS_REG36__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG37
+#define RTAVFS_REG37__RTAVFSFSMRIPPLECOUNTERSDONECNT__SHIFT                                                   0x0
+#define RTAVFS_REG37__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG37__RTAVFSFSMRIPPLECOUNTERSDONECNT_MASK                                                     0x0000FFFFL
+#define RTAVFS_REG37__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG38
+#define RTAVFS_REG38__RTAVFSFSMCPOFINALRESULTREADYCNT__SHIFT                                                  0x0
+#define RTAVFS_REG38__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG38__RTAVFSFSMCPOFINALRESULTREADYCNT_MASK                                                    0x0000FFFFL
+#define RTAVFS_REG38__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG39
+#define RTAVFS_REG39__RTAVFSFSMVOLTCODEREADYCNT__SHIFT                                                        0x0
+#define RTAVFS_REG39__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG39__RTAVFSFSMVOLTCODEREADYCNT_MASK                                                          0x0000FFFFL
+#define RTAVFS_REG39__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG40
+#define RTAVFS_REG40__RTAVFSFSMTARGETVOLTAGEREADYCNT__SHIFT                                                   0x0
+#define RTAVFS_REG40__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG40__RTAVFSFSMTARGETVOLTAGEREADYCNT_MASK                                                     0x0000FFFFL
+#define RTAVFS_REG40__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG41
+#define RTAVFS_REG41__RTAVFSFSMSTOPCPOSCNT__SHIFT                                                             0x0
+#define RTAVFS_REG41__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG41__RTAVFSFSMSTOPCPOSCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG41__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG42
+#define RTAVFS_REG42__RTAVFSFSMWAITFORACKCNT__SHIFT                                                           0x0
+#define RTAVFS_REG42__RESERVED__SHIFT                                                                         0x10
+#define RTAVFS_REG42__RTAVFSFSMWAITFORACKCNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG42__RESERVED_MASK                                                                           0xFFFF0000L
+//RTAVFS_REG43
+#define RTAVFS_REG43__RTAVFSKP0__SHIFT                                                                        0x0
+#define RTAVFS_REG43__RTAVFSKP1__SHIFT                                                                        0x4
+#define RTAVFS_REG43__RTAVFSKP2__SHIFT                                                                        0x8
+#define RTAVFS_REG43__RTAVFSKP3__SHIFT                                                                        0xc
+#define RTAVFS_REG43__RTAVFSKI0__SHIFT                                                                        0x10
+#define RTAVFS_REG43__RTAVFSKI1__SHIFT                                                                        0x14
+#define RTAVFS_REG43__RTAVFSKI2__SHIFT                                                                        0x18
+#define RTAVFS_REG43__RTAVFSKI3__SHIFT                                                                        0x1c
+#define RTAVFS_REG43__RTAVFSKP0_MASK                                                                          0x0000000FL
+#define RTAVFS_REG43__RTAVFSKP1_MASK                                                                          0x000000F0L
+#define RTAVFS_REG43__RTAVFSKP2_MASK                                                                          0x00000F00L
+#define RTAVFS_REG43__RTAVFSKP3_MASK                                                                          0x0000F000L
+#define RTAVFS_REG43__RTAVFSKI0_MASK                                                                          0x000F0000L
+#define RTAVFS_REG43__RTAVFSKI1_MASK                                                                          0x00F00000L
+#define RTAVFS_REG43__RTAVFSKI2_MASK                                                                          0x0F000000L
+#define RTAVFS_REG43__RTAVFSKI3_MASK                                                                          0xF0000000L
+//RTAVFS_REG44
+#define RTAVFS_REG44__RTAVFSV1__SHIFT                                                                         0x0
+#define RTAVFS_REG44__RTAVFSV2__SHIFT                                                                         0xa
+#define RTAVFS_REG44__RTAVFSV3__SHIFT                                                                         0x14
+#define RTAVFS_REG44__RTAVFSUSEBINARYSEARCH__SHIFT                                                            0x1e
+#define RTAVFS_REG44__RTAVFSVOLTCODEHWCAL__SHIFT                                                              0x1f
+#define RTAVFS_REG44__RTAVFSV1_MASK                                                                           0x000003FFL
+#define RTAVFS_REG44__RTAVFSV2_MASK                                                                           0x000FFC00L
+#define RTAVFS_REG44__RTAVFSV3_MASK                                                                           0x3FF00000L
+#define RTAVFS_REG44__RTAVFSUSEBINARYSEARCH_MASK                                                              0x40000000L
+#define RTAVFS_REG44__RTAVFSVOLTCODEHWCAL_MASK                                                                0x80000000L
+//RTAVFS_REG45
+#define RTAVFS_REG45__RTAVFSVRBLEEDCNTRL__SHIFT                                                               0x0
+#define RTAVFS_REG45__RTAVFSVRENABLE__SHIFT                                                                   0x1
+#define RTAVFS_REG45__RTAVFSVOLTCODEOVERRIDE__SHIFT                                                           0x2
+#define RTAVFS_REG45__RTAVFSVOLTCODEOVERRIDESEL__SHIFT                                                        0xc
+#define RTAVFS_REG45__RTAVFSLOWPWREN__SHIFT                                                                   0xd
+#define RTAVFS_REG45__RTAVFSUREGENABLE__SHIFT                                                                 0xe
+#define RTAVFS_REG45__RTAVFSBGENABLE__SHIFT                                                                   0xf
+#define RTAVFS_REG45__RTAVFSENABLEVDDRETSENSING__SHIFT                                                        0x10
+#define RTAVFS_REG45__RESERVED__SHIFT                                                                         0x11
+#define RTAVFS_REG45__RTAVFSVRBLEEDCNTRL_MASK                                                                 0x00000001L
+#define RTAVFS_REG45__RTAVFSVRENABLE_MASK                                                                     0x00000002L
+#define RTAVFS_REG45__RTAVFSVOLTCODEOVERRIDE_MASK                                                             0x00000FFCL
+#define RTAVFS_REG45__RTAVFSVOLTCODEOVERRIDESEL_MASK                                                          0x00001000L
+#define RTAVFS_REG45__RTAVFSLOWPWREN_MASK                                                                     0x00002000L
+#define RTAVFS_REG45__RTAVFSUREGENABLE_MASK                                                                   0x00004000L
+#define RTAVFS_REG45__RTAVFSBGENABLE_MASK                                                                     0x00008000L
+#define RTAVFS_REG45__RTAVFSENABLEVDDRETSENSING_MASK                                                          0x00010000L
+#define RTAVFS_REG45__RESERVED_MASK                                                                           0xFFFE0000L
+//RTAVFS_REG46
+#define RTAVFS_REG46__RTAVFSKP__SHIFT                                                                         0x0
+#define RTAVFS_REG46__RTAVFSKI__SHIFT                                                                         0x4
+#define RTAVFS_REG46__RTAVFSPIENABLEANTIWINDUP__SHIFT                                                         0x8
+#define RTAVFS_REG46__RTAVFSPISHIFT__SHIFT                                                                    0x9
+#define RTAVFS_REG46__RTAVFSPIERREN__SHIFT                                                                    0xd
+#define RTAVFS_REG46__RTAVFSPISHIFTOUT__SHIFT                                                                 0xe
+#define RTAVFS_REG46__RTAVFSUSELUTKPKI__SHIFT                                                                 0x12
+#define RTAVFS_REG46__RESERVED__SHIFT                                                                         0x13
+#define RTAVFS_REG46__RTAVFSKP_MASK                                                                           0x0000000FL
+#define RTAVFS_REG46__RTAVFSKI_MASK                                                                           0x000000F0L
+#define RTAVFS_REG46__RTAVFSPIENABLEANTIWINDUP_MASK                                                           0x00000100L
+#define RTAVFS_REG46__RTAVFSPISHIFT_MASK                                                                      0x00001E00L
+#define RTAVFS_REG46__RTAVFSPIERREN_MASK                                                                      0x00002000L
+#define RTAVFS_REG46__RTAVFSPISHIFTOUT_MASK                                                                   0x0003C000L
+#define RTAVFS_REG46__RTAVFSUSELUTKPKI_MASK                                                                   0x00040000L
+#define RTAVFS_REG46__RESERVED_MASK                                                                           0xFFF80000L
+//RTAVFS_REG47
+#define RTAVFS_REG47__RTAVFSVOLTCODEPIMIN__SHIFT                                                              0x0
+#define RTAVFS_REG47__RTAVFSVOLTCODEPIMAX__SHIFT                                                              0xa
+#define RTAVFS_REG47__RTAVFSPIERRMASK__SHIFT                                                                  0x14
+#define RTAVFS_REG47__RTAVFSFORCEDISABLEPI__SHIFT                                                             0x1b
+#define RTAVFS_REG47__RESERVED__SHIFT                                                                         0x1c
+#define RTAVFS_REG47__RTAVFSVOLTCODEPIMIN_MASK                                                                0x000003FFL
+#define RTAVFS_REG47__RTAVFSVOLTCODEPIMAX_MASK                                                                0x000FFC00L
+#define RTAVFS_REG47__RTAVFSPIERRMASK_MASK                                                                    0x07F00000L
+#define RTAVFS_REG47__RTAVFSFORCEDISABLEPI_MASK                                                               0x08000000L
+#define RTAVFS_REG47__RESERVED_MASK                                                                           0xF0000000L
+//RTAVFS_REG48
+#define RTAVFS_REG48__RTAVFSPILOOPNITERATIONS__SHIFT                                                          0x0
+#define RTAVFS_REG48__RTAVFSPIERRTHRESHOLD__SHIFT                                                             0x10
+#define RTAVFS_REG48__RTAVFSPILOOPNITERATIONS_MASK                                                            0x0000FFFFL
+#define RTAVFS_REG48__RTAVFSPIERRTHRESHOLD_MASK                                                               0xFFFF0000L
+//RTAVFS_REG49
+#define RTAVFS_REG49__RTAVFSPSMRSTAVGVDD__SHIFT                                                               0x0
+#define RTAVFS_REG49__RTAVFSPSMMEASMAXVDD__SHIFT                                                              0x1
+#define RTAVFS_REG49__RTAVFSPSMCLKDIVVDD__SHIFT                                                               0x2
+#define RTAVFS_REG49__RTAVFSPSMAVGDIVVDD__SHIFT                                                               0x4
+#define RTAVFS_REG49__RTAVFSPSMOSCENVDD__SHIFT                                                                0xa
+#define RTAVFS_REG49__RTAVFSPSMAVGENVDD__SHIFT                                                                0xb
+#define RTAVFS_REG49__RTAVFSPSMRSTMINMAXVDD__SHIFT                                                            0xc
+#define RTAVFS_REG49__RESERVED__SHIFT                                                                         0xd
+#define RTAVFS_REG49__RTAVFSPSMRSTAVGVDD_MASK                                                                 0x00000001L
+#define RTAVFS_REG49__RTAVFSPSMMEASMAXVDD_MASK                                                                0x00000002L
+#define RTAVFS_REG49__RTAVFSPSMCLKDIVVDD_MASK                                                                 0x0000000CL
+#define RTAVFS_REG49__RTAVFSPSMAVGDIVVDD_MASK                                                                 0x000003F0L
+#define RTAVFS_REG49__RTAVFSPSMOSCENVDD_MASK                                                                  0x00000400L
+#define RTAVFS_REG49__RTAVFSPSMAVGENVDD_MASK                                                                  0x00000800L
+#define RTAVFS_REG49__RTAVFSPSMRSTMINMAXVDD_MASK                                                              0x00001000L
+#define RTAVFS_REG49__RESERVED_MASK                                                                           0xFFFFE000L
+//RTAVFS_REG50
+#define RTAVFS_REG50__RTAVFSPSMRSTAVGVREG__SHIFT                                                              0x0
+#define RTAVFS_REG50__RTAVFSPSMMEASMAXVREG__SHIFT                                                             0x1
+#define RTAVFS_REG50__RTAVFSPSMCLKDIVVREG__SHIFT                                                              0x2
+#define RTAVFS_REG50__RTAVFSPSMAVGDIVVREG__SHIFT                                                              0x4
+#define RTAVFS_REG50__RTAVFSPSMOSCENVREG__SHIFT                                                               0xa
+#define RTAVFS_REG50__RTAVFSPSMAVGENVREG__SHIFT                                                               0xb
+#define RTAVFS_REG50__RTAVFSPSMRSTMINMAXVREG__SHIFT                                                           0xc
+#define RTAVFS_REG50__RESERVED__SHIFT                                                                         0xd
+#define RTAVFS_REG50__RTAVFSPSMRSTAVGVREG_MASK                                                                0x00000001L
+#define RTAVFS_REG50__RTAVFSPSMMEASMAXVREG_MASK                                                               0x00000002L
+#define RTAVFS_REG50__RTAVFSPSMCLKDIVVREG_MASK                                                                0x0000000CL
+#define RTAVFS_REG50__RTAVFSPSMAVGDIVVREG_MASK                                                                0x000003F0L
+#define RTAVFS_REG50__RTAVFSPSMOSCENVREG_MASK                                                                 0x00000400L
+#define RTAVFS_REG50__RTAVFSPSMAVGENVREG_MASK                                                                 0x00000800L
+#define RTAVFS_REG50__RTAVFSPSMRSTMINMAXVREG_MASK                                                             0x00001000L
+#define RTAVFS_REG50__RESERVED_MASK                                                                           0xFFFFE000L
+//RTAVFS_REG51
+#define RTAVFS_REG51__RTAVFSAVFSENABLE__SHIFT                                                                 0x0
+#define RTAVFS_REG51__RTAVFSCPOTURNONDELAY__SHIFT                                                             0x1
+#define RTAVFS_REG51__RTAVFSSELECTMINMAX__SHIFT                                                               0x5
+#define RTAVFS_REG51__RTAVFSSELECTPERPATHSCALING__SHIFT                                                       0x6
+#define RTAVFS_REG51__RTAVFSADDVOLTCODEGUARDBAND__SHIFT                                                       0x7
+#define RTAVFS_REG51__RTAVFSSENDAVGPSMTOPSMOUT__SHIFT                                                         0x8
+#define RTAVFS_REG51__RTAVFSUPDATEANCHORVOLTAGES__SHIFT                                                       0x9
+#define RTAVFS_REG51__RTAVFSSENDVDDTOPSMOUT__SHIFT                                                            0xa
+#define RTAVFS_REG51__RESERVED__SHIFT                                                                         0xb
+#define RTAVFS_REG51__RTAVFSAVFSENABLE_MASK                                                                   0x00000001L
+#define RTAVFS_REG51__RTAVFSCPOTURNONDELAY_MASK                                                               0x0000001EL
+#define RTAVFS_REG51__RTAVFSSELECTMINMAX_MASK                                                                 0x00000020L
+#define RTAVFS_REG51__RTAVFSSELECTPERPATHSCALING_MASK                                                         0x00000040L
+#define RTAVFS_REG51__RTAVFSADDVOLTCODEGUARDBAND_MASK                                                         0x00000080L
+#define RTAVFS_REG51__RTAVFSSENDAVGPSMTOPSMOUT_MASK                                                           0x00000100L
+#define RTAVFS_REG51__RTAVFSUPDATEANCHORVOLTAGES_MASK                                                         0x00000200L
+#define RTAVFS_REG51__RTAVFSSENDVDDTOPSMOUT_MASK                                                              0x00000400L
+#define RTAVFS_REG51__RESERVED_MASK                                                                           0xFFFFF800L
+//RTAVFS_REG52
+#define RTAVFS_REG52__RTAVFSMINMAXPSMVDD__SHIFT                                                               0x0
+#define RTAVFS_REG52__RTAVFSAVGPSMVDD__SHIFT                                                                  0xe
+#define RTAVFS_REG52__RESERVED__SHIFT                                                                         0x1c
+#define RTAVFS_REG52__RTAVFSMINMAXPSMVDD_MASK                                                                 0x00003FFFL
+#define RTAVFS_REG52__RTAVFSAVGPSMVDD_MASK                                                                    0x0FFFC000L
+#define RTAVFS_REG52__RESERVED_MASK                                                                           0xF0000000L
+//RTAVFS_REG53
+#define RTAVFS_REG53__RTAVFSMINMAXPSMVREG__SHIFT                                                              0x0
+#define RTAVFS_REG53__RTAVFSAVGPSMVREG__SHIFT                                                                 0xe
+#define RTAVFS_REG53__RESERVED__SHIFT                                                                         0x1c
+#define RTAVFS_REG53__RTAVFSMINMAXPSMVREG_MASK                                                                0x00003FFFL
+#define RTAVFS_REG53__RTAVFSAVGPSMVREG_MASK                                                                   0x0FFFC000L
+#define RTAVFS_REG53__RESERVED_MASK                                                                           0xF0000000L
+//RTAVFS_REG54
+#define RTAVFS_REG54__RTAVFSCPO0_STARTCNT__SHIFT                                                              0x0
+#define RTAVFS_REG54__RTAVFSCPO0_STOPCNT__SHIFT                                                               0x10
+#define RTAVFS_REG54__RTAVFSCPO0_STARTCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG54__RTAVFSCPO0_STOPCNT_MASK                                                                 0xFFFF0000L
+//RTAVFS_REG55
+#define RTAVFS_REG55__RTAVFSCPO1_STARTCNT__SHIFT                                                              0x0
+#define RTAVFS_REG55__RTAVFSCPO1_STOPCNT__SHIFT                                                               0x10
+#define RTAVFS_REG55__RTAVFSCPO1_STARTCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG55__RTAVFSCPO1_STOPCNT_MASK                                                                 0xFFFF0000L
+//RTAVFS_REG56
+#define RTAVFS_REG56__RTAVFSCPO2_STARTCNT__SHIFT                                                              0x0
+#define RTAVFS_REG56__RTAVFSCPO2_STOPCNT__SHIFT                                                               0x10
+#define RTAVFS_REG56__RTAVFSCPO2_STARTCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG56__RTAVFSCPO2_STOPCNT_MASK                                                                 0xFFFF0000L
+//RTAVFS_REG57
+#define RTAVFS_REG57__RTAVFSCPO3_STARTCNT__SHIFT                                                              0x0
+#define RTAVFS_REG57__RTAVFSCPO3_STOPCNT__SHIFT                                                               0x10
+#define RTAVFS_REG57__RTAVFSCPO3_STARTCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG57__RTAVFSCPO3_STOPCNT_MASK                                                                 0xFFFF0000L
+//RTAVFS_REG58
+#define RTAVFS_REG58__RTAVFSCPO4_STARTCNT__SHIFT                                                              0x0
+#define RTAVFS_REG58__RTAVFSCPO4_STOPCNT__SHIFT                                                               0x10
+#define RTAVFS_REG58__RTAVFSCPO4_STARTCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG58__RTAVFSCPO4_STOPCNT_MASK                                                                 0xFFFF0000L
+//RTAVFS_REG59
+#define RTAVFS_REG59__RTAVFSCPO5_STARTCNT__SHIFT                                                              0x0
+#define RTAVFS_REG59__RTAVFSCPO5_STOPCNT__SHIFT                                                               0x10
+#define RTAVFS_REG59__RTAVFSCPO5_STARTCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG59__RTAVFSCPO5_STOPCNT_MASK                                                                 0xFFFF0000L
+//RTAVFS_REG60
+#define RTAVFS_REG60__RTAVFSCPO6_STARTCNT__SHIFT                                                              0x0
+#define RTAVFS_REG60__RTAVFSCPO6_STOPCNT__SHIFT                                                               0x10
+#define RTAVFS_REG60__RTAVFSCPO6_STARTCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG60__RTAVFSCPO6_STOPCNT_MASK                                                                 0xFFFF0000L
+//RTAVFS_REG61
+#define RTAVFS_REG61__RTAVFSCPO7_STARTCNT__SHIFT                                                              0x0
+#define RTAVFS_REG61__RTAVFSCPO7_STOPCNT__SHIFT                                                               0x10
+#define RTAVFS_REG61__RTAVFSCPO7_STARTCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG61__RTAVFSCPO7_STOPCNT_MASK                                                                 0xFFFF0000L
+//RTAVFS_REG62
+#define RTAVFS_REG62__RTAVFSCPO8_STARTCNT__SHIFT                                                              0x0
+#define RTAVFS_REG62__RTAVFSCPO8_STOPCNT__SHIFT                                                               0x10
+#define RTAVFS_REG62__RTAVFSCPO8_STARTCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG62__RTAVFSCPO8_STOPCNT_MASK                                                                 0xFFFF0000L
+//RTAVFS_REG63
+#define RTAVFS_REG63__RTAVFSCPO9_STARTCNT__SHIFT                                                              0x0
+#define RTAVFS_REG63__RTAVFSCPO9_STOPCNT__SHIFT                                                               0x10
+#define RTAVFS_REG63__RTAVFSCPO9_STARTCNT_MASK                                                                0x0000FFFFL
+#define RTAVFS_REG63__RTAVFSCPO9_STOPCNT_MASK                                                                 0xFFFF0000L
+//RTAVFS_REG64
+#define RTAVFS_REG64__RTAVFSCPO10_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG64__RTAVFSCPO10_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG64__RTAVFSCPO10_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG64__RTAVFSCPO10_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG65
+#define RTAVFS_REG65__RTAVFSCPO11_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG65__RTAVFSCPO11_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG65__RTAVFSCPO11_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG65__RTAVFSCPO11_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG66
+#define RTAVFS_REG66__RTAVFSCPO12_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG66__RTAVFSCPO12_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG66__RTAVFSCPO12_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG66__RTAVFSCPO12_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG67
+#define RTAVFS_REG67__RTAVFSCPO13_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG67__RTAVFSCPO13_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG67__RTAVFSCPO13_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG67__RTAVFSCPO13_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG68
+#define RTAVFS_REG68__RTAVFSCPO14_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG68__RTAVFSCPO14_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG68__RTAVFSCPO14_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG68__RTAVFSCPO14_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG69
+#define RTAVFS_REG69__RTAVFSCPO15_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG69__RTAVFSCPO15_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG69__RTAVFSCPO15_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG69__RTAVFSCPO15_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG70
+#define RTAVFS_REG70__RTAVFSCPO16_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG70__RTAVFSCPO16_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG70__RTAVFSCPO16_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG70__RTAVFSCPO16_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG71
+#define RTAVFS_REG71__RTAVFSCPO17_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG71__RTAVFSCPO17_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG71__RTAVFSCPO17_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG71__RTAVFSCPO17_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG72
+#define RTAVFS_REG72__RTAVFSCPO18_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG72__RTAVFSCPO18_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG72__RTAVFSCPO18_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG72__RTAVFSCPO18_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG73
+#define RTAVFS_REG73__RTAVFSCPO19_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG73__RTAVFSCPO19_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG73__RTAVFSCPO19_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG73__RTAVFSCPO19_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG74
+#define RTAVFS_REG74__RTAVFSCPO20_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG74__RTAVFSCPO20_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG74__RTAVFSCPO20_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG74__RTAVFSCPO20_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG75
+#define RTAVFS_REG75__RTAVFSCPO21_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG75__RTAVFSCPO21_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG75__RTAVFSCPO21_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG75__RTAVFSCPO21_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG76
+#define RTAVFS_REG76__RTAVFSCPO22_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG76__RTAVFSCPO22_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG76__RTAVFSCPO22_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG76__RTAVFSCPO22_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG77
+#define RTAVFS_REG77__RTAVFSCPO23_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG77__RTAVFSCPO23_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG77__RTAVFSCPO23_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG77__RTAVFSCPO23_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG78
+#define RTAVFS_REG78__RTAVFSCPO24_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG78__RTAVFSCPO24_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG78__RTAVFSCPO24_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG78__RTAVFSCPO24_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG79
+#define RTAVFS_REG79__RTAVFSCPO25_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG79__RTAVFSCPO25_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG79__RTAVFSCPO25_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG79__RTAVFSCPO25_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG80
+#define RTAVFS_REG80__RTAVFSCPO26_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG80__RTAVFSCPO26_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG80__RTAVFSCPO26_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG80__RTAVFSCPO26_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG81
+#define RTAVFS_REG81__RTAVFSCPO27_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG81__RTAVFSCPO27_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG81__RTAVFSCPO27_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG81__RTAVFSCPO27_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG82
+#define RTAVFS_REG82__RTAVFSCPO28_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG82__RTAVFSCPO28_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG82__RTAVFSCPO28_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG82__RTAVFSCPO28_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG83
+#define RTAVFS_REG83__RTAVFSCPO29_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG83__RTAVFSCPO29_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG83__RTAVFSCPO29_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG83__RTAVFSCPO29_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG84
+#define RTAVFS_REG84__RTAVFSCPO30_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG84__RTAVFSCPO30_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG84__RTAVFSCPO30_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG84__RTAVFSCPO30_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG85
+#define RTAVFS_REG85__RTAVFSCPO31_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG85__RTAVFSCPO31_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG85__RTAVFSCPO31_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG85__RTAVFSCPO31_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG86
+#define RTAVFS_REG86__RTAVFSCPO32_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG86__RTAVFSCPO32_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG86__RTAVFSCPO32_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG86__RTAVFSCPO32_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG87
+#define RTAVFS_REG87__RTAVFSCPO33_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG87__RTAVFSCPO33_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG87__RTAVFSCPO33_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG87__RTAVFSCPO33_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG88
+#define RTAVFS_REG88__RTAVFSCPO34_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG88__RTAVFSCPO34_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG88__RTAVFSCPO34_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG88__RTAVFSCPO34_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG89
+#define RTAVFS_REG89__RTAVFSCPO35_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG89__RTAVFSCPO35_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG89__RTAVFSCPO35_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG89__RTAVFSCPO35_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG90
+#define RTAVFS_REG90__RTAVFSCPO36_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG90__RTAVFSCPO36_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG90__RTAVFSCPO36_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG90__RTAVFSCPO36_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG91
+#define RTAVFS_REG91__RTAVFSCPO37_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG91__RTAVFSCPO37_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG91__RTAVFSCPO37_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG91__RTAVFSCPO37_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG92
+#define RTAVFS_REG92__RTAVFSCPO38_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG92__RTAVFSCPO38_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG92__RTAVFSCPO38_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG92__RTAVFSCPO38_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG93
+#define RTAVFS_REG93__RTAVFSCPO39_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG93__RTAVFSCPO39_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG93__RTAVFSCPO39_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG93__RTAVFSCPO39_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG94
+#define RTAVFS_REG94__RTAVFSCPO40_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG94__RTAVFSCPO40_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG94__RTAVFSCPO40_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG94__RTAVFSCPO40_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG95
+#define RTAVFS_REG95__RTAVFSCPO41_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG95__RTAVFSCPO41_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG95__RTAVFSCPO41_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG95__RTAVFSCPO41_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG96
+#define RTAVFS_REG96__RTAVFSCPO42_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG96__RTAVFSCPO42_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG96__RTAVFSCPO42_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG96__RTAVFSCPO42_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG97
+#define RTAVFS_REG97__RTAVFSCPO43_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG97__RTAVFSCPO43_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG97__RTAVFSCPO43_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG97__RTAVFSCPO43_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG98
+#define RTAVFS_REG98__RTAVFSCPO44_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG98__RTAVFSCPO44_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG98__RTAVFSCPO44_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG98__RTAVFSCPO44_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG99
+#define RTAVFS_REG99__RTAVFSCPO45_STARTCNT__SHIFT                                                             0x0
+#define RTAVFS_REG99__RTAVFSCPO45_STOPCNT__SHIFT                                                              0x10
+#define RTAVFS_REG99__RTAVFSCPO45_STARTCNT_MASK                                                               0x0000FFFFL
+#define RTAVFS_REG99__RTAVFSCPO45_STOPCNT_MASK                                                                0xFFFF0000L
+//RTAVFS_REG100
+#define RTAVFS_REG100__RTAVFSCPO46_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG100__RTAVFSCPO46_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG100__RTAVFSCPO46_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG100__RTAVFSCPO46_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG101
+#define RTAVFS_REG101__RTAVFSCPO47_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG101__RTAVFSCPO47_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG101__RTAVFSCPO47_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG101__RTAVFSCPO47_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG102
+#define RTAVFS_REG102__RTAVFSCPO48_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG102__RTAVFSCPO48_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG102__RTAVFSCPO48_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG102__RTAVFSCPO48_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG103
+#define RTAVFS_REG103__RTAVFSCPO49_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG103__RTAVFSCPO49_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG103__RTAVFSCPO49_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG103__RTAVFSCPO49_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG104
+#define RTAVFS_REG104__RTAVFSCPO50_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG104__RTAVFSCPO50_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG104__RTAVFSCPO50_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG104__RTAVFSCPO50_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG105
+#define RTAVFS_REG105__RTAVFSCPO51_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG105__RTAVFSCPO51_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG105__RTAVFSCPO51_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG105__RTAVFSCPO51_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG106
+#define RTAVFS_REG106__RTAVFSCPO52_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG106__RTAVFSCPO52_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG106__RTAVFSCPO52_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG106__RTAVFSCPO52_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG107
+#define RTAVFS_REG107__RTAVFSCPO53_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG107__RTAVFSCPO53_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG107__RTAVFSCPO53_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG107__RTAVFSCPO53_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG108
+#define RTAVFS_REG108__RTAVFSCPO54_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG108__RTAVFSCPO54_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG108__RTAVFSCPO54_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG108__RTAVFSCPO54_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG109
+#define RTAVFS_REG109__RTAVFSCPO55_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG109__RTAVFSCPO55_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG109__RTAVFSCPO55_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG109__RTAVFSCPO55_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG110
+#define RTAVFS_REG110__RTAVFSCPO56_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG110__RTAVFSCPO56_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG110__RTAVFSCPO56_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG110__RTAVFSCPO56_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG111
+#define RTAVFS_REG111__RTAVFSCPO57_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG111__RTAVFSCPO57_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG111__RTAVFSCPO57_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG111__RTAVFSCPO57_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG112
+#define RTAVFS_REG112__RTAVFSCPO58_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG112__RTAVFSCPO58_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG112__RTAVFSCPO58_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG112__RTAVFSCPO58_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG113
+#define RTAVFS_REG113__RTAVFSCPO59_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG113__RTAVFSCPO59_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG113__RTAVFSCPO59_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG113__RTAVFSCPO59_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG114
+#define RTAVFS_REG114__RTAVFSCPO60_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG114__RTAVFSCPO60_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG114__RTAVFSCPO60_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG114__RTAVFSCPO60_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG115
+#define RTAVFS_REG115__RTAVFSCPO61_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG115__RTAVFSCPO61_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG115__RTAVFSCPO61_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG115__RTAVFSCPO61_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG116
+#define RTAVFS_REG116__RTAVFSCPO62_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG116__RTAVFSCPO62_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG116__RTAVFSCPO62_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG116__RTAVFSCPO62_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG117
+#define RTAVFS_REG117__RTAVFSCPO63_STARTCNT__SHIFT                                                            0x0
+#define RTAVFS_REG117__RTAVFSCPO63_STOPCNT__SHIFT                                                             0x10
+#define RTAVFS_REG117__RTAVFSCPO63_STARTCNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG117__RTAVFSCPO63_STOPCNT_MASK                                                               0xFFFF0000L
+//RTAVFS_REG118
+#define RTAVFS_REG118__RTAVFSCPOEN0__SHIFT                                                                    0x0
+#define RTAVFS_REG118__RTAVFSCPOEN0_MASK                                                                      0xFFFFFFFFL
+//RTAVFS_REG119
+#define RTAVFS_REG119__RTAVFSCPOEN1__SHIFT                                                                    0x0
+#define RTAVFS_REG119__RTAVFSCPOEN1_MASK                                                                      0xFFFFFFFFL
+//RTAVFS_REG120
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV0__SHIFT                                                                0x0
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV1__SHIFT                                                                0x2
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV2__SHIFT                                                                0x4
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV3__SHIFT                                                                0x6
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV4__SHIFT                                                                0x8
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV5__SHIFT                                                                0xa
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV6__SHIFT                                                                0xc
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV7__SHIFT                                                                0xe
+#define RTAVFS_REG120__RTAVFSCPOAVGDIVFINAL__SHIFT                                                            0x10
+#define RTAVFS_REG120__RESERVED__SHIFT                                                                        0x12
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV0_MASK                                                                  0x00000003L
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV1_MASK                                                                  0x0000000CL
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV2_MASK                                                                  0x00000030L
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV3_MASK                                                                  0x000000C0L
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV4_MASK                                                                  0x00000300L
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV5_MASK                                                                  0x00000C00L
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV6_MASK                                                                  0x00003000L
+#define RTAVFS_REG120__RTAVFSCPOAVGDIV7_MASK                                                                  0x0000C000L
+#define RTAVFS_REG120__RTAVFSCPOAVGDIVFINAL_MASK                                                              0x00030000L
+#define RTAVFS_REG120__RESERVED_MASK                                                                          0xFFFC0000L
+//RTAVFS_REG121
+#define RTAVFS_REG121__RTAVFSZONE0INUSE__SHIFT                                                                0x0
+#define RTAVFS_REG121__RTAVFSZONE1INUSE__SHIFT                                                                0x1
+#define RTAVFS_REG121__RTAVFSZONE2INUSE__SHIFT                                                                0x2
+#define RTAVFS_REG121__RTAVFSZONE3INUSE__SHIFT                                                                0x3
+#define RTAVFS_REG121__RTAVFSZONE4INUSE__SHIFT                                                                0x4
+#define RTAVFS_REG121__RTAVFSRESERVED__SHIFT                                                                  0x5
+#define RTAVFS_REG121__RTAVFSERRORCODE__SHIFT                                                                 0x1c
+#define RTAVFS_REG121__RTAVFSZONE0INUSE_MASK                                                                  0x00000001L
+#define RTAVFS_REG121__RTAVFSZONE1INUSE_MASK                                                                  0x00000002L
+#define RTAVFS_REG121__RTAVFSZONE2INUSE_MASK                                                                  0x00000004L
+#define RTAVFS_REG121__RTAVFSZONE3INUSE_MASK                                                                  0x00000008L
+#define RTAVFS_REG121__RTAVFSZONE4INUSE_MASK                                                                  0x00000010L
+#define RTAVFS_REG121__RTAVFSRESERVED_MASK                                                                    0x0FFFFFE0L
+#define RTAVFS_REG121__RTAVFSERRORCODE_MASK                                                                   0xF0000000L
+//RTAVFS_REG122
+#define RTAVFS_REG122__RTAVFSCPO0_RIPPLECNT__SHIFT                                                            0x0
+#define RTAVFS_REG122__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG122__RTAVFSCPO0_RIPPLECNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG122__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG123
+#define RTAVFS_REG123__RTAVFSCPO1_RIPPLECNT__SHIFT                                                            0x0
+#define RTAVFS_REG123__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG123__RTAVFSCPO1_RIPPLECNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG123__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG124
+#define RTAVFS_REG124__RTAVFSCPO2_RIPPLECNT__SHIFT                                                            0x0
+#define RTAVFS_REG124__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG124__RTAVFSCPO2_RIPPLECNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG124__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG125
+#define RTAVFS_REG125__RTAVFSCPO3_RIPPLECNT__SHIFT                                                            0x0
+#define RTAVFS_REG125__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG125__RTAVFSCPO3_RIPPLECNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG125__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG126
+#define RTAVFS_REG126__RTAVFSCPO4_RIPPLECNT__SHIFT                                                            0x0
+#define RTAVFS_REG126__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG126__RTAVFSCPO4_RIPPLECNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG126__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG127
+#define RTAVFS_REG127__RTAVFSCPO5_RIPPLECNT__SHIFT                                                            0x0
+#define RTAVFS_REG127__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG127__RTAVFSCPO5_RIPPLECNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG127__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG128
+#define RTAVFS_REG128__RTAVFSCPO6_RIPPLECNT__SHIFT                                                            0x0
+#define RTAVFS_REG128__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG128__RTAVFSCPO6_RIPPLECNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG128__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG129
+#define RTAVFS_REG129__RTAVFSCPO7_RIPPLECNT__SHIFT                                                            0x0
+#define RTAVFS_REG129__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG129__RTAVFSCPO7_RIPPLECNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG129__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG130
+#define RTAVFS_REG130__RTAVFSCPO8_RIPPLECNT__SHIFT                                                            0x0
+#define RTAVFS_REG130__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG130__RTAVFSCPO8_RIPPLECNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG130__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG131
+#define RTAVFS_REG131__RTAVFSCPO9_RIPPLECNT__SHIFT                                                            0x0
+#define RTAVFS_REG131__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG131__RTAVFSCPO9_RIPPLECNT_MASK                                                              0x0000FFFFL
+#define RTAVFS_REG131__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG132
+#define RTAVFS_REG132__RTAVFSCPO10_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG132__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG132__RTAVFSCPO10_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG132__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG133
+#define RTAVFS_REG133__RTAVFSCPO11_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG133__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG133__RTAVFSCPO11_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG133__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG134
+#define RTAVFS_REG134__RTAVFSCPO12_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG134__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG134__RTAVFSCPO12_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG134__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG135
+#define RTAVFS_REG135__RTAVFSCPO13_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG135__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG135__RTAVFSCPO13_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG135__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG136
+#define RTAVFS_REG136__RTAVFSCPO14_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG136__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG136__RTAVFSCPO14_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG136__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG137
+#define RTAVFS_REG137__RTAVFSCPO15_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG137__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG137__RTAVFSCPO15_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG137__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG138
+#define RTAVFS_REG138__RTAVFSCPO16_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG138__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG138__RTAVFSCPO16_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG138__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG139
+#define RTAVFS_REG139__RTAVFSCPO17_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG139__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG139__RTAVFSCPO17_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG139__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG140
+#define RTAVFS_REG140__RTAVFSCPO18_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG140__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG140__RTAVFSCPO18_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG140__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG141
+#define RTAVFS_REG141__RTAVFSCPO19_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG141__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG141__RTAVFSCPO19_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG141__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG142
+#define RTAVFS_REG142__RTAVFSCPO20_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG142__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG142__RTAVFSCPO20_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG142__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG143
+#define RTAVFS_REG143__RTAVFSCPO21_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG143__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG143__RTAVFSCPO21_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG143__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG144
+#define RTAVFS_REG144__RTAVFSCPO22_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG144__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG144__RTAVFSCPO22_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG144__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG145
+#define RTAVFS_REG145__RTAVFSCPO23_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG145__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG145__RTAVFSCPO23_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG145__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG146
+#define RTAVFS_REG146__RTAVFSCPO24_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG146__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG146__RTAVFSCPO24_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG146__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG147
+#define RTAVFS_REG147__RTAVFSCPO25_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG147__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG147__RTAVFSCPO25_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG147__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG148
+#define RTAVFS_REG148__RTAVFSCPO26_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG148__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG148__RTAVFSCPO26_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG148__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG149
+#define RTAVFS_REG149__RTAVFSCPO27_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG149__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG149__RTAVFSCPO27_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG149__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG150
+#define RTAVFS_REG150__RTAVFSCPO28_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG150__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG150__RTAVFSCPO28_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG150__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG151
+#define RTAVFS_REG151__RTAVFSCPO29_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG151__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG151__RTAVFSCPO29_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG151__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG152
+#define RTAVFS_REG152__RTAVFSCPO30_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG152__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG152__RTAVFSCPO30_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG152__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG153
+#define RTAVFS_REG153__RTAVFSCPO31_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG153__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG153__RTAVFSCPO31_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG153__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG154
+#define RTAVFS_REG154__RTAVFSCPO32_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG154__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG154__RTAVFSCPO32_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG154__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG155
+#define RTAVFS_REG155__RTAVFSCPO33_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG155__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG155__RTAVFSCPO33_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG155__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG156
+#define RTAVFS_REG156__RTAVFSCPO34_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG156__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG156__RTAVFSCPO34_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG156__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG157
+#define RTAVFS_REG157__RTAVFSCPO35_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG157__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG157__RTAVFSCPO35_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG157__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG158
+#define RTAVFS_REG158__RTAVFSCPO36_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG158__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG158__RTAVFSCPO36_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG158__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG159
+#define RTAVFS_REG159__RTAVFSCPO37_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG159__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG159__RTAVFSCPO37_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG159__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG160
+#define RTAVFS_REG160__RTAVFSCPO38_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG160__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG160__RTAVFSCPO38_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG160__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG161
+#define RTAVFS_REG161__RTAVFSCPO39_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG161__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG161__RTAVFSCPO39_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG161__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG162
+#define RTAVFS_REG162__RTAVFSCPO40_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG162__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG162__RTAVFSCPO40_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG162__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG163
+#define RTAVFS_REG163__RTAVFSCPO41_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG163__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG163__RTAVFSCPO41_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG163__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG164
+#define RTAVFS_REG164__RTAVFSCPO42_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG164__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG164__RTAVFSCPO42_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG164__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG165
+#define RTAVFS_REG165__RTAVFSCPO43_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG165__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG165__RTAVFSCPO43_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG165__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG166
+#define RTAVFS_REG166__RTAVFSCPO44_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG166__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG166__RTAVFSCPO44_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG166__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG167
+#define RTAVFS_REG167__RTAVFSCPO45_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG167__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG167__RTAVFSCPO45_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG167__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG168
+#define RTAVFS_REG168__RTAVFSCPO46_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG168__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG168__RTAVFSCPO46_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG168__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG169
+#define RTAVFS_REG169__RTAVFSCPO47_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG169__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG169__RTAVFSCPO47_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG169__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG170
+#define RTAVFS_REG170__RTAVFSCPO48_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG170__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG170__RTAVFSCPO48_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG170__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG171
+#define RTAVFS_REG171__RTAVFSCPO49_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG171__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG171__RTAVFSCPO49_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG171__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG172
+#define RTAVFS_REG172__RTAVFSCPO50_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG172__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG172__RTAVFSCPO50_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG172__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG173
+#define RTAVFS_REG173__RTAVFSCPO51_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG173__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG173__RTAVFSCPO51_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG173__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG174
+#define RTAVFS_REG174__RTAVFSCPO52_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG174__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG174__RTAVFSCPO52_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG174__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG175
+#define RTAVFS_REG175__RTAVFSCPO53_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG175__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG175__RTAVFSCPO53_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG175__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG176
+#define RTAVFS_REG176__RTAVFSCPO54_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG176__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG176__RTAVFSCPO54_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG176__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG177
+#define RTAVFS_REG177__RTAVFSCPO55_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG177__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG177__RTAVFSCPO55_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG177__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG178
+#define RTAVFS_REG178__RTAVFSCPO56_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG178__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG178__RTAVFSCPO56_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG178__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG179
+#define RTAVFS_REG179__RTAVFSCPO57_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG179__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG179__RTAVFSCPO57_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG179__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG180
+#define RTAVFS_REG180__RTAVFSCPO58_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG180__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG180__RTAVFSCPO58_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG180__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG181
+#define RTAVFS_REG181__RTAVFSCPO59_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG181__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG181__RTAVFSCPO59_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG181__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG182
+#define RTAVFS_REG182__RTAVFSCPO60_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG182__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG182__RTAVFSCPO60_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG182__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG183
+#define RTAVFS_REG183__RTAVFSCPO61_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG183__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG183__RTAVFSCPO61_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG183__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG184
+#define RTAVFS_REG184__RTAVFSCPO62_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG184__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG184__RTAVFSCPO62_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG184__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG185
+#define RTAVFS_REG185__RTAVFSCPO63_RIPPLECNT__SHIFT                                                           0x0
+#define RTAVFS_REG185__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG185__RTAVFSCPO63_RIPPLECNT_MASK                                                             0x0000FFFFL
+#define RTAVFS_REG185__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG186
+#define RTAVFS_REG186__RTAVFSTARGETFREQCNTOVERRIDE__SHIFT                                                     0x0
+#define RTAVFS_REG186__RTAVFSTARGETFREQCNTOVERRIDESEL__SHIFT                                                  0x10
+#define RTAVFS_REG186__RESERVED__SHIFT                                                                        0x11
+#define RTAVFS_REG186__RTAVFSTARGETFREQCNTOVERRIDE_MASK                                                       0x0000FFFFL
+#define RTAVFS_REG186__RTAVFSTARGETFREQCNTOVERRIDESEL_MASK                                                    0x00010000L
+#define RTAVFS_REG186__RESERVED_MASK                                                                          0xFFFE0000L
+//RTAVFS_REG187
+#define RTAVFS_REG187__RTAVFSCURRENTFREQCNTOVERRIDE__SHIFT                                                    0x0
+#define RTAVFS_REG187__RTAVFSCURRENTFREQCNTOVERRIDESEL__SHIFT                                                 0x10
+#define RTAVFS_REG187__RESERVED__SHIFT                                                                        0x11
+#define RTAVFS_REG187__RTAVFSCURRENTFREQCNTOVERRIDE_MASK                                                      0x0000FFFFL
+#define RTAVFS_REG187__RTAVFSCURRENTFREQCNTOVERRIDESEL_MASK                                                   0x00010000L
+#define RTAVFS_REG187__RESERVED_MASK                                                                          0xFFFE0000L
+//RTAVFS_REG189
+#define RTAVFS_REG189__RTAVFSVOLTCODEFROMPI__SHIFT                                                            0x0
+#define RTAVFS_REG189__RTAVFSVOLTCODEFROMBINARYSEARCH__SHIFT                                                  0xa
+#define RTAVFS_REG189__RTAVFSVDDREGON__SHIFT                                                                  0x14
+#define RTAVFS_REG189__RTAVFSVDDABOVEVDDRET__SHIFT                                                            0x15
+#define RTAVFS_REG189__RESERVED__SHIFT                                                                        0x16
+#define RTAVFS_REG189__RTAVFSVOLTCODEFROMPI_MASK                                                              0x000003FFL
+#define RTAVFS_REG189__RTAVFSVOLTCODEFROMBINARYSEARCH_MASK                                                    0x000FFC00L
+#define RTAVFS_REG189__RTAVFSVDDREGON_MASK                                                                    0x00100000L
+#define RTAVFS_REG189__RTAVFSVDDABOVEVDDRET_MASK                                                              0x00200000L
+#define RTAVFS_REG189__RESERVED_MASK                                                                          0xFFC00000L
+//RTAVFS_REG190
+#define RTAVFS_REG190__RTAVFSIGNORERLCREQ__SHIFT                                                              0x0
+#define RTAVFS_REG190__RTAVFSRIPPLECOUNTEROUTSEL__SHIFT                                                       0x1
+#define RTAVFS_REG190__RTAVFSRUNLOOP__SHIFT                                                                   0x6
+#define RTAVFS_REG190__RTAVFSSAVECPOWEIGHTS__SHIFT                                                            0x7
+#define RTAVFS_REG190__RTAVFSRESTORECPOWEIGHTS__SHIFT                                                         0x8
+#define RTAVFS_REG190__RTAVFSRESETRETENTIONREGS__SHIFT                                                        0x9
+#define RTAVFS_REG190__RESERVED__SHIFT                                                                        0xa
+#define RTAVFS_REG190__RTAVFSIGNORERLCREQ_MASK                                                                0x00000001L
+#define RTAVFS_REG190__RTAVFSRIPPLECOUNTEROUTSEL_MASK                                                         0x0000003EL
+#define RTAVFS_REG190__RTAVFSRUNLOOP_MASK                                                                     0x00000040L
+#define RTAVFS_REG190__RTAVFSSAVECPOWEIGHTS_MASK                                                              0x00000080L
+#define RTAVFS_REG190__RTAVFSRESTORECPOWEIGHTS_MASK                                                           0x00000100L
+#define RTAVFS_REG190__RTAVFSRESETRETENTIONREGS_MASK                                                          0x00000200L
+#define RTAVFS_REG190__RESERVED_MASK                                                                          0xFFFFFC00L
+//RTAVFS_REG191
+#define RTAVFS_REG191__RTAVFSSTOPATSTARTUP__SHIFT                                                             0x0
+#define RTAVFS_REG191__RTAVFSSTOPATIDLE__SHIFT                                                                0x1
+#define RTAVFS_REG191__RTAVFSSTOPATRESETCPORIPPLECOUNTERS__SHIFT                                              0x2
+#define RTAVFS_REG191__RTAVFSSTOPATSTARTCPOS__SHIFT                                                           0x3
+#define RTAVFS_REG191__RTAVFSSTOPATSTARTRIPPLECOUNTERS__SHIFT                                                 0x4
+#define RTAVFS_REG191__RTAVFSSTOPATRIPPLECOUNTERSDONE__SHIFT                                                  0x5
+#define RTAVFS_REG191__RTAVFSSTOPATCPOFINALRESULTREADY__SHIFT                                                 0x6
+#define RTAVFS_REG191__RTAVFSSTOPATVOLTCODEREADY__SHIFT                                                       0x7
+#define RTAVFS_REG191__RTAVFSSTOPATTARGETVOLATGEREADY__SHIFT                                                  0x8
+#define RTAVFS_REG191__RTAVFSSTOPATSTOPCPOS__SHIFT                                                            0x9
+#define RTAVFS_REG191__RTAVFSSTOPATWAITFORACK__SHIFT                                                          0xa
+#define RTAVFS_REG191__RESERVED__SHIFT                                                                        0xb
+#define RTAVFS_REG191__RTAVFSSTOPATSTARTUP_MASK                                                               0x00000001L
+#define RTAVFS_REG191__RTAVFSSTOPATIDLE_MASK                                                                  0x00000002L
+#define RTAVFS_REG191__RTAVFSSTOPATRESETCPORIPPLECOUNTERS_MASK                                                0x00000004L
+#define RTAVFS_REG191__RTAVFSSTOPATSTARTCPOS_MASK                                                             0x00000008L
+#define RTAVFS_REG191__RTAVFSSTOPATSTARTRIPPLECOUNTERS_MASK                                                   0x00000010L
+#define RTAVFS_REG191__RTAVFSSTOPATRIPPLECOUNTERSDONE_MASK                                                    0x00000020L
+#define RTAVFS_REG191__RTAVFSSTOPATCPOFINALRESULTREADY_MASK                                                   0x00000040L
+#define RTAVFS_REG191__RTAVFSSTOPATVOLTCODEREADY_MASK                                                         0x00000080L
+#define RTAVFS_REG191__RTAVFSSTOPATTARGETVOLATGEREADY_MASK                                                    0x00000100L
+#define RTAVFS_REG191__RTAVFSSTOPATSTOPCPOS_MASK                                                              0x00000200L
+#define RTAVFS_REG191__RTAVFSSTOPATWAITFORACK_MASK                                                            0x00000400L
+#define RTAVFS_REG191__RESERVED_MASK                                                                          0xFFFFF800L
+//RTAVFS_REG192
+#define RTAVFS_REG192__RTAVFSAVFSSCALEDCPOCOUNT__SHIFT                                                        0x0
+#define RTAVFS_REG192__RTAVFSAVFSFINALMINCPOCOUNT__SHIFT                                                      0x10
+#define RTAVFS_REG192__RTAVFSAVFSSCALEDCPOCOUNT_MASK                                                          0x0000FFFFL
+#define RTAVFS_REG192__RTAVFSAVFSFINALMINCPOCOUNT_MASK                                                        0xFFFF0000L
+//RTAVFS_REG193
+#define RTAVFS_REG193__RTAVFSFSMSTATE__SHIFT                                                                  0x0
+#define RTAVFS_REG193__RESERVED__SHIFT                                                                        0x10
+#define RTAVFS_REG193__RTAVFSFSMSTATE_MASK                                                                    0x0000FFFFL
+#define RTAVFS_REG193__RESERVED_MASK                                                                          0xFFFF0000L
+//RTAVFS_REG194
+#define RTAVFS_REG194__RTAVFSRIPPLECNTREAD__SHIFT                                                             0x0
+#define RTAVFS_REG194__RTAVFSRIPPLECNTREAD_MASK                                                               0xFFFFFFFFL
+
+
+// addressBlock: sqind
+//SQ_DEBUG_STS_LOCAL
+#define SQ_DEBUG_STS_LOCAL__BUSY__SHIFT                                                                       0x0
+#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL__SHIFT                                                                 0x4
+#define SQ_DEBUG_STS_LOCAL__SQ_BUSY__SHIFT                                                                    0xc
+#define SQ_DEBUG_STS_LOCAL__IS_BUSY__SHIFT                                                                    0xd
+#define SQ_DEBUG_STS_LOCAL__IB_BUSY__SHIFT                                                                    0xe
+#define SQ_DEBUG_STS_LOCAL__ARB_BUSY__SHIFT                                                                   0xf
+#define SQ_DEBUG_STS_LOCAL__EXP_BUSY__SHIFT                                                                   0x10
+#define SQ_DEBUG_STS_LOCAL__BRMSG_BUSY__SHIFT                                                                 0x11
+#define SQ_DEBUG_STS_LOCAL__VM_BUSY__SHIFT                                                                    0x12
+#define SQ_DEBUG_STS_LOCAL__BUSY_MASK                                                                         0x00000001L
+#define SQ_DEBUG_STS_LOCAL__WAVE_LEVEL_MASK                                                                   0x000003F0L
+#define SQ_DEBUG_STS_LOCAL__SQ_BUSY_MASK                                                                      0x00001000L
+#define SQ_DEBUG_STS_LOCAL__IS_BUSY_MASK                                                                      0x00002000L
+#define SQ_DEBUG_STS_LOCAL__IB_BUSY_MASK                                                                      0x00004000L
+#define SQ_DEBUG_STS_LOCAL__ARB_BUSY_MASK                                                                     0x00008000L
+#define SQ_DEBUG_STS_LOCAL__EXP_BUSY_MASK                                                                     0x00010000L
+#define SQ_DEBUG_STS_LOCAL__BRMSG_BUSY_MASK                                                                   0x00020000L
+#define SQ_DEBUG_STS_LOCAL__VM_BUSY_MASK                                                                      0x00040000L
+//SQ_DEBUG_CTRL_LOCAL
+#define SQ_DEBUG_CTRL_LOCAL__UNUSED__SHIFT                                                                    0x0
+#define SQ_DEBUG_CTRL_LOCAL__UNUSED_MASK                                                                      0x000000FFL
+//SQ_WAVE_ACTIVE
+#define SQ_WAVE_ACTIVE__WAVE_SLOT__SHIFT                                                                      0x0
+#define SQ_WAVE_ACTIVE__WAVE_SLOT_MASK                                                                        0x000FFFFFL
+//SQ_WAVE_VALID_AND_IDLE
+#define SQ_WAVE_VALID_AND_IDLE__WAVE_SLOT__SHIFT                                                              0x0
+#define SQ_WAVE_VALID_AND_IDLE__WAVE_SLOT_MASK                                                                0x000FFFFFL
+//SQ_WAVE_MODE
+#define SQ_WAVE_MODE__FP_ROUND__SHIFT                                                                         0x0
+#define SQ_WAVE_MODE__FP_DENORM__SHIFT                                                                        0x4
+#define SQ_WAVE_MODE__DX10_CLAMP__SHIFT                                                                       0x8
+#define SQ_WAVE_MODE__IEEE__SHIFT                                                                             0x9
+#define SQ_WAVE_MODE__LOD_CLAMPED__SHIFT                                                                      0xa
+#define SQ_WAVE_MODE__TRAP_AFTER_INST_EN__SHIFT                                                               0xb
+#define SQ_WAVE_MODE__EXCP_EN__SHIFT                                                                          0xc
+#define SQ_WAVE_MODE__WAVE_END__SHIFT                                                                         0x15
+#define SQ_WAVE_MODE__FP16_OVFL__SHIFT                                                                        0x17
+#define SQ_WAVE_MODE__DISABLE_PERF__SHIFT                                                                     0x1b
+#define SQ_WAVE_MODE__FP_ROUND_MASK                                                                           0x0000000FL
+#define SQ_WAVE_MODE__FP_DENORM_MASK                                                                          0x000000F0L
+#define SQ_WAVE_MODE__DX10_CLAMP_MASK                                                                         0x00000100L
+#define SQ_WAVE_MODE__IEEE_MASK                                                                               0x00000200L
+#define SQ_WAVE_MODE__LOD_CLAMPED_MASK                                                                        0x00000400L
+#define SQ_WAVE_MODE__TRAP_AFTER_INST_EN_MASK                                                                 0x00000800L
+#define SQ_WAVE_MODE__EXCP_EN_MASK                                                                            0x001FF000L
+#define SQ_WAVE_MODE__WAVE_END_MASK                                                                           0x00200000L
+#define SQ_WAVE_MODE__FP16_OVFL_MASK                                                                          0x00800000L
+#define SQ_WAVE_MODE__DISABLE_PERF_MASK                                                                       0x08000000L
+//SQ_WAVE_STATUS
+#define SQ_WAVE_STATUS__SCC__SHIFT                                                                            0x0
+#define SQ_WAVE_STATUS__SPI_PRIO__SHIFT                                                                       0x1
+#define SQ_WAVE_STATUS__USER_PRIO__SHIFT                                                                      0x3
+#define SQ_WAVE_STATUS__PRIV__SHIFT                                                                           0x5
+#define SQ_WAVE_STATUS__TRAP_EN__SHIFT                                                                        0x6
+#define SQ_WAVE_STATUS__TTRACE_EN__SHIFT                                                                      0x7
+#define SQ_WAVE_STATUS__EXPORT_RDY__SHIFT                                                                     0x8
+#define SQ_WAVE_STATUS__EXECZ__SHIFT                                                                          0x9
+#define SQ_WAVE_STATUS__VCCZ__SHIFT                                                                           0xa
+#define SQ_WAVE_STATUS__IN_TG__SHIFT                                                                          0xb
+#define SQ_WAVE_STATUS__IN_BARRIER__SHIFT                                                                     0xc
+#define SQ_WAVE_STATUS__HALT__SHIFT                                                                           0xd
+#define SQ_WAVE_STATUS__TRAP__SHIFT                                                                           0xe
+#define SQ_WAVE_STATUS__TTRACE_SIMD_EN__SHIFT                                                                 0xf
+#define SQ_WAVE_STATUS__VALID__SHIFT                                                                          0x10
+#define SQ_WAVE_STATUS__ECC_ERR__SHIFT                                                                        0x11
+#define SQ_WAVE_STATUS__SKIP_EXPORT__SHIFT                                                                    0x12
+#define SQ_WAVE_STATUS__PERF_EN__SHIFT                                                                        0x13
+#define SQ_WAVE_STATUS__COND_DBG_USER__SHIFT                                                                  0x14
+#define SQ_WAVE_STATUS__COND_DBG_SYS__SHIFT                                                                   0x15
+#define SQ_WAVE_STATUS__OREO_CONFLICT__SHIFT                                                                  0x16
+#define SQ_WAVE_STATUS__FATAL_HALT__SHIFT                                                                     0x17
+#define SQ_WAVE_STATUS__NO_VGPRS__SHIFT                                                                       0x18
+#define SQ_WAVE_STATUS__LDS_PARAM_READY__SHIFT                                                                0x19
+#define SQ_WAVE_STATUS__MUST_GS_ALLOC__SHIFT                                                                  0x1a
+#define SQ_WAVE_STATUS__MUST_EXPORT__SHIFT                                                                    0x1b
+#define SQ_WAVE_STATUS__IDLE__SHIFT                                                                           0x1c
+#define SQ_WAVE_STATUS__SCRATCH_EN__SHIFT                                                                     0x1d
+#define SQ_WAVE_STATUS__SCC_MASK                                                                              0x00000001L
+#define SQ_WAVE_STATUS__SPI_PRIO_MASK                                                                         0x00000006L
+#define SQ_WAVE_STATUS__USER_PRIO_MASK                                                                        0x00000018L
+#define SQ_WAVE_STATUS__PRIV_MASK                                                                             0x00000020L
+#define SQ_WAVE_STATUS__TRAP_EN_MASK                                                                          0x00000040L
+#define SQ_WAVE_STATUS__TTRACE_EN_MASK                                                                        0x00000080L
+#define SQ_WAVE_STATUS__EXPORT_RDY_MASK                                                                       0x00000100L
+#define SQ_WAVE_STATUS__EXECZ_MASK                                                                            0x00000200L
+#define SQ_WAVE_STATUS__VCCZ_MASK                                                                             0x00000400L
+#define SQ_WAVE_STATUS__IN_TG_MASK                                                                            0x00000800L
+#define SQ_WAVE_STATUS__IN_BARRIER_MASK                                                                       0x00001000L
+#define SQ_WAVE_STATUS__HALT_MASK                                                                             0x00002000L
+#define SQ_WAVE_STATUS__TRAP_MASK                                                                             0x00004000L
+#define SQ_WAVE_STATUS__TTRACE_SIMD_EN_MASK                                                                   0x00008000L
+#define SQ_WAVE_STATUS__VALID_MASK                                                                            0x00010000L
+#define SQ_WAVE_STATUS__ECC_ERR_MASK                                                                          0x00020000L
+#define SQ_WAVE_STATUS__SKIP_EXPORT_MASK                                                                      0x00040000L
+#define SQ_WAVE_STATUS__PERF_EN_MASK                                                                          0x00080000L
+#define SQ_WAVE_STATUS__COND_DBG_USER_MASK                                                                    0x00100000L
+#define SQ_WAVE_STATUS__COND_DBG_SYS_MASK                                                                     0x00200000L
+#define SQ_WAVE_STATUS__OREO_CONFLICT_MASK                                                                    0x00400000L
+#define SQ_WAVE_STATUS__FATAL_HALT_MASK                                                                       0x00800000L
+#define SQ_WAVE_STATUS__NO_VGPRS_MASK                                                                         0x01000000L
+#define SQ_WAVE_STATUS__LDS_PARAM_READY_MASK                                                                  0x02000000L
+#define SQ_WAVE_STATUS__MUST_GS_ALLOC_MASK                                                                    0x04000000L
+#define SQ_WAVE_STATUS__MUST_EXPORT_MASK                                                                      0x08000000L
+#define SQ_WAVE_STATUS__IDLE_MASK                                                                             0x10000000L
+#define SQ_WAVE_STATUS__SCRATCH_EN_MASK                                                                       0x20000000L
+//SQ_WAVE_TRAPSTS
+#define SQ_WAVE_TRAPSTS__EXCP__SHIFT                                                                          0x0
+#define SQ_WAVE_TRAPSTS__SAVECTX__SHIFT                                                                       0xa
+#define SQ_WAVE_TRAPSTS__ILLEGAL_INST__SHIFT                                                                  0xb
+#define SQ_WAVE_TRAPSTS__EXCP_HI__SHIFT                                                                       0xc
+#define SQ_WAVE_TRAPSTS__BUFFER_OOB__SHIFT                                                                    0xf
+#define SQ_WAVE_TRAPSTS__HOST_TRAP__SHIFT                                                                     0x10
+#define SQ_WAVE_TRAPSTS__WAVESTART__SHIFT                                                                     0x11
+#define SQ_WAVE_TRAPSTS__WAVE_END__SHIFT                                                                      0x12
+#define SQ_WAVE_TRAPSTS__PERF_SNAPSHOT__SHIFT                                                                 0x13
+#define SQ_WAVE_TRAPSTS__TRAP_AFTER_INST__SHIFT                                                               0x14
+#define SQ_WAVE_TRAPSTS__UTC_ERROR__SHIFT                                                                     0x1c
+#define SQ_WAVE_TRAPSTS__EXCP_MASK                                                                            0x000001FFL
+#define SQ_WAVE_TRAPSTS__SAVECTX_MASK                                                                         0x00000400L
+#define SQ_WAVE_TRAPSTS__ILLEGAL_INST_MASK                                                                    0x00000800L
+#define SQ_WAVE_TRAPSTS__EXCP_HI_MASK                                                                         0x00007000L
+#define SQ_WAVE_TRAPSTS__BUFFER_OOB_MASK                                                                      0x00008000L
+#define SQ_WAVE_TRAPSTS__HOST_TRAP_MASK                                                                       0x00010000L
+#define SQ_WAVE_TRAPSTS__WAVESTART_MASK                                                                       0x00020000L
+#define SQ_WAVE_TRAPSTS__WAVE_END_MASK                                                                        0x00040000L
+#define SQ_WAVE_TRAPSTS__PERF_SNAPSHOT_MASK                                                                   0x00080000L
+#define SQ_WAVE_TRAPSTS__TRAP_AFTER_INST_MASK                                                                 0x00100000L
+#define SQ_WAVE_TRAPSTS__UTC_ERROR_MASK                                                                       0x10000000L
+//SQ_WAVE_GPR_ALLOC
+#define SQ_WAVE_GPR_ALLOC__VGPR_BASE__SHIFT                                                                   0x0
+#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE__SHIFT                                                                   0xc
+#define SQ_WAVE_GPR_ALLOC__VGPR_BASE_MASK                                                                     0x000001FFL
+#define SQ_WAVE_GPR_ALLOC__VGPR_SIZE_MASK                                                                     0x000FF000L
+//SQ_WAVE_LDS_ALLOC
+#define SQ_WAVE_LDS_ALLOC__LDS_BASE__SHIFT                                                                    0x0
+#define SQ_WAVE_LDS_ALLOC__LDS_SIZE__SHIFT                                                                    0xc
+#define SQ_WAVE_LDS_ALLOC__VGPR_SHARED_SIZE__SHIFT                                                            0x18
+#define SQ_WAVE_LDS_ALLOC__LDS_BASE_MASK                                                                      0x000001FFL
+#define SQ_WAVE_LDS_ALLOC__LDS_SIZE_MASK                                                                      0x001FF000L
+#define SQ_WAVE_LDS_ALLOC__VGPR_SHARED_SIZE_MASK                                                              0x0F000000L
+//SQ_WAVE_IB_STS
+#define SQ_WAVE_IB_STS__EXP_CNT__SHIFT                                                                        0x0
+#define SQ_WAVE_IB_STS__LGKM_CNT__SHIFT                                                                       0x4
+#define SQ_WAVE_IB_STS__VM_CNT__SHIFT                                                                         0xa
+#define SQ_WAVE_IB_STS__VS_CNT__SHIFT                                                                         0x1a
+#define SQ_WAVE_IB_STS__EXP_CNT_MASK                                                                          0x00000007L
+#define SQ_WAVE_IB_STS__LGKM_CNT_MASK                                                                         0x000003F0L
+#define SQ_WAVE_IB_STS__VM_CNT_MASK                                                                           0x0000FC00L
+#define SQ_WAVE_IB_STS__VS_CNT_MASK                                                                           0xFC000000L
+//SQ_WAVE_PC_LO
+#define SQ_WAVE_PC_LO__PC_LO__SHIFT                                                                           0x0
+#define SQ_WAVE_PC_LO__PC_LO_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_PC_HI
+#define SQ_WAVE_PC_HI__PC_HI__SHIFT                                                                           0x0
+#define SQ_WAVE_PC_HI__PC_HI_MASK                                                                             0x0000FFFFL
+//SQ_WAVE_IB_DBG1
+#define SQ_WAVE_IB_DBG1__WAVE_IDLE__SHIFT                                                                     0x18
+#define SQ_WAVE_IB_DBG1__MISC_CNT__SHIFT                                                                      0x19
+#define SQ_WAVE_IB_DBG1__WAVE_IDLE_MASK                                                                       0x01000000L
+#define SQ_WAVE_IB_DBG1__MISC_CNT_MASK                                                                        0xFE000000L
+//SQ_WAVE_FLUSH_IB
+#define SQ_WAVE_FLUSH_IB__UNUSED__SHIFT                                                                       0x0
+#define SQ_WAVE_FLUSH_IB__UNUSED_MASK                                                                         0xFFFFFFFFL
+//SQ_WAVE_FLAT_SCRATCH_LO
+#define SQ_WAVE_FLAT_SCRATCH_LO__DATA__SHIFT                                                                  0x0
+#define SQ_WAVE_FLAT_SCRATCH_LO__DATA_MASK                                                                    0xFFFFFFFFL
+//SQ_WAVE_FLAT_SCRATCH_HI
+#define SQ_WAVE_FLAT_SCRATCH_HI__DATA__SHIFT                                                                  0x0
+#define SQ_WAVE_FLAT_SCRATCH_HI__DATA_MASK                                                                    0xFFFFFFFFL
+//SQ_WAVE_HW_ID1
+#define SQ_WAVE_HW_ID1__WAVE_ID__SHIFT                                                                        0x0
+#define SQ_WAVE_HW_ID1__SIMD_ID__SHIFT                                                                        0x8
+#define SQ_WAVE_HW_ID1__WGP_ID__SHIFT                                                                         0xa
+#define SQ_WAVE_HW_ID1__SA_ID__SHIFT                                                                          0x10
+#define SQ_WAVE_HW_ID1__SE_ID__SHIFT                                                                          0x12
+#define SQ_WAVE_HW_ID1__DP_RATE__SHIFT                                                                        0x1d
+#define SQ_WAVE_HW_ID1__WAVE_ID_MASK                                                                          0x0000001FL
+#define SQ_WAVE_HW_ID1__SIMD_ID_MASK                                                                          0x00000300L
+#define SQ_WAVE_HW_ID1__WGP_ID_MASK                                                                           0x00003C00L
+#define SQ_WAVE_HW_ID1__SA_ID_MASK                                                                            0x00010000L
+#define SQ_WAVE_HW_ID1__SE_ID_MASK                                                                            0x001C0000L
+#define SQ_WAVE_HW_ID1__DP_RATE_MASK                                                                          0xE0000000L
+//SQ_WAVE_HW_ID2
+#define SQ_WAVE_HW_ID2__QUEUE_ID__SHIFT                                                                       0x0
+#define SQ_WAVE_HW_ID2__PIPE_ID__SHIFT                                                                        0x4
+#define SQ_WAVE_HW_ID2__ME_ID__SHIFT                                                                          0x8
+#define SQ_WAVE_HW_ID2__STATE_ID__SHIFT                                                                       0xc
+#define SQ_WAVE_HW_ID2__WG_ID__SHIFT                                                                          0x10
+#define SQ_WAVE_HW_ID2__VM_ID__SHIFT                                                                          0x18
+#define SQ_WAVE_HW_ID2__QUEUE_ID_MASK                                                                         0x0000000FL
+#define SQ_WAVE_HW_ID2__PIPE_ID_MASK                                                                          0x00000030L
+#define SQ_WAVE_HW_ID2__ME_ID_MASK                                                                            0x00000300L
+#define SQ_WAVE_HW_ID2__STATE_ID_MASK                                                                         0x00007000L
+#define SQ_WAVE_HW_ID2__WG_ID_MASK                                                                            0x001F0000L
+#define SQ_WAVE_HW_ID2__VM_ID_MASK                                                                            0x0F000000L
+//SQ_WAVE_POPS_PACKER
+#define SQ_WAVE_POPS_PACKER__POPS_EN__SHIFT                                                                   0x0
+#define SQ_WAVE_POPS_PACKER__POPS_PACKER_ID__SHIFT                                                            0x1
+#define SQ_WAVE_POPS_PACKER__POPS_EN_MASK                                                                     0x00000001L
+#define SQ_WAVE_POPS_PACKER__POPS_PACKER_ID_MASK                                                              0x00000006L
+//SQ_WAVE_SCHED_MODE
+#define SQ_WAVE_SCHED_MODE__DEP_MODE__SHIFT                                                                   0x0
+#define SQ_WAVE_SCHED_MODE__DEP_MODE_MASK                                                                     0x00000003L
+//SQ_WAVE_IB_STS2
+#define SQ_WAVE_IB_STS2__INST_PREFETCH__SHIFT                                                                 0x0
+#define SQ_WAVE_IB_STS2__MEM_ORDER__SHIFT                                                                     0x8
+#define SQ_WAVE_IB_STS2__FWD_PROGRESS__SHIFT                                                                  0xa
+#define SQ_WAVE_IB_STS2__WAVE64__SHIFT                                                                        0xb
+#define SQ_WAVE_IB_STS2__INST_PREFETCH_MASK                                                                   0x00000003L
+#define SQ_WAVE_IB_STS2__MEM_ORDER_MASK                                                                       0x00000300L
+#define SQ_WAVE_IB_STS2__FWD_PROGRESS_MASK                                                                    0x00000400L
+#define SQ_WAVE_IB_STS2__WAVE64_MASK                                                                          0x00000800L
+//SQ_WAVE_SHADER_CYCLES
+#define SQ_WAVE_SHADER_CYCLES__CYCLES__SHIFT                                                                  0x0
+#define SQ_WAVE_SHADER_CYCLES__CYCLES_MASK                                                                    0x000FFFFFL
+//SQ_WAVE_TTMP0
+#define SQ_WAVE_TTMP0__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP0__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP1
+#define SQ_WAVE_TTMP1__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP1__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP2
+#define SQ_WAVE_TTMP2__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP2__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP3
+#define SQ_WAVE_TTMP3__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP3__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP4
+#define SQ_WAVE_TTMP4__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP4__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP5
+#define SQ_WAVE_TTMP5__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP5__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP6
+#define SQ_WAVE_TTMP6__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP6__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP7
+#define SQ_WAVE_TTMP7__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP7__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP8
+#define SQ_WAVE_TTMP8__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP8__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP9
+#define SQ_WAVE_TTMP9__DATA__SHIFT                                                                            0x0
+#define SQ_WAVE_TTMP9__DATA_MASK                                                                              0xFFFFFFFFL
+//SQ_WAVE_TTMP10
+#define SQ_WAVE_TTMP10__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP10__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP11
+#define SQ_WAVE_TTMP11__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP11__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP12
+#define SQ_WAVE_TTMP12__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP12__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP13
+#define SQ_WAVE_TTMP13__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP13__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP14
+#define SQ_WAVE_TTMP14__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP14__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_TTMP15
+#define SQ_WAVE_TTMP15__DATA__SHIFT                                                                           0x0
+#define SQ_WAVE_TTMP15__DATA_MASK                                                                             0xFFFFFFFFL
+//SQ_WAVE_M0
+#define SQ_WAVE_M0__M0__SHIFT                                                                                 0x0
+#define SQ_WAVE_M0__M0_MASK                                                                                   0xFFFFFFFFL
+//SQ_WAVE_EXEC_LO
+#define SQ_WAVE_EXEC_LO__EXEC_LO__SHIFT                                                                       0x0
+#define SQ_WAVE_EXEC_LO__EXEC_LO_MASK                                                                         0xFFFFFFFFL
+//SQ_WAVE_EXEC_HI
+#define SQ_WAVE_EXEC_HI__EXEC_HI__SHIFT                                                                       0x0
+#define SQ_WAVE_EXEC_HI__EXEC_HI_MASK                                                                         0xFFFFFFFFL
+
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h
index d8632ccf3494..c488d4a50cf4 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_offset.h
@@ -4409,6 +4409,10 @@
 #define mmVMSHAREDPF0_MC_VM_XGMI_LFB_SIZE_BASE_IDX                                                     1
 #define mmVMSHAREDPF0_MC_VM_CACHEABLE_DRAM_CNTL                                                        0x0af9
 #define mmVMSHAREDPF0_MC_VM_CACHEABLE_DRAM_CNTL_BASE_IDX                                               1
+#define mmMC_VM_XGMI_LFB_CNTL                                                                          0x0823
+#define mmMC_VM_XGMI_LFB_CNTL_BASE_IDX                                                                 0
+#define mmMC_VM_XGMI_LFB_SIZE                                                                          0x0824
+#define mmMC_VM_XGMI_LFB_SIZE_BASE_IDX                                                                 0
 
 
 // addressBlock: mmhub_utcl2_vmsharedvcdec
diff --git a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h
index 111a71b434e2..2969fbf282b7 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/mmhub/mmhub_9_4_1_sh_mask.h
@@ -26728,6 +26728,14 @@
 //VMSHAREDPF0_MC_VM_CACHEABLE_DRAM_CNTL
 #define VMSHAREDPF0_MC_VM_CACHEABLE_DRAM_CNTL__ENABLE_CACHEABLE_DRAM_ADDRESS_APERTURE__SHIFT                  0x0
 #define VMSHAREDPF0_MC_VM_CACHEABLE_DRAM_CNTL__ENABLE_CACHEABLE_DRAM_ADDRESS_APERTURE_MASK                    0x00000001L
+//MC_VM_XGMI_LFB_CNTL
+#define MC_VM_XGMI_LFB_CNTL__PF_LFB_REGION__SHIFT                                                             0x0
+#define MC_VM_XGMI_LFB_CNTL__PF_MAX_REGION__SHIFT                                                             0x3
+#define MC_VM_XGMI_LFB_CNTL__PF_LFB_REGION_MASK                                                               0x00000007L
+#define MC_VM_XGMI_LFB_CNTL__PF_MAX_REGION_MASK                                                               0x00000038L
+//MC_VM_XGMI_LFB_SIZE
+#define MC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE__SHIFT                                                               0x0
+#define MC_VM_XGMI_LFB_SIZE__PF_LFB_SIZE_MASK                                                                 0x0000FFFFL
 
 
 // addressBlock: mmhub_utcl2_vmsharedvcdec
diff --git a/drivers/gpu/drm/amd/include/kgd_pp_interface.h b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
index 7e3231c2191c..a40ead44778a 100644
--- a/drivers/gpu/drm/amd/include/kgd_pp_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_pp_interface.h
@@ -824,4 +824,62 @@ struct gpu_metrics_v2_2 {
 	uint64_t			indep_throttle_status;
 };
 
+struct gpu_metrics_v2_3 {
+	struct metrics_table_header	common_header;
+
+	/* Temperature */
+	uint16_t			temperature_gfx; // gfx temperature on APUs
+	uint16_t			temperature_soc; // soc temperature on APUs
+	uint16_t			temperature_core[8]; // CPU core temperature on APUs
+	uint16_t			temperature_l3[2];
+
+	/* Utilization */
+	uint16_t			average_gfx_activity;
+	uint16_t			average_mm_activity; // UVD or VCN
+
+	/* Driver attached timestamp (in ns) */
+	uint64_t			system_clock_counter;
+
+	/* Power/Energy */
+	uint16_t			average_socket_power; // dGPU + APU power on A + A platform
+	uint16_t			average_cpu_power;
+	uint16_t			average_soc_power;
+	uint16_t			average_gfx_power;
+	uint16_t			average_core_power[8]; // CPU core power on APUs
+
+	/* Average clocks */
+	uint16_t			average_gfxclk_frequency;
+	uint16_t			average_socclk_frequency;
+	uint16_t			average_uclk_frequency;
+	uint16_t			average_fclk_frequency;
+	uint16_t			average_vclk_frequency;
+	uint16_t			average_dclk_frequency;
+
+	/* Current clocks */
+	uint16_t			current_gfxclk;
+	uint16_t			current_socclk;
+	uint16_t			current_uclk;
+	uint16_t			current_fclk;
+	uint16_t			current_vclk;
+	uint16_t			current_dclk;
+	uint16_t			current_coreclk[8]; // CPU core clocks
+	uint16_t			current_l3clk[2];
+
+	/* Throttle status (ASIC dependent) */
+	uint32_t			throttle_status;
+
+	/* Fans */
+	uint16_t			fan_pwm;
+
+	uint16_t			padding[3];
+
+	/* Throttle status (ASIC independent) */
+	uint64_t			indep_throttle_status;
+
+	/* Average Temperature */
+	uint16_t			average_temperature_gfx; // average gfx temperature on APUs
+	uint16_t			average_temperature_soc; // average soc temperature on APUs
+	uint16_t			average_temperature_core[8]; // average CPU core temperature on APUs
+	uint16_t			average_temperature_l3[2];
+};
 #endif
diff --git a/drivers/gpu/drm/amd/include/mes_v11_api_def.h b/drivers/gpu/drm/amd/include/mes_v11_api_def.h
index 50bfa513cb35..7e85cdc5bd34 100644
--- a/drivers/gpu/drm/amd/include/mes_v11_api_def.h
+++ b/drivers/gpu/drm/amd/include/mes_v11_api_def.h
@@ -269,7 +269,8 @@ union MESAPI__ADD_QUEUE {
 			uint32_t map_kiq_utility_queue  : 1;
 			uint32_t is_kfd_process		: 1;
 			uint32_t trap_en		: 1;
-			uint32_t reserved		: 21;
+			uint32_t is_aql_queue		: 1;
+			uint32_t reserved		: 20;
 		};
 		struct MES_API_STATUS		api_status;
 		uint64_t                        tma_addr;
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
index 956b6ce81c84..1b300c569faf 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_dpm.c
@@ -668,6 +668,51 @@ int amdgpu_dpm_wait_for_event(struct amdgpu_device *adev,
 	return ret;
 }
 
+int amdgpu_dpm_set_residency_gfxoff(struct amdgpu_device *adev, bool value)
+{
+	struct smu_context *smu = adev->powerplay.pp_handle;
+	int ret = 0;
+
+	if (!is_support_sw_smu(adev))
+		return -EOPNOTSUPP;
+
+	mutex_lock(&adev->pm.mutex);
+	ret = smu_set_residency_gfxoff(smu, value);
+	mutex_unlock(&adev->pm.mutex);
+
+	return ret;
+}
+
+int amdgpu_dpm_get_residency_gfxoff(struct amdgpu_device *adev, u32 *value)
+{
+	struct smu_context *smu = adev->powerplay.pp_handle;
+	int ret = 0;
+
+	if (!is_support_sw_smu(adev))
+		return -EOPNOTSUPP;
+
+	mutex_lock(&adev->pm.mutex);
+	ret = smu_get_residency_gfxoff(smu, value);
+	mutex_unlock(&adev->pm.mutex);
+
+	return ret;
+}
+
+int amdgpu_dpm_get_entrycount_gfxoff(struct amdgpu_device *adev, u64 *value)
+{
+	struct smu_context *smu = adev->powerplay.pp_handle;
+	int ret = 0;
+
+	if (!is_support_sw_smu(adev))
+		return -EOPNOTSUPP;
+
+	mutex_lock(&adev->pm.mutex);
+	ret = smu_get_entrycount_gfxoff(smu, value);
+	mutex_unlock(&adev->pm.mutex);
+
+	return ret;
+}
+
 int amdgpu_dpm_get_status_gfxoff(struct amdgpu_device *adev, uint32_t *value)
 {
 	struct smu_context *smu = adev->powerplay.pp_handle;
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
index 5e318b3f6c0f..948cc75376f8 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
@@ -3405,9 +3405,6 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
 
 void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev)
 {
-	if (adev->pm.dpm_enabled == 0)
-		return;
-
 	if (adev->pm.int_hwmon_dev)
 		hwmon_device_unregister(adev->pm.int_hwmon_dev);
 
diff --git a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
index 65624d091ed2..cb5b9df78b4d 100644
--- a/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
+++ b/drivers/gpu/drm/amd/pm/inc/amdgpu_dpm.h
@@ -435,6 +435,9 @@ int amdgpu_dpm_set_soft_freq_range(struct amdgpu_device *adev,
 int amdgpu_dpm_write_watermarks_table(struct amdgpu_device *adev);
 int amdgpu_dpm_wait_for_event(struct amdgpu_device *adev, enum smu_event_type event,
 		       uint64_t event_arg);
+int amdgpu_dpm_get_residency_gfxoff(struct amdgpu_device *adev, u32 *value);
+int amdgpu_dpm_set_residency_gfxoff(struct amdgpu_device *adev, bool value);
+int amdgpu_dpm_get_entrycount_gfxoff(struct amdgpu_device *adev, u64 *value);
 int amdgpu_dpm_get_status_gfxoff(struct amdgpu_device *adev, uint32_t *value);
 uint64_t amdgpu_dpm_get_thermal_throttling_counter(struct amdgpu_device *adev);
 void amdgpu_dpm_gfx_state_change(struct amdgpu_device *adev,
diff --git a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
index 1eb4e613b27a..ec055858eb95 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c
@@ -1485,6 +1485,7 @@ static int pp_get_prv_buffer_details(void *handle, void **addr, size_t *size)
 {
 	struct pp_hwmgr *hwmgr = handle;
 	struct amdgpu_device *adev = hwmgr->adev;
+	int err;
 
 	if (!addr || !size)
 		return -EINVAL;
@@ -1492,7 +1493,9 @@ static int pp_get_prv_buffer_details(void *handle, void **addr, size_t *size)
 	*addr = NULL;
 	*size = 0;
 	if (adev->pm.smu_prv_buffer) {
-		amdgpu_bo_kmap(adev->pm.smu_prv_buffer, addr);
+		err = amdgpu_bo_kmap(adev->pm.smu_prv_buffer, addr);
+		if (err)
+			return err;
 		*size = adev->pm.smu_prv_buffer_size;
 	}
 
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c
index dad3e3741a4e..190af79f3236 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega10_thermal.c
@@ -67,22 +67,21 @@ int vega10_fan_ctrl_get_fan_speed_info(struct pp_hwmgr *hwmgr,
 int vega10_fan_ctrl_get_fan_speed_pwm(struct pp_hwmgr *hwmgr,
 		uint32_t *speed)
 {
-	uint32_t current_rpm;
-	uint32_t percent = 0;
-
-	if (hwmgr->thermal_controller.fanInfo.bNoFan)
-		return 0;
+	struct amdgpu_device *adev = hwmgr->adev;
+	uint32_t duty100, duty;
+	uint64_t tmp64;
 
-	if (vega10_get_current_rpm(hwmgr, &current_rpm))
-		return -1;
+	duty100 = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_FDO_CTRL1),
+				CG_FDO_CTRL1, FMAX_DUTY100);
+	duty = REG_GET_FIELD(RREG32_SOC15(THM, 0, mmCG_THERMAL_STATUS),
+				CG_THERMAL_STATUS, FDO_PWM_DUTY);
 
-	if (hwmgr->thermal_controller.
-			advanceFanControlParameters.usMaxFanRPM != 0)
-		percent = current_rpm * 255 /
-			hwmgr->thermal_controller.
-			advanceFanControlParameters.usMaxFanRPM;
+	if (!duty100)
+		return -EINVAL;
 
-	*speed = MIN(percent, 255);
+	tmp64 = (uint64_t)duty * 255;
+	do_div(tmp64, duty100);
+	*speed = MIN((uint32_t)tmp64, 255);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c
index 1e79baab753e..bd54fbd393b9 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega12_processpptables.c
@@ -195,7 +195,6 @@ static int init_powerplay_table_information(
 	struct phm_ppt_v3_information *pptable_information =
 		(struct phm_ppt_v3_information *)hwmgr->pptable;
 	uint32_t disable_power_control = 0;
-	int result;
 
 	hwmgr->thermal_controller.ucType = powerplay_table->ucThermalControllerType;
 	pptable_information->uc_thermal_controller_type = powerplay_table->ucThermalControllerType;
@@ -257,9 +256,7 @@ static int init_powerplay_table_information(
 	if (pptable_information->smc_pptable == NULL)
 		return -ENOMEM;
 
-	result = append_vbios_pptable(hwmgr, (pptable_information->smc_pptable));
-
-	return result;
+	return append_vbios_pptable(hwmgr, (pptable_information->smc_pptable));
 }
 
 static int vega12_pp_tables_initialize(struct pp_hwmgr *hwmgr)
diff --git a/drivers/gpu/drm/amd/pm/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/pm/powerplay/inc/hardwaremanager.h
index 6e0be6027705..01a7d66864f2 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/inc/hardwaremanager.h
+++ b/drivers/gpu/drm/amd/pm/powerplay/inc/hardwaremanager.h
@@ -401,8 +401,6 @@ extern int phm_powerdown_uvd(struct pp_hwmgr *hwmgr);
 extern int phm_setup_asic(struct pp_hwmgr *hwmgr);
 extern int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr);
 extern int phm_disable_dynamic_state_management(struct pp_hwmgr *hwmgr);
-extern bool phm_is_hw_access_blocked(struct pp_hwmgr *hwmgr);
-extern int phm_block_hw_access(struct pp_hwmgr *hwmgr, bool block);
 extern int phm_set_power_state(struct pp_hwmgr *hwmgr,
 		    const struct pp_hw_power_state *pcurrent_state,
 		 const struct pp_hw_power_state *pnew_power_state);
diff --git a/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.c b/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.c
index 45214a364baa..e7ed2a7adf8f 100644
--- a/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.c
+++ b/drivers/gpu/drm/amd/pm/powerplay/smumgr/polaris10_smumgr.c
@@ -2567,15 +2567,13 @@ static uint8_t polaris10_get_memory_modile_index(struct pp_hwmgr *hwmgr)
 
 static int polaris10_initialize_mc_reg_table(struct pp_hwmgr *hwmgr)
 {
-	int result;
 	struct polaris10_smumgr *smu_data = (struct polaris10_smumgr *)(hwmgr->smu_backend);
 	pp_atomctrl_mc_reg_table *mc_reg_table = &smu_data->mc_reg_table;
 	uint8_t module_index = polaris10_get_memory_modile_index(hwmgr);
 
 	memset(mc_reg_table, 0, sizeof(pp_atomctrl_mc_reg_table));
-	result = atomctrl_initialize_mc_reg_table_v2_2(hwmgr, module_index, mc_reg_table);
 
-	return result;
+	return atomctrl_initialize_mc_reg_table_v2_2(hwmgr, module_index, mc_reg_table);
 }
 
 static bool polaris10_is_dpm_running(struct pp_hwmgr *hwmgr)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
index 7510d470b864..13c5c7f1ecb9 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
@@ -90,6 +90,30 @@ static int smu_sys_set_pp_feature_mask(void *handle,
 	return smu_set_pp_feature_mask(smu, new_mask);
 }
 
+int smu_set_residency_gfxoff(struct smu_context *smu, bool value)
+{
+	if (!smu->ppt_funcs->set_gfx_off_residency)
+		return -EINVAL;
+
+	return smu_set_gfx_off_residency(smu, value);
+}
+
+int smu_get_residency_gfxoff(struct smu_context *smu, u32 *value)
+{
+	if (!smu->ppt_funcs->get_gfx_off_residency)
+		return -EINVAL;
+
+	return smu_get_gfx_off_residency(smu, value);
+}
+
+int smu_get_entrycount_gfxoff(struct smu_context *smu, u64 *value)
+{
+	if (!smu->ppt_funcs->get_gfx_off_entrycount)
+		return -EINVAL;
+
+	return smu_get_gfx_off_entrycount(smu, value);
+}
+
 int smu_get_status_gfxoff(struct smu_context *smu, uint32_t *value)
 {
 	if (!smu->ppt_funcs->get_gfx_off_status)
@@ -581,6 +605,7 @@ static int smu_set_funcs(struct amdgpu_device *adev)
 		smu->od_enabled = true;
 		break;
 	case IP_VERSION(13, 0, 0):
+	case IP_VERSION(13, 0, 10):
 		smu_v13_0_0_set_ppt_funcs(smu);
 		break;
 	case IP_VERSION(13, 0, 7):
@@ -1576,6 +1601,7 @@ static int smu_suspend(void *handle)
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	struct smu_context *smu = adev->powerplay.pp_handle;
 	int ret;
+	uint64_t count;
 
 	if (amdgpu_sriov_vf(adev)&& !amdgpu_sriov_is_pp_one_vf(adev))
 		return 0;
@@ -1593,6 +1619,14 @@ static int smu_suspend(void *handle)
 
 	smu_set_gfx_cgpg(smu, false);
 
+	/*
+	 * pwfw resets entrycount when device is suspended, so we save the
+	 * last value to be used when we resume to keep it consistent
+	 */
+	ret = smu_get_entrycount_gfxoff(smu, &count);
+	if (!ret)
+		adev->gfx.gfx_off_entrycount = count;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
index b81c657c7386..e2fa3b066b96 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/amdgpu_smu.h
@@ -1112,6 +1112,22 @@ struct pptable_funcs {
 	uint32_t (*get_gfx_off_status)(struct smu_context *smu);
 
 	/**
+	 * @gfx_off_entrycount: total GFXOFF entry count at the time of
+	 * query since system power-up
+	 */
+	u32 (*get_gfx_off_entrycount)(struct smu_context *smu, uint64_t *entrycount);
+
+	/**
+	 * @set_gfx_off_residency: set 1 to start logging, 0 to stop logging
+	 */
+	u32 (*set_gfx_off_residency)(struct smu_context *smu, bool start);
+
+	/**
+	 * @get_gfx_off_residency: Average GFXOFF residency % during the logging interval
+	 */
+	u32 (*get_gfx_off_residency)(struct smu_context *smu, uint32_t *residency);
+
+	/**
 	 * @register_irq_handler: Register interupt request handlers.
 	 */
 	int (*register_irq_handler)(struct smu_context *smu);
@@ -1454,6 +1470,12 @@ int smu_set_ac_dc(struct smu_context *smu);
 
 int smu_allow_xgmi_power_down(struct smu_context *smu, bool en);
 
+int smu_get_entrycount_gfxoff(struct smu_context *smu, u64 *value);
+
+int smu_get_residency_gfxoff(struct smu_context *smu, u32 *value);
+
+int smu_set_residency_gfxoff(struct smu_context *smu, bool value);
+
 int smu_get_status_gfxoff(struct smu_context *smu, uint32_t *value);
 
 int smu_handle_passthrough_sbr(struct smu_context *smu, bool enable);
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.h
index d2e10a724560..82cf9e563065 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_0_7_ppsmc.h
@@ -137,7 +137,7 @@
 #define PPSMC_MSG_DisallowGpo                    0x56
 
 #define PPSMC_MSG_Enable2ndUSB20Port             0x57
-
-#define PPSMC_Message_Count                      0x58
+#define PPSMC_MSG_DriverMode2Reset               0x5D
+#define PPSMC_Message_Count                      0x5E
 
 #endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_5_ppsmc.h b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_5_ppsmc.h
index fe130a497d6c..7471e2df2828 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_5_ppsmc.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/pmfw_if/smu_v11_5_ppsmc.h
@@ -108,7 +108,10 @@
 #define PPSMC_MSG_SetSlowPPTLimit                      0x4A
 #define PPSMC_MSG_GetFastPPTLimit                      0x4B
 #define PPSMC_MSG_GetSlowPPTLimit                      0x4C
-#define PPSMC_Message_Count                            0x4D
+#define PPSMC_MSG_GetGfxOffStatus		       0x50
+#define PPSMC_MSG_GetGfxOffEntryCount		       0x51
+#define PPSMC_MSG_LogGfxOffResidency		       0x52
+#define PPSMC_Message_Count                            0x53
 
 //Argument for PPSMC_MSG_GfxDeviceDriverReset
 enum {
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
index 19084a4fcb2b..58098b82df66 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_types.h
@@ -235,7 +235,11 @@
 	__SMU_DUMMY_MAP(UnforceGfxVid),           \
 	__SMU_DUMMY_MAP(HeavySBR),			\
 	__SMU_DUMMY_MAP(SetBadHBMPagesRetiredFlagsPerChannel), \
-	__SMU_DUMMY_MAP(EnableGfxImu),
+	__SMU_DUMMY_MAP(EnableGfxImu), \
+	__SMU_DUMMY_MAP(DriverMode2Reset), \
+	__SMU_DUMMY_MAP(GetGfxOffStatus),		 \
+	__SMU_DUMMY_MAP(GetGfxOffEntryCount),		 \
+	__SMU_DUMMY_MAP(LogGfxOffResidency),
 
 #undef __SMU_DUMMY_MAP
 #define __SMU_DUMMY_MAP(type)	SMU_MSG_##type
diff --git a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
index f442bf085a31..9d62ea2af132 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/inc/smu_v13_0.h
@@ -32,6 +32,7 @@
 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_5 0x04
 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_0 0x30
 #define SMU13_DRIVER_IF_VERSION_SMU_V13_0_7 0x2C
+#define SMU13_DRIVER_IF_VERSION_SMU_V13_0_10 0x1D
 
 #define SMU13_MODE1_RESET_WAIT_TIME_IN_MS 500  //500ms
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
index 644ea150e075..74996a8fb671 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c
@@ -154,6 +154,7 @@ static struct cmn2asic_msg_mapping sienna_cichlid_message_map[SMU_MSG_MAX_COUNT]
 	MSG_MAP(SetGpoFeaturePMask,		PPSMC_MSG_SetGpoFeaturePMask,          0),
 	MSG_MAP(DisallowGpo,			PPSMC_MSG_DisallowGpo,                 0),
 	MSG_MAP(Enable2ndUSB20Port,		PPSMC_MSG_Enable2ndUSB20Port,          0),
+	MSG_MAP(DriverMode2Reset,		PPSMC_MSG_DriverMode2Reset,	       0),
 };
 
 static struct cmn2asic_mapping sienna_cichlid_clk_map[SMU_CLK_COUNT] = {
@@ -4265,6 +4266,57 @@ static int sienna_cichlid_stb_get_data_direct(struct smu_context *smu,
 	return 0;
 }
 
+static bool sienna_cichlid_is_mode2_reset_supported(struct smu_context *smu)
+{
+	return true;
+}
+
+static int sienna_cichlid_mode2_reset(struct smu_context *smu)
+{
+	u32 smu_version;
+	int ret = 0, index;
+	struct amdgpu_device *adev = smu->adev;
+	int timeout = 100;
+
+	smu_cmn_get_smc_version(smu, NULL, &smu_version);
+
+	index = smu_cmn_to_asic_specific_index(smu, CMN2ASIC_MAPPING_MSG,
+						SMU_MSG_DriverMode2Reset);
+
+	mutex_lock(&smu->message_lock);
+
+	ret = smu_cmn_send_msg_without_waiting(smu, (uint16_t)index,
+					       SMU_RESET_MODE_2);
+
+	ret = smu_cmn_wait_for_response(smu);
+	while (ret != 0 && timeout) {
+		ret = smu_cmn_wait_for_response(smu);
+		/* Wait a bit more time for getting ACK */
+		if (ret != 0) {
+			--timeout;
+			usleep_range(500, 1000);
+			continue;
+		} else {
+			break;
+		}
+	}
+
+	if (!timeout) {
+		dev_err(adev->dev,
+			"failed to send mode2 message \tparam: 0x%08x response %#x\n",
+			SMU_RESET_MODE_2, ret);
+		goto out;
+	}
+
+	dev_info(smu->adev->dev, "restore config space...\n");
+	/* Restore the config space saved during init */
+	amdgpu_device_load_pci_state(adev->pdev);
+out:
+	mutex_unlock(&smu->message_lock);
+
+	return ret;
+}
+
 static const struct pptable_funcs sienna_cichlid_ppt_funcs = {
 	.get_allowed_feature_mask = sienna_cichlid_get_allowed_feature_mask,
 	.set_default_dpm_table = sienna_cichlid_set_default_dpm_table,
@@ -4360,6 +4412,8 @@ static const struct pptable_funcs sienna_cichlid_ppt_funcs = {
 	.get_default_config_table_settings = sienna_cichlid_get_default_config_table_settings,
 	.set_config_table = sienna_cichlid_set_config_table,
 	.get_unique_id = sienna_cichlid_get_unique_id,
+	.mode2_reset_is_support = sienna_cichlid_is_mode2_reset_supported,
+	.mode2_reset = sienna_cichlid_mode2_reset,
 };
 
 void sienna_cichlid_set_ppt_funcs(struct smu_context *smu)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
index 89504ff8e9ed..cb10c7e31264 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c
@@ -138,6 +138,9 @@ static struct cmn2asic_msg_mapping vangogh_message_map[SMU_MSG_MAX_COUNT] = {
 	MSG_MAP(SetSlowPPTLimit,                    PPSMC_MSG_SetSlowPPTLimit,						0),
 	MSG_MAP(GetFastPPTLimit,                    PPSMC_MSG_GetFastPPTLimit,						0),
 	MSG_MAP(GetSlowPPTLimit,                    PPSMC_MSG_GetSlowPPTLimit,						0),
+	MSG_MAP(GetGfxOffStatus,		    PPSMC_MSG_GetGfxOffStatus,						0),
+	MSG_MAP(GetGfxOffEntryCount,		    PPSMC_MSG_GetGfxOffEntryCount,					0),
+	MSG_MAP(LogGfxOffResidency,		    PPSMC_MSG_LogGfxOffResidency,					0),
 };
 
 static struct cmn2asic_mapping vangogh_feature_mask_map[SMU_FEATURE_COUNT] = {
@@ -220,14 +223,13 @@ static int vangogh_tables_init(struct smu_context *smu)
 {
 	struct smu_table_context *smu_table = &smu->smu_table;
 	struct smu_table *tables = smu_table->tables;
-	struct amdgpu_device *adev = smu->adev;
 	uint32_t if_version;
+	uint32_t smu_version;
 	uint32_t ret = 0;
 
-	ret = smu_cmn_get_smc_version(smu, &if_version, NULL);
+	ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version);
 	if (ret) {
-		dev_err(adev->dev, "Failed to get smu if version!\n");
-		goto err0_out;
+		return ret;
 	}
 
 	SMU_TABLE_INIT(tables, SMU_TABLE_WATERMARKS, sizeof(Watermarks_t),
@@ -252,7 +254,10 @@ static int vangogh_tables_init(struct smu_context *smu)
 		goto err0_out;
 	smu_table->metrics_time = 0;
 
-	smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v2_2);
+	if (smu_version >= 0x043F3E00)
+		smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v2_3);
+	else
+		smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v2_2);
 	smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL);
 	if (!smu_table->gpu_metrics_table)
 		goto err1_out;
@@ -1645,6 +1650,63 @@ static int vangogh_set_watermarks_table(struct smu_context *smu,
 	return 0;
 }
 
+static ssize_t vangogh_get_legacy_gpu_metrics_v2_3(struct smu_context *smu,
+				      void **table)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct gpu_metrics_v2_3 *gpu_metrics =
+		(struct gpu_metrics_v2_3 *)smu_table->gpu_metrics_table;
+	SmuMetrics_legacy_t metrics;
+	int ret = 0;
+
+	ret = smu_cmn_get_metrics_table(smu, &metrics, true);
+	if (ret)
+		return ret;
+
+	smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 3);
+
+	gpu_metrics->temperature_gfx = metrics.GfxTemperature;
+	gpu_metrics->temperature_soc = metrics.SocTemperature;
+	memcpy(&gpu_metrics->temperature_core[0],
+		&metrics.CoreTemperature[0],
+		sizeof(uint16_t) * 4);
+	gpu_metrics->temperature_l3[0] = metrics.L3Temperature[0];
+
+	gpu_metrics->average_gfx_activity = metrics.GfxActivity;
+	gpu_metrics->average_mm_activity = metrics.UvdActivity;
+
+	gpu_metrics->average_socket_power = metrics.CurrentSocketPower;
+	gpu_metrics->average_cpu_power = metrics.Power[0];
+	gpu_metrics->average_soc_power = metrics.Power[1];
+	gpu_metrics->average_gfx_power = metrics.Power[2];
+	memcpy(&gpu_metrics->average_core_power[0],
+		&metrics.CorePower[0],
+		sizeof(uint16_t) * 4);
+
+	gpu_metrics->average_gfxclk_frequency = metrics.GfxclkFrequency;
+	gpu_metrics->average_socclk_frequency = metrics.SocclkFrequency;
+	gpu_metrics->average_uclk_frequency = metrics.MemclkFrequency;
+	gpu_metrics->average_fclk_frequency = metrics.MemclkFrequency;
+	gpu_metrics->average_vclk_frequency = metrics.VclkFrequency;
+	gpu_metrics->average_dclk_frequency = metrics.DclkFrequency;
+
+	memcpy(&gpu_metrics->current_coreclk[0],
+		&metrics.CoreFrequency[0],
+		sizeof(uint16_t) * 4);
+	gpu_metrics->current_l3clk[0] = metrics.L3Frequency[0];
+
+	gpu_metrics->throttle_status = metrics.ThrottlerStatus;
+	gpu_metrics->indep_throttle_status =
+			smu_cmn_get_indep_throttler_status(metrics.ThrottlerStatus,
+							   vangogh_throttler_map);
+
+	gpu_metrics->system_clock_counter = ktime_get_boottime_ns();
+
+	*table = (void *)gpu_metrics;
+
+	return sizeof(struct gpu_metrics_v2_3);
+}
+
 static ssize_t vangogh_get_legacy_gpu_metrics(struct smu_context *smu,
 				      void **table)
 {
@@ -1702,6 +1764,77 @@ static ssize_t vangogh_get_legacy_gpu_metrics(struct smu_context *smu,
 	return sizeof(struct gpu_metrics_v2_2);
 }
 
+static ssize_t vangogh_get_gpu_metrics_v2_3(struct smu_context *smu,
+				      void **table)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	struct gpu_metrics_v2_3 *gpu_metrics =
+		(struct gpu_metrics_v2_3 *)smu_table->gpu_metrics_table;
+	SmuMetrics_t metrics;
+	int ret = 0;
+
+	ret = smu_cmn_get_metrics_table(smu, &metrics, true);
+	if (ret)
+		return ret;
+
+	smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 3);
+
+	gpu_metrics->temperature_gfx = metrics.Current.GfxTemperature;
+	gpu_metrics->temperature_soc = metrics.Current.SocTemperature;
+	memcpy(&gpu_metrics->temperature_core[0],
+		&metrics.Current.CoreTemperature[0],
+		sizeof(uint16_t) * 4);
+	gpu_metrics->temperature_l3[0] = metrics.Current.L3Temperature[0];
+
+	gpu_metrics->average_temperature_gfx = metrics.Average.GfxTemperature;
+	gpu_metrics->average_temperature_soc = metrics.Average.SocTemperature;
+	memcpy(&gpu_metrics->average_temperature_core[0],
+		&metrics.Average.CoreTemperature[0],
+		sizeof(uint16_t) * 4);
+	gpu_metrics->average_temperature_l3[0] = metrics.Average.L3Temperature[0];
+
+	gpu_metrics->average_gfx_activity = metrics.Current.GfxActivity;
+	gpu_metrics->average_mm_activity = metrics.Current.UvdActivity;
+
+	gpu_metrics->average_socket_power = metrics.Current.CurrentSocketPower;
+	gpu_metrics->average_cpu_power = metrics.Current.Power[0];
+	gpu_metrics->average_soc_power = metrics.Current.Power[1];
+	gpu_metrics->average_gfx_power = metrics.Current.Power[2];
+	memcpy(&gpu_metrics->average_core_power[0],
+		&metrics.Average.CorePower[0],
+		sizeof(uint16_t) * 4);
+
+	gpu_metrics->average_gfxclk_frequency = metrics.Average.GfxclkFrequency;
+	gpu_metrics->average_socclk_frequency = metrics.Average.SocclkFrequency;
+	gpu_metrics->average_uclk_frequency = metrics.Average.MemclkFrequency;
+	gpu_metrics->average_fclk_frequency = metrics.Average.MemclkFrequency;
+	gpu_metrics->average_vclk_frequency = metrics.Average.VclkFrequency;
+	gpu_metrics->average_dclk_frequency = metrics.Average.DclkFrequency;
+
+	gpu_metrics->current_gfxclk = metrics.Current.GfxclkFrequency;
+	gpu_metrics->current_socclk = metrics.Current.SocclkFrequency;
+	gpu_metrics->current_uclk = metrics.Current.MemclkFrequency;
+	gpu_metrics->current_fclk = metrics.Current.MemclkFrequency;
+	gpu_metrics->current_vclk = metrics.Current.VclkFrequency;
+	gpu_metrics->current_dclk = metrics.Current.DclkFrequency;
+
+	memcpy(&gpu_metrics->current_coreclk[0],
+		&metrics.Current.CoreFrequency[0],
+		sizeof(uint16_t) * 4);
+	gpu_metrics->current_l3clk[0] = metrics.Current.L3Frequency[0];
+
+	gpu_metrics->throttle_status = metrics.Current.ThrottlerStatus;
+	gpu_metrics->indep_throttle_status =
+			smu_cmn_get_indep_throttler_status(metrics.Current.ThrottlerStatus,
+							   vangogh_throttler_map);
+
+	gpu_metrics->system_clock_counter = ktime_get_boottime_ns();
+
+	*table = (void *)gpu_metrics;
+
+	return sizeof(struct gpu_metrics_v2_3);
+}
+
 static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu,
 				      void **table)
 {
@@ -1769,20 +1902,26 @@ static ssize_t vangogh_get_gpu_metrics(struct smu_context *smu,
 static ssize_t vangogh_common_get_gpu_metrics(struct smu_context *smu,
 				      void **table)
 {
-	struct amdgpu_device *adev = smu->adev;
 	uint32_t if_version;
+	uint32_t smu_version;
 	int ret = 0;
 
-	ret = smu_cmn_get_smc_version(smu, &if_version, NULL);
+	ret = smu_cmn_get_smc_version(smu, &if_version, &smu_version);
 	if (ret) {
-		dev_err(adev->dev, "Failed to get smu if version!\n");
 		return ret;
 	}
 
-	if (if_version < 0x3)
-		ret = vangogh_get_legacy_gpu_metrics(smu, table);
-	else
-		ret = vangogh_get_gpu_metrics(smu, table);
+	if (smu_version >= 0x043F3E00) {
+		if (if_version < 0x3)
+			ret = vangogh_get_legacy_gpu_metrics_v2_3(smu, table);
+		else
+			ret = vangogh_get_gpu_metrics_v2_3(smu, table);
+	} else {
+		if (if_version < 0x3)
+			ret = vangogh_get_legacy_gpu_metrics(smu, table);
+		else
+			ret = vangogh_get_gpu_metrics(smu, table);
+	}
 
 	return ret;
 }
@@ -2200,6 +2339,76 @@ static int vangogh_set_power_limit(struct smu_context *smu,
 	return ret;
 }
 
+/**
+ * vangogh_set_gfxoff_residency
+ *
+ * @smu: amdgpu_device pointer
+ * @start: start/stop residency log
+ *
+ * This function will be used to log gfxoff residency
+ *
+ *
+ * Returns standard response codes.
+ */
+static u32 vangogh_set_gfxoff_residency(struct smu_context *smu, bool start)
+{
+	int ret = 0;
+	u32 residency;
+	struct amdgpu_device *adev = smu->adev;
+
+	if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
+		return 0;
+
+	ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_LogGfxOffResidency,
+					      start, &residency);
+
+	if (!start)
+		adev->gfx.gfx_off_residency = residency;
+
+	return ret;
+}
+
+/**
+ * vangogh_get_gfxoff_residency
+ *
+ * @smu: amdgpu_device pointer
+ *
+ * This function will be used to get gfxoff residency.
+ *
+ * Returns standard response codes.
+ */
+static u32 vangogh_get_gfxoff_residency(struct smu_context *smu, uint32_t *residency)
+{
+	struct amdgpu_device *adev = smu->adev;
+
+	*residency = adev->gfx.gfx_off_residency;
+
+	return 0;
+}
+
+/**
+ * vangogh_get_gfxoff_entrycount - get gfxoff entry count
+ *
+ * @smu: amdgpu_device pointer
+ *
+ * This function will be used to get gfxoff entry count
+ *
+ * Returns standard response codes.
+ */
+static u32 vangogh_get_gfxoff_entrycount(struct smu_context *smu, uint64_t *entrycount)
+{
+	int ret = 0, value = 0;
+	struct amdgpu_device *adev = smu->adev;
+
+	if (!(adev->pm.pp_feature & PP_GFXOFF_MASK))
+		return 0;
+
+	ret = smu_cmn_send_smc_msg(smu, SMU_MSG_GetGfxOffEntryCount, &value);
+	*entrycount = value + adev->gfx.gfx_off_entrycount;
+
+	return ret;
+}
+
 static const struct pptable_funcs vangogh_ppt_funcs = {
 
 	.check_fw_status = smu_v11_0_check_fw_status,
@@ -2237,6 +2446,9 @@ static const struct pptable_funcs vangogh_ppt_funcs = {
 	.mode2_reset = vangogh_mode2_reset,
 	.gfx_off_control = smu_v11_0_gfx_off_control,
 	.get_gfx_off_status = vangogh_get_gfxoff_status,
+	.get_gfx_off_entrycount = vangogh_get_gfxoff_entrycount,
+	.get_gfx_off_residency = vangogh_get_gfxoff_residency,
+	.set_gfx_off_residency = vangogh_set_gfxoff_residency,
 	.get_ppt_limit = vangogh_get_ppt_limit,
 	.get_power_limit = vangogh_get_power_limit,
 	.set_power_limit = vangogh_set_power_limit,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
index 93f9b8377539..93fffdbab4f0 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0.c
@@ -59,6 +59,7 @@
 MODULE_FIRMWARE("amdgpu/aldebaran_smc.bin");
 MODULE_FIRMWARE("amdgpu/smu_13_0_0.bin");
 MODULE_FIRMWARE("amdgpu/smu_13_0_7.bin");
+MODULE_FIRMWARE("amdgpu/smu_13_0_10.bin");
 
 #define mmMP1_SMN_C2PMSG_66                                                                            0x0282
 #define mmMP1_SMN_C2PMSG_66_BASE_IDX                                                                   0
@@ -303,6 +304,9 @@ int smu_v13_0_check_fw_version(struct smu_context *smu)
 	case IP_VERSION(13, 0, 5):
 		smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_5;
 		break;
+	case IP_VERSION(13, 0, 10):
+		smu->smc_driver_if_version = SMU13_DRIVER_IF_VERSION_SMU_V13_0_10;
+		break;
 	default:
 		dev_err(adev->dev, "smu unsupported IP version: 0x%x.\n",
 			adev->ip_versions[MP1_HWIP][0]);
@@ -451,6 +455,8 @@ int smu_v13_0_setup_pptable(struct smu_context *smu)
 	} else {
 		pptable_id = smu->smu_table.boot_values.pp_table_id;
 
+		if (adev->ip_versions[MP1_HWIP][0] == IP_VERSION(13, 0, 10))
+			pptable_id = 6666;
 	}
 
 	/* force using vbios pptable in sriov mode */
@@ -1062,6 +1068,9 @@ int smu_v13_0_enable_thermal_alert(struct smu_context *smu)
 {
 	int ret = 0;
 
+	if (!smu->irq_source.num_types)
+		return 0;
+
 	ret = amdgpu_irq_get(smu->adev, &smu->irq_source, 0);
 	if (ret)
 		return ret;
@@ -1071,6 +1080,9 @@ int smu_v13_0_enable_thermal_alert(struct smu_context *smu)
 
 int smu_v13_0_disable_thermal_alert(struct smu_context *smu)
 {
+	if (!smu->irq_source.num_types)
+		return 0;
+
 	return amdgpu_irq_put(smu->adev, &smu->irq_source, 0);
 }
 
@@ -1442,6 +1454,9 @@ int smu_v13_0_register_irq_handler(struct smu_context *smu)
 	struct amdgpu_irq_src *irq_src = &smu->irq_source;
 	int ret = 0;
 
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
 	irq_src->num_types = 1;
 	irq_src->funcs = &smu_v13_0_irq_funcs;
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
index 096327513dd0..1d454485e0d9 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_0_ppt.c
@@ -239,82 +239,47 @@ smu_v13_0_0_get_allowed_feature_mask(struct smu_context *smu,
 				  uint32_t *feature_mask, uint32_t num)
 {
 	struct amdgpu_device *adev = smu->adev;
+	u32 smu_version;
 
 	if (num > 2)
 		return -EINVAL;
 
-	memset(feature_mask, 0, sizeof(uint32_t) * num);
-
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_FW_DATA_READ_BIT);
+	memset(feature_mask, 0xff, sizeof(uint32_t) * num);
 
-	if (adev->pm.pp_feature & PP_SCLK_DPM_MASK) {
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT);
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_IMU_BIT);
+	if (!(adev->pm.pp_feature & PP_SCLK_DPM_MASK)) {
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_GFXCLK_BIT);
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFX_IMU_BIT);
 	}
 
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_MM_DPM_BIT);
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_VCN_BIT);
-
-	if ((adev->pg_flags & AMD_PG_SUPPORT_ATHUB) &&
-	    (adev->pg_flags & AMD_PG_SUPPORT_MMHUB))
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_ATHUB_MMHUB_PG_BIT);
+	if (!(adev->pg_flags & AMD_PG_SUPPORT_ATHUB) ||
+	    !(adev->pg_flags & AMD_PG_SUPPORT_MMHUB))
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_ATHUB_MMHUB_PG_BIT);
 
-	if (adev->pm.pp_feature & PP_SOCCLK_DPM_MASK)
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT);
-
-#if 0
-	if (adev->pm.pp_feature & PP_GFXOFF_MASK)
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFXOFF_BIT);
-#endif
+	if (!(adev->pm.pp_feature & PP_SOCCLK_DPM_MASK))
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_SOCCLK_BIT);
 
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_THROTTLERS_BIT);
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_FAN_CONTROL_BIT);
-
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DF_CSTATE_BIT);
-
-	if (adev->pm.pp_feature & PP_MCLK_DPM_MASK) {
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_UCLK_BIT);
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_VMEMP_SCALING_BIT);
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_VDDIO_MEM_SCALING_BIT);
+	/* PMFW 78.58 contains a critical fix for gfxoff feature */
+	smu_cmn_get_smc_version(smu, NULL, &smu_version);
+	if ((smu_version < 0x004e3a00) ||
+	     !(adev->pm.pp_feature & PP_GFXOFF_MASK))
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFXOFF_BIT);
+
+	if (!(adev->pm.pp_feature & PP_MCLK_DPM_MASK)) {
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_UCLK_BIT);
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VMEMP_SCALING_BIT);
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_VDDIO_MEM_SCALING_BIT);
 	}
 
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_MEM_TEMP_READ_BIT);
-
-	if (adev->pm.pp_feature & PP_SCLK_DEEP_SLEEP_MASK)
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_GFXCLK_BIT);
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_SOCCLK_BIT);
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_SOC_MPCLK_DS_BIT);
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_BACO_MPCLK_DS_BIT);
+	if (!(adev->pm.pp_feature & PP_SCLK_DEEP_SLEEP_MASK))
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_GFXCLK_BIT);
 
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_FCLK_BIT);
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_DCN_BIT);
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_DCFCLK_BIT);
-
-	if (adev->pm.pp_feature & PP_PCIE_DPM_MASK) {
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_LINK_BIT);
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_LCLK_BIT);
+	if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK)) {
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DPM_LINK_BIT);
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_DS_LCLK_BIT);
 	}
 
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_BACO_BIT);
-
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_MP0CLK_BIT);
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_FW_DSTATE_BIT);
-
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_OUT_OF_BAND_MONITOR_BIT);
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_SOC_CG_BIT);
-
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_FCLK_BIT);
-
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_FW_CTF_BIT);
-
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DS_UCLK_BIT);
-
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_VR0HOT_BIT);
-
-	*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_DPM_GFX_POWER_OPTIMIZER_BIT);
-
-	if (adev->pm.pp_feature & PP_ULV_MASK)
-		*(uint64_t *)feature_mask |= FEATURE_MASK(FEATURE_GFX_ULV_BIT);
+	if (!(adev->pm.pp_feature & PP_ULV_MASK))
+		*(uint64_t *)feature_mask &= ~FEATURE_MASK(FEATURE_GFX_ULV_BIT);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
index 15e4298c7cc8..e4f8f90ac5aa 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
@@ -969,6 +969,9 @@ void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev)
 	case METRICS_VERSION(2, 2):
 		structure_size = sizeof(struct gpu_metrics_v2_2);
 		break;
+	case METRICS_VERSION(2, 3):
+		structure_size = sizeof(struct gpu_metrics_v2_3);
+		break;
 	default:
 		return;
 	}
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_internal.h b/drivers/gpu/drm/amd/pm/swsmu/smu_internal.h
index 7469bbfce1fb..ceb13c838067 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_internal.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_internal.h
@@ -47,6 +47,9 @@
 #define smu_notify_memory_pool_location(smu)				smu_ppt_funcs(notify_memory_pool_location, 0, smu)
 #define smu_gfx_off_control(smu, enable)				smu_ppt_funcs(gfx_off_control, 0, smu, enable)
 #define smu_get_gfx_off_status(smu)						smu_ppt_funcs(get_gfx_off_status, 0, smu)
+#define smu_get_gfx_off_entrycount(smu, value)						smu_ppt_funcs(get_gfx_off_entrycount, 0, smu, value)
+#define smu_get_gfx_off_residency(smu, value)						smu_ppt_funcs(get_gfx_off_residency, 0, smu, value)
+#define smu_set_gfx_off_residency(smu, value)						smu_ppt_funcs(set_gfx_off_residency, 0, smu, value)
 #define smu_set_last_dcef_min_deep_sleep_clk(smu)			smu_ppt_funcs(set_last_dcef_min_deep_sleep_clk, 0, smu)
 #define smu_system_features_control(smu, en)				smu_ppt_funcs(system_features_control, 0, smu, en)
 #define smu_init_max_sustainable_clocks(smu)				smu_ppt_funcs(init_max_sustainable_clocks, 0, smu)
diff --git a/drivers/gpu/drm/arm/Kconfig b/drivers/gpu/drm/arm/Kconfig
index 6e3f1d600541..c1b89274d2a4 100644
--- a/drivers/gpu/drm/arm/Kconfig
+++ b/drivers/gpu/drm/arm/Kconfig
@@ -6,7 +6,7 @@ config DRM_HDLCD
 	depends on DRM && OF && (ARM || ARM64 || COMPILE_TEST)
 	depends on COMMON_CLK
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	help
 	  Choose this option if you have an ARM High Definition Colour LCD
 	  controller.
@@ -27,7 +27,7 @@ config DRM_MALI_DISPLAY
 	depends on DRM && OF && (ARM || ARM64 || COMPILE_TEST)
 	depends on COMMON_CLK
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select VIDEOMODE_HELPERS
 	help
 	  Choose this option if you want to compile the ARM Mali Display
diff --git a/drivers/gpu/drm/arm/display/Kconfig b/drivers/gpu/drm/arm/display/Kconfig
index e91598b60781..4acc4285a4eb 100644
--- a/drivers/gpu/drm/arm/display/Kconfig
+++ b/drivers/gpu/drm/arm/display/Kconfig
@@ -4,7 +4,7 @@ config DRM_KOMEDA
 	depends on DRM && OF
 	depends on COMMON_CLK
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select VIDEOMODE_HELPERS
 	help
 	  Choose this option if you want to compile the ARM Komeda display
diff --git a/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.c b/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.c
index daa1faccd3e7..6c56f5662bc7 100644
--- a/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.c
+++ b/drivers/gpu/drm/arm/display/komeda/d71/d71_dev.c
@@ -310,8 +310,7 @@ static int d71_reset(struct d71_dev *d71)
 	u32 __iomem *gcu = d71->gcu_addr;
 	int ret;
 
-	malidp_write32_mask(gcu, BLK_CONTROL,
-			    GCU_CONTROL_SRST, GCU_CONTROL_SRST);
+	malidp_write32(gcu, BLK_CONTROL, GCU_CONTROL_SRST);
 
 	ret = dp_wait_cond(!(malidp_read32(gcu, BLK_CONTROL) & GCU_CONTROL_SRST),
 			   100, 1000, 10000);
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
index 59172acb9738..4cc07d6bb9d8 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
@@ -11,7 +11,6 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
 
@@ -235,7 +234,7 @@ void komeda_crtc_handle_event(struct komeda_crtc   *kcrtc,
 			crtc->state->event = NULL;
 			drm_crtc_send_vblank_event(crtc, event);
 		} else {
-			DRM_WARN("CRTC[%d]: FLIP happen but no pending commit.\n",
+			DRM_WARN("CRTC[%d]: FLIP happened but no pending commit.\n",
 				 drm_crtc_index(&kcrtc->base));
 		}
 		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
@@ -286,7 +285,7 @@ komeda_crtc_atomic_enable(struct drm_crtc *crtc,
 	komeda_crtc_do_flush(crtc, old);
 }
 
-static void
+void
 komeda_crtc_flush_and_wait_for_flip_done(struct komeda_crtc *kcrtc,
 					 struct completion *input_flip_done)
 {
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_drv.c b/drivers/gpu/drm/arm/display/komeda/komeda_drv.c
index ba16895690f1..9fce4239d4ad 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_drv.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_drv.c
@@ -9,6 +9,7 @@
 #include <linux/platform_device.h>
 #include <linux/component.h>
 #include <linux/pm_runtime.h>
+#include <drm/drm_fb_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_of.h>
 #include "komeda_dev.h"
@@ -72,6 +73,7 @@ static int komeda_bind(struct device *dev)
 	}
 
 	dev_set_drvdata(dev, mdrv);
+	drm_fbdev_generic_setup(&mdrv->kms->base, 32);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c b/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c
index 3c372d2deb0a..df5da5a44755 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_framebuffer.c
@@ -5,9 +5,9 @@
  *
  */
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_gem.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 
 #include "komeda_framebuffer.h"
@@ -137,7 +137,7 @@ komeda_fb_none_afbc_size_check(struct komeda_dev *mdev, struct komeda_fb *kfb,
 		}
 
 		min_size = komeda_fb_get_pixel_addr(kfb, 0, fb->height, i)
-			 - to_drm_gem_cma_obj(obj)->paddr;
+			 - to_drm_gem_dma_obj(obj)->dma_addr;
 		if (obj->size < min_size) {
 			DRM_DEBUG_KMS("The fb->obj[%d] size: 0x%zx lower than the minimum requirement: 0x%llx.\n",
 				      i, obj->size, min_size);
@@ -239,7 +239,7 @@ dma_addr_t
 komeda_fb_get_pixel_addr(struct komeda_fb *kfb, int x, int y, int plane)
 {
 	struct drm_framebuffer *fb = &kfb->base;
-	const struct drm_gem_cma_object *obj;
+	const struct drm_gem_dma_object *obj;
 	u32 offset, plane_x, plane_y, block_w, block_sz;
 
 	if (plane >= fb->format->num_planes) {
@@ -247,7 +247,7 @@ komeda_fb_get_pixel_addr(struct komeda_fb *kfb, int x, int y, int plane)
 		return -EINVAL;
 	}
 
-	obj = drm_fb_cma_get_gem_obj(fb, plane);
+	obj = drm_fb_dma_get_gem_obj(fb, plane);
 
 	offset = fb->offsets[plane];
 	if (!fb->modifier) {
@@ -260,7 +260,7 @@ komeda_fb_get_pixel_addr(struct komeda_fb *kfb, int x, int y, int plane)
 			+ plane_y * fb->pitches[plane];
 	}
 
-	return obj->paddr + offset;
+	return obj->dma_addr + offset;
 }
 
 /* if the fb can be supported by a specific layer */
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_kms.c b/drivers/gpu/drm/arm/display/komeda/komeda_kms.c
index 93b7f09b96ca..451746ebbe71 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_kms.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_kms.c
@@ -11,7 +11,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_probe_helper.h>
@@ -21,9 +21,9 @@
 #include "komeda_framebuffer.h"
 #include "komeda_kms.h"
 
-DEFINE_DRM_GEM_CMA_FOPS(komeda_cma_fops);
+DEFINE_DRM_GEM_DMA_FOPS(komeda_cma_fops);
 
-static int komeda_gem_cma_dumb_create(struct drm_file *file,
+static int komeda_gem_dma_dumb_create(struct drm_file *file,
 				      struct drm_device *dev,
 				      struct drm_mode_create_dumb *args)
 {
@@ -32,7 +32,7 @@ static int komeda_gem_cma_dumb_create(struct drm_file *file,
 
 	args->pitch = ALIGN(pitch, mdev->chip.bus_width);
 
-	return drm_gem_cma_dumb_create_internal(file, dev, args);
+	return drm_gem_dma_dumb_create_internal(file, dev, args);
 }
 
 static irqreturn_t komeda_kms_irq_handler(int irq, void *data)
@@ -60,7 +60,7 @@ static irqreturn_t komeda_kms_irq_handler(int irq, void *data)
 static const struct drm_driver komeda_kms_driver = {
 	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.lastclose			= drm_fb_helper_lastclose,
-	DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(komeda_gem_cma_dumb_create),
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(komeda_gem_dma_dumb_create),
 	.fops = &komeda_cma_fops,
 	.name = "komeda",
 	.desc = "Arm Komeda Display Processor driver",
@@ -69,6 +69,25 @@ static const struct drm_driver komeda_kms_driver = {
 	.minor = 1,
 };
 
+static void komeda_kms_atomic_commit_hw_done(struct drm_atomic_state *state)
+{
+	struct drm_device *dev = state->dev;
+	struct komeda_kms_dev *kms = to_kdev(dev);
+	int i;
+
+	for (i = 0; i < kms->n_crtcs; i++) {
+		struct komeda_crtc *kcrtc = &kms->crtcs[i];
+
+		if (kcrtc->base.state->active) {
+			struct completion *flip_done = NULL;
+			if (kcrtc->base.state->event)
+				flip_done = kcrtc->base.state->event->base.completion;
+			komeda_crtc_flush_and_wait_for_flip_done(kcrtc, flip_done);
+		}
+	}
+	drm_atomic_helper_commit_hw_done(state);
+}
+
 static void komeda_kms_commit_tail(struct drm_atomic_state *old_state)
 {
 	struct drm_device *dev = old_state->dev;
@@ -81,7 +100,7 @@ static void komeda_kms_commit_tail(struct drm_atomic_state *old_state)
 
 	drm_atomic_helper_commit_modeset_enables(dev, old_state);
 
-	drm_atomic_helper_commit_hw_done(old_state);
+	komeda_kms_atomic_commit_hw_done(old_state);
 
 	drm_atomic_helper_wait_for_flip_done(dev, old_state);
 
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_kms.h b/drivers/gpu/drm/arm/display/komeda/komeda_kms.h
index 7889e380ab23..7339339ef6b8 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_kms.h
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_kms.h
@@ -183,6 +183,8 @@ void komeda_kms_cleanup_private_objs(struct komeda_kms_dev *kms);
 
 void komeda_crtc_handle_event(struct komeda_crtc   *kcrtc,
 			      struct komeda_events *evts);
+void komeda_crtc_flush_and_wait_for_flip_done(struct komeda_crtc *kcrtc,
+					      struct completion *input_flip_done);
 
 struct komeda_kms_dev *komeda_kms_attach(struct komeda_dev *mdev);
 void komeda_kms_detach(struct komeda_kms_dev *kms);
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c
index e672b9cffee3..3276a3e82c62 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c
@@ -1271,7 +1271,7 @@ int komeda_release_unclaimed_resources(struct komeda_pipeline *pipe,
 	return 0;
 }
 
-/* Since standalong disabled components must be disabled separately and in the
+/* Since standalone disabled components must be disabled separately and in the
  * last, So a complete disable operation may needs to call pipeline_disable
  * twice (two phase disabling).
  * Phase 1: disable the common components, flush it.
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_plane.c b/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
index dff22dec54b5..c20ff72f0ae5 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
@@ -7,7 +7,6 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 #include "komeda_dev.h"
 #include "komeda_kms.h"
diff --git a/drivers/gpu/drm/arm/hdlcd_crtc.c b/drivers/gpu/drm/arm/hdlcd_crtc.c
index afc9cd856501..7030339fa232 100644
--- a/drivers/gpu/drm/arm/hdlcd_crtc.c
+++ b/drivers/gpu/drm/arm/hdlcd_crtc.c
@@ -18,12 +18,11 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_of.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
@@ -252,8 +251,8 @@ static int hdlcd_plane_atomic_check(struct drm_plane *plane,
 			return -EINVAL;
 		return drm_atomic_helper_check_plane_state(new_plane_state,
 							   crtc_state,
-							   DRM_PLANE_HELPER_NO_SCALING,
-							   DRM_PLANE_HELPER_NO_SCALING,
+							   DRM_PLANE_NO_SCALING,
+							   DRM_PLANE_NO_SCALING,
 							   false, true);
 	}
 
@@ -274,7 +273,7 @@ static void hdlcd_plane_atomic_update(struct drm_plane *plane,
 		return;
 
 	dest_h = drm_rect_height(&new_plane_state->dst);
-	scanout_start = drm_fb_cma_get_gem_addr(fb, new_plane_state, 0);
+	scanout_start = drm_fb_dma_get_gem_addr(fb, new_plane_state, 0);
 
 	hdlcd = plane->dev->dev_private;
 	hdlcd_write(hdlcd, HDLCD_REG_FB_LINE_LENGTH, fb->pitches[0]);
diff --git a/drivers/gpu/drm/arm/hdlcd_drv.c b/drivers/gpu/drm/arm/hdlcd_drv.c
index e89ae0ec60eb..a032003c340c 100644
--- a/drivers/gpu/drm/arm/hdlcd_drv.c
+++ b/drivers/gpu/drm/arm/hdlcd_drv.c
@@ -21,13 +21,13 @@
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
 
+#include <drm/drm_aperture.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_debugfs.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper.h>
 #include <drm/drm_module.h>
@@ -40,8 +40,7 @@
 
 static irqreturn_t hdlcd_irq(int irq, void *arg)
 {
-	struct drm_device *drm = arg;
-	struct hdlcd_drm_private *hdlcd = drm->dev_private;
+	struct hdlcd_drm_private *hdlcd = arg;
 	unsigned long irq_status;
 
 	irq_status = hdlcd_read(hdlcd, HDLCD_REG_INT_STATUS);
@@ -69,61 +68,32 @@ static irqreturn_t hdlcd_irq(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
-static void hdlcd_irq_preinstall(struct drm_device *drm)
+static int hdlcd_irq_install(struct hdlcd_drm_private *hdlcd)
 {
-	struct hdlcd_drm_private *hdlcd = drm->dev_private;
+	int ret;
+
 	/* Ensure interrupts are disabled */
 	hdlcd_write(hdlcd, HDLCD_REG_INT_MASK, 0);
 	hdlcd_write(hdlcd, HDLCD_REG_INT_CLEAR, ~0);
-}
-
-static void hdlcd_irq_postinstall(struct drm_device *drm)
-{
-#ifdef CONFIG_DEBUG_FS
-	struct hdlcd_drm_private *hdlcd = drm->dev_private;
-	unsigned long irq_mask = hdlcd_read(hdlcd, HDLCD_REG_INT_MASK);
-
-	/* enable debug interrupts */
-	irq_mask |= HDLCD_DEBUG_INT_MASK;
-
-	hdlcd_write(hdlcd, HDLCD_REG_INT_MASK, irq_mask);
-#endif
-}
-
-static int hdlcd_irq_install(struct drm_device *drm, int irq)
-{
-	int ret;
-
-	if (irq == IRQ_NOTCONNECTED)
-		return -ENOTCONN;
-
-	hdlcd_irq_preinstall(drm);
 
-	ret = request_irq(irq, hdlcd_irq, 0, drm->driver->name, drm);
+	ret = request_irq(hdlcd->irq, hdlcd_irq, 0, "hdlcd", hdlcd);
 	if (ret)
 		return ret;
 
-	hdlcd_irq_postinstall(drm);
+#ifdef CONFIG_DEBUG_FS
+	/* enable debug interrupts */
+	hdlcd_write(hdlcd, HDLCD_REG_INT_MASK, HDLCD_DEBUG_INT_MASK);
+#endif
 
 	return 0;
 }
 
-static void hdlcd_irq_uninstall(struct drm_device *drm)
+static void hdlcd_irq_uninstall(struct hdlcd_drm_private *hdlcd)
 {
-	struct hdlcd_drm_private *hdlcd = drm->dev_private;
 	/* disable all the interrupts that we might have enabled */
-	unsigned long irq_mask = hdlcd_read(hdlcd, HDLCD_REG_INT_MASK);
-
-#ifdef CONFIG_DEBUG_FS
-	/* disable debug interrupts */
-	irq_mask &= ~HDLCD_DEBUG_INT_MASK;
-#endif
-
-	/* disable vsync interrupts */
-	irq_mask &= ~HDLCD_INTERRUPT_VSYNC;
-	hdlcd_write(hdlcd, HDLCD_REG_INT_MASK, irq_mask);
+	hdlcd_write(hdlcd, HDLCD_REG_INT_MASK, 0);
 
-	free_irq(hdlcd->irq, drm);
+	free_irq(hdlcd->irq, hdlcd);
 }
 
 static int hdlcd_load(struct drm_device *drm, unsigned long flags)
@@ -183,7 +153,7 @@ static int hdlcd_load(struct drm_device *drm, unsigned long flags)
 		goto irq_fail;
 	hdlcd->irq = ret;
 
-	ret = hdlcd_irq_install(drm, hdlcd->irq);
+	ret = hdlcd_irq_install(hdlcd);
 	if (ret < 0) {
 		DRM_ERROR("failed to install IRQ handler\n");
 		goto irq_fail;
@@ -255,11 +225,11 @@ static void hdlcd_debugfs_init(struct drm_minor *minor)
 }
 #endif
 
-DEFINE_DRM_GEM_CMA_FOPS(fops);
+DEFINE_DRM_GEM_DMA_FOPS(fops);
 
 static const struct drm_driver hdlcd_driver = {
 	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init = hdlcd_debugfs_init,
 #endif
@@ -314,6 +284,15 @@ static int hdlcd_drm_bind(struct device *dev)
 		goto err_vblank;
 	}
 
+	/*
+	 * If EFI left us running, take over from simple framebuffer
+	 * drivers. Read HDLCD_REG_COMMAND to see if we are enabled.
+	 */
+	if (hdlcd_read(hdlcd, HDLCD_REG_COMMAND)) {
+		hdlcd_write(hdlcd, HDLCD_REG_COMMAND, 0);
+		drm_aperture_remove_framebuffers(false, &hdlcd_driver);
+	}
+
 	drm_mode_config_reset(drm);
 	drm_kms_helper_poll_init(drm);
 
@@ -335,7 +314,7 @@ err_pm_active:
 err_unload:
 	of_node_put(hdlcd->crtc.port);
 	hdlcd->crtc.port = NULL;
-	hdlcd_irq_uninstall(drm);
+	hdlcd_irq_uninstall(hdlcd);
 	of_reserved_mem_device_release(drm->dev);
 err_free:
 	drm_mode_config_cleanup(drm);
@@ -357,7 +336,7 @@ static void hdlcd_drm_unbind(struct device *dev)
 	hdlcd->crtc.port = NULL;
 	pm_runtime_get_sync(dev);
 	drm_atomic_helper_shutdown(drm);
-	hdlcd_irq_uninstall(drm);
+	hdlcd_irq_uninstall(hdlcd);
 	pm_runtime_put(dev);
 	if (pm_runtime_enabled(dev))
 		pm_runtime_disable(dev);
diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c
index d5aef21426cf..1d0b0c54ccc7 100644
--- a/drivers/gpu/drm/arm/malidp_drv.c
+++ b/drivers/gpu/drm/arm/malidp_drv.c
@@ -19,10 +19,9 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper.h>
 #include <drm/drm_module.h>
@@ -457,7 +456,7 @@ static int malidp_irq_init(struct platform_device *pdev)
 	return 0;
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(fops);
+DEFINE_DRM_GEM_DMA_FOPS(fops);
 
 static int malidp_dumb_create(struct drm_file *file_priv,
 			      struct drm_device *drm,
@@ -469,7 +468,7 @@ static int malidp_dumb_create(struct drm_file *file_priv,
 
 	args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), alignment);
 
-	return drm_gem_cma_dumb_create_internal(file_priv, drm, args);
+	return drm_gem_dma_dumb_create_internal(file_priv, drm, args);
 }
 
 #ifdef CONFIG_DEBUG_FS
@@ -566,7 +565,7 @@ static void malidp_debugfs_init(struct drm_minor *minor)
 
 static const struct drm_driver malidp_driver = {
 	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
-	DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(malidp_dumb_create),
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(malidp_dumb_create),
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init = malidp_debugfs_init,
 #endif
diff --git a/drivers/gpu/drm/arm/malidp_mw.c b/drivers/gpu/drm/arm/malidp_mw.c
index b66ca5b33a7f..ef76d0e6ee2f 100644
--- a/drivers/gpu/drm/arm/malidp_mw.c
+++ b/drivers/gpu/drm/arm/malidp_mw.c
@@ -10,10 +10,10 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_writeback.h>
 
@@ -160,7 +160,7 @@ malidp_mw_encoder_atomic_check(struct drm_encoder *encoder,
 
 	n_planes = fb->format->num_planes;
 	for (i = 0; i < n_planes; i++) {
-		struct drm_gem_cma_object *obj = drm_fb_cma_get_gem_obj(fb, i);
+		struct drm_gem_dma_object *obj = drm_fb_dma_get_gem_obj(fb, i);
 		/* memory write buffers are never rotated */
 		u8 alignment = malidp_hw_get_pitch_align(malidp->dev, 0);
 
@@ -170,7 +170,7 @@ malidp_mw_encoder_atomic_check(struct drm_encoder *encoder,
 			return -EINVAL;
 		}
 		mw_state->pitches[i] = fb->pitches[i];
-		mw_state->addrs[i] = obj->paddr + fb->offsets[i];
+		mw_state->addrs[i] = obj->dma_addr + fb->offsets[i];
 	}
 	mw_state->n_planes = n_planes;
 
diff --git a/drivers/gpu/drm/arm/malidp_planes.c b/drivers/gpu/drm/arm/malidp_planes.c
index 8a9562642d16..45f5e35e7f24 100644
--- a/drivers/gpu/drm/arm/malidp_planes.c
+++ b/drivers/gpu/drm/arm/malidp_planes.c
@@ -13,12 +13,11 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 
 #include "malidp_hw.h"
@@ -334,15 +333,15 @@ static bool malidp_check_pages_threshold(struct malidp_plane_state *ms,
 
 	for (i = 0; i < ms->n_planes; i++) {
 		struct drm_gem_object *obj;
-		struct drm_gem_cma_object *cma_obj;
+		struct drm_gem_dma_object *dma_obj;
 		struct sg_table *sgt;
 		struct scatterlist *sgl;
 
 		obj = drm_gem_fb_get_obj(ms->base.fb, i);
-		cma_obj = to_drm_gem_cma_obj(obj);
+		dma_obj = to_drm_gem_dma_obj(obj);
 
-		if (cma_obj->sgt)
-			sgt = cma_obj->sgt;
+		if (dma_obj->sgt)
+			sgt = dma_obj->sgt;
 		else
 			sgt = obj->funcs->get_sg_table(obj);
 
@@ -353,14 +352,14 @@ static bool malidp_check_pages_threshold(struct malidp_plane_state *ms,
 
 		while (sgl) {
 			if (sgl->length < pgsize) {
-				if (!cma_obj->sgt)
+				if (!dma_obj->sgt)
 					kfree(sgt);
 				return false;
 			}
 
 			sgl = sg_next(sgl);
 		}
-		if (!cma_obj->sgt)
+		if (!dma_obj->sgt)
 			kfree(sgt);
 	}
 
@@ -715,7 +714,7 @@ static void malidp_set_plane_base_addr(struct drm_framebuffer *fb,
 				       struct malidp_plane *mp,
 				       int plane_index)
 {
-	dma_addr_t paddr;
+	dma_addr_t dma_addr;
 	u16 ptr;
 	struct drm_plane *plane = &mp->base;
 	bool afbc = fb->modifier ? true : false;
@@ -723,27 +722,27 @@ static void malidp_set_plane_base_addr(struct drm_framebuffer *fb,
 	ptr = mp->layer->ptr + (plane_index << 4);
 
 	/*
-	 * drm_fb_cma_get_gem_addr() alters the physical base address of the
+	 * drm_fb_dma_get_gem_addr() alters the physical base address of the
 	 * framebuffer as per the plane's src_x, src_y co-ordinates (ie to
 	 * take care of source cropping).
 	 * For AFBC, this is not needed as the cropping is handled by _AD_CROP_H
 	 * and _AD_CROP_V registers.
 	 */
 	if (!afbc) {
-		paddr = drm_fb_cma_get_gem_addr(fb, plane->state,
-						plane_index);
+		dma_addr = drm_fb_dma_get_gem_addr(fb, plane->state,
+						   plane_index);
 	} else {
-		struct drm_gem_cma_object *obj;
+		struct drm_gem_dma_object *obj;
 
-		obj = drm_fb_cma_get_gem_obj(fb, plane_index);
+		obj = drm_fb_dma_get_gem_obj(fb, plane_index);
 
 		if (WARN_ON(!obj))
 			return;
-		paddr = obj->paddr;
+		dma_addr = obj->dma_addr;
 	}
 
-	malidp_hw_write(mp->hwdev, lower_32_bits(paddr), ptr);
-	malidp_hw_write(mp->hwdev, upper_32_bits(paddr), ptr + 4);
+	malidp_hw_write(mp->hwdev, lower_32_bits(dma_addr), ptr);
+	malidp_hw_write(mp->hwdev, upper_32_bits(dma_addr), ptr + 4);
 }
 
 static void malidp_de_set_plane_afbc(struct drm_plane *plane)
diff --git a/drivers/gpu/drm/arm/malidp_regs.h b/drivers/gpu/drm/arm/malidp_regs.h
index 514c50dcb74d..3bc16db70ddb 100644
--- a/drivers/gpu/drm/arm/malidp_regs.h
+++ b/drivers/gpu/drm/arm/malidp_regs.h
@@ -145,7 +145,7 @@
 #define     MALIDP_SE_COEFFTAB_DATA_MASK	0x3fff
 #define     MALIDP_SE_SET_COEFFTAB_DATA(x) \
 		((x) & MALIDP_SE_COEFFTAB_DATA_MASK)
-/* Enhance coeffents reigster offset */
+/* Enhance coefficients register offset */
 #define MALIDP_SE_IMAGE_ENH			0x3C
 /* ENH_LIMITS offset 0x0 */
 #define     MALIDP_SE_ENH_LOW_LEVEL		24
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index b7bb90ae787f..15dd667aa2e7 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -12,7 +12,6 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
diff --git a/drivers/gpu/drm/armada/armada_gem.c b/drivers/gpu/drm/armada/armada_gem.c
index 147abf1a3968..5430265ad458 100644
--- a/drivers/gpu/drm/armada/armada_gem.c
+++ b/drivers/gpu/drm/armada/armada_gem.c
@@ -107,11 +107,11 @@ armada_gem_linear_back(struct drm_device *dev, struct armada_gem_object *obj)
 	}
 
 	/*
-	 * We could grab something from CMA if it's enabled, but that
+	 * We could grab something from DMA if it's enabled, but that
 	 * involves building in a problem:
 	 *
-	 * CMA's interface uses dma_alloc_coherent(), which provides us
-	 * with an CPU virtual address and a device address.
+	 * GEM DMA helper interface uses dma_alloc_coherent(), which provides
+	 * us with an CPU virtual address and a device address.
 	 *
 	 * The CPU virtual address may be either an address in the kernel
 	 * direct mapped region (for example, as it would be on x86) or
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
index 424250535fed..f21eb8fb76d8 100644
--- a/drivers/gpu/drm/armada/armada_overlay.c
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -298,12 +298,6 @@ fail:
 	return ret;
 }
 
-static void armada_ovl_plane_destroy(struct drm_plane *plane)
-{
-	drm_plane_cleanup(plane);
-	kfree(plane);
-}
-
 static void armada_overlay_reset(struct drm_plane *plane)
 {
 	struct armada_overlay_state *state;
@@ -468,7 +462,7 @@ static int armada_overlay_get_property(struct drm_plane *plane,
 static const struct drm_plane_funcs armada_ovl_plane_funcs = {
 	.update_plane	= armada_overlay_plane_update,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.destroy	= armada_ovl_plane_destroy,
+	.destroy	= drm_plane_helper_destroy,
 	.reset		= armada_overlay_reset,
 	.atomic_duplicate_state = armada_overlay_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
diff --git a/drivers/gpu/drm/armada/armada_plane.c b/drivers/gpu/drm/armada/armada_plane.c
index 959d7f0a5108..cc47c032dbc1 100644
--- a/drivers/gpu/drm/armada/armada_plane.c
+++ b/drivers/gpu/drm/armada/armada_plane.c
@@ -288,7 +288,7 @@ struct drm_plane_state *armada_plane_duplicate_state(struct drm_plane *plane)
 static const struct drm_plane_funcs armada_primary_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.destroy	= drm_primary_helper_destroy,
+	.destroy	= drm_plane_helper_destroy,
 	.reset		= armada_plane_reset,
 	.atomic_duplicate_state = armada_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
diff --git a/drivers/gpu/drm/aspeed/Kconfig b/drivers/gpu/drm/aspeed/Kconfig
index 024ccab14f88..8137c39b057b 100644
--- a/drivers/gpu/drm/aspeed/Kconfig
+++ b/drivers/gpu/drm/aspeed/Kconfig
@@ -5,7 +5,7 @@ config DRM_ASPEED_GFX
 	depends on (COMPILE_TEST || ARCH_ASPEED)
 	depends on MMU
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DMA_CMA if HAVE_DMA_CONTIGUOUS
 	select CMA if HAVE_DMA_CONTIGUOUS
 	select MFD_SYSCON
diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c b/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c
index f3788d7d82d6..55a3444a51d8 100644
--- a/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c
+++ b/drivers/gpu/drm/aspeed/aspeed_gfx_crtc.c
@@ -7,11 +7,11 @@
 
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_vblank.h>
@@ -168,7 +168,7 @@ static void aspeed_gfx_pipe_update(struct drm_simple_display_pipe *pipe,
 	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_framebuffer *fb = pipe->plane.state->fb;
 	struct drm_pending_vblank_event *event;
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 
 	spin_lock_irq(&crtc->dev->event_lock);
 	event = crtc->state->event;
@@ -185,10 +185,10 @@ static void aspeed_gfx_pipe_update(struct drm_simple_display_pipe *pipe,
 	if (!fb)
 		return;
 
-	gem = drm_fb_cma_get_gem_obj(fb, 0);
+	gem = drm_fb_dma_get_gem_obj(fb, 0);
 	if (!gem)
 		return;
-	writel(gem->paddr, priv->base + CRT_ADDR);
+	writel(gem->dma_addr, priv->base + CRT_ADDR);
 }
 
 static int aspeed_gfx_enable_vblank(struct drm_simple_display_pipe *pipe)
diff --git a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
index 7780b72de9e8..a94f1a9e8f40 100644
--- a/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
+++ b/drivers/gpu/drm/aspeed/aspeed_gfx_drv.c
@@ -16,9 +16,8 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_probe_helper.h>
@@ -246,11 +245,11 @@ static void aspeed_gfx_unload(struct drm_device *drm)
 	drm_kms_helper_poll_fini(drm);
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(fops);
+DEFINE_DRM_GEM_DMA_FOPS(fops);
 
 static const struct drm_driver aspeed_gfx_driver = {
 	.driver_features        = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 	.fops = &fops,
 	.name = "aspeed-gfx-drm",
 	.desc = "ASPEED GFX DRM",
diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c
index 760b27971557..b9392f31e629 100644
--- a/drivers/gpu/drm/ast/ast_drv.c
+++ b/drivers/gpu/drm/ast/ast_drv.c
@@ -39,7 +39,7 @@
 
 #include "ast_drv.h"
 
-int ast_modeset = -1;
+static int ast_modeset = -1;
 
 MODULE_PARM_DESC(modeset, "Disable/Enable modesetting");
 module_param_named(modeset, ast_modeset, int, 0400);
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index 214b10178454..1bc0220e6783 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -42,7 +42,6 @@
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_gem_vram_helper.h>
 #include <drm/drm_managed.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
 
@@ -114,6 +113,9 @@ static bool ast_get_vbios_mode_info(const struct drm_format_info *format,
 	case 1024:
 		vbios_mode->enh_table = &res_1024x768[refresh_rate_index];
 		break;
+	case 1152:
+		vbios_mode->enh_table = &res_1152x864[refresh_rate_index];
+		break;
 	case 1280:
 		if (mode->crtc_vdisplay == 800)
 			vbios_mode->enh_table = &res_1280x800[refresh_rate_index];
@@ -311,7 +313,7 @@ static void ast_set_crtc_reg(struct ast_private *ast,
 	u8 jreg05 = 0, jreg07 = 0, jreg09 = 0, jregAC = 0, jregAD = 0, jregAE = 0;
 	u16 temp, precache = 0;
 
-	if ((ast->chip == AST2500) &&
+	if ((ast->chip == AST2500 || ast->chip == AST2600) &&
 	    (vbios_mode->enh_table->flags & AST2500PreCatchCRT))
 		precache = 40;
 
@@ -352,6 +354,12 @@ static void ast_set_crtc_reg(struct ast_private *ast,
 	ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0xAC, 0x00, jregAC);
 	ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0xAD, 0x00, jregAD);
 
+	// Workaround for HSync Time non octave pixels (1920x1080@60Hz HSync 44 pixels);
+	if ((ast->chip == AST2600) && (mode->crtc_vdisplay == 1080))
+		ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0xFC, 0xFD, 0x02);
+	else
+		ast_set_index_reg_mask(ast, AST_IO_CRTC_PORT, 0xFC, 0xFD, 0x00);
+
 	/* vert timings */
 	temp = (mode->crtc_vtotal) - 2;
 	if (temp & 0x100)
@@ -429,7 +437,7 @@ static void ast_set_dclk_reg(struct ast_private *ast,
 {
 	const struct ast_vbios_dclk_info *clk_info;
 
-	if (ast->chip == AST2500)
+	if ((ast->chip == AST2500) || (ast->chip == AST2600))
 		clk_info = &dclk_table_ast2500[vbios_mode->enh_table->dclk_index];
 	else
 		clk_info = &dclk_table[vbios_mode->enh_table->dclk_index];
@@ -555,8 +563,8 @@ static int ast_primary_plane_helper_atomic_check(struct drm_plane *plane,
 						   new_plane_state->crtc);
 
 	ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, true);
 	if (ret)
 		return ret;
@@ -779,8 +787,8 @@ static int ast_cursor_plane_helper_atomic_check(struct drm_plane *plane,
 						   new_plane_state->crtc);
 
 	ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  true, true);
 	if (ret)
 		return ret;
@@ -1058,6 +1066,8 @@ ast_crtc_helper_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode
 			return MODE_OK;
 		if ((mode->hdisplay == 1600) && (mode->vdisplay == 900))
 			return MODE_OK;
+		if ((mode->hdisplay == 1152) && (mode->vdisplay == 864))
+			return MODE_OK;
 
 		if ((ast->chip == AST2100) || (ast->chip == AST2200) ||
 		    (ast->chip == AST2300) || (ast->chip == AST2400) ||
@@ -1090,6 +1100,10 @@ ast_crtc_helper_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode
 		if (mode->vdisplay == 768)
 			status = MODE_OK;
 		break;
+	case 1152:
+		if (mode->vdisplay == 864)
+			status = MODE_OK;
+		break;
 	case 1280:
 		if (mode->vdisplay == 1024)
 			status = MODE_OK;
diff --git a/drivers/gpu/drm/ast/ast_tables.h b/drivers/gpu/drm/ast/ast_tables.h
index dbe1cc620f6e..0378c9bc079b 100644
--- a/drivers/gpu/drm/ast/ast_tables.h
+++ b/drivers/gpu/drm/ast/ast_tables.h
@@ -272,6 +272,13 @@ static const struct ast_vbios_enhtable res_1600x1200[] = {
 	 (SyncPP | Charx8Dot), 0xFF, 1, 0x33 },
 };
 
+static const struct ast_vbios_enhtable res_1152x864[] = {
+	{1600, 1152, 64, 128,  900,  864, 1, 3, VCLK108,	/* 75Hz */
+	 (SyncPP | Charx8Dot | NewModeInfo), 75, 1, 0x3B },
+	{1600, 1152, 64, 128,  900,  864, 1, 3, VCLK108,	/* end */
+	 (SyncPP | Charx8Dot | NewModeInfo), 0xFF, 1, 0x3B },
+};
+
 /* 16:9 */
 static const struct ast_vbios_enhtable res_1360x768[] = {
 	{1792, 1360, 64, 112, 795, 768, 3, 6, VCLK85_5,		/* 60Hz */
diff --git a/drivers/gpu/drm/atmel-hlcdc/Kconfig b/drivers/gpu/drm/atmel-hlcdc/Kconfig
index 8ae679f1a518..3bdbab3a6333 100644
--- a/drivers/gpu/drm/atmel-hlcdc/Kconfig
+++ b/drivers/gpu/drm/atmel-hlcdc/Kconfig
@@ -2,7 +2,7 @@
 config DRM_ATMEL_HLCDC
 	tristate "DRM Support for ATMEL HLCDC Display Controller"
 	depends on DRM && OF && COMMON_CLK && MFD_ATMEL_HLCDC && ARM
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_KMS_HELPER
 	select DRM_PANEL
 	help
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
index 651e3c109360..f7e7f4e919c7 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
@@ -20,7 +20,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_probe_helper.h>
@@ -730,11 +730,11 @@ static void atmel_hlcdc_dc_unload(struct drm_device *dev)
 	clk_disable_unprepare(dc->hlcdc->periph_clk);
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(fops);
+DEFINE_DRM_GEM_DMA_FOPS(fops);
 
 static const struct drm_driver atmel_hlcdc_dc_driver = {
 	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 	.fops = &fops,
 	.name = "atmel-hlcdc",
 	.desc = "Atmel HLCD Controller DRM",
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
index 2306ceb3e999..daa508504f47 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
@@ -12,11 +12,10 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 
 #include "atmel_hlcdc_dc.h"
 
@@ -449,9 +448,9 @@ static void atmel_hlcdc_plane_update_buffers(struct atmel_hlcdc_plane *plane,
 	sr = atmel_hlcdc_layer_read_reg(&plane->layer, ATMEL_HLCDC_LAYER_CHSR);
 
 	for (i = 0; i < state->nplanes; i++) {
-		struct drm_gem_cma_object *gem = drm_fb_cma_get_gem_obj(fb, i);
+		struct drm_gem_dma_object *gem = drm_fb_dma_get_gem_obj(fb, i);
 
-		state->dscrs[i]->addr = gem->paddr + state->offsets[i];
+		state->dscrs[i]->addr = gem->dma_addr + state->offsets[i];
 
 		atmel_hlcdc_layer_write_reg(&plane->layer,
 					    ATMEL_HLCDC_LAYER_PLANE_HEAD(i),
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511.h b/drivers/gpu/drm/bridge/adv7511/adv7511.h
index a031a0cd1f18..94de73cbeb2d 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511.h
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511.h
@@ -394,10 +394,7 @@ void adv7511_cec_irq_process(struct adv7511 *adv7511, unsigned int irq1);
 #else
 static inline int adv7511_cec_init(struct device *dev, struct adv7511 *adv7511)
 {
-	unsigned int offset = adv7511->type == ADV7533 ?
-						ADV7533_REG_CEC_OFFSET : 0;
-
-	regmap_write(adv7511->regmap, ADV7511_REG_CEC_CTRL + offset,
+	regmap_write(adv7511->regmap, ADV7511_REG_CEC_CTRL,
 		     ADV7511_CEC_CTRL_POWER_DOWN);
 	return 0;
 }
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_cec.c b/drivers/gpu/drm/bridge/adv7511/adv7511_cec.c
index 0b266f28f150..99964f5a5457 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511_cec.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_cec.c
@@ -359,7 +359,7 @@ int adv7511_cec_init(struct device *dev, struct adv7511 *adv7511)
 		goto err_cec_alloc;
 	}
 
-	regmap_write(adv7511->regmap, ADV7511_REG_CEC_CTRL + offset, 0);
+	regmap_write(adv7511->regmap, ADV7511_REG_CEC_CTRL, 0);
 	/* cec soft reset */
 	regmap_write(adv7511->regmap_cec,
 		     ADV7511_REG_CEC_SOFT_RESET + offset, 0x01);
@@ -386,7 +386,7 @@ err_cec_alloc:
 	dev_info(dev, "Initializing CEC failed with error %d, disabling CEC\n",
 		 ret);
 err_cec_parse_dt:
-	regmap_write(adv7511->regmap, ADV7511_REG_CEC_CTRL + offset,
+	regmap_write(adv7511->regmap, ADV7511_REG_CEC_CTRL,
 		     ADV7511_CEC_CTRL_POWER_DOWN);
 	return ret == -EPROBE_DEFER ? ret : 0;
 }
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
index 38bf28720f3a..f887200e8abc 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
@@ -1336,13 +1336,10 @@ uninit_regulators:
 	return ret;
 }
 
-static int adv7511_remove(struct i2c_client *i2c)
+static void adv7511_remove(struct i2c_client *i2c)
 {
 	struct adv7511 *adv7511 = i2c_get_clientdata(i2c);
 
-	i2c_unregister_device(adv7511->i2c_cec);
-	clk_disable_unprepare(adv7511->cec_clk);
-
 	adv7511_uninit_regulators(adv7511);
 
 	drm_bridge_remove(&adv7511->bridge);
@@ -1350,11 +1347,11 @@ static int adv7511_remove(struct i2c_client *i2c)
 	adv7511_audio_exit(adv7511);
 
 	cec_unregister_adapter(adv7511->cec_adap);
+	i2c_unregister_device(adv7511->i2c_cec);
+	clk_disable_unprepare(adv7511->cec_clk);
 
 	i2c_unregister_device(adv7511->i2c_packet);
 	i2c_unregister_device(adv7511->i2c_edid);
-
-	return 0;
 }
 
 static const struct i2c_device_id adv7511_i2c_ids[] = {
diff --git a/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c b/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c
index ae3d6e9a606c..660a54857929 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix-anx6345.c
@@ -787,7 +787,7 @@ err_unregister_i2c:
 	return err;
 }
 
-static int anx6345_i2c_remove(struct i2c_client *client)
+static void anx6345_i2c_remove(struct i2c_client *client)
 {
 	struct anx6345 *anx6345 = i2c_get_clientdata(client);
 
@@ -798,8 +798,6 @@ static int anx6345_i2c_remove(struct i2c_client *client)
 	kfree(anx6345->edid);
 
 	mutex_destroy(&anx6345->lock);
-
-	return 0;
 }
 
 static const struct i2c_device_id anx6345_id[] = {
diff --git a/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c b/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c
index d2fc8676fab6..5997049fde5b 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix-anx78xx.c
@@ -1357,7 +1357,7 @@ err_unregister_i2c:
 	return err;
 }
 
-static int anx78xx_i2c_remove(struct i2c_client *client)
+static void anx78xx_i2c_remove(struct i2c_client *client)
 {
 	struct anx78xx *anx78xx = i2c_get_clientdata(client);
 
@@ -1366,8 +1366,6 @@ static int anx78xx_i2c_remove(struct i2c_client *client)
 	unregister_i2c_dummy_clients(anx78xx);
 
 	kfree(anx78xx->edid);
-
-	return 0;
 }
 
 static const struct i2c_device_id anx78xx_id[] = {
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
index 8aadcc0aa90b..df9370e0ff23 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
@@ -1864,12 +1864,6 @@ EXPORT_SYMBOL_GPL(analogix_dp_remove);
 int analogix_dp_suspend(struct analogix_dp_device *dp)
 {
 	clk_disable_unprepare(dp->clock);
-
-	if (dp->plat_data->panel) {
-		if (drm_panel_unprepare(dp->plat_data->panel))
-			DRM_ERROR("failed to turnoff the panel\n");
-	}
-
 	return 0;
 }
 EXPORT_SYMBOL_GPL(analogix_dp_suspend);
@@ -1884,13 +1878,6 @@ int analogix_dp_resume(struct analogix_dp_device *dp)
 		return ret;
 	}
 
-	if (dp->plat_data->panel) {
-		if (drm_panel_prepare(dp->plat_data->panel)) {
-			DRM_ERROR("failed to setup the panel\n");
-			return -EBUSY;
-		}
-	}
-
 	return 0;
 }
 EXPORT_SYMBOL_GPL(analogix_dp_resume);
diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c
index d1f1d525aeb6..b0ff1ecb80a5 100644
--- a/drivers/gpu/drm/bridge/analogix/anx7625.c
+++ b/drivers/gpu/drm/bridge/analogix/anx7625.c
@@ -1440,6 +1440,20 @@ static void anx7625_start_dp_work(struct anx7625_data *ctx)
 
 static int anx7625_read_hpd_status_p0(struct anx7625_data *ctx)
 {
+	int ret;
+
+	/* Set irq detect window to 2ms */
+	ret = anx7625_reg_write(ctx, ctx->i2c.tx_p2_client,
+				HPD_DET_TIMER_BIT0_7, HPD_TIME & 0xFF);
+	ret |= anx7625_reg_write(ctx, ctx->i2c.tx_p2_client,
+				 HPD_DET_TIMER_BIT8_15,
+				 (HPD_TIME >> 8) & 0xFF);
+	ret |= anx7625_reg_write(ctx, ctx->i2c.tx_p2_client,
+				 HPD_DET_TIMER_BIT16_23,
+				 (HPD_TIME >> 16) & 0xFF);
+	if (ret < 0)
+		return ret;
+
 	return anx7625_reg_read(ctx, ctx->i2c.rx_p0_client, SYSTEM_STSTUS);
 }
 
@@ -1642,6 +1656,7 @@ static int anx7625_parse_dt(struct device *dev,
 	anx7625_get_swing_setting(dev, pdata);
 
 	pdata->is_dpi = 0; /* default dsi mode */
+	of_node_put(pdata->mipi_host_node);
 	pdata->mipi_host_node = of_graph_get_remote_node(np, 0, 0);
 	if (!pdata->mipi_host_node) {
 		DRM_DEV_ERROR(dev, "fail to get internal panel.\n");
@@ -1796,8 +1811,13 @@ static int anx7625_audio_hw_params(struct device *dev, void *data,
 	int wl, ch, rate;
 	int ret = 0;
 
-	if (fmt->fmt != HDMI_DSP_A) {
-		DRM_DEV_ERROR(dev, "only supports DSP_A\n");
+	if (anx7625_sink_detect(ctx) == connector_status_disconnected) {
+		DRM_DEV_DEBUG_DRIVER(dev, "DP not connected\n");
+		return 0;
+	}
+
+	if (fmt->fmt != HDMI_DSP_A && fmt->fmt != HDMI_I2S) {
+		DRM_DEV_ERROR(dev, "only supports DSP_A & I2S\n");
 		return -EINVAL;
 	}
 
@@ -1805,10 +1825,16 @@ static int anx7625_audio_hw_params(struct device *dev, void *data,
 			     params->sample_rate, params->sample_width,
 			     params->cea.channels);
 
-	ret |= anx7625_write_and_or(ctx, ctx->i2c.tx_p2_client,
-				    AUDIO_CHANNEL_STATUS_6,
-				    ~I2S_SLAVE_MODE,
-				    TDM_SLAVE_MODE);
+	if (fmt->fmt == HDMI_DSP_A)
+		ret = anx7625_write_and_or(ctx, ctx->i2c.tx_p2_client,
+					   AUDIO_CHANNEL_STATUS_6,
+					   ~I2S_SLAVE_MODE,
+					   TDM_SLAVE_MODE);
+	else
+		ret = anx7625_write_and_or(ctx, ctx->i2c.tx_p2_client,
+					   AUDIO_CHANNEL_STATUS_6,
+					   ~TDM_SLAVE_MODE,
+					   I2S_SLAVE_MODE);
 
 	/* Word length */
 	switch (params->sample_width) {
@@ -2689,7 +2715,7 @@ free_hdcp_wq:
 	return ret;
 }
 
-static int anx7625_i2c_remove(struct i2c_client *client)
+static void anx7625_i2c_remove(struct i2c_client *client)
 {
 	struct anx7625_data *platform = i2c_get_clientdata(client);
 
@@ -2709,8 +2735,6 @@ static int anx7625_i2c_remove(struct i2c_client *client)
 
 	if (platform->pdata.audio_en)
 		anx7625_unregister_audio(platform);
-
-	return 0;
 }
 
 static const struct i2c_device_id anx7625_id[] = {
diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.h b/drivers/gpu/drm/bridge/analogix/anx7625.h
index e257a84db962..14f33d6be289 100644
--- a/drivers/gpu/drm/bridge/analogix/anx7625.h
+++ b/drivers/gpu/drm/bridge/analogix/anx7625.h
@@ -132,6 +132,12 @@
 #define I2S_SLAVE_MODE 0x08
 #define AUDIO_LAYOUT   0x01
 
+#define HPD_DET_TIMER_BIT0_7   0xea
+#define HPD_DET_TIMER_BIT8_15  0xeb
+#define HPD_DET_TIMER_BIT16_23 0xec
+/* HPD debounce time 2ms for 27M clock */
+#define HPD_TIME               54000
+
 #define AUDIO_CONTROL_REGISTER 0xe6
 #define TDM_TIMING_MODE 0x08
 
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
index ab63e7b11944..31442a922502 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
@@ -2605,7 +2605,8 @@ static int cdns_mhdp_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 
 	cancel_work_sync(&mhdp->modeset_retry_work);
-	flush_scheduled_work();
+	flush_work(&mhdp->hpd_work);
+	/* Ignoring mhdp->hdcp.check_work and mhdp->hdcp.prop_work here. */
 
 	clk_disable_unprepare(mhdp->clk);
 
diff --git a/drivers/gpu/drm/bridge/chipone-icn6211.c b/drivers/gpu/drm/bridge/chipone-icn6211.c
index 481c86b2406e..bf920c3503aa 100644
--- a/drivers/gpu/drm/bridge/chipone-icn6211.c
+++ b/drivers/gpu/drm/bridge/chipone-icn6211.c
@@ -11,6 +11,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bits.h>
+#include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
@@ -151,6 +152,8 @@ struct chipone {
 	struct regulator *vdd1;
 	struct regulator *vdd2;
 	struct regulator *vdd3;
+	struct clk *refclk;
+	unsigned long refclk_rate;
 	bool interface_i2c;
 };
 
@@ -259,7 +262,7 @@ static void chipone_configure_pll(struct chipone *icn,
 
 	/*
 	 * DSI byte clock frequency (input into PLL) is calculated as:
-	 *  DSI_CLK = mode clock * bpp / dsi_data_lanes / 8
+	 *  DSI_CLK = HS clock / 4
 	 *
 	 * DPI pixel clock frequency (output from PLL) is mode clock.
 	 *
@@ -273,8 +276,10 @@ static void chipone_configure_pll(struct chipone *icn,
 	 * It seems the PLL input clock after applying P pre-divider have
 	 * to be lower than 20 MHz.
 	 */
-	fin = mode_clock * mipi_dsi_pixel_format_to_bpp(icn->dsi->format) /
-	      icn->dsi->lanes / 8; /* in Hz */
+	if (icn->refclk)
+		fin = icn->refclk_rate;
+	else
+		fin = icn->dsi->hs_rate / 4; /* in Hz */
 
 	/* Minimum value of P predivider for PLL input in 5..20 MHz */
 	p_min = clamp(DIV_ROUND_UP(fin, 20000000), 1U, 31U);
@@ -319,16 +324,18 @@ static void chipone_configure_pll(struct chipone *icn,
 	best_p_pot = !(best_p & 1);
 
 	dev_dbg(icn->dev,
-		"PLL: P[3:0]=%d P[4]=2*%d M=%d S[7:5]=2^%d delta=%d => DSI f_in=%d Hz ; DPI f_out=%d Hz\n",
+		"PLL: P[3:0]=%d P[4]=2*%d M=%d S[7:5]=2^%d delta=%d => DSI f_in(%s)=%d Hz ; DPI f_out=%d Hz\n",
 		best_p >> best_p_pot, best_p_pot, best_m, best_s + 1,
-		min_delta, fin, (fin * best_m) / (best_p << (best_s + 1)));
+		min_delta, icn->refclk ? "EXT" : "DSI", fin,
+		(fin * best_m) / (best_p << (best_s + 1)));
 
 	ref_div = PLL_REF_DIV_P(best_p >> best_p_pot) | PLL_REF_DIV_S(best_s);
 	if (best_p_pot)	/* Prefer /2 pre-divider */
 		ref_div |= PLL_REF_DIV_Pe;
 
-	/* Clock source selection fixed to MIPI DSI clock lane */
-	chipone_writeb(icn, PLL_CTRL(6), PLL_CTRL_6_MIPI_CLK);
+	/* Clock source selection either external clock or MIPI DSI clock lane */
+	chipone_writeb(icn, PLL_CTRL(6),
+		       icn->refclk ? PLL_CTRL_6_EXTERNAL : PLL_CTRL_6_MIPI_CLK);
 	chipone_writeb(icn, PLL_REF_DIV, ref_div);
 	chipone_writeb(icn, PLL_INT(0), best_m);
 }
@@ -464,6 +471,11 @@ static void chipone_atomic_pre_enable(struct drm_bridge *bridge,
 				      "failed to enable VDD3 regulator: %d\n", ret);
 	}
 
+	ret = clk_prepare_enable(icn->refclk);
+	if (ret)
+		DRM_DEV_ERROR(icn->dev,
+			      "failed to enable RECLK clock: %d\n", ret);
+
 	gpiod_set_value(icn->enable_gpio, 1);
 
 	usleep_range(10000, 11000);
@@ -474,6 +486,8 @@ static void chipone_atomic_post_disable(struct drm_bridge *bridge,
 {
 	struct chipone *icn = bridge_to_chipone(bridge);
 
+	clk_disable_unprepare(icn->refclk);
+
 	if (icn->vdd1)
 		regulator_disable(icn->vdd1);
 
@@ -515,6 +529,8 @@ static int chipone_dsi_attach(struct chipone *icn)
 	dsi->format = MIPI_DSI_FMT_RGB888;
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
 			  MIPI_DSI_MODE_LPM | MIPI_DSI_MODE_NO_EOT_PACKET;
+	dsi->hs_rate = 500000000;
+	dsi->lp_rate = 16000000;
 
 	ret = mipi_dsi_attach(dsi);
 	if (ret < 0)
@@ -617,6 +633,20 @@ static int chipone_parse_dt(struct chipone *icn)
 	struct device *dev = icn->dev;
 	int ret;
 
+	icn->refclk = devm_clk_get_optional(dev, "refclk");
+	if (IS_ERR(icn->refclk)) {
+		ret = PTR_ERR(icn->refclk);
+		DRM_DEV_ERROR(dev, "failed to get REFCLK clock: %d\n", ret);
+		return ret;
+	} else if (icn->refclk) {
+		icn->refclk_rate = clk_get_rate(icn->refclk);
+		if (icn->refclk_rate < 10000000 || icn->refclk_rate > 154000000) {
+			DRM_DEV_ERROR(dev, "REFCLK out of range: %ld Hz\n",
+				      icn->refclk_rate);
+			return -EINVAL;
+		}
+	}
+
 	icn->vdd1 = devm_regulator_get_optional(dev, "vdd1");
 	if (IS_ERR(icn->vdd1)) {
 		ret = PTR_ERR(icn->vdd1);
@@ -735,14 +765,12 @@ static int chipone_i2c_probe(struct i2c_client *client,
 	return chipone_dsi_host_attach(icn);
 }
 
-static int chipone_dsi_remove(struct mipi_dsi_device *dsi)
+static void chipone_dsi_remove(struct mipi_dsi_device *dsi)
 {
 	struct chipone *icn = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_bridge_remove(&icn->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id chipone_of_match[] = {
diff --git a/drivers/gpu/drm/bridge/chrontel-ch7033.c b/drivers/gpu/drm/bridge/chrontel-ch7033.c
index ba060277c3fd..b94f39a86846 100644
--- a/drivers/gpu/drm/bridge/chrontel-ch7033.c
+++ b/drivers/gpu/drm/bridge/chrontel-ch7033.c
@@ -583,14 +583,12 @@ static int ch7033_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int ch7033_remove(struct i2c_client *client)
+static void ch7033_remove(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
 	struct ch7033_priv *priv = dev_get_drvdata(dev);
 
 	drm_bridge_remove(&priv->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id ch7033_dt_ids[] = {
diff --git a/drivers/gpu/drm/bridge/cros-ec-anx7688.c b/drivers/gpu/drm/bridge/cros-ec-anx7688.c
index 0f6d907432e3..fa91bdeddef0 100644
--- a/drivers/gpu/drm/bridge/cros-ec-anx7688.c
+++ b/drivers/gpu/drm/bridge/cros-ec-anx7688.c
@@ -159,13 +159,11 @@ static int cros_ec_anx7688_bridge_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int cros_ec_anx7688_bridge_remove(struct i2c_client *client)
+static void cros_ec_anx7688_bridge_remove(struct i2c_client *client)
 {
 	struct cros_ec_anx7688 *anx7688 = i2c_get_clientdata(client);
 
 	drm_bridge_remove(&anx7688->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id cros_ec_anx7688_bridge_match_table[] = {
diff --git a/drivers/gpu/drm/bridge/ite-it6505.c b/drivers/gpu/drm/bridge/ite-it6505.c
index 4b673c4792d7..dfe4351c9bdd 100644
--- a/drivers/gpu/drm/bridge/ite-it6505.c
+++ b/drivers/gpu/drm/bridge/ite-it6505.c
@@ -506,6 +506,9 @@ static int it6505_read(struct it6505 *it6505, unsigned int reg_addr)
 	int err;
 	struct device *dev = &it6505->client->dev;
 
+	if (!it6505->powered)
+		return -ENODEV;
+
 	err = regmap_read(it6505->regmap, reg_addr, &value);
 	if (err < 0) {
 		dev_err(dev, "read failed reg[0x%x] err: %d", reg_addr, err);
@@ -521,6 +524,9 @@ static int it6505_write(struct it6505 *it6505, unsigned int reg_addr,
 	int err;
 	struct device *dev = &it6505->client->dev;
 
+	if (!it6505->powered)
+		return -ENODEV;
+
 	err = regmap_write(it6505->regmap, reg_addr, reg_val);
 
 	if (err < 0) {
@@ -538,6 +544,9 @@ static int it6505_set_bits(struct it6505 *it6505, unsigned int reg,
 	int err;
 	struct device *dev = &it6505->client->dev;
 
+	if (!it6505->powered)
+		return -ENODEV;
+
 	err = regmap_update_bits(it6505->regmap, reg, mask, value);
 	if (err < 0) {
 		dev_err(dev, "write reg[0x%x] = 0x%x mask = 0x%x failed err %d",
@@ -554,7 +563,7 @@ static void it6505_debug_print(struct it6505 *it6505, unsigned int reg,
 	struct device *dev = &it6505->client->dev;
 	int val;
 
-	if (likely(!(__drm_debug & DRM_UT_DRIVER)))
+	if (!drm_debug_enabled(DRM_UT_DRIVER))
 		return;
 
 	val = it6505_read(it6505, reg);
@@ -682,7 +691,7 @@ static void it6505_calc_video_info(struct it6505 *it6505)
 	DRM_DEV_DEBUG_DRIVER(dev, "hactive_start:%d, vactive_start:%d",
 			     hdes, vdes);
 
-	for (i = 0; i < 10; i++) {
+	for (i = 0; i < 3; i++) {
 		it6505_set_bits(it6505, REG_DATA_CTRL0, ENABLE_PCLK_COUNTER,
 				ENABLE_PCLK_COUNTER);
 		usleep_range(10000, 15000);
@@ -699,7 +708,7 @@ static void it6505_calc_video_info(struct it6505 *it6505)
 		return;
 	}
 
-	sum /= 10;
+	sum /= 3;
 	pclk = 13500 * 2048 / sum;
 	it6505->video_info.clock = pclk;
 	it6505->video_info.hdisplay = hdew;
@@ -2341,8 +2350,6 @@ static void it6505_irq_hpd(struct it6505 *it6505)
 
 		if (!it6505_get_video_status(it6505))
 			it6505_video_reset(it6505);
-
-		it6505_calc_video_info(it6505);
 	} else {
 		memset(it6505->dpcd, 0, sizeof(it6505->dpcd));
 
@@ -2559,13 +2566,12 @@ static int it6505_poweron(struct it6505 *it6505)
 		usleep_range(10000, 20000);
 	}
 
+	it6505->powered = true;
 	it6505_reset_logic(it6505);
 	it6505_int_mask_enable(it6505);
 	it6505_init(it6505);
 	it6505_lane_off(it6505);
 
-	it6505->powered = true;
-
 	return 0;
 }
 
@@ -2954,6 +2960,9 @@ static void it6505_bridge_atomic_enable(struct drm_bridge *bridge,
 
 	it6505_int_mask_enable(it6505);
 	it6505_video_reset(it6505);
+
+	it6505_drm_dp_link_set_power(&it6505->aux, &it6505->link,
+				     DP_SET_POWER_D0);
 }
 
 static void it6505_bridge_atomic_disable(struct drm_bridge *bridge,
@@ -2965,9 +2974,9 @@ static void it6505_bridge_atomic_disable(struct drm_bridge *bridge,
 	DRM_DEV_DEBUG_DRIVER(dev, "start");
 
 	if (it6505->powered) {
-		it6505_video_disable(it6505);
 		it6505_drm_dp_link_set_power(&it6505->aux, &it6505->link,
 					     DP_SET_POWER_D3);
+		it6505_video_disable(it6505);
 	}
 }
 
@@ -3044,7 +3053,7 @@ static int it6505_init_pdata(struct it6505 *it6505)
 		return PTR_ERR(pdata->ovdd);
 	}
 
-	pdata->gpiod_reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	pdata->gpiod_reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
 	if (IS_ERR(pdata->gpiod_reset)) {
 		dev_err(dev, "gpiod_reset gpio not found");
 		return PTR_ERR(pdata->gpiod_reset);
@@ -3316,7 +3325,7 @@ static int it6505_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int it6505_i2c_remove(struct i2c_client *client)
+static void it6505_i2c_remove(struct i2c_client *client)
 {
 	struct it6505 *it6505 = i2c_get_clientdata(client);
 
@@ -3324,8 +3333,6 @@ static int it6505_i2c_remove(struct i2c_client *client)
 	drm_dp_aux_unregister(&it6505->aux);
 	it6505_debugfs_remove(it6505);
 	it6505_poweroff(it6505);
-
-	return 0;
 }
 
 static const struct i2c_device_id it6505_id[] = {
diff --git a/drivers/gpu/drm/bridge/ite-it66121.c b/drivers/gpu/drm/bridge/ite-it66121.c
index 44278d54d35d..4f6f1deba28c 100644
--- a/drivers/gpu/drm/bridge/ite-it66121.c
+++ b/drivers/gpu/drm/bridge/ite-it66121.c
@@ -1623,15 +1623,13 @@ static int it66121_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int it66121_remove(struct i2c_client *client)
+static void it66121_remove(struct i2c_client *client)
 {
 	struct it66121_ctx *ctx = i2c_get_clientdata(client);
 
 	ite66121_power_off(ctx);
 	drm_bridge_remove(&ctx->bridge);
 	mutex_destroy(&ctx->lock);
-
-	return 0;
 }
 
 static const struct of_device_id it66121_dt_match[] = {
diff --git a/drivers/gpu/drm/bridge/lontium-lt8912b.c b/drivers/gpu/drm/bridge/lontium-lt8912b.c
index 28bad30dc4e5..a98efef0ba0e 100644
--- a/drivers/gpu/drm/bridge/lontium-lt8912b.c
+++ b/drivers/gpu/drm/bridge/lontium-lt8912b.c
@@ -165,30 +165,38 @@ static int lt8912_write_rxlogicres_config(struct lt8912 *lt)
 	return ret;
 };
 
+/* enable LVDS output with some hardcoded configuration, not required for the HDMI output */
 static int lt8912_write_lvds_config(struct lt8912 *lt)
 {
 	const struct reg_sequence seq[] = {
+		// lvds power up
 		{0x44, 0x30},
 		{0x51, 0x05},
-		{0x50, 0x24},
-		{0x51, 0x2d},
-		{0x52, 0x04},
-		{0x69, 0x0e},
+
+		// core pll bypass
+		{0x50, 0x24}, // cp=50uA
+		{0x51, 0x2d}, // Pix_clk as reference, second order passive LPF PLL
+		{0x52, 0x04}, // loopdiv=0, use second-order PLL
+		{0x69, 0x0e}, // CP_PRESET_DIV_RATIO
 		{0x69, 0x8e},
 		{0x6a, 0x00},
-		{0x6c, 0xb8},
+		{0x6c, 0xb8}, // RGD_CP_SOFT_K_EN,RGD_CP_SOFT_K[13:8]
 		{0x6b, 0x51},
-		{0x04, 0xfb},
+
+		{0x04, 0xfb}, // core pll reset
 		{0x04, 0xff},
-		{0x7f, 0x00},
-		{0xa8, 0x13},
-		{0x02, 0xf7},
+
+		// scaler bypass
+		{0x7f, 0x00}, // disable scaler
+		{0xa8, 0x13}, // 0x13: JEIDA, 0x33: VESA
+
+		{0x02, 0xf7}, // lvds pll reset
 		{0x02, 0xff},
 		{0x03, 0xcf},
 		{0x03, 0xff},
 	};
 
-	return regmap_multi_reg_write(lt->regmap[I2C_CEC_DSI], seq, ARRAY_SIZE(seq));
+	return regmap_multi_reg_write(lt->regmap[I2C_MAIN], seq, ARRAY_SIZE(seq));
 };
 
 static inline struct lt8912 *bridge_to_lt8912(struct drm_bridge *b)
@@ -268,7 +276,7 @@ static int lt8912_video_setup(struct lt8912 *lt)
 	u32 hactive, h_total, hpw, hfp, hbp;
 	u32 vactive, v_total, vpw, vfp, vbp;
 	u8 settle = 0x08;
-	int ret;
+	int ret, hsync_activehigh, vsync_activehigh;
 
 	if (!lt)
 		return -EINVAL;
@@ -278,12 +286,14 @@ static int lt8912_video_setup(struct lt8912 *lt)
 	hpw = lt->mode.hsync_len;
 	hbp = lt->mode.hback_porch;
 	h_total = hactive + hfp + hpw + hbp;
+	hsync_activehigh = lt->mode.flags & DISPLAY_FLAGS_HSYNC_HIGH;
 
 	vactive = lt->mode.vactive;
 	vfp = lt->mode.vfront_porch;
 	vpw = lt->mode.vsync_len;
 	vbp = lt->mode.vback_porch;
 	v_total = vactive + vfp + vpw + vbp;
+	vsync_activehigh = lt->mode.flags & DISPLAY_FLAGS_VSYNC_HIGH;
 
 	if (vactive <= 600)
 		settle = 0x04;
@@ -317,6 +327,13 @@ static int lt8912_video_setup(struct lt8912 *lt)
 	ret |= regmap_write(lt->regmap[I2C_CEC_DSI], 0x3e, hfp & 0xff);
 	ret |= regmap_write(lt->regmap[I2C_CEC_DSI], 0x3f, hfp >> 8);
 
+	ret |= regmap_update_bits(lt->regmap[I2C_MAIN], 0xab, BIT(0),
+				  vsync_activehigh ? BIT(0) : 0);
+	ret |= regmap_update_bits(lt->regmap[I2C_MAIN], 0xab, BIT(1),
+				  hsync_activehigh ? BIT(1) : 0);
+	ret |= regmap_update_bits(lt->regmap[I2C_MAIN], 0xb2, BIT(0),
+				  lt->connector.display_info.is_hdmi ? BIT(0) : 0);
+
 	return ret;
 }
 
@@ -714,7 +731,7 @@ err_dt_parse:
 	return ret;
 }
 
-static int lt8912_remove(struct i2c_client *client)
+static void lt8912_remove(struct i2c_client *client)
 {
 	struct lt8912 *lt = i2c_get_clientdata(client);
 
@@ -722,7 +739,6 @@ static int lt8912_remove(struct i2c_client *client)
 	drm_bridge_remove(&lt->bridge);
 	lt8912_free_i2c(lt);
 	lt8912_put_dt(lt);
-	return 0;
 }
 
 static const struct of_device_id lt8912_dt_match[] = {
diff --git a/drivers/gpu/drm/bridge/lontium-lt9211.c b/drivers/gpu/drm/bridge/lontium-lt9211.c
index 9a3e90427d12..933ca028d612 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9211.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9211.c
@@ -766,13 +766,11 @@ static int lt9211_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int lt9211_remove(struct i2c_client *client)
+static void lt9211_remove(struct i2c_client *client)
 {
 	struct lt9211 *ctx = i2c_get_clientdata(client);
 
 	drm_bridge_remove(&ctx->bridge);
-
-	return 0;
 }
 
 static struct i2c_device_id lt9211_id[] = {
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611.c b/drivers/gpu/drm/bridge/lontium-lt9611.c
index 8a60e83482a0..7c0a99173b39 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611.c
@@ -813,13 +813,14 @@ static int lt9611_connector_init(struct drm_bridge *bridge, struct lt9611 *lt961
 
 	drm_connector_helper_add(&lt9611->connector,
 				 &lt9611_bridge_connector_helper_funcs);
-	drm_connector_attach_encoder(&lt9611->connector, bridge->encoder);
 
 	if (!bridge->encoder) {
 		DRM_ERROR("Parent encoder object not found");
 		return -ENODEV;
 	}
 
+	drm_connector_attach_encoder(&lt9611->connector, bridge->encoder);
+
 	return 0;
 }
 
@@ -1216,7 +1217,7 @@ err_of_put:
 	return ret;
 }
 
-static int lt9611_remove(struct i2c_client *client)
+static void lt9611_remove(struct i2c_client *client)
 {
 	struct lt9611 *lt9611 = i2c_get_clientdata(client);
 
@@ -1228,8 +1229,6 @@ static int lt9611_remove(struct i2c_client *client)
 
 	of_node_put(lt9611->dsi1_node);
 	of_node_put(lt9611->dsi0_node);
-
-	return 0;
 }
 
 static struct i2c_device_id lt9611_id[] = {
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
index fdf12d4c6416..fa1ee6264d92 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
@@ -978,7 +978,7 @@ err_of_put:
 	return ret;
 }
 
-static int lt9611uxc_remove(struct i2c_client *client)
+static void lt9611uxc_remove(struct i2c_client *client)
 {
 	struct lt9611uxc *lt9611uxc = i2c_get_clientdata(client);
 
@@ -993,8 +993,6 @@ static int lt9611uxc_remove(struct i2c_client *client)
 
 	of_node_put(lt9611uxc->dsi1_node);
 	of_node_put(lt9611uxc->dsi0_node);
-
-	return 0;
 }
 
 static struct i2c_device_id lt9611uxc_id[] = {
diff --git a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
index cce98bf2a4e7..97359f807bfc 100644
--- a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
+++ b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
@@ -296,7 +296,9 @@ static void ge_b850v3_lvds_remove(void)
 	 * This check is to avoid both the drivers
 	 * removing the bridge in their remove() function
 	 */
-	if (!ge_b850v3_lvds_ptr)
+	if (!ge_b850v3_lvds_ptr ||
+	    !ge_b850v3_lvds_ptr->stdp2690_i2c ||
+		!ge_b850v3_lvds_ptr->stdp4028_i2c)
 		goto out;
 
 	drm_bridge_remove(&ge_b850v3_lvds_ptr->bridge);
@@ -355,11 +357,9 @@ static int stdp4028_ge_b850v3_fw_probe(struct i2c_client *stdp4028_i2c,
 	return ge_b850v3_register();
 }
 
-static int stdp4028_ge_b850v3_fw_remove(struct i2c_client *stdp4028_i2c)
+static void stdp4028_ge_b850v3_fw_remove(struct i2c_client *stdp4028_i2c)
 {
 	ge_b850v3_lvds_remove();
-
-	return 0;
 }
 
 static const struct i2c_device_id stdp4028_ge_b850v3_fw_i2c_table[] = {
@@ -405,11 +405,9 @@ static int stdp2690_ge_b850v3_fw_probe(struct i2c_client *stdp2690_i2c,
 	return ge_b850v3_register();
 }
 
-static int stdp2690_ge_b850v3_fw_remove(struct i2c_client *stdp2690_i2c)
+static void stdp2690_ge_b850v3_fw_remove(struct i2c_client *stdp2690_i2c)
 {
 	ge_b850v3_lvds_remove();
-
-	return 0;
 }
 
 static const struct i2c_device_id stdp2690_ge_b850v3_fw_i2c_table[] = {
diff --git a/drivers/gpu/drm/bridge/nxp-ptn3460.c b/drivers/gpu/drm/bridge/nxp-ptn3460.c
index 1ab91f4e057b..0851101a8c72 100644
--- a/drivers/gpu/drm/bridge/nxp-ptn3460.c
+++ b/drivers/gpu/drm/bridge/nxp-ptn3460.c
@@ -315,13 +315,11 @@ static int ptn3460_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int ptn3460_remove(struct i2c_client *client)
+static void ptn3460_remove(struct i2c_client *client)
 {
 	struct ptn3460_bridge *ptn_bridge = i2c_get_clientdata(client);
 
 	drm_bridge_remove(&ptn_bridge->bridge);
-
-	return 0;
 }
 
 static const struct i2c_device_id ptn3460_i2c_table[] = {
diff --git a/drivers/gpu/drm/bridge/panel.c b/drivers/gpu/drm/bridge/panel.c
index 4277bf4f032b..216af76d0042 100644
--- a/drivers/gpu/drm/bridge/panel.c
+++ b/drivers/gpu/drm/bridge/panel.c
@@ -8,6 +8,7 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_connector.h>
 #include <drm/drm_encoder.h>
+#include <drm/drm_managed.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
@@ -367,6 +368,44 @@ struct drm_bridge *devm_drm_panel_bridge_add_typed(struct device *dev,
 }
 EXPORT_SYMBOL(devm_drm_panel_bridge_add_typed);
 
+static void drmm_drm_panel_bridge_release(struct drm_device *drm, void *ptr)
+{
+	struct drm_bridge *bridge = ptr;
+
+	drm_panel_bridge_remove(bridge);
+}
+
+/**
+ * drmm_panel_bridge_add - Creates a DRM-managed &drm_bridge and
+ *                         &drm_connector that just calls the
+ *                         appropriate functions from &drm_panel.
+ *
+ * @drm: DRM device to tie the bridge lifetime to
+ * @panel: The drm_panel being wrapped.  Must be non-NULL.
+ *
+ * This is the DRM-managed version of drm_panel_bridge_add() which
+ * automatically calls drm_panel_bridge_remove() when @dev is cleaned
+ * up.
+ */
+struct drm_bridge *drmm_panel_bridge_add(struct drm_device *drm,
+					 struct drm_panel *panel)
+{
+	struct drm_bridge *bridge;
+	int ret;
+
+	bridge = drm_panel_bridge_add_typed(panel, panel->connector_type);
+	if (IS_ERR(bridge))
+		return bridge;
+
+	ret = drmm_add_action_or_reset(drm, drmm_drm_panel_bridge_release,
+				       bridge);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return bridge;
+}
+EXPORT_SYMBOL(drmm_panel_bridge_add);
+
 /**
  * drm_panel_bridge_connector - return the connector for the panel bridge
  * @bridge: The drm_bridge.
@@ -420,4 +459,39 @@ struct drm_bridge *devm_drm_of_get_bridge(struct device *dev,
 	return bridge;
 }
 EXPORT_SYMBOL(devm_drm_of_get_bridge);
+
+/**
+ * drmm_of_get_bridge - Return next bridge in the chain
+ * @drm: device to tie the bridge lifetime to
+ * @np: device tree node containing encoder output ports
+ * @port: port in the device tree node
+ * @endpoint: endpoint in the device tree node
+ *
+ * Given a DT node's port and endpoint number, finds the connected node
+ * and returns the associated bridge if any, or creates and returns a
+ * drm panel bridge instance if a panel is connected.
+ *
+ * Returns a drmm managed pointer to the bridge if successful, or an error
+ * pointer otherwise.
+ */
+struct drm_bridge *drmm_of_get_bridge(struct drm_device *drm,
+				      struct device_node *np,
+				      u32 port, u32 endpoint)
+{
+	struct drm_bridge *bridge;
+	struct drm_panel *panel;
+	int ret;
+
+	ret = drm_of_find_panel_or_bridge(np, port, endpoint,
+					  &panel, &bridge);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (panel)
+		bridge = drmm_panel_bridge_add(drm, panel);
+
+	return bridge;
+}
+EXPORT_SYMBOL(drmm_of_get_bridge);
+
 #endif
diff --git a/drivers/gpu/drm/bridge/parade-ps8622.c b/drivers/gpu/drm/bridge/parade-ps8622.c
index b5750e5f71d7..309de802863d 100644
--- a/drivers/gpu/drm/bridge/parade-ps8622.c
+++ b/drivers/gpu/drm/bridge/parade-ps8622.c
@@ -520,14 +520,12 @@ static int ps8622_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int ps8622_remove(struct i2c_client *client)
+static void ps8622_remove(struct i2c_client *client)
 {
 	struct ps8622_bridge *ps8622 = i2c_get_clientdata(client);
 
 	backlight_device_unregister(ps8622->bl);
 	drm_bridge_remove(&ps8622->bridge);
-
-	return 0;
 }
 
 static const struct i2c_device_id ps8622_i2c_table[] = {
diff --git a/drivers/gpu/drm/bridge/parade-ps8640.c b/drivers/gpu/drm/bridge/parade-ps8640.c
index 31e88cb39f8a..d7483c13c569 100644
--- a/drivers/gpu/drm/bridge/parade-ps8640.c
+++ b/drivers/gpu/drm/bridge/parade-ps8640.c
@@ -375,6 +375,11 @@ static int __maybe_unused ps8640_resume(struct device *dev)
 	gpiod_set_value(ps_bridge->gpio_reset, 1);
 	usleep_range(2000, 2500);
 	gpiod_set_value(ps_bridge->gpio_reset, 0);
+	/* Double reset for T4 and T5 */
+	msleep(50);
+	gpiod_set_value(ps_bridge->gpio_reset, 1);
+	msleep(50);
+	gpiod_set_value(ps_bridge->gpio_reset, 0);
 
 	/*
 	 * Mystery 200 ms delay for the "MCU to be ready". It's unclear if
@@ -631,8 +636,8 @@ static int ps8640_probe(struct i2c_client *client)
 	if (!ps_bridge)
 		return -ENOMEM;
 
-	ps_bridge->supplies[0].supply = "vdd33";
-	ps_bridge->supplies[1].supply = "vdd12";
+	ps_bridge->supplies[0].supply = "vdd12";
+	ps_bridge->supplies[1].supply = "vdd33";
 	ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ps_bridge->supplies),
 				      ps_bridge->supplies);
 	if (ret)
diff --git a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c
index 7ab38d734ad6..878fb7d3732b 100644
--- a/drivers/gpu/drm/bridge/sii902x.c
+++ b/drivers/gpu/drm/bridge/sii902x.c
@@ -1145,7 +1145,7 @@ static int sii902x_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int sii902x_remove(struct i2c_client *client)
+static void sii902x_remove(struct i2c_client *client)
 
 {
 	struct sii902x *sii902x = i2c_get_clientdata(client);
@@ -1154,8 +1154,6 @@ static int sii902x_remove(struct i2c_client *client)
 	drm_bridge_remove(&sii902x->bridge);
 	regulator_bulk_disable(ARRAY_SIZE(sii902x->supplies),
 			       sii902x->supplies);
-
-	return 0;
 }
 
 static const struct of_device_id sii902x_dt_ids[] = {
diff --git a/drivers/gpu/drm/bridge/sii9234.c b/drivers/gpu/drm/bridge/sii9234.c
index 15c98a7bd81c..5b3061d4b5c3 100644
--- a/drivers/gpu/drm/bridge/sii9234.c
+++ b/drivers/gpu/drm/bridge/sii9234.c
@@ -936,14 +936,12 @@ static int sii9234_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int sii9234_remove(struct i2c_client *client)
+static void sii9234_remove(struct i2c_client *client)
 {
 	struct sii9234 *ctx = i2c_get_clientdata(client);
 
 	sii9234_cable_out(ctx);
 	drm_bridge_remove(&ctx->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id sii9234_dt_match[] = {
diff --git a/drivers/gpu/drm/bridge/sil-sii8620.c b/drivers/gpu/drm/bridge/sil-sii8620.c
index ab0bce4a988c..511982a1cedb 100644
--- a/drivers/gpu/drm/bridge/sil-sii8620.c
+++ b/drivers/gpu/drm/bridge/sil-sii8620.c
@@ -2346,7 +2346,7 @@ static int sii8620_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int sii8620_remove(struct i2c_client *client)
+static void sii8620_remove(struct i2c_client *client)
 {
 	struct sii8620 *ctx = i2c_get_clientdata(client);
 
@@ -2360,8 +2360,6 @@ static int sii8620_remove(struct i2c_client *client)
 		sii8620_cable_out(ctx);
 	}
 	drm_bridge_remove(&ctx->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id sii8620_dt_match[] = {
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
index 7d2ed0ed2fe2..4efb62bcdb63 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-ahb-audio.c
@@ -542,8 +542,8 @@ static int snd_dw_hdmi_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
-	strlcpy(card->driver, DRIVER_NAME, sizeof(card->driver));
-	strlcpy(card->shortname, "DW-HDMI", sizeof(card->shortname));
+	strscpy(card->driver, DRIVER_NAME, sizeof(card->driver));
+	strscpy(card->shortname, "DW-HDMI", sizeof(card->shortname));
 	snprintf(card->longname, sizeof(card->longname),
 		 "%s rev 0x%02x, irq %d", card->shortname, revision,
 		 data->irq);
@@ -561,7 +561,7 @@ static int snd_dw_hdmi_probe(struct platform_device *pdev)
 
 	dw->pcm = pcm;
 	pcm->private_data = dw;
-	strlcpy(pcm->name, DRIVER_NAME, sizeof(pcm->name));
+	strscpy(pcm->name, DRIVER_NAME, sizeof(pcm->name));
 	snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &snd_dw_hdmi_ops);
 
 	/*
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
index 25a60eb4d67c..40d8ca37f5bc 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
@@ -3096,6 +3096,7 @@ static irqreturn_t dw_hdmi_irq(int irq, void *dev_id)
 {
 	struct dw_hdmi *hdmi = dev_id;
 	u8 intr_stat, phy_int_pol, phy_pol_mask, phy_stat;
+	enum drm_connector_status status = connector_status_unknown;
 
 	intr_stat = hdmi_readb(hdmi, HDMI_IH_PHY_STAT0);
 	phy_int_pol = hdmi_readb(hdmi, HDMI_PHY_POL0);
@@ -3134,13 +3135,15 @@ static irqreturn_t dw_hdmi_irq(int irq, void *dev_id)
 			cec_notifier_phys_addr_invalidate(hdmi->cec_notifier);
 			mutex_unlock(&hdmi->cec_notifier_mutex);
 		}
-	}
 
-	if (intr_stat & HDMI_IH_PHY_STAT0_HPD) {
-		enum drm_connector_status status = phy_int_pol & HDMI_PHY_HPD
-						 ? connector_status_connected
-						 : connector_status_disconnected;
+		if (phy_stat & HDMI_PHY_HPD)
+			status = connector_status_connected;
+
+		if (!(phy_stat & (HDMI_PHY_HPD | HDMI_PHY_RX_SENSE)))
+			status = connector_status_disconnected;
+	}
 
+	if (status != connector_status_unknown) {
 		dev_dbg(hdmi->dev, "EVENT=%s\n",
 			status == connector_status_connected ?
 			"plugin" : "plugout");
diff --git a/drivers/gpu/drm/bridge/tc358762.c b/drivers/gpu/drm/bridge/tc358762.c
index 40439da4db49..7f4fce1aa998 100644
--- a/drivers/gpu/drm/bridge/tc358762.c
+++ b/drivers/gpu/drm/bridge/tc358762.c
@@ -241,14 +241,12 @@ static int tc358762_probe(struct mipi_dsi_device *dsi)
 	return ret;
 }
 
-static int tc358762_remove(struct mipi_dsi_device *dsi)
+static void tc358762_remove(struct mipi_dsi_device *dsi)
 {
 	struct tc358762 *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_bridge_remove(&ctx->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id tc358762_of_match[] = {
diff --git a/drivers/gpu/drm/bridge/tc358764.c b/drivers/gpu/drm/bridge/tc358764.c
index fdfb14aca926..53259c12d777 100644
--- a/drivers/gpu/drm/bridge/tc358764.c
+++ b/drivers/gpu/drm/bridge/tc358764.c
@@ -381,14 +381,12 @@ static int tc358764_probe(struct mipi_dsi_device *dsi)
 	return ret;
 }
 
-static int tc358764_remove(struct mipi_dsi_device *dsi)
+static void tc358764_remove(struct mipi_dsi_device *dsi)
 {
 	struct tc358764 *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_bridge_remove(&ctx->bridge);
-
-	return 0;
 }
 
 static const struct of_device_id tc358764_of_match[] = {
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index 02bd757a8987..2a58eb271f70 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -889,6 +889,7 @@ static int tc_set_edp_video_mode(struct tc_data *tc,
 	u32 dp0_syncval;
 	u32 bits_per_pixel = 24;
 	u32 in_bw, out_bw;
+	u32 dpipxlfmt;
 
 	/*
 	 * Recommended maximum number of symbols transferred in a transfer unit:
@@ -938,10 +939,15 @@ static int tc_set_edp_video_mode(struct tc_data *tc,
 	if (ret)
 		return ret;
 
-	ret = regmap_write(tc->regmap, DPIPXLFMT,
-			   VS_POL_ACTIVE_LOW | HS_POL_ACTIVE_LOW |
-			   DE_POL_ACTIVE_HIGH | SUB_CFG_TYPE_CONFIG1 |
-			   DPI_BPP_RGB888);
+	dpipxlfmt = DE_POL_ACTIVE_HIGH | SUB_CFG_TYPE_CONFIG1 | DPI_BPP_RGB888;
+
+	if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+		dpipxlfmt |= VS_POL_ACTIVE_LOW;
+
+	if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+		dpipxlfmt |= HS_POL_ACTIVE_LOW;
+
+	ret = regmap_write(tc->regmap, DPIPXLFMT, dpipxlfmt);
 	if (ret)
 		return ret;
 
@@ -1244,7 +1250,13 @@ static int tc_main_link_disable(struct tc_data *tc)
 	if (ret)
 		return ret;
 
-	return regmap_write(tc->regmap, DP0CTL, 0);
+	ret = regmap_write(tc->regmap, DP0CTL, 0);
+	if (ret)
+		return ret;
+
+	return regmap_update_bits(tc->regmap, DP_PHY_CTRL,
+				  PHY_M0_RST | PHY_M1_RST | PHY_M0_EN,
+				  PHY_M0_RST | PHY_M1_RST);
 }
 
 static int tc_dsi_rx_enable(struct tc_data *tc)
@@ -1252,10 +1264,10 @@ static int tc_dsi_rx_enable(struct tc_data *tc)
 	u32 value;
 	int ret;
 
-	regmap_write(tc->regmap, PPI_D0S_CLRSIPOCOUNT, 3);
-	regmap_write(tc->regmap, PPI_D1S_CLRSIPOCOUNT, 3);
-	regmap_write(tc->regmap, PPI_D2S_CLRSIPOCOUNT, 3);
-	regmap_write(tc->regmap, PPI_D3S_CLRSIPOCOUNT, 3);
+	regmap_write(tc->regmap, PPI_D0S_CLRSIPOCOUNT, 5);
+	regmap_write(tc->regmap, PPI_D1S_CLRSIPOCOUNT, 5);
+	regmap_write(tc->regmap, PPI_D2S_CLRSIPOCOUNT, 5);
+	regmap_write(tc->regmap, PPI_D3S_CLRSIPOCOUNT, 5);
 	regmap_write(tc->regmap, PPI_D0S_ATMR, 0);
 	regmap_write(tc->regmap, PPI_D1S_ATMR, 0);
 	regmap_write(tc->regmap, PPI_TX_RX_TA, TTA_GET | TTA_SURE);
@@ -1496,41 +1508,16 @@ tc_edp_bridge_atomic_disable(struct drm_bridge *bridge,
 		dev_err(tc->dev, "main link disable error: %d\n", ret);
 }
 
-static bool tc_bridge_mode_fixup(struct drm_bridge *bridge,
-				 const struct drm_display_mode *mode,
-				 struct drm_display_mode *adj)
-{
-	/* Fixup sync polarities, both hsync and vsync are active low */
-	adj->flags = mode->flags;
-	adj->flags |= (DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC);
-	adj->flags &= ~(DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC);
-
-	return true;
-}
-
-static int tc_common_atomic_check(struct drm_bridge *bridge,
-				  struct drm_bridge_state *bridge_state,
-				  struct drm_crtc_state *crtc_state,
-				  struct drm_connector_state *conn_state,
-				  const unsigned int max_khz)
-{
-	tc_bridge_mode_fixup(bridge, &crtc_state->mode,
-			     &crtc_state->adjusted_mode);
-
-	if (crtc_state->adjusted_mode.clock > max_khz)
-		return -EINVAL;
-
-	return 0;
-}
-
 static int tc_dpi_atomic_check(struct drm_bridge *bridge,
 			       struct drm_bridge_state *bridge_state,
 			       struct drm_crtc_state *crtc_state,
 			       struct drm_connector_state *conn_state)
 {
 	/* DSI->DPI interface clock limitation: upto 100 MHz */
-	return tc_common_atomic_check(bridge, bridge_state, crtc_state,
-				      conn_state, 100000);
+	if (crtc_state->adjusted_mode.clock > 100000)
+		return -EINVAL;
+
+	return 0;
 }
 
 static int tc_edp_atomic_check(struct drm_bridge *bridge,
@@ -1539,8 +1526,10 @@ static int tc_edp_atomic_check(struct drm_bridge *bridge,
 			       struct drm_connector_state *conn_state)
 {
 	/* DPI->(e)DP interface clock limitation: upto 154 MHz */
-	return tc_common_atomic_check(bridge, bridge_state, crtc_state,
-				      conn_state, 154000);
+	if (crtc_state->adjusted_mode.clock > 154000)
+		return -EINVAL;
+
+	return 0;
 }
 
 static enum drm_mode_status
@@ -1783,7 +1772,6 @@ static const struct drm_bridge_funcs tc_edp_bridge_funcs = {
 	.atomic_check = tc_edp_atomic_check,
 	.atomic_enable = tc_edp_bridge_atomic_enable,
 	.atomic_disable = tc_edp_bridge_atomic_disable,
-	.mode_fixup = tc_bridge_mode_fixup,
 	.detect = tc_bridge_detect,
 	.get_edid = tc_get_edid,
 	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
@@ -1925,22 +1913,23 @@ static int tc_mipi_dsi_host_attach(struct tc_data *tc)
 static int tc_probe_dpi_bridge_endpoint(struct tc_data *tc)
 {
 	struct device *dev = tc->dev;
+	struct drm_bridge *bridge;
 	struct drm_panel *panel;
 	int ret;
 
 	/* port@1 is the DPI input/output port */
-	ret = drm_of_find_panel_or_bridge(dev->of_node, 1, 0, &panel, NULL);
+	ret = drm_of_find_panel_or_bridge(dev->of_node, 1, 0, &panel, &bridge);
 	if (ret && ret != -ENODEV)
 		return ret;
 
 	if (panel) {
-		struct drm_bridge *panel_bridge;
-
-		panel_bridge = devm_drm_panel_bridge_add(dev, panel);
-		if (IS_ERR(panel_bridge))
-			return PTR_ERR(panel_bridge);
+		bridge = devm_drm_panel_bridge_add(dev, panel);
+		if (IS_ERR(bridge))
+			return PTR_ERR(bridge);
+	}
 
-		tc->panel_bridge = panel_bridge;
+	if (bridge) {
+		tc->panel_bridge = bridge;
 		tc->bridge.type = DRM_MODE_CONNECTOR_DPI;
 		tc->bridge.funcs = &tc_dpi_bridge_funcs;
 
@@ -2010,9 +1999,10 @@ static int tc_probe_bridge_endpoint(struct tc_data *tc)
 
 	for_each_endpoint_of_node(dev->of_node, node) {
 		of_graph_parse_endpoint(node, &endpoint);
-		if (endpoint.port > 2)
+		if (endpoint.port > 2) {
+			of_node_put(node);
 			return -EINVAL;
-
+		}
 		mode |= BIT(endpoint.port);
 	}
 
@@ -2194,13 +2184,11 @@ static int tc_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	return 0;
 }
 
-static int tc_remove(struct i2c_client *client)
+static void tc_remove(struct i2c_client *client)
 {
 	struct tc_data *tc = i2c_get_clientdata(client);
 
 	drm_bridge_remove(&tc->bridge);
-
-	return 0;
 }
 
 static const struct i2c_device_id tc358767_i2c_ids[] = {
diff --git a/drivers/gpu/drm/bridge/tc358768.c b/drivers/gpu/drm/bridge/tc358768.c
index fd585bf925fe..4c4b77ce8aba 100644
--- a/drivers/gpu/drm/bridge/tc358768.c
+++ b/drivers/gpu/drm/bridge/tc358768.c
@@ -1072,13 +1072,11 @@ static int tc358768_i2c_probe(struct i2c_client *client,
 	return mipi_dsi_host_register(&priv->dsi_host);
 }
 
-static int tc358768_i2c_remove(struct i2c_client *client)
+static void tc358768_i2c_remove(struct i2c_client *client)
 {
 	struct tc358768_priv *priv = i2c_get_clientdata(client);
 
 	mipi_dsi_host_unregister(&priv->dsi_host);
-
-	return 0;
 }
 
 static struct i2c_driver tc358768_driver = {
diff --git a/drivers/gpu/drm/bridge/tc358775.c b/drivers/gpu/drm/bridge/tc358775.c
index f1c6e62b0e1d..02dc12b8151e 100644
--- a/drivers/gpu/drm/bridge/tc358775.c
+++ b/drivers/gpu/drm/bridge/tc358775.c
@@ -704,13 +704,11 @@ err_bridge_remove:
 	return ret;
 }
 
-static int tc_remove(struct i2c_client *client)
+static void tc_remove(struct i2c_client *client)
 {
 	struct tc_data *tc = i2c_get_clientdata(client);
 
 	drm_bridge_remove(&tc->bridge);
-
-	return 0;
 }
 
 static const struct i2c_device_id tc358775_i2c_ids[] = {
diff --git a/drivers/gpu/drm/bridge/ti-dlpc3433.c b/drivers/gpu/drm/bridge/ti-dlpc3433.c
index cef454862b67..186a9e2ff24d 100644
--- a/drivers/gpu/drm/bridge/ti-dlpc3433.c
+++ b/drivers/gpu/drm/bridge/ti-dlpc3433.c
@@ -379,14 +379,12 @@ err_remove_bridge:
 	return ret;
 }
 
-static int dlpc3433_remove(struct i2c_client *client)
+static void dlpc3433_remove(struct i2c_client *client)
 {
 	struct dlpc *dlpc = i2c_get_clientdata(client);
 
 	drm_bridge_remove(&dlpc->bridge);
 	of_node_put(dlpc->host_node);
-
-	return 0;
 }
 
 static const struct i2c_device_id dlpc3433_id[] = {
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
index 14e7aa77e758..7ba9467fff12 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
@@ -708,13 +708,11 @@ err_remove_bridge:
 	return ret;
 }
 
-static int sn65dsi83_remove(struct i2c_client *client)
+static void sn65dsi83_remove(struct i2c_client *client)
 {
 	struct sn65dsi83 *ctx = i2c_get_clientdata(client);
 
 	drm_bridge_remove(&ctx->bridge);
-
-	return 0;
 }
 
 static struct i2c_device_id sn65dsi83_id[] = {
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c
index d6dd4d99a229..3c3561942eb6 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c
@@ -29,6 +29,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_bridge_connector.h>
+#include <drm/drm_edid.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
@@ -68,6 +69,7 @@
 #define  BPP_18_RGB				BIT(0)
 #define SN_HPD_DISABLE_REG			0x5C
 #define  HPD_DISABLE				BIT(0)
+#define  HPD_DEBOUNCED_STATE			BIT(4)
 #define SN_GPIO_IO_REG				0x5E
 #define  SN_GPIO_INPUT_SHIFT			4
 #define  SN_GPIO_OUTPUT_SHIFT			0
@@ -92,6 +94,8 @@
 #define SN_DATARATE_CONFIG_REG			0x94
 #define  DP_DATARATE_MASK			GENMASK(7, 5)
 #define  DP_DATARATE(x)				((x) << 5)
+#define SN_TRAINING_SETTING_REG			0x95
+#define  SCRAMBLE_DISABLE			BIT(4)
 #define SN_ML_TX_MODE_REG			0x96
 #define  ML_TX_MAIN_LINK_OFF			0
 #define  ML_TX_NORMAL_MODE			BIT(0)
@@ -698,11 +702,6 @@ static int ti_sn_bridge_attach(struct drm_bridge *bridge,
 	struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge);
 	int ret;
 
-	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) {
-		DRM_ERROR("Fix bridge driver to make connector optional!");
-		return -EINVAL;
-	}
-
 	pdata->aux.drm_dev = bridge->dev;
 	ret = drm_dp_aux_register(&pdata->aux);
 	if (ret < 0) {
@@ -710,15 +709,18 @@ static int ti_sn_bridge_attach(struct drm_bridge *bridge,
 		return ret;
 	}
 
-	/* We never want the next bridge to *also* create a connector: */
-	flags |= DRM_BRIDGE_ATTACH_NO_CONNECTOR;
-
-	/* Attach the next bridge */
+	/*
+	 * Attach the next bridge.
+	 * We never want the next bridge to *also* create a connector.
+	 */
 	ret = drm_bridge_attach(bridge->encoder, pdata->next_bridge,
-				&pdata->bridge, flags);
+				&pdata->bridge, flags | DRM_BRIDGE_ATTACH_NO_CONNECTOR);
 	if (ret < 0)
 		goto err_initted_aux;
 
+	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)
+		return 0;
+
 	pdata->connector = drm_bridge_connector_init(pdata->bridge.dev,
 						     pdata->bridge.encoder);
 	if (IS_ERR(pdata->connector)) {
@@ -749,6 +751,29 @@ ti_sn_bridge_mode_valid(struct drm_bridge *bridge,
 	if (mode->clock > 594000)
 		return MODE_CLOCK_HIGH;
 
+	/*
+	 * The front and back porch registers are 8 bits, and pulse width
+	 * registers are 15 bits, so reject any modes with larger periods.
+	 */
+
+	if ((mode->hsync_start - mode->hdisplay) > 0xff)
+		return MODE_HBLANK_WIDE;
+
+	if ((mode->vsync_start - mode->vdisplay) > 0xff)
+		return MODE_VBLANK_WIDE;
+
+	if ((mode->hsync_end - mode->hsync_start) > 0x7fff)
+		return MODE_HSYNC_WIDE;
+
+	if ((mode->vsync_end - mode->vsync_start) > 0x7fff)
+		return MODE_VSYNC_WIDE;
+
+	if ((mode->htotal - mode->hsync_end) > 0xff)
+		return MODE_HBLANK_WIDE;
+
+	if ((mode->vtotal - mode->vsync_end) > 0xff)
+		return MODE_VBLANK_WIDE;
+
 	return MODE_OK;
 }
 
@@ -779,9 +804,9 @@ static void ti_sn_bridge_set_dsi_rate(struct ti_sn65dsi86 *pdata)
 	regmap_write(pdata->regmap, SN_DSIA_CLK_FREQ_REG, val);
 }
 
-static unsigned int ti_sn_bridge_get_bpp(struct ti_sn65dsi86 *pdata)
+static unsigned int ti_sn_bridge_get_bpp(struct drm_connector *connector)
 {
-	if (pdata->connector->display_info.bpc <= 6)
+	if (connector->display_info.bpc <= 6)
 		return 18;
 	else
 		return 24;
@@ -796,7 +821,7 @@ static const unsigned int ti_sn_bridge_dp_rate_lut[] = {
 	0, 1620, 2160, 2430, 2700, 3240, 4320, 5400
 };
 
-static int ti_sn_bridge_calc_min_dp_rate_idx(struct ti_sn65dsi86 *pdata)
+static int ti_sn_bridge_calc_min_dp_rate_idx(struct ti_sn65dsi86 *pdata, unsigned int bpp)
 {
 	unsigned int bit_rate_khz, dp_rate_mhz;
 	unsigned int i;
@@ -804,7 +829,7 @@ static int ti_sn_bridge_calc_min_dp_rate_idx(struct ti_sn65dsi86 *pdata)
 		&pdata->bridge.encoder->crtc->state->adjusted_mode;
 
 	/* Calculate minimum bit rate based on our pixel clock. */
-	bit_rate_khz = mode->clock * ti_sn_bridge_get_bpp(pdata);
+	bit_rate_khz = mode->clock * bpp;
 
 	/* Calculate minimum DP data rate, taking 80% as per DP spec */
 	dp_rate_mhz = DIV_ROUND_UP(bit_rate_khz * DP_CLK_FUDGE_NUM,
@@ -1016,12 +1041,21 @@ static void ti_sn_bridge_atomic_enable(struct drm_bridge *bridge,
 				       struct drm_bridge_state *old_bridge_state)
 {
 	struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge);
+	struct drm_connector *connector;
 	const char *last_err_str = "No supported DP rate";
 	unsigned int valid_rates;
 	int dp_rate_idx;
 	unsigned int val;
 	int ret = -EINVAL;
 	int max_dp_lanes;
+	unsigned int bpp;
+
+	connector = drm_atomic_get_new_connector_for_encoder(old_bridge_state->base.state,
+							     bridge->encoder);
+	if (!connector) {
+		dev_err_ratelimited(pdata->dev, "Could not get the connector\n");
+		return;
+	}
 
 	max_dp_lanes = ti_sn_get_max_lanes(pdata);
 	pdata->dp_lanes = min(pdata->dp_lanes, max_dp_lanes);
@@ -1040,15 +1074,27 @@ static void ti_sn_bridge_atomic_enable(struct drm_bridge *bridge,
 
 	/*
 	 * The SN65DSI86 only supports ASSR Display Authentication method and
-	 * this method is enabled by default. An eDP panel must support this
+	 * this method is enabled for eDP panels. An eDP panel must support this
 	 * authentication method. We need to enable this method in the eDP panel
 	 * at DisplayPort address 0x0010A prior to link training.
+	 *
+	 * As only ASSR is supported by SN65DSI86, for full DisplayPort displays
+	 * we need to disable the scrambler.
 	 */
-	drm_dp_dpcd_writeb(&pdata->aux, DP_EDP_CONFIGURATION_SET,
-			   DP_ALTERNATE_SCRAMBLER_RESET_ENABLE);
+	if (pdata->bridge.type == DRM_MODE_CONNECTOR_eDP) {
+		drm_dp_dpcd_writeb(&pdata->aux, DP_EDP_CONFIGURATION_SET,
+				   DP_ALTERNATE_SCRAMBLER_RESET_ENABLE);
 
+		regmap_update_bits(pdata->regmap, SN_TRAINING_SETTING_REG,
+				   SCRAMBLE_DISABLE, 0);
+	} else {
+		regmap_update_bits(pdata->regmap, SN_TRAINING_SETTING_REG,
+				   SCRAMBLE_DISABLE, SCRAMBLE_DISABLE);
+	}
+
+	bpp = ti_sn_bridge_get_bpp(connector);
 	/* Set the DP output format (18 bpp or 24 bpp) */
-	val = (ti_sn_bridge_get_bpp(pdata) == 18) ? BPP_18_RGB : 0;
+	val = bpp == 18 ? BPP_18_RGB : 0;
 	regmap_update_bits(pdata->regmap, SN_DATA_FORMAT_REG, BPP_18_RGB, val);
 
 	/* DP lane config */
@@ -1059,7 +1105,7 @@ static void ti_sn_bridge_atomic_enable(struct drm_bridge *bridge,
 	valid_rates = ti_sn_bridge_read_valid_rates(pdata);
 
 	/* Train until we run out of rates */
-	for (dp_rate_idx = ti_sn_bridge_calc_min_dp_rate_idx(pdata);
+	for (dp_rate_idx = ti_sn_bridge_calc_min_dp_rate_idx(pdata, bpp);
 	     dp_rate_idx < ARRAY_SIZE(ti_sn_bridge_dp_rate_lut);
 	     dp_rate_idx++) {
 		if (!(valid_rates & BIT(dp_rate_idx)))
@@ -1114,10 +1160,33 @@ static void ti_sn_bridge_atomic_post_disable(struct drm_bridge *bridge,
 	pm_runtime_put_sync(pdata->dev);
 }
 
+static enum drm_connector_status ti_sn_bridge_detect(struct drm_bridge *bridge)
+{
+	struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge);
+	int val = 0;
+
+	pm_runtime_get_sync(pdata->dev);
+	regmap_read(pdata->regmap, SN_HPD_DISABLE_REG, &val);
+	pm_runtime_put_autosuspend(pdata->dev);
+
+	return val & HPD_DEBOUNCED_STATE ? connector_status_connected
+					 : connector_status_disconnected;
+}
+
+static struct edid *ti_sn_bridge_get_edid(struct drm_bridge *bridge,
+					  struct drm_connector *connector)
+{
+	struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge);
+
+	return drm_get_edid(connector, &pdata->aux.ddc);
+}
+
 static const struct drm_bridge_funcs ti_sn_bridge_funcs = {
 	.attach = ti_sn_bridge_attach,
 	.detach = ti_sn_bridge_detach,
 	.mode_valid = ti_sn_bridge_mode_valid,
+	.get_edid = ti_sn_bridge_get_edid,
+	.detect = ti_sn_bridge_detect,
 	.atomic_pre_enable = ti_sn_bridge_atomic_pre_enable,
 	.atomic_enable = ti_sn_bridge_atomic_enable,
 	.atomic_disable = ti_sn_bridge_atomic_disable,
@@ -1198,10 +1267,9 @@ static int ti_sn_bridge_probe(struct auxiliary_device *adev,
 	int ret;
 
 	pdata->next_bridge = devm_drm_of_get_bridge(pdata->dev, np, 1, 0);
-	if (IS_ERR(pdata->next_bridge)) {
-		DRM_ERROR("failed to create panel bridge\n");
-		return PTR_ERR(pdata->next_bridge);
-	}
+	if (IS_ERR(pdata->next_bridge))
+		return dev_err_probe(pdata->dev, PTR_ERR(pdata->next_bridge),
+				     "failed to create panel bridge\n");
 
 	ti_sn_bridge_parse_lanes(pdata, np);
 
@@ -1211,6 +1279,11 @@ static int ti_sn_bridge_probe(struct auxiliary_device *adev,
 
 	pdata->bridge.funcs = &ti_sn_bridge_funcs;
 	pdata->bridge.of_node = np;
+	pdata->bridge.type = pdata->next_bridge->type == DRM_MODE_CONNECTOR_DisplayPort
+			   ? DRM_MODE_CONNECTOR_DisplayPort : DRM_MODE_CONNECTOR_eDP;
+
+	if (pdata->bridge.type == DRM_MODE_CONNECTOR_DisplayPort)
+		pdata->bridge.ops = DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_DETECT;
 
 	drm_bridge_add(&pdata->bridge);
 
diff --git a/drivers/gpu/drm/bridge/ti-tfp410.c b/drivers/gpu/drm/bridge/ti-tfp410.c
index 401fe61217c7..b9635abbad16 100644
--- a/drivers/gpu/drm/bridge/ti-tfp410.c
+++ b/drivers/gpu/drm/bridge/ti-tfp410.c
@@ -394,11 +394,9 @@ static int tfp410_i2c_probe(struct i2c_client *client,
 	return tfp410_init(&client->dev, true);
 }
 
-static int tfp410_i2c_remove(struct i2c_client *client)
+static void tfp410_i2c_remove(struct i2c_client *client)
 {
 	tfp410_fini(&client->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id tfp410_i2c_ids[] = {
diff --git a/drivers/gpu/drm/display/drm_dp_helper.c b/drivers/gpu/drm/display/drm_dp_helper.c
index e5bab236b3ae..9f055d9710ea 100644
--- a/drivers/gpu/drm/display/drm_dp_helper.c
+++ b/drivers/gpu/drm/display/drm_dp_helper.c
@@ -390,6 +390,38 @@ void drm_dp_link_train_channel_eq_delay(const struct drm_dp_aux *aux,
 }
 EXPORT_SYMBOL(drm_dp_link_train_channel_eq_delay);
 
+/**
+ * drm_dp_phy_name() - Get the name of the given DP PHY
+ * @dp_phy: The DP PHY identifier
+ *
+ * Given the @dp_phy, get a user friendly name of the DP PHY, either "DPRX" or
+ * "LTTPR <N>", or "<INVALID DP PHY>" on errors. The returned string is always
+ * non-NULL and valid.
+ *
+ * Returns: Name of the DP PHY.
+ */
+const char *drm_dp_phy_name(enum drm_dp_phy dp_phy)
+{
+	static const char * const phy_names[] = {
+		[DP_PHY_DPRX] = "DPRX",
+		[DP_PHY_LTTPR1] = "LTTPR 1",
+		[DP_PHY_LTTPR2] = "LTTPR 2",
+		[DP_PHY_LTTPR3] = "LTTPR 3",
+		[DP_PHY_LTTPR4] = "LTTPR 4",
+		[DP_PHY_LTTPR5] = "LTTPR 5",
+		[DP_PHY_LTTPR6] = "LTTPR 6",
+		[DP_PHY_LTTPR7] = "LTTPR 7",
+		[DP_PHY_LTTPR8] = "LTTPR 8",
+	};
+
+	if (dp_phy < 0 || dp_phy >= ARRAY_SIZE(phy_names) ||
+	    WARN_ON(!phy_names[dp_phy]))
+		return "<INVALID DP PHY>";
+
+	return phy_names[dp_phy];
+}
+EXPORT_SYMBOL(drm_dp_phy_name);
+
 void drm_dp_lttpr_link_train_clock_recovery_delay(void)
 {
 	usleep_range(100, 200);
@@ -1597,7 +1629,7 @@ static int drm_dp_aux_reply_duration(const struct drm_dp_aux_msg *msg)
 
 /*
  * Calculate the length of the i2c transfer in usec, assuming
- * the i2c bus speed is as specified. Gives the the "worst"
+ * the i2c bus speed is as specified. Gives the "worst"
  * case estimate, ie. successful while as long as possible.
  * Doesn't account the "MOT" bit, and instead assumes each
  * message includes a START, ADDRESS and STOP. Neither does it
@@ -2638,17 +2670,8 @@ int drm_dp_set_phy_test_pattern(struct drm_dp_aux *aux,
 				struct drm_dp_phy_test_params *data, u8 dp_rev)
 {
 	int err, i;
-	u8 link_config[2];
 	u8 test_pattern;
 
-	link_config[0] = drm_dp_link_rate_to_bw_code(data->link_rate);
-	link_config[1] = data->num_lanes;
-	if (data->enhanced_frame_cap)
-		link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
-	err = drm_dp_dpcd_write(aux, DP_LINK_BW_SET, link_config, 2);
-	if (err < 0)
-		return err;
-
 	test_pattern = data->phy_pattern;
 	if (dp_rev < 0x12) {
 		test_pattern = (test_pattern << 2) &
diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c
index 57e65423e50d..ecd22c038c8c 100644
--- a/drivers/gpu/drm/display/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c
@@ -68,8 +68,7 @@ static bool dump_dp_payload_table(struct drm_dp_mst_topology_mgr *mgr,
 static void drm_dp_mst_topology_put_port(struct drm_dp_mst_port *port);
 
 static int drm_dp_dpcd_write_payload(struct drm_dp_mst_topology_mgr *mgr,
-				     int id,
-				     struct drm_dp_payload *payload);
+				     int id, u8 start_slot, u8 num_slots);
 
 static int drm_dp_send_dpcd_read(struct drm_dp_mst_topology_mgr *mgr,
 				 struct drm_dp_mst_port *port,
@@ -1235,57 +1234,6 @@ build_query_stream_enc_status(struct drm_dp_sideband_msg_tx *msg, u8 stream_id,
 	return 0;
 }
 
-static int drm_dp_mst_assign_payload_id(struct drm_dp_mst_topology_mgr *mgr,
-					struct drm_dp_vcpi *vcpi)
-{
-	int ret, vcpi_ret;
-
-	mutex_lock(&mgr->payload_lock);
-	ret = find_first_zero_bit(&mgr->payload_mask, mgr->max_payloads + 1);
-	if (ret > mgr->max_payloads) {
-		ret = -EINVAL;
-		drm_dbg_kms(mgr->dev, "out of payload ids %d\n", ret);
-		goto out_unlock;
-	}
-
-	vcpi_ret = find_first_zero_bit(&mgr->vcpi_mask, mgr->max_payloads + 1);
-	if (vcpi_ret > mgr->max_payloads) {
-		ret = -EINVAL;
-		drm_dbg_kms(mgr->dev, "out of vcpi ids %d\n", ret);
-		goto out_unlock;
-	}
-
-	set_bit(ret, &mgr->payload_mask);
-	set_bit(vcpi_ret, &mgr->vcpi_mask);
-	vcpi->vcpi = vcpi_ret + 1;
-	mgr->proposed_vcpis[ret - 1] = vcpi;
-out_unlock:
-	mutex_unlock(&mgr->payload_lock);
-	return ret;
-}
-
-static void drm_dp_mst_put_payload_id(struct drm_dp_mst_topology_mgr *mgr,
-				      int vcpi)
-{
-	int i;
-
-	if (vcpi == 0)
-		return;
-
-	mutex_lock(&mgr->payload_lock);
-	drm_dbg_kms(mgr->dev, "putting payload %d\n", vcpi);
-	clear_bit(vcpi - 1, &mgr->vcpi_mask);
-
-	for (i = 0; i < mgr->max_payloads; i++) {
-		if (mgr->proposed_vcpis[i] &&
-		    mgr->proposed_vcpis[i]->vcpi == vcpi) {
-			mgr->proposed_vcpis[i] = NULL;
-			clear_bit(i + 1, &mgr->payload_mask);
-		}
-	}
-	mutex_unlock(&mgr->payload_lock);
-}
-
 static bool check_txmsg_state(struct drm_dp_mst_topology_mgr *mgr,
 			      struct drm_dp_sideband_msg_tx *txmsg)
 {
@@ -1738,6 +1686,20 @@ drm_dp_mst_dump_port_topology_history(struct drm_dp_mst_port *port) {}
 #define save_port_topology_ref(port, type)
 #endif
 
+struct drm_dp_mst_atomic_payload *
+drm_atomic_get_mst_payload_state(struct drm_dp_mst_topology_state *state,
+				 struct drm_dp_mst_port *port)
+{
+	struct drm_dp_mst_atomic_payload *payload;
+
+	list_for_each_entry(payload, &state->payloads, next)
+		if (payload->port == port)
+			return payload;
+
+	return NULL;
+}
+EXPORT_SYMBOL(drm_atomic_get_mst_payload_state);
+
 static void drm_dp_destroy_mst_branch_device(struct kref *kref)
 {
 	struct drm_dp_mst_branch *mstb =
@@ -2496,7 +2458,7 @@ fail_put:
 	return ret;
 }
 
-static void
+static int
 drm_dp_mst_handle_conn_stat(struct drm_dp_mst_branch *mstb,
 			    struct drm_dp_connection_status_notify *conn_stat)
 {
@@ -2509,7 +2471,7 @@ drm_dp_mst_handle_conn_stat(struct drm_dp_mst_branch *mstb,
 
 	port = drm_dp_get_port(mstb, conn_stat->port_number);
 	if (!port)
-		return;
+		return 0;
 
 	if (port->connector) {
 		if (!port->input && conn_stat->input_port) {
@@ -2562,8 +2524,7 @@ drm_dp_mst_handle_conn_stat(struct drm_dp_mst_branch *mstb,
 
 out:
 	drm_dp_mst_topology_put_port(port);
-	if (dowork)
-		queue_work(system_long_wq, &mstb->mgr->work);
+	return dowork;
 }
 
 static struct drm_dp_mst_branch *drm_dp_get_mst_branch_device(struct drm_dp_mst_topology_mgr *mgr,
@@ -3240,6 +3201,8 @@ int drm_dp_send_query_stream_enc_status(struct drm_dp_mst_topology_mgr *mgr,
 		struct drm_dp_mst_port *port,
 		struct drm_dp_query_stream_enc_status_ack_reply *status)
 {
+	struct drm_dp_mst_topology_state *state;
+	struct drm_dp_mst_atomic_payload *payload;
 	struct drm_dp_sideband_msg_tx *txmsg;
 	u8 nonce[7];
 	int ret;
@@ -3256,6 +3219,10 @@ int drm_dp_send_query_stream_enc_status(struct drm_dp_mst_topology_mgr *mgr,
 
 	get_random_bytes(nonce, sizeof(nonce));
 
+	drm_modeset_lock(&mgr->base.lock, NULL);
+	state = to_drm_dp_mst_topology_state(mgr->base.state);
+	payload = drm_atomic_get_mst_payload_state(state, port);
+
 	/*
 	 * "Source device targets the QUERY_STREAM_ENCRYPTION_STATUS message
 	 *  transaction at the MST Branch device directly connected to the
@@ -3263,7 +3230,7 @@ int drm_dp_send_query_stream_enc_status(struct drm_dp_mst_topology_mgr *mgr,
 	 */
 	txmsg->dst = mgr->mst_primary;
 
-	build_query_stream_enc_status(txmsg, port->vcpi.vcpi, nonce);
+	build_query_stream_enc_status(txmsg, payload->vcpi, nonce);
 
 	drm_dp_queue_down_tx(mgr, txmsg);
 
@@ -3280,6 +3247,7 @@ int drm_dp_send_query_stream_enc_status(struct drm_dp_mst_topology_mgr *mgr,
 	memcpy(status, &txmsg->reply.u.enc_status, sizeof(*status));
 
 out:
+	drm_modeset_unlock(&mgr->base.lock);
 	drm_dp_mst_topology_put_port(port);
 out_get_port:
 	kfree(txmsg);
@@ -3288,238 +3256,162 @@ out_get_port:
 EXPORT_SYMBOL(drm_dp_send_query_stream_enc_status);
 
 static int drm_dp_create_payload_step1(struct drm_dp_mst_topology_mgr *mgr,
-				       int id,
-				       struct drm_dp_payload *payload)
+				       struct drm_dp_mst_atomic_payload *payload)
 {
-	int ret;
-
-	ret = drm_dp_dpcd_write_payload(mgr, id, payload);
-	if (ret < 0) {
-		payload->payload_state = 0;
-		return ret;
-	}
-	payload->payload_state = DP_PAYLOAD_LOCAL;
-	return 0;
+	return drm_dp_dpcd_write_payload(mgr, payload->vcpi, payload->vc_start_slot,
+					 payload->time_slots);
 }
 
 static int drm_dp_create_payload_step2(struct drm_dp_mst_topology_mgr *mgr,
-				       struct drm_dp_mst_port *port,
-				       int id,
-				       struct drm_dp_payload *payload)
+				       struct drm_dp_mst_atomic_payload *payload)
 {
 	int ret;
+	struct drm_dp_mst_port *port = drm_dp_mst_topology_get_port_validated(mgr, payload->port);
 
-	ret = drm_dp_payload_send_msg(mgr, port, id, port->vcpi.pbn);
-	if (ret < 0)
-		return ret;
-	payload->payload_state = DP_PAYLOAD_REMOTE;
+	if (!port)
+		return -EIO;
+
+	ret = drm_dp_payload_send_msg(mgr, port, payload->vcpi, payload->pbn);
+	drm_dp_mst_topology_put_port(port);
 	return ret;
 }
 
 static int drm_dp_destroy_payload_step1(struct drm_dp_mst_topology_mgr *mgr,
-					struct drm_dp_mst_port *port,
-					int id,
-					struct drm_dp_payload *payload)
+					struct drm_dp_mst_topology_state *mst_state,
+					struct drm_dp_mst_atomic_payload *payload)
 {
 	drm_dbg_kms(mgr->dev, "\n");
+
 	/* it's okay for these to fail */
-	if (port) {
-		drm_dp_payload_send_msg(mgr, port, id, 0);
-	}
+	drm_dp_payload_send_msg(mgr, payload->port, payload->vcpi, 0);
+	drm_dp_dpcd_write_payload(mgr, payload->vcpi, payload->vc_start_slot, 0);
 
-	drm_dp_dpcd_write_payload(mgr, id, payload);
-	payload->payload_state = DP_PAYLOAD_DELETE_LOCAL;
-	return 0;
-}
-
-static int drm_dp_destroy_payload_step2(struct drm_dp_mst_topology_mgr *mgr,
-					int id,
-					struct drm_dp_payload *payload)
-{
-	payload->payload_state = 0;
 	return 0;
 }
 
 /**
- * drm_dp_update_payload_part1() - Execute payload update part 1
- * @mgr: manager to use.
- * @start_slot: this is the cur slot
- *
- * NOTE: start_slot is a temporary workaround for non-atomic drivers,
- * this will be removed when non-atomic mst helpers are moved out of the helper
+ * drm_dp_add_payload_part1() - Execute payload update part 1
+ * @mgr: Manager to use.
+ * @mst_state: The MST atomic state
+ * @payload: The payload to write
  *
- * This iterates over all proposed virtual channels, and tries to
- * allocate space in the link for them. For 0->slots transitions,
- * this step just writes the VCPI to the MST device. For slots->0
- * transitions, this writes the updated VCPIs and removes the
- * remote VC payloads.
+ * Determines the starting time slot for the given payload, and programs the VCPI for this payload
+ * into hardware. After calling this, the driver should generate ACT and payload packets.
  *
- * after calling this the driver should generate ACT and payload
- * packets.
+ * Returns: 0 on success, error code on failure. In the event that this fails,
+ * @payload.vc_start_slot will also be set to -1.
  */
-int drm_dp_update_payload_part1(struct drm_dp_mst_topology_mgr *mgr, int start_slot)
+int drm_dp_add_payload_part1(struct drm_dp_mst_topology_mgr *mgr,
+			     struct drm_dp_mst_topology_state *mst_state,
+			     struct drm_dp_mst_atomic_payload *payload)
 {
-	struct drm_dp_payload req_payload;
 	struct drm_dp_mst_port *port;
-	int i, j;
-	int cur_slots = start_slot;
-	bool skip;
+	int ret;
 
-	mutex_lock(&mgr->payload_lock);
-	for (i = 0; i < mgr->max_payloads; i++) {
-		struct drm_dp_vcpi *vcpi = mgr->proposed_vcpis[i];
-		struct drm_dp_payload *payload = &mgr->payloads[i];
-		bool put_port = false;
+	port = drm_dp_mst_topology_get_port_validated(mgr, payload->port);
+	if (!port)
+		return 0;
 
-		/* solve the current payloads - compare to the hw ones
-		   - update the hw view */
-		req_payload.start_slot = cur_slots;
-		if (vcpi) {
-			port = container_of(vcpi, struct drm_dp_mst_port,
-					    vcpi);
+	if (mgr->payload_count == 0)
+		mgr->next_start_slot = mst_state->start_slot;
 
-			mutex_lock(&mgr->lock);
-			skip = !drm_dp_mst_port_downstream_of_branch(port, mgr->mst_primary);
-			mutex_unlock(&mgr->lock);
+	payload->vc_start_slot = mgr->next_start_slot;
 
-			if (skip) {
-				drm_dbg_kms(mgr->dev,
-					    "Virtual channel %d is not in current topology\n",
-					    i);
-				continue;
-			}
-			/* Validated ports don't matter if we're releasing
-			 * VCPI
-			 */
-			if (vcpi->num_slots) {
-				port = drm_dp_mst_topology_get_port_validated(
-				    mgr, port);
-				if (!port) {
-					if (vcpi->num_slots == payload->num_slots) {
-						cur_slots += vcpi->num_slots;
-						payload->start_slot = req_payload.start_slot;
-						continue;
-					} else {
-						drm_dbg_kms(mgr->dev,
-							    "Fail:set payload to invalid sink");
-						mutex_unlock(&mgr->payload_lock);
-						return -EINVAL;
-					}
-				}
-				put_port = true;
-			}
+	ret = drm_dp_create_payload_step1(mgr, payload);
+	drm_dp_mst_topology_put_port(port);
+	if (ret < 0) {
+		drm_warn(mgr->dev, "Failed to create MST payload for port %p: %d\n",
+			 payload->port, ret);
+		payload->vc_start_slot = -1;
+		return ret;
+	}
 
-			req_payload.num_slots = vcpi->num_slots;
-			req_payload.vcpi = vcpi->vcpi;
-		} else {
-			port = NULL;
-			req_payload.num_slots = 0;
-		}
+	mgr->payload_count++;
+	mgr->next_start_slot += payload->time_slots;
 
-		payload->start_slot = req_payload.start_slot;
-		/* work out what is required to happen with this payload */
-		if (payload->num_slots != req_payload.num_slots) {
-
-			/* need to push an update for this payload */
-			if (req_payload.num_slots) {
-				drm_dp_create_payload_step1(mgr, vcpi->vcpi,
-							    &req_payload);
-				payload->num_slots = req_payload.num_slots;
-				payload->vcpi = req_payload.vcpi;
-
-			} else if (payload->num_slots) {
-				payload->num_slots = 0;
-				drm_dp_destroy_payload_step1(mgr, port,
-							     payload->vcpi,
-							     payload);
-				req_payload.payload_state =
-					payload->payload_state;
-				payload->start_slot = 0;
-			}
-			payload->payload_state = req_payload.payload_state;
-		}
-		cur_slots += req_payload.num_slots;
+	return 0;
+}
+EXPORT_SYMBOL(drm_dp_add_payload_part1);
 
-		if (put_port)
-			drm_dp_mst_topology_put_port(port);
-	}
+/**
+ * drm_dp_remove_payload() - Remove an MST payload
+ * @mgr: Manager to use.
+ * @mst_state: The MST atomic state
+ * @payload: The payload to write
+ *
+ * Removes a payload from an MST topology if it was successfully assigned a start slot. Also updates
+ * the starting time slots of all other payloads which would have been shifted towards the start of
+ * the VC table as a result. After calling this, the driver should generate ACT and payload packets.
+ */
+void drm_dp_remove_payload(struct drm_dp_mst_topology_mgr *mgr,
+			   struct drm_dp_mst_topology_state *mst_state,
+			   struct drm_dp_mst_atomic_payload *payload)
+{
+	struct drm_dp_mst_atomic_payload *pos;
+	bool send_remove = false;
 
-	for (i = 0; i < mgr->max_payloads; /* do nothing */) {
-		if (mgr->payloads[i].payload_state != DP_PAYLOAD_DELETE_LOCAL) {
-			i++;
-			continue;
-		}
+	/* We failed to make the payload, so nothing to do */
+	if (payload->vc_start_slot == -1)
+		return;
 
-		drm_dbg_kms(mgr->dev, "removing payload %d\n", i);
-		for (j = i; j < mgr->max_payloads - 1; j++) {
-			mgr->payloads[j] = mgr->payloads[j + 1];
-			mgr->proposed_vcpis[j] = mgr->proposed_vcpis[j + 1];
+	mutex_lock(&mgr->lock);
+	send_remove = drm_dp_mst_port_downstream_of_branch(payload->port, mgr->mst_primary);
+	mutex_unlock(&mgr->lock);
 
-			if (mgr->proposed_vcpis[j] &&
-			    mgr->proposed_vcpis[j]->num_slots) {
-				set_bit(j + 1, &mgr->payload_mask);
-			} else {
-				clear_bit(j + 1, &mgr->payload_mask);
-			}
-		}
+	if (send_remove)
+		drm_dp_destroy_payload_step1(mgr, mst_state, payload);
+	else
+		drm_dbg_kms(mgr->dev, "Payload for VCPI %d not in topology, not sending remove\n",
+			    payload->vcpi);
 
-		memset(&mgr->payloads[mgr->max_payloads - 1], 0,
-		       sizeof(struct drm_dp_payload));
-		mgr->proposed_vcpis[mgr->max_payloads - 1] = NULL;
-		clear_bit(mgr->max_payloads, &mgr->payload_mask);
+	list_for_each_entry(pos, &mst_state->payloads, next) {
+		if (pos != payload && pos->vc_start_slot > payload->vc_start_slot)
+			pos->vc_start_slot -= payload->time_slots;
 	}
-	mutex_unlock(&mgr->payload_lock);
+	payload->vc_start_slot = -1;
 
-	return 0;
+	mgr->payload_count--;
+	mgr->next_start_slot -= payload->time_slots;
 }
-EXPORT_SYMBOL(drm_dp_update_payload_part1);
+EXPORT_SYMBOL(drm_dp_remove_payload);
 
 /**
- * drm_dp_update_payload_part2() - Execute payload update part 2
- * @mgr: manager to use.
+ * drm_dp_add_payload_part2() - Execute payload update part 2
+ * @mgr: Manager to use.
+ * @state: The global atomic state
+ * @payload: The payload to update
+ *
+ * If @payload was successfully assigned a starting time slot by drm_dp_add_payload_part1(), this
+ * function will send the sideband messages to finish allocating this payload.
  *
- * This iterates over all proposed virtual channels, and tries to
- * allocate space in the link for them. For 0->slots transitions,
- * this step writes the remote VC payload commands. For slots->0
- * this just resets some internal state.
+ * Returns: 0 on success, negative error code on failure.
  */
-int drm_dp_update_payload_part2(struct drm_dp_mst_topology_mgr *mgr)
+int drm_dp_add_payload_part2(struct drm_dp_mst_topology_mgr *mgr,
+			     struct drm_atomic_state *state,
+			     struct drm_dp_mst_atomic_payload *payload)
 {
-	struct drm_dp_mst_port *port;
-	int i;
 	int ret = 0;
-	bool skip;
 
-	mutex_lock(&mgr->payload_lock);
-	for (i = 0; i < mgr->max_payloads; i++) {
-
-		if (!mgr->proposed_vcpis[i])
-			continue;
-
-		port = container_of(mgr->proposed_vcpis[i], struct drm_dp_mst_port, vcpi);
-
-		mutex_lock(&mgr->lock);
-		skip = !drm_dp_mst_port_downstream_of_branch(port, mgr->mst_primary);
-		mutex_unlock(&mgr->lock);
-
-		if (skip)
-			continue;
+	/* Skip failed payloads */
+	if (payload->vc_start_slot == -1) {
+		drm_dbg_kms(state->dev, "Part 1 of payload creation for %s failed, skipping part 2\n",
+			    payload->port->connector->name);
+		return -EIO;
+	}
 
-		drm_dbg_kms(mgr->dev, "payload %d %d\n", i, mgr->payloads[i].payload_state);
-		if (mgr->payloads[i].payload_state == DP_PAYLOAD_LOCAL) {
-			ret = drm_dp_create_payload_step2(mgr, port, mgr->proposed_vcpis[i]->vcpi, &mgr->payloads[i]);
-		} else if (mgr->payloads[i].payload_state == DP_PAYLOAD_DELETE_LOCAL) {
-			ret = drm_dp_destroy_payload_step2(mgr, mgr->proposed_vcpis[i]->vcpi, &mgr->payloads[i]);
-		}
-		if (ret) {
-			mutex_unlock(&mgr->payload_lock);
-			return ret;
-		}
+	ret = drm_dp_create_payload_step2(mgr, payload);
+	if (ret < 0) {
+		if (!payload->delete)
+			drm_err(mgr->dev, "Step 2 of creating MST payload for %p failed: %d\n",
+				payload->port, ret);
+		else
+			drm_dbg_kms(mgr->dev, "Step 2 of removing MST payload for %p failed: %d\n",
+				    payload->port, ret);
 	}
-	mutex_unlock(&mgr->payload_lock);
-	return 0;
+
+	return ret;
 }
-EXPORT_SYMBOL(drm_dp_update_payload_part2);
+EXPORT_SYMBOL(drm_dp_add_payload_part2);
 
 static int drm_dp_send_dpcd_read(struct drm_dp_mst_topology_mgr *mgr,
 				 struct drm_dp_mst_port *port,
@@ -3699,7 +3591,6 @@ int drm_dp_mst_topology_mgr_set_mst(struct drm_dp_mst_topology_mgr *mgr, bool ms
 	int ret = 0;
 	struct drm_dp_mst_branch *mstb = NULL;
 
-	mutex_lock(&mgr->payload_lock);
 	mutex_lock(&mgr->lock);
 	if (mst_state == mgr->mst_state)
 		goto out_unlock;
@@ -3707,10 +3598,6 @@ int drm_dp_mst_topology_mgr_set_mst(struct drm_dp_mst_topology_mgr *mgr, bool ms
 	mgr->mst_state = mst_state;
 	/* set the device into MST mode */
 	if (mst_state) {
-		struct drm_dp_payload reset_pay;
-		int lane_count;
-		int link_rate;
-
 		WARN_ON(mgr->mst_primary);
 
 		/* get dpcd info */
@@ -3721,16 +3608,6 @@ int drm_dp_mst_topology_mgr_set_mst(struct drm_dp_mst_topology_mgr *mgr, bool ms
 			goto out_unlock;
 		}
 
-		lane_count = min_t(int, mgr->dpcd[2] & DP_MAX_LANE_COUNT_MASK, mgr->max_lane_count);
-		link_rate = min_t(int, drm_dp_bw_code_to_link_rate(mgr->dpcd[1]), mgr->max_link_rate);
-		mgr->pbn_div = drm_dp_get_vc_payload_bw(mgr,
-							link_rate,
-							lane_count);
-		if (mgr->pbn_div == 0) {
-			ret = -EINVAL;
-			goto out_unlock;
-		}
-
 		/* add initial branch device at LCT 1 */
 		mstb = drm_dp_add_mst_branch_device(1, NULL);
 		if (mstb == NULL) {
@@ -3750,9 +3627,8 @@ int drm_dp_mst_topology_mgr_set_mst(struct drm_dp_mst_topology_mgr *mgr, bool ms
 		if (ret < 0)
 			goto out_unlock;
 
-		reset_pay.start_slot = 0;
-		reset_pay.num_slots = 0x3f;
-		drm_dp_dpcd_write_payload(mgr, 0, &reset_pay);
+		/* Write reset payload */
+		drm_dp_dpcd_write_payload(mgr, 0, 0, 0x3f);
 
 		queue_work(system_long_wq, &mgr->work);
 
@@ -3764,19 +3640,11 @@ int drm_dp_mst_topology_mgr_set_mst(struct drm_dp_mst_topology_mgr *mgr, bool ms
 		/* this can fail if the device is gone */
 		drm_dp_dpcd_writeb(mgr->aux, DP_MSTM_CTRL, 0);
 		ret = 0;
-		memset(mgr->payloads, 0,
-		       mgr->max_payloads * sizeof(mgr->payloads[0]));
-		memset(mgr->proposed_vcpis, 0,
-		       mgr->max_payloads * sizeof(mgr->proposed_vcpis[0]));
-		mgr->payload_mask = 0;
-		set_bit(0, &mgr->payload_mask);
-		mgr->vcpi_mask = 0;
 		mgr->payload_id_table_cleared = false;
 	}
 
 out_unlock:
 	mutex_unlock(&mgr->lock);
-	mutex_unlock(&mgr->payload_lock);
 	if (mstb)
 		drm_dp_mst_topology_put_mstb(mstb);
 	return ret;
@@ -4047,7 +3915,7 @@ drm_dp_mst_process_up_req(struct drm_dp_mst_topology_mgr *mgr,
 	struct drm_dp_mst_branch *mstb = NULL;
 	struct drm_dp_sideband_msg_req_body *msg = &up_req->msg;
 	struct drm_dp_sideband_msg_hdr *hdr = &up_req->hdr;
-	bool hotplug = false;
+	bool hotplug = false, dowork = false;
 
 	if (hdr->broadcast) {
 		const u8 *guid = NULL;
@@ -4070,11 +3938,14 @@ drm_dp_mst_process_up_req(struct drm_dp_mst_topology_mgr *mgr,
 
 	/* TODO: Add missing handler for DP_RESOURCE_STATUS_NOTIFY events */
 	if (msg->req_type == DP_CONNECTION_STATUS_NOTIFY) {
-		drm_dp_mst_handle_conn_stat(mstb, &msg->u.conn_stat);
+		dowork = drm_dp_mst_handle_conn_stat(mstb, &msg->u.conn_stat);
 		hotplug = true;
 	}
 
 	drm_dp_mst_topology_put_mstb(mstb);
+
+	if (dowork)
+		queue_work(system_long_wq, &mgr->work);
 	return hotplug;
 }
 
@@ -4293,341 +4164,352 @@ struct edid *drm_dp_mst_get_edid(struct drm_connector *connector, struct drm_dp_
 EXPORT_SYMBOL(drm_dp_mst_get_edid);
 
 /**
- * drm_dp_find_vcpi_slots() - Find VCPI slots for this PBN value
- * @mgr: manager to use
- * @pbn: payload bandwidth to convert into slots.
- *
- * Calculate the number of VCPI slots that will be required for the given PBN
- * value. This function is deprecated, and should not be used in atomic
- * drivers.
- *
- * RETURNS:
- * The total slots required for this port, or error.
- */
-int drm_dp_find_vcpi_slots(struct drm_dp_mst_topology_mgr *mgr,
-			   int pbn)
-{
-	int num_slots;
-
-	num_slots = DIV_ROUND_UP(pbn, mgr->pbn_div);
-
-	/* max. time slots - one slot for MTP header */
-	if (num_slots > 63)
-		return -ENOSPC;
-	return num_slots;
-}
-EXPORT_SYMBOL(drm_dp_find_vcpi_slots);
-
-static int drm_dp_init_vcpi(struct drm_dp_mst_topology_mgr *mgr,
-			    struct drm_dp_vcpi *vcpi, int pbn, int slots)
-{
-	int ret;
-
-	vcpi->pbn = pbn;
-	vcpi->aligned_pbn = slots * mgr->pbn_div;
-	vcpi->num_slots = slots;
-
-	ret = drm_dp_mst_assign_payload_id(mgr, vcpi);
-	if (ret < 0)
-		return ret;
-	return 0;
-}
-
-/**
- * drm_dp_atomic_find_vcpi_slots() - Find and add VCPI slots to the state
+ * drm_dp_atomic_find_time_slots() - Find and add time slots to the state
  * @state: global atomic state
  * @mgr: MST topology manager for the port
- * @port: port to find vcpi slots for
+ * @port: port to find time slots for
  * @pbn: bandwidth required for the mode in PBN
- * @pbn_div: divider for DSC mode that takes FEC into account
  *
- * Allocates VCPI slots to @port, replacing any previous VCPI allocations it
- * may have had. Any atomic drivers which support MST must call this function
- * in their &drm_encoder_helper_funcs.atomic_check() callback to change the
- * current VCPI allocation for the new state, but only when
- * &drm_crtc_state.mode_changed or &drm_crtc_state.connectors_changed is set
- * to ensure compatibility with userspace applications that still use the
- * legacy modesetting UAPI.
+ * Allocates time slots to @port, replacing any previous time slot allocations it may
+ * have had. Any atomic drivers which support MST must call this function in
+ * their &drm_encoder_helper_funcs.atomic_check() callback unconditionally to
+ * change the current time slot allocation for the new state, and ensure the MST
+ * atomic state is added whenever the state of payloads in the topology changes.
  *
  * Allocations set by this function are not checked against the bandwidth
  * restraints of @mgr until the driver calls drm_dp_mst_atomic_check().
  *
  * Additionally, it is OK to call this function multiple times on the same
  * @port as needed. It is not OK however, to call this function and
- * drm_dp_atomic_release_vcpi_slots() in the same atomic check phase.
+ * drm_dp_atomic_release_time_slots() in the same atomic check phase.
  *
  * See also:
- * drm_dp_atomic_release_vcpi_slots()
+ * drm_dp_atomic_release_time_slots()
  * drm_dp_mst_atomic_check()
  *
  * Returns:
  * Total slots in the atomic state assigned for this port, or a negative error
  * code if the port no longer exists
  */
-int drm_dp_atomic_find_vcpi_slots(struct drm_atomic_state *state,
+int drm_dp_atomic_find_time_slots(struct drm_atomic_state *state,
 				  struct drm_dp_mst_topology_mgr *mgr,
-				  struct drm_dp_mst_port *port, int pbn,
-				  int pbn_div)
+				  struct drm_dp_mst_port *port, int pbn)
 {
 	struct drm_dp_mst_topology_state *topology_state;
-	struct drm_dp_vcpi_allocation *pos, *vcpi = NULL;
-	int prev_slots, prev_bw, req_slots;
+	struct drm_dp_mst_atomic_payload *payload = NULL;
+	struct drm_connector_state *conn_state;
+	int prev_slots = 0, prev_bw = 0, req_slots;
 
 	topology_state = drm_atomic_get_mst_topology_state(state, mgr);
 	if (IS_ERR(topology_state))
 		return PTR_ERR(topology_state);
 
-	/* Find the current allocation for this port, if any */
-	list_for_each_entry(pos, &topology_state->vcpis, next) {
-		if (pos->port == port) {
-			vcpi = pos;
-			prev_slots = vcpi->vcpi;
-			prev_bw = vcpi->pbn;
+	conn_state = drm_atomic_get_new_connector_state(state, port->connector);
+	topology_state->pending_crtc_mask |= drm_crtc_mask(conn_state->crtc);
 
-			/*
-			 * This should never happen, unless the driver tries
-			 * releasing and allocating the same VCPI allocation,
-			 * which is an error
-			 */
-			if (WARN_ON(!prev_slots)) {
-				drm_err(mgr->dev,
-					"cannot allocate and release VCPI on [MST PORT:%p] in the same state\n",
-					port);
-				return -EINVAL;
-			}
+	/* Find the current allocation for this port, if any */
+	payload = drm_atomic_get_mst_payload_state(topology_state, port);
+	if (payload) {
+		prev_slots = payload->time_slots;
+		prev_bw = payload->pbn;
 
-			break;
+		/*
+		 * This should never happen, unless the driver tries
+		 * releasing and allocating the same timeslot allocation,
+		 * which is an error
+		 */
+		if (drm_WARN_ON(mgr->dev, payload->delete)) {
+			drm_err(mgr->dev,
+				"cannot allocate and release time slots on [MST PORT:%p] in the same state\n",
+				port);
+			return -EINVAL;
 		}
 	}
-	if (!vcpi) {
-		prev_slots = 0;
-		prev_bw = 0;
-	}
-
-	if (pbn_div <= 0)
-		pbn_div = mgr->pbn_div;
 
-	req_slots = DIV_ROUND_UP(pbn, pbn_div);
+	req_slots = DIV_ROUND_UP(pbn, topology_state->pbn_div);
 
-	drm_dbg_atomic(mgr->dev, "[CONNECTOR:%d:%s] [MST PORT:%p] VCPI %d -> %d\n",
+	drm_dbg_atomic(mgr->dev, "[CONNECTOR:%d:%s] [MST PORT:%p] TU %d -> %d\n",
 		       port->connector->base.id, port->connector->name,
 		       port, prev_slots, req_slots);
 	drm_dbg_atomic(mgr->dev, "[CONNECTOR:%d:%s] [MST PORT:%p] PBN %d -> %d\n",
 		       port->connector->base.id, port->connector->name,
 		       port, prev_bw, pbn);
 
-	/* Add the new allocation to the state */
-	if (!vcpi) {
-		vcpi = kzalloc(sizeof(*vcpi), GFP_KERNEL);
-		if (!vcpi)
+	/* Add the new allocation to the state, note the VCPI isn't assigned until the end */
+	if (!payload) {
+		payload = kzalloc(sizeof(*payload), GFP_KERNEL);
+		if (!payload)
 			return -ENOMEM;
 
 		drm_dp_mst_get_port_malloc(port);
-		vcpi->port = port;
-		list_add(&vcpi->next, &topology_state->vcpis);
+		payload->port = port;
+		payload->vc_start_slot = -1;
+		list_add(&payload->next, &topology_state->payloads);
 	}
-	vcpi->vcpi = req_slots;
-	vcpi->pbn = pbn;
+	payload->time_slots = req_slots;
+	payload->pbn = pbn;
 
 	return req_slots;
 }
-EXPORT_SYMBOL(drm_dp_atomic_find_vcpi_slots);
+EXPORT_SYMBOL(drm_dp_atomic_find_time_slots);
 
 /**
- * drm_dp_atomic_release_vcpi_slots() - Release allocated vcpi slots
+ * drm_dp_atomic_release_time_slots() - Release allocated time slots
  * @state: global atomic state
  * @mgr: MST topology manager for the port
- * @port: The port to release the VCPI slots from
+ * @port: The port to release the time slots from
  *
- * Releases any VCPI slots that have been allocated to a port in the atomic
- * state. Any atomic drivers which support MST must call this function in
- * their &drm_connector_helper_funcs.atomic_check() callback when the
- * connector will no longer have VCPI allocated (e.g. because its CRTC was
- * removed) when it had VCPI allocated in the previous atomic state.
+ * Releases any time slots that have been allocated to a port in the atomic
+ * state. Any atomic drivers which support MST must call this function
+ * unconditionally in their &drm_connector_helper_funcs.atomic_check() callback.
+ * This helper will check whether time slots would be released by the new state and
+ * respond accordingly, along with ensuring the MST state is always added to the
+ * atomic state whenever a new state would modify the state of payloads on the
+ * topology.
  *
  * It is OK to call this even if @port has been removed from the system.
  * Additionally, it is OK to call this function multiple times on the same
  * @port as needed. It is not OK however, to call this function and
- * drm_dp_atomic_find_vcpi_slots() on the same @port in a single atomic check
+ * drm_dp_atomic_find_time_slots() on the same @port in a single atomic check
  * phase.
  *
  * See also:
- * drm_dp_atomic_find_vcpi_slots()
+ * drm_dp_atomic_find_time_slots()
  * drm_dp_mst_atomic_check()
  *
  * Returns:
- * 0 if all slots for this port were added back to
- * &drm_dp_mst_topology_state.avail_slots or negative error code
+ * 0 on success, negative error code otherwise
  */
-int drm_dp_atomic_release_vcpi_slots(struct drm_atomic_state *state,
+int drm_dp_atomic_release_time_slots(struct drm_atomic_state *state,
 				     struct drm_dp_mst_topology_mgr *mgr,
 				     struct drm_dp_mst_port *port)
 {
 	struct drm_dp_mst_topology_state *topology_state;
-	struct drm_dp_vcpi_allocation *pos;
-	bool found = false;
+	struct drm_dp_mst_atomic_payload *payload;
+	struct drm_connector_state *old_conn_state, *new_conn_state;
+	bool update_payload = true;
+
+	old_conn_state = drm_atomic_get_old_connector_state(state, port->connector);
+	if (!old_conn_state->crtc)
+		return 0;
+
+	/* If the CRTC isn't disabled by this state, don't release it's payload */
+	new_conn_state = drm_atomic_get_new_connector_state(state, port->connector);
+	if (new_conn_state->crtc) {
+		struct drm_crtc_state *crtc_state =
+			drm_atomic_get_new_crtc_state(state, new_conn_state->crtc);
+
+		/* No modeset means no payload changes, so it's safe to not pull in the MST state */
+		if (!crtc_state || !drm_atomic_crtc_needs_modeset(crtc_state))
+			return 0;
+
+		if (!crtc_state->mode_changed && !crtc_state->connectors_changed)
+			update_payload = false;
+	}
 
 	topology_state = drm_atomic_get_mst_topology_state(state, mgr);
 	if (IS_ERR(topology_state))
 		return PTR_ERR(topology_state);
 
-	list_for_each_entry(pos, &topology_state->vcpis, next) {
-		if (pos->port == port) {
-			found = true;
-			break;
-		}
-	}
-	if (WARN_ON(!found)) {
-		drm_err(mgr->dev, "no VCPI for [MST PORT:%p] found in mst state %p\n",
+	topology_state->pending_crtc_mask |= drm_crtc_mask(old_conn_state->crtc);
+	if (!update_payload)
+		return 0;
+
+	payload = drm_atomic_get_mst_payload_state(topology_state, port);
+	if (WARN_ON(!payload)) {
+		drm_err(mgr->dev, "No payload for [MST PORT:%p] found in mst state %p\n",
 			port, &topology_state->base);
 		return -EINVAL;
 	}
 
-	drm_dbg_atomic(mgr->dev, "[MST PORT:%p] VCPI %d -> 0\n", port, pos->vcpi);
-	if (pos->vcpi) {
+	if (new_conn_state->crtc)
+		return 0;
+
+	drm_dbg_atomic(mgr->dev, "[MST PORT:%p] TU %d -> 0\n", port, payload->time_slots);
+	if (!payload->delete) {
 		drm_dp_mst_put_port_malloc(port);
-		pos->vcpi = 0;
-		pos->pbn = 0;
+		payload->pbn = 0;
+		payload->delete = true;
+		topology_state->payload_mask &= ~BIT(payload->vcpi - 1);
 	}
 
 	return 0;
 }
-EXPORT_SYMBOL(drm_dp_atomic_release_vcpi_slots);
+EXPORT_SYMBOL(drm_dp_atomic_release_time_slots);
 
 /**
- * drm_dp_mst_update_slots() - updates the slot info depending on the DP ecoding format
- * @mst_state: mst_state to update
- * @link_encoding_cap: the ecoding format on the link
- */
-void drm_dp_mst_update_slots(struct drm_dp_mst_topology_state *mst_state, uint8_t link_encoding_cap)
-{
-	if (link_encoding_cap == DP_CAP_ANSI_128B132B) {
-		mst_state->total_avail_slots = 64;
-		mst_state->start_slot = 0;
-	} else {
-		mst_state->total_avail_slots = 63;
-		mst_state->start_slot = 1;
-	}
-
-	DRM_DEBUG_KMS("%s encoding format on mst_state 0x%p\n",
-		      (link_encoding_cap == DP_CAP_ANSI_128B132B) ? "128b/132b":"8b/10b",
-		      mst_state);
-}
-EXPORT_SYMBOL(drm_dp_mst_update_slots);
-
-/**
- * drm_dp_mst_allocate_vcpi() - Allocate a virtual channel
- * @mgr: manager for this port
- * @port: port to allocate a virtual channel for.
- * @pbn: payload bandwidth number to request
- * @slots: returned number of slots for this PBN.
+ * drm_dp_mst_atomic_setup_commit() - setup_commit hook for MST helpers
+ * @state: global atomic state
+ *
+ * This function saves all of the &drm_crtc_commit structs in an atomic state that touch any CRTCs
+ * currently assigned to an MST topology. Drivers must call this hook from their
+ * &drm_mode_config_helper_funcs.atomic_commit_setup hook.
+ *
+ * Returns:
+ * 0 if all CRTC commits were retrieved successfully, negative error code otherwise
  */
-bool drm_dp_mst_allocate_vcpi(struct drm_dp_mst_topology_mgr *mgr,
-			      struct drm_dp_mst_port *port, int pbn, int slots)
+int drm_dp_mst_atomic_setup_commit(struct drm_atomic_state *state)
 {
-	int ret;
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_topology_state *mst_state;
+	struct drm_crtc *crtc;
+	struct drm_crtc_state *crtc_state;
+	int i, j, commit_idx, num_commit_deps;
 
-	if (slots < 0)
-		return false;
+	for_each_new_mst_mgr_in_state(state, mgr, mst_state, i) {
+		if (!mst_state->pending_crtc_mask)
+			continue;
 
-	port = drm_dp_mst_topology_get_port_validated(mgr, port);
-	if (!port)
-		return false;
+		num_commit_deps = hweight32(mst_state->pending_crtc_mask);
+		mst_state->commit_deps = kmalloc_array(num_commit_deps,
+						       sizeof(*mst_state->commit_deps), GFP_KERNEL);
+		if (!mst_state->commit_deps)
+			return -ENOMEM;
+		mst_state->num_commit_deps = num_commit_deps;
 
-	if (port->vcpi.vcpi > 0) {
-		drm_dbg_kms(mgr->dev,
-			    "payload: vcpi %d already allocated for pbn %d - requested pbn %d\n",
-			    port->vcpi.vcpi, port->vcpi.pbn, pbn);
-		if (pbn == port->vcpi.pbn) {
-			drm_dp_mst_topology_put_port(port);
-			return true;
+		commit_idx = 0;
+		for_each_new_crtc_in_state(state, crtc, crtc_state, j) {
+			if (mst_state->pending_crtc_mask & drm_crtc_mask(crtc)) {
+				mst_state->commit_deps[commit_idx++] =
+					drm_crtc_commit_get(crtc_state->commit);
+			}
 		}
 	}
 
-	ret = drm_dp_init_vcpi(mgr, &port->vcpi, pbn, slots);
-	if (ret) {
-		drm_dbg_kms(mgr->dev, "failed to init vcpi slots=%d ret=%d\n",
-			    DIV_ROUND_UP(pbn, mgr->pbn_div), ret);
-		drm_dp_mst_topology_put_port(port);
-		goto out;
-	}
-	drm_dbg_kms(mgr->dev, "initing vcpi for pbn=%d slots=%d\n", pbn, port->vcpi.num_slots);
-
-	/* Keep port allocated until its payload has been removed */
-	drm_dp_mst_get_port_malloc(port);
-	drm_dp_mst_topology_put_port(port);
-	return true;
-out:
-	return false;
+	return 0;
 }
-EXPORT_SYMBOL(drm_dp_mst_allocate_vcpi);
+EXPORT_SYMBOL(drm_dp_mst_atomic_setup_commit);
 
-int drm_dp_mst_get_vcpi_slots(struct drm_dp_mst_topology_mgr *mgr, struct drm_dp_mst_port *port)
+/**
+ * drm_dp_mst_atomic_wait_for_dependencies() - Wait for all pending commits on MST topologies,
+ * prepare new MST state for commit
+ * @state: global atomic state
+ *
+ * Goes through any MST topologies in this atomic state, and waits for any pending commits which
+ * touched CRTCs that were/are on an MST topology to be programmed to hardware and flipped to before
+ * returning. This is to prevent multiple non-blocking commits affecting an MST topology from racing
+ * with eachother by forcing them to be executed sequentially in situations where the only resources
+ * the modeset objects in these commits share are an MST topology.
+ *
+ * This function also prepares the new MST state for commit by performing some state preparation
+ * which can't be done until this point, such as reading back the final VC start slots (which are
+ * determined at commit-time) from the previous state.
+ *
+ * All MST drivers must call this function after calling drm_atomic_helper_wait_for_dependencies(),
+ * or whatever their equivalent of that is.
+ */
+void drm_dp_mst_atomic_wait_for_dependencies(struct drm_atomic_state *state)
 {
-	int slots = 0;
+	struct drm_dp_mst_topology_state *old_mst_state, *new_mst_state;
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_atomic_payload *old_payload, *new_payload;
+	int i, j, ret;
+
+	for_each_oldnew_mst_mgr_in_state(state, mgr, old_mst_state, new_mst_state, i) {
+		for (j = 0; j < old_mst_state->num_commit_deps; j++) {
+			ret = drm_crtc_commit_wait(old_mst_state->commit_deps[j]);
+			if (ret < 0)
+				drm_err(state->dev, "Failed to wait for %s: %d\n",
+					old_mst_state->commit_deps[j]->crtc->name, ret);
+		}
 
-	port = drm_dp_mst_topology_get_port_validated(mgr, port);
-	if (!port)
-		return slots;
+		/* Now that previous state is committed, it's safe to copy over the start slot
+		 * assignments
+		 */
+		list_for_each_entry(old_payload, &old_mst_state->payloads, next) {
+			if (old_payload->delete)
+				continue;
 
-	slots = port->vcpi.num_slots;
-	drm_dp_mst_topology_put_port(port);
-	return slots;
+			new_payload = drm_atomic_get_mst_payload_state(new_mst_state,
+								       old_payload->port);
+			new_payload->vc_start_slot = old_payload->vc_start_slot;
+		}
+	}
 }
-EXPORT_SYMBOL(drm_dp_mst_get_vcpi_slots);
+EXPORT_SYMBOL(drm_dp_mst_atomic_wait_for_dependencies);
 
 /**
- * drm_dp_mst_reset_vcpi_slots() - Reset number of slots to 0 for VCPI
- * @mgr: manager for this port
- * @port: unverified pointer to a port.
+ * drm_dp_mst_root_conn_atomic_check() - Serialize CRTC commits on MST-capable connectors operating
+ * in SST mode
+ * @new_conn_state: The new connector state of the &drm_connector
+ * @mgr: The MST topology manager for the &drm_connector
+ *
+ * Since MST uses fake &drm_encoder structs, the generic atomic modesetting code isn't able to
+ * serialize non-blocking commits happening on the real DP connector of an MST topology switching
+ * into/away from MST mode - as the CRTC on the real DP connector and the CRTCs on the connector's
+ * MST topology will never share the same &drm_encoder.
+ *
+ * This function takes care of this serialization issue, by checking a root MST connector's atomic
+ * state to determine if it is about to have a modeset - and then pulling in the MST topology state
+ * if so, along with adding any relevant CRTCs to &drm_dp_mst_topology_state.pending_crtc_mask.
  *
- * This just resets the number of slots for the ports VCPI for later programming.
+ * Drivers implementing MST must call this function from the
+ * &drm_connector_helper_funcs.atomic_check hook of any physical DP &drm_connector capable of
+ * driving MST sinks.
+ *
+ * Returns:
+ * 0 on success, negative error code otherwise
  */
-void drm_dp_mst_reset_vcpi_slots(struct drm_dp_mst_topology_mgr *mgr, struct drm_dp_mst_port *port)
+int drm_dp_mst_root_conn_atomic_check(struct drm_connector_state *new_conn_state,
+				      struct drm_dp_mst_topology_mgr *mgr)
 {
-	/*
-	 * A port with VCPI will remain allocated until its VCPI is
-	 * released, no verified ref needed
-	 */
+	struct drm_atomic_state *state = new_conn_state->state;
+	struct drm_connector_state *old_conn_state =
+		drm_atomic_get_old_connector_state(state, new_conn_state->connector);
+	struct drm_crtc_state *crtc_state;
+	struct drm_dp_mst_topology_state *mst_state = NULL;
+
+	if (new_conn_state->crtc) {
+		crtc_state = drm_atomic_get_new_crtc_state(state, new_conn_state->crtc);
+		if (crtc_state && drm_atomic_crtc_needs_modeset(crtc_state)) {
+			mst_state = drm_atomic_get_mst_topology_state(state, mgr);
+			if (IS_ERR(mst_state))
+				return PTR_ERR(mst_state);
+
+			mst_state->pending_crtc_mask |= drm_crtc_mask(new_conn_state->crtc);
+		}
+	}
+
+	if (old_conn_state->crtc) {
+		crtc_state = drm_atomic_get_new_crtc_state(state, old_conn_state->crtc);
+		if (crtc_state && drm_atomic_crtc_needs_modeset(crtc_state)) {
+			if (!mst_state) {
+				mst_state = drm_atomic_get_mst_topology_state(state, mgr);
+				if (IS_ERR(mst_state))
+					return PTR_ERR(mst_state);
+			}
+
+			mst_state->pending_crtc_mask |= drm_crtc_mask(old_conn_state->crtc);
+		}
+	}
 
-	port->vcpi.num_slots = 0;
+	return 0;
 }
-EXPORT_SYMBOL(drm_dp_mst_reset_vcpi_slots);
+EXPORT_SYMBOL(drm_dp_mst_root_conn_atomic_check);
 
 /**
- * drm_dp_mst_deallocate_vcpi() - deallocate a VCPI
- * @mgr: manager for this port
- * @port: port to deallocate vcpi for
- *
- * This can be called unconditionally, regardless of whether
- * drm_dp_mst_allocate_vcpi() succeeded or not.
+ * drm_dp_mst_update_slots() - updates the slot info depending on the DP ecoding format
+ * @mst_state: mst_state to update
+ * @link_encoding_cap: the ecoding format on the link
  */
-void drm_dp_mst_deallocate_vcpi(struct drm_dp_mst_topology_mgr *mgr,
-				struct drm_dp_mst_port *port)
+void drm_dp_mst_update_slots(struct drm_dp_mst_topology_state *mst_state, uint8_t link_encoding_cap)
 {
-	bool skip;
-
-	if (!port->vcpi.vcpi)
-		return;
-
-	mutex_lock(&mgr->lock);
-	skip = !drm_dp_mst_port_downstream_of_branch(port, mgr->mst_primary);
-	mutex_unlock(&mgr->lock);
-
-	if (skip)
-		return;
+	if (link_encoding_cap == DP_CAP_ANSI_128B132B) {
+		mst_state->total_avail_slots = 64;
+		mst_state->start_slot = 0;
+	} else {
+		mst_state->total_avail_slots = 63;
+		mst_state->start_slot = 1;
+	}
 
-	drm_dp_mst_put_payload_id(mgr, port->vcpi.vcpi);
-	port->vcpi.num_slots = 0;
-	port->vcpi.pbn = 0;
-	port->vcpi.aligned_pbn = 0;
-	port->vcpi.vcpi = 0;
-	drm_dp_mst_put_port_malloc(port);
+	DRM_DEBUG_KMS("%s encoding format on mst_state 0x%p\n",
+		      (link_encoding_cap == DP_CAP_ANSI_128B132B) ? "128b/132b":"8b/10b",
+		      mst_state);
 }
-EXPORT_SYMBOL(drm_dp_mst_deallocate_vcpi);
+EXPORT_SYMBOL(drm_dp_mst_update_slots);
 
 static int drm_dp_dpcd_write_payload(struct drm_dp_mst_topology_mgr *mgr,
-				     int id, struct drm_dp_payload *payload)
+				     int id, u8 start_slot, u8 num_slots)
 {
 	u8 payload_alloc[3], status;
 	int ret;
@@ -4637,8 +4519,8 @@ static int drm_dp_dpcd_write_payload(struct drm_dp_mst_topology_mgr *mgr,
 			   DP_PAYLOAD_TABLE_UPDATED);
 
 	payload_alloc[0] = id;
-	payload_alloc[1] = payload->start_slot;
-	payload_alloc[2] = payload->num_slots;
+	payload_alloc[1] = start_slot;
+	payload_alloc[2] = num_slots;
 
 	ret = drm_dp_dpcd_write(mgr->aux, DP_PAYLOAD_ALLOCATE_SET, payload_alloc, 3);
 	if (ret != 3) {
@@ -4853,8 +4735,9 @@ static void fetch_monitor_name(struct drm_dp_mst_topology_mgr *mgr,
 void drm_dp_mst_dump_topology(struct seq_file *m,
 			      struct drm_dp_mst_topology_mgr *mgr)
 {
-	int i;
-	struct drm_dp_mst_port *port;
+	struct drm_dp_mst_topology_state *state;
+	struct drm_dp_mst_atomic_payload *payload;
+	int i, ret;
 
 	mutex_lock(&mgr->lock);
 	if (mgr->mst_primary)
@@ -4863,36 +4746,35 @@ void drm_dp_mst_dump_topology(struct seq_file *m,
 	/* dump VCPIs */
 	mutex_unlock(&mgr->lock);
 
-	mutex_lock(&mgr->payload_lock);
-	seq_printf(m, "\n*** VCPI Info ***\n");
-	seq_printf(m, "payload_mask: %lx, vcpi_mask: %lx, max_payloads: %d\n", mgr->payload_mask, mgr->vcpi_mask, mgr->max_payloads);
+	ret = drm_modeset_lock_single_interruptible(&mgr->base.lock);
+	if (ret < 0)
+		return;
+
+	state = to_drm_dp_mst_topology_state(mgr->base.state);
+	seq_printf(m, "\n*** Atomic state info ***\n");
+	seq_printf(m, "payload_mask: %x, max_payloads: %d, start_slot: %u, pbn_div: %d\n",
+		   state->payload_mask, mgr->max_payloads, state->start_slot, state->pbn_div);
 
-	seq_printf(m, "\n|   idx   |  port # |  vcp_id | # slots |     sink name     |\n");
+	seq_printf(m, "\n| idx | port | vcpi | slots | pbn | dsc |     sink name     |\n");
 	for (i = 0; i < mgr->max_payloads; i++) {
-		if (mgr->proposed_vcpis[i]) {
+		list_for_each_entry(payload, &state->payloads, next) {
 			char name[14];
 
-			port = container_of(mgr->proposed_vcpis[i], struct drm_dp_mst_port, vcpi);
-			fetch_monitor_name(mgr, port, name, sizeof(name));
-			seq_printf(m, "%10d%10d%10d%10d%20s\n",
+			if (payload->vcpi != i || payload->delete)
+				continue;
+
+			fetch_monitor_name(mgr, payload->port, name, sizeof(name));
+			seq_printf(m, " %5d %6d %6d %02d - %02d %5d %5s %19s\n",
 				   i,
-				   port->port_num,
-				   port->vcpi.vcpi,
-				   port->vcpi.num_slots,
+				   payload->port->port_num,
+				   payload->vcpi,
+				   payload->vc_start_slot,
+				   payload->vc_start_slot + payload->time_slots - 1,
+				   payload->pbn,
+				   payload->dsc_enabled ? "Y" : "N",
 				   (*name != 0) ? name : "Unknown");
-		} else
-			seq_printf(m, "%6d - Unused\n", i);
-	}
-	seq_printf(m, "\n*** Payload Info ***\n");
-	seq_printf(m, "|   idx   |  state  |  start slot  | # slots |\n");
-	for (i = 0; i < mgr->max_payloads; i++) {
-		seq_printf(m, "%10d%10d%15d%10d\n",
-			   i,
-			   mgr->payloads[i].payload_state,
-			   mgr->payloads[i].start_slot,
-			   mgr->payloads[i].num_slots);
+		}
 	}
-	mutex_unlock(&mgr->payload_lock);
 
 	seq_printf(m, "\n*** DPCD Info ***\n");
 	mutex_lock(&mgr->lock);
@@ -4907,14 +4789,14 @@ void drm_dp_mst_dump_topology(struct seq_file *m,
 		seq_printf(m, "dpcd: %*ph\n", DP_RECEIVER_CAP_SIZE, buf);
 
 		ret = drm_dp_dpcd_read(mgr->aux, DP_FAUX_CAP, buf, 2);
-		if (ret) {
+		if (ret != 2) {
 			seq_printf(m, "faux/mst read failed\n");
 			goto out;
 		}
 		seq_printf(m, "faux/mst: %*ph\n", 2, buf);
 
 		ret = drm_dp_dpcd_read(mgr->aux, DP_MSTM_CTRL, buf, 1);
-		if (ret) {
+		if (ret != 1) {
 			seq_printf(m, "mst ctrl read failed\n");
 			goto out;
 		}
@@ -4922,7 +4804,7 @@ void drm_dp_mst_dump_topology(struct seq_file *m,
 
 		/* dump the standard OUI branch header */
 		ret = drm_dp_dpcd_read(mgr->aux, DP_BRANCH_OUI, buf, DP_BRANCH_OUI_HEADER_SIZE);
-		if (ret) {
+		if (ret != DP_BRANCH_OUI_HEADER_SIZE) {
 			seq_printf(m, "branch oui read failed\n");
 			goto out;
 		}
@@ -4938,7 +4820,7 @@ void drm_dp_mst_dump_topology(struct seq_file *m,
 
 out:
 	mutex_unlock(&mgr->lock);
-
+	drm_modeset_unlock(&mgr->base.lock);
 }
 EXPORT_SYMBOL(drm_dp_mst_dump_topology);
 
@@ -5060,7 +4942,7 @@ drm_dp_mst_duplicate_state(struct drm_private_obj *obj)
 {
 	struct drm_dp_mst_topology_state *state, *old_state =
 		to_dp_mst_topology_state(obj->state);
-	struct drm_dp_vcpi_allocation *pos, *vcpi;
+	struct drm_dp_mst_atomic_payload *pos, *payload;
 
 	state = kmemdup(old_state, sizeof(*state), GFP_KERNEL);
 	if (!state)
@@ -5068,25 +4950,28 @@ drm_dp_mst_duplicate_state(struct drm_private_obj *obj)
 
 	__drm_atomic_helper_private_obj_duplicate_state(obj, &state->base);
 
-	INIT_LIST_HEAD(&state->vcpis);
+	INIT_LIST_HEAD(&state->payloads);
+	state->commit_deps = NULL;
+	state->num_commit_deps = 0;
+	state->pending_crtc_mask = 0;
 
-	list_for_each_entry(pos, &old_state->vcpis, next) {
-		/* Prune leftover freed VCPI allocations */
-		if (!pos->vcpi)
+	list_for_each_entry(pos, &old_state->payloads, next) {
+		/* Prune leftover freed timeslot allocations */
+		if (pos->delete)
 			continue;
 
-		vcpi = kmemdup(pos, sizeof(*vcpi), GFP_KERNEL);
-		if (!vcpi)
+		payload = kmemdup(pos, sizeof(*payload), GFP_KERNEL);
+		if (!payload)
 			goto fail;
 
-		drm_dp_mst_get_port_malloc(vcpi->port);
-		list_add(&vcpi->next, &state->vcpis);
+		drm_dp_mst_get_port_malloc(payload->port);
+		list_add(&payload->next, &state->payloads);
 	}
 
 	return &state->base;
 
 fail:
-	list_for_each_entry_safe(pos, vcpi, &state->vcpis, next) {
+	list_for_each_entry_safe(pos, payload, &state->payloads, next) {
 		drm_dp_mst_put_port_malloc(pos->port);
 		kfree(pos);
 	}
@@ -5100,15 +4985,20 @@ static void drm_dp_mst_destroy_state(struct drm_private_obj *obj,
 {
 	struct drm_dp_mst_topology_state *mst_state =
 		to_dp_mst_topology_state(state);
-	struct drm_dp_vcpi_allocation *pos, *tmp;
+	struct drm_dp_mst_atomic_payload *pos, *tmp;
+	int i;
 
-	list_for_each_entry_safe(pos, tmp, &mst_state->vcpis, next) {
-		/* We only keep references to ports with non-zero VCPIs */
-		if (pos->vcpi)
+	list_for_each_entry_safe(pos, tmp, &mst_state->payloads, next) {
+		/* We only keep references to ports with active payloads */
+		if (!pos->delete)
 			drm_dp_mst_put_port_malloc(pos->port);
 		kfree(pos);
 	}
 
+	for (i = 0; i < mst_state->num_commit_deps; i++)
+		drm_crtc_commit_put(mst_state->commit_deps[i]);
+
+	kfree(mst_state->commit_deps);
 	kfree(mst_state);
 }
 
@@ -5135,7 +5025,7 @@ static int
 drm_dp_mst_atomic_check_mstb_bw_limit(struct drm_dp_mst_branch *mstb,
 				      struct drm_dp_mst_topology_state *state)
 {
-	struct drm_dp_vcpi_allocation *vcpi;
+	struct drm_dp_mst_atomic_payload *payload;
 	struct drm_dp_mst_port *port;
 	int pbn_used = 0, ret;
 	bool found = false;
@@ -5143,9 +5033,9 @@ drm_dp_mst_atomic_check_mstb_bw_limit(struct drm_dp_mst_branch *mstb,
 	/* Check that we have at least one port in our state that's downstream
 	 * of this branch, otherwise we can skip this branch
 	 */
-	list_for_each_entry(vcpi, &state->vcpis, next) {
-		if (!vcpi->pbn ||
-		    !drm_dp_mst_port_downstream_of_branch(vcpi->port, mstb))
+	list_for_each_entry(payload, &state->payloads, next) {
+		if (!payload->pbn ||
+		    !drm_dp_mst_port_downstream_of_branch(payload->port, mstb))
 			continue;
 
 		found = true;
@@ -5176,25 +5066,15 @@ static int
 drm_dp_mst_atomic_check_port_bw_limit(struct drm_dp_mst_port *port,
 				      struct drm_dp_mst_topology_state *state)
 {
-	struct drm_dp_vcpi_allocation *vcpi;
+	struct drm_dp_mst_atomic_payload *payload;
 	int pbn_used = 0;
 
 	if (port->pdt == DP_PEER_DEVICE_NONE)
 		return 0;
 
 	if (drm_dp_mst_is_end_device(port->pdt, port->mcs)) {
-		bool found = false;
-
-		list_for_each_entry(vcpi, &state->vcpis, next) {
-			if (vcpi->port != port)
-				continue;
-			if (!vcpi->pbn)
-				return 0;
-
-			found = true;
-			break;
-		}
-		if (!found)
+		payload = drm_atomic_get_mst_payload_state(state, port);
+		if (!payload)
 			return 0;
 
 		/*
@@ -5208,7 +5088,7 @@ drm_dp_mst_atomic_check_port_bw_limit(struct drm_dp_mst_port *port,
 			return -EINVAL;
 		}
 
-		pbn_used = vcpi->pbn;
+		pbn_used = payload->pbn;
 	} else {
 		pbn_used = drm_dp_mst_atomic_check_mstb_bw_limit(port->mstb,
 								 state);
@@ -5230,28 +5110,28 @@ drm_dp_mst_atomic_check_port_bw_limit(struct drm_dp_mst_port *port,
 }
 
 static inline int
-drm_dp_mst_atomic_check_vcpi_alloc_limit(struct drm_dp_mst_topology_mgr *mgr,
-					 struct drm_dp_mst_topology_state *mst_state)
+drm_dp_mst_atomic_check_payload_alloc_limits(struct drm_dp_mst_topology_mgr *mgr,
+					     struct drm_dp_mst_topology_state *mst_state)
 {
-	struct drm_dp_vcpi_allocation *vcpi;
+	struct drm_dp_mst_atomic_payload *payload;
 	int avail_slots = mst_state->total_avail_slots, payload_count = 0;
 
-	list_for_each_entry(vcpi, &mst_state->vcpis, next) {
-		/* Releasing VCPI is always OK-even if the port is gone */
-		if (!vcpi->vcpi) {
-			drm_dbg_atomic(mgr->dev, "[MST PORT:%p] releases all VCPI slots\n",
-				       vcpi->port);
+	list_for_each_entry(payload, &mst_state->payloads, next) {
+		/* Releasing payloads is always OK-even if the port is gone */
+		if (payload->delete) {
+			drm_dbg_atomic(mgr->dev, "[MST PORT:%p] releases all time slots\n",
+				       payload->port);
 			continue;
 		}
 
-		drm_dbg_atomic(mgr->dev, "[MST PORT:%p] requires %d vcpi slots\n",
-			       vcpi->port, vcpi->vcpi);
+		drm_dbg_atomic(mgr->dev, "[MST PORT:%p] requires %d time slots\n",
+			       payload->port, payload->time_slots);
 
-		avail_slots -= vcpi->vcpi;
+		avail_slots -= payload->time_slots;
 		if (avail_slots < 0) {
 			drm_dbg_atomic(mgr->dev,
-				       "[MST PORT:%p] not enough VCPI slots in mst state %p (avail=%d)\n",
-				       vcpi->port, mst_state, avail_slots + vcpi->vcpi);
+				       "[MST PORT:%p] not enough time slots in mst state %p (avail=%d)\n",
+				       payload->port, mst_state, avail_slots + payload->time_slots);
 			return -ENOSPC;
 		}
 
@@ -5261,9 +5141,22 @@ drm_dp_mst_atomic_check_vcpi_alloc_limit(struct drm_dp_mst_topology_mgr *mgr,
 				       mgr, mst_state, mgr->max_payloads);
 			return -EINVAL;
 		}
+
+		/* Assign a VCPI */
+		if (!payload->vcpi) {
+			payload->vcpi = ffz(mst_state->payload_mask) + 1;
+			drm_dbg_atomic(mgr->dev, "[MST PORT:%p] assigned VCPI #%d\n",
+				       payload->port, payload->vcpi);
+			mst_state->payload_mask |= BIT(payload->vcpi - 1);
+		}
 	}
-	drm_dbg_atomic(mgr->dev, "[MST MGR:%p] mst state %p VCPI avail=%d used=%d\n",
-		       mgr, mst_state, avail_slots, mst_state->total_avail_slots - avail_slots);
+
+	if (!payload_count)
+		mst_state->pbn_div = 0;
+
+	drm_dbg_atomic(mgr->dev, "[MST MGR:%p] mst state %p TU pbn_div=%d avail=%d used=%d\n",
+		       mgr, mst_state, mst_state->pbn_div, avail_slots,
+		       mst_state->total_avail_slots - avail_slots);
 
 	return 0;
 }
@@ -5284,7 +5177,7 @@ drm_dp_mst_atomic_check_vcpi_alloc_limit(struct drm_dp_mst_topology_mgr *mgr,
 int drm_dp_mst_add_affected_dsc_crtcs(struct drm_atomic_state *state, struct drm_dp_mst_topology_mgr *mgr)
 {
 	struct drm_dp_mst_topology_state *mst_state;
-	struct drm_dp_vcpi_allocation *pos;
+	struct drm_dp_mst_atomic_payload *pos;
 	struct drm_connector *connector;
 	struct drm_connector_state *conn_state;
 	struct drm_crtc *crtc;
@@ -5295,7 +5188,7 @@ int drm_dp_mst_add_affected_dsc_crtcs(struct drm_atomic_state *state, struct drm
 	if (IS_ERR(mst_state))
 		return -EINVAL;
 
-	list_for_each_entry(pos, &mst_state->vcpis, next) {
+	list_for_each_entry(pos, &mst_state->payloads, next) {
 
 		connector = pos->port->connector;
 
@@ -5334,7 +5227,6 @@ EXPORT_SYMBOL(drm_dp_mst_add_affected_dsc_crtcs);
  * @state: Pointer to the new drm_atomic_state
  * @port: Pointer to the affected MST Port
  * @pbn: Newly recalculated bw required for link with DSC enabled
- * @pbn_div: Divider to calculate correct number of pbn per slot
  * @enable: Boolean flag to enable or disable DSC on the port
  *
  * This function enables DSC on the given Port
@@ -5345,54 +5237,46 @@ EXPORT_SYMBOL(drm_dp_mst_add_affected_dsc_crtcs);
  */
 int drm_dp_mst_atomic_enable_dsc(struct drm_atomic_state *state,
 				 struct drm_dp_mst_port *port,
-				 int pbn, int pbn_div,
-				 bool enable)
+				 int pbn, bool enable)
 {
 	struct drm_dp_mst_topology_state *mst_state;
-	struct drm_dp_vcpi_allocation *pos;
-	bool found = false;
-	int vcpi = 0;
+	struct drm_dp_mst_atomic_payload *payload;
+	int time_slots = 0;
 
 	mst_state = drm_atomic_get_mst_topology_state(state, port->mgr);
-
 	if (IS_ERR(mst_state))
 		return PTR_ERR(mst_state);
 
-	list_for_each_entry(pos, &mst_state->vcpis, next) {
-		if (pos->port == port) {
-			found = true;
-			break;
-		}
-	}
-
-	if (!found) {
+	payload = drm_atomic_get_mst_payload_state(mst_state, port);
+	if (!payload) {
 		drm_dbg_atomic(state->dev,
-			       "[MST PORT:%p] Couldn't find VCPI allocation in mst state %p\n",
+			       "[MST PORT:%p] Couldn't find payload in mst state %p\n",
 			       port, mst_state);
 		return -EINVAL;
 	}
 
-	if (pos->dsc_enabled == enable) {
+	if (payload->dsc_enabled == enable) {
 		drm_dbg_atomic(state->dev,
-			       "[MST PORT:%p] DSC flag is already set to %d, returning %d VCPI slots\n",
-			       port, enable, pos->vcpi);
-		vcpi = pos->vcpi;
+			       "[MST PORT:%p] DSC flag is already set to %d, returning %d time slots\n",
+			       port, enable, payload->time_slots);
+		time_slots = payload->time_slots;
 	}
 
 	if (enable) {
-		vcpi = drm_dp_atomic_find_vcpi_slots(state, port->mgr, port, pbn, pbn_div);
+		time_slots = drm_dp_atomic_find_time_slots(state, port->mgr, port, pbn);
 		drm_dbg_atomic(state->dev,
-			       "[MST PORT:%p] Enabling DSC flag, reallocating %d VCPI slots on the port\n",
-			       port, vcpi);
-		if (vcpi < 0)
+			       "[MST PORT:%p] Enabling DSC flag, reallocating %d time slots on the port\n",
+			       port, time_slots);
+		if (time_slots < 0)
 			return -EINVAL;
 	}
 
-	pos->dsc_enabled = enable;
+	payload->dsc_enabled = enable;
 
-	return vcpi;
+	return time_slots;
 }
 EXPORT_SYMBOL(drm_dp_mst_atomic_enable_dsc);
+
 /**
  * drm_dp_mst_atomic_check - Check that the new state of an MST topology in an
  * atomic update is valid
@@ -5400,15 +5284,15 @@ EXPORT_SYMBOL(drm_dp_mst_atomic_enable_dsc);
  *
  * Checks the given topology state for an atomic update to ensure that it's
  * valid. This includes checking whether there's enough bandwidth to support
- * the new VCPI allocations in the atomic update.
+ * the new timeslot allocations in the atomic update.
  *
  * Any atomic drivers supporting DP MST must make sure to call this after
  * checking the rest of their state in their
  * &drm_mode_config_funcs.atomic_check() callback.
  *
  * See also:
- * drm_dp_atomic_find_vcpi_slots()
- * drm_dp_atomic_release_vcpi_slots()
+ * drm_dp_atomic_find_time_slots()
+ * drm_dp_atomic_release_time_slots()
  *
  * Returns:
  *
@@ -5424,7 +5308,7 @@ int drm_dp_mst_atomic_check(struct drm_atomic_state *state)
 		if (!mgr->mst_state)
 			continue;
 
-		ret = drm_dp_mst_atomic_check_vcpi_alloc_limit(mgr, mst_state);
+		ret = drm_dp_mst_atomic_check_payload_alloc_limits(mgr, mst_state);
 		if (ret)
 			break;
 
@@ -5450,7 +5334,6 @@ EXPORT_SYMBOL(drm_dp_mst_topology_state_funcs);
 
 /**
  * drm_atomic_get_mst_topology_state: get MST topology state
- *
  * @state: global atomic state
  * @mgr: MST topology manager, also the private object in this case
  *
@@ -5470,14 +5353,37 @@ struct drm_dp_mst_topology_state *drm_atomic_get_mst_topology_state(struct drm_a
 EXPORT_SYMBOL(drm_atomic_get_mst_topology_state);
 
 /**
+ * drm_atomic_get_new_mst_topology_state: get new MST topology state in atomic state, if any
+ * @state: global atomic state
+ * @mgr: MST topology manager, also the private object in this case
+ *
+ * This function wraps drm_atomic_get_priv_obj_state() passing in the MST atomic
+ * state vtable so that the private object state returned is that of a MST
+ * topology object.
+ *
+ * Returns:
+ *
+ * The MST topology state, or NULL if there's no topology state for this MST mgr
+ * in the global atomic state
+ */
+struct drm_dp_mst_topology_state *
+drm_atomic_get_new_mst_topology_state(struct drm_atomic_state *state,
+				      struct drm_dp_mst_topology_mgr *mgr)
+{
+	struct drm_private_state *priv_state =
+		drm_atomic_get_new_private_obj_state(state, &mgr->base);
+
+	return priv_state ? to_dp_mst_topology_state(priv_state) : NULL;
+}
+EXPORT_SYMBOL(drm_atomic_get_new_mst_topology_state);
+
+/**
  * drm_dp_mst_topology_mgr_init - initialise a topology manager
  * @mgr: manager struct to initialise
  * @dev: device providing this structure - for i2c addition.
  * @aux: DP helper aux channel to talk to this device
  * @max_dpcd_transaction_bytes: hw specific DPCD transaction limit
  * @max_payloads: maximum number of payloads this GPU can source
- * @max_lane_count: maximum number of lanes this GPU supports
- * @max_link_rate: maximum link rate per lane this GPU supports in kHz
  * @conn_base_id: the connector object ID the MST device is connected to.
  *
  * Return 0 for success, or negative error code on failure
@@ -5485,14 +5391,12 @@ EXPORT_SYMBOL(drm_atomic_get_mst_topology_state);
 int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr,
 				 struct drm_device *dev, struct drm_dp_aux *aux,
 				 int max_dpcd_transaction_bytes, int max_payloads,
-				 int max_lane_count, int max_link_rate,
 				 int conn_base_id)
 {
 	struct drm_dp_mst_topology_state *mst_state;
 
 	mutex_init(&mgr->lock);
 	mutex_init(&mgr->qlock);
-	mutex_init(&mgr->payload_lock);
 	mutex_init(&mgr->delayed_destroy_lock);
 	mutex_init(&mgr->up_req_lock);
 	mutex_init(&mgr->probe_lock);
@@ -5522,19 +5426,7 @@ int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr,
 	mgr->aux = aux;
 	mgr->max_dpcd_transaction_bytes = max_dpcd_transaction_bytes;
 	mgr->max_payloads = max_payloads;
-	mgr->max_lane_count = max_lane_count;
-	mgr->max_link_rate = max_link_rate;
 	mgr->conn_base_id = conn_base_id;
-	if (max_payloads + 1 > sizeof(mgr->payload_mask) * 8 ||
-	    max_payloads + 1 > sizeof(mgr->vcpi_mask) * 8)
-		return -EINVAL;
-	mgr->payloads = kcalloc(max_payloads, sizeof(struct drm_dp_payload), GFP_KERNEL);
-	if (!mgr->payloads)
-		return -ENOMEM;
-	mgr->proposed_vcpis = kcalloc(max_payloads, sizeof(struct drm_dp_vcpi *), GFP_KERNEL);
-	if (!mgr->proposed_vcpis)
-		return -ENOMEM;
-	set_bit(0, &mgr->payload_mask);
 
 	mst_state = kzalloc(sizeof(*mst_state), GFP_KERNEL);
 	if (mst_state == NULL)
@@ -5544,7 +5436,7 @@ int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr,
 	mst_state->start_slot = 1;
 
 	mst_state->mgr = mgr;
-	INIT_LIST_HEAD(&mst_state->vcpis);
+	INIT_LIST_HEAD(&mst_state->payloads);
 
 	drm_atomic_private_obj_init(dev, &mgr->base,
 				    &mst_state->base,
@@ -5567,19 +5459,12 @@ void drm_dp_mst_topology_mgr_destroy(struct drm_dp_mst_topology_mgr *mgr)
 		destroy_workqueue(mgr->delayed_destroy_wq);
 		mgr->delayed_destroy_wq = NULL;
 	}
-	mutex_lock(&mgr->payload_lock);
-	kfree(mgr->payloads);
-	mgr->payloads = NULL;
-	kfree(mgr->proposed_vcpis);
-	mgr->proposed_vcpis = NULL;
-	mutex_unlock(&mgr->payload_lock);
 	mgr->dev = NULL;
 	mgr->aux = NULL;
 	drm_atomic_private_obj_fini(&mgr->base);
 	mgr->funcs = NULL;
 
 	mutex_destroy(&mgr->delayed_destroy_lock);
-	mutex_destroy(&mgr->payload_lock);
 	mutex_destroy(&mgr->qlock);
 	mutex_destroy(&mgr->lock);
 	mutex_destroy(&mgr->up_req_lock);
@@ -5908,8 +5793,10 @@ struct drm_dp_aux *drm_dp_mst_dsc_aux_for_port(struct drm_dp_mst_port *port)
 		/* Enpoint decompression with DP-to-DP peer device */
 		if ((endpoint_dsc & DP_DSC_DECOMPRESSION_IS_SUPPORTED) &&
 		    (endpoint_fec & DP_FEC_CAPABLE) &&
-		    (upstream_dsc & 0x2) /* DSC passthrough */)
+		    (upstream_dsc & DP_DSC_PASSTHROUGH_IS_SUPPORTED)) {
+			port->passthrough_aux = &immediate_upstream_port->aux;
 			return &port->aux;
+		}
 
 		/* Virtual DPCD decompression with DP-to-DP peer device */
 		return &immediate_upstream_port->aux;
diff --git a/drivers/gpu/drm/display/drm_scdc_helper.c b/drivers/gpu/drm/display/drm_scdc_helper.c
index 81881e81ceae..c3ad4ab2b456 100644
--- a/drivers/gpu/drm/display/drm_scdc_helper.c
+++ b/drivers/gpu/drm/display/drm_scdc_helper.c
@@ -35,6 +35,19 @@
  * HDMI 2.0 specification. It is a point-to-point protocol that allows the
  * HDMI source and HDMI sink to exchange data. The same I2C interface that
  * is used to access EDID serves as the transport mechanism for SCDC.
+ *
+ * Note: The SCDC status is going to be lost when the display is
+ * disconnected. This can happen physically when the user disconnects
+ * the cable, but also when a display is switched on (such as waking up
+ * a TV).
+ *
+ * This is further complicated by the fact that, upon a disconnection /
+ * reconnection, KMS won't change the mode on its own. This means that
+ * one can't just rely on setting the SCDC status on enable, but also
+ * has to track the connector status changes using interrupts and
+ * restore the SCDC status. The typical solution for this is to trigger an
+ * empty modeset in drm_connector_helper_funcs.detect_ctx(), like what vc4 does
+ * in vc4_hdmi_reset_link().
  */
 
 #define SCDC_I2C_SLAVE_ADDRESS 0x54
diff --git a/drivers/gpu/drm/drm_aperture.c b/drivers/gpu/drm/drm_aperture.c
index fdb7d5c17ba1..3b8fdeeafd53 100644
--- a/drivers/gpu/drm/drm_aperture.c
+++ b/drivers/gpu/drm/drm_aperture.c
@@ -74,7 +74,7 @@
  * given framebuffer memory. Ownership of the framebuffer memory is achieved
  * by calling devm_aperture_acquire_from_firmware(). On success, the driver
  * is the owner of the framebuffer range. The function fails if the
- * framebuffer is already by another driver. See below for an example.
+ * framebuffer is already owned by another driver. See below for an example.
  *
  * .. code-block:: c
  *
@@ -112,7 +112,7 @@
  *
  * The generic driver is now subject to forced removal by other drivers. This
  * only works for platform drivers that support hot unplug.
- * When a driver calls drm_aperture_remove_conflicting_framebuffers() et al
+ * When a driver calls drm_aperture_remove_conflicting_framebuffers() et al.
  * for the registered framebuffer range, the aperture helpers call
  * platform_device_unregister() and the generic driver unloads itself. It
  * may not access the device's registers, framebuffer memory, ROM, etc
@@ -164,7 +164,7 @@ EXPORT_SYMBOL(devm_aperture_acquire_from_firmware);
  * @primary: also kick vga16fb if present
  * @req_driver: requesting DRM driver
  *
- * This function removes graphics device drivers which use memory range described by
+ * This function removes graphics device drivers which use the memory range described by
  * @base and @size.
  *
  * Returns:
@@ -182,8 +182,8 @@ EXPORT_SYMBOL(drm_aperture_remove_conflicting_framebuffers);
  * @pdev: PCI device
  * @req_driver: requesting DRM driver
  *
- * This function removes graphics device drivers using memory range configured
- * for any of @pdev's memory bars. The function assumes that PCI device with
+ * This function removes graphics device drivers using the memory range configured
+ * for any of @pdev's memory bars. The function assumes that a PCI device with
  * shadowed ROM drives a primary display and so kicks out vga16fb.
  *
  * Returns:
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index 8bf41aa24068..98cc3137c062 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -38,7 +38,6 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_self_refresh_helper.h>
 #include <drm/drm_vblank.h>
@@ -703,8 +702,12 @@ drm_atomic_helper_check_modeset(struct drm_device *dev,
 
 		if (funcs->atomic_check)
 			ret = funcs->atomic_check(connector, state);
-		if (ret)
+		if (ret) {
+			drm_dbg_atomic(dev,
+				       "[CONNECTOR:%d:%s] driver check failed\n",
+				       connector->base.id, connector->name);
 			return ret;
+		}
 
 		connectors_mask |= BIT(i);
 	}
@@ -746,8 +749,12 @@ drm_atomic_helper_check_modeset(struct drm_device *dev,
 
 		if (funcs->atomic_check)
 			ret = funcs->atomic_check(connector, state);
-		if (ret)
+		if (ret) {
+			drm_dbg_atomic(dev,
+				       "[CONNECTOR:%d:%s] driver check failed\n",
+				       connector->base.id, connector->name);
 			return ret;
+		}
 	}
 
 	/*
@@ -779,6 +786,45 @@ drm_atomic_helper_check_modeset(struct drm_device *dev,
 EXPORT_SYMBOL(drm_atomic_helper_check_modeset);
 
 /**
+ * drm_atomic_helper_check_wb_encoder_state() - Check writeback encoder state
+ * @encoder: encoder state to check
+ * @conn_state: connector state to check
+ *
+ * Checks if the writeback connector state is valid, and returns an error if it
+ * isn't.
+ *
+ * RETURNS:
+ * Zero for success or -errno
+ */
+int
+drm_atomic_helper_check_wb_encoder_state(struct drm_encoder *encoder,
+					 struct drm_connector_state *conn_state)
+{
+	struct drm_writeback_job *wb_job = conn_state->writeback_job;
+	struct drm_property_blob *pixel_format_blob;
+	struct drm_framebuffer *fb;
+	size_t i, nformats;
+	u32 *formats;
+
+	if (!wb_job || !wb_job->fb)
+		return 0;
+
+	pixel_format_blob = wb_job->connector->pixel_formats_blob_ptr;
+	nformats = pixel_format_blob->length / sizeof(u32);
+	formats = pixel_format_blob->data;
+	fb = wb_job->fb;
+
+	for (i = 0; i < nformats; i++)
+		if (fb->format->format == formats[i])
+			return 0;
+
+	drm_dbg_kms(encoder->dev, "Invalid pixel format %p4cc\n", &fb->format->format);
+
+	return -EINVAL;
+}
+EXPORT_SYMBOL(drm_atomic_helper_check_wb_encoder_state);
+
+/**
  * drm_atomic_helper_check_plane_state() - Check plane state for validity
  * @plane_state: plane state to check
  * @crtc_state: CRTC state to check
@@ -1789,7 +1835,7 @@ int drm_atomic_helper_async_check(struct drm_device *dev,
 	struct drm_plane_state *old_plane_state = NULL;
 	struct drm_plane_state *new_plane_state = NULL;
 	const struct drm_plane_helper_funcs *funcs;
-	int i, n_planes = 0;
+	int i, ret, n_planes = 0;
 
 	for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
 		if (drm_atomic_crtc_needs_modeset(crtc_state))
@@ -1800,19 +1846,34 @@ int drm_atomic_helper_async_check(struct drm_device *dev,
 		n_planes++;
 
 	/* FIXME: we support only single plane updates for now */
-	if (n_planes != 1)
+	if (n_planes != 1) {
+		drm_dbg_atomic(dev,
+			       "only single plane async updates are supported\n");
 		return -EINVAL;
+	}
 
 	if (!new_plane_state->crtc ||
-	    old_plane_state->crtc != new_plane_state->crtc)
+	    old_plane_state->crtc != new_plane_state->crtc) {
+		drm_dbg_atomic(dev,
+			       "[PLANE:%d:%s] async update cannot change CRTC\n",
+			       plane->base.id, plane->name);
 		return -EINVAL;
+	}
 
 	funcs = plane->helper_private;
-	if (!funcs->atomic_async_update)
+	if (!funcs->atomic_async_update) {
+		drm_dbg_atomic(dev,
+			       "[PLANE:%d:%s] driver does not support async updates\n",
+			       plane->base.id, plane->name);
 		return -EINVAL;
+	}
 
-	if (new_plane_state->fence)
+	if (new_plane_state->fence) {
+		drm_dbg_atomic(dev,
+			       "[PLANE:%d:%s] missing fence for async update\n",
+			       plane->base.id, plane->name);
 		return -EINVAL;
+	}
 
 	/*
 	 * Don't do an async update if there is an outstanding commit modifying
@@ -1827,7 +1888,12 @@ int drm_atomic_helper_async_check(struct drm_device *dev,
 		return -EBUSY;
 	}
 
-	return funcs->atomic_async_check(plane, state);
+	ret = funcs->atomic_async_check(plane, state);
+	if (ret != 0)
+		drm_dbg_atomic(dev,
+			       "[PLANE:%d:%s] driver async check failed\n",
+			       plane->base.id, plane->name);
+	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_async_check);
 
diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c
index 6e433d465f41..cf92a9ae8034 100644
--- a/drivers/gpu/drm/drm_auth.c
+++ b/drivers/gpu/drm/drm_auth.c
@@ -140,14 +140,14 @@ struct drm_master *drm_master_create(struct drm_device *dev)
 
 	kref_init(&master->refcount);
 	drm_master_legacy_init(master);
-	idr_init(&master->magic_map);
+	idr_init_base(&master->magic_map, 1);
 	master->dev = dev;
 
 	/* initialize the tree of output resource lessees */
 	INIT_LIST_HEAD(&master->lessees);
 	INIT_LIST_HEAD(&master->lessee_list);
 	idr_init(&master->leases);
-	idr_init(&master->lessee_idr);
+	idr_init_base(&master->lessee_idr, 1);
 
 	return master;
 }
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index 6abf7a2407e9..1545c50fd1c8 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -847,8 +847,8 @@ static int select_bus_fmt_recursive(struct drm_bridge *first_bridge,
 				    struct drm_connector_state *conn_state,
 				    u32 out_bus_fmt)
 {
+	unsigned int i, num_in_bus_fmts = 0;
 	struct drm_bridge_state *cur_state;
-	unsigned int num_in_bus_fmts, i;
 	struct drm_bridge *prev_bridge;
 	u32 *in_bus_fmts;
 	int ret;
@@ -969,7 +969,7 @@ drm_atomic_bridge_chain_select_bus_fmts(struct drm_bridge *bridge,
 	struct drm_connector *conn = conn_state->connector;
 	struct drm_encoder *encoder = bridge->encoder;
 	struct drm_bridge_state *last_bridge_state;
-	unsigned int i, num_out_bus_fmts;
+	unsigned int i, num_out_bus_fmts = 0;
 	struct drm_bridge *last_bridge;
 	u32 *out_bus_fmts;
 	int ret = 0;
diff --git a/drivers/gpu/drm/drm_client.c b/drivers/gpu/drm/drm_client.c
index af3b7395bf69..2b230b4d6942 100644
--- a/drivers/gpu/drm/drm_client.c
+++ b/drivers/gpu/drm/drm_client.c
@@ -264,7 +264,7 @@ drm_client_buffer_create(struct drm_client_dev *client, u32 width, u32 height, u
 
 	dumb_args.width = width;
 	dumb_args.height = height;
-	dumb_args.bpp = info->cpp[0] * 8;
+	dumb_args.bpp = drm_format_info_bpp(info, 0);
 	ret = drm_mode_create_dumb(dev, &dumb_args, client->file);
 	if (ret)
 		goto err_delete;
@@ -373,7 +373,7 @@ static int drm_client_buffer_addfb(struct drm_client_buffer *buffer,
 	int ret;
 
 	info = drm_format_info(format);
-	fb_req.bpp = info->cpp[0] * 8;
+	fb_req.bpp = drm_format_info_bpp(info, 0);
 	fb_req.depth = info->depth;
 	fb_req.width = width;
 	fb_req.height = height;
diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c
index 17c6c3eefcd6..d021497841b8 100644
--- a/drivers/gpu/drm/drm_color_mgmt.c
+++ b/drivers/gpu/drm/drm_color_mgmt.c
@@ -575,7 +575,7 @@ int drm_plane_create_color_properties(struct drm_plane *plane,
 		len++;
 	}
 
-	prop = drm_property_create_enum(dev, 0,	"COLOR_RANGE",
+	prop = drm_property_create_enum(dev, 0, "COLOR_RANGE",
 					enum_list, len);
 	if (!prop)
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 1ab083b35e3b..e3142c8142b3 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -22,15 +22,16 @@
 
 #include <drm/drm_auth.h>
 #include <drm/drm_connector.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_encoder.h>
+#include <drm/drm_file.h>
+#include <drm/drm_managed.h>
 #include <drm/drm_panel.h>
-#include <drm/drm_utils.h>
 #include <drm/drm_print.h>
-#include <drm/drm_drv.h>
-#include <drm/drm_file.h>
 #include <drm/drm_privacy_screen_consumer.h>
 #include <drm/drm_sysfs.h>
+#include <drm/drm_utils.h>
 
 #include <linux/fb.h>
 #include <linux/uaccess.h>
@@ -214,23 +215,11 @@ void drm_connector_free_work_fn(struct work_struct *work)
 	}
 }
 
-/**
- * drm_connector_init - Init a preallocated connector
- * @dev: DRM device
- * @connector: the connector to init
- * @funcs: callbacks for this connector
- * @connector_type: user visible type of the connector
- *
- * Initialises a preallocated connector. Connectors should be
- * subclassed as part of driver connector objects.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_connector_init(struct drm_device *dev,
-		       struct drm_connector *connector,
-		       const struct drm_connector_funcs *funcs,
-		       int connector_type)
+static int __drm_connector_init(struct drm_device *dev,
+				struct drm_connector *connector,
+				const struct drm_connector_funcs *funcs,
+				int connector_type,
+				struct i2c_adapter *ddc)
 {
 	struct drm_mode_config *config = &dev->mode_config;
 	int ret;
@@ -278,6 +267,9 @@ int drm_connector_init(struct drm_device *dev,
 		goto out_put_type_id;
 	}
 
+	/* provide ddc symlink in sysfs */
+	connector->ddc = ddc;
+
 	INIT_LIST_HEAD(&connector->global_connector_list_entry);
 	INIT_LIST_HEAD(&connector->probed_modes);
 	INIT_LIST_HEAD(&connector->modes);
@@ -334,6 +326,38 @@ out_put:
 
 	return ret;
 }
+
+/**
+ * drm_connector_init - Init a preallocated connector
+ * @dev: DRM device
+ * @connector: the connector to init
+ * @funcs: callbacks for this connector
+ * @connector_type: user visible type of the connector
+ *
+ * Initialises a preallocated connector. Connectors should be
+ * subclassed as part of driver connector objects.
+ *
+ * At driver unload time the driver's &drm_connector_funcs.destroy hook
+ * should call drm_connector_cleanup() and free the connector structure.
+ * The connector structure should not be allocated with devm_kzalloc().
+ *
+ * Note: consider using drmm_connector_init() instead of
+ * drm_connector_init() to let the DRM managed resource infrastructure
+ * take care of cleanup and deallocation.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_connector_init(struct drm_device *dev,
+		       struct drm_connector *connector,
+		       const struct drm_connector_funcs *funcs,
+		       int connector_type)
+{
+	if (drm_WARN_ON(dev, !(funcs && funcs->destroy)))
+		return -EINVAL;
+
+	return __drm_connector_init(dev, connector, funcs, connector_type, NULL);
+}
 EXPORT_SYMBOL(drm_connector_init);
 
 /**
@@ -347,8 +371,16 @@ EXPORT_SYMBOL(drm_connector_init);
  * Initialises a preallocated connector. Connectors should be
  * subclassed as part of driver connector objects.
  *
+ * At driver unload time the driver's &drm_connector_funcs.destroy hook
+ * should call drm_connector_cleanup() and free the connector structure.
+ * The connector structure should not be allocated with devm_kzalloc().
+ *
  * Ensures that the ddc field of the connector is correctly set.
  *
+ * Note: consider using drmm_connector_init() instead of
+ * drm_connector_init_with_ddc() to let the DRM managed resource
+ * infrastructure take care of cleanup and deallocation.
+ *
  * Returns:
  * Zero on success, error code on failure.
  */
@@ -358,18 +390,63 @@ int drm_connector_init_with_ddc(struct drm_device *dev,
 				int connector_type,
 				struct i2c_adapter *ddc)
 {
+	if (drm_WARN_ON(dev, !(funcs && funcs->destroy)))
+		return -EINVAL;
+
+	return __drm_connector_init(dev, connector, funcs, connector_type, ddc);
+}
+EXPORT_SYMBOL(drm_connector_init_with_ddc);
+
+static void drm_connector_cleanup_action(struct drm_device *dev,
+					 void *ptr)
+{
+	struct drm_connector *connector = ptr;
+
+	drm_connector_cleanup(connector);
+}
+
+/**
+ * drmm_connector_init - Init a preallocated connector
+ * @dev: DRM device
+ * @connector: the connector to init
+ * @funcs: callbacks for this connector
+ * @connector_type: user visible type of the connector
+ * @ddc: optional pointer to the associated ddc adapter
+ *
+ * Initialises a preallocated connector. Connectors should be
+ * subclassed as part of driver connector objects.
+ *
+ * Cleanup is automatically handled with a call to
+ * drm_connector_cleanup() in a DRM-managed action.
+ *
+ * The connector structure should be allocated with drmm_kzalloc().
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drmm_connector_init(struct drm_device *dev,
+			struct drm_connector *connector,
+			const struct drm_connector_funcs *funcs,
+			int connector_type,
+			struct i2c_adapter *ddc)
+{
 	int ret;
 
-	ret = drm_connector_init(dev, connector, funcs, connector_type);
+	if (drm_WARN_ON(dev, funcs && funcs->destroy))
+		return -EINVAL;
+
+	ret = __drm_connector_init(dev, connector, funcs, connector_type, NULL);
 	if (ret)
 		return ret;
 
-	/* provide ddc symlink in sysfs */
-	connector->ddc = ddc;
+	ret = drmm_add_action_or_reset(dev, drm_connector_cleanup_action,
+				       connector);
+	if (ret)
+		return ret;
 
-	return ret;
+	return 0;
 }
-EXPORT_SYMBOL(drm_connector_init_with_ddc);
+EXPORT_SYMBOL(drmm_connector_init);
 
 /**
  * drm_connector_attach_edid_property - attach edid property.
@@ -517,6 +594,9 @@ EXPORT_SYMBOL(drm_connector_cleanup);
  * e.g. DP MST connectors. All other connectors will be registered automatically
  * when calling drm_dev_register().
  *
+ * When the connector is no longer available, callers must call
+ * drm_connector_unregister().
+ *
  * Returns:
  * Zero on success, error code on failure.
  */
@@ -573,9 +653,8 @@ EXPORT_SYMBOL(drm_connector_register);
  * @connector: the connector to unregister
  *
  * Unregister userspace interfaces for a connector. Only call this for
- * connectors which have registered explicitly by calling drm_dev_register(),
- * since connectors are unregistered automatically when drm_dev_unregister() is
- * called.
+ * connectors which have been registered explicitly by calling
+ * drm_connector_register().
  */
 void drm_connector_unregister(struct drm_connector *connector)
 {
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index cad2a7e5166f..df9bf3c9206e 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -343,9 +343,10 @@ static int __drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *
  * The @primary and @cursor planes are only relevant for legacy uAPI, see
  * &drm_crtc.primary and &drm_crtc.cursor.
  *
- * Note: consider using drmm_crtc_alloc_with_planes() instead of
- * drm_crtc_init_with_planes() to let the DRM managed resource infrastructure
- * take care of cleanup and deallocation.
+ * Note: consider using drmm_crtc_alloc_with_planes() or
+ * drmm_crtc_init_with_planes() instead of drm_crtc_init_with_planes()
+ * to let the DRM managed resource infrastructure take care of cleanup
+ * and deallocation.
  *
  * Returns:
  * Zero on success, error code on failure.
@@ -370,14 +371,88 @@ int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
 }
 EXPORT_SYMBOL(drm_crtc_init_with_planes);
 
-static void drmm_crtc_alloc_with_planes_cleanup(struct drm_device *dev,
-						void *ptr)
+static void drmm_crtc_init_with_planes_cleanup(struct drm_device *dev,
+					       void *ptr)
 {
 	struct drm_crtc *crtc = ptr;
 
 	drm_crtc_cleanup(crtc);
 }
 
+__printf(6, 0)
+static int __drmm_crtc_init_with_planes(struct drm_device *dev,
+					struct drm_crtc *crtc,
+					struct drm_plane *primary,
+					struct drm_plane *cursor,
+					const struct drm_crtc_funcs *funcs,
+					const char *name,
+					va_list args)
+{
+	int ret;
+
+	drm_WARN_ON(dev, funcs && funcs->destroy);
+
+	ret = __drm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs,
+					  name, args);
+	if (ret)
+		return ret;
+
+	ret = drmm_add_action_or_reset(dev, drmm_crtc_init_with_planes_cleanup,
+				       crtc);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+/**
+ * drmm_crtc_init_with_planes - Initialise a new CRTC object with
+ *    specified primary and cursor planes.
+ * @dev: DRM device
+ * @crtc: CRTC object to init
+ * @primary: Primary plane for CRTC
+ * @cursor: Cursor plane for CRTC
+ * @funcs: callbacks for the new CRTC
+ * @name: printf style format string for the CRTC name, or NULL for default name
+ *
+ * Inits a new object created as base part of a driver crtc object. Drivers
+ * should use this function instead of drm_crtc_init(), which is only provided
+ * for backwards compatibility with drivers which do not yet support universal
+ * planes). For really simple hardware which has only 1 plane look at
+ * drm_simple_display_pipe_init() instead.
+ *
+ * Cleanup is automatically handled through registering
+ * drmm_crtc_cleanup() with drmm_add_action(). The crtc structure should
+ * be allocated with drmm_kzalloc().
+ *
+ * The @drm_crtc_funcs.destroy hook must be NULL.
+ *
+ * The @primary and @cursor planes are only relevant for legacy uAPI, see
+ * &drm_crtc.primary and &drm_crtc.cursor.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drmm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
+			       struct drm_plane *primary,
+			       struct drm_plane *cursor,
+			       const struct drm_crtc_funcs *funcs,
+			       const char *name, ...)
+{
+	va_list ap;
+	int ret;
+
+	va_start(ap, name);
+	ret = __drmm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs,
+					   name, ap);
+	va_end(ap);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+EXPORT_SYMBOL(drmm_crtc_init_with_planes);
+
 void *__drmm_crtc_alloc_with_planes(struct drm_device *dev,
 				    size_t size, size_t offset,
 				    struct drm_plane *primary,
@@ -400,17 +475,12 @@ void *__drmm_crtc_alloc_with_planes(struct drm_device *dev,
 	crtc = container + offset;
 
 	va_start(ap, name);
-	ret = __drm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs,
-					  name, ap);
+	ret = __drmm_crtc_init_with_planes(dev, crtc, primary, cursor, funcs,
+					   name, ap);
 	va_end(ap);
 	if (ret)
 		return ERR_PTR(ret);
 
-	ret = drmm_add_action_or_reset(dev, drmm_crtc_alloc_with_planes_cleanup,
-				       crtc);
-	if (ret)
-		return ERR_PTR(ret);
-
 	return container;
 }
 EXPORT_SYMBOL(__drmm_crtc_alloc_with_planes);
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 8a6d54515f92..457448cc60f7 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -45,7 +45,6 @@
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
 
diff --git a/drivers/gpu/drm/drm_damage_helper.c b/drivers/gpu/drm/drm_damage_helper.c
index 937b699ac2a8..d8b2955e88fd 100644
--- a/drivers/gpu/drm/drm_damage_helper.c
+++ b/drivers/gpu/drm/drm_damage_helper.c
@@ -224,6 +224,7 @@ drm_atomic_helper_damage_iter_init(struct drm_atomic_helper_damage_iter *iter,
 				   const struct drm_plane_state *old_state,
 				   const struct drm_plane_state *state)
 {
+	struct drm_rect src;
 	memset(iter, 0, sizeof(*iter));
 
 	if (!state || !state->crtc || !state->fb || !state->visible)
@@ -233,10 +234,12 @@ drm_atomic_helper_damage_iter_init(struct drm_atomic_helper_damage_iter *iter,
 	iter->num_clips = drm_plane_get_damage_clips_count(state);
 
 	/* Round down for x1/y1 and round up for x2/y2 to catch all pixels */
-	iter->plane_src.x1 = state->src.x1 >> 16;
-	iter->plane_src.y1 = state->src.y1 >> 16;
-	iter->plane_src.x2 = (state->src.x2 >> 16) + !!(state->src.x2 & 0xFFFF);
-	iter->plane_src.y2 = (state->src.y2 >> 16) + !!(state->src.y2 & 0xFFFF);
+	src = drm_plane_state_src(state);
+
+	iter->plane_src.x1 = src.x1 >> 16;
+	iter->plane_src.y1 = src.y1 >> 16;
+	iter->plane_src.x2 = (src.x2 >> 16) + !!(src.x2 & 0xFFFF);
+	iter->plane_src.y2 = (src.y2 >> 16) + !!(src.y2 & 0xFFFF);
 
 	if (!iter->clips || !drm_rect_equals(&state->src, &old_state->src)) {
 		iter->clips = NULL;
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index eaa819381281..4005dab6147d 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -5165,6 +5165,51 @@ static void fixup_detailed_cea_mode_clock(struct drm_display_mode *mode)
 	mode->clock = clock;
 }
 
+static void drm_calculate_luminance_range(struct drm_connector *connector)
+{
+	struct hdr_static_metadata *hdr_metadata = &connector->hdr_sink_metadata.hdmi_type1;
+	struct drm_luminance_range_info *luminance_range =
+		&connector->display_info.luminance_range;
+	static const u8 pre_computed_values[] = {
+		50, 51, 52, 53, 55, 56, 57, 58, 59, 61, 62, 63, 65, 66, 68, 69,
+		71, 72, 74, 75, 77, 79, 81, 82, 84, 86, 88, 90, 92, 94, 96, 98
+	};
+	u32 max_avg, min_cll, max, min, q, r;
+
+	if (!(hdr_metadata->metadata_type & BIT(HDMI_STATIC_METADATA_TYPE1)))
+		return;
+
+	max_avg = hdr_metadata->max_fall;
+	min_cll = hdr_metadata->min_cll;
+
+	/*
+	 * From the specification (CTA-861-G), for calculating the maximum
+	 * luminance we need to use:
+	 *	Luminance = 50*2**(CV/32)
+	 * Where CV is a one-byte value.
+	 * For calculating this expression we may need float point precision;
+	 * to avoid this complexity level, we take advantage that CV is divided
+	 * by a constant. From the Euclids division algorithm, we know that CV
+	 * can be written as: CV = 32*q + r. Next, we replace CV in the
+	 * Luminance expression and get 50*(2**q)*(2**(r/32)), hence we just
+	 * need to pre-compute the value of r/32. For pre-computing the values
+	 * We just used the following Ruby line:
+	 *	(0...32).each {|cv| puts (50*2**(cv/32.0)).round}
+	 * The results of the above expressions can be verified at
+	 * pre_computed_values.
+	 */
+	q = max_avg >> 5;
+	r = max_avg % 32;
+	max = (1 << q) * pre_computed_values[r];
+
+	/* min luminance: maxLum * (CV/255)^2 / 100 */
+	q = DIV_ROUND_CLOSEST(min_cll, 255);
+	min = max * DIV_ROUND_CLOSEST((q * q), 100);
+
+	luminance_range->min_luminance = min;
+	luminance_range->max_luminance = max;
+}
+
 static uint8_t eotf_supported(const u8 *edid_ext)
 {
 	return edid_ext[2] &
@@ -5196,8 +5241,12 @@ drm_parse_hdr_metadata_block(struct drm_connector *connector, const u8 *db)
 		connector->hdr_sink_metadata.hdmi_type1.max_cll = db[4];
 	if (len >= 5)
 		connector->hdr_sink_metadata.hdmi_type1.max_fall = db[5];
-	if (len >= 6)
+	if (len >= 6) {
 		connector->hdr_sink_metadata.hdmi_type1.min_cll = db[6];
+
+		/* Calculate only when all values are available */
+		drm_calculate_luminance_range(connector);
+	}
 }
 
 static void
@@ -6113,6 +6162,7 @@ static void drm_reset_display_info(struct drm_connector *connector)
 
 	info->non_desktop = 0;
 	memset(&info->monitor_range, 0, sizeof(info->monitor_range));
+	memset(&info->luminance_range, 0, sizeof(info->luminance_range));
 
 	info->mso_stream_count = 0;
 	info->mso_pixel_overlap = 0;
diff --git a/drivers/gpu/drm/drm_encoder.c b/drivers/gpu/drm/drm_encoder.c
index a940024c8087..1143bc7f3252 100644
--- a/drivers/gpu/drm/drm_encoder.c
+++ b/drivers/gpu/drm/drm_encoder.c
@@ -27,6 +27,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_managed.h>
+#include <drm/drm_print.h>
 
 #include "drm_crtc_internal.h"
 
@@ -148,9 +149,9 @@ out_put:
  * the encoder structure. The encoder structure should not be allocated with
  * devm_kzalloc().
  *
- * Note: consider using drmm_encoder_alloc() instead of drm_encoder_init() to
- * let the DRM managed resource infrastructure take care of cleanup and
- * deallocation.
+ * Note: consider using drmm_encoder_alloc() or drmm_encoder_init()
+ * instead of drm_encoder_init() to let the DRM managed resource
+ * infrastructure take care of cleanup and deallocation.
  *
  * Returns:
  * Zero on success, error code on failure.
@@ -212,6 +213,30 @@ static void drmm_encoder_alloc_release(struct drm_device *dev, void *ptr)
 	drm_encoder_cleanup(encoder);
 }
 
+__printf(5, 0)
+static int __drmm_encoder_init(struct drm_device *dev,
+			       struct drm_encoder *encoder,
+			       const struct drm_encoder_funcs *funcs,
+			       int encoder_type,
+			       const char *name,
+			       va_list args)
+{
+	int ret;
+
+	if (drm_WARN_ON(dev, funcs && funcs->destroy))
+		return -EINVAL;
+
+	ret = __drm_encoder_init(dev, encoder, funcs, encoder_type, name, args);
+	if (ret)
+		return ret;
+
+	ret = drmm_add_action_or_reset(dev, drmm_encoder_alloc_release, encoder);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
 void *__drmm_encoder_alloc(struct drm_device *dev, size_t size, size_t offset,
 			   const struct drm_encoder_funcs *funcs,
 			   int encoder_type, const char *name, ...)
@@ -221,9 +246,6 @@ void *__drmm_encoder_alloc(struct drm_device *dev, size_t size, size_t offset,
 	va_list ap;
 	int ret;
 
-	if (WARN_ON(funcs && funcs->destroy))
-		return ERR_PTR(-EINVAL);
-
 	container = drmm_kzalloc(dev, size, GFP_KERNEL);
 	if (!container)
 		return ERR_PTR(-ENOMEM);
@@ -231,19 +253,50 @@ void *__drmm_encoder_alloc(struct drm_device *dev, size_t size, size_t offset,
 	encoder = container + offset;
 
 	va_start(ap, name);
-	ret = __drm_encoder_init(dev, encoder, funcs, encoder_type, name, ap);
+	ret = __drmm_encoder_init(dev, encoder, funcs, encoder_type, name, ap);
 	va_end(ap);
 	if (ret)
 		return ERR_PTR(ret);
 
-	ret = drmm_add_action_or_reset(dev, drmm_encoder_alloc_release, encoder);
-	if (ret)
-		return ERR_PTR(ret);
-
 	return container;
 }
 EXPORT_SYMBOL(__drmm_encoder_alloc);
 
+/**
+ * drmm_encoder_init - Initialize a preallocated encoder
+ * @dev: drm device
+ * @encoder: the encoder to init
+ * @funcs: callbacks for this encoder (optional)
+ * @encoder_type: user visible type of the encoder
+ * @name: printf style format string for the encoder name, or NULL for default name
+ *
+ * Initializes a preallocated encoder. Encoder should be subclassed as
+ * part of driver encoder objects. Cleanup is automatically handled
+ * through registering drm_encoder_cleanup() with drmm_add_action(). The
+ * encoder structure should be allocated with drmm_kzalloc().
+ *
+ * The @drm_encoder_funcs.destroy hook must be NULL.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drmm_encoder_init(struct drm_device *dev, struct drm_encoder *encoder,
+		      const struct drm_encoder_funcs *funcs,
+		      int encoder_type, const char *name, ...)
+{
+	va_list ap;
+	int ret;
+
+	va_start(ap, name);
+	ret = __drmm_encoder_init(dev, encoder, funcs, encoder_type, name, ap);
+	va_end(ap);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+EXPORT_SYMBOL(drmm_encoder_init);
+
 static struct drm_crtc *drm_encoder_get_crtc(struct drm_encoder *encoder)
 {
 	struct drm_connector *connector;
diff --git a/drivers/gpu/drm/drm_fb_cma_helper.c b/drivers/gpu/drm/drm_fb_dma_helper.c
index 69c57273b184..3b535ad1b07c 100644
--- a/drivers/gpu/drm/drm_fb_cma_helper.c
+++ b/drivers/gpu/drm/drm_fb_dma_helper.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * drm kms/fb cma (contiguous memory allocator) helper functions
+ * drm kms/fb dma helper functions
  *
  * Copyright (C) 2012 Analog Devices Inc.
  *   Author: Lars-Peter Clausen <lars@metafoo.de>
@@ -10,35 +10,40 @@
  */
 
 #include <drm/drm_damage_helper.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_plane.h>
 #include <linux/dma-mapping.h>
 #include <linux/module.h>
 
 /**
- * DOC: framebuffer cma helper functions
+ * DOC: framebuffer dma helper functions
  *
- * Provides helper functions for creating a cma (contiguous memory allocator)
- * backed framebuffer.
+ * Provides helper functions for creating a DMA-contiguous framebuffer.
+ *
+ * Depending on the platform, the buffers may be physically non-contiguous and
+ * mapped through an IOMMU or a similar mechanism, or allocated from
+ * physically-contiguous memory (using, for instance, CMA or a pool of memory
+ * reserved at early boot). This is handled behind the scenes by the DMA mapping
+ * API.
  *
  * drm_gem_fb_create() is used in the &drm_mode_config_funcs.fb_create
- * callback function to create a cma backed framebuffer.
+ * callback function to create a DMA-contiguous framebuffer.
  */
 
 /**
- * drm_fb_cma_get_gem_obj() - Get CMA GEM object for framebuffer
+ * drm_fb_dma_get_gem_obj() - Get DMA GEM object for framebuffer
  * @fb: The framebuffer
  * @plane: Which plane
  *
- * Return the CMA GEM object for given framebuffer.
+ * Return the DMA GEM object for given framebuffer.
  *
  * This function will usually be called from the CRTC callback functions.
  */
-struct drm_gem_cma_object *drm_fb_cma_get_gem_obj(struct drm_framebuffer *fb,
+struct drm_gem_dma_object *drm_fb_dma_get_gem_obj(struct drm_framebuffer *fb,
 						  unsigned int plane)
 {
 	struct drm_gem_object *gem;
@@ -47,27 +52,27 @@ struct drm_gem_cma_object *drm_fb_cma_get_gem_obj(struct drm_framebuffer *fb,
 	if (!gem)
 		return NULL;
 
-	return to_drm_gem_cma_obj(gem);
+	return to_drm_gem_dma_obj(gem);
 }
-EXPORT_SYMBOL_GPL(drm_fb_cma_get_gem_obj);
+EXPORT_SYMBOL_GPL(drm_fb_dma_get_gem_obj);
 
 /**
- * drm_fb_cma_get_gem_addr() - Get physical address for framebuffer, for pixel
+ * drm_fb_dma_get_gem_addr() - Get DMA (bus) address for framebuffer, for pixel
  * formats where values are grouped in blocks this will get you the beginning of
  * the block
  * @fb: The framebuffer
  * @state: Which state of drm plane
  * @plane: Which plane
- * Return the CMA GEM address for given framebuffer.
+ * Return the DMA GEM address for given framebuffer.
  *
  * This function will usually be called from the PLANE callback functions.
  */
-dma_addr_t drm_fb_cma_get_gem_addr(struct drm_framebuffer *fb,
+dma_addr_t drm_fb_dma_get_gem_addr(struct drm_framebuffer *fb,
 				   struct drm_plane_state *state,
 				   unsigned int plane)
 {
-	struct drm_gem_cma_object *obj;
-	dma_addr_t paddr;
+	struct drm_gem_dma_object *obj;
+	dma_addr_t dma_addr;
 	u8 h_div = 1, v_div = 1;
 	u32 block_w = drm_format_info_block_width(fb->format, plane);
 	u32 block_h = drm_format_info_block_height(fb->format, plane);
@@ -77,11 +82,11 @@ dma_addr_t drm_fb_cma_get_gem_addr(struct drm_framebuffer *fb,
 	u32 block_start_y;
 	u32 num_hblocks;
 
-	obj = drm_fb_cma_get_gem_obj(fb, plane);
+	obj = drm_fb_dma_get_gem_obj(fb, plane);
 	if (!obj)
 		return 0;
 
-	paddr = obj->paddr + fb->offsets[plane];
+	dma_addr = obj->dma_addr + fb->offsets[plane];
 
 	if (plane > 0) {
 		h_div = fb->format->hsub;
@@ -93,43 +98,43 @@ dma_addr_t drm_fb_cma_get_gem_addr(struct drm_framebuffer *fb,
 	block_start_y = (sample_y / block_h) * block_h;
 	num_hblocks = sample_x / block_w;
 
-	paddr += fb->pitches[plane] * block_start_y;
-	paddr += block_size * num_hblocks;
+	dma_addr += fb->pitches[plane] * block_start_y;
+	dma_addr += block_size * num_hblocks;
 
-	return paddr;
+	return dma_addr;
 }
-EXPORT_SYMBOL_GPL(drm_fb_cma_get_gem_addr);
+EXPORT_SYMBOL_GPL(drm_fb_dma_get_gem_addr);
 
 /**
- * drm_fb_cma_sync_non_coherent - Sync GEM object to non-coherent backing
+ * drm_fb_dma_sync_non_coherent - Sync GEM object to non-coherent backing
  *	memory
  * @drm: DRM device
  * @old_state: Old plane state
  * @state: New plane state
  *
  * This function can be used by drivers that use damage clips and have
- * CMA GEM objects backed by non-coherent memory. Calling this function
+ * DMA GEM objects backed by non-coherent memory. Calling this function
  * in a plane's .atomic_update ensures that all the data in the backing
  * memory have been written to RAM.
  */
-void drm_fb_cma_sync_non_coherent(struct drm_device *drm,
+void drm_fb_dma_sync_non_coherent(struct drm_device *drm,
 				  struct drm_plane_state *old_state,
 				  struct drm_plane_state *state)
 {
 	const struct drm_format_info *finfo = state->fb->format;
 	struct drm_atomic_helper_damage_iter iter;
-	const struct drm_gem_cma_object *cma_obj;
+	const struct drm_gem_dma_object *dma_obj;
 	unsigned int offset, i;
 	struct drm_rect clip;
 	dma_addr_t daddr;
 	size_t nb_bytes;
 
 	for (i = 0; i < finfo->num_planes; i++) {
-		cma_obj = drm_fb_cma_get_gem_obj(state->fb, i);
-		if (!cma_obj->map_noncoherent)
+		dma_obj = drm_fb_dma_get_gem_obj(state->fb, i);
+		if (!dma_obj->map_noncoherent)
 			continue;
 
-		daddr = drm_fb_cma_get_gem_addr(state->fb, state, i);
+		daddr = drm_fb_dma_get_gem_addr(state->fb, state, i);
 		drm_atomic_helper_damage_iter_init(&iter, old_state, state);
 
 		drm_atomic_for_each_plane_damage(&iter, &clip) {
@@ -142,4 +147,4 @@ void drm_fb_cma_sync_non_coherent(struct drm_device *drm,
 		}
 	}
 }
-EXPORT_SYMBOL_GPL(drm_fb_cma_sync_non_coherent);
+EXPORT_SYMBOL_GPL(drm_fb_dma_sync_non_coherent);
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 2d4cee6a10ff..71edb80fe0fb 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -377,12 +377,31 @@ static void drm_fb_helper_damage_blit_real(struct drm_fb_helper *fb_helper,
 					   struct iosys_map *dst)
 {
 	struct drm_framebuffer *fb = fb_helper->fb;
-	unsigned int cpp = fb->format->cpp[0];
-	size_t offset = clip->y1 * fb->pitches[0] + clip->x1 * cpp;
-	void *src = fb_helper->fbdev->screen_buffer + offset;
-	size_t len = (clip->x2 - clip->x1) * cpp;
+	size_t offset = clip->y1 * fb->pitches[0];
+	size_t len = clip->x2 - clip->x1;
 	unsigned int y;
+	void *src;
 
+	switch (drm_format_info_bpp(fb->format, 0)) {
+	case 1:
+		offset += clip->x1 / 8;
+		len = DIV_ROUND_UP(len + clip->x1 % 8, 8);
+		break;
+	case 2:
+		offset += clip->x1 / 4;
+		len = DIV_ROUND_UP(len + clip->x1 % 4, 4);
+		break;
+	case 4:
+		offset += clip->x1 / 2;
+		len = DIV_ROUND_UP(len + clip->x1 % 2, 2);
+		break;
+	default:
+		offset += clip->x1 * fb->format->cpp[0];
+		len *= fb->format->cpp[0];
+		break;
+	}
+
+	src = fb_helper->fbdev->screen_buffer + offset;
 	iosys_map_incr(dst, offset); /* go to first pixel within clip rect */
 
 	for (y = clip->y1; y < clip->y2; y++) {
@@ -1274,19 +1293,23 @@ static bool drm_fb_pixel_format_equal(const struct fb_var_screeninfo *var_1,
 }
 
 static void drm_fb_helper_fill_pixel_fmt(struct fb_var_screeninfo *var,
-					 u8 depth)
+					 const struct drm_format_info *format)
 {
-	switch (depth) {
-	case 8:
+	u8 depth = format->depth;
+
+	if (format->is_color_indexed) {
 		var->red.offset = 0;
 		var->green.offset = 0;
 		var->blue.offset = 0;
-		var->red.length = 8; /* 8bit DAC */
-		var->green.length = 8;
-		var->blue.length = 8;
+		var->red.length = depth;
+		var->green.length = depth;
+		var->blue.length = depth;
 		var->transp.offset = 0;
 		var->transp.length = 0;
-		break;
+		return;
+	}
+
+	switch (depth) {
 	case 15:
 		var->red.offset = 10;
 		var->green.offset = 5;
@@ -1341,7 +1364,9 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var,
 {
 	struct drm_fb_helper *fb_helper = info->par;
 	struct drm_framebuffer *fb = fb_helper->fb;
+	const struct drm_format_info *format = fb->format;
 	struct drm_device *dev = fb_helper->dev;
+	unsigned int bpp;
 
 	if (in_dbg_master())
 		return -EINVAL;
@@ -1351,22 +1376,33 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var,
 		var->pixclock = 0;
 	}
 
-	if ((drm_format_info_block_width(fb->format, 0) > 1) ||
-	    (drm_format_info_block_height(fb->format, 0) > 1))
-		return -EINVAL;
+	switch (format->format) {
+	case DRM_FORMAT_C1:
+	case DRM_FORMAT_C2:
+	case DRM_FORMAT_C4:
+		/* supported format with sub-byte pixels */
+		break;
+
+	default:
+		if ((drm_format_info_block_width(format, 0) > 1) ||
+		    (drm_format_info_block_height(format, 0) > 1))
+			return -EINVAL;
+		break;
+	}
 
 	/*
 	 * Changes struct fb_var_screeninfo are currently not pushed back
 	 * to KMS, hence fail if different settings are requested.
 	 */
-	if (var->bits_per_pixel > fb->format->cpp[0] * 8 ||
+	bpp = drm_format_info_bpp(format, 0);
+	if (var->bits_per_pixel > bpp ||
 	    var->xres > fb->width || var->yres > fb->height ||
 	    var->xres_virtual > fb->width || var->yres_virtual > fb->height) {
 		drm_dbg_kms(dev, "fb requested width/height/bpp can't fit in current fb "
 			  "request %dx%d-%d (virtual %dx%d) > %dx%d-%d\n",
 			  var->xres, var->yres, var->bits_per_pixel,
 			  var->xres_virtual, var->yres_virtual,
-			  fb->width, fb->height, fb->format->cpp[0] * 8);
+			  fb->width, fb->height, bpp);
 		return -EINVAL;
 	}
 
@@ -1381,13 +1417,13 @@ int drm_fb_helper_check_var(struct fb_var_screeninfo *var,
 	    !var->blue.length    && !var->transp.length   &&
 	    !var->red.msb_right  && !var->green.msb_right &&
 	    !var->blue.msb_right && !var->transp.msb_right) {
-		drm_fb_helper_fill_pixel_fmt(var, fb->format->depth);
+		drm_fb_helper_fill_pixel_fmt(var, format);
 	}
 
 	/*
 	 * Likewise, bits_per_pixel should be rounded up to a supported value.
 	 */
-	var->bits_per_pixel = fb->format->cpp[0] * 8;
+	var->bits_per_pixel = bpp;
 
 	/*
 	 * drm fbdev emulation doesn't support changing the pixel format at all,
@@ -1723,11 +1759,11 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper,
 }
 
 static void drm_fb_helper_fill_fix(struct fb_info *info, uint32_t pitch,
-				   uint32_t depth)
+				   bool is_color_indexed)
 {
 	info->fix.type = FB_TYPE_PACKED_PIXELS;
-	info->fix.visual = depth == 8 ? FB_VISUAL_PSEUDOCOLOR :
-		FB_VISUAL_TRUECOLOR;
+	info->fix.visual = is_color_indexed ? FB_VISUAL_PSEUDOCOLOR
+					    : FB_VISUAL_TRUECOLOR;
 	info->fix.mmio_start = 0;
 	info->fix.mmio_len = 0;
 	info->fix.type_aux = 0;
@@ -1744,19 +1780,31 @@ static void drm_fb_helper_fill_var(struct fb_info *info,
 				   uint32_t fb_width, uint32_t fb_height)
 {
 	struct drm_framebuffer *fb = fb_helper->fb;
+	const struct drm_format_info *format = fb->format;
+
+	switch (format->format) {
+	case DRM_FORMAT_C1:
+	case DRM_FORMAT_C2:
+	case DRM_FORMAT_C4:
+		/* supported format with sub-byte pixels */
+		break;
+
+	default:
+		WARN_ON((drm_format_info_block_width(format, 0) > 1) ||
+			(drm_format_info_block_height(format, 0) > 1));
+		break;
+	}
 
-	WARN_ON((drm_format_info_block_width(fb->format, 0) > 1) ||
-		(drm_format_info_block_height(fb->format, 0) > 1));
 	info->pseudo_palette = fb_helper->pseudo_palette;
 	info->var.xres_virtual = fb->width;
 	info->var.yres_virtual = fb->height;
-	info->var.bits_per_pixel = fb->format->cpp[0] * 8;
+	info->var.bits_per_pixel = drm_format_info_bpp(format, 0);
 	info->var.accel_flags = FB_ACCELF_TEXT;
 	info->var.xoffset = 0;
 	info->var.yoffset = 0;
 	info->var.activate = FB_ACTIVATE_NOW;
 
-	drm_fb_helper_fill_pixel_fmt(&info->var, fb->format->depth);
+	drm_fb_helper_fill_pixel_fmt(&info->var, format);
 
 	info->var.xres = fb_width;
 	info->var.yres = fb_height;
@@ -1781,7 +1829,8 @@ void drm_fb_helper_fill_info(struct fb_info *info,
 {
 	struct drm_framebuffer *fb = fb_helper->fb;
 
-	drm_fb_helper_fill_fix(info, fb->pitches[0], fb->format->depth);
+	drm_fb_helper_fill_fix(info, fb->pitches[0],
+			       fb->format->is_color_indexed);
 	drm_fb_helper_fill_var(info, fb_helper,
 			       sizes->fb_width, sizes->fb_height);
 
diff --git a/drivers/gpu/drm/drm_file.c b/drivers/gpu/drm/drm_file.c
index dc7d2e5b16c8..a8b4d918e9a3 100644
--- a/drivers/gpu/drm/drm_file.c
+++ b/drivers/gpu/drm/drm_file.c
@@ -48,11 +48,6 @@
 #include "drm_internal.h"
 #include "drm_legacy.h"
 
-#if defined(CONFIG_MMU) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
-#include <uapi/asm/mman.h>
-#include <drm/drm_vma_manager.h>
-#endif
-
 /* from BKL pushdown */
 DEFINE_MUTEX(drm_global_mutex);
 
@@ -131,7 +126,7 @@ bool drm_dev_needs_global_mutex(struct drm_device *dev)
  *     };
  *
  * For plain GEM based drivers there is the DEFINE_DRM_GEM_FOPS() macro, and for
- * CMA based drivers there is the DEFINE_DRM_GEM_CMA_FOPS() macro to make this
+ * DMA based drivers there is the DEFINE_DRM_GEM_DMA_FOPS() macro to make this
  * simpler.
  *
  * The driver's &file_operations must be stored in &drm_driver.fops.
@@ -912,139 +907,3 @@ struct file *mock_drm_getfile(struct drm_minor *minor, unsigned int flags)
 	return file;
 }
 EXPORT_SYMBOL_FOR_TESTS_ONLY(mock_drm_getfile);
-
-#ifdef CONFIG_MMU
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-/*
- * drm_addr_inflate() attempts to construct an aligned area by inflating
- * the area size and skipping the unaligned start of the area.
- * adapted from shmem_get_unmapped_area()
- */
-static unsigned long drm_addr_inflate(unsigned long addr,
-				      unsigned long len,
-				      unsigned long pgoff,
-				      unsigned long flags,
-				      unsigned long huge_size)
-{
-	unsigned long offset, inflated_len;
-	unsigned long inflated_addr;
-	unsigned long inflated_offset;
-
-	offset = (pgoff << PAGE_SHIFT) & (huge_size - 1);
-	if (offset && offset + len < 2 * huge_size)
-		return addr;
-	if ((addr & (huge_size - 1)) == offset)
-		return addr;
-
-	inflated_len = len + huge_size - PAGE_SIZE;
-	if (inflated_len > TASK_SIZE)
-		return addr;
-	if (inflated_len < len)
-		return addr;
-
-	inflated_addr = current->mm->get_unmapped_area(NULL, 0, inflated_len,
-						       0, flags);
-	if (IS_ERR_VALUE(inflated_addr))
-		return addr;
-	if (inflated_addr & ~PAGE_MASK)
-		return addr;
-
-	inflated_offset = inflated_addr & (huge_size - 1);
-	inflated_addr += offset - inflated_offset;
-	if (inflated_offset > offset)
-		inflated_addr += huge_size;
-
-	if (inflated_addr > TASK_SIZE - len)
-		return addr;
-
-	return inflated_addr;
-}
-
-/**
- * drm_get_unmapped_area() - Get an unused user-space virtual memory area
- * suitable for huge page table entries.
- * @file: The struct file representing the address space being mmap()'d.
- * @uaddr: Start address suggested by user-space.
- * @len: Length of the area.
- * @pgoff: The page offset into the address space.
- * @flags: mmap flags
- * @mgr: The address space manager used by the drm driver. This argument can
- * probably be removed at some point when all drivers use the same
- * address space manager.
- *
- * This function attempts to find an unused user-space virtual memory area
- * that can accommodate the size we want to map, and that is properly
- * aligned to facilitate huge page table entries matching actual
- * huge pages or huge page aligned memory in buffer objects. Buffer objects
- * are assumed to start at huge page boundary pfns (io memory) or be
- * populated by huge pages aligned to the start of the buffer object
- * (system- or coherent memory). Adapted from shmem_get_unmapped_area.
- *
- * Return: aligned user-space address.
- */
-unsigned long drm_get_unmapped_area(struct file *file,
-				    unsigned long uaddr, unsigned long len,
-				    unsigned long pgoff, unsigned long flags,
-				    struct drm_vma_offset_manager *mgr)
-{
-	unsigned long addr;
-	unsigned long inflated_addr;
-	struct drm_vma_offset_node *node;
-
-	if (len > TASK_SIZE)
-		return -ENOMEM;
-
-	/*
-	 * @pgoff is the file page-offset the huge page boundaries of
-	 * which typically aligns to physical address huge page boundaries.
-	 * That's not true for DRM, however, where physical address huge
-	 * page boundaries instead are aligned with the offset from
-	 * buffer object start. So adjust @pgoff to be the offset from
-	 * buffer object start.
-	 */
-	drm_vma_offset_lock_lookup(mgr);
-	node = drm_vma_offset_lookup_locked(mgr, pgoff, 1);
-	if (node)
-		pgoff -= node->vm_node.start;
-	drm_vma_offset_unlock_lookup(mgr);
-
-	addr = current->mm->get_unmapped_area(file, uaddr, len, pgoff, flags);
-	if (IS_ERR_VALUE(addr))
-		return addr;
-	if (addr & ~PAGE_MASK)
-		return addr;
-	if (addr > TASK_SIZE - len)
-		return addr;
-
-	if (len < HPAGE_PMD_SIZE)
-		return addr;
-	if (flags & MAP_FIXED)
-		return addr;
-	/*
-	 * Our priority is to support MAP_SHARED mapped hugely;
-	 * and support MAP_PRIVATE mapped hugely too, until it is COWed.
-	 * But if caller specified an address hint, respect that as before.
-	 */
-	if (uaddr)
-		return addr;
-
-	inflated_addr = drm_addr_inflate(addr, len, pgoff, flags,
-					 HPAGE_PMD_SIZE);
-
-	if (IS_ENABLED(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) &&
-	    len >= HPAGE_PUD_SIZE)
-		inflated_addr = drm_addr_inflate(inflated_addr, len, pgoff,
-						 flags, HPAGE_PUD_SIZE);
-	return inflated_addr;
-}
-#else /* CONFIG_TRANSPARENT_HUGEPAGE */
-unsigned long drm_get_unmapped_area(struct file *file,
-				    unsigned long uaddr, unsigned long len,
-				    unsigned long pgoff, unsigned long flags,
-				    struct drm_vma_offset_manager *mgr)
-{
-	return current->mm->get_unmapped_area(file, uaddr, len, pgoff, flags);
-}
-#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
-EXPORT_SYMBOL_GPL(drm_get_unmapped_area);
-#endif /* CONFIG_MMU */
diff --git a/drivers/gpu/drm/drm_format_helper.c b/drivers/gpu/drm/drm_format_helper.c
index a3ccd8bc966f..e2f76621453c 100644
--- a/drivers/gpu/drm/drm_format_helper.c
+++ b/drivers/gpu/drm/drm_format_helper.c
@@ -8,9 +8,10 @@
  * (at your option) any later version.
  */
 
+#include <linux/io.h>
+#include <linux/iosys-map.h>
 #include <linux/module.h>
 #include <linux/slab.h>
-#include <linux/io.h>
 
 #include <drm/drm_device.h>
 #include <drm/drm_format_helper.h>
@@ -40,11 +41,11 @@ unsigned int drm_fb_clip_offset(unsigned int pitch, const struct drm_format_info
 }
 EXPORT_SYMBOL(drm_fb_clip_offset);
 
-/* TODO: Make this functon work with multi-plane formats. */
-static int drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_pixsize,
-		       const void *vaddr, const struct drm_framebuffer *fb,
-		       const struct drm_rect *clip, bool vaddr_cached_hint,
-		       void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
+/* TODO: Make this function work with multi-plane formats. */
+static int __drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_pixsize,
+			 const void *vaddr, const struct drm_framebuffer *fb,
+			 const struct drm_rect *clip, bool vaddr_cached_hint,
+			 void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
 {
 	unsigned long linepixels = drm_rect_width(clip);
 	unsigned long lines = drm_rect_height(clip);
@@ -54,7 +55,7 @@ static int drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_pix
 	const void *sbuf;
 
 	/*
-	 * Some source buffers, such as CMA memory, use write-combine
+	 * Some source buffers, such as DMA memory, use write-combine
 	 * caching, so reads are uncached. Speed up access by fetching
 	 * one line at a time.
 	 */
@@ -83,11 +84,11 @@ static int drm_fb_xfrm(void *dst, unsigned long dst_pitch, unsigned long dst_pix
 	return 0;
 }
 
-/* TODO: Make this functon work with multi-plane formats. */
-static int drm_fb_xfrm_toio(void __iomem *dst, unsigned long dst_pitch, unsigned long dst_pixsize,
-			    const void *vaddr, const struct drm_framebuffer *fb,
-			    const struct drm_rect *clip, bool vaddr_cached_hint,
-			    void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
+/* TODO: Make this function work with multi-plane formats. */
+static int __drm_fb_xfrm_toio(void __iomem *dst, unsigned long dst_pitch, unsigned long dst_pixsize,
+			      const void *vaddr, const struct drm_framebuffer *fb,
+			      const struct drm_rect *clip, bool vaddr_cached_hint,
+			      void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
 {
 	unsigned long linepixels = drm_rect_width(clip);
 	unsigned long lines = drm_rect_height(clip);
@@ -128,65 +129,82 @@ static int drm_fb_xfrm_toio(void __iomem *dst, unsigned long dst_pitch, unsigned
 	return 0;
 }
 
-/**
- * drm_fb_memcpy - Copy clip buffer
- * @dst: Destination buffer
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @vaddr: Source buffer
- * @fb: DRM framebuffer
- * @clip: Clip rectangle area to copy
- *
- * This function does not apply clipping on dst, i.e. the destination
- * is at the top-left corner.
- */
-void drm_fb_memcpy(void *dst, unsigned int dst_pitch, const void *vaddr,
-		   const struct drm_framebuffer *fb, const struct drm_rect *clip)
+/* TODO: Make this function work with multi-plane formats. */
+static int drm_fb_xfrm(struct iosys_map *dst,
+		       const unsigned int *dst_pitch, const u8 *dst_pixsize,
+		       const struct iosys_map *src, const struct drm_framebuffer *fb,
+		       const struct drm_rect *clip, bool vaddr_cached_hint,
+		       void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels))
 {
-	unsigned int cpp = fb->format->cpp[0];
-	size_t len = (clip->x2 - clip->x1) * cpp;
-	unsigned int y, lines = clip->y2 - clip->y1;
+	static const unsigned int default_dst_pitch[DRM_FORMAT_MAX_PLANES] = {
+		0, 0, 0, 0
+	};
 
 	if (!dst_pitch)
-		dst_pitch = len;
+		dst_pitch = default_dst_pitch;
 
-	vaddr += clip_offset(clip, fb->pitches[0], cpp);
-	for (y = 0; y < lines; y++) {
-		memcpy(dst, vaddr, len);
-		vaddr += fb->pitches[0];
-		dst += dst_pitch;
-	}
+	/* TODO: handle src in I/O memory here */
+	if (dst[0].is_iomem)
+		return __drm_fb_xfrm_toio(dst[0].vaddr_iomem, dst_pitch[0], dst_pixsize[0],
+					  src[0].vaddr, fb, clip, vaddr_cached_hint, xfrm_line);
+	else
+		return __drm_fb_xfrm(dst[0].vaddr, dst_pitch[0], dst_pixsize[0],
+				     src[0].vaddr, fb, clip, vaddr_cached_hint, xfrm_line);
 }
-EXPORT_SYMBOL(drm_fb_memcpy);
 
 /**
- * drm_fb_memcpy_toio - Copy clip buffer
- * @dst: Destination buffer (iomem)
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @vaddr: Source buffer
+ * drm_fb_memcpy - Copy clip buffer
+ * @dst: Array of destination buffers
+ * @dst_pitch: Array of numbers of bytes between the start of two consecutive scanlines
+ *             within @dst; can be NULL if scanlines are stored next to each other.
+ * @src: Array of source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
  *
- * This function does not apply clipping on dst, i.e. the destination
- * is at the top-left corner.
+ * This function copies parts of a framebuffer to display memory. Destination and
+ * framebuffer formats must match. No conversion takes place. The parameters @dst,
+ * @dst_pitch and @src refer to arrays. Each array must have at least as many entries
+ * as there are planes in @fb's format. Each entry stores the value for the format's
+ * respective color plane at the same index.
+ *
+ * This function does not apply clipping on @dst (i.e. the destination is at the
+ * top-left corner).
  */
-void drm_fb_memcpy_toio(void __iomem *dst, unsigned int dst_pitch, const void *vaddr,
-			const struct drm_framebuffer *fb, const struct drm_rect *clip)
+void drm_fb_memcpy(struct iosys_map *dst, const unsigned int *dst_pitch,
+		   const struct iosys_map *src, const struct drm_framebuffer *fb,
+		   const struct drm_rect *clip)
 {
-	unsigned int cpp = fb->format->cpp[0];
-	size_t len = (clip->x2 - clip->x1) * cpp;
-	unsigned int y, lines = clip->y2 - clip->y1;
+	static const unsigned int default_dst_pitch[DRM_FORMAT_MAX_PLANES] = {
+		0, 0, 0, 0
+	};
 
-	if (!dst_pitch)
-		dst_pitch = len;
+	const struct drm_format_info *format = fb->format;
+	unsigned int i, y, lines = drm_rect_height(clip);
 
-	vaddr += clip_offset(clip, fb->pitches[0], cpp);
-	for (y = 0; y < lines; y++) {
-		memcpy_toio(dst, vaddr, len);
-		vaddr += fb->pitches[0];
-		dst += dst_pitch;
+	if (!dst_pitch)
+		dst_pitch = default_dst_pitch;
+
+	for (i = 0; i < format->num_planes; ++i) {
+		unsigned int bpp_i = drm_format_info_bpp(format, i);
+		unsigned int cpp_i = DIV_ROUND_UP(bpp_i, 8);
+		size_t len_i = DIV_ROUND_UP(drm_rect_width(clip) * bpp_i, 8);
+		unsigned int dst_pitch_i = dst_pitch[i];
+		struct iosys_map dst_i = dst[i];
+		struct iosys_map src_i = src[i];
+
+		if (!dst_pitch_i)
+			dst_pitch_i = len_i;
+
+		iosys_map_incr(&src_i, clip_offset(clip, fb->pitches[i], cpp_i));
+		for (y = 0; y < lines; y++) {
+			/* TODO: handle src_i in I/O memory here */
+			iosys_map_memcpy_to(&dst_i, 0, src_i.vaddr, len_i);
+			iosys_map_incr(&src_i, fb->pitches[i]);
+			iosys_map_incr(&dst_i, dst_pitch_i);
+		}
 	}
 }
-EXPORT_SYMBOL(drm_fb_memcpy_toio);
+EXPORT_SYMBOL(drm_fb_memcpy);
 
 static void drm_fb_swab16_line(void *dbuf, const void *sbuf, unsigned int pixels)
 {
@@ -210,37 +228,47 @@ static void drm_fb_swab32_line(void *dbuf, const void *sbuf, unsigned int pixels
 
 /**
  * drm_fb_swab - Swap bytes into clip buffer
- * @dst: Destination buffer
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @src: Source buffer
+ * @dst: Array of destination buffers
+ * @dst_pitch: Array of numbers of bytes between the start of two consecutive scanlines
+ *             within @dst; can be NULL if scanlines are stored next to each other.
+ * @src: Array of source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
  * @cached: Source buffer is mapped cached (eg. not write-combined)
  *
- * If @cached is false a temporary buffer is used to cache one pixel line at a
- * time to speed up slow uncached reads.
+ * This function copies parts of a framebuffer to display memory and swaps per-pixel
+ * bytes during the process. Destination and framebuffer formats must match. The
+ * parameters @dst, @dst_pitch and @src refer to arrays. Each array must have at
+ * least as many entries as there are planes in @fb's format. Each entry stores the
+ * value for the format's respective color plane at the same index. If @cached is
+ * false a temporary buffer is used to cache one pixel line at a time to speed up
+ * slow uncached reads.
  *
- * This function does not apply clipping on dst, i.e. the destination
- * is at the top-left corner.
+ * This function does not apply clipping on @dst (i.e. the destination is at the
+ * top-left corner).
  */
-void drm_fb_swab(void *dst, unsigned int dst_pitch, const void *src,
-		 const struct drm_framebuffer *fb, const struct drm_rect *clip,
-		 bool cached)
+void drm_fb_swab(struct iosys_map *dst, const unsigned int *dst_pitch,
+		 const struct iosys_map *src, const struct drm_framebuffer *fb,
+		 const struct drm_rect *clip, bool cached)
 {
-	u8 cpp = fb->format->cpp[0];
+	const struct drm_format_info *format = fb->format;
+	u8 cpp = DIV_ROUND_UP(drm_format_info_bpp(format, 0), 8);
+	void (*swab_line)(void *dbuf, const void *sbuf, unsigned int npixels);
 
 	switch (cpp) {
 	case 4:
-		drm_fb_xfrm(dst, dst_pitch, cpp, src, fb, clip, cached, drm_fb_swab32_line);
+		swab_line = drm_fb_swab32_line;
 		break;
 	case 2:
-		drm_fb_xfrm(dst, dst_pitch, cpp, src, fb, clip, cached, drm_fb_swab16_line);
+		swab_line = drm_fb_swab16_line;
 		break;
 	default:
 		drm_warn_once(fb->dev, "Format %p4cc has unsupported pixel size.\n",
-			      &fb->format->format);
-		break;
+			      &format->format);
+		return;
 	}
+
+	drm_fb_xfrm(dst, dst_pitch, &cpp, src, fb, clip, cached, swab_line);
 }
 EXPORT_SYMBOL(drm_fb_swab);
 
@@ -261,32 +289,50 @@ static void drm_fb_xrgb8888_to_rgb332_line(void *dbuf, const void *sbuf, unsigne
 
 /**
  * drm_fb_xrgb8888_to_rgb332 - Convert XRGB8888 to RGB332 clip buffer
- * @dst: RGB332 destination buffer
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @src: XRGB8888 source buffer
+ * @dst: Array of RGB332 destination buffers
+ * @dst_pitch: Array of numbers of bytes between the start of two consecutive scanlines
+ *             within @dst; can be NULL if scanlines are stored next to each other.
+ * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
  *
- * Drivers can use this function for RGB332 devices that don't natively support XRGB8888.
+ * This function copies parts of a framebuffer to display memory and converts the
+ * color format during the process. Destination and framebuffer formats must match. The
+ * parameters @dst, @dst_pitch and @src refer to arrays. Each array must have at
+ * least as many entries as there are planes in @fb's format. Each entry stores the
+ * value for the format's respective color plane at the same index.
+ *
+ * This function does not apply clipping on @dst (i.e. the destination is at the
+ * top-left corner).
+ *
+ * Drivers can use this function for RGB332 devices that don't support XRGB8888 natively.
  */
-void drm_fb_xrgb8888_to_rgb332(void *dst, unsigned int dst_pitch, const void *src,
-			       const struct drm_framebuffer *fb, const struct drm_rect *clip)
+void drm_fb_xrgb8888_to_rgb332(struct iosys_map *dst, const unsigned int *dst_pitch,
+			       const struct iosys_map *src, const struct drm_framebuffer *fb,
+			       const struct drm_rect *clip)
 {
-	drm_fb_xfrm(dst, dst_pitch, 1, src, fb, clip, false, drm_fb_xrgb8888_to_rgb332_line);
+	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
+		1,
+	};
+
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+		    drm_fb_xrgb8888_to_rgb332_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb332);
 
 static void drm_fb_xrgb8888_to_rgb565_line(void *dbuf, const void *sbuf, unsigned int pixels)
 {
 	u16 *dbuf16 = dbuf;
-	const u32 *sbuf32 = sbuf;
+	const __le32 *sbuf32 = sbuf;
 	unsigned int x;
 	u16 val16;
+	u32 pix;
 
 	for (x = 0; x < pixels; x++) {
-		val16 = ((sbuf32[x] & 0x00F80000) >> 8) |
-			((sbuf32[x] & 0x0000FC00) >> 5) |
-			((sbuf32[x] & 0x000000F8) >> 3);
+		pix = le32_to_cpu(sbuf32[x]);
+		val16 = ((pix & 0x00F80000) >> 8) |
+			((pix & 0x0000FC00) >> 5) |
+			((pix & 0x000000F8) >> 3);
 		dbuf16[x] = val16;
 	}
 }
@@ -295,146 +341,143 @@ static void drm_fb_xrgb8888_to_rgb565_swab_line(void *dbuf, const void *sbuf,
 						unsigned int pixels)
 {
 	u16 *dbuf16 = dbuf;
-	const u32 *sbuf32 = sbuf;
+	const __le32 *sbuf32 = sbuf;
 	unsigned int x;
 	u16 val16;
+	u32 pix;
 
 	for (x = 0; x < pixels; x++) {
-		val16 = ((sbuf32[x] & 0x00F80000) >> 8) |
-			((sbuf32[x] & 0x0000FC00) >> 5) |
-			((sbuf32[x] & 0x000000F8) >> 3);
+		pix = le32_to_cpu(sbuf32[x]);
+		val16 = ((pix & 0x00F80000) >> 8) |
+			((pix & 0x0000FC00) >> 5) |
+			((pix & 0x000000F8) >> 3);
 		dbuf16[x] = swab16(val16);
 	}
 }
 
 /**
  * drm_fb_xrgb8888_to_rgb565 - Convert XRGB8888 to RGB565 clip buffer
- * @dst: RGB565 destination buffer
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @vaddr: XRGB8888 source buffer
+ * @dst: Array of RGB565 destination buffers
+ * @dst_pitch: Array of numbers of bytes between the start of two consecutive scanlines
+ *             within @dst; can be NULL if scanlines are stored next to each other.
+ * @src: Array of XRGB8888 source buffer
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
  * @swab: Swap bytes
  *
- * Drivers can use this function for RGB565 devices that don't natively
- * support XRGB8888.
- */
-void drm_fb_xrgb8888_to_rgb565(void *dst, unsigned int dst_pitch, const void *vaddr,
-			       const struct drm_framebuffer *fb, const struct drm_rect *clip,
-			       bool swab)
-{
-	if (swab)
-		drm_fb_xfrm(dst, dst_pitch, 2, vaddr, fb, clip, false,
-			    drm_fb_xrgb8888_to_rgb565_swab_line);
-	else
-		drm_fb_xfrm(dst, dst_pitch, 2, vaddr, fb, clip, false,
-			    drm_fb_xrgb8888_to_rgb565_line);
-}
-EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565);
-
-/**
- * drm_fb_xrgb8888_to_rgb565_toio - Convert XRGB8888 to RGB565 clip buffer
- * @dst: RGB565 destination buffer (iomem)
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @vaddr: XRGB8888 source buffer
- * @fb: DRM framebuffer
- * @clip: Clip rectangle area to copy
- * @swab: Swap bytes
+ * This function copies parts of a framebuffer to display memory and converts the
+ * color format during the process. Destination and framebuffer formats must match. The
+ * parameters @dst, @dst_pitch and @src refer to arrays. Each array must have at
+ * least as many entries as there are planes in @fb's format. Each entry stores the
+ * value for the format's respective color plane at the same index.
  *
- * Drivers can use this function for RGB565 devices that don't natively
- * support XRGB8888.
+ * This function does not apply clipping on @dst (i.e. the destination is at the
+ * top-left corner).
+ *
+ * Drivers can use this function for RGB565 devices that don't support XRGB8888 natively.
  */
-void drm_fb_xrgb8888_to_rgb565_toio(void __iomem *dst, unsigned int dst_pitch,
-				    const void *vaddr, const struct drm_framebuffer *fb,
-				    const struct drm_rect *clip, bool swab)
+void drm_fb_xrgb8888_to_rgb565(struct iosys_map *dst, const unsigned int *dst_pitch,
+			       const struct iosys_map *src, const struct drm_framebuffer *fb,
+			       const struct drm_rect *clip, bool swab)
 {
+	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
+		2,
+	};
+
+	void (*xfrm_line)(void *dbuf, const void *sbuf, unsigned int npixels);
+
 	if (swab)
-		drm_fb_xfrm_toio(dst, dst_pitch, 2, vaddr, fb, clip, false,
-				 drm_fb_xrgb8888_to_rgb565_swab_line);
+		xfrm_line = drm_fb_xrgb8888_to_rgb565_swab_line;
 	else
-		drm_fb_xfrm_toio(dst, dst_pitch, 2, vaddr, fb, clip, false,
-				 drm_fb_xrgb8888_to_rgb565_line);
+		xfrm_line = drm_fb_xrgb8888_to_rgb565_line;
+
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false, xfrm_line);
 }
-EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565_toio);
+EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb565);
 
 static void drm_fb_xrgb8888_to_rgb888_line(void *dbuf, const void *sbuf, unsigned int pixels)
 {
 	u8 *dbuf8 = dbuf;
-	const u32 *sbuf32 = sbuf;
+	const __le32 *sbuf32 = sbuf;
 	unsigned int x;
+	u32 pix;
 
 	for (x = 0; x < pixels; x++) {
-		*dbuf8++ = (sbuf32[x] & 0x000000FF) >>  0;
-		*dbuf8++ = (sbuf32[x] & 0x0000FF00) >>  8;
-		*dbuf8++ = (sbuf32[x] & 0x00FF0000) >> 16;
+		pix = le32_to_cpu(sbuf32[x]);
+		*dbuf8++ = (pix & 0x000000FF) >>  0;
+		*dbuf8++ = (pix & 0x0000FF00) >>  8;
+		*dbuf8++ = (pix & 0x00FF0000) >> 16;
 	}
 }
 
 /**
  * drm_fb_xrgb8888_to_rgb888 - Convert XRGB8888 to RGB888 clip buffer
- * @dst: RGB888 destination buffer
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @src: XRGB8888 source buffer
+ * @dst: Array of RGB888 destination buffers
+ * @dst_pitch: Array of numbers of bytes between the start of two consecutive scanlines
+ *             within @dst; can be NULL if scanlines are stored next to each other.
+ * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
  *
- * Drivers can use this function for RGB888 devices that don't natively
- * support XRGB8888.
- */
-void drm_fb_xrgb8888_to_rgb888(void *dst, unsigned int dst_pitch, const void *src,
-			       const struct drm_framebuffer *fb, const struct drm_rect *clip)
-{
-	drm_fb_xfrm(dst, dst_pitch, 3, src, fb, clip, false, drm_fb_xrgb8888_to_rgb888_line);
-}
-EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb888);
-
-/**
- * drm_fb_xrgb8888_to_rgb888_toio - Convert XRGB8888 to RGB888 clip buffer
- * @dst: RGB565 destination buffer (iomem)
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @vaddr: XRGB8888 source buffer
- * @fb: DRM framebuffer
- * @clip: Clip rectangle area to copy
+ * This function copies parts of a framebuffer to display memory and converts the
+ * color format during the process. Destination and framebuffer formats must match. The
+ * parameters @dst, @dst_pitch and @src refer to arrays. Each array must have at
+ * least as many entries as there are planes in @fb's format. Each entry stores the
+ * value for the format's respective color plane at the same index.
+ *
+ * This function does not apply clipping on @dst (i.e. the destination is at the
+ * top-left corner).
  *
  * Drivers can use this function for RGB888 devices that don't natively
  * support XRGB8888.
  */
-void drm_fb_xrgb8888_to_rgb888_toio(void __iomem *dst, unsigned int dst_pitch,
-				    const void *vaddr, const struct drm_framebuffer *fb,
-				    const struct drm_rect *clip)
+void drm_fb_xrgb8888_to_rgb888(struct iosys_map *dst, const unsigned int *dst_pitch,
+			       const struct iosys_map *src, const struct drm_framebuffer *fb,
+			       const struct drm_rect *clip)
 {
-	drm_fb_xfrm_toio(dst, dst_pitch, 3, vaddr, fb, clip, false,
-			 drm_fb_xrgb8888_to_rgb888_line);
+	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
+		3,
+	};
+
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+		    drm_fb_xrgb8888_to_rgb888_line);
 }
-EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb888_toio);
+EXPORT_SYMBOL(drm_fb_xrgb8888_to_rgb888);
 
 static void drm_fb_rgb565_to_xrgb8888_line(void *dbuf, const void *sbuf, unsigned int pixels)
 {
-	u32 *dbuf32 = dbuf;
-	const u16 *sbuf16 = sbuf;
+	__le32 *dbuf32 = dbuf;
+	const __le16 *sbuf16 = sbuf;
 	unsigned int x;
 
-	for (x = 0; x < pixels; x++, ++sbuf16, ++dbuf32) {
-		u32 val32 = ((*sbuf16 & 0xf800) << 8) |
-			    ((*sbuf16 & 0x07e0) << 5) |
-			    ((*sbuf16 & 0x001f) << 3);
-		*dbuf32 = 0xff000000 | val32 |
-			  ((val32 >> 3) & 0x00070007) |
-			  ((val32 >> 2) & 0x00000300);
+	for (x = 0; x < pixels; x++) {
+		u16 val16 = le16_to_cpu(sbuf16[x]);
+		u32 val32 = ((val16 & 0xf800) << 8) |
+			    ((val16 & 0x07e0) << 5) |
+			    ((val16 & 0x001f) << 3);
+		val32 = 0xff000000 | val32 |
+			((val32 >> 3) & 0x00070007) |
+			((val32 >> 2) & 0x00000300);
+		dbuf32[x] = cpu_to_le32(val32);
 	}
 }
 
-static void drm_fb_rgb565_to_xrgb8888_toio(void __iomem *dst, unsigned int dst_pitch,
-					   const void *vaddr, const struct drm_framebuffer *fb,
-					   const struct drm_rect *clip)
+static void drm_fb_rgb565_to_xrgb8888(struct iosys_map *dst, const unsigned int *dst_pitch,
+				      const struct iosys_map *src,
+				      const struct drm_framebuffer *fb,
+				      const struct drm_rect *clip)
 {
-	drm_fb_xfrm_toio(dst, dst_pitch, 4, vaddr, fb, clip, false,
-			 drm_fb_rgb565_to_xrgb8888_line);
+	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
+		4,
+	};
+
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+		    drm_fb_rgb565_to_xrgb8888_line);
 }
 
 static void drm_fb_rgb888_to_xrgb8888_line(void *dbuf, const void *sbuf, unsigned int pixels)
 {
-	u32 *dbuf32 = dbuf;
+	__le32 *dbuf32 = dbuf;
 	const u8 *sbuf8 = sbuf;
 	unsigned int x;
 
@@ -442,117 +485,159 @@ static void drm_fb_rgb888_to_xrgb8888_line(void *dbuf, const void *sbuf, unsigne
 		u8 r = *sbuf8++;
 		u8 g = *sbuf8++;
 		u8 b = *sbuf8++;
-		*dbuf32++ = 0xff000000 | (r << 16) | (g << 8) | b;
+		u32 pix = 0xff000000 | (r << 16) | (g << 8) | b;
+		dbuf32[x] = cpu_to_le32(pix);
 	}
 }
 
-static void drm_fb_rgb888_to_xrgb8888_toio(void __iomem *dst, unsigned int dst_pitch,
-					   const void *vaddr, const struct drm_framebuffer *fb,
-					   const struct drm_rect *clip)
+static void drm_fb_rgb888_to_xrgb8888(struct iosys_map *dst, const unsigned int *dst_pitch,
+				      const struct iosys_map *src,
+				      const struct drm_framebuffer *fb,
+				      const struct drm_rect *clip)
 {
-	drm_fb_xfrm_toio(dst, dst_pitch, 4, vaddr, fb, clip, false,
-			 drm_fb_rgb888_to_xrgb8888_line);
+	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
+		4,
+	};
+
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+		    drm_fb_rgb888_to_xrgb8888_line);
 }
 
 static void drm_fb_xrgb8888_to_xrgb2101010_line(void *dbuf, const void *sbuf, unsigned int pixels)
 {
-	u32 *dbuf32 = dbuf;
-	const u32 *sbuf32 = sbuf;
+	__le32 *dbuf32 = dbuf;
+	const __le32 *sbuf32 = sbuf;
 	unsigned int x;
 	u32 val32;
+	u32 pix;
 
 	for (x = 0; x < pixels; x++) {
-		val32 = ((sbuf32[x] & 0x000000FF) << 2) |
-			((sbuf32[x] & 0x0000FF00) << 4) |
-			((sbuf32[x] & 0x00FF0000) << 6);
-		*dbuf32++ = val32 | ((val32 >> 8) & 0x00300C03);
+		pix = le32_to_cpu(sbuf32[x]);
+		val32 = ((pix & 0x000000FF) << 2) |
+			((pix & 0x0000FF00) << 4) |
+			((pix & 0x00FF0000) << 6);
+		pix = val32 | ((val32 >> 8) & 0x00300C03);
+		*dbuf32++ = cpu_to_le32(pix);
 	}
 }
 
 /**
- * drm_fb_xrgb8888_to_xrgb2101010_toio - Convert XRGB8888 to XRGB2101010 clip
- * buffer
- * @dst: XRGB2101010 destination buffer (iomem)
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @vaddr: XRGB8888 source buffer
+ * drm_fb_xrgb8888_to_xrgb2101010 - Convert XRGB8888 to XRGB2101010 clip buffer
+ * @dst: Array of XRGB2101010 destination buffers
+ * @dst_pitch: Array of numbers of bytes between the start of two consecutive scanlines
+ *             within @dst; can be NULL if scanlines are stored next to each other.
+ * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
  *
- * Drivers can use this function for XRGB2101010 devices that don't natively
- * support XRGB8888.
+ * This function copies parts of a framebuffer to display memory and converts the
+ * color format during the process. Destination and framebuffer formats must match. The
+ * parameters @dst, @dst_pitch and @src refer to arrays. Each array must have at
+ * least as many entries as there are planes in @fb's format. Each entry stores the
+ * value for the format's respective color plane at the same index.
+ *
+ * This function does not apply clipping on @dst (i.e. the destination is at the
+ * top-left corner).
+ *
+ * Drivers can use this function for XRGB2101010 devices that don't support XRGB8888
+ * natively.
  */
-void drm_fb_xrgb8888_to_xrgb2101010_toio(void __iomem *dst,
-					 unsigned int dst_pitch, const void *vaddr,
-					 const struct drm_framebuffer *fb,
-					 const struct drm_rect *clip)
+void drm_fb_xrgb8888_to_xrgb2101010(struct iosys_map *dst, const unsigned int *dst_pitch,
+				    const struct iosys_map *src, const struct drm_framebuffer *fb,
+				    const struct drm_rect *clip)
 {
-	drm_fb_xfrm_toio(dst, dst_pitch, 4, vaddr, fb, clip, false,
-			 drm_fb_xrgb8888_to_xrgb2101010_line);
+	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
+		4,
+	};
+
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+		    drm_fb_xrgb8888_to_xrgb2101010_line);
 }
-EXPORT_SYMBOL(drm_fb_xrgb8888_to_xrgb2101010_toio);
+EXPORT_SYMBOL(drm_fb_xrgb8888_to_xrgb2101010);
 
 static void drm_fb_xrgb8888_to_gray8_line(void *dbuf, const void *sbuf, unsigned int pixels)
 {
 	u8 *dbuf8 = dbuf;
-	const u32 *sbuf32 = sbuf;
+	const __le32 *sbuf32 = sbuf;
 	unsigned int x;
 
 	for (x = 0; x < pixels; x++) {
-		u8 r = (*sbuf32 & 0x00ff0000) >> 16;
-		u8 g = (*sbuf32 & 0x0000ff00) >> 8;
-		u8 b =  *sbuf32 & 0x000000ff;
+		u32 pix = le32_to_cpu(sbuf32[x]);
+		u8 r = (pix & 0x00ff0000) >> 16;
+		u8 g = (pix & 0x0000ff00) >> 8;
+		u8 b =  pix & 0x000000ff;
 
 		/* ITU BT.601: Y = 0.299 R + 0.587 G + 0.114 B */
 		*dbuf8++ = (3 * r + 6 * g + b) / 10;
-		sbuf32++;
 	}
 }
 
 /**
  * drm_fb_xrgb8888_to_gray8 - Convert XRGB8888 to grayscale
- * @dst: 8-bit grayscale destination buffer
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @vaddr: XRGB8888 source buffer
+ * @dst: Array of 8-bit grayscale destination buffers
+ * @dst_pitch: Array of numbers of bytes between the start of two consecutive scanlines
+ *             within @dst; can be NULL if scanlines are stored next to each other.
+ * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
  *
- * Drm doesn't have native monochrome or grayscale support.
- * Such drivers can announce the commonly supported XR24 format to userspace
- * and use this function to convert to the native format.
+ * This function copies parts of a framebuffer to display memory and converts the
+ * color format during the process. Destination and framebuffer formats must match. The
+ * parameters @dst, @dst_pitch and @src refer to arrays. Each array must have at
+ * least as many entries as there are planes in @fb's format. Each entry stores the
+ * value for the format's respective color plane at the same index.
  *
- * Monochrome drivers will use the most significant bit,
- * where 1 means foreground color and 0 background color.
+ * This function does not apply clipping on @dst (i.e. the destination is at the
+ * top-left corner).
  *
- * ITU BT.601 is used for the RGB -> luma (brightness) conversion.
+ * DRM doesn't have native monochrome or grayscale support. Drivers can use this
+ * function for grayscale devices that don't support XRGB8888 natively.Such
+ * drivers can announce the commonly supported XR24 format to userspace and use
+ * this function to convert to the native format. Monochrome drivers will use the
+ * most significant bit, where 1 means foreground color and 0 background color.
+ * ITU BT.601 is being used for the RGB -> luma (brightness) conversion.
  */
-void drm_fb_xrgb8888_to_gray8(void *dst, unsigned int dst_pitch, const void *vaddr,
-			      const struct drm_framebuffer *fb, const struct drm_rect *clip)
+void drm_fb_xrgb8888_to_gray8(struct iosys_map *dst, const unsigned int *dst_pitch,
+			      const struct iosys_map *src, const struct drm_framebuffer *fb,
+			      const struct drm_rect *clip)
 {
-	drm_fb_xfrm(dst, dst_pitch, 1, vaddr, fb, clip, false, drm_fb_xrgb8888_to_gray8_line);
+	static const u8 dst_pixsize[DRM_FORMAT_MAX_PLANES] = {
+		1,
+	};
+
+	drm_fb_xfrm(dst, dst_pitch, dst_pixsize, src, fb, clip, false,
+		    drm_fb_xrgb8888_to_gray8_line);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_gray8);
 
 /**
- * drm_fb_blit_toio - Copy parts of a framebuffer to display memory
- * @dst:	The display memory to copy to
- * @dst_pitch:	Number of bytes between two consecutive scanlines within dst
+ * drm_fb_blit - Copy parts of a framebuffer to display memory
+ * @dst:	Array of display-memory addresses to copy to
+ * @dst_pitch: Array of numbers of bytes between the start of two consecutive scanlines
+ *             within @dst; can be NULL if scanlines are stored next to each other.
  * @dst_format:	FOURCC code of the display's color format
- * @vmap:	The framebuffer memory to copy from
+ * @src:	The framebuffer memory to copy from
  * @fb:		The framebuffer to copy from
  * @clip:	Clip rectangle area to copy
  *
  * This function copies parts of a framebuffer to display memory. If the
  * formats of the display and the framebuffer mismatch, the blit function
- * will attempt to convert between them.
+ * will attempt to convert between them during the process. The parameters @dst,
+ * @dst_pitch and @src refer to arrays. Each array must have at least as many
+ * entries as there are planes in @dst_format's format. Each entry stores the
+ * value for the format's respective color plane at the same index.
+ *
+ * This function does not apply clipping on @dst (i.e. the destination is at the
+ * top-left corner).
  *
  * Returns:
  * 0 on success, or
  * -EINVAL if the color-format conversion failed, or
  * a negative error code otherwise.
  */
-int drm_fb_blit_toio(void __iomem *dst, unsigned int dst_pitch, uint32_t dst_format,
-		     const void *vmap, const struct drm_framebuffer *fb,
-		     const struct drm_rect *clip)
+int drm_fb_blit(struct iosys_map *dst, const unsigned int *dst_pitch, uint32_t dst_format,
+		const struct iosys_map *src, const struct drm_framebuffer *fb,
+		const struct drm_rect *clip)
 {
 	uint32_t fb_format = fb->format->format;
 
@@ -567,30 +652,30 @@ int drm_fb_blit_toio(void __iomem *dst, unsigned int dst_pitch, uint32_t dst_for
 		dst_format = DRM_FORMAT_XRGB2101010;
 
 	if (dst_format == fb_format) {
-		drm_fb_memcpy_toio(dst, dst_pitch, vmap, fb, clip);
+		drm_fb_memcpy(dst, dst_pitch, src, fb, clip);
 		return 0;
 
 	} else if (dst_format == DRM_FORMAT_RGB565) {
 		if (fb_format == DRM_FORMAT_XRGB8888) {
-			drm_fb_xrgb8888_to_rgb565_toio(dst, dst_pitch, vmap, fb, clip, false);
+			drm_fb_xrgb8888_to_rgb565(dst, dst_pitch, src, fb, clip, false);
 			return 0;
 		}
 	} else if (dst_format == DRM_FORMAT_RGB888) {
 		if (fb_format == DRM_FORMAT_XRGB8888) {
-			drm_fb_xrgb8888_to_rgb888_toio(dst, dst_pitch, vmap, fb, clip);
+			drm_fb_xrgb8888_to_rgb888(dst, dst_pitch, src, fb, clip);
 			return 0;
 		}
 	} else if (dst_format == DRM_FORMAT_XRGB8888) {
 		if (fb_format == DRM_FORMAT_RGB888) {
-			drm_fb_rgb888_to_xrgb8888_toio(dst, dst_pitch, vmap, fb, clip);
+			drm_fb_rgb888_to_xrgb8888(dst, dst_pitch, src, fb, clip);
 			return 0;
 		} else if (fb_format == DRM_FORMAT_RGB565) {
-			drm_fb_rgb565_to_xrgb8888_toio(dst, dst_pitch, vmap, fb, clip);
+			drm_fb_rgb565_to_xrgb8888(dst, dst_pitch, src, fb, clip);
 			return 0;
 		}
 	} else if (dst_format == DRM_FORMAT_XRGB2101010) {
 		if (fb_format == DRM_FORMAT_XRGB8888) {
-			drm_fb_xrgb8888_to_xrgb2101010_toio(dst, dst_pitch, vmap, fb, clip);
+			drm_fb_xrgb8888_to_xrgb2101010(dst, dst_pitch, src, fb, clip);
 			return 0;
 		}
 	}
@@ -600,8 +685,7 @@ int drm_fb_blit_toio(void __iomem *dst, unsigned int dst_pitch, uint32_t dst_for
 
 	return -EINVAL;
 }
-EXPORT_SYMBOL(drm_fb_blit_toio);
-
+EXPORT_SYMBOL(drm_fb_blit);
 
 static void drm_fb_gray8_to_mono_line(void *dbuf, const void *sbuf, unsigned int pixels)
 {
@@ -622,49 +706,67 @@ static void drm_fb_gray8_to_mono_line(void *dbuf, const void *sbuf, unsigned int
 
 /**
  * drm_fb_xrgb8888_to_mono - Convert XRGB8888 to monochrome
- * @dst: monochrome destination buffer (0=black, 1=white)
- * @dst_pitch: Number of bytes between two consecutive scanlines within dst
- * @vaddr: XRGB8888 source buffer
+ * @dst: Array of monochrome destination buffers (0=black, 1=white)
+ * @dst_pitch: Array of numbers of bytes between the start of two consecutive scanlines
+ *             within @dst; can be NULL if scanlines are stored next to each other.
+ * @src: Array of XRGB8888 source buffers
  * @fb: DRM framebuffer
  * @clip: Clip rectangle area to copy
  *
- * DRM doesn't have native monochrome support.
- * Such drivers can announce the commonly supported XR24 format to userspace
- * and use this function to convert to the native format.
+ * This function copies parts of a framebuffer to display memory and converts the
+ * color format during the process. Destination and framebuffer formats must match. The
+ * parameters @dst, @dst_pitch and @src refer to arrays. Each array must have at
+ * least as many entries as there are planes in @fb's format. Each entry stores the
+ * value for the format's respective color plane at the same index.
+ *
+ * This function does not apply clipping on @dst (i.e. the destination is at the
+ * top-left corner). The first pixel (upper left corner of the clip rectangle) will
+ * be converted and copied to the first bit (LSB) in the first byte of the monochrome
+ * destination buffer. If the caller requires that the first pixel in a byte must
+ * be located at an x-coordinate that is a multiple of 8, then the caller must take
+ * care itself of supplying a suitable clip rectangle.
+ *
+ * DRM doesn't have native monochrome support. Drivers can use this function for
+ * monochrome devices that don't support XRGB8888 natively. Such drivers can
+ * announce the commonly supported XR24 format to userspace and use this function
+ * to convert to the native format.
  *
  * This function uses drm_fb_xrgb8888_to_gray8() to convert to grayscale and
  * then the result is converted from grayscale to monochrome.
- *
- * The first pixel (upper left corner of the clip rectangle) will be converted
- * and copied to the first bit (LSB) in the first byte of the monochrome
- * destination buffer.
- * If the caller requires that the first pixel in a byte must be located at an
- * x-coordinate that is a multiple of 8, then the caller must take care itself
- * of supplying a suitable clip rectangle.
  */
-void drm_fb_xrgb8888_to_mono(void *dst, unsigned int dst_pitch, const void *vaddr,
-			     const struct drm_framebuffer *fb, const struct drm_rect *clip)
+void drm_fb_xrgb8888_to_mono(struct iosys_map *dst, const unsigned int *dst_pitch,
+			     const struct iosys_map *src, const struct drm_framebuffer *fb,
+			     const struct drm_rect *clip)
 {
+	static const unsigned int default_dst_pitch[DRM_FORMAT_MAX_PLANES] = {
+		0, 0, 0, 0
+	};
 	unsigned int linepixels = drm_rect_width(clip);
 	unsigned int lines = drm_rect_height(clip);
 	unsigned int cpp = fb->format->cpp[0];
 	unsigned int len_src32 = linepixels * cpp;
 	struct drm_device *dev = fb->dev;
+	void *vaddr = src[0].vaddr;
+	unsigned int dst_pitch_0;
 	unsigned int y;
-	u8 *mono = dst, *gray8;
+	u8 *mono = dst[0].vaddr, *gray8;
 	u32 *src32;
 
 	if (drm_WARN_ON(dev, fb->format->format != DRM_FORMAT_XRGB8888))
 		return;
 
+	if (!dst_pitch)
+		dst_pitch = default_dst_pitch;
+	dst_pitch_0 = dst_pitch[0];
+
 	/*
 	 * The mono destination buffer contains 1 bit per pixel
 	 */
-	if (!dst_pitch)
-		dst_pitch = DIV_ROUND_UP(linepixels, 8);
+	if (!dst_pitch_0)
+		dst_pitch_0 = DIV_ROUND_UP(linepixels, 8);
 
 	/*
-	 * The cma memory is write-combined so reads are uncached.
+	 * The dma memory is write-combined so reads are uncached.
 	 * Speed up by fetching one line at a time.
 	 *
 	 * Also, format conversion from XR24 to monochrome are done
@@ -686,9 +788,117 @@ void drm_fb_xrgb8888_to_mono(void *dst, unsigned int dst_pitch, const void *vadd
 		drm_fb_xrgb8888_to_gray8_line(gray8, src32, linepixels);
 		drm_fb_gray8_to_mono_line(mono, gray8, linepixels);
 		vaddr += fb->pitches[0];
-		mono += dst_pitch;
+		mono += dst_pitch_0;
 	}
 
 	kfree(src32);
 }
 EXPORT_SYMBOL(drm_fb_xrgb8888_to_mono);
+
+static bool is_listed_fourcc(const uint32_t *fourccs, size_t nfourccs, uint32_t fourcc)
+{
+	const uint32_t *fourccs_end = fourccs + nfourccs;
+
+	while (fourccs < fourccs_end) {
+		if (*fourccs == fourcc)
+			return true;
+		++fourccs;
+	}
+	return false;
+}
+
+/**
+ * drm_fb_build_fourcc_list - Filters a list of supported color formats against
+ *                            the device's native formats
+ * @dev: DRM device
+ * @native_fourccs: 4CC codes of natively supported color formats
+ * @native_nfourccs: The number of entries in @native_fourccs
+ * @driver_fourccs: 4CC codes of all driver-supported color formats
+ * @driver_nfourccs: The number of entries in @driver_fourccs
+ * @fourccs_out: Returns 4CC codes of supported color formats
+ * @nfourccs_out: The number of available entries in @fourccs_out
+ *
+ * This function create a list of supported color format from natively
+ * supported formats and the emulated formats.
+ * At a minimum, most userspace programs expect at least support for
+ * XRGB8888 on the primary plane. Devices that have to emulate the
+ * format, and possibly others, can use drm_fb_build_fourcc_list() to
+ * create a list of supported color formats. The returned list can
+ * be handed over to drm_universal_plane_init() et al. Native formats
+ * will go before emulated formats. Other heuristics might be applied
+ * to optimize the order. Formats near the beginning of the list are
+ * usually preferred over formats near the end of the list.
+ *
+ * Returns:
+ * The number of color-formats 4CC codes returned in @fourccs_out.
+ */
+size_t drm_fb_build_fourcc_list(struct drm_device *dev,
+				const u32 *native_fourccs, size_t native_nfourccs,
+				const u32 *driver_fourccs, size_t driver_nfourccs,
+				u32 *fourccs_out, size_t nfourccs_out)
+{
+	u32 *fourccs = fourccs_out;
+	const u32 *fourccs_end = fourccs_out + nfourccs_out;
+	bool found_native = false;
+	size_t i;
+
+	/*
+	 * The device's native formats go first.
+	 */
+
+	for (i = 0; i < native_nfourccs; ++i) {
+		u32 fourcc = native_fourccs[i];
+
+		if (is_listed_fourcc(fourccs_out, fourccs - fourccs_out, fourcc)) {
+			continue; /* skip duplicate entries */
+		} else if (fourccs == fourccs_end) {
+			drm_warn(dev, "Ignoring native format %p4cc\n", &fourcc);
+			continue; /* end of available output buffer */
+		}
+
+		drm_dbg_kms(dev, "adding native format %p4cc\n", &fourcc);
+
+		if (!found_native)
+			found_native = is_listed_fourcc(driver_fourccs, driver_nfourccs, fourcc);
+		*fourccs = fourcc;
+		++fourccs;
+	}
+
+	/*
+	 * The plane's atomic_update helper converts the framebuffer's color format
+	 * to a native format when copying to device memory.
+	 *
+	 * If there is not a single format supported by both, device and
+	 * driver, the native formats are likely not supported by the conversion
+	 * helpers. Therefore *only* support the native formats and add a
+	 * conversion helper ASAP.
+	 */
+	if (!found_native) {
+		drm_warn(dev, "Format conversion helpers required to add extra formats.\n");
+		goto out;
+	}
+
+	/*
+	 * The extra formats, emulated by the driver, go second.
+	 */
+
+	for (i = 0; (i < driver_nfourccs) && (fourccs < fourccs_end); ++i) {
+		u32 fourcc = driver_fourccs[i];
+
+		if (is_listed_fourcc(fourccs_out, fourccs - fourccs_out, fourcc)) {
+			continue; /* skip duplicate and native entries */
+		} else if (fourccs == fourccs_end) {
+			drm_warn(dev, "Ignoring emulated format %p4cc\n", &fourcc);
+			continue; /* end of available output buffer */
+		}
+
+		drm_dbg_kms(dev, "adding emulated format %p4cc\n", &fourcc);
+
+		*fourccs = fourcc;
+		++fourccs;
+	}
+
+out:
+	return fourccs - fourccs_out;
+}
+EXPORT_SYMBOL(drm_fb_build_fourcc_list);
diff --git a/drivers/gpu/drm/drm_fourcc.c b/drivers/gpu/drm/drm_fourcc.c
index 07741b678798..e09331bb3bc7 100644
--- a/drivers/gpu/drm/drm_fourcc.c
+++ b/drivers/gpu/drm/drm_fourcc.c
@@ -43,6 +43,21 @@ uint32_t drm_mode_legacy_fb_format(uint32_t bpp, uint32_t depth)
 	uint32_t fmt = DRM_FORMAT_INVALID;
 
 	switch (bpp) {
+	case 1:
+		if (depth == 1)
+			fmt = DRM_FORMAT_C1;
+		break;
+
+	case 2:
+		if (depth == 2)
+			fmt = DRM_FORMAT_C2;
+		break;
+
+	case 4:
+		if (depth == 4)
+			fmt = DRM_FORMAT_C4;
+		break;
+
 	case 8:
 		if (depth == 8)
 			fmt = DRM_FORMAT_C8;
@@ -132,7 +147,26 @@ EXPORT_SYMBOL(drm_driver_legacy_fb_format);
 const struct drm_format_info *__drm_format_info(u32 format)
 {
 	static const struct drm_format_info formats[] = {
-		{ .format = DRM_FORMAT_C8,		.depth = 8,  .num_planes = 1, .cpp = { 1, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_C1,		.depth = 1,  .num_planes = 1,
+		  .char_per_block = { 1, }, .block_w = { 8, }, .block_h = { 1, }, .hsub = 1, .vsub = 1, .is_color_indexed = true },
+		{ .format = DRM_FORMAT_C2,		.depth = 2,  .num_planes = 1,
+		  .char_per_block = { 1, }, .block_w = { 4, }, .block_h = { 1, }, .hsub = 1, .vsub = 1, .is_color_indexed = true },
+		{ .format = DRM_FORMAT_C4,		.depth = 4,  .num_planes = 1,
+		  .char_per_block = { 1, }, .block_w = { 2, }, .block_h = { 1, }, .hsub = 1, .vsub = 1, .is_color_indexed = true },
+		{ .format = DRM_FORMAT_C8,		.depth = 8,  .num_planes = 1, .cpp = { 1, 0, 0 }, .hsub = 1, .vsub = 1, .is_color_indexed = true },
+		{ .format = DRM_FORMAT_D1,		.depth = 1,  .num_planes = 1,
+		  .char_per_block = { 1, }, .block_w = { 8, }, .block_h = { 1, }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_D2,		.depth = 2,  .num_planes = 1,
+		  .char_per_block = { 1, }, .block_w = { 4, }, .block_h = { 1, }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_D4,		.depth = 4,  .num_planes = 1,
+		  .char_per_block = { 1, }, .block_w = { 2, }, .block_h = { 1, }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_D8,		.depth = 8,  .num_planes = 1, .cpp = { 1, 0, 0 }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_R1,		.depth = 1,  .num_planes = 1,
+		  .char_per_block = { 1, }, .block_w = { 8, }, .block_h = { 1, }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_R2,		.depth = 2,  .num_planes = 1,
+		  .char_per_block = { 1, }, .block_w = { 4, }, .block_h = { 1, }, .hsub = 1, .vsub = 1 },
+		{ .format = DRM_FORMAT_R4,		.depth = 4,  .num_planes = 1,
+		  .char_per_block = { 1, }, .block_w = { 2, }, .block_h = { 1, }, .hsub = 1, .vsub = 1 },
 		{ .format = DRM_FORMAT_R8,		.depth = 8,  .num_planes = 1, .cpp = { 1, 0, 0 }, .hsub = 1, .vsub = 1 },
 		{ .format = DRM_FORMAT_R10,		.depth = 10, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
 		{ .format = DRM_FORMAT_R12,		.depth = 12, .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 1, .vsub = 1 },
@@ -371,6 +405,25 @@ unsigned int drm_format_info_block_height(const struct drm_format_info *info,
 EXPORT_SYMBOL(drm_format_info_block_height);
 
 /**
+ * drm_format_info_bpp - number of bits per pixel
+ * @info: pixel format info
+ * @plane: plane index
+ *
+ * Returns:
+ * The actual number of bits per pixel, depending on the plane index.
+ */
+unsigned int drm_format_info_bpp(const struct drm_format_info *info, int plane)
+{
+	if (!info || plane < 0 || plane >= info->num_planes)
+		return 0;
+
+	return info->char_per_block[plane] * 8 /
+	       (drm_format_info_block_width(info, plane) *
+		drm_format_info_block_height(info, plane));
+}
+EXPORT_SYMBOL(drm_format_info_bpp);
+
+/**
  * drm_format_info_min_pitch - computes the minimum required pitch in bytes
  * @info: pixel format info
  * @plane: plane index
diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c
index 4562a8b86579..2dd97473ca10 100644
--- a/drivers/gpu/drm/drm_framebuffer.c
+++ b/drivers/gpu/drm/drm_framebuffer.c
@@ -87,13 +87,13 @@ int drm_framebuffer_check_src_coords(uint32_t src_x, uint32_t src_y,
 	    src_x > fb_width - src_w ||
 	    src_h > fb_height ||
 	    src_y > fb_height - src_h) {
-		DRM_DEBUG_KMS("Invalid source coordinates "
-			      "%u.%06ux%u.%06u+%u.%06u+%u.%06u (fb %ux%u)\n",
-			      src_w >> 16, ((src_w & 0xffff) * 15625) >> 10,
-			      src_h >> 16, ((src_h & 0xffff) * 15625) >> 10,
-			      src_x >> 16, ((src_x & 0xffff) * 15625) >> 10,
-			      src_y >> 16, ((src_y & 0xffff) * 15625) >> 10,
-			      fb->width, fb->height);
+		drm_dbg_kms(fb->dev, "Invalid source coordinates "
+			    "%u.%06ux%u.%06u+%u.%06u+%u.%06u (fb %ux%u)\n",
+			    src_w >> 16, ((src_w & 0xffff) * 15625) >> 10,
+			    src_h >> 16, ((src_h & 0xffff) * 15625) >> 10,
+			    src_x >> 16, ((src_x & 0xffff) * 15625) >> 10,
+			    src_y >> 16, ((src_y & 0xffff) * 15625) >> 10,
+			    fb->width, fb->height);
 		return -ENOSPC;
 	}
 
@@ -125,7 +125,7 @@ int drm_mode_addfb(struct drm_device *dev, struct drm_mode_fb_cmd *or,
 
 	r.pixel_format = drm_driver_legacy_fb_format(dev, or->bpp, or->depth);
 	if (r.pixel_format == DRM_FORMAT_INVALID) {
-		DRM_DEBUG("bad {bpp:%d, depth:%d}\n", or->bpp, or->depth);
+		drm_dbg_kms(dev, "bad {bpp:%d, depth:%d}\n", or->bpp, or->depth);
 		return -EINVAL;
 	}
 
@@ -177,18 +177,18 @@ static int framebuffer_check(struct drm_device *dev,
 
 	/* check if the format is supported at all */
 	if (!__drm_format_info(r->pixel_format)) {
-		DRM_DEBUG_KMS("bad framebuffer format %p4cc\n",
-			      &r->pixel_format);
+		drm_dbg_kms(dev, "bad framebuffer format %p4cc\n",
+			    &r->pixel_format);
 		return -EINVAL;
 	}
 
 	if (r->width == 0) {
-		DRM_DEBUG_KMS("bad framebuffer width %u\n", r->width);
+		drm_dbg_kms(dev, "bad framebuffer width %u\n", r->width);
 		return -EINVAL;
 	}
 
 	if (r->height == 0) {
-		DRM_DEBUG_KMS("bad framebuffer height %u\n", r->height);
+		drm_dbg_kms(dev, "bad framebuffer height %u\n", r->height);
 		return -EINVAL;
 	}
 
@@ -202,12 +202,12 @@ static int framebuffer_check(struct drm_device *dev,
 		u64 min_pitch = drm_format_info_min_pitch(info, i, width);
 
 		if (!block_size && (r->modifier[i] == DRM_FORMAT_MOD_LINEAR)) {
-			DRM_DEBUG_KMS("Format requires non-linear modifier for plane %d\n", i);
+			drm_dbg_kms(dev, "Format requires non-linear modifier for plane %d\n", i);
 			return -EINVAL;
 		}
 
 		if (!r->handles[i]) {
-			DRM_DEBUG_KMS("no buffer object handle for plane %d\n", i);
+			drm_dbg_kms(dev, "no buffer object handle for plane %d\n", i);
 			return -EINVAL;
 		}
 
@@ -218,20 +218,20 @@ static int framebuffer_check(struct drm_device *dev,
 			return -ERANGE;
 
 		if (block_size && r->pitches[i] < min_pitch) {
-			DRM_DEBUG_KMS("bad pitch %u for plane %d\n", r->pitches[i], i);
+			drm_dbg_kms(dev, "bad pitch %u for plane %d\n", r->pitches[i], i);
 			return -EINVAL;
 		}
 
 		if (r->modifier[i] && !(r->flags & DRM_MODE_FB_MODIFIERS)) {
-			DRM_DEBUG_KMS("bad fb modifier %llu for plane %d\n",
-				      r->modifier[i], i);
+			drm_dbg_kms(dev, "bad fb modifier %llu for plane %d\n",
+				    r->modifier[i], i);
 			return -EINVAL;
 		}
 
 		if (r->flags & DRM_MODE_FB_MODIFIERS &&
 		    r->modifier[i] != r->modifier[0]) {
-			DRM_DEBUG_KMS("bad fb modifier %llu for plane %d\n",
-				      r->modifier[i], i);
+			drm_dbg_kms(dev, "bad fb modifier %llu for plane %d\n",
+				    r->modifier[i], i);
 			return -EINVAL;
 		}
 
@@ -244,7 +244,7 @@ static int framebuffer_check(struct drm_device *dev,
 			if (r->pixel_format != DRM_FORMAT_NV12 ||
 					width % 128 || height % 32 ||
 					r->pitches[i] % 128) {
-				DRM_DEBUG_KMS("bad modifier data for plane %d\n", i);
+				drm_dbg_kms(dev, "bad modifier data for plane %d\n", i);
 				return -EINVAL;
 			}
 			break;
@@ -256,7 +256,7 @@ static int framebuffer_check(struct drm_device *dev,
 
 	for (i = info->num_planes; i < 4; i++) {
 		if (r->modifier[i]) {
-			DRM_DEBUG_KMS("non-zero modifier for unused plane %d\n", i);
+			drm_dbg_kms(dev, "non-zero modifier for unused plane %d\n", i);
 			return -EINVAL;
 		}
 
@@ -265,17 +265,17 @@ static int framebuffer_check(struct drm_device *dev,
 			continue;
 
 		if (r->handles[i]) {
-			DRM_DEBUG_KMS("buffer object handle for unused plane %d\n", i);
+			drm_dbg_kms(dev, "buffer object handle for unused plane %d\n", i);
 			return -EINVAL;
 		}
 
 		if (r->pitches[i]) {
-			DRM_DEBUG_KMS("non-zero pitch for unused plane %d\n", i);
+			drm_dbg_kms(dev, "non-zero pitch for unused plane %d\n", i);
 			return -EINVAL;
 		}
 
 		if (r->offsets[i]) {
-			DRM_DEBUG_KMS("non-zero offset for unused plane %d\n", i);
+			drm_dbg_kms(dev, "non-zero offset for unused plane %d\n", i);
 			return -EINVAL;
 		}
 	}
@@ -293,24 +293,24 @@ drm_internal_framebuffer_create(struct drm_device *dev,
 	int ret;
 
 	if (r->flags & ~(DRM_MODE_FB_INTERLACED | DRM_MODE_FB_MODIFIERS)) {
-		DRM_DEBUG_KMS("bad framebuffer flags 0x%08x\n", r->flags);
+		drm_dbg_kms(dev, "bad framebuffer flags 0x%08x\n", r->flags);
 		return ERR_PTR(-EINVAL);
 	}
 
 	if ((config->min_width > r->width) || (r->width > config->max_width)) {
-		DRM_DEBUG_KMS("bad framebuffer width %d, should be >= %d && <= %d\n",
-			  r->width, config->min_width, config->max_width);
+		drm_dbg_kms(dev, "bad framebuffer width %d, should be >= %d && <= %d\n",
+			    r->width, config->min_width, config->max_width);
 		return ERR_PTR(-EINVAL);
 	}
 	if ((config->min_height > r->height) || (r->height > config->max_height)) {
-		DRM_DEBUG_KMS("bad framebuffer height %d, should be >= %d && <= %d\n",
-			  r->height, config->min_height, config->max_height);
+		drm_dbg_kms(dev, "bad framebuffer height %d, should be >= %d && <= %d\n",
+			    r->height, config->min_height, config->max_height);
 		return ERR_PTR(-EINVAL);
 	}
 
 	if (r->flags & DRM_MODE_FB_MODIFIERS &&
 	    dev->mode_config.fb_modifiers_not_supported) {
-		DRM_DEBUG_KMS("driver does not support fb modifiers\n");
+		drm_dbg_kms(dev, "driver does not support fb modifiers\n");
 		return ERR_PTR(-EINVAL);
 	}
 
@@ -320,7 +320,7 @@ drm_internal_framebuffer_create(struct drm_device *dev,
 
 	fb = dev->mode_config.funcs->fb_create(dev, file_priv, r);
 	if (IS_ERR(fb)) {
-		DRM_DEBUG_KMS("could not create framebuffer\n");
+		drm_dbg_kms(dev, "could not create framebuffer\n");
 		return fb;
 	}
 
@@ -356,7 +356,7 @@ int drm_mode_addfb2(struct drm_device *dev,
 	if (IS_ERR(fb))
 		return PTR_ERR(fb);
 
-	DRM_DEBUG_KMS("[FB:%d]\n", fb->base.id);
+	drm_dbg_kms(dev, "[FB:%d]\n", fb->base.id);
 	r->fb_id = fb->base.id;
 
 	/* Transfer ownership to the filp for reaping on close */
@@ -384,7 +384,7 @@ int drm_mode_addfb2_ioctl(struct drm_device *dev,
 		 * then.  So block it to make userspace fallback to
 		 * ADDFB.
 		 */
-		DRM_DEBUG_KMS("addfb2 broken on bigendian");
+		drm_dbg_kms(dev, "addfb2 broken on bigendian");
 		return -EOPNOTSUPP;
 	}
 #endif
@@ -530,7 +530,7 @@ int drm_mode_getfb(struct drm_device *dev,
 	r->height = fb->height;
 	r->width = fb->width;
 	r->depth = fb->format->depth;
-	r->bpp = fb->format->cpp[0] * 8;
+	r->bpp = drm_format_info_bpp(fb->format, 0);
 	r->pitch = fb->pitches[0];
 
 	/* GET_FB() is an unprivileged ioctl so we must not return a
@@ -935,7 +935,7 @@ EXPORT_SYMBOL(drm_framebuffer_unregister_private);
  * the id and get back -EINVAL. Obviously no concern at driver unload time.
  *
  * Also, the framebuffer will not be removed from the lookup idr - for
- * user-created framebuffers this will happen in in the rmfb ioctl. For
+ * user-created framebuffers this will happen in the rmfb ioctl. For
  * driver-private objects (e.g. for fbdev) drivers need to explicitly call
  * drm_framebuffer_unregister_private.
  */
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index ad068865ba20..8b68a3c1e6ab 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -165,6 +165,7 @@ void drm_gem_private_object_init(struct drm_device *dev,
 		obj->resv = &obj->_resv;
 
 	drm_vma_node_reset(&obj->vma_node);
+	INIT_LIST_HEAD(&obj->lru_node);
 }
 EXPORT_SYMBOL(drm_gem_private_object_init);
 
@@ -936,6 +937,7 @@ drm_gem_object_release(struct drm_gem_object *obj)
 
 	dma_resv_fini(&obj->_resv);
 	drm_gem_free_mmap_offset(obj);
+	drm_gem_lru_remove(obj);
 }
 EXPORT_SYMBOL(drm_gem_object_release);
 
@@ -1259,3 +1261,171 @@ drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
 	ww_acquire_fini(acquire_ctx);
 }
 EXPORT_SYMBOL(drm_gem_unlock_reservations);
+
+/**
+ * drm_gem_lru_init - initialize a LRU
+ *
+ * @lru: The LRU to initialize
+ * @lock: The lock protecting the LRU
+ */
+void
+drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock)
+{
+	lru->lock = lock;
+	lru->count = 0;
+	INIT_LIST_HEAD(&lru->list);
+}
+EXPORT_SYMBOL(drm_gem_lru_init);
+
+static void
+drm_gem_lru_remove_locked(struct drm_gem_object *obj)
+{
+	obj->lru->count -= obj->size >> PAGE_SHIFT;
+	WARN_ON(obj->lru->count < 0);
+	list_del(&obj->lru_node);
+	obj->lru = NULL;
+}
+
+/**
+ * drm_gem_lru_remove - remove object from whatever LRU it is in
+ *
+ * If the object is currently in any LRU, remove it.
+ *
+ * @obj: The GEM object to remove from current LRU
+ */
+void
+drm_gem_lru_remove(struct drm_gem_object *obj)
+{
+	struct drm_gem_lru *lru = obj->lru;
+
+	if (!lru)
+		return;
+
+	mutex_lock(lru->lock);
+	drm_gem_lru_remove_locked(obj);
+	mutex_unlock(lru->lock);
+}
+EXPORT_SYMBOL(drm_gem_lru_remove);
+
+static void
+drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj)
+{
+	lockdep_assert_held_once(lru->lock);
+
+	if (obj->lru)
+		drm_gem_lru_remove_locked(obj);
+
+	lru->count += obj->size >> PAGE_SHIFT;
+	list_add_tail(&obj->lru_node, &lru->list);
+	obj->lru = lru;
+}
+
+/**
+ * drm_gem_lru_move_tail - move the object to the tail of the LRU
+ *
+ * If the object is already in this LRU it will be moved to the
+ * tail.  Otherwise it will be removed from whichever other LRU
+ * it is in (if any) and moved into this LRU.
+ *
+ * @lru: The LRU to move the object into.
+ * @obj: The GEM object to move into this LRU
+ */
+void
+drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj)
+{
+	mutex_lock(lru->lock);
+	drm_gem_lru_move_tail_locked(lru, obj);
+	mutex_unlock(lru->lock);
+}
+EXPORT_SYMBOL(drm_gem_lru_move_tail);
+
+/**
+ * drm_gem_lru_scan - helper to implement shrinker.scan_objects
+ *
+ * If the shrink callback succeeds, it is expected that the driver
+ * move the object out of this LRU.
+ *
+ * If the LRU possibly contain active buffers, it is the responsibility
+ * of the shrink callback to check for this (ie. dma_resv_test_signaled())
+ * or if necessary block until the buffer becomes idle.
+ *
+ * @lru: The LRU to scan
+ * @nr_to_scan: The number of pages to try to reclaim
+ * @shrink: Callback to try to shrink/reclaim the object.
+ */
+unsigned long
+drm_gem_lru_scan(struct drm_gem_lru *lru, unsigned nr_to_scan,
+		 bool (*shrink)(struct drm_gem_object *obj))
+{
+	struct drm_gem_lru still_in_lru;
+	struct drm_gem_object *obj;
+	unsigned freed = 0;
+
+	drm_gem_lru_init(&still_in_lru, lru->lock);
+
+	mutex_lock(lru->lock);
+
+	while (freed < nr_to_scan) {
+		obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node);
+
+		if (!obj)
+			break;
+
+		drm_gem_lru_move_tail_locked(&still_in_lru, obj);
+
+		/*
+		 * If it's in the process of being freed, gem_object->free()
+		 * may be blocked on lock waiting to remove it.  So just
+		 * skip it.
+		 */
+		if (!kref_get_unless_zero(&obj->refcount))
+			continue;
+
+		/*
+		 * Now that we own a reference, we can drop the lock for the
+		 * rest of the loop body, to reduce contention with other
+		 * code paths that need the LRU lock
+		 */
+		mutex_unlock(lru->lock);
+
+		/*
+		 * Note that this still needs to be trylock, since we can
+		 * hit shrinker in response to trying to get backing pages
+		 * for this obj (ie. while it's lock is already held)
+		 */
+		if (!dma_resv_trylock(obj->resv))
+			goto tail;
+
+		if (shrink(obj)) {
+			freed += obj->size >> PAGE_SHIFT;
+
+			/*
+			 * If we succeeded in releasing the object's backing
+			 * pages, we expect the driver to have moved the object
+			 * out of this LRU
+			 */
+			WARN_ON(obj->lru == &still_in_lru);
+			WARN_ON(obj->lru == lru);
+		}
+
+		dma_resv_unlock(obj->resv);
+
+tail:
+		drm_gem_object_put(obj);
+		mutex_lock(lru->lock);
+	}
+
+	/*
+	 * Move objects we've skipped over out of the temporary still_in_lru
+	 * back into this LRU
+	 */
+	list_for_each_entry (obj, &still_in_lru.list, lru_node)
+		obj->lru = lru;
+	list_splice_tail(&still_in_lru.list, &lru->list);
+	lru->count += still_in_lru.count;
+
+	mutex_unlock(lru->lock);
+
+	return freed;
+}
+EXPORT_SYMBOL(drm_gem_lru_scan);
diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_dma_helper.c
index 42abee9a0f4f..f6901ff97bbb 100644
--- a/drivers/gpu/drm/drm_gem_cma_helper.c
+++ b/drivers/gpu/drm/drm_gem_dma_helper.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * drm gem CMA (contiguous memory allocator) helper functions
+ * drm gem DMA helper functions
  *
  * Copyright (C) 2012 Sascha Hauer, Pengutronix
  *
@@ -20,20 +20,17 @@
 #include <drm/drm.h>
 #include <drm/drm_device.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_vma_manager.h>
 
 /**
- * DOC: cma helpers
+ * DOC: dma helpers
  *
- * The DRM GEM/CMA helpers are a means to provide buffer objects that are
+ * The DRM GEM/DMA helpers are a means to provide buffer objects that are
  * presented to the device as a contiguous chunk of memory. This is useful
  * for devices that do not support scatter-gather DMA (either directly or
  * by using an intimately attached IOMMU).
  *
- * Despite the name, the DRM GEM/CMA helpers are not hardwired to use the
- * Contiguous Memory Allocator (CMA).
- *
  * For devices that access the memory bus through an (external) IOMMU then
  * the buffer objects are allocated using a traditional page-based
  * allocator and may be scattered through physical memory. However they
@@ -44,36 +41,36 @@
  * objects that are physically contiguous in memory.
  *
  * For GEM callback helpers in struct &drm_gem_object functions, see likewise
- * named functions with an _object_ infix (e.g., drm_gem_cma_object_vmap() wraps
- * drm_gem_cma_vmap()). These helpers perform the necessary type conversion.
+ * named functions with an _object_ infix (e.g., drm_gem_dma_object_vmap() wraps
+ * drm_gem_dma_vmap()). These helpers perform the necessary type conversion.
  */
 
-static const struct drm_gem_object_funcs drm_gem_cma_default_funcs = {
-	.free = drm_gem_cma_object_free,
-	.print_info = drm_gem_cma_object_print_info,
-	.get_sg_table = drm_gem_cma_object_get_sg_table,
-	.vmap = drm_gem_cma_object_vmap,
-	.mmap = drm_gem_cma_object_mmap,
-	.vm_ops = &drm_gem_cma_vm_ops,
+static const struct drm_gem_object_funcs drm_gem_dma_default_funcs = {
+	.free = drm_gem_dma_object_free,
+	.print_info = drm_gem_dma_object_print_info,
+	.get_sg_table = drm_gem_dma_object_get_sg_table,
+	.vmap = drm_gem_dma_object_vmap,
+	.mmap = drm_gem_dma_object_mmap,
+	.vm_ops = &drm_gem_dma_vm_ops,
 };
 
 /**
- * __drm_gem_cma_create - Create a GEM CMA object without allocating memory
+ * __drm_gem_dma_create - Create a GEM DMA object without allocating memory
  * @drm: DRM device
  * @size: size of the object to allocate
  * @private: true if used for internal purposes
  *
- * This function creates and initializes a GEM CMA object of the given size,
+ * This function creates and initializes a GEM DMA object of the given size,
  * but doesn't allocate any memory to back the object.
  *
  * Returns:
- * A struct drm_gem_cma_object * on success or an ERR_PTR()-encoded negative
+ * A struct drm_gem_dma_object * on success or an ERR_PTR()-encoded negative
  * error code on failure.
  */
-static struct drm_gem_cma_object *
-__drm_gem_cma_create(struct drm_device *drm, size_t size, bool private)
+static struct drm_gem_dma_object *
+__drm_gem_dma_create(struct drm_device *drm, size_t size, bool private)
 {
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	struct drm_gem_object *gem_obj;
 	int ret = 0;
 
@@ -81,22 +78,22 @@ __drm_gem_cma_create(struct drm_device *drm, size_t size, bool private)
 		gem_obj = drm->driver->gem_create_object(drm, size);
 		if (IS_ERR(gem_obj))
 			return ERR_CAST(gem_obj);
-		cma_obj = to_drm_gem_cma_obj(gem_obj);
+		dma_obj = to_drm_gem_dma_obj(gem_obj);
 	} else {
-		cma_obj = kzalloc(sizeof(*cma_obj), GFP_KERNEL);
-		if (!cma_obj)
+		dma_obj = kzalloc(sizeof(*dma_obj), GFP_KERNEL);
+		if (!dma_obj)
 			return ERR_PTR(-ENOMEM);
-		gem_obj = &cma_obj->base;
+		gem_obj = &dma_obj->base;
 	}
 
 	if (!gem_obj->funcs)
-		gem_obj->funcs = &drm_gem_cma_default_funcs;
+		gem_obj->funcs = &drm_gem_dma_default_funcs;
 
 	if (private) {
 		drm_gem_private_object_init(drm, gem_obj, size);
 
 		/* Always use writecombine for dma-buf mappings */
-		cma_obj->map_noncoherent = false;
+		dma_obj->map_noncoherent = false;
 	} else {
 		ret = drm_gem_object_init(drm, gem_obj, size);
 	}
@@ -109,19 +106,19 @@ __drm_gem_cma_create(struct drm_device *drm, size_t size, bool private)
 		goto error;
 	}
 
-	return cma_obj;
+	return dma_obj;
 
 error:
-	kfree(cma_obj);
+	kfree(dma_obj);
 	return ERR_PTR(ret);
 }
 
 /**
- * drm_gem_cma_create - allocate an object with the given size
+ * drm_gem_dma_create - allocate an object with the given size
  * @drm: DRM device
  * @size: size of the object to allocate
  *
- * This function creates a CMA GEM object and allocates memory as backing store.
+ * This function creates a DMA GEM object and allocates memory as backing store.
  * The allocated memory will occupy a contiguous chunk of bus address space.
  *
  * For devices that are directly connected to the memory bus then the allocated
@@ -131,78 +128,79 @@ error:
  * requirements.
  *
  * Returns:
- * A struct drm_gem_cma_object * on success or an ERR_PTR()-encoded negative
+ * A struct drm_gem_dma_object * on success or an ERR_PTR()-encoded negative
  * error code on failure.
  */
-struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm,
+struct drm_gem_dma_object *drm_gem_dma_create(struct drm_device *drm,
 					      size_t size)
 {
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	int ret;
 
 	size = round_up(size, PAGE_SIZE);
 
-	cma_obj = __drm_gem_cma_create(drm, size, false);
-	if (IS_ERR(cma_obj))
-		return cma_obj;
+	dma_obj = __drm_gem_dma_create(drm, size, false);
+	if (IS_ERR(dma_obj))
+		return dma_obj;
 
-	if (cma_obj->map_noncoherent) {
-		cma_obj->vaddr = dma_alloc_noncoherent(drm->dev, size,
-						       &cma_obj->paddr,
+	if (dma_obj->map_noncoherent) {
+		dma_obj->vaddr = dma_alloc_noncoherent(drm->dev, size,
+						       &dma_obj->dma_addr,
 						       DMA_TO_DEVICE,
 						       GFP_KERNEL | __GFP_NOWARN);
 	} else {
-		cma_obj->vaddr = dma_alloc_wc(drm->dev, size, &cma_obj->paddr,
+		dma_obj->vaddr = dma_alloc_wc(drm->dev, size,
+					      &dma_obj->dma_addr,
 					      GFP_KERNEL | __GFP_NOWARN);
 	}
-	if (!cma_obj->vaddr) {
+	if (!dma_obj->vaddr) {
 		drm_dbg(drm, "failed to allocate buffer with size %zu\n",
 			 size);
 		ret = -ENOMEM;
 		goto error;
 	}
 
-	return cma_obj;
+	return dma_obj;
 
 error:
-	drm_gem_object_put(&cma_obj->base);
+	drm_gem_object_put(&dma_obj->base);
 	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL_GPL(drm_gem_cma_create);
+EXPORT_SYMBOL_GPL(drm_gem_dma_create);
 
 /**
- * drm_gem_cma_create_with_handle - allocate an object with the given size and
+ * drm_gem_dma_create_with_handle - allocate an object with the given size and
  *     return a GEM handle to it
  * @file_priv: DRM file-private structure to register the handle for
  * @drm: DRM device
  * @size: size of the object to allocate
  * @handle: return location for the GEM handle
  *
- * This function creates a CMA GEM object, allocating a chunk of memory as
+ * This function creates a DMA GEM object, allocating a chunk of memory as
  * backing store. The GEM object is then added to the list of object associated
  * with the given file and a handle to it is returned.
  *
  * The allocated memory will occupy a contiguous chunk of bus address space.
- * See drm_gem_cma_create() for more details.
+ * See drm_gem_dma_create() for more details.
  *
  * Returns:
- * A struct drm_gem_cma_object * on success or an ERR_PTR()-encoded negative
+ * A struct drm_gem_dma_object * on success or an ERR_PTR()-encoded negative
  * error code on failure.
  */
-static struct drm_gem_cma_object *
-drm_gem_cma_create_with_handle(struct drm_file *file_priv,
+static struct drm_gem_dma_object *
+drm_gem_dma_create_with_handle(struct drm_file *file_priv,
 			       struct drm_device *drm, size_t size,
 			       uint32_t *handle)
 {
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	struct drm_gem_object *gem_obj;
 	int ret;
 
-	cma_obj = drm_gem_cma_create(drm, size);
-	if (IS_ERR(cma_obj))
-		return cma_obj;
+	dma_obj = drm_gem_dma_create(drm, size);
+	if (IS_ERR(dma_obj))
+		return dma_obj;
 
-	gem_obj = &cma_obj->base;
+	gem_obj = &dma_obj->base;
 
 	/*
 	 * allocate a id of idr table where the obj is registered
@@ -214,44 +212,44 @@ drm_gem_cma_create_with_handle(struct drm_file *file_priv,
 	if (ret)
 		return ERR_PTR(ret);
 
-	return cma_obj;
+	return dma_obj;
 }
 
 /**
- * drm_gem_cma_free - free resources associated with a CMA GEM object
- * @cma_obj: CMA GEM object to free
+ * drm_gem_dma_free - free resources associated with a DMA GEM object
+ * @dma_obj: DMA GEM object to free
  *
- * This function frees the backing memory of the CMA GEM object, cleans up the
+ * This function frees the backing memory of the DMA GEM object, cleans up the
  * GEM object state and frees the memory used to store the object itself.
  * If the buffer is imported and the virtual address is set, it is released.
  */
-void drm_gem_cma_free(struct drm_gem_cma_object *cma_obj)
+void drm_gem_dma_free(struct drm_gem_dma_object *dma_obj)
 {
-	struct drm_gem_object *gem_obj = &cma_obj->base;
-	struct iosys_map map = IOSYS_MAP_INIT_VADDR(cma_obj->vaddr);
+	struct drm_gem_object *gem_obj = &dma_obj->base;
+	struct iosys_map map = IOSYS_MAP_INIT_VADDR(dma_obj->vaddr);
 
 	if (gem_obj->import_attach) {
-		if (cma_obj->vaddr)
+		if (dma_obj->vaddr)
 			dma_buf_vunmap(gem_obj->import_attach->dmabuf, &map);
-		drm_prime_gem_destroy(gem_obj, cma_obj->sgt);
-	} else if (cma_obj->vaddr) {
-		if (cma_obj->map_noncoherent)
-			dma_free_noncoherent(gem_obj->dev->dev, cma_obj->base.size,
-					     cma_obj->vaddr, cma_obj->paddr,
+		drm_prime_gem_destroy(gem_obj, dma_obj->sgt);
+	} else if (dma_obj->vaddr) {
+		if (dma_obj->map_noncoherent)
+			dma_free_noncoherent(gem_obj->dev->dev, dma_obj->base.size,
+					     dma_obj->vaddr, dma_obj->dma_addr,
 					     DMA_TO_DEVICE);
 		else
-			dma_free_wc(gem_obj->dev->dev, cma_obj->base.size,
-				    cma_obj->vaddr, cma_obj->paddr);
+			dma_free_wc(gem_obj->dev->dev, dma_obj->base.size,
+				    dma_obj->vaddr, dma_obj->dma_addr);
 	}
 
 	drm_gem_object_release(gem_obj);
 
-	kfree(cma_obj);
+	kfree(dma_obj);
 }
-EXPORT_SYMBOL_GPL(drm_gem_cma_free);
+EXPORT_SYMBOL_GPL(drm_gem_dma_free);
 
 /**
- * drm_gem_cma_dumb_create_internal - create a dumb buffer object
+ * drm_gem_dma_dumb_create_internal - create a dumb buffer object
  * @file_priv: DRM file-private structure to create the dumb buffer for
  * @drm: DRM device
  * @args: IOCTL data
@@ -264,12 +262,12 @@ EXPORT_SYMBOL_GPL(drm_gem_cma_free);
  * Returns:
  * 0 on success or a negative error code on failure.
  */
-int drm_gem_cma_dumb_create_internal(struct drm_file *file_priv,
+int drm_gem_dma_dumb_create_internal(struct drm_file *file_priv,
 				     struct drm_device *drm,
 				     struct drm_mode_create_dumb *args)
 {
 	unsigned int min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 
 	if (args->pitch < min_pitch)
 		args->pitch = min_pitch;
@@ -277,14 +275,14 @@ int drm_gem_cma_dumb_create_internal(struct drm_file *file_priv,
 	if (args->size < args->pitch * args->height)
 		args->size = args->pitch * args->height;
 
-	cma_obj = drm_gem_cma_create_with_handle(file_priv, drm, args->size,
+	dma_obj = drm_gem_dma_create_with_handle(file_priv, drm, args->size,
 						 &args->handle);
-	return PTR_ERR_OR_ZERO(cma_obj);
+	return PTR_ERR_OR_ZERO(dma_obj);
 }
-EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_create_internal);
+EXPORT_SYMBOL_GPL(drm_gem_dma_dumb_create_internal);
 
 /**
- * drm_gem_cma_dumb_create - create a dumb buffer object
+ * drm_gem_dma_dumb_create - create a dumb buffer object
  * @file_priv: DRM file-private structure to create the dumb buffer for
  * @drm: DRM device
  * @args: IOCTL data
@@ -296,35 +294,35 @@ EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_create_internal);
  *
  * For hardware with additional restrictions, drivers can adjust the fields
  * set up by userspace and pass the IOCTL data along to the
- * drm_gem_cma_dumb_create_internal() function.
+ * drm_gem_dma_dumb_create_internal() function.
  *
  * Returns:
  * 0 on success or a negative error code on failure.
  */
-int drm_gem_cma_dumb_create(struct drm_file *file_priv,
+int drm_gem_dma_dumb_create(struct drm_file *file_priv,
 			    struct drm_device *drm,
 			    struct drm_mode_create_dumb *args)
 {
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 
 	args->pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
 	args->size = args->pitch * args->height;
 
-	cma_obj = drm_gem_cma_create_with_handle(file_priv, drm, args->size,
+	dma_obj = drm_gem_dma_create_with_handle(file_priv, drm, args->size,
 						 &args->handle);
-	return PTR_ERR_OR_ZERO(cma_obj);
+	return PTR_ERR_OR_ZERO(dma_obj);
 }
-EXPORT_SYMBOL_GPL(drm_gem_cma_dumb_create);
+EXPORT_SYMBOL_GPL(drm_gem_dma_dumb_create);
 
-const struct vm_operations_struct drm_gem_cma_vm_ops = {
+const struct vm_operations_struct drm_gem_dma_vm_ops = {
 	.open = drm_gem_vm_open,
 	.close = drm_gem_vm_close,
 };
-EXPORT_SYMBOL_GPL(drm_gem_cma_vm_ops);
+EXPORT_SYMBOL_GPL(drm_gem_dma_vm_ops);
 
 #ifndef CONFIG_MMU
 /**
- * drm_gem_cma_get_unmapped_area - propose address for mapping in noMMU cases
+ * drm_gem_dma_get_unmapped_area - propose address for mapping in noMMU cases
  * @filp: file object
  * @addr: memory address
  * @len: buffer size
@@ -339,13 +337,13 @@ EXPORT_SYMBOL_GPL(drm_gem_cma_vm_ops);
  * Returns:
  * mapping address on success or a negative error code on failure.
  */
-unsigned long drm_gem_cma_get_unmapped_area(struct file *filp,
+unsigned long drm_gem_dma_get_unmapped_area(struct file *filp,
 					    unsigned long addr,
 					    unsigned long len,
 					    unsigned long pgoff,
 					    unsigned long flags)
 {
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	struct drm_gem_object *obj = NULL;
 	struct drm_file *priv = filp->private_data;
 	struct drm_device *dev = priv->minor->dev;
@@ -384,35 +382,35 @@ unsigned long drm_gem_cma_get_unmapped_area(struct file *filp,
 		return -EACCES;
 	}
 
-	cma_obj = to_drm_gem_cma_obj(obj);
+	dma_obj = to_drm_gem_dma_obj(obj);
 
 	drm_gem_object_put(obj);
 
-	return cma_obj->vaddr ? (unsigned long)cma_obj->vaddr : -EINVAL;
+	return dma_obj->vaddr ? (unsigned long)dma_obj->vaddr : -EINVAL;
 }
-EXPORT_SYMBOL_GPL(drm_gem_cma_get_unmapped_area);
+EXPORT_SYMBOL_GPL(drm_gem_dma_get_unmapped_area);
 #endif
 
 /**
- * drm_gem_cma_print_info() - Print &drm_gem_cma_object info for debugfs
- * @cma_obj: CMA GEM object
+ * drm_gem_dma_print_info() - Print &drm_gem_dma_object info for debugfs
+ * @dma_obj: DMA GEM object
  * @p: DRM printer
  * @indent: Tab indentation level
  *
- * This function prints paddr and vaddr for use in e.g. debugfs output.
+ * This function prints dma_addr and vaddr for use in e.g. debugfs output.
  */
-void drm_gem_cma_print_info(const struct drm_gem_cma_object *cma_obj,
+void drm_gem_dma_print_info(const struct drm_gem_dma_object *dma_obj,
 			    struct drm_printer *p, unsigned int indent)
 {
-	drm_printf_indent(p, indent, "paddr=%pad\n", &cma_obj->paddr);
-	drm_printf_indent(p, indent, "vaddr=%p\n", cma_obj->vaddr);
+	drm_printf_indent(p, indent, "dma_addr=%pad\n", &dma_obj->dma_addr);
+	drm_printf_indent(p, indent, "vaddr=%p\n", dma_obj->vaddr);
 }
-EXPORT_SYMBOL(drm_gem_cma_print_info);
+EXPORT_SYMBOL(drm_gem_dma_print_info);
 
 /**
- * drm_gem_cma_get_sg_table - provide a scatter/gather table of pinned
- *     pages for a CMA GEM object
- * @cma_obj: CMA GEM object
+ * drm_gem_dma_get_sg_table - provide a scatter/gather table of pinned
+ *     pages for a DMA GEM object
+ * @dma_obj: DMA GEM object
  *
  * This function exports a scatter/gather table by calling the standard
  * DMA mapping API.
@@ -420,9 +418,9 @@ EXPORT_SYMBOL(drm_gem_cma_print_info);
  * Returns:
  * A pointer to the scatter/gather table of pinned pages or NULL on failure.
  */
-struct sg_table *drm_gem_cma_get_sg_table(struct drm_gem_cma_object *cma_obj)
+struct sg_table *drm_gem_dma_get_sg_table(struct drm_gem_dma_object *dma_obj)
 {
-	struct drm_gem_object *obj = &cma_obj->base;
+	struct drm_gem_object *obj = &dma_obj->base;
 	struct sg_table *sgt;
 	int ret;
 
@@ -430,8 +428,8 @@ struct sg_table *drm_gem_cma_get_sg_table(struct drm_gem_cma_object *cma_obj)
 	if (!sgt)
 		return ERR_PTR(-ENOMEM);
 
-	ret = dma_get_sgtable(obj->dev->dev, sgt, cma_obj->vaddr,
-			      cma_obj->paddr, obj->size);
+	ret = dma_get_sgtable(obj->dev->dev, sgt, dma_obj->vaddr,
+			      dma_obj->dma_addr, obj->size);
 	if (ret < 0)
 		goto out;
 
@@ -441,10 +439,10 @@ out:
 	kfree(sgt);
 	return ERR_PTR(ret);
 }
-EXPORT_SYMBOL_GPL(drm_gem_cma_get_sg_table);
+EXPORT_SYMBOL_GPL(drm_gem_dma_get_sg_table);
 
 /**
- * drm_gem_cma_prime_import_sg_table - produce a CMA GEM object from another
+ * drm_gem_dma_prime_import_sg_table - produce a DMA GEM object from another
  *     driver's scatter/gather table of pinned pages
  * @dev: device to import into
  * @attach: DMA-BUF attachment
@@ -453,7 +451,7 @@ EXPORT_SYMBOL_GPL(drm_gem_cma_get_sg_table);
  * This function imports a scatter/gather table exported via DMA-BUF by
  * another driver. Imported buffers must be physically contiguous in memory
  * (i.e. the scatter/gather table must contain a single entry). Drivers that
- * use the CMA helpers should set this as their
+ * use the DMA helpers should set this as their
  * &drm_driver.gem_prime_import_sg_table callback.
  *
  * Returns:
@@ -461,56 +459,57 @@ EXPORT_SYMBOL_GPL(drm_gem_cma_get_sg_table);
  * error code on failure.
  */
 struct drm_gem_object *
-drm_gem_cma_prime_import_sg_table(struct drm_device *dev,
+drm_gem_dma_prime_import_sg_table(struct drm_device *dev,
 				  struct dma_buf_attachment *attach,
 				  struct sg_table *sgt)
 {
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 
 	/* check if the entries in the sg_table are contiguous */
 	if (drm_prime_get_contiguous_size(sgt) < attach->dmabuf->size)
 		return ERR_PTR(-EINVAL);
 
-	/* Create a CMA GEM buffer. */
-	cma_obj = __drm_gem_cma_create(dev, attach->dmabuf->size, true);
-	if (IS_ERR(cma_obj))
-		return ERR_CAST(cma_obj);
+	/* Create a DMA GEM buffer. */
+	dma_obj = __drm_gem_dma_create(dev, attach->dmabuf->size, true);
+	if (IS_ERR(dma_obj))
+		return ERR_CAST(dma_obj);
 
-	cma_obj->paddr = sg_dma_address(sgt->sgl);
-	cma_obj->sgt = sgt;
+	dma_obj->dma_addr = sg_dma_address(sgt->sgl);
+	dma_obj->sgt = sgt;
 
-	DRM_DEBUG_PRIME("dma_addr = %pad, size = %zu\n", &cma_obj->paddr, attach->dmabuf->size);
+	DRM_DEBUG_PRIME("dma_addr = %pad, size = %zu\n", &dma_obj->dma_addr,
+			attach->dmabuf->size);
 
-	return &cma_obj->base;
+	return &dma_obj->base;
 }
-EXPORT_SYMBOL_GPL(drm_gem_cma_prime_import_sg_table);
+EXPORT_SYMBOL_GPL(drm_gem_dma_prime_import_sg_table);
 
 /**
- * drm_gem_cma_vmap - map a CMA GEM object into the kernel's virtual
+ * drm_gem_dma_vmap - map a DMA GEM object into the kernel's virtual
  *     address space
- * @cma_obj: CMA GEM object
- * @map: Returns the kernel virtual address of the CMA GEM object's backing
+ * @dma_obj: DMA GEM object
+ * @map: Returns the kernel virtual address of the DMA GEM object's backing
  *       store.
  *
  * This function maps a buffer into the kernel's virtual address space.
- * Since the CMA buffers are already mapped into the kernel virtual address
+ * Since the DMA buffers are already mapped into the kernel virtual address
  * space this simply returns the cached virtual address.
  *
  * Returns:
  * 0 on success, or a negative error code otherwise.
  */
-int drm_gem_cma_vmap(struct drm_gem_cma_object *cma_obj,
+int drm_gem_dma_vmap(struct drm_gem_dma_object *dma_obj,
 		     struct iosys_map *map)
 {
-	iosys_map_set_vaddr(map, cma_obj->vaddr);
+	iosys_map_set_vaddr(map, dma_obj->vaddr);
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(drm_gem_cma_vmap);
+EXPORT_SYMBOL_GPL(drm_gem_dma_vmap);
 
 /**
- * drm_gem_cma_mmap - memory-map an exported CMA GEM object
- * @cma_obj: CMA GEM object
+ * drm_gem_dma_mmap - memory-map an exported DMA GEM object
+ * @dma_obj: DMA GEM object
  * @vma: VMA for the area to be mapped
  *
  * This function maps a buffer into a userspace process's address space.
@@ -520,9 +519,9 @@ EXPORT_SYMBOL_GPL(drm_gem_cma_vmap);
  * Returns:
  * 0 on success or a negative error code on failure.
  */
-int drm_gem_cma_mmap(struct drm_gem_cma_object *cma_obj, struct vm_area_struct *vma)
+int drm_gem_dma_mmap(struct drm_gem_dma_object *dma_obj, struct vm_area_struct *vma)
 {
-	struct drm_gem_object *obj = &cma_obj->base;
+	struct drm_gem_object *obj = &dma_obj->base;
 	int ret;
 
 	/*
@@ -534,37 +533,38 @@ int drm_gem_cma_mmap(struct drm_gem_cma_object *cma_obj, struct vm_area_struct *
 	vma->vm_flags &= ~VM_PFNMAP;
 	vma->vm_flags |= VM_DONTEXPAND;
 
-	if (cma_obj->map_noncoherent) {
+	if (dma_obj->map_noncoherent) {
 		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
 
-		ret = dma_mmap_pages(cma_obj->base.dev->dev,
+		ret = dma_mmap_pages(dma_obj->base.dev->dev,
 				     vma, vma->vm_end - vma->vm_start,
-				     virt_to_page(cma_obj->vaddr));
+				     virt_to_page(dma_obj->vaddr));
 	} else {
-		ret = dma_mmap_wc(cma_obj->base.dev->dev, vma, cma_obj->vaddr,
-				  cma_obj->paddr, vma->vm_end - vma->vm_start);
+		ret = dma_mmap_wc(dma_obj->base.dev->dev, vma, dma_obj->vaddr,
+				  dma_obj->dma_addr,
+				  vma->vm_end - vma->vm_start);
 	}
 	if (ret)
 		drm_gem_vm_close(vma);
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(drm_gem_cma_mmap);
+EXPORT_SYMBOL_GPL(drm_gem_dma_mmap);
 
 /**
- * drm_gem_cma_prime_import_sg_table_vmap - PRIME import another driver's
+ * drm_gem_dma_prime_import_sg_table_vmap - PRIME import another driver's
  *	scatter/gather table and get the virtual address of the buffer
  * @dev: DRM device
  * @attach: DMA-BUF attachment
  * @sgt: Scatter/gather table of pinned pages
  *
  * This function imports a scatter/gather table using
- * drm_gem_cma_prime_import_sg_table() and uses dma_buf_vmap() to get the kernel
- * virtual address. This ensures that a CMA GEM object always has its virtual
+ * drm_gem_dma_prime_import_sg_table() and uses dma_buf_vmap() to get the kernel
+ * virtual address. This ensures that a DMA GEM object always has its virtual
  * address set. This address is released when the object is freed.
  *
  * This function can be used as the &drm_driver.gem_prime_import_sg_table
- * callback. The &DRM_GEM_CMA_DRIVER_OPS_VMAP macro provides a shortcut to set
+ * callback. The &DRM_GEM_DMA_DRIVER_OPS_VMAP macro provides a shortcut to set
  * the necessary DRM driver operations.
  *
  * Returns:
@@ -572,11 +572,11 @@ EXPORT_SYMBOL_GPL(drm_gem_cma_mmap);
  * error code on failure.
  */
 struct drm_gem_object *
-drm_gem_cma_prime_import_sg_table_vmap(struct drm_device *dev,
+drm_gem_dma_prime_import_sg_table_vmap(struct drm_device *dev,
 				       struct dma_buf_attachment *attach,
 				       struct sg_table *sgt)
 {
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	struct drm_gem_object *obj;
 	struct iosys_map map;
 	int ret;
@@ -587,19 +587,19 @@ drm_gem_cma_prime_import_sg_table_vmap(struct drm_device *dev,
 		return ERR_PTR(ret);
 	}
 
-	obj = drm_gem_cma_prime_import_sg_table(dev, attach, sgt);
+	obj = drm_gem_dma_prime_import_sg_table(dev, attach, sgt);
 	if (IS_ERR(obj)) {
 		dma_buf_vunmap(attach->dmabuf, &map);
 		return obj;
 	}
 
-	cma_obj = to_drm_gem_cma_obj(obj);
-	cma_obj->vaddr = map.vaddr;
+	dma_obj = to_drm_gem_dma_obj(obj);
+	dma_obj->vaddr = map.vaddr;
 
 	return obj;
 }
-EXPORT_SYMBOL(drm_gem_cma_prime_import_sg_table_vmap);
+EXPORT_SYMBOL(drm_gem_dma_prime_import_sg_table_vmap);
 
-MODULE_DESCRIPTION("DRM CMA memory-management helpers");
+MODULE_DESCRIPTION("DRM DMA memory-management helpers");
 MODULE_IMPORT_NS(DMA_BUF);
 MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/drm_gem_framebuffer_helper.c b/drivers/gpu/drm/drm_gem_framebuffer_helper.c
index 61339a9cd010..880a4975507f 100644
--- a/drivers/gpu/drm/drm_gem_framebuffer_helper.c
+++ b/drivers/gpu/drm/drm_gem_framebuffer_helper.c
@@ -490,6 +490,8 @@ void drm_gem_fb_end_cpu_access(struct drm_framebuffer *fb, enum dma_data_directi
 }
 EXPORT_SYMBOL(drm_gem_fb_end_cpu_access);
 
+// TODO Drop this function and replace by drm_format_info_bpp() once all
+// DRM_FORMAT_* provide proper block info in drivers/gpu/drm/drm_fourcc.c
 static __u32 drm_gem_afbc_get_bpp(struct drm_device *dev,
 				  const struct drm_mode_fb_cmd2 *mode_cmd)
 {
@@ -497,11 +499,6 @@ static __u32 drm_gem_afbc_get_bpp(struct drm_device *dev,
 
 	info = drm_get_format_info(dev, mode_cmd);
 
-	/* use whatever a driver has set */
-	if (info->cpp[0])
-		return info->cpp[0] * 8;
-
-	/* guess otherwise */
 	switch (info->format) {
 	case DRM_FORMAT_YUV420_8BIT:
 		return 12;
@@ -510,11 +507,8 @@ static __u32 drm_gem_afbc_get_bpp(struct drm_device *dev,
 	case DRM_FORMAT_VUY101010:
 		return 30;
 	default:
-		break;
+		return drm_format_info_bpp(info, 0);
 	}
-
-	/* all attempts failed */
-	return 0;
 }
 
 static int drm_gem_afbc_min_size(struct drm_device *dev,
diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c
index 904fc893c905..35138f8a375c 100644
--- a/drivers/gpu/drm/drm_gem_shmem_helper.c
+++ b/drivers/gpu/drm/drm_gem_shmem_helper.c
@@ -663,7 +663,7 @@ EXPORT_SYMBOL(drm_gem_shmem_print_info);
  * drm_gem_shmem_get_pages_sgt() instead.
  *
  * Returns:
- * A pointer to the scatter/gather table of pinned pages or NULL on failure.
+ * A pointer to the scatter/gather table of pinned pages or error pointer on failure.
  */
 struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem)
 {
diff --git a/drivers/gpu/drm/drm_gem_vram_helper.c b/drivers/gpu/drm/drm_gem_vram_helper.c
index d607043716d3..125160b534be 100644
--- a/drivers/gpu/drm/drm_gem_vram_helper.c
+++ b/drivers/gpu/drm/drm_gem_vram_helper.c
@@ -226,9 +226,9 @@ struct drm_gem_vram_object *drm_gem_vram_create(struct drm_device *dev,
 	 * A failing ttm_bo_init will call ttm_buffer_object_destroy
 	 * to release gbo->bo.base and kfree gbo.
 	 */
-	ret = ttm_bo_init(bdev, &gbo->bo, size, ttm_bo_type_device,
-			  &gbo->placement, pg_align, false, NULL, NULL,
-			  ttm_buffer_object_destroy);
+	ret = ttm_bo_init_validate(bdev, &gbo->bo, ttm_bo_type_device,
+				   &gbo->placement, pg_align, false, NULL, NULL,
+				   ttm_buffer_object_destroy);
 	if (ret)
 		return ERR_PTR(ret);
 
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index 8faad23dc1d8..ca2a6e6101dc 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -472,7 +472,13 @@ EXPORT_SYMBOL(drm_invalid_op);
  */
 static int drm_copy_field(char __user *buf, size_t *buf_len, const char *value)
 {
-	int len;
+	size_t len;
+
+	/* don't attempt to copy a NULL pointer */
+	if (WARN_ONCE(!value, "BUG: the value to copy was not set!")) {
+		*buf_len = 0;
+		return 0;
+	}
 
 	/* don't overflow userbuf */
 	len = strlen(value);
diff --git a/drivers/gpu/drm/drm_mipi_dbi.c b/drivers/gpu/drm/drm_mipi_dbi.c
index 2f61f53d472f..a6ac56580876 100644
--- a/drivers/gpu/drm/drm_mipi_dbi.c
+++ b/drivers/gpu/drm/drm_mipi_dbi.c
@@ -205,7 +205,7 @@ int mipi_dbi_buf_copy(void *dst, struct drm_framebuffer *fb,
 	struct drm_gem_object *gem = drm_gem_fb_get_obj(fb, 0);
 	struct iosys_map map[DRM_FORMAT_MAX_PLANES];
 	struct iosys_map data[DRM_FORMAT_MAX_PLANES];
-	void *src;
+	struct iosys_map dst_map = IOSYS_MAP_INIT_VADDR(dst);
 	int ret;
 
 	ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE);
@@ -215,17 +215,16 @@ int mipi_dbi_buf_copy(void *dst, struct drm_framebuffer *fb,
 	ret = drm_gem_fb_vmap(fb, map, data);
 	if (ret)
 		goto out_drm_gem_fb_end_cpu_access;
-	src = data[0].vaddr; /* TODO: Use mapping abstraction properly */
 
 	switch (fb->format->format) {
 	case DRM_FORMAT_RGB565:
 		if (swap)
-			drm_fb_swab(dst, 0, src, fb, clip, !gem->import_attach);
+			drm_fb_swab(&dst_map, NULL, data, fb, clip, !gem->import_attach);
 		else
-			drm_fb_memcpy(dst, 0, src, fb, clip);
+			drm_fb_memcpy(&dst_map, NULL, data, fb, clip);
 		break;
 	case DRM_FORMAT_XRGB8888:
-		drm_fb_xrgb8888_to_rgb565(dst, 0, src, fb, clip, swap);
+		drm_fb_xrgb8888_to_rgb565(&dst_map, NULL, data, fb, clip, swap);
 		break;
 	default:
 		drm_err_once(fb->dev, "Format is not supported: %p4cc\n",
@@ -311,6 +310,24 @@ err_drm_dev_exit:
 }
 
 /**
+ * mipi_dbi_pipe_mode_valid - MIPI DBI mode-valid helper
+ * @pipe: Simple display pipe
+ * @mode: The mode to test
+ *
+ * This function validates a given display mode against the MIPI DBI's hardware
+ * display. Drivers can use this as their &drm_simple_display_pipe_funcs->mode_valid
+ * callback.
+ */
+enum drm_mode_status mipi_dbi_pipe_mode_valid(struct drm_simple_display_pipe *pipe,
+					      const struct drm_display_mode *mode)
+{
+	struct mipi_dbi_dev *dbidev = drm_to_mipi_dbi_dev(pipe->crtc.dev);
+
+	return drm_crtc_helper_mode_valid_fixed(&pipe->crtc, mode, &dbidev->mode);
+}
+EXPORT_SYMBOL(mipi_dbi_pipe_mode_valid);
+
+/**
  * mipi_dbi_pipe_update - Display pipe update helper
  * @pipe: Simple display pipe
  * @old_state: Old plane state
@@ -416,26 +433,8 @@ EXPORT_SYMBOL(mipi_dbi_pipe_disable);
 static int mipi_dbi_connector_get_modes(struct drm_connector *connector)
 {
 	struct mipi_dbi_dev *dbidev = drm_to_mipi_dbi_dev(connector->dev);
-	struct drm_display_mode *mode;
-
-	mode = drm_mode_duplicate(connector->dev, &dbidev->mode);
-	if (!mode) {
-		DRM_ERROR("Failed to duplicate mode\n");
-		return 0;
-	}
-
-	if (mode->name[0] == '\0')
-		drm_mode_set_name(mode);
-
-	mode->type |= DRM_MODE_TYPE_PREFERRED;
-	drm_mode_probed_add(connector, mode);
-
-	if (mode->width_mm) {
-		connector->display_info.width_mm = mode->width_mm;
-		connector->display_info.height_mm = mode->height_mm;
-	}
 
-	return 1;
+	return drm_connector_helper_get_modes_fixed(connector, &dbidev->mode);
 }
 
 static const struct drm_connector_helper_funcs mipi_dbi_connector_hfuncs = {
@@ -1136,7 +1135,7 @@ int mipi_dbi_spi_init(struct spi_device *spi, struct mipi_dbi *dbi,
 	/*
 	 * Even though it's not the SPI device that does DMA (the master does),
 	 * the dma mask is necessary for the dma_alloc_wc() in the GEM code
-	 * (e.g., drm_gem_cma_create()). The dma_addr returned will be a physical
+	 * (e.g., drm_gem_dma_create()). The dma_addr returned will be a physical
 	 * address which might be different from the bus address, but this is
 	 * not a problem since the address will not be used.
 	 * The virtual address is used in the transfer and the SPI core
diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c
index c40bde96cfdf..3ec02748d56f 100644
--- a/drivers/gpu/drm/drm_mipi_dsi.c
+++ b/drivers/gpu/drm/drm_mipi_dsi.c
@@ -346,6 +346,7 @@ static int mipi_dsi_remove_device_fn(struct device *dev, void *priv)
 {
 	struct mipi_dsi_device *dsi = to_mipi_dsi_device(dev);
 
+	mipi_dsi_detach(dsi);
 	mipi_dsi_device_unregister(dsi);
 
 	return 0;
@@ -1236,7 +1237,9 @@ static int mipi_dsi_drv_remove(struct device *dev)
 	struct mipi_dsi_driver *drv = to_mipi_dsi_driver(dev->driver);
 	struct mipi_dsi_device *dsi = to_mipi_dsi_device(dev);
 
-	return drv->remove(dsi);
+	drv->remove(dsi);
+
+	return 0;
 }
 
 static void mipi_dsi_drv_shutdown(struct device *dev)
diff --git a/drivers/gpu/drm/drm_mode_config.c b/drivers/gpu/drm/drm_mode_config.c
index 59b34f07cfce..939d621c9ad4 100644
--- a/drivers/gpu/drm/drm_mode_config.c
+++ b/drivers/gpu/drm/drm_mode_config.c
@@ -151,6 +151,9 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
 	count = 0;
 	connector_id = u64_to_user_ptr(card_res->connector_id_ptr);
 	drm_for_each_connector_iter(connector, &conn_iter) {
+		if (connector->registration_state != DRM_CONNECTOR_REGISTERED)
+			continue;
+
 		/* only expose writeback connectors if userspace understands them */
 		if (!file_priv->writeback_connectors &&
 		    (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK))
@@ -412,8 +415,8 @@ int drmm_mode_config_init(struct drm_device *dev)
 	INIT_LIST_HEAD(&dev->mode_config.property_blob_list);
 	INIT_LIST_HEAD(&dev->mode_config.plane_list);
 	INIT_LIST_HEAD(&dev->mode_config.privobj_list);
-	idr_init(&dev->mode_config.object_idr);
-	idr_init(&dev->mode_config.tile_idr);
+	idr_init_base(&dev->mode_config.object_idr, 1);
+	idr_init_base(&dev->mode_config.tile_idr, 1);
 	ida_init(&dev->mode_config.connector_ida);
 	spin_lock_init(&dev->mode_config.connector_list_lock);
 
diff --git a/drivers/gpu/drm/drm_modeset_helper.c b/drivers/gpu/drm/drm_modeset_helper.c
index 0f08319453b2..f858dfedf2cf 100644
--- a/drivers/gpu/drm/drm_modeset_helper.c
+++ b/drivers/gpu/drm/drm_modeset_helper.c
@@ -100,45 +100,16 @@ EXPORT_SYMBOL(drm_helper_mode_fill_fb_struct);
  * This is the minimal list of formats that seem to be safe for modeset use
  * with all current DRM drivers.  Most hardware can actually support more
  * formats than this and drivers may specify a more accurate list when
- * creating the primary plane.  However drivers that still call
- * drm_plane_init() will use this minimal format list as the default.
+ * creating the primary plane.
  */
 static const uint32_t safe_modeset_formats[] = {
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_ARGB8888,
 };
 
-static struct drm_plane *create_primary_plane(struct drm_device *dev)
-{
-	struct drm_plane *primary;
-	int ret;
-
-	primary = kzalloc(sizeof(*primary), GFP_KERNEL);
-	if (primary == NULL) {
-		DRM_DEBUG_KMS("Failed to allocate primary plane\n");
-		return NULL;
-	}
-
-	/*
-	 * Remove the format_default field from drm_plane when dropping
-	 * this helper.
-	 */
-	primary->format_default = true;
-
-	/* possible_crtc's will be filled in later by crtc_init */
-	ret = drm_universal_plane_init(dev, primary, 0,
-				       &drm_primary_helper_funcs,
-				       safe_modeset_formats,
-				       ARRAY_SIZE(safe_modeset_formats),
-				       NULL,
-				       DRM_PLANE_TYPE_PRIMARY, NULL);
-	if (ret) {
-		kfree(primary);
-		primary = NULL;
-	}
-
-	return primary;
-}
+static const struct drm_plane_funcs primary_plane_funcs = {
+	DRM_PLANE_NON_ATOMIC_FUNCS,
+};
 
 /**
  * drm_crtc_init - Legacy CRTC initialization function
@@ -171,10 +142,33 @@ int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
 		  const struct drm_crtc_funcs *funcs)
 {
 	struct drm_plane *primary;
+	int ret;
+
+	/* possible_crtc's will be filled in later by crtc_init */
+	primary = __drm_universal_plane_alloc(dev, sizeof(*primary), 0, 0,
+					      &primary_plane_funcs,
+					      safe_modeset_formats,
+					      ARRAY_SIZE(safe_modeset_formats),
+					      NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (IS_ERR(primary))
+		return PTR_ERR(primary);
 
-	primary = create_primary_plane(dev);
-	return drm_crtc_init_with_planes(dev, crtc, primary, NULL, funcs,
-					 NULL);
+	/*
+	 * Remove the format_default field from drm_plane when dropping
+	 * this helper.
+	 */
+	primary->format_default = true;
+
+	ret = drm_crtc_init_with_planes(dev, crtc, primary, NULL, funcs, NULL);
+	if (ret)
+		goto err_drm_plane_cleanup;
+
+	return 0;
+
+err_drm_plane_cleanup:
+	drm_plane_cleanup(primary);
+	kfree(primary);
+	return ret;
 }
 EXPORT_SYMBOL(drm_crtc_init);
 
diff --git a/drivers/gpu/drm/drm_panel_orientation_quirks.c b/drivers/gpu/drm/drm_panel_orientation_quirks.c
index fc1728d46ac2..8a0c0e0bb5bd 100644
--- a/drivers/gpu/drm/drm_panel_orientation_quirks.c
+++ b/drivers/gpu/drm/drm_panel_orientation_quirks.c
@@ -103,6 +103,12 @@ static const struct drm_dmi_panel_orientation_data lcd800x1280_rightside_up = {
 	.orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP,
 };
 
+static const struct drm_dmi_panel_orientation_data lcd1080x1920_leftside_up = {
+	.width = 1080,
+	.height = 1920,
+	.orientation = DRM_MODE_PANEL_ORIENTATION_LEFT_UP,
+};
+
 static const struct drm_dmi_panel_orientation_data lcd1200x1920_rightside_up = {
 	.width = 1200,
 	.height = 1920,
@@ -128,6 +134,12 @@ static const struct dmi_system_id orientation_data[] = {
 		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "One S1003"),
 		},
 		.driver_data = (void *)&lcd800x1280_rightside_up,
+	}, {	/* Anbernic Win600 */
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "Anbernic"),
+		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Win600"),
+		},
+		.driver_data = (void *)&lcd720x1280_rightside_up,
 	}, {	/* Asus T100HA */
 		.matches = {
 		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
@@ -152,6 +164,12 @@ static const struct dmi_system_id orientation_data[] = {
 		  DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "AYA NEO 2021"),
 		},
 		.driver_data = (void *)&lcd800x1280_rightside_up,
+	}, {	/* AYA NEO AIR */
+		.matches = {
+		  DMI_EXACT_MATCH(DMI_SYS_VENDOR, "AYANEO"),
+		  DMI_MATCH(DMI_BOARD_NAME, "AIR"),
+		},
+		.driver_data = (void *)&lcd1080x1920_leftside_up,
 	}, {	/* AYA NEO NEXT */
 		.matches = {
 		  DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "AYANEO"),
diff --git a/drivers/gpu/drm/drm_plane.c b/drivers/gpu/drm/drm_plane.c
index 726f2f163c26..33357629a7f5 100644
--- a/drivers/gpu/drm/drm_plane.c
+++ b/drivers/gpu/drm/drm_plane.c
@@ -448,6 +448,44 @@ void *__drmm_universal_plane_alloc(struct drm_device *dev, size_t size,
 }
 EXPORT_SYMBOL(__drmm_universal_plane_alloc);
 
+void *__drm_universal_plane_alloc(struct drm_device *dev, size_t size,
+				  size_t offset, uint32_t possible_crtcs,
+				  const struct drm_plane_funcs *funcs,
+				  const uint32_t *formats, unsigned int format_count,
+				  const uint64_t *format_modifiers,
+				  enum drm_plane_type type,
+				  const char *name, ...)
+{
+	void *container;
+	struct drm_plane *plane;
+	va_list ap;
+	int ret;
+
+	if (drm_WARN_ON(dev, !funcs))
+		return ERR_PTR(-EINVAL);
+
+	container = kzalloc(size, GFP_KERNEL);
+	if (!container)
+		return ERR_PTR(-ENOMEM);
+
+	plane = container + offset;
+
+	va_start(ap, name);
+	ret = __drm_universal_plane_init(dev, plane, possible_crtcs, funcs,
+					 formats, format_count, format_modifiers,
+					 type, name, ap);
+	va_end(ap);
+	if (ret)
+		goto err_kfree;
+
+	return container;
+
+err_kfree:
+	kfree(container);
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL(__drm_universal_plane_alloc);
+
 int drm_plane_register_all(struct drm_device *dev)
 {
 	unsigned int num_planes = 0;
@@ -483,38 +521,6 @@ void drm_plane_unregister_all(struct drm_device *dev)
 }
 
 /**
- * drm_plane_init - Initialize a legacy plane
- * @dev: DRM device
- * @plane: plane object to init
- * @possible_crtcs: bitmask of possible CRTCs
- * @funcs: callbacks for the new plane
- * @formats: array of supported formats (DRM_FORMAT\_\*)
- * @format_count: number of elements in @formats
- * @is_primary: plane type (primary vs overlay)
- *
- * Legacy API to initialize a DRM plane.
- *
- * New drivers should call drm_universal_plane_init() instead.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
-		   uint32_t possible_crtcs,
-		   const struct drm_plane_funcs *funcs,
-		   const uint32_t *formats, unsigned int format_count,
-		   bool is_primary)
-{
-	enum drm_plane_type type;
-
-	type = is_primary ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
-	return drm_universal_plane_init(dev, plane, possible_crtcs, funcs,
-					formats, format_count,
-					NULL, type, NULL);
-}
-EXPORT_SYMBOL(drm_plane_init);
-
-/**
  * drm_plane_cleanup - Clean up the core plane usage
  * @plane: plane to cleanup
  *
diff --git a/drivers/gpu/drm/drm_plane_helper.c b/drivers/gpu/drm/drm_plane_helper.c
index 838b32b70bce..865bd999b187 100644
--- a/drivers/gpu/drm/drm_plane_helper.c
+++ b/drivers/gpu/drm/drm_plane_helper.c
@@ -30,8 +30,10 @@
 #include <drm/drm_atomic_uapi.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_device.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_plane_helper.h>
+#include <drm/drm_print.h>
 #include <drm/drm_rect.h>
 
 #define SUBPIXEL_MASK 0xffff
@@ -145,13 +147,36 @@ static int drm_plane_helper_check_update(struct drm_plane *plane,
 	return 0;
 }
 
-static int drm_primary_helper_update(struct drm_plane *plane, struct drm_crtc *crtc,
-				     struct drm_framebuffer *fb,
-				     int crtc_x, int crtc_y,
-				     unsigned int crtc_w, unsigned int crtc_h,
-				     uint32_t src_x, uint32_t src_y,
-				     uint32_t src_w, uint32_t src_h,
-				     struct drm_modeset_acquire_ctx *ctx)
+/**
+ * drm_plane_helper_update_primary - Helper for updating primary planes
+ * @plane: plane to update
+ * @crtc: the plane's new CRTC
+ * @fb: the plane's new framebuffer
+ * @crtc_x: x coordinate within CRTC
+ * @crtc_y: y coordinate within CRTC
+ * @crtc_w: width coordinate within CRTC
+ * @crtc_h: height coordinate within CRTC
+ * @src_x: x coordinate within source
+ * @src_y: y coordinate within source
+ * @src_w: width coordinate within source
+ * @src_h: height coordinate within source
+ * @ctx: modeset locking context
+ *
+ * This helper validates the given parameters and updates the primary plane.
+ *
+ * This function is only useful for non-atomic modesetting. Don't use
+ * it in new drivers.
+ *
+ * Returns:
+ * Zero on success, or an errno code otherwise.
+ */
+int drm_plane_helper_update_primary(struct drm_plane *plane, struct drm_crtc *crtc,
+				    struct drm_framebuffer *fb,
+				    int crtc_x, int crtc_y,
+				    unsigned int crtc_w, unsigned int crtc_h,
+				    uint32_t src_x, uint32_t src_y,
+				    uint32_t src_w, uint32_t src_h,
+				    struct drm_modeset_acquire_ctx *ctx)
 {
 	struct drm_mode_set set = {
 		.crtc = crtc,
@@ -172,15 +197,19 @@ static int drm_primary_helper_update(struct drm_plane *plane, struct drm_crtc *c
 		.x2 = crtc_x + crtc_w,
 		.y2 = crtc_y + crtc_h,
 	};
+	struct drm_device *dev = plane->dev;
 	struct drm_connector **connector_list;
 	int num_connectors, ret;
 	bool visible;
 
+	if (drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev)))
+		return -EINVAL;
+
 	ret = drm_plane_helper_check_update(plane, crtc, fb,
 					    &src, &dest,
 					    DRM_MODE_ROTATE_0,
-					    DRM_PLANE_HELPER_NO_SCALING,
-					    DRM_PLANE_HELPER_NO_SCALING,
+					    DRM_PLANE_NO_SCALING,
+					    DRM_PLANE_NO_SCALING,
 					    false, false, &visible);
 	if (ret)
 		return ret;
@@ -218,31 +247,74 @@ static int drm_primary_helper_update(struct drm_plane *plane, struct drm_crtc *c
 	kfree(connector_list);
 	return ret;
 }
+EXPORT_SYMBOL(drm_plane_helper_update_primary);
 
-static int drm_primary_helper_disable(struct drm_plane *plane,
-				      struct drm_modeset_acquire_ctx *ctx)
+/**
+ * drm_plane_helper_disable_primary - Helper for disabling primary planes
+ * @plane: plane to disable
+ * @ctx: modeset locking context
+ *
+ * This helper returns an error when trying to disable the primary
+ * plane.
+ *
+ * This function is only useful for non-atomic modesetting. Don't use
+ * it in new drivers.
+ *
+ * Returns:
+ * An errno code.
+ */
+int drm_plane_helper_disable_primary(struct drm_plane *plane,
+				     struct drm_modeset_acquire_ctx *ctx)
 {
+	struct drm_device *dev = plane->dev;
+
+	drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));
+
 	return -EINVAL;
 }
+EXPORT_SYMBOL(drm_plane_helper_disable_primary);
 
 /**
- * drm_primary_helper_destroy() - Helper for primary plane destruction
+ * drm_plane_helper_destroy() - Helper for primary plane destruction
  * @plane: plane to destroy
  *
  * Provides a default plane destroy handler for primary planes.  This handler
  * is called during CRTC destruction.  We disable the primary plane, remove
  * it from the DRM plane list, and deallocate the plane structure.
  */
-void drm_primary_helper_destroy(struct drm_plane *plane)
+void drm_plane_helper_destroy(struct drm_plane *plane)
 {
 	drm_plane_cleanup(plane);
 	kfree(plane);
 }
-EXPORT_SYMBOL(drm_primary_helper_destroy);
-
-const struct drm_plane_funcs drm_primary_helper_funcs = {
-	.update_plane = drm_primary_helper_update,
-	.disable_plane = drm_primary_helper_disable,
-	.destroy = drm_primary_helper_destroy,
-};
-EXPORT_SYMBOL(drm_primary_helper_funcs);
+EXPORT_SYMBOL(drm_plane_helper_destroy);
+
+/**
+ * drm_plane_helper_atomic_check() - Helper to check plane atomic-state
+ * @plane: plane to check
+ * @state: atomic state object
+ *
+ * Provides a default plane-state check handler for planes whose atomic-state
+ * scale and positioning are not expected to change since the plane is always
+ * a fullscreen scanout buffer.
+ *
+ * This is often the case for the primary plane of simple framebuffers.
+ *
+ * RETURNS:
+ * Zero on success, or an errno code otherwise.
+ */
+int drm_plane_helper_atomic_check(struct drm_plane *plane, struct drm_atomic_state *state)
+{
+	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_crtc *new_crtc = new_plane_state->crtc;
+	struct drm_crtc_state *new_crtc_state = NULL;
+
+	if (new_crtc)
+		new_crtc_state = drm_atomic_get_new_crtc_state(state, new_crtc);
+
+	return drm_atomic_helper_check_plane_state(new_plane_state, new_crtc_state,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   false, false);
+}
+EXPORT_SYMBOL(drm_plane_helper_atomic_check);
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index bb427c5a4f1f..69b0b2b9cc1c 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -1015,6 +1015,30 @@ bool drm_helper_hpd_irq_event(struct drm_device *dev)
 EXPORT_SYMBOL(drm_helper_hpd_irq_event);
 
 /**
+ * drm_crtc_helper_mode_valid_fixed - Validates a display mode
+ * @crtc: the crtc
+ * @mode: the mode to validate
+ * @fixed_mode: the display hardware's mode
+ *
+ * Returns:
+ * MODE_OK on success, or another mode-status code otherwise.
+ */
+enum drm_mode_status drm_crtc_helper_mode_valid_fixed(struct drm_crtc *crtc,
+						      const struct drm_display_mode *mode,
+						      const struct drm_display_mode *fixed_mode)
+{
+	if (mode->hdisplay != fixed_mode->hdisplay && mode->vdisplay != fixed_mode->vdisplay)
+		return MODE_ONE_SIZE;
+	else if (mode->hdisplay != fixed_mode->hdisplay)
+		return MODE_ONE_WIDTH;
+	else if (mode->vdisplay != fixed_mode->vdisplay)
+		return MODE_ONE_HEIGHT;
+
+	return MODE_OK;
+}
+EXPORT_SYMBOL(drm_crtc_helper_mode_valid_fixed);
+
+/**
  * drm_connector_helper_get_modes_from_ddc - Updates the connector's EDID
  *                                           property from the connector's
  *                                           DDC channel
@@ -1051,6 +1075,46 @@ int drm_connector_helper_get_modes_from_ddc(struct drm_connector *connector)
 EXPORT_SYMBOL(drm_connector_helper_get_modes_from_ddc);
 
 /**
+ * drm_connector_helper_get_modes_fixed - Duplicates a display mode for a connector
+ * @connector: the connector
+ * @fixed_mode: the display hardware's mode
+ *
+ * This function duplicates a display modes for a connector. Drivers for hardware
+ * that only supports a single fixed mode can use this function in their connector's
+ * get_modes helper.
+ *
+ * Returns:
+ * The number of created modes.
+ */
+int drm_connector_helper_get_modes_fixed(struct drm_connector *connector,
+					 const struct drm_display_mode *fixed_mode)
+{
+	struct drm_device *dev = connector->dev;
+	struct drm_display_mode *mode;
+
+	mode = drm_mode_duplicate(dev, fixed_mode);
+	if (!mode) {
+		drm_err(dev, "Failed to duplicate mode " DRM_MODE_FMT "\n",
+			DRM_MODE_ARG(fixed_mode));
+		return 0;
+	}
+
+	if (mode->name[0] == '\0')
+		drm_mode_set_name(mode);
+
+	mode->type |= DRM_MODE_TYPE_PREFERRED;
+	drm_mode_probed_add(connector, mode);
+
+	if (mode->width_mm)
+		connector->display_info.width_mm = mode->width_mm;
+	if (mode->height_mm)
+		connector->display_info.height_mm = mode->height_mm;
+
+	return 1;
+}
+EXPORT_SYMBOL(drm_connector_helper_get_modes_fixed);
+
+/**
  * drm_connector_helper_get_modes - Read EDID and update connector.
  * @connector: The connector
  *
diff --git a/drivers/gpu/drm/drm_simple_kms_helper.c b/drivers/gpu/drm/drm_simple_kms_helper.c
index 36633590ebf3..e9f782119d3d 100644
--- a/drivers/gpu/drm/drm_simple_kms_helper.c
+++ b/drivers/gpu/drm/drm_simple_kms_helper.c
@@ -12,7 +12,6 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_managed.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
 
@@ -223,8 +222,8 @@ static int drm_simple_kms_plane_atomic_check(struct drm_plane *plane,
 						   &pipe->crtc);
 
 	ret = drm_atomic_helper_check_plane_state(plane_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, false);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index 66e5f1e34044..7c3aa77186d3 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -9,7 +9,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/exynos_drm.h>
 
 #include "exynos_drm_crtc.h"
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index 10b0036f8a2e..b7c11bdce2c8 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -893,7 +893,7 @@ static int hdmi_get_modes(struct drm_connector *connector)
 	if (!edid)
 		return -ENODEV;
 
-	hdata->dvi_mode = !drm_detect_hdmi_monitor(edid);
+	hdata->dvi_mode = !connector->display_info.is_hdmi;
 	DRM_DEV_DEBUG_KMS(hdata->dev, "%s : width[%d] x height[%d]\n",
 			  (hdata->dvi_mode ? "dvi monitor" : "hdmi monitor"),
 			  edid->width_cm, edid->height_cm);
@@ -922,8 +922,8 @@ static int hdmi_find_phy_conf(struct hdmi_context *hdata, u32 pixel_clock)
 	return -EINVAL;
 }
 
-static int hdmi_mode_valid(struct drm_connector *connector,
-			struct drm_display_mode *mode)
+static enum drm_mode_status hdmi_mode_valid(struct drm_connector *connector,
+					    struct drm_display_mode *mode)
 {
 	struct hdmi_context *hdata = connector_to_hdmi(connector);
 	int ret;
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c
index 65260a658684..8d333db813b7 100644
--- a/drivers/gpu/drm/exynos/exynos_mixer.c
+++ b/drivers/gpu/drm/exynos/exynos_mixer.c
@@ -1045,7 +1045,7 @@ static void mixer_atomic_disable(struct exynos_drm_crtc *crtc)
 	clear_bit(MXR_BIT_POWERED, &ctx->flags);
 }
 
-static int mixer_mode_valid(struct exynos_drm_crtc *crtc,
+static enum drm_mode_status mixer_mode_valid(struct exynos_drm_crtc *crtc,
 		const struct drm_display_mode *mode)
 {
 	struct mixer_context *ctx = crtc->ctx;
diff --git a/drivers/gpu/drm/fsl-dcu/Kconfig b/drivers/gpu/drm/fsl-dcu/Kconfig
index e95e96c565ba..5ca71ef87325 100644
--- a/drivers/gpu/drm/fsl-dcu/Kconfig
+++ b/drivers/gpu/drm/fsl-dcu/Kconfig
@@ -3,7 +3,7 @@ config DRM_FSL_DCU
 	tristate "DRM Support for Freescale DCU"
 	depends on DRM && OF && ARM && COMMON_CLK
 	select BACKLIGHT_CLASS_DEVICE
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_KMS_HELPER
 	select DRM_PANEL
 	select REGMAP_MMIO
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
index 7a503bf08d0f..b4acc3422ba4 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
@@ -20,9 +20,8 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_modeset_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_probe_helper.h>
@@ -150,13 +149,13 @@ static void fsl_dcu_unload(struct drm_device *dev)
 	dev->dev_private = NULL;
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(fsl_dcu_drm_fops);
+DEFINE_DRM_GEM_DMA_FOPS(fsl_dcu_drm_fops);
 
 static const struct drm_driver fsl_dcu_drm_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.load			= fsl_dcu_load,
 	.unload			= fsl_dcu_unload,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 	.fops			= &fsl_dcu_drm_fops,
 	.name			= "fsl-dcu-drm",
 	.desc			= "Freescale DCU DRM",
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_kms.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_kms.c
index d763f53f480c..5b47000738e4 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_kms.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_kms.c
@@ -6,7 +6,6 @@
  */
 
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_probe_helper.h>
 
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
index 0cd527f0c146..794a87d16f88 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
@@ -10,10 +10,10 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 
@@ -84,7 +84,7 @@ static void fsl_dcu_drm_plane_atomic_update(struct drm_plane *plane,
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
 	struct drm_framebuffer *fb = plane->state->fb;
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 	unsigned int alpha = DCU_LAYER_AB_NONE, bpp;
 	int index;
 
@@ -95,7 +95,7 @@ static void fsl_dcu_drm_plane_atomic_update(struct drm_plane *plane,
 	if (index < 0)
 		return;
 
-	gem = drm_fb_cma_get_gem_obj(fb, 0);
+	gem = drm_fb_dma_get_gem_obj(fb, 0);
 
 	switch (fb->format->format) {
 	case DRM_FORMAT_RGB565:
@@ -136,7 +136,7 @@ static void fsl_dcu_drm_plane_atomic_update(struct drm_plane *plane,
 		     DCU_LAYER_POSY(new_state->crtc_y) |
 		     DCU_LAYER_POSX(new_state->crtc_x));
 	regmap_write(fsl_dev->regmap,
-		     DCU_CTRLDESCLN(index, 3), gem->paddr);
+		     DCU_CTRLDESCLN(index, 3), gem->dma_addr);
 	regmap_write(fsl_dev->regmap, DCU_CTRLDESCLN(index, 4),
 		     DCU_LAYER_EN |
 		     DCU_LAYER_TRANS(0xff) |
@@ -171,16 +171,10 @@ static const struct drm_plane_helper_funcs fsl_dcu_drm_plane_helper_funcs = {
 	.atomic_update = fsl_dcu_drm_plane_atomic_update,
 };
 
-static void fsl_dcu_drm_plane_destroy(struct drm_plane *plane)
-{
-	drm_plane_cleanup(plane);
-	kfree(plane);
-}
-
 static const struct drm_plane_funcs fsl_dcu_drm_plane_funcs = {
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
-	.destroy = fsl_dcu_drm_plane_destroy,
+	.destroy = drm_plane_helper_destroy,
 	.disable_plane = drm_atomic_helper_disable_plane,
 	.reset = drm_atomic_helper_plane_reset,
 	.update_plane = drm_atomic_helper_update_plane,
diff --git a/drivers/gpu/drm/gma500/Kconfig b/drivers/gpu/drm/gma500/Kconfig
index 0cff20265f97..807b989e3c77 100644
--- a/drivers/gpu/drm/gma500/Kconfig
+++ b/drivers/gpu/drm/gma500/Kconfig
@@ -7,6 +7,8 @@ config DRM_GMA500
 	select ACPI_VIDEO if ACPI
 	select BACKLIGHT_CLASS_DEVICE if ACPI
 	select INPUT if ACPI
+	select X86_PLATFORM_DEVICES if ACPI
+	select ACPI_WMI if ACPI
 	help
 	  Say yes for an experimental 2D KMS framebuffer driver for the
 	  Intel GMA500 (Poulsbo), Intel GMA600 (Moorestown/Oak Trail) and
diff --git a/drivers/gpu/drm/gma500/backlight.c b/drivers/gpu/drm/gma500/backlight.c
index 46b9c0f13d6d..577a4987b193 100644
--- a/drivers/gpu/drm/gma500/backlight.c
+++ b/drivers/gpu/drm/gma500/backlight.c
@@ -7,75 +7,109 @@
  * Authors: Eric Knopp
  */
 
+#include <acpi/video.h>
+
 #include "psb_drv.h"
 #include "psb_intel_reg.h"
 #include "psb_intel_drv.h"
 #include "intel_bios.h"
 #include "power.h"
 
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
-static void do_gma_backlight_set(struct drm_device *dev)
-{
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	backlight_update_status(dev_priv->backlight_device);
-}
-#endif
-
 void gma_backlight_enable(struct drm_device *dev)
 {
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
+
 	dev_priv->backlight_enabled = true;
-	if (dev_priv->backlight_device) {
-		dev_priv->backlight_device->props.brightness = dev_priv->backlight_level;
-		do_gma_backlight_set(dev);
-	}
-#endif	
+	dev_priv->ops->backlight_set(dev, dev_priv->backlight_level);
 }
 
 void gma_backlight_disable(struct drm_device *dev)
 {
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
+
 	dev_priv->backlight_enabled = false;
-	if (dev_priv->backlight_device) {
-		dev_priv->backlight_device->props.brightness = 0;
-		do_gma_backlight_set(dev);
-	}
-#endif
+	dev_priv->ops->backlight_set(dev, 0);
 }
 
 void gma_backlight_set(struct drm_device *dev, int v)
 {
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
+
 	dev_priv->backlight_level = v;
-	if (dev_priv->backlight_device && dev_priv->backlight_enabled) {
-		dev_priv->backlight_device->props.brightness = v;
-		do_gma_backlight_set(dev);
-	}
-#endif
+	if (dev_priv->backlight_enabled)
+		dev_priv->ops->backlight_set(dev, v);
+}
+
+static int gma_backlight_get_brightness(struct backlight_device *bd)
+{
+	struct drm_device *dev = bl_get_data(bd);
+	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
+
+	if (dev_priv->ops->backlight_get)
+		return dev_priv->ops->backlight_get(dev);
+
+	return dev_priv->backlight_level;
 }
 
+static int gma_backlight_update_status(struct backlight_device *bd)
+{
+	struct drm_device *dev = bl_get_data(bd);
+	int level = backlight_get_brightness(bd);
+
+	/* Percentage 1-100% being valid */
+	if (level < 1)
+		level = 1;
+
+	gma_backlight_set(dev, level);
+	return 0;
+}
+
+static const struct backlight_ops gma_backlight_ops __maybe_unused = {
+	.get_brightness = gma_backlight_get_brightness,
+	.update_status  = gma_backlight_update_status,
+};
+
 int gma_backlight_init(struct drm_device *dev)
 {
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
+	struct backlight_properties props __maybe_unused = {};
+	int ret;
+
 	dev_priv->backlight_enabled = true;
-	return dev_priv->ops->backlight_init(dev);
-#else
-	return 0;
+	dev_priv->backlight_level = 100;
+
+	ret = dev_priv->ops->backlight_init(dev);
+	if (ret)
+		return ret;
+
+	if (!acpi_video_backlight_use_native()) {
+		drm_info(dev, "Skipping %s backlight registration\n",
+			 dev_priv->ops->backlight_name);
+		return 0;
+	}
+
+#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
+	props.brightness = dev_priv->backlight_level;
+	props.max_brightness = PSB_MAX_BRIGHTNESS;
+	props.type = BACKLIGHT_RAW;
+
+	dev_priv->backlight_device =
+		backlight_device_register(dev_priv->ops->backlight_name,
+					  dev->dev, dev,
+					  &gma_backlight_ops, &props);
+	if (IS_ERR(dev_priv->backlight_device))
+		return PTR_ERR(dev_priv->backlight_device);
 #endif
+
+	return 0;
 }
 
 void gma_backlight_exit(struct drm_device *dev)
 {
 #ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	if (dev_priv->backlight_device) {
-		dev_priv->backlight_device->props.brightness = 0;
-		backlight_update_status(dev_priv->backlight_device);
+
+	if (dev_priv->backlight_device)
 		backlight_device_unregister(dev_priv->backlight_device);
-	}
 #endif
 }
diff --git a/drivers/gpu/drm/gma500/cdv_device.c b/drivers/gpu/drm/gma500/cdv_device.c
index ce96234f3df2..3065596257e9 100644
--- a/drivers/gpu/drm/gma500/cdv_device.c
+++ b/drivers/gpu/drm/gma500/cdv_device.c
@@ -5,7 +5,6 @@
  *
  **************************************************************************/
 
-#include <linux/backlight.h>
 #include <linux/delay.h>
 
 #include <drm/drm.h>
@@ -62,14 +61,10 @@ static int cdv_output_init(struct drm_device *dev)
 	return 0;
 }
 
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
-
 /*
  *	Cedartrail Backlght Interfaces
  */
 
-static struct backlight_device *cdv_backlight_device;
-
 static int cdv_backlight_combination_mode(struct drm_device *dev)
 {
 	return REG_READ(BLC_PWM_CTL2) & PWM_LEGACY_MODE;
@@ -92,9 +87,8 @@ static u32 cdv_get_max_backlight(struct drm_device *dev)
 	return max;
 }
 
-static int cdv_get_brightness(struct backlight_device *bd)
+static int cdv_get_brightness(struct drm_device *dev)
 {
-	struct drm_device *dev = bl_get_data(bd);
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
 	u32 val = REG_READ(BLC_PWM_CTL) & BACKLIGHT_DUTY_CYCLE_MASK;
 
@@ -106,20 +100,13 @@ static int cdv_get_brightness(struct backlight_device *bd)
 		val *= lbpc;
 	}
 	return (val * 100)/cdv_get_max_backlight(dev);
-
 }
 
-static int cdv_set_brightness(struct backlight_device *bd)
+static void cdv_set_brightness(struct drm_device *dev, int level)
 {
-	struct drm_device *dev = bl_get_data(bd);
 	struct pci_dev *pdev = to_pci_dev(dev->dev);
-	int level = bd->props.brightness;
 	u32 blc_pwm_ctl;
 
-	/* Percentage 1-100% being valid */
-	if (level < 1)
-		level = 1;
-
 	level *= cdv_get_max_backlight(dev);
 	level /= 100;
 
@@ -136,38 +123,18 @@ static int cdv_set_brightness(struct backlight_device *bd)
 	blc_pwm_ctl = REG_READ(BLC_PWM_CTL) & ~BACKLIGHT_DUTY_CYCLE_MASK;
 	REG_WRITE(BLC_PWM_CTL, (blc_pwm_ctl |
 				(level << BACKLIGHT_DUTY_CYCLE_SHIFT)));
-	return 0;
 }
 
-static const struct backlight_ops cdv_ops = {
-	.get_brightness = cdv_get_brightness,
-	.update_status  = cdv_set_brightness,
-};
-
 static int cdv_backlight_init(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	struct backlight_properties props;
-
-	memset(&props, 0, sizeof(struct backlight_properties));
-	props.max_brightness = 100;
-	props.type = BACKLIGHT_PLATFORM;
-
-	cdv_backlight_device = backlight_device_register("psb-bl",
-					NULL, (void *)dev, &cdv_ops, &props);
-	if (IS_ERR(cdv_backlight_device))
-		return PTR_ERR(cdv_backlight_device);
-
-	cdv_backlight_device->props.brightness =
-			cdv_get_brightness(cdv_backlight_device);
-	backlight_update_status(cdv_backlight_device);
-	dev_priv->backlight_device = cdv_backlight_device;
-	dev_priv->backlight_enabled = true;
+
+	dev_priv->backlight_level = cdv_get_brightness(dev);
+	cdv_set_brightness(dev, dev_priv->backlight_level);
+
 	return 0;
 }
 
-#endif
-
 /*
  *	Provide the Cedarview specific chip logic and low level methods
  *	for power management
@@ -613,9 +580,10 @@ const struct psb_ops cdv_chip_ops = {
 	.hotplug = cdv_hotplug_event,
 	.hotplug_enable = cdv_hotplug_enable,
 
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
 	.backlight_init = cdv_backlight_init,
-#endif
+	.backlight_get = cdv_get_brightness,
+	.backlight_set = cdv_set_brightness,
+	.backlight_name = "psb-bl",
 
 	.init_pm = cdv_init_pm,
 	.save_regs = cdv_save_display_registers,
diff --git a/drivers/gpu/drm/gma500/cdv_intel_dp.c b/drivers/gpu/drm/gma500/cdv_intel_dp.c
index bb2e9d64018a..53b967282d6a 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_dp.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_dp.c
@@ -115,7 +115,7 @@ i2c_algo_dp_aux_stop(struct i2c_adapter *adapter, bool reading)
 
 /*
  * Write a single byte to the current I2C address, the
- * the I2C link must be running or this returns -EIO
+ * I2C link must be running or this returns -EIO
  */
 static int
 i2c_algo_dp_aux_put_byte(struct i2c_adapter *adapter, u8 byte)
diff --git a/drivers/gpu/drm/gma500/gma_display.c b/drivers/gpu/drm/gma500/gma_display.c
index 2f52eceda3a1..fe7b8436f87a 100644
--- a/drivers/gpu/drm/gma500/gma_display.c
+++ b/drivers/gpu/drm/gma500/gma_display.c
@@ -555,28 +555,11 @@ int gma_crtc_page_flip(struct drm_crtc *crtc,
 	return ret;
 }
 
-int gma_crtc_set_config(struct drm_mode_set *set,
-			struct drm_modeset_acquire_ctx *ctx)
-{
-	struct drm_device *dev = set->crtc->dev;
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	int ret;
-
-	if (!dev_priv->rpm_enabled)
-		return drm_crtc_helper_set_config(set, ctx);
-
-	pm_runtime_forbid(dev->dev);
-	ret = drm_crtc_helper_set_config(set, ctx);
-	pm_runtime_allow(dev->dev);
-
-	return ret;
-}
-
 const struct drm_crtc_funcs gma_crtc_funcs = {
 	.cursor_set = gma_crtc_cursor_set,
 	.cursor_move = gma_crtc_cursor_move,
 	.gamma_set = gma_crtc_gamma_set,
-	.set_config = gma_crtc_set_config,
+	.set_config = drm_crtc_helper_set_config,
 	.destroy = gma_crtc_destroy,
 	.page_flip = gma_crtc_page_flip,
 	.enable_vblank = gma_crtc_enable_vblank,
diff --git a/drivers/gpu/drm/gma500/gma_display.h b/drivers/gpu/drm/gma500/gma_display.h
index 113cf048105e..c8b611a2f6c6 100644
--- a/drivers/gpu/drm/gma500/gma_display.h
+++ b/drivers/gpu/drm/gma500/gma_display.h
@@ -69,8 +69,6 @@ extern int gma_crtc_page_flip(struct drm_crtc *crtc,
 			      struct drm_pending_vblank_event *event,
 			      uint32_t page_flip_flags,
 			      struct drm_modeset_acquire_ctx *ctx);
-extern int gma_crtc_set_config(struct drm_mode_set *set,
-			       struct drm_modeset_acquire_ctx *ctx);
 
 extern void gma_crtc_save(struct drm_crtc *crtc);
 extern void gma_crtc_restore(struct drm_crtc *crtc);
diff --git a/drivers/gpu/drm/gma500/oaktrail_crtc.c b/drivers/gpu/drm/gma500/oaktrail_crtc.c
index 6004390d647a..64761f46b434 100644
--- a/drivers/gpu/drm/gma500/oaktrail_crtc.c
+++ b/drivers/gpu/drm/gma500/oaktrail_crtc.c
@@ -310,7 +310,7 @@ static void oaktrail_crtc_dpms(struct drm_crtc *crtc, int mode)
 						   temp & ~PIPEACONF_ENABLE, i);
 				REG_READ_WITH_AUX(map->conf, i);
 			}
-			/* Wait for for the pipe disable to take effect. */
+			/* Wait for the pipe disable to take effect. */
 			gma_wait_for_vblank(dev);
 
 			temp = REG_READ_WITH_AUX(map->dpll, i);
diff --git a/drivers/gpu/drm/gma500/oaktrail_device.c b/drivers/gpu/drm/gma500/oaktrail_device.c
index f90e628cb482..2531959d3d77 100644
--- a/drivers/gpu/drm/gma500/oaktrail_device.c
+++ b/drivers/gpu/drm/gma500/oaktrail_device.c
@@ -5,7 +5,6 @@
  *
  **************************************************************************/
 
-#include <linux/backlight.h>
 #include <linux/delay.h>
 #include <linux/dmi.h>
 #include <linux/module.h>
@@ -37,29 +36,18 @@ static int oaktrail_output_init(struct drm_device *dev)
  *	Provide the low level interfaces for the Moorestown backlight
  */
 
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
-
 #define MRST_BLC_MAX_PWM_REG_FREQ	    0xFFFF
 #define BLC_PWM_PRECISION_FACTOR 100	/* 10000000 */
 #define BLC_PWM_FREQ_CALC_CONSTANT 32
 #define MHz 1000000
 #define BLC_ADJUSTMENT_MAX 100
 
-static struct backlight_device *oaktrail_backlight_device;
-static int oaktrail_brightness;
-
-static int oaktrail_set_brightness(struct backlight_device *bd)
+static void oaktrail_set_brightness(struct drm_device *dev, int level)
 {
-	struct drm_device *dev = bl_get_data(oaktrail_backlight_device);
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	int level = bd->props.brightness;
 	u32 blc_pwm_ctl;
 	u32 max_pwm_blc;
 
-	/* Percentage 1-100% being valid */
-	if (level < 1)
-		level = 1;
-
 	if (gma_power_begin(dev, 0)) {
 		/* Calculate and set the brightness value */
 		max_pwm_blc = REG_READ(BLC_PWM_CTL) >> 16;
@@ -82,19 +70,9 @@ static int oaktrail_set_brightness(struct backlight_device *bd)
 		REG_WRITE(BLC_PWM_CTL, (max_pwm_blc << 16) | blc_pwm_ctl);
 		gma_power_end(dev);
 	}
-	oaktrail_brightness = level;
-	return 0;
-}
-
-static int oaktrail_get_brightness(struct backlight_device *bd)
-{
-	/* return locally cached var instead of HW read (due to DPST etc.) */
-	/* FIXME: ideally return actual value in case firmware fiddled with
-	   it */
-	return oaktrail_brightness;
 }
 
-static int device_backlight_init(struct drm_device *dev)
+static int oaktrail_backlight_init(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
 	unsigned long core_clock;
@@ -123,44 +101,11 @@ static int device_backlight_init(struct drm_device *dev)
 		REG_WRITE(BLC_PWM_CTL, value | (value << 16));
 		gma_power_end(dev);
 	}
-	return 0;
-}
-
-static const struct backlight_ops oaktrail_ops = {
-	.get_brightness = oaktrail_get_brightness,
-	.update_status  = oaktrail_set_brightness,
-};
 
-static int oaktrail_backlight_init(struct drm_device *dev)
-{
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	int ret;
-	struct backlight_properties props;
-
-	memset(&props, 0, sizeof(struct backlight_properties));
-	props.max_brightness = 100;
-	props.type = BACKLIGHT_PLATFORM;
-
-	oaktrail_backlight_device = backlight_device_register("oaktrail-bl",
-				NULL, (void *)dev, &oaktrail_ops, &props);
-
-	if (IS_ERR(oaktrail_backlight_device))
-		return PTR_ERR(oaktrail_backlight_device);
-
-	ret = device_backlight_init(dev);
-	if (ret < 0) {
-		backlight_device_unregister(oaktrail_backlight_device);
-		return ret;
-	}
-	oaktrail_backlight_device->props.brightness = 100;
-	oaktrail_backlight_device->props.max_brightness = 100;
-	backlight_update_status(oaktrail_backlight_device);
-	dev_priv->backlight_device = oaktrail_backlight_device;
+	oaktrail_set_brightness(dev, PSB_MAX_BRIGHTNESS);
 	return 0;
 }
 
-#endif
-
 /*
  *	Provide the Moorestown specific chip logic and low level methods
  *	for power management
@@ -545,9 +490,9 @@ const struct psb_ops oaktrail_chip_ops = {
 
 	.output_init = oaktrail_output_init,
 
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
 	.backlight_init = oaktrail_backlight_init,
-#endif
+	.backlight_set = oaktrail_set_brightness,
+	.backlight_name = "oaktrail-bl",
 
 	.save_regs = oaktrail_save_display_registers,
 	.restore_regs = oaktrail_restore_display_registers,
diff --git a/drivers/gpu/drm/gma500/oaktrail_lvds.c b/drivers/gpu/drm/gma500/oaktrail_lvds.c
index 4d98df189e10..75b4eb1c8884 100644
--- a/drivers/gpu/drm/gma500/oaktrail_lvds.c
+++ b/drivers/gpu/drm/gma500/oaktrail_lvds.c
@@ -61,7 +61,6 @@ static void oaktrail_lvds_set_power(struct drm_device *dev,
 			pp_status = REG_READ(PP_STATUS);
 		} while (pp_status & PP_ON);
 		dev_priv->is_lvds_on = false;
-		pm_request_idle(dev->dev);
 	}
 	gma_power_end(dev);
 }
diff --git a/drivers/gpu/drm/gma500/opregion.c b/drivers/gpu/drm/gma500/opregion.c
index dc494df71a48..0c271072af63 100644
--- a/drivers/gpu/drm/gma500/opregion.c
+++ b/drivers/gpu/drm/gma500/opregion.c
@@ -150,21 +150,17 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
 {
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
 	struct opregion_asle *asle = dev_priv->opregion.asle;
-	struct backlight_device *bd = dev_priv->backlight_device;
 
 	DRM_DEBUG_DRIVER("asle set backlight %x\n", bclp);
 
 	if (!(bclp & ASLE_BCLP_VALID))
 		return ASLE_BACKLIGHT_FAILED;
 
-	if (bd == NULL)
-		return ASLE_BACKLIGHT_FAILED;
-
 	bclp &= ASLE_BCLP_MSK;
 	if (bclp > 255)
 		return ASLE_BACKLIGHT_FAILED;
 
-	gma_backlight_set(dev, bclp * bd->props.max_brightness / 255);
+	gma_backlight_set(dev, bclp * PSB_MAX_BRIGHTNESS / 255);
 
 	asle->cblv = (bclp * 0x64) / 0xff | ASLE_CBLV_VALID;
 
diff --git a/drivers/gpu/drm/gma500/power.c b/drivers/gpu/drm/gma500/power.c
index 66873085d450..186af29bea6f 100644
--- a/drivers/gpu/drm/gma500/power.c
+++ b/drivers/gpu/drm/gma500/power.c
@@ -37,9 +37,6 @@
 #include <linux/mutex.h>
 #include <linux/pm_runtime.h>
 
-static struct mutex power_mutex;	/* Serialize power ops */
-static DEFINE_SPINLOCK(power_ctrl_lock);	/* Serialize power claim */
-
 /**
  *	gma_power_init		-	initialise power manager
  *	@dev: our device
@@ -54,13 +51,23 @@ void gma_power_init(struct drm_device *dev)
 	dev_priv->apm_base = dev_priv->apm_reg & 0xffff;
 	dev_priv->ospm_base &= 0xffff;
 
-	dev_priv->display_power = true;	/* We start active */
-	dev_priv->display_count = 0;	/* Currently no users */
-	dev_priv->suspended = false;	/* And not suspended */
-	mutex_init(&power_mutex);
-
 	if (dev_priv->ops->init_pm)
 		dev_priv->ops->init_pm(dev);
+
+	/*
+	 * Runtime pm support is broken atm. So for now unconditionally
+	 * call pm_runtime_get() here and put it again in psb_driver_unload()
+	 *
+	 * To fix this we need to call pm_runtime_get() once for each active
+	 * pipe at boot and then put() / get() for each pipe disable / enable
+	 * so that the device gets runtime suspended when no pipes are active.
+	 * Once this is in place the pm_runtime_get() below should be replaced
+	 * by a pm_runtime_allow() call to undo the pm_runtime_forbid() from
+	 * pci_pm_init().
+	 */
+	pm_runtime_get(dev->dev);
+
+	dev_priv->pm_initialized = true;
 }
 
 /**
@@ -71,8 +78,12 @@ void gma_power_init(struct drm_device *dev)
  */
 void gma_power_uninit(struct drm_device *dev)
 {
-	pm_runtime_disable(dev->dev);
-	pm_runtime_set_suspended(dev->dev);
+	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
+
+	if (!dev_priv->pm_initialized)
+		return;
+
+	pm_runtime_put_noidle(dev->dev);
 }
 
 /**
@@ -85,11 +96,8 @@ static void gma_suspend_display(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
 
-	if (dev_priv->suspended)
-		return;
 	dev_priv->ops->save_regs(dev);
 	dev_priv->ops->power_down(dev);
-	dev_priv->display_power = false;
 }
 
 /**
@@ -106,8 +114,6 @@ static void gma_resume_display(struct pci_dev *pdev)
 
 	/* turn on the display power island */
 	dev_priv->ops->power_up(dev);
-	dev_priv->suspended = false;
-	dev_priv->display_power = true;
 
 	PSB_WVDC32(dev_priv->pge_ctl | _PSB_PGETBL_ENABLED, PSB_PGETBL_CTL);
 	pci_write_config_word(pdev, PSB_GMCH_CTRL,
@@ -131,9 +137,6 @@ static void gma_suspend_pci(struct pci_dev *pdev)
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
 	int bsm, vbt;
 
-	if (dev_priv->suspended)
-		return;
-
 	pci_save_state(pdev);
 	pci_read_config_dword(pdev, 0x5C, &bsm);
 	dev_priv->regs.saveBSM = bsm;
@@ -142,8 +145,6 @@ static void gma_suspend_pci(struct pci_dev *pdev)
 
 	pci_disable_device(pdev);
 	pci_set_power_state(pdev, PCI_D3hot);
-
-	dev_priv->suspended = true;
 }
 
 /**
@@ -153,26 +154,17 @@ static void gma_suspend_pci(struct pci_dev *pdev)
  *	Perform the resume processing on our PCI device state - rewrite
  *	register state and re-enable the PCI device
  */
-static bool gma_resume_pci(struct pci_dev *pdev)
+static int gma_resume_pci(struct pci_dev *pdev)
 {
 	struct drm_device *dev = pci_get_drvdata(pdev);
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	int ret;
-
-	if (!dev_priv->suspended)
-		return true;
 
 	pci_set_power_state(pdev, PCI_D0);
 	pci_restore_state(pdev);
 	pci_write_config_dword(pdev, 0x5c, dev_priv->regs.saveBSM);
 	pci_write_config_dword(pdev, 0xFC, dev_priv->regs.saveVBT);
-	ret = pci_enable_device(pdev);
 
-	if (ret != 0)
-		dev_err(&pdev->dev, "pci_enable failed: %d\n", ret);
-	else
-		dev_priv->suspended = false;
-	return !dev_priv->suspended;
+	return pci_enable_device(pdev);
 }
 
 /**
@@ -187,20 +179,10 @@ int gma_power_suspend(struct device *_dev)
 {
 	struct pci_dev *pdev = to_pci_dev(_dev);
 	struct drm_device *dev = pci_get_drvdata(pdev);
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
 
-	mutex_lock(&power_mutex);
-	if (!dev_priv->suspended) {
-		if (dev_priv->display_count) {
-			mutex_unlock(&power_mutex);
-			dev_err(dev->dev, "GPU hardware busy, cannot suspend\n");
-			return -EBUSY;
-		}
-		gma_irq_uninstall(dev);
-		gma_suspend_display(dev);
-		gma_suspend_pci(pdev);
-	}
-	mutex_unlock(&power_mutex);
+	gma_irq_uninstall(dev);
+	gma_suspend_display(dev);
+	gma_suspend_pci(pdev);
 	return 0;
 }
 
@@ -215,27 +197,13 @@ int gma_power_resume(struct device *_dev)
 	struct pci_dev *pdev = to_pci_dev(_dev);
 	struct drm_device *dev = pci_get_drvdata(pdev);
 
-	mutex_lock(&power_mutex);
 	gma_resume_pci(pdev);
 	gma_resume_display(pdev);
 	gma_irq_install(dev);
-	mutex_unlock(&power_mutex);
 	return 0;
 }
 
 /**
- *	gma_power_is_on		-	returne true if power is on
- *	@dev: our DRM device
- *
- *	Returns true if the display island power is on at this moment
- */
-bool gma_power_is_on(struct drm_device *dev)
-{
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	return dev_priv->display_power;
-}
-
-/**
  *	gma_power_begin		-	begin requiring power
  *	@dev: our DRM device
  *	@force_on: true to force power on
@@ -245,35 +213,10 @@ bool gma_power_is_on(struct drm_device *dev)
  */
 bool gma_power_begin(struct drm_device *dev, bool force_on)
 {
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	struct pci_dev *pdev = to_pci_dev(dev->dev);
-	int ret;
-	unsigned long flags;
-
-	spin_lock_irqsave(&power_ctrl_lock, flags);
-	/* Power already on ? */
-	if (dev_priv->display_power) {
-		dev_priv->display_count++;
-		pm_runtime_get(dev->dev);
-		spin_unlock_irqrestore(&power_ctrl_lock, flags);
-		return true;
-	}
-	if (force_on == false)
-		goto out_false;
-
-	/* Ok power up needed */
-	ret = gma_resume_pci(pdev);
-	if (ret == 0) {
-		gma_irq_preinstall(dev);
-		gma_irq_postinstall(dev);
-		pm_runtime_get(dev->dev);
-		dev_priv->display_count++;
-		spin_unlock_irqrestore(&power_ctrl_lock, flags);
-		return true;
-	}
-out_false:
-	spin_unlock_irqrestore(&power_ctrl_lock, flags);
-	return false;
+	if (force_on)
+		return pm_runtime_resume_and_get(dev->dev) == 0;
+	else
+		return pm_runtime_get_if_in_use(dev->dev) == 1;
 }
 
 /**
@@ -285,46 +228,5 @@ out_false:
  */
 void gma_power_end(struct drm_device *dev)
 {
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	unsigned long flags;
-	spin_lock_irqsave(&power_ctrl_lock, flags);
-	dev_priv->display_count--;
-	WARN_ON(dev_priv->display_count < 0);
-	spin_unlock_irqrestore(&power_ctrl_lock, flags);
 	pm_runtime_put(dev->dev);
 }
-
-int psb_runtime_suspend(struct device *dev)
-{
-	return gma_power_suspend(dev);
-}
-
-int psb_runtime_resume(struct device *dev)
-{
-	return gma_power_resume(dev);
-}
-
-int psb_runtime_idle(struct device *dev)
-{
-	struct drm_device *drmdev = pci_get_drvdata(to_pci_dev(dev));
-	struct drm_psb_private *dev_priv = to_drm_psb_private(drmdev);
-	if (dev_priv->display_count)
-		return 0;
-	else
-		return 1;
-}
-
-int gma_power_thaw(struct device *_dev)
-{
-	return gma_power_resume(_dev);
-}
-
-int gma_power_freeze(struct device *_dev)
-{
-	return gma_power_suspend(_dev);
-}
-
-int gma_power_restore(struct device *_dev)
-{
-	return gma_power_resume(_dev);
-}
diff --git a/drivers/gpu/drm/gma500/power.h b/drivers/gpu/drm/gma500/power.h
index 0c89c4d6ec20..063328d66652 100644
--- a/drivers/gpu/drm/gma500/power.h
+++ b/drivers/gpu/drm/gma500/power.h
@@ -43,9 +43,6 @@ void gma_power_uninit(struct drm_device *dev);
  */
 int gma_power_suspend(struct device *dev);
 int gma_power_resume(struct device *dev);
-int gma_power_thaw(struct device *dev);
-int gma_power_freeze(struct device *dev);
-int gma_power_restore(struct device *_dev);
 
 /*
  * These are the functions the driver should use to wrap all hw access
@@ -54,19 +51,4 @@ int gma_power_restore(struct device *_dev);
 bool gma_power_begin(struct drm_device *dev, bool force);
 void gma_power_end(struct drm_device *dev);
 
-/*
- * Use this function to do an instantaneous check for if the hw is on.
- * Only use this in cases where you know the mutex is already held such
- * as in irq install/uninstall and you need to
- * prevent a deadlock situation.  Otherwise use gma_power_begin().
- */
-bool gma_power_is_on(struct drm_device *dev);
-
-/*
- * GFX-Runtime PM callbacks
- */
-int psb_runtime_suspend(struct device *dev);
-int psb_runtime_resume(struct device *dev);
-int psb_runtime_idle(struct device *dev);
-
 #endif /*_PSB_POWERMGMT_H_*/
diff --git a/drivers/gpu/drm/gma500/psb_device.c b/drivers/gpu/drm/gma500/psb_device.c
index 71534f4ca834..3c294c38bdb4 100644
--- a/drivers/gpu/drm/gma500/psb_device.c
+++ b/drivers/gpu/drm/gma500/psb_device.c
@@ -5,8 +5,6 @@
  *
  **************************************************************************/
 
-#include <linux/backlight.h>
-
 #include <drm/drm.h>
 
 #include "gma_device.h"
@@ -24,8 +22,6 @@ static int psb_output_init(struct drm_device *dev)
 	return 0;
 }
 
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
-
 /*
  *	Poulsbo Backlight Interfaces
  */
@@ -41,18 +37,6 @@ static int psb_output_init(struct drm_device *dev)
 #define PSB_BACKLIGHT_PWM_POLARITY_BIT_CLEAR (0xFFFE)
 #define PSB_BACKLIGHT_PWM_CTL_SHIFT	(16)
 
-static int psb_brightness;
-static struct backlight_device *psb_backlight_device;
-
-static int psb_get_brightness(struct backlight_device *bd)
-{
-	/* return locally cached var instead of HW read (due to DPST etc.) */
-	/* FIXME: ideally return actual value in case firmware fiddled with
-	   it */
-	return psb_brightness;
-}
-
-
 static int psb_backlight_setup(struct drm_device *dev)
 {
 	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
@@ -86,62 +70,13 @@ static int psb_backlight_setup(struct drm_device *dev)
 		REG_WRITE(BLC_PWM_CTL,
 			(value << PSB_BACKLIGHT_PWM_CTL_SHIFT) | (value));
 	}
-	return 0;
-}
-
-static int psb_set_brightness(struct backlight_device *bd)
-{
-	struct drm_device *dev = bl_get_data(psb_backlight_device);
-	int level = bd->props.brightness;
-
-	/* Percentage 1-100% being valid */
-	if (level < 1)
-		level = 1;
-
-	psb_intel_lvds_set_brightness(dev, level);
-	psb_brightness = level;
-	return 0;
-}
-
-static const struct backlight_ops psb_ops = {
-	.get_brightness = psb_get_brightness,
-	.update_status  = psb_set_brightness,
-};
-
-static int psb_backlight_init(struct drm_device *dev)
-{
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	int ret;
-	struct backlight_properties props;
-
-	memset(&props, 0, sizeof(struct backlight_properties));
-	props.max_brightness = 100;
-	props.type = BACKLIGHT_PLATFORM;
-
-	psb_backlight_device = backlight_device_register("psb-bl",
-					NULL, (void *)dev, &psb_ops, &props);
-	if (IS_ERR(psb_backlight_device))
-		return PTR_ERR(psb_backlight_device);
-
-	ret = psb_backlight_setup(dev);
-	if (ret < 0) {
-		backlight_device_unregister(psb_backlight_device);
-		psb_backlight_device = NULL;
-		return ret;
-	}
-	psb_backlight_device->props.brightness = 100;
-	psb_backlight_device->props.max_brightness = 100;
-	backlight_update_status(psb_backlight_device);
-	dev_priv->backlight_device = psb_backlight_device;
 
+	psb_intel_lvds_set_brightness(dev, PSB_MAX_BRIGHTNESS);
 	/* This must occur after the backlight is properly initialised */
 	psb_lid_timer_init(dev_priv);
-
 	return 0;
 }
 
-#endif
-
 /*
  *	Provide the Poulsbo specific chip logic and low level methods
  *	for power management
@@ -345,9 +280,9 @@ const struct psb_ops psb_chip_ops = {
 
 	.output_init = psb_output_init,
 
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
-	.backlight_init = psb_backlight_init,
-#endif
+	.backlight_init = psb_backlight_setup,
+	.backlight_set = psb_intel_lvds_set_brightness,
+	.backlight_name = "psb-bl",
 
 	.init_pm = psb_init_pm,
 	.save_regs = psb_save_display_registers,
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index 54e756b48606..cd9c73f5a64a 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -169,8 +169,7 @@ static void psb_driver_unload(struct drm_device *dev)
 
 	/* TODO: Kill vblank etc here */
 
-	if (dev_priv->backlight_device)
-		gma_backlight_exit(dev);
+	gma_backlight_exit(dev);
 	psb_modeset_cleanup(dev);
 
 	gma_irq_uninstall(dev);
@@ -399,6 +398,8 @@ static int psb_driver_load(struct drm_device *dev, unsigned long flags)
 		if (gma_encoder->type == INTEL_OUTPUT_LVDS ||
 		    gma_encoder->type == INTEL_OUTPUT_MIPI) {
 			ret = gma_backlight_init(dev);
+			if (ret == 0)
+				acpi_video_register_backlight();
 			break;
 		}
 	}
@@ -407,11 +408,6 @@ static int psb_driver_load(struct drm_device *dev, unsigned long flags)
 	if (ret)
 		return ret;
 	psb_intel_opregion_enable_asle(dev);
-#if 0
-	/* Enable runtime pm at last */
-	pm_runtime_enable(dev->dev);
-	pm_runtime_set_active(dev->dev);
-#endif
 
 	return devm_add_action_or_reset(dev->dev, psb_device_release, dev);
 
@@ -420,33 +416,6 @@ out_err:
 	return ret;
 }
 
-static inline void get_brightness(struct backlight_device *bd)
-{
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
-	if (bd) {
-		bd->props.brightness = bd->ops->get_brightness(bd);
-		backlight_update_status(bd);
-	}
-#endif
-}
-
-static long psb_unlocked_ioctl(struct file *filp, unsigned int cmd,
-			       unsigned long arg)
-{
-	struct drm_file *file_priv = filp->private_data;
-	struct drm_device *dev = file_priv->minor->dev;
-	struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
-	static unsigned int runtime_allowed;
-
-	if (runtime_allowed == 1 && dev_priv->is_lvds_on) {
-		runtime_allowed++;
-		pm_runtime_allow(dev->dev);
-		dev_priv->rpm_enabled = 1;
-	}
-	return drm_ioctl(filp, cmd, arg);
-	/* FIXME: do we need to wrap the other side of this */
-}
-
 static int psb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	struct drm_psb_private *dev_priv;
@@ -493,22 +462,13 @@ static void psb_pci_remove(struct pci_dev *pdev)
 	drm_dev_unregister(dev);
 }
 
-static const struct dev_pm_ops psb_pm_ops = {
-	.resume = gma_power_resume,
-	.suspend = gma_power_suspend,
-	.thaw = gma_power_thaw,
-	.freeze = gma_power_freeze,
-	.restore = gma_power_restore,
-	.runtime_suspend = psb_runtime_suspend,
-	.runtime_resume = psb_runtime_resume,
-	.runtime_idle = psb_runtime_idle,
-};
+static DEFINE_RUNTIME_DEV_PM_OPS(psb_pm_ops, gma_power_suspend, gma_power_resume, NULL);
 
 static const struct file_operations psb_gem_fops = {
 	.owner = THIS_MODULE,
 	.open = drm_open,
 	.release = drm_release,
-	.unlocked_ioctl = psb_unlocked_ioctl,
+	.unlocked_ioctl = drm_ioctl,
 	.compat_ioctl = drm_compat_ioctl,
 	.mmap = drm_gem_mmap,
 	.poll = drm_poll,
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index 731cc356c07a..ae544b69fc47 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -172,6 +172,8 @@
 #define PSB_WATCHDOG_DELAY (HZ * 2)
 #define PSB_LID_DELAY (HZ / 10)
 
+#define PSB_MAX_BRIGHTNESS		100
+
 #define PSB_PWR_STATE_ON		1
 #define PSB_PWR_STATE_OFF		2
 
@@ -426,9 +428,7 @@ struct drm_psb_private {
 	spinlock_t irqmask_lock;
 
 	/* Power */
-	bool suspended;
-	bool display_power;
-	int display_count;
+	bool pm_initialized;
 
 	/* Modesetting */
 	struct psb_intel_mode_device mode_dev;
@@ -486,9 +486,6 @@ struct drm_psb_private {
 	unsigned int core_freq;
 	uint32_t iLVDS_enable;
 
-	/* Runtime PM state */
-	int rpm_enabled;
-
 	/* MID specific */
 	bool use_msi;
 	bool has_gct;
@@ -527,10 +524,6 @@ struct drm_psb_private {
 
 	struct drm_fb_helper *fb_helper;
 
-	/* Panel brightness */
-	int brightness;
-	int brightness_adjusted;
-
 	bool dsr_enable;
 	u32 dsr_fb_update;
 	bool dpi_panel_on[3];
@@ -599,10 +592,13 @@ struct psb_ops {
 	void (*disable_sr)(struct drm_device *dev);
 
 	void (*lvds_bl_power)(struct drm_device *dev, bool on);
-#ifdef CONFIG_BACKLIGHT_CLASS_DEVICE
+
 	/* Backlight */
 	int (*backlight_init)(struct drm_device *dev);
-#endif
+	void (*backlight_set)(struct drm_device *dev, int level);
+	int (*backlight_get)(struct drm_device *dev);
+	const char *backlight_name;
+
 	int i2c_bus;		/* I2C bus identifier for Moorestown */
 };
 
diff --git a/drivers/gpu/drm/gma500/psb_intel_display.c b/drivers/gpu/drm/gma500/psb_intel_display.c
index 9a5ea06a1a8e..531c1781a8fb 100644
--- a/drivers/gpu/drm/gma500/psb_intel_display.c
+++ b/drivers/gpu/drm/gma500/psb_intel_display.c
@@ -9,8 +9,6 @@
 #include <linux/delay.h>
 #include <linux/i2c.h>
 
-#include <drm/drm_plane_helper.h>
-
 #include "framebuffer.h"
 #include "gem.h"
 #include "gma_display.h"
diff --git a/drivers/gpu/drm/gma500/psb_intel_drv.h b/drivers/gpu/drm/gma500/psb_intel_drv.h
index 8ccba116821b..8a1111fe714b 100644
--- a/drivers/gpu/drm/gma500/psb_intel_drv.h
+++ b/drivers/gpu/drm/gma500/psb_intel_drv.h
@@ -197,8 +197,6 @@ extern void psb_intel_lvds_set_brightness(struct drm_device *dev, int level);
 extern void oaktrail_lvds_init(struct drm_device *dev,
 			   struct psb_intel_mode_device *mode_dev);
 extern void oaktrail_wait_for_INTR_PKT_SENT(struct drm_device *dev);
-extern void oaktrail_dsi_init(struct drm_device *dev,
-			   struct psb_intel_mode_device *mode_dev);
 struct gma_i2c_chan *oaktrail_lvds_i2c_init(struct drm_device *dev);
 extern void mid_dsi_init(struct drm_device *dev,
 		    struct psb_intel_mode_device *mode_dev, int dsi_num);
@@ -219,9 +217,6 @@ extern struct drm_crtc *psb_intel_get_crtc_from_pipe(struct drm_device *dev,
 						 int pipe);
 extern struct drm_connector *psb_intel_sdvo_find(struct drm_device *dev,
 					     int sdvoB);
-extern int psb_intel_sdvo_supports_hotplug(struct drm_connector *connector);
-extern void psb_intel_sdvo_set_hotplug(struct drm_connector *connector,
-				   int enable);
 extern int intelfb_probe(struct drm_device *dev);
 extern int intelfb_remove(struct drm_device *dev,
 			  struct drm_framebuffer *fb);
diff --git a/drivers/gpu/drm/gma500/psb_intel_sdvo.c b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
index a85aace25548..bdced46dd333 100644
--- a/drivers/gpu/drm/gma500/psb_intel_sdvo.c
+++ b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
@@ -400,26 +400,38 @@ static const struct _sdvo_cmd_name {
 #define IS_SDVOB(reg)	(reg == SDVOB)
 #define SDVO_NAME(svdo) (IS_SDVOB((svdo)->sdvo_reg) ? "SDVOB" : "SDVOC")
 
-static void psb_intel_sdvo_debug_write(struct psb_intel_sdvo *psb_intel_sdvo, u8 cmd,
-				   const void *args, int args_len)
+static void psb_intel_sdvo_debug_write(struct psb_intel_sdvo *psb_intel_sdvo,
+				       u8 cmd, const void *args, int args_len)
 {
-	int i;
+	struct drm_device *dev = psb_intel_sdvo->base.base.dev;
+	int i, pos = 0;
+	char buffer[73];
+
+#define BUF_PRINT(args...) \
+	pos += snprintf(buffer + pos, max_t(int, sizeof(buffer) - pos, 0), args)
+
+	for (i = 0; i < args_len; i++) {
+		BUF_PRINT("%02X ", ((u8 *)args)[i]);
+	}
+
+	for (; i < 8; i++) {
+		BUF_PRINT("   ");
+	}
 
-	DRM_DEBUG_KMS("%s: W: %02X ",
-				SDVO_NAME(psb_intel_sdvo), cmd);
-	for (i = 0; i < args_len; i++)
-		DRM_DEBUG_KMS("%02X ", ((u8 *)args)[i]);
-	for (; i < 8; i++)
-		DRM_DEBUG_KMS("   ");
 	for (i = 0; i < ARRAY_SIZE(sdvo_cmd_names); i++) {
 		if (cmd == sdvo_cmd_names[i].cmd) {
-			DRM_DEBUG_KMS("(%s)", sdvo_cmd_names[i].name);
+			BUF_PRINT("(%s)", sdvo_cmd_names[i].name);
 			break;
 		}
 	}
+
 	if (i == ARRAY_SIZE(sdvo_cmd_names))
-		DRM_DEBUG_KMS("(%02X)", cmd);
-	DRM_DEBUG_KMS("\n");
+		BUF_PRINT("(%02X)", cmd);
+
+	drm_WARN_ON(dev, pos >= sizeof(buffer) - 1);
+#undef BUF_PRINT
+
+	DRM_DEBUG_KMS("%s: W: %02X %s\n", SDVO_NAME(psb_intel_sdvo), cmd, buffer);
 }
 
 static const char *cmd_status_names[] = {
@@ -490,13 +502,13 @@ static bool psb_intel_sdvo_write_cmd(struct psb_intel_sdvo *psb_intel_sdvo, u8 c
 }
 
 static bool psb_intel_sdvo_read_response(struct psb_intel_sdvo *psb_intel_sdvo,
-				     void *response, int response_len)
+					 void *response, int response_len)
 {
+	struct drm_device *dev = psb_intel_sdvo->base.base.dev;
+	char buffer[73];
+	int i, pos = 0;
 	u8 retry = 5;
 	u8 status;
-	int i;
-
-	DRM_DEBUG_KMS("%s: R: ", SDVO_NAME(psb_intel_sdvo));
 
 	/*
 	 * The documentation states that all commands will be
@@ -520,10 +532,13 @@ static bool psb_intel_sdvo_read_response(struct psb_intel_sdvo *psb_intel_sdvo,
 			goto log_fail;
 	}
 
+#define BUF_PRINT(args...) \
+	pos += snprintf(buffer + pos, max_t(int, sizeof(buffer) - pos, 0), args)
+
 	if (status <= SDVO_CMD_STATUS_SCALING_NOT_SUPP)
-		DRM_DEBUG_KMS("(%s)", cmd_status_names[status]);
+		BUF_PRINT("(%s)", cmd_status_names[status]);
 	else
-		DRM_DEBUG_KMS("(??? %d)", status);
+		BUF_PRINT("(??? %d)", status);
 
 	if (status != SDVO_CMD_STATUS_SUCCESS)
 		goto log_fail;
@@ -534,13 +549,18 @@ static bool psb_intel_sdvo_read_response(struct psb_intel_sdvo *psb_intel_sdvo,
 					  SDVO_I2C_RETURN_0 + i,
 					  &((u8 *)response)[i]))
 			goto log_fail;
-		DRM_DEBUG_KMS(" %02X", ((u8 *)response)[i]);
+		BUF_PRINT(" %02X", ((u8 *)response)[i]);
 	}
-	DRM_DEBUG_KMS("\n");
+
+	drm_WARN_ON(dev, pos >= sizeof(buffer) - 1);
+#undef BUF_PRINT
+
+	DRM_DEBUG_KMS("%s: R: %s\n", SDVO_NAME(psb_intel_sdvo), buffer);
 	return true;
 
 log_fail:
-	DRM_DEBUG_KMS("... failed\n");
+	DRM_DEBUG_KMS("%s: R: ... failed %s\n",
+		      SDVO_NAME(psb_intel_sdvo), buffer);
 	return false;
 }
 
diff --git a/drivers/gpu/drm/gma500/psb_irq.c b/drivers/gpu/drm/gma500/psb_irq.c
index 038f18ed0a95..d421031462df 100644
--- a/drivers/gpu/drm/gma500/psb_irq.c
+++ b/drivers/gpu/drm/gma500/psb_irq.c
@@ -228,7 +228,7 @@ static irqreturn_t gma_irq_handler(int irq, void *arg)
 	vdc_stat &= dev_priv->vdc_irq_mask;
 	spin_unlock(&dev_priv->irqmask_lock);
 
-	if (dsp_int && gma_power_is_on(dev)) {
+	if (dsp_int) {
 		gma_vdc_interrupt(dev, vdc_stat);
 		handled = 1;
 	}
@@ -264,13 +264,12 @@ void gma_irq_preinstall(struct drm_device *dev)
 
 	spin_lock_irqsave(&dev_priv->irqmask_lock, irqflags);
 
-	if (gma_power_is_on(dev)) {
-		PSB_WVDC32(0xFFFFFFFF, PSB_HWSTAM);
-		PSB_WVDC32(0x00000000, PSB_INT_MASK_R);
-		PSB_WVDC32(0x00000000, PSB_INT_ENABLE_R);
-		PSB_WSGX32(0x00000000, PSB_CR_EVENT_HOST_ENABLE);
-		PSB_RSGX32(PSB_CR_EVENT_HOST_ENABLE);
-	}
+	PSB_WVDC32(0xFFFFFFFF, PSB_HWSTAM);
+	PSB_WVDC32(0x00000000, PSB_INT_MASK_R);
+	PSB_WVDC32(0x00000000, PSB_INT_ENABLE_R);
+	PSB_WSGX32(0x00000000, PSB_CR_EVENT_HOST_ENABLE);
+	PSB_RSGX32(PSB_CR_EVENT_HOST_ENABLE);
+
 	if (dev->vblank[0].enabled)
 		dev_priv->vdc_irq_mask |= _PSB_VSYNC_PIPEA_FLAG;
 	if (dev->vblank[1].enabled)
diff --git a/drivers/gpu/drm/gud/gud_pipe.c b/drivers/gpu/drm/gud/gud_pipe.c
index 4873f9799f41..7c6dc2bcd14a 100644
--- a/drivers/gpu/drm/gud/gud_pipe.c
+++ b/drivers/gpu/drm/gud/gud_pipe.c
@@ -59,6 +59,7 @@ static size_t gud_xrgb8888_to_r124(u8 *dst, const struct drm_format_info *format
 	unsigned int bits_per_pixel = 8 / block_width;
 	unsigned int x, y, width, height;
 	u8 pix, *pix8, *block = dst; /* Assign to silence compiler warning */
+	struct iosys_map dst_map, vmap;
 	size_t len;
 	void *buf;
 
@@ -74,7 +75,9 @@ static size_t gud_xrgb8888_to_r124(u8 *dst, const struct drm_format_info *format
 	if (!buf)
 		return 0;
 
-	drm_fb_xrgb8888_to_gray8(buf, 0, src, fb, rect);
+	iosys_map_set_vaddr(&dst_map, buf);
+	iosys_map_set_vaddr(&vmap, src);
+	drm_fb_xrgb8888_to_gray8(&dst_map, NULL, &vmap, fb, rect);
 	pix8 = buf;
 
 	for (y = 0; y < height; y++) {
@@ -105,7 +108,8 @@ static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
 	unsigned int bits_per_pixel = 8 / block_width;
 	u8 r, g, b, pix, *block = dst; /* Assign to silence compiler warning */
 	unsigned int x, y, width;
-	u32 *pix32;
+	__le32 *sbuf32;
+	u32 pix32;
 	size_t len;
 
 	/* Start on a byte boundary */
@@ -114,8 +118,8 @@ static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
 	len = drm_format_info_min_pitch(format, 0, width) * drm_rect_height(rect);
 
 	for (y = rect->y1; y < rect->y2; y++) {
-		pix32 = src + (y * fb->pitches[0]);
-		pix32 += rect->x1;
+		sbuf32 = src + (y * fb->pitches[0]);
+		sbuf32 += rect->x1;
 
 		for (x = 0; x < width; x++) {
 			unsigned int pixpos = x % block_width; /* within byte from the left */
@@ -126,9 +130,10 @@ static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
 				*block = 0;
 			}
 
-			r = *pix32 >> 16;
-			g = *pix32 >> 8;
-			b = *pix32++;
+			pix32 = le32_to_cpu(*sbuf32++);
+			r = pix32 >> 16;
+			g = pix32 >> 8;
+			b = pix32;
 
 			switch (format->format) {
 			case GUD_DRM_FORMAT_XRGB1111:
@@ -154,6 +159,7 @@ static int gud_prep_flush(struct gud_device *gdrm, struct drm_framebuffer *fb,
 	u8 compression = gdrm->compression;
 	struct iosys_map map[DRM_FORMAT_MAX_PLANES];
 	struct iosys_map map_data[DRM_FORMAT_MAX_PLANES];
+	struct iosys_map dst;
 	void *vaddr, *buf;
 	size_t pitch, len;
 	int ret = 0;
@@ -177,6 +183,7 @@ retry:
 		buf = gdrm->compress_buf;
 	else
 		buf = gdrm->bulk_buf;
+	iosys_map_set_vaddr(&dst, buf);
 
 	/*
 	 * Imported buffers are assumed to be write-combined and thus uncached
@@ -190,23 +197,24 @@ retry:
 				goto end_cpu_access;
 			}
 		} else if (format->format == DRM_FORMAT_R8) {
-			drm_fb_xrgb8888_to_gray8(buf, 0, vaddr, fb, rect);
+			drm_fb_xrgb8888_to_gray8(&dst, NULL, map_data, fb, rect);
 		} else if (format->format == DRM_FORMAT_RGB332) {
-			drm_fb_xrgb8888_to_rgb332(buf, 0, vaddr, fb, rect);
+			drm_fb_xrgb8888_to_rgb332(&dst, NULL, map_data, fb, rect);
 		} else if (format->format == DRM_FORMAT_RGB565) {
-			drm_fb_xrgb8888_to_rgb565(buf, 0, vaddr, fb, rect, gud_is_big_endian());
+			drm_fb_xrgb8888_to_rgb565(&dst, NULL, map_data, fb, rect,
+						  gud_is_big_endian());
 		} else if (format->format == DRM_FORMAT_RGB888) {
-			drm_fb_xrgb8888_to_rgb888(buf, 0, vaddr, fb, rect);
+			drm_fb_xrgb8888_to_rgb888(&dst, NULL, map_data, fb, rect);
 		} else {
 			len = gud_xrgb8888_to_color(buf, format, vaddr, fb, rect);
 		}
 	} else if (gud_is_big_endian() && format->cpp[0] > 1) {
-		drm_fb_swab(buf, 0, vaddr, fb, rect, !import_attach);
+		drm_fb_swab(&dst, NULL, map_data, fb, rect, !import_attach);
 	} else if (compression && !import_attach && pitch == fb->pitches[0]) {
 		/* can compress directly from the framebuffer */
 		buf = vaddr + rect->y1 * pitch;
 	} else {
-		drm_fb_memcpy(buf, 0, vaddr, fb, rect);
+		drm_fb_memcpy(&dst, NULL, map_data, fb, rect);
 	}
 
 	memset(req, 0, sizeof(*req));
diff --git a/drivers/gpu/drm/hisilicon/kirin/Kconfig b/drivers/gpu/drm/hisilicon/kirin/Kconfig
index b770f7662830..c5265675bf0c 100644
--- a/drivers/gpu/drm/hisilicon/kirin/Kconfig
+++ b/drivers/gpu/drm/hisilicon/kirin/Kconfig
@@ -3,7 +3,7 @@ config DRM_HISI_KIRIN
 	tristate "DRM Support for Hisilicon Kirin series SoCs Platform"
 	depends on DRM && OF && ARM64
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_MIPI_DSI
 	help
 	  Choose this option if you have a hisilicon Kirin chipsets(hi6220).
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
index 61c29c2834e6..871f79a6b17e 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
@@ -24,11 +24,10 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 #include <drm/drm_gem_framebuffer_helper.h>
@@ -549,13 +548,13 @@ static const struct drm_crtc_funcs ade_crtc_funcs = {
 static void ade_rdma_set(void __iomem *base, struct drm_framebuffer *fb,
 			 u32 ch, u32 y, u32 in_h, u32 fmt)
 {
-	struct drm_gem_cma_object *obj = drm_fb_cma_get_gem_obj(fb, 0);
+	struct drm_gem_dma_object *obj = drm_fb_dma_get_gem_obj(fb, 0);
 	u32 reg_ctrl, reg_addr, reg_size, reg_stride, reg_space, reg_en;
 	u32 stride = fb->pitches[0];
-	u32 addr = (u32)obj->paddr + y * stride;
+	u32 addr = (u32) obj->dma_addr + y * stride;
 
 	DRM_DEBUG_DRIVER("rdma%d: (y=%d, height=%d), stride=%d, paddr=0x%x\n",
-			 ch + 1, y, in_h, stride, (u32)obj->paddr);
+			 ch + 1, y, in_h, stride, (u32) obj->dma_addr);
 	DRM_DEBUG_DRIVER("addr=0x%x, fb:%dx%d, pixel_format=%d(%p4cc)\n",
 			 addr, fb->width, fb->height, fmt,
 			 &fb->format->format);
@@ -920,12 +919,12 @@ static const struct drm_mode_config_funcs ade_mode_config_funcs = {
 
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(ade_fops);
+DEFINE_DRM_GEM_DMA_FOPS(ade_fops);
 
 static const struct drm_driver ade_driver = {
 	.driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops = &ade_fops,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 	.name = "kirin",
 	.desc = "Hisilicon Kirin620 SoC DRM Driver",
 	.date = "20150718",
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
index 2af51df6dca7..73ee7f25f734 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
@@ -19,9 +19,8 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_of.h>
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c b/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
index b8e64dd8d3a6..28e732f94bf2 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
+++ b/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
@@ -21,19 +21,18 @@
 #include "hyperv_drm.h"
 
 static int hyperv_blit_to_vram_rect(struct drm_framebuffer *fb,
-				    const struct iosys_map *map,
+				    const struct iosys_map *vmap,
 				    struct drm_rect *rect)
 {
 	struct hyperv_drm_device *hv = to_hv(fb->dev);
-	void __iomem *dst = hv->vram;
-	void *vmap = map->vaddr; /* TODO: Use mapping abstraction properly */
+	struct iosys_map dst = IOSYS_MAP_INIT_VADDR_IOMEM(hv->vram);
 	int idx;
 
 	if (!drm_dev_enter(&hv->dev, &idx))
 		return -ENODEV;
 
-	dst += drm_fb_clip_offset(fb->pitches[0], fb->format, rect);
-	drm_fb_memcpy_toio(dst, fb->pitches[0], vmap, fb, rect);
+	iosys_map_incr(&dst, drm_fb_clip_offset(fb->pitches[0], fb->format, rect));
+	drm_fb_memcpy(&dst, fb->pitches, vmap, fb, rect);
 
 	drm_dev_exit(idx);
 
diff --git a/drivers/gpu/drm/i2c/ch7006_drv.c b/drivers/gpu/drm/i2c/ch7006_drv.c
index b91e48d2190d..578b738859b9 100644
--- a/drivers/gpu/drm/i2c/ch7006_drv.c
+++ b/drivers/gpu/drm/i2c/ch7006_drv.c
@@ -417,11 +417,9 @@ fail:
 	return -ENODEV;
 }
 
-static int ch7006_remove(struct i2c_client *client)
+static void ch7006_remove(struct i2c_client *client)
 {
 	ch7006_dbg(client, "\n");
-
-	return 0;
 }
 
 static int ch7006_resume(struct device *dev)
diff --git a/drivers/gpu/drm/i2c/sil164_drv.c b/drivers/gpu/drm/i2c/sil164_drv.c
index 741886b54419..1bc0b5de4499 100644
--- a/drivers/gpu/drm/i2c/sil164_drv.c
+++ b/drivers/gpu/drm/i2c/sil164_drv.c
@@ -370,12 +370,6 @@ sil164_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	return 0;
 }
 
-static int
-sil164_remove(struct i2c_client *client)
-{
-	return 0;
-}
-
 static struct i2c_client *
 sil164_detect_slave(struct i2c_client *client)
 {
@@ -427,7 +421,6 @@ MODULE_DEVICE_TABLE(i2c, sil164_ids);
 static struct drm_i2c_encoder_driver sil164_driver = {
 	.i2c_driver = {
 		.probe = sil164_probe,
-		.remove = sil164_remove,
 		.driver = {
 			.name = "sil164",
 		},
diff --git a/drivers/gpu/drm/i2c/tda9950.c b/drivers/gpu/drm/i2c/tda9950.c
index 5b03fdd1eaa4..9ed54e7ccff2 100644
--- a/drivers/gpu/drm/i2c/tda9950.c
+++ b/drivers/gpu/drm/i2c/tda9950.c
@@ -478,14 +478,12 @@ static int tda9950_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tda9950_remove(struct i2c_client *client)
+static void tda9950_remove(struct i2c_client *client)
 {
 	struct tda9950_priv *priv = i2c_get_clientdata(client);
 
 	cec_notifier_cec_adap_unregister(priv->notify, priv->adap);
 	cec_unregister_adapter(priv->adap);
-
-	return 0;
 }
 
 static struct i2c_device_id tda9950_ids[] = {
diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c
index f8eb6f69be05..d444e7fffb54 100644
--- a/drivers/gpu/drm/i2c/tda998x_drv.c
+++ b/drivers/gpu/drm/i2c/tda998x_drv.c
@@ -2076,11 +2076,10 @@ tda998x_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	return ret;
 }
 
-static int tda998x_remove(struct i2c_client *client)
+static void tda998x_remove(struct i2c_client *client)
 {
 	component_del(&client->dev, &tda998x_ops);
 	tda998x_destroy(&client->dev);
-	return 0;
 }
 
 #ifdef CONFIG_OF
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
index 7ae3b7d67fcf..3efce05d7b57 100644
--- a/drivers/gpu/drm/i915/Kconfig
+++ b/drivers/gpu/drm/i915/Kconfig
@@ -23,6 +23,8 @@ config DRM_I915
 	# but for select to work, need to select ACPI_VIDEO's dependencies, ick
 	select BACKLIGHT_CLASS_DEVICE if ACPI
 	select INPUT if ACPI
+	select X86_PLATFORM_DEVICES if ACPI
+	select ACPI_WMI if ACPI
 	select ACPI_VIDEO if ACPI
 	select ACPI_BUTTON if ACPI
 	select SYNC_FILE
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 522ef9b4aff3..a26edcdadc21 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -123,6 +123,7 @@ gt-y += \
 	gt/intel_ring.o \
 	gt/intel_ring_submission.o \
 	gt/intel_rps.o \
+	gt/intel_sa_media.o \
 	gt/intel_sseu.o \
 	gt/intel_sseu_debugfs.o \
 	gt/intel_timeline.o \
@@ -257,7 +258,8 @@ i915-y += \
 	display/intel_vga.o \
 	display/i9xx_plane.o \
 	display/skl_scaler.o \
-	display/skl_universal_plane.o
+	display/skl_universal_plane.o \
+	display/skl_watermark.o
 i915-$(CONFIG_ACPI) += \
 	display/intel_acpi.o \
 	display/intel_opregion.o
diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c
index 82ad8fe7440c..e3e3d27ffb53 100644
--- a/drivers/gpu/drm/i915/display/g4x_dp.c
+++ b/drivers/gpu/drm/i915/display/g4x_dp.c
@@ -1169,7 +1169,7 @@ intel_dp_hotplug(struct intel_encoder *encoder,
 static bool ibx_digital_port_connected(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	u32 bit = dev_priv->hotplug.pch_hpd[encoder->hpd_pin];
+	u32 bit = dev_priv->display.hotplug.pch_hpd[encoder->hpd_pin];
 
 	return intel_de_read(dev_priv, SDEISR) & bit;
 }
@@ -1223,7 +1223,7 @@ static bool gm45_digital_port_connected(struct intel_encoder *encoder)
 static bool ilk_digital_port_connected(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	u32 bit = dev_priv->hotplug.hpd[encoder->hpd_pin];
+	u32 bit = dev_priv->display.hotplug.hpd[encoder->hpd_pin];
 
 	return intel_de_read(dev_priv, DEISR) & bit;
 }
diff --git a/drivers/gpu/drm/i915/display/hsw_ips.c b/drivers/gpu/drm/i915/display/hsw_ips.c
index 861dcd2eb890..a5be4af792cb 100644
--- a/drivers/gpu/drm/i915/display/hsw_ips.c
+++ b/drivers/gpu/drm/i915/display/hsw_ips.c
@@ -202,7 +202,7 @@ bool hsw_crtc_state_ips_capable(const struct intel_crtc_state *crtc_state)
 	 * Should measure whether using a lower cdclk w/o IPS
 	 */
 	if (IS_BROADWELL(i915) &&
-	    crtc_state->pixel_rate > i915->max_cdclk_freq * 95 / 100)
+	    crtc_state->pixel_rate > i915->display.cdclk.max_cdclk_freq * 95 / 100)
 		return false;
 
 	return true;
diff --git a/drivers/gpu/drm/i915/display/i9xx_plane.c b/drivers/gpu/drm/i915/display/i9xx_plane.c
index 592e5adfed8b..5afbe3e98ee8 100644
--- a/drivers/gpu/drm/i915/display/i9xx_plane.c
+++ b/drivers/gpu/drm/i915/display/i9xx_plane.c
@@ -7,7 +7,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 
 #include "intel_atomic.h"
 #include "intel_atomic_plane.h"
@@ -126,7 +125,7 @@ static struct intel_fbc *i9xx_plane_fbc(struct drm_i915_private *dev_priv,
 					enum i9xx_plane_id i9xx_plane)
 {
 	if (i9xx_plane_has_fbc(dev_priv, i9xx_plane))
-		return dev_priv->fbc[INTEL_FBC_A];
+		return dev_priv->display.fbc[INTEL_FBC_A];
 	else
 		return NULL;
 }
@@ -326,8 +325,8 @@ i9xx_plane_check(struct intel_crtc_state *crtc_state,
 		return ret;
 
 	ret = intel_atomic_plane_check_clipping(plane_state, crtc_state,
-						DRM_PLANE_HELPER_NO_SCALING,
-						DRM_PLANE_HELPER_NO_SCALING,
+						DRM_PLANE_NO_SCALING,
+						DRM_PLANE_NO_SCALING,
 						i9xx_plane_has_windowing(plane));
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c
index 1390729401a0..ed4d93942dbd 100644
--- a/drivers/gpu/drm/i915/display/icl_dsi.c
+++ b/drivers/gpu/drm/i915/display/icl_dsi.c
@@ -33,6 +33,7 @@
 #include "icl_dsi_regs.h"
 #include "intel_atomic.h"
 #include "intel_backlight.h"
+#include "intel_backlight_regs.h"
 #include "intel_combo_phy.h"
 #include "intel_combo_phy_regs.h"
 #include "intel_connector.h"
@@ -641,13 +642,13 @@ static void gen11_dsi_gate_clocks(struct intel_encoder *encoder)
 	u32 tmp;
 	enum phy phy;
 
-	mutex_lock(&dev_priv->dpll.lock);
+	mutex_lock(&dev_priv->display.dpll.lock);
 	tmp = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0);
 	for_each_dsi_phy(phy, intel_dsi->phys)
 		tmp |= ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy);
 
 	intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, tmp);
-	mutex_unlock(&dev_priv->dpll.lock);
+	mutex_unlock(&dev_priv->display.dpll.lock);
 }
 
 static void gen11_dsi_ungate_clocks(struct intel_encoder *encoder)
@@ -657,13 +658,13 @@ static void gen11_dsi_ungate_clocks(struct intel_encoder *encoder)
 	u32 tmp;
 	enum phy phy;
 
-	mutex_lock(&dev_priv->dpll.lock);
+	mutex_lock(&dev_priv->display.dpll.lock);
 	tmp = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0);
 	for_each_dsi_phy(phy, intel_dsi->phys)
 		tmp &= ~ICL_DPCLKA_CFGCR0_DDI_CLK_OFF(phy);
 
 	intel_de_write(dev_priv, ICL_DPCLKA_CFGCR0, tmp);
-	mutex_unlock(&dev_priv->dpll.lock);
+	mutex_unlock(&dev_priv->display.dpll.lock);
 }
 
 static bool gen11_dsi_is_clock_enabled(struct intel_encoder *encoder)
@@ -693,7 +694,7 @@ static void gen11_dsi_map_pll(struct intel_encoder *encoder,
 	enum phy phy;
 	u32 val;
 
-	mutex_lock(&dev_priv->dpll.lock);
+	mutex_lock(&dev_priv->display.dpll.lock);
 
 	val = intel_de_read(dev_priv, ICL_DPCLKA_CFGCR0);
 	for_each_dsi_phy(phy, intel_dsi->phys) {
@@ -709,7 +710,7 @@ static void gen11_dsi_map_pll(struct intel_encoder *encoder,
 
 	intel_de_posting_read(dev_priv, ICL_DPCLKA_CFGCR0);
 
-	mutex_unlock(&dev_priv->dpll.lock);
+	mutex_unlock(&dev_priv->display.dpll.lock);
 }
 
 static void
@@ -2075,13 +2076,9 @@ void icl_dsi_init(struct drm_i915_private *dev_priv)
 	if (drm_WARN_ON(&dev_priv->drm, intel_connector->panel.vbt.dsi.bl_ports & ~intel_dsi->ports))
 		intel_connector->panel.vbt.dsi.bl_ports &= intel_dsi->ports;
 
-	intel_dsi->dcs_backlight_ports = intel_connector->panel.vbt.dsi.bl_ports;
-
 	if (drm_WARN_ON(&dev_priv->drm, intel_connector->panel.vbt.dsi.cabc_ports & ~intel_dsi->ports))
 		intel_connector->panel.vbt.dsi.cabc_ports &= intel_dsi->ports;
 
-	intel_dsi->dcs_cabc_ports = intel_connector->panel.vbt.dsi.cabc_ports;
-
 	for_each_dsi_port(port, intel_dsi->ports) {
 		struct intel_dsi_host *host;
 
diff --git a/drivers/gpu/drm/i915/display/intel_acpi.c b/drivers/gpu/drm/i915/display/intel_acpi.c
index e78430001f07..9df78e7caa2b 100644
--- a/drivers/gpu/drm/i915/display/intel_acpi.c
+++ b/drivers/gpu/drm/i915/display/intel_acpi.c
@@ -7,6 +7,7 @@
 
 #include <linux/pci.h>
 #include <linux/acpi.h>
+#include <acpi/video.h>
 
 #include "i915_drv.h"
 #include "intel_acpi.h"
@@ -331,3 +332,29 @@ void intel_acpi_assign_connector_fwnodes(struct drm_i915_private *i915)
 	 */
 	fwnode_handle_put(fwnode);
 }
+
+void intel_acpi_video_register(struct drm_i915_private *i915)
+{
+	struct drm_connector_list_iter conn_iter;
+	struct drm_connector *connector;
+
+	acpi_video_register();
+
+	/*
+	 * If i915 is driving an internal panel without registering its native
+	 * backlight handler try to register the acpi_video backlight.
+	 * For panels not driven by i915 another GPU driver may still register
+	 * a native backlight later and acpi_video_register_backlight() should
+	 * only be called after any native backlights have been registered.
+	 */
+	drm_connector_list_iter_begin(&i915->drm, &conn_iter);
+	drm_for_each_connector_iter(connector, &conn_iter) {
+		struct intel_panel *panel = &to_intel_connector(connector)->panel;
+
+		if (panel->backlight.funcs && !panel->backlight.device) {
+			acpi_video_register_backlight();
+			break;
+		}
+	}
+	drm_connector_list_iter_end(&conn_iter);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_acpi.h b/drivers/gpu/drm/i915/display/intel_acpi.h
index 4a760a2baed9..6a0007452f95 100644
--- a/drivers/gpu/drm/i915/display/intel_acpi.h
+++ b/drivers/gpu/drm/i915/display/intel_acpi.h
@@ -14,6 +14,7 @@ void intel_unregister_dsm_handler(void);
 void intel_dsm_get_bios_data_funcs_supported(struct drm_i915_private *i915);
 void intel_acpi_device_id_update(struct drm_i915_private *i915);
 void intel_acpi_assign_connector_fwnodes(struct drm_i915_private *i915);
+void intel_acpi_video_register(struct drm_i915_private *i915);
 #else
 static inline void intel_register_dsm_handler(void) { return; }
 static inline void intel_unregister_dsm_handler(void) { return; }
@@ -23,6 +24,8 @@ static inline
 void intel_acpi_device_id_update(struct drm_i915_private *i915) { return; }
 static inline
 void intel_acpi_assign_connector_fwnodes(struct drm_i915_private *i915) { return; }
+static inline
+void intel_acpi_video_register(struct drm_i915_private *i915) { return; }
 #endif /* CONFIG_ACPI */
 
 #endif /* __INTEL_ACPI_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_atomic.c b/drivers/gpu/drm/i915/display/intel_atomic.c
index 40da7910f845..18f0a5ae3bac 100644
--- a/drivers/gpu/drm/i915/display/intel_atomic.c
+++ b/drivers/gpu/drm/i915/display/intel_atomic.c
@@ -32,7 +32,6 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 
 #include "i915_drv.h"
 #include "i915_reg.h"
@@ -63,9 +62,9 @@ int intel_digital_connector_atomic_get_property(struct drm_connector *connector,
 	struct intel_digital_connector_state *intel_conn_state =
 		to_intel_digital_connector_state(state);
 
-	if (property == dev_priv->force_audio_property)
+	if (property == dev_priv->display.properties.force_audio)
 		*val = intel_conn_state->force_audio;
-	else if (property == dev_priv->broadcast_rgb_property)
+	else if (property == dev_priv->display.properties.broadcast_rgb)
 		*val = intel_conn_state->broadcast_rgb;
 	else {
 		drm_dbg_atomic(&dev_priv->drm,
@@ -96,12 +95,12 @@ int intel_digital_connector_atomic_set_property(struct drm_connector *connector,
 	struct intel_digital_connector_state *intel_conn_state =
 		to_intel_digital_connector_state(state);
 
-	if (property == dev_priv->force_audio_property) {
+	if (property == dev_priv->display.properties.force_audio) {
 		intel_conn_state->force_audio = val;
 		return 0;
 	}
 
-	if (property == dev_priv->broadcast_rgb_property) {
+	if (property == dev_priv->display.properties.broadcast_rgb) {
 		intel_conn_state->broadcast_rgb = val;
 		return 0;
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_atomic_plane.c b/drivers/gpu/drm/i915/display/intel_atomic_plane.c
index efe8591619e3..aaa6708256d5 100644
--- a/drivers/gpu/drm/i915/display/intel_atomic_plane.c
+++ b/drivers/gpu/drm/i915/display/intel_atomic_plane.c
@@ -33,7 +33,6 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 
 #include "gt/intel_rps.h"
 
@@ -43,9 +42,9 @@
 #include "intel_display_types.h"
 #include "intel_fb.h"
 #include "intel_fb_pin.h"
-#include "intel_pm.h"
 #include "intel_sprite.h"
 #include "skl_scaler.h"
+#include "skl_watermark.h"
 
 static void intel_plane_state_reset(struct intel_plane_state *plane_state,
 				    struct intel_plane *plane)
diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c
index 6c9ee905f132..aacbc6da84ef 100644
--- a/drivers/gpu/drm/i915/display/intel_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_audio.c
@@ -393,7 +393,7 @@ hsw_dp_audio_config_update(struct intel_encoder *encoder,
 			   const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct i915_audio_component *acomp = dev_priv->audio.component;
+	struct i915_audio_component *acomp = dev_priv->display.audio.component;
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	enum port port = encoder->port;
 	const struct dp_aud_n_m *nm;
@@ -441,7 +441,7 @@ hsw_hdmi_audio_config_update(struct intel_encoder *encoder,
 			     const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct i915_audio_component *acomp = dev_priv->audio.component;
+	struct i915_audio_component *acomp = dev_priv->display.audio.component;
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 	enum port port = encoder->port;
 	int n, rate;
@@ -496,7 +496,7 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder,
 	enum transcoder cpu_transcoder = old_crtc_state->cpu_transcoder;
 	u32 tmp;
 
-	mutex_lock(&dev_priv->audio.mutex);
+	mutex_lock(&dev_priv->display.audio.mutex);
 
 	/* Disable timestamps */
 	tmp = intel_de_read(dev_priv, HSW_AUD_CFG(cpu_transcoder));
@@ -514,7 +514,7 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder,
 	tmp &= ~AUDIO_OUTPUT_ENABLE(cpu_transcoder);
 	intel_de_write(dev_priv, HSW_AUD_PIN_ELD_CP_VLD, tmp);
 
-	mutex_unlock(&dev_priv->audio.mutex);
+	mutex_unlock(&dev_priv->display.audio.mutex);
 }
 
 static unsigned int calc_hblank_early_prog(struct intel_encoder *encoder,
@@ -532,7 +532,7 @@ static unsigned int calc_hblank_early_prog(struct intel_encoder *encoder,
 	h_total = crtc_state->hw.adjusted_mode.crtc_htotal;
 	pixel_clk = crtc_state->hw.adjusted_mode.crtc_clock;
 	vdsc_bpp = crtc_state->dsc.compressed_bpp;
-	cdclk = i915->cdclk.hw.cdclk;
+	cdclk = i915->display.cdclk.hw.cdclk;
 	/* fec= 0.972261, using rounding multiplier of 1000000 */
 	fec_coeff = 972261;
 	link_clk = crtc_state->port_clock;
@@ -639,7 +639,7 @@ static void hsw_audio_codec_enable(struct intel_encoder *encoder,
 	u32 tmp;
 	int len, i;
 
-	mutex_lock(&dev_priv->audio.mutex);
+	mutex_lock(&dev_priv->display.audio.mutex);
 
 	/* Enable Audio WA for 4k DSC usecases */
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP))
@@ -677,7 +677,7 @@ static void hsw_audio_codec_enable(struct intel_encoder *encoder,
 	/* Enable timestamps */
 	hsw_audio_config_update(encoder, crtc_state);
 
-	mutex_unlock(&dev_priv->audio.mutex);
+	mutex_unlock(&dev_priv->display.audio.mutex);
 }
 
 static void ilk_audio_codec_disable(struct intel_encoder *encoder,
@@ -814,7 +814,7 @@ void intel_audio_codec_enable(struct intel_encoder *encoder,
 			      const struct drm_connector_state *conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct i915_audio_component *acomp = dev_priv->audio.component;
+	struct i915_audio_component *acomp = dev_priv->display.audio.component;
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_connector *connector = conn_state->connector;
 	const struct drm_display_mode *adjusted_mode =
@@ -838,17 +838,17 @@ void intel_audio_codec_enable(struct intel_encoder *encoder,
 
 	connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2;
 
-	if (dev_priv->audio.funcs)
-		dev_priv->audio.funcs->audio_codec_enable(encoder,
-							  crtc_state,
-							  conn_state);
+	if (dev_priv->display.funcs.audio)
+		dev_priv->display.funcs.audio->audio_codec_enable(encoder,
+								  crtc_state,
+								  conn_state);
 
-	mutex_lock(&dev_priv->audio.mutex);
+	mutex_lock(&dev_priv->display.audio.mutex);
 	encoder->audio_connector = connector;
 
 	/* referred in audio callbacks */
-	dev_priv->audio.encoder_map[pipe] = encoder;
-	mutex_unlock(&dev_priv->audio.mutex);
+	dev_priv->display.audio.encoder_map[pipe] = encoder;
+	mutex_unlock(&dev_priv->display.audio.mutex);
 
 	if (acomp && acomp->base.audio_ops &&
 	    acomp->base.audio_ops->pin_eld_notify) {
@@ -878,7 +878,7 @@ void intel_audio_codec_disable(struct intel_encoder *encoder,
 			       const struct drm_connector_state *old_conn_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct i915_audio_component *acomp = dev_priv->audio.component;
+	struct i915_audio_component *acomp = dev_priv->display.audio.component;
 	struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->uapi.crtc);
 	struct drm_connector *connector = old_conn_state->connector;
 	enum port port = encoder->port;
@@ -891,15 +891,15 @@ void intel_audio_codec_disable(struct intel_encoder *encoder,
 		    connector->base.id, connector->name,
 		    encoder->base.base.id, encoder->base.name, pipe_name(pipe));
 
-	if (dev_priv->audio.funcs)
-		dev_priv->audio.funcs->audio_codec_disable(encoder,
-							   old_crtc_state,
-							   old_conn_state);
+	if (dev_priv->display.funcs.audio)
+		dev_priv->display.funcs.audio->audio_codec_disable(encoder,
+								   old_crtc_state,
+								   old_conn_state);
 
-	mutex_lock(&dev_priv->audio.mutex);
+	mutex_lock(&dev_priv->display.audio.mutex);
 	encoder->audio_connector = NULL;
-	dev_priv->audio.encoder_map[pipe] = NULL;
-	mutex_unlock(&dev_priv->audio.mutex);
+	dev_priv->display.audio.encoder_map[pipe] = NULL;
+	mutex_unlock(&dev_priv->display.audio.mutex);
 
 	if (acomp && acomp->base.audio_ops &&
 	    acomp->base.audio_ops->pin_eld_notify) {
@@ -935,13 +935,13 @@ static const struct intel_audio_funcs hsw_audio_funcs = {
 void intel_audio_hooks_init(struct drm_i915_private *dev_priv)
 {
 	if (IS_G4X(dev_priv)) {
-		dev_priv->audio.funcs = &g4x_audio_funcs;
+		dev_priv->display.funcs.audio = &g4x_audio_funcs;
 	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
-		dev_priv->audio.funcs = &ilk_audio_funcs;
+		dev_priv->display.funcs.audio = &ilk_audio_funcs;
 	} else if (IS_HASWELL(dev_priv) || DISPLAY_VER(dev_priv) >= 8) {
-		dev_priv->audio.funcs = &hsw_audio_funcs;
+		dev_priv->display.funcs.audio = &hsw_audio_funcs;
 	} else if (HAS_PCH_SPLIT(dev_priv)) {
-		dev_priv->audio.funcs = &ilk_audio_funcs;
+		dev_priv->display.funcs.audio = &ilk_audio_funcs;
 	}
 }
 
@@ -971,7 +971,7 @@ void intel_audio_cdclk_change_post(struct drm_i915_private *i915)
 	struct aud_ts_cdclk_m_n aud_ts;
 
 	if (DISPLAY_VER(i915) >= 13) {
-		get_aud_ts_cdclk_m_n(i915->cdclk.hw.ref, i915->cdclk.hw.cdclk, &aud_ts);
+		get_aud_ts_cdclk_m_n(i915->display.cdclk.hw.ref, i915->display.cdclk.hw.cdclk, &aud_ts);
 
 		intel_de_write(i915, AUD_TS_CDCLK_N, aud_ts.n);
 		intel_de_write(i915, AUD_TS_CDCLK_M, aud_ts.m | AUD_TS_CDCLK_M_EN);
@@ -1046,13 +1046,13 @@ static unsigned long i915_audio_component_get_power(struct device *kdev)
 
 	ret = intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO_PLAYBACK);
 
-	if (dev_priv->audio.power_refcount++ == 0) {
+	if (dev_priv->display.audio.power_refcount++ == 0) {
 		if (DISPLAY_VER(dev_priv) >= 9) {
 			intel_de_write(dev_priv, AUD_FREQ_CNTRL,
-				       dev_priv->audio.freq_cntrl);
+				       dev_priv->display.audio.freq_cntrl);
 			drm_dbg_kms(&dev_priv->drm,
 				    "restored AUD_FREQ_CNTRL to 0x%x\n",
-				    dev_priv->audio.freq_cntrl);
+				    dev_priv->display.audio.freq_cntrl);
 		}
 
 		/* Force CDCLK to 2*BCLK as long as we need audio powered. */
@@ -1073,7 +1073,7 @@ static void i915_audio_component_put_power(struct device *kdev,
 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
 
 	/* Stop forcing CDCLK to 2*BCLK if no need for audio to be powered. */
-	if (--dev_priv->audio.power_refcount == 0)
+	if (--dev_priv->display.audio.power_refcount == 0)
 		if (IS_GEMINILAKE(dev_priv))
 			glk_force_audio_cdclk(dev_priv, false);
 
@@ -1119,7 +1119,7 @@ static int i915_audio_component_get_cdclk_freq(struct device *kdev)
 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_DDI(dev_priv)))
 		return -ENODEV;
 
-	return dev_priv->cdclk.hw.cdclk;
+	return dev_priv->display.cdclk.hw.cdclk;
 }
 
 /*
@@ -1140,10 +1140,10 @@ static struct intel_encoder *get_saved_enc(struct drm_i915_private *dev_priv,
 	/* MST */
 	if (pipe >= 0) {
 		if (drm_WARN_ON(&dev_priv->drm,
-				pipe >= ARRAY_SIZE(dev_priv->audio.encoder_map)))
+				pipe >= ARRAY_SIZE(dev_priv->display.audio.encoder_map)))
 			return NULL;
 
-		encoder = dev_priv->audio.encoder_map[pipe];
+		encoder = dev_priv->display.audio.encoder_map[pipe];
 		/*
 		 * when bootup, audio driver may not know it is
 		 * MST or not. So it will poll all the port & pipe
@@ -1159,7 +1159,7 @@ static struct intel_encoder *get_saved_enc(struct drm_i915_private *dev_priv,
 		return NULL;
 
 	for_each_pipe(dev_priv, pipe) {
-		encoder = dev_priv->audio.encoder_map[pipe];
+		encoder = dev_priv->display.audio.encoder_map[pipe];
 		if (encoder == NULL)
 			continue;
 
@@ -1177,7 +1177,7 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
 						int pipe, int rate)
 {
 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
-	struct i915_audio_component *acomp = dev_priv->audio.component;
+	struct i915_audio_component *acomp = dev_priv->display.audio.component;
 	struct intel_encoder *encoder;
 	struct intel_crtc *crtc;
 	unsigned long cookie;
@@ -1187,7 +1187,7 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
 		return 0;
 
 	cookie = i915_audio_component_get_power(kdev);
-	mutex_lock(&dev_priv->audio.mutex);
+	mutex_lock(&dev_priv->display.audio.mutex);
 
 	/* 1. get the pipe */
 	encoder = get_saved_enc(dev_priv, port, pipe);
@@ -1206,7 +1206,7 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
 	hsw_audio_config_update(encoder, crtc->config);
 
  unlock:
-	mutex_unlock(&dev_priv->audio.mutex);
+	mutex_unlock(&dev_priv->display.audio.mutex);
 	i915_audio_component_put_power(kdev, cookie);
 	return err;
 }
@@ -1220,13 +1220,13 @@ static int i915_audio_component_get_eld(struct device *kdev, int port,
 	const u8 *eld;
 	int ret = -EINVAL;
 
-	mutex_lock(&dev_priv->audio.mutex);
+	mutex_lock(&dev_priv->display.audio.mutex);
 
 	intel_encoder = get_saved_enc(dev_priv, port, pipe);
 	if (!intel_encoder) {
 		drm_dbg_kms(&dev_priv->drm, "Not valid for port %c\n",
 			    port_name(port));
-		mutex_unlock(&dev_priv->audio.mutex);
+		mutex_unlock(&dev_priv->display.audio.mutex);
 		return ret;
 	}
 
@@ -1238,7 +1238,7 @@ static int i915_audio_component_get_eld(struct device *kdev, int port,
 		memcpy(buf, eld, min(max_bytes, ret));
 	}
 
-	mutex_unlock(&dev_priv->audio.mutex);
+	mutex_unlock(&dev_priv->display.audio.mutex);
 	return ret;
 }
 
@@ -1273,7 +1273,7 @@ static int i915_audio_component_bind(struct device *i915_kdev,
 	BUILD_BUG_ON(MAX_PORTS != I915_MAX_PORTS);
 	for (i = 0; i < ARRAY_SIZE(acomp->aud_sample_rate); i++)
 		acomp->aud_sample_rate[i] = 0;
-	dev_priv->audio.component = acomp;
+	dev_priv->display.audio.component = acomp;
 	drm_modeset_unlock_all(&dev_priv->drm);
 
 	return 0;
@@ -1288,14 +1288,14 @@ static void i915_audio_component_unbind(struct device *i915_kdev,
 	drm_modeset_lock_all(&dev_priv->drm);
 	acomp->base.ops = NULL;
 	acomp->base.dev = NULL;
-	dev_priv->audio.component = NULL;
+	dev_priv->display.audio.component = NULL;
 	drm_modeset_unlock_all(&dev_priv->drm);
 
 	device_link_remove(hda_kdev, i915_kdev);
 
-	if (dev_priv->audio.power_refcount)
+	if (dev_priv->display.audio.power_refcount)
 		drm_err(&dev_priv->drm, "audio power refcount %d after unbind\n",
-			dev_priv->audio.power_refcount);
+			dev_priv->display.audio.power_refcount);
 }
 
 static const struct component_ops i915_audio_component_bind_ops = {
@@ -1359,13 +1359,13 @@ static void i915_audio_component_init(struct drm_i915_private *dev_priv)
 		drm_dbg_kms(&dev_priv->drm, "use AUD_FREQ_CNTRL of 0x%x (init value 0x%x)\n",
 			    aud_freq, aud_freq_init);
 
-		dev_priv->audio.freq_cntrl = aud_freq;
+		dev_priv->display.audio.freq_cntrl = aud_freq;
 	}
 
 	/* init with current cdclk */
 	intel_audio_cdclk_change_post(dev_priv);
 
-	dev_priv->audio.component_registered = true;
+	dev_priv->display.audio.component_registered = true;
 }
 
 /**
@@ -1377,11 +1377,11 @@ static void i915_audio_component_init(struct drm_i915_private *dev_priv)
  */
 static void i915_audio_component_cleanup(struct drm_i915_private *dev_priv)
 {
-	if (!dev_priv->audio.component_registered)
+	if (!dev_priv->display.audio.component_registered)
 		return;
 
 	component_del(dev_priv->drm.dev, &i915_audio_component_bind_ops);
-	dev_priv->audio.component_registered = false;
+	dev_priv->display.audio.component_registered = false;
 }
 
 /**
@@ -1403,7 +1403,7 @@ void intel_audio_init(struct drm_i915_private *dev_priv)
  */
 void intel_audio_deinit(struct drm_i915_private *dev_priv)
 {
-	if ((dev_priv)->audio.lpe.platdev != NULL)
+	if (dev_priv->display.audio.lpe.platdev != NULL)
 		intel_lpe_audio_teardown(dev_priv);
 	else
 		i915_audio_component_cleanup(dev_priv);
diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c
index f5e1d692976e..beba39a38c87 100644
--- a/drivers/gpu/drm/i915/display/intel_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_backlight.c
@@ -8,7 +8,10 @@
 #include <linux/pwm.h>
 #include <linux/string_helpers.h>
 
+#include <acpi/video.h>
+
 #include "intel_backlight.h"
+#include "intel_backlight_regs.h"
 #include "intel_connector.h"
 #include "intel_de.h"
 #include "intel_display_types.h"
@@ -17,6 +20,7 @@
 #include "intel_panel.h"
 #include "intel_pci_config.h"
 #include "intel_pps.h"
+#include "intel_quirks.h"
 
 /**
  * scale - scale values from one range to another
@@ -87,7 +91,7 @@ u32 intel_backlight_invert_pwm_level(struct intel_connector *connector, u32 val)
 		return val;
 
 	if (dev_priv->params.invert_brightness > 0 ||
-	    dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS) {
+	    intel_has_quirk(dev_priv, QUIRK_INVERT_BRIGHTNESS)) {
 		return panel->backlight.pwm_level_max - val + panel->backlight.pwm_level_min;
 	}
 
@@ -127,7 +131,7 @@ u32 intel_backlight_level_from_pwm(struct intel_connector *connector, u32 val)
 			 panel->backlight.max == 0 || panel->backlight.pwm_level_max == 0);
 
 	if (dev_priv->params.invert_brightness > 0 ||
-	    (dev_priv->params.invert_brightness == 0 && dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS))
+	    (dev_priv->params.invert_brightness == 0 && intel_has_quirk(dev_priv, QUIRK_INVERT_BRIGHTNESS)))
 		val = panel->backlight.pwm_level_max - (val - panel->backlight.pwm_level_min);
 
 	return scale(val, panel->backlight.pwm_level_min, panel->backlight.pwm_level_max,
@@ -304,7 +308,7 @@ void intel_backlight_set_acpi(const struct drm_connector_state *conn_state,
 	if (!panel->backlight.present || !conn_state->crtc)
 		return;
 
-	mutex_lock(&dev_priv->backlight_lock);
+	mutex_lock(&dev_priv->display.backlight.lock);
 
 	drm_WARN_ON(&dev_priv->drm, panel->backlight.max == 0);
 
@@ -320,7 +324,7 @@ void intel_backlight_set_acpi(const struct drm_connector_state *conn_state,
 	if (panel->backlight.enabled)
 		intel_panel_actually_set_backlight(conn_state, hw_level);
 
-	mutex_unlock(&dev_priv->backlight_lock);
+	mutex_unlock(&dev_priv->display.backlight.lock);
 }
 
 static void lpt_disable_backlight(const struct drm_connector_state *old_conn_state, u32 level)
@@ -464,14 +468,14 @@ void intel_backlight_disable(const struct drm_connector_state *old_conn_state)
 		return;
 	}
 
-	mutex_lock(&dev_priv->backlight_lock);
+	mutex_lock(&dev_priv->display.backlight.lock);
 
 	if (panel->backlight.device)
 		panel->backlight.device->props.power = FB_BLANK_POWERDOWN;
 	panel->backlight.enabled = false;
 	panel->backlight.funcs->disable(old_conn_state, 0);
 
-	mutex_unlock(&dev_priv->backlight_lock);
+	mutex_unlock(&dev_priv->display.backlight.lock);
 }
 
 static void lpt_enable_backlight(const struct intel_crtc_state *crtc_state,
@@ -814,11 +818,11 @@ void intel_backlight_enable(const struct intel_crtc_state *crtc_state,
 
 	drm_dbg_kms(&dev_priv->drm, "pipe %c\n", pipe_name(pipe));
 
-	mutex_lock(&dev_priv->backlight_lock);
+	mutex_lock(&dev_priv->display.backlight.lock);
 
 	__intel_backlight_enable(crtc_state, conn_state);
 
-	mutex_unlock(&dev_priv->backlight_lock);
+	mutex_unlock(&dev_priv->display.backlight.lock);
 }
 
 #if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE)
@@ -828,12 +832,12 @@ static u32 intel_panel_get_backlight(struct intel_connector *connector)
 	struct intel_panel *panel = &connector->panel;
 	u32 val = 0;
 
-	mutex_lock(&dev_priv->backlight_lock);
+	mutex_lock(&dev_priv->display.backlight.lock);
 
 	if (panel->backlight.enabled)
 		val = panel->backlight.funcs->get(connector, intel_connector_get_pipe(connector));
 
-	mutex_unlock(&dev_priv->backlight_lock);
+	mutex_unlock(&dev_priv->display.backlight.lock);
 
 	drm_dbg_kms(&dev_priv->drm, "get backlight PWM = %d\n", val);
 	return val;
@@ -861,7 +865,7 @@ static void intel_panel_set_backlight(const struct drm_connector_state *conn_sta
 	if (!panel->backlight.present)
 		return;
 
-	mutex_lock(&dev_priv->backlight_lock);
+	mutex_lock(&dev_priv->display.backlight.lock);
 
 	drm_WARN_ON(&dev_priv->drm, panel->backlight.max == 0);
 
@@ -871,7 +875,7 @@ static void intel_panel_set_backlight(const struct drm_connector_state *conn_sta
 	if (panel->backlight.enabled)
 		intel_panel_actually_set_backlight(conn_state, hw_level);
 
-	mutex_unlock(&dev_priv->backlight_lock);
+	mutex_unlock(&dev_priv->display.backlight.lock);
 }
 
 static int intel_backlight_device_update_status(struct backlight_device *bd)
@@ -951,6 +955,11 @@ int intel_backlight_device_register(struct intel_connector *connector)
 
 	WARN_ON(panel->backlight.max == 0);
 
+	if (!acpi_video_backlight_use_native()) {
+		drm_info(&i915->drm, "Skipping intel_backlight registration\n");
+		return 0;
+	}
+
 	memset(&props, 0, sizeof(props));
 	props.type = BACKLIGHT_RAW;
 
@@ -1112,7 +1121,7 @@ static u32 i9xx_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 	if (IS_PINEVIEW(dev_priv))
 		clock = KHz(RUNTIME_INFO(dev_priv)->rawclk_freq);
 	else
-		clock = KHz(dev_priv->cdclk.hw.cdclk);
+		clock = KHz(dev_priv->display.cdclk.hw.cdclk);
 
 	return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * 32);
 }
@@ -1130,7 +1139,7 @@ static u32 i965_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 	if (IS_G4X(dev_priv))
 		clock = KHz(RUNTIME_INFO(dev_priv)->rawclk_freq);
 	else
-		clock = KHz(dev_priv->cdclk.hw.cdclk);
+		clock = KHz(dev_priv->display.cdclk.hw.cdclk);
 
 	return DIV_ROUND_CLOSEST(clock, pwm_freq_hz * 128);
 }
@@ -1590,11 +1599,11 @@ void intel_backlight_update(struct intel_atomic_state *state,
 	if (!panel->backlight.present)
 		return;
 
-	mutex_lock(&dev_priv->backlight_lock);
+	mutex_lock(&dev_priv->display.backlight.lock);
 	if (!panel->backlight.enabled)
 		__intel_backlight_enable(crtc_state, conn_state);
 
-	mutex_unlock(&dev_priv->backlight_lock);
+	mutex_unlock(&dev_priv->display.backlight.lock);
 }
 
 int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe)
@@ -1604,7 +1613,7 @@ int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe)
 	int ret;
 
 	if (!connector->panel.vbt.backlight.present) {
-		if (dev_priv->quirks & QUIRK_BACKLIGHT_PRESENT) {
+		if (intel_has_quirk(dev_priv, QUIRK_BACKLIGHT_PRESENT)) {
 			drm_dbg_kms(&dev_priv->drm,
 				    "no backlight present per VBT, but present per quirk\n");
 		} else {
@@ -1619,9 +1628,9 @@ int intel_backlight_setup(struct intel_connector *connector, enum pipe pipe)
 		return -ENODEV;
 
 	/* set level and max in panel struct */
-	mutex_lock(&dev_priv->backlight_lock);
+	mutex_lock(&dev_priv->display.backlight.lock);
 	ret = panel->backlight.funcs->setup(connector, pipe);
-	mutex_unlock(&dev_priv->backlight_lock);
+	mutex_unlock(&dev_priv->display.backlight.lock);
 
 	if (ret) {
 		drm_dbg_kms(&dev_priv->drm,
@@ -1776,7 +1785,7 @@ void intel_backlight_init_funcs(struct intel_panel *panel)
 		if (intel_dp_aux_init_backlight_funcs(connector) == 0)
 			return;
 
-		if (!(dev_priv->quirks & QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK))
+		if (!intel_has_quirk(dev_priv, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK))
 			connector->panel.backlight.power = intel_pps_backlight_power;
 	}
 
diff --git a/drivers/gpu/drm/i915/display/intel_backlight_regs.h b/drivers/gpu/drm/i915/display/intel_backlight_regs.h
new file mode 100644
index 000000000000..50c1210f6d5d
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_backlight_regs.h
@@ -0,0 +1,124 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#ifndef __INTEL_BACKLIGHT_REGS_H__
+#define __INTEL_BACKLIGHT_REGS_H__
+
+#include "i915_reg_defs.h"
+
+#define _VLV_BLC_PWM_CTL2_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61250)
+#define _VLV_BLC_PWM_CTL2_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61350)
+#define VLV_BLC_PWM_CTL2(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \
+					 _VLV_BLC_PWM_CTL2_B)
+
+#define _VLV_BLC_PWM_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61254)
+#define _VLV_BLC_PWM_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61354)
+#define VLV_BLC_PWM_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL_A, \
+					_VLV_BLC_PWM_CTL_B)
+
+#define _VLV_BLC_HIST_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61260)
+#define _VLV_BLC_HIST_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61360)
+#define VLV_BLC_HIST_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_HIST_CTL_A, \
+					 _VLV_BLC_HIST_CTL_B)
+
+/* Backlight control */
+#define BLC_PWM_CTL2	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61250) /* 965+ only */
+#define   BLM_PWM_ENABLE		(1 << 31)
+#define   BLM_COMBINATION_MODE		(1 << 30) /* gen4 only */
+#define   BLM_PIPE_SELECT		(1 << 29)
+#define   BLM_PIPE_SELECT_IVB		(3 << 29)
+#define   BLM_PIPE_A			(0 << 29)
+#define   BLM_PIPE_B			(1 << 29)
+#define   BLM_PIPE_C			(2 << 29) /* ivb + */
+#define   BLM_TRANSCODER_A		BLM_PIPE_A /* hsw */
+#define   BLM_TRANSCODER_B		BLM_PIPE_B
+#define   BLM_TRANSCODER_C		BLM_PIPE_C
+#define   BLM_TRANSCODER_EDP		(3 << 29)
+#define   BLM_PIPE(pipe)		((pipe) << 29)
+#define   BLM_POLARITY_I965		(1 << 28) /* gen4 only */
+#define   BLM_PHASE_IN_INTERUPT_STATUS	(1 << 26)
+#define   BLM_PHASE_IN_ENABLE		(1 << 25)
+#define   BLM_PHASE_IN_INTERUPT_ENABL	(1 << 24)
+#define   BLM_PHASE_IN_TIME_BASE_SHIFT	(16)
+#define   BLM_PHASE_IN_TIME_BASE_MASK	(0xff << 16)
+#define   BLM_PHASE_IN_COUNT_SHIFT	(8)
+#define   BLM_PHASE_IN_COUNT_MASK	(0xff << 8)
+#define   BLM_PHASE_IN_INCR_SHIFT	(0)
+#define   BLM_PHASE_IN_INCR_MASK	(0xff << 0)
+#define BLC_PWM_CTL	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61254)
+/*
+ * This is the most significant 15 bits of the number of backlight cycles in a
+ * complete cycle of the modulated backlight control.
+ *
+ * The actual value is this field multiplied by two.
+ */
+#define   BACKLIGHT_MODULATION_FREQ_SHIFT	(17)
+#define   BACKLIGHT_MODULATION_FREQ_MASK	(0x7fff << 17)
+#define   BLM_LEGACY_MODE			(1 << 16) /* gen2 only */
+/*
+ * This is the number of cycles out of the backlight modulation cycle for which
+ * the backlight is on.
+ *
+ * This field must be no greater than the number of cycles in the complete
+ * backlight modulation cycle.
+ */
+#define   BACKLIGHT_DUTY_CYCLE_SHIFT		(0)
+#define   BACKLIGHT_DUTY_CYCLE_MASK		(0xffff)
+#define   BACKLIGHT_DUTY_CYCLE_MASK_PNV		(0xfffe)
+#define   BLM_POLARITY_PNV			(1 << 0) /* pnv only */
+
+#define BLC_HIST_CTL	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61260)
+#define  BLM_HISTOGRAM_ENABLE			(1 << 31)
+
+/* New registers for PCH-split platforms. Safe where new bits show up, the
+ * register layout machtes with gen4 BLC_PWM_CTL[12]. */
+#define BLC_PWM_CPU_CTL2	_MMIO(0x48250)
+#define BLC_PWM_CPU_CTL		_MMIO(0x48254)
+
+#define HSW_BLC_PWM2_CTL	_MMIO(0x48350)
+
+/* PCH CTL1 is totally different, all but the below bits are reserved. CTL2 is
+ * like the normal CTL from gen4 and earlier. Hooray for confusing naming. */
+#define BLC_PWM_PCH_CTL1	_MMIO(0xc8250)
+#define   BLM_PCH_PWM_ENABLE			(1 << 31)
+#define   BLM_PCH_OVERRIDE_ENABLE		(1 << 30)
+#define   BLM_PCH_POLARITY			(1 << 29)
+#define BLC_PWM_PCH_CTL2	_MMIO(0xc8254)
+
+/* BXT backlight register definition. */
+#define _BXT_BLC_PWM_CTL1			0xC8250
+#define   BXT_BLC_PWM_ENABLE			(1 << 31)
+#define   BXT_BLC_PWM_POLARITY			(1 << 29)
+#define _BXT_BLC_PWM_FREQ1			0xC8254
+#define _BXT_BLC_PWM_DUTY1			0xC8258
+
+#define _BXT_BLC_PWM_CTL2			0xC8350
+#define _BXT_BLC_PWM_FREQ2			0xC8354
+#define _BXT_BLC_PWM_DUTY2			0xC8358
+
+#define BXT_BLC_PWM_CTL(controller)    _MMIO_PIPE(controller,		\
+					_BXT_BLC_PWM_CTL1, _BXT_BLC_PWM_CTL2)
+#define BXT_BLC_PWM_FREQ(controller)   _MMIO_PIPE(controller, \
+					_BXT_BLC_PWM_FREQ1, _BXT_BLC_PWM_FREQ2)
+#define BXT_BLC_PWM_DUTY(controller)   _MMIO_PIPE(controller, \
+					_BXT_BLC_PWM_DUTY1, _BXT_BLC_PWM_DUTY2)
+
+/* Utility pin */
+#define UTIL_PIN_CTL			_MMIO(0x48400)
+#define   UTIL_PIN_ENABLE		(1 << 31)
+#define   UTIL_PIN_PIPE_MASK		(3 << 29)
+#define   UTIL_PIN_PIPE(x)		((x) << 29)
+#define   UTIL_PIN_MODE_MASK		(0xf << 24)
+#define   UTIL_PIN_MODE_DATA		(0 << 24)
+#define   UTIL_PIN_MODE_PWM		(1 << 24)
+#define   UTIL_PIN_MODE_VBLANK		(4 << 24)
+#define   UTIL_PIN_MODE_VSYNC		(5 << 24)
+#define   UTIL_PIN_MODE_EYE_LEVEL	(8 << 24)
+#define   UTIL_PIN_OUTPUT_DATA		(1 << 23)
+#define   UTIL_PIN_POLARITY		(1 << 22)
+#define   UTIL_PIN_DIRECTION_INPUT	(1 << 19)
+#define   UTIL_PIN_INPUT_DATA		(1 << 16)
+
+#endif /* __INTEL_BACKLIGHT_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_bios.c b/drivers/gpu/drm/i915/display/intel_bios.c
index 7d6eb9ad7a02..28bdb936cd1f 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.c
+++ b/drivers/gpu/drm/i915/display/intel_bios.c
@@ -135,18 +135,6 @@ static u32 raw_block_offset(const void *bdb, enum bdb_block_id section_id)
 	return block - bdb;
 }
 
-/* size of the block excluding the header */
-static u32 raw_block_size(const void *bdb, enum bdb_block_id section_id)
-{
-	const void *block;
-
-	block = find_raw_section(bdb, section_id);
-	if (!block)
-		return 0;
-
-	return get_blocksize(block);
-}
-
 struct bdb_block_entry {
 	struct list_head node;
 	enum bdb_block_id section_id;
@@ -159,7 +147,7 @@ find_section(struct drm_i915_private *i915,
 {
 	struct bdb_block_entry *entry;
 
-	list_for_each_entry(entry, &i915->vbt.bdb_blocks, node) {
+	list_for_each_entry(entry, &i915->display.vbt.bdb_blocks, node) {
 		if (entry->section_id == section_id)
 			return entry->data + 3;
 	}
@@ -231,9 +219,14 @@ static bool validate_lfp_data_ptrs(const void *bdb,
 {
 	int fp_timing_size, dvo_timing_size, panel_pnp_id_size, panel_name_size;
 	int data_block_size, lfp_data_size;
+	const void *data_block;
 	int i;
 
-	data_block_size = raw_block_size(bdb, BDB_LVDS_LFP_DATA);
+	data_block = find_raw_section(bdb, BDB_LVDS_LFP_DATA);
+	if (!data_block)
+		return false;
+
+	data_block_size = get_blocksize(data_block);
 	if (data_block_size == 0)
 		return false;
 
@@ -261,21 +254,6 @@ static bool validate_lfp_data_ptrs(const void *bdb,
 	if (16 * lfp_data_size > data_block_size)
 		return false;
 
-	/*
-	 * Except for vlv/chv machines all real VBTs seem to have 6
-	 * unaccounted bytes in the fp_timing table. And it doesn't
-	 * appear to be a really intentional hole as the fp_timing
-	 * 0xffff terminator is always within those 6 missing bytes.
-	 */
-	if (fp_timing_size + dvo_timing_size + panel_pnp_id_size != lfp_data_size &&
-	    fp_timing_size + 6 + dvo_timing_size + panel_pnp_id_size != lfp_data_size)
-		return false;
-
-	if (ptrs->ptr[0].fp_timing.offset + fp_timing_size > ptrs->ptr[0].dvo_timing.offset ||
-	    ptrs->ptr[0].dvo_timing.offset + dvo_timing_size != ptrs->ptr[0].panel_pnp_id.offset ||
-	    ptrs->ptr[0].panel_pnp_id.offset + panel_pnp_id_size != lfp_data_size)
-		return false;
-
 	/* make sure the table entries have uniform size */
 	for (i = 1; i < 16; i++) {
 		if (ptrs->ptr[i].fp_timing.table_size != fp_timing_size ||
@@ -289,6 +267,23 @@ static bool validate_lfp_data_ptrs(const void *bdb,
 			return false;
 	}
 
+	/*
+	 * Except for vlv/chv machines all real VBTs seem to have 6
+	 * unaccounted bytes in the fp_timing table. And it doesn't
+	 * appear to be a really intentional hole as the fp_timing
+	 * 0xffff terminator is always within those 6 missing bytes.
+	 */
+	if (fp_timing_size + 6 + dvo_timing_size + panel_pnp_id_size == lfp_data_size)
+		fp_timing_size += 6;
+
+	if (fp_timing_size + dvo_timing_size + panel_pnp_id_size != lfp_data_size)
+		return false;
+
+	if (ptrs->ptr[0].fp_timing.offset + fp_timing_size != ptrs->ptr[0].dvo_timing.offset ||
+	    ptrs->ptr[0].dvo_timing.offset + dvo_timing_size != ptrs->ptr[0].panel_pnp_id.offset ||
+	    ptrs->ptr[0].panel_pnp_id.offset + panel_pnp_id_size != lfp_data_size)
+		return false;
+
 	/* make sure the tables fit inside the data block */
 	for (i = 0; i < 16; i++) {
 		if (ptrs->ptr[i].fp_timing.offset + fp_timing_size > data_block_size ||
@@ -300,6 +295,15 @@ static bool validate_lfp_data_ptrs(const void *bdb,
 	if (ptrs->panel_name.offset + 16 * panel_name_size > data_block_size)
 		return false;
 
+	/* make sure fp_timing terminators are present at expected locations */
+	for (i = 0; i < 16; i++) {
+		const u16 *t = data_block + ptrs->ptr[i].fp_timing.offset +
+			fp_timing_size - 2;
+
+		if (*t != 0xffff)
+			return false;
+	}
+
 	return true;
 }
 
@@ -333,18 +337,6 @@ static bool fixup_lfp_data_ptrs(const void *bdb, void *ptrs_block)
 	return validate_lfp_data_ptrs(bdb, ptrs);
 }
 
-static const void *find_fp_timing_terminator(const u8 *data, int size)
-{
-	int i;
-
-	for (i = 0; i < size - 1; i++) {
-		if (data[i] == 0xff && data[i+1] == 0xff)
-			return &data[i];
-	}
-
-	return NULL;
-}
-
 static int make_lfp_data_ptr(struct lvds_lfp_data_ptr_table *table,
 			     int table_size, int total_size)
 {
@@ -368,11 +360,22 @@ static void next_lfp_data_ptr(struct lvds_lfp_data_ptr_table *next,
 static void *generate_lfp_data_ptrs(struct drm_i915_private *i915,
 				    const void *bdb)
 {
-	int i, size, table_size, block_size, offset;
-	const void *t0, *t1, *block;
+	int i, size, table_size, block_size, offset, fp_timing_size;
 	struct bdb_lvds_lfp_data_ptrs *ptrs;
+	const void *block;
 	void *ptrs_block;
 
+	/*
+	 * The hardcoded fp_timing_size is only valid for
+	 * modernish VBTs. All older VBTs definitely should
+	 * include block 41 and thus we don't need to
+	 * generate one.
+	 */
+	if (i915->display.vbt.version < 155)
+		return NULL;
+
+	fp_timing_size = 38;
+
 	block = find_raw_section(bdb, BDB_LVDS_LFP_DATA);
 	if (!block)
 		return NULL;
@@ -381,17 +384,8 @@ static void *generate_lfp_data_ptrs(struct drm_i915_private *i915,
 
 	block_size = get_blocksize(block);
 
-	size = block_size;
-	t0 = find_fp_timing_terminator(block, size);
-	if (!t0)
-		return NULL;
-
-	size -= t0 - block - 2;
-	t1 = find_fp_timing_terminator(t0 + 2, size);
-	if (!t1)
-		return NULL;
-
-	size = t1 - t0;
+	size = fp_timing_size + sizeof(struct lvds_dvo_timing) +
+		sizeof(struct lvds_pnp_id);
 	if (size * 16 > block_size)
 		return NULL;
 
@@ -409,7 +403,7 @@ static void *generate_lfp_data_ptrs(struct drm_i915_private *i915,
 	table_size = sizeof(struct lvds_dvo_timing);
 	size = make_lfp_data_ptr(&ptrs->ptr[0].dvo_timing, table_size, size);
 
-	table_size = t0 - block + 2;
+	table_size = fp_timing_size;
 	size = make_lfp_data_ptr(&ptrs->ptr[0].fp_timing, table_size, size);
 
 	if (ptrs->ptr[0].fp_timing.table_size)
@@ -424,14 +418,14 @@ static void *generate_lfp_data_ptrs(struct drm_i915_private *i915,
 		return NULL;
 	}
 
-	size = t1 - t0;
+	size = fp_timing_size + sizeof(struct lvds_dvo_timing) +
+		sizeof(struct lvds_pnp_id);
 	for (i = 1; i < 16; i++) {
 		next_lfp_data_ptr(&ptrs->ptr[i].fp_timing, &ptrs->ptr[i-1].fp_timing, size);
 		next_lfp_data_ptr(&ptrs->ptr[i].dvo_timing, &ptrs->ptr[i-1].dvo_timing, size);
 		next_lfp_data_ptr(&ptrs->ptr[i].panel_pnp_id, &ptrs->ptr[i-1].panel_pnp_id, size);
 	}
 
-	size = t1 - t0;
 	table_size = sizeof(struct lvds_lfp_panel_name);
 
 	if (16 * (size + table_size) <= block_size) {
@@ -508,7 +502,7 @@ init_bdb_block(struct drm_i915_private *i915,
 		return;
 	}
 
-	list_add_tail(&entry->node, &i915->vbt.bdb_blocks);
+	list_add_tail(&entry->node, &i915->display.vbt.bdb_blocks);
 }
 
 static void init_bdb_blocks(struct drm_i915_private *i915,
@@ -611,6 +605,19 @@ get_lfp_data_tail(const struct bdb_lvds_lfp_data *data,
 		return NULL;
 }
 
+static void dump_pnp_id(struct drm_i915_private *i915,
+			const struct lvds_pnp_id *pnp_id,
+			const char *name)
+{
+	u16 mfg_name = be16_to_cpu((__force __be16)pnp_id->mfg_name);
+	char vend[4];
+
+	drm_dbg_kms(&i915->drm, "%s PNPID mfg: %s (0x%x), prod: %u, serial: %u, week: %d, year: %d\n",
+		    name, drm_edid_decode_mfg_id(mfg_name, vend),
+		    pnp_id->mfg_name, pnp_id->product_code, pnp_id->serial,
+		    pnp_id->mfg_week, pnp_id->mfg_year + 1990);
+}
+
 static int opregion_get_panel_type(struct drm_i915_private *i915,
 				   const struct intel_bios_encoder_data *devdata,
 				   const struct edid *edid)
@@ -662,6 +669,8 @@ static int pnpid_get_panel_type(struct drm_i915_private *i915,
 	edid_id_nodate.mfg_week = 0;
 	edid_id_nodate.mfg_year = 0;
 
+	dump_pnp_id(i915, edid_id, "EDID");
+
 	ptrs = find_section(i915, BDB_LVDS_LFP_DATA_PTRS);
 	if (!ptrs)
 		return -1;
@@ -868,6 +877,7 @@ parse_lfp_data(struct drm_i915_private *i915,
 	const struct bdb_lvds_lfp_data *data;
 	const struct bdb_lvds_lfp_data_tail *tail;
 	const struct bdb_lvds_lfp_data_ptrs *ptrs;
+	const struct lvds_pnp_id *pnp_id;
 	int panel_type = panel->vbt.panel_type;
 
 	ptrs = find_section(i915, BDB_LVDS_LFP_DATA_PTRS);
@@ -881,11 +891,18 @@ parse_lfp_data(struct drm_i915_private *i915,
 	if (!panel->vbt.lfp_lvds_vbt_mode)
 		parse_lfp_panel_dtd(i915, panel, data, ptrs);
 
+	pnp_id = get_lvds_pnp_id(data, ptrs, panel_type);
+	dump_pnp_id(i915, pnp_id, "Panel");
+
 	tail = get_lfp_data_tail(data, ptrs);
 	if (!tail)
 		return;
 
-	if (i915->vbt.version >= 188) {
+	drm_dbg_kms(&i915->drm, "Panel name: %.*s\n",
+		    (int)sizeof(tail->panel_name[0].name),
+		    tail->panel_name[panel_type].name);
+
+	if (i915->display.vbt.version >= 188) {
 		panel->vbt.seamless_drrs_min_refresh_rate =
 			tail->seamless_drrs_min_refresh_rate[panel_type];
 		drm_dbg_kms(&i915->drm,
@@ -911,7 +928,7 @@ parse_generic_dtd(struct drm_i915_private *i915,
 	 * first on VBT >= 229, but still fall back to trying the old LFP
 	 * block if that fails.
 	 */
-	if (i915->vbt.version < 229)
+	if (i915->display.vbt.version < 229)
 		return;
 
 	generic_dtd = find_section(i915, BDB_GENERIC_DTD);
@@ -1015,12 +1032,12 @@ parse_lfp_backlight(struct drm_i915_private *i915,
 	}
 
 	panel->vbt.backlight.type = INTEL_BACKLIGHT_DISPLAY_DDI;
-	if (i915->vbt.version >= 191) {
+	if (i915->display.vbt.version >= 191) {
 		size_t exp_size;
 
-		if (i915->vbt.version >= 236)
+		if (i915->display.vbt.version >= 236)
 			exp_size = sizeof(struct bdb_lfp_backlight_data);
-		else if (i915->vbt.version >= 234)
+		else if (i915->display.vbt.version >= 234)
 			exp_size = EXP_BDB_LFP_BL_DATA_SIZE_REV_234;
 		else
 			exp_size = EXP_BDB_LFP_BL_DATA_SIZE_REV_191;
@@ -1037,14 +1054,14 @@ parse_lfp_backlight(struct drm_i915_private *i915,
 	panel->vbt.backlight.pwm_freq_hz = entry->pwm_freq_hz;
 	panel->vbt.backlight.active_low_pwm = entry->active_low_pwm;
 
-	if (i915->vbt.version >= 234) {
+	if (i915->display.vbt.version >= 234) {
 		u16 min_level;
 		bool scale;
 
 		level = backlight_data->brightness_level[panel_type].level;
 		min_level = backlight_data->brightness_min_level[panel_type].level;
 
-		if (i915->vbt.version >= 236)
+		if (i915->display.vbt.version >= 236)
 			scale = backlight_data->brightness_precision_bits[panel_type] == 16;
 		else
 			scale = level > 255;
@@ -1141,37 +1158,37 @@ parse_general_features(struct drm_i915_private *i915)
 	if (!general)
 		return;
 
-	i915->vbt.int_tv_support = general->int_tv_support;
+	i915->display.vbt.int_tv_support = general->int_tv_support;
 	/* int_crt_support can't be trusted on earlier platforms */
-	if (i915->vbt.version >= 155 &&
+	if (i915->display.vbt.version >= 155 &&
 	    (HAS_DDI(i915) || IS_VALLEYVIEW(i915)))
-		i915->vbt.int_crt_support = general->int_crt_support;
-	i915->vbt.lvds_use_ssc = general->enable_ssc;
-	i915->vbt.lvds_ssc_freq =
+		i915->display.vbt.int_crt_support = general->int_crt_support;
+	i915->display.vbt.lvds_use_ssc = general->enable_ssc;
+	i915->display.vbt.lvds_ssc_freq =
 		intel_bios_ssc_frequency(i915, general->ssc_freq);
-	i915->vbt.display_clock_mode = general->display_clock_mode;
-	i915->vbt.fdi_rx_polarity_inverted = general->fdi_rx_polarity_inverted;
-	if (i915->vbt.version >= 181) {
-		i915->vbt.orientation = general->rotate_180 ?
+	i915->display.vbt.display_clock_mode = general->display_clock_mode;
+	i915->display.vbt.fdi_rx_polarity_inverted = general->fdi_rx_polarity_inverted;
+	if (i915->display.vbt.version >= 181) {
+		i915->display.vbt.orientation = general->rotate_180 ?
 			DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP :
 			DRM_MODE_PANEL_ORIENTATION_NORMAL;
 	} else {
-		i915->vbt.orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
+		i915->display.vbt.orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN;
 	}
 
-	if (i915->vbt.version >= 249 && general->afc_startup_config) {
-		i915->vbt.override_afc_startup = true;
-		i915->vbt.override_afc_startup_val = general->afc_startup_config == 0x1 ? 0x0 : 0x7;
+	if (i915->display.vbt.version >= 249 && general->afc_startup_config) {
+		i915->display.vbt.override_afc_startup = true;
+		i915->display.vbt.override_afc_startup_val = general->afc_startup_config == 0x1 ? 0x0 : 0x7;
 	}
 
 	drm_dbg_kms(&i915->drm,
 		    "BDB_GENERAL_FEATURES int_tv_support %d int_crt_support %d lvds_use_ssc %d lvds_ssc_freq %d display_clock_mode %d fdi_rx_polarity_inverted %d\n",
-		    i915->vbt.int_tv_support,
-		    i915->vbt.int_crt_support,
-		    i915->vbt.lvds_use_ssc,
-		    i915->vbt.lvds_ssc_freq,
-		    i915->vbt.display_clock_mode,
-		    i915->vbt.fdi_rx_polarity_inverted);
+		    i915->display.vbt.int_tv_support,
+		    i915->display.vbt.int_crt_support,
+		    i915->display.vbt.lvds_use_ssc,
+		    i915->display.vbt.lvds_ssc_freq,
+		    i915->display.vbt.display_clock_mode,
+		    i915->display.vbt.fdi_rx_polarity_inverted);
 }
 
 static const struct child_device_config *
@@ -1197,7 +1214,7 @@ parse_sdvo_device_mapping(struct drm_i915_private *i915)
 		return;
 	}
 
-	list_for_each_entry(devdata, &i915->vbt.display_devices, node) {
+	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
 		child = &devdata->child;
 
 		if (child->slave_addr != SLAVE_ADDR1 &&
@@ -1221,7 +1238,7 @@ parse_sdvo_device_mapping(struct drm_i915_private *i915)
 			    child->slave_addr,
 			    (child->dvo_port == DEVICE_PORT_DVOB) ?
 			    "SDVOB" : "SDVOC");
-		mapping = &i915->vbt.sdvo_mappings[child->dvo_port - 1];
+		mapping = &i915->display.vbt.sdvo_mappings[child->dvo_port - 1];
 		if (!mapping->initialized) {
 			mapping->dvo_port = child->dvo_port;
 			mapping->slave_addr = child->slave_addr;
@@ -1272,7 +1289,7 @@ parse_driver_features(struct drm_i915_private *i915)
 		 * interpretation, but real world VBTs seem to.
 		 */
 		if (driver->lvds_config != BDB_DRIVER_FEATURE_INT_LVDS)
-			i915->vbt.int_lvds_support = 0;
+			i915->display.vbt.int_lvds_support = 0;
 	} else {
 		/*
 		 * FIXME it's not clear which BDB version has the LVDS config
@@ -1285,10 +1302,10 @@ parse_driver_features(struct drm_i915_private *i915)
 		 * in the wild with the bits correctly populated. Version
 		 * 108 (on i85x) does not have the bits correctly populated.
 		 */
-		if (i915->vbt.version >= 134 &&
+		if (i915->display.vbt.version >= 134 &&
 		    driver->lvds_config != BDB_DRIVER_FEATURE_INT_LVDS &&
 		    driver->lvds_config != BDB_DRIVER_FEATURE_INT_SDVO_LVDS)
-			i915->vbt.int_lvds_support = 0;
+			i915->display.vbt.int_lvds_support = 0;
 	}
 }
 
@@ -1302,7 +1319,7 @@ parse_panel_driver_features(struct drm_i915_private *i915,
 	if (!driver)
 		return;
 
-	if (i915->vbt.version < 228) {
+	if (i915->display.vbt.version < 228) {
 		drm_dbg_kms(&i915->drm, "DRRS State Enabled:%d\n",
 			    driver->drrs_enabled);
 		/*
@@ -1335,7 +1352,7 @@ parse_power_conservation_features(struct drm_i915_private *i915,
 
 	panel->vbt.vrr = true; /* matches Windows behaviour */
 
-	if (i915->vbt.version < 228)
+	if (i915->display.vbt.version < 228)
 		return;
 
 	power = find_section(i915, BDB_LFP_POWER);
@@ -1361,10 +1378,10 @@ parse_power_conservation_features(struct drm_i915_private *i915,
 			panel->vbt.drrs_type = DRRS_TYPE_NONE;
 	}
 
-	if (i915->vbt.version >= 232)
+	if (i915->display.vbt.version >= 232)
 		panel->vbt.edp.hobl = panel_bool(power->hobl, panel_type);
 
-	if (i915->vbt.version >= 233)
+	if (i915->display.vbt.version >= 233)
 		panel->vbt.vrr = panel_bool(power->vrr_feature_enabled,
 					    panel_type);
 }
@@ -1400,7 +1417,7 @@ parse_edp(struct drm_i915_private *i915,
 
 	panel->vbt.edp.pps = *edp_pps;
 
-	if (i915->vbt.version >= 224) {
+	if (i915->display.vbt.version >= 224) {
 		panel->vbt.edp.rate =
 			edp->edp_fast_link_training_rate[panel_type] * 20;
 	} else {
@@ -1479,7 +1496,7 @@ parse_edp(struct drm_i915_private *i915,
 		break;
 	}
 
-	if (i915->vbt.version >= 173) {
+	if (i915->display.vbt.version >= 173) {
 		u8 vswing;
 
 		/* Don't read from VBT if module parameter has valid value*/
@@ -1495,7 +1512,7 @@ parse_edp(struct drm_i915_private *i915,
 	panel->vbt.edp.drrs_msa_timing_delay =
 		panel_bits(edp->sdrrs_msa_timing_delay, panel_type, 2);
 
-	if (i915->vbt.version >= 244)
+	if (i915->display.vbt.version >= 244)
 		panel->vbt.edp.max_link_rate =
 			edp->edp_max_port_link_rate[panel_type] * 20;
 }
@@ -1527,7 +1544,7 @@ parse_psr(struct drm_i915_private *i915,
 	 * New psr options 0=500us, 1=100us, 2=2500us, 3=0us
 	 * Old decimal value is wake up time in multiples of 100 us.
 	 */
-	if (i915->vbt.version >= 205 &&
+	if (i915->display.vbt.version >= 205 &&
 	    (DISPLAY_VER(i915) >= 9 && !IS_BROXTON(i915))) {
 		switch (psr_table->tp1_wakeup_time) {
 		case 0:
@@ -1573,7 +1590,7 @@ parse_psr(struct drm_i915_private *i915,
 		panel->vbt.psr.tp2_tp3_wakeup_time_us = psr_table->tp2_tp3_wakeup_time * 100;
 	}
 
-	if (i915->vbt.version >= 226) {
+	if (i915->display.vbt.version >= 226) {
 		u32 wakeup_time = psr->psr2_tp2_tp3_wakeup_time;
 
 		wakeup_time = panel_bits(wakeup_time, panel_type, 2);
@@ -1605,7 +1622,7 @@ static void parse_dsi_backlight_ports(struct drm_i915_private *i915,
 {
 	enum port port_bc = DISPLAY_VER(i915) >= 11 ? PORT_B : PORT_C;
 
-	if (!panel->vbt.dsi.config->dual_link || i915->vbt.version < 197) {
+	if (!panel->vbt.dsi.config->dual_link || i915->display.vbt.version < 197) {
 		panel->vbt.dsi.bl_ports = BIT(port);
 		if (panel->vbt.dsi.config->cabc_supported)
 			panel->vbt.dsi.cabc_ports = BIT(port);
@@ -2060,7 +2077,7 @@ parse_compression_parameters(struct drm_i915_private *i915)
 	u16 block_size;
 	int index;
 
-	if (i915->vbt.version < 198)
+	if (i915->display.vbt.version < 198)
 		return;
 
 	params = find_section(i915, BDB_COMPRESSION_PARAMETERS);
@@ -2080,7 +2097,7 @@ parse_compression_parameters(struct drm_i915_private *i915)
 		}
 	}
 
-	list_for_each_entry(devdata, &i915->vbt.display_devices, node) {
+	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
 		child = &devdata->child;
 
 		if (!child->compression_enable)
@@ -2214,7 +2231,7 @@ static enum port get_port_by_ddc_pin(struct drm_i915_private *i915, u8 ddc_pin)
 		return PORT_NONE;
 
 	for_each_port(port) {
-		devdata = i915->vbt.ports[port];
+		devdata = i915->display.vbt.ports[port];
 
 		if (devdata && ddc_pin == devdata->child.ddc_pin)
 			return port;
@@ -2263,7 +2280,7 @@ static void sanitize_ddc_pin(struct intel_bios_encoder_data *devdata,
 	 * there are real machines (eg. Asrock B250M-HDV) where VBT has both
 	 * port A and port E with the same AUX ch and we must pick port E :(
 	 */
-	child = &i915->vbt.ports[p]->child;
+	child = &i915->display.vbt.ports[p]->child;
 
 	child->device_type &= ~DEVICE_TYPE_TMDS_DVI_SIGNALING;
 	child->device_type |= DEVICE_TYPE_NOT_HDMI_OUTPUT;
@@ -2280,7 +2297,7 @@ static enum port get_port_by_aux_ch(struct drm_i915_private *i915, u8 aux_ch)
 		return PORT_NONE;
 
 	for_each_port(port) {
-		devdata = i915->vbt.ports[port];
+		devdata = i915->display.vbt.ports[port];
 
 		if (devdata && aux_ch == devdata->child.aux_channel)
 			return port;
@@ -2315,7 +2332,7 @@ static void sanitize_aux_ch(struct intel_bios_encoder_data *devdata,
 	 * there are real machines (eg. Asrock B250M-HDV) where VBT has both
 	 * port A and port E with the same AUX ch and we must pick port E :(
 	 */
-	child = &i915->vbt.ports[p]->child;
+	child = &i915->display.vbt.ports[p]->child;
 
 	child->device_type &= ~DEVICE_TYPE_DISPLAYPORT_OUTPUT;
 	child->aux_channel = 0;
@@ -2427,7 +2444,7 @@ static enum port dvo_port_to_port(struct drm_i915_private *i915,
 		[PORT_TC4] = { DVO_PORT_HDMII, DVO_PORT_DPI, -1 },
 	};
 
-	if (DISPLAY_VER(i915) == 13)
+	if (DISPLAY_VER(i915) >= 13)
 		return __dvo_port_to_port(ARRAY_SIZE(xelpd_port_mapping),
 					  ARRAY_SIZE(xelpd_port_mapping[0]),
 					  xelpd_port_mapping,
@@ -2489,15 +2506,23 @@ static int parse_bdb_216_dp_max_link_rate(const int vbt_max_link_rate)
 
 static int _intel_bios_dp_max_link_rate(const struct intel_bios_encoder_data *devdata)
 {
-	if (!devdata || devdata->i915->vbt.version < 216)
+	if (!devdata || devdata->i915->display.vbt.version < 216)
 		return 0;
 
-	if (devdata->i915->vbt.version >= 230)
+	if (devdata->i915->display.vbt.version >= 230)
 		return parse_bdb_230_dp_max_link_rate(devdata->child.dp_max_link_rate);
 	else
 		return parse_bdb_216_dp_max_link_rate(devdata->child.dp_max_link_rate);
 }
 
+static int _intel_bios_dp_max_lane_count(const struct intel_bios_encoder_data *devdata)
+{
+	if (!devdata || devdata->i915->display.vbt.version < 244)
+		return 0;
+
+	return devdata->child.dp_max_lane_count + 1;
+}
+
 static void sanitize_device_type(struct intel_bios_encoder_data *devdata,
 				 enum port port)
 {
@@ -2553,7 +2578,7 @@ intel_bios_encoder_supports_edp(const struct intel_bios_encoder_data *devdata)
 
 static int _intel_bios_hdmi_level_shift(const struct intel_bios_encoder_data *devdata)
 {
-	if (!devdata || devdata->i915->vbt.version < 158)
+	if (!devdata || devdata->i915->display.vbt.version < 158)
 		return -1;
 
 	return devdata->child.hdmi_level_shifter_value;
@@ -2561,7 +2586,7 @@ static int _intel_bios_hdmi_level_shift(const struct intel_bios_encoder_data *de
 
 static int _intel_bios_max_tmds_clock(const struct intel_bios_encoder_data *devdata)
 {
-	if (!devdata || devdata->i915->vbt.version < 204)
+	if (!devdata || devdata->i915->display.vbt.version < 204)
 		return 0;
 
 	switch (devdata->child.hdmi_max_data_rate) {
@@ -2670,7 +2695,7 @@ static void parse_ddi_port(struct intel_bios_encoder_data *devdata)
 		return;
 	}
 
-	if (i915->vbt.ports[port]) {
+	if (i915->display.vbt.ports[port]) {
 		drm_dbg_kms(&i915->drm,
 			    "More than one child device for port %c in VBT, using the first.\n",
 			    port_name(port));
@@ -2685,7 +2710,7 @@ static void parse_ddi_port(struct intel_bios_encoder_data *devdata)
 	if (intel_bios_encoder_supports_dp(devdata))
 		sanitize_aux_ch(devdata, port);
 
-	i915->vbt.ports[port] = devdata;
+	i915->display.vbt.ports[port] = devdata;
 }
 
 static bool has_ddi_port_info(struct drm_i915_private *i915)
@@ -2701,12 +2726,12 @@ static void parse_ddi_ports(struct drm_i915_private *i915)
 	if (!has_ddi_port_info(i915))
 		return;
 
-	list_for_each_entry(devdata, &i915->vbt.display_devices, node)
+	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node)
 		parse_ddi_port(devdata);
 
 	for_each_port(port) {
-		if (i915->vbt.ports[port])
-			print_ddi_port(i915->vbt.ports[port], port);
+		if (i915->display.vbt.ports[port])
+			print_ddi_port(i915->display.vbt.ports[port], port);
 	}
 }
 
@@ -2739,33 +2764,33 @@ parse_general_definitions(struct drm_i915_private *i915)
 	bus_pin = defs->crt_ddc_gmbus_pin;
 	drm_dbg_kms(&i915->drm, "crt_ddc_bus_pin: %d\n", bus_pin);
 	if (intel_gmbus_is_valid_pin(i915, bus_pin))
-		i915->vbt.crt_ddc_pin = bus_pin;
+		i915->display.vbt.crt_ddc_pin = bus_pin;
 
-	if (i915->vbt.version < 106) {
+	if (i915->display.vbt.version < 106) {
 		expected_size = 22;
-	} else if (i915->vbt.version < 111) {
+	} else if (i915->display.vbt.version < 111) {
 		expected_size = 27;
-	} else if (i915->vbt.version < 195) {
+	} else if (i915->display.vbt.version < 195) {
 		expected_size = LEGACY_CHILD_DEVICE_CONFIG_SIZE;
-	} else if (i915->vbt.version == 195) {
+	} else if (i915->display.vbt.version == 195) {
 		expected_size = 37;
-	} else if (i915->vbt.version <= 215) {
+	} else if (i915->display.vbt.version <= 215) {
 		expected_size = 38;
-	} else if (i915->vbt.version <= 237) {
+	} else if (i915->display.vbt.version <= 237) {
 		expected_size = 39;
 	} else {
 		expected_size = sizeof(*child);
 		BUILD_BUG_ON(sizeof(*child) < 39);
 		drm_dbg(&i915->drm,
 			"Expected child device config size for VBT version %u not known; assuming %u\n",
-			i915->vbt.version, expected_size);
+			i915->display.vbt.version, expected_size);
 	}
 
 	/* Flag an error for unexpected size, but continue anyway. */
 	if (defs->child_dev_size != expected_size)
 		drm_err(&i915->drm,
 			"Unexpected child device config size %u (expected %u for VBT version %u)\n",
-			defs->child_dev_size, expected_size, i915->vbt.version);
+			defs->child_dev_size, expected_size, i915->display.vbt.version);
 
 	/* The legacy sized child device config is the minimum we need. */
 	if (defs->child_dev_size < LEGACY_CHILD_DEVICE_CONFIG_SIZE) {
@@ -2801,10 +2826,10 @@ parse_general_definitions(struct drm_i915_private *i915)
 		memcpy(&devdata->child, child,
 		       min_t(size_t, defs->child_dev_size, sizeof(*child)));
 
-		list_add_tail(&devdata->node, &i915->vbt.display_devices);
+		list_add_tail(&devdata->node, &i915->display.vbt.display_devices);
 	}
 
-	if (list_empty(&i915->vbt.display_devices))
+	if (list_empty(&i915->display.vbt.display_devices))
 		drm_dbg_kms(&i915->drm,
 			    "no child dev is parsed from VBT\n");
 }
@@ -2813,25 +2838,25 @@ parse_general_definitions(struct drm_i915_private *i915)
 static void
 init_vbt_defaults(struct drm_i915_private *i915)
 {
-	i915->vbt.crt_ddc_pin = GMBUS_PIN_VGADDC;
+	i915->display.vbt.crt_ddc_pin = GMBUS_PIN_VGADDC;
 
 	/* general features */
-	i915->vbt.int_tv_support = 1;
-	i915->vbt.int_crt_support = 1;
+	i915->display.vbt.int_tv_support = 1;
+	i915->display.vbt.int_crt_support = 1;
 
 	/* driver features */
-	i915->vbt.int_lvds_support = 1;
+	i915->display.vbt.int_lvds_support = 1;
 
 	/* Default to using SSC */
-	i915->vbt.lvds_use_ssc = 1;
+	i915->display.vbt.lvds_use_ssc = 1;
 	/*
 	 * Core/SandyBridge/IvyBridge use alternative (120MHz) reference
 	 * clock for LVDS.
 	 */
-	i915->vbt.lvds_ssc_freq = intel_bios_ssc_frequency(i915,
-							   !HAS_PCH_SPLIT(i915));
+	i915->display.vbt.lvds_ssc_freq = intel_bios_ssc_frequency(i915,
+								   !HAS_PCH_SPLIT(i915));
 	drm_dbg_kms(&i915->drm, "Set default to SSC at %d kHz\n",
-		    i915->vbt.lvds_ssc_freq);
+		    i915->display.vbt.lvds_ssc_freq);
 }
 
 /* Common defaults which may be overridden by VBT. */
@@ -2892,7 +2917,7 @@ init_vbt_missing_defaults(struct drm_i915_private *i915)
 		if (port == PORT_A)
 			child->device_type |= DEVICE_TYPE_INTERNAL_CONNECTOR;
 
-		list_add_tail(&devdata->node, &i915->vbt.display_devices);
+		list_add_tail(&devdata->node, &i915->display.vbt.display_devices);
 
 		drm_dbg_kms(&i915->drm,
 			    "Generating default VBT child device with type 0x04%x on port %c\n",
@@ -2900,7 +2925,7 @@ init_vbt_missing_defaults(struct drm_i915_private *i915)
 	}
 
 	/* Bypass some minimum baseline VBT version checks */
-	i915->vbt.version = 155;
+	i915->display.vbt.version = 155;
 }
 
 static const struct bdb_header *get_bdb_header(const struct vbt_header *vbt)
@@ -3087,12 +3112,12 @@ err_unmap_oprom:
  */
 void intel_bios_init(struct drm_i915_private *i915)
 {
-	const struct vbt_header *vbt = i915->opregion.vbt;
+	const struct vbt_header *vbt = i915->display.opregion.vbt;
 	struct vbt_header *oprom_vbt = NULL;
 	const struct bdb_header *bdb;
 
-	INIT_LIST_HEAD(&i915->vbt.display_devices);
-	INIT_LIST_HEAD(&i915->vbt.bdb_blocks);
+	INIT_LIST_HEAD(&i915->display.vbt.display_devices);
+	INIT_LIST_HEAD(&i915->display.vbt.bdb_blocks);
 
 	if (!HAS_DISPLAY(i915)) {
 		drm_dbg_kms(&i915->drm,
@@ -3120,11 +3145,11 @@ void intel_bios_init(struct drm_i915_private *i915)
 		goto out;
 
 	bdb = get_bdb_header(vbt);
-	i915->vbt.version = bdb->version;
+	i915->display.vbt.version = bdb->version;
 
 	drm_dbg_kms(&i915->drm,
 		    "VBT signature \"%.*s\", BDB version %d\n",
-		    (int)sizeof(vbt->signature), vbt->signature, i915->vbt.version);
+		    (int)sizeof(vbt->signature), vbt->signature, i915->display.vbt.version);
 
 	init_bdb_blocks(i915, bdb);
 
@@ -3181,13 +3206,13 @@ void intel_bios_driver_remove(struct drm_i915_private *i915)
 	struct intel_bios_encoder_data *devdata, *nd;
 	struct bdb_block_entry *entry, *ne;
 
-	list_for_each_entry_safe(devdata, nd, &i915->vbt.display_devices, node) {
+	list_for_each_entry_safe(devdata, nd, &i915->display.vbt.display_devices, node) {
 		list_del(&devdata->node);
 		kfree(devdata->dsc);
 		kfree(devdata);
 	}
 
-	list_for_each_entry_safe(entry, ne, &i915->vbt.bdb_blocks, node) {
+	list_for_each_entry_safe(entry, ne, &i915->display.vbt.bdb_blocks, node) {
 		list_del(&entry->node);
 		kfree(entry);
 	}
@@ -3221,13 +3246,13 @@ bool intel_bios_is_tv_present(struct drm_i915_private *i915)
 	const struct intel_bios_encoder_data *devdata;
 	const struct child_device_config *child;
 
-	if (!i915->vbt.int_tv_support)
+	if (!i915->display.vbt.int_tv_support)
 		return false;
 
-	if (list_empty(&i915->vbt.display_devices))
+	if (list_empty(&i915->display.vbt.display_devices))
 		return true;
 
-	list_for_each_entry(devdata, &i915->vbt.display_devices, node) {
+	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
 		child = &devdata->child;
 
 		/*
@@ -3264,10 +3289,10 @@ bool intel_bios_is_lvds_present(struct drm_i915_private *i915, u8 *i2c_pin)
 	const struct intel_bios_encoder_data *devdata;
 	const struct child_device_config *child;
 
-	if (list_empty(&i915->vbt.display_devices))
+	if (list_empty(&i915->display.vbt.display_devices))
 		return true;
 
-	list_for_each_entry(devdata, &i915->vbt.display_devices, node) {
+	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
 		child = &devdata->child;
 
 		/* If the device type is not LFP, continue.
@@ -3294,7 +3319,7 @@ bool intel_bios_is_lvds_present(struct drm_i915_private *i915, u8 *i2c_pin)
 		 * additional data.  Trust that if the VBT was written into
 		 * the OpRegion then they have validated the LVDS's existence.
 		 */
-		if (i915->opregion.vbt)
+		if (i915->display.opregion.vbt)
 			return true;
 	}
 
@@ -3313,7 +3338,7 @@ bool intel_bios_is_port_present(struct drm_i915_private *i915, enum port port)
 	if (WARN_ON(!has_ddi_port_info(i915)))
 		return true;
 
-	return i915->vbt.ports[port];
+	return i915->display.vbt.ports[port];
 }
 
 /**
@@ -3373,7 +3398,7 @@ bool intel_bios_is_dsi_present(struct drm_i915_private *i915,
 	const struct child_device_config *child;
 	u8 dvo_port;
 
-	list_for_each_entry(devdata, &i915->vbt.display_devices, node) {
+	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
 		child = &devdata->child;
 
 		if (!(child->device_type & DEVICE_TYPE_MIPI_OUTPUT))
@@ -3472,7 +3497,7 @@ bool intel_bios_get_dsc_params(struct intel_encoder *encoder,
 	const struct intel_bios_encoder_data *devdata;
 	const struct child_device_config *child;
 
-	list_for_each_entry(devdata, &i915->vbt.display_devices, node) {
+	list_for_each_entry(devdata, &i915->display.vbt.display_devices, node) {
 		child = &devdata->child;
 
 		if (!(child->device_type & DEVICE_TYPE_MIPI_OUTPUT))
@@ -3503,7 +3528,7 @@ bool
 intel_bios_is_port_hpd_inverted(const struct drm_i915_private *i915,
 				enum port port)
 {
-	const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port];
+	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port];
 
 	if (drm_WARN_ON_ONCE(&i915->drm,
 			     !IS_GEMINILAKE(i915) && !IS_BROXTON(i915)))
@@ -3523,7 +3548,7 @@ bool
 intel_bios_is_lspcon_present(const struct drm_i915_private *i915,
 			     enum port port)
 {
-	const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port];
+	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port];
 
 	return HAS_LSPCON(i915) && devdata && devdata->child.lspcon;
 }
@@ -3539,7 +3564,7 @@ bool
 intel_bios_is_lane_reversal_needed(const struct drm_i915_private *i915,
 				   enum port port)
 {
-	const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port];
+	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port];
 
 	return devdata && devdata->child.lane_reversal;
 }
@@ -3547,7 +3572,7 @@ intel_bios_is_lane_reversal_needed(const struct drm_i915_private *i915,
 enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915,
 				   enum port port)
 {
-	const struct intel_bios_encoder_data *devdata = i915->vbt.ports[port];
+	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[port];
 	enum aux_ch aux_ch;
 
 	if (!devdata || !devdata->child.aux_channel) {
@@ -3585,7 +3610,7 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915,
 			aux_ch = AUX_CH_C;
 		break;
 	case DP_AUX_D:
-		if (DISPLAY_VER(i915) == 13)
+		if (DISPLAY_VER(i915) >= 13)
 			aux_ch = AUX_CH_D_XELPD;
 		else if (IS_ALDERLAKE_S(i915))
 			aux_ch = AUX_CH_USBC3;
@@ -3595,7 +3620,7 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915,
 			aux_ch = AUX_CH_D;
 		break;
 	case DP_AUX_E:
-		if (DISPLAY_VER(i915) == 13)
+		if (DISPLAY_VER(i915) >= 13)
 			aux_ch = AUX_CH_E_XELPD;
 		else if (IS_ALDERLAKE_S(i915))
 			aux_ch = AUX_CH_USBC4;
@@ -3603,25 +3628,25 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915,
 			aux_ch = AUX_CH_E;
 		break;
 	case DP_AUX_F:
-		if (DISPLAY_VER(i915) == 13)
+		if (DISPLAY_VER(i915) >= 13)
 			aux_ch = AUX_CH_USBC1;
 		else
 			aux_ch = AUX_CH_F;
 		break;
 	case DP_AUX_G:
-		if (DISPLAY_VER(i915) == 13)
+		if (DISPLAY_VER(i915) >= 13)
 			aux_ch = AUX_CH_USBC2;
 		else
 			aux_ch = AUX_CH_G;
 		break;
 	case DP_AUX_H:
-		if (DISPLAY_VER(i915) == 13)
+		if (DISPLAY_VER(i915) >= 13)
 			aux_ch = AUX_CH_USBC3;
 		else
 			aux_ch = AUX_CH_H;
 		break;
 	case DP_AUX_I:
-		if (DISPLAY_VER(i915) == 13)
+		if (DISPLAY_VER(i915) >= 13)
 			aux_ch = AUX_CH_USBC4;
 		else
 			aux_ch = AUX_CH_I;
@@ -3641,7 +3666,7 @@ enum aux_ch intel_bios_port_aux_ch(struct drm_i915_private *i915,
 int intel_bios_max_tmds_clock(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port];
+	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port];
 
 	return _intel_bios_max_tmds_clock(devdata);
 }
@@ -3650,14 +3675,14 @@ int intel_bios_max_tmds_clock(struct intel_encoder *encoder)
 int intel_bios_hdmi_level_shift(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port];
+	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port];
 
 	return _intel_bios_hdmi_level_shift(devdata);
 }
 
 int intel_bios_encoder_dp_boost_level(const struct intel_bios_encoder_data *devdata)
 {
-	if (!devdata || devdata->i915->vbt.version < 196 || !devdata->child.iboost)
+	if (!devdata || devdata->i915->display.vbt.version < 196 || !devdata->child.iboost)
 		return 0;
 
 	return translate_iboost(devdata->child.dp_iboost_level);
@@ -3665,7 +3690,7 @@ int intel_bios_encoder_dp_boost_level(const struct intel_bios_encoder_data *devd
 
 int intel_bios_encoder_hdmi_boost_level(const struct intel_bios_encoder_data *devdata)
 {
-	if (!devdata || devdata->i915->vbt.version < 196 || !devdata->child.iboost)
+	if (!devdata || devdata->i915->display.vbt.version < 196 || !devdata->child.iboost)
 		return 0;
 
 	return translate_iboost(devdata->child.hdmi_iboost_level);
@@ -3674,15 +3699,23 @@ int intel_bios_encoder_hdmi_boost_level(const struct intel_bios_encoder_data *de
 int intel_bios_dp_max_link_rate(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port];
+	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port];
 
 	return _intel_bios_dp_max_link_rate(devdata);
 }
 
+int intel_bios_dp_max_lane_count(struct intel_encoder *encoder)
+{
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port];
+
+	return _intel_bios_dp_max_lane_count(devdata);
+}
+
 int intel_bios_alternate_ddc_pin(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	const struct intel_bios_encoder_data *devdata = i915->vbt.ports[encoder->port];
+	const struct intel_bios_encoder_data *devdata = i915->display.vbt.ports[encoder->port];
 
 	if (!devdata || !devdata->child.ddc_pin)
 		return 0;
@@ -3692,16 +3725,16 @@ int intel_bios_alternate_ddc_pin(struct intel_encoder *encoder)
 
 bool intel_bios_encoder_supports_typec_usb(const struct intel_bios_encoder_data *devdata)
 {
-	return devdata->i915->vbt.version >= 195 && devdata->child.dp_usb_type_c;
+	return devdata->i915->display.vbt.version >= 195 && devdata->child.dp_usb_type_c;
 }
 
 bool intel_bios_encoder_supports_tbt(const struct intel_bios_encoder_data *devdata)
 {
-	return devdata->i915->vbt.version >= 209 && devdata->child.tbt;
+	return devdata->i915->display.vbt.version >= 209 && devdata->child.tbt;
 }
 
 const struct intel_bios_encoder_data *
 intel_bios_encoder_data_lookup(struct drm_i915_private *i915, enum port port)
 {
-	return i915->vbt.ports[port];
+	return i915->display.vbt.ports[port];
 }
diff --git a/drivers/gpu/drm/i915/display/intel_bios.h b/drivers/gpu/drm/i915/display/intel_bios.h
index e47582b0de0a..e375405a7828 100644
--- a/drivers/gpu/drm/i915/display/intel_bios.h
+++ b/drivers/gpu/drm/i915/display/intel_bios.h
@@ -258,6 +258,7 @@ bool intel_bios_get_dsc_params(struct intel_encoder *encoder,
 int intel_bios_max_tmds_clock(struct intel_encoder *encoder);
 int intel_bios_hdmi_level_shift(struct intel_encoder *encoder);
 int intel_bios_dp_max_link_rate(struct intel_encoder *encoder);
+int intel_bios_dp_max_lane_count(struct intel_encoder *encoder);
 int intel_bios_alternate_ddc_pin(struct intel_encoder *encoder);
 bool intel_bios_port_supports_typec_usb(struct drm_i915_private *i915, enum port port);
 bool intel_bios_port_supports_tbt(struct drm_i915_private *i915, enum port port);
diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c
index 3699869ab2db..4ace026b29bd 100644
--- a/drivers/gpu/drm/i915/display/intel_bw.c
+++ b/drivers/gpu/drm/i915/display/intel_bw.c
@@ -5,15 +5,17 @@
 
 #include <drm/drm_atomic_state_helper.h>
 
+#include "i915_drv.h"
 #include "i915_reg.h"
 #include "i915_utils.h"
 #include "intel_atomic.h"
 #include "intel_bw.h"
 #include "intel_cdclk.h"
+#include "intel_display_core.h"
 #include "intel_display_types.h"
+#include "skl_watermark.h"
 #include "intel_mchbar_regs.h"
 #include "intel_pcode.h"
-#include "intel_pm.h"
 
 /* Parameters for Qclk Geyserville (QGV) */
 struct intel_qgv_point {
@@ -137,6 +139,42 @@ int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
 	return 0;
 }
 
+static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv,
+				   struct intel_qgv_point *sp, int point)
+{
+	u32 val, val2;
+	u16 dclk;
+
+	val = intel_uncore_read(&dev_priv->uncore,
+				MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
+	val2 = intel_uncore_read(&dev_priv->uncore,
+				 MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
+	dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
+	sp->dclk = DIV_ROUND_UP((16667 * dclk), 1000);
+	sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
+	sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
+
+	sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
+	sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
+
+	sp->t_rc = sp->t_rp + sp->t_ras;
+
+	return 0;
+}
+
+static int
+intel_read_qgv_point_info(struct drm_i915_private *dev_priv,
+			  struct intel_qgv_point *sp,
+			  int point)
+{
+	if (DISPLAY_VER(dev_priv) >= 14)
+		return mtl_read_qgv_point_info(dev_priv, sp, point);
+	else if (IS_DG1(dev_priv))
+		return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point);
+	else
+		return icl_pcode_read_qgv_point_info(dev_priv, sp, point);
+}
+
 static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
 			      struct intel_qgv_info *qi,
 			      bool is_y_tile)
@@ -147,7 +185,32 @@ static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
 	qi->num_points = dram_info->num_qgv_points;
 	qi->num_psf_points = dram_info->num_psf_gv_points;
 
-	if (DISPLAY_VER(dev_priv) >= 12)
+	if (DISPLAY_VER(dev_priv) >= 14) {
+		switch (dram_info->type) {
+		case INTEL_DRAM_DDR4:
+			qi->t_bl = 4;
+			qi->max_numchannels = 2;
+			qi->channel_width = 64;
+			qi->deinterleave = 2;
+			break;
+		case INTEL_DRAM_DDR5:
+			qi->t_bl = 8;
+			qi->max_numchannels = 4;
+			qi->channel_width = 32;
+			qi->deinterleave = 2;
+			break;
+		case INTEL_DRAM_LPDDR4:
+		case INTEL_DRAM_LPDDR5:
+			qi->t_bl = 16;
+			qi->max_numchannels = 8;
+			qi->channel_width = 16;
+			qi->deinterleave = 4;
+			break;
+		default:
+			MISSING_CASE(dram_info->type);
+			return -EINVAL;
+		}
+	} else if (DISPLAY_VER(dev_priv) >= 12) {
 		switch (dram_info->type) {
 		case INTEL_DRAM_DDR4:
 			qi->t_bl = is_y_tile ? 8 : 4;
@@ -181,7 +244,7 @@ static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
 			qi->max_numchannels = 1;
 			break;
 		}
-	else if (DISPLAY_VER(dev_priv) == 11) {
+	} else if (DISPLAY_VER(dev_priv) == 11) {
 		qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
 		qi->max_numchannels = 1;
 	}
@@ -193,11 +256,7 @@ static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
 	for (i = 0; i < qi->num_points; i++) {
 		struct intel_qgv_point *sp = &qi->points[i];
 
-		if (IS_DG1(dev_priv))
-			ret = dg1_mchbar_read_qgv_point_info(dev_priv, sp, i);
-		else
-			ret = icl_pcode_read_qgv_point_info(dev_priv, sp, i);
-
+		ret = intel_read_qgv_point_info(dev_priv, sp, i);
 		if (ret)
 			return ret;
 
@@ -284,6 +343,13 @@ static const struct intel_sa_info adlp_sa_info = {
 	.derating = 20,
 };
 
+static const struct intel_sa_info mtl_sa_info = {
+	.deburst = 32,
+	.deprogbwlimit = 38, /* GB/s */
+	.displayrtids = 256,
+	.derating = 20,
+};
+
 static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
 {
 	struct intel_qgv_info qi = {};
@@ -292,7 +358,7 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel
 	int ipqdepth, ipqdepthpch = 16;
 	int dclk_max;
 	int maxdebw;
-	int num_groups = ARRAY_SIZE(dev_priv->max_bw);
+	int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
 	int i, ret;
 
 	ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
@@ -308,7 +374,7 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel
 	qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 
 	for (i = 0; i < num_groups; i++) {
-		struct intel_bw_info *bi = &dev_priv->max_bw[i];
+		struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
 		int clpchgroup;
 		int j;
 
@@ -346,9 +412,9 @@ static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel
 	 * as it will fail and pointless anyway.
 	 */
 	if (qi.num_points == 1)
-		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
+		dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 	else
-		dev_priv->sagv_status = I915_SAGV_ENABLED;
+		dev_priv->display.sagv.status = I915_SAGV_ENABLED;
 
 	return 0;
 }
@@ -363,7 +429,7 @@ static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel
 	int dclk_max;
 	int maxdebw, peakbw;
 	int clperchgroup;
-	int num_groups = ARRAY_SIZE(dev_priv->max_bw);
+	int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
 	int i, ret;
 
 	ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
@@ -399,7 +465,7 @@ static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel
 	clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
 
 	for (i = 0; i < num_groups; i++) {
-		struct intel_bw_info *bi = &dev_priv->max_bw[i];
+		struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
 		struct intel_bw_info *bi_next;
 		int clpchgroup;
 		int j;
@@ -407,7 +473,7 @@ static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel
 		clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
 
 		if (i < num_groups - 1) {
-			bi_next = &dev_priv->max_bw[i + 1];
+			bi_next = &dev_priv->display.bw.max[i + 1];
 
 			if (clpchgroup < clperchgroup)
 				bi_next->num_planes = (ipqdepth - clpchgroup) /
@@ -458,9 +524,9 @@ static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel
 	 * as it will fail and pointless anyway.
 	 */
 	if (qi.num_points == 1)
-		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
+		dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 	else
-		dev_priv->sagv_status = I915_SAGV_ENABLED;
+		dev_priv->display.sagv.status = I915_SAGV_ENABLED;
 
 	return 0;
 }
@@ -468,7 +534,7 @@ static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel
 static void dg2_get_bw_info(struct drm_i915_private *i915)
 {
 	unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000;
-	int num_groups = ARRAY_SIZE(i915->max_bw);
+	int num_groups = ARRAY_SIZE(i915->display.bw.max);
 	int i;
 
 	/*
@@ -479,7 +545,7 @@ static void dg2_get_bw_info(struct drm_i915_private *i915)
 	 * whereas DG2-G11 platforms have 38 GB/s.
 	 */
 	for (i = 0; i < num_groups; i++) {
-		struct intel_bw_info *bi = &i915->max_bw[i];
+		struct intel_bw_info *bi = &i915->display.bw.max[i];
 
 		bi->num_planes = 1;
 		/* Need only one dummy QGV point per group */
@@ -487,7 +553,7 @@ static void dg2_get_bw_info(struct drm_i915_private *i915)
 		bi->deratedbw[0] = deratedbw;
 	}
 
-	i915->sagv_status = I915_SAGV_NOT_CONTROLLED;
+	i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
 }
 
 static unsigned int icl_max_bw(struct drm_i915_private *dev_priv,
@@ -500,9 +566,9 @@ static unsigned int icl_max_bw(struct drm_i915_private *dev_priv,
 	 */
 	num_planes = max(1, num_planes);
 
-	for (i = 0; i < ARRAY_SIZE(dev_priv->max_bw); i++) {
+	for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) {
 		const struct intel_bw_info *bi =
-			&dev_priv->max_bw[i];
+			&dev_priv->display.bw.max[i];
 
 		/*
 		 * Pcode will not expose all QGV points when
@@ -528,9 +594,9 @@ static unsigned int tgl_max_bw(struct drm_i915_private *dev_priv,
 	 */
 	num_planes = max(1, num_planes);
 
-	for (i = ARRAY_SIZE(dev_priv->max_bw) - 1; i >= 0; i--) {
+	for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) {
 		const struct intel_bw_info *bi =
-			&dev_priv->max_bw[i];
+			&dev_priv->display.bw.max[i];
 
 		/*
 		 * Pcode will not expose all QGV points when
@@ -543,14 +609,14 @@ static unsigned int tgl_max_bw(struct drm_i915_private *dev_priv,
 			return bi->deratedbw[qgv_point];
 	}
 
-	return dev_priv->max_bw[0].deratedbw[qgv_point];
+	return dev_priv->display.bw.max[0].deratedbw[qgv_point];
 }
 
 static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
 			       int psf_gv_point)
 {
 	const struct intel_bw_info *bi =
-			&dev_priv->max_bw[0];
+			&dev_priv->display.bw.max[0];
 
 	return bi->psf_bw[psf_gv_point];
 }
@@ -560,7 +626,9 @@ void intel_bw_init_hw(struct drm_i915_private *dev_priv)
 	if (!HAS_DISPLAY(dev_priv))
 		return;
 
-	if (IS_DG2(dev_priv))
+	if (DISPLAY_VER(dev_priv) >= 14)
+		tgl_get_bw_info(dev_priv, &mtl_sa_info);
+	else if (IS_DG2(dev_priv))
 		dg2_get_bw_info(dev_priv);
 	else if (IS_ALDERLAKE_P(dev_priv))
 		tgl_get_bw_info(dev_priv, &adlp_sa_info);
@@ -669,7 +737,7 @@ intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 	struct intel_global_state *bw_state;
 
-	bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->bw_obj);
+	bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj);
 
 	return to_intel_bw_state(bw_state);
 }
@@ -680,7 +748,7 @@ intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 	struct intel_global_state *bw_state;
 
-	bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->bw_obj);
+	bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj);
 
 	return to_intel_bw_state(bw_state);
 }
@@ -691,7 +759,7 @@ intel_atomic_get_bw_state(struct intel_atomic_state *state)
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 	struct intel_global_state *bw_state;
 
-	bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->bw_obj);
+	bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj);
 	if (IS_ERR(bw_state))
 		return ERR_CAST(bw_state);
 
@@ -898,8 +966,8 @@ int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
 
 static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
 {
-	unsigned int num_psf_gv_points = i915->max_bw[0].num_psf_gv_points;
-	unsigned int num_qgv_points = i915->max_bw[0].num_qgv_points;
+	unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
+	unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
 	u16 qgv_points = 0, psf_points = 0;
 
 	/*
@@ -972,8 +1040,8 @@ int intel_bw_atomic_check(struct intel_atomic_state *state)
 	int i, ret;
 	u16 qgv_points = 0, psf_points = 0;
 	unsigned int max_bw_point = 0, max_bw = 0;
-	unsigned int num_qgv_points = dev_priv->max_bw[0].num_qgv_points;
-	unsigned int num_psf_gv_points = dev_priv->max_bw[0].num_psf_gv_points;
+	unsigned int num_qgv_points = dev_priv->display.bw.max[0].num_qgv_points;
+	unsigned int num_psf_gv_points = dev_priv->display.bw.max[0].num_psf_gv_points;
 	bool changed = false;
 
 	/* FIXME earlier gens need some checks too */
@@ -1128,7 +1196,7 @@ int intel_bw_init(struct drm_i915_private *dev_priv)
 	if (!state)
 		return -ENOMEM;
 
-	intel_atomic_global_obj_init(dev_priv, &dev_priv->bw_obj,
+	intel_atomic_global_obj_init(dev_priv, &dev_priv->display.bw.obj,
 				     &state->base, &intel_bw_funcs);
 
 	return 0;
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c
index 6e80162632dd..ed05070b7307 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.c
@@ -79,26 +79,26 @@ struct intel_cdclk_funcs {
 void intel_cdclk_get_cdclk(struct drm_i915_private *dev_priv,
 			   struct intel_cdclk_config *cdclk_config)
 {
-	dev_priv->cdclk_funcs->get_cdclk(dev_priv, cdclk_config);
+	dev_priv->display.funcs.cdclk->get_cdclk(dev_priv, cdclk_config);
 }
 
 static void intel_cdclk_set_cdclk(struct drm_i915_private *dev_priv,
 				  const struct intel_cdclk_config *cdclk_config,
 				  enum pipe pipe)
 {
-	dev_priv->cdclk_funcs->set_cdclk(dev_priv, cdclk_config, pipe);
+	dev_priv->display.funcs.cdclk->set_cdclk(dev_priv, cdclk_config, pipe);
 }
 
 static int intel_cdclk_modeset_calc_cdclk(struct drm_i915_private *dev_priv,
 					  struct intel_cdclk_state *cdclk_config)
 {
-	return dev_priv->cdclk_funcs->modeset_calc_cdclk(cdclk_config);
+	return dev_priv->display.funcs.cdclk->modeset_calc_cdclk(cdclk_config);
 }
 
 static u8 intel_cdclk_calc_voltage_level(struct drm_i915_private *dev_priv,
 					 int cdclk)
 {
-	return dev_priv->cdclk_funcs->calc_voltage_level(cdclk);
+	return dev_priv->display.funcs.cdclk->calc_voltage_level(cdclk);
 }
 
 static void fixed_133mhz_get_cdclk(struct drm_i915_private *dev_priv,
@@ -548,7 +548,7 @@ static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 	else
 		default_credits = PFI_CREDIT(8);
 
-	if (dev_priv->cdclk.hw.cdclk >= dev_priv->czclk_freq) {
+	if (dev_priv->display.cdclk.hw.cdclk >= dev_priv->czclk_freq) {
 		/* CHV suggested value is 31 or 63 */
 		if (IS_CHERRYVIEW(dev_priv))
 			credits = PFI_CREDIT_63;
@@ -1026,7 +1026,7 @@ static void skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
 	if (intel_de_wait_for_set(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 5))
 		drm_err(&dev_priv->drm, "DPLL0 not locked\n");
 
-	dev_priv->cdclk.hw.vco = vco;
+	dev_priv->display.cdclk.hw.vco = vco;
 
 	/* We'll want to keep using the current vco from now on. */
 	skl_set_preferred_cdclk_vco(dev_priv, vco);
@@ -1040,7 +1040,7 @@ static void skl_dpll0_disable(struct drm_i915_private *dev_priv)
 	if (intel_de_wait_for_clear(dev_priv, LCPLL1_CTL, LCPLL_PLL_LOCK, 1))
 		drm_err(&dev_priv->drm, "Couldn't disable DPLL0\n");
 
-	dev_priv->cdclk.hw.vco = 0;
+	dev_priv->display.cdclk.hw.vco = 0;
 }
 
 static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
@@ -1049,7 +1049,7 @@ static u32 skl_cdclk_freq_sel(struct drm_i915_private *dev_priv,
 	switch (cdclk) {
 	default:
 		drm_WARN_ON(&dev_priv->drm,
-			    cdclk != dev_priv->cdclk.hw.bypass);
+			    cdclk != dev_priv->display.cdclk.hw.bypass);
 		drm_WARN_ON(&dev_priv->drm, vco != 0);
 		fallthrough;
 	case 308571:
@@ -1098,13 +1098,13 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv,
 
 	freq_select = skl_cdclk_freq_sel(dev_priv, cdclk, vco);
 
-	if (dev_priv->cdclk.hw.vco != 0 &&
-	    dev_priv->cdclk.hw.vco != vco)
+	if (dev_priv->display.cdclk.hw.vco != 0 &&
+	    dev_priv->display.cdclk.hw.vco != vco)
 		skl_dpll0_disable(dev_priv);
 
 	cdclk_ctl = intel_de_read(dev_priv, CDCLK_CTL);
 
-	if (dev_priv->cdclk.hw.vco != vco) {
+	if (dev_priv->display.cdclk.hw.vco != vco) {
 		/* Wa Display #1183: skl,kbl,cfl */
 		cdclk_ctl &= ~(CDCLK_FREQ_SEL_MASK | CDCLK_FREQ_DECIMAL_MASK);
 		cdclk_ctl |= freq_select | skl_cdclk_decimal(cdclk);
@@ -1116,7 +1116,7 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv,
 	intel_de_write(dev_priv, CDCLK_CTL, cdclk_ctl);
 	intel_de_posting_read(dev_priv, CDCLK_CTL);
 
-	if (dev_priv->cdclk.hw.vco != vco)
+	if (dev_priv->display.cdclk.hw.vco != vco)
 		skl_dpll0_enable(dev_priv, vco);
 
 	/* Wa Display #1183: skl,kbl,cfl */
@@ -1151,11 +1151,11 @@ static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
 		goto sanitize;
 
 	intel_update_cdclk(dev_priv);
-	intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK");
+	intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
 
 	/* Is PLL enabled and locked ? */
-	if (dev_priv->cdclk.hw.vco == 0 ||
-	    dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
+	if (dev_priv->display.cdclk.hw.vco == 0 ||
+	    dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
 		goto sanitize;
 
 	/* DPLL okay; verify the cdclock
@@ -1166,7 +1166,7 @@ static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
 	 */
 	cdctl = intel_de_read(dev_priv, CDCLK_CTL);
 	expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
-		skl_cdclk_decimal(dev_priv->cdclk.hw.cdclk);
+		skl_cdclk_decimal(dev_priv->display.cdclk.hw.cdclk);
 	if (cdctl == expected)
 		/* All well; nothing to sanitize */
 		return;
@@ -1175,9 +1175,9 @@ sanitize:
 	drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
 
 	/* force cdclk programming */
-	dev_priv->cdclk.hw.cdclk = 0;
+	dev_priv->display.cdclk.hw.cdclk = 0;
 	/* force full PLL disable + enable */
-	dev_priv->cdclk.hw.vco = -1;
+	dev_priv->display.cdclk.hw.vco = -1;
 }
 
 static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
@@ -1186,19 +1186,19 @@ static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
 
 	skl_sanitize_cdclk(dev_priv);
 
-	if (dev_priv->cdclk.hw.cdclk != 0 &&
-	    dev_priv->cdclk.hw.vco != 0) {
+	if (dev_priv->display.cdclk.hw.cdclk != 0 &&
+	    dev_priv->display.cdclk.hw.vco != 0) {
 		/*
 		 * Use the current vco as our initial
 		 * guess as to what the preferred vco is.
 		 */
 		if (dev_priv->skl_preferred_vco_freq == 0)
 			skl_set_preferred_cdclk_vco(dev_priv,
-						    dev_priv->cdclk.hw.vco);
+						    dev_priv->display.cdclk.hw.vco);
 		return;
 	}
 
-	cdclk_config = dev_priv->cdclk.hw;
+	cdclk_config = dev_priv->display.cdclk.hw;
 
 	cdclk_config.vco = dev_priv->skl_preferred_vco_freq;
 	if (cdclk_config.vco == 0)
@@ -1211,7 +1211,7 @@ static void skl_cdclk_init_hw(struct drm_i915_private *dev_priv)
 
 static void skl_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
 {
-	struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
+	struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
 
 	cdclk_config.cdclk = cdclk_config.bypass;
 	cdclk_config.vco = 0;
@@ -1352,35 +1352,35 @@ static const struct intel_cdclk_vals dg2_cdclk_table[] = {
 
 static int bxt_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
 {
-	const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
+	const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
 	int i;
 
 	for (i = 0; table[i].refclk; i++)
-		if (table[i].refclk == dev_priv->cdclk.hw.ref &&
+		if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
 		    table[i].cdclk >= min_cdclk)
 			return table[i].cdclk;
 
 	drm_WARN(&dev_priv->drm, 1,
 		 "Cannot satisfy minimum cdclk %d with refclk %u\n",
-		 min_cdclk, dev_priv->cdclk.hw.ref);
+		 min_cdclk, dev_priv->display.cdclk.hw.ref);
 	return 0;
 }
 
 static int bxt_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
 {
-	const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
+	const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
 	int i;
 
-	if (cdclk == dev_priv->cdclk.hw.bypass)
+	if (cdclk == dev_priv->display.cdclk.hw.bypass)
 		return 0;
 
 	for (i = 0; table[i].refclk; i++)
-		if (table[i].refclk == dev_priv->cdclk.hw.ref &&
+		if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
 		    table[i].cdclk == cdclk)
-			return dev_priv->cdclk.hw.ref * table[i].ratio;
+			return dev_priv->display.cdclk.hw.ref * table[i].ratio;
 
 	drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
-		 cdclk, dev_priv->cdclk.hw.ref);
+		 cdclk, dev_priv->display.cdclk.hw.ref);
 	return 0;
 }
 
@@ -1554,12 +1554,12 @@ static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
 				    BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
 		drm_err(&dev_priv->drm, "timeout waiting for DE PLL unlock\n");
 
-	dev_priv->cdclk.hw.vco = 0;
+	dev_priv->display.cdclk.hw.vco = 0;
 }
 
 static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
 {
-	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
+	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
 
 	intel_de_rmw(dev_priv, BXT_DE_PLL_CTL,
 		     BXT_DE_PLL_RATIO_MASK, BXT_DE_PLL_RATIO(ratio));
@@ -1571,7 +1571,7 @@ static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
 				  BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
 		drm_err(&dev_priv->drm, "timeout waiting for DE PLL lock\n");
 
-	dev_priv->cdclk.hw.vco = vco;
+	dev_priv->display.cdclk.hw.vco = vco;
 }
 
 static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
@@ -1583,12 +1583,12 @@ static void icl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
 	if (intel_de_wait_for_clear(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
 		drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL unlock\n");
 
-	dev_priv->cdclk.hw.vco = 0;
+	dev_priv->display.cdclk.hw.vco = 0;
 }
 
 static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
 {
-	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
+	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
 	u32 val;
 
 	val = ICL_CDCLK_PLL_RATIO(ratio);
@@ -1601,12 +1601,12 @@ static void icl_cdclk_pll_enable(struct drm_i915_private *dev_priv, int vco)
 	if (intel_de_wait_for_set(dev_priv, BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 1))
 		drm_err(&dev_priv->drm, "timeout waiting for CDCLK PLL lock\n");
 
-	dev_priv->cdclk.hw.vco = vco;
+	dev_priv->display.cdclk.hw.vco = vco;
 }
 
 static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
 {
-	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk.hw.ref);
+	int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->display.cdclk.hw.ref);
 	u32 val;
 
 	/* Write PLL ratio without disabling */
@@ -1625,7 +1625,7 @@ static void adlp_cdclk_pll_crawl(struct drm_i915_private *dev_priv, int vco)
 	val &= ~BXT_DE_PLL_FREQ_REQ;
 	intel_de_write(dev_priv, BXT_DE_PLL_ENABLE, val);
 
-	dev_priv->cdclk.hw.vco = vco;
+	dev_priv->display.cdclk.hw.vco = vco;
 }
 
 static u32 bxt_cdclk_cd2x_pipe(struct drm_i915_private *dev_priv, enum pipe pipe)
@@ -1655,7 +1655,7 @@ static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
 	switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
 	default:
 		drm_WARN_ON(&dev_priv->drm,
-			    cdclk != dev_priv->cdclk.hw.bypass);
+			    cdclk != dev_priv->display.cdclk.hw.bypass);
 		drm_WARN_ON(&dev_priv->drm, vco != 0);
 		fallthrough;
 	case 2:
@@ -1672,19 +1672,19 @@ static u32 bxt_cdclk_cd2x_div_sel(struct drm_i915_private *dev_priv,
 static u32 cdclk_squash_waveform(struct drm_i915_private *dev_priv,
 				 int cdclk)
 {
-	const struct intel_cdclk_vals *table = dev_priv->cdclk.table;
+	const struct intel_cdclk_vals *table = dev_priv->display.cdclk.table;
 	int i;
 
-	if (cdclk == dev_priv->cdclk.hw.bypass)
+	if (cdclk == dev_priv->display.cdclk.hw.bypass)
 		return 0;
 
 	for (i = 0; table[i].refclk; i++)
-		if (table[i].refclk == dev_priv->cdclk.hw.ref &&
+		if (table[i].refclk == dev_priv->display.cdclk.hw.ref &&
 		    table[i].cdclk == cdclk)
 			return table[i].waveform;
 
 	drm_WARN(&dev_priv->drm, 1, "cdclk %d not valid for refclk %u\n",
-		 cdclk, dev_priv->cdclk.hw.ref);
+		 cdclk, dev_priv->display.cdclk.hw.ref);
 
 	return 0xffff;
 }
@@ -1721,22 +1721,22 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
 		return;
 	}
 
-	if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->cdclk.hw.vco > 0 && vco > 0) {
-		if (dev_priv->cdclk.hw.vco != vco)
+	if (HAS_CDCLK_CRAWL(dev_priv) && dev_priv->display.cdclk.hw.vco > 0 && vco > 0) {
+		if (dev_priv->display.cdclk.hw.vco != vco)
 			adlp_cdclk_pll_crawl(dev_priv, vco);
 	} else if (DISPLAY_VER(dev_priv) >= 11) {
-		if (dev_priv->cdclk.hw.vco != 0 &&
-		    dev_priv->cdclk.hw.vco != vco)
+		if (dev_priv->display.cdclk.hw.vco != 0 &&
+		    dev_priv->display.cdclk.hw.vco != vco)
 			icl_cdclk_pll_disable(dev_priv);
 
-		if (dev_priv->cdclk.hw.vco != vco)
+		if (dev_priv->display.cdclk.hw.vco != vco)
 			icl_cdclk_pll_enable(dev_priv, vco);
 	} else {
-		if (dev_priv->cdclk.hw.vco != 0 &&
-		    dev_priv->cdclk.hw.vco != vco)
+		if (dev_priv->display.cdclk.hw.vco != 0 &&
+		    dev_priv->display.cdclk.hw.vco != vco)
 			bxt_de_pll_disable(dev_priv);
 
-		if (dev_priv->cdclk.hw.vco != vco)
+		if (dev_priv->display.cdclk.hw.vco != vco)
 			bxt_de_pll_enable(dev_priv, vco);
 	}
 
@@ -1803,7 +1803,7 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
 		 * Can't read out the voltage level :(
 		 * Let's just assume everything is as expected.
 		 */
-		dev_priv->cdclk.hw.voltage_level = cdclk_config->voltage_level;
+		dev_priv->display.cdclk.hw.voltage_level = cdclk_config->voltage_level;
 }
 
 static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
@@ -1812,10 +1812,10 @@ static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
 	int cdclk, clock, vco;
 
 	intel_update_cdclk(dev_priv);
-	intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK");
+	intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
 
-	if (dev_priv->cdclk.hw.vco == 0 ||
-	    dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.bypass)
+	if (dev_priv->display.cdclk.hw.vco == 0 ||
+	    dev_priv->display.cdclk.hw.cdclk == dev_priv->display.cdclk.hw.bypass)
 		goto sanitize;
 
 	/* DPLL okay; verify the cdclock
@@ -1833,32 +1833,32 @@ static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
 	cdctl &= ~bxt_cdclk_cd2x_pipe(dev_priv, INVALID_PIPE);
 
 	/* Make sure this is a legal cdclk value for the platform */
-	cdclk = bxt_calc_cdclk(dev_priv, dev_priv->cdclk.hw.cdclk);
-	if (cdclk != dev_priv->cdclk.hw.cdclk)
+	cdclk = bxt_calc_cdclk(dev_priv, dev_priv->display.cdclk.hw.cdclk);
+	if (cdclk != dev_priv->display.cdclk.hw.cdclk)
 		goto sanitize;
 
 	/* Make sure the VCO is correct for the cdclk */
 	vco = bxt_calc_cdclk_pll_vco(dev_priv, cdclk);
-	if (vco != dev_priv->cdclk.hw.vco)
+	if (vco != dev_priv->display.cdclk.hw.vco)
 		goto sanitize;
 
 	expected = skl_cdclk_decimal(cdclk);
 
 	/* Figure out what CD2X divider we should be using for this cdclk */
 	if (has_cdclk_squasher(dev_priv))
-		clock = dev_priv->cdclk.hw.vco / 2;
+		clock = dev_priv->display.cdclk.hw.vco / 2;
 	else
-		clock = dev_priv->cdclk.hw.cdclk;
+		clock = dev_priv->display.cdclk.hw.cdclk;
 
 	expected |= bxt_cdclk_cd2x_div_sel(dev_priv, clock,
-					   dev_priv->cdclk.hw.vco);
+					   dev_priv->display.cdclk.hw.vco);
 
 	/*
 	 * Disable SSA Precharge when CD clock frequency < 500 MHz,
 	 * enable otherwise.
 	 */
 	if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
-	    dev_priv->cdclk.hw.cdclk >= 500000)
+	    dev_priv->display.cdclk.hw.cdclk >= 500000)
 		expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
 
 	if (cdctl == expected)
@@ -1869,10 +1869,10 @@ sanitize:
 	drm_dbg_kms(&dev_priv->drm, "Sanitizing cdclk programmed by pre-os\n");
 
 	/* force cdclk programming */
-	dev_priv->cdclk.hw.cdclk = 0;
+	dev_priv->display.cdclk.hw.cdclk = 0;
 
 	/* force full PLL disable + enable */
-	dev_priv->cdclk.hw.vco = -1;
+	dev_priv->display.cdclk.hw.vco = -1;
 }
 
 static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
@@ -1881,11 +1881,11 @@ static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
 
 	bxt_sanitize_cdclk(dev_priv);
 
-	if (dev_priv->cdclk.hw.cdclk != 0 &&
-	    dev_priv->cdclk.hw.vco != 0)
+	if (dev_priv->display.cdclk.hw.cdclk != 0 &&
+	    dev_priv->display.cdclk.hw.vco != 0)
 		return;
 
-	cdclk_config = dev_priv->cdclk.hw;
+	cdclk_config = dev_priv->display.cdclk.hw;
 
 	/*
 	 * FIXME:
@@ -1902,7 +1902,7 @@ static void bxt_cdclk_init_hw(struct drm_i915_private *dev_priv)
 
 static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
 {
-	struct intel_cdclk_config cdclk_config = dev_priv->cdclk.hw;
+	struct intel_cdclk_config cdclk_config = dev_priv->display.cdclk.hw;
 
 	cdclk_config.cdclk = cdclk_config.bypass;
 	cdclk_config.vco = 0;
@@ -1916,7 +1916,7 @@ static void bxt_cdclk_uninit_hw(struct drm_i915_private *dev_priv)
  * intel_cdclk_init_hw - Initialize CDCLK hardware
  * @i915: i915 device
  *
- * Initialize CDCLK. This consists mainly of initializing dev_priv->cdclk.hw and
+ * Initialize CDCLK. This consists mainly of initializing dev_priv->display.cdclk.hw and
  * sanitizing the state of the hardware if needed. This is generally done only
  * during the display core initialization sequence, after which the DMC will
  * take care of turning CDCLK off/on as needed.
@@ -2077,10 +2077,10 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv,
 {
 	struct intel_encoder *encoder;
 
-	if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config))
+	if (!intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config))
 		return;
 
-	if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->cdclk_funcs->set_cdclk))
+	if (drm_WARN_ON_ONCE(&dev_priv->drm, !dev_priv->display.funcs.cdclk->set_cdclk))
 		return;
 
 	intel_cdclk_dump_config(dev_priv, cdclk_config, "Changing CDCLK to");
@@ -2098,12 +2098,12 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv,
 	 * functions use cdclk. Not all platforms/ports do,
 	 * but we'll lock them all for simplicity.
 	 */
-	mutex_lock(&dev_priv->gmbus_mutex);
+	mutex_lock(&dev_priv->display.gmbus.mutex);
 	for_each_intel_dp(&dev_priv->drm, encoder) {
 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 
 		mutex_lock_nest_lock(&intel_dp->aux.hw_mutex,
-				     &dev_priv->gmbus_mutex);
+				     &dev_priv->display.gmbus.mutex);
 	}
 
 	intel_cdclk_set_cdclk(dev_priv, cdclk_config, pipe);
@@ -2113,7 +2113,7 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv,
 
 		mutex_unlock(&intel_dp->aux.hw_mutex);
 	}
-	mutex_unlock(&dev_priv->gmbus_mutex);
+	mutex_unlock(&dev_priv->display.gmbus.mutex);
 
 	for_each_intel_encoder_with_psr(&dev_priv->drm, encoder) {
 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
@@ -2124,9 +2124,9 @@ static void intel_set_cdclk(struct drm_i915_private *dev_priv,
 	intel_audio_cdclk_change_post(dev_priv);
 
 	if (drm_WARN(&dev_priv->drm,
-		     intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_config),
+		     intel_cdclk_changed(&dev_priv->display.cdclk.hw, cdclk_config),
 		     "cdclk state doesn't match!\n")) {
-		intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "[hw state]");
+		intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "[hw state]");
 		intel_cdclk_dump_config(dev_priv, cdclk_config, "[sw state]");
 	}
 }
@@ -2300,7 +2300,7 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
 		min_cdclk = max(min_cdclk, (int)crtc_state->pixel_rate);
 
 	/*
-	 * HACK. Currently for TGL platforms we calculate
+	 * HACK. Currently for TGL/DG2 platforms we calculate
 	 * min_cdclk initially based on pixel_rate divided
 	 * by 2, accounting for also plane requirements,
 	 * however in some cases the lowest possible CDCLK
@@ -2308,14 +2308,14 @@ int intel_crtc_compute_min_cdclk(const struct intel_crtc_state *crtc_state)
 	 * Explicitly stating here that this seems to be currently
 	 * rather a Hack, than final solution.
 	 */
-	if (IS_TIGERLAKE(dev_priv)) {
+	if (IS_TIGERLAKE(dev_priv) || IS_DG2(dev_priv)) {
 		/*
 		 * Clamp to max_cdclk_freq in case pixel rate is higher,
 		 * in order not to break an 8K, but still leave W/A at place.
 		 */
 		min_cdclk = max_t(int, min_cdclk,
 				  min_t(int, crtc_state->pixel_rate,
-					dev_priv->max_cdclk_freq));
+					dev_priv->display.cdclk.max_cdclk_freq));
 	}
 
 	return min_cdclk;
@@ -2368,10 +2368,10 @@ static int intel_compute_min_cdclk(struct intel_cdclk_state *cdclk_state)
 	for_each_pipe(dev_priv, pipe)
 		min_cdclk = max(cdclk_state->min_cdclk[pipe], min_cdclk);
 
-	if (min_cdclk > dev_priv->max_cdclk_freq) {
+	if (min_cdclk > dev_priv->display.cdclk.max_cdclk_freq) {
 		drm_dbg_kms(&dev_priv->drm,
 			    "required cdclk (%d kHz) exceeds max (%d kHz)\n",
-			    min_cdclk, dev_priv->max_cdclk_freq);
+			    min_cdclk, dev_priv->display.cdclk.max_cdclk_freq);
 		return -EINVAL;
 	}
 
@@ -2643,7 +2643,7 @@ intel_atomic_get_cdclk_state(struct intel_atomic_state *state)
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
 	struct intel_global_state *cdclk_state;
 
-	cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->cdclk.obj);
+	cdclk_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.cdclk.obj);
 	if (IS_ERR(cdclk_state))
 		return ERR_CAST(cdclk_state);
 
@@ -2693,7 +2693,7 @@ int intel_cdclk_init(struct drm_i915_private *dev_priv)
 	if (!cdclk_state)
 		return -ENOMEM;
 
-	intel_atomic_global_obj_init(dev_priv, &dev_priv->cdclk.obj,
+	intel_atomic_global_obj_init(dev_priv, &dev_priv->display.cdclk.obj,
 				     &cdclk_state->base, &intel_cdclk_funcs);
 
 	return 0;
@@ -2799,7 +2799,7 @@ int intel_modeset_calc_cdclk(struct intel_atomic_state *state)
 
 static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
 {
-	int max_cdclk_freq = dev_priv->max_cdclk_freq;
+	int max_cdclk_freq = dev_priv->display.cdclk.max_cdclk_freq;
 
 	if (DISPLAY_VER(dev_priv) >= 10)
 		return 2 * max_cdclk_freq;
@@ -2825,19 +2825,19 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
 void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
 {
 	if (IS_JSL_EHL(dev_priv)) {
-		if (dev_priv->cdclk.hw.ref == 24000)
-			dev_priv->max_cdclk_freq = 552000;
+		if (dev_priv->display.cdclk.hw.ref == 24000)
+			dev_priv->display.cdclk.max_cdclk_freq = 552000;
 		else
-			dev_priv->max_cdclk_freq = 556800;
+			dev_priv->display.cdclk.max_cdclk_freq = 556800;
 	} else if (DISPLAY_VER(dev_priv) >= 11) {
-		if (dev_priv->cdclk.hw.ref == 24000)
-			dev_priv->max_cdclk_freq = 648000;
+		if (dev_priv->display.cdclk.hw.ref == 24000)
+			dev_priv->display.cdclk.max_cdclk_freq = 648000;
 		else
-			dev_priv->max_cdclk_freq = 652800;
+			dev_priv->display.cdclk.max_cdclk_freq = 652800;
 	} else if (IS_GEMINILAKE(dev_priv)) {
-		dev_priv->max_cdclk_freq = 316800;
+		dev_priv->display.cdclk.max_cdclk_freq = 316800;
 	} else if (IS_BROXTON(dev_priv)) {
-		dev_priv->max_cdclk_freq = 624000;
+		dev_priv->display.cdclk.max_cdclk_freq = 624000;
 	} else if (DISPLAY_VER(dev_priv) == 9) {
 		u32 limit = intel_de_read(dev_priv, SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
 		int max_cdclk, vco;
@@ -2859,7 +2859,7 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
 		else
 			max_cdclk = 308571;
 
-		dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
+		dev_priv->display.cdclk.max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
 	} else if (IS_BROADWELL(dev_priv))  {
 		/*
 		 * FIXME with extra cooling we can allow
@@ -2868,26 +2868,26 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
 		 * available? PCI ID, VTB, something else?
 		 */
 		if (intel_de_read(dev_priv, FUSE_STRAP) & HSW_CDCLK_LIMIT)
-			dev_priv->max_cdclk_freq = 450000;
+			dev_priv->display.cdclk.max_cdclk_freq = 450000;
 		else if (IS_BDW_ULX(dev_priv))
-			dev_priv->max_cdclk_freq = 450000;
+			dev_priv->display.cdclk.max_cdclk_freq = 450000;
 		else if (IS_BDW_ULT(dev_priv))
-			dev_priv->max_cdclk_freq = 540000;
+			dev_priv->display.cdclk.max_cdclk_freq = 540000;
 		else
-			dev_priv->max_cdclk_freq = 675000;
+			dev_priv->display.cdclk.max_cdclk_freq = 675000;
 	} else if (IS_CHERRYVIEW(dev_priv)) {
-		dev_priv->max_cdclk_freq = 320000;
+		dev_priv->display.cdclk.max_cdclk_freq = 320000;
 	} else if (IS_VALLEYVIEW(dev_priv)) {
-		dev_priv->max_cdclk_freq = 400000;
+		dev_priv->display.cdclk.max_cdclk_freq = 400000;
 	} else {
 		/* otherwise assume cdclk is fixed */
-		dev_priv->max_cdclk_freq = dev_priv->cdclk.hw.cdclk;
+		dev_priv->display.cdclk.max_cdclk_freq = dev_priv->display.cdclk.hw.cdclk;
 	}
 
 	dev_priv->max_dotclk_freq = intel_compute_max_dotclk(dev_priv);
 
 	drm_dbg(&dev_priv->drm, "Max CD clock rate: %d kHz\n",
-		dev_priv->max_cdclk_freq);
+		dev_priv->display.cdclk.max_cdclk_freq);
 
 	drm_dbg(&dev_priv->drm, "Max dotclock rate: %d kHz\n",
 		dev_priv->max_dotclk_freq);
@@ -2901,7 +2901,7 @@ void intel_update_max_cdclk(struct drm_i915_private *dev_priv)
  */
 void intel_update_cdclk(struct drm_i915_private *dev_priv)
 {
-	intel_cdclk_get_cdclk(dev_priv, &dev_priv->cdclk.hw);
+	intel_cdclk_get_cdclk(dev_priv, &dev_priv->display.cdclk.hw);
 
 	/*
 	 * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
@@ -2911,7 +2911,7 @@ void intel_update_cdclk(struct drm_i915_private *dev_priv)
 	 */
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		intel_de_write(dev_priv, GMBUSFREQ_VLV,
-			       DIV_ROUND_UP(dev_priv->cdclk.hw.cdclk, 1000));
+			       DIV_ROUND_UP(dev_priv->display.cdclk.hw.cdclk, 1000));
 }
 
 static int dg1_rawclk(struct drm_i915_private *dev_priv)
@@ -3036,6 +3036,13 @@ u32 intel_read_rawclk(struct drm_i915_private *dev_priv)
 
 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_DG1)
 		freq = dg1_rawclk(dev_priv);
+	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_MTP)
+		/*
+		 * MTL always uses a 38.4 MHz rawclk.  The bspec tells us
+		 * "RAWCLK_FREQ defaults to the values for 38.4 and does
+		 * not need to be programmed."
+		 */
+		freq = 38400;
 	else if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
 		freq = cnp_rawclk(dev_priv);
 	else if (HAS_PCH_SPLIT(dev_priv))
@@ -3187,78 +3194,78 @@ static const struct intel_cdclk_funcs i830_cdclk_funcs = {
 void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv)
 {
 	if (IS_DG2(dev_priv)) {
-		dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
-		dev_priv->cdclk.table = dg2_cdclk_table;
+		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
+		dev_priv->display.cdclk.table = dg2_cdclk_table;
 	} else if (IS_ALDERLAKE_P(dev_priv)) {
-		dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
 		/* Wa_22011320316:adl-p[a0] */
 		if (IS_ADLP_DISPLAY_STEP(dev_priv, STEP_A0, STEP_B0))
-			dev_priv->cdclk.table = adlp_a_step_cdclk_table;
+			dev_priv->display.cdclk.table = adlp_a_step_cdclk_table;
 		else
-			dev_priv->cdclk.table = adlp_cdclk_table;
+			dev_priv->display.cdclk.table = adlp_cdclk_table;
 	} else if (IS_ROCKETLAKE(dev_priv)) {
-		dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
-		dev_priv->cdclk.table = rkl_cdclk_table;
+		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
+		dev_priv->display.cdclk.table = rkl_cdclk_table;
 	} else if (DISPLAY_VER(dev_priv) >= 12) {
-		dev_priv->cdclk_funcs = &tgl_cdclk_funcs;
-		dev_priv->cdclk.table = icl_cdclk_table;
+		dev_priv->display.funcs.cdclk = &tgl_cdclk_funcs;
+		dev_priv->display.cdclk.table = icl_cdclk_table;
 	} else if (IS_JSL_EHL(dev_priv)) {
-		dev_priv->cdclk_funcs = &ehl_cdclk_funcs;
-		dev_priv->cdclk.table = icl_cdclk_table;
+		dev_priv->display.funcs.cdclk = &ehl_cdclk_funcs;
+		dev_priv->display.cdclk.table = icl_cdclk_table;
 	} else if (DISPLAY_VER(dev_priv) >= 11) {
-		dev_priv->cdclk_funcs = &icl_cdclk_funcs;
-		dev_priv->cdclk.table = icl_cdclk_table;
+		dev_priv->display.funcs.cdclk = &icl_cdclk_funcs;
+		dev_priv->display.cdclk.table = icl_cdclk_table;
 	} else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) {
-		dev_priv->cdclk_funcs = &bxt_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &bxt_cdclk_funcs;
 		if (IS_GEMINILAKE(dev_priv))
-			dev_priv->cdclk.table = glk_cdclk_table;
+			dev_priv->display.cdclk.table = glk_cdclk_table;
 		else
-			dev_priv->cdclk.table = bxt_cdclk_table;
+			dev_priv->display.cdclk.table = bxt_cdclk_table;
 	} else if (DISPLAY_VER(dev_priv) == 9) {
-		dev_priv->cdclk_funcs = &skl_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &skl_cdclk_funcs;
 	} else if (IS_BROADWELL(dev_priv)) {
-		dev_priv->cdclk_funcs = &bdw_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &bdw_cdclk_funcs;
 	} else if (IS_HASWELL(dev_priv)) {
-		dev_priv->cdclk_funcs = &hsw_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &hsw_cdclk_funcs;
 	} else if (IS_CHERRYVIEW(dev_priv)) {
-		dev_priv->cdclk_funcs = &chv_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &chv_cdclk_funcs;
 	} else if (IS_VALLEYVIEW(dev_priv)) {
-		dev_priv->cdclk_funcs = &vlv_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &vlv_cdclk_funcs;
 	} else if (IS_SANDYBRIDGE(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
-		dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
 	} else if (IS_IRONLAKE(dev_priv)) {
-		dev_priv->cdclk_funcs = &ilk_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &ilk_cdclk_funcs;
 	} else if (IS_GM45(dev_priv)) {
-		dev_priv->cdclk_funcs = &gm45_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &gm45_cdclk_funcs;
 	} else if (IS_G45(dev_priv)) {
-		dev_priv->cdclk_funcs = &g33_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
 	} else if (IS_I965GM(dev_priv)) {
-		dev_priv->cdclk_funcs = &i965gm_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &i965gm_cdclk_funcs;
 	} else if (IS_I965G(dev_priv)) {
-		dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
 	} else if (IS_PINEVIEW(dev_priv)) {
-		dev_priv->cdclk_funcs = &pnv_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &pnv_cdclk_funcs;
 	} else if (IS_G33(dev_priv)) {
-		dev_priv->cdclk_funcs = &g33_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &g33_cdclk_funcs;
 	} else if (IS_I945GM(dev_priv)) {
-		dev_priv->cdclk_funcs = &i945gm_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &i945gm_cdclk_funcs;
 	} else if (IS_I945G(dev_priv)) {
-		dev_priv->cdclk_funcs = &fixed_400mhz_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &fixed_400mhz_cdclk_funcs;
 	} else if (IS_I915GM(dev_priv)) {
-		dev_priv->cdclk_funcs = &i915gm_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &i915gm_cdclk_funcs;
 	} else if (IS_I915G(dev_priv)) {
-		dev_priv->cdclk_funcs = &i915g_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &i915g_cdclk_funcs;
 	} else if (IS_I865G(dev_priv)) {
-		dev_priv->cdclk_funcs = &i865g_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &i865g_cdclk_funcs;
 	} else if (IS_I85X(dev_priv)) {
-		dev_priv->cdclk_funcs = &i85x_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &i85x_cdclk_funcs;
 	} else if (IS_I845G(dev_priv)) {
-		dev_priv->cdclk_funcs = &i845g_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &i845g_cdclk_funcs;
 	} else if (IS_I830(dev_priv)) {
-		dev_priv->cdclk_funcs = &i830_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
 	}
 
-	if (drm_WARN(&dev_priv->drm, !dev_priv->cdclk_funcs,
+	if (drm_WARN(&dev_priv->drm, !dev_priv->display.funcs.cdclk,
 		     "Unknown platform. Assuming i830\n"))
-		dev_priv->cdclk_funcs = &i830_cdclk_funcs;
+		dev_priv->display.funcs.cdclk = &i830_cdclk_funcs;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.h b/drivers/gpu/drm/i915/display/intel_cdclk.h
index b535cf6a7d9e..c674879a84a5 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.h
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.h
@@ -77,9 +77,9 @@ intel_atomic_get_cdclk_state(struct intel_atomic_state *state);
 
 #define to_intel_cdclk_state(x) container_of((x), struct intel_cdclk_state, base)
 #define intel_atomic_get_old_cdclk_state(state) \
-	to_intel_cdclk_state(intel_atomic_get_old_global_obj_state(state, &to_i915(state->base.dev)->cdclk.obj))
+	to_intel_cdclk_state(intel_atomic_get_old_global_obj_state(state, &to_i915(state->base.dev)->display.cdclk.obj))
 #define intel_atomic_get_new_cdclk_state(state) \
-	to_intel_cdclk_state(intel_atomic_get_new_global_obj_state(state, &to_i915(state->base.dev)->cdclk.obj))
+	to_intel_cdclk_state(intel_atomic_get_new_global_obj_state(state, &to_i915(state->base.dev)->display.cdclk.obj))
 
 int intel_cdclk_init(struct drm_i915_private *dev_priv);
 
diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c
index 9583d17e858d..6bda4274eae9 100644
--- a/drivers/gpu/drm/i915/display/intel_color.c
+++ b/drivers/gpu/drm/i915/display/intel_color.c
@@ -26,6 +26,7 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_dpll.h"
+#include "intel_dsb.h"
 #include "vlv_dsi_pll.h"
 
 struct intel_color_funcs {
@@ -1167,22 +1168,22 @@ void intel_color_load_luts(const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 
-	dev_priv->color_funcs->load_luts(crtc_state);
+	dev_priv->display.funcs.color->load_luts(crtc_state);
 }
 
 void intel_color_commit_noarm(const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 
-	if (dev_priv->color_funcs->color_commit_noarm)
-		dev_priv->color_funcs->color_commit_noarm(crtc_state);
+	if (dev_priv->display.funcs.color->color_commit_noarm)
+		dev_priv->display.funcs.color->color_commit_noarm(crtc_state);
 }
 
 void intel_color_commit_arm(const struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 
-	dev_priv->color_funcs->color_commit_arm(crtc_state);
+	dev_priv->display.funcs.color->color_commit_arm(crtc_state);
 }
 
 static bool intel_can_preload_luts(const struct intel_crtc_state *new_crtc_state)
@@ -1238,15 +1239,15 @@ int intel_color_check(struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 
-	return dev_priv->color_funcs->color_check(crtc_state);
+	return dev_priv->display.funcs.color->color_check(crtc_state);
 }
 
 void intel_color_get_config(struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 
-	if (dev_priv->color_funcs->read_luts)
-		dev_priv->color_funcs->read_luts(crtc_state);
+	if (dev_priv->display.funcs.color->read_luts)
+		dev_priv->display.funcs.color->read_luts(crtc_state);
 }
 
 static bool need_plane_update(struct intel_plane *plane,
@@ -2225,28 +2226,28 @@ void intel_color_init(struct intel_crtc *crtc)
 
 	if (HAS_GMCH(dev_priv)) {
 		if (IS_CHERRYVIEW(dev_priv)) {
-			dev_priv->color_funcs = &chv_color_funcs;
+			dev_priv->display.funcs.color = &chv_color_funcs;
 		} else if (DISPLAY_VER(dev_priv) >= 4) {
-			dev_priv->color_funcs = &i965_color_funcs;
+			dev_priv->display.funcs.color = &i965_color_funcs;
 		} else {
-			dev_priv->color_funcs = &i9xx_color_funcs;
+			dev_priv->display.funcs.color = &i9xx_color_funcs;
 		}
 	} else {
 		if (DISPLAY_VER(dev_priv) >= 11)
-			dev_priv->color_funcs = &icl_color_funcs;
+			dev_priv->display.funcs.color = &icl_color_funcs;
 		else if (DISPLAY_VER(dev_priv) == 10)
-			dev_priv->color_funcs = &glk_color_funcs;
+			dev_priv->display.funcs.color = &glk_color_funcs;
 		else if (DISPLAY_VER(dev_priv) == 9)
-			dev_priv->color_funcs = &skl_color_funcs;
+			dev_priv->display.funcs.color = &skl_color_funcs;
 		else if (DISPLAY_VER(dev_priv) == 8)
-			dev_priv->color_funcs = &bdw_color_funcs;
+			dev_priv->display.funcs.color = &bdw_color_funcs;
 		else if (DISPLAY_VER(dev_priv) == 7) {
 			if (IS_HASWELL(dev_priv))
-				dev_priv->color_funcs = &hsw_color_funcs;
+				dev_priv->display.funcs.color = &hsw_color_funcs;
 			else
-				dev_priv->color_funcs = &ivb_color_funcs;
+				dev_priv->display.funcs.color = &ivb_color_funcs;
 		} else
-			dev_priv->color_funcs = &ilk_color_funcs;
+			dev_priv->display.funcs.color = &ilk_color_funcs;
 	}
 
 	drm_crtc_enable_color_mgmt(&crtc->base,
diff --git a/drivers/gpu/drm/i915/display/intel_connector.c b/drivers/gpu/drm/i915/display/intel_connector.c
index 1dcc268927a2..6d5cbeb8df4d 100644
--- a/drivers/gpu/drm/i915/display/intel_connector.c
+++ b/drivers/gpu/drm/i915/display/intel_connector.c
@@ -229,7 +229,7 @@ intel_attach_force_audio_property(struct drm_connector *connector)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_property *prop;
 
-	prop = dev_priv->force_audio_property;
+	prop = dev_priv->display.properties.force_audio;
 	if (prop == NULL) {
 		prop = drm_property_create_enum(dev, 0,
 					   "audio",
@@ -238,7 +238,7 @@ intel_attach_force_audio_property(struct drm_connector *connector)
 		if (prop == NULL)
 			return;
 
-		dev_priv->force_audio_property = prop;
+		dev_priv->display.properties.force_audio = prop;
 	}
 	drm_object_attach_property(&connector->base, prop, 0);
 }
@@ -256,7 +256,7 @@ intel_attach_broadcast_rgb_property(struct drm_connector *connector)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_property *prop;
 
-	prop = dev_priv->broadcast_rgb_property;
+	prop = dev_priv->display.properties.broadcast_rgb;
 	if (prop == NULL) {
 		prop = drm_property_create_enum(dev, DRM_MODE_PROP_ENUM,
 					   "Broadcast RGB",
@@ -265,7 +265,7 @@ intel_attach_broadcast_rgb_property(struct drm_connector *connector)
 		if (prop == NULL)
 			return;
 
-		dev_priv->broadcast_rgb_property = prop;
+		dev_priv->display.properties.broadcast_rgb = prop;
 	}
 
 	drm_object_attach_property(&connector->base, prop, 0);
diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c
index 6a3893c8ff22..4a8ff2f97608 100644
--- a/drivers/gpu/drm/i915/display/intel_crt.c
+++ b/drivers/gpu/drm/i915/display/intel_crt.c
@@ -46,6 +46,7 @@
 #include "intel_gmbus.h"
 #include "intel_hotplug.h"
 #include "intel_pch_display.h"
+#include "intel_pch_refclk.h"
 
 /* Here's the desired hotplug mode */
 #define ADPA_HOTPLUG_BITS (ADPA_CRT_HOTPLUG_PERIOD_128 |		\
@@ -444,6 +445,8 @@ static int hsw_crt_compute_config(struct intel_encoder *encoder,
 	/* FDI must always be 2.7 GHz */
 	pipe_config->port_clock = 135000 * 2;
 
+	adjusted_mode->crtc_clock = lpt_iclkip(pipe_config);
+
 	return 0;
 }
 
@@ -643,9 +646,7 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector)
 	struct i2c_adapter *i2c;
 	bool ret = false;
 
-	BUG_ON(crt->base.type != INTEL_OUTPUT_ANALOG);
-
-	i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin);
+	i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->display.vbt.crt_ddc_pin);
 	edid = intel_crt_get_edid(connector, i2c);
 
 	if (edid) {
@@ -931,7 +932,7 @@ static int intel_crt_get_modes(struct drm_connector *connector)
 	wakeref = intel_display_power_get(dev_priv,
 					  intel_encoder->power_domain);
 
-	i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin);
+	i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->display.vbt.crt_ddc_pin);
 	ret = intel_crt_ddc_get_modes(connector, i2c);
 	if (ret || !IS_G4X(dev_priv))
 		goto out;
@@ -1110,8 +1111,8 @@ void intel_crt_init(struct drm_i915_private *dev_priv)
 		u32 fdi_config = FDI_RX_POLARITY_REVERSED_LPT |
 				 FDI_RX_LINK_REVERSAL_OVERRIDE;
 
-		dev_priv->fdi_rx_config = intel_de_read(dev_priv,
-							FDI_RX_CTL(PIPE_A)) & fdi_config;
+		dev_priv->display.fdi.rx_config = intel_de_read(dev_priv,
+								FDI_RX_CTL(PIPE_A)) & fdi_config;
 	}
 
 	intel_crt_reset(&crt->base.base);
diff --git a/drivers/gpu/drm/i915/display/intel_crtc.c b/drivers/gpu/drm/i915/display/intel_crtc.c
index 4442aa355f86..6792a9056f46 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc.c
@@ -9,7 +9,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_plane.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_vblank_work.h>
 
 #include "i915_irq.h"
diff --git a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
index 4ca6e9493ff2..e9212f69c360 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
@@ -134,8 +134,8 @@ static void intel_dump_plane_state(const struct intel_plane_state *plane_state)
 		    plane->base.base.id, plane->base.name,
 		    fb->base.id, fb->width, fb->height, &fb->format->format,
 		    fb->modifier, str_yes_no(plane_state->uapi.visible));
-	drm_dbg_kms(&i915->drm, "\trotation: 0x%x, scaler: %d\n",
-		    plane_state->hw.rotation, plane_state->scaler_id);
+	drm_dbg_kms(&i915->drm, "\trotation: 0x%x, scaler: %d, scaling_filter: %d\n",
+		    plane_state->hw.rotation, plane_state->scaler_id, plane_state->hw.scaling_filter);
 	if (plane_state->uapi.visible)
 		drm_dbg_kms(&i915->drm,
 			    "\tsrc: " DRM_RECT_FP_FMT " dst: " DRM_RECT_FMT "\n",
@@ -262,10 +262,11 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
 
 	if (DISPLAY_VER(i915) >= 9)
 		drm_dbg_kms(&i915->drm,
-			    "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
+			    "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d, scaling_filter: %d\n",
 			    crtc->num_scalers,
 			    pipe_config->scaler_state.scaler_users,
-			    pipe_config->scaler_state.scaler_id);
+			    pipe_config->scaler_state.scaler_id,
+			    pipe_config->hw.scaling_filter);
 
 	if (HAS_GMCH(i915))
 		drm_dbg_kms(&i915->drm,
diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c
index c2797ad2d313..87899e89b3a7 100644
--- a/drivers/gpu/drm/i915/display/intel_cursor.c
+++ b/drivers/gpu/drm/i915/display/intel_cursor.c
@@ -8,7 +8,6 @@
 #include <drm/drm_atomic_uapi.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_damage_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_fourcc.h>
 
 #include "intel_atomic.h"
@@ -20,9 +19,9 @@
 #include "intel_fb.h"
 #include "intel_fb_pin.h"
 #include "intel_frontbuffer.h"
-#include "intel_pm.h"
 #include "intel_psr.h"
 #include "intel_sprite.h"
+#include "skl_watermark.h"
 
 /* Cursor formats */
 static const u32 intel_cursor_formats[] = {
@@ -144,8 +143,8 @@ static int intel_check_cursor(struct intel_crtc_state *crtc_state,
 	}
 
 	ret = intel_atomic_plane_check_clipping(plane_state, crtc_state,
-						DRM_PLANE_HELPER_NO_SCALING,
-						DRM_PLANE_HELPER_NO_SCALING,
+						DRM_PLANE_NO_SCALING,
+						DRM_PLANE_NO_SCALING,
 						true);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
index 2330604b0bcc..da8472cdc135 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi.c
@@ -57,6 +57,7 @@
 #include "intel_lspcon.h"
 #include "intel_pps.h"
 #include "intel_psr.h"
+#include "intel_quirks.h"
 #include "intel_snps_phy.h"
 #include "intel_sprite.h"
 #include "intel_tc.h"
@@ -323,28 +324,6 @@ static int icl_calc_tbt_pll_link(struct drm_i915_private *dev_priv,
 	}
 }
 
-int intel_crtc_dotclock(const struct intel_crtc_state *pipe_config)
-{
-	int dotclock;
-
-	if (intel_crtc_has_dp_encoder(pipe_config))
-		dotclock = intel_dotclock_calculate(pipe_config->port_clock,
-						    &pipe_config->dp_m_n);
-	else if (pipe_config->has_hdmi_sink && pipe_config->pipe_bpp > 24)
-		dotclock = pipe_config->port_clock * 24 / pipe_config->pipe_bpp;
-	else
-		dotclock = pipe_config->port_clock;
-
-	if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 &&
-	    !intel_crtc_has_dp_encoder(pipe_config))
-		dotclock *= 2;
-
-	if (pipe_config->pixel_multiplier)
-		dotclock /= pipe_config->pixel_multiplier;
-
-	return dotclock;
-}
-
 static void ddi_dotclock_get(struct intel_crtc_state *pipe_config)
 {
 	/* CRT dotclock is determined via other means */
@@ -631,7 +610,7 @@ void intel_ddi_disable_transcoder_func(const struct intel_crtc_state *crtc_state
 
 	intel_de_write(dev_priv, TRANS_DDI_FUNC_CTL(cpu_transcoder), ctl);
 
-	if (dev_priv->quirks & QUIRK_INCREASE_DDI_DISABLED_TIME &&
+	if (intel_has_quirk(dev_priv, QUIRK_INCREASE_DDI_DISABLED_TIME) &&
 	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
 		drm_dbg_kms(&dev_priv->drm,
 			    "Quirk Increase DDI disabled time\n");
@@ -1425,7 +1404,7 @@ hsw_set_signal_levels(struct intel_encoder *encoder,
 static void _icl_ddi_enable_clock(struct drm_i915_private *i915, i915_reg_t reg,
 				  u32 clk_sel_mask, u32 clk_sel, u32 clk_off)
 {
-	mutex_lock(&i915->dpll.lock);
+	mutex_lock(&i915->display.dpll.lock);
 
 	intel_de_rmw(i915, reg, clk_sel_mask, clk_sel);
 
@@ -1435,17 +1414,17 @@ static void _icl_ddi_enable_clock(struct drm_i915_private *i915, i915_reg_t reg,
 	 */
 	intel_de_rmw(i915, reg, clk_off, 0);
 
-	mutex_unlock(&i915->dpll.lock);
+	mutex_unlock(&i915->display.dpll.lock);
 }
 
 static void _icl_ddi_disable_clock(struct drm_i915_private *i915, i915_reg_t reg,
 				   u32 clk_off)
 {
-	mutex_lock(&i915->dpll.lock);
+	mutex_lock(&i915->display.dpll.lock);
 
 	intel_de_rmw(i915, reg, 0, clk_off);
 
-	mutex_unlock(&i915->dpll.lock);
+	mutex_unlock(&i915->display.dpll.lock);
 }
 
 static bool _icl_ddi_is_clock_enabled(struct drm_i915_private *i915, i915_reg_t reg,
@@ -1720,12 +1699,12 @@ static void icl_ddi_tc_enable_clock(struct intel_encoder *encoder,
 	intel_de_write(i915, DDI_CLK_SEL(port),
 		       icl_pll_to_ddi_clk_sel(encoder, crtc_state));
 
-	mutex_lock(&i915->dpll.lock);
+	mutex_lock(&i915->display.dpll.lock);
 
 	intel_de_rmw(i915, ICL_DPCLKA_CFGCR0,
 		     ICL_DPCLKA_CFGCR0_TC_CLK_OFF(tc_port), 0);
 
-	mutex_unlock(&i915->dpll.lock);
+	mutex_unlock(&i915->display.dpll.lock);
 }
 
 static void icl_ddi_tc_disable_clock(struct intel_encoder *encoder)
@@ -1734,12 +1713,12 @@ static void icl_ddi_tc_disable_clock(struct intel_encoder *encoder)
 	enum tc_port tc_port = intel_port_to_tc(i915, encoder->port);
 	enum port port = encoder->port;
 
-	mutex_lock(&i915->dpll.lock);
+	mutex_lock(&i915->display.dpll.lock);
 
 	intel_de_rmw(i915, ICL_DPCLKA_CFGCR0,
 		     0, ICL_DPCLKA_CFGCR0_TC_CLK_OFF(tc_port));
 
-	mutex_unlock(&i915->dpll.lock);
+	mutex_unlock(&i915->display.dpll.lock);
 
 	intel_de_write(i915, DDI_CLK_SEL(port), DDI_CLK_SEL_NONE);
 }
@@ -1824,7 +1803,7 @@ static void skl_ddi_enable_clock(struct intel_encoder *encoder,
 	if (drm_WARN_ON(&i915->drm, !pll))
 		return;
 
-	mutex_lock(&i915->dpll.lock);
+	mutex_lock(&i915->display.dpll.lock);
 
 	intel_de_rmw(i915, DPLL_CTRL2,
 		     DPLL_CTRL2_DDI_CLK_OFF(port) |
@@ -1832,7 +1811,7 @@ static void skl_ddi_enable_clock(struct intel_encoder *encoder,
 		     DPLL_CTRL2_DDI_CLK_SEL(pll->info->id, port) |
 		     DPLL_CTRL2_DDI_SEL_OVERRIDE(port));
 
-	mutex_unlock(&i915->dpll.lock);
+	mutex_unlock(&i915->display.dpll.lock);
 }
 
 static void skl_ddi_disable_clock(struct intel_encoder *encoder)
@@ -1840,12 +1819,12 @@ static void skl_ddi_disable_clock(struct intel_encoder *encoder)
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	enum port port = encoder->port;
 
-	mutex_lock(&i915->dpll.lock);
+	mutex_lock(&i915->display.dpll.lock);
 
 	intel_de_rmw(i915, DPLL_CTRL2,
 		     0, DPLL_CTRL2_DDI_CLK_OFF(port));
 
-	mutex_unlock(&i915->dpll.lock);
+	mutex_unlock(&i915->display.dpll.lock);
 }
 
 static bool skl_ddi_is_clock_enabled(struct intel_encoder *encoder)
@@ -2691,10 +2670,14 @@ static void intel_ddi_post_disable_hdmi(struct intel_atomic_state *state,
 	dig_port->set_infoframes(encoder, false,
 				 old_crtc_state, old_conn_state);
 
-	intel_ddi_disable_pipe_clock(old_crtc_state);
+	if (DISPLAY_VER(dev_priv) < 12)
+		intel_ddi_disable_pipe_clock(old_crtc_state);
 
 	intel_disable_ddi_buf(encoder, old_crtc_state);
 
+	if (DISPLAY_VER(dev_priv) >= 12)
+		intel_ddi_disable_pipe_clock(old_crtc_state);
+
 	intel_display_power_put(dev_priv,
 				dig_port->ddi_io_power_domain,
 				fetch_and_zero(&dig_port->ddi_io_wakeref));
@@ -2862,6 +2845,8 @@ static void intel_enable_ddi_hdmi(struct intel_atomic_state *state,
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct drm_connector *connector = conn_state->connector;
 	enum port port = encoder->port;
+	enum phy phy = intel_port_to_phy(dev_priv, port);
+	u32 buf_ctl;
 
 	if (!intel_hdmi_handle_sink_scrambling(encoder, connector,
 					       crtc_state->hdmi_high_tmds_clock_ratio,
@@ -2919,8 +2904,12 @@ static void intel_enable_ddi_hdmi(struct intel_atomic_state *state,
 	 * On ADL_P the PHY link rate and lane count must be programmed but
 	 * these are both 0 for HDMI.
 	 */
-	intel_de_write(dev_priv, DDI_BUF_CTL(port),
-		       dig_port->saved_port_bits | DDI_BUF_CTL_ENABLE);
+	buf_ctl = dig_port->saved_port_bits | DDI_BUF_CTL_ENABLE;
+	if (IS_ALDERLAKE_P(dev_priv) && intel_phy_is_tc(dev_priv, phy)) {
+		drm_WARN_ON(&dev_priv->drm, !intel_tc_port_in_legacy_mode(dig_port));
+		buf_ctl |= DDI_BUF_CTL_TC_PHY_OWNERSHIP;
+	}
+	intel_de_write(dev_priv, DDI_BUF_CTL(port), buf_ctl);
 
 	intel_audio_codec_enable(encoder, crtc_state, conn_state);
 }
@@ -3611,10 +3600,22 @@ static void intel_ddi_sync_state(struct intel_encoder *encoder,
 static bool intel_ddi_initial_fastset_check(struct intel_encoder *encoder,
 					    struct intel_crtc_state *crtc_state)
 {
-	if (intel_crtc_has_dp_encoder(crtc_state))
-		return intel_dp_initial_fastset_check(encoder, crtc_state);
+	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
+	enum phy phy = intel_port_to_phy(i915, encoder->port);
+	bool fastset = true;
 
-	return true;
+	if (intel_phy_is_tc(i915, phy)) {
+		drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s] Forcing full modeset to compute TC port DPLLs\n",
+			    encoder->base.base.id, encoder->base.name);
+		crtc_state->uapi.mode_changed = true;
+		fastset = false;
+	}
+
+	if (intel_crtc_has_dp_encoder(crtc_state) &&
+	    !intel_dp_initial_fastset_check(encoder, crtc_state))
+		fastset = false;
+
+	return fastset;
 }
 
 static enum intel_output_type
@@ -4028,7 +4029,7 @@ intel_ddi_hotplug(struct intel_encoder *encoder,
 static bool lpt_digital_port_connected(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	u32 bit = dev_priv->hotplug.pch_hpd[encoder->hpd_pin];
+	u32 bit = dev_priv->display.hotplug.pch_hpd[encoder->hpd_pin];
 
 	return intel_de_read(dev_priv, SDEISR) & bit;
 }
@@ -4036,7 +4037,7 @@ static bool lpt_digital_port_connected(struct intel_encoder *encoder)
 static bool hsw_digital_port_connected(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	u32 bit = dev_priv->hotplug.hpd[encoder->hpd_pin];
+	u32 bit = dev_priv->display.hotplug.hpd[encoder->hpd_pin];
 
 	return intel_de_read(dev_priv, DEISR) & bit;
 }
@@ -4044,7 +4045,7 @@ static bool hsw_digital_port_connected(struct intel_encoder *encoder)
 static bool bdw_digital_port_connected(struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	u32 bit = dev_priv->hotplug.hpd[encoder->hpd_pin];
+	u32 bit = dev_priv->display.hotplug.hpd[encoder->hpd_pin];
 
 	return intel_de_read(dev_priv, GEN8_DE_PORT_ISR) & bit;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index fc5d94862ef3..dd008ba8afe3 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -41,7 +41,6 @@
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_privacy_screen_consumer.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_rect.h>
@@ -92,6 +91,7 @@
 #include "intel_dmc.h"
 #include "intel_dp_link_training.h"
 #include "intel_dpt.h"
+#include "intel_dsb.h"
 #include "intel_fbc.h"
 #include "intel_fbdev.h"
 #include "intel_fdi.h"
@@ -118,6 +118,7 @@
 #include "i9xx_plane.h"
 #include "skl_scaler.h"
 #include "skl_universal_plane.h"
+#include "skl_watermark.h"
 #include "vlv_dsi.h"
 #include "vlv_dsi_pll.h"
 #include "vlv_dsi_regs.h"
@@ -164,16 +165,16 @@ static void ilk_pfit_enable(const struct intel_crtc_state *crtc_state);
  */
 void intel_update_watermarks(struct drm_i915_private *dev_priv)
 {
-	if (dev_priv->wm_disp->update_wm)
-		dev_priv->wm_disp->update_wm(dev_priv);
+	if (dev_priv->display.funcs.wm->update_wm)
+		dev_priv->display.funcs.wm->update_wm(dev_priv);
 }
 
 static int intel_compute_pipe_wm(struct intel_atomic_state *state,
 				 struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (dev_priv->wm_disp->compute_pipe_wm)
-		return dev_priv->wm_disp->compute_pipe_wm(state, crtc);
+	if (dev_priv->display.funcs.wm->compute_pipe_wm)
+		return dev_priv->display.funcs.wm->compute_pipe_wm(state, crtc);
 	return 0;
 }
 
@@ -181,20 +182,20 @@ static int intel_compute_intermediate_wm(struct intel_atomic_state *state,
 					 struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (!dev_priv->wm_disp->compute_intermediate_wm)
+	if (!dev_priv->display.funcs.wm->compute_intermediate_wm)
 		return 0;
 	if (drm_WARN_ON(&dev_priv->drm,
-			!dev_priv->wm_disp->compute_pipe_wm))
+			!dev_priv->display.funcs.wm->compute_pipe_wm))
 		return 0;
-	return dev_priv->wm_disp->compute_intermediate_wm(state, crtc);
+	return dev_priv->display.funcs.wm->compute_intermediate_wm(state, crtc);
 }
 
 static bool intel_initial_watermarks(struct intel_atomic_state *state,
 				     struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (dev_priv->wm_disp->initial_watermarks) {
-		dev_priv->wm_disp->initial_watermarks(state, crtc);
+	if (dev_priv->display.funcs.wm->initial_watermarks) {
+		dev_priv->display.funcs.wm->initial_watermarks(state, crtc);
 		return true;
 	}
 	return false;
@@ -204,23 +205,23 @@ static void intel_atomic_update_watermarks(struct intel_atomic_state *state,
 					   struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (dev_priv->wm_disp->atomic_update_watermarks)
-		dev_priv->wm_disp->atomic_update_watermarks(state, crtc);
+	if (dev_priv->display.funcs.wm->atomic_update_watermarks)
+		dev_priv->display.funcs.wm->atomic_update_watermarks(state, crtc);
 }
 
 static void intel_optimize_watermarks(struct intel_atomic_state *state,
 				      struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (dev_priv->wm_disp->optimize_watermarks)
-		dev_priv->wm_disp->optimize_watermarks(state, crtc);
+	if (dev_priv->display.funcs.wm->optimize_watermarks)
+		dev_priv->display.funcs.wm->optimize_watermarks(state, crtc);
 }
 
 static int intel_compute_global_watermarks(struct intel_atomic_state *state)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	if (dev_priv->wm_disp->compute_global_watermarks)
-		return dev_priv->wm_disp->compute_global_watermarks(state);
+	if (dev_priv->display.funcs.wm->compute_global_watermarks)
+		return dev_priv->display.funcs.wm->compute_global_watermarks(state);
 	return 0;
 }
 
@@ -619,7 +620,10 @@ void intel_disable_transcoder(const struct intel_crtc_state *old_crtc_state)
 	if (!IS_I830(dev_priv))
 		val &= ~PIPECONF_ENABLE;
 
-	if (DISPLAY_VER(dev_priv) >= 12)
+	if (DISPLAY_VER(dev_priv) >= 14)
+		intel_de_rmw(dev_priv, MTL_CHICKEN_TRANS(cpu_transcoder),
+			     FECSTALL_DIS_DPTSTREAM_DPTTG, 0);
+	else if (DISPLAY_VER(dev_priv) >= 12)
 		intel_de_rmw(dev_priv, CHICKEN_TRANS(cpu_transcoder),
 			     FECSTALL_DIS_DPTSTREAM_DPTTG, 0);
 
@@ -671,7 +675,7 @@ bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
 
 	return DISPLAY_VER(dev_priv) < 4 ||
 		(plane->fbc &&
-		 plane_state->view.gtt.type == I915_GGTT_VIEW_NORMAL);
+		 plane_state->view.gtt.type == I915_GTT_VIEW_NORMAL);
 }
 
 /*
@@ -1487,7 +1491,7 @@ static void intel_encoders_update_prepare(struct intel_atomic_state *state)
 	 * Make sure the DPLL state is up-to-date for fastset TypeC ports after non-blocking commits.
 	 * TODO: Update the DPLL state for all cases in the encoder->update_prepare() hook.
 	 */
-	if (i915->dpll.mgr) {
+	if (i915->display.dpll.mgr) {
 		for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
 			if (intel_crtc_needs_modeset(new_crtc_state))
 				continue;
@@ -1839,7 +1843,9 @@ static void hsw_set_frame_start_delay(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	i915_reg_t reg = CHICKEN_TRANS(crtc_state->cpu_transcoder);
+	enum transcoder transcoder = crtc_state->cpu_transcoder;
+	i915_reg_t reg = DISPLAY_VER(dev_priv) >= 14 ? MTL_CHICKEN_TRANS(transcoder) :
+			 CHICKEN_TRANS(transcoder);
 	u32 val;
 
 	val = intel_de_read(dev_priv, reg);
@@ -2081,22 +2087,20 @@ bool intel_phy_is_combo(struct drm_i915_private *dev_priv, enum phy phy)
 {
 	if (phy == PHY_NONE)
 		return false;
-	else if (IS_DG2(dev_priv))
-		/*
-		 * DG2 outputs labelled as "combo PHY" in the bspec use
-		 * SNPS PHYs with completely different programming,
-		 * hence we always return false here.
-		 */
-		return false;
 	else if (IS_ALDERLAKE_S(dev_priv))
 		return phy <= PHY_E;
 	else if (IS_DG1(dev_priv) || IS_ROCKETLAKE(dev_priv))
 		return phy <= PHY_D;
 	else if (IS_JSL_EHL(dev_priv))
 		return phy <= PHY_C;
-	else if (DISPLAY_VER(dev_priv) >= 11)
+	else if (IS_ALDERLAKE_P(dev_priv) || IS_DISPLAY_VER(dev_priv, 11, 12))
 		return phy <= PHY_B;
 	else
+		/*
+		 * DG2 outputs labelled as "combo PHY" in the bspec use
+		 * SNPS PHYs with completely different programming,
+		 * hence we always return false here.
+		 */
 		return false;
 }
 
@@ -2402,7 +2406,7 @@ static void i9xx_crtc_disable(struct intel_atomic_state *state,
 	if (DISPLAY_VER(dev_priv) != 2)
 		intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
 
-	if (!dev_priv->wm_disp->initial_watermarks)
+	if (!dev_priv->display.funcs.wm->initial_watermarks)
 		intel_update_watermarks(dev_priv);
 
 	/* clock the pipe down to 640x480@60 to potentially save power */
@@ -2661,7 +2665,7 @@ static int intel_crtc_compute_pipe_mode(struct intel_crtc_state *crtc_state)
 	intel_mode_from_crtc_timings(pipe_mode, pipe_mode);
 
 	if (DISPLAY_VER(i915) < 4) {
-		clock_limit = i915->max_cdclk_freq * 9 / 10;
+		clock_limit = i915->display.cdclk.max_cdclk_freq * 9 / 10;
 
 		/*
 		 * Enable double wide mode when the dot clock
@@ -2693,6 +2697,10 @@ static int intel_crtc_compute_config(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	int ret;
 
+	ret = intel_dpll_crtc_compute_clock(state, crtc);
+	if (ret)
+		return ret;
+
 	ret = intel_crtc_compute_pipe_src(crtc_state);
 	if (ret)
 		return ret;
@@ -2719,19 +2727,11 @@ intel_reduce_m_n_ratio(u32 *num, u32 *den)
 	}
 }
 
-static void compute_m_n(unsigned int m, unsigned int n,
-			u32 *ret_m, u32 *ret_n,
-			bool constant_n)
+static void compute_m_n(u32 *ret_m, u32 *ret_n,
+			u32 m, u32 n, u32 constant_n)
 {
-	/*
-	 * Several DP dongles in particular seem to be fussy about
-	 * too large link M/N values. Give N value as 0x8000 that
-	 * should be acceptable by specific devices. 0x8000 is the
-	 * specified fixed N value for asynchronous clock mode,
-	 * which the devices expect also in synchronous clock mode.
-	 */
 	if (constant_n)
-		*ret_n = DP_LINK_CONSTANT_N_VALUE;
+		*ret_n = constant_n;
 	else
 		*ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
 
@@ -2743,22 +2743,28 @@ void
 intel_link_compute_m_n(u16 bits_per_pixel, int nlanes,
 		       int pixel_clock, int link_clock,
 		       struct intel_link_m_n *m_n,
-		       bool constant_n, bool fec_enable)
+		       bool fec_enable)
 {
 	u32 data_clock = bits_per_pixel * pixel_clock;
 
 	if (fec_enable)
 		data_clock = intel_dp_mode_to_fec_clock(data_clock);
 
+	/*
+	 * Windows/BIOS uses fixed M/N values always. Follow suit.
+	 *
+	 * Also several DP dongles in particular seem to be fussy
+	 * about too large link M/N values. Presumably the 20bit
+	 * value used by Windows/BIOS is acceptable to everyone.
+	 */
 	m_n->tu = 64;
-	compute_m_n(data_clock,
-		    link_clock * nlanes * 8,
-		    &m_n->data_m, &m_n->data_n,
-		    constant_n);
+	compute_m_n(&m_n->data_m, &m_n->data_n,
+		    data_clock, link_clock * nlanes * 8,
+		    0x8000000);
 
-	compute_m_n(pixel_clock, link_clock,
-		    &m_n->link_m, &m_n->link_n,
-		    constant_n);
+	compute_m_n(&m_n->link_m, &m_n->link_n,
+		    pixel_clock, link_clock,
+		    0x80000);
 }
 
 static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
@@ -2774,12 +2780,12 @@ static void intel_panel_sanitize_ssc(struct drm_i915_private *dev_priv)
 						       PCH_DREF_CONTROL) &
 			DREF_SSC1_ENABLE;
 
-		if (dev_priv->vbt.lvds_use_ssc != bios_lvds_use_ssc) {
+		if (dev_priv->display.vbt.lvds_use_ssc != bios_lvds_use_ssc) {
 			drm_dbg_kms(&dev_priv->drm,
 				    "SSC %s by BIOS, overriding VBT which says %s\n",
 				    str_enabled_disabled(bios_lvds_use_ssc),
-				    str_enabled_disabled(dev_priv->vbt.lvds_use_ssc));
-			dev_priv->vbt.lvds_use_ssc = bios_lvds_use_ssc;
+				    str_enabled_disabled(dev_priv->display.vbt.lvds_use_ssc));
+			dev_priv->display.vbt.lvds_use_ssc = bios_lvds_use_ssc;
 		}
 	}
 }
@@ -4127,7 +4133,9 @@ static bool hsw_get_pipe_config(struct intel_crtc *crtc,
 	}
 
 	if (!transcoder_is_dsi(pipe_config->cpu_transcoder)) {
-		tmp = intel_de_read(dev_priv, CHICKEN_TRANS(pipe_config->cpu_transcoder));
+		tmp = intel_de_read(dev_priv, DISPLAY_VER(dev_priv) >= 14 ?
+				    MTL_CHICKEN_TRANS(pipe_config->cpu_transcoder) :
+				    CHICKEN_TRANS(pipe_config->cpu_transcoder));
 
 		pipe_config->framestart_delay = REG_FIELD_GET(HSW_FRAME_START_DELAY_MASK, tmp) + 1;
 	} else {
@@ -4146,7 +4154,7 @@ bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state)
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 
-	if (!i915->display->get_pipe_config(crtc, crtc_state))
+	if (!i915->display.funcs.display->get_pipe_config(crtc, crtc_state))
 		return false;
 
 	crtc_state->hw.active = true;
@@ -4375,7 +4383,7 @@ static int i9xx_pll_refclk(struct drm_device *dev,
 	u32 dpll = pipe_config->dpll_hw_state.dpll;
 
 	if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
-		return dev_priv->vbt.lvds_ssc_freq;
+		return dev_priv->display.vbt.lvds_ssc_freq;
 	else if (HAS_PCH_SPLIT(dev_priv))
 		return 120000;
 	else if (DISPLAY_VER(dev_priv) != 2)
@@ -4493,7 +4501,31 @@ int intel_dotclock_calculate(int link_freq,
 	if (!m_n->link_n)
 		return 0;
 
-	return div_u64(mul_u32_u32(m_n->link_m, link_freq), m_n->link_n);
+	return DIV_ROUND_UP_ULL(mul_u32_u32(m_n->link_m, link_freq),
+				m_n->link_n);
+}
+
+int intel_crtc_dotclock(const struct intel_crtc_state *pipe_config)
+{
+	int dotclock;
+
+	if (intel_crtc_has_dp_encoder(pipe_config))
+		dotclock = intel_dotclock_calculate(pipe_config->port_clock,
+						    &pipe_config->dp_m_n);
+	else if (pipe_config->has_hdmi_sink && pipe_config->pipe_bpp > 24)
+		dotclock = DIV_ROUND_CLOSEST(pipe_config->port_clock * 24,
+					     pipe_config->pipe_bpp);
+	else
+		dotclock = pipe_config->port_clock;
+
+	if (pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 &&
+	    !intel_crtc_has_dp_encoder(pipe_config))
+		dotclock *= 2;
+
+	if (pipe_config->pixel_multiplier)
+		dotclock /= pipe_config->pixel_multiplier;
+
+	return dotclock;
 }
 
 /* Returns the currently programmed mode of the given encoder. */
@@ -4754,7 +4786,7 @@ static u16 skl_linetime_wm(const struct intel_crtc_state *crtc_state)
 
 	/* Display WA #1135: BXT:ALL GLK:ALL */
 	if ((IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv)) &&
-	    dev_priv->ipc_enabled)
+	    skl_watermark_ipc_enabled(dev_priv))
 		linetime_wm /= 2;
 
 	return min(linetime_wm, 0x1ff);
@@ -4800,10 +4832,6 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state,
 		crtc_state->update_wm_post = true;
 
 	if (mode_changed) {
-		ret = intel_dpll_crtc_compute_clock(state, crtc);
-		if (ret)
-			return ret;
-
 		ret = intel_dpll_crtc_get_shared_dpll(state, crtc);
 		if (ret)
 			return ret;
@@ -5368,46 +5396,14 @@ bool intel_fuzzy_clock_check(int clock1, int clock2)
 }
 
 static bool
-intel_compare_m_n(unsigned int m, unsigned int n,
-		  unsigned int m2, unsigned int n2,
-		  bool exact)
-{
-	if (m == m2 && n == n2)
-		return true;
-
-	if (exact || !m || !n || !m2 || !n2)
-		return false;
-
-	BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
-
-	if (n > n2) {
-		while (n > n2) {
-			m2 <<= 1;
-			n2 <<= 1;
-		}
-	} else if (n < n2) {
-		while (n < n2) {
-			m <<= 1;
-			n <<= 1;
-		}
-	}
-
-	if (n != n2)
-		return false;
-
-	return intel_fuzzy_clock_check(m, m2);
-}
-
-static bool
 intel_compare_link_m_n(const struct intel_link_m_n *m_n,
-		       const struct intel_link_m_n *m2_n2,
-		       bool exact)
+		       const struct intel_link_m_n *m2_n2)
 {
 	return m_n->tu == m2_n2->tu &&
-		intel_compare_m_n(m_n->data_m, m_n->data_n,
-				  m2_n2->data_m, m2_n2->data_n, exact) &&
-		intel_compare_m_n(m_n->link_m, m_n->link_n,
-				  m2_n2->link_m, m2_n2->link_n, exact);
+		m_n->data_m == m2_n2->data_m &&
+		m_n->data_n == m2_n2->data_n &&
+		m_n->link_m == m2_n2->link_m &&
+		m_n->link_n == m2_n2->link_n;
 }
 
 static bool
@@ -5601,8 +5597,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 
 #define PIPE_CONF_CHECK_M_N(name) do { \
 	if (!intel_compare_link_m_n(&current_config->name, \
-				    &pipe_config->name,\
-				    !fastset)) { \
+				    &pipe_config->name)) { \
 		pipe_config_mismatch(fastset, crtc, __stringify(name), \
 				     "(expected tu %i data %i/%i link %i/%i, " \
 				     "found tu %i, data %i/%i link %i/%i)", \
@@ -5649,9 +5644,9 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
  */
 #define PIPE_CONF_CHECK_M_N_ALT(name, alt_name) do { \
 	if (!intel_compare_link_m_n(&current_config->name, \
-				    &pipe_config->name, !fastset) && \
+				    &pipe_config->name) && \
 	    !intel_compare_link_m_n(&current_config->alt_name, \
-				    &pipe_config->name, !fastset)) { \
+				    &pipe_config->name)) { \
 		pipe_config_mismatch(fastset, crtc, __stringify(name), \
 				     "(expected tu %i data %i/%i link %i/%i, " \
 				     "or tu %i data %i/%i link %i/%i, " \
@@ -5686,16 +5681,6 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	} \
 } while (0)
 
-#define PIPE_CONF_CHECK_CLOCK_FUZZY(name) do { \
-	if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
-		pipe_config_mismatch(fastset, crtc, __stringify(name), \
-				     "(expected %i, found %i)", \
-				     current_config->name, \
-				     pipe_config->name); \
-		ret = false; \
-	} \
-} while (0)
-
 #define PIPE_CONF_CHECK_INFOFRAME(name) do { \
 	if (!intel_compare_infoframe(&current_config->infoframes.name, \
 				     &pipe_config->infoframes.name)) { \
@@ -5751,8 +5736,9 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	PIPE_CONF_CHECK_I(lane_count);
 	PIPE_CONF_CHECK_X(lane_lat_optim_mask);
 
-	if (DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv)) {
-		PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
+	if (HAS_DOUBLE_BUFFERED_M_N(dev_priv)) {
+		if (!fastset || !pipe_config->seamless_m_n)
+			PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
 	} else {
 		PIPE_CONF_CHECK_M_N(dp_m_n);
 		PIPE_CONF_CHECK_M_N(dp_m2_n2);
@@ -5814,7 +5800,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 		PIPE_CONF_CHECK_RECT(pch_pfit.dst);
 
 		PIPE_CONF_CHECK_I(scaler_state.scaler_id);
-		PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
+		PIPE_CONF_CHECK_I(pixel_rate);
 
 		PIPE_CONF_CHECK_X(gamma_mode);
 		if (IS_CHERRYVIEW(dev_priv))
@@ -5841,7 +5827,7 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 
 	PIPE_CONF_CHECK_BOOL(double_wide);
 
-	if (dev_priv->dpll.mgr) {
+	if (dev_priv->display.dpll.mgr) {
 		PIPE_CONF_CHECK_P(shared_dpll);
 
 		PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
@@ -5884,9 +5870,11 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 	if (IS_G4X(dev_priv) || DISPLAY_VER(dev_priv) >= 5)
 		PIPE_CONF_CHECK_I(pipe_bpp);
 
-	PIPE_CONF_CHECK_CLOCK_FUZZY(hw.pipe_mode.crtc_clock);
-	PIPE_CONF_CHECK_CLOCK_FUZZY(hw.adjusted_mode.crtc_clock);
-	PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
+	if (!fastset || !pipe_config->seamless_m_n) {
+		PIPE_CONF_CHECK_I(hw.pipe_mode.crtc_clock);
+		PIPE_CONF_CHECK_I(hw.adjusted_mode.crtc_clock);
+	}
+	PIPE_CONF_CHECK_I(port_clock);
 
 	PIPE_CONF_CHECK_I(min_voltage_level);
 
@@ -5928,7 +5916,6 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 #undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
 #undef PIPE_CONF_CHECK_P
 #undef PIPE_CONF_CHECK_FLAGS
-#undef PIPE_CONF_CHECK_CLOCK_FUZZY
 #undef PIPE_CONF_CHECK_COLOR_LUT
 #undef PIPE_CONF_CHECK_TIMINGS
 #undef PIPE_CONF_CHECK_RECT
@@ -6050,20 +6037,6 @@ void intel_crtc_update_active_timings(const struct intel_crtc_state *crtc_state)
 	}
 }
 
-static void intel_modeset_clear_plls(struct intel_atomic_state *state)
-{
-	struct intel_crtc_state *new_crtc_state;
-	struct intel_crtc *crtc;
-	int i;
-
-	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
-		if (!intel_crtc_needs_modeset(new_crtc_state))
-			continue;
-
-		intel_release_shared_dplls(state, crtc);
-	}
-}
-
 /*
  * This implements the workaround described in the "notes" section of the mode
  * set sequence documentation. When going from no pipes or single pipe to
@@ -6164,23 +6137,6 @@ static void intel_crtc_check_fastset(const struct intel_crtc_state *old_crtc_sta
 	new_crtc_state->update_pipe = true;
 }
 
-static void intel_crtc_copy_fastset(const struct intel_crtc_state *old_crtc_state,
-				    struct intel_crtc_state *new_crtc_state)
-{
-	/*
-	 * If we're not doing the full modeset we want to
-	 * keep the current M/N values as they may be
-	 * sufficiently different to the computed values
-	 * to cause problems.
-	 *
-	 * FIXME: should really copy more fuzzy state here
-	 */
-	new_crtc_state->fdi_m_n = old_crtc_state->fdi_m_n;
-	new_crtc_state->dp_m_n = old_crtc_state->dp_m_n;
-	new_crtc_state->dp_m2_n2 = old_crtc_state->dp_m2_n2;
-	new_crtc_state->has_drrs = old_crtc_state->has_drrs;
-}
-
 static int intel_crtc_add_planes_to_state(struct intel_atomic_state *state,
 					  struct intel_crtc *crtc,
 					  u8 plane_ids_mask)
@@ -6837,9 +6793,11 @@ static int intel_atomic_check(struct drm_device *dev,
 		if (!intel_crtc_needs_modeset(new_crtc_state))
 			continue;
 
-		ret = intel_modeset_pipe_config_late(state, crtc);
-		if (ret)
-			goto fail;
+		if (new_crtc_state->hw.enable) {
+			ret = intel_modeset_pipe_config_late(state, crtc);
+			if (ret)
+				goto fail;
+		}
 
 		intel_crtc_check_fastset(old_crtc_state, new_crtc_state);
 	}
@@ -6890,15 +6848,12 @@ static int intel_atomic_check(struct drm_device *dev,
 
 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
 					    new_crtc_state, i) {
-		if (intel_crtc_needs_modeset(new_crtc_state)) {
-			any_ms = true;
+		if (!intel_crtc_needs_modeset(new_crtc_state))
 			continue;
-		}
 
-		if (!new_crtc_state->update_pipe)
-			continue;
+		any_ms = true;
 
-		intel_crtc_copy_fastset(old_crtc_state, new_crtc_state);
+		intel_release_shared_dplls(state, crtc);
 	}
 
 	if (any_ms && !check_digital_port_conflicts(state)) {
@@ -6939,8 +6894,6 @@ static int intel_atomic_check(struct drm_device *dev,
 		ret = intel_modeset_calc_cdclk(state);
 		if (ret)
 			return ret;
-
-		intel_modeset_clear_plls(state);
 	}
 
 	ret = intel_atomic_check_crtcs(state);
@@ -7059,6 +7012,10 @@ static void intel_pipe_fastset(const struct intel_crtc_state *old_crtc_state,
 	if (DISPLAY_VER(dev_priv) >= 9 ||
 	    IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
 		hsw_set_linetime_wm(new_crtc_state);
+
+	if (new_crtc_state->seamless_m_n)
+		intel_cpu_transcoder_set_m1_n1(crtc, new_crtc_state->cpu_transcoder,
+					       &new_crtc_state->dp_m_n);
 }
 
 static void commit_pipe_pre_planes(struct intel_atomic_state *state,
@@ -7121,7 +7078,7 @@ static void intel_enable_crtc(struct intel_atomic_state *state,
 
 	intel_crtc_update_active_timings(new_crtc_state);
 
-	dev_priv->display->crtc_enable(state, crtc);
+	dev_priv->display.funcs.display->crtc_enable(state, crtc);
 
 	if (intel_crtc_is_bigjoiner_slave(new_crtc_state))
 		return;
@@ -7200,7 +7157,7 @@ static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
 	 */
 	intel_crtc_disable_pipe_crc(crtc);
 
-	dev_priv->display->crtc_disable(state, crtc);
+	dev_priv->display.funcs.display->crtc_disable(state, crtc);
 	crtc->active = false;
 	intel_fbc_disable(crtc);
 	intel_disable_shared_dpll(old_crtc_state);
@@ -7411,7 +7368,7 @@ static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
 	struct intel_atomic_state *state, *next;
 	struct llist_node *freed;
 
-	freed = llist_del_all(&dev_priv->atomic_helper.free_list);
+	freed = llist_del_all(&dev_priv->display.atomic_helper.free_list);
 	llist_for_each_entry_safe(state, next, freed, freed)
 		drm_atomic_state_put(&state->base);
 }
@@ -7419,7 +7376,7 @@ static void intel_atomic_helper_free_state(struct drm_i915_private *dev_priv)
 static void intel_atomic_helper_free_state_worker(struct work_struct *work)
 {
 	struct drm_i915_private *dev_priv =
-		container_of(work, typeof(*dev_priv), atomic_helper.free_work);
+		container_of(work, typeof(*dev_priv), display.atomic_helper.free_work);
 
 	intel_atomic_helper_free_state(dev_priv);
 }
@@ -7532,6 +7489,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state)
 	intel_atomic_commit_fence_wait(state);
 
 	drm_atomic_helper_wait_for_dependencies(&state->base);
+	drm_dp_mst_atomic_wait_for_dependencies(&state->base);
 
 	if (state->modeset)
 		wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_MODESET);
@@ -7588,7 +7546,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state)
 	}
 
 	/* Now enable the clocks, plane, pipe, and connectors that we set up. */
-	dev_priv->display->commit_modeset_enables(state);
+	dev_priv->display.funcs.display->commit_modeset_enables(state);
 
 	intel_encoders_update_complete(state);
 
@@ -7711,7 +7669,7 @@ intel_atomic_commit_ready(struct i915_sw_fence *fence,
 	case FENCE_FREE:
 		{
 			struct intel_atomic_helper *helper =
-				&to_i915(state->base.dev)->atomic_helper;
+				&to_i915(state->base.dev)->display.atomic_helper;
 
 			if (llist_add(&state->freed, &helper->free_list))
 				schedule_work(&helper->free_work);
@@ -7814,12 +7772,12 @@ static int intel_atomic_commit(struct drm_device *dev,
 
 	i915_sw_fence_commit(&state->commit_ready);
 	if (nonblock && state->modeset) {
-		queue_work(dev_priv->modeset_wq, &state->base.commit_work);
+		queue_work(dev_priv->display.wq.modeset, &state->base.commit_work);
 	} else if (nonblock) {
-		queue_work(dev_priv->flip_wq, &state->base.commit_work);
+		queue_work(dev_priv->display.wq.flip, &state->base.commit_work);
 	} else {
 		if (state->modeset)
-			flush_workqueue(dev_priv->modeset_wq);
+			flush_workqueue(dev_priv->display.wq.modeset);
 		intel_atomic_commit_tail(state);
 	}
 
@@ -7925,7 +7883,7 @@ static bool intel_ddi_crt_present(struct drm_i915_private *dev_priv)
 	if (intel_de_read(dev_priv, DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
 		return false;
 
-	if (!dev_priv->vbt.int_crt_support)
+	if (!dev_priv->display.vbt.int_crt_support)
 		return false;
 
 	return true;
@@ -8060,7 +8018,7 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv)
 	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		bool has_edp, has_port;
 
-		if (IS_VALLEYVIEW(dev_priv) && dev_priv->vbt.int_crt_support)
+		if (IS_VALLEYVIEW(dev_priv) && dev_priv->display.vbt.int_crt_support)
 			intel_crt_init(dev_priv);
 
 		/*
@@ -8319,7 +8277,7 @@ static const struct drm_mode_config_funcs intel_mode_funcs = {
 	.atomic_state_free = intel_atomic_state_free,
 };
 
-static const struct drm_i915_display_funcs skl_display_funcs = {
+static const struct intel_display_funcs skl_display_funcs = {
 	.get_pipe_config = hsw_get_pipe_config,
 	.crtc_enable = hsw_crtc_enable,
 	.crtc_disable = hsw_crtc_disable,
@@ -8327,7 +8285,7 @@ static const struct drm_i915_display_funcs skl_display_funcs = {
 	.get_initial_plane_config = skl_get_initial_plane_config,
 };
 
-static const struct drm_i915_display_funcs ddi_display_funcs = {
+static const struct intel_display_funcs ddi_display_funcs = {
 	.get_pipe_config = hsw_get_pipe_config,
 	.crtc_enable = hsw_crtc_enable,
 	.crtc_disable = hsw_crtc_disable,
@@ -8335,7 +8293,7 @@ static const struct drm_i915_display_funcs ddi_display_funcs = {
 	.get_initial_plane_config = i9xx_get_initial_plane_config,
 };
 
-static const struct drm_i915_display_funcs pch_split_display_funcs = {
+static const struct intel_display_funcs pch_split_display_funcs = {
 	.get_pipe_config = ilk_get_pipe_config,
 	.crtc_enable = ilk_crtc_enable,
 	.crtc_disable = ilk_crtc_disable,
@@ -8343,7 +8301,7 @@ static const struct drm_i915_display_funcs pch_split_display_funcs = {
 	.get_initial_plane_config = i9xx_get_initial_plane_config,
 };
 
-static const struct drm_i915_display_funcs vlv_display_funcs = {
+static const struct intel_display_funcs vlv_display_funcs = {
 	.get_pipe_config = i9xx_get_pipe_config,
 	.crtc_enable = valleyview_crtc_enable,
 	.crtc_disable = i9xx_crtc_disable,
@@ -8351,7 +8309,7 @@ static const struct drm_i915_display_funcs vlv_display_funcs = {
 	.get_initial_plane_config = i9xx_get_initial_plane_config,
 };
 
-static const struct drm_i915_display_funcs i9xx_display_funcs = {
+static const struct intel_display_funcs i9xx_display_funcs = {
 	.get_pipe_config = i9xx_get_pipe_config,
 	.crtc_enable = i9xx_crtc_enable,
 	.crtc_disable = i9xx_crtc_disable,
@@ -8374,16 +8332,16 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv)
 	intel_dpll_init_clock_hook(dev_priv);
 
 	if (DISPLAY_VER(dev_priv) >= 9) {
-		dev_priv->display = &skl_display_funcs;
+		dev_priv->display.funcs.display = &skl_display_funcs;
 	} else if (HAS_DDI(dev_priv)) {
-		dev_priv->display = &ddi_display_funcs;
+		dev_priv->display.funcs.display = &ddi_display_funcs;
 	} else if (HAS_PCH_SPLIT(dev_priv)) {
-		dev_priv->display = &pch_split_display_funcs;
+		dev_priv->display.funcs.display = &pch_split_display_funcs;
 	} else if (IS_CHERRYVIEW(dev_priv) ||
 		   IS_VALLEYVIEW(dev_priv)) {
-		dev_priv->display = &vlv_display_funcs;
+		dev_priv->display.funcs.display = &vlv_display_funcs;
 	} else {
-		dev_priv->display = &i9xx_display_funcs;
+		dev_priv->display.funcs.display = &i9xx_display_funcs;
 	}
 
 	intel_fdi_init_hook(dev_priv);
@@ -8396,11 +8354,11 @@ void intel_modeset_init_hw(struct drm_i915_private *i915)
 	if (!HAS_DISPLAY(i915))
 		return;
 
-	cdclk_state = to_intel_cdclk_state(i915->cdclk.obj.state);
+	cdclk_state = to_intel_cdclk_state(i915->display.cdclk.obj.state);
 
 	intel_update_cdclk(i915);
-	intel_cdclk_dump_config(i915, &i915->cdclk.hw, "Current CDCLK");
-	cdclk_state->logical = cdclk_state->actual = i915->cdclk.hw;
+	intel_cdclk_dump_config(i915, &i915->display.cdclk.hw, "Current CDCLK");
+	cdclk_state->logical = cdclk_state->actual = i915->display.cdclk.hw;
 }
 
 static int sanitize_watermarks_add_affected(struct drm_atomic_state *state)
@@ -8456,7 +8414,7 @@ static void sanitize_watermarks(struct drm_i915_private *dev_priv)
 	int i;
 
 	/* Only supported on platforms that use atomic watermark design */
-	if (!dev_priv->wm_disp->optimize_watermarks)
+	if (!dev_priv->display.funcs.wm->optimize_watermarks)
 		return;
 
 	state = drm_atomic_state_alloc(&dev_priv->drm);
@@ -8600,6 +8558,10 @@ out:
 	return ret;
 }
 
+static const struct drm_mode_config_helper_funcs intel_mode_config_funcs = {
+	.atomic_commit_setup = drm_dp_mst_atomic_setup_commit,
+};
+
 static void intel_mode_config_init(struct drm_i915_private *i915)
 {
 	struct drm_mode_config *mode_config = &i915->drm.mode_config;
@@ -8614,6 +8576,7 @@ static void intel_mode_config_init(struct drm_i915_private *i915)
 	mode_config->prefer_shadow = 1;
 
 	mode_config->funcs = &intel_mode_funcs;
+	mode_config->helper_private = &intel_mode_config_funcs;
 
 	mode_config->async_page_flip = HAS_ASYNC_FLIPS(i915);
 
@@ -8683,11 +8646,9 @@ int intel_modeset_init_noirq(struct drm_i915_private *i915)
 
 	intel_dmc_ucode_init(i915);
 
-	i915->modeset_wq = alloc_ordered_workqueue("i915_modeset", 0);
-	i915->flip_wq = alloc_workqueue("i915_flip", WQ_HIGHPRI |
-					WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
-
-	i915->window2_delay = 0; /* No DSB so no window2 delay */
+	i915->display.wq.modeset = alloc_ordered_workqueue("i915_modeset", 0);
+	i915->display.wq.flip = alloc_workqueue("i915_flip", WQ_HIGHPRI |
+						WQ_UNBOUND, WQ_UNBOUND_MAX_ACTIVE);
 
 	intel_mode_config_init(i915);
 
@@ -8703,8 +8664,8 @@ int intel_modeset_init_noirq(struct drm_i915_private *i915)
 	if (ret)
 		goto cleanup_vga_client_pw_domain_dmc;
 
-	init_llist_head(&i915->atomic_helper.free_list);
-	INIT_WORK(&i915->atomic_helper.free_work,
+	init_llist_head(&i915->display.atomic_helper.free_list);
+	INIT_WORK(&i915->display.atomic_helper.free_work,
 		  intel_atomic_helper_free_state_worker);
 
 	intel_init_quirks(i915);
@@ -8764,7 +8725,7 @@ int intel_modeset_init_nogem(struct drm_i915_private *i915)
 
 	intel_hdcp_component_init(i915);
 
-	if (i915->max_cdclk_freq == 0)
+	if (i915->display.cdclk.max_cdclk_freq == 0)
 		intel_update_max_cdclk(i915);
 
 	/*
@@ -8828,7 +8789,7 @@ int intel_modeset_init(struct drm_i915_private *i915)
 	intel_hpd_init(i915);
 	intel_hpd_poll_disable(i915);
 
-	intel_init_ipc(i915);
+	skl_watermark_ipc_init(i915);
 
 	return 0;
 }
@@ -8959,7 +8920,7 @@ void intel_display_resume(struct drm_device *dev)
 	if (!ret)
 		ret = __intel_display_resume(i915, state, &ctx);
 
-	intel_enable_ipc(i915);
+	skl_watermark_ipc_update(i915);
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
 
@@ -8994,11 +8955,18 @@ void intel_modeset_driver_remove(struct drm_i915_private *i915)
 	if (!HAS_DISPLAY(i915))
 		return;
 
-	flush_workqueue(i915->flip_wq);
-	flush_workqueue(i915->modeset_wq);
+	flush_workqueue(i915->display.wq.flip);
+	flush_workqueue(i915->display.wq.modeset);
+
+	flush_work(&i915->display.atomic_helper.free_work);
+	drm_WARN_ON(&i915->drm, !llist_empty(&i915->display.atomic_helper.free_list));
 
-	flush_work(&i915->atomic_helper.free_work);
-	drm_WARN_ON(&i915->drm, !llist_empty(&i915->atomic_helper.free_list));
+	/*
+	 * MST topology needs to be suspended so we don't have any calls to
+	 * fbdev after it's finalized. MST will be destroyed later as part of
+	 * drm_mode_config_cleanup()
+	 */
+	intel_dp_mst_suspend(i915);
 }
 
 /* part #2: call after irq uninstall */
@@ -9013,13 +8981,6 @@ void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915)
 	 */
 	intel_hpd_poll_fini(i915);
 
-	/*
-	 * MST topology needs to be suspended so we don't have any calls to
-	 * fbdev after it's finalized. MST will be destroyed later as part of
-	 * drm_mode_config_cleanup()
-	 */
-	intel_dp_mst_suspend(i915);
-
 	/* poll work can call into fbdev, hence clean that up afterwards */
 	intel_fbdev_fini(i915);
 
@@ -9036,8 +8997,8 @@ void intel_modeset_driver_remove_noirq(struct drm_i915_private *i915)
 
 	intel_gmbus_teardown(i915);
 
-	destroy_workqueue(i915->flip_wq);
-	destroy_workqueue(i915->modeset_wq);
+	destroy_workqueue(i915->display.wq.flip);
+	destroy_workqueue(i915->display.wq.modeset);
 
 	intel_fbc_cleanup(i915);
 }
@@ -9084,7 +9045,7 @@ void intel_display_driver_register(struct drm_i915_private *i915)
 
 	/* Must be done after probing outputs */
 	intel_opregion_register(i915);
-	acpi_video_register();
+	intel_acpi_video_register(i915);
 
 	intel_audio_init(i915);
 
diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h
index fa5371036239..884e8e67b17c 100644
--- a/drivers/gpu/drm/i915/display/intel_display.h
+++ b/drivers/gpu/drm/i915/display/intel_display.h
@@ -45,7 +45,7 @@ struct drm_modeset_acquire_ctx;
 struct drm_plane;
 struct drm_plane_state;
 struct i915_address_space;
-struct i915_ggtt_view;
+struct i915_gtt_view;
 struct intel_atomic_state;
 struct intel_crtc;
 struct intel_crtc_state;
@@ -375,7 +375,7 @@ enum hpd_pin {
 
 #define for_each_pipe(__dev_priv, __p) \
 	for ((__p) = 0; (__p) < I915_MAX_PIPES; (__p)++) \
-		for_each_if(INTEL_INFO(__dev_priv)->display.pipe_mask & BIT(__p))
+		for_each_if(RUNTIME_INFO(__dev_priv)->pipe_mask & BIT(__p))
 
 #define for_each_pipe_masked(__dev_priv, __p, __mask) \
 	for_each_pipe(__dev_priv, __p) \
@@ -383,7 +383,7 @@ enum hpd_pin {
 
 #define for_each_cpu_transcoder(__dev_priv, __t) \
 	for ((__t) = 0; (__t) < I915_MAX_TRANSCODERS; (__t)++)	\
-		for_each_if (INTEL_INFO(__dev_priv)->display.cpu_transcoder_mask & BIT(__t))
+		for_each_if (RUNTIME_INFO(__dev_priv)->cpu_transcoder_mask & BIT(__t))
 
 #define for_each_cpu_transcoder_masked(__dev_priv, __t, __mask) \
 	for_each_cpu_transcoder(__dev_priv, __t) \
@@ -547,7 +547,7 @@ u8 intel_calc_active_pipes(struct intel_atomic_state *state,
 void intel_link_compute_m_n(u16 bpp, int nlanes,
 			    int pixel_clock, int link_clock,
 			    struct intel_link_m_n *m_n,
-			    bool constant_n, bool fec_enable);
+			    bool fec_enable);
 u32 intel_plane_fb_max_stride(struct drm_i915_private *dev_priv,
 			      u32 pixel_format, u64 modifier);
 enum drm_mode_status
diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h
new file mode 100644
index 000000000000..96cf994b0ad1
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_display_core.h
@@ -0,0 +1,418 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#ifndef __INTEL_DISPLAY_CORE_H__
+#define __INTEL_DISPLAY_CORE_H__
+
+#include <linux/list.h>
+#include <linux/llist.h>
+#include <linux/mutex.h>
+#include <linux/types.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+
+#include <drm/drm_connector.h>
+
+#include "intel_cdclk.h"
+#include "intel_display.h"
+#include "intel_display_power.h"
+#include "intel_dmc.h"
+#include "intel_dpll_mgr.h"
+#include "intel_fbc.h"
+#include "intel_global_state.h"
+#include "intel_gmbus.h"
+#include "intel_opregion.h"
+#include "intel_pm_types.h"
+
+struct drm_i915_private;
+struct drm_property;
+struct i915_audio_component;
+struct i915_hdcp_comp_master;
+struct intel_atomic_state;
+struct intel_audio_funcs;
+struct intel_bios_encoder_data;
+struct intel_cdclk_funcs;
+struct intel_cdclk_vals;
+struct intel_color_funcs;
+struct intel_crtc;
+struct intel_crtc_state;
+struct intel_dpll_funcs;
+struct intel_dpll_mgr;
+struct intel_fbdev;
+struct intel_fdi_funcs;
+struct intel_hotplug_funcs;
+struct intel_initial_plane_config;
+struct intel_overlay;
+
+/* Amount of SAGV/QGV points, BSpec precisely defines this */
+#define I915_NUM_QGV_POINTS 8
+
+/* Amount of PSF GV points, BSpec precisely defines this */
+#define I915_NUM_PSF_GV_POINTS 3
+
+struct intel_display_funcs {
+	/*
+	 * Returns the active state of the crtc, and if the crtc is active,
+	 * fills out the pipe-config with the hw state.
+	 */
+	bool (*get_pipe_config)(struct intel_crtc *,
+				struct intel_crtc_state *);
+	void (*get_initial_plane_config)(struct intel_crtc *,
+					 struct intel_initial_plane_config *);
+	void (*crtc_enable)(struct intel_atomic_state *state,
+			    struct intel_crtc *crtc);
+	void (*crtc_disable)(struct intel_atomic_state *state,
+			     struct intel_crtc *crtc);
+	void (*commit_modeset_enables)(struct intel_atomic_state *state);
+};
+
+/* functions used for watermark calcs for display. */
+struct intel_wm_funcs {
+	/* update_wm is for legacy wm management */
+	void (*update_wm)(struct drm_i915_private *dev_priv);
+	int (*compute_pipe_wm)(struct intel_atomic_state *state,
+			       struct intel_crtc *crtc);
+	int (*compute_intermediate_wm)(struct intel_atomic_state *state,
+				       struct intel_crtc *crtc);
+	void (*initial_watermarks)(struct intel_atomic_state *state,
+				   struct intel_crtc *crtc);
+	void (*atomic_update_watermarks)(struct intel_atomic_state *state,
+					 struct intel_crtc *crtc);
+	void (*optimize_watermarks)(struct intel_atomic_state *state,
+				    struct intel_crtc *crtc);
+	int (*compute_global_watermarks)(struct intel_atomic_state *state);
+};
+
+struct intel_audio {
+	/* hda/i915 audio component */
+	struct i915_audio_component *component;
+	bool component_registered;
+	/* mutex for audio/video sync */
+	struct mutex mutex;
+	int power_refcount;
+	u32 freq_cntrl;
+
+	/* Used to save the pipe-to-encoder mapping for audio */
+	struct intel_encoder *encoder_map[I915_MAX_PIPES];
+
+	/* necessary resource sharing with HDMI LPE audio driver. */
+	struct {
+		struct platform_device *platdev;
+		int irq;
+	} lpe;
+};
+
+/*
+ * dpll and cdclk state is protected by connection_mutex dpll.lock serializes
+ * intel_{prepare,enable,disable}_shared_dpll.  Must be global rather than per
+ * dpll, because on some platforms plls share registers.
+ */
+struct intel_dpll {
+	struct mutex lock;
+
+	int num_shared_dpll;
+	struct intel_shared_dpll shared_dplls[I915_NUM_PLLS];
+	const struct intel_dpll_mgr *mgr;
+
+	struct {
+		int nssc;
+		int ssc;
+	} ref_clks;
+};
+
+struct intel_frontbuffer_tracking {
+	spinlock_t lock;
+
+	/*
+	 * Tracking bits for delayed frontbuffer flushing du to gpu activity or
+	 * scheduled flips.
+	 */
+	unsigned busy_bits;
+	unsigned flip_bits;
+};
+
+struct intel_hotplug {
+	struct delayed_work hotplug_work;
+
+	const u32 *hpd, *pch_hpd;
+
+	struct {
+		unsigned long last_jiffies;
+		int count;
+		enum {
+			HPD_ENABLED = 0,
+			HPD_DISABLED = 1,
+			HPD_MARK_DISABLED = 2
+		} state;
+	} stats[HPD_NUM_PINS];
+	u32 event_bits;
+	u32 retry_bits;
+	struct delayed_work reenable_work;
+
+	u32 long_port_mask;
+	u32 short_port_mask;
+	struct work_struct dig_port_work;
+
+	struct work_struct poll_init_work;
+	bool poll_enabled;
+
+	unsigned int hpd_storm_threshold;
+	/* Whether or not to count short HPD IRQs in HPD storms */
+	u8 hpd_short_storm_enabled;
+
+	/*
+	 * if we get a HPD irq from DP and a HPD irq from non-DP
+	 * the non-DP HPD could block the workqueue on a mode config
+	 * mutex getting, that userspace may have taken. However
+	 * userspace is waiting on the DP workqueue to run which is
+	 * blocked behind the non-DP one.
+	 */
+	struct workqueue_struct *dp_wq;
+};
+
+struct intel_vbt_data {
+	/* bdb version */
+	u16 version;
+
+	/* Feature bits */
+	unsigned int int_tv_support:1;
+	unsigned int int_crt_support:1;
+	unsigned int lvds_use_ssc:1;
+	unsigned int int_lvds_support:1;
+	unsigned int display_clock_mode:1;
+	unsigned int fdi_rx_polarity_inverted:1;
+	int lvds_ssc_freq;
+	enum drm_panel_orientation orientation;
+
+	bool override_afc_startup;
+	u8 override_afc_startup_val;
+
+	int crt_ddc_pin;
+
+	struct list_head display_devices;
+	struct list_head bdb_blocks;
+
+	struct intel_bios_encoder_data *ports[I915_MAX_PORTS]; /* Non-NULL if port present. */
+	struct sdvo_device_mapping {
+		u8 initialized;
+		u8 dvo_port;
+		u8 slave_addr;
+		u8 dvo_wiring;
+		u8 i2c_pin;
+		u8 ddc_pin;
+	} sdvo_mappings[2];
+};
+
+struct intel_wm {
+	/*
+	 * Raw watermark latency values:
+	 * in 0.1us units for WM0,
+	 * in 0.5us units for WM1+.
+	 */
+	/* primary */
+	u16 pri_latency[5];
+	/* sprite */
+	u16 spr_latency[5];
+	/* cursor */
+	u16 cur_latency[5];
+	/*
+	 * Raw watermark memory latency values
+	 * for SKL for all 8 levels
+	 * in 1us units.
+	 */
+	u16 skl_latency[8];
+
+	/* current hardware state */
+	union {
+		struct ilk_wm_values hw;
+		struct vlv_wm_values vlv;
+		struct g4x_wm_values g4x;
+	};
+
+	u8 max_level;
+
+	/*
+	 * Should be held around atomic WM register writing; also
+	 * protects * intel_crtc->wm.active and
+	 * crtc_state->wm.need_postvbl_update.
+	 */
+	struct mutex wm_mutex;
+
+	bool ipc_enabled;
+};
+
+struct intel_display {
+	/* Display functions */
+	struct {
+		/* Top level crtc-ish functions */
+		const struct intel_display_funcs *display;
+
+		/* Display CDCLK functions */
+		const struct intel_cdclk_funcs *cdclk;
+
+		/* Display pll funcs */
+		const struct intel_dpll_funcs *dpll;
+
+		/* irq display functions */
+		const struct intel_hotplug_funcs *hotplug;
+
+		/* pm display functions */
+		const struct intel_wm_funcs *wm;
+
+		/* fdi display functions */
+		const struct intel_fdi_funcs *fdi;
+
+		/* Display internal color functions */
+		const struct intel_color_funcs *color;
+
+		/* Display internal audio functions */
+		const struct intel_audio_funcs *audio;
+	} funcs;
+
+	/* Grouping using anonymous structs. Keep sorted. */
+	struct intel_atomic_helper {
+		struct llist_head free_list;
+		struct work_struct free_work;
+	} atomic_helper;
+
+	struct {
+		/* backlight registers and fields in struct intel_panel */
+		struct mutex lock;
+	} backlight;
+
+	struct {
+		struct intel_global_obj obj;
+
+		struct intel_bw_info {
+			/* for each QGV point */
+			unsigned int deratedbw[I915_NUM_QGV_POINTS];
+			/* for each PSF GV point */
+			unsigned int psf_bw[I915_NUM_PSF_GV_POINTS];
+			u8 num_qgv_points;
+			u8 num_psf_gv_points;
+			u8 num_planes;
+		} max[6];
+	} bw;
+
+	struct {
+		/* The current hardware cdclk configuration */
+		struct intel_cdclk_config hw;
+
+		/* cdclk, divider, and ratio table from bspec */
+		const struct intel_cdclk_vals *table;
+
+		struct intel_global_obj obj;
+
+		unsigned int max_cdclk_freq;
+	} cdclk;
+
+	struct {
+		/* The current hardware dbuf configuration */
+		u8 enabled_slices;
+
+		struct intel_global_obj obj;
+	} dbuf;
+
+	struct {
+		/* VLV/CHV/BXT/GLK DSI MMIO register base address */
+		u32 mmio_base;
+	} dsi;
+
+	struct {
+		/* list of fbdev register on this device */
+		struct intel_fbdev *fbdev;
+		struct work_struct suspend_work;
+	} fbdev;
+
+	struct {
+		unsigned int pll_freq;
+		u32 rx_config;
+	} fdi;
+
+	struct {
+		/*
+		 * Base address of where the gmbus and gpio blocks are located
+		 * (either on PCH or on SoC for platforms without PCH).
+		 */
+		u32 mmio_base;
+
+		/*
+		 * gmbus.mutex protects against concurrent usage of the single
+		 * hw gmbus controller on different i2c buses.
+		 */
+		struct mutex mutex;
+
+		struct intel_gmbus *bus[GMBUS_NUM_PINS];
+
+		wait_queue_head_t wait_queue;
+	} gmbus;
+
+	struct {
+		struct i915_hdcp_comp_master *master;
+		bool comp_added;
+
+		/* Mutex to protect the above hdcp component related values. */
+		struct mutex comp_mutex;
+	} hdcp;
+
+	struct {
+		struct i915_power_domains domains;
+
+		/* Shadow for DISPLAY_PHY_CONTROL which can't be safely read */
+		u32 chv_phy_control;
+
+		/* perform PHY state sanity checks? */
+		bool chv_phy_assert[2];
+	} power;
+
+	struct {
+		u32 mmio_base;
+
+		/* protects panel power sequencer state */
+		struct mutex mutex;
+	} pps;
+
+	struct {
+		struct drm_property *broadcast_rgb;
+		struct drm_property *force_audio;
+	} properties;
+
+	struct {
+		unsigned long mask;
+	} quirks;
+
+	struct {
+		enum {
+			I915_SAGV_UNKNOWN = 0,
+			I915_SAGV_DISABLED,
+			I915_SAGV_ENABLED,
+			I915_SAGV_NOT_CONTROLLED
+		} status;
+
+		u32 block_time_us;
+	} sagv;
+
+	struct {
+		/* ordered wq for modesets */
+		struct workqueue_struct *modeset;
+
+		/* unbound hipri wq for page flips/plane updates */
+		struct workqueue_struct *flip;
+	} wq;
+
+	/* Grouping using named structs. Keep sorted. */
+	struct intel_audio audio;
+	struct intel_dmc dmc;
+	struct intel_dpll dpll;
+	struct intel_fbc *fbc[I915_MAX_FBCS];
+	struct intel_frontbuffer_tracking fb_tracking;
+	struct intel_hotplug hotplug;
+	struct intel_opregion opregion;
+	struct intel_overlay *overlay;
+	struct intel_vbt_data vbt;
+	struct intel_wm wm;
+};
+
+#endif /* __INTEL_DISPLAY_CORE_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
index 6c3954479047..7c7253a2541c 100644
--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
@@ -26,6 +26,7 @@
 #include "intel_pm.h"
 #include "intel_psr.h"
 #include "intel_sprite.h"
+#include "skl_watermark.h"
 
 static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)
 {
@@ -37,10 +38,10 @@ static int i915_frontbuffer_tracking(struct seq_file *m, void *unused)
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
 
 	seq_printf(m, "FB tracking busy bits: 0x%08x\n",
-		   dev_priv->fb_tracking.busy_bits);
+		   dev_priv->display.fb_tracking.busy_bits);
 
 	seq_printf(m, "FB tracking flip bits: 0x%08x\n",
-		   dev_priv->fb_tracking.flip_bits);
+		   dev_priv->display.fb_tracking.flip_bits);
 
 	return 0;
 }
@@ -103,7 +104,8 @@ static int i915_sr_status(struct seq_file *m, void *unused)
 
 static int i915_opregion(struct seq_file *m, void *unused)
 {
-	struct intel_opregion *opregion = &node_to_i915(m->private)->opregion;
+	struct drm_i915_private *i915 = node_to_i915(m->private);
+	struct intel_opregion *opregion = &i915->display.opregion;
 
 	if (opregion->header)
 		seq_write(m, opregion->header, OPREGION_SIZE);
@@ -113,7 +115,8 @@ static int i915_opregion(struct seq_file *m, void *unused)
 
 static int i915_vbt(struct seq_file *m, void *unused)
 {
-	struct intel_opregion *opregion = &node_to_i915(m->private)->opregion;
+	struct drm_i915_private *i915 = node_to_i915(m->private);
+	struct intel_opregion *opregion = &i915->display.opregion;
 
 	if (opregion->vbt)
 		seq_write(m, opregion->vbt, opregion->vbt_size);
@@ -129,7 +132,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
 	struct drm_framebuffer *drm_fb;
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
-	fbdev_fb = intel_fbdev_framebuffer(dev_priv->fbdev);
+	fbdev_fb = intel_fbdev_framebuffer(dev_priv->display.fbdev.fbdev);
 	if (fbdev_fb) {
 		seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ",
 			   fbdev_fb->base.width,
@@ -722,10 +725,11 @@ static void intel_scaler_info(struct seq_file *m, struct intel_crtc *crtc)
 
 	/* Not all platformas have a scaler */
 	if (num_scalers) {
-		seq_printf(m, "\tnum_scalers=%d, scaler_users=%x scaler_id=%d",
+		seq_printf(m, "\tnum_scalers=%d, scaler_users=%x scaler_id=%d scaling_filter=%d",
 			   num_scalers,
 			   crtc_state->scaler_state.scaler_users,
-			   crtc_state->scaler_state.scaler_id);
+			   crtc_state->scaler_state.scaler_id,
+			   crtc_state->hw.scaling_filter);
 
 		for (i = 0; i < num_scalers; i++) {
 			const struct intel_scaler *sc =
@@ -932,11 +936,11 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 	drm_modeset_lock_all(dev);
 
 	seq_printf(m, "PLL refclks: non-SSC: %d kHz, SSC: %d kHz\n",
-		   dev_priv->dpll.ref_clks.nssc,
-		   dev_priv->dpll.ref_clks.ssc);
+		   dev_priv->display.dpll.ref_clks.nssc,
+		   dev_priv->display.dpll.ref_clks.ssc);
 
-	for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) {
-		struct intel_shared_dpll *pll = &dev_priv->dpll.shared_dplls[i];
+	for (i = 0; i < dev_priv->display.dpll.num_shared_dpll; i++) {
+		struct intel_shared_dpll *pll = &dev_priv->display.dpll.shared_dplls[i];
 
 		seq_printf(m, "DPLL%i: %s, id: %i\n", i, pll->info->name,
 			   pll->info->id);
@@ -979,58 +983,6 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 	return 0;
 }
 
-static int i915_ipc_status_show(struct seq_file *m, void *data)
-{
-	struct drm_i915_private *dev_priv = m->private;
-
-	seq_printf(m, "Isochronous Priority Control: %s\n",
-			str_yes_no(dev_priv->ipc_enabled));
-	return 0;
-}
-
-static int i915_ipc_status_open(struct inode *inode, struct file *file)
-{
-	struct drm_i915_private *dev_priv = inode->i_private;
-
-	if (!HAS_IPC(dev_priv))
-		return -ENODEV;
-
-	return single_open(file, i915_ipc_status_show, dev_priv);
-}
-
-static ssize_t i915_ipc_status_write(struct file *file, const char __user *ubuf,
-				     size_t len, loff_t *offp)
-{
-	struct seq_file *m = file->private_data;
-	struct drm_i915_private *dev_priv = m->private;
-	intel_wakeref_t wakeref;
-	bool enable;
-	int ret;
-
-	ret = kstrtobool_from_user(ubuf, len, &enable);
-	if (ret < 0)
-		return ret;
-
-	with_intel_runtime_pm(&dev_priv->runtime_pm, wakeref) {
-		if (!dev_priv->ipc_enabled && enable)
-			drm_info(&dev_priv->drm,
-				 "Enabling IPC: WM will be proper only after next commit\n");
-		dev_priv->ipc_enabled = enable;
-		intel_enable_ipc(dev_priv);
-	}
-
-	return len;
-}
-
-static const struct file_operations i915_ipc_status_fops = {
-	.owner = THIS_MODULE,
-	.open = i915_ipc_status_open,
-	.read = seq_read,
-	.llseek = seq_lseek,
-	.release = single_release,
-	.write = i915_ipc_status_write
-};
-
 static int i915_ddb_info(struct seq_file *m, void *unused)
 {
 	struct drm_i915_private *dev_priv = node_to_i915(m->private);
@@ -1427,9 +1379,9 @@ static int pri_wm_latency_show(struct seq_file *m, void *data)
 	const u16 *latencies;
 
 	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->wm.skl_latency;
+		latencies = dev_priv->display.wm.skl_latency;
 	else
-		latencies = dev_priv->wm.pri_latency;
+		latencies = dev_priv->display.wm.pri_latency;
 
 	wm_latency_show(m, latencies);
 
@@ -1442,9 +1394,9 @@ static int spr_wm_latency_show(struct seq_file *m, void *data)
 	const u16 *latencies;
 
 	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->wm.skl_latency;
+		latencies = dev_priv->display.wm.skl_latency;
 	else
-		latencies = dev_priv->wm.spr_latency;
+		latencies = dev_priv->display.wm.spr_latency;
 
 	wm_latency_show(m, latencies);
 
@@ -1457,9 +1409,9 @@ static int cur_wm_latency_show(struct seq_file *m, void *data)
 	const u16 *latencies;
 
 	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->wm.skl_latency;
+		latencies = dev_priv->display.wm.skl_latency;
 	else
-		latencies = dev_priv->wm.cur_latency;
+		latencies = dev_priv->display.wm.cur_latency;
 
 	wm_latency_show(m, latencies);
 
@@ -1550,9 +1502,9 @@ static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf,
 	u16 *latencies;
 
 	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->wm.skl_latency;
+		latencies = dev_priv->display.wm.skl_latency;
 	else
-		latencies = dev_priv->wm.pri_latency;
+		latencies = dev_priv->display.wm.pri_latency;
 
 	return wm_latency_write(file, ubuf, len, offp, latencies);
 }
@@ -1565,9 +1517,9 @@ static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf,
 	u16 *latencies;
 
 	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->wm.skl_latency;
+		latencies = dev_priv->display.wm.skl_latency;
 	else
-		latencies = dev_priv->wm.spr_latency;
+		latencies = dev_priv->display.wm.spr_latency;
 
 	return wm_latency_write(file, ubuf, len, offp, latencies);
 }
@@ -1580,9 +1532,9 @@ static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf,
 	u16 *latencies;
 
 	if (DISPLAY_VER(dev_priv) >= 9)
-		latencies = dev_priv->wm.skl_latency;
+		latencies = dev_priv->display.wm.skl_latency;
 	else
-		latencies = dev_priv->wm.cur_latency;
+		latencies = dev_priv->display.wm.cur_latency;
 
 	return wm_latency_write(file, ubuf, len, offp, latencies);
 }
@@ -1617,14 +1569,14 @@ static const struct file_operations i915_cur_wm_latency_fops = {
 static int i915_hpd_storm_ctl_show(struct seq_file *m, void *data)
 {
 	struct drm_i915_private *dev_priv = m->private;
-	struct i915_hotplug *hotplug = &dev_priv->hotplug;
+	struct intel_hotplug *hotplug = &dev_priv->display.hotplug;
 
 	/* Synchronize with everything first in case there's been an HPD
 	 * storm, but we haven't finished handling it in the kernel yet
 	 */
 	intel_synchronize_irq(dev_priv);
-	flush_work(&dev_priv->hotplug.dig_port_work);
-	flush_delayed_work(&dev_priv->hotplug.hotplug_work);
+	flush_work(&dev_priv->display.hotplug.dig_port_work);
+	flush_delayed_work(&dev_priv->display.hotplug.hotplug_work);
 
 	seq_printf(m, "Threshold: %d\n", hotplug->hpd_storm_threshold);
 	seq_printf(m, "Detected: %s\n",
@@ -1639,7 +1591,7 @@ static ssize_t i915_hpd_storm_ctl_write(struct file *file,
 {
 	struct seq_file *m = file->private_data;
 	struct drm_i915_private *dev_priv = m->private;
-	struct i915_hotplug *hotplug = &dev_priv->hotplug;
+	struct intel_hotplug *hotplug = &dev_priv->display.hotplug;
 	unsigned int new_threshold;
 	int i;
 	char *newline;
@@ -1678,7 +1630,7 @@ static ssize_t i915_hpd_storm_ctl_write(struct file *file,
 	spin_unlock_irq(&dev_priv->irq_lock);
 
 	/* Re-enable hpd immediately if we were in an irq storm */
-	flush_delayed_work(&dev_priv->hotplug.reenable_work);
+	flush_delayed_work(&dev_priv->display.hotplug.reenable_work);
 
 	return len;
 }
@@ -1702,7 +1654,7 @@ static int i915_hpd_short_storm_ctl_show(struct seq_file *m, void *data)
 	struct drm_i915_private *dev_priv = m->private;
 
 	seq_printf(m, "Enabled: %s\n",
-		   str_yes_no(dev_priv->hotplug.hpd_short_storm_enabled));
+		   str_yes_no(dev_priv->display.hotplug.hpd_short_storm_enabled));
 
 	return 0;
 }
@@ -1720,7 +1672,7 @@ static ssize_t i915_hpd_short_storm_ctl_write(struct file *file,
 {
 	struct seq_file *m = file->private_data;
 	struct drm_i915_private *dev_priv = m->private;
-	struct i915_hotplug *hotplug = &dev_priv->hotplug;
+	struct intel_hotplug *hotplug = &dev_priv->display.hotplug;
 	char *newline;
 	char tmp[16];
 	int i;
@@ -1756,7 +1708,7 @@ static ssize_t i915_hpd_short_storm_ctl_write(struct file *file,
 	spin_unlock_irq(&dev_priv->irq_lock);
 
 	/* Re-enable hpd immediately if we were in an irq storm */
-	flush_delayed_work(&dev_priv->hotplug.reenable_work);
+	flush_delayed_work(&dev_priv->display.hotplug.reenable_work);
 
 	return len;
 }
@@ -1907,7 +1859,6 @@ static const struct {
 	{"i915_dp_test_active", &i915_displayport_test_active_fops},
 	{"i915_hpd_storm_ctl", &i915_hpd_storm_ctl_fops},
 	{"i915_hpd_short_storm_ctl", &i915_hpd_short_storm_ctl_fops},
-	{"i915_ipc_status", &i915_ipc_status_fops},
 	{"i915_drrs_ctl", &i915_drrs_ctl_fops},
 	{"i915_edp_psr_debug", &i915_edp_psr_debug_fops},
 };
@@ -1931,6 +1882,7 @@ void intel_display_debugfs_register(struct drm_i915_private *i915)
 
 	intel_dmc_debugfs_register(i915);
 	intel_fbc_debugfs_register(i915);
+	skl_watermark_ipc_debugfs_register(i915);
 }
 
 static int i915_panel_show(struct seq_file *m, void *data)
@@ -2137,7 +2089,7 @@ static const struct file_operations i915_dsc_fec_support_fops = {
 	.write = i915_dsc_fec_support_write
 };
 
-static int i915_dsc_bpp_show(struct seq_file *m, void *data)
+static int i915_dsc_bpc_show(struct seq_file *m, void *data)
 {
 	struct drm_connector *connector = m->private;
 	struct drm_device *dev = connector->dev;
@@ -2160,14 +2112,14 @@ static int i915_dsc_bpp_show(struct seq_file *m, void *data)
 	}
 
 	crtc_state = to_intel_crtc_state(crtc->state);
-	seq_printf(m, "Compressed_BPP: %d\n", crtc_state->dsc.compressed_bpp);
+	seq_printf(m, "Input_BPC: %d\n", crtc_state->dsc.config.bits_per_component);
 
 out:	drm_modeset_unlock(&dev->mode_config.connection_mutex);
 
 	return ret;
 }
 
-static ssize_t i915_dsc_bpp_write(struct file *file,
+static ssize_t i915_dsc_bpc_write(struct file *file,
 				  const char __user *ubuf,
 				  size_t len, loff_t *offp)
 {
@@ -2175,33 +2127,32 @@ static ssize_t i915_dsc_bpp_write(struct file *file,
 		((struct seq_file *)file->private_data)->private;
 	struct intel_encoder *encoder = intel_attached_encoder(to_intel_connector(connector));
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-	int dsc_bpp = 0;
+	int dsc_bpc = 0;
 	int ret;
 
-	ret = kstrtoint_from_user(ubuf, len, 0, &dsc_bpp);
+	ret = kstrtoint_from_user(ubuf, len, 0, &dsc_bpc);
 	if (ret < 0)
 		return ret;
 
-	intel_dp->force_dsc_bpp = dsc_bpp;
+	intel_dp->force_dsc_bpc = dsc_bpc;
 	*offp += len;
 
 	return len;
 }
 
-static int i915_dsc_bpp_open(struct inode *inode,
+static int i915_dsc_bpc_open(struct inode *inode,
 			     struct file *file)
 {
-	return single_open(file, i915_dsc_bpp_show,
-			   inode->i_private);
+	return single_open(file, i915_dsc_bpc_show, inode->i_private);
 }
 
-static const struct file_operations i915_dsc_bpp_fops = {
+static const struct file_operations i915_dsc_bpc_fops = {
 	.owner = THIS_MODULE,
-	.open = i915_dsc_bpp_open,
+	.open = i915_dsc_bpc_open,
 	.read = seq_read,
 	.llseek = seq_lseek,
 	.release = single_release,
-	.write = i915_dsc_bpp_write
+	.write = i915_dsc_bpc_write
 };
 
 /*
@@ -2271,8 +2222,8 @@ void intel_connector_debugfs_add(struct intel_connector *intel_connector)
 		debugfs_create_file("i915_dsc_fec_support", 0644, root,
 				    connector, &i915_dsc_fec_support_fops);
 
-		debugfs_create_file("i915_dsc_bpp", 0644, root,
-				    connector, &i915_dsc_bpp_fops);
+		debugfs_create_file("i915_dsc_bpc", 0644, root,
+				    connector, &i915_dsc_bpc_fops);
 	}
 
 	if (connector->connector_type == DRM_MODE_CONNECTOR_DSI ||
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c
index 589af257edeb..1e608b9e5055 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power.c
@@ -7,6 +7,7 @@
 
 #include "i915_drv.h"
 #include "i915_irq.h"
+#include "intel_backlight_regs.h"
 #include "intel_cdclk.h"
 #include "intel_combo_phy.h"
 #include "intel_de.h"
@@ -18,8 +19,8 @@
 #include "intel_mchbar_regs.h"
 #include "intel_pch_refclk.h"
 #include "intel_pcode.h"
-#include "intel_pm.h"
 #include "intel_snps_phy.h"
+#include "skl_watermark.h"
 #include "vlv_sideband.h"
 
 #define for_each_power_domain_well(__dev_priv, __power_well, __domain)	\
@@ -243,7 +244,7 @@ bool intel_display_power_is_enabled(struct drm_i915_private *dev_priv,
 	struct i915_power_domains *power_domains;
 	bool ret;
 
-	power_domains = &dev_priv->power_domains;
+	power_domains = &dev_priv->display.power.domains;
 
 	mutex_lock(&power_domains->lock);
 	ret = __intel_display_power_is_enabled(dev_priv, domain);
@@ -268,7 +269,7 @@ sanitize_target_dc_state(struct drm_i915_private *dev_priv,
 		if (target_dc_state != states[i])
 			continue;
 
-		if (dev_priv->dmc.allowed_dc_mask & target_dc_state)
+		if (dev_priv->display.dmc.allowed_dc_mask & target_dc_state)
 			break;
 
 		target_dc_state = states[i + 1];
@@ -291,7 +292,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
 {
 	struct i915_power_well *power_well;
 	bool dc_off_enabled;
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 
 	mutex_lock(&power_domains->lock);
 	power_well = lookup_power_well(dev_priv, SKL_DISP_DC_OFF);
@@ -301,7 +302,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
 
 	state = sanitize_target_dc_state(dev_priv, state);
 
-	if (state == dev_priv->dmc.target_dc_state)
+	if (state == dev_priv->display.dmc.target_dc_state)
 		goto unlock;
 
 	dc_off_enabled = intel_power_well_is_enabled(dev_priv, power_well);
@@ -312,7 +313,7 @@ void intel_display_power_set_target_dc_state(struct drm_i915_private *dev_priv,
 	if (!dc_off_enabled)
 		intel_power_well_enable(dev_priv, power_well);
 
-	dev_priv->dmc.target_dc_state = state;
+	dev_priv->display.dmc.target_dc_state = state;
 
 	if (!dc_off_enabled)
 		intel_power_well_disable(dev_priv, power_well);
@@ -339,7 +340,7 @@ assert_async_put_domain_masks_disjoint(struct i915_power_domains *power_domains)
 {
 	struct drm_i915_private *i915 = container_of(power_domains,
 						     struct drm_i915_private,
-						     power_domains);
+						     display.power.domains);
 
 	return !drm_WARN_ON(&i915->drm,
 			    bitmap_intersects(power_domains->async_put_domains[0].bits,
@@ -352,7 +353,7 @@ __async_put_domains_state_ok(struct i915_power_domains *power_domains)
 {
 	struct drm_i915_private *i915 = container_of(power_domains,
 						     struct drm_i915_private,
-						     power_domains);
+						     display.power.domains);
 	struct intel_power_domain_mask async_put_mask;
 	enum intel_display_power_domain domain;
 	bool err = false;
@@ -375,7 +376,7 @@ static void print_power_domains(struct i915_power_domains *power_domains,
 {
 	struct drm_i915_private *i915 = container_of(power_domains,
 						     struct drm_i915_private,
-						     power_domains);
+						     display.power.domains);
 	enum intel_display_power_domain domain;
 
 	drm_dbg(&i915->drm, "%s (%d):\n", prefix, bitmap_weight(mask->bits, POWER_DOMAIN_NUM));
@@ -390,7 +391,7 @@ print_async_put_domains_state(struct i915_power_domains *power_domains)
 {
 	struct drm_i915_private *i915 = container_of(power_domains,
 						     struct drm_i915_private,
-						     power_domains);
+						     display.power.domains);
 
 	drm_dbg(&i915->drm, "async_put_wakeref %u\n",
 		power_domains->async_put_wakeref);
@@ -445,7 +446,7 @@ static bool
 intel_display_power_grab_async_put_ref(struct drm_i915_private *dev_priv,
 				       enum intel_display_power_domain domain)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct intel_power_domain_mask async_put_mask;
 	bool ret = false;
 
@@ -474,7 +475,7 @@ static void
 __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
 				 enum intel_display_power_domain domain)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct i915_power_well *power_well;
 
 	if (intel_display_power_grab_async_put_ref(dev_priv, domain))
@@ -501,7 +502,7 @@ __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
 intel_wakeref_t intel_display_power_get(struct drm_i915_private *dev_priv,
 					enum intel_display_power_domain domain)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	intel_wakeref_t wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm);
 
 	mutex_lock(&power_domains->lock);
@@ -527,7 +528,7 @@ intel_wakeref_t
 intel_display_power_get_if_enabled(struct drm_i915_private *dev_priv,
 				   enum intel_display_power_domain domain)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	intel_wakeref_t wakeref;
 	bool is_enabled;
 
@@ -563,7 +564,7 @@ __intel_display_power_put_domain(struct drm_i915_private *dev_priv,
 	const char *name = intel_display_power_domain_str(domain);
 	struct intel_power_domain_mask async_put_mask;
 
-	power_domains = &dev_priv->power_domains;
+	power_domains = &dev_priv->display.power.domains;
 
 	drm_WARN(&dev_priv->drm, !power_domains->domain_use_count[domain],
 		 "Use count on domain %s is already zero\n",
@@ -583,7 +584,7 @@ __intel_display_power_put_domain(struct drm_i915_private *dev_priv,
 static void __intel_display_power_put(struct drm_i915_private *dev_priv,
 				      enum intel_display_power_domain domain)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 
 	mutex_lock(&power_domains->lock);
 	__intel_display_power_put_domain(dev_priv, domain);
@@ -596,7 +597,7 @@ queue_async_put_domains_work(struct i915_power_domains *power_domains,
 {
 	struct drm_i915_private *i915 = container_of(power_domains,
 						     struct drm_i915_private,
-						     power_domains);
+						     display.power.domains);
 	drm_WARN_ON(&i915->drm, power_domains->async_put_wakeref);
 	power_domains->async_put_wakeref = wakeref;
 	drm_WARN_ON(&i915->drm, !queue_delayed_work(system_unbound_wq,
@@ -610,7 +611,7 @@ release_async_put_domains(struct i915_power_domains *power_domains,
 {
 	struct drm_i915_private *dev_priv =
 		container_of(power_domains, struct drm_i915_private,
-			     power_domains);
+			     display.power.domains);
 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
 	enum intel_display_power_domain domain;
 	intel_wakeref_t wakeref;
@@ -637,8 +638,8 @@ intel_display_power_put_async_work(struct work_struct *work)
 {
 	struct drm_i915_private *dev_priv =
 		container_of(work, struct drm_i915_private,
-			     power_domains.async_put_work.work);
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+			     display.power.domains.async_put_work.work);
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
 	intel_wakeref_t new_work_wakeref = intel_runtime_pm_get_raw(rpm);
 	intel_wakeref_t old_work_wakeref = 0;
@@ -698,7 +699,7 @@ void __intel_display_power_put_async(struct drm_i915_private *i915,
 				     enum intel_display_power_domain domain,
 				     intel_wakeref_t wakeref)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 	struct intel_runtime_pm *rpm = &i915->runtime_pm;
 	intel_wakeref_t work_wakeref = intel_runtime_pm_get_raw(rpm);
 
@@ -746,7 +747,7 @@ out_verify:
  */
 void intel_display_power_flush_work(struct drm_i915_private *i915)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 	struct intel_power_domain_mask async_put_mask;
 	intel_wakeref_t work_wakeref;
 
@@ -779,7 +780,7 @@ out_verify:
 static void
 intel_display_power_flush_work_sync(struct drm_i915_private *i915)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 
 	intel_display_power_flush_work(i915);
 	cancel_delayed_work_sync(&power_domains->async_put_work);
@@ -908,7 +909,7 @@ static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
 		return 0;
 
 	if (IS_DG2(dev_priv))
-		max_dc = 0;
+		max_dc = 1;
 	else if (IS_DG1(dev_priv))
 		max_dc = 3;
 	else if (DISPLAY_VER(dev_priv) >= 12)
@@ -976,15 +977,15 @@ static u32 get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
  */
 int intel_power_domains_init(struct drm_i915_private *dev_priv)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 
 	dev_priv->params.disable_power_well =
 		sanitize_disable_power_well_option(dev_priv,
 						   dev_priv->params.disable_power_well);
-	dev_priv->dmc.allowed_dc_mask =
+	dev_priv->display.dmc.allowed_dc_mask =
 		get_allowed_dc_mask(dev_priv, dev_priv->params.enable_dc);
 
-	dev_priv->dmc.target_dc_state =
+	dev_priv->display.dmc.target_dc_state =
 		sanitize_target_dc_state(dev_priv, DC_STATE_EN_UPTO_DC6);
 
 	mutex_init(&power_domains->lock);
@@ -1003,12 +1004,12 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
  */
 void intel_power_domains_cleanup(struct drm_i915_private *dev_priv)
 {
-	intel_display_power_map_cleanup(&dev_priv->power_domains);
+	intel_display_power_map_cleanup(&dev_priv->display.power.domains);
 }
 
 static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct i915_power_well *power_well;
 
 	mutex_lock(&power_domains->lock);
@@ -1037,7 +1038,7 @@ static void gen9_dbuf_slice_set(struct drm_i915_private *dev_priv,
 void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv,
 			     u8 req_slices)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	u8 slice_mask = INTEL_INFO(dev_priv)->display.dbuf.slice_mask;
 	enum dbuf_slice slice;
 
@@ -1060,14 +1061,14 @@ void gen9_dbuf_slices_update(struct drm_i915_private *dev_priv,
 	for_each_dbuf_slice(dev_priv, slice)
 		gen9_dbuf_slice_set(dev_priv, slice, req_slices & BIT(slice));
 
-	dev_priv->dbuf.enabled_slices = req_slices;
+	dev_priv->display.dbuf.enabled_slices = req_slices;
 
 	mutex_unlock(&power_domains->lock);
 }
 
 static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
 {
-	dev_priv->dbuf.enabled_slices =
+	dev_priv->display.dbuf.enabled_slices =
 		intel_enabled_dbuf_slices_mask(dev_priv);
 
 	/*
@@ -1075,7 +1076,7 @@ static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
 	 * figure out later which slices we have and what we need.
 	 */
 	gen9_dbuf_slices_update(dev_priv, BIT(DBUF_S1) |
-				dev_priv->dbuf.enabled_slices);
+				dev_priv->display.dbuf.enabled_slices);
 }
 
 static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
@@ -1101,7 +1102,7 @@ static void icl_mbus_init(struct drm_i915_private *dev_priv)
 	unsigned long abox_regs = INTEL_INFO(dev_priv)->display.abox_mask;
 	u32 mask, val, i;
 
-	if (IS_ALDERLAKE_P(dev_priv))
+	if (IS_ALDERLAKE_P(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
 		return;
 
 	mask = MBUS_ABOX_BT_CREDIT_POOL1_MASK |
@@ -1309,7 +1310,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
 	intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
 
 	intel_update_cdclk(dev_priv);
-	intel_cdclk_dump_config(dev_priv, &dev_priv->cdclk.hw, "Current CDCLK");
+	intel_cdclk_dump_config(dev_priv, &dev_priv->display.cdclk.hw, "Current CDCLK");
 }
 
 /*
@@ -1381,6 +1382,9 @@ static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
 		reset_bits = RESET_PCH_HANDSHAKE_ENABLE;
 	}
 
+	if (DISPLAY_VER(dev_priv) >= 14)
+		reset_bits |= MTL_RESET_PICA_HANDSHAKE_EN;
+
 	val = intel_de_read(dev_priv, reg);
 
 	if (enable)
@@ -1394,7 +1398,7 @@ static void intel_pch_reset_handshake(struct drm_i915_private *dev_priv,
 static void skl_display_core_init(struct drm_i915_private *dev_priv,
 				  bool resume)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct i915_power_well *well;
 
 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
@@ -1426,13 +1430,14 @@ static void skl_display_core_init(struct drm_i915_private *dev_priv,
 
 static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct i915_power_well *well;
 
 	if (!HAS_DISPLAY(dev_priv))
 		return;
 
 	gen9_disable_dc_states(dev_priv);
+	/* TODO: disable DMC program */
 
 	gen9_dbuf_disable(dev_priv);
 
@@ -1459,7 +1464,7 @@ static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
 
 static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct i915_power_well *well;
 
 	gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
@@ -1493,13 +1498,14 @@ static void bxt_display_core_init(struct drm_i915_private *dev_priv, bool resume
 
 static void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct i915_power_well *well;
 
 	if (!HAS_DISPLAY(dev_priv))
 		return;
 
 	gen9_disable_dc_states(dev_priv);
+	/* TODO: disable DMC program */
 
 	gen9_dbuf_disable(dev_priv);
 
@@ -1601,7 +1607,7 @@ static void tgl_bw_buddy_init(struct drm_i915_private *dev_priv)
 static void icl_display_core_init(struct drm_i915_private *dev_priv,
 				  bool resume)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct i915_power_well *well;
 	u32 val;
 
@@ -1668,13 +1674,14 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv,
 
 static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	struct i915_power_well *well;
 
 	if (!HAS_DISPLAY(dev_priv))
 		return;
 
 	gen9_disable_dc_states(dev_priv);
+	intel_dmc_disable_program(dev_priv);
 
 	/* 1. Disable all display engine functions -> aready done */
 
@@ -1712,7 +1719,7 @@ static void chv_phy_control_init(struct drm_i915_private *dev_priv)
 	 * power well state and lane status to reconstruct the
 	 * expected initial value.
 	 */
-	dev_priv->chv_phy_control =
+	dev_priv->display.power.chv_phy_control =
 		PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY0) |
 		PHY_LDO_SEQ_DELAY(PHY_LDO_DELAY_600NS, DPIO_PHY1) |
 		PHY_CH_POWER_MODE(PHY_CH_DEEP_PSR, DPIO_PHY0, DPIO_CH0) |
@@ -1734,27 +1741,27 @@ static void chv_phy_control_init(struct drm_i915_private *dev_priv)
 		if (mask == 0xf)
 			mask = 0x0;
 		else
-			dev_priv->chv_phy_control |=
+			dev_priv->display.power.chv_phy_control |=
 				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH0);
 
-		dev_priv->chv_phy_control |=
+		dev_priv->display.power.chv_phy_control |=
 			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH0);
 
 		mask = (status & DPLL_PORTC_READY_MASK) >> 4;
 		if (mask == 0xf)
 			mask = 0x0;
 		else
-			dev_priv->chv_phy_control |=
+			dev_priv->display.power.chv_phy_control |=
 				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY0, DPIO_CH1);
 
-		dev_priv->chv_phy_control |=
+		dev_priv->display.power.chv_phy_control |=
 			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY0, DPIO_CH1);
 
-		dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
+		dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY0);
 
-		dev_priv->chv_phy_assert[DPIO_PHY0] = false;
+		dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = false;
 	} else {
-		dev_priv->chv_phy_assert[DPIO_PHY0] = true;
+		dev_priv->display.power.chv_phy_assert[DPIO_PHY0] = true;
 	}
 
 	if (intel_power_well_is_enabled(dev_priv, cmn_d)) {
@@ -1766,21 +1773,21 @@ static void chv_phy_control_init(struct drm_i915_private *dev_priv)
 		if (mask == 0xf)
 			mask = 0x0;
 		else
-			dev_priv->chv_phy_control |=
+			dev_priv->display.power.chv_phy_control |=
 				PHY_CH_POWER_DOWN_OVRD_EN(DPIO_PHY1, DPIO_CH0);
 
-		dev_priv->chv_phy_control |=
+		dev_priv->display.power.chv_phy_control |=
 			PHY_CH_POWER_DOWN_OVRD(mask, DPIO_PHY1, DPIO_CH0);
 
-		dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
+		dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(DPIO_PHY1);
 
-		dev_priv->chv_phy_assert[DPIO_PHY1] = false;
+		dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = false;
 	} else {
-		dev_priv->chv_phy_assert[DPIO_PHY1] = true;
+		dev_priv->display.power.chv_phy_assert[DPIO_PHY1] = true;
 	}
 
 	drm_dbg_kms(&dev_priv->drm, "Initial PHY_CONTROL=0x%08x\n",
-		    dev_priv->chv_phy_control);
+		    dev_priv->display.power.chv_phy_control);
 
 	/* Defer application of initial phy_control to enabling the powerwell */
 }
@@ -1864,7 +1871,7 @@ static void intel_power_domains_verify_state(struct drm_i915_private *dev_priv);
  */
 void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 
 	power_domains->initializing = true;
 
@@ -1905,8 +1912,8 @@ void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
 	/* Disable power support if the user asked so. */
 	if (!i915->params.disable_power_well) {
 		drm_WARN_ON(&i915->drm, power_domains->disable_wakeref);
-		i915->power_domains.disable_wakeref = intel_display_power_get(i915,
-									      POWER_DOMAIN_INIT);
+		i915->display.power.domains.disable_wakeref = intel_display_power_get(i915,
+										      POWER_DOMAIN_INIT);
 	}
 	intel_power_domains_sync_hw(i915);
 
@@ -1927,12 +1934,12 @@ void intel_power_domains_init_hw(struct drm_i915_private *i915, bool resume)
 void intel_power_domains_driver_remove(struct drm_i915_private *i915)
 {
 	intel_wakeref_t wakeref __maybe_unused =
-		fetch_and_zero(&i915->power_domains.init_wakeref);
+		fetch_and_zero(&i915->display.power.domains.init_wakeref);
 
 	/* Remove the refcount we took to keep power well support disabled. */
 	if (!i915->params.disable_power_well)
 		intel_display_power_put(i915, POWER_DOMAIN_INIT,
-					fetch_and_zero(&i915->power_domains.disable_wakeref));
+					fetch_and_zero(&i915->display.power.domains.disable_wakeref));
 
 	intel_display_power_flush_work_sync(i915);
 
@@ -1954,7 +1961,7 @@ void intel_power_domains_driver_remove(struct drm_i915_private *i915)
  */
 void intel_power_domains_sanitize_state(struct drm_i915_private *i915)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 	struct i915_power_well *power_well;
 
 	mutex_lock(&power_domains->lock);
@@ -1988,7 +1995,7 @@ void intel_power_domains_sanitize_state(struct drm_i915_private *i915)
 void intel_power_domains_enable(struct drm_i915_private *i915)
 {
 	intel_wakeref_t wakeref __maybe_unused =
-		fetch_and_zero(&i915->power_domains.init_wakeref);
+		fetch_and_zero(&i915->display.power.domains.init_wakeref);
 
 	intel_display_power_put(i915, POWER_DOMAIN_INIT, wakeref);
 	intel_power_domains_verify_state(i915);
@@ -2003,7 +2010,7 @@ void intel_power_domains_enable(struct drm_i915_private *i915)
  */
 void intel_power_domains_disable(struct drm_i915_private *i915)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 
 	drm_WARN_ON(&i915->drm, power_domains->init_wakeref);
 	power_domains->init_wakeref =
@@ -2026,7 +2033,7 @@ void intel_power_domains_disable(struct drm_i915_private *i915)
 void intel_power_domains_suspend(struct drm_i915_private *i915,
 				 enum i915_drm_suspend_mode suspend_mode)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 	intel_wakeref_t wakeref __maybe_unused =
 		fetch_and_zero(&power_domains->init_wakeref);
 
@@ -2039,7 +2046,7 @@ void intel_power_domains_suspend(struct drm_i915_private *i915,
 	 * resources as required and also enable deeper system power states
 	 * that would be blocked if the firmware was inactive.
 	 */
-	if (!(i915->dmc.allowed_dc_mask & DC_STATE_EN_DC9) &&
+	if (!(i915->display.dmc.allowed_dc_mask & DC_STATE_EN_DC9) &&
 	    suspend_mode == I915_DRM_SUSPEND_IDLE &&
 	    intel_dmc_has_payload(i915)) {
 		intel_display_power_flush_work(i915);
@@ -2053,7 +2060,7 @@ void intel_power_domains_suspend(struct drm_i915_private *i915,
 	 */
 	if (!i915->params.disable_power_well)
 		intel_display_power_put(i915, POWER_DOMAIN_INIT,
-					fetch_and_zero(&i915->power_domains.disable_wakeref));
+					fetch_and_zero(&i915->display.power.domains.disable_wakeref));
 
 	intel_display_power_flush_work(i915);
 	intel_power_domains_verify_state(i915);
@@ -2080,7 +2087,7 @@ void intel_power_domains_suspend(struct drm_i915_private *i915,
  */
 void intel_power_domains_resume(struct drm_i915_private *i915)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 
 	if (power_domains->display_core_suspended) {
 		intel_power_domains_init_hw(i915, true);
@@ -2098,7 +2105,7 @@ void intel_power_domains_resume(struct drm_i915_private *i915)
 
 static void intel_power_domains_dump_info(struct drm_i915_private *i915)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 	struct i915_power_well *power_well;
 
 	for_each_power_well(i915, power_well) {
@@ -2126,7 +2133,7 @@ static void intel_power_domains_dump_info(struct drm_i915_private *i915)
  */
 static void intel_power_domains_verify_state(struct drm_i915_private *i915)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 	struct i915_power_well *power_well;
 	bool dump_domain_info;
 
@@ -2232,10 +2239,10 @@ void intel_display_power_resume(struct drm_i915_private *i915)
 		bxt_disable_dc9(i915);
 		icl_display_core_init(i915, true);
 		if (intel_dmc_has_payload(i915)) {
-			if (i915->dmc.allowed_dc_mask &
+			if (i915->display.dmc.allowed_dc_mask &
 			    DC_STATE_EN_UPTO_DC6)
 				skl_enable_dc6(i915);
-			else if (i915->dmc.allowed_dc_mask &
+			else if (i915->display.dmc.allowed_dc_mask &
 				 DC_STATE_EN_UPTO_DC5)
 				gen9_enable_dc5(i915);
 		}
@@ -2243,7 +2250,7 @@ void intel_display_power_resume(struct drm_i915_private *i915)
 		bxt_disable_dc9(i915);
 		bxt_display_core_init(i915, true);
 		if (intel_dmc_has_payload(i915) &&
-		    (i915->dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
+		    (i915->display.dmc.allowed_dc_mask & DC_STATE_EN_UPTO_DC5))
 			gen9_enable_dc5(i915);
 	} else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) {
 		hsw_disable_pc8(i915);
@@ -2252,7 +2259,7 @@ void intel_display_power_resume(struct drm_i915_private *i915)
 
 void intel_display_power_debug(struct drm_i915_private *i915, struct seq_file *m)
 {
-	struct i915_power_domains *power_domains = &i915->power_domains;
+	struct i915_power_domains *power_domains = &i915->display.power.domains;
 	int i;
 
 	mutex_lock(&power_domains->lock);
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_map.c b/drivers/gpu/drm/i915/display/intel_display_power_map.c
index 97b367f39f35..dc04afc6cc8f 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_map.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_map.c
@@ -1350,6 +1350,117 @@ static const struct i915_power_well_desc_list xelpd_power_wells[] = {
 	I915_PW_DESCRIPTORS(xelpd_power_wells_main),
 };
 
+/*
+ * MTL is based on XELPD power domains with the exception of power gating for:
+ * - DDI_IO (moved to PLL logic)
+ * - AUX and AUX_IO functionality and register access for USBC1-4 (PICA always-on)
+ */
+#define XELPDP_PW_2_POWER_DOMAINS \
+	XELPD_PW_B_POWER_DOMAINS, \
+	XELPD_PW_C_POWER_DOMAINS, \
+	XELPD_PW_D_POWER_DOMAINS, \
+	POWER_DOMAIN_AUDIO_PLAYBACK, \
+	POWER_DOMAIN_VGA, \
+	POWER_DOMAIN_PORT_DDI_LANES_TC1, \
+	POWER_DOMAIN_PORT_DDI_LANES_TC2, \
+	POWER_DOMAIN_PORT_DDI_LANES_TC3, \
+	POWER_DOMAIN_PORT_DDI_LANES_TC4
+
+I915_DECL_PW_DOMAINS(xelpdp_pwdoms_pw_2,
+	XELPDP_PW_2_POWER_DOMAINS,
+	POWER_DOMAIN_INIT);
+
+I915_DECL_PW_DOMAINS(xelpdp_pwdoms_dc_off,
+	XELPDP_PW_2_POWER_DOMAINS,
+	POWER_DOMAIN_AUDIO_MMIO,
+	POWER_DOMAIN_MODESET,
+	POWER_DOMAIN_AUX_A,
+	POWER_DOMAIN_AUX_B,
+	POWER_DOMAIN_INIT);
+
+I915_DECL_PW_DOMAINS(xelpdp_pwdoms_aux_tc1,
+	POWER_DOMAIN_AUX_USBC1,
+	POWER_DOMAIN_AUX_TBT1);
+
+I915_DECL_PW_DOMAINS(xelpdp_pwdoms_aux_tc2,
+	POWER_DOMAIN_AUX_USBC2,
+	POWER_DOMAIN_AUX_TBT2);
+
+I915_DECL_PW_DOMAINS(xelpdp_pwdoms_aux_tc3,
+	POWER_DOMAIN_AUX_USBC3,
+	POWER_DOMAIN_AUX_TBT3);
+
+I915_DECL_PW_DOMAINS(xelpdp_pwdoms_aux_tc4,
+	POWER_DOMAIN_AUX_USBC4,
+	POWER_DOMAIN_AUX_TBT4);
+
+static const struct i915_power_well_desc xelpdp_power_wells_main[] = {
+	{
+		.instances = &I915_PW_INSTANCES(
+			I915_PW("DC_off", &xelpdp_pwdoms_dc_off,
+				.id = SKL_DISP_DC_OFF),
+		),
+		.ops = &gen9_dc_off_power_well_ops,
+	}, {
+		.instances = &I915_PW_INSTANCES(
+			I915_PW("PW_2", &xelpdp_pwdoms_pw_2,
+				.hsw.idx = ICL_PW_CTL_IDX_PW_2,
+				.id = SKL_DISP_PW_2),
+		),
+		.ops = &hsw_power_well_ops,
+		.has_vga = true,
+		.has_fuses = true,
+	}, {
+		.instances = &I915_PW_INSTANCES(
+			I915_PW("PW_A", &xelpd_pwdoms_pw_a,
+				.hsw.idx = XELPD_PW_CTL_IDX_PW_A),
+		),
+		.ops = &hsw_power_well_ops,
+		.irq_pipe_mask = BIT(PIPE_A),
+		.has_fuses = true,
+	}, {
+		.instances = &I915_PW_INSTANCES(
+			I915_PW("PW_B", &xelpd_pwdoms_pw_b,
+				.hsw.idx = XELPD_PW_CTL_IDX_PW_B),
+		),
+		.ops = &hsw_power_well_ops,
+		.irq_pipe_mask = BIT(PIPE_B),
+		.has_fuses = true,
+	}, {
+		.instances = &I915_PW_INSTANCES(
+			I915_PW("PW_C", &xelpd_pwdoms_pw_c,
+				.hsw.idx = XELPD_PW_CTL_IDX_PW_C),
+		),
+		.ops = &hsw_power_well_ops,
+		.irq_pipe_mask = BIT(PIPE_C),
+		.has_fuses = true,
+	}, {
+		.instances = &I915_PW_INSTANCES(
+			I915_PW("PW_D", &xelpd_pwdoms_pw_d,
+				.hsw.idx = XELPD_PW_CTL_IDX_PW_D),
+		),
+		.ops = &hsw_power_well_ops,
+		.irq_pipe_mask = BIT(PIPE_D),
+		.has_fuses = true,
+	}, {
+		.instances = &I915_PW_INSTANCES(
+			I915_PW("AUX_A", &icl_pwdoms_aux_a, .xelpdp.aux_ch = AUX_CH_A),
+			I915_PW("AUX_B", &icl_pwdoms_aux_b, .xelpdp.aux_ch = AUX_CH_B),
+			I915_PW("AUX_TC1", &xelpdp_pwdoms_aux_tc1, .xelpdp.aux_ch = AUX_CH_USBC1),
+			I915_PW("AUX_TC2", &xelpdp_pwdoms_aux_tc2, .xelpdp.aux_ch = AUX_CH_USBC2),
+			I915_PW("AUX_TC3", &xelpdp_pwdoms_aux_tc3, .xelpdp.aux_ch = AUX_CH_USBC3),
+			I915_PW("AUX_TC4", &xelpdp_pwdoms_aux_tc4, .xelpdp.aux_ch = AUX_CH_USBC4),
+		),
+		.ops = &xelpdp_aux_power_well_ops,
+	},
+};
+
+static const struct i915_power_well_desc_list xelpdp_power_wells[] = {
+	I915_PW_DESCRIPTORS(i9xx_power_wells_always_on),
+	I915_PW_DESCRIPTORS(icl_power_wells_pw_1),
+	I915_PW_DESCRIPTORS(xelpdp_power_wells_main),
+};
+
 static void init_power_well_domains(const struct i915_power_well_instance *inst,
 				    struct i915_power_well *power_well)
 {
@@ -1388,7 +1499,7 @@ __set_power_wells(struct i915_power_domains *power_domains,
 {
 	struct drm_i915_private *i915 = container_of(power_domains,
 						     struct drm_i915_private,
-						     power_domains);
+						     display.power.domains);
 	u64 power_well_ids = 0;
 	const struct i915_power_well_desc_list *desc_list;
 	const struct i915_power_well_desc *desc;
@@ -1447,7 +1558,7 @@ int intel_display_power_map_init(struct i915_power_domains *power_domains)
 {
 	struct drm_i915_private *i915 = container_of(power_domains,
 						     struct drm_i915_private,
-						     power_domains);
+						     display.power.domains);
 	/*
 	 * The enabling order will be from lower to higher indexed wells,
 	 * the disabling order is reversed.
@@ -1457,7 +1568,9 @@ int intel_display_power_map_init(struct i915_power_domains *power_domains)
 		return 0;
 	}
 
-	if (DISPLAY_VER(i915) >= 13)
+	if (DISPLAY_VER(i915) >= 14)
+		return set_power_wells(power_domains, xelpdp_power_wells);
+	else if (DISPLAY_VER(i915) >= 13)
 		return set_power_wells(power_domains, xelpd_power_wells);
 	else if (IS_DG1(i915))
 		return set_power_wells(power_domains, dg1_power_wells);
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c
index 91cfd5890f46..df7ee4969ef1 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c
@@ -5,6 +5,7 @@
 
 #include "i915_drv.h"
 #include "i915_irq.h"
+#include "intel_backlight_regs.h"
 #include "intel_combo_phy.h"
 #include "intel_combo_phy_regs.h"
 #include "intel_crt.h"
@@ -16,10 +17,10 @@
 #include "intel_dpll.h"
 #include "intel_hotplug.h"
 #include "intel_pcode.h"
-#include "intel_pm.h"
 #include "intel_pps.h"
 #include "intel_tc.h"
 #include "intel_vga.h"
+#include "skl_watermark.h"
 #include "vlv_sideband.h"
 #include "vlv_sideband_reg.h"
 
@@ -84,7 +85,7 @@ lookup_power_well(struct drm_i915_private *i915,
 	drm_WARN(&i915->drm, 1,
 		 "Power well %d not defined for this platform\n",
 		 power_well_id);
-	return &i915->power_domains.power_wells[0];
+	return &i915->display.power.domains.power_wells[0];
 }
 
 void intel_power_well_enable(struct drm_i915_private *i915,
@@ -710,8 +711,8 @@ void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv)
 
 	drm_dbg_kms(&dev_priv->drm,
 		    "Resetting DC state tracking from %02x to %02x\n",
-		    dev_priv->dmc.dc_state, val);
-	dev_priv->dmc.dc_state = val;
+		    dev_priv->display.dmc.dc_state, val);
+	dev_priv->display.dmc.dc_state = val;
 }
 
 /**
@@ -746,8 +747,8 @@ void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state)
 		return;
 
 	if (drm_WARN_ON_ONCE(&dev_priv->drm,
-			     state & ~dev_priv->dmc.allowed_dc_mask))
-		state &= dev_priv->dmc.allowed_dc_mask;
+			     state & ~dev_priv->display.dmc.allowed_dc_mask))
+		state &= dev_priv->display.dmc.allowed_dc_mask;
 
 	val = intel_de_read(dev_priv, DC_STATE_EN);
 	mask = gen9_dc_mask(dev_priv);
@@ -755,16 +756,16 @@ void gen9_set_dc_state(struct drm_i915_private *dev_priv, u32 state)
 		    val & mask, state);
 
 	/* Check if DMC is ignoring our DC state requests */
-	if ((val & mask) != dev_priv->dmc.dc_state)
+	if ((val & mask) != dev_priv->display.dmc.dc_state)
 		drm_err(&dev_priv->drm, "DC state mismatch (0x%x -> 0x%x)\n",
-			dev_priv->dmc.dc_state, val & mask);
+			dev_priv->display.dmc.dc_state, val & mask);
 
 	val &= ~mask;
 	val |= state;
 
 	gen9_write_dc_state(dev_priv, val);
 
-	dev_priv->dmc.dc_state = val & mask;
+	dev_priv->display.dmc.dc_state = val & mask;
 }
 
 static void tgl_enable_dc3co(struct drm_i915_private *dev_priv)
@@ -945,7 +946,7 @@ static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
 static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
 {
 	u8 hw_enabled_dbuf_slices = intel_enabled_dbuf_slices_mask(dev_priv);
-	u8 enabled_dbuf_slices = dev_priv->dbuf.enabled_slices;
+	u8 enabled_dbuf_slices = dev_priv->display.dbuf.enabled_slices;
 
 	drm_WARN(&dev_priv->drm,
 		 hw_enabled_dbuf_slices != enabled_dbuf_slices,
@@ -958,7 +959,7 @@ void gen9_disable_dc_states(struct drm_i915_private *dev_priv)
 {
 	struct intel_cdclk_config cdclk_config = {};
 
-	if (dev_priv->dmc.target_dc_state == DC_STATE_EN_DC3CO) {
+	if (dev_priv->display.dmc.target_dc_state == DC_STATE_EN_DC3CO) {
 		tgl_disable_dc3co(dev_priv);
 		return;
 	}
@@ -971,7 +972,7 @@ void gen9_disable_dc_states(struct drm_i915_private *dev_priv)
 	intel_cdclk_get_cdclk(dev_priv, &cdclk_config);
 	/* Can't read out voltage_level so can't use intel_cdclk_changed() */
 	drm_WARN_ON(&dev_priv->drm,
-		    intel_cdclk_needs_modeset(&dev_priv->cdclk.hw,
+		    intel_cdclk_needs_modeset(&dev_priv->display.cdclk.hw,
 					      &cdclk_config));
 
 	gen9_assert_dbuf_enabled(dev_priv);
@@ -1000,7 +1001,7 @@ static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
 	if (!intel_dmc_has_payload(dev_priv))
 		return;
 
-	switch (dev_priv->dmc.target_dc_state) {
+	switch (dev_priv->display.dmc.target_dc_state) {
 	case DC_STATE_EN_DC3CO:
 		tgl_enable_dc3co(dev_priv);
 		break;
@@ -1156,10 +1157,10 @@ static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
 	 * (and never recovering) in this case. intel_dsi_post_disable() will
 	 * clear it when we turn off the display.
 	 */
-	val = intel_de_read(dev_priv, DSPCLK_GATE_D);
+	val = intel_de_read(dev_priv, DSPCLK_GATE_D(dev_priv));
 	val &= DPOUNIT_CLOCK_GATE_DISABLE;
 	val |= VRHUNIT_CLOCK_GATE_DISABLE;
-	intel_de_write(dev_priv, DSPCLK_GATE_D, val);
+	intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), val);
 
 	/*
 	 * Disable trickle feed and enable pnd deadline calculation
@@ -1207,7 +1208,7 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 	 * During driver initialization/resume we can avoid restoring the
 	 * part of the HW/SW state that will be inited anyway explicitly.
 	 */
-	if (dev_priv->power_domains.initializing)
+	if (dev_priv->display.power.domains.initializing)
 		return;
 
 	intel_hpd_init(dev_priv);
@@ -1302,7 +1303,7 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
 		lookup_power_well(dev_priv, VLV_DISP_PW_DPIO_CMN_BC);
 	struct i915_power_well *cmn_d =
 		lookup_power_well(dev_priv, CHV_DISP_PW_DPIO_CMN_D);
-	u32 phy_control = dev_priv->chv_phy_control;
+	u32 phy_control = dev_priv->display.power.chv_phy_control;
 	u32 phy_status = 0;
 	u32 phy_status_mask = 0xffffffff;
 
@@ -1313,7 +1314,7 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
 	 * reset (ie. the power well has been disabled at
 	 * least once).
 	 */
-	if (!dev_priv->chv_phy_assert[DPIO_PHY0])
+	if (!dev_priv->display.power.chv_phy_assert[DPIO_PHY0])
 		phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY0, DPIO_CH0) |
 				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 0) |
 				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH0, 1) |
@@ -1321,7 +1322,7 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
 				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 0) |
 				     PHY_STATUS_SPLINE_LDO(DPIO_PHY0, DPIO_CH1, 1));
 
-	if (!dev_priv->chv_phy_assert[DPIO_PHY1])
+	if (!dev_priv->display.power.chv_phy_assert[DPIO_PHY1])
 		phy_status_mask &= ~(PHY_STATUS_CMN_LDO(DPIO_PHY1, DPIO_CH0) |
 				     PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 0) |
 				     PHY_STATUS_SPLINE_LDO(DPIO_PHY1, DPIO_CH0, 1));
@@ -1397,7 +1398,7 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
 		drm_err(&dev_priv->drm,
 			"Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
 			intel_de_read(dev_priv, DISPLAY_PHY_STATUS) & phy_status_mask,
-			phy_status, dev_priv->chv_phy_control);
+			phy_status, dev_priv->display.power.chv_phy_control);
 }
 
 #undef BITS_SET
@@ -1457,13 +1458,13 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 
 	vlv_dpio_put(dev_priv);
 
-	dev_priv->chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
+	dev_priv->display.power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
 	intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
-		       dev_priv->chv_phy_control);
+		       dev_priv->display.power.chv_phy_control);
 
 	drm_dbg_kms(&dev_priv->drm,
 		    "Enabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
-		    phy, dev_priv->chv_phy_control);
+		    phy, dev_priv->display.power.chv_phy_control);
 
 	assert_chv_phy_status(dev_priv);
 }
@@ -1487,18 +1488,18 @@ static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
 		assert_pll_disabled(dev_priv, PIPE_C);
 	}
 
-	dev_priv->chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
+	dev_priv->display.power.chv_phy_control &= ~PHY_COM_LANE_RESET_DEASSERT(phy);
 	intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
-		       dev_priv->chv_phy_control);
+		       dev_priv->display.power.chv_phy_control);
 
 	vlv_set_power_well(dev_priv, power_well, false);
 
 	drm_dbg_kms(&dev_priv->drm,
 		    "Disabled DPIO PHY%d (PHY_CONTROL=0x%08x)\n",
-		    phy, dev_priv->chv_phy_control);
+		    phy, dev_priv->display.power.chv_phy_control);
 
 	/* PHY is fully reset now, so we can enable the PHY state asserts */
-	dev_priv->chv_phy_assert[phy] = true;
+	dev_priv->display.power.chv_phy_assert[phy] = true;
 
 	assert_chv_phy_status(dev_priv);
 }
@@ -1516,7 +1517,7 @@ static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpi
 	 * reset (ie. the power well has been disabled at
 	 * least once).
 	 */
-	if (!dev_priv->chv_phy_assert[phy])
+	if (!dev_priv->display.power.chv_phy_assert[phy])
 		return;
 
 	if (ch == DPIO_CH0)
@@ -1570,27 +1571,27 @@ static void assert_chv_phy_powergate(struct drm_i915_private *dev_priv, enum dpi
 bool chv_phy_powergate_ch(struct drm_i915_private *dev_priv, enum dpio_phy phy,
 			  enum dpio_channel ch, bool override)
 {
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	bool was_override;
 
 	mutex_lock(&power_domains->lock);
 
-	was_override = dev_priv->chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
+	was_override = dev_priv->display.power.chv_phy_control & PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
 
 	if (override == was_override)
 		goto out;
 
 	if (override)
-		dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
+		dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
 	else
-		dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
+		dev_priv->display.power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
 
 	intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
-		       dev_priv->chv_phy_control);
+		       dev_priv->display.power.chv_phy_control);
 
 	drm_dbg_kms(&dev_priv->drm,
 		    "Power gating DPIO PHY%d CH%d (DPIO_PHY_CONTROL=0x%08x)\n",
-		    phy, ch, dev_priv->chv_phy_control);
+		    phy, ch, dev_priv->display.power.chv_phy_control);
 
 	assert_chv_phy_status(dev_priv);
 
@@ -1604,26 +1605,26 @@ void chv_phy_powergate_lanes(struct intel_encoder *encoder,
 			     bool override, unsigned int mask)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct i915_power_domains *power_domains = &dev_priv->power_domains;
+	struct i915_power_domains *power_domains = &dev_priv->display.power.domains;
 	enum dpio_phy phy = vlv_dig_port_to_phy(enc_to_dig_port(encoder));
 	enum dpio_channel ch = vlv_dig_port_to_channel(enc_to_dig_port(encoder));
 
 	mutex_lock(&power_domains->lock);
 
-	dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
-	dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
+	dev_priv->display.power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD(0xf, phy, ch);
+	dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD(mask, phy, ch);
 
 	if (override)
-		dev_priv->chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
+		dev_priv->display.power.chv_phy_control |= PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
 	else
-		dev_priv->chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
+		dev_priv->display.power.chv_phy_control &= ~PHY_CH_POWER_DOWN_OVRD_EN(phy, ch);
 
 	intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
-		       dev_priv->chv_phy_control);
+		       dev_priv->display.power.chv_phy_control);
 
 	drm_dbg_kms(&dev_priv->drm,
 		    "Power gating DPIO PHY%d CH%d lanes 0x%x (PHY_CONTROL=0x%08x)\n",
-		    phy, ch, mask, dev_priv->chv_phy_control);
+		    phy, ch, mask, dev_priv->display.power.chv_phy_control);
 
 	assert_chv_phy_status(dev_priv);
 
@@ -1701,7 +1702,7 @@ static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
 					struct i915_power_well *power_well)
 {
 	intel_de_write(dev_priv, DISPLAY_PHY_CONTROL,
-		       dev_priv->chv_phy_control);
+		       dev_priv->display.power.chv_phy_control);
 }
 
 static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
@@ -1797,6 +1798,43 @@ tgl_tc_cold_off_power_well_is_enabled(struct drm_i915_private *dev_priv,
 	return intel_power_well_refcount(power_well);
 }
 
+static void xelpdp_aux_power_well_enable(struct drm_i915_private *dev_priv,
+					 struct i915_power_well *power_well)
+{
+	enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
+
+	intel_de_rmw(dev_priv, XELPDP_DP_AUX_CH_CTL(aux_ch),
+		     XELPDP_DP_AUX_CH_CTL_POWER_REQUEST,
+		     XELPDP_DP_AUX_CH_CTL_POWER_REQUEST);
+
+	/*
+	 * The power status flag cannot be used to determine whether aux
+	 * power wells have finished powering up.  Instead we're
+	 * expected to just wait a fixed 600us after raising the request
+	 * bit.
+	 */
+	usleep_range(600, 1200);
+}
+
+static void xelpdp_aux_power_well_disable(struct drm_i915_private *dev_priv,
+					  struct i915_power_well *power_well)
+{
+	enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
+
+	intel_de_rmw(dev_priv, XELPDP_DP_AUX_CH_CTL(aux_ch),
+		     XELPDP_DP_AUX_CH_CTL_POWER_REQUEST,
+		     0);
+	usleep_range(10, 30);
+}
+
+static bool xelpdp_aux_power_well_enabled(struct drm_i915_private *dev_priv,
+					  struct i915_power_well *power_well)
+{
+	enum aux_ch aux_ch = i915_power_well_instance(power_well)->xelpdp.aux_ch;
+
+	return intel_de_read(dev_priv, XELPDP_DP_AUX_CH_CTL(aux_ch)) &
+		XELPDP_DP_AUX_CH_CTL_POWER_STATUS;
+}
 
 const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
 	.sync_hw = i9xx_power_well_sync_hw_noop,
@@ -1910,3 +1948,10 @@ const struct i915_power_well_ops tgl_tc_cold_off_ops = {
 	.disable = tgl_tc_cold_off_power_well_disable,
 	.is_enabled = tgl_tc_cold_off_power_well_is_enabled,
 };
+
+const struct i915_power_well_ops xelpdp_aux_power_well_ops = {
+	.sync_hw = i9xx_power_well_sync_hw_noop,
+	.enable = xelpdp_aux_power_well_enable,
+	.disable = xelpdp_aux_power_well_disable,
+	.is_enabled = xelpdp_aux_power_well_enabled,
+};
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.h b/drivers/gpu/drm/i915/display/intel_display_power_well.h
index d0624642dcb6..e13b521e322a 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.h
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.h
@@ -14,15 +14,15 @@ struct drm_i915_private;
 struct i915_power_well;
 
 #define for_each_power_well(__dev_priv, __power_well)				\
-	for ((__power_well) = (__dev_priv)->power_domains.power_wells;	\
-	     (__power_well) - (__dev_priv)->power_domains.power_wells <	\
-		(__dev_priv)->power_domains.power_well_count;		\
+	for ((__power_well) = (__dev_priv)->display.power.domains.power_wells;	\
+	     (__power_well) - (__dev_priv)->display.power.domains.power_wells <	\
+		(__dev_priv)->display.power.domains.power_well_count;		\
 	     (__power_well)++)
 
 #define for_each_power_well_reverse(__dev_priv, __power_well)			\
-	for ((__power_well) = (__dev_priv)->power_domains.power_wells +		\
-			      (__dev_priv)->power_domains.power_well_count - 1;	\
-	     (__power_well) - (__dev_priv)->power_domains.power_wells >= 0;	\
+	for ((__power_well) = (__dev_priv)->display.power.domains.power_wells +		\
+			      (__dev_priv)->display.power.domains.power_well_count - 1;	\
+	     (__power_well) - (__dev_priv)->display.power.domains.power_wells >= 0;	\
 	     (__power_well)--)
 
 /*
@@ -80,6 +80,9 @@ struct i915_power_well_instance {
 			 */
 			u8 idx;
 		} hsw;
+		struct {
+			u8 aux_ch;
+		} xelpdp;
 	};
 };
 
@@ -169,5 +172,6 @@ extern const struct i915_power_well_ops vlv_dpio_power_well_ops;
 extern const struct i915_power_well_ops icl_aux_power_well_ops;
 extern const struct i915_power_well_ops icl_ddi_power_well_ops;
 extern const struct i915_power_well_ops tgl_tc_cold_off_ops;
+extern const struct i915_power_well_ops xelpdp_aux_power_well_ops;
 
 #endif
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index 0da9b208d56e..298d00a11f47 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -105,7 +105,7 @@ struct intel_fb_view {
 	 * In the normal view the FB object's backing store sg list is used
 	 * directly and hence the remap information here is not used.
 	 */
-	struct i915_ggtt_view gtt;
+	struct i915_gtt_view gtt;
 
 	/*
 	 * The GTT view (gtt.type) specific information for each FB color
@@ -1130,6 +1130,7 @@ struct intel_crtc_state {
 	/* m2_n2 for eDP downclock */
 	struct intel_link_m_n dp_m2_n2;
 	bool has_drrs;
+	bool seamless_m_n;
 
 	/* PSR is supported but might not be enabled due the lack of enabled planes */
 	bool has_psr;
@@ -1712,7 +1713,7 @@ struct intel_dp {
 
 	/* Display stream compression testing */
 	bool force_dsc_en;
-	int force_dsc_bpp;
+	int force_dsc_bpc;
 
 	bool hobl_failed;
 	bool hobl_active;
diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c
index fa9ef591b885..e52ecc0738a6 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc.c
+++ b/drivers/gpu/drm/i915/display/intel_dmc.c
@@ -52,8 +52,8 @@
 
 #define DISPLAY_VER12_DMC_MAX_FW_SIZE	ICL_DMC_MAX_FW_SIZE
 
-#define DG2_DMC_PATH			DMC_PATH(dg2, 2, 06)
-#define DG2_DMC_VERSION_REQUIRED	DMC_VERSION(2, 06)
+#define DG2_DMC_PATH			DMC_PATH(dg2, 2, 07)
+#define DG2_DMC_VERSION_REQUIRED	DMC_VERSION(2, 07)
 MODULE_FIRMWARE(DG2_DMC_PATH);
 
 #define ADLP_DMC_PATH			DMC_PATH(adlp, 2, 16)
@@ -250,7 +250,7 @@ struct stepping_info {
 
 static bool has_dmc_id_fw(struct drm_i915_private *i915, int dmc_id)
 {
-	return i915->dmc.dmc_info[dmc_id].payload;
+	return i915->display.dmc.dmc_info[dmc_id].payload;
 }
 
 bool intel_dmc_has_payload(struct drm_i915_private *i915)
@@ -277,6 +277,17 @@ static void gen9_set_dc_state_debugmask(struct drm_i915_private *dev_priv)
 	intel_de_posting_read(dev_priv, DC_STATE_DEBUG);
 }
 
+static void disable_event_handler(struct drm_i915_private *i915,
+				  i915_reg_t ctl_reg, i915_reg_t htp_reg)
+{
+	intel_de_write(i915, ctl_reg,
+		       REG_FIELD_PREP(DMC_EVT_CTL_TYPE_MASK,
+				      DMC_EVT_CTL_TYPE_EDGE_0_1) |
+		       REG_FIELD_PREP(DMC_EVT_CTL_EVENT_ID_MASK,
+				      DMC_EVT_CTL_EVENT_ID_FALSE));
+	intel_de_write(i915, htp_reg, 0);
+}
+
 static void
 disable_flip_queue_event(struct drm_i915_private *i915,
 			 i915_reg_t ctl_reg, i915_reg_t htp_reg)
@@ -299,12 +310,7 @@ disable_flip_queue_event(struct drm_i915_private *i915,
 		return;
 	}
 
-	intel_de_write(i915, ctl_reg,
-		       REG_FIELD_PREP(DMC_EVT_CTL_TYPE_MASK,
-				      DMC_EVT_CTL_TYPE_EDGE_0_1) |
-		       REG_FIELD_PREP(DMC_EVT_CTL_EVENT_ID_MASK,
-				      DMC_EVT_CTL_EVENT_ID_FALSE));
-	intel_de_write(i915, htp_reg, 0);
+	disable_event_handler(i915, ctl_reg, htp_reg);
 }
 
 static bool
@@ -356,6 +362,51 @@ disable_all_flip_queue_events(struct drm_i915_private *i915)
 	}
 }
 
+static void disable_all_event_handlers(struct drm_i915_private *i915)
+{
+	int id;
+
+	/* TODO: disable the event handlers on pre-GEN12 platforms as well */
+	if (DISPLAY_VER(i915) < 12)
+		return;
+
+	for (id = DMC_FW_MAIN; id < DMC_FW_MAX; id++) {
+		int handler;
+
+		if (!has_dmc_id_fw(i915, id))
+			continue;
+
+		for (handler = 0; handler < DMC_EVENT_HANDLER_COUNT_GEN12; handler++)
+			disable_event_handler(i915,
+					      DMC_EVT_CTL(i915, id, handler),
+					      DMC_EVT_HTP(i915, id, handler));
+	}
+}
+
+static void pipedmc_clock_gating_wa(struct drm_i915_private *i915, bool enable)
+{
+	enum pipe pipe;
+
+	if (DISPLAY_VER(i915) != 13)
+		return;
+
+	/*
+	 * Wa_16015201720:adl-p,dg2
+	 * The WA requires clock gating to be disabled all the time
+	 * for pipe A and B.
+	 * For pipe C and D clock gating needs to be disabled only
+	 * during initializing the firmware.
+	 */
+	if (enable)
+		for (pipe = PIPE_A; pipe <= PIPE_D; pipe++)
+			intel_de_rmw(i915, CLKGATE_DIS_PSL_EXT(pipe),
+				     0, PIPEDMC_GATING_DIS);
+	else
+		for (pipe = PIPE_C; pipe <= PIPE_D; pipe++)
+			intel_de_rmw(i915, CLKGATE_DIS_PSL_EXT(pipe),
+				     PIPEDMC_GATING_DIS, 0);
+}
+
 /**
  * intel_dmc_load_program() - write the firmware from memory to register.
  * @dev_priv: i915 drm device.
@@ -366,12 +417,16 @@ disable_all_flip_queue_events(struct drm_i915_private *i915)
  */
 void intel_dmc_load_program(struct drm_i915_private *dev_priv)
 {
-	struct intel_dmc *dmc = &dev_priv->dmc;
+	struct intel_dmc *dmc = &dev_priv->display.dmc;
 	u32 id, i;
 
 	if (!intel_dmc_has_payload(dev_priv))
 		return;
 
+	pipedmc_clock_gating_wa(dev_priv, true);
+
+	disable_all_event_handlers(dev_priv);
+
 	assert_rpm_wakelock_held(&dev_priv->runtime_pm);
 
 	preempt_disable();
@@ -393,7 +448,7 @@ void intel_dmc_load_program(struct drm_i915_private *dev_priv)
 		}
 	}
 
-	dev_priv->dmc.dc_state = 0;
+	dev_priv->display.dmc.dc_state = 0;
 
 	gen9_set_dc_state_debugmask(dev_priv);
 
@@ -403,12 +458,31 @@ void intel_dmc_load_program(struct drm_i915_private *dev_priv)
 	 * here.
 	 */
 	disable_all_flip_queue_events(dev_priv);
+
+	pipedmc_clock_gating_wa(dev_priv, false);
+}
+
+/**
+ * intel_dmc_disable_program() - disable the firmware
+ * @i915: i915 drm device
+ *
+ * Disable all event handlers in the firmware, making sure the firmware is
+ * inactive after the display is uninitialized.
+ */
+void intel_dmc_disable_program(struct drm_i915_private *i915)
+{
+	if (!intel_dmc_has_payload(i915))
+		return;
+
+	pipedmc_clock_gating_wa(i915, true);
+	disable_all_event_handlers(i915);
+	pipedmc_clock_gating_wa(i915, false);
 }
 
 void assert_dmc_loaded(struct drm_i915_private *i915)
 {
 	drm_WARN_ONCE(&i915->drm,
-		      !intel_de_read(i915, DMC_PROGRAM(i915->dmc.dmc_info[DMC_FW_MAIN].start_mmioaddr, 0)),
+		      !intel_de_read(i915, DMC_PROGRAM(i915->display.dmc.dmc_info[DMC_FW_MAIN].start_mmioaddr, 0)),
 		      "DMC program storage start is NULL\n");
 	drm_WARN_ONCE(&i915->drm, !intel_de_read(i915, DMC_SSP_BASE),
 		      "DMC SSP Base Not fine\n");
@@ -445,7 +519,7 @@ static void dmc_set_fw_offset(struct intel_dmc *dmc,
 {
 	unsigned int i, id;
 
-	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc);
+	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc);
 
 	for (i = 0; i < num_entries; i++) {
 		id = package_ver <= 1 ? DMC_FW_MAIN : fw_info[i].dmc_id;
@@ -473,7 +547,7 @@ static bool dmc_mmio_addr_sanity_check(struct intel_dmc *dmc,
 				       const u32 *mmioaddr, u32 mmio_count,
 				       int header_ver, u8 dmc_id)
 {
-	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc);
+	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc);
 	u32 start_range, end_range;
 	int i;
 
@@ -511,7 +585,7 @@ static u32 parse_dmc_fw_header(struct intel_dmc *dmc,
 			       const struct intel_dmc_header_base *dmc_header,
 			       size_t rem_size, u8 dmc_id)
 {
-	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc);
+	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc);
 	struct dmc_fw_info *dmc_info = &dmc->dmc_info[dmc_id];
 	unsigned int header_len_bytes, dmc_header_size, payload_size, i;
 	const u32 *mmioaddr, *mmiodata;
@@ -622,7 +696,7 @@ parse_dmc_fw_package(struct intel_dmc *dmc,
 		     const struct stepping_info *si,
 		     size_t rem_size)
 {
-	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc);
+	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc);
 	u32 package_size = sizeof(struct intel_package_header);
 	u32 num_entries, max_entries;
 	const struct intel_fw_info *fw_info;
@@ -676,7 +750,7 @@ static u32 parse_dmc_fw_css(struct intel_dmc *dmc,
 			    struct intel_css_header *css_header,
 			    size_t rem_size)
 {
-	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), dmc);
+	struct drm_i915_private *i915 = container_of(dmc, typeof(*i915), display.dmc);
 
 	if (rem_size < sizeof(struct intel_css_header)) {
 		drm_err(&i915->drm, "Truncated DMC firmware, refusing.\n");
@@ -713,7 +787,7 @@ static void parse_dmc_fw(struct drm_i915_private *dev_priv,
 	struct intel_css_header *css_header;
 	struct intel_package_header *package_header;
 	struct intel_dmc_header_base *dmc_header;
-	struct intel_dmc *dmc = &dev_priv->dmc;
+	struct intel_dmc *dmc = &dev_priv->display.dmc;
 	struct stepping_info display_info = { '*', '*'};
 	const struct stepping_info *si = intel_get_stepping_info(dev_priv, &display_info);
 	u32 readcount = 0;
@@ -740,7 +814,7 @@ static void parse_dmc_fw(struct drm_i915_private *dev_priv,
 	readcount += r;
 
 	for (id = 0; id < DMC_FW_MAX; id++) {
-		if (!dev_priv->dmc.dmc_info[id].present)
+		if (!dev_priv->display.dmc.dmc_info[id].present)
 			continue;
 
 		offset = readcount + dmc->dmc_info[id].dmc_offset * 4;
@@ -756,15 +830,15 @@ static void parse_dmc_fw(struct drm_i915_private *dev_priv,
 
 static void intel_dmc_runtime_pm_get(struct drm_i915_private *dev_priv)
 {
-	drm_WARN_ON(&dev_priv->drm, dev_priv->dmc.wakeref);
-	dev_priv->dmc.wakeref =
+	drm_WARN_ON(&dev_priv->drm, dev_priv->display.dmc.wakeref);
+	dev_priv->display.dmc.wakeref =
 		intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
 }
 
 static void intel_dmc_runtime_pm_put(struct drm_i915_private *dev_priv)
 {
 	intel_wakeref_t wakeref __maybe_unused =
-		fetch_and_zero(&dev_priv->dmc.wakeref);
+		fetch_and_zero(&dev_priv->display.dmc.wakeref);
 
 	intel_display_power_put(dev_priv, POWER_DOMAIN_INIT, wakeref);
 }
@@ -775,10 +849,10 @@ static void dmc_load_work_fn(struct work_struct *work)
 	struct intel_dmc *dmc;
 	const struct firmware *fw = NULL;
 
-	dev_priv = container_of(work, typeof(*dev_priv), dmc.work);
-	dmc = &dev_priv->dmc;
+	dev_priv = container_of(work, typeof(*dev_priv), display.dmc.work);
+	dmc = &dev_priv->display.dmc;
 
-	request_firmware(&fw, dev_priv->dmc.fw_path, dev_priv->drm.dev);
+	request_firmware(&fw, dev_priv->display.dmc.fw_path, dev_priv->drm.dev);
 	parse_dmc_fw(dev_priv, fw);
 
 	if (intel_dmc_has_payload(dev_priv)) {
@@ -787,7 +861,7 @@ static void dmc_load_work_fn(struct work_struct *work)
 
 		drm_info(&dev_priv->drm,
 			 "Finished loading DMC firmware %s (v%u.%u)\n",
-			 dev_priv->dmc.fw_path, DMC_VERSION_MAJOR(dmc->version),
+			 dev_priv->display.dmc.fw_path, DMC_VERSION_MAJOR(dmc->version),
 			 DMC_VERSION_MINOR(dmc->version));
 	} else {
 		drm_notice(&dev_priv->drm,
@@ -810,9 +884,9 @@ static void dmc_load_work_fn(struct work_struct *work)
  */
 void intel_dmc_ucode_init(struct drm_i915_private *dev_priv)
 {
-	struct intel_dmc *dmc = &dev_priv->dmc;
+	struct intel_dmc *dmc = &dev_priv->display.dmc;
 
-	INIT_WORK(&dev_priv->dmc.work, dmc_load_work_fn);
+	INIT_WORK(&dev_priv->display.dmc.work, dmc_load_work_fn);
 
 	if (!HAS_DMC(dev_priv))
 		return;
@@ -895,7 +969,7 @@ void intel_dmc_ucode_init(struct drm_i915_private *dev_priv)
 	}
 
 	drm_dbg_kms(&dev_priv->drm, "Loading %s\n", dmc->fw_path);
-	schedule_work(&dev_priv->dmc.work);
+	schedule_work(&dev_priv->display.dmc.work);
 }
 
 /**
@@ -911,7 +985,7 @@ void intel_dmc_ucode_suspend(struct drm_i915_private *dev_priv)
 	if (!HAS_DMC(dev_priv))
 		return;
 
-	flush_work(&dev_priv->dmc.work);
+	flush_work(&dev_priv->display.dmc.work);
 
 	/* Drop the reference held in case DMC isn't loaded. */
 	if (!intel_dmc_has_payload(dev_priv))
@@ -953,16 +1027,16 @@ void intel_dmc_ucode_fini(struct drm_i915_private *dev_priv)
 		return;
 
 	intel_dmc_ucode_suspend(dev_priv);
-	drm_WARN_ON(&dev_priv->drm, dev_priv->dmc.wakeref);
+	drm_WARN_ON(&dev_priv->drm, dev_priv->display.dmc.wakeref);
 
 	for (id = 0; id < DMC_FW_MAX; id++)
-		kfree(dev_priv->dmc.dmc_info[id].payload);
+		kfree(dev_priv->display.dmc.dmc_info[id].payload);
 }
 
 void intel_dmc_print_error_state(struct drm_i915_error_state_buf *m,
 				 struct drm_i915_private *i915)
 {
-	struct intel_dmc *dmc = &i915->dmc;
+	struct intel_dmc *dmc = &i915->display.dmc;
 
 	if (!HAS_DMC(i915))
 		return;
@@ -984,7 +1058,7 @@ static int intel_dmc_debugfs_status_show(struct seq_file *m, void *unused)
 	if (!HAS_DMC(i915))
 		return -ENODEV;
 
-	dmc = &i915->dmc;
+	dmc = &i915->display.dmc;
 
 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 
diff --git a/drivers/gpu/drm/i915/display/intel_dmc.h b/drivers/gpu/drm/i915/display/intel_dmc.h
index 41091aee3b47..67e03315ef99 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc.h
+++ b/drivers/gpu/drm/i915/display/intel_dmc.h
@@ -47,6 +47,7 @@ struct intel_dmc {
 
 void intel_dmc_ucode_init(struct drm_i915_private *i915);
 void intel_dmc_load_program(struct drm_i915_private *i915);
+void intel_dmc_disable_program(struct drm_i915_private *i915);
 void intel_dmc_ucode_fini(struct drm_i915_private *i915);
 void intel_dmc_ucode_suspend(struct drm_i915_private *i915);
 void intel_dmc_ucode_resume(struct drm_i915_private *i915);
diff --git a/drivers/gpu/drm/i915/display/intel_dmc_regs.h b/drivers/gpu/drm/i915/display/intel_dmc_regs.h
index 238620b55966..5e5e41644ddf 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_dmc_regs.h
@@ -28,6 +28,8 @@
 #define _DMC_REG(i915, dmc_id, reg) \
 	((reg) - __DMC_REG_MMIO_BASE + _DMC_REG_MMIO_BASE(i915, dmc_id))
 
+#define DMC_EVENT_HANDLER_COUNT_GEN12	8
+
 #define _DMC_EVT_HTP_0			0x8f004
 
 #define DMC_EVT_HTP(i915, dmc_id, handler) \
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 3ed7eeacc706..c9be61d2348e 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -286,11 +286,22 @@ static int intel_dp_max_common_rate(struct intel_dp *intel_dp)
 	return intel_dp_common_rate(intel_dp, intel_dp->num_common_rates - 1);
 }
 
+static int intel_dp_max_source_lane_count(struct intel_digital_port *dig_port)
+{
+	int vbt_max_lanes = intel_bios_dp_max_lane_count(&dig_port->base);
+	int max_lanes = dig_port->max_lanes;
+
+	if (vbt_max_lanes)
+		max_lanes = min(max_lanes, vbt_max_lanes);
+
+	return max_lanes;
+}
+
 /* Theoretical max between source and sink */
 static int intel_dp_max_common_lane_count(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	int source_max = dig_port->max_lanes;
+	int source_max = intel_dp_max_source_lane_count(dig_port);
 	int sink_max = intel_dp->max_sink_lane_count;
 	int fia_max = intel_tc_port_fia_max_lane_count(dig_port);
 	int lttpr_max = drm_dp_lttpr_max_lane_count(intel_dp->lttpr_common_caps);
@@ -694,7 +705,7 @@ static u16 intel_dp_dsc_get_output_bpp(struct drm_i915_private *i915,
 
 	if (bigjoiner) {
 		u32 max_bpp_bigjoiner =
-			i915->max_cdclk_freq * 48 /
+			i915->display.cdclk.max_cdclk_freq * 48 /
 			intel_dp_mode_to_fec_clock(mode_clock);
 
 		bits_per_pixel = min(bits_per_pixel, max_bpp_bigjoiner);
@@ -1286,21 +1297,45 @@ intel_dp_adjust_compliance_config(struct intel_dp *intel_dp,
 	}
 }
 
+static bool has_seamless_m_n(struct intel_connector *connector)
+{
+	struct drm_i915_private *i915 = to_i915(connector->base.dev);
+
+	/*
+	 * Seamless M/N reprogramming only implemented
+	 * for BDW+ double buffered M/N registers so far.
+	 */
+	return HAS_DOUBLE_BUFFERED_M_N(i915) &&
+		intel_panel_drrs_type(connector) == DRRS_TYPE_SEAMLESS;
+}
+
+static int intel_dp_mode_clock(const struct intel_crtc_state *crtc_state,
+			       const struct drm_connector_state *conn_state)
+{
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
+	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+
+	/* FIXME a bit of a mess wrt clock vs. crtc_clock */
+	if (has_seamless_m_n(connector))
+		return intel_panel_highest_mode(connector, adjusted_mode)->clock;
+	else
+		return adjusted_mode->crtc_clock;
+}
+
 /* Optimize link config in order: max bpp, min clock, min lanes */
 static int
 intel_dp_compute_link_config_wide(struct intel_dp *intel_dp,
 				  struct intel_crtc_state *pipe_config,
+				  const struct drm_connector_state *conn_state,
 				  const struct link_config_limits *limits)
 {
-	struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
-	int bpp, i, lane_count;
+	int bpp, i, lane_count, clock = intel_dp_mode_clock(pipe_config, conn_state);
 	int mode_rate, link_rate, link_avail;
 
 	for (bpp = limits->max_bpp; bpp >= limits->min_bpp; bpp -= 2 * 3) {
 		int output_bpp = intel_dp_output_bpp(pipe_config->output_format, bpp);
 
-		mode_rate = intel_dp_link_required(adjusted_mode->crtc_clock,
-						   output_bpp);
+		mode_rate = intel_dp_link_required(clock, output_bpp);
 
 		for (i = 0; i < intel_dp->num_common_rates; i++) {
 			link_rate = intel_dp_common_rate(intel_dp, i);
@@ -1351,7 +1386,18 @@ static int intel_dp_dsc_compute_bpp(struct intel_dp *intel_dp, u8 max_req_bpc)
 	return 0;
 }
 
-#define DSC_SUPPORTED_VERSION_MIN		1
+static int intel_dp_source_dsc_version_minor(struct intel_dp *intel_dp)
+{
+	struct drm_i915_private *i915 = dp_to_i915(intel_dp);
+
+	return DISPLAY_VER(i915) >= 14 ? 2 : 1;
+}
+
+static int intel_dp_sink_dsc_version_minor(struct intel_dp *intel_dp)
+{
+	return (intel_dp->dsc_dpcd[DP_DSC_REV - DP_DSC_SUPPORT] & DP_DSC_MINOR_MASK) >>
+		DP_DSC_MINOR_SHIFT;
+}
 
 static int intel_dp_dsc_compute_params(struct intel_encoder *encoder,
 				       struct intel_crtc_state *crtc_state)
@@ -1391,9 +1437,8 @@ static int intel_dp_dsc_compute_params(struct intel_encoder *encoder,
 		(intel_dp->dsc_dpcd[DP_DSC_REV - DP_DSC_SUPPORT] &
 		 DP_DSC_MAJOR_MASK) >> DP_DSC_MAJOR_SHIFT;
 	vdsc_cfg->dsc_version_minor =
-		min(DSC_SUPPORTED_VERSION_MIN,
-		    (intel_dp->dsc_dpcd[DP_DSC_REV - DP_DSC_SUPPORT] &
-		     DP_DSC_MINOR_MASK) >> DP_DSC_MINOR_SHIFT);
+		min(intel_dp_source_dsc_version_minor(intel_dp),
+		    intel_dp_sink_dsc_version_minor(intel_dp));
 
 	vdsc_cfg->convert_rgb = intel_dp->dsc_dpcd[DP_DSC_DEC_COLOR_FORMAT_CAP - DP_DSC_SUPPORT] &
 		DP_DSC_RGB;
@@ -1439,6 +1484,11 @@ static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 
 	pipe_bpp = intel_dp_dsc_compute_bpp(intel_dp, conn_state->max_requested_bpc);
 
+	if (intel_dp->force_dsc_bpc) {
+		pipe_bpp = intel_dp->force_dsc_bpc * 3;
+		drm_dbg_kms(&dev_priv->drm, "Input DSC BPP forced to %d", pipe_bpp);
+	}
+
 	/* Min Input BPC for ICL+ is 8 */
 	if (pipe_bpp < 8 * 3) {
 		drm_dbg_kms(&dev_priv->drm,
@@ -1490,28 +1540,12 @@ static int intel_dp_dsc_compute_config(struct intel_dp *intel_dp,
 		pipe_config->dsc.slice_count = dsc_dp_slice_count;
 	}
 
-	/* As of today we support DSC for only RGB */
-	if (intel_dp->force_dsc_bpp) {
-		if (intel_dp->force_dsc_bpp >= 8 &&
-		    intel_dp->force_dsc_bpp < pipe_bpp) {
-			drm_dbg_kms(&dev_priv->drm,
-				    "DSC BPP forced to %d",
-				    intel_dp->force_dsc_bpp);
-			pipe_config->dsc.compressed_bpp =
-						intel_dp->force_dsc_bpp;
-		} else {
-			drm_dbg_kms(&dev_priv->drm,
-				    "Invalid DSC BPP %d",
-				    intel_dp->force_dsc_bpp);
-		}
-	}
-
 	/*
 	 * VDSC engine operates at 1 Pixel per clock, so if peak pixel rate
 	 * is greater than the maximum Cdclock and if slice count is even
 	 * then we need to use 2 VDSC instances.
 	 */
-	if (adjusted_mode->crtc_clock > dev_priv->max_cdclk_freq ||
+	if (adjusted_mode->crtc_clock > dev_priv->display.cdclk.max_cdclk_freq ||
 	    pipe_config->bigjoiner_pipes) {
 		if (pipe_config->dsc.slice_count < 2) {
 			drm_dbg_kms(&dev_priv->drm,
@@ -1601,7 +1635,7 @@ intel_dp_compute_link_config(struct intel_encoder *encoder,
 	 * Optimize for slow and wide for everything, because there are some
 	 * eDP 1.3 and 1.4 panels don't work well with fast and narrow.
 	 */
-	ret = intel_dp_compute_link_config_wide(intel_dp, pipe_config, &limits);
+	ret = intel_dp_compute_link_config_wide(intel_dp, pipe_config, conn_state, &limits);
 
 	if (ret || joiner_needs_dsc || intel_dp->force_dsc_en) {
 		drm_dbg_kms(&i915->drm, "Try DSC (fallback=%s, joiner=%s, force=%s)\n",
@@ -1844,8 +1878,7 @@ intel_dp_compute_hdr_metadata_infoframe_sdp(struct intel_dp *intel_dp,
 static bool cpu_transcoder_has_drrs(struct drm_i915_private *i915,
 				    enum transcoder cpu_transcoder)
 {
-	/* M1/N1 is double buffered */
-	if (DISPLAY_VER(i915) >= 9 || IS_BROADWELL(i915))
+	if (HAS_DOUBLE_BUFFERED_M_N(i915))
 		return true;
 
 	return intel_cpu_transcoder_has_m2_n2(i915, cpu_transcoder);
@@ -1883,13 +1916,16 @@ static bool can_enable_drrs(struct intel_connector *connector,
 static void
 intel_dp_drrs_compute_config(struct intel_connector *connector,
 			     struct intel_crtc_state *pipe_config,
-			     int output_bpp, bool constant_n)
+			     int output_bpp)
 {
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	const struct drm_display_mode *downclock_mode =
 		intel_panel_downclock_mode(connector, &pipe_config->hw.adjusted_mode);
 	int pixel_clock;
 
+	if (has_seamless_m_n(connector))
+		pipe_config->seamless_m_n = true;
+
 	if (!can_enable_drrs(connector, pipe_config, downclock_mode)) {
 		if (intel_cpu_transcoder_has_m2_n2(i915, pipe_config->cpu_transcoder))
 			intel_zero_m_n(&pipe_config->dp_m2_n2);
@@ -1907,7 +1943,7 @@ intel_dp_drrs_compute_config(struct intel_connector *connector,
 
 	intel_link_compute_m_n(output_bpp, pipe_config->lane_count, pixel_clock,
 			       pipe_config->port_clock, &pipe_config->dp_m2_n2,
-			       constant_n, pipe_config->fec_enable);
+			       pipe_config->fec_enable);
 
 	/* FIXME: abstract this better */
 	if (pipe_config->splitter.enable)
@@ -1982,7 +2018,6 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 	const struct drm_display_mode *fixed_mode;
 	struct intel_connector *connector = intel_dp->attached_connector;
-	bool constant_n = drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_CONSTANT_N);
 	int ret = 0, output_bpp;
 
 	if (HAS_PCH_SPLIT(dev_priv) && !HAS_DDI(dev_priv) && encoder->port != PORT_A)
@@ -2061,7 +2096,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 			       adjusted_mode->crtc_clock,
 			       pipe_config->port_clock,
 			       &pipe_config->dp_m_n,
-			       constant_n, pipe_config->fec_enable);
+			       pipe_config->fec_enable);
 
 	/* FIXME: abstract this better */
 	if (pipe_config->splitter.enable)
@@ -2072,8 +2107,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 
 	intel_vrr_compute_config(pipe_config, conn_state);
 	intel_psr_compute_config(intel_dp, pipe_config, conn_state);
-	intel_dp_drrs_compute_config(connector, pipe_config,
-				     output_bpp, constant_n);
+	intel_dp_drrs_compute_config(connector, pipe_config, output_bpp);
 	intel_dp_compute_vsc_sdp(intel_dp, pipe_config, conn_state);
 	intel_dp_compute_hdr_metadata_infoframe_sdp(intel_dp, pipe_config, conn_state);
 
@@ -4967,12 +5001,21 @@ static int intel_dp_connector_atomic_check(struct drm_connector *conn,
 {
 	struct drm_i915_private *dev_priv = to_i915(conn->dev);
 	struct intel_atomic_state *state = to_intel_atomic_state(_state);
+	struct drm_connector_state *conn_state = drm_atomic_get_new_connector_state(_state, conn);
+	struct intel_connector *intel_conn = to_intel_connector(conn);
+	struct intel_dp *intel_dp = enc_to_intel_dp(intel_conn->encoder);
 	int ret;
 
 	ret = intel_digital_connector_atomic_check(conn, &state->base);
 	if (ret)
 		return ret;
 
+	if (intel_dp_mst_source_support(intel_dp)) {
+		ret = drm_dp_mst_root_conn_atomic_check(conn_state, &intel_dp->mst_mgr);
+		if (ret)
+			return ret;
+	}
+
 	/*
 	 * We don't enable port sync on BDW due to missing w/as and
 	 * due to not having adjusted the modeset sequence appropriately.
@@ -4998,9 +5041,9 @@ static void intel_dp_oob_hotplug_event(struct drm_connector *connector)
 	struct drm_i915_private *i915 = to_i915(connector->dev);
 
 	spin_lock_irq(&i915->irq_lock);
-	i915->hotplug.event_bits |= BIT(encoder->hpd_pin);
+	i915->display.hotplug.event_bits |= BIT(encoder->hpd_pin);
 	spin_unlock_irq(&i915->irq_lock);
-	queue_delayed_work(system_wq, &i915->hotplug.hotplug_work, 0);
+	queue_delayed_work(system_wq, &i915->display.hotplug.hotplug_work, 0);
 }
 
 static const struct drm_connector_funcs intel_dp_connector_funcs = {
@@ -5158,7 +5201,7 @@ intel_edp_add_properties(struct intel_dp *intel_dp)
 		return;
 
 	drm_connector_set_panel_orientation_with_quirk(&connector->base,
-						       i915->vbt.orientation,
+						       i915->display.vbt.orientation,
 						       fixed_mode->hdisplay,
 						       fixed_mode->vdisplay);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c
index 2bc119374555..48c375c65a41 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c
@@ -42,7 +42,7 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp)
 	bool done;
 
 #define C (((status = intel_uncore_read_notrace(&i915->uncore, ch_ctl)) & DP_AUX_CH_CTL_SEND_BUSY) == 0)
-	done = wait_event_timeout(i915->gmbus_wait_queue, C,
+	done = wait_event_timeout(i915->display.gmbus.wait_queue, C,
 				  msecs_to_jiffies_timeout(timeout_ms));
 
 	/* just trace the final value */
@@ -86,7 +86,7 @@ static u32 ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 	 * divide by 2000 and use that
 	 */
 	if (dig_port->aux_ch == AUX_CH_A)
-		freq = dev_priv->cdclk.hw.cdclk;
+		freq = dev_priv->display.cdclk.hw.cdclk;
 	else
 		freq = RUNTIME_INFO(dev_priv)->rawclk_freq;
 	return DIV_ROUND_CLOSEST(freq, 2000);
@@ -150,6 +150,7 @@ static u32 skl_get_aux_send_ctl(struct intel_dp *intel_dp,
 				u32 unused)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	struct drm_i915_private *i915 =	to_i915(dig_port->base.base.dev);
 	u32 ret;
 
 	/*
@@ -170,6 +171,13 @@ static u32 skl_get_aux_send_ctl(struct intel_dp *intel_dp,
 	if (intel_tc_port_in_tbt_alt_mode(dig_port))
 		ret |= DP_AUX_CH_CTL_TBT_IO;
 
+	/*
+	 * Power request bit is already set during aux power well enable.
+	 * Preserve the bit across aux transactions.
+	 */
+	if (DISPLAY_VER(i915) >= 14)
+		ret |= XELPDP_DP_AUX_CH_CTL_POWER_REQUEST;
+
 	return ret;
 }
 
@@ -629,6 +637,46 @@ static i915_reg_t tgl_aux_data_reg(struct intel_dp *intel_dp, int index)
 	}
 }
 
+static i915_reg_t xelpdp_aux_ctl_reg(struct intel_dp *intel_dp)
+{
+	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	enum aux_ch aux_ch = dig_port->aux_ch;
+
+	switch (aux_ch) {
+	case AUX_CH_A:
+	case AUX_CH_B:
+	case AUX_CH_USBC1:
+	case AUX_CH_USBC2:
+	case AUX_CH_USBC3:
+	case AUX_CH_USBC4:
+		return XELPDP_DP_AUX_CH_CTL(aux_ch);
+	default:
+		MISSING_CASE(aux_ch);
+		return XELPDP_DP_AUX_CH_CTL(AUX_CH_A);
+	}
+}
+
+static i915_reg_t xelpdp_aux_data_reg(struct intel_dp *intel_dp, int index)
+{
+	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
+	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
+	enum aux_ch aux_ch = dig_port->aux_ch;
+
+	switch (aux_ch) {
+	case AUX_CH_A:
+	case AUX_CH_B:
+	case AUX_CH_USBC1:
+	case AUX_CH_USBC2:
+	case AUX_CH_USBC3:
+	case AUX_CH_USBC4:
+		return XELPDP_DP_AUX_CH_DATA(aux_ch, index);
+	default:
+		MISSING_CASE(aux_ch);
+		return XELPDP_DP_AUX_CH_DATA(AUX_CH_A, index);
+	}
+}
+
 void intel_dp_aux_fini(struct intel_dp *intel_dp)
 {
 	if (cpu_latency_qos_request_active(&intel_dp->pm_qos))
@@ -644,7 +692,10 @@ void intel_dp_aux_init(struct intel_dp *intel_dp)
 	struct intel_encoder *encoder = &dig_port->base;
 	enum aux_ch aux_ch = dig_port->aux_ch;
 
-	if (DISPLAY_VER(dev_priv) >= 12) {
+	if (DISPLAY_VER(dev_priv) >= 14) {
+		intel_dp->aux_ch_ctl_reg = xelpdp_aux_ctl_reg;
+		intel_dp->aux_ch_data_reg = xelpdp_aux_data_reg;
+	} else if (DISPLAY_VER(dev_priv) >= 12) {
 		intel_dp->aux_ch_ctl_reg = tgl_aux_ctl_reg;
 		intel_dp->aux_ch_data_reg = tgl_aux_data_reg;
 	} else if (DISPLAY_VER(dev_priv) >= 9) {
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
index c92d5bb2326a..83af95bce98d 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
@@ -278,6 +278,8 @@ intel_dp_aux_hdr_setup_backlight(struct intel_connector *connector, enum pipe pi
 {
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	struct intel_panel *panel = &connector->panel;
+	struct drm_luminance_range_info *luminance_range =
+		&connector->base.display_info.luminance_range;
 	int ret;
 
 	if (panel->backlight.edp.intel.sdr_uses_aux) {
@@ -293,8 +295,17 @@ intel_dp_aux_hdr_setup_backlight(struct intel_connector *connector, enum pipe pi
 		}
 	}
 
-	panel->backlight.max = 512;
-	panel->backlight.min = 0;
+	if (luminance_range->max_luminance) {
+		panel->backlight.max = luminance_range->max_luminance;
+		panel->backlight.min = luminance_range->min_luminance;
+	} else {
+		panel->backlight.max = 512;
+		panel->backlight.min = 0;
+	}
+
+	drm_dbg_kms(&i915->drm, "Using backlight range %d..%d\n", panel->backlight.min,
+		    panel->backlight.max);
+
 	panel->backlight.level = intel_dp_aux_hdr_get_backlight(connector, pipe);
 	panel->backlight.enabled = panel->backlight.level != 0;
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c
index a7640dbcf00e..88689124c013 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_hdcp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_hdcp.c
@@ -17,6 +17,7 @@
 #include "intel_dp.h"
 #include "intel_dp_hdcp.h"
 #include "intel_hdcp.h"
+#include "intel_hdcp_regs.h"
 
 static unsigned int transcoder_to_stream_enc_status(enum transcoder cpu_transcoder)
 {
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
index d213d8ad1ea5..3d3efcf02011 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
@@ -37,17 +37,6 @@ static void intel_dp_reset_lttpr_count(struct intel_dp *intel_dp)
 				    DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV] = 0;
 }
 
-static const char *intel_dp_phy_name(enum drm_dp_phy dp_phy,
-				     char *buf, size_t buf_size)
-{
-	if (dp_phy == DP_PHY_DPRX)
-		snprintf(buf, buf_size, "DPRX");
-	else
-		snprintf(buf, buf_size, "LTTPR %d", dp_phy - DP_PHY_LTTPR1 + 1);
-
-	return buf;
-}
-
 static u8 *intel_dp_lttpr_phy_caps(struct intel_dp *intel_dp,
 				   enum drm_dp_phy dp_phy)
 {
@@ -60,20 +49,19 @@ static void intel_dp_read_lttpr_phy_caps(struct intel_dp *intel_dp,
 {
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	u8 *phy_caps = intel_dp_lttpr_phy_caps(intel_dp, dp_phy);
-	char phy_name[10];
-
-	intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name));
 
 	if (drm_dp_read_lttpr_phy_caps(&intel_dp->aux, dpcd, dp_phy, phy_caps) < 0) {
 		drm_dbg_kms(&dp_to_i915(intel_dp)->drm,
 			    "[ENCODER:%d:%s][%s] failed to read the PHY caps\n",
-			    encoder->base.base.id, encoder->base.name, phy_name);
+			    encoder->base.base.id, encoder->base.name,
+			    drm_dp_phy_name(dp_phy));
 		return;
 	}
 
 	drm_dbg_kms(&dp_to_i915(intel_dp)->drm,
 		    "[ENCODER:%d:%s][%s] PHY capabilities: %*ph\n",
-		    encoder->base.base.id, encoder->base.name, phy_name,
+		    encoder->base.base.id, encoder->base.name,
+		    drm_dp_phy_name(dp_phy),
 		    (int)sizeof(intel_dp->lttpr_phy_caps[0]),
 		    phy_caps);
 }
@@ -423,14 +411,13 @@ intel_dp_get_adjust_train(struct intel_dp *intel_dp,
 {
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	char phy_name[10];
 	int lane;
 
 	if (intel_dp_is_uhbr(crtc_state)) {
 		drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] 128b/132b, lanes: %d, "
 			    "TX FFE request: " TRAIN_REQ_FMT "\n",
 			    encoder->base.base.id, encoder->base.name,
-			    intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)),
+			    drm_dp_phy_name(dp_phy),
 			    crtc_state->lane_count,
 			    TRAIN_REQ_TX_FFE_ARGS(link_status));
 	} else {
@@ -438,7 +425,7 @@ intel_dp_get_adjust_train(struct intel_dp *intel_dp,
 			    "vswing request: " TRAIN_REQ_FMT ", "
 			    "pre-emphasis request: " TRAIN_REQ_FMT "\n",
 			    encoder->base.base.id, encoder->base.name,
-			    intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)),
+			    drm_dp_phy_name(dp_phy),
 			    crtc_state->lane_count,
 			    TRAIN_REQ_VSWING_ARGS(link_status),
 			    TRAIN_REQ_PREEMPH_ARGS(link_status));
@@ -503,13 +490,12 @@ intel_dp_program_link_training_pattern(struct intel_dp *intel_dp,
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
 	u8 train_pat = intel_dp_training_pattern_symbol(dp_train_pat);
-	char phy_name[10];
 
 	if (train_pat != DP_TRAINING_PATTERN_DISABLE)
 		drm_dbg_kms(&i915->drm,
 			    "[ENCODER:%d:%s][%s] Using DP training pattern TPS%c\n",
 			    encoder->base.base.id, encoder->base.name,
-			    intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)),
+			    drm_dp_phy_name(dp_phy),
 			    dp_training_pattern_name(train_pat));
 
 	intel_dp->set_link_train(intel_dp, crtc_state, dp_train_pat);
@@ -546,13 +532,12 @@ void intel_dp_set_signal_levels(struct intel_dp *intel_dp,
 {
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	char phy_name[10];
 
 	if (intel_dp_is_uhbr(crtc_state)) {
 		drm_dbg_kms(&i915->drm, "[ENCODER:%d:%s][%s] 128b/132b, lanes: %d, "
 			    "TX FFE presets: " TRAIN_SET_FMT "\n",
 			    encoder->base.base.id, encoder->base.name,
-			    intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)),
+			    drm_dp_phy_name(dp_phy),
 			    crtc_state->lane_count,
 			    TRAIN_SET_TX_FFE_ARGS(intel_dp->train_set));
 	} else {
@@ -560,7 +545,7 @@ void intel_dp_set_signal_levels(struct intel_dp *intel_dp,
 			    "vswing levels: " TRAIN_SET_FMT ", "
 			    "pre-emphasis levels: " TRAIN_SET_FMT "\n",
 			    encoder->base.base.id, encoder->base.name,
-			    intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)),
+			    drm_dp_phy_name(dp_phy),
 			    crtc_state->lane_count,
 			    TRAIN_SET_VSWING_ARGS(intel_dp->train_set),
 			    TRAIN_SET_PREEMPH_ARGS(intel_dp->train_set));
@@ -754,12 +739,11 @@ intel_dp_dump_link_status(struct intel_dp *intel_dp, enum drm_dp_phy dp_phy,
 {
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	struct drm_i915_private *i915 = to_i915(encoder->base.dev);
-	char phy_name[10];
 
 	drm_dbg_kms(&i915->drm,
 		    "[ENCODER:%d:%s][%s] ln0_1:0x%x ln2_3:0x%x align:0x%x sink:0x%x adj_req0_1:0x%x adj_req2_3:0x%x\n",
 		    encoder->base.base.id, encoder->base.name,
-		    intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)),
+		    drm_dp_phy_name(dp_phy),
 		    link_status[0], link_status[1], link_status[2],
 		    link_status[3], link_status[4], link_status[5]);
 }
@@ -779,21 +763,19 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp,
 	int voltage_tries, cr_tries, max_cr_tries;
 	u8 link_status[DP_LINK_STATUS_SIZE];
 	bool max_vswing_reached = false;
-	char phy_name[10];
 	int delay_us;
 
 	delay_us = drm_dp_read_clock_recovery_delay(&intel_dp->aux,
 						    intel_dp->dpcd, dp_phy,
 						    intel_dp_is_uhbr(crtc_state));
 
-	intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name));
-
 	/* clock recovery */
 	if (!intel_dp_reset_link_train(intel_dp, crtc_state, dp_phy,
 				       DP_TRAINING_PATTERN_1 |
 				       DP_LINK_SCRAMBLING_DISABLE)) {
 		drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed to enable link training\n",
-			encoder->base.base.id, encoder->base.name, phy_name);
+			encoder->base.base.id, encoder->base.name,
+			drm_dp_phy_name(dp_phy));
 		return false;
 	}
 
@@ -817,14 +799,16 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp,
 		if (drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, dp_phy,
 						     link_status) < 0) {
 			drm_err(&i915->drm, "[ENCODER:%d:%s][%s] Failed to get link status\n",
-				encoder->base.base.id, encoder->base.name, phy_name);
+				encoder->base.base.id, encoder->base.name,
+				drm_dp_phy_name(dp_phy));
 			return false;
 		}
 
 		if (drm_dp_clock_recovery_ok(link_status, crtc_state->lane_count)) {
 			drm_dbg_kms(&i915->drm,
 				    "[ENCODER:%d:%s][%s] Clock recovery OK\n",
-				    encoder->base.base.id, encoder->base.name, phy_name);
+				    encoder->base.base.id, encoder->base.name,
+				    drm_dp_phy_name(dp_phy));
 			return true;
 		}
 
@@ -832,7 +816,8 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp,
 			intel_dp_dump_link_status(intel_dp, dp_phy, link_status);
 			drm_dbg_kms(&i915->drm,
 				    "[ENCODER:%d:%s][%s] Same voltage tried 5 times\n",
-				    encoder->base.base.id, encoder->base.name, phy_name);
+				    encoder->base.base.id, encoder->base.name,
+				    drm_dp_phy_name(dp_phy));
 			return false;
 		}
 
@@ -840,7 +825,8 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp,
 			intel_dp_dump_link_status(intel_dp, dp_phy, link_status);
 			drm_dbg_kms(&i915->drm,
 				    "[ENCODER:%d:%s][%s] Max Voltage Swing reached\n",
-				    encoder->base.base.id, encoder->base.name, phy_name);
+				    encoder->base.base.id, encoder->base.name,
+				    drm_dp_phy_name(dp_phy));
 			return false;
 		}
 
@@ -850,7 +836,8 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp,
 		if (!intel_dp_update_link_train(intel_dp, crtc_state, dp_phy)) {
 			drm_err(&i915->drm,
 				"[ENCODER:%d:%s][%s] Failed to update link training\n",
-				encoder->base.base.id, encoder->base.name, phy_name);
+				encoder->base.base.id, encoder->base.name,
+				drm_dp_phy_name(dp_phy));
 			return false;
 		}
 
@@ -868,7 +855,8 @@ intel_dp_link_training_clock_recovery(struct intel_dp *intel_dp,
 	intel_dp_dump_link_status(intel_dp, dp_phy, link_status);
 	drm_err(&i915->drm,
 		"[ENCODER:%d:%s][%s] Failed clock recovery %d times, giving up!\n",
-		encoder->base.base.id, encoder->base.name, phy_name, max_cr_tries);
+		encoder->base.base.id, encoder->base.name,
+		drm_dp_phy_name(dp_phy), max_cr_tries);
 
 	return false;
 }
@@ -946,15 +934,12 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp,
 	u32 training_pattern;
 	u8 link_status[DP_LINK_STATUS_SIZE];
 	bool channel_eq = false;
-	char phy_name[10];
 	int delay_us;
 
 	delay_us = drm_dp_read_channel_eq_delay(&intel_dp->aux,
 						intel_dp->dpcd, dp_phy,
 						intel_dp_is_uhbr(crtc_state));
 
-	intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name));
-
 	training_pattern = intel_dp_training_pattern(intel_dp, crtc_state, dp_phy);
 	/* Scrambling is disabled for TPS2/3 and enabled for TPS4 */
 	if (training_pattern != DP_TRAINING_PATTERN_4)
@@ -966,7 +951,7 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp,
 		drm_err(&i915->drm,
 			"[ENCODER:%d:%s][%s] Failed to start channel equalization\n",
 			encoder->base.base.id, encoder->base.name,
-			phy_name);
+			drm_dp_phy_name(dp_phy));
 		return false;
 	}
 
@@ -977,7 +962,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp,
 						     link_status) < 0) {
 			drm_err(&i915->drm,
 				"[ENCODER:%d:%s][%s] Failed to get link status\n",
-				encoder->base.base.id, encoder->base.name, phy_name);
+				encoder->base.base.id, encoder->base.name,
+				drm_dp_phy_name(dp_phy));
 			break;
 		}
 
@@ -988,7 +974,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp,
 			drm_dbg_kms(&i915->drm,
 				    "[ENCODER:%d:%s][%s] Clock recovery check failed, cannot "
 				    "continue channel equalization\n",
-				    encoder->base.base.id, encoder->base.name, phy_name);
+				    encoder->base.base.id, encoder->base.name,
+				    drm_dp_phy_name(dp_phy));
 			break;
 		}
 
@@ -997,7 +984,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp,
 			channel_eq = true;
 			drm_dbg_kms(&i915->drm,
 				    "[ENCODER:%d:%s][%s] Channel EQ done. DP Training successful\n",
-				    encoder->base.base.id, encoder->base.name, phy_name);
+				    encoder->base.base.id, encoder->base.name,
+				    drm_dp_phy_name(dp_phy));
 			break;
 		}
 
@@ -1007,7 +995,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp,
 		if (!intel_dp_update_link_train(intel_dp, crtc_state, dp_phy)) {
 			drm_err(&i915->drm,
 				"[ENCODER:%d:%s][%s] Failed to update link training\n",
-				encoder->base.base.id, encoder->base.name, phy_name);
+				encoder->base.base.id, encoder->base.name,
+				drm_dp_phy_name(dp_phy));
 			break;
 		}
 	}
@@ -1017,7 +1006,8 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp,
 		intel_dp_dump_link_status(intel_dp, dp_phy, link_status);
 		drm_dbg_kms(&i915->drm,
 			    "[ENCODER:%d:%s][%s] Channel equalization failed 5 times\n",
-			    encoder->base.base.id, encoder->base.name, phy_name);
+			    encoder->base.base.id, encoder->base.name,
+			    drm_dp_phy_name(dp_phy));
 	}
 
 	return channel_eq;
@@ -1092,7 +1082,6 @@ intel_dp_link_train_phy(struct intel_dp *intel_dp,
 {
 	struct intel_connector *connector = intel_dp->attached_connector;
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
-	char phy_name[10];
 	bool ret = false;
 
 	if (!intel_dp_link_training_clock_recovery(intel_dp, crtc_state, dp_phy))
@@ -1108,7 +1097,7 @@ out:
 		    "[CONNECTOR:%d:%s][ENCODER:%d:%s][%s] Link Training %s at link rate = %d, lane count = %d\n",
 		    connector->base.base.id, connector->base.name,
 		    encoder->base.base.id, encoder->base.name,
-		    intel_dp_phy_name(dp_phy, phy_name, sizeof(phy_name)),
+		    drm_dp_phy_name(dp_phy),
 		    ret ? "passed" : "failed",
 		    crtc_state->port_clock, crtc_state->lane_count);
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c
index 14d2a64193b2..03604a37931c 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c
@@ -52,30 +52,36 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder,
 	struct drm_atomic_state *state = crtc_state->uapi.state;
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder);
 	struct intel_dp *intel_dp = &intel_mst->primary->dp;
+	struct drm_dp_mst_topology_state *mst_state;
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
 	const struct drm_display_mode *adjusted_mode =
 		&crtc_state->hw.adjusted_mode;
-	bool constant_n = drm_dp_has_quirk(&intel_dp->desc, DP_DPCD_QUIRK_CONSTANT_N);
 	int bpp, slots = -EINVAL;
 
+	mst_state = drm_atomic_get_mst_topology_state(state, &intel_dp->mst_mgr);
+	if (IS_ERR(mst_state))
+		return PTR_ERR(mst_state);
+
 	crtc_state->lane_count = limits->max_lane_count;
 	crtc_state->port_clock = limits->max_rate;
 
+	// TODO: Handle pbn_div changes by adding a new MST helper
+	if (!mst_state->pbn_div) {
+		mst_state->pbn_div = drm_dp_get_vc_payload_bw(&intel_dp->mst_mgr,
+							      limits->max_rate,
+							      limits->max_lane_count);
+	}
+
 	for (bpp = limits->max_bpp; bpp >= limits->min_bpp; bpp -= 2 * 3) {
 		crtc_state->pipe_bpp = bpp;
 
 		crtc_state->pbn = drm_dp_calc_pbn_mode(adjusted_mode->crtc_clock,
 						       crtc_state->pipe_bpp,
 						       false);
-
-		slots = drm_dp_atomic_find_vcpi_slots(state, &intel_dp->mst_mgr,
-						      connector->port,
-						      crtc_state->pbn,
-						      drm_dp_get_vc_payload_bw(&intel_dp->mst_mgr,
-									       crtc_state->port_clock,
-									       crtc_state->lane_count));
+		slots = drm_dp_atomic_find_time_slots(state, &intel_dp->mst_mgr,
+						      connector->port, crtc_state->pbn);
 		if (slots == -EDEADLK)
 			return slots;
 		if (slots >= 0)
@@ -93,7 +99,7 @@ static int intel_dp_mst_compute_link_config(struct intel_encoder *encoder,
 			       adjusted_mode->crtc_clock,
 			       crtc_state->port_clock,
 			       &crtc_state->dp_m_n,
-			       constant_n, crtc_state->fec_enable);
+			       crtc_state->fec_enable);
 	crtc_state->dp_m_n.tu = slots;
 
 	return 0;
@@ -308,14 +314,8 @@ intel_dp_mst_atomic_check(struct drm_connector *connector,
 			  struct drm_atomic_state *_state)
 {
 	struct intel_atomic_state *state = to_intel_atomic_state(_state);
-	struct drm_connector_state *new_conn_state =
-		drm_atomic_get_new_connector_state(&state->base, connector);
-	struct drm_connector_state *old_conn_state =
-		drm_atomic_get_old_connector_state(&state->base, connector);
 	struct intel_connector *intel_connector =
 		to_intel_connector(connector);
-	struct drm_crtc *new_crtc = new_conn_state->crtc;
-	struct drm_dp_mst_topology_mgr *mgr;
 	int ret;
 
 	ret = intel_digital_connector_atomic_check(connector, &state->base);
@@ -326,28 +326,9 @@ intel_dp_mst_atomic_check(struct drm_connector *connector,
 	if (ret)
 		return ret;
 
-	if (!old_conn_state->crtc)
-		return 0;
-
-	/* We only want to free VCPI if this state disables the CRTC on this
-	 * connector
-	 */
-	if (new_crtc) {
-		struct intel_crtc *crtc = to_intel_crtc(new_crtc);
-		struct intel_crtc_state *crtc_state =
-			intel_atomic_get_new_crtc_state(state, crtc);
-
-		if (!crtc_state ||
-		    !drm_atomic_crtc_needs_modeset(&crtc_state->uapi) ||
-		    crtc_state->uapi.enable)
-			return 0;
-	}
-
-	mgr = &enc_to_mst(to_intel_encoder(old_conn_state->best_encoder))->primary->dp.mst_mgr;
-	ret = drm_dp_atomic_release_vcpi_slots(&state->base, mgr,
-					       intel_connector->port);
-
-	return ret;
+	return drm_dp_atomic_release_time_slots(&state->base,
+						&intel_connector->mst_port->mst_mgr,
+						intel_connector->port);
 }
 
 static void clear_act_sent(struct intel_encoder *encoder,
@@ -383,21 +364,17 @@ static void intel_mst_disable_dp(struct intel_atomic_state *state,
 	struct intel_dp *intel_dp = &dig_port->dp;
 	struct intel_connector *connector =
 		to_intel_connector(old_conn_state->connector);
+	struct drm_dp_mst_topology_state *mst_state =
+		drm_atomic_get_mst_topology_state(&state->base, &intel_dp->mst_mgr);
 	struct drm_i915_private *i915 = to_i915(connector->base.dev);
-	int start_slot = intel_dp_is_uhbr(old_crtc_state) ? 0 : 1;
-	int ret;
 
 	drm_dbg_kms(&i915->drm, "active links %d\n",
 		    intel_dp->active_mst_links);
 
 	intel_hdcp_disable(intel_mst->connector);
 
-	drm_dp_mst_reset_vcpi_slots(&intel_dp->mst_mgr, connector->port);
-
-	ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr, start_slot);
-	if (ret) {
-		drm_dbg_kms(&i915->drm, "failed to update payload %d\n", ret);
-	}
+	drm_dp_remove_payload(&intel_dp->mst_mgr, mst_state,
+			      drm_atomic_get_mst_payload_state(mst_state, connector->port));
 
 	intel_audio_codec_disable(encoder, old_crtc_state, old_conn_state);
 }
@@ -425,8 +402,6 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 
 	intel_disable_transcoder(old_crtc_state);
 
-	drm_dp_update_payload_part2(&intel_dp->mst_mgr);
-
 	clear_act_sent(encoder, old_crtc_state);
 
 	intel_de_rmw(dev_priv, TRANS_DDI_FUNC_CTL(old_crtc_state->cpu_transcoder),
@@ -434,8 +409,6 @@ static void intel_mst_post_disable_dp(struct intel_atomic_state *state,
 
 	wait_for_act_sent(encoder, old_crtc_state);
 
-	drm_dp_mst_deallocate_vcpi(&intel_dp->mst_mgr, connector->port);
-
 	intel_ddi_disable_transcoder_func(old_crtc_state);
 
 	if (DISPLAY_VER(dev_priv) >= 9)
@@ -502,7 +475,8 @@ static void intel_mst_pre_enable_dp(struct intel_atomic_state *state,
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
-	int start_slot = intel_dp_is_uhbr(pipe_config) ? 0 : 1;
+	struct drm_dp_mst_topology_state *mst_state =
+		drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr);
 	int ret;
 	bool first_mst_stream;
 
@@ -528,16 +502,13 @@ static void intel_mst_pre_enable_dp(struct intel_atomic_state *state,
 		dig_port->base.pre_enable(state, &dig_port->base,
 						pipe_config, NULL);
 
-	ret = drm_dp_mst_allocate_vcpi(&intel_dp->mst_mgr,
-				       connector->port,
-				       pipe_config->pbn,
-				       pipe_config->dp_m_n.tu);
-	if (!ret)
-		drm_err(&dev_priv->drm, "failed to allocate vcpi\n");
-
 	intel_dp->active_mst_links++;
 
-	ret = drm_dp_update_payload_part1(&intel_dp->mst_mgr, start_slot);
+	ret = drm_dp_add_payload_part1(&intel_dp->mst_mgr, mst_state,
+				       drm_atomic_get_mst_payload_state(mst_state, connector->port));
+	if (ret < 0)
+		drm_err(&dev_priv->drm, "Failed to create MST payload for %s: %d\n",
+			connector->base.name, ret);
 
 	/*
 	 * Before Gen 12 this is not done as part of
@@ -560,7 +531,10 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state,
 	struct intel_dp_mst_encoder *intel_mst = enc_to_mst(encoder);
 	struct intel_digital_port *dig_port = intel_mst->primary;
 	struct intel_dp *intel_dp = &dig_port->dp;
+	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct drm_dp_mst_topology_state *mst_state =
+		drm_atomic_get_new_mst_topology_state(&state->base, &intel_dp->mst_mgr);
 	enum transcoder trans = pipe_config->cpu_transcoder;
 
 	drm_WARN_ON(&dev_priv->drm, pipe_config->has_pch_encoder);
@@ -588,9 +562,13 @@ static void intel_mst_enable_dp(struct intel_atomic_state *state,
 
 	wait_for_act_sent(encoder, pipe_config);
 
-	drm_dp_update_payload_part2(&intel_dp->mst_mgr);
+	drm_dp_add_payload_part2(&intel_dp->mst_mgr, &state->base,
+				 drm_atomic_get_mst_payload_state(mst_state, connector->port));
 
-	if (DISPLAY_VER(dev_priv) >= 12 && pipe_config->fec_enable)
+	if (DISPLAY_VER(dev_priv) >= 14 && pipe_config->fec_enable)
+		intel_de_rmw(dev_priv, MTL_CHICKEN_TRANS(trans), 0,
+			     FECSTALL_DIS_DPTSTREAM_DPTTG);
+	else if (DISPLAY_VER(dev_priv) >= 12 && pipe_config->fec_enable)
 		intel_de_rmw(dev_priv, CHICKEN_TRANS(trans), 0,
 			     FECSTALL_DIS_DPTSTREAM_DPTTG);
 
@@ -972,8 +950,6 @@ intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_base_id)
 	struct intel_dp *intel_dp = &dig_port->dp;
 	enum port port = dig_port->base.port;
 	int ret;
-	int max_source_rate =
-		intel_dp->source_rates[intel_dp->num_source_rates - 1];
 
 	if (!HAS_DP_MST(i915) || intel_dp_is_edp(intel_dp))
 		return 0;
@@ -989,10 +965,7 @@ intel_dp_mst_encoder_init(struct intel_digital_port *dig_port, int conn_base_id)
 	/* create encoders */
 	intel_dp_create_fake_mst_encoders(dig_port);
 	ret = drm_dp_mst_topology_mgr_init(&intel_dp->mst_mgr, &i915->drm,
-					   &intel_dp->aux, 16, 3,
-					   dig_port->max_lanes,
-					   max_source_rate,
-					   conn_base_id);
+					   &intel_dp->aux, 16, 3, conn_base_id);
 	if (ret) {
 		intel_dp->mst_mgr.cbs = NULL;
 		return ret;
diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.c b/drivers/gpu/drm/i915/display/intel_dpio_phy.c
index cc6abe761f5e..8732b8722ed7 100644
--- a/drivers/gpu/drm/i915/display/intel_dpio_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.c
@@ -484,7 +484,7 @@ void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 	enum dpio_phy rcomp_phy = phy_info->rcomp_phy;
 	bool was_enabled;
 
-	lockdep_assert_held(&dev_priv->power_domains.lock);
+	lockdep_assert_held(&dev_priv->display.power.domains.lock);
 
 	was_enabled = true;
 	if (rcomp_phy != -1)
diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c
index 5262f16b45ac..b15ba78d64d6 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll.c
@@ -938,12 +938,25 @@ static int hsw_crtc_compute_clock(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_encoder *encoder =
 		intel_get_crtc_new_encoder(state, crtc_state);
+	int ret;
 
 	if (DISPLAY_VER(dev_priv) < 11 &&
 	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI))
 		return 0;
 
-	return intel_compute_shared_dplls(state, crtc, encoder);
+	ret = intel_compute_shared_dplls(state, crtc, encoder);
+	if (ret)
+		return ret;
+
+	/* FIXME this is a mess */
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI))
+		return 0;
+
+	/* CRT dotclock is determined via other means */
+	if (!crtc_state->has_pch_encoder)
+		crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
+
+	return 0;
 }
 
 static int hsw_crtc_get_shared_dpll(struct intel_atomic_state *state,
@@ -969,8 +982,15 @@ static int dg2_crtc_compute_clock(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_encoder *encoder =
 		intel_get_crtc_new_encoder(state, crtc_state);
+	int ret;
+
+	ret = intel_mpllb_calc_state(crtc_state, encoder);
+	if (ret)
+		return ret;
+
+	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
 
-	return intel_mpllb_calc_state(crtc_state, encoder);
+	return 0;
 }
 
 static bool ilk_needs_fb_cb_tune(const struct dpll *dpll, int factor)
@@ -991,7 +1011,7 @@ static void ilk_update_pll_dividers(struct intel_crtc_state *crtc_state,
 	factor = 21;
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 		if ((intel_panel_use_ssc(dev_priv) &&
-		     dev_priv->vbt.lvds_ssc_freq == 100000) ||
+		     dev_priv->display.vbt.lvds_ssc_freq == 100000) ||
 		    (HAS_PCH_IBX(dev_priv) &&
 		     intel_is_dual_link_lvds(dev_priv)))
 			factor = 25;
@@ -1096,6 +1116,7 @@ static int ilk_crtc_compute_clock(struct intel_atomic_state *state,
 		intel_atomic_get_new_crtc_state(state, crtc);
 	const struct intel_limit *limit;
 	int refclk = 120000;
+	int ret;
 
 	/* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
 	if (!crtc_state->has_pch_encoder)
@@ -1105,8 +1126,8 @@ static int ilk_crtc_compute_clock(struct intel_atomic_state *state,
 		if (intel_panel_use_ssc(dev_priv)) {
 			drm_dbg_kms(&dev_priv->drm,
 				    "using SSC reference clock of %d kHz\n",
-				    dev_priv->vbt.lvds_ssc_freq);
-			refclk = dev_priv->vbt.lvds_ssc_freq;
+				    dev_priv->display.vbt.lvds_ssc_freq);
+			refclk = dev_priv->display.vbt.lvds_ssc_freq;
 		}
 
 		if (intel_is_dual_link_lvds(dev_priv)) {
@@ -1132,7 +1153,14 @@ static int ilk_crtc_compute_clock(struct intel_atomic_state *state,
 	ilk_compute_dpll(crtc_state, &crtc_state->dpll,
 			 &crtc_state->dpll);
 
-	return intel_compute_shared_dplls(state, crtc, NULL);
+	ret = intel_compute_shared_dplls(state, crtc, NULL);
+	if (ret)
+		return ret;
+
+	crtc_state->port_clock = crtc_state->dpll.dot;
+	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
+
+	return ret;
 }
 
 static int ilk_crtc_get_shared_dpll(struct intel_atomic_state *state,
@@ -1198,6 +1226,13 @@ static int chv_crtc_compute_clock(struct intel_atomic_state *state,
 
 	chv_compute_dpll(crtc_state);
 
+	/* FIXME this is a mess */
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI))
+		return 0;
+
+	crtc_state->port_clock = crtc_state->dpll.dot;
+	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
+
 	return 0;
 }
 
@@ -1217,6 +1252,13 @@ static int vlv_crtc_compute_clock(struct intel_atomic_state *state,
 
 	vlv_compute_dpll(crtc_state);
 
+	/* FIXME this is a mess */
+	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI))
+		return 0;
+
+	crtc_state->port_clock = crtc_state->dpll.dot;
+	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
+
 	return 0;
 }
 
@@ -1231,7 +1273,7 @@ static int g4x_crtc_compute_clock(struct intel_atomic_state *state,
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 		if (intel_panel_use_ssc(dev_priv)) {
-			refclk = dev_priv->vbt.lvds_ssc_freq;
+			refclk = dev_priv->display.vbt.lvds_ssc_freq;
 			drm_dbg_kms(&dev_priv->drm,
 				    "using SSC reference clock of %d kHz\n",
 				    refclk);
@@ -1259,6 +1301,11 @@ static int g4x_crtc_compute_clock(struct intel_atomic_state *state,
 	i9xx_compute_dpll(crtc_state, &crtc_state->dpll,
 			  &crtc_state->dpll);
 
+	crtc_state->port_clock = crtc_state->dpll.dot;
+	/* FIXME this is a mess */
+	if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_TVOUT))
+		crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
+
 	return 0;
 }
 
@@ -1273,7 +1320,7 @@ static int pnv_crtc_compute_clock(struct intel_atomic_state *state,
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 		if (intel_panel_use_ssc(dev_priv)) {
-			refclk = dev_priv->vbt.lvds_ssc_freq;
+			refclk = dev_priv->display.vbt.lvds_ssc_freq;
 			drm_dbg_kms(&dev_priv->drm,
 				    "using SSC reference clock of %d kHz\n",
 				    refclk);
@@ -1292,6 +1339,9 @@ static int pnv_crtc_compute_clock(struct intel_atomic_state *state,
 	i9xx_compute_dpll(crtc_state, &crtc_state->dpll,
 			  &crtc_state->dpll);
 
+	crtc_state->port_clock = crtc_state->dpll.dot;
+	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
+
 	return 0;
 }
 
@@ -1306,7 +1356,7 @@ static int i9xx_crtc_compute_clock(struct intel_atomic_state *state,
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 		if (intel_panel_use_ssc(dev_priv)) {
-			refclk = dev_priv->vbt.lvds_ssc_freq;
+			refclk = dev_priv->display.vbt.lvds_ssc_freq;
 			drm_dbg_kms(&dev_priv->drm,
 				    "using SSC reference clock of %d kHz\n",
 				    refclk);
@@ -1325,6 +1375,11 @@ static int i9xx_crtc_compute_clock(struct intel_atomic_state *state,
 	i9xx_compute_dpll(crtc_state, &crtc_state->dpll,
 			  &crtc_state->dpll);
 
+	crtc_state->port_clock = crtc_state->dpll.dot;
+	/* FIXME this is a mess */
+	if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_TVOUT))
+		crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
+
 	return 0;
 }
 
@@ -1339,7 +1394,7 @@ static int i8xx_crtc_compute_clock(struct intel_atomic_state *state,
 
 	if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 		if (intel_panel_use_ssc(dev_priv)) {
-			refclk = dev_priv->vbt.lvds_ssc_freq;
+			refclk = dev_priv->display.vbt.lvds_ssc_freq;
 			drm_dbg_kms(&dev_priv->drm,
 				    "using SSC reference clock of %d kHz\n",
 				    refclk);
@@ -1360,6 +1415,9 @@ static int i8xx_crtc_compute_clock(struct intel_atomic_state *state,
 	i8xx_compute_dpll(crtc_state, &crtc_state->dpll,
 			  &crtc_state->dpll);
 
+	crtc_state->port_clock = crtc_state->dpll.dot;
+	crtc_state->hw.adjusted_mode.crtc_clock = intel_crtc_dotclock(crtc_state);
+
 	return 0;
 }
 
@@ -1411,16 +1469,13 @@ int intel_dpll_crtc_compute_clock(struct intel_atomic_state *state,
 
 	drm_WARN_ON(&i915->drm, !intel_crtc_needs_modeset(crtc_state));
 
-	if (drm_WARN_ON(&i915->drm, crtc_state->shared_dpll))
-		return 0;
-
 	memset(&crtc_state->dpll_hw_state, 0,
 	       sizeof(crtc_state->dpll_hw_state));
 
 	if (!crtc_state->hw.enable)
 		return 0;
 
-	ret = i915->dpll_funcs->crtc_compute_clock(state, crtc);
+	ret = i915->display.funcs.dpll->crtc_compute_clock(state, crtc);
 	if (ret) {
 		drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] Couldn't calculate DPLL settings\n",
 			    crtc->base.base.id, crtc->base.name);
@@ -1439,17 +1494,15 @@ int intel_dpll_crtc_get_shared_dpll(struct intel_atomic_state *state,
 	int ret;
 
 	drm_WARN_ON(&i915->drm, !intel_crtc_needs_modeset(crtc_state));
+	drm_WARN_ON(&i915->drm, !crtc_state->hw.enable && crtc_state->shared_dpll);
 
-	if (drm_WARN_ON(&i915->drm, crtc_state->shared_dpll))
-		return 0;
-
-	if (!crtc_state->hw.enable)
+	if (!crtc_state->hw.enable || crtc_state->shared_dpll)
 		return 0;
 
-	if (!i915->dpll_funcs->crtc_get_shared_dpll)
+	if (!i915->display.funcs.dpll->crtc_get_shared_dpll)
 		return 0;
 
-	ret = i915->dpll_funcs->crtc_get_shared_dpll(state, crtc);
+	ret = i915->display.funcs.dpll->crtc_get_shared_dpll(state, crtc);
 	if (ret) {
 		drm_dbg_kms(&i915->drm, "[CRTC:%d:%s] Couldn't get a shared DPLL\n",
 			    crtc->base.base.id, crtc->base.name);
@@ -1463,23 +1516,23 @@ void
 intel_dpll_init_clock_hook(struct drm_i915_private *dev_priv)
 {
 	if (IS_DG2(dev_priv))
-		dev_priv->dpll_funcs = &dg2_dpll_funcs;
+		dev_priv->display.funcs.dpll = &dg2_dpll_funcs;
 	else if (DISPLAY_VER(dev_priv) >= 9 || HAS_DDI(dev_priv))
-		dev_priv->dpll_funcs = &hsw_dpll_funcs;
+		dev_priv->display.funcs.dpll = &hsw_dpll_funcs;
 	else if (HAS_PCH_SPLIT(dev_priv))
-		dev_priv->dpll_funcs = &ilk_dpll_funcs;
+		dev_priv->display.funcs.dpll = &ilk_dpll_funcs;
 	else if (IS_CHERRYVIEW(dev_priv))
-		dev_priv->dpll_funcs = &chv_dpll_funcs;
+		dev_priv->display.funcs.dpll = &chv_dpll_funcs;
 	else if (IS_VALLEYVIEW(dev_priv))
-		dev_priv->dpll_funcs = &vlv_dpll_funcs;
+		dev_priv->display.funcs.dpll = &vlv_dpll_funcs;
 	else if (IS_G4X(dev_priv))
-		dev_priv->dpll_funcs = &g4x_dpll_funcs;
+		dev_priv->display.funcs.dpll = &g4x_dpll_funcs;
 	else if (IS_PINEVIEW(dev_priv))
-		dev_priv->dpll_funcs = &pnv_dpll_funcs;
+		dev_priv->display.funcs.dpll = &pnv_dpll_funcs;
 	else if (DISPLAY_VER(dev_priv) != 2)
-		dev_priv->dpll_funcs = &i9xx_dpll_funcs;
+		dev_priv->display.funcs.dpll = &i9xx_dpll_funcs;
 	else
-		dev_priv->dpll_funcs = &i8xx_dpll_funcs;
+		dev_priv->display.funcs.dpll = &i8xx_dpll_funcs;
 }
 
 static bool i9xx_has_pps(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
index 118598c9a809..e5fb66a5dd02 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
@@ -113,8 +113,8 @@ intel_atomic_duplicate_dpll_state(struct drm_i915_private *dev_priv,
 	enum intel_dpll_id i;
 
 	/* Copy shared dpll state */
-	for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) {
-		struct intel_shared_dpll *pll = &dev_priv->dpll.shared_dplls[i];
+	for (i = 0; i < dev_priv->display.dpll.num_shared_dpll; i++) {
+		struct intel_shared_dpll *pll = &dev_priv->display.dpll.shared_dplls[i];
 
 		shared_dpll[i] = pll->state;
 	}
@@ -149,7 +149,7 @@ struct intel_shared_dpll *
 intel_get_shared_dpll_by_id(struct drm_i915_private *dev_priv,
 			    enum intel_dpll_id id)
 {
-	return &dev_priv->dpll.shared_dplls[id];
+	return &dev_priv->display.dpll.shared_dplls[id];
 }
 
 /**
@@ -164,11 +164,11 @@ enum intel_dpll_id
 intel_get_shared_dpll_id(struct drm_i915_private *dev_priv,
 			 struct intel_shared_dpll *pll)
 {
-	long pll_idx = pll - dev_priv->dpll.shared_dplls;
+	long pll_idx = pll - dev_priv->display.dpll.shared_dplls;
 
 	if (drm_WARN_ON(&dev_priv->drm,
 			pll_idx < 0 ||
-			pll_idx >= dev_priv->dpll.num_shared_dpll))
+			pll_idx >= dev_priv->display.dpll.num_shared_dpll))
 		return -1;
 
 	return pll_idx;
@@ -245,7 +245,7 @@ void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state)
 	if (drm_WARN_ON(&dev_priv->drm, pll == NULL))
 		return;
 
-	mutex_lock(&dev_priv->dpll.lock);
+	mutex_lock(&dev_priv->display.dpll.lock);
 	old_mask = pll->active_mask;
 
 	if (drm_WARN_ON(&dev_priv->drm, !(pll->state.pipe_mask & pipe_mask)) ||
@@ -271,7 +271,7 @@ void intel_enable_shared_dpll(const struct intel_crtc_state *crtc_state)
 	pll->on = true;
 
 out:
-	mutex_unlock(&dev_priv->dpll.lock);
+	mutex_unlock(&dev_priv->display.dpll.lock);
 }
 
 /**
@@ -294,7 +294,7 @@ void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state)
 	if (pll == NULL)
 		return;
 
-	mutex_lock(&dev_priv->dpll.lock);
+	mutex_lock(&dev_priv->display.dpll.lock);
 	if (drm_WARN(&dev_priv->drm, !(pll->active_mask & pipe_mask),
 		     "%s not used by [CRTC:%d:%s]\n", pll->info->name,
 		     crtc->base.base.id, crtc->base.name))
@@ -317,7 +317,7 @@ void intel_disable_shared_dpll(const struct intel_crtc_state *crtc_state)
 	pll->on = false;
 
 out:
-	mutex_unlock(&dev_priv->dpll.lock);
+	mutex_unlock(&dev_priv->display.dpll.lock);
 }
 
 static struct intel_shared_dpll *
@@ -336,7 +336,7 @@ intel_find_shared_dpll(struct intel_atomic_state *state,
 	drm_WARN_ON(&dev_priv->drm, dpll_mask & ~(BIT(I915_NUM_PLLS) - 1));
 
 	for_each_set_bit(i, &dpll_mask, I915_NUM_PLLS) {
-		pll = &dev_priv->dpll.shared_dplls[i];
+		pll = &dev_priv->display.dpll.shared_dplls[i];
 
 		/* Only want to check enabled timings first */
 		if (shared_dpll[i].pipe_mask == 0) {
@@ -436,9 +436,9 @@ void intel_shared_dpll_swap_state(struct intel_atomic_state *state)
 	if (!state->dpll_set)
 		return;
 
-	for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) {
+	for (i = 0; i < dev_priv->display.dpll.num_shared_dpll; i++) {
 		struct intel_shared_dpll *pll =
-			&dev_priv->dpll.shared_dplls[i];
+			&dev_priv->display.dpll.shared_dplls[i];
 
 		swap(pll->state, shared_dpll[i]);
 	}
@@ -537,7 +537,7 @@ static int ibx_get_dpll(struct intel_atomic_state *state,
 	if (HAS_PCH_IBX(dev_priv)) {
 		/* Ironlake PCH has a fixed PLL->PCH pipe mapping. */
 		i = (enum intel_dpll_id) crtc->pipe;
-		pll = &dev_priv->dpll.shared_dplls[i];
+		pll = &dev_priv->display.dpll.shared_dplls[i];
 
 		drm_dbg_kms(&dev_priv->drm,
 			    "[CRTC:%d:%s] using pre-allocated %s\n",
@@ -905,37 +905,6 @@ hsw_ddi_calculate_wrpll(int clock /* in Hz */,
 	*r2_out = best.r2;
 }
 
-static int
-hsw_ddi_wrpll_compute_dpll(struct intel_atomic_state *state,
-			   struct intel_crtc *crtc)
-{
-	struct intel_crtc_state *crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-	unsigned int p, n2, r2;
-
-	hsw_ddi_calculate_wrpll(crtc_state->port_clock * 1000, &r2, &n2, &p);
-
-	crtc_state->dpll_hw_state.wrpll =
-		WRPLL_PLL_ENABLE | WRPLL_REF_LCPLL |
-		WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) |
-		WRPLL_DIVIDER_POST(p);
-
-	return 0;
-}
-
-static struct intel_shared_dpll *
-hsw_ddi_wrpll_get_dpll(struct intel_atomic_state *state,
-		       struct intel_crtc *crtc)
-{
-	struct intel_crtc_state *crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-
-	return intel_find_shared_dpll(state, crtc,
-				      &crtc_state->dpll_hw_state,
-				      BIT(DPLL_ID_WRPLL2) |
-				      BIT(DPLL_ID_WRPLL1));
-}
-
 static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv,
 				  const struct intel_shared_dpll *pll,
 				  const struct intel_dpll_hw_state *pll_state)
@@ -948,7 +917,7 @@ static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv,
 	case WRPLL_REF_SPECIAL_HSW:
 		/* Muxed-SSC for BDW, non-SSC for non-ULT HSW. */
 		if (IS_HASWELL(dev_priv) && !IS_HSW_ULT(dev_priv)) {
-			refclk = dev_priv->dpll.ref_clks.nssc;
+			refclk = dev_priv->display.dpll.ref_clks.nssc;
 			break;
 		}
 		fallthrough;
@@ -958,7 +927,7 @@ static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv,
 		 * code only cares about 5% accuracy, and spread is a max of
 		 * 0.5% downspread.
 		 */
-		refclk = dev_priv->dpll.ref_clks.ssc;
+		refclk = dev_priv->display.dpll.ref_clks.ssc;
 		break;
 	case WRPLL_REF_LCPLL:
 		refclk = 2700000;
@@ -977,6 +946,41 @@ static int hsw_ddi_wrpll_get_freq(struct drm_i915_private *dev_priv,
 }
 
 static int
+hsw_ddi_wrpll_compute_dpll(struct intel_atomic_state *state,
+			   struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	struct intel_crtc_state *crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	unsigned int p, n2, r2;
+
+	hsw_ddi_calculate_wrpll(crtc_state->port_clock * 1000, &r2, &n2, &p);
+
+	crtc_state->dpll_hw_state.wrpll =
+		WRPLL_PLL_ENABLE | WRPLL_REF_LCPLL |
+		WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) |
+		WRPLL_DIVIDER_POST(p);
+
+	crtc_state->port_clock = hsw_ddi_wrpll_get_freq(i915, NULL,
+							&crtc_state->dpll_hw_state);
+
+	return 0;
+}
+
+static struct intel_shared_dpll *
+hsw_ddi_wrpll_get_dpll(struct intel_atomic_state *state,
+		       struct intel_crtc *crtc)
+{
+	struct intel_crtc_state *crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+
+	return intel_find_shared_dpll(state, crtc,
+				      &crtc_state->dpll_hw_state,
+				      BIT(DPLL_ID_WRPLL2) |
+				      BIT(DPLL_ID_WRPLL1));
+}
+
+static int
 hsw_ddi_lcpll_compute_dpll(struct intel_crtc_state *crtc_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
@@ -1145,12 +1149,12 @@ static int hsw_get_dpll(struct intel_atomic_state *state,
 
 static void hsw_update_dpll_ref_clks(struct drm_i915_private *i915)
 {
-	i915->dpll.ref_clks.ssc = 135000;
+	i915->display.dpll.ref_clks.ssc = 135000;
 	/* Non-SSC is only used on non-ULT HSW. */
 	if (intel_de_read(i915, FUSE_STRAP3) & HSW_REF_CLK_SELECT)
-		i915->dpll.ref_clks.nssc = 24000;
+		i915->display.dpll.ref_clks.nssc = 24000;
 	else
-		i915->dpll.ref_clks.nssc = 135000;
+		i915->display.dpll.ref_clks.nssc = 135000;
 }
 
 static void hsw_dump_hw_state(struct drm_i915_private *dev_priv,
@@ -1618,48 +1622,11 @@ skip_remaining_dividers:
 	return 0;
 }
 
-static int skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state)
-{
-	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
-	struct skl_wrpll_params wrpll_params = {};
-	u32 ctrl1, cfgcr1, cfgcr2;
-	int ret;
-
-	/*
-	 * See comment in intel_dpll_hw_state to understand why we always use 0
-	 * as the DPLL id in this function.
-	 */
-	ctrl1 = DPLL_CTRL1_OVERRIDE(0);
-
-	ctrl1 |= DPLL_CTRL1_HDMI_MODE(0);
-
-	ret = skl_ddi_calculate_wrpll(crtc_state->port_clock * 1000,
-				      i915->dpll.ref_clks.nssc, &wrpll_params);
-	if (ret)
-		return ret;
-
-	cfgcr1 = DPLL_CFGCR1_FREQ_ENABLE |
-		DPLL_CFGCR1_DCO_FRACTION(wrpll_params.dco_fraction) |
-		wrpll_params.dco_integer;
-
-	cfgcr2 = DPLL_CFGCR2_QDIV_RATIO(wrpll_params.qdiv_ratio) |
-		DPLL_CFGCR2_QDIV_MODE(wrpll_params.qdiv_mode) |
-		DPLL_CFGCR2_KDIV(wrpll_params.kdiv) |
-		DPLL_CFGCR2_PDIV(wrpll_params.pdiv) |
-		wrpll_params.central_freq;
-
-	crtc_state->dpll_hw_state.ctrl1 = ctrl1;
-	crtc_state->dpll_hw_state.cfgcr1 = cfgcr1;
-	crtc_state->dpll_hw_state.cfgcr2 = cfgcr2;
-
-	return 0;
-}
-
 static int skl_ddi_wrpll_get_freq(struct drm_i915_private *i915,
 				  const struct intel_shared_dpll *pll,
 				  const struct intel_dpll_hw_state *pll_state)
 {
-	int ref_clock = i915->dpll.ref_clks.nssc;
+	int ref_clock = i915->display.dpll.ref_clks.nssc;
 	u32 p0, p1, p2, dco_freq;
 
 	p0 = pll_state->cfgcr2 & DPLL_CFGCR2_PDIV_MASK;
@@ -1726,6 +1693,46 @@ static int skl_ddi_wrpll_get_freq(struct drm_i915_private *i915,
 	return dco_freq / (p0 * p1 * p2 * 5);
 }
 
+static int skl_ddi_hdmi_pll_dividers(struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	struct skl_wrpll_params wrpll_params = {};
+	u32 ctrl1, cfgcr1, cfgcr2;
+	int ret;
+
+	/*
+	 * See comment in intel_dpll_hw_state to understand why we always use 0
+	 * as the DPLL id in this function.
+	 */
+	ctrl1 = DPLL_CTRL1_OVERRIDE(0);
+
+	ctrl1 |= DPLL_CTRL1_HDMI_MODE(0);
+
+	ret = skl_ddi_calculate_wrpll(crtc_state->port_clock * 1000,
+				      i915->display.dpll.ref_clks.nssc, &wrpll_params);
+	if (ret)
+		return ret;
+
+	cfgcr1 = DPLL_CFGCR1_FREQ_ENABLE |
+		DPLL_CFGCR1_DCO_FRACTION(wrpll_params.dco_fraction) |
+		wrpll_params.dco_integer;
+
+	cfgcr2 = DPLL_CFGCR2_QDIV_RATIO(wrpll_params.qdiv_ratio) |
+		DPLL_CFGCR2_QDIV_MODE(wrpll_params.qdiv_mode) |
+		DPLL_CFGCR2_KDIV(wrpll_params.kdiv) |
+		DPLL_CFGCR2_PDIV(wrpll_params.pdiv) |
+		wrpll_params.central_freq;
+
+	crtc_state->dpll_hw_state.ctrl1 = ctrl1;
+	crtc_state->dpll_hw_state.cfgcr1 = cfgcr1;
+	crtc_state->dpll_hw_state.cfgcr2 = cfgcr2;
+
+	crtc_state->port_clock = skl_ddi_wrpll_get_freq(i915, NULL,
+							&crtc_state->dpll_hw_state);
+
+	return 0;
+}
+
 static int
 skl_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state)
 {
@@ -1858,7 +1865,7 @@ static int skl_ddi_pll_get_freq(struct drm_i915_private *i915,
 static void skl_update_dpll_ref_clks(struct drm_i915_private *i915)
 {
 	/* No SSC ref */
-	i915->dpll.ref_clks.nssc = i915->cdclk.hw.ref;
+	i915->display.dpll.ref_clks.nssc = i915->display.cdclk.hw.ref;
 }
 
 static void skl_dump_hw_state(struct drm_i915_private *dev_priv,
@@ -2171,7 +2178,7 @@ static void bxt_ddi_dp_pll_dividers(struct intel_crtc_state *crtc_state,
 		}
 	}
 
-	chv_calc_dpll_params(i915->dpll.ref_clks.nssc, clk_div);
+	chv_calc_dpll_params(i915->display.dpll.ref_clks.nssc, clk_div);
 
 	drm_WARN_ON(&i915->drm, clk_div->vco == 0 ||
 		    clk_div->dot != crtc_state->port_clock);
@@ -2245,6 +2252,23 @@ static int bxt_ddi_set_dpll_hw_state(struct intel_crtc_state *crtc_state,
 	return 0;
 }
 
+static int bxt_ddi_pll_get_freq(struct drm_i915_private *i915,
+				const struct intel_shared_dpll *pll,
+				const struct intel_dpll_hw_state *pll_state)
+{
+	struct dpll clock;
+
+	clock.m1 = 2;
+	clock.m2 = REG_FIELD_GET(PORT_PLL_M2_INT_MASK, pll_state->pll0) << 22;
+	if (pll_state->pll3 & PORT_PLL_M2_FRAC_ENABLE)
+		clock.m2 |= REG_FIELD_GET(PORT_PLL_M2_FRAC_MASK, pll_state->pll2);
+	clock.n = REG_FIELD_GET(PORT_PLL_N_MASK, pll_state->pll1);
+	clock.p1 = REG_FIELD_GET(PORT_PLL_P1_MASK, pll_state->ebb0);
+	clock.p2 = REG_FIELD_GET(PORT_PLL_P2_MASK, pll_state->ebb0);
+
+	return chv_calc_dpll_params(i915->display.dpll.ref_clks.nssc, &clock);
+}
+
 static int
 bxt_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state)
 {
@@ -2258,28 +2282,20 @@ bxt_ddi_dp_set_dpll_hw_state(struct intel_crtc_state *crtc_state)
 static int
 bxt_ddi_hdmi_set_dpll_hw_state(struct intel_crtc_state *crtc_state)
 {
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
 	struct dpll clk_div = {};
+	int ret;
 
 	bxt_ddi_hdmi_pll_dividers(crtc_state, &clk_div);
 
-	return bxt_ddi_set_dpll_hw_state(crtc_state, &clk_div);
-}
-
-static int bxt_ddi_pll_get_freq(struct drm_i915_private *i915,
-				const struct intel_shared_dpll *pll,
-				const struct intel_dpll_hw_state *pll_state)
-{
-	struct dpll clock;
+	ret = bxt_ddi_set_dpll_hw_state(crtc_state, &clk_div);
+	if (ret)
+		return ret;
 
-	clock.m1 = 2;
-	clock.m2 = REG_FIELD_GET(PORT_PLL_M2_INT_MASK, pll_state->pll0) << 22;
-	if (pll_state->pll3 & PORT_PLL_M2_FRAC_ENABLE)
-		clock.m2 |= REG_FIELD_GET(PORT_PLL_M2_FRAC_MASK, pll_state->pll2);
-	clock.n = REG_FIELD_GET(PORT_PLL_N_MASK, pll_state->pll1);
-	clock.p1 = REG_FIELD_GET(PORT_PLL_P1_MASK, pll_state->ebb0);
-	clock.p2 = REG_FIELD_GET(PORT_PLL_P2_MASK, pll_state->ebb0);
+	crtc_state->port_clock = bxt_ddi_pll_get_freq(i915, NULL,
+						      &crtc_state->dpll_hw_state);
 
-	return chv_calc_dpll_params(i915->dpll.ref_clks.nssc, &clock);
+	return 0;
 }
 
 static int bxt_compute_dpll(struct intel_atomic_state *state,
@@ -2324,8 +2340,8 @@ static int bxt_get_dpll(struct intel_atomic_state *state,
 
 static void bxt_update_dpll_ref_clks(struct drm_i915_private *i915)
 {
-	i915->dpll.ref_clks.ssc = 100000;
-	i915->dpll.ref_clks.nssc = 100000;
+	i915->display.dpll.ref_clks.ssc = 100000;
+	i915->display.dpll.ref_clks.nssc = 100000;
 	/* DSI non-SSC ref 19.2MHz */
 }
 
@@ -2468,7 +2484,7 @@ ehl_combo_pll_div_frac_wa_needed(struct drm_i915_private *i915)
 	return ((IS_PLATFORM(i915, INTEL_ELKHARTLAKE) &&
 		 IS_JSL_EHL_DISPLAY_STEP(i915, STEP_B0, STEP_FOREVER)) ||
 		 IS_TIGERLAKE(i915) || IS_ALDERLAKE_S(i915) || IS_ALDERLAKE_P(i915)) &&
-		 i915->dpll.ref_clks.nssc == 38400;
+		 i915->display.dpll.ref_clks.nssc == 38400;
 }
 
 struct icl_combo_pll_params {
@@ -2562,7 +2578,7 @@ static int icl_calc_dp_combo_pll(struct intel_crtc_state *crtc_state,
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 	const struct icl_combo_pll_params *params =
-		dev_priv->dpll.ref_clks.nssc == 24000 ?
+		dev_priv->display.dpll.ref_clks.nssc == 24000 ?
 		icl_dp_combo_pll_24MHz_values :
 		icl_dp_combo_pll_19_2MHz_values;
 	int clock = crtc_state->port_clock;
@@ -2585,9 +2601,9 @@ static int icl_calc_tbt_pll(struct intel_crtc_state *crtc_state,
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 
 	if (DISPLAY_VER(dev_priv) >= 12) {
-		switch (dev_priv->dpll.ref_clks.nssc) {
+		switch (dev_priv->display.dpll.ref_clks.nssc) {
 		default:
-			MISSING_CASE(dev_priv->dpll.ref_clks.nssc);
+			MISSING_CASE(dev_priv->display.dpll.ref_clks.nssc);
 			fallthrough;
 		case 19200:
 		case 38400:
@@ -2598,9 +2614,9 @@ static int icl_calc_tbt_pll(struct intel_crtc_state *crtc_state,
 			break;
 		}
 	} else {
-		switch (dev_priv->dpll.ref_clks.nssc) {
+		switch (dev_priv->display.dpll.ref_clks.nssc) {
 		default:
-			MISSING_CASE(dev_priv->dpll.ref_clks.nssc);
+			MISSING_CASE(dev_priv->display.dpll.ref_clks.nssc);
 			fallthrough;
 		case 19200:
 		case 38400:
@@ -2630,7 +2646,7 @@ static int icl_ddi_tbt_pll_get_freq(struct drm_i915_private *i915,
 
 static int icl_wrpll_ref_clock(struct drm_i915_private *i915)
 {
-	int ref_clock = i915->dpll.ref_clks.nssc;
+	int ref_clock = i915->display.dpll.ref_clks.nssc;
 
 	/*
 	 * For ICL+, the spec states: if reference frequency is 38.4,
@@ -2769,8 +2785,8 @@ static void icl_calc_dpll_state(struct drm_i915_private *i915,
 	else
 		pll_state->cfgcr1 |= DPLL_CFGCR1_CENTRAL_FREQ_8400;
 
-	if (i915->vbt.override_afc_startup)
-		pll_state->div0 = TGL_DPLL0_DIV0_AFC_STARTUP(i915->vbt.override_afc_startup_val);
+	if (i915->display.vbt.override_afc_startup)
+		pll_state->div0 = TGL_DPLL0_DIV0_AFC_STARTUP(i915->display.vbt.override_afc_startup_val);
 }
 
 static int icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc,
@@ -2857,7 +2873,7 @@ static int icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
 				 struct intel_dpll_hw_state *pll_state)
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
-	int refclk_khz = dev_priv->dpll.ref_clks.nssc;
+	int refclk_khz = dev_priv->display.dpll.ref_clks.nssc;
 	int clock = crtc_state->port_clock;
 	u32 dco_khz, m1div, m2div_int, m2div_rem, m2div_frac;
 	u32 iref_ndiv, iref_trim, iref_pulse_w;
@@ -2965,8 +2981,8 @@ static int icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
 					 DKL_PLL_DIV0_PROP_COEFF(prop_coeff) |
 					 DKL_PLL_DIV0_FBPREDIV(m1div) |
 					 DKL_PLL_DIV0_FBDIV_INT(m2div_int);
-		if (dev_priv->vbt.override_afc_startup) {
-			u8 val = dev_priv->vbt.override_afc_startup_val;
+		if (dev_priv->display.vbt.override_afc_startup) {
+			u8 val = dev_priv->display.vbt.override_afc_startup_val;
 
 			pll_state->mg_pll_div0 |= DKL_PLL_DIV0_AFC_STARTUP(val);
 		}
@@ -3063,7 +3079,7 @@ static int icl_ddi_mg_pll_get_freq(struct drm_i915_private *dev_priv,
 	u32 m1, m2_int, m2_frac, div1, div2, ref_clock;
 	u64 tmp;
 
-	ref_clock = dev_priv->dpll.ref_clks.nssc;
+	ref_clock = dev_priv->display.dpll.ref_clks.nssc;
 
 	if (DISPLAY_VER(dev_priv) >= 12) {
 		m1 = pll_state->mg_pll_div0 & DKL_PLL_DIV0_FBPREDIV_MASK;
@@ -3197,6 +3213,12 @@ static int icl_compute_combo_phy_dpll(struct intel_atomic_state *state,
 
 	icl_calc_dpll_state(dev_priv, &pll_params, &port_dpll->hw_state);
 
+	/* this is mainly for the fastset check */
+	icl_set_active_port_dpll(crtc_state, ICL_PORT_DPLL_DEFAULT);
+
+	crtc_state->port_clock = icl_ddi_combo_pll_get_freq(dev_priv, NULL,
+							    &port_dpll->hw_state);
+
 	return 0;
 }
 
@@ -3282,6 +3304,12 @@ static int icl_compute_tc_phy_dplls(struct intel_atomic_state *state,
 	if (ret)
 		return ret;
 
+	/* this is mainly for the fastset check */
+	icl_set_active_port_dpll(crtc_state, ICL_PORT_DPLL_MG_PHY);
+
+	crtc_state->port_clock = icl_ddi_mg_pll_get_freq(dev_priv, NULL,
+							 &port_dpll->hw_state);
+
 	return 0;
 }
 
@@ -3440,7 +3468,7 @@ static bool mg_pll_get_hw_state(struct drm_i915_private *dev_priv,
 	hw_state->mg_pll_tdc_coldst_bias =
 		intel_de_read(dev_priv, MG_PLL_TDC_COLDST_BIAS(tc_port));
 
-	if (dev_priv->dpll.ref_clks.nssc == 38400) {
+	if (dev_priv->display.dpll.ref_clks.nssc == 38400) {
 		hw_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART;
 		hw_state->mg_pll_bias_mask = 0;
 	} else {
@@ -3502,7 +3530,7 @@ static bool dkl_pll_get_hw_state(struct drm_i915_private *dev_priv,
 
 	hw_state->mg_pll_div0 = intel_de_read(dev_priv, DKL_PLL_DIV0(tc_port));
 	val = DKL_PLL_DIV0_MASK;
-	if (dev_priv->vbt.override_afc_startup)
+	if (dev_priv->display.vbt.override_afc_startup)
 		val |= DKL_PLL_DIV0_AFC_STARTUP_MASK;
 	hw_state->mg_pll_div0 &= val;
 
@@ -3566,7 +3594,7 @@ static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv,
 						 TGL_DPLL_CFGCR0(id));
 		hw_state->cfgcr1 = intel_de_read(dev_priv,
 						 TGL_DPLL_CFGCR1(id));
-		if (dev_priv->vbt.override_afc_startup) {
+		if (dev_priv->display.vbt.override_afc_startup) {
 			hw_state->div0 = intel_de_read(dev_priv, TGL_DPLL0_DIV0(id));
 			hw_state->div0 &= TGL_DPLL0_DIV0_AFC_STARTUP_MASK;
 		}
@@ -3638,9 +3666,9 @@ static void icl_dpll_write(struct drm_i915_private *dev_priv,
 
 	intel_de_write(dev_priv, cfgcr0_reg, hw_state->cfgcr0);
 	intel_de_write(dev_priv, cfgcr1_reg, hw_state->cfgcr1);
-	drm_WARN_ON_ONCE(&dev_priv->drm, dev_priv->vbt.override_afc_startup &&
+	drm_WARN_ON_ONCE(&dev_priv->drm, dev_priv->display.vbt.override_afc_startup &&
 			 !i915_mmio_reg_valid(div0_reg));
-	if (dev_priv->vbt.override_afc_startup &&
+	if (dev_priv->display.vbt.override_afc_startup &&
 	    i915_mmio_reg_valid(div0_reg))
 		intel_de_rmw(dev_priv, div0_reg, TGL_DPLL0_DIV0_AFC_STARTUP_MASK,
 			     hw_state->div0);
@@ -3732,7 +3760,7 @@ static void dkl_pll_write(struct drm_i915_private *dev_priv,
 	intel_de_write(dev_priv, DKL_CLKTOP2_HSCLKCTL(tc_port), val);
 
 	val = DKL_PLL_DIV0_MASK;
-	if (dev_priv->vbt.override_afc_startup)
+	if (dev_priv->display.vbt.override_afc_startup)
 		val |= DKL_PLL_DIV0_AFC_STARTUP_MASK;
 	intel_de_rmw(dev_priv, DKL_PLL_DIV0(tc_port), val,
 		     hw_state->mg_pll_div0);
@@ -3967,7 +3995,7 @@ static void mg_pll_disable(struct drm_i915_private *dev_priv,
 static void icl_update_dpll_ref_clks(struct drm_i915_private *i915)
 {
 	/* No SSC ref */
-	i915->dpll.ref_clks.nssc = i915->cdclk.hw.ref;
+	i915->display.dpll.ref_clks.nssc = i915->display.cdclk.hw.ref;
 }
 
 static void icl_dump_hw_state(struct drm_i915_private *dev_priv,
@@ -4192,22 +4220,24 @@ void intel_shared_dpll_init(struct drm_i915_private *dev_priv)
 		dpll_mgr = &pch_pll_mgr;
 
 	if (!dpll_mgr) {
-		dev_priv->dpll.num_shared_dpll = 0;
+		dev_priv->display.dpll.num_shared_dpll = 0;
 		return;
 	}
 
 	dpll_info = dpll_mgr->dpll_info;
 
 	for (i = 0; dpll_info[i].name; i++) {
+		if (drm_WARN_ON(&dev_priv->drm,
+				i >= ARRAY_SIZE(dev_priv->display.dpll.shared_dplls)))
+			break;
+
 		drm_WARN_ON(&dev_priv->drm, i != dpll_info[i].id);
-		dev_priv->dpll.shared_dplls[i].info = &dpll_info[i];
+		dev_priv->display.dpll.shared_dplls[i].info = &dpll_info[i];
 	}
 
-	dev_priv->dpll.mgr = dpll_mgr;
-	dev_priv->dpll.num_shared_dpll = i;
-	mutex_init(&dev_priv->dpll.lock);
-
-	BUG_ON(dev_priv->dpll.num_shared_dpll > I915_NUM_PLLS);
+	dev_priv->display.dpll.mgr = dpll_mgr;
+	dev_priv->display.dpll.num_shared_dpll = i;
+	mutex_init(&dev_priv->display.dpll.lock);
 }
 
 /**
@@ -4229,7 +4259,7 @@ int intel_compute_shared_dplls(struct intel_atomic_state *state,
 			       struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr;
+	const struct intel_dpll_mgr *dpll_mgr = dev_priv->display.dpll.mgr;
 
 	if (drm_WARN_ON(&dev_priv->drm, !dpll_mgr))
 		return -EINVAL;
@@ -4262,7 +4292,7 @@ int intel_reserve_shared_dplls(struct intel_atomic_state *state,
 			       struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr;
+	const struct intel_dpll_mgr *dpll_mgr = dev_priv->display.dpll.mgr;
 
 	if (drm_WARN_ON(&dev_priv->drm, !dpll_mgr))
 		return -EINVAL;
@@ -4285,7 +4315,7 @@ void intel_release_shared_dplls(struct intel_atomic_state *state,
 				struct intel_crtc *crtc)
 {
 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr;
+	const struct intel_dpll_mgr *dpll_mgr = dev_priv->display.dpll.mgr;
 
 	/*
 	 * FIXME: this function is called for every platform having a
@@ -4314,7 +4344,7 @@ void intel_update_active_dpll(struct intel_atomic_state *state,
 			      struct intel_encoder *encoder)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	const struct intel_dpll_mgr *dpll_mgr = dev_priv->dpll.mgr;
+	const struct intel_dpll_mgr *dpll_mgr = dev_priv->display.dpll.mgr;
 
 	if (drm_WARN_ON(&dev_priv->drm, !dpll_mgr))
 		return;
@@ -4385,16 +4415,16 @@ static void readout_dpll_hw_state(struct drm_i915_private *i915,
 
 void intel_dpll_update_ref_clks(struct drm_i915_private *i915)
 {
-	if (i915->dpll.mgr && i915->dpll.mgr->update_ref_clks)
-		i915->dpll.mgr->update_ref_clks(i915);
+	if (i915->display.dpll.mgr && i915->display.dpll.mgr->update_ref_clks)
+		i915->display.dpll.mgr->update_ref_clks(i915);
 }
 
 void intel_dpll_readout_hw_state(struct drm_i915_private *i915)
 {
 	int i;
 
-	for (i = 0; i < i915->dpll.num_shared_dpll; i++)
-		readout_dpll_hw_state(i915, &i915->dpll.shared_dplls[i]);
+	for (i = 0; i < i915->display.dpll.num_shared_dpll; i++)
+		readout_dpll_hw_state(i915, &i915->display.dpll.shared_dplls[i]);
 }
 
 static void sanitize_dpll_state(struct drm_i915_private *i915,
@@ -4420,8 +4450,8 @@ void intel_dpll_sanitize_state(struct drm_i915_private *i915)
 {
 	int i;
 
-	for (i = 0; i < i915->dpll.num_shared_dpll; i++)
-		sanitize_dpll_state(i915, &i915->dpll.shared_dplls[i]);
+	for (i = 0; i < i915->display.dpll.num_shared_dpll; i++)
+		sanitize_dpll_state(i915, &i915->display.dpll.shared_dplls[i]);
 }
 
 /**
@@ -4434,8 +4464,8 @@ void intel_dpll_sanitize_state(struct drm_i915_private *i915)
 void intel_dpll_dump_hw_state(struct drm_i915_private *dev_priv,
 			      const struct intel_dpll_hw_state *hw_state)
 {
-	if (dev_priv->dpll.mgr) {
-		dev_priv->dpll.mgr->dump_hw_state(dev_priv, hw_state);
+	if (dev_priv->display.dpll.mgr) {
+		dev_priv->display.dpll.mgr->dump_hw_state(dev_priv, hw_state);
 	} else {
 		/* fallback for platforms that don't use the shared dpll
 		 * infrastructure
@@ -4533,7 +4563,7 @@ void intel_shared_dpll_verify_disabled(struct drm_i915_private *i915)
 {
 	int i;
 
-	for (i = 0; i < i915->dpll.num_shared_dpll; i++)
-		verify_single_dpll_state(i915, &i915->dpll.shared_dplls[i],
+	for (i = 0; i < i915->display.dpll.num_shared_dpll; i++)
+		verify_single_dpll_state(i915, &i915->display.dpll.shared_dplls[i],
 					 NULL, NULL);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c
index c4affcb216fd..fc9c3e41c333 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb.c
@@ -9,6 +9,36 @@
 #include "i915_drv.h"
 #include "intel_de.h"
 #include "intel_display_types.h"
+#include "intel_dsb.h"
+
+struct i915_vma;
+
+enum dsb_id {
+	INVALID_DSB = -1,
+	DSB1,
+	DSB2,
+	DSB3,
+	MAX_DSB_PER_PIPE
+};
+
+struct intel_dsb {
+	enum dsb_id id;
+	u32 *cmd_buf;
+	struct i915_vma *vma;
+
+	/*
+	 * free_pos will point the first free entry position
+	 * and help in calculating tail of command buffer.
+	 */
+	int free_pos;
+
+	/*
+	 * ins_start_offset will help to store start address of the dsb
+	 * instuction and help in identifying the batch of auto-increment
+	 * register.
+	 */
+	u32 ins_start_offset;
+};
 
 #define DSB_BUF_SIZE    (2 * PAGE_SIZE)
 
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.h b/drivers/gpu/drm/i915/display/intel_dsb.h
index 6cb9c580cdca..74dd2b3343bb 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.h
+++ b/drivers/gpu/drm/i915/display/intel_dsb.h
@@ -11,34 +11,6 @@
 #include "i915_reg_defs.h"
 
 struct intel_crtc_state;
-struct i915_vma;
-
-enum dsb_id {
-	INVALID_DSB = -1,
-	DSB1,
-	DSB2,
-	DSB3,
-	MAX_DSB_PER_PIPE
-};
-
-struct intel_dsb {
-	enum dsb_id id;
-	u32 *cmd_buf;
-	struct i915_vma *vma;
-
-	/*
-	 * free_pos will point the first free entry position
-	 * and help in calculating tail of command buffer.
-	 */
-	int free_pos;
-
-	/*
-	 * ins_start_offset will help to store start address of the dsb
-	 * instuction and help in identifying the batch of auto-increment
-	 * register.
-	 */
-	u32 ins_start_offset;
-};
 
 void intel_dsb_prepare(struct intel_crtc_state *crtc_state);
 void intel_dsb_cleanup(struct intel_crtc_state *crtc_state);
diff --git a/drivers/gpu/drm/i915/display/intel_dsi.c b/drivers/gpu/drm/i915/display/intel_dsi.c
index 35e121cd226c..5efdd471ac2b 100644
--- a/drivers/gpu/drm/i915/display/intel_dsi.c
+++ b/drivers/gpu/drm/i915/display/intel_dsi.c
@@ -106,7 +106,7 @@ intel_dsi_get_panel_orientation(struct intel_connector *connector)
 	if (orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN)
 		return orientation;
 
-	orientation = dev_priv->vbt.orientation;
+	orientation = dev_priv->display.vbt.orientation;
 	if (orientation != DRM_MODE_PANEL_ORIENTATION_UNKNOWN)
 		return orientation;
 
diff --git a/drivers/gpu/drm/i915/display/intel_dsi.h b/drivers/gpu/drm/i915/display/intel_dsi.h
index eafef0a87fea..ce80bd8be519 100644
--- a/drivers/gpu/drm/i915/display/intel_dsi.h
+++ b/drivers/gpu/drm/i915/display/intel_dsi.h
@@ -89,9 +89,6 @@ struct intel_dsi {
 	u8 escape_clk_div;
 	u8 dual_link;
 
-	u16 dcs_backlight_ports;
-	u16 dcs_cabc_ports;
-
 	/* RGB or BGR */
 	bool bgr_enabled;
 
diff --git a/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c b/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c
index 1bc7118c56a2..20e466d843ce 100644
--- a/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_dsi_dcs_backlight.c
@@ -53,7 +53,7 @@ static u32 dcs_get_backlight(struct intel_connector *connector, enum pipe unused
 	enum port port;
 	size_t len = panel->backlight.max > U8_MAX ? 2 : 1;
 
-	for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) {
+	for_each_dsi_port(port, panel->vbt.dsi.bl_ports) {
 		dsi_device = intel_dsi->dsi_hosts[port]->device;
 		mipi_dsi_dcs_read(dsi_device, MIPI_DCS_GET_DISPLAY_BRIGHTNESS,
 				  &data, len);
@@ -80,7 +80,7 @@ static void dcs_set_backlight(const struct drm_connector_state *conn_state, u32
 		data[1] = level;
 	}
 
-	for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) {
+	for_each_dsi_port(port, panel->vbt.dsi.bl_ports) {
 		dsi_device = intel_dsi->dsi_hosts[port]->device;
 		mode_flags = dsi_device->mode_flags;
 		dsi_device->mode_flags &= ~MIPI_DSI_MODE_LPM;
@@ -93,12 +93,13 @@ static void dcs_set_backlight(const struct drm_connector_state *conn_state, u32
 static void dcs_disable_backlight(const struct drm_connector_state *conn_state, u32 level)
 {
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(to_intel_encoder(conn_state->best_encoder));
+	struct intel_panel *panel = &to_intel_connector(conn_state->connector)->panel;
 	struct mipi_dsi_device *dsi_device;
 	enum port port;
 
 	dcs_set_backlight(conn_state, 0);
 
-	for_each_dsi_port(port, intel_dsi->dcs_cabc_ports) {
+	for_each_dsi_port(port, panel->vbt.dsi.cabc_ports) {
 		u8 cabc = POWER_SAVE_OFF;
 
 		dsi_device = intel_dsi->dsi_hosts[port]->device;
@@ -106,7 +107,7 @@ static void dcs_disable_backlight(const struct drm_connector_state *conn_state,
 				   &cabc, sizeof(cabc));
 	}
 
-	for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) {
+	for_each_dsi_port(port, panel->vbt.dsi.bl_ports) {
 		u8 ctrl = 0;
 
 		dsi_device = intel_dsi->dsi_hosts[port]->device;
@@ -127,10 +128,11 @@ static void dcs_enable_backlight(const struct intel_crtc_state *crtc_state,
 				 const struct drm_connector_state *conn_state, u32 level)
 {
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(to_intel_encoder(conn_state->best_encoder));
+	struct intel_panel *panel = &to_intel_connector(conn_state->connector)->panel;
 	struct mipi_dsi_device *dsi_device;
 	enum port port;
 
-	for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) {
+	for_each_dsi_port(port, panel->vbt.dsi.bl_ports) {
 		u8 ctrl = 0;
 
 		dsi_device = intel_dsi->dsi_hosts[port]->device;
@@ -146,7 +148,7 @@ static void dcs_enable_backlight(const struct intel_crtc_state *crtc_state,
 				   &ctrl, sizeof(ctrl));
 	}
 
-	for_each_dsi_port(port, intel_dsi->dcs_cabc_ports) {
+	for_each_dsi_port(port, panel->vbt.dsi.cabc_ports) {
 		u8 cabc = POWER_SAVE_MEDIUM;
 
 		dsi_device = intel_dsi->dsi_hosts[port]->device;
diff --git a/drivers/gpu/drm/i915/display/intel_dvo_dev.h b/drivers/gpu/drm/i915/display/intel_dvo_dev.h
index d96c3cc46e50..50205f064d93 100644
--- a/drivers/gpu/drm/i915/display/intel_dvo_dev.h
+++ b/drivers/gpu/drm/i915/display/intel_dvo_dev.h
@@ -75,8 +75,8 @@ struct intel_dvo_dev_ops {
 	 *
 	 * \return MODE_OK if the mode is valid, or another MODE_* otherwise.
 	 */
-	int (*mode_valid)(struct intel_dvo_device *dvo,
-			  struct drm_display_mode *mode);
+	enum drm_mode_status (*mode_valid)(struct intel_dvo_device *dvo,
+					   struct drm_display_mode *mode);
 
 	/*
 	 * Callback for preparing mode changes on an output
diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c
index b191915ab351..eefa33c555ac 100644
--- a/drivers/gpu/drm/i915/display/intel_fb.c
+++ b/drivers/gpu/drm/i915/display/intel_fb.c
@@ -1395,7 +1395,7 @@ static u32 calc_plane_remap_info(const struct intel_framebuffer *fb, int color_p
 			       plane_view_height_tiles(fb, color_plane, dims, y));
 	}
 
-	if (view->gtt.type == I915_GGTT_VIEW_ROTATED) {
+	if (view->gtt.type == I915_GTT_VIEW_ROTATED) {
 		drm_WARN_ON(&i915->drm, remap_info->linear);
 		check_array_bounds(i915, view->gtt.rotated.plane, color_plane);
 
@@ -1420,7 +1420,7 @@ static u32 calc_plane_remap_info(const struct intel_framebuffer *fb, int color_p
 		/* rotate the tile dimensions to match the GTT view */
 		swap(tile_width, tile_height);
 	} else {
-		drm_WARN_ON(&i915->drm, view->gtt.type != I915_GGTT_VIEW_REMAPPED);
+		drm_WARN_ON(&i915->drm, view->gtt.type != I915_GTT_VIEW_REMAPPED);
 
 		check_array_bounds(i915, view->gtt.remapped.plane, color_plane);
 
@@ -1503,12 +1503,12 @@ calc_plane_normal_size(const struct intel_framebuffer *fb, int color_plane,
 }
 
 static void intel_fb_view_init(struct drm_i915_private *i915, struct intel_fb_view *view,
-			       enum i915_ggtt_view_type view_type)
+			       enum i915_gtt_view_type view_type)
 {
 	memset(view, 0, sizeof(*view));
 	view->gtt.type = view_type;
 
-	if (view_type == I915_GGTT_VIEW_REMAPPED && IS_ALDERLAKE_P(i915))
+	if (view_type == I915_GTT_VIEW_REMAPPED && IS_ALDERLAKE_P(i915))
 		view->gtt.remapped.plane_alignment = SZ_2M / PAGE_SIZE;
 }
 
@@ -1530,16 +1530,16 @@ int intel_fill_fb_info(struct drm_i915_private *i915, struct intel_framebuffer *
 	int i, num_planes = fb->base.format->num_planes;
 	unsigned int tile_size = intel_tile_size(i915);
 
-	intel_fb_view_init(i915, &fb->normal_view, I915_GGTT_VIEW_NORMAL);
+	intel_fb_view_init(i915, &fb->normal_view, I915_GTT_VIEW_NORMAL);
 
 	drm_WARN_ON(&i915->drm,
 		    intel_fb_supports_90_270_rotation(fb) &&
 		    intel_fb_needs_pot_stride_remap(fb));
 
 	if (intel_fb_supports_90_270_rotation(fb))
-		intel_fb_view_init(i915, &fb->rotated_view, I915_GGTT_VIEW_ROTATED);
+		intel_fb_view_init(i915, &fb->rotated_view, I915_GTT_VIEW_ROTATED);
 	if (intel_fb_needs_pot_stride_remap(fb))
-		intel_fb_view_init(i915, &fb->remapped_view, I915_GGTT_VIEW_REMAPPED);
+		intel_fb_view_init(i915, &fb->remapped_view, I915_GTT_VIEW_REMAPPED);
 
 	for (i = 0; i < num_planes; i++) {
 		struct fb_plane_view_dims view_dims;
@@ -1620,8 +1620,8 @@ static void intel_plane_remap_gtt(struct intel_plane_state *plane_state)
 	u32 gtt_offset = 0;
 
 	intel_fb_view_init(i915, &plane_state->view,
-			   drm_rotation_90_or_270(rotation) ? I915_GGTT_VIEW_ROTATED :
-							      I915_GGTT_VIEW_REMAPPED);
+			   drm_rotation_90_or_270(rotation) ? I915_GTT_VIEW_ROTATED :
+							      I915_GTT_VIEW_REMAPPED);
 
 	src_x = plane_state->uapi.src.x1 >> 16;
 	src_y = plane_state->uapi.src.y1 >> 16;
diff --git a/drivers/gpu/drm/i915/display/intel_fb_pin.c b/drivers/gpu/drm/i915/display/intel_fb_pin.c
index bd6e7c98e751..c86e5d4ee016 100644
--- a/drivers/gpu/drm/i915/display/intel_fb_pin.c
+++ b/drivers/gpu/drm/i915/display/intel_fb_pin.c
@@ -18,7 +18,7 @@
 
 static struct i915_vma *
 intel_pin_fb_obj_dpt(struct drm_framebuffer *fb,
-		     const struct i915_ggtt_view *view,
+		     const struct i915_gtt_view *view,
 		     bool uses_fence,
 		     unsigned long *out_flags,
 		     struct i915_address_space *vm)
@@ -79,7 +79,7 @@ err:
 struct i915_vma *
 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
 			   bool phys_cursor,
-			   const struct i915_ggtt_view *view,
+			   const struct i915_gtt_view *view,
 			   bool uses_fence,
 			   unsigned long *out_flags)
 {
diff --git a/drivers/gpu/drm/i915/display/intel_fb_pin.h b/drivers/gpu/drm/i915/display/intel_fb_pin.h
index e4fcd0218d9d..de0efaa25905 100644
--- a/drivers/gpu/drm/i915/display/intel_fb_pin.h
+++ b/drivers/gpu/drm/i915/display/intel_fb_pin.h
@@ -11,12 +11,12 @@
 struct drm_framebuffer;
 struct i915_vma;
 struct intel_plane_state;
-struct i915_ggtt_view;
+struct i915_gtt_view;
 
 struct i915_vma *
 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
 			   bool phys_cursor,
-			   const struct i915_ggtt_view *view,
+			   const struct i915_gtt_view *view,
 			   bool uses_fence,
 			   unsigned long *out_flags);
 
diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c
index 16537830ccf0..f38175304928 100644
--- a/drivers/gpu/drm/i915/display/intel_fbc.c
+++ b/drivers/gpu/drm/i915/display/intel_fbc.c
@@ -55,11 +55,11 @@
 
 #define for_each_fbc_id(__dev_priv, __fbc_id) \
 	for ((__fbc_id) = INTEL_FBC_A; (__fbc_id) < I915_MAX_FBCS; (__fbc_id)++) \
-		for_each_if(INTEL_INFO(__dev_priv)->display.fbc_mask & BIT(__fbc_id))
+		for_each_if(RUNTIME_INFO(__dev_priv)->fbc_mask & BIT(__fbc_id))
 
 #define for_each_intel_fbc(__dev_priv, __fbc, __fbc_id) \
 	for_each_fbc_id((__dev_priv), (__fbc_id)) \
-		for_each_if((__fbc) = (__dev_priv)->fbc[(__fbc_id)])
+		for_each_if((__fbc) = (__dev_priv)->display.fbc[(__fbc_id)])
 
 struct intel_fbc_funcs {
 	void (*activate)(struct intel_fbc *fbc);
@@ -1098,6 +1098,12 @@ static int intel_fbc_check_plane(struct intel_atomic_state *state,
 		return 0;
 	}
 
+	/* Wa_14016291713 */
+	if (IS_DISPLAY_VER(i915, 12, 13) && crtc_state->has_psr) {
+		plane_state->no_fbc_reason = "PSR1 enabled (Wa_14016291713)";
+		return 0;
+	}
+
 	if (!pixel_format_is_valid(plane_state)) {
 		plane_state->no_fbc_reason = "pixel format not supported";
 		return 0;
@@ -1704,17 +1710,17 @@ void intel_fbc_init(struct drm_i915_private *i915)
 	enum intel_fbc_id fbc_id;
 
 	if (!drm_mm_initialized(&i915->mm.stolen))
-		mkwrite_device_info(i915)->display.fbc_mask = 0;
+		RUNTIME_INFO(i915)->fbc_mask = 0;
 
 	if (need_fbc_vtd_wa(i915))
-		mkwrite_device_info(i915)->display.fbc_mask = 0;
+		RUNTIME_INFO(i915)->fbc_mask = 0;
 
 	i915->params.enable_fbc = intel_sanitize_fbc_option(i915);
 	drm_dbg_kms(&i915->drm, "Sanitized enable_fbc value: %d\n",
 		    i915->params.enable_fbc);
 
 	for_each_fbc_id(i915, fbc_id)
-		i915->fbc[fbc_id] = intel_fbc_create(i915, fbc_id);
+		i915->display.fbc[fbc_id] = intel_fbc_create(i915, fbc_id);
 }
 
 /**
@@ -1834,7 +1840,7 @@ void intel_fbc_debugfs_register(struct drm_i915_private *i915)
 	struct drm_minor *minor = i915->drm.primary;
 	struct intel_fbc *fbc;
 
-	fbc = i915->fbc[INTEL_FBC_A];
+	fbc = i915->display.fbc[INTEL_FBC_A];
 	if (fbc)
 		intel_fbc_debugfs_add(fbc, minor->debugfs_root);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_fbc.h b/drivers/gpu/drm/i915/display/intel_fbc.h
index db60143295ec..4adb98afe6ff 100644
--- a/drivers/gpu/drm/i915/display/intel_fbc.h
+++ b/drivers/gpu/drm/i915/display/intel_fbc.h
@@ -19,6 +19,7 @@ struct intel_plane_state;
 
 enum intel_fbc_id {
 	INTEL_FBC_A,
+	INTEL_FBC_B,
 
 	I915_MAX_FBCS,
 };
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c
index 221336178991..112aa0447a0d 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/display/intel_fbdev.c
@@ -198,8 +198,8 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 	struct i915_ggtt *ggtt = to_gt(dev_priv)->ggtt;
-	const struct i915_ggtt_view view = {
-		.type = I915_GGTT_VIEW_NORMAL,
+	const struct i915_gtt_view view = {
+		.type = I915_GTT_VIEW_NORMAL,
 	};
 	intel_wakeref_t wakeref;
 	struct fb_info *info;
@@ -210,6 +210,12 @@ static int intelfb_create(struct drm_fb_helper *helper,
 	struct drm_i915_gem_object *obj;
 	int ret;
 
+	mutex_lock(&ifbdev->hpd_lock);
+	ret = ifbdev->hpd_suspended ? -EAGAIN : 0;
+	mutex_unlock(&ifbdev->hpd_lock);
+	if (ret)
+		return ret;
+
 	if (intel_fb &&
 	    (sizes->fb_width > intel_fb->base.width ||
 	     sizes->fb_height > intel_fb->base.height)) {
@@ -500,7 +506,7 @@ static void intel_fbdev_suspend_worker(struct work_struct *work)
 {
 	intel_fbdev_set_suspend(&container_of(work,
 					      struct drm_i915_private,
-					      fbdev_suspend_work)->drm,
+					      display.fbdev.suspend_work)->drm,
 				FBINFO_STATE_RUNNING,
 				true);
 }
@@ -530,8 +536,8 @@ int intel_fbdev_init(struct drm_device *dev)
 		return ret;
 	}
 
-	dev_priv->fbdev = ifbdev;
-	INIT_WORK(&dev_priv->fbdev_suspend_work, intel_fbdev_suspend_worker);
+	dev_priv->display.fbdev.fbdev = ifbdev;
+	INIT_WORK(&dev_priv->display.fbdev.suspend_work, intel_fbdev_suspend_worker);
 
 	return 0;
 }
@@ -548,7 +554,7 @@ static void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
 
 void intel_fbdev_initial_config_async(struct drm_device *dev)
 {
-	struct intel_fbdev *ifbdev = to_i915(dev)->fbdev;
+	struct intel_fbdev *ifbdev = to_i915(dev)->display.fbdev.fbdev;
 
 	if (!ifbdev)
 		return;
@@ -568,12 +574,13 @@ static void intel_fbdev_sync(struct intel_fbdev *ifbdev)
 
 void intel_fbdev_unregister(struct drm_i915_private *dev_priv)
 {
-	struct intel_fbdev *ifbdev = dev_priv->fbdev;
+	struct intel_fbdev *ifbdev = dev_priv->display.fbdev.fbdev;
 
 	if (!ifbdev)
 		return;
 
-	cancel_work_sync(&dev_priv->fbdev_suspend_work);
+	intel_fbdev_set_suspend(&dev_priv->drm, FBINFO_STATE_SUSPENDED, true);
+
 	if (!current_is_async())
 		intel_fbdev_sync(ifbdev);
 
@@ -582,7 +589,7 @@ void intel_fbdev_unregister(struct drm_i915_private *dev_priv)
 
 void intel_fbdev_fini(struct drm_i915_private *dev_priv)
 {
-	struct intel_fbdev *ifbdev = fetch_and_zero(&dev_priv->fbdev);
+	struct intel_fbdev *ifbdev = fetch_and_zero(&dev_priv->display.fbdev.fbdev);
 
 	if (!ifbdev)
 		return;
@@ -596,7 +603,7 @@ void intel_fbdev_fini(struct drm_i915_private *dev_priv)
  */
 static void intel_fbdev_hpd_set_suspend(struct drm_i915_private *i915, int state)
 {
-	struct intel_fbdev *ifbdev = i915->fbdev;
+	struct intel_fbdev *ifbdev = i915->display.fbdev.fbdev;
 	bool send_hpd = false;
 
 	mutex_lock(&ifbdev->hpd_lock);
@@ -614,11 +621,11 @@ static void intel_fbdev_hpd_set_suspend(struct drm_i915_private *i915, int state
 void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct intel_fbdev *ifbdev = dev_priv->fbdev;
+	struct intel_fbdev *ifbdev = dev_priv->display.fbdev.fbdev;
 	struct fb_info *info;
 
 	if (!ifbdev || !ifbdev->vma)
-		return;
+		goto set_suspend;
 
 	info = ifbdev->helper.fbdev;
 
@@ -631,7 +638,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous
 		 * ourselves, so only flush outstanding work upon suspend!
 		 */
 		if (state != FBINFO_STATE_RUNNING)
-			flush_work(&dev_priv->fbdev_suspend_work);
+			flush_work(&dev_priv->display.fbdev.suspend_work);
 
 		console_lock();
 	} else {
@@ -645,7 +652,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous
 			/* Don't block our own workqueue as this can
 			 * be run in parallel with other i915.ko tasks.
 			 */
-			schedule_work(&dev_priv->fbdev_suspend_work);
+			schedule_work(&dev_priv->display.fbdev.suspend_work);
 			return;
 		}
 	}
@@ -661,12 +668,13 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous
 	drm_fb_helper_set_suspend(&ifbdev->helper, state);
 	console_unlock();
 
+set_suspend:
 	intel_fbdev_hpd_set_suspend(dev_priv, state);
 }
 
 void intel_fbdev_output_poll_changed(struct drm_device *dev)
 {
-	struct intel_fbdev *ifbdev = to_i915(dev)->fbdev;
+	struct intel_fbdev *ifbdev = to_i915(dev)->display.fbdev.fbdev;
 	bool send_hpd;
 
 	if (!ifbdev)
@@ -685,7 +693,7 @@ void intel_fbdev_output_poll_changed(struct drm_device *dev)
 
 void intel_fbdev_restore_mode(struct drm_device *dev)
 {
-	struct intel_fbdev *ifbdev = to_i915(dev)->fbdev;
+	struct intel_fbdev *ifbdev = to_i915(dev)->display.fbdev.fbdev;
 
 	if (!ifbdev)
 		return;
diff --git a/drivers/gpu/drm/i915/display/intel_fdi.c b/drivers/gpu/drm/i915/display/intel_fdi.c
index 67d2484afbaa..7f47e5c85c81 100644
--- a/drivers/gpu/drm/i915/display/intel_fdi.c
+++ b/drivers/gpu/drm/i915/display/intel_fdi.c
@@ -113,7 +113,7 @@ void intel_fdi_link_train(struct intel_crtc *crtc,
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
-	dev_priv->fdi_funcs->fdi_link_train(crtc, crtc_state);
+	dev_priv->display.funcs.fdi->fdi_link_train(crtc, crtc_state);
 }
 
 /* units of 100MHz */
@@ -210,14 +210,14 @@ void intel_fdi_pll_freq_update(struct drm_i915_private *i915)
 		u32 fdi_pll_clk =
 			intel_de_read(i915, FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
 
-		i915->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
+		i915->display.fdi.pll_freq = (fdi_pll_clk + 2) * 10000;
 	} else if (IS_SANDYBRIDGE(i915) || IS_IVYBRIDGE(i915)) {
-		i915->fdi_pll_freq = 270000;
+		i915->display.fdi.pll_freq = 270000;
 	} else {
 		return;
 	}
 
-	drm_dbg(&i915->drm, "FDI PLL freq=%d\n", i915->fdi_pll_freq);
+	drm_dbg(&i915->drm, "FDI PLL freq=%d\n", i915->display.fdi.pll_freq);
 }
 
 int intel_fdi_link_freq(struct drm_i915_private *i915,
@@ -226,7 +226,7 @@ int intel_fdi_link_freq(struct drm_i915_private *i915,
 	if (HAS_DDI(i915))
 		return pipe_config->port_clock; /* SPLL */
 	else
-		return i915->fdi_pll_freq;
+		return i915->display.fdi.pll_freq;
 }
 
 int ilk_fdi_compute_config(struct intel_crtc *crtc,
@@ -256,7 +256,7 @@ retry:
 	pipe_config->fdi_lanes = lane;
 
 	intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
-			       link_bw, &pipe_config->fdi_m_n, false, false);
+			       link_bw, &pipe_config->fdi_m_n, false);
 
 	ret = ilk_check_fdi_lanes(dev, crtc->pipe, pipe_config);
 	if (ret == -EDEADLK)
@@ -789,7 +789,7 @@ void hsw_fdi_link_train(struct intel_encoder *encoder,
 		       FDI_RX_PWRDN_LANE1_VAL(2) | FDI_RX_PWRDN_LANE0_VAL(2) | FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
 
 	/* Enable the PCH Receiver FDI PLL */
-	rx_ctl_val = dev_priv->fdi_rx_config | FDI_RX_ENHANCE_FRAME_ENABLE |
+	rx_ctl_val = dev_priv->display.fdi.rx_config | FDI_RX_ENHANCE_FRAME_ENABLE |
 		     FDI_RX_PLL_ENABLE |
 		     FDI_DP_PORT_WIDTH(crtc_state->fdi_lanes);
 	intel_de_write(dev_priv, FDI_RX_CTL(PIPE_A), rx_ctl_val);
@@ -1066,11 +1066,11 @@ void
 intel_fdi_init_hook(struct drm_i915_private *dev_priv)
 {
 	if (IS_IRONLAKE(dev_priv)) {
-		dev_priv->fdi_funcs = &ilk_funcs;
+		dev_priv->display.funcs.fdi = &ilk_funcs;
 	} else if (IS_SANDYBRIDGE(dev_priv)) {
-		dev_priv->fdi_funcs = &gen6_funcs;
+		dev_priv->display.funcs.fdi = &gen6_funcs;
 	} else if (IS_IVYBRIDGE(dev_priv)) {
 		/* FIXME: detect B0+ stepping and use auto training */
-		dev_priv->fdi_funcs = &ivb_funcs;
+		dev_priv->display.funcs.fdi = &ivb_funcs;
 	}
 }
diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.c b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
index 791248f812aa..d80e3e8a9b01 100644
--- a/drivers/gpu/drm/i915/display/intel_frontbuffer.c
+++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.c
@@ -81,9 +81,9 @@ static void frontbuffer_flush(struct drm_i915_private *i915,
 			      enum fb_op_origin origin)
 {
 	/* Delay flushing when rings are still busy.*/
-	spin_lock(&i915->fb_tracking.lock);
-	frontbuffer_bits &= ~i915->fb_tracking.busy_bits;
-	spin_unlock(&i915->fb_tracking.lock);
+	spin_lock(&i915->display.fb_tracking.lock);
+	frontbuffer_bits &= ~i915->display.fb_tracking.busy_bits;
+	spin_unlock(&i915->display.fb_tracking.lock);
 
 	if (!frontbuffer_bits)
 		return;
@@ -111,11 +111,11 @@ static void frontbuffer_flush(struct drm_i915_private *i915,
 void intel_frontbuffer_flip_prepare(struct drm_i915_private *i915,
 				    unsigned frontbuffer_bits)
 {
-	spin_lock(&i915->fb_tracking.lock);
-	i915->fb_tracking.flip_bits |= frontbuffer_bits;
+	spin_lock(&i915->display.fb_tracking.lock);
+	i915->display.fb_tracking.flip_bits |= frontbuffer_bits;
 	/* Remove stale busy bits due to the old buffer. */
-	i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
-	spin_unlock(&i915->fb_tracking.lock);
+	i915->display.fb_tracking.busy_bits &= ~frontbuffer_bits;
+	spin_unlock(&i915->display.fb_tracking.lock);
 }
 
 /**
@@ -131,11 +131,11 @@ void intel_frontbuffer_flip_prepare(struct drm_i915_private *i915,
 void intel_frontbuffer_flip_complete(struct drm_i915_private *i915,
 				     unsigned frontbuffer_bits)
 {
-	spin_lock(&i915->fb_tracking.lock);
+	spin_lock(&i915->display.fb_tracking.lock);
 	/* Mask any cancelled flips. */
-	frontbuffer_bits &= i915->fb_tracking.flip_bits;
-	i915->fb_tracking.flip_bits &= ~frontbuffer_bits;
-	spin_unlock(&i915->fb_tracking.lock);
+	frontbuffer_bits &= i915->display.fb_tracking.flip_bits;
+	i915->display.fb_tracking.flip_bits &= ~frontbuffer_bits;
+	spin_unlock(&i915->display.fb_tracking.lock);
 
 	if (frontbuffer_bits)
 		frontbuffer_flush(i915, frontbuffer_bits, ORIGIN_FLIP);
@@ -155,10 +155,10 @@ void intel_frontbuffer_flip_complete(struct drm_i915_private *i915,
 void intel_frontbuffer_flip(struct drm_i915_private *i915,
 			    unsigned frontbuffer_bits)
 {
-	spin_lock(&i915->fb_tracking.lock);
+	spin_lock(&i915->display.fb_tracking.lock);
 	/* Remove stale busy bits due to the old buffer. */
-	i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
-	spin_unlock(&i915->fb_tracking.lock);
+	i915->display.fb_tracking.busy_bits &= ~frontbuffer_bits;
+	spin_unlock(&i915->display.fb_tracking.lock);
 
 	frontbuffer_flush(i915, frontbuffer_bits, ORIGIN_FLIP);
 }
@@ -170,10 +170,10 @@ void __intel_fb_invalidate(struct intel_frontbuffer *front,
 	struct drm_i915_private *i915 = to_i915(front->obj->base.dev);
 
 	if (origin == ORIGIN_CS) {
-		spin_lock(&i915->fb_tracking.lock);
-		i915->fb_tracking.busy_bits |= frontbuffer_bits;
-		i915->fb_tracking.flip_bits &= ~frontbuffer_bits;
-		spin_unlock(&i915->fb_tracking.lock);
+		spin_lock(&i915->display.fb_tracking.lock);
+		i915->display.fb_tracking.busy_bits |= frontbuffer_bits;
+		i915->display.fb_tracking.flip_bits &= ~frontbuffer_bits;
+		spin_unlock(&i915->display.fb_tracking.lock);
 	}
 
 	trace_intel_frontbuffer_invalidate(frontbuffer_bits, origin);
@@ -191,11 +191,11 @@ void __intel_fb_flush(struct intel_frontbuffer *front,
 	struct drm_i915_private *i915 = to_i915(front->obj->base.dev);
 
 	if (origin == ORIGIN_CS) {
-		spin_lock(&i915->fb_tracking.lock);
+		spin_lock(&i915->display.fb_tracking.lock);
 		/* Filter out new bits since rendering started. */
-		frontbuffer_bits &= i915->fb_tracking.busy_bits;
-		i915->fb_tracking.busy_bits &= ~frontbuffer_bits;
-		spin_unlock(&i915->fb_tracking.lock);
+		frontbuffer_bits &= i915->display.fb_tracking.busy_bits;
+		i915->display.fb_tracking.busy_bits &= ~frontbuffer_bits;
+		spin_unlock(&i915->display.fb_tracking.lock);
 	}
 
 	if (frontbuffer_bits)
@@ -221,7 +221,7 @@ static void frontbuffer_retire(struct i915_active *ref)
 }
 
 static void frontbuffer_release(struct kref *ref)
-	__releases(&to_i915(front->obj->base.dev)->fb_tracking.lock)
+	__releases(&to_i915(front->obj->base.dev)->display.fb_tracking.lock)
 {
 	struct intel_frontbuffer *front =
 		container_of(ref, typeof(*front), ref);
@@ -238,7 +238,7 @@ static void frontbuffer_release(struct kref *ref)
 	spin_unlock(&obj->vma.lock);
 
 	RCU_INIT_POINTER(obj->frontbuffer, NULL);
-	spin_unlock(&to_i915(obj->base.dev)->fb_tracking.lock);
+	spin_unlock(&to_i915(obj->base.dev)->display.fb_tracking.lock);
 
 	i915_active_fini(&front->write);
 
@@ -268,7 +268,7 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj)
 			 frontbuffer_retire,
 			 I915_ACTIVE_RETIRE_SLEEPS);
 
-	spin_lock(&i915->fb_tracking.lock);
+	spin_lock(&i915->display.fb_tracking.lock);
 	if (rcu_access_pointer(obj->frontbuffer)) {
 		kfree(front);
 		front = rcu_dereference_protected(obj->frontbuffer, true);
@@ -277,7 +277,7 @@ intel_frontbuffer_get(struct drm_i915_gem_object *obj)
 		i915_gem_object_get(obj);
 		rcu_assign_pointer(obj->frontbuffer, front);
 	}
-	spin_unlock(&i915->fb_tracking.lock);
+	spin_unlock(&i915->display.fb_tracking.lock);
 
 	return front;
 }
@@ -286,7 +286,7 @@ void intel_frontbuffer_put(struct intel_frontbuffer *front)
 {
 	kref_put_lock(&front->ref,
 		      frontbuffer_release,
-		      &to_i915(front->obj->base.dev)->fb_tracking.lock);
+		      &to_i915(front->obj->base.dev)->display.fb_tracking.lock);
 }
 
 /**
@@ -311,6 +311,8 @@ void intel_frontbuffer_track(struct intel_frontbuffer *old,
 	 */
 	BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
 		     BITS_PER_TYPE(atomic_t));
+	BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > 32);
+	BUILD_BUG_ON(I915_MAX_PLANES > INTEL_FRONTBUFFER_BITS_PER_PIPE);
 
 	if (old) {
 		drm_WARN_ON(old->obj->base.dev,
diff --git a/drivers/gpu/drm/i915/display/intel_frontbuffer.h b/drivers/gpu/drm/i915/display/intel_frontbuffer.h
index ff0c37b079aa..3c474ed937fb 100644
--- a/drivers/gpu/drm/i915/display/intel_frontbuffer.h
+++ b/drivers/gpu/drm/i915/display/intel_frontbuffer.h
@@ -25,6 +25,7 @@
 #define __INTEL_FRONTBUFFER_H__
 
 #include <linux/atomic.h>
+#include <linux/bits.h>
 #include <linux/kref.h>
 
 #include "gem/i915_gem_object_types.h"
@@ -48,6 +49,23 @@ struct intel_frontbuffer {
 	struct rcu_head rcu;
 };
 
+/*
+ * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is
+ * considered to be the frontbuffer for the given plane interface-wise. This
+ * doesn't mean that the hw necessarily already scans it out, but that any
+ * rendering (by the cpu or gpu) will land in the frontbuffer eventually.
+ *
+ * We have one bit per pipe and per scanout plane type.
+ */
+#define INTEL_FRONTBUFFER_BITS_PER_PIPE 8
+#define INTEL_FRONTBUFFER(pipe, plane_id) \
+	BIT((plane_id) + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe));
+#define INTEL_FRONTBUFFER_OVERLAY(pipe) \
+	BIT(INTEL_FRONTBUFFER_BITS_PER_PIPE - 1 + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))
+#define INTEL_FRONTBUFFER_ALL_MASK(pipe) \
+	GENMASK(INTEL_FRONTBUFFER_BITS_PER_PIPE * ((pipe) + 1) - 1,	\
+		INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))
+
 void intel_frontbuffer_flip_prepare(struct drm_i915_private *i915,
 				    unsigned frontbuffer_bits);
 void intel_frontbuffer_flip_complete(struct drm_i915_private *i915,
diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c
index a6ba7fb72339..74443f57f62d 100644
--- a/drivers/gpu/drm/i915/display/intel_gmbus.c
+++ b/drivers/gpu/drm/i915/display/intel_gmbus.c
@@ -37,6 +37,7 @@
 #include "intel_de.h"
 #include "intel_display_types.h"
 #include "intel_gmbus.h"
+#include "intel_gmbus_regs.h"
 
 struct intel_gmbus {
 	struct i2c_adapter adapter;
@@ -45,7 +46,7 @@ struct intel_gmbus {
 	u32 reg0;
 	i915_reg_t gpio_reg;
 	struct i2c_algo_bit_data bit_algo;
-	struct drm_i915_private *dev_priv;
+	struct drm_i915_private *i915;
 };
 
 struct gmbus_pin {
@@ -116,6 +117,18 @@ static const struct gmbus_pin gmbus_pins_dg2[] = {
 	[GMBUS_PIN_9_TC1_ICP] = { "tc1", GPIOJ },
 };
 
+static const struct gmbus_pin gmbus_pins_mtp[] = {
+	[GMBUS_PIN_1_BXT] = { "dpa", GPIOB },
+	[GMBUS_PIN_2_BXT] = { "dpb", GPIOC },
+	[GMBUS_PIN_3_BXT] = { "dpc", GPIOD },
+	[GMBUS_PIN_4_CNP] = { "dpd", GPIOE },
+	[GMBUS_PIN_5_MTP] = { "dpe", GPIOF },
+	[GMBUS_PIN_9_TC1_ICP] = { "tc1", GPIOJ },
+	[GMBUS_PIN_10_TC2_ICP] = { "tc2", GPIOK },
+	[GMBUS_PIN_11_TC3_ICP] = { "tc3", GPIOL },
+	[GMBUS_PIN_12_TC4_ICP] = { "tc4", GPIOM },
+};
+
 static const struct gmbus_pin *get_gmbus_pin(struct drm_i915_private *i915,
 					     unsigned int pin)
 {
@@ -128,6 +141,9 @@ static const struct gmbus_pin *get_gmbus_pin(struct drm_i915_private *i915,
 	} else if (INTEL_PCH_TYPE(i915) >= PCH_DG1) {
 		pins = gmbus_pins_dg1;
 		size = ARRAY_SIZE(gmbus_pins_dg1);
+	} else if (INTEL_PCH_TYPE(i915) >= PCH_MTP) {
+		pins = gmbus_pins_mtp;
+		size = ARRAY_SIZE(gmbus_pins_mtp);
 	} else if (INTEL_PCH_TYPE(i915) >= PCH_ICP) {
 		pins = gmbus_pins_icp;
 		size = ARRAY_SIZE(gmbus_pins_icp);
@@ -170,55 +186,55 @@ to_intel_gmbus(struct i2c_adapter *i2c)
 }
 
 void
-intel_gmbus_reset(struct drm_i915_private *dev_priv)
+intel_gmbus_reset(struct drm_i915_private *i915)
 {
-	intel_de_write(dev_priv, GMBUS0, 0);
-	intel_de_write(dev_priv, GMBUS4, 0);
+	intel_de_write(i915, GMBUS0(i915), 0);
+	intel_de_write(i915, GMBUS4(i915), 0);
 }
 
-static void pnv_gmbus_clock_gating(struct drm_i915_private *dev_priv,
+static void pnv_gmbus_clock_gating(struct drm_i915_private *i915,
 				   bool enable)
 {
 	u32 val;
 
 	/* When using bit bashing for I2C, this bit needs to be set to 1 */
-	val = intel_de_read(dev_priv, DSPCLK_GATE_D);
+	val = intel_de_read(i915, DSPCLK_GATE_D(i915));
 	if (!enable)
 		val |= PNV_GMBUSUNIT_CLOCK_GATE_DISABLE;
 	else
 		val &= ~PNV_GMBUSUNIT_CLOCK_GATE_DISABLE;
-	intel_de_write(dev_priv, DSPCLK_GATE_D, val);
+	intel_de_write(i915, DSPCLK_GATE_D(i915), val);
 }
 
-static void pch_gmbus_clock_gating(struct drm_i915_private *dev_priv,
+static void pch_gmbus_clock_gating(struct drm_i915_private *i915,
 				   bool enable)
 {
 	u32 val;
 
-	val = intel_de_read(dev_priv, SOUTH_DSPCLK_GATE_D);
+	val = intel_de_read(i915, SOUTH_DSPCLK_GATE_D);
 	if (!enable)
 		val |= PCH_GMBUSUNIT_CLOCK_GATE_DISABLE;
 	else
 		val &= ~PCH_GMBUSUNIT_CLOCK_GATE_DISABLE;
-	intel_de_write(dev_priv, SOUTH_DSPCLK_GATE_D, val);
+	intel_de_write(i915, SOUTH_DSPCLK_GATE_D, val);
 }
 
-static void bxt_gmbus_clock_gating(struct drm_i915_private *dev_priv,
+static void bxt_gmbus_clock_gating(struct drm_i915_private *i915,
 				   bool enable)
 {
 	u32 val;
 
-	val = intel_de_read(dev_priv, GEN9_CLKGATE_DIS_4);
+	val = intel_de_read(i915, GEN9_CLKGATE_DIS_4);
 	if (!enable)
 		val |= BXT_GMBUS_GATING_DIS;
 	else
 		val &= ~BXT_GMBUS_GATING_DIS;
-	intel_de_write(dev_priv, GEN9_CLKGATE_DIS_4, val);
+	intel_de_write(i915, GEN9_CLKGATE_DIS_4, val);
 }
 
 static u32 get_reserved(struct intel_gmbus *bus)
 {
-	struct drm_i915_private *i915 = bus->dev_priv;
+	struct drm_i915_private *i915 = bus->i915;
 	struct intel_uncore *uncore = &i915->uncore;
 	u32 reserved = 0;
 
@@ -234,7 +250,7 @@ static u32 get_reserved(struct intel_gmbus *bus)
 static int get_clock(void *data)
 {
 	struct intel_gmbus *bus = data;
-	struct intel_uncore *uncore = &bus->dev_priv->uncore;
+	struct intel_uncore *uncore = &bus->i915->uncore;
 	u32 reserved = get_reserved(bus);
 
 	intel_uncore_write_notrace(uncore,
@@ -249,7 +265,7 @@ static int get_clock(void *data)
 static int get_data(void *data)
 {
 	struct intel_gmbus *bus = data;
-	struct intel_uncore *uncore = &bus->dev_priv->uncore;
+	struct intel_uncore *uncore = &bus->i915->uncore;
 	u32 reserved = get_reserved(bus);
 
 	intel_uncore_write_notrace(uncore,
@@ -264,7 +280,7 @@ static int get_data(void *data)
 static void set_clock(void *data, int state_high)
 {
 	struct intel_gmbus *bus = data;
-	struct intel_uncore *uncore = &bus->dev_priv->uncore;
+	struct intel_uncore *uncore = &bus->i915->uncore;
 	u32 reserved = get_reserved(bus);
 	u32 clock_bits;
 
@@ -283,7 +299,7 @@ static void set_clock(void *data, int state_high)
 static void set_data(void *data, int state_high)
 {
 	struct intel_gmbus *bus = data;
-	struct intel_uncore *uncore = &bus->dev_priv->uncore;
+	struct intel_uncore *uncore = &bus->i915->uncore;
 	u32 reserved = get_reserved(bus);
 	u32 data_bits;
 
@@ -301,12 +317,12 @@ static int
 intel_gpio_pre_xfer(struct i2c_adapter *adapter)
 {
 	struct intel_gmbus *bus = to_intel_gmbus(adapter);
-	struct drm_i915_private *dev_priv = bus->dev_priv;
+	struct drm_i915_private *i915 = bus->i915;
 
-	intel_gmbus_reset(dev_priv);
+	intel_gmbus_reset(i915);
 
-	if (IS_PINEVIEW(dev_priv))
-		pnv_gmbus_clock_gating(dev_priv, false);
+	if (IS_PINEVIEW(i915))
+		pnv_gmbus_clock_gating(i915, false);
 
 	set_data(bus, 1);
 	set_clock(bus, 1);
@@ -318,13 +334,13 @@ static void
 intel_gpio_post_xfer(struct i2c_adapter *adapter)
 {
 	struct intel_gmbus *bus = to_intel_gmbus(adapter);
-	struct drm_i915_private *dev_priv = bus->dev_priv;
+	struct drm_i915_private *i915 = bus->i915;
 
 	set_data(bus, 1);
 	set_clock(bus, 1);
 
-	if (IS_PINEVIEW(dev_priv))
-		pnv_gmbus_clock_gating(dev_priv, true);
+	if (IS_PINEVIEW(i915))
+		pnv_gmbus_clock_gating(i915, true);
 }
 
 static void
@@ -356,7 +372,7 @@ static bool has_gmbus_irq(struct drm_i915_private *i915)
 	return HAS_GMBUS_IRQ(i915) && intel_irqs_enabled(i915);
 }
 
-static int gmbus_wait(struct drm_i915_private *dev_priv, u32 status, u32 irq_en)
+static int gmbus_wait(struct drm_i915_private *i915, u32 status, u32 irq_en)
 {
 	DEFINE_WAIT(wait);
 	u32 gmbus2;
@@ -366,21 +382,21 @@ static int gmbus_wait(struct drm_i915_private *dev_priv, u32 status, u32 irq_en)
 	 * we also need to check for NAKs besides the hw ready/idle signal, we
 	 * need to wake up periodically and check that ourselves.
 	 */
-	if (!has_gmbus_irq(dev_priv))
+	if (!has_gmbus_irq(i915))
 		irq_en = 0;
 
-	add_wait_queue(&dev_priv->gmbus_wait_queue, &wait);
-	intel_de_write_fw(dev_priv, GMBUS4, irq_en);
+	add_wait_queue(&i915->display.gmbus.wait_queue, &wait);
+	intel_de_write_fw(i915, GMBUS4(i915), irq_en);
 
 	status |= GMBUS_SATOER;
-	ret = wait_for_us((gmbus2 = intel_de_read_fw(dev_priv, GMBUS2)) & status,
+	ret = wait_for_us((gmbus2 = intel_de_read_fw(i915, GMBUS2(i915))) & status,
 			  2);
 	if (ret)
-		ret = wait_for((gmbus2 = intel_de_read_fw(dev_priv, GMBUS2)) & status,
+		ret = wait_for((gmbus2 = intel_de_read_fw(i915, GMBUS2(i915))) & status,
 			       50);
 
-	intel_de_write_fw(dev_priv, GMBUS4, 0);
-	remove_wait_queue(&dev_priv->gmbus_wait_queue, &wait);
+	intel_de_write_fw(i915, GMBUS4(i915), 0);
+	remove_wait_queue(&i915->display.gmbus.wait_queue, &wait);
 
 	if (gmbus2 & GMBUS_SATOER)
 		return -ENXIO;
@@ -389,7 +405,7 @@ static int gmbus_wait(struct drm_i915_private *dev_priv, u32 status, u32 irq_en)
 }
 
 static int
-gmbus_wait_idle(struct drm_i915_private *dev_priv)
+gmbus_wait_idle(struct drm_i915_private *i915)
 {
 	DEFINE_WAIT(wait);
 	u32 irq_enable;
@@ -397,35 +413,35 @@ gmbus_wait_idle(struct drm_i915_private *dev_priv)
 
 	/* Important: The hw handles only the first bit, so set only one! */
 	irq_enable = 0;
-	if (has_gmbus_irq(dev_priv))
+	if (has_gmbus_irq(i915))
 		irq_enable = GMBUS_IDLE_EN;
 
-	add_wait_queue(&dev_priv->gmbus_wait_queue, &wait);
-	intel_de_write_fw(dev_priv, GMBUS4, irq_enable);
+	add_wait_queue(&i915->display.gmbus.wait_queue, &wait);
+	intel_de_write_fw(i915, GMBUS4(i915), irq_enable);
 
-	ret = intel_wait_for_register_fw(&dev_priv->uncore,
-					 GMBUS2, GMBUS_ACTIVE, 0,
+	ret = intel_wait_for_register_fw(&i915->uncore,
+					 GMBUS2(i915), GMBUS_ACTIVE, 0,
 					 10);
 
-	intel_de_write_fw(dev_priv, GMBUS4, 0);
-	remove_wait_queue(&dev_priv->gmbus_wait_queue, &wait);
+	intel_de_write_fw(i915, GMBUS4(i915), 0);
+	remove_wait_queue(&i915->display.gmbus.wait_queue, &wait);
 
 	return ret;
 }
 
-static unsigned int gmbus_max_xfer_size(struct drm_i915_private *dev_priv)
+static unsigned int gmbus_max_xfer_size(struct drm_i915_private *i915)
 {
-	return DISPLAY_VER(dev_priv) >= 9 ? GEN9_GMBUS_BYTE_COUNT_MAX :
+	return DISPLAY_VER(i915) >= 9 ? GEN9_GMBUS_BYTE_COUNT_MAX :
 	       GMBUS_BYTE_COUNT_MAX;
 }
 
 static int
-gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv,
+gmbus_xfer_read_chunk(struct drm_i915_private *i915,
 		      unsigned short addr, u8 *buf, unsigned int len,
 		      u32 gmbus0_reg, u32 gmbus1_index)
 {
 	unsigned int size = len;
-	bool burst_read = len > gmbus_max_xfer_size(dev_priv);
+	bool burst_read = len > gmbus_max_xfer_size(i915);
 	bool extra_byte_added = false;
 
 	if (burst_read) {
@@ -438,21 +454,21 @@ gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv,
 			len++;
 		}
 		size = len % 256 + 256;
-		intel_de_write_fw(dev_priv, GMBUS0,
+		intel_de_write_fw(i915, GMBUS0(i915),
 				  gmbus0_reg | GMBUS_BYTE_CNT_OVERRIDE);
 	}
 
-	intel_de_write_fw(dev_priv, GMBUS1,
+	intel_de_write_fw(i915, GMBUS1(i915),
 			  gmbus1_index | GMBUS_CYCLE_WAIT | (size << GMBUS_BYTE_COUNT_SHIFT) | (addr << GMBUS_SLAVE_ADDR_SHIFT) | GMBUS_SLAVE_READ | GMBUS_SW_RDY);
 	while (len) {
 		int ret;
 		u32 val, loop = 0;
 
-		ret = gmbus_wait(dev_priv, GMBUS_HW_RDY, GMBUS_HW_RDY_EN);
+		ret = gmbus_wait(i915, GMBUS_HW_RDY, GMBUS_HW_RDY_EN);
 		if (ret)
 			return ret;
 
-		val = intel_de_read_fw(dev_priv, GMBUS3);
+		val = intel_de_read_fw(i915, GMBUS3(i915));
 		do {
 			if (extra_byte_added && len == 1)
 				break;
@@ -463,7 +479,7 @@ gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv,
 
 		if (burst_read && len == size - 4)
 			/* Reset the override bit */
-			intel_de_write_fw(dev_priv, GMBUS0, gmbus0_reg);
+			intel_de_write_fw(i915, GMBUS0(i915), gmbus0_reg);
 	}
 
 	return 0;
@@ -480,7 +496,7 @@ gmbus_xfer_read_chunk(struct drm_i915_private *dev_priv,
 #define INTEL_GMBUS_BURST_READ_MAX_LEN		767U
 
 static int
-gmbus_xfer_read(struct drm_i915_private *dev_priv, struct i2c_msg *msg,
+gmbus_xfer_read(struct drm_i915_private *i915, struct i2c_msg *msg,
 		u32 gmbus0_reg, u32 gmbus1_index)
 {
 	u8 *buf = msg->buf;
@@ -489,12 +505,12 @@ gmbus_xfer_read(struct drm_i915_private *dev_priv, struct i2c_msg *msg,
 	int ret;
 
 	do {
-		if (HAS_GMBUS_BURST_READ(dev_priv))
+		if (HAS_GMBUS_BURST_READ(i915))
 			len = min(rx_size, INTEL_GMBUS_BURST_READ_MAX_LEN);
 		else
-			len = min(rx_size, gmbus_max_xfer_size(dev_priv));
+			len = min(rx_size, gmbus_max_xfer_size(i915));
 
-		ret = gmbus_xfer_read_chunk(dev_priv, msg->addr, buf, len,
+		ret = gmbus_xfer_read_chunk(i915, msg->addr, buf, len,
 					    gmbus0_reg, gmbus1_index);
 		if (ret)
 			return ret;
@@ -507,7 +523,7 @@ gmbus_xfer_read(struct drm_i915_private *dev_priv, struct i2c_msg *msg,
 }
 
 static int
-gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv,
+gmbus_xfer_write_chunk(struct drm_i915_private *i915,
 		       unsigned short addr, u8 *buf, unsigned int len,
 		       u32 gmbus1_index)
 {
@@ -520,8 +536,8 @@ gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv,
 		len -= 1;
 	}
 
-	intel_de_write_fw(dev_priv, GMBUS3, val);
-	intel_de_write_fw(dev_priv, GMBUS1,
+	intel_de_write_fw(i915, GMBUS3(i915), val);
+	intel_de_write_fw(i915, GMBUS1(i915),
 			  gmbus1_index | GMBUS_CYCLE_WAIT | (chunk_size << GMBUS_BYTE_COUNT_SHIFT) | (addr << GMBUS_SLAVE_ADDR_SHIFT) | GMBUS_SLAVE_WRITE | GMBUS_SW_RDY);
 	while (len) {
 		int ret;
@@ -531,9 +547,9 @@ gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv,
 			val |= *buf++ << (8 * loop);
 		} while (--len && ++loop < 4);
 
-		intel_de_write_fw(dev_priv, GMBUS3, val);
+		intel_de_write_fw(i915, GMBUS3(i915), val);
 
-		ret = gmbus_wait(dev_priv, GMBUS_HW_RDY, GMBUS_HW_RDY_EN);
+		ret = gmbus_wait(i915, GMBUS_HW_RDY, GMBUS_HW_RDY_EN);
 		if (ret)
 			return ret;
 	}
@@ -542,7 +558,7 @@ gmbus_xfer_write_chunk(struct drm_i915_private *dev_priv,
 }
 
 static int
-gmbus_xfer_write(struct drm_i915_private *dev_priv, struct i2c_msg *msg,
+gmbus_xfer_write(struct drm_i915_private *i915, struct i2c_msg *msg,
 		 u32 gmbus1_index)
 {
 	u8 *buf = msg->buf;
@@ -551,9 +567,9 @@ gmbus_xfer_write(struct drm_i915_private *dev_priv, struct i2c_msg *msg,
 	int ret;
 
 	do {
-		len = min(tx_size, gmbus_max_xfer_size(dev_priv));
+		len = min(tx_size, gmbus_max_xfer_size(i915));
 
-		ret = gmbus_xfer_write_chunk(dev_priv, msg->addr, buf, len,
+		ret = gmbus_xfer_write_chunk(i915, msg->addr, buf, len,
 					     gmbus1_index);
 		if (ret)
 			return ret;
@@ -580,7 +596,7 @@ gmbus_is_index_xfer(struct i2c_msg *msgs, int i, int num)
 }
 
 static int
-gmbus_index_xfer(struct drm_i915_private *dev_priv, struct i2c_msg *msgs,
+gmbus_index_xfer(struct drm_i915_private *i915, struct i2c_msg *msgs,
 		 u32 gmbus0_reg)
 {
 	u32 gmbus1_index = 0;
@@ -596,17 +612,17 @@ gmbus_index_xfer(struct drm_i915_private *dev_priv, struct i2c_msg *msgs,
 
 	/* GMBUS5 holds 16-bit index */
 	if (gmbus5)
-		intel_de_write_fw(dev_priv, GMBUS5, gmbus5);
+		intel_de_write_fw(i915, GMBUS5(i915), gmbus5);
 
 	if (msgs[1].flags & I2C_M_RD)
-		ret = gmbus_xfer_read(dev_priv, &msgs[1], gmbus0_reg,
+		ret = gmbus_xfer_read(i915, &msgs[1], gmbus0_reg,
 				      gmbus1_index);
 	else
-		ret = gmbus_xfer_write(dev_priv, &msgs[1], gmbus1_index);
+		ret = gmbus_xfer_write(i915, &msgs[1], gmbus1_index);
 
 	/* Clear GMBUS5 after each index transfer */
 	if (gmbus5)
-		intel_de_write_fw(dev_priv, GMBUS5, 0);
+		intel_de_write_fw(i915, GMBUS5(i915), 0);
 
 	return ret;
 }
@@ -616,34 +632,34 @@ do_gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num,
 	      u32 gmbus0_source)
 {
 	struct intel_gmbus *bus = to_intel_gmbus(adapter);
-	struct drm_i915_private *dev_priv = bus->dev_priv;
+	struct drm_i915_private *i915 = bus->i915;
 	int i = 0, inc, try = 0;
 	int ret = 0;
 
 	/* Display WA #0868: skl,bxt,kbl,cfl,glk */
-	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
-		bxt_gmbus_clock_gating(dev_priv, false);
-	else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_CNP(dev_priv))
-		pch_gmbus_clock_gating(dev_priv, false);
+	if (IS_GEMINILAKE(i915) || IS_BROXTON(i915))
+		bxt_gmbus_clock_gating(i915, false);
+	else if (HAS_PCH_SPT(i915) || HAS_PCH_CNP(i915))
+		pch_gmbus_clock_gating(i915, false);
 
 retry:
-	intel_de_write_fw(dev_priv, GMBUS0, gmbus0_source | bus->reg0);
+	intel_de_write_fw(i915, GMBUS0(i915), gmbus0_source | bus->reg0);
 
 	for (; i < num; i += inc) {
 		inc = 1;
 		if (gmbus_is_index_xfer(msgs, i, num)) {
-			ret = gmbus_index_xfer(dev_priv, &msgs[i],
+			ret = gmbus_index_xfer(i915, &msgs[i],
 					       gmbus0_source | bus->reg0);
 			inc = 2; /* an index transmission is two msgs */
 		} else if (msgs[i].flags & I2C_M_RD) {
-			ret = gmbus_xfer_read(dev_priv, &msgs[i],
+			ret = gmbus_xfer_read(i915, &msgs[i],
 					      gmbus0_source | bus->reg0, 0);
 		} else {
-			ret = gmbus_xfer_write(dev_priv, &msgs[i], 0);
+			ret = gmbus_xfer_write(i915, &msgs[i], 0);
 		}
 
 		if (!ret)
-			ret = gmbus_wait(dev_priv,
+			ret = gmbus_wait(i915,
 					 GMBUS_HW_WAIT_PHASE, GMBUS_HW_WAIT_EN);
 		if (ret == -ETIMEDOUT)
 			goto timeout;
@@ -655,19 +671,19 @@ retry:
 	 * a STOP on the very first cycle. To simplify the code we
 	 * unconditionally generate the STOP condition with an additional gmbus
 	 * cycle. */
-	intel_de_write_fw(dev_priv, GMBUS1, GMBUS_CYCLE_STOP | GMBUS_SW_RDY);
+	intel_de_write_fw(i915, GMBUS1(i915), GMBUS_CYCLE_STOP | GMBUS_SW_RDY);
 
 	/* Mark the GMBUS interface as disabled after waiting for idle.
 	 * We will re-enable it at the start of the next xfer,
 	 * till then let it sleep.
 	 */
-	if (gmbus_wait_idle(dev_priv)) {
-		drm_dbg_kms(&dev_priv->drm,
+	if (gmbus_wait_idle(i915)) {
+		drm_dbg_kms(&i915->drm,
 			    "GMBUS [%s] timed out waiting for idle\n",
 			    adapter->name);
 		ret = -ETIMEDOUT;
 	}
-	intel_de_write_fw(dev_priv, GMBUS0, 0);
+	intel_de_write_fw(i915, GMBUS0(i915), 0);
 	ret = ret ?: i;
 	goto out;
 
@@ -686,8 +702,8 @@ clear_err:
 	 * it's slow responding and only answers on the 2nd retry.
 	 */
 	ret = -ENXIO;
-	if (gmbus_wait_idle(dev_priv)) {
-		drm_dbg_kms(&dev_priv->drm,
+	if (gmbus_wait_idle(i915)) {
+		drm_dbg_kms(&i915->drm,
 			    "GMBUS [%s] timed out after NAK\n",
 			    adapter->name);
 		ret = -ETIMEDOUT;
@@ -697,11 +713,11 @@ clear_err:
 	 * of resetting the GMBUS controller and so clearing the
 	 * BUS_ERROR raised by the slave's NAK.
 	 */
-	intel_de_write_fw(dev_priv, GMBUS1, GMBUS_SW_CLR_INT);
-	intel_de_write_fw(dev_priv, GMBUS1, 0);
-	intel_de_write_fw(dev_priv, GMBUS0, 0);
+	intel_de_write_fw(i915, GMBUS1(i915), GMBUS_SW_CLR_INT);
+	intel_de_write_fw(i915, GMBUS1(i915), 0);
+	intel_de_write_fw(i915, GMBUS0(i915), 0);
 
-	drm_dbg_kms(&dev_priv->drm, "GMBUS [%s] NAK for addr: %04x %c(%d)\n",
+	drm_dbg_kms(&i915->drm, "GMBUS [%s] NAK for addr: %04x %c(%d)\n",
 		    adapter->name, msgs[i].addr,
 		    (msgs[i].flags & I2C_M_RD) ? 'r' : 'w', msgs[i].len);
 
@@ -712,7 +728,7 @@ clear_err:
 	 * drm_do_probe_ddc_edid, which bails out on the first -ENXIO.
 	 */
 	if (ret == -ENXIO && i == 0 && try++ == 0) {
-		drm_dbg_kms(&dev_priv->drm,
+		drm_dbg_kms(&i915->drm,
 			    "GMBUS [%s] NAK on first message, retry\n",
 			    adapter->name);
 		goto retry;
@@ -721,10 +737,10 @@ clear_err:
 	goto out;
 
 timeout:
-	drm_dbg_kms(&dev_priv->drm,
+	drm_dbg_kms(&i915->drm,
 		    "GMBUS [%s] timed out, falling back to bit banging on pin %d\n",
 		    bus->adapter.name, bus->reg0 & 0xff);
-	intel_de_write_fw(dev_priv, GMBUS0, 0);
+	intel_de_write_fw(i915, GMBUS0(i915), 0);
 
 	/*
 	 * Hardware may not support GMBUS over these pins? Try GPIO bitbanging
@@ -734,10 +750,10 @@ timeout:
 
 out:
 	/* Display WA #0868: skl,bxt,kbl,cfl,glk */
-	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
-		bxt_gmbus_clock_gating(dev_priv, true);
-	else if (HAS_PCH_SPT(dev_priv) || HAS_PCH_CNP(dev_priv))
-		pch_gmbus_clock_gating(dev_priv, true);
+	if (IS_GEMINILAKE(i915) || IS_BROXTON(i915))
+		bxt_gmbus_clock_gating(i915, true);
+	else if (HAS_PCH_SPT(i915) || HAS_PCH_CNP(i915))
+		pch_gmbus_clock_gating(i915, true);
 
 	return ret;
 }
@@ -746,11 +762,11 @@ static int
 gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 {
 	struct intel_gmbus *bus = to_intel_gmbus(adapter);
-	struct drm_i915_private *dev_priv = bus->dev_priv;
+	struct drm_i915_private *i915 = bus->i915;
 	intel_wakeref_t wakeref;
 	int ret;
 
-	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
+	wakeref = intel_display_power_get(i915, POWER_DOMAIN_GMBUS);
 
 	if (bus->force_bit) {
 		ret = i2c_bit_algo.master_xfer(adapter, msgs, num);
@@ -762,7 +778,7 @@ gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 			bus->force_bit |= GMBUS_FORCE_BIT_RETRY;
 	}
 
-	intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS, wakeref);
+	intel_display_power_put(i915, POWER_DOMAIN_GMBUS, wakeref);
 
 	return ret;
 }
@@ -770,7 +786,7 @@ gmbus_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs, int num)
 int intel_gmbus_output_aksv(struct i2c_adapter *adapter)
 {
 	struct intel_gmbus *bus = to_intel_gmbus(adapter);
-	struct drm_i915_private *dev_priv = bus->dev_priv;
+	struct drm_i915_private *i915 = bus->i915;
 	u8 cmd = DRM_HDCP_DDC_AKSV;
 	u8 buf[DRM_HDCP_KSV_LEN] = { 0 };
 	struct i2c_msg msgs[] = {
@@ -790,8 +806,8 @@ int intel_gmbus_output_aksv(struct i2c_adapter *adapter)
 	intel_wakeref_t wakeref;
 	int ret;
 
-	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_GMBUS);
-	mutex_lock(&dev_priv->gmbus_mutex);
+	wakeref = intel_display_power_get(i915, POWER_DOMAIN_GMBUS);
+	mutex_lock(&i915->display.gmbus.mutex);
 
 	/*
 	 * In order to output Aksv to the receiver, use an indexed write to
@@ -800,8 +816,8 @@ int intel_gmbus_output_aksv(struct i2c_adapter *adapter)
 	 */
 	ret = do_gmbus_xfer(adapter, msgs, ARRAY_SIZE(msgs), GMBUS_AKSV_SELECT);
 
-	mutex_unlock(&dev_priv->gmbus_mutex);
-	intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS, wakeref);
+	mutex_unlock(&i915->display.gmbus.mutex);
+	intel_display_power_put(i915, POWER_DOMAIN_GMBUS, wakeref);
 
 	return ret;
 }
@@ -824,27 +840,27 @@ static void gmbus_lock_bus(struct i2c_adapter *adapter,
 			   unsigned int flags)
 {
 	struct intel_gmbus *bus = to_intel_gmbus(adapter);
-	struct drm_i915_private *dev_priv = bus->dev_priv;
+	struct drm_i915_private *i915 = bus->i915;
 
-	mutex_lock(&dev_priv->gmbus_mutex);
+	mutex_lock(&i915->display.gmbus.mutex);
 }
 
 static int gmbus_trylock_bus(struct i2c_adapter *adapter,
 			     unsigned int flags)
 {
 	struct intel_gmbus *bus = to_intel_gmbus(adapter);
-	struct drm_i915_private *dev_priv = bus->dev_priv;
+	struct drm_i915_private *i915 = bus->i915;
 
-	return mutex_trylock(&dev_priv->gmbus_mutex);
+	return mutex_trylock(&i915->display.gmbus.mutex);
 }
 
 static void gmbus_unlock_bus(struct i2c_adapter *adapter,
 			     unsigned int flags)
 {
 	struct intel_gmbus *bus = to_intel_gmbus(adapter);
-	struct drm_i915_private *dev_priv = bus->dev_priv;
+	struct drm_i915_private *i915 = bus->i915;
 
-	mutex_unlock(&dev_priv->gmbus_mutex);
+	mutex_unlock(&i915->display.gmbus.mutex);
 }
 
 static const struct i2c_lock_operations gmbus_lock_ops = {
@@ -855,31 +871,31 @@ static const struct i2c_lock_operations gmbus_lock_ops = {
 
 /**
  * intel_gmbus_setup - instantiate all Intel i2c GMBuses
- * @dev_priv: i915 device private
+ * @i915: i915 device private
  */
-int intel_gmbus_setup(struct drm_i915_private *dev_priv)
+int intel_gmbus_setup(struct drm_i915_private *i915)
 {
-	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
+	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 	unsigned int pin;
 	int ret;
 
-	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		dev_priv->gpio_mmio_base = VLV_DISPLAY_BASE;
-	else if (!HAS_GMCH(dev_priv))
+	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
+		i915->display.gmbus.mmio_base = VLV_DISPLAY_BASE;
+	else if (!HAS_GMCH(i915))
 		/*
 		 * Broxton uses the same PCH offsets for South Display Engine,
 		 * even though it doesn't have a PCH.
 		 */
-		dev_priv->gpio_mmio_base = PCH_DISPLAY_BASE;
+		i915->display.gmbus.mmio_base = PCH_DISPLAY_BASE;
 
-	mutex_init(&dev_priv->gmbus_mutex);
-	init_waitqueue_head(&dev_priv->gmbus_wait_queue);
+	mutex_init(&i915->display.gmbus.mutex);
+	init_waitqueue_head(&i915->display.gmbus.wait_queue);
 
-	for (pin = 0; pin < ARRAY_SIZE(dev_priv->gmbus); pin++) {
+	for (pin = 0; pin < ARRAY_SIZE(i915->display.gmbus.bus); pin++) {
 		const struct gmbus_pin *gmbus_pin;
 		struct intel_gmbus *bus;
 
-		gmbus_pin = get_gmbus_pin(dev_priv, pin);
+		gmbus_pin = get_gmbus_pin(i915, pin);
 		if (!gmbus_pin)
 			continue;
 
@@ -896,7 +912,7 @@ int intel_gmbus_setup(struct drm_i915_private *dev_priv)
 			 "i915 gmbus %s", gmbus_pin->name);
 
 		bus->adapter.dev.parent = &pdev->dev;
-		bus->dev_priv = dev_priv;
+		bus->i915 = i915;
 
 		bus->adapter.algo = &gmbus_algorithm;
 		bus->adapter.lock_ops = &gmbus_lock_ops;
@@ -911,10 +927,10 @@ int intel_gmbus_setup(struct drm_i915_private *dev_priv)
 		bus->reg0 = pin | GMBUS_RATE_100KHZ;
 
 		/* gmbus seems to be broken on i830 */
-		if (IS_I830(dev_priv))
+		if (IS_I830(i915))
 			bus->force_bit = 1;
 
-		intel_gpio_setup(bus, GPIO(gmbus_pin->gpio));
+		intel_gpio_setup(bus, GPIO(i915, gmbus_pin->gpio));
 
 		ret = i2c_add_adapter(&bus->adapter);
 		if (ret) {
@@ -922,43 +938,43 @@ int intel_gmbus_setup(struct drm_i915_private *dev_priv)
 			goto err;
 		}
 
-		dev_priv->gmbus[pin] = bus;
+		i915->display.gmbus.bus[pin] = bus;
 	}
 
-	intel_gmbus_reset(dev_priv);
+	intel_gmbus_reset(i915);
 
 	return 0;
 
 err:
-	intel_gmbus_teardown(dev_priv);
+	intel_gmbus_teardown(i915);
 
 	return ret;
 }
 
-struct i2c_adapter *intel_gmbus_get_adapter(struct drm_i915_private *dev_priv,
+struct i2c_adapter *intel_gmbus_get_adapter(struct drm_i915_private *i915,
 					    unsigned int pin)
 {
-	if (drm_WARN_ON(&dev_priv->drm, pin >= ARRAY_SIZE(dev_priv->gmbus) ||
-			!dev_priv->gmbus[pin]))
+	if (drm_WARN_ON(&i915->drm, pin >= ARRAY_SIZE(i915->display.gmbus.bus) ||
+			!i915->display.gmbus.bus[pin]))
 		return NULL;
 
-	return &dev_priv->gmbus[pin]->adapter;
+	return &i915->display.gmbus.bus[pin]->adapter;
 }
 
 void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit)
 {
 	struct intel_gmbus *bus = to_intel_gmbus(adapter);
-	struct drm_i915_private *dev_priv = bus->dev_priv;
+	struct drm_i915_private *i915 = bus->i915;
 
-	mutex_lock(&dev_priv->gmbus_mutex);
+	mutex_lock(&i915->display.gmbus.mutex);
 
 	bus->force_bit += force_bit ? 1 : -1;
-	drm_dbg_kms(&dev_priv->drm,
+	drm_dbg_kms(&i915->drm,
 		    "%sabling bit-banging on %s. force bit now %d\n",
 		    force_bit ? "en" : "dis", adapter->name,
 		    bus->force_bit);
 
-	mutex_unlock(&dev_priv->gmbus_mutex);
+	mutex_unlock(&i915->display.gmbus.mutex);
 }
 
 bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter)
@@ -968,20 +984,20 @@ bool intel_gmbus_is_forced_bit(struct i2c_adapter *adapter)
 	return bus->force_bit;
 }
 
-void intel_gmbus_teardown(struct drm_i915_private *dev_priv)
+void intel_gmbus_teardown(struct drm_i915_private *i915)
 {
 	unsigned int pin;
 
-	for (pin = 0; pin < ARRAY_SIZE(dev_priv->gmbus); pin++) {
+	for (pin = 0; pin < ARRAY_SIZE(i915->display.gmbus.bus); pin++) {
 		struct intel_gmbus *bus;
 
-		bus = dev_priv->gmbus[pin];
+		bus = i915->display.gmbus.bus[pin];
 		if (!bus)
 			continue;
 
 		i2c_del_adapter(&bus->adapter);
 
 		kfree(bus);
-		dev_priv->gmbus[pin] = NULL;
+		i915->display.gmbus.bus[pin] = NULL;
 	}
 }
diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.h b/drivers/gpu/drm/i915/display/intel_gmbus.h
index 8edc2e99cf53..20f704bd4e70 100644
--- a/drivers/gpu/drm/i915/display/intel_gmbus.h
+++ b/drivers/gpu/drm/i915/display/intel_gmbus.h
@@ -24,6 +24,7 @@ struct i2c_adapter;
 #define GMBUS_PIN_2_BXT		2
 #define GMBUS_PIN_3_BXT		3
 #define GMBUS_PIN_4_CNP		4
+#define GMBUS_PIN_5_MTP		5
 #define GMBUS_PIN_9_TC1_ICP	9
 #define GMBUS_PIN_10_TC2_ICP	10
 #define GMBUS_PIN_11_TC3_ICP	11
diff --git a/drivers/gpu/drm/i915/display/intel_gmbus_regs.h b/drivers/gpu/drm/i915/display/intel_gmbus_regs.h
new file mode 100644
index 000000000000..53aacbda983c
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_gmbus_regs.h
@@ -0,0 +1,81 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#ifndef __INTEL_GMBUS_REGS_H__
+#define __INTEL_GMBUS_REGS_H__
+
+#include "i915_reg_defs.h"
+
+#define GMBUS_MMIO_BASE(__i915) ((__i915)->display.gmbus.mmio_base)
+
+#define GPIO(__i915, gpio)	_MMIO(GMBUS_MMIO_BASE(__i915) + 0x5010 + 4 * (gpio))
+#define   GPIO_CLOCK_DIR_MASK		(1 << 0)
+#define   GPIO_CLOCK_DIR_IN		(0 << 1)
+#define   GPIO_CLOCK_DIR_OUT		(1 << 1)
+#define   GPIO_CLOCK_VAL_MASK		(1 << 2)
+#define   GPIO_CLOCK_VAL_OUT		(1 << 3)
+#define   GPIO_CLOCK_VAL_IN		(1 << 4)
+#define   GPIO_CLOCK_PULLUP_DISABLE	(1 << 5)
+#define   GPIO_DATA_DIR_MASK		(1 << 8)
+#define   GPIO_DATA_DIR_IN		(0 << 9)
+#define   GPIO_DATA_DIR_OUT		(1 << 9)
+#define   GPIO_DATA_VAL_MASK		(1 << 10)
+#define   GPIO_DATA_VAL_OUT		(1 << 11)
+#define   GPIO_DATA_VAL_IN		(1 << 12)
+#define   GPIO_DATA_PULLUP_DISABLE	(1 << 13)
+
+/* clock/port select */
+#define GMBUS0(__i915)		_MMIO(GMBUS_MMIO_BASE(__i915) + 0x5100)
+#define   GMBUS_AKSV_SELECT		(1 << 11)
+#define   GMBUS_RATE_100KHZ		(0 << 8)
+#define   GMBUS_RATE_50KHZ		(1 << 8)
+#define   GMBUS_RATE_400KHZ		(2 << 8) /* reserved on Pineview */
+#define   GMBUS_RATE_1MHZ		(3 << 8) /* reserved on Pineview */
+#define   GMBUS_HOLD_EXT		(1 << 7) /* 300ns hold time, rsvd on Pineview */
+#define   GMBUS_BYTE_CNT_OVERRIDE	(1 << 6)
+
+/* command/status */
+#define GMBUS1(__i915)		_MMIO(GMBUS_MMIO_BASE(__i915) + 0x5104)
+#define   GMBUS_SW_CLR_INT		(1 << 31)
+#define   GMBUS_SW_RDY			(1 << 30)
+#define   GMBUS_ENT			(1 << 29) /* enable timeout */
+#define   GMBUS_CYCLE_NONE		(0 << 25)
+#define   GMBUS_CYCLE_WAIT		(1 << 25)
+#define   GMBUS_CYCLE_INDEX		(2 << 25)
+#define   GMBUS_CYCLE_STOP		(4 << 25)
+#define   GMBUS_BYTE_COUNT_SHIFT	16
+#define   GMBUS_BYTE_COUNT_MAX		256U
+#define   GEN9_GMBUS_BYTE_COUNT_MAX	511U
+#define   GMBUS_SLAVE_INDEX_SHIFT	8
+#define   GMBUS_SLAVE_ADDR_SHIFT	1
+#define   GMBUS_SLAVE_READ		(1 << 0)
+#define   GMBUS_SLAVE_WRITE		(0 << 0)
+
+/* status */
+#define GMBUS2(__i915)		_MMIO(GMBUS_MMIO_BASE(__i915) + 0x5108)
+#define   GMBUS_INUSE			(1 << 15)
+#define   GMBUS_HW_WAIT_PHASE		(1 << 14)
+#define   GMBUS_STALL_TIMEOUT		(1 << 13)
+#define   GMBUS_INT			(1 << 12)
+#define   GMBUS_HW_RDY			(1 << 11)
+#define   GMBUS_SATOER			(1 << 10)
+#define   GMBUS_ACTIVE			(1 << 9)
+
+/* data buffer bytes 3-0 */
+#define GMBUS3(__i915)		_MMIO(GMBUS_MMIO_BASE(__i915) + 0x510c)
+
+/* interrupt mask (Pineview+) */
+#define GMBUS4(__i915)		_MMIO(GMBUS_MMIO_BASE(__i915) + 0x5110)
+#define   GMBUS_SLAVE_TIMEOUT_EN	(1 << 4)
+#define   GMBUS_NAK_EN			(1 << 3)
+#define   GMBUS_IDLE_EN			(1 << 2)
+#define   GMBUS_HW_WAIT_EN		(1 << 1)
+#define   GMBUS_HW_RDY_EN		(1 << 0)
+
+/* byte index */
+#define GMBUS5(__i915)		_MMIO(GMBUS_MMIO_BASE(__i915) + 0x5120)
+#define   GMBUS_2BYTE_INDEX_EN		(1 << 31)
+
+#endif /* __INTEL_GMBUS_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c
index 8ea66a2e1b09..6406fd487ee5 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp.c
+++ b/drivers/gpu/drm/i915/display/intel_hdcp.c
@@ -23,6 +23,7 @@
 #include "intel_display_power_well.h"
 #include "intel_display_types.h"
 #include "intel_hdcp.h"
+#include "intel_hdcp_regs.h"
 #include "intel_pcode.h"
 
 #define KEY_LOAD_TRIES	5
@@ -30,8 +31,30 @@
 
 static int intel_conn_to_vcpi(struct intel_connector *connector)
 {
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_atomic_payload *payload;
+	struct drm_dp_mst_topology_state *mst_state;
+	int vcpi = 0;
+
 	/* For HDMI this is forced to be 0x0. For DP SST also this is 0x0. */
-	return connector->port	? connector->port->vcpi.vcpi : 0;
+	if (!connector->port)
+		return 0;
+	mgr = connector->port->mgr;
+
+	drm_modeset_lock(&mgr->base.lock, NULL);
+	mst_state = to_drm_dp_mst_topology_state(mgr->base.state);
+	payload = drm_atomic_get_mst_payload_state(mst_state, connector->port);
+	if (drm_WARN_ON(mgr->dev, !payload))
+		goto out;
+
+	vcpi = payload->vcpi;
+	if (drm_WARN_ON(mgr->dev, vcpi < 0)) {
+		vcpi = 0;
+		goto out;
+	}
+out:
+	drm_modeset_unlock(&mgr->base.lock);
+	return vcpi;
 }
 
 /*
@@ -187,12 +210,12 @@ bool intel_hdcp2_capable(struct intel_connector *connector)
 		return false;
 
 	/* MEI interface is solid */
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	if (!dev_priv->hdcp_comp_added ||  !dev_priv->hdcp_master) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	if (!dev_priv->display.hdcp.comp_added ||  !dev_priv->display.hdcp.master) {
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return false;
 	}
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	/* Sink's capability for HDCP2.2 */
 	hdcp->shim->hdcp_2_2_capable(dig_port, &capable);
@@ -1109,8 +1132,8 @@ static void intel_hdcp_prop_work(struct work_struct *work)
 
 bool is_hdcp_supported(struct drm_i915_private *dev_priv, enum port port)
 {
-	return INTEL_INFO(dev_priv)->display.has_hdcp &&
-			(DISPLAY_VER(dev_priv) >= 12 || port < PORT_E);
+	return RUNTIME_INFO(dev_priv)->has_hdcp &&
+		(DISPLAY_VER(dev_priv) >= 12 || port < PORT_E);
 }
 
 static int
@@ -1123,11 +1146,11 @@ hdcp2_prepare_ake_init(struct intel_connector *connector,
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
@@ -1135,7 +1158,7 @@ hdcp2_prepare_ake_init(struct intel_connector *connector,
 	if (ret)
 		drm_dbg_kms(&dev_priv->drm, "Prepare_ake_init failed. %d\n",
 			    ret);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -1153,11 +1176,11 @@ hdcp2_verify_rx_cert_prepare_km(struct intel_connector *connector,
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
@@ -1167,7 +1190,7 @@ hdcp2_verify_rx_cert_prepare_km(struct intel_connector *connector,
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Verify rx_cert failed. %d\n",
 			    ret);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -1181,18 +1204,18 @@ static int hdcp2_verify_hprime(struct intel_connector *connector,
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
 	ret = comp->ops->verify_hprime(comp->mei_dev, data, rx_hprime);
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Verify hprime failed. %d\n", ret);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -1207,11 +1230,11 @@ hdcp2_store_pairing_info(struct intel_connector *connector,
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
@@ -1219,7 +1242,7 @@ hdcp2_store_pairing_info(struct intel_connector *connector,
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Store pairing info failed. %d\n",
 			    ret);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -1234,11 +1257,11 @@ hdcp2_prepare_lc_init(struct intel_connector *connector,
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
@@ -1246,7 +1269,7 @@ hdcp2_prepare_lc_init(struct intel_connector *connector,
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Prepare lc_init failed. %d\n",
 			    ret);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -1261,11 +1284,11 @@ hdcp2_verify_lprime(struct intel_connector *connector,
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
@@ -1273,7 +1296,7 @@ hdcp2_verify_lprime(struct intel_connector *connector,
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Verify L_Prime failed. %d\n",
 			    ret);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -1287,11 +1310,11 @@ static int hdcp2_prepare_skey(struct intel_connector *connector,
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
@@ -1299,7 +1322,7 @@ static int hdcp2_prepare_skey(struct intel_connector *connector,
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Get session key failed. %d\n",
 			    ret);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -1316,11 +1339,11 @@ hdcp2_verify_rep_topology_prepare_ack(struct intel_connector *connector,
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
@@ -1330,7 +1353,7 @@ hdcp2_verify_rep_topology_prepare_ack(struct intel_connector *connector,
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm,
 			    "Verify rep topology failed. %d\n", ret);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -1345,18 +1368,18 @@ hdcp2_verify_mprime(struct intel_connector *connector,
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
 	ret = comp->ops->verify_mprime(comp->mei_dev, data, stream_ready);
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Verify mprime failed. %d\n", ret);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -1369,11 +1392,11 @@ static int hdcp2_authenticate_port(struct intel_connector *connector)
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
@@ -1381,7 +1404,7 @@ static int hdcp2_authenticate_port(struct intel_connector *connector)
 	if (ret < 0)
 		drm_dbg_kms(&dev_priv->drm, "Enable hdcp auth failed. %d\n",
 			    ret);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -1393,17 +1416,17 @@ static int hdcp2_close_mei_session(struct intel_connector *connector)
 	struct i915_hdcp_comp_master *comp;
 	int ret;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	comp = dev_priv->hdcp_master;
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	comp = dev_priv->display.hdcp.master;
 
 	if (!comp || !comp->ops) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return -EINVAL;
 	}
 
 	ret = comp->ops->close_hdcp_session(comp->mei_dev,
 					     &dig_port->hdcp_port_data);
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return ret;
 }
@@ -2121,10 +2144,10 @@ static int i915_hdcp_component_bind(struct device *i915_kdev,
 	struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev);
 
 	drm_dbg(&dev_priv->drm, "I915 HDCP comp bind\n");
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	dev_priv->hdcp_master = (struct i915_hdcp_comp_master *)data;
-	dev_priv->hdcp_master->mei_dev = mei_kdev;
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	dev_priv->display.hdcp.master = (struct i915_hdcp_comp_master *)data;
+	dev_priv->display.hdcp.master->mei_dev = mei_kdev;
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	return 0;
 }
@@ -2135,9 +2158,9 @@ static void i915_hdcp_component_unbind(struct device *i915_kdev,
 	struct drm_i915_private *dev_priv = kdev_to_i915(i915_kdev);
 
 	drm_dbg(&dev_priv->drm, "I915 HDCP comp unbind\n");
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	dev_priv->hdcp_master = NULL;
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	dev_priv->display.hdcp.master = NULL;
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 }
 
 static const struct component_ops i915_hdcp_component_ops = {
@@ -2228,19 +2251,19 @@ void intel_hdcp_component_init(struct drm_i915_private *dev_priv)
 	if (!is_hdcp2_supported(dev_priv))
 		return;
 
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	drm_WARN_ON(&dev_priv->drm, dev_priv->hdcp_comp_added);
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	drm_WARN_ON(&dev_priv->drm, dev_priv->display.hdcp.comp_added);
 
-	dev_priv->hdcp_comp_added = true;
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	dev_priv->display.hdcp.comp_added = true;
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 	ret = component_add_typed(dev_priv->drm.dev, &i915_hdcp_component_ops,
 				  I915_COMPONENT_HDCP);
 	if (ret < 0) {
 		drm_dbg_kms(&dev_priv->drm, "Failed at component add(%d)\n",
 			    ret);
-		mutex_lock(&dev_priv->hdcp_comp_mutex);
-		dev_priv->hdcp_comp_added = false;
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+		mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+		dev_priv->display.hdcp.comp_added = false;
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return;
 	}
 }
@@ -2453,14 +2476,14 @@ void intel_hdcp_update_pipe(struct intel_atomic_state *state,
 
 void intel_hdcp_component_fini(struct drm_i915_private *dev_priv)
 {
-	mutex_lock(&dev_priv->hdcp_comp_mutex);
-	if (!dev_priv->hdcp_comp_added) {
-		mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	mutex_lock(&dev_priv->display.hdcp.comp_mutex);
+	if (!dev_priv->display.hdcp.comp_added) {
+		mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 		return;
 	}
 
-	dev_priv->hdcp_comp_added = false;
-	mutex_unlock(&dev_priv->hdcp_comp_mutex);
+	dev_priv->display.hdcp.comp_added = false;
+	mutex_unlock(&dev_priv->display.hdcp.comp_mutex);
 
 	component_del(dev_priv->drm.dev, &i915_hdcp_component_ops);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp_regs.h b/drivers/gpu/drm/i915/display/intel_hdcp_regs.h
new file mode 100644
index 000000000000..2a3733e8966c
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_hdcp_regs.h
@@ -0,0 +1,270 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#ifndef __INTEL_HDCP_REGS_H__
+#define __INTEL_HDCP_REGS_H__
+
+#include "i915_reg_defs.h"
+
+/* HDCP Key Registers */
+#define HDCP_KEY_CONF			_MMIO(0x66c00)
+#define  HDCP_AKSV_SEND_TRIGGER		REG_BIT(31)
+#define  HDCP_CLEAR_KEYS_TRIGGER	REG_BIT(30)
+#define  HDCP_KEY_LOAD_TRIGGER		REG_BIT(8)
+#define HDCP_KEY_STATUS			_MMIO(0x66c04)
+#define  HDCP_FUSE_IN_PROGRESS		REG_BIT(7)
+#define  HDCP_FUSE_ERROR		REG_BIT(6)
+#define  HDCP_FUSE_DONE			REG_BIT(5)
+#define  HDCP_KEY_LOAD_STATUS		REG_BIT(1)
+#define  HDCP_KEY_LOAD_DONE		REG_BIT(0)
+#define HDCP_AKSV_LO			_MMIO(0x66c10)
+#define HDCP_AKSV_HI			_MMIO(0x66c14)
+
+/* HDCP Repeater Registers */
+#define HDCP_REP_CTL			_MMIO(0x66d00)
+#define  HDCP_TRANSA_REP_PRESENT	REG_BIT(31)
+#define  HDCP_TRANSB_REP_PRESENT	REG_BIT(30)
+#define  HDCP_TRANSC_REP_PRESENT	REG_BIT(29)
+#define  HDCP_TRANSD_REP_PRESENT	REG_BIT(28)
+#define  HDCP_DDIB_REP_PRESENT		REG_BIT(30)
+#define  HDCP_DDIA_REP_PRESENT		REG_BIT(29)
+#define  HDCP_DDIC_REP_PRESENT		REG_BIT(28)
+#define  HDCP_DDID_REP_PRESENT		REG_BIT(27)
+#define  HDCP_DDIF_REP_PRESENT		REG_BIT(26)
+#define  HDCP_DDIE_REP_PRESENT		REG_BIT(25)
+#define  HDCP_TRANSA_SHA1_M0		(1 << 20)
+#define  HDCP_TRANSB_SHA1_M0		(2 << 20)
+#define  HDCP_TRANSC_SHA1_M0		(3 << 20)
+#define  HDCP_TRANSD_SHA1_M0		(4 << 20)
+#define  HDCP_DDIB_SHA1_M0		(1 << 20)
+#define  HDCP_DDIA_SHA1_M0		(2 << 20)
+#define  HDCP_DDIC_SHA1_M0		(3 << 20)
+#define  HDCP_DDID_SHA1_M0		(4 << 20)
+#define  HDCP_DDIF_SHA1_M0		(5 << 20)
+#define  HDCP_DDIE_SHA1_M0		(6 << 20) /* Bspec says 5? */
+#define  HDCP_SHA1_BUSY			REG_BIT(16)
+#define  HDCP_SHA1_READY		REG_BIT(17)
+#define  HDCP_SHA1_COMPLETE		REG_BIT(18)
+#define  HDCP_SHA1_V_MATCH		REG_BIT(19)
+#define  HDCP_SHA1_TEXT_32		(1 << 1)
+#define  HDCP_SHA1_COMPLETE_HASH	(2 << 1)
+#define  HDCP_SHA1_TEXT_24		(4 << 1)
+#define  HDCP_SHA1_TEXT_16		(5 << 1)
+#define  HDCP_SHA1_TEXT_8		(6 << 1)
+#define  HDCP_SHA1_TEXT_0		(7 << 1)
+#define HDCP_SHA_V_PRIME_H0		_MMIO(0x66d04)
+#define HDCP_SHA_V_PRIME_H1		_MMIO(0x66d08)
+#define HDCP_SHA_V_PRIME_H2		_MMIO(0x66d0C)
+#define HDCP_SHA_V_PRIME_H3		_MMIO(0x66d10)
+#define HDCP_SHA_V_PRIME_H4		_MMIO(0x66d14)
+#define HDCP_SHA_V_PRIME(h)		_MMIO((0x66d04 + (h) * 4))
+#define HDCP_SHA_TEXT			_MMIO(0x66d18)
+
+/* HDCP Auth Registers */
+#define _PORTA_HDCP_AUTHENC		0x66800
+#define _PORTB_HDCP_AUTHENC		0x66500
+#define _PORTC_HDCP_AUTHENC		0x66600
+#define _PORTD_HDCP_AUTHENC		0x66700
+#define _PORTE_HDCP_AUTHENC		0x66A00
+#define _PORTF_HDCP_AUTHENC		0x66900
+#define _PORT_HDCP_AUTHENC(port, x)	_MMIO(_PICK(port, \
+					  _PORTA_HDCP_AUTHENC, \
+					  _PORTB_HDCP_AUTHENC, \
+					  _PORTC_HDCP_AUTHENC, \
+					  _PORTD_HDCP_AUTHENC, \
+					  _PORTE_HDCP_AUTHENC, \
+					  _PORTF_HDCP_AUTHENC) + (x))
+#define PORT_HDCP_CONF(port)		_PORT_HDCP_AUTHENC(port, 0x0)
+#define _TRANSA_HDCP_CONF		0x66400
+#define _TRANSB_HDCP_CONF		0x66500
+#define TRANS_HDCP_CONF(trans)		_MMIO_TRANS(trans, _TRANSA_HDCP_CONF, \
+						    _TRANSB_HDCP_CONF)
+#define HDCP_CONF(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP_CONF(trans) : \
+					 PORT_HDCP_CONF(port))
+
+#define  HDCP_CONF_CAPTURE_AN		REG_BIT(0)
+#define  HDCP_CONF_AUTH_AND_ENC		(REG_BIT(1) | REG_BIT(0))
+#define PORT_HDCP_ANINIT(port)		_PORT_HDCP_AUTHENC(port, 0x4)
+#define _TRANSA_HDCP_ANINIT		0x66404
+#define _TRANSB_HDCP_ANINIT		0x66504
+#define TRANS_HDCP_ANINIT(trans)	_MMIO_TRANS(trans, \
+						    _TRANSA_HDCP_ANINIT, \
+						    _TRANSB_HDCP_ANINIT)
+#define HDCP_ANINIT(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP_ANINIT(trans) : \
+					 PORT_HDCP_ANINIT(port))
+
+#define PORT_HDCP_ANLO(port)		_PORT_HDCP_AUTHENC(port, 0x8)
+#define _TRANSA_HDCP_ANLO		0x66408
+#define _TRANSB_HDCP_ANLO		0x66508
+#define TRANS_HDCP_ANLO(trans)		_MMIO_TRANS(trans, _TRANSA_HDCP_ANLO, \
+						    _TRANSB_HDCP_ANLO)
+#define HDCP_ANLO(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP_ANLO(trans) : \
+					 PORT_HDCP_ANLO(port))
+
+#define PORT_HDCP_ANHI(port)		_PORT_HDCP_AUTHENC(port, 0xC)
+#define _TRANSA_HDCP_ANHI		0x6640C
+#define _TRANSB_HDCP_ANHI		0x6650C
+#define TRANS_HDCP_ANHI(trans)		_MMIO_TRANS(trans, _TRANSA_HDCP_ANHI, \
+						    _TRANSB_HDCP_ANHI)
+#define HDCP_ANHI(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP_ANHI(trans) : \
+					 PORT_HDCP_ANHI(port))
+
+#define PORT_HDCP_BKSVLO(port)		_PORT_HDCP_AUTHENC(port, 0x10)
+#define _TRANSA_HDCP_BKSVLO		0x66410
+#define _TRANSB_HDCP_BKSVLO		0x66510
+#define TRANS_HDCP_BKSVLO(trans)	_MMIO_TRANS(trans, \
+						    _TRANSA_HDCP_BKSVLO, \
+						    _TRANSB_HDCP_BKSVLO)
+#define HDCP_BKSVLO(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP_BKSVLO(trans) : \
+					 PORT_HDCP_BKSVLO(port))
+
+#define PORT_HDCP_BKSVHI(port)		_PORT_HDCP_AUTHENC(port, 0x14)
+#define _TRANSA_HDCP_BKSVHI		0x66414
+#define _TRANSB_HDCP_BKSVHI		0x66514
+#define TRANS_HDCP_BKSVHI(trans)	_MMIO_TRANS(trans, \
+						    _TRANSA_HDCP_BKSVHI, \
+						    _TRANSB_HDCP_BKSVHI)
+#define HDCP_BKSVHI(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP_BKSVHI(trans) : \
+					 PORT_HDCP_BKSVHI(port))
+
+#define PORT_HDCP_RPRIME(port)		_PORT_HDCP_AUTHENC(port, 0x18)
+#define _TRANSA_HDCP_RPRIME		0x66418
+#define _TRANSB_HDCP_RPRIME		0x66518
+#define TRANS_HDCP_RPRIME(trans)	_MMIO_TRANS(trans, \
+						    _TRANSA_HDCP_RPRIME, \
+						    _TRANSB_HDCP_RPRIME)
+#define HDCP_RPRIME(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP_RPRIME(trans) : \
+					 PORT_HDCP_RPRIME(port))
+
+#define PORT_HDCP_STATUS(port)		_PORT_HDCP_AUTHENC(port, 0x1C)
+#define _TRANSA_HDCP_STATUS		0x6641C
+#define _TRANSB_HDCP_STATUS		0x6651C
+#define TRANS_HDCP_STATUS(trans)	_MMIO_TRANS(trans, \
+						    _TRANSA_HDCP_STATUS, \
+						    _TRANSB_HDCP_STATUS)
+#define HDCP_STATUS(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP_STATUS(trans) : \
+					 PORT_HDCP_STATUS(port))
+
+#define  HDCP_STATUS_STREAM_A_ENC	REG_BIT(31)
+#define  HDCP_STATUS_STREAM_B_ENC	REG_BIT(30)
+#define  HDCP_STATUS_STREAM_C_ENC	REG_BIT(29)
+#define  HDCP_STATUS_STREAM_D_ENC	REG_BIT(28)
+#define  HDCP_STATUS_AUTH		REG_BIT(21)
+#define  HDCP_STATUS_ENC		REG_BIT(20)
+#define  HDCP_STATUS_RI_MATCH		REG_BIT(19)
+#define  HDCP_STATUS_R0_READY		REG_BIT(18)
+#define  HDCP_STATUS_AN_READY		REG_BIT(17)
+#define  HDCP_STATUS_CIPHER		REG_BIT(16)
+#define  HDCP_STATUS_FRAME_CNT(x)	(((x) >> 8) & 0xff)
+
+/* HDCP2.2 Registers */
+#define _PORTA_HDCP2_BASE		0x66800
+#define _PORTB_HDCP2_BASE		0x66500
+#define _PORTC_HDCP2_BASE		0x66600
+#define _PORTD_HDCP2_BASE		0x66700
+#define _PORTE_HDCP2_BASE		0x66A00
+#define _PORTF_HDCP2_BASE		0x66900
+#define _PORT_HDCP2_BASE(port, x)	_MMIO(_PICK((port), \
+					  _PORTA_HDCP2_BASE, \
+					  _PORTB_HDCP2_BASE, \
+					  _PORTC_HDCP2_BASE, \
+					  _PORTD_HDCP2_BASE, \
+					  _PORTE_HDCP2_BASE, \
+					  _PORTF_HDCP2_BASE) + (x))
+
+#define PORT_HDCP2_AUTH(port)		_PORT_HDCP2_BASE(port, 0x98)
+#define _TRANSA_HDCP2_AUTH		0x66498
+#define _TRANSB_HDCP2_AUTH		0x66598
+#define TRANS_HDCP2_AUTH(trans)		_MMIO_TRANS(trans, _TRANSA_HDCP2_AUTH, \
+						    _TRANSB_HDCP2_AUTH)
+#define   AUTH_LINK_AUTHENTICATED	REG_BIT(31)
+#define   AUTH_LINK_TYPE		REG_BIT(30)
+#define   AUTH_FORCE_CLR_INPUTCTR	REG_BIT(19)
+#define   AUTH_CLR_KEYS			REG_BIT(18)
+#define HDCP2_AUTH(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP2_AUTH(trans) : \
+					 PORT_HDCP2_AUTH(port))
+
+#define PORT_HDCP2_CTL(port)		_PORT_HDCP2_BASE(port, 0xB0)
+#define _TRANSA_HDCP2_CTL		0x664B0
+#define _TRANSB_HDCP2_CTL		0x665B0
+#define TRANS_HDCP2_CTL(trans)		_MMIO_TRANS(trans, _TRANSA_HDCP2_CTL, \
+						    _TRANSB_HDCP2_CTL)
+#define   CTL_LINK_ENCRYPTION_REQ	REG_BIT(31)
+#define HDCP2_CTL(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP2_CTL(trans) : \
+					 PORT_HDCP2_CTL(port))
+
+#define PORT_HDCP2_STATUS(port)		_PORT_HDCP2_BASE(port, 0xB4)
+#define _TRANSA_HDCP2_STATUS		0x664B4
+#define _TRANSB_HDCP2_STATUS		0x665B4
+#define TRANS_HDCP2_STATUS(trans)	_MMIO_TRANS(trans, \
+						    _TRANSA_HDCP2_STATUS, \
+						    _TRANSB_HDCP2_STATUS)
+#define   LINK_TYPE_STATUS		REG_BIT(22)
+#define   LINK_AUTH_STATUS		REG_BIT(21)
+#define   LINK_ENCRYPTION_STATUS	REG_BIT(20)
+#define HDCP2_STATUS(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP2_STATUS(trans) : \
+					 PORT_HDCP2_STATUS(port))
+
+#define _PIPEA_HDCP2_STREAM_STATUS	0x668C0
+#define _PIPEB_HDCP2_STREAM_STATUS	0x665C0
+#define _PIPEC_HDCP2_STREAM_STATUS	0x666C0
+#define _PIPED_HDCP2_STREAM_STATUS	0x667C0
+#define PIPE_HDCP2_STREAM_STATUS(pipe)		_MMIO(_PICK((pipe), \
+						      _PIPEA_HDCP2_STREAM_STATUS, \
+						      _PIPEB_HDCP2_STREAM_STATUS, \
+						      _PIPEC_HDCP2_STREAM_STATUS, \
+						      _PIPED_HDCP2_STREAM_STATUS))
+
+#define _TRANSA_HDCP2_STREAM_STATUS		0x664C0
+#define _TRANSB_HDCP2_STREAM_STATUS		0x665C0
+#define TRANS_HDCP2_STREAM_STATUS(trans)	_MMIO_TRANS(trans, \
+						    _TRANSA_HDCP2_STREAM_STATUS, \
+						    _TRANSB_HDCP2_STREAM_STATUS)
+#define   STREAM_ENCRYPTION_STATUS	REG_BIT(31)
+#define   STREAM_TYPE_STATUS		REG_BIT(30)
+#define HDCP2_STREAM_STATUS(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP2_STREAM_STATUS(trans) : \
+					 PIPE_HDCP2_STREAM_STATUS(pipe))
+
+#define _PORTA_HDCP2_AUTH_STREAM		0x66F00
+#define _PORTB_HDCP2_AUTH_STREAM		0x66F04
+#define PORT_HDCP2_AUTH_STREAM(port)	_MMIO_PORT(port, \
+						   _PORTA_HDCP2_AUTH_STREAM, \
+						   _PORTB_HDCP2_AUTH_STREAM)
+#define _TRANSA_HDCP2_AUTH_STREAM		0x66F00
+#define _TRANSB_HDCP2_AUTH_STREAM		0x66F04
+#define TRANS_HDCP2_AUTH_STREAM(trans)	_MMIO_TRANS(trans, \
+						    _TRANSA_HDCP2_AUTH_STREAM, \
+						    _TRANSB_HDCP2_AUTH_STREAM)
+#define   AUTH_STREAM_TYPE		REG_BIT(31)
+#define HDCP2_AUTH_STREAM(dev_priv, trans, port) \
+					(GRAPHICS_VER(dev_priv) >= 12 ? \
+					 TRANS_HDCP2_AUTH_STREAM(trans) : \
+					 PORT_HDCP2_AUTH_STREAM(port))
+
+#endif /* __INTEL_HDCP_REGS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index ebd91aa69dd2..7816b2a33fee 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -50,6 +50,7 @@
 #include "intel_dp.h"
 #include "intel_gmbus.h"
 #include "intel_hdcp.h"
+#include "intel_hdcp_regs.h"
 #include "intel_hdmi.h"
 #include "intel_lspcon.h"
 #include "intel_panel.h"
@@ -1891,7 +1892,7 @@ int intel_hdmi_tmds_clock(int clock, int bpc, bool ycbcr420_output)
 	 *  1.5x for 12bpc
 	 *  1.25x for 10bpc
 	 */
-	return clock * bpc / 8;
+	return DIV_ROUND_CLOSEST(clock * bpc, 8);
 }
 
 static bool intel_hdmi_source_bpc_possible(struct drm_i915_private *i915, int bpc)
@@ -2001,6 +2002,15 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
 		clock *= 2;
 	}
 
+	/*
+	 * HDMI2.1 requires higher resolution modes like 8k60, 4K120 to be
+	 * enumerated only if FRL is supported. Current platforms do not support
+	 * FRL so prune the higher resolution modes that require doctclock more
+	 * than 600MHz.
+	 */
+	if (clock > 600000)
+		return MODE_CLOCK_HIGH;
+
 	ycbcr_420_only = drm_mode_is_420_only(&connector->display_info, mode);
 
 	status = intel_hdmi_mode_clock_valid(connector, clock, has_hdmi_sink, ycbcr_420_only);
diff --git a/drivers/gpu/drm/i915/display/intel_hotplug.c b/drivers/gpu/drm/i915/display/intel_hotplug.c
index 5f8b4f481cff..f7a2f485b177 100644
--- a/drivers/gpu/drm/i915/display/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/display/intel_hotplug.c
@@ -119,13 +119,13 @@ intel_connector_hpd_pin(struct intel_connector *connector)
  * responsible for further action.
  *
  * The number of IRQs that are allowed within @HPD_STORM_DETECT_PERIOD is
- * stored in @dev_priv->hotplug.hpd_storm_threshold which defaults to
+ * stored in @dev_priv->display.hotplug.hpd_storm_threshold which defaults to
  * @HPD_STORM_DEFAULT_THRESHOLD. Long IRQs count as +10 to this threshold, and
  * short IRQs count as +1. If this threshold is exceeded, it's considered an
  * IRQ storm and the IRQ state is set to @HPD_MARK_DISABLED.
  *
  * By default, most systems will only count long IRQs towards
- * &dev_priv->hotplug.hpd_storm_threshold. However, some older systems also
+ * &dev_priv->display.hotplug.hpd_storm_threshold. However, some older systems also
  * suffer from short IRQ storms and must also track these. Because short IRQ
  * storms are naturally caused by sideband interactions with DP MST devices,
  * short IRQ detection is only enabled for systems without DP MST support.
@@ -140,7 +140,7 @@ intel_connector_hpd_pin(struct intel_connector *connector)
 static bool intel_hpd_irq_storm_detect(struct drm_i915_private *dev_priv,
 				       enum hpd_pin pin, bool long_hpd)
 {
-	struct i915_hotplug *hpd = &dev_priv->hotplug;
+	struct intel_hotplug *hpd = &dev_priv->display.hotplug;
 	unsigned long start = hpd->stats[pin].last_jiffies;
 	unsigned long end = start + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD);
 	const int increment = long_hpd ? 10 : 1;
@@ -148,7 +148,7 @@ static bool intel_hpd_irq_storm_detect(struct drm_i915_private *dev_priv,
 	bool storm = false;
 
 	if (!threshold ||
-	    (!long_hpd && !dev_priv->hotplug.hpd_short_storm_enabled))
+	    (!long_hpd && !dev_priv->display.hotplug.hpd_short_storm_enabled))
 		return false;
 
 	if (!time_in_range(jiffies, start, end)) {
@@ -191,7 +191,7 @@ intel_hpd_irq_storm_switch_to_polling(struct drm_i915_private *dev_priv)
 
 		pin = intel_connector_hpd_pin(connector);
 		if (pin == HPD_NONE ||
-		    dev_priv->hotplug.stats[pin].state != HPD_MARK_DISABLED)
+		    dev_priv->display.hotplug.stats[pin].state != HPD_MARK_DISABLED)
 			continue;
 
 		drm_info(&dev_priv->drm,
@@ -199,7 +199,7 @@ intel_hpd_irq_storm_switch_to_polling(struct drm_i915_private *dev_priv)
 			 "switching from hotplug detection to polling\n",
 			 connector->base.name);
 
-		dev_priv->hotplug.stats[pin].state = HPD_DISABLED;
+		dev_priv->display.hotplug.stats[pin].state = HPD_DISABLED;
 		connector->base.polled = DRM_CONNECTOR_POLL_CONNECT |
 			DRM_CONNECTOR_POLL_DISCONNECT;
 		hpd_disabled = true;
@@ -209,7 +209,7 @@ intel_hpd_irq_storm_switch_to_polling(struct drm_i915_private *dev_priv)
 	/* Enable polling and queue hotplug re-enabling. */
 	if (hpd_disabled) {
 		drm_kms_helper_poll_enable(dev);
-		mod_delayed_work(system_wq, &dev_priv->hotplug.reenable_work,
+		mod_delayed_work(system_wq, &dev_priv->display.hotplug.reenable_work,
 				 msecs_to_jiffies(HPD_STORM_REENABLE_DELAY));
 	}
 }
@@ -218,7 +218,7 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work)
 {
 	struct drm_i915_private *dev_priv =
 		container_of(work, typeof(*dev_priv),
-			     hotplug.reenable_work.work);
+			     display.hotplug.reenable_work.work);
 	struct drm_device *dev = &dev_priv->drm;
 	struct drm_connector_list_iter conn_iter;
 	struct intel_connector *connector;
@@ -233,7 +233,7 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work)
 	for_each_intel_connector_iter(connector, &conn_iter) {
 		pin = intel_connector_hpd_pin(connector);
 		if (pin == HPD_NONE ||
-		    dev_priv->hotplug.stats[pin].state != HPD_DISABLED)
+		    dev_priv->display.hotplug.stats[pin].state != HPD_DISABLED)
 			continue;
 
 		if (connector->base.polled != connector->polled)
@@ -245,8 +245,8 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work)
 	drm_connector_list_iter_end(&conn_iter);
 
 	for_each_hpd_pin(pin) {
-		if (dev_priv->hotplug.stats[pin].state == HPD_DISABLED)
-			dev_priv->hotplug.stats[pin].state = HPD_ENABLED;
+		if (dev_priv->display.hotplug.stats[pin].state == HPD_DISABLED)
+			dev_priv->display.hotplug.stats[pin].state = HPD_ENABLED;
 	}
 
 	intel_hpd_irq_setup(dev_priv);
@@ -297,16 +297,16 @@ static bool intel_encoder_has_hpd_pulse(struct intel_encoder *encoder)
 static void i915_digport_work_func(struct work_struct *work)
 {
 	struct drm_i915_private *dev_priv =
-		container_of(work, struct drm_i915_private, hotplug.dig_port_work);
+		container_of(work, struct drm_i915_private, display.hotplug.dig_port_work);
 	u32 long_port_mask, short_port_mask;
 	struct intel_encoder *encoder;
 	u32 old_bits = 0;
 
 	spin_lock_irq(&dev_priv->irq_lock);
-	long_port_mask = dev_priv->hotplug.long_port_mask;
-	dev_priv->hotplug.long_port_mask = 0;
-	short_port_mask = dev_priv->hotplug.short_port_mask;
-	dev_priv->hotplug.short_port_mask = 0;
+	long_port_mask = dev_priv->display.hotplug.long_port_mask;
+	dev_priv->display.hotplug.long_port_mask = 0;
+	short_port_mask = dev_priv->display.hotplug.short_port_mask;
+	dev_priv->display.hotplug.short_port_mask = 0;
 	spin_unlock_irq(&dev_priv->irq_lock);
 
 	for_each_intel_encoder(&dev_priv->drm, encoder) {
@@ -335,9 +335,9 @@ static void i915_digport_work_func(struct work_struct *work)
 
 	if (old_bits) {
 		spin_lock_irq(&dev_priv->irq_lock);
-		dev_priv->hotplug.event_bits |= old_bits;
+		dev_priv->display.hotplug.event_bits |= old_bits;
 		spin_unlock_irq(&dev_priv->irq_lock);
-		queue_delayed_work(system_wq, &dev_priv->hotplug.hotplug_work, 0);
+		queue_delayed_work(system_wq, &dev_priv->display.hotplug.hotplug_work, 0);
 	}
 }
 
@@ -353,10 +353,10 @@ void intel_hpd_trigger_irq(struct intel_digital_port *dig_port)
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
 
 	spin_lock_irq(&i915->irq_lock);
-	i915->hotplug.short_port_mask |= BIT(dig_port->base.port);
+	i915->display.hotplug.short_port_mask |= BIT(dig_port->base.port);
 	spin_unlock_irq(&i915->irq_lock);
 
-	queue_work(i915->hotplug.dp_wq, &i915->hotplug.dig_port_work);
+	queue_work(i915->display.hotplug.dp_wq, &i915->display.hotplug.dig_port_work);
 }
 
 /*
@@ -366,7 +366,7 @@ static void i915_hotplug_work_func(struct work_struct *work)
 {
 	struct drm_i915_private *dev_priv =
 		container_of(work, struct drm_i915_private,
-			     hotplug.hotplug_work.work);
+			     display.hotplug.hotplug_work.work);
 	struct drm_device *dev = &dev_priv->drm;
 	struct drm_connector_list_iter conn_iter;
 	struct intel_connector *connector;
@@ -379,10 +379,10 @@ static void i915_hotplug_work_func(struct work_struct *work)
 
 	spin_lock_irq(&dev_priv->irq_lock);
 
-	hpd_event_bits = dev_priv->hotplug.event_bits;
-	dev_priv->hotplug.event_bits = 0;
-	hpd_retry_bits = dev_priv->hotplug.retry_bits;
-	dev_priv->hotplug.retry_bits = 0;
+	hpd_event_bits = dev_priv->display.hotplug.event_bits;
+	dev_priv->display.hotplug.event_bits = 0;
+	hpd_retry_bits = dev_priv->display.hotplug.retry_bits;
+	dev_priv->display.hotplug.retry_bits = 0;
 
 	/* Enable polling for connectors which had HPD IRQ storms */
 	intel_hpd_irq_storm_switch_to_polling(dev_priv);
@@ -435,10 +435,10 @@ static void i915_hotplug_work_func(struct work_struct *work)
 	retry &= ~changed;
 	if (retry) {
 		spin_lock_irq(&dev_priv->irq_lock);
-		dev_priv->hotplug.retry_bits |= retry;
+		dev_priv->display.hotplug.retry_bits |= retry;
 		spin_unlock_irq(&dev_priv->irq_lock);
 
-		mod_delayed_work(system_wq, &dev_priv->hotplug.hotplug_work,
+		mod_delayed_work(system_wq, &dev_priv->display.hotplug.hotplug_work,
 				 msecs_to_jiffies(HPD_RETRY_DELAY));
 	}
 }
@@ -502,10 +502,10 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 
 		if (long_hpd) {
 			long_hpd_pulse_mask |= BIT(pin);
-			dev_priv->hotplug.long_port_mask |= BIT(port);
+			dev_priv->display.hotplug.long_port_mask |= BIT(port);
 		} else {
 			short_hpd_pulse_mask |= BIT(pin);
-			dev_priv->hotplug.short_port_mask |= BIT(port);
+			dev_priv->display.hotplug.short_port_mask |= BIT(port);
 		}
 	}
 
@@ -516,7 +516,7 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 		if (!(BIT(pin) & pin_mask))
 			continue;
 
-		if (dev_priv->hotplug.stats[pin].state == HPD_DISABLED) {
+		if (dev_priv->display.hotplug.stats[pin].state == HPD_DISABLED) {
 			/*
 			 * On GMCH platforms the interrupt mask bits only
 			 * prevent irq generation, not the setting of the
@@ -529,7 +529,7 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 			continue;
 		}
 
-		if (dev_priv->hotplug.stats[pin].state != HPD_ENABLED)
+		if (dev_priv->display.hotplug.stats[pin].state != HPD_ENABLED)
 			continue;
 
 		/*
@@ -540,13 +540,13 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 		if (((short_hpd_pulse_mask | long_hpd_pulse_mask) & BIT(pin))) {
 			long_hpd = long_hpd_pulse_mask & BIT(pin);
 		} else {
-			dev_priv->hotplug.event_bits |= BIT(pin);
+			dev_priv->display.hotplug.event_bits |= BIT(pin);
 			long_hpd = true;
 			queue_hp = true;
 		}
 
 		if (intel_hpd_irq_storm_detect(dev_priv, pin, long_hpd)) {
-			dev_priv->hotplug.event_bits &= ~BIT(pin);
+			dev_priv->display.hotplug.event_bits &= ~BIT(pin);
 			storm_detected = true;
 			queue_hp = true;
 		}
@@ -567,9 +567,9 @@ void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
 	 * deadlock.
 	 */
 	if (queue_dig)
-		queue_work(dev_priv->hotplug.dp_wq, &dev_priv->hotplug.dig_port_work);
+		queue_work(dev_priv->display.hotplug.dp_wq, &dev_priv->display.hotplug.dig_port_work);
 	if (queue_hp)
-		queue_delayed_work(system_wq, &dev_priv->hotplug.hotplug_work, 0);
+		queue_delayed_work(system_wq, &dev_priv->display.hotplug.hotplug_work, 0);
 }
 
 /**
@@ -594,8 +594,8 @@ void intel_hpd_init(struct drm_i915_private *dev_priv)
 		return;
 
 	for_each_hpd_pin(i) {
-		dev_priv->hotplug.stats[i].count = 0;
-		dev_priv->hotplug.stats[i].state = HPD_ENABLED;
+		dev_priv->display.hotplug.stats[i].count = 0;
+		dev_priv->display.hotplug.stats[i].state = HPD_ENABLED;
 	}
 
 	/*
@@ -611,7 +611,7 @@ static void i915_hpd_poll_init_work(struct work_struct *work)
 {
 	struct drm_i915_private *dev_priv =
 		container_of(work, struct drm_i915_private,
-			     hotplug.poll_init_work);
+			     display.hotplug.poll_init_work);
 	struct drm_device *dev = &dev_priv->drm;
 	struct drm_connector_list_iter conn_iter;
 	struct intel_connector *connector;
@@ -619,7 +619,7 @@ static void i915_hpd_poll_init_work(struct work_struct *work)
 
 	mutex_lock(&dev->mode_config.mutex);
 
-	enabled = READ_ONCE(dev_priv->hotplug.poll_enabled);
+	enabled = READ_ONCE(dev_priv->display.hotplug.poll_enabled);
 
 	drm_connector_list_iter_begin(dev, &conn_iter);
 	for_each_intel_connector_iter(connector, &conn_iter) {
@@ -672,7 +672,7 @@ void intel_hpd_poll_enable(struct drm_i915_private *dev_priv)
 	    !INTEL_DISPLAY_ENABLED(dev_priv))
 		return;
 
-	WRITE_ONCE(dev_priv->hotplug.poll_enabled, true);
+	WRITE_ONCE(dev_priv->display.hotplug.poll_enabled, true);
 
 	/*
 	 * We might already be holding dev->mode_config.mutex, so do this in a
@@ -680,7 +680,7 @@ void intel_hpd_poll_enable(struct drm_i915_private *dev_priv)
 	 * As well, there's no issue if we race here since we always reschedule
 	 * this worker anyway
 	 */
-	schedule_work(&dev_priv->hotplug.poll_init_work);
+	schedule_work(&dev_priv->display.hotplug.poll_init_work);
 }
 
 /**
@@ -707,17 +707,17 @@ void intel_hpd_poll_disable(struct drm_i915_private *dev_priv)
 	if (!HAS_DISPLAY(dev_priv))
 		return;
 
-	WRITE_ONCE(dev_priv->hotplug.poll_enabled, false);
-	schedule_work(&dev_priv->hotplug.poll_init_work);
+	WRITE_ONCE(dev_priv->display.hotplug.poll_enabled, false);
+	schedule_work(&dev_priv->display.hotplug.poll_init_work);
 }
 
 void intel_hpd_init_work(struct drm_i915_private *dev_priv)
 {
-	INIT_DELAYED_WORK(&dev_priv->hotplug.hotplug_work,
+	INIT_DELAYED_WORK(&dev_priv->display.hotplug.hotplug_work,
 			  i915_hotplug_work_func);
-	INIT_WORK(&dev_priv->hotplug.dig_port_work, i915_digport_work_func);
-	INIT_WORK(&dev_priv->hotplug.poll_init_work, i915_hpd_poll_init_work);
-	INIT_DELAYED_WORK(&dev_priv->hotplug.reenable_work,
+	INIT_WORK(&dev_priv->display.hotplug.dig_port_work, i915_digport_work_func);
+	INIT_WORK(&dev_priv->display.hotplug.poll_init_work, i915_hpd_poll_init_work);
+	INIT_DELAYED_WORK(&dev_priv->display.hotplug.reenable_work,
 			  intel_hpd_irq_storm_reenable_work);
 }
 
@@ -728,17 +728,17 @@ void intel_hpd_cancel_work(struct drm_i915_private *dev_priv)
 
 	spin_lock_irq(&dev_priv->irq_lock);
 
-	dev_priv->hotplug.long_port_mask = 0;
-	dev_priv->hotplug.short_port_mask = 0;
-	dev_priv->hotplug.event_bits = 0;
-	dev_priv->hotplug.retry_bits = 0;
+	dev_priv->display.hotplug.long_port_mask = 0;
+	dev_priv->display.hotplug.short_port_mask = 0;
+	dev_priv->display.hotplug.event_bits = 0;
+	dev_priv->display.hotplug.retry_bits = 0;
 
 	spin_unlock_irq(&dev_priv->irq_lock);
 
-	cancel_work_sync(&dev_priv->hotplug.dig_port_work);
-	cancel_delayed_work_sync(&dev_priv->hotplug.hotplug_work);
-	cancel_work_sync(&dev_priv->hotplug.poll_init_work);
-	cancel_delayed_work_sync(&dev_priv->hotplug.reenable_work);
+	cancel_work_sync(&dev_priv->display.hotplug.dig_port_work);
+	cancel_delayed_work_sync(&dev_priv->display.hotplug.hotplug_work);
+	cancel_work_sync(&dev_priv->display.hotplug.poll_init_work);
+	cancel_delayed_work_sync(&dev_priv->display.hotplug.reenable_work);
 }
 
 bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin)
@@ -749,8 +749,8 @@ bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin)
 		return false;
 
 	spin_lock_irq(&dev_priv->irq_lock);
-	if (dev_priv->hotplug.stats[pin].state == HPD_ENABLED) {
-		dev_priv->hotplug.stats[pin].state = HPD_DISABLED;
+	if (dev_priv->display.hotplug.stats[pin].state == HPD_ENABLED) {
+		dev_priv->display.hotplug.stats[pin].state = HPD_DISABLED;
 		ret = true;
 	}
 	spin_unlock_irq(&dev_priv->irq_lock);
@@ -764,6 +764,6 @@ void intel_hpd_enable(struct drm_i915_private *dev_priv, enum hpd_pin pin)
 		return;
 
 	spin_lock_irq(&dev_priv->irq_lock);
-	dev_priv->hotplug.stats[pin].state = HPD_ENABLED;
+	dev_priv->display.hotplug.stats[pin].state = HPD_ENABLED;
 	spin_unlock_irq(&dev_priv->irq_lock);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_lpe_audio.c b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
index 4970bf146c4a..dca6003ccac8 100644
--- a/drivers/gpu/drm/i915/display/intel_lpe_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
@@ -73,8 +73,9 @@
 #include "i915_drv.h"
 #include "intel_de.h"
 #include "intel_lpe_audio.h"
+#include "intel_pci_config.h"
 
-#define HAS_LPE_AUDIO(dev_priv) ((dev_priv)->audio.lpe.platdev != NULL)
+#define HAS_LPE_AUDIO(dev_priv) ((dev_priv)->display.audio.lpe.platdev != NULL)
 
 static struct platform_device *
 lpe_audio_platdev_create(struct drm_i915_private *dev_priv)
@@ -96,13 +97,13 @@ lpe_audio_platdev_create(struct drm_i915_private *dev_priv)
 		return ERR_PTR(-ENOMEM);
 	}
 
-	rsc[0].start    = rsc[0].end = dev_priv->audio.lpe.irq;
+	rsc[0].start    = rsc[0].end = dev_priv->display.audio.lpe.irq;
 	rsc[0].flags    = IORESOURCE_IRQ;
 	rsc[0].name     = "hdmi-lpe-audio-irq";
 
-	rsc[1].start    = pci_resource_start(pdev, 0) +
+	rsc[1].start    = pci_resource_start(pdev, GTTMMADR_BAR) +
 		I915_HDMI_LPE_AUDIO_BASE;
-	rsc[1].end      = pci_resource_start(pdev, 0) +
+	rsc[1].end      = pci_resource_start(pdev, GTTMMADR_BAR) +
 		I915_HDMI_LPE_AUDIO_BASE + I915_HDMI_LPE_AUDIO_SIZE - 1;
 	rsc[1].flags    = IORESOURCE_MEM;
 	rsc[1].name     = "hdmi-lpe-audio-mmio";
@@ -148,7 +149,7 @@ static void lpe_audio_platdev_destroy(struct drm_i915_private *dev_priv)
 	 * than us fiddle with its internals.
 	 */
 
-	platform_device_unregister(dev_priv->audio.lpe.platdev);
+	platform_device_unregister(dev_priv->display.audio.lpe.platdev);
 }
 
 static void lpe_audio_irq_unmask(struct irq_data *d)
@@ -167,7 +168,7 @@ static struct irq_chip lpe_audio_irqchip = {
 
 static int lpe_audio_irq_init(struct drm_i915_private *dev_priv)
 {
-	int irq = dev_priv->audio.lpe.irq;
+	int irq = dev_priv->display.audio.lpe.irq;
 
 	drm_WARN_ON(&dev_priv->drm, !intel_irqs_enabled(dev_priv));
 	irq_set_chip_and_handler_name(irq,
@@ -204,15 +205,15 @@ static int lpe_audio_setup(struct drm_i915_private *dev_priv)
 {
 	int ret;
 
-	dev_priv->audio.lpe.irq = irq_alloc_desc(0);
-	if (dev_priv->audio.lpe.irq < 0) {
+	dev_priv->display.audio.lpe.irq = irq_alloc_desc(0);
+	if (dev_priv->display.audio.lpe.irq < 0) {
 		drm_err(&dev_priv->drm, "Failed to allocate IRQ desc: %d\n",
-			dev_priv->audio.lpe.irq);
-		ret = dev_priv->audio.lpe.irq;
+			dev_priv->display.audio.lpe.irq);
+		ret = dev_priv->display.audio.lpe.irq;
 		goto err;
 	}
 
-	drm_dbg(&dev_priv->drm, "irq = %d\n", dev_priv->audio.lpe.irq);
+	drm_dbg(&dev_priv->drm, "irq = %d\n", dev_priv->display.audio.lpe.irq);
 
 	ret = lpe_audio_irq_init(dev_priv);
 
@@ -223,10 +224,10 @@ static int lpe_audio_setup(struct drm_i915_private *dev_priv)
 		goto err_free_irq;
 	}
 
-	dev_priv->audio.lpe.platdev = lpe_audio_platdev_create(dev_priv);
+	dev_priv->display.audio.lpe.platdev = lpe_audio_platdev_create(dev_priv);
 
-	if (IS_ERR(dev_priv->audio.lpe.platdev)) {
-		ret = PTR_ERR(dev_priv->audio.lpe.platdev);
+	if (IS_ERR(dev_priv->display.audio.lpe.platdev)) {
+		ret = PTR_ERR(dev_priv->display.audio.lpe.platdev);
 		drm_err(&dev_priv->drm,
 			"Failed to create lpe audio platform device: %d\n",
 			ret);
@@ -241,10 +242,10 @@ static int lpe_audio_setup(struct drm_i915_private *dev_priv)
 
 	return 0;
 err_free_irq:
-	irq_free_desc(dev_priv->audio.lpe.irq);
+	irq_free_desc(dev_priv->display.audio.lpe.irq);
 err:
-	dev_priv->audio.lpe.irq = -1;
-	dev_priv->audio.lpe.platdev = NULL;
+	dev_priv->display.audio.lpe.irq = -1;
+	dev_priv->display.audio.lpe.platdev = NULL;
 	return ret;
 }
 
@@ -262,7 +263,7 @@ void intel_lpe_audio_irq_handler(struct drm_i915_private *dev_priv)
 	if (!HAS_LPE_AUDIO(dev_priv))
 		return;
 
-	ret = generic_handle_irq(dev_priv->audio.lpe.irq);
+	ret = generic_handle_irq(dev_priv->display.audio.lpe.irq);
 	if (ret)
 		drm_err_ratelimited(&dev_priv->drm,
 				    "error handling LPE audio irq: %d\n", ret);
@@ -303,10 +304,10 @@ void intel_lpe_audio_teardown(struct drm_i915_private *dev_priv)
 
 	lpe_audio_platdev_destroy(dev_priv);
 
-	irq_free_desc(dev_priv->audio.lpe.irq);
+	irq_free_desc(dev_priv->display.audio.lpe.irq);
 
-	dev_priv->audio.lpe.irq = -1;
-	dev_priv->audio.lpe.platdev = NULL;
+	dev_priv->display.audio.lpe.irq = -1;
+	dev_priv->display.audio.lpe.platdev = NULL;
 }
 
 /**
@@ -333,7 +334,7 @@ void intel_lpe_audio_notify(struct drm_i915_private *dev_priv,
 	if (!HAS_LPE_AUDIO(dev_priv))
 		return;
 
-	pdata = dev_get_platdata(&dev_priv->audio.lpe.platdev->dev);
+	pdata = dev_get_platdata(&dev_priv->display.audio.lpe.platdev->dev);
 	ppdata = &pdata->port[port - PORT_B];
 
 	spin_lock_irqsave(&pdata->lpe_audio_slock, irqflags);
@@ -361,7 +362,7 @@ void intel_lpe_audio_notify(struct drm_i915_private *dev_priv,
 	}
 
 	if (pdata->notify_audio_lpe)
-		pdata->notify_audio_lpe(dev_priv->audio.lpe.platdev, port - PORT_B);
+		pdata->notify_audio_lpe(dev_priv->display.audio.lpe.platdev, port - PORT_B);
 
 	spin_unlock_irqrestore(&pdata->lpe_audio_slock, irqflags);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c
index 730480ac3300..9aa38e8141b5 100644
--- a/drivers/gpu/drm/i915/display/intel_lvds.c
+++ b/drivers/gpu/drm/i915/display/intel_lvds.c
@@ -837,12 +837,12 @@ void intel_lvds_init(struct drm_i915_private *dev_priv)
 
 	/* Skip init on machines we know falsely report LVDS */
 	if (dmi_check_system(intel_no_lvds)) {
-		drm_WARN(dev, !dev_priv->vbt.int_lvds_support,
+		drm_WARN(dev, !dev_priv->display.vbt.int_lvds_support,
 			 "Useless DMI match. Internal LVDS support disabled by VBT\n");
 		return;
 	}
 
-	if (!dev_priv->vbt.int_lvds_support) {
+	if (!dev_priv->display.vbt.int_lvds_support) {
 		drm_dbg_kms(&dev_priv->drm,
 			    "Internal LVDS support disabled by VBT\n");
 		return;
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_setup.c b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
index f0e04d3904c6..cbfabd58b75a 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_setup.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
@@ -23,6 +23,7 @@
 #include "intel_modeset_setup.h"
 #include "intel_pch_display.h"
 #include "intel_pm.h"
+#include "skl_watermark.h"
 
 static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
 					struct drm_modeset_acquire_ctx *ctx)
@@ -30,11 +31,11 @@ static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
 	struct intel_encoder *encoder;
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 	struct intel_bw_state *bw_state =
-		to_intel_bw_state(i915->bw_obj.state);
+		to_intel_bw_state(i915->display.bw.obj.state);
 	struct intel_cdclk_state *cdclk_state =
-		to_intel_cdclk_state(i915->cdclk.obj.state);
+		to_intel_cdclk_state(i915->display.cdclk.obj.state);
 	struct intel_dbuf_state *dbuf_state =
-		to_intel_dbuf_state(i915->dbuf.obj.state);
+		to_intel_dbuf_state(i915->display.dbuf.obj.state);
 	struct intel_crtc_state *crtc_state =
 		to_intel_crtc_state(crtc->base.state);
 	struct intel_plane *plane;
@@ -70,7 +71,7 @@ static void intel_crtc_disable_noatomic(struct intel_crtc *crtc,
 
 	drm_WARN_ON(&i915->drm, IS_ERR(temp_crtc_state) || ret);
 
-	i915->display->crtc_disable(to_intel_atomic_state(state), crtc);
+	i915->display.funcs.display->crtc_disable(to_intel_atomic_state(state), crtc);
 
 	drm_atomic_state_put(state);
 
@@ -415,9 +416,9 @@ static void readout_plane_state(struct drm_i915_private *i915)
 static void intel_modeset_readout_hw_state(struct drm_i915_private *i915)
 {
 	struct intel_cdclk_state *cdclk_state =
-		to_intel_cdclk_state(i915->cdclk.obj.state);
+		to_intel_cdclk_state(i915->display.cdclk.obj.state);
 	struct intel_dbuf_state *dbuf_state =
-		to_intel_dbuf_state(i915->dbuf.obj.state);
+		to_intel_dbuf_state(i915->display.dbuf.obj.state);
 	enum pipe pipe;
 	struct intel_crtc *crtc;
 	struct intel_encoder *encoder;
@@ -535,7 +536,7 @@ static void intel_modeset_readout_hw_state(struct drm_i915_private *i915)
 
 	for_each_intel_crtc(&i915->drm, crtc) {
 		struct intel_bw_state *bw_state =
-			to_intel_bw_state(i915->bw_obj.state);
+			to_intel_bw_state(i915->display.bw.obj.state);
 		struct intel_crtc_state *crtc_state =
 			to_intel_crtc_state(crtc->base.state);
 		struct intel_plane *plane;
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_verify.c b/drivers/gpu/drm/i915/display/intel_modeset_verify.c
index a91586d77cb6..0fdcf2e6d57f 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_verify.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_verify.c
@@ -15,8 +15,8 @@
 #include "intel_display_types.h"
 #include "intel_fdi.h"
 #include "intel_modeset_verify.h"
-#include "intel_pm.h"
 #include "intel_snps_phy.h"
+#include "skl_watermark.h"
 
 /*
  * Cross check the actual hw state with our own modeset state tracking (and its
@@ -94,10 +94,10 @@ static void intel_pipe_config_sanity_check(struct drm_i915_private *dev_priv,
 
 		/*
 		 * FDI already provided one idea for the dotclock.
-		 * Yell if the encoder disagrees.
+		 * Yell if the encoder disagrees. Allow for slight
+		 * rounding differences.
 		 */
-		drm_WARN(&dev_priv->drm,
-			 !intel_fuzzy_clock_check(fdi_dotclock, dotclock),
+		drm_WARN(&dev_priv->drm, abs(fdi_dotclock - dotclock) > 1,
 			 "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
 			 fdi_dotclock, dotclock);
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c
index 1c0c745c142d..caa07ef34f21 100644
--- a/drivers/gpu/drm/i915/display/intel_opregion.c
+++ b/drivers/gpu/drm/i915/display/intel_opregion.c
@@ -252,7 +252,7 @@ struct opregion_asle_ext {
 
 static int check_swsci_function(struct drm_i915_private *i915, u32 function)
 {
-	struct opregion_swsci *swsci = i915->opregion.swsci;
+	struct opregion_swsci *swsci = i915->display.opregion.swsci;
 	u32 main_function, sub_function;
 
 	if (!swsci)
@@ -265,11 +265,11 @@ static int check_swsci_function(struct drm_i915_private *i915, u32 function)
 
 	/* Check if we can call the function. See swsci_setup for details. */
 	if (main_function == SWSCI_SBCB) {
-		if ((i915->opregion.swsci_sbcb_sub_functions &
+		if ((i915->display.opregion.swsci_sbcb_sub_functions &
 		     (1 << sub_function)) == 0)
 			return -EINVAL;
 	} else if (main_function == SWSCI_GBDA) {
-		if ((i915->opregion.swsci_gbda_sub_functions &
+		if ((i915->display.opregion.swsci_gbda_sub_functions &
 		     (1 << sub_function)) == 0)
 			return -EINVAL;
 	}
@@ -280,7 +280,7 @@ static int check_swsci_function(struct drm_i915_private *i915, u32 function)
 static int swsci(struct drm_i915_private *dev_priv,
 		 u32 function, u32 parm, u32 *parm_out)
 {
-	struct opregion_swsci *swsci = dev_priv->opregion.swsci;
+	struct opregion_swsci *swsci = dev_priv->display.opregion.swsci;
 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 	u32 scic, dslp;
 	u16 swsci_val;
@@ -462,7 +462,7 @@ static u32 asle_set_backlight(struct drm_i915_private *dev_priv, u32 bclp)
 {
 	struct intel_connector *connector;
 	struct drm_connector_list_iter conn_iter;
-	struct opregion_asle *asle = dev_priv->opregion.asle;
+	struct opregion_asle *asle = dev_priv->display.opregion.asle;
 	struct drm_device *dev = &dev_priv->drm;
 
 	drm_dbg(&dev_priv->drm, "bclp = 0x%08x\n", bclp);
@@ -586,8 +586,8 @@ static void asle_work(struct work_struct *work)
 	struct intel_opregion *opregion =
 		container_of(work, struct intel_opregion, asle_work);
 	struct drm_i915_private *dev_priv =
-		container_of(opregion, struct drm_i915_private, opregion);
-	struct opregion_asle *asle = dev_priv->opregion.asle;
+		container_of(opregion, struct drm_i915_private, display.opregion);
+	struct opregion_asle *asle = dev_priv->display.opregion.asle;
 	u32 aslc_stat = 0;
 	u32 aslc_req;
 
@@ -635,8 +635,8 @@ static void asle_work(struct work_struct *work)
 
 void intel_opregion_asle_intr(struct drm_i915_private *dev_priv)
 {
-	if (dev_priv->opregion.asle)
-		schedule_work(&dev_priv->opregion.asle_work);
+	if (dev_priv->display.opregion.asle)
+		schedule_work(&dev_priv->display.opregion.asle_work);
 }
 
 #define ACPI_EV_DISPLAY_SWITCH (1<<0)
@@ -692,7 +692,7 @@ static void set_did(struct intel_opregion *opregion, int i, u32 val)
 
 static void intel_didl_outputs(struct drm_i915_private *dev_priv)
 {
-	struct intel_opregion *opregion = &dev_priv->opregion;
+	struct intel_opregion *opregion = &dev_priv->display.opregion;
 	struct intel_connector *connector;
 	struct drm_connector_list_iter conn_iter;
 	int i = 0, max_outputs;
@@ -731,7 +731,7 @@ static void intel_didl_outputs(struct drm_i915_private *dev_priv)
 
 static void intel_setup_cadls(struct drm_i915_private *dev_priv)
 {
-	struct intel_opregion *opregion = &dev_priv->opregion;
+	struct intel_opregion *opregion = &dev_priv->display.opregion;
 	struct intel_connector *connector;
 	struct drm_connector_list_iter conn_iter;
 	int i = 0;
@@ -761,7 +761,7 @@ static void intel_setup_cadls(struct drm_i915_private *dev_priv)
 
 static void swsci_setup(struct drm_i915_private *dev_priv)
 {
-	struct intel_opregion *opregion = &dev_priv->opregion;
+	struct intel_opregion *opregion = &dev_priv->display.opregion;
 	bool requested_callbacks = false;
 	u32 tmp;
 
@@ -839,7 +839,7 @@ static const struct dmi_system_id intel_no_opregion_vbt[] = {
 
 static int intel_load_vbt_firmware(struct drm_i915_private *dev_priv)
 {
-	struct intel_opregion *opregion = &dev_priv->opregion;
+	struct intel_opregion *opregion = &dev_priv->display.opregion;
 	const struct firmware *fw = NULL;
 	const char *name = dev_priv->params.vbt_firmware;
 	int ret;
@@ -879,7 +879,7 @@ static int intel_load_vbt_firmware(struct drm_i915_private *dev_priv)
 
 int intel_opregion_setup(struct drm_i915_private *dev_priv)
 {
-	struct intel_opregion *opregion = &dev_priv->opregion;
+	struct intel_opregion *opregion = &dev_priv->display.opregion;
 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 	u32 asls, mboxes;
 	char buf[sizeof(OPREGION_SIGNATURE)];
@@ -1106,7 +1106,7 @@ struct edid *intel_opregion_get_edid(struct intel_connector *intel_connector)
 {
 	struct drm_connector *connector = &intel_connector->base;
 	struct drm_i915_private *i915 = to_i915(connector->dev);
-	struct intel_opregion *opregion = &i915->opregion;
+	struct intel_opregion *opregion = &i915->display.opregion;
 	const void *in_edid;
 	const struct edid *edid;
 	struct edid *new_edid;
@@ -1141,7 +1141,7 @@ struct edid *intel_opregion_get_edid(struct intel_connector *intel_connector)
 
 bool intel_opregion_headless_sku(struct drm_i915_private *i915)
 {
-	struct intel_opregion *opregion = &i915->opregion;
+	struct intel_opregion *opregion = &i915->display.opregion;
 	struct opregion_header *header = opregion->header;
 
 	if (!header || header->over.major < 2 ||
@@ -1153,7 +1153,7 @@ bool intel_opregion_headless_sku(struct drm_i915_private *i915)
 
 void intel_opregion_register(struct drm_i915_private *i915)
 {
-	struct intel_opregion *opregion = &i915->opregion;
+	struct intel_opregion *opregion = &i915->display.opregion;
 
 	if (!opregion->header)
 		return;
@@ -1169,7 +1169,7 @@ void intel_opregion_register(struct drm_i915_private *i915)
 
 void intel_opregion_resume(struct drm_i915_private *i915)
 {
-	struct intel_opregion *opregion = &i915->opregion;
+	struct intel_opregion *opregion = &i915->display.opregion;
 
 	if (!opregion->header)
 		return;
@@ -1200,7 +1200,7 @@ void intel_opregion_resume(struct drm_i915_private *i915)
 
 void intel_opregion_suspend(struct drm_i915_private *i915, pci_power_t state)
 {
-	struct intel_opregion *opregion = &i915->opregion;
+	struct intel_opregion *opregion = &i915->display.opregion;
 
 	if (!opregion->header)
 		return;
@@ -1210,7 +1210,7 @@ void intel_opregion_suspend(struct drm_i915_private *i915, pci_power_t state)
 	if (opregion->asle)
 		opregion->asle->ardy = ASLE_ARDY_NOT_READY;
 
-	cancel_work_sync(&i915->opregion.asle_work);
+	cancel_work_sync(&i915->display.opregion.asle_work);
 
 	if (opregion->acpi)
 		opregion->acpi->drdy = 0;
@@ -1218,7 +1218,7 @@ void intel_opregion_suspend(struct drm_i915_private *i915, pci_power_t state)
 
 void intel_opregion_unregister(struct drm_i915_private *i915)
 {
-	struct intel_opregion *opregion = &i915->opregion;
+	struct intel_opregion *opregion = &i915->display.opregion;
 
 	intel_opregion_suspend(i915, PCI_D1);
 
diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c
index 79ed8bd04a07..c12bdca8da9b 100644
--- a/drivers/gpu/drm/i915/display/intel_overlay.c
+++ b/drivers/gpu/drm/i915/display/intel_overlay.c
@@ -211,9 +211,9 @@ static void i830_overlay_clock_gating(struct drm_i915_private *dev_priv,
 
 	/* WA_OVERLAY_CLKGATE:alm */
 	if (enable)
-		intel_de_write(dev_priv, DSPCLK_GATE_D, 0);
+		intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), 0);
 	else
-		intel_de_write(dev_priv, DSPCLK_GATE_D,
+		intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv),
 			       OVRUNIT_CLOCK_GATE_DISABLE);
 
 	/* WA_DISABLE_L2CACHE_CLOCK_GATING:alm */
@@ -487,7 +487,7 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
 
 void intel_overlay_reset(struct drm_i915_private *dev_priv)
 {
-	struct intel_overlay *overlay = dev_priv->overlay;
+	struct intel_overlay *overlay = dev_priv->display.overlay;
 
 	if (!overlay)
 		return;
@@ -1113,7 +1113,7 @@ int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
 	struct drm_i915_gem_object *new_bo;
 	int ret;
 
-	overlay = dev_priv->overlay;
+	overlay = dev_priv->display.overlay;
 	if (!overlay) {
 		drm_dbg(&dev_priv->drm, "userspace bug: no overlay\n");
 		return -ENODEV;
@@ -1273,7 +1273,7 @@ int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data,
 	struct intel_overlay *overlay;
 	int ret;
 
-	overlay = dev_priv->overlay;
+	overlay = dev_priv->display.overlay;
 	if (!overlay) {
 		drm_dbg(&dev_priv->drm, "userspace bug: no overlay\n");
 		return -ENODEV;
@@ -1416,7 +1416,7 @@ void intel_overlay_setup(struct drm_i915_private *dev_priv)
 	update_polyphase_filter(overlay->regs);
 	update_reg_attrs(overlay, overlay->regs);
 
-	dev_priv->overlay = overlay;
+	dev_priv->display.overlay = overlay;
 	drm_info(&dev_priv->drm, "Initialized overlay support.\n");
 	return;
 
@@ -1428,7 +1428,7 @@ void intel_overlay_cleanup(struct drm_i915_private *dev_priv)
 {
 	struct intel_overlay *overlay;
 
-	overlay = fetch_and_zero(&dev_priv->overlay);
+	overlay = fetch_and_zero(&dev_priv->display.overlay);
 	if (!overlay)
 		return;
 
@@ -1457,7 +1457,7 @@ struct intel_overlay_error_state {
 struct intel_overlay_error_state *
 intel_overlay_capture_error_state(struct drm_i915_private *dev_priv)
 {
-	struct intel_overlay *overlay = dev_priv->overlay;
+	struct intel_overlay *overlay = dev_priv->display.overlay;
 	struct intel_overlay_error_state *error;
 
 	if (!overlay || !overlay->active)
diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c
index 237a40623dd7..a3a3f9fe4342 100644
--- a/drivers/gpu/drm/i915/display/intel_panel.c
+++ b/drivers/gpu/drm/i915/display/intel_panel.c
@@ -37,13 +37,14 @@
 #include "intel_display_types.h"
 #include "intel_drrs.h"
 #include "intel_panel.h"
+#include "intel_quirks.h"
 
 bool intel_panel_use_ssc(struct drm_i915_private *i915)
 {
 	if (i915->params.panel_use_ssc >= 0)
 		return i915->params.panel_use_ssc != 0;
-	return i915->vbt.lvds_use_ssc
-		&& !(i915->quirks & QUIRK_LVDS_SSC_DISABLE);
+	return i915->display.vbt.lvds_use_ssc &&
+		!intel_has_quirk(i915, QUIRK_LVDS_SSC_DISABLE);
 }
 
 const struct drm_display_mode *
@@ -81,15 +82,14 @@ static bool is_alt_drrs_mode(const struct drm_display_mode *mode,
 		mode->clock != preferred_mode->clock;
 }
 
-static bool is_alt_vrr_mode(const struct drm_display_mode *mode,
-			    const struct drm_display_mode *preferred_mode)
+static bool is_alt_fixed_mode(const struct drm_display_mode *mode,
+			      const struct drm_display_mode *preferred_mode)
 {
 	return drm_mode_match(mode, preferred_mode,
 			      DRM_MODE_MATCH_FLAGS |
 			      DRM_MODE_MATCH_3D_FLAGS) &&
 		mode->hdisplay == preferred_mode->hdisplay &&
-		mode->vdisplay == preferred_mode->vdisplay &&
-		mode->clock != preferred_mode->clock;
+		mode->vdisplay == preferred_mode->vdisplay;
 }
 
 const struct drm_display_mode *
@@ -114,6 +114,21 @@ intel_panel_downclock_mode(struct intel_connector *connector,
 	return best_mode;
 }
 
+const struct drm_display_mode *
+intel_panel_highest_mode(struct intel_connector *connector,
+			 const struct drm_display_mode *adjusted_mode)
+{
+	const struct drm_display_mode *fixed_mode, *best_mode = adjusted_mode;
+
+	/* pick the fixed_mode that has the highest clock */
+	list_for_each_entry(fixed_mode, &connector->panel.fixed_modes, head) {
+		if (fixed_mode->clock > best_mode->clock)
+			best_mode = fixed_mode;
+	}
+
+	return best_mode;
+}
+
 int intel_panel_get_modes(struct intel_connector *connector)
 {
 	const struct drm_display_mode *fixed_mode;
@@ -172,19 +187,7 @@ int intel_panel_compute_config(struct intel_connector *connector,
 	return 0;
 }
 
-static bool is_alt_fixed_mode(const struct drm_display_mode *mode,
-			      const struct drm_display_mode *preferred_mode,
-			      bool has_vrr)
-{
-	/* is_alt_drrs_mode() is a subset of is_alt_vrr_mode() */
-	if (has_vrr)
-		return is_alt_vrr_mode(mode, preferred_mode);
-	else
-		return is_alt_drrs_mode(mode, preferred_mode);
-}
-
-static void intel_panel_add_edid_alt_fixed_modes(struct intel_connector *connector,
-						 bool has_vrr)
+static void intel_panel_add_edid_alt_fixed_modes(struct intel_connector *connector)
 {
 	struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 	const struct drm_display_mode *preferred_mode =
@@ -192,7 +195,7 @@ static void intel_panel_add_edid_alt_fixed_modes(struct intel_connector *connect
 	struct drm_display_mode *mode, *next;
 
 	list_for_each_entry_safe(mode, next, &connector->base.probed_modes, head) {
-		if (!is_alt_fixed_mode(mode, preferred_mode, has_vrr))
+		if (!is_alt_fixed_mode(mode, preferred_mode))
 			continue;
 
 		drm_dbg_kms(&dev_priv->drm,
@@ -255,7 +258,7 @@ void intel_panel_add_edid_fixed_modes(struct intel_connector *connector,
 {
 	intel_panel_add_edid_preferred_mode(connector);
 	if (intel_panel_preferred_fixed_mode(connector) && (has_drrs || has_vrr))
-		intel_panel_add_edid_alt_fixed_modes(connector, has_vrr);
+		intel_panel_add_edid_alt_fixed_modes(connector);
 	intel_panel_destroy_probed_modes(connector);
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_panel.h b/drivers/gpu/drm/i915/display/intel_panel.h
index b087c0c3cc6d..eff3ffd3d082 100644
--- a/drivers/gpu/drm/i915/display/intel_panel.h
+++ b/drivers/gpu/drm/i915/display/intel_panel.h
@@ -31,6 +31,9 @@ intel_panel_fixed_mode(struct intel_connector *connector,
 const struct drm_display_mode *
 intel_panel_downclock_mode(struct intel_connector *connector,
 			   const struct drm_display_mode *adjusted_mode);
+const struct drm_display_mode *
+intel_panel_highest_mode(struct intel_connector *connector,
+			 const struct drm_display_mode *adjusted_mode);
 int intel_panel_get_modes(struct intel_connector *connector);
 enum drrs_type intel_panel_drrs_type(struct intel_connector *connector);
 enum drm_mode_status
diff --git a/drivers/gpu/drm/i915/display/intel_pch_refclk.c b/drivers/gpu/drm/i915/display/intel_pch_refclk.c
index 9934c8a9e240..a66097cdc1e0 100644
--- a/drivers/gpu/drm/i915/display/intel_pch_refclk.c
+++ b/drivers/gpu/drm/i915/display/intel_pch_refclk.c
@@ -167,6 +167,15 @@ static void lpt_compute_iclkip(struct iclkip_params *p, int clock)
 	}
 }
 
+int lpt_iclkip(const struct intel_crtc_state *crtc_state)
+{
+	struct iclkip_params p;
+
+	lpt_compute_iclkip(&p, crtc_state->hw.adjusted_mode.crtc_clock);
+
+	return lpt_iclkip_freq(&p);
+}
+
 /* Program iCLKIP clock to the desired frequency */
 void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
 {
@@ -179,6 +188,7 @@ void lpt_program_iclkip(const struct intel_crtc_state *crtc_state)
 	lpt_disable_iclkip(dev_priv);
 
 	lpt_compute_iclkip(&p, clock);
+	drm_WARN_ON(&dev_priv->drm, lpt_iclkip_freq(&p) != clock);
 
 	/* This should not happen with any sane values */
 	drm_WARN_ON(&dev_priv->drm, SBI_SSCDIVINTPHASE_DIVSEL(p.divsel) &
@@ -514,7 +524,7 @@ static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv)
 	}
 
 	if (HAS_PCH_IBX(dev_priv)) {
-		has_ck505 = dev_priv->vbt.display_clock_mode;
+		has_ck505 = dev_priv->display.vbt.display_clock_mode;
 		can_ssc = has_ck505;
 	} else {
 		has_ck505 = false;
@@ -522,7 +532,7 @@ static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv)
 	}
 
 	/* Check if any DPLLs are using the SSC source */
-	for (i = 0; i < dev_priv->dpll.num_shared_dpll; i++) {
+	for (i = 0; i < dev_priv->display.dpll.num_shared_dpll; i++) {
 		u32 temp = intel_de_read(dev_priv, PCH_DPLL(i));
 
 		if (!(temp & DPLL_VCO_ENABLE))
@@ -654,7 +664,7 @@ static void ilk_init_pch_refclk(struct drm_i915_private *dev_priv)
 		}
 	}
 
-	BUG_ON(val != final);
+	drm_WARN_ON(&dev_priv->drm, val != final);
 }
 
 /*
diff --git a/drivers/gpu/drm/i915/display/intel_pch_refclk.h b/drivers/gpu/drm/i915/display/intel_pch_refclk.h
index 12ab2c75a800..9bcf56629f24 100644
--- a/drivers/gpu/drm/i915/display/intel_pch_refclk.h
+++ b/drivers/gpu/drm/i915/display/intel_pch_refclk.h
@@ -14,6 +14,7 @@ struct intel_crtc_state;
 void lpt_program_iclkip(const struct intel_crtc_state *crtc_state);
 void lpt_disable_iclkip(struct drm_i915_private *dev_priv);
 int lpt_get_iclkip(struct drm_i915_private *dev_priv);
+int lpt_iclkip(const struct intel_crtc_state *crtc_state);
 
 void intel_init_pch_refclk(struct drm_i915_private *dev_priv);
 void lpt_disable_clkout_dp(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/display/intel_plane_initial.c b/drivers/gpu/drm/i915/display/intel_plane_initial.c
index d10f27d0b7b0..76be796df255 100644
--- a/drivers/gpu/drm/i915/display/intel_plane_initial.c
+++ b/drivers/gpu/drm/i915/display/intel_plane_initial.c
@@ -311,7 +311,7 @@ void intel_crtc_initial_plane_config(struct intel_crtc *crtc)
 	 * can even allow for smooth boot transitions if the BIOS
 	 * fb is large enough for the active pipe configuration.
 	 */
-	dev_priv->display->get_initial_plane_config(crtc, &plane_config);
+	dev_priv->display.funcs.display->get_initial_plane_config(crtc, &plane_config);
 
 	/*
 	 * If the fb is shared between multiple heads, we'll
diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c
index 1b21a341962f..21944f5bf3a8 100644
--- a/drivers/gpu/drm/i915/display/intel_pps.c
+++ b/drivers/gpu/drm/i915/display/intel_pps.c
@@ -12,6 +12,7 @@
 #include "intel_dpll.h"
 #include "intel_lvds.h"
 #include "intel_pps.h"
+#include "intel_quirks.h"
 
 static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv,
 				      enum pipe pipe);
@@ -28,7 +29,7 @@ intel_wakeref_t intel_pps_lock(struct intel_dp *intel_dp)
 	 * See intel_pps_reset_all() why we need a power domain reference here.
 	 */
 	wakeref = intel_display_power_get(dev_priv, POWER_DOMAIN_DISPLAY_CORE);
-	mutex_lock(&dev_priv->pps_mutex);
+	mutex_lock(&dev_priv->display.pps.mutex);
 
 	return wakeref;
 }
@@ -38,7 +39,7 @@ intel_wakeref_t intel_pps_unlock(struct intel_dp *intel_dp,
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 
-	mutex_unlock(&dev_priv->pps_mutex);
+	mutex_unlock(&dev_priv->display.pps.mutex);
 	intel_display_power_put(dev_priv, POWER_DOMAIN_DISPLAY_CORE, wakeref);
 
 	return 0;
@@ -163,7 +164,7 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum pipe pipe;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	/* We should never land here with regular DP ports */
 	drm_WARN_ON(&dev_priv->drm, !intel_dp_is_edp(intel_dp));
@@ -212,7 +213,7 @@ bxt_power_sequencer_idx(struct intel_dp *intel_dp)
 	struct intel_connector *connector = intel_dp->attached_connector;
 	int backlight_controller = connector->panel.vbt.backlight.controller;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	/* We should never land here with regular DP ports */
 	drm_WARN_ON(&dev_priv->drm, !intel_dp_is_edp(intel_dp));
@@ -282,7 +283,7 @@ vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp)
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum port port = dig_port->base.port;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	/* try to find a pipe with this port selected */
 	/* first pick one where the panel is on */
@@ -407,7 +408,7 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 	    intel_dp->pps.pps_pipe == INVALID_PIPE)
@@ -420,7 +421,7 @@ static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	if ((IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
 	    intel_dp->pps.pps_pipe == INVALID_PIPE)
@@ -463,7 +464,7 @@ static void wait_panel_status(struct intel_dp *intel_dp,
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	i915_reg_t pp_stat_reg, pp_ctrl_reg;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	intel_pps_verify_state(intel_dp);
 
@@ -556,7 +557,7 @@ static  u32 ilk_get_pp_control(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	u32 control;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	control = intel_de_read(dev_priv, _pp_ctrl_reg(intel_dp));
 	if (drm_WARN_ON(&dev_priv->drm, !HAS_DDI(dev_priv) &&
@@ -580,7 +581,7 @@ bool intel_pps_vdd_on_unlocked(struct intel_dp *intel_dp)
 	i915_reg_t pp_stat_reg, pp_ctrl_reg;
 	bool need_to_disable = !intel_dp->pps.want_panel_vdd;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	if (!intel_dp_is_edp(intel_dp))
 		return false;
@@ -657,7 +658,7 @@ static void intel_pps_vdd_off_sync_unlocked(struct intel_dp *intel_dp)
 	u32 pp;
 	i915_reg_t pp_stat_reg, pp_ctrl_reg;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	drm_WARN_ON(&dev_priv->drm, intel_dp->pps.want_panel_vdd);
 
@@ -748,7 +749,7 @@ void intel_pps_vdd_off_unlocked(struct intel_dp *intel_dp, bool sync)
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	if (!intel_dp_is_edp(intel_dp))
 		return;
@@ -771,7 +772,7 @@ void intel_pps_on_unlocked(struct intel_dp *intel_dp)
 	u32 pp;
 	i915_reg_t pp_ctrl_reg;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	if (!intel_dp_is_edp(intel_dp))
 		return;
@@ -832,7 +833,7 @@ void intel_pps_off_unlocked(struct intel_dp *intel_dp)
 	u32 pp;
 	i915_reg_t pp_ctrl_reg;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	if (!intel_dp_is_edp(intel_dp))
 		return;
@@ -991,7 +992,7 @@ static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv,
 {
 	struct intel_encoder *encoder;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	for_each_intel_dp(&dev_priv->drm, encoder) {
 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
@@ -1021,7 +1022,7 @@ void vlv_pps_init(struct intel_encoder *encoder,
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	drm_WARN_ON(&dev_priv->drm, intel_dp->pps.active_pipe != INVALID_PIPE);
 
@@ -1064,7 +1065,7 @@ static void pps_vdd_init(struct intel_dp *intel_dp)
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	if (!edp_have_panel_vdd(intel_dp))
 		return;
@@ -1176,7 +1177,7 @@ static void pps_init_delays_bios(struct intel_dp *intel_dp,
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	if (!pps_delays_valid(&intel_dp->pps.bios_pps_delays))
 		intel_pps_readout_hw_state(intel_dp, &intel_dp->pps.bios_pps_delays);
@@ -1202,7 +1203,7 @@ static void pps_init_delays_vbt(struct intel_dp *intel_dp,
 	 * just fails to power back on. Increasing the delay to 800ms
 	 * seems sufficient to avoid this problem.
 	 */
-	if (dev_priv->quirks & QUIRK_INCREASE_T12_DELAY) {
+	if (intel_has_quirk(dev_priv, QUIRK_INCREASE_T12_DELAY)) {
 		vbt->t11_t12 = max_t(u16, vbt->t11_t12, 1300 * 10);
 		drm_dbg_kms(&dev_priv->drm,
 			    "Increasing T12 panel delay as per the quirk to %d\n",
@@ -1223,7 +1224,7 @@ static void pps_init_delays_spec(struct intel_dp *intel_dp,
 {
 	struct drm_i915_private *dev_priv = dp_to_i915(intel_dp);
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	/* Upper limits from eDP 1.3 spec. Note that we use the clunky units of
 	 * our hw here, which are all in 100usec. */
@@ -1246,7 +1247,7 @@ static void pps_init_delays(struct intel_dp *intel_dp)
 	struct edp_power_seq cur, vbt, spec,
 		*final = &intel_dp->pps.pps_delays;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	/* already initialized? */
 	if (pps_delays_valid(final))
@@ -1312,7 +1313,7 @@ static void pps_init_registers(struct intel_dp *intel_dp, bool force_disable_vdd
 	enum port port = dp_to_dig_port(intel_dp)->base.port;
 	const struct edp_power_seq *seq = &intel_dp->pps.pps_delays;
 
-	lockdep_assert_held(&dev_priv->pps_mutex);
+	lockdep_assert_held(&dev_priv->display.pps.mutex);
 
 	intel_pps_get_registers(intel_dp, &regs);
 
@@ -1487,11 +1488,11 @@ void intel_pps_unlock_regs_wa(struct drm_i915_private *dev_priv)
 void intel_pps_setup(struct drm_i915_private *i915)
 {
 	if (HAS_PCH_SPLIT(i915) || IS_GEMINILAKE(i915) || IS_BROXTON(i915))
-		i915->pps_mmio_base = PCH_PPS_BASE;
+		i915->display.pps.mmio_base = PCH_PPS_BASE;
 	else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
-		i915->pps_mmio_base = VLV_PPS_BASE;
+		i915->display.pps.mmio_base = VLV_PPS_BASE;
 	else
-		i915->pps_mmio_base = PPS_BASE;
+		i915->display.pps.mmio_base = PPS_BASE;
 }
 
 void assert_pps_unlocked(struct drm_i915_private *dev_priv, enum pipe pipe)
diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
index e6a870641cd2..9def8d9fade6 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.c
+++ b/drivers/gpu/drm/i915/display/intel_psr.c
@@ -706,7 +706,7 @@ tgl_dc3co_exitline_compute_config(struct intel_dp *intel_dp,
 	if (crtc_state->enable_psr2_sel_fetch)
 		return;
 
-	if (!(dev_priv->dmc.allowed_dc_mask & DC_STATE_EN_DC3CO))
+	if (!(dev_priv->display.dmc.allowed_dc_mask & DC_STATE_EN_DC3CO))
 		return;
 
 	if (!dc3co_is_pipe_port_compatible(intel_dp, crtc_state))
@@ -805,13 +805,14 @@ static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_d
 	hblank_total = adjusted_mode->crtc_hblank_end - adjusted_mode->crtc_hblank_start;
 	hblank_ns = div_u64(1000000ULL * hblank_total, adjusted_mode->crtc_clock);
 
-	/* From spec: (72 / number of lanes) * 1000 / symbol clock frequency MHz */
-	req_ns = (72 / crtc_state->lane_count) * 1000 / (crtc_state->port_clock / 1000);
+	/* From spec: ((60 / number of lanes) + 11) * 1000 / symbol clock frequency MHz */
+	req_ns = ((60 / crtc_state->lane_count) + 11) * 1000 / (crtc_state->port_clock / 1000);
 
 	if ((hblank_ns - req_ns) > 100)
 		return true;
 
-	if (DISPLAY_VER(dev_priv) < 13 || intel_dp->edp_dpcd[0] < DP_EDP_14b)
+	/* Not supported <13 / Wa_22012279113:adl-p */
+	if (DISPLAY_VER(dev_priv) <= 13 || intel_dp->edp_dpcd[0] < DP_EDP_14b)
 		return false;
 
 	crtc_state->req_psr2_sdp_prior_scanline = true;
@@ -1721,8 +1722,6 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 					     new_plane_state, i) {
 		struct drm_rect src, damaged_area = { .x1 = 0, .y1 = -1,
 						      .x2 = INT_MAX };
-		struct drm_atomic_helper_damage_iter iter;
-		struct drm_rect clip;
 
 		if (new_plane_state->uapi.crtc != crtc_state->uapi.crtc)
 			continue;
@@ -1767,22 +1766,18 @@ int intel_psr2_sel_fetch_update(struct intel_atomic_state *state,
 			continue;
 		}
 
-		drm_rect_fp_to_int(&src, &new_plane_state->uapi.src);
+		src = drm_plane_state_src(&new_plane_state->uapi);
+		drm_rect_fp_to_int(&src, &src);
 
-		drm_atomic_helper_damage_iter_init(&iter,
-						   &old_plane_state->uapi,
-						   &new_plane_state->uapi);
-		drm_atomic_for_each_plane_damage(&iter, &clip) {
-			if (drm_rect_intersect(&clip, &src))
-				clip_area_update(&damaged_area, &clip,
-						 &crtc_state->pipe_src);
-		}
-
-		if (damaged_area.y1 == -1)
+		if (!drm_atomic_helper_damage_merged(&old_plane_state->uapi,
+						     &new_plane_state->uapi, &damaged_area))
 			continue;
 
 		damaged_area.y1 += new_plane_state->uapi.dst.y1 - src.y1;
 		damaged_area.y2 += new_plane_state->uapi.dst.y1 - src.y1;
+		damaged_area.x1 += new_plane_state->uapi.dst.x1 - src.x1;
+		damaged_area.x2 += new_plane_state->uapi.dst.x1 - src.x1;
+
 		clip_area_update(&pipe_clip, &damaged_area, &crtc_state->pipe_src);
 	}
 
@@ -1863,7 +1858,9 @@ void intel_psr_pre_plane_update(struct intel_atomic_state *state,
 				struct intel_crtc *crtc)
 {
 	struct drm_i915_private *i915 = to_i915(state->base.dev);
-	const struct intel_crtc_state *crtc_state =
+	const struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(state, crtc);
+	const struct intel_crtc_state *new_crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
 	struct intel_encoder *encoder;
 
@@ -1871,7 +1868,7 @@ void intel_psr_pre_plane_update(struct intel_atomic_state *state,
 		return;
 
 	for_each_intel_encoder_mask_with_psr(state->base.dev, encoder,
-					     crtc_state->uapi.encoder_mask) {
+					     old_crtc_state->uapi.encoder_mask) {
 		struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 		struct intel_psr *psr = &intel_dp->psr;
 		bool needs_to_disable = false;
@@ -1884,10 +1881,10 @@ void intel_psr_pre_plane_update(struct intel_atomic_state *state,
 		 * - All planes will go inactive
 		 * - Changing between PSR versions
 		 */
-		needs_to_disable |= intel_crtc_needs_modeset(crtc_state);
-		needs_to_disable |= !crtc_state->has_psr;
-		needs_to_disable |= !crtc_state->active_planes;
-		needs_to_disable |= crtc_state->has_psr2 != psr->psr2_enabled;
+		needs_to_disable |= intel_crtc_needs_modeset(new_crtc_state);
+		needs_to_disable |= !new_crtc_state->has_psr;
+		needs_to_disable |= !new_crtc_state->active_planes;
+		needs_to_disable |= new_crtc_state->has_psr2 != psr->psr2_enabled;
 
 		if (psr->enabled && needs_to_disable)
 			intel_psr_disable_locked(intel_dp);
diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c
index e415cd7c0b84..6e48d3bcdfec 100644
--- a/drivers/gpu/drm/i915/display/intel_quirks.c
+++ b/drivers/gpu/drm/i915/display/intel_quirks.c
@@ -9,12 +9,17 @@
 #include "intel_display_types.h"
 #include "intel_quirks.h"
 
+static void intel_set_quirk(struct drm_i915_private *i915, enum intel_quirk_id quirk)
+{
+	i915->display.quirks.mask |= BIT(quirk);
+}
+
 /*
  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
  */
 static void quirk_ssc_force_disable(struct drm_i915_private *i915)
 {
-	i915->quirks |= QUIRK_LVDS_SSC_DISABLE;
+	intel_set_quirk(i915, QUIRK_LVDS_SSC_DISABLE);
 	drm_info(&i915->drm, "applying lvds SSC disable quirk\n");
 }
 
@@ -24,14 +29,14 @@ static void quirk_ssc_force_disable(struct drm_i915_private *i915)
  */
 static void quirk_invert_brightness(struct drm_i915_private *i915)
 {
-	i915->quirks |= QUIRK_INVERT_BRIGHTNESS;
+	intel_set_quirk(i915, QUIRK_INVERT_BRIGHTNESS);
 	drm_info(&i915->drm, "applying inverted panel brightness quirk\n");
 }
 
 /* Some VBT's incorrectly indicate no backlight is present */
 static void quirk_backlight_present(struct drm_i915_private *i915)
 {
-	i915->quirks |= QUIRK_BACKLIGHT_PRESENT;
+	intel_set_quirk(i915, QUIRK_BACKLIGHT_PRESENT);
 	drm_info(&i915->drm, "applying backlight present quirk\n");
 }
 
@@ -40,7 +45,7 @@ static void quirk_backlight_present(struct drm_i915_private *i915)
  */
 static void quirk_increase_t12_delay(struct drm_i915_private *i915)
 {
-	i915->quirks |= QUIRK_INCREASE_T12_DELAY;
+	intel_set_quirk(i915, QUIRK_INCREASE_T12_DELAY);
 	drm_info(&i915->drm, "Applying T12 delay quirk\n");
 }
 
@@ -50,13 +55,13 @@ static void quirk_increase_t12_delay(struct drm_i915_private *i915)
  */
 static void quirk_increase_ddi_disabled_time(struct drm_i915_private *i915)
 {
-	i915->quirks |= QUIRK_INCREASE_DDI_DISABLED_TIME;
+	intel_set_quirk(i915, QUIRK_INCREASE_DDI_DISABLED_TIME);
 	drm_info(&i915->drm, "Applying Increase DDI Disabled quirk\n");
 }
 
 static void quirk_no_pps_backlight_power_hook(struct drm_i915_private *i915)
 {
-	i915->quirks |= QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK;
+	intel_set_quirk(i915, QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK);
 	drm_info(&i915->drm, "Applying no pps backlight power quirk\n");
 }
 
@@ -216,3 +221,8 @@ void intel_init_quirks(struct drm_i915_private *i915)
 			intel_dmi_quirks[i].hook(i915);
 	}
 }
+
+bool intel_has_quirk(struct drm_i915_private *i915, enum intel_quirk_id quirk)
+{
+	return i915->display.quirks.mask & BIT(quirk);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_quirks.h b/drivers/gpu/drm/i915/display/intel_quirks.h
index b0fcff142a56..10a4d163149f 100644
--- a/drivers/gpu/drm/i915/display/intel_quirks.h
+++ b/drivers/gpu/drm/i915/display/intel_quirks.h
@@ -6,8 +6,20 @@
 #ifndef __INTEL_QUIRKS_H__
 #define __INTEL_QUIRKS_H__
 
+#include <linux/types.h>
+
 struct drm_i915_private;
 
-void intel_init_quirks(struct drm_i915_private *dev_priv);
+enum intel_quirk_id {
+	QUIRK_BACKLIGHT_PRESENT,
+	QUIRK_INCREASE_DDI_DISABLED_TIME,
+	QUIRK_INCREASE_T12_DELAY,
+	QUIRK_INVERT_BRIGHTNESS,
+	QUIRK_LVDS_SSC_DISABLE,
+	QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK,
+};
+
+void intel_init_quirks(struct drm_i915_private *i915);
+bool intel_has_quirk(struct drm_i915_private *i915, enum intel_quirk_id quirk);
 
 #endif /* __INTEL_QUIRKS_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c
index 19122bc6d2ab..f5b744bef18f 100644
--- a/drivers/gpu/drm/i915/display/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/display/intel_sdvo.c
@@ -2016,7 +2016,7 @@ intel_sdvo_get_analog_edid(struct drm_connector *connector)
 
 	return drm_get_edid(connector,
 			    intel_gmbus_get_adapter(dev_priv,
-						    dev_priv->vbt.crt_ddc_pin));
+						    dev_priv->display.vbt.crt_ddc_pin));
 }
 
 static enum drm_connector_status
@@ -2581,9 +2581,9 @@ intel_sdvo_select_ddc_bus(struct drm_i915_private *dev_priv,
 	struct sdvo_device_mapping *mapping;
 
 	if (sdvo->port == PORT_B)
-		mapping = &dev_priv->vbt.sdvo_mappings[0];
+		mapping = &dev_priv->display.vbt.sdvo_mappings[0];
 	else
-		mapping = &dev_priv->vbt.sdvo_mappings[1];
+		mapping = &dev_priv->display.vbt.sdvo_mappings[1];
 
 	if (mapping->initialized)
 		sdvo->ddc_bus = 1 << ((mapping->ddc_pin & 0xf0) >> 4);
@@ -2599,9 +2599,9 @@ intel_sdvo_select_i2c_bus(struct drm_i915_private *dev_priv,
 	u8 pin;
 
 	if (sdvo->port == PORT_B)
-		mapping = &dev_priv->vbt.sdvo_mappings[0];
+		mapping = &dev_priv->display.vbt.sdvo_mappings[0];
 	else
-		mapping = &dev_priv->vbt.sdvo_mappings[1];
+		mapping = &dev_priv->display.vbt.sdvo_mappings[1];
 
 	if (mapping->initialized &&
 	    intel_gmbus_is_valid_pin(dev_priv, mapping->i2c_pin))
@@ -2639,11 +2639,11 @@ intel_sdvo_get_slave_addr(struct drm_i915_private *dev_priv,
 	struct sdvo_device_mapping *my_mapping, *other_mapping;
 
 	if (sdvo->port == PORT_B) {
-		my_mapping = &dev_priv->vbt.sdvo_mappings[0];
-		other_mapping = &dev_priv->vbt.sdvo_mappings[1];
+		my_mapping = &dev_priv->display.vbt.sdvo_mappings[0];
+		other_mapping = &dev_priv->display.vbt.sdvo_mappings[1];
 	} else {
-		my_mapping = &dev_priv->vbt.sdvo_mappings[1];
-		other_mapping = &dev_priv->vbt.sdvo_mappings[0];
+		my_mapping = &dev_priv->display.vbt.sdvo_mappings[1];
+		other_mapping = &dev_priv->display.vbt.sdvo_mappings[0];
 	}
 
 	/* If the BIOS described our SDVO device, take advantage of it. */
diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.c b/drivers/gpu/drm/i915/display/intel_snps_phy.c
index 0bdbedc67d7d..937cefd6f78f 100644
--- a/drivers/gpu/drm/i915/display/intel_snps_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_snps_phy.c
@@ -518,6 +518,1086 @@ static const struct intel_mpllb_state dg2_hdmi_148_5 = {
 };
 
 /* values in the below table are calculted using the algo */
+static const struct intel_mpllb_state dg2_hdmi_25200 = {
+	.clock = 25200,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 5) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 128) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 41943) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 2621),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_27027 = {
+	.clock = 27027,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 5) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 140) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 31876) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 46555),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_28320 = {
+	.clock = 28320,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 5) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 148) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 40894) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 30408),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_30240 = {
+	.clock = 30240,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 5) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 160) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 50331) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 42466),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_31500 = {
+	.clock = 31500,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 68) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 26214),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_36000 = {
+	.clock = 36000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 82) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 39321) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 39320),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_40000 = {
+	.clock = 40000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 96) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_49500 = {
+	.clock = 49500,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 1),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 126) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 13107) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 13107),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_50000 = {
+	.clock = 50000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 1),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 128) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_57284 = {
+	.clock = 57284,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 150) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 42886) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 49701),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_58000 = {
+	.clock = 58000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 4) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 152) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 52428) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 52427),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_65000 = {
+	.clock = 65000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 72) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_71000 = {
+	.clock = 71000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 80) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 52428) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 52427),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_74176 = {
+	.clock = 74176,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 22334) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 43829),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_75000 = {
+	.clock = 75000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 88) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_78750 = {
+	.clock = 78750,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 94) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_85500 = {
+	.clock = 85500,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 104) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 26214),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_88750 = {
+	.clock = 88750,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 1),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 110) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_106500 = {
+	.clock = 106500,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 138) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 13107) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 13107),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_108000 = {
+	.clock = 108000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 140) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 26214),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_115500 = {
+	.clock = 115500,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 152) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 26214),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_119000 = {
+	.clock = 119000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 3) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 158) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 13107) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 13107),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_135000 = {
+	.clock = 135000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 76) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 0),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_138500 = {
+	.clock = 138500,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 78) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 26214) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 26214),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_147160 = {
+	.clock = 147160,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 84) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 56623) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 6815),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_148352 = {
+	.clock = 148352,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 22334) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 43829),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_154000 = {
+	.clock = 154000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 13) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 90) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 39321) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 39320),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_162000 = {
+	.clock = 162000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 2),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 96) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 52428) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 52427),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_209800 = {
+	.clock = 209800,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 134) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 60293) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 7864),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_262750 = {
+	.clock = 262750,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 72) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 36044) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 52427),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_268500 = {
+	.clock = 268500,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 7) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 74) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 45875) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 13107),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_296703 = {
+	.clock = 296703,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 22321) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 36804),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_241500 = {
+	.clock = 241500,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 160) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 39321) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 39320),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_497750 = {
+	.clock = 497750,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 15) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 0),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 166) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 36044) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 52427),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_592000 = {
+	.clock = 592000,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 13107) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 13107),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
+static const struct intel_mpllb_state dg2_hdmi_593407 = {
+	.clock = 593407,
+	.ref_control =
+		REG_FIELD_PREP(SNPS_PHY_REF_CONTROL_REF_RANGE, 3),
+	.mpllb_cp =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT, 6) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP, 14) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_INT_GS, 64) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_CP_PROP_GS, 124),
+	.mpllb_div =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_DIV5_CLK_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_TX_CLK_DIV, 0) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_PMIX_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_V2I, 2) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FREQ_VCO, 3),
+	.mpllb_div2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_REF_CLK_DIV, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_MULTIPLIER, 86) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_HDMI_DIV, 1),
+	.mpllb_fracn1 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_CGG_UPDATE_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_EN, 1) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_DEN, 65535),
+	.mpllb_fracn2 =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_QUOT, 22328) |
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_FRACN_REM, 7549),
+	.mpllb_sscen =
+		REG_FIELD_PREP(SNPS_PHY_MPLLB_SSC_UP_SPREAD, 1),
+};
+
 static const struct intel_mpllb_state dg2_hdmi_297 = {
 	.clock = 297000,
 	.ref_control =
@@ -584,6 +1664,42 @@ static const struct intel_mpllb_state * const dg2_hdmi_tables[] = {
 	&dg2_hdmi_148_5,
 	&dg2_hdmi_297,
 	&dg2_hdmi_594,
+	&dg2_hdmi_25200,
+	&dg2_hdmi_27027,
+	&dg2_hdmi_28320,
+	&dg2_hdmi_30240,
+	&dg2_hdmi_31500,
+	&dg2_hdmi_36000,
+	&dg2_hdmi_40000,
+	&dg2_hdmi_49500,
+	&dg2_hdmi_50000,
+	&dg2_hdmi_57284,
+	&dg2_hdmi_58000,
+	&dg2_hdmi_65000,
+	&dg2_hdmi_71000,
+	&dg2_hdmi_74176,
+	&dg2_hdmi_75000,
+	&dg2_hdmi_78750,
+	&dg2_hdmi_85500,
+	&dg2_hdmi_88750,
+	&dg2_hdmi_106500,
+	&dg2_hdmi_108000,
+	&dg2_hdmi_115500,
+	&dg2_hdmi_119000,
+	&dg2_hdmi_135000,
+	&dg2_hdmi_138500,
+	&dg2_hdmi_147160,
+	&dg2_hdmi_148352,
+	&dg2_hdmi_154000,
+	&dg2_hdmi_162000,
+	&dg2_hdmi_209800,
+	&dg2_hdmi_241500,
+	&dg2_hdmi_262750,
+	&dg2_hdmi_268500,
+	&dg2_hdmi_296703,
+	&dg2_hdmi_497750,
+	&dg2_hdmi_592000,
+	&dg2_hdmi_593407,
 	NULL,
 };
 
diff --git a/drivers/gpu/drm/i915/display/intel_sprite.c b/drivers/gpu/drm/i915/display/intel_sprite.c
index 2713faad0625..7649c50b5445 100644
--- a/drivers/gpu/drm/i915/display/intel_sprite.c
+++ b/drivers/gpu/drm/i915/display/intel_sprite.c
@@ -39,7 +39,6 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_rect.h>
 
 #include "i915_drv.h"
@@ -1355,8 +1354,8 @@ g4x_sprite_check(struct intel_crtc_state *crtc_state,
 {
 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
-	int min_scale = DRM_PLANE_HELPER_NO_SCALING;
-	int max_scale = DRM_PLANE_HELPER_NO_SCALING;
+	int min_scale = DRM_PLANE_NO_SCALING;
+	int max_scale = DRM_PLANE_NO_SCALING;
 	int ret;
 
 	if (g4x_fb_scalable(plane_state->hw.fb)) {
@@ -1426,8 +1425,8 @@ vlv_sprite_check(struct intel_crtc_state *crtc_state,
 		return ret;
 
 	ret = intel_atomic_plane_check_clipping(plane_state, crtc_state,
-						DRM_PLANE_HELPER_NO_SCALING,
-						DRM_PLANE_HELPER_NO_SCALING,
+						DRM_PLANE_NO_SCALING,
+						DRM_PLANE_NO_SCALING,
 						true);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c
index 6773840f6cc7..e5af955b5600 100644
--- a/drivers/gpu/drm/i915/display/intel_tc.c
+++ b/drivers/gpu/drm/i915/display/intel_tc.c
@@ -246,7 +246,7 @@ static u32 icl_tc_port_live_status_mask(struct intel_digital_port *dig_port)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
 	struct intel_uncore *uncore = &i915->uncore;
-	u32 isr_bit = i915->hotplug.pch_hpd[dig_port->base.hpd_pin];
+	u32 isr_bit = i915->display.hotplug.pch_hpd[dig_port->base.hpd_pin];
 	u32 mask = 0;
 	u32 val;
 
@@ -279,7 +279,7 @@ static u32 adl_tc_port_live_status_mask(struct intel_digital_port *dig_port)
 {
 	struct drm_i915_private *i915 = to_i915(dig_port->base.base.dev);
 	enum tc_port tc_port = intel_port_to_tc(i915, dig_port->base.port);
-	u32 isr_bit = i915->hotplug.pch_hpd[dig_port->base.hpd_pin];
+	u32 isr_bit = i915->display.hotplug.pch_hpd[dig_port->base.hpd_pin];
 	struct intel_uncore *uncore = &i915->uncore;
 	u32 val, mask = 0;
 
diff --git a/drivers/gpu/drm/i915/display/intel_tv.c b/drivers/gpu/drm/i915/display/intel_tv.c
index 9379f3463344..dcf89d701f0f 100644
--- a/drivers/gpu/drm/i915/display/intel_tv.c
+++ b/drivers/gpu/drm/i915/display/intel_tv.c
@@ -39,6 +39,7 @@
 #include "intel_crtc.h"
 #include "intel_de.h"
 #include "intel_display_types.h"
+#include "intel_dpll.h"
 #include "intel_hotplug.h"
 #include "intel_tv.h"
 
@@ -982,10 +983,10 @@ intel_tv_mode_vdisplay(const struct tv_mode *tv_mode)
 
 static void
 intel_tv_mode_to_mode(struct drm_display_mode *mode,
-		      const struct tv_mode *tv_mode)
+		      const struct tv_mode *tv_mode,
+		      int clock)
 {
-	mode->clock = tv_mode->clock /
-		(tv_mode->oversample >> !tv_mode->progressive);
+	mode->clock = clock / (tv_mode->oversample >> !tv_mode->progressive);
 
 	/*
 	 * tv_mode horizontal timings:
@@ -1143,7 +1144,7 @@ intel_tv_get_config(struct intel_encoder *encoder,
 	xsize = tmp >> 16;
 	ysize = tmp & 0xffff;
 
-	intel_tv_mode_to_mode(&mode, &tv_mode);
+	intel_tv_mode_to_mode(&mode, &tv_mode, pipe_config->port_clock);
 
 	drm_dbg_kms(&dev_priv->drm, "TV mode: " DRM_MODE_FMT "\n",
 		    DRM_MODE_ARG(&mode));
@@ -1184,6 +1185,9 @@ intel_tv_compute_config(struct intel_encoder *encoder,
 			struct intel_crtc_state *pipe_config,
 			struct drm_connector_state *conn_state)
 {
+	struct intel_atomic_state *state =
+		to_intel_atomic_state(pipe_config->uapi.state);
+	struct intel_crtc *crtc = to_intel_crtc(pipe_config->uapi.crtc);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_tv_connector_state *tv_conn_state =
 		to_intel_tv_connector_state(conn_state);
@@ -1192,6 +1196,7 @@ intel_tv_compute_config(struct intel_encoder *encoder,
 		&pipe_config->hw.adjusted_mode;
 	int hdisplay = adjusted_mode->crtc_hdisplay;
 	int vdisplay = adjusted_mode->crtc_vdisplay;
+	int ret;
 
 	if (!tv_mode)
 		return -EINVAL;
@@ -1206,7 +1211,13 @@ intel_tv_compute_config(struct intel_encoder *encoder,
 
 	pipe_config->port_clock = tv_mode->clock;
 
-	intel_tv_mode_to_mode(adjusted_mode, tv_mode);
+	ret = intel_dpll_crtc_compute_clock(state, crtc);
+	if (ret)
+		return ret;
+
+	pipe_config->clock_set = true;
+
+	intel_tv_mode_to_mode(adjusted_mode, tv_mode, pipe_config->port_clock);
 	drm_mode_set_crtcinfo(adjusted_mode, 0);
 
 	if (intel_tv_source_too_wide(dev_priv, hdisplay) ||
@@ -1804,7 +1815,7 @@ intel_tv_get_modes(struct drm_connector *connector)
 		 * about the actual timings of the mode. We
 		 * do ignore the margins though.
 		 */
-		intel_tv_mode_to_mode(mode, tv_mode);
+		intel_tv_mode_to_mode(mode, tv_mode, tv_mode->clock);
 		if (count == 0) {
 			drm_dbg_kms(&dev_priv->drm, "TV mode: " DRM_MODE_FMT "\n",
 				    DRM_MODE_ARG(mode));
diff --git a/drivers/gpu/drm/i915/display/intel_vbt_defs.h b/drivers/gpu/drm/i915/display/intel_vbt_defs.h
index 509b0a419c20..a9f44abfc9fc 100644
--- a/drivers/gpu/drm/i915/display/intel_vbt_defs.h
+++ b/drivers/gpu/drm/i915/display/intel_vbt_defs.h
@@ -76,6 +76,20 @@ struct bdb_header {
 } __packed;
 
 /*
+ * BDB version number dependencies are documented as:
+ *
+ * <start>+
+ *    indicates the field was introduced in version <start>
+ *    and is still valid
+ *
+ * <start>-<end>
+ *    indicates the field was introduced in version <start>
+ *    and obsoleted in version <end>+1.
+ *
+ * ??? indicates the specific version number is unknown
+ */
+
+/*
  * There are several types of BIOS data blocks (BDBs), each block has
  * an ID and size in the first 3 bytes (ID in first, size in next 2).
  * Known types are listed below.
@@ -144,12 +158,12 @@ struct bdb_general_features {
         /* bits 3 */
 	u8 disable_smooth_vision:1;
 	u8 single_dvi:1;
-	u8 rotate_180:1;					/* 181 */
+	u8 rotate_180:1;					/* 181+ */
 	u8 fdi_rx_polarity_inverted:1;
-	u8 vbios_extended_mode:1;				/* 160 */
-	u8 copy_ilfp_dtd_to_sdvo_lvds_dtd:1;			/* 160 */
-	u8 panel_best_fit_timing:1;				/* 160 */
-	u8 ignore_strap_state:1;				/* 160 */
+	u8 vbios_extended_mode:1;				/* 160+ */
+	u8 copy_ilfp_dtd_to_sdvo_lvds_dtd:1;			/* 160+ */
+	u8 panel_best_fit_timing:1;				/* 160+ */
+	u8 ignore_strap_state:1;				/* 160+ */
 
         /* bits 4 */
 	u8 legacy_monitor_detect;
@@ -164,11 +178,11 @@ struct bdb_general_features {
 	u8 rsvd11:2; /* finish byte */
 
 	/* bits 6 */
-	u8 tc_hpd_retry_timeout:7; /* 242 */
+	u8 tc_hpd_retry_timeout:7;				/* 242+ */
 	u8 rsvd12:1;
 
 	/* bits 7 */
-	u8 afc_startup_config:2;/* 249 */
+	u8 afc_startup_config:2;				/* 249+ */
 	u8 rsvd13:6;
 } __packed;
 
@@ -183,6 +197,15 @@ struct bdb_general_features {
 #define GPIO_PIN_ADD_DDC_I2C	0x06 /* "ADDCARD DDC/I2C GPIO pins" */
 
 /* Device handle */
+#define DEVICE_HANDLE_CRT	0x0001
+#define DEVICE_HANDLE_EFP1	0x0004
+#define DEVICE_HANDLE_EFP2	0x0040
+#define DEVICE_HANDLE_EFP3	0x0020
+#define DEVICE_HANDLE_EFP4	0x0010 /* 194+ */
+#define DEVICE_HANDLE_EFP5	0x0002 /* 215+ */
+#define DEVICE_HANDLE_EFP6	0x0001 /* 217+ */
+#define DEVICE_HANDLE_EFP7	0x0100 /* 217+ */
+#define DEVICE_HANDLE_EFP8	0x0200 /* 217+ */
 #define DEVICE_HANDLE_LFP1	0x0008
 #define DEVICE_HANDLE_LFP2	0x0080
 
@@ -275,27 +298,27 @@ struct bdb_general_features {
 #define DVO_PORT_DPC		8
 #define DVO_PORT_DPD		9
 #define DVO_PORT_DPA		10
-#define DVO_PORT_DPE		11				/* 193 */
-#define DVO_PORT_HDMIE		12				/* 193 */
+#define DVO_PORT_DPE		11				/* 193+ */
+#define DVO_PORT_HDMIE		12				/* 193+ */
 #define DVO_PORT_DPF		13				/* N/A */
 #define DVO_PORT_HDMIF		14				/* N/A */
-#define DVO_PORT_DPG		15				/* 217 */
-#define DVO_PORT_HDMIG		16				/* 217 */
-#define DVO_PORT_DPH		17				/* 217 */
-#define DVO_PORT_HDMIH		18				/* 217 */
-#define DVO_PORT_DPI		19				/* 217 */
-#define DVO_PORT_HDMII		20				/* 217 */
-#define DVO_PORT_MIPIA		21				/* 171 */
-#define DVO_PORT_MIPIB		22				/* 171 */
-#define DVO_PORT_MIPIC		23				/* 171 */
-#define DVO_PORT_MIPID		24				/* 171 */
-
-#define HDMI_MAX_DATA_RATE_PLATFORM	0			/* 204 */
-#define HDMI_MAX_DATA_RATE_297		1			/* 204 */
-#define HDMI_MAX_DATA_RATE_165		2			/* 204 */
-#define HDMI_MAX_DATA_RATE_594		3			/* 249 */
-#define HDMI_MAX_DATA_RATE_340		4			/* 249 */
-#define HDMI_MAX_DATA_RATE_300		5			/* 249 */
+#define DVO_PORT_DPG		15				/* 217+ */
+#define DVO_PORT_HDMIG		16				/* 217+ */
+#define DVO_PORT_DPH		17				/* 217+ */
+#define DVO_PORT_HDMIH		18				/* 217+ */
+#define DVO_PORT_DPI		19				/* 217+ */
+#define DVO_PORT_HDMII		20				/* 217+ */
+#define DVO_PORT_MIPIA		21				/* 171+ */
+#define DVO_PORT_MIPIB		22				/* 171+ */
+#define DVO_PORT_MIPIC		23				/* 171+ */
+#define DVO_PORT_MIPID		24				/* 171+ */
+
+#define HDMI_MAX_DATA_RATE_PLATFORM	0			/* 204+ */
+#define HDMI_MAX_DATA_RATE_297		1			/* 204+ */
+#define HDMI_MAX_DATA_RATE_165		2			/* 204+ */
+#define HDMI_MAX_DATA_RATE_594		3			/* 249+ */
+#define HDMI_MAX_DATA_RATE_340		4			/* 249+ */
+#define HDMI_MAX_DATA_RATE_300		5			/* 249+ */
 
 #define LEGACY_CHILD_DEVICE_CONFIG_SIZE		33
 
@@ -362,10 +385,10 @@ enum vbt_gmbus_ddi {
  * basically any of the fields to ensure the correct interpretation for the BDB
  * version in question.
  *
- * When we copy the child device configs to dev_priv->vbt.child_dev, we reserve
- * space for the full structure below, and initialize the tail not actually
- * present in VBT to zeros. Accessing those fields is fine, as long as the
- * default zero is taken into account, again according to the BDB version.
+ * When we copy the child device configs to dev_priv->display.vbt.child_dev, we
+ * reserve space for the full structure below, and initialize the tail not
+ * actually present in VBT to zeros. Accessing those fields is fine, as long as
+ * the default zero is taken into account, again according to the BDB version.
  *
  * BDB versions 155 and below are considered legacy, and version 155 seems to be
  * a baseline for some of the VBT documentation. When adding new fields, please
@@ -379,20 +402,30 @@ struct child_device_config {
 		u8  device_id[10]; /* ascii string */
 		struct {
 			u8 i2c_speed;
-			u8 dp_onboard_redriver;			/* 158 */
-			u8 dp_ondock_redriver;			/* 158 */
-			u8 hdmi_level_shifter_value:5;		/* 169 */
-			u8 hdmi_max_data_rate:3;		/* 204 */
-			u16 dtd_buf_ptr;			/* 161 */
-			u8 edidless_efp:1;			/* 161 */
-			u8 compression_enable:1;		/* 198 */
-			u8 compression_method_cps:1;		/* 198 */
-			u8 ganged_edp:1;			/* 202 */
-			u8 reserved0:4;
-			u8 compression_structure_index:4;	/* 198 */
-			u8 reserved1:4;
-			u8 slave_port;				/* 202 */
-			u8 reserved2;
+			u8 dp_onboard_redriver_preemph:3;	/* 158+ */
+			u8 dp_onboard_redriver_vswing:3;	/* 158+ */
+			u8 dp_onboard_redriver_present:1;	/* 158+ */
+			u8 reserved0:1;
+			u8 dp_ondock_redriver_preemph:3;	/* 158+ */
+			u8 dp_ondock_redriver_vswing:3;		/* 158+ */
+			u8 dp_ondock_redriver_present:1;	/* 158+ */
+			u8 reserved1:1;
+			u8 hdmi_level_shifter_value:5;		/* 158+ */
+			u8 hdmi_max_data_rate:3;		/* 204+ */
+			u16 dtd_buf_ptr;			/* 161+ */
+			u8 edidless_efp:1;			/* 161+ */
+			u8 compression_enable:1;		/* 198+ */
+			u8 compression_method_cps:1;		/* 198+ */
+			u8 ganged_edp:1;			/* 202+ */
+			u8 lttpr_non_transparent:1;		/* 235+ */
+			u8 disable_compression_for_ext_disp:1;	/* 251+ */
+			u8 reserved2:2;
+			u8 compression_structure_index:4;	/* 198+ */
+			u8 reserved3:4;
+			u8 hdmi_max_frl_rate:4;			/* 237+ */
+			u8 hdmi_max_frl_rate_valid:1;		/* 237+ */
+			u8 reserved4:3;				/* 237+ */
+			u8 reserved5;
 		} __packed;
 	} __packed;
 
@@ -412,16 +445,16 @@ struct child_device_config {
 			u8 ddc2_pin;
 		} __packed;
 		struct {
-			u8 efp_routed:1;			/* 158 */
-			u8 lane_reversal:1;			/* 184 */
-			u8 lspcon:1;				/* 192 */
-			u8 iboost:1;				/* 196 */
-			u8 hpd_invert:1;			/* 196 */
-			u8 use_vbt_vswing:1;			/* 218 */
-			u8 flag_reserved:2;
-			u8 hdmi_support:1;			/* 158 */
-			u8 dp_support:1;			/* 158 */
-			u8 tmds_support:1;			/* 158 */
+			u8 efp_routed:1;			/* 158+ */
+			u8 lane_reversal:1;			/* 184+ */
+			u8 lspcon:1;				/* 192+ */
+			u8 iboost:1;				/* 196+ */
+			u8 hpd_invert:1;			/* 196+ */
+			u8 use_vbt_vswing:1;			/* 218+ */
+			u8 dp_max_lane_count:2;			/* 244+ */
+			u8 hdmi_support:1;			/* 158+ */
+			u8 dp_support:1;			/* 158+ */
+			u8 tmds_support:1;			/* 158+ */
 			u8 support_reserved:5;
 			u8 aux_channel;
 			u8 dongle_detect;
@@ -429,7 +462,7 @@ struct child_device_config {
 	} __packed;
 
 	u8 pipe_cap:2;
-	u8 sdvo_stall:1;					/* 158 */
+	u8 sdvo_stall:1;					/* 158+ */
 	u8 hpd_status:2;
 	u8 integrated_encoder:1;
 	u8 capabilities_reserved:2;
@@ -437,21 +470,21 @@ struct child_device_config {
 
 	union {
 		u8 dvo2_wiring;
-		u8 mipi_bridge_type;				/* 171 */
+		u8 mipi_bridge_type;				/* 171+ */
 	} __packed;
 
 	u16 extended_type;
 	u8 dvo_function;
-	u8 dp_usb_type_c:1;					/* 195 */
-	u8 tbt:1;						/* 209 */
-	u8 flags2_reserved:2;					/* 195 */
-	u8 dp_port_trace_length:4;				/* 209 */
-	u8 dp_gpio_index;					/* 195 */
-	u16 dp_gpio_pin_num;					/* 195 */
-	u8 dp_iboost_level:4;					/* 196 */
-	u8 hdmi_iboost_level:4;					/* 196 */
-	u8 dp_max_link_rate:3;					/* 216/230 GLK+ */
-	u8 dp_max_link_rate_reserved:5;				/* 216/230 */
+	u8 dp_usb_type_c:1;					/* 195+ */
+	u8 tbt:1;						/* 209+ */
+	u8 flags2_reserved:2;					/* 195+ */
+	u8 dp_port_trace_length:4;				/* 209+ */
+	u8 dp_gpio_index;					/* 195+ */
+	u16 dp_gpio_pin_num;					/* 195+ */
+	u8 dp_iboost_level:4;					/* 196+ */
+	u8 hdmi_iboost_level:4;					/* 196+ */
+	u8 dp_max_link_rate:3;					/* 216+ */
+	u8 dp_max_link_rate_reserved:5;				/* 216+ */
 } __packed;
 
 struct bdb_general_definitions {
@@ -459,7 +492,7 @@ struct bdb_general_definitions {
 	u8 crt_ddc_gmbus_pin;
 
 	/* DPMS bits */
-	u8 dpms_acpi:1;
+	u8 dpms_non_acpi:1;
 	u8 skip_boot_crt_detect:1;
 	u8 dpms_aim:1;
 	u8 rsvd1:5; /* finish byte */
@@ -488,25 +521,25 @@ struct bdb_general_definitions {
 
 struct psr_table {
 	/* Feature bits */
-	u8 full_link:1;
-	u8 require_aux_to_wakeup:1;
+	u8 full_link:1;						/* 165+ */
+	u8 require_aux_to_wakeup:1;				/* 165+ */
 	u8 feature_bits_rsvd:6;
 
 	/* Wait times */
-	u8 idle_frames:4;
-	u8 lines_to_wait:3;
+	u8 idle_frames:4;					/* 165+ */
+	u8 lines_to_wait:3;					/* 165+ */
 	u8 wait_times_rsvd:1;
 
 	/* TP wake up time in multiple of 100 */
-	u16 tp1_wakeup_time;
-	u16 tp2_tp3_wakeup_time;
+	u16 tp1_wakeup_time;					/* 165+ */
+	u16 tp2_tp3_wakeup_time;				/* 165+ */
 } __packed;
 
 struct bdb_psr {
 	struct psr_table psr_table[16];
 
 	/* PSR2 TP2/TP3 wakeup time for 16 panels */
-	u32 psr2_tp2_tp3_wakeup_time;
+	u32 psr2_tp2_tp3_wakeup_time;				/* 226+ */
 } __packed;
 
 /*
@@ -519,9 +552,10 @@ struct bdb_psr {
 #define BDB_DRIVER_FEATURE_INT_SDVO_LVDS	3
 
 struct bdb_driver_features {
+	/* Driver bits */
 	u8 boot_dev_algorithm:1;
-	u8 block_display_switch:1;
-	u8 allow_display_switch:1;
+	u8 allow_display_switch_dvd:1;
+	u8 allow_display_switch_dos:1;
 	u8 hotplug_dvo:1;
 	u8 dual_view_zoom:1;
 	u8 int15h_hook:1;
@@ -533,6 +567,7 @@ struct bdb_driver_features {
 	u8 boot_mode_bpp;
 	u8 boot_mode_refresh;
 
+	/* Extended Driver Bits 1 */
 	u16 enable_lfp_primary:1;
 	u16 selective_mode_pruning:1;
 	u16 dual_frequency:1;
@@ -548,29 +583,40 @@ struct bdb_driver_features {
 	u16 tv_hotplug:1;
 	u16 hdmi_config:2;
 
-	u8 static_display:1;
-	u8 reserved2:7;
+	/* Driver Flags 1 */
+	u8 static_display:1;					/* 163+ */
+	u8 embedded_platform:1;					/* 163+ */
+	u8 display_subsystem_enable:1;				/* 163+ */
+	u8 reserved0:5;
+
 	u16 legacy_crt_max_x;
 	u16 legacy_crt_max_y;
 	u8 legacy_crt_max_refresh;
 
-	u8 hdmi_termination;
-	u8 custom_vbt_version;
-	/* Driver features data block */
-	u16 rmpm_enabled:1;
-	u16 s2ddt_enabled:1;
-	u16 dpst_enabled:1;
-	u16 bltclt_enabled:1;
-	u16 adb_enabled:1;
-	u16 drrs_enabled:1;
-	u16 grs_enabled:1;
-	u16 gpmt_enabled:1;
-	u16 tbt_enabled:1;
-	u16 psr_enabled:1;
-	u16 ips_enabled:1;
-	u16 reserved3:1;
-	u16 dmrrs_enabled:1;
-	u16 reserved4:2;
+	/* Extended Driver Bits 2 */
+	u8 hdmi_termination:1;
+	u8 cea861d_hdmi_support:1;
+	u8 self_refresh_enable:1;
+	u8 reserved1:5;
+
+	u8 custom_vbt_version;					/* 155+ */
+
+	/* Driver Feature Flags */
+	u16 rmpm_enabled:1;					/* 165+ */
+	u16 s2ddt_enabled:1;					/* 165+ */
+	u16 dpst_enabled:1;					/* 165-227 */
+	u16 bltclt_enabled:1;					/* 165+ */
+	u16 adb_enabled:1;					/* 165-227 */
+	u16 drrs_enabled:1;					/* 165-227 */
+	u16 grs_enabled:1;					/* 165+ */
+	u16 gpmt_enabled:1;					/* 165+ */
+	u16 tbt_enabled:1;					/* 165+ */
+	u16 psr_enabled:1;					/* 165-227 */
+	u16 ips_enabled:1;					/* 165+ */
+	u16 dpfs_enabled:1;					/* 165+ */
+	u16 dmrrs_enabled:1;					/* 174-227 */
+	u16 adt_enabled:1;					/* ???-228 */
+	u16 hpd_wake:1;						/* 201-240 */
 	u16 pc_feature_valid:1;
 } __packed;
 
@@ -657,7 +703,7 @@ struct bdb_sdvo_panel_dtds {
 
 
 struct edp_fast_link_params {
-	u8 rate:4;
+	u8 rate:4;						/* ???-223 */
 	u8 lanes:4;
 	u8 preemphasis:4;
 	u8 vswing:4;
@@ -690,18 +736,18 @@ struct bdb_edp {
 	u32 sdrrs_msa_timing_delay;
 
 	/* ith bit indicates enabled/disabled for (i+1)th panel */
-	u16 edp_s3d_feature;					/* 162 */
-	u16 edp_t3_optimization;				/* 165 */
-	u64 edp_vswing_preemph;					/* 173 */
-	u16 fast_link_training;					/* 182 */
-	u16 dpcd_600h_write_required;				/* 185 */
-	struct edp_pwm_delays pwm_delays[16];			/* 186 */
-	u16 full_link_params_provided;				/* 199 */
-	struct edp_full_link_params full_link_params[16];	/* 199 */
-	u16 apical_enable;					/* 203 */
-	struct edp_apical_params apical_params[16];		/* 203 */
-	u16 edp_fast_link_training_rate[16];			/* 224 */
-	u16 edp_max_port_link_rate[16];				/* 244 */
+	u16 edp_s3d_feature;					/* 162+ */
+	u16 edp_t3_optimization;				/* 165+ */
+	u64 edp_vswing_preemph;					/* 173+ */
+	u16 fast_link_training;					/* 182+ */
+	u16 dpcd_600h_write_required;				/* 185+ */
+	struct edp_pwm_delays pwm_delays[16];			/* 186+ */
+	u16 full_link_params_provided;				/* 199+ */
+	struct edp_full_link_params full_link_params[16];	/* 199+ */
+	u16 apical_enable;					/* 203+ */
+	struct edp_apical_params apical_params[16];		/* 203+ */
+	u16 edp_fast_link_training_rate[16];			/* 224+ */
+	u16 edp_max_port_link_rate[16];				/* 244+ */
 } __packed;
 
 /*
@@ -710,14 +756,14 @@ struct bdb_edp {
 
 struct bdb_lvds_options {
 	u8 panel_type;
-	u8 panel_type2;						/* 212 */
+	u8 panel_type2;						/* 212+ */
 	/* LVDS capabilities, stored in a dword */
 	u8 pfit_mode:2;
 	u8 pfit_text_mode_enhanced:1;
 	u8 pfit_gfx_mode_enhanced:1;
 	u8 pfit_ratio_auto:1;
 	u8 pixel_dither:1;
-	u8 lvds_edid:1;
+	u8 lvds_edid:1;						/* ???-240 */
 	u8 rsvd2:1;
 	u8 rsvd4;
 	/* LVDS Panel channel bits stored here */
@@ -731,11 +777,11 @@ struct bdb_lvds_options {
 	/* LVDS panel type bits stored here */
 	u32 dps_panel_type_bits;
 	/* LVDS backlight control type bits stored here */
-	u32 blt_control_type_bits;
+	u32 blt_control_type_bits;				/* ???-240 */
 
-	u16 lcdvcc_s0_enable;					/* 200 */
-	u32 rotation;						/* 228 */
-	u32 position;						/* 240 */
+	u16 lcdvcc_s0_enable;					/* 200+ */
+	u32 rotation;						/* 228+ */
+	u32 position;						/* 240+ */
 } __packed;
 
 /*
@@ -756,7 +802,7 @@ struct lvds_lfp_data_ptr {
 struct bdb_lvds_lfp_data_ptrs {
 	u8 lvds_entries;
 	struct lvds_lfp_data_ptr ptr[16];
-	struct lvds_lfp_data_ptr_table panel_name; /* 156-163? */
+	struct lvds_lfp_data_ptr_table panel_name;		/* (156-163?)+ */
 } __packed;
 
 /*
@@ -808,20 +854,20 @@ struct lvds_lfp_panel_name {
 } __packed;
 
 struct lvds_lfp_black_border {
-	u8 top; /* 227 */
-	u8 bottom; /* 227 */
-	u8 left; /* 238 */
-	u8 right; /* 238 */
+	u8 top;		/* 227+ */
+	u8 bottom;	/* 227+ */
+	u8 left;	/* 238+ */
+	u8 right;	/* 238+ */
 } __packed;
 
 struct bdb_lvds_lfp_data_tail {
-	struct lvds_lfp_panel_name panel_name[16]; /* 156-163? */
-	u16 scaling_enable; /* 187 */
-	u8 seamless_drrs_min_refresh_rate[16]; /* 188 */
-	u8 pixel_overlap_count[16]; /* 208 */
-	struct lvds_lfp_black_border black_border[16]; /* 227 */
-	u16 dual_lfp_port_sync_enable; /* 231 */
-	u16 gpu_dithering_for_banding_artifacts; /* 245 */
+	struct lvds_lfp_panel_name panel_name[16];		/* (156-163?)+ */
+	u16 scaling_enable;					/* 187+ */
+	u8 seamless_drrs_min_refresh_rate[16];			/* 188+ */
+	u8 pixel_overlap_count[16];				/* 208+ */
+	struct lvds_lfp_black_border black_border[16];		/* 227+ */
+	u16 dual_lfp_port_sync_enable;				/* 231+ */
+	u16 gpu_dithering_for_banding_artifacts;		/* 245+ */
 } __packed;
 
 /*
@@ -836,7 +882,7 @@ struct lfp_backlight_data_entry {
 	u8 active_low_pwm:1;
 	u8 obsolete1:5;
 	u16 pwm_freq_hz;
-	u8 min_brightness; /* Obsolete from 234+ */
+	u8 min_brightness;					/* ???-233 */
 	u8 obsolete2;
 	u8 obsolete3;
 } __packed;
@@ -859,7 +905,7 @@ struct lfp_brightness_level {
 struct bdb_lfp_backlight_data {
 	u8 entry_size;
 	struct lfp_backlight_data_entry data[16];
-	u8 level[16]; /* Obsolete from 234+ */
+	u8 level[16];							/* ???-233 */
 	struct lfp_backlight_control_method backlight_control[16];
 	struct lfp_brightness_level brightness_level[16];		/* 234+ */
 	struct lfp_brightness_level brightness_min_level[16];		/* 234+ */
@@ -874,8 +920,8 @@ struct lfp_power_features {
 	u8 reserved1:1;
 	u8 power_conservation_pref:3;
 	u8 reserved2:1;
-	u8 lace_enabled_status:1;
-	u8 lace_support:1;
+	u8 lace_enabled_status:1;					/* 210+ */
+	u8 lace_support:1;						/* 210+ */
 	u8 als_enable:1;
 } __packed;
 
@@ -895,24 +941,24 @@ struct aggressiveness_profile2_entry {
 } __packed;
 
 struct bdb_lfp_power {
-	struct lfp_power_features features;
+	struct lfp_power_features features;				/* ???-227 */
 	struct als_data_entry als[5];
-	u8 lace_aggressiveness_profile:3;
+	u8 lace_aggressiveness_profile:3;				/* 210-227 */
 	u8 reserved1:5;
-	u16 dpst;
-	u16 psr;
-	u16 drrs;
-	u16 lace_support;
-	u16 adt;
-	u16 dmrrs;
-	u16 adb;
-	u16 lace_enabled_status;
-	struct aggressiveness_profile_entry aggressiveness[16];
-	u16 hobl; /* 232+ */
-	u16 vrr_feature_enabled; /* 233+ */
-	u16 elp; /* 247+ */
-	u16 opst; /* 247+ */
-	struct aggressiveness_profile2_entry aggressiveness2[16]; /* 247+ */
+	u16 dpst;							/* 228+ */
+	u16 psr;							/* 228+ */
+	u16 drrs;							/* 228+ */
+	u16 lace_support;						/* 228+ */
+	u16 adt;							/* 228+ */
+	u16 dmrrs;							/* 228+ */
+	u16 adb;							/* 228+ */
+	u16 lace_enabled_status;					/* 228+ */
+	struct aggressiveness_profile_entry aggressiveness[16];		/* 228+ */
+	u16 hobl;							/* 232+ */
+	u16 vrr_feature_enabled;					/* 233+ */
+	u16 elp;							/* 247+ */
+	u16 opst;							/* 247+ */
+	struct aggressiveness_profile2_entry aggressiveness2[16];	/* 247+ */
 } __packed;
 
 /*
@@ -922,10 +968,10 @@ struct bdb_lfp_power {
 #define MAX_MIPI_CONFIGURATIONS	6
 
 struct bdb_mipi_config {
-	struct mipi_config config[MAX_MIPI_CONFIGURATIONS]; /* 175 */
-	struct mipi_pps_data pps[MAX_MIPI_CONFIGURATIONS]; /* 177 */
-	struct edp_pwm_delays pwm_delays[MAX_MIPI_CONFIGURATIONS]; /* 186 */
-	u8 pmic_i2c_bus_number[MAX_MIPI_CONFIGURATIONS]; /* 190 */
+	struct mipi_config config[MAX_MIPI_CONFIGURATIONS];		/* 175+ */
+	struct mipi_pps_data pps[MAX_MIPI_CONFIGURATIONS];		/* 177+ */
+	struct edp_pwm_delays pwm_delays[MAX_MIPI_CONFIGURATIONS];	/* 186+ */
+	u8 pmic_i2c_bus_number[MAX_MIPI_CONFIGURATIONS];		/* 190+ */
 } __packed;
 
 /*
diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c
index ca530f0733e0..269f9792390d 100644
--- a/drivers/gpu/drm/i915/display/intel_vdsc.c
+++ b/drivers/gpu/drm/i915/display/intel_vdsc.c
@@ -344,7 +344,7 @@ bool intel_dsc_source_support(const struct intel_crtc_state *crtc_state)
 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 
-	if (!INTEL_INFO(i915)->display.has_dsc)
+	if (!RUNTIME_INFO(i915)->has_dsc)
 		return false;
 
 	if (DISPLAY_VER(i915) >= 12)
@@ -596,6 +596,8 @@ static void intel_dsc_pps_configure(const struct intel_crtc_state *crtc_state)
 		DSC_VER_MIN_SHIFT |
 		vdsc_cfg->bits_per_component << DSC_BPC_SHIFT |
 		vdsc_cfg->line_buf_depth << DSC_LINE_BUF_DEPTH_SHIFT;
+	if (vdsc_cfg->dsc_version_minor == 2)
+		pps_val |= DSC_ALT_ICH_SEL;
 	if (vdsc_cfg->block_pred_enable)
 		pps_val |= DSC_BLOCK_PREDICTION;
 	if (vdsc_cfg->convert_rgb)
diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c
index 04250a0fec3c..5eac99021875 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr.c
+++ b/drivers/gpu/drm/i915/display/intel_vrr.c
@@ -142,11 +142,16 @@ intel_vrr_compute_config(struct intel_crtc_state *crtc_state,
 	 * For XE_LPD+, we use guardband and pipeline override
 	 * is deprecated.
 	 */
-	if (DISPLAY_VER(i915) >= 13)
+	if (DISPLAY_VER(i915) >= 13) {
+		/*
+		 * FIXME: Subtract Window2 delay from below value.
+		 *
+		 * Window2 specifies time required to program DSB (Window2) in
+		 * number of scan lines. Assuming 0 for no DSB.
+		 */
 		crtc_state->vrr.guardband =
-			crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay -
-			i915->window2_delay;
-	else
+			crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay;
+	} else {
 		/*
 		 * FIXME: s/4/framestart_delay/ to get consistent
 		 * earliest/latest points for register latching regardless
@@ -159,6 +164,7 @@ intel_vrr_compute_config(struct intel_crtc_state *crtc_state,
 		 */
 		crtc_state->vrr.pipeline_full =
 			min(255, crtc_state->vrr.vmin - adjusted_mode->crtc_vdisplay - 4 - 1);
+	}
 
 	crtc_state->mode_flags |= I915_MODE_FLAG_VRR;
 }
diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c
index c11e15a93164..7cb713043408 100644
--- a/drivers/gpu/drm/i915/display/skl_universal_plane.c
+++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c
@@ -7,7 +7,6 @@
 #include <drm/drm_blend.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 
 #include "i915_drv.h"
 #include "intel_atomic_plane.h"
@@ -15,11 +14,11 @@
 #include "intel_display_types.h"
 #include "intel_fb.h"
 #include "intel_fbc.h"
-#include "intel_pm.h"
 #include "intel_psr.h"
 #include "intel_sprite.h"
 #include "skl_scaler.h"
 #include "skl_universal_plane.h"
+#include "skl_watermark.h"
 #include "pxp/intel_pxp.h"
 
 static const u32 skl_plane_formats[] = {
@@ -1856,8 +1855,8 @@ static int skl_plane_check(struct intel_crtc_state *crtc_state,
 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	const struct drm_framebuffer *fb = plane_state->hw.fb;
-	int min_scale = DRM_PLANE_HELPER_NO_SCALING;
-	int max_scale = DRM_PLANE_HELPER_NO_SCALING;
+	int min_scale = DRM_PLANE_NO_SCALING;
+	int max_scale = DRM_PLANE_NO_SCALING;
 	int ret;
 
 	ret = skl_plane_check_fb(crtc_state, plane_state);
@@ -1929,7 +1928,7 @@ static enum intel_fbc_id skl_fbc_id_for_pipe(enum pipe pipe)
 static bool skl_plane_has_fbc(struct drm_i915_private *dev_priv,
 			      enum intel_fbc_id fbc_id, enum plane_id plane_id)
 {
-	if ((INTEL_INFO(dev_priv)->display.fbc_mask & BIT(fbc_id)) == 0)
+	if ((RUNTIME_INFO(dev_priv)->fbc_mask & BIT(fbc_id)) == 0)
 		return false;
 
 	return plane_id == PLANE_PRIMARY;
@@ -1941,7 +1940,7 @@ static struct intel_fbc *skl_plane_fbc(struct drm_i915_private *dev_priv,
 	enum intel_fbc_id fbc_id = skl_fbc_id_for_pipe(pipe);
 
 	if (skl_plane_has_fbc(dev_priv, fbc_id, plane_id))
-		return dev_priv->fbc[fbc_id];
+		return dev_priv->display.fbc[fbc_id];
 	else
 		return NULL;
 }
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c
new file mode 100644
index 000000000000..01b0932757ed
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/skl_watermark.c
@@ -0,0 +1,3562 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#include <drm/drm_blend.h>
+
+#include "intel_atomic.h"
+#include "intel_atomic_plane.h"
+#include "intel_bw.h"
+#include "intel_de.h"
+#include "intel_display.h"
+#include "intel_display_power.h"
+#include "intel_display_types.h"
+#include "intel_fb.h"
+#include "skl_watermark.h"
+
+#include "i915_drv.h"
+#include "i915_fixed.h"
+#include "i915_reg.h"
+#include "intel_pcode.h"
+#include "intel_pm.h"
+
+static void skl_sagv_disable(struct drm_i915_private *i915);
+
+/* Stores plane specific WM parameters */
+struct skl_wm_params {
+	bool x_tiled, y_tiled;
+	bool rc_surface;
+	bool is_planar;
+	u32 width;
+	u8 cpp;
+	u32 plane_pixel_rate;
+	u32 y_min_scanlines;
+	u32 plane_bytes_per_line;
+	uint_fixed_16_16_t plane_blocks_per_line;
+	uint_fixed_16_16_t y_tile_minimum;
+	u32 linetime_us;
+	u32 dbuf_block_size;
+};
+
+u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915)
+{
+	u8 enabled_slices = 0;
+	enum dbuf_slice slice;
+
+	for_each_dbuf_slice(i915, slice) {
+		if (intel_uncore_read(&i915->uncore,
+				      DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
+			enabled_slices |= BIT(slice);
+	}
+
+	return enabled_slices;
+}
+
+/*
+ * FIXME: We still don't have the proper code detect if we need to apply the WA,
+ * so assume we'll always need it in order to avoid underruns.
+ */
+static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
+{
+	return DISPLAY_VER(i915) == 9;
+}
+
+static bool
+intel_has_sagv(struct drm_i915_private *i915)
+{
+	return DISPLAY_VER(i915) >= 9 && !IS_LP(i915) &&
+		i915->display.sagv.status != I915_SAGV_NOT_CONTROLLED;
+}
+
+static u32
+intel_sagv_block_time(struct drm_i915_private *i915)
+{
+	if (DISPLAY_VER(i915) >= 14) {
+		u32 val;
+
+		val = intel_uncore_read(&i915->uncore, MTL_LATENCY_SAGV);
+
+		return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
+	} else if (DISPLAY_VER(i915) >= 12) {
+		u32 val = 0;
+		int ret;
+
+		ret = snb_pcode_read(&i915->uncore,
+				     GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
+				     &val, NULL);
+		if (ret) {
+			drm_dbg_kms(&i915->drm, "Couldn't read SAGV block time!\n");
+			return 0;
+		}
+
+		return val;
+	} else if (DISPLAY_VER(i915) == 11) {
+		return 10;
+	} else if (DISPLAY_VER(i915) == 9 && !IS_LP(i915)) {
+		return 30;
+	} else {
+		return 0;
+	}
+}
+
+static void intel_sagv_init(struct drm_i915_private *i915)
+{
+	if (!intel_has_sagv(i915))
+		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
+
+	/*
+	 * Probe to see if we have working SAGV control.
+	 * For icl+ this was already determined by intel_bw_init_hw().
+	 */
+	if (DISPLAY_VER(i915) < 11)
+		skl_sagv_disable(i915);
+
+	drm_WARN_ON(&i915->drm, i915->display.sagv.status == I915_SAGV_UNKNOWN);
+
+	i915->display.sagv.block_time_us = intel_sagv_block_time(i915);
+
+	drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
+		    str_yes_no(intel_has_sagv(i915)), i915->display.sagv.block_time_us);
+
+	/* avoid overflow when adding with wm0 latency/etc. */
+	if (drm_WARN(&i915->drm, i915->display.sagv.block_time_us > U16_MAX,
+		     "Excessive SAGV block time %u, ignoring\n",
+		     i915->display.sagv.block_time_us))
+		i915->display.sagv.block_time_us = 0;
+
+	if (!intel_has_sagv(i915))
+		i915->display.sagv.block_time_us = 0;
+}
+
+/*
+ * SAGV dynamically adjusts the system agent voltage and clock frequencies
+ * depending on power and performance requirements. The display engine access
+ * to system memory is blocked during the adjustment time. Because of the
+ * blocking time, having this enabled can cause full system hangs and/or pipe
+ * underruns if we don't meet all of the following requirements:
+ *
+ *  - <= 1 pipe enabled
+ *  - All planes can enable watermarks for latencies >= SAGV engine block time
+ *  - We're not using an interlaced display configuration
+ */
+static void skl_sagv_enable(struct drm_i915_private *i915)
+{
+	int ret;
+
+	if (!intel_has_sagv(i915))
+		return;
+
+	if (i915->display.sagv.status == I915_SAGV_ENABLED)
+		return;
+
+	drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
+	ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
+			      GEN9_SAGV_ENABLE);
+
+	/* We don't need to wait for SAGV when enabling */
+
+	/*
+	 * Some skl systems, pre-release machines in particular,
+	 * don't actually have SAGV.
+	 */
+	if (IS_SKYLAKE(i915) && ret == -ENXIO) {
+		drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
+		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
+		return;
+	} else if (ret < 0) {
+		drm_err(&i915->drm, "Failed to enable SAGV\n");
+		return;
+	}
+
+	i915->display.sagv.status = I915_SAGV_ENABLED;
+}
+
+static void skl_sagv_disable(struct drm_i915_private *i915)
+{
+	int ret;
+
+	if (!intel_has_sagv(i915))
+		return;
+
+	if (i915->display.sagv.status == I915_SAGV_DISABLED)
+		return;
+
+	drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
+	/* bspec says to keep retrying for at least 1 ms */
+	ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
+				GEN9_SAGV_DISABLE,
+				GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
+				1);
+	/*
+	 * Some skl systems, pre-release machines in particular,
+	 * don't actually have SAGV.
+	 */
+	if (IS_SKYLAKE(i915) && ret == -ENXIO) {
+		drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
+		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
+		return;
+	} else if (ret < 0) {
+		drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
+		return;
+	}
+
+	i915->display.sagv.status = I915_SAGV_DISABLED;
+}
+
+static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_bw_state *new_bw_state =
+		intel_atomic_get_new_bw_state(state);
+
+	if (!new_bw_state)
+		return;
+
+	if (!intel_can_enable_sagv(i915, new_bw_state))
+		skl_sagv_disable(i915);
+}
+
+static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_bw_state *new_bw_state =
+		intel_atomic_get_new_bw_state(state);
+
+	if (!new_bw_state)
+		return;
+
+	if (intel_can_enable_sagv(i915, new_bw_state))
+		skl_sagv_enable(i915);
+}
+
+static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_bw_state *old_bw_state =
+		intel_atomic_get_old_bw_state(state);
+	const struct intel_bw_state *new_bw_state =
+		intel_atomic_get_new_bw_state(state);
+	u16 old_mask, new_mask;
+
+	if (!new_bw_state)
+		return;
+
+	old_mask = old_bw_state->qgv_points_mask;
+	new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
+
+	if (old_mask == new_mask)
+		return;
+
+	WARN_ON(!new_bw_state->base.changed);
+
+	drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
+		    old_mask, new_mask);
+
+	/*
+	 * Restrict required qgv points before updating the configuration.
+	 * According to BSpec we can't mask and unmask qgv points at the same
+	 * time. Also masking should be done before updating the configuration
+	 * and unmasking afterwards.
+	 */
+	icl_pcode_restrict_qgv_points(i915, new_mask);
+}
+
+static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_bw_state *old_bw_state =
+		intel_atomic_get_old_bw_state(state);
+	const struct intel_bw_state *new_bw_state =
+		intel_atomic_get_new_bw_state(state);
+	u16 old_mask, new_mask;
+
+	if (!new_bw_state)
+		return;
+
+	old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
+	new_mask = new_bw_state->qgv_points_mask;
+
+	if (old_mask == new_mask)
+		return;
+
+	WARN_ON(!new_bw_state->base.changed);
+
+	drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
+		    old_mask, new_mask);
+
+	/*
+	 * Allow required qgv points after updating the configuration.
+	 * According to BSpec we can't mask and unmask qgv points at the same
+	 * time. Also masking should be done before updating the configuration
+	 * and unmasking afterwards.
+	 */
+	icl_pcode_restrict_qgv_points(i915, new_mask);
+}
+
+void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+
+	/*
+	 * Just return if we can't control SAGV or don't have it.
+	 * This is different from situation when we have SAGV but just can't
+	 * afford it due to DBuf limitation - in case if SAGV is completely
+	 * disabled in a BIOS, we are not even allowed to send a PCode request,
+	 * as it will throw an error. So have to check it here.
+	 */
+	if (!intel_has_sagv(i915))
+		return;
+
+	if (DISPLAY_VER(i915) >= 11)
+		icl_sagv_pre_plane_update(state);
+	else
+		skl_sagv_pre_plane_update(state);
+}
+
+void intel_sagv_post_plane_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+
+	/*
+	 * Just return if we can't control SAGV or don't have it.
+	 * This is different from situation when we have SAGV but just can't
+	 * afford it due to DBuf limitation - in case if SAGV is completely
+	 * disabled in a BIOS, we are not even allowed to send a PCode request,
+	 * as it will throw an error. So have to check it here.
+	 */
+	if (!intel_has_sagv(i915))
+		return;
+
+	if (DISPLAY_VER(i915) >= 11)
+		icl_sagv_post_plane_update(state);
+	else
+		skl_sagv_post_plane_update(state);
+}
+
+static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	enum plane_id plane_id;
+	int max_level = INT_MAX;
+
+	if (!intel_has_sagv(i915))
+		return false;
+
+	if (!crtc_state->hw.active)
+		return true;
+
+	if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
+		return false;
+
+	for_each_plane_id_on_crtc(crtc, plane_id) {
+		const struct skl_plane_wm *wm =
+			&crtc_state->wm.skl.optimal.planes[plane_id];
+		int level;
+
+		/* Skip this plane if it's not enabled */
+		if (!wm->wm[0].enable)
+			continue;
+
+		/* Find the highest enabled wm level for this plane */
+		for (level = ilk_wm_max_level(i915);
+		     !wm->wm[level].enable; --level)
+		     { }
+
+		/* Highest common enabled wm level for all planes */
+		max_level = min(level, max_level);
+	}
+
+	/* No enabled planes? */
+	if (max_level == INT_MAX)
+		return true;
+
+	for_each_plane_id_on_crtc(crtc, plane_id) {
+		const struct skl_plane_wm *wm =
+			&crtc_state->wm.skl.optimal.planes[plane_id];
+
+		/*
+		 * All enabled planes must have enabled a common wm level that
+		 * can tolerate memory latencies higher than sagv_block_time_us
+		 */
+		if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
+			return false;
+	}
+
+	return true;
+}
+
+static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	enum plane_id plane_id;
+
+	if (!crtc_state->hw.active)
+		return true;
+
+	for_each_plane_id_on_crtc(crtc, plane_id) {
+		const struct skl_plane_wm *wm =
+			&crtc_state->wm.skl.optimal.planes[plane_id];
+
+		if (wm->wm[0].enable && !wm->sagv.wm0.enable)
+			return false;
+	}
+
+	return true;
+}
+
+static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+
+	if (DISPLAY_VER(i915) >= 12)
+		return tgl_crtc_can_enable_sagv(crtc_state);
+	else
+		return skl_crtc_can_enable_sagv(crtc_state);
+}
+
+bool intel_can_enable_sagv(struct drm_i915_private *i915,
+			   const struct intel_bw_state *bw_state)
+{
+	if (DISPLAY_VER(i915) < 11 &&
+	    bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
+		return false;
+
+	return bw_state->pipe_sagv_reject == 0;
+}
+
+static int intel_compute_sagv_mask(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	int ret;
+	struct intel_crtc *crtc;
+	struct intel_crtc_state *new_crtc_state;
+	struct intel_bw_state *new_bw_state = NULL;
+	const struct intel_bw_state *old_bw_state = NULL;
+	int i;
+
+	for_each_new_intel_crtc_in_state(state, crtc,
+					 new_crtc_state, i) {
+		new_bw_state = intel_atomic_get_bw_state(state);
+		if (IS_ERR(new_bw_state))
+			return PTR_ERR(new_bw_state);
+
+		old_bw_state = intel_atomic_get_old_bw_state(state);
+
+		if (intel_crtc_can_enable_sagv(new_crtc_state))
+			new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
+		else
+			new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
+	}
+
+	if (!new_bw_state)
+		return 0;
+
+	new_bw_state->active_pipes =
+		intel_calc_active_pipes(state, old_bw_state->active_pipes);
+
+	if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
+		ret = intel_atomic_lock_global_state(&new_bw_state->base);
+		if (ret)
+			return ret;
+	}
+
+	if (intel_can_enable_sagv(i915, new_bw_state) !=
+	    intel_can_enable_sagv(i915, old_bw_state)) {
+		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
+		if (ret)
+			return ret;
+	} else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
+		ret = intel_atomic_lock_global_state(&new_bw_state->base);
+		if (ret)
+			return ret;
+	}
+
+	for_each_new_intel_crtc_in_state(state, crtc,
+					 new_crtc_state, i) {
+		struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
+
+		/*
+		 * We store use_sagv_wm in the crtc state rather than relying on
+		 * that bw state since we have no convenient way to get at the
+		 * latter from the plane commit hooks (especially in the legacy
+		 * cursor case)
+		 */
+		pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(i915) &&
+			DISPLAY_VER(i915) >= 12 &&
+			intel_can_enable_sagv(i915, new_bw_state);
+	}
+
+	return 0;
+}
+
+static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
+			      u16 start, u16 end)
+{
+	entry->start = start;
+	entry->end = end;
+
+	return end;
+}
+
+static int intel_dbuf_slice_size(struct drm_i915_private *i915)
+{
+	return INTEL_INFO(i915)->display.dbuf.size /
+		hweight8(INTEL_INFO(i915)->display.dbuf.slice_mask);
+}
+
+static void
+skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
+			 struct skl_ddb_entry *ddb)
+{
+	int slice_size = intel_dbuf_slice_size(i915);
+
+	if (!slice_mask) {
+		ddb->start = 0;
+		ddb->end = 0;
+		return;
+	}
+
+	ddb->start = (ffs(slice_mask) - 1) * slice_size;
+	ddb->end = fls(slice_mask) * slice_size;
+
+	WARN_ON(ddb->start >= ddb->end);
+	WARN_ON(ddb->end > INTEL_INFO(i915)->display.dbuf.size);
+}
+
+static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
+{
+	struct skl_ddb_entry ddb;
+
+	if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
+		slice_mask = BIT(DBUF_S1);
+	else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
+		slice_mask = BIT(DBUF_S3);
+
+	skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
+
+	return ddb.start;
+}
+
+u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
+			    const struct skl_ddb_entry *entry)
+{
+	int slice_size = intel_dbuf_slice_size(i915);
+	enum dbuf_slice start_slice, end_slice;
+	u8 slice_mask = 0;
+
+	if (!skl_ddb_entry_size(entry))
+		return 0;
+
+	start_slice = entry->start / slice_size;
+	end_slice = (entry->end - 1) / slice_size;
+
+	/*
+	 * Per plane DDB entry can in a really worst case be on multiple slices
+	 * but single entry is anyway contigious.
+	 */
+	while (start_slice <= end_slice) {
+		slice_mask |= BIT(start_slice);
+		start_slice++;
+	}
+
+	return slice_mask;
+}
+
+static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
+{
+	const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
+	int hdisplay, vdisplay;
+
+	if (!crtc_state->hw.active)
+		return 0;
+
+	/*
+	 * Watermark/ddb requirement highly depends upon width of the
+	 * framebuffer, So instead of allocating DDB equally among pipes
+	 * distribute DDB based on resolution/width of the display.
+	 */
+	drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
+
+	return hdisplay;
+}
+
+static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
+				    enum pipe for_pipe,
+				    unsigned int *weight_start,
+				    unsigned int *weight_end,
+				    unsigned int *weight_total)
+{
+	struct drm_i915_private *i915 =
+		to_i915(dbuf_state->base.state->base.dev);
+	enum pipe pipe;
+
+	*weight_start = 0;
+	*weight_end = 0;
+	*weight_total = 0;
+
+	for_each_pipe(i915, pipe) {
+		int weight = dbuf_state->weight[pipe];
+
+		/*
+		 * Do not account pipes using other slice sets
+		 * luckily as of current BSpec slice sets do not partially
+		 * intersect(pipes share either same one slice or same slice set
+		 * i.e no partial intersection), so it is enough to check for
+		 * equality for now.
+		 */
+		if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
+			continue;
+
+		*weight_total += weight;
+		if (pipe < for_pipe) {
+			*weight_start += weight;
+			*weight_end += weight;
+		} else if (pipe == for_pipe) {
+			*weight_end += weight;
+		}
+	}
+}
+
+static int
+skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	unsigned int weight_total, weight_start, weight_end;
+	const struct intel_dbuf_state *old_dbuf_state =
+		intel_atomic_get_old_dbuf_state(state);
+	struct intel_dbuf_state *new_dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+	struct intel_crtc_state *crtc_state;
+	struct skl_ddb_entry ddb_slices;
+	enum pipe pipe = crtc->pipe;
+	unsigned int mbus_offset = 0;
+	u32 ddb_range_size;
+	u32 dbuf_slice_mask;
+	u32 start, end;
+	int ret;
+
+	if (new_dbuf_state->weight[pipe] == 0) {
+		skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
+		goto out;
+	}
+
+	dbuf_slice_mask = new_dbuf_state->slices[pipe];
+
+	skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices);
+	mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask);
+	ddb_range_size = skl_ddb_entry_size(&ddb_slices);
+
+	intel_crtc_dbuf_weights(new_dbuf_state, pipe,
+				&weight_start, &weight_end, &weight_total);
+
+	start = ddb_range_size * weight_start / weight_total;
+	end = ddb_range_size * weight_end / weight_total;
+
+	skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
+			   ddb_slices.start - mbus_offset + start,
+			   ddb_slices.start - mbus_offset + end);
+
+out:
+	if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
+	    skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
+				&new_dbuf_state->ddb[pipe]))
+		return 0;
+
+	ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
+	if (ret)
+		return ret;
+
+	crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
+
+	/*
+	 * Used for checking overlaps, so we need absolute
+	 * offsets instead of MBUS relative offsets.
+	 */
+	crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
+	crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
+
+	drm_dbg_kms(&i915->drm,
+		    "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
+		    crtc->base.base.id, crtc->base.name,
+		    old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
+		    old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
+		    new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
+		    old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
+
+	return 0;
+}
+
+static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
+				 int width, const struct drm_format_info *format,
+				 u64 modifier, unsigned int rotation,
+				 u32 plane_pixel_rate, struct skl_wm_params *wp,
+				 int color_plane);
+
+static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
+				 struct intel_plane *plane,
+				 int level,
+				 unsigned int latency,
+				 const struct skl_wm_params *wp,
+				 const struct skl_wm_level *result_prev,
+				 struct skl_wm_level *result /* out */);
+
+static unsigned int
+skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
+		      int num_active)
+{
+	struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	int level, max_level = ilk_wm_max_level(i915);
+	struct skl_wm_level wm = {};
+	int ret, min_ddb_alloc = 0;
+	struct skl_wm_params wp;
+
+	ret = skl_compute_wm_params(crtc_state, 256,
+				    drm_format_info(DRM_FORMAT_ARGB8888),
+				    DRM_FORMAT_MOD_LINEAR,
+				    DRM_MODE_ROTATE_0,
+				    crtc_state->pixel_rate, &wp, 0);
+	drm_WARN_ON(&i915->drm, ret);
+
+	for (level = 0; level <= max_level; level++) {
+		unsigned int latency = i915->display.wm.skl_latency[level];
+
+		skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
+		if (wm.min_ddb_alloc == U16_MAX)
+			break;
+
+		min_ddb_alloc = wm.min_ddb_alloc;
+	}
+
+	return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
+}
+
+static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
+{
+	skl_ddb_entry_init(entry,
+			   REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
+			   REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
+	if (entry->end)
+		entry->end++;
+}
+
+static void
+skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
+			   const enum pipe pipe,
+			   const enum plane_id plane_id,
+			   struct skl_ddb_entry *ddb,
+			   struct skl_ddb_entry *ddb_y)
+{
+	u32 val;
+
+	/* Cursor doesn't support NV12/planar, so no extra calculation needed */
+	if (plane_id == PLANE_CURSOR) {
+		val = intel_uncore_read(&i915->uncore, CUR_BUF_CFG(pipe));
+		skl_ddb_entry_init_from_hw(ddb, val);
+		return;
+	}
+
+	val = intel_uncore_read(&i915->uncore, PLANE_BUF_CFG(pipe, plane_id));
+	skl_ddb_entry_init_from_hw(ddb, val);
+
+	if (DISPLAY_VER(i915) >= 11)
+		return;
+
+	val = intel_uncore_read(&i915->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
+	skl_ddb_entry_init_from_hw(ddb_y, val);
+}
+
+static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
+				      struct skl_ddb_entry *ddb,
+				      struct skl_ddb_entry *ddb_y)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	enum intel_display_power_domain power_domain;
+	enum pipe pipe = crtc->pipe;
+	intel_wakeref_t wakeref;
+	enum plane_id plane_id;
+
+	power_domain = POWER_DOMAIN_PIPE(pipe);
+	wakeref = intel_display_power_get_if_enabled(i915, power_domain);
+	if (!wakeref)
+		return;
+
+	for_each_plane_id_on_crtc(crtc, plane_id)
+		skl_ddb_get_hw_plane_state(i915, pipe,
+					   plane_id,
+					   &ddb[plane_id],
+					   &ddb_y[plane_id]);
+
+	intel_display_power_put(i915, power_domain, wakeref);
+}
+
+struct dbuf_slice_conf_entry {
+	u8 active_pipes;
+	u8 dbuf_mask[I915_MAX_PIPES];
+	bool join_mbus;
+};
+
+/*
+ * Table taken from Bspec 12716
+ * Pipes do have some preferred DBuf slice affinity,
+ * plus there are some hardcoded requirements on how
+ * those should be distributed for multipipe scenarios.
+ * For more DBuf slices algorithm can get even more messy
+ * and less readable, so decided to use a table almost
+ * as is from BSpec itself - that way it is at least easier
+ * to compare, change and check.
+ */
+static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
+/* Autogenerated with igt/tools/intel_dbuf_map tool: */
+{
+	{
+		.active_pipes = BIT(PIPE_A),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_B] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_C] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_C] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1),
+			[PIPE_C] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_B] = BIT(DBUF_S1),
+			[PIPE_C] = BIT(DBUF_S2),
+		},
+	},
+	{}
+};
+
+/*
+ * Table taken from Bspec 49255
+ * Pipes do have some preferred DBuf slice affinity,
+ * plus there are some hardcoded requirements on how
+ * those should be distributed for multipipe scenarios.
+ * For more DBuf slices algorithm can get even more messy
+ * and less readable, so decided to use a table almost
+ * as is from BSpec itself - that way it is at least easier
+ * to compare, change and check.
+ */
+static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
+/* Autogenerated with igt/tools/intel_dbuf_map tool: */
+{
+	{
+		.active_pipes = BIT(PIPE_A),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S2),
+			[PIPE_B] = BIT(DBUF_S1),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_C] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1),
+			[PIPE_C] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_B] = BIT(DBUF_S1),
+			[PIPE_C] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_D] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1),
+			[PIPE_D] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_B] = BIT(DBUF_S1),
+			[PIPE_D] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_C] = BIT(DBUF_S1),
+			[PIPE_D] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_C] = BIT(DBUF_S2),
+			[PIPE_D] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1),
+			[PIPE_C] = BIT(DBUF_S2),
+			[PIPE_D] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_B] = BIT(DBUF_S1),
+			[PIPE_C] = BIT(DBUF_S2),
+			[PIPE_D] = BIT(DBUF_S2),
+		},
+	},
+	{}
+};
+
+static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
+	{
+		.active_pipes = BIT(PIPE_A),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_B] = BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_B] = BIT(DBUF_S2),
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_B] = BIT(DBUF_S2),
+			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_C] = BIT(DBUF_S3),
+			[PIPE_D] = BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_C] = BIT(DBUF_S3),
+			[PIPE_D] = BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_C] = BIT(DBUF_S3),
+			[PIPE_D] = BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1),
+			[PIPE_B] = BIT(DBUF_S2),
+			[PIPE_C] = BIT(DBUF_S3),
+			[PIPE_D] = BIT(DBUF_S4),
+		},
+	},
+	{}
+};
+
+static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
+	/*
+	 * Keep the join_mbus cases first so check_mbus_joined()
+	 * will prefer them over the !join_mbus cases.
+	 */
+	{
+		.active_pipes = BIT(PIPE_A),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+		.join_mbus = true,
+	},
+	{
+		.active_pipes = BIT(PIPE_B),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+		.join_mbus = true,
+	},
+	{
+		.active_pipes = BIT(PIPE_A),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+		.join_mbus = false,
+	},
+	{
+		.active_pipes = BIT(PIPE_B),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+		.join_mbus = false,
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
+			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
+			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{
+		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
+		.dbuf_mask = {
+			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
+			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
+			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
+			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
+		},
+	},
+	{}
+
+};
+
+static bool check_mbus_joined(u8 active_pipes,
+			      const struct dbuf_slice_conf_entry *dbuf_slices)
+{
+	int i;
+
+	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
+		if (dbuf_slices[i].active_pipes == active_pipes)
+			return dbuf_slices[i].join_mbus;
+	}
+	return false;
+}
+
+static bool adlp_check_mbus_joined(u8 active_pipes)
+{
+	return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
+}
+
+static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
+			      const struct dbuf_slice_conf_entry *dbuf_slices)
+{
+	int i;
+
+	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
+		if (dbuf_slices[i].active_pipes == active_pipes &&
+		    dbuf_slices[i].join_mbus == join_mbus)
+			return dbuf_slices[i].dbuf_mask[pipe];
+	}
+	return 0;
+}
+
+/*
+ * This function finds an entry with same enabled pipe configuration and
+ * returns correspondent DBuf slice mask as stated in BSpec for particular
+ * platform.
+ */
+static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
+{
+	/*
+	 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
+	 * required calculating "pipe ratio" in order to determine
+	 * if one or two slices can be used for single pipe configurations
+	 * as additional constraint to the existing table.
+	 * However based on recent info, it should be not "pipe ratio"
+	 * but rather ratio between pixel_rate and cdclk with additional
+	 * constants, so for now we are using only table until this is
+	 * clarified. Also this is the reason why crtc_state param is
+	 * still here - we will need it once those additional constraints
+	 * pop up.
+	 */
+	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
+				   icl_allowed_dbufs);
+}
+
+static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
+{
+	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
+				   tgl_allowed_dbufs);
+}
+
+static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
+{
+	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
+				   adlp_allowed_dbufs);
+}
+
+static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
+{
+	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
+				   dg2_allowed_dbufs);
+}
+
+static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	enum pipe pipe = crtc->pipe;
+
+	if (IS_DG2(i915))
+		return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
+	else if (DISPLAY_VER(i915) >= 13)
+		return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
+	else if (DISPLAY_VER(i915) == 12)
+		return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
+	else if (DISPLAY_VER(i915) == 11)
+		return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
+	/*
+	 * For anything else just return one slice yet.
+	 * Should be extended for other platforms.
+	 */
+	return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
+}
+
+static bool
+use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
+		     struct intel_plane *plane)
+{
+	struct drm_i915_private *i915 = to_i915(plane->base.dev);
+
+	return DISPLAY_VER(i915) >= 13 &&
+	       crtc_state->uapi.async_flip &&
+	       plane->async_flip;
+}
+
+static u64
+skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	enum plane_id plane_id;
+	u64 data_rate = 0;
+
+	for_each_plane_id_on_crtc(crtc, plane_id) {
+		if (plane_id == PLANE_CURSOR)
+			continue;
+
+		data_rate += crtc_state->rel_data_rate[plane_id];
+
+		if (DISPLAY_VER(i915) < 11)
+			data_rate += crtc_state->rel_data_rate_y[plane_id];
+	}
+
+	return data_rate;
+}
+
+static const struct skl_wm_level *
+skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
+		   enum plane_id plane_id,
+		   int level)
+{
+	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
+
+	if (level == 0 && pipe_wm->use_sagv_wm)
+		return &wm->sagv.wm0;
+
+	return &wm->wm[level];
+}
+
+static const struct skl_wm_level *
+skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
+		   enum plane_id plane_id)
+{
+	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
+
+	if (pipe_wm->use_sagv_wm)
+		return &wm->sagv.trans_wm;
+
+	return &wm->trans_wm;
+}
+
+/*
+ * We only disable the watermarks for each plane if
+ * they exceed the ddb allocation of said plane. This
+ * is done so that we don't end up touching cursor
+ * watermarks needlessly when some other plane reduces
+ * our max possible watermark level.
+ *
+ * Bspec has this to say about the PLANE_WM enable bit:
+ * "All the watermarks at this level for all enabled
+ *  planes must be enabled before the level will be used."
+ * So this is actually safe to do.
+ */
+static void
+skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
+{
+	if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
+		memset(wm, 0, sizeof(*wm));
+}
+
+static void
+skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
+			const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
+{
+	if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
+	    uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
+		memset(wm, 0, sizeof(*wm));
+		memset(uv_wm, 0, sizeof(*uv_wm));
+	}
+}
+
+static bool icl_need_wm1_wa(struct drm_i915_private *i915,
+			    enum plane_id plane_id)
+{
+	/*
+	 * Wa_1408961008:icl, ehl
+	 * Wa_14012656716:tgl, adl
+	 * Underruns with WM1+ disabled
+	 */
+	return DISPLAY_VER(i915) == 11 ||
+	       (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR);
+}
+
+struct skl_plane_ddb_iter {
+	u64 data_rate;
+	u16 start, size;
+};
+
+static void
+skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
+		       struct skl_ddb_entry *ddb,
+		       const struct skl_wm_level *wm,
+		       u64 data_rate)
+{
+	u16 size, extra = 0;
+
+	if (data_rate) {
+		extra = min_t(u16, iter->size,
+			      DIV64_U64_ROUND_UP(iter->size * data_rate,
+						 iter->data_rate));
+		iter->size -= extra;
+		iter->data_rate -= data_rate;
+	}
+
+	/*
+	 * Keep ddb entry of all disabled planes explicitly zeroed
+	 * to avoid skl_ddb_add_affected_planes() adding them to
+	 * the state when other planes change their allocations.
+	 */
+	size = wm->min_ddb_alloc + extra;
+	if (size)
+		iter->start = skl_ddb_entry_init(ddb, iter->start,
+						 iter->start + size);
+}
+
+static int
+skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
+			    struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct intel_crtc_state *crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	const struct intel_dbuf_state *dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+	const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
+	int num_active = hweight8(dbuf_state->active_pipes);
+	struct skl_plane_ddb_iter iter;
+	enum plane_id plane_id;
+	u16 cursor_size;
+	u32 blocks;
+	int level;
+
+	/* Clear the partitioning for disabled planes. */
+	memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
+	memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
+
+	if (!crtc_state->hw.active)
+		return 0;
+
+	iter.start = alloc->start;
+	iter.size = skl_ddb_entry_size(alloc);
+	if (iter.size == 0)
+		return 0;
+
+	/* Allocate fixed number of blocks for cursor. */
+	cursor_size = skl_cursor_allocation(crtc_state, num_active);
+	iter.size -= cursor_size;
+	skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
+			   alloc->end - cursor_size, alloc->end);
+
+	iter.data_rate = skl_total_relative_data_rate(crtc_state);
+
+	/*
+	 * Find the highest watermark level for which we can satisfy the block
+	 * requirement of active planes.
+	 */
+	for (level = ilk_wm_max_level(i915); level >= 0; level--) {
+		blocks = 0;
+		for_each_plane_id_on_crtc(crtc, plane_id) {
+			const struct skl_plane_wm *wm =
+				&crtc_state->wm.skl.optimal.planes[plane_id];
+
+			if (plane_id == PLANE_CURSOR) {
+				const struct skl_ddb_entry *ddb =
+					&crtc_state->wm.skl.plane_ddb[plane_id];
+
+				if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
+					drm_WARN_ON(&i915->drm,
+						    wm->wm[level].min_ddb_alloc != U16_MAX);
+					blocks = U32_MAX;
+					break;
+				}
+				continue;
+			}
+
+			blocks += wm->wm[level].min_ddb_alloc;
+			blocks += wm->uv_wm[level].min_ddb_alloc;
+		}
+
+		if (blocks <= iter.size) {
+			iter.size -= blocks;
+			break;
+		}
+	}
+
+	if (level < 0) {
+		drm_dbg_kms(&i915->drm,
+			    "Requested display configuration exceeds system DDB limitations");
+		drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
+			    blocks, iter.size);
+		return -EINVAL;
+	}
+
+	/* avoid the WARN later when we don't allocate any extra DDB */
+	if (iter.data_rate == 0)
+		iter.size = 0;
+
+	/*
+	 * Grant each plane the blocks it requires at the highest achievable
+	 * watermark level, plus an extra share of the leftover blocks
+	 * proportional to its relative data rate.
+	 */
+	for_each_plane_id_on_crtc(crtc, plane_id) {
+		struct skl_ddb_entry *ddb =
+			&crtc_state->wm.skl.plane_ddb[plane_id];
+		struct skl_ddb_entry *ddb_y =
+			&crtc_state->wm.skl.plane_ddb_y[plane_id];
+		const struct skl_plane_wm *wm =
+			&crtc_state->wm.skl.optimal.planes[plane_id];
+
+		if (plane_id == PLANE_CURSOR)
+			continue;
+
+		if (DISPLAY_VER(i915) < 11 &&
+		    crtc_state->nv12_planes & BIT(plane_id)) {
+			skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
+					       crtc_state->rel_data_rate_y[plane_id]);
+			skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
+					       crtc_state->rel_data_rate[plane_id]);
+		} else {
+			skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
+					       crtc_state->rel_data_rate[plane_id]);
+		}
+	}
+	drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
+
+	/*
+	 * When we calculated watermark values we didn't know how high
+	 * of a level we'd actually be able to hit, so we just marked
+	 * all levels as "enabled."  Go back now and disable the ones
+	 * that aren't actually possible.
+	 */
+	for (level++; level <= ilk_wm_max_level(i915); level++) {
+		for_each_plane_id_on_crtc(crtc, plane_id) {
+			const struct skl_ddb_entry *ddb =
+				&crtc_state->wm.skl.plane_ddb[plane_id];
+			const struct skl_ddb_entry *ddb_y =
+				&crtc_state->wm.skl.plane_ddb_y[plane_id];
+			struct skl_plane_wm *wm =
+				&crtc_state->wm.skl.optimal.planes[plane_id];
+
+			if (DISPLAY_VER(i915) < 11 &&
+			    crtc_state->nv12_planes & BIT(plane_id))
+				skl_check_nv12_wm_level(&wm->wm[level],
+							&wm->uv_wm[level],
+							ddb_y, ddb);
+			else
+				skl_check_wm_level(&wm->wm[level], ddb);
+
+			if (icl_need_wm1_wa(i915, plane_id) &&
+			    level == 1 && wm->wm[0].enable) {
+				wm->wm[level].blocks = wm->wm[0].blocks;
+				wm->wm[level].lines = wm->wm[0].lines;
+				wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
+			}
+		}
+	}
+
+	/*
+	 * Go back and disable the transition and SAGV watermarks
+	 * if it turns out we don't have enough DDB blocks for them.
+	 */
+	for_each_plane_id_on_crtc(crtc, plane_id) {
+		const struct skl_ddb_entry *ddb =
+			&crtc_state->wm.skl.plane_ddb[plane_id];
+		const struct skl_ddb_entry *ddb_y =
+			&crtc_state->wm.skl.plane_ddb_y[plane_id];
+		struct skl_plane_wm *wm =
+			&crtc_state->wm.skl.optimal.planes[plane_id];
+
+		if (DISPLAY_VER(i915) < 11 &&
+		    crtc_state->nv12_planes & BIT(plane_id)) {
+			skl_check_wm_level(&wm->trans_wm, ddb_y);
+		} else {
+			WARN_ON(skl_ddb_entry_size(ddb_y));
+
+			skl_check_wm_level(&wm->trans_wm, ddb);
+		}
+
+		skl_check_wm_level(&wm->sagv.wm0, ddb);
+		skl_check_wm_level(&wm->sagv.trans_wm, ddb);
+	}
+
+	return 0;
+}
+
+/*
+ * The max latency should be 257 (max the punit can code is 255 and we add 2us
+ * for the read latency) and cpp should always be <= 8, so that
+ * should allow pixel_rate up to ~2 GHz which seems sufficient since max
+ * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
+ */
+static uint_fixed_16_16_t
+skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
+	       u8 cpp, u32 latency, u32 dbuf_block_size)
+{
+	u32 wm_intermediate_val;
+	uint_fixed_16_16_t ret;
+
+	if (latency == 0)
+		return FP_16_16_MAX;
+
+	wm_intermediate_val = latency * pixel_rate * cpp;
+	ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
+
+	if (DISPLAY_VER(i915) >= 10)
+		ret = add_fixed16_u32(ret, 1);
+
+	return ret;
+}
+
+static uint_fixed_16_16_t
+skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
+	       uint_fixed_16_16_t plane_blocks_per_line)
+{
+	u32 wm_intermediate_val;
+	uint_fixed_16_16_t ret;
+
+	if (latency == 0)
+		return FP_16_16_MAX;
+
+	wm_intermediate_val = latency * pixel_rate;
+	wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
+					   pipe_htotal * 1000);
+	ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
+	return ret;
+}
+
+static uint_fixed_16_16_t
+intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	u32 pixel_rate;
+	u32 crtc_htotal;
+	uint_fixed_16_16_t linetime_us;
+
+	if (!crtc_state->hw.active)
+		return u32_to_fixed16(0);
+
+	pixel_rate = crtc_state->pixel_rate;
+
+	if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
+		return u32_to_fixed16(0);
+
+	crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
+	linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
+
+	return linetime_us;
+}
+
+static int
+skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
+		      int width, const struct drm_format_info *format,
+		      u64 modifier, unsigned int rotation,
+		      u32 plane_pixel_rate, struct skl_wm_params *wp,
+		      int color_plane)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	u32 interm_pbpl;
+
+	/* only planar format has two planes */
+	if (color_plane == 1 &&
+	    !intel_format_info_is_yuv_semiplanar(format, modifier)) {
+		drm_dbg_kms(&i915->drm,
+			    "Non planar format have single plane\n");
+		return -EINVAL;
+	}
+
+	wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
+		      modifier == I915_FORMAT_MOD_4_TILED ||
+		      modifier == I915_FORMAT_MOD_Yf_TILED ||
+		      modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+		      modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
+	wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
+	wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
+			 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
+	wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
+
+	wp->width = width;
+	if (color_plane == 1 && wp->is_planar)
+		wp->width /= 2;
+
+	wp->cpp = format->cpp[color_plane];
+	wp->plane_pixel_rate = plane_pixel_rate;
+
+	if (DISPLAY_VER(i915) >= 11 &&
+	    modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
+		wp->dbuf_block_size = 256;
+	else
+		wp->dbuf_block_size = 512;
+
+	if (drm_rotation_90_or_270(rotation)) {
+		switch (wp->cpp) {
+		case 1:
+			wp->y_min_scanlines = 16;
+			break;
+		case 2:
+			wp->y_min_scanlines = 8;
+			break;
+		case 4:
+			wp->y_min_scanlines = 4;
+			break;
+		default:
+			MISSING_CASE(wp->cpp);
+			return -EINVAL;
+		}
+	} else {
+		wp->y_min_scanlines = 4;
+	}
+
+	if (skl_needs_memory_bw_wa(i915))
+		wp->y_min_scanlines *= 2;
+
+	wp->plane_bytes_per_line = wp->width * wp->cpp;
+	if (wp->y_tiled) {
+		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
+					   wp->y_min_scanlines,
+					   wp->dbuf_block_size);
+
+		if (DISPLAY_VER(i915) >= 10)
+			interm_pbpl++;
+
+		wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
+							wp->y_min_scanlines);
+	} else {
+		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
+					   wp->dbuf_block_size);
+
+		if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
+			interm_pbpl++;
+
+		wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
+	}
+
+	wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
+					     wp->plane_blocks_per_line);
+
+	wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state));
+
+	return 0;
+}
+
+static int
+skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
+			    const struct intel_plane_state *plane_state,
+			    struct skl_wm_params *wp, int color_plane)
+{
+	const struct drm_framebuffer *fb = plane_state->hw.fb;
+	int width;
+
+	/*
+	 * Src coordinates are already rotated by 270 degrees for
+	 * the 90/270 degree plane rotation cases (to match the
+	 * GTT mapping), hence no need to account for rotation here.
+	 */
+	width = drm_rect_width(&plane_state->uapi.src) >> 16;
+
+	return skl_compute_wm_params(crtc_state, width,
+				     fb->format, fb->modifier,
+				     plane_state->hw.rotation,
+				     intel_plane_pixel_rate(crtc_state, plane_state),
+				     wp, color_plane);
+}
+
+static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
+{
+	if (DISPLAY_VER(i915) >= 10)
+		return true;
+
+	/* The number of lines are ignored for the level 0 watermark. */
+	return level > 0;
+}
+
+static int skl_wm_max_lines(struct drm_i915_private *i915)
+{
+	if (DISPLAY_VER(i915) >= 13)
+		return 255;
+	else
+		return 31;
+}
+
+static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
+				 struct intel_plane *plane,
+				 int level,
+				 unsigned int latency,
+				 const struct skl_wm_params *wp,
+				 const struct skl_wm_level *result_prev,
+				 struct skl_wm_level *result /* out */)
+{
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	uint_fixed_16_16_t method1, method2;
+	uint_fixed_16_16_t selected_result;
+	u32 blocks, lines, min_ddb_alloc = 0;
+
+	if (latency == 0 ||
+	    (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
+		/* reject it */
+		result->min_ddb_alloc = U16_MAX;
+		return;
+	}
+
+	/*
+	 * WaIncreaseLatencyIPCEnabled: kbl,cfl
+	 * Display WA #1141: kbl,cfl
+	 */
+	if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
+	    skl_watermark_ipc_enabled(i915))
+		latency += 4;
+
+	if (skl_needs_memory_bw_wa(i915) && wp->x_tiled)
+		latency += 15;
+
+	method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
+				 wp->cpp, latency, wp->dbuf_block_size);
+	method2 = skl_wm_method2(wp->plane_pixel_rate,
+				 crtc_state->hw.pipe_mode.crtc_htotal,
+				 latency,
+				 wp->plane_blocks_per_line);
+
+	if (wp->y_tiled) {
+		selected_result = max_fixed16(method2, wp->y_tile_minimum);
+	} else {
+		if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
+		     wp->dbuf_block_size < 1) &&
+		     (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
+			selected_result = method2;
+		} else if (latency >= wp->linetime_us) {
+			if (DISPLAY_VER(i915) == 9)
+				selected_result = min_fixed16(method1, method2);
+			else
+				selected_result = method2;
+		} else {
+			selected_result = method1;
+		}
+	}
+
+	blocks = fixed16_to_u32_round_up(selected_result) + 1;
+	/*
+	 * Lets have blocks at minimum equivalent to plane_blocks_per_line
+	 * as there will be at minimum one line for lines configuration. This
+	 * is a work around for FIFO underruns observed with resolutions like
+	 * 4k 60 Hz in single channel DRAM configurations.
+	 *
+	 * As per the Bspec 49325, if the ddb allocation can hold at least
+	 * one plane_blocks_per_line, we should have selected method2 in
+	 * the above logic. Assuming that modern versions have enough dbuf
+	 * and method2 guarantees blocks equivalent to at least 1 line,
+	 * select the blocks as plane_blocks_per_line.
+	 *
+	 * TODO: Revisit the logic when we have better understanding on DRAM
+	 * channels' impact on the level 0 memory latency and the relevant
+	 * wm calculations.
+	 */
+	if (skl_wm_has_lines(i915, level))
+		blocks = max(blocks,
+			     fixed16_to_u32_round_up(wp->plane_blocks_per_line));
+	lines = div_round_up_fixed16(selected_result,
+				     wp->plane_blocks_per_line);
+
+	if (DISPLAY_VER(i915) == 9) {
+		/* Display WA #1125: skl,bxt,kbl */
+		if (level == 0 && wp->rc_surface)
+			blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
+
+		/* Display WA #1126: skl,bxt,kbl */
+		if (level >= 1 && level <= 7) {
+			if (wp->y_tiled) {
+				blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
+				lines += wp->y_min_scanlines;
+			} else {
+				blocks++;
+			}
+
+			/*
+			 * Make sure result blocks for higher latency levels are
+			 * at least as high as level below the current level.
+			 * Assumption in DDB algorithm optimization for special
+			 * cases. Also covers Display WA #1125 for RC.
+			 */
+			if (result_prev->blocks > blocks)
+				blocks = result_prev->blocks;
+		}
+	}
+
+	if (DISPLAY_VER(i915) >= 11) {
+		if (wp->y_tiled) {
+			int extra_lines;
+
+			if (lines % wp->y_min_scanlines == 0)
+				extra_lines = wp->y_min_scanlines;
+			else
+				extra_lines = wp->y_min_scanlines * 2 -
+					lines % wp->y_min_scanlines;
+
+			min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
+								 wp->plane_blocks_per_line);
+		} else {
+			min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
+		}
+	}
+
+	if (!skl_wm_has_lines(i915, level))
+		lines = 0;
+
+	if (lines > skl_wm_max_lines(i915)) {
+		/* reject it */
+		result->min_ddb_alloc = U16_MAX;
+		return;
+	}
+
+	/*
+	 * If lines is valid, assume we can use this watermark level
+	 * for now.  We'll come back and disable it after we calculate the
+	 * DDB allocation if it turns out we don't actually have enough
+	 * blocks to satisfy it.
+	 */
+	result->blocks = blocks;
+	result->lines = lines;
+	/* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
+	result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
+	result->enable = true;
+
+	if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
+		result->can_sagv = latency >= i915->display.sagv.block_time_us;
+}
+
+static void
+skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
+		      struct intel_plane *plane,
+		      const struct skl_wm_params *wm_params,
+		      struct skl_wm_level *levels)
+{
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	int level, max_level = ilk_wm_max_level(i915);
+	struct skl_wm_level *result_prev = &levels[0];
+
+	for (level = 0; level <= max_level; level++) {
+		struct skl_wm_level *result = &levels[level];
+		unsigned int latency = i915->display.wm.skl_latency[level];
+
+		skl_compute_plane_wm(crtc_state, plane, level, latency,
+				     wm_params, result_prev, result);
+
+		result_prev = result;
+	}
+}
+
+static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
+				struct intel_plane *plane,
+				const struct skl_wm_params *wm_params,
+				struct skl_plane_wm *plane_wm)
+{
+	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
+	struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
+	struct skl_wm_level *levels = plane_wm->wm;
+	unsigned int latency = 0;
+
+	if (i915->display.sagv.block_time_us)
+		latency = i915->display.sagv.block_time_us + i915->display.wm.skl_latency[0];
+
+	skl_compute_plane_wm(crtc_state, plane, 0, latency,
+			     wm_params, &levels[0],
+			     sagv_wm);
+}
+
+static void skl_compute_transition_wm(struct drm_i915_private *i915,
+				      struct skl_wm_level *trans_wm,
+				      const struct skl_wm_level *wm0,
+				      const struct skl_wm_params *wp)
+{
+	u16 trans_min, trans_amount, trans_y_tile_min;
+	u16 wm0_blocks, trans_offset, blocks;
+
+	/* Transition WM don't make any sense if ipc is disabled */
+	if (!skl_watermark_ipc_enabled(i915))
+		return;
+
+	/*
+	 * WaDisableTWM:skl,kbl,cfl,bxt
+	 * Transition WM are not recommended by HW team for GEN9
+	 */
+	if (DISPLAY_VER(i915) == 9)
+		return;
+
+	if (DISPLAY_VER(i915) >= 11)
+		trans_min = 4;
+	else
+		trans_min = 14;
+
+	/* Display WA #1140: glk,cnl */
+	if (DISPLAY_VER(i915) == 10)
+		trans_amount = 0;
+	else
+		trans_amount = 10; /* This is configurable amount */
+
+	trans_offset = trans_min + trans_amount;
+
+	/*
+	 * The spec asks for Selected Result Blocks for wm0 (the real value),
+	 * not Result Blocks (the integer value). Pay attention to the capital
+	 * letters. The value wm_l0->blocks is actually Result Blocks, but
+	 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
+	 * and since we later will have to get the ceiling of the sum in the
+	 * transition watermarks calculation, we can just pretend Selected
+	 * Result Blocks is Result Blocks minus 1 and it should work for the
+	 * current platforms.
+	 */
+	wm0_blocks = wm0->blocks - 1;
+
+	if (wp->y_tiled) {
+		trans_y_tile_min =
+			(u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
+		blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
+	} else {
+		blocks = wm0_blocks + trans_offset;
+	}
+	blocks++;
+
+	/*
+	 * Just assume we can enable the transition watermark.  After
+	 * computing the DDB we'll come back and disable it if that
+	 * assumption turns out to be false.
+	 */
+	trans_wm->blocks = blocks;
+	trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
+	trans_wm->enable = true;
+}
+
+static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
+				     const struct intel_plane_state *plane_state,
+				     struct intel_plane *plane, int color_plane)
+{
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
+	struct skl_wm_params wm_params;
+	int ret;
+
+	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
+					  &wm_params, color_plane);
+	if (ret)
+		return ret;
+
+	skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
+
+	skl_compute_transition_wm(i915, &wm->trans_wm,
+				  &wm->wm[0], &wm_params);
+
+	if (DISPLAY_VER(i915) >= 12) {
+		tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
+
+		skl_compute_transition_wm(i915, &wm->sagv.trans_wm,
+					  &wm->sagv.wm0, &wm_params);
+	}
+
+	return 0;
+}
+
+static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
+				 const struct intel_plane_state *plane_state,
+				 struct intel_plane *plane)
+{
+	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
+	struct skl_wm_params wm_params;
+	int ret;
+
+	wm->is_planar = true;
+
+	/* uv plane watermarks must also be validated for NV12/Planar */
+	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
+					  &wm_params, 1);
+	if (ret)
+		return ret;
+
+	skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
+
+	return 0;
+}
+
+static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
+			      const struct intel_plane_state *plane_state)
+{
+	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
+	enum plane_id plane_id = plane->id;
+	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
+	const struct drm_framebuffer *fb = plane_state->hw.fb;
+	int ret;
+
+	memset(wm, 0, sizeof(*wm));
+
+	if (!intel_wm_plane_visible(crtc_state, plane_state))
+		return 0;
+
+	ret = skl_build_plane_wm_single(crtc_state, plane_state,
+					plane, 0);
+	if (ret)
+		return ret;
+
+	if (fb->format->is_yuv && fb->format->num_planes > 1) {
+		ret = skl_build_plane_wm_uv(crtc_state, plane_state,
+					    plane);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
+			      const struct intel_plane_state *plane_state)
+{
+	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
+	struct drm_i915_private *i915 = to_i915(plane->base.dev);
+	enum plane_id plane_id = plane->id;
+	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
+	int ret;
+
+	/* Watermarks calculated in master */
+	if (plane_state->planar_slave)
+		return 0;
+
+	memset(wm, 0, sizeof(*wm));
+
+	if (plane_state->planar_linked_plane) {
+		const struct drm_framebuffer *fb = plane_state->hw.fb;
+
+		drm_WARN_ON(&i915->drm,
+			    !intel_wm_plane_visible(crtc_state, plane_state));
+		drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
+			    fb->format->num_planes == 1);
+
+		ret = skl_build_plane_wm_single(crtc_state, plane_state,
+						plane_state->planar_linked_plane, 0);
+		if (ret)
+			return ret;
+
+		ret = skl_build_plane_wm_single(crtc_state, plane_state,
+						plane, 1);
+		if (ret)
+			return ret;
+	} else if (intel_wm_plane_visible(crtc_state, plane_state)) {
+		ret = skl_build_plane_wm_single(crtc_state, plane_state,
+						plane, 0);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int skl_build_pipe_wm(struct intel_atomic_state *state,
+			     struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct intel_crtc_state *crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	const struct intel_plane_state *plane_state;
+	struct intel_plane *plane;
+	int ret, i;
+
+	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
+		/*
+		 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
+		 * instead but we don't populate that correctly for NV12 Y
+		 * planes so for now hack this.
+		 */
+		if (plane->pipe != crtc->pipe)
+			continue;
+
+		if (DISPLAY_VER(i915) >= 11)
+			ret = icl_build_plane_wm(crtc_state, plane_state);
+		else
+			ret = skl_build_plane_wm(crtc_state, plane_state);
+		if (ret)
+			return ret;
+	}
+
+	crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
+
+	return 0;
+}
+
+static void skl_ddb_entry_write(struct drm_i915_private *i915,
+				i915_reg_t reg,
+				const struct skl_ddb_entry *entry)
+{
+	if (entry->end)
+		intel_de_write_fw(i915, reg,
+				  PLANE_BUF_END(entry->end - 1) |
+				  PLANE_BUF_START(entry->start));
+	else
+		intel_de_write_fw(i915, reg, 0);
+}
+
+static void skl_write_wm_level(struct drm_i915_private *i915,
+			       i915_reg_t reg,
+			       const struct skl_wm_level *level)
+{
+	u32 val = 0;
+
+	if (level->enable)
+		val |= PLANE_WM_EN;
+	if (level->ignore_lines)
+		val |= PLANE_WM_IGNORE_LINES;
+	val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks);
+	val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
+
+	intel_de_write_fw(i915, reg, val);
+}
+
+void skl_write_plane_wm(struct intel_plane *plane,
+			const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(plane->base.dev);
+	int level, max_level = ilk_wm_max_level(i915);
+	enum plane_id plane_id = plane->id;
+	enum pipe pipe = plane->pipe;
+	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
+	const struct skl_ddb_entry *ddb =
+		&crtc_state->wm.skl.plane_ddb[plane_id];
+	const struct skl_ddb_entry *ddb_y =
+		&crtc_state->wm.skl.plane_ddb_y[plane_id];
+
+	for (level = 0; level <= max_level; level++)
+		skl_write_wm_level(i915, PLANE_WM(pipe, plane_id, level),
+				   skl_plane_wm_level(pipe_wm, plane_id, level));
+
+	skl_write_wm_level(i915, PLANE_WM_TRANS(pipe, plane_id),
+			   skl_plane_trans_wm(pipe_wm, plane_id));
+
+	if (HAS_HW_SAGV_WM(i915)) {
+		const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
+
+		skl_write_wm_level(i915, PLANE_WM_SAGV(pipe, plane_id),
+				   &wm->sagv.wm0);
+		skl_write_wm_level(i915, PLANE_WM_SAGV_TRANS(pipe, plane_id),
+				   &wm->sagv.trans_wm);
+	}
+
+	skl_ddb_entry_write(i915,
+			    PLANE_BUF_CFG(pipe, plane_id), ddb);
+
+	if (DISPLAY_VER(i915) < 11)
+		skl_ddb_entry_write(i915,
+				    PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y);
+}
+
+void skl_write_cursor_wm(struct intel_plane *plane,
+			 const struct intel_crtc_state *crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(plane->base.dev);
+	int level, max_level = ilk_wm_max_level(i915);
+	enum plane_id plane_id = plane->id;
+	enum pipe pipe = plane->pipe;
+	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
+	const struct skl_ddb_entry *ddb =
+		&crtc_state->wm.skl.plane_ddb[plane_id];
+
+	for (level = 0; level <= max_level; level++)
+		skl_write_wm_level(i915, CUR_WM(pipe, level),
+				   skl_plane_wm_level(pipe_wm, plane_id, level));
+
+	skl_write_wm_level(i915, CUR_WM_TRANS(pipe),
+			   skl_plane_trans_wm(pipe_wm, plane_id));
+
+	if (HAS_HW_SAGV_WM(i915)) {
+		const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
+
+		skl_write_wm_level(i915, CUR_WM_SAGV(pipe),
+				   &wm->sagv.wm0);
+		skl_write_wm_level(i915, CUR_WM_SAGV_TRANS(pipe),
+				   &wm->sagv.trans_wm);
+	}
+
+	skl_ddb_entry_write(i915, CUR_BUF_CFG(pipe), ddb);
+}
+
+static bool skl_wm_level_equals(const struct skl_wm_level *l1,
+				const struct skl_wm_level *l2)
+{
+	return l1->enable == l2->enable &&
+		l1->ignore_lines == l2->ignore_lines &&
+		l1->lines == l2->lines &&
+		l1->blocks == l2->blocks;
+}
+
+static bool skl_plane_wm_equals(struct drm_i915_private *i915,
+				const struct skl_plane_wm *wm1,
+				const struct skl_plane_wm *wm2)
+{
+	int level, max_level = ilk_wm_max_level(i915);
+
+	for (level = 0; level <= max_level; level++) {
+		/*
+		 * We don't check uv_wm as the hardware doesn't actually
+		 * use it. It only gets used for calculating the required
+		 * ddb allocation.
+		 */
+		if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
+			return false;
+	}
+
+	return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
+		skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
+		skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
+}
+
+static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
+				    const struct skl_ddb_entry *b)
+{
+	return a->start < b->end && b->start < a->end;
+}
+
+static void skl_ddb_entry_union(struct skl_ddb_entry *a,
+				const struct skl_ddb_entry *b)
+{
+	if (a->end && b->end) {
+		a->start = min(a->start, b->start);
+		a->end = max(a->end, b->end);
+	} else if (b->end) {
+		a->start = b->start;
+		a->end = b->end;
+	}
+}
+
+bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
+				 const struct skl_ddb_entry *entries,
+				 int num_entries, int ignore_idx)
+{
+	int i;
+
+	for (i = 0; i < num_entries; i++) {
+		if (i != ignore_idx &&
+		    skl_ddb_entries_overlap(ddb, &entries[i]))
+			return true;
+	}
+
+	return false;
+}
+
+static int
+skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
+			    struct intel_crtc_state *new_crtc_state)
+{
+	struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
+	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct intel_plane *plane;
+
+	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
+		struct intel_plane_state *plane_state;
+		enum plane_id plane_id = plane->id;
+
+		if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
+					&new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
+		    skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
+					&new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
+			continue;
+
+		plane_state = intel_atomic_get_plane_state(state, plane);
+		if (IS_ERR(plane_state))
+			return PTR_ERR(plane_state);
+
+		new_crtc_state->update_planes |= BIT(plane_id);
+	}
+
+	return 0;
+}
+
+static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
+{
+	struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev);
+	u8 enabled_slices;
+	enum pipe pipe;
+
+	/*
+	 * FIXME: For now we always enable slice S1 as per
+	 * the Bspec display initialization sequence.
+	 */
+	enabled_slices = BIT(DBUF_S1);
+
+	for_each_pipe(i915, pipe)
+		enabled_slices |= dbuf_state->slices[pipe];
+
+	return enabled_slices;
+}
+
+static int
+skl_compute_ddb(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_dbuf_state *old_dbuf_state;
+	struct intel_dbuf_state *new_dbuf_state = NULL;
+	const struct intel_crtc_state *old_crtc_state;
+	struct intel_crtc_state *new_crtc_state;
+	struct intel_crtc *crtc;
+	int ret, i;
+
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+		new_dbuf_state = intel_atomic_get_dbuf_state(state);
+		if (IS_ERR(new_dbuf_state))
+			return PTR_ERR(new_dbuf_state);
+
+		old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
+		break;
+	}
+
+	if (!new_dbuf_state)
+		return 0;
+
+	new_dbuf_state->active_pipes =
+		intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
+
+	if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
+		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
+		if (ret)
+			return ret;
+	}
+
+	if (HAS_MBUS_JOINING(i915))
+		new_dbuf_state->joined_mbus =
+			adlp_check_mbus_joined(new_dbuf_state->active_pipes);
+
+	for_each_intel_crtc(&i915->drm, crtc) {
+		enum pipe pipe = crtc->pipe;
+
+		new_dbuf_state->slices[pipe] =
+			skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
+						new_dbuf_state->joined_mbus);
+
+		if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
+			continue;
+
+		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
+		if (ret)
+			return ret;
+	}
+
+	new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
+
+	if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
+	    old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
+		ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
+		if (ret)
+			return ret;
+
+		if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
+			/* TODO: Implement vblank synchronized MBUS joining changes */
+			ret = intel_modeset_all_pipes(state);
+			if (ret)
+				return ret;
+		}
+
+		drm_dbg_kms(&i915->drm,
+			    "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
+			    old_dbuf_state->enabled_slices,
+			    new_dbuf_state->enabled_slices,
+			    INTEL_INFO(i915)->display.dbuf.slice_mask,
+			    str_yes_no(old_dbuf_state->joined_mbus),
+			    str_yes_no(new_dbuf_state->joined_mbus));
+	}
+
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+		enum pipe pipe = crtc->pipe;
+
+		new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
+
+		if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
+			continue;
+
+		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
+		if (ret)
+			return ret;
+	}
+
+	for_each_intel_crtc(&i915->drm, crtc) {
+		ret = skl_crtc_allocate_ddb(state, crtc);
+		if (ret)
+			return ret;
+	}
+
+	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
+					    new_crtc_state, i) {
+		ret = skl_crtc_allocate_plane_ddb(state, crtc);
+		if (ret)
+			return ret;
+
+		ret = skl_ddb_add_affected_planes(old_crtc_state,
+						  new_crtc_state);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static char enast(bool enable)
+{
+	return enable ? '*' : ' ';
+}
+
+static void
+skl_print_wm_changes(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_crtc_state *old_crtc_state;
+	const struct intel_crtc_state *new_crtc_state;
+	struct intel_plane *plane;
+	struct intel_crtc *crtc;
+	int i;
+
+	if (!drm_debug_enabled(DRM_UT_KMS))
+		return;
+
+	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
+					    new_crtc_state, i) {
+		const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
+
+		old_pipe_wm = &old_crtc_state->wm.skl.optimal;
+		new_pipe_wm = &new_crtc_state->wm.skl.optimal;
+
+		for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
+			enum plane_id plane_id = plane->id;
+			const struct skl_ddb_entry *old, *new;
+
+			old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
+			new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
+
+			if (skl_ddb_entry_equal(old, new))
+				continue;
+
+			drm_dbg_kms(&i915->drm,
+				    "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
+				    plane->base.base.id, plane->base.name,
+				    old->start, old->end, new->start, new->end,
+				    skl_ddb_entry_size(old), skl_ddb_entry_size(new));
+		}
+
+		for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
+			enum plane_id plane_id = plane->id;
+			const struct skl_plane_wm *old_wm, *new_wm;
+
+			old_wm = &old_pipe_wm->planes[plane_id];
+			new_wm = &new_pipe_wm->planes[plane_id];
+
+			if (skl_plane_wm_equals(i915, old_wm, new_wm))
+				continue;
+
+			drm_dbg_kms(&i915->drm,
+				    "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
+				    " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
+				    plane->base.base.id, plane->base.name,
+				    enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
+				    enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
+				    enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
+				    enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
+				    enast(old_wm->trans_wm.enable),
+				    enast(old_wm->sagv.wm0.enable),
+				    enast(old_wm->sagv.trans_wm.enable),
+				    enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
+				    enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
+				    enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
+				    enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
+				    enast(new_wm->trans_wm.enable),
+				    enast(new_wm->sagv.wm0.enable),
+				    enast(new_wm->sagv.trans_wm.enable));
+
+			drm_dbg_kms(&i915->drm,
+				    "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
+				      " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
+				    plane->base.base.id, plane->base.name,
+				    enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
+				    enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
+				    enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
+				    enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
+				    enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
+				    enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
+				    enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
+				    enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
+				    enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
+				    enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
+				    enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
+				    enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
+				    enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
+				    enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
+				    enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
+				    enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
+				    enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
+				    enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
+				    enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
+				    enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
+				    enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
+				    enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
+
+			drm_dbg_kms(&i915->drm,
+				    "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
+				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
+				    plane->base.base.id, plane->base.name,
+				    old_wm->wm[0].blocks, old_wm->wm[1].blocks,
+				    old_wm->wm[2].blocks, old_wm->wm[3].blocks,
+				    old_wm->wm[4].blocks, old_wm->wm[5].blocks,
+				    old_wm->wm[6].blocks, old_wm->wm[7].blocks,
+				    old_wm->trans_wm.blocks,
+				    old_wm->sagv.wm0.blocks,
+				    old_wm->sagv.trans_wm.blocks,
+				    new_wm->wm[0].blocks, new_wm->wm[1].blocks,
+				    new_wm->wm[2].blocks, new_wm->wm[3].blocks,
+				    new_wm->wm[4].blocks, new_wm->wm[5].blocks,
+				    new_wm->wm[6].blocks, new_wm->wm[7].blocks,
+				    new_wm->trans_wm.blocks,
+				    new_wm->sagv.wm0.blocks,
+				    new_wm->sagv.trans_wm.blocks);
+
+			drm_dbg_kms(&i915->drm,
+				    "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
+				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
+				    plane->base.base.id, plane->base.name,
+				    old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
+				    old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
+				    old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
+				    old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
+				    old_wm->trans_wm.min_ddb_alloc,
+				    old_wm->sagv.wm0.min_ddb_alloc,
+				    old_wm->sagv.trans_wm.min_ddb_alloc,
+				    new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
+				    new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
+				    new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
+				    new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
+				    new_wm->trans_wm.min_ddb_alloc,
+				    new_wm->sagv.wm0.min_ddb_alloc,
+				    new_wm->sagv.trans_wm.min_ddb_alloc);
+		}
+	}
+}
+
+static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
+					 const struct skl_pipe_wm *old_pipe_wm,
+					 const struct skl_pipe_wm *new_pipe_wm)
+{
+	struct drm_i915_private *i915 = to_i915(plane->base.dev);
+	int level, max_level = ilk_wm_max_level(i915);
+
+	for (level = 0; level <= max_level; level++) {
+		/*
+		 * We don't check uv_wm as the hardware doesn't actually
+		 * use it. It only gets used for calculating the required
+		 * ddb allocation.
+		 */
+		if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
+					 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
+			return false;
+	}
+
+	if (HAS_HW_SAGV_WM(i915)) {
+		const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
+		const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
+
+		if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
+		    !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
+			return false;
+	}
+
+	return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
+				   skl_plane_trans_wm(new_pipe_wm, plane->id));
+}
+
+/*
+ * To make sure the cursor watermark registers are always consistent
+ * with our computed state the following scenario needs special
+ * treatment:
+ *
+ * 1. enable cursor
+ * 2. move cursor entirely offscreen
+ * 3. disable cursor
+ *
+ * Step 2. does call .disable_plane() but does not zero the watermarks
+ * (since we consider an offscreen cursor still active for the purposes
+ * of watermarks). Step 3. would not normally call .disable_plane()
+ * because the actual plane visibility isn't changing, and we don't
+ * deallocate the cursor ddb until the pipe gets disabled. So we must
+ * force step 3. to call .disable_plane() to update the watermark
+ * registers properly.
+ *
+ * Other planes do not suffer from this issues as their watermarks are
+ * calculated based on the actual plane visibility. The only time this
+ * can trigger for the other planes is during the initial readout as the
+ * default value of the watermarks registers is not zero.
+ */
+static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
+				      struct intel_crtc *crtc)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	const struct intel_crtc_state *old_crtc_state =
+		intel_atomic_get_old_crtc_state(state, crtc);
+	struct intel_crtc_state *new_crtc_state =
+		intel_atomic_get_new_crtc_state(state, crtc);
+	struct intel_plane *plane;
+
+	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
+		struct intel_plane_state *plane_state;
+		enum plane_id plane_id = plane->id;
+
+		/*
+		 * Force a full wm update for every plane on modeset.
+		 * Required because the reset value of the wm registers
+		 * is non-zero, whereas we want all disabled planes to
+		 * have zero watermarks. So if we turn off the relevant
+		 * power well the hardware state will go out of sync
+		 * with the software state.
+		 */
+		if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
+		    skl_plane_selected_wm_equals(plane,
+						 &old_crtc_state->wm.skl.optimal,
+						 &new_crtc_state->wm.skl.optimal))
+			continue;
+
+		plane_state = intel_atomic_get_plane_state(state, plane);
+		if (IS_ERR(plane_state))
+			return PTR_ERR(plane_state);
+
+		new_crtc_state->update_planes |= BIT(plane_id);
+	}
+
+	return 0;
+}
+
+static int
+skl_compute_wm(struct intel_atomic_state *state)
+{
+	struct intel_crtc *crtc;
+	struct intel_crtc_state *new_crtc_state;
+	int ret, i;
+
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+		ret = skl_build_pipe_wm(state, crtc);
+		if (ret)
+			return ret;
+	}
+
+	ret = skl_compute_ddb(state);
+	if (ret)
+		return ret;
+
+	ret = intel_compute_sagv_mask(state);
+	if (ret)
+		return ret;
+
+	/*
+	 * skl_compute_ddb() will have adjusted the final watermarks
+	 * based on how much ddb is available. Now we can actually
+	 * check if the final watermarks changed.
+	 */
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+		ret = skl_wm_add_affected_planes(state, crtc);
+		if (ret)
+			return ret;
+	}
+
+	skl_print_wm_changes(state);
+
+	return 0;
+}
+
+static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
+{
+	level->enable = val & PLANE_WM_EN;
+	level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
+	level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
+	level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
+}
+
+static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
+				     struct skl_pipe_wm *out)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	enum pipe pipe = crtc->pipe;
+	int level, max_level;
+	enum plane_id plane_id;
+	u32 val;
+
+	max_level = ilk_wm_max_level(i915);
+
+	for_each_plane_id_on_crtc(crtc, plane_id) {
+		struct skl_plane_wm *wm = &out->planes[plane_id];
+
+		for (level = 0; level <= max_level; level++) {
+			if (plane_id != PLANE_CURSOR)
+				val = intel_uncore_read(&i915->uncore, PLANE_WM(pipe, plane_id, level));
+			else
+				val = intel_uncore_read(&i915->uncore, CUR_WM(pipe, level));
+
+			skl_wm_level_from_reg_val(val, &wm->wm[level]);
+		}
+
+		if (plane_id != PLANE_CURSOR)
+			val = intel_uncore_read(&i915->uncore, PLANE_WM_TRANS(pipe, plane_id));
+		else
+			val = intel_uncore_read(&i915->uncore, CUR_WM_TRANS(pipe));
+
+		skl_wm_level_from_reg_val(val, &wm->trans_wm);
+
+		if (HAS_HW_SAGV_WM(i915)) {
+			if (plane_id != PLANE_CURSOR)
+				val = intel_uncore_read(&i915->uncore,
+							PLANE_WM_SAGV(pipe, plane_id));
+			else
+				val = intel_uncore_read(&i915->uncore,
+							CUR_WM_SAGV(pipe));
+
+			skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
+
+			if (plane_id != PLANE_CURSOR)
+				val = intel_uncore_read(&i915->uncore,
+							PLANE_WM_SAGV_TRANS(pipe, plane_id));
+			else
+				val = intel_uncore_read(&i915->uncore,
+							CUR_WM_SAGV_TRANS(pipe));
+
+			skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
+		} else if (DISPLAY_VER(i915) >= 12) {
+			wm->sagv.wm0 = wm->wm[0];
+			wm->sagv.trans_wm = wm->trans_wm;
+		}
+	}
+}
+
+void skl_wm_get_hw_state(struct drm_i915_private *i915)
+{
+	struct intel_dbuf_state *dbuf_state =
+		to_intel_dbuf_state(i915->display.dbuf.obj.state);
+	struct intel_crtc *crtc;
+
+	if (HAS_MBUS_JOINING(i915))
+		dbuf_state->joined_mbus = intel_de_read(i915, MBUS_CTL) & MBUS_JOIN;
+
+	for_each_intel_crtc(&i915->drm, crtc) {
+		struct intel_crtc_state *crtc_state =
+			to_intel_crtc_state(crtc->base.state);
+		enum pipe pipe = crtc->pipe;
+		unsigned int mbus_offset;
+		enum plane_id plane_id;
+		u8 slices;
+
+		memset(&crtc_state->wm.skl.optimal, 0,
+		       sizeof(crtc_state->wm.skl.optimal));
+		if (crtc_state->hw.active)
+			skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
+		crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
+
+		memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
+
+		for_each_plane_id_on_crtc(crtc, plane_id) {
+			struct skl_ddb_entry *ddb =
+				&crtc_state->wm.skl.plane_ddb[plane_id];
+			struct skl_ddb_entry *ddb_y =
+				&crtc_state->wm.skl.plane_ddb_y[plane_id];
+
+			if (!crtc_state->hw.active)
+				continue;
+
+			skl_ddb_get_hw_plane_state(i915, crtc->pipe,
+						   plane_id, ddb, ddb_y);
+
+			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
+			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
+		}
+
+		dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
+
+		/*
+		 * Used for checking overlaps, so we need absolute
+		 * offsets instead of MBUS relative offsets.
+		 */
+		slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
+						 dbuf_state->joined_mbus);
+		mbus_offset = mbus_ddb_offset(i915, slices);
+		crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
+		crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
+
+		/* The slices actually used by the planes on the pipe */
+		dbuf_state->slices[pipe] =
+			skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb);
+
+		drm_dbg_kms(&i915->drm,
+			    "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
+			    crtc->base.base.id, crtc->base.name,
+			    dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
+			    dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
+			    str_yes_no(dbuf_state->joined_mbus));
+	}
+
+	dbuf_state->enabled_slices = i915->display.dbuf.enabled_slices;
+}
+
+static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
+{
+	const struct intel_dbuf_state *dbuf_state =
+		to_intel_dbuf_state(i915->display.dbuf.obj.state);
+	struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
+	struct intel_crtc *crtc;
+
+	for_each_intel_crtc(&i915->drm, crtc) {
+		const struct intel_crtc_state *crtc_state =
+			to_intel_crtc_state(crtc->base.state);
+
+		entries[crtc->pipe] = crtc_state->wm.skl.ddb;
+	}
+
+	for_each_intel_crtc(&i915->drm, crtc) {
+		const struct intel_crtc_state *crtc_state =
+			to_intel_crtc_state(crtc->base.state);
+		u8 slices;
+
+		slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
+						 dbuf_state->joined_mbus);
+		if (dbuf_state->slices[crtc->pipe] & ~slices)
+			return true;
+
+		if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
+						I915_MAX_PIPES, crtc->pipe))
+			return true;
+	}
+
+	return false;
+}
+
+void skl_wm_sanitize(struct drm_i915_private *i915)
+{
+	struct intel_crtc *crtc;
+
+	/*
+	 * On TGL/RKL (at least) the BIOS likes to assign the planes
+	 * to the wrong DBUF slices. This will cause an infinite loop
+	 * in skl_commit_modeset_enables() as it can't find a way to
+	 * transition between the old bogus DBUF layout to the new
+	 * proper DBUF layout without DBUF allocation overlaps between
+	 * the planes (which cannot be allowed or else the hardware
+	 * may hang). If we detect a bogus DBUF layout just turn off
+	 * all the planes so that skl_commit_modeset_enables() can
+	 * simply ignore them.
+	 */
+	if (!skl_dbuf_is_misconfigured(i915))
+		return;
+
+	drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
+
+	for_each_intel_crtc(&i915->drm, crtc) {
+		struct intel_plane *plane = to_intel_plane(crtc->base.primary);
+		const struct intel_plane_state *plane_state =
+			to_intel_plane_state(plane->base.state);
+		struct intel_crtc_state *crtc_state =
+			to_intel_crtc_state(crtc->base.state);
+
+		if (plane_state->uapi.visible)
+			intel_plane_disable_noatomic(crtc, plane);
+
+		drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
+
+		memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
+	}
+}
+
+void intel_wm_state_verify(struct intel_crtc *crtc,
+			   struct intel_crtc_state *new_crtc_state)
+{
+	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
+	struct skl_hw_state {
+		struct skl_ddb_entry ddb[I915_MAX_PLANES];
+		struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
+		struct skl_pipe_wm wm;
+	} *hw;
+	const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
+	int level, max_level = ilk_wm_max_level(i915);
+	struct intel_plane *plane;
+	u8 hw_enabled_slices;
+
+	if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
+		return;
+
+	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
+	if (!hw)
+		return;
+
+	skl_pipe_wm_get_hw_state(crtc, &hw->wm);
+
+	skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y);
+
+	hw_enabled_slices = intel_enabled_dbuf_slices_mask(i915);
+
+	if (DISPLAY_VER(i915) >= 11 &&
+	    hw_enabled_slices != i915->display.dbuf.enabled_slices)
+		drm_err(&i915->drm,
+			"mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
+			i915->display.dbuf.enabled_slices,
+			hw_enabled_slices);
+
+	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
+		const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
+		const struct skl_wm_level *hw_wm_level, *sw_wm_level;
+
+		/* Watermarks */
+		for (level = 0; level <= max_level; level++) {
+			hw_wm_level = &hw->wm.planes[plane->id].wm[level];
+			sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
+
+			if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
+				continue;
+
+			drm_err(&i915->drm,
+				"[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+				plane->base.base.id, plane->base.name, level,
+				sw_wm_level->enable,
+				sw_wm_level->blocks,
+				sw_wm_level->lines,
+				hw_wm_level->enable,
+				hw_wm_level->blocks,
+				hw_wm_level->lines);
+		}
+
+		hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
+		sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
+
+		if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
+			drm_err(&i915->drm,
+				"[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+				plane->base.base.id, plane->base.name,
+				sw_wm_level->enable,
+				sw_wm_level->blocks,
+				sw_wm_level->lines,
+				hw_wm_level->enable,
+				hw_wm_level->blocks,
+				hw_wm_level->lines);
+		}
+
+		hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
+		sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
+
+		if (HAS_HW_SAGV_WM(i915) &&
+		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
+			drm_err(&i915->drm,
+				"[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+				plane->base.base.id, plane->base.name,
+				sw_wm_level->enable,
+				sw_wm_level->blocks,
+				sw_wm_level->lines,
+				hw_wm_level->enable,
+				hw_wm_level->blocks,
+				hw_wm_level->lines);
+		}
+
+		hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
+		sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
+
+		if (HAS_HW_SAGV_WM(i915) &&
+		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
+			drm_err(&i915->drm,
+				"[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+				plane->base.base.id, plane->base.name,
+				sw_wm_level->enable,
+				sw_wm_level->blocks,
+				sw_wm_level->lines,
+				hw_wm_level->enable,
+				hw_wm_level->blocks,
+				hw_wm_level->lines);
+		}
+
+		/* DDB */
+		hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
+		sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
+
+		if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
+			drm_err(&i915->drm,
+				"[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
+				plane->base.base.id, plane->base.name,
+				sw_ddb_entry->start, sw_ddb_entry->end,
+				hw_ddb_entry->start, hw_ddb_entry->end);
+		}
+	}
+
+	kfree(hw);
+}
+
+bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
+{
+	return i915->display.wm.ipc_enabled;
+}
+
+void skl_watermark_ipc_update(struct drm_i915_private *i915)
+{
+	if (!HAS_IPC(i915))
+		return;
+
+	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL2, DISP_IPC_ENABLE,
+			 skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
+}
+
+static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
+{
+	/* Display WA #0477 WaDisableIPC: skl */
+	if (IS_SKYLAKE(i915))
+		return false;
+
+	/* Display WA #1141: SKL:all KBL:all CFL */
+	if (IS_KABYLAKE(i915) ||
+	    IS_COFFEELAKE(i915) ||
+	    IS_COMETLAKE(i915))
+		return i915->dram_info.symmetric_memory;
+
+	return true;
+}
+
+void skl_watermark_ipc_init(struct drm_i915_private *i915)
+{
+	if (!HAS_IPC(i915))
+		return;
+
+	i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
+
+	skl_watermark_ipc_update(i915);
+}
+
+static void
+adjust_wm_latency(struct drm_i915_private *i915,
+		  u16 wm[], int max_level, int read_latency)
+{
+	bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
+	int i, level;
+
+	/*
+	 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
+	 * need to be disabled. We make sure to sanitize the values out
+	 * of the punit to satisfy this requirement.
+	 */
+	for (level = 1; level <= max_level; level++) {
+		if (wm[level] == 0) {
+			for (i = level + 1; i <= max_level; i++)
+				wm[i] = 0;
+
+			max_level = level - 1;
+			break;
+		}
+	}
+
+	/*
+	 * WaWmMemoryReadLatency
+	 *
+	 * punit doesn't take into account the read latency so we need
+	 * to add proper adjustement to each valid level we retrieve
+	 * from the punit when level 0 response data is 0us.
+	 */
+	if (wm[0] == 0) {
+		for (level = 0; level <= max_level; level++)
+			wm[level] += read_latency;
+	}
+
+	/*
+	 * WA Level-0 adjustment for 16GB DIMMs: SKL+
+	 * If we could not get dimm info enable this WA to prevent from
+	 * any underrun. If not able to get Dimm info assume 16GB dimm
+	 * to avoid any underrun.
+	 */
+	if (wm_lv_0_adjust_needed)
+		wm[0] += 1;
+}
+
+static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
+{
+	struct intel_uncore *uncore = &i915->uncore;
+	int max_level = ilk_wm_max_level(i915);
+	u32 val;
+
+	val = intel_uncore_read(uncore, MTL_LATENCY_LP0_LP1);
+	wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
+	wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
+
+	val = intel_uncore_read(uncore, MTL_LATENCY_LP2_LP3);
+	wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
+	wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
+
+	val = intel_uncore_read(uncore, MTL_LATENCY_LP4_LP5);
+	wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
+	wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
+
+	adjust_wm_latency(i915, wm, max_level, 6);
+}
+
+static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
+{
+	int max_level = ilk_wm_max_level(i915);
+	int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
+	int mult = IS_DG2(i915) ? 2 : 1;
+	u32 val;
+	int ret;
+
+	/* read the first set of memory latencies[0:3] */
+	val = 0; /* data0 to be programmed to 0 for first set */
+	ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
+	if (ret) {
+		drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
+		return;
+	}
+
+	wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
+	wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
+	wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
+	wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
+
+	/* read the second set of memory latencies[4:7] */
+	val = 1; /* data0 to be programmed to 1 for second set */
+	ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
+	if (ret) {
+		drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
+		return;
+	}
+
+	wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
+	wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
+	wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
+	wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
+
+	adjust_wm_latency(i915, wm, max_level, read_latency);
+}
+
+static void skl_setup_wm_latency(struct drm_i915_private *i915)
+{
+	if (DISPLAY_VER(i915) >= 14)
+		mtl_read_wm_latency(i915, i915->display.wm.skl_latency);
+	else
+		skl_read_wm_latency(i915, i915->display.wm.skl_latency);
+
+	intel_print_wm_latency(i915, "Gen9 Plane", i915->display.wm.skl_latency);
+}
+
+static const struct intel_wm_funcs skl_wm_funcs = {
+	.compute_global_watermarks = skl_compute_wm,
+};
+
+void skl_wm_init(struct drm_i915_private *i915)
+{
+	intel_sagv_init(i915);
+
+	skl_setup_wm_latency(i915);
+
+	i915->display.funcs.wm = &skl_wm_funcs;
+}
+
+static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
+{
+	struct intel_dbuf_state *dbuf_state;
+
+	dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
+	if (!dbuf_state)
+		return NULL;
+
+	return &dbuf_state->base;
+}
+
+static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
+				     struct intel_global_state *state)
+{
+	kfree(state);
+}
+
+static const struct intel_global_state_funcs intel_dbuf_funcs = {
+	.atomic_duplicate_state = intel_dbuf_duplicate_state,
+	.atomic_destroy_state = intel_dbuf_destroy_state,
+};
+
+struct intel_dbuf_state *
+intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	struct intel_global_state *dbuf_state;
+
+	dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj);
+	if (IS_ERR(dbuf_state))
+		return ERR_CAST(dbuf_state);
+
+	return to_intel_dbuf_state(dbuf_state);
+}
+
+int intel_dbuf_init(struct drm_i915_private *i915)
+{
+	struct intel_dbuf_state *dbuf_state;
+
+	dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
+	if (!dbuf_state)
+		return -ENOMEM;
+
+	intel_atomic_global_obj_init(i915, &i915->display.dbuf.obj,
+				     &dbuf_state->base, &intel_dbuf_funcs);
+
+	return 0;
+}
+
+/*
+ * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
+ * update the request state of all DBUS slices.
+ */
+static void update_mbus_pre_enable(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	u32 mbus_ctl, dbuf_min_tracker_val;
+	enum dbuf_slice slice;
+	const struct intel_dbuf_state *dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+
+	if (!HAS_MBUS_JOINING(i915))
+		return;
+
+	/*
+	 * TODO: Implement vblank synchronized MBUS joining changes.
+	 * Must be properly coordinated with dbuf reprogramming.
+	 */
+	if (dbuf_state->joined_mbus) {
+		mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
+			MBUS_JOIN_PIPE_SELECT_NONE;
+		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
+	} else {
+		mbus_ctl = MBUS_HASHING_MODE_2x2 |
+			MBUS_JOIN_PIPE_SELECT_NONE;
+		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
+	}
+
+	intel_de_rmw(i915, MBUS_CTL,
+		     MBUS_HASHING_MODE_MASK | MBUS_JOIN |
+		     MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
+
+	for_each_dbuf_slice(i915, slice)
+		intel_de_rmw(i915, DBUF_CTL_S(slice),
+			     DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
+			     dbuf_min_tracker_val);
+}
+
+void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_dbuf_state *new_dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+	const struct intel_dbuf_state *old_dbuf_state =
+		intel_atomic_get_old_dbuf_state(state);
+
+	if (!new_dbuf_state ||
+	    (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
+	     new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
+		return;
+
+	WARN_ON(!new_dbuf_state->base.changed);
+
+	update_mbus_pre_enable(state);
+	gen9_dbuf_slices_update(i915,
+				old_dbuf_state->enabled_slices |
+				new_dbuf_state->enabled_slices);
+}
+
+void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_dbuf_state *new_dbuf_state =
+		intel_atomic_get_new_dbuf_state(state);
+	const struct intel_dbuf_state *old_dbuf_state =
+		intel_atomic_get_old_dbuf_state(state);
+
+	if (!new_dbuf_state ||
+	    (new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices &&
+	     new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus))
+		return;
+
+	WARN_ON(!new_dbuf_state->base.changed);
+
+	gen9_dbuf_slices_update(i915,
+				new_dbuf_state->enabled_slices);
+}
+
+static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
+{
+	switch (pipe) {
+	case PIPE_A:
+		return !(active_pipes & BIT(PIPE_D));
+	case PIPE_D:
+		return !(active_pipes & BIT(PIPE_A));
+	case PIPE_B:
+		return !(active_pipes & BIT(PIPE_C));
+	case PIPE_C:
+		return !(active_pipes & BIT(PIPE_B));
+	default: /* to suppress compiler warning */
+		MISSING_CASE(pipe);
+		break;
+	}
+
+	return false;
+}
+
+void intel_mbus_dbox_update(struct intel_atomic_state *state)
+{
+	struct drm_i915_private *i915 = to_i915(state->base.dev);
+	const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
+	const struct intel_crtc_state *new_crtc_state;
+	const struct intel_crtc *crtc;
+	u32 val = 0;
+	int i;
+
+	if (DISPLAY_VER(i915) < 11)
+		return;
+
+	new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
+	old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
+	if (!new_dbuf_state ||
+	    (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
+	     new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
+		return;
+
+	if (DISPLAY_VER(i915) >= 14)
+		val |= MBUS_DBOX_I_CREDIT(2);
+
+	if (DISPLAY_VER(i915) >= 12) {
+		val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
+		val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
+		val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
+	}
+
+	if (DISPLAY_VER(i915) >= 14)
+		val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(12) :
+						     MBUS_DBOX_A_CREDIT(8);
+	else if (IS_ALDERLAKE_P(i915))
+		/* Wa_22010947358:adl-p */
+		val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
+						     MBUS_DBOX_A_CREDIT(4);
+	else
+		val |= MBUS_DBOX_A_CREDIT(2);
+
+	if (DISPLAY_VER(i915) >= 14) {
+		val |= MBUS_DBOX_B_CREDIT(0xA);
+	} else if (IS_ALDERLAKE_P(i915)) {
+		val |= MBUS_DBOX_BW_CREDIT(2);
+		val |= MBUS_DBOX_B_CREDIT(8);
+	} else if (DISPLAY_VER(i915) >= 12) {
+		val |= MBUS_DBOX_BW_CREDIT(2);
+		val |= MBUS_DBOX_B_CREDIT(12);
+	} else {
+		val |= MBUS_DBOX_BW_CREDIT(1);
+		val |= MBUS_DBOX_B_CREDIT(8);
+	}
+
+	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
+		u32 pipe_val = val;
+
+		if (!new_crtc_state->hw.active)
+			continue;
+
+		if (DISPLAY_VER(i915) >= 14) {
+			if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe,
+							      new_dbuf_state->active_pipes))
+				pipe_val |= MBUS_DBOX_BW_8CREDITS_MTL;
+			else
+				pipe_val |= MBUS_DBOX_BW_4CREDITS_MTL;
+		}
+
+		intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), pipe_val);
+	}
+}
+
+static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
+{
+	struct drm_i915_private *i915 = m->private;
+
+	seq_printf(m, "Isochronous Priority Control: %s\n",
+		   str_yes_no(skl_watermark_ipc_enabled(i915)));
+	return 0;
+}
+
+static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
+{
+	struct drm_i915_private *i915 = inode->i_private;
+
+	return single_open(file, skl_watermark_ipc_status_show, i915);
+}
+
+static ssize_t skl_watermark_ipc_status_write(struct file *file,
+					      const char __user *ubuf,
+					      size_t len, loff_t *offp)
+{
+	struct seq_file *m = file->private_data;
+	struct drm_i915_private *i915 = m->private;
+	intel_wakeref_t wakeref;
+	bool enable;
+	int ret;
+
+	ret = kstrtobool_from_user(ubuf, len, &enable);
+	if (ret < 0)
+		return ret;
+
+	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
+		if (!skl_watermark_ipc_enabled(i915) && enable)
+			drm_info(&i915->drm,
+				 "Enabling IPC: WM will be proper only after next commit\n");
+		i915->display.wm.ipc_enabled = enable;
+		skl_watermark_ipc_update(i915);
+	}
+
+	return len;
+}
+
+static const struct file_operations skl_watermark_ipc_status_fops = {
+	.owner = THIS_MODULE,
+	.open = skl_watermark_ipc_status_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release,
+	.write = skl_watermark_ipc_status_write
+};
+
+void skl_watermark_ipc_debugfs_register(struct drm_i915_private *i915)
+{
+	struct drm_minor *minor = i915->drm.primary;
+
+	if (!HAS_IPC(i915))
+		return;
+
+	debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
+			    &skl_watermark_ipc_status_fops);
+}
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.h b/drivers/gpu/drm/i915/display/skl_watermark.h
new file mode 100644
index 000000000000..7a5a4e67cd73
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/skl_watermark.h
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#ifndef __SKL_WATERMARK_H__
+#define __SKL_WATERMARK_H__
+
+#include <linux/types.h>
+
+#include "intel_display.h"
+#include "intel_global_state.h"
+#include "intel_pm_types.h"
+
+struct drm_i915_private;
+struct intel_atomic_state;
+struct intel_bw_state;
+struct intel_crtc;
+struct intel_crtc_state;
+struct intel_plane;
+
+u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *i915);
+
+void intel_sagv_pre_plane_update(struct intel_atomic_state *state);
+void intel_sagv_post_plane_update(struct intel_atomic_state *state);
+bool intel_can_enable_sagv(struct drm_i915_private *i915,
+			   const struct intel_bw_state *bw_state);
+
+u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
+			    const struct skl_ddb_entry *entry);
+
+void skl_write_plane_wm(struct intel_plane *plane,
+			const struct intel_crtc_state *crtc_state);
+void skl_write_cursor_wm(struct intel_plane *plane,
+			 const struct intel_crtc_state *crtc_state);
+
+bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
+				 const struct skl_ddb_entry *entries,
+				 int num_entries, int ignore_idx);
+
+void skl_wm_get_hw_state(struct drm_i915_private *i915);
+void skl_wm_sanitize(struct drm_i915_private *i915);
+
+void intel_wm_state_verify(struct intel_crtc *crtc,
+			   struct intel_crtc_state *new_crtc_state);
+
+void skl_watermark_ipc_init(struct drm_i915_private *i915);
+void skl_watermark_ipc_update(struct drm_i915_private *i915);
+bool skl_watermark_ipc_enabled(struct drm_i915_private *i915);
+void skl_watermark_ipc_debugfs_register(struct drm_i915_private *i915);
+
+void skl_wm_init(struct drm_i915_private *i915);
+
+struct intel_dbuf_state {
+	struct intel_global_state base;
+
+	struct skl_ddb_entry ddb[I915_MAX_PIPES];
+	unsigned int weight[I915_MAX_PIPES];
+	u8 slices[I915_MAX_PIPES];
+	u8 enabled_slices;
+	u8 active_pipes;
+	bool joined_mbus;
+};
+
+struct intel_dbuf_state *
+intel_atomic_get_dbuf_state(struct intel_atomic_state *state);
+
+#define to_intel_dbuf_state(x) container_of((x), struct intel_dbuf_state, base)
+#define intel_atomic_get_old_dbuf_state(state) \
+	to_intel_dbuf_state(intel_atomic_get_old_global_obj_state(state, &to_i915(state->base.dev)->display.dbuf.obj))
+#define intel_atomic_get_new_dbuf_state(state) \
+	to_intel_dbuf_state(intel_atomic_get_new_global_obj_state(state, &to_i915(state->base.dev)->display.dbuf.obj))
+
+int intel_dbuf_init(struct drm_i915_private *i915);
+void intel_dbuf_pre_plane_update(struct intel_atomic_state *state);
+void intel_dbuf_post_plane_update(struct intel_atomic_state *state);
+void intel_mbus_dbox_update(struct intel_atomic_state *state);
+
+#endif /* __SKL_WATERMARK_H__ */
+
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c
index 35136d26e517..b3f5ca280ef2 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi.c
@@ -822,9 +822,9 @@ static void intel_dsi_pre_enable(struct intel_atomic_state *state,
 		u32 val;
 
 		/* Disable DPOunit clock gating, can stall pipe */
-		val = intel_de_read(dev_priv, DSPCLK_GATE_D);
+		val = intel_de_read(dev_priv, DSPCLK_GATE_D(dev_priv));
 		val |= DPOUNIT_CLOCK_GATE_DISABLE;
-		intel_de_write(dev_priv, DSPCLK_GATE_D, val);
+		intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), val);
 	}
 
 	if (!IS_GEMINILAKE(dev_priv))
@@ -998,9 +998,9 @@ static void intel_dsi_post_disable(struct intel_atomic_state *state,
 
 		vlv_dsi_pll_disable(encoder);
 
-		val = intel_de_read(dev_priv, DSPCLK_GATE_D);
+		val = intel_de_read(dev_priv, DSPCLK_GATE_D(dev_priv));
 		val &= ~DPOUNIT_CLOCK_GATE_DISABLE;
-		intel_de_write(dev_priv, DSPCLK_GATE_D, val);
+		intel_de_write(dev_priv, DSPCLK_GATE_D(dev_priv), val);
 	}
 
 	/* Assert reset */
@@ -1277,13 +1277,12 @@ static void intel_dsi_get_config(struct intel_encoder *encoder,
 		pclk = vlv_dsi_get_pclk(encoder, pipe_config);
 	}
 
-	if (intel_dsi->dual_link)
-		pclk *= 2;
+	pipe_config->port_clock = pclk;
 
-	if (pclk) {
-		pipe_config->hw.adjusted_mode.crtc_clock = pclk;
-		pipe_config->port_clock = pclk;
-	}
+	/* FIXME definitely not right for burst/cmd mode/pixel overlap */
+	pipe_config->hw.adjusted_mode.crtc_clock = pclk;
+	if (intel_dsi->dual_link)
+		pipe_config->hw.adjusted_mode.crtc_clock *= 2;
 }
 
 /* return txclkesc cycles in terms of divider and duration in us */
@@ -1872,9 +1871,9 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv)
 		return;
 
 	if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
-		dev_priv->mipi_mmio_base = BXT_MIPI_BASE;
+		dev_priv->display.dsi.mmio_base = BXT_MIPI_BASE;
 	else
-		dev_priv->mipi_mmio_base = VLV_MIPI_BASE;
+		dev_priv->display.dsi.mmio_base = VLV_MIPI_BASE;
 
 	intel_dsi = kzalloc(sizeof(*intel_dsi), GFP_KERNEL);
 	if (!intel_dsi)
@@ -1936,13 +1935,9 @@ void vlv_dsi_init(struct drm_i915_private *dev_priv)
 	if (drm_WARN_ON(&dev_priv->drm, intel_connector->panel.vbt.dsi.bl_ports & ~intel_dsi->ports))
 		intel_connector->panel.vbt.dsi.bl_ports &= intel_dsi->ports;
 
-	intel_dsi->dcs_backlight_ports = intel_connector->panel.vbt.dsi.bl_ports;
-
 	if (drm_WARN_ON(&dev_priv->drm, intel_connector->panel.vbt.dsi.cabc_ports & ~intel_dsi->ports))
 		intel_connector->panel.vbt.dsi.cabc_ports &= intel_dsi->ports;
 
-	intel_dsi->dcs_cabc_ports = intel_connector->panel.vbt.dsi.cabc_ports;
-
 	/* Create a DSI host (and a device) for each port. */
 	for_each_dsi_port(port, intel_dsi->ports) {
 		struct intel_dsi_host *host;
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
index 5894b0138343..af7402127cd9 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
@@ -113,6 +113,61 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv,
 	return 0;
 }
 
+static int vlv_dsi_pclk(struct intel_encoder *encoder,
+			struct intel_crtc_state *config)
+{
+	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
+	int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
+	u32 dsi_clock;
+	u32 pll_ctl, pll_div;
+	u32 m = 0, p = 0, n;
+	int refclk = IS_CHERRYVIEW(dev_priv) ? 100000 : 25000;
+	int i;
+
+	pll_ctl = config->dsi_pll.ctrl;
+	pll_div = config->dsi_pll.div;
+
+	/* mask out other bits and extract the P1 divisor */
+	pll_ctl &= DSI_PLL_P1_POST_DIV_MASK;
+	pll_ctl = pll_ctl >> (DSI_PLL_P1_POST_DIV_SHIFT - 2);
+
+	/* N1 divisor */
+	n = (pll_div & DSI_PLL_N1_DIV_MASK) >> DSI_PLL_N1_DIV_SHIFT;
+	n = 1 << n; /* register has log2(N1) */
+
+	/* mask out the other bits and extract the M1 divisor */
+	pll_div &= DSI_PLL_M1_DIV_MASK;
+	pll_div = pll_div >> DSI_PLL_M1_DIV_SHIFT;
+
+	while (pll_ctl) {
+		pll_ctl = pll_ctl >> 1;
+		p++;
+	}
+	p--;
+
+	if (!p) {
+		drm_err(&dev_priv->drm, "wrong P1 divisor\n");
+		return 0;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(lfsr_converts); i++) {
+		if (lfsr_converts[i] == pll_div)
+			break;
+	}
+
+	if (i == ARRAY_SIZE(lfsr_converts)) {
+		drm_err(&dev_priv->drm, "wrong m_seed programmed\n");
+		return 0;
+	}
+
+	m = i + 62;
+
+	dsi_clock = (m * refclk) / (p * n);
+
+	return DIV_ROUND_CLOSEST(dsi_clock * intel_dsi->lane_count, bpp);
+}
+
 /*
  * XXX: The muxing and gating is hard coded for now. Need to add support for
  * sharing PLLs with two DSI outputs.
@@ -122,8 +177,7 @@ int vlv_dsi_pll_compute(struct intel_encoder *encoder,
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
-	int ret;
-	u32 dsi_clk;
+	int pclk, dsi_clk, ret;
 
 	dsi_clk = dsi_clk_from_pclk(intel_dsi->pclk, intel_dsi->pixel_format,
 				    intel_dsi->lane_count);
@@ -145,6 +199,14 @@ int vlv_dsi_pll_compute(struct intel_encoder *encoder,
 	drm_dbg_kms(&dev_priv->drm, "dsi pll div %08x, ctrl %08x\n",
 		    config->dsi_pll.div, config->dsi_pll.ctrl);
 
+	pclk = vlv_dsi_pclk(encoder, config);
+	config->port_clock = pclk;
+
+	/* FIXME definitely not right for burst/cmd mode/pixel overlap */
+	config->hw.adjusted_mode.crtc_clock = pclk;
+	if (intel_dsi->dual_link)
+		config->hw.adjusted_mode.crtc_clock *= 2;
+
 	return 0;
 }
 
@@ -262,13 +324,7 @@ u32 vlv_dsi_get_pclk(struct intel_encoder *encoder,
 		     struct intel_crtc_state *config)
 {
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
-	int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
-	u32 dsi_clock, pclk;
 	u32 pll_ctl, pll_div;
-	u32 m = 0, p = 0, n;
-	int refclk = IS_CHERRYVIEW(dev_priv) ? 100000 : 25000;
-	int i;
 
 	drm_dbg_kms(&dev_priv->drm, "\n");
 
@@ -280,65 +336,31 @@ u32 vlv_dsi_get_pclk(struct intel_encoder *encoder,
 	config->dsi_pll.ctrl = pll_ctl & ~DSI_PLL_LOCK;
 	config->dsi_pll.div = pll_div;
 
-	/* mask out other bits and extract the P1 divisor */
-	pll_ctl &= DSI_PLL_P1_POST_DIV_MASK;
-	pll_ctl = pll_ctl >> (DSI_PLL_P1_POST_DIV_SHIFT - 2);
-
-	/* N1 divisor */
-	n = (pll_div & DSI_PLL_N1_DIV_MASK) >> DSI_PLL_N1_DIV_SHIFT;
-	n = 1 << n; /* register has log2(N1) */
-
-	/* mask out the other bits and extract the M1 divisor */
-	pll_div &= DSI_PLL_M1_DIV_MASK;
-	pll_div = pll_div >> DSI_PLL_M1_DIV_SHIFT;
-
-	while (pll_ctl) {
-		pll_ctl = pll_ctl >> 1;
-		p++;
-	}
-	p--;
-
-	if (!p) {
-		drm_err(&dev_priv->drm, "wrong P1 divisor\n");
-		return 0;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(lfsr_converts); i++) {
-		if (lfsr_converts[i] == pll_div)
-			break;
-	}
-
-	if (i == ARRAY_SIZE(lfsr_converts)) {
-		drm_err(&dev_priv->drm, "wrong m_seed programmed\n");
-		return 0;
-	}
-
-	m = i + 62;
+	return vlv_dsi_pclk(encoder, config);
+}
 
-	dsi_clock = (m * refclk) / (p * n);
+static int bxt_dsi_pclk(struct intel_encoder *encoder,
+			const struct intel_crtc_state *config)
+{
+	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
+	int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
+	u32 dsi_ratio, dsi_clk;
 
-	pclk = DIV_ROUND_CLOSEST(dsi_clock * intel_dsi->lane_count, bpp);
+	dsi_ratio = config->dsi_pll.ctrl & BXT_DSI_PLL_RATIO_MASK;
+	dsi_clk = (dsi_ratio * BXT_REF_CLOCK_KHZ) / 2;
 
-	return pclk;
+	return DIV_ROUND_CLOSEST(dsi_clk * intel_dsi->lane_count, bpp);
 }
 
 u32 bxt_dsi_get_pclk(struct intel_encoder *encoder,
 		     struct intel_crtc_state *config)
 {
-	u32 pclk;
-	u32 dsi_clk;
-	u32 dsi_ratio;
-	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-	int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
+	u32 pclk;
 
 	config->dsi_pll.ctrl = intel_de_read(dev_priv, BXT_DSI_PLL_CTL);
 
-	dsi_ratio = config->dsi_pll.ctrl & BXT_DSI_PLL_RATIO_MASK;
-
-	dsi_clk = (dsi_ratio * BXT_REF_CLOCK_KHZ) / 2;
-
-	pclk = DIV_ROUND_CLOSEST(dsi_clk * intel_dsi->lane_count, bpp);
+	pclk = bxt_dsi_pclk(encoder, config);
 
 	drm_dbg(&dev_priv->drm, "Calculated pclk=%u\n", pclk);
 	return pclk;
@@ -463,6 +485,7 @@ int bxt_dsi_pll_compute(struct intel_encoder *encoder,
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
 	u8 dsi_ratio, dsi_ratio_min, dsi_ratio_max;
 	u32 dsi_clk;
+	int pclk;
 
 	dsi_clk = dsi_clk_from_pclk(intel_dsi->pclk, intel_dsi->pixel_format,
 				    intel_dsi->lane_count);
@@ -502,6 +525,14 @@ int bxt_dsi_pll_compute(struct intel_encoder *encoder,
 	if (IS_BROXTON(dev_priv) && dsi_ratio <= 50)
 		config->dsi_pll.ctrl |= BXT_DSI_PLL_PVD_RATIO_1;
 
+	pclk = bxt_dsi_pclk(encoder, config);
+	config->port_clock = pclk;
+
+	/* FIXME definitely not right for burst/cmd mode/pixel overlap */
+	config->hw.adjusted_mode.crtc_clock = pclk;
+	if (intel_dsi->dual_link)
+		config->hw.adjusted_mode.crtc_clock *= 2;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_regs.h b/drivers/gpu/drm/i915/display/vlv_dsi_regs.h
index 356e51515346..e065b8f2ee08 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi_regs.h
+++ b/drivers/gpu/drm/i915/display/vlv_dsi_regs.h
@@ -11,6 +11,8 @@
 #define VLV_MIPI_BASE			VLV_DISPLAY_BASE
 #define BXT_MIPI_BASE			0x60000
 
+#define _MIPI_MMIO_BASE(__i915) ((__i915)->display.dsi.mmio_base)
+
 #define _MIPI_PORT(port, a, c)	(((port) == PORT_A) ? a : c)	/* ports A and C only */
 #define _MMIO_MIPI(port, a, c)	_MMIO(_MIPI_PORT(port, a, c))
 
@@ -96,8 +98,8 @@
 
 /* MIPI DSI Controller and D-PHY registers */
 
-#define _MIPIA_DEVICE_READY		(dev_priv->mipi_mmio_base + 0xb000)
-#define _MIPIC_DEVICE_READY		(dev_priv->mipi_mmio_base + 0xb800)
+#define _MIPIA_DEVICE_READY		(_MIPI_MMIO_BASE(dev_priv) + 0xb000)
+#define _MIPIC_DEVICE_READY		(_MIPI_MMIO_BASE(dev_priv) + 0xb800)
 #define MIPI_DEVICE_READY(port)		_MMIO_MIPI(port, _MIPIA_DEVICE_READY, _MIPIC_DEVICE_READY)
 #define  BUS_POSSESSION					(1 << 3) /* set to give bus to receiver */
 #define  ULPS_STATE_MASK				(3 << 1)
@@ -106,11 +108,11 @@
 #define  ULPS_STATE_NORMAL_OPERATION			(0 << 1)
 #define  DEVICE_READY					(1 << 0)
 
-#define _MIPIA_INTR_STAT		(dev_priv->mipi_mmio_base + 0xb004)
-#define _MIPIC_INTR_STAT		(dev_priv->mipi_mmio_base + 0xb804)
+#define _MIPIA_INTR_STAT		(_MIPI_MMIO_BASE(dev_priv) + 0xb004)
+#define _MIPIC_INTR_STAT		(_MIPI_MMIO_BASE(dev_priv) + 0xb804)
 #define MIPI_INTR_STAT(port)		_MMIO_MIPI(port, _MIPIA_INTR_STAT, _MIPIC_INTR_STAT)
-#define _MIPIA_INTR_EN			(dev_priv->mipi_mmio_base + 0xb008)
-#define _MIPIC_INTR_EN			(dev_priv->mipi_mmio_base + 0xb808)
+#define _MIPIA_INTR_EN			(_MIPI_MMIO_BASE(dev_priv) + 0xb008)
+#define _MIPIC_INTR_EN			(_MIPI_MMIO_BASE(dev_priv) + 0xb808)
 #define MIPI_INTR_EN(port)		_MMIO_MIPI(port, _MIPIA_INTR_EN, _MIPIC_INTR_EN)
 #define  TEARING_EFFECT					(1 << 31)
 #define  SPL_PKT_SENT_INTERRUPT				(1 << 30)
@@ -145,8 +147,8 @@
 #define  RXSOT_SYNC_ERROR				(1 << 1)
 #define  RXSOT_ERROR					(1 << 0)
 
-#define _MIPIA_DSI_FUNC_PRG		(dev_priv->mipi_mmio_base + 0xb00c)
-#define _MIPIC_DSI_FUNC_PRG		(dev_priv->mipi_mmio_base + 0xb80c)
+#define _MIPIA_DSI_FUNC_PRG		(_MIPI_MMIO_BASE(dev_priv) + 0xb00c)
+#define _MIPIC_DSI_FUNC_PRG		(_MIPI_MMIO_BASE(dev_priv) + 0xb80c)
 #define MIPI_DSI_FUNC_PRG(port)		_MMIO_MIPI(port, _MIPIA_DSI_FUNC_PRG, _MIPIC_DSI_FUNC_PRG)
 #define  CMD_MODE_DATA_WIDTH_MASK			(7 << 13)
 #define  CMD_MODE_NOT_SUPPORTED				(0 << 13)
@@ -168,76 +170,76 @@
 #define  DATA_LANES_PRG_REG_SHIFT			0
 #define  DATA_LANES_PRG_REG_MASK			(7 << 0)
 
-#define _MIPIA_HS_TX_TIMEOUT		(dev_priv->mipi_mmio_base + 0xb010)
-#define _MIPIC_HS_TX_TIMEOUT		(dev_priv->mipi_mmio_base + 0xb810)
+#define _MIPIA_HS_TX_TIMEOUT		(_MIPI_MMIO_BASE(dev_priv) + 0xb010)
+#define _MIPIC_HS_TX_TIMEOUT		(_MIPI_MMIO_BASE(dev_priv) + 0xb810)
 #define MIPI_HS_TX_TIMEOUT(port)	_MMIO_MIPI(port, _MIPIA_HS_TX_TIMEOUT, _MIPIC_HS_TX_TIMEOUT)
 #define  HIGH_SPEED_TX_TIMEOUT_COUNTER_MASK		0xffffff
 
-#define _MIPIA_LP_RX_TIMEOUT		(dev_priv->mipi_mmio_base + 0xb014)
-#define _MIPIC_LP_RX_TIMEOUT		(dev_priv->mipi_mmio_base + 0xb814)
+#define _MIPIA_LP_RX_TIMEOUT		(_MIPI_MMIO_BASE(dev_priv) + 0xb014)
+#define _MIPIC_LP_RX_TIMEOUT		(_MIPI_MMIO_BASE(dev_priv) + 0xb814)
 #define MIPI_LP_RX_TIMEOUT(port)	_MMIO_MIPI(port, _MIPIA_LP_RX_TIMEOUT, _MIPIC_LP_RX_TIMEOUT)
 #define  LOW_POWER_RX_TIMEOUT_COUNTER_MASK		0xffffff
 
-#define _MIPIA_TURN_AROUND_TIMEOUT	(dev_priv->mipi_mmio_base + 0xb018)
-#define _MIPIC_TURN_AROUND_TIMEOUT	(dev_priv->mipi_mmio_base + 0xb818)
+#define _MIPIA_TURN_AROUND_TIMEOUT	(_MIPI_MMIO_BASE(dev_priv) + 0xb018)
+#define _MIPIC_TURN_AROUND_TIMEOUT	(_MIPI_MMIO_BASE(dev_priv) + 0xb818)
 #define MIPI_TURN_AROUND_TIMEOUT(port)	_MMIO_MIPI(port, _MIPIA_TURN_AROUND_TIMEOUT, _MIPIC_TURN_AROUND_TIMEOUT)
 #define  TURN_AROUND_TIMEOUT_MASK			0x3f
 
-#define _MIPIA_DEVICE_RESET_TIMER	(dev_priv->mipi_mmio_base + 0xb01c)
-#define _MIPIC_DEVICE_RESET_TIMER	(dev_priv->mipi_mmio_base + 0xb81c)
+#define _MIPIA_DEVICE_RESET_TIMER	(_MIPI_MMIO_BASE(dev_priv) + 0xb01c)
+#define _MIPIC_DEVICE_RESET_TIMER	(_MIPI_MMIO_BASE(dev_priv) + 0xb81c)
 #define MIPI_DEVICE_RESET_TIMER(port)	_MMIO_MIPI(port, _MIPIA_DEVICE_RESET_TIMER, _MIPIC_DEVICE_RESET_TIMER)
 #define  DEVICE_RESET_TIMER_MASK			0xffff
 
-#define _MIPIA_DPI_RESOLUTION		(dev_priv->mipi_mmio_base + 0xb020)
-#define _MIPIC_DPI_RESOLUTION		(dev_priv->mipi_mmio_base + 0xb820)
+#define _MIPIA_DPI_RESOLUTION		(_MIPI_MMIO_BASE(dev_priv) + 0xb020)
+#define _MIPIC_DPI_RESOLUTION		(_MIPI_MMIO_BASE(dev_priv) + 0xb820)
 #define MIPI_DPI_RESOLUTION(port)	_MMIO_MIPI(port, _MIPIA_DPI_RESOLUTION, _MIPIC_DPI_RESOLUTION)
 #define  VERTICAL_ADDRESS_SHIFT				16
 #define  VERTICAL_ADDRESS_MASK				(0xffff << 16)
 #define  HORIZONTAL_ADDRESS_SHIFT			0
 #define  HORIZONTAL_ADDRESS_MASK			0xffff
 
-#define _MIPIA_DBI_FIFO_THROTTLE	(dev_priv->mipi_mmio_base + 0xb024)
-#define _MIPIC_DBI_FIFO_THROTTLE	(dev_priv->mipi_mmio_base + 0xb824)
+#define _MIPIA_DBI_FIFO_THROTTLE	(_MIPI_MMIO_BASE(dev_priv) + 0xb024)
+#define _MIPIC_DBI_FIFO_THROTTLE	(_MIPI_MMIO_BASE(dev_priv) + 0xb824)
 #define MIPI_DBI_FIFO_THROTTLE(port)	_MMIO_MIPI(port, _MIPIA_DBI_FIFO_THROTTLE, _MIPIC_DBI_FIFO_THROTTLE)
 #define  DBI_FIFO_EMPTY_HALF				(0 << 0)
 #define  DBI_FIFO_EMPTY_QUARTER				(1 << 0)
 #define  DBI_FIFO_EMPTY_7_LOCATIONS			(2 << 0)
 
 /* regs below are bits 15:0 */
-#define _MIPIA_HSYNC_PADDING_COUNT	(dev_priv->mipi_mmio_base + 0xb028)
-#define _MIPIC_HSYNC_PADDING_COUNT	(dev_priv->mipi_mmio_base + 0xb828)
+#define _MIPIA_HSYNC_PADDING_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb028)
+#define _MIPIC_HSYNC_PADDING_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb828)
 #define MIPI_HSYNC_PADDING_COUNT(port)	_MMIO_MIPI(port, _MIPIA_HSYNC_PADDING_COUNT, _MIPIC_HSYNC_PADDING_COUNT)
 
-#define _MIPIA_HBP_COUNT		(dev_priv->mipi_mmio_base + 0xb02c)
-#define _MIPIC_HBP_COUNT		(dev_priv->mipi_mmio_base + 0xb82c)
+#define _MIPIA_HBP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb02c)
+#define _MIPIC_HBP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb82c)
 #define MIPI_HBP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_HBP_COUNT, _MIPIC_HBP_COUNT)
 
-#define _MIPIA_HFP_COUNT		(dev_priv->mipi_mmio_base + 0xb030)
-#define _MIPIC_HFP_COUNT		(dev_priv->mipi_mmio_base + 0xb830)
+#define _MIPIA_HFP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb030)
+#define _MIPIC_HFP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb830)
 #define MIPI_HFP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_HFP_COUNT, _MIPIC_HFP_COUNT)
 
-#define _MIPIA_HACTIVE_AREA_COUNT	(dev_priv->mipi_mmio_base + 0xb034)
-#define _MIPIC_HACTIVE_AREA_COUNT	(dev_priv->mipi_mmio_base + 0xb834)
+#define _MIPIA_HACTIVE_AREA_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb034)
+#define _MIPIC_HACTIVE_AREA_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb834)
 #define MIPI_HACTIVE_AREA_COUNT(port)	_MMIO_MIPI(port, _MIPIA_HACTIVE_AREA_COUNT, _MIPIC_HACTIVE_AREA_COUNT)
 
-#define _MIPIA_VSYNC_PADDING_COUNT	(dev_priv->mipi_mmio_base + 0xb038)
-#define _MIPIC_VSYNC_PADDING_COUNT	(dev_priv->mipi_mmio_base + 0xb838)
+#define _MIPIA_VSYNC_PADDING_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb038)
+#define _MIPIC_VSYNC_PADDING_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb838)
 #define MIPI_VSYNC_PADDING_COUNT(port)	_MMIO_MIPI(port, _MIPIA_VSYNC_PADDING_COUNT, _MIPIC_VSYNC_PADDING_COUNT)
 
-#define _MIPIA_VBP_COUNT		(dev_priv->mipi_mmio_base + 0xb03c)
-#define _MIPIC_VBP_COUNT		(dev_priv->mipi_mmio_base + 0xb83c)
+#define _MIPIA_VBP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb03c)
+#define _MIPIC_VBP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb83c)
 #define MIPI_VBP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_VBP_COUNT, _MIPIC_VBP_COUNT)
 
-#define _MIPIA_VFP_COUNT		(dev_priv->mipi_mmio_base + 0xb040)
-#define _MIPIC_VFP_COUNT		(dev_priv->mipi_mmio_base + 0xb840)
+#define _MIPIA_VFP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb040)
+#define _MIPIC_VFP_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb840)
 #define MIPI_VFP_COUNT(port)		_MMIO_MIPI(port, _MIPIA_VFP_COUNT, _MIPIC_VFP_COUNT)
 
-#define _MIPIA_HIGH_LOW_SWITCH_COUNT	(dev_priv->mipi_mmio_base + 0xb044)
-#define _MIPIC_HIGH_LOW_SWITCH_COUNT	(dev_priv->mipi_mmio_base + 0xb844)
+#define _MIPIA_HIGH_LOW_SWITCH_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb044)
+#define _MIPIC_HIGH_LOW_SWITCH_COUNT	(_MIPI_MMIO_BASE(dev_priv) + 0xb844)
 #define MIPI_HIGH_LOW_SWITCH_COUNT(port)	_MMIO_MIPI(port,	_MIPIA_HIGH_LOW_SWITCH_COUNT, _MIPIC_HIGH_LOW_SWITCH_COUNT)
 
-#define _MIPIA_DPI_CONTROL		(dev_priv->mipi_mmio_base + 0xb048)
-#define _MIPIC_DPI_CONTROL		(dev_priv->mipi_mmio_base + 0xb848)
+#define _MIPIA_DPI_CONTROL		(_MIPI_MMIO_BASE(dev_priv) + 0xb048)
+#define _MIPIC_DPI_CONTROL		(_MIPI_MMIO_BASE(dev_priv) + 0xb848)
 #define MIPI_DPI_CONTROL(port)		_MMIO_MIPI(port, _MIPIA_DPI_CONTROL, _MIPIC_DPI_CONTROL)
 #define  DPI_LP_MODE					(1 << 6)
 #define  BACKLIGHT_OFF					(1 << 5)
@@ -247,27 +249,27 @@
 #define  TURN_ON					(1 << 1)
 #define  SHUTDOWN					(1 << 0)
 
-#define _MIPIA_DPI_DATA			(dev_priv->mipi_mmio_base + 0xb04c)
-#define _MIPIC_DPI_DATA			(dev_priv->mipi_mmio_base + 0xb84c)
+#define _MIPIA_DPI_DATA			(_MIPI_MMIO_BASE(dev_priv) + 0xb04c)
+#define _MIPIC_DPI_DATA			(_MIPI_MMIO_BASE(dev_priv) + 0xb84c)
 #define MIPI_DPI_DATA(port)		_MMIO_MIPI(port, _MIPIA_DPI_DATA, _MIPIC_DPI_DATA)
 #define  COMMAND_BYTE_SHIFT				0
 #define  COMMAND_BYTE_MASK				(0x3f << 0)
 
-#define _MIPIA_INIT_COUNT		(dev_priv->mipi_mmio_base + 0xb050)
-#define _MIPIC_INIT_COUNT		(dev_priv->mipi_mmio_base + 0xb850)
+#define _MIPIA_INIT_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb050)
+#define _MIPIC_INIT_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb850)
 #define MIPI_INIT_COUNT(port)		_MMIO_MIPI(port, _MIPIA_INIT_COUNT, _MIPIC_INIT_COUNT)
 #define  MASTER_INIT_TIMER_SHIFT			0
 #define  MASTER_INIT_TIMER_MASK				(0xffff << 0)
 
-#define _MIPIA_MAX_RETURN_PKT_SIZE	(dev_priv->mipi_mmio_base + 0xb054)
-#define _MIPIC_MAX_RETURN_PKT_SIZE	(dev_priv->mipi_mmio_base + 0xb854)
+#define _MIPIA_MAX_RETURN_PKT_SIZE	(_MIPI_MMIO_BASE(dev_priv) + 0xb054)
+#define _MIPIC_MAX_RETURN_PKT_SIZE	(_MIPI_MMIO_BASE(dev_priv) + 0xb854)
 #define MIPI_MAX_RETURN_PKT_SIZE(port)	_MMIO_MIPI(port, \
 			_MIPIA_MAX_RETURN_PKT_SIZE, _MIPIC_MAX_RETURN_PKT_SIZE)
 #define  MAX_RETURN_PKT_SIZE_SHIFT			0
 #define  MAX_RETURN_PKT_SIZE_MASK			(0x3ff << 0)
 
-#define _MIPIA_VIDEO_MODE_FORMAT	(dev_priv->mipi_mmio_base + 0xb058)
-#define _MIPIC_VIDEO_MODE_FORMAT	(dev_priv->mipi_mmio_base + 0xb858)
+#define _MIPIA_VIDEO_MODE_FORMAT	(_MIPI_MMIO_BASE(dev_priv) + 0xb058)
+#define _MIPIC_VIDEO_MODE_FORMAT	(_MIPI_MMIO_BASE(dev_priv) + 0xb858)
 #define MIPI_VIDEO_MODE_FORMAT(port)	_MMIO_MIPI(port, _MIPIA_VIDEO_MODE_FORMAT, _MIPIC_VIDEO_MODE_FORMAT)
 #define  RANDOM_DPI_DISPLAY_RESOLUTION			(1 << 4)
 #define  DISABLE_VIDEO_BTA				(1 << 3)
@@ -276,8 +278,8 @@
 #define  VIDEO_MODE_NON_BURST_WITH_SYNC_EVENTS		(2 << 0)
 #define  VIDEO_MODE_BURST				(3 << 0)
 
-#define _MIPIA_EOT_DISABLE		(dev_priv->mipi_mmio_base + 0xb05c)
-#define _MIPIC_EOT_DISABLE		(dev_priv->mipi_mmio_base + 0xb85c)
+#define _MIPIA_EOT_DISABLE		(_MIPI_MMIO_BASE(dev_priv) + 0xb05c)
+#define _MIPIC_EOT_DISABLE		(_MIPI_MMIO_BASE(dev_priv) + 0xb85c)
 #define MIPI_EOT_DISABLE(port)		_MMIO_MIPI(port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE)
 #define  BXT_DEFEATURE_DPI_FIFO_CTR			(1 << 9)
 #define  BXT_DPHY_DEFEATURE_EN				(1 << 8)
@@ -290,35 +292,35 @@
 #define  CLOCKSTOP					(1 << 1)
 #define  EOT_DISABLE					(1 << 0)
 
-#define _MIPIA_LP_BYTECLK		(dev_priv->mipi_mmio_base + 0xb060)
-#define _MIPIC_LP_BYTECLK		(dev_priv->mipi_mmio_base + 0xb860)
+#define _MIPIA_LP_BYTECLK		(_MIPI_MMIO_BASE(dev_priv) + 0xb060)
+#define _MIPIC_LP_BYTECLK		(_MIPI_MMIO_BASE(dev_priv) + 0xb860)
 #define MIPI_LP_BYTECLK(port)		_MMIO_MIPI(port, _MIPIA_LP_BYTECLK, _MIPIC_LP_BYTECLK)
 #define  LP_BYTECLK_SHIFT				0
 #define  LP_BYTECLK_MASK				(0xffff << 0)
 
-#define _MIPIA_TLPX_TIME_COUNT		(dev_priv->mipi_mmio_base + 0xb0a4)
-#define _MIPIC_TLPX_TIME_COUNT		(dev_priv->mipi_mmio_base + 0xb8a4)
+#define _MIPIA_TLPX_TIME_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb0a4)
+#define _MIPIC_TLPX_TIME_COUNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb8a4)
 #define MIPI_TLPX_TIME_COUNT(port)	 _MMIO_MIPI(port, _MIPIA_TLPX_TIME_COUNT, _MIPIC_TLPX_TIME_COUNT)
 
-#define _MIPIA_CLK_LANE_TIMING		(dev_priv->mipi_mmio_base + 0xb098)
-#define _MIPIC_CLK_LANE_TIMING		(dev_priv->mipi_mmio_base + 0xb898)
+#define _MIPIA_CLK_LANE_TIMING		(_MIPI_MMIO_BASE(dev_priv) + 0xb098)
+#define _MIPIC_CLK_LANE_TIMING		(_MIPI_MMIO_BASE(dev_priv) + 0xb898)
 #define MIPI_CLK_LANE_TIMING(port)	 _MMIO_MIPI(port, _MIPIA_CLK_LANE_TIMING, _MIPIC_CLK_LANE_TIMING)
 
 /* bits 31:0 */
-#define _MIPIA_LP_GEN_DATA		(dev_priv->mipi_mmio_base + 0xb064)
-#define _MIPIC_LP_GEN_DATA		(dev_priv->mipi_mmio_base + 0xb864)
+#define _MIPIA_LP_GEN_DATA		(_MIPI_MMIO_BASE(dev_priv) + 0xb064)
+#define _MIPIC_LP_GEN_DATA		(_MIPI_MMIO_BASE(dev_priv) + 0xb864)
 #define MIPI_LP_GEN_DATA(port)		_MMIO_MIPI(port, _MIPIA_LP_GEN_DATA, _MIPIC_LP_GEN_DATA)
 
 /* bits 31:0 */
-#define _MIPIA_HS_GEN_DATA		(dev_priv->mipi_mmio_base + 0xb068)
-#define _MIPIC_HS_GEN_DATA		(dev_priv->mipi_mmio_base + 0xb868)
+#define _MIPIA_HS_GEN_DATA		(_MIPI_MMIO_BASE(dev_priv) + 0xb068)
+#define _MIPIC_HS_GEN_DATA		(_MIPI_MMIO_BASE(dev_priv) + 0xb868)
 #define MIPI_HS_GEN_DATA(port)		_MMIO_MIPI(port, _MIPIA_HS_GEN_DATA, _MIPIC_HS_GEN_DATA)
 
-#define _MIPIA_LP_GEN_CTRL		(dev_priv->mipi_mmio_base + 0xb06c)
-#define _MIPIC_LP_GEN_CTRL		(dev_priv->mipi_mmio_base + 0xb86c)
+#define _MIPIA_LP_GEN_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb06c)
+#define _MIPIC_LP_GEN_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb86c)
 #define MIPI_LP_GEN_CTRL(port)		_MMIO_MIPI(port, _MIPIA_LP_GEN_CTRL, _MIPIC_LP_GEN_CTRL)
-#define _MIPIA_HS_GEN_CTRL		(dev_priv->mipi_mmio_base + 0xb070)
-#define _MIPIC_HS_GEN_CTRL		(dev_priv->mipi_mmio_base + 0xb870)
+#define _MIPIA_HS_GEN_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb070)
+#define _MIPIC_HS_GEN_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb870)
 #define MIPI_HS_GEN_CTRL(port)		_MMIO_MIPI(port, _MIPIA_HS_GEN_CTRL, _MIPIC_HS_GEN_CTRL)
 #define  LONG_PACKET_WORD_COUNT_SHIFT			8
 #define  LONG_PACKET_WORD_COUNT_MASK			(0xffff << 8)
@@ -330,8 +332,8 @@
 #define  DATA_TYPE_MASK					(0x3f << 0)
 /* data type values, see include/video/mipi_display.h */
 
-#define _MIPIA_GEN_FIFO_STAT		(dev_priv->mipi_mmio_base + 0xb074)
-#define _MIPIC_GEN_FIFO_STAT		(dev_priv->mipi_mmio_base + 0xb874)
+#define _MIPIA_GEN_FIFO_STAT		(_MIPI_MMIO_BASE(dev_priv) + 0xb074)
+#define _MIPIC_GEN_FIFO_STAT		(_MIPI_MMIO_BASE(dev_priv) + 0xb874)
 #define MIPI_GEN_FIFO_STAT(port)	_MMIO_MIPI(port, _MIPIA_GEN_FIFO_STAT, _MIPIC_GEN_FIFO_STAT)
 #define  DPI_FIFO_EMPTY					(1 << 28)
 #define  DBI_FIFO_EMPTY					(1 << 27)
@@ -348,15 +350,15 @@
 #define  HS_DATA_FIFO_HALF_EMPTY			(1 << 1)
 #define  HS_DATA_FIFO_FULL				(1 << 0)
 
-#define _MIPIA_HS_LS_DBI_ENABLE		(dev_priv->mipi_mmio_base + 0xb078)
-#define _MIPIC_HS_LS_DBI_ENABLE		(dev_priv->mipi_mmio_base + 0xb878)
+#define _MIPIA_HS_LS_DBI_ENABLE		(_MIPI_MMIO_BASE(dev_priv) + 0xb078)
+#define _MIPIC_HS_LS_DBI_ENABLE		(_MIPI_MMIO_BASE(dev_priv) + 0xb878)
 #define MIPI_HS_LP_DBI_ENABLE(port)	_MMIO_MIPI(port, _MIPIA_HS_LS_DBI_ENABLE, _MIPIC_HS_LS_DBI_ENABLE)
 #define  DBI_HS_LP_MODE_MASK				(1 << 0)
 #define  DBI_LP_MODE					(1 << 0)
 #define  DBI_HS_MODE					(0 << 0)
 
-#define _MIPIA_DPHY_PARAM		(dev_priv->mipi_mmio_base + 0xb080)
-#define _MIPIC_DPHY_PARAM		(dev_priv->mipi_mmio_base + 0xb880)
+#define _MIPIA_DPHY_PARAM		(_MIPI_MMIO_BASE(dev_priv) + 0xb080)
+#define _MIPIC_DPHY_PARAM		(_MIPI_MMIO_BASE(dev_priv) + 0xb880)
 #define MIPI_DPHY_PARAM(port)		_MMIO_MIPI(port, _MIPIA_DPHY_PARAM, _MIPIC_DPHY_PARAM)
 #define  EXIT_ZERO_COUNT_SHIFT				24
 #define  EXIT_ZERO_COUNT_MASK				(0x3f << 24)
@@ -367,34 +369,34 @@
 #define  PREPARE_COUNT_SHIFT				0
 #define  PREPARE_COUNT_MASK				(0x3f << 0)
 
-#define _MIPIA_DBI_BW_CTRL		(dev_priv->mipi_mmio_base + 0xb084)
-#define _MIPIC_DBI_BW_CTRL		(dev_priv->mipi_mmio_base + 0xb884)
+#define _MIPIA_DBI_BW_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb084)
+#define _MIPIC_DBI_BW_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb884)
 #define MIPI_DBI_BW_CTRL(port)		_MMIO_MIPI(port, _MIPIA_DBI_BW_CTRL, _MIPIC_DBI_BW_CTRL)
 
-#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT		(dev_priv->mipi_mmio_base + 0xb088)
-#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT		(dev_priv->mipi_mmio_base + 0xb888)
+#define _MIPIA_CLK_LANE_SWITCH_TIME_CNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb088)
+#define _MIPIC_CLK_LANE_SWITCH_TIME_CNT		(_MIPI_MMIO_BASE(dev_priv) + 0xb888)
 #define MIPI_CLK_LANE_SWITCH_TIME_CNT(port)	_MMIO_MIPI(port, _MIPIA_CLK_LANE_SWITCH_TIME_CNT, _MIPIC_CLK_LANE_SWITCH_TIME_CNT)
 #define  LP_HS_SSW_CNT_SHIFT				16
 #define  LP_HS_SSW_CNT_MASK				(0xffff << 16)
 #define  HS_LP_PWR_SW_CNT_SHIFT				0
 #define  HS_LP_PWR_SW_CNT_MASK				(0xffff << 0)
 
-#define _MIPIA_STOP_STATE_STALL		(dev_priv->mipi_mmio_base + 0xb08c)
-#define _MIPIC_STOP_STATE_STALL		(dev_priv->mipi_mmio_base + 0xb88c)
+#define _MIPIA_STOP_STATE_STALL		(_MIPI_MMIO_BASE(dev_priv) + 0xb08c)
+#define _MIPIC_STOP_STATE_STALL		(_MIPI_MMIO_BASE(dev_priv) + 0xb88c)
 #define MIPI_STOP_STATE_STALL(port)	_MMIO_MIPI(port, _MIPIA_STOP_STATE_STALL, _MIPIC_STOP_STATE_STALL)
 #define  STOP_STATE_STALL_COUNTER_SHIFT			0
 #define  STOP_STATE_STALL_COUNTER_MASK			(0xff << 0)
 
-#define _MIPIA_INTR_STAT_REG_1		(dev_priv->mipi_mmio_base + 0xb090)
-#define _MIPIC_INTR_STAT_REG_1		(dev_priv->mipi_mmio_base + 0xb890)
+#define _MIPIA_INTR_STAT_REG_1		(_MIPI_MMIO_BASE(dev_priv) + 0xb090)
+#define _MIPIC_INTR_STAT_REG_1		(_MIPI_MMIO_BASE(dev_priv) + 0xb890)
 #define MIPI_INTR_STAT_REG_1(port)	_MMIO_MIPI(port, _MIPIA_INTR_STAT_REG_1, _MIPIC_INTR_STAT_REG_1)
-#define _MIPIA_INTR_EN_REG_1		(dev_priv->mipi_mmio_base + 0xb094)
-#define _MIPIC_INTR_EN_REG_1		(dev_priv->mipi_mmio_base + 0xb894)
+#define _MIPIA_INTR_EN_REG_1		(_MIPI_MMIO_BASE(dev_priv) + 0xb094)
+#define _MIPIC_INTR_EN_REG_1		(_MIPI_MMIO_BASE(dev_priv) + 0xb894)
 #define MIPI_INTR_EN_REG_1(port)	_MMIO_MIPI(port, _MIPIA_INTR_EN_REG_1, _MIPIC_INTR_EN_REG_1)
 #define  RX_CONTENTION_DETECTED				(1 << 0)
 
 /* XXX: only pipe A ?!? */
-#define MIPIA_DBI_TYPEC_CTRL		(dev_priv->mipi_mmio_base + 0xb100)
+#define MIPIA_DBI_TYPEC_CTRL		(_MIPI_MMIO_BASE(dev_priv) + 0xb100)
 #define  DBI_TYPEC_ENABLE				(1 << 31)
 #define  DBI_TYPEC_WIP					(1 << 30)
 #define  DBI_TYPEC_OPTION_SHIFT				28
@@ -407,8 +409,8 @@
 
 /* MIPI adapter registers */
 
-#define _MIPIA_CTRL			(dev_priv->mipi_mmio_base + 0xb104)
-#define _MIPIC_CTRL			(dev_priv->mipi_mmio_base + 0xb904)
+#define _MIPIA_CTRL			(_MIPI_MMIO_BASE(dev_priv) + 0xb104)
+#define _MIPIC_CTRL			(_MIPI_MMIO_BASE(dev_priv) + 0xb904)
 #define MIPI_CTRL(port)			_MMIO_MIPI(port, _MIPIA_CTRL, _MIPIC_CTRL)
 #define  ESCAPE_CLOCK_DIVIDER_SHIFT			5 /* A only */
 #define  ESCAPE_CLOCK_DIVIDER_MASK			(3 << 5)
@@ -440,21 +442,21 @@
 #define  GLK_MIPIIO_PORT_POWERED			(1 << 1) /* RO */
 #define  GLK_MIPIIO_ENABLE				(1 << 0)
 
-#define _MIPIA_DATA_ADDRESS		(dev_priv->mipi_mmio_base + 0xb108)
-#define _MIPIC_DATA_ADDRESS		(dev_priv->mipi_mmio_base + 0xb908)
+#define _MIPIA_DATA_ADDRESS		(_MIPI_MMIO_BASE(dev_priv) + 0xb108)
+#define _MIPIC_DATA_ADDRESS		(_MIPI_MMIO_BASE(dev_priv) + 0xb908)
 #define MIPI_DATA_ADDRESS(port)		_MMIO_MIPI(port, _MIPIA_DATA_ADDRESS, _MIPIC_DATA_ADDRESS)
 #define  DATA_MEM_ADDRESS_SHIFT				5
 #define  DATA_MEM_ADDRESS_MASK				(0x7ffffff << 5)
 #define  DATA_VALID					(1 << 0)
 
-#define _MIPIA_DATA_LENGTH		(dev_priv->mipi_mmio_base + 0xb10c)
-#define _MIPIC_DATA_LENGTH		(dev_priv->mipi_mmio_base + 0xb90c)
+#define _MIPIA_DATA_LENGTH		(_MIPI_MMIO_BASE(dev_priv) + 0xb10c)
+#define _MIPIC_DATA_LENGTH		(_MIPI_MMIO_BASE(dev_priv) + 0xb90c)
 #define MIPI_DATA_LENGTH(port)		_MMIO_MIPI(port, _MIPIA_DATA_LENGTH, _MIPIC_DATA_LENGTH)
 #define  DATA_LENGTH_SHIFT				0
 #define  DATA_LENGTH_MASK				(0xfffff << 0)
 
-#define _MIPIA_COMMAND_ADDRESS		(dev_priv->mipi_mmio_base + 0xb110)
-#define _MIPIC_COMMAND_ADDRESS		(dev_priv->mipi_mmio_base + 0xb910)
+#define _MIPIA_COMMAND_ADDRESS		(_MIPI_MMIO_BASE(dev_priv) + 0xb110)
+#define _MIPIC_COMMAND_ADDRESS		(_MIPI_MMIO_BASE(dev_priv) + 0xb910)
 #define MIPI_COMMAND_ADDRESS(port)	_MMIO_MIPI(port, _MIPIA_COMMAND_ADDRESS, _MIPIC_COMMAND_ADDRESS)
 #define  COMMAND_MEM_ADDRESS_SHIFT			5
 #define  COMMAND_MEM_ADDRESS_MASK			(0x7ffffff << 5)
@@ -462,18 +464,18 @@
 #define  MEMORY_WRITE_DATA_FROM_PIPE_RENDERING		(1 << 1)
 #define  COMMAND_VALID					(1 << 0)
 
-#define _MIPIA_COMMAND_LENGTH		(dev_priv->mipi_mmio_base + 0xb114)
-#define _MIPIC_COMMAND_LENGTH		(dev_priv->mipi_mmio_base + 0xb914)
+#define _MIPIA_COMMAND_LENGTH		(_MIPI_MMIO_BASE(dev_priv) + 0xb114)
+#define _MIPIC_COMMAND_LENGTH		(_MIPI_MMIO_BASE(dev_priv) + 0xb914)
 #define MIPI_COMMAND_LENGTH(port)	_MMIO_MIPI(port, _MIPIA_COMMAND_LENGTH, _MIPIC_COMMAND_LENGTH)
 #define  COMMAND_LENGTH_SHIFT(n)			(8 * (n)) /* n: 0...3 */
 #define  COMMAND_LENGTH_MASK(n)				(0xff << (8 * (n)))
 
-#define _MIPIA_READ_DATA_RETURN0	(dev_priv->mipi_mmio_base + 0xb118)
-#define _MIPIC_READ_DATA_RETURN0	(dev_priv->mipi_mmio_base + 0xb918)
+#define _MIPIA_READ_DATA_RETURN0	(_MIPI_MMIO_BASE(dev_priv) + 0xb118)
+#define _MIPIC_READ_DATA_RETURN0	(_MIPI_MMIO_BASE(dev_priv) + 0xb918)
 #define MIPI_READ_DATA_RETURN(port, n) _MMIO(_MIPI(port, _MIPIA_READ_DATA_RETURN0, _MIPIC_READ_DATA_RETURN0) + 4 * (n)) /* n: 0...7 */
 
-#define _MIPIA_READ_DATA_VALID		(dev_priv->mipi_mmio_base + 0xb138)
-#define _MIPIC_READ_DATA_VALID		(dev_priv->mipi_mmio_base + 0xb938)
+#define _MIPIA_READ_DATA_VALID		(_MIPI_MMIO_BASE(dev_priv) + 0xb138)
+#define _MIPIC_READ_DATA_VALID		(_MIPI_MMIO_BASE(dev_priv) + 0xb938)
 #define MIPI_READ_DATA_VALID(port)	_MMIO_MIPI(port, _MIPIA_READ_DATA_VALID, _MIPIC_READ_DATA_VALID)
 #define  READ_DATA_VALID(n)				(1 << (n))
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_domain.c b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
index 1674b0c5802b..d44a152ce680 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_domain.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_domain.c
@@ -397,7 +397,7 @@ struct i915_vma *
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 				     struct i915_gem_ww_ctx *ww,
 				     u32 alignment,
-				     const struct i915_ggtt_view *view,
+				     const struct i915_gtt_view *view,
 				     unsigned int flags)
 {
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
@@ -434,7 +434,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 	 */
 	vma = ERR_PTR(-ENOSPC);
 	if ((flags & PIN_MAPPABLE) == 0 &&
-	    (!view || view->type == I915_GGTT_VIEW_NORMAL))
+	    (!view || view->type == I915_GTT_VIEW_NORMAL))
 		vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0, alignment,
 						  flags | PIN_MAPPABLE |
 						  PIN_NONBLOCK);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h
index 1b88ea13435c..5a7a14e85c3f 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_lmem.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_lmem.h
@@ -12,8 +12,6 @@ struct drm_i915_private;
 struct drm_i915_gem_object;
 struct intel_memory_region;
 
-extern const struct drm_i915_gem_object_ops i915_gem_lmem_obj_ops;
-
 void __iomem *
 i915_gem_object_lmem_io_map(struct drm_i915_gem_object *obj,
 			    unsigned long n,
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
index 0c5c43852e24..73d9eda1d6b7 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c
@@ -194,17 +194,17 @@ int i915_gem_mmap_gtt_version(void)
 	return 4;
 }
 
-static inline struct i915_ggtt_view
+static inline struct i915_gtt_view
 compute_partial_view(const struct drm_i915_gem_object *obj,
 		     pgoff_t page_offset,
 		     unsigned int chunk)
 {
-	struct i915_ggtt_view view;
+	struct i915_gtt_view view;
 
 	if (i915_gem_object_is_tiled(obj))
 		chunk = roundup(chunk, tile_row_pages(obj) ?: 1);
 
-	view.type = I915_GGTT_VIEW_PARTIAL;
+	view.type = I915_GTT_VIEW_PARTIAL;
 	view.partial.offset = rounddown(page_offset, chunk);
 	view.partial.size =
 		min_t(unsigned int, chunk,
@@ -212,7 +212,7 @@ compute_partial_view(const struct drm_i915_gem_object *obj,
 
 	/* If the partial covers the entire object, just create a normal VMA. */
 	if (chunk >= obj->base.size >> PAGE_SHIFT)
-		view.type = I915_GGTT_VIEW_NORMAL;
+		view.type = I915_GTT_VIEW_NORMAL;
 
 	return view;
 }
@@ -341,12 +341,12 @@ retry:
 					  PIN_NOEVICT);
 	if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {
 		/* Use a partial view if it is bigger than available space */
-		struct i915_ggtt_view view =
+		struct i915_gtt_view view =
 			compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
 		unsigned int flags;
 
 		flags = PIN_MAPPABLE | PIN_NOSEARCH;
-		if (view.type == I915_GGTT_VIEW_NORMAL)
+		if (view.type == I915_GTT_VIEW_NORMAL)
 			flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
 
 		/*
@@ -357,7 +357,7 @@ retry:
 		vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
 		if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {
 			flags = PIN_MAPPABLE;
-			view.type = I915_GGTT_VIEW_PARTIAL;
+			view.type = I915_GTT_VIEW_PARTIAL;
 			vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
 		}
 
@@ -394,7 +394,7 @@ retry:
 
 	/* Finally, remap it using the new GTT offset */
 	ret = remap_io_mapping(area,
-			       area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
+			       area->vm_start + (vma->gtt_view.partial.offset << PAGE_SHIFT),
 			       (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT,
 			       min_t(u64, vma->size, area->vm_end - area->vm_start),
 			       &ggtt->iomap);
@@ -413,7 +413,7 @@ retry:
 	vma->mmo = mmo;
 
 	if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
-		intel_wakeref_auto(&to_gt(i915)->ggtt->userfault_wakeref,
+		intel_wakeref_auto(&to_gt(i915)->userfault_wakeref,
 				   msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
 
 	if (write) {
@@ -550,6 +550,20 @@ out:
 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 }
 
+void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj)
+{
+	struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
+	struct ttm_device *bdev = bo->bdev;
+
+	drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
+
+	if (obj->userfault_count) {
+		/* rpm wakeref provide exclusive access */
+		list_del(&obj->userfault_link);
+		obj->userfault_count = 0;
+	}
+}
+
 void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj)
 {
 	struct i915_mmap_offset *mmo, *mn;
@@ -573,6 +587,13 @@ void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj)
 		spin_lock(&obj->mmo.lock);
 	}
 	spin_unlock(&obj->mmo.lock);
+
+	if (obj->userfault_count) {
+		mutex_lock(&to_gt(to_i915(obj->base.dev))->lmem_userfault_lock);
+		list_del(&obj->userfault_link);
+		mutex_unlock(&to_gt(to_i915(obj->base.dev))->lmem_userfault_lock);
+		obj->userfault_count = 0;
+	}
 }
 
 static struct i915_mmap_offset *
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.h b/drivers/gpu/drm/i915/gem/i915_gem_mman.h
index efee9e0d2508..1fa91b3033b3 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_mman.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.h
@@ -27,6 +27,7 @@ int i915_gem_dumb_mmap_offset(struct drm_file *file_priv,
 void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj);
 void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj);
 
+void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj);
 void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj);
 
 #endif
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.c b/drivers/gpu/drm/i915/gem/i915_gem_object.c
index 85482a04d158..7ff9c7877bec 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.c
@@ -238,7 +238,7 @@ static void __i915_gem_object_free_mmaps(struct drm_i915_gem_object *obj)
 {
 	/* Skip serialisation and waking the device if known to be not used. */
 
-	if (obj->userfault_count)
+	if (obj->userfault_count && !IS_DGFX(to_i915(obj->base.dev)))
 		i915_gem_object_release_mmap_gtt(obj);
 
 	if (!RB_EMPTY_ROOT(&obj->mmo.offsets)) {
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object.h b/drivers/gpu/drm/i915/gem/i915_gem_object.h
index 6f0a3ce35567..7317d4102955 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object.h
@@ -543,7 +543,7 @@ struct i915_vma * __must_check
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 				     struct i915_gem_ww_ctx *ww,
 				     u32 alignment,
-				     const struct i915_ggtt_view *view,
+				     const struct i915_gtt_view *view,
 				     unsigned int flags);
 
 void i915_gem_object_make_unshrinkable(struct drm_i915_gem_object *obj);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
index 9f6b14ec189a..40305e2bcd49 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+++ b/drivers/gpu/drm/i915/gem/i915_gem_object_types.h
@@ -298,7 +298,8 @@ struct drm_i915_gem_object {
 	};
 
 	/**
-	 * Whether the object is currently in the GGTT mmap.
+	 * Whether the object is currently in the GGTT or any other supported
+	 * fake offset mmap backed by lmem.
 	 */
 	unsigned int userfault_count;
 	struct list_head userfault_link;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
index 8357dbdcab5c..4df50b049cea 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c
@@ -20,7 +20,7 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
 				 unsigned int sg_page_sizes)
 {
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
-	unsigned long supported = INTEL_INFO(i915)->page_sizes;
+	unsigned long supported = RUNTIME_INFO(i915)->page_sizes;
 	bool shrinkable;
 	int i;
 
@@ -66,7 +66,7 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
 	shrinkable = i915_gem_object_is_shrinkable(obj);
 
 	if (i915_gem_object_is_tiled(obj) &&
-	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
+	    i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {
 		GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
 		i915_gem_object_set_tiling_quirk(obj);
 		GEM_BUG_ON(!list_empty(&obj->mm.link));
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_pm.c
index 00359ec9d58b..3428f735e786 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_pm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.c
@@ -24,7 +24,7 @@ void i915_gem_suspend(struct drm_i915_private *i915)
 {
 	GEM_TRACE("%s\n", dev_name(i915->drm.dev));
 
-	intel_wakeref_auto(&to_gt(i915)->ggtt->userfault_wakeref, 0);
+	intel_wakeref_auto(&to_gt(i915)->userfault_wakeref, 0);
 	flush_workqueue(i915->wq);
 
 	/*
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
index 4eed3dd90ba8..f42ca1179f37 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_shmem.c
@@ -75,7 +75,7 @@ int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st,
 	if (size > resource_size(&mr->region))
 		return -ENOMEM;
 
-	if (sg_alloc_table(st, page_count, GFP_KERNEL))
+	if (sg_alloc_table(st, page_count, GFP_KERNEL | __GFP_NOWARN))
 		return -ENOMEM;
 
 	/*
@@ -137,7 +137,7 @@ int shmem_sg_alloc_table(struct drm_i915_private *i915, struct sg_table *st,
 				 * trigger the out-of-memory killer and for
 				 * this we want __GFP_RETRY_MAYFAIL.
 				 */
-				gfp |= __GFP_RETRY_MAYFAIL;
+				gfp |= __GFP_RETRY_MAYFAIL | __GFP_NOWARN;
 			}
 		} while (1);
 
@@ -209,7 +209,7 @@ static int shmem_get_pages(struct drm_i915_gem_object *obj)
 	GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
 
 rebuild_st:
-	st = kmalloc(sizeof(*st), GFP_KERNEL);
+	st = kmalloc(sizeof(*st), GFP_KERNEL | __GFP_NOWARN);
 	if (!st)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
index 166d0a4b9e8c..acc561c0f0aa 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
@@ -18,10 +18,12 @@
 #include "gt/intel_region_lmem.h"
 #include "i915_drv.h"
 #include "i915_gem_stolen.h"
+#include "i915_pci.h"
 #include "i915_reg.h"
 #include "i915_utils.h"
 #include "i915_vgpu.h"
 #include "intel_mchbar_regs.h"
+#include "intel_pci_config.h"
 
 /*
  * The BIOS typically reserves some of the system's memory for the exclusive
@@ -428,48 +430,29 @@ static int i915_gem_init_stolen(struct intel_memory_region *mem)
 	reserved_base = stolen_top;
 	reserved_size = 0;
 
-	switch (GRAPHICS_VER(i915)) {
-	case 2:
-	case 3:
-		break;
-	case 4:
-		if (!IS_G4X(i915))
-			break;
-		fallthrough;
-	case 5:
-		g4x_get_stolen_reserved(i915, uncore,
+	if (GRAPHICS_VER(i915) >= 11) {
+		icl_get_stolen_reserved(i915, uncore,
 					&reserved_base, &reserved_size);
-		break;
-	case 6:
-		gen6_get_stolen_reserved(i915, uncore,
-					 &reserved_base, &reserved_size);
-		break;
-	case 7:
-		if (IS_VALLEYVIEW(i915))
-			vlv_get_stolen_reserved(i915, uncore,
-						&reserved_base, &reserved_size);
-		else
-			gen7_get_stolen_reserved(i915, uncore,
-						 &reserved_base, &reserved_size);
-		break;
-	case 8:
-	case 9:
+	} else if (GRAPHICS_VER(i915) >= 8) {
 		if (IS_LP(i915))
 			chv_get_stolen_reserved(i915, uncore,
 						&reserved_base, &reserved_size);
 		else
 			bdw_get_stolen_reserved(i915, uncore,
 						&reserved_base, &reserved_size);
-		break;
-	default:
-		MISSING_CASE(GRAPHICS_VER(i915));
-		fallthrough;
-	case 11:
-	case 12:
-		icl_get_stolen_reserved(i915, uncore,
-					&reserved_base,
-					&reserved_size);
-		break;
+	} else if (GRAPHICS_VER(i915) >= 7) {
+		if (IS_VALLEYVIEW(i915))
+			vlv_get_stolen_reserved(i915, uncore,
+						&reserved_base, &reserved_size);
+		else
+			gen7_get_stolen_reserved(i915, uncore,
+						 &reserved_base, &reserved_size);
+	} else if (GRAPHICS_VER(i915) >= 6) {
+		gen6_get_stolen_reserved(i915, uncore,
+					 &reserved_base, &reserved_size);
+	} else if (GRAPHICS_VER(i915) >= 5 || IS_G4X(i915)) {
+		g4x_get_stolen_reserved(i915, uncore,
+					&reserved_base, &reserved_size);
 	}
 
 	/*
@@ -827,10 +810,13 @@ i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type,
 	if (WARN_ON_ONCE(instance))
 		return ERR_PTR(-ENODEV);
 
+	if (!i915_pci_resource_valid(pdev, GEN12_LMEM_BAR))
+		return ERR_PTR(-ENXIO);
+
 	/* Use DSM base address instead for stolen memory */
 	dsm_base = intel_uncore_read64(uncore, GEN12_DSMBASE);
 	if (IS_DG1(uncore->i915)) {
-		lmem_size = pci_resource_len(pdev, 2);
+		lmem_size = pci_resource_len(pdev, GEN12_LMEM_BAR);
 		if (WARN_ON(lmem_size < dsm_base))
 			return ERR_PTR(-ENODEV);
 	} else {
@@ -842,11 +828,11 @@ i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type,
 	}
 
 	dsm_size = lmem_size - dsm_base;
-	if (pci_resource_len(pdev, 2) < lmem_size) {
+	if (pci_resource_len(pdev, GEN12_LMEM_BAR) < lmem_size) {
 		io_start = 0;
 		io_size = 0;
 	} else {
-		io_start = pci_resource_start(pdev, 2) + dsm_base;
+		io_start = pci_resource_start(pdev, GEN12_LMEM_BAR) + dsm_base;
 		io_size = dsm_size;
 	}
 
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
index 85518b28cd72..fd42b89b7162 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c
@@ -278,7 +278,7 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
 	 */
 	if (i915_gem_object_has_pages(obj) &&
 	    obj->mm.madv == I915_MADV_WILLNEED &&
-	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
+	    i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {
 		if (tiling == I915_TILING_NONE) {
 			GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
 			i915_gem_object_clear_tiling_quirk(obj);
@@ -458,7 +458,7 @@ i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
 	}
 
 	/* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
-	if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
+	if (dev_priv->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
 		args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;
 	else
 		args->phys_swizzle_mode = args->swizzle_mode;
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
index 6f3ab7ade41a..e3fc38dd5db0 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm.c
@@ -361,7 +361,6 @@ static bool i915_ttm_eviction_valuable(struct ttm_buffer_object *bo,
 				       const struct ttm_place *place)
 {
 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
-	struct ttm_resource *res = bo->resource;
 
 	if (!obj)
 		return false;
@@ -378,45 +377,7 @@ static bool i915_ttm_eviction_valuable(struct ttm_buffer_object *bo,
 	if (!i915_gem_object_evictable(obj))
 		return false;
 
-	switch (res->mem_type) {
-	case I915_PL_LMEM0: {
-		struct ttm_resource_manager *man =
-			ttm_manager_type(bo->bdev, res->mem_type);
-		struct i915_ttm_buddy_resource *bman_res =
-			to_ttm_buddy_resource(res);
-		struct drm_buddy *mm = bman_res->mm;
-		struct drm_buddy_block *block;
-
-		if (!place->fpfn && !place->lpfn)
-			return true;
-
-		GEM_BUG_ON(!place->lpfn);
-
-		/*
-		 * If we just want something mappable then we can quickly check
-		 * if the current victim resource is using any of the CPU
-		 * visible portion.
-		 */
-		if (!place->fpfn &&
-		    place->lpfn == i915_ttm_buddy_man_visible_size(man))
-			return bman_res->used_visible_size > 0;
-
-		/* Real range allocation */
-		list_for_each_entry(block, &bman_res->blocks, link) {
-			unsigned long fpfn =
-				drm_buddy_block_offset(block) >> PAGE_SHIFT;
-			unsigned long lpfn = fpfn +
-				(drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
-
-			if (place->fpfn < lpfn && place->lpfn > fpfn)
-				return true;
-		}
-		return false;
-	} default:
-		break;
-	}
-
-	return true;
+	return ttm_bo_eviction_valuable(bo, place);
 }
 
 static void i915_ttm_evict_flags(struct ttm_buffer_object *bo,
@@ -548,9 +509,18 @@ static int i915_ttm_shrink(struct drm_i915_gem_object *obj, unsigned int flags)
 static void i915_ttm_delete_mem_notify(struct ttm_buffer_object *bo)
 {
 	struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
+	intel_wakeref_t wakeref = 0;
+
+	if (bo->resource && likely(obj)) {
+		/* ttm_bo_release() already has dma_resv_lock */
+		if (i915_ttm_cpu_maps_iomem(bo->resource))
+			wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm);
 
-	if (likely(obj)) {
 		__i915_gem_object_pages_fini(obj);
+
+		if (wakeref)
+			intel_runtime_pm_put(&to_i915(obj->base.dev)->runtime_pm, wakeref);
+
 		i915_ttm_free_cached_io_rsgt(obj);
 	}
 }
@@ -1020,6 +990,7 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf)
 	struct ttm_buffer_object *bo = area->vm_private_data;
 	struct drm_device *dev = bo->base.dev;
 	struct drm_i915_gem_object *obj;
+	intel_wakeref_t wakeref = 0;
 	vm_fault_t ret;
 	int idx;
 
@@ -1041,6 +1012,9 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf)
 		return VM_FAULT_SIGBUS;
 	}
 
+	if (i915_ttm_cpu_maps_iomem(bo->resource))
+		wakeref = intel_runtime_pm_get(&to_i915(obj->base.dev)->runtime_pm);
+
 	if (!i915_ttm_resource_mappable(bo->resource)) {
 		int err = -ENODEV;
 		int i;
@@ -1062,7 +1036,8 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf)
 		if (err) {
 			drm_dbg(dev, "Unable to make resource CPU accessible\n");
 			dma_resv_unlock(bo->base.resv);
-			return VM_FAULT_SIGBUS;
+			ret = VM_FAULT_SIGBUS;
+			goto out_rpm;
 		}
 	}
 
@@ -1073,12 +1048,30 @@ static vm_fault_t vm_fault_ttm(struct vm_fault *vmf)
 	} else {
 		ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
 	}
+
 	if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
-		return ret;
+		goto out_rpm;
+
+	/* ttm_bo_vm_reserve() already has dma_resv_lock */
+	if (ret == VM_FAULT_NOPAGE && wakeref && !obj->userfault_count) {
+		obj->userfault_count = 1;
+		mutex_lock(&to_gt(to_i915(obj->base.dev))->lmem_userfault_lock);
+		list_add(&obj->userfault_link, &to_gt(to_i915(obj->base.dev))->lmem_userfault_list);
+		mutex_unlock(&to_gt(to_i915(obj->base.dev))->lmem_userfault_lock);
+	}
+
+	if (wakeref & CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
+		intel_wakeref_auto(&to_gt(to_i915(obj->base.dev))->userfault_wakeref,
+				   msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
 
 	i915_ttm_adjust_lru(obj);
 
 	dma_resv_unlock(bo->base.resv);
+
+out_rpm:
+	if (wakeref)
+		intel_runtime_pm_put(&to_i915(obj->base.dev)->runtime_pm, wakeref);
+
 	return ret;
 }
 
@@ -1242,9 +1235,8 @@ int __i915_gem_ttm_object_init(struct intel_memory_region *mem,
 	 * Similarly, in delayed_destroy, we can't call ttm_bo_put()
 	 * until successful initialization.
 	 */
-	ret = ttm_bo_init_reserved(&i915->bdev, i915_gem_to_ttm(obj), size,
-				   bo_type, &i915_sys_placement,
-				   page_size >> PAGE_SHIFT,
+	ret = ttm_bo_init_reserved(&i915->bdev, i915_gem_to_ttm(obj), bo_type,
+				   &i915_sys_placement, page_size >> PAGE_SHIFT,
 				   &ctx, NULL, NULL, i915_ttm_bo_destroy);
 	if (ret)
 		return i915_ttm_err_to_gem(ret);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c
index 9aad84059d56..07e49f22f2de 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c
@@ -79,7 +79,12 @@ static int i915_ttm_backup(struct i915_gem_apply_to_region *apply,
 		goto out_no_populate;
 
 	err = i915_gem_obj_copy_ttm(backup, obj, pm_apply->allow_gpu, false);
-	GEM_WARN_ON(err);
+	if (err) {
+		drm_err(&i915->drm,
+			"Unable to copy from device to system memory, err:%pe\n",
+			ERR_PTR(err));
+		goto out_no_populate;
+	}
 	ttm_bo_wait_ctx(backup_bo, &ctx);
 
 	obj->ttm.backup = backup;
diff --git a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
index 72ce2c9f42fd..c570cf780079 100644
--- a/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/gem/selftests/huge_pages.c
@@ -358,7 +358,7 @@ fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single)
 static int igt_check_page_sizes(struct i915_vma *vma)
 {
 	struct drm_i915_private *i915 = vma->vm->i915;
-	unsigned int supported = INTEL_INFO(i915)->page_sizes;
+	unsigned int supported = RUNTIME_INFO(i915)->page_sizes;
 	struct drm_i915_gem_object *obj = vma->obj;
 	int err;
 
@@ -419,7 +419,7 @@ static int igt_mock_exhaust_device_supported_pages(void *arg)
 {
 	struct i915_ppgtt *ppgtt = arg;
 	struct drm_i915_private *i915 = ppgtt->vm.i915;
-	unsigned int saved_mask = INTEL_INFO(i915)->page_sizes;
+	unsigned int saved_mask = RUNTIME_INFO(i915)->page_sizes;
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
 	int i, j, single;
@@ -438,7 +438,7 @@ static int igt_mock_exhaust_device_supported_pages(void *arg)
 				combination |= page_sizes[j];
 		}
 
-		mkwrite_device_info(i915)->page_sizes = combination;
+		RUNTIME_INFO(i915)->page_sizes = combination;
 
 		for (single = 0; single <= 1; ++single) {
 			obj = fake_huge_pages_object(i915, combination, !!single);
@@ -485,7 +485,7 @@ static int igt_mock_exhaust_device_supported_pages(void *arg)
 out_put:
 	i915_gem_object_put(obj);
 out_device:
-	mkwrite_device_info(i915)->page_sizes = saved_mask;
+	RUNTIME_INFO(i915)->page_sizes = saved_mask;
 
 	return err;
 }
@@ -495,7 +495,7 @@ static int igt_mock_memory_region_huge_pages(void *arg)
 	const unsigned int flags[] = { 0, I915_BO_ALLOC_CONTIGUOUS };
 	struct i915_ppgtt *ppgtt = arg;
 	struct drm_i915_private *i915 = ppgtt->vm.i915;
-	unsigned long supported = INTEL_INFO(i915)->page_sizes;
+	unsigned long supported = RUNTIME_INFO(i915)->page_sizes;
 	struct intel_memory_region *mem;
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
@@ -573,7 +573,7 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg)
 {
 	struct i915_ppgtt *ppgtt = arg;
 	struct drm_i915_private *i915 = ppgtt->vm.i915;
-	unsigned long supported = INTEL_INFO(i915)->page_sizes;
+	unsigned long supported = RUNTIME_INFO(i915)->page_sizes;
 	struct drm_i915_gem_object *obj;
 	int bit;
 	int err;
@@ -1390,7 +1390,7 @@ out_put:
 static int igt_ppgtt_sanity_check(void *arg)
 {
 	struct drm_i915_private *i915 = arg;
-	unsigned int supported = INTEL_INFO(i915)->page_sizes;
+	unsigned int supported = RUNTIME_INFO(i915)->page_sizes;
 	struct {
 		igt_create_fn fn;
 		unsigned int flags;
@@ -1764,8 +1764,8 @@ int i915_gem_huge_page_mock_selftests(void)
 		return -ENOMEM;
 
 	/* Pretend to be a device which supports the 48b PPGTT */
-	mkwrite_device_info(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL;
-	mkwrite_device_info(dev_priv)->ppgtt_size = 48;
+	RUNTIME_INFO(dev_priv)->ppgtt_type = INTEL_PPGTT_FULL;
+	RUNTIME_INFO(dev_priv)->ppgtt_size = 48;
 
 	ppgtt = i915_ppgtt_create(to_gt(dev_priv), 0);
 	if (IS_ERR(ppgtt)) {
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
index 3cfc621ef363..9a6a6b5b722b 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c
@@ -711,7 +711,7 @@ static bool bad_swizzling(struct drm_i915_private *i915)
 {
 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 
-	if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
+	if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
 		return true;
 
 	if (has_bit17_swizzle(ggtt->bit_6_swizzle_x) ||
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c
index 13b088cc787e..a666d7e610f5 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c
@@ -434,5 +434,5 @@ int i915_gem_coherency_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_gem_coherency),
 	};
 
-	return i915_subtests(tests, i915);
+	return i915_live_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c
index 62c61af77a42..51ed824b020c 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c
@@ -476,5 +476,5 @@ int i915_gem_dmabuf_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_dmabuf_import_same_driver_lmem_smem),
 	};
 
-	return i915_subtests(tests, i915);
+	return i915_live_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
index 3ced9948a331..b73c91aa5450 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c
@@ -93,7 +93,7 @@ static int check_partial_mapping(struct drm_i915_gem_object *obj,
 {
 	const unsigned long npages = obj->base.size / PAGE_SIZE;
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
-	struct i915_ggtt_view view;
+	struct i915_gtt_view view;
 	struct i915_vma *vma;
 	unsigned long page;
 	u32 __iomem *io;
@@ -210,7 +210,7 @@ static int check_partial_mappings(struct drm_i915_gem_object *obj,
 	}
 
 	for_each_prime_number_from(page, 1, npages) {
-		struct i915_ggtt_view view =
+		struct i915_gtt_view view =
 			compute_partial_view(obj, page, MIN_CHUNK_PAGES);
 		u32 __iomem *io;
 		struct page *p;
@@ -367,7 +367,7 @@ static int igt_partial_tiling(void *arg)
 		unsigned int pitch;
 		struct tile tile;
 
-		if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
+		if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
 			/*
 			 * The swizzling pattern is actually unknown as it
 			 * varies based on physical address of each page.
@@ -464,7 +464,7 @@ static int igt_smoke_tiling(void *arg)
 	 * Remember to look at the st_seed if we see a flip-flop in BAT!
 	 */
 
-	if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
+	if (i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
 		return 0;
 
 	obj = huge_gem_object(i915,
@@ -1844,5 +1844,5 @@ int i915_gem_mman_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_mmap_gpu),
 	};
 
-	return i915_subtests(tests, i915);
+	return i915_live_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c
index fe0a890775e2..bdf5bb40ccf1 100644
--- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c
@@ -95,5 +95,5 @@ int i915_gem_object_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_gem_huge),
 	};
 
-	return i915_subtests(tests, i915);
+	return i915_live_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c
index 1bb766c79dcb..5aaacc53fa4c 100644
--- a/drivers/gpu/drm/i915/gt/gen6_ppgtt.c
+++ b/drivers/gpu/drm/i915/gt/gen6_ppgtt.c
@@ -247,6 +247,7 @@ err_scratch1:
 	i915_gem_object_put(vm->scratch[1]);
 err_scratch0:
 	i915_gem_object_put(vm->scratch[0]);
+	vm->scratch[0] = NULL;
 	return ret;
 }
 
@@ -268,9 +269,10 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 	gen6_ppgtt_free_pd(ppgtt);
 	free_scratch(vm);
 
-	mutex_destroy(&ppgtt->flush);
+	if (ppgtt->base.pd)
+		free_pd(&ppgtt->base.vm, ppgtt->base.pd);
 
-	free_pd(&ppgtt->base.vm, ppgtt->base.pd);
+	mutex_destroy(&ppgtt->flush);
 }
 
 static void pd_vma_bind(struct i915_address_space *vm,
@@ -449,19 +451,17 @@ struct i915_ppgtt *gen6_ppgtt_create(struct intel_gt *gt)
 
 	err = gen6_ppgtt_init_scratch(ppgtt);
 	if (err)
-		goto err_free;
+		goto err_put;
 
 	ppgtt->base.pd = gen6_alloc_top_pd(ppgtt);
 	if (IS_ERR(ppgtt->base.pd)) {
 		err = PTR_ERR(ppgtt->base.pd);
-		goto err_scratch;
+		goto err_put;
 	}
 
 	return &ppgtt->base;
 
-err_scratch:
-	free_scratch(&ppgtt->base.vm);
-err_free:
-	kfree(ppgtt);
+err_put:
+	i915_vm_put(&ppgtt->base.vm);
 	return ERR_PTR(err);
 }
diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
index 98645797962f..e49fa6fa6aee 100644
--- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.c
@@ -165,10 +165,12 @@ static u32 preparser_disable(bool state)
 	return MI_ARB_CHECK | 1 << 8 | state;
 }
 
-u32 *gen12_emit_aux_table_inv(u32 *cs, const i915_reg_t inv_reg)
+u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv_reg)
 {
+	u32 gsi_offset = gt->uncore->gsi_offset;
+
 	*cs++ = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN;
-	*cs++ = i915_mmio_reg_offset(inv_reg);
+	*cs++ = i915_mmio_reg_offset(inv_reg) + gsi_offset;
 	*cs++ = AUX_INV;
 	*cs++ = MI_NOOP;
 
@@ -254,7 +256,8 @@ int gen12_emit_flush_rcs(struct i915_request *rq, u32 mode)
 
 		if (!HAS_FLAT_CCS(rq->engine->i915)) {
 			/* hsdes: 1809175790 */
-			cs = gen12_emit_aux_table_inv(cs, GEN12_GFX_CCS_AUX_NV);
+			cs = gen12_emit_aux_table_inv(rq->engine->gt,
+						      cs, GEN12_GFX_CCS_AUX_NV);
 		}
 
 		*cs++ = preparser_disable(false);
@@ -313,9 +316,11 @@ int gen12_emit_flush_xcs(struct i915_request *rq, u32 mode)
 
 	if (aux_inv) { /* hsdes: 1809175790 */
 		if (rq->engine->class == VIDEO_DECODE_CLASS)
-			cs = gen12_emit_aux_table_inv(cs, GEN12_VD0_AUX_NV);
+			cs = gen12_emit_aux_table_inv(rq->engine->gt,
+						      cs, GEN12_VD0_AUX_NV);
 		else
-			cs = gen12_emit_aux_table_inv(cs, GEN12_VE0_AUX_NV);
+			cs = gen12_emit_aux_table_inv(rq->engine->gt,
+						      cs, GEN12_VE0_AUX_NV);
 	}
 
 	if (mode & EMIT_INVALIDATE)
diff --git a/drivers/gpu/drm/i915/gt/gen8_engine_cs.h b/drivers/gpu/drm/i915/gt/gen8_engine_cs.h
index 32e3d2b831bb..e4d24c811dd6 100644
--- a/drivers/gpu/drm/i915/gt/gen8_engine_cs.h
+++ b/drivers/gpu/drm/i915/gt/gen8_engine_cs.h
@@ -13,6 +13,7 @@
 #include "intel_gt_regs.h"
 #include "intel_gpu_commands.h"
 
+struct intel_gt;
 struct i915_request;
 
 int gen8_emit_flush_rcs(struct i915_request *rq, u32 mode);
@@ -45,7 +46,7 @@ u32 *gen8_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs);
 u32 *gen11_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs);
 u32 *gen12_emit_fini_breadcrumb_rcs(struct i915_request *rq, u32 *cs);
 
-u32 *gen12_emit_aux_table_inv(u32 *cs, const i915_reg_t inv_reg);
+u32 *gen12_emit_aux_table_inv(struct intel_gt *gt, u32 *cs, const i915_reg_t inv_reg);
 
 static inline u32 *
 __gen8_emit_pipe_control(u32 *batch, u32 flags0, u32 flags1, u32 offset)
diff --git a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c
index c7bd5d71b03e..2128b7a72a25 100644
--- a/drivers/gpu/drm/i915/gt/gen8_ppgtt.c
+++ b/drivers/gpu/drm/i915/gt/gen8_ppgtt.c
@@ -196,7 +196,10 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 	if (intel_vgpu_active(vm->i915))
 		gen8_ppgtt_notify_vgt(ppgtt, false);
 
-	__gen8_ppgtt_cleanup(vm, ppgtt->pd, gen8_pd_top_count(vm), vm->top);
+	if (ppgtt->pd)
+		__gen8_ppgtt_cleanup(vm, ppgtt->pd,
+				     gen8_pd_top_count(vm), vm->top);
+
 	free_scratch(vm);
 }
 
@@ -803,8 +806,10 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 		struct drm_i915_gem_object *obj;
 
 		obj = vm->alloc_pt_dma(vm, I915_GTT_PAGE_SIZE_4K);
-		if (IS_ERR(obj))
+		if (IS_ERR(obj)) {
+			ret = PTR_ERR(obj);
 			goto free_scratch;
+		}
 
 		ret = map_pt_dma(vm, obj);
 		if (ret) {
@@ -823,7 +828,8 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 free_scratch:
 	while (i--)
 		i915_gem_object_put(vm->scratch[i]);
-	return -ENOMEM;
+	vm->scratch[0] = NULL;
+	return ret;
 }
 
 static int gen8_preallocate_top_level_pdp(struct i915_ppgtt *ppgtt)
@@ -901,6 +907,7 @@ err_pd:
 struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt,
 				     unsigned long lmem_pt_obj_flags)
 {
+	struct i915_page_directory *pd;
 	struct i915_ppgtt *ppgtt;
 	int err;
 
@@ -946,21 +953,7 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt,
 		ppgtt->vm.alloc_scratch_dma = alloc_pt_dma;
 	}
 
-	err = gen8_init_scratch(&ppgtt->vm);
-	if (err)
-		goto err_free;
-
-	ppgtt->pd = gen8_alloc_top_pd(&ppgtt->vm);
-	if (IS_ERR(ppgtt->pd)) {
-		err = PTR_ERR(ppgtt->pd);
-		goto err_free_scratch;
-	}
-
-	if (!i915_vm_is_4lvl(&ppgtt->vm)) {
-		err = gen8_preallocate_top_level_pdp(ppgtt);
-		if (err)
-			goto err_free_pd;
-	}
+	ppgtt->vm.pte_encode = gen8_pte_encode;
 
 	ppgtt->vm.bind_async_flags = I915_VMA_LOCAL_BIND;
 	ppgtt->vm.insert_entries = gen8_ppgtt_insert;
@@ -971,22 +964,31 @@ struct i915_ppgtt *gen8_ppgtt_create(struct intel_gt *gt,
 	ppgtt->vm.allocate_va_range = gen8_ppgtt_alloc;
 	ppgtt->vm.clear_range = gen8_ppgtt_clear;
 	ppgtt->vm.foreach = gen8_ppgtt_foreach;
+	ppgtt->vm.cleanup = gen8_ppgtt_cleanup;
 
-	ppgtt->vm.pte_encode = gen8_pte_encode;
+	err = gen8_init_scratch(&ppgtt->vm);
+	if (err)
+		goto err_put;
+
+	pd = gen8_alloc_top_pd(&ppgtt->vm);
+	if (IS_ERR(pd)) {
+		err = PTR_ERR(pd);
+		goto err_put;
+	}
+	ppgtt->pd = pd;
+
+	if (!i915_vm_is_4lvl(&ppgtt->vm)) {
+		err = gen8_preallocate_top_level_pdp(ppgtt);
+		if (err)
+			goto err_put;
+	}
 
 	if (intel_vgpu_active(gt->i915))
 		gen8_ppgtt_notify_vgt(ppgtt, true);
 
-	ppgtt->vm.cleanup = gen8_ppgtt_cleanup;
-
 	return ppgtt;
 
-err_free_pd:
-	__gen8_ppgtt_cleanup(&ppgtt->vm, ppgtt->pd,
-			     gen8_pd_top_count(&ppgtt->vm), ppgtt->vm.top);
-err_free_scratch:
-	free_scratch(&ppgtt->vm);
-err_free:
-	kfree(ppgtt);
+err_put:
+	i915_vm_put(&ppgtt->vm);
 	return ERR_PTR(err);
 }
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
index 37fa813af766..1f7188129cd1 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c
@@ -654,16 +654,83 @@ bool gen11_vdbox_has_sfc(struct intel_gt *gt,
 	 */
 	if ((gt->info.sfc_mask & BIT(physical_vdbox / 2)) == 0)
 		return false;
-	else if (GRAPHICS_VER(i915) == 12)
+	else if (MEDIA_VER(i915) >= 12)
 		return (physical_vdbox % 2 == 0) ||
 			!(BIT(physical_vdbox - 1) & vdbox_mask);
-	else if (GRAPHICS_VER(i915) == 11)
+	else if (MEDIA_VER(i915) == 11)
 		return logical_vdbox % 2 == 0;
 
-	MISSING_CASE(GRAPHICS_VER(i915));
 	return false;
 }
 
+static void engine_mask_apply_media_fuses(struct intel_gt *gt)
+{
+	struct drm_i915_private *i915 = gt->i915;
+	unsigned int logical_vdbox = 0;
+	unsigned int i;
+	u32 media_fuse, fuse1;
+	u16 vdbox_mask;
+	u16 vebox_mask;
+
+	if (MEDIA_VER(gt->i915) < 11)
+		return;
+
+	/*
+	 * On newer platforms the fusing register is called 'enable' and has
+	 * enable semantics, while on older platforms it is called 'disable'
+	 * and bits have disable semantices.
+	 */
+	media_fuse = intel_uncore_read(gt->uncore, GEN11_GT_VEBOX_VDBOX_DISABLE);
+	if (MEDIA_VER_FULL(i915) < IP_VER(12, 50))
+		media_fuse = ~media_fuse;
+
+	vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK;
+	vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >>
+		      GEN11_GT_VEBOX_DISABLE_SHIFT;
+
+	if (MEDIA_VER_FULL(i915) >= IP_VER(12, 50)) {
+		fuse1 = intel_uncore_read(gt->uncore, HSW_PAVP_FUSE1);
+		gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1);
+	} else {
+		gt->info.sfc_mask = ~0;
+	}
+
+	for (i = 0; i < I915_MAX_VCS; i++) {
+		if (!HAS_ENGINE(gt, _VCS(i))) {
+			vdbox_mask &= ~BIT(i);
+			continue;
+		}
+
+		if (!(BIT(i) & vdbox_mask)) {
+			gt->info.engine_mask &= ~BIT(_VCS(i));
+			drm_dbg(&i915->drm, "vcs%u fused off\n", i);
+			continue;
+		}
+
+		if (gen11_vdbox_has_sfc(gt, i, logical_vdbox, vdbox_mask))
+			gt->info.vdbox_sfc_access |= BIT(i);
+		logical_vdbox++;
+	}
+	drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n",
+		vdbox_mask, VDBOX_MASK(gt));
+	GEM_BUG_ON(vdbox_mask != VDBOX_MASK(gt));
+
+	for (i = 0; i < I915_MAX_VECS; i++) {
+		if (!HAS_ENGINE(gt, _VECS(i))) {
+			vebox_mask &= ~BIT(i);
+			continue;
+		}
+
+		if (!(BIT(i) & vebox_mask)) {
+			gt->info.engine_mask &= ~BIT(_VECS(i));
+			drm_dbg(&i915->drm, "vecs%u fused off\n", i);
+		}
+	}
+	drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n",
+		vebox_mask, VEBOX_MASK(gt));
+	GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt));
+}
+
 static void engine_mask_apply_compute_fuses(struct intel_gt *gt)
 {
 	struct drm_i915_private *i915 = gt->i915;
@@ -672,7 +739,10 @@ static void engine_mask_apply_compute_fuses(struct intel_gt *gt)
 	unsigned long ccs_mask;
 	unsigned int i;
 
-	if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
+	if (GRAPHICS_VER(i915) < 11)
+		return;
+
+	if (hweight32(CCS_MASK(gt)) <= 1)
 		return;
 
 	ccs_mask = intel_slicemask_from_xehp_dssmask(info->sseu.compute_subslice_mask,
@@ -694,6 +764,10 @@ static void engine_mask_apply_copy_fuses(struct intel_gt *gt)
 	unsigned long meml3_mask;
 	unsigned long quad;
 
+	if (!(GRAPHICS_VER_FULL(i915) >= IP_VER(12, 60) &&
+	      GRAPHICS_VER_FULL(i915) < IP_VER(12, 70)))
+		return;
+
 	meml3_mask = intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3);
 	meml3_mask = REG_FIELD_GET(GEN12_MEML3_EN_MASK, meml3_mask);
 
@@ -727,75 +801,11 @@ static void engine_mask_apply_copy_fuses(struct intel_gt *gt)
  */
 static intel_engine_mask_t init_engine_mask(struct intel_gt *gt)
 {
-	struct drm_i915_private *i915 = gt->i915;
 	struct intel_gt_info *info = &gt->info;
-	struct intel_uncore *uncore = gt->uncore;
-	unsigned int logical_vdbox = 0;
-	unsigned int i;
-	u32 media_fuse, fuse1;
-	u16 vdbox_mask;
-	u16 vebox_mask;
-
-	info->engine_mask = INTEL_INFO(i915)->platform_engine_mask;
-
-	if (GRAPHICS_VER(i915) < 11)
-		return info->engine_mask;
 
-	/*
-	 * On newer platforms the fusing register is called 'enable' and has
-	 * enable semantics, while on older platforms it is called 'disable'
-	 * and bits have disable semantices.
-	 */
-	media_fuse = intel_uncore_read(uncore, GEN11_GT_VEBOX_VDBOX_DISABLE);
-	if (GRAPHICS_VER_FULL(i915) < IP_VER(12, 50))
-		media_fuse = ~media_fuse;
-
-	vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK;
-	vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >>
-		      GEN11_GT_VEBOX_DISABLE_SHIFT;
-
-	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50)) {
-		fuse1 = intel_uncore_read(uncore, HSW_PAVP_FUSE1);
-		gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1);
-	} else {
-		gt->info.sfc_mask = ~0;
-	}
-
-	for (i = 0; i < I915_MAX_VCS; i++) {
-		if (!HAS_ENGINE(gt, _VCS(i))) {
-			vdbox_mask &= ~BIT(i);
-			continue;
-		}
-
-		if (!(BIT(i) & vdbox_mask)) {
-			info->engine_mask &= ~BIT(_VCS(i));
-			drm_dbg(&i915->drm, "vcs%u fused off\n", i);
-			continue;
-		}
-
-		if (gen11_vdbox_has_sfc(gt, i, logical_vdbox, vdbox_mask))
-			gt->info.vdbox_sfc_access |= BIT(i);
-		logical_vdbox++;
-	}
-	drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n",
-		vdbox_mask, VDBOX_MASK(gt));
-	GEM_BUG_ON(vdbox_mask != VDBOX_MASK(gt));
-
-	for (i = 0; i < I915_MAX_VECS; i++) {
-		if (!HAS_ENGINE(gt, _VECS(i))) {
-			vebox_mask &= ~BIT(i);
-			continue;
-		}
-
-		if (!(BIT(i) & vebox_mask)) {
-			info->engine_mask &= ~BIT(_VECS(i));
-			drm_dbg(&i915->drm, "vecs%u fused off\n", i);
-		}
-	}
-	drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n",
-		vebox_mask, VEBOX_MASK(gt));
-	GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt));
+	GEM_BUG_ON(!info->engine_mask);
 
+	engine_mask_apply_media_fuses(gt);
 	engine_mask_apply_compute_fuses(gt);
 	engine_mask_apply_copy_fuses(gt);
 
@@ -1688,9 +1698,9 @@ bool intel_engine_irq_enable(struct intel_engine_cs *engine)
 		return false;
 
 	/* Caller disables interrupts */
-	spin_lock(&engine->gt->irq_lock);
+	spin_lock(engine->gt->irq_lock);
 	engine->irq_enable(engine);
-	spin_unlock(&engine->gt->irq_lock);
+	spin_unlock(engine->gt->irq_lock);
 
 	return true;
 }
@@ -1701,9 +1711,9 @@ void intel_engine_irq_disable(struct intel_engine_cs *engine)
 		return;
 
 	/* Caller disables interrupts */
-	spin_lock(&engine->gt->irq_lock);
+	spin_lock(engine->gt->irq_lock);
 	engine->irq_disable(engine);
-	spin_unlock(&engine->gt->irq_lock);
+	spin_unlock(engine->gt->irq_lock);
 }
 
 void intel_engines_reset_default_submission(struct intel_gt *gt)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_regs.h b/drivers/gpu/drm/i915/gt/intel_engine_regs.h
index 889f0df3940b..fe1a0d5fd4b1 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_regs.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_regs.h
@@ -110,6 +110,7 @@
 #define RING_SBBSTATE(base)			_MMIO((base) + 0x118) /* hsw+ */
 #define RING_SBBADDR_UDW(base)			_MMIO((base) + 0x11c) /* gen8+ */
 #define RING_BBADDR(base)			_MMIO((base) + 0x140)
+#define RING_BB_OFFSET(base)			_MMIO((base) + 0x158)
 #define RING_BBADDR_UDW(base)			_MMIO((base) + 0x168) /* gen8+ */
 #define CCID(base)				_MMIO((base) + 0x180)
 #define   CCID_EN				BIT(0)
diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h
index 633a7e5dba3b..6b5d4ea22b67 100644
--- a/drivers/gpu/drm/i915/gt/intel_engine_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h
@@ -166,6 +166,21 @@ struct intel_engine_execlists {
 	struct timer_list preempt;
 
 	/**
+	 * @preempt_target: active request at the time of the preemption request
+	 *
+	 * We force a preemption to occur if the pending contexts have not
+	 * been promoted to active upon receipt of the CS ack event within
+	 * the timeout. This timeout maybe chosen based on the target,
+	 * using a very short timeout if the context is no longer schedulable.
+	 * That short timeout may not be applicable to other contexts, so
+	 * if a context switch should happen within before the preemption
+	 * timeout, we may shoot early at an innocent context. To prevent this,
+	 * we record which context was active at the time of the preemption
+	 * request and only reset that context upon the timeout.
+	 */
+	const struct i915_request *preempt_target;
+
+	/**
 	 * @ccid: identifier for contexts submitted to this engine
 	 */
 	u32 ccid;
diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
index 4b909cb88cdf..c718e6dc40b5 100644
--- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
+++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c
@@ -1241,6 +1241,9 @@ static unsigned long active_preempt_timeout(struct intel_engine_cs *engine,
 	if (!rq)
 		return 0;
 
+	/* Only allow ourselves to force reset the currently active context */
+	engine->execlists.preempt_target = rq;
+
 	/* Force a fast reset for terminated contexts (ignoring sysfs!) */
 	if (unlikely(intel_context_is_banned(rq->context) || bad_request(rq)))
 		return INTEL_CONTEXT_BANNED_PREEMPT_TIMEOUT_MS;
@@ -2427,8 +2430,24 @@ static void execlists_submission_tasklet(struct tasklet_struct *t)
 	GEM_BUG_ON(inactive - post > ARRAY_SIZE(post));
 
 	if (unlikely(preempt_timeout(engine))) {
+		const struct i915_request *rq = *engine->execlists.active;
+
+		/*
+		 * If after the preempt-timeout expired, we are still on the
+		 * same active request/context as before we initiated the
+		 * preemption, reset the engine.
+		 *
+		 * However, if we have processed a CS event to switch contexts,
+		 * but not yet processed the CS event for the pending
+		 * preemption, reset the timer allowing the new context to
+		 * gracefully exit.
+		 */
 		cancel_timer(&engine->execlists.preempt);
-		engine->execlists.error_interrupt |= ERROR_PREEMPT;
+		if (rq == engine->execlists.preempt_target)
+			engine->execlists.error_interrupt |= ERROR_PREEMPT;
+		else
+			set_timer_ms(&engine->execlists.preempt,
+				     active_preempt_timeout(engine, rq));
 	}
 
 	if (unlikely(READ_ONCE(engine->execlists.error_interrupt))) {
diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c
index 15a915bb4088..30cf5c3369d9 100644
--- a/drivers/gpu/drm/i915/gt/intel_ggtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c
@@ -16,7 +16,9 @@
 #include "intel_ggtt_gmch.h"
 #include "intel_gt.h"
 #include "intel_gt_regs.h"
+#include "intel_pci_config.h"
 #include "i915_drv.h"
+#include "i915_pci.h"
 #include "i915_scatterlist.h"
 #include "i915_utils.h"
 #include "i915_vgpu.h"
@@ -869,8 +871,8 @@ static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size)
 	u32 pte_flags;
 	int ret;
 
-	GEM_WARN_ON(pci_resource_len(pdev, 0) != gen6_gttmmadr_size(i915));
-	phys_addr = pci_resource_start(pdev, 0) + gen6_gttadr_offset(i915);
+	GEM_WARN_ON(pci_resource_len(pdev, GTTMMADR_BAR) != gen6_gttmmadr_size(i915));
+	phys_addr = pci_resource_start(pdev, GTTMMADR_BAR) + gen6_gttadr_offset(i915);
 
 	/*
 	 * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range
@@ -930,7 +932,10 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 	u16 snb_gmch_ctl;
 
 	if (!HAS_LMEM(i915)) {
-		ggtt->gmadr = pci_resource(pdev, 2);
+		if (!i915_pci_resource_valid(pdev, GTT_APERTURE_BAR))
+			return -ENXIO;
+
+		ggtt->gmadr = pci_resource(pdev, GTT_APERTURE_BAR);
 		ggtt->mappable_end = resource_size(&ggtt->gmadr);
 	}
 
@@ -1084,7 +1089,10 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt)
 	unsigned int size;
 	u16 snb_gmch_ctl;
 
-	ggtt->gmadr = pci_resource(pdev, 2);
+	if (!i915_pci_resource_valid(pdev, GTT_APERTURE_BAR))
+		return -ENXIO;
+
+	ggtt->gmadr = pci_resource(pdev, GTT_APERTURE_BAR);
 	ggtt->mappable_end = resource_size(&ggtt->gmadr);
 
 	/*
diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c
index 6ebda3d65086..ea775e601686 100644
--- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c
+++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c
@@ -727,7 +727,7 @@ static void detect_bit_6_swizzle(struct i915_ggtt *ggtt)
 		 * bit17 dependent, and so we need to also prevent the pages
 		 * from being moved.
 		 */
-		i915->quirks |= QUIRK_PIN_SWIZZLED_PAGES;
+		i915->gem_quirks |= GEM_QUIRK_PIN_SWIZZLED_PAGES;
 		swizzle_x = I915_BIT_6_SWIZZLE_NONE;
 		swizzle_y = I915_BIT_6_SWIZZLE_NONE;
 	}
@@ -842,7 +842,6 @@ void intel_ggtt_init_fences(struct i915_ggtt *ggtt)
 
 	INIT_LIST_HEAD(&ggtt->fence_list);
 	INIT_LIST_HEAD(&ggtt->userfault_list);
-	intel_wakeref_auto_init(&ggtt->userfault_wakeref, uncore->rpm);
 
 	detect_bit_6_swizzle(ggtt);
 
diff --git a/drivers/gpu/drm/i915/gt/intel_gsc.c b/drivers/gpu/drm/i915/gt/intel_gsc.c
index 0e494028b81d..7af6db3194dd 100644
--- a/drivers/gpu/drm/i915/gt/intel_gsc.c
+++ b/drivers/gpu/drm/i915/gt/intel_gsc.c
@@ -7,6 +7,7 @@
 #include <linux/mei_aux.h>
 #include "i915_drv.h"
 #include "i915_reg.h"
+#include "gem/i915_gem_region.h"
 #include "gt/intel_gsc.h"
 #include "gt/intel_gt.h"
 
@@ -36,10 +37,56 @@ static int gsc_irq_init(int irq)
 	return irq_set_chip_data(irq, NULL);
 }
 
+static int
+gsc_ext_om_alloc(struct intel_gsc *gsc, struct intel_gsc_intf *intf, size_t size)
+{
+	struct intel_gt *gt = gsc_to_gt(gsc);
+	struct drm_i915_gem_object *obj;
+	int err;
+
+	obj = i915_gem_object_create_lmem(gt->i915, size,
+					  I915_BO_ALLOC_CONTIGUOUS |
+					  I915_BO_ALLOC_CPU_CLEAR);
+	if (IS_ERR(obj)) {
+		drm_err(&gt->i915->drm, "Failed to allocate gsc memory\n");
+		return PTR_ERR(obj);
+	}
+
+	err = i915_gem_object_pin_pages_unlocked(obj);
+	if (err) {
+		drm_err(&gt->i915->drm, "Failed to pin pages for gsc memory\n");
+		goto out_put;
+	}
+
+	intf->gem_obj = obj;
+
+	return 0;
+
+out_put:
+	i915_gem_object_put(obj);
+	return err;
+}
+
+static void gsc_ext_om_destroy(struct intel_gsc_intf *intf)
+{
+	struct drm_i915_gem_object *obj = fetch_and_zero(&intf->gem_obj);
+
+	if (!obj)
+		return;
+
+	if (i915_gem_object_has_pinned_pages(obj))
+		i915_gem_object_unpin_pages(obj);
+
+	i915_gem_object_put(obj);
+}
+
 struct gsc_def {
 	const char *name;
 	unsigned long bar;
 	size_t bar_size;
+	bool use_polling;
+	bool slow_firmware;
+	size_t lmem_size;
 };
 
 /* gsc resources and definitions (HECI1 and HECI2) */
@@ -54,11 +101,25 @@ static const struct gsc_def gsc_def_dg1[] = {
 	}
 };
 
+static const struct gsc_def gsc_def_xehpsdv[] = {
+	{
+		/* HECI1 not enabled on the device. */
+	},
+	{
+		.name = "mei-gscfi",
+		.bar = DG1_GSC_HECI2_BASE,
+		.bar_size = GSC_BAR_LENGTH,
+		.use_polling = true,
+		.slow_firmware = true,
+	}
+};
+
 static const struct gsc_def gsc_def_dg2[] = {
 	{
 		.name = "mei-gsc",
 		.bar = DG2_GSC_HECI1_BASE,
 		.bar_size = GSC_BAR_LENGTH,
+		.lmem_size = SZ_4M,
 	},
 	{
 		.name = "mei-gscfi",
@@ -75,26 +136,32 @@ static void gsc_release_dev(struct device *dev)
 	kfree(adev);
 }
 
-static void gsc_destroy_one(struct intel_gsc_intf *intf)
+static void gsc_destroy_one(struct drm_i915_private *i915,
+			    struct intel_gsc *gsc, unsigned int intf_id)
 {
+	struct intel_gsc_intf *intf = &gsc->intf[intf_id];
+
 	if (intf->adev) {
 		auxiliary_device_delete(&intf->adev->aux_dev);
 		auxiliary_device_uninit(&intf->adev->aux_dev);
 		intf->adev = NULL;
 	}
+
 	if (intf->irq >= 0)
 		irq_free_desc(intf->irq);
 	intf->irq = -1;
+
+	gsc_ext_om_destroy(intf);
 }
 
-static void gsc_init_one(struct drm_i915_private *i915,
-			 struct intel_gsc_intf *intf,
+static void gsc_init_one(struct drm_i915_private *i915, struct intel_gsc *gsc,
 			 unsigned int intf_id)
 {
 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 	struct mei_aux_device *adev;
 	struct auxiliary_device *aux_dev;
 	const struct gsc_def *def;
+	struct intel_gsc_intf *intf = &gsc->intf[intf_id];
 	int ret;
 
 	intf->irq = -1;
@@ -105,6 +172,8 @@ static void gsc_init_one(struct drm_i915_private *i915,
 
 	if (IS_DG1(i915)) {
 		def = &gsc_def_dg1[intf_id];
+	} else if (IS_XEHPSDV(i915)) {
+		def = &gsc_def_xehpsdv[intf_id];
 	} else if (IS_DG2(i915)) {
 		def = &gsc_def_dg2[intf_id];
 	} else {
@@ -117,10 +186,14 @@ static void gsc_init_one(struct drm_i915_private *i915,
 		return;
 	}
 
+	/* skip irq initialization */
+	if (def->use_polling)
+		goto add_device;
+
 	intf->irq = irq_alloc_desc(0);
 	if (intf->irq < 0) {
 		drm_err(&i915->drm, "gsc irq error %d\n", intf->irq);
-		return;
+		goto fail;
 	}
 
 	ret = gsc_irq_init(intf->irq);
@@ -129,16 +202,31 @@ static void gsc_init_one(struct drm_i915_private *i915,
 		goto fail;
 	}
 
+add_device:
 	adev = kzalloc(sizeof(*adev), GFP_KERNEL);
 	if (!adev)
 		goto fail;
 
+	if (def->lmem_size) {
+		drm_dbg(&i915->drm, "setting up GSC lmem\n");
+
+		if (gsc_ext_om_alloc(gsc, intf, def->lmem_size)) {
+			drm_err(&i915->drm, "setting up gsc extended operational memory failed\n");
+			kfree(adev);
+			goto fail;
+		}
+
+		adev->ext_op_mem.start = i915_gem_object_get_dma_address(intf->gem_obj, 0);
+		adev->ext_op_mem.end = adev->ext_op_mem.start + def->lmem_size;
+	}
+
 	adev->irq = intf->irq;
 	adev->bar.parent = &pdev->resource[0];
 	adev->bar.start = def->bar + pdev->resource[0].start;
 	adev->bar.end = adev->bar.start + def->bar_size - 1;
 	adev->bar.flags = IORESOURCE_MEM;
 	adev->bar.desc = IORES_DESC_NONE;
+	adev->slow_firmware = def->slow_firmware;
 
 	aux_dev = &adev->aux_dev;
 	aux_dev->name = def->name;
@@ -165,7 +253,7 @@ static void gsc_init_one(struct drm_i915_private *i915,
 
 	return;
 fail:
-	gsc_destroy_one(intf);
+	gsc_destroy_one(i915, gsc, intf->id);
 }
 
 static void gsc_irq_handler(struct intel_gt *gt, unsigned int intf_id)
@@ -182,10 +270,8 @@ static void gsc_irq_handler(struct intel_gt *gt, unsigned int intf_id)
 		return;
 	}
 
-	if (gt->gsc.intf[intf_id].irq < 0) {
-		drm_err_ratelimited(&gt->i915->drm, "GSC irq: irq not set");
+	if (gt->gsc.intf[intf_id].irq < 0)
 		return;
-	}
 
 	ret = generic_handle_irq(gt->gsc.intf[intf_id].irq);
 	if (ret)
@@ -208,7 +294,7 @@ void intel_gsc_init(struct intel_gsc *gsc, struct drm_i915_private *i915)
 		return;
 
 	for (i = 0; i < INTEL_GSC_NUM_INTERFACES; i++)
-		gsc_init_one(i915, &gsc->intf[i], i);
+		gsc_init_one(i915, gsc, i);
 }
 
 void intel_gsc_fini(struct intel_gsc *gsc)
@@ -220,5 +306,5 @@ void intel_gsc_fini(struct intel_gsc *gsc)
 		return;
 
 	for (i = 0; i < INTEL_GSC_NUM_INTERFACES; i++)
-		gsc_destroy_one(&gsc->intf[i]);
+		gsc_destroy_one(gt->i915, gsc, i);
 }
diff --git a/drivers/gpu/drm/i915/gt/intel_gsc.h b/drivers/gpu/drm/i915/gt/intel_gsc.h
index 68582f912b21..fcac1775e9c3 100644
--- a/drivers/gpu/drm/i915/gt/intel_gsc.h
+++ b/drivers/gpu/drm/i915/gt/intel_gsc.h
@@ -20,11 +20,14 @@ struct mei_aux_device;
 
 /**
  * struct intel_gsc - graphics security controller
+ *
+ * @gem_obj: scratch memory GSC operations
  * @intf : gsc interface
  */
 struct intel_gsc {
 	struct intel_gsc_intf {
 		struct mei_aux_device *adev;
+		struct drm_i915_gem_object *gem_obj;
 		int irq;
 		unsigned int id;
 	} intf[INTEL_GSC_NUM_INTERFACES];
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index f435e06125aa..d0b03a928b9a 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -26,18 +26,22 @@
 #include "intel_gt_requests.h"
 #include "intel_migrate.h"
 #include "intel_mocs.h"
+#include "intel_pci_config.h"
 #include "intel_pm.h"
 #include "intel_rc6.h"
 #include "intel_renderstate.h"
 #include "intel_rps.h"
+#include "intel_sa_media.h"
 #include "intel_gt_sysfs.h"
 #include "intel_uncore.h"
 #include "shmem_utils.h"
 
-static void __intel_gt_init_early(struct intel_gt *gt)
+void intel_gt_common_init_early(struct intel_gt *gt)
 {
-	spin_lock_init(&gt->irq_lock);
+	spin_lock_init(gt->irq_lock);
 
+	INIT_LIST_HEAD(&gt->lmem_userfault_list);
+	mutex_init(&gt->lmem_userfault_lock);
 	INIT_LIST_HEAD(&gt->closed_vma);
 	spin_lock_init(&gt->closed_lock);
 
@@ -57,14 +61,19 @@ static void __intel_gt_init_early(struct intel_gt *gt)
 }
 
 /* Preliminary initialization of Tile 0 */
-void intel_root_gt_init_early(struct drm_i915_private *i915)
+int intel_root_gt_init_early(struct drm_i915_private *i915)
 {
 	struct intel_gt *gt = to_gt(i915);
 
 	gt->i915 = i915;
 	gt->uncore = &i915->uncore;
+	gt->irq_lock = drmm_kzalloc(&i915->drm, sizeof(*gt->irq_lock), GFP_KERNEL);
+	if (!gt->irq_lock)
+		return -ENOMEM;
 
-	__intel_gt_init_early(gt);
+	intel_gt_common_init_early(gt);
+
+	return 0;
 }
 
 static int intel_gt_probe_lmem(struct intel_gt *gt)
@@ -780,26 +789,25 @@ static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr)
 	int ret;
 
 	if (!gt_is_root(gt)) {
-		struct intel_uncore_mmio_debug *mmio_debug;
 		struct intel_uncore *uncore;
+		spinlock_t *irq_lock;
 
-		uncore = kzalloc(sizeof(*uncore), GFP_KERNEL);
+		uncore = drmm_kzalloc(&gt->i915->drm, sizeof(*uncore), GFP_KERNEL);
 		if (!uncore)
 			return -ENOMEM;
 
-		mmio_debug = kzalloc(sizeof(*mmio_debug), GFP_KERNEL);
-		if (!mmio_debug) {
-			kfree(uncore);
+		irq_lock = drmm_kzalloc(&gt->i915->drm, sizeof(*irq_lock), GFP_KERNEL);
+		if (!irq_lock)
 			return -ENOMEM;
-		}
 
 		gt->uncore = uncore;
-		gt->uncore->debug = mmio_debug;
+		gt->irq_lock = irq_lock;
 
-		__intel_gt_init_early(gt);
+		intel_gt_common_init_early(gt);
 	}
 
 	intel_uncore_init_early(gt->uncore, gt);
+	intel_wakeref_auto_init(&gt->userfault_wakeref, gt->uncore->rpm);
 
 	ret = intel_uncore_setup_mmio(gt->uncore, phys_addr);
 	if (ret)
@@ -810,27 +818,17 @@ static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr)
 	return 0;
 }
 
-static void
-intel_gt_tile_cleanup(struct intel_gt *gt)
-{
-	intel_uncore_cleanup_mmio(gt->uncore);
-
-	if (!gt_is_root(gt)) {
-		kfree(gt->uncore->debug);
-		kfree(gt->uncore);
-		kfree(gt);
-	}
-}
-
 int intel_gt_probe_all(struct drm_i915_private *i915)
 {
 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
 	struct intel_gt *gt = &i915->gt0;
+	const struct intel_gt_definition *gtdef;
 	phys_addr_t phys_addr;
 	unsigned int mmio_bar;
+	unsigned int i;
 	int ret;
 
-	mmio_bar = GRAPHICS_VER(i915) == 2 ? 1 : 0;
+	mmio_bar = GRAPHICS_VER(i915) == 2 ? GEN2_GTTMMADR_BAR : GTTMMADR_BAR;
 	phys_addr = pci_resource_start(pdev, mmio_bar);
 
 	/*
@@ -838,14 +836,74 @@ int intel_gt_probe_all(struct drm_i915_private *i915)
 	 * and it has been already initialized early during probe
 	 * in i915_driver_probe()
 	 */
+	gt->i915 = i915;
+	gt->name = "Primary GT";
+	gt->info.engine_mask = RUNTIME_INFO(i915)->platform_engine_mask;
+
+	drm_dbg(&i915->drm, "Setting up %s\n", gt->name);
 	ret = intel_gt_tile_setup(gt, phys_addr);
 	if (ret)
 		return ret;
 
 	i915->gt[0] = gt;
 
-	/* TODO: add more tiles */
+	if (!HAS_EXTRA_GT_LIST(i915))
+		return 0;
+
+	for (i = 1, gtdef = &INTEL_INFO(i915)->extra_gt_list[i - 1];
+	     gtdef->name != NULL;
+	     i++, gtdef = &INTEL_INFO(i915)->extra_gt_list[i - 1]) {
+		gt = drmm_kzalloc(&i915->drm, sizeof(*gt), GFP_KERNEL);
+		if (!gt) {
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		gt->i915 = i915;
+		gt->name = gtdef->name;
+		gt->type = gtdef->type;
+		gt->info.engine_mask = gtdef->engine_mask;
+		gt->info.id = i;
+
+		drm_dbg(&i915->drm, "Setting up %s\n", gt->name);
+		if (GEM_WARN_ON(range_overflows_t(resource_size_t,
+						  gtdef->mapping_base,
+						  SZ_16M,
+						  pci_resource_len(pdev, mmio_bar)))) {
+			ret = -ENODEV;
+			goto err;
+		}
+
+		switch (gtdef->type) {
+		case GT_TILE:
+			ret = intel_gt_tile_setup(gt, phys_addr + gtdef->mapping_base);
+			break;
+
+		case GT_MEDIA:
+			ret = intel_sa_mediagt_setup(gt, phys_addr + gtdef->mapping_base,
+						     gtdef->gsi_offset);
+			break;
+
+		case GT_PRIMARY:
+			/* Primary GT should not appear in extra GT list */
+		default:
+			MISSING_CASE(gtdef->type);
+			ret = -ENODEV;
+		}
+
+		if (ret)
+			goto err;
+
+		i915->gt[i] = gt;
+	}
+
 	return 0;
+
+err:
+	i915_probe_error(i915, "Failed to initialize %s! (%d)\n", gtdef->name, ret);
+	intel_gt_release_all(i915);
+
+	return ret;
 }
 
 int intel_gt_tiles_init(struct drm_i915_private *i915)
@@ -868,10 +926,8 @@ void intel_gt_release_all(struct drm_i915_private *i915)
 	struct intel_gt *gt;
 	unsigned int id;
 
-	for_each_gt(gt, i915, id) {
-		intel_gt_tile_cleanup(gt);
+	for_each_gt(gt, i915, id)
 		i915->gt[id] = NULL;
-	}
 }
 
 void intel_gt_info_print(const struct intel_gt_info *info,
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.h b/drivers/gpu/drm/i915/gt/intel_gt.h
index 40b06adf509a..2ee582e287c8 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt.h
@@ -44,7 +44,8 @@ static inline struct intel_gt *gsc_to_gt(struct intel_gsc *gsc)
 	return container_of(gsc, struct intel_gt, gsc);
 }
 
-void intel_root_gt_init_early(struct drm_i915_private *i915);
+void intel_gt_common_init_early(struct intel_gt *gt);
+int intel_root_gt_init_early(struct drm_i915_private *i915);
 int intel_gt_assign_ggtt(struct intel_gt *gt);
 int intel_gt_init_mmio(struct intel_gt *gt);
 int __must_check intel_gt_init_hw(struct intel_gt *gt);
@@ -54,7 +55,6 @@ void intel_gt_driver_register(struct intel_gt *gt);
 void intel_gt_driver_unregister(struct intel_gt *gt);
 void intel_gt_driver_remove(struct intel_gt *gt);
 void intel_gt_driver_release(struct intel_gt *gt);
-
 void intel_gt_driver_late_release_all(struct drm_i915_private *i915);
 
 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout);
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c
index d5d1b04dbcad..3f656d3dba9a 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_clock_utils.c
@@ -26,26 +26,6 @@ static u32 read_reference_ts_freq(struct intel_uncore *uncore)
 	return base_freq + frac_freq;
 }
 
-static u32 gen9_get_crystal_clock_freq(struct intel_uncore *uncore,
-				       u32 rpm_config_reg)
-{
-	u32 f19_2_mhz = 19200000;
-	u32 f24_mhz = 24000000;
-	u32 crystal_clock =
-		(rpm_config_reg & GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK) >>
-		GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT;
-
-	switch (crystal_clock) {
-	case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ:
-		return f19_2_mhz;
-	case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ:
-		return f24_mhz;
-	default:
-		MISSING_CASE(crystal_clock);
-		return 0;
-	}
-}
-
 static u32 gen11_get_crystal_clock_freq(struct intel_uncore *uncore,
 					u32 rpm_config_reg)
 {
@@ -72,98 +52,106 @@ static u32 gen11_get_crystal_clock_freq(struct intel_uncore *uncore,
 	}
 }
 
-static u32 read_clock_frequency(struct intel_uncore *uncore)
+static u32 gen11_read_clock_frequency(struct intel_uncore *uncore)
 {
-	u32 f12_5_mhz = 12500000;
-	u32 f19_2_mhz = 19200000;
-	u32 f24_mhz = 24000000;
+	u32 ctc_reg = intel_uncore_read(uncore, CTC_MODE);
+	u32 freq = 0;
+
+	/*
+	 * Note that on gen11+, the clock frequency may be reconfigured.
+	 * We do not, and we assume nobody else does.
+	 *
+	 * First figure out the reference frequency. There are 2 ways
+	 * we can compute the frequency, either through the
+	 * TIMESTAMP_OVERRIDE register or through RPM_CONFIG. CTC_MODE
+	 * tells us which one we should use.
+	 */
+	if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) {
+		freq = read_reference_ts_freq(uncore);
+	} else {
+		u32 c0 = intel_uncore_read(uncore, RPM_CONFIG0);
+
+		freq = gen11_get_crystal_clock_freq(uncore, c0);
 
-	if (GRAPHICS_VER(uncore->i915) <= 4) {
-		/*
-		 * PRMs say:
-		 *
-		 *     "The value in this register increments once every 16
-		 *      hclks." (through the “Clocking Configuration�
-		 *      (“CLKCFG�) MCHBAR register)
-		 */
-		return RUNTIME_INFO(uncore->i915)->rawclk_freq * 1000 / 16;
-	} else if (GRAPHICS_VER(uncore->i915) <= 8) {
 		/*
-		 * PRMs say:
-		 *
-		 *     "The PCU TSC counts 10ns increments; this timestamp
-		 *      reflects bits 38:3 of the TSC (i.e. 80ns granularity,
-		 *      rolling over every 1.5 hours).
+		 * Now figure out how the command stream's timestamp
+		 * register increments from this frequency (it might
+		 * increment only every few clock cycle).
 		 */
-		return f12_5_mhz;
-	} else if (GRAPHICS_VER(uncore->i915) <= 9) {
-		u32 ctc_reg = intel_uncore_read(uncore, CTC_MODE);
-		u32 freq = 0;
-
-		if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) {
-			freq = read_reference_ts_freq(uncore);
-		} else {
-			freq = IS_GEN9_LP(uncore->i915) ? f19_2_mhz : f24_mhz;
-
-			/*
-			 * Now figure out how the command stream's timestamp
-			 * register increments from this frequency (it might
-			 * increment only every few clock cycle).
-			 */
-			freq >>= 3 - ((ctc_reg & CTC_SHIFT_PARAMETER_MASK) >>
-				      CTC_SHIFT_PARAMETER_SHIFT);
-		}
-
-		return freq;
-	} else if (GRAPHICS_VER(uncore->i915) <= 12) {
-		u32 ctc_reg = intel_uncore_read(uncore, CTC_MODE);
-		u32 freq = 0;
+		freq >>= 3 - ((c0 & GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >>
+			      GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT);
+	}
+
+	return freq;
+}
+
+static u32 gen9_read_clock_frequency(struct intel_uncore *uncore)
+{
+	u32 ctc_reg = intel_uncore_read(uncore, CTC_MODE);
+	u32 freq = 0;
+
+	if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) {
+		freq = read_reference_ts_freq(uncore);
+	} else {
+		freq = IS_GEN9_LP(uncore->i915) ? 19200000 : 24000000;
 
 		/*
-		 * First figure out the reference frequency. There are 2 ways
-		 * we can compute the frequency, either through the
-		 * TIMESTAMP_OVERRIDE register or through RPM_CONFIG. CTC_MODE
-		 * tells us which one we should use.
+		 * Now figure out how the command stream's timestamp
+		 * register increments from this frequency (it might
+		 * increment only every few clock cycle).
 		 */
-		if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) {
-			freq = read_reference_ts_freq(uncore);
-		} else {
-			u32 c0 = intel_uncore_read(uncore, RPM_CONFIG0);
-
-			if (GRAPHICS_VER(uncore->i915) >= 11)
-				freq = gen11_get_crystal_clock_freq(uncore, c0);
-			else
-				freq = gen9_get_crystal_clock_freq(uncore, c0);
-
-			/*
-			 * Now figure out how the command stream's timestamp
-			 * register increments from this frequency (it might
-			 * increment only every few clock cycle).
-			 */
-			freq >>= 3 - ((c0 & GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >>
-				      GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT);
-		}
-
-		return freq;
+		freq >>= 3 - ((ctc_reg & CTC_SHIFT_PARAMETER_MASK) >>
+			      CTC_SHIFT_PARAMETER_SHIFT);
 	}
 
-	MISSING_CASE("Unknown gen, unable to read command streamer timestamp frequency\n");
-	return 0;
+	return freq;
 }
 
-void intel_gt_init_clock_frequency(struct intel_gt *gt)
+static u32 gen5_read_clock_frequency(struct intel_uncore *uncore)
 {
 	/*
-	 * Note that on gen11+, the clock frequency may be reconfigured.
-	 * We do not, and we assume nobody else does.
+	 * PRMs say:
+	 *
+	 *     "The PCU TSC counts 10ns increments; this timestamp
+	 *      reflects bits 38:3 of the TSC (i.e. 80ns granularity,
+	 *      rolling over every 1.5 hours).
+	 */
+	return 12500000;
+}
+
+static u32 gen2_read_clock_frequency(struct intel_uncore *uncore)
+{
+	/*
+	 * PRMs say:
+	 *
+	 *     "The value in this register increments once every 16
+	 *      hclks." (through the “Clocking Configuration�
+	 *      (“CLKCFG�) MCHBAR register)
 	 */
+	return RUNTIME_INFO(uncore->i915)->rawclk_freq * 1000 / 16;
+}
+
+static u32 read_clock_frequency(struct intel_uncore *uncore)
+{
+	if (GRAPHICS_VER(uncore->i915) >= 11)
+		return gen11_read_clock_frequency(uncore);
+	else if (GRAPHICS_VER(uncore->i915) >= 9)
+		return gen9_read_clock_frequency(uncore);
+	else if (GRAPHICS_VER(uncore->i915) >= 5)
+		return gen5_read_clock_frequency(uncore);
+	else
+		return gen2_read_clock_frequency(uncore);
+}
+
+void intel_gt_init_clock_frequency(struct intel_gt *gt)
+{
 	gt->clock_frequency = read_clock_frequency(gt->uncore);
-	if (gt->clock_frequency)
-		gt->clock_period_ns = intel_gt_clock_interval_to_ns(gt, 1);
 
 	/* Icelake appears to use another fixed frequency for CTX_TIMESTAMP */
 	if (GRAPHICS_VER(gt->i915) == 11)
 		gt->clock_period_ns = NSEC_PER_SEC / 13750000;
+	else if (gt->clock_frequency)
+		gt->clock_period_ns = intel_gt_clock_interval_to_ns(gt, 1);
 
 	GT_TRACE(gt,
 		 "Using clock frequency: %dkHz, period: %dns, wrap: %lldms\n",
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_irq.c
index 3a72d4fd0214..f26882fdc24c 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_irq.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_irq.c
@@ -29,7 +29,7 @@ gen11_gt_engine_identity(struct intel_gt *gt,
 	u32 timeout_ts;
 	u32 ident;
 
-	lockdep_assert_held(&gt->irq_lock);
+	lockdep_assert_held(gt->irq_lock);
 
 	raw_reg_write(regs, GEN11_IIR_REG_SELECTOR(bank), BIT(bit));
 
@@ -59,11 +59,17 @@ static void
 gen11_other_irq_handler(struct intel_gt *gt, const u8 instance,
 			const u16 iir)
 {
+	struct intel_gt *media_gt = gt->i915->media_gt;
+
 	if (instance == OTHER_GUC_INSTANCE)
 		return guc_irq_handler(&gt->uc.guc, iir);
+	if (instance == OTHER_MEDIA_GUC_INSTANCE && media_gt)
+		return guc_irq_handler(&media_gt->uc.guc, iir);
 
 	if (instance == OTHER_GTPM_INSTANCE)
 		return gen11_rps_irq_handler(&gt->rps, iir);
+	if (instance == OTHER_MEDIA_GTPM_INSTANCE && media_gt)
+		return gen11_rps_irq_handler(&media_gt->rps, iir);
 
 	if (instance == OTHER_KCR_INSTANCE)
 		return intel_pxp_irq_handler(&gt->pxp, iir);
@@ -81,6 +87,18 @@ gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
 {
 	struct intel_engine_cs *engine;
 
+	/*
+	 * Platforms with standalone media have their media engines in another
+	 * GT.
+	 */
+	if (MEDIA_VER(gt->i915) >= 13 &&
+	    (class == VIDEO_DECODE_CLASS || class == VIDEO_ENHANCEMENT_CLASS)) {
+		if (!gt->i915->media_gt)
+			goto err;
+
+		gt = gt->i915->media_gt;
+	}
+
 	if (instance <= MAX_ENGINE_INSTANCE)
 		engine = gt->engine_class[class][instance];
 	else
@@ -89,6 +107,7 @@ gen11_engine_irq_handler(struct intel_gt *gt, const u8 class,
 	if (likely(engine))
 		return intel_engine_cs_irq(engine, iir);
 
+err:
 	WARN_ONCE(1, "unhandled engine interrupt class=0x%x, instance=0x%x\n",
 		  class, instance);
 }
@@ -120,7 +139,7 @@ gen11_gt_bank_handler(struct intel_gt *gt, const unsigned int bank)
 	unsigned long intr_dw;
 	unsigned int bit;
 
-	lockdep_assert_held(&gt->irq_lock);
+	lockdep_assert_held(gt->irq_lock);
 
 	intr_dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 
@@ -138,14 +157,14 @@ void gen11_gt_irq_handler(struct intel_gt *gt, const u32 master_ctl)
 {
 	unsigned int bank;
 
-	spin_lock(&gt->irq_lock);
+	spin_lock(gt->irq_lock);
 
 	for (bank = 0; bank < 2; bank++) {
 		if (master_ctl & GEN11_GT_DW_IRQ(bank))
 			gen11_gt_bank_handler(gt, bank);
 	}
 
-	spin_unlock(&gt->irq_lock);
+	spin_unlock(gt->irq_lock);
 }
 
 bool gen11_gt_reset_one_iir(struct intel_gt *gt,
@@ -154,7 +173,7 @@ bool gen11_gt_reset_one_iir(struct intel_gt *gt,
 	void __iomem * const regs = gt->uncore->regs;
 	u32 dw;
 
-	lockdep_assert_held(&gt->irq_lock);
+	lockdep_assert_held(gt->irq_lock);
 
 	dw = raw_reg_read(regs, GEN11_GT_INTR_DW(bank));
 	if (dw & BIT(bit)) {
@@ -310,9 +329,9 @@ static void gen7_parity_error_irq_handler(struct intel_gt *gt, u32 iir)
 	if (!HAS_L3_DPF(gt->i915))
 		return;
 
-	spin_lock(&gt->irq_lock);
+	spin_lock(gt->irq_lock);
 	gen5_gt_disable_irq(gt, GT_PARITY_ERROR(gt->i915));
-	spin_unlock(&gt->irq_lock);
+	spin_unlock(gt->irq_lock);
 
 	if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
 		gt->i915->l3_parity.which_slice |= 1 << 1;
@@ -434,7 +453,7 @@ static void gen5_gt_update_irq(struct intel_gt *gt,
 			       u32 interrupt_mask,
 			       u32 enabled_irq_mask)
 {
-	lockdep_assert_held(&gt->irq_lock);
+	lockdep_assert_held(gt->irq_lock);
 
 	GEM_BUG_ON(enabled_irq_mask & ~interrupt_mask);
 
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm.h b/drivers/gpu/drm/i915/gt/intel_gt_pm.h
index a334787a4939..6c9a46452364 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm.h
@@ -55,6 +55,14 @@ static inline void intel_gt_pm_might_put(struct intel_gt *gt)
 	for (tmp = 1, intel_gt_pm_get(gt); tmp; \
 	     intel_gt_pm_put(gt), tmp = 0)
 
+/**
+ * with_intel_gt_pm_if_awake - if GT is PM awake, get a reference to prevent
+ *	it to sleep, run some code and then asynchrously put the reference
+ *	away.
+ *
+ * @gt: pointer to the gt
+ * @wf: pointer to a temporary wakeref.
+ */
 #define with_intel_gt_pm_if_awake(gt, wf) \
 	for (wf = intel_gt_pm_get_if_awake(gt); wf; intel_gt_pm_put_async(gt), wf = 0)
 
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
index 40bdd4cb629f..108b9e76c32e 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
@@ -504,8 +504,8 @@ void intel_gt_pm_frequency_dump(struct intel_gt *gt, struct drm_printer *p)
 		drm_puts(p, "no P-state info available\n");
 	}
 
-	drm_printf(p, "Current CD clock frequency: %d kHz\n", i915->cdclk.hw.cdclk);
-	drm_printf(p, "Max CD clock frequency: %d kHz\n", i915->max_cdclk_freq);
+	drm_printf(p, "Current CD clock frequency: %d kHz\n", i915->display.cdclk.hw.cdclk);
+	drm_printf(p, "Max CD clock frequency: %d kHz\n", i915->display.cdclk.max_cdclk_freq);
 	drm_printf(p, "Max pixel clock frequency: %d kHz\n", i915->max_dotclk_freq);
 
 	intel_runtime_pm_put(uncore->rpm, wakeref);
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c
index 11060f5a4c89..52f2a28b2058 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_irq.c
@@ -37,7 +37,7 @@ static void gen6_gt_pm_update_irq(struct intel_gt *gt,
 
 	WARN_ON(enabled_irq_mask & ~interrupt_mask);
 
-	lockdep_assert_held(&gt->irq_lock);
+	lockdep_assert_held(gt->irq_lock);
 
 	new_val = gt->pm_imr;
 	new_val &= ~interrupt_mask;
@@ -64,7 +64,7 @@ void gen6_gt_pm_reset_iir(struct intel_gt *gt, u32 reset_mask)
 	struct intel_uncore *uncore = gt->uncore;
 	i915_reg_t reg = GRAPHICS_VER(gt->i915) >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
 
-	lockdep_assert_held(&gt->irq_lock);
+	lockdep_assert_held(gt->irq_lock);
 
 	intel_uncore_write(uncore, reg, reset_mask);
 	intel_uncore_write(uncore, reg, reset_mask);
@@ -92,7 +92,7 @@ static void write_pm_ier(struct intel_gt *gt)
 
 void gen6_gt_pm_enable_irq(struct intel_gt *gt, u32 enable_mask)
 {
-	lockdep_assert_held(&gt->irq_lock);
+	lockdep_assert_held(gt->irq_lock);
 
 	gt->pm_ier |= enable_mask;
 	write_pm_ier(gt);
@@ -101,7 +101,7 @@ void gen6_gt_pm_enable_irq(struct intel_gt *gt, u32 enable_mask)
 
 void gen6_gt_pm_disable_irq(struct intel_gt *gt, u32 disable_mask)
 {
-	lockdep_assert_held(&gt->irq_lock);
+	lockdep_assert_held(gt->irq_lock);
 
 	gt->pm_ier &= ~disable_mask;
 	gen6_gt_pm_mask_irq(gt, disable_mask);
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_regs.h b/drivers/gpu/drm/i915/gt/intel_gt_regs.h
index 60d6eb5f245b..2275ee47da95 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_regs.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_regs.h
@@ -259,6 +259,9 @@
 #define   GEN9_PREEMPT_GPGPU_COMMAND_LEVEL	GEN9_PREEMPT_GPGPU_LEVEL(1, 0)
 #define   GEN9_PREEMPT_GPGPU_LEVEL_MASK		GEN9_PREEMPT_GPGPU_LEVEL(1, 1)
 
+#define DRAW_WATERMARK				_MMIO(0x26c0)
+#define   VERT_WM_VAL				REG_GENMASK(9, 0)
+
 #define GEN12_GLOBAL_MOCS(i)			_MMIO(0x4000 + (i) * 4) /* Global MOCS regs */
 
 #define RENDER_HWS_PGA_GEN7			_MMIO(0x4080)
@@ -374,6 +377,9 @@
 #define CHICKEN_RASTER_1			_MMIO(0x6204)
 #define   DIS_SF_ROUND_NEAREST_EVEN		REG_BIT(8)
 
+#define CHICKEN_RASTER_2			_MMIO(0x6208)
+#define   TBIMR_FAST_CLIP			REG_BIT(5)
+
 #define VFLSKPD					_MMIO(0x62a8)
 #define   DIS_OVER_FETCH_CACHE			REG_BIT(1)
 #define   DIS_MULT_MISS_RD_SQUASH		REG_BIT(0)
@@ -1007,6 +1013,8 @@
 #define   GEN11_LSN_UNSLCVC_GAFS_HALF_CL2_MAXALLOC	(1 << 9)
 #define   GEN11_LSN_UNSLCVC_GAFS_HALF_SF_MAXALLOC	(1 << 7)
 
+#define GUCPMTIMESTAMP				_MMIO(0xc3e8)
+
 #define __GEN9_RCS0_MOCS0			0xc800
 #define GEN9_GFX_MOCS(i)			_MMIO(__GEN9_RCS0_MOCS0 + (i) * 4)
 #define __GEN9_VCS0_MOCS0			0xc900
@@ -1078,6 +1086,7 @@
 
 #define GEN10_SAMPLER_MODE			_MMIO(0xe18c)
 #define   ENABLE_SMALLPL			REG_BIT(15)
+#define   SC_DISABLE_POWER_OPTIMIZATION_EBB	REG_BIT(9)
 #define   GEN11_SAMPLER_ENABLE_HEADLESS_MSG	REG_BIT(5)
 
 #define GEN9_HALF_SLICE_CHICKEN7		_MMIO(0xe194)
@@ -1101,6 +1110,8 @@
 #define   GEN12_DISABLE_TDL_PUSH		REG_BIT(9)
 #define   GEN11_DIS_PICK_2ND_EU			REG_BIT(7)
 #define   GEN12_DISABLE_HDR_PAST_PAYLOAD_HOLD_FIX	REG_BIT(4)
+#define   THREAD_EX_ARB_MODE			REG_GENMASK(3, 2)
+#define   THREAD_EX_ARB_MODE_RR_AFTER_DEP	REG_FIELD_PREP(THREAD_EX_ARB_MODE, 0x2)
 
 #define HSW_ROW_CHICKEN3			_MMIO(0xe49c)
 #define   HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE	(1 << 6)
@@ -1123,6 +1134,8 @@
 
 #define RT_CTRL					_MMIO(0xe530)
 #define   DIS_NULL_QUERY			REG_BIT(10)
+#define   STACKID_CTRL				REG_GENMASK(6, 5)
+#define   STACKID_CTRL_512			REG_FIELD_PREP(STACKID_CTRL, 0x2)
 
 #define EU_PERF_CNTL1				_MMIO(0xe558)
 #define EU_PERF_CNTL5				_MMIO(0xe55c)
@@ -1541,6 +1554,8 @@
 #define   OTHER_GTPM_INSTANCE			1
 #define   OTHER_KCR_INSTANCE			4
 #define   OTHER_GSC_INSTANCE			6
+#define   OTHER_MEDIA_GUC_INSTANCE		16
+#define   OTHER_MEDIA_GTPM_INSTANCE		17
 
 #define GEN11_IIR_REG_SELECTOR(x)		_MMIO(0x190070 + ((x) * 4))
 
@@ -1565,4 +1580,12 @@
 
 #define GEN12_SFC_DONE(n)			_MMIO(0x1cc000 + (n) * 0x1000)
 
+/*
+ * Standalone Media's non-engine GT registers are located at their regular GT
+ * offsets plus 0x380000.  This extra offset is stored inside the intel_uncore
+ * structure so that the existing code can be used for both GTs without
+ * modification.
+ */
+#define MTL_MEDIA_GSI_BASE			0x380000
+
 #endif /* __INTEL_GT_REGS__ */
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_sysfs.c b/drivers/gpu/drm/i915/gt/intel_gt_sysfs.c
index 9e4ebf53379b..d651ccd0ab20 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_sysfs.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_sysfs.c
@@ -22,11 +22,6 @@ bool is_object_gt(struct kobject *kobj)
 	return !strncmp(kobj->name, "gt", 2);
 }
 
-static struct intel_gt *kobj_to_gt(struct kobject *kobj)
-{
-	return container_of(kobj, struct intel_gt, sysfs_gt);
-}
-
 struct intel_gt *intel_gt_sysfs_get_drvdata(struct device *dev,
 					    const char *name)
 {
@@ -101,6 +96,10 @@ void intel_gt_sysfs_register(struct intel_gt *gt)
 				 gt->i915->sysfs_gt, "gt%d", gt->info.id))
 		goto exit_fail;
 
+	gt->sysfs_defaults = kobject_create_and_add(".defaults", &gt->sysfs_gt);
+	if (!gt->sysfs_defaults)
+		goto exit_fail;
+
 	intel_gt_sysfs_pm_init(gt, &gt->sysfs_gt);
 
 	return;
@@ -113,5 +112,6 @@ exit_fail:
 
 void intel_gt_sysfs_unregister(struct intel_gt *gt)
 {
+	kobject_put(gt->sysfs_defaults);
 	kobject_put(&gt->sysfs_gt);
 }
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_sysfs.h b/drivers/gpu/drm/i915/gt/intel_gt_sysfs.h
index a99aa7e8b01a..6232923a420d 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_sysfs.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_sysfs.h
@@ -10,6 +10,7 @@
 #include <linux/kobject.h>
 
 #include "i915_gem.h" /* GEM_BUG_ON() */
+#include "intel_gt_types.h"
 
 struct intel_gt;
 
@@ -22,6 +23,11 @@ intel_gt_create_kobj(struct intel_gt *gt,
 		     struct kobject *dir,
 		     const char *name);
 
+static inline struct intel_gt *kobj_to_gt(struct kobject *kobj)
+{
+	return container_of(kobj, struct intel_gt, sysfs_gt);
+}
+
 void intel_gt_sysfs_register(struct intel_gt *gt);
 void intel_gt_sysfs_unregister(struct intel_gt *gt);
 struct intel_gt *intel_gt_sysfs_get_drvdata(struct device *dev,
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c b/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c
index 73a8b46e0234..180dd6f3ef57 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_sysfs_pm.c
@@ -545,8 +545,7 @@ static INTEL_GT_RPS_BOOL_ATTR_RO(throttle_reason_ratl, RATL_MASK);
 static INTEL_GT_RPS_BOOL_ATTR_RO(throttle_reason_vr_thermalert, VR_THERMALERT_MASK);
 static INTEL_GT_RPS_BOOL_ATTR_RO(throttle_reason_vr_tdc, VR_TDC_MASK);
 
-static const struct attribute *freq_attrs[] = {
-	&dev_attr_punit_req_freq_mhz.attr,
+static const struct attribute *throttle_reason_attrs[] = {
 	&attr_throttle_reason_status.attr,
 	&attr_throttle_reason_pl1.attr,
 	&attr_throttle_reason_pl2.attr,
@@ -727,6 +726,34 @@ static const struct attribute *media_perf_power_attrs[] = {
 	NULL
 };
 
+static ssize_t
+default_min_freq_mhz_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	struct intel_gt *gt = kobj_to_gt(kobj->parent);
+
+	return sysfs_emit(buf, "%u\n", gt->defaults.min_freq);
+}
+
+static struct kobj_attribute default_min_freq_mhz =
+__ATTR(rps_min_freq_mhz, 0444, default_min_freq_mhz_show, NULL);
+
+static ssize_t
+default_max_freq_mhz_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	struct intel_gt *gt = kobj_to_gt(kobj->parent);
+
+	return sysfs_emit(buf, "%u\n", gt->defaults.max_freq);
+}
+
+static struct kobj_attribute default_max_freq_mhz =
+__ATTR(rps_max_freq_mhz, 0444, default_max_freq_mhz_show, NULL);
+
+static const struct attribute * const rps_defaults_attrs[] = {
+	&default_min_freq_mhz.attr,
+	&default_max_freq_mhz.attr,
+	NULL
+};
+
 static int intel_sysfs_rps_init(struct intel_gt *gt, struct kobject *kobj,
 				const struct attribute * const *attrs)
 {
@@ -763,12 +790,20 @@ void intel_gt_sysfs_pm_init(struct intel_gt *gt, struct kobject *kobj)
 	if (!is_object_gt(kobj))
 		return;
 
-	ret = sysfs_create_files(kobj, freq_attrs);
+	ret = sysfs_create_file(kobj, &dev_attr_punit_req_freq_mhz.attr);
 	if (ret)
 		drm_warn(&gt->i915->drm,
-			 "failed to create gt%u throttle sysfs files (%pe)",
+			 "failed to create gt%u punit_req_freq_mhz sysfs (%pe)",
 			 gt->info.id, ERR_PTR(ret));
 
+	if (GRAPHICS_VER(gt->i915) >= 11) {
+		ret = sysfs_create_files(kobj, throttle_reason_attrs);
+		if (ret)
+			drm_warn(&gt->i915->drm,
+				 "failed to create gt%u throttle sysfs files (%pe)",
+				 gt->info.id, ERR_PTR(ret));
+	}
+
 	if (HAS_MEDIA_RATIO_MODE(gt->i915) && intel_uc_uses_guc_slpc(&gt->uc)) {
 		ret = sysfs_create_files(kobj, media_perf_power_attrs);
 		if (ret)
@@ -776,4 +811,10 @@ void intel_gt_sysfs_pm_init(struct intel_gt *gt, struct kobject *kobj)
 				 "failed to create gt%u media_perf_power_attrs sysfs (%pe)\n",
 				 gt->info.id, ERR_PTR(ret));
 	}
+
+	ret = sysfs_create_files(gt->sysfs_defaults, rps_defaults_attrs);
+	if (ret)
+		drm_warn(&gt->i915->drm,
+			 "failed to add gt%u rps defaults (%pe)\n",
+			 gt->info.id, ERR_PTR(ret));
 }
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_types.h b/drivers/gpu/drm/i915/gt/intel_gt_types.h
index 3804a583382b..f19c2de77ff6 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_types.h
+++ b/drivers/gpu/drm/i915/gt/intel_gt_types.h
@@ -76,8 +76,22 @@ enum intel_submission_method {
 	INTEL_SUBMISSION_GUC,
 };
 
+struct gt_defaults {
+	u32 min_freq;
+	u32 max_freq;
+};
+
+enum intel_gt_type {
+	GT_PRIMARY,
+	GT_TILE,
+	GT_MEDIA,
+};
+
 struct intel_gt {
 	struct drm_i915_private *i915;
+	const char *name;
+	enum intel_gt_type type;
+
 	struct intel_uncore *uncore;
 	struct i915_ggtt *ggtt;
 
@@ -127,6 +141,20 @@ struct intel_gt {
 	struct intel_wakeref wakeref;
 	atomic_t user_wakeref;
 
+	/**
+	 *  Protects access to lmem usefault list.
+	 *  It is required, if we are outside of the runtime suspend path,
+	 *  access to @lmem_userfault_list requires always first grabbing the
+	 *  runtime pm, to ensure we can't race against runtime suspend.
+	 *  Once we have that we also need to grab @lmem_userfault_lock,
+	 *  at which point we have exclusive access.
+	 *  The runtime suspend path is special since it doesn't really hold any locks,
+	 *  but instead has exclusive access by virtue of all other accesses requiring
+	 *  holding the runtime pm wakeref.
+	 */
+	struct mutex lmem_userfault_lock;
+	struct list_head lmem_userfault_list;
+
 	struct list_head closed_vma;
 	spinlock_t closed_lock; /* guards the list of closed_vma */
 
@@ -142,6 +170,9 @@ struct intel_gt {
 	 */
 	intel_wakeref_t awake;
 
+	/* Manual runtime pm autosuspend delay for user GGTT/lmem mmaps */
+	struct intel_wakeref_auto userfault_wakeref;
+
 	u32 clock_frequency;
 	u32 clock_period_ns;
 
@@ -149,7 +180,7 @@ struct intel_gt {
 	struct intel_rc6 rc6;
 	struct intel_rps rps;
 
-	spinlock_t irq_lock;
+	spinlock_t *irq_lock;
 	u32 gt_imr;
 	u32 pm_ier;
 	u32 pm_imr;
@@ -251,6 +282,18 @@ struct intel_gt {
 
 	/* gt/gtN sysfs */
 	struct kobject sysfs_gt;
+
+	/* sysfs defaults per gt */
+	struct gt_defaults defaults;
+	struct kobject *sysfs_defaults;
+};
+
+struct intel_gt_definition {
+	enum intel_gt_type type;
+	char *name;
+	u32 mapping_base;
+	u32 gsi_offset;
+	intel_engine_mask_t engine_mask;
 };
 
 enum intel_gt_scratch_field {
diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.c b/drivers/gpu/drm/i915/gt/intel_gtt.c
index b67831833c9a..2eaeba14319e 100644
--- a/drivers/gpu/drm/i915/gt/intel_gtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gtt.c
@@ -405,6 +405,9 @@ void free_scratch(struct i915_address_space *vm)
 {
 	int i;
 
+	if (!vm->scratch[0])
+		return;
+
 	for (i = 0; i <= vm->top; i++)
 		i915_gem_object_put(vm->scratch[i]);
 }
diff --git a/drivers/gpu/drm/i915/gt/intel_gtt.h b/drivers/gpu/drm/i915/gt/intel_gtt.h
index e639434e97fd..c0ca53cba9f0 100644
--- a/drivers/gpu/drm/i915/gt/intel_gtt.h
+++ b/drivers/gpu/drm/i915/gt/intel_gtt.h
@@ -386,9 +386,6 @@ struct i915_ggtt {
 	 */
 	struct list_head userfault_list;
 
-	/* Manual runtime pm autosuspend delay for user GGTT mmaps */
-	struct intel_wakeref_auto userfault_wakeref;
-
 	struct mutex error_mutex;
 	struct drm_mm_node error_capture;
 	struct drm_mm_node uc_fw;
diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c
index eec73c66406c..3955292483a6 100644
--- a/drivers/gpu/drm/i915/gt/intel_lrc.c
+++ b/drivers/gpu/drm/i915/gt/intel_lrc.c
@@ -662,6 +662,21 @@ static int lrc_ring_mi_mode(const struct intel_engine_cs *engine)
 		return -1;
 }
 
+static int lrc_ring_bb_offset(const struct intel_engine_cs *engine)
+{
+	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
+		return 0x80;
+	else if (GRAPHICS_VER(engine->i915) >= 12)
+		return 0x70;
+	else if (GRAPHICS_VER(engine->i915) >= 9)
+		return 0x64;
+	else if (GRAPHICS_VER(engine->i915) >= 8 &&
+		 engine->class == RENDER_CLASS)
+		return 0xc4;
+	else
+		return -1;
+}
+
 static int lrc_ring_gpr0(const struct intel_engine_cs *engine)
 {
 	if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
@@ -768,6 +783,7 @@ static void init_common_regs(u32 * const regs,
 			     bool inhibit)
 {
 	u32 ctl;
+	int loc;
 
 	ctl = _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH);
 	ctl |= _MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
@@ -779,6 +795,10 @@ static void init_common_regs(u32 * const regs,
 	regs[CTX_CONTEXT_CONTROL] = ctl;
 
 	regs[CTX_TIMESTAMP] = ce->stats.runtime.last;
+
+	loc = lrc_ring_bb_offset(engine);
+	if (loc != -1)
+		regs[loc + 1] = 0;
 }
 
 static void init_wa_bb_regs(u32 * const regs,
@@ -1242,6 +1262,23 @@ dg2_emit_rcs_hang_wabb(const struct intel_context *ce, u32 *cs)
 	return cs;
 }
 
+/*
+ * The bspec's tuning guide asks us to program a vertical watermark value of
+ * 0x3FF.  However this register is not saved/restored properly by the
+ * hardware, so we're required to apply the desired value via INDIRECT_CTX
+ * batch buffer to ensure the value takes effect properly.  All other bits
+ * in this register should remain at 0 (the hardware default).
+ */
+static u32 *
+dg2_emit_draw_watermark_setting(u32 *cs)
+{
+	*cs++ = MI_LOAD_REGISTER_IMM(1);
+	*cs++ = i915_mmio_reg_offset(DRAW_WATERMARK);
+	*cs++ = REG_FIELD_PREP(VERT_WM_VAL, 0x3FF);
+
+	return cs;
+}
+
 static u32 *
 gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs)
 {
@@ -1261,7 +1298,12 @@ gen12_emit_indirect_ctx_rcs(const struct intel_context *ce, u32 *cs)
 
 	/* hsdes: 1809175790 */
 	if (!HAS_FLAT_CCS(ce->engine->i915))
-		cs = gen12_emit_aux_table_inv(cs, GEN12_GFX_CCS_AUX_NV);
+		cs = gen12_emit_aux_table_inv(ce->engine->gt,
+					      cs, GEN12_GFX_CCS_AUX_NV);
+
+	/* Wa_16014892111 */
+	if (IS_DG2(ce->engine->i915))
+		cs = dg2_emit_draw_watermark_setting(cs);
 
 	return cs;
 }
@@ -1283,9 +1325,11 @@ gen12_emit_indirect_ctx_xcs(const struct intel_context *ce, u32 *cs)
 	/* hsdes: 1809175790 */
 	if (!HAS_FLAT_CCS(ce->engine->i915)) {
 		if (ce->engine->class == VIDEO_DECODE_CLASS)
-			cs = gen12_emit_aux_table_inv(cs, GEN12_VD0_AUX_NV);
+			cs = gen12_emit_aux_table_inv(ce->engine->gt,
+						      cs, GEN12_VD0_AUX_NV);
 		else if (ce->engine->class == VIDEO_ENHANCEMENT_CLASS)
-			cs = gen12_emit_aux_table_inv(cs, GEN12_VE0_AUX_NV);
+			cs = gen12_emit_aux_table_inv(ce->engine->gt,
+						      cs, GEN12_VE0_AUX_NV);
 	}
 
 	return cs;
diff --git a/drivers/gpu/drm/i915/gt/intel_migrate.c b/drivers/gpu/drm/i915/gt/intel_migrate.c
index 933648cc90ff..aaaf1906026c 100644
--- a/drivers/gpu/drm/i915/gt/intel_migrate.c
+++ b/drivers/gpu/drm/i915/gt/intel_migrate.c
@@ -511,44 +511,16 @@ static inline u32 *i915_flush_dw(u32 *cmd, u32 flags)
 	return cmd;
 }
 
-static u32 calc_ctrl_surf_instr_size(struct drm_i915_private *i915, int size)
-{
-	u32 num_cmds, num_blks, total_size;
-
-	if (!GET_CCS_BYTES(i915, size))
-		return 0;
-
-	/*
-	 * XY_CTRL_SURF_COPY_BLT transfers CCS in 256 byte
-	 * blocks. one XY_CTRL_SURF_COPY_BLT command can
-	 * transfer upto 1024 blocks.
-	 */
-	num_blks = DIV_ROUND_UP(GET_CCS_BYTES(i915, size),
-				NUM_CCS_BYTES_PER_BLOCK);
-	num_cmds = DIV_ROUND_UP(num_blks, NUM_CCS_BLKS_PER_XFER);
-	total_size = XY_CTRL_SURF_INSTR_SIZE * num_cmds;
-
-	/*
-	 * Adding a flush before and after XY_CTRL_SURF_COPY_BLT
-	 */
-	total_size += 2 * MI_FLUSH_DW_SIZE;
-
-	return total_size;
-}
-
 static int emit_copy_ccs(struct i915_request *rq,
 			 u32 dst_offset, u8 dst_access,
 			 u32 src_offset, u8 src_access, int size)
 {
 	struct drm_i915_private *i915 = rq->engine->i915;
 	int mocs = rq->engine->gt->mocs.uc_index << 1;
-	u32 num_ccs_blks, ccs_ring_size;
+	u32 num_ccs_blks;
 	u32 *cs;
 
-	ccs_ring_size = calc_ctrl_surf_instr_size(i915, size);
-	WARN_ON(!ccs_ring_size);
-
-	cs = intel_ring_begin(rq, round_up(ccs_ring_size, 2));
+	cs = intel_ring_begin(rq, 12);
 	if (IS_ERR(cs))
 		return PTR_ERR(cs);
 
@@ -583,8 +555,7 @@ static int emit_copy_ccs(struct i915_request *rq,
 		FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs);
 
 	cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS);
-	if (ccs_ring_size & 1)
-		*cs++ = MI_NOOP;
+	*cs++ = MI_NOOP;
 
 	intel_ring_advance(rq, cs);
 
@@ -645,7 +616,7 @@ static u64 scatter_list_length(struct scatterlist *sg)
 	while (sg && sg_dma_len(sg)) {
 		len += sg_dma_len(sg);
 		sg = sg_next(sg);
-	};
+	}
 
 	return len;
 }
diff --git a/drivers/gpu/drm/i915/gt/intel_ppgtt.c b/drivers/gpu/drm/i915/gt/intel_ppgtt.c
index 6ee8d1127016..7ecfa672f738 100644
--- a/drivers/gpu/drm/i915/gt/intel_ppgtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_ppgtt.c
@@ -312,7 +312,7 @@ void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt,
 	ppgtt->vm.gt = gt;
 	ppgtt->vm.i915 = i915;
 	ppgtt->vm.dma = i915->drm.dev;
-	ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size);
+	ppgtt->vm.total = BIT_ULL(RUNTIME_INFO(i915)->ppgtt_size);
 	ppgtt->vm.lmem_pt_obj_flags = lmem_pt_obj_flags;
 
 	dma_resv_init(&ppgtt->vm._resv);
diff --git a/drivers/gpu/drm/i915/gt/intel_region_lmem.c b/drivers/gpu/drm/i915/gt/intel_region_lmem.c
index aa6aed837194..f3ad93db0b21 100644
--- a/drivers/gpu/drm/i915/gt/intel_region_lmem.c
+++ b/drivers/gpu/drm/i915/gt/intel_region_lmem.c
@@ -4,8 +4,10 @@
  */
 
 #include "i915_drv.h"
+#include "i915_pci.h"
 #include "i915_reg.h"
 #include "intel_memory_region.h"
+#include "intel_pci_config.h"
 #include "intel_region_lmem.h"
 #include "intel_region_ttm.h"
 #include "gem/i915_gem_lmem.h"
@@ -45,7 +47,6 @@ _resize_bar(struct drm_i915_private *i915, int resno, resource_size_t size)
 	drm_info(&i915->drm, "BAR%d resized to %dM\n", resno, 1 << bar_size);
 }
 
-#define LMEM_BAR_NUM 2
 static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t lmem_size)
 {
 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);
@@ -56,15 +57,14 @@ static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t
 	u32 pci_cmd;
 	int i;
 
-	current_size = roundup_pow_of_two(pci_resource_len(pdev, LMEM_BAR_NUM));
+	current_size = roundup_pow_of_two(pci_resource_len(pdev, GEN12_LMEM_BAR));
 
 	if (i915->params.lmem_bar_size) {
 		u32 bar_sizes;
 
 		rebar_size = i915->params.lmem_bar_size *
 			(resource_size_t)SZ_1M;
-		bar_sizes = pci_rebar_get_possible_sizes(pdev,
-							 LMEM_BAR_NUM);
+		bar_sizes = pci_rebar_get_possible_sizes(pdev, GEN12_LMEM_BAR);
 
 		if (rebar_size == current_size)
 			return;
@@ -107,7 +107,7 @@ static void i915_resize_lmem_bar(struct drm_i915_private *i915, resource_size_t
 	pci_write_config_dword(pdev, PCI_COMMAND,
 			       pci_cmd & ~PCI_COMMAND_MEMORY);
 
-	_resize_bar(i915, LMEM_BAR_NUM, rebar_size);
+	_resize_bar(i915, GEN12_LMEM_BAR, rebar_size);
 
 	pci_assign_unassigned_bus_resources(pdev->bus);
 	pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd);
@@ -202,6 +202,9 @@ static struct intel_memory_region *setup_lmem(struct intel_gt *gt)
 	if (!IS_DGFX(i915))
 		return ERR_PTR(-ENODEV);
 
+	if (!i915_pci_resource_valid(pdev, GEN12_LMEM_BAR))
+		return ERR_PTR(-ENXIO);
+
 	if (HAS_FLAT_CCS(i915)) {
 		resource_size_t lmem_range;
 		u64 tile_stolen, flat_ccs_base;
@@ -236,8 +239,8 @@ static struct intel_memory_region *setup_lmem(struct intel_gt *gt)
 				  mul_u32_u32(i915->params.lmem_size, SZ_1M));
 	}
 
-	io_start = pci_resource_start(pdev, 2);
-	io_size = min(pci_resource_len(pdev, 2), lmem_size);
+	io_start = pci_resource_start(pdev, GEN12_LMEM_BAR);
+	io_size = min(pci_resource_len(pdev, GEN12_LMEM_BAR), lmem_size);
 	if (!io_size)
 		return ERR_PTR(-EIO);
 
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index c68d36fb5bbd..b36674356986 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -776,7 +776,7 @@ static void revoke_mmaps(struct intel_gt *gt)
 			continue;
 
 		node = &vma->mmo->vma_node;
-		vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
+		vma_offset = vma->gtt_view.partial.offset << PAGE_SHIFT;
 
 		unmap_mapping_range(gt->i915->drm.anon_inode->i_mapping,
 				    drm_vma_node_offset_addr(node) + vma_offset,
@@ -1281,9 +1281,6 @@ static void intel_gt_reset_global(struct intel_gt *gt,
 	intel_wedge_on_timeout(&w, gt, 5 * HZ) {
 		intel_display_prepare_reset(gt->i915);
 
-		/* Flush everyone using a resource about to be clobbered */
-		synchronize_srcu_expedited(&gt->reset.backoff_srcu);
-
 		intel_gt_reset(gt, engine_mask, reason);
 
 		intel_display_finish_reset(gt->i915);
@@ -1392,6 +1389,9 @@ void intel_gt_handle_error(struct intel_gt *gt,
 		}
 	}
 
+	/* Flush everyone using a resource about to be clobbered */
+	synchronize_srcu_expedited(&gt->reset.backoff_srcu);
+
 	intel_gt_reset_global(gt, engine_mask, msg);
 
 	if (!intel_uc_uses_guc_submission(&gt->uc)) {
diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c
index 7bb967034679..6b86250c31ab 100644
--- a/drivers/gpu/drm/i915/gt/intel_rps.c
+++ b/drivers/gpu/drm/i915/gt/intel_rps.c
@@ -194,9 +194,9 @@ static void rps_enable_interrupts(struct intel_rps *rps)
 
 	rps_reset_ei(rps);
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	gen6_gt_pm_enable_irq(gt, rps->pm_events);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 
 	intel_uncore_write(gt->uncore,
 			   GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
@@ -217,14 +217,14 @@ static void rps_reset_interrupts(struct intel_rps *rps)
 {
 	struct intel_gt *gt = rps_to_gt(rps);
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	if (GRAPHICS_VER(gt->i915) >= 11)
 		gen11_rps_reset_interrupts(rps);
 	else
 		gen6_rps_reset_interrupts(rps);
 
 	rps->pm_iir = 0;
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 }
 
 static void rps_disable_interrupts(struct intel_rps *rps)
@@ -234,9 +234,9 @@ static void rps_disable_interrupts(struct intel_rps *rps)
 	intel_uncore_write(gt->uncore,
 			   GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 
 	intel_synchronize_irq(gt->i915);
 
@@ -1107,7 +1107,12 @@ void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *c
 		caps->min_freq = (rp_state_cap >>  0) & 0xff;
 	} else {
 		caps->rp0_freq = (rp_state_cap >>  0) & 0xff;
-		caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
+		if (GRAPHICS_VER(i915) >= 10)
+			caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
+						       intel_uncore_read(to_gt(i915)->uncore,
+						       GEN10_FREQ_INFO_REC));
+		else
+			caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
 		caps->min_freq = (rp_state_cap >> 16) & 0xff;
 	}
 
@@ -1546,6 +1551,9 @@ void intel_rps_disable(struct intel_rps *rps)
 {
 	struct drm_i915_private *i915 = rps_to_i915(rps);
 
+	if (!intel_rps_is_enabled(rps))
+		return;
+
 	intel_rps_clear_enabled(rps);
 	intel_rps_clear_interrupts(rps);
 	intel_rps_clear_timer(rps);
@@ -1789,10 +1797,10 @@ static void rps_work(struct work_struct *work)
 	int new_freq, adj, min, max;
 	u32 pm_iir = 0;
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
 	client_boost = atomic_read(&rps->num_waiters);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 
 	/* Make sure we didn't queue anything we're not going to process. */
 	if (!pm_iir && !client_boost)
@@ -1865,9 +1873,9 @@ static void rps_work(struct work_struct *work)
 	mutex_unlock(&rps->lock);
 
 out:
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	gen6_gt_pm_unmask_irq(gt, rps->pm_events);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 }
 
 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
@@ -1875,7 +1883,7 @@ void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
 	struct intel_gt *gt = rps_to_gt(rps);
 	const u32 events = rps->pm_events & pm_iir;
 
-	lockdep_assert_held(&gt->irq_lock);
+	lockdep_assert_held(gt->irq_lock);
 
 	if (unlikely(!events))
 		return;
@@ -1895,7 +1903,7 @@ void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
 
 	events = pm_iir & rps->pm_events;
 	if (events) {
-		spin_lock(&gt->irq_lock);
+		spin_lock(gt->irq_lock);
 
 		GT_TRACE(gt, "irq events:%x\n", events);
 
@@ -1903,7 +1911,7 @@ void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
 		rps->pm_iir |= events;
 
 		schedule_work(&rps->work);
-		spin_unlock(&gt->irq_lock);
+		spin_unlock(gt->irq_lock);
 	}
 
 	if (GRAPHICS_VER(gt->i915) >= 8)
@@ -1979,7 +1987,9 @@ void intel_rps_init(struct intel_rps *rps)
 
 	/* Derive initial user preferences/limits from the hardware limits */
 	rps->max_freq_softlimit = rps->max_freq;
+	rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
 	rps->min_freq_softlimit = rps->min_freq;
+	rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
 
 	/* After setting max-softlimit, find the overclock max freq */
 	if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
diff --git a/drivers/gpu/drm/i915/gt/intel_sa_media.c b/drivers/gpu/drm/i915/gt/intel_sa_media.c
new file mode 100644
index 000000000000..e8f3d18c12b8
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/intel_sa_media.c
@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2021 Intel Corporation
+ */
+
+#include <drm/drm_managed.h>
+
+#include "i915_drv.h"
+#include "gt/intel_gt.h"
+#include "gt/intel_sa_media.h"
+
+int intel_sa_mediagt_setup(struct intel_gt *gt, phys_addr_t phys_addr,
+			   u32 gsi_offset)
+{
+	struct drm_i915_private *i915 = gt->i915;
+	struct intel_uncore *uncore;
+
+	uncore = drmm_kzalloc(&i915->drm, sizeof(*uncore), GFP_KERNEL);
+	if (!uncore)
+		return -ENOMEM;
+
+	uncore->gsi_offset = gsi_offset;
+
+	gt->irq_lock = to_gt(i915)->irq_lock;
+	intel_gt_common_init_early(gt);
+	intel_uncore_init_early(uncore, gt);
+
+	/*
+	 * Standalone media shares the general MMIO space with the primary
+	 * GT.  We'll re-use the primary GT's mapping.
+	 */
+	uncore->regs = i915->uncore.regs;
+	if (drm_WARN_ON(&i915->drm, uncore->regs == NULL))
+		return -EIO;
+
+	gt->uncore = uncore;
+	gt->phys_addr = phys_addr;
+
+	/*
+	 * For current platforms we can assume there's only a single
+	 * media GT and cache it for quick lookup.
+	 */
+	drm_WARN_ON(&i915->drm, i915->media_gt);
+	i915->media_gt = gt;
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/i915/gt/intel_sa_media.h b/drivers/gpu/drm/i915/gt/intel_sa_media.h
new file mode 100644
index 000000000000..3afb310de932
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/intel_sa_media.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2021 Intel Corporation
+ */
+#ifndef __INTEL_SA_MEDIA__
+#define __INTEL_SA_MEDIA__
+
+#include <linux/types.h>
+
+struct intel_gt;
+
+int intel_sa_mediagt_setup(struct intel_gt *gt, phys_addr_t phys_addr,
+			   u32 gsi_offset);
+
+#endif /* __INTEL_SA_MEDIA_H__ */
diff --git a/drivers/gpu/drm/i915/gt/intel_sseu.c b/drivers/gpu/drm/i915/gt/intel_sseu.c
index c6d3050604c8..66f21c735d54 100644
--- a/drivers/gpu/drm/i915/gt/intel_sseu.c
+++ b/drivers/gpu/drm/i915/gt/intel_sseu.c
@@ -382,7 +382,6 @@ static void cherryview_sseu_info_init(struct intel_gt *gt)
 static void gen9_sseu_info_init(struct intel_gt *gt)
 {
 	struct drm_i915_private *i915 = gt->i915;
-	struct intel_device_info *info = mkwrite_device_info(i915);
 	struct sseu_dev_info *sseu = &gt->info.sseu;
 	struct intel_uncore *uncore = gt->uncore;
 	u32 fuse2, eu_disable, subslice_mask;
@@ -471,10 +470,10 @@ static void gen9_sseu_info_init(struct intel_gt *gt)
 
 	if (IS_GEN9_LP(i915)) {
 #define IS_SS_DISABLED(ss)	(!(sseu->subslice_mask.hsw[0] & BIT(ss)))
-		info->has_pooled_eu = hweight8(sseu->subslice_mask.hsw[0]) == 3;
+		RUNTIME_INFO(i915)->has_pooled_eu = hweight8(sseu->subslice_mask.hsw[0]) == 3;
 
 		sseu->min_eu_in_pool = 0;
-		if (info->has_pooled_eu) {
+		if (HAS_POOLED_EU(i915)) {
 			if (IS_SS_DISABLED(2) || IS_SS_DISABLED(0))
 				sseu->min_eu_in_pool = 3;
 			else if (IS_SS_DISABLED(1))
diff --git a/drivers/gpu/drm/i915/gt/intel_workarounds.c b/drivers/gpu/drm/i915/gt/intel_workarounds.c
index e8111fce56d0..6d2003d598e6 100644
--- a/drivers/gpu/drm/i915/gt/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/gt/intel_workarounds.c
@@ -568,6 +568,7 @@ static void icl_ctx_workarounds_init(struct intel_engine_cs *engine,
 static void dg2_ctx_gt_tuning_init(struct intel_engine_cs *engine,
 				   struct i915_wa_list *wal)
 {
+	wa_masked_en(wal, CHICKEN_RASTER_2, TBIMR_FAST_CLIP);
 	wa_write_clr_set(wal, GEN11_L3SQCREG5, L3_PWM_TIMER_INIT_VAL_MASK,
 			 REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f));
 	wa_add(wal,
@@ -2102,13 +2103,6 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 		/* Wa_1509235366:dg2 */
 		wa_write_or(wal, GEN12_GAMCNTRL_CTRL, INVALIDATION_BROADCAST_MODE_DIS |
 			    GLOBAL_INVALIDATION_MODE);
-
-		/*
-		 * The following are not actually "workarounds" but rather
-		 * recommended tuning settings documented in the bspec's
-		 * performance guide section.
-		 */
-		wa_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS);
 	}
 
 	if (IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) {
@@ -2119,6 +2113,13 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 		wa_write_or(wal, LSC_CHICKEN_BIT_0_UDW, DIS_CHAIN_2XSIMD8);
 	}
 
+	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_FOREVER) ||
+	    IS_DG2_G11(i915) || IS_DG2_G12(i915)) {
+		/* Wa_1509727124:dg2 */
+		wa_masked_en(wal, GEN10_SAMPLER_MODE,
+			     SC_DISABLE_POWER_OPTIMIZATION_EBB);
+	}
+
 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0) ||
 	    IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0)) {
 		/* Wa_14012419201:dg2 */
@@ -2195,15 +2196,6 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 		wa_write_or(wal, XEHP_L3NODEARBCFG, XEHP_LNESPARE);
 	}
 
-	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_C0) ||
-	    IS_DG2_G11(i915)) {
-		/* Wa_22012654132:dg2 */
-		wa_add(wal, GEN10_CACHE_MODE_SS, 0,
-		       _MASKED_BIT_ENABLE(ENABLE_PREFETCH_INTO_IC),
-		       0 /* write-only, so skip validation */,
-		       true);
-	}
-
 	/* Wa_14013202645:dg2 */
 	if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_B0, STEP_C0) ||
 	    IS_DG2_GRAPHICS_STEP(i915, G11, STEP_A0, STEP_B0))
@@ -2397,7 +2389,7 @@ rcs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 			     FF_DOP_CLOCK_GATE_DISABLE);
 	}
 
-	if (HAS_PERCTX_PREEMPT_CTRL(i915)) {
+	if (IS_GRAPHICS_VER(i915, 9, 12)) {
 		/* FtrPerCtxtPreemptionGranularityControl:skl,bxt,kbl,cfl,cnl,icl,tgl */
 		wa_masked_en(wal,
 			     GEN7_FF_SLICE_CS_CHICKEN1,
@@ -2670,6 +2662,56 @@ ccs_engine_wa_init(struct intel_engine_cs *engine, struct i915_wa_list *wal)
 }
 
 /*
+ * The bspec performance guide has recommended MMIO tuning settings.  These
+ * aren't truly "workarounds" but we want to program them with the same
+ * workaround infrastructure to ensure that they're automatically added to
+ * the GuC save/restore lists, re-applied at the right times, and checked for
+ * any conflicting programming requested by real workarounds.
+ *
+ * Programming settings should be added here only if their registers are not
+ * part of an engine's register state context.  If a register is part of a
+ * context, then any tuning settings should be programmed in an appropriate
+ * function invoked by __intel_engine_init_ctx_wa().
+ */
+static void
+add_render_compute_tuning_settings(struct drm_i915_private *i915,
+				   struct i915_wa_list *wal)
+{
+	if (IS_PONTEVECCHIO(i915)) {
+		wa_write(wal, XEHPC_L3SCRUB,
+			 SCRUB_CL_DWNGRADE_SHARED | SCRUB_RATE_4B_PER_CLK);
+	}
+
+	if (IS_DG2(i915)) {
+		wa_write_or(wal, XEHP_L3SCQREG7, BLEND_FILL_CACHING_OPT_DIS);
+		wa_write_clr_set(wal, RT_CTRL, STACKID_CTRL, STACKID_CTRL_512);
+
+		/*
+		 * This is also listed as Wa_22012654132 for certain DG2
+		 * steppings, but the tuning setting programming is a superset
+		 * since it applies to all DG2 variants and steppings.
+		 *
+		 * Note that register 0xE420 is write-only and cannot be read
+		 * back for verification on DG2 (due to Wa_14012342262), so
+		 * we need to explicitly skip the readback.
+		 */
+		wa_add(wal, GEN10_CACHE_MODE_SS, 0,
+		       _MASKED_BIT_ENABLE(ENABLE_PREFETCH_INTO_IC),
+		       0 /* write-only, so skip validation */,
+		       true);
+	}
+
+	/*
+	 * This tuning setting proves beneficial only on ATS-M designs; the
+	 * default "age based" setting is optimal on regular DG2 and other
+	 * platforms.
+	 */
+	if (INTEL_INFO(i915)->tuning_thread_rr_after_dep)
+		wa_masked_field_set(wal, GEN9_ROW_CHICKEN4, THREAD_EX_ARB_MODE,
+				    THREAD_EX_ARB_MODE_RR_AFTER_DEP);
+}
+
+/*
  * The workarounds in this function apply to shared registers in
  * the general render reset domain that aren't tied to a
  * specific engine.  Since all render+compute engines get reset
@@ -2683,14 +2725,9 @@ general_render_compute_wa_init(struct intel_engine_cs *engine, struct i915_wa_li
 {
 	struct drm_i915_private *i915 = engine->i915;
 
-	if (IS_PONTEVECCHIO(i915)) {
-		/*
-		 * The following is not actually a "workaround" but rather
-		 * a recommended tuning setting documented in the bspec's
-		 * performance guide section.
-		 */
-		wa_write(wal, XEHPC_L3SCRUB, SCRUB_CL_DWNGRADE_SHARED | SCRUB_RATE_4B_PER_CLK);
+	add_render_compute_tuning_settings(i915, wal);
 
+	if (IS_PONTEVECCHIO(i915)) {
 		/* Wa_16016694945 */
 		wa_masked_en(wal, XEHPC_LNCFMISCCFGREG0, XEHPC_OVRLSCCC);
 	}
diff --git a/drivers/gpu/drm/i915/gt/selftest_execlists.c b/drivers/gpu/drm/i915/gt/selftest_execlists.c
index 09f8cd2d0e2c..1e08b2473b99 100644
--- a/drivers/gpu/drm/i915/gt/selftest_execlists.c
+++ b/drivers/gpu/drm/i915/gt/selftest_execlists.c
@@ -2077,7 +2077,7 @@ static int __cancel_active0(struct live_preempt_cancel *arg)
 		goto out;
 	}
 
-	intel_context_set_banned(rq->context);
+	intel_context_ban(rq->context, rq);
 	err = intel_engine_pulse(arg->engine);
 	if (err)
 		goto out;
@@ -2136,7 +2136,7 @@ static int __cancel_active1(struct live_preempt_cancel *arg)
 	if (err)
 		goto out;
 
-	intel_context_set_banned(rq[1]->context);
+	intel_context_ban(rq[1]->context, rq[1]);
 	err = intel_engine_pulse(arg->engine);
 	if (err)
 		goto out;
@@ -2219,7 +2219,7 @@ static int __cancel_queued(struct live_preempt_cancel *arg)
 	if (err)
 		goto out;
 
-	intel_context_set_banned(rq[2]->context);
+	intel_context_ban(rq[2]->context, rq[2]);
 	err = intel_engine_pulse(arg->engine);
 	if (err)
 		goto out;
@@ -2234,7 +2234,13 @@ static int __cancel_queued(struct live_preempt_cancel *arg)
 		goto out;
 	}
 
-	if (rq[1]->fence.error != 0) {
+	/*
+	 * The behavior between having semaphores and not is different. With
+	 * semaphores the subsequent request is on the hardware and not cancelled
+	 * while without the request is held in the driver and cancelled.
+	 */
+	if (intel_engine_has_semaphores(rq[1]->engine) &&
+	    rq[1]->fence.error != 0) {
 		pr_err("Normal inflight1 request did not complete\n");
 		err = -EINVAL;
 		goto out;
@@ -2282,7 +2288,7 @@ static int __cancel_hostile(struct live_preempt_cancel *arg)
 		goto out;
 	}
 
-	intel_context_set_banned(rq->context);
+	intel_context_ban(rq->context, rq);
 	err = intel_engine_pulse(arg->engine); /* force reset */
 	if (err)
 		goto out;
diff --git a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
index 6493265d5f64..7f3bb1d34dfb 100644
--- a/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
+++ b/drivers/gpu/drm/i915/gt/selftest_hangcheck.c
@@ -1302,13 +1302,15 @@ static int igt_reset_wait(void *arg)
 {
 	struct intel_gt *gt = arg;
 	struct i915_gpu_error *global = &gt->i915->gpu_error;
-	struct intel_engine_cs *engine = gt->engine[RCS0];
+	struct intel_engine_cs *engine;
 	struct i915_request *rq;
 	unsigned int reset_count;
 	struct hang h;
 	long timeout;
 	int err;
 
+	engine = intel_selftest_find_any_engine(gt);
+
 	if (!engine || !intel_engine_can_store_dword(engine))
 		return 0;
 
@@ -1432,7 +1434,7 @@ static int __igt_reset_evict_vma(struct intel_gt *gt,
 				 int (*fn)(void *),
 				 unsigned int flags)
 {
-	struct intel_engine_cs *engine = gt->engine[RCS0];
+	struct intel_engine_cs *engine;
 	struct drm_i915_gem_object *obj;
 	struct task_struct *tsk = NULL;
 	struct i915_request *rq;
@@ -1444,6 +1446,8 @@ static int __igt_reset_evict_vma(struct intel_gt *gt,
 	if (!gt->ggtt->num_fences && flags & EXEC_OBJECT_NEEDS_FENCE)
 		return 0;
 
+	engine = intel_selftest_find_any_engine(gt);
+
 	if (!engine || !intel_engine_can_store_dword(engine))
 		return 0;
 
@@ -1819,12 +1823,14 @@ static int igt_handle_error(void *arg)
 {
 	struct intel_gt *gt = arg;
 	struct i915_gpu_error *global = &gt->i915->gpu_error;
-	struct intel_engine_cs *engine = gt->engine[RCS0];
+	struct intel_engine_cs *engine;
 	struct hang h;
 	struct i915_request *rq;
 	struct i915_gpu_coredump *error;
 	int err;
 
+	engine = intel_selftest_find_any_engine(gt);
+
 	/* Check that we can issue a global GPU and engine reset */
 
 	if (!intel_has_reset_engine(gt))
diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c
index 1109088fe8f6..82d3f8058995 100644
--- a/drivers/gpu/drm/i915/gt/selftest_lrc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c
@@ -27,6 +27,9 @@
 #define NUM_GPR 16
 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
 
+#define LRI_HEADER MI_INSTR(0x22, 0)
+#define LRI_LENGTH_MASK GENMASK(7, 0)
+
 static struct i915_vma *create_scratch(struct intel_gt *gt)
 {
 	return __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
@@ -202,7 +205,7 @@ static int live_lrc_layout(void *arg)
 				continue;
 			}
 
-			if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
+			if ((lri & GENMASK(31, 23)) != LRI_HEADER) {
 				pr_err("%s: Expected LRI command at dword %d, found %08x\n",
 				       engine->name, dw, lri);
 				err = -EINVAL;
@@ -357,6 +360,11 @@ static int live_lrc_fixed(void *arg)
 				lrc_ring_cmd_buf_cctl(engine),
 				"RING_CMD_BUF_CCTL"
 			},
+			{
+				i915_mmio_reg_offset(RING_BB_OFFSET(engine->mmio_base)),
+				lrc_ring_bb_offset(engine),
+				"RING_BB_OFFSET"
+			},
 			{ },
 		}, *t;
 		u32 *hw;
@@ -987,18 +995,40 @@ store_context(struct intel_context *ce, struct i915_vma *scratch)
 	hw = defaults;
 	hw += LRC_STATE_OFFSET / sizeof(*hw);
 	do {
-		u32 len = hw[dw] & 0x7f;
+		u32 len = hw[dw] & LRI_LENGTH_MASK;
+
+		/*
+		 * Keep it simple, skip parsing complex commands
+		 *
+		 * At present, there are no more MI_LOAD_REGISTER_IMM
+		 * commands after the first 3D state command. Rather
+		 * than include a table (see i915_cmd_parser.c) of all
+		 * the possible commands and their instruction lengths
+		 * (or mask for variable length instructions), assume
+		 * we have gathered the complete list of registers and
+		 * bail out.
+		 */
+		if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT)
+			break;
 
 		if (hw[dw] == 0) {
 			dw++;
 			continue;
 		}
 
-		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
+		if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) {
+			/* Assume all other MI commands match LRI length mask */
 			dw += len + 2;
 			continue;
 		}
 
+		if (!len) {
+			pr_err("%s: invalid LRI found in context image\n",
+			       ce->engine->name);
+			igt_hexdump(defaults, PAGE_SIZE);
+			break;
+		}
+
 		dw++;
 		len = (len + 1) / 2;
 		while (len--) {
@@ -1150,18 +1180,29 @@ static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
 	hw = defaults;
 	hw += LRC_STATE_OFFSET / sizeof(*hw);
 	do {
-		u32 len = hw[dw] & 0x7f;
+		u32 len = hw[dw] & LRI_LENGTH_MASK;
+
+		/* For simplicity, break parsing at the first complex command */
+		if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT)
+			break;
 
 		if (hw[dw] == 0) {
 			dw++;
 			continue;
 		}
 
-		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
+		if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) {
 			dw += len + 2;
 			continue;
 		}
 
+		if (!len) {
+			pr_err("%s: invalid LRI found in context image\n",
+			       ce->engine->name);
+			igt_hexdump(defaults, PAGE_SIZE);
+			break;
+		}
+
 		dw++;
 		len = (len + 1) / 2;
 		*cs++ = MI_LOAD_REGISTER_IMM(len);
@@ -1292,18 +1333,29 @@ static int compare_isolation(struct intel_engine_cs *engine,
 	hw = defaults;
 	hw += LRC_STATE_OFFSET / sizeof(*hw);
 	do {
-		u32 len = hw[dw] & 0x7f;
+		u32 len = hw[dw] & LRI_LENGTH_MASK;
+
+		/* For simplicity, break parsing at the first complex command */
+		if ((hw[dw] >> INSTR_CLIENT_SHIFT) != INSTR_MI_CLIENT)
+			break;
 
 		if (hw[dw] == 0) {
 			dw++;
 			continue;
 		}
 
-		if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
+		if ((hw[dw] & GENMASK(31, 23)) != LRI_HEADER) {
 			dw += len + 2;
 			continue;
 		}
 
+		if (!len) {
+			pr_err("%s: invalid LRI found in context image\n",
+			       engine->name);
+			igt_hexdump(defaults, PAGE_SIZE);
+			break;
+		}
+
 		dw++;
 		len = (len + 1) / 2;
 		while (len--) {
@@ -1343,6 +1395,30 @@ err_A0:
 	return err;
 }
 
+static struct i915_vma *
+create_result_vma(struct i915_address_space *vm, unsigned long sz)
+{
+	struct i915_vma *vma;
+	void *ptr;
+
+	vma = create_user_vma(vm, sz);
+	if (IS_ERR(vma))
+		return vma;
+
+	/* Set the results to a known value distinct from the poison */
+	ptr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
+	if (IS_ERR(ptr)) {
+		i915_vma_put(vma);
+		return ERR_CAST(ptr);
+	}
+
+	memset(ptr, POISON_INUSE, vma->size);
+	i915_gem_object_flush_map(vma->obj);
+	i915_gem_object_unpin_map(vma->obj);
+
+	return vma;
+}
+
 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
 {
 	u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
@@ -1361,13 +1437,13 @@ static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
 		goto err_A;
 	}
 
-	ref[0] = create_user_vma(A->vm, SZ_64K);
+	ref[0] = create_result_vma(A->vm, SZ_64K);
 	if (IS_ERR(ref[0])) {
 		err = PTR_ERR(ref[0]);
 		goto err_B;
 	}
 
-	ref[1] = create_user_vma(A->vm, SZ_64K);
+	ref[1] = create_result_vma(A->vm, SZ_64K);
 	if (IS_ERR(ref[1])) {
 		err = PTR_ERR(ref[1]);
 		goto err_ref0;
@@ -1389,13 +1465,13 @@ static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
 	}
 	i915_request_put(rq);
 
-	result[0] = create_user_vma(A->vm, SZ_64K);
+	result[0] = create_result_vma(A->vm, SZ_64K);
 	if (IS_ERR(result[0])) {
 		err = PTR_ERR(result[0]);
 		goto err_ref1;
 	}
 
-	result[1] = create_user_vma(A->vm, SZ_64K);
+	result[1] = create_result_vma(A->vm, SZ_64K);
 	if (IS_ERR(result[1])) {
 		err = PTR_ERR(result[1]);
 		goto err_result0;
@@ -1408,18 +1484,17 @@ static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
 	}
 
 	err = poison_registers(B, poison, sema);
-	if (err) {
-		WRITE_ONCE(*sema, -1);
-		i915_request_put(rq);
-		goto err_result1;
-	}
-
-	if (i915_request_wait(rq, 0, HZ / 2) < 0) {
-		i915_request_put(rq);
+	if (err == 0 && i915_request_wait(rq, 0, HZ / 2) < 0) {
+		pr_err("%s(%s): wait for results timed out\n",
+		       __func__, engine->name);
 		err = -ETIME;
-		goto err_result1;
 	}
+
+	/* Always cancel the semaphore wait, just in case the GPU gets stuck */
+	WRITE_ONCE(*sema, -1);
 	i915_request_put(rq);
+	if (err)
+		goto err_result1;
 
 	err = compare_isolation(engine, ref, result, A, poison);
 
diff --git a/drivers/gpu/drm/i915/gt/selftest_slpc.c b/drivers/gpu/drm/i915/gt/selftest_slpc.c
index ac29691e0b1a..f8a1d27df272 100644
--- a/drivers/gpu/drm/i915/gt/selftest_slpc.c
+++ b/drivers/gpu/drm/i915/gt/selftest_slpc.c
@@ -166,6 +166,15 @@ static int run_test(struct intel_gt *gt, int test_type)
 		return -EIO;
 	}
 
+	/*
+	 * FIXME: With efficient frequency enabled, GuC can request
+	 * frequencies higher than the SLPC max. While this is fixed
+	 * in GuC, we level set these tests with RPn as min.
+	 */
+	err = slpc_set_min_freq(slpc, slpc->min_freq);
+	if (err)
+		return err;
+
 	if (slpc->min_freq == slpc->rp0_freq) {
 		pr_err("Min/Max are fused to the same value\n");
 		return -EINVAL;
diff --git a/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h b/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h
index df83c1cc7c7a..28b8387f97b7 100644
--- a/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h
+++ b/drivers/gpu/drm/i915/gt/uc/abi/guc_communication_ctb_abi.h
@@ -37,6 +37,7 @@
  *  |   |       |   - _`GUC_CTB_STATUS_OVERFLOW` = 1 (head/tail too large)     |
  *  |   |       |   - _`GUC_CTB_STATUS_UNDERFLOW` = 2 (truncated message)      |
  *  |   |       |   - _`GUC_CTB_STATUS_MISMATCH` = 4 (head/tail modified)      |
+ *  |   |       |   - _`GUC_CTB_STATUS_UNUSED` = 8 (CTB is not in use)         |
  *  +---+-------+--------------------------------------------------------------+
  *  |...|       | RESERVED = MBZ                                               |
  *  +---+-------+--------------------------------------------------------------+
@@ -49,9 +50,10 @@ struct guc_ct_buffer_desc {
 	u32 tail;
 	u32 status;
 #define GUC_CTB_STATUS_NO_ERROR				0
-#define GUC_CTB_STATUS_OVERFLOW				(1 << 0)
-#define GUC_CTB_STATUS_UNDERFLOW			(1 << 1)
-#define GUC_CTB_STATUS_MISMATCH				(1 << 2)
+#define GUC_CTB_STATUS_OVERFLOW				BIT(0)
+#define GUC_CTB_STATUS_UNDERFLOW			BIT(1)
+#define GUC_CTB_STATUS_MISMATCH				BIT(2)
+#define GUC_CTB_STATUS_UNUSED				BIT(3)
 	u32 reserved[13];
 } __packed;
 static_assert(sizeof(struct guc_ct_buffer_desc) == 64);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc.c
index 2706a8c65090..bac06e3d6f2c 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.c
@@ -82,9 +82,9 @@ static void gen9_reset_guc_interrupts(struct intel_guc *guc)
 
 	assert_rpm_wakelock_held(&gt->i915->runtime_pm);
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	gen6_gt_pm_reset_iir(gt, gt->pm_guc_events);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 }
 
 static void gen9_enable_guc_interrupts(struct intel_guc *guc)
@@ -93,11 +93,11 @@ static void gen9_enable_guc_interrupts(struct intel_guc *guc)
 
 	assert_rpm_wakelock_held(&gt->i915->runtime_pm);
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	WARN_ON_ONCE(intel_uncore_read(gt->uncore, GEN8_GT_IIR(2)) &
 		     gt->pm_guc_events);
 	gen6_gt_pm_enable_irq(gt, gt->pm_guc_events);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 }
 
 static void gen9_disable_guc_interrupts(struct intel_guc *guc)
@@ -106,11 +106,11 @@ static void gen9_disable_guc_interrupts(struct intel_guc *guc)
 
 	assert_rpm_wakelock_held(&gt->i915->runtime_pm);
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 
 	gen6_gt_pm_disable_irq(gt, gt->pm_guc_events);
 
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 	intel_synchronize_irq(gt->i915);
 
 	gen9_reset_guc_interrupts(guc);
@@ -120,9 +120,9 @@ static void gen11_reset_guc_interrupts(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	gen11_gt_reset_one_iir(gt, 0, GEN11_GUC);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 }
 
 static void gen11_enable_guc_interrupts(struct intel_guc *guc)
@@ -130,25 +130,25 @@ static void gen11_enable_guc_interrupts(struct intel_guc *guc)
 	struct intel_gt *gt = guc_to_gt(guc);
 	u32 events = REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST);
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_GUC));
 	intel_uncore_write(gt->uncore,
 			   GEN11_GUC_SG_INTR_ENABLE, events);
 	intel_uncore_write(gt->uncore,
 			   GEN11_GUC_SG_INTR_MASK, ~events);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 }
 
 static void gen11_disable_guc_interrupts(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 
 	intel_uncore_write(gt->uncore, GEN11_GUC_SG_INTR_MASK, ~0);
 	intel_uncore_write(gt->uncore, GEN11_GUC_SG_INTR_ENABLE, 0);
 
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 	intel_synchronize_irq(gt->i915);
 
 	gen11_reset_guc_interrupts(guc);
@@ -224,53 +224,22 @@ static u32 guc_ctl_feature_flags(struct intel_guc *guc)
 
 static u32 guc_ctl_log_params_flags(struct intel_guc *guc)
 {
-	u32 offset = intel_guc_ggtt_offset(guc, guc->log.vma) >> PAGE_SHIFT;
-	u32 flags;
-
-	#if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0)
-	#define LOG_UNIT SZ_1M
-	#define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS
-	#else
-	#define LOG_UNIT SZ_4K
-	#define LOG_FLAG 0
-	#endif
-
-	#if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0)
-	#define CAPTURE_UNIT SZ_1M
-	#define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS
-	#else
-	#define CAPTURE_UNIT SZ_4K
-	#define CAPTURE_FLAG 0
-	#endif
-
-	BUILD_BUG_ON(!CRASH_BUFFER_SIZE);
-	BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT));
-	BUILD_BUG_ON(!DEBUG_BUFFER_SIZE);
-	BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT));
-	BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE);
-	BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT));
-
-	BUILD_BUG_ON((CRASH_BUFFER_SIZE / LOG_UNIT - 1) >
-			(GUC_LOG_CRASH_MASK >> GUC_LOG_CRASH_SHIFT));
-	BUILD_BUG_ON((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) >
-			(GUC_LOG_DEBUG_MASK >> GUC_LOG_DEBUG_SHIFT));
-	BUILD_BUG_ON((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) >
-			(GUC_LOG_CAPTURE_MASK >> GUC_LOG_CAPTURE_SHIFT));
+	struct intel_guc_log *log = &guc->log;
+	u32 offset, flags;
+
+	GEM_BUG_ON(!log->sizes_initialised);
+
+	offset = intel_guc_ggtt_offset(guc, log->vma) >> PAGE_SHIFT;
 
 	flags = GUC_LOG_VALID |
 		GUC_LOG_NOTIFY_ON_HALF_FULL |
-		CAPTURE_FLAG |
-		LOG_FLAG |
-		((CRASH_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_CRASH_SHIFT) |
-		((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_DEBUG_SHIFT) |
-		((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) << GUC_LOG_CAPTURE_SHIFT) |
+		log->sizes[GUC_LOG_SECTIONS_DEBUG].flag |
+		log->sizes[GUC_LOG_SECTIONS_CAPTURE].flag |
+		(log->sizes[GUC_LOG_SECTIONS_CRASH].count << GUC_LOG_CRASH_SHIFT) |
+		(log->sizes[GUC_LOG_SECTIONS_DEBUG].count << GUC_LOG_DEBUG_SHIFT) |
+		(log->sizes[GUC_LOG_SECTIONS_CAPTURE].count << GUC_LOG_CAPTURE_SHIFT) |
 		(offset << GUC_LOG_BUF_ADDR_SHIFT);
 
-	#undef LOG_UNIT
-	#undef LOG_FLAG
-	#undef CAPTURE_UNIT
-	#undef CAPTURE_FLAG
-
 	return flags;
 }
 
@@ -389,6 +358,23 @@ void intel_guc_write_params(struct intel_guc *guc)
 	intel_uncore_forcewake_put(uncore, FORCEWAKE_GT);
 }
 
+void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p)
+{
+	struct intel_gt *gt = guc_to_gt(guc);
+	intel_wakeref_t wakeref;
+	u32 stamp = 0;
+	u64 ktime;
+
+	with_intel_runtime_pm(&gt->i915->runtime_pm, wakeref)
+		stamp = intel_uncore_read(gt->uncore, GUCPMTIMESTAMP);
+	ktime = ktime_get_boottime_ns();
+
+	drm_printf(p, "Kernel timestamp: 0x%08llX [%llu]\n", ktime, ktime);
+	drm_printf(p, "GuC timestamp: 0x%08X [%u]\n", stamp, stamp);
+	drm_printf(p, "CS timestamp frequency: %u Hz, %u ns\n",
+		   gt->clock_frequency, gt->clock_period_ns);
+}
+
 int intel_guc_init(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc.h b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
index a7acffbf15d1..804133df1ac9 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc.h
@@ -464,4 +464,6 @@ void intel_guc_load_status(struct intel_guc *guc, struct drm_printer *p);
 
 void intel_guc_write_barrier(struct intel_guc *guc);
 
+void intel_guc_dump_time_info(struct intel_guc *guc, struct drm_printer *p);
+
 #endif
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c
index ba7541f3ca61..74cbe8eaf531 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ads.c
@@ -464,7 +464,11 @@ static void fill_engine_enable_masks(struct intel_gt *gt,
 }
 
 #define LR_HW_CONTEXT_SIZE (80 * sizeof(u32))
-#define LRC_SKIP_SIZE (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SIZE)
+#define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32))
+#define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50) ? \
+				    XEHP_LR_HW_CONTEXT_SIZE : \
+				    LR_HW_CONTEXT_SIZE)
+#define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915))
 static int guc_prep_golden_context(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
@@ -525,7 +529,7 @@ static int guc_prep_golden_context(struct intel_guc *guc)
 		 * on all engines).
 		 */
 		ads_blob_write(guc, ads.eng_state_size[guc_class],
-			       real_size - LRC_SKIP_SIZE);
+			       real_size - LRC_SKIP_SIZE(gt->i915));
 		ads_blob_write(guc, ads.golden_context_lrca[guc_class],
 			       addr_ggtt);
 
@@ -599,7 +603,7 @@ static void guc_init_golden_context(struct intel_guc *guc)
 		}
 
 		GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) !=
-			   real_size - LRC_SKIP_SIZE);
+			   real_size - LRC_SKIP_SIZE(gt->i915));
 		GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt);
 
 		addr_ggtt += alloc_size;
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c
index 75257bd20ff0..8f1165146013 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.c
@@ -600,10 +600,8 @@ intel_guc_capture_getnullheader(struct intel_guc *guc,
 	return 0;
 }
 
-#define GUC_CAPTURE_OVERBUFFER_MULTIPLIER 3
-
-int
-intel_guc_capture_output_min_size_est(struct intel_guc *guc)
+static int
+guc_capture_output_min_size_est(struct intel_guc *guc)
 {
 	struct intel_gt *gt = guc_to_gt(guc);
 	struct intel_engine_cs *engine;
@@ -623,13 +621,8 @@ intel_guc_capture_output_min_size_est(struct intel_guc *guc)
 	 * For each engine instance, there would be 1 x guc_state_capture_group_t output
 	 * followed by 3 x guc_state_capture_t lists. The latter is how the register
 	 * dumps are split across different register types (where the '3' are global vs class
-	 * vs instance). Finally, let's multiply the whole thing by 3x (just so we are
-	 * not limited to just 1 round of data in a worst case full register dump log)
-	 *
-	 * NOTE: intel_guc_log that allocates the log buffer would round this size up to
-	 * a power of two.
+	 * vs instance).
 	 */
-
 	for_each_engine(engine, gt, id) {
 		worst_min_size += sizeof(struct guc_state_capture_group_header_t) +
 					 (3 * sizeof(struct guc_state_capture_header_t));
@@ -649,7 +642,31 @@ intel_guc_capture_output_min_size_est(struct intel_guc *guc)
 
 	worst_min_size += (num_regs * sizeof(struct guc_mmio_reg));
 
-	return (worst_min_size * GUC_CAPTURE_OVERBUFFER_MULTIPLIER);
+	return worst_min_size;
+}
+
+/*
+ * Add on a 3x multiplier to allow for multiple back-to-back captures occurring
+ * before the i915 can read the data out and process it
+ */
+#define GUC_CAPTURE_OVERBUFFER_MULTIPLIER 3
+
+static void check_guc_capture_size(struct intel_guc *guc)
+{
+	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
+	int min_size = guc_capture_output_min_size_est(guc);
+	int spare_size = min_size * GUC_CAPTURE_OVERBUFFER_MULTIPLIER;
+	u32 buffer_size = intel_guc_log_section_size_capture(&guc->log);
+
+	if (min_size < 0)
+		drm_warn(&i915->drm, "Failed to calculate GuC error state capture buffer minimum size: %d!\n",
+			 min_size);
+	else if (min_size > buffer_size)
+		drm_warn(&i915->drm, "GuC error state capture buffer is too small: %d < %d\n",
+			 buffer_size, min_size);
+	else if (spare_size > buffer_size)
+		drm_notice(&i915->drm, "GuC error state capture buffer maybe too small: %d < %d (min = %d)\n",
+			   buffer_size, spare_size, min_size);
 }
 
 /*
@@ -1278,7 +1295,8 @@ static void __guc_capture_process_output(struct intel_guc *guc)
 
 	log_buf_state = guc->log.buf_addr +
 			(sizeof(struct guc_log_buffer_state) * GUC_CAPTURE_LOG_BUFFER);
-	src_data = guc->log.buf_addr + intel_guc_get_log_buffer_offset(GUC_CAPTURE_LOG_BUFFER);
+	src_data = guc->log.buf_addr +
+		   intel_guc_get_log_buffer_offset(&guc->log, GUC_CAPTURE_LOG_BUFFER);
 
 	/*
 	 * Make a copy of the state structure, inside GuC log buffer
@@ -1286,7 +1304,7 @@ static void __guc_capture_process_output(struct intel_guc *guc)
 	 * from it multiple times.
 	 */
 	memcpy(&log_buf_state_local, log_buf_state, sizeof(struct guc_log_buffer_state));
-	buffer_size = intel_guc_get_log_buffer_size(GUC_CAPTURE_LOG_BUFFER);
+	buffer_size = intel_guc_get_log_buffer_size(&guc->log, GUC_CAPTURE_LOG_BUFFER);
 	read_offset = log_buf_state_local.read_ptr;
 	write_offset = log_buf_state_local.sampled_write_ptr;
 	full_count = log_buf_state_local.buffer_full_cnt;
@@ -1365,33 +1383,22 @@ guc_capture_reg_to_str(const struct intel_guc *guc, u32 owner, u32 type,
 	return NULL;
 }
 
-#ifdef CONFIG_DRM_I915_DEBUG_GUC
-#define __out(a, ...) \
-	do { \
-		drm_warn((&(a)->i915->drm), __VA_ARGS__); \
-		i915_error_printf((a), __VA_ARGS__); \
-	} while (0)
-#else
-#define __out(a, ...) \
-	i915_error_printf(a, __VA_ARGS__)
-#endif
-
 #define GCAP_PRINT_INTEL_ENG_INFO(ebuf, eng) \
 	do { \
-		__out(ebuf, "    i915-Eng-Name: %s command stream\n", \
-		      (eng)->name); \
-		__out(ebuf, "    i915-Eng-Inst-Class: 0x%02x\n", (eng)->class); \
-		__out(ebuf, "    i915-Eng-Inst-Id: 0x%02x\n", (eng)->instance); \
-		__out(ebuf, "    i915-Eng-LogicalMask: 0x%08x\n", \
-		      (eng)->logical_mask); \
+		i915_error_printf(ebuf, "    i915-Eng-Name: %s command stream\n", \
+				  (eng)->name); \
+		i915_error_printf(ebuf, "    i915-Eng-Inst-Class: 0x%02x\n", (eng)->class); \
+		i915_error_printf(ebuf, "    i915-Eng-Inst-Id: 0x%02x\n", (eng)->instance); \
+		i915_error_printf(ebuf, "    i915-Eng-LogicalMask: 0x%08x\n", \
+				  (eng)->logical_mask); \
 	} while (0)
 
 #define GCAP_PRINT_GUC_INST_INFO(ebuf, node) \
 	do { \
-		__out(ebuf, "    GuC-Engine-Inst-Id: 0x%08x\n", \
-		      (node)->eng_inst); \
-		__out(ebuf, "    GuC-Context-Id: 0x%08x\n", (node)->guc_id); \
-		__out(ebuf, "    LRCA: 0x%08x\n", (node)->lrca); \
+		i915_error_printf(ebuf, "    GuC-Engine-Inst-Id: 0x%08x\n", \
+				  (node)->eng_inst); \
+		i915_error_printf(ebuf, "    GuC-Context-Id: 0x%08x\n", (node)->guc_id); \
+		i915_error_printf(ebuf, "    LRCA: 0x%08x\n", (node)->lrca); \
 	} while (0)
 
 int intel_guc_capture_print_engine_node(struct drm_i915_error_state_buf *ebuf,
@@ -1423,57 +1430,57 @@ int intel_guc_capture_print_engine_node(struct drm_i915_error_state_buf *ebuf,
 
 	guc = &ee->engine->gt->uc.guc;
 
-	__out(ebuf, "global --- GuC Error Capture on %s command stream:\n",
-	      ee->engine->name);
+	i915_error_printf(ebuf, "global --- GuC Error Capture on %s command stream:\n",
+			  ee->engine->name);
 
 	node = ee->guc_capture_node;
 	if (!node) {
-		__out(ebuf, "  No matching ee-node\n");
+		i915_error_printf(ebuf, "  No matching ee-node\n");
 		return 0;
 	}
 
-	__out(ebuf, "Coverage:  %s\n", grptype[node->is_partial]);
+	i915_error_printf(ebuf, "Coverage:  %s\n", grptype[node->is_partial]);
 
 	for (i = GUC_CAPTURE_LIST_TYPE_GLOBAL; i < GUC_CAPTURE_LIST_TYPE_MAX; ++i) {
-		__out(ebuf, "  RegListType: %s\n",
-		      datatype[i % GUC_CAPTURE_LIST_TYPE_MAX]);
-		__out(ebuf, "    Owner-Id: %d\n", node->reginfo[i].vfid);
+		i915_error_printf(ebuf, "  RegListType: %s\n",
+				  datatype[i % GUC_CAPTURE_LIST_TYPE_MAX]);
+		i915_error_printf(ebuf, "    Owner-Id: %d\n", node->reginfo[i].vfid);
 
 		switch (i) {
 		case GUC_CAPTURE_LIST_TYPE_GLOBAL:
 		default:
 			break;
 		case GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS:
-			__out(ebuf, "    GuC-Eng-Class: %d\n", node->eng_class);
-			__out(ebuf, "    i915-Eng-Class: %d\n",
-			      guc_class_to_engine_class(node->eng_class));
+			i915_error_printf(ebuf, "    GuC-Eng-Class: %d\n", node->eng_class);
+			i915_error_printf(ebuf, "    i915-Eng-Class: %d\n",
+					  guc_class_to_engine_class(node->eng_class));
 			break;
 		case GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE:
 			eng = intel_guc_lookup_engine(guc, node->eng_class, node->eng_inst);
 			if (eng)
 				GCAP_PRINT_INTEL_ENG_INFO(ebuf, eng);
 			else
-				__out(ebuf, "    i915-Eng-Lookup Fail!\n");
+				i915_error_printf(ebuf, "    i915-Eng-Lookup Fail!\n");
 			GCAP_PRINT_GUC_INST_INFO(ebuf, node);
 			break;
 		}
 
 		numregs = node->reginfo[i].num_regs;
-		__out(ebuf, "    NumRegs: %d\n", numregs);
+		i915_error_printf(ebuf, "    NumRegs: %d\n", numregs);
 		j = 0;
 		while (numregs--) {
 			regs = node->reginfo[i].regs;
 			str = guc_capture_reg_to_str(guc, GUC_CAPTURE_LIST_INDEX_PF, i,
 						     node->eng_class, 0, regs[j].offset, &is_ext);
 			if (!str)
-				__out(ebuf, "      REG-0x%08x", regs[j].offset);
+				i915_error_printf(ebuf, "      REG-0x%08x", regs[j].offset);
 			else
-				__out(ebuf, "      %s", str);
+				i915_error_printf(ebuf, "      %s", str);
 			if (is_ext)
-				__out(ebuf, "[%ld][%ld]",
-				      FIELD_GET(GUC_REGSET_STEERING_GROUP, regs[j].flags),
-				      FIELD_GET(GUC_REGSET_STEERING_INSTANCE, regs[j].flags));
-			__out(ebuf, ":  0x%08x\n", regs[j].value);
+				i915_error_printf(ebuf, "[%ld][%ld]",
+					FIELD_GET(GUC_REGSET_STEERING_GROUP, regs[j].flags),
+					FIELD_GET(GUC_REGSET_STEERING_INSTANCE, regs[j].flags));
+			i915_error_printf(ebuf, ":  0x%08x\n", regs[j].value);
 			++j;
 		}
 	}
@@ -1580,5 +1587,7 @@ int intel_guc_capture_init(struct intel_guc *guc)
 	INIT_LIST_HEAD(&guc->capture->outlist);
 	INIT_LIST_HEAD(&guc->capture->cachelist);
 
+	check_guc_capture_size(guc);
+
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.h
index d3d7bd0b6db6..fbd3713c7832 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_capture.h
@@ -21,7 +21,6 @@ int intel_guc_capture_print_engine_node(struct drm_i915_error_state_buf *m,
 void intel_guc_capture_get_matching_node(struct intel_gt *gt, struct intel_engine_coredump *ee,
 					 struct intel_context *ce);
 void intel_guc_capture_process(struct intel_guc *guc);
-int intel_guc_capture_output_min_size_est(struct intel_guc *guc);
 int intel_guc_capture_getlist(struct intel_guc *guc, u32 owner, u32 type, u32 classid,
 			      void **outptr);
 int intel_guc_capture_getlistsize(struct intel_guc *guc, u32 owner, u32 type, u32 classid,
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
index f01325cd1b62..2b22065e87bf 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_ct.c
@@ -455,6 +455,7 @@ corrupted:
 
 /**
  * wait_for_ct_request_update - Wait for CT request state update.
+ * @ct:		pointer to CT
  * @req:	pointer to pending request
  * @status:	placeholder for status
  *
@@ -467,9 +468,10 @@ corrupted:
  * *	0 response received (status is valid)
  * *	-ETIMEDOUT no response within hardcoded timeout
  */
-static int wait_for_ct_request_update(struct ct_request *req, u32 *status)
+static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status)
 {
 	int err;
+	bool ct_enabled;
 
 	/*
 	 * Fast commands should complete in less than 10us, so sample quickly
@@ -481,12 +483,15 @@ static int wait_for_ct_request_update(struct ct_request *req, u32 *status)
 #define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10
 #define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000
 #define done \
-	(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \
+	(!(ct_enabled = intel_guc_ct_enabled(ct)) || \
+	 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \
 	 GUC_HXG_ORIGIN_GUC)
 	err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS);
 	if (err)
 		err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS);
 #undef done
+	if (!ct_enabled)
+		err = -ENODEV;
 
 	*status = req->status;
 	return err;
@@ -703,11 +708,18 @@ retry:
 
 	intel_guc_notify(ct_to_guc(ct));
 
-	err = wait_for_ct_request_update(&request, status);
+	err = wait_for_ct_request_update(ct, &request, status);
 	g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
 	if (unlikely(err)) {
-		CT_ERROR(ct, "No response for request %#x (fence %u)\n",
-			 action[0], request.fence);
+		if (err == -ENODEV)
+			/* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress.
+			 * In this case, output is debug rather than error info
+			 */
+			CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n",
+				 action[0], request.fence);
+		else
+			CT_ERROR(ct, "No response for request %#x (fence %u)\n",
+				 action[0], request.fence);
 		goto unlink;
 	}
 
@@ -771,8 +783,9 @@ int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len,
 
 	ret = ct_send(ct, action, len, response_buf, response_buf_size, &status);
 	if (unlikely(ret < 0)) {
-		CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n",
-			 action[0], ERR_PTR(ret), status);
+		if (ret != -ENODEV)
+			CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n",
+				 action[0], ERR_PTR(ret), status);
 	} else if (unlikely(ret)) {
 		CT_DEBUG(ct, "send action %#x returned %d (%#x)\n",
 			 action[0], ret, ret);
@@ -816,8 +829,22 @@ static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg)
 	if (unlikely(ctb->broken))
 		return -EPIPE;
 
-	if (unlikely(desc->status))
-		goto corrupted;
+	if (unlikely(desc->status)) {
+		u32 status = desc->status;
+
+		if (status & GUC_CTB_STATUS_UNUSED) {
+			/*
+			 * Potentially valid if a CLIENT_RESET request resulted in
+			 * contexts/engines being reset. But should never happen as
+			 * no contexts should be active when CLIENT_RESET is sent.
+			 */
+			CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n");
+			status &= ~GUC_CTB_STATUS_UNUSED;
+		}
+
+		if (status)
+			goto corrupted;
+	}
 
 	GEM_BUG_ON(head > size);
 
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c
index 25b2d7ce6640..55d3ef93e86f 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.c
@@ -13,8 +13,163 @@
 #include "intel_guc_capture.h"
 #include "intel_guc_log.h"
 
+#if defined(CONFIG_DRM_I915_DEBUG_GUC)
+#define GUC_LOG_DEFAULT_CRASH_BUFFER_SIZE	SZ_2M
+#define GUC_LOG_DEFAULT_DEBUG_BUFFER_SIZE	SZ_16M
+#define GUC_LOG_DEFAULT_CAPTURE_BUFFER_SIZE	SZ_4M
+#elif defined(CONFIG_DRM_I915_DEBUG_GEM)
+#define GUC_LOG_DEFAULT_CRASH_BUFFER_SIZE	SZ_1M
+#define GUC_LOG_DEFAULT_DEBUG_BUFFER_SIZE	SZ_2M
+#define GUC_LOG_DEFAULT_CAPTURE_BUFFER_SIZE	SZ_4M
+#else
+#define GUC_LOG_DEFAULT_CRASH_BUFFER_SIZE	SZ_8K
+#define GUC_LOG_DEFAULT_DEBUG_BUFFER_SIZE	SZ_64K
+#define GUC_LOG_DEFAULT_CAPTURE_BUFFER_SIZE	SZ_2M
+#endif
+
 static void guc_log_copy_debuglogs_for_relay(struct intel_guc_log *log);
 
+struct guc_log_section {
+	u32 max;
+	u32 flag;
+	u32 default_val;
+	const char *name;
+};
+
+static void _guc_log_init_sizes(struct intel_guc_log *log)
+{
+	struct intel_guc *guc = log_to_guc(log);
+	struct drm_i915_private *i915 = guc_to_gt(guc)->i915;
+	static const struct guc_log_section sections[GUC_LOG_SECTIONS_LIMIT] = {
+		{
+			GUC_LOG_CRASH_MASK >> GUC_LOG_CRASH_SHIFT,
+			GUC_LOG_LOG_ALLOC_UNITS,
+			GUC_LOG_DEFAULT_CRASH_BUFFER_SIZE,
+			"crash dump"
+		},
+		{
+			GUC_LOG_DEBUG_MASK >> GUC_LOG_DEBUG_SHIFT,
+			GUC_LOG_LOG_ALLOC_UNITS,
+			GUC_LOG_DEFAULT_DEBUG_BUFFER_SIZE,
+			"debug",
+		},
+		{
+			GUC_LOG_CAPTURE_MASK >> GUC_LOG_CAPTURE_SHIFT,
+			GUC_LOG_CAPTURE_ALLOC_UNITS,
+			GUC_LOG_DEFAULT_CAPTURE_BUFFER_SIZE,
+			"capture",
+		}
+	};
+	int i;
+
+	for (i = 0; i < GUC_LOG_SECTIONS_LIMIT; i++)
+		log->sizes[i].bytes = sections[i].default_val;
+
+	/* If debug size > 1MB then bump default crash size to keep the same units */
+	if (log->sizes[GUC_LOG_SECTIONS_DEBUG].bytes >= SZ_1M &&
+	    GUC_LOG_DEFAULT_CRASH_BUFFER_SIZE < SZ_1M)
+		log->sizes[GUC_LOG_SECTIONS_CRASH].bytes = SZ_1M;
+
+	/* Prepare the GuC API structure fields: */
+	for (i = 0; i < GUC_LOG_SECTIONS_LIMIT; i++) {
+		/* Convert to correct units */
+		if ((log->sizes[i].bytes % SZ_1M) == 0) {
+			log->sizes[i].units = SZ_1M;
+			log->sizes[i].flag = sections[i].flag;
+		} else {
+			log->sizes[i].units = SZ_4K;
+			log->sizes[i].flag = 0;
+		}
+
+		if (!IS_ALIGNED(log->sizes[i].bytes, log->sizes[i].units))
+			drm_err(&i915->drm, "Mis-aligned GuC log %s size: 0x%X vs 0x%X!",
+				sections[i].name, log->sizes[i].bytes, log->sizes[i].units);
+		log->sizes[i].count = log->sizes[i].bytes / log->sizes[i].units;
+
+		if (!log->sizes[i].count) {
+			drm_err(&i915->drm, "Zero GuC log %s size!", sections[i].name);
+		} else {
+			/* Size is +1 unit */
+			log->sizes[i].count--;
+		}
+
+		/* Clip to field size */
+		if (log->sizes[i].count > sections[i].max) {
+			drm_err(&i915->drm, "GuC log %s size too large: %d vs %d!",
+				sections[i].name, log->sizes[i].count + 1, sections[i].max + 1);
+			log->sizes[i].count = sections[i].max;
+		}
+	}
+
+	if (log->sizes[GUC_LOG_SECTIONS_CRASH].units != log->sizes[GUC_LOG_SECTIONS_DEBUG].units) {
+		drm_err(&i915->drm, "Unit mis-match for GuC log crash and debug sections: %d vs %d!",
+			log->sizes[GUC_LOG_SECTIONS_CRASH].units,
+			log->sizes[GUC_LOG_SECTIONS_DEBUG].units);
+		log->sizes[GUC_LOG_SECTIONS_CRASH].units = log->sizes[GUC_LOG_SECTIONS_DEBUG].units;
+		log->sizes[GUC_LOG_SECTIONS_CRASH].count = 0;
+	}
+
+	log->sizes_initialised = true;
+}
+
+static void guc_log_init_sizes(struct intel_guc_log *log)
+{
+	if (log->sizes_initialised)
+		return;
+
+	_guc_log_init_sizes(log);
+}
+
+static u32 intel_guc_log_section_size_crash(struct intel_guc_log *log)
+{
+	guc_log_init_sizes(log);
+
+	return log->sizes[GUC_LOG_SECTIONS_CRASH].bytes;
+}
+
+static u32 intel_guc_log_section_size_debug(struct intel_guc_log *log)
+{
+	guc_log_init_sizes(log);
+
+	return log->sizes[GUC_LOG_SECTIONS_DEBUG].bytes;
+}
+
+u32 intel_guc_log_section_size_capture(struct intel_guc_log *log)
+{
+	guc_log_init_sizes(log);
+
+	return log->sizes[GUC_LOG_SECTIONS_CAPTURE].bytes;
+}
+
+static u32 intel_guc_log_size(struct intel_guc_log *log)
+{
+	/*
+	 *  GuC Log buffer Layout:
+	 *
+	 *  NB: Ordering must follow "enum guc_log_buffer_type".
+	 *
+	 *  +===============================+ 00B
+	 *  |      Debug state header       |
+	 *  +-------------------------------+ 32B
+	 *  |    Crash dump state header    |
+	 *  +-------------------------------+ 64B
+	 *  |     Capture state header      |
+	 *  +-------------------------------+ 96B
+	 *  |                               |
+	 *  +===============================+ PAGE_SIZE (4KB)
+	 *  |          Debug logs           |
+	 *  +===============================+ + DEBUG_SIZE
+	 *  |        Crash Dump logs        |
+	 *  +===============================+ + CRASH_SIZE
+	 *  |         Capture logs          |
+	 *  +===============================+ + CAPTURE_SIZE
+	 */
+	return PAGE_SIZE +
+		intel_guc_log_section_size_crash(log) +
+		intel_guc_log_section_size_debug(log) +
+		intel_guc_log_section_size_capture(log);
+}
+
 /**
  * DOC: GuC firmware log
  *
@@ -139,7 +294,8 @@ static void guc_move_to_next_buf(struct intel_guc_log *log)
 	smp_wmb();
 
 	/* All data has been written, so now move the offset of sub buffer. */
-	relay_reserve(log->relay.channel, log->vma->obj->base.size - CAPTURE_BUFFER_SIZE);
+	relay_reserve(log->relay.channel, log->vma->obj->base.size -
+					  intel_guc_log_section_size_capture(log));
 
 	/* Switch to the next sub buffer */
 	relay_flush(log->relay.channel);
@@ -184,15 +340,16 @@ bool intel_guc_check_log_buf_overflow(struct intel_guc_log *log,
 	return overflow;
 }
 
-unsigned int intel_guc_get_log_buffer_size(enum guc_log_buffer_type type)
+unsigned int intel_guc_get_log_buffer_size(struct intel_guc_log *log,
+					   enum guc_log_buffer_type type)
 {
 	switch (type) {
 	case GUC_DEBUG_LOG_BUFFER:
-		return DEBUG_BUFFER_SIZE;
+		return intel_guc_log_section_size_debug(log);
 	case GUC_CRASH_DUMP_LOG_BUFFER:
-		return CRASH_BUFFER_SIZE;
+		return intel_guc_log_section_size_crash(log);
 	case GUC_CAPTURE_LOG_BUFFER:
-		return CAPTURE_BUFFER_SIZE;
+		return intel_guc_log_section_size_capture(log);
 	default:
 		MISSING_CASE(type);
 	}
@@ -200,7 +357,8 @@ unsigned int intel_guc_get_log_buffer_size(enum guc_log_buffer_type type)
 	return 0;
 }
 
-size_t intel_guc_get_log_buffer_offset(enum guc_log_buffer_type type)
+size_t intel_guc_get_log_buffer_offset(struct intel_guc_log *log,
+				       enum guc_log_buffer_type type)
 {
 	enum guc_log_buffer_type i;
 	size_t offset = PAGE_SIZE;/* for the log_buffer_states */
@@ -208,7 +366,7 @@ size_t intel_guc_get_log_buffer_offset(enum guc_log_buffer_type type)
 	for (i = GUC_DEBUG_LOG_BUFFER; i < GUC_MAX_LOG_BUFFER; ++i) {
 		if (i == type)
 			break;
-		offset += intel_guc_get_log_buffer_size(i);
+		offset += intel_guc_get_log_buffer_size(log, i);
 	}
 
 	return offset;
@@ -259,7 +417,7 @@ static void _guc_log_copy_debuglogs_for_relay(struct intel_guc_log *log)
 		 */
 		memcpy(&log_buf_state_local, log_buf_state,
 		       sizeof(struct guc_log_buffer_state));
-		buffer_size = intel_guc_get_log_buffer_size(type);
+		buffer_size = intel_guc_get_log_buffer_size(log, type);
 		read_offset = log_buf_state_local.read_ptr;
 		write_offset = log_buf_state_local.sampled_write_ptr;
 		full_cnt = log_buf_state_local.buffer_full_cnt;
@@ -374,7 +532,7 @@ static int guc_log_relay_create(struct intel_guc_log *log)
 	  * Keep the size of sub buffers same as shared log buffer
 	  * but GuC log-events excludes the error-state-capture logs
 	  */
-	subbuf_size = log->vma->size - CAPTURE_BUFFER_SIZE;
+	subbuf_size = log->vma->size - intel_guc_log_section_size_capture(log);
 
 	/*
 	 * Store up to 8 snapshots, which is large enough to buffer sufficient
@@ -461,32 +619,7 @@ int intel_guc_log_create(struct intel_guc_log *log)
 
 	GEM_BUG_ON(log->vma);
 
-	/*
-	 *  GuC Log buffer Layout
-	 * (this ordering must follow "enum guc_log_buffer_type" definition)
-	 *
-	 *  +===============================+ 00B
-	 *  |      Debug state header       |
-	 *  +-------------------------------+ 32B
-	 *  |    Crash dump state header    |
-	 *  +-------------------------------+ 64B
-	 *  |     Capture state header      |
-	 *  +-------------------------------+ 96B
-	 *  |                               |
-	 *  +===============================+ PAGE_SIZE (4KB)
-	 *  |          Debug logs           |
-	 *  +===============================+ + DEBUG_SIZE
-	 *  |        Crash Dump logs        |
-	 *  +===============================+ + CRASH_SIZE
-	 *  |         Capture logs          |
-	 *  +===============================+ + CAPTURE_SIZE
-	 */
-	if (intel_guc_capture_output_min_size_est(guc) > CAPTURE_BUFFER_SIZE)
-		DRM_WARN("GuC log buffer for state_capture maybe too small. %d < %d\n",
-			 CAPTURE_BUFFER_SIZE, intel_guc_capture_output_min_size_est(guc));
-
-	guc_log_size = PAGE_SIZE + CRASH_BUFFER_SIZE + DEBUG_BUFFER_SIZE +
-		       CAPTURE_BUFFER_SIZE;
+	guc_log_size = intel_guc_log_size(log);
 
 	vma = intel_guc_allocate_vma(guc, guc_log_size);
 	if (IS_ERR(vma)) {
@@ -749,8 +882,9 @@ int intel_guc_log_dump(struct intel_guc_log *log, struct drm_printer *p,
 	struct intel_guc *guc = log_to_guc(log);
 	struct intel_uc *uc = container_of(guc, struct intel_uc, guc);
 	struct drm_i915_gem_object *obj = NULL;
-	u32 *map;
-	int i = 0;
+	void *map;
+	u32 *page;
+	int i, j;
 
 	if (!intel_guc_is_supported(guc))
 		return -ENODEV;
@@ -763,21 +897,34 @@ int intel_guc_log_dump(struct intel_guc_log *log, struct drm_printer *p,
 	if (!obj)
 		return 0;
 
+	page = (u32 *)__get_free_page(GFP_KERNEL);
+	if (!page)
+		return -ENOMEM;
+
+	intel_guc_dump_time_info(guc, p);
+
 	map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(map)) {
 		DRM_DEBUG("Failed to pin object\n");
 		drm_puts(p, "(log data unaccessible)\n");
+		free_page((unsigned long)page);
 		return PTR_ERR(map);
 	}
 
-	for (i = 0; i < obj->base.size / sizeof(u32); i += 4)
-		drm_printf(p, "0x%08x 0x%08x 0x%08x 0x%08x\n",
-			   *(map + i), *(map + i + 1),
-			   *(map + i + 2), *(map + i + 3));
+	for (i = 0; i < obj->base.size; i += PAGE_SIZE) {
+		if (!i915_memcpy_from_wc(page, map + i, PAGE_SIZE))
+			memcpy(page, map + i, PAGE_SIZE);
+
+		for (j = 0; j < PAGE_SIZE / sizeof(u32); j += 4)
+			drm_printf(p, "0x%08x 0x%08x 0x%08x 0x%08x\n",
+				   *(page + j + 0), *(page + j + 1),
+				   *(page + j + 2), *(page + j + 3));
+	}
 
 	drm_puts(p, "\n");
 
 	i915_gem_object_unpin_map(obj);
+	free_page((unsigned long)page);
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h
index 18007e639be9..02127703be80 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_log.h
@@ -15,20 +15,6 @@
 
 struct intel_guc;
 
-#if defined(CONFIG_DRM_I915_DEBUG_GUC)
-#define CRASH_BUFFER_SIZE	SZ_2M
-#define DEBUG_BUFFER_SIZE	SZ_16M
-#define CAPTURE_BUFFER_SIZE	SZ_4M
-#elif defined(CONFIG_DRM_I915_DEBUG_GEM)
-#define CRASH_BUFFER_SIZE	SZ_1M
-#define DEBUG_BUFFER_SIZE	SZ_2M
-#define CAPTURE_BUFFER_SIZE	SZ_1M
-#else
-#define CRASH_BUFFER_SIZE	SZ_8K
-#define DEBUG_BUFFER_SIZE	SZ_64K
-#define CAPTURE_BUFFER_SIZE	SZ_16K
-#endif
-
 /*
  * While we're using plain log level in i915, GuC controls are much more...
  * "elaborate"? We have a couple of bits for verbosity, separate bit for actual
@@ -46,10 +32,30 @@ struct intel_guc;
 #define GUC_VERBOSITY_TO_LOG_LEVEL(x)	((x) + 2)
 #define GUC_LOG_LEVEL_MAX GUC_VERBOSITY_TO_LOG_LEVEL(GUC_LOG_VERBOSITY_MAX)
 
+enum {
+	GUC_LOG_SECTIONS_CRASH,
+	GUC_LOG_SECTIONS_DEBUG,
+	GUC_LOG_SECTIONS_CAPTURE,
+	GUC_LOG_SECTIONS_LIMIT
+};
+
 struct intel_guc_log {
 	u32 level;
+
+	/* Allocation settings */
+	struct {
+		s32 bytes;	/* Size in bytes */
+		s32 units;	/* GuC API units - 1MB or 4KB */
+		s32 count;	/* Number of API units */
+		u32 flag;	/* GuC API units flag */
+	} sizes[GUC_LOG_SECTIONS_LIMIT];
+	bool sizes_initialised;
+
+	/* Combined buffer allocation */
 	struct i915_vma *vma;
 	void *buf_addr;
+
+	/* RelayFS support */
 	struct {
 		bool buf_in_use;
 		bool started;
@@ -58,6 +64,7 @@ struct intel_guc_log {
 		struct mutex lock;
 		u32 full_count;
 	} relay;
+
 	/* logging related stats */
 	struct {
 		u32 sampled_overflow;
@@ -69,8 +76,9 @@ struct intel_guc_log {
 void intel_guc_log_init_early(struct intel_guc_log *log);
 bool intel_guc_check_log_buf_overflow(struct intel_guc_log *log, enum guc_log_buffer_type type,
 				      unsigned int full_cnt);
-unsigned int intel_guc_get_log_buffer_size(enum guc_log_buffer_type type);
-size_t intel_guc_get_log_buffer_offset(enum guc_log_buffer_type type);
+unsigned int intel_guc_get_log_buffer_size(struct intel_guc_log *log,
+					   enum guc_log_buffer_type type);
+size_t intel_guc_get_log_buffer_offset(struct intel_guc_log *log, enum guc_log_buffer_type type);
 int intel_guc_log_create(struct intel_guc_log *log);
 void intel_guc_log_destroy(struct intel_guc_log *log);
 
@@ -92,4 +100,6 @@ void intel_guc_log_info(struct intel_guc_log *log, struct drm_printer *p);
 int intel_guc_log_dump(struct intel_guc_log *log, struct drm_printer *p,
 		       bool dump_load_err);
 
+u32 intel_guc_log_section_size_capture(struct intel_guc_log *log);
+
 #endif
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h
index 8dc063f087eb..a7092f711e9c 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_reg.h
@@ -102,6 +102,10 @@
 #define   GUC_SEND_TRIGGER		  (1<<0)
 #define GEN11_GUC_HOST_INTERRUPT	_MMIO(0x1901f0)
 
+#define GEN12_GUC_SEM_INTR_ENABLES	_MMIO(0xc71c)
+#define   GUC_SEM_INTR_ROUTE_TO_GUC	BIT(31)
+#define   GUC_SEM_INTR_ENABLE_ALL	(0xff)
+
 #define GUC_NUM_DOORBELLS		256
 
 /* format of the HW-monitored doorbell cacheline */
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
index ec9c4ca0f615..fdd895f73f9f 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
@@ -137,17 +137,6 @@ static int guc_action_slpc_set_param(struct intel_guc *guc, u8 id, u32 value)
 	return ret > 0 ? -EPROTO : ret;
 }
 
-static int guc_action_slpc_unset_param(struct intel_guc *guc, u8 id)
-{
-	u32 request[] = {
-		GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST,
-		SLPC_EVENT(SLPC_EVENT_PARAMETER_UNSET, 1),
-		id,
-	};
-
-	return intel_guc_send(guc, request, ARRAY_SIZE(request));
-}
-
 static bool slpc_is_running(struct intel_guc_slpc *slpc)
 {
 	return slpc_get_state(slpc) == SLPC_GLOBAL_STATE_RUNNING;
@@ -201,16 +190,6 @@ static int slpc_set_param(struct intel_guc_slpc *slpc, u8 id, u32 value)
 	return ret;
 }
 
-static int slpc_unset_param(struct intel_guc_slpc *slpc,
-			    u8 id)
-{
-	struct intel_guc *guc = slpc_to_guc(slpc);
-
-	GEM_BUG_ON(id >= SLPC_MAX_PARAM);
-
-	return guc_action_slpc_unset_param(guc, id);
-}
-
 static int slpc_force_min_freq(struct intel_guc_slpc *slpc, u32 freq)
 {
 	struct drm_i915_private *i915 = slpc_to_i915(slpc);
@@ -488,23 +467,33 @@ int intel_guc_slpc_set_min_freq(struct intel_guc_slpc *slpc, u32 val)
 
 	/* Need a lock now since waitboost can be modifying min as well */
 	mutex_lock(&slpc->lock);
-
-	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
-
-		ret = slpc_set_param(slpc,
-				     SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ,
-				     val);
-
-		/* Return standardized err code for sysfs calls */
-		if (ret)
-			ret = -EIO;
+	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
+
+	/* Ignore efficient freq if lower min freq is requested */
+	ret = slpc_set_param(slpc,
+			     SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY,
+			     val < slpc->rp1_freq);
+	if (ret) {
+		i915_probe_error(i915, "Failed to toggle efficient freq (%pe)\n",
+				 ERR_PTR(ret));
+		goto out;
 	}
 
+	ret = slpc_set_param(slpc,
+			     SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ,
+			     val);
+
 	if (!ret)
 		slpc->min_freq_softlimit = val;
 
+out:
+	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 	mutex_unlock(&slpc->lock);
 
+	/* Return standardized err code for sysfs calls */
+	if (ret)
+		ret = -EIO;
+
 	return ret;
 }
 
@@ -575,45 +564,28 @@ static int slpc_set_softlimits(struct intel_guc_slpc *slpc)
 	 * unless they have deviated from defaults, in which case,
 	 * we retain the values and set min/max accordingly.
 	 */
-	if (!slpc->max_freq_softlimit)
+	if (!slpc->max_freq_softlimit) {
 		slpc->max_freq_softlimit = slpc->rp0_freq;
-	else if (slpc->max_freq_softlimit != slpc->rp0_freq)
+		slpc_to_gt(slpc)->defaults.max_freq = slpc->max_freq_softlimit;
+	} else if (slpc->max_freq_softlimit != slpc->rp0_freq) {
 		ret = intel_guc_slpc_set_max_freq(slpc,
 						  slpc->max_freq_softlimit);
+	}
 
 	if (unlikely(ret))
 		return ret;
 
-	if (!slpc->min_freq_softlimit)
-		slpc->min_freq_softlimit = slpc->min_freq;
-	else if (slpc->min_freq_softlimit != slpc->min_freq)
+	if (!slpc->min_freq_softlimit) {
+		ret = intel_guc_slpc_get_min_freq(slpc, &slpc->min_freq_softlimit);
+		if (unlikely(ret))
+			return ret;
+		slpc_to_gt(slpc)->defaults.min_freq = slpc->min_freq_softlimit;
+	} else if (slpc->min_freq_softlimit != slpc->min_freq) {
 		return intel_guc_slpc_set_min_freq(slpc,
 						   slpc->min_freq_softlimit);
-
-	return 0;
-}
-
-static int slpc_ignore_eff_freq(struct intel_guc_slpc *slpc, bool ignore)
-{
-	int ret = 0;
-
-	if (ignore) {
-		ret = slpc_set_param(slpc,
-				     SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY,
-				     ignore);
-		if (!ret)
-			return slpc_set_param(slpc,
-					      SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ,
-					      slpc->min_freq);
-	} else {
-		ret = slpc_unset_param(slpc,
-				       SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY);
-		if (!ret)
-			return slpc_unset_param(slpc,
-						SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ);
 	}
 
-	return ret;
+	return 0;
 }
 
 static int slpc_use_fused_rp0(struct intel_guc_slpc *slpc)
@@ -675,14 +647,6 @@ int intel_guc_slpc_enable(struct intel_guc_slpc *slpc)
 
 	slpc_get_rp_values(slpc);
 
-	/* Ignore efficient freq and set min to platform min */
-	ret = slpc_ignore_eff_freq(slpc, true);
-	if (unlikely(ret)) {
-		i915_probe_error(i915, "Failed to set SLPC min to RPn (%pe)\n",
-				 ERR_PTR(ret));
-		return ret;
-	}
-
 	/* Set SLPC max limit to RP0 */
 	ret = slpc_use_fused_rp0(slpc);
 	if (unlikely(ret)) {
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
index 3e91f44829e9..22ba66e48a9b 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c
@@ -1537,8 +1537,8 @@ void intel_guc_submission_reset_prepare(struct intel_guc *guc)
 	__reset_guc_busyness_stats(guc);
 
 	/* Flush IRQ handler */
-	spin_lock_irq(&guc_to_gt(guc)->irq_lock);
-	spin_unlock_irq(&guc_to_gt(guc)->irq_lock);
+	spin_lock_irq(guc_to_gt(guc)->irq_lock);
+	spin_unlock_irq(guc_to_gt(guc)->irq_lock);
 
 	guc_flush_submissions(guc);
 	guc_flush_destroyed_contexts(guc);
@@ -1873,7 +1873,7 @@ int intel_guc_submission_init(struct intel_guc *guc)
 	if (guc->submission_initialized)
 		return 0;
 
-	if (guc->fw.major_ver_found < 70) {
+	if (GET_UC_VER(guc) < MAKE_UC_VER(70, 0, 0)) {
 		ret = guc_lrc_desc_pool_create_v69(guc);
 		if (ret)
 			return ret;
@@ -2308,7 +2308,7 @@ static int register_context(struct intel_context *ce, bool loop)
 	GEM_BUG_ON(intel_context_is_child(ce));
 	trace_intel_context_register(ce);
 
-	if (guc->fw.major_ver_found >= 70)
+	if (GET_UC_VER(guc) >= MAKE_UC_VER(70, 0, 0))
 		ret = register_context_v70(guc, ce, loop);
 	else
 		ret = register_context_v69(guc, ce, loop);
@@ -2320,7 +2320,7 @@ static int register_context(struct intel_context *ce, bool loop)
 		set_context_registered(ce);
 		spin_unlock_irqrestore(&ce->guc_state.lock, flags);
 
-		if (guc->fw.major_ver_found >= 70)
+		if (GET_UC_VER(guc) >= MAKE_UC_VER(70, 0, 0))
 			guc_context_policy_init_v70(ce, loop);
 	}
 
@@ -2425,7 +2425,6 @@ static int guc_context_policy_init_v70(struct intel_context *ce, bool loop)
 	struct context_policy policy;
 	u32 execution_quantum;
 	u32 preemption_timeout;
-	bool missing = false;
 	unsigned long flags;
 	int ret;
 
@@ -2443,32 +2442,9 @@ static int guc_context_policy_init_v70(struct intel_context *ce, bool loop)
 		__guc_context_policy_add_preempt_to_idle(&policy, 1);
 
 	ret = __guc_context_set_context_policies(guc, &policy, loop);
-	missing = ret != 0;
-
-	if (!missing && intel_context_is_parent(ce)) {
-		struct intel_context *child;
-
-		for_each_child(ce, child) {
-			__guc_context_policy_start_klv(&policy, child->guc_id.id);
-
-			if (engine->flags & I915_ENGINE_WANT_FORCED_PREEMPTION)
-				__guc_context_policy_add_preempt_to_idle(&policy, 1);
-
-			child->guc_state.prio = ce->guc_state.prio;
-			__guc_context_policy_add_priority(&policy, ce->guc_state.prio);
-			__guc_context_policy_add_execution_quantum(&policy, execution_quantum);
-			__guc_context_policy_add_preemption_timeout(&policy, preemption_timeout);
-
-			ret = __guc_context_set_context_policies(guc, &policy, loop);
-			if (ret) {
-				missing = true;
-				break;
-			}
-		}
-	}
 
 	spin_lock_irqsave(&ce->guc_state.lock, flags);
-	if (missing)
+	if (ret != 0)
 		set_context_policy_required(ce);
 	else
 		clr_context_policy_required(ce);
@@ -2950,7 +2926,7 @@ static void __guc_context_set_preemption_timeout(struct intel_guc *guc,
 						 u16 guc_id,
 						 u32 preemption_timeout)
 {
-	if (guc->fw.major_ver_found >= 70) {
+	if (GET_UC_VER(guc) >= MAKE_UC_VER(70, 0, 0)) {
 		struct context_policy policy;
 
 		__guc_context_policy_start_klv(&policy, guc_id);
@@ -3215,7 +3191,7 @@ static int guc_context_alloc(struct intel_context *ce)
 static void __guc_context_set_prio(struct intel_guc *guc,
 				   struct intel_context *ce)
 {
-	if (guc->fw.major_ver_found >= 70) {
+	if (GET_UC_VER(guc) >= MAKE_UC_VER(70, 0, 0)) {
 		struct context_policy policy;
 
 		__guc_context_policy_start_klv(&policy, ce->guc_id.id);
@@ -4203,13 +4179,27 @@ int intel_guc_submission_setup(struct intel_engine_cs *engine)
 
 void intel_guc_submission_enable(struct intel_guc *guc)
 {
+	struct intel_gt *gt = guc_to_gt(guc);
+
+	/* Enable and route to GuC */
+	if (GRAPHICS_VER(gt->i915) >= 12)
+		intel_uncore_write(gt->uncore, GEN12_GUC_SEM_INTR_ENABLES,
+				   GUC_SEM_INTR_ROUTE_TO_GUC |
+				   GUC_SEM_INTR_ENABLE_ALL);
+
 	guc_init_lrc_mapping(guc);
 	guc_init_engine_stats(guc);
 }
 
 void intel_guc_submission_disable(struct intel_guc *guc)
 {
+	struct intel_gt *gt = guc_to_gt(guc);
+
 	/* Note: By the time we're here, GuC may have already been reset */
+
+	/* Disable and route to host */
+	if (GRAPHICS_VER(gt->i915) >= 12)
+		intel_uncore_write(gt->uncore, GEN12_GUC_SEM_INTR_ENABLES, 0x0);
 }
 
 static bool __guc_submission_supported(struct intel_guc *guc)
@@ -5175,4 +5165,5 @@ bool intel_guc_virtual_engine_has_heartbeat(const struct intel_engine_cs *ve)
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftest_guc.c"
 #include "selftest_guc_multi_lrc.c"
+#include "selftest_guc_hangcheck.c"
 #endif
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc.c b/drivers/gpu/drm/i915/gt/uc/intel_uc.c
index f2e7c82985ef..dbd048b77e19 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc.c
@@ -245,9 +245,9 @@ static int guc_enable_communication(struct intel_guc *guc)
 	intel_guc_enable_interrupts(guc);
 
 	/* check for CT messages received before we enabled interrupts */
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	intel_guc_ct_event_handler(&guc->ct);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 
 	drm_dbg(&i915->drm, "GuC communication enabled\n");
 
@@ -435,9 +435,11 @@ static void print_fw_ver(struct intel_uc *uc, struct intel_uc_fw *fw)
 {
 	struct drm_i915_private *i915 = uc_to_gt(uc)->i915;
 
-	drm_info(&i915->drm, "%s firmware %s version %u.%u\n",
-		 intel_uc_fw_type_repr(fw->type), fw->path,
-		 fw->major_ver_found, fw->minor_ver_found);
+	drm_info(&i915->drm, "%s firmware %s version %u.%u.%u\n",
+		 intel_uc_fw_type_repr(fw->type), fw->file_selected.path,
+		 fw->file_selected.major_ver,
+		 fw->file_selected.minor_ver,
+		 fw->file_selected.patch_ver);
 }
 
 static int __uc_init_hw(struct intel_uc *uc)
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
index 56a0d80f88ba..b91ad4aede1f 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c
@@ -41,7 +41,7 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw,
 		"%s firmware -> %s\n",
 		intel_uc_fw_type_repr(uc_fw->type),
 		status == INTEL_UC_FIRMWARE_SELECTED ?
-		uc_fw->path : intel_uc_fw_status_repr(status));
+		uc_fw->file_selected.path : intel_uc_fw_status_repr(status));
 }
 #endif
 
@@ -51,84 +51,153 @@ void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw,
  *
  * Note that RKL and ADL-S have the same GuC/HuC device ID's and use the same
  * firmware as TGL.
+ *
+ * Version numbers:
+ * Originally, the driver required an exact match major/minor/patch furmware
+ * file and only supported that one version for any given platform. However,
+ * the new direction from upstream is to be backwards compatible with all
+ * prior releases and to be as flexible as possible as to what firmware is
+ * loaded.
+ *
+ * For GuC, the major version number signifies a backwards breaking API change.
+ * So, new format GuC firmware files are labelled by their major version only.
+ * For HuC, there is no KMD interaction, hence no version matching requirement.
+ * So, new format HuC firmware files have no version number at all.
+ *
+ * All of which means that the table below must keep all old format files with
+ * full three point version number. But newer files have reduced requirements.
+ * Having said that, the driver still needs to track the minor version number
+ * for GuC at least. As it is useful to report to the user that they are not
+ * running with a recent enough version for all KMD supported features,
+ * security fixes, etc. to be enabled.
+ */
+#define INTEL_GUC_FIRMWARE_DEFS(fw_def, guc_maj, guc_mmp) \
+	fw_def(DG2,          0, guc_maj(dg2,  70, 5)) \
+	fw_def(ALDERLAKE_P,  0, guc_maj(adlp, 70, 5)) \
+	fw_def(ALDERLAKE_P,  0, guc_mmp(adlp, 70, 1, 1)) \
+	fw_def(ALDERLAKE_P,  0, guc_mmp(adlp, 69, 0, 3)) \
+	fw_def(ALDERLAKE_S,  0, guc_maj(tgl,  70, 5)) \
+	fw_def(ALDERLAKE_S,  0, guc_mmp(tgl,  70, 1, 1)) \
+	fw_def(ALDERLAKE_S,  0, guc_mmp(tgl,  69, 0, 3)) \
+	fw_def(DG1,          0, guc_maj(dg1,  70, 5)) \
+	fw_def(ROCKETLAKE,   0, guc_mmp(tgl,  70, 1, 1)) \
+	fw_def(TIGERLAKE,    0, guc_mmp(tgl,  70, 1, 1)) \
+	fw_def(JASPERLAKE,   0, guc_mmp(ehl,  70, 1, 1)) \
+	fw_def(ELKHARTLAKE,  0, guc_mmp(ehl,  70, 1, 1)) \
+	fw_def(ICELAKE,      0, guc_mmp(icl,  70, 1, 1)) \
+	fw_def(COMETLAKE,    5, guc_mmp(cml,  70, 1, 1)) \
+	fw_def(COMETLAKE,    0, guc_mmp(kbl,  70, 1, 1)) \
+	fw_def(COFFEELAKE,   0, guc_mmp(kbl,  70, 1, 1)) \
+	fw_def(GEMINILAKE,   0, guc_mmp(glk,  70, 1, 1)) \
+	fw_def(KABYLAKE,     0, guc_mmp(kbl,  70, 1, 1)) \
+	fw_def(BROXTON,      0, guc_mmp(bxt,  70, 1, 1)) \
+	fw_def(SKYLAKE,      0, guc_mmp(skl,  70, 1, 1))
+
+#define INTEL_HUC_FIRMWARE_DEFS(fw_def, huc_raw, huc_mmp) \
+	fw_def(ALDERLAKE_P,  0, huc_raw(tgl)) \
+	fw_def(ALDERLAKE_P,  0, huc_mmp(tgl,  7, 9, 3)) \
+	fw_def(ALDERLAKE_S,  0, huc_raw(tgl)) \
+	fw_def(ALDERLAKE_S,  0, huc_mmp(tgl,  7, 9, 3)) \
+	fw_def(DG1,          0, huc_raw(dg1)) \
+	fw_def(ROCKETLAKE,   0, huc_mmp(tgl,  7, 9, 3)) \
+	fw_def(TIGERLAKE,    0, huc_mmp(tgl,  7, 9, 3)) \
+	fw_def(JASPERLAKE,   0, huc_mmp(ehl,  9, 0, 0)) \
+	fw_def(ELKHARTLAKE,  0, huc_mmp(ehl,  9, 0, 0)) \
+	fw_def(ICELAKE,      0, huc_mmp(icl,  9, 0, 0)) \
+	fw_def(COMETLAKE,    5, huc_mmp(cml,  4, 0, 0)) \
+	fw_def(COMETLAKE,    0, huc_mmp(kbl,  4, 0, 0)) \
+	fw_def(COFFEELAKE,   0, huc_mmp(kbl,  4, 0, 0)) \
+	fw_def(GEMINILAKE,   0, huc_mmp(glk,  4, 0, 0)) \
+	fw_def(KABYLAKE,     0, huc_mmp(kbl,  4, 0, 0)) \
+	fw_def(BROXTON,      0, huc_mmp(bxt,  2, 0, 0)) \
+	fw_def(SKYLAKE,      0, huc_mmp(skl,  2, 0, 0))
+
+/*
+ * Set of macros for producing a list of filenames from the above table.
  */
-#define INTEL_GUC_FIRMWARE_DEFS(fw_def, guc_def) \
-	fw_def(DG2,          0, guc_def(dg2,  70, 1, 2)) \
-	fw_def(ALDERLAKE_P,  0, guc_def(adlp, 70, 1, 1)) \
-	fw_def(ALDERLAKE_S,  0, guc_def(tgl,  70, 1, 1)) \
-	fw_def(DG1,          0, guc_def(dg1,  70, 1, 1)) \
-	fw_def(ROCKETLAKE,   0, guc_def(tgl,  70, 1, 1)) \
-	fw_def(TIGERLAKE,    0, guc_def(tgl,  70, 1, 1)) \
-	fw_def(JASPERLAKE,   0, guc_def(ehl,  70, 1, 1)) \
-	fw_def(ELKHARTLAKE,  0, guc_def(ehl,  70, 1, 1)) \
-	fw_def(ICELAKE,      0, guc_def(icl,  70, 1, 1)) \
-	fw_def(COMETLAKE,    5, guc_def(cml,  70, 1, 1)) \
-	fw_def(COMETLAKE,    0, guc_def(kbl,  70, 1, 1)) \
-	fw_def(COFFEELAKE,   0, guc_def(kbl,  70, 1, 1)) \
-	fw_def(GEMINILAKE,   0, guc_def(glk,  70, 1, 1)) \
-	fw_def(KABYLAKE,     0, guc_def(kbl,  70, 1, 1)) \
-	fw_def(BROXTON,      0, guc_def(bxt,  70, 1, 1)) \
-	fw_def(SKYLAKE,      0, guc_def(skl,  70, 1, 1))
-
-#define INTEL_GUC_FIRMWARE_DEFS_FALLBACK(fw_def, guc_def) \
-	fw_def(ALDERLAKE_P,  0, guc_def(adlp, 69, 0, 3)) \
-	fw_def(ALDERLAKE_S,  0, guc_def(tgl,  69, 0, 3))
-
-#define INTEL_HUC_FIRMWARE_DEFS(fw_def, huc_def) \
-	fw_def(ALDERLAKE_P,  0, huc_def(tgl,  7, 9, 3)) \
-	fw_def(ALDERLAKE_S,  0, huc_def(tgl,  7, 9, 3)) \
-	fw_def(DG1,          0, huc_def(dg1,  7, 9, 3)) \
-	fw_def(ROCKETLAKE,   0, huc_def(tgl,  7, 9, 3)) \
-	fw_def(TIGERLAKE,    0, huc_def(tgl,  7, 9, 3)) \
-	fw_def(JASPERLAKE,   0, huc_def(ehl,  9, 0, 0)) \
-	fw_def(ELKHARTLAKE,  0, huc_def(ehl,  9, 0, 0)) \
-	fw_def(ICELAKE,      0, huc_def(icl,  9, 0, 0)) \
-	fw_def(COMETLAKE,    5, huc_def(cml,  4, 0, 0)) \
-	fw_def(COMETLAKE,    0, huc_def(kbl,  4, 0, 0)) \
-	fw_def(COFFEELAKE,   0, huc_def(kbl,  4, 0, 0)) \
-	fw_def(GEMINILAKE,   0, huc_def(glk,  4, 0, 0)) \
-	fw_def(KABYLAKE,     0, huc_def(kbl,  4, 0, 0)) \
-	fw_def(BROXTON,      0, huc_def(bxt,  2, 0, 0)) \
-	fw_def(SKYLAKE,      0, huc_def(skl,  2, 0, 0))
-
-#define __MAKE_UC_FW_PATH(prefix_, name_, major_, minor_, patch_) \
+#define __MAKE_UC_FW_PATH_BLANK(prefix_, name_) \
+	"i915/" \
+	__stringify(prefix_) name_ ".bin"
+
+#define __MAKE_UC_FW_PATH_MAJOR(prefix_, name_, major_) \
+	"i915/" \
+	__stringify(prefix_) name_ \
+	__stringify(major_) ".bin"
+
+#define __MAKE_UC_FW_PATH_MMP(prefix_, name_, major_, minor_, patch_) \
 	"i915/" \
 	__stringify(prefix_) name_ \
 	__stringify(major_) "." \
 	__stringify(minor_) "." \
 	__stringify(patch_) ".bin"
 
-#define MAKE_GUC_FW_PATH(prefix_, major_, minor_, patch_) \
-	__MAKE_UC_FW_PATH(prefix_, "_guc_", major_, minor_, patch_)
+/* Minor for internal driver use, not part of file name */
+#define MAKE_GUC_FW_PATH_MAJOR(prefix_, major_, minor_) \
+	__MAKE_UC_FW_PATH_MAJOR(prefix_, "_guc_", major_)
+
+#define MAKE_GUC_FW_PATH_MMP(prefix_, major_, minor_, patch_) \
+	__MAKE_UC_FW_PATH_MMP(prefix_, "_guc_", major_, minor_, patch_)
 
-#define MAKE_HUC_FW_PATH(prefix_, major_, minor_, bld_num_) \
-	__MAKE_UC_FW_PATH(prefix_, "_huc_", major_, minor_, bld_num_)
+#define MAKE_HUC_FW_PATH_BLANK(prefix_) \
+	__MAKE_UC_FW_PATH_BLANK(prefix_, "_huc")
 
-/* All blobs need to be declared via MODULE_FIRMWARE() */
+#define MAKE_HUC_FW_PATH_MMP(prefix_, major_, minor_, patch_) \
+	__MAKE_UC_FW_PATH_MMP(prefix_, "_huc_", major_, minor_, patch_)
+
+/*
+ * All blobs need to be declared via MODULE_FIRMWARE().
+ * This first expansion of the table macros is solely to provide
+ * that declaration.
+ */
 #define INTEL_UC_MODULE_FW(platform_, revid_, uc_) \
 	MODULE_FIRMWARE(uc_);
 
-INTEL_GUC_FIRMWARE_DEFS(INTEL_UC_MODULE_FW, MAKE_GUC_FW_PATH)
-INTEL_GUC_FIRMWARE_DEFS_FALLBACK(INTEL_UC_MODULE_FW, MAKE_GUC_FW_PATH)
-INTEL_HUC_FIRMWARE_DEFS(INTEL_UC_MODULE_FW, MAKE_HUC_FW_PATH)
+INTEL_GUC_FIRMWARE_DEFS(INTEL_UC_MODULE_FW, MAKE_GUC_FW_PATH_MAJOR, MAKE_GUC_FW_PATH_MMP)
+INTEL_HUC_FIRMWARE_DEFS(INTEL_UC_MODULE_FW, MAKE_HUC_FW_PATH_BLANK, MAKE_HUC_FW_PATH_MMP)
 
-/* The below structs and macros are used to iterate across the list of blobs */
+/*
+ * The next expansion of the table macros (in __uc_fw_auto_select below) provides
+ * actual data structures with both the filename and the version information.
+ * These structure arrays are then iterated over to the list of suitable files
+ * for the current platform and to then attempt to load those files, in the order
+ * listed, until one is successfully found.
+ */
 struct __packed uc_fw_blob {
+	const char *path;
+	bool legacy;
 	u8 major;
 	u8 minor;
-	const char *path;
+	u8 patch;
 };
 
-#define UC_FW_BLOB(major_, minor_, path_) \
-	{ .major = major_, .minor = minor_, .path = path_ }
+#define UC_FW_BLOB_BASE(major_, minor_, patch_, path_) \
+	.major = major_, \
+	.minor = minor_, \
+	.patch = patch_, \
+	.path = path_,
+
+#define UC_FW_BLOB_NEW(major_, minor_, patch_, path_) \
+	{ UC_FW_BLOB_BASE(major_, minor_, patch_, path_) \
+	  .legacy = false }
 
-#define GUC_FW_BLOB(prefix_, major_, minor_, patch_) \
-	UC_FW_BLOB(major_, minor_, \
-		   MAKE_GUC_FW_PATH(prefix_, major_, minor_, patch_))
+#define UC_FW_BLOB_OLD(major_, minor_, patch_, path_) \
+	{ UC_FW_BLOB_BASE(major_, minor_, patch_, path_) \
+	  .legacy = true }
 
-#define HUC_FW_BLOB(prefix_, major_, minor_, bld_num_) \
-	UC_FW_BLOB(major_, minor_, \
-		   MAKE_HUC_FW_PATH(prefix_, major_, minor_, bld_num_))
+#define GUC_FW_BLOB(prefix_, major_, minor_) \
+	UC_FW_BLOB_NEW(major_, minor_, 0, \
+		       MAKE_GUC_FW_PATH_MAJOR(prefix_, major_, minor_))
+
+#define GUC_FW_BLOB_MMP(prefix_, major_, minor_, patch_) \
+	UC_FW_BLOB_OLD(major_, minor_, patch_, \
+		       MAKE_GUC_FW_PATH_MMP(prefix_, major_, minor_, patch_))
+
+#define HUC_FW_BLOB(prefix_) \
+	UC_FW_BLOB_NEW(0, 0, 0, MAKE_HUC_FW_PATH_BLANK(prefix_))
+
+#define HUC_FW_BLOB_MMP(prefix_, major_, minor_, patch_) \
+	UC_FW_BLOB_OLD(major_, minor_, patch_, \
+		       MAKE_HUC_FW_PATH_MMP(prefix_, major_, minor_, patch_))
 
 struct __packed uc_fw_platform_requirement {
 	enum intel_platform p;
@@ -152,23 +221,22 @@ static void
 __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw)
 {
 	static const struct uc_fw_platform_requirement blobs_guc[] = {
-		INTEL_GUC_FIRMWARE_DEFS(MAKE_FW_LIST, GUC_FW_BLOB)
-	};
-	static const struct uc_fw_platform_requirement blobs_guc_fallback[] = {
-		INTEL_GUC_FIRMWARE_DEFS_FALLBACK(MAKE_FW_LIST, GUC_FW_BLOB)
+		INTEL_GUC_FIRMWARE_DEFS(MAKE_FW_LIST, GUC_FW_BLOB, GUC_FW_BLOB_MMP)
 	};
 	static const struct uc_fw_platform_requirement blobs_huc[] = {
-		INTEL_HUC_FIRMWARE_DEFS(MAKE_FW_LIST, HUC_FW_BLOB)
+		INTEL_HUC_FIRMWARE_DEFS(MAKE_FW_LIST, HUC_FW_BLOB, HUC_FW_BLOB_MMP)
 	};
 	static const struct fw_blobs_by_type blobs_all[INTEL_UC_FW_NUM_TYPES] = {
 		[INTEL_UC_FW_TYPE_GUC] = { blobs_guc, ARRAY_SIZE(blobs_guc) },
 		[INTEL_UC_FW_TYPE_HUC] = { blobs_huc, ARRAY_SIZE(blobs_huc) },
 	};
+	static bool verified;
 	const struct uc_fw_platform_requirement *fw_blobs;
 	enum intel_platform p = INTEL_INFO(i915)->platform;
 	u32 fw_count;
 	u8 rev = INTEL_REVID(i915);
 	int i;
+	bool found;
 
 	/*
 	 * The only difference between the ADL GuC FWs is the HWConfig support.
@@ -183,50 +251,102 @@ __uc_fw_auto_select(struct drm_i915_private *i915, struct intel_uc_fw *uc_fw)
 	fw_blobs = blobs_all[uc_fw->type].blobs;
 	fw_count = blobs_all[uc_fw->type].count;
 
+	found = false;
 	for (i = 0; i < fw_count && p <= fw_blobs[i].p; i++) {
-		if (p == fw_blobs[i].p && rev >= fw_blobs[i].rev) {
-			const struct uc_fw_blob *blob = &fw_blobs[i].blob;
-			uc_fw->path = blob->path;
-			uc_fw->wanted_path = blob->path;
-			uc_fw->major_ver_wanted = blob->major;
-			uc_fw->minor_ver_wanted = blob->minor;
-			break;
-		}
-	}
+		const struct uc_fw_blob *blob = &fw_blobs[i].blob;
 
-	if (uc_fw->type == INTEL_UC_FW_TYPE_GUC) {
-		const struct uc_fw_platform_requirement *blobs = blobs_guc_fallback;
-		u32 count = ARRAY_SIZE(blobs_guc_fallback);
+		if (p != fw_blobs[i].p)
+			continue;
 
-		for (i = 0; i < count && p <= blobs[i].p; i++) {
-			if (p == blobs[i].p && rev >= blobs[i].rev) {
-				const struct uc_fw_blob *blob = &blobs[i].blob;
+		if (rev < fw_blobs[i].rev)
+			continue;
 
-				uc_fw->fallback.path = blob->path;
-				uc_fw->fallback.major_ver = blob->major;
-				uc_fw->fallback.minor_ver = blob->minor;
-				break;
-			}
+		if (uc_fw->file_selected.path) {
+			if (uc_fw->file_selected.path == blob->path)
+				uc_fw->file_selected.path = NULL;
+
+			continue;
 		}
+
+		uc_fw->file_selected.path = blob->path;
+		uc_fw->file_wanted.path = blob->path;
+		uc_fw->file_wanted.major_ver = blob->major;
+		uc_fw->file_wanted.minor_ver = blob->minor;
+		found = true;
+		break;
+	}
+
+	if (!found && uc_fw->file_selected.path) {
+		/* Failed to find a match for the last attempt?! */
+		uc_fw->file_selected.path = NULL;
 	}
 
 	/* make sure the list is ordered as expected */
-	if (IS_ENABLED(CONFIG_DRM_I915_SELFTEST)) {
+	if (IS_ENABLED(CONFIG_DRM_I915_SELFTEST) && !verified) {
+		verified = true;
+
 		for (i = 1; i < fw_count; i++) {
+			/* Next platform is good: */
 			if (fw_blobs[i].p < fw_blobs[i - 1].p)
 				continue;
 
+			/* Next platform revision is good: */
 			if (fw_blobs[i].p == fw_blobs[i - 1].p &&
 			    fw_blobs[i].rev < fw_blobs[i - 1].rev)
 				continue;
 
-			pr_err("invalid FW blob order: %s r%u comes before %s r%u\n",
-			       intel_platform_name(fw_blobs[i - 1].p),
-			       fw_blobs[i - 1].rev,
-			       intel_platform_name(fw_blobs[i].p),
-			       fw_blobs[i].rev);
+			/* Platform/revision must be in order: */
+			if (fw_blobs[i].p != fw_blobs[i - 1].p ||
+			    fw_blobs[i].rev != fw_blobs[i - 1].rev)
+				goto bad;
+
+			/* Next major version is good: */
+			if (fw_blobs[i].blob.major < fw_blobs[i - 1].blob.major)
+				continue;
+
+			/* New must be before legacy: */
+			if (!fw_blobs[i].blob.legacy && fw_blobs[i - 1].blob.legacy)
+				goto bad;
+
+			/* New to legacy also means 0.0 to X.Y (HuC), or X.0 to X.Y (GuC) */
+			if (fw_blobs[i].blob.legacy && !fw_blobs[i - 1].blob.legacy) {
+				if (!fw_blobs[i - 1].blob.major)
+					continue;
+
+				if (fw_blobs[i].blob.major == fw_blobs[i - 1].blob.major)
+					continue;
+			}
+
+			/* Major versions must be in order: */
+			if (fw_blobs[i].blob.major != fw_blobs[i - 1].blob.major)
+				goto bad;
+
+			/* Next minor version is good: */
+			if (fw_blobs[i].blob.minor < fw_blobs[i - 1].blob.minor)
+				continue;
+
+			/* Minor versions must be in order: */
+			if (fw_blobs[i].blob.minor != fw_blobs[i - 1].blob.minor)
+				goto bad;
+
+			/* Patch versions must be in order: */
+			if (fw_blobs[i].blob.patch <= fw_blobs[i - 1].blob.patch)
+				continue;
 
-			uc_fw->path = NULL;
+bad:
+			drm_err(&i915->drm, "Invalid FW blob order: %s r%u %s%d.%d.%d comes before %s r%u %s%d.%d.%d\n",
+				intel_platform_name(fw_blobs[i - 1].p), fw_blobs[i - 1].rev,
+				fw_blobs[i - 1].blob.legacy ? "L" : "v",
+				fw_blobs[i - 1].blob.major,
+				fw_blobs[i - 1].blob.minor,
+				fw_blobs[i - 1].blob.patch,
+				intel_platform_name(fw_blobs[i].p), fw_blobs[i].rev,
+				fw_blobs[i].blob.legacy ? "L" : "v",
+				fw_blobs[i].blob.major,
+				fw_blobs[i].blob.minor,
+				fw_blobs[i].blob.patch);
+
+			uc_fw->file_selected.path = NULL;
 		}
 	}
 }
@@ -259,7 +379,7 @@ static void __uc_fw_user_override(struct drm_i915_private *i915, struct intel_uc
 	}
 
 	if (unlikely(path)) {
-		uc_fw->path = path;
+		uc_fw->file_selected.path = path;
 		uc_fw->user_overridden = true;
 	}
 }
@@ -283,7 +403,7 @@ void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw,
 	 */
 	BUILD_BUG_ON(INTEL_UC_FIRMWARE_UNINITIALIZED);
 	GEM_BUG_ON(uc_fw->status);
-	GEM_BUG_ON(uc_fw->path);
+	GEM_BUG_ON(uc_fw->file_selected.path);
 
 	uc_fw->type = type;
 
@@ -292,7 +412,7 @@ void intel_uc_fw_init_early(struct intel_uc_fw *uc_fw,
 		__uc_fw_user_override(i915, uc_fw);
 	}
 
-	intel_uc_fw_change_status(uc_fw, uc_fw->path ? *uc_fw->path ?
+	intel_uc_fw_change_status(uc_fw, uc_fw->file_selected.path ? *uc_fw->file_selected.path ?
 				  INTEL_UC_FIRMWARE_SELECTED :
 				  INTEL_UC_FIRMWARE_DISABLED :
 				  INTEL_UC_FIRMWARE_NOT_SUPPORTED);
@@ -305,32 +425,32 @@ static void __force_fw_fetch_failures(struct intel_uc_fw *uc_fw, int e)
 
 	if (i915_inject_probe_error(i915, e)) {
 		/* non-existing blob */
-		uc_fw->path = "<invalid>";
+		uc_fw->file_selected.path = "<invalid>";
 		uc_fw->user_overridden = user;
 	} else if (i915_inject_probe_error(i915, e)) {
 		/* require next major version */
-		uc_fw->major_ver_wanted += 1;
-		uc_fw->minor_ver_wanted = 0;
+		uc_fw->file_wanted.major_ver += 1;
+		uc_fw->file_wanted.minor_ver = 0;
 		uc_fw->user_overridden = user;
 	} else if (i915_inject_probe_error(i915, e)) {
 		/* require next minor version */
-		uc_fw->minor_ver_wanted += 1;
+		uc_fw->file_wanted.minor_ver += 1;
 		uc_fw->user_overridden = user;
-	} else if (uc_fw->major_ver_wanted &&
+	} else if (uc_fw->file_wanted.major_ver &&
 		   i915_inject_probe_error(i915, e)) {
 		/* require prev major version */
-		uc_fw->major_ver_wanted -= 1;
-		uc_fw->minor_ver_wanted = 0;
+		uc_fw->file_wanted.major_ver -= 1;
+		uc_fw->file_wanted.minor_ver = 0;
 		uc_fw->user_overridden = user;
-	} else if (uc_fw->minor_ver_wanted &&
+	} else if (uc_fw->file_wanted.minor_ver &&
 		   i915_inject_probe_error(i915, e)) {
 		/* require prev minor version - hey, this should work! */
-		uc_fw->minor_ver_wanted -= 1;
+		uc_fw->file_wanted.minor_ver -= 1;
 		uc_fw->user_overridden = user;
 	} else if (user && i915_inject_probe_error(i915, e)) {
 		/* officially unsupported platform */
-		uc_fw->major_ver_wanted = 0;
-		uc_fw->minor_ver_wanted = 0;
+		uc_fw->file_wanted.major_ver = 0;
+		uc_fw->file_wanted.minor_ver = 0;
 		uc_fw->user_overridden = true;
 	}
 }
@@ -339,10 +459,12 @@ static int check_gsc_manifest(const struct firmware *fw,
 			      struct intel_uc_fw *uc_fw)
 {
 	u32 *dw = (u32 *)fw->data;
-	u32 version = dw[HUC_GSC_VERSION_DW];
+	u32 version_hi = dw[HUC_GSC_VERSION_HI_DW];
+	u32 version_lo = dw[HUC_GSC_VERSION_LO_DW];
 
-	uc_fw->major_ver_found = FIELD_GET(HUC_GSC_MAJOR_VER_MASK, version);
-	uc_fw->minor_ver_found = FIELD_GET(HUC_GSC_MINOR_VER_MASK, version);
+	uc_fw->file_selected.major_ver = FIELD_GET(HUC_GSC_MAJOR_VER_HI_MASK, version_hi);
+	uc_fw->file_selected.minor_ver = FIELD_GET(HUC_GSC_MINOR_VER_HI_MASK, version_hi);
+	uc_fw->file_selected.patch_ver = FIELD_GET(HUC_GSC_PATCH_VER_LO_MASK, version_lo);
 
 	return 0;
 }
@@ -357,7 +479,7 @@ static int check_ccs_header(struct drm_i915_private *i915,
 	/* Check the size of the blob before examining buffer contents */
 	if (unlikely(fw->size < sizeof(struct uc_css_header))) {
 		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu < %zu\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
+			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
 			 fw->size, sizeof(struct uc_css_header));
 		return -ENODATA;
 	}
@@ -370,7 +492,7 @@ static int check_ccs_header(struct drm_i915_private *i915,
 	if (unlikely(size != sizeof(struct uc_css_header))) {
 		drm_warn(&i915->drm,
 			 "%s firmware %s: unexpected header size: %zu != %zu\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
+			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
 			 fw->size, sizeof(struct uc_css_header));
 		return -EPROTO;
 	}
@@ -385,7 +507,7 @@ static int check_ccs_header(struct drm_i915_private *i915,
 	size = sizeof(struct uc_css_header) + uc_fw->ucode_size + uc_fw->rsa_size;
 	if (unlikely(fw->size < size)) {
 		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu < %zu\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
+			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
 			 fw->size, size);
 		return -ENOEXEC;
 	}
@@ -394,16 +516,18 @@ static int check_ccs_header(struct drm_i915_private *i915,
 	size = __intel_uc_fw_get_upload_size(uc_fw);
 	if (unlikely(size >= i915->wopcm.size)) {
 		drm_warn(&i915->drm, "%s firmware %s: invalid size: %zu > %zu\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
+			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
 			 size, (size_t)i915->wopcm.size);
 		return -E2BIG;
 	}
 
 	/* Get version numbers from the CSS header */
-	uc_fw->major_ver_found = FIELD_GET(CSS_SW_VERSION_UC_MAJOR,
-					   css->sw_version);
-	uc_fw->minor_ver_found = FIELD_GET(CSS_SW_VERSION_UC_MINOR,
-					   css->sw_version);
+	uc_fw->file_selected.major_ver = FIELD_GET(CSS_SW_VERSION_UC_MAJOR,
+						   css->sw_version);
+	uc_fw->file_selected.minor_ver = FIELD_GET(CSS_SW_VERSION_UC_MINOR,
+						   css->sw_version);
+	uc_fw->file_selected.patch_ver = FIELD_GET(CSS_SW_VERSION_UC_PATCH,
+						   css->sw_version);
 
 	if (uc_fw->type == INTEL_UC_FW_TYPE_GUC)
 		uc_fw->private_data_size = css->private_data_size;
@@ -422,9 +546,11 @@ static int check_ccs_header(struct drm_i915_private *i915,
 int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw)
 {
 	struct drm_i915_private *i915 = __uc_fw_to_gt(uc_fw)->i915;
+	struct intel_uc_fw_file file_ideal;
 	struct device *dev = i915->drm.dev;
 	struct drm_i915_gem_object *obj;
 	const struct firmware *fw = NULL;
+	bool old_ver = false;
 	int err;
 
 	GEM_BUG_ON(!i915->wopcm.size);
@@ -437,24 +563,32 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw)
 	__force_fw_fetch_failures(uc_fw, -EINVAL);
 	__force_fw_fetch_failures(uc_fw, -ESTALE);
 
-	err = firmware_request_nowarn(&fw, uc_fw->path, dev);
-	if (err && !intel_uc_fw_is_overridden(uc_fw) && uc_fw->fallback.path) {
-		err = firmware_request_nowarn(&fw, uc_fw->fallback.path, dev);
-		if (!err) {
-			drm_notice(&i915->drm,
-				   "%s firmware %s is recommended, but only %s was found\n",
-				   intel_uc_fw_type_repr(uc_fw->type),
-				   uc_fw->wanted_path,
-				   uc_fw->fallback.path);
-			drm_info(&i915->drm,
-				 "Consider updating your linux-firmware pkg or downloading from %s\n",
-				 INTEL_UC_FIRMWARE_URL);
-
-			uc_fw->path = uc_fw->fallback.path;
-			uc_fw->major_ver_wanted = uc_fw->fallback.major_ver;
-			uc_fw->minor_ver_wanted = uc_fw->fallback.minor_ver;
+	err = firmware_request_nowarn(&fw, uc_fw->file_selected.path, dev);
+	memcpy(&file_ideal, &uc_fw->file_wanted, sizeof(file_ideal));
+
+	/* Any error is terminal if overriding. Don't bother searching for older versions */
+	if (err && intel_uc_fw_is_overridden(uc_fw))
+		goto fail;
+
+	while (err == -ENOENT) {
+		old_ver = true;
+
+		__uc_fw_auto_select(i915, uc_fw);
+		if (!uc_fw->file_selected.path) {
+			/*
+			 * No more options! But set the path back to something
+			 * valid just in case it gets dereferenced.
+			 */
+			uc_fw->file_selected.path = file_ideal.path;
+
+			/* Also, preserve the version that was really wanted */
+			memcpy(&uc_fw->file_wanted, &file_ideal, sizeof(uc_fw->file_wanted));
+			break;
 		}
+
+		err = firmware_request_nowarn(&fw, uc_fw->file_selected.path, dev);
 	}
+
 	if (err)
 		goto fail;
 
@@ -465,18 +599,39 @@ int intel_uc_fw_fetch(struct intel_uc_fw *uc_fw)
 	if (err)
 		goto fail;
 
-	if (uc_fw->major_ver_found != uc_fw->major_ver_wanted ||
-	    uc_fw->minor_ver_found < uc_fw->minor_ver_wanted) {
-		drm_notice(&i915->drm, "%s firmware %s: unexpected version: %u.%u != %u.%u\n",
-			   intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
-			   uc_fw->major_ver_found, uc_fw->minor_ver_found,
-			   uc_fw->major_ver_wanted, uc_fw->minor_ver_wanted);
-		if (!intel_uc_fw_is_overridden(uc_fw)) {
-			err = -ENOEXEC;
-			goto fail;
+	if (uc_fw->file_wanted.major_ver) {
+		/* Check the file's major version was as it claimed */
+		if (uc_fw->file_selected.major_ver != uc_fw->file_wanted.major_ver) {
+			drm_notice(&i915->drm, "%s firmware %s: unexpected version: %u.%u != %u.%u\n",
+				   intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
+				   uc_fw->file_selected.major_ver, uc_fw->file_selected.minor_ver,
+				   uc_fw->file_wanted.major_ver, uc_fw->file_wanted.minor_ver);
+			if (!intel_uc_fw_is_overridden(uc_fw)) {
+				err = -ENOEXEC;
+				goto fail;
+			}
+		} else {
+			if (uc_fw->file_selected.minor_ver < uc_fw->file_wanted.minor_ver)
+				old_ver = true;
 		}
 	}
 
+	if (old_ver) {
+		/* Preserve the version that was really wanted */
+		memcpy(&uc_fw->file_wanted, &file_ideal, sizeof(uc_fw->file_wanted));
+
+		drm_notice(&i915->drm,
+			   "%s firmware %s (%d.%d) is recommended, but only %s (%d.%d) was found\n",
+			   intel_uc_fw_type_repr(uc_fw->type),
+			   uc_fw->file_wanted.path,
+			   uc_fw->file_wanted.major_ver, uc_fw->file_wanted.minor_ver,
+			   uc_fw->file_selected.path,
+			   uc_fw->file_selected.major_ver, uc_fw->file_selected.minor_ver);
+		drm_info(&i915->drm,
+			 "Consider updating your linux-firmware pkg or downloading from %s\n",
+			 INTEL_UC_FIRMWARE_URL);
+	}
+
 	if (HAS_LMEM(i915)) {
 		obj = i915_gem_object_create_lmem_from_data(i915, fw->data, fw->size);
 		if (!IS_ERR(obj))
@@ -503,7 +658,7 @@ fail:
 				  INTEL_UC_FIRMWARE_ERROR);
 
 	i915_probe_error(i915, "%s firmware %s: fetch failed with error %d\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path, err);
+			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path, err);
 	drm_info(&i915->drm, "%s firmware(s) can be downloaded from %s\n",
 		 intel_uc_fw_type_repr(uc_fw->type), INTEL_UC_FIRMWARE_URL);
 
@@ -645,7 +800,7 @@ int intel_uc_fw_upload(struct intel_uc_fw *uc_fw, u32 dst_offset, u32 dma_flags)
 
 fail:
 	i915_probe_error(gt->i915, "Failed to load %s firmware %s (%d)\n",
-			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->path,
+			 intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path,
 			 err);
 	intel_uc_fw_change_status(uc_fw, INTEL_UC_FIRMWARE_LOAD_FAIL);
 	return err;
@@ -863,19 +1018,34 @@ size_t intel_uc_fw_copy_rsa(struct intel_uc_fw *uc_fw, void *dst, u32 max_len)
  */
 void intel_uc_fw_dump(const struct intel_uc_fw *uc_fw, struct drm_printer *p)
 {
+	u32 ver_sel, ver_want;
+
 	drm_printf(p, "%s firmware: %s\n",
-		   intel_uc_fw_type_repr(uc_fw->type), uc_fw->wanted_path);
-	if (uc_fw->fallback.path) {
-		drm_printf(p, "%s firmware fallback: %s\n",
-			   intel_uc_fw_type_repr(uc_fw->type), uc_fw->fallback.path);
-		drm_printf(p, "fallback selected: %s\n",
-			   str_yes_no(uc_fw->path == uc_fw->fallback.path));
-	}
+		   intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_selected.path);
+	if (uc_fw->file_selected.path != uc_fw->file_wanted.path)
+		drm_printf(p, "%s firmware wanted: %s\n",
+			   intel_uc_fw_type_repr(uc_fw->type), uc_fw->file_wanted.path);
 	drm_printf(p, "\tstatus: %s\n",
 		   intel_uc_fw_status_repr(uc_fw->status));
-	drm_printf(p, "\tversion: wanted %u.%u, found %u.%u\n",
-		   uc_fw->major_ver_wanted, uc_fw->minor_ver_wanted,
-		   uc_fw->major_ver_found, uc_fw->minor_ver_found);
+	ver_sel = MAKE_UC_VER(uc_fw->file_selected.major_ver,
+			      uc_fw->file_selected.minor_ver,
+			      uc_fw->file_selected.patch_ver);
+	ver_want = MAKE_UC_VER(uc_fw->file_wanted.major_ver,
+			       uc_fw->file_wanted.minor_ver,
+			       uc_fw->file_wanted.patch_ver);
+	if (ver_sel < ver_want)
+		drm_printf(p, "\tversion: wanted %u.%u.%u, found %u.%u.%u\n",
+			   uc_fw->file_wanted.major_ver,
+			   uc_fw->file_wanted.minor_ver,
+			   uc_fw->file_wanted.patch_ver,
+			   uc_fw->file_selected.major_ver,
+			   uc_fw->file_selected.minor_ver,
+			   uc_fw->file_selected.patch_ver);
+	else
+		drm_printf(p, "\tversion: found %u.%u.%u\n",
+			   uc_fw->file_selected.major_ver,
+			   uc_fw->file_selected.minor_ver,
+			   uc_fw->file_selected.patch_ver);
 	drm_printf(p, "\tuCode: %u bytes\n", uc_fw->ucode_size);
 	drm_printf(p, "\tRSA: %u bytes\n", uc_fw->rsa_size);
 }
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h
index 7aa2644400b9..cb586f7df270 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw.h
@@ -65,6 +65,18 @@ enum intel_uc_fw_type {
 #define INTEL_UC_FW_NUM_TYPES 2
 
 /*
+ * The firmware build process will generate a version header file with major and
+ * minor version defined. The versions are built into CSS header of firmware.
+ * i915 kernel driver set the minimal firmware version required per platform.
+ */
+struct intel_uc_fw_file {
+	const char *path;
+	u16 major_ver;
+	u16 minor_ver;
+	u16 patch_ver;
+};
+
+/*
  * This structure encapsulates all the data needed during the process
  * of fetching, caching, and loading the firmware image into the uC.
  */
@@ -74,11 +86,12 @@ struct intel_uc_fw {
 		const enum intel_uc_fw_status status;
 		enum intel_uc_fw_status __status; /* no accidental overwrites */
 	};
-	const char *wanted_path;
-	const char *path;
+	struct intel_uc_fw_file file_wanted;
+	struct intel_uc_fw_file file_selected;
 	bool user_overridden;
 	size_t size;
 	struct drm_i915_gem_object *obj;
+
 	/**
 	 * @dummy: A vma used in binding the uc fw to ggtt. We can't define this
 	 * vma on the stack as it can lead to a stack overflow, so we define it
@@ -89,30 +102,18 @@ struct intel_uc_fw {
 	struct i915_vma_resource dummy;
 	struct i915_vma *rsa_data;
 
-	/*
-	 * The firmware build process will generate a version header file with major and
-	 * minor version defined. The versions are built into CSS header of firmware.
-	 * i915 kernel driver set the minimal firmware version required per platform.
-	 */
-	u16 major_ver_wanted;
-	u16 minor_ver_wanted;
-	u16 major_ver_found;
-	u16 minor_ver_found;
-
-	struct {
-		const char *path;
-		u16 major_ver;
-		u16 minor_ver;
-	} fallback;
-
 	u32 rsa_size;
 	u32 ucode_size;
-
 	u32 private_data_size;
 
 	bool loaded_via_gsc;
 };
 
+#define MAKE_UC_VER(maj, min, pat)	((pat) | ((min) << 8) | ((maj) << 16))
+#define GET_UC_VER(uc)			(MAKE_UC_VER((uc)->fw.file_selected.major_ver, \
+						     (uc)->fw.file_selected.minor_ver, \
+						     (uc)->fw.file_selected.patch_ver))
+
 #ifdef CONFIG_DRM_I915_DEBUG_GUC
 void intel_uc_fw_change_status(struct intel_uc_fw *uc_fw,
 			       enum intel_uc_fw_status status);
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h
index b05e0e35b734..7a411178bdbf 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h
+++ b/drivers/gpu/drm/i915/gt/uc/intel_uc_fw_abi.h
@@ -83,8 +83,10 @@ struct uc_css_header {
 } __packed;
 static_assert(sizeof(struct uc_css_header) == 128);
 
-#define HUC_GSC_VERSION_DW		44
-#define   HUC_GSC_MAJOR_VER_MASK	(0xFF << 0)
-#define   HUC_GSC_MINOR_VER_MASK	(0xFF << 16)
+#define HUC_GSC_VERSION_HI_DW		44
+#define   HUC_GSC_MAJOR_VER_HI_MASK	(0xFF << 0)
+#define   HUC_GSC_MINOR_VER_HI_MASK	(0xFF << 16)
+#define HUC_GSC_VERSION_LO_DW		45
+#define   HUC_GSC_PATCH_VER_LO_MASK	(0xFF << 0)
 
 #endif /* _INTEL_UC_FW_ABI_H */
diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
index 1df71d0796ae..e28518fe8b90 100644
--- a/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
+++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc.c
@@ -54,6 +54,9 @@ static int intel_guc_scrub_ctbs(void *arg)
 	struct intel_engine_cs *engine;
 	struct intel_context *ce;
 
+	if (!intel_has_gpu_reset(gt))
+		return 0;
+
 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
 	engine = intel_selftest_find_any_engine(gt);
 
@@ -62,7 +65,7 @@ static int intel_guc_scrub_ctbs(void *arg)
 		ce = intel_context_create(engine);
 		if (IS_ERR(ce)) {
 			ret = PTR_ERR(ce);
-			pr_err("Failed to create context, %d: %d\n", i, ret);
+			drm_err(&gt->i915->drm, "Failed to create context, %d: %d\n", i, ret);
 			goto err;
 		}
 
@@ -83,7 +86,7 @@ static int intel_guc_scrub_ctbs(void *arg)
 
 		if (IS_ERR(rq)) {
 			ret = PTR_ERR(rq);
-			pr_err("Failed to create request, %d: %d\n", i, ret);
+			drm_err(&gt->i915->drm, "Failed to create request, %d: %d\n", i, ret);
 			goto err;
 		}
 
@@ -93,7 +96,7 @@ static int intel_guc_scrub_ctbs(void *arg)
 	for (i = 0; i < 3; ++i) {
 		ret = i915_request_wait(last[i], 0, HZ);
 		if (ret < 0) {
-			pr_err("Last request failed to complete: %d\n", ret);
+			drm_err(&gt->i915->drm, "Last request failed to complete: %d\n", ret);
 			goto err;
 		}
 		i915_request_put(last[i]);
@@ -110,7 +113,7 @@ static int intel_guc_scrub_ctbs(void *arg)
 	/* GT will not idle if G2H are lost */
 	ret = intel_gt_wait_for_idle(gt, HZ);
 	if (ret < 0) {
-		pr_err("GT failed to idle: %d\n", ret);
+		drm_err(&gt->i915->drm, "GT failed to idle: %d\n", ret);
 		goto err;
 	}
 
@@ -150,7 +153,7 @@ static int intel_guc_steal_guc_ids(void *arg)
 
 	ce = kcalloc(GUC_MAX_CONTEXT_ID, sizeof(*ce), GFP_KERNEL);
 	if (!ce) {
-		pr_err("Context array allocation failed\n");
+		drm_err(&gt->i915->drm, "Context array allocation failed\n");
 		return -ENOMEM;
 	}
 
@@ -164,24 +167,24 @@ static int intel_guc_steal_guc_ids(void *arg)
 	if (IS_ERR(ce[context_index])) {
 		ret = PTR_ERR(ce[context_index]);
 		ce[context_index] = NULL;
-		pr_err("Failed to create context: %d\n", ret);
+		drm_err(&gt->i915->drm, "Failed to create context: %d\n", ret);
 		goto err_wakeref;
 	}
 	ret = igt_spinner_init(&spin, engine->gt);
 	if (ret) {
-		pr_err("Failed to create spinner: %d\n", ret);
+		drm_err(&gt->i915->drm, "Failed to create spinner: %d\n", ret);
 		goto err_contexts;
 	}
 	spin_rq = igt_spinner_create_request(&spin, ce[context_index],
 					     MI_ARB_CHECK);
 	if (IS_ERR(spin_rq)) {
 		ret = PTR_ERR(spin_rq);
-		pr_err("Failed to create spinner request: %d\n", ret);
+		drm_err(&gt->i915->drm, "Failed to create spinner request: %d\n", ret);
 		goto err_contexts;
 	}
 	ret = request_add_spin(spin_rq, &spin);
 	if (ret) {
-		pr_err("Failed to add Spinner request: %d\n", ret);
+		drm_err(&gt->i915->drm, "Failed to add Spinner request: %d\n", ret);
 		goto err_spin_rq;
 	}
 
@@ -191,7 +194,7 @@ static int intel_guc_steal_guc_ids(void *arg)
 		if (IS_ERR(ce[context_index])) {
 			ret = PTR_ERR(ce[context_index--]);
 			ce[context_index] = NULL;
-			pr_err("Failed to create context: %d\n", ret);
+			drm_err(&gt->i915->drm, "Failed to create context: %d\n", ret);
 			goto err_spin_rq;
 		}
 
@@ -200,8 +203,8 @@ static int intel_guc_steal_guc_ids(void *arg)
 			ret = PTR_ERR(rq);
 			rq = NULL;
 			if (ret != -EAGAIN) {
-				pr_err("Failed to create request, %d: %d\n",
-				       context_index, ret);
+				drm_err(&gt->i915->drm, "Failed to create request, %d: %d\n",
+					context_index, ret);
 				goto err_spin_rq;
 			}
 		} else {
@@ -215,7 +218,7 @@ static int intel_guc_steal_guc_ids(void *arg)
 	igt_spinner_end(&spin);
 	ret = intel_selftest_wait_for_rq(spin_rq);
 	if (ret) {
-		pr_err("Spin request failed to complete: %d\n", ret);
+		drm_err(&gt->i915->drm, "Spin request failed to complete: %d\n", ret);
 		i915_request_put(last);
 		goto err_spin_rq;
 	}
@@ -227,7 +230,7 @@ static int intel_guc_steal_guc_ids(void *arg)
 	ret = i915_request_wait(last, 0, HZ * 30);
 	i915_request_put(last);
 	if (ret < 0) {
-		pr_err("Last request failed to complete: %d\n", ret);
+		drm_err(&gt->i915->drm, "Last request failed to complete: %d\n", ret);
 		goto err_spin_rq;
 	}
 
@@ -235,7 +238,7 @@ static int intel_guc_steal_guc_ids(void *arg)
 	rq = nop_user_request(ce[context_index], NULL);
 	if (IS_ERR(rq)) {
 		ret = PTR_ERR(rq);
-		pr_err("Failed to steal guc_id, %d: %d\n", context_index, ret);
+		drm_err(&gt->i915->drm, "Failed to steal guc_id, %d: %d\n", context_index, ret);
 		goto err_spin_rq;
 	}
 
@@ -243,21 +246,20 @@ static int intel_guc_steal_guc_ids(void *arg)
 	ret = i915_request_wait(rq, 0, HZ);
 	i915_request_put(rq);
 	if (ret < 0) {
-		pr_err("Request with stolen guc_id failed to complete: %d\n",
-		       ret);
+		drm_err(&gt->i915->drm, "Request with stolen guc_id failed to complete: %d\n", ret);
 		goto err_spin_rq;
 	}
 
 	/* Wait for idle */
 	ret = intel_gt_wait_for_idle(gt, HZ * 30);
 	if (ret < 0) {
-		pr_err("GT failed to idle: %d\n", ret);
+		drm_err(&gt->i915->drm, "GT failed to idle: %d\n", ret);
 		goto err_spin_rq;
 	}
 
 	/* Verify a guc_id was stolen */
 	if (guc->number_guc_id_stolen == number_guc_id_stolen) {
-		pr_err("No guc_id was stolen");
+		drm_err(&gt->i915->drm, "No guc_id was stolen");
 		ret = -EINVAL;
 	} else {
 		ret = 0;
diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c
new file mode 100644
index 000000000000..01f8cd3c3134
--- /dev/null
+++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc_hangcheck.c
@@ -0,0 +1,159 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#include "selftests/igt_spinner.h"
+#include "selftests/igt_reset.h"
+#include "selftests/intel_scheduler_helpers.h"
+#include "gt/intel_engine_heartbeat.h"
+#include "gem/selftests/mock_context.h"
+
+#define BEAT_INTERVAL	100
+
+static struct i915_request *nop_request(struct intel_engine_cs *engine)
+{
+	struct i915_request *rq;
+
+	rq = intel_engine_create_kernel_request(engine);
+	if (IS_ERR(rq))
+		return rq;
+
+	i915_request_get(rq);
+	i915_request_add(rq);
+
+	return rq;
+}
+
+static int intel_hang_guc(void *arg)
+{
+	struct intel_gt *gt = arg;
+	int ret = 0;
+	struct i915_gem_context *ctx;
+	struct intel_context *ce;
+	struct igt_spinner spin;
+	struct i915_request *rq;
+	intel_wakeref_t wakeref;
+	struct i915_gpu_error *global = &gt->i915->gpu_error;
+	struct intel_engine_cs *engine;
+	unsigned int reset_count;
+	u32 guc_status;
+	u32 old_beat;
+
+	ctx = kernel_context(gt->i915, NULL);
+	if (IS_ERR(ctx)) {
+		drm_err(&gt->i915->drm, "Failed get kernel context: %ld\n", PTR_ERR(ctx));
+		return PTR_ERR(ctx);
+	}
+
+	wakeref = intel_runtime_pm_get(gt->uncore->rpm);
+
+	ce = intel_context_create(gt->engine[BCS0]);
+	if (IS_ERR(ce)) {
+		ret = PTR_ERR(ce);
+		drm_err(&gt->i915->drm, "Failed to create spinner request: %d\n", ret);
+		goto err;
+	}
+
+	engine = ce->engine;
+	reset_count = i915_reset_count(global);
+
+	old_beat = engine->props.heartbeat_interval_ms;
+	ret = intel_engine_set_heartbeat(engine, BEAT_INTERVAL);
+	if (ret) {
+		drm_err(&gt->i915->drm, "Failed to boost heatbeat interval: %d\n", ret);
+		goto err;
+	}
+
+	ret = igt_spinner_init(&spin, engine->gt);
+	if (ret) {
+		drm_err(&gt->i915->drm, "Failed to create spinner: %d\n", ret);
+		goto err;
+	}
+
+	rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
+	intel_context_put(ce);
+	if (IS_ERR(rq)) {
+		ret = PTR_ERR(rq);
+		drm_err(&gt->i915->drm, "Failed to create spinner request: %d\n", ret);
+		goto err_spin;
+	}
+
+	ret = request_add_spin(rq, &spin);
+	if (ret) {
+		i915_request_put(rq);
+		drm_err(&gt->i915->drm, "Failed to add Spinner request: %d\n", ret);
+		goto err_spin;
+	}
+
+	ret = intel_reset_guc(gt);
+	if (ret) {
+		i915_request_put(rq);
+		drm_err(&gt->i915->drm, "Failed to reset GuC, ret = %d\n", ret);
+		goto err_spin;
+	}
+
+	guc_status = intel_uncore_read(gt->uncore, GUC_STATUS);
+	if (!(guc_status & GS_MIA_IN_RESET)) {
+		i915_request_put(rq);
+		drm_err(&gt->i915->drm, "GuC failed to reset: status = 0x%08X\n", guc_status);
+		ret = -EIO;
+		goto err_spin;
+	}
+
+	/* Wait for the heartbeat to cause a reset */
+	ret = intel_selftest_wait_for_rq(rq);
+	i915_request_put(rq);
+	if (ret) {
+		drm_err(&gt->i915->drm, "Request failed to complete: %d\n", ret);
+		goto err_spin;
+	}
+
+	if (i915_reset_count(global) == reset_count) {
+		drm_err(&gt->i915->drm, "Failed to record a GPU reset\n");
+		ret = -EINVAL;
+		goto err_spin;
+	}
+
+err_spin:
+	igt_spinner_end(&spin);
+	igt_spinner_fini(&spin);
+	intel_engine_set_heartbeat(engine, old_beat);
+
+	if (ret == 0) {
+		rq = nop_request(engine);
+		if (IS_ERR(rq)) {
+			ret = PTR_ERR(rq);
+			goto err;
+		}
+
+		ret = intel_selftest_wait_for_rq(rq);
+		i915_request_put(rq);
+		if (ret) {
+			drm_err(&gt->i915->drm, "No-op failed to complete: %d\n", ret);
+			goto err;
+		}
+	}
+
+err:
+	intel_runtime_pm_put(gt->uncore->rpm, wakeref);
+	kernel_context_close(ctx);
+
+	return ret;
+}
+
+int intel_guc_hang_check(struct drm_i915_private *i915)
+{
+	static const struct i915_subtest tests[] = {
+		SUBTEST(intel_hang_guc),
+	};
+	struct intel_gt *gt = to_gt(i915);
+
+	if (intel_gt_is_wedged(gt))
+		return 0;
+
+	if (!intel_uc_uses_guc_submission(&gt->uc))
+		return 0;
+
+	return intel_gt_live_subtests(tests, gt);
+}
diff --git a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
index 812220a43df8..d17982c36d25 100644
--- a/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
+++ b/drivers/gpu/drm/i915/gt/uc/selftest_guc_multi_lrc.c
@@ -115,30 +115,30 @@ static int __intel_guc_multi_lrc_basic(struct intel_gt *gt, unsigned int class)
 
 	parent = multi_lrc_create_parent(gt, class, 0);
 	if (IS_ERR(parent)) {
-		pr_err("Failed creating contexts: %ld", PTR_ERR(parent));
+		drm_err(&gt->i915->drm, "Failed creating contexts: %ld", PTR_ERR(parent));
 		return PTR_ERR(parent);
 	} else if (!parent) {
-		pr_debug("Not enough engines in class: %d", class);
+		drm_dbg(&gt->i915->drm, "Not enough engines in class: %d", class);
 		return 0;
 	}
 
 	rq = multi_lrc_nop_request(parent);
 	if (IS_ERR(rq)) {
 		ret = PTR_ERR(rq);
-		pr_err("Failed creating requests: %d", ret);
+		drm_err(&gt->i915->drm, "Failed creating requests: %d", ret);
 		goto out;
 	}
 
 	ret = intel_selftest_wait_for_rq(rq);
 	if (ret)
-		pr_err("Failed waiting on request: %d", ret);
+		drm_err(&gt->i915->drm, "Failed waiting on request: %d", ret);
 
 	i915_request_put(rq);
 
 	if (ret >= 0) {
 		ret = intel_gt_wait_for_idle(gt, HZ * 5);
 		if (ret < 0)
-			pr_err("GT failed to idle: %d\n", ret);
+			drm_err(&gt->i915->drm, "GT failed to idle: %d\n", ret);
 	}
 
 out:
diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c
index dad3a6054335..eef3bba8a41b 100644
--- a/drivers/gpu/drm/i915/gvt/cfg_space.c
+++ b/drivers/gpu/drm/i915/gvt/cfg_space.c
@@ -33,6 +33,7 @@
 
 #include "i915_drv.h"
 #include "gvt.h"
+#include "intel_pci_config.h"
 
 enum {
 	INTEL_GVT_PCI_BAR_GTTMMIO = 0,
@@ -353,9 +354,9 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu,
 	memset(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_OPREGION, 0, 4);
 
 	vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].size =
-		pci_resource_len(pdev, 0);
+		pci_resource_len(pdev, GTTMMADR_BAR);
 	vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].size =
-		pci_resource_len(pdev, 2);
+		pci_resource_len(pdev, GTT_APERTURE_BAR);
 
 	memset(vgpu_cfg_space(vgpu) + PCI_ROM_ADDRESS, 0, 4);
 
diff --git a/drivers/gpu/drm/i915/gvt/edid.c b/drivers/gpu/drm/i915/gvt/edid.c
index a30ba2d7b7ba..1b509c1a1e33 100644
--- a/drivers/gpu/drm/i915/gvt/edid.c
+++ b/drivers/gpu/drm/i915/gvt/edid.c
@@ -32,9 +32,10 @@
  *
  */
 
+#include "display/intel_gmbus_regs.h"
+#include "gvt.h"
 #include "i915_drv.h"
 #include "i915_reg.h"
-#include "gvt.h"
 
 #define GMBUS1_TOTAL_BYTES_SHIFT 16
 #define GMBUS1_TOTAL_BYTES_MASK 0x1ff
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index 61423da36710..daac2050d77d 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -498,7 +498,7 @@ static u32 bdw_vgpu_get_dp_bitrate(struct intel_vgpu *vgpu, enum port port)
 
 		switch (wrpll_ctl & WRPLL_REF_MASK) {
 		case WRPLL_REF_PCH_SSC:
-			refclk = vgpu->gvt->gt->i915->dpll.ref_clks.ssc;
+			refclk = vgpu->gvt->gt->i915->display.dpll.ref_clks.ssc;
 			break;
 		case WRPLL_REF_LCPLL:
 			refclk = 2700000;
@@ -529,7 +529,7 @@ out:
 static u32 bxt_vgpu_get_dp_bitrate(struct intel_vgpu *vgpu, enum port port)
 {
 	u32 dp_br = 0;
-	int refclk = vgpu->gvt->gt->i915->dpll.ref_clks.nssc;
+	int refclk = vgpu->gvt->gt->i915->display.dpll.ref_clks.nssc;
 	enum dpio_phy phy = DPIO_PHY0;
 	enum dpio_channel ch = DPIO_CH0;
 	struct dpll clock = {0};
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 94e5c29d2ee3..ae987e92251d 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -66,8 +66,7 @@ static int i915_capabilities(struct seq_file *m, void *data)
 
 	seq_printf(m, "pch: %d\n", INTEL_PCH_TYPE(i915));
 
-	intel_device_info_print_static(INTEL_INFO(i915), &p);
-	intel_device_info_print_runtime(RUNTIME_INFO(i915), &p);
+	intel_device_info_print(INTEL_INFO(i915), RUNTIME_INFO(i915), &p);
 	i915_print_iommu_status(i915, &p);
 	intel_gt_info_print(&to_gt(i915)->info, &p);
 	intel_driver_caps_print(&i915->caps, &p);
@@ -188,47 +187,47 @@ i915_debugfs_describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 			   stringify_page_sizes(vma->resource->page_sizes_gtt,
 						NULL, 0));
 		if (i915_vma_is_ggtt(vma) || i915_vma_is_dpt(vma)) {
-			switch (vma->ggtt_view.type) {
-			case I915_GGTT_VIEW_NORMAL:
+			switch (vma->gtt_view.type) {
+			case I915_GTT_VIEW_NORMAL:
 				seq_puts(m, ", normal");
 				break;
 
-			case I915_GGTT_VIEW_PARTIAL:
+			case I915_GTT_VIEW_PARTIAL:
 				seq_printf(m, ", partial [%08llx+%x]",
-					   vma->ggtt_view.partial.offset << PAGE_SHIFT,
-					   vma->ggtt_view.partial.size << PAGE_SHIFT);
+					   vma->gtt_view.partial.offset << PAGE_SHIFT,
+					   vma->gtt_view.partial.size << PAGE_SHIFT);
 				break;
 
-			case I915_GGTT_VIEW_ROTATED:
+			case I915_GTT_VIEW_ROTATED:
 				seq_printf(m, ", rotated [(%ux%u, src_stride=%u, dst_stride=%u, offset=%u), (%ux%u, src_stride=%u, dst_stride=%u, offset=%u)]",
-					   vma->ggtt_view.rotated.plane[0].width,
-					   vma->ggtt_view.rotated.plane[0].height,
-					   vma->ggtt_view.rotated.plane[0].src_stride,
-					   vma->ggtt_view.rotated.plane[0].dst_stride,
-					   vma->ggtt_view.rotated.plane[0].offset,
-					   vma->ggtt_view.rotated.plane[1].width,
-					   vma->ggtt_view.rotated.plane[1].height,
-					   vma->ggtt_view.rotated.plane[1].src_stride,
-					   vma->ggtt_view.rotated.plane[1].dst_stride,
-					   vma->ggtt_view.rotated.plane[1].offset);
+					   vma->gtt_view.rotated.plane[0].width,
+					   vma->gtt_view.rotated.plane[0].height,
+					   vma->gtt_view.rotated.plane[0].src_stride,
+					   vma->gtt_view.rotated.plane[0].dst_stride,
+					   vma->gtt_view.rotated.plane[0].offset,
+					   vma->gtt_view.rotated.plane[1].width,
+					   vma->gtt_view.rotated.plane[1].height,
+					   vma->gtt_view.rotated.plane[1].src_stride,
+					   vma->gtt_view.rotated.plane[1].dst_stride,
+					   vma->gtt_view.rotated.plane[1].offset);
 				break;
 
-			case I915_GGTT_VIEW_REMAPPED:
+			case I915_GTT_VIEW_REMAPPED:
 				seq_printf(m, ", remapped [(%ux%u, src_stride=%u, dst_stride=%u, offset=%u), (%ux%u, src_stride=%u, dst_stride=%u, offset=%u)]",
-					   vma->ggtt_view.remapped.plane[0].width,
-					   vma->ggtt_view.remapped.plane[0].height,
-					   vma->ggtt_view.remapped.plane[0].src_stride,
-					   vma->ggtt_view.remapped.plane[0].dst_stride,
-					   vma->ggtt_view.remapped.plane[0].offset,
-					   vma->ggtt_view.remapped.plane[1].width,
-					   vma->ggtt_view.remapped.plane[1].height,
-					   vma->ggtt_view.remapped.plane[1].src_stride,
-					   vma->ggtt_view.remapped.plane[1].dst_stride,
-					   vma->ggtt_view.remapped.plane[1].offset);
+					   vma->gtt_view.remapped.plane[0].width,
+					   vma->gtt_view.remapped.plane[0].height,
+					   vma->gtt_view.remapped.plane[0].src_stride,
+					   vma->gtt_view.remapped.plane[0].dst_stride,
+					   vma->gtt_view.remapped.plane[0].offset,
+					   vma->gtt_view.remapped.plane[1].width,
+					   vma->gtt_view.remapped.plane[1].height,
+					   vma->gtt_view.remapped.plane[1].src_stride,
+					   vma->gtt_view.remapped.plane[1].dst_stride,
+					   vma->gtt_view.remapped.plane[1].offset);
 				break;
 
 			default:
-				MISSING_CASE(vma->ggtt_view.type);
+				MISSING_CASE(vma->gtt_view.type);
 				break;
 			}
 		}
@@ -411,7 +410,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
 	seq_printf(m, "bit6 swizzle for Y-tiling = %s\n",
 		   swizzle_string(to_gt(dev_priv)->ggtt->bit_6_swizzle_y));
 
-	if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
+	if (dev_priv->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES)
 		seq_puts(m, "L-shaped memory detected\n");
 
 	/* On BDW+, swizzling is not used. See detect_bit_6_swizzle() */
@@ -493,7 +492,7 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused)
 		seq_puts(m, "Runtime power management not supported\n");
 
 	seq_printf(m, "Runtime power status: %s\n",
-		   str_enabled_disabled(!dev_priv->power_domains.init_wakeref));
+		   str_enabled_disabled(!dev_priv->display.power.domains.init_wakeref));
 
 	seq_printf(m, "GPU idle: %s\n", str_yes_no(!to_gt(dev_priv)->awake));
 	seq_printf(m, "IRQs disabled: %s\n",
diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c
index deb8a8b76965..c459eb362c47 100644
--- a/drivers/gpu/drm/i915/i915_driver.c
+++ b/drivers/gpu/drm/i915/i915_driver.c
@@ -61,6 +61,7 @@
 #include "display/intel_pps.h"
 #include "display/intel_sprite.h"
 #include "display/intel_vga.h"
+#include "display/skl_watermark.h"
 
 #include "gem/i915_gem_context.h"
 #include "gem/i915_gem_create.h"
@@ -105,6 +106,12 @@ static const char irst_name[] = "INT3392";
 
 static const struct drm_driver i915_drm_driver;
 
+static void i915_release_bridge_dev(struct drm_device *dev,
+				    void *bridge)
+{
+	pci_dev_put(bridge);
+}
+
 static int i915_get_bridge_dev(struct drm_i915_private *dev_priv)
 {
 	int domain = pci_domain_nr(to_pci_dev(dev_priv->drm.dev)->bus);
@@ -115,7 +122,9 @@ static int i915_get_bridge_dev(struct drm_i915_private *dev_priv)
 		drm_err(&dev_priv->drm, "bridge device not found\n");
 		return -EIO;
 	}
-	return 0;
+
+	return drmm_add_action_or_reset(&dev_priv->drm, i915_release_bridge_dev,
+					dev_priv->bridge_dev);
 }
 
 /* Allocate space for the MCH regs if needed, return nonzero on error */
@@ -252,8 +261,8 @@ static int i915_workqueues_init(struct drm_i915_private *dev_priv)
 	if (dev_priv->wq == NULL)
 		goto out_err;
 
-	dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
-	if (dev_priv->hotplug.dp_wq == NULL)
+	dev_priv->display.hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
+	if (dev_priv->display.hotplug.dp_wq == NULL)
 		goto out_free_wq;
 
 	return 0;
@@ -268,7 +277,7 @@ out_err:
 
 static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
 {
-	destroy_workqueue(dev_priv->hotplug.dp_wq);
+	destroy_workqueue(dev_priv->display.hotplug.dp_wq);
 	destroy_workqueue(dev_priv->wq);
 }
 
@@ -302,8 +311,13 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
 
 static void sanitize_gpu(struct drm_i915_private *i915)
 {
-	if (!INTEL_INFO(i915)->gpu_reset_clobbers_display)
-		__intel_gt_reset(to_gt(i915), ALL_ENGINES);
+	if (!INTEL_INFO(i915)->gpu_reset_clobbers_display) {
+		struct intel_gt *gt;
+		unsigned int i;
+
+		for_each_gt(gt, i915, i)
+			__intel_gt_reset(gt, ALL_ENGINES);
+	}
 }
 
 /**
@@ -326,19 +340,19 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
 	intel_device_info_subplatform_init(dev_priv);
 	intel_step_init(dev_priv);
 
-	intel_uncore_mmio_debug_init_early(&dev_priv->mmio_debug);
+	intel_uncore_mmio_debug_init_early(dev_priv);
 
 	spin_lock_init(&dev_priv->irq_lock);
 	spin_lock_init(&dev_priv->gpu_error.lock);
-	mutex_init(&dev_priv->backlight_lock);
+	mutex_init(&dev_priv->display.backlight.lock);
 
 	mutex_init(&dev_priv->sb_lock);
 	cpu_latency_qos_add_request(&dev_priv->sb_qos, PM_QOS_DEFAULT_VALUE);
 
-	mutex_init(&dev_priv->audio.mutex);
-	mutex_init(&dev_priv->wm.wm_mutex);
-	mutex_init(&dev_priv->pps_mutex);
-	mutex_init(&dev_priv->hdcp_comp_mutex);
+	mutex_init(&dev_priv->display.audio.mutex);
+	mutex_init(&dev_priv->display.wm.wm_mutex);
+	mutex_init(&dev_priv->display.pps.mutex);
+	mutex_init(&dev_priv->display.hdcp.comp_mutex);
 
 	i915_memcpy_init_early(dev_priv);
 	intel_runtime_pm_init_early(&dev_priv->runtime_pm);
@@ -357,7 +371,9 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
 
 	intel_wopcm_init_early(&dev_priv->wopcm);
 
-	intel_root_gt_init_early(dev_priv);
+	ret = intel_root_gt_init_early(dev_priv);
+	if (ret < 0)
+		goto err_rootgt;
 
 	i915_drm_clients_init(&dev_priv->clients, dev_priv);
 
@@ -382,6 +398,7 @@ err_gem:
 	i915_gem_cleanup_early(dev_priv);
 	intel_gt_driver_late_release_all(dev_priv);
 	i915_drm_clients_fini(&dev_priv->clients);
+err_rootgt:
 	intel_region_ttm_device_fini(dev_priv);
 err_ttm:
 	vlv_suspend_cleanup(dev_priv);
@@ -423,7 +440,8 @@ static void i915_driver_late_release(struct drm_i915_private *dev_priv)
  */
 static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
 {
-	int ret;
+	struct intel_gt *gt;
+	int ret, i;
 
 	if (i915_inject_probe_failure(dev_priv))
 		return -ENODEV;
@@ -432,17 +450,27 @@ static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
 	if (ret < 0)
 		return ret;
 
-	ret = intel_uncore_init_mmio(&dev_priv->uncore);
-	if (ret)
-		return ret;
+	for_each_gt(gt, dev_priv, i) {
+		ret = intel_uncore_init_mmio(gt->uncore);
+		if (ret)
+			return ret;
+
+		ret = drmm_add_action_or_reset(&dev_priv->drm,
+					       intel_uncore_fini_mmio,
+					       gt->uncore);
+		if (ret)
+			return ret;
+	}
 
 	/* Try to make sure MCHBAR is enabled before poking at it */
 	intel_setup_mchbar(dev_priv);
 	intel_device_info_runtime_init(dev_priv);
 
-	ret = intel_gt_init_mmio(to_gt(dev_priv));
-	if (ret)
-		goto err_uncore;
+	for_each_gt(gt, dev_priv, i) {
+		ret = intel_gt_init_mmio(gt);
+		if (ret)
+			goto err_uncore;
+	}
 
 	/* As early as possible, scrub existing GPU state before clobbering */
 	sanitize_gpu(dev_priv);
@@ -451,8 +479,6 @@ static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
 
 err_uncore:
 	intel_teardown_mchbar(dev_priv);
-	intel_uncore_fini_mmio(&dev_priv->uncore);
-	pci_dev_put(dev_priv->bridge_dev);
 
 	return ret;
 }
@@ -464,8 +490,6 @@ err_uncore:
 static void i915_driver_mmio_release(struct drm_i915_private *dev_priv)
 {
 	intel_teardown_mchbar(dev_priv);
-	intel_uncore_fini_mmio(&dev_priv->uncore);
-	pci_dev_put(dev_priv->bridge_dev);
 }
 
 /**
@@ -715,6 +739,8 @@ static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)
 static void i915_driver_register(struct drm_i915_private *dev_priv)
 {
 	struct drm_device *dev = &dev_priv->drm;
+	struct intel_gt *gt;
+	unsigned int i;
 
 	i915_gem_driver_register(dev_priv);
 	i915_pmu_register(dev_priv);
@@ -734,7 +760,8 @@ static void i915_driver_register(struct drm_i915_private *dev_priv)
 	/* Depends on sysfs having been initialized */
 	i915_perf_register(dev_priv);
 
-	intel_gt_driver_register(to_gt(dev_priv));
+	for_each_gt(gt, dev_priv, i)
+		intel_gt_driver_register(gt);
 
 	intel_display_driver_register(dev_priv);
 
@@ -753,6 +780,9 @@ static void i915_driver_register(struct drm_i915_private *dev_priv)
  */
 static void i915_driver_unregister(struct drm_i915_private *dev_priv)
 {
+	struct intel_gt *gt;
+	unsigned int i;
+
 	i915_switcheroo_unregister(dev_priv);
 
 	intel_unregister_dsm_handler();
@@ -762,7 +792,8 @@ static void i915_driver_unregister(struct drm_i915_private *dev_priv)
 
 	intel_display_driver_unregister(dev_priv);
 
-	intel_gt_driver_unregister(to_gt(dev_priv));
+	for_each_gt(gt, dev_priv, i)
+		intel_gt_driver_unregister(gt);
 
 	i915_perf_unregister(dev_priv);
 	i915_pmu_unregister(dev_priv);
@@ -784,6 +815,8 @@ static void i915_welcome_messages(struct drm_i915_private *dev_priv)
 {
 	if (drm_debug_enabled(DRM_UT_DRIVER)) {
 		struct drm_printer p = drm_debug_printer("i915 device info:");
+		struct intel_gt *gt;
+		unsigned int i;
 
 		drm_printf(&p, "pciid=0x%04x rev=0x%02x platform=%s (subplatform=0x%x) gen=%i\n",
 			   INTEL_DEVID(dev_priv),
@@ -793,10 +826,11 @@ static void i915_welcome_messages(struct drm_i915_private *dev_priv)
 					     INTEL_INFO(dev_priv)->platform),
 			   GRAPHICS_VER(dev_priv));
 
-		intel_device_info_print_static(INTEL_INFO(dev_priv), &p);
-		intel_device_info_print_runtime(RUNTIME_INFO(dev_priv), &p);
+		intel_device_info_print(INTEL_INFO(dev_priv),
+					RUNTIME_INFO(dev_priv), &p);
 		i915_print_iommu_status(dev_priv, &p);
-		intel_gt_info_print(&to_gt(dev_priv)->info, &p);
+		for_each_gt(gt, dev_priv, i)
+			intel_gt_info_print(&gt->info, &p);
 	}
 
 	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG))
@@ -814,6 +848,7 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent)
 	const struct intel_device_info *match_info =
 		(struct intel_device_info *)ent->driver_data;
 	struct intel_device_info *device_info;
+	struct intel_runtime_info *runtime;
 	struct drm_i915_private *i915;
 
 	i915 = devm_drm_dev_alloc(&pdev->dev, &i915_drm_driver,
@@ -829,7 +864,11 @@ i915_driver_create(struct pci_dev *pdev, const struct pci_device_id *ent)
 	/* Setup the write-once "constant" device info */
 	device_info = mkwrite_device_info(i915);
 	memcpy(device_info, match_info, sizeof(*device_info));
-	RUNTIME_INFO(i915)->device_id = pdev->device;
+
+	/* Initialize initial runtime info from static const data and pdev. */
+	runtime = RUNTIME_INFO(i915);
+	memcpy(runtime, &INTEL_INFO(i915)->__runtime, sizeof(*runtime));
+	runtime->device_id = pdev->device;
 
 	return i915;
 }
@@ -948,7 +987,9 @@ out_fini:
 
 void i915_driver_remove(struct drm_i915_private *i915)
 {
-	disable_rpm_wakeref_asserts(&i915->runtime_pm);
+	intel_wakeref_t wakeref;
+
+	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 
 	i915_driver_unregister(i915);
 
@@ -972,18 +1013,19 @@ void i915_driver_remove(struct drm_i915_private *i915)
 
 	i915_driver_hw_remove(i915);
 
-	enable_rpm_wakeref_asserts(&i915->runtime_pm);
+	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 }
 
 static void i915_driver_release(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
+	intel_wakeref_t wakeref;
 
 	if (!dev_priv->do_release)
 		return;
 
-	disable_rpm_wakeref_asserts(rpm);
+	wakeref = intel_runtime_pm_get(rpm);
 
 	i915_gem_driver_release(dev_priv);
 
@@ -994,7 +1036,8 @@ static void i915_driver_release(struct drm_device *dev)
 
 	i915_driver_mmio_release(dev_priv);
 
-	enable_rpm_wakeref_asserts(rpm);
+	intel_runtime_pm_put(rpm, wakeref);
+
 	intel_runtime_pm_driver_release(rpm);
 
 	i915_driver_late_release(dev_priv);
@@ -1206,13 +1249,15 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
-	int ret;
+	struct intel_gt *gt;
+	int ret, i;
 
 	disable_rpm_wakeref_asserts(rpm);
 
 	i915_gem_suspend_late(dev_priv);
 
-	intel_uncore_suspend(&dev_priv->uncore);
+	for_each_gt(gt, dev_priv, i)
+		intel_uncore_suspend(gt->uncore);
 
 	intel_power_domains_suspend(dev_priv,
 				    get_suspend_mode(dev_priv, hibernation));
@@ -1344,7 +1389,8 @@ static int i915_drm_resume_early(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
-	int ret;
+	struct intel_gt *gt;
+	int ret, i;
 
 	/*
 	 * We have a resume ordering issue with the snd-hda driver also
@@ -1398,9 +1444,10 @@ static int i915_drm_resume_early(struct drm_device *dev)
 		drm_err(&dev_priv->drm,
 			"Resume prepare failed: %d, continuing anyway\n", ret);
 
-	intel_uncore_resume_early(&dev_priv->uncore);
-
-	intel_gt_check_and_clear_faults(to_gt(dev_priv));
+	for_each_gt(gt, dev_priv, i) {
+		intel_uncore_resume_early(gt->uncore);
+		intel_gt_check_and_clear_faults(gt);
+	}
 
 	intel_display_power_resume_early(dev_priv);
 
@@ -1580,7 +1627,8 @@ static int intel_runtime_suspend(struct device *kdev)
 {
 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
-	int ret;
+	struct intel_gt *gt;
+	int ret, i;
 
 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))
 		return -ENODEV;
@@ -1595,11 +1643,13 @@ static int intel_runtime_suspend(struct device *kdev)
 	 */
 	i915_gem_runtime_suspend(dev_priv);
 
-	intel_gt_runtime_suspend(to_gt(dev_priv));
+	for_each_gt(gt, dev_priv, i)
+		intel_gt_runtime_suspend(gt);
 
 	intel_runtime_pm_disable_interrupts(dev_priv);
 
-	intel_uncore_suspend(&dev_priv->uncore);
+	for_each_gt(gt, dev_priv, i)
+		intel_uncore_suspend(gt->uncore);
 
 	intel_display_power_suspend(dev_priv);
 
@@ -1663,7 +1713,8 @@ static int intel_runtime_resume(struct device *kdev)
 {
 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
-	int ret;
+	struct intel_gt *gt;
+	int ret, i;
 
 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))
 		return -ENODEV;
@@ -1683,7 +1734,8 @@ static int intel_runtime_resume(struct device *kdev)
 
 	ret = vlv_resume_prepare(dev_priv, true);
 
-	intel_uncore_runtime_resume(&dev_priv->uncore);
+	for_each_gt(gt, dev_priv, i)
+		intel_uncore_runtime_resume(gt->uncore);
 
 	intel_runtime_pm_enable_interrupts(dev_priv);
 
@@ -1691,7 +1743,8 @@ static int intel_runtime_resume(struct device *kdev)
 	 * No point of rolling back things in case of an error, as the best
 	 * we can do is to hope that things will still work (and disable RPM).
 	 */
-	intel_gt_runtime_resume(to_gt(dev_priv));
+	for_each_gt(gt, dev_priv, i)
+		intel_gt_runtime_resume(gt);
 
 	/*
 	 * On VLV/CHV display interrupts are part of the display
@@ -1703,7 +1756,7 @@ static int intel_runtime_resume(struct device *kdev)
 		intel_hpd_poll_disable(dev_priv);
 	}
 
-	intel_enable_ipc(dev_priv);
+	skl_watermark_ipc_update(dev_priv);
 
 	enable_rpm_wakeref_asserts(rpm);
 
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 086bbe8945d6..bdc81db76dbd 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -34,20 +34,10 @@
 
 #include <linux/pm_qos.h>
 
-#include <drm/drm_connector.h>
 #include <drm/ttm/ttm_device.h>
 
-#include "display/intel_cdclk.h"
 #include "display/intel_display.h"
-#include "display/intel_display_power.h"
-#include "display/intel_dmc.h"
-#include "display/intel_dpll_mgr.h"
-#include "display/intel_dsb.h"
-#include "display/intel_fbc.h"
-#include "display/intel_frontbuffer.h"
-#include "display/intel_global_state.h"
-#include "display/intel_gmbus.h"
-#include "display/intel_opregion.h"
+#include "display/intel_display_core.h"
 
 #include "gem/i915_gem_context_types.h"
 #include "gem/i915_gem_lmem.h"
@@ -70,80 +60,24 @@
 #include "intel_device_info.h"
 #include "intel_memory_region.h"
 #include "intel_pch.h"
-#include "intel_pm_types.h"
 #include "intel_runtime_pm.h"
 #include "intel_step.h"
 #include "intel_uncore.h"
 #include "intel_wopcm.h"
 
-struct dpll;
 struct drm_i915_clock_gating_funcs;
 struct drm_i915_gem_object;
 struct drm_i915_private;
-struct intel_atomic_state;
-struct intel_audio_funcs;
-struct intel_cdclk_config;
-struct intel_cdclk_funcs;
-struct intel_cdclk_state;
-struct intel_cdclk_vals;
-struct intel_color_funcs;
 struct intel_connector;
-struct intel_crtc;
 struct intel_dp;
-struct intel_dpll_funcs;
 struct intel_encoder;
-struct intel_fbdev;
-struct intel_fdi_funcs;
-struct intel_gmbus;
-struct intel_hotplug_funcs;
-struct intel_initial_plane_config;
 struct intel_limit;
-struct intel_overlay;
 struct intel_overlay_error_state;
 struct vlv_s0ix_state;
 
 /* Threshold == 5 for long IRQs, 50 for short */
 #define HPD_STORM_DEFAULT_THRESHOLD 50
 
-struct i915_hotplug {
-	struct delayed_work hotplug_work;
-
-	const u32 *hpd, *pch_hpd;
-
-	struct {
-		unsigned long last_jiffies;
-		int count;
-		enum {
-			HPD_ENABLED = 0,
-			HPD_DISABLED = 1,
-			HPD_MARK_DISABLED = 2
-		} state;
-	} stats[HPD_NUM_PINS];
-	u32 event_bits;
-	u32 retry_bits;
-	struct delayed_work reenable_work;
-
-	u32 long_port_mask;
-	u32 short_port_mask;
-	struct work_struct dig_port_work;
-
-	struct work_struct poll_init_work;
-	bool poll_enabled;
-
-	unsigned int hpd_storm_threshold;
-	/* Whether or not to count short HPD IRQs in HPD storms */
-	u8 hpd_short_storm_enabled;
-
-	/*
-	 * if we get a HPD irq from DP and a HPD irq from non-DP
-	 * the non-DP HPD could block the workqueue on a mode config
-	 * mutex getting, that userspace may have taken. However
-	 * userspace is waiting on the DP workqueue to run which is
-	 * blocked behind the non-DP one.
-	 */
-	struct workqueue_struct *dp_wq;
-};
-
 #define I915_GEM_GPU_DOMAINS \
 	(I915_GEM_DOMAIN_RENDER | \
 	 I915_GEM_DOMAIN_SAMPLER | \
@@ -151,55 +85,9 @@ struct i915_hotplug {
 	 I915_GEM_DOMAIN_INSTRUCTION | \
 	 I915_GEM_DOMAIN_VERTEX)
 
-struct sdvo_device_mapping {
-	u8 initialized;
-	u8 dvo_port;
-	u8 slave_addr;
-	u8 dvo_wiring;
-	u8 i2c_pin;
-	u8 ddc_pin;
-};
-
-/* functions used for watermark calcs for display. */
-struct drm_i915_wm_disp_funcs {
-	/* update_wm is for legacy wm management */
-	void (*update_wm)(struct drm_i915_private *dev_priv);
-	int (*compute_pipe_wm)(struct intel_atomic_state *state,
-			       struct intel_crtc *crtc);
-	int (*compute_intermediate_wm)(struct intel_atomic_state *state,
-				       struct intel_crtc *crtc);
-	void (*initial_watermarks)(struct intel_atomic_state *state,
-				   struct intel_crtc *crtc);
-	void (*atomic_update_watermarks)(struct intel_atomic_state *state,
-					 struct intel_crtc *crtc);
-	void (*optimize_watermarks)(struct intel_atomic_state *state,
-				    struct intel_crtc *crtc);
-	int (*compute_global_watermarks)(struct intel_atomic_state *state);
-};
-
-struct drm_i915_display_funcs {
-	/* Returns the active state of the crtc, and if the crtc is active,
-	 * fills out the pipe-config with the hw state. */
-	bool (*get_pipe_config)(struct intel_crtc *,
-				struct intel_crtc_state *);
-	void (*get_initial_plane_config)(struct intel_crtc *,
-					 struct intel_initial_plane_config *);
-	void (*crtc_enable)(struct intel_atomic_state *state,
-			    struct intel_crtc *crtc);
-	void (*crtc_disable)(struct intel_atomic_state *state,
-			     struct intel_crtc *crtc);
-	void (*commit_modeset_enables)(struct intel_atomic_state *state);
-};
-
 #define I915_COLOR_UNEVICTABLE (-1) /* a non-vma sharing the address space */
 
-#define QUIRK_LVDS_SSC_DISABLE (1<<1)
-#define QUIRK_INVERT_BRIGHTNESS (1<<2)
-#define QUIRK_BACKLIGHT_PRESENT (1<<3)
-#define QUIRK_PIN_SWIZZLED_PAGES (1<<5)
-#define QUIRK_INCREASE_T12_DELAY (1<<6)
-#define QUIRK_INCREASE_DDI_DISABLED_TIME (1<<7)
-#define QUIRK_NO_PPS_BACKLIGHT_POWER_HOOK (1<<8)
+#define GEM_QUIRK_PIN_SWIZZLED_PAGES	BIT(0)
 
 struct i915_suspend_saved_registers {
 	u32 saveDSPARB;
@@ -289,51 +177,8 @@ i915_fence_timeout(const struct drm_i915_private *i915)
 	return i915_fence_context_timeout(i915, U64_MAX);
 }
 
-/* Amount of SAGV/QGV points, BSpec precisely defines this */
-#define I915_NUM_QGV_POINTS 8
-
 #define HAS_HW_SAGV_WM(i915) (DISPLAY_VER(i915) >= 13 && !IS_DGFX(i915))
 
-/* Amount of PSF GV points, BSpec precisely defines this */
-#define I915_NUM_PSF_GV_POINTS 3
-
-struct intel_vbt_data {
-	/* bdb version */
-	u16 version;
-
-	/* Feature bits */
-	unsigned int int_tv_support:1;
-	unsigned int int_crt_support:1;
-	unsigned int lvds_use_ssc:1;
-	unsigned int int_lvds_support:1;
-	unsigned int display_clock_mode:1;
-	unsigned int fdi_rx_polarity_inverted:1;
-	int lvds_ssc_freq;
-	enum drm_panel_orientation orientation;
-
-	bool override_afc_startup;
-	u8 override_afc_startup_val;
-
-	int crt_ddc_pin;
-
-	struct list_head display_devices;
-	struct list_head bdb_blocks;
-
-	struct intel_bios_encoder_data *ports[I915_MAX_PORTS]; /* Non-NULL if port present. */
-	struct sdvo_device_mapping sdvo_mappings[2];
-};
-
-struct i915_frontbuffer_tracking {
-	spinlock_t lock;
-
-	/*
-	 * Tracking bits for delayed frontbuffer flushing du to gpu activity or
-	 * scheduled flips.
-	 */
-	unsigned busy_bits;
-	unsigned flip_bits;
-};
-
 struct i915_virtual_gpu {
 	struct mutex lock; /* serialises sending of g2v_notify command pkts */
 	bool active;
@@ -348,32 +193,11 @@ struct i915_selftest_stash {
 	struct ida mock_region_instances;
 };
 
-/* intel_audio.c private */
-struct intel_audio_private {
-	/* Display internal audio functions */
-	const struct intel_audio_funcs *funcs;
-
-	/* hda/i915 audio component */
-	struct i915_audio_component *component;
-	bool component_registered;
-	/* mutex for audio/video sync */
-	struct mutex mutex;
-	int power_refcount;
-	u32 freq_cntrl;
-
-	/* Used to save the pipe-to-encoder mapping for audio */
-	struct intel_encoder *encoder_map[I915_MAX_PIPES];
-
-	/* necessary resource sharing with HDMI LPE audio driver. */
-	struct {
-		struct platform_device *platdev;
-		int irq;
-	} lpe;
-};
-
 struct drm_i915_private {
 	struct drm_device drm;
 
+	struct intel_display display;
+
 	/* FIXME: Device release actions should all be moved to drmm_ */
 	bool do_release;
 
@@ -417,27 +241,6 @@ struct drm_i915_private {
 
 	struct intel_wopcm wopcm;
 
-	struct intel_dmc dmc;
-
-	struct intel_gmbus *gmbus[GMBUS_NUM_PINS];
-
-	/** gmbus_mutex protects against concurrent usage of the single hw gmbus
-	 * controller on different i2c buses. */
-	struct mutex gmbus_mutex;
-
-	/**
-	 * Base address of where the gmbus and gpio blocks are located (either
-	 * on PCH or on SoC for platforms without PCH).
-	 */
-	u32 gpio_mmio_base;
-
-	/* MMIO base address for MIPI regs */
-	u32 mipi_mmio_base;
-
-	u32 pps_mmio_base;
-
-	wait_queue_head_t gmbus_wait_queue;
-
 	struct pci_dev *bridge_dev;
 
 	struct rb_root uabi_engines;
@@ -461,48 +264,15 @@ struct drm_i915_private {
 	};
 	u32 pipestat_irq_mask[I915_MAX_PIPES];
 
-	struct i915_hotplug hotplug;
-	struct intel_fbc *fbc[I915_MAX_FBCS];
-	struct intel_opregion opregion;
-	struct intel_vbt_data vbt;
-
 	bool preserve_bios_swizzle;
 
-	/* overlay */
-	struct intel_overlay *overlay;
-
-	/* backlight registers and fields in struct intel_panel */
-	struct mutex backlight_lock;
-
-	/* protects panel power sequencer state */
-	struct mutex pps_mutex;
-
 	unsigned int fsb_freq, mem_freq, is_ddr3;
 	unsigned int skl_preferred_vco_freq;
-	unsigned int max_cdclk_freq;
 
 	unsigned int max_dotclk_freq;
 	unsigned int hpll_freq;
-	unsigned int fdi_pll_freq;
 	unsigned int czclk_freq;
 
-	struct {
-		/* The current hardware cdclk configuration */
-		struct intel_cdclk_config hw;
-
-		/* cdclk, divider, and ratio table from bspec */
-		const struct intel_cdclk_vals *table;
-
-		struct intel_global_obj obj;
-	} cdclk;
-
-	struct {
-		/* The current hardware dbuf configuration */
-		u8 enabled_slices;
-
-		struct intel_global_obj obj;
-	} dbuf;
-
 	/**
 	 * wq - Driver workqueue for GEM.
 	 *
@@ -512,40 +282,14 @@ struct drm_i915_private {
 	 */
 	struct workqueue_struct *wq;
 
-	/* ordered wq for modesets */
-	struct workqueue_struct *modeset_wq;
-	/* unbound hipri wq for page flips/plane updates */
-	struct workqueue_struct *flip_wq;
-
 	/* pm private clock gating functions */
 	const struct drm_i915_clock_gating_funcs *clock_gating_funcs;
 
-	/* pm display functions */
-	const struct drm_i915_wm_disp_funcs *wm_disp;
-
-	/* irq display functions */
-	const struct intel_hotplug_funcs *hotplug_funcs;
-
-	/* fdi display functions */
-	const struct intel_fdi_funcs *fdi_funcs;
-
-	/* display pll funcs */
-	const struct intel_dpll_funcs *dpll_funcs;
-
-	/* Display functions */
-	const struct drm_i915_display_funcs *display;
-
-	/* Display internal color functions */
-	const struct intel_color_funcs *color_funcs;
-
-	/* Display CDCLK functions */
-	const struct intel_cdclk_funcs *cdclk_funcs;
-
 	/* PCH chipset type */
 	enum intel_pch pch_type;
 	unsigned short pch_id;
 
-	unsigned long quirks;
+	unsigned long gem_quirks;
 
 	struct drm_atomic_state *modeset_restore_state;
 	struct drm_modeset_acquire_ctx reset_ctx;
@@ -554,34 +298,8 @@ struct drm_i915_private {
 
 	/* Kernel Modesetting */
 
-	/**
-	 * dpll and cdclk state is protected by connection_mutex
-	 * dpll.lock serializes intel_{prepare,enable,disable}_shared_dpll.
-	 * Must be global rather than per dpll, because on some platforms plls
-	 * share registers.
-	 */
-	struct {
-		struct mutex lock;
-
-		int num_shared_dpll;
-		struct intel_shared_dpll shared_dplls[I915_NUM_PLLS];
-		const struct intel_dpll_mgr *mgr;
-
-		struct {
-			int nssc;
-			int ssc;
-		} ref_clks;
-	} dpll;
-
 	struct list_head global_obj_list;
 
-	struct i915_frontbuffer_tracking fb_tracking;
-
-	struct intel_atomic_helper {
-		struct llist_head free_list;
-		struct work_struct free_work;
-	} atomic_helper;
-
 	bool mchbar_need_disable;
 
 	struct intel_l3_parity l3_parity;
@@ -600,21 +318,8 @@ struct drm_i915_private {
 	 */
 	u32 edram_size_mb;
 
-	struct i915_power_domains power_domains;
-
 	struct i915_gpu_error gpu_error;
 
-	/* list of fbdev register on this device */
-	struct intel_fbdev *fbdev;
-	struct work_struct fbdev_suspend_work;
-
-	struct drm_property *broadcast_rgb_property;
-	struct drm_property *force_audio_property;
-
-	u32 fdi_rx_config;
-
-	/* Shadow for DISPLAY_PHY_CONTROL which can't be safely read */
-	u32 chv_phy_control;
 	/*
 	 * Shadows for CHV DPLL_MD regs to keep the state
 	 * checker somewhat working in the presence hardware
@@ -627,51 +332,6 @@ struct drm_i915_private {
 	struct i915_suspend_saved_registers regfile;
 	struct vlv_s0ix_state *vlv_s0ix_state;
 
-	enum {
-		I915_SAGV_UNKNOWN = 0,
-		I915_SAGV_DISABLED,
-		I915_SAGV_ENABLED,
-		I915_SAGV_NOT_CONTROLLED
-	} sagv_status;
-
-	u32 sagv_block_time_us;
-
-	struct {
-		/*
-		 * Raw watermark latency values:
-		 * in 0.1us units for WM0,
-		 * in 0.5us units for WM1+.
-		 */
-		/* primary */
-		u16 pri_latency[5];
-		/* sprite */
-		u16 spr_latency[5];
-		/* cursor */
-		u16 cur_latency[5];
-		/*
-		 * Raw watermark memory latency values
-		 * for SKL for all 8 levels
-		 * in 1us units.
-		 */
-		u16 skl_latency[8];
-
-		/* current hardware state */
-		union {
-			struct ilk_wm_values hw;
-			struct vlv_wm_values vlv;
-			struct g4x_wm_values g4x;
-		};
-
-		u8 max_level;
-
-		/*
-		 * Should be held around atomic WM register writing; also
-		 * protects * intel_crtc->wm.active and
-		 * crtc_state->wm.need_postvbl_update.
-		 */
-		struct mutex wm_mutex;
-	} wm;
-
 	struct dram_info {
 		bool wm_lv_0_adjust_needed;
 		u8 num_channels;
@@ -689,18 +349,6 @@ struct drm_i915_private {
 		u8 num_psf_gv_points;
 	} dram_info;
 
-	struct intel_bw_info {
-		/* for each QGV point */
-		unsigned int deratedbw[I915_NUM_QGV_POINTS];
-		/* for each PSF GV point */
-		unsigned int psf_bw[I915_NUM_PSF_GV_POINTS];
-		u8 num_qgv_points;
-		u8 num_psf_gv_points;
-		u8 num_planes;
-	} max_bw[6];
-
-	struct intel_global_obj bw_obj;
-
 	struct intel_runtime_pm runtime_pm;
 
 	struct i915_perf perf;
@@ -716,6 +364,9 @@ struct drm_i915_private {
 
 	struct kobject *sysfs_gt;
 
+	/* Quick lookup of media GT (current platforms only have one) */
+	struct intel_gt *media_gt;
+
 	struct {
 		struct i915_gem_contexts {
 			spinlock_t lock; /* locks list */
@@ -733,9 +384,6 @@ struct drm_i915_private {
 		struct file *mmap_singleton;
 	} gem;
 
-	/* Window2 specifies time required to program DSB (Window2) in number of scan lines */
-	u8 window2_delay;
-
 	u8 pch_ssc_use;
 
 	/* For i915gm/i945gm vblank irq workaround */
@@ -743,31 +391,16 @@ struct drm_i915_private {
 
 	bool irq_enabled;
 
-	union {
-		/* perform PHY state sanity checks? */
-		bool chv_phy_assert[2];
-
-		/*
-		 * DG2: Mask of PHYs that were not calibrated by the firmware
-		 * and should not be used.
-		 */
-		u8 snps_phy_failed_calibration;
-	};
-
-	bool ipc_enabled;
-
-	struct intel_audio_private audio;
+	/*
+	 * DG2: Mask of PHYs that were not calibrated by the firmware
+	 * and should not be used.
+	 */
+	u8 snps_phy_failed_calibration;
 
 	struct i915_pmu pmu;
 
 	struct i915_drm_clients clients;
 
-	struct i915_hdcp_comp_master *hdcp_master;
-	bool hdcp_comp_added;
-
-	/* Mutex to protect the above hdcp component related values. */
-	struct mutex hdcp_comp_mutex;
-
 	/* The TTM device structure. */
 	struct ttm_device bdev;
 
@@ -826,28 +459,6 @@ static inline struct intel_gt *to_gt(struct drm_i915_private *i915)
 	     (engine__) && (engine__)->uabi_class == (class__); \
 	     (engine__) = rb_to_uabi_engine(rb_next(&(engine__)->uabi_node)))
 
-#define I915_GTT_OFFSET_NONE ((u32)-1)
-
-/*
- * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is
- * considered to be the frontbuffer for the given plane interface-wise. This
- * doesn't mean that the hw necessarily already scans it out, but that any
- * rendering (by the cpu or gpu) will land in the frontbuffer eventually.
- *
- * We have one bit per pipe and per scanout plane type.
- */
-#define INTEL_FRONTBUFFER_BITS_PER_PIPE 8
-#define INTEL_FRONTBUFFER(pipe, plane_id) ({ \
-	BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES > 32); \
-	BUILD_BUG_ON(I915_MAX_PLANES > INTEL_FRONTBUFFER_BITS_PER_PIPE); \
-	BIT((plane_id) + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe)); \
-})
-#define INTEL_FRONTBUFFER_OVERLAY(pipe) \
-	BIT(INTEL_FRONTBUFFER_BITS_PER_PIPE - 1 + INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))
-#define INTEL_FRONTBUFFER_ALL_MASK(pipe) \
-	GENMASK(INTEL_FRONTBUFFER_BITS_PER_PIPE * ((pipe) + 1) - 1, \
-		INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))
-
 #define INTEL_INFO(dev_priv)	(&(dev_priv)->__info)
 #define RUNTIME_INFO(dev_priv)	(&(dev_priv)->__runtime)
 #define DRIVER_CAPS(dev_priv)	(&(dev_priv)->caps)
@@ -856,19 +467,19 @@ static inline struct intel_gt *to_gt(struct drm_i915_private *i915)
 
 #define IP_VER(ver, rel)		((ver) << 8 | (rel))
 
-#define GRAPHICS_VER(i915)		(INTEL_INFO(i915)->graphics.ver)
-#define GRAPHICS_VER_FULL(i915)		IP_VER(INTEL_INFO(i915)->graphics.ver, \
-					       INTEL_INFO(i915)->graphics.rel)
+#define GRAPHICS_VER(i915)		(RUNTIME_INFO(i915)->graphics.ip.ver)
+#define GRAPHICS_VER_FULL(i915)		IP_VER(RUNTIME_INFO(i915)->graphics.ip.ver, \
+					       RUNTIME_INFO(i915)->graphics.ip.rel)
 #define IS_GRAPHICS_VER(i915, from, until) \
 	(GRAPHICS_VER(i915) >= (from) && GRAPHICS_VER(i915) <= (until))
 
-#define MEDIA_VER(i915)			(INTEL_INFO(i915)->media.ver)
-#define MEDIA_VER_FULL(i915)		IP_VER(INTEL_INFO(i915)->media.ver, \
-					       INTEL_INFO(i915)->media.rel)
+#define MEDIA_VER(i915)			(RUNTIME_INFO(i915)->media.ip.ver)
+#define MEDIA_VER_FULL(i915)		IP_VER(RUNTIME_INFO(i915)->media.ip.ver, \
+					       RUNTIME_INFO(i915)->media.ip.rel)
 #define IS_MEDIA_VER(i915, from, until) \
 	(MEDIA_VER(i915) >= (from) && MEDIA_VER(i915) <= (until))
 
-#define DISPLAY_VER(i915)	(INTEL_INFO(i915)->display.ver)
+#define DISPLAY_VER(i915)	(RUNTIME_INFO(i915)->display.ip.ver)
 #define IS_DISPLAY_VER(i915, from, until) \
 	(DISPLAY_VER(i915) >= (from) && DISPLAY_VER(i915) <= (until))
 
@@ -1210,7 +821,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 
 #define HAS_EXECLISTS(dev_priv) HAS_LOGICAL_RING_CONTEXTS(dev_priv)
 
-#define INTEL_PPGTT(dev_priv) (INTEL_INFO(dev_priv)->ppgtt_type)
+#define INTEL_PPGTT(dev_priv) (RUNTIME_INFO(dev_priv)->ppgtt_type)
 #define HAS_PPGTT(dev_priv) \
 	(INTEL_PPGTT(dev_priv) != INTEL_PPGTT_NONE)
 #define HAS_FULL_PPGTT(dev_priv) \
@@ -1218,7 +829,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 
 #define HAS_PAGE_SIZES(dev_priv, sizes) ({ \
 	GEM_BUG_ON((sizes) == 0); \
-	((sizes) & ~INTEL_INFO(dev_priv)->page_sizes) == 0; \
+	((sizes) & ~RUNTIME_INFO(dev_priv)->page_sizes) == 0; \
 })
 
 #define HAS_OVERLAY(dev_priv)		 (INTEL_INFO(dev_priv)->display.has_overlay)
@@ -1249,13 +860,15 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 #define I915_HAS_HOTPLUG(dev_priv)	(INTEL_INFO(dev_priv)->display.has_hotplug)
 
 #define HAS_FW_BLC(dev_priv)	(DISPLAY_VER(dev_priv) > 2)
-#define HAS_FBC(dev_priv)	(INTEL_INFO(dev_priv)->display.fbc_mask != 0)
+#define HAS_FBC(dev_priv)	(RUNTIME_INFO(dev_priv)->fbc_mask != 0)
 #define HAS_CUR_FBC(dev_priv)	(!HAS_GMCH(dev_priv) && DISPLAY_VER(dev_priv) >= 7)
 
 #define HAS_IPS(dev_priv)	(IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv))
 
 #define HAS_DP_MST(dev_priv)	(INTEL_INFO(dev_priv)->display.has_dp_mst)
-#define HAS_DP20(dev_priv)	(IS_DG2(dev_priv))
+#define HAS_DP20(dev_priv)	(IS_DG2(dev_priv) || DISPLAY_VER(dev_priv) >= 14)
+
+#define HAS_DOUBLE_BUFFERED_M_N(dev_priv)	(DISPLAY_VER(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
 
 #define HAS_CDCLK_CRAWL(dev_priv)	 (INTEL_INFO(dev_priv)->display.has_cdclk_crawl)
 #define HAS_DDI(dev_priv)		 (INTEL_INFO(dev_priv)->display.has_ddi)
@@ -1264,7 +877,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 #define HAS_PSR_HW_TRACKING(dev_priv) \
 	(INTEL_INFO(dev_priv)->display.has_psr_hw_tracking)
 #define HAS_PSR2_SEL_FETCH(dev_priv)	 (DISPLAY_VER(dev_priv) >= 12)
-#define HAS_TRANSCODER(dev_priv, trans)	 ((INTEL_INFO(dev_priv)->display.cpu_transcoder_mask & BIT(trans)) != 0)
+#define HAS_TRANSCODER(dev_priv, trans)	 ((RUNTIME_INFO(dev_priv)->cpu_transcoder_mask & BIT(trans)) != 0)
 
 #define HAS_RC6(dev_priv)		 (INTEL_INFO(dev_priv)->has_rc6)
 #define HAS_RC6p(dev_priv)		 (INTEL_INFO(dev_priv)->has_rc6p)
@@ -1272,7 +885,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 
 #define HAS_RPS(dev_priv)	(INTEL_INFO(dev_priv)->has_rps)
 
-#define HAS_DMC(dev_priv)	(INTEL_INFO(dev_priv)->display.has_dmc)
+#define HAS_DMC(dev_priv)	(RUNTIME_INFO(dev_priv)->has_dmc)
 
 #define HAS_HECI_PXP(dev_priv) \
 	(INTEL_INFO(dev_priv)->has_heci_pxp)
@@ -1302,9 +915,11 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 
 #define HAS_IPC(dev_priv)		 (INTEL_INFO(dev_priv)->display.has_ipc)
 
-#define HAS_REGION(i915, i) (INTEL_INFO(i915)->memory_regions & (i))
+#define HAS_REGION(i915, i) (RUNTIME_INFO(i915)->memory_regions & (i))
 #define HAS_LMEM(i915) HAS_REGION(i915, REGION_LMEM)
 
+#define HAS_EXTRA_GT_LIST(dev_priv)   (INTEL_INFO(dev_priv)->extra_gt_list)
+
 /*
  * Platform has the dedicated compression control state for each lmem surfaces
  * stored in lmem to support the 3D and media compression formats.
@@ -1313,7 +928,7 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 
 #define HAS_GT_UC(dev_priv)	(INTEL_INFO(dev_priv)->has_gt_uc)
 
-#define HAS_POOLED_EU(dev_priv)	(INTEL_INFO(dev_priv)->has_pooled_eu)
+#define HAS_POOLED_EU(dev_priv)	(RUNTIME_INFO(dev_priv)->has_pooled_eu)
 
 #define HAS_GLOBAL_MOCS_REGISTERS(dev_priv)	(INTEL_INFO(dev_priv)->has_global_mocs)
 
@@ -1335,9 +950,9 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 #define GT_FREQUENCY_MULTIPLIER 50
 #define GEN9_FREQ_SCALER 3
 
-#define INTEL_NUM_PIPES(dev_priv) (hweight8(INTEL_INFO(dev_priv)->display.pipe_mask))
+#define INTEL_NUM_PIPES(dev_priv) (hweight8(RUNTIME_INFO(dev_priv)->pipe_mask))
 
-#define HAS_DISPLAY(dev_priv) (INTEL_INFO(dev_priv)->display.pipe_mask != 0)
+#define HAS_DISPLAY(dev_priv) (RUNTIME_INFO(dev_priv)->pipe_mask != 0)
 
 #define HAS_VRR(i915)	(DISPLAY_VER(i915) >= 11)
 
@@ -1352,91 +967,15 @@ IS_SUBPLATFORM(const struct drm_i915_private *i915,
 #define HAS_GUC_DEPRIVILEGE(dev_priv) \
 	(INTEL_INFO(dev_priv)->has_guc_deprivilege)
 
-#define HAS_PERCTX_PREEMPT_CTRL(i915) \
-	((GRAPHICS_VER(i915) >= 9) &&  GRAPHICS_VER_FULL(i915) < IP_VER(12, 55))
-
 #define HAS_D12_PLANE_MINIMIZATION(dev_priv) (IS_ROCKETLAKE(dev_priv) || \
 					      IS_ALDERLAKE_S(dev_priv))
 
-#define HAS_MBUS_JOINING(i915) (IS_ALDERLAKE_P(i915))
+#define HAS_MBUS_JOINING(i915) (IS_ALDERLAKE_P(i915) || DISPLAY_VER(i915) >= 14)
 
 #define HAS_3D_PIPELINE(i915)	(INTEL_INFO(i915)->has_3d_pipeline)
 
 #define HAS_ONE_EU_PER_FUSE_BIT(i915)	(INTEL_INFO(i915)->has_one_eu_per_fuse_bit)
 
-/* i915_gem.c */
-void i915_gem_init_early(struct drm_i915_private *dev_priv);
-void i915_gem_cleanup_early(struct drm_i915_private *dev_priv);
-
-static inline void i915_gem_drain_freed_objects(struct drm_i915_private *i915)
-{
-	/*
-	 * A single pass should suffice to release all the freed objects (along
-	 * most call paths) , but be a little more paranoid in that freeing
-	 * the objects does take a little amount of time, during which the rcu
-	 * callbacks could have added new objects into the freed list, and
-	 * armed the work again.
-	 */
-	while (atomic_read(&i915->mm.free_count)) {
-		flush_work(&i915->mm.free_work);
-		flush_delayed_work(&i915->bdev.wq);
-		rcu_barrier();
-	}
-}
-
-static inline void i915_gem_drain_workqueue(struct drm_i915_private *i915)
-{
-	/*
-	 * Similar to objects above (see i915_gem_drain_freed-objects), in
-	 * general we have workers that are armed by RCU and then rearm
-	 * themselves in their callbacks. To be paranoid, we need to
-	 * drain the workqueue a second time after waiting for the RCU
-	 * grace period so that we catch work queued via RCU from the first
-	 * pass. As neither drain_workqueue() nor flush_workqueue() report
-	 * a result, we make an assumption that we only don't require more
-	 * than 3 passes to catch all _recursive_ RCU delayed work.
-	 *
-	 */
-	int pass = 3;
-	do {
-		flush_workqueue(i915->wq);
-		rcu_barrier();
-		i915_gem_drain_freed_objects(i915);
-	} while (--pass);
-	drain_workqueue(i915->wq);
-}
-
-struct i915_vma * __must_check
-i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
-			    struct i915_gem_ww_ctx *ww,
-			    const struct i915_ggtt_view *view,
-			    u64 size, u64 alignment, u64 flags);
-
-struct i915_vma * __must_check
-i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
-			 const struct i915_ggtt_view *view,
-			 u64 size, u64 alignment, u64 flags);
-
-int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
-			   unsigned long flags);
-#define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0)
-#define I915_GEM_OBJECT_UNBIND_BARRIER BIT(1)
-#define I915_GEM_OBJECT_UNBIND_TEST BIT(2)
-#define I915_GEM_OBJECT_UNBIND_VM_TRYLOCK BIT(3)
-#define I915_GEM_OBJECT_UNBIND_ASYNC BIT(4)
-
-void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv);
-
-int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno);
-
-int __must_check i915_gem_init(struct drm_i915_private *dev_priv);
-void i915_gem_driver_register(struct drm_i915_private *i915);
-void i915_gem_driver_unregister(struct drm_i915_private *i915);
-void i915_gem_driver_remove(struct drm_i915_private *dev_priv);
-void i915_gem_driver_release(struct drm_i915_private *dev_priv);
-
-int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file);
-
 /* intel_device_info.c */
 static inline struct intel_device_info *
 mkwrite_device_info(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index b605d0ceaefa..2bdddb61ebd7 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -842,6 +842,10 @@ void i915_gem_runtime_suspend(struct drm_i915_private *i915)
 				 &to_gt(i915)->ggtt->userfault_list, userfault_link)
 		__i915_gem_object_release_mmap_gtt(obj);
 
+	list_for_each_entry_safe(obj, on,
+				 &to_gt(i915)->lmem_userfault_list, userfault_link)
+		i915_gem_object_runtime_pm_release_mmap_offset(obj);
+
 	/*
 	 * The fence will be lost when the device powers down. If any were
 	 * in use by hardware (i.e. they are pinned), we should not be powering
@@ -885,7 +889,7 @@ static void discard_ggtt_vma(struct i915_vma *vma)
 struct i915_vma *
 i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
 			    struct i915_gem_ww_ctx *ww,
-			    const struct i915_ggtt_view *view,
+			    const struct i915_gtt_view *view,
 			    u64 size, u64 alignment, u64 flags)
 {
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
@@ -896,7 +900,7 @@ i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
 	GEM_WARN_ON(!ww);
 
 	if (flags & PIN_MAPPABLE &&
-	    (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
+	    (!view || view->type == I915_GTT_VIEW_NORMAL)) {
 		/*
 		 * If the required space is larger than the available
 		 * aperture, we will not able to find a slot for the
@@ -987,7 +991,7 @@ new_vma:
 
 struct i915_vma * __must_check
 i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
-			 const struct i915_ggtt_view *view,
+			 const struct i915_gtt_view *view,
 			 u64 size, u64 alignment, u64 flags)
 {
 	struct i915_gem_ww_ctx ww;
@@ -1035,7 +1039,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 
 	if (i915_gem_object_has_pages(obj) &&
 	    i915_gem_object_is_tiled(obj) &&
-	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
+	    i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {
 		if (obj->mm.madv == I915_MADV_WILLNEED) {
 			GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
 			i915_gem_object_clear_tiling_quirk(obj);
@@ -1085,14 +1089,50 @@ out:
 	return err;
 }
 
+/*
+ * A single pass should suffice to release all the freed objects (along most
+ * call paths), but be a little more paranoid in that freeing the objects does
+ * take a little amount of time, during which the rcu callbacks could have added
+ * new objects into the freed list, and armed the work again.
+ */
+void i915_gem_drain_freed_objects(struct drm_i915_private *i915)
+{
+	while (atomic_read(&i915->mm.free_count)) {
+		flush_work(&i915->mm.free_work);
+		flush_delayed_work(&i915->bdev.wq);
+		rcu_barrier();
+	}
+}
+
+/*
+ * Similar to objects above (see i915_gem_drain_freed-objects), in general we
+ * have workers that are armed by RCU and then rearm themselves in their
+ * callbacks. To be paranoid, we need to drain the workqueue a second time after
+ * waiting for the RCU grace period so that we catch work queued via RCU from
+ * the first pass. As neither drain_workqueue() nor flush_workqueue() report a
+ * result, we make an assumption that we only don't require more than 3 passes
+ * to catch all _recursive_ RCU delayed work.
+ */
+void i915_gem_drain_workqueue(struct drm_i915_private *i915)
+{
+	int i;
+
+	for (i = 0; i < 3; i++) {
+		flush_workqueue(i915->wq);
+		rcu_barrier();
+		i915_gem_drain_freed_objects(i915);
+	}
+
+	drain_workqueue(i915->wq);
+}
+
 int i915_gem_init(struct drm_i915_private *dev_priv)
 {
 	int ret;
 
 	/* We need to fallback to 4K pages if host doesn't support huge gtt. */
 	if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
-		mkwrite_device_info(dev_priv)->page_sizes =
-			I915_GTT_PAGE_SIZE_4K;
+		RUNTIME_INFO(dev_priv)->page_sizes = I915_GTT_PAGE_SIZE_4K;
 
 	ret = i915_gem_init_userptr(dev_priv);
 	if (ret)
@@ -1173,7 +1213,7 @@ void i915_gem_driver_unregister(struct drm_i915_private *i915)
 
 void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
 {
-	intel_wakeref_auto_fini(&to_gt(dev_priv)->ggtt->userfault_wakeref);
+	intel_wakeref_auto_fini(&to_gt(dev_priv)->userfault_wakeref);
 
 	i915_gem_suspend_late(dev_priv);
 	intel_gt_driver_remove(to_gt(dev_priv));
@@ -1214,7 +1254,7 @@ void i915_gem_init_early(struct drm_i915_private *dev_priv)
 	i915_gem_init__mm(dev_priv);
 	i915_gem_init__contexts(dev_priv);
 
-	spin_lock_init(&dev_priv->fb_tracking.lock);
+	spin_lock_init(&dev_priv->display.fb_tracking.lock);
 }
 
 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index 68d8d52bd541..a5cdf6662d01 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -26,12 +26,55 @@
 #define __I915_GEM_H__
 
 #include <linux/bug.h>
+#include <linux/types.h>
 
 #include <drm/drm_drv.h>
 
 #include "i915_utils.h"
 
+struct drm_file;
+struct drm_i915_gem_object;
 struct drm_i915_private;
+struct i915_gem_ww_ctx;
+struct i915_gtt_view;
+struct i915_vma;
+
+void i915_gem_init_early(struct drm_i915_private *i915);
+void i915_gem_cleanup_early(struct drm_i915_private *i915);
+
+void i915_gem_drain_freed_objects(struct drm_i915_private *i915);
+void i915_gem_drain_workqueue(struct drm_i915_private *i915);
+
+struct i915_vma * __must_check
+i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
+			    struct i915_gem_ww_ctx *ww,
+			    const struct i915_gtt_view *view,
+			    u64 size, u64 alignment, u64 flags);
+
+struct i915_vma * __must_check
+i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
+			 const struct i915_gtt_view *view,
+			 u64 size, u64 alignment, u64 flags);
+
+int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
+			   unsigned long flags);
+#define I915_GEM_OBJECT_UNBIND_ACTIVE BIT(0)
+#define I915_GEM_OBJECT_UNBIND_BARRIER BIT(1)
+#define I915_GEM_OBJECT_UNBIND_TEST BIT(2)
+#define I915_GEM_OBJECT_UNBIND_VM_TRYLOCK BIT(3)
+#define I915_GEM_OBJECT_UNBIND_ASYNC BIT(4)
+
+void i915_gem_runtime_suspend(struct drm_i915_private *i915);
+
+int __must_check i915_gem_init(struct drm_i915_private *i915);
+void i915_gem_driver_register(struct drm_i915_private *i915);
+void i915_gem_driver_unregister(struct drm_i915_private *i915);
+void i915_gem_driver_remove(struct drm_i915_private *i915);
+void i915_gem_driver_release(struct drm_i915_private *i915);
+
+int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file);
+
+/* FIXME: All of the below belong somewhere else. */
 
 #ifdef CONFIG_DRM_I915_DEBUG_GEM
 
diff --git a/drivers/gpu/drm/i915/i915_getparam.c b/drivers/gpu/drm/i915/i915_getparam.c
index 6fd15b39570c..342c8ca6414e 100644
--- a/drivers/gpu/drm/i915/i915_getparam.c
+++ b/drivers/gpu/drm/i915/i915_getparam.c
@@ -36,7 +36,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data,
 		value = to_gt(i915)->ggtt->num_fences;
 		break;
 	case I915_PARAM_HAS_OVERLAY:
-		value = !!i915->overlay;
+		value = !!i915->display.overlay;
 		break;
 	case I915_PARAM_HAS_BSD:
 		value = !!intel_engine_lookup_user(i915,
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 32e92651ef7c..9ea2fe34e7d3 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -646,8 +646,7 @@ static void err_print_capabilities(struct drm_i915_error_state_buf *m,
 {
 	struct drm_printer p = i915_error_printer(m);
 
-	intel_device_info_print_static(&error->device_info, &p);
-	intel_device_info_print_runtime(&error->runtime_info, &p);
+	intel_device_info_print(&error->device_info, &error->runtime_info, &p);
 	intel_driver_caps_print(&error->driver_caps, &p);
 }
 
@@ -671,6 +670,18 @@ static void err_print_pciid(struct drm_i915_error_state_buf *m,
 		   pdev->subsystem_device);
 }
 
+static void err_print_guc_ctb(struct drm_i915_error_state_buf *m,
+			      const char *name,
+			      const struct intel_ctb_coredump *ctb)
+{
+	if (!ctb->size)
+		return;
+
+	err_printf(m, "GuC %s CTB: raw: 0x%08X, 0x%08X/%08X, cached: 0x%08X/%08X, desc = 0x%08X, buf = 0x%08X x 0x%08X\n",
+		   name, ctb->raw_status, ctb->raw_head, ctb->raw_tail,
+		   ctb->head, ctb->tail, ctb->desc_offset, ctb->cmds_offset, ctb->size);
+}
+
 static void err_print_uc(struct drm_i915_error_state_buf *m,
 			 const struct intel_uc_coredump *error_uc)
 {
@@ -678,7 +689,12 @@ static void err_print_uc(struct drm_i915_error_state_buf *m,
 
 	intel_uc_fw_dump(&error_uc->guc_fw, &p);
 	intel_uc_fw_dump(&error_uc->huc_fw, &p);
-	intel_gpu_error_print_vma(m, NULL, error_uc->guc_log);
+	err_printf(m, "GuC timestamp: 0x%08x\n", error_uc->guc.timestamp);
+	intel_gpu_error_print_vma(m, NULL, error_uc->guc.vma_log);
+	err_printf(m, "GuC CTB fence: %d\n", error_uc->guc.last_fence);
+	err_print_guc_ctb(m, "Send", error_uc->guc.ctb + 0);
+	err_print_guc_ctb(m, "Recv", error_uc->guc.ctb + 1);
+	intel_gpu_error_print_vma(m, NULL, error_uc->guc.vma_ctb);
 }
 
 static void err_free_sgl(struct scatterlist *sgl)
@@ -720,6 +736,8 @@ static void err_print_gt_global_nonguc(struct drm_i915_error_state_buf *m,
 	int i;
 
 	err_printf(m, "GT awake: %s\n", str_yes_no(gt->awake));
+	err_printf(m, "CS timestamp frequency: %u Hz, %d ns\n",
+		   gt->clock_frequency, gt->clock_period_ns);
 	err_printf(m, "EIR: 0x%08x\n", gt->eir);
 	err_printf(m, "PGTBL_ER: 0x%08x\n", gt->pgtbl_er);
 
@@ -851,7 +869,7 @@ static void __err_print_to_sgl(struct drm_i915_error_state_buf *m,
 	if (error->gt) {
 		bool print_guc_capture = false;
 
-		if (error->gt->uc && error->gt->uc->is_guc_capture)
+		if (error->gt->uc && error->gt->uc->guc.is_guc_capture)
 			print_guc_capture = true;
 
 		err_print_gt_display(m, error->gt);
@@ -1004,9 +1022,12 @@ static void cleanup_params(struct i915_gpu_coredump *error)
 
 static void cleanup_uc(struct intel_uc_coredump *uc)
 {
-	kfree(uc->guc_fw.path);
-	kfree(uc->huc_fw.path);
-	i915_vma_coredump_free(uc->guc_log);
+	kfree(uc->guc_fw.file_selected.path);
+	kfree(uc->huc_fw.file_selected.path);
+	kfree(uc->guc_fw.file_wanted.path);
+	kfree(uc->huc_fw.file_wanted.path);
+	i915_vma_coredump_free(uc->guc.vma_log);
+	i915_vma_coredump_free(uc->guc.vma_ctb);
 
 	kfree(uc);
 }
@@ -1655,6 +1676,23 @@ gt_record_engines(struct intel_gt_coredump *gt,
 	}
 }
 
+static void gt_record_guc_ctb(struct intel_ctb_coredump *saved,
+			      const struct intel_guc_ct_buffer *ctb,
+			      const void *blob_ptr, struct intel_guc *guc)
+{
+	if (!ctb || !ctb->desc)
+		return;
+
+	saved->raw_status = ctb->desc->status;
+	saved->raw_head = ctb->desc->head;
+	saved->raw_tail = ctb->desc->tail;
+	saved->head = ctb->head;
+	saved->tail = ctb->tail;
+	saved->size = ctb->size;
+	saved->desc_offset = ((void *)ctb->desc) - blob_ptr;
+	saved->cmds_offset = ((void *)ctb->cmds) - blob_ptr;
+}
+
 static struct intel_uc_coredump *
 gt_record_uc(struct intel_gt_coredump *gt,
 	     struct i915_vma_compress *compress)
@@ -1669,14 +1707,26 @@ gt_record_uc(struct intel_gt_coredump *gt,
 	memcpy(&error_uc->guc_fw, &uc->guc.fw, sizeof(uc->guc.fw));
 	memcpy(&error_uc->huc_fw, &uc->huc.fw, sizeof(uc->huc.fw));
 
-	/* Non-default firmware paths will be specified by the modparam.
-	 * As modparams are generally accesible from the userspace make
-	 * explicit copies of the firmware paths.
+	error_uc->guc_fw.file_selected.path = kstrdup(uc->guc.fw.file_selected.path, ALLOW_FAIL);
+	error_uc->huc_fw.file_selected.path = kstrdup(uc->huc.fw.file_selected.path, ALLOW_FAIL);
+	error_uc->guc_fw.file_wanted.path = kstrdup(uc->guc.fw.file_wanted.path, ALLOW_FAIL);
+	error_uc->huc_fw.file_wanted.path = kstrdup(uc->huc.fw.file_wanted.path, ALLOW_FAIL);
+
+	/*
+	 * Save the GuC log and include a timestamp reference for converting the
+	 * log times to system times (in conjunction with the error->boottime and
+	 * gt->clock_frequency fields saved elsewhere).
 	 */
-	error_uc->guc_fw.path = kstrdup(uc->guc.fw.path, ALLOW_FAIL);
-	error_uc->huc_fw.path = kstrdup(uc->huc.fw.path, ALLOW_FAIL);
-	error_uc->guc_log = create_vma_coredump(gt->_gt, uc->guc.log.vma,
-						"GuC log buffer", compress);
+	error_uc->guc.timestamp = intel_uncore_read(gt->_gt->uncore, GUCPMTIMESTAMP);
+	error_uc->guc.vma_log = create_vma_coredump(gt->_gt, uc->guc.log.vma,
+						    "GuC log buffer", compress);
+	error_uc->guc.vma_ctb = create_vma_coredump(gt->_gt, uc->guc.ct.vma,
+						    "GuC CT buffer", compress);
+	error_uc->guc.last_fence = uc->guc.ct.requests.last_fence;
+	gt_record_guc_ctb(error_uc->guc.ctb + 0, &uc->guc.ct.ctbs.send,
+			  uc->guc.ct.ctbs.send.desc, (struct intel_guc *)&uc->guc);
+	gt_record_guc_ctb(error_uc->guc.ctb + 1, &uc->guc.ct.ctbs.recv,
+			  uc->guc.ct.ctbs.send.desc, (struct intel_guc *)&uc->guc);
 
 	return error_uc;
 }
@@ -1833,6 +1883,8 @@ static void gt_record_global_regs(struct intel_gt_coredump *gt)
 static void gt_record_info(struct intel_gt_coredump *gt)
 {
 	memcpy(&gt->info, &gt->_gt->info, sizeof(struct intel_gt_info));
+	gt->clock_frequency = gt->_gt->clock_frequency;
+	gt->clock_period_ns = gt->_gt->clock_period_ns;
 }
 
 /*
@@ -2027,9 +2079,9 @@ __i915_gpu_coredump(struct intel_gt *gt, intel_engine_mask_t engine_mask, u32 du
 			error->gt->uc = gt_record_uc(error->gt, compress);
 			if (error->gt->uc) {
 				if (dump_flags & CORE_DUMP_FLAG_IS_GUC_CAPTURE)
-					error->gt->uc->is_guc_capture = true;
+					error->gt->uc->guc.is_guc_capture = true;
 				else
-					GEM_BUG_ON(error->gt->uc->is_guc_capture);
+					GEM_BUG_ON(error->gt->uc->guc.is_guc_capture);
 			}
 		}
 
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h
index 55a143b92d10..efc75cc2ffdb 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.h
+++ b/drivers/gpu/drm/i915/i915_gpu_error.h
@@ -125,6 +125,15 @@ struct intel_engine_coredump {
 	struct intel_engine_coredump *next;
 };
 
+struct intel_ctb_coredump {
+	u32 raw_head, head;
+	u32 raw_tail, tail;
+	u32 raw_status;
+	u32 desc_offset;
+	u32 cmds_offset;
+	u32 size;
+};
+
 struct intel_gt_coredump {
 	const struct intel_gt *_gt;
 	bool awake;
@@ -150,6 +159,8 @@ struct intel_gt_coredump {
 	u32 gtt_cache;
 	u32 aux_err; /* gen12 */
 	u32 gam_done; /* gen12 */
+	u32 clock_frequency;
+	u32 clock_period_ns;
 
 	/* Display related */
 	u32 derrmr;
@@ -163,8 +174,14 @@ struct intel_gt_coredump {
 	struct intel_uc_coredump {
 		struct intel_uc_fw guc_fw;
 		struct intel_uc_fw huc_fw;
-		struct i915_vma_coredump *guc_log;
-		bool is_guc_capture;
+		struct guc_info {
+			struct intel_ctb_coredump ctb[2];
+			struct i915_vma_coredump *vma_ctb;
+			struct i915_vma_coredump *vma_log;
+			u32 timestamp;
+			u16 last_fence;
+			bool is_guc_capture;
+		} guc;
 	} *uc;
 
 	struct intel_gt_coredump *next;
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 73cebc6aa650..86a42d9e8041 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -65,7 +65,7 @@
 
 /*
  * Interrupt statistic for PMU. Increments the counter only if the
- * interrupt originated from the the GPU so interrupts from a device which
+ * interrupt originated from the GPU so interrupts from a device which
  * shares the interrupt line are not accounted.
  */
 static inline void pmu_irq_stats(struct drm_i915_private *i915,
@@ -185,7 +185,7 @@ static const u32 hpd_sde_dg1[HPD_NUM_PINS] = {
 
 static void intel_hpd_init_pins(struct drm_i915_private *dev_priv)
 {
-	struct i915_hotplug *hpd = &dev_priv->hotplug;
+	struct intel_hotplug *hpd = &dev_priv->display.hotplug;
 
 	if (HAS_GMCH(dev_priv)) {
 		if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
@@ -595,7 +595,7 @@ void i915_disable_pipestat(struct drm_i915_private *dev_priv,
 
 static bool i915_has_asle(struct drm_i915_private *dev_priv)
 {
-	if (!dev_priv->opregion.asle)
+	if (!dev_priv->display.opregion.asle)
 		return false;
 
 	return IS_PINEVIEW(dev_priv) || IS_MOBILE(dev_priv);
@@ -1104,9 +1104,9 @@ static void ivb_parity_work(struct work_struct *work)
 
 out:
 	drm_WARN_ON(&dev_priv->drm, dev_priv->l3_parity.which_slice);
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	gen5_gt_enable_irq(gt, GT_PARITY_ERROR(dev_priv));
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 
 	mutex_unlock(&dev_priv->drm.struct_mutex);
 }
@@ -1272,7 +1272,7 @@ static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
 	u32 enabled_irqs = 0;
 
 	for_each_intel_encoder(&dev_priv->drm, encoder)
-		if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
+		if (dev_priv->display.hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
 			enabled_irqs |= hpd[encoder->hpd_pin];
 
 	return enabled_irqs;
@@ -1304,12 +1304,12 @@ static u32 intel_hpd_hotplug_enables(struct drm_i915_private *i915,
 
 static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
 {
-	wake_up_all(&dev_priv->gmbus_wait_queue);
+	wake_up_all(&dev_priv->display.gmbus.wait_queue);
 }
 
 static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
 {
-	wake_up_all(&dev_priv->gmbus_wait_queue);
+	wake_up_all(&dev_priv->display.gmbus.wait_queue);
 }
 
 #if defined(CONFIG_DEBUG_FS)
@@ -1637,7 +1637,7 @@ static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
 	if (hotplug_trigger) {
 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
 				   hotplug_trigger, hotplug_trigger,
-				   dev_priv->hotplug.hpd,
+				   dev_priv->display.hotplug.hpd,
 				   i9xx_port_hotplug_long_detect);
 
 		intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
@@ -1841,7 +1841,7 @@ static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
 
 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
 			   hotplug_trigger, dig_hotplug_reg,
-			   dev_priv->hotplug.pch_hpd,
+			   dev_priv->display.hotplug.pch_hpd,
 			   pch_port_hotplug_long_detect);
 
 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
@@ -1986,7 +1986,7 @@ static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 
 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
 				   ddi_hotplug_trigger, dig_hotplug_reg,
-				   dev_priv->hotplug.pch_hpd,
+				   dev_priv->display.hotplug.pch_hpd,
 				   icp_ddi_port_hotplug_long_detect);
 	}
 
@@ -1998,7 +1998,7 @@ static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 
 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
 				   tc_hotplug_trigger, dig_hotplug_reg,
-				   dev_priv->hotplug.pch_hpd,
+				   dev_priv->display.hotplug.pch_hpd,
 				   icp_tc_port_hotplug_long_detect);
 	}
 
@@ -2024,7 +2024,7 @@ static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 
 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
 				   hotplug_trigger, dig_hotplug_reg,
-				   dev_priv->hotplug.pch_hpd,
+				   dev_priv->display.hotplug.pch_hpd,
 				   spt_port_hotplug_long_detect);
 	}
 
@@ -2036,7 +2036,7 @@ static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 
 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
 				   hotplug2_trigger, dig_hotplug_reg,
-				   dev_priv->hotplug.pch_hpd,
+				   dev_priv->display.hotplug.pch_hpd,
 				   spt_port_hotplug2_long_detect);
 	}
 
@@ -2057,7 +2057,7 @@ static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
 
 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
 			   hotplug_trigger, dig_hotplug_reg,
-			   dev_priv->hotplug.hpd,
+			   dev_priv->display.hotplug.hpd,
 			   ilk_port_hotplug_long_detect);
 
 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
@@ -2237,7 +2237,7 @@ static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
 
 	intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
 			   hotplug_trigger, dig_hotplug_reg,
-			   dev_priv->hotplug.hpd,
+			   dev_priv->display.hotplug.hpd,
 			   bxt_port_hotplug_long_detect);
 
 	intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
@@ -2257,7 +2257,7 @@ static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
 
 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
 				   trigger_tc, dig_hotplug_reg,
-				   dev_priv->hotplug.hpd,
+				   dev_priv->display.hotplug.hpd,
 				   gen11_port_hotplug_long_detect);
 	}
 
@@ -2269,7 +2269,7 @@ static void gen11_hpd_irq_handler(struct drm_i915_private *dev_priv, u32 iir)
 
 		intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
 				   trigger_tbt, dig_hotplug_reg,
-				   dev_priv->hotplug.hpd,
+				   dev_priv->display.hotplug.hpd,
 				   gen11_port_hotplug_long_detect);
 	}
 
@@ -2653,9 +2653,9 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
 }
 
 static u32
-gen11_gu_misc_irq_ack(struct intel_gt *gt, const u32 master_ctl)
+gen11_gu_misc_irq_ack(struct drm_i915_private *i915, const u32 master_ctl)
 {
-	void __iomem * const regs = gt->uncore->regs;
+	void __iomem * const regs = i915->uncore.regs;
 	u32 iir;
 
 	if (!(master_ctl & GEN11_GU_MISC_IRQ))
@@ -2669,10 +2669,10 @@ gen11_gu_misc_irq_ack(struct intel_gt *gt, const u32 master_ctl)
 }
 
 static void
-gen11_gu_misc_irq_handler(struct intel_gt *gt, const u32 iir)
+gen11_gu_misc_irq_handler(struct drm_i915_private *i915, const u32 iir)
 {
 	if (iir & GEN11_GU_MISC_GSE)
-		intel_opregion_asle_intr(gt->i915);
+		intel_opregion_asle_intr(i915);
 }
 
 static inline u32 gen11_master_intr_disable(void __iomem * const regs)
@@ -2736,11 +2736,11 @@ static irqreturn_t gen11_irq_handler(int irq, void *arg)
 	if (master_ctl & GEN11_DISPLAY_IRQ)
 		gen11_display_irq_handler(i915);
 
-	gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl);
+	gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl);
 
 	gen11_master_intr_enable(regs);
 
-	gen11_gu_misc_irq_handler(gt, gu_misc_iir);
+	gen11_gu_misc_irq_handler(i915, gu_misc_iir);
 
 	pmu_irq_stats(i915, IRQ_HANDLED);
 
@@ -2801,11 +2801,11 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg)
 	if (master_ctl & GEN11_DISPLAY_IRQ)
 		gen11_display_irq_handler(i915);
 
-	gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl);
+	gu_misc_iir = gen11_gu_misc_irq_ack(i915, master_ctl);
 
 	dg1_master_intr_enable(regs);
 
-	gen11_gu_misc_irq_handler(gt, gu_misc_iir);
+	gen11_gu_misc_irq_handler(i915, gu_misc_iir);
 
 	pmu_irq_stats(i915, IRQ_HANDLED);
 
@@ -3313,8 +3313,8 @@ static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
 {
 	u32 hotplug_irqs, enabled_irqs;
 
-	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
-	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
+	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd);
+	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd);
 
 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
 
@@ -3383,8 +3383,8 @@ static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv)
 {
 	u32 hotplug_irqs, enabled_irqs;
 
-	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
-	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
+	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd);
+	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd);
 
 	if (INTEL_PCH_TYPE(dev_priv) <= PCH_TGP)
 		intel_uncore_write(&dev_priv->uncore, SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
@@ -3460,8 +3460,8 @@ static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
 	u32 hotplug_irqs, enabled_irqs;
 	u32 val;
 
-	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
-	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
+	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.hpd);
+	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.hpd);
 
 	val = intel_uncore_read(&dev_priv->uncore, GEN11_DE_HPD_IMR);
 	val &= ~hotplug_irqs;
@@ -3538,8 +3538,8 @@ static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
 	if (INTEL_PCH_TYPE(dev_priv) >= PCH_CNP)
 		intel_uncore_write(&dev_priv->uncore, SHPD_FILTER_CNT, SHPD_FILTER_CNT_500_ADJ);
 
-	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
-	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.pch_hpd);
+	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd);
+	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.pch_hpd);
 
 	ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
 
@@ -3578,8 +3578,8 @@ static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
 {
 	u32 hotplug_irqs, enabled_irqs;
 
-	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
-	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
+	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.hpd);
+	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.hpd);
 
 	if (DISPLAY_VER(dev_priv) >= 8)
 		bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
@@ -3636,8 +3636,8 @@ static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
 {
 	u32 hotplug_irqs, enabled_irqs;
 
-	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->hotplug.hpd);
-	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->hotplug.hpd);
+	enabled_irqs = intel_hpd_enabled_irqs(dev_priv, dev_priv->display.hotplug.hpd);
+	hotplug_irqs = intel_hpd_hotplug_irqs(dev_priv, dev_priv->display.hotplug.hpd);
 
 	bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
 
@@ -4370,8 +4370,8 @@ HPD_FUNCS(ilk);
 
 void intel_hpd_irq_setup(struct drm_i915_private *i915)
 {
-	if (i915->display_irqs_enabled && i915->hotplug_funcs)
-		i915->hotplug_funcs->hpd_irq_setup(i915);
+	if (i915->display_irqs_enabled && i915->display.funcs.hotplug)
+		i915->display.funcs.hotplug->hpd_irq_setup(i915);
 }
 
 /**
@@ -4413,33 +4413,33 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 		dev_priv->display_irqs_enabled = false;
 
-	dev_priv->hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
+	dev_priv->display.hotplug.hpd_storm_threshold = HPD_STORM_DEFAULT_THRESHOLD;
 	/* If we have MST support, we want to avoid doing short HPD IRQ storm
 	 * detection, as short HPD storms will occur as a natural part of
 	 * sideband messaging with MST.
 	 * On older platforms however, IRQ storms can occur with both long and
 	 * short pulses, as seen on some G4x systems.
 	 */
-	dev_priv->hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv);
+	dev_priv->display.hotplug.hpd_short_storm_enabled = !HAS_DP_MST(dev_priv);
 
 	if (HAS_GMCH(dev_priv)) {
 		if (I915_HAS_HOTPLUG(dev_priv))
-			dev_priv->hotplug_funcs = &i915_hpd_funcs;
+			dev_priv->display.funcs.hotplug = &i915_hpd_funcs;
 	} else {
 		if (HAS_PCH_DG2(dev_priv))
-			dev_priv->hotplug_funcs = &icp_hpd_funcs;
+			dev_priv->display.funcs.hotplug = &icp_hpd_funcs;
 		else if (HAS_PCH_DG1(dev_priv))
-			dev_priv->hotplug_funcs = &dg1_hpd_funcs;
+			dev_priv->display.funcs.hotplug = &dg1_hpd_funcs;
 		else if (DISPLAY_VER(dev_priv) >= 11)
-			dev_priv->hotplug_funcs = &gen11_hpd_funcs;
+			dev_priv->display.funcs.hotplug = &gen11_hpd_funcs;
 		else if (IS_GEMINILAKE(dev_priv) || IS_BROXTON(dev_priv))
-			dev_priv->hotplug_funcs = &bxt_hpd_funcs;
+			dev_priv->display.funcs.hotplug = &bxt_hpd_funcs;
 		else if (INTEL_PCH_TYPE(dev_priv) >= PCH_ICP)
-			dev_priv->hotplug_funcs = &icp_hpd_funcs;
+			dev_priv->display.funcs.hotplug = &icp_hpd_funcs;
 		else if (INTEL_PCH_TYPE(dev_priv) >= PCH_SPT)
-			dev_priv->hotplug_funcs = &spt_hpd_funcs;
+			dev_priv->display.funcs.hotplug = &spt_hpd_funcs;
 		else
-			dev_priv->hotplug_funcs = &ilk_hpd_funcs;
+			dev_priv->display.funcs.hotplug = &ilk_hpd_funcs;
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index aacc10f2e73f..cd4487a1d3be 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -26,16 +26,22 @@
 #include <drm/drm_drv.h>
 #include <drm/i915_pciids.h>
 
+#include "gt/intel_gt_regs.h"
+#include "gt/intel_sa_media.h"
+
 #include "i915_driver.h"
 #include "i915_drv.h"
 #include "i915_pci.h"
 #include "i915_reg.h"
+#include "intel_pci_config.h"
 
 #define PLATFORM(x) .platform = (x)
 #define GEN(x) \
-	.graphics.ver = (x), \
-	.media.ver = (x), \
-	.display.ver = (x)
+	.__runtime.graphics.ip.ver = (x), \
+	.__runtime.media.ip.ver = (x), \
+	.__runtime.display.ip.ver = (x)
+
+#define NO_DISPLAY .__runtime.pipe_mask = 0
 
 #define I845_PIPE_OFFSETS \
 	.display.pipe_offsets = { \
@@ -159,16 +165,16 @@
 /* Keep in gen based order, and chronological order within a gen */
 
 #define GEN_DEFAULT_PAGE_SIZES \
-	.page_sizes = I915_GTT_PAGE_SIZE_4K
+	.__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K
 
 #define GEN_DEFAULT_REGIONS \
-	.memory_regions = REGION_SMEM | REGION_STOLEN_SMEM
+	.__runtime.memory_regions = REGION_SMEM | REGION_STOLEN_SMEM
 
 #define I830_FEATURES \
 	GEN(2), \
 	.is_mobile = 1, \
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \
 	.display.has_overlay = 1, \
 	.display.cursor_needs_physical = 1, \
 	.display.overlay_needs_physical = 1, \
@@ -177,7 +183,7 @@
 	.has_3d_pipeline = 1, \
 	.hws_needs_physical = 1, \
 	.unfenced_needs_alignment = 1, \
-	.platform_engine_mask = BIT(RCS0), \
+	.__runtime.platform_engine_mask = BIT(RCS0), \
 	.has_snoop = true, \
 	.has_coherent_ggtt = false, \
 	.dma_mask_size = 32, \
@@ -189,8 +195,8 @@
 
 #define I845_FEATURES \
 	GEN(2), \
-	.display.pipe_mask = BIT(PIPE_A), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A), \
+	.__runtime.pipe_mask = BIT(PIPE_A), \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A), \
 	.display.has_overlay = 1, \
 	.display.overlay_needs_physical = 1, \
 	.display.has_gmch = 1, \
@@ -198,7 +204,7 @@
 	.gpu_reset_clobbers_display = true, \
 	.hws_needs_physical = 1, \
 	.unfenced_needs_alignment = 1, \
-	.platform_engine_mask = BIT(RCS0), \
+	.__runtime.platform_engine_mask = BIT(RCS0), \
 	.has_snoop = true, \
 	.has_coherent_ggtt = false, \
 	.dma_mask_size = 32, \
@@ -221,22 +227,22 @@ static const struct intel_device_info i845g_info = {
 static const struct intel_device_info i85x_info = {
 	I830_FEATURES,
 	PLATFORM(INTEL_I85X),
-	.display.fbc_mask = BIT(INTEL_FBC_A),
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A),
 };
 
 static const struct intel_device_info i865g_info = {
 	I845_FEATURES,
 	PLATFORM(INTEL_I865G),
-	.display.fbc_mask = BIT(INTEL_FBC_A),
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A),
 };
 
 #define GEN3_FEATURES \
 	GEN(3), \
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \
 	.display.has_gmch = 1, \
 	.gpu_reset_clobbers_display = true, \
-	.platform_engine_mask = BIT(RCS0), \
+	.__runtime.platform_engine_mask = BIT(RCS0), \
 	.has_3d_pipeline = 1, \
 	.has_snoop = true, \
 	.has_coherent_ggtt = true, \
@@ -266,7 +272,7 @@ static const struct intel_device_info i915gm_info = {
 	.display.has_overlay = 1,
 	.display.overlay_needs_physical = 1,
 	.display.supports_tv = 1,
-	.display.fbc_mask = BIT(INTEL_FBC_A),
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A),
 	.hws_needs_physical = 1,
 	.unfenced_needs_alignment = 1,
 };
@@ -291,7 +297,7 @@ static const struct intel_device_info i945gm_info = {
 	.display.has_overlay = 1,
 	.display.overlay_needs_physical = 1,
 	.display.supports_tv = 1,
-	.display.fbc_mask = BIT(INTEL_FBC_A),
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A),
 	.hws_needs_physical = 1,
 	.unfenced_needs_alignment = 1,
 };
@@ -323,12 +329,12 @@ static const struct intel_device_info pnv_m_info = {
 
 #define GEN4_FEATURES \
 	GEN(4), \
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \
 	.display.has_hotplug = 1, \
 	.display.has_gmch = 1, \
 	.gpu_reset_clobbers_display = true, \
-	.platform_engine_mask = BIT(RCS0), \
+	.__runtime.platform_engine_mask = BIT(RCS0), \
 	.has_3d_pipeline = 1, \
 	.has_snoop = true, \
 	.has_coherent_ggtt = true, \
@@ -351,7 +357,7 @@ static const struct intel_device_info i965gm_info = {
 	GEN4_FEATURES,
 	PLATFORM(INTEL_I965GM),
 	.is_mobile = 1,
-	.display.fbc_mask = BIT(INTEL_FBC_A),
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A),
 	.display.has_overlay = 1,
 	.display.supports_tv = 1,
 	.hws_needs_physical = 1,
@@ -361,7 +367,7 @@ static const struct intel_device_info i965gm_info = {
 static const struct intel_device_info g45_info = {
 	GEN4_FEATURES,
 	PLATFORM(INTEL_G45),
-	.platform_engine_mask = BIT(RCS0) | BIT(VCS0),
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0),
 	.gpu_reset_clobbers_display = false,
 };
 
@@ -369,18 +375,18 @@ static const struct intel_device_info gm45_info = {
 	GEN4_FEATURES,
 	PLATFORM(INTEL_GM45),
 	.is_mobile = 1,
-	.display.fbc_mask = BIT(INTEL_FBC_A),
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A),
 	.display.supports_tv = 1,
-	.platform_engine_mask = BIT(RCS0) | BIT(VCS0),
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0),
 	.gpu_reset_clobbers_display = false,
 };
 
 #define GEN5_FEATURES \
 	GEN(5), \
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \
 	.display.has_hotplug = 1, \
-	.platform_engine_mask = BIT(RCS0) | BIT(VCS0), \
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0), \
 	.has_3d_pipeline = 1, \
 	.has_snoop = true, \
 	.has_coherent_ggtt = true, \
@@ -403,16 +409,16 @@ static const struct intel_device_info ilk_m_info = {
 	PLATFORM(INTEL_IRONLAKE),
 	.is_mobile = 1,
 	.has_rps = true,
-	.display.fbc_mask = BIT(INTEL_FBC_A),
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A),
 };
 
 #define GEN6_FEATURES \
 	GEN(6), \
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B), \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B), \
 	.display.has_hotplug = 1, \
-	.display.fbc_mask = BIT(INTEL_FBC_A), \
-	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A), \
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \
 	.has_3d_pipeline = 1, \
 	.has_coherent_ggtt = true, \
 	.has_llc = 1, \
@@ -420,8 +426,8 @@ static const struct intel_device_info ilk_m_info = {
 	.has_rc6p = 1, \
 	.has_rps = true, \
 	.dma_mask_size = 40, \
-	.ppgtt_type = INTEL_PPGTT_ALIASING, \
-	.ppgtt_size = 31, \
+	.__runtime.ppgtt_type = INTEL_PPGTT_ALIASING, \
+	.__runtime.ppgtt_size = 31, \
 	I9XX_PIPE_OFFSETS, \
 	I9XX_CURSOR_OFFSETS, \
 	ILK_COLORS, \
@@ -460,11 +466,11 @@ static const struct intel_device_info snb_m_gt2_info = {
 
 #define GEN7_FEATURES  \
 	GEN(7), \
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), \
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C), \
 	.display.has_hotplug = 1, \
-	.display.fbc_mask = BIT(INTEL_FBC_A), \
-	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A), \
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0), \
 	.has_3d_pipeline = 1, \
 	.has_coherent_ggtt = true, \
 	.has_llc = 1, \
@@ -473,8 +479,8 @@ static const struct intel_device_info snb_m_gt2_info = {
 	.has_reset_engine = true, \
 	.has_rps = true, \
 	.dma_mask_size = 40, \
-	.ppgtt_type = INTEL_PPGTT_ALIASING, \
-	.ppgtt_size = 31, \
+	.__runtime.ppgtt_type = INTEL_PPGTT_ALIASING, \
+	.__runtime.ppgtt_size = 31, \
 	IVB_PIPE_OFFSETS, \
 	IVB_CURSOR_OFFSETS, \
 	IVB_COLORS, \
@@ -515,9 +521,8 @@ static const struct intel_device_info ivb_m_gt2_info = {
 static const struct intel_device_info ivb_q_info = {
 	GEN7_FEATURES,
 	PLATFORM(INTEL_IVYBRIDGE),
+	NO_DISPLAY,
 	.gt = 2,
-	.display.pipe_mask = 0, /* legal, last one wins */
-	.display.cpu_transcoder_mask = 0,
 	.has_l3_dpf = 1,
 };
 
@@ -525,8 +530,8 @@ static const struct intel_device_info vlv_info = {
 	PLATFORM(INTEL_VALLEYVIEW),
 	GEN(7),
 	.is_lp = 1,
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B),
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B),
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B),
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B),
 	.has_runtime_pm = 1,
 	.has_rc6 = 1,
 	.has_reset_engine = true,
@@ -534,11 +539,11 @@ static const struct intel_device_info vlv_info = {
 	.display.has_gmch = 1,
 	.display.has_hotplug = 1,
 	.dma_mask_size = 40,
-	.ppgtt_type = INTEL_PPGTT_ALIASING,
-	.ppgtt_size = 31,
+	.__runtime.ppgtt_type = INTEL_PPGTT_ALIASING,
+	.__runtime.ppgtt_size = 31,
 	.has_snoop = true,
 	.has_coherent_ggtt = false,
-	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0),
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0),
 	.display.mmio_offset = VLV_DISPLAY_BASE,
 	I9XX_PIPE_OFFSETS,
 	I9XX_CURSOR_OFFSETS,
@@ -549,8 +554,8 @@ static const struct intel_device_info vlv_info = {
 
 #define G75_FEATURES  \
 	GEN7_FEATURES, \
-	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \
 		BIT(TRANSCODER_C) | BIT(TRANSCODER_EDP), \
 	.display.has_ddi = 1, \
 	.display.has_fpga_dbg = 1, \
@@ -584,8 +589,8 @@ static const struct intel_device_info hsw_gt3_info = {
 	GEN(8), \
 	.has_logical_ring_contexts = 1, \
 	.dma_mask_size = 39, \
-	.ppgtt_type = INTEL_PPGTT_FULL, \
-	.ppgtt_size = 48, \
+	.__runtime.ppgtt_type = INTEL_PPGTT_FULL, \
+	.__runtime.ppgtt_size = 48, \
 	.has_64bit_reloc = 1
 
 #define BDW_PLATFORM \
@@ -613,18 +618,18 @@ static const struct intel_device_info bdw_rsvd_info = {
 static const struct intel_device_info bdw_gt3_info = {
 	BDW_PLATFORM,
 	.gt = 3,
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 };
 
 static const struct intel_device_info chv_info = {
 	PLATFORM(INTEL_CHERRYVIEW),
 	GEN(8),
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C),
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | BIT(TRANSCODER_C),
 	.display.has_hotplug = 1,
 	.is_lp = 1,
-	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0),
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0),
 	.has_64bit_reloc = 1,
 	.has_runtime_pm = 1,
 	.has_rc6 = 1,
@@ -632,8 +637,8 @@ static const struct intel_device_info chv_info = {
 	.has_logical_ring_contexts = 1,
 	.display.has_gmch = 1,
 	.dma_mask_size = 39,
-	.ppgtt_type = INTEL_PPGTT_FULL,
-	.ppgtt_size = 32,
+	.__runtime.ppgtt_type = INTEL_PPGTT_FULL,
+	.__runtime.ppgtt_size = 32,
 	.has_reset_engine = 1,
 	.has_snoop = true,
 	.has_coherent_ggtt = false,
@@ -646,16 +651,16 @@ static const struct intel_device_info chv_info = {
 };
 
 #define GEN9_DEFAULT_PAGE_SIZES \
-	.page_sizes = I915_GTT_PAGE_SIZE_4K | \
-		      I915_GTT_PAGE_SIZE_64K
+	.__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \
+		I915_GTT_PAGE_SIZE_64K
 
 #define GEN9_FEATURES \
 	GEN8_FEATURES, \
 	GEN(9), \
 	GEN9_DEFAULT_PAGE_SIZES, \
-	.display.has_dmc = 1, \
+	.__runtime.has_dmc = 1, \
 	.has_gt_uc = 1, \
-	.display.has_hdcp = 1, \
+	.__runtime.has_hdcp = 1, \
 	.display.has_ipc = 1, \
 	.display.has_psr = 1, \
 	.display.has_psr_hw_tracking = 1, \
@@ -678,7 +683,7 @@ static const struct intel_device_info skl_gt2_info = {
 
 #define SKL_GT3_PLUS_PLATFORM \
 	SKL_PLATFORM, \
-	.platform_engine_mask = \
+	.__runtime.platform_engine_mask = \
 		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1)
 
 
@@ -697,29 +702,29 @@ static const struct intel_device_info skl_gt4_info = {
 	.is_lp = 1, \
 	.display.dbuf.slice_mask = BIT(DBUF_S1), \
 	.display.has_hotplug = 1, \
-	.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0), \
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C), \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \
 		BIT(TRANSCODER_C) | BIT(TRANSCODER_EDP) | \
 		BIT(TRANSCODER_DSI_A) | BIT(TRANSCODER_DSI_C), \
 	.has_3d_pipeline = 1, \
 	.has_64bit_reloc = 1, \
 	.display.has_ddi = 1, \
 	.display.has_fpga_dbg = 1, \
-	.display.fbc_mask = BIT(INTEL_FBC_A), \
-	.display.has_hdcp = 1, \
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A), \
+	.__runtime.has_hdcp = 1, \
 	.display.has_psr = 1, \
 	.display.has_psr_hw_tracking = 1, \
 	.has_runtime_pm = 1, \
-	.display.has_dmc = 1, \
+	.__runtime.has_dmc = 1, \
 	.has_rc6 = 1, \
 	.has_rps = true, \
 	.display.has_dp_mst = 1, \
 	.has_logical_ring_contexts = 1, \
 	.has_gt_uc = 1, \
 	.dma_mask_size = 39, \
-	.ppgtt_type = INTEL_PPGTT_FULL, \
-	.ppgtt_size = 48, \
+	.__runtime.ppgtt_type = INTEL_PPGTT_FULL, \
+	.__runtime.ppgtt_size = 48, \
 	.has_reset_engine = 1, \
 	.has_snoop = true, \
 	.has_coherent_ggtt = false, \
@@ -739,7 +744,7 @@ static const struct intel_device_info bxt_info = {
 static const struct intel_device_info glk_info = {
 	GEN9_LP_FEATURES,
 	PLATFORM(INTEL_GEMINILAKE),
-	.display.ver = 10,
+	.__runtime.display.ip.ver = 10,
 	.display.dbuf.size = 1024 - 4, /* 4 blocks for bypass path allocation */
 	GLK_COLORS,
 };
@@ -761,7 +766,7 @@ static const struct intel_device_info kbl_gt2_info = {
 static const struct intel_device_info kbl_gt3_info = {
 	KBL_PLATFORM,
 	.gt = 3,
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 };
 
@@ -782,7 +787,7 @@ static const struct intel_device_info cfl_gt2_info = {
 static const struct intel_device_info cfl_gt3_info = {
 	CFL_PLATFORM,
 	.gt = 3,
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(VCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS1),
 };
 
@@ -801,15 +806,15 @@ static const struct intel_device_info cml_gt2_info = {
 };
 
 #define GEN11_DEFAULT_PAGE_SIZES \
-	.page_sizes = I915_GTT_PAGE_SIZE_4K | \
-		      I915_GTT_PAGE_SIZE_64K | \
-		      I915_GTT_PAGE_SIZE_2M
+	.__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \
+		I915_GTT_PAGE_SIZE_64K |		\
+		I915_GTT_PAGE_SIZE_2M
 
 #define GEN11_FEATURES \
 	GEN9_FEATURES, \
 	GEN11_DEFAULT_PAGE_SIZES, \
 	.display.abox_mask = BIT(0), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \
 		BIT(TRANSCODER_C) | BIT(TRANSCODER_EDP) | \
 		BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1), \
 	.display.pipe_offsets = { \
@@ -832,37 +837,37 @@ static const struct intel_device_info cml_gt2_info = {
 	ICL_COLORS, \
 	.display.dbuf.size = 2048, \
 	.display.dbuf.slice_mask = BIT(DBUF_S1) | BIT(DBUF_S2), \
-	.display.has_dsc = 1, \
+	.__runtime.has_dsc = 1, \
 	.has_coherent_ggtt = false, \
 	.has_logical_ring_elsq = 1
 
 static const struct intel_device_info icl_info = {
 	GEN11_FEATURES,
 	PLATFORM(INTEL_ICELAKE),
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 };
 
 static const struct intel_device_info ehl_info = {
 	GEN11_FEATURES,
 	PLATFORM(INTEL_ELKHARTLAKE),
-	.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0),
-	.ppgtt_size = 36,
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0),
+	.__runtime.ppgtt_size = 36,
 };
 
 static const struct intel_device_info jsl_info = {
 	GEN11_FEATURES,
 	PLATFORM(INTEL_JASPERLAKE),
-	.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0),
-	.ppgtt_size = 36,
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(VCS0) | BIT(VECS0),
+	.__runtime.ppgtt_size = 36,
 };
 
 #define GEN12_FEATURES \
 	GEN11_FEATURES, \
 	GEN(12), \
 	.display.abox_mask = GENMASK(2, 1), \
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), \
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D), \
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) | \
 		BIT(TRANSCODER_C) | BIT(TRANSCODER_D) | \
 		BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1), \
 	.display.pipe_offsets = { \
@@ -890,7 +895,7 @@ static const struct intel_device_info tgl_info = {
 	GEN12_FEATURES,
 	PLATFORM(INTEL_TIGERLAKE),
 	.display.has_modular_fia = 1,
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 };
 
@@ -898,17 +903,17 @@ static const struct intel_device_info rkl_info = {
 	GEN12_FEATURES,
 	PLATFORM(INTEL_ROCKETLAKE),
 	.display.abox_mask = BIT(0),
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
 		BIT(TRANSCODER_C),
 	.display.has_hti = 1,
 	.display.has_psr_hw_tracking = 0,
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0),
 };
 
 #define DGFX_FEATURES \
-	.memory_regions = REGION_SMEM | REGION_LMEM | REGION_STOLEN_LMEM, \
+	.__runtime.memory_regions = REGION_SMEM | REGION_LMEM | REGION_STOLEN_LMEM, \
 	.has_llc = 0, \
 	.has_pxp = 0, \
 	.has_snoop = 1, \
@@ -918,24 +923,24 @@ static const struct intel_device_info rkl_info = {
 static const struct intel_device_info dg1_info = {
 	GEN12_FEATURES,
 	DGFX_FEATURES,
-	.graphics.rel = 10,
+	.__runtime.graphics.ip.rel = 10,
 	PLATFORM(INTEL_DG1),
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
 	.require_force_probe = 1,
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) |
 		BIT(VCS0) | BIT(VCS2),
 	/* Wa_16011227922 */
-	.ppgtt_size = 47,
+	.__runtime.ppgtt_size = 47,
 };
 
 static const struct intel_device_info adl_s_info = {
 	GEN12_FEATURES,
 	PLATFORM(INTEL_ALDERLAKE_S),
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
 	.display.has_hti = 1,
 	.display.has_psr_hw_tracking = 0,
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
 	.dma_mask_size = 39,
 };
@@ -951,18 +956,18 @@ static const struct intel_device_info adl_s_info = {
 	.display.dbuf.slice_mask = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) |	\
 		BIT(DBUF_S4),							\
 	.display.has_ddi = 1,							\
-	.display.has_dmc = 1,							\
+	.__runtime.has_dmc = 1,							\
 	.display.has_dp_mst = 1,						\
 	.display.has_dsb = 1,							\
-	.display.has_dsc = 1,							\
-	.display.fbc_mask = BIT(INTEL_FBC_A),					\
+	.__runtime.has_dsc = 1,							\
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A),					\
 	.display.has_fpga_dbg = 1,						\
-	.display.has_hdcp = 1,							\
+	.__runtime.has_hdcp = 1,						\
 	.display.has_hotplug = 1,						\
 	.display.has_ipc = 1,							\
 	.display.has_psr = 1,							\
-	.display.ver = 13,							\
-	.display.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),	\
+	.__runtime.display.ip.ver = 13,							\
+	.__runtime.pipe_mask = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),	\
 	.display.pipe_offsets = {						\
 		[TRANSCODER_A] = PIPE_A_OFFSET,					\
 		[TRANSCODER_B] = PIPE_B_OFFSET,					\
@@ -985,28 +990,28 @@ static const struct intel_device_info adl_p_info = {
 	GEN12_FEATURES,
 	XE_LPD_FEATURES,
 	PLATFORM(INTEL_ALDERLAKE_P),
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
 			       BIT(TRANSCODER_C) | BIT(TRANSCODER_D) |
 			       BIT(TRANSCODER_DSI_0) | BIT(TRANSCODER_DSI_1),
 	.display.has_cdclk_crawl = 1,
 	.display.has_modular_fia = 1,
 	.display.has_psr_hw_tracking = 0,
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(BCS0) | BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
-	.ppgtt_size = 48,
+	.__runtime.ppgtt_size = 48,
 	.dma_mask_size = 39,
 };
 
 #undef GEN
 
 #define XE_HP_PAGE_SIZES \
-	.page_sizes = I915_GTT_PAGE_SIZE_4K | \
-		      I915_GTT_PAGE_SIZE_64K | \
-		      I915_GTT_PAGE_SIZE_2M
+	.__runtime.page_sizes = I915_GTT_PAGE_SIZE_4K | \
+		I915_GTT_PAGE_SIZE_64K |		\
+		I915_GTT_PAGE_SIZE_2M
 
 #define XE_HP_FEATURES \
-	.graphics.ver = 12, \
-	.graphics.rel = 50, \
+	.__runtime.graphics.ip.ver = 12, \
+	.__runtime.graphics.ip.rel = 50, \
 	XE_HP_PAGE_SIZES, \
 	.dma_mask_size = 46, \
 	.has_3d_pipeline = 1, \
@@ -1022,12 +1027,12 @@ static const struct intel_device_info adl_p_info = {
 	.has_reset_engine = 1, \
 	.has_rps = 1, \
 	.has_runtime_pm = 1, \
-	.ppgtt_size = 48, \
-	.ppgtt_type = INTEL_PPGTT_FULL
+	.__runtime.ppgtt_size = 48, \
+	.__runtime.ppgtt_type = INTEL_PPGTT_FULL
 
 #define XE_HPM_FEATURES \
-	.media.ver = 12, \
-	.media.rel = 50
+	.__runtime.media.ip.ver = 12, \
+	.__runtime.media.ip.rel = 50
 
 __maybe_unused
 static const struct intel_device_info xehpsdv_info = {
@@ -1035,11 +1040,11 @@ static const struct intel_device_info xehpsdv_info = {
 	XE_HPM_FEATURES,
 	DGFX_FEATURES,
 	PLATFORM(INTEL_XEHPSDV),
-	.display = { },
+	NO_DISPLAY,
 	.has_64k_pages = 1,
 	.needs_compact_pt = 1,
 	.has_media_ratio_mode = 1,
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(RCS0) | BIT(BCS0) |
 		BIT(VECS0) | BIT(VECS1) | BIT(VECS2) | BIT(VECS3) |
 		BIT(VCS0) | BIT(VCS1) | BIT(VCS2) | BIT(VCS3) |
@@ -1052,8 +1057,8 @@ static const struct intel_device_info xehpsdv_info = {
 	XE_HP_FEATURES, \
 	XE_HPM_FEATURES, \
 	DGFX_FEATURES, \
-	.graphics.rel = 55, \
-	.media.rel = 55, \
+	.__runtime.graphics.ip.rel = 55, \
+	.__runtime.media.ip.rel = 55, \
 	PLATFORM(INTEL_DG2), \
 	.has_4tile = 1, \
 	.has_64k_pages = 1, \
@@ -1061,7 +1066,7 @@ static const struct intel_device_info xehpsdv_info = {
 	.has_heci_pxp = 1, \
 	.needs_compact_pt = 1, \
 	.has_media_ratio_mode = 1, \
-	.platform_engine_mask = \
+	.__runtime.platform_engine_mask = \
 		BIT(RCS0) | BIT(BCS0) | \
 		BIT(VECS0) | BIT(VECS1) | \
 		BIT(VCS0) | BIT(VCS2) | \
@@ -1070,15 +1075,16 @@ static const struct intel_device_info xehpsdv_info = {
 static const struct intel_device_info dg2_info = {
 	DG2_FEATURES,
 	XE_LPD_FEATURES,
-	.display.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
+	.__runtime.cpu_transcoder_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
 			       BIT(TRANSCODER_C) | BIT(TRANSCODER_D),
 	.require_force_probe = 1,
 };
 
 static const struct intel_device_info ats_m_info = {
 	DG2_FEATURES,
-	.display = { 0 },
+	NO_DISPLAY,
 	.require_force_probe = 1,
+	.tuning_thread_rr_after_dep = 1,
 };
 
 #define XE_HPC_FEATURES \
@@ -1095,12 +1101,12 @@ static const struct intel_device_info pvc_info = {
 	XE_HPC_FEATURES,
 	XE_HPM_FEATURES,
 	DGFX_FEATURES,
-	.graphics.rel = 60,
-	.media.rel = 60,
+	.__runtime.graphics.ip.rel = 60,
+	.__runtime.media.ip.rel = 60,
 	PLATFORM(INTEL_PONTEVECCHIO),
-	.display = { 0 },
+	NO_DISPLAY,
 	.has_flat_ccs = 0,
-	.platform_engine_mask =
+	.__runtime.platform_engine_mask =
 		BIT(BCS0) |
 		BIT(VCS0) |
 		BIT(CCS0) | BIT(CCS1) | BIT(CCS2) | BIT(CCS3),
@@ -1109,8 +1115,19 @@ static const struct intel_device_info pvc_info = {
 
 #define XE_LPDP_FEATURES	\
 	XE_LPD_FEATURES,	\
-	.display.ver = 14,	\
-	.display.has_cdclk_crawl = 1
+	.__runtime.display.ip.ver = 14,	\
+	.display.has_cdclk_crawl = 1, \
+	.__runtime.fbc_mask = BIT(INTEL_FBC_A) | BIT(INTEL_FBC_B)
+
+static const struct intel_gt_definition xelpmp_extra_gt[] = {
+	{
+		.type = GT_MEDIA,
+		.name = "Standalone Media GT",
+		.gsi_offset = MTL_MEDIA_GSI_BASE,
+		.engine_mask = BIT(VECS0) | BIT(VCS0) | BIT(VCS2),
+	},
+	{}
+};
 
 __maybe_unused
 static const struct intel_device_info mtl_info = {
@@ -1120,15 +1137,16 @@ static const struct intel_device_info mtl_info = {
 	 * Real graphics IP version will be obtained from hardware GMD_ID
 	 * register.  Value provided here is just for sanity checking.
 	 */
-	.graphics.ver = 12,
-	.graphics.rel = 70,
-	.media.ver = 13,
+	.__runtime.graphics.ip.ver = 12,
+	.__runtime.graphics.ip.rel = 70,
+	.__runtime.media.ip.ver = 13,
 	PLATFORM(INTEL_METEORLAKE),
 	.display.has_modular_fia = 1,
+	.extra_gt_list = xelpmp_extra_gt,
 	.has_flat_ccs = 0,
 	.has_snoop = 1,
-	.memory_regions = REGION_SMEM | REGION_STOLEN_LMEM,
-	.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(CCS0),
+	.__runtime.memory_regions = REGION_SMEM | REGION_STOLEN_LMEM,
+	.__runtime.platform_engine_mask = BIT(RCS0) | BIT(BCS0) | BIT(CCS0),
 	.require_force_probe = 1,
 };
 
@@ -1262,6 +1280,27 @@ static bool force_probe(u16 device_id, const char *devices)
 	return ret;
 }
 
+bool i915_pci_resource_valid(struct pci_dev *pdev, int bar)
+{
+	if (!pci_resource_flags(pdev, bar))
+		return false;
+
+	if (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET)
+		return false;
+
+	if (!pci_resource_len(pdev, bar))
+		return false;
+
+	return true;
+}
+
+static bool intel_mmio_bar_valid(struct pci_dev *pdev, struct intel_device_info *intel_info)
+{
+	int gttmmaddr_bar = intel_info->__runtime.graphics.ip.ver == 2 ? GEN2_GTTMMADR_BAR : GTTMMADR_BAR;
+
+	return i915_pci_resource_valid(pdev, gttmmaddr_bar);
+}
+
 static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	struct intel_device_info *intel_info =
@@ -1287,6 +1326,9 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (PCI_FUNC(pdev->devfn))
 		return -ENODEV;
 
+	if (!intel_mmio_bar_valid(pdev, intel_info))
+		return -ENXIO;
+
 	/* Detect if we need to wait for other drivers early on */
 	if (intel_modeset_probe_defer(pdev))
 		return -EPROBE_DEFER;
diff --git a/drivers/gpu/drm/i915/i915_pci.h b/drivers/gpu/drm/i915/i915_pci.h
index ee048c238174..8dfe19f9a775 100644
--- a/drivers/gpu/drm/i915/i915_pci.h
+++ b/drivers/gpu/drm/i915/i915_pci.h
@@ -6,7 +6,13 @@
 #ifndef __I915_PCI_H__
 #define __I915_PCI_H__
 
+#include <linux/types.h>
+
+struct pci_dev;
+
 int i915_pci_register_driver(void);
 void i915_pci_unregister_driver(void);
 
+bool i915_pci_resource_valid(struct pci_dev *pdev, int bar);
+
 #endif /* __I915_PCI_H__ */
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index f3c23fe9ad9c..0defbb43ceea 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -1376,7 +1376,8 @@ static void i915_oa_stream_destroy(struct i915_perf_stream *stream)
 {
 	struct i915_perf *perf = stream->perf;
 
-	BUG_ON(stream != perf->exclusive_stream);
+	if (WARN_ON(stream != perf->exclusive_stream))
+		return;
 
 	/*
 	 * Unset exclusive_stream first, it will be checked while disabling
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 135d04c2d41c..1a9bd829fc7e 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -1125,8 +1125,12 @@
 #define MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN	REG_BIT(16) /* tgl+ */
 #define MBUS_DBOX_BW_CREDIT_MASK		REG_GENMASK(15, 14)
 #define MBUS_DBOX_BW_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, x)
+#define MBUS_DBOX_BW_4CREDITS_MTL		REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, 0x2)
+#define MBUS_DBOX_BW_8CREDITS_MTL		REG_FIELD_PREP(MBUS_DBOX_BW_CREDIT_MASK, 0x3)
 #define MBUS_DBOX_B_CREDIT_MASK			REG_GENMASK(12, 8)
 #define MBUS_DBOX_B_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_B_CREDIT_MASK, x)
+#define MBUS_DBOX_I_CREDIT_MASK			REG_GENMASK(7, 5)
+#define MBUS_DBOX_I_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_I_CREDIT_MASK, x)
 #define MBUS_DBOX_A_CREDIT_MASK			REG_GENMASK(3, 0)
 #define MBUS_DBOX_A_CREDIT(x)			REG_FIELD_PREP(MBUS_DBOX_A_CREDIT_MASK, x)
 
@@ -1462,69 +1466,6 @@
 #define   FBC_REND_CACHE_CLEAN		REG_BIT(1)
 
 /*
- * GPIO regs
- */
-#define GPIO(gpio)		_MMIO(dev_priv->gpio_mmio_base + 0x5010 + \
-				      4 * (gpio))
-
-# define GPIO_CLOCK_DIR_MASK		(1 << 0)
-# define GPIO_CLOCK_DIR_IN		(0 << 1)
-# define GPIO_CLOCK_DIR_OUT		(1 << 1)
-# define GPIO_CLOCK_VAL_MASK		(1 << 2)
-# define GPIO_CLOCK_VAL_OUT		(1 << 3)
-# define GPIO_CLOCK_VAL_IN		(1 << 4)
-# define GPIO_CLOCK_PULLUP_DISABLE	(1 << 5)
-# define GPIO_DATA_DIR_MASK		(1 << 8)
-# define GPIO_DATA_DIR_IN		(0 << 9)
-# define GPIO_DATA_DIR_OUT		(1 << 9)
-# define GPIO_DATA_VAL_MASK		(1 << 10)
-# define GPIO_DATA_VAL_OUT		(1 << 11)
-# define GPIO_DATA_VAL_IN		(1 << 12)
-# define GPIO_DATA_PULLUP_DISABLE	(1 << 13)
-
-#define GMBUS0			_MMIO(dev_priv->gpio_mmio_base + 0x5100) /* clock/port select */
-#define   GMBUS_AKSV_SELECT	(1 << 11)
-#define   GMBUS_RATE_100KHZ	(0 << 8)
-#define   GMBUS_RATE_50KHZ	(1 << 8)
-#define   GMBUS_RATE_400KHZ	(2 << 8) /* reserved on Pineview */
-#define   GMBUS_RATE_1MHZ	(3 << 8) /* reserved on Pineview */
-#define   GMBUS_HOLD_EXT	(1 << 7) /* 300ns hold time, rsvd on Pineview */
-#define   GMBUS_BYTE_CNT_OVERRIDE (1 << 6)
-
-#define GMBUS1			_MMIO(dev_priv->gpio_mmio_base + 0x5104) /* command/status */
-#define   GMBUS_SW_CLR_INT	(1 << 31)
-#define   GMBUS_SW_RDY		(1 << 30)
-#define   GMBUS_ENT		(1 << 29) /* enable timeout */
-#define   GMBUS_CYCLE_NONE	(0 << 25)
-#define   GMBUS_CYCLE_WAIT	(1 << 25)
-#define   GMBUS_CYCLE_INDEX	(2 << 25)
-#define   GMBUS_CYCLE_STOP	(4 << 25)
-#define   GMBUS_BYTE_COUNT_SHIFT 16
-#define   GMBUS_BYTE_COUNT_MAX   256U
-#define   GEN9_GMBUS_BYTE_COUNT_MAX 511U
-#define   GMBUS_SLAVE_INDEX_SHIFT 8
-#define   GMBUS_SLAVE_ADDR_SHIFT 1
-#define   GMBUS_SLAVE_READ	(1 << 0)
-#define   GMBUS_SLAVE_WRITE	(0 << 0)
-#define GMBUS2			_MMIO(dev_priv->gpio_mmio_base + 0x5108) /* status */
-#define   GMBUS_INUSE		(1 << 15)
-#define   GMBUS_HW_WAIT_PHASE	(1 << 14)
-#define   GMBUS_STALL_TIMEOUT	(1 << 13)
-#define   GMBUS_INT		(1 << 12)
-#define   GMBUS_HW_RDY		(1 << 11)
-#define   GMBUS_SATOER		(1 << 10)
-#define   GMBUS_ACTIVE		(1 << 9)
-#define GMBUS3			_MMIO(dev_priv->gpio_mmio_base + 0x510c) /* data buffer bytes 3-0 */
-#define GMBUS4			_MMIO(dev_priv->gpio_mmio_base + 0x5110) /* interrupt mask (Pineview+) */
-#define   GMBUS_SLAVE_TIMEOUT_EN (1 << 4)
-#define   GMBUS_NAK_EN		(1 << 3)
-#define   GMBUS_IDLE_EN		(1 << 2)
-#define   GMBUS_HW_WAIT_EN	(1 << 1)
-#define   GMBUS_HW_RDY_EN	(1 << 0)
-#define GMBUS5			_MMIO(dev_priv->gpio_mmio_base + 0x5120) /* byte index */
-#define   GMBUS_2BYTE_INDEX_EN	(1 << 31)
-
-/*
  * Clock control & power management
  */
 #define _DPLL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x6014)
@@ -1700,7 +1641,7 @@
 #define  DSTATE_PLL_D3_OFF			(1 << 3)
 #define  DSTATE_GFX_CLOCK_GATING		(1 << 1)
 #define  DSTATE_DOT_CLOCK_GATING		(1 << 0)
-#define DSPCLK_GATE_D	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x6200)
+#define DSPCLK_GATE_D(__i915)		_MMIO(DISPLAY_MMIO_BASE(__i915) + 0x6200)
 # define DPUNIT_B_CLOCK_GATE_DISABLE		(1 << 30) /* 965 */
 # define VSUNIT_CLOCK_GATE_DISABLE		(1 << 29) /* 965 */
 # define VRHUNIT_CLOCK_GATE_DISABLE		(1 << 28) /* 965 */
@@ -1916,6 +1857,13 @@
 #define CLKGATE_DIS_PSL(pipe) \
 	_MMIO_PIPE(pipe, _CLKGATE_DIS_PSL_A, _CLKGATE_DIS_PSL_B)
 
+#define _CLKGATE_DIS_PSL_EXT_A		0x4654C
+#define _CLKGATE_DIS_PSL_EXT_B		0x46550
+#define   PIPEDMC_GATING_DIS		REG_BIT(12)
+
+#define CLKGATE_DIS_PSL_EXT(pipe) \
+	_MMIO_PIPE(pipe, _CLKGATE_DIS_PSL_EXT_A, _CLKGATE_DIS_PSL_EXT_B)
+
 /*
  * Display engine regs
  */
@@ -2822,7 +2770,7 @@
 #define VLV_PPS_BASE			(VLV_DISPLAY_BASE + PPS_BASE)
 #define PCH_PPS_BASE			0xC7200
 
-#define _MMIO_PPS(pps_idx, reg)		_MMIO(dev_priv->pps_mmio_base -	\
+#define _MMIO_PPS(pps_idx, reg)		_MMIO(dev_priv->display.pps.mmio_base -	\
 					      PPS_BASE + (reg) +	\
 					      (pps_idx) * 0x100)
 
@@ -2918,118 +2866,6 @@
 
 #define PFIT_AUTO_RATIOS _MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61238)
 
-#define _VLV_BLC_PWM_CTL2_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61250)
-#define _VLV_BLC_PWM_CTL2_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61350)
-#define VLV_BLC_PWM_CTL2(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \
-					 _VLV_BLC_PWM_CTL2_B)
-
-#define _VLV_BLC_PWM_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61254)
-#define _VLV_BLC_PWM_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61354)
-#define VLV_BLC_PWM_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_PWM_CTL_A, \
-					_VLV_BLC_PWM_CTL_B)
-
-#define _VLV_BLC_HIST_CTL_A (DISPLAY_MMIO_BASE(dev_priv) + 0x61260)
-#define _VLV_BLC_HIST_CTL_B (DISPLAY_MMIO_BASE(dev_priv) + 0x61360)
-#define VLV_BLC_HIST_CTL(pipe) _MMIO_PIPE(pipe, _VLV_BLC_HIST_CTL_A, \
-					 _VLV_BLC_HIST_CTL_B)
-
-/* Backlight control */
-#define BLC_PWM_CTL2	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61250) /* 965+ only */
-#define   BLM_PWM_ENABLE		(1 << 31)
-#define   BLM_COMBINATION_MODE		(1 << 30) /* gen4 only */
-#define   BLM_PIPE_SELECT		(1 << 29)
-#define   BLM_PIPE_SELECT_IVB		(3 << 29)
-#define   BLM_PIPE_A			(0 << 29)
-#define   BLM_PIPE_B			(1 << 29)
-#define   BLM_PIPE_C			(2 << 29) /* ivb + */
-#define   BLM_TRANSCODER_A		BLM_PIPE_A /* hsw */
-#define   BLM_TRANSCODER_B		BLM_PIPE_B
-#define   BLM_TRANSCODER_C		BLM_PIPE_C
-#define   BLM_TRANSCODER_EDP		(3 << 29)
-#define   BLM_PIPE(pipe)		((pipe) << 29)
-#define   BLM_POLARITY_I965		(1 << 28) /* gen4 only */
-#define   BLM_PHASE_IN_INTERUPT_STATUS	(1 << 26)
-#define   BLM_PHASE_IN_ENABLE		(1 << 25)
-#define   BLM_PHASE_IN_INTERUPT_ENABL	(1 << 24)
-#define   BLM_PHASE_IN_TIME_BASE_SHIFT	(16)
-#define   BLM_PHASE_IN_TIME_BASE_MASK	(0xff << 16)
-#define   BLM_PHASE_IN_COUNT_SHIFT	(8)
-#define   BLM_PHASE_IN_COUNT_MASK	(0xff << 8)
-#define   BLM_PHASE_IN_INCR_SHIFT	(0)
-#define   BLM_PHASE_IN_INCR_MASK	(0xff << 0)
-#define BLC_PWM_CTL	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61254)
-/*
- * This is the most significant 15 bits of the number of backlight cycles in a
- * complete cycle of the modulated backlight control.
- *
- * The actual value is this field multiplied by two.
- */
-#define   BACKLIGHT_MODULATION_FREQ_SHIFT	(17)
-#define   BACKLIGHT_MODULATION_FREQ_MASK	(0x7fff << 17)
-#define   BLM_LEGACY_MODE			(1 << 16) /* gen2 only */
-/*
- * This is the number of cycles out of the backlight modulation cycle for which
- * the backlight is on.
- *
- * This field must be no greater than the number of cycles in the complete
- * backlight modulation cycle.
- */
-#define   BACKLIGHT_DUTY_CYCLE_SHIFT		(0)
-#define   BACKLIGHT_DUTY_CYCLE_MASK		(0xffff)
-#define   BACKLIGHT_DUTY_CYCLE_MASK_PNV		(0xfffe)
-#define   BLM_POLARITY_PNV			(1 << 0) /* pnv only */
-
-#define BLC_HIST_CTL	_MMIO(DISPLAY_MMIO_BASE(dev_priv) + 0x61260)
-#define  BLM_HISTOGRAM_ENABLE			(1 << 31)
-
-/* New registers for PCH-split platforms. Safe where new bits show up, the
- * register layout machtes with gen4 BLC_PWM_CTL[12]. */
-#define BLC_PWM_CPU_CTL2	_MMIO(0x48250)
-#define BLC_PWM_CPU_CTL		_MMIO(0x48254)
-
-#define HSW_BLC_PWM2_CTL	_MMIO(0x48350)
-
-/* PCH CTL1 is totally different, all but the below bits are reserved. CTL2 is
- * like the normal CTL from gen4 and earlier. Hooray for confusing naming. */
-#define BLC_PWM_PCH_CTL1	_MMIO(0xc8250)
-#define   BLM_PCH_PWM_ENABLE			(1 << 31)
-#define   BLM_PCH_OVERRIDE_ENABLE		(1 << 30)
-#define   BLM_PCH_POLARITY			(1 << 29)
-#define BLC_PWM_PCH_CTL2	_MMIO(0xc8254)
-
-#define UTIL_PIN_CTL			_MMIO(0x48400)
-#define   UTIL_PIN_ENABLE		(1 << 31)
-#define   UTIL_PIN_PIPE_MASK		(3 << 29)
-#define   UTIL_PIN_PIPE(x)		((x) << 29)
-#define   UTIL_PIN_MODE_MASK		(0xf << 24)
-#define   UTIL_PIN_MODE_DATA		(0 << 24)
-#define   UTIL_PIN_MODE_PWM		(1 << 24)
-#define   UTIL_PIN_MODE_VBLANK		(4 << 24)
-#define   UTIL_PIN_MODE_VSYNC		(5 << 24)
-#define   UTIL_PIN_MODE_EYE_LEVEL	(8 << 24)
-#define   UTIL_PIN_OUTPUT_DATA		(1 << 23)
-#define   UTIL_PIN_POLARITY		(1 << 22)
-#define   UTIL_PIN_DIRECTION_INPUT	(1 << 19)
-#define   UTIL_PIN_INPUT_DATA		(1 << 16)
-
-/* BXT backlight register definition. */
-#define _BXT_BLC_PWM_CTL1			0xC8250
-#define   BXT_BLC_PWM_ENABLE			(1 << 31)
-#define   BXT_BLC_PWM_POLARITY			(1 << 29)
-#define _BXT_BLC_PWM_FREQ1			0xC8254
-#define _BXT_BLC_PWM_DUTY1			0xC8258
-
-#define _BXT_BLC_PWM_CTL2			0xC8350
-#define _BXT_BLC_PWM_FREQ2			0xC8354
-#define _BXT_BLC_PWM_DUTY2			0xC8358
-
-#define BXT_BLC_PWM_CTL(controller)    _MMIO_PIPE(controller,		\
-					_BXT_BLC_PWM_CTL1, _BXT_BLC_PWM_CTL2)
-#define BXT_BLC_PWM_FREQ(controller)   _MMIO_PIPE(controller, \
-					_BXT_BLC_PWM_FREQ1, _BXT_BLC_PWM_FREQ2)
-#define BXT_BLC_PWM_DUTY(controller)   _MMIO_PIPE(controller, \
-					_BXT_BLC_PWM_DUTY1, _BXT_BLC_PWM_DUTY2)
-
 #define PCH_GTC_CTL		_MMIO(0xe7000)
 #define   PCH_GTC_ENABLE	(1 << 31)
 
@@ -3619,6 +3455,34 @@
 #define DP_AUX_CH_CTL(aux_ch)	_MMIO_PORT(aux_ch, _DPA_AUX_CH_CTL, _DPB_AUX_CH_CTL)
 #define DP_AUX_CH_DATA(aux_ch, i)	_MMIO(_PORT(aux_ch, _DPA_AUX_CH_DATA1, _DPB_AUX_CH_DATA1) + (i) * 4) /* 5 registers */
 
+#define _XELPDP_USBC1_AUX_CH_CTL	0x16F210
+#define _XELPDP_USBC2_AUX_CH_CTL	0x16F410
+#define _XELPDP_USBC3_AUX_CH_CTL	0x16F610
+#define _XELPDP_USBC4_AUX_CH_CTL	0x16F810
+
+#define XELPDP_DP_AUX_CH_CTL(aux_ch)		_MMIO(_PICK(aux_ch, \
+						       _DPA_AUX_CH_CTL, \
+						       _DPB_AUX_CH_CTL, \
+						       0, /* port/aux_ch C is non-existent */ \
+						       _XELPDP_USBC1_AUX_CH_CTL, \
+						       _XELPDP_USBC2_AUX_CH_CTL, \
+						       _XELPDP_USBC3_AUX_CH_CTL, \
+						       _XELPDP_USBC4_AUX_CH_CTL))
+
+#define _XELPDP_USBC1_AUX_CH_DATA1      0x16F214
+#define _XELPDP_USBC2_AUX_CH_DATA1      0x16F414
+#define _XELPDP_USBC3_AUX_CH_DATA1      0x16F614
+#define _XELPDP_USBC4_AUX_CH_DATA1      0x16F814
+
+#define XELPDP_DP_AUX_CH_DATA(aux_ch, i)	_MMIO(_PICK(aux_ch, \
+						       _DPA_AUX_CH_DATA1, \
+						       _DPB_AUX_CH_DATA1, \
+						       0, /* port/aux_ch C is non-existent */ \
+						       _XELPDP_USBC1_AUX_CH_DATA1, \
+						       _XELPDP_USBC2_AUX_CH_DATA1, \
+						       _XELPDP_USBC3_AUX_CH_DATA1, \
+						       _XELPDP_USBC4_AUX_CH_DATA1) + (i) * 4)
+
 #define   DP_AUX_CH_CTL_SEND_BUSY	    (1 << 31)
 #define   DP_AUX_CH_CTL_DONE		    (1 << 30)
 #define   DP_AUX_CH_CTL_INTERRUPT	    (1 << 29)
@@ -3631,6 +3495,8 @@
 #define   DP_AUX_CH_CTL_RECEIVE_ERROR	    (1 << 25)
 #define   DP_AUX_CH_CTL_MESSAGE_SIZE_MASK    (0x1f << 20)
 #define   DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT   20
+#define   XELPDP_DP_AUX_CH_CTL_POWER_REQUEST REG_BIT(19)
+#define   XELPDP_DP_AUX_CH_CTL_POWER_STATUS  REG_BIT(18)
 #define   DP_AUX_CH_CTL_PRECHARGE_2US_MASK   (0xf << 16)
 #define   DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT  16
 #define   DP_AUX_CH_CTL_AUX_AKSV_SELECT	    (1 << 15)
@@ -5862,6 +5728,13 @@
 					    [TRANSCODER_B] = _CHICKEN_TRANS_B, \
 					    [TRANSCODER_C] = _CHICKEN_TRANS_C, \
 					    [TRANSCODER_D] = _CHICKEN_TRANS_D))
+
+#define _MTL_CHICKEN_TRANS_A	0x604e0
+#define _MTL_CHICKEN_TRANS_B	0x614e0
+#define MTL_CHICKEN_TRANS(trans)	_MMIO_TRANS((trans), \
+						    _MTL_CHICKEN_TRANS_A, \
+						    _MTL_CHICKEN_TRANS_B)
+
 #define  HSW_FRAME_START_DELAY_MASK	REG_GENMASK(28, 27)
 #define  HSW_FRAME_START_DELAY(x)	REG_FIELD_PREP(HSW_FRAME_START_DELAY_MASK, x)
 #define  VSC_DATA_SEL_SOFTWARE_CONTROL	REG_BIT(25) /* GLK */
@@ -5926,7 +5799,8 @@
 							 _BW_BUDDY1_PAGE_MASK))
 
 #define HSW_NDE_RSTWRN_OPT	_MMIO(0x46408)
-#define  RESET_PCH_HANDSHAKE_ENABLE	(1 << 4)
+#define  MTL_RESET_PICA_HANDSHAKE_EN	REG_BIT(6)
+#define  RESET_PCH_HANDSHAKE_ENABLE	REG_BIT(4)
 
 #define GEN8_CHICKEN_DCPR_1			_MMIO(0x46430)
 #define   SKL_SELECT_ALTERNATE_DC_EXIT		REG_BIT(30)
@@ -6718,10 +6592,10 @@
 #define     GEN6_DECODE_RC6_VID(vids)		(((vids) * 5) + 245)
 #define   BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ	0x18
 #define   GEN9_PCODE_READ_MEM_LATENCY		0x6
-#define     GEN9_MEM_LATENCY_LEVEL_MASK		0xFF
-#define     GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT	8
-#define     GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT	16
-#define     GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT	24
+#define     GEN9_MEM_LATENCY_LEVEL_3_7_MASK	REG_GENMASK(31, 24)
+#define     GEN9_MEM_LATENCY_LEVEL_2_6_MASK	REG_GENMASK(23, 16)
+#define     GEN9_MEM_LATENCY_LEVEL_1_5_MASK	REG_GENMASK(15, 8)
+#define     GEN9_MEM_LATENCY_LEVEL_0_4_MASK	REG_GENMASK(7, 0)
 #define   SKL_PCODE_LOAD_HDCP_KEYS		0x5
 #define   SKL_PCODE_CDCLK_CONTROL		0x7
 #define     SKL_CDCLK_PREPARE_FOR_CHANGE	0x3
@@ -6937,265 +6811,6 @@ enum skl_power_gate {
 #define   ICL_AUX_ANAOVRD1_LDO_BYPASS	(1 << 7)
 #define   ICL_AUX_ANAOVRD1_ENABLE	(1 << 0)
 
-/* HDCP Key Registers */
-#define HDCP_KEY_CONF			_MMIO(0x66c00)
-#define  HDCP_AKSV_SEND_TRIGGER		BIT(31)
-#define  HDCP_CLEAR_KEYS_TRIGGER	BIT(30)
-#define  HDCP_KEY_LOAD_TRIGGER		BIT(8)
-#define HDCP_KEY_STATUS			_MMIO(0x66c04)
-#define  HDCP_FUSE_IN_PROGRESS		BIT(7)
-#define  HDCP_FUSE_ERROR		BIT(6)
-#define  HDCP_FUSE_DONE			BIT(5)
-#define  HDCP_KEY_LOAD_STATUS		BIT(1)
-#define  HDCP_KEY_LOAD_DONE		BIT(0)
-#define HDCP_AKSV_LO			_MMIO(0x66c10)
-#define HDCP_AKSV_HI			_MMIO(0x66c14)
-
-/* HDCP Repeater Registers */
-#define HDCP_REP_CTL			_MMIO(0x66d00)
-#define  HDCP_TRANSA_REP_PRESENT	BIT(31)
-#define  HDCP_TRANSB_REP_PRESENT	BIT(30)
-#define  HDCP_TRANSC_REP_PRESENT	BIT(29)
-#define  HDCP_TRANSD_REP_PRESENT	BIT(28)
-#define  HDCP_DDIB_REP_PRESENT		BIT(30)
-#define  HDCP_DDIA_REP_PRESENT		BIT(29)
-#define  HDCP_DDIC_REP_PRESENT		BIT(28)
-#define  HDCP_DDID_REP_PRESENT		BIT(27)
-#define  HDCP_DDIF_REP_PRESENT		BIT(26)
-#define  HDCP_DDIE_REP_PRESENT		BIT(25)
-#define  HDCP_TRANSA_SHA1_M0		(1 << 20)
-#define  HDCP_TRANSB_SHA1_M0		(2 << 20)
-#define  HDCP_TRANSC_SHA1_M0		(3 << 20)
-#define  HDCP_TRANSD_SHA1_M0		(4 << 20)
-#define  HDCP_DDIB_SHA1_M0		(1 << 20)
-#define  HDCP_DDIA_SHA1_M0		(2 << 20)
-#define  HDCP_DDIC_SHA1_M0		(3 << 20)
-#define  HDCP_DDID_SHA1_M0		(4 << 20)
-#define  HDCP_DDIF_SHA1_M0		(5 << 20)
-#define  HDCP_DDIE_SHA1_M0		(6 << 20) /* Bspec says 5? */
-#define  HDCP_SHA1_BUSY			BIT(16)
-#define  HDCP_SHA1_READY		BIT(17)
-#define  HDCP_SHA1_COMPLETE		BIT(18)
-#define  HDCP_SHA1_V_MATCH		BIT(19)
-#define  HDCP_SHA1_TEXT_32		(1 << 1)
-#define  HDCP_SHA1_COMPLETE_HASH	(2 << 1)
-#define  HDCP_SHA1_TEXT_24		(4 << 1)
-#define  HDCP_SHA1_TEXT_16		(5 << 1)
-#define  HDCP_SHA1_TEXT_8		(6 << 1)
-#define  HDCP_SHA1_TEXT_0		(7 << 1)
-#define HDCP_SHA_V_PRIME_H0		_MMIO(0x66d04)
-#define HDCP_SHA_V_PRIME_H1		_MMIO(0x66d08)
-#define HDCP_SHA_V_PRIME_H2		_MMIO(0x66d0C)
-#define HDCP_SHA_V_PRIME_H3		_MMIO(0x66d10)
-#define HDCP_SHA_V_PRIME_H4		_MMIO(0x66d14)
-#define HDCP_SHA_V_PRIME(h)		_MMIO((0x66d04 + (h) * 4))
-#define HDCP_SHA_TEXT			_MMIO(0x66d18)
-
-/* HDCP Auth Registers */
-#define _PORTA_HDCP_AUTHENC		0x66800
-#define _PORTB_HDCP_AUTHENC		0x66500
-#define _PORTC_HDCP_AUTHENC		0x66600
-#define _PORTD_HDCP_AUTHENC		0x66700
-#define _PORTE_HDCP_AUTHENC		0x66A00
-#define _PORTF_HDCP_AUTHENC		0x66900
-#define _PORT_HDCP_AUTHENC(port, x)	_MMIO(_PICK(port, \
-					  _PORTA_HDCP_AUTHENC, \
-					  _PORTB_HDCP_AUTHENC, \
-					  _PORTC_HDCP_AUTHENC, \
-					  _PORTD_HDCP_AUTHENC, \
-					  _PORTE_HDCP_AUTHENC, \
-					  _PORTF_HDCP_AUTHENC) + (x))
-#define PORT_HDCP_CONF(port)		_PORT_HDCP_AUTHENC(port, 0x0)
-#define _TRANSA_HDCP_CONF		0x66400
-#define _TRANSB_HDCP_CONF		0x66500
-#define TRANS_HDCP_CONF(trans)		_MMIO_TRANS(trans, _TRANSA_HDCP_CONF, \
-						    _TRANSB_HDCP_CONF)
-#define HDCP_CONF(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP_CONF(trans) : \
-					 PORT_HDCP_CONF(port))
-
-#define  HDCP_CONF_CAPTURE_AN		BIT(0)
-#define  HDCP_CONF_AUTH_AND_ENC		(BIT(1) | BIT(0))
-#define PORT_HDCP_ANINIT(port)		_PORT_HDCP_AUTHENC(port, 0x4)
-#define _TRANSA_HDCP_ANINIT		0x66404
-#define _TRANSB_HDCP_ANINIT		0x66504
-#define TRANS_HDCP_ANINIT(trans)	_MMIO_TRANS(trans, \
-						    _TRANSA_HDCP_ANINIT, \
-						    _TRANSB_HDCP_ANINIT)
-#define HDCP_ANINIT(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP_ANINIT(trans) : \
-					 PORT_HDCP_ANINIT(port))
-
-#define PORT_HDCP_ANLO(port)		_PORT_HDCP_AUTHENC(port, 0x8)
-#define _TRANSA_HDCP_ANLO		0x66408
-#define _TRANSB_HDCP_ANLO		0x66508
-#define TRANS_HDCP_ANLO(trans)		_MMIO_TRANS(trans, _TRANSA_HDCP_ANLO, \
-						    _TRANSB_HDCP_ANLO)
-#define HDCP_ANLO(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP_ANLO(trans) : \
-					 PORT_HDCP_ANLO(port))
-
-#define PORT_HDCP_ANHI(port)		_PORT_HDCP_AUTHENC(port, 0xC)
-#define _TRANSA_HDCP_ANHI		0x6640C
-#define _TRANSB_HDCP_ANHI		0x6650C
-#define TRANS_HDCP_ANHI(trans)		_MMIO_TRANS(trans, _TRANSA_HDCP_ANHI, \
-						    _TRANSB_HDCP_ANHI)
-#define HDCP_ANHI(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP_ANHI(trans) : \
-					 PORT_HDCP_ANHI(port))
-
-#define PORT_HDCP_BKSVLO(port)		_PORT_HDCP_AUTHENC(port, 0x10)
-#define _TRANSA_HDCP_BKSVLO		0x66410
-#define _TRANSB_HDCP_BKSVLO		0x66510
-#define TRANS_HDCP_BKSVLO(trans)	_MMIO_TRANS(trans, \
-						    _TRANSA_HDCP_BKSVLO, \
-						    _TRANSB_HDCP_BKSVLO)
-#define HDCP_BKSVLO(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP_BKSVLO(trans) : \
-					 PORT_HDCP_BKSVLO(port))
-
-#define PORT_HDCP_BKSVHI(port)		_PORT_HDCP_AUTHENC(port, 0x14)
-#define _TRANSA_HDCP_BKSVHI		0x66414
-#define _TRANSB_HDCP_BKSVHI		0x66514
-#define TRANS_HDCP_BKSVHI(trans)	_MMIO_TRANS(trans, \
-						    _TRANSA_HDCP_BKSVHI, \
-						    _TRANSB_HDCP_BKSVHI)
-#define HDCP_BKSVHI(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP_BKSVHI(trans) : \
-					 PORT_HDCP_BKSVHI(port))
-
-#define PORT_HDCP_RPRIME(port)		_PORT_HDCP_AUTHENC(port, 0x18)
-#define _TRANSA_HDCP_RPRIME		0x66418
-#define _TRANSB_HDCP_RPRIME		0x66518
-#define TRANS_HDCP_RPRIME(trans)	_MMIO_TRANS(trans, \
-						    _TRANSA_HDCP_RPRIME, \
-						    _TRANSB_HDCP_RPRIME)
-#define HDCP_RPRIME(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP_RPRIME(trans) : \
-					 PORT_HDCP_RPRIME(port))
-
-#define PORT_HDCP_STATUS(port)		_PORT_HDCP_AUTHENC(port, 0x1C)
-#define _TRANSA_HDCP_STATUS		0x6641C
-#define _TRANSB_HDCP_STATUS		0x6651C
-#define TRANS_HDCP_STATUS(trans)	_MMIO_TRANS(trans, \
-						    _TRANSA_HDCP_STATUS, \
-						    _TRANSB_HDCP_STATUS)
-#define HDCP_STATUS(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP_STATUS(trans) : \
-					 PORT_HDCP_STATUS(port))
-
-#define  HDCP_STATUS_STREAM_A_ENC	BIT(31)
-#define  HDCP_STATUS_STREAM_B_ENC	BIT(30)
-#define  HDCP_STATUS_STREAM_C_ENC	BIT(29)
-#define  HDCP_STATUS_STREAM_D_ENC	BIT(28)
-#define  HDCP_STATUS_AUTH		BIT(21)
-#define  HDCP_STATUS_ENC		BIT(20)
-#define  HDCP_STATUS_RI_MATCH		BIT(19)
-#define  HDCP_STATUS_R0_READY		BIT(18)
-#define  HDCP_STATUS_AN_READY		BIT(17)
-#define  HDCP_STATUS_CIPHER		BIT(16)
-#define  HDCP_STATUS_FRAME_CNT(x)	(((x) >> 8) & 0xff)
-
-/* HDCP2.2 Registers */
-#define _PORTA_HDCP2_BASE		0x66800
-#define _PORTB_HDCP2_BASE		0x66500
-#define _PORTC_HDCP2_BASE		0x66600
-#define _PORTD_HDCP2_BASE		0x66700
-#define _PORTE_HDCP2_BASE		0x66A00
-#define _PORTF_HDCP2_BASE		0x66900
-#define _PORT_HDCP2_BASE(port, x)	_MMIO(_PICK((port), \
-					  _PORTA_HDCP2_BASE, \
-					  _PORTB_HDCP2_BASE, \
-					  _PORTC_HDCP2_BASE, \
-					  _PORTD_HDCP2_BASE, \
-					  _PORTE_HDCP2_BASE, \
-					  _PORTF_HDCP2_BASE) + (x))
-
-#define PORT_HDCP2_AUTH(port)		_PORT_HDCP2_BASE(port, 0x98)
-#define _TRANSA_HDCP2_AUTH		0x66498
-#define _TRANSB_HDCP2_AUTH		0x66598
-#define TRANS_HDCP2_AUTH(trans)		_MMIO_TRANS(trans, _TRANSA_HDCP2_AUTH, \
-						    _TRANSB_HDCP2_AUTH)
-#define   AUTH_LINK_AUTHENTICATED	BIT(31)
-#define   AUTH_LINK_TYPE		BIT(30)
-#define   AUTH_FORCE_CLR_INPUTCTR	BIT(19)
-#define   AUTH_CLR_KEYS			BIT(18)
-#define HDCP2_AUTH(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP2_AUTH(trans) : \
-					 PORT_HDCP2_AUTH(port))
-
-#define PORT_HDCP2_CTL(port)		_PORT_HDCP2_BASE(port, 0xB0)
-#define _TRANSA_HDCP2_CTL		0x664B0
-#define _TRANSB_HDCP2_CTL		0x665B0
-#define TRANS_HDCP2_CTL(trans)		_MMIO_TRANS(trans, _TRANSA_HDCP2_CTL, \
-						    _TRANSB_HDCP2_CTL)
-#define   CTL_LINK_ENCRYPTION_REQ	BIT(31)
-#define HDCP2_CTL(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP2_CTL(trans) : \
-					 PORT_HDCP2_CTL(port))
-
-#define PORT_HDCP2_STATUS(port)		_PORT_HDCP2_BASE(port, 0xB4)
-#define _TRANSA_HDCP2_STATUS		0x664B4
-#define _TRANSB_HDCP2_STATUS		0x665B4
-#define TRANS_HDCP2_STATUS(trans)	_MMIO_TRANS(trans, \
-						    _TRANSA_HDCP2_STATUS, \
-						    _TRANSB_HDCP2_STATUS)
-#define   LINK_TYPE_STATUS		BIT(22)
-#define   LINK_AUTH_STATUS		BIT(21)
-#define   LINK_ENCRYPTION_STATUS	BIT(20)
-#define HDCP2_STATUS(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP2_STATUS(trans) : \
-					 PORT_HDCP2_STATUS(port))
-
-#define _PIPEA_HDCP2_STREAM_STATUS	0x668C0
-#define _PIPEB_HDCP2_STREAM_STATUS	0x665C0
-#define _PIPEC_HDCP2_STREAM_STATUS	0x666C0
-#define _PIPED_HDCP2_STREAM_STATUS	0x667C0
-#define PIPE_HDCP2_STREAM_STATUS(pipe)		_MMIO(_PICK((pipe), \
-						      _PIPEA_HDCP2_STREAM_STATUS, \
-						      _PIPEB_HDCP2_STREAM_STATUS, \
-						      _PIPEC_HDCP2_STREAM_STATUS, \
-						      _PIPED_HDCP2_STREAM_STATUS))
-
-#define _TRANSA_HDCP2_STREAM_STATUS		0x664C0
-#define _TRANSB_HDCP2_STREAM_STATUS		0x665C0
-#define TRANS_HDCP2_STREAM_STATUS(trans)	_MMIO_TRANS(trans, \
-						    _TRANSA_HDCP2_STREAM_STATUS, \
-						    _TRANSB_HDCP2_STREAM_STATUS)
-#define   STREAM_ENCRYPTION_STATUS	BIT(31)
-#define   STREAM_TYPE_STATUS		BIT(30)
-#define HDCP2_STREAM_STATUS(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP2_STREAM_STATUS(trans) : \
-					 PIPE_HDCP2_STREAM_STATUS(pipe))
-
-#define _PORTA_HDCP2_AUTH_STREAM		0x66F00
-#define _PORTB_HDCP2_AUTH_STREAM		0x66F04
-#define PORT_HDCP2_AUTH_STREAM(port)	_MMIO_PORT(port, \
-						   _PORTA_HDCP2_AUTH_STREAM, \
-						   _PORTB_HDCP2_AUTH_STREAM)
-#define _TRANSA_HDCP2_AUTH_STREAM		0x66F00
-#define _TRANSB_HDCP2_AUTH_STREAM		0x66F04
-#define TRANS_HDCP2_AUTH_STREAM(trans)	_MMIO_TRANS(trans, \
-						    _TRANSA_HDCP2_AUTH_STREAM, \
-						    _TRANSB_HDCP2_AUTH_STREAM)
-#define   AUTH_STREAM_TYPE		BIT(31)
-#define HDCP2_AUTH_STREAM(dev_priv, trans, port) \
-					(GRAPHICS_VER(dev_priv) >= 12 ? \
-					 TRANS_HDCP2_AUTH_STREAM(trans) : \
-					 PORT_HDCP2_AUTH_STREAM(port))
-
 /* Per-pipe DDI Function Control */
 #define _TRANS_DDI_FUNC_CTL_A		0x60400
 #define _TRANS_DDI_FUNC_CTL_B		0x61400
@@ -7503,16 +7118,16 @@ enum skl_power_gate {
 
 /* CDCLK_CTL */
 #define CDCLK_CTL			_MMIO(0x46000)
-#define  CDCLK_FREQ_SEL_MASK		(3 << 26)
-#define  CDCLK_FREQ_450_432		(0 << 26)
-#define  CDCLK_FREQ_540			(1 << 26)
-#define  CDCLK_FREQ_337_308		(2 << 26)
-#define  CDCLK_FREQ_675_617		(3 << 26)
-#define  BXT_CDCLK_CD2X_DIV_SEL_MASK	(3 << 22)
-#define  BXT_CDCLK_CD2X_DIV_SEL_1	(0 << 22)
-#define  BXT_CDCLK_CD2X_DIV_SEL_1_5	(1 << 22)
-#define  BXT_CDCLK_CD2X_DIV_SEL_2	(2 << 22)
-#define  BXT_CDCLK_CD2X_DIV_SEL_4	(3 << 22)
+#define  CDCLK_FREQ_SEL_MASK		REG_GENMASK(27, 26)
+#define  CDCLK_FREQ_450_432		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 0)
+#define  CDCLK_FREQ_540		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 1)
+#define  CDCLK_FREQ_337_308		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 2)
+#define  CDCLK_FREQ_675_617		REG_FIELD_PREP(CDCLK_FREQ_SEL_MASK, 3)
+#define  BXT_CDCLK_CD2X_DIV_SEL_MASK	REG_GENMASK(23, 22)
+#define  BXT_CDCLK_CD2X_DIV_SEL_1	REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 0)
+#define  BXT_CDCLK_CD2X_DIV_SEL_1_5	REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 1)
+#define  BXT_CDCLK_CD2X_DIV_SEL_2	REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 2)
+#define  BXT_CDCLK_CD2X_DIV_SEL_4	REG_FIELD_PREP(BXT_CDCLK_CD2X_DIV_SEL_MASK, 3)
 #define  BXT_CDCLK_CD2X_PIPE(pipe)	((pipe) << 20)
 #define  CDCLK_DIVMUX_CD_OVERRIDE	(1 << 19)
 #define  BXT_CDCLK_CD2X_PIPE_NONE	BXT_CDCLK_CD2X_PIPE(3)
@@ -8367,6 +7982,7 @@ enum skl_power_gate {
 #define ICL_DSC1_PICTURE_PARAMETER_SET_0(pipe)	_MMIO_PIPE((pipe) - PIPE_B, \
 							   _ICL_DSC1_PICTURE_PARAMETER_SET_0_PB, \
 							   _ICL_DSC1_PICTURE_PARAMETER_SET_0_PC)
+#define  DSC_ALT_ICH_SEL		(1 << 20)
 #define  DSC_VBR_ENABLE			(1 << 19)
 #define  DSC_422_ENABLE			(1 << 18)
 #define  DSC_COLOR_SPACE_CONVERSION	(1 << 17)
@@ -8717,4 +8333,27 @@ enum skl_power_gate {
 #define GEN12_CULLBIT2			_MMIO(0x7030)
 #define GEN12_STATE_ACK_DEBUG		_MMIO(0x20BC)
 
+#define MTL_LATENCY_LP0_LP1		_MMIO(0x45780)
+#define MTL_LATENCY_LP2_LP3		_MMIO(0x45784)
+#define MTL_LATENCY_LP4_LP5		_MMIO(0x45788)
+#define  MTL_LATENCY_LEVEL_EVEN_MASK	REG_GENMASK(12, 0)
+#define  MTL_LATENCY_LEVEL_ODD_MASK	REG_GENMASK(28, 16)
+
+#define MTL_LATENCY_SAGV		_MMIO(0x4578b)
+#define   MTL_LATENCY_QCLK_SAGV		REG_GENMASK(12, 0)
+
+#define MTL_MEM_SS_INFO_GLOBAL			_MMIO(0x45700)
+#define   MTL_N_OF_ENABLED_QGV_POINTS_MASK	REG_GENMASK(11, 8)
+#define   MTL_N_OF_POPULATED_CH_MASK		REG_GENMASK(7, 4)
+#define   MTL_DDR_TYPE_MASK			REG_GENMASK(3, 0)
+
+#define MTL_MEM_SS_INFO_QGV_POINT_LOW(point)	 _MMIO(0x45710 + (point) * 2)
+#define   MTL_TRCD_MASK			REG_GENMASK(31, 24)
+#define   MTL_TRP_MASK			REG_GENMASK(23, 16)
+#define   MTL_DCLK_MASK			REG_GENMASK(15, 0)
+
+#define MTL_MEM_SS_INFO_QGV_POINT_HIGH(point)	 _MMIO(0x45714 + (point) * 2)
+#define   MTL_TRAS_MASK			REG_GENMASK(16, 8)
+#define   MTL_TRDPRE_MASK		REG_GENMASK(7, 0)
+
 #endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.c b/drivers/gpu/drm/i915/i915_sw_fence.c
index ae984c66c48a..6fc0d1b89690 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.c
+++ b/drivers/gpu/drm/i915/i915_sw_fence.c
@@ -241,8 +241,6 @@ void __i915_sw_fence_init(struct i915_sw_fence *fence,
 			  const char *name,
 			  struct lock_class_key *key)
 {
-	BUG_ON(!fn);
-
 	__init_waitqueue_head(&fence->wait, name, key);
 	fence->fn = fn;
 #ifdef CONFIG_DRM_I915_SW_FENCE_CHECK_DAG
diff --git a/drivers/gpu/drm/i915/i915_sw_fence.h b/drivers/gpu/drm/i915/i915_sw_fence.h
index a7c603bc1b01..619fc5a22f0c 100644
--- a/drivers/gpu/drm/i915/i915_sw_fence.h
+++ b/drivers/gpu/drm/i915/i915_sw_fence.h
@@ -48,11 +48,15 @@ void __i915_sw_fence_init(struct i915_sw_fence *fence,
 do {								\
 	static struct lock_class_key __key;			\
 								\
+	BUILD_BUG_ON((fn) == NULL);				\
 	__i915_sw_fence_init((fence), (fn), #fence, &__key);	\
 } while (0)
 #else
 #define i915_sw_fence_init(fence, fn)				\
-	__i915_sw_fence_init((fence), (fn), NULL, NULL)
+do {								\
+	BUILD_BUG_ON((fn) == NULL);				\
+	__i915_sw_fence_init((fence), (fn), NULL, NULL);	\
+} while (0)
 #endif
 
 void i915_sw_fence_reinit(struct i915_sw_fence *fence);
diff --git a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
index 427de1aaab36..e19452f0e100 100644
--- a/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
+++ b/drivers/gpu/drm/i915/i915_ttm_buddy_manager.c
@@ -173,6 +173,77 @@ static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man,
 	kfree(bman_res);
 }
 
+static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man,
+					  struct ttm_resource *res,
+					  const struct ttm_place *place,
+					  size_t size)
+{
+	struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
+	struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
+	struct drm_buddy *mm = &bman->mm;
+	struct drm_buddy_block *block;
+
+	if (!place->fpfn && !place->lpfn)
+		return true;
+
+	GEM_BUG_ON(!place->lpfn);
+
+	/*
+	 * If we just want something mappable then we can quickly check
+	 * if the current victim resource is using any of the CPU
+	 * visible portion.
+	 */
+	if (!place->fpfn &&
+	    place->lpfn == i915_ttm_buddy_man_visible_size(man))
+		return bman_res->used_visible_size > 0;
+
+	/* Check each drm buddy block individually */
+	list_for_each_entry(block, &bman_res->blocks, link) {
+		unsigned long fpfn =
+			drm_buddy_block_offset(block) >> PAGE_SHIFT;
+		unsigned long lpfn = fpfn +
+			(drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
+
+		if (place->fpfn < lpfn && place->lpfn > fpfn)
+			return true;
+	}
+
+	return false;
+}
+
+static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man,
+					  struct ttm_resource *res,
+					  const struct ttm_place *place,
+					  size_t size)
+{
+	struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res);
+	struct i915_ttm_buddy_manager *bman = to_buddy_manager(man);
+	struct drm_buddy *mm = &bman->mm;
+	struct drm_buddy_block *block;
+
+	if (!place->fpfn && !place->lpfn)
+		return true;
+
+	GEM_BUG_ON(!place->lpfn);
+
+	if (!place->fpfn &&
+	    place->lpfn == i915_ttm_buddy_man_visible_size(man))
+		return bman_res->used_visible_size == res->num_pages;
+
+	/* Check each drm buddy block individually */
+	list_for_each_entry(block, &bman_res->blocks, link) {
+		unsigned long fpfn =
+			drm_buddy_block_offset(block) >> PAGE_SHIFT;
+		unsigned long lpfn = fpfn +
+			(drm_buddy_block_size(mm, block) >> PAGE_SHIFT);
+
+		if (fpfn < place->fpfn || lpfn > place->lpfn)
+			return false;
+	}
+
+	return true;
+}
+
 static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man,
 				     struct drm_printer *printer)
 {
@@ -200,6 +271,8 @@ static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man,
 static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = {
 	.alloc = i915_ttm_buddy_man_alloc,
 	.free = i915_ttm_buddy_man_free,
+	.intersects = i915_ttm_buddy_man_intersects,
+	.compatible = i915_ttm_buddy_man_compatible,
 	.debug = i915_ttm_buddy_man_debug,
 };
 
diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h
index c10d68cdc3ca..6c14d13364bf 100644
--- a/drivers/gpu/drm/i915/i915_utils.h
+++ b/drivers/gpu/drm/i915/i915_utils.h
@@ -360,10 +360,6 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms)
 #define KHz(x) (1000 * (x))
 #define MHz(x) KHz(1000 * (x))
 
-#define KBps(x) (1000 * (x))
-#define MBps(x) KBps(1000 * (x))
-#define GBps(x) ((u64)1000 * MBps((x)))
-
 void add_taint_for_CI(struct drm_i915_private *i915, unsigned int taint);
 static inline void __add_taint_for_CI(unsigned int taint)
 {
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 373582cfd8f3..f17c09ead7d7 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -109,7 +109,7 @@ static void __i915_vma_retire(struct i915_active *ref)
 static struct i915_vma *
 vma_create(struct drm_i915_gem_object *obj,
 	   struct i915_address_space *vm,
-	   const struct i915_ggtt_view *view)
+	   const struct i915_gtt_view *view)
 {
 	struct i915_vma *pos = ERR_PTR(-E2BIG);
 	struct i915_vma *vma;
@@ -141,9 +141,9 @@ vma_create(struct drm_i915_gem_object *obj,
 	INIT_LIST_HEAD(&vma->obj_link);
 	RB_CLEAR_NODE(&vma->obj_node);
 
-	if (view && view->type != I915_GGTT_VIEW_NORMAL) {
-		vma->ggtt_view = *view;
-		if (view->type == I915_GGTT_VIEW_PARTIAL) {
+	if (view && view->type != I915_GTT_VIEW_NORMAL) {
+		vma->gtt_view = *view;
+		if (view->type == I915_GTT_VIEW_PARTIAL) {
 			GEM_BUG_ON(range_overflows_t(u64,
 						     view->partial.offset,
 						     view->partial.size,
@@ -151,10 +151,10 @@ vma_create(struct drm_i915_gem_object *obj,
 			vma->size = view->partial.size;
 			vma->size <<= PAGE_SHIFT;
 			GEM_BUG_ON(vma->size > obj->base.size);
-		} else if (view->type == I915_GGTT_VIEW_ROTATED) {
+		} else if (view->type == I915_GTT_VIEW_ROTATED) {
 			vma->size = intel_rotation_info_size(&view->rotated);
 			vma->size <<= PAGE_SHIFT;
-		} else if (view->type == I915_GGTT_VIEW_REMAPPED) {
+		} else if (view->type == I915_GTT_VIEW_REMAPPED) {
 			vma->size = intel_remapped_info_size(&view->remapped);
 			vma->size <<= PAGE_SHIFT;
 		}
@@ -248,7 +248,7 @@ err_vma:
 static struct i915_vma *
 i915_vma_lookup(struct drm_i915_gem_object *obj,
 	   struct i915_address_space *vm,
-	   const struct i915_ggtt_view *view)
+	   const struct i915_gtt_view *view)
 {
 	struct rb_node *rb;
 
@@ -286,7 +286,7 @@ i915_vma_lookup(struct drm_i915_gem_object *obj,
 struct i915_vma *
 i915_vma_instance(struct drm_i915_gem_object *obj,
 		  struct i915_address_space *vm,
-		  const struct i915_ggtt_view *view)
+		  const struct i915_gtt_view *view)
 {
 	struct i915_vma *vma;
 
@@ -1203,7 +1203,7 @@ err_st_alloc:
 }
 
 static noinline struct sg_table *
-intel_partial_pages(const struct i915_ggtt_view *view,
+intel_partial_pages(const struct i915_gtt_view *view,
 		    struct drm_i915_gem_object *obj)
 {
 	struct sg_table *st;
@@ -1247,33 +1247,33 @@ __i915_vma_get_pages(struct i915_vma *vma)
 	 */
 	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj));
 
-	switch (vma->ggtt_view.type) {
+	switch (vma->gtt_view.type) {
 	default:
-		GEM_BUG_ON(vma->ggtt_view.type);
+		GEM_BUG_ON(vma->gtt_view.type);
 		fallthrough;
-	case I915_GGTT_VIEW_NORMAL:
+	case I915_GTT_VIEW_NORMAL:
 		pages = vma->obj->mm.pages;
 		break;
 
-	case I915_GGTT_VIEW_ROTATED:
+	case I915_GTT_VIEW_ROTATED:
 		pages =
-			intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj);
+			intel_rotate_pages(&vma->gtt_view.rotated, vma->obj);
 		break;
 
-	case I915_GGTT_VIEW_REMAPPED:
+	case I915_GTT_VIEW_REMAPPED:
 		pages =
-			intel_remap_pages(&vma->ggtt_view.remapped, vma->obj);
+			intel_remap_pages(&vma->gtt_view.remapped, vma->obj);
 		break;
 
-	case I915_GGTT_VIEW_PARTIAL:
-		pages = intel_partial_pages(&vma->ggtt_view, vma->obj);
+	case I915_GTT_VIEW_PARTIAL:
+		pages = intel_partial_pages(&vma->gtt_view, vma->obj);
 		break;
 	}
 
 	if (IS_ERR(pages)) {
 		drm_err(&vma->vm->i915->drm,
 			"Failed to get pages for VMA view type %u (%ld)!\n",
-			vma->ggtt_view.type, PTR_ERR(pages));
+			vma->gtt_view.type, PTR_ERR(pages));
 		return PTR_ERR(pages);
 	}
 
@@ -1806,7 +1806,7 @@ void i915_vma_revoke_mmap(struct i915_vma *vma)
 	GEM_BUG_ON(!vma->obj->userfault_count);
 
 	node = &vma->mmo->vma_node;
-	vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
+	vma_offset = vma->gtt_view.partial.offset << PAGE_SHIFT;
 	unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
 			    drm_vma_node_offset_addr(node) + vma_offset,
 			    vma->size,
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 33a58f605d75..aecd9c64486b 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -43,7 +43,7 @@
 struct i915_vma *
 i915_vma_instance(struct drm_i915_gem_object *obj,
 		  struct i915_address_space *vm,
-		  const struct i915_ggtt_view *view);
+		  const struct i915_gtt_view *view);
 
 void i915_vma_unpin_and_release(struct i915_vma **p_vma, unsigned int flags);
 #define I915_VMA_RELEASE_MAP BIT(0)
@@ -160,7 +160,7 @@ static inline void i915_vma_put(struct i915_vma *vma)
 static inline long
 i915_vma_compare(struct i915_vma *vma,
 		 struct i915_address_space *vm,
-		 const struct i915_ggtt_view *view)
+		 const struct i915_gtt_view *view)
 {
 	ptrdiff_t cmp;
 
@@ -170,8 +170,8 @@ i915_vma_compare(struct i915_vma *vma,
 	if (cmp)
 		return cmp;
 
-	BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL != 0);
-	cmp = vma->ggtt_view.type;
+	BUILD_BUG_ON(I915_GTT_VIEW_NORMAL != 0);
+	cmp = vma->gtt_view.type;
 	if (!view)
 		return cmp;
 
@@ -181,7 +181,7 @@ i915_vma_compare(struct i915_vma *vma,
 
 	assert_i915_gem_gtt_types();
 
-	/* ggtt_view.type also encodes its size so that we both distinguish
+	/* gtt_view.type also encodes its size so that we both distinguish
 	 * different views using it as a "type" and also use a compact (no
 	 * accessing of uninitialised padding bytes) memcmp without storing
 	 * an extra parameter or adding more code.
@@ -191,14 +191,14 @@ i915_vma_compare(struct i915_vma *vma,
 	 * we assert above that all branches have the same address, and that
 	 * each branch has a unique type/size.
 	 */
-	BUILD_BUG_ON(I915_GGTT_VIEW_NORMAL >= I915_GGTT_VIEW_PARTIAL);
-	BUILD_BUG_ON(I915_GGTT_VIEW_PARTIAL >= I915_GGTT_VIEW_ROTATED);
-	BUILD_BUG_ON(I915_GGTT_VIEW_ROTATED >= I915_GGTT_VIEW_REMAPPED);
+	BUILD_BUG_ON(I915_GTT_VIEW_NORMAL >= I915_GTT_VIEW_PARTIAL);
+	BUILD_BUG_ON(I915_GTT_VIEW_PARTIAL >= I915_GTT_VIEW_ROTATED);
+	BUILD_BUG_ON(I915_GTT_VIEW_ROTATED >= I915_GTT_VIEW_REMAPPED);
 	BUILD_BUG_ON(offsetof(typeof(*view), rotated) !=
 		     offsetof(typeof(*view), partial));
 	BUILD_BUG_ON(offsetof(typeof(*view), rotated) !=
 		     offsetof(typeof(*view), remapped));
-	return memcmp(&vma->ggtt_view.partial, &view->partial, view->type);
+	return memcmp(&vma->gtt_view.partial, &view->partial, view->type);
 }
 
 struct i915_vma_work *i915_vma_work(void);
diff --git a/drivers/gpu/drm/i915/i915_vma_resource.c b/drivers/gpu/drm/i915/i915_vma_resource.c
index 5a67995ea5fe..de1342dbfa12 100644
--- a/drivers/gpu/drm/i915/i915_vma_resource.c
+++ b/drivers/gpu/drm/i915/i915_vma_resource.c
@@ -216,6 +216,10 @@ i915_vma_resource_fence_notify(struct i915_sw_fence *fence,
 /**
  * i915_vma_resource_unbind - Unbind a vma resource
  * @vma_res: The vma resource to unbind.
+ * @tlb: pointer to vma->obj->mm.tlb associated with the resource
+ *	 to be stored at vma_res->tlb. When not-NULL, it will be used
+ *	 to do TLB cache invalidation before freeing a VMA resource.
+ *	 Used only for async unbind.
  *
  * At this point this function does little more than publish a fence that
  * signals immediately unless signaling is held back.
diff --git a/drivers/gpu/drm/i915/i915_vma_types.h b/drivers/gpu/drm/i915/i915_vma_types.h
index be6e028c3b57..ec0f6c9f57d0 100644
--- a/drivers/gpu/drm/i915/i915_vma_types.h
+++ b/drivers/gpu/drm/i915/i915_vma_types.h
@@ -67,30 +67,30 @@ enum i915_cache_level;
  * Implementation and usage
  *
  * GGTT views are implemented using VMAs and are distinguished via enum
- * i915_ggtt_view_type and struct i915_ggtt_view.
+ * i915_gtt_view_type and struct i915_gtt_view.
  *
  * A new flavour of core GEM functions which work with GGTT bound objects were
  * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
- * renaming  in large amounts of code. They take the struct i915_ggtt_view
+ * renaming  in large amounts of code. They take the struct i915_gtt_view
  * parameter encapsulating all metadata required to implement a view.
  *
  * As a helper for callers which are only interested in the normal view,
- * globally const i915_ggtt_view_normal singleton instance exists. All old core
+ * globally const i915_gtt_view_normal singleton instance exists. All old core
  * GEM API functions, the ones not taking the view parameter, are operating on,
  * or with the normal GGTT view.
  *
  * Code wanting to add or use a new GGTT view needs to:
  *
  * 1. Add a new enum with a suitable name.
- * 2. Extend the metadata in the i915_ggtt_view structure if required.
+ * 2. Extend the metadata in the i915_gtt_view structure if required.
  * 3. Add support to i915_get_vma_pages().
  *
  * New views are required to build a scatter-gather table from within the
- * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
+ * i915_get_vma_pages function. This table is stored in the vma.gtt_view and
  * exists for the lifetime of an VMA.
  *
  * Core API is designed to have copy semantics which means that passed in
- * struct i915_ggtt_view does not need to be persistent (left around after
+ * struct i915_gtt_view does not need to be persistent (left around after
  * calling the core API functions).
  *
  */
@@ -130,11 +130,11 @@ struct intel_partial_info {
 	unsigned int size;
 } __packed;
 
-enum i915_ggtt_view_type {
-	I915_GGTT_VIEW_NORMAL = 0,
-	I915_GGTT_VIEW_ROTATED = sizeof(struct intel_rotation_info),
-	I915_GGTT_VIEW_PARTIAL = sizeof(struct intel_partial_info),
-	I915_GGTT_VIEW_REMAPPED = sizeof(struct intel_remapped_info),
+enum i915_gtt_view_type {
+	I915_GTT_VIEW_NORMAL = 0,
+	I915_GTT_VIEW_ROTATED = sizeof(struct intel_rotation_info),
+	I915_GTT_VIEW_PARTIAL = sizeof(struct intel_partial_info),
+	I915_GTT_VIEW_REMAPPED = sizeof(struct intel_remapped_info),
 };
 
 static inline void assert_i915_gem_gtt_types(void)
@@ -152,18 +152,18 @@ static inline void assert_i915_gem_gtt_types(void)
 	/* As we encode the size of each branch inside the union into its type,
 	 * we have to be careful that each branch has a unique size.
 	 */
-	switch ((enum i915_ggtt_view_type)0) {
-	case I915_GGTT_VIEW_NORMAL:
-	case I915_GGTT_VIEW_PARTIAL:
-	case I915_GGTT_VIEW_ROTATED:
-	case I915_GGTT_VIEW_REMAPPED:
+	switch ((enum i915_gtt_view_type)0) {
+	case I915_GTT_VIEW_NORMAL:
+	case I915_GTT_VIEW_PARTIAL:
+	case I915_GTT_VIEW_ROTATED:
+	case I915_GTT_VIEW_REMAPPED:
 		/* gcc complains if these are identical cases */
 		break;
 	}
 }
 
-struct i915_ggtt_view {
-	enum i915_ggtt_view_type type;
+struct i915_gtt_view {
+	enum i915_gtt_view_type type;
 	union {
 		/* Members need to contain no holes/padding */
 		struct intel_partial_info partial;
@@ -280,11 +280,11 @@ struct i915_vma {
 	/**
 	 * Support different GGTT views into the same object.
 	 * This means there can be multiple VMA mappings per object and per VM.
-	 * i915_ggtt_view_type is used to distinguish between those entries.
-	 * The default one of zero (I915_GGTT_VIEW_NORMAL) is default and also
+	 * i915_gtt_view_type is used to distinguish between those entries.
+	 * The default one of zero (I915_GTT_VIEW_NORMAL) is default and also
 	 * assumed in GEM functions which take no ggtt view parameter.
 	 */
-	struct i915_ggtt_view ggtt_view;
+	struct i915_gtt_view gtt_view;
 
 	/** This object's place on the active/inactive lists */
 	struct list_head vm_link;
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index d98fbbd589aa..20575eb77ea7 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -88,46 +88,57 @@ const char *intel_platform_name(enum intel_platform platform)
 	return platform_names[platform];
 }
 
-void intel_device_info_print_static(const struct intel_device_info *info,
-				    struct drm_printer *p)
+void intel_device_info_print(const struct intel_device_info *info,
+			     const struct intel_runtime_info *runtime,
+			     struct drm_printer *p)
 {
-	if (info->graphics.rel)
-		drm_printf(p, "graphics version: %u.%02u\n", info->graphics.ver,
-			   info->graphics.rel);
+	if (runtime->graphics.ip.rel)
+		drm_printf(p, "graphics version: %u.%02u\n",
+			   runtime->graphics.ip.ver,
+			   runtime->graphics.ip.rel);
 	else
-		drm_printf(p, "graphics version: %u\n", info->graphics.ver);
+		drm_printf(p, "graphics version: %u\n",
+			   runtime->graphics.ip.ver);
 
-	if (info->media.rel)
-		drm_printf(p, "media version: %u.%02u\n", info->media.ver, info->media.rel);
+	if (runtime->media.ip.rel)
+		drm_printf(p, "media version: %u.%02u\n",
+			   runtime->media.ip.ver,
+			   runtime->media.ip.rel);
 	else
-		drm_printf(p, "media version: %u\n", info->media.ver);
+		drm_printf(p, "media version: %u\n",
+			   runtime->media.ip.ver);
 
-	if (info->display.rel)
-		drm_printf(p, "display version: %u.%02u\n", info->display.ver, info->display.rel);
+	if (runtime->display.ip.rel)
+		drm_printf(p, "display version: %u.%02u\n",
+			   runtime->display.ip.ver,
+			   runtime->display.ip.rel);
 	else
-		drm_printf(p, "display version: %u\n", info->display.ver);
+		drm_printf(p, "display version: %u\n",
+			   runtime->display.ip.ver);
 
 	drm_printf(p, "gt: %d\n", info->gt);
-	drm_printf(p, "memory-regions: %x\n", info->memory_regions);
-	drm_printf(p, "page-sizes: %x\n", info->page_sizes);
+	drm_printf(p, "memory-regions: %x\n", runtime->memory_regions);
+	drm_printf(p, "page-sizes: %x\n", runtime->page_sizes);
 	drm_printf(p, "platform: %s\n", intel_platform_name(info->platform));
-	drm_printf(p, "ppgtt-size: %d\n", info->ppgtt_size);
-	drm_printf(p, "ppgtt-type: %d\n", info->ppgtt_type);
+	drm_printf(p, "ppgtt-size: %d\n", runtime->ppgtt_size);
+	drm_printf(p, "ppgtt-type: %d\n", runtime->ppgtt_type);
 	drm_printf(p, "dma_mask_size: %u\n", info->dma_mask_size);
 
 #define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, str_yes_no(info->name))
 	DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG);
 #undef PRINT_FLAG
 
+	drm_printf(p, "has_pooled_eu: %s\n", str_yes_no(runtime->has_pooled_eu));
+
 #define PRINT_FLAG(name) drm_printf(p, "%s: %s\n", #name, str_yes_no(info->display.name))
 	DEV_INFO_DISPLAY_FOR_EACH_FLAG(PRINT_FLAG);
 #undef PRINT_FLAG
-}
 
-void intel_device_info_print_runtime(const struct intel_runtime_info *info,
-				     struct drm_printer *p)
-{
-	drm_printf(p, "rawclk rate: %u kHz\n", info->rawclk_freq);
+	drm_printf(p, "has_hdcp: %s\n", str_yes_no(runtime->has_hdcp));
+	drm_printf(p, "has_dmc: %s\n", str_yes_no(runtime->has_dmc));
+	drm_printf(p, "has_dsc: %s\n", str_yes_no(runtime->has_dsc));
+
+	drm_printf(p, "rawclk rate: %u kHz\n", runtime->rawclk_freq);
 }
 
 #undef INTEL_VGA_DEVICE
@@ -364,55 +375,55 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 		     !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) {
 			drm_info(&dev_priv->drm,
 				 "Display fused off, disabling\n");
-			info->display.pipe_mask = 0;
-			info->display.cpu_transcoder_mask = 0;
-			info->display.fbc_mask = 0;
+			runtime->pipe_mask = 0;
+			runtime->cpu_transcoder_mask = 0;
+			runtime->fbc_mask = 0;
 		} else if (fuse_strap & IVB_PIPE_C_DISABLE) {
 			drm_info(&dev_priv->drm, "PipeC fused off\n");
-			info->display.pipe_mask &= ~BIT(PIPE_C);
-			info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_C);
+			runtime->pipe_mask &= ~BIT(PIPE_C);
+			runtime->cpu_transcoder_mask &= ~BIT(TRANSCODER_C);
 		}
 	} else if (HAS_DISPLAY(dev_priv) && DISPLAY_VER(dev_priv) >= 9) {
 		u32 dfsm = intel_de_read(dev_priv, SKL_DFSM);
 
 		if (dfsm & SKL_DFSM_PIPE_A_DISABLE) {
-			info->display.pipe_mask &= ~BIT(PIPE_A);
-			info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_A);
-			info->display.fbc_mask &= ~BIT(INTEL_FBC_A);
+			runtime->pipe_mask &= ~BIT(PIPE_A);
+			runtime->cpu_transcoder_mask &= ~BIT(TRANSCODER_A);
+			runtime->fbc_mask &= ~BIT(INTEL_FBC_A);
 		}
 		if (dfsm & SKL_DFSM_PIPE_B_DISABLE) {
-			info->display.pipe_mask &= ~BIT(PIPE_B);
-			info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_B);
+			runtime->pipe_mask &= ~BIT(PIPE_B);
+			runtime->cpu_transcoder_mask &= ~BIT(TRANSCODER_B);
 		}
 		if (dfsm & SKL_DFSM_PIPE_C_DISABLE) {
-			info->display.pipe_mask &= ~BIT(PIPE_C);
-			info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_C);
+			runtime->pipe_mask &= ~BIT(PIPE_C);
+			runtime->cpu_transcoder_mask &= ~BIT(TRANSCODER_C);
 		}
 
 		if (DISPLAY_VER(dev_priv) >= 12 &&
 		    (dfsm & TGL_DFSM_PIPE_D_DISABLE)) {
-			info->display.pipe_mask &= ~BIT(PIPE_D);
-			info->display.cpu_transcoder_mask &= ~BIT(TRANSCODER_D);
+			runtime->pipe_mask &= ~BIT(PIPE_D);
+			runtime->cpu_transcoder_mask &= ~BIT(TRANSCODER_D);
 		}
 
 		if (dfsm & SKL_DFSM_DISPLAY_HDCP_DISABLE)
-			info->display.has_hdcp = 0;
+			runtime->has_hdcp = 0;
 
 		if (dfsm & SKL_DFSM_DISPLAY_PM_DISABLE)
-			info->display.fbc_mask = 0;
+			runtime->fbc_mask = 0;
 
 		if (DISPLAY_VER(dev_priv) >= 11 && (dfsm & ICL_DFSM_DMC_DISABLE))
-			info->display.has_dmc = 0;
+			runtime->has_dmc = 0;
 
 		if (DISPLAY_VER(dev_priv) >= 10 &&
 		    (dfsm & GLK_DFSM_DISPLAY_DSC_DISABLE))
-			info->display.has_dsc = 0;
+			runtime->has_dsc = 0;
 	}
 
 	if (GRAPHICS_VER(dev_priv) == 6 && i915_vtd_active(dev_priv)) {
 		drm_info(&dev_priv->drm,
 			 "Disabling ppGTT for VT-d support\n");
-		info->ppgtt_type = INTEL_PPGTT_NONE;
+		runtime->ppgtt_type = INTEL_PPGTT_NONE;
 	}
 
 	runtime->rawclk_freq = intel_read_rawclk(dev_priv);
@@ -422,8 +433,14 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
 		dev_priv->drm.driver_features &= ~(DRIVER_MODESET |
 						   DRIVER_ATOMIC);
 		memset(&info->display, 0, sizeof(info->display));
+
+		runtime->cpu_transcoder_mask = 0;
 		memset(runtime->num_sprites, 0, sizeof(runtime->num_sprites));
 		memset(runtime->num_scalers, 0, sizeof(runtime->num_scalers));
+		runtime->fbc_mask = 0;
+		runtime->has_hdcp = false;
+		runtime->has_dmc = false;
+		runtime->has_dsc = false;
 	}
 }
 
diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h
index 23bf230aa104..d638235e1d26 100644
--- a/drivers/gpu/drm/i915/intel_device_info.h
+++ b/drivers/gpu/drm/i915/intel_device_info.h
@@ -37,6 +37,7 @@
 
 struct drm_printer;
 struct drm_i915_private;
+struct intel_gt_definition;
 
 /* Keep in gen based order, and chronological order within a gen */
 enum intel_platform {
@@ -164,7 +165,6 @@ enum intel_ppgtt_type {
 	func(has_media_ratio_mode); \
 	func(has_mslice_steering); \
 	func(has_one_eu_per_fuse_bit); \
-	func(has_pooled_eu); \
 	func(has_pxp); \
 	func(has_rc6); \
 	func(has_rc6p); \
@@ -172,6 +172,7 @@ enum intel_ppgtt_type {
 	func(has_runtime_pm); \
 	func(has_snoop); \
 	func(has_coherent_ggtt); \
+	func(tuning_thread_rr_after_dep); \
 	func(unfenced_needs_alignment); \
 	func(hws_needs_physical);
 
@@ -179,14 +180,11 @@ enum intel_ppgtt_type {
 	/* Keep in alphabetical order */ \
 	func(cursor_needs_physical); \
 	func(has_cdclk_crawl); \
-	func(has_dmc); \
 	func(has_ddi); \
 	func(has_dp_mst); \
 	func(has_dsb); \
-	func(has_dsc); \
 	func(has_fpga_dbg); \
 	func(has_gmch); \
-	func(has_hdcp); \
 	func(has_hotplug); \
 	func(has_hti); \
 	func(has_ipc); \
@@ -202,23 +200,67 @@ struct ip_version {
 	u8 rel;
 };
 
-struct intel_device_info {
-	struct ip_version graphics;
-	struct ip_version media;
+struct intel_runtime_info {
+	struct {
+		struct ip_version ip;
+	} graphics;
+	struct {
+		struct ip_version ip;
+	} media;
+	struct {
+		struct ip_version ip;
+	} display;
+
+	/*
+	 * Platform mask is used for optimizing or-ed IS_PLATFORM calls into
+	 * single runtime conditionals, and also to provide groundwork for
+	 * future per platform, or per SKU build optimizations.
+	 *
+	 * Array can be extended when necessary if the corresponding
+	 * BUILD_BUG_ON is hit.
+	 */
+	u32 platform_mask[2];
+
+	u16 device_id;
 
 	intel_engine_mask_t platform_engine_mask; /* Engines supported by the HW */
 
-	enum intel_platform platform;
+	u32 rawclk_freq;
 
-	unsigned int dma_mask_size; /* available DMA address bits */
+	struct intel_step_info step;
+
+	unsigned int page_sizes; /* page sizes supported by the HW */
 
 	enum intel_ppgtt_type ppgtt_type;
 	unsigned int ppgtt_size; /* log2, e.g. 31/32/48 bits */
 
-	unsigned int page_sizes; /* page sizes supported by the HW */
-
 	u32 memory_regions; /* regions supported by the HW */
 
+	bool has_pooled_eu;
+
+	/* display */
+	struct {
+		u8 pipe_mask;
+		u8 cpu_transcoder_mask;
+
+		u8 num_sprites[I915_MAX_PIPES];
+		u8 num_scalers[I915_MAX_PIPES];
+
+		u8 fbc_mask;
+
+		bool has_hdcp;
+		bool has_dmc;
+		bool has_dsc;
+	};
+};
+
+struct intel_device_info {
+	enum intel_platform platform;
+
+	unsigned int dma_mask_size; /* available DMA address bits */
+
+	const struct intel_gt_definition *extra_gt_list;
+
 	u8 gt; /* GT number, 0 if undefined */
 
 #define DEFINE_FLAG(name) u8 name:1
@@ -226,12 +268,6 @@ struct intel_device_info {
 #undef DEFINE_FLAG
 
 	struct {
-		u8 ver;
-		u8 rel;
-
-		u8 pipe_mask;
-		u8 cpu_transcoder_mask;
-		u8 fbc_mask;
 		u8 abox_mask;
 
 		struct {
@@ -258,27 +294,11 @@ struct intel_device_info {
 			u32 gamma_lut_tests;
 		} color;
 	} display;
-};
 
-struct intel_runtime_info {
 	/*
-	 * Platform mask is used for optimizing or-ed IS_PLATFORM calls into
-	 * into single runtime conditionals, and also to provide groundwork
-	 * for future per platform, or per SKU build optimizations.
-	 *
-	 * Array can be extended when necessary if the corresponding
-	 * BUILD_BUG_ON is hit.
+	 * Initial runtime info. Do not access outside of i915_driver_create().
 	 */
-	u32 platform_mask[2];
-
-	u16 device_id;
-
-	u8 num_sprites[I915_MAX_PIPES];
-	u8 num_scalers[I915_MAX_PIPES];
-
-	u32 rawclk_freq;
-
-	struct intel_step_info step;
+	const struct intel_runtime_info __runtime;
 };
 
 struct intel_driver_caps {
@@ -291,10 +311,9 @@ const char *intel_platform_name(enum intel_platform platform);
 void intel_device_info_subplatform_init(struct drm_i915_private *dev_priv);
 void intel_device_info_runtime_init(struct drm_i915_private *dev_priv);
 
-void intel_device_info_print_static(const struct intel_device_info *info,
-				    struct drm_printer *p);
-void intel_device_info_print_runtime(const struct intel_runtime_info *info,
-				     struct drm_printer *p);
+void intel_device_info_print(const struct intel_device_info *info,
+			     const struct intel_runtime_info *runtime,
+			     struct drm_printer *p);
 
 void intel_driver_caps_print(const struct intel_driver_caps *caps,
 			     struct drm_printer *p);
diff --git a/drivers/gpu/drm/i915/intel_dram.c b/drivers/gpu/drm/i915/intel_dram.c
index 437447119770..2403ccd52c74 100644
--- a/drivers/gpu/drm/i915/intel_dram.c
+++ b/drivers/gpu/drm/i915/intel_dram.c
@@ -466,6 +466,43 @@ static int gen12_get_dram_info(struct drm_i915_private *i915)
 	return icl_pcode_read_mem_global_info(i915);
 }
 
+static int xelpdp_get_dram_info(struct drm_i915_private *i915)
+{
+	u32 val = intel_uncore_read(&i915->uncore, MTL_MEM_SS_INFO_GLOBAL);
+	struct dram_info *dram_info = &i915->dram_info;
+
+	val = REG_FIELD_GET(MTL_DDR_TYPE_MASK, val);
+	switch (val) {
+	case 0:
+		dram_info->type = INTEL_DRAM_DDR4;
+		break;
+	case 1:
+		dram_info->type = INTEL_DRAM_DDR5;
+		break;
+	case 2:
+		dram_info->type = INTEL_DRAM_LPDDR5;
+		break;
+	case 3:
+		dram_info->type = INTEL_DRAM_LPDDR4;
+		break;
+	case 4:
+		dram_info->type = INTEL_DRAM_DDR3;
+		break;
+	case 5:
+		dram_info->type = INTEL_DRAM_LPDDR3;
+		break;
+	default:
+		MISSING_CASE(val);
+		return -EINVAL;
+	}
+
+	dram_info->num_channels = REG_FIELD_GET(MTL_N_OF_POPULATED_CH_MASK, val);
+	dram_info->num_qgv_points = REG_FIELD_GET(MTL_N_OF_ENABLED_QGV_POINTS_MASK, val);
+	/* PSF GV points not supported in D14+ */
+
+	return 0;
+}
+
 void intel_dram_detect(struct drm_i915_private *i915)
 {
 	struct dram_info *dram_info = &i915->dram_info;
@@ -480,7 +517,9 @@ void intel_dram_detect(struct drm_i915_private *i915)
 	 */
 	dram_info->wm_lv_0_adjust_needed = !IS_GEN9_LP(i915);
 
-	if (GRAPHICS_VER(i915) >= 12)
+	if (DISPLAY_VER(i915) >= 14)
+		ret = xelpdp_get_dram_info(i915);
+	else if (GRAPHICS_VER(i915) >= 12)
 		ret = gen12_get_dram_info(i915);
 	else if (GRAPHICS_VER(i915) >= 11)
 		ret = gen11_get_dram_info(i915);
diff --git a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
index 5595639d0033..8279dc580a3e 100644
--- a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
+++ b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
@@ -4,6 +4,7 @@
  */
 
 #include "display/intel_audio_regs.h"
+#include "display/intel_backlight_regs.h"
 #include "display/intel_dmc_regs.h"
 #include "display/vlv_dsi_pll_regs.h"
 #include "gt/intel_gt_regs.h"
diff --git a/drivers/gpu/drm/i915/intel_mchbar_regs.h b/drivers/gpu/drm/i915/intel_mchbar_regs.h
index 2aad2f0cc8db..ffc702b79579 100644
--- a/drivers/gpu/drm/i915/intel_mchbar_regs.h
+++ b/drivers/gpu/drm/i915/intel_mchbar_regs.h
@@ -196,6 +196,9 @@
 #define   RP1_CAP_MASK				REG_GENMASK(15, 8)
 #define   RPN_CAP_MASK				REG_GENMASK(23, 16)
 
+#define GEN10_FREQ_INFO_REC			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5ef0)
+#define   RPE_MASK				REG_GENMASK(15, 8)
+
 /* snb MCH registers for priority tuning */
 #define MCH_SSKPD				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5d10)
 #define   SSKPD_NEW_WM0_MASK_HSW		REG_GENMASK64(63, 56)
diff --git a/drivers/gpu/drm/i915/intel_pch.c b/drivers/gpu/drm/i915/intel_pch.c
index 0fec25be146a..ba9843cb1b13 100644
--- a/drivers/gpu/drm/i915/intel_pch.c
+++ b/drivers/gpu/drm/i915/intel_pch.c
@@ -138,6 +138,11 @@ intel_pch_type(const struct drm_i915_private *dev_priv, unsigned short id)
 		drm_WARN_ON(&dev_priv->drm, !IS_ALDERLAKE_S(dev_priv) &&
 			    !IS_ALDERLAKE_P(dev_priv));
 		return PCH_ADP;
+	case INTEL_PCH_MTP_DEVICE_ID_TYPE:
+	case INTEL_PCH_MTP2_DEVICE_ID_TYPE:
+		drm_dbg_kms(&dev_priv->drm, "Found Meteor Lake PCH\n");
+		drm_WARN_ON(&dev_priv->drm, !IS_METEORLAKE(dev_priv));
+		return PCH_MTP;
 	default:
 		return PCH_NONE;
 	}
@@ -166,7 +171,9 @@ intel_virt_detect_pch(const struct drm_i915_private *dev_priv,
 	 * make an educated guess as to which PCH is really there.
 	 */
 
-	if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv))
+	if (IS_METEORLAKE(dev_priv))
+		id = INTEL_PCH_MTP_DEVICE_ID_TYPE;
+	else if (IS_ALDERLAKE_S(dev_priv) || IS_ALDERLAKE_P(dev_priv))
 		id = INTEL_PCH_ADP_DEVICE_ID_TYPE;
 	else if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv))
 		id = INTEL_PCH_TGP_DEVICE_ID_TYPE;
diff --git a/drivers/gpu/drm/i915/intel_pch.h b/drivers/gpu/drm/i915/intel_pch.h
index 7c8ce9781d1a..32aff5a70d04 100644
--- a/drivers/gpu/drm/i915/intel_pch.h
+++ b/drivers/gpu/drm/i915/intel_pch.h
@@ -25,6 +25,7 @@ enum intel_pch {
 	PCH_ICP,	/* Ice Lake/Jasper Lake PCH */
 	PCH_TGP,	/* Tiger Lake/Mule Creek Canyon PCH */
 	PCH_ADP,	/* Alder Lake PCH */
+	PCH_MTP,	/* Meteor Lake PCH */
 
 	/* Fake PCHs, functionality handled on the same PCI dev */
 	PCH_DG1 = 1024,
@@ -57,12 +58,15 @@ enum intel_pch {
 #define INTEL_PCH_ADP2_DEVICE_ID_TYPE		0x5180
 #define INTEL_PCH_ADP3_DEVICE_ID_TYPE		0x7A00
 #define INTEL_PCH_ADP4_DEVICE_ID_TYPE		0x5480
+#define INTEL_PCH_MTP_DEVICE_ID_TYPE		0x7E00
+#define INTEL_PCH_MTP2_DEVICE_ID_TYPE		0xAE00
 #define INTEL_PCH_P2X_DEVICE_ID_TYPE		0x7100
 #define INTEL_PCH_P3X_DEVICE_ID_TYPE		0x7000
 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE		0x2900 /* qemu q35 has 2918 */
 
 #define INTEL_PCH_TYPE(dev_priv)		((dev_priv)->pch_type)
 #define INTEL_PCH_ID(dev_priv)			((dev_priv)->pch_id)
+#define HAS_PCH_MTP(dev_priv)			(INTEL_PCH_TYPE(dev_priv) == PCH_MTP)
 #define HAS_PCH_DG2(dev_priv)			(INTEL_PCH_TYPE(dev_priv) == PCH_DG2)
 #define HAS_PCH_ADP(dev_priv)			(INTEL_PCH_TYPE(dev_priv) == PCH_ADP)
 #define HAS_PCH_DG1(dev_priv)			(INTEL_PCH_TYPE(dev_priv) == PCH_DG1)
diff --git a/drivers/gpu/drm/i915/intel_pci_config.h b/drivers/gpu/drm/i915/intel_pci_config.h
index 12cd9d4f23de..4977a524ce6f 100644
--- a/drivers/gpu/drm/i915/intel_pci_config.h
+++ b/drivers/gpu/drm/i915/intel_pci_config.h
@@ -6,6 +6,13 @@
 #ifndef __INTEL_PCI_CONFIG_H__
 #define __INTEL_PCI_CONFIG_H__
 
+/* PCI BARs */
+#define GTTMMADR_BAR				0
+#define GEN2_GTTMMADR_BAR			1
+#define GFXMEM_BAR				2
+#define GTT_APERTURE_BAR			GFXMEM_BAR
+#define GEN12_LMEM_BAR				GFXMEM_BAR
+
 /* BSM in include/drm/i915_drm.h */
 
 #define MCHBAR_I915				0x44
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 9fe4b583cc28..8f86f56e7ca4 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -25,61 +25,22 @@
  *
  */
 
-#include <linux/module.h>
-#include <linux/string_helpers.h>
-#include <linux/pm_runtime.h>
-
-#include <drm/drm_atomic_helper.h>
-#include <drm/drm_blend.h>
-#include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
-
-#include "display/intel_atomic.h"
-#include "display/intel_atomic_plane.h"
-#include "display/intel_bw.h"
 #include "display/intel_de.h"
 #include "display/intel_display_trace.h"
-#include "display/intel_display_types.h"
-#include "display/intel_fb.h"
-#include "display/intel_fbc.h"
-#include "display/intel_sprite.h"
-#include "display/skl_universal_plane.h"
+#include "display/skl_watermark.h"
 
 #include "gt/intel_engine_regs.h"
 #include "gt/intel_gt_regs.h"
-#include "gt/intel_llc.h"
 
 #include "i915_drv.h"
-#include "i915_fixed.h"
-#include "i915_irq.h"
 #include "intel_mchbar_regs.h"
-#include "intel_pcode.h"
 #include "intel_pm.h"
 #include "vlv_sideband.h"
-#include "../../../platform/x86/intel_ips.h"
-
-static void skl_sagv_disable(struct drm_i915_private *dev_priv);
 
 struct drm_i915_clock_gating_funcs {
 	void (*init_clock_gating)(struct drm_i915_private *i915);
 };
 
-/* Stores plane specific WM parameters */
-struct skl_wm_params {
-	bool x_tiled, y_tiled;
-	bool rc_surface;
-	bool is_planar;
-	u32 width;
-	u8 cpp;
-	u32 plane_pixel_rate;
-	u32 y_min_scanlines;
-	u32 plane_bytes_per_line;
-	uint_fixed_16_16_t plane_blocks_per_line;
-	uint_fixed_16_16_t y_tile_minimum;
-	u32 linetime_us;
-	u32 dbuf_block_size;
-};
-
 /* used in computing the new watermarks state */
 struct intel_wm_config {
 	unsigned int num_pipes_active;
@@ -469,13 +430,13 @@ bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 {
 	bool ret;
 
-	mutex_lock(&dev_priv->wm.wm_mutex);
+	mutex_lock(&dev_priv->display.wm.wm_mutex);
 	ret = _intel_set_memory_cxsr(dev_priv, enable);
 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-		dev_priv->wm.vlv.cxsr = enable;
+		dev_priv->display.wm.vlv.cxsr = enable;
 	else if (IS_G4X(dev_priv))
-		dev_priv->wm.g4x.cxsr = enable;
-	mutex_unlock(&dev_priv->wm.wm_mutex);
+		dev_priv->display.wm.g4x.cxsr = enable;
+	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 
 	return ret;
 }
@@ -835,11 +796,11 @@ static bool is_enabling(int old, int new, int threshold)
 
 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 {
-	return dev_priv->wm.max_level + 1;
+	return dev_priv->display.wm.max_level + 1;
 }
 
-static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
-				   const struct intel_plane_state *plane_state)
+bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
+			    const struct intel_plane_state *plane_state)
 {
 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 
@@ -1094,11 +1055,11 @@ static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
 {
 	/* all latencies in usec */
-	dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
-	dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
-	dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
+	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
+	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
+	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
 
-	dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
+	dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL;
 }
 
 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
@@ -1151,7 +1112,7 @@ static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	const struct drm_display_mode *pipe_mode =
 		&crtc_state->hw.pipe_mode;
-	unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
+	unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
 	unsigned int pixel_rate, htotal, cpp, width, wm;
 
 	if (latency == 0)
@@ -1325,7 +1286,7 @@ static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
 {
 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
 
-	if (level > dev_priv->wm.max_level)
+	if (level > dev_priv->display.wm.max_level)
 		return false;
 
 	return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
@@ -1584,7 +1545,7 @@ static void g4x_merge_wm(struct drm_i915_private *dev_priv,
 
 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
 {
-	struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
+	struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
 	struct g4x_wm_values new_wm = {};
 
 	g4x_merge_wm(dev_priv, &new_wm);
@@ -1610,10 +1571,10 @@ static void g4x_initial_watermarks(struct intel_atomic_state *state,
 	const struct intel_crtc_state *crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
 
-	mutex_lock(&dev_priv->wm.wm_mutex);
+	mutex_lock(&dev_priv->display.wm.wm_mutex);
 	crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
 	g4x_program_watermarks(dev_priv);
-	mutex_unlock(&dev_priv->wm.wm_mutex);
+	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 }
 
 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
@@ -1626,10 +1587,10 @@ static void g4x_optimize_watermarks(struct intel_atomic_state *state,
 	if (!crtc_state->wm.need_postvbl_update)
 		return;
 
-	mutex_lock(&dev_priv->wm.wm_mutex);
+	mutex_lock(&dev_priv->display.wm.wm_mutex);
 	crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
 	g4x_program_watermarks(dev_priv);
-	mutex_unlock(&dev_priv->wm.wm_mutex);
+	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 }
 
 /* latency must be in 0.1us units. */
@@ -1651,15 +1612,15 @@ static unsigned int vlv_wm_method2(unsigned int pixel_rate,
 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
 {
 	/* all latencies in usec */
-	dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
+	dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
 
-	dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
+	dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2;
 
 	if (IS_CHERRYVIEW(dev_priv)) {
-		dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
-		dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
+		dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
+		dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
 
-		dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
+		dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
 	}
 }
 
@@ -1673,7 +1634,7 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
 		&crtc_state->hw.pipe_mode;
 	unsigned int pixel_rate, htotal, cpp, width, wm;
 
-	if (dev_priv->wm.pri_latency[level] == 0)
+	if (dev_priv->display.wm.pri_latency[level] == 0)
 		return USHRT_MAX;
 
 	if (!intel_wm_plane_visible(crtc_state, plane_state))
@@ -1694,7 +1655,7 @@ static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
 		wm = 63;
 	} else {
 		wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
-				    dev_priv->wm.pri_latency[level] * 10);
+				    dev_priv->display.wm.pri_latency[level] * 10);
 	}
 
 	return min_t(unsigned int, wm, USHRT_MAX);
@@ -2159,7 +2120,7 @@ static void vlv_merge_wm(struct drm_i915_private *dev_priv,
 	struct intel_crtc *crtc;
 	int num_active_pipes = 0;
 
-	wm->level = dev_priv->wm.max_level;
+	wm->level = dev_priv->display.wm.max_level;
 	wm->cxsr = true;
 
 	for_each_intel_crtc(&dev_priv->drm, crtc) {
@@ -2198,7 +2159,7 @@ static void vlv_merge_wm(struct drm_i915_private *dev_priv,
 
 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
 {
-	struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
+	struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
 	struct vlv_wm_values new_wm = {};
 
 	vlv_merge_wm(dev_priv, &new_wm);
@@ -2236,10 +2197,10 @@ static void vlv_initial_watermarks(struct intel_atomic_state *state,
 	const struct intel_crtc_state *crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
 
-	mutex_lock(&dev_priv->wm.wm_mutex);
+	mutex_lock(&dev_priv->display.wm.wm_mutex);
 	crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
 	vlv_program_watermarks(dev_priv);
-	mutex_unlock(&dev_priv->wm.wm_mutex);
+	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 }
 
 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
@@ -2252,10 +2213,10 @@ static void vlv_optimize_watermarks(struct intel_atomic_state *state,
 	if (!crtc_state->wm.need_postvbl_update)
 		return;
 
-	mutex_lock(&dev_priv->wm.wm_mutex);
+	mutex_lock(&dev_priv->display.wm.wm_mutex);
 	crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
 	vlv_program_watermarks(dev_priv);
-	mutex_unlock(&dev_priv->wm.wm_mutex);
+	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 }
 
 static void i965_update_wm(struct drm_i915_private *dev_priv)
@@ -2836,9 +2797,9 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
 				 const struct intel_plane_state *curstate,
 				 struct intel_wm_level *result)
 {
-	u16 pri_latency = dev_priv->wm.pri_latency[level];
-	u16 spr_latency = dev_priv->wm.spr_latency[level];
-	u16 cur_latency = dev_priv->wm.cur_latency[level];
+	u16 pri_latency = dev_priv->display.wm.pri_latency[level];
+	u16 spr_latency = dev_priv->display.wm.spr_latency[level];
+	u16 cur_latency = dev_priv->display.wm.cur_latency[level];
 
 	/* WM1+ latency values stored in 0.5us units */
 	if (level > 0) {
@@ -2862,119 +2823,43 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
 	result->enable = true;
 }
 
-static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
-				  u16 wm[])
+static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
 {
-	struct intel_uncore *uncore = &dev_priv->uncore;
-
-	if (DISPLAY_VER(dev_priv) >= 9) {
-		u32 val;
-		int ret, i;
-		int level, max_level = ilk_wm_max_level(dev_priv);
-		int mult = IS_DG2(dev_priv) ? 2 : 1;
-
-		/* read the first set of memory latencies[0:3] */
-		val = 0; /* data0 to be programmed to 0 for first set */
-		ret = snb_pcode_read(&dev_priv->uncore, GEN9_PCODE_READ_MEM_LATENCY,
-				     &val, NULL);
-
-		if (ret) {
-			drm_err(&dev_priv->drm,
-				"SKL Mailbox read error = %d\n", ret);
-			return;
-		}
+	u64 sskpd;
 
-		wm[0] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
-		wm[1] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
-				GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
-		wm[2] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
-				GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
-		wm[3] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
-				GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
-
-		/* read the second set of memory latencies[4:7] */
-		val = 1; /* data0 to be programmed to 1 for second set */
-		ret = snb_pcode_read(&dev_priv->uncore, GEN9_PCODE_READ_MEM_LATENCY,
-				     &val, NULL);
-		if (ret) {
-			drm_err(&dev_priv->drm,
-				"SKL Mailbox read error = %d\n", ret);
-			return;
-		}
+	sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
 
-		wm[4] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
-		wm[5] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
-				GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
-		wm[6] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
-				GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
-		wm[7] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
-				GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
+	wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
+	if (wm[0] == 0)
+		wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
+	wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
+	wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
+	wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
+	wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
+}
 
-		/*
-		 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
-		 * need to be disabled. We make sure to sanitize the values out
-		 * of the punit to satisfy this requirement.
-		 */
-		for (level = 1; level <= max_level; level++) {
-			if (wm[level] == 0) {
-				for (i = level + 1; i <= max_level; i++)
-					wm[i] = 0;
+static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
+{
+	u32 sskpd;
 
-				max_level = level - 1;
+	sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
 
-				break;
-			}
-		}
+	wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
+	wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
+	wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
+	wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
+}
 
-		/*
-		 * WaWmMemoryReadLatency
-		 *
-		 * punit doesn't take into account the read latency so we need
-		 * to add proper adjustement to each valid level we retrieve
-		 * from the punit when level 0 response data is 0us.
-		 */
-		if (wm[0] == 0) {
-			u8 adjust = DISPLAY_VER(dev_priv) >= 12 ? 3 : 2;
+static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
+{
+	u32 mltr;
 
-			for (level = 0; level <= max_level; level++)
-				wm[level] += adjust;
-		}
+	mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
 
-		/*
-		 * WA Level-0 adjustment for 16GB DIMMs: SKL+
-		 * If we could not get dimm info enable this WA to prevent from
-		 * any underrun. If not able to get Dimm info assume 16GB dimm
-		 * to avoid any underrun.
-		 */
-		if (dev_priv->dram_info.wm_lv_0_adjust_needed)
-			wm[0] += 1;
-	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
-		u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
-
-		wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
-		if (wm[0] == 0)
-			wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
-		wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
-		wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
-		wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
-		wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
-	} else if (DISPLAY_VER(dev_priv) >= 6) {
-		u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
-
-		wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
-		wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
-		wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
-		wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
-	} else if (DISPLAY_VER(dev_priv) >= 5) {
-		u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
-
-		/* ILK primary LP0 latency is 700 ns */
-		wm[0] = 7;
-		wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
-		wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
-	} else {
-		MISSING_CASE(INTEL_DEVID(dev_priv));
-	}
+	/* ILK primary LP0 latency is 700 ns */
+	wm[0] = 7;
+	wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
+	wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
 }
 
 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
@@ -3008,9 +2893,8 @@ int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
 		return 2;
 }
 
-static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
-				   const char *name,
-				   const u16 wm[])
+void intel_print_wm_latency(struct drm_i915_private *dev_priv,
+			    const char *name, const u16 wm[])
 {
 	int level, max_level = ilk_wm_max_level(dev_priv);
 
@@ -3062,18 +2946,18 @@ static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
 	 * The BIOS provided WM memory latency values are often
 	 * inadequate for high resolution displays. Adjust them.
 	 */
-	changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12);
-	changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12);
-	changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
+	changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
+	changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
+	changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
 
 	if (!changed)
 		return;
 
 	drm_dbg_kms(&dev_priv->drm,
 		    "WM latency values increased to avoid potential underruns\n");
-	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
-	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
-	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
+	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
+	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
+	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
 }
 
 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
@@ -3089,37 +2973,42 @@ static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
 	 * interrupts only. To play it safe we disable LP3
 	 * watermarks entirely.
 	 */
-	if (dev_priv->wm.pri_latency[3] == 0 &&
-	    dev_priv->wm.spr_latency[3] == 0 &&
-	    dev_priv->wm.cur_latency[3] == 0)
+	if (dev_priv->display.wm.pri_latency[3] == 0 &&
+	    dev_priv->display.wm.spr_latency[3] == 0 &&
+	    dev_priv->display.wm.cur_latency[3] == 0)
 		return;
 
-	dev_priv->wm.pri_latency[3] = 0;
-	dev_priv->wm.spr_latency[3] = 0;
-	dev_priv->wm.cur_latency[3] = 0;
+	dev_priv->display.wm.pri_latency[3] = 0;
+	dev_priv->display.wm.spr_latency[3] = 0;
+	dev_priv->display.wm.cur_latency[3] = 0;
 
 	drm_dbg_kms(&dev_priv->drm,
 		    "LP3 watermarks disabled due to potential for lost interrupts\n");
-	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
-	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
-	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
+	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
+	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
+	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
 }
 
 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
 {
-	intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
+	if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
+		hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
+	else if (DISPLAY_VER(dev_priv) >= 6)
+		snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
+	else
+		ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
 
-	memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
-	       sizeof(dev_priv->wm.pri_latency));
-	memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
-	       sizeof(dev_priv->wm.pri_latency));
+	memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
+	       sizeof(dev_priv->display.wm.pri_latency));
+	memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
+	       sizeof(dev_priv->display.wm.pri_latency));
 
-	intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
-	intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
+	intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
+	intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
 
-	intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
-	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
-	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
+	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
+	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
+	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
 
 	if (DISPLAY_VER(dev_priv) == 6) {
 		snb_wm_latency_quirk(dev_priv);
@@ -3127,12 +3016,6 @@ static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
 	}
 }
 
-static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
-{
-	intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
-	intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
-}
-
 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
 				 struct intel_pipe_wm *pipe_wm)
 {
@@ -3387,7 +3270,7 @@ static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
 	if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
 		return 2 * level;
 	else
-		return dev_priv->wm.pri_latency[level];
+		return dev_priv->display.wm.pri_latency[level];
 }
 
 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
@@ -3539,7 +3422,7 @@ static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
 			       unsigned int dirty)
 {
-	struct ilk_wm_values *previous = &dev_priv->wm.hw;
+	struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
 	bool changed = false;
 
 	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
@@ -3573,7 +3456,7 @@ static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
 				struct ilk_wm_values *results)
 {
-	struct ilk_wm_values *previous = &dev_priv->wm.hw;
+	struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
 	unsigned int dirty;
 	u32 val;
 
@@ -3635,7 +3518,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
 	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
 		intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
 
-	dev_priv->wm.hw = *results;
+	dev_priv->display.wm.hw = *results;
 }
 
 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
@@ -3643,2765 +3526,6 @@ bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
 	return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
 }
 
-u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
-{
-	u8 enabled_slices = 0;
-	enum dbuf_slice slice;
-
-	for_each_dbuf_slice(dev_priv, slice) {
-		if (intel_uncore_read(&dev_priv->uncore,
-				      DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
-			enabled_slices |= BIT(slice);
-	}
-
-	return enabled_slices;
-}
-
-/*
- * FIXME: We still don't have the proper code detect if we need to apply the WA,
- * so assume we'll always need it in order to avoid underruns.
- */
-static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
-{
-	return DISPLAY_VER(dev_priv) == 9;
-}
-
-static bool
-intel_has_sagv(struct drm_i915_private *dev_priv)
-{
-	return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) &&
-		dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
-}
-
-static u32
-intel_sagv_block_time(struct drm_i915_private *dev_priv)
-{
-	if (DISPLAY_VER(dev_priv) >= 12) {
-		u32 val = 0;
-		int ret;
-
-		ret = snb_pcode_read(&dev_priv->uncore,
-				     GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
-				     &val, NULL);
-		if (ret) {
-			drm_dbg_kms(&dev_priv->drm, "Couldn't read SAGV block time!\n");
-			return 0;
-		}
-
-		return val;
-	} else if (DISPLAY_VER(dev_priv) == 11) {
-		return 10;
-	} else if (DISPLAY_VER(dev_priv) == 9 && !IS_LP(dev_priv)) {
-		return 30;
-	} else {
-		return 0;
-	}
-}
-
-static void intel_sagv_init(struct drm_i915_private *i915)
-{
-	if (!intel_has_sagv(i915))
-		i915->sagv_status = I915_SAGV_NOT_CONTROLLED;
-
-	/*
-	 * Probe to see if we have working SAGV control.
-	 * For icl+ this was already determined by intel_bw_init_hw().
-	 */
-	if (DISPLAY_VER(i915) < 11)
-		skl_sagv_disable(i915);
-
-	drm_WARN_ON(&i915->drm, i915->sagv_status == I915_SAGV_UNKNOWN);
-
-	i915->sagv_block_time_us = intel_sagv_block_time(i915);
-
-	drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
-		    str_yes_no(intel_has_sagv(i915)), i915->sagv_block_time_us);
-
-	/* avoid overflow when adding with wm0 latency/etc. */
-	if (drm_WARN(&i915->drm, i915->sagv_block_time_us > U16_MAX,
-		     "Excessive SAGV block time %u, ignoring\n",
-		     i915->sagv_block_time_us))
-		i915->sagv_block_time_us = 0;
-
-	if (!intel_has_sagv(i915))
-		i915->sagv_block_time_us = 0;
-}
-
-/*
- * SAGV dynamically adjusts the system agent voltage and clock frequencies
- * depending on power and performance requirements. The display engine access
- * to system memory is blocked during the adjustment time. Because of the
- * blocking time, having this enabled can cause full system hangs and/or pipe
- * underruns if we don't meet all of the following requirements:
- *
- *  - <= 1 pipe enabled
- *  - All planes can enable watermarks for latencies >= SAGV engine block time
- *  - We're not using an interlaced display configuration
- */
-static void skl_sagv_enable(struct drm_i915_private *dev_priv)
-{
-	int ret;
-
-	if (!intel_has_sagv(dev_priv))
-		return;
-
-	if (dev_priv->sagv_status == I915_SAGV_ENABLED)
-		return;
-
-	drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
-	ret = snb_pcode_write(&dev_priv->uncore, GEN9_PCODE_SAGV_CONTROL,
-			      GEN9_SAGV_ENABLE);
-
-	/* We don't need to wait for SAGV when enabling */
-
-	/*
-	 * Some skl systems, pre-release machines in particular,
-	 * don't actually have SAGV.
-	 */
-	if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
-		drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
-		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
-		return;
-	} else if (ret < 0) {
-		drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
-		return;
-	}
-
-	dev_priv->sagv_status = I915_SAGV_ENABLED;
-}
-
-static void skl_sagv_disable(struct drm_i915_private *dev_priv)
-{
-	int ret;
-
-	if (!intel_has_sagv(dev_priv))
-		return;
-
-	if (dev_priv->sagv_status == I915_SAGV_DISABLED)
-		return;
-
-	drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
-	/* bspec says to keep retrying for at least 1 ms */
-	ret = skl_pcode_request(&dev_priv->uncore, GEN9_PCODE_SAGV_CONTROL,
-				GEN9_SAGV_DISABLE,
-				GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
-				1);
-	/*
-	 * Some skl systems, pre-release machines in particular,
-	 * don't actually have SAGV.
-	 */
-	if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
-		drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
-		dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
-		return;
-	} else if (ret < 0) {
-		drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
-		return;
-	}
-
-	dev_priv->sagv_status = I915_SAGV_DISABLED;
-}
-
-static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
-	const struct intel_bw_state *new_bw_state =
-		intel_atomic_get_new_bw_state(state);
-
-	if (!new_bw_state)
-		return;
-
-	if (!intel_can_enable_sagv(i915, new_bw_state))
-		skl_sagv_disable(i915);
-}
-
-static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
-	const struct intel_bw_state *new_bw_state =
-		intel_atomic_get_new_bw_state(state);
-
-	if (!new_bw_state)
-		return;
-
-	if (intel_can_enable_sagv(i915, new_bw_state))
-		skl_sagv_enable(i915);
-}
-
-static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	const struct intel_bw_state *old_bw_state =
-		intel_atomic_get_old_bw_state(state);
-	const struct intel_bw_state *new_bw_state =
-		intel_atomic_get_new_bw_state(state);
-	u16 old_mask, new_mask;
-
-	if (!new_bw_state)
-		return;
-
-	old_mask = old_bw_state->qgv_points_mask;
-	new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
-
-	if (old_mask == new_mask)
-		return;
-
-	WARN_ON(!new_bw_state->base.changed);
-
-	drm_dbg_kms(&dev_priv->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
-		    old_mask, new_mask);
-
-	/*
-	 * Restrict required qgv points before updating the configuration.
-	 * According to BSpec we can't mask and unmask qgv points at the same
-	 * time. Also masking should be done before updating the configuration
-	 * and unmasking afterwards.
-	 */
-	icl_pcode_restrict_qgv_points(dev_priv, new_mask);
-}
-
-static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	const struct intel_bw_state *old_bw_state =
-		intel_atomic_get_old_bw_state(state);
-	const struct intel_bw_state *new_bw_state =
-		intel_atomic_get_new_bw_state(state);
-	u16 old_mask, new_mask;
-
-	if (!new_bw_state)
-		return;
-
-	old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
-	new_mask = new_bw_state->qgv_points_mask;
-
-	if (old_mask == new_mask)
-		return;
-
-	WARN_ON(!new_bw_state->base.changed);
-
-	drm_dbg_kms(&dev_priv->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
-		    old_mask, new_mask);
-
-	/*
-	 * Allow required qgv points after updating the configuration.
-	 * According to BSpec we can't mask and unmask qgv points at the same
-	 * time. Also masking should be done before updating the configuration
-	 * and unmasking afterwards.
-	 */
-	icl_pcode_restrict_qgv_points(dev_priv, new_mask);
-}
-
-void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
-
-	/*
-	 * Just return if we can't control SAGV or don't have it.
-	 * This is different from situation when we have SAGV but just can't
-	 * afford it due to DBuf limitation - in case if SAGV is completely
-	 * disabled in a BIOS, we are not even allowed to send a PCode request,
-	 * as it will throw an error. So have to check it here.
-	 */
-	if (!intel_has_sagv(i915))
-		return;
-
-	if (DISPLAY_VER(i915) >= 11)
-		icl_sagv_pre_plane_update(state);
-	else
-		skl_sagv_pre_plane_update(state);
-}
-
-void intel_sagv_post_plane_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
-
-	/*
-	 * Just return if we can't control SAGV or don't have it.
-	 * This is different from situation when we have SAGV but just can't
-	 * afford it due to DBuf limitation - in case if SAGV is completely
-	 * disabled in a BIOS, we are not even allowed to send a PCode request,
-	 * as it will throw an error. So have to check it here.
-	 */
-	if (!intel_has_sagv(i915))
-		return;
-
-	if (DISPLAY_VER(i915) >= 11)
-		icl_sagv_post_plane_update(state);
-	else
-		skl_sagv_post_plane_update(state);
-}
-
-static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
-{
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum plane_id plane_id;
-	int max_level = INT_MAX;
-
-	if (!intel_has_sagv(dev_priv))
-		return false;
-
-	if (!crtc_state->hw.active)
-		return true;
-
-	if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
-		return false;
-
-	for_each_plane_id_on_crtc(crtc, plane_id) {
-		const struct skl_plane_wm *wm =
-			&crtc_state->wm.skl.optimal.planes[plane_id];
-		int level;
-
-		/* Skip this plane if it's not enabled */
-		if (!wm->wm[0].enable)
-			continue;
-
-		/* Find the highest enabled wm level for this plane */
-		for (level = ilk_wm_max_level(dev_priv);
-		     !wm->wm[level].enable; --level)
-		     { }
-
-		/* Highest common enabled wm level for all planes */
-		max_level = min(level, max_level);
-	}
-
-	/* No enabled planes? */
-	if (max_level == INT_MAX)
-		return true;
-
-	for_each_plane_id_on_crtc(crtc, plane_id) {
-		const struct skl_plane_wm *wm =
-			&crtc_state->wm.skl.optimal.planes[plane_id];
-
-		/*
-		 * All enabled planes must have enabled a common wm level that
-		 * can tolerate memory latencies higher than sagv_block_time_us
-		 */
-		if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
-			return false;
-	}
-
-	return true;
-}
-
-static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
-{
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	enum plane_id plane_id;
-
-	if (!crtc_state->hw.active)
-		return true;
-
-	for_each_plane_id_on_crtc(crtc, plane_id) {
-		const struct skl_plane_wm *wm =
-			&crtc_state->wm.skl.optimal.planes[plane_id];
-
-		if (wm->wm[0].enable && !wm->sagv.wm0.enable)
-			return false;
-	}
-
-	return true;
-}
-
-static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
-{
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-
-	if (DISPLAY_VER(dev_priv) >= 12)
-		return tgl_crtc_can_enable_sagv(crtc_state);
-	else
-		return skl_crtc_can_enable_sagv(crtc_state);
-}
-
-bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
-			   const struct intel_bw_state *bw_state)
-{
-	if (DISPLAY_VER(dev_priv) < 11 &&
-	    bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
-		return false;
-
-	return bw_state->pipe_sagv_reject == 0;
-}
-
-static int intel_compute_sagv_mask(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	int ret;
-	struct intel_crtc *crtc;
-	struct intel_crtc_state *new_crtc_state;
-	struct intel_bw_state *new_bw_state = NULL;
-	const struct intel_bw_state *old_bw_state = NULL;
-	int i;
-
-	for_each_new_intel_crtc_in_state(state, crtc,
-					 new_crtc_state, i) {
-		new_bw_state = intel_atomic_get_bw_state(state);
-		if (IS_ERR(new_bw_state))
-			return PTR_ERR(new_bw_state);
-
-		old_bw_state = intel_atomic_get_old_bw_state(state);
-
-		if (intel_crtc_can_enable_sagv(new_crtc_state))
-			new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
-		else
-			new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
-	}
-
-	if (!new_bw_state)
-		return 0;
-
-	new_bw_state->active_pipes =
-		intel_calc_active_pipes(state, old_bw_state->active_pipes);
-
-	if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
-		ret = intel_atomic_lock_global_state(&new_bw_state->base);
-		if (ret)
-			return ret;
-	}
-
-	if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
-	    intel_can_enable_sagv(dev_priv, old_bw_state)) {
-		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
-		if (ret)
-			return ret;
-	} else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
-		ret = intel_atomic_lock_global_state(&new_bw_state->base);
-		if (ret)
-			return ret;
-	}
-
-	for_each_new_intel_crtc_in_state(state, crtc,
-					 new_crtc_state, i) {
-		struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
-
-		/*
-		 * We store use_sagv_wm in the crtc state rather than relying on
-		 * that bw state since we have no convenient way to get at the
-		 * latter from the plane commit hooks (especially in the legacy
-		 * cursor case)
-		 */
-		pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) &&
-			DISPLAY_VER(dev_priv) >= 12 &&
-			intel_can_enable_sagv(dev_priv, new_bw_state);
-	}
-
-	return 0;
-}
-
-static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
-			      u16 start, u16 end)
-{
-	entry->start = start;
-	entry->end = end;
-
-	return end;
-}
-
-static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
-{
-	return INTEL_INFO(dev_priv)->display.dbuf.size /
-		hweight8(INTEL_INFO(dev_priv)->display.dbuf.slice_mask);
-}
-
-static void
-skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
-			 struct skl_ddb_entry *ddb)
-{
-	int slice_size = intel_dbuf_slice_size(dev_priv);
-
-	if (!slice_mask) {
-		ddb->start = 0;
-		ddb->end = 0;
-		return;
-	}
-
-	ddb->start = (ffs(slice_mask) - 1) * slice_size;
-	ddb->end = fls(slice_mask) * slice_size;
-
-	WARN_ON(ddb->start >= ddb->end);
-	WARN_ON(ddb->end > INTEL_INFO(dev_priv)->display.dbuf.size);
-}
-
-static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
-{
-	struct skl_ddb_entry ddb;
-
-	if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
-		slice_mask = BIT(DBUF_S1);
-	else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
-		slice_mask = BIT(DBUF_S3);
-
-	skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
-
-	return ddb.start;
-}
-
-u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
-			    const struct skl_ddb_entry *entry)
-{
-	int slice_size = intel_dbuf_slice_size(dev_priv);
-	enum dbuf_slice start_slice, end_slice;
-	u8 slice_mask = 0;
-
-	if (!skl_ddb_entry_size(entry))
-		return 0;
-
-	start_slice = entry->start / slice_size;
-	end_slice = (entry->end - 1) / slice_size;
-
-	/*
-	 * Per plane DDB entry can in a really worst case be on multiple slices
-	 * but single entry is anyway contigious.
-	 */
-	while (start_slice <= end_slice) {
-		slice_mask |= BIT(start_slice);
-		start_slice++;
-	}
-
-	return slice_mask;
-}
-
-static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
-{
-	const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
-	int hdisplay, vdisplay;
-
-	if (!crtc_state->hw.active)
-		return 0;
-
-	/*
-	 * Watermark/ddb requirement highly depends upon width of the
-	 * framebuffer, So instead of allocating DDB equally among pipes
-	 * distribute DDB based on resolution/width of the display.
-	 */
-	drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
-
-	return hdisplay;
-}
-
-static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
-				    enum pipe for_pipe,
-				    unsigned int *weight_start,
-				    unsigned int *weight_end,
-				    unsigned int *weight_total)
-{
-	struct drm_i915_private *dev_priv =
-		to_i915(dbuf_state->base.state->base.dev);
-	enum pipe pipe;
-
-	*weight_start = 0;
-	*weight_end = 0;
-	*weight_total = 0;
-
-	for_each_pipe(dev_priv, pipe) {
-		int weight = dbuf_state->weight[pipe];
-
-		/*
-		 * Do not account pipes using other slice sets
-		 * luckily as of current BSpec slice sets do not partially
-		 * intersect(pipes share either same one slice or same slice set
-		 * i.e no partial intersection), so it is enough to check for
-		 * equality for now.
-		 */
-		if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
-			continue;
-
-		*weight_total += weight;
-		if (pipe < for_pipe) {
-			*weight_start += weight;
-			*weight_end += weight;
-		} else if (pipe == for_pipe) {
-			*weight_end += weight;
-		}
-	}
-}
-
-static int
-skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	unsigned int weight_total, weight_start, weight_end;
-	const struct intel_dbuf_state *old_dbuf_state =
-		intel_atomic_get_old_dbuf_state(state);
-	struct intel_dbuf_state *new_dbuf_state =
-		intel_atomic_get_new_dbuf_state(state);
-	struct intel_crtc_state *crtc_state;
-	struct skl_ddb_entry ddb_slices;
-	enum pipe pipe = crtc->pipe;
-	unsigned int mbus_offset = 0;
-	u32 ddb_range_size;
-	u32 dbuf_slice_mask;
-	u32 start, end;
-	int ret;
-
-	if (new_dbuf_state->weight[pipe] == 0) {
-		skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
-		goto out;
-	}
-
-	dbuf_slice_mask = new_dbuf_state->slices[pipe];
-
-	skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
-	mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask);
-	ddb_range_size = skl_ddb_entry_size(&ddb_slices);
-
-	intel_crtc_dbuf_weights(new_dbuf_state, pipe,
-				&weight_start, &weight_end, &weight_total);
-
-	start = ddb_range_size * weight_start / weight_total;
-	end = ddb_range_size * weight_end / weight_total;
-
-	skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
-			   ddb_slices.start - mbus_offset + start,
-			   ddb_slices.start - mbus_offset + end);
-
-out:
-	if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
-	    skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
-				&new_dbuf_state->ddb[pipe]))
-		return 0;
-
-	ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
-	if (ret)
-		return ret;
-
-	crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
-	if (IS_ERR(crtc_state))
-		return PTR_ERR(crtc_state);
-
-	/*
-	 * Used for checking overlaps, so we need absolute
-	 * offsets instead of MBUS relative offsets.
-	 */
-	crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
-	crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
-
-	drm_dbg_kms(&dev_priv->drm,
-		    "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
-		    crtc->base.base.id, crtc->base.name,
-		    old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
-		    old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
-		    new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
-		    old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
-
-	return 0;
-}
-
-static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
-				 int width, const struct drm_format_info *format,
-				 u64 modifier, unsigned int rotation,
-				 u32 plane_pixel_rate, struct skl_wm_params *wp,
-				 int color_plane);
-
-static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
-				 struct intel_plane *plane,
-				 int level,
-				 unsigned int latency,
-				 const struct skl_wm_params *wp,
-				 const struct skl_wm_level *result_prev,
-				 struct skl_wm_level *result /* out */);
-
-static unsigned int
-skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
-		      int num_active)
-{
-	struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
-	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
-	int level, max_level = ilk_wm_max_level(dev_priv);
-	struct skl_wm_level wm = {};
-	int ret, min_ddb_alloc = 0;
-	struct skl_wm_params wp;
-
-	ret = skl_compute_wm_params(crtc_state, 256,
-				    drm_format_info(DRM_FORMAT_ARGB8888),
-				    DRM_FORMAT_MOD_LINEAR,
-				    DRM_MODE_ROTATE_0,
-				    crtc_state->pixel_rate, &wp, 0);
-	drm_WARN_ON(&dev_priv->drm, ret);
-
-	for (level = 0; level <= max_level; level++) {
-		unsigned int latency = dev_priv->wm.skl_latency[level];
-
-		skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
-		if (wm.min_ddb_alloc == U16_MAX)
-			break;
-
-		min_ddb_alloc = wm.min_ddb_alloc;
-	}
-
-	return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
-}
-
-static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
-{
-	skl_ddb_entry_init(entry,
-			   REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
-			   REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
-	if (entry->end)
-		entry->end++;
-}
-
-static void
-skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
-			   const enum pipe pipe,
-			   const enum plane_id plane_id,
-			   struct skl_ddb_entry *ddb,
-			   struct skl_ddb_entry *ddb_y)
-{
-	u32 val;
-
-	/* Cursor doesn't support NV12/planar, so no extra calculation needed */
-	if (plane_id == PLANE_CURSOR) {
-		val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
-		skl_ddb_entry_init_from_hw(ddb, val);
-		return;
-	}
-
-	val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
-	skl_ddb_entry_init_from_hw(ddb, val);
-
-	if (DISPLAY_VER(dev_priv) >= 11)
-		return;
-
-	val = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
-	skl_ddb_entry_init_from_hw(ddb_y, val);
-}
-
-static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
-				      struct skl_ddb_entry *ddb,
-				      struct skl_ddb_entry *ddb_y)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum intel_display_power_domain power_domain;
-	enum pipe pipe = crtc->pipe;
-	intel_wakeref_t wakeref;
-	enum plane_id plane_id;
-
-	power_domain = POWER_DOMAIN_PIPE(pipe);
-	wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
-	if (!wakeref)
-		return;
-
-	for_each_plane_id_on_crtc(crtc, plane_id)
-		skl_ddb_get_hw_plane_state(dev_priv, pipe,
-					   plane_id,
-					   &ddb[plane_id],
-					   &ddb_y[plane_id]);
-
-	intel_display_power_put(dev_priv, power_domain, wakeref);
-}
-
-struct dbuf_slice_conf_entry {
-	u8 active_pipes;
-	u8 dbuf_mask[I915_MAX_PIPES];
-	bool join_mbus;
-};
-
-/*
- * Table taken from Bspec 12716
- * Pipes do have some preferred DBuf slice affinity,
- * plus there are some hardcoded requirements on how
- * those should be distributed for multipipe scenarios.
- * For more DBuf slices algorithm can get even more messy
- * and less readable, so decided to use a table almost
- * as is from BSpec itself - that way it is at least easier
- * to compare, change and check.
- */
-static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
-/* Autogenerated with igt/tools/intel_dbuf_map tool: */
-{
-	{
-		.active_pipes = BIT(PIPE_A),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_B] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_C] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_C] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1),
-			[PIPE_C] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_B] = BIT(DBUF_S1),
-			[PIPE_C] = BIT(DBUF_S2),
-		},
-	},
-	{}
-};
-
-/*
- * Table taken from Bspec 49255
- * Pipes do have some preferred DBuf slice affinity,
- * plus there are some hardcoded requirements on how
- * those should be distributed for multipipe scenarios.
- * For more DBuf slices algorithm can get even more messy
- * and less readable, so decided to use a table almost
- * as is from BSpec itself - that way it is at least easier
- * to compare, change and check.
- */
-static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
-/* Autogenerated with igt/tools/intel_dbuf_map tool: */
-{
-	{
-		.active_pipes = BIT(PIPE_A),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S2),
-			[PIPE_B] = BIT(DBUF_S1),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_C] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1),
-			[PIPE_C] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_B] = BIT(DBUF_S1),
-			[PIPE_C] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_D] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1),
-			[PIPE_D] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_B] = BIT(DBUF_S1),
-			[PIPE_D] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_C] = BIT(DBUF_S1),
-			[PIPE_D] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_C] = BIT(DBUF_S2),
-			[PIPE_D] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1),
-			[PIPE_C] = BIT(DBUF_S2),
-			[PIPE_D] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_B] = BIT(DBUF_S1),
-			[PIPE_C] = BIT(DBUF_S2),
-			[PIPE_D] = BIT(DBUF_S2),
-		},
-	},
-	{}
-};
-
-static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
-	{
-		.active_pipes = BIT(PIPE_A),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_B] = BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_B] = BIT(DBUF_S2),
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_B] = BIT(DBUF_S2),
-			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_C] = BIT(DBUF_S3),
-			[PIPE_D] = BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_C] = BIT(DBUF_S3),
-			[PIPE_D] = BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_C] = BIT(DBUF_S3),
-			[PIPE_D] = BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1),
-			[PIPE_B] = BIT(DBUF_S2),
-			[PIPE_C] = BIT(DBUF_S3),
-			[PIPE_D] = BIT(DBUF_S4),
-		},
-	},
-	{}
-};
-
-static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
-	/*
-	 * Keep the join_mbus cases first so check_mbus_joined()
-	 * will prefer them over the !join_mbus cases.
-	 */
-	{
-		.active_pipes = BIT(PIPE_A),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-		.join_mbus = true,
-	},
-	{
-		.active_pipes = BIT(PIPE_B),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-		.join_mbus = true,
-	},
-	{
-		.active_pipes = BIT(PIPE_A),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-		.join_mbus = false,
-	},
-	{
-		.active_pipes = BIT(PIPE_B),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-		.join_mbus = false,
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
-			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
-			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{
-		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
-		.dbuf_mask = {
-			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
-			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
-			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
-			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
-		},
-	},
-	{}
-
-};
-
-static bool check_mbus_joined(u8 active_pipes,
-			      const struct dbuf_slice_conf_entry *dbuf_slices)
-{
-	int i;
-
-	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
-		if (dbuf_slices[i].active_pipes == active_pipes)
-			return dbuf_slices[i].join_mbus;
-	}
-	return false;
-}
-
-static bool adlp_check_mbus_joined(u8 active_pipes)
-{
-	return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
-}
-
-static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
-			      const struct dbuf_slice_conf_entry *dbuf_slices)
-{
-	int i;
-
-	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
-		if (dbuf_slices[i].active_pipes == active_pipes &&
-		    dbuf_slices[i].join_mbus == join_mbus)
-			return dbuf_slices[i].dbuf_mask[pipe];
-	}
-	return 0;
-}
-
-/*
- * This function finds an entry with same enabled pipe configuration and
- * returns correspondent DBuf slice mask as stated in BSpec for particular
- * platform.
- */
-static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
-{
-	/*
-	 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
-	 * required calculating "pipe ratio" in order to determine
-	 * if one or two slices can be used for single pipe configurations
-	 * as additional constraint to the existing table.
-	 * However based on recent info, it should be not "pipe ratio"
-	 * but rather ratio between pixel_rate and cdclk with additional
-	 * constants, so for now we are using only table until this is
-	 * clarified. Also this is the reason why crtc_state param is
-	 * still here - we will need it once those additional constraints
-	 * pop up.
-	 */
-	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
-				   icl_allowed_dbufs);
-}
-
-static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
-{
-	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
-				   tgl_allowed_dbufs);
-}
-
-static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
-{
-	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
-				   adlp_allowed_dbufs);
-}
-
-static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
-{
-	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
-				   dg2_allowed_dbufs);
-}
-
-static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum pipe pipe = crtc->pipe;
-
-	if (IS_DG2(dev_priv))
-		return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
-	else if (IS_ALDERLAKE_P(dev_priv))
-		return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
-	else if (DISPLAY_VER(dev_priv) == 12)
-		return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
-	else if (DISPLAY_VER(dev_priv) == 11)
-		return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
-	/*
-	 * For anything else just return one slice yet.
-	 * Should be extended for other platforms.
-	 */
-	return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
-}
-
-static bool
-use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
-		     struct intel_plane *plane)
-{
-	struct drm_i915_private *i915 = to_i915(plane->base.dev);
-
-	return DISPLAY_VER(i915) >= 13 &&
-	       crtc_state->uapi.async_flip &&
-	       plane->async_flip;
-}
-
-static u64
-skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
-{
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
-	enum plane_id plane_id;
-	u64 data_rate = 0;
-
-	for_each_plane_id_on_crtc(crtc, plane_id) {
-		if (plane_id == PLANE_CURSOR)
-			continue;
-
-		data_rate += crtc_state->rel_data_rate[plane_id];
-
-		if (DISPLAY_VER(i915) < 11)
-			data_rate += crtc_state->rel_data_rate_y[plane_id];
-	}
-
-	return data_rate;
-}
-
-static const struct skl_wm_level *
-skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
-		   enum plane_id plane_id,
-		   int level)
-{
-	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
-
-	if (level == 0 && pipe_wm->use_sagv_wm)
-		return &wm->sagv.wm0;
-
-	return &wm->wm[level];
-}
-
-static const struct skl_wm_level *
-skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
-		   enum plane_id plane_id)
-{
-	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
-
-	if (pipe_wm->use_sagv_wm)
-		return &wm->sagv.trans_wm;
-
-	return &wm->trans_wm;
-}
-
-/*
- * We only disable the watermarks for each plane if
- * they exceed the ddb allocation of said plane. This
- * is done so that we don't end up touching cursor
- * watermarks needlessly when some other plane reduces
- * our max possible watermark level.
- *
- * Bspec has this to say about the PLANE_WM enable bit:
- * "All the watermarks at this level for all enabled
- *  planes must be enabled before the level will be used."
- * So this is actually safe to do.
- */
-static void
-skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
-{
-	if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
-		memset(wm, 0, sizeof(*wm));
-}
-
-static void
-skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
-			const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
-{
-	if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
-	    uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
-		memset(wm, 0, sizeof(*wm));
-		memset(uv_wm, 0, sizeof(*uv_wm));
-	}
-}
-
-static bool icl_need_wm1_wa(struct drm_i915_private *i915,
-			    enum plane_id plane_id)
-{
-	/*
-	 * Wa_1408961008:icl, ehl
-	 * Wa_14012656716:tgl, adl
-	 * Underruns with WM1+ disabled
-	 */
-	return DISPLAY_VER(i915) == 11 ||
-	       (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR);
-}
-
-struct skl_plane_ddb_iter {
-	u64 data_rate;
-	u16 start, size;
-};
-
-static void
-skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
-		       struct skl_ddb_entry *ddb,
-		       const struct skl_wm_level *wm,
-		       u64 data_rate)
-{
-	u16 size, extra = 0;
-
-	if (data_rate) {
-		extra = min_t(u16, iter->size,
-			      DIV64_U64_ROUND_UP(iter->size * data_rate,
-						 iter->data_rate));
-		iter->size -= extra;
-		iter->data_rate -= data_rate;
-	}
-
-	/*
-	 * Keep ddb entry of all disabled planes explicitly zeroed
-	 * to avoid skl_ddb_add_affected_planes() adding them to
-	 * the state when other planes change their allocations.
-	 */
-	size = wm->min_ddb_alloc + extra;
-	if (size)
-		iter->start = skl_ddb_entry_init(ddb, iter->start,
-						 iter->start + size);
-}
-
-static int
-skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
-			    struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct intel_crtc_state *crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-	const struct intel_dbuf_state *dbuf_state =
-		intel_atomic_get_new_dbuf_state(state);
-	const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
-	int num_active = hweight8(dbuf_state->active_pipes);
-	struct skl_plane_ddb_iter iter;
-	enum plane_id plane_id;
-	u16 cursor_size;
-	u32 blocks;
-	int level;
-
-	/* Clear the partitioning for disabled planes. */
-	memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
-	memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
-
-	if (!crtc_state->hw.active)
-		return 0;
-
-	iter.start = alloc->start;
-	iter.size = skl_ddb_entry_size(alloc);
-	if (iter.size == 0)
-		return 0;
-
-	/* Allocate fixed number of blocks for cursor. */
-	cursor_size = skl_cursor_allocation(crtc_state, num_active);
-	iter.size -= cursor_size;
-	skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
-			   alloc->end - cursor_size, alloc->end);
-
-	iter.data_rate = skl_total_relative_data_rate(crtc_state);
-
-	/*
-	 * Find the highest watermark level for which we can satisfy the block
-	 * requirement of active planes.
-	 */
-	for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
-		blocks = 0;
-		for_each_plane_id_on_crtc(crtc, plane_id) {
-			const struct skl_plane_wm *wm =
-				&crtc_state->wm.skl.optimal.planes[plane_id];
-
-			if (plane_id == PLANE_CURSOR) {
-				const struct skl_ddb_entry *ddb =
-					&crtc_state->wm.skl.plane_ddb[plane_id];
-
-				if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
-					drm_WARN_ON(&dev_priv->drm,
-						    wm->wm[level].min_ddb_alloc != U16_MAX);
-					blocks = U32_MAX;
-					break;
-				}
-				continue;
-			}
-
-			blocks += wm->wm[level].min_ddb_alloc;
-			blocks += wm->uv_wm[level].min_ddb_alloc;
-		}
-
-		if (blocks <= iter.size) {
-			iter.size -= blocks;
-			break;
-		}
-	}
-
-	if (level < 0) {
-		drm_dbg_kms(&dev_priv->drm,
-			    "Requested display configuration exceeds system DDB limitations");
-		drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
-			    blocks, iter.size);
-		return -EINVAL;
-	}
-
-	/* avoid the WARN later when we don't allocate any extra DDB */
-	if (iter.data_rate == 0)
-		iter.size = 0;
-
-	/*
-	 * Grant each plane the blocks it requires at the highest achievable
-	 * watermark level, plus an extra share of the leftover blocks
-	 * proportional to its relative data rate.
-	 */
-	for_each_plane_id_on_crtc(crtc, plane_id) {
-		struct skl_ddb_entry *ddb =
-			&crtc_state->wm.skl.plane_ddb[plane_id];
-		struct skl_ddb_entry *ddb_y =
-			&crtc_state->wm.skl.plane_ddb_y[plane_id];
-		const struct skl_plane_wm *wm =
-			&crtc_state->wm.skl.optimal.planes[plane_id];
-
-		if (plane_id == PLANE_CURSOR)
-			continue;
-
-		if (DISPLAY_VER(dev_priv) < 11 &&
-		    crtc_state->nv12_planes & BIT(plane_id)) {
-			skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
-					       crtc_state->rel_data_rate_y[plane_id]);
-			skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
-					       crtc_state->rel_data_rate[plane_id]);
-		} else {
-			skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
-					       crtc_state->rel_data_rate[plane_id]);
-		}
-	}
-	drm_WARN_ON(&dev_priv->drm, iter.size != 0 || iter.data_rate != 0);
-
-	/*
-	 * When we calculated watermark values we didn't know how high
-	 * of a level we'd actually be able to hit, so we just marked
-	 * all levels as "enabled."  Go back now and disable the ones
-	 * that aren't actually possible.
-	 */
-	for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
-		for_each_plane_id_on_crtc(crtc, plane_id) {
-			const struct skl_ddb_entry *ddb =
-				&crtc_state->wm.skl.plane_ddb[plane_id];
-			const struct skl_ddb_entry *ddb_y =
-				&crtc_state->wm.skl.plane_ddb_y[plane_id];
-			struct skl_plane_wm *wm =
-				&crtc_state->wm.skl.optimal.planes[plane_id];
-
-			if (DISPLAY_VER(dev_priv) < 11 &&
-			    crtc_state->nv12_planes & BIT(plane_id))
-				skl_check_nv12_wm_level(&wm->wm[level],
-							&wm->uv_wm[level],
-							ddb_y, ddb);
-			else
-				skl_check_wm_level(&wm->wm[level], ddb);
-
-			if (icl_need_wm1_wa(dev_priv, plane_id) &&
-			    level == 1 && wm->wm[0].enable) {
-				wm->wm[level].blocks = wm->wm[0].blocks;
-				wm->wm[level].lines = wm->wm[0].lines;
-				wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
-			}
-		}
-	}
-
-	/*
-	 * Go back and disable the transition and SAGV watermarks
-	 * if it turns out we don't have enough DDB blocks for them.
-	 */
-	for_each_plane_id_on_crtc(crtc, plane_id) {
-		const struct skl_ddb_entry *ddb =
-			&crtc_state->wm.skl.plane_ddb[plane_id];
-		const struct skl_ddb_entry *ddb_y =
-			&crtc_state->wm.skl.plane_ddb_y[plane_id];
-		struct skl_plane_wm *wm =
-			&crtc_state->wm.skl.optimal.planes[plane_id];
-
-		if (DISPLAY_VER(dev_priv) < 11 &&
-		    crtc_state->nv12_planes & BIT(plane_id)) {
-			skl_check_wm_level(&wm->trans_wm, ddb_y);
-		} else {
-			WARN_ON(skl_ddb_entry_size(ddb_y));
-
-			skl_check_wm_level(&wm->trans_wm, ddb);
-		}
-
-		skl_check_wm_level(&wm->sagv.wm0, ddb);
-		skl_check_wm_level(&wm->sagv.trans_wm, ddb);
-	}
-
-	return 0;
-}
-
-/*
- * The max latency should be 257 (max the punit can code is 255 and we add 2us
- * for the read latency) and cpp should always be <= 8, so that
- * should allow pixel_rate up to ~2 GHz which seems sufficient since max
- * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
-*/
-static uint_fixed_16_16_t
-skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
-	       u8 cpp, u32 latency, u32 dbuf_block_size)
-{
-	u32 wm_intermediate_val;
-	uint_fixed_16_16_t ret;
-
-	if (latency == 0)
-		return FP_16_16_MAX;
-
-	wm_intermediate_val = latency * pixel_rate * cpp;
-	ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
-
-	if (DISPLAY_VER(dev_priv) >= 10)
-		ret = add_fixed16_u32(ret, 1);
-
-	return ret;
-}
-
-static uint_fixed_16_16_t
-skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
-	       uint_fixed_16_16_t plane_blocks_per_line)
-{
-	u32 wm_intermediate_val;
-	uint_fixed_16_16_t ret;
-
-	if (latency == 0)
-		return FP_16_16_MAX;
-
-	wm_intermediate_val = latency * pixel_rate;
-	wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
-					   pipe_htotal * 1000);
-	ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
-	return ret;
-}
-
-static uint_fixed_16_16_t
-intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
-	u32 pixel_rate;
-	u32 crtc_htotal;
-	uint_fixed_16_16_t linetime_us;
-
-	if (!crtc_state->hw.active)
-		return u32_to_fixed16(0);
-
-	pixel_rate = crtc_state->pixel_rate;
-
-	if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
-		return u32_to_fixed16(0);
-
-	crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
-	linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
-
-	return linetime_us;
-}
-
-static int
-skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
-		      int width, const struct drm_format_info *format,
-		      u64 modifier, unsigned int rotation,
-		      u32 plane_pixel_rate, struct skl_wm_params *wp,
-		      int color_plane)
-{
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	u32 interm_pbpl;
-
-	/* only planar format has two planes */
-	if (color_plane == 1 &&
-	    !intel_format_info_is_yuv_semiplanar(format, modifier)) {
-		drm_dbg_kms(&dev_priv->drm,
-			    "Non planar format have single plane\n");
-		return -EINVAL;
-	}
-
-	wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
-		      modifier == I915_FORMAT_MOD_4_TILED ||
-		      modifier == I915_FORMAT_MOD_Yf_TILED ||
-		      modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
-		      modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
-	wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
-	wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
-			 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
-	wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
-
-	wp->width = width;
-	if (color_plane == 1 && wp->is_planar)
-		wp->width /= 2;
-
-	wp->cpp = format->cpp[color_plane];
-	wp->plane_pixel_rate = plane_pixel_rate;
-
-	if (DISPLAY_VER(dev_priv) >= 11 &&
-	    modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
-		wp->dbuf_block_size = 256;
-	else
-		wp->dbuf_block_size = 512;
-
-	if (drm_rotation_90_or_270(rotation)) {
-		switch (wp->cpp) {
-		case 1:
-			wp->y_min_scanlines = 16;
-			break;
-		case 2:
-			wp->y_min_scanlines = 8;
-			break;
-		case 4:
-			wp->y_min_scanlines = 4;
-			break;
-		default:
-			MISSING_CASE(wp->cpp);
-			return -EINVAL;
-		}
-	} else {
-		wp->y_min_scanlines = 4;
-	}
-
-	if (skl_needs_memory_bw_wa(dev_priv))
-		wp->y_min_scanlines *= 2;
-
-	wp->plane_bytes_per_line = wp->width * wp->cpp;
-	if (wp->y_tiled) {
-		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
-					   wp->y_min_scanlines,
-					   wp->dbuf_block_size);
-
-		if (DISPLAY_VER(dev_priv) >= 10)
-			interm_pbpl++;
-
-		wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
-							wp->y_min_scanlines);
-	} else {
-		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
-					   wp->dbuf_block_size);
-
-		if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
-			interm_pbpl++;
-
-		wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
-	}
-
-	wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
-					     wp->plane_blocks_per_line);
-
-	wp->linetime_us = fixed16_to_u32_round_up(
-					intel_get_linetime_us(crtc_state));
-
-	return 0;
-}
-
-static int
-skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
-			    const struct intel_plane_state *plane_state,
-			    struct skl_wm_params *wp, int color_plane)
-{
-	const struct drm_framebuffer *fb = plane_state->hw.fb;
-	int width;
-
-	/*
-	 * Src coordinates are already rotated by 270 degrees for
-	 * the 90/270 degree plane rotation cases (to match the
-	 * GTT mapping), hence no need to account for rotation here.
-	 */
-	width = drm_rect_width(&plane_state->uapi.src) >> 16;
-
-	return skl_compute_wm_params(crtc_state, width,
-				     fb->format, fb->modifier,
-				     plane_state->hw.rotation,
-				     intel_plane_pixel_rate(crtc_state, plane_state),
-				     wp, color_plane);
-}
-
-static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
-{
-	if (DISPLAY_VER(dev_priv) >= 10)
-		return true;
-
-	/* The number of lines are ignored for the level 0 watermark. */
-	return level > 0;
-}
-
-static int skl_wm_max_lines(struct drm_i915_private *dev_priv)
-{
-	if (DISPLAY_VER(dev_priv) >= 13)
-		return 255;
-	else
-		return 31;
-}
-
-static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
-				 struct intel_plane *plane,
-				 int level,
-				 unsigned int latency,
-				 const struct skl_wm_params *wp,
-				 const struct skl_wm_level *result_prev,
-				 struct skl_wm_level *result /* out */)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
-	uint_fixed_16_16_t method1, method2;
-	uint_fixed_16_16_t selected_result;
-	u32 blocks, lines, min_ddb_alloc = 0;
-
-	if (latency == 0 ||
-	    (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
-		/* reject it */
-		result->min_ddb_alloc = U16_MAX;
-		return;
-	}
-
-	/*
-	 * WaIncreaseLatencyIPCEnabled: kbl,cfl
-	 * Display WA #1141: kbl,cfl
-	 */
-	if ((IS_KABYLAKE(dev_priv) ||
-	     IS_COFFEELAKE(dev_priv) ||
-	     IS_COMETLAKE(dev_priv)) &&
-	    dev_priv->ipc_enabled)
-		latency += 4;
-
-	if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
-		latency += 15;
-
-	method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
-				 wp->cpp, latency, wp->dbuf_block_size);
-	method2 = skl_wm_method2(wp->plane_pixel_rate,
-				 crtc_state->hw.pipe_mode.crtc_htotal,
-				 latency,
-				 wp->plane_blocks_per_line);
-
-	if (wp->y_tiled) {
-		selected_result = max_fixed16(method2, wp->y_tile_minimum);
-	} else {
-		if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
-		     wp->dbuf_block_size < 1) &&
-		     (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
-			selected_result = method2;
-		} else if (latency >= wp->linetime_us) {
-			if (DISPLAY_VER(dev_priv) == 9)
-				selected_result = min_fixed16(method1, method2);
-			else
-				selected_result = method2;
-		} else {
-			selected_result = method1;
-		}
-	}
-
-	blocks = fixed16_to_u32_round_up(selected_result) + 1;
-	/*
-	 * Lets have blocks at minimum equivalent to plane_blocks_per_line
-	 * as there will be at minimum one line for lines configuration. This
-	 * is a work around for FIFO underruns observed with resolutions like
-	 * 4k 60 Hz in single channel DRAM configurations.
-	 *
-	 * As per the Bspec 49325, if the ddb allocation can hold at least
-	 * one plane_blocks_per_line, we should have selected method2 in
-	 * the above logic. Assuming that modern versions have enough dbuf
-	 * and method2 guarantees blocks equivalent to at least 1 line,
-	 * select the blocks as plane_blocks_per_line.
-	 *
-	 * TODO: Revisit the logic when we have better understanding on DRAM
-	 * channels' impact on the level 0 memory latency and the relevant
-	 * wm calculations.
-	 */
-	if (skl_wm_has_lines(dev_priv, level))
-		blocks = max(blocks,
-			     fixed16_to_u32_round_up(wp->plane_blocks_per_line));
-	lines = div_round_up_fixed16(selected_result,
-				     wp->plane_blocks_per_line);
-
-	if (DISPLAY_VER(dev_priv) == 9) {
-		/* Display WA #1125: skl,bxt,kbl */
-		if (level == 0 && wp->rc_surface)
-			blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
-
-		/* Display WA #1126: skl,bxt,kbl */
-		if (level >= 1 && level <= 7) {
-			if (wp->y_tiled) {
-				blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
-				lines += wp->y_min_scanlines;
-			} else {
-				blocks++;
-			}
-
-			/*
-			 * Make sure result blocks for higher latency levels are
-			 * atleast as high as level below the current level.
-			 * Assumption in DDB algorithm optimization for special
-			 * cases. Also covers Display WA #1125 for RC.
-			 */
-			if (result_prev->blocks > blocks)
-				blocks = result_prev->blocks;
-		}
-	}
-
-	if (DISPLAY_VER(dev_priv) >= 11) {
-		if (wp->y_tiled) {
-			int extra_lines;
-
-			if (lines % wp->y_min_scanlines == 0)
-				extra_lines = wp->y_min_scanlines;
-			else
-				extra_lines = wp->y_min_scanlines * 2 -
-					lines % wp->y_min_scanlines;
-
-			min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
-								 wp->plane_blocks_per_line);
-		} else {
-			min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
-		}
-	}
-
-	if (!skl_wm_has_lines(dev_priv, level))
-		lines = 0;
-
-	if (lines > skl_wm_max_lines(dev_priv)) {
-		/* reject it */
-		result->min_ddb_alloc = U16_MAX;
-		return;
-	}
-
-	/*
-	 * If lines is valid, assume we can use this watermark level
-	 * for now.  We'll come back and disable it after we calculate the
-	 * DDB allocation if it turns out we don't actually have enough
-	 * blocks to satisfy it.
-	 */
-	result->blocks = blocks;
-	result->lines = lines;
-	/* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
-	result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
-	result->enable = true;
-
-	if (DISPLAY_VER(dev_priv) < 12 && dev_priv->sagv_block_time_us)
-		result->can_sagv = latency >= dev_priv->sagv_block_time_us;
-}
-
-static void
-skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
-		      struct intel_plane *plane,
-		      const struct skl_wm_params *wm_params,
-		      struct skl_wm_level *levels)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
-	int level, max_level = ilk_wm_max_level(dev_priv);
-	struct skl_wm_level *result_prev = &levels[0];
-
-	for (level = 0; level <= max_level; level++) {
-		struct skl_wm_level *result = &levels[level];
-		unsigned int latency = dev_priv->wm.skl_latency[level];
-
-		skl_compute_plane_wm(crtc_state, plane, level, latency,
-				     wm_params, result_prev, result);
-
-		result_prev = result;
-	}
-}
-
-static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
-				struct intel_plane *plane,
-				const struct skl_wm_params *wm_params,
-				struct skl_plane_wm *plane_wm)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
-	struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
-	struct skl_wm_level *levels = plane_wm->wm;
-	unsigned int latency = 0;
-
-	if (dev_priv->sagv_block_time_us)
-		latency = dev_priv->sagv_block_time_us + dev_priv->wm.skl_latency[0];
-
-	skl_compute_plane_wm(crtc_state, plane, 0, latency,
-			     wm_params, &levels[0],
-			     sagv_wm);
-}
-
-static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
-				      struct skl_wm_level *trans_wm,
-				      const struct skl_wm_level *wm0,
-				      const struct skl_wm_params *wp)
-{
-	u16 trans_min, trans_amount, trans_y_tile_min;
-	u16 wm0_blocks, trans_offset, blocks;
-
-	/* Transition WM don't make any sense if ipc is disabled */
-	if (!dev_priv->ipc_enabled)
-		return;
-
-	/*
-	 * WaDisableTWM:skl,kbl,cfl,bxt
-	 * Transition WM are not recommended by HW team for GEN9
-	 */
-	if (DISPLAY_VER(dev_priv) == 9)
-		return;
-
-	if (DISPLAY_VER(dev_priv) >= 11)
-		trans_min = 4;
-	else
-		trans_min = 14;
-
-	/* Display WA #1140: glk,cnl */
-	if (DISPLAY_VER(dev_priv) == 10)
-		trans_amount = 0;
-	else
-		trans_amount = 10; /* This is configurable amount */
-
-	trans_offset = trans_min + trans_amount;
-
-	/*
-	 * The spec asks for Selected Result Blocks for wm0 (the real value),
-	 * not Result Blocks (the integer value). Pay attention to the capital
-	 * letters. The value wm_l0->blocks is actually Result Blocks, but
-	 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
-	 * and since we later will have to get the ceiling of the sum in the
-	 * transition watermarks calculation, we can just pretend Selected
-	 * Result Blocks is Result Blocks minus 1 and it should work for the
-	 * current platforms.
-	 */
-	wm0_blocks = wm0->blocks - 1;
-
-	if (wp->y_tiled) {
-		trans_y_tile_min =
-			(u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
-		blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
-	} else {
-		blocks = wm0_blocks + trans_offset;
-	}
-	blocks++;
-
-	/*
-	 * Just assume we can enable the transition watermark.  After
-	 * computing the DDB we'll come back and disable it if that
-	 * assumption turns out to be false.
-	 */
-	trans_wm->blocks = blocks;
-	trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
-	trans_wm->enable = true;
-}
-
-static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
-				     const struct intel_plane_state *plane_state,
-				     struct intel_plane *plane, int color_plane)
-{
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
-	struct skl_wm_params wm_params;
-	int ret;
-
-	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
-					  &wm_params, color_plane);
-	if (ret)
-		return ret;
-
-	skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
-
-	skl_compute_transition_wm(dev_priv, &wm->trans_wm,
-				  &wm->wm[0], &wm_params);
-
-	if (DISPLAY_VER(dev_priv) >= 12) {
-		tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
-
-		skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
-					  &wm->sagv.wm0, &wm_params);
-	}
-
-	return 0;
-}
-
-static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
-				 const struct intel_plane_state *plane_state,
-				 struct intel_plane *plane)
-{
-	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
-	struct skl_wm_params wm_params;
-	int ret;
-
-	wm->is_planar = true;
-
-	/* uv plane watermarks must also be validated for NV12/Planar */
-	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
-					  &wm_params, 1);
-	if (ret)
-		return ret;
-
-	skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
-
-	return 0;
-}
-
-static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
-			      const struct intel_plane_state *plane_state)
-{
-	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
-	enum plane_id plane_id = plane->id;
-	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
-	const struct drm_framebuffer *fb = plane_state->hw.fb;
-	int ret;
-
-	memset(wm, 0, sizeof(*wm));
-
-	if (!intel_wm_plane_visible(crtc_state, plane_state))
-		return 0;
-
-	ret = skl_build_plane_wm_single(crtc_state, plane_state,
-					plane, 0);
-	if (ret)
-		return ret;
-
-	if (fb->format->is_yuv && fb->format->num_planes > 1) {
-		ret = skl_build_plane_wm_uv(crtc_state, plane_state,
-					    plane);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
-			      const struct intel_plane_state *plane_state)
-{
-	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
-	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
-	enum plane_id plane_id = plane->id;
-	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
-	int ret;
-
-	/* Watermarks calculated in master */
-	if (plane_state->planar_slave)
-		return 0;
-
-	memset(wm, 0, sizeof(*wm));
-
-	if (plane_state->planar_linked_plane) {
-		const struct drm_framebuffer *fb = plane_state->hw.fb;
-
-		drm_WARN_ON(&dev_priv->drm,
-			    !intel_wm_plane_visible(crtc_state, plane_state));
-		drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
-			    fb->format->num_planes == 1);
-
-		ret = skl_build_plane_wm_single(crtc_state, plane_state,
-						plane_state->planar_linked_plane, 0);
-		if (ret)
-			return ret;
-
-		ret = skl_build_plane_wm_single(crtc_state, plane_state,
-						plane, 1);
-		if (ret)
-			return ret;
-	} else if (intel_wm_plane_visible(crtc_state, plane_state)) {
-		ret = skl_build_plane_wm_single(crtc_state, plane_state,
-						plane, 0);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static int skl_build_pipe_wm(struct intel_atomic_state *state,
-			     struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct intel_crtc_state *crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-	const struct intel_plane_state *plane_state;
-	struct intel_plane *plane;
-	int ret, i;
-
-	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
-		/*
-		 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
-		 * instead but we don't populate that correctly for NV12 Y
-		 * planes so for now hack this.
-		 */
-		if (plane->pipe != crtc->pipe)
-			continue;
-
-		if (DISPLAY_VER(dev_priv) >= 11)
-			ret = icl_build_plane_wm(crtc_state, plane_state);
-		else
-			ret = skl_build_plane_wm(crtc_state, plane_state);
-		if (ret)
-			return ret;
-	}
-
-	crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
-
-	return 0;
-}
-
-static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
-				i915_reg_t reg,
-				const struct skl_ddb_entry *entry)
-{
-	if (entry->end)
-		intel_de_write_fw(dev_priv, reg,
-				  PLANE_BUF_END(entry->end - 1) |
-				  PLANE_BUF_START(entry->start));
-	else
-		intel_de_write_fw(dev_priv, reg, 0);
-}
-
-static void skl_write_wm_level(struct drm_i915_private *dev_priv,
-			       i915_reg_t reg,
-			       const struct skl_wm_level *level)
-{
-	u32 val = 0;
-
-	if (level->enable)
-		val |= PLANE_WM_EN;
-	if (level->ignore_lines)
-		val |= PLANE_WM_IGNORE_LINES;
-	val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks);
-	val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
-
-	intel_de_write_fw(dev_priv, reg, val);
-}
-
-void skl_write_plane_wm(struct intel_plane *plane,
-			const struct intel_crtc_state *crtc_state)
-{
-	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
-	int level, max_level = ilk_wm_max_level(dev_priv);
-	enum plane_id plane_id = plane->id;
-	enum pipe pipe = plane->pipe;
-	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
-	const struct skl_ddb_entry *ddb =
-		&crtc_state->wm.skl.plane_ddb[plane_id];
-	const struct skl_ddb_entry *ddb_y =
-		&crtc_state->wm.skl.plane_ddb_y[plane_id];
-
-	for (level = 0; level <= max_level; level++)
-		skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
-				   skl_plane_wm_level(pipe_wm, plane_id, level));
-
-	skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
-			   skl_plane_trans_wm(pipe_wm, plane_id));
-
-	if (HAS_HW_SAGV_WM(dev_priv)) {
-		const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
-
-		skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id),
-				   &wm->sagv.wm0);
-		skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id),
-				   &wm->sagv.trans_wm);
-	}
-
-	skl_ddb_entry_write(dev_priv,
-			    PLANE_BUF_CFG(pipe, plane_id), ddb);
-
-	if (DISPLAY_VER(dev_priv) < 11)
-		skl_ddb_entry_write(dev_priv,
-				    PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y);
-}
-
-void skl_write_cursor_wm(struct intel_plane *plane,
-			 const struct intel_crtc_state *crtc_state)
-{
-	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
-	int level, max_level = ilk_wm_max_level(dev_priv);
-	enum plane_id plane_id = plane->id;
-	enum pipe pipe = plane->pipe;
-	const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
-	const struct skl_ddb_entry *ddb =
-		&crtc_state->wm.skl.plane_ddb[plane_id];
-
-	for (level = 0; level <= max_level; level++)
-		skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
-				   skl_plane_wm_level(pipe_wm, plane_id, level));
-
-	skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
-			   skl_plane_trans_wm(pipe_wm, plane_id));
-
-	if (HAS_HW_SAGV_WM(dev_priv)) {
-		const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
-
-		skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe),
-				   &wm->sagv.wm0);
-		skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe),
-				   &wm->sagv.trans_wm);
-	}
-
-	skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
-}
-
-static bool skl_wm_level_equals(const struct skl_wm_level *l1,
-				const struct skl_wm_level *l2)
-{
-	return l1->enable == l2->enable &&
-		l1->ignore_lines == l2->ignore_lines &&
-		l1->lines == l2->lines &&
-		l1->blocks == l2->blocks;
-}
-
-static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
-				const struct skl_plane_wm *wm1,
-				const struct skl_plane_wm *wm2)
-{
-	int level, max_level = ilk_wm_max_level(dev_priv);
-
-	for (level = 0; level <= max_level; level++) {
-		/*
-		 * We don't check uv_wm as the hardware doesn't actually
-		 * use it. It only gets used for calculating the required
-		 * ddb allocation.
-		 */
-		if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
-			return false;
-	}
-
-	return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
-		skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
-		skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
-}
-
-static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
-				    const struct skl_ddb_entry *b)
-{
-	return a->start < b->end && b->start < a->end;
-}
-
-static void skl_ddb_entry_union(struct skl_ddb_entry *a,
-				const struct skl_ddb_entry *b)
-{
-	if (a->end && b->end) {
-		a->start = min(a->start, b->start);
-		a->end = max(a->end, b->end);
-	} else if (b->end) {
-		a->start = b->start;
-		a->end = b->end;
-	}
-}
-
-bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
-				 const struct skl_ddb_entry *entries,
-				 int num_entries, int ignore_idx)
-{
-	int i;
-
-	for (i = 0; i < num_entries; i++) {
-		if (i != ignore_idx &&
-		    skl_ddb_entries_overlap(ddb, &entries[i]))
-			return true;
-	}
-
-	return false;
-}
-
-static int
-skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
-			    struct intel_crtc_state *new_crtc_state)
-{
-	struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
-	struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct intel_plane *plane;
-
-	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
-		struct intel_plane_state *plane_state;
-		enum plane_id plane_id = plane->id;
-
-		if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
-					&new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
-		    skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
-					&new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
-			continue;
-
-		plane_state = intel_atomic_get_plane_state(state, plane);
-		if (IS_ERR(plane_state))
-			return PTR_ERR(plane_state);
-
-		new_crtc_state->update_planes |= BIT(plane_id);
-	}
-
-	return 0;
-}
-
-static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
-{
-	struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
-	u8 enabled_slices;
-	enum pipe pipe;
-
-	/*
-	 * FIXME: For now we always enable slice S1 as per
-	 * the Bspec display initialization sequence.
-	 */
-	enabled_slices = BIT(DBUF_S1);
-
-	for_each_pipe(dev_priv, pipe)
-		enabled_slices |= dbuf_state->slices[pipe];
-
-	return enabled_slices;
-}
-
-static int
-skl_compute_ddb(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	const struct intel_dbuf_state *old_dbuf_state;
-	struct intel_dbuf_state *new_dbuf_state = NULL;
-	const struct intel_crtc_state *old_crtc_state;
-	struct intel_crtc_state *new_crtc_state;
-	struct intel_crtc *crtc;
-	int ret, i;
-
-	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
-		new_dbuf_state = intel_atomic_get_dbuf_state(state);
-		if (IS_ERR(new_dbuf_state))
-			return PTR_ERR(new_dbuf_state);
-
-		old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
-		break;
-	}
-
-	if (!new_dbuf_state)
-		return 0;
-
-	new_dbuf_state->active_pipes =
-		intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
-
-	if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
-		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
-		if (ret)
-			return ret;
-	}
-
-	if (HAS_MBUS_JOINING(dev_priv))
-		new_dbuf_state->joined_mbus =
-			adlp_check_mbus_joined(new_dbuf_state->active_pipes);
-
-	for_each_intel_crtc(&dev_priv->drm, crtc) {
-		enum pipe pipe = crtc->pipe;
-
-		new_dbuf_state->slices[pipe] =
-			skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
-						new_dbuf_state->joined_mbus);
-
-		if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
-			continue;
-
-		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
-		if (ret)
-			return ret;
-	}
-
-	new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
-
-	if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
-	    old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
-		ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
-		if (ret)
-			return ret;
-
-		if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
-			/* TODO: Implement vblank synchronized MBUS joining changes */
-			ret = intel_modeset_all_pipes(state);
-			if (ret)
-				return ret;
-		}
-
-		drm_dbg_kms(&dev_priv->drm,
-			    "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
-			    old_dbuf_state->enabled_slices,
-			    new_dbuf_state->enabled_slices,
-			    INTEL_INFO(dev_priv)->display.dbuf.slice_mask,
-			    str_yes_no(old_dbuf_state->joined_mbus),
-			    str_yes_no(new_dbuf_state->joined_mbus));
-	}
-
-	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
-		enum pipe pipe = crtc->pipe;
-
-		new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
-
-		if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
-			continue;
-
-		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
-		if (ret)
-			return ret;
-	}
-
-	for_each_intel_crtc(&dev_priv->drm, crtc) {
-		ret = skl_crtc_allocate_ddb(state, crtc);
-		if (ret)
-			return ret;
-	}
-
-	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
-					    new_crtc_state, i) {
-		ret = skl_crtc_allocate_plane_ddb(state, crtc);
-		if (ret)
-			return ret;
-
-		ret = skl_ddb_add_affected_planes(old_crtc_state,
-						  new_crtc_state);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-static char enast(bool enable)
-{
-	return enable ? '*' : ' ';
-}
-
-static void
-skl_print_wm_changes(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	const struct intel_crtc_state *old_crtc_state;
-	const struct intel_crtc_state *new_crtc_state;
-	struct intel_plane *plane;
-	struct intel_crtc *crtc;
-	int i;
-
-	if (!drm_debug_enabled(DRM_UT_KMS))
-		return;
-
-	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
-					    new_crtc_state, i) {
-		const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
-
-		old_pipe_wm = &old_crtc_state->wm.skl.optimal;
-		new_pipe_wm = &new_crtc_state->wm.skl.optimal;
-
-		for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
-			enum plane_id plane_id = plane->id;
-			const struct skl_ddb_entry *old, *new;
-
-			old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
-			new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
-
-			if (skl_ddb_entry_equal(old, new))
-				continue;
-
-			drm_dbg_kms(&dev_priv->drm,
-				    "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
-				    plane->base.base.id, plane->base.name,
-				    old->start, old->end, new->start, new->end,
-				    skl_ddb_entry_size(old), skl_ddb_entry_size(new));
-		}
-
-		for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
-			enum plane_id plane_id = plane->id;
-			const struct skl_plane_wm *old_wm, *new_wm;
-
-			old_wm = &old_pipe_wm->planes[plane_id];
-			new_wm = &new_pipe_wm->planes[plane_id];
-
-			if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
-				continue;
-
-			drm_dbg_kms(&dev_priv->drm,
-				    "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
-				    " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
-				    plane->base.base.id, plane->base.name,
-				    enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
-				    enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
-				    enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
-				    enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
-				    enast(old_wm->trans_wm.enable),
-				    enast(old_wm->sagv.wm0.enable),
-				    enast(old_wm->sagv.trans_wm.enable),
-				    enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
-				    enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
-				    enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
-				    enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
-				    enast(new_wm->trans_wm.enable),
-				    enast(new_wm->sagv.wm0.enable),
-				    enast(new_wm->sagv.trans_wm.enable));
-
-			drm_dbg_kms(&dev_priv->drm,
-				    "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
-				      " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
-				    plane->base.base.id, plane->base.name,
-				    enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
-				    enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
-				    enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
-				    enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
-				    enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
-				    enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
-				    enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
-				    enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
-				    enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
-				    enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
-				    enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
-				    enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
-				    enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
-				    enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
-				    enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
-				    enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
-				    enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
-				    enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
-				    enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
-				    enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
-				    enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
-				    enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
-
-			drm_dbg_kms(&dev_priv->drm,
-				    "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
-				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
-				    plane->base.base.id, plane->base.name,
-				    old_wm->wm[0].blocks, old_wm->wm[1].blocks,
-				    old_wm->wm[2].blocks, old_wm->wm[3].blocks,
-				    old_wm->wm[4].blocks, old_wm->wm[5].blocks,
-				    old_wm->wm[6].blocks, old_wm->wm[7].blocks,
-				    old_wm->trans_wm.blocks,
-				    old_wm->sagv.wm0.blocks,
-				    old_wm->sagv.trans_wm.blocks,
-				    new_wm->wm[0].blocks, new_wm->wm[1].blocks,
-				    new_wm->wm[2].blocks, new_wm->wm[3].blocks,
-				    new_wm->wm[4].blocks, new_wm->wm[5].blocks,
-				    new_wm->wm[6].blocks, new_wm->wm[7].blocks,
-				    new_wm->trans_wm.blocks,
-				    new_wm->sagv.wm0.blocks,
-				    new_wm->sagv.trans_wm.blocks);
-
-			drm_dbg_kms(&dev_priv->drm,
-				    "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
-				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
-				    plane->base.base.id, plane->base.name,
-				    old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
-				    old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
-				    old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
-				    old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
-				    old_wm->trans_wm.min_ddb_alloc,
-				    old_wm->sagv.wm0.min_ddb_alloc,
-				    old_wm->sagv.trans_wm.min_ddb_alloc,
-				    new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
-				    new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
-				    new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
-				    new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
-				    new_wm->trans_wm.min_ddb_alloc,
-				    new_wm->sagv.wm0.min_ddb_alloc,
-				    new_wm->sagv.trans_wm.min_ddb_alloc);
-		}
-	}
-}
-
-static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
-					 const struct skl_pipe_wm *old_pipe_wm,
-					 const struct skl_pipe_wm *new_pipe_wm)
-{
-	struct drm_i915_private *i915 = to_i915(plane->base.dev);
-	int level, max_level = ilk_wm_max_level(i915);
-
-	for (level = 0; level <= max_level; level++) {
-		/*
-		 * We don't check uv_wm as the hardware doesn't actually
-		 * use it. It only gets used for calculating the required
-		 * ddb allocation.
-		 */
-		if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
-					 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
-			return false;
-	}
-
-	if (HAS_HW_SAGV_WM(i915)) {
-		const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
-		const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
-
-		if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
-		    !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
-			return false;
-	}
-
-	return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
-				   skl_plane_trans_wm(new_pipe_wm, plane->id));
-}
-
-/*
- * To make sure the cursor watermark registers are always consistent
- * with our computed state the following scenario needs special
- * treatment:
- *
- * 1. enable cursor
- * 2. move cursor entirely offscreen
- * 3. disable cursor
- *
- * Step 2. does call .disable_plane() but does not zero the watermarks
- * (since we consider an offscreen cursor still active for the purposes
- * of watermarks). Step 3. would not normally call .disable_plane()
- * because the actual plane visibility isn't changing, and we don't
- * deallocate the cursor ddb until the pipe gets disabled. So we must
- * force step 3. to call .disable_plane() to update the watermark
- * registers properly.
- *
- * Other planes do not suffer from this issues as their watermarks are
- * calculated based on the actual plane visibility. The only time this
- * can trigger for the other planes is during the initial readout as the
- * default value of the watermarks registers is not zero.
- */
-static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
-				      struct intel_crtc *crtc)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	const struct intel_crtc_state *old_crtc_state =
-		intel_atomic_get_old_crtc_state(state, crtc);
-	struct intel_crtc_state *new_crtc_state =
-		intel_atomic_get_new_crtc_state(state, crtc);
-	struct intel_plane *plane;
-
-	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
-		struct intel_plane_state *plane_state;
-		enum plane_id plane_id = plane->id;
-
-		/*
-		 * Force a full wm update for every plane on modeset.
-		 * Required because the reset value of the wm registers
-		 * is non-zero, whereas we want all disabled planes to
-		 * have zero watermarks. So if we turn off the relevant
-		 * power well the hardware state will go out of sync
-		 * with the software state.
-		 */
-		if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
-		    skl_plane_selected_wm_equals(plane,
-						 &old_crtc_state->wm.skl.optimal,
-						 &new_crtc_state->wm.skl.optimal))
-			continue;
-
-		plane_state = intel_atomic_get_plane_state(state, plane);
-		if (IS_ERR(plane_state))
-			return PTR_ERR(plane_state);
-
-		new_crtc_state->update_planes |= BIT(plane_id);
-	}
-
-	return 0;
-}
-
-static int
-skl_compute_wm(struct intel_atomic_state *state)
-{
-	struct intel_crtc *crtc;
-	struct intel_crtc_state *new_crtc_state;
-	int ret, i;
-
-	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
-		ret = skl_build_pipe_wm(state, crtc);
-		if (ret)
-			return ret;
-	}
-
-	ret = skl_compute_ddb(state);
-	if (ret)
-		return ret;
-
-	ret = intel_compute_sagv_mask(state);
-	if (ret)
-		return ret;
-
-	/*
-	 * skl_compute_ddb() will have adjusted the final watermarks
-	 * based on how much ddb is available. Now we can actually
-	 * check if the final watermarks changed.
-	 */
-	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
-		ret = skl_wm_add_affected_planes(state, crtc);
-		if (ret)
-			return ret;
-	}
-
-	skl_print_wm_changes(state);
-
-	return 0;
-}
-
 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
 				  struct intel_wm_config *config)
 {
@@ -6459,10 +3583,10 @@ static void ilk_initial_watermarks(struct intel_atomic_state *state,
 	const struct intel_crtc_state *crtc_state =
 		intel_atomic_get_new_crtc_state(state, crtc);
 
-	mutex_lock(&dev_priv->wm.wm_mutex);
+	mutex_lock(&dev_priv->display.wm.wm_mutex);
 	crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
 	ilk_program_watermarks(dev_priv);
-	mutex_unlock(&dev_priv->wm.wm_mutex);
+	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 }
 
 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
@@ -6475,216 +3599,17 @@ static void ilk_optimize_watermarks(struct intel_atomic_state *state,
 	if (!crtc_state->wm.need_postvbl_update)
 		return;
 
-	mutex_lock(&dev_priv->wm.wm_mutex);
+	mutex_lock(&dev_priv->display.wm.wm_mutex);
 	crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
 	ilk_program_watermarks(dev_priv);
-	mutex_unlock(&dev_priv->wm.wm_mutex);
-}
-
-static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
-{
-	level->enable = val & PLANE_WM_EN;
-	level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
-	level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
-	level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
-}
-
-static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
-				     struct skl_pipe_wm *out)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	enum pipe pipe = crtc->pipe;
-	int level, max_level;
-	enum plane_id plane_id;
-	u32 val;
-
-	max_level = ilk_wm_max_level(dev_priv);
-
-	for_each_plane_id_on_crtc(crtc, plane_id) {
-		struct skl_plane_wm *wm = &out->planes[plane_id];
-
-		for (level = 0; level <= max_level; level++) {
-			if (plane_id != PLANE_CURSOR)
-				val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
-			else
-				val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
-
-			skl_wm_level_from_reg_val(val, &wm->wm[level]);
-		}
-
-		if (plane_id != PLANE_CURSOR)
-			val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
-		else
-			val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
-
-		skl_wm_level_from_reg_val(val, &wm->trans_wm);
-
-		if (HAS_HW_SAGV_WM(dev_priv)) {
-			if (plane_id != PLANE_CURSOR)
-				val = intel_uncore_read(&dev_priv->uncore,
-							PLANE_WM_SAGV(pipe, plane_id));
-			else
-				val = intel_uncore_read(&dev_priv->uncore,
-							CUR_WM_SAGV(pipe));
-
-			skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
-
-			if (plane_id != PLANE_CURSOR)
-				val = intel_uncore_read(&dev_priv->uncore,
-							PLANE_WM_SAGV_TRANS(pipe, plane_id));
-			else
-				val = intel_uncore_read(&dev_priv->uncore,
-							CUR_WM_SAGV_TRANS(pipe));
-
-			skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
-		} else if (DISPLAY_VER(dev_priv) >= 12) {
-			wm->sagv.wm0 = wm->wm[0];
-			wm->sagv.trans_wm = wm->trans_wm;
-		}
-	}
-}
-
-void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
-{
-	struct intel_dbuf_state *dbuf_state =
-		to_intel_dbuf_state(dev_priv->dbuf.obj.state);
-	struct intel_crtc *crtc;
-
-	if (HAS_MBUS_JOINING(dev_priv))
-		dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
-
-	for_each_intel_crtc(&dev_priv->drm, crtc) {
-		struct intel_crtc_state *crtc_state =
-			to_intel_crtc_state(crtc->base.state);
-		enum pipe pipe = crtc->pipe;
-		unsigned int mbus_offset;
-		enum plane_id plane_id;
-		u8 slices;
-
-		memset(&crtc_state->wm.skl.optimal, 0,
-		       sizeof(crtc_state->wm.skl.optimal));
-		if (crtc_state->hw.active)
-			skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
-		crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
-
-		memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
-
-		for_each_plane_id_on_crtc(crtc, plane_id) {
-			struct skl_ddb_entry *ddb =
-				&crtc_state->wm.skl.plane_ddb[plane_id];
-			struct skl_ddb_entry *ddb_y =
-				&crtc_state->wm.skl.plane_ddb_y[plane_id];
-
-			if (!crtc_state->hw.active)
-				continue;
-
-			skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
-						   plane_id, ddb, ddb_y);
-
-			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
-			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
-		}
-
-		dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
-
-		/*
-		 * Used for checking overlaps, so we need absolute
-		 * offsets instead of MBUS relative offsets.
-		 */
-		slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
-						 dbuf_state->joined_mbus);
-		mbus_offset = mbus_ddb_offset(dev_priv, slices);
-		crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
-		crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
-
-		/* The slices actually used by the planes on the pipe */
-		dbuf_state->slices[pipe] =
-			skl_ddb_dbuf_slice_mask(dev_priv, &crtc_state->wm.skl.ddb);
-
-		drm_dbg_kms(&dev_priv->drm,
-			    "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
-			    crtc->base.base.id, crtc->base.name,
-			    dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
-			    dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
-			    str_yes_no(dbuf_state->joined_mbus));
-	}
-
-	dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
-}
-
-static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
-{
-	const struct intel_dbuf_state *dbuf_state =
-		to_intel_dbuf_state(i915->dbuf.obj.state);
-	struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
-	struct intel_crtc *crtc;
-
-	for_each_intel_crtc(&i915->drm, crtc) {
-		const struct intel_crtc_state *crtc_state =
-			to_intel_crtc_state(crtc->base.state);
-
-		entries[crtc->pipe] = crtc_state->wm.skl.ddb;
-	}
-
-	for_each_intel_crtc(&i915->drm, crtc) {
-		const struct intel_crtc_state *crtc_state =
-			to_intel_crtc_state(crtc->base.state);
-		u8 slices;
-
-		slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
-						 dbuf_state->joined_mbus);
-		if (dbuf_state->slices[crtc->pipe] & ~slices)
-			return true;
-
-		if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
-						I915_MAX_PIPES, crtc->pipe))
-			return true;
-	}
-
-	return false;
-}
-
-void skl_wm_sanitize(struct drm_i915_private *i915)
-{
-	struct intel_crtc *crtc;
-
-	/*
-	 * On TGL/RKL (at least) the BIOS likes to assign the planes
-	 * to the wrong DBUF slices. This will cause an infinite loop
-	 * in skl_commit_modeset_enables() as it can't find a way to
-	 * transition between the old bogus DBUF layout to the new
-	 * proper DBUF layout without DBUF allocation overlaps between
-	 * the planes (which cannot be allowed or else the hardware
-	 * may hang). If we detect a bogus DBUF layout just turn off
-	 * all the planes so that skl_commit_modeset_enables() can
-	 * simply ignore them.
-	 */
-	if (!skl_dbuf_is_misconfigured(i915))
-		return;
-
-	drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
-
-	for_each_intel_crtc(&i915->drm, crtc) {
-		struct intel_plane *plane = to_intel_plane(crtc->base.primary);
-		const struct intel_plane_state *plane_state =
-			to_intel_plane_state(plane->base.state);
-		struct intel_crtc_state *crtc_state =
-			to_intel_crtc_state(crtc->base.state);
-
-		if (plane_state->uapi.visible)
-			intel_plane_disable_noatomic(crtc, plane);
-
-		drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
-
-		memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
-	}
+	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 }
 
 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
 {
 	struct drm_device *dev = crtc->base.dev;
 	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct ilk_wm_values *hw = &dev_priv->wm.hw;
+	struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
 	struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
 	struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
 	enum pipe pipe = crtc->pipe;
@@ -6832,7 +3757,7 @@ static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
 
 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
 {
-	struct g4x_wm_values *wm = &dev_priv->wm.g4x;
+	struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
 	struct intel_crtc *crtc;
 
 	g4x_read_wm_values(dev_priv, wm);
@@ -6926,7 +3851,7 @@ void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
 	struct intel_plane *plane;
 	struct intel_crtc *crtc;
 
-	mutex_lock(&dev_priv->wm.wm_mutex);
+	mutex_lock(&dev_priv->display.wm.wm_mutex);
 
 	for_each_intel_plane(&dev_priv->drm, plane) {
 		struct intel_crtc *crtc =
@@ -6974,12 +3899,12 @@ void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
 
 	g4x_program_watermarks(dev_priv);
 
-	mutex_unlock(&dev_priv->wm.wm_mutex);
+	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 }
 
 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
 {
-	struct vlv_wm_values *wm = &dev_priv->wm.vlv;
+	struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
 	struct intel_crtc *crtc;
 	u32 val;
 
@@ -7013,7 +3938,7 @@ void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
 			drm_dbg_kms(&dev_priv->drm,
 				    "Punit not acking DDR DVFS request, "
 				    "assuming DDR DVFS is disabled\n");
-			dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
+			dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5;
 		} else {
 			val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 			if ((val & FORCE_DDR_HIGH_FREQ) == 0)
@@ -7082,7 +4007,7 @@ void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
 	struct intel_plane *plane;
 	struct intel_crtc *crtc;
 
-	mutex_lock(&dev_priv->wm.wm_mutex);
+	mutex_lock(&dev_priv->display.wm.wm_mutex);
 
 	for_each_intel_plane(&dev_priv->drm, plane) {
 		struct intel_crtc *crtc =
@@ -7123,7 +4048,7 @@ void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
 
 	vlv_program_watermarks(dev_priv);
 
-	mutex_unlock(&dev_priv->wm.wm_mutex);
+	mutex_unlock(&dev_priv->display.wm.wm_mutex);
 }
 
 /*
@@ -7144,7 +4069,7 @@ static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
 
 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
 {
-	struct ilk_wm_values *hw = &dev_priv->wm.hw;
+	struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
 	struct intel_crtc *crtc;
 
 	ilk_init_lp_watermarks(dev_priv);
@@ -7173,168 +4098,6 @@ void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
 		!(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
 }
 
-void intel_wm_state_verify(struct intel_crtc *crtc,
-			   struct intel_crtc_state *new_crtc_state)
-{
-	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-	struct skl_hw_state {
-		struct skl_ddb_entry ddb[I915_MAX_PLANES];
-		struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
-		struct skl_pipe_wm wm;
-	} *hw;
-	const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
-	int level, max_level = ilk_wm_max_level(dev_priv);
-	struct intel_plane *plane;
-	u8 hw_enabled_slices;
-
-	if (DISPLAY_VER(dev_priv) < 9 || !new_crtc_state->hw.active)
-		return;
-
-	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
-	if (!hw)
-		return;
-
-	skl_pipe_wm_get_hw_state(crtc, &hw->wm);
-
-	skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y);
-
-	hw_enabled_slices = intel_enabled_dbuf_slices_mask(dev_priv);
-
-	if (DISPLAY_VER(dev_priv) >= 11 &&
-	    hw_enabled_slices != dev_priv->dbuf.enabled_slices)
-		drm_err(&dev_priv->drm,
-			"mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
-			dev_priv->dbuf.enabled_slices,
-			hw_enabled_slices);
-
-	for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
-		const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
-		const struct skl_wm_level *hw_wm_level, *sw_wm_level;
-
-		/* Watermarks */
-		for (level = 0; level <= max_level; level++) {
-			hw_wm_level = &hw->wm.planes[plane->id].wm[level];
-			sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
-
-			if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
-				continue;
-
-			drm_err(&dev_priv->drm,
-				"[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
-				plane->base.base.id, plane->base.name, level,
-				sw_wm_level->enable,
-				sw_wm_level->blocks,
-				sw_wm_level->lines,
-				hw_wm_level->enable,
-				hw_wm_level->blocks,
-				hw_wm_level->lines);
-		}
-
-		hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
-		sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
-
-		if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
-			drm_err(&dev_priv->drm,
-				"[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
-				plane->base.base.id, plane->base.name,
-				sw_wm_level->enable,
-				sw_wm_level->blocks,
-				sw_wm_level->lines,
-				hw_wm_level->enable,
-				hw_wm_level->blocks,
-				hw_wm_level->lines);
-		}
-
-		hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
-		sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
-
-		if (HAS_HW_SAGV_WM(dev_priv) &&
-		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
-			drm_err(&dev_priv->drm,
-				"[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
-				plane->base.base.id, plane->base.name,
-				sw_wm_level->enable,
-				sw_wm_level->blocks,
-				sw_wm_level->lines,
-				hw_wm_level->enable,
-				hw_wm_level->blocks,
-				hw_wm_level->lines);
-		}
-
-		hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
-		sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
-
-		if (HAS_HW_SAGV_WM(dev_priv) &&
-		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
-			drm_err(&dev_priv->drm,
-				"[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
-				plane->base.base.id, plane->base.name,
-				sw_wm_level->enable,
-				sw_wm_level->blocks,
-				sw_wm_level->lines,
-				hw_wm_level->enable,
-				hw_wm_level->blocks,
-				hw_wm_level->lines);
-		}
-
-		/* DDB */
-		hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
-		sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
-
-		if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
-			drm_err(&dev_priv->drm,
-				"[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
-				plane->base.base.id, plane->base.name,
-				sw_ddb_entry->start, sw_ddb_entry->end,
-				hw_ddb_entry->start, hw_ddb_entry->end);
-		}
-	}
-
-	kfree(hw);
-}
-
-void intel_enable_ipc(struct drm_i915_private *dev_priv)
-{
-	u32 val;
-
-	if (!HAS_IPC(dev_priv))
-		return;
-
-	val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
-
-	if (dev_priv->ipc_enabled)
-		val |= DISP_IPC_ENABLE;
-	else
-		val &= ~DISP_IPC_ENABLE;
-
-	intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
-}
-
-static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
-{
-	/* Display WA #0477 WaDisableIPC: skl */
-	if (IS_SKYLAKE(dev_priv))
-		return false;
-
-	/* Display WA #1141: SKL:all KBL:all CFL */
-	if (IS_KABYLAKE(dev_priv) ||
-	    IS_COFFEELAKE(dev_priv) ||
-	    IS_COMETLAKE(dev_priv))
-		return dev_priv->dram_info.symmetric_memory;
-
-	return true;
-}
-
-void intel_init_ipc(struct drm_i915_private *dev_priv)
-{
-	if (!HAS_IPC(dev_priv))
-		return;
-
-	dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
-
-	intel_enable_ipc(dev_priv);
-}
-
 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
 {
 	/*
@@ -7442,7 +4205,7 @@ static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
 		val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
 		val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 		val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
-		if (dev_priv->vbt.fdi_rx_polarity_inverted)
+		if (dev_priv->display.vbt.fdi_rx_polarity_inverted)
 			val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
 		val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
@@ -7593,9 +4356,8 @@ static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
 
 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-	/* Wa_1409120013:tgl,rkl,adl-s,dg1,dg2 */
-	if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv) ||
-	    IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv) || IS_DG2(dev_priv))
+	/* Wa_1409120013 */
+	if (DISPLAY_VER(dev_priv) == 12)
 		intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 				   DPFC_CHICKEN_COMP_DUMMY_PIXEL);
 
@@ -7972,7 +4734,7 @@ static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
 		OVCUNIT_CLOCK_GATE_DISABLE;
 	if (IS_GM45(dev_priv))
 		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
-	intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
+	intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D(dev_priv), dspclk_gate);
 
 	g4x_disable_trickle_feed(dev_priv);
 }
@@ -7983,7 +4745,7 @@ static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
 
 	intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
 	intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
-	intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
+	intel_uncore_write(uncore, DSPCLK_GATE_D(dev_priv), 0);
 	intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
 	intel_uncore_write16(uncore, DEUC, 0);
 	intel_uncore_write(uncore,
@@ -8175,18 +4937,14 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
 	}
 }
 
-static const struct drm_i915_wm_disp_funcs skl_wm_funcs = {
-	.compute_global_watermarks = skl_compute_wm,
-};
-
-static const struct drm_i915_wm_disp_funcs ilk_wm_funcs = {
+static const struct intel_wm_funcs ilk_wm_funcs = {
 	.compute_pipe_wm = ilk_compute_pipe_wm,
 	.compute_intermediate_wm = ilk_compute_intermediate_wm,
 	.initial_watermarks = ilk_initial_watermarks,
 	.optimize_watermarks = ilk_optimize_watermarks,
 };
 
-static const struct drm_i915_wm_disp_funcs vlv_wm_funcs = {
+static const struct intel_wm_funcs vlv_wm_funcs = {
 	.compute_pipe_wm = vlv_compute_pipe_wm,
 	.compute_intermediate_wm = vlv_compute_intermediate_wm,
 	.initial_watermarks = vlv_initial_watermarks,
@@ -8194,67 +4952,67 @@ static const struct drm_i915_wm_disp_funcs vlv_wm_funcs = {
 	.atomic_update_watermarks = vlv_atomic_update_fifo,
 };
 
-static const struct drm_i915_wm_disp_funcs g4x_wm_funcs = {
+static const struct intel_wm_funcs g4x_wm_funcs = {
 	.compute_pipe_wm = g4x_compute_pipe_wm,
 	.compute_intermediate_wm = g4x_compute_intermediate_wm,
 	.initial_watermarks = g4x_initial_watermarks,
 	.optimize_watermarks = g4x_optimize_watermarks,
 };
 
-static const struct drm_i915_wm_disp_funcs pnv_wm_funcs = {
+static const struct intel_wm_funcs pnv_wm_funcs = {
 	.update_wm = pnv_update_wm,
 };
 
-static const struct drm_i915_wm_disp_funcs i965_wm_funcs = {
+static const struct intel_wm_funcs i965_wm_funcs = {
 	.update_wm = i965_update_wm,
 };
 
-static const struct drm_i915_wm_disp_funcs i9xx_wm_funcs = {
+static const struct intel_wm_funcs i9xx_wm_funcs = {
 	.update_wm = i9xx_update_wm,
 };
 
-static const struct drm_i915_wm_disp_funcs i845_wm_funcs = {
+static const struct intel_wm_funcs i845_wm_funcs = {
 	.update_wm = i845_update_wm,
 };
 
-static const struct drm_i915_wm_disp_funcs nop_funcs = {
+static const struct intel_wm_funcs nop_funcs = {
 };
 
 /* Set up chip specific power management-related functions */
 void intel_init_pm(struct drm_i915_private *dev_priv)
 {
+	if (DISPLAY_VER(dev_priv) >= 9) {
+		skl_wm_init(dev_priv);
+		return;
+	}
+
 	/* For cxsr */
 	if (IS_PINEVIEW(dev_priv))
 		pnv_get_mem_freq(dev_priv);
 	else if (GRAPHICS_VER(dev_priv) == 5)
 		ilk_get_mem_freq(dev_priv);
 
-	intel_sagv_init(dev_priv);
-
 	/* For FIFO watermark updates */
-	if (DISPLAY_VER(dev_priv) >= 9) {
-		skl_setup_wm_latency(dev_priv);
-		dev_priv->wm_disp = &skl_wm_funcs;
-	} else if (HAS_PCH_SPLIT(dev_priv)) {
+	if (HAS_PCH_SPLIT(dev_priv)) {
 		ilk_setup_wm_latency(dev_priv);
 
-		if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->wm.pri_latency[1] &&
-		     dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
-		    (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] &&
-		     dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
-			dev_priv->wm_disp = &ilk_wm_funcs;
+		if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] &&
+		     dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) ||
+		    (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] &&
+		     dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) {
+			dev_priv->display.funcs.wm = &ilk_wm_funcs;
 		} else {
 			drm_dbg_kms(&dev_priv->drm,
 				    "Failed to read display plane latency. "
 				    "Disable CxSR\n");
-			dev_priv->wm_disp = &nop_funcs;
+			dev_priv->display.funcs.wm = &nop_funcs;
 		}
 	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 		vlv_setup_wm_latency(dev_priv);
-		dev_priv->wm_disp = &vlv_wm_funcs;
+		dev_priv->display.funcs.wm = &vlv_wm_funcs;
 	} else if (IS_G4X(dev_priv)) {
 		g4x_setup_wm_latency(dev_priv);
-		dev_priv->wm_disp = &g4x_wm_funcs;
+		dev_priv->display.funcs.wm = &g4x_wm_funcs;
 	} else if (IS_PINEVIEW(dev_priv)) {
 		if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 					    dev_priv->is_ddr3,
@@ -8268,22 +5026,22 @@ void intel_init_pm(struct drm_i915_private *dev_priv)
 				 dev_priv->fsb_freq, dev_priv->mem_freq);
 			/* Disable CxSR and never update its watermark again */
 			intel_set_memory_cxsr(dev_priv, false);
-			dev_priv->wm_disp = &nop_funcs;
+			dev_priv->display.funcs.wm = &nop_funcs;
 		} else
-			dev_priv->wm_disp = &pnv_wm_funcs;
+			dev_priv->display.funcs.wm = &pnv_wm_funcs;
 	} else if (DISPLAY_VER(dev_priv) == 4) {
-		dev_priv->wm_disp = &i965_wm_funcs;
+		dev_priv->display.funcs.wm = &i965_wm_funcs;
 	} else if (DISPLAY_VER(dev_priv) == 3) {
-		dev_priv->wm_disp = &i9xx_wm_funcs;
+		dev_priv->display.funcs.wm = &i9xx_wm_funcs;
 	} else if (DISPLAY_VER(dev_priv) == 2) {
 		if (INTEL_NUM_PIPES(dev_priv) == 1)
-			dev_priv->wm_disp = &i845_wm_funcs;
+			dev_priv->display.funcs.wm = &i845_wm_funcs;
 		else
-			dev_priv->wm_disp = &i9xx_wm_funcs;
+			dev_priv->display.funcs.wm = &i9xx_wm_funcs;
 	} else {
 		drm_err(&dev_priv->drm,
 			"unexpected fall-through in %s\n", __func__);
-		dev_priv->wm_disp = &nop_funcs;
+		dev_priv->display.funcs.wm = &nop_funcs;
 	}
 }
 
@@ -8292,183 +5050,3 @@ void intel_pm_setup(struct drm_i915_private *dev_priv)
 	dev_priv->runtime_pm.suspended = false;
 	atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
 }
-
-static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
-{
-	struct intel_dbuf_state *dbuf_state;
-
-	dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
-	if (!dbuf_state)
-		return NULL;
-
-	return &dbuf_state->base;
-}
-
-static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
-				     struct intel_global_state *state)
-{
-	kfree(state);
-}
-
-static const struct intel_global_state_funcs intel_dbuf_funcs = {
-	.atomic_duplicate_state = intel_dbuf_duplicate_state,
-	.atomic_destroy_state = intel_dbuf_destroy_state,
-};
-
-struct intel_dbuf_state *
-intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	struct intel_global_state *dbuf_state;
-
-	dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
-	if (IS_ERR(dbuf_state))
-		return ERR_CAST(dbuf_state);
-
-	return to_intel_dbuf_state(dbuf_state);
-}
-
-int intel_dbuf_init(struct drm_i915_private *dev_priv)
-{
-	struct intel_dbuf_state *dbuf_state;
-
-	dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
-	if (!dbuf_state)
-		return -ENOMEM;
-
-	intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
-				     &dbuf_state->base, &intel_dbuf_funcs);
-
-	return 0;
-}
-
-/*
- * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
- * update the request state of all DBUS slices.
- */
-static void update_mbus_pre_enable(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	u32 mbus_ctl, dbuf_min_tracker_val;
-	enum dbuf_slice slice;
-	const struct intel_dbuf_state *dbuf_state =
-		intel_atomic_get_new_dbuf_state(state);
-
-	if (!HAS_MBUS_JOINING(dev_priv))
-		return;
-
-	/*
-	 * TODO: Implement vblank synchronized MBUS joining changes.
-	 * Must be properly coordinated with dbuf reprogramming.
-	 */
-	if (dbuf_state->joined_mbus) {
-		mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
-			MBUS_JOIN_PIPE_SELECT_NONE;
-		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
-	} else {
-		mbus_ctl = MBUS_HASHING_MODE_2x2 |
-			MBUS_JOIN_PIPE_SELECT_NONE;
-		dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
-	}
-
-	intel_de_rmw(dev_priv, MBUS_CTL,
-		     MBUS_HASHING_MODE_MASK | MBUS_JOIN |
-		     MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
-
-	for_each_dbuf_slice(dev_priv, slice)
-		intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
-			     DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
-			     dbuf_min_tracker_val);
-}
-
-void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	const struct intel_dbuf_state *new_dbuf_state =
-		intel_atomic_get_new_dbuf_state(state);
-	const struct intel_dbuf_state *old_dbuf_state =
-		intel_atomic_get_old_dbuf_state(state);
-
-	if (!new_dbuf_state ||
-	    ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
-	    && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
-		return;
-
-	WARN_ON(!new_dbuf_state->base.changed);
-
-	update_mbus_pre_enable(state);
-	gen9_dbuf_slices_update(dev_priv,
-				old_dbuf_state->enabled_slices |
-				new_dbuf_state->enabled_slices);
-}
-
-void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *dev_priv = to_i915(state->base.dev);
-	const struct intel_dbuf_state *new_dbuf_state =
-		intel_atomic_get_new_dbuf_state(state);
-	const struct intel_dbuf_state *old_dbuf_state =
-		intel_atomic_get_old_dbuf_state(state);
-
-	if (!new_dbuf_state ||
-	    ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
-	    && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
-		return;
-
-	WARN_ON(!new_dbuf_state->base.changed);
-
-	gen9_dbuf_slices_update(dev_priv,
-				new_dbuf_state->enabled_slices);
-}
-
-void intel_mbus_dbox_update(struct intel_atomic_state *state)
-{
-	struct drm_i915_private *i915 = to_i915(state->base.dev);
-	const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
-	const struct intel_crtc_state *new_crtc_state;
-	const struct intel_crtc *crtc;
-	u32 val = 0;
-	int i;
-
-	if (DISPLAY_VER(i915) < 11)
-		return;
-
-	new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
-	old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
-	if (!new_dbuf_state ||
-	    (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
-	     new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
-		return;
-
-	if (DISPLAY_VER(i915) >= 12) {
-		val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
-		val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
-		val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
-	}
-
-	/* Wa_22010947358:adl-p */
-	if (IS_ALDERLAKE_P(i915))
-		val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
-						     MBUS_DBOX_A_CREDIT(4);
-	else
-		val |= MBUS_DBOX_A_CREDIT(2);
-
-	if (IS_ALDERLAKE_P(i915)) {
-		val |= MBUS_DBOX_BW_CREDIT(2);
-		val |= MBUS_DBOX_B_CREDIT(8);
-	} else if (DISPLAY_VER(i915) >= 12) {
-		val |= MBUS_DBOX_BW_CREDIT(2);
-		val |= MBUS_DBOX_B_CREDIT(12);
-	} else {
-		val |= MBUS_DBOX_BW_CREDIT(1);
-		val |= MBUS_DBOX_B_CREDIT(8);
-	}
-
-	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
-		if (!new_crtc_state->hw.active ||
-		    !intel_crtc_needs_modeset(new_crtc_state))
-			continue;
-
-		intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), val);
-	}
-}
diff --git a/drivers/gpu/drm/i915/intel_pm.h b/drivers/gpu/drm/i915/intel_pm.h
index 945503ae493e..c09b872d65c8 100644
--- a/drivers/gpu/drm/i915/intel_pm.h
+++ b/drivers/gpu/drm/i915/intel_pm.h
@@ -8,22 +8,9 @@
 
 #include <linux/types.h>
 
-#include "display/intel_display.h"
-#include "display/intel_global_state.h"
-
-#include "i915_drv.h"
-
-struct drm_device;
 struct drm_i915_private;
-struct i915_request;
-struct intel_atomic_state;
-struct intel_bw_state;
-struct intel_crtc;
 struct intel_crtc_state;
-struct intel_plane;
-struct skl_ddb_entry;
-struct skl_pipe_wm;
-struct skl_wm_level;
+struct intel_plane_state;
 
 void intel_init_clock_gating(struct drm_i915_private *dev_priv);
 void intel_suspend_hw(struct drm_i915_private *dev_priv);
@@ -34,56 +21,14 @@ void intel_pm_setup(struct drm_i915_private *dev_priv);
 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv);
 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv);
 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv);
-void skl_wm_get_hw_state(struct drm_i915_private *dev_priv);
-void intel_wm_state_verify(struct intel_crtc *crtc,
-			   struct intel_crtc_state *new_crtc_state);
-u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv);
-void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv);
-u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
-			    const struct skl_ddb_entry *entry);
 void g4x_wm_sanitize(struct drm_i915_private *dev_priv);
 void vlv_wm_sanitize(struct drm_i915_private *dev_priv);
-void skl_wm_sanitize(struct drm_i915_private *dev_priv);
-bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
-			   const struct intel_bw_state *bw_state);
-void intel_sagv_pre_plane_update(struct intel_atomic_state *state);
-void intel_sagv_post_plane_update(struct intel_atomic_state *state);
-bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
-				 const struct skl_ddb_entry *entries,
-				 int num_entries, int ignore_idx);
-void skl_write_plane_wm(struct intel_plane *plane,
-			const struct intel_crtc_state *crtc_state);
-void skl_write_cursor_wm(struct intel_plane *plane,
-			 const struct intel_crtc_state *crtc_state);
 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv);
-void intel_init_ipc(struct drm_i915_private *dev_priv);
-void intel_enable_ipc(struct drm_i915_private *dev_priv);
+bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
+			    const struct intel_plane_state *plane_state);
+void intel_print_wm_latency(struct drm_i915_private *dev_priv,
+			    const char *name, const u16 wm[]);
 
 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable);
 
-struct intel_dbuf_state {
-	struct intel_global_state base;
-
-	struct skl_ddb_entry ddb[I915_MAX_PIPES];
-	unsigned int weight[I915_MAX_PIPES];
-	u8 slices[I915_MAX_PIPES];
-	u8 enabled_slices;
-	u8 active_pipes;
-	bool joined_mbus;
-};
-
-struct intel_dbuf_state *
-intel_atomic_get_dbuf_state(struct intel_atomic_state *state);
-
-#define to_intel_dbuf_state(x) container_of((x), struct intel_dbuf_state, base)
-#define intel_atomic_get_old_dbuf_state(state) \
-	to_intel_dbuf_state(intel_atomic_get_old_global_obj_state(state, &to_i915(state->base.dev)->dbuf.obj))
-#define intel_atomic_get_new_dbuf_state(state) \
-	to_intel_dbuf_state(intel_atomic_get_new_global_obj_state(state, &to_i915(state->base.dev)->dbuf.obj))
-
-int intel_dbuf_init(struct drm_i915_private *dev_priv);
-void intel_dbuf_pre_plane_update(struct intel_atomic_state *state);
-void intel_dbuf_post_plane_update(struct intel_atomic_state *state);
-void intel_mbus_dbox_update(struct intel_atomic_state *state);
-
 #endif /* __INTEL_PM_H__ */
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index a852c471d1b3..5cd423c7b646 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -21,6 +21,7 @@
  * IN THE SOFTWARE.
  */
 
+#include <drm/drm_managed.h>
 #include <linux/pm_runtime.h>
 
 #include "gt/intel_engine_regs.h"
@@ -44,29 +45,47 @@ fw_domains_get(struct intel_uncore *uncore, enum forcewake_domains fw_domains)
 }
 
 void
-intel_uncore_mmio_debug_init_early(struct intel_uncore_mmio_debug *mmio_debug)
+intel_uncore_mmio_debug_init_early(struct drm_i915_private *i915)
 {
-	spin_lock_init(&mmio_debug->lock);
-	mmio_debug->unclaimed_mmio_check = 1;
+	spin_lock_init(&i915->mmio_debug.lock);
+	i915->mmio_debug.unclaimed_mmio_check = 1;
+
+	i915->uncore.debug = &i915->mmio_debug;
 }
 
-static void mmio_debug_suspend(struct intel_uncore_mmio_debug *mmio_debug)
+static void mmio_debug_suspend(struct intel_uncore *uncore)
 {
-	lockdep_assert_held(&mmio_debug->lock);
+	if (!uncore->debug)
+		return;
+
+	spin_lock(&uncore->debug->lock);
 
 	/* Save and disable mmio debugging for the user bypass */
-	if (!mmio_debug->suspend_count++) {
-		mmio_debug->saved_mmio_check = mmio_debug->unclaimed_mmio_check;
-		mmio_debug->unclaimed_mmio_check = 0;
+	if (!uncore->debug->suspend_count++) {
+		uncore->debug->saved_mmio_check = uncore->debug->unclaimed_mmio_check;
+		uncore->debug->unclaimed_mmio_check = 0;
 	}
+
+	spin_unlock(&uncore->debug->lock);
 }
 
-static void mmio_debug_resume(struct intel_uncore_mmio_debug *mmio_debug)
+static bool check_for_unclaimed_mmio(struct intel_uncore *uncore);
+
+static void mmio_debug_resume(struct intel_uncore *uncore)
 {
-	lockdep_assert_held(&mmio_debug->lock);
+	if (!uncore->debug)
+		return;
+
+	spin_lock(&uncore->debug->lock);
+
+	if (!--uncore->debug->suspend_count)
+		uncore->debug->unclaimed_mmio_check = uncore->debug->saved_mmio_check;
+
+	if (check_for_unclaimed_mmio(uncore))
+		drm_info(&uncore->i915->drm,
+			 "Invalid mmio detected during user access\n");
 
-	if (!--mmio_debug->suspend_count)
-		mmio_debug->unclaimed_mmio_check = mmio_debug->saved_mmio_check;
+	spin_unlock(&uncore->debug->lock);
 }
 
 static const char * const forcewake_domain_names[] = {
@@ -112,8 +131,11 @@ fw_domain_reset(const struct intel_uncore_forcewake_domain *d)
 	 * trying to reset here does exist at this point (engines could be fused
 	 * off in ICL+), so no waiting for acks
 	 */
-	/* WaRsClearFWBitsAtReset:bdw,skl */
-	fw_clear(d, 0xffff);
+	/* WaRsClearFWBitsAtReset */
+	if (GRAPHICS_VER(d->uncore->i915) >= 12)
+		fw_clear(d, 0xefff);
+	else
+		fw_clear(d, 0xffff);
 }
 
 static inline void
@@ -674,9 +696,7 @@ void intel_uncore_forcewake_user_get(struct intel_uncore *uncore)
 	spin_lock_irq(&uncore->lock);
 	if (!uncore->user_forcewake_count++) {
 		intel_uncore_forcewake_get__locked(uncore, FORCEWAKE_ALL);
-		spin_lock(&uncore->debug->lock);
-		mmio_debug_suspend(uncore->debug);
-		spin_unlock(&uncore->debug->lock);
+		mmio_debug_suspend(uncore);
 	}
 	spin_unlock_irq(&uncore->lock);
 }
@@ -692,14 +712,7 @@ void intel_uncore_forcewake_user_put(struct intel_uncore *uncore)
 {
 	spin_lock_irq(&uncore->lock);
 	if (!--uncore->user_forcewake_count) {
-		spin_lock(&uncore->debug->lock);
-		mmio_debug_resume(uncore->debug);
-
-		if (check_for_unclaimed_mmio(uncore))
-			drm_info(&uncore->i915->drm,
-				 "Invalid mmio detected during user access\n");
-		spin_unlock(&uncore->debug->lock);
-
+		mmio_debug_resume(uncore);
 		intel_uncore_forcewake_put__locked(uncore, FORCEWAKE_ALL);
 	}
 	spin_unlock_irq(&uncore->lock);
@@ -915,6 +928,9 @@ find_fw_domain(struct intel_uncore *uncore, u32 offset)
 {
 	const struct intel_forcewake_range *entry;
 
+	if (IS_GSI_REG(offset))
+		offset += uncore->gsi_offset;
+
 	entry = BSEARCH(offset,
 			uncore->fw_domains_table,
 			uncore->fw_domains_table_entries,
@@ -1130,6 +1146,9 @@ static bool is_shadowed(struct intel_uncore *uncore, u32 offset)
 	if (drm_WARN_ON(&uncore->i915->drm, !uncore->shadowed_reg_table))
 		return false;
 
+	if (IS_GSI_REG(offset))
+		offset += uncore->gsi_offset;
+
 	return BSEARCH(offset,
 		       uncore->shadowed_reg_table,
 		       uncore->shadowed_reg_table_entries,
@@ -1701,7 +1720,7 @@ unclaimed_reg_debug(struct intel_uncore *uncore,
 		    const bool read,
 		    const bool before)
 {
-	if (likely(!uncore->i915->params.mmio_debug))
+	if (likely(!uncore->i915->params.mmio_debug) || !uncore->debug)
 		return;
 
 	/* interrupts are disabled and re-enabled around uncore->lock usage */
@@ -1982,8 +2001,8 @@ static int __fw_domain_init(struct intel_uncore *uncore,
 
 	d->uncore = uncore;
 	d->wake_count = 0;
-	d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set);
-	d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack);
+	d->reg_set = uncore->regs + i915_mmio_reg_offset(reg_set) + uncore->gsi_offset;
+	d->reg_ack = uncore->regs + i915_mmio_reg_offset(reg_ack) + uncore->gsi_offset;
 
 	d->id = domain_id;
 
@@ -2067,7 +2086,7 @@ static int intel_uncore_fw_domains_init(struct intel_uncore *uncore)
 
 	if (GRAPHICS_VER(i915) >= 11) {
 		/* we'll prune the domains of missing engines later */
-		intel_engine_mask_t emask = INTEL_INFO(i915)->platform_engine_mask;
+		intel_engine_mask_t emask = RUNTIME_INFO(i915)->platform_engine_mask;
 		int i;
 
 		uncore->fw_get_funcs = &uncore_get_fallback;
@@ -2220,6 +2239,11 @@ static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
 	return NOTIFY_OK;
 }
 
+static void uncore_unmap_mmio(struct drm_device *drm, void *regs)
+{
+	iounmap(regs);
+}
+
 int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr)
 {
 	struct drm_i915_private *i915 = uncore->i915;
@@ -2232,14 +2256,15 @@ int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr)
 	 * clobbering the GTT which we want ioremap_wc instead. Fortunately,
 	 * the register BAR remains the same size for all the earlier
 	 * generations up to Ironlake.
-	 * For dgfx chips register range is expanded to 4MB.
+	 * For dgfx chips register range is expanded to 4MB, and this larger
+	 * range is also used for integrated gpus beginning with Meteor Lake.
 	 */
-	if (GRAPHICS_VER(i915) < 5)
-		mmio_size = 512 * 1024;
-	else if (IS_DGFX(i915))
+	if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
 		mmio_size = 4 * 1024 * 1024;
-	else
+	else if (GRAPHICS_VER(i915) >= 5)
 		mmio_size = 2 * 1024 * 1024;
+	else
+		mmio_size = 512 * 1024;
 
 	uncore->regs = ioremap(phys_addr, mmio_size);
 	if (uncore->regs == NULL) {
@@ -2247,12 +2272,7 @@ int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr)
 		return -EIO;
 	}
 
-	return 0;
-}
-
-void intel_uncore_cleanup_mmio(struct intel_uncore *uncore)
-{
-	iounmap(uncore->regs);
+	return drmm_add_action_or_reset(&i915->drm, uncore_unmap_mmio, uncore->regs);
 }
 
 void intel_uncore_init_early(struct intel_uncore *uncore,
@@ -2262,7 +2282,6 @@ void intel_uncore_init_early(struct intel_uncore *uncore,
 	uncore->i915 = gt->i915;
 	uncore->gt = gt;
 	uncore->rpm = &gt->i915->runtime_pm;
-	uncore->debug = &gt->i915->mmio_debug;
 }
 
 static void uncore_raw_init(struct intel_uncore *uncore)
@@ -2442,8 +2461,11 @@ void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore,
 	}
 }
 
-void intel_uncore_fini_mmio(struct intel_uncore *uncore)
+/* Called via drm-managed action */
+void intel_uncore_fini_mmio(struct drm_device *dev, void *data)
 {
+	struct intel_uncore *uncore = data;
+
 	if (intel_uncore_has_forcewake(uncore)) {
 		iosf_mbi_punit_acquire();
 		iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
@@ -2573,6 +2595,9 @@ bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore)
 {
 	bool ret;
 
+	if (!uncore->debug)
+		return false;
+
 	spin_lock_irq(&uncore->debug->lock);
 	ret = check_for_unclaimed_mmio(uncore);
 	spin_unlock_irq(&uncore->debug->lock);
@@ -2585,6 +2610,9 @@ intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore)
 {
 	bool ret = false;
 
+	if (drm_WARN_ON(&uncore->i915->drm, !uncore->debug))
+		return false;
+
 	spin_lock_irq(&uncore->debug->lock);
 
 	if (unlikely(uncore->debug->unclaimed_mmio_check <= 0))
diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h
index b1fa912a65e7..5022bac80b67 100644
--- a/drivers/gpu/drm/i915/intel_uncore.h
+++ b/drivers/gpu/drm/i915/intel_uncore.h
@@ -33,6 +33,7 @@
 
 #include "i915_reg_defs.h"
 
+struct drm_device;
 struct drm_i915_private;
 struct intel_runtime_pm;
 struct intel_uncore;
@@ -135,6 +136,16 @@ struct intel_uncore {
 
 	spinlock_t lock; /** lock is also taken in irq contexts. */
 
+	/*
+	 * Do we need to apply an additional offset to reach the beginning
+	 * of the basic non-engine GT registers (referred to as "GSI" on
+	 * newer platforms, or "GT block" on older platforms)?  If so, we'll
+	 * track that here and apply it transparently to registers in the
+	 * appropriate range to maintain compatibility with our existing
+	 * register definitions and GT code.
+	 */
+	u32 gsi_offset;
+
 	unsigned int flags;
 #define UNCORE_HAS_FORCEWAKE		BIT(0)
 #define UNCORE_HAS_FPGA_DBG_UNCLAIMED	BIT(1)
@@ -210,8 +221,7 @@ intel_uncore_has_fifo(const struct intel_uncore *uncore)
 	return uncore->flags & UNCORE_HAS_FIFO;
 }
 
-void
-intel_uncore_mmio_debug_init_early(struct intel_uncore_mmio_debug *mmio_debug);
+void intel_uncore_mmio_debug_init_early(struct drm_i915_private *i915);
 void intel_uncore_init_early(struct intel_uncore *uncore,
 			     struct intel_gt *gt);
 int intel_uncore_setup_mmio(struct intel_uncore *uncore, phys_addr_t phys_addr);
@@ -221,7 +231,7 @@ void intel_uncore_prune_engine_fw_domains(struct intel_uncore *uncore,
 bool intel_uncore_unclaimed_mmio(struct intel_uncore *uncore);
 bool intel_uncore_arm_unclaimed_mmio_detection(struct intel_uncore *uncore);
 void intel_uncore_cleanup_mmio(struct intel_uncore *uncore);
-void intel_uncore_fini_mmio(struct intel_uncore *uncore);
+void intel_uncore_fini_mmio(struct drm_device *dev, void *data);
 void intel_uncore_suspend(struct intel_uncore *uncore);
 void intel_uncore_resume_early(struct intel_uncore *uncore);
 void intel_uncore_runtime_resume(struct intel_uncore *uncore);
@@ -294,19 +304,27 @@ intel_wait_for_register_fw(struct intel_uncore *uncore,
 					    2, timeout_ms, NULL);
 }
 
+#define IS_GSI_REG(reg) ((reg) < 0x40000)
+
 /* register access functions */
 #define __raw_read(x__, s__) \
 static inline u##x__ __raw_uncore_read##x__(const struct intel_uncore *uncore, \
 					    i915_reg_t reg) \
 { \
-	return read##s__(uncore->regs + i915_mmio_reg_offset(reg)); \
+	u32 offset = i915_mmio_reg_offset(reg); \
+	if (IS_GSI_REG(offset)) \
+		offset += uncore->gsi_offset; \
+	return read##s__(uncore->regs + offset); \
 }
 
 #define __raw_write(x__, s__) \
 static inline void __raw_uncore_write##x__(const struct intel_uncore *uncore, \
 					   i915_reg_t reg, u##x__ val) \
 { \
-	write##s__(val, uncore->regs + i915_mmio_reg_offset(reg)); \
+	u32 offset = i915_mmio_reg_offset(reg); \
+	if (IS_GSI_REG(offset)) \
+		offset += uncore->gsi_offset; \
+	write##s__(val, uncore->regs + offset); \
 }
 __raw_read(8, b)
 __raw_read(16, w)
@@ -447,6 +465,18 @@ static inline int intel_uncore_write_and_verify(struct intel_uncore *uncore,
 	return (reg_val & mask) != expected_val ? -EINVAL : 0;
 }
 
+/*
+ * The raw_reg_{read,write} macros are intended as a micro-optimization for
+ * interrupt handlers so that the pointer indirection on uncore->regs can
+ * be computed once (and presumably cached in a register) instead of generating
+ * extra load instructions for each MMIO access.
+ *
+ * Given that these macros are only intended for non-GSI interrupt registers
+ * (and the goal is to avoid extra instructions generated by the compiler),
+ * these macros do not account for uncore->gsi_offset.  Any caller that needs
+ * to use these macros on a GSI register is responsible for adding the
+ * appropriate GSI offset to the 'base' parameter.
+ */
 #define raw_reg_read(base, reg) \
 	readl(base + i915_mmio_reg_offset(reg))
 #define raw_reg_write(base, reg, value) \
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp.c b/drivers/gpu/drm/i915/pxp/intel_pxp.c
index 15311eaed848..69cdaaddc4a9 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp.c
@@ -169,11 +169,23 @@ static void pxp_queue_termination(struct intel_pxp *pxp)
 	 * We want to get the same effect as if we received a termination
 	 * interrupt, so just pretend that we did.
 	 */
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	intel_pxp_mark_termination_in_progress(pxp);
 	pxp->session_events |= PXP_TERMINATION_REQUEST;
 	queue_work(system_unbound_wq, &pxp->session_work);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
+}
+
+static bool pxp_component_bound(struct intel_pxp *pxp)
+{
+	bool bound = false;
+
+	mutex_lock(&pxp->tee_mutex);
+	if (pxp->pxp_component)
+		bound = true;
+	mutex_unlock(&pxp->tee_mutex);
+
+	return bound;
 }
 
 /*
@@ -187,6 +199,9 @@ int intel_pxp_start(struct intel_pxp *pxp)
 	if (!intel_pxp_is_enabled(pxp))
 		return -ENODEV;
 
+	if (wait_for(pxp_component_bound(pxp), 250))
+		return -ENXIO;
+
 	mutex_lock(&pxp->arb_mutex);
 
 	if (pxp->arb_is_valid)
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c b/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c
index e888b5124a07..4359e8be4101 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_debugfs.c
@@ -47,9 +47,9 @@ static int pxp_terminate_set(void *data, u64 val)
 		return -ENODEV;
 
 	/* simulate a termination interrupt */
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	intel_pxp_irq_handler(pxp, GEN12_DISPLAY_PXP_STATE_TERMINATED_INTERRUPT);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 
 	if (!wait_for_completion_timeout(&pxp->termination,
 					 msecs_to_jiffies(100)))
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c
index 04745f914407..c28be430718a 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_irq.c
@@ -25,7 +25,7 @@ void intel_pxp_irq_handler(struct intel_pxp *pxp, u16 iir)
 	if (GEM_WARN_ON(!intel_pxp_is_enabled(pxp)))
 		return;
 
-	lockdep_assert_held(&gt->irq_lock);
+	lockdep_assert_held(gt->irq_lock);
 
 	if (unlikely(!iir))
 		return;
@@ -55,16 +55,16 @@ static inline void __pxp_set_interrupts(struct intel_gt *gt, u32 interrupts)
 
 static inline void pxp_irq_reset(struct intel_gt *gt)
 {
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	gen11_gt_reset_one_iir(gt, 0, GEN11_KCR);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 }
 
 void intel_pxp_irq_enable(struct intel_pxp *pxp)
 {
 	struct intel_gt *gt = pxp_to_gt(pxp);
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 
 	if (!pxp->irq_enabled)
 		WARN_ON_ONCE(gen11_gt_reset_one_iir(gt, 0, GEN11_KCR));
@@ -72,7 +72,7 @@ void intel_pxp_irq_enable(struct intel_pxp *pxp)
 	__pxp_set_interrupts(gt, GEN12_PXP_INTERRUPTS);
 	pxp->irq_enabled = true;
 
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 }
 
 void intel_pxp_irq_disable(struct intel_pxp *pxp)
@@ -88,12 +88,12 @@ void intel_pxp_irq_disable(struct intel_pxp *pxp)
 	 */
 	GEM_WARN_ON(intel_pxp_is_active(pxp));
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 
 	pxp->irq_enabled = false;
 	__pxp_set_interrupts(gt, 0);
 
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 	intel_synchronize_irq(gt->i915);
 
 	pxp_irq_reset(gt);
diff --git a/drivers/gpu/drm/i915/pxp/intel_pxp_session.c b/drivers/gpu/drm/i915/pxp/intel_pxp_session.c
index 92b00b4de240..1bb5b5249157 100644
--- a/drivers/gpu/drm/i915/pxp/intel_pxp_session.c
+++ b/drivers/gpu/drm/i915/pxp/intel_pxp_session.c
@@ -144,9 +144,9 @@ void intel_pxp_session_work(struct work_struct *work)
 	intel_wakeref_t wakeref;
 	u32 events = 0;
 
-	spin_lock_irq(&gt->irq_lock);
+	spin_lock_irq(gt->irq_lock);
 	events = fetch_and_zero(&pxp->session_events);
-	spin_unlock_irq(&gt->irq_lock);
+	spin_unlock_irq(gt->irq_lock);
 
 	if (!events)
 		return;
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index ab9f17fc85bc..e050a2de5fd1 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -1080,7 +1080,7 @@ static int misaligned_case(struct i915_address_space *vm, struct intel_memory_re
 	bool is_stolen = mr->type == INTEL_MEMORY_STOLEN_SYSTEM ||
 			 mr->type == INTEL_MEMORY_STOLEN_LOCAL;
 
-	obj = i915_gem_object_create_region(mr, size, 0, 0);
+	obj = i915_gem_object_create_region(mr, size, 0, I915_BO_ALLOC_GPU_ONLY);
 	if (IS_ERR(obj)) {
 		/* if iGVT-g or DMAR is active, stolen mem will be uninitialized */
 		if (PTR_ERR(obj) == -ENODEV && is_stolen)
@@ -2324,5 +2324,5 @@ int i915_gem_gtt_live_selftests(struct drm_i915_private *i915)
 
 	GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total));
 
-	return i915_subtests(tests, i915);
+	return i915_live_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
index bdd290f2bf3c..aaf8a380e5c7 100644
--- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
@@ -49,5 +49,6 @@ selftest(perf, i915_perf_live_selftests)
 selftest(slpc, intel_slpc_live_selftests)
 selftest(guc, intel_guc_live_selftests)
 selftest(guc_multi_lrc, intel_guc_multi_lrc_live_selftests)
+selftest(guc_hang, intel_guc_hang_check)
 /* Here be dragons: keep last to run last! */
 selftest(late_gt_pm, intel_gt_pm_late_selftests)
diff --git a/drivers/gpu/drm/i915/selftests/i915_perf.c b/drivers/gpu/drm/i915/selftests/i915_perf.c
index 88db2e3d81d0..429c6d73b159 100644
--- a/drivers/gpu/drm/i915/selftests/i915_perf.c
+++ b/drivers/gpu/drm/i915/selftests/i915_perf.c
@@ -431,7 +431,7 @@ int i915_perf_live_selftests(struct drm_i915_private *i915)
 	if (err)
 		return err;
 
-	err = i915_subtests(tests, i915);
+	err = i915_live_subtests(tests, i915);
 
 	destroy_empty_config(&i915->perf);
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c
index c56a0c2cd2f7..818a4909c1f3 100644
--- a/drivers/gpu/drm/i915/selftests/i915_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_request.c
@@ -971,7 +971,7 @@ static struct i915_vma *empty_batch(struct drm_i915_private *i915)
 	if (err)
 		goto err;
 
-	/* Force the wait wait now to avoid including it in the benchmark */
+	/* Force the wait now to avoid including it in the benchmark */
 	err = i915_vma_sync(vma);
 	if (err)
 		goto err_pin;
@@ -1821,7 +1821,7 @@ int i915_request_live_selftests(struct drm_i915_private *i915)
 	if (intel_gt_is_wedged(to_gt(i915)))
 		return 0;
 
-	return i915_subtests(tests, i915);
+	return i915_live_subtests(tests, i915);
 }
 
 static int switch_to_kernel_sync(struct intel_context *ce, int err)
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index 6921ba128015..71b52d5efef4 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -51,9 +51,9 @@ static bool assert_vma(struct i915_vma *vma,
 		ok = false;
 	}
 
-	if (vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) {
+	if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) {
 		pr_err("VMA created with wrong type [%d]\n",
-		       vma->ggtt_view.type);
+		       vma->gtt_view.type);
 		ok = false;
 	}
 
@@ -63,7 +63,7 @@ static bool assert_vma(struct i915_vma *vma,
 static struct i915_vma *
 checked_vma_instance(struct drm_i915_gem_object *obj,
 		     struct i915_address_space *vm,
-		     const struct i915_ggtt_view *view)
+		     const struct i915_gtt_view *view)
 {
 	struct i915_vma *vma;
 	bool ok = true;
@@ -91,7 +91,7 @@ checked_vma_instance(struct drm_i915_gem_object *obj,
 	}
 
 	if (i915_vma_compare(vma, vma->vm,
-			     i915_vma_is_ggtt(vma) ? &vma->ggtt_view : NULL)) {
+			     i915_vma_is_ggtt(vma) ? &vma->gtt_view : NULL)) {
 		pr_err("i915_vma_compare failed with itself\n");
 		return ERR_PTR(-EINVAL);
 	}
@@ -530,12 +530,12 @@ assert_remapped(struct drm_i915_gem_object *obj,
 	return sg;
 }
 
-static unsigned int remapped_size(enum i915_ggtt_view_type view_type,
+static unsigned int remapped_size(enum i915_gtt_view_type view_type,
 				  const struct intel_remapped_plane_info *a,
 				  const struct intel_remapped_plane_info *b)
 {
 
-	if (view_type == I915_GGTT_VIEW_ROTATED)
+	if (view_type == I915_GTT_VIEW_ROTATED)
 		return a->dst_stride * a->width + b->dst_stride * b->width;
 	else
 		return a->dst_stride * a->height + b->dst_stride * b->height;
@@ -569,9 +569,9 @@ static int igt_vma_rotate_remap(void *arg)
 
 		{ }
 	}, *a, *b;
-	enum i915_ggtt_view_type types[] = {
-		I915_GGTT_VIEW_ROTATED,
-		I915_GGTT_VIEW_REMAPPED,
+	enum i915_gtt_view_type types[] = {
+		I915_GTT_VIEW_ROTATED,
+		I915_GTT_VIEW_REMAPPED,
 		0,
 	}, *t;
 	const unsigned int max_pages = 64;
@@ -588,7 +588,7 @@ static int igt_vma_rotate_remap(void *arg)
 	for (t = types; *t; t++) {
 	for (a = planes; a->width; a++) {
 		for (b = planes + ARRAY_SIZE(planes); b-- != planes; ) {
-			struct i915_ggtt_view view = {
+			struct i915_gtt_view view = {
 				.type = *t,
 				.remapped.plane[0] = *a,
 				.remapped.plane[1] = *b,
@@ -602,11 +602,11 @@ static int igt_vma_rotate_remap(void *arg)
 			max_offset = max_pages - max_offset;
 
 			if (!plane_info[0].dst_stride)
-				plane_info[0].dst_stride = view.type == I915_GGTT_VIEW_ROTATED ?
+				plane_info[0].dst_stride = view.type == I915_GTT_VIEW_ROTATED ?
 									plane_info[0].height :
 									plane_info[0].width;
 			if (!plane_info[1].dst_stride)
-				plane_info[1].dst_stride = view.type == I915_GGTT_VIEW_ROTATED ?
+				plane_info[1].dst_stride = view.type == I915_GTT_VIEW_ROTATED ?
 									plane_info[1].height :
 									plane_info[1].width;
 
@@ -630,7 +630,7 @@ static int igt_vma_rotate_remap(void *arg)
 
 					expected_pages = remapped_size(view.type, &plane_info[0], &plane_info[1]);
 
-					if (view.type == I915_GGTT_VIEW_ROTATED &&
+					if (view.type == I915_GTT_VIEW_ROTATED &&
 					    vma->size != expected_pages * PAGE_SIZE) {
 						pr_err("VMA is wrong size, expected %lu, found %llu\n",
 						       PAGE_SIZE * expected_pages, vma->size);
@@ -638,7 +638,7 @@ static int igt_vma_rotate_remap(void *arg)
 						goto out_object;
 					}
 
-					if (view.type == I915_GGTT_VIEW_REMAPPED &&
+					if (view.type == I915_GTT_VIEW_REMAPPED &&
 					    vma->size > expected_pages * PAGE_SIZE) {
 						pr_err("VMA is wrong size, expected %lu, found %llu\n",
 						       PAGE_SIZE * expected_pages, vma->size);
@@ -668,13 +668,13 @@ static int igt_vma_rotate_remap(void *arg)
 
 					sg = vma->pages->sgl;
 					for (n = 0; n < ARRAY_SIZE(view.rotated.plane); n++) {
-						if (view.type == I915_GGTT_VIEW_ROTATED)
+						if (view.type == I915_GTT_VIEW_ROTATED)
 							sg = assert_rotated(obj, &view.rotated, n, sg);
 						else
 							sg = assert_remapped(obj, &view.remapped, n, sg);
 						if (IS_ERR(sg)) {
 							pr_err("Inconsistent %s VMA pages for plane %d: [(%d, %d, %d, %d, %d), (%d, %d, %d, %d, %d)]\n",
-							       view.type == I915_GGTT_VIEW_ROTATED ?
+							       view.type == I915_GTT_VIEW_ROTATED ?
 							       "rotated" : "remapped", n,
 							       plane_info[0].width,
 							       plane_info[0].height,
@@ -741,7 +741,7 @@ static bool assert_partial(struct drm_i915_gem_object *obj,
 }
 
 static bool assert_pin(struct i915_vma *vma,
-		       struct i915_ggtt_view *view,
+		       struct i915_gtt_view *view,
 		       u64 size,
 		       const char *name)
 {
@@ -759,8 +759,8 @@ static bool assert_pin(struct i915_vma *vma,
 		ok = false;
 	}
 
-	if (view && view->type != I915_GGTT_VIEW_NORMAL) {
-		if (memcmp(&vma->ggtt_view, view, sizeof(*view))) {
+	if (view && view->type != I915_GTT_VIEW_NORMAL) {
+		if (memcmp(&vma->gtt_view, view, sizeof(*view))) {
 			pr_err("(%s) VMA mismatch upon creation!\n",
 			       name);
 			ok = false;
@@ -772,9 +772,9 @@ static bool assert_pin(struct i915_vma *vma,
 			ok = false;
 		}
 	} else {
-		if (vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) {
+		if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) {
 			pr_err("Not the normal ggtt view! Found %d\n",
-			       vma->ggtt_view.type);
+			       vma->gtt_view.type);
 			ok = false;
 		}
 
@@ -818,14 +818,14 @@ static int igt_vma_partial(void *arg)
 		nvma = 0;
 		for_each_prime_number_from(sz, 1, npages) {
 			for_each_prime_number_from(offset, 0, npages - sz) {
-				struct i915_ggtt_view view;
+				struct i915_gtt_view view;
 
-				view.type = I915_GGTT_VIEW_PARTIAL;
+				view.type = I915_GTT_VIEW_PARTIAL;
 				view.partial.offset = offset;
 				view.partial.size = sz;
 
 				if (sz == npages)
-					view.type = I915_GGTT_VIEW_NORMAL;
+					view.type = I915_GTT_VIEW_NORMAL;
 
 				vma = checked_vma_instance(obj, vm, &view);
 				if (IS_ERR(vma)) {
@@ -976,9 +976,9 @@ static int igt_vma_remapped_gtt(void *arg)
 
 		{ }
 	}, *p;
-	enum i915_ggtt_view_type types[] = {
-		I915_GGTT_VIEW_ROTATED,
-		I915_GGTT_VIEW_REMAPPED,
+	enum i915_gtt_view_type types[] = {
+		I915_GTT_VIEW_ROTATED,
+		I915_GTT_VIEW_REMAPPED,
 		0,
 	}, *t;
 	struct drm_i915_gem_object *obj;
@@ -996,7 +996,7 @@ static int igt_vma_remapped_gtt(void *arg)
 
 	for (t = types; *t; t++) {
 		for (p = planes; p->width; p++) {
-			struct i915_ggtt_view view = {
+			struct i915_gtt_view view = {
 				.type = *t,
 				.rotated.plane[0] = *p,
 			};
@@ -1012,7 +1012,7 @@ static int igt_vma_remapped_gtt(void *arg)
 				goto out;
 
 			if (!plane_info[0].dst_stride)
-				plane_info[0].dst_stride = *t == I915_GGTT_VIEW_ROTATED ?
+				plane_info[0].dst_stride = *t == I915_GTT_VIEW_ROTATED ?
 								 p->height : p->width;
 
 			vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
@@ -1021,7 +1021,7 @@ static int igt_vma_remapped_gtt(void *arg)
 				goto out;
 			}
 
-			GEM_BUG_ON(vma->ggtt_view.type != *t);
+			GEM_BUG_ON(vma->gtt_view.type != *t);
 
 			map = i915_vma_pin_iomap(vma);
 			i915_vma_unpin(vma);
@@ -1035,7 +1035,7 @@ static int igt_vma_remapped_gtt(void *arg)
 					unsigned int offset;
 					u32 val = y << 16 | x;
 
-					if (*t == I915_GGTT_VIEW_ROTATED)
+					if (*t == I915_GTT_VIEW_ROTATED)
 						offset = (x * plane_info[0].dst_stride + y) * PAGE_SIZE;
 					else
 						offset = (y * plane_info[0].dst_stride + x) * PAGE_SIZE;
@@ -1052,7 +1052,7 @@ static int igt_vma_remapped_gtt(void *arg)
 				goto out;
 			}
 
-			GEM_BUG_ON(vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL);
+			GEM_BUG_ON(vma->gtt_view.type != I915_GTT_VIEW_NORMAL);
 
 			map = i915_vma_pin_iomap(vma);
 			i915_vma_unpin(vma);
@@ -1067,7 +1067,7 @@ static int igt_vma_remapped_gtt(void *arg)
 					u32 exp = y << 16 | x;
 					u32 val;
 
-					if (*t == I915_GGTT_VIEW_ROTATED)
+					if (*t == I915_GTT_VIEW_ROTATED)
 						src_idx = rotated_index(&view.rotated, 0, x, y);
 					else
 						src_idx = remapped_index(&view.remapped, 0, x, y);
@@ -1076,7 +1076,7 @@ static int igt_vma_remapped_gtt(void *arg)
 					val = ioread32(&map[offset / sizeof(*map)]);
 					if (val != exp) {
 						pr_err("%s VMA write test failed, expected 0x%x, found 0x%x\n",
-						       *t == I915_GGTT_VIEW_ROTATED ? "Rotated" : "Remapped",
+						       *t == I915_GTT_VIEW_ROTATED ? "Rotated" : "Remapped",
 						       exp, val);
 						i915_vma_unpin_iomap(vma);
 						err = -EINVAL;
@@ -1103,5 +1103,5 @@ int i915_vma_live_selftests(struct drm_i915_private *i915)
 		SUBTEST(igt_vma_remapped_gtt),
 	};
 
-	return i915_subtests(tests, i915);
+	return i915_live_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 9c31a16f8380..fff11c90f1fa 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -115,6 +115,7 @@ static struct dev_pm_domain pm_domain = {
 static void mock_gt_probe(struct drm_i915_private *i915)
 {
 	i915->gt[0] = &i915->gt0;
+	i915->gt[0]->name = "Mock GT";
 }
 
 struct drm_i915_private *mock_gem_device(void)
@@ -172,14 +173,14 @@ struct drm_i915_private *mock_gem_device(void)
 	/* Using the global GTT may ask questions about KMS users, so prepare */
 	drm_mode_config_init(&i915->drm);
 
-	mkwrite_device_info(i915)->graphics.ver = -1;
+	RUNTIME_INFO(i915)->graphics.ip.ver = -1;
 
-	mkwrite_device_info(i915)->page_sizes =
+	RUNTIME_INFO(i915)->page_sizes =
 		I915_GTT_PAGE_SIZE_4K |
 		I915_GTT_PAGE_SIZE_64K |
 		I915_GTT_PAGE_SIZE_2M;
 
-	mkwrite_device_info(i915)->memory_regions = REGION_SMEM;
+	RUNTIME_INFO(i915)->memory_regions = REGION_SMEM;
 	intel_memory_regions_hw_probe(i915);
 
 	spin_lock_init(&i915->gpu_error.lock);
@@ -209,7 +210,7 @@ struct drm_i915_private *mock_gem_device(void)
 	mock_init_ggtt(to_gt(i915));
 	to_gt(i915)->vm = i915_vm_get(&to_gt(i915)->ggtt->vm);
 
-	mkwrite_device_info(i915)->platform_engine_mask = BIT(0);
+	RUNTIME_INFO(i915)->platform_engine_mask = BIT(0);
 	to_gt(i915)->info.engine_mask = BIT(0);
 
 	to_gt(i915)->engine[RCS0] = mock_engine(i915, "mock", RCS0);
diff --git a/drivers/gpu/drm/imx/Kconfig b/drivers/gpu/drm/imx/Kconfig
index bb9738c7c825..975de4ff7313 100644
--- a/drivers/gpu/drm/imx/Kconfig
+++ b/drivers/gpu/drm/imx/Kconfig
@@ -3,7 +3,7 @@ config DRM_IMX
 	tristate "DRM Support for Freescale i.MX"
 	select DRM_KMS_HELPER
 	select VIDEOMODE_HELPERS
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_KMS_HELPER
 	depends on DRM && (ARCH_MXC || ARCH_MULTIPLATFORM || COMPILE_TEST)
 	depends on IMX_IPUV3_CORE
diff --git a/drivers/gpu/drm/imx/dcss/Kconfig b/drivers/gpu/drm/imx/dcss/Kconfig
index 5c2b2277afbf..3ffc061d392b 100644
--- a/drivers/gpu/drm/imx/dcss/Kconfig
+++ b/drivers/gpu/drm/imx/dcss/Kconfig
@@ -2,7 +2,7 @@ config DRM_IMX_DCSS
 	tristate "i.MX8MQ DCSS"
 	select IMX_IRQSTEER
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select VIDEOMODE_HELPERS
 	depends on DRM && ARCH_MXC && ARM64
 	help
diff --git a/drivers/gpu/drm/imx/dcss/dcss-kms.c b/drivers/gpu/drm/imx/dcss/dcss-kms.c
index 8cf3352d8858..b4f82ebca532 100644
--- a/drivers/gpu/drm/imx/dcss/dcss-kms.c
+++ b/drivers/gpu/drm/imx/dcss/dcss-kms.c
@@ -8,7 +8,7 @@
 #include <drm/drm_bridge_connector.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_of.h>
 #include <drm/drm_probe_helper.h>
@@ -17,7 +17,7 @@
 #include "dcss-dev.h"
 #include "dcss-kms.h"
 
-DEFINE_DRM_GEM_CMA_FOPS(dcss_cma_fops);
+DEFINE_DRM_GEM_DMA_FOPS(dcss_cma_fops);
 
 static const struct drm_mode_config_funcs dcss_drm_mode_config_funcs = {
 	.fb_create = drm_gem_fb_create,
@@ -28,7 +28,7 @@ static const struct drm_mode_config_funcs dcss_drm_mode_config_funcs = {
 
 static const struct drm_driver dcss_kms_driver = {
 	.driver_features	= DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 	.fops			= &dcss_cma_fops,
 	.name			= "imx-dcss",
 	.desc			= "i.MX8MQ Display Subsystem",
diff --git a/drivers/gpu/drm/imx/dcss/dcss-plane.c b/drivers/gpu/drm/imx/dcss/dcss-plane.c
index c29f343f33e5..ab6d32bad756 100644
--- a/drivers/gpu/drm/imx/dcss/dcss-plane.c
+++ b/drivers/gpu/drm/imx/dcss/dcss-plane.c
@@ -6,10 +6,10 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 
 #include "dcss-dev.h"
 #include "dcss-kms.h"
@@ -147,7 +147,7 @@ static int dcss_plane_atomic_check(struct drm_plane *plane,
 	struct dcss_dev *dcss = plane->dev->dev_private;
 	struct drm_framebuffer *fb = new_plane_state->fb;
 	bool is_primary_plane = plane->type == DRM_PLANE_TYPE_PRIMARY;
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	struct drm_crtc_state *crtc_state;
 	int hdisplay, vdisplay;
 	int min, max;
@@ -156,8 +156,8 @@ static int dcss_plane_atomic_check(struct drm_plane *plane,
 	if (!fb || !new_plane_state->crtc)
 		return 0;
 
-	cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
-	WARN_ON(!cma_obj);
+	dma_obj = drm_fb_dma_get_gem_obj(fb, 0);
+	WARN_ON(!dma_obj);
 
 	crtc_state = drm_atomic_get_existing_crtc_state(state,
 							new_plane_state->crtc);
@@ -218,26 +218,26 @@ static void dcss_plane_atomic_set_base(struct dcss_plane *dcss_plane)
 	struct dcss_dev *dcss = plane->dev->dev_private;
 	struct drm_framebuffer *fb = state->fb;
 	const struct drm_format_info *format = fb->format;
-	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+	struct drm_gem_dma_object *dma_obj = drm_fb_dma_get_gem_obj(fb, 0);
 	unsigned long p1_ba = 0, p2_ba = 0;
 
 	if (!format->is_yuv ||
 	    format->format == DRM_FORMAT_NV12 ||
 	    format->format == DRM_FORMAT_NV21)
-		p1_ba = cma_obj->paddr + fb->offsets[0] +
+		p1_ba = dma_obj->dma_addr + fb->offsets[0] +
 			fb->pitches[0] * (state->src.y1 >> 16) +
 			format->char_per_block[0] * (state->src.x1 >> 16);
 	else if (format->format == DRM_FORMAT_UYVY ||
 		 format->format == DRM_FORMAT_VYUY ||
 		 format->format == DRM_FORMAT_YUYV ||
 		 format->format == DRM_FORMAT_YVYU)
-		p1_ba = cma_obj->paddr + fb->offsets[0] +
+		p1_ba = dma_obj->dma_addr + fb->offsets[0] +
 			fb->pitches[0] * (state->src.y1 >> 16) +
 			2 * format->char_per_block[0] * (state->src.x1 >> 17);
 
 	if (format->format == DRM_FORMAT_NV12 ||
 	    format->format == DRM_FORMAT_NV21)
-		p2_ba = cma_obj->paddr + fb->offsets[1] +
+		p2_ba = dma_obj->dma_addr + fb->offsets[1] +
 			(((fb->pitches[1] >> 1) * (state->src.y1 >> 17) +
 			(state->src.x1 >> 17)) << 1);
 
diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c
index a57812ec36b1..8dd8b0f912af 100644
--- a/drivers/gpu/drm/imx/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/imx-drm-core.c
@@ -16,13 +16,11 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_of.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
@@ -34,7 +32,7 @@
 static int legacyfb_depth = 16;
 module_param(legacyfb_depth, int, 0444);
 
-DEFINE_DRM_GEM_CMA_FOPS(imx_drm_driver_fops);
+DEFINE_DRM_GEM_DMA_FOPS(imx_drm_driver_fops);
 
 void imx_drm_connector_destroy(struct drm_connector *connector)
 {
@@ -154,7 +152,7 @@ static int imx_drm_dumb_create(struct drm_file *file_priv,
 
 	args->width = ALIGN(width, 8);
 
-	ret = drm_gem_cma_dumb_create(file_priv, drm, args);
+	ret = drm_gem_dma_dumb_create(file_priv, drm, args);
 	if (ret)
 		return ret;
 
@@ -164,7 +162,7 @@ static int imx_drm_dumb_create(struct drm_file *file_priv,
 
 static const struct drm_driver imx_drm_driver = {
 	.driver_features	= DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
-	DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(imx_drm_dumb_create),
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(imx_drm_dumb_create),
 	.ioctls			= imx_drm_ioctls,
 	.num_ioctls		= ARRAY_SIZE(imx_drm_ioctls),
 	.fops			= &imx_drm_driver_fops,
diff --git a/drivers/gpu/drm/imx/imx-drm.h b/drivers/gpu/drm/imx/imx-drm.h
index c3e1a3f14d30..e721bebda2bd 100644
--- a/drivers/gpu/drm/imx/imx-drm.h
+++ b/drivers/gpu/drm/imx/imx-drm.h
@@ -32,7 +32,7 @@ extern struct platform_driver ipu_drm_driver;
 
 void imx_drm_mode_config_init(struct drm_device *drm);
 
-struct drm_gem_cma_object *imx_drm_fb_get_obj(struct drm_framebuffer *fb);
+struct drm_gem_dma_object *imx_drm_fb_get_obj(struct drm_framebuffer *fb);
 
 int imx_drm_encoder_parse_of(struct drm_device *drm,
 	struct drm_encoder *encoder, struct device_node *np);
diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c
index f7863d6dea80..5f26090b0c98 100644
--- a/drivers/gpu/drm/imx/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3-crtc.c
@@ -18,8 +18,7 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_fb_cma_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c
index ea5f594955df..dba4f7d81d69 100644
--- a/drivers/gpu/drm/imx/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3-plane.c
@@ -8,13 +8,12 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
-#include <drm/drm_plane_helper.h>
 
 #include <video/imx-ipu-v3.h>
 
@@ -126,14 +125,14 @@ static inline unsigned long
 drm_plane_state_to_eba(struct drm_plane_state *state, int plane)
 {
 	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	int x = state->src.x1 >> 16;
 	int y = state->src.y1 >> 16;
 
-	cma_obj = drm_fb_cma_get_gem_obj(fb, plane);
-	BUG_ON(!cma_obj);
+	dma_obj = drm_fb_dma_get_gem_obj(fb, plane);
+	BUG_ON(!dma_obj);
 
-	return cma_obj->paddr + fb->offsets[plane] + fb->pitches[plane] * y +
+	return dma_obj->dma_addr + fb->offsets[plane] + fb->pitches[plane] * y +
 	       fb->format->cpp[plane] * x;
 }
 
@@ -141,18 +140,18 @@ static inline unsigned long
 drm_plane_state_to_ubo(struct drm_plane_state *state)
 {
 	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	unsigned long eba = drm_plane_state_to_eba(state, 0);
 	int x = state->src.x1 >> 16;
 	int y = state->src.y1 >> 16;
 
-	cma_obj = drm_fb_cma_get_gem_obj(fb, 1);
-	BUG_ON(!cma_obj);
+	dma_obj = drm_fb_dma_get_gem_obj(fb, 1);
+	BUG_ON(!dma_obj);
 
 	x /= fb->format->hsub;
 	y /= fb->format->vsub;
 
-	return cma_obj->paddr + fb->offsets[1] + fb->pitches[1] * y +
+	return dma_obj->dma_addr + fb->offsets[1] + fb->pitches[1] * y +
 	       fb->format->cpp[1] * x - eba;
 }
 
@@ -160,18 +159,18 @@ static inline unsigned long
 drm_plane_state_to_vbo(struct drm_plane_state *state)
 {
 	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	unsigned long eba = drm_plane_state_to_eba(state, 0);
 	int x = state->src.x1 >> 16;
 	int y = state->src.y1 >> 16;
 
-	cma_obj = drm_fb_cma_get_gem_obj(fb, 2);
-	BUG_ON(!cma_obj);
+	dma_obj = drm_fb_dma_get_gem_obj(fb, 2);
+	BUG_ON(!dma_obj);
 
 	x /= fb->format->hsub;
 	y /= fb->format->vsub;
 
-	return cma_obj->paddr + fb->offsets[2] + fb->pitches[2] * y +
+	return dma_obj->dma_addr + fb->offsets[2] + fb->pitches[2] * y +
 	       fb->format->cpp[2] * x - eba;
 }
 
@@ -393,8 +392,8 @@ static int ipu_plane_atomic_check(struct drm_plane *plane,
 		return -EINVAL;
 
 	ret = drm_atomic_helper_check_plane_state(new_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  can_position, true);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/ingenic/Kconfig b/drivers/gpu/drm/ingenic/Kconfig
index 090830bcbde7..a53f475d33df 100644
--- a/drivers/gpu/drm/ingenic/Kconfig
+++ b/drivers/gpu/drm/ingenic/Kconfig
@@ -8,7 +8,7 @@ config DRM_INGENIC
 	select DRM_BRIDGE
 	select DRM_PANEL_BRIDGE
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select VT_HW_CONSOLE_BINDING if FRAMEBUFFER_CONSOLE
 	help
 	  Choose this option for DRM support for the Ingenic SoCs.
diff --git a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
index eb8208bfe5ab..ab0515d2c420 100644
--- a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
+++ b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
@@ -30,8 +30,8 @@
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_encoder.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
@@ -41,7 +41,6 @@
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_plane.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
@@ -482,8 +481,8 @@ static int ingenic_drm_plane_atomic_check(struct drm_plane *plane,
 		return PTR_ERR(priv_state);
 
 	ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  priv->soc_info->has_osd,
 						  true);
 	if (ret)
@@ -670,12 +669,12 @@ static void ingenic_drm_plane_atomic_update(struct drm_plane *plane,
 
 	if (newstate && newstate->fb) {
 		if (priv->soc_info->map_noncoherent)
-			drm_fb_cma_sync_non_coherent(&priv->drm, oldstate, newstate);
+			drm_fb_dma_sync_non_coherent(&priv->drm, oldstate, newstate);
 
 		crtc_state = newstate->crtc->state;
 		plane_id = !!(priv->soc_info->has_osd && plane != &priv->f0);
 
-		addr = drm_fb_cma_get_gem_addr(newstate->fb, newstate, 0);
+		addr = drm_fb_dma_get_gem_addr(newstate->fb, newstate, 0);
 		width = newstate->src_w >> 16;
 		height = newstate->src_h >> 16;
 		cpp = newstate->fb->format->cpp[0];
@@ -915,7 +914,7 @@ static struct drm_gem_object *
 ingenic_drm_gem_create_object(struct drm_device *drm, size_t size)
 {
 	struct ingenic_drm *priv = drm_device_get_priv(drm);
-	struct drm_gem_cma_object *obj;
+	struct drm_gem_dma_object *obj;
 
 	obj = kzalloc(sizeof(*obj), GFP_KERNEL);
 	if (!obj)
@@ -948,7 +947,7 @@ static void ingenic_drm_destroy_state(struct drm_private_obj *obj,
 	kfree(priv_state);
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(ingenic_drm_fops);
+DEFINE_DRM_GEM_DMA_FOPS(ingenic_drm_fops);
 
 static const struct drm_driver ingenic_drm_driver_data = {
 	.driver_features	= DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
@@ -961,7 +960,7 @@ static const struct drm_driver ingenic_drm_driver_data = {
 
 	.fops			= &ingenic_drm_fops,
 	.gem_create_object	= ingenic_drm_gem_create_object,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 };
 
 static const struct drm_plane_funcs ingenic_drm_primary_plane_funcs = {
@@ -1464,21 +1463,22 @@ static int ingenic_drm_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static int __maybe_unused ingenic_drm_suspend(struct device *dev)
+static int ingenic_drm_suspend(struct device *dev)
 {
 	struct ingenic_drm *priv = dev_get_drvdata(dev);
 
 	return drm_mode_config_helper_suspend(&priv->drm);
 }
 
-static int __maybe_unused ingenic_drm_resume(struct device *dev)
+static int ingenic_drm_resume(struct device *dev)
 {
 	struct ingenic_drm *priv = dev_get_drvdata(dev);
 
 	return drm_mode_config_helper_resume(&priv->drm);
 }
 
-static SIMPLE_DEV_PM_OPS(ingenic_drm_pm_ops, ingenic_drm_suspend, ingenic_drm_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(ingenic_drm_pm_ops,
+				ingenic_drm_suspend, ingenic_drm_resume);
 
 static const u32 jz4740_formats[] = {
 	DRM_FORMAT_XRGB1555,
@@ -1541,6 +1541,32 @@ static const struct jz_soc_info jz4725b_soc_info = {
 	.num_formats_f0 = ARRAY_SIZE(jz4725b_formats_f0),
 };
 
+static const struct jz_soc_info jz4760_soc_info = {
+	.needs_dev_clk = false,
+	.has_osd = true,
+	.map_noncoherent = false,
+	.max_width = 1280,
+	.max_height = 720,
+	.max_burst = JZ_LCD_CTRL_BURST_32,
+	.formats_f1 = jz4770_formats_f1,
+	.num_formats_f1 = ARRAY_SIZE(jz4770_formats_f1),
+	.formats_f0 = jz4770_formats_f0,
+	.num_formats_f0 = ARRAY_SIZE(jz4770_formats_f0),
+};
+
+static const struct jz_soc_info jz4760b_soc_info = {
+	.needs_dev_clk = false,
+	.has_osd = true,
+	.map_noncoherent = false,
+	.max_width = 1280,
+	.max_height = 720,
+	.max_burst = JZ_LCD_CTRL_BURST_64,
+	.formats_f1 = jz4770_formats_f1,
+	.num_formats_f1 = ARRAY_SIZE(jz4770_formats_f1),
+	.formats_f0 = jz4770_formats_f0,
+	.num_formats_f0 = ARRAY_SIZE(jz4770_formats_f0),
+};
+
 static const struct jz_soc_info jz4770_soc_info = {
 	.needs_dev_clk = false,
 	.has_osd = true,
@@ -1572,6 +1598,8 @@ static const struct jz_soc_info jz4780_soc_info = {
 static const struct of_device_id ingenic_drm_of_match[] = {
 	{ .compatible = "ingenic,jz4740-lcd", .data = &jz4740_soc_info },
 	{ .compatible = "ingenic,jz4725b-lcd", .data = &jz4725b_soc_info },
+	{ .compatible = "ingenic,jz4760-lcd", .data = &jz4760_soc_info },
+	{ .compatible = "ingenic,jz4760b-lcd", .data = &jz4760b_soc_info },
 	{ .compatible = "ingenic,jz4770-lcd", .data = &jz4770_soc_info },
 	{ .compatible = "ingenic,jz4780-lcd", .data = &jz4780_soc_info },
 	{ /* sentinel */ },
@@ -1581,7 +1609,7 @@ MODULE_DEVICE_TABLE(of, ingenic_drm_of_match);
 static struct platform_driver ingenic_drm_driver = {
 	.driver = {
 		.name = "ingenic-drm",
-		.pm = pm_ptr(&ingenic_drm_pm_ops),
+		.pm = pm_sleep_ptr(&ingenic_drm_pm_ops),
 		.of_match_table = of_match_ptr(ingenic_drm_of_match),
 	},
 	.probe = ingenic_drm_probe,
@@ -1616,4 +1644,4 @@ module_exit(ingenic_drm_exit);
 
 MODULE_AUTHOR("Paul Cercueil <paul@crapouillou.net>");
 MODULE_DESCRIPTION("DRM driver for the Ingenic SoCs\n");
-MODULE_LICENSE("GPL v2");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/ingenic/ingenic-ipu.c b/drivers/gpu/drm/ingenic/ingenic-ipu.c
index 32a50935aa6d..7a43505011a5 100644
--- a/drivers/gpu/drm/ingenic/ingenic-ipu.c
+++ b/drivers/gpu/drm/ingenic/ingenic-ipu.c
@@ -22,14 +22,13 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_plane.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_property.h>
 #include <drm/drm_vblank.h>
 
@@ -363,15 +362,15 @@ static void ingenic_ipu_plane_atomic_update(struct drm_plane *plane,
 	}
 
 	if (ingenic_drm_map_noncoherent(ipu->master))
-		drm_fb_cma_sync_non_coherent(ipu->drm, oldstate, newstate);
+		drm_fb_dma_sync_non_coherent(ipu->drm, oldstate, newstate);
 
 	/* New addresses will be committed in vblank handler... */
-	ipu->addr_y = drm_fb_cma_get_gem_addr(newstate->fb, newstate, 0);
+	ipu->addr_y = drm_fb_dma_get_gem_addr(newstate->fb, newstate, 0);
 	if (finfo->num_planes > 1)
-		ipu->addr_u = drm_fb_cma_get_gem_addr(newstate->fb, newstate,
+		ipu->addr_u = drm_fb_dma_get_gem_addr(newstate->fb, newstate,
 						      1);
 	if (finfo->num_planes > 2)
-		ipu->addr_v = drm_fb_cma_get_gem_addr(newstate->fb, newstate,
+		ipu->addr_v = drm_fb_dma_get_gem_addr(newstate->fb, newstate,
 						      2);
 
 	if (!needs_modeset)
@@ -697,10 +696,12 @@ ingenic_ipu_plane_atomic_set_property(struct drm_plane *plane,
 {
 	struct ingenic_ipu *ipu = plane_to_ingenic_ipu(plane);
 	struct drm_crtc_state *crtc_state;
+	bool mode_changed;
 
 	if (property != ipu->sharpness_prop)
 		return -EINVAL;
 
+	mode_changed = val != ipu->sharpness;
 	ipu->sharpness = val;
 
 	if (state->crtc) {
@@ -708,7 +709,7 @@ ingenic_ipu_plane_atomic_set_property(struct drm_plane *plane,
 		if (WARN_ON(!crtc_state))
 			return -EINVAL;
 
-		crtc_state->mode_changed = true;
+		crtc_state->mode_changed |= mode_changed;
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/kmb/Kconfig b/drivers/gpu/drm/kmb/Kconfig
index 5fdd43dad507..fd011367db1d 100644
--- a/drivers/gpu/drm/kmb/Kconfig
+++ b/drivers/gpu/drm/kmb/Kconfig
@@ -3,7 +3,7 @@ config DRM_KMB_DISPLAY
 	depends on DRM
 	depends on ARCH_KEEMBAY || COMPILE_TEST
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_MIPI_DSI
 	help
 	Choose this option if you have Intel's KeemBay SOC which integrates
diff --git a/drivers/gpu/drm/kmb/kmb_drv.c b/drivers/gpu/drm/kmb/kmb_drv.c
index 76fef0880504..2382ccb3ee99 100644
--- a/drivers/gpu/drm/kmb/kmb_drv.c
+++ b/drivers/gpu/drm/kmb/kmb_drv.c
@@ -16,7 +16,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_probe_helper.h>
@@ -433,14 +433,14 @@ static void kmb_irq_uninstall(struct drm_device *drm)
 	free_irq(kmb->irq_lcd, drm);
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(fops);
+DEFINE_DRM_GEM_DMA_FOPS(fops);
 
 static const struct drm_driver kmb_driver = {
 	.driver_features = DRIVER_GEM |
 	    DRIVER_MODESET | DRIVER_ATOMIC,
 	/* GEM Operations */
 	.fops = &fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.name = "kmb-drm",
 	.desc = "KEEMBAY DISPLAY DRIVER",
 	.date = DRIVER_DATE,
diff --git a/drivers/gpu/drm/kmb/kmb_plane.c b/drivers/gpu/drm/kmb/kmb_plane.c
index 89d055a089a6..a42f63f6f957 100644
--- a/drivers/gpu/drm/kmb/kmb_plane.c
+++ b/drivers/gpu/drm/kmb/kmb_plane.c
@@ -8,13 +8,12 @@
 #include <drm/drm_blend.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
-#include <drm/drm_plane_helper.h>
 
 #include "kmb_drv.h"
 #include "kmb_plane.h"
@@ -136,8 +135,8 @@ static int kmb_plane_atomic_check(struct drm_plane *plane,
 						   new_plane_state->crtc);
 	return drm_atomic_helper_check_plane_state(new_plane_state,
 						   crtc_state,
-						   DRM_PLANE_HELPER_NO_SCALING,
-						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
 						   can_position, true);
 }
 
@@ -404,7 +403,7 @@ static void kmb_plane_atomic_update(struct drm_plane *plane,
 	kmb_write_lcd(kmb, LCD_LAYERn_DMA_LINE_WIDTH(plane_id),
 		      (width * fb->format->cpp[0]));
 
-	addr[Y_PLANE] = drm_fb_cma_get_gem_addr(fb, new_plane_state, 0);
+	addr[Y_PLANE] = drm_fb_dma_get_gem_addr(fb, new_plane_state, 0);
 	kmb_write_lcd(kmb, LCD_LAYERn_DMA_START_ADDR(plane_id),
 		      addr[Y_PLANE] + fb->offsets[0]);
 	val = get_pixel_format(fb->format->format);
@@ -416,7 +415,7 @@ static void kmb_plane_atomic_update(struct drm_plane *plane,
 		kmb_write_lcd(kmb, LCD_LAYERn_DMA_CB_LINE_WIDTH(plane_id),
 			      (width * fb->format->cpp[0]));
 
-		addr[U_PLANE] = drm_fb_cma_get_gem_addr(fb, new_plane_state,
+		addr[U_PLANE] = drm_fb_dma_get_gem_addr(fb, new_plane_state,
 							U_PLANE);
 		/* check if Cb/Cr is swapped*/
 		if (num_planes == 3 && (val & LCD_LAYER_CRCB_ORDER))
@@ -437,7 +436,7 @@ static void kmb_plane_atomic_update(struct drm_plane *plane,
 				      LCD_LAYERn_DMA_CR_LINE_WIDTH(plane_id),
 				      ((width) * fb->format->cpp[0]));
 
-			addr[V_PLANE] = drm_fb_cma_get_gem_addr(fb,
+			addr[V_PLANE] = drm_fb_dma_get_gem_addr(fb,
 								new_plane_state,
 								V_PLANE);
 
diff --git a/drivers/gpu/drm/logicvc/Kconfig b/drivers/gpu/drm/logicvc/Kconfig
index 300b2be07385..fa7a88368809 100644
--- a/drivers/gpu/drm/logicvc/Kconfig
+++ b/drivers/gpu/drm/logicvc/Kconfig
@@ -3,7 +3,7 @@ config DRM_LOGICVC
 	depends on DRM
 	depends on OF || COMPILE_TEST
 	select DRM_KMS_HELPER
-	select DRM_KMS_CMA_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_KMS_DMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	help
 	  DRM display driver for the logiCVC programmable logic block from Xylon
diff --git a/drivers/gpu/drm/logicvc/logicvc_crtc.c b/drivers/gpu/drm/logicvc/logicvc_crtc.c
index c94bb9bb456b..43a675d03808 100644
--- a/drivers/gpu/drm/logicvc/logicvc_crtc.c
+++ b/drivers/gpu/drm/logicvc/logicvc_crtc.c
@@ -12,7 +12,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
 
diff --git a/drivers/gpu/drm/logicvc/logicvc_drm.c b/drivers/gpu/drm/logicvc/logicvc_drm.c
index 65a050176c33..cc9a4e965f77 100644
--- a/drivers/gpu/drm/logicvc/logicvc_drm.c
+++ b/drivers/gpu/drm/logicvc/logicvc_drm.c
@@ -18,7 +18,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_print.h>
 
 #include "logicvc_crtc.h"
@@ -29,9 +29,9 @@
 #include "logicvc_of.h"
 #include "logicvc_regs.h"
 
-DEFINE_DRM_GEM_CMA_FOPS(logicvc_drm_fops);
+DEFINE_DRM_GEM_DMA_FOPS(logicvc_drm_fops);
 
-static int logicvc_drm_gem_cma_dumb_create(struct drm_file *file_priv,
+static int logicvc_drm_gem_dma_dumb_create(struct drm_file *file_priv,
 					   struct drm_device *drm_dev,
 					   struct drm_mode_create_dumb *args)
 {
@@ -40,7 +40,7 @@ static int logicvc_drm_gem_cma_dumb_create(struct drm_file *file_priv,
 	/* Stride is always fixed to its configuration value. */
 	args->pitch = logicvc->config.row_stride * DIV_ROUND_UP(args->bpp, 8);
 
-	return drm_gem_cma_dumb_create_internal(file_priv, drm_dev, args);
+	return drm_gem_dma_dumb_create_internal(file_priv, drm_dev, args);
 }
 
 static struct drm_driver logicvc_drm_driver = {
@@ -54,7 +54,7 @@ static struct drm_driver logicvc_drm_driver = {
 	.major				= 1,
 	.minor				= 0,
 
-	DRM_GEM_CMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE(logicvc_drm_gem_cma_dumb_create),
+	DRM_GEM_DMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE(logicvc_drm_gem_dma_dumb_create),
 };
 
 static struct regmap_config logicvc_drm_regmap_config = {
diff --git a/drivers/gpu/drm/logicvc/logicvc_interface.c b/drivers/gpu/drm/logicvc/logicvc_interface.c
index c73592f6c406..815cebb4c4ca 100644
--- a/drivers/gpu/drm/logicvc/logicvc_interface.c
+++ b/drivers/gpu/drm/logicvc/logicvc_interface.c
@@ -12,7 +12,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_encoder.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
diff --git a/drivers/gpu/drm/logicvc/logicvc_layer.c b/drivers/gpu/drm/logicvc/logicvc_layer.c
index 441e3cfce4cf..464000aea765 100644
--- a/drivers/gpu/drm/logicvc/logicvc_layer.c
+++ b/drivers/gpu/drm/logicvc/logicvc_layer.c
@@ -10,11 +10,10 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_plane.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 
 #include "logicvc_crtc.h"
@@ -117,8 +116,8 @@ static int logicvc_plane_atomic_check(struct drm_plane *drm_plane,
 		}
 	}
 
-	min_scale = DRM_PLANE_HELPER_NO_SCALING;
-	max_scale = DRM_PLANE_HELPER_NO_SCALING;
+	min_scale = DRM_PLANE_NO_SCALING;
+	max_scale = DRM_PLANE_NO_SCALING;
 
 	can_position = (drm_plane->type == DRM_PLANE_TYPE_OVERLAY &&
 			layer->index != (logicvc->config.layers_count - 1) &&
@@ -158,7 +157,7 @@ static void logicvc_plane_atomic_update(struct drm_plane *drm_plane,
 		     new_state->crtc_h - 1);
 
 	if (logicvc->caps->layer_address) {
-		phys_addr_t fb_addr = drm_fb_cma_get_gem_addr(fb, new_state, 0);
+		phys_addr_t fb_addr = drm_fb_dma_get_gem_addr(fb, new_state, 0);
 
 		regmap_write(logicvc->regmap, LOGICVC_LAYER_ADDRESS_REG(index),
 			     fb_addr);
@@ -281,7 +280,7 @@ int logicvc_layer_buffer_find_setup(struct logicvc_drm *logicvc,
 		return -ENOMEM;
 	}
 
-	fb_addr = drm_fb_cma_get_gem_addr(fb, state, 0);
+	fb_addr = drm_fb_dma_get_gem_addr(fb, state, 0);
 	if (fb_addr < logicvc->reserved_mem_base) {
 		drm_err(drm_dev,
 			"Framebuffer memory below reserved memory base!\n");
diff --git a/drivers/gpu/drm/logicvc/logicvc_mode.c b/drivers/gpu/drm/logicvc/logicvc_mode.c
index 11940704f644..d8207ffda1af 100644
--- a/drivers/gpu/drm/logicvc/logicvc_mode.c
+++ b/drivers/gpu/drm/logicvc/logicvc_mode.c
@@ -10,9 +10,8 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_mode_config.h>
 #include <drm/drm_panel.h>
diff --git a/drivers/gpu/drm/mcde/Kconfig b/drivers/gpu/drm/mcde/Kconfig
index d0bf1bc8da3f..4f3d68e11bc1 100644
--- a/drivers/gpu/drm/mcde/Kconfig
+++ b/drivers/gpu/drm/mcde/Kconfig
@@ -10,7 +10,7 @@ config DRM_MCDE
 	select DRM_BRIDGE
 	select DRM_PANEL_BRIDGE
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select VT_HW_CONSOLE_BINDING if FRAMEBUFFER_CONSOLE
 	help
 	  Choose this option for DRM support for the ST-Ericsson MCDE
diff --git a/drivers/gpu/drm/mcde/mcde_display.c b/drivers/gpu/drm/mcde/mcde_display.c
index 4df477540d07..52043a12a2e8 100644
--- a/drivers/gpu/drm/mcde/mcde_display.c
+++ b/drivers/gpu/drm/mcde/mcde_display.c
@@ -11,11 +11,11 @@
 #include <linux/media-bus-format.h>
 
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_bridge.h>
@@ -165,7 +165,7 @@ static int mcde_display_check(struct drm_simple_display_pipe *pipe,
 	struct drm_framebuffer *fb = pstate->fb;
 
 	if (fb) {
-		u32 offset = drm_fb_cma_get_gem_addr(fb, pstate, 0);
+		u32 offset = drm_fb_dma_get_gem_addr(fb, pstate, 0);
 
 		/* FB base address must be dword aligned. */
 		if (offset & 3) {
@@ -1424,7 +1424,7 @@ static void mcde_display_update(struct drm_simple_display_pipe *pipe,
 	 * from the DRM core before the display is enabled.
 	 */
 	if (fb) {
-		mcde_set_extsrc(mcde, drm_fb_cma_get_gem_addr(fb, pstate, 0));
+		mcde_set_extsrc(mcde, drm_fb_dma_get_gem_addr(fb, pstate, 0));
 		dev_info_once(mcde->dev, "first update of display contents\n");
 		/*
 		 * Usually the flow is already active, unless we are in
diff --git a/drivers/gpu/drm/mcde/mcde_drv.c b/drivers/gpu/drm/mcde/mcde_drv.c
index e601baa87e55..1c4482ad507d 100644
--- a/drivers/gpu/drm/mcde/mcde_drv.c
+++ b/drivers/gpu/drm/mcde/mcde_drv.c
@@ -37,7 +37,7 @@
  * (effectively using channels 0..3) for concurrent use.
  *
  * In the current DRM/KMS setup, we use one external source, one overlay,
- * one FIFO and one formatter which we connect to the simple CMA framebuffer
+ * one FIFO and one formatter which we connect to the simple DMA framebuffer
  * helpers. We then provide a bridge to the DSI port, and on the DSI port
  * bridge we connect hang a panel bridge or other bridge. This may be subject
  * to change as we exploit more of the hardware capabilities.
@@ -68,10 +68,10 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_of.h>
@@ -198,7 +198,7 @@ static int mcde_modeset_init(struct drm_device *drm)
 	return 0;
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(drm_fops);
+DEFINE_DRM_GEM_DMA_FOPS(drm_fops);
 
 static const struct drm_driver mcde_drm_driver = {
 	.driver_features =
@@ -212,7 +212,7 @@ static const struct drm_driver mcde_drm_driver = {
 	.major = 1,
 	.minor = 0,
 	.patchlevel = 0,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 };
 
 static int mcde_drm_bind(struct device *dev)
diff --git a/drivers/gpu/drm/mediatek/Kconfig b/drivers/gpu/drm/mediatek/Kconfig
index 2976d21e9a34..369e495d0c3e 100644
--- a/drivers/gpu/drm/mediatek/Kconfig
+++ b/drivers/gpu/drm/mediatek/Kconfig
@@ -7,7 +7,7 @@ config DRM_MEDIATEK
 	depends on HAVE_ARM_SMCCC
 	depends on OF
 	depends on MTK_MMSYS
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
 	select DRM_PANEL
@@ -21,6 +21,15 @@ config DRM_MEDIATEK
 	  This driver provides kernel mode setting and
 	  buffer management to userspace.
 
+config DRM_MEDIATEK_DP
+	tristate "DRM DPTX Support for MediaTek SoCs"
+	depends on DRM_MEDIATEK
+	select PHY_MTK_DP
+	select DRM_DISPLAY_HELPER
+	select DRM_DISPLAY_DP_HELPER
+	help
+	  DRM/KMS Display Port driver for MediaTek SoCs.
+
 config DRM_MEDIATEK_HDMI
 	tristate "DRM HDMI Support for Mediatek SoCs"
 	depends on DRM_MEDIATEK
diff --git a/drivers/gpu/drm/mediatek/Makefile b/drivers/gpu/drm/mediatek/Makefile
index 6e604a933ed0..3517d1c65cd7 100644
--- a/drivers/gpu/drm/mediatek/Makefile
+++ b/drivers/gpu/drm/mediatek/Makefile
@@ -23,3 +23,5 @@ mediatek-drm-hdmi-objs := mtk_cec.o \
 			  mtk_hdmi_ddc.o
 
 obj-$(CONFIG_DRM_MEDIATEK_HDMI) += mediatek-drm-hdmi.o
+
+obj-$(CONFIG_DRM_MEDIATEK_DP) += mtk_dp.o
diff --git a/drivers/gpu/drm/mediatek/mtk_dp.c b/drivers/gpu/drm/mediatek/mtk_dp.c
new file mode 100644
index 000000000000..9d085c05c49c
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_dp.c
@@ -0,0 +1,2663 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2019-2022 MediaTek Inc.
+ * Copyright (c) 2022 BayLibre
+ */
+
+#include <drm/display/drm_dp.h>
+#include <drm/display/drm_dp_helper.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_print.h>
+#include <drm/drm_probe_helper.h>
+#include <linux/arm-smccc.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/media-bus-format.h>
+#include <linux/nvmem-consumer.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/soc/mediatek/mtk_sip_svc.h>
+#include <sound/hdmi-codec.h>
+#include <video/videomode.h>
+
+#include "mtk_dp_reg.h"
+
+#define MTK_DP_SIP_CONTROL_AARCH32	MTK_SIP_SMC_CMD(0x523)
+#define MTK_DP_SIP_ATF_EDP_VIDEO_UNMUTE	(BIT(0) | BIT(5))
+#define MTK_DP_SIP_ATF_VIDEO_UNMUTE	BIT(5)
+
+#define MTK_DP_THREAD_CABLE_STATE_CHG	BIT(0)
+#define MTK_DP_THREAD_HPD_EVENT		BIT(1)
+
+#define MTK_DP_4P1T 4
+#define MTK_DP_HDE 2
+#define MTK_DP_PIX_PER_ADDR 2
+#define MTK_DP_AUX_WAIT_REPLY_COUNT 20
+#define MTK_DP_TBC_BUF_READ_START_ADDR 0x8
+#define MTK_DP_TRAIN_VOLTAGE_LEVEL_RETRY 5
+#define MTK_DP_TRAIN_DOWNSCALE_RETRY 10
+#define MTK_DP_VERSION 0x11
+#define MTK_DP_SDP_AUI 0x4
+
+enum {
+	MTK_DP_CAL_GLB_BIAS_TRIM = 0,
+	MTK_DP_CAL_CLKTX_IMPSE,
+	MTK_DP_CAL_LN_TX_IMPSEL_PMOS_0,
+	MTK_DP_CAL_LN_TX_IMPSEL_PMOS_1,
+	MTK_DP_CAL_LN_TX_IMPSEL_PMOS_2,
+	MTK_DP_CAL_LN_TX_IMPSEL_PMOS_3,
+	MTK_DP_CAL_LN_TX_IMPSEL_NMOS_0,
+	MTK_DP_CAL_LN_TX_IMPSEL_NMOS_1,
+	MTK_DP_CAL_LN_TX_IMPSEL_NMOS_2,
+	MTK_DP_CAL_LN_TX_IMPSEL_NMOS_3,
+	MTK_DP_CAL_MAX,
+};
+
+struct mtk_dp_train_info {
+	bool sink_ssc;
+	bool cable_plugged_in;
+	/* link_rate is in multiple of 0.27Gbps */
+	int link_rate;
+	int lane_count;
+	unsigned int channel_eq_pattern;
+};
+
+struct mtk_dp_audio_cfg {
+	bool detect_monitor;
+	int sad_count;
+	int sample_rate;
+	int word_length_bits;
+	int channels;
+};
+
+struct mtk_dp_info {
+	enum dp_pixelformat format;
+	struct videomode vm;
+	struct mtk_dp_audio_cfg audio_cur_cfg;
+};
+
+struct mtk_dp_efuse_fmt {
+	unsigned short idx;
+	unsigned short shift;
+	unsigned short mask;
+	unsigned short min_val;
+	unsigned short max_val;
+	unsigned short default_val;
+};
+
+struct mtk_dp {
+	bool enabled;
+	bool need_debounce;
+	u8 max_lanes;
+	u8 max_linkrate;
+	u8 rx_cap[DP_RECEIVER_CAP_SIZE];
+	u32 cal_data[MTK_DP_CAL_MAX];
+	u32 irq_thread_handle;
+	/* irq_thread_lock is used to protect irq_thread_handle */
+	spinlock_t irq_thread_lock;
+
+	struct device *dev;
+	struct drm_bridge bridge;
+	struct drm_bridge *next_bridge;
+	struct drm_connector *conn;
+	struct drm_device *drm_dev;
+	struct drm_dp_aux aux;
+
+	const struct mtk_dp_data *data;
+	struct mtk_dp_info info;
+	struct mtk_dp_train_info train_info;
+
+	struct platform_device *phy_dev;
+	struct phy *phy;
+	struct regmap *regs;
+	struct timer_list debounce_timer;
+
+	/* For audio */
+	bool audio_enable;
+	hdmi_codec_plugged_cb plugged_cb;
+	struct platform_device *audio_pdev;
+
+	struct device *codec_dev;
+	/* protect the plugged_cb as it's used in both bridge ops and audio */
+	struct mutex update_plugged_status_lock;
+};
+
+struct mtk_dp_data {
+	int bridge_type;
+	unsigned int smc_cmd;
+	const struct mtk_dp_efuse_fmt *efuse_fmt;
+	bool audio_supported;
+};
+
+static const struct mtk_dp_efuse_fmt mt8195_edp_efuse_fmt[MTK_DP_CAL_MAX] = {
+	[MTK_DP_CAL_GLB_BIAS_TRIM] = {
+		.idx = 3,
+		.shift = 27,
+		.mask = 0x1f,
+		.min_val = 1,
+		.max_val = 0x1e,
+		.default_val = 0xf,
+	},
+	[MTK_DP_CAL_CLKTX_IMPSE] = {
+		.idx = 0,
+		.shift = 9,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_0] = {
+		.idx = 2,
+		.shift = 28,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_1] = {
+		.idx = 2,
+		.shift = 20,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_2] = {
+		.idx = 2,
+		.shift = 12,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_3] = {
+		.idx = 2,
+		.shift = 4,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_0] = {
+		.idx = 2,
+		.shift = 24,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_1] = {
+		.idx = 2,
+		.shift = 16,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_2] = {
+		.idx = 2,
+		.shift = 8,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_3] = {
+		.idx = 2,
+		.shift = 0,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+};
+
+static const struct mtk_dp_efuse_fmt mt8195_dp_efuse_fmt[MTK_DP_CAL_MAX] = {
+	[MTK_DP_CAL_GLB_BIAS_TRIM] = {
+		.idx = 0,
+		.shift = 27,
+		.mask = 0x1f,
+		.min_val = 1,
+		.max_val = 0x1e,
+		.default_val = 0xf,
+	},
+	[MTK_DP_CAL_CLKTX_IMPSE] = {
+		.idx = 0,
+		.shift = 13,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_0] = {
+		.idx = 1,
+		.shift = 28,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_1] = {
+		.idx = 1,
+		.shift = 20,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_2] = {
+		.idx = 1,
+		.shift = 12,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_3] = {
+		.idx = 1,
+		.shift = 4,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_0] = {
+		.idx = 1,
+		.shift = 24,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_1] = {
+		.idx = 1,
+		.shift = 16,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_2] = {
+		.idx = 1,
+		.shift = 8,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+	[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_3] = {
+		.idx = 1,
+		.shift = 0,
+		.mask = 0xf,
+		.min_val = 1,
+		.max_val = 0xe,
+		.default_val = 0x8,
+	},
+};
+
+static struct regmap_config mtk_dp_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = SEC_OFFSET + 0x90,
+	.name = "mtk-dp-registers",
+};
+
+static struct mtk_dp *mtk_dp_from_bridge(struct drm_bridge *b)
+{
+	return container_of(b, struct mtk_dp, bridge);
+}
+
+static u32 mtk_dp_read(struct mtk_dp *mtk_dp, u32 offset)
+{
+	u32 read_val;
+	int ret;
+
+	ret = regmap_read(mtk_dp->regs, offset, &read_val);
+	if (ret) {
+		dev_err(mtk_dp->dev, "Failed to read register 0x%x: %d\n",
+			offset, ret);
+		return 0;
+	}
+
+	return read_val;
+}
+
+static int mtk_dp_write(struct mtk_dp *mtk_dp, u32 offset, u32 val)
+{
+	int ret = regmap_write(mtk_dp->regs, offset, val);
+
+	if (ret)
+		dev_err(mtk_dp->dev,
+			"Failed to write register 0x%x with value 0x%x\n",
+			offset, val);
+	return ret;
+}
+
+static int mtk_dp_update_bits(struct mtk_dp *mtk_dp, u32 offset,
+			      u32 val, u32 mask)
+{
+	int ret = regmap_update_bits(mtk_dp->regs, offset, mask, val);
+
+	if (ret)
+		dev_err(mtk_dp->dev,
+			"Failed to update register 0x%x with value 0x%x, mask 0x%x\n",
+			offset, val, mask);
+	return ret;
+}
+
+static void mtk_dp_bulk_16bit_write(struct mtk_dp *mtk_dp, u32 offset, u8 *buf,
+				    size_t length)
+{
+	int i;
+
+	/* 2 bytes per register */
+	for (i = 0; i < length; i += 2) {
+		u32 val = buf[i] | (i + 1 < length ? buf[i + 1] << 8 : 0);
+
+		if (mtk_dp_write(mtk_dp, offset + i * 2, val))
+			return;
+	}
+}
+
+static void mtk_dp_msa_bypass_enable(struct mtk_dp *mtk_dp, bool enable)
+{
+	u32 mask = HTOTAL_SEL_DP_ENC0_P0 | VTOTAL_SEL_DP_ENC0_P0 |
+		   HSTART_SEL_DP_ENC0_P0 | VSTART_SEL_DP_ENC0_P0 |
+		   HWIDTH_SEL_DP_ENC0_P0 | VHEIGHT_SEL_DP_ENC0_P0 |
+		   HSP_SEL_DP_ENC0_P0 | HSW_SEL_DP_ENC0_P0 |
+		   VSP_SEL_DP_ENC0_P0 | VSW_SEL_DP_ENC0_P0;
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3030, enable ? 0 : mask, mask);
+}
+
+static void mtk_dp_set_msa(struct mtk_dp *mtk_dp)
+{
+	struct drm_display_mode mode;
+	struct videomode *vm = &mtk_dp->info.vm;
+
+	drm_display_mode_from_videomode(vm, &mode);
+
+	/* horizontal */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3010,
+			   mode.htotal, HTOTAL_SW_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3018,
+			   vm->hsync_len + vm->hback_porch,
+			   HSTART_SW_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3028,
+			   vm->hsync_len, HSW_SW_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3028,
+			   0, HSP_SW_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3020,
+			   vm->hactive, HWIDTH_SW_DP_ENC0_P0_MASK);
+
+	/* vertical */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3014,
+			   mode.vtotal, VTOTAL_SW_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_301C,
+			   vm->vsync_len + vm->vback_porch,
+			   VSTART_SW_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_302C,
+			   vm->vsync_len, VSW_SW_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_302C,
+			   0, VSP_SW_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3024,
+			   vm->vactive, VHEIGHT_SW_DP_ENC0_P0_MASK);
+
+	/* horizontal */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3064,
+			   vm->hactive, HDE_NUM_LAST_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3154,
+			   mode.htotal, PGEN_HTOTAL_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3158,
+			   vm->hfront_porch,
+			   PGEN_HSYNC_RISING_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_315C,
+			   vm->hsync_len,
+			   PGEN_HSYNC_PULSE_WIDTH_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3160,
+			   vm->hback_porch + vm->hsync_len,
+			   PGEN_HFDE_START_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3164,
+			   vm->hactive,
+			   PGEN_HFDE_ACTIVE_WIDTH_DP_ENC0_P0_MASK);
+
+	/* vertical */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3168,
+			   mode.vtotal,
+			   PGEN_VTOTAL_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_316C,
+			   vm->vfront_porch,
+			   PGEN_VSYNC_RISING_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3170,
+			   vm->vsync_len,
+			   PGEN_VSYNC_PULSE_WIDTH_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3174,
+			   vm->vback_porch + vm->vsync_len,
+			   PGEN_VFDE_START_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3178,
+			   vm->vactive,
+			   PGEN_VFDE_ACTIVE_WIDTH_DP_ENC0_P0_MASK);
+}
+
+static int mtk_dp_set_color_format(struct mtk_dp *mtk_dp,
+				   enum dp_pixelformat color_format)
+{
+	u32 val;
+
+	/* update MISC0 */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3034,
+			   color_format << DP_TEST_COLOR_FORMAT_SHIFT,
+			   DP_TEST_COLOR_FORMAT_MASK);
+
+	switch (color_format) {
+	case DP_PIXELFORMAT_YUV422:
+		val = PIXEL_ENCODE_FORMAT_DP_ENC0_P0_YCBCR422;
+		break;
+	case DP_PIXELFORMAT_RGB:
+		val = PIXEL_ENCODE_FORMAT_DP_ENC0_P0_RGB;
+		break;
+	default:
+		drm_warn(mtk_dp->drm_dev, "Unsupported color format: %d\n",
+			 color_format);
+		return -EINVAL;
+	}
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_303C,
+			   val, PIXEL_ENCODE_FORMAT_DP_ENC0_P0_MASK);
+	return 0;
+}
+
+static void mtk_dp_set_color_depth(struct mtk_dp *mtk_dp)
+{
+	/* Only support 8 bits currently */
+	/* Update MISC0 */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3034,
+			   DP_MSA_MISC_8_BPC, DP_TEST_BIT_DEPTH_MASK);
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_303C,
+			   VIDEO_COLOR_DEPTH_DP_ENC0_P0_8BIT,
+			   VIDEO_COLOR_DEPTH_DP_ENC0_P0_MASK);
+}
+
+static void mtk_dp_config_mn_mode(struct mtk_dp *mtk_dp)
+{
+	/* 0: hw mode, 1: sw mode */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3004,
+			   0, VIDEO_M_CODE_SEL_DP_ENC0_P0_MASK);
+}
+
+static void mtk_dp_set_sram_read_start(struct mtk_dp *mtk_dp, u32 val)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_303C,
+			   val, SRAM_START_READ_THRD_DP_ENC0_P0_MASK);
+}
+
+static void mtk_dp_setup_encoder(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_303C,
+			   VIDEO_MN_GEN_EN_DP_ENC0_P0,
+			   VIDEO_MN_GEN_EN_DP_ENC0_P0);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3040,
+			   SDP_DOWN_CNT_DP_ENC0_P0_VAL,
+			   SDP_DOWN_CNT_INIT_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_3364,
+			   SDP_DOWN_CNT_IN_HBLANK_DP_ENC1_P0_VAL,
+			   SDP_DOWN_CNT_INIT_IN_HBLANK_DP_ENC1_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_3300,
+			   VIDEO_AFIFO_RDY_SEL_DP_ENC1_P0_VAL << 8,
+			   VIDEO_AFIFO_RDY_SEL_DP_ENC1_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_3364,
+			   FIFO_READ_START_POINT_DP_ENC1_P0_VAL << 12,
+			   FIFO_READ_START_POINT_DP_ENC1_P0_MASK);
+	mtk_dp_write(mtk_dp, MTK_DP_ENC1_P0_3368, DP_ENC1_P0_3368_VAL);
+}
+
+static void mtk_dp_pg_enable(struct mtk_dp *mtk_dp, bool enable)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3038,
+			   enable ? VIDEO_SOURCE_SEL_DP_ENC0_P0_MASK : 0,
+			   VIDEO_SOURCE_SEL_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_31B0,
+			   PGEN_PATTERN_SEL_VAL << 4, PGEN_PATTERN_SEL_MASK);
+}
+
+static void mtk_dp_audio_setup_channels(struct mtk_dp *mtk_dp,
+					struct mtk_dp_audio_cfg *cfg)
+{
+	u32 channel_enable_bits;
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_3324,
+			   AUDIO_SOURCE_MUX_DP_ENC1_P0_DPRX,
+			   AUDIO_SOURCE_MUX_DP_ENC1_P0_MASK);
+
+	/* audio channel count change reset */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_33F4,
+			   DP_ENC_DUMMY_RW_1, DP_ENC_DUMMY_RW_1);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_3304,
+			   AU_PRTY_REGEN_DP_ENC1_P0_MASK |
+			   AU_CH_STS_REGEN_DP_ENC1_P0_MASK |
+			   AUDIO_SAMPLE_PRSENT_REGEN_DP_ENC1_P0_MASK,
+			   AU_PRTY_REGEN_DP_ENC1_P0_MASK |
+			   AU_CH_STS_REGEN_DP_ENC1_P0_MASK |
+			   AUDIO_SAMPLE_PRSENT_REGEN_DP_ENC1_P0_MASK);
+
+	switch (cfg->channels) {
+	case 2:
+		channel_enable_bits = AUDIO_2CH_SEL_DP_ENC0_P0_MASK |
+				      AUDIO_2CH_EN_DP_ENC0_P0_MASK;
+		break;
+	case 8:
+	default:
+		channel_enable_bits = AUDIO_8CH_SEL_DP_ENC0_P0_MASK |
+				      AUDIO_8CH_EN_DP_ENC0_P0_MASK;
+		break;
+	}
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3088,
+			   channel_enable_bits | AU_EN_DP_ENC0_P0,
+			   AUDIO_2CH_SEL_DP_ENC0_P0_MASK |
+			   AUDIO_2CH_EN_DP_ENC0_P0_MASK |
+			   AUDIO_8CH_SEL_DP_ENC0_P0_MASK |
+			   AUDIO_8CH_EN_DP_ENC0_P0_MASK |
+			   AU_EN_DP_ENC0_P0);
+
+	/* audio channel count change reset */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_33F4, 0, DP_ENC_DUMMY_RW_1);
+
+	/* enable audio reset */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_33F4,
+			   DP_ENC_DUMMY_RW_1_AUDIO_RST_EN,
+			   DP_ENC_DUMMY_RW_1_AUDIO_RST_EN);
+}
+
+static void mtk_dp_audio_channel_status_set(struct mtk_dp *mtk_dp,
+					    struct mtk_dp_audio_cfg *cfg)
+{
+	struct snd_aes_iec958 iec = { 0 };
+
+	switch (cfg->sample_rate) {
+	case 32000:
+		iec.status[3] = IEC958_AES3_CON_FS_32000;
+		break;
+	case 44100:
+		iec.status[3] = IEC958_AES3_CON_FS_44100;
+		break;
+	case 48000:
+		iec.status[3] = IEC958_AES3_CON_FS_48000;
+		break;
+	case 88200:
+		iec.status[3] = IEC958_AES3_CON_FS_88200;
+		break;
+	case 96000:
+		iec.status[3] = IEC958_AES3_CON_FS_96000;
+		break;
+	case 192000:
+		iec.status[3] = IEC958_AES3_CON_FS_192000;
+		break;
+	default:
+		iec.status[3] = IEC958_AES3_CON_FS_NOTID;
+		break;
+	}
+
+	switch (cfg->word_length_bits) {
+	case 16:
+		iec.status[4] = IEC958_AES4_CON_WORDLEN_20_16;
+		break;
+	case 20:
+		iec.status[4] = IEC958_AES4_CON_WORDLEN_20_16 |
+				IEC958_AES4_CON_MAX_WORDLEN_24;
+		break;
+	case 24:
+		iec.status[4] = IEC958_AES4_CON_WORDLEN_24_20 |
+				IEC958_AES4_CON_MAX_WORDLEN_24;
+		break;
+	default:
+		iec.status[4] = IEC958_AES4_CON_WORDLEN_NOTID;
+	}
+
+	/* IEC 60958 consumer channel status bits */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_308C,
+			   0, CH_STATUS_0_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3090,
+			   iec.status[3] << 8, CH_STATUS_1_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3094,
+			   iec.status[4], CH_STATUS_2_DP_ENC0_P0_MASK);
+}
+
+static void mtk_dp_audio_sdp_asp_set_channels(struct mtk_dp *mtk_dp,
+					      int channels)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_312C,
+			   (min(8, channels) - 1) << 8,
+			   ASP_HB2_DP_ENC0_P0_MASK | ASP_HB3_DP_ENC0_P0_MASK);
+}
+
+static void mtk_dp_audio_set_divider(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_30BC,
+			   AUDIO_M_CODE_MULT_DIV_SEL_DP_ENC0_P0_DIV_2,
+			   AUDIO_M_CODE_MULT_DIV_SEL_DP_ENC0_P0_MASK);
+}
+
+static void mtk_dp_sdp_trigger_aui(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_3280,
+			   MTK_DP_SDP_AUI, SDP_PACKET_TYPE_DP_ENC1_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_3280,
+			   SDP_PACKET_W_DP_ENC1_P0, SDP_PACKET_W_DP_ENC1_P0);
+}
+
+static void mtk_dp_sdp_set_data(struct mtk_dp *mtk_dp, u8 *data_bytes)
+{
+	mtk_dp_bulk_16bit_write(mtk_dp, MTK_DP_ENC1_P0_3200,
+				data_bytes, 0x10);
+}
+
+static void mtk_dp_sdp_set_header_aui(struct mtk_dp *mtk_dp,
+				      struct dp_sdp_header *header)
+{
+	u32 db_addr = MTK_DP_ENC0_P0_30D8 + (MTK_DP_SDP_AUI - 1) * 8;
+
+	mtk_dp_bulk_16bit_write(mtk_dp, db_addr, (u8 *)header, 4);
+}
+
+static void mtk_dp_disable_sdp_aui(struct mtk_dp *mtk_dp)
+{
+	/* Disable periodic send */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_30A8 & 0xfffc, 0,
+			   0xff << ((MTK_DP_ENC0_P0_30A8 & 3) * 8));
+}
+
+static void mtk_dp_setup_sdp_aui(struct mtk_dp *mtk_dp,
+				 struct dp_sdp *sdp)
+{
+	u32 shift;
+
+	mtk_dp_sdp_set_data(mtk_dp, sdp->db);
+	mtk_dp_sdp_set_header_aui(mtk_dp, &sdp->sdp_header);
+	mtk_dp_disable_sdp_aui(mtk_dp);
+
+	shift = (MTK_DP_ENC0_P0_30A8 & 3) * 8;
+
+	mtk_dp_sdp_trigger_aui(mtk_dp);
+	/* Enable periodic sending */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_30A8 & 0xfffc,
+			   0x05 << shift, 0xff << shift);
+}
+
+static void mtk_dp_aux_irq_clear(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_write(mtk_dp, MTK_DP_AUX_P0_3640, DP_AUX_P0_3640_VAL);
+}
+
+static void mtk_dp_aux_set_cmd(struct mtk_dp *mtk_dp, u8 cmd, u32 addr)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3644,
+			   cmd, MCU_REQUEST_COMMAND_AUX_TX_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3648,
+			   addr, MCU_REQUEST_ADDRESS_LSB_AUX_TX_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_364C,
+			   addr >> 16, MCU_REQUEST_ADDRESS_MSB_AUX_TX_P0_MASK);
+}
+
+static void mtk_dp_aux_clear_fifo(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3650,
+			   MCU_ACK_TRAN_COMPLETE_AUX_TX_P0,
+			   MCU_ACK_TRAN_COMPLETE_AUX_TX_P0 |
+			   PHY_FIFO_RST_AUX_TX_P0_MASK |
+			   MCU_REQ_DATA_NUM_AUX_TX_P0_MASK);
+}
+
+static void mtk_dp_aux_request_ready(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3630,
+			   AUX_TX_REQUEST_READY_AUX_TX_P0,
+			   AUX_TX_REQUEST_READY_AUX_TX_P0);
+}
+
+static void mtk_dp_aux_fill_write_fifo(struct mtk_dp *mtk_dp, u8 *buf,
+				       size_t length)
+{
+	mtk_dp_bulk_16bit_write(mtk_dp, MTK_DP_AUX_P0_3708, buf, length);
+}
+
+static void mtk_dp_aux_read_rx_fifo(struct mtk_dp *mtk_dp, u8 *buf,
+				    size_t length, int read_delay)
+{
+	int read_pos;
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3620,
+			   0, AUX_RD_MODE_AUX_TX_P0_MASK);
+
+	for (read_pos = 0; read_pos < length; read_pos++) {
+		mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3620,
+				   AUX_RX_FIFO_READ_PULSE_TX_P0,
+				   AUX_RX_FIFO_READ_PULSE_TX_P0);
+
+		/* Hardware needs time to update the data */
+		usleep_range(read_delay, read_delay * 2);
+		buf[read_pos] = (u8)(mtk_dp_read(mtk_dp, MTK_DP_AUX_P0_3620) &
+				     AUX_RX_FIFO_READ_DATA_AUX_TX_P0_MASK);
+	}
+}
+
+static void mtk_dp_aux_set_length(struct mtk_dp *mtk_dp, size_t length)
+{
+	if (length > 0) {
+		mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3650,
+				   (length - 1) << 12,
+				   MCU_REQ_DATA_NUM_AUX_TX_P0_MASK);
+		mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_362C,
+				   0,
+				   AUX_NO_LENGTH_AUX_TX_P0 |
+				   AUX_TX_AUXTX_OV_EN_AUX_TX_P0_MASK |
+				   AUX_RESERVED_RW_0_AUX_TX_P0_MASK);
+	} else {
+		mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_362C,
+				   AUX_NO_LENGTH_AUX_TX_P0,
+				   AUX_NO_LENGTH_AUX_TX_P0 |
+				   AUX_TX_AUXTX_OV_EN_AUX_TX_P0_MASK |
+				   AUX_RESERVED_RW_0_AUX_TX_P0_MASK);
+	}
+}
+
+static int mtk_dp_aux_wait_for_completion(struct mtk_dp *mtk_dp, bool is_read)
+{
+	int wait_reply = MTK_DP_AUX_WAIT_REPLY_COUNT;
+
+	while (--wait_reply) {
+		u32 aux_irq_status;
+
+		if (is_read) {
+			u32 fifo_status = mtk_dp_read(mtk_dp, MTK_DP_AUX_P0_3618);
+
+			if (fifo_status &
+			    (AUX_RX_FIFO_WRITE_POINTER_AUX_TX_P0_MASK |
+			     AUX_RX_FIFO_FULL_AUX_TX_P0_MASK)) {
+				return 0;
+			}
+		}
+
+		aux_irq_status = mtk_dp_read(mtk_dp, MTK_DP_AUX_P0_3640);
+		if (aux_irq_status & AUX_RX_AUX_RECV_COMPLETE_IRQ_AUX_TX_P0)
+			return 0;
+
+		if (aux_irq_status & AUX_400US_TIMEOUT_IRQ_AUX_TX_P0)
+			return -ETIMEDOUT;
+
+		/* Give the hardware a chance to reach completion before retrying */
+		usleep_range(100, 500);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int mtk_dp_aux_do_transfer(struct mtk_dp *mtk_dp, bool is_read, u8 cmd,
+				  u32 addr, u8 *buf, size_t length)
+{
+	int ret;
+	u32 reply_cmd;
+
+	if (is_read && (length > DP_AUX_MAX_PAYLOAD_BYTES ||
+			(cmd == DP_AUX_NATIVE_READ && !length)))
+		return -EINVAL;
+
+	if (!is_read)
+		mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3704,
+				   AUX_TX_FIFO_NEW_MODE_EN_AUX_TX_P0,
+				   AUX_TX_FIFO_NEW_MODE_EN_AUX_TX_P0);
+
+	/* We need to clear fifo and irq before sending commands to the sink device. */
+	mtk_dp_aux_clear_fifo(mtk_dp);
+	mtk_dp_aux_irq_clear(mtk_dp);
+
+	mtk_dp_aux_set_cmd(mtk_dp, cmd, addr);
+	mtk_dp_aux_set_length(mtk_dp, length);
+
+	if (!is_read) {
+		if (length)
+			mtk_dp_aux_fill_write_fifo(mtk_dp, buf, length);
+
+		mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3704,
+				   AUX_TX_FIFO_WDATA_NEW_MODE_T_AUX_TX_P0_MASK,
+				   AUX_TX_FIFO_WDATA_NEW_MODE_T_AUX_TX_P0_MASK);
+	}
+
+	mtk_dp_aux_request_ready(mtk_dp);
+
+	/* Wait for feedback from sink device. */
+	ret = mtk_dp_aux_wait_for_completion(mtk_dp, is_read);
+
+	reply_cmd = mtk_dp_read(mtk_dp, MTK_DP_AUX_P0_3624) &
+		    AUX_RX_REPLY_COMMAND_AUX_TX_P0_MASK;
+
+	if (ret || reply_cmd) {
+		u32 phy_status = mtk_dp_read(mtk_dp, MTK_DP_AUX_P0_3628) &
+				 AUX_RX_PHY_STATE_AUX_TX_P0_MASK;
+		if (phy_status != AUX_RX_PHY_STATE_AUX_TX_P0_RX_IDLE) {
+			drm_err(mtk_dp->drm_dev,
+				"AUX Rx Aux hang, need SW reset\n");
+			return -EIO;
+		}
+
+		return -ETIMEDOUT;
+	}
+
+	if (!length) {
+		mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_362C,
+				   0,
+				   AUX_NO_LENGTH_AUX_TX_P0 |
+				   AUX_TX_AUXTX_OV_EN_AUX_TX_P0_MASK |
+				   AUX_RESERVED_RW_0_AUX_TX_P0_MASK);
+	} else if (is_read) {
+		int read_delay;
+
+		if (cmd == (DP_AUX_I2C_READ | DP_AUX_I2C_MOT) ||
+		    cmd == DP_AUX_I2C_READ)
+			read_delay = 500;
+		else
+			read_delay = 100;
+
+		mtk_dp_aux_read_rx_fifo(mtk_dp, buf, length, read_delay);
+	}
+
+	return 0;
+}
+
+static void mtk_dp_set_swing_pre_emphasis(struct mtk_dp *mtk_dp, int lane_num,
+					  int swing_val, int preemphasis)
+{
+	u32 lane_shift = lane_num * DP_TX1_VOLT_SWING_SHIFT;
+
+	dev_dbg(mtk_dp->dev,
+		"link training: swing_val = 0x%x, pre-emphasis = 0x%x\n",
+		swing_val, preemphasis);
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_SWING_EMP,
+			   swing_val << (DP_TX0_VOLT_SWING_SHIFT + lane_shift),
+			   DP_TX0_VOLT_SWING_MASK << lane_shift);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_SWING_EMP,
+			   preemphasis << (DP_TX0_PRE_EMPH_SHIFT + lane_shift),
+			   DP_TX0_PRE_EMPH_MASK << lane_shift);
+}
+
+static void mtk_dp_reset_swing_pre_emphasis(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_SWING_EMP,
+			   0,
+			   DP_TX0_VOLT_SWING_MASK |
+			   DP_TX1_VOLT_SWING_MASK |
+			   DP_TX2_VOLT_SWING_MASK |
+			   DP_TX3_VOLT_SWING_MASK |
+			   DP_TX0_PRE_EMPH_MASK |
+			   DP_TX1_PRE_EMPH_MASK |
+			   DP_TX2_PRE_EMPH_MASK |
+			   DP_TX3_PRE_EMPH_MASK);
+}
+
+static u32 mtk_dp_swirq_get_clear(struct mtk_dp *mtk_dp)
+{
+	u32 irq_status = mtk_dp_read(mtk_dp, MTK_DP_TRANS_P0_35D0) &
+			 SW_IRQ_FINAL_STATUS_DP_TRANS_P0_MASK;
+
+	if (irq_status) {
+		mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_35C8,
+				   irq_status, SW_IRQ_CLR_DP_TRANS_P0_MASK);
+		mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_35C8,
+				   0, SW_IRQ_CLR_DP_TRANS_P0_MASK);
+	}
+
+	return irq_status;
+}
+
+static u32 mtk_dp_hwirq_get_clear(struct mtk_dp *mtk_dp)
+{
+	u32 irq_status = (mtk_dp_read(mtk_dp, MTK_DP_TRANS_P0_3418) &
+			  IRQ_STATUS_DP_TRANS_P0_MASK) >> 12;
+
+	if (irq_status) {
+		mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_3418,
+				   irq_status, IRQ_CLR_DP_TRANS_P0_MASK);
+		mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_3418,
+				   0, IRQ_CLR_DP_TRANS_P0_MASK);
+	}
+
+	return irq_status;
+}
+
+static void mtk_dp_hwirq_enable(struct mtk_dp *mtk_dp, bool enable)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_3418,
+			   enable ? 0 :
+			   IRQ_MASK_DP_TRANS_P0_DISC_IRQ |
+			   IRQ_MASK_DP_TRANS_P0_CONN_IRQ |
+			   IRQ_MASK_DP_TRANS_P0_INT_IRQ,
+			   IRQ_MASK_DP_TRANS_P0_MASK);
+}
+
+static void mtk_dp_initialize_settings(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_342C,
+			   XTAL_FREQ_DP_TRANS_P0_DEFAULT,
+			   XTAL_FREQ_DP_TRANS_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_3540,
+			   FEC_CLOCK_EN_MODE_DP_TRANS_P0,
+			   FEC_CLOCK_EN_MODE_DP_TRANS_P0);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_31EC,
+			   AUDIO_CH_SRC_SEL_DP_ENC0_P0,
+			   AUDIO_CH_SRC_SEL_DP_ENC0_P0);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_304C,
+			   0, SDP_VSYNC_RISING_MASK_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_IRQ_MASK,
+			   IRQ_MASK_AUX_TOP_IRQ, IRQ_MASK_AUX_TOP_IRQ);
+}
+
+static void mtk_dp_initialize_hpd_detect_settings(struct mtk_dp *mtk_dp)
+{
+	u32 val;
+	/* Debounce threshold */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_3410,
+			   8, HPD_DEB_THD_DP_TRANS_P0_MASK);
+
+	val = (HPD_INT_THD_DP_TRANS_P0_LOWER_500US |
+	       HPD_INT_THD_DP_TRANS_P0_UPPER_1100US) << 4;
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_3410,
+			   val, HPD_INT_THD_DP_TRANS_P0_MASK);
+
+	/*
+	 * Connect threshold 1.5ms + 5 x 0.1ms = 2ms
+	 * Disconnect threshold 1.5ms + 5 x 0.1ms = 2ms
+	 */
+	val = (5 << 8) | (5 << 12);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_3410,
+			   val,
+			   HPD_DISC_THD_DP_TRANS_P0_MASK |
+			   HPD_CONN_THD_DP_TRANS_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_3430,
+			   HPD_INT_THD_ECO_DP_TRANS_P0_HIGH_BOUND_EXT,
+			   HPD_INT_THD_ECO_DP_TRANS_P0_MASK);
+}
+
+static void mtk_dp_initialize_aux_settings(struct mtk_dp *mtk_dp)
+{
+	/* modify timeout threshold = 0x1595 */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_360C,
+			   AUX_TIMEOUT_THR_AUX_TX_P0_VAL,
+			   AUX_TIMEOUT_THR_AUX_TX_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3658,
+			   0, AUX_TX_OV_EN_AUX_TX_P0_MASK);
+	/* 25 for 26M */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3634,
+			   AUX_TX_OVER_SAMPLE_RATE_FOR_26M << 8,
+			   AUX_TX_OVER_SAMPLE_RATE_AUX_TX_P0_MASK);
+	/* 13 for 26M */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3614,
+			   AUX_RX_UI_CNT_THR_AUX_FOR_26M,
+			   AUX_RX_UI_CNT_THR_AUX_TX_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_37C8,
+			   MTK_ATOP_EN_AUX_TX_P0,
+			   MTK_ATOP_EN_AUX_TX_P0);
+}
+
+static void mtk_dp_initialize_digital_settings(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_304C,
+			   0, VBID_VIDEO_MUTE_DP_ENC0_P0_MASK);
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_3368,
+			   BS2BS_MODE_DP_ENC1_P0_VAL << 12,
+			   BS2BS_MODE_DP_ENC1_P0_MASK);
+
+	/* dp tx encoder reset all sw */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3004,
+			   DP_TX_ENCODER_4P_RESET_SW_DP_ENC0_P0,
+			   DP_TX_ENCODER_4P_RESET_SW_DP_ENC0_P0);
+
+	/* Wait for sw reset to complete */
+	usleep_range(1000, 5000);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3004,
+			   0, DP_TX_ENCODER_4P_RESET_SW_DP_ENC0_P0);
+}
+
+static void mtk_dp_digital_sw_reset(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_340C,
+			   DP_TX_TRANSMITTER_4P_RESET_SW_DP_TRANS_P0,
+			   DP_TX_TRANSMITTER_4P_RESET_SW_DP_TRANS_P0);
+
+	/* Wait for sw reset to complete */
+	usleep_range(1000, 5000);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_340C,
+			   0, DP_TX_TRANSMITTER_4P_RESET_SW_DP_TRANS_P0);
+}
+
+static void mtk_dp_set_lanes(struct mtk_dp *mtk_dp, int lanes)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_35F0,
+			   lanes == 0 ? 0 : DP_TRANS_DUMMY_RW_0,
+			   DP_TRANS_DUMMY_RW_0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3000,
+			   lanes, LANE_NUM_DP_ENC0_P0_MASK);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_34A4,
+			   lanes << 2, LANE_NUM_DP_TRANS_P0_MASK);
+}
+
+static void mtk_dp_get_calibration_data(struct mtk_dp *mtk_dp)
+{
+	const struct mtk_dp_efuse_fmt *fmt;
+	struct device *dev = mtk_dp->dev;
+	struct nvmem_cell *cell;
+	u32 *cal_data = mtk_dp->cal_data;
+	u32 *buf;
+	int i;
+	size_t len;
+
+	cell = nvmem_cell_get(dev, "dp_calibration_data");
+	if (IS_ERR(cell)) {
+		dev_warn(dev, "Failed to get nvmem cell dp_calibration_data\n");
+		goto use_default_val;
+	}
+
+	buf = (u32 *)nvmem_cell_read(cell, &len);
+	nvmem_cell_put(cell);
+
+	if (IS_ERR(buf) || ((len / sizeof(u32)) != 4)) {
+		dev_warn(dev, "Failed to read nvmem_cell_read\n");
+
+		if (!IS_ERR(buf))
+			kfree(buf);
+
+		goto use_default_val;
+	}
+
+	for (i = 0; i < MTK_DP_CAL_MAX; i++) {
+		fmt = &mtk_dp->data->efuse_fmt[i];
+		cal_data[i] = (buf[fmt->idx] >> fmt->shift) & fmt->mask;
+
+		if (cal_data[i] < fmt->min_val || cal_data[i] > fmt->max_val) {
+			dev_warn(mtk_dp->dev, "Invalid efuse data, idx = %d\n", i);
+			kfree(buf);
+			goto use_default_val;
+		}
+	}
+	kfree(buf);
+
+	return;
+
+use_default_val:
+	dev_warn(mtk_dp->dev, "Use default calibration data\n");
+	for (i = 0; i < MTK_DP_CAL_MAX; i++)
+		cal_data[i] = mtk_dp->data->efuse_fmt[i].default_val;
+}
+
+static void mtk_dp_set_calibration_data(struct mtk_dp *mtk_dp)
+{
+	u32 *cal_data = mtk_dp->cal_data;
+
+	mtk_dp_update_bits(mtk_dp, DP_PHY_GLB_DPAUX_TX,
+			   cal_data[MTK_DP_CAL_CLKTX_IMPSE] << 20,
+			   RG_CKM_PT0_CKTX_IMPSEL);
+	mtk_dp_update_bits(mtk_dp, DP_PHY_GLB_BIAS_GEN_00,
+			   cal_data[MTK_DP_CAL_GLB_BIAS_TRIM] << 16,
+			   RG_XTP_GLB_BIAS_INTR_CTRL);
+	mtk_dp_update_bits(mtk_dp, DP_PHY_LANE_TX_0,
+			   cal_data[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_0] << 12,
+			   RG_XTP_LN0_TX_IMPSEL_PMOS);
+	mtk_dp_update_bits(mtk_dp, DP_PHY_LANE_TX_0,
+			   cal_data[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_0] << 16,
+			   RG_XTP_LN0_TX_IMPSEL_NMOS);
+	mtk_dp_update_bits(mtk_dp, DP_PHY_LANE_TX_1,
+			   cal_data[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_1] << 12,
+			   RG_XTP_LN1_TX_IMPSEL_PMOS);
+	mtk_dp_update_bits(mtk_dp, DP_PHY_LANE_TX_1,
+			   cal_data[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_1] << 16,
+			   RG_XTP_LN1_TX_IMPSEL_NMOS);
+	mtk_dp_update_bits(mtk_dp, DP_PHY_LANE_TX_2,
+			   cal_data[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_2] << 12,
+			   RG_XTP_LN2_TX_IMPSEL_PMOS);
+	mtk_dp_update_bits(mtk_dp, DP_PHY_LANE_TX_2,
+			   cal_data[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_2] << 16,
+			   RG_XTP_LN2_TX_IMPSEL_NMOS);
+	mtk_dp_update_bits(mtk_dp, DP_PHY_LANE_TX_3,
+			   cal_data[MTK_DP_CAL_LN_TX_IMPSEL_PMOS_3] << 12,
+			   RG_XTP_LN3_TX_IMPSEL_PMOS);
+	mtk_dp_update_bits(mtk_dp, DP_PHY_LANE_TX_3,
+			   cal_data[MTK_DP_CAL_LN_TX_IMPSEL_NMOS_3] << 16,
+			   RG_XTP_LN3_TX_IMPSEL_NMOS);
+}
+
+static int mtk_dp_phy_configure(struct mtk_dp *mtk_dp,
+				u32 link_rate, int lane_count)
+{
+	int ret;
+	union phy_configure_opts phy_opts = {
+		.dp = {
+			.link_rate = drm_dp_bw_code_to_link_rate(link_rate) / 100,
+			.set_rate = 1,
+			.lanes = lane_count,
+			.set_lanes = 1,
+			.ssc = mtk_dp->train_info.sink_ssc,
+		}
+	};
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_PWR_STATE, DP_PWR_STATE_BANDGAP,
+			   DP_PWR_STATE_MASK);
+
+	ret = phy_configure(mtk_dp->phy, &phy_opts);
+	if (ret)
+		return ret;
+
+	mtk_dp_set_calibration_data(mtk_dp);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_PWR_STATE,
+			   DP_PWR_STATE_BANDGAP_TPLL_LANE, DP_PWR_STATE_MASK);
+
+	return 0;
+}
+
+static void mtk_dp_set_idle_pattern(struct mtk_dp *mtk_dp, bool enable)
+{
+	u32 val = POST_MISC_DATA_LANE0_OV_DP_TRANS_P0_MASK |
+		  POST_MISC_DATA_LANE1_OV_DP_TRANS_P0_MASK |
+		  POST_MISC_DATA_LANE2_OV_DP_TRANS_P0_MASK |
+		  POST_MISC_DATA_LANE3_OV_DP_TRANS_P0_MASK;
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_3580,
+			   enable ? val : 0, val);
+}
+
+static void mtk_dp_train_set_pattern(struct mtk_dp *mtk_dp, int pattern)
+{
+	/* TPS1 */
+	if (pattern == 1)
+		mtk_dp_set_idle_pattern(mtk_dp, false);
+
+	mtk_dp_update_bits(mtk_dp,
+			   MTK_DP_TRANS_P0_3400,
+			   pattern ? BIT(pattern - 1) << 12 : 0,
+			   PATTERN1_EN_DP_TRANS_P0_MASK |
+			   PATTERN2_EN_DP_TRANS_P0_MASK |
+			   PATTERN3_EN_DP_TRANS_P0_MASK |
+			   PATTERN4_EN_DP_TRANS_P0_MASK);
+}
+
+static void mtk_dp_set_enhanced_frame_mode(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3000,
+			   ENHANCED_FRAME_EN_DP_ENC0_P0,
+			   ENHANCED_FRAME_EN_DP_ENC0_P0);
+}
+
+static void mtk_dp_training_set_scramble(struct mtk_dp *mtk_dp, bool enable)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TRANS_P0_3404,
+			   enable ? DP_SCR_EN_DP_TRANS_P0_MASK : 0,
+			   DP_SCR_EN_DP_TRANS_P0_MASK);
+}
+
+static void mtk_dp_video_mute(struct mtk_dp *mtk_dp, bool enable)
+{
+	struct arm_smccc_res res;
+	u32 val = VIDEO_MUTE_SEL_DP_ENC0_P0 |
+		  (enable ? VIDEO_MUTE_SW_DP_ENC0_P0 : 0);
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3000,
+			   val,
+			   VIDEO_MUTE_SEL_DP_ENC0_P0 |
+			   VIDEO_MUTE_SW_DP_ENC0_P0);
+
+	arm_smccc_smc(MTK_DP_SIP_CONTROL_AARCH32,
+		      mtk_dp->data->smc_cmd, enable,
+		      0, 0, 0, 0, 0, &res);
+
+	dev_dbg(mtk_dp->dev, "smc cmd: 0x%x, p1: %s, ret: 0x%lx-0x%lx\n",
+		mtk_dp->data->smc_cmd, enable ? "enable" : "disable", res.a0, res.a1);
+}
+
+static void mtk_dp_audio_mute(struct mtk_dp *mtk_dp, bool mute)
+{
+	u32 val[3];
+
+	if (mute) {
+		val[0] = VBID_AUDIO_MUTE_FLAG_SW_DP_ENC0_P0 |
+			 VBID_AUDIO_MUTE_FLAG_SEL_DP_ENC0_P0;
+		val[1] = 0;
+		val[2] = 0;
+	} else {
+		val[0] = 0;
+		val[1] = AU_EN_DP_ENC0_P0;
+		/* Send one every two frames */
+		val[2] = 0x0F;
+	}
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3030,
+			   val[0],
+			   VBID_AUDIO_MUTE_FLAG_SW_DP_ENC0_P0 |
+			   VBID_AUDIO_MUTE_FLAG_SEL_DP_ENC0_P0);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3088,
+			   val[1], AU_EN_DP_ENC0_P0);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_30A4,
+			   val[2], AU_TS_CFG_DP_ENC0_P0_MASK);
+}
+
+static void mtk_dp_power_enable(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_RESET_AND_PROBE,
+			   0, SW_RST_B_PHYD);
+
+	/* Wait for power enable */
+	usleep_range(10, 200);
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_RESET_AND_PROBE,
+			   SW_RST_B_PHYD, SW_RST_B_PHYD);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_PWR_STATE,
+			   DP_PWR_STATE_BANDGAP_TPLL, DP_PWR_STATE_MASK);
+	mtk_dp_write(mtk_dp, MTK_DP_1040,
+		     RG_DPAUX_RX_VALID_DEGLITCH_EN | RG_XTP_GLB_CKDET_EN |
+		     RG_DPAUX_RX_EN);
+	mtk_dp_update_bits(mtk_dp, MTK_DP_0034, 0, DA_CKM_CKTX0_EN_FORCE_EN);
+}
+
+static void mtk_dp_power_disable(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_write(mtk_dp, MTK_DP_TOP_PWR_STATE, 0);
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_0034,
+			   DA_CKM_CKTX0_EN_FORCE_EN, DA_CKM_CKTX0_EN_FORCE_EN);
+
+	/* Disable RX */
+	mtk_dp_write(mtk_dp, MTK_DP_1040, 0);
+	mtk_dp_write(mtk_dp, MTK_DP_TOP_MEM_PD,
+		     0x550 | FUSE_SEL | MEM_ISO_EN);
+}
+
+static void mtk_dp_initialize_priv_data(struct mtk_dp *mtk_dp)
+{
+	mtk_dp->train_info.link_rate = DP_LINK_BW_5_4;
+	mtk_dp->train_info.lane_count = mtk_dp->max_lanes;
+	mtk_dp->train_info.cable_plugged_in = false;
+
+	mtk_dp->info.format = DP_PIXELFORMAT_RGB;
+	memset(&mtk_dp->info.vm, 0, sizeof(struct videomode));
+	mtk_dp->audio_enable = false;
+}
+
+static void mtk_dp_sdp_set_down_cnt_init(struct mtk_dp *mtk_dp,
+					 u32 sram_read_start)
+{
+	u32 sdp_down_cnt_init = 0;
+	struct drm_display_mode mode;
+	struct videomode *vm = &mtk_dp->info.vm;
+
+	drm_display_mode_from_videomode(vm, &mode);
+
+	if (mode.clock > 0)
+		sdp_down_cnt_init = sram_read_start *
+				    mtk_dp->train_info.link_rate * 2700 * 8 /
+				    (mode.clock * 4);
+
+	switch (mtk_dp->train_info.lane_count) {
+	case 1:
+		sdp_down_cnt_init = max_t(u32, sdp_down_cnt_init, 0x1A);
+		break;
+	case 2:
+		/* case for LowResolution && High Audio Sample Rate */
+		sdp_down_cnt_init = max_t(u32, sdp_down_cnt_init, 0x10);
+		sdp_down_cnt_init += mode.vtotal <= 525 ? 4 : 0;
+		break;
+	case 4:
+	default:
+		sdp_down_cnt_init = max_t(u32, sdp_down_cnt_init, 6);
+		break;
+	}
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC0_P0_3040,
+			   sdp_down_cnt_init,
+			   SDP_DOWN_CNT_INIT_DP_ENC0_P0_MASK);
+}
+
+static void mtk_dp_sdp_set_down_cnt_init_in_hblank(struct mtk_dp *mtk_dp)
+{
+	int pix_clk_mhz;
+	u32 dc_offset;
+	u32 spd_down_cnt_init = 0;
+	struct drm_display_mode mode;
+	struct videomode *vm = &mtk_dp->info.vm;
+
+	drm_display_mode_from_videomode(vm, &mode);
+
+	pix_clk_mhz = mtk_dp->info.format == DP_PIXELFORMAT_YUV420 ?
+		      mode.clock / 2000 : mode.clock / 1000;
+
+	switch (mtk_dp->train_info.lane_count) {
+	case 1:
+		spd_down_cnt_init = 0x20;
+		break;
+	case 2:
+		dc_offset = (mode.vtotal <= 525) ? 0x14 : 0x00;
+		spd_down_cnt_init = 0x18 + dc_offset;
+		break;
+	case 4:
+	default:
+		dc_offset = (mode.vtotal <= 525) ? 0x08 : 0x00;
+		if (pix_clk_mhz > mtk_dp->train_info.link_rate * 27)
+			spd_down_cnt_init = 0x8;
+		else
+			spd_down_cnt_init = 0x10 + dc_offset;
+		break;
+	}
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_ENC1_P0_3364, spd_down_cnt_init,
+			   SDP_DOWN_CNT_INIT_IN_HBLANK_DP_ENC1_P0_MASK);
+}
+
+static void mtk_dp_setup_tu(struct mtk_dp *mtk_dp)
+{
+	u32 sram_read_start = min_t(u32, MTK_DP_TBC_BUF_READ_START_ADDR,
+				    mtk_dp->info.vm.hactive /
+				    mtk_dp->train_info.lane_count /
+				    MTK_DP_4P1T / MTK_DP_HDE /
+				    MTK_DP_PIX_PER_ADDR);
+	mtk_dp_set_sram_read_start(mtk_dp, sram_read_start);
+	mtk_dp_setup_encoder(mtk_dp);
+	mtk_dp_sdp_set_down_cnt_init_in_hblank(mtk_dp);
+	mtk_dp_sdp_set_down_cnt_init(mtk_dp, sram_read_start);
+}
+
+static void mtk_dp_set_tx_out(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_setup_tu(mtk_dp);
+}
+
+static void mtk_dp_train_update_swing_pre(struct mtk_dp *mtk_dp, int lanes,
+					  u8 dpcd_adjust_req[2])
+{
+	int lane;
+
+	for (lane = 0; lane < lanes; ++lane) {
+		u8 val;
+		u8 swing;
+		u8 preemphasis;
+		int index = lane / 2;
+		int shift = lane % 2 ? DP_ADJUST_VOLTAGE_SWING_LANE1_SHIFT : 0;
+
+		swing = (dpcd_adjust_req[index] >> shift) &
+			DP_ADJUST_VOLTAGE_SWING_LANE0_MASK;
+		preemphasis = ((dpcd_adjust_req[index] >> shift) &
+			       DP_ADJUST_PRE_EMPHASIS_LANE0_MASK) >>
+			      DP_ADJUST_PRE_EMPHASIS_LANE0_SHIFT;
+		val = swing << DP_TRAIN_VOLTAGE_SWING_SHIFT |
+		      preemphasis << DP_TRAIN_PRE_EMPHASIS_SHIFT;
+
+		if (swing == DP_TRAIN_VOLTAGE_SWING_LEVEL_3)
+			val |= DP_TRAIN_MAX_SWING_REACHED;
+		if (preemphasis == 3)
+			val |= DP_TRAIN_MAX_PRE_EMPHASIS_REACHED;
+
+		mtk_dp_set_swing_pre_emphasis(mtk_dp, lane, swing, preemphasis);
+		drm_dp_dpcd_writeb(&mtk_dp->aux, DP_TRAINING_LANE0_SET + lane,
+				   val);
+	}
+}
+
+static void mtk_dp_pattern(struct mtk_dp *mtk_dp, bool is_tps1)
+{
+	int pattern;
+	unsigned int aux_offset;
+
+	if (is_tps1) {
+		pattern = 1;
+		aux_offset = DP_LINK_SCRAMBLING_DISABLE | DP_TRAINING_PATTERN_1;
+	} else {
+		aux_offset = mtk_dp->train_info.channel_eq_pattern;
+
+		switch (mtk_dp->train_info.channel_eq_pattern) {
+		case DP_TRAINING_PATTERN_4:
+			pattern = 4;
+			break;
+		case DP_TRAINING_PATTERN_3:
+			pattern = 3;
+			aux_offset |= DP_LINK_SCRAMBLING_DISABLE;
+			break;
+		case DP_TRAINING_PATTERN_2:
+		default:
+			pattern = 2;
+			aux_offset |= DP_LINK_SCRAMBLING_DISABLE;
+			break;
+		}
+	}
+
+	mtk_dp_train_set_pattern(mtk_dp, pattern);
+	drm_dp_dpcd_writeb(&mtk_dp->aux, DP_TRAINING_PATTERN_SET, aux_offset);
+}
+
+static int mtk_dp_train_setting(struct mtk_dp *mtk_dp, u8 target_link_rate,
+				u8 target_lane_count)
+{
+	int ret;
+
+	drm_dp_dpcd_writeb(&mtk_dp->aux, DP_LINK_BW_SET, target_link_rate);
+	drm_dp_dpcd_writeb(&mtk_dp->aux, DP_LANE_COUNT_SET,
+			   target_lane_count | DP_LANE_COUNT_ENHANCED_FRAME_EN);
+
+	if (mtk_dp->train_info.sink_ssc)
+		drm_dp_dpcd_writeb(&mtk_dp->aux, DP_DOWNSPREAD_CTRL,
+				   DP_SPREAD_AMP_0_5);
+
+	mtk_dp_set_lanes(mtk_dp, target_lane_count / 2);
+	ret = mtk_dp_phy_configure(mtk_dp, target_link_rate, target_lane_count);
+	if (ret)
+		return ret;
+
+	dev_dbg(mtk_dp->dev,
+		"Link train target_link_rate = 0x%x, target_lane_count = 0x%x\n",
+		target_link_rate, target_lane_count);
+
+	return 0;
+}
+
+static int mtk_dp_train_cr(struct mtk_dp *mtk_dp, u8 target_lane_count)
+{
+	u8 lane_adjust[2] = {};
+	u8 link_status[DP_LINK_STATUS_SIZE] = {};
+	u8 prev_lane_adjust = 0xff;
+	int train_retries = 0;
+	int voltage_retries = 0;
+
+	mtk_dp_pattern(mtk_dp, true);
+
+	/* In DP spec 1.4, the retry count of CR is defined as 10. */
+	do {
+		train_retries++;
+		if (!mtk_dp->train_info.cable_plugged_in) {
+			mtk_dp_train_set_pattern(mtk_dp, 0);
+			return -ENODEV;
+		}
+
+		drm_dp_dpcd_read(&mtk_dp->aux, DP_ADJUST_REQUEST_LANE0_1,
+				 lane_adjust, sizeof(lane_adjust));
+		mtk_dp_train_update_swing_pre(mtk_dp, target_lane_count,
+					      lane_adjust);
+
+		drm_dp_link_train_clock_recovery_delay(&mtk_dp->aux,
+						       mtk_dp->rx_cap);
+
+		/* check link status from sink device */
+		drm_dp_dpcd_read_link_status(&mtk_dp->aux, link_status);
+		if (drm_dp_clock_recovery_ok(link_status,
+					     target_lane_count)) {
+			dev_dbg(mtk_dp->dev, "Link train CR pass\n");
+			return 0;
+		}
+
+		/*
+		 * In DP spec 1.4, if current voltage level is the same
+		 * with previous voltage level, we need to retry 5 times.
+		 */
+		if (prev_lane_adjust == link_status[4]) {
+			voltage_retries++;
+			/*
+			 * Condition of CR fail:
+			 * 1. Failed to pass CR using the same voltage
+			 *    level over five times.
+			 * 2. Failed to pass CR when the current voltage
+			 *    level is the same with previous voltage
+			 *    level and reach max voltage level (3).
+			 */
+			if (voltage_retries > MTK_DP_TRAIN_VOLTAGE_LEVEL_RETRY ||
+			    (prev_lane_adjust & DP_ADJUST_VOLTAGE_SWING_LANE0_MASK) == 3) {
+				dev_dbg(mtk_dp->dev, "Link train CR fail\n");
+				break;
+			}
+		} else {
+			/*
+			 * If the voltage level is changed, we need to
+			 * re-calculate this retry count.
+			 */
+			voltage_retries = 0;
+		}
+		prev_lane_adjust = link_status[4];
+	} while (train_retries < MTK_DP_TRAIN_DOWNSCALE_RETRY);
+
+	/* Failed to train CR, and disable pattern. */
+	drm_dp_dpcd_writeb(&mtk_dp->aux, DP_TRAINING_PATTERN_SET,
+			   DP_TRAINING_PATTERN_DISABLE);
+	mtk_dp_train_set_pattern(mtk_dp, 0);
+
+	return -ETIMEDOUT;
+}
+
+static int mtk_dp_train_eq(struct mtk_dp *mtk_dp, u8 target_lane_count)
+{
+	u8 lane_adjust[2] = {};
+	u8 link_status[DP_LINK_STATUS_SIZE] = {};
+	int train_retries = 0;
+
+	mtk_dp_pattern(mtk_dp, false);
+
+	do {
+		train_retries++;
+		if (!mtk_dp->train_info.cable_plugged_in) {
+			mtk_dp_train_set_pattern(mtk_dp, 0);
+			return -ENODEV;
+		}
+
+		drm_dp_dpcd_read(&mtk_dp->aux, DP_ADJUST_REQUEST_LANE0_1,
+				 lane_adjust, sizeof(lane_adjust));
+		mtk_dp_train_update_swing_pre(mtk_dp, target_lane_count,
+					      lane_adjust);
+
+		drm_dp_link_train_channel_eq_delay(&mtk_dp->aux,
+						   mtk_dp->rx_cap);
+
+		/* check link status from sink device */
+		drm_dp_dpcd_read_link_status(&mtk_dp->aux, link_status);
+		if (drm_dp_channel_eq_ok(link_status, target_lane_count)) {
+			dev_dbg(mtk_dp->dev, "Link train EQ pass\n");
+
+			/* Training done, and disable pattern. */
+			drm_dp_dpcd_writeb(&mtk_dp->aux, DP_TRAINING_PATTERN_SET,
+					   DP_TRAINING_PATTERN_DISABLE);
+			mtk_dp_train_set_pattern(mtk_dp, 0);
+			return 0;
+		}
+		dev_dbg(mtk_dp->dev, "Link train EQ fail\n");
+	} while (train_retries < MTK_DP_TRAIN_DOWNSCALE_RETRY);
+
+	/* Failed to train EQ, and disable pattern. */
+	drm_dp_dpcd_writeb(&mtk_dp->aux, DP_TRAINING_PATTERN_SET,
+			   DP_TRAINING_PATTERN_DISABLE);
+	mtk_dp_train_set_pattern(mtk_dp, 0);
+
+	return -ETIMEDOUT;
+}
+
+static int mtk_dp_parse_capabilities(struct mtk_dp *mtk_dp)
+{
+	u8 val;
+	ssize_t ret;
+
+	drm_dp_read_dpcd_caps(&mtk_dp->aux, mtk_dp->rx_cap);
+
+	if (drm_dp_tps4_supported(mtk_dp->rx_cap))
+		mtk_dp->train_info.channel_eq_pattern = DP_TRAINING_PATTERN_4;
+	else if (drm_dp_tps3_supported(mtk_dp->rx_cap))
+		mtk_dp->train_info.channel_eq_pattern = DP_TRAINING_PATTERN_3;
+	else
+		mtk_dp->train_info.channel_eq_pattern = DP_TRAINING_PATTERN_2;
+
+	mtk_dp->train_info.sink_ssc = drm_dp_max_downspread(mtk_dp->rx_cap);
+
+	ret = drm_dp_dpcd_readb(&mtk_dp->aux, DP_MSTM_CAP, &val);
+	if (ret < 1) {
+		drm_err(mtk_dp->drm_dev, "Read mstm cap failed\n");
+		return ret == 0 ? -EIO : ret;
+	}
+
+	if (val & DP_MST_CAP) {
+		/* Clear DP_DEVICE_SERVICE_IRQ_VECTOR_ESI0 */
+		ret = drm_dp_dpcd_readb(&mtk_dp->aux,
+					DP_DEVICE_SERVICE_IRQ_VECTOR_ESI0,
+					&val);
+		if (ret < 1) {
+			drm_err(mtk_dp->drm_dev, "Read irq vector failed\n");
+			return ret == 0 ? -EIO : ret;
+		}
+
+		if (val)
+			drm_dp_dpcd_writeb(&mtk_dp->aux,
+					   DP_DEVICE_SERVICE_IRQ_VECTOR_ESI0,
+					   val);
+	}
+
+	return 0;
+}
+
+static bool mtk_dp_edid_parse_audio_capabilities(struct mtk_dp *mtk_dp,
+						 struct mtk_dp_audio_cfg *cfg)
+{
+	if (!mtk_dp->data->audio_supported)
+		return false;
+
+	if (mtk_dp->info.audio_cur_cfg.sad_count <= 0) {
+		drm_info(mtk_dp->drm_dev, "The SADs is NULL\n");
+		return false;
+	}
+
+	return true;
+}
+
+static void mtk_dp_train_change_mode(struct mtk_dp *mtk_dp)
+{
+	phy_reset(mtk_dp->phy);
+	mtk_dp_reset_swing_pre_emphasis(mtk_dp);
+}
+
+static int mtk_dp_training(struct mtk_dp *mtk_dp)
+{
+	int ret;
+	u8 lane_count, link_rate, train_limit, max_link_rate;
+
+	link_rate = min_t(u8, mtk_dp->max_linkrate,
+			  mtk_dp->rx_cap[DP_MAX_LINK_RATE]);
+	max_link_rate = link_rate;
+	lane_count = min_t(u8, mtk_dp->max_lanes,
+			   drm_dp_max_lane_count(mtk_dp->rx_cap));
+
+	/*
+	 * TPS are generated by the hardware pattern generator. From the
+	 * hardware setting we need to disable this scramble setting before
+	 * use the TPS pattern generator.
+	 */
+	mtk_dp_training_set_scramble(mtk_dp, false);
+
+	for (train_limit = 6; train_limit > 0; train_limit--) {
+		mtk_dp_train_change_mode(mtk_dp);
+
+		ret = mtk_dp_train_setting(mtk_dp, link_rate, lane_count);
+		if (ret)
+			return ret;
+
+		ret = mtk_dp_train_cr(mtk_dp, lane_count);
+		if (ret == -ENODEV) {
+			return ret;
+		} else if (ret) {
+			/* reduce link rate */
+			switch (link_rate) {
+			case DP_LINK_BW_1_62:
+				lane_count = lane_count / 2;
+				link_rate = max_link_rate;
+				if (lane_count == 0)
+					return -EIO;
+				break;
+			case DP_LINK_BW_2_7:
+				link_rate = DP_LINK_BW_1_62;
+				break;
+			case DP_LINK_BW_5_4:
+				link_rate = DP_LINK_BW_2_7;
+				break;
+			case DP_LINK_BW_8_1:
+				link_rate = DP_LINK_BW_5_4;
+				break;
+			default:
+				return -EINVAL;
+			};
+			continue;
+		}
+
+		ret = mtk_dp_train_eq(mtk_dp, lane_count);
+		if (ret == -ENODEV) {
+			return ret;
+		} else if (ret) {
+			/* reduce lane count */
+			if (lane_count == 0)
+				return -EIO;
+			lane_count /= 2;
+			continue;
+		}
+
+		/* if we can run to this, training is done. */
+		break;
+	}
+
+	if (train_limit == 0)
+		return -ETIMEDOUT;
+
+	mtk_dp->train_info.link_rate = link_rate;
+	mtk_dp->train_info.lane_count = lane_count;
+
+	/*
+	 * After training done, we need to output normal stream instead of TPS,
+	 * so we need to enable scramble.
+	 */
+	mtk_dp_training_set_scramble(mtk_dp, true);
+	mtk_dp_set_enhanced_frame_mode(mtk_dp);
+
+	return 0;
+}
+
+static void mtk_dp_video_enable(struct mtk_dp *mtk_dp, bool enable)
+{
+	/* the mute sequence is different between enable and disable */
+	if (enable) {
+		mtk_dp_msa_bypass_enable(mtk_dp, false);
+		mtk_dp_pg_enable(mtk_dp, false);
+		mtk_dp_set_tx_out(mtk_dp);
+		mtk_dp_video_mute(mtk_dp, false);
+	} else {
+		mtk_dp_video_mute(mtk_dp, true);
+		mtk_dp_pg_enable(mtk_dp, true);
+		mtk_dp_msa_bypass_enable(mtk_dp, true);
+	}
+}
+
+static void mtk_dp_audio_sdp_setup(struct mtk_dp *mtk_dp,
+				   struct mtk_dp_audio_cfg *cfg)
+{
+	struct dp_sdp sdp;
+	struct hdmi_audio_infoframe frame;
+
+	hdmi_audio_infoframe_init(&frame);
+	frame.coding_type = HDMI_AUDIO_CODING_TYPE_PCM;
+	frame.channels = cfg->channels;
+	frame.sample_frequency = cfg->sample_rate;
+
+	switch (cfg->word_length_bits) {
+	case 16:
+		frame.sample_size = HDMI_AUDIO_SAMPLE_SIZE_16;
+		break;
+	case 20:
+		frame.sample_size = HDMI_AUDIO_SAMPLE_SIZE_20;
+		break;
+	case 24:
+	default:
+		frame.sample_size = HDMI_AUDIO_SAMPLE_SIZE_24;
+		break;
+	}
+
+	hdmi_audio_infoframe_pack_for_dp(&frame, &sdp, MTK_DP_VERSION);
+
+	mtk_dp_audio_sdp_asp_set_channels(mtk_dp, cfg->channels);
+	mtk_dp_setup_sdp_aui(mtk_dp, &sdp);
+}
+
+static void mtk_dp_audio_setup(struct mtk_dp *mtk_dp,
+			       struct mtk_dp_audio_cfg *cfg)
+{
+	mtk_dp_audio_sdp_setup(mtk_dp, cfg);
+	mtk_dp_audio_channel_status_set(mtk_dp, cfg);
+
+	mtk_dp_audio_setup_channels(mtk_dp, cfg);
+	mtk_dp_audio_set_divider(mtk_dp);
+}
+
+static int mtk_dp_video_config(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_config_mn_mode(mtk_dp);
+	mtk_dp_set_msa(mtk_dp);
+	mtk_dp_set_color_depth(mtk_dp);
+	return mtk_dp_set_color_format(mtk_dp, mtk_dp->info.format);
+}
+
+static void mtk_dp_init_port(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_set_idle_pattern(mtk_dp, true);
+	mtk_dp_initialize_priv_data(mtk_dp);
+
+	mtk_dp_initialize_settings(mtk_dp);
+	mtk_dp_initialize_aux_settings(mtk_dp);
+	mtk_dp_initialize_digital_settings(mtk_dp);
+
+	mtk_dp_update_bits(mtk_dp, MTK_DP_AUX_P0_3690,
+			   RX_REPLY_COMPLETE_MODE_AUX_TX_P0,
+			   RX_REPLY_COMPLETE_MODE_AUX_TX_P0);
+	mtk_dp_initialize_hpd_detect_settings(mtk_dp);
+
+	mtk_dp_digital_sw_reset(mtk_dp);
+}
+
+static irqreturn_t mtk_dp_hpd_event_thread(int hpd, void *dev)
+{
+	struct mtk_dp *mtk_dp = dev;
+	unsigned long flags;
+	u32 status;
+
+	if (mtk_dp->need_debounce && mtk_dp->train_info.cable_plugged_in)
+		msleep(100);
+
+	spin_lock_irqsave(&mtk_dp->irq_thread_lock, flags);
+	status = mtk_dp->irq_thread_handle;
+	mtk_dp->irq_thread_handle = 0;
+	spin_unlock_irqrestore(&mtk_dp->irq_thread_lock, flags);
+
+	if (status & MTK_DP_THREAD_CABLE_STATE_CHG) {
+		drm_helper_hpd_irq_event(mtk_dp->bridge.dev);
+
+		if (!mtk_dp->train_info.cable_plugged_in) {
+			mtk_dp_disable_sdp_aui(mtk_dp);
+			memset(&mtk_dp->info.audio_cur_cfg, 0,
+			       sizeof(mtk_dp->info.audio_cur_cfg));
+
+			mtk_dp->need_debounce = false;
+			mod_timer(&mtk_dp->debounce_timer,
+				  jiffies + msecs_to_jiffies(100) - 1);
+		}
+	}
+
+	if (status & MTK_DP_THREAD_HPD_EVENT)
+		dev_dbg(mtk_dp->dev, "Receive IRQ from sink devices\n");
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t mtk_dp_hpd_event(int hpd, void *dev)
+{
+	struct mtk_dp *mtk_dp = dev;
+	bool cable_sta_chg = false;
+	unsigned long flags;
+	u32 irq_status = mtk_dp_swirq_get_clear(mtk_dp) |
+			 mtk_dp_hwirq_get_clear(mtk_dp);
+
+	if (!irq_status)
+		return IRQ_HANDLED;
+
+	spin_lock_irqsave(&mtk_dp->irq_thread_lock, flags);
+
+	if (irq_status & MTK_DP_HPD_INTERRUPT)
+		mtk_dp->irq_thread_handle |= MTK_DP_THREAD_HPD_EVENT;
+
+	/* Cable state is changed. */
+	if (irq_status != MTK_DP_HPD_INTERRUPT) {
+		mtk_dp->irq_thread_handle |= MTK_DP_THREAD_CABLE_STATE_CHG;
+		cable_sta_chg = true;
+	}
+
+	spin_unlock_irqrestore(&mtk_dp->irq_thread_lock, flags);
+
+	if (cable_sta_chg) {
+		if (!!(mtk_dp_read(mtk_dp, MTK_DP_TRANS_P0_3414) &
+		       HPD_DB_DP_TRANS_P0_MASK))
+			mtk_dp->train_info.cable_plugged_in = true;
+		else
+			mtk_dp->train_info.cable_plugged_in = false;
+	}
+
+	return IRQ_WAKE_THREAD;
+}
+
+static int mtk_dp_dt_parse(struct mtk_dp *mtk_dp,
+			   struct platform_device *pdev)
+{
+	struct device_node *endpoint;
+	struct device *dev = &pdev->dev;
+	int ret;
+	void __iomem *base;
+	u32 linkrate;
+	int len;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	mtk_dp->regs = devm_regmap_init_mmio(dev, base, &mtk_dp_regmap_config);
+	if (IS_ERR(mtk_dp->regs))
+		return PTR_ERR(mtk_dp->regs);
+
+	endpoint = of_graph_get_endpoint_by_regs(pdev->dev.of_node, 1, -1);
+	len = of_property_count_elems_of_size(endpoint,
+					      "data-lanes", sizeof(u32));
+	if (len < 0 || len > 4 || len == 3) {
+		dev_err(dev, "invalid data lane size: %d\n", len);
+		return -EINVAL;
+	}
+
+	mtk_dp->max_lanes = len;
+
+	ret = device_property_read_u32(dev, "max-linkrate-mhz", &linkrate);
+	if (ret) {
+		dev_err(dev, "failed to read max linkrate: %d\n", ret);
+		return ret;
+	}
+
+	mtk_dp->max_linkrate = drm_dp_link_rate_to_bw_code(linkrate * 100);
+
+	return 0;
+}
+
+static void mtk_dp_update_plugged_status(struct mtk_dp *mtk_dp)
+{
+	mutex_lock(&mtk_dp->update_plugged_status_lock);
+	if (mtk_dp->plugged_cb && mtk_dp->codec_dev)
+		mtk_dp->plugged_cb(mtk_dp->codec_dev,
+				   mtk_dp->enabled &
+				   mtk_dp->info.audio_cur_cfg.detect_monitor);
+	mutex_unlock(&mtk_dp->update_plugged_status_lock);
+}
+
+static enum drm_connector_status mtk_dp_bdg_detect(struct drm_bridge *bridge)
+{
+	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
+	enum drm_connector_status ret = connector_status_disconnected;
+	bool enabled = mtk_dp->enabled;
+	u8 sink_count = 0;
+
+	if (!mtk_dp->train_info.cable_plugged_in)
+		return ret;
+
+	if (!enabled) {
+		/* power on aux */
+		mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_PWR_STATE,
+				   DP_PWR_STATE_BANDGAP_TPLL_LANE,
+				   DP_PWR_STATE_MASK);
+
+		/* power on panel */
+		drm_dp_dpcd_writeb(&mtk_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
+		usleep_range(2000, 5000);
+	}
+	/*
+	 * Some dongles still source HPD when they do not connect to any
+	 * sink device. To avoid this, we need to read the sink count
+	 * to make sure we do connect to sink devices. After this detect
+	 * function, we just need to check the HPD connection to check
+	 * whether we connect to a sink device.
+	 */
+	drm_dp_dpcd_readb(&mtk_dp->aux, DP_SINK_COUNT, &sink_count);
+	if (DP_GET_SINK_COUNT(sink_count))
+		ret = connector_status_connected;
+
+	if (!enabled) {
+		/* power off panel */
+		drm_dp_dpcd_writeb(&mtk_dp->aux, DP_SET_POWER, DP_SET_POWER_D3);
+		usleep_range(2000, 3000);
+
+		/* power off aux */
+		mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_PWR_STATE,
+				   DP_PWR_STATE_BANDGAP_TPLL,
+				   DP_PWR_STATE_MASK);
+	}
+
+	return ret;
+}
+
+static struct edid *mtk_dp_get_edid(struct drm_bridge *bridge,
+				    struct drm_connector *connector)
+{
+	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
+	bool enabled = mtk_dp->enabled;
+	struct edid *new_edid = NULL;
+	struct mtk_dp_audio_cfg *audio_caps = &mtk_dp->info.audio_cur_cfg;
+	struct cea_sad *sads;
+
+	if (!enabled) {
+		drm_bridge_chain_pre_enable(bridge);
+
+		/* power on aux */
+		mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_PWR_STATE,
+				   DP_PWR_STATE_BANDGAP_TPLL_LANE,
+				   DP_PWR_STATE_MASK);
+
+		/* power on panel */
+		drm_dp_dpcd_writeb(&mtk_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
+		usleep_range(2000, 5000);
+	}
+
+	new_edid = drm_get_edid(connector, &mtk_dp->aux.ddc);
+
+	/*
+	 * Parse capability here to let atomic_get_input_bus_fmts and
+	 * mode_valid use the capability to calculate sink bitrates.
+	 */
+	if (mtk_dp_parse_capabilities(mtk_dp)) {
+		drm_err(mtk_dp->drm_dev, "Can't parse capabilities\n");
+		new_edid = NULL;
+	}
+
+	if (new_edid) {
+		audio_caps->sad_count = drm_edid_to_sad(new_edid, &sads);
+		audio_caps->detect_monitor = drm_detect_monitor_audio(new_edid);
+	}
+
+	if (!enabled) {
+		/* power off panel */
+		drm_dp_dpcd_writeb(&mtk_dp->aux, DP_SET_POWER, DP_SET_POWER_D3);
+		usleep_range(2000, 3000);
+
+		/* power off aux */
+		mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_PWR_STATE,
+				   DP_PWR_STATE_BANDGAP_TPLL,
+				   DP_PWR_STATE_MASK);
+
+		drm_bridge_chain_post_disable(bridge);
+	}
+
+	return new_edid;
+}
+
+static ssize_t mtk_dp_aux_transfer(struct drm_dp_aux *mtk_aux,
+				   struct drm_dp_aux_msg *msg)
+{
+	struct mtk_dp *mtk_dp;
+	bool is_read;
+	u8 request;
+	size_t accessed_bytes = 0;
+	int ret;
+
+	mtk_dp = container_of(mtk_aux, struct mtk_dp, aux);
+
+	if (!mtk_dp->train_info.cable_plugged_in) {
+		ret = -EAGAIN;
+		goto err;
+	}
+
+	switch (msg->request) {
+	case DP_AUX_I2C_MOT:
+	case DP_AUX_I2C_WRITE:
+	case DP_AUX_NATIVE_WRITE:
+	case DP_AUX_I2C_WRITE_STATUS_UPDATE:
+	case DP_AUX_I2C_WRITE_STATUS_UPDATE | DP_AUX_I2C_MOT:
+		request = msg->request & ~DP_AUX_I2C_WRITE_STATUS_UPDATE;
+		is_read = false;
+		break;
+	case DP_AUX_I2C_READ:
+	case DP_AUX_NATIVE_READ:
+	case DP_AUX_I2C_READ | DP_AUX_I2C_MOT:
+		request = msg->request;
+		is_read = true;
+		break;
+	default:
+		drm_err(mtk_aux->drm_dev, "invalid aux cmd = %d\n",
+			msg->request);
+		ret = -EINVAL;
+		goto err;
+	}
+
+	do {
+		size_t to_access = min_t(size_t, DP_AUX_MAX_PAYLOAD_BYTES,
+					 msg->size - accessed_bytes);
+
+		ret = mtk_dp_aux_do_transfer(mtk_dp, is_read, request,
+					     msg->address + accessed_bytes,
+					     msg->buffer + accessed_bytes,
+					     to_access);
+
+		if (ret) {
+			drm_info(mtk_dp->drm_dev,
+				 "Failed to do AUX transfer: %d\n", ret);
+			goto err;
+		}
+		accessed_bytes += to_access;
+	} while (accessed_bytes < msg->size);
+
+	msg->reply = DP_AUX_NATIVE_REPLY_ACK | DP_AUX_I2C_REPLY_ACK;
+	return msg->size;
+err:
+	msg->reply = DP_AUX_NATIVE_REPLY_NACK | DP_AUX_I2C_REPLY_NACK;
+	return ret;
+}
+
+static int mtk_dp_poweron(struct mtk_dp *mtk_dp)
+{
+	int ret;
+
+	ret = phy_init(mtk_dp->phy);
+	if (ret) {
+		dev_err(mtk_dp->dev, "Failed to initialize phy: %d\n", ret);
+		return ret;
+	}
+
+	mtk_dp_init_port(mtk_dp);
+	mtk_dp_power_enable(mtk_dp);
+
+	return 0;
+}
+
+static void mtk_dp_poweroff(struct mtk_dp *mtk_dp)
+{
+	mtk_dp_power_disable(mtk_dp);
+	phy_exit(mtk_dp->phy);
+}
+
+static int mtk_dp_bridge_attach(struct drm_bridge *bridge,
+				enum drm_bridge_attach_flags flags)
+{
+	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
+	int ret;
+
+	if (!(flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)) {
+		dev_err(mtk_dp->dev, "Driver does not provide a connector!");
+		return -EINVAL;
+	}
+
+	mtk_dp->aux.drm_dev = bridge->dev;
+	ret = drm_dp_aux_register(&mtk_dp->aux);
+	if (ret) {
+		dev_err(mtk_dp->dev,
+			"failed to register DP AUX channel: %d\n", ret);
+		return ret;
+	}
+
+	ret = mtk_dp_poweron(mtk_dp);
+	if (ret)
+		goto err_aux_register;
+
+	if (mtk_dp->next_bridge) {
+		ret = drm_bridge_attach(bridge->encoder, mtk_dp->next_bridge,
+					&mtk_dp->bridge, flags);
+		if (ret) {
+			drm_warn(mtk_dp->drm_dev,
+				 "Failed to attach external bridge: %d\n", ret);
+			goto err_bridge_attach;
+		}
+	}
+
+	mtk_dp->drm_dev = bridge->dev;
+
+	mtk_dp_hwirq_enable(mtk_dp, true);
+
+	return 0;
+
+err_bridge_attach:
+	mtk_dp_poweroff(mtk_dp);
+err_aux_register:
+	drm_dp_aux_unregister(&mtk_dp->aux);
+	return ret;
+}
+
+static void mtk_dp_bridge_detach(struct drm_bridge *bridge)
+{
+	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
+
+	mtk_dp_hwirq_enable(mtk_dp, false);
+	mtk_dp->drm_dev = NULL;
+	mtk_dp_poweroff(mtk_dp);
+	drm_dp_aux_unregister(&mtk_dp->aux);
+}
+
+static void mtk_dp_bridge_atomic_enable(struct drm_bridge *bridge,
+					struct drm_bridge_state *old_state)
+{
+	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
+	int ret;
+
+	mtk_dp->conn = drm_atomic_get_new_connector_for_encoder(old_state->base.state,
+								bridge->encoder);
+	if (!mtk_dp->conn) {
+		drm_err(mtk_dp->drm_dev,
+			"Can't enable bridge as connector is missing\n");
+		return;
+	}
+
+	/* power on aux */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_PWR_STATE,
+			   DP_PWR_STATE_BANDGAP_TPLL_LANE,
+			   DP_PWR_STATE_MASK);
+
+	if (mtk_dp->train_info.cable_plugged_in) {
+		drm_dp_dpcd_writeb(&mtk_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
+		usleep_range(2000, 5000);
+	}
+
+	/* Training */
+	ret = mtk_dp_training(mtk_dp);
+	if (ret) {
+		drm_err(mtk_dp->drm_dev, "Training failed, %d\n", ret);
+		goto power_off_aux;
+	}
+
+	ret = mtk_dp_video_config(mtk_dp);
+	if (ret)
+		goto power_off_aux;
+
+	mtk_dp_video_enable(mtk_dp, true);
+
+	mtk_dp->audio_enable =
+		mtk_dp_edid_parse_audio_capabilities(mtk_dp,
+						     &mtk_dp->info.audio_cur_cfg);
+	if (mtk_dp->audio_enable) {
+		mtk_dp_audio_setup(mtk_dp, &mtk_dp->info.audio_cur_cfg);
+		mtk_dp_audio_mute(mtk_dp, false);
+	} else {
+		memset(&mtk_dp->info.audio_cur_cfg, 0,
+		       sizeof(mtk_dp->info.audio_cur_cfg));
+	}
+
+	mtk_dp->enabled = true;
+	mtk_dp_update_plugged_status(mtk_dp);
+
+	return;
+power_off_aux:
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_PWR_STATE,
+			   DP_PWR_STATE_BANDGAP_TPLL,
+			   DP_PWR_STATE_MASK);
+}
+
+static void mtk_dp_bridge_atomic_disable(struct drm_bridge *bridge,
+					 struct drm_bridge_state *old_state)
+{
+	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
+
+	mtk_dp->enabled = false;
+	mtk_dp_update_plugged_status(mtk_dp);
+	mtk_dp_video_enable(mtk_dp, false);
+	mtk_dp_audio_mute(mtk_dp, true);
+
+	if (mtk_dp->train_info.cable_plugged_in) {
+		drm_dp_dpcd_writeb(&mtk_dp->aux, DP_SET_POWER, DP_SET_POWER_D3);
+		usleep_range(2000, 3000);
+	}
+
+	/* power off aux */
+	mtk_dp_update_bits(mtk_dp, MTK_DP_TOP_PWR_STATE,
+			   DP_PWR_STATE_BANDGAP_TPLL,
+			   DP_PWR_STATE_MASK);
+
+	/* Ensure the sink is muted */
+	msleep(20);
+}
+
+static enum drm_mode_status
+mtk_dp_bridge_mode_valid(struct drm_bridge *bridge,
+			 const struct drm_display_info *info,
+			 const struct drm_display_mode *mode)
+{
+	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
+	u32 bpp = info->color_formats & DRM_COLOR_FORMAT_YCBCR422 ? 16 : 24;
+	u32 rate = min_t(u32, drm_dp_max_link_rate(mtk_dp->rx_cap) *
+			      drm_dp_max_lane_count(mtk_dp->rx_cap),
+			 drm_dp_bw_code_to_link_rate(mtk_dp->max_linkrate) *
+			 mtk_dp->max_lanes);
+
+	if (rate < mode->clock * bpp / 8)
+		return MODE_CLOCK_HIGH;
+
+	return MODE_OK;
+}
+
+static u32 *mtk_dp_bridge_atomic_get_output_bus_fmts(struct drm_bridge *bridge,
+						     struct drm_bridge_state *bridge_state,
+						     struct drm_crtc_state *crtc_state,
+						     struct drm_connector_state *conn_state,
+						     unsigned int *num_output_fmts)
+{
+	u32 *output_fmts;
+
+	*num_output_fmts = 0;
+	output_fmts = kmalloc(sizeof(*output_fmts), GFP_KERNEL);
+	if (!output_fmts)
+		return NULL;
+	*num_output_fmts = 1;
+	output_fmts[0] = MEDIA_BUS_FMT_FIXED;
+	return output_fmts;
+}
+
+static const u32 mt8195_input_fmts[] = {
+	MEDIA_BUS_FMT_RGB888_1X24,
+	MEDIA_BUS_FMT_YUV8_1X24,
+	MEDIA_BUS_FMT_YUYV8_1X16,
+};
+
+static u32 *mtk_dp_bridge_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
+						    struct drm_bridge_state *bridge_state,
+						    struct drm_crtc_state *crtc_state,
+						    struct drm_connector_state *conn_state,
+						    u32 output_fmt,
+						    unsigned int *num_input_fmts)
+{
+	u32 *input_fmts;
+	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
+	struct drm_display_mode *mode = &crtc_state->adjusted_mode;
+	struct drm_display_info *display_info =
+		&conn_state->connector->display_info;
+	u32 rate = min_t(u32, drm_dp_max_link_rate(mtk_dp->rx_cap) *
+			      drm_dp_max_lane_count(mtk_dp->rx_cap),
+			 drm_dp_bw_code_to_link_rate(mtk_dp->max_linkrate) *
+			 mtk_dp->max_lanes);
+
+	*num_input_fmts = 0;
+
+	/*
+	 * If the linkrate is smaller than datarate of RGB888, larger than
+	 * datarate of YUV422 and sink device supports YUV422, we output YUV422
+	 * format. Use this condition, we can support more resolution.
+	 */
+	if ((rate < (mode->clock * 24 / 8)) &&
+	    (rate > (mode->clock * 16 / 8)) &&
+	    (display_info->color_formats & DRM_COLOR_FORMAT_YCBCR422)) {
+		input_fmts = kcalloc(1, sizeof(*input_fmts), GFP_KERNEL);
+		if (!input_fmts)
+			return NULL;
+		*num_input_fmts = 1;
+		input_fmts[0] = MEDIA_BUS_FMT_YUYV8_1X16;
+	} else {
+		input_fmts = kcalloc(ARRAY_SIZE(mt8195_input_fmts),
+				     sizeof(*input_fmts),
+				     GFP_KERNEL);
+		if (!input_fmts)
+			return NULL;
+
+		*num_input_fmts = ARRAY_SIZE(mt8195_input_fmts);
+		memcpy(input_fmts, mt8195_input_fmts, sizeof(mt8195_input_fmts));
+	}
+
+	return input_fmts;
+}
+
+static int mtk_dp_bridge_atomic_check(struct drm_bridge *bridge,
+				      struct drm_bridge_state *bridge_state,
+				      struct drm_crtc_state *crtc_state,
+				      struct drm_connector_state *conn_state)
+{
+	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
+	struct drm_crtc *crtc = conn_state->crtc;
+	unsigned int input_bus_format;
+
+	input_bus_format = bridge_state->input_bus_cfg.format;
+
+	dev_dbg(mtk_dp->dev, "input format 0x%04x, output format 0x%04x\n",
+		bridge_state->input_bus_cfg.format,
+		 bridge_state->output_bus_cfg.format);
+
+	if (input_bus_format == MEDIA_BUS_FMT_YUYV8_1X16)
+		mtk_dp->info.format = DP_PIXELFORMAT_YUV422;
+	else
+		mtk_dp->info.format = DP_PIXELFORMAT_RGB;
+
+	if (!crtc) {
+		drm_err(mtk_dp->drm_dev,
+			"Can't enable bridge as connector state doesn't have a crtc\n");
+		return -EINVAL;
+	}
+
+	drm_display_mode_to_videomode(&crtc_state->adjusted_mode, &mtk_dp->info.vm);
+
+	return 0;
+}
+
+static const struct drm_bridge_funcs mtk_dp_bridge_funcs = {
+	.atomic_check = mtk_dp_bridge_atomic_check,
+	.atomic_duplicate_state = drm_atomic_helper_bridge_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_bridge_destroy_state,
+	.atomic_get_output_bus_fmts = mtk_dp_bridge_atomic_get_output_bus_fmts,
+	.atomic_get_input_bus_fmts = mtk_dp_bridge_atomic_get_input_bus_fmts,
+	.atomic_reset = drm_atomic_helper_bridge_reset,
+	.attach = mtk_dp_bridge_attach,
+	.detach = mtk_dp_bridge_detach,
+	.atomic_enable = mtk_dp_bridge_atomic_enable,
+	.atomic_disable = mtk_dp_bridge_atomic_disable,
+	.mode_valid = mtk_dp_bridge_mode_valid,
+	.get_edid = mtk_dp_get_edid,
+	.detect = mtk_dp_bdg_detect,
+};
+
+static void mtk_dp_debounce_timer(struct timer_list *t)
+{
+	struct mtk_dp *mtk_dp = from_timer(mtk_dp, t, debounce_timer);
+
+	mtk_dp->need_debounce = true;
+}
+
+/*
+ * HDMI audio codec callbacks
+ */
+static int mtk_dp_audio_hw_params(struct device *dev, void *data,
+				  struct hdmi_codec_daifmt *daifmt,
+				  struct hdmi_codec_params *params)
+{
+	struct mtk_dp *mtk_dp = dev_get_drvdata(dev);
+
+	if (!mtk_dp->enabled) {
+		dev_err(mtk_dp->dev, "%s, DP is not ready!\n", __func__);
+		return -ENODEV;
+	}
+
+	mtk_dp->info.audio_cur_cfg.channels = params->cea.channels;
+	mtk_dp->info.audio_cur_cfg.sample_rate = params->sample_rate;
+
+	mtk_dp_audio_setup(mtk_dp, &mtk_dp->info.audio_cur_cfg);
+
+	return 0;
+}
+
+static int mtk_dp_audio_startup(struct device *dev, void *data)
+{
+	struct mtk_dp *mtk_dp = dev_get_drvdata(dev);
+
+	mtk_dp_audio_mute(mtk_dp, false);
+
+	return 0;
+}
+
+static void mtk_dp_audio_shutdown(struct device *dev, void *data)
+{
+	struct mtk_dp *mtk_dp = dev_get_drvdata(dev);
+
+	mtk_dp_audio_mute(mtk_dp, true);
+}
+
+static int mtk_dp_audio_get_eld(struct device *dev, void *data, uint8_t *buf,
+				size_t len)
+{
+	struct mtk_dp *mtk_dp = dev_get_drvdata(dev);
+
+	if (mtk_dp->enabled)
+		memcpy(buf, mtk_dp->conn->eld, len);
+	else
+		memset(buf, 0, len);
+
+	return 0;
+}
+
+static int mtk_dp_audio_hook_plugged_cb(struct device *dev, void *data,
+					hdmi_codec_plugged_cb fn,
+					struct device *codec_dev)
+{
+	struct mtk_dp *mtk_dp = data;
+
+	mutex_lock(&mtk_dp->update_plugged_status_lock);
+	mtk_dp->plugged_cb = fn;
+	mtk_dp->codec_dev = codec_dev;
+	mutex_unlock(&mtk_dp->update_plugged_status_lock);
+
+	mtk_dp_update_plugged_status(mtk_dp);
+
+	return 0;
+}
+
+static const struct hdmi_codec_ops mtk_dp_audio_codec_ops = {
+	.hw_params = mtk_dp_audio_hw_params,
+	.audio_startup = mtk_dp_audio_startup,
+	.audio_shutdown = mtk_dp_audio_shutdown,
+	.get_eld = mtk_dp_audio_get_eld,
+	.hook_plugged_cb = mtk_dp_audio_hook_plugged_cb,
+	.no_capture_mute = 1,
+};
+
+static int mtk_dp_register_audio_driver(struct device *dev)
+{
+	struct mtk_dp *mtk_dp = dev_get_drvdata(dev);
+	struct hdmi_codec_pdata codec_data = {
+		.ops = &mtk_dp_audio_codec_ops,
+		.max_i2s_channels = 8,
+		.i2s = 1,
+		.data = mtk_dp,
+	};
+
+	mtk_dp->audio_pdev = platform_device_register_data(dev,
+							   HDMI_CODEC_DRV_NAME,
+							   PLATFORM_DEVID_AUTO,
+							   &codec_data,
+							   sizeof(codec_data));
+	return PTR_ERR_OR_ZERO(mtk_dp->audio_pdev);
+}
+
+static int mtk_dp_probe(struct platform_device *pdev)
+{
+	struct mtk_dp *mtk_dp;
+	struct device *dev = &pdev->dev;
+	int ret, irq_num;
+
+	mtk_dp = devm_kzalloc(dev, sizeof(*mtk_dp), GFP_KERNEL);
+	if (!mtk_dp)
+		return -ENOMEM;
+
+	mtk_dp->dev = dev;
+	mtk_dp->data = (struct mtk_dp_data *)of_device_get_match_data(dev);
+
+	irq_num = platform_get_irq(pdev, 0);
+	if (irq_num < 0)
+		return dev_err_probe(dev, irq_num,
+				     "failed to request dp irq resource\n");
+
+	mtk_dp->next_bridge = devm_drm_of_get_bridge(dev, dev->of_node, 1, 0);
+	if (IS_ERR(mtk_dp->next_bridge) &&
+	    PTR_ERR(mtk_dp->next_bridge) == -ENODEV)
+		mtk_dp->next_bridge = NULL;
+	else if (IS_ERR(mtk_dp->next_bridge))
+		return dev_err_probe(dev, PTR_ERR(mtk_dp->next_bridge),
+				     "Failed to get bridge\n");
+
+	ret = mtk_dp_dt_parse(mtk_dp, pdev);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to parse dt\n");
+
+	drm_dp_aux_init(&mtk_dp->aux);
+	mtk_dp->aux.name = "aux_mtk_dp";
+	mtk_dp->aux.transfer = mtk_dp_aux_transfer;
+
+	spin_lock_init(&mtk_dp->irq_thread_lock);
+
+	ret = devm_request_threaded_irq(dev, irq_num, mtk_dp_hpd_event,
+					mtk_dp_hpd_event_thread,
+					IRQ_TYPE_LEVEL_HIGH, dev_name(dev),
+					mtk_dp);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to request mediatek dptx irq\n");
+
+	mutex_init(&mtk_dp->update_plugged_status_lock);
+
+	platform_set_drvdata(pdev, mtk_dp);
+
+	if (mtk_dp->data->audio_supported) {
+		ret = mtk_dp_register_audio_driver(dev);
+		if (ret) {
+			dev_err(dev, "Failed to register audio driver: %d\n",
+				ret);
+			return ret;
+		}
+	}
+
+	mtk_dp->phy_dev = platform_device_register_data(dev, "mediatek-dp-phy",
+							PLATFORM_DEVID_AUTO,
+							&mtk_dp->regs,
+							sizeof(struct regmap *));
+	if (IS_ERR(mtk_dp->phy_dev))
+		return dev_err_probe(dev, PTR_ERR(mtk_dp->phy_dev),
+				     "Failed to create device mediatek-dp-phy\n");
+
+	mtk_dp_get_calibration_data(mtk_dp);
+
+	mtk_dp->phy = devm_phy_get(&mtk_dp->phy_dev->dev, "dp");
+
+	if (IS_ERR(mtk_dp->phy)) {
+		platform_device_unregister(mtk_dp->phy_dev);
+		return dev_err_probe(dev, PTR_ERR(mtk_dp->phy),
+				     "Failed to get phy\n");
+	}
+
+	mtk_dp->bridge.funcs = &mtk_dp_bridge_funcs;
+	mtk_dp->bridge.of_node = dev->of_node;
+
+	mtk_dp->bridge.ops =
+		DRM_BRIDGE_OP_DETECT | DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_HPD;
+	mtk_dp->bridge.type = mtk_dp->data->bridge_type;
+
+	drm_bridge_add(&mtk_dp->bridge);
+
+	mtk_dp->need_debounce = true;
+	timer_setup(&mtk_dp->debounce_timer, mtk_dp_debounce_timer, 0);
+
+	pm_runtime_enable(dev);
+	pm_runtime_get_sync(dev);
+
+	return 0;
+}
+
+static int mtk_dp_remove(struct platform_device *pdev)
+{
+	struct mtk_dp *mtk_dp = platform_get_drvdata(pdev);
+
+	pm_runtime_put(&pdev->dev);
+	pm_runtime_disable(&pdev->dev);
+	del_timer_sync(&mtk_dp->debounce_timer);
+	drm_bridge_remove(&mtk_dp->bridge);
+	platform_device_unregister(mtk_dp->phy_dev);
+	if (mtk_dp->audio_pdev)
+		platform_device_unregister(mtk_dp->audio_pdev);
+
+	return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int mtk_dp_suspend(struct device *dev)
+{
+	struct mtk_dp *mtk_dp = dev_get_drvdata(dev);
+
+	mtk_dp_power_disable(mtk_dp);
+	mtk_dp_hwirq_enable(mtk_dp, false);
+	pm_runtime_put_sync(dev);
+
+	return 0;
+}
+
+static int mtk_dp_resume(struct device *dev)
+{
+	struct mtk_dp *mtk_dp = dev_get_drvdata(dev);
+
+	pm_runtime_get_sync(dev);
+	mtk_dp_init_port(mtk_dp);
+	mtk_dp_hwirq_enable(mtk_dp, true);
+	mtk_dp_power_enable(mtk_dp);
+
+	return 0;
+}
+#endif
+
+static SIMPLE_DEV_PM_OPS(mtk_dp_pm_ops, mtk_dp_suspend, mtk_dp_resume);
+
+static const struct mtk_dp_data mt8195_edp_data = {
+	.bridge_type = DRM_MODE_CONNECTOR_eDP,
+	.smc_cmd = MTK_DP_SIP_ATF_EDP_VIDEO_UNMUTE,
+	.efuse_fmt = mt8195_edp_efuse_fmt,
+	.audio_supported = false,
+};
+
+static const struct mtk_dp_data mt8195_dp_data = {
+	.bridge_type = DRM_MODE_CONNECTOR_DisplayPort,
+	.smc_cmd = MTK_DP_SIP_ATF_VIDEO_UNMUTE,
+	.efuse_fmt = mt8195_dp_efuse_fmt,
+	.audio_supported = true,
+};
+
+static const struct of_device_id mtk_dp_of_match[] = {
+	{
+		.compatible = "mediatek,mt8195-edp-tx",
+		.data = &mt8195_edp_data,
+	},
+	{
+		.compatible = "mediatek,mt8195-dp-tx",
+		.data = &mt8195_dp_data,
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, mtk_dp_of_match);
+
+static struct platform_driver mtk_dp_driver = {
+	.probe = mtk_dp_probe,
+	.remove = mtk_dp_remove,
+	.driver = {
+		.name = "mediatek-drm-dp",
+		.of_match_table = mtk_dp_of_match,
+		.pm = &mtk_dp_pm_ops,
+	},
+};
+
+module_platform_driver(mtk_dp_driver);
+
+MODULE_AUTHOR("Jitao Shi <jitao.shi@mediatek.com>");
+MODULE_AUTHOR("Markus Schneider-Pargmann <msp@baylibre.com>");
+MODULE_AUTHOR("Bo-Chen Chen <rex-bc.chen@mediatek.com>");
+MODULE_DESCRIPTION("MediaTek DisplayPort Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/mediatek/mtk_dp_reg.h b/drivers/gpu/drm/mediatek/mtk_dp_reg.h
new file mode 100644
index 000000000000..84e38cef03c2
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_dp_reg.h
@@ -0,0 +1,350 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2019-2022 MediaTek Inc.
+ * Copyright (c) 2022 BayLibre
+ */
+#ifndef _MTK_DP_REG_H_
+#define _MTK_DP_REG_H_
+
+#define SEC_OFFSET	0x4000
+
+#define MTK_DP_HPD_DISCONNECT		BIT(1)
+#define MTK_DP_HPD_CONNECT		BIT(2)
+#define MTK_DP_HPD_INTERRUPT		BIT(3)
+
+/* offset: 0x0 */
+#define DP_PHY_GLB_BIAS_GEN_00		0x0
+#define RG_XTP_GLB_BIAS_INTR_CTRL		GENMASK(20, 16)
+#define DP_PHY_GLB_DPAUX_TX		0x8
+#define RG_CKM_PT0_CKTX_IMPSEL			GENMASK(23, 20)
+#define MTK_DP_0034			0x34
+#define DA_XTP_GLB_CKDET_EN_FORCE_VAL		BIT(15)
+#define DA_XTP_GLB_CKDET_EN_FORCE_EN		BIT(14)
+#define DA_CKM_INTCKTX_EN_FORCE_VAL		BIT(13)
+#define DA_CKM_INTCKTX_EN_FORCE_EN		BIT(12)
+#define DA_CKM_CKTX0_EN_FORCE_VAL		BIT(11)
+#define DA_CKM_CKTX0_EN_FORCE_EN		BIT(10)
+#define DA_CKM_XTAL_CK_FORCE_VAL		BIT(9)
+#define DA_CKM_XTAL_CK_FORCE_EN			BIT(8)
+#define DA_CKM_BIAS_LPF_EN_FORCE_VAL		BIT(7)
+#define DA_CKM_BIAS_LPF_EN_FORCE_EN		BIT(6)
+#define DA_CKM_BIAS_EN_FORCE_VAL		BIT(5)
+#define DA_CKM_BIAS_EN_FORCE_EN			BIT(4)
+#define DA_XTP_GLB_AVD10_ON_FORCE_VAL		BIT(3)
+#define DA_XTP_GLB_AVD10_ON_FORCE		BIT(2)
+#define DA_XTP_GLB_LDO_EN_FORCE_VAL		BIT(1)
+#define DA_XTP_GLB_LDO_EN_FORCE_EN		BIT(0)
+#define DP_PHY_LANE_TX_0		0x104
+#define RG_XTP_LN0_TX_IMPSEL_PMOS		GENMASK(15, 12)
+#define RG_XTP_LN0_TX_IMPSEL_NMOS		GENMASK(19, 16)
+#define DP_PHY_LANE_TX_1		0x204
+#define RG_XTP_LN1_TX_IMPSEL_PMOS		GENMASK(15, 12)
+#define RG_XTP_LN1_TX_IMPSEL_NMOS		GENMASK(19, 16)
+#define DP_PHY_LANE_TX_2		0x304
+#define RG_XTP_LN2_TX_IMPSEL_PMOS		GENMASK(15, 12)
+#define RG_XTP_LN2_TX_IMPSEL_NMOS		GENMASK(19, 16)
+#define DP_PHY_LANE_TX_3		0x404
+#define RG_XTP_LN3_TX_IMPSEL_PMOS		GENMASK(15, 12)
+#define RG_XTP_LN3_TX_IMPSEL_NMOS		GENMASK(19, 16)
+#define MTK_DP_1040			0x1040
+#define RG_DPAUX_RX_VALID_DEGLITCH_EN		BIT(2)
+#define RG_XTP_GLB_CKDET_EN			BIT(1)
+#define RG_DPAUX_RX_EN				BIT(0)
+
+/* offset: TOP_OFFSET (0x2000) */
+#define MTK_DP_TOP_PWR_STATE		0x2000
+#define DP_PWR_STATE_MASK			GENMASK(1, 0)
+#define DP_PWR_STATE_BANDGAP			BIT(0)
+#define DP_PWR_STATE_BANDGAP_TPLL		BIT(1)
+#define DP_PWR_STATE_BANDGAP_TPLL_LANE		GENMASK(1, 0)
+#define MTK_DP_TOP_SWING_EMP		0x2004
+#define DP_TX0_VOLT_SWING_MASK			GENMASK(1, 0)
+#define DP_TX0_VOLT_SWING_SHIFT			0
+#define DP_TX0_PRE_EMPH_MASK			GENMASK(3, 2)
+#define DP_TX0_PRE_EMPH_SHIFT			2
+#define DP_TX1_VOLT_SWING_MASK			GENMASK(9, 8)
+#define DP_TX1_VOLT_SWING_SHIFT			8
+#define DP_TX1_PRE_EMPH_MASK			GENMASK(11, 10)
+#define DP_TX2_VOLT_SWING_MASK			GENMASK(17, 16)
+#define DP_TX2_PRE_EMPH_MASK			GENMASK(19, 18)
+#define DP_TX3_VOLT_SWING_MASK			GENMASK(25, 24)
+#define DP_TX3_PRE_EMPH_MASK			GENMASK(27, 26)
+#define MTK_DP_TOP_RESET_AND_PROBE	0x2020
+#define SW_RST_B_PHYD				BIT(4)
+#define MTK_DP_TOP_IRQ_MASK		0x202c
+#define IRQ_MASK_AUX_TOP_IRQ			BIT(2)
+#define MTK_DP_TOP_MEM_PD		0x2038
+#define MEM_ISO_EN				BIT(0)
+#define FUSE_SEL				BIT(2)
+
+/* offset: ENC0_OFFSET (0x3000) */
+#define MTK_DP_ENC0_P0_3000			0x3000
+#define LANE_NUM_DP_ENC0_P0_MASK			GENMASK(1, 0)
+#define VIDEO_MUTE_SW_DP_ENC0_P0			BIT(2)
+#define VIDEO_MUTE_SEL_DP_ENC0_P0			BIT(3)
+#define ENHANCED_FRAME_EN_DP_ENC0_P0			BIT(4)
+#define MTK_DP_ENC0_P0_3004			0x3004
+#define VIDEO_M_CODE_SEL_DP_ENC0_P0_MASK		BIT(8)
+#define DP_TX_ENCODER_4P_RESET_SW_DP_ENC0_P0		BIT(9)
+#define MTK_DP_ENC0_P0_3010			0x3010
+#define HTOTAL_SW_DP_ENC0_P0_MASK			GENMASK(15, 0)
+#define MTK_DP_ENC0_P0_3014			0x3014
+#define VTOTAL_SW_DP_ENC0_P0_MASK			GENMASK(15, 0)
+#define MTK_DP_ENC0_P0_3018			0x3018
+#define HSTART_SW_DP_ENC0_P0_MASK			GENMASK(15, 0)
+#define MTK_DP_ENC0_P0_301C			0x301c
+#define VSTART_SW_DP_ENC0_P0_MASK			GENMASK(15, 0)
+#define MTK_DP_ENC0_P0_3020			0x3020
+#define HWIDTH_SW_DP_ENC0_P0_MASK			GENMASK(15, 0)
+#define MTK_DP_ENC0_P0_3024			0x3024
+#define VHEIGHT_SW_DP_ENC0_P0_MASK			GENMASK(15, 0)
+#define MTK_DP_ENC0_P0_3028			0x3028
+#define HSW_SW_DP_ENC0_P0_MASK				GENMASK(14, 0)
+#define HSP_SW_DP_ENC0_P0_MASK				BIT(15)
+#define MTK_DP_ENC0_P0_302C			0x302c
+#define VSW_SW_DP_ENC0_P0_MASK				GENMASK(14, 0)
+#define VSP_SW_DP_ENC0_P0_MASK				BIT(15)
+#define MTK_DP_ENC0_P0_3030			0x3030
+#define HTOTAL_SEL_DP_ENC0_P0				BIT(0)
+#define VTOTAL_SEL_DP_ENC0_P0				BIT(1)
+#define HSTART_SEL_DP_ENC0_P0				BIT(2)
+#define VSTART_SEL_DP_ENC0_P0				BIT(3)
+#define HWIDTH_SEL_DP_ENC0_P0				BIT(4)
+#define VHEIGHT_SEL_DP_ENC0_P0				BIT(5)
+#define HSP_SEL_DP_ENC0_P0				BIT(6)
+#define HSW_SEL_DP_ENC0_P0				BIT(7)
+#define VSP_SEL_DP_ENC0_P0				BIT(8)
+#define VSW_SEL_DP_ENC0_P0				BIT(9)
+#define VBID_AUDIO_MUTE_FLAG_SW_DP_ENC0_P0		BIT(11)
+#define VBID_AUDIO_MUTE_FLAG_SEL_DP_ENC0_P0		BIT(12)
+#define MTK_DP_ENC0_P0_3034			0x3034
+#define MTK_DP_ENC0_P0_3038			0x3038
+#define VIDEO_SOURCE_SEL_DP_ENC0_P0_MASK		BIT(11)
+#define MTK_DP_ENC0_P0_303C			0x303c
+#define SRAM_START_READ_THRD_DP_ENC0_P0_MASK		GENMASK(5, 0)
+#define VIDEO_COLOR_DEPTH_DP_ENC0_P0_MASK		GENMASK(10, 8)
+#define VIDEO_COLOR_DEPTH_DP_ENC0_P0_16BIT		(0 << 8)
+#define VIDEO_COLOR_DEPTH_DP_ENC0_P0_12BIT		(1 << 8)
+#define VIDEO_COLOR_DEPTH_DP_ENC0_P0_10BIT		(2 << 8)
+#define VIDEO_COLOR_DEPTH_DP_ENC0_P0_8BIT		(3 << 8)
+#define VIDEO_COLOR_DEPTH_DP_ENC0_P0_6BIT		(4 << 8)
+#define PIXEL_ENCODE_FORMAT_DP_ENC0_P0_MASK		GENMASK(14, 12)
+#define PIXEL_ENCODE_FORMAT_DP_ENC0_P0_RGB		(0 << 12)
+#define PIXEL_ENCODE_FORMAT_DP_ENC0_P0_YCBCR422		(1 << 12)
+#define PIXEL_ENCODE_FORMAT_DP_ENC0_P0_YCBCR420		(2 << 12)
+#define VIDEO_MN_GEN_EN_DP_ENC0_P0			BIT(15)
+#define MTK_DP_ENC0_P0_3040			0x3040
+#define SDP_DOWN_CNT_DP_ENC0_P0_VAL			0x20
+#define SDP_DOWN_CNT_INIT_DP_ENC0_P0_MASK		GENMASK(11, 0)
+#define MTK_DP_ENC0_P0_304C			0x304c
+#define VBID_VIDEO_MUTE_DP_ENC0_P0_MASK			BIT(2)
+#define SDP_VSYNC_RISING_MASK_DP_ENC0_P0_MASK		BIT(8)
+#define MTK_DP_ENC0_P0_3064			0x3064
+#define HDE_NUM_LAST_DP_ENC0_P0_MASK			GENMASK(15, 0)
+#define MTK_DP_ENC0_P0_3088			0x3088
+#define AU_EN_DP_ENC0_P0				BIT(6)
+#define AUDIO_8CH_EN_DP_ENC0_P0_MASK			BIT(7)
+#define AUDIO_8CH_SEL_DP_ENC0_P0_MASK			BIT(8)
+#define AUDIO_2CH_EN_DP_ENC0_P0_MASK			BIT(14)
+#define AUDIO_2CH_SEL_DP_ENC0_P0_MASK			BIT(15)
+#define MTK_DP_ENC0_P0_308C			0x308c
+#define CH_STATUS_0_DP_ENC0_P0_MASK			GENMASK(15, 0)
+#define MTK_DP_ENC0_P0_3090			0x3090
+#define CH_STATUS_1_DP_ENC0_P0_MASK			GENMASK(15, 0)
+#define MTK_DP_ENC0_P0_3094			0x3094
+#define CH_STATUS_2_DP_ENC0_P0_MASK			GENMASK(7, 0)
+#define MTK_DP_ENC0_P0_30A4			0x30a4
+#define AU_TS_CFG_DP_ENC0_P0_MASK			GENMASK(7, 0)
+#define MTK_DP_ENC0_P0_30A8			0x30a8
+#define MTK_DP_ENC0_P0_30BC			0x30bc
+#define ISRC_CONT_DP_ENC0_P0				BIT(0)
+#define AUDIO_M_CODE_MULT_DIV_SEL_DP_ENC0_P0_MASK	GENMASK(10, 8)
+#define AUDIO_M_CODE_MULT_DIV_SEL_DP_ENC0_P0_MUL_2	(1 << 8)
+#define AUDIO_M_CODE_MULT_DIV_SEL_DP_ENC0_P0_MUL_4	(2 << 8)
+#define AUDIO_M_CODE_MULT_DIV_SEL_DP_ENC0_P0_MUL_8	(3 << 8)
+#define AUDIO_M_CODE_MULT_DIV_SEL_DP_ENC0_P0_DIV_2	(5 << 8)
+#define AUDIO_M_CODE_MULT_DIV_SEL_DP_ENC0_P0_DIV_4	(6 << 8)
+#define AUDIO_M_CODE_MULT_DIV_SEL_DP_ENC0_P0_DIV_8	(7 << 8)
+#define MTK_DP_ENC0_P0_30D8			0x30d8
+#define MTK_DP_ENC0_P0_312C			0x312c
+#define ASP_HB2_DP_ENC0_P0_MASK				GENMASK(7, 0)
+#define ASP_HB3_DP_ENC0_P0_MASK				GENMASK(15, 8)
+#define MTK_DP_ENC0_P0_3154			0x3154
+#define PGEN_HTOTAL_DP_ENC0_P0_MASK			GENMASK(13, 0)
+#define MTK_DP_ENC0_P0_3158			0x3158
+#define PGEN_HSYNC_RISING_DP_ENC0_P0_MASK		GENMASK(13, 0)
+#define MTK_DP_ENC0_P0_315C			0x315c
+#define PGEN_HSYNC_PULSE_WIDTH_DP_ENC0_P0_MASK		GENMASK(13, 0)
+#define MTK_DP_ENC0_P0_3160			0x3160
+#define PGEN_HFDE_START_DP_ENC0_P0_MASK			GENMASK(13, 0)
+#define MTK_DP_ENC0_P0_3164			0x3164
+#define PGEN_HFDE_ACTIVE_WIDTH_DP_ENC0_P0_MASK		GENMASK(13, 0)
+#define MTK_DP_ENC0_P0_3168			0x3168
+#define PGEN_VTOTAL_DP_ENC0_P0_MASK			GENMASK(12, 0)
+#define MTK_DP_ENC0_P0_316C			0x316c
+#define PGEN_VSYNC_RISING_DP_ENC0_P0_MASK		GENMASK(12, 0)
+#define MTK_DP_ENC0_P0_3170			0x3170
+#define PGEN_VSYNC_PULSE_WIDTH_DP_ENC0_P0_MASK		GENMASK(12, 0)
+#define MTK_DP_ENC0_P0_3174			0x3174
+#define PGEN_VFDE_START_DP_ENC0_P0_MASK			GENMASK(12, 0)
+#define MTK_DP_ENC0_P0_3178			0x3178
+#define PGEN_VFDE_ACTIVE_WIDTH_DP_ENC0_P0_MASK		GENMASK(12, 0)
+#define MTK_DP_ENC0_P0_31B0			0x31b0
+#define PGEN_PATTERN_SEL_VAL				4
+#define PGEN_PATTERN_SEL_MASK				GENMASK(6, 4)
+#define MTK_DP_ENC0_P0_31EC			0x31ec
+#define AUDIO_CH_SRC_SEL_DP_ENC0_P0			BIT(4)
+#define ISRC1_HB3_DP_ENC0_P0_MASK			GENMASK(15, 8)
+
+/* offset: ENC1_OFFSET (0x3200) */
+#define MTK_DP_ENC1_P0_3200			0x3200
+#define MTK_DP_ENC1_P0_3280			0x3280
+#define SDP_PACKET_TYPE_DP_ENC1_P0_MASK			GENMASK(4, 0)
+#define SDP_PACKET_W_DP_ENC1_P0				BIT(5)
+#define SDP_PACKET_W_DP_ENC1_P0_MASK			BIT(5)
+#define MTK_DP_ENC1_P0_3300			0x3300
+#define VIDEO_AFIFO_RDY_SEL_DP_ENC1_P0_VAL		2
+#define VIDEO_AFIFO_RDY_SEL_DP_ENC1_P0_MASK		GENMASK(9, 8)
+#define MTK_DP_ENC1_P0_3304			0x3304
+#define AU_PRTY_REGEN_DP_ENC1_P0_MASK			BIT(8)
+#define AU_CH_STS_REGEN_DP_ENC1_P0_MASK			BIT(9)
+#define AUDIO_SAMPLE_PRSENT_REGEN_DP_ENC1_P0_MASK	BIT(12)
+#define MTK_DP_ENC1_P0_3324			0x3324
+#define AUDIO_SOURCE_MUX_DP_ENC1_P0_MASK		GENMASK(9, 8)
+#define AUDIO_SOURCE_MUX_DP_ENC1_P0_DPRX		0
+#define MTK_DP_ENC1_P0_3364			0x3364
+#define SDP_DOWN_CNT_IN_HBLANK_DP_ENC1_P0_VAL		0x20
+#define SDP_DOWN_CNT_INIT_IN_HBLANK_DP_ENC1_P0_MASK	GENMASK(11, 0)
+#define FIFO_READ_START_POINT_DP_ENC1_P0_VAL		4
+#define FIFO_READ_START_POINT_DP_ENC1_P0_MASK		GENMASK(15, 12)
+#define MTK_DP_ENC1_P0_3368			0x3368
+#define VIDEO_SRAM_FIFO_CNT_RESET_SEL_DP_ENC1_P0	BIT(0)
+#define VIDEO_STABLE_CNT_THRD_DP_ENC1_P0		BIT(4)
+#define SDP_DP13_EN_DP_ENC1_P0				BIT(8)
+#define BS2BS_MODE_DP_ENC1_P0				BIT(12)
+#define BS2BS_MODE_DP_ENC1_P0_MASK			GENMASK(13, 12)
+#define BS2BS_MODE_DP_ENC1_P0_VAL			1
+#define DP_ENC1_P0_3368_VAL				(VIDEO_SRAM_FIFO_CNT_RESET_SEL_DP_ENC1_P0 | \
+							 VIDEO_STABLE_CNT_THRD_DP_ENC1_P0 | \
+							 SDP_DP13_EN_DP_ENC1_P0 | \
+							 BS2BS_MODE_DP_ENC1_P0)
+#define MTK_DP_ENC1_P0_33F4			0x33f4
+#define DP_ENC_DUMMY_RW_1_AUDIO_RST_EN			BIT(0)
+#define DP_ENC_DUMMY_RW_1				BIT(9)
+
+/* offset: TRANS_OFFSET (0x3400) */
+#define MTK_DP_TRANS_P0_3400				0x3400
+#define PATTERN1_EN_DP_TRANS_P0_MASK				BIT(12)
+#define PATTERN2_EN_DP_TRANS_P0_MASK				BIT(13)
+#define PATTERN3_EN_DP_TRANS_P0_MASK				BIT(14)
+#define PATTERN4_EN_DP_TRANS_P0_MASK				BIT(15)
+#define MTK_DP_TRANS_P0_3404				0x3404
+#define DP_SCR_EN_DP_TRANS_P0_MASK				BIT(0)
+#define MTK_DP_TRANS_P0_340C				0x340c
+#define DP_TX_TRANSMITTER_4P_RESET_SW_DP_TRANS_P0		BIT(13)
+#define MTK_DP_TRANS_P0_3410				0x3410
+#define HPD_DEB_THD_DP_TRANS_P0_MASK				GENMASK(3, 0)
+#define HPD_INT_THD_DP_TRANS_P0_MASK				GENMASK(7, 4)
+#define HPD_INT_THD_DP_TRANS_P0_LOWER_500US			(2 << 4)
+#define HPD_INT_THD_DP_TRANS_P0_UPPER_1100US			(2 << 6)
+#define HPD_DISC_THD_DP_TRANS_P0_MASK				GENMASK(11, 8)
+#define HPD_CONN_THD_DP_TRANS_P0_MASK				GENMASK(15, 12)
+#define MTK_DP_TRANS_P0_3414				0x3414
+#define HPD_DB_DP_TRANS_P0_MASK					BIT(2)
+#define MTK_DP_TRANS_P0_3418				0x3418
+#define IRQ_CLR_DP_TRANS_P0_MASK				GENMASK(3, 0)
+#define IRQ_MASK_DP_TRANS_P0_MASK				GENMASK(7, 4)
+#define IRQ_MASK_DP_TRANS_P0_DISC_IRQ				(BIT(1) << 4)
+#define IRQ_MASK_DP_TRANS_P0_CONN_IRQ				(BIT(2) << 4)
+#define IRQ_MASK_DP_TRANS_P0_INT_IRQ				(BIT(3) << 4)
+#define IRQ_STATUS_DP_TRANS_P0_MASK				GENMASK(15, 12)
+#define MTK_DP_TRANS_P0_342C				0x342c
+#define XTAL_FREQ_DP_TRANS_P0_DEFAULT				(BIT(0) | BIT(3) | BIT(5) | BIT(6))
+#define XTAL_FREQ_DP_TRANS_P0_MASK				GENMASK(7, 0)
+#define MTK_DP_TRANS_P0_3430				0x3430
+#define HPD_INT_THD_ECO_DP_TRANS_P0_MASK			GENMASK(1, 0)
+#define HPD_INT_THD_ECO_DP_TRANS_P0_HIGH_BOUND_EXT		BIT(1)
+#define MTK_DP_TRANS_P0_34A4				0x34a4
+#define LANE_NUM_DP_TRANS_P0_MASK				GENMASK(3, 2)
+#define MTK_DP_TRANS_P0_3540				0x3540
+#define FEC_EN_DP_TRANS_P0_MASK					BIT(0)
+#define FEC_CLOCK_EN_MODE_DP_TRANS_P0				BIT(3)
+#define MTK_DP_TRANS_P0_3580				0x3580
+#define POST_MISC_DATA_LANE0_OV_DP_TRANS_P0_MASK		BIT(8)
+#define POST_MISC_DATA_LANE1_OV_DP_TRANS_P0_MASK		BIT(9)
+#define POST_MISC_DATA_LANE2_OV_DP_TRANS_P0_MASK		BIT(10)
+#define POST_MISC_DATA_LANE3_OV_DP_TRANS_P0_MASK		BIT(11)
+#define MTK_DP_TRANS_P0_35C8				0x35c8
+#define SW_IRQ_CLR_DP_TRANS_P0_MASK				GENMASK(15, 0)
+#define SW_IRQ_STATUS_DP_TRANS_P0_MASK				GENMASK(15, 0)
+#define MTK_DP_TRANS_P0_35D0				0x35d0
+#define SW_IRQ_FINAL_STATUS_DP_TRANS_P0_MASK			GENMASK(15, 0)
+#define MTK_DP_TRANS_P0_35F0				0x35f0
+#define DP_TRANS_DUMMY_RW_0					BIT(3)
+#define DP_TRANS_DUMMY_RW_0_MASK				GENMASK(3, 2)
+
+/* offset: AUX_OFFSET (0x3600) */
+#define MTK_DP_AUX_P0_360C			0x360c
+#define AUX_TIMEOUT_THR_AUX_TX_P0_MASK			GENMASK(12, 0)
+#define AUX_TIMEOUT_THR_AUX_TX_P0_VAL			0x1595
+#define MTK_DP_AUX_P0_3614			0x3614
+#define AUX_RX_UI_CNT_THR_AUX_TX_P0_MASK		GENMASK(6, 0)
+#define AUX_RX_UI_CNT_THR_AUX_FOR_26M			13
+#define MTK_DP_AUX_P0_3618			0x3618
+#define AUX_RX_FIFO_FULL_AUX_TX_P0_MASK			BIT(9)
+#define AUX_RX_FIFO_WRITE_POINTER_AUX_TX_P0_MASK	GENMASK(3, 0)
+#define MTK_DP_AUX_P0_3620			0x3620
+#define AUX_RD_MODE_AUX_TX_P0_MASK			BIT(9)
+#define AUX_RX_FIFO_READ_PULSE_TX_P0			BIT(8)
+#define AUX_RX_FIFO_READ_DATA_AUX_TX_P0_MASK		GENMASK(7, 0)
+#define MTK_DP_AUX_P0_3624			0x3624
+#define AUX_RX_REPLY_COMMAND_AUX_TX_P0_MASK		GENMASK(3, 0)
+#define MTK_DP_AUX_P0_3628			0x3628
+#define AUX_RX_PHY_STATE_AUX_TX_P0_MASK			GENMASK(9, 0)
+#define AUX_RX_PHY_STATE_AUX_TX_P0_RX_IDLE		BIT(0)
+#define MTK_DP_AUX_P0_362C			0x362c
+#define AUX_NO_LENGTH_AUX_TX_P0				BIT(0)
+#define AUX_TX_AUXTX_OV_EN_AUX_TX_P0_MASK		BIT(1)
+#define AUX_RESERVED_RW_0_AUX_TX_P0_MASK		GENMASK(15, 2)
+#define MTK_DP_AUX_P0_3630			0x3630
+#define AUX_TX_REQUEST_READY_AUX_TX_P0			BIT(3)
+#define MTK_DP_AUX_P0_3634			0x3634
+#define AUX_TX_OVER_SAMPLE_RATE_AUX_TX_P0_MASK		GENMASK(15, 8)
+#define AUX_TX_OVER_SAMPLE_RATE_FOR_26M			25
+#define MTK_DP_AUX_P0_3640			0x3640
+#define AUX_RX_AUX_RECV_COMPLETE_IRQ_AUX_TX_P0		BIT(6)
+#define AUX_RX_EDID_RECV_COMPLETE_IRQ_AUX_TX_P0		BIT(5)
+#define AUX_RX_MCCS_RECV_COMPLETE_IRQ_AUX_TX_P0		BIT(4)
+#define AUX_RX_CMD_RECV_IRQ_AUX_TX_P0			BIT(3)
+#define AUX_RX_ADDR_RECV_IRQ_AUX_TX_P0			BIT(2)
+#define AUX_RX_DATA_RECV_IRQ_AUX_TX_P0			BIT(1)
+#define AUX_400US_TIMEOUT_IRQ_AUX_TX_P0			BIT(0)
+#define DP_AUX_P0_3640_VAL				(AUX_400US_TIMEOUT_IRQ_AUX_TX_P0 | \
+							 AUX_RX_DATA_RECV_IRQ_AUX_TX_P0 | \
+							 AUX_RX_ADDR_RECV_IRQ_AUX_TX_P0 | \
+							 AUX_RX_CMD_RECV_IRQ_AUX_TX_P0 | \
+							 AUX_RX_MCCS_RECV_COMPLETE_IRQ_AUX_TX_P0 | \
+							 AUX_RX_EDID_RECV_COMPLETE_IRQ_AUX_TX_P0 | \
+							 AUX_RX_AUX_RECV_COMPLETE_IRQ_AUX_TX_P0)
+#define MTK_DP_AUX_P0_3644			0x3644
+#define MCU_REQUEST_COMMAND_AUX_TX_P0_MASK		GENMASK(3, 0)
+#define MTK_DP_AUX_P0_3648			0x3648
+#define MCU_REQUEST_ADDRESS_LSB_AUX_TX_P0_MASK		GENMASK(15, 0)
+#define MTK_DP_AUX_P0_364C			0x364c
+#define MCU_REQUEST_ADDRESS_MSB_AUX_TX_P0_MASK		GENMASK(3, 0)
+#define MTK_DP_AUX_P0_3650			0x3650
+#define MCU_REQ_DATA_NUM_AUX_TX_P0_MASK			GENMASK(15, 12)
+#define PHY_FIFO_RST_AUX_TX_P0_MASK			BIT(9)
+#define MCU_ACK_TRAN_COMPLETE_AUX_TX_P0			BIT(8)
+#define MTK_DP_AUX_P0_3658			0x3658
+#define AUX_TX_OV_EN_AUX_TX_P0_MASK			BIT(0)
+#define MTK_DP_AUX_P0_3690			0x3690
+#define RX_REPLY_COMPLETE_MODE_AUX_TX_P0		BIT(8)
+#define MTK_DP_AUX_P0_3704			0x3704
+#define AUX_TX_FIFO_WDATA_NEW_MODE_T_AUX_TX_P0_MASK	BIT(1)
+#define AUX_TX_FIFO_NEW_MODE_EN_AUX_TX_P0		BIT(2)
+#define MTK_DP_AUX_P0_3708			0x3708
+#define MTK_DP_AUX_P0_37C8			0x37c8
+#define MTK_ATOP_EN_AUX_TX_P0				BIT(0)
+
+#endif /*_MTK_DP_REG_H_*/
diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c
index 630a4e301ef6..508a6d994e83 100644
--- a/drivers/gpu/drm/mediatek/mtk_dpi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dpi.c
@@ -11,7 +11,6 @@
 #include <linux/media-bus-format.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
-#include <linux/of_gpio.h>
 #include <linux/of_graph.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
index 42cc7052b050..112615817dcb 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_crtc.c
@@ -15,7 +15,6 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
index 0e4c77724b05..91f58db5915f 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
@@ -20,7 +20,7 @@
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_of.h>
 #include <drm/drm_probe_helper.h>
@@ -833,11 +833,8 @@ static int mtk_drm_sys_prepare(struct device *dev)
 {
 	struct mtk_drm_private *private = dev_get_drvdata(dev);
 	struct drm_device *drm = private->drm;
-	int ret;
-
-	ret = drm_mode_config_helper_suspend(drm);
 
-	return ret;
+	return drm_mode_config_helper_suspend(drm);
 }
 
 static void mtk_drm_sys_complete(struct device *dev)
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_gem.c b/drivers/gpu/drm/mediatek/mtk_drm_gem.c
index 139d7724c6d0..47e96b0289f9 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_gem.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_gem.c
@@ -8,7 +8,7 @@
 #include <drm/drm.h>
 #include <drm/drm_device.h>
 #include <drm/drm_gem.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_prime.h>
 
 #include "mtk_drm_drv.h"
@@ -22,7 +22,7 @@ static const struct drm_gem_object_funcs mtk_drm_gem_object_funcs = {
 	.vmap = mtk_drm_gem_prime_vmap,
 	.vunmap = mtk_drm_gem_prime_vunmap,
 	.mmap = mtk_drm_gem_object_mmap,
-	.vm_ops = &drm_gem_cma_vm_ops,
+	.vm_ops = &drm_gem_dma_vm_ops,
 };
 
 static struct mtk_drm_gem_obj *mtk_drm_gem_init(struct drm_device *dev,
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.c b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
index 5c0d9ce69931..2f5e007dd380 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_plane.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
@@ -11,7 +11,6 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 
 #include "mtk_drm_crtc.h"
 #include "mtk_drm_ddp_comp.h"
@@ -108,8 +107,8 @@ static int mtk_plane_atomic_async_check(struct drm_plane *plane,
 		crtc_state = new_plane_state->crtc->state;
 
 	return drm_atomic_helper_check_plane_state(plane->state, crtc_state,
-						   DRM_PLANE_HELPER_NO_SCALING,
-						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
 						   true, true);
 }
 
@@ -202,8 +201,8 @@ static int mtk_plane_atomic_check(struct drm_plane *plane,
 
 	return drm_atomic_helper_check_plane_state(new_plane_state,
 						   crtc_state,
-						   DRM_PLANE_HELPER_NO_SCALING,
-						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
 						   true, true);
 }
 
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c
index 3196189429bc..4c80b6896dc3 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c
@@ -16,7 +16,6 @@
 #include <linux/mutex.h>
 #include <linux/of_platform.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/of_graph.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
diff --git a/drivers/gpu/drm/meson/Kconfig b/drivers/gpu/drm/meson/Kconfig
index 6c70fc3214af..823909da87db 100644
--- a/drivers/gpu/drm/meson/Kconfig
+++ b/drivers/gpu/drm/meson/Kconfig
@@ -4,7 +4,7 @@ config DRM_MESON
 	depends on DRM && OF && (ARM || ARM64)
 	depends on ARCH_MESON || COMPILE_TEST
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_DISPLAY_CONNECTOR
 	select VIDEOMODE_HELPERS
 	select REGMAP_MMIO
diff --git a/drivers/gpu/drm/meson/meson_drv.c b/drivers/gpu/drm/meson/meson_drv.c
index bd4ca11d3ff5..3b24a924b7b9 100644
--- a/drivers/gpu/drm/meson/meson_drv.c
+++ b/drivers/gpu/drm/meson/meson_drv.c
@@ -19,7 +19,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_module.h>
@@ -87,16 +87,16 @@ static int meson_dumb_create(struct drm_file *file, struct drm_device *dev,
 	args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), SZ_64);
 	args->size = PAGE_ALIGN(args->pitch * args->height);
 
-	return drm_gem_cma_dumb_create_internal(file, dev, args);
+	return drm_gem_dma_dumb_create_internal(file, dev, args);
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(fops);
+DEFINE_DRM_GEM_DMA_FOPS(fops);
 
 static const struct drm_driver meson_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 
-	/* CMA Ops */
-	DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(meson_dumb_create),
+	/* DMA Ops */
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(meson_dumb_create),
 
 	/* Misc */
 	.fops			= &fops,
@@ -388,10 +388,14 @@ static void meson_drv_unbind(struct device *dev)
 	drm_dev_unregister(drm);
 	drm_kms_helper_poll_fini(drm);
 	drm_atomic_helper_shutdown(drm);
-	component_unbind_all(dev, drm);
 	free_irq(priv->vsync_irq, drm);
 	drm_dev_put(drm);
 
+	meson_encoder_hdmi_remove(priv);
+	meson_encoder_cvbs_remove(priv);
+
+	component_unbind_all(dev, drm);
+
 	if (priv->afbcd.ops)
 		priv->afbcd.ops->exit(priv);
 }
@@ -493,6 +497,13 @@ static int meson_drv_probe(struct platform_device *pdev)
 	return 0;
 };
 
+static int meson_drv_remove(struct platform_device *pdev)
+{
+	component_master_del(&pdev->dev, &meson_drv_master_ops);
+
+	return 0;
+}
+
 static struct meson_drm_match_data meson_drm_gxbb_data = {
 	.compat = VPU_COMPATIBLE_GXBB,
 };
@@ -530,6 +541,7 @@ static const struct dev_pm_ops meson_drv_pm_ops = {
 
 static struct platform_driver meson_drm_platform_driver = {
 	.probe      = meson_drv_probe,
+	.remove     = meson_drv_remove,
 	.shutdown   = meson_drv_shutdown,
 	.driver     = {
 		.name	= "meson-drm",
diff --git a/drivers/gpu/drm/meson/meson_drv.h b/drivers/gpu/drm/meson/meson_drv.h
index 177dac3ca3be..c62ee358456f 100644
--- a/drivers/gpu/drm/meson/meson_drv.h
+++ b/drivers/gpu/drm/meson/meson_drv.h
@@ -25,6 +25,12 @@ enum vpu_compatible {
 	VPU_COMPATIBLE_G12A = 3,
 };
 
+enum {
+	MESON_ENC_CVBS = 0,
+	MESON_ENC_HDMI,
+	MESON_ENC_LAST,
+};
+
 struct meson_drm_match_data {
 	enum vpu_compatible compat;
 	struct meson_afbcd_ops *afbcd_ops;
@@ -51,6 +57,7 @@ struct meson_drm {
 	struct drm_crtc *crtc;
 	struct drm_plane *primary_plane;
 	struct drm_plane *overlay_plane;
+	void *encoders[MESON_ENC_LAST];
 
 	const struct meson_drm_soc_limits *limits;
 
diff --git a/drivers/gpu/drm/meson/meson_encoder_cvbs.c b/drivers/gpu/drm/meson/meson_encoder_cvbs.c
index 8110a6e39320..5675bc2a92cf 100644
--- a/drivers/gpu/drm/meson/meson_encoder_cvbs.c
+++ b/drivers/gpu/drm/meson/meson_encoder_cvbs.c
@@ -281,5 +281,18 @@ int meson_encoder_cvbs_init(struct meson_drm *priv)
 	}
 	drm_connector_attach_encoder(connector, &meson_encoder_cvbs->encoder);
 
+	priv->encoders[MESON_ENC_CVBS] = meson_encoder_cvbs;
+
 	return 0;
 }
+
+void meson_encoder_cvbs_remove(struct meson_drm *priv)
+{
+	struct meson_encoder_cvbs *meson_encoder_cvbs;
+
+	if (priv->encoders[MESON_ENC_CVBS]) {
+		meson_encoder_cvbs = priv->encoders[MESON_ENC_CVBS];
+		drm_bridge_remove(&meson_encoder_cvbs->bridge);
+		drm_bridge_remove(meson_encoder_cvbs->next_bridge);
+	}
+}
diff --git a/drivers/gpu/drm/meson/meson_encoder_cvbs.h b/drivers/gpu/drm/meson/meson_encoder_cvbs.h
index 61d9d183ce7f..09710fec3c66 100644
--- a/drivers/gpu/drm/meson/meson_encoder_cvbs.h
+++ b/drivers/gpu/drm/meson/meson_encoder_cvbs.h
@@ -25,5 +25,6 @@ struct meson_cvbs_mode {
 extern struct meson_cvbs_mode meson_cvbs_modes[MESON_CVBS_MODES_COUNT];
 
 int meson_encoder_cvbs_init(struct meson_drm *priv);
+void meson_encoder_cvbs_remove(struct meson_drm *priv);
 
 #endif /* __MESON_VENC_CVBS_H */
diff --git a/drivers/gpu/drm/meson/meson_encoder_hdmi.c b/drivers/gpu/drm/meson/meson_encoder_hdmi.c
index 2f616c55c271..53231bfdf7e2 100644
--- a/drivers/gpu/drm/meson/meson_encoder_hdmi.c
+++ b/drivers/gpu/drm/meson/meson_encoder_hdmi.c
@@ -452,6 +452,8 @@ int meson_encoder_hdmi_init(struct meson_drm *priv)
 		meson_encoder_hdmi->cec_notifier = notifier;
 	}
 
+	priv->encoders[MESON_ENC_HDMI] = meson_encoder_hdmi;
+
 	dev_dbg(priv->dev, "HDMI encoder initialized\n");
 
 	return 0;
@@ -460,3 +462,14 @@ err_put_node:
 	of_node_put(remote);
 	return ret;
 }
+
+void meson_encoder_hdmi_remove(struct meson_drm *priv)
+{
+	struct meson_encoder_hdmi *meson_encoder_hdmi;
+
+	if (priv->encoders[MESON_ENC_HDMI]) {
+		meson_encoder_hdmi = priv->encoders[MESON_ENC_HDMI];
+		drm_bridge_remove(&meson_encoder_hdmi->bridge);
+		drm_bridge_remove(meson_encoder_hdmi->next_bridge);
+	}
+}
diff --git a/drivers/gpu/drm/meson/meson_encoder_hdmi.h b/drivers/gpu/drm/meson/meson_encoder_hdmi.h
index ed19494f0956..a6cd38eb5f71 100644
--- a/drivers/gpu/drm/meson/meson_encoder_hdmi.h
+++ b/drivers/gpu/drm/meson/meson_encoder_hdmi.h
@@ -8,5 +8,6 @@
 #define __MESON_ENCODER_HDMI_H
 
 int meson_encoder_hdmi_init(struct meson_drm *priv);
+void meson_encoder_hdmi_remove(struct meson_drm *priv);
 
 #endif /* __MESON_ENCODER_HDMI_H */
diff --git a/drivers/gpu/drm/meson/meson_overlay.c b/drivers/gpu/drm/meson/meson_overlay.c
index b4a0518c1028..7f98de38842b 100644
--- a/drivers/gpu/drm/meson/meson_overlay.c
+++ b/drivers/gpu/drm/meson/meson_overlay.c
@@ -11,12 +11,11 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 
 #include "meson_overlay.h"
 #include "meson_registers.h"
@@ -477,7 +476,7 @@ static void meson_overlay_atomic_update(struct drm_plane *plane,
 									   plane);
 	struct drm_framebuffer *fb = new_state->fb;
 	struct meson_drm *priv = meson_overlay->priv;
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 	unsigned long flags;
 	bool interlace_mode;
 
@@ -651,8 +650,8 @@ static void meson_overlay_atomic_update(struct drm_plane *plane,
 
 	switch (priv->viu.vd1_planes) {
 	case 3:
-		gem = drm_fb_cma_get_gem_obj(fb, 2);
-		priv->viu.vd1_addr2 = gem->paddr + fb->offsets[2];
+		gem = drm_fb_dma_get_gem_obj(fb, 2);
+		priv->viu.vd1_addr2 = gem->dma_addr + fb->offsets[2];
 		priv->viu.vd1_stride2 = fb->pitches[2];
 		priv->viu.vd1_height2 =
 			drm_format_info_plane_height(fb->format,
@@ -663,8 +662,8 @@ static void meson_overlay_atomic_update(struct drm_plane *plane,
 			 priv->viu.vd1_height2);
 		fallthrough;
 	case 2:
-		gem = drm_fb_cma_get_gem_obj(fb, 1);
-		priv->viu.vd1_addr1 = gem->paddr + fb->offsets[1];
+		gem = drm_fb_dma_get_gem_obj(fb, 1);
+		priv->viu.vd1_addr1 = gem->dma_addr + fb->offsets[1];
 		priv->viu.vd1_stride1 = fb->pitches[1];
 		priv->viu.vd1_height1 =
 			drm_format_info_plane_height(fb->format,
@@ -675,8 +674,8 @@ static void meson_overlay_atomic_update(struct drm_plane *plane,
 			 priv->viu.vd1_height1);
 		fallthrough;
 	case 1:
-		gem = drm_fb_cma_get_gem_obj(fb, 0);
-		priv->viu.vd1_addr0 = gem->paddr + fb->offsets[0];
+		gem = drm_fb_dma_get_gem_obj(fb, 0);
+		priv->viu.vd1_addr0 = gem->dma_addr + fb->offsets[0];
 		priv->viu.vd1_stride0 = fb->pitches[0];
 		priv->viu.vd1_height0 =
 			drm_format_info_plane_height(fb->format,
diff --git a/drivers/gpu/drm/meson/meson_plane.c b/drivers/gpu/drm/meson/meson_plane.c
index 03acc68abf2c..815dfe30492b 100644
--- a/drivers/gpu/drm/meson/meson_plane.c
+++ b/drivers/gpu/drm/meson/meson_plane.c
@@ -15,12 +15,11 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 
 #include "meson_plane.h"
 #include "meson_registers.h"
@@ -95,7 +94,7 @@ static int meson_plane_atomic_check(struct drm_plane *plane,
 	return drm_atomic_helper_check_plane_state(new_plane_state,
 						   crtc_state,
 						   FRAC_16_16(1, 5),
-						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
 						   false, true);
 }
 
@@ -140,7 +139,7 @@ static void meson_plane_atomic_update(struct drm_plane *plane,
 	struct drm_rect dest = drm_plane_state_dest(new_state);
 	struct meson_drm *priv = meson_plane->priv;
 	struct drm_framebuffer *fb = new_state->fb;
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 	unsigned long flags;
 	int vsc_ini_rcv_num, vsc_ini_rpt_p0_num;
 	int vsc_bot_rcv_num, vsc_bot_rpt_p0_num;
@@ -366,9 +365,9 @@ static void meson_plane_atomic_update(struct drm_plane *plane,
 	}
 
 	/* Update Canvas with buffer address */
-	gem = drm_fb_cma_get_gem_obj(fb, 0);
+	gem = drm_fb_dma_get_gem_obj(fb, 0);
 
-	priv->viu.osd1_addr = gem->paddr;
+	priv->viu.osd1_addr = gem->dma_addr;
 	priv->viu.osd1_stride = fb->pitches[0];
 	priv->viu.osd1_height = fb->height;
 	priv->viu.osd1_width = fb->width;
diff --git a/drivers/gpu/drm/mgag200/Makefile b/drivers/gpu/drm/mgag200/Makefile
index 89558549c3af..182e224c460d 100644
--- a/drivers/gpu/drm/mgag200/Makefile
+++ b/drivers/gpu/drm/mgag200/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 mgag200-y := \
+	mgag200_bmc.o \
 	mgag200_drv.o \
 	mgag200_g200.o \
 	mgag200_g200eh.o \
@@ -10,7 +11,6 @@ mgag200-y := \
 	mgag200_g200se.o \
 	mgag200_g200wb.o \
 	mgag200_i2c.o \
-	mgag200_mode.o \
-	mgag200_pll.o
+	mgag200_mode.o
 
 obj-$(CONFIG_DRM_MGAG200) += mgag200.o
diff --git a/drivers/gpu/drm/mgag200/mgag200_bmc.c b/drivers/gpu/drm/mgag200/mgag200_bmc.c
new file mode 100644
index 000000000000..2ba2e3c5086a
--- /dev/null
+++ b/drivers/gpu/drm/mgag200/mgag200_bmc.c
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/delay.h>
+
+#include "mgag200_drv.h"
+
+void mgag200_bmc_disable_vidrst(struct mga_device *mdev)
+{
+	u8 tmp;
+	int iter_max;
+
+	/*
+	 * 1 - The first step is to inform the BMC of an upcoming mode
+	 * change. We are putting the misc<0> to output.
+	 */
+
+	WREG8(DAC_INDEX, MGA1064_GEN_IO_CTL);
+	tmp = RREG8(DAC_DATA);
+	tmp |= 0x10;
+	WREG_DAC(MGA1064_GEN_IO_CTL, tmp);
+
+	/* we are putting a 1 on the misc<0> line */
+	WREG8(DAC_INDEX, MGA1064_GEN_IO_DATA);
+	tmp = RREG8(DAC_DATA);
+	tmp |= 0x10;
+	WREG_DAC(MGA1064_GEN_IO_DATA, tmp);
+
+	/*
+	 * 2- Second step to mask any further scan request. This is
+	 * done by asserting the remfreqmsk bit (XSPAREREG<7>)
+	 */
+
+	WREG8(DAC_INDEX, MGA1064_SPAREREG);
+	tmp = RREG8(DAC_DATA);
+	tmp |= 0x80;
+	WREG_DAC(MGA1064_SPAREREG, tmp);
+
+	/*
+	 * 3a- The third step is to verify if there is an active scan.
+	 * We are waiting for a 0 on remhsyncsts <XSPAREREG<0>).
+	 */
+	iter_max = 300;
+	while (!(tmp & 0x1) && iter_max) {
+		WREG8(DAC_INDEX, MGA1064_SPAREREG);
+		tmp = RREG8(DAC_DATA);
+		udelay(1000);
+		iter_max--;
+	}
+
+	/*
+	 * 3b- This step occurs only if the remove is actually
+	 * scanning. We are waiting for the end of the frame which is
+	 * a 1 on remvsyncsts (XSPAREREG<1>)
+	 */
+	if (iter_max) {
+		iter_max = 300;
+		while ((tmp & 0x2) && iter_max) {
+			WREG8(DAC_INDEX, MGA1064_SPAREREG);
+			tmp = RREG8(DAC_DATA);
+			udelay(1000);
+			iter_max--;
+		}
+	}
+}
+
+void mgag200_bmc_enable_vidrst(struct mga_device *mdev)
+{
+	u8 tmp;
+
+	/* Ensure that the vrsten and hrsten are set */
+	WREG8(MGAREG_CRTCEXT_INDEX, 1);
+	tmp = RREG8(MGAREG_CRTCEXT_DATA);
+	WREG8(MGAREG_CRTCEXT_DATA, tmp | 0x88);
+
+	/* Assert rstlvl2 */
+	WREG8(DAC_INDEX, MGA1064_REMHEADCTL2);
+	tmp = RREG8(DAC_DATA);
+	tmp |= 0x8;
+	WREG8(DAC_DATA, tmp);
+
+	udelay(10);
+
+	/* Deassert rstlvl2 */
+	tmp &= ~0x08;
+	WREG8(DAC_INDEX, MGA1064_REMHEADCTL2);
+	WREG8(DAC_DATA, tmp);
+
+	/* Remove mask of scan request */
+	WREG8(DAC_INDEX, MGA1064_SPAREREG);
+	tmp = RREG8(DAC_DATA);
+	tmp &= ~0x80;
+	WREG8(DAC_DATA, tmp);
+
+	/* Put back a 0 on the misc<0> line */
+	WREG8(DAC_INDEX, MGA1064_GEN_IO_DATA);
+	tmp = RREG8(DAC_DATA);
+	tmp &= ~0x10;
+	WREG_DAC(MGA1064_GEN_IO_DATA, tmp);
+}
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.c b/drivers/gpu/drm/mgag200/mgag200_drv.c
index a222bf76804f..ece6cd102dbb 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.c
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.c
@@ -155,15 +155,16 @@ int mgag200_device_preinit(struct mga_device *mdev)
 	return 0;
 }
 
-int mgag200_device_init(struct mga_device *mdev, enum mga_type type,
-			const struct mgag200_device_info *info)
+int mgag200_device_init(struct mga_device *mdev,
+			const struct mgag200_device_info *info,
+			const struct mgag200_device_funcs *funcs)
 {
 	struct drm_device *dev = &mdev->base;
 	u8 crtcext3, misc;
 	int ret;
 
 	mdev->info = info;
-	mdev->type = type;
+	mdev->funcs = funcs;
 
 	ret = drmm_mutex_init(dev, &mdev->rmmio_lock);
 	if (ret)
@@ -226,29 +227,29 @@ mgag200_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	switch (type) {
 	case G200_PCI:
 	case G200_AGP:
-		mdev = mgag200_g200_device_create(pdev, &mgag200_driver, type);
+		mdev = mgag200_g200_device_create(pdev, &mgag200_driver);
 		break;
 	case G200_SE_A:
 	case G200_SE_B:
 		mdev = mgag200_g200se_device_create(pdev, &mgag200_driver, type);
 		break;
 	case G200_WB:
-		mdev = mgag200_g200wb_device_create(pdev, &mgag200_driver, type);
+		mdev = mgag200_g200wb_device_create(pdev, &mgag200_driver);
 		break;
 	case G200_EV:
-		mdev = mgag200_g200ev_device_create(pdev, &mgag200_driver, type);
+		mdev = mgag200_g200ev_device_create(pdev, &mgag200_driver);
 		break;
 	case G200_EH:
-		mdev = mgag200_g200eh_device_create(pdev, &mgag200_driver, type);
+		mdev = mgag200_g200eh_device_create(pdev, &mgag200_driver);
 		break;
 	case G200_EH3:
-		mdev = mgag200_g200eh3_device_create(pdev, &mgag200_driver, type);
+		mdev = mgag200_g200eh3_device_create(pdev, &mgag200_driver);
 		break;
 	case G200_ER:
-		mdev = mgag200_g200er_device_create(pdev, &mgag200_driver, type);
+		mdev = mgag200_g200er_device_create(pdev, &mgag200_driver);
 		break;
 	case G200_EW3:
-		mdev = mgag200_g200ew3_device_create(pdev, &mgag200_driver, type);
+		mdev = mgag200_g200ew3_device_create(pdev, &mgag200_driver);
 		break;
 	default:
 		dev_err(&pdev->dev, "Device type %d is unsupported\n", type);
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h
index 301c4ab46539..f0c2349404b4 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.h
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.h
@@ -15,11 +15,13 @@
 
 #include <video/vga.h>
 
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_gem_shmem_helper.h>
-#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_plane.h>
 
 #include "mgag200_reg.h"
 
@@ -123,11 +125,39 @@
 #define MGA_MISC_OUT 0x1fc2
 #define MGA_MISC_IN 0x1fcc
 
+/*
+ * TODO: This is a pretty large set of default values for all kinds of
+ *       settings. It should be split and set in the various DRM helpers,
+ *       such as the CRTC reset or atomic_enable helpers. The PLL values
+ *       probably belong to each model's PLL code.
+ */
+#define MGAG200_DAC_DEFAULT(xvrefctrl, xpixclkctrl, xmiscctrl, xsyspllm, xsysplln, xsyspllp)	\
+	/* 0x00: */        0,    0,    0,    0,    0,    0, 0x00,    0,				\
+	/* 0x08: */        0,    0,    0,    0,    0,    0,    0,    0,				\
+	/* 0x10: */        0,    0,    0,    0,    0,    0,    0,    0,				\
+	/* 0x18: */     (xvrefctrl),								\
+	/* 0x19: */        0,									\
+	/* 0x1a: */     (xpixclkctrl),								\
+	/* 0x1b: */     0xff, 0xbf, 0x20,							\
+	/* 0x1e: */	(xmiscctrl),								\
+	/* 0x1f: */	0x20,									\
+	/* 0x20: */     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,				\
+	/* 0x28: */     0x00, 0x00, 0x00, 0x00,							\
+	/* 0x2c: */     (xsyspllm),								\
+	/* 0x2d: */     (xsysplln),								\
+	/* 0x2e: */     (xsyspllp),								\
+	/* 0x2f: */     0x40,									\
+	/* 0x30: */     0x00, 0xb0, 0x00, 0xc2, 0x34, 0x14, 0x02, 0x83,				\
+	/* 0x38: */     0x00, 0x93, 0x00, 0x77, 0x00, 0x00, 0x00, 0x3a,				\
+	/* 0x40: */        0,    0,    0,    0,    0,    0,    0,    0,				\
+	/* 0x48: */        0,    0,    0,    0,    0,    0,    0,    0				\
+
+#define MGAG200_LUT_SIZE 256
+
 #define MGAG200_MAX_FB_HEIGHT 4096
 #define MGAG200_MAX_FB_WIDTH 4096
 
 struct mga_device;
-struct mgag200_pll;
 
 /*
  * Stores parameters for programming the PLLs
@@ -146,20 +176,12 @@ struct mgag200_pll_values {
 	unsigned int s;
 };
 
-struct mgag200_pll_funcs {
-	int (*compute)(struct mgag200_pll *pll, long clock, struct mgag200_pll_values *pllc);
-	void (*update)(struct mgag200_pll *pll, const struct mgag200_pll_values *pllc);
-};
-
-struct mgag200_pll {
-	struct mga_device *mdev;
-
-	const struct mgag200_pll_funcs *funcs;
-};
-
 struct mgag200_crtc_state {
 	struct drm_crtc_state base;
 
+	/* Primary-plane format; required for modesetting and color mgmt. */
+	const struct drm_format_info *format;
+
 	struct mgag200_pll_values pixpllc;
 };
 
@@ -188,8 +210,6 @@ enum mga_type {
 	G200_EW3,
 };
 
-#define IS_G200_SE(mdev) (mdev->type == G200_SE_A || mdev->type == G200_SE_B)
-
 struct mgag200_device_info {
 	u16 max_hdisplay;
 	u16 max_vdisplay;
@@ -230,10 +250,39 @@ struct mgag200_device_info {
 		.bug_no_startadd = (_bug_no_startadd), \
 	}
 
+struct mgag200_device_funcs {
+	/*
+	 * Disables an external reset source (i.e., BMC) before programming
+	 * a new display mode.
+	 */
+	void (*disable_vidrst)(struct mga_device *mdev);
+
+	/*
+	 * Enables an external reset source (i.e., BMC) after programming
+	 * a new display mode.
+	 */
+	void (*enable_vidrst)(struct mga_device *mdev);
+
+	/*
+	 * Validate that the given state can be programmed into PIXPLLC. On
+	 * success, the calculated parameters should be stored in the CRTC's
+	 * state in struct @mgag200_crtc_state.pixpllc.
+	 */
+	int (*pixpllc_atomic_check)(struct drm_crtc *crtc, struct drm_atomic_state *new_state);
+
+	/*
+	 * Program PIXPLLC from the CRTC state. The parameters should have been
+	 * stored in struct @mgag200_crtc_state.pixpllc by the corresponding
+	 * implementation of @pixpllc_atomic_check.
+	 */
+	void (*pixpllc_atomic_update)(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
+};
+
 struct mga_device {
 	struct drm_device base;
 
 	const struct mgag200_device_info *info;
+	const struct mgag200_device_funcs *funcs;
 
 	struct resource			*rmmio_res;
 	void __iomem			*rmmio;
@@ -243,12 +292,11 @@ struct mga_device {
 	void __iomem			*vram;
 	resource_size_t			vram_available;
 
-	enum mga_type			type;
-
-	struct mgag200_pll pixpll;
+	struct drm_plane primary_plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
 	struct mga_i2c_chan i2c;
 	struct drm_connector connector;
-	struct drm_simple_display_pipe display_pipe;
 };
 
 static inline struct mga_device *to_mga_device(struct drm_device *dev)
@@ -287,35 +335,113 @@ int mgag200_init_pci_options(struct pci_dev *pdev, u32 option, u32 option2);
 resource_size_t mgag200_probe_vram(void __iomem *mem, resource_size_t size);
 resource_size_t mgag200_device_probe_vram(struct mga_device *mdev);
 int mgag200_device_preinit(struct mga_device *mdev);
-int mgag200_device_init(struct mga_device *mdev, enum mga_type type,
-			const struct mgag200_device_info *info);
+int mgag200_device_init(struct mga_device *mdev,
+			const struct mgag200_device_info *info,
+			const struct mgag200_device_funcs *funcs);
 
 				/* mgag200_<device type>.c */
-struct mga_device *mgag200_g200_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-					      enum mga_type type);
+struct mga_device *mgag200_g200_device_create(struct pci_dev *pdev, const struct drm_driver *drv);
 struct mga_device *mgag200_g200se_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
 						enum mga_type type);
-struct mga_device *mgag200_g200wb_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-						enum mga_type type);
-struct mga_device *mgag200_g200ev_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-						enum mga_type type);
-struct mga_device *mgag200_g200eh_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-						enum mga_type type);
-struct mga_device *mgag200_g200eh3_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-						 enum mga_type type);
-struct mga_device *mgag200_g200er_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-						enum mga_type type);
-struct mga_device *mgag200_g200ew3_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-						 enum mga_type type);
+void mgag200_g200wb_init_registers(struct mga_device *mdev);
+void mgag200_g200wb_pixpllc_atomic_update(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
+struct mga_device *mgag200_g200wb_device_create(struct pci_dev *pdev, const struct drm_driver *drv);
+struct mga_device *mgag200_g200ev_device_create(struct pci_dev *pdev, const struct drm_driver *drv);
+void mgag200_g200eh_init_registers(struct mga_device *mdev);
+void mgag200_g200eh_pixpllc_atomic_update(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
+struct mga_device *mgag200_g200eh_device_create(struct pci_dev *pdev,
+						const struct drm_driver *drv);
+struct mga_device *mgag200_g200eh3_device_create(struct pci_dev *pdev,
+						 const struct drm_driver *drv);
+struct mga_device *mgag200_g200er_device_create(struct pci_dev *pdev,
+						const struct drm_driver *drv);
+struct mga_device *mgag200_g200ew3_device_create(struct pci_dev *pdev,
+						 const struct drm_driver *drv);
 
-				/* mgag200_mode.c */
-resource_size_t mgag200_device_probe_vram(struct mga_device *mdev);
-int mgag200_modeset_init(struct mga_device *mdev, resource_size_t vram_fb_available);
+/*
+ * mgag200_mode.c
+ */
+
+struct drm_crtc;
+struct drm_crtc_state;
+struct drm_display_mode;
+struct drm_plane;
+struct drm_atomic_state;
+
+extern const uint32_t mgag200_primary_plane_formats[];
+extern const size_t   mgag200_primary_plane_formats_size;
+extern const uint64_t mgag200_primary_plane_fmtmods[];
+
+int mgag200_primary_plane_helper_atomic_check(struct drm_plane *plane,
+					      struct drm_atomic_state *new_state);
+void mgag200_primary_plane_helper_atomic_update(struct drm_plane *plane,
+						struct drm_atomic_state *old_state);
+void mgag200_primary_plane_helper_atomic_disable(struct drm_plane *plane,
+						 struct drm_atomic_state *old_state);
+#define MGAG200_PRIMARY_PLANE_HELPER_FUNCS \
+	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS, \
+	.atomic_check = mgag200_primary_plane_helper_atomic_check, \
+	.atomic_update = mgag200_primary_plane_helper_atomic_update, \
+	.atomic_disable = mgag200_primary_plane_helper_atomic_disable
+
+#define MGAG200_PRIMARY_PLANE_FUNCS \
+	.update_plane = drm_atomic_helper_update_plane, \
+	.disable_plane = drm_atomic_helper_disable_plane, \
+	.destroy = drm_plane_cleanup, \
+	DRM_GEM_SHADOW_PLANE_FUNCS
+
+enum drm_mode_status mgag200_crtc_helper_mode_valid(struct drm_crtc *crtc,
+						    const struct drm_display_mode *mode);
+int mgag200_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *new_state);
+void mgag200_crtc_helper_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
+void mgag200_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
+void mgag200_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
+
+#define MGAG200_CRTC_HELPER_FUNCS \
+	.mode_valid = mgag200_crtc_helper_mode_valid, \
+	.atomic_check = mgag200_crtc_helper_atomic_check, \
+	.atomic_flush = mgag200_crtc_helper_atomic_flush, \
+	.atomic_enable = mgag200_crtc_helper_atomic_enable, \
+	.atomic_disable = mgag200_crtc_helper_atomic_disable
+
+void mgag200_crtc_reset(struct drm_crtc *crtc);
+struct drm_crtc_state *mgag200_crtc_atomic_duplicate_state(struct drm_crtc *crtc);
+void mgag200_crtc_atomic_destroy_state(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state);
+
+#define MGAG200_CRTC_FUNCS \
+	.reset = mgag200_crtc_reset, \
+	.destroy = drm_crtc_cleanup, \
+	.set_config = drm_atomic_helper_set_config, \
+	.page_flip = drm_atomic_helper_page_flip, \
+	.atomic_duplicate_state = mgag200_crtc_atomic_duplicate_state, \
+	.atomic_destroy_state = mgag200_crtc_atomic_destroy_state
+
+#define MGAG200_DAC_ENCODER_FUNCS \
+	.destroy = drm_encoder_cleanup
+
+int mgag200_vga_connector_helper_get_modes(struct drm_connector *connector);
+
+#define MGAG200_VGA_CONNECTOR_HELPER_FUNCS \
+	.get_modes  = mgag200_vga_connector_helper_get_modes
+
+#define MGAG200_VGA_CONNECTOR_FUNCS \
+	.reset                  = drm_atomic_helper_connector_reset, \
+	.fill_modes             = drm_helper_probe_single_connector_modes, \
+	.destroy                = drm_connector_cleanup, \
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state, \
+	.atomic_destroy_state   = drm_atomic_helper_connector_destroy_state
+
+void mgag200_set_mode_regs(struct mga_device *mdev, const struct drm_display_mode *mode);
+void mgag200_set_format_regs(struct mga_device *mdev, const struct drm_format_info *format);
+void mgag200_enable_display(struct mga_device *mdev);
+void mgag200_init_registers(struct mga_device *mdev);
+int mgag200_mode_config_init(struct mga_device *mdev, resource_size_t vram_available);
+
+				/* mgag200_bmc.c */
+void mgag200_bmc_disable_vidrst(struct mga_device *mdev);
+void mgag200_bmc_enable_vidrst(struct mga_device *mdev);
 
 				/* mgag200_i2c.c */
 int mgag200_i2c_init(struct mga_device *mdev, struct mga_i2c_chan *i2c);
 
-				/* mgag200_pll.c */
-int mgag200_pixpll_init(struct mgag200_pll *pixpll, struct mga_device *mdev);
-
 #endif				/* __MGAG200_DRV_H__ */
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200.c b/drivers/gpu/drm/mgag200/mgag200_g200.c
index 674385921b7f..bf5d7fe525a3 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200.c
@@ -3,7 +3,11 @@
 #include <linux/pci.h>
 #include <linux/vmalloc.h>
 
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_probe_helper.h>
 
 #include "mgag200_drv.h"
 
@@ -30,6 +34,235 @@ static int mgag200_g200_init_pci_options(struct pci_dev *pdev)
 	return mgag200_init_pci_options(pdev, option, 0x00008000);
 }
 
+static void mgag200_g200_init_registers(struct mgag200_g200_device *g200)
+{
+	static const u8 dacvalue[] = {
+		MGAG200_DAC_DEFAULT(0x00, 0xc9, 0x1f,
+				    0x04, 0x2d, 0x19)
+	};
+
+	struct mga_device *mdev = &g200->base;
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(dacvalue); ++i) {
+		if ((i <= 0x17) ||
+		    (i == 0x1b) ||
+		    (i == 0x1c) ||
+		    ((i >= 0x1f) && (i <= 0x29)) ||
+		    ((i >= 0x30) && (i <= 0x37)))
+			continue;
+		WREG_DAC(i, dacvalue[i]);
+	}
+
+	mgag200_init_registers(mdev);
+}
+
+/*
+ * PIXPLLC
+ */
+
+static int mgag200_g200_pixpllc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *new_state)
+{
+	static const int post_div_max = 7;
+	static const int in_div_min = 1;
+	static const int in_div_max = 6;
+	static const int feed_div_min = 7;
+	static const int feed_div_max = 127;
+
+	struct drm_device *dev = crtc->dev;
+	struct mgag200_g200_device *g200 = to_mgag200_g200_device(dev);
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	struct mgag200_crtc_state *new_mgag200_crtc_state = to_mgag200_crtc_state(new_crtc_state);
+	long clock = new_crtc_state->mode.clock;
+	struct mgag200_pll_values *pixpllc = &new_mgag200_crtc_state->pixpllc;
+	u8 testp, testm, testn;
+	u8 n = 0, m = 0, p, s;
+	long f_vco;
+	long computed;
+	long delta, tmp_delta;
+	long ref_clk = g200->ref_clk;
+	long p_clk_min = g200->pclk_min;
+	long p_clk_max = g200->pclk_max;
+
+	if (clock > p_clk_max) {
+		drm_err(dev, "Pixel Clock %ld too high\n", clock);
+		return -EINVAL;
+	}
+
+	if (clock < p_clk_min >> 3)
+		clock = p_clk_min >> 3;
+
+	f_vco = clock;
+	for (testp = 0;
+	     testp <= post_div_max && f_vco < p_clk_min;
+	     testp = (testp << 1) + 1, f_vco <<= 1)
+		;
+	p = testp + 1;
+
+	delta = clock;
+
+	for (testm = in_div_min; testm <= in_div_max; testm++) {
+		for (testn = feed_div_min; testn <= feed_div_max; testn++) {
+			computed = ref_clk * (testn + 1) / (testm + 1);
+			if (computed < f_vco)
+				tmp_delta = f_vco - computed;
+			else
+				tmp_delta = computed - f_vco;
+			if (tmp_delta < delta) {
+				delta = tmp_delta;
+				m = testm + 1;
+				n = testn + 1;
+			}
+		}
+	}
+	f_vco = ref_clk * n / m;
+	if (f_vco < 100000)
+		s = 0;
+	else if (f_vco < 140000)
+		s = 1;
+	else if (f_vco < 180000)
+		s = 2;
+	else
+		s = 3;
+
+	drm_dbg_kms(dev, "clock: %ld vco: %ld m: %d n: %d p: %d s: %d\n",
+		    clock, f_vco, m, n, p, s);
+
+	pixpllc->m = m;
+	pixpllc->n = n;
+	pixpllc->p = p;
+	pixpllc->s = s;
+
+	return 0;
+}
+
+static void mgag200_g200_pixpllc_atomic_update(struct drm_crtc *crtc,
+					       struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	struct mgag200_pll_values *pixpllc = &mgag200_crtc_state->pixpllc;
+	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
+	u8 xpixpllcm, xpixpllcn, xpixpllcp;
+
+	pixpllcm = pixpllc->m - 1;
+	pixpllcn = pixpllc->n - 1;
+	pixpllcp = pixpllc->p - 1;
+	pixpllcs = pixpllc->s;
+
+	xpixpllcm = pixpllcm;
+	xpixpllcn = pixpllcn;
+	xpixpllcp = (pixpllcs << 3) | pixpllcp;
+
+	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
+
+	WREG_DAC(MGA1064_PIX_PLLC_M, xpixpllcm);
+	WREG_DAC(MGA1064_PIX_PLLC_N, xpixpllcn);
+	WREG_DAC(MGA1064_PIX_PLLC_P, xpixpllcp);
+}
+
+/*
+ * Mode-setting pipeline
+ */
+
+static const struct drm_plane_helper_funcs mgag200_g200_primary_plane_helper_funcs = {
+	MGAG200_PRIMARY_PLANE_HELPER_FUNCS,
+};
+
+static const struct drm_plane_funcs mgag200_g200_primary_plane_funcs = {
+	MGAG200_PRIMARY_PLANE_FUNCS,
+};
+
+static const struct drm_crtc_helper_funcs mgag200_g200_crtc_helper_funcs = {
+	MGAG200_CRTC_HELPER_FUNCS,
+};
+
+static const struct drm_crtc_funcs mgag200_g200_crtc_funcs = {
+	MGAG200_CRTC_FUNCS,
+};
+
+static const struct drm_encoder_funcs mgag200_g200_dac_encoder_funcs = {
+	MGAG200_DAC_ENCODER_FUNCS,
+};
+
+static const struct drm_connector_helper_funcs mgag200_g200_vga_connector_helper_funcs = {
+	MGAG200_VGA_CONNECTOR_HELPER_FUNCS,
+};
+
+static const struct drm_connector_funcs mgag200_g200_vga_connector_funcs = {
+	MGAG200_VGA_CONNECTOR_FUNCS,
+};
+
+static int mgag200_g200_pipeline_init(struct mga_device *mdev)
+{
+	struct drm_device *dev = &mdev->base;
+	struct drm_plane *primary_plane = &mdev->primary_plane;
+	struct drm_crtc *crtc = &mdev->crtc;
+	struct drm_encoder *encoder = &mdev->encoder;
+	struct mga_i2c_chan *i2c = &mdev->i2c;
+	struct drm_connector *connector = &mdev->connector;
+	int ret;
+
+	ret = drm_universal_plane_init(dev, primary_plane, 0,
+				       &mgag200_g200_primary_plane_funcs,
+				       mgag200_primary_plane_formats,
+				       mgag200_primary_plane_formats_size,
+				       mgag200_primary_plane_fmtmods,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		drm_err(dev, "drm_universal_plane_init() failed: %d\n", ret);
+		return ret;
+	}
+	drm_plane_helper_add(primary_plane, &mgag200_g200_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&mgag200_g200_crtc_funcs, NULL);
+	if (ret) {
+		drm_err(dev, "drm_crtc_init_with_planes() failed: %d\n", ret);
+		return ret;
+	}
+	drm_crtc_helper_add(crtc, &mgag200_g200_crtc_helper_funcs);
+
+	/* FIXME: legacy gamma tables, but atomic gamma doesn't work without */
+	drm_mode_crtc_set_gamma_size(crtc, MGAG200_LUT_SIZE);
+	drm_crtc_enable_color_mgmt(crtc, 0, false, MGAG200_LUT_SIZE);
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+	ret = drm_encoder_init(dev, encoder, &mgag200_g200_dac_encoder_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
+	if (ret) {
+		drm_err(dev, "drm_encoder_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = mgag200_i2c_init(mdev, i2c);
+	if (ret) {
+		drm_err(dev, "failed to add DDC bus: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_connector_init_with_ddc(dev, connector,
+					  &mgag200_g200_vga_connector_funcs,
+					  DRM_MODE_CONNECTOR_VGA,
+					  &i2c->adapter);
+	if (ret) {
+		drm_err(dev, "drm_connector_init_with_ddc() failed: %d\n", ret);
+		return ret;
+	}
+	drm_connector_helper_add(connector, &mgag200_g200_vga_connector_helper_funcs);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret) {
+		drm_err(dev, "drm_connector_attach_encoder() failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * DRM Device
  */
@@ -160,8 +393,12 @@ out:
 	pci_unmap_rom(pdev, rom);
 }
 
-struct mga_device *mgag200_g200_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-					      enum mga_type type)
+static const struct mgag200_device_funcs mgag200_g200_device_funcs = {
+	.pixpllc_atomic_check = mgag200_g200_pixpllc_atomic_check,
+	.pixpllc_atomic_update = mgag200_g200_pixpllc_atomic_update,
+};
+
+struct mga_device *mgag200_g200_device_create(struct pci_dev *pdev, const struct drm_driver *drv)
 {
 	struct mgag200_g200_device *g200;
 	struct mga_device *mdev;
@@ -187,15 +424,24 @@ struct mga_device *mgag200_g200_device_create(struct pci_dev *pdev, const struct
 
 	mgag200_g200_init_refclk(g200);
 
-	ret = mgag200_device_init(mdev, type, &mgag200_g200_device_info);
+	ret = mgag200_device_init(mdev, &mgag200_g200_device_info,
+				  &mgag200_g200_device_funcs);
 	if (ret)
 		return ERR_PTR(ret);
 
+	mgag200_g200_init_registers(g200);
+
 	vram_available = mgag200_device_probe_vram(mdev);
 
-	ret = mgag200_modeset_init(mdev, vram_available);
+	ret = mgag200_mode_config_init(mdev, vram_available);
 	if (ret)
 		return ERR_PTR(ret);
 
+	ret = mgag200_g200_pipeline_init(mdev);
+	if (ret)
+		return ERR_PTR(ret);
+
+	drm_mode_config_reset(dev);
+
 	return mdev;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200eh.c b/drivers/gpu/drm/mgag200/mgag200_g200eh.c
index 1b9a22728744..fad62453a91d 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200eh.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200eh.c
@@ -1,11 +1,267 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
+#include <linux/delay.h>
 #include <linux/pci.h>
 
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_probe_helper.h>
 
 #include "mgag200_drv.h"
 
+void mgag200_g200eh_init_registers(struct mga_device *mdev)
+{
+	static const u8 dacvalue[] = {
+		MGAG200_DAC_DEFAULT(0x00, 0xc9,
+				    MGA1064_MISC_CTL_VGA8 | MGA1064_MISC_CTL_DAC_RAM_CS,
+				    0x00, 0x00, 0x00)
+	};
+
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(dacvalue); i++) {
+		if ((i <= 0x17) ||
+		    (i == 0x1b) ||
+		    (i == 0x1c) ||
+		    ((i >= 0x1f) && (i <= 0x29)) ||
+		    ((i >= 0x30) && (i <= 0x37)) ||
+		    ((i >= 0x44) && (i <= 0x4e)))
+			continue;
+		WREG_DAC(i, dacvalue[i]);
+	}
+
+	mgag200_init_registers(mdev);
+}
+
+/*
+ * PIXPLLC
+ */
+
+static int mgag200_g200eh_pixpllc_atomic_check(struct drm_crtc *crtc,
+					       struct drm_atomic_state *new_state)
+{
+	static const unsigned int vcomax = 800000;
+	static const unsigned int vcomin = 400000;
+	static const unsigned int pllreffreq = 33333;
+
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	struct mgag200_crtc_state *new_mgag200_crtc_state = to_mgag200_crtc_state(new_crtc_state);
+	long clock = new_crtc_state->mode.clock;
+	struct mgag200_pll_values *pixpllc = &new_mgag200_crtc_state->pixpllc;
+	unsigned int delta, tmpdelta;
+	unsigned int testp, testm, testn;
+	unsigned int p, m, n, s;
+	unsigned int computed;
+
+	m = n = p = s = 0;
+	delta = 0xffffffff;
+
+	for (testp = 16; testp > 0; testp >>= 1) {
+		if (clock * testp > vcomax)
+			continue;
+		if (clock * testp < vcomin)
+			continue;
+
+		for (testm = 1; testm < 33; testm++) {
+			for (testn = 17; testn < 257; testn++) {
+				computed = (pllreffreq * testn) / (testm * testp);
+				if (computed > clock)
+					tmpdelta = computed - clock;
+				else
+					tmpdelta = clock - computed;
+				if (tmpdelta < delta) {
+					delta = tmpdelta;
+					n = testn;
+					m = testm;
+					p = testp;
+				}
+			}
+		}
+	}
+
+	pixpllc->m = m;
+	pixpllc->n = n;
+	pixpllc->p = p;
+	pixpllc->s = s;
+
+	return 0;
+}
+
+void mgag200_g200eh_pixpllc_atomic_update(struct drm_crtc *crtc,
+					  struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	struct mgag200_pll_values *pixpllc = &mgag200_crtc_state->pixpllc;
+	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
+	u8 xpixpllcm, xpixpllcn, xpixpllcp, tmp;
+	int i, j, tmpcount, vcount;
+	bool pll_locked = false;
+
+	pixpllcm = pixpllc->m - 1;
+	pixpllcn = pixpllc->n - 1;
+	pixpllcp = pixpllc->p - 1;
+	pixpllcs = pixpllc->s;
+
+	xpixpllcm = ((pixpllcn & BIT(8)) >> 1) | pixpllcm;
+	xpixpllcn = pixpllcn;
+	xpixpllcp = (pixpllcs << 3) | pixpllcp;
+
+	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
+
+	for (i = 0; i <= 32 && pll_locked == false; i++) {
+		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+		tmp = RREG8(DAC_DATA);
+		tmp |= MGA1064_PIX_CLK_CTL_CLK_DIS;
+		WREG8(DAC_DATA, tmp);
+
+		tmp = RREG8(MGAREG_MEM_MISC_READ);
+		tmp |= 0x3 << 2;
+		WREG8(MGAREG_MEM_MISC_WRITE, tmp);
+
+		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+		tmp = RREG8(DAC_DATA);
+		tmp |= MGA1064_PIX_CLK_CTL_CLK_POW_DOWN;
+		WREG8(DAC_DATA, tmp);
+
+		udelay(500);
+
+		WREG_DAC(MGA1064_EH_PIX_PLLC_M, xpixpllcm);
+		WREG_DAC(MGA1064_EH_PIX_PLLC_N, xpixpllcn);
+		WREG_DAC(MGA1064_EH_PIX_PLLC_P, xpixpllcp);
+
+		udelay(500);
+
+		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+		tmp = RREG8(DAC_DATA);
+		tmp &= ~MGA1064_PIX_CLK_CTL_SEL_MSK;
+		tmp |= MGA1064_PIX_CLK_CTL_SEL_PLL;
+		WREG8(DAC_DATA, tmp);
+
+		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+		tmp = RREG8(DAC_DATA);
+		tmp &= ~MGA1064_PIX_CLK_CTL_CLK_DIS;
+		tmp &= ~MGA1064_PIX_CLK_CTL_CLK_POW_DOWN;
+		WREG8(DAC_DATA, tmp);
+
+		vcount = RREG8(MGAREG_VCOUNT);
+
+		for (j = 0; j < 30 && pll_locked == false; j++) {
+			tmpcount = RREG8(MGAREG_VCOUNT);
+			if (tmpcount < vcount)
+				vcount = 0;
+			if ((tmpcount - vcount) > 2)
+				pll_locked = true;
+			else
+				udelay(5);
+		}
+	}
+}
+
+/*
+ * Mode-setting pipeline
+ */
+
+static const struct drm_plane_helper_funcs mgag200_g200eh_primary_plane_helper_funcs = {
+	MGAG200_PRIMARY_PLANE_HELPER_FUNCS,
+};
+
+static const struct drm_plane_funcs mgag200_g200eh_primary_plane_funcs = {
+	MGAG200_PRIMARY_PLANE_FUNCS,
+};
+
+static const struct drm_crtc_helper_funcs mgag200_g200eh_crtc_helper_funcs = {
+	MGAG200_CRTC_HELPER_FUNCS,
+};
+
+static const struct drm_crtc_funcs mgag200_g200eh_crtc_funcs = {
+	MGAG200_CRTC_FUNCS,
+};
+
+static const struct drm_encoder_funcs mgag200_g200eh_dac_encoder_funcs = {
+	MGAG200_DAC_ENCODER_FUNCS,
+};
+
+static const struct drm_connector_helper_funcs mgag200_g200eh_vga_connector_helper_funcs = {
+	MGAG200_VGA_CONNECTOR_HELPER_FUNCS,
+};
+
+static const struct drm_connector_funcs mgag200_g200eh_vga_connector_funcs = {
+	MGAG200_VGA_CONNECTOR_FUNCS,
+};
+
+static int mgag200_g200eh_pipeline_init(struct mga_device *mdev)
+{
+	struct drm_device *dev = &mdev->base;
+	struct drm_plane *primary_plane = &mdev->primary_plane;
+	struct drm_crtc *crtc = &mdev->crtc;
+	struct drm_encoder *encoder = &mdev->encoder;
+	struct mga_i2c_chan *i2c = &mdev->i2c;
+	struct drm_connector *connector = &mdev->connector;
+	int ret;
+
+	ret = drm_universal_plane_init(dev, primary_plane, 0,
+				       &mgag200_g200eh_primary_plane_funcs,
+				       mgag200_primary_plane_formats,
+				       mgag200_primary_plane_formats_size,
+				       mgag200_primary_plane_fmtmods,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		drm_err(dev, "drm_universal_plane_init() failed: %d\n", ret);
+		return ret;
+	}
+	drm_plane_helper_add(primary_plane, &mgag200_g200eh_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&mgag200_g200eh_crtc_funcs, NULL);
+	if (ret) {
+		drm_err(dev, "drm_crtc_init_with_planes() failed: %d\n", ret);
+		return ret;
+	}
+	drm_crtc_helper_add(crtc, &mgag200_g200eh_crtc_helper_funcs);
+
+	/* FIXME: legacy gamma tables, but atomic gamma doesn't work without */
+	drm_mode_crtc_set_gamma_size(crtc, MGAG200_LUT_SIZE);
+	drm_crtc_enable_color_mgmt(crtc, 0, false, MGAG200_LUT_SIZE);
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+	ret = drm_encoder_init(dev, encoder, &mgag200_g200eh_dac_encoder_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
+	if (ret) {
+		drm_err(dev, "drm_encoder_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = mgag200_i2c_init(mdev, i2c);
+	if (ret) {
+		drm_err(dev, "failed to add DDC bus: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_connector_init_with_ddc(dev, connector,
+					  &mgag200_g200eh_vga_connector_funcs,
+					  DRM_MODE_CONNECTOR_VGA,
+					  &i2c->adapter);
+	if (ret) {
+		drm_err(dev, "drm_connector_init_with_ddc() failed: %d\n", ret);
+		return ret;
+	}
+	drm_connector_helper_add(connector, &mgag200_g200eh_vga_connector_helper_funcs);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret) {
+		drm_err(dev, "drm_connector_attach_encoder() failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * DRM device
  */
@@ -13,8 +269,12 @@
 static const struct mgag200_device_info mgag200_g200eh_device_info =
 	MGAG200_DEVICE_INFO_INIT(2048, 2048, 37500, false, 1, 0, false);
 
-struct mga_device *mgag200_g200eh_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-						enum mga_type type)
+static const struct mgag200_device_funcs mgag200_g200eh_device_funcs = {
+	.pixpllc_atomic_check = mgag200_g200eh_pixpllc_atomic_check,
+	.pixpllc_atomic_update = mgag200_g200eh_pixpllc_atomic_update,
+};
+
+struct mga_device *mgag200_g200eh_device_create(struct pci_dev *pdev, const struct drm_driver *drv)
 {
 	struct mga_device *mdev;
 	struct drm_device *dev;
@@ -36,15 +296,24 @@ struct mga_device *mgag200_g200eh_device_create(struct pci_dev *pdev, const stru
 	if (ret)
 		return ERR_PTR(ret);
 
-	ret = mgag200_device_init(mdev, type, &mgag200_g200eh_device_info);
+	ret = mgag200_device_init(mdev, &mgag200_g200eh_device_info,
+				  &mgag200_g200eh_device_funcs);
 	if (ret)
 		return ERR_PTR(ret);
 
+	mgag200_g200eh_init_registers(mdev);
+
 	vram_available = mgag200_device_probe_vram(mdev);
 
-	ret = mgag200_modeset_init(mdev, vram_available);
+	ret = mgag200_mode_config_init(mdev, vram_available);
 	if (ret)
 		return ERR_PTR(ret);
 
+	ret = mgag200_g200eh_pipeline_init(mdev);
+	if (ret)
+		return ERR_PTR(ret);
+
+	drm_mode_config_reset(dev);
+
 	return mdev;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200eh3.c b/drivers/gpu/drm/mgag200/mgag200_g200eh3.c
index 438cda1b14c9..0f7d8112cd49 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200eh3.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200eh3.c
@@ -2,20 +2,184 @@
 
 #include <linux/pci.h>
 
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_probe_helper.h>
 
 #include "mgag200_drv.h"
 
 /*
+ * PIXPLLC
+ */
+
+static int mgag200_g200eh3_pixpllc_atomic_check(struct drm_crtc *crtc,
+						struct drm_atomic_state *new_state)
+{
+	static const unsigned int vcomax = 3000000;
+	static const unsigned int vcomin = 1500000;
+	static const unsigned int pllreffreq = 25000;
+
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	struct mgag200_crtc_state *new_mgag200_crtc_state = to_mgag200_crtc_state(new_crtc_state);
+	long clock = new_crtc_state->mode.clock;
+	struct mgag200_pll_values *pixpllc = &new_mgag200_crtc_state->pixpllc;
+	unsigned int delta, tmpdelta;
+	unsigned int testp, testm, testn;
+	unsigned int p, m, n, s;
+	unsigned int computed;
+
+	m = n = p = s = 0;
+	delta = 0xffffffff;
+	testp = 0;
+
+	for (testm = 150; testm >= 6; testm--) {
+		if (clock * testm > vcomax)
+			continue;
+		if (clock * testm < vcomin)
+			continue;
+		for (testn = 120; testn >= 60; testn--) {
+			computed = (pllreffreq * testn) / testm;
+			if (computed > clock)
+				tmpdelta = computed - clock;
+			else
+				tmpdelta = clock - computed;
+			if (tmpdelta < delta) {
+				delta = tmpdelta;
+				n = testn + 1;
+				m = testm + 1;
+				p = testp + 1;
+			}
+			if (delta == 0)
+				break;
+		}
+		if (delta == 0)
+			break;
+	}
+
+	pixpllc->m = m;
+	pixpllc->n = n;
+	pixpllc->p = p;
+	pixpllc->s = s;
+
+	return 0;
+}
+
+/*
+ * Mode-setting pipeline
+ */
+
+static const struct drm_plane_helper_funcs mgag200_g200eh3_primary_plane_helper_funcs = {
+	MGAG200_PRIMARY_PLANE_HELPER_FUNCS,
+};
+
+static const struct drm_plane_funcs mgag200_g200eh3_primary_plane_funcs = {
+	MGAG200_PRIMARY_PLANE_FUNCS,
+};
+
+static const struct drm_crtc_helper_funcs mgag200_g200eh3_crtc_helper_funcs = {
+	MGAG200_CRTC_HELPER_FUNCS,
+};
+
+static const struct drm_crtc_funcs mgag200_g200eh3_crtc_funcs = {
+	MGAG200_CRTC_FUNCS,
+};
+
+static const struct drm_encoder_funcs mgag200_g200eh3_dac_encoder_funcs = {
+	MGAG200_DAC_ENCODER_FUNCS,
+};
+
+static const struct drm_connector_helper_funcs mgag200_g200eh3_vga_connector_helper_funcs = {
+	MGAG200_VGA_CONNECTOR_HELPER_FUNCS,
+};
+
+static const struct drm_connector_funcs mgag200_g200eh3_vga_connector_funcs = {
+	MGAG200_VGA_CONNECTOR_FUNCS,
+};
+
+static int mgag200_g200eh3_pipeline_init(struct mga_device *mdev)
+{
+	struct drm_device *dev = &mdev->base;
+	struct drm_plane *primary_plane = &mdev->primary_plane;
+	struct drm_crtc *crtc = &mdev->crtc;
+	struct drm_encoder *encoder = &mdev->encoder;
+	struct mga_i2c_chan *i2c = &mdev->i2c;
+	struct drm_connector *connector = &mdev->connector;
+	int ret;
+
+	ret = drm_universal_plane_init(dev, primary_plane, 0,
+				       &mgag200_g200eh3_primary_plane_funcs,
+				       mgag200_primary_plane_formats,
+				       mgag200_primary_plane_formats_size,
+				       mgag200_primary_plane_fmtmods,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		drm_err(dev, "drm_universal_plane_init() failed: %d\n", ret);
+		return ret;
+	}
+	drm_plane_helper_add(primary_plane, &mgag200_g200eh3_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&mgag200_g200eh3_crtc_funcs, NULL);
+	if (ret) {
+		drm_err(dev, "drm_crtc_init_with_planes() failed: %d\n", ret);
+		return ret;
+	}
+	drm_crtc_helper_add(crtc, &mgag200_g200eh3_crtc_helper_funcs);
+
+	/* FIXME: legacy gamma tables, but atomic gamma doesn't work without */
+	drm_mode_crtc_set_gamma_size(crtc, MGAG200_LUT_SIZE);
+	drm_crtc_enable_color_mgmt(crtc, 0, false, MGAG200_LUT_SIZE);
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+	ret = drm_encoder_init(dev, encoder, &mgag200_g200eh3_dac_encoder_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
+	if (ret) {
+		drm_err(dev, "drm_encoder_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = mgag200_i2c_init(mdev, i2c);
+	if (ret) {
+		drm_err(dev, "failed to add DDC bus: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_connector_init_with_ddc(dev, connector,
+					  &mgag200_g200eh3_vga_connector_funcs,
+					  DRM_MODE_CONNECTOR_VGA,
+					  &i2c->adapter);
+	if (ret) {
+		drm_err(dev, "drm_connector_init_with_ddc() failed: %d\n", ret);
+		return ret;
+	}
+	drm_connector_helper_add(connector, &mgag200_g200eh3_vga_connector_helper_funcs);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret) {
+		drm_err(dev, "drm_connector_attach_encoder() failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+/*
  * DRM device
  */
 
 static const struct mgag200_device_info mgag200_g200eh3_device_info =
 	MGAG200_DEVICE_INFO_INIT(2048, 2048, 0, false, 1, 0, false);
 
+static const struct mgag200_device_funcs mgag200_g200eh3_device_funcs = {
+	.pixpllc_atomic_check = mgag200_g200eh3_pixpllc_atomic_check,
+	.pixpllc_atomic_update = mgag200_g200eh_pixpllc_atomic_update, // same as G200EH
+};
+
 struct mga_device *mgag200_g200eh3_device_create(struct pci_dev *pdev,
-						 const struct drm_driver *drv,
-						 enum mga_type type)
+						 const struct drm_driver *drv)
 {
 	struct mga_device *mdev;
 	struct drm_device *dev;
@@ -37,15 +201,24 @@ struct mga_device *mgag200_g200eh3_device_create(struct pci_dev *pdev,
 	if (ret)
 		return ERR_PTR(ret);
 
-	ret = mgag200_device_init(mdev, type, &mgag200_g200eh3_device_info);
+	ret = mgag200_device_init(mdev, &mgag200_g200eh3_device_info,
+				  &mgag200_g200eh3_device_funcs);
 	if (ret)
 		return ERR_PTR(ret);
 
+	mgag200_g200eh_init_registers(mdev); // same as G200EH
+
 	vram_available = mgag200_device_probe_vram(mdev);
 
-	ret = mgag200_modeset_init(mdev, vram_available);
+	ret = mgag200_mode_config_init(mdev, vram_available);
 	if (ret)
 		return ERR_PTR(ret);
 
+	ret = mgag200_g200eh3_pipeline_init(mdev);
+	if (ret)
+		return ERR_PTR(ret);
+
+	drm_mode_config_reset(dev);
+
 	return mdev;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200er.c b/drivers/gpu/drm/mgag200/mgag200_g200er.c
index 0790d4e6463d..bce267e0f7de 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200er.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200er.c
@@ -1,11 +1,305 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
+#include <linux/delay.h>
 #include <linux/pci.h>
 
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_probe_helper.h>
 
 #include "mgag200_drv.h"
 
+static void mgag200_g200er_init_registers(struct mga_device *mdev)
+{
+	static const u8 dacvalue[] = {
+		MGAG200_DAC_DEFAULT(0x00, 0xc9, 0x1f, 0x00, 0x00, 0x00)
+	};
+
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(dacvalue); i++) {
+		if ((i <= 0x17) ||
+		    (i == 0x1b) ||
+		    (i == 0x1c) ||
+		    ((i >= 0x1f) && (i <= 0x29)) ||
+		    ((i >= 0x30) && (i <= 0x37)))
+			continue;
+		WREG_DAC(i, dacvalue[i]);
+	}
+
+	WREG_DAC(0x90, 0); /* G200ER specific */
+
+	mgag200_init_registers(mdev);
+
+	WREG_ECRT(0x24, 0x5); /* G200ER specific */
+}
+
+static void mgag200_g200er_reset_tagfifo(struct mga_device *mdev)
+{
+	static const uint32_t RESET_FLAG = 0x00200000; /* undocumented magic value */
+	u32 memctl;
+
+	memctl = RREG32(MGAREG_MEMCTL);
+
+	memctl |= RESET_FLAG;
+	WREG32(MGAREG_MEMCTL, memctl);
+
+	udelay(1000);
+
+	memctl &= ~RESET_FLAG;
+	WREG32(MGAREG_MEMCTL, memctl);
+}
+
+/*
+ * PIXPLLC
+ */
+
+static int mgag200_g200er_pixpllc_atomic_check(struct drm_crtc *crtc,
+					       struct drm_atomic_state *new_state)
+{
+	static const unsigned int vcomax = 1488000;
+	static const unsigned int vcomin = 1056000;
+	static const unsigned int pllreffreq = 48000;
+	static const unsigned int m_div_val[] = { 1, 2, 4, 8 };
+
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	struct mgag200_crtc_state *new_mgag200_crtc_state = to_mgag200_crtc_state(new_crtc_state);
+	long clock = new_crtc_state->mode.clock;
+	struct mgag200_pll_values *pixpllc = &new_mgag200_crtc_state->pixpllc;
+	unsigned int delta, tmpdelta;
+	int testr, testn, testm, testo;
+	unsigned int p, m, n, s;
+	unsigned int computed, vco;
+
+	m = n = p = s = 0;
+	delta = 0xffffffff;
+
+	for (testr = 0; testr < 4; testr++) {
+		if (delta == 0)
+			break;
+		for (testn = 5; testn < 129; testn++) {
+			if (delta == 0)
+				break;
+			for (testm = 3; testm >= 0; testm--) {
+				if (delta == 0)
+					break;
+				for (testo = 5; testo < 33; testo++) {
+					vco = pllreffreq * (testn + 1) /
+						(testr + 1);
+					if (vco < vcomin)
+						continue;
+					if (vco > vcomax)
+						continue;
+					computed = vco / (m_div_val[testm] * (testo + 1));
+					if (computed > clock)
+						tmpdelta = computed - clock;
+					else
+						tmpdelta = clock - computed;
+					if (tmpdelta < delta) {
+						delta = tmpdelta;
+						m = (testm | (testo << 3)) + 1;
+						n = testn + 1;
+						p = testr + 1;
+						s = testr;
+					}
+				}
+			}
+		}
+	}
+
+	pixpllc->m = m;
+	pixpllc->n = n;
+	pixpllc->p = p;
+	pixpllc->s = s;
+
+	return 0;
+}
+
+static void mgag200_g200er_pixpllc_atomic_update(struct drm_crtc *crtc,
+						 struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	struct mgag200_pll_values *pixpllc = &mgag200_crtc_state->pixpllc;
+	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
+	u8 xpixpllcm, xpixpllcn, xpixpllcp, tmp;
+
+	pixpllcm = pixpllc->m - 1;
+	pixpllcn = pixpllc->n - 1;
+	pixpllcp = pixpllc->p - 1;
+	pixpllcs = pixpllc->s;
+
+	xpixpllcm = pixpllcm;
+	xpixpllcn = pixpllcn;
+	xpixpllcp = (pixpllcs << 3) | pixpllcp;
+
+	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
+
+	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+	tmp = RREG8(DAC_DATA);
+	tmp |= MGA1064_PIX_CLK_CTL_CLK_DIS;
+	WREG8(DAC_DATA, tmp);
+
+	WREG8(DAC_INDEX, MGA1064_REMHEADCTL);
+	tmp = RREG8(DAC_DATA);
+	tmp |= MGA1064_REMHEADCTL_CLKDIS;
+	WREG8(DAC_DATA, tmp);
+
+	tmp = RREG8(MGAREG_MEM_MISC_READ);
+	tmp |= (0x3<<2) | 0xc0;
+	WREG8(MGAREG_MEM_MISC_WRITE, tmp);
+
+	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+	tmp = RREG8(DAC_DATA);
+	tmp &= ~MGA1064_PIX_CLK_CTL_CLK_DIS;
+	tmp |= MGA1064_PIX_CLK_CTL_CLK_POW_DOWN;
+	WREG8(DAC_DATA, tmp);
+
+	udelay(500);
+
+	WREG_DAC(MGA1064_ER_PIX_PLLC_N, xpixpllcn);
+	WREG_DAC(MGA1064_ER_PIX_PLLC_M, xpixpllcm);
+	WREG_DAC(MGA1064_ER_PIX_PLLC_P, xpixpllcp);
+
+	udelay(50);
+}
+
+/*
+ * Mode-setting pipeline
+ */
+
+static const struct drm_plane_helper_funcs mgag200_g200er_primary_plane_helper_funcs = {
+	MGAG200_PRIMARY_PLANE_HELPER_FUNCS,
+};
+
+static const struct drm_plane_funcs mgag200_g200er_primary_plane_funcs = {
+	MGAG200_PRIMARY_PLANE_FUNCS,
+};
+
+static void mgag200_g200er_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+						     struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	const struct mgag200_device_funcs *funcs = mdev->funcs;
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	const struct drm_format_info *format = mgag200_crtc_state->format;
+
+	if (funcs->disable_vidrst)
+		funcs->disable_vidrst(mdev);
+
+	mgag200_set_format_regs(mdev, format);
+	mgag200_set_mode_regs(mdev, adjusted_mode);
+
+	if (funcs->pixpllc_atomic_update)
+		funcs->pixpllc_atomic_update(crtc, old_state);
+
+	mgag200_g200er_reset_tagfifo(mdev);
+
+	mgag200_enable_display(mdev);
+
+	if (funcs->enable_vidrst)
+		funcs->enable_vidrst(mdev);
+}
+
+static const struct drm_crtc_helper_funcs mgag200_g200er_crtc_helper_funcs = {
+	.mode_valid = mgag200_crtc_helper_mode_valid,
+	.atomic_check = mgag200_crtc_helper_atomic_check,
+	.atomic_flush = mgag200_crtc_helper_atomic_flush,
+	.atomic_enable = mgag200_g200er_crtc_helper_atomic_enable,
+	.atomic_disable = mgag200_crtc_helper_atomic_disable
+};
+
+static const struct drm_crtc_funcs mgag200_g200er_crtc_funcs = {
+	MGAG200_CRTC_FUNCS,
+};
+
+static const struct drm_encoder_funcs mgag200_g200er_dac_encoder_funcs = {
+	MGAG200_DAC_ENCODER_FUNCS,
+};
+
+static const struct drm_connector_helper_funcs mgag200_g200er_vga_connector_helper_funcs = {
+	MGAG200_VGA_CONNECTOR_HELPER_FUNCS,
+};
+
+static const struct drm_connector_funcs mgag200_g200er_vga_connector_funcs = {
+	MGAG200_VGA_CONNECTOR_FUNCS,
+};
+
+static int mgag200_g200er_pipeline_init(struct mga_device *mdev)
+{
+	struct drm_device *dev = &mdev->base;
+	struct drm_plane *primary_plane = &mdev->primary_plane;
+	struct drm_crtc *crtc = &mdev->crtc;
+	struct drm_encoder *encoder = &mdev->encoder;
+	struct mga_i2c_chan *i2c = &mdev->i2c;
+	struct drm_connector *connector = &mdev->connector;
+	int ret;
+
+	ret = drm_universal_plane_init(dev, primary_plane, 0,
+				       &mgag200_g200er_primary_plane_funcs,
+				       mgag200_primary_plane_formats,
+				       mgag200_primary_plane_formats_size,
+				       mgag200_primary_plane_fmtmods,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		drm_err(dev, "drm_universal_plane_init() failed: %d\n", ret);
+		return ret;
+	}
+	drm_plane_helper_add(primary_plane, &mgag200_g200er_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&mgag200_g200er_crtc_funcs, NULL);
+	if (ret) {
+		drm_err(dev, "drm_crtc_init_with_planes() failed: %d\n", ret);
+		return ret;
+	}
+	drm_crtc_helper_add(crtc, &mgag200_g200er_crtc_helper_funcs);
+
+	/* FIXME: legacy gamma tables, but atomic gamma doesn't work without */
+	drm_mode_crtc_set_gamma_size(crtc, MGAG200_LUT_SIZE);
+	drm_crtc_enable_color_mgmt(crtc, 0, false, MGAG200_LUT_SIZE);
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+	ret = drm_encoder_init(dev, encoder, &mgag200_g200er_dac_encoder_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
+	if (ret) {
+		drm_err(dev, "drm_encoder_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = mgag200_i2c_init(mdev, i2c);
+	if (ret) {
+		drm_err(dev, "failed to add DDC bus: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_connector_init_with_ddc(dev, connector,
+					  &mgag200_g200er_vga_connector_funcs,
+					  DRM_MODE_CONNECTOR_VGA,
+					  &i2c->adapter);
+	if (ret) {
+		drm_err(dev, "drm_connector_init_with_ddc() failed: %d\n", ret);
+		return ret;
+	}
+	drm_connector_helper_add(connector, &mgag200_g200er_vga_connector_helper_funcs);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret) {
+		drm_err(dev, "drm_connector_attach_encoder() failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * DRM device
  */
@@ -13,8 +307,12 @@
 static const struct mgag200_device_info mgag200_g200er_device_info =
 	MGAG200_DEVICE_INFO_INIT(2048, 2048, 55000, false, 1, 0, false);
 
-struct mga_device *mgag200_g200er_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-						enum mga_type type)
+static const struct mgag200_device_funcs mgag200_g200er_device_funcs = {
+	.pixpllc_atomic_check = mgag200_g200er_pixpllc_atomic_check,
+	.pixpllc_atomic_update = mgag200_g200er_pixpllc_atomic_update,
+};
+
+struct mga_device *mgag200_g200er_device_create(struct pci_dev *pdev, const struct drm_driver *drv)
 {
 	struct mga_device *mdev;
 	struct drm_device *dev;
@@ -32,15 +330,24 @@ struct mga_device *mgag200_g200er_device_create(struct pci_dev *pdev, const stru
 	if (ret)
 		return ERR_PTR(ret);
 
-	ret = mgag200_device_init(mdev, type, &mgag200_g200er_device_info);
+	ret = mgag200_device_init(mdev, &mgag200_g200er_device_info,
+				  &mgag200_g200er_device_funcs);
 	if (ret)
 		return ERR_PTR(ret);
 
+	mgag200_g200er_init_registers(mdev);
+
 	vram_available = mgag200_device_probe_vram(mdev);
 
-	ret = mgag200_modeset_init(mdev, vram_available);
+	ret = mgag200_mode_config_init(mdev, vram_available);
 	if (ret)
 		return ERR_PTR(ret);
 
+	ret = mgag200_g200er_pipeline_init(mdev);
+	if (ret)
+		return ERR_PTR(ret);
+
+	drm_mode_config_reset(dev);
+
 	return mdev;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200ev.c b/drivers/gpu/drm/mgag200/mgag200_g200ev.c
index 5353422d0eef..ac957f42abe1 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200ev.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200ev.c
@@ -1,11 +1,306 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
+#include <linux/delay.h>
 #include <linux/pci.h>
 
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_probe_helper.h>
 
 #include "mgag200_drv.h"
 
+static void mgag200_g200ev_init_registers(struct mga_device *mdev)
+{
+	static const u8 dacvalue[] = {
+		MGAG200_DAC_DEFAULT(0x00,
+				    MGA1064_PIX_CLK_CTL_SEL_PLL,
+				    MGA1064_MISC_CTL_VGA8 | MGA1064_MISC_CTL_DAC_RAM_CS,
+				    0x00, 0x00, 0x00)
+	};
+
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(dacvalue); i++) {
+		if ((i <= 0x17) ||
+		    (i == 0x1b) ||
+		    (i == 0x1c) ||
+		    ((i >= 0x1f) && (i <= 0x29)) ||
+		    ((i >= 0x30) && (i <= 0x37)) ||
+		    ((i >= 0x44) && (i <= 0x4e)))
+			continue;
+		WREG_DAC(i, dacvalue[i]);
+	}
+
+	mgag200_init_registers(mdev);
+}
+
+static void mgag200_g200ev_set_hiprilvl(struct mga_device *mdev)
+{
+	WREG_ECRT(0x06, 0x00);
+}
+
+/*
+ * PIXPLLC
+ */
+
+static int mgag200_g200ev_pixpllc_atomic_check(struct drm_crtc *crtc,
+					       struct drm_atomic_state *new_state)
+{
+	static const unsigned int vcomax = 550000;
+	static const unsigned int vcomin = 150000;
+	static const unsigned int pllreffreq = 50000;
+
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	struct mgag200_crtc_state *new_mgag200_crtc_state = to_mgag200_crtc_state(new_crtc_state);
+	long clock = new_crtc_state->mode.clock;
+	struct mgag200_pll_values *pixpllc = &new_mgag200_crtc_state->pixpllc;
+	unsigned int delta, tmpdelta;
+	unsigned int testp, testm, testn;
+	unsigned int p, m, n, s;
+	unsigned int computed;
+
+	m = n = p = s = 0;
+	delta = 0xffffffff;
+
+	for (testp = 16; testp > 0; testp--) {
+		if (clock * testp > vcomax)
+			continue;
+		if (clock * testp < vcomin)
+			continue;
+
+		for (testn = 1; testn < 257; testn++) {
+			for (testm = 1; testm < 17; testm++) {
+				computed = (pllreffreq * testn) /
+					(testm * testp);
+				if (computed > clock)
+					tmpdelta = computed - clock;
+				else
+					tmpdelta = clock - computed;
+				if (tmpdelta < delta) {
+					delta = tmpdelta;
+					n = testn;
+					m = testm;
+					p = testp;
+				}
+			}
+		}
+	}
+
+	pixpllc->m = m;
+	pixpllc->n = n;
+	pixpllc->p = p;
+	pixpllc->s = s;
+
+	return 0;
+}
+
+static void mgag200_g200ev_pixpllc_atomic_update(struct drm_crtc *crtc,
+						 struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	struct mgag200_pll_values *pixpllc = &mgag200_crtc_state->pixpllc;
+	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
+	u8 xpixpllcm, xpixpllcn, xpixpllcp, tmp;
+
+	pixpllcm = pixpllc->m - 1;
+	pixpllcn = pixpllc->n - 1;
+	pixpllcp = pixpllc->p - 1;
+	pixpllcs = pixpllc->s;
+
+	xpixpllcm = pixpllcm;
+	xpixpllcn = pixpllcn;
+	xpixpllcp = (pixpllcs << 3) | pixpllcp;
+
+	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
+
+	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+	tmp = RREG8(DAC_DATA);
+	tmp |= MGA1064_PIX_CLK_CTL_CLK_DIS;
+	WREG8(DAC_DATA, tmp);
+
+	tmp = RREG8(MGAREG_MEM_MISC_READ);
+	tmp |= 0x3 << 2;
+	WREG8(MGAREG_MEM_MISC_WRITE, tmp);
+
+	WREG8(DAC_INDEX, MGA1064_PIX_PLL_STAT);
+	tmp = RREG8(DAC_DATA);
+	WREG8(DAC_DATA, tmp & ~0x40);
+
+	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+	tmp = RREG8(DAC_DATA);
+	tmp |= MGA1064_PIX_CLK_CTL_CLK_POW_DOWN;
+	WREG8(DAC_DATA, tmp);
+
+	WREG_DAC(MGA1064_EV_PIX_PLLC_M, xpixpllcm);
+	WREG_DAC(MGA1064_EV_PIX_PLLC_N, xpixpllcn);
+	WREG_DAC(MGA1064_EV_PIX_PLLC_P, xpixpllcp);
+
+	udelay(50);
+
+	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+	tmp = RREG8(DAC_DATA);
+	tmp &= ~MGA1064_PIX_CLK_CTL_CLK_POW_DOWN;
+	WREG8(DAC_DATA, tmp);
+
+	udelay(500);
+
+	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+	tmp = RREG8(DAC_DATA);
+	tmp &= ~MGA1064_PIX_CLK_CTL_SEL_MSK;
+	tmp |= MGA1064_PIX_CLK_CTL_SEL_PLL;
+	WREG8(DAC_DATA, tmp);
+
+	WREG8(DAC_INDEX, MGA1064_PIX_PLL_STAT);
+	tmp = RREG8(DAC_DATA);
+	WREG8(DAC_DATA, tmp | 0x40);
+
+	tmp = RREG8(MGAREG_MEM_MISC_READ);
+	tmp |= (0x3 << 2);
+	WREG8(MGAREG_MEM_MISC_WRITE, tmp);
+
+	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+	tmp = RREG8(DAC_DATA);
+	tmp &= ~MGA1064_PIX_CLK_CTL_CLK_DIS;
+	WREG8(DAC_DATA, tmp);
+}
+
+/*
+ * Mode-setting pipeline
+ */
+
+static const struct drm_plane_helper_funcs mgag200_g200ev_primary_plane_helper_funcs = {
+	MGAG200_PRIMARY_PLANE_HELPER_FUNCS,
+};
+
+static const struct drm_plane_funcs mgag200_g200ev_primary_plane_funcs = {
+	MGAG200_PRIMARY_PLANE_FUNCS,
+};
+
+static void mgag200_g200ev_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+						     struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	const struct mgag200_device_funcs *funcs = mdev->funcs;
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	const struct drm_format_info *format = mgag200_crtc_state->format;
+
+	if (funcs->disable_vidrst)
+		funcs->disable_vidrst(mdev);
+
+	mgag200_set_format_regs(mdev, format);
+	mgag200_set_mode_regs(mdev, adjusted_mode);
+
+	if (funcs->pixpllc_atomic_update)
+		funcs->pixpllc_atomic_update(crtc, old_state);
+
+	mgag200_g200ev_set_hiprilvl(mdev);
+
+	mgag200_enable_display(mdev);
+
+	if (funcs->enable_vidrst)
+		funcs->enable_vidrst(mdev);
+}
+
+static const struct drm_crtc_helper_funcs mgag200_g200ev_crtc_helper_funcs = {
+	.mode_valid = mgag200_crtc_helper_mode_valid,
+	.atomic_check = mgag200_crtc_helper_atomic_check,
+	.atomic_flush = mgag200_crtc_helper_atomic_flush,
+	.atomic_enable = mgag200_g200ev_crtc_helper_atomic_enable,
+	.atomic_disable = mgag200_crtc_helper_atomic_disable
+};
+
+static const struct drm_crtc_funcs mgag200_g200ev_crtc_funcs = {
+	MGAG200_CRTC_FUNCS,
+};
+
+static const struct drm_encoder_funcs mgag200_g200ev_dac_encoder_funcs = {
+	MGAG200_DAC_ENCODER_FUNCS,
+};
+
+static const struct drm_connector_helper_funcs mgag200_g200ev_vga_connector_helper_funcs = {
+	MGAG200_VGA_CONNECTOR_HELPER_FUNCS,
+};
+
+static const struct drm_connector_funcs mgag200_g200ev_vga_connector_funcs = {
+	MGAG200_VGA_CONNECTOR_FUNCS,
+};
+
+static int mgag200_g200ev_pipeline_init(struct mga_device *mdev)
+{
+	struct drm_device *dev = &mdev->base;
+	struct drm_plane *primary_plane = &mdev->primary_plane;
+	struct drm_crtc *crtc = &mdev->crtc;
+	struct drm_encoder *encoder = &mdev->encoder;
+	struct mga_i2c_chan *i2c = &mdev->i2c;
+	struct drm_connector *connector = &mdev->connector;
+	int ret;
+
+	ret = drm_universal_plane_init(dev, primary_plane, 0,
+				       &mgag200_g200ev_primary_plane_funcs,
+				       mgag200_primary_plane_formats,
+				       mgag200_primary_plane_formats_size,
+				       mgag200_primary_plane_fmtmods,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		drm_err(dev, "drm_universal_plane_init() failed: %d\n", ret);
+		return ret;
+	}
+	drm_plane_helper_add(primary_plane, &mgag200_g200ev_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&mgag200_g200ev_crtc_funcs, NULL);
+	if (ret) {
+		drm_err(dev, "drm_crtc_init_with_planes() failed: %d\n", ret);
+		return ret;
+	}
+	drm_crtc_helper_add(crtc, &mgag200_g200ev_crtc_helper_funcs);
+
+	/* FIXME: legacy gamma tables, but atomic gamma doesn't work without */
+	drm_mode_crtc_set_gamma_size(crtc, MGAG200_LUT_SIZE);
+	drm_crtc_enable_color_mgmt(crtc, 0, false, MGAG200_LUT_SIZE);
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+	ret = drm_encoder_init(dev, encoder, &mgag200_g200ev_dac_encoder_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
+	if (ret) {
+		drm_err(dev, "drm_encoder_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = mgag200_i2c_init(mdev, i2c);
+	if (ret) {
+		drm_err(dev, "failed to add DDC bus: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_connector_init_with_ddc(dev, connector,
+					  &mgag200_g200ev_vga_connector_funcs,
+					  DRM_MODE_CONNECTOR_VGA,
+					  &i2c->adapter);
+	if (ret) {
+		drm_err(dev, "drm_connector_init_with_ddc() failed: %d\n", ret);
+		return ret;
+	}
+	drm_connector_helper_add(connector, &mgag200_g200ev_vga_connector_helper_funcs);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret) {
+		drm_err(dev, "drm_connector_attach_encoder() failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * DRM device
  */
@@ -13,8 +308,12 @@
 static const struct mgag200_device_info mgag200_g200ev_device_info =
 	MGAG200_DEVICE_INFO_INIT(2048, 2048, 32700, false, 0, 1, false);
 
-struct mga_device *mgag200_g200ev_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-						enum mga_type type)
+static const struct mgag200_device_funcs mgag200_g200ev_device_funcs = {
+	.pixpllc_atomic_check = mgag200_g200ev_pixpllc_atomic_check,
+	.pixpllc_atomic_update = mgag200_g200ev_pixpllc_atomic_update,
+};
+
+struct mga_device *mgag200_g200ev_device_create(struct pci_dev *pdev, const struct drm_driver *drv)
 {
 	struct mga_device *mdev;
 	struct drm_device *dev;
@@ -36,15 +335,24 @@ struct mga_device *mgag200_g200ev_device_create(struct pci_dev *pdev, const stru
 	if (ret)
 		return ERR_PTR(ret);
 
-	ret = mgag200_device_init(mdev, type, &mgag200_g200ev_device_info);
+	ret = mgag200_device_init(mdev, &mgag200_g200ev_device_info,
+				  &mgag200_g200ev_device_funcs);
 	if (ret)
 		return ERR_PTR(ret);
 
+	mgag200_g200ev_init_registers(mdev);
+
 	vram_available = mgag200_device_probe_vram(mdev);
 
-	ret = mgag200_modeset_init(mdev, vram_available);
+	ret = mgag200_mode_config_init(mdev, vram_available);
 	if (ret)
 		return ERR_PTR(ret);
 
+	ret = mgag200_g200ev_pipeline_init(mdev);
+	if (ret)
+		return ERR_PTR(ret);
+
+	drm_mode_config_reset(dev);
+
 	return mdev;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200ew3.c b/drivers/gpu/drm/mgag200/mgag200_g200ew3.c
index 3bfc1324cf78..170934414d7d 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200ew3.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200ew3.c
@@ -2,10 +2,179 @@
 
 #include <linux/pci.h>
 
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_probe_helper.h>
 
 #include "mgag200_drv.h"
 
+static void mgag200_g200ew3_init_registers(struct mga_device *mdev)
+{
+	mgag200_g200wb_init_registers(mdev); // same as G200WB
+
+	WREG_ECRT(0x34, 0x5); // G200EW3 specific
+}
+
+/*
+ * PIXPLLC
+ */
+
+static int mgag200_g200ew3_pixpllc_atomic_check(struct drm_crtc *crtc,
+						struct drm_atomic_state *new_state)
+{
+	static const unsigned int vcomax = 800000;
+	static const unsigned int vcomin = 400000;
+	static const unsigned int pllreffreq = 25000;
+
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	struct mgag200_crtc_state *new_mgag200_crtc_state = to_mgag200_crtc_state(new_crtc_state);
+	long clock = new_crtc_state->mode.clock;
+	struct mgag200_pll_values *pixpllc = &new_mgag200_crtc_state->pixpllc;
+	unsigned int delta, tmpdelta;
+	unsigned int testp, testm, testn, testp2;
+	unsigned int p, m, n, s;
+	unsigned int computed;
+
+	m = n = p = s = 0;
+	delta = 0xffffffff;
+
+	for (testp = 1; testp < 8; testp++) {
+		for (testp2 = 1; testp2 < 8; testp2++) {
+			if (testp < testp2)
+				continue;
+			if ((clock * testp * testp2) > vcomax)
+				continue;
+			if ((clock * testp * testp2) < vcomin)
+				continue;
+			for (testm = 1; testm < 26; testm++) {
+				for (testn = 32; testn < 2048 ; testn++) {
+					computed = (pllreffreq * testn) / (testm * testp * testp2);
+					if (computed > clock)
+						tmpdelta = computed - clock;
+					else
+						tmpdelta = clock - computed;
+					if (tmpdelta < delta) {
+						delta = tmpdelta;
+						m = testm + 1;
+						n = testn + 1;
+						p = testp + 1;
+						s = testp2;
+					}
+				}
+			}
+		}
+	}
+
+	pixpllc->m = m;
+	pixpllc->n = n;
+	pixpllc->p = p;
+	pixpllc->s = s;
+
+	return 0;
+}
+
+/*
+ * Mode-setting pipeline
+ */
+
+static const struct drm_plane_helper_funcs mgag200_g200ew3_primary_plane_helper_funcs = {
+	MGAG200_PRIMARY_PLANE_HELPER_FUNCS,
+};
+
+static const struct drm_plane_funcs mgag200_g200ew3_primary_plane_funcs = {
+	MGAG200_PRIMARY_PLANE_FUNCS,
+};
+
+static const struct drm_crtc_helper_funcs mgag200_g200ew3_crtc_helper_funcs = {
+	MGAG200_CRTC_HELPER_FUNCS,
+};
+
+static const struct drm_crtc_funcs mgag200_g200ew3_crtc_funcs = {
+	MGAG200_CRTC_FUNCS,
+};
+
+static const struct drm_encoder_funcs mgag200_g200ew3_dac_encoder_funcs = {
+	MGAG200_DAC_ENCODER_FUNCS,
+};
+
+static const struct drm_connector_helper_funcs mgag200_g200ew3_vga_connector_helper_funcs = {
+	MGAG200_VGA_CONNECTOR_HELPER_FUNCS,
+};
+
+static const struct drm_connector_funcs mgag200_g200ew3_vga_connector_funcs = {
+	MGAG200_VGA_CONNECTOR_FUNCS,
+};
+
+static int mgag200_g200ew3_pipeline_init(struct mga_device *mdev)
+{
+	struct drm_device *dev = &mdev->base;
+	struct drm_plane *primary_plane = &mdev->primary_plane;
+	struct drm_crtc *crtc = &mdev->crtc;
+	struct drm_encoder *encoder = &mdev->encoder;
+	struct mga_i2c_chan *i2c = &mdev->i2c;
+	struct drm_connector *connector = &mdev->connector;
+	int ret;
+
+	ret = drm_universal_plane_init(dev, primary_plane, 0,
+				       &mgag200_g200ew3_primary_plane_funcs,
+				       mgag200_primary_plane_formats,
+				       mgag200_primary_plane_formats_size,
+				       mgag200_primary_plane_fmtmods,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		drm_err(dev, "drm_universal_plane_init() failed: %d\n", ret);
+		return ret;
+	}
+	drm_plane_helper_add(primary_plane, &mgag200_g200ew3_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&mgag200_g200ew3_crtc_funcs, NULL);
+	if (ret) {
+		drm_err(dev, "drm_crtc_init_with_planes() failed: %d\n", ret);
+		return ret;
+	}
+	drm_crtc_helper_add(crtc, &mgag200_g200ew3_crtc_helper_funcs);
+
+	/* FIXME: legacy gamma tables, but atomic gamma doesn't work without */
+	drm_mode_crtc_set_gamma_size(crtc, MGAG200_LUT_SIZE);
+	drm_crtc_enable_color_mgmt(crtc, 0, false, MGAG200_LUT_SIZE);
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+	ret = drm_encoder_init(dev, encoder, &mgag200_g200ew3_dac_encoder_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
+	if (ret) {
+		drm_err(dev, "drm_encoder_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = mgag200_i2c_init(mdev, i2c);
+	if (ret) {
+		drm_err(dev, "failed to add DDC bus: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_connector_init_with_ddc(dev, connector,
+					  &mgag200_g200ew3_vga_connector_funcs,
+					  DRM_MODE_CONNECTOR_VGA,
+					  &i2c->adapter);
+	if (ret) {
+		drm_err(dev, "drm_connector_init_with_ddc() failed: %d\n", ret);
+		return ret;
+	}
+	drm_connector_helper_add(connector, &mgag200_g200ew3_vga_connector_helper_funcs);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret) {
+		drm_err(dev, "drm_connector_attach_encoder() failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * DRM device
  */
@@ -13,6 +182,13 @@
 static const struct mgag200_device_info mgag200_g200ew3_device_info =
 	MGAG200_DEVICE_INFO_INIT(2048, 2048, 0, true, 0, 1, false);
 
+static const struct mgag200_device_funcs mgag200_g200ew3_device_funcs = {
+	.disable_vidrst = mgag200_bmc_disable_vidrst,
+	.enable_vidrst = mgag200_bmc_enable_vidrst,
+	.pixpllc_atomic_check = mgag200_g200ew3_pixpllc_atomic_check,
+	.pixpllc_atomic_update = mgag200_g200wb_pixpllc_atomic_update, // same as G200WB
+};
+
 static resource_size_t mgag200_g200ew3_device_probe_vram(struct mga_device *mdev)
 {
 	resource_size_t vram_size = resource_size(mdev->vram_res);
@@ -23,8 +199,7 @@ static resource_size_t mgag200_g200ew3_device_probe_vram(struct mga_device *mdev
 }
 
 struct mga_device *mgag200_g200ew3_device_create(struct pci_dev *pdev,
-						 const struct drm_driver *drv,
-						 enum mga_type type)
+						 const struct drm_driver *drv)
 {
 	struct mga_device *mdev;
 	struct drm_device *dev;
@@ -46,15 +221,24 @@ struct mga_device *mgag200_g200ew3_device_create(struct pci_dev *pdev,
 	if (ret)
 		return ERR_PTR(ret);
 
-	ret = mgag200_device_init(mdev, type, &mgag200_g200ew3_device_info);
+	ret = mgag200_device_init(mdev, &mgag200_g200ew3_device_info,
+				  &mgag200_g200ew3_device_funcs);
 	if (ret)
 		return ERR_PTR(ret);
 
+	mgag200_g200ew3_init_registers(mdev);
+
 	vram_available = mgag200_g200ew3_device_probe_vram(mdev);
 
-	ret = mgag200_modeset_init(mdev, vram_available);
+	ret = mgag200_mode_config_init(mdev, vram_available);
 	if (ret)
 		return ERR_PTR(ret);
 
+	ret = mgag200_g200ew3_pipeline_init(mdev);
+	if (ret)
+		return ERR_PTR(ret);
+
+	drm_mode_config_reset(dev);
+
 	return mdev;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200se.c b/drivers/gpu/drm/mgag200/mgag200_g200se.c
index 0a3e66695e22..be389ed91cbd 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200se.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200se.c
@@ -1,8 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
+#include <linux/delay.h>
 #include <linux/pci.h>
 
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_probe_helper.h>
 
 #include "mgag200_drv.h"
 
@@ -28,6 +33,404 @@ static int mgag200_g200se_init_pci_options(struct pci_dev *pdev)
 	return mgag200_init_pci_options(pdev, option, 0x00008000);
 }
 
+static void mgag200_g200se_init_registers(struct mgag200_g200se_device *g200se)
+{
+	static const u8 dacvalue[] = {
+		MGAG200_DAC_DEFAULT(0x03,
+				    MGA1064_PIX_CLK_CTL_SEL_PLL,
+				    MGA1064_MISC_CTL_DAC_EN |
+				    MGA1064_MISC_CTL_VGA8 |
+				    MGA1064_MISC_CTL_DAC_RAM_CS,
+				    0x00, 0x00, 0x00)
+	};
+
+	struct mga_device *mdev = &g200se->base;
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(dacvalue); i++) {
+		if ((i <= 0x17) ||
+		    (i == 0x1b) ||
+		    (i == 0x1c) ||
+		    ((i >= 0x1f) && (i <= 0x29)) ||
+		    ((i == 0x2c) || (i == 0x2d) || (i == 0x2e)) ||
+		    ((i >= 0x30) && (i <= 0x37)))
+			continue;
+		WREG_DAC(i, dacvalue[i]);
+	}
+
+	mgag200_init_registers(mdev);
+}
+
+static void mgag200_g200se_set_hiprilvl(struct mga_device *mdev,
+					const struct drm_display_mode *mode,
+					const struct drm_format_info *format)
+{
+	struct mgag200_g200se_device *g200se = to_mgag200_g200se_device(&mdev->base);
+	unsigned int hiprilvl;
+	u8 crtcext6;
+
+	if  (g200se->unique_rev_id >= 0x04) {
+		hiprilvl = 0;
+	} else if (g200se->unique_rev_id >= 0x02) {
+		unsigned int bpp;
+		unsigned long mb;
+
+		if (format->cpp[0] * 8 > 16)
+			bpp = 32;
+		else if (format->cpp[0] * 8 > 8)
+			bpp = 16;
+		else
+			bpp = 8;
+
+		mb = (mode->clock * bpp) / 1000;
+		if (mb > 3100)
+			hiprilvl = 0;
+		else if (mb > 2600)
+			hiprilvl = 1;
+		else if (mb > 1900)
+			hiprilvl = 2;
+		else if (mb > 1160)
+			hiprilvl = 3;
+		else if (mb > 440)
+			hiprilvl = 4;
+		else
+			hiprilvl = 5;
+
+	} else if (g200se->unique_rev_id >= 0x01) {
+		hiprilvl = 3;
+	} else {
+		hiprilvl = 4;
+	}
+
+	crtcext6 = hiprilvl; /* implicitly sets maxhipri to 0 */
+
+	WREG_ECRT(0x06, crtcext6);
+}
+
+/*
+ * PIXPLLC
+ */
+
+static int mgag200_g200se_00_pixpllc_atomic_check(struct drm_crtc *crtc,
+						  struct drm_atomic_state *new_state)
+{
+	static const unsigned int vcomax = 320000;
+	static const unsigned int vcomin = 160000;
+	static const unsigned int pllreffreq = 25000;
+
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	struct mgag200_crtc_state *new_mgag200_crtc_state = to_mgag200_crtc_state(new_crtc_state);
+	long clock = new_crtc_state->mode.clock;
+	struct mgag200_pll_values *pixpllc = &new_mgag200_crtc_state->pixpllc;
+	unsigned int delta, tmpdelta, permitteddelta;
+	unsigned int testp, testm, testn;
+	unsigned int p, m, n, s;
+	unsigned int computed;
+
+	m = n = p = s = 0;
+	delta = 0xffffffff;
+	permitteddelta = clock * 5 / 1000;
+
+	for (testp = 8; testp > 0; testp /= 2) {
+		if (clock * testp > vcomax)
+			continue;
+		if (clock * testp < vcomin)
+			continue;
+
+		for (testn = 17; testn < 256; testn++) {
+			for (testm = 1; testm < 32; testm++) {
+				computed = (pllreffreq * testn) / (testm * testp);
+				if (computed > clock)
+					tmpdelta = computed - clock;
+				else
+					tmpdelta = clock - computed;
+				if (tmpdelta < delta) {
+					delta = tmpdelta;
+					m = testm;
+					n = testn;
+					p = testp;
+				}
+			}
+		}
+	}
+
+	if (delta > permitteddelta) {
+		pr_warn("PLL delta too large\n");
+		return -EINVAL;
+	}
+
+	pixpllc->m = m;
+	pixpllc->n = n;
+	pixpllc->p = p;
+	pixpllc->s = s;
+
+	return 0;
+}
+
+static void mgag200_g200se_00_pixpllc_atomic_update(struct drm_crtc *crtc,
+						    struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	struct mgag200_pll_values *pixpllc = &mgag200_crtc_state->pixpllc;
+	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
+	u8 xpixpllcm, xpixpllcn, xpixpllcp;
+
+	pixpllcm = pixpllc->m - 1;
+	pixpllcn = pixpllc->n - 1;
+	pixpllcp = pixpllc->p - 1;
+	pixpllcs = pixpllc->s;
+
+	xpixpllcm = pixpllcm | ((pixpllcn & BIT(8)) >> 1);
+	xpixpllcn = pixpllcn;
+	xpixpllcp = (pixpllcs << 3) | pixpllcp;
+
+	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
+
+	WREG_DAC(MGA1064_PIX_PLLC_M, xpixpllcm);
+	WREG_DAC(MGA1064_PIX_PLLC_N, xpixpllcn);
+	WREG_DAC(MGA1064_PIX_PLLC_P, xpixpllcp);
+}
+
+static int mgag200_g200se_04_pixpllc_atomic_check(struct drm_crtc *crtc,
+						  struct drm_atomic_state *new_state)
+{
+	static const unsigned int vcomax = 1600000;
+	static const unsigned int vcomin = 800000;
+	static const unsigned int pllreffreq = 25000;
+	static const unsigned int pvalues_e4[] = {16, 14, 12, 10, 8, 6, 4, 2, 1};
+
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	struct mgag200_crtc_state *new_mgag200_crtc_state = to_mgag200_crtc_state(new_crtc_state);
+	long clock = new_crtc_state->mode.clock;
+	struct mgag200_pll_values *pixpllc = &new_mgag200_crtc_state->pixpllc;
+	unsigned int delta, tmpdelta, permitteddelta;
+	unsigned int testp, testm, testn;
+	unsigned int p, m, n, s;
+	unsigned int computed;
+	unsigned int fvv;
+	unsigned int i;
+
+	m = n = p = s = 0;
+	delta = 0xffffffff;
+
+	if (clock < 25000)
+		clock = 25000;
+	clock = clock * 2;
+
+	/* Permited delta is 0.5% as VESA Specification */
+	permitteddelta = clock * 5 / 1000;
+
+	for (i = 0 ; i < ARRAY_SIZE(pvalues_e4); i++) {
+		testp = pvalues_e4[i];
+
+		if ((clock * testp) > vcomax)
+			continue;
+		if ((clock * testp) < vcomin)
+			continue;
+
+		for (testn = 50; testn <= 256; testn++) {
+			for (testm = 1; testm <= 32; testm++) {
+				computed = (pllreffreq * testn) / (testm * testp);
+				if (computed > clock)
+					tmpdelta = computed - clock;
+				else
+					tmpdelta = clock - computed;
+
+				if (tmpdelta < delta) {
+					delta = tmpdelta;
+					m = testm;
+					n = testn;
+					p = testp;
+				}
+			}
+		}
+	}
+
+	fvv = pllreffreq * n / m;
+	fvv = (fvv - 800000) / 50000;
+	if (fvv > 15)
+		fvv = 15;
+	s = fvv << 1;
+
+	if (delta > permitteddelta) {
+		pr_warn("PLL delta too large\n");
+		return -EINVAL;
+	}
+
+	pixpllc->m = m;
+	pixpllc->n = n;
+	pixpllc->p = p;
+	pixpllc->s = s;
+
+	return 0;
+}
+
+static void mgag200_g200se_04_pixpllc_atomic_update(struct drm_crtc *crtc,
+						    struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	struct mgag200_pll_values *pixpllc = &mgag200_crtc_state->pixpllc;
+	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
+	u8 xpixpllcm, xpixpllcn, xpixpllcp;
+
+	pixpllcm = pixpllc->m - 1;
+	pixpllcn = pixpllc->n - 1;
+	pixpllcp = pixpllc->p - 1;
+	pixpllcs = pixpllc->s;
+
+	xpixpllcm = pixpllcm | ((pixpllcn & BIT(8)) >> 1);
+	xpixpllcn = pixpllcn;
+	xpixpllcp = (pixpllcs << 3) | pixpllcp;
+
+	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
+
+	WREG_DAC(MGA1064_PIX_PLLC_M, xpixpllcm);
+	WREG_DAC(MGA1064_PIX_PLLC_N, xpixpllcn);
+	WREG_DAC(MGA1064_PIX_PLLC_P, xpixpllcp);
+
+	WREG_DAC(0x1a, 0x09);
+	msleep(20);
+	WREG_DAC(0x1a, 0x01);
+}
+
+/*
+ * Mode-setting pipeline
+ */
+
+static const struct drm_plane_helper_funcs mgag200_g200se_primary_plane_helper_funcs = {
+	MGAG200_PRIMARY_PLANE_HELPER_FUNCS,
+};
+
+static const struct drm_plane_funcs mgag200_g200se_primary_plane_funcs = {
+	MGAG200_PRIMARY_PLANE_FUNCS,
+};
+
+static void mgag200_g200se_crtc_helper_atomic_enable(struct drm_crtc *crtc,
+						     struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	const struct mgag200_device_funcs *funcs = mdev->funcs;
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	const struct drm_format_info *format = mgag200_crtc_state->format;
+
+	if (funcs->disable_vidrst)
+		funcs->disable_vidrst(mdev);
+
+	mgag200_set_format_regs(mdev, format);
+	mgag200_set_mode_regs(mdev, adjusted_mode);
+
+	if (funcs->pixpllc_atomic_update)
+		funcs->pixpllc_atomic_update(crtc, old_state);
+
+	mgag200_g200se_set_hiprilvl(mdev, adjusted_mode, format);
+
+	mgag200_enable_display(mdev);
+
+	if (funcs->enable_vidrst)
+		funcs->enable_vidrst(mdev);
+}
+
+static const struct drm_crtc_helper_funcs mgag200_g200se_crtc_helper_funcs = {
+	.mode_valid = mgag200_crtc_helper_mode_valid,
+	.atomic_check = mgag200_crtc_helper_atomic_check,
+	.atomic_flush = mgag200_crtc_helper_atomic_flush,
+	.atomic_enable = mgag200_g200se_crtc_helper_atomic_enable,
+	.atomic_disable = mgag200_crtc_helper_atomic_disable
+};
+
+static const struct drm_crtc_funcs mgag200_g200se_crtc_funcs = {
+	MGAG200_CRTC_FUNCS,
+};
+
+static const struct drm_encoder_funcs mgag200_g200se_dac_encoder_funcs = {
+	MGAG200_DAC_ENCODER_FUNCS,
+};
+
+static const struct drm_connector_helper_funcs mgag200_g200se_vga_connector_helper_funcs = {
+	MGAG200_VGA_CONNECTOR_HELPER_FUNCS,
+};
+
+static const struct drm_connector_funcs mgag200_g200se_vga_connector_funcs = {
+	MGAG200_VGA_CONNECTOR_FUNCS,
+};
+
+static int mgag200_g200se_pipeline_init(struct mga_device *mdev)
+{
+	struct drm_device *dev = &mdev->base;
+	struct drm_plane *primary_plane = &mdev->primary_plane;
+	struct drm_crtc *crtc = &mdev->crtc;
+	struct drm_encoder *encoder = &mdev->encoder;
+	struct mga_i2c_chan *i2c = &mdev->i2c;
+	struct drm_connector *connector = &mdev->connector;
+	int ret;
+
+	ret = drm_universal_plane_init(dev, primary_plane, 0,
+				       &mgag200_g200se_primary_plane_funcs,
+				       mgag200_primary_plane_formats,
+				       mgag200_primary_plane_formats_size,
+				       mgag200_primary_plane_fmtmods,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		drm_err(dev, "drm_universal_plane_init() failed: %d\n", ret);
+		return ret;
+	}
+	drm_plane_helper_add(primary_plane, &mgag200_g200se_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&mgag200_g200se_crtc_funcs, NULL);
+	if (ret) {
+		drm_err(dev, "drm_crtc_init_with_planes() failed: %d\n", ret);
+		return ret;
+	}
+	drm_crtc_helper_add(crtc, &mgag200_g200se_crtc_helper_funcs);
+
+	/* FIXME: legacy gamma tables, but atomic gamma doesn't work without */
+	drm_mode_crtc_set_gamma_size(crtc, MGAG200_LUT_SIZE);
+	drm_crtc_enable_color_mgmt(crtc, 0, false, MGAG200_LUT_SIZE);
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+	ret = drm_encoder_init(dev, encoder, &mgag200_g200se_dac_encoder_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
+	if (ret) {
+		drm_err(dev, "drm_encoder_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = mgag200_i2c_init(mdev, i2c);
+	if (ret) {
+		drm_err(dev, "failed to add DDC bus: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_connector_init_with_ddc(dev, connector,
+					  &mgag200_g200se_vga_connector_funcs,
+					  DRM_MODE_CONNECTOR_VGA,
+					  &i2c->adapter);
+	if (ret) {
+		drm_err(dev, "drm_connector_init_with_ddc() failed: %d\n", ret);
+		return ret;
+	}
+	drm_connector_helper_add(connector, &mgag200_g200se_vga_connector_helper_funcs);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret) {
+		drm_err(dev, "drm_connector_attach_encoder() failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * DRM device
  */
@@ -65,11 +468,22 @@ static int mgag200_g200se_init_unique_rev_id(struct mgag200_g200se_device *g200s
 	return 0;
 }
 
+static const struct mgag200_device_funcs mgag200_g200se_00_device_funcs = {
+	.pixpllc_atomic_check = mgag200_g200se_00_pixpllc_atomic_check,
+	.pixpllc_atomic_update = mgag200_g200se_00_pixpllc_atomic_update,
+};
+
+static const struct mgag200_device_funcs mgag200_g200se_04_device_funcs = {
+	.pixpllc_atomic_check = mgag200_g200se_04_pixpllc_atomic_check,
+	.pixpllc_atomic_update = mgag200_g200se_04_pixpllc_atomic_update,
+};
+
 struct mga_device *mgag200_g200se_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
 						enum mga_type type)
 {
 	struct mgag200_g200se_device *g200se;
 	const struct mgag200_device_info *info;
+	const struct mgag200_device_funcs *funcs;
 	struct mga_device *mdev;
 	struct drm_device *dev;
 	resource_size_t vram_available;
@@ -116,15 +530,28 @@ struct mga_device *mgag200_g200se_device_create(struct pci_dev *pdev, const stru
 		return ERR_PTR(-EINVAL);
 	}
 
-	ret = mgag200_device_init(mdev, type, info);
+	if (g200se->unique_rev_id >= 0x04)
+		funcs = &mgag200_g200se_04_device_funcs;
+	else
+		funcs = &mgag200_g200se_00_device_funcs;
+
+	ret = mgag200_device_init(mdev, info, funcs);
 	if (ret)
 		return ERR_PTR(ret);
 
+	mgag200_g200se_init_registers(g200se);
+
 	vram_available = mgag200_device_probe_vram(mdev);
 
-	ret = mgag200_modeset_init(mdev, vram_available);
+	ret = mgag200_mode_config_init(mdev, vram_available);
 	if (ret)
 		return ERR_PTR(ret);
 
+	ret = mgag200_g200se_pipeline_init(mdev);
+	if (ret)
+		return ERR_PTR(ret);
+
+	drm_mode_config_reset(dev);
+
 	return mdev;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200wb.c b/drivers/gpu/drm/mgag200/mgag200_g200wb.c
index c8450ac8eaec..9baa727ac6f9 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200wb.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200wb.c
@@ -1,11 +1,314 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
+#include <linux/delay.h>
 #include <linux/pci.h>
 
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_gem_atomic_helper.h>
+#include <drm/drm_probe_helper.h>
 
 #include "mgag200_drv.h"
 
+void mgag200_g200wb_init_registers(struct mga_device *mdev)
+{
+	static const u8 dacvalue[] = {
+		MGAG200_DAC_DEFAULT(0x07, 0xc9, 0x1f, 0x00, 0x00, 0x00)
+	};
+
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(dacvalue); i++) {
+		if ((i <= 0x17) ||
+		    (i == 0x1b) ||
+		    (i == 0x1c) ||
+		    ((i >= 0x1f) && (i <= 0x29)) ||
+		    ((i >= 0x30) && (i <= 0x37)) ||
+		    ((i >= 0x44) && (i <= 0x4e)))
+			continue;
+		WREG_DAC(i, dacvalue[i]);
+	}
+
+	mgag200_init_registers(mdev);
+}
+
+/*
+ * PIXPLLC
+ */
+
+static int mgag200_g200wb_pixpllc_atomic_check(struct drm_crtc *crtc,
+					       struct drm_atomic_state *new_state)
+{
+	static const unsigned int vcomax = 550000;
+	static const unsigned int vcomin = 150000;
+	static const unsigned int pllreffreq = 48000;
+
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	struct mgag200_crtc_state *new_mgag200_crtc_state = to_mgag200_crtc_state(new_crtc_state);
+	long clock = new_crtc_state->mode.clock;
+	struct mgag200_pll_values *pixpllc = &new_mgag200_crtc_state->pixpllc;
+	unsigned int delta, tmpdelta;
+	unsigned int testp, testm, testn;
+	unsigned int p, m, n, s;
+	unsigned int computed;
+
+	m = n = p = s = 0;
+	delta = 0xffffffff;
+
+	for (testp = 1; testp < 9; testp++) {
+		if (clock * testp > vcomax)
+			continue;
+		if (clock * testp < vcomin)
+			continue;
+
+		for (testm = 1; testm < 17; testm++) {
+			for (testn = 1; testn < 151; testn++) {
+				computed = (pllreffreq * testn) / (testm * testp);
+				if (computed > clock)
+					tmpdelta = computed - clock;
+				else
+					tmpdelta = clock - computed;
+				if (tmpdelta < delta) {
+					delta = tmpdelta;
+					n = testn;
+					m = testm;
+					p = testp;
+					s = 0;
+				}
+			}
+		}
+	}
+
+	pixpllc->m = m;
+	pixpllc->n = n;
+	pixpllc->p = p;
+	pixpllc->s = s;
+
+	return 0;
+}
+
+void mgag200_g200wb_pixpllc_atomic_update(struct drm_crtc *crtc,
+					  struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	struct mgag200_pll_values *pixpllc = &mgag200_crtc_state->pixpllc;
+	bool pll_locked = false;
+	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
+	u8 xpixpllcm, xpixpllcn, xpixpllcp, tmp;
+	int i, j, tmpcount, vcount;
+
+	pixpllcm = pixpllc->m - 1;
+	pixpllcn = pixpllc->n - 1;
+	pixpllcp = pixpllc->p - 1;
+	pixpllcs = pixpllc->s;
+
+	xpixpllcm = ((pixpllcn & BIT(8)) >> 1) | pixpllcm;
+	xpixpllcn = pixpllcn;
+	xpixpllcp = ((pixpllcn & GENMASK(10, 9)) >> 3) | (pixpllcs << 3) | pixpllcp;
+
+	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
+
+	for (i = 0; i <= 32 && pll_locked == false; i++) {
+		if (i > 0) {
+			WREG8(MGAREG_CRTC_INDEX, 0x1e);
+			tmp = RREG8(MGAREG_CRTC_DATA);
+			if (tmp < 0xff)
+				WREG8(MGAREG_CRTC_DATA, tmp+1);
+		}
+
+		/* set pixclkdis to 1 */
+		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+		tmp = RREG8(DAC_DATA);
+		tmp |= MGA1064_PIX_CLK_CTL_CLK_DIS;
+		WREG8(DAC_DATA, tmp);
+
+		WREG8(DAC_INDEX, MGA1064_REMHEADCTL);
+		tmp = RREG8(DAC_DATA);
+		tmp |= MGA1064_REMHEADCTL_CLKDIS;
+		WREG8(DAC_DATA, tmp);
+
+		/* select PLL Set C */
+		tmp = RREG8(MGAREG_MEM_MISC_READ);
+		tmp |= 0x3 << 2;
+		WREG8(MGAREG_MEM_MISC_WRITE, tmp);
+
+		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+		tmp = RREG8(DAC_DATA);
+		tmp |= MGA1064_PIX_CLK_CTL_CLK_POW_DOWN | 0x80;
+		WREG8(DAC_DATA, tmp);
+
+		udelay(500);
+
+		/* reset the PLL */
+		WREG8(DAC_INDEX, MGA1064_VREF_CTL);
+		tmp = RREG8(DAC_DATA);
+		tmp &= ~0x04;
+		WREG8(DAC_DATA, tmp);
+
+		udelay(50);
+
+		/* program pixel pll register */
+		WREG_DAC(MGA1064_WB_PIX_PLLC_N, xpixpllcn);
+		WREG_DAC(MGA1064_WB_PIX_PLLC_M, xpixpllcm);
+		WREG_DAC(MGA1064_WB_PIX_PLLC_P, xpixpllcp);
+
+		udelay(50);
+
+		/* turn pll on */
+		WREG8(DAC_INDEX, MGA1064_VREF_CTL);
+		tmp = RREG8(DAC_DATA);
+		tmp |= 0x04;
+		WREG_DAC(MGA1064_VREF_CTL, tmp);
+
+		udelay(500);
+
+		/* select the pixel pll */
+		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+		tmp = RREG8(DAC_DATA);
+		tmp &= ~MGA1064_PIX_CLK_CTL_SEL_MSK;
+		tmp |= MGA1064_PIX_CLK_CTL_SEL_PLL;
+		WREG8(DAC_DATA, tmp);
+
+		WREG8(DAC_INDEX, MGA1064_REMHEADCTL);
+		tmp = RREG8(DAC_DATA);
+		tmp &= ~MGA1064_REMHEADCTL_CLKSL_MSK;
+		tmp |= MGA1064_REMHEADCTL_CLKSL_PLL;
+		WREG8(DAC_DATA, tmp);
+
+		/* reset dotclock rate bit */
+		WREG8(MGAREG_SEQ_INDEX, 1);
+		tmp = RREG8(MGAREG_SEQ_DATA);
+		tmp &= ~0x8;
+		WREG8(MGAREG_SEQ_DATA, tmp);
+
+		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
+		tmp = RREG8(DAC_DATA);
+		tmp &= ~MGA1064_PIX_CLK_CTL_CLK_DIS;
+		WREG8(DAC_DATA, tmp);
+
+		vcount = RREG8(MGAREG_VCOUNT);
+
+		for (j = 0; j < 30 && pll_locked == false; j++) {
+			tmpcount = RREG8(MGAREG_VCOUNT);
+			if (tmpcount < vcount)
+				vcount = 0;
+			if ((tmpcount - vcount) > 2)
+				pll_locked = true;
+			else
+				udelay(5);
+		}
+	}
+
+	WREG8(DAC_INDEX, MGA1064_REMHEADCTL);
+	tmp = RREG8(DAC_DATA);
+	tmp &= ~MGA1064_REMHEADCTL_CLKDIS;
+	WREG_DAC(MGA1064_REMHEADCTL, tmp);
+}
+
+/*
+ * Mode-setting pipeline
+ */
+
+static const struct drm_plane_helper_funcs mgag200_g200wb_primary_plane_helper_funcs = {
+	MGAG200_PRIMARY_PLANE_HELPER_FUNCS,
+};
+
+static const struct drm_plane_funcs mgag200_g200wb_primary_plane_funcs = {
+	MGAG200_PRIMARY_PLANE_FUNCS,
+};
+
+static const struct drm_crtc_helper_funcs mgag200_g200wb_crtc_helper_funcs = {
+	MGAG200_CRTC_HELPER_FUNCS,
+};
+
+static const struct drm_crtc_funcs mgag200_g200wb_crtc_funcs = {
+	MGAG200_CRTC_FUNCS,
+};
+
+static const struct drm_encoder_funcs mgag200_g200wb_dac_encoder_funcs = {
+	MGAG200_DAC_ENCODER_FUNCS,
+};
+
+static const struct drm_connector_helper_funcs mgag200_g200wb_vga_connector_helper_funcs = {
+	MGAG200_VGA_CONNECTOR_HELPER_FUNCS,
+};
+
+static const struct drm_connector_funcs mgag200_g200wb_vga_connector_funcs = {
+	MGAG200_VGA_CONNECTOR_FUNCS,
+};
+
+static int mgag200_g200wb_pipeline_init(struct mga_device *mdev)
+{
+	struct drm_device *dev = &mdev->base;
+	struct drm_plane *primary_plane = &mdev->primary_plane;
+	struct drm_crtc *crtc = &mdev->crtc;
+	struct drm_encoder *encoder = &mdev->encoder;
+	struct mga_i2c_chan *i2c = &mdev->i2c;
+	struct drm_connector *connector = &mdev->connector;
+	int ret;
+
+	ret = drm_universal_plane_init(dev, primary_plane, 0,
+				       &mgag200_g200wb_primary_plane_funcs,
+				       mgag200_primary_plane_formats,
+				       mgag200_primary_plane_formats_size,
+				       mgag200_primary_plane_fmtmods,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		drm_err(dev, "drm_universal_plane_init() failed: %d\n", ret);
+		return ret;
+	}
+	drm_plane_helper_add(primary_plane, &mgag200_g200wb_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&mgag200_g200wb_crtc_funcs, NULL);
+	if (ret) {
+		drm_err(dev, "drm_crtc_init_with_planes() failed: %d\n", ret);
+		return ret;
+	}
+	drm_crtc_helper_add(crtc, &mgag200_g200wb_crtc_helper_funcs);
+
+	/* FIXME: legacy gamma tables, but atomic gamma doesn't work without */
+	drm_mode_crtc_set_gamma_size(crtc, MGAG200_LUT_SIZE);
+	drm_crtc_enable_color_mgmt(crtc, 0, false, MGAG200_LUT_SIZE);
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+	ret = drm_encoder_init(dev, encoder, &mgag200_g200wb_dac_encoder_funcs,
+			       DRM_MODE_ENCODER_DAC, NULL);
+	if (ret) {
+		drm_err(dev, "drm_encoder_init() failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = mgag200_i2c_init(mdev, i2c);
+	if (ret) {
+		drm_err(dev, "failed to add DDC bus: %d\n", ret);
+		return ret;
+	}
+
+	ret = drm_connector_init_with_ddc(dev, connector,
+					  &mgag200_g200wb_vga_connector_funcs,
+					  DRM_MODE_CONNECTOR_VGA,
+					  &i2c->adapter);
+	if (ret) {
+		drm_err(dev, "drm_connector_init_with_ddc() failed: %d\n", ret);
+		return ret;
+	}
+	drm_connector_helper_add(connector, &mgag200_g200wb_vga_connector_helper_funcs);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret) {
+		drm_err(dev, "drm_connector_attach_encoder() failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * DRM device
  */
@@ -13,8 +316,14 @@
 static const struct mgag200_device_info mgag200_g200wb_device_info =
 	MGAG200_DEVICE_INFO_INIT(1280, 1024, 31877, true, 0, 1, false);
 
-struct mga_device *mgag200_g200wb_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
-						enum mga_type type)
+static const struct mgag200_device_funcs mgag200_g200wb_device_funcs = {
+	.disable_vidrst = mgag200_bmc_disable_vidrst,
+	.enable_vidrst = mgag200_bmc_enable_vidrst,
+	.pixpllc_atomic_check = mgag200_g200wb_pixpllc_atomic_check,
+	.pixpllc_atomic_update = mgag200_g200wb_pixpllc_atomic_update,
+};
+
+struct mga_device *mgag200_g200wb_device_create(struct pci_dev *pdev, const struct drm_driver *drv)
 {
 	struct mga_device *mdev;
 	struct drm_device *dev;
@@ -36,15 +345,24 @@ struct mga_device *mgag200_g200wb_device_create(struct pci_dev *pdev, const stru
 	if (ret)
 		return ERR_PTR(ret);
 
-	ret = mgag200_device_init(mdev, type, &mgag200_g200wb_device_info);
+	ret = mgag200_device_init(mdev, &mgag200_g200wb_device_info,
+				  &mgag200_g200wb_device_funcs);
 	if (ret)
 		return ERR_PTR(ret);
 
+	mgag200_g200wb_init_registers(mdev);
+
 	vram_available = mgag200_device_probe_vram(mdev);
 
-	ret = mgag200_modeset_init(mdev, vram_available);
+	ret = mgag200_mode_config_init(mdev, vram_available);
 	if (ret)
 		return ERR_PTR(ret);
 
+	ret = mgag200_g200wb_pipeline_init(mdev);
+	if (ret)
+		return ERR_PTR(ret);
+
+	drm_mode_config_reset(dev);
+
 	return mdev;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index 225cca2ed60e..bbab2549243a 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -11,24 +11,19 @@
 #include <linux/delay.h>
 #include <linux/iosys-map.h>
 
+#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_atomic_state_helper.h>
-#include <drm/drm_crtc_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_format_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #include "mgag200_drv.h"
 
-#define MGAG200_LUT_SIZE 256
-
 /*
  * This file contains setup code for the CRTC.
  */
@@ -132,95 +127,6 @@ static inline void mga_wait_busy(struct mga_device *mdev)
 	} while ((status & 0x01) && time_before(jiffies, timeout));
 }
 
-static void mgag200_g200wb_hold_bmc(struct mga_device *mdev)
-{
-	u8 tmp;
-	int iter_max;
-
-	/* 1- The first step is to warn the BMC of an upcoming mode change.
-	 * We are putting the misc<0> to output.*/
-
-	WREG8(DAC_INDEX, MGA1064_GEN_IO_CTL);
-	tmp = RREG8(DAC_DATA);
-	tmp |= 0x10;
-	WREG_DAC(MGA1064_GEN_IO_CTL, tmp);
-
-	/* we are putting a 1 on the misc<0> line */
-	WREG8(DAC_INDEX, MGA1064_GEN_IO_DATA);
-	tmp = RREG8(DAC_DATA);
-	tmp |= 0x10;
-	WREG_DAC(MGA1064_GEN_IO_DATA, tmp);
-
-	/* 2- Second step to mask and further scan request
-	 * This will be done by asserting the remfreqmsk bit (XSPAREREG<7>)
-	 */
-	WREG8(DAC_INDEX, MGA1064_SPAREREG);
-	tmp = RREG8(DAC_DATA);
-	tmp |= 0x80;
-	WREG_DAC(MGA1064_SPAREREG, tmp);
-
-	/* 3a- the third step is to verifu if there is an active scan
-	 * We are searching for a 0 on remhsyncsts <XSPAREREG<0>)
-	 */
-	iter_max = 300;
-	while (!(tmp & 0x1) && iter_max) {
-		WREG8(DAC_INDEX, MGA1064_SPAREREG);
-		tmp = RREG8(DAC_DATA);
-		udelay(1000);
-		iter_max--;
-	}
-
-	/* 3b- this step occurs only if the remove is actually scanning
-	 * we are waiting for the end of the frame which is a 1 on
-	 * remvsyncsts (XSPAREREG<1>)
-	 */
-	if (iter_max) {
-		iter_max = 300;
-		while ((tmp & 0x2) && iter_max) {
-			WREG8(DAC_INDEX, MGA1064_SPAREREG);
-			tmp = RREG8(DAC_DATA);
-			udelay(1000);
-			iter_max--;
-		}
-	}
-}
-
-static void mgag200_g200wb_release_bmc(struct mga_device *mdev)
-{
-	u8 tmp;
-
-	/* 1- The first step is to ensure that the vrsten and hrsten are set */
-	WREG8(MGAREG_CRTCEXT_INDEX, 1);
-	tmp = RREG8(MGAREG_CRTCEXT_DATA);
-	WREG8(MGAREG_CRTCEXT_DATA, tmp | 0x88);
-
-	/* 2- second step is to assert the rstlvl2 */
-	WREG8(DAC_INDEX, MGA1064_REMHEADCTL2);
-	tmp = RREG8(DAC_DATA);
-	tmp |= 0x8;
-	WREG8(DAC_DATA, tmp);
-
-	/* wait 10 us */
-	udelay(10);
-
-	/* 3- deassert rstlvl2 */
-	tmp &= ~0x08;
-	WREG8(DAC_INDEX, MGA1064_REMHEADCTL2);
-	WREG8(DAC_DATA, tmp);
-
-	/* 4- remove mask of scan request */
-	WREG8(DAC_INDEX, MGA1064_SPAREREG);
-	tmp = RREG8(DAC_DATA);
-	tmp &= ~0x80;
-	WREG8(DAC_DATA, tmp);
-
-	/* 5- put back a 0 on the misc<0> line */
-	WREG8(DAC_INDEX, MGA1064_GEN_IO_DATA);
-	tmp = RREG8(DAC_DATA);
-	tmp &= ~0x10;
-	WREG_DAC(MGA1064_GEN_IO_DATA, tmp);
-}
-
 /*
  * This is how the framebuffer base address is stored in g200 cards:
  *   * Assume @offset is the gpu_addr variable of the framebuffer object
@@ -267,86 +173,10 @@ static void mgag200_set_startadd(struct mga_device *mdev,
 	WREG_ECRT(0x00, crtcext0);
 }
 
-static void mgag200_set_dac_regs(struct mga_device *mdev)
-{
-	size_t i;
-	u8 dacvalue[] = {
-		/* 0x00: */        0,    0,    0,    0,    0,    0, 0x00,    0,
-		/* 0x08: */        0,    0,    0,    0,    0,    0,    0,    0,
-		/* 0x10: */        0,    0,    0,    0,    0,    0,    0,    0,
-		/* 0x18: */     0x00,    0, 0xC9, 0xFF, 0xBF, 0x20, 0x1F, 0x20,
-		/* 0x20: */     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
-		/* 0x28: */     0x00, 0x00, 0x00, 0x00,    0,    0,    0, 0x40,
-		/* 0x30: */     0x00, 0xB0, 0x00, 0xC2, 0x34, 0x14, 0x02, 0x83,
-		/* 0x38: */     0x00, 0x93, 0x00, 0x77, 0x00, 0x00, 0x00, 0x3A,
-		/* 0x40: */        0,    0,    0,    0,    0,    0,    0,    0,
-		/* 0x48: */        0,    0,    0,    0,    0,    0,    0,    0
-	};
-
-	switch (mdev->type) {
-	case G200_PCI:
-	case G200_AGP:
-		dacvalue[MGA1064_SYS_PLL_M] = 0x04;
-		dacvalue[MGA1064_SYS_PLL_N] = 0x2D;
-		dacvalue[MGA1064_SYS_PLL_P] = 0x19;
-		break;
-	case G200_SE_A:
-	case G200_SE_B:
-		dacvalue[MGA1064_VREF_CTL] = 0x03;
-		dacvalue[MGA1064_PIX_CLK_CTL] = MGA1064_PIX_CLK_CTL_SEL_PLL;
-		dacvalue[MGA1064_MISC_CTL] = MGA1064_MISC_CTL_DAC_EN |
-					     MGA1064_MISC_CTL_VGA8 |
-					     MGA1064_MISC_CTL_DAC_RAM_CS;
-		break;
-	case G200_WB:
-	case G200_EW3:
-		dacvalue[MGA1064_VREF_CTL] = 0x07;
-		break;
-	case G200_EV:
-		dacvalue[MGA1064_PIX_CLK_CTL] = MGA1064_PIX_CLK_CTL_SEL_PLL;
-		dacvalue[MGA1064_MISC_CTL] = MGA1064_MISC_CTL_VGA8 |
-					     MGA1064_MISC_CTL_DAC_RAM_CS;
-		break;
-	case G200_EH:
-	case G200_EH3:
-		dacvalue[MGA1064_MISC_CTL] = MGA1064_MISC_CTL_VGA8 |
-					     MGA1064_MISC_CTL_DAC_RAM_CS;
-		break;
-	case G200_ER:
-		break;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(dacvalue); i++) {
-		if ((i <= 0x17) ||
-		    (i == 0x1b) ||
-		    (i == 0x1c) ||
-		    ((i >= 0x1f) && (i <= 0x29)) ||
-		    ((i >= 0x30) && (i <= 0x37)))
-			continue;
-		if (IS_G200_SE(mdev) &&
-		    ((i == 0x2c) || (i == 0x2d) || (i == 0x2e)))
-			continue;
-		if ((mdev->type == G200_EV ||
-		    mdev->type == G200_WB ||
-		    mdev->type == G200_EH ||
-		    mdev->type == G200_EW3 ||
-		    mdev->type == G200_EH3) &&
-		    (i >= 0x44) && (i <= 0x4e))
-			continue;
-
-		WREG_DAC(i, dacvalue[i]);
-	}
-
-	if (mdev->type == G200_ER)
-		WREG_DAC(0x90, 0);
-}
-
-static void mgag200_init_regs(struct mga_device *mdev)
+void mgag200_init_registers(struct mga_device *mdev)
 {
 	u8 crtc11, misc;
 
-	mgag200_set_dac_regs(mdev);
-
 	WREG_SEQ(2, 0x0f);
 	WREG_SEQ(3, 0x00);
 	WREG_SEQ(4, 0x0e);
@@ -364,19 +194,12 @@ static void mgag200_init_regs(struct mga_device *mdev)
 		    MGAREG_CRTC11_VINTCLR);
 	WREG_CRT(0x11, crtc11);
 
-	if (mdev->type == G200_ER)
-		WREG_ECRT(0x24, 0x5);
-
-	if (mdev->type == G200_EW3)
-		WREG_ECRT(0x34, 0x5);
-
 	misc = RREG8(MGA_MISC_IN);
 	misc |= MGAREG_MISC_IOADSEL;
 	WREG8(MGA_MISC_OUT, misc);
 }
 
-static void mgag200_set_mode_regs(struct mga_device *mdev,
-				  const struct drm_display_mode *mode)
+void mgag200_set_mode_regs(struct mga_device *mdev, const struct drm_display_mode *mode)
 {
 	const struct mgag200_device_info *info = mdev->info;
 	unsigned int hdisplay, hsyncstart, hsyncend, htotal;
@@ -500,11 +323,9 @@ static void mgag200_set_offset(struct mga_device *mdev,
 	WREG_ECRT(0x00, crtcext0);
 }
 
-static void mgag200_set_format_regs(struct mga_device *mdev,
-				    const struct drm_framebuffer *fb)
+void mgag200_set_format_regs(struct mga_device *mdev, const struct drm_format_info *format)
 {
 	struct drm_device *dev = &mdev->base;
-	const struct drm_format_info *format = fb->format;
 	unsigned int bpp, bppshift, scale;
 	u8 crtcext3, xmulctrl;
 
@@ -565,76 +386,9 @@ static void mgag200_set_format_regs(struct mga_device *mdev,
 	WREG_ECRT(3, crtcext3);
 }
 
-static void mgag200_g200er_reset_tagfifo(struct mga_device *mdev)
+void mgag200_enable_display(struct mga_device *mdev)
 {
-	static uint32_t RESET_FLAG = 0x00200000; /* undocumented magic value */
-	u32 memctl;
-
-	memctl = RREG32(MGAREG_MEMCTL);
-
-	memctl |= RESET_FLAG;
-	WREG32(MGAREG_MEMCTL, memctl);
-
-	udelay(1000);
-
-	memctl &= ~RESET_FLAG;
-	WREG32(MGAREG_MEMCTL, memctl);
-}
-
-static void mgag200_g200se_set_hiprilvl(struct mga_device *mdev,
-					const struct drm_display_mode *mode,
-					const struct drm_framebuffer *fb)
-{
-	struct mgag200_g200se_device *g200se = to_mgag200_g200se_device(&mdev->base);
-	unsigned int hiprilvl;
-	u8 crtcext6;
-
-	if  (g200se->unique_rev_id >= 0x04) {
-		hiprilvl = 0;
-	} else if (g200se->unique_rev_id >= 0x02) {
-		unsigned int bpp;
-		unsigned long mb;
-
-		if (fb->format->cpp[0] * 8 > 16)
-			bpp = 32;
-		else if (fb->format->cpp[0] * 8 > 8)
-			bpp = 16;
-		else
-			bpp = 8;
-
-		mb = (mode->clock * bpp) / 1000;
-		if (mb > 3100)
-			hiprilvl = 0;
-		else if (mb > 2600)
-			hiprilvl = 1;
-		else if (mb > 1900)
-			hiprilvl = 2;
-		else if (mb > 1160)
-			hiprilvl = 3;
-		else if (mb > 440)
-			hiprilvl = 4;
-		else
-			hiprilvl = 5;
-
-	} else if (g200se->unique_rev_id >= 0x01) {
-		hiprilvl = 3;
-	} else {
-		hiprilvl = 4;
-	}
-
-	crtcext6 = hiprilvl; /* implicitly sets maxhipri to 0 */
-
-	WREG_ECRT(0x06, crtcext6);
-}
-
-static void mgag200_g200ev_set_hiprilvl(struct mga_device *mdev)
-{
-	WREG_ECRT(0x06, 0x00);
-}
-
-static void mgag200_enable_display(struct mga_device *mdev)
-{
-	u8 seq0, seq1, crtcext1;
+	u8 seq0, crtcext1;
 
 	RREG_SEQ(0x00, seq0);
 	seq0 |= MGAREG_SEQ0_SYNCRST |
@@ -648,12 +402,6 @@ static void mgag200_enable_display(struct mga_device *mdev)
 	mga_wait_vsync(mdev);
 	mga_wait_busy(mdev);
 
-	RREG_SEQ(0x01, seq1);
-	seq1 &= ~MGAREG_SEQ1_SCROFF;
-	WREG_SEQ(0x01, seq1);
-
-	msleep(20);
-
 	RREG_ECRT(0x01, crtcext1);
 	crtcext1 &= ~MGAREG_CRTCEXT1_VSYNCOFF;
 	crtcext1 &= ~MGAREG_CRTCEXT1_HSYNCOFF;
@@ -662,7 +410,7 @@ static void mgag200_enable_display(struct mga_device *mdev)
 
 static void mgag200_disable_display(struct mga_device *mdev)
 {
-	u8 seq0, seq1, crtcext1;
+	u8 seq0, crtcext1;
 
 	RREG_SEQ(0x00, seq0);
 	seq0 &= ~MGAREG_SEQ0_SYNCRST;
@@ -675,59 +423,127 @@ static void mgag200_disable_display(struct mga_device *mdev)
 	mga_wait_vsync(mdev);
 	mga_wait_busy(mdev);
 
-	RREG_SEQ(0x01, seq1);
-	seq1 |= MGAREG_SEQ1_SCROFF;
-	WREG_SEQ(0x01, seq1);
-
-	msleep(20);
-
 	RREG_ECRT(0x01, crtcext1);
 	crtcext1 |= MGAREG_CRTCEXT1_VSYNCOFF |
 		    MGAREG_CRTCEXT1_HSYNCOFF;
 	WREG_ECRT(0x01, crtcext1);
 }
 
+static void mgag200_handle_damage(struct mga_device *mdev, const struct iosys_map *vmap,
+				  struct drm_framebuffer *fb, struct drm_rect *clip)
+{
+	struct iosys_map dst = IOSYS_MAP_INIT_VADDR_IOMEM(mdev->vram);
+
+	iosys_map_incr(&dst, drm_fb_clip_offset(fb->pitches[0], fb->format, clip));
+	drm_fb_memcpy(&dst, fb->pitches, vmap, fb, clip);
+}
+
 /*
- * Connector
+ * Primary plane
  */
 
-static int mgag200_vga_connector_helper_get_modes(struct drm_connector *connector)
+const uint32_t mgag200_primary_plane_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_RGB565,
+	DRM_FORMAT_RGB888,
+};
+
+const size_t mgag200_primary_plane_formats_size = ARRAY_SIZE(mgag200_primary_plane_formats);
+
+const uint64_t mgag200_primary_plane_fmtmods[] = {
+	DRM_FORMAT_MOD_LINEAR,
+	DRM_FORMAT_MOD_INVALID
+};
+
+int mgag200_primary_plane_helper_atomic_check(struct drm_plane *plane,
+					      struct drm_atomic_state *new_state)
 {
-	struct mga_device *mdev = to_mga_device(connector->dev);
+	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(new_state, plane);
+	struct drm_framebuffer *new_fb = new_plane_state->fb;
+	struct drm_framebuffer *fb = NULL;
+	struct drm_crtc *new_crtc = new_plane_state->crtc;
+	struct drm_crtc_state *new_crtc_state = NULL;
+	struct mgag200_crtc_state *new_mgag200_crtc_state;
 	int ret;
 
-	/*
-	 * Protect access to I/O registers from concurrent modesetting
-	 * by acquiring the I/O-register lock.
-	 */
-	mutex_lock(&mdev->rmmio_lock);
-	ret = drm_connector_helper_get_modes_from_ddc(connector);
-	mutex_unlock(&mdev->rmmio_lock);
+	if (new_crtc)
+		new_crtc_state = drm_atomic_get_new_crtc_state(new_state, new_crtc);
 
-	return ret;
+	ret = drm_atomic_helper_check_plane_state(new_plane_state, new_crtc_state,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  false, true);
+	if (ret)
+		return ret;
+	else if (!new_plane_state->visible)
+		return 0;
+
+	if (plane->state)
+		fb = plane->state->fb;
+
+	if (!fb || (fb->format != new_fb->format))
+		new_crtc_state->mode_changed = true; /* update PLL settings */
+
+	new_mgag200_crtc_state = to_mgag200_crtc_state(new_crtc_state);
+	new_mgag200_crtc_state->format = new_fb->format;
+
+	return 0;
 }
 
-static const struct drm_connector_helper_funcs mga_vga_connector_helper_funcs = {
-	.get_modes  = mgag200_vga_connector_helper_get_modes,
-};
+void mgag200_primary_plane_helper_atomic_update(struct drm_plane *plane,
+						struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = plane->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	struct drm_plane_state *plane_state = plane->state;
+	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(old_state, plane);
+	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
+	struct drm_framebuffer *fb = plane_state->fb;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_rect damage;
+	u8 seq1;
 
-static const struct drm_connector_funcs mga_vga_connector_funcs = {
-	.reset                  = drm_atomic_helper_connector_reset,
-	.fill_modes             = drm_helper_probe_single_connector_modes,
-	.destroy                = drm_connector_cleanup,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state   = drm_atomic_helper_connector_destroy_state,
-};
+	if (!fb)
+		return;
+
+	drm_atomic_helper_damage_iter_init(&iter, old_plane_state, plane_state);
+	drm_atomic_for_each_plane_damage(&iter, &damage) {
+		mgag200_handle_damage(mdev, shadow_plane_state->data, fb, &damage);
+	}
+
+	/* Always scanout image at VRAM offset 0 */
+	mgag200_set_startadd(mdev, (u32)0);
+	mgag200_set_offset(mdev, fb);
+
+	if (!old_plane_state->crtc && plane_state->crtc) { // enabling
+		RREG_SEQ(0x01, seq1);
+		seq1 &= ~MGAREG_SEQ1_SCROFF;
+		WREG_SEQ(0x01, seq1);
+		msleep(20);
+	}
+}
+
+void mgag200_primary_plane_helper_atomic_disable(struct drm_plane *plane,
+						 struct drm_atomic_state *old_state)
+{
+	struct drm_device *dev = plane->dev;
+	struct mga_device *mdev = to_mga_device(dev);
+	u8 seq1;
+
+	RREG_SEQ(0x01, seq1);
+	seq1 |= MGAREG_SEQ1_SCROFF;
+	WREG_SEQ(0x01, seq1);
+	msleep(20);
+}
 
 /*
- * Simple Display Pipe
+ * CRTC
  */
 
-static enum drm_mode_status
-mgag200_simple_display_pipe_mode_valid(struct drm_simple_display_pipe *pipe,
-				       const struct drm_display_mode *mode)
+enum drm_mode_status mgag200_crtc_helper_mode_valid(struct drm_crtc *crtc,
+						    const struct drm_display_mode *mode)
 {
-	struct mga_device *mdev = to_mga_device(pipe->crtc.dev);
+	struct mga_device *mdev = to_mga_device(crtc->dev);
 	const struct mgag200_device_info *info = mdev->info;
 
 	/*
@@ -754,167 +570,112 @@ mgag200_simple_display_pipe_mode_valid(struct drm_simple_display_pipe *pipe,
 	return MODE_OK;
 }
 
-static void
-mgag200_handle_damage(struct mga_device *mdev, struct drm_framebuffer *fb,
-		      struct drm_rect *clip, const struct iosys_map *map)
-{
-	void __iomem *dst = mdev->vram;
-	void *vmap = map->vaddr; /* TODO: Use mapping abstraction properly */
-
-	dst += drm_fb_clip_offset(fb->pitches[0], fb->format, clip);
-	drm_fb_memcpy_toio(dst, fb->pitches[0], vmap, fb, clip);
-}
-
-static void
-mgag200_simple_display_pipe_enable(struct drm_simple_display_pipe *pipe,
-				   struct drm_crtc_state *crtc_state,
-				   struct drm_plane_state *plane_state)
+int mgag200_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *new_state)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
-	struct mgag200_pll *pixpll = &mdev->pixpll;
-	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
-	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
-	struct drm_framebuffer *fb = plane_state->fb;
-	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
-	struct drm_rect fullscreen = {
-		.x1 = 0,
-		.x2 = fb->width,
-		.y1 = 0,
-		.y2 = fb->height,
-	};
-
-	/*
-	 * Concurrent operations could possibly trigger a call to
-	 * drm_connector_helper_funcs.get_modes by trying to read the
-	 * display modes. Protect access to I/O registers by acquiring
-	 * the I/O-register lock.
-	 */
-	mutex_lock(&mdev->rmmio_lock);
-
-	if (mdev->type == G200_WB || mdev->type == G200_EW3)
-		mgag200_g200wb_hold_bmc(mdev);
-
-	mgag200_set_format_regs(mdev, fb);
-	mgag200_set_mode_regs(mdev, adjusted_mode);
-
-	pixpll->funcs->update(pixpll, &mgag200_crtc_state->pixpllc);
-
-	if (mdev->type == G200_ER)
-		mgag200_g200er_reset_tagfifo(mdev);
-
-	if (IS_G200_SE(mdev))
-		mgag200_g200se_set_hiprilvl(mdev, adjusted_mode, fb);
-	else if (mdev->type == G200_EV)
-		mgag200_g200ev_set_hiprilvl(mdev);
-
-	if (mdev->type == G200_WB || mdev->type == G200_EW3)
-		mgag200_g200wb_release_bmc(mdev);
+	const struct mgag200_device_funcs *funcs = mdev->funcs;
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	struct drm_property_blob *new_gamma_lut = new_crtc_state->gamma_lut;
+	int ret;
 
-	if (crtc_state->gamma_lut)
-		mgag200_crtc_set_gamma(mdev, fb->format, crtc_state->gamma_lut->data);
-	else
-		mgag200_crtc_set_gamma_linear(mdev, fb->format);
+	ret = drm_atomic_helper_check_crtc_state(new_crtc_state, false);
+	if (ret)
+		return ret;
 
-	mgag200_enable_display(mdev);
+	if (!new_crtc_state->enable)
+		return 0;
 
-	mgag200_handle_damage(mdev, fb, &fullscreen, &shadow_plane_state->data[0]);
+	if (new_crtc_state->mode_changed) {
+		if (funcs->pixpllc_atomic_check) {
+			ret = funcs->pixpllc_atomic_check(crtc, new_state);
+			if (ret)
+				return ret;
+		}
+	}
 
-	/* Always scanout image at VRAM offset 0 */
-	mgag200_set_startadd(mdev, (u32)0);
-	mgag200_set_offset(mdev, fb);
+	if (new_crtc_state->color_mgmt_changed && new_gamma_lut) {
+		if (new_gamma_lut->length != MGAG200_LUT_SIZE * sizeof(struct drm_color_lut)) {
+			drm_dbg(dev, "Wrong size for gamma_lut %zu\n", new_gamma_lut->length);
+			return -EINVAL;
+		}
+	}
 
-	mutex_unlock(&mdev->rmmio_lock);
+	return drm_atomic_add_affected_planes(new_state, crtc);
 }
 
-static void
-mgag200_simple_display_pipe_disable(struct drm_simple_display_pipe *pipe)
+void mgag200_crtc_helper_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *old_state)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct mga_device *mdev = to_mga_device(crtc->dev);
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
+	struct drm_device *dev = crtc->dev;
+	struct mga_device *mdev = to_mga_device(dev);
 
-	mgag200_disable_display(mdev);
+	if (crtc_state->enable && crtc_state->color_mgmt_changed) {
+		const struct drm_format_info *format = mgag200_crtc_state->format;
+
+		if (crtc_state->gamma_lut)
+			mgag200_crtc_set_gamma(mdev, format, crtc_state->gamma_lut->data);
+		else
+			mgag200_crtc_set_gamma_linear(mdev, format);
+	}
 }
 
-static int
-mgag200_simple_display_pipe_check(struct drm_simple_display_pipe *pipe,
-				  struct drm_plane_state *plane_state,
-				  struct drm_crtc_state *crtc_state)
+void mgag200_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *old_state)
 {
-	struct drm_plane *plane = plane_state->plane;
-	struct drm_device *dev = plane->dev;
+	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
-	struct mgag200_pll *pixpll = &mdev->pixpll;
+	const struct mgag200_device_funcs *funcs = mdev->funcs;
+	struct drm_crtc_state *crtc_state = crtc->state;
+	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
 	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
-	struct drm_framebuffer *new_fb = plane_state->fb;
-	struct drm_framebuffer *fb = NULL;
-	int ret;
+	const struct drm_format_info *format = mgag200_crtc_state->format;
 
-	if (!new_fb)
-		return 0;
+	if (funcs->disable_vidrst)
+		funcs->disable_vidrst(mdev);
 
-	if (plane->state)
-		fb = plane->state->fb;
+	mgag200_set_format_regs(mdev, format);
+	mgag200_set_mode_regs(mdev, adjusted_mode);
 
-	if (!fb || (fb->format != new_fb->format))
-		crtc_state->mode_changed = true; /* update PLL settings */
+	if (funcs->pixpllc_atomic_update)
+		funcs->pixpllc_atomic_update(crtc, old_state);
 
-	if (crtc_state->mode_changed) {
-		ret = pixpll->funcs->compute(pixpll, crtc_state->mode.clock,
-					     &mgag200_crtc_state->pixpllc);
-		if (ret)
-			return ret;
-	}
+	mgag200_enable_display(mdev);
 
-	if (crtc_state->color_mgmt_changed && crtc_state->gamma_lut) {
-		if (crtc_state->gamma_lut->length !=
-		    MGAG200_LUT_SIZE * sizeof(struct drm_color_lut)) {
-			drm_err(dev, "Wrong size for gamma_lut %zu\n",
-				crtc_state->gamma_lut->length);
-			return -EINVAL;
-		}
-	}
-	return 0;
+	if (funcs->enable_vidrst)
+		funcs->enable_vidrst(mdev);
 }
 
-static void
-mgag200_simple_display_pipe_update(struct drm_simple_display_pipe *pipe,
-				   struct drm_plane_state *old_state)
+void mgag200_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *old_state)
 {
-	struct drm_plane *plane = &pipe->plane;
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct drm_device *dev = plane->dev;
-	struct mga_device *mdev = to_mga_device(dev);
-	struct drm_plane_state *state = plane->state;
-	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(state);
-	struct drm_framebuffer *fb = state->fb;
-	struct drm_rect damage;
-	struct drm_atomic_helper_damage_iter iter;
+	struct mga_device *mdev = to_mga_device(crtc->dev);
+	const struct mgag200_device_funcs *funcs = mdev->funcs;
 
-	if (!fb)
-		return;
+	if (funcs->disable_vidrst)
+		funcs->disable_vidrst(mdev);
 
-	mutex_lock(&mdev->rmmio_lock);
+	mgag200_disable_display(mdev);
 
-	if (crtc->state->color_mgmt_changed && crtc->state->gamma_lut)
-		mgag200_crtc_set_gamma(mdev, fb->format, crtc->state->gamma_lut->data);
+	if (funcs->enable_vidrst)
+		funcs->enable_vidrst(mdev);
+}
 
-	drm_atomic_helper_damage_iter_init(&iter, old_state, state);
-	drm_atomic_for_each_plane_damage(&iter, &damage) {
-		mgag200_handle_damage(mdev, fb, &damage, &shadow_plane_state->data[0]);
-	}
-	/* Always scanout image at VRAM offset 0 */
-	mgag200_set_startadd(mdev, (u32)0);
-	mgag200_set_offset(mdev, fb);
+void mgag200_crtc_reset(struct drm_crtc *crtc)
+{
+	struct mgag200_crtc_state *mgag200_crtc_state;
 
-	mutex_unlock(&mdev->rmmio_lock);
+	if (crtc->state)
+		crtc->funcs->atomic_destroy_state(crtc, crtc->state);
+
+	mgag200_crtc_state = kzalloc(sizeof(*mgag200_crtc_state), GFP_KERNEL);
+	if (mgag200_crtc_state)
+		__drm_atomic_helper_crtc_reset(crtc, &mgag200_crtc_state->base);
+	else
+		__drm_atomic_helper_crtc_reset(crtc, NULL);
 }
 
-static struct drm_crtc_state *
-mgag200_simple_display_pipe_duplicate_crtc_state(struct drm_simple_display_pipe *pipe)
+struct drm_crtc_state *mgag200_crtc_atomic_duplicate_state(struct drm_crtc *crtc)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
 	struct drm_crtc_state *crtc_state = crtc->state;
 	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
 	struct mgag200_crtc_state *new_mgag200_crtc_state;
@@ -927,14 +688,14 @@ mgag200_simple_display_pipe_duplicate_crtc_state(struct drm_simple_display_pipe
 		return NULL;
 	__drm_atomic_helper_crtc_duplicate_state(crtc, &new_mgag200_crtc_state->base);
 
+	new_mgag200_crtc_state->format = mgag200_crtc_state->format;
 	memcpy(&new_mgag200_crtc_state->pixpllc, &mgag200_crtc_state->pixpllc,
 	       sizeof(new_mgag200_crtc_state->pixpllc));
 
 	return &new_mgag200_crtc_state->base;
 }
 
-static void mgag200_simple_display_pipe_destroy_crtc_state(struct drm_simple_display_pipe *pipe,
-							   struct drm_crtc_state *crtc_state)
+void mgag200_crtc_atomic_destroy_state(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state)
 {
 	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
 
@@ -942,50 +703,49 @@ static void mgag200_simple_display_pipe_destroy_crtc_state(struct drm_simple_dis
 	kfree(mgag200_crtc_state);
 }
 
-static void mgag200_simple_display_pipe_reset_crtc(struct drm_simple_display_pipe *pipe)
+/*
+ * Connector
+ */
+
+int mgag200_vga_connector_helper_get_modes(struct drm_connector *connector)
 {
-	struct drm_crtc *crtc = &pipe->crtc;
-	struct mgag200_crtc_state *mgag200_crtc_state;
+	struct mga_device *mdev = to_mga_device(connector->dev);
+	int ret;
 
-	if (crtc->state) {
-		mgag200_simple_display_pipe_destroy_crtc_state(pipe, crtc->state);
-		crtc->state = NULL; /* must be set to NULL here */
-	}
+	/*
+	 * Protect access to I/O registers from concurrent modesetting
+	 * by acquiring the I/O-register lock.
+	 */
+	mutex_lock(&mdev->rmmio_lock);
+	ret = drm_connector_helper_get_modes_from_ddc(connector);
+	mutex_unlock(&mdev->rmmio_lock);
 
-	mgag200_crtc_state = kzalloc(sizeof(*mgag200_crtc_state), GFP_KERNEL);
-	if (!mgag200_crtc_state)
-		return;
-	__drm_atomic_helper_crtc_reset(crtc, &mgag200_crtc_state->base);
+	return ret;
 }
 
-static const struct drm_simple_display_pipe_funcs
-mgag200_simple_display_pipe_funcs = {
-	.mode_valid = mgag200_simple_display_pipe_mode_valid,
-	.enable	    = mgag200_simple_display_pipe_enable,
-	.disable    = mgag200_simple_display_pipe_disable,
-	.check	    = mgag200_simple_display_pipe_check,
-	.update	    = mgag200_simple_display_pipe_update,
-	.reset_crtc = mgag200_simple_display_pipe_reset_crtc,
-	.duplicate_crtc_state = mgag200_simple_display_pipe_duplicate_crtc_state,
-	.destroy_crtc_state = mgag200_simple_display_pipe_destroy_crtc_state,
-	DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS,
-};
-
-static const uint32_t mgag200_simple_display_pipe_formats[] = {
-	DRM_FORMAT_XRGB8888,
-	DRM_FORMAT_RGB565,
-	DRM_FORMAT_RGB888,
-};
-
-static const uint64_t mgag200_simple_display_pipe_fmtmods[] = {
-	DRM_FORMAT_MOD_LINEAR,
-	DRM_FORMAT_MOD_INVALID
-};
-
 /*
  * Mode config
  */
 
+static void mgag200_mode_config_helper_atomic_commit_tail(struct drm_atomic_state *state)
+{
+	struct mga_device *mdev = to_mga_device(state->dev);
+
+	/*
+	 * Concurrent operations could possibly trigger a call to
+	 * drm_connector_helper_funcs.get_modes by trying to read the
+	 * display modes. Protect access to I/O registers by acquiring
+	 * the I/O-register lock.
+	 */
+	mutex_lock(&mdev->rmmio_lock);
+	drm_atomic_helper_commit_tail(state);
+	mutex_unlock(&mdev->rmmio_lock);
+}
+
+static const struct drm_mode_config_helper_funcs mgag200_mode_config_helper_funcs = {
+	.atomic_commit_tail = mgag200_mode_config_helper_atomic_commit_tail,
+};
+
 /* Calculates a mode's required memory bandwidth (in KiB/sec). */
 static uint32_t mgag200_calculate_mode_bandwidth(const struct drm_display_mode *mode,
 						 unsigned int bits_per_pixel)
@@ -1048,23 +808,16 @@ static const struct drm_mode_config_funcs mgag200_mode_config_funcs = {
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-int mgag200_modeset_init(struct mga_device *mdev, resource_size_t vram_available)
+int mgag200_mode_config_init(struct mga_device *mdev, resource_size_t vram_available)
 {
 	struct drm_device *dev = &mdev->base;
-	struct mga_i2c_chan *i2c = &mdev->i2c;
-	struct drm_connector *connector = &mdev->connector;
-	struct drm_simple_display_pipe *pipe = &mdev->display_pipe;
-	size_t format_count = ARRAY_SIZE(mgag200_simple_display_pipe_formats);
 	int ret;
 
-	mgag200_init_regs(mdev);
-
 	mdev->vram_available = vram_available;
 
 	ret = drmm_mode_config_init(dev);
 	if (ret) {
-		drm_err(dev, "drmm_mode_config_init() failed, error %d\n",
-			ret);
+		drm_err(dev, "drmm_mode_config_init() failed: %d\n", ret);
 		return ret;
 	}
 
@@ -1073,48 +826,7 @@ int mgag200_modeset_init(struct mga_device *mdev, resource_size_t vram_available
 	dev->mode_config.preferred_depth = 24;
 	dev->mode_config.fb_base = mdev->vram_res->start;
 	dev->mode_config.funcs = &mgag200_mode_config_funcs;
-
-	ret = mgag200_i2c_init(mdev, i2c);
-	if (ret) {
-		drm_err(dev, "failed to add DDC bus: %d\n", ret);
-		return ret;
-	}
-
-	ret = drm_connector_init_with_ddc(dev, connector,
-					  &mga_vga_connector_funcs,
-					  DRM_MODE_CONNECTOR_VGA,
-					  &i2c->adapter);
-	if (ret) {
-		drm_err(dev, "drm_connector_init_with_ddc() failed: %d\n", ret);
-		return ret;
-	}
-	drm_connector_helper_add(connector, &mga_vga_connector_helper_funcs);
-
-	ret = mgag200_pixpll_init(&mdev->pixpll, mdev);
-	if (ret)
-		return ret;
-
-	ret = drm_simple_display_pipe_init(dev, pipe,
-					   &mgag200_simple_display_pipe_funcs,
-					   mgag200_simple_display_pipe_formats,
-					   format_count,
-					   mgag200_simple_display_pipe_fmtmods,
-					   connector);
-	if (ret) {
-		drm_err(dev,
-			"drm_simple_display_pipe_init() failed, error %d\n",
-			ret);
-		return ret;
-	}
-
-	drm_plane_enable_fb_damage_clips(&pipe->plane);
-
-	/* FIXME: legacy gamma tables, but atomic gamma doesn't work without */
-	drm_mode_crtc_set_gamma_size(&pipe->crtc, MGAG200_LUT_SIZE);
-
-	drm_crtc_enable_color_mgmt(&pipe->crtc, 0, false, MGAG200_LUT_SIZE);
-
-	drm_mode_config_reset(dev);
+	dev->mode_config.helper_private = &mgag200_mode_config_helper_funcs;
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_pll.c b/drivers/gpu/drm/mgag200/mgag200_pll.c
deleted file mode 100644
index 8065ca5d8de9..000000000000
--- a/drivers/gpu/drm/mgag200/mgag200_pll.c
+++ /dev/null
@@ -1,997 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-
-#include <linux/delay.h>
-
-#include "mgag200_drv.h"
-
-/*
- * G200
- */
-
-static int mgag200_pixpll_compute_g200(struct mgag200_pll *pixpll, long clock,
-				       struct mgag200_pll_values *pixpllc)
-{
-	struct mga_device *mdev = pixpll->mdev;
-	struct drm_device *dev = &mdev->base;
-	struct mgag200_g200_device *g200 = to_mgag200_g200_device(dev);
-	const int post_div_max = 7;
-	const int in_div_min = 1;
-	const int in_div_max = 6;
-	const int feed_div_min = 7;
-	const int feed_div_max = 127;
-	u8 testp, testm, testn;
-	u8 n = 0, m = 0, p, s;
-	long f_vco;
-	long computed;
-	long delta, tmp_delta;
-	long ref_clk = g200->ref_clk;
-	long p_clk_min = g200->pclk_min;
-	long p_clk_max = g200->pclk_max;
-
-	if (clock > p_clk_max) {
-		drm_err(dev, "Pixel Clock %ld too high\n", clock);
-		return -EINVAL;
-	}
-
-	if (clock < p_clk_min >> 3)
-		clock = p_clk_min >> 3;
-
-	f_vco = clock;
-	for (testp = 0;
-	     testp <= post_div_max && f_vco < p_clk_min;
-	     testp = (testp << 1) + 1, f_vco <<= 1)
-		;
-	p = testp + 1;
-
-	delta = clock;
-
-	for (testm = in_div_min; testm <= in_div_max; testm++) {
-		for (testn = feed_div_min; testn <= feed_div_max; testn++) {
-			computed = ref_clk * (testn + 1) / (testm + 1);
-			if (computed < f_vco)
-				tmp_delta = f_vco - computed;
-			else
-				tmp_delta = computed - f_vco;
-			if (tmp_delta < delta) {
-				delta = tmp_delta;
-				m = testm + 1;
-				n = testn + 1;
-			}
-		}
-	}
-	f_vco = ref_clk * n / m;
-	if (f_vco < 100000)
-		s = 0;
-	else if (f_vco < 140000)
-		s = 1;
-	else if (f_vco < 180000)
-		s = 2;
-	else
-		s = 3;
-
-	drm_dbg_kms(dev, "clock: %ld vco: %ld m: %d n: %d p: %d s: %d\n",
-		    clock, f_vco, m, n, p, s);
-
-	pixpllc->m = m;
-	pixpllc->n = n;
-	pixpllc->p = p;
-	pixpllc->s = s;
-
-	return 0;
-}
-
-static void
-mgag200_pixpll_update_g200(struct mgag200_pll *pixpll, const struct mgag200_pll_values *pixpllc)
-{
-	struct mga_device *mdev = pixpll->mdev;
-	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
-	u8 xpixpllcm, xpixpllcn, xpixpllcp;
-
-	pixpllcm = pixpllc->m - 1;
-	pixpllcn = pixpllc->n - 1;
-	pixpllcp = pixpllc->p - 1;
-	pixpllcs = pixpllc->s;
-
-	xpixpllcm = pixpllcm;
-	xpixpllcn = pixpllcn;
-	xpixpllcp = (pixpllcs << 3) | pixpllcp;
-
-	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
-
-	WREG_DAC(MGA1064_PIX_PLLC_M, xpixpllcm);
-	WREG_DAC(MGA1064_PIX_PLLC_N, xpixpllcn);
-	WREG_DAC(MGA1064_PIX_PLLC_P, xpixpllcp);
-}
-
-static const struct mgag200_pll_funcs mgag200_pixpll_funcs_g200 = {
-	.compute = mgag200_pixpll_compute_g200,
-	.update = mgag200_pixpll_update_g200,
-};
-
-/*
- * G200SE
- */
-
-static int mgag200_pixpll_compute_g200se_00(struct mgag200_pll *pixpll, long clock,
-					    struct mgag200_pll_values *pixpllc)
-{
-	static const unsigned int vcomax = 320000;
-	static const unsigned int vcomin = 160000;
-	static const unsigned int pllreffreq = 25000;
-
-	unsigned int delta, tmpdelta, permitteddelta;
-	unsigned int testp, testm, testn;
-	unsigned int p, m, n, s;
-	unsigned int computed;
-
-	m = n = p = s = 0;
-	delta = 0xffffffff;
-	permitteddelta = clock * 5 / 1000;
-
-	for (testp = 8; testp > 0; testp /= 2) {
-		if (clock * testp > vcomax)
-			continue;
-		if (clock * testp < vcomin)
-			continue;
-
-		for (testn = 17; testn < 256; testn++) {
-			for (testm = 1; testm < 32; testm++) {
-				computed = (pllreffreq * testn) / (testm * testp);
-				if (computed > clock)
-					tmpdelta = computed - clock;
-				else
-					tmpdelta = clock - computed;
-				if (tmpdelta < delta) {
-					delta = tmpdelta;
-					m = testm;
-					n = testn;
-					p = testp;
-				}
-			}
-		}
-	}
-
-	if (delta > permitteddelta) {
-		pr_warn("PLL delta too large\n");
-		return -EINVAL;
-	}
-
-	pixpllc->m = m;
-	pixpllc->n = n;
-	pixpllc->p = p;
-	pixpllc->s = s;
-
-	return 0;
-}
-
-static void mgag200_pixpll_update_g200se_00(struct mgag200_pll *pixpll,
-					    const struct mgag200_pll_values *pixpllc)
-{
-	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
-	u8 xpixpllcm, xpixpllcn, xpixpllcp;
-	struct mga_device *mdev = pixpll->mdev;
-
-	pixpllcm = pixpllc->m - 1;
-	pixpllcn = pixpllc->n - 1;
-	pixpllcp = pixpllc->p - 1;
-	pixpllcs = pixpllc->s;
-
-	xpixpllcm = pixpllcm | ((pixpllcn & BIT(8)) >> 1);
-	xpixpllcn = pixpllcn;
-	xpixpllcp = (pixpllcs << 3) | pixpllcp;
-
-	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
-
-	WREG_DAC(MGA1064_PIX_PLLC_M, xpixpllcm);
-	WREG_DAC(MGA1064_PIX_PLLC_N, xpixpllcn);
-	WREG_DAC(MGA1064_PIX_PLLC_P, xpixpllcp);
-}
-
-static int mgag200_pixpll_compute_g200se_04(struct mgag200_pll *pixpll, long clock,
-					    struct mgag200_pll_values *pixpllc)
-{
-	static const unsigned int vcomax = 1600000;
-	static const unsigned int vcomin = 800000;
-	static const unsigned int pllreffreq = 25000;
-	static const unsigned int pvalues_e4[] = {16, 14, 12, 10, 8, 6, 4, 2, 1};
-
-	unsigned int delta, tmpdelta, permitteddelta;
-	unsigned int testp, testm, testn;
-	unsigned int p, m, n, s;
-	unsigned int computed;
-	unsigned int fvv;
-	unsigned int i;
-
-	m = n = p = s = 0;
-	delta = 0xffffffff;
-
-	if (clock < 25000)
-		clock = 25000;
-	clock = clock * 2;
-
-	/* Permited delta is 0.5% as VESA Specification */
-	permitteddelta = clock * 5 / 1000;
-
-	for (i = 0 ; i < ARRAY_SIZE(pvalues_e4); i++) {
-		testp = pvalues_e4[i];
-
-		if ((clock * testp) > vcomax)
-			continue;
-		if ((clock * testp) < vcomin)
-			continue;
-
-		for (testn = 50; testn <= 256; testn++) {
-			for (testm = 1; testm <= 32; testm++) {
-				computed = (pllreffreq * testn) / (testm * testp);
-				if (computed > clock)
-					tmpdelta = computed - clock;
-				else
-					tmpdelta = clock - computed;
-
-				if (tmpdelta < delta) {
-					delta = tmpdelta;
-					m = testm;
-					n = testn;
-					p = testp;
-				}
-			}
-		}
-	}
-
-	fvv = pllreffreq * n / m;
-	fvv = (fvv - 800000) / 50000;
-	if (fvv > 15)
-		fvv = 15;
-	s = fvv << 1;
-
-	if (delta > permitteddelta) {
-		pr_warn("PLL delta too large\n");
-		return -EINVAL;
-	}
-
-	pixpllc->m = m;
-	pixpllc->n = n;
-	pixpllc->p = p;
-	pixpllc->s = s;
-
-	return 0;
-}
-
-static void mgag200_pixpll_update_g200se_04(struct mgag200_pll *pixpll,
-					    const struct mgag200_pll_values *pixpllc)
-{
-	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
-	u8 xpixpllcm, xpixpllcn, xpixpllcp;
-	struct mga_device *mdev = pixpll->mdev;
-
-	pixpllcm = pixpllc->m - 1;
-	pixpllcn = pixpllc->n - 1;
-	pixpllcp = pixpllc->p - 1;
-	pixpllcs = pixpllc->s;
-
-	xpixpllcm = pixpllcm | ((pixpllcn & BIT(8)) >> 1);
-	xpixpllcn = pixpllcn;
-	xpixpllcp = (pixpllcs << 3) | pixpllcp;
-
-	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
-
-	WREG_DAC(MGA1064_PIX_PLLC_M, xpixpllcm);
-	WREG_DAC(MGA1064_PIX_PLLC_N, xpixpllcn);
-	WREG_DAC(MGA1064_PIX_PLLC_P, xpixpllcp);
-
-	WREG_DAC(0x1a, 0x09);
-	msleep(20);
-	WREG_DAC(0x1a, 0x01);
-}
-
-static const struct mgag200_pll_funcs mgag200_pixpll_funcs_g200se_00 = {
-	.compute = mgag200_pixpll_compute_g200se_00,
-	.update = mgag200_pixpll_update_g200se_00,
-};
-
-static const struct mgag200_pll_funcs mgag200_pixpll_funcs_g200se_04 = {
-	.compute = mgag200_pixpll_compute_g200se_04,
-	.update = mgag200_pixpll_update_g200se_04,
-};
-
-/*
- * G200WB
- */
-
-static int mgag200_pixpll_compute_g200wb(struct mgag200_pll *pixpll, long clock,
-					 struct mgag200_pll_values *pixpllc)
-{
-	static const unsigned int vcomax = 550000;
-	static const unsigned int vcomin = 150000;
-	static const unsigned int pllreffreq = 48000;
-
-	unsigned int delta, tmpdelta;
-	unsigned int testp, testm, testn;
-	unsigned int p, m, n, s;
-	unsigned int computed;
-
-	m = n = p = s = 0;
-	delta = 0xffffffff;
-
-	for (testp = 1; testp < 9; testp++) {
-		if (clock * testp > vcomax)
-			continue;
-		if (clock * testp < vcomin)
-			continue;
-
-		for (testm = 1; testm < 17; testm++) {
-			for (testn = 1; testn < 151; testn++) {
-				computed = (pllreffreq * testn) / (testm * testp);
-				if (computed > clock)
-					tmpdelta = computed - clock;
-				else
-					tmpdelta = clock - computed;
-				if (tmpdelta < delta) {
-					delta = tmpdelta;
-					n = testn;
-					m = testm;
-					p = testp;
-					s = 0;
-				}
-			}
-		}
-	}
-
-	pixpllc->m = m;
-	pixpllc->n = n;
-	pixpllc->p = p;
-	pixpllc->s = s;
-
-	return 0;
-}
-
-static void
-mgag200_pixpll_update_g200wb(struct mgag200_pll *pixpll, const struct mgag200_pll_values *pixpllc)
-{
-	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
-	u8 xpixpllcm, xpixpllcn, xpixpllcp, tmp;
-	int i, j, tmpcount, vcount;
-	struct mga_device *mdev = pixpll->mdev;
-	bool pll_locked = false;
-
-	pixpllcm = pixpllc->m - 1;
-	pixpllcn = pixpllc->n - 1;
-	pixpllcp = pixpllc->p - 1;
-	pixpllcs = pixpllc->s;
-
-	xpixpllcm = ((pixpllcn & BIT(8)) >> 1) | pixpllcm;
-	xpixpllcn = pixpllcn;
-	xpixpllcp = ((pixpllcn & GENMASK(10, 9)) >> 3) | (pixpllcs << 3) | pixpllcp;
-
-	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
-
-	for (i = 0; i <= 32 && pll_locked == false; i++) {
-		if (i > 0) {
-			WREG8(MGAREG_CRTC_INDEX, 0x1e);
-			tmp = RREG8(MGAREG_CRTC_DATA);
-			if (tmp < 0xff)
-				WREG8(MGAREG_CRTC_DATA, tmp+1);
-		}
-
-		/* set pixclkdis to 1 */
-		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-		tmp = RREG8(DAC_DATA);
-		tmp |= MGA1064_PIX_CLK_CTL_CLK_DIS;
-		WREG8(DAC_DATA, tmp);
-
-		WREG8(DAC_INDEX, MGA1064_REMHEADCTL);
-		tmp = RREG8(DAC_DATA);
-		tmp |= MGA1064_REMHEADCTL_CLKDIS;
-		WREG8(DAC_DATA, tmp);
-
-		/* select PLL Set C */
-		tmp = RREG8(MGAREG_MEM_MISC_READ);
-		tmp |= 0x3 << 2;
-		WREG8(MGAREG_MEM_MISC_WRITE, tmp);
-
-		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-		tmp = RREG8(DAC_DATA);
-		tmp |= MGA1064_PIX_CLK_CTL_CLK_POW_DOWN | 0x80;
-		WREG8(DAC_DATA, tmp);
-
-		udelay(500);
-
-		/* reset the PLL */
-		WREG8(DAC_INDEX, MGA1064_VREF_CTL);
-		tmp = RREG8(DAC_DATA);
-		tmp &= ~0x04;
-		WREG8(DAC_DATA, tmp);
-
-		udelay(50);
-
-		/* program pixel pll register */
-		WREG_DAC(MGA1064_WB_PIX_PLLC_N, xpixpllcn);
-		WREG_DAC(MGA1064_WB_PIX_PLLC_M, xpixpllcm);
-		WREG_DAC(MGA1064_WB_PIX_PLLC_P, xpixpllcp);
-
-		udelay(50);
-
-		/* turn pll on */
-		WREG8(DAC_INDEX, MGA1064_VREF_CTL);
-		tmp = RREG8(DAC_DATA);
-		tmp |= 0x04;
-		WREG_DAC(MGA1064_VREF_CTL, tmp);
-
-		udelay(500);
-
-		/* select the pixel pll */
-		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-		tmp = RREG8(DAC_DATA);
-		tmp &= ~MGA1064_PIX_CLK_CTL_SEL_MSK;
-		tmp |= MGA1064_PIX_CLK_CTL_SEL_PLL;
-		WREG8(DAC_DATA, tmp);
-
-		WREG8(DAC_INDEX, MGA1064_REMHEADCTL);
-		tmp = RREG8(DAC_DATA);
-		tmp &= ~MGA1064_REMHEADCTL_CLKSL_MSK;
-		tmp |= MGA1064_REMHEADCTL_CLKSL_PLL;
-		WREG8(DAC_DATA, tmp);
-
-		/* reset dotclock rate bit */
-		WREG8(MGAREG_SEQ_INDEX, 1);
-		tmp = RREG8(MGAREG_SEQ_DATA);
-		tmp &= ~0x8;
-		WREG8(MGAREG_SEQ_DATA, tmp);
-
-		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-		tmp = RREG8(DAC_DATA);
-		tmp &= ~MGA1064_PIX_CLK_CTL_CLK_DIS;
-		WREG8(DAC_DATA, tmp);
-
-		vcount = RREG8(MGAREG_VCOUNT);
-
-		for (j = 0; j < 30 && pll_locked == false; j++) {
-			tmpcount = RREG8(MGAREG_VCOUNT);
-			if (tmpcount < vcount)
-				vcount = 0;
-			if ((tmpcount - vcount) > 2)
-				pll_locked = true;
-			else
-				udelay(5);
-		}
-	}
-
-	WREG8(DAC_INDEX, MGA1064_REMHEADCTL);
-	tmp = RREG8(DAC_DATA);
-	tmp &= ~MGA1064_REMHEADCTL_CLKDIS;
-	WREG_DAC(MGA1064_REMHEADCTL, tmp);
-}
-
-static const struct mgag200_pll_funcs mgag200_pixpll_funcs_g200wb = {
-	.compute = mgag200_pixpll_compute_g200wb,
-	.update = mgag200_pixpll_update_g200wb,
-};
-
-/*
- * G200EV
- */
-
-static int mgag200_pixpll_compute_g200ev(struct mgag200_pll *pixpll, long clock,
-					 struct mgag200_pll_values *pixpllc)
-{
-	static const unsigned int vcomax = 550000;
-	static const unsigned int vcomin = 150000;
-	static const unsigned int pllreffreq = 50000;
-
-	unsigned int delta, tmpdelta;
-	unsigned int testp, testm, testn;
-	unsigned int p, m, n, s;
-	unsigned int computed;
-
-	m = n = p = s = 0;
-	delta = 0xffffffff;
-
-	for (testp = 16; testp > 0; testp--) {
-		if (clock * testp > vcomax)
-			continue;
-		if (clock * testp < vcomin)
-			continue;
-
-		for (testn = 1; testn < 257; testn++) {
-			for (testm = 1; testm < 17; testm++) {
-				computed = (pllreffreq * testn) /
-					(testm * testp);
-				if (computed > clock)
-					tmpdelta = computed - clock;
-				else
-					tmpdelta = clock - computed;
-				if (tmpdelta < delta) {
-					delta = tmpdelta;
-					n = testn;
-					m = testm;
-					p = testp;
-				}
-			}
-		}
-	}
-
-	pixpllc->m = m;
-	pixpllc->n = n;
-	pixpllc->p = p;
-	pixpllc->s = s;
-
-	return 0;
-}
-
-static void
-mgag200_pixpll_update_g200ev(struct mgag200_pll *pixpll, const struct mgag200_pll_values *pixpllc)
-{
-	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
-	u8 xpixpllcm, xpixpllcn, xpixpllcp, tmp;
-	struct mga_device *mdev = pixpll->mdev;
-
-	pixpllcm = pixpllc->m - 1;
-	pixpllcn = pixpllc->n - 1;
-	pixpllcp = pixpllc->p - 1;
-	pixpllcs = pixpllc->s;
-
-	xpixpllcm = pixpllcm;
-	xpixpllcn = pixpllcn;
-	xpixpllcp = (pixpllcs << 3) | pixpllcp;
-
-	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
-
-	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-	tmp = RREG8(DAC_DATA);
-	tmp |= MGA1064_PIX_CLK_CTL_CLK_DIS;
-	WREG8(DAC_DATA, tmp);
-
-	tmp = RREG8(MGAREG_MEM_MISC_READ);
-	tmp |= 0x3 << 2;
-	WREG8(MGAREG_MEM_MISC_WRITE, tmp);
-
-	WREG8(DAC_INDEX, MGA1064_PIX_PLL_STAT);
-	tmp = RREG8(DAC_DATA);
-	WREG8(DAC_DATA, tmp & ~0x40);
-
-	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-	tmp = RREG8(DAC_DATA);
-	tmp |= MGA1064_PIX_CLK_CTL_CLK_POW_DOWN;
-	WREG8(DAC_DATA, tmp);
-
-	WREG_DAC(MGA1064_EV_PIX_PLLC_M, xpixpllcm);
-	WREG_DAC(MGA1064_EV_PIX_PLLC_N, xpixpllcn);
-	WREG_DAC(MGA1064_EV_PIX_PLLC_P, xpixpllcp);
-
-	udelay(50);
-
-	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-	tmp = RREG8(DAC_DATA);
-	tmp &= ~MGA1064_PIX_CLK_CTL_CLK_POW_DOWN;
-	WREG8(DAC_DATA, tmp);
-
-	udelay(500);
-
-	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-	tmp = RREG8(DAC_DATA);
-	tmp &= ~MGA1064_PIX_CLK_CTL_SEL_MSK;
-	tmp |= MGA1064_PIX_CLK_CTL_SEL_PLL;
-	WREG8(DAC_DATA, tmp);
-
-	WREG8(DAC_INDEX, MGA1064_PIX_PLL_STAT);
-	tmp = RREG8(DAC_DATA);
-	WREG8(DAC_DATA, tmp | 0x40);
-
-	tmp = RREG8(MGAREG_MEM_MISC_READ);
-	tmp |= (0x3 << 2);
-	WREG8(MGAREG_MEM_MISC_WRITE, tmp);
-
-	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-	tmp = RREG8(DAC_DATA);
-	tmp &= ~MGA1064_PIX_CLK_CTL_CLK_DIS;
-	WREG8(DAC_DATA, tmp);
-}
-
-static const struct mgag200_pll_funcs mgag200_pixpll_funcs_g200ev = {
-	.compute = mgag200_pixpll_compute_g200ev,
-	.update = mgag200_pixpll_update_g200ev,
-};
-
-/*
- * G200EH
- */
-
-static int mgag200_pixpll_compute_g200eh(struct mgag200_pll *pixpll, long clock,
-					 struct mgag200_pll_values *pixpllc)
-{
-	static const unsigned int vcomax = 800000;
-	static const unsigned int vcomin = 400000;
-	static const unsigned int pllreffreq = 33333;
-
-	unsigned int delta, tmpdelta;
-	unsigned int testp, testm, testn;
-	unsigned int p, m, n, s;
-	unsigned int computed;
-
-	m = n = p = s = 0;
-	delta = 0xffffffff;
-
-	for (testp = 16; testp > 0; testp >>= 1) {
-		if (clock * testp > vcomax)
-			continue;
-		if (clock * testp < vcomin)
-			continue;
-
-		for (testm = 1; testm < 33; testm++) {
-			for (testn = 17; testn < 257; testn++) {
-				computed = (pllreffreq * testn) / (testm * testp);
-				if (computed > clock)
-					tmpdelta = computed - clock;
-				else
-					tmpdelta = clock - computed;
-				if (tmpdelta < delta) {
-					delta = tmpdelta;
-					n = testn;
-					m = testm;
-					p = testp;
-				}
-			}
-		}
-	}
-
-	pixpllc->m = m;
-	pixpllc->n = n;
-	pixpllc->p = p;
-	pixpllc->s = s;
-
-	return 0;
-}
-
-static void
-mgag200_pixpll_update_g200eh(struct mgag200_pll *pixpll, const struct mgag200_pll_values *pixpllc)
-{
-	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
-	u8 xpixpllcm, xpixpllcn, xpixpllcp, tmp;
-	int i, j, tmpcount, vcount;
-	struct mga_device *mdev = pixpll->mdev;
-	bool pll_locked = false;
-
-	pixpllcm = pixpllc->m - 1;
-	pixpllcn = pixpllc->n - 1;
-	pixpllcp = pixpllc->p - 1;
-	pixpllcs = pixpllc->s;
-
-	xpixpllcm = ((pixpllcn & BIT(8)) >> 1) | pixpllcm;
-	xpixpllcn = pixpllcn;
-	xpixpllcp = (pixpllcs << 3) | pixpllcp;
-
-	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
-
-	for (i = 0; i <= 32 && pll_locked == false; i++) {
-		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-		tmp = RREG8(DAC_DATA);
-		tmp |= MGA1064_PIX_CLK_CTL_CLK_DIS;
-		WREG8(DAC_DATA, tmp);
-
-		tmp = RREG8(MGAREG_MEM_MISC_READ);
-		tmp |= 0x3 << 2;
-		WREG8(MGAREG_MEM_MISC_WRITE, tmp);
-
-		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-		tmp = RREG8(DAC_DATA);
-		tmp |= MGA1064_PIX_CLK_CTL_CLK_POW_DOWN;
-		WREG8(DAC_DATA, tmp);
-
-		udelay(500);
-
-		WREG_DAC(MGA1064_EH_PIX_PLLC_M, xpixpllcm);
-		WREG_DAC(MGA1064_EH_PIX_PLLC_N, xpixpllcn);
-		WREG_DAC(MGA1064_EH_PIX_PLLC_P, xpixpllcp);
-
-		udelay(500);
-
-		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-		tmp = RREG8(DAC_DATA);
-		tmp &= ~MGA1064_PIX_CLK_CTL_SEL_MSK;
-		tmp |= MGA1064_PIX_CLK_CTL_SEL_PLL;
-		WREG8(DAC_DATA, tmp);
-
-		WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-		tmp = RREG8(DAC_DATA);
-		tmp &= ~MGA1064_PIX_CLK_CTL_CLK_DIS;
-		tmp &= ~MGA1064_PIX_CLK_CTL_CLK_POW_DOWN;
-		WREG8(DAC_DATA, tmp);
-
-		vcount = RREG8(MGAREG_VCOUNT);
-
-		for (j = 0; j < 30 && pll_locked == false; j++) {
-			tmpcount = RREG8(MGAREG_VCOUNT);
-			if (tmpcount < vcount)
-				vcount = 0;
-			if ((tmpcount - vcount) > 2)
-				pll_locked = true;
-			else
-				udelay(5);
-		}
-	}
-}
-
-static const struct mgag200_pll_funcs mgag200_pixpll_funcs_g200eh = {
-	.compute = mgag200_pixpll_compute_g200eh,
-	.update = mgag200_pixpll_update_g200eh,
-};
-
-/*
- * G200EH3
- */
-
-static int mgag200_pixpll_compute_g200eh3(struct mgag200_pll *pixpll, long clock,
-					  struct mgag200_pll_values *pixpllc)
-{
-	static const unsigned int vcomax = 3000000;
-	static const unsigned int vcomin = 1500000;
-	static const unsigned int pllreffreq = 25000;
-
-	unsigned int delta, tmpdelta;
-	unsigned int testp, testm, testn;
-	unsigned int p, m, n, s;
-	unsigned int computed;
-
-	m = n = p = s = 0;
-	delta = 0xffffffff;
-	testp = 0;
-
-	for (testm = 150; testm >= 6; testm--) {
-		if (clock * testm > vcomax)
-			continue;
-		if (clock * testm < vcomin)
-			continue;
-		for (testn = 120; testn >= 60; testn--) {
-			computed = (pllreffreq * testn) / testm;
-			if (computed > clock)
-				tmpdelta = computed - clock;
-			else
-				tmpdelta = clock - computed;
-			if (tmpdelta < delta) {
-				delta = tmpdelta;
-				n = testn + 1;
-				m = testm + 1;
-				p = testp + 1;
-			}
-			if (delta == 0)
-				break;
-		}
-		if (delta == 0)
-			break;
-	}
-
-	pixpllc->m = m;
-	pixpllc->n = n;
-	pixpllc->p = p;
-	pixpllc->s = s;
-
-	return 0;
-}
-
-static const struct mgag200_pll_funcs mgag200_pixpll_funcs_g200eh3 = {
-	.compute = mgag200_pixpll_compute_g200eh3,
-	.update = mgag200_pixpll_update_g200eh, // same as G200EH
-};
-
-/*
- * G200ER
- */
-
-static int mgag200_pixpll_compute_g200er(struct mgag200_pll *pixpll, long clock,
-					 struct mgag200_pll_values *pixpllc)
-{
-	static const unsigned int vcomax = 1488000;
-	static const unsigned int vcomin = 1056000;
-	static const unsigned int pllreffreq = 48000;
-	static const unsigned int m_div_val[] = { 1, 2, 4, 8 };
-
-	unsigned int delta, tmpdelta;
-	int testr, testn, testm, testo;
-	unsigned int p, m, n, s;
-	unsigned int computed, vco;
-
-	m = n = p = s = 0;
-	delta = 0xffffffff;
-
-	for (testr = 0; testr < 4; testr++) {
-		if (delta == 0)
-			break;
-		for (testn = 5; testn < 129; testn++) {
-			if (delta == 0)
-				break;
-			for (testm = 3; testm >= 0; testm--) {
-				if (delta == 0)
-					break;
-				for (testo = 5; testo < 33; testo++) {
-					vco = pllreffreq * (testn + 1) /
-						(testr + 1);
-					if (vco < vcomin)
-						continue;
-					if (vco > vcomax)
-						continue;
-					computed = vco / (m_div_val[testm] * (testo + 1));
-					if (computed > clock)
-						tmpdelta = computed - clock;
-					else
-						tmpdelta = clock - computed;
-					if (tmpdelta < delta) {
-						delta = tmpdelta;
-						m = (testm | (testo << 3)) + 1;
-						n = testn + 1;
-						p = testr + 1;
-						s = testr;
-					}
-				}
-			}
-		}
-	}
-
-	pixpllc->m = m;
-	pixpllc->n = n;
-	pixpllc->p = p;
-	pixpllc->s = s;
-
-	return 0;
-}
-
-static void
-mgag200_pixpll_update_g200er(struct mgag200_pll *pixpll, const struct mgag200_pll_values *pixpllc)
-{
-	unsigned int pixpllcm, pixpllcn, pixpllcp, pixpllcs;
-	u8 xpixpllcm, xpixpllcn, xpixpllcp, tmp;
-	struct mga_device *mdev = pixpll->mdev;
-
-	pixpllcm = pixpllc->m - 1;
-	pixpllcn = pixpllc->n - 1;
-	pixpllcp = pixpllc->p - 1;
-	pixpllcs = pixpllc->s;
-
-	xpixpllcm = pixpllcm;
-	xpixpllcn = pixpllcn;
-	xpixpllcp = (pixpllcs << 3) | pixpllcp;
-
-	WREG_MISC_MASKED(MGAREG_MISC_CLKSEL_MGA, MGAREG_MISC_CLKSEL_MASK);
-
-	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-	tmp = RREG8(DAC_DATA);
-	tmp |= MGA1064_PIX_CLK_CTL_CLK_DIS;
-	WREG8(DAC_DATA, tmp);
-
-	WREG8(DAC_INDEX, MGA1064_REMHEADCTL);
-	tmp = RREG8(DAC_DATA);
-	tmp |= MGA1064_REMHEADCTL_CLKDIS;
-	WREG8(DAC_DATA, tmp);
-
-	tmp = RREG8(MGAREG_MEM_MISC_READ);
-	tmp |= (0x3<<2) | 0xc0;
-	WREG8(MGAREG_MEM_MISC_WRITE, tmp);
-
-	WREG8(DAC_INDEX, MGA1064_PIX_CLK_CTL);
-	tmp = RREG8(DAC_DATA);
-	tmp &= ~MGA1064_PIX_CLK_CTL_CLK_DIS;
-	tmp |= MGA1064_PIX_CLK_CTL_CLK_POW_DOWN;
-	WREG8(DAC_DATA, tmp);
-
-	udelay(500);
-
-	WREG_DAC(MGA1064_ER_PIX_PLLC_N, xpixpllcn);
-	WREG_DAC(MGA1064_ER_PIX_PLLC_M, xpixpllcm);
-	WREG_DAC(MGA1064_ER_PIX_PLLC_P, xpixpllcp);
-
-	udelay(50);
-}
-
-static const struct mgag200_pll_funcs mgag200_pixpll_funcs_g200er = {
-	.compute = mgag200_pixpll_compute_g200er,
-	.update = mgag200_pixpll_update_g200er,
-};
-
-/*
- * G200EW3
- */
-
-static int mgag200_pixpll_compute_g200ew3(struct mgag200_pll *pixpll, long clock,
-					  struct mgag200_pll_values *pixpllc)
-{
-	static const unsigned int vcomax = 800000;
-	static const unsigned int vcomin = 400000;
-	static const unsigned int pllreffreq = 25000;
-
-	unsigned int delta, tmpdelta;
-	unsigned int testp, testm, testn, testp2;
-	unsigned int p, m, n, s;
-	unsigned int computed;
-
-	m = n = p = s = 0;
-	delta = 0xffffffff;
-
-	for (testp = 1; testp < 8; testp++) {
-		for (testp2 = 1; testp2 < 8; testp2++) {
-			if (testp < testp2)
-				continue;
-			if ((clock * testp * testp2) > vcomax)
-				continue;
-			if ((clock * testp * testp2) < vcomin)
-				continue;
-			for (testm = 1; testm < 26; testm++) {
-				for (testn = 32; testn < 2048 ; testn++) {
-					computed = (pllreffreq * testn) / (testm * testp * testp2);
-					if (computed > clock)
-						tmpdelta = computed - clock;
-					else
-						tmpdelta = clock - computed;
-					if (tmpdelta < delta) {
-						delta = tmpdelta;
-						m = testm + 1;
-						n = testn + 1;
-						p = testp + 1;
-						s = testp2;
-					}
-				}
-			}
-		}
-	}
-
-	pixpllc->m = m;
-	pixpllc->n = n;
-	pixpllc->p = p;
-	pixpllc->s = s;
-
-	return 0;
-}
-
-static const struct mgag200_pll_funcs mgag200_pixpll_funcs_g200ew3 = {
-	.compute = mgag200_pixpll_compute_g200ew3,
-	.update = mgag200_pixpll_update_g200wb, // same as G200WB
-};
-
-/*
- * PLL initialization
- */
-
-int mgag200_pixpll_init(struct mgag200_pll *pixpll, struct mga_device *mdev)
-{
-	struct drm_device *dev = &mdev->base;
-	struct mgag200_g200se_device *g200se;
-
-	pixpll->mdev = mdev;
-
-	switch (mdev->type) {
-	case G200_PCI:
-	case G200_AGP:
-		pixpll->funcs = &mgag200_pixpll_funcs_g200;
-		break;
-	case G200_SE_A:
-	case G200_SE_B:
-		g200se = to_mgag200_g200se_device(dev);
-
-		if (g200se->unique_rev_id >= 0x04)
-			pixpll->funcs = &mgag200_pixpll_funcs_g200se_04;
-		else
-			pixpll->funcs = &mgag200_pixpll_funcs_g200se_00;
-		break;
-	case G200_WB:
-		pixpll->funcs = &mgag200_pixpll_funcs_g200wb;
-		break;
-	case G200_EV:
-		pixpll->funcs = &mgag200_pixpll_funcs_g200ev;
-		break;
-	case G200_EH:
-		pixpll->funcs = &mgag200_pixpll_funcs_g200eh;
-		break;
-	case G200_EH3:
-		pixpll->funcs = &mgag200_pixpll_funcs_g200eh3;
-		break;
-	case G200_ER:
-		pixpll->funcs = &mgag200_pixpll_funcs_g200er;
-		break;
-	case G200_EW3:
-		pixpll->funcs = &mgag200_pixpll_funcs_g200ew3;
-		break;
-	default:
-		drm_err(dev, "unknown device type %d\n", mdev->type);
-		return -ENODEV;
-	}
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/mgag200/mgag200_reg.h b/drivers/gpu/drm/mgag200/mgag200_reg.h
index 99a9ab7d9119..1019ffd6c260 100644
--- a/drivers/gpu/drm/mgag200/mgag200_reg.h
+++ b/drivers/gpu/drm/mgag200/mgag200_reg.h
@@ -96,7 +96,7 @@
 #define MGAREG_SRCORG		0x2cb4
 #define MGAREG_DSTORG		0x2cb8
 
-/* add or or this to one of the previous "power registers" to start
+/* add or this to one of the previous "power registers" to start
    the drawing engine */
 
 #define MGAREG_EXEC		0x0100
diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
index 0ab0e1dd8bbb..2c8b9899625b 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
@@ -68,7 +68,7 @@ static void a3xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 
 	/* BIT(31) of CACHE_FLUSH_TS triggers CACHE_FLUSH_TS IRQ from GPU */
 	OUT_PKT3(ring, CP_EVENT_WRITE, 3);
-	OUT_RING(ring, CACHE_FLUSH_TS | BIT(31));
+	OUT_RING(ring, CACHE_FLUSH_TS | CP_EVENT_WRITE_0_IRQ);
 	OUT_RING(ring, rbmemptr(ring, fence));
 	OUT_RING(ring, submit->seqno);
 
diff --git a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
index 0c6b2a6d0b4c..7cb8d9849c07 100644
--- a/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a4xx_gpu.c
@@ -62,7 +62,7 @@ static void a4xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 
 	/* BIT(31) of CACHE_FLUSH_TS triggers CACHE_FLUSH_TS IRQ from GPU */
 	OUT_PKT3(ring, CP_EVENT_WRITE, 3);
-	OUT_RING(ring, CACHE_FLUSH_TS | BIT(31));
+	OUT_RING(ring, CACHE_FLUSH_TS | CP_EVENT_WRITE_0_IRQ);
 	OUT_RING(ring, rbmemptr(ring, fence));
 	OUT_RING(ring, submit->seqno);
 
diff --git a/drivers/gpu/drm/msm/adreno/a6xx.xml.h b/drivers/gpu/drm/msm/adreno/a6xx.xml.h
index b03e2c413ab1..beea4a7fc1df 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx.xml.h
@@ -1413,6 +1413,10 @@ static inline uint32_t REG_A6XX_RBBM_PERFCTR_RBBM_SEL(uint32_t i0) { return 0x00
 
 #define REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL			0x00000011
 
+#define REG_A6XX_RBBM_GBIF_HALT					0x00000016
+
+#define REG_A6XX_RBBM_GBIF_HALT_ACK				0x00000017
+
 #define REG_A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD			0x0000001c
 #define A6XX_RBBM_WAIT_FOR_GPU_IDLE_CMD_WAIT_GPU_IDLE		0x00000001
 
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
index 310a317885a1..e033d6a67a20 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -873,9 +873,47 @@ static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu)
 		(val & 1), 100, 1000);
 }
 
+#define GBIF_CLIENT_HALT_MASK             BIT(0)
+#define GBIF_ARB_HALT_MASK                BIT(1)
+
+static void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu)
+{
+	struct msm_gpu *gpu = &adreno_gpu->base;
+
+	if (!a6xx_has_gbif(adreno_gpu)) {
+		gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf);
+		spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) &
+								0xf) == 0xf);
+		gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
+
+		return;
+	}
+
+	/* Halt the gx side of GBIF */
+	gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 1);
+	spin_until(gpu_read(gpu, REG_A6XX_RBBM_GBIF_HALT_ACK) & 1);
+
+	/* Halt new client requests on GBIF */
+	gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK);
+	spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
+			(GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK);
+
+	/* Halt all AXI requests on GBIF */
+	gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK);
+	spin_until((gpu_read(gpu,  REG_A6XX_GBIF_HALT_ACK) &
+			(GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK);
+
+	/* The GBIF halt needs to be explicitly cleared */
+	gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0);
+}
+
 /* Force the GMU off in case it isn't responsive */
 static void a6xx_gmu_force_off(struct a6xx_gmu *gmu)
 {
+	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
+	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
+	struct msm_gpu *gpu = &adreno_gpu->base;
+
 	/* Flush all the queues */
 	a6xx_hfi_stop(gmu);
 
@@ -887,6 +925,15 @@ static void a6xx_gmu_force_off(struct a6xx_gmu *gmu)
 
 	/* Make sure there are no outstanding RPMh votes */
 	a6xx_gmu_rpmh_off(gmu);
+
+	/* Halt the gmu cm3 core */
+	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
+
+	a6xx_bus_clear_pending_transactions(adreno_gpu);
+
+	/* Reset GPU core blocks */
+	gpu_write(gpu, REG_A6XX_RBBM_SW_RESET_CMD, 1);
+	udelay(100);
 }
 
 static void a6xx_gmu_set_initial_freq(struct msm_gpu *gpu, struct a6xx_gmu *gmu)
@@ -1014,36 +1061,6 @@ bool a6xx_gmu_isidle(struct a6xx_gmu *gmu)
 	return true;
 }
 
-#define GBIF_CLIENT_HALT_MASK             BIT(0)
-#define GBIF_ARB_HALT_MASK                BIT(1)
-
-static void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu)
-{
-	struct msm_gpu *gpu = &adreno_gpu->base;
-
-	if (!a6xx_has_gbif(adreno_gpu)) {
-		gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf);
-		spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) &
-								0xf) == 0xf);
-		gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
-
-		return;
-	}
-
-	/* Halt new client requests on GBIF */
-	gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK);
-	spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
-			(GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK);
-
-	/* Halt all AXI requests on GBIF */
-	gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK);
-	spin_until((gpu_read(gpu,  REG_A6XX_GBIF_HALT_ACK) &
-			(GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK);
-
-	/* The GBIF halt needs to be explicitly cleared */
-	gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0);
-}
-
 /* Gracefully try to shut down the GMU and by extension the GPU */
 static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu)
 {
@@ -1069,7 +1086,11 @@ static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu)
 		a6xx_bus_clear_pending_transactions(adreno_gpu);
 
 		/* tell the GMU we want to slumber */
-		a6xx_gmu_notify_slumber(gmu);
+		ret = a6xx_gmu_notify_slumber(gmu);
+		if (ret) {
+			a6xx_gmu_force_off(gmu);
+			return;
+		}
 
 		ret = gmu_poll_timeout(gmu,
 			REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, val,
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index 4d501100b9e4..fdc578016e0b 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -10,6 +10,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/devfreq.h>
+#include <linux/reset.h>
 #include <linux/soc/qcom/llcc-qcom.h>
 
 #define GPU_PAS_ID 13
@@ -146,7 +147,7 @@ static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu,
 	 */
 
 	OUT_PKT7(ring, CP_EVENT_WRITE, 1);
-	OUT_RING(ring, 0x31);
+	OUT_RING(ring, CACHE_INVALIDATE);
 
 	if (!sysprof) {
 		/*
@@ -987,6 +988,10 @@ static int hw_init(struct msm_gpu *gpu)
 	/* Make sure the GMU keeps the GPU on while we set it up */
 	a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
 
+	/* Clear GBIF halt in case GX domain was not collapsed */
+	if (a6xx_has_gbif(adreno_gpu))
+		gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 0);
+
 	gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);
 
 	/*
@@ -1261,7 +1266,7 @@ static void a6xx_recover(struct msm_gpu *gpu)
 {
 	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
 	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
-	int i;
+	int i, active_submits;
 
 	adreno_dump_info(gpu);
 
@@ -1272,14 +1277,46 @@ static void a6xx_recover(struct msm_gpu *gpu)
 	if (hang_debug)
 		a6xx_dump(gpu);
 
+	/* Halt SQE first */
+	gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 3);
+
 	/*
 	 * Turn off keep alive that might have been enabled by the hang
 	 * interrupt
 	 */
 	gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0);
 
-	gpu->funcs->pm_suspend(gpu);
-	gpu->funcs->pm_resume(gpu);
+	pm_runtime_dont_use_autosuspend(&gpu->pdev->dev);
+
+	/* active_submit won't change until we make a submission */
+	mutex_lock(&gpu->active_lock);
+	active_submits = gpu->active_submits;
+
+	/*
+	 * Temporarily clear active_submits count to silence a WARN() in the
+	 * runtime suspend cb
+	 */
+	gpu->active_submits = 0;
+
+	/* Drop the rpm refcount from active submits */
+	if (active_submits)
+		pm_runtime_put(&gpu->pdev->dev);
+
+	/* And the final one from recover worker */
+	pm_runtime_put_sync(&gpu->pdev->dev);
+
+	/* Call into gpucc driver to poll for cx gdsc collapse */
+	reset_control_reset(gpu->cx_collapse);
+
+	pm_runtime_use_autosuspend(&gpu->pdev->dev);
+
+	if (active_submits)
+		pm_runtime_get(&gpu->pdev->dev);
+
+	pm_runtime_get_sync(&gpu->pdev->dev);
+
+	gpu->active_submits = active_submits;
+	mutex_unlock(&gpu->active_lock);
 
 	msm_gpu_hw_init(gpu);
 }
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
index 781dcd3fb283..13ce321283ff 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
@@ -412,7 +412,6 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 	struct dpu_format *format;
 	struct dpu_hw_ctl *ctl = mixer->lm_ctl;
 
-	u32 flush_mask;
 	uint32_t stage_idx, lm_idx;
 	int zpos_cnt[DPU_STAGE_MAX + 1] = { 0 };
 	bool bg_alpha_enable = false;
@@ -420,6 +419,8 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 
 	memset(fetch_active, 0, sizeof(fetch_active));
 	drm_atomic_crtc_for_each_plane(plane, crtc) {
+		enum dpu_sspp sspp_idx;
+
 		state = plane->state;
 		if (!state)
 			continue;
@@ -430,14 +431,14 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 		pstate = to_dpu_plane_state(state);
 		fb = state->fb;
 
-		dpu_plane_get_ctl_flush(plane, ctl, &flush_mask);
-		set_bit(dpu_plane_pipe(plane), fetch_active);
+		sspp_idx = dpu_plane_pipe(plane);
+		set_bit(sspp_idx, fetch_active);
 
 		DRM_DEBUG_ATOMIC("crtc %d stage:%d - plane %d sspp %d fb %d\n",
 				crtc->base.id,
 				pstate->stage,
 				plane->base.id,
-				dpu_plane_pipe(plane) - SSPP_VIG0,
+				sspp_idx - SSPP_VIG0,
 				state->fb ? state->fb->base.id : -1);
 
 		format = to_dpu_format(msm_framebuffer_format(pstate->base.fb));
@@ -447,13 +448,13 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 
 		stage_idx = zpos_cnt[pstate->stage]++;
 		stage_cfg->stage[pstate->stage][stage_idx] =
-					dpu_plane_pipe(plane);
+					sspp_idx;
 		stage_cfg->multirect_index[pstate->stage][stage_idx] =
 					pstate->multirect_index;
 
 		trace_dpu_crtc_setup_mixer(DRMID(crtc), DRMID(plane),
 					   state, pstate, stage_idx,
-					   dpu_plane_pipe(plane) - SSPP_VIG0,
+					   sspp_idx - SSPP_VIG0,
 					   format->base.pixel_format,
 					   fb ? fb->modifier : 0);
 
@@ -462,7 +463,8 @@ static void _dpu_crtc_blend_setup_mixer(struct drm_crtc *crtc,
 			_dpu_crtc_setup_blend_cfg(mixer + lm_idx,
 						pstate, format);
 
-			mixer[lm_idx].flush_mask |= flush_mask;
+			mixer[lm_idx].lm_ctl->ops.update_pending_flush_sspp(mixer[lm_idx].lm_ctl,
+									    sspp_idx);
 
 			if (bg_alpha_enable && !format->alpha_enable)
 				mixer[lm_idx].mixer_op_mode = 0;
@@ -496,7 +498,6 @@ static void _dpu_crtc_blend_setup(struct drm_crtc *crtc)
 
 	for (i = 0; i < cstate->num_mixers; i++) {
 		mixer[i].mixer_op_mode = 0;
-		mixer[i].flush_mask = 0;
 		if (mixer[i].lm_ctl->ops.clear_all_blendstages)
 			mixer[i].lm_ctl->ops.clear_all_blendstages(
 					mixer[i].lm_ctl);
@@ -513,17 +514,14 @@ static void _dpu_crtc_blend_setup(struct drm_crtc *crtc)
 
 		lm->ops.setup_alpha_out(lm, mixer[i].mixer_op_mode);
 
-		mixer[i].flush_mask |= ctl->ops.get_bitmask_mixer(ctl,
-			mixer[i].hw_lm->idx);
-
 		/* stage config flush mask */
-		ctl->ops.update_pending_flush(ctl, mixer[i].flush_mask);
+		ctl->ops.update_pending_flush_mixer(ctl,
+			mixer[i].hw_lm->idx);
 
-		DRM_DEBUG_ATOMIC("lm %d, op_mode 0x%X, ctl %d, flush mask 0x%x\n",
+		DRM_DEBUG_ATOMIC("lm %d, op_mode 0x%X, ctl %d\n",
 			mixer[i].hw_lm->idx - LM_0,
 			mixer[i].mixer_op_mode,
-			ctl->idx - CTL_0,
-			mixer[i].flush_mask);
+			ctl->idx - CTL_0);
 
 		ctl->ops.setup_blendstage(ctl, mixer[i].hw_lm->idx,
 			&stage_cfg);
@@ -767,16 +765,9 @@ static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc)
 			dspp->ops.setup_pcc(dspp, &cfg);
 		}
 
-		mixer[i].flush_mask |= ctl->ops.get_bitmask_dspp(ctl,
-			mixer[i].hw_dspp->idx);
-
 		/* stage config flush mask */
-		ctl->ops.update_pending_flush(ctl, mixer[i].flush_mask);
-
-		DRM_DEBUG_ATOMIC("lm %d, ctl %d, flush mask 0x%x\n",
-			mixer[i].hw_lm->idx - DSPP_0,
-			ctl->idx - CTL_0,
-			mixer[i].flush_mask);
+		ctl->ops.update_pending_flush_dspp(ctl,
+			mixer[i].hw_dspp->idx);
 	}
 }
 
@@ -1235,17 +1226,8 @@ static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
 	}
 
 	for (i = 1; i < SSPP_MAX; i++) {
-		if (pipe_staged[i]) {
+		if (pipe_staged[i])
 			dpu_plane_clear_multirect(pipe_staged[i]);
-
-			if (is_dpu_plane_virtual(pipe_staged[i]->plane)) {
-				DPU_ERROR(
-					"r1 only virt plane:%d not supported\n",
-					pipe_staged[i]->plane->base.id);
-				rc  = -EINVAL;
-				goto end;
-			}
-		}
 	}
 
 	z_pos = -1;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h
index 9b67645c2574..539b68b1626a 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.h
@@ -97,7 +97,6 @@ struct dpu_crtc_mixer {
 	struct dpu_hw_ctl *lm_ctl;
 	struct dpu_hw_dspp *hw_dspp;
 	u32 mixer_op_mode;
-	u32 flush_mask;
 };
 
 /**
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
index 52a626117f70..9c6817b5a194 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
@@ -162,7 +162,7 @@ enum dpu_enc_rc_states {
  * @vsync_event_work:		worker to handle vsync event for autorefresh
  * @topology:                   topology of the display
  * @idle_timeout:		idle timeout duration in milliseconds
- * @dsc:			msm_display_dsc_config pointer, for DSC-enabled encoders
+ * @dsc:			drm_dsc_config pointer, for DSC-enabled encoders
  */
 struct dpu_encoder_virt {
 	struct drm_encoder base;
@@ -208,7 +208,7 @@ struct dpu_encoder_virt {
 	bool wide_bus_en;
 
 	/* DSC configuration */
-	struct msm_display_dsc_config *dsc;
+	struct drm_dsc_config *dsc;
 };
 
 #define to_dpu_encoder_virt(x) container_of(x, struct dpu_encoder_virt, base)
@@ -1791,12 +1791,12 @@ static void dpu_encoder_vsync_event_work_handler(struct kthread_work *work)
 }
 
 static u32
-dpu_encoder_dsc_initial_line_calc(struct msm_display_dsc_config *dsc,
+dpu_encoder_dsc_initial_line_calc(struct drm_dsc_config *dsc,
 				  u32 enc_ip_width)
 {
 	int ssm_delay, total_pixels, soft_slice_per_enc;
 
-	soft_slice_per_enc = enc_ip_width / dsc->drm->slice_width;
+	soft_slice_per_enc = enc_ip_width / dsc->slice_width;
 
 	/*
 	 * minimum number of initial line pixels is a sum of:
@@ -1808,16 +1808,16 @@ dpu_encoder_dsc_initial_line_calc(struct msm_display_dsc_config *dsc,
 	 * 5. 6 additional pixels as the output of the rate buffer is
 	 *    48 bits wide
 	 */
-	ssm_delay = ((dsc->drm->bits_per_component < 10) ? 84 : 92);
-	total_pixels = ssm_delay * 3 + dsc->drm->initial_xmit_delay + 47;
+	ssm_delay = ((dsc->bits_per_component < 10) ? 84 : 92);
+	total_pixels = ssm_delay * 3 + dsc->initial_xmit_delay + 47;
 	if (soft_slice_per_enc > 1)
 		total_pixels += (ssm_delay * 3);
-	return DIV_ROUND_UP(total_pixels, dsc->drm->slice_width);
+	return DIV_ROUND_UP(total_pixels, dsc->slice_width);
 }
 
 static void dpu_encoder_dsc_pipe_cfg(struct dpu_hw_dsc *hw_dsc,
 				     struct dpu_hw_pingpong *hw_pp,
-				     struct msm_display_dsc_config *dsc,
+				     struct drm_dsc_config *dsc,
 				     u32 common_mode,
 				     u32 initial_lines)
 {
@@ -1835,7 +1835,7 @@ static void dpu_encoder_dsc_pipe_cfg(struct dpu_hw_dsc *hw_dsc,
 }
 
 static void dpu_encoder_prep_dsc(struct dpu_encoder_virt *dpu_enc,
-				 struct msm_display_dsc_config *dsc)
+				 struct drm_dsc_config *dsc)
 {
 	/* coding only for 2LM, 2enc, 1 dsc config */
 	struct dpu_encoder_phys *enc_master = dpu_enc->cur_master;
@@ -1858,14 +1858,15 @@ static void dpu_encoder_prep_dsc(struct dpu_encoder_virt *dpu_enc,
 		}
 	}
 
-	pic_width = dsc->drm->pic_width;
+	dsc_common_mode = 0;
+	pic_width = dsc->pic_width;
 
 	dsc_common_mode = DSC_MODE_MULTIPLEX | DSC_MODE_SPLIT_PANEL;
 	if (enc_master->intf_mode == INTF_MODE_VIDEO)
 		dsc_common_mode |= DSC_MODE_VIDEO;
 
-	this_frame_slices = pic_width / dsc->drm->slice_width;
-	intf_ip_w = this_frame_slices * dsc->drm->slice_width;
+	this_frame_slices = pic_width / dsc->slice_width;
+	intf_ip_w = this_frame_slices * dsc->slice_width;
 
 	/*
 	 * dsc merge case: when using 2 encoders for the same stream,
@@ -1980,7 +1981,6 @@ static void dpu_encoder_helper_reset_mixers(struct dpu_encoder_phys *phys_enc)
 {
 	struct dpu_hw_mixer_cfg mixer;
 	int i, num_lm;
-	u32 flush_mask = 0;
 	struct dpu_global_state *global_state;
 	struct dpu_hw_blk *hw_lm[2];
 	struct dpu_hw_mixer *hw_mixer[2];
@@ -1999,9 +1999,8 @@ static void dpu_encoder_helper_reset_mixers(struct dpu_encoder_phys *phys_enc)
 
 	for (i = 0; i < num_lm; i++) {
 		hw_mixer[i] = to_dpu_hw_mixer(hw_lm[i]);
-		flush_mask = phys_enc->hw_ctl->ops.get_bitmask_mixer(ctl, hw_mixer[i]->idx);
-		if (phys_enc->hw_ctl->ops.update_pending_flush)
-			phys_enc->hw_ctl->ops.update_pending_flush(ctl, flush_mask);
+		if (phys_enc->hw_ctl->ops.update_pending_flush_mixer)
+			phys_enc->hw_ctl->ops.update_pending_flush_mixer(ctl, hw_mixer[i]->idx);
 
 		/* clear all blendstages */
 		if (phys_enc->hw_ctl->ops.setup_blendstage)
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
index d4d1ecd416e3..9e7236ef34e6 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
@@ -36,7 +36,7 @@ struct msm_display_info {
 	uint32_t h_tile_instance[MAX_H_TILES_PER_DISPLAY];
 	bool is_cmd_mode;
 	bool is_te_using_watchdog_timer;
-	struct msm_display_dsc_config *dsc;
+	struct drm_dsc_config *dsc;
 };
 
 /**
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
index 0239a811d5ec..27f029fdc682 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
@@ -1333,7 +1333,7 @@ static const struct dpu_vbif_dynamic_ot_cfg msm8998_ot_rdwr_cfg[] = {
 
 static const struct dpu_vbif_cfg msm8998_vbif[] = {
 	{
-	.name = "vbif_0", .id = VBIF_0,
+	.name = "vbif_rt", .id = VBIF_RT,
 	.base = 0, .len = 0x1040,
 	.default_ot_rd_limit = 32,
 	.default_ot_wr_limit = 32,
@@ -1363,7 +1363,7 @@ static const struct dpu_vbif_cfg msm8998_vbif[] = {
 
 static const struct dpu_vbif_cfg sdm845_vbif[] = {
 	{
-	.name = "vbif_0", .id = VBIF_0,
+	.name = "vbif_rt", .id = VBIF_RT,
 	.base = 0, .len = 0x1040,
 	.features = BIT(DPU_VBIF_QOS_REMAP),
 	.xin_halt_timeout = 0x4000,
@@ -1939,11 +1939,6 @@ static const struct dpu_mdss_hw_cfg_handler cfg_handler[] = {
 const struct dpu_mdss_cfg *dpu_hw_catalog_init(u32 hw_rev)
 {
 	int i;
-	struct dpu_mdss_cfg *dpu_cfg;
-
-	dpu_cfg = kzalloc(sizeof(*dpu_cfg), GFP_KERNEL);
-	if (!dpu_cfg)
-		return ERR_PTR(-ENOMEM);
 
 	for (i = 0; i < ARRAY_SIZE(cfg_handler); i++) {
 		if (cfg_handler[i].hw_rev == hw_rev)
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
index 71fe4c505f5b..38aa38ab1568 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
@@ -76,7 +76,7 @@ enum {
 
 /**
  * MDP TOP BLOCK features
- * @DPU_MDP_PANIC_PER_PIPE Panic configuration needs to be be done per pipe
+ * @DPU_MDP_PANIC_PER_PIPE Panic configuration needs to be done per pipe
  * @DPU_MDP_10BIT_SUPPORT, Chipset supports 10 bit pixel formats
  * @DPU_MDP_BWC,           MDSS HW supports Bandwidth compression.
  * @DPU_MDP_UBWC_1_0,      This chipsets supports Universal Bandwidth
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
index e12b7fa48a7b..a35ecb6676c8 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.c
@@ -150,92 +150,84 @@ static inline void dpu_hw_ctl_trigger_flush(struct dpu_hw_ctl *ctx)
 	DPU_REG_WRITE(&ctx->hw, CTL_FLUSH, ctx->pending_flush_mask);
 }
 
-static uint32_t dpu_hw_ctl_get_bitmask_sspp(struct dpu_hw_ctl *ctx,
+static void dpu_hw_ctl_update_pending_flush_sspp(struct dpu_hw_ctl *ctx,
 	enum dpu_sspp sspp)
 {
-	uint32_t flushbits = 0;
-
 	switch (sspp) {
 	case SSPP_VIG0:
-		flushbits =  BIT(0);
+		ctx->pending_flush_mask |=  BIT(0);
 		break;
 	case SSPP_VIG1:
-		flushbits = BIT(1);
+		ctx->pending_flush_mask |= BIT(1);
 		break;
 	case SSPP_VIG2:
-		flushbits = BIT(2);
+		ctx->pending_flush_mask |= BIT(2);
 		break;
 	case SSPP_VIG3:
-		flushbits = BIT(18);
+		ctx->pending_flush_mask |= BIT(18);
 		break;
 	case SSPP_RGB0:
-		flushbits = BIT(3);
+		ctx->pending_flush_mask |= BIT(3);
 		break;
 	case SSPP_RGB1:
-		flushbits = BIT(4);
+		ctx->pending_flush_mask |= BIT(4);
 		break;
 	case SSPP_RGB2:
-		flushbits = BIT(5);
+		ctx->pending_flush_mask |= BIT(5);
 		break;
 	case SSPP_RGB3:
-		flushbits = BIT(19);
+		ctx->pending_flush_mask |= BIT(19);
 		break;
 	case SSPP_DMA0:
-		flushbits = BIT(11);
+		ctx->pending_flush_mask |= BIT(11);
 		break;
 	case SSPP_DMA1:
-		flushbits = BIT(12);
+		ctx->pending_flush_mask |= BIT(12);
 		break;
 	case SSPP_DMA2:
-		flushbits = BIT(24);
+		ctx->pending_flush_mask |= BIT(24);
 		break;
 	case SSPP_DMA3:
-		flushbits = BIT(25);
+		ctx->pending_flush_mask |= BIT(25);
 		break;
 	case SSPP_CURSOR0:
-		flushbits = BIT(22);
+		ctx->pending_flush_mask |= BIT(22);
 		break;
 	case SSPP_CURSOR1:
-		flushbits = BIT(23);
+		ctx->pending_flush_mask |= BIT(23);
 		break;
 	default:
 		break;
 	}
-
-	return flushbits;
 }
 
-static uint32_t dpu_hw_ctl_get_bitmask_mixer(struct dpu_hw_ctl *ctx,
+static void dpu_hw_ctl_update_pending_flush_mixer(struct dpu_hw_ctl *ctx,
 	enum dpu_lm lm)
 {
-	uint32_t flushbits = 0;
-
 	switch (lm) {
 	case LM_0:
-		flushbits = BIT(6);
+		ctx->pending_flush_mask |= BIT(6);
 		break;
 	case LM_1:
-		flushbits = BIT(7);
+		ctx->pending_flush_mask |= BIT(7);
 		break;
 	case LM_2:
-		flushbits = BIT(8);
+		ctx->pending_flush_mask |= BIT(8);
 		break;
 	case LM_3:
-		flushbits = BIT(9);
+		ctx->pending_flush_mask |= BIT(9);
 		break;
 	case LM_4:
-		flushbits = BIT(10);
+		ctx->pending_flush_mask |= BIT(10);
 		break;
 	case LM_5:
-		flushbits = BIT(20);
+		ctx->pending_flush_mask |= BIT(20);
 		break;
 	default:
-		return -EINVAL;
+		break;
 	}
 
-	flushbits |= CTL_FLUSH_MASK_CTL;
-
-	return flushbits;
+	ctx->pending_flush_mask |= CTL_FLUSH_MASK_CTL;
 }
 
 static void dpu_hw_ctl_update_pending_flush_intf(struct dpu_hw_ctl *ctx,
@@ -294,29 +286,25 @@ static void dpu_hw_ctl_update_pending_flush_merge_3d_v1(struct dpu_hw_ctl *ctx,
 	ctx->pending_flush_mask |= BIT(MERGE_3D_IDX);
 }
 
-static uint32_t dpu_hw_ctl_get_bitmask_dspp(struct dpu_hw_ctl *ctx,
+static void dpu_hw_ctl_update_pending_flush_dspp(struct dpu_hw_ctl *ctx,
 	enum dpu_dspp dspp)
 {
-	uint32_t flushbits = 0;
-
 	switch (dspp) {
 	case DSPP_0:
-		flushbits = BIT(13);
+		ctx->pending_flush_mask |= BIT(13);
 		break;
 	case DSPP_1:
-		flushbits = BIT(14);
+		ctx->pending_flush_mask |= BIT(14);
 		break;
 	case DSPP_2:
-		flushbits = BIT(15);
+		ctx->pending_flush_mask |= BIT(15);
 		break;
 	case DSPP_3:
-		flushbits = BIT(21);
+		ctx->pending_flush_mask |= BIT(21);
 		break;
 	default:
-		return 0;
+		break;
 	}
-
-	return flushbits;
 }
 
 static u32 dpu_hw_ctl_poll_reset_status(struct dpu_hw_ctl *ctx, u32 timeout_us)
@@ -685,9 +673,9 @@ static void _setup_ctl_ops(struct dpu_hw_ctl_ops *ops,
 	ops->wait_reset_status = dpu_hw_ctl_wait_reset_status;
 	ops->clear_all_blendstages = dpu_hw_ctl_clear_all_blendstages;
 	ops->setup_blendstage = dpu_hw_ctl_setup_blendstage;
-	ops->get_bitmask_sspp = dpu_hw_ctl_get_bitmask_sspp;
-	ops->get_bitmask_mixer = dpu_hw_ctl_get_bitmask_mixer;
-	ops->get_bitmask_dspp = dpu_hw_ctl_get_bitmask_dspp;
+	ops->update_pending_flush_sspp = dpu_hw_ctl_update_pending_flush_sspp;
+	ops->update_pending_flush_mixer = dpu_hw_ctl_update_pending_flush_mixer;
+	ops->update_pending_flush_dspp = dpu_hw_ctl_update_pending_flush_dspp;
 	if (cap & BIT(DPU_CTL_FETCH_ACTIVE))
 		ops->set_active_pipes = dpu_hw_ctl_set_fetch_pipe_active;
 };
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
index 7d9ad6a3f9f6..96c012ec8467 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_ctl.h
@@ -130,6 +130,32 @@ struct dpu_hw_ctl_ops {
 		enum dpu_merge_3d blk);
 
 	/**
+	 * OR in the given flushbits to the cached pending_flush_mask
+	 * No effect on hardware
+	 * @ctx       : ctl path ctx pointer
+	 * @blk       : SSPP block index
+	 */
+	void (*update_pending_flush_sspp)(struct dpu_hw_ctl *ctx,
+		enum dpu_sspp blk);
+
+	/**
+	 * OR in the given flushbits to the cached pending_flush_mask
+	 * No effect on hardware
+	 * @ctx       : ctl path ctx pointer
+	 * @blk       : LM block index
+	 */
+	void (*update_pending_flush_mixer)(struct dpu_hw_ctl *ctx,
+		enum dpu_lm blk);
+
+	/**
+	 * OR in the given flushbits to the cached pending_flush_mask
+	 * No effect on hardware
+	 * @ctx       : ctl path ctx pointer
+	 * @blk       : DSPP block index
+	 */
+	void (*update_pending_flush_dspp)(struct dpu_hw_ctl *ctx,
+		enum dpu_dspp blk);
+	/**
 	 * Write the value of the pending_flush_mask to hardware
 	 * @ctx       : ctl path ctx pointer
 	 */
@@ -171,15 +197,6 @@ struct dpu_hw_ctl_ops {
 	 */
 	int (*wait_reset_status)(struct dpu_hw_ctl *ctx);
 
-	uint32_t (*get_bitmask_sspp)(struct dpu_hw_ctl *ctx,
-		enum dpu_sspp blk);
-
-	uint32_t (*get_bitmask_mixer)(struct dpu_hw_ctl *ctx,
-		enum dpu_lm blk);
-
-	uint32_t (*get_bitmask_dspp)(struct dpu_hw_ctl *ctx,
-		enum dpu_dspp blk);
-
 	/**
 	 * Set all blend stages to disabled
 	 * @ctx       : ctl path ctx pointer
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c
index 411689ae6382..f2ddcfb6f7ee 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.c
@@ -37,7 +37,7 @@ static void dpu_hw_dsc_disable(struct dpu_hw_dsc *dsc)
 }
 
 static void dpu_hw_dsc_config(struct dpu_hw_dsc *hw_dsc,
-			      struct msm_display_dsc_config *dsc,
+			      struct drm_dsc_config *dsc,
 			      u32 mode,
 			      u32 initial_lines)
 {
@@ -52,89 +52,89 @@ static void dpu_hw_dsc_config(struct dpu_hw_dsc *hw_dsc,
 	if (is_cmd_mode)
 		initial_lines += 1;
 
-	slice_last_group_size = 3 - (dsc->drm->slice_width % 3);
+	slice_last_group_size = 3 - (dsc->slice_width % 3);
 	data = (initial_lines << 20);
 	data |= ((slice_last_group_size - 1) << 18);
 	/* bpp is 6.4 format, 4 LSBs bits are for fractional part */
-	data |= dsc->drm->bits_per_pixel << 12;
-	lsb = dsc->drm->bits_per_pixel % 4;
-	bpp = dsc->drm->bits_per_pixel / 4;
+	data |= dsc->bits_per_pixel << 12;
+	lsb = dsc->bits_per_pixel % 4;
+	bpp = dsc->bits_per_pixel / 4;
 	bpp *= 4;
 	bpp <<= 4;
 	bpp |= lsb;
 
 	data |= bpp << 8;
-	data |= (dsc->drm->block_pred_enable << 7);
-	data |= (dsc->drm->line_buf_depth << 3);
-	data |= (dsc->drm->simple_422 << 2);
-	data |= (dsc->drm->convert_rgb << 1);
-	data |= dsc->drm->bits_per_component;
+	data |= (dsc->block_pred_enable << 7);
+	data |= (dsc->line_buf_depth << 3);
+	data |= (dsc->simple_422 << 2);
+	data |= (dsc->convert_rgb << 1);
+	data |= dsc->bits_per_component;
 
 	DPU_REG_WRITE(c, DSC_ENC, data);
 
-	data = dsc->drm->pic_width << 16;
-	data |= dsc->drm->pic_height;
+	data = dsc->pic_width << 16;
+	data |= dsc->pic_height;
 	DPU_REG_WRITE(c, DSC_PICTURE, data);
 
-	data = dsc->drm->slice_width << 16;
-	data |= dsc->drm->slice_height;
+	data = dsc->slice_width << 16;
+	data |= dsc->slice_height;
 	DPU_REG_WRITE(c, DSC_SLICE, data);
 
-	data = dsc->drm->slice_chunk_size << 16;
+	data = dsc->slice_chunk_size << 16;
 	DPU_REG_WRITE(c, DSC_CHUNK_SIZE, data);
 
-	data = dsc->drm->initial_dec_delay << 16;
-	data |= dsc->drm->initial_xmit_delay;
+	data = dsc->initial_dec_delay << 16;
+	data |= dsc->initial_xmit_delay;
 	DPU_REG_WRITE(c, DSC_DELAY, data);
 
-	data = dsc->drm->initial_scale_value;
+	data = dsc->initial_scale_value;
 	DPU_REG_WRITE(c, DSC_SCALE_INITIAL, data);
 
-	data = dsc->drm->scale_decrement_interval;
+	data = dsc->scale_decrement_interval;
 	DPU_REG_WRITE(c, DSC_SCALE_DEC_INTERVAL, data);
 
-	data = dsc->drm->scale_increment_interval;
+	data = dsc->scale_increment_interval;
 	DPU_REG_WRITE(c, DSC_SCALE_INC_INTERVAL, data);
 
-	data = dsc->drm->first_line_bpg_offset;
+	data = dsc->first_line_bpg_offset;
 	DPU_REG_WRITE(c, DSC_FIRST_LINE_BPG_OFFSET, data);
 
-	data = dsc->drm->nfl_bpg_offset << 16;
-	data |= dsc->drm->slice_bpg_offset;
+	data = dsc->nfl_bpg_offset << 16;
+	data |= dsc->slice_bpg_offset;
 	DPU_REG_WRITE(c, DSC_BPG_OFFSET, data);
 
-	data = dsc->drm->initial_offset << 16;
-	data |= dsc->drm->final_offset;
+	data = dsc->initial_offset << 16;
+	data |= dsc->final_offset;
 	DPU_REG_WRITE(c, DSC_DSC_OFFSET, data);
 
-	det_thresh_flatness = 7 + 2 * (dsc->drm->bits_per_component - 8);
+	det_thresh_flatness = 7 + 2 * (dsc->bits_per_component - 8);
 	data = det_thresh_flatness << 10;
-	data |= dsc->drm->flatness_max_qp << 5;
-	data |= dsc->drm->flatness_min_qp;
+	data |= dsc->flatness_max_qp << 5;
+	data |= dsc->flatness_min_qp;
 	DPU_REG_WRITE(c, DSC_FLATNESS, data);
 
-	data = dsc->drm->rc_model_size;
+	data = dsc->rc_model_size;
 	DPU_REG_WRITE(c, DSC_RC_MODEL_SIZE, data);
 
-	data = dsc->drm->rc_tgt_offset_low << 18;
-	data |= dsc->drm->rc_tgt_offset_high << 14;
-	data |= dsc->drm->rc_quant_incr_limit1 << 9;
-	data |= dsc->drm->rc_quant_incr_limit0 << 4;
-	data |= dsc->drm->rc_edge_factor;
+	data = dsc->rc_tgt_offset_low << 18;
+	data |= dsc->rc_tgt_offset_high << 14;
+	data |= dsc->rc_quant_incr_limit1 << 9;
+	data |= dsc->rc_quant_incr_limit0 << 4;
+	data |= dsc->rc_edge_factor;
 	DPU_REG_WRITE(c, DSC_RC, data);
 }
 
 static void dpu_hw_dsc_config_thresh(struct dpu_hw_dsc *hw_dsc,
-				     struct msm_display_dsc_config *dsc)
+				     struct drm_dsc_config *dsc)
 {
-	struct drm_dsc_rc_range_parameters *rc = dsc->drm->rc_range_params;
+	struct drm_dsc_rc_range_parameters *rc = dsc->rc_range_params;
 	struct dpu_hw_blk_reg_map *c = &hw_dsc->hw;
 	u32 off;
 	int i;
 
 	off = DSC_RC_BUF_THRESH;
 	for (i = 0; i < DSC_NUM_BUF_RANGES - 1 ; i++) {
-		DPU_REG_WRITE(c, off, dsc->drm->rc_buf_thresh[i]);
+		DPU_REG_WRITE(c, off, dsc->rc_buf_thresh[i]);
 		off += 4;
 	}
 
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h
index 45e4118f1fa2..c0b77fe1a696 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_dsc.h
@@ -31,7 +31,7 @@ struct dpu_hw_dsc_ops {
 	 * @initial_lines: amount of initial lines to be used
 	 */
 	void (*dsc_config)(struct dpu_hw_dsc *hw_dsc,
-			   struct msm_display_dsc_config *dsc,
+			   struct drm_dsc_config *dsc,
 			   u32 mode,
 			   u32 initial_lines);
 
@@ -41,7 +41,7 @@ struct dpu_hw_dsc_ops {
 	 * @dsc: panel dsc parameters
 	 */
 	void (*dsc_config_thresh)(struct dpu_hw_dsc *hw_dsc,
-				  struct msm_display_dsc_config *dsc);
+				  struct drm_dsc_config *dsc);
 };
 
 struct dpu_hw_dsc {
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
index 9f402be55fbf..d3b0ed0a9c6c 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_mdss.h
@@ -273,11 +273,9 @@ enum dpu_wd_timer {
 };
 
 enum dpu_vbif {
-	VBIF_0,
-	VBIF_1,
+	VBIF_RT,
+	VBIF_NRT,
 	VBIF_MAX,
-	VBIF_RT = VBIF_0,
-	VBIF_NRT = VBIF_1
 };
 
 /**
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
index 102c21bb4192..691c471b08c2 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.c
@@ -780,8 +780,7 @@ static const struct dpu_sspp_cfg *_sspp_offset(enum dpu_sspp sspp,
 }
 
 struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx,
-		void __iomem *addr, const struct dpu_mdss_cfg *catalog,
-		bool is_virtual_pipe)
+		void __iomem *addr, const struct dpu_mdss_cfg *catalog)
 {
 	struct dpu_hw_pipe *hw_pipe;
 	const struct dpu_sspp_cfg *cfg;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
index 78b1bc9e004f..0c95b7e64f6c 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_sspp.h
@@ -377,11 +377,9 @@ struct dpu_kms;
  * @idx:  Pipe index for which driver object is required
  * @addr: Mapped register io address of MDP
  * @catalog : Pointer to mdss catalog data
- * @is_virtual_pipe: is this pipe virtual pipe
  */
 struct dpu_hw_pipe *dpu_hw_sspp_init(enum dpu_sspp idx,
-		void __iomem *addr, const struct dpu_mdss_cfg *catalog,
-		bool is_virtual_pipe);
+		void __iomem *addr, const struct dpu_mdss_cfg *catalog);
 
 /**
  * dpu_hw_sspp_destroy(): Destroys SSPP driver context
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
index 008e1420e6e5..5e6e2626151e 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
@@ -384,12 +384,9 @@ static int dpu_kms_parse_data_bus_icc_path(struct dpu_kms *dpu_kms)
 	struct icc_path *path1;
 	struct drm_device *dev = dpu_kms->dev;
 	struct device *dpu_dev = dev->dev;
-	struct device *mdss_dev = dpu_dev->parent;
 
-	/* Interconnects are a part of MDSS device tree binding, not the
-	 * MDP/DPU device. */
-	path0 = of_icc_get(mdss_dev, "mdp0-mem");
-	path1 = of_icc_get(mdss_dev, "mdp1-mem");
+	path0 = msm_icc_get(dpu_dev, "mdp0-mem");
+	path1 = msm_icc_get(dpu_dev, "mdp1-mem");
 
 	if (IS_ERR_OR_NULL(path0))
 		return PTR_ERR_OR_ZERO(path0);
@@ -782,7 +779,7 @@ static int _dpu_kms_drm_obj_init(struct dpu_kms *dpu_kms)
 			  catalog->sspp[i].features & BIT(DPU_SSPP_CURSOR));
 
 		plane = dpu_plane_init(dev, catalog->sspp[i].id, type,
-				       (1UL << max_crtc_count) - 1, 0);
+				       (1UL << max_crtc_count) - 1);
 		if (IS_ERR(plane)) {
 			DPU_ERROR("dpu_plane_init failed\n");
 			ret = PTR_ERR(plane);
@@ -826,12 +823,10 @@ static void _dpu_kms_hw_destroy(struct dpu_kms *dpu_kms)
 	_dpu_kms_mmu_destroy(dpu_kms);
 
 	if (dpu_kms->catalog) {
-		for (i = 0; i < dpu_kms->catalog->vbif_count; i++) {
-			u32 vbif_idx = dpu_kms->catalog->vbif[i].id;
-
-			if ((vbif_idx < VBIF_MAX) && dpu_kms->hw_vbif[vbif_idx]) {
-				dpu_hw_vbif_destroy(dpu_kms->hw_vbif[vbif_idx]);
-				dpu_kms->hw_vbif[vbif_idx] = NULL;
+		for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) {
+			if (dpu_kms->hw_vbif[i]) {
+				dpu_hw_vbif_destroy(dpu_kms->hw_vbif[i]);
+				dpu_kms->hw_vbif[i] = NULL;
 			}
 		}
 	}
@@ -902,12 +897,10 @@ static void dpu_kms_mdp_snapshot(struct msm_disp_state *disp_state, struct msm_k
 	int i;
 	struct dpu_kms *dpu_kms;
 	const struct dpu_mdss_cfg *cat;
-	struct dpu_hw_mdp *top;
 
 	dpu_kms = to_dpu_kms(kms);
 
 	cat = dpu_kms->catalog;
-	top = dpu_kms->hw_mdp;
 
 	pm_runtime_get_sync(&dpu_kms->pdev->dev);
 
@@ -1113,12 +1106,10 @@ static int dpu_kms_hw_init(struct msm_kms *kms)
 	for (i = 0; i < dpu_kms->catalog->vbif_count; i++) {
 		u32 vbif_idx = dpu_kms->catalog->vbif[i].id;
 
-		dpu_kms->hw_vbif[i] = dpu_hw_vbif_init(vbif_idx,
+		dpu_kms->hw_vbif[vbif_idx] = dpu_hw_vbif_init(vbif_idx,
 				dpu_kms->vbif[vbif_idx], dpu_kms->catalog);
-		if (IS_ERR_OR_NULL(dpu_kms->hw_vbif[vbif_idx])) {
+		if (IS_ERR(dpu_kms->hw_vbif[vbif_idx])) {
 			rc = PTR_ERR(dpu_kms->hw_vbif[vbif_idx]);
-			if (!dpu_kms->hw_vbif[vbif_idx])
-				rc = -EINVAL;
 			DPU_ERROR("failed to init vbif %d: %d\n", vbif_idx, rc);
 			dpu_kms->hw_vbif[vbif_idx] = NULL;
 			goto power_error;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
index a617a3d8b1bc..658005f609f4 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
@@ -91,7 +91,7 @@ enum dpu_plane_qos {
 /*
  * struct dpu_plane - local dpu plane structure
  * @aspace: address space pointer
- * @mplane_list: List of multirect planes of the same pipe
+ * @csc_ptr: Points to dpu_csc_cfg structure to use for current
  * @catalog: Points to dpu catalog structure
  * @revalidate: force revalidation of all the plane properties
  */
@@ -106,8 +106,6 @@ struct dpu_plane {
 	uint32_t color_fill;
 	bool is_error;
 	bool is_rt_pipe;
-	bool is_virtual;
-	struct list_head mplane_list;
 	const struct dpu_mdss_cfg *catalog;
 };
 
@@ -225,7 +223,7 @@ static void _dpu_plane_calc_clk(struct drm_plane *plane, struct dpu_hw_pipe_cfg
 static int _dpu_plane_calc_fill_level(struct drm_plane *plane,
 		const struct dpu_format *fmt, u32 src_width)
 {
-	struct dpu_plane *pdpu, *tmp;
+	struct dpu_plane *pdpu;
 	struct dpu_plane_state *pstate;
 	u32 fixed_buff_size;
 	u32 total_fl;
@@ -239,19 +237,7 @@ static int _dpu_plane_calc_fill_level(struct drm_plane *plane,
 	pstate = to_dpu_plane_state(plane->state);
 	fixed_buff_size = pdpu->catalog->caps->pixel_ram_size;
 
-	list_for_each_entry(tmp, &pdpu->mplane_list, mplane_list) {
-		u32 tmp_width;
-
-		if (!tmp->base.state->visible)
-			continue;
-		tmp_width = drm_rect_width(&tmp->base.state->src) >> 16;
-		DPU_DEBUG("plane%d/%d src_width:%d/%d\n",
-				pdpu->base.base.id, tmp->base.base.id,
-				src_width,
-				tmp_width);
-		src_width = max_t(u32, src_width,
-				  tmp_width);
-	}
+	/* FIXME: in multirect case account for the src_width of all the planes */
 
 	if (fmt->fetch_planes == DPU_PLANE_PSEUDO_PLANAR) {
 		if (fmt->chroma_sample == DPU_CHROMA_420) {
@@ -854,13 +840,8 @@ int dpu_plane_validate_multirect_v2(struct dpu_multirect_plane_states *plane)
 	}
 
 done:
-	if (dpu_plane[R0]->is_virtual) {
-		pstate[R0]->multirect_index = DPU_SSPP_RECT_1;
-		pstate[R1]->multirect_index = DPU_SSPP_RECT_0;
-	} else {
-		pstate[R0]->multirect_index = DPU_SSPP_RECT_0;
-		pstate[R1]->multirect_index = DPU_SSPP_RECT_1;
-	}
+	pstate[R0]->multirect_index = DPU_SSPP_RECT_0;
+	pstate[R1]->multirect_index = DPU_SSPP_RECT_1;
 
 	DPU_DEBUG_PLANE(dpu_plane[R0], "R0: %d - %d\n",
 		pstate[R0]->multirect_mode, pstate[R0]->multirect_index);
@@ -869,18 +850,6 @@ done:
 	return 0;
 }
 
-/**
- * dpu_plane_get_ctl_flush - get control flush for the given plane
- * @plane: Pointer to drm plane structure
- * @ctl: Pointer to hardware control driver
- * @flush_sspp: Pointer to sspp flush control word
- */
-void dpu_plane_get_ctl_flush(struct drm_plane *plane, struct dpu_hw_ctl *ctl,
-		u32 *flush_sspp)
-{
-	*flush_sspp = ctl->ops.get_bitmask_sspp(ctl, dpu_plane_pipe(plane));
-}
-
 static int dpu_plane_prepare_fb(struct drm_plane *plane,
 		struct drm_plane_state *new_state)
 {
@@ -1266,19 +1235,13 @@ static void dpu_plane_sspp_atomic_update(struct drm_plane *plane)
 
 static void _dpu_plane_atomic_disable(struct drm_plane *plane)
 {
-	struct dpu_plane *pdpu = to_dpu_plane(plane);
 	struct drm_plane_state *state = plane->state;
 	struct dpu_plane_state *pstate = to_dpu_plane_state(state);
 
-	trace_dpu_plane_disable(DRMID(plane), is_dpu_plane_virtual(plane),
+	trace_dpu_plane_disable(DRMID(plane), false,
 				pstate->multirect_mode);
 
 	pstate->pending = true;
-
-	if (is_dpu_plane_virtual(plane) &&
-			pdpu->pipe_hw && pdpu->pipe_hw->ops.setup_multirect)
-		pdpu->pipe_hw->ops.setup_multirect(pdpu->pipe_hw,
-				DPU_SSPP_RECT_SOLO, DPU_SSPP_MULTIRECT_NONE);
 }
 
 static void dpu_plane_atomic_update(struct drm_plane *plane,
@@ -1493,22 +1456,16 @@ enum dpu_sspp dpu_plane_pipe(struct drm_plane *plane)
 	return plane ? to_dpu_plane(plane)->pipe : SSPP_NONE;
 }
 
-bool is_dpu_plane_virtual(struct drm_plane *plane)
-{
-	return plane ? to_dpu_plane(plane)->is_virtual : false;
-}
-
 /* initialize plane */
 struct drm_plane *dpu_plane_init(struct drm_device *dev,
 		uint32_t pipe, enum drm_plane_type type,
-		unsigned long possible_crtcs, u32 master_plane_id)
+		unsigned long possible_crtcs)
 {
-	struct drm_plane *plane = NULL, *master_plane = NULL;
+	struct drm_plane *plane = NULL;
 	const uint32_t *format_list;
 	struct dpu_plane *pdpu;
 	struct msm_drm_private *priv = dev->dev_private;
 	struct dpu_kms *kms = to_dpu_kms(priv->kms);
-	int zpos_max = DPU_ZPOS_MAX;
 	uint32_t num_formats;
 	uint32_t supported_rotations;
 	int ret = -EINVAL;
@@ -1524,18 +1481,9 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev,
 	/* cache local stuff for later */
 	plane = &pdpu->base;
 	pdpu->pipe = pipe;
-	pdpu->is_virtual = (master_plane_id != 0);
-	INIT_LIST_HEAD(&pdpu->mplane_list);
-	master_plane = drm_plane_find(dev, NULL, master_plane_id);
-	if (master_plane) {
-		struct dpu_plane *mpdpu = to_dpu_plane(master_plane);
-
-		list_add_tail(&pdpu->mplane_list, &mpdpu->mplane_list);
-	}
 
 	/* initialize underlying h/w driver */
-	pdpu->pipe_hw = dpu_hw_sspp_init(pipe, kms->mmio, kms->catalog,
-							master_plane_id != 0);
+	pdpu->pipe_hw = dpu_hw_sspp_init(pipe, kms->mmio, kms->catalog);
 	if (IS_ERR(pdpu->pipe_hw)) {
 		DPU_ERROR("[%u]SSPP init failed\n", pipe);
 		ret = PTR_ERR(pdpu->pipe_hw);
@@ -1545,14 +1493,8 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev,
 		goto clean_sspp;
 	}
 
-	if (pdpu->is_virtual) {
-		format_list = pdpu->pipe_hw->cap->sblk->virt_format_list;
-		num_formats = pdpu->pipe_hw->cap->sblk->virt_num_formats;
-	}
-	else {
-		format_list = pdpu->pipe_hw->cap->sblk->format_list;
-		num_formats = pdpu->pipe_hw->cap->sblk->num_formats;
-	}
+	format_list = pdpu->pipe_hw->cap->sblk->format_list;
+	num_formats = pdpu->pipe_hw->cap->sblk->num_formats;
 
 	ret = drm_universal_plane_init(dev, plane, 0xff, &dpu_plane_funcs,
 				format_list, num_formats,
@@ -1562,14 +1504,7 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev,
 
 	pdpu->catalog = kms->catalog;
 
-	if (kms->catalog->mixer_count &&
-		kms->catalog->mixer[0].sblk->maxblendstages) {
-		zpos_max = kms->catalog->mixer[0].sblk->maxblendstages - 1;
-		if (zpos_max > DPU_STAGE_MAX - DPU_STAGE_0 - 1)
-			zpos_max = DPU_STAGE_MAX - DPU_STAGE_0 - 1;
-	}
-
-	ret = drm_plane_create_zpos_property(plane, 0, 0, zpos_max);
+	ret = drm_plane_create_zpos_property(plane, 0, 0, DPU_ZPOS_MAX);
 	if (ret)
 		DPU_ERROR("failed to install zpos property, rc = %d\n", ret);
 
@@ -1594,15 +1529,14 @@ struct drm_plane *dpu_plane_init(struct drm_device *dev,
 
 	mutex_init(&pdpu->lock);
 
-	DPU_DEBUG("%s created for pipe:%u id:%u virtual:%u\n", plane->name,
-					pipe, plane->base.id, master_plane_id);
+	DPU_DEBUG("%s created for pipe:%u id:%u\n", plane->name,
+					pipe, plane->base.id);
 	return plane;
 
 clean_sspp:
 	if (pdpu && pdpu->pipe_hw)
 		dpu_hw_sspp_destroy(pdpu->pipe_hw);
 clean_plane:
-	list_del(&pdpu->mplane_list);
 	kfree(pdpu);
 	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
index e1463107a6fc..b7b1b05199c2 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
@@ -65,23 +65,6 @@ struct dpu_multirect_plane_states {
 enum dpu_sspp dpu_plane_pipe(struct drm_plane *plane);
 
 /**
- * is_dpu_plane_virtual - check for virtual plane
- * @plane: Pointer to DRM plane object
- * returns: true - if the plane is virtual
- *          false - if the plane is primary
- */
-bool is_dpu_plane_virtual(struct drm_plane *plane);
-
-/**
- * dpu_plane_get_ctl_flush - get control flush mask
- * @plane:   Pointer to DRM plane object
- * @ctl: Pointer to control hardware
- * @flush_sspp: Pointer to sspp flush control word
- */
-void dpu_plane_get_ctl_flush(struct drm_plane *plane, struct dpu_hw_ctl *ctl,
-		u32 *flush_sspp);
-
-/**
  * dpu_plane_flush - final plane operations before commit flush
  * @plane: Pointer to drm plane structure
  */
@@ -99,14 +82,11 @@ void dpu_plane_set_error(struct drm_plane *plane, bool error);
  * @pipe:  dpu hardware pipe identifier
  * @type:  Plane type - PRIMARY/OVERLAY/CURSOR
  * @possible_crtcs: bitmask of crtc that can be attached to the given pipe
- * @master_plane_id: primary plane id of a multirect pipe. 0 value passed for
- *                   a regular plane initialization. A non-zero primary plane
- *                   id will be passed for a virtual pipe initialization.
  *
  */
 struct drm_plane *dpu_plane_init(struct drm_device *dev,
 		uint32_t pipe, enum drm_plane_type type,
-		unsigned long possible_crtcs, u32 master_plane_id);
+		unsigned long possible_crtcs);
 
 /**
  * dpu_plane_validate_multirecti_v2 - validate the multirect planes
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c
index 21d20373eb8b..1305e250b71e 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_vbif.c
@@ -11,6 +11,26 @@
 #include "dpu_hw_vbif.h"
 #include "dpu_trace.h"
 
+static struct dpu_hw_vbif *dpu_get_vbif(struct dpu_kms *dpu_kms, enum dpu_vbif vbif_idx)
+{
+	if (vbif_idx < ARRAY_SIZE(dpu_kms->hw_vbif))
+		return dpu_kms->hw_vbif[vbif_idx];
+
+	return NULL;
+}
+
+static const char *dpu_vbif_name(enum dpu_vbif idx)
+{
+	switch (idx) {
+	case VBIF_RT:
+		return "VBIF_RT";
+	case VBIF_NRT:
+		return "VBIF_NRT";
+	default:
+		return "??";
+	}
+}
+
 /**
  * _dpu_vbif_wait_for_xin_halt - wait for the xin to halt
  * @vbif:	Pointer to hardware vbif driver
@@ -42,12 +62,12 @@ static int _dpu_vbif_wait_for_xin_halt(struct dpu_hw_vbif *vbif, u32 xin_id)
 
 	if (!status) {
 		rc = -ETIMEDOUT;
-		DPU_ERROR("VBIF %d client %d not halting. TIMEDOUT.\n",
-				vbif->idx - VBIF_0, xin_id);
+		DPU_ERROR("%s client %d not halting. TIMEDOUT.\n",
+				dpu_vbif_name(vbif->idx), xin_id);
 	} else {
 		rc = 0;
-		DRM_DEBUG_ATOMIC("VBIF %d client %d is halted\n",
-				vbif->idx - VBIF_0, xin_id);
+		DRM_DEBUG_ATOMIC("%s client %d is halted\n",
+				dpu_vbif_name(vbif->idx), xin_id);
 	}
 
 	return rc;
@@ -87,8 +107,8 @@ static void _dpu_vbif_apply_dynamic_ot_limit(struct dpu_hw_vbif *vbif,
 		}
 	}
 
-	DRM_DEBUG_ATOMIC("vbif:%d xin:%d w:%d h:%d fps:%d pps:%llu ot:%u\n",
-			vbif->idx - VBIF_0, params->xin_id,
+	DRM_DEBUG_ATOMIC("%s xin:%d w:%d h:%d fps:%d pps:%llu ot:%u\n",
+			dpu_vbif_name(vbif->idx), params->xin_id,
 			params->width, params->height, params->frame_rate,
 			pps, *ot_lim);
 }
@@ -133,8 +153,8 @@ static u32 _dpu_vbif_get_ot_limit(struct dpu_hw_vbif *vbif,
 	}
 
 exit:
-	DRM_DEBUG_ATOMIC("vbif:%d xin:%d ot_lim:%d\n",
-			vbif->idx - VBIF_0, params->xin_id, ot_lim);
+	DRM_DEBUG_ATOMIC("%s xin:%d ot_lim:%d\n",
+			dpu_vbif_name(vbif->idx), params->xin_id, ot_lim);
 	return ot_lim;
 }
 
@@ -148,20 +168,15 @@ exit:
 void dpu_vbif_set_ot_limit(struct dpu_kms *dpu_kms,
 		struct dpu_vbif_set_ot_params *params)
 {
-	struct dpu_hw_vbif *vbif = NULL;
+	struct dpu_hw_vbif *vbif;
 	struct dpu_hw_mdp *mdp;
 	bool forced_on = false;
 	u32 ot_lim;
-	int ret, i;
+	int ret;
 
 	mdp = dpu_kms->hw_mdp;
 
-	for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) {
-		if (dpu_kms->hw_vbif[i] &&
-				dpu_kms->hw_vbif[i]->idx == params->vbif_idx)
-			vbif = dpu_kms->hw_vbif[i];
-	}
-
+	vbif = dpu_get_vbif(dpu_kms, params->vbif_idx);
 	if (!vbif || !mdp) {
 		DRM_DEBUG_ATOMIC("invalid arguments vbif %d mdp %d\n",
 				vbif != NULL, mdp != NULL);
@@ -204,7 +219,7 @@ void dpu_vbif_set_ot_limit(struct dpu_kms *dpu_kms,
 void dpu_vbif_set_qos_remap(struct dpu_kms *dpu_kms,
 		struct dpu_vbif_set_qos_params *params)
 {
-	struct dpu_hw_vbif *vbif = NULL;
+	struct dpu_hw_vbif *vbif;
 	struct dpu_hw_mdp *mdp;
 	bool forced_on = false;
 	const struct dpu_vbif_qos_tbl *qos_tbl;
@@ -216,13 +231,7 @@ void dpu_vbif_set_qos_remap(struct dpu_kms *dpu_kms,
 	}
 	mdp = dpu_kms->hw_mdp;
 
-	for (i = 0; i < ARRAY_SIZE(dpu_kms->hw_vbif); i++) {
-		if (dpu_kms->hw_vbif[i] &&
-				dpu_kms->hw_vbif[i]->idx == params->vbif_idx) {
-			vbif = dpu_kms->hw_vbif[i];
-			break;
-		}
-	}
+	vbif = dpu_get_vbif(dpu_kms, params->vbif_idx);
 
 	if (!vbif || !vbif->cap) {
 		DPU_ERROR("invalid vbif %d\n", params->vbif_idx);
@@ -245,8 +254,8 @@ void dpu_vbif_set_qos_remap(struct dpu_kms *dpu_kms,
 	forced_on = mdp->ops.setup_clk_force_ctrl(mdp, params->clk_ctrl, true);
 
 	for (i = 0; i < qos_tbl->npriority_lvl; i++) {
-		DRM_DEBUG_ATOMIC("vbif:%d xin:%d lvl:%d/%d\n",
-				params->vbif_idx, params->xin_id, i,
+		DRM_DEBUG_ATOMIC("%s xin:%d lvl:%d/%d\n",
+				dpu_vbif_name(params->vbif_idx), params->xin_id, i,
 				qos_tbl->priority_lvl[i]);
 		vbif->ops.set_qos_remap(vbif, params->xin_id, i,
 				qos_tbl->priority_lvl[i]);
@@ -266,8 +275,8 @@ void dpu_vbif_clear_errors(struct dpu_kms *dpu_kms)
 		if (vbif && vbif->ops.clear_errors) {
 			vbif->ops.clear_errors(vbif, &pnd, &src);
 			if (pnd || src) {
-				DRM_DEBUG_KMS("VBIF %d: pnd 0x%X, src 0x%X\n",
-					      vbif->idx - VBIF_0, pnd, src);
+				DRM_DEBUG_KMS("%s: pnd 0x%X, src 0x%X\n",
+					      dpu_vbif_name(vbif->idx), pnd, src);
 			}
 		}
 	}
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
index d2a48caf9d27..b0d21838a134 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
@@ -902,12 +902,9 @@ fail:
 
 static int mdp5_setup_interconnect(struct platform_device *pdev)
 {
-	/* Interconnects are a part of MDSS device tree binding, not the
-	 * MDP5 device. */
-	struct device *mdss_dev = pdev->dev.parent;
-	struct icc_path *path0 = of_icc_get(mdss_dev, "mdp0-mem");
-	struct icc_path *path1 = of_icc_get(mdss_dev, "mdp1-mem");
-	struct icc_path *path_rot = of_icc_get(mdss_dev, "rotator-mem");
+	struct icc_path *path0 = msm_icc_get(&pdev->dev, "mdp0-mem");
+	struct icc_path *path1 = msm_icc_get(&pdev->dev, "mdp1-mem");
+	struct icc_path *path_rot = msm_icc_get(&pdev->dev, "rotator-mem");
 
 	if (IS_ERR(path0))
 		return PTR_ERR(path0);
diff --git a/drivers/gpu/drm/msm/dp/dp_catalog.c b/drivers/gpu/drm/msm/dp/dp_catalog.c
index 7257515871a9..676279d0ca8d 100644
--- a/drivers/gpu/drm/msm/dp/dp_catalog.c
+++ b/drivers/gpu/drm/msm/dp/dp_catalog.c
@@ -431,7 +431,7 @@ void dp_catalog_ctrl_config_msa(struct dp_catalog *dp_catalog,
 
 	if (rate == link_rate_hbr3)
 		pixel_div = 6;
-	else if (rate == 1620000 || rate == 270000)
+	else if (rate == 162000 || rate == 270000)
 		pixel_div = 2;
 	else if (rate == link_rate_hbr2)
 		pixel_div = 4;
diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c
index 013ca02e17cb..3854c9f1f7e9 100644
--- a/drivers/gpu/drm/msm/dp/dp_ctrl.c
+++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c
@@ -1238,8 +1238,6 @@ static int dp_ctrl_link_train_2(struct dp_ctrl_private *ctrl,
 	return -ETIMEDOUT;
 }
 
-static int dp_ctrl_reinitialize_mainlink(struct dp_ctrl_private *ctrl);
-
 static int dp_ctrl_link_train(struct dp_ctrl_private *ctrl,
 			int *training_step)
 {
@@ -1358,25 +1356,7 @@ static int dp_ctrl_enable_mainlink_clocks(struct dp_ctrl_private *ctrl)
 	if (ret)
 		DRM_ERROR("Unable to start link clocks. ret=%d\n", ret);
 
-	drm_dbg_dp(ctrl->drm_dev, "link rate=%d pixel_clk=%d\n",
-		ctrl->link->link_params.rate, ctrl->dp_ctrl.pixel_rate);
-
-	return ret;
-}
-
-static int dp_ctrl_enable_stream_clocks(struct dp_ctrl_private *ctrl)
-{
-	int ret = 0;
-
-	dp_ctrl_set_clock_rate(ctrl, DP_STREAM_PM, "stream_pixel",
-					ctrl->dp_ctrl.pixel_rate * 1000);
-
-	ret = dp_power_clk_enable(ctrl->power, DP_STREAM_PM, true);
-	if (ret)
-		DRM_ERROR("Unabled to start pixel clocks. ret=%d\n", ret);
-
-	drm_dbg_dp(ctrl->drm_dev, "link rate=%d pixel_clk=%d\n",
-			ctrl->link->link_params.rate, ctrl->dp_ctrl.pixel_rate);
+	drm_dbg_dp(ctrl->drm_dev, "link rate=%d\n", ctrl->link->link_params.rate);
 
 	return ret;
 }
@@ -1520,8 +1500,6 @@ static int dp_ctrl_link_maintenance(struct dp_ctrl_private *ctrl)
 	ctrl->link->phy_params.p_level = 0;
 	ctrl->link->phy_params.v_level = 0;
 
-	ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock;
-
 	ret = dp_ctrl_setup_main_link(ctrl, &training_step);
 	if (ret)
 		goto end;
@@ -1535,38 +1513,6 @@ end:
 	return ret;
 }
 
-static int dp_ctrl_on_stream_phy_test_report(struct dp_ctrl *dp_ctrl);
-
-static int dp_ctrl_process_phy_test_request(struct dp_ctrl_private *ctrl)
-{
-	int ret = 0;
-
-	if (!ctrl->link->phy_params.phy_test_pattern_sel) {
-		drm_dbg_dp(ctrl->drm_dev,
-			"no test pattern selected by sink\n");
-		return ret;
-	}
-
-	/*
-	 * The global reset will need DP link related clocks to be
-	 * running. Add the global reset just before disabling the
-	 * link clocks and core clocks.
-	 */
-	ret = dp_ctrl_off(&ctrl->dp_ctrl);
-	if (ret) {
-		DRM_ERROR("failed to disable DP controller\n");
-		return ret;
-	}
-
-	ret = dp_ctrl_on_link(&ctrl->dp_ctrl);
-	if (!ret)
-		ret = dp_ctrl_on_stream_phy_test_report(&ctrl->dp_ctrl);
-	else
-		DRM_ERROR("failed to enable DP link controller\n");
-
-	return ret;
-}
-
 static bool dp_ctrl_send_phy_test_pattern(struct dp_ctrl_private *ctrl)
 {
 	bool success = false;
@@ -1619,6 +1565,48 @@ static bool dp_ctrl_send_phy_test_pattern(struct dp_ctrl_private *ctrl)
 	return success;
 }
 
+static int dp_ctrl_process_phy_test_request(struct dp_ctrl_private *ctrl)
+{
+	int ret;
+	unsigned long pixel_rate;
+
+	if (!ctrl->link->phy_params.phy_test_pattern_sel) {
+		drm_dbg_dp(ctrl->drm_dev,
+			"no test pattern selected by sink\n");
+		return 0;
+	}
+
+	/*
+	 * The global reset will need DP link related clocks to be
+	 * running. Add the global reset just before disabling the
+	 * link clocks and core clocks.
+	 */
+	ret = dp_ctrl_off(&ctrl->dp_ctrl);
+	if (ret) {
+		DRM_ERROR("failed to disable DP controller\n");
+		return ret;
+	}
+
+	ret = dp_ctrl_on_link(&ctrl->dp_ctrl);
+	if (ret) {
+		DRM_ERROR("failed to enable DP link controller\n");
+		return ret;
+	}
+
+	pixel_rate = ctrl->panel->dp_mode.drm_mode.clock;
+	dp_ctrl_set_clock_rate(ctrl, DP_STREAM_PM, "stream_pixel", pixel_rate * 1000);
+
+	ret = dp_power_clk_enable(ctrl->power, DP_STREAM_PM, true);
+	if (ret) {
+		DRM_ERROR("Failed to start pixel clocks. ret=%d\n", ret);
+		return ret;
+	}
+
+	dp_ctrl_send_phy_test_pattern(ctrl);
+
+	return 0;
+}
+
 void dp_ctrl_handle_sink_request(struct dp_ctrl *dp_ctrl)
 {
 	struct dp_ctrl_private *ctrl;
@@ -1689,11 +1677,12 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl)
 {
 	int rc = 0;
 	struct dp_ctrl_private *ctrl;
-	u32 rate = 0;
+	u32 rate;
 	int link_train_max_retries = 5;
 	u32 const phy_cts_pixel_clk_khz = 148500;
 	u8 link_status[DP_LINK_STATUS_SIZE];
 	unsigned int training_step;
+	unsigned long pixel_rate;
 
 	if (!dp_ctrl)
 		return -EINVAL;
@@ -1701,25 +1690,24 @@ int dp_ctrl_on_link(struct dp_ctrl *dp_ctrl)
 	ctrl = container_of(dp_ctrl, struct dp_ctrl_private, dp_ctrl);
 
 	rate = ctrl->panel->link_info.rate;
+	pixel_rate = ctrl->panel->dp_mode.drm_mode.clock;
 
 	dp_power_clk_enable(ctrl->power, DP_CORE_PM, true);
 
 	if (ctrl->link->sink_request & DP_TEST_LINK_PHY_TEST_PATTERN) {
 		drm_dbg_dp(ctrl->drm_dev,
 				"using phy test link parameters\n");
-		if (!ctrl->panel->dp_mode.drm_mode.clock)
-			ctrl->dp_ctrl.pixel_rate = phy_cts_pixel_clk_khz;
+		if (!pixel_rate)
+			pixel_rate = phy_cts_pixel_clk_khz;
 	} else {
 		ctrl->link->link_params.rate = rate;
 		ctrl->link->link_params.num_lanes =
 			ctrl->panel->link_info.num_lanes;
-		ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock;
 	}
 
-	drm_dbg_dp(ctrl->drm_dev, "rate=%d, num_lanes=%d, pixel_rate=%d\n",
+	drm_dbg_dp(ctrl->drm_dev, "rate=%d, num_lanes=%d, pixel_rate=%lu\n",
 		ctrl->link->link_params.rate, ctrl->link->link_params.num_lanes,
-		ctrl->dp_ctrl.pixel_rate);
-
+		pixel_rate);
 
 	rc = dp_ctrl_enable_mainlink_clocks(ctrl);
 	if (rc)
@@ -1816,31 +1804,12 @@ static int dp_ctrl_link_retrain(struct dp_ctrl_private *ctrl)
 	return dp_ctrl_setup_main_link(ctrl, &training_step);
 }
 
-static int dp_ctrl_on_stream_phy_test_report(struct dp_ctrl *dp_ctrl)
-{
-	int ret;
-	struct dp_ctrl_private *ctrl;
-
-	ctrl = container_of(dp_ctrl, struct dp_ctrl_private, dp_ctrl);
-
-	ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock;
-
-	ret = dp_ctrl_enable_stream_clocks(ctrl);
-	if (ret) {
-		DRM_ERROR("Failed to start pixel clocks. ret=%d\n", ret);
-		return ret;
-	}
-
-	dp_ctrl_send_phy_test_pattern(ctrl);
-
-	return 0;
-}
-
 int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl, bool force_link_train)
 {
 	int ret = 0;
 	bool mainlink_ready = false;
 	struct dp_ctrl_private *ctrl;
+	unsigned long pixel_rate;
 	unsigned long pixel_rate_orig;
 
 	if (!dp_ctrl)
@@ -1848,15 +1817,14 @@ int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl, bool force_link_train)
 
 	ctrl = container_of(dp_ctrl, struct dp_ctrl_private, dp_ctrl);
 
-	ctrl->dp_ctrl.pixel_rate = ctrl->panel->dp_mode.drm_mode.clock;
+	pixel_rate = pixel_rate_orig = ctrl->panel->dp_mode.drm_mode.clock;
 
-	pixel_rate_orig = ctrl->dp_ctrl.pixel_rate;
 	if (dp_ctrl->wide_bus_en)
-		ctrl->dp_ctrl.pixel_rate >>= 1;
+		pixel_rate >>= 1;
 
-	drm_dbg_dp(ctrl->drm_dev, "rate=%d, num_lanes=%d, pixel_rate=%d\n",
+	drm_dbg_dp(ctrl->drm_dev, "rate=%d, num_lanes=%d, pixel_rate=%lu\n",
 		ctrl->link->link_params.rate,
-		ctrl->link->link_params.num_lanes, ctrl->dp_ctrl.pixel_rate);
+		ctrl->link->link_params.num_lanes, pixel_rate);
 
 	if (!dp_power_clk_status(ctrl->power, DP_CTRL_PM)) { /* link clk is off */
 		ret = dp_ctrl_enable_mainlink_clocks(ctrl);
@@ -1866,9 +1834,11 @@ int dp_ctrl_on_stream(struct dp_ctrl *dp_ctrl, bool force_link_train)
 		}
 	}
 
-	ret = dp_ctrl_enable_stream_clocks(ctrl);
+	dp_ctrl_set_clock_rate(ctrl, DP_STREAM_PM, "stream_pixel", pixel_rate * 1000);
+
+	ret = dp_power_clk_enable(ctrl->power, DP_STREAM_PM, true);
 	if (ret) {
-		DRM_ERROR("Failed to start pixel clocks. ret=%d\n", ret);
+		DRM_ERROR("Unable to start pixel clocks. ret=%d\n", ret);
 		goto end;
 	}
 
diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.h b/drivers/gpu/drm/msm/dp/dp_ctrl.h
index b563e2e3bfe5..9f29734af81c 100644
--- a/drivers/gpu/drm/msm/dp/dp_ctrl.h
+++ b/drivers/gpu/drm/msm/dp/dp_ctrl.h
@@ -16,7 +16,6 @@
 struct dp_ctrl {
 	bool orientation;
 	atomic_t aborted;
-	u32 pixel_rate;
 	bool wide_bus_en;
 };
 
diff --git a/drivers/gpu/drm/msm/dp/dp_link.c b/drivers/gpu/drm/msm/dp/dp_link.c
index 36f0af02749f..36bb6191d2f0 100644
--- a/drivers/gpu/drm/msm/dp/dp_link.c
+++ b/drivers/gpu/drm/msm/dp/dp_link.c
@@ -786,7 +786,7 @@ static int dp_link_process_link_training_request(struct dp_link_private *link)
 			link->request.test_lane_count);
 
 	link->dp_link.link_params.num_lanes = link->request.test_lane_count;
-	link->dp_link.link_params.rate = 
+	link->dp_link.link_params.rate =
 		drm_dp_bw_code_to_link_rate(link->request.test_link_rate);
 
 	return 0;
@@ -965,8 +965,7 @@ static int dp_link_process_link_status_update(struct dp_link_private *link)
 	if (channel_eq_done && clock_recovery_done)
 		return -EINVAL;
 
-
-       return 0;
+	return 0;
 }
 
 /**
diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c
index 1625328fa430..39bbabb5daf6 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.c
+++ b/drivers/gpu/drm/msm/dsi/dsi.c
@@ -6,14 +6,6 @@
 #include "dsi.h"
 #include "dsi_cfg.h"
 
-struct drm_encoder *msm_dsi_get_encoder(struct msm_dsi *msm_dsi)
-{
-	if (!msm_dsi || !msm_dsi_device_connected(msm_dsi))
-		return NULL;
-
-	return msm_dsi->encoder;
-}
-
 bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi)
 {
 	unsigned long host_flags = msm_dsi_host_get_mode_flags(msm_dsi->host);
@@ -21,7 +13,7 @@ bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi)
 	return !(host_flags & MIPI_DSI_MODE_VIDEO);
 }
 
-struct msm_display_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi)
+struct drm_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi)
 {
 	return msm_dsi_host_get_dsc_config(msm_dsi->host);
 }
@@ -220,7 +212,6 @@ int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev,
 			 struct drm_encoder *encoder)
 {
 	struct msm_drm_private *priv;
-	struct drm_bridge *ext_bridge;
 	int ret;
 
 	if (WARN_ON(!encoder) || WARN_ON(!msm_dsi) || WARN_ON(!dev))
@@ -254,26 +245,10 @@ int msm_dsi_modeset_init(struct msm_dsi *msm_dsi, struct drm_device *dev,
 		goto fail;
 	}
 
-	/*
-	 * check if the dsi encoder output is connected to a panel or an
-	 * external bridge. We create a connector only if we're connected to a
-	 * drm_panel device. When we're connected to an external bridge, we
-	 * assume that the drm_bridge driver will create the connector itself.
-	 */
-	ext_bridge = msm_dsi_host_get_bridge(msm_dsi->host);
-
-	if (ext_bridge)
-		msm_dsi->connector =
-			msm_dsi_manager_ext_bridge_init(msm_dsi->id);
-	else
-		msm_dsi->connector =
-			msm_dsi_manager_connector_init(msm_dsi->id);
-
-	if (IS_ERR(msm_dsi->connector)) {
-		ret = PTR_ERR(msm_dsi->connector);
+	ret = msm_dsi_manager_ext_bridge_init(msm_dsi->id);
+	if (ret) {
 		DRM_DEV_ERROR(dev->dev,
 			"failed to create dsi connector: %d\n", ret);
-		msm_dsi->connector = NULL;
 		goto fail;
 	}
 
@@ -287,12 +262,6 @@ fail:
 		msm_dsi->bridge = NULL;
 	}
 
-	/* don't destroy connector if we didn't make it */
-	if (msm_dsi->connector && !msm_dsi->external_bridge)
-		msm_dsi->connector->funcs->destroy(msm_dsi->connector);
-
-	msm_dsi->connector = NULL;
-
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi.h b/drivers/gpu/drm/msm/dsi/dsi.h
index 580a1e6358bf..2a96b4fe7839 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.h
+++ b/drivers/gpu/drm/msm/dsi/dsi.h
@@ -12,7 +12,6 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_mipi_dsi.h>
-#include <drm/drm_panel.h>
 
 #include "msm_drv.h"
 #include "disp/msm_disp_snapshot.h"
@@ -30,27 +29,12 @@ enum msm_dsi_phy_usecase {
 	MSM_DSI_PHY_SLAVE,
 };
 
-#define DSI_DEV_REGULATOR_MAX	8
 #define DSI_BUS_CLK_MAX		4
 
-/* Regulators for DSI devices */
-struct dsi_reg_entry {
-	char name[32];
-	int enable_load;
-	int disable_load;
-};
-
-struct dsi_reg_config {
-	int num;
-	struct dsi_reg_entry regs[DSI_DEV_REGULATOR_MAX];
-};
-
 struct msm_dsi {
 	struct drm_device *dev;
 	struct platform_device *pdev;
 
-	/* connector managed by us when we're connected to a drm_panel */
-	struct drm_connector *connector;
 	/* internal dsi bridge attached to MDP interface */
 	struct drm_bridge *bridge;
 
@@ -58,10 +42,8 @@ struct msm_dsi {
 	struct msm_dsi_phy *phy;
 
 	/*
-	 * panel/external_bridge connected to dsi bridge output, only one of the
-	 * two can be valid at a time
+	 * external_bridge connected to dsi bridge output
 	 */
-	struct drm_panel *panel;
 	struct drm_bridge *external_bridge;
 
 	struct device *phy_dev;
@@ -76,8 +58,7 @@ struct msm_dsi {
 /* dsi manager */
 struct drm_bridge *msm_dsi_manager_bridge_init(u8 id);
 void msm_dsi_manager_bridge_destroy(struct drm_bridge *bridge);
-struct drm_connector *msm_dsi_manager_connector_init(u8 id);
-struct drm_connector *msm_dsi_manager_ext_bridge_init(u8 id);
+int msm_dsi_manager_ext_bridge_init(u8 id);
 int msm_dsi_manager_cmd_xfer(int id, const struct mipi_dsi_msg *msg);
 bool msm_dsi_manager_cmd_xfer_trigger(int id, u32 dma_base, u32 len);
 int msm_dsi_manager_register(struct msm_dsi *msm_dsi);
@@ -87,11 +68,9 @@ void msm_dsi_manager_tpg_enable(void);
 /* msm dsi */
 static inline bool msm_dsi_device_connected(struct msm_dsi *msm_dsi)
 {
-	return msm_dsi->panel || msm_dsi->external_bridge;
+	return msm_dsi->external_bridge;
 }
 
-struct drm_encoder *msm_dsi_get_encoder(struct msm_dsi *msm_dsi);
-
 /* dsi host */
 struct msm_dsi_host;
 int msm_dsi_host_xfer_prepare(struct mipi_dsi_host *host,
@@ -116,9 +95,7 @@ int msm_dsi_host_set_display_mode(struct mipi_dsi_host *host,
 				  const struct drm_display_mode *mode);
 enum drm_mode_status msm_dsi_host_check_dsc(struct mipi_dsi_host *host,
 					    const struct drm_display_mode *mode);
-struct drm_panel *msm_dsi_host_get_panel(struct mipi_dsi_host *host);
 unsigned long msm_dsi_host_get_mode_flags(struct mipi_dsi_host *host);
-struct drm_bridge *msm_dsi_host_get_bridge(struct mipi_dsi_host *host);
 int msm_dsi_host_register(struct mipi_dsi_host *host);
 void msm_dsi_host_unregister(struct mipi_dsi_host *host);
 void msm_dsi_host_set_phy_mode(struct mipi_dsi_host *host,
@@ -154,7 +131,7 @@ int dsi_calc_clk_rate_v2(struct msm_dsi_host *msm_host, bool is_bonded_dsi);
 int dsi_calc_clk_rate_6g(struct msm_dsi_host *msm_host, bool is_bonded_dsi);
 void msm_dsi_host_snapshot(struct msm_disp_state *disp_state, struct mipi_dsi_host *host);
 void msm_dsi_host_test_pattern_en(struct mipi_dsi_host *host);
-struct msm_display_dsc_config *msm_dsi_host_get_dsc_config(struct mipi_dsi_host *host);
+struct drm_dsc_config *msm_dsi_host_get_dsc_config(struct mipi_dsi_host *host);
 
 /* dsi phy */
 struct msm_dsi_phy;
diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.c b/drivers/gpu/drm/msm/dsi/dsi_cfg.c
index 72c018e26f47..7e97c239ed48 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_cfg.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.c
@@ -9,16 +9,16 @@ static const char * const dsi_v2_bus_clk_names[] = {
 	"core_mmss", "iface", "bus",
 };
 
+static const struct regulator_bulk_data apq8064_dsi_regulators[] = {
+	{ .supply = "vdda", .init_load_uA = 100000 },	/* 1.2 V */
+	{ .supply = "avdd", .init_load_uA = 10000 },	/* 3.0 V */
+	{ .supply = "vddio", .init_load_uA = 100000 },	/* 1.8 V */
+};
+
 static const struct msm_dsi_config apq8064_dsi_cfg = {
 	.io_offset = 0,
-	.reg_cfg = {
-		.num = 3,
-		.regs = {
-			{"vdda", 100000, 100},	/* 1.2 V */
-			{"avdd", 10000, 100},	/* 3.0 V */
-			{"vddio", 100000, 100},	/* 1.8 V */
-		},
-	},
+	.regulator_data = apq8064_dsi_regulators,
+	.num_regulators = ARRAY_SIZE(apq8064_dsi_regulators),
 	.bus_clk_names = dsi_v2_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_v2_bus_clk_names),
 	.io_start = { 0x4700000, 0x5800000 },
@@ -29,16 +29,16 @@ static const char * const dsi_6g_bus_clk_names[] = {
 	"mdp_core", "iface", "bus", "core_mmss",
 };
 
+static const struct regulator_bulk_data msm8974_apq8084_regulators[] = {
+	{ .supply = "vdd", .init_load_uA = 150000 },	/* 3.0 V */
+	{ .supply = "vdda", .init_load_uA = 100000 },	/* 1.2 V */
+	{ .supply = "vddio", .init_load_uA = 100000 },	/* 1.8 V */
+};
+
 static const struct msm_dsi_config msm8974_apq8084_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 3,
-		.regs = {
-			{"vdd", 150000, 100},	/* 3.0 V */
-			{"vdda", 100000, 100},	/* 1.2 V */
-			{"vddio", 100000, 100},	/* 1.8 V */
-		},
-	},
+	.regulator_data = msm8974_apq8084_regulators,
+	.num_regulators = ARRAY_SIZE(msm8974_apq8084_regulators),
 	.bus_clk_names = dsi_6g_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names),
 	.io_start = { 0xfd922800, 0xfd922b00 },
@@ -49,15 +49,15 @@ static const char * const dsi_8916_bus_clk_names[] = {
 	"mdp_core", "iface", "bus",
 };
 
+static const struct regulator_bulk_data msm8916_dsi_regulators[] = {
+	{ .supply = "vdda", .init_load_uA = 100000 },	/* 1.2 V */
+	{ .supply = "vddio", .init_load_uA = 100000 },	/* 1.8 V */
+};
+
 static const struct msm_dsi_config msm8916_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 2,
-		.regs = {
-			{"vdda", 100000, 100},	/* 1.2 V */
-			{"vddio", 100000, 100},	/* 1.8 V */
-		},
-	},
+	.regulator_data = msm8916_dsi_regulators,
+	.num_regulators = ARRAY_SIZE(msm8916_dsi_regulators),
 	.bus_clk_names = dsi_8916_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_8916_bus_clk_names),
 	.io_start = { 0x1a98000 },
@@ -68,34 +68,34 @@ static const char * const dsi_8976_bus_clk_names[] = {
 	"mdp_core", "iface", "bus",
 };
 
+static const struct regulator_bulk_data msm8976_dsi_regulators[] = {
+	{ .supply = "vdda", .init_load_uA = 100000 },	/* 1.2 V */
+	{ .supply = "vddio", .init_load_uA = 100000 },	/* 1.8 V */
+};
+
 static const struct msm_dsi_config msm8976_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 2,
-		.regs = {
-			{"vdda", 100000, 100},	/* 1.2 V */
-			{"vddio", 100000, 100},	/* 1.8 V */
-		},
-	},
+	.regulator_data = msm8976_dsi_regulators,
+	.num_regulators = ARRAY_SIZE(msm8976_dsi_regulators),
 	.bus_clk_names = dsi_8976_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_8976_bus_clk_names),
 	.io_start = { 0x1a94000, 0x1a96000 },
 	.num_dsi = 2,
 };
 
+static const struct regulator_bulk_data msm8994_dsi_regulators[] = {
+	{ .supply = "vdda", .init_load_uA = 100000 },	/* 1.25 V */
+	{ .supply = "vddio", .init_load_uA = 100000 },	/* 1.8 V */
+	{ .supply = "vcca", .init_load_uA = 10000 },	/* 1.0 V */
+	{ .supply = "vdd", .init_load_uA = 100000 },	/* 1.8 V */
+	{ .supply = "lab_reg", .init_load_uA = -1 },
+	{ .supply = "ibb_reg", .init_load_uA = -1 },
+};
+
 static const struct msm_dsi_config msm8994_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 6,
-		.regs = {
-			{"vdda", 100000, 100},	/* 1.25 V */
-			{"vddio", 100000, 100},	/* 1.8 V */
-			{"vcca", 10000, 100},	/* 1.0 V */
-			{"vdd", 100000, 100},	/* 1.8 V */
-			{"lab_reg", -1, -1},
-			{"ibb_reg", -1, -1},
-		},
-	},
+	.regulator_data = msm8994_dsi_regulators,
+	.num_regulators = ARRAY_SIZE(msm8994_dsi_regulators),
 	.bus_clk_names = dsi_6g_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names),
 	.io_start = { 0xfd998000, 0xfd9a0000 },
@@ -106,16 +106,16 @@ static const char * const dsi_8996_bus_clk_names[] = {
 	"mdp_core", "iface", "bus", "core_mmss",
 };
 
+static const struct regulator_bulk_data msm8996_dsi_regulators[] = {
+	{ .supply = "vdda", .init_load_uA = 18160 },	/* 1.25 V */
+	{ .supply = "vcca", .init_load_uA = 17000 },	/* 0.925 V */
+	{ .supply = "vddio", .init_load_uA = 100000 },	/* 1.8 V */
+};
+
 static const struct msm_dsi_config msm8996_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 3,
-		.regs = {
-			{"vdda", 18160, 1 },	/* 1.25 V */
-			{"vcca", 17000, 32 },	/* 0.925 V */
-			{"vddio", 100000, 100 },/* 1.8 V */
-		},
-	},
+	.regulator_data = msm8996_dsi_regulators,
+	.num_regulators = ARRAY_SIZE(msm8996_dsi_regulators),
 	.bus_clk_names = dsi_8996_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_8996_bus_clk_names),
 	.io_start = { 0x994000, 0x996000 },
@@ -126,15 +126,15 @@ static const char * const dsi_msm8998_bus_clk_names[] = {
 	"iface", "bus", "core",
 };
 
+static const struct regulator_bulk_data msm8998_dsi_regulators[] = {
+	{ .supply = "vdd", .init_load_uA = 367000 },	/* 0.9 V */
+	{ .supply = "vdda", .init_load_uA = 62800 },	/* 1.2 V */
+};
+
 static const struct msm_dsi_config msm8998_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 2,
-		.regs = {
-			{"vdd", 367000, 16 },	/* 0.9 V */
-			{"vdda", 62800, 2 },	/* 1.2 V */
-		},
-	},
+	.regulator_data = msm8998_dsi_regulators,
+	.num_regulators = ARRAY_SIZE(msm8998_dsi_regulators),
 	.bus_clk_names = dsi_msm8998_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_msm8998_bus_clk_names),
 	.io_start = { 0xc994000, 0xc996000 },
@@ -145,14 +145,14 @@ static const char * const dsi_sdm660_bus_clk_names[] = {
 	"iface", "bus", "core", "core_mmss",
 };
 
+static const struct regulator_bulk_data sdm660_dsi_regulators[] = {
+	{ .supply = "vdda", .init_load_uA = 12560 },	/* 1.2 V */
+};
+
 static const struct msm_dsi_config sdm660_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vdda", 12560, 4 },	/* 1.2 V */
-		},
-	},
+	.regulator_data = sdm660_dsi_regulators,
+	.num_regulators = ARRAY_SIZE(sdm660_dsi_regulators),
 	.bus_clk_names = dsi_sdm660_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_sdm660_bus_clk_names),
 	.io_start = { 0xc994000, 0xc996000 },
@@ -167,28 +167,28 @@ static const char * const dsi_sc7180_bus_clk_names[] = {
 	"iface", "bus",
 };
 
+static const struct regulator_bulk_data sdm845_dsi_regulators[] = {
+	{ .supply = "vdda", .init_load_uA = 21800 },	/* 1.2 V */
+};
+
 static const struct msm_dsi_config sdm845_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vdda", 21800, 4 },	/* 1.2 V */
-		},
-	},
+	.regulator_data = sdm845_dsi_regulators,
+	.num_regulators = ARRAY_SIZE(sdm845_dsi_regulators),
 	.bus_clk_names = dsi_sdm845_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_sdm845_bus_clk_names),
 	.io_start = { 0xae94000, 0xae96000 },
 	.num_dsi = 2,
 };
 
+static const struct regulator_bulk_data sc7180_dsi_regulators[] = {
+	{ .supply = "vdda", .init_load_uA = 21800 },	/* 1.2 V */
+};
+
 static const struct msm_dsi_config sc7180_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vdda", 21800, 4 },	/* 1.2 V */
-		},
-	},
+	.regulator_data = sc7180_dsi_regulators,
+	.num_regulators = ARRAY_SIZE(sc7180_dsi_regulators),
 	.bus_clk_names = dsi_sc7180_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_sc7180_bus_clk_names),
 	.io_start = { 0xae94000 },
@@ -199,14 +199,14 @@ static const char * const dsi_sc7280_bus_clk_names[] = {
 	"iface", "bus",
 };
 
+static const struct regulator_bulk_data sc7280_dsi_regulators[] = {
+	{ .supply = "vdda", .init_load_uA = 8350 },	/* 1.2 V */
+};
+
 static const struct msm_dsi_config sc7280_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vdda", 8350, 0 },	/* 1.2 V */
-		},
-	},
+	.regulator_data = sc7280_dsi_regulators,
+	.num_regulators = ARRAY_SIZE(sc7280_dsi_regulators),
 	.bus_clk_names = dsi_sc7280_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_sc7280_bus_clk_names),
 	.io_start = { 0xae94000 },
@@ -217,14 +217,14 @@ static const char * const dsi_qcm2290_bus_clk_names[] = {
 	"iface", "bus",
 };
 
+static const struct regulator_bulk_data qcm2290_dsi_cfg_regulators[] = {
+	{ .supply = "vdda", .init_load_uA = 21800 },	/* 1.2 V */
+};
+
 static const struct msm_dsi_config qcm2290_dsi_cfg = {
 	.io_offset = DSI_6G_REG_SHIFT,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vdda", 21800, 4 },	/* 1.2 V */
-		},
-	},
+	.regulator_data = qcm2290_dsi_cfg_regulators,
+	.num_regulators = ARRAY_SIZE(qcm2290_dsi_cfg_regulators),
 	.bus_clk_names = dsi_qcm2290_bus_clk_names,
 	.num_bus_clks = ARRAY_SIZE(dsi_qcm2290_bus_clk_names),
 	.io_start = { 0x5e94000 },
diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.h b/drivers/gpu/drm/msm/dsi/dsi_cfg.h
index fe54a999968b..8f04e685a74e 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_cfg.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.h
@@ -32,7 +32,8 @@
 
 struct msm_dsi_config {
 	u32 io_offset;
-	struct dsi_reg_config reg_cfg;
+	const struct regulator_bulk_data *regulator_data;
+	int num_regulators;
 	const char * const *bus_clk_names;
 	const int num_bus_clks;
 	const resource_size_t io_start[DSI_MAX];
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index a34078497af1..7fbf391c024f 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -33,7 +33,7 @@
 
 #define DSI_RESET_TOGGLE_DELAY_MS 20
 
-static int dsi_populate_dsc_params(struct msm_display_dsc_config *dsc);
+static int dsi_populate_dsc_params(struct drm_dsc_config *dsc);
 
 static int dsi_get_version(const void __iomem *base, u32 *major, u32 *minor)
 {
@@ -108,7 +108,7 @@ struct msm_dsi_host {
 
 	void __iomem *ctrl_base;
 	phys_addr_t ctrl_size;
-	struct regulator_bulk_data supplies[DSI_DEV_REGULATOR_MAX];
+	struct regulator_bulk_data *supplies;
 
 	int num_bus_clks;
 	struct clk_bulk_data bus_clks[DSI_BUS_CLK_MAX];
@@ -144,7 +144,6 @@ struct msm_dsi_host {
 
 	u32 err_work_state;
 	struct work_struct err_work;
-	struct work_struct hpd_work;
 	struct workqueue_struct *workqueue;
 
 	/* DSI 6G TX buffer*/
@@ -161,10 +160,9 @@ struct msm_dsi_host {
 	struct regmap *sfpb;
 
 	struct drm_display_mode *mode;
-	struct msm_display_dsc_config *dsc;
+	struct drm_dsc_config *dsc;
 
 	/* connected device info */
-	struct device_node *device_node;
 	unsigned int channel;
 	unsigned int lanes;
 	enum mipi_dsi_pixel_format format;
@@ -205,9 +203,6 @@ static inline void dsi_write(struct msm_dsi_host *msm_host, u32 reg, u32 data)
 	msm_writel(data, msm_host->ctrl_base + reg);
 }
 
-static int dsi_host_regulator_enable(struct msm_dsi_host *msm_host);
-static void dsi_host_regulator_disable(struct msm_dsi_host *msm_host);
-
 static const struct msm_dsi_cfg_handler *dsi_get_config(
 						struct msm_dsi_host *msm_host)
 {
@@ -258,76 +253,6 @@ static inline struct msm_dsi_host *to_msm_dsi_host(struct mipi_dsi_host *host)
 	return container_of(host, struct msm_dsi_host, base);
 }
 
-static void dsi_host_regulator_disable(struct msm_dsi_host *msm_host)
-{
-	struct regulator_bulk_data *s = msm_host->supplies;
-	const struct dsi_reg_entry *regs = msm_host->cfg_hnd->cfg->reg_cfg.regs;
-	int num = msm_host->cfg_hnd->cfg->reg_cfg.num;
-	int i;
-
-	DBG("");
-	for (i = num - 1; i >= 0; i--)
-		if (regs[i].disable_load >= 0)
-			regulator_set_load(s[i].consumer,
-					   regs[i].disable_load);
-
-	regulator_bulk_disable(num, s);
-}
-
-static int dsi_host_regulator_enable(struct msm_dsi_host *msm_host)
-{
-	struct regulator_bulk_data *s = msm_host->supplies;
-	const struct dsi_reg_entry *regs = msm_host->cfg_hnd->cfg->reg_cfg.regs;
-	int num = msm_host->cfg_hnd->cfg->reg_cfg.num;
-	int ret, i;
-
-	DBG("");
-	for (i = 0; i < num; i++) {
-		if (regs[i].enable_load >= 0) {
-			ret = regulator_set_load(s[i].consumer,
-						 regs[i].enable_load);
-			if (ret < 0) {
-				pr_err("regulator %d set op mode failed, %d\n",
-					i, ret);
-				goto fail;
-			}
-		}
-	}
-
-	ret = regulator_bulk_enable(num, s);
-	if (ret < 0) {
-		pr_err("regulator enable failed, %d\n", ret);
-		goto fail;
-	}
-
-	return 0;
-
-fail:
-	for (i--; i >= 0; i--)
-		regulator_set_load(s[i].consumer, regs[i].disable_load);
-	return ret;
-}
-
-static int dsi_regulator_init(struct msm_dsi_host *msm_host)
-{
-	struct regulator_bulk_data *s = msm_host->supplies;
-	const struct dsi_reg_entry *regs = msm_host->cfg_hnd->cfg->reg_cfg.regs;
-	int num = msm_host->cfg_hnd->cfg->reg_cfg.num;
-	int i, ret;
-
-	for (i = 0; i < num; i++)
-		s[i].supply = regs[i].name;
-
-	ret = devm_regulator_bulk_get(&msm_host->pdev->dev, num, s);
-	if (ret < 0) {
-		pr_err("%s: failed to init regulator, ret=%d\n",
-						__func__, ret);
-		return ret;
-	}
-
-	return 0;
-}
-
 int dsi_clk_init_v2(struct msm_dsi_host *msm_host)
 {
 	struct platform_device *pdev = msm_host->pdev;
@@ -916,7 +841,7 @@ static void dsi_ctrl_config(struct msm_dsi_host *msm_host, bool enable,
 
 static void dsi_update_dsc_timing(struct msm_dsi_host *msm_host, bool is_cmd_mode, u32 hdisplay)
 {
-	struct msm_display_dsc_config *dsc = msm_host->dsc;
+	struct drm_dsc_config *dsc = msm_host->dsc;
 	u32 reg, intf_width, reg_ctrl, reg_ctrl2;
 	u32 slice_per_intf, total_bytes_per_intf;
 	u32 pkt_per_line;
@@ -927,24 +852,24 @@ static void dsi_update_dsc_timing(struct msm_dsi_host *msm_host, bool is_cmd_mod
 	 * compress mode registers
 	 */
 	intf_width = hdisplay;
-	slice_per_intf = DIV_ROUND_UP(intf_width, dsc->drm->slice_width);
+	slice_per_intf = DIV_ROUND_UP(intf_width, dsc->slice_width);
 
 	/* If slice_per_pkt is greater than slice_per_intf
 	 * then default to 1. This can happen during partial
 	 * update.
 	 */
-	if (slice_per_intf > dsc->drm->slice_count)
-		dsc->drm->slice_count = 1;
+	if (slice_per_intf > dsc->slice_count)
+		dsc->slice_count = 1;
 
-	slice_per_intf = DIV_ROUND_UP(hdisplay, dsc->drm->slice_width);
-	bytes_in_slice = DIV_ROUND_UP(dsc->drm->slice_width * dsc->drm->bits_per_pixel, 8);
+	slice_per_intf = DIV_ROUND_UP(hdisplay, dsc->slice_width);
+	bytes_in_slice = DIV_ROUND_UP(dsc->slice_width * dsc->bits_per_pixel, 8);
 
-	dsc->drm->slice_chunk_size = bytes_in_slice;
+	dsc->slice_chunk_size = bytes_in_slice;
 
 	total_bytes_per_intf = bytes_in_slice * slice_per_intf;
 
 	eol_byte_num = total_bytes_per_intf % 3;
-	pkt_per_line = slice_per_intf / dsc->drm->slice_count;
+	pkt_per_line = slice_per_intf / dsc->slice_count;
 
 	if (is_cmd_mode) /* packet data type */
 		reg = DSI_COMMAND_COMPRESSION_MODE_CTRL_STREAM0_DATATYPE(MIPI_DSI_DCS_LONG_WRITE);
@@ -1009,7 +934,7 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi)
 	}
 
 	if (msm_host->dsc) {
-		struct msm_display_dsc_config *dsc = msm_host->dsc;
+		struct drm_dsc_config *dsc = msm_host->dsc;
 
 		/* update dsc params with timing params */
 		if (!dsc || !mode->hdisplay || !mode->vdisplay) {
@@ -1018,9 +943,9 @@ static void dsi_timing_setup(struct msm_dsi_host *msm_host, bool is_bonded_dsi)
 			return;
 		}
 
-		dsc->drm->pic_width = mode->hdisplay;
-		dsc->drm->pic_height = mode->vdisplay;
-		DBG("Mode %dx%d\n", dsc->drm->pic_width, dsc->drm->pic_height);
+		dsc->pic_width = mode->hdisplay;
+		dsc->pic_height = mode->vdisplay;
+		DBG("Mode %dx%d\n", dsc->pic_width, dsc->pic_height);
 
 		/* we do the calculations for dsc parameters here so that
 		 * panel can use these parameters
@@ -1500,14 +1425,6 @@ static int dsi_cmds2buf_tx(struct msm_dsi_host *msm_host,
 	return len;
 }
 
-static void dsi_hpd_worker(struct work_struct *work)
-{
-	struct msm_dsi_host *msm_host =
-		container_of(work, struct msm_dsi_host, hpd_work);
-
-	drm_helper_hpd_irq_event(msm_host->dev);
-}
-
 static void dsi_err_worker(struct work_struct *work)
 {
 	struct msm_dsi_host *msm_host =
@@ -1686,6 +1603,8 @@ static int dsi_host_attach(struct mipi_dsi_host *host,
 	msm_host->lanes = dsi->lanes;
 	msm_host->format = dsi->format;
 	msm_host->mode_flags = dsi->mode_flags;
+	if (dsi->dsc)
+		msm_host->dsc = dsi->dsc;
 
 	/* Some gpios defined in panel DT need to be controlled by host */
 	ret = dsi_host_init_panel_gpios(msm_host, &dsi->dev);
@@ -1697,8 +1616,6 @@ static int dsi_host_attach(struct mipi_dsi_host *host,
 		return ret;
 
 	DBG("id=%d", msm_host->id);
-	if (msm_host->dev)
-		queue_work(msm_host->workqueue, &msm_host->hpd_work);
 
 	return 0;
 }
@@ -1710,11 +1627,7 @@ static int dsi_host_detach(struct mipi_dsi_host *host,
 
 	dsi_dev_detach(msm_host->pdev);
 
-	msm_host->device_node = NULL;
-
 	DBG("id=%d", msm_host->id);
-	if (msm_host->dev)
-		queue_work(msm_host->workqueue, &msm_host->hpd_work);
 
 	return 0;
 }
@@ -1841,7 +1754,7 @@ static char bpg_offset[DSC_NUM_BUF_RANGES] = {
 	2, 0, 0, -2, -4, -6, -8, -8, -8, -10, -10, -12, -12, -12, -12
 };
 
-static int dsi_populate_dsc_params(struct msm_display_dsc_config *dsc)
+static int dsi_populate_dsc_params(struct drm_dsc_config *dsc)
 {
 	int mux_words_size;
 	int groups_per_line, groups_total;
@@ -1854,98 +1767,98 @@ static int dsi_populate_dsc_params(struct msm_display_dsc_config *dsc)
 	int final_value, final_scale;
 	int i;
 
-	dsc->drm->rc_model_size = 8192;
-	dsc->drm->first_line_bpg_offset = 12;
-	dsc->drm->rc_edge_factor = 6;
-	dsc->drm->rc_tgt_offset_high = 3;
-	dsc->drm->rc_tgt_offset_low = 3;
-	dsc->drm->simple_422 = 0;
-	dsc->drm->convert_rgb = 1;
-	dsc->drm->vbr_enable = 0;
+	dsc->rc_model_size = 8192;
+	dsc->first_line_bpg_offset = 12;
+	dsc->rc_edge_factor = 6;
+	dsc->rc_tgt_offset_high = 3;
+	dsc->rc_tgt_offset_low = 3;
+	dsc->simple_422 = 0;
+	dsc->convert_rgb = 1;
+	dsc->vbr_enable = 0;
 
 	/* handle only bpp = bpc = 8 */
 	for (i = 0; i < DSC_NUM_BUF_RANGES - 1 ; i++)
-		dsc->drm->rc_buf_thresh[i] = dsi_dsc_rc_buf_thresh[i];
+		dsc->rc_buf_thresh[i] = dsi_dsc_rc_buf_thresh[i];
 
 	for (i = 0; i < DSC_NUM_BUF_RANGES; i++) {
-		dsc->drm->rc_range_params[i].range_min_qp = min_qp[i];
-		dsc->drm->rc_range_params[i].range_max_qp = max_qp[i];
-		dsc->drm->rc_range_params[i].range_bpg_offset = bpg_offset[i];
+		dsc->rc_range_params[i].range_min_qp = min_qp[i];
+		dsc->rc_range_params[i].range_max_qp = max_qp[i];
+		dsc->rc_range_params[i].range_bpg_offset = bpg_offset[i];
 	}
 
-	dsc->drm->initial_offset = 6144; /* Not bpp 12 */
-	if (dsc->drm->bits_per_pixel != 8)
-		dsc->drm->initial_offset = 2048;	/* bpp = 12 */
+	dsc->initial_offset = 6144; /* Not bpp 12 */
+	if (dsc->bits_per_pixel != 8)
+		dsc->initial_offset = 2048;	/* bpp = 12 */
 
 	mux_words_size = 48;		/* bpc == 8/10 */
-	if (dsc->drm->bits_per_component == 12)
+	if (dsc->bits_per_component == 12)
 		mux_words_size = 64;
 
-	dsc->drm->initial_xmit_delay = 512;
-	dsc->drm->initial_scale_value = 32;
-	dsc->drm->first_line_bpg_offset = 12;
-	dsc->drm->line_buf_depth = dsc->drm->bits_per_component + 1;
+	dsc->initial_xmit_delay = 512;
+	dsc->initial_scale_value = 32;
+	dsc->first_line_bpg_offset = 12;
+	dsc->line_buf_depth = dsc->bits_per_component + 1;
 
 	/* bpc 8 */
-	dsc->drm->flatness_min_qp = 3;
-	dsc->drm->flatness_max_qp = 12;
-	dsc->drm->rc_quant_incr_limit0 = 11;
-	dsc->drm->rc_quant_incr_limit1 = 11;
-	dsc->drm->mux_word_size = DSC_MUX_WORD_SIZE_8_10_BPC;
+	dsc->flatness_min_qp = 3;
+	dsc->flatness_max_qp = 12;
+	dsc->rc_quant_incr_limit0 = 11;
+	dsc->rc_quant_incr_limit1 = 11;
+	dsc->mux_word_size = DSC_MUX_WORD_SIZE_8_10_BPC;
 
 	/* FIXME: need to call drm_dsc_compute_rc_parameters() so that rest of
 	 * params are calculated
 	 */
-	groups_per_line = DIV_ROUND_UP(dsc->drm->slice_width, 3);
-	dsc->drm->slice_chunk_size = dsc->drm->slice_width * dsc->drm->bits_per_pixel / 8;
-	if ((dsc->drm->slice_width * dsc->drm->bits_per_pixel) % 8)
-		dsc->drm->slice_chunk_size++;
+	groups_per_line = DIV_ROUND_UP(dsc->slice_width, 3);
+	dsc->slice_chunk_size = dsc->slice_width * dsc->bits_per_pixel / 8;
+	if ((dsc->slice_width * dsc->bits_per_pixel) % 8)
+		dsc->slice_chunk_size++;
 
 	/* rbs-min */
-	min_rate_buffer_size =  dsc->drm->rc_model_size - dsc->drm->initial_offset +
-				dsc->drm->initial_xmit_delay * dsc->drm->bits_per_pixel +
-				groups_per_line * dsc->drm->first_line_bpg_offset;
+	min_rate_buffer_size =  dsc->rc_model_size - dsc->initial_offset +
+				dsc->initial_xmit_delay * dsc->bits_per_pixel +
+				groups_per_line * dsc->first_line_bpg_offset;
 
-	hrd_delay = DIV_ROUND_UP(min_rate_buffer_size, dsc->drm->bits_per_pixel);
+	hrd_delay = DIV_ROUND_UP(min_rate_buffer_size, dsc->bits_per_pixel);
 
-	dsc->drm->initial_dec_delay = hrd_delay - dsc->drm->initial_xmit_delay;
+	dsc->initial_dec_delay = hrd_delay - dsc->initial_xmit_delay;
 
-	dsc->drm->initial_scale_value = 8 * dsc->drm->rc_model_size /
-				       (dsc->drm->rc_model_size - dsc->drm->initial_offset);
+	dsc->initial_scale_value = 8 * dsc->rc_model_size /
+				       (dsc->rc_model_size - dsc->initial_offset);
 
-	slice_bits = 8 * dsc->drm->slice_chunk_size * dsc->drm->slice_height;
+	slice_bits = 8 * dsc->slice_chunk_size * dsc->slice_height;
 
-	groups_total = groups_per_line * dsc->drm->slice_height;
+	groups_total = groups_per_line * dsc->slice_height;
 
-	data = dsc->drm->first_line_bpg_offset * 2048;
+	data = dsc->first_line_bpg_offset * 2048;
 
-	dsc->drm->nfl_bpg_offset = DIV_ROUND_UP(data, (dsc->drm->slice_height - 1));
+	dsc->nfl_bpg_offset = DIV_ROUND_UP(data, (dsc->slice_height - 1));
 
-	pre_num_extra_mux_bits = 3 * (mux_words_size + (4 * dsc->drm->bits_per_component + 4) - 2);
+	pre_num_extra_mux_bits = 3 * (mux_words_size + (4 * dsc->bits_per_component + 4) - 2);
 
 	num_extra_mux_bits = pre_num_extra_mux_bits - (mux_words_size -
 			     ((slice_bits - pre_num_extra_mux_bits) % mux_words_size));
 
-	data = 2048 * (dsc->drm->rc_model_size - dsc->drm->initial_offset + num_extra_mux_bits);
-	dsc->drm->slice_bpg_offset = DIV_ROUND_UP(data, groups_total);
+	data = 2048 * (dsc->rc_model_size - dsc->initial_offset + num_extra_mux_bits);
+	dsc->slice_bpg_offset = DIV_ROUND_UP(data, groups_total);
 
 	/* bpp * 16 + 0.5 */
-	data = dsc->drm->bits_per_pixel * 16;
+	data = dsc->bits_per_pixel * 16;
 	data *= 2;
 	data++;
 	data /= 2;
 	target_bpp_x16 = data;
 
-	data = (dsc->drm->initial_xmit_delay * target_bpp_x16) / 16;
-	final_value =  dsc->drm->rc_model_size - data + num_extra_mux_bits;
-	dsc->drm->final_offset = final_value;
+	data = (dsc->initial_xmit_delay * target_bpp_x16) / 16;
+	final_value =  dsc->rc_model_size - data + num_extra_mux_bits;
+	dsc->final_offset = final_value;
 
-	final_scale = 8 * dsc->drm->rc_model_size / (dsc->drm->rc_model_size - final_value);
+	final_scale = 8 * dsc->rc_model_size / (dsc->rc_model_size - final_value);
 
-	data = (final_scale - 9) * (dsc->drm->nfl_bpg_offset + dsc->drm->slice_bpg_offset);
-	dsc->drm->scale_increment_interval = (2048 * dsc->drm->final_offset) / data;
+	data = (final_scale - 9) * (dsc->nfl_bpg_offset + dsc->slice_bpg_offset);
+	dsc->scale_increment_interval = (2048 * dsc->final_offset) / data;
 
-	dsc->drm->scale_decrement_interval = groups_per_line / (dsc->drm->initial_scale_value - 8);
+	dsc->scale_decrement_interval = groups_per_line / (dsc->initial_scale_value - 8);
 
 	return 0;
 }
@@ -1954,7 +1867,7 @@ static int dsi_host_parse_dt(struct msm_dsi_host *msm_host)
 {
 	struct device *dev = &msm_host->pdev->dev;
 	struct device_node *np = dev->of_node;
-	struct device_node *endpoint, *device_node;
+	struct device_node *endpoint;
 	int ret = 0;
 
 	/*
@@ -1977,16 +1890,6 @@ static int dsi_host_parse_dt(struct msm_dsi_host *msm_host)
 		goto err;
 	}
 
-	/* Get panel node from the output port's endpoint data */
-	device_node = of_graph_get_remote_node(np, 1, 0);
-	if (!device_node) {
-		DRM_DEV_DEBUG(dev, "%s: no valid device\n", __func__);
-		ret = -ENODEV;
-		goto err;
-	}
-
-	msm_host->device_node = device_node;
-
 	if (of_property_read_bool(np, "syscon-sfpb")) {
 		msm_host->sfpb = syscon_regmap_lookup_by_phandle(np,
 					"syscon-sfpb");
@@ -1997,8 +1900,6 @@ static int dsi_host_parse_dt(struct msm_dsi_host *msm_host)
 		}
 	}
 
-	of_node_put(device_node);
-
 err:
 	of_node_put(endpoint);
 
@@ -2028,6 +1929,7 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 {
 	struct msm_dsi_host *msm_host = NULL;
 	struct platform_device *pdev = msm_dsi->pdev;
+	const struct msm_dsi_config *cfg;
 	int ret;
 
 	msm_host = devm_kzalloc(&pdev->dev, sizeof(*msm_host), GFP_KERNEL);
@@ -2060,6 +1962,7 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 		pr_err("%s: get config failed\n", __func__);
 		goto fail;
 	}
+	cfg = msm_host->cfg_hnd->cfg;
 
 	msm_host->id = dsi_host_get_id(msm_host);
 	if (msm_host->id < 0) {
@@ -2069,13 +1972,13 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 	}
 
 	/* fixup base address by io offset */
-	msm_host->ctrl_base += msm_host->cfg_hnd->cfg->io_offset;
+	msm_host->ctrl_base += cfg->io_offset;
 
-	ret = dsi_regulator_init(msm_host);
-	if (ret) {
-		pr_err("%s: regulator init failed\n", __func__);
+	ret = devm_regulator_bulk_get_const(&pdev->dev, cfg->num_regulators,
+					    cfg->regulator_data,
+					    &msm_host->supplies);
+	if (ret)
 		goto fail;
-	}
 
 	ret = dsi_clk_init(msm_host);
 	if (ret) {
@@ -2126,7 +2029,6 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 	/* setup workqueue */
 	msm_host->workqueue = alloc_ordered_workqueue("dsi_drm_work", 0);
 	INIT_WORK(&msm_host->err_work, dsi_err_worker);
-	INIT_WORK(&msm_host->hpd_work, dsi_hpd_worker);
 
 	msm_dsi->id = msm_host->id;
 
@@ -2159,23 +2061,9 @@ int msm_dsi_host_modeset_init(struct mipi_dsi_host *host,
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
 	const struct msm_dsi_cfg_handler *cfg_hnd = msm_host->cfg_hnd;
-	struct drm_panel *panel;
 	int ret;
 
 	msm_host->dev = dev;
-	panel = msm_dsi_host_get_panel(&msm_host->base);
-
-	if (!IS_ERR(panel) && panel->dsc) {
-		struct msm_display_dsc_config *dsc = msm_host->dsc;
-
-		if (!dsc) {
-			dsc = devm_kzalloc(&msm_host->pdev->dev, sizeof(*dsc), GFP_KERNEL);
-			if (!dsc)
-				return -ENOMEM;
-			dsc->drm = panel->dsc;
-			msm_host->dsc = dsc;
-		}
-	}
 
 	ret = cfg_hnd->ops->tx_buf_alloc(msm_host, SZ_4K);
 	if (ret) {
@@ -2556,7 +2444,8 @@ int msm_dsi_host_power_on(struct mipi_dsi_host *host,
 
 	msm_dsi_sfpb_config(msm_host, true);
 
-	ret = dsi_host_regulator_enable(msm_host);
+	ret = regulator_bulk_enable(msm_host->cfg_hnd->cfg->num_regulators,
+				    msm_host->supplies);
 	if (ret) {
 		pr_err("%s:Failed to enable vregs.ret=%d\n",
 			__func__, ret);
@@ -2596,7 +2485,8 @@ fail_disable_clk:
 	cfg_hnd->ops->link_clk_disable(msm_host);
 	pm_runtime_put(&msm_host->pdev->dev);
 fail_disable_reg:
-	dsi_host_regulator_disable(msm_host);
+	regulator_bulk_disable(msm_host->cfg_hnd->cfg->num_regulators,
+			       msm_host->supplies);
 unlock_ret:
 	mutex_unlock(&msm_host->dev_mutex);
 	return ret;
@@ -2623,7 +2513,8 @@ int msm_dsi_host_power_off(struct mipi_dsi_host *host)
 	cfg_hnd->ops->link_clk_disable(msm_host);
 	pm_runtime_put(&msm_host->pdev->dev);
 
-	dsi_host_regulator_disable(msm_host);
+	regulator_bulk_disable(msm_host->cfg_hnd->cfg->num_regulators,
+			       msm_host->supplies);
 
 	msm_dsi_sfpb_config(msm_host, false);
 
@@ -2659,45 +2550,33 @@ enum drm_mode_status msm_dsi_host_check_dsc(struct mipi_dsi_host *host,
 					    const struct drm_display_mode *mode)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
-	struct msm_display_dsc_config *dsc = msm_host->dsc;
+	struct drm_dsc_config *dsc = msm_host->dsc;
 	int pic_width = mode->hdisplay;
 	int pic_height = mode->vdisplay;
 
 	if (!msm_host->dsc)
 		return MODE_OK;
 
-	if (pic_width % dsc->drm->slice_width) {
+	if (pic_width % dsc->slice_width) {
 		pr_err("DSI: pic_width %d has to be multiple of slice %d\n",
-		       pic_width, dsc->drm->slice_width);
+		       pic_width, dsc->slice_width);
 		return MODE_H_ILLEGAL;
 	}
 
-	if (pic_height % dsc->drm->slice_height) {
+	if (pic_height % dsc->slice_height) {
 		pr_err("DSI: pic_height %d has to be multiple of slice %d\n",
-		       pic_height, dsc->drm->slice_height);
+		       pic_height, dsc->slice_height);
 		return MODE_V_ILLEGAL;
 	}
 
 	return MODE_OK;
 }
 
-struct drm_panel *msm_dsi_host_get_panel(struct mipi_dsi_host *host)
-{
-	return of_drm_find_panel(to_msm_dsi_host(host)->device_node);
-}
-
 unsigned long msm_dsi_host_get_mode_flags(struct mipi_dsi_host *host)
 {
 	return to_msm_dsi_host(host)->mode_flags;
 }
 
-struct drm_bridge *msm_dsi_host_get_bridge(struct mipi_dsi_host *host)
-{
-	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
-
-	return of_drm_find_bridge(msm_host->device_node);
-}
-
 void msm_dsi_host_snapshot(struct msm_disp_state *disp_state, struct mipi_dsi_host *host)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
@@ -2771,7 +2650,7 @@ void msm_dsi_host_test_pattern_en(struct mipi_dsi_host *host)
 				DSI_TEST_PATTERN_GEN_CMD_STREAM0_TRIGGER_SW_TRIGGER);
 }
 
-struct msm_display_dsc_config *msm_dsi_host_get_dsc_config(struct mipi_dsi_host *host)
+struct drm_dsc_config *msm_dsi_host_get_dsc_config(struct mipi_dsi_host *host)
 {
 	struct msm_dsi_host *msm_host = to_msm_dsi_host(host);
 
diff --git a/drivers/gpu/drm/msm/dsi/dsi_manager.c b/drivers/gpu/drm/msm/dsi/dsi_manager.c
index cb84d185d73a..3a1417397283 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_manager.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_manager.c
@@ -141,14 +141,11 @@ static int enable_phy(struct msm_dsi *msm_dsi,
 		      struct msm_dsi_phy_shared_timings *shared_timings)
 {
 	struct msm_dsi_phy_clk_request clk_req;
-	int ret;
 	bool is_bonded_dsi = IS_BONDED_DSI();
 
 	msm_dsi_host_get_phy_clk_req(msm_dsi->host, &clk_req, is_bonded_dsi);
 
-	ret = msm_dsi_phy_enable(msm_dsi->phy, &clk_req, shared_timings);
-
-	return ret;
+	return msm_dsi_phy_enable(msm_dsi->phy, &clk_req, shared_timings);
 }
 
 static int
@@ -214,39 +211,26 @@ static void dsi_mgr_phy_disable(int id)
 	}
 }
 
-struct dsi_connector {
-	struct drm_connector base;
-	int id;
-};
-
 struct dsi_bridge {
 	struct drm_bridge base;
 	int id;
 };
 
-#define to_dsi_connector(x) container_of(x, struct dsi_connector, base)
 #define to_dsi_bridge(x) container_of(x, struct dsi_bridge, base)
 
-static inline int dsi_mgr_connector_get_id(struct drm_connector *connector)
-{
-	struct dsi_connector *dsi_connector = to_dsi_connector(connector);
-	return dsi_connector->id;
-}
-
 static int dsi_mgr_bridge_get_id(struct drm_bridge *bridge)
 {
 	struct dsi_bridge *dsi_bridge = to_dsi_bridge(bridge);
 	return dsi_bridge->id;
 }
 
-static int msm_dsi_manager_panel_init(struct drm_connector *conn, u8 id)
+static void msm_dsi_manager_set_split_display(u8 id)
 {
-	struct msm_drm_private *priv = conn->dev->dev_private;
-	struct msm_kms *kms = priv->kms;
 	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
 	struct msm_dsi *other_dsi = dsi_mgr_get_other_dsi(id);
+	struct msm_drm_private *priv = msm_dsi->dev->dev_private;
+	struct msm_kms *kms = priv->kms;
 	struct msm_dsi *master_dsi, *slave_dsi;
-	struct drm_panel *panel;
 
 	if (IS_BONDED_DSI() && !IS_MASTER_DSI_LINK(id)) {
 		master_dsi = other_dsi;
@@ -256,89 +240,18 @@ static int msm_dsi_manager_panel_init(struct drm_connector *conn, u8 id)
 		slave_dsi = other_dsi;
 	}
 
-	/*
-	 * There is only 1 panel in the global panel list for bonded DSI mode.
-	 * Therefore slave dsi should get the drm_panel instance from master
-	 * dsi.
-	 */
-	panel = msm_dsi_host_get_panel(master_dsi->host);
-	if (IS_ERR(panel)) {
-		DRM_ERROR("Could not find panel for %u (%ld)\n", msm_dsi->id,
-			  PTR_ERR(panel));
-		return PTR_ERR(panel);
-	}
-
-	if (!panel || !IS_BONDED_DSI())
-		goto out;
-
-	drm_object_attach_property(&conn->base,
-				   conn->dev->mode_config.tile_property, 0);
+	if (!msm_dsi->external_bridge || !IS_BONDED_DSI())
+		return;
 
 	/*
 	 * Set split display info to kms once bonded DSI panel is connected to
 	 * both hosts.
 	 */
-	if (other_dsi && other_dsi->panel && kms->funcs->set_split_display) {
+	if (other_dsi && other_dsi->external_bridge && kms->funcs->set_split_display) {
 		kms->funcs->set_split_display(kms, master_dsi->encoder,
 					      slave_dsi->encoder,
 					      msm_dsi_is_cmd_mode(msm_dsi));
 	}
-
-out:
-	msm_dsi->panel = panel;
-	return 0;
-}
-
-static enum drm_connector_status dsi_mgr_connector_detect(
-		struct drm_connector *connector, bool force)
-{
-	int id = dsi_mgr_connector_get_id(connector);
-	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
-
-	return msm_dsi->panel ? connector_status_connected :
-		connector_status_disconnected;
-}
-
-static void dsi_mgr_connector_destroy(struct drm_connector *connector)
-{
-	struct dsi_connector *dsi_connector = to_dsi_connector(connector);
-
-	DBG("");
-
-	drm_connector_cleanup(connector);
-
-	kfree(dsi_connector);
-}
-
-static int dsi_mgr_connector_get_modes(struct drm_connector *connector)
-{
-	int id = dsi_mgr_connector_get_id(connector);
-	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
-	struct drm_panel *panel = msm_dsi->panel;
-	int num;
-
-	if (!panel)
-		return 0;
-
-	/*
-	 * In bonded DSI mode, we have one connector that can be
-	 * attached to the drm_panel.
-	 */
-	num = drm_panel_get_modes(panel, connector);
-	if (!num)
-		return 0;
-
-	return num;
-}
-
-static struct drm_encoder *
-dsi_mgr_connector_best_encoder(struct drm_connector *connector)
-{
-	int id = dsi_mgr_connector_get_id(connector);
-	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
-
-	DBG("");
-	return msm_dsi_get_encoder(msm_dsi);
 }
 
 static void dsi_mgr_bridge_power_on(struct drm_bridge *bridge)
@@ -403,7 +316,6 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge)
 	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
 	struct msm_dsi *msm_dsi1 = dsi_mgr_get_dsi(DSI_1);
 	struct mipi_dsi_host *host = msm_dsi->host;
-	struct drm_panel *panel = msm_dsi->panel;
 	bool is_bonded_dsi = IS_BONDED_DSI();
 	int ret;
 
@@ -418,18 +330,6 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge)
 	if (!dsi_mgr_power_on_early(bridge))
 		dsi_mgr_bridge_power_on(bridge);
 
-	/* Always call panel functions once, because even for dual panels,
-	 * there is only one drm_panel instance.
-	 */
-	if (panel) {
-		ret = drm_panel_prepare(panel);
-		if (ret) {
-			pr_err("%s: prepare panel %d failed, %d\n", __func__,
-								id, ret);
-			goto panel_prep_fail;
-		}
-	}
-
 	ret = msm_dsi_host_enable(host);
 	if (ret) {
 		pr_err("%s: enable host %d failed, %d\n", __func__, id, ret);
@@ -449,9 +349,6 @@ static void dsi_mgr_bridge_pre_enable(struct drm_bridge *bridge)
 host1_en_fail:
 	msm_dsi_host_disable(host);
 host_en_fail:
-	if (panel)
-		drm_panel_unprepare(panel);
-panel_prep_fail:
 
 	return;
 }
@@ -469,62 +366,12 @@ void msm_dsi_manager_tpg_enable(void)
 	}
 }
 
-static void dsi_mgr_bridge_enable(struct drm_bridge *bridge)
-{
-	int id = dsi_mgr_bridge_get_id(bridge);
-	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
-	struct drm_panel *panel = msm_dsi->panel;
-	bool is_bonded_dsi = IS_BONDED_DSI();
-	int ret;
-
-	DBG("id=%d", id);
-	if (!msm_dsi_device_connected(msm_dsi))
-		return;
-
-	/* Do nothing with the host if it is slave-DSI in case of bonded DSI */
-	if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id))
-		return;
-
-	if (panel) {
-		ret = drm_panel_enable(panel);
-		if (ret) {
-			pr_err("%s: enable panel %d failed, %d\n", __func__, id,
-									ret);
-		}
-	}
-}
-
-static void dsi_mgr_bridge_disable(struct drm_bridge *bridge)
-{
-	int id = dsi_mgr_bridge_get_id(bridge);
-	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
-	struct drm_panel *panel = msm_dsi->panel;
-	bool is_bonded_dsi = IS_BONDED_DSI();
-	int ret;
-
-	DBG("id=%d", id);
-	if (!msm_dsi_device_connected(msm_dsi))
-		return;
-
-	/* Do nothing with the host if it is slave-DSI in case of bonded DSI */
-	if (is_bonded_dsi && !IS_MASTER_DSI_LINK(id))
-		return;
-
-	if (panel) {
-		ret = drm_panel_disable(panel);
-		if (ret)
-			pr_err("%s: Panel %d OFF failed, %d\n", __func__, id,
-									ret);
-	}
-}
-
 static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge)
 {
 	int id = dsi_mgr_bridge_get_id(bridge);
 	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
 	struct msm_dsi *msm_dsi1 = dsi_mgr_get_dsi(DSI_1);
 	struct mipi_dsi_host *host = msm_dsi->host;
-	struct drm_panel *panel = msm_dsi->panel;
 	bool is_bonded_dsi = IS_BONDED_DSI();
 	int ret;
 
@@ -551,13 +398,6 @@ static void dsi_mgr_bridge_post_disable(struct drm_bridge *bridge)
 			pr_err("%s: host1 disable failed, %d\n", __func__, ret);
 	}
 
-	if (panel) {
-		ret = drm_panel_unprepare(panel);
-		if (ret)
-			pr_err("%s: Panel %d unprepare failed,%d\n", __func__,
-								id, ret);
-	}
-
 	msm_dsi_host_disable_irq(host);
 	if (is_bonded_dsi && msm_dsi1)
 		msm_dsi_host_disable_irq(msm_dsi1->host);
@@ -614,76 +454,13 @@ static enum drm_mode_status dsi_mgr_bridge_mode_valid(struct drm_bridge *bridge,
 	return msm_dsi_host_check_dsc(host, mode);
 }
 
-static const struct drm_connector_funcs dsi_mgr_connector_funcs = {
-	.detect = dsi_mgr_connector_detect,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = dsi_mgr_connector_destroy,
-	.reset = drm_atomic_helper_connector_reset,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-};
-
-static const struct drm_connector_helper_funcs dsi_mgr_conn_helper_funcs = {
-	.get_modes = dsi_mgr_connector_get_modes,
-	.best_encoder = dsi_mgr_connector_best_encoder,
-};
-
 static const struct drm_bridge_funcs dsi_mgr_bridge_funcs = {
 	.pre_enable = dsi_mgr_bridge_pre_enable,
-	.enable = dsi_mgr_bridge_enable,
-	.disable = dsi_mgr_bridge_disable,
 	.post_disable = dsi_mgr_bridge_post_disable,
 	.mode_set = dsi_mgr_bridge_mode_set,
 	.mode_valid = dsi_mgr_bridge_mode_valid,
 };
 
-/* initialize connector when we're connected to a drm_panel */
-struct drm_connector *msm_dsi_manager_connector_init(u8 id)
-{
-	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
-	struct drm_connector *connector = NULL;
-	struct dsi_connector *dsi_connector;
-	int ret;
-
-	dsi_connector = kzalloc(sizeof(*dsi_connector), GFP_KERNEL);
-	if (!dsi_connector)
-		return ERR_PTR(-ENOMEM);
-
-	dsi_connector->id = id;
-
-	connector = &dsi_connector->base;
-
-	ret = drm_connector_init(msm_dsi->dev, connector,
-			&dsi_mgr_connector_funcs, DRM_MODE_CONNECTOR_DSI);
-	if (ret)
-		return ERR_PTR(ret);
-
-	drm_connector_helper_add(connector, &dsi_mgr_conn_helper_funcs);
-
-	/* Enable HPD to let hpd event is handled
-	 * when panel is attached to the host.
-	 */
-	connector->polled = DRM_CONNECTOR_POLL_HPD;
-
-	/* Display driver doesn't support interlace now. */
-	connector->interlace_allowed = 0;
-	connector->doublescan_allowed = 0;
-
-	drm_connector_attach_encoder(connector, msm_dsi->encoder);
-
-	ret = msm_dsi_manager_panel_init(connector, id);
-	if (ret) {
-		DRM_DEV_ERROR(msm_dsi->dev->dev, "init panel failed %d\n", ret);
-		goto fail;
-	}
-
-	return connector;
-
-fail:
-	connector->funcs->destroy(connector);
-	return ERR_PTR(ret);
-}
-
 /* initialize bridge */
 struct drm_bridge *msm_dsi_manager_bridge_init(u8 id)
 {
@@ -722,18 +499,21 @@ fail:
 	return ERR_PTR(ret);
 }
 
-struct drm_connector *msm_dsi_manager_ext_bridge_init(u8 id)
+int msm_dsi_manager_ext_bridge_init(u8 id)
 {
 	struct msm_dsi *msm_dsi = dsi_mgr_get_dsi(id);
 	struct drm_device *dev = msm_dsi->dev;
-	struct drm_connector *connector;
 	struct drm_encoder *encoder;
 	struct drm_bridge *int_bridge, *ext_bridge;
 	int ret;
 
 	int_bridge = msm_dsi->bridge;
-	ext_bridge = msm_dsi->external_bridge =
-			msm_dsi_host_get_bridge(msm_dsi->host);
+	ext_bridge = devm_drm_of_get_bridge(&msm_dsi->pdev->dev,
+					    msm_dsi->pdev->dev.of_node, 1, 0);
+	if (IS_ERR(ext_bridge))
+		return PTR_ERR(ext_bridge);
+
+	msm_dsi->external_bridge = ext_bridge;
 
 	encoder = msm_dsi->encoder;
 
@@ -745,36 +525,32 @@ struct drm_connector *msm_dsi_manager_ext_bridge_init(u8 id)
 	ret = drm_bridge_attach(encoder, ext_bridge, int_bridge,
 			DRM_BRIDGE_ATTACH_NO_CONNECTOR);
 	if (ret == -EINVAL) {
-		struct drm_connector *connector;
-		struct list_head *connector_list;
-
-		/* link the internal dsi bridge to the external bridge */
-		drm_bridge_attach(encoder, ext_bridge, int_bridge, 0);
-
 		/*
-		 * we need the drm_connector created by the external bridge
-		 * driver (or someone else) to feed it to our driver's
-		 * priv->connector[] list, mainly for msm_fbdev_init()
+		 * link the internal dsi bridge to the external bridge,
+		 * connector is created by the next bridge.
 		 */
-		connector_list = &dev->mode_config.connector_list;
+		ret = drm_bridge_attach(encoder, ext_bridge, int_bridge, 0);
+		if (ret < 0)
+			return ret;
+	} else {
+		struct drm_connector *connector;
 
-		list_for_each_entry(connector, connector_list, head) {
-			if (drm_connector_has_possible_encoder(connector, encoder))
-				return connector;
+		/* We are in charge of the connector, create one now. */
+		connector = drm_bridge_connector_init(dev, encoder);
+		if (IS_ERR(connector)) {
+			DRM_ERROR("Unable to create bridge connector\n");
+			return PTR_ERR(connector);
 		}
 
-		return ERR_PTR(-ENODEV);
-	}
-
-	connector = drm_bridge_connector_init(dev, encoder);
-	if (IS_ERR(connector)) {
-		DRM_ERROR("Unable to create bridge connector\n");
-		return ERR_CAST(connector);
+		ret = drm_connector_attach_encoder(connector, encoder);
+		if (ret < 0)
+			return ret;
 	}
 
-	drm_connector_attach_encoder(connector, encoder);
+	/* The pipeline is ready, ping encoders if necessary */
+	msm_dsi_manager_set_split_display(id);
 
-	return connector;
+	return 0;
 }
 
 void msm_dsi_manager_bridge_destroy(struct drm_bridge *bridge)
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
index 56dfa2d24be1..7fc0975cb869 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
@@ -507,82 +507,6 @@ int msm_dsi_cphy_timing_calc_v4(struct msm_dsi_dphy_timing *timing,
 	return 0;
 }
 
-static int dsi_phy_regulator_init(struct msm_dsi_phy *phy)
-{
-	struct regulator_bulk_data *s = phy->supplies;
-	const struct dsi_reg_entry *regs = phy->cfg->reg_cfg.regs;
-	struct device *dev = &phy->pdev->dev;
-	int num = phy->cfg->reg_cfg.num;
-	int i, ret;
-
-	for (i = 0; i < num; i++)
-		s[i].supply = regs[i].name;
-
-	ret = devm_regulator_bulk_get(dev, num, s);
-	if (ret < 0) {
-		if (ret != -EPROBE_DEFER) {
-			DRM_DEV_ERROR(dev,
-				      "%s: failed to init regulator, ret=%d\n",
-				      __func__, ret);
-		}
-
-		return ret;
-	}
-
-	return 0;
-}
-
-static void dsi_phy_regulator_disable(struct msm_dsi_phy *phy)
-{
-	struct regulator_bulk_data *s = phy->supplies;
-	const struct dsi_reg_entry *regs = phy->cfg->reg_cfg.regs;
-	int num = phy->cfg->reg_cfg.num;
-	int i;
-
-	DBG("");
-	for (i = num - 1; i >= 0; i--)
-		if (regs[i].disable_load >= 0)
-			regulator_set_load(s[i].consumer, regs[i].disable_load);
-
-	regulator_bulk_disable(num, s);
-}
-
-static int dsi_phy_regulator_enable(struct msm_dsi_phy *phy)
-{
-	struct regulator_bulk_data *s = phy->supplies;
-	const struct dsi_reg_entry *regs = phy->cfg->reg_cfg.regs;
-	struct device *dev = &phy->pdev->dev;
-	int num = phy->cfg->reg_cfg.num;
-	int ret, i;
-
-	DBG("");
-	for (i = 0; i < num; i++) {
-		if (regs[i].enable_load >= 0) {
-			ret = regulator_set_load(s[i].consumer,
-							regs[i].enable_load);
-			if (ret < 0) {
-				DRM_DEV_ERROR(dev,
-					"regulator %d set op mode failed, %d\n",
-					i, ret);
-				goto fail;
-			}
-		}
-	}
-
-	ret = regulator_bulk_enable(num, s);
-	if (ret < 0) {
-		DRM_DEV_ERROR(dev, "regulator enable failed, %d\n", ret);
-		goto fail;
-	}
-
-	return 0;
-
-fail:
-	for (i--; i >= 0; i--)
-		regulator_set_load(s[i].consumer, regs[i].disable_load);
-	return ret;
-}
-
 static int dsi_phy_enable_resource(struct msm_dsi_phy *phy)
 {
 	struct device *dev = &phy->pdev->dev;
@@ -697,12 +621,9 @@ static int dsi_phy_driver_probe(struct platform_device *pdev)
 	phy->pdev = pdev;
 
 	phy->id = dsi_phy_get_id(phy);
-	if (phy->id < 0) {
-		ret = phy->id;
-		DRM_DEV_ERROR(dev, "%s: couldn't identify PHY index, %d\n",
-			__func__, ret);
-		goto fail;
-	}
+	if (phy->id < 0)
+		return dev_err_probe(dev, phy->id,
+				     "Couldn't identify PHY index\n");
 
 	phy->regulator_ldo_mode = of_property_read_bool(dev->of_node,
 				"qcom,dsi-phy-regulator-ldo-mode");
@@ -710,86 +631,71 @@ static int dsi_phy_driver_probe(struct platform_device *pdev)
 		phy->cphy_mode = (phy_type == PHY_TYPE_CPHY);
 
 	phy->base = msm_ioremap_size(pdev, "dsi_phy", &phy->base_size);
-	if (IS_ERR(phy->base)) {
-		DRM_DEV_ERROR(dev, "%s: failed to map phy base\n", __func__);
-		ret = -ENOMEM;
-		goto fail;
-	}
+	if (IS_ERR(phy->base))
+		return dev_err_probe(dev, PTR_ERR(phy->base),
+				     "Failed to map phy base\n");
 
 	phy->pll_base = msm_ioremap_size(pdev, "dsi_pll", &phy->pll_size);
-	if (IS_ERR(phy->pll_base)) {
-		DRM_DEV_ERROR(&pdev->dev, "%s: failed to map pll base\n", __func__);
-		ret = -ENOMEM;
-		goto fail;
-	}
+	if (IS_ERR(phy->pll_base))
+		return dev_err_probe(dev, PTR_ERR(phy->pll_base),
+				     "Failed to map pll base\n");
 
 	if (phy->cfg->has_phy_lane) {
 		phy->lane_base = msm_ioremap_size(pdev, "dsi_phy_lane", &phy->lane_size);
-		if (IS_ERR(phy->lane_base)) {
-			DRM_DEV_ERROR(&pdev->dev, "%s: failed to map phy lane base\n", __func__);
-			ret = -ENOMEM;
-			goto fail;
-		}
+		if (IS_ERR(phy->lane_base))
+			return dev_err_probe(dev, PTR_ERR(phy->lane_base),
+					     "Failed to map phy lane base\n");
 	}
 
 	if (phy->cfg->has_phy_regulator) {
 		phy->reg_base = msm_ioremap_size(pdev, "dsi_phy_regulator", &phy->reg_size);
-		if (IS_ERR(phy->reg_base)) {
-			DRM_DEV_ERROR(&pdev->dev, "%s: failed to map phy regulator base\n", __func__);
-			ret = -ENOMEM;
-			goto fail;
-		}
+		if (IS_ERR(phy->reg_base))
+			return dev_err_probe(dev, PTR_ERR(phy->reg_base),
+					     "Failed to map phy regulator base\n");
 	}
 
 	if (phy->cfg->ops.parse_dt_properties) {
 		ret = phy->cfg->ops.parse_dt_properties(phy);
 		if (ret)
-			goto fail;
+			return ret;
 	}
 
-	ret = dsi_phy_regulator_init(phy);
+	ret = devm_regulator_bulk_get_const(dev, phy->cfg->num_regulators,
+					    phy->cfg->regulator_data,
+					    &phy->supplies);
 	if (ret)
-		goto fail;
+		return ret;
 
 	phy->ahb_clk = msm_clk_get(pdev, "iface");
-	if (IS_ERR(phy->ahb_clk)) {
-		DRM_DEV_ERROR(dev, "%s: Unable to get ahb clk\n", __func__);
-		ret = PTR_ERR(phy->ahb_clk);
-		goto fail;
-	}
+	if (IS_ERR(phy->ahb_clk))
+		return dev_err_probe(dev, PTR_ERR(phy->ahb_clk),
+				     "Unable to get ahb clk\n");
 
 	/* PLL init will call into clk_register which requires
 	 * register access, so we need to enable power and ahb clock.
 	 */
 	ret = dsi_phy_enable_resource(phy);
 	if (ret)
-		goto fail;
+		return ret;
 
 	if (phy->cfg->ops.pll_init) {
 		ret = phy->cfg->ops.pll_init(phy);
-		if (ret) {
-			DRM_DEV_INFO(dev,
-				"%s: pll init failed: %d, need separate pll clk driver\n",
-				__func__, ret);
-			goto fail;
-		}
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "PLL init failed; need separate clk driver\n");
 	}
 
 	ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get,
 				     phy->provided_clocks);
-	if (ret) {
-		DRM_DEV_ERROR(dev, "%s: failed to register clk provider: %d\n", __func__, ret);
-		goto fail;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to register clk provider\n");
 
 	dsi_phy_disable_resource(phy);
 
 	platform_set_drvdata(pdev, phy);
 
 	return 0;
-
-fail:
-	return ret;
 }
 
 static struct platform_driver dsi_phy_platform_driver = {
@@ -829,7 +735,7 @@ int msm_dsi_phy_enable(struct msm_dsi_phy *phy,
 		goto res_en_fail;
 	}
 
-	ret = dsi_phy_regulator_enable(phy);
+	ret = regulator_bulk_enable(phy->cfg->num_regulators, phy->supplies);
 	if (ret) {
 		DRM_DEV_ERROR(dev, "%s: regulator enable failed, %d\n",
 			__func__, ret);
@@ -866,7 +772,7 @@ pll_restor_fail:
 	if (phy->cfg->ops.disable)
 		phy->cfg->ops.disable(phy);
 phy_en_fail:
-	dsi_phy_regulator_disable(phy);
+	regulator_bulk_disable(phy->cfg->num_regulators, phy->supplies);
 reg_en_fail:
 	dsi_phy_disable_resource(phy);
 res_en_fail:
@@ -880,7 +786,7 @@ void msm_dsi_phy_disable(struct msm_dsi_phy *phy)
 
 	phy->cfg->ops.disable(phy);
 
-	dsi_phy_regulator_disable(phy);
+	regulator_bulk_disable(phy->cfg->num_regulators, phy->supplies);
 	dsi_phy_disable_resource(phy);
 }
 
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
index dc91b43d5a38..60a99c6525b2 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
@@ -29,7 +29,8 @@ struct msm_dsi_phy_ops {
 };
 
 struct msm_dsi_phy_cfg {
-	struct dsi_reg_config reg_cfg;
+	const struct regulator_bulk_data *regulator_data;
+	int num_regulators;
 	struct msm_dsi_phy_ops ops;
 
 	unsigned long	min_pll_rate;
@@ -98,7 +99,7 @@ struct msm_dsi_phy {
 	int id;
 
 	struct clk *ahb_clk;
-	struct regulator_bulk_data supplies[DSI_DEV_REGULATOR_MAX];
+	struct regulator_bulk_data *supplies;
 
 	struct msm_dsi_dphy_timing timing;
 	const struct msm_dsi_phy_cfg *cfg;
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c
index 08b015ea1b1e..27b592c776a3 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_10nm.c
@@ -188,19 +188,19 @@ static void dsi_pll_ssc_commit(struct dsi_pll_10nm *pll, struct dsi_pll_config *
 		pr_debug("SSC is enabled\n");
 
 		dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_STEPSIZE_LOW_1,
-			  config->ssc_stepsize & 0xff);
+			      config->ssc_stepsize & 0xff);
 		dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_STEPSIZE_HIGH_1,
-			  config->ssc_stepsize >> 8);
+			      config->ssc_stepsize >> 8);
 		dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_DIV_PER_LOW_1,
-			  config->ssc_div_per & 0xff);
+			      config->ssc_div_per & 0xff);
 		dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_DIV_PER_HIGH_1,
-			  config->ssc_div_per >> 8);
+			      config->ssc_div_per >> 8);
 		dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_DIV_ADJPER_LOW_1,
-			  config->ssc_adj_per & 0xff);
+			      config->ssc_adj_per & 0xff);
 		dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_DIV_ADJPER_HIGH_1,
-			  config->ssc_adj_per >> 8);
+			      config->ssc_adj_per >> 8);
 		dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_SSC_CONTROL,
-			  SSC_EN | (config->ssc_center ? SSC_CENTER : 0));
+			      SSC_EN | (config->ssc_center ? SSC_CENTER : 0));
 	}
 }
 
@@ -215,16 +215,19 @@ static void dsi_pll_config_hzindep_reg(struct dsi_pll_10nm *pll)
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FEEDBACK_DIVIDER, 0x4e);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_CALIBRATION_SETTINGS, 0x40);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_BAND_SEL_CAL_SETTINGS_THREE,
-		  0xba);
-	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FREQ_DETECT_SETTINGS_ONE, 0x0c);
+		      0xba);
+	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FREQ_DETECT_SETTINGS_ONE,
+		      0x0c);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_OUTDIV, 0x00);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_CORE_OVERRIDE, 0x00);
-	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_DIGITAL_TIMERS_TWO, 0x08);
+	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_DIGITAL_TIMERS_TWO,
+		      0x08);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_PROP_GAIN_RATE_1, 0x08);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_BAND_SET_RATE_1, 0xc0);
-	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_INT_GAIN_IFILT_BAND_1, 0xfa);
+	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_INT_GAIN_IFILT_BAND_1,
+		      0xfa);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_FL_INT_GAIN_PFILT_BAND_1,
-		  0x4c);
+		      0x4c);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_LOCK_OVERRIDE, 0x80);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PFILT, 0x29);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_IFILT, 0x3f);
@@ -236,18 +239,18 @@ static void dsi_pll_commit(struct dsi_pll_10nm *pll, struct dsi_pll_config *conf
 
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_CORE_INPUT_OVERRIDE, 0x12);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_DECIMAL_DIV_START_1,
-		  config->decimal_div_start);
+		      config->decimal_div_start);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FRAC_DIV_START_LOW_1,
-		  config->frac_div_start & 0xff);
+		      config->frac_div_start & 0xff);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FRAC_DIV_START_MID_1,
-		  (config->frac_div_start & 0xff00) >> 8);
+		      (config->frac_div_start & 0xff00) >> 8);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_FRAC_DIV_START_HIGH_1,
-		  (config->frac_div_start & 0x30000) >> 16);
+		      (config->frac_div_start & 0x30000) >> 16);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_LOCKDET_RATE_1, 64);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_PLL_LOCK_DELAY, 0x06);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_CMODE, 0x10);
 	dsi_phy_write(base + REG_DSI_10nm_PHY_PLL_CLOCK_INVERTERS,
-		  config->pll_clock_inverters);
+		      config->pll_clock_inverters);
 }
 
 static int dsi_pll_10nm_vco_set_rate(struct clk_hw *hw, unsigned long rate,
@@ -306,7 +309,7 @@ static void dsi_pll_disable_pll_bias(struct dsi_pll_10nm *pll)
 
 	dsi_phy_write(pll->phy->pll_base + REG_DSI_10nm_PHY_PLL_SYSTEM_MUXES, 0);
 	dsi_phy_write(pll->phy->base + REG_DSI_10nm_PHY_CMN_CTRL_0,
-		  data & ~BIT(5));
+		      data & ~BIT(5));
 	ndelay(250);
 }
 
@@ -315,7 +318,7 @@ static void dsi_pll_enable_pll_bias(struct dsi_pll_10nm *pll)
 	u32 data = dsi_phy_read(pll->phy->base + REG_DSI_10nm_PHY_CMN_CTRL_0);
 
 	dsi_phy_write(pll->phy->base + REG_DSI_10nm_PHY_CMN_CTRL_0,
-		  data | BIT(5));
+		      data | BIT(5));
 	dsi_phy_write(pll->phy->pll_base + REG_DSI_10nm_PHY_PLL_SYSTEM_MUXES, 0xc0);
 	ndelay(250);
 }
@@ -326,7 +329,7 @@ static void dsi_pll_disable_global_clk(struct dsi_pll_10nm *pll)
 
 	data = dsi_phy_read(pll->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG1);
 	dsi_phy_write(pll->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG1,
-		  data & ~BIT(5));
+		      data & ~BIT(5));
 }
 
 static void dsi_pll_enable_global_clk(struct dsi_pll_10nm *pll)
@@ -335,7 +338,7 @@ static void dsi_pll_enable_global_clk(struct dsi_pll_10nm *pll)
 
 	data = dsi_phy_read(pll->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG1);
 	dsi_phy_write(pll->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG1,
-		  data | BIT(5));
+		      data | BIT(5));
 }
 
 static int dsi_pll_10nm_vco_prepare(struct clk_hw *hw)
@@ -356,7 +359,7 @@ static int dsi_pll_10nm_vco_prepare(struct clk_hw *hw)
 
 	/* Start PLL */
 	dsi_phy_write(pll_10nm->phy->base + REG_DSI_10nm_PHY_CMN_PLL_CNTRL,
-		  0x01);
+		      0x01);
 
 	/*
 	 * ensure all PLL configurations are written prior to checking
@@ -378,10 +381,10 @@ static int dsi_pll_10nm_vco_prepare(struct clk_hw *hw)
 		dsi_pll_enable_global_clk(pll_10nm->slave);
 
 	dsi_phy_write(pll_10nm->phy->base + REG_DSI_10nm_PHY_CMN_RBUF_CTRL,
-		  0x01);
+		      0x01);
 	if (pll_10nm->slave)
 		dsi_phy_write(pll_10nm->slave->phy->base +
-			  REG_DSI_10nm_PHY_CMN_RBUF_CTRL, 0x01);
+			      REG_DSI_10nm_PHY_CMN_RBUF_CTRL, 0x01);
 
 error:
 	return rc;
@@ -486,7 +489,7 @@ static void dsi_10nm_pll_save_state(struct msm_dsi_phy *phy)
 	u32 cmn_clk_cfg0, cmn_clk_cfg1;
 
 	cached->pll_out_div = dsi_phy_read(pll_10nm->phy->pll_base +
-				       REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE);
+			REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE);
 	cached->pll_out_div &= 0x3;
 
 	cmn_clk_cfg0 = dsi_phy_read(phy_base + REG_DSI_10nm_PHY_CMN_CLK_CFG0);
@@ -515,7 +518,7 @@ static int dsi_10nm_pll_restore_state(struct msm_dsi_phy *phy)
 	dsi_phy_write(pll_10nm->phy->pll_base + REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE, val);
 
 	dsi_phy_write(phy_base + REG_DSI_10nm_PHY_CMN_CLK_CFG0,
-		  cached->bit_clk_div | (cached->pix_clk_div << 4));
+		      cached->bit_clk_div | (cached->pix_clk_div << 4));
 
 	val = dsi_phy_read(phy_base + REG_DSI_10nm_PHY_CMN_CLK_CFG1);
 	val &= ~0x3;
@@ -571,64 +574,59 @@ static int dsi_10nm_set_usecase(struct msm_dsi_phy *phy)
  */
 static int pll_10nm_register(struct dsi_pll_10nm *pll_10nm, struct clk_hw **provided_clocks)
 {
-	char clk_name[32], parent[32], vco_name[32];
-	char parent2[32], parent3[32], parent4[32];
+	char clk_name[32];
 	struct clk_init_data vco_init = {
 		.parent_data = &(const struct clk_parent_data) {
 			.fw_name = "ref",
 		},
 		.num_parents = 1,
-		.name = vco_name,
+		.name = clk_name,
 		.flags = CLK_IGNORE_UNUSED,
 		.ops = &clk_ops_dsi_pll_10nm_vco,
 	};
 	struct device *dev = &pll_10nm->phy->pdev->dev;
-	struct clk_hw *hw;
+	struct clk_hw *hw, *pll_out_div, *pll_bit, *pll_by_2_bit;
+	struct clk_hw *pll_post_out_div, *pclk_mux;
 	int ret;
 
 	DBG("DSI%d", pll_10nm->phy->id);
 
-	snprintf(vco_name, 32, "dsi%dvco_clk", pll_10nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dvco_clk", pll_10nm->phy->id);
 	pll_10nm->clk_hw.init = &vco_init;
 
 	ret = devm_clk_hw_register(dev, &pll_10nm->clk_hw);
 	if (ret)
 		return ret;
 
-	snprintf(clk_name, 32, "dsi%d_pll_out_div_clk", pll_10nm->phy->id);
-	snprintf(parent, 32, "dsi%dvco_clk", pll_10nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_out_div_clk", pll_10nm->phy->id);
 
-	hw = devm_clk_hw_register_divider(dev, clk_name,
-				     parent, CLK_SET_RATE_PARENT,
-				     pll_10nm->phy->pll_base +
-				     REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE,
-				     0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL);
-	if (IS_ERR(hw)) {
-		ret = PTR_ERR(hw);
+	pll_out_div = devm_clk_hw_register_divider_parent_hw(dev, clk_name,
+			&pll_10nm->clk_hw, CLK_SET_RATE_PARENT,
+			pll_10nm->phy->pll_base +
+				REG_DSI_10nm_PHY_PLL_PLL_OUTDIV_RATE,
+			0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL);
+	if (IS_ERR(pll_out_div)) {
+		ret = PTR_ERR(pll_out_div);
 		goto fail;
 	}
 
-	snprintf(clk_name, 32, "dsi%d_pll_bit_clk", pll_10nm->phy->id);
-	snprintf(parent, 32, "dsi%d_pll_out_div_clk", pll_10nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_bit_clk", pll_10nm->phy->id);
 
 	/* BIT CLK: DIV_CTRL_3_0 */
-	hw = devm_clk_hw_register_divider(dev, clk_name, parent,
-				     CLK_SET_RATE_PARENT,
-				     pll_10nm->phy->base +
-				     REG_DSI_10nm_PHY_CMN_CLK_CFG0,
-				     0, 4, CLK_DIVIDER_ONE_BASED,
-				     &pll_10nm->postdiv_lock);
-	if (IS_ERR(hw)) {
-		ret = PTR_ERR(hw);
+	pll_bit = devm_clk_hw_register_divider_parent_hw(dev, clk_name,
+			pll_out_div, CLK_SET_RATE_PARENT,
+			pll_10nm->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG0,
+			0, 4, CLK_DIVIDER_ONE_BASED, &pll_10nm->postdiv_lock);
+	if (IS_ERR(pll_bit)) {
+		ret = PTR_ERR(pll_bit);
 		goto fail;
 	}
 
-	snprintf(clk_name, 32, "dsi%d_phy_pll_out_byteclk", pll_10nm->phy->id);
-	snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_10nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_phy_pll_out_byteclk", pll_10nm->phy->id);
 
 	/* DSI Byte clock = VCO_CLK / OUT_DIV / BIT_DIV / 8 */
-	hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent,
-					  CLK_SET_RATE_PARENT, 1, 8);
+	hw = devm_clk_hw_register_fixed_factor_parent_hw(dev, clk_name,
+			pll_bit, CLK_SET_RATE_PARENT, 1, 8);
 	if (IS_ERR(hw)) {
 		ret = PTR_ERR(hw);
 		goto fail;
@@ -636,52 +634,45 @@ static int pll_10nm_register(struct dsi_pll_10nm *pll_10nm, struct clk_hw **prov
 
 	provided_clocks[DSI_BYTE_PLL_CLK] = hw;
 
-	snprintf(clk_name, 32, "dsi%d_pll_by_2_bit_clk", pll_10nm->phy->id);
-	snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_10nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_by_2_bit_clk", pll_10nm->phy->id);
 
-	hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent,
-					  0, 1, 2);
-	if (IS_ERR(hw)) {
-		ret = PTR_ERR(hw);
+	pll_by_2_bit = devm_clk_hw_register_fixed_factor_parent_hw(dev,
+			clk_name, pll_bit, 0, 1, 2);
+	if (IS_ERR(pll_by_2_bit)) {
+		ret = PTR_ERR(pll_by_2_bit);
 		goto fail;
 	}
 
-	snprintf(clk_name, 32, "dsi%d_pll_post_out_div_clk", pll_10nm->phy->id);
-	snprintf(parent, 32, "dsi%d_pll_out_div_clk", pll_10nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_post_out_div_clk", pll_10nm->phy->id);
 
-	hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent,
-					  0, 1, 4);
-	if (IS_ERR(hw)) {
-		ret = PTR_ERR(hw);
+	pll_post_out_div = devm_clk_hw_register_fixed_factor_parent_hw(dev,
+			clk_name, pll_out_div, 0, 1, 4);
+	if (IS_ERR(pll_post_out_div)) {
+		ret = PTR_ERR(pll_post_out_div);
 		goto fail;
 	}
 
-	snprintf(clk_name, 32, "dsi%d_pclk_mux", pll_10nm->phy->id);
-	snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_10nm->phy->id);
-	snprintf(parent2, 32, "dsi%d_pll_by_2_bit_clk", pll_10nm->phy->id);
-	snprintf(parent3, 32, "dsi%d_pll_out_div_clk", pll_10nm->phy->id);
-	snprintf(parent4, 32, "dsi%d_pll_post_out_div_clk", pll_10nm->phy->id);
-
-	hw = devm_clk_hw_register_mux(dev, clk_name,
-				 ((const char *[]){
-				 parent, parent2, parent3, parent4
-				 }), 4, 0, pll_10nm->phy->base +
-				 REG_DSI_10nm_PHY_CMN_CLK_CFG1,
-				 0, 2, 0, NULL);
-	if (IS_ERR(hw)) {
-		ret = PTR_ERR(hw);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_pclk_mux", pll_10nm->phy->id);
+
+	pclk_mux = devm_clk_hw_register_mux_parent_hws(dev, clk_name,
+			((const struct clk_hw *[]){
+				pll_bit,
+				pll_by_2_bit,
+				pll_out_div,
+				pll_post_out_div,
+			}), 4, 0, pll_10nm->phy->base +
+				REG_DSI_10nm_PHY_CMN_CLK_CFG1, 0, 2, 0, NULL);
+	if (IS_ERR(pclk_mux)) {
+		ret = PTR_ERR(pclk_mux);
 		goto fail;
 	}
 
-	snprintf(clk_name, 32, "dsi%d_phy_pll_out_dsiclk", pll_10nm->phy->id);
-	snprintf(parent, 32, "dsi%d_pclk_mux", pll_10nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_phy_pll_out_dsiclk", pll_10nm->phy->id);
 
 	/* PIX CLK DIV : DIV_CTRL_7_4*/
-	hw = devm_clk_hw_register_divider(dev, clk_name, parent,
-				     0, pll_10nm->phy->base +
-					REG_DSI_10nm_PHY_CMN_CLK_CFG0,
-				     4, 4, CLK_DIVIDER_ONE_BASED,
-				     &pll_10nm->postdiv_lock);
+	hw = devm_clk_hw_register_divider_parent_hw(dev, clk_name, pclk_mux,
+			0, pll_10nm->phy->base + REG_DSI_10nm_PHY_CMN_CLK_CFG0,
+			4, 4, CLK_DIVIDER_ONE_BASED, &pll_10nm->postdiv_lock);
 	if (IS_ERR(hw)) {
 		ret = PTR_ERR(hw);
 		goto fail;
@@ -1028,14 +1019,14 @@ static int dsi_10nm_phy_parse_dt(struct msm_dsi_phy *phy)
 	return 0;
 }
 
+static const struct regulator_bulk_data dsi_phy_10nm_regulators[] = {
+	{ .supply = "vdds", .init_load_uA = 36000 },
+};
+
 const struct msm_dsi_phy_cfg dsi_phy_10nm_cfgs = {
 	.has_phy_lane = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vdds", 36000, 32},
-		},
-	},
+	.regulator_data = dsi_phy_10nm_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_10nm_regulators),
 	.ops = {
 		.enable = dsi_10nm_phy_enable,
 		.disable = dsi_10nm_phy_disable,
@@ -1052,12 +1043,8 @@ const struct msm_dsi_phy_cfg dsi_phy_10nm_cfgs = {
 
 const struct msm_dsi_phy_cfg dsi_phy_10nm_8998_cfgs = {
 	.has_phy_lane = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vdds", 36000, 32},
-		},
-	},
+	.regulator_data = dsi_phy_10nm_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_10nm_regulators),
 	.ops = {
 		.enable = dsi_10nm_phy_enable,
 		.disable = dsi_10nm_phy_disable,
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c
index 8199c53567f4..0f8f4ca46429 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_14nm.c
@@ -711,7 +711,7 @@ static int dsi_14nm_pll_restore_state(struct msm_dsi_phy *phy)
 					cached_state->vco_rate, 0);
 	if (ret) {
 		DRM_DEV_ERROR(&pll_14nm->phy->pdev->dev,
-			"restore vco rate failed. ret=%d\n", ret);
+			      "restore vco rate failed. ret=%d\n", ret);
 		return ret;
 	}
 
@@ -764,14 +764,14 @@ static int dsi_14nm_set_usecase(struct msm_dsi_phy *phy)
 
 static struct clk_hw *pll_14nm_postdiv_register(struct dsi_pll_14nm *pll_14nm,
 						const char *name,
-						const char *parent_name,
+						const struct clk_hw *parent_hw,
 						unsigned long flags,
 						u8 shift)
 {
 	struct dsi_pll_14nm_postdiv *pll_postdiv;
 	struct device *dev = &pll_14nm->phy->pdev->dev;
 	struct clk_init_data postdiv_init = {
-		.parent_names = (const char *[]) { parent_name },
+		.parent_hws = (const struct clk_hw *[]) { parent_hw },
 		.num_parents = 1,
 		.name = name,
 		.flags = flags,
@@ -800,72 +800,70 @@ static struct clk_hw *pll_14nm_postdiv_register(struct dsi_pll_14nm *pll_14nm,
 
 static int pll_14nm_register(struct dsi_pll_14nm *pll_14nm, struct clk_hw **provided_clocks)
 {
-	char clk_name[32], parent[32], vco_name[32];
+	char clk_name[32];
 	struct clk_init_data vco_init = {
 		.parent_data = &(const struct clk_parent_data) {
 			.fw_name = "ref",
 		},
 		.num_parents = 1,
-		.name = vco_name,
+		.name = clk_name,
 		.flags = CLK_IGNORE_UNUSED,
 		.ops = &clk_ops_dsi_pll_14nm_vco,
 	};
 	struct device *dev = &pll_14nm->phy->pdev->dev;
-	struct clk_hw *hw;
+	struct clk_hw *hw, *n1_postdiv, *n1_postdivby2;
 	int ret;
 
 	DBG("DSI%d", pll_14nm->phy->id);
 
-	snprintf(vco_name, 32, "dsi%dvco_clk", pll_14nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dvco_clk", pll_14nm->phy->id);
 	pll_14nm->clk_hw.init = &vco_init;
 
 	ret = devm_clk_hw_register(dev, &pll_14nm->clk_hw);
 	if (ret)
 		return ret;
 
-	snprintf(clk_name, 32, "dsi%dn1_postdiv_clk", pll_14nm->phy->id);
-	snprintf(parent, 32, "dsi%dvco_clk", pll_14nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dn1_postdiv_clk", pll_14nm->phy->id);
 
 	/* N1 postdiv, bits 0-3 in REG_DSI_14nm_PHY_CMN_CLK_CFG0 */
-	hw = pll_14nm_postdiv_register(pll_14nm, clk_name, parent,
-				       CLK_SET_RATE_PARENT, 0);
-	if (IS_ERR(hw))
-		return PTR_ERR(hw);
+	n1_postdiv = pll_14nm_postdiv_register(pll_14nm, clk_name,
+			&pll_14nm->clk_hw, CLK_SET_RATE_PARENT, 0);
+	if (IS_ERR(n1_postdiv))
+		return PTR_ERR(n1_postdiv);
 
-	snprintf(clk_name, 32, "dsi%dpllbyte", pll_14nm->phy->id);
-	snprintf(parent, 32, "dsi%dn1_postdiv_clk", pll_14nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dpllbyte", pll_14nm->phy->id);
 
 	/* DSI Byte clock = VCO_CLK / N1 / 8 */
-	hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent,
-					  CLK_SET_RATE_PARENT, 1, 8);
+	hw = devm_clk_hw_register_fixed_factor_parent_hw(dev, clk_name,
+			n1_postdiv, CLK_SET_RATE_PARENT, 1, 8);
 	if (IS_ERR(hw))
 		return PTR_ERR(hw);
 
 	provided_clocks[DSI_BYTE_PLL_CLK] = hw;
 
-	snprintf(clk_name, 32, "dsi%dn1_postdivby2_clk", pll_14nm->phy->id);
-	snprintf(parent, 32, "dsi%dn1_postdiv_clk", pll_14nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dn1_postdivby2_clk", pll_14nm->phy->id);
 
 	/*
 	 * Skip the mux for now, force DSICLK_SEL to 1, Add a /2 divider
 	 * on the way. Don't let it set parent.
 	 */
-	hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, 0, 1, 2);
-	if (IS_ERR(hw))
-		return PTR_ERR(hw);
+	n1_postdivby2 = devm_clk_hw_register_fixed_factor_parent_hw(dev,
+			clk_name, n1_postdiv, 0, 1, 2);
+	if (IS_ERR(n1_postdivby2))
+		return PTR_ERR(n1_postdivby2);
 
-	snprintf(clk_name, 32, "dsi%dpll", pll_14nm->phy->id);
-	snprintf(parent, 32, "dsi%dn1_postdivby2_clk", pll_14nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dpll", pll_14nm->phy->id);
 
 	/* DSI pixel clock = VCO_CLK / N1 / 2 / N2
 	 * This is the output of N2 post-divider, bits 4-7 in
 	 * REG_DSI_14nm_PHY_CMN_CLK_CFG0. Don't let it set parent.
 	 */
-	hw = pll_14nm_postdiv_register(pll_14nm, clk_name, parent, 0, 4);
+	hw = pll_14nm_postdiv_register(pll_14nm, clk_name, n1_postdivby2,
+			0, 4);
 	if (IS_ERR(hw))
 		return PTR_ERR(hw);
 
-	provided_clocks[DSI_PIXEL_PLL_CLK]	= hw;
+	provided_clocks[DSI_PIXEL_PLL_CLK] = hw;
 
 	return 0;
 }
@@ -952,7 +950,8 @@ static int dsi_14nm_phy_enable(struct msm_dsi_phy *phy,
 
 	if (msm_dsi_dphy_timing_calc_v2(timing, clk_req)) {
 		DRM_DEV_ERROR(&phy->pdev->dev,
-			"%s: D-PHY timing calculation failed\n", __func__);
+			      "%s: D-PHY timing calculation failed\n",
+			      __func__);
 		return -EINVAL;
 	}
 
@@ -1005,7 +1004,7 @@ static int dsi_14nm_phy_enable(struct msm_dsi_phy *phy,
 	ret = dsi_14nm_set_usecase(phy);
 	if (ret) {
 		DRM_DEV_ERROR(&phy->pdev->dev, "%s: set pll usecase failed, %d\n",
-			__func__, ret);
+			      __func__, ret);
 		return ret;
 	}
 
@@ -1024,14 +1023,18 @@ static void dsi_14nm_phy_disable(struct msm_dsi_phy *phy)
 	wmb();
 }
 
+static const struct regulator_bulk_data dsi_phy_14nm_17mA_regulators[] = {
+	{ .supply = "vcca", .init_load_uA = 17000 },
+};
+
+static const struct regulator_bulk_data dsi_phy_14nm_73p4mA_regulators[] = {
+	{ .supply = "vcca", .init_load_uA = 73400 },
+};
+
 const struct msm_dsi_phy_cfg dsi_phy_14nm_cfgs = {
 	.has_phy_lane = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vcca", 17000, 32},
-		},
-	},
+	.regulator_data = dsi_phy_14nm_17mA_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_14nm_17mA_regulators),
 	.ops = {
 		.enable = dsi_14nm_phy_enable,
 		.disable = dsi_14nm_phy_disable,
@@ -1047,12 +1050,8 @@ const struct msm_dsi_phy_cfg dsi_phy_14nm_cfgs = {
 
 const struct msm_dsi_phy_cfg dsi_phy_14nm_660_cfgs = {
 	.has_phy_lane = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vcca", 73400, 32},
-		},
-	},
+	.regulator_data = dsi_phy_14nm_73p4mA_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_14nm_73p4mA_regulators),
 	.ops = {
 		.enable = dsi_14nm_phy_enable,
 		.disable = dsi_14nm_phy_disable,
@@ -1068,12 +1067,8 @@ const struct msm_dsi_phy_cfg dsi_phy_14nm_660_cfgs = {
 
 const struct msm_dsi_phy_cfg dsi_phy_14nm_8953_cfgs = {
 	.has_phy_lane = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vcca", 17000, 32},
-		},
-	},
+	.regulator_data = dsi_phy_14nm_17mA_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_14nm_17mA_regulators),
 	.ops = {
 		.enable = dsi_14nm_phy_enable,
 		.disable = dsi_14nm_phy_disable,
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c
index ee7c418a1c29..c9752b991744 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c
@@ -129,15 +129,15 @@ static void dsi_20nm_phy_disable(struct msm_dsi_phy *phy)
 	dsi_20nm_phy_regulator_ctrl(phy, false);
 }
 
+static const struct regulator_bulk_data dsi_phy_20nm_regulators[] = {
+	{ .supply = "vddio", .init_load_uA = 100000 },	/* 1.8 V */
+	{ .supply = "vcca", .init_load_uA = 10000 },	/* 1.0 V */
+};
+
 const struct msm_dsi_phy_cfg dsi_phy_20nm_cfgs = {
 	.has_phy_regulator = true,
-	.reg_cfg = {
-		.num = 2,
-		.regs = {
-			{"vddio", 100000, 100},	/* 1.8 V */
-			{"vcca", 10000, 100},	/* 1.0 V */
-		},
-	},
+	.regulator_data = dsi_phy_20nm_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_20nm_regulators),
 	.ops = {
 		.enable = dsi_20nm_phy_enable,
 		.disable = dsi_20nm_phy_disable,
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c
index 48eab80b548e..4c1bf55c5f38 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c
@@ -104,7 +104,7 @@ static void pll_28nm_software_reset(struct dsi_pll_28nm *pll_28nm)
 	 * reset bit off and back on.
 	 */
 	dsi_phy_write_udelay(base + REG_DSI_28nm_PHY_PLL_TEST_CFG,
-			DSI_28nm_PHY_PLL_TEST_CFG_PLL_SW_RESET, 1);
+			     DSI_28nm_PHY_PLL_TEST_CFG_PLL_SW_RESET, 1);
 	dsi_phy_write_udelay(base + REG_DSI_28nm_PHY_PLL_TEST_CFG, 0x00, 1);
 }
 
@@ -201,9 +201,9 @@ static int dsi_pll_28nm_clk_set_rate(struct clk_hw *hw, unsigned long rate,
 
 	dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_SDM_CFG1, sdm_cfg1);
 	dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_SDM_CFG2,
-		DSI_28nm_PHY_PLL_SDM_CFG2_FREQ_SEED_7_0(sdm_cfg2));
+		      DSI_28nm_PHY_PLL_SDM_CFG2_FREQ_SEED_7_0(sdm_cfg2));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_SDM_CFG3,
-		DSI_28nm_PHY_PLL_SDM_CFG3_FREQ_SEED_15_8(sdm_cfg3));
+		      DSI_28nm_PHY_PLL_SDM_CFG3_FREQ_SEED_15_8(sdm_cfg3));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_SDM_CFG4, 0x00);
 
 	/* Add hardware recommended delay for correct PLL configuration */
@@ -316,12 +316,12 @@ static int _dsi_pll_28nm_vco_prepare_hpm(struct dsi_pll_28nm *pll_28nm)
 	for (i = 0; i < 2; i++) {
 		/* DSI Uniphy lock detect setting */
 		dsi_phy_write_udelay(base + REG_DSI_28nm_PHY_PLL_LKDET_CFG2,
-				0x0c, 100);
+				     0x0c, 100);
 		dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_LKDET_CFG2, 0x0d);
 
 		/* poll for PLL ready status */
-		locked = pll_28nm_poll_for_ready(pll_28nm,
-						max_reads, timeout_us);
+		locked = pll_28nm_poll_for_ready(pll_28nm, max_reads,
+						 timeout_us);
 		if (locked)
 			break;
 
@@ -508,28 +508,28 @@ static int dsi_28nm_pll_restore_state(struct msm_dsi_phy *phy)
 	}
 
 	dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_POSTDIV3_CFG,
-			cached_state->postdiv3);
+		      cached_state->postdiv3);
 	dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_POSTDIV1_CFG,
-			cached_state->postdiv1);
+		      cached_state->postdiv1);
 	dsi_phy_write(base + REG_DSI_28nm_PHY_PLL_VREG_CFG,
-			cached_state->byte_mux);
+		      cached_state->byte_mux);
 
 	return 0;
 }
 
 static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **provided_clocks)
 {
-	char clk_name[32], parent1[32], parent2[32], vco_name[32];
+	char clk_name[32];
 	struct clk_init_data vco_init = {
 		.parent_data = &(const struct clk_parent_data) {
 			.fw_name = "ref", .name = "xo",
 		},
 		.num_parents = 1,
-		.name = vco_name,
+		.name = clk_name,
 		.flags = CLK_IGNORE_UNUSED,
 	};
 	struct device *dev = &pll_28nm->phy->pdev->dev;
-	struct clk_hw *hw;
+	struct clk_hw *hw, *analog_postdiv, *indirect_path_div2, *byte_mux;
 	int ret;
 
 	DBG("%d", pll_28nm->phy->id);
@@ -539,55 +539,49 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **prov
 	else
 		vco_init.ops = &clk_ops_dsi_pll_28nm_vco_hpm;
 
-	snprintf(vco_name, 32, "dsi%dvco_clk", pll_28nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dvco_clk", pll_28nm->phy->id);
 	pll_28nm->clk_hw.init = &vco_init;
 	ret = devm_clk_hw_register(dev, &pll_28nm->clk_hw);
 	if (ret)
 		return ret;
 
-	snprintf(clk_name, 32, "dsi%danalog_postdiv_clk", pll_28nm->phy->id);
-	snprintf(parent1, 32, "dsi%dvco_clk", pll_28nm->phy->id);
-	hw = devm_clk_hw_register_divider(dev, clk_name,
-			parent1, CLK_SET_RATE_PARENT,
+	snprintf(clk_name, sizeof(clk_name), "dsi%danalog_postdiv_clk", pll_28nm->phy->id);
+	analog_postdiv = devm_clk_hw_register_divider_parent_hw(dev, clk_name,
+			&pll_28nm->clk_hw, CLK_SET_RATE_PARENT,
 			pll_28nm->phy->pll_base +
-			REG_DSI_28nm_PHY_PLL_POSTDIV1_CFG,
+				REG_DSI_28nm_PHY_PLL_POSTDIV1_CFG,
 			0, 4, 0, NULL);
-	if (IS_ERR(hw))
-		return PTR_ERR(hw);
-
-	snprintf(clk_name, 32, "dsi%dindirect_path_div2_clk", pll_28nm->phy->id);
-	snprintf(parent1, 32, "dsi%danalog_postdiv_clk", pll_28nm->phy->id);
-	hw = devm_clk_hw_register_fixed_factor(dev, clk_name,
-			parent1, CLK_SET_RATE_PARENT,
-			1, 2);
-	if (IS_ERR(hw))
-		return PTR_ERR(hw);
-
-	snprintf(clk_name, 32, "dsi%dpll", pll_28nm->phy->id);
-	snprintf(parent1, 32, "dsi%dvco_clk", pll_28nm->phy->id);
-	hw = devm_clk_hw_register_divider(dev, clk_name,
-				parent1, 0, pll_28nm->phy->pll_base +
+	if (IS_ERR(analog_postdiv))
+		return PTR_ERR(analog_postdiv);
+
+	snprintf(clk_name, sizeof(clk_name), "dsi%dindirect_path_div2_clk", pll_28nm->phy->id);
+	indirect_path_div2 = devm_clk_hw_register_fixed_factor_parent_hw(dev,
+			clk_name, analog_postdiv, CLK_SET_RATE_PARENT, 1, 2);
+	if (IS_ERR(indirect_path_div2))
+		return PTR_ERR(indirect_path_div2);
+
+	snprintf(clk_name, sizeof(clk_name), "dsi%dpll", pll_28nm->phy->id);
+	hw = devm_clk_hw_register_divider_parent_hw(dev, clk_name,
+			&pll_28nm->clk_hw, 0, pll_28nm->phy->pll_base +
 				REG_DSI_28nm_PHY_PLL_POSTDIV3_CFG,
-				0, 8, 0, NULL);
+			0, 8, 0, NULL);
 	if (IS_ERR(hw))
 		return PTR_ERR(hw);
 	provided_clocks[DSI_PIXEL_PLL_CLK] = hw;
 
-	snprintf(clk_name, 32, "dsi%dbyte_mux", pll_28nm->phy->id);
-	snprintf(parent1, 32, "dsi%dvco_clk", pll_28nm->phy->id);
-	snprintf(parent2, 32, "dsi%dindirect_path_div2_clk", pll_28nm->phy->id);
-	hw = devm_clk_hw_register_mux(dev, clk_name,
-			((const char *[]){
-				parent1, parent2
+	snprintf(clk_name, sizeof(clk_name), "dsi%dbyte_mux", pll_28nm->phy->id);
+	byte_mux = devm_clk_hw_register_mux_parent_hws(dev, clk_name,
+			((const struct clk_hw *[]){
+				&pll_28nm->clk_hw,
+				indirect_path_div2,
 			}), 2, CLK_SET_RATE_PARENT, pll_28nm->phy->pll_base +
-			REG_DSI_28nm_PHY_PLL_VREG_CFG, 1, 1, 0, NULL);
-	if (IS_ERR(hw))
-		return PTR_ERR(hw);
+				REG_DSI_28nm_PHY_PLL_VREG_CFG, 1, 1, 0, NULL);
+	if (IS_ERR(byte_mux))
+		return PTR_ERR(byte_mux);
 
-	snprintf(clk_name, 32, "dsi%dpllbyte", pll_28nm->phy->id);
-	snprintf(parent1, 32, "dsi%dbyte_mux", pll_28nm->phy->id);
-	hw = devm_clk_hw_register_fixed_factor(dev, clk_name,
-				parent1, CLK_SET_RATE_PARENT, 1, 4);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dpllbyte", pll_28nm->phy->id);
+	hw = devm_clk_hw_register_fixed_factor_parent_hw(dev, clk_name,
+			byte_mux, CLK_SET_RATE_PARENT, 1, 4);
 	if (IS_ERR(hw))
 		return PTR_ERR(hw);
 	provided_clocks[DSI_BYTE_PLL_CLK] = hw;
@@ -627,31 +621,31 @@ static void dsi_28nm_dphy_set_timing(struct msm_dsi_phy *phy,
 	void __iomem *base = phy->base;
 
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_0,
-		DSI_28nm_PHY_TIMING_CTRL_0_CLK_ZERO(timing->clk_zero));
+		      DSI_28nm_PHY_TIMING_CTRL_0_CLK_ZERO(timing->clk_zero));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_1,
-		DSI_28nm_PHY_TIMING_CTRL_1_CLK_TRAIL(timing->clk_trail));
+		      DSI_28nm_PHY_TIMING_CTRL_1_CLK_TRAIL(timing->clk_trail));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_2,
-		DSI_28nm_PHY_TIMING_CTRL_2_CLK_PREPARE(timing->clk_prepare));
+		      DSI_28nm_PHY_TIMING_CTRL_2_CLK_PREPARE(timing->clk_prepare));
 	if (timing->clk_zero & BIT(8))
 		dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_3,
-			DSI_28nm_PHY_TIMING_CTRL_3_CLK_ZERO_8);
+			      DSI_28nm_PHY_TIMING_CTRL_3_CLK_ZERO_8);
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_4,
-		DSI_28nm_PHY_TIMING_CTRL_4_HS_EXIT(timing->hs_exit));
+		      DSI_28nm_PHY_TIMING_CTRL_4_HS_EXIT(timing->hs_exit));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_5,
-		DSI_28nm_PHY_TIMING_CTRL_5_HS_ZERO(timing->hs_zero));
+		      DSI_28nm_PHY_TIMING_CTRL_5_HS_ZERO(timing->hs_zero));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_6,
-		DSI_28nm_PHY_TIMING_CTRL_6_HS_PREPARE(timing->hs_prepare));
+		      DSI_28nm_PHY_TIMING_CTRL_6_HS_PREPARE(timing->hs_prepare));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_7,
-		DSI_28nm_PHY_TIMING_CTRL_7_HS_TRAIL(timing->hs_trail));
+		      DSI_28nm_PHY_TIMING_CTRL_7_HS_TRAIL(timing->hs_trail));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_8,
-		DSI_28nm_PHY_TIMING_CTRL_8_HS_RQST(timing->hs_rqst));
+		      DSI_28nm_PHY_TIMING_CTRL_8_HS_RQST(timing->hs_rqst));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_9,
-		DSI_28nm_PHY_TIMING_CTRL_9_TA_GO(timing->ta_go) |
-		DSI_28nm_PHY_TIMING_CTRL_9_TA_SURE(timing->ta_sure));
+		      DSI_28nm_PHY_TIMING_CTRL_9_TA_GO(timing->ta_go) |
+		      DSI_28nm_PHY_TIMING_CTRL_9_TA_SURE(timing->ta_sure));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_10,
-		DSI_28nm_PHY_TIMING_CTRL_10_TA_GET(timing->ta_get));
+		      DSI_28nm_PHY_TIMING_CTRL_10_TA_GET(timing->ta_get));
 	dsi_phy_write(base + REG_DSI_28nm_PHY_TIMING_CTRL_11,
-		DSI_28nm_PHY_TIMING_CTRL_11_TRIG3_CMD(0));
+		      DSI_28nm_PHY_TIMING_CTRL_11_TRIG3_CMD(0));
 }
 
 static void dsi_28nm_phy_regulator_enable_dcdc(struct msm_dsi_phy *phy)
@@ -713,7 +707,8 @@ static int dsi_28nm_phy_enable(struct msm_dsi_phy *phy,
 
 	if (msm_dsi_dphy_timing_calc(timing, clk_req)) {
 		DRM_DEV_ERROR(&phy->pdev->dev,
-			"%s: D-PHY timing calculation failed\n", __func__);
+			      "%s: D-PHY timing calculation failed\n",
+			      __func__);
 		return -EINVAL;
 	}
 
@@ -769,14 +764,14 @@ static void dsi_28nm_phy_disable(struct msm_dsi_phy *phy)
 	wmb();
 }
 
+static const struct regulator_bulk_data dsi_phy_28nm_regulators[] = {
+	{ .supply = "vddio", .init_load_uA = 100000 },
+};
+
 const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_cfgs = {
 	.has_phy_regulator = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vddio", 100000, 100},
-		},
-	},
+	.regulator_data = dsi_phy_28nm_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_28nm_regulators),
 	.ops = {
 		.enable = dsi_28nm_phy_enable,
 		.disable = dsi_28nm_phy_disable,
@@ -792,12 +787,8 @@ const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_cfgs = {
 
 const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_famb_cfgs = {
 	.has_phy_regulator = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vddio", 100000, 100},
-		},
-	},
+	.regulator_data = dsi_phy_28nm_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_28nm_regulators),
 	.ops = {
 		.enable = dsi_28nm_phy_enable,
 		.disable = dsi_28nm_phy_disable,
@@ -813,12 +804,8 @@ const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_famb_cfgs = {
 
 const struct msm_dsi_phy_cfg dsi_phy_28nm_lp_cfgs = {
 	.has_phy_regulator = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vddio", 100000, 100},	/* 1.8 V */
-		},
-	},
+	.regulator_data = dsi_phy_28nm_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_28nm_regulators),
 	.ops = {
 		.enable = dsi_28nm_phy_enable,
 		.disable = dsi_28nm_phy_disable,
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
index fc56cdcc9ad6..26c08047e20c 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
@@ -104,29 +104,29 @@ static int dsi_pll_28nm_clk_set_rate(struct clk_hw *hw, unsigned long rate,
 	fb_divider = (temp * VCO_PREF_DIV_RATIO) / val;
 	fb_divider = fb_divider / 2 - 1;
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_1,
-			fb_divider & 0xff);
+		      fb_divider & 0xff);
 
 	val = dsi_phy_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_2);
 
 	val |= (fb_divider >> 8) & 0x07;
 
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_2,
-			val);
+		      val);
 
 	val = dsi_phy_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_3);
 
 	val |= (VCO_PREF_DIV_RATIO - 1) & 0x3f;
 
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_3,
-			val);
+		      val);
 
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_6,
-			0xf);
+		      0xf);
 
 	val = dsi_phy_read(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8);
 	val |= 0x7 << 4;
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8,
-			val);
+		      val);
 
 	return 0;
 }
@@ -206,7 +206,7 @@ static int dsi_pll_28nm_vco_prepare(struct clk_hw *hw)
 
 	/* enable the PLL */
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_0,
-			DSI_28nm_8960_PHY_PLL_CTRL_0_ENABLE);
+		      DSI_28nm_8960_PHY_PLL_CTRL_0_ENABLE);
 
 	locked = pll_28nm_poll_for_ready(pll_28nm, max_reads, timeout_us);
 
@@ -367,23 +367,23 @@ static int dsi_28nm_pll_restore_state(struct msm_dsi_phy *phy)
 					cached_state->vco_rate, 0);
 	if (ret) {
 		DRM_DEV_ERROR(&pll_28nm->phy->pdev->dev,
-			"restore vco rate failed. ret=%d\n", ret);
+			      "restore vco rate failed. ret=%d\n", ret);
 		return ret;
 	}
 
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_10,
-			cached_state->postdiv3);
+		      cached_state->postdiv3);
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_9,
-			cached_state->postdiv2);
+		      cached_state->postdiv2);
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_PLL_CTRL_8,
-			cached_state->postdiv1);
+		      cached_state->postdiv1);
 
 	return 0;
 }
 
 static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **provided_clocks)
 {
-	char *clk_name, *parent_name, *vco_name;
+	char clk_name[32];
 	struct clk_init_data vco_init = {
 		.parent_data = &(const struct clk_parent_data) {
 			.fw_name = "ref",
@@ -404,20 +404,8 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **prov
 	if (!bytediv)
 		return -ENOMEM;
 
-	vco_name = devm_kzalloc(dev, 32, GFP_KERNEL);
-	if (!vco_name)
-		return -ENOMEM;
-
-	parent_name = devm_kzalloc(dev, 32, GFP_KERNEL);
-	if (!parent_name)
-		return -ENOMEM;
-
-	clk_name = devm_kzalloc(dev, 32, GFP_KERNEL);
-	if (!clk_name)
-		return -ENOMEM;
-
-	snprintf(vco_name, 32, "dsi%dvco_clk", pll_28nm->phy->id);
-	vco_init.name = vco_name;
+	snprintf(clk_name, sizeof(clk_name), "dsi%dvco_clk", pll_28nm->phy->id);
+	vco_init.name = clk_name;
 
 	pll_28nm->clk_hw.init = &vco_init;
 
@@ -429,13 +417,14 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **prov
 	bytediv->hw.init = &bytediv_init;
 	bytediv->reg = pll_28nm->phy->pll_base + REG_DSI_28nm_8960_PHY_PLL_CTRL_9;
 
-	snprintf(parent_name, 32, "dsi%dvco_clk", pll_28nm->phy->id);
-	snprintf(clk_name, 32, "dsi%dpllbyte", pll_28nm->phy->id + 1);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dpllbyte", pll_28nm->phy->id + 1);
 
 	bytediv_init.name = clk_name;
 	bytediv_init.ops = &clk_bytediv_ops;
 	bytediv_init.flags = CLK_SET_RATE_PARENT;
-	bytediv_init.parent_names = (const char * const *) &parent_name;
+	bytediv_init.parent_hws = (const struct clk_hw*[]){
+		&pll_28nm->clk_hw,
+	};
 	bytediv_init.num_parents = 1;
 
 	/* DIV2 */
@@ -444,12 +433,12 @@ static int pll_28nm_register(struct dsi_pll_28nm *pll_28nm, struct clk_hw **prov
 		return ret;
 	provided_clocks[DSI_BYTE_PLL_CLK] = &bytediv->hw;
 
-	snprintf(clk_name, 32, "dsi%dpll", pll_28nm->phy->id + 1);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dpll", pll_28nm->phy->id + 1);
 	/* DIV3 */
-	hw = devm_clk_hw_register_divider(dev, clk_name,
-				parent_name, 0, pll_28nm->phy->pll_base +
+	hw = devm_clk_hw_register_divider_parent_hw(dev, clk_name,
+			&pll_28nm->clk_hw, 0, pll_28nm->phy->pll_base +
 				REG_DSI_28nm_8960_PHY_PLL_CTRL_10,
-				0, 8, 0, NULL);
+			0, 8, 0, NULL);
 	if (IS_ERR(hw))
 		return PTR_ERR(hw);
 	provided_clocks[DSI_PIXEL_PLL_CLK] = hw;
@@ -489,29 +478,29 @@ static void dsi_28nm_dphy_set_timing(struct msm_dsi_phy *phy,
 	void __iomem *base = phy->base;
 
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_0,
-		DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO(timing->clk_zero));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_0_CLK_ZERO(timing->clk_zero));
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_1,
-		DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL(timing->clk_trail));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_1_CLK_TRAIL(timing->clk_trail));
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_2,
-		DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE(timing->clk_prepare));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_2_CLK_PREPARE(timing->clk_prepare));
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_3, 0x0);
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_4,
-		DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT(timing->hs_exit));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_4_HS_EXIT(timing->hs_exit));
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_5,
-		DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO(timing->hs_zero));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_5_HS_ZERO(timing->hs_zero));
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_6,
-		DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE(timing->hs_prepare));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_6_HS_PREPARE(timing->hs_prepare));
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_7,
-		DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL(timing->hs_trail));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_7_HS_TRAIL(timing->hs_trail));
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_8,
-		DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST(timing->hs_rqst));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_8_HS_RQST(timing->hs_rqst));
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_9,
-		DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO(timing->ta_go) |
-		DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE(timing->ta_sure));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_GO(timing->ta_go) |
+		      DSI_28nm_8960_PHY_TIMING_CTRL_9_TA_SURE(timing->ta_sure));
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_10,
-		DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET(timing->ta_get));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_10_TA_GET(timing->ta_get));
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_TIMING_CTRL_11,
-		DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD(0));
+		      DSI_28nm_8960_PHY_TIMING_CTRL_11_TRIG3_CMD(0));
 }
 
 static void dsi_28nm_phy_regulator_init(struct msm_dsi_phy *phy)
@@ -523,7 +512,7 @@ static void dsi_28nm_phy_regulator_init(struct msm_dsi_phy *phy)
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_2, 1);
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_3, 0);
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CTRL_4,
-		0x100);
+		      0x100);
 }
 
 static void dsi_28nm_phy_regulator_ctrl(struct msm_dsi_phy *phy)
@@ -544,7 +533,7 @@ static void dsi_28nm_phy_calibration(struct msm_dsi_phy *phy)
 	int i = 5000;
 
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_REGULATOR_CAL_PWR_CFG,
-			0x3);
+		      0x3);
 
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_SW_CFG_2, 0x0);
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_MISC_CAL_HW_CFG_1, 0x5a);
@@ -577,11 +566,11 @@ static void dsi_28nm_phy_lane_config(struct msm_dsi_phy *phy)
 		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_CFG_1(i), 0x45);
 		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_CFG_2(i), 0x00);
 		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_TEST_DATAPATH(i),
-			0x00);
+			      0x00);
 		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_TEST_STR_0(i),
-			0x01);
+			      0x01);
 		dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LN_TEST_STR_1(i),
-			0x66);
+			      0x66);
 	}
 
 	dsi_phy_write(base + REG_DSI_28nm_8960_PHY_LNCK_CFG_0, 0x40);
@@ -602,7 +591,8 @@ static int dsi_28nm_phy_enable(struct msm_dsi_phy *phy,
 
 	if (msm_dsi_dphy_timing_calc(timing, clk_req)) {
 		DRM_DEV_ERROR(&phy->pdev->dev,
-			"%s: D-PHY timing calculation failed\n", __func__);
+			      "%s: D-PHY timing calculation failed\n",
+			      __func__);
 		return -EINVAL;
 	}
 
@@ -648,14 +638,14 @@ static void dsi_28nm_phy_disable(struct msm_dsi_phy *phy)
 	wmb();
 }
 
+static const struct regulator_bulk_data dsi_phy_28nm_8960_regulators[] = {
+	{ .supply = "vddio", .init_load_uA = 100000 },	/* 1.8 V */
+};
+
 const struct msm_dsi_phy_cfg dsi_phy_28nm_8960_cfgs = {
 	.has_phy_regulator = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vddio", 100000, 100},	/* 1.8 V */
-		},
-	},
+	.regulator_data = dsi_phy_28nm_8960_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_28nm_8960_regulators),
 	.ops = {
 		.enable = dsi_28nm_phy_enable,
 		.disable = dsi_28nm_phy_disable,
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
index 66ed1919a1db..9e7fa7d88ead 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
@@ -176,19 +176,19 @@ static void dsi_pll_ssc_commit(struct dsi_pll_7nm *pll, struct dsi_pll_config *c
 		pr_debug("SSC is enabled\n");
 
 		dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_STEPSIZE_LOW_1,
-			  config->ssc_stepsize & 0xff);
+			      config->ssc_stepsize & 0xff);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_STEPSIZE_HIGH_1,
-			  config->ssc_stepsize >> 8);
+			      config->ssc_stepsize >> 8);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_DIV_PER_LOW_1,
-			  config->ssc_div_per & 0xff);
+			      config->ssc_div_per & 0xff);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_DIV_PER_HIGH_1,
-			  config->ssc_div_per >> 8);
+			      config->ssc_div_per >> 8);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_ADJPER_LOW_1,
-			  config->ssc_adj_per & 0xff);
+			      config->ssc_adj_per & 0xff);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_ADJPER_HIGH_1,
-			  config->ssc_adj_per >> 8);
+			      config->ssc_adj_per >> 8);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_SSC_CONTROL,
-			  SSC_EN | (config->ssc_center ? SSC_CENTER : 0));
+			      SSC_EN | (config->ssc_center ? SSC_CENTER : 0));
 	}
 }
 
@@ -208,7 +208,7 @@ static void dsi_pll_config_hzindep_reg(struct dsi_pll_7nm *pll)
 	}
 
 	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_FIVE_1,
-		  analog_controls_five_1);
+		      analog_controls_five_1);
 	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_VCO_CONFIG_1, vco_config_1);
 	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_FIVE, 0x01);
 	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_ANALOG_CONTROLS_TWO, 0x03);
@@ -245,17 +245,20 @@ static void dsi_pll_commit(struct dsi_pll_7nm *pll, struct dsi_pll_config *confi
 	void __iomem *base = pll->phy->pll_base;
 
 	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CORE_INPUT_OVERRIDE, 0x12);
-	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_DECIMAL_DIV_START_1, config->decimal_div_start);
+	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_DECIMAL_DIV_START_1,
+		      config->decimal_div_start);
 	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_LOW_1,
-		  config->frac_div_start & 0xff);
+		      config->frac_div_start & 0xff);
 	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_MID_1,
-		  (config->frac_div_start & 0xff00) >> 8);
+		      (config->frac_div_start & 0xff00) >> 8);
 	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_FRAC_DIV_START_HIGH_1,
-		  (config->frac_div_start & 0x30000) >> 16);
+		      (config->frac_div_start & 0x30000) >> 16);
 	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_PLL_LOCKDET_RATE_1, 0x40);
 	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_PLL_LOCK_DELAY, 0x06);
-	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CMODE_1, pll->phy->cphy_mode ? 0x00 : 0x10);
-	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CLOCK_INVERTERS, config->pll_clock_inverters);
+	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CMODE_1,
+		      pll->phy->cphy_mode ? 0x00 : 0x10);
+	dsi_phy_write(base + REG_DSI_7nm_PHY_PLL_CLOCK_INVERTERS,
+		      config->pll_clock_inverters);
 }
 
 static int dsi_pll_7nm_vco_set_rate(struct clk_hw *hw, unsigned long rate,
@@ -341,7 +344,7 @@ static void dsi_pll_enable_global_clk(struct dsi_pll_7nm *pll)
 
 	data = dsi_phy_read(pll->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1);
 	dsi_phy_write(pll->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1,
-		  data | BIT(5) | BIT(4));
+		      data | BIT(5) | BIT(4));
 }
 
 static void dsi_pll_phy_dig_reset(struct dsi_pll_7nm *pll)
@@ -500,7 +503,7 @@ static void dsi_7nm_pll_save_state(struct msm_dsi_phy *phy)
 	u32 cmn_clk_cfg0, cmn_clk_cfg1;
 
 	cached->pll_out_div = dsi_phy_read(pll_7nm->phy->pll_base +
-				       REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE);
+			REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE);
 	cached->pll_out_div &= 0x3;
 
 	cmn_clk_cfg0 = dsi_phy_read(phy_base + REG_DSI_7nm_PHY_CMN_CLK_CFG0);
@@ -529,7 +532,7 @@ static int dsi_7nm_pll_restore_state(struct msm_dsi_phy *phy)
 	dsi_phy_write(pll_7nm->phy->pll_base + REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE, val);
 
 	dsi_phy_write(phy_base + REG_DSI_7nm_PHY_CMN_CLK_CFG0,
-		  cached->bit_clk_div | (cached->pix_clk_div << 4));
+		      cached->bit_clk_div | (cached->pix_clk_div << 4));
 
 	val = dsi_phy_read(phy_base + REG_DSI_7nm_PHY_CMN_CLK_CFG1);
 	val &= ~0x3;
@@ -585,65 +588,60 @@ static int dsi_7nm_set_usecase(struct msm_dsi_phy *phy)
  */
 static int pll_7nm_register(struct dsi_pll_7nm *pll_7nm, struct clk_hw **provided_clocks)
 {
-	char clk_name[32], parent[32], vco_name[32];
-	char parent2[32];
+	char clk_name[32];
 	struct clk_init_data vco_init = {
 		.parent_data = &(const struct clk_parent_data) {
 			.fw_name = "ref",
 		},
 		.num_parents = 1,
-		.name = vco_name,
+		.name = clk_name,
 		.flags = CLK_IGNORE_UNUSED,
 		.ops = &clk_ops_dsi_pll_7nm_vco,
 	};
 	struct device *dev = &pll_7nm->phy->pdev->dev;
-	struct clk_hw *hw;
+	struct clk_hw *hw, *pll_out_div, *pll_bit, *pll_by_2_bit;
+	struct clk_hw *pll_post_out_div, *phy_pll_out_dsi_parent;
 	int ret;
 
 	DBG("DSI%d", pll_7nm->phy->id);
 
-	snprintf(vco_name, 32, "dsi%dvco_clk", pll_7nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%dvco_clk", pll_7nm->phy->id);
 	pll_7nm->clk_hw.init = &vco_init;
 
 	ret = devm_clk_hw_register(dev, &pll_7nm->clk_hw);
 	if (ret)
 		return ret;
 
-	snprintf(clk_name, 32, "dsi%d_pll_out_div_clk", pll_7nm->phy->id);
-	snprintf(parent, 32, "dsi%dvco_clk", pll_7nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_out_div_clk", pll_7nm->phy->id);
 
-	hw = devm_clk_hw_register_divider(dev, clk_name,
-				     parent, CLK_SET_RATE_PARENT,
-				     pll_7nm->phy->pll_base +
-				     REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE,
-				     0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL);
-	if (IS_ERR(hw)) {
-		ret = PTR_ERR(hw);
+	pll_out_div = devm_clk_hw_register_divider_parent_hw(dev, clk_name,
+			&pll_7nm->clk_hw, CLK_SET_RATE_PARENT,
+			pll_7nm->phy->pll_base +
+				REG_DSI_7nm_PHY_PLL_PLL_OUTDIV_RATE,
+			0, 2, CLK_DIVIDER_POWER_OF_TWO, NULL);
+	if (IS_ERR(pll_out_div)) {
+		ret = PTR_ERR(pll_out_div);
 		goto fail;
 	}
 
-	snprintf(clk_name, 32, "dsi%d_pll_bit_clk", pll_7nm->phy->id);
-	snprintf(parent, 32, "dsi%d_pll_out_div_clk", pll_7nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_bit_clk", pll_7nm->phy->id);
 
 	/* BIT CLK: DIV_CTRL_3_0 */
-	hw = devm_clk_hw_register_divider(dev, clk_name, parent,
-				     CLK_SET_RATE_PARENT,
-				     pll_7nm->phy->base +
-				     REG_DSI_7nm_PHY_CMN_CLK_CFG0,
-				     0, 4, CLK_DIVIDER_ONE_BASED,
-				     &pll_7nm->postdiv_lock);
-	if (IS_ERR(hw)) {
-		ret = PTR_ERR(hw);
+	pll_bit = devm_clk_hw_register_divider_parent_hw(dev, clk_name,
+			pll_out_div, CLK_SET_RATE_PARENT,
+			pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG0,
+			0, 4, CLK_DIVIDER_ONE_BASED, &pll_7nm->postdiv_lock);
+	if (IS_ERR(pll_bit)) {
+		ret = PTR_ERR(pll_bit);
 		goto fail;
 	}
 
-	snprintf(clk_name, 32, "dsi%d_phy_pll_out_byteclk", pll_7nm->phy->id);
-	snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_7nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_phy_pll_out_byteclk", pll_7nm->phy->id);
 
 	/* DSI Byte clock = VCO_CLK / OUT_DIV / BIT_DIV / 8 */
-	hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent,
-					  CLK_SET_RATE_PARENT, 1,
-					  pll_7nm->phy->cphy_mode ? 7 : 8);
+	hw = devm_clk_hw_register_fixed_factor_parent_hw(dev, clk_name,
+			pll_bit, CLK_SET_RATE_PARENT, 1,
+			pll_7nm->phy->cphy_mode ? 7 : 8);
 	if (IS_ERR(hw)) {
 		ret = PTR_ERR(hw);
 		goto fail;
@@ -651,25 +649,25 @@ static int pll_7nm_register(struct dsi_pll_7nm *pll_7nm, struct clk_hw **provide
 
 	provided_clocks[DSI_BYTE_PLL_CLK] = hw;
 
-	snprintf(clk_name, 32, "dsi%d_pll_by_2_bit_clk", pll_7nm->phy->id);
-	snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_7nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_by_2_bit_clk", pll_7nm->phy->id);
 
-	hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent,
-					  0, 1, 2);
-	if (IS_ERR(hw)) {
-		ret = PTR_ERR(hw);
+	pll_by_2_bit = devm_clk_hw_register_fixed_factor_parent_hw(dev,
+			clk_name, pll_bit, 0, 1, 2);
+	if (IS_ERR(pll_by_2_bit)) {
+		ret = PTR_ERR(pll_by_2_bit);
 		goto fail;
 	}
 
-	snprintf(clk_name, 32, "dsi%d_pll_post_out_div_clk", pll_7nm->phy->id);
-	snprintf(parent, 32, "dsi%d_pll_out_div_clk", pll_7nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_pll_post_out_div_clk", pll_7nm->phy->id);
 
 	if (pll_7nm->phy->cphy_mode)
-		hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, 0, 2, 7);
+		pll_post_out_div = devm_clk_hw_register_fixed_factor_parent_hw(
+				dev, clk_name, pll_out_div, 0, 2, 7);
 	else
-		hw = devm_clk_hw_register_fixed_factor(dev, clk_name, parent, 0, 1, 4);
-	if (IS_ERR(hw)) {
-		ret = PTR_ERR(hw);
+		pll_post_out_div = devm_clk_hw_register_fixed_factor_parent_hw(
+				dev, clk_name, pll_out_div, 0, 1, 4);
+	if (IS_ERR(pll_post_out_div)) {
+		ret = PTR_ERR(pll_post_out_div);
 		goto fail;
 	}
 
@@ -682,34 +680,32 @@ static int pll_7nm_register(struct dsi_pll_7nm *pll_7nm, struct clk_hw **provide
 		data = dsi_phy_read(pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1);
 		dsi_phy_write(pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG1, data | 3);
 
-		snprintf(parent, 32, "dsi%d_pll_post_out_div_clk", pll_7nm->phy->id);
+		phy_pll_out_dsi_parent = pll_post_out_div;
 	} else {
-		snprintf(clk_name, 32, "dsi%d_pclk_mux", pll_7nm->phy->id);
-		snprintf(parent, 32, "dsi%d_pll_bit_clk", pll_7nm->phy->id);
-		snprintf(parent2, 32, "dsi%d_pll_by_2_bit_clk", pll_7nm->phy->id);
-
-		hw = devm_clk_hw_register_mux(dev, clk_name,
-					((const char *[]){
-					parent, parent2,
-					}), 2, 0, pll_7nm->phy->base +
+		snprintf(clk_name, sizeof(clk_name), "dsi%d_pclk_mux", pll_7nm->phy->id);
+
+		hw = devm_clk_hw_register_mux_parent_hws(dev, clk_name,
+				((const struct clk_hw *[]){
+					pll_bit,
+					pll_by_2_bit,
+				}), 2, 0, pll_7nm->phy->base +
 					REG_DSI_7nm_PHY_CMN_CLK_CFG1,
-					0, 1, 0, NULL);
+				0, 1, 0, NULL);
 		if (IS_ERR(hw)) {
 			ret = PTR_ERR(hw);
 			goto fail;
 		}
 
-		snprintf(parent, 32, "dsi%d_pclk_mux", pll_7nm->phy->id);
+		phy_pll_out_dsi_parent = hw;
 	}
 
-	snprintf(clk_name, 32, "dsi%d_phy_pll_out_dsiclk", pll_7nm->phy->id);
+	snprintf(clk_name, sizeof(clk_name), "dsi%d_phy_pll_out_dsiclk", pll_7nm->phy->id);
 
 	/* PIX CLK DIV : DIV_CTRL_7_4*/
-	hw = devm_clk_hw_register_divider(dev, clk_name, parent,
-				     0, pll_7nm->phy->base +
-					REG_DSI_7nm_PHY_CMN_CLK_CFG0,
-				     4, 4, CLK_DIVIDER_ONE_BASED,
-				     &pll_7nm->postdiv_lock);
+	hw = devm_clk_hw_register_divider_parent_hw(dev, clk_name,
+			phy_pll_out_dsi_parent, 0,
+			pll_7nm->phy->base + REG_DSI_7nm_PHY_CMN_CLK_CFG0,
+			4, 4, CLK_DIVIDER_ONE_BASED, &pll_7nm->postdiv_lock);
 	if (IS_ERR(hw)) {
 		ret = PTR_ERR(hw);
 		goto fail;
@@ -841,7 +837,7 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy,
 		ret = msm_dsi_dphy_timing_calc_v4(timing, clk_req);
 	if (ret) {
 		DRM_DEV_ERROR(&phy->pdev->dev,
-			"%s: PHY timing calculation failed\n", __func__);
+			      "%s: PHY timing calculation failed\n", __func__);
 		return -EINVAL;
 	}
 
@@ -960,10 +956,10 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy,
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_0, 0x00);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_4, timing->hs_exit);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_5,
-			timing->shared_timings.clk_pre);
+			      timing->shared_timings.clk_pre);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_6, timing->clk_prepare);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_7,
-			timing->shared_timings.clk_post);
+			      timing->shared_timings.clk_post);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_8, timing->hs_rqst);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_9, 0x02);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_10, 0x04);
@@ -982,9 +978,9 @@ static int dsi_7nm_phy_enable(struct msm_dsi_phy *phy,
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_10, 0x04);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_11, 0x00);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_12,
-			timing->shared_timings.clk_pre);
+			      timing->shared_timings.clk_pre);
 		dsi_phy_write(base + REG_DSI_7nm_PHY_CMN_TIMING_CTRL_13,
-			timing->shared_timings.clk_post);
+			      timing->shared_timings.clk_post);
 	}
 
 	/* DSI lane settings */
@@ -1036,14 +1032,18 @@ static void dsi_7nm_phy_disable(struct msm_dsi_phy *phy)
 	DBG("DSI%d PHY disabled", phy->id);
 }
 
+static const struct regulator_bulk_data dsi_phy_7nm_36mA_regulators[] = {
+	{ .supply = "vdds", .init_load_uA = 36000 },
+};
+
+static const struct regulator_bulk_data dsi_phy_7nm_37750uA_regulators[] = {
+	{ .supply = "vdds", .init_load_uA = 37550 },
+};
+
 const struct msm_dsi_phy_cfg dsi_phy_7nm_cfgs = {
 	.has_phy_lane = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vdds", 36000, 32},
-		},
-	},
+	.regulator_data = dsi_phy_7nm_36mA_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_7nm_36mA_regulators),
 	.ops = {
 		.enable = dsi_7nm_phy_enable,
 		.disable = dsi_7nm_phy_disable,
@@ -1065,12 +1065,8 @@ const struct msm_dsi_phy_cfg dsi_phy_7nm_cfgs = {
 
 const struct msm_dsi_phy_cfg dsi_phy_7nm_8150_cfgs = {
 	.has_phy_lane = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vdds", 36000, 32},
-		},
-	},
+	.regulator_data = dsi_phy_7nm_36mA_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_7nm_36mA_regulators),
 	.ops = {
 		.enable = dsi_7nm_phy_enable,
 		.disable = dsi_7nm_phy_disable,
@@ -1087,12 +1083,8 @@ const struct msm_dsi_phy_cfg dsi_phy_7nm_8150_cfgs = {
 
 const struct msm_dsi_phy_cfg dsi_phy_7nm_7280_cfgs = {
 	.has_phy_lane = true,
-	.reg_cfg = {
-		.num = 1,
-		.regs = {
-			{"vdds", 37550, 0},
-		},
-	},
+	.regulator_data = dsi_phy_7nm_37750uA_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_7nm_37750uA_regulators),
 	.ops = {
 		.enable = dsi_7nm_phy_enable,
 		.disable = dsi_7nm_phy_disable,
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c b/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c
index b06d9d25a189..4dd055416620 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_phy_8996.c
@@ -691,15 +691,13 @@ static const struct clk_ops hdmi_8996_pll_ops = {
 	.is_enabled = hdmi_8996_pll_is_enabled,
 };
 
-static const char * const hdmi_pll_parents[] = {
-	"xo",
-};
-
 static const struct clk_init_data pll_init = {
 	.name = "hdmipll",
 	.ops = &hdmi_8996_pll_ops,
-	.parent_names = hdmi_pll_parents,
-	.num_parents = ARRAY_SIZE(hdmi_pll_parents),
+	.parent_data = (const struct clk_parent_data[]){
+		{ .fw_name = "xo", .name = "xo_board" },
+	},
+	.num_parents = 1,
 	.flags = CLK_IGNORE_UNUSED,
 };
 
@@ -707,8 +705,7 @@ int msm_hdmi_pll_8996_init(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct hdmi_pll_8996 *pll;
-	struct clk *clk;
-	int i;
+	int i, ret;
 
 	pll = devm_kzalloc(dev, sizeof(*pll), GFP_KERNEL);
 	if (!pll)
@@ -735,10 +732,16 @@ int msm_hdmi_pll_8996_init(struct platform_device *pdev)
 	}
 	pll->clk_hw.init = &pll_init;
 
-	clk = devm_clk_register(dev, &pll->clk_hw);
-	if (IS_ERR(clk)) {
+	ret = devm_clk_hw_register(dev, &pll->clk_hw);
+	if (ret) {
 		DRM_DEV_ERROR(dev, "failed to register pll clock\n");
-		return -EINVAL;
+		return ret;
+	}
+
+	ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_simple_get, &pll->clk_hw);
+	if (ret) {
+		DRM_DEV_ERROR(dev, "%s: failed to register clk provider: %d\n", __func__, ret);
+		return ret;
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/msm/msm_debugfs.c b/drivers/gpu/drm/msm/msm_debugfs.c
index 7d2dab260f86..95f4374ae21c 100644
--- a/drivers/gpu/drm/msm/msm_debugfs.c
+++ b/drivers/gpu/drm/msm/msm_debugfs.c
@@ -7,6 +7,7 @@
 #ifdef CONFIG_DEBUG_FS
 
 #include <linux/debugfs.h>
+#include <linux/fault-inject.h>
 
 #include <drm/drm_debugfs.h>
 #include <drm/drm_file.h>
@@ -326,6 +327,13 @@ void msm_debugfs_init(struct drm_minor *minor)
 
 	if (priv->kms && priv->kms->funcs->debugfs_init)
 		priv->kms->funcs->debugfs_init(priv->kms, minor);
+
+#ifdef CONFIG_FAULT_INJECTION
+	fault_create_debugfs_attr("fail_gem_alloc", minor->debugfs_root,
+				  &fail_gem_alloc);
+	fault_create_debugfs_attr("fail_gem_iova", minor->debugfs_root,
+				  &fail_gem_iova);
+#endif
 }
 #endif
 
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 16884db272de..28034c21f6bc 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/dma-mapping.h>
+#include <linux/fault-inject.h>
 #include <linux/kthread.h>
 #include <linux/sched/mm.h>
 #include <linux/uaccess.h>
@@ -78,6 +79,11 @@ static bool modeset = true;
 MODULE_PARM_DESC(modeset, "Use kernel modesetting [KMS] (1=on (default), 0=disable)");
 module_param(modeset, bool, 0600);
 
+#ifdef CONFIG_FAULT_INJECTION
+DECLARE_FAULT_ATTR(fail_gem_alloc);
+DECLARE_FAULT_ATTR(fail_gem_iova);
+#endif
+
 static irqreturn_t msm_irq(int irq, void *arg)
 {
 	struct drm_device *dev = arg;
@@ -355,7 +361,7 @@ static int msm_init_vram(struct drm_device *dev)
 		DRM_INFO("using VRAM carveout: %lx@%pa\n", size, &r.start);
 
 		/* if we have no IOMMU, then we need to use carveout allocator.
-		 * Grab the entire CMA chunk carved out in early startup in
+		 * Grab the entire DMA chunk carved out in early startup in
 		 * mach-msm:
 		 */
 	} else if (!msm_use_mmu(dev)) {
@@ -418,14 +424,18 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv)
 	INIT_LIST_HEAD(&priv->objects);
 	mutex_init(&priv->obj_lock);
 
-	INIT_LIST_HEAD(&priv->inactive_willneed);
-	INIT_LIST_HEAD(&priv->inactive_dontneed);
-	INIT_LIST_HEAD(&priv->inactive_unpinned);
-	mutex_init(&priv->mm_lock);
+	/*
+	 * Initialize the LRUs:
+	 */
+	mutex_init(&priv->lru.lock);
+	drm_gem_lru_init(&priv->lru.unbacked, &priv->lru.lock);
+	drm_gem_lru_init(&priv->lru.pinned,   &priv->lru.lock);
+	drm_gem_lru_init(&priv->lru.willneed, &priv->lru.lock);
+	drm_gem_lru_init(&priv->lru.dontneed, &priv->lru.lock);
 
 	/* Teach lockdep about lock ordering wrt. shrinker: */
 	fs_reclaim_acquire(GFP_KERNEL);
-	might_lock(&priv->mm_lock);
+	might_lock(&priv->lru.lock);
 	fs_reclaim_release(GFP_KERNEL);
 
 	drm_mode_config_init(ddev);
@@ -699,6 +709,9 @@ static int msm_ioctl_gem_new(struct drm_device *dev, void *data,
 		flags |= MSM_BO_WC;
 	}
 
+	if (should_fail(&fail_gem_alloc, args->size))
+		return -ENOMEM;
+
 	return msm_gem_new_handle(dev, file, args->size,
 			args->flags, &args->handle, NULL);
 }
@@ -760,6 +773,9 @@ static int msm_ioctl_gem_info_iova(struct drm_device *dev,
 	if (!priv->gpu)
 		return -EINVAL;
 
+	if (should_fail(&fail_gem_iova, obj->size))
+		return -ENOMEM;
+
 	/*
 	 * Don't pin the memory here - just get an address so that userspace can
 	 * be productive
@@ -781,6 +797,9 @@ static int msm_ioctl_gem_info_set_iova(struct drm_device *dev,
 	if (priv->gpu->aspace == ctx->aspace)
 		return -EOPNOTSUPP;
 
+	if (should_fail(&fail_gem_iova, obj->size))
+		return -ENOMEM;
+
 	return msm_gem_set_iova(obj, ctx->aspace, iova);
 }
 
@@ -885,13 +904,13 @@ static int wait_fence(struct msm_gpu_submitqueue *queue, uint32_t fence_id,
 	 * retired, so if the fence is not found it means there is nothing
 	 * to wait for
 	 */
-	ret = mutex_lock_interruptible(&queue->lock);
+	ret = mutex_lock_interruptible(&queue->idr_lock);
 	if (ret)
 		return ret;
 	fence = idr_find(&queue->fence_idr, fence_id);
 	if (fence)
 		fence = dma_fence_get_rcu(fence);
-	mutex_unlock(&queue->lock);
+	mutex_unlock(&queue->idr_lock);
 
 	if (!fence)
 		return 0;
@@ -1244,10 +1263,15 @@ void msm_drv_shutdown(struct platform_device *pdev)
 	struct msm_drm_private *priv = platform_get_drvdata(pdev);
 	struct drm_device *drm = priv ? priv->dev : NULL;
 
-	if (!priv || !priv->kms)
-		return;
-
-	drm_atomic_helper_shutdown(drm);
+	/*
+	 * Shutdown the hw if we're far enough along where things might be on.
+	 * If we run this too early, we'll end up panicking in any variety of
+	 * places. Since we don't register the drm device until late in
+	 * msm_drm_init, drm_dev->registered is used as an indicator that the
+	 * shutdown will be successful.
+	 */
+	if (drm && drm->registered)
+		drm_atomic_helper_shutdown(drm);
 }
 
 static struct platform_driver msm_platform_driver = {
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index b3689a2d27d7..b2ea262296a4 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -27,13 +27,19 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/display/drm_dsc.h>
 #include <drm/msm_drm.h>
 #include <drm/drm_gem.h>
 
+#ifdef CONFIG_FAULT_INJECTION
+extern struct fault_attr fail_gem_alloc;
+extern struct fault_attr fail_gem_iova;
+#else
+#  define should_fail(attr, size) 0
+#endif
+
 struct msm_kms;
 struct msm_gpu;
 struct msm_mmu;
@@ -96,11 +102,6 @@ struct msm_drm_thread {
 	struct kthread_worker *worker;
 };
 
-/* DSC config */
-struct msm_display_dsc_config {
-	struct drm_dsc_config *drm;
-};
-
 struct msm_drm_private {
 
 	struct drm_device *dev;
@@ -142,28 +143,60 @@ struct msm_drm_private {
 	struct mutex obj_lock;
 
 	/**
-	 * LRUs of inactive GEM objects.  Every bo is either in one of the
-	 * inactive lists (depending on whether or not it is shrinkable) or
-	 * gpu->active_list (for the gpu it is active on[1]), or transiently
-	 * on a temporary list as the shrinker is running.
+	 * lru:
 	 *
-	 * Note that inactive_willneed also contains pinned and vmap'd bos,
-	 * but the number of pinned-but-not-active objects is small (scanout
-	 * buffers, ringbuffer, etc).
+	 * The various LRU's that a GEM object is in at various stages of
+	 * it's lifetime.  Objects start out in the unbacked LRU.  When
+	 * pinned (for scannout or permanently mapped GPU buffers, like
+	 * ringbuffer, memptr, fw, etc) it moves to the pinned LRU.  When
+	 * unpinned, it moves into willneed or dontneed LRU depending on
+	 * madvise state.  When backing pages are evicted (willneed) or
+	 * purged (dontneed) it moves back into the unbacked LRU.
 	 *
-	 * These lists are protected by mm_lock (which should be acquired
-	 * before per GEM object lock).  One should *not* hold mm_lock in
-	 * get_pages()/vmap()/etc paths, as they can trigger the shrinker.
-	 *
-	 * [1] if someone ever added support for the old 2d cores, there could be
-	 *     more than one gpu object
+	 * The dontneed LRU is considered by the shrinker for objects
+	 * that are candidate for purging, and the willneed LRU is
+	 * considered for objects that could be evicted.
 	 */
-	struct list_head inactive_willneed;  /* inactive + potentially unpin/evictable */
-	struct list_head inactive_dontneed;  /* inactive + shrinkable */
-	struct list_head inactive_unpinned;  /* inactive + purged or unpinned */
-	long shrinkable_count;               /* write access under mm_lock */
-	long evictable_count;                /* write access under mm_lock */
-	struct mutex mm_lock;
+	struct {
+		/**
+		 * unbacked:
+		 *
+		 * The LRU for GEM objects without backing pages allocated.
+		 * This mostly exists so that objects are always is one
+		 * LRU.
+		 */
+		struct drm_gem_lru unbacked;
+
+		/**
+		 * pinned:
+		 *
+		 * The LRU for pinned GEM objects
+		 */
+		struct drm_gem_lru pinned;
+
+		/**
+		 * willneed:
+		 *
+		 * The LRU for unpinned GEM objects which are in madvise
+		 * WILLNEED state (ie. can be evicted)
+		 */
+		struct drm_gem_lru willneed;
+
+		/**
+		 * dontneed:
+		 *
+		 * The LRU for unpinned GEM objects which are in madvise
+		 * DONTNEED state (ie. can be purged)
+		 */
+		struct drm_gem_lru dontneed;
+
+		/**
+		 * lock:
+		 *
+		 * Protects manipulation of all of the LRUs.
+		 */
+		struct mutex lock;
+	} lru;
 
 	struct workqueue_struct *wq;
 
@@ -290,7 +323,7 @@ void msm_dsi_snapshot(struct msm_disp_state *disp_state, struct msm_dsi *msm_dsi
 bool msm_dsi_is_cmd_mode(struct msm_dsi *msm_dsi);
 bool msm_dsi_is_bonded_dsi(struct msm_dsi *msm_dsi);
 bool msm_dsi_is_master_dsi(struct msm_dsi *msm_dsi);
-struct msm_display_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi);
+struct drm_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi);
 #else
 static inline void __init msm_dsi_register(void)
 {
@@ -320,7 +353,7 @@ static inline bool msm_dsi_is_master_dsi(struct msm_dsi *msm_dsi)
 	return false;
 }
 
-static inline struct msm_display_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi)
+static inline struct drm_dsc_config *msm_dsi_get_dsc_config(struct msm_dsi *msm_dsi)
 {
 	return NULL;
 }
@@ -433,6 +466,8 @@ void __iomem *msm_ioremap_size(struct platform_device *pdev, const char *name,
 		phys_addr_t *size);
 void __iomem *msm_ioremap_quiet(struct platform_device *pdev, const char *name);
 
+struct icc_path *msm_icc_get(struct device *dev, const char *name);
+
 #define msm_writel(data, addr) writel((data), (addr))
 #define msm_readl(addr) readl((addr))
 
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 8ddbd2e001d4..1dee0d18abbb 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -19,7 +19,7 @@
 #include "msm_gpu.h"
 #include "msm_mmu.h"
 
-static void update_inactive(struct msm_gem_object *msm_obj);
+static void update_lru(struct drm_gem_object *obj);
 
 static dma_addr_t physaddr(struct drm_gem_object *obj)
 {
@@ -97,7 +97,7 @@ static struct page **get_pages(struct drm_gem_object *obj)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	if (!msm_obj->pages) {
 		struct drm_device *dev = obj->dev;
@@ -132,7 +132,7 @@ static struct page **get_pages(struct drm_gem_object *obj)
 		if (msm_obj->flags & MSM_BO_WC)
 			sync_for_device(msm_obj);
 
-		update_inactive(msm_obj);
+		update_lru(obj);
 	}
 
 	return msm_obj->pages;
@@ -174,40 +174,45 @@ static void put_pages(struct drm_gem_object *obj)
 			put_pages_vram(obj);
 
 		msm_obj->pages = NULL;
+		update_lru(obj);
 	}
 }
 
-struct page **msm_gem_get_pages(struct drm_gem_object *obj)
+static struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct page **p;
 
-	msm_gem_lock(obj);
+	msm_gem_assert_locked(obj);
 
 	if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {
-		msm_gem_unlock(obj);
 		return ERR_PTR(-EBUSY);
 	}
 
 	p = get_pages(obj);
-
 	if (!IS_ERR(p)) {
-		msm_obj->pin_count++;
-		update_inactive(msm_obj);
+		to_msm_bo(obj)->pin_count++;
+		update_lru(obj);
 	}
 
-	msm_gem_unlock(obj);
 	return p;
 }
 
-void msm_gem_put_pages(struct drm_gem_object *obj)
+struct page **msm_gem_pin_pages(struct drm_gem_object *obj)
 {
-	struct msm_gem_object *msm_obj = to_msm_bo(obj);
+	struct page **p;
 
 	msm_gem_lock(obj);
-	msm_obj->pin_count--;
-	GEM_WARN_ON(msm_obj->pin_count < 0);
-	update_inactive(msm_obj);
+	p = msm_gem_pin_pages_locked(obj);
+	msm_gem_unlock(obj);
+
+	return p;
+}
+
+void msm_gem_unpin_pages(struct drm_gem_object *obj)
+{
+	msm_gem_lock(obj);
+	msm_gem_unpin_locked(obj);
 	msm_gem_unlock(obj);
 }
 
@@ -273,7 +278,7 @@ static uint64_t mmap_offset(struct drm_gem_object *obj)
 	struct drm_device *dev = obj->dev;
 	int ret;
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	/* Make it mmapable */
 	ret = drm_gem_create_mmap_offset(obj);
@@ -302,7 +307,7 @@ static struct msm_gem_vma *add_vma(struct drm_gem_object *obj,
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct msm_gem_vma *vma;
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	vma = kzalloc(sizeof(*vma), GFP_KERNEL);
 	if (!vma)
@@ -321,7 +326,7 @@ static struct msm_gem_vma *lookup_vma(struct drm_gem_object *obj,
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct msm_gem_vma *vma;
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	list_for_each_entry(vma, &msm_obj->vmas, list) {
 		if (vma->aspace == aspace)
@@ -352,7 +357,7 @@ put_iova_spaces(struct drm_gem_object *obj, bool close)
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct msm_gem_vma *vma;
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	list_for_each_entry(vma, &msm_obj->vmas, list) {
 		if (vma->aspace) {
@@ -370,7 +375,7 @@ put_iova_vmas(struct drm_gem_object *obj)
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct msm_gem_vma *vma, *tmp;
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	list_for_each_entry_safe(vma, tmp, &msm_obj->vmas, list) {
 		del_vma(vma);
@@ -383,7 +388,7 @@ static struct msm_gem_vma *get_vma_locked(struct drm_gem_object *obj,
 {
 	struct msm_gem_vma *vma;
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	vma = lookup_vma(obj, aspace);
 
@@ -423,19 +428,18 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct msm_gem_vma *vma)
 	if (msm_obj->flags & MSM_BO_CACHED_COHERENT)
 		prot |= IOMMU_CACHE;
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	if (GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED))
 		return -EBUSY;
 
-	pages = get_pages(obj);
+	pages = msm_gem_pin_pages_locked(obj);
 	if (IS_ERR(pages))
 		return PTR_ERR(pages);
 
 	ret = msm_gem_map_vma(vma->aspace, vma, prot, msm_obj->sgt, obj->size);
-
-	if (!ret)
-		msm_obj->pin_count++;
+	if (ret)
+		msm_gem_unpin_locked(obj);
 
 	return ret;
 }
@@ -444,12 +448,12 @@ void msm_gem_unpin_locked(struct drm_gem_object *obj)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	msm_obj->pin_count--;
 	GEM_WARN_ON(msm_obj->pin_count < 0);
 
-	update_inactive(msm_obj);
+	update_lru(obj);
 }
 
 struct msm_gem_vma *msm_gem_get_vma_locked(struct drm_gem_object *obj,
@@ -465,7 +469,7 @@ static int get_and_pin_iova_range_locked(struct drm_gem_object *obj,
 	struct msm_gem_vma *vma;
 	int ret;
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	vma = get_vma_locked(obj, aspace, range_start, range_end);
 	if (IS_ERR(vma))
@@ -626,7 +630,7 @@ static void *get_vaddr(struct drm_gem_object *obj, unsigned madv)
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	int ret = 0;
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	if (obj->import_attach)
 		return ERR_PTR(-ENODEV);
@@ -658,7 +662,7 @@ static void *get_vaddr(struct drm_gem_object *obj, unsigned madv)
 			goto fail;
 		}
 
-		update_inactive(msm_obj);
+		update_lru(obj);
 	}
 
 	return msm_obj->vaddr;
@@ -699,7 +703,7 @@ void msm_gem_put_vaddr_locked(struct drm_gem_object *obj)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 	GEM_WARN_ON(msm_obj->vmap_count < 1);
 
 	msm_obj->vmap_count--;
@@ -729,8 +733,7 @@ int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
 	/* If the obj is inactive, we might need to move it
 	 * between inactive lists
 	 */
-	if (msm_obj->active_count == 0)
-		update_inactive(msm_obj);
+	update_lru(obj);
 
 	msm_gem_unlock(obj);
 
@@ -742,7 +745,7 @@ void msm_gem_purge(struct drm_gem_object *obj)
 	struct drm_device *dev = obj->dev;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 	GEM_WARN_ON(!is_purgeable(msm_obj));
 
 	/* Get rid of any iommu mapping(s): */
@@ -757,7 +760,6 @@ void msm_gem_purge(struct drm_gem_object *obj)
 	put_iova_vmas(obj);
 
 	msm_obj->madv = __MSM_MADV_PURGED;
-	update_inactive(msm_obj);
 
 	drm_gem_free_mmap_offset(obj);
 
@@ -780,10 +782,8 @@ void msm_gem_evict(struct drm_gem_object *obj)
 	struct drm_device *dev = obj->dev;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 	GEM_WARN_ON(is_unevictable(msm_obj));
-	GEM_WARN_ON(!msm_obj->evictable);
-	GEM_WARN_ON(msm_obj->active_count);
 
 	/* Get rid of any iommu mapping(s): */
 	put_iova_spaces(obj, false);
@@ -791,15 +791,13 @@ void msm_gem_evict(struct drm_gem_object *obj)
 	drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping);
 
 	put_pages(obj);
-
-	update_inactive(msm_obj);
 }
 
 void msm_gem_vunmap(struct drm_gem_object *obj)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(obj);
 
 	if (!msm_obj->vaddr || GEM_WARN_ON(!is_vunmapable(msm_obj)))
 		return;
@@ -808,66 +806,37 @@ void msm_gem_vunmap(struct drm_gem_object *obj)
 	msm_obj->vaddr = NULL;
 }
 
-void msm_gem_active_get(struct drm_gem_object *obj, struct msm_gpu *gpu)
+static void update_lru(struct drm_gem_object *obj)
 {
-	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	struct msm_drm_private *priv = obj->dev->dev_private;
-
-	might_sleep();
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
-	GEM_WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED);
-	GEM_WARN_ON(msm_obj->dontneed);
-
-	if (msm_obj->active_count++ == 0) {
-		mutex_lock(&priv->mm_lock);
-		if (msm_obj->evictable)
-			mark_unevictable(msm_obj);
-		list_move_tail(&msm_obj->mm_list, &gpu->active_list);
-		mutex_unlock(&priv->mm_lock);
-	}
-}
-
-void msm_gem_active_put(struct drm_gem_object *obj)
-{
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
-	might_sleep();
-	GEM_WARN_ON(!msm_gem_is_locked(obj));
+	msm_gem_assert_locked(&msm_obj->base);
 
-	if (--msm_obj->active_count == 0) {
-		update_inactive(msm_obj);
+	if (!msm_obj->pages) {
+		GEM_WARN_ON(msm_obj->pin_count);
+		GEM_WARN_ON(msm_obj->vmap_count);
+
+		drm_gem_lru_move_tail(&priv->lru.unbacked, obj);
+	} else if (msm_obj->pin_count || msm_obj->vmap_count) {
+		drm_gem_lru_move_tail(&priv->lru.pinned, obj);
+	} else if (msm_obj->madv == MSM_MADV_WILLNEED) {
+		drm_gem_lru_move_tail(&priv->lru.willneed, obj);
+	} else {
+		GEM_WARN_ON(msm_obj->madv != MSM_MADV_DONTNEED);
+
+		drm_gem_lru_move_tail(&priv->lru.dontneed, obj);
 	}
 }
 
-static void update_inactive(struct msm_gem_object *msm_obj)
+bool msm_gem_active(struct drm_gem_object *obj)
 {
-	struct msm_drm_private *priv = msm_obj->base.dev->dev_private;
-
-	GEM_WARN_ON(!msm_gem_is_locked(&msm_obj->base));
+	msm_gem_assert_locked(obj);
 
-	if (msm_obj->active_count != 0)
-		return;
-
-	mutex_lock(&priv->mm_lock);
-
-	if (msm_obj->dontneed)
-		mark_unpurgeable(msm_obj);
-	if (msm_obj->evictable)
-		mark_unevictable(msm_obj);
-
-	list_del(&msm_obj->mm_list);
-	if ((msm_obj->madv == MSM_MADV_WILLNEED) && msm_obj->sgt) {
-		list_add_tail(&msm_obj->mm_list, &priv->inactive_willneed);
-		mark_evictable(msm_obj);
-	} else if (msm_obj->madv == MSM_MADV_DONTNEED) {
-		list_add_tail(&msm_obj->mm_list, &priv->inactive_dontneed);
-		mark_purgeable(msm_obj);
-	} else {
-		GEM_WARN_ON((msm_obj->madv != __MSM_MADV_PURGED) && msm_obj->sgt);
-		list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned);
-	}
+	if (to_msm_bo(obj)->pin_count)
+		return true;
 
-	mutex_unlock(&priv->mm_lock);
+	return !dma_resv_test_signaled(obj->resv, dma_resv_usage_rw(true));
 }
 
 int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout)
@@ -910,7 +879,7 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m,
 	stats->all.count++;
 	stats->all.size += obj->size;
 
-	if (is_active(msm_obj)) {
+	if (msm_gem_active(obj)) {
 		stats->active.count++;
 		stats->active.size += obj->size;
 	}
@@ -938,7 +907,7 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m,
 	}
 
 	seq_printf(m, "%08x: %c %2d (%2d) %08llx %p",
-			msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',
+			msm_obj->flags, msm_gem_active(obj) ? 'A' : 'I',
 			obj->name, kref_read(&obj->refcount),
 			off, msm_obj->vaddr);
 
@@ -1015,15 +984,6 @@ static void msm_gem_free_object(struct drm_gem_object *obj)
 	list_del(&msm_obj->node);
 	mutex_unlock(&priv->obj_lock);
 
-	mutex_lock(&priv->mm_lock);
-	if (msm_obj->dontneed)
-		mark_unpurgeable(msm_obj);
-	list_del(&msm_obj->mm_list);
-	mutex_unlock(&priv->mm_lock);
-
-	/* object should not be on active list: */
-	GEM_WARN_ON(is_active(msm_obj));
-
 	put_iova_spaces(obj, true);
 
 	if (obj->import_attach) {
@@ -1183,13 +1143,6 @@ struct drm_gem_object *msm_gem_new(struct drm_device *dev, uint32_t size, uint32
 
 		to_msm_bo(obj)->vram_node = &vma->node;
 
-		/* Call chain get_pages() -> update_inactive() tries to
-		 * access msm_obj->mm_list, but it is not initialized yet.
-		 * To avoid NULL pointer dereference error, initialize
-		 * mm_list to be empty.
-		 */
-		INIT_LIST_HEAD(&msm_obj->mm_list);
-
 		msm_gem_lock(obj);
 		pages = get_pages(obj);
 		msm_gem_unlock(obj);
@@ -1212,9 +1165,7 @@ struct drm_gem_object *msm_gem_new(struct drm_device *dev, uint32_t size, uint32
 		mapping_set_gfp_mask(obj->filp->f_mapping, GFP_HIGHUSER);
 	}
 
-	mutex_lock(&priv->mm_lock);
-	list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned);
-	mutex_unlock(&priv->mm_lock);
+	drm_gem_lru_move_tail(&priv->lru.unbacked, obj);
 
 	mutex_lock(&priv->obj_lock);
 	list_add_tail(&msm_obj->node, &priv->objects);
@@ -1270,9 +1221,7 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev,
 
 	msm_gem_unlock(obj);
 
-	mutex_lock(&priv->mm_lock);
-	list_add_tail(&msm_obj->mm_list, &priv->inactive_unpinned);
-	mutex_unlock(&priv->mm_lock);
+	drm_gem_lru_move_tail(&priv->lru.pinned, obj);
 
 	mutex_lock(&priv->obj_lock);
 	list_add_tail(&msm_obj->node, &priv->objects);
diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h
index 432032ad4aed..c4844cf3a585 100644
--- a/drivers/gpu/drm/msm/msm_gem.h
+++ b/drivers/gpu/drm/msm/msm_gem.h
@@ -94,16 +94,6 @@ struct msm_gem_object {
 	uint8_t madv;
 
 	/**
-	 * Is object on inactive_dontneed list (ie. counted in priv->shrinkable_count)?
-	 */
-	bool dontneed : 1;
-
-	/**
-	 * Is object evictable (ie. counted in priv->evictable_count)?
-	 */
-	bool evictable : 1;
-
-	/**
 	 * count of active vmap'ing
 	 */
 	uint8_t vmap_count;
@@ -114,17 +104,6 @@ struct msm_gem_object {
 	 */
 	struct list_head node;
 
-	/**
-	 * An object is either:
-	 *  inactive - on priv->inactive_dontneed or priv->inactive_willneed
-	 *     (depending on purgeability status)
-	 *  active   - on one one of the gpu's active_list..  well, at
-	 *     least for now we don't have (I don't think) hw sync between
-	 *     2d and 3d one devices which have both, meaning we need to
-	 *     block on submit if a bo is already on other ring
-	 */
-	struct list_head mm_list;
-
 	struct page **pages;
 	struct sg_table *sgt;
 	void *vaddr;
@@ -138,7 +117,6 @@ struct msm_gem_object {
 
 	char name[32]; /* Identifier to print for the debugfs files */
 
-	int active_count;
 	int pin_count;
 };
 #define to_msm_bo(x) container_of(x, struct msm_gem_object, base)
@@ -159,8 +137,8 @@ int msm_gem_get_and_pin_iova(struct drm_gem_object *obj,
 		struct msm_gem_address_space *aspace, uint64_t *iova);
 void msm_gem_unpin_iova(struct drm_gem_object *obj,
 		struct msm_gem_address_space *aspace);
-struct page **msm_gem_get_pages(struct drm_gem_object *obj);
-void msm_gem_put_pages(struct drm_gem_object *obj);
+struct page **msm_gem_pin_pages(struct drm_gem_object *obj);
+void msm_gem_unpin_pages(struct drm_gem_object *obj);
 int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
 		struct drm_mode_create_dumb *args);
 int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
@@ -171,8 +149,7 @@ void *msm_gem_get_vaddr_active(struct drm_gem_object *obj);
 void msm_gem_put_vaddr_locked(struct drm_gem_object *obj);
 void msm_gem_put_vaddr(struct drm_gem_object *obj);
 int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv);
-void msm_gem_active_get(struct drm_gem_object *obj, struct msm_gpu *gpu);
-void msm_gem_active_put(struct drm_gem_object *obj);
+bool msm_gem_active(struct drm_gem_object *obj);
 int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout);
 int msm_gem_cpu_fini(struct drm_gem_object *obj);
 int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
@@ -208,12 +185,6 @@ msm_gem_lock(struct drm_gem_object *obj)
 	dma_resv_lock(obj->resv, NULL);
 }
 
-static inline bool __must_check
-msm_gem_trylock(struct drm_gem_object *obj)
-{
-	return dma_resv_trylock(obj->resv);
-}
-
 static inline int
 msm_gem_lock_interruptible(struct drm_gem_object *obj)
 {
@@ -226,8 +197,8 @@ msm_gem_unlock(struct drm_gem_object *obj)
 	dma_resv_unlock(obj->resv);
 }
 
-static inline bool
-msm_gem_is_locked(struct drm_gem_object *obj)
+static inline void
+msm_gem_assert_locked(struct drm_gem_object *obj)
 {
 	/*
 	 * Destroying the object is a special case.. msm_gem_free_object()
@@ -241,13 +212,10 @@ msm_gem_is_locked(struct drm_gem_object *obj)
 	 * Unfortunately lockdep is not aware of this detail.  So when the
 	 * refcount drops to zero, we pretend it is already locked.
 	 */
-	return dma_resv_is_locked(obj->resv) || (kref_read(&obj->refcount) == 0);
-}
-
-static inline bool is_active(struct msm_gem_object *msm_obj)
-{
-	GEM_WARN_ON(!msm_gem_is_locked(&msm_obj->base));
-	return msm_obj->active_count;
+	lockdep_assert_once(
+		(kref_read(&obj->refcount) == 0) ||
+		(lockdep_is_held(&obj->resv->lock.base) != LOCK_STATE_NOT_HELD)
+	);
 }
 
 /* imported/exported objects are not purgeable: */
@@ -264,81 +232,15 @@ static inline bool is_purgeable(struct msm_gem_object *msm_obj)
 
 static inline bool is_vunmapable(struct msm_gem_object *msm_obj)
 {
-	GEM_WARN_ON(!msm_gem_is_locked(&msm_obj->base));
+	msm_gem_assert_locked(&msm_obj->base);
 	return (msm_obj->vmap_count == 0) && msm_obj->vaddr;
 }
 
-static inline void mark_purgeable(struct msm_gem_object *msm_obj)
-{
-	struct msm_drm_private *priv = msm_obj->base.dev->dev_private;
-
-	GEM_WARN_ON(!mutex_is_locked(&priv->mm_lock));
-
-	if (is_unpurgeable(msm_obj))
-		return;
-
-	if (GEM_WARN_ON(msm_obj->dontneed))
-		return;
-
-	priv->shrinkable_count += msm_obj->base.size >> PAGE_SHIFT;
-	msm_obj->dontneed = true;
-}
-
-static inline void mark_unpurgeable(struct msm_gem_object *msm_obj)
-{
-	struct msm_drm_private *priv = msm_obj->base.dev->dev_private;
-
-	GEM_WARN_ON(!mutex_is_locked(&priv->mm_lock));
-
-	if (is_unpurgeable(msm_obj))
-		return;
-
-	if (GEM_WARN_ON(!msm_obj->dontneed))
-		return;
-
-	priv->shrinkable_count -= msm_obj->base.size >> PAGE_SHIFT;
-	GEM_WARN_ON(priv->shrinkable_count < 0);
-	msm_obj->dontneed = false;
-}
-
 static inline bool is_unevictable(struct msm_gem_object *msm_obj)
 {
 	return is_unpurgeable(msm_obj) || msm_obj->vaddr;
 }
 
-static inline void mark_evictable(struct msm_gem_object *msm_obj)
-{
-	struct msm_drm_private *priv = msm_obj->base.dev->dev_private;
-
-	WARN_ON(!mutex_is_locked(&priv->mm_lock));
-
-	if (is_unevictable(msm_obj))
-		return;
-
-	if (WARN_ON(msm_obj->evictable))
-		return;
-
-	priv->evictable_count += msm_obj->base.size >> PAGE_SHIFT;
-	msm_obj->evictable = true;
-}
-
-static inline void mark_unevictable(struct msm_gem_object *msm_obj)
-{
-	struct msm_drm_private *priv = msm_obj->base.dev->dev_private;
-
-	WARN_ON(!mutex_is_locked(&priv->mm_lock));
-
-	if (is_unevictable(msm_obj))
-		return;
-
-	if (WARN_ON(!msm_obj->evictable))
-		return;
-
-	priv->evictable_count -= msm_obj->base.size >> PAGE_SHIFT;
-	WARN_ON(priv->evictable_count < 0);
-	msm_obj->evictable = false;
-}
-
 void msm_gem_purge(struct drm_gem_object *obj);
 void msm_gem_evict(struct drm_gem_object *obj);
 void msm_gem_vunmap(struct drm_gem_object *obj);
@@ -390,9 +292,8 @@ struct msm_gem_submit {
 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
 #define BO_VALID	0x8000	/* is current addr in cmdstream correct/valid? */
 #define BO_LOCKED	0x4000	/* obj lock is held */
-#define BO_ACTIVE	0x2000	/* active refcnt is held */
-#define BO_OBJ_PINNED	0x1000	/* obj (pages) is pinned and on active list */
-#define BO_VMA_PINNED	0x0800	/* vma (virtual address) is pinned */
+#define BO_OBJ_PINNED	0x2000	/* obj (pages) is pinned and on active list */
+#define BO_VMA_PINNED	0x1000	/* vma (virtual address) is pinned */
 		uint32_t flags;
 		union {
 			struct msm_gem_object *obj;
diff --git a/drivers/gpu/drm/msm/msm_gem_prime.c b/drivers/gpu/drm/msm/msm_gem_prime.c
index dcc8a573bc76..c1d91863df05 100644
--- a/drivers/gpu/drm/msm/msm_gem_prime.c
+++ b/drivers/gpu/drm/msm/msm_gem_prime.c
@@ -63,12 +63,12 @@ struct drm_gem_object *msm_gem_prime_import_sg_table(struct drm_device *dev,
 int msm_gem_prime_pin(struct drm_gem_object *obj)
 {
 	if (!obj->import_attach)
-		msm_gem_get_pages(obj);
+		msm_gem_pin_pages(obj);
 	return 0;
 }
 
 void msm_gem_prime_unpin(struct drm_gem_object *obj)
 {
 	if (!obj->import_attach)
-		msm_gem_put_pages(obj);
+		msm_gem_unpin_pages(obj);
 }
diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c
index 0317055e3253..1de14e67f96b 100644
--- a/drivers/gpu/drm/msm/msm_gem_shrinker.c
+++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c
@@ -24,103 +24,77 @@ static bool can_swap(void)
 	return enable_eviction && get_nr_swap_pages() > 0;
 }
 
+static bool can_block(struct shrink_control *sc)
+{
+	if (!(sc->gfp_mask & __GFP_DIRECT_RECLAIM))
+		return false;
+	return current_is_kswapd() || (sc->gfp_mask & __GFP_RECLAIM);
+}
+
 static unsigned long
 msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
 {
 	struct msm_drm_private *priv =
 		container_of(shrinker, struct msm_drm_private, shrinker);
-	unsigned count = priv->shrinkable_count;
+	unsigned count = priv->lru.dontneed.count;
 
 	if (can_swap())
-		count += priv->evictable_count;
+		count += priv->lru.willneed.count;
 
 	return count;
 }
 
 static bool
-purge(struct msm_gem_object *msm_obj)
+purge(struct drm_gem_object *obj)
 {
-	if (!is_purgeable(msm_obj))
+	if (!is_purgeable(to_msm_bo(obj)))
 		return false;
 
-	/*
-	 * This will move the obj out of still_in_list to
-	 * the purged list
-	 */
-	msm_gem_purge(&msm_obj->base);
+	if (msm_gem_active(obj))
+		return false;
+
+	msm_gem_purge(obj);
 
 	return true;
 }
 
 static bool
-evict(struct msm_gem_object *msm_obj)
+evict(struct drm_gem_object *obj)
 {
-	if (is_unevictable(msm_obj))
+	if (is_unevictable(to_msm_bo(obj)))
+		return false;
+
+	if (msm_gem_active(obj))
 		return false;
 
-	msm_gem_evict(&msm_obj->base);
+	msm_gem_evict(obj);
 
 	return true;
 }
 
-static unsigned long
-scan(struct msm_drm_private *priv, unsigned nr_to_scan, struct list_head *list,
-		bool (*shrink)(struct msm_gem_object *msm_obj))
+static bool
+wait_for_idle(struct drm_gem_object *obj)
 {
-	unsigned freed = 0;
-	struct list_head still_in_list;
-
-	INIT_LIST_HEAD(&still_in_list);
-
-	mutex_lock(&priv->mm_lock);
-
-	while (freed < nr_to_scan) {
-		struct msm_gem_object *msm_obj = list_first_entry_or_null(
-				list, typeof(*msm_obj), mm_list);
-
-		if (!msm_obj)
-			break;
-
-		list_move_tail(&msm_obj->mm_list, &still_in_list);
-
-		/*
-		 * If it is in the process of being freed, msm_gem_free_object
-		 * can be blocked on mm_lock waiting to remove it.  So just
-		 * skip it.
-		 */
-		if (!kref_get_unless_zero(&msm_obj->base.refcount))
-			continue;
-
-		/*
-		 * Now that we own a reference, we can drop mm_lock for the
-		 * rest of the loop body, to reduce contention with the
-		 * retire_submit path (which could make more objects purgeable)
-		 */
-
-		mutex_unlock(&priv->mm_lock);
-
-		/*
-		 * Note that this still needs to be trylock, since we can
-		 * hit shrinker in response to trying to get backing pages
-		 * for this obj (ie. while it's lock is already held)
-		 */
-		if (!msm_gem_trylock(&msm_obj->base))
-			goto tail;
-
-		if (shrink(msm_obj))
-			freed += msm_obj->base.size >> PAGE_SHIFT;
+	enum dma_resv_usage usage = dma_resv_usage_rw(true);
+	return dma_resv_wait_timeout(obj->resv, usage, false, 1000) > 0;
+}
 
-		msm_gem_unlock(&msm_obj->base);
+static bool
+active_purge(struct drm_gem_object *obj)
+{
+	if (!wait_for_idle(obj))
+		return false;
 
-tail:
-		drm_gem_object_put(&msm_obj->base);
-		mutex_lock(&priv->mm_lock);
-	}
+	return purge(obj);
+}
 
-	list_splice_tail(&still_in_list, list);
-	mutex_unlock(&priv->mm_lock);
+static bool
+active_evict(struct drm_gem_object *obj)
+{
+	if (!wait_for_idle(obj))
+		return false;
 
-	return freed;
+	return evict(obj);
 }
 
 static unsigned long
@@ -128,21 +102,34 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 {
 	struct msm_drm_private *priv =
 		container_of(shrinker, struct msm_drm_private, shrinker);
-	unsigned long freed;
-
-	freed = scan(priv, sc->nr_to_scan, &priv->inactive_dontneed, purge);
-
-	if (freed > 0)
-		trace_msm_gem_purge(freed << PAGE_SHIFT);
-
-	if (can_swap() && freed < sc->nr_to_scan) {
-		int evicted = scan(priv, sc->nr_to_scan - freed,
-				&priv->inactive_willneed, evict);
+	struct {
+		struct drm_gem_lru *lru;
+		bool (*shrink)(struct drm_gem_object *obj);
+		bool cond;
+		unsigned long freed;
+	} stages[] = {
+		/* Stages of progressively more aggressive/expensive reclaim: */
+		{ &priv->lru.dontneed, purge,        true },
+		{ &priv->lru.willneed, evict,        can_swap() },
+		{ &priv->lru.dontneed, active_purge, can_block(sc) },
+		{ &priv->lru.willneed, active_evict, can_swap() && can_block(sc) },
+	};
+	long nr = sc->nr_to_scan;
+	unsigned long freed = 0;
 
-		if (evicted > 0)
-			trace_msm_gem_evict(evicted << PAGE_SHIFT);
+	for (unsigned i = 0; (nr > 0) && (i < ARRAY_SIZE(stages)); i++) {
+		if (!stages[i].cond)
+			continue;
+		stages[i].freed =
+			drm_gem_lru_scan(stages[i].lru, nr, stages[i].shrink);
+		nr -= stages[i].freed;
+		freed += stages[i].freed;
+	}
 
-		freed += evicted;
+	if (freed) {
+		trace_msm_gem_shrink(sc->nr_to_scan, stages[0].freed,
+				     stages[1].freed, stages[2].freed,
+				     stages[3].freed);
 	}
 
 	return (freed > 0) ? freed : SHRINK_STOP;
@@ -173,12 +160,12 @@ msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan)
 static const int vmap_shrink_limit = 15;
 
 static bool
-vmap_shrink(struct msm_gem_object *msm_obj)
+vmap_shrink(struct drm_gem_object *obj)
 {
-	if (!is_vunmapable(msm_obj))
+	if (!is_vunmapable(to_msm_bo(obj)))
 		return false;
 
-	msm_gem_vunmap(&msm_obj->base);
+	msm_gem_vunmap(obj);
 
 	return true;
 }
@@ -188,17 +175,18 @@ msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
 {
 	struct msm_drm_private *priv =
 		container_of(nb, struct msm_drm_private, vmap_notifier);
-	struct list_head *mm_lists[] = {
-		&priv->inactive_dontneed,
-		&priv->inactive_willneed,
-		priv->gpu ? &priv->gpu->active_list : NULL,
+	struct drm_gem_lru *lrus[] = {
+		&priv->lru.dontneed,
+		&priv->lru.willneed,
+		&priv->lru.pinned,
 		NULL,
 	};
 	unsigned idx, unmapped = 0;
 
-	for (idx = 0; mm_lists[idx] && unmapped < vmap_shrink_limit; idx++) {
-		unmapped += scan(priv, vmap_shrink_limit - unmapped,
-				mm_lists[idx], vmap_shrink);
+	for (idx = 0; lrus[idx] && unmapped < vmap_shrink_limit; idx++) {
+		unmapped += drm_gem_lru_scan(lrus[idx],
+					     vmap_shrink_limit - unmapped,
+					     vmap_shrink);
 	}
 
 	*(unsigned long *)ptr += unmapped;
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index c9e4aeb14f4a..5599d93ec0d2 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -26,6 +26,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,
 		struct msm_gpu_submitqueue *queue, uint32_t nr_bos,
 		uint32_t nr_cmds)
 {
+	static atomic_t ident = ATOMIC_INIT(0);
 	struct msm_gem_submit *submit;
 	uint64_t sz;
 	int ret;
@@ -36,7 +37,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,
 	if (sz > SIZE_MAX)
 		return ERR_PTR(-ENOMEM);
 
-	submit = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY);
+	submit = kzalloc(sz, GFP_KERNEL);
 	if (!submit)
 		return ERR_PTR(-ENOMEM);
 
@@ -52,9 +53,13 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,
 	submit->gpu = gpu;
 	submit->cmd = (void *)&submit->bos[nr_bos];
 	submit->queue = queue;
+	submit->pid = get_pid(task_pid(current));
 	submit->ring = gpu->rb[queue->ring_nr];
 	submit->fault_dumped = false;
 
+	/* Get a unique identifier for the submission for logging purposes */
+	submit->ident = atomic_inc_return(&ident) - 1;
+
 	INIT_LIST_HEAD(&submit->node);
 
 	return submit;
@@ -67,9 +72,9 @@ void __msm_gem_submit_destroy(struct kref *kref)
 	unsigned i;
 
 	if (submit->fence_id) {
-		mutex_lock(&submit->queue->lock);
+		mutex_lock(&submit->queue->idr_lock);
 		idr_remove(&submit->queue->fence_idr, submit->fence_id);
-		mutex_unlock(&submit->queue->lock);
+		mutex_unlock(&submit->queue->idr_lock);
 	}
 
 	dma_fence_put(submit->user_fence);
@@ -238,17 +243,13 @@ static void submit_cleanup_bo(struct msm_gem_submit *submit, int i,
 	if (flags & BO_OBJ_PINNED)
 		msm_gem_unpin_locked(obj);
 
-	if (flags & BO_ACTIVE)
-		msm_gem_active_put(obj);
-
 	if (flags & BO_LOCKED)
 		dma_resv_unlock(obj->resv);
 }
 
 static void submit_unlock_unpin_bo(struct msm_gem_submit *submit, int i)
 {
-	unsigned cleanup_flags = BO_VMA_PINNED | BO_OBJ_PINNED |
-				 BO_ACTIVE | BO_LOCKED;
+	unsigned cleanup_flags = BO_VMA_PINNED | BO_OBJ_PINNED | BO_LOCKED;
 	submit_cleanup_bo(submit, i, cleanup_flags);
 
 	if (!(submit->bos[i].flags & BO_VALID))
@@ -353,18 +354,6 @@ static int submit_pin_objects(struct msm_gem_submit *submit)
 
 	submit->valid = true;
 
-	/*
-	 * Increment active_count first, so if under memory pressure, we
-	 * don't inadvertently evict a bo needed by the submit in order
-	 * to pin an earlier bo in the same submit.
-	 */
-	for (i = 0; i < submit->nr_bos; i++) {
-		struct drm_gem_object *obj = &submit->bos[i].obj->base;
-
-		msm_gem_active_get(obj, submit->gpu);
-		submit->bos[i].flags |= BO_ACTIVE;
-	}
-
 	for (i = 0; i < submit->nr_bos; i++) {
 		struct drm_gem_object *obj = &submit->bos[i].obj->base;
 		struct msm_gem_vma *vma;
@@ -512,11 +501,11 @@ out:
  */
 static void submit_cleanup(struct msm_gem_submit *submit, bool error)
 {
-	unsigned cleanup_flags = BO_LOCKED;
+	unsigned cleanup_flags = BO_LOCKED | BO_OBJ_PINNED;
 	unsigned i;
 
 	if (error)
-		cleanup_flags |= BO_VMA_PINNED | BO_OBJ_PINNED | BO_ACTIVE;
+		cleanup_flags |= BO_VMA_PINNED;
 
 	for (i = 0; i < submit->nr_bos; i++) {
 		struct msm_gem_object *msm_obj = submit->bos[i].obj;
@@ -533,10 +522,6 @@ void msm_submit_retire(struct msm_gem_submit *submit)
 	for (i = 0; i < submit->nr_bos; i++) {
 		struct drm_gem_object *obj = &submit->bos[i].obj->base;
 
-		msm_gem_lock(obj);
-		/* Note, VMA already fence-unpinned before submit: */
-		submit_cleanup_bo(submit, i, BO_OBJ_PINNED | BO_ACTIVE);
-		msm_gem_unlock(obj);
 		drm_gem_object_put(obj);
 	}
 }
@@ -718,7 +703,6 @@ static void msm_process_post_deps(struct msm_submit_post_dep *post_deps,
 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 		struct drm_file *file)
 {
-	static atomic_t ident = ATOMIC_INIT(0);
 	struct msm_drm_private *priv = dev->dev_private;
 	struct drm_msm_gem_submit *args = data;
 	struct msm_file_private *ctx = file->driver_priv;
@@ -729,10 +713,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 	struct msm_submit_post_dep *post_deps = NULL;
 	struct drm_syncobj **syncobjs_to_reset = NULL;
 	int out_fence_fd = -1;
-	struct pid *pid = get_pid(task_pid(current));
 	bool has_ww_ticket = false;
 	unsigned i;
-	int ret, submitid;
+	int ret;
 
 	if (!gpu)
 		return -ENXIO;
@@ -764,35 +747,26 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 	if (!queue)
 		return -ENOENT;
 
-	/* Get a unique identifier for the submission for logging purposes */
-	submitid = atomic_inc_return(&ident) - 1;
-
 	ring = gpu->rb[queue->ring_nr];
-	trace_msm_gpu_submit(pid_nr(pid), ring->id, submitid,
-		args->nr_bos, args->nr_cmds);
-
-	ret = mutex_lock_interruptible(&queue->lock);
-	if (ret)
-		goto out_post_unlock;
 
 	if (args->flags & MSM_SUBMIT_FENCE_FD_OUT) {
 		out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
 		if (out_fence_fd < 0) {
 			ret = out_fence_fd;
-			goto out_unlock;
+			return ret;
 		}
 	}
 
-	submit = submit_create(dev, gpu, queue, args->nr_bos,
-		args->nr_cmds);
-	if (IS_ERR(submit)) {
-		ret = PTR_ERR(submit);
-		submit = NULL;
-		goto out_unlock;
-	}
+	submit = submit_create(dev, gpu, queue, args->nr_bos, args->nr_cmds);
+	if (IS_ERR(submit))
+		return PTR_ERR(submit);
+
+	trace_msm_gpu_submit(pid_nr(submit->pid), ring->id, submit->ident,
+		args->nr_bos, args->nr_cmds);
 
-	submit->pid = pid;
-	submit->ident = submitid;
+	ret = mutex_lock_interruptible(&queue->lock);
+	if (ret)
+		goto out_post_unlock;
 
 	if (args->flags & MSM_SUBMIT_SUDO)
 		submit->in_rb = true;
@@ -887,6 +861,8 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 
 	submit->nr_cmds = i;
 
+	mutex_lock(&queue->idr_lock);
+
 	/*
 	 * If using userspace provided seqno fence, validate that the id
 	 * is available before arming sched job.  Since access to fence_idr
@@ -895,6 +871,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 	 */
 	if ((args->flags & MSM_SUBMIT_FENCE_SN_IN) &&
 			idr_find(&queue->fence_idr, args->fence)) {
+		mutex_unlock(&queue->idr_lock);
 		ret = -EINVAL;
 		goto out;
 	}
@@ -927,6 +904,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
 						    submit->user_fence, 1,
 						    INT_MAX, GFP_KERNEL);
 	}
+
+	mutex_unlock(&queue->idr_lock);
+
 	if (submit->fence_id < 0) {
 		ret = submit->fence_id;
 		submit->fence_id = 0;
@@ -965,9 +945,9 @@ out_unlock:
 	if (ret && (out_fence_fd >= 0))
 		put_unused_fd(out_fence_fd);
 	mutex_unlock(&queue->lock);
+out_post_unlock:
 	if (submit)
 		msm_gem_submit_put(submit);
-out_post_unlock:
 	if (!IS_ERR_OR_NULL(post_deps)) {
 		for (i = 0; i < args->nr_out_syncobjs; ++i) {
 			kfree(post_deps[i].chain);
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index c2bfcf3f1f40..0098ee8438aa 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -16,6 +16,7 @@
 #include <generated/utsrelease.h>
 #include <linux/string_helpers.h>
 #include <linux/devcoredump.h>
+#include <linux/reset.h>
 #include <linux/sched/task.h>
 
 /*
@@ -394,7 +395,6 @@ static void recover_worker(struct kthread_work *work)
 	/* Record the crash state */
 	pm_runtime_get_sync(&gpu->pdev->dev);
 	msm_gpu_crashstate_capture(gpu, submit, comm, cmd);
-	pm_runtime_put_sync(&gpu->pdev->dev);
 
 	kfree(cmd);
 	kfree(comm);
@@ -423,9 +423,7 @@ static void recover_worker(struct kthread_work *work)
 		/* retire completed submits, plus the one that hung: */
 		retire_submits(gpu);
 
-		pm_runtime_get_sync(&gpu->pdev->dev);
 		gpu->funcs->recover(gpu);
-		pm_runtime_put_sync(&gpu->pdev->dev);
 
 		/*
 		 * Replay all remaining submits starting with highest priority
@@ -442,6 +440,8 @@ static void recover_worker(struct kthread_work *work)
 		}
 	}
 
+	pm_runtime_put(&gpu->pdev->dev);
+
 	mutex_unlock(&gpu->lock);
 
 	msm_gpu_retire(gpu);
@@ -664,11 +664,12 @@ static void retire_submit(struct msm_gpu *gpu, struct msm_ringbuffer *ring,
 	mutex_lock(&gpu->active_lock);
 	gpu->active_submits--;
 	WARN_ON(gpu->active_submits < 0);
-	if (!gpu->active_submits)
+	if (!gpu->active_submits) {
 		msm_devfreq_idle(gpu);
-	mutex_unlock(&gpu->active_lock);
+		pm_runtime_put_autosuspend(&gpu->pdev->dev);
+	}
 
-	pm_runtime_put_autosuspend(&gpu->pdev->dev);
+	mutex_unlock(&gpu->active_lock);
 
 	msm_gem_submit_put(submit);
 }
@@ -757,14 +758,17 @@ void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
 
 	/* Update devfreq on transition from idle->active: */
 	mutex_lock(&gpu->active_lock);
-	if (!gpu->active_submits)
+	if (!gpu->active_submits) {
+		pm_runtime_get(&gpu->pdev->dev);
 		msm_devfreq_active(gpu);
+	}
 	gpu->active_submits++;
 	mutex_unlock(&gpu->active_lock);
 
 	gpu->funcs->submit(gpu, submit);
 	gpu->cur_ctx_seqno = submit->queue->ctx->seqno;
 
+	pm_runtime_put(&gpu->pdev->dev);
 	hangcheck_timer_reset(gpu);
 }
 
@@ -846,7 +850,6 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 
 	sched_set_fifo_low(gpu->worker->task);
 
-	INIT_LIST_HEAD(&gpu->active_list);
 	mutex_init(&gpu->active_lock);
 	mutex_init(&gpu->lock);
 	init_waitqueue_head(&gpu->retire_event);
@@ -901,6 +904,9 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 	if (IS_ERR(gpu->gpu_cx))
 		gpu->gpu_cx = NULL;
 
+	gpu->cx_collapse = devm_reset_control_get_optional_exclusive(&pdev->dev,
+			"cx_collapse");
+
 	gpu->pdev = pdev;
 	platform_set_drvdata(pdev, &gpu->adreno_smmu);
 
@@ -974,8 +980,6 @@ void msm_gpu_cleanup(struct msm_gpu *gpu)
 
 	DBG("%s", gpu->name);
 
-	WARN_ON(!list_empty(&gpu->active_list));
-
 	for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) {
 		msm_ringbuffer_destroy(gpu->rb[i]);
 		gpu->rb[i] = NULL;
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index 4d935fedd2ac..ff911e7305ce 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -13,6 +13,7 @@
 #include <linux/interconnect.h>
 #include <linux/pm_opp.h>
 #include <linux/regulator/consumer.h>
+#include <linux/reset.h>
 
 #include "msm_drv.h"
 #include "msm_fence.h"
@@ -187,12 +188,6 @@ struct msm_gpu {
 	 */
 	int cur_ctx_seqno;
 
-	/*
-	 * List of GEM active objects on this gpu.  Protected by
-	 * msm_drm_private::mm_lock
-	 */
-	struct list_head active_list;
-
 	/**
 	 * lock:
 	 *
@@ -277,6 +272,9 @@ struct msm_gpu {
 	bool hw_apriv;
 
 	struct thermal_cooling_device *cooling;
+
+	/* To poll for cx gdsc collapse during gpu recovery */
+	struct reset_control *cx_collapse;
 };
 
 static inline struct msm_gpu *dev_to_gpu(struct device *dev)
@@ -466,7 +464,8 @@ static inline int msm_gpu_convert_priority(struct msm_gpu *gpu, int prio,
  * @node:      node in the context's list of submitqueues
  * @fence_idr: maps fence-id to dma_fence for userspace visible fence
  *             seqno, protected by submitqueue lock
- * @lock:      submitqueue lock
+ * @idr_lock:  for serializing access to fence_idr
+ * @lock:      submitqueue lock for serializing submits on a queue
  * @ref:       reference count
  * @entity:    the submit job-queue
  */
@@ -479,6 +478,7 @@ struct msm_gpu_submitqueue {
 	struct msm_file_private *ctx;
 	struct list_head node;
 	struct idr fence_idr;
+	struct mutex idr_lock;
 	struct mutex lock;
 	struct kref ref;
 	struct drm_sched_entity *entity;
diff --git a/drivers/gpu/drm/msm/msm_gpu_trace.h b/drivers/gpu/drm/msm/msm_gpu_trace.h
index ca0b08d7875b..ac40d857bc45 100644
--- a/drivers/gpu/drm/msm/msm_gpu_trace.h
+++ b/drivers/gpu/drm/msm/msm_gpu_trace.h
@@ -115,29 +115,27 @@ TRACE_EVENT(msm_gmu_freq_change,
 );
 
 
-TRACE_EVENT(msm_gem_purge,
-		TP_PROTO(u32 bytes),
-		TP_ARGS(bytes),
+TRACE_EVENT(msm_gem_shrink,
+		TP_PROTO(u32 nr_to_scan, u32 purged, u32 evicted,
+			 u32 active_purged, u32 active_evicted),
+		TP_ARGS(nr_to_scan, purged, evicted, active_purged, active_evicted),
 		TP_STRUCT__entry(
-			__field(u32, bytes)
+			__field(u32, nr_to_scan)
+			__field(u32, purged)
+			__field(u32, evicted)
+			__field(u32, active_purged)
+			__field(u32, active_evicted)
 			),
 		TP_fast_assign(
-			__entry->bytes = bytes;
+			__entry->nr_to_scan = nr_to_scan;
+			__entry->purged = purged;
+			__entry->evicted = evicted;
+			__entry->active_purged = active_purged;
+			__entry->active_evicted = active_evicted;
 			),
-		TP_printk("Purging %u bytes", __entry->bytes)
-);
-
-
-TRACE_EVENT(msm_gem_evict,
-		TP_PROTO(u32 bytes),
-		TP_ARGS(bytes),
-		TP_STRUCT__entry(
-			__field(u32, bytes)
-			),
-		TP_fast_assign(
-			__entry->bytes = bytes;
-			),
-		TP_printk("Evicting %u bytes", __entry->bytes)
+		TP_printk("nr_to_scan=%u pg, purged=%u pg, evicted=%u pg, active_purged=%u pg, active_evicted=%u pg",
+			  __entry->nr_to_scan, __entry->purged, __entry->evicted,
+			  __entry->active_purged, __entry->active_evicted)
 );
 
 
diff --git a/drivers/gpu/drm/msm/msm_io_utils.c b/drivers/gpu/drm/msm/msm_io_utils.c
index 7b504617833a..d02cd29ce829 100644
--- a/drivers/gpu/drm/msm/msm_io_utils.c
+++ b/drivers/gpu/drm/msm/msm_io_utils.c
@@ -5,6 +5,8 @@
  * Author: Rob Clark <robdclark@gmail.com>
  */
 
+#include <linux/interconnect.h>
+
 #include "msm_drv.h"
 
 /*
@@ -124,3 +126,23 @@ void msm_hrtimer_work_init(struct msm_hrtimer_work *work,
 	work->worker = worker;
 	kthread_init_work(&work->work, fn);
 }
+
+struct icc_path *msm_icc_get(struct device *dev, const char *name)
+{
+	struct device *mdss_dev = dev->parent;
+	struct icc_path *path;
+
+	path = of_icc_get(dev, name);
+	if (path)
+		return path;
+
+	/*
+	 * If there are no interconnects attached to the corresponding device
+	 * node, of_icc_get() will return NULL.
+	 *
+	 * If the MDP5/DPU device node doesn't have interconnects, lookup the
+	 * path in the parent (MDSS) device.
+	 */
+	return of_icc_get(mdss_dev, name);
+
+}
diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c
index a54ed354578b..5577cea7c009 100644
--- a/drivers/gpu/drm/msm/msm_iommu.c
+++ b/drivers/gpu/drm/msm/msm_iommu.c
@@ -21,6 +21,7 @@ struct msm_iommu_pagetable {
 	struct msm_mmu base;
 	struct msm_mmu *parent;
 	struct io_pgtable_ops *pgtbl_ops;
+	unsigned long pgsize_bitmap;	/* Bitmap of page sizes in use */
 	phys_addr_t ttbr;
 	u32 asid;
 };
@@ -29,23 +30,84 @@ static struct msm_iommu_pagetable *to_pagetable(struct msm_mmu *mmu)
 	return container_of(mmu, struct msm_iommu_pagetable, base);
 }
 
+/* based on iommu_pgsize() in iommu.c: */
+static size_t calc_pgsize(struct msm_iommu_pagetable *pagetable,
+			   unsigned long iova, phys_addr_t paddr,
+			   size_t size, size_t *count)
+{
+	unsigned int pgsize_idx, pgsize_idx_next;
+	unsigned long pgsizes;
+	size_t offset, pgsize, pgsize_next;
+	unsigned long addr_merge = paddr | iova;
+
+	/* Page sizes supported by the hardware and small enough for @size */
+	pgsizes = pagetable->pgsize_bitmap & GENMASK(__fls(size), 0);
+
+	/* Constrain the page sizes further based on the maximum alignment */
+	if (likely(addr_merge))
+		pgsizes &= GENMASK(__ffs(addr_merge), 0);
+
+	/* Make sure we have at least one suitable page size */
+	BUG_ON(!pgsizes);
+
+	/* Pick the biggest page size remaining */
+	pgsize_idx = __fls(pgsizes);
+	pgsize = BIT(pgsize_idx);
+	if (!count)
+		return pgsize;
+
+	/* Find the next biggest support page size, if it exists */
+	pgsizes = pagetable->pgsize_bitmap & ~GENMASK(pgsize_idx, 0);
+	if (!pgsizes)
+		goto out_set_count;
+
+	pgsize_idx_next = __ffs(pgsizes);
+	pgsize_next = BIT(pgsize_idx_next);
+
+	/*
+	 * There's no point trying a bigger page size unless the virtual
+	 * and physical addresses are similarly offset within the larger page.
+	 */
+	if ((iova ^ paddr) & (pgsize_next - 1))
+		goto out_set_count;
+
+	/* Calculate the offset to the next page size alignment boundary */
+	offset = pgsize_next - (addr_merge & (pgsize_next - 1));
+
+	/*
+	 * If size is big enough to accommodate the larger page, reduce
+	 * the number of smaller pages.
+	 */
+	if (offset + pgsize_next <= size)
+		size = offset;
+
+out_set_count:
+	*count = size >> pgsize_idx;
+	return pgsize;
+}
+
 static int msm_iommu_pagetable_unmap(struct msm_mmu *mmu, u64 iova,
 		size_t size)
 {
 	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
 	struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
-	size_t unmapped = 0;
 
-	/* Unmap the block one page at a time */
 	while (size) {
-		unmapped += ops->unmap(ops, iova, 4096, NULL);
-		iova += 4096;
-		size -= 4096;
+		size_t unmapped, pgsize, count;
+
+		pgsize = calc_pgsize(pagetable, iova, iova, size, &count);
+
+		unmapped = ops->unmap_pages(ops, iova, pgsize, count, NULL);
+		if (!unmapped)
+			break;
+
+		iova += unmapped;
+		size -= unmapped;
 	}
 
 	iommu_flush_iotlb_all(to_msm_iommu(pagetable->parent)->domain);
 
-	return (unmapped == size) ? 0 : -EINVAL;
+	return (size == 0) ? 0 : -EINVAL;
 }
 
 static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova,
@@ -54,7 +116,6 @@ static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova,
 	struct msm_iommu_pagetable *pagetable = to_pagetable(mmu);
 	struct io_pgtable_ops *ops = pagetable->pgtbl_ops;
 	struct scatterlist *sg;
-	size_t mapped = 0;
 	u64 addr = iova;
 	unsigned int i;
 
@@ -62,17 +123,26 @@ static int msm_iommu_pagetable_map(struct msm_mmu *mmu, u64 iova,
 		size_t size = sg->length;
 		phys_addr_t phys = sg_phys(sg);
 
-		/* Map the block one page at a time */
 		while (size) {
-			if (ops->map(ops, addr, phys, 4096, prot, GFP_KERNEL)) {
-				msm_iommu_pagetable_unmap(mmu, iova, mapped);
+			size_t pgsize, count, mapped = 0;
+			int ret;
+
+			pgsize = calc_pgsize(pagetable, addr, phys, size, &count);
+
+			ret = ops->map_pages(ops, addr, phys, pgsize, count,
+					     prot, GFP_KERNEL, &mapped);
+
+			/* map_pages could fail after mapping some of the pages,
+			 * so update the counters before error handling.
+			 */
+			phys += mapped;
+			addr += mapped;
+			size -= mapped;
+
+			if (ret) {
+				msm_iommu_pagetable_unmap(mmu, iova, addr - iova);
 				return -EINVAL;
 			}
-
-			phys += 4096;
-			addr += 4096;
-			size -= 4096;
-			mapped += 4096;
 		}
 	}
 
@@ -207,6 +277,7 @@ struct msm_mmu *msm_iommu_pagetable_create(struct msm_mmu *parent)
 
 	/* Needed later for TLB flush */
 	pagetable->parent = parent;
+	pagetable->pgsize_bitmap = ttbr0_cfg.pgsize_bitmap;
 	pagetable->ttbr = ttbr0_cfg.arm_lpae_s1_cfg.ttbr;
 
 	/*
diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c
index 56eecb4a72dc..cad4c3525f0b 100644
--- a/drivers/gpu/drm/msm/msm_ringbuffer.c
+++ b/drivers/gpu/drm/msm/msm_ringbuffer.c
@@ -29,8 +29,6 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)
 		msm_gem_unlock(obj);
 	}
 
-	pm_runtime_get_sync(&gpu->pdev->dev);
-
 	/* TODO move submit path over to using a per-ring lock.. */
 	mutex_lock(&gpu->lock);
 
@@ -38,8 +36,6 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)
 
 	mutex_unlock(&gpu->lock);
 
-	pm_runtime_put(&gpu->pdev->dev);
-
 	return dma_fence_get(submit->hw_fence);
 }
 
diff --git a/drivers/gpu/drm/msm/msm_submitqueue.c b/drivers/gpu/drm/msm/msm_submitqueue.c
index f486a3cd4e55..c6929e205b51 100644
--- a/drivers/gpu/drm/msm/msm_submitqueue.c
+++ b/drivers/gpu/drm/msm/msm_submitqueue.c
@@ -200,6 +200,7 @@ int msm_submitqueue_create(struct drm_device *drm, struct msm_file_private *ctx,
 		*id = queue->id;
 
 	idr_init(&queue->fence_idr);
+	mutex_init(&queue->idr_lock);
 	mutex_init(&queue->lock);
 
 	list_add_tail(&queue->node, &ctx->submitqueues);
diff --git a/drivers/gpu/drm/mxsfb/Kconfig b/drivers/gpu/drm/mxsfb/Kconfig
index 873551b4552f..116f8168bda4 100644
--- a/drivers/gpu/drm/mxsfb/Kconfig
+++ b/drivers/gpu/drm/mxsfb/Kconfig
@@ -10,7 +10,7 @@ config DRM_MXSFB
 	depends on COMMON_CLK
 	select DRM_MXS
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_PANEL
 	select DRM_PANEL_BRIDGE
 	help
@@ -26,7 +26,7 @@ config DRM_IMX_LCDIF
 	depends on COMMON_CLK
 	select DRM_MXS
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_PANEL
 	select DRM_PANEL_BRIDGE
 	help
diff --git a/drivers/gpu/drm/mxsfb/lcdif_drv.c b/drivers/gpu/drm/mxsfb/lcdif_drv.c
index befad33dcb95..075002ed6fb0 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_drv.c
+++ b/drivers/gpu/drm/mxsfb/lcdif_drv.c
@@ -8,7 +8,6 @@
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
 #include <linux/io.h>
-#include <linux/iopoll.h>
 #include <linux/module.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
@@ -16,11 +15,9 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
-#include <drm/drm_connector.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_fourcc.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_mode_config.h>
 #include <drm/drm_module.h>
@@ -45,23 +42,11 @@ static int lcdif_attach_bridge(struct lcdif_drm_private *lcdif)
 {
 	struct drm_device *drm = lcdif->drm;
 	struct drm_bridge *bridge;
-	struct drm_panel *panel;
 	int ret;
 
-	ret = drm_of_find_panel_or_bridge(drm->dev->of_node, 0, 0, &panel,
-					  &bridge);
-	if (ret)
-		return ret;
-
-	if (panel) {
-		bridge = devm_drm_panel_bridge_add_typed(drm->dev, panel,
-							 DRM_MODE_CONNECTOR_DPI);
-		if (IS_ERR(bridge))
-			return PTR_ERR(bridge);
-	}
-
-	if (!bridge)
-		return -ENODEV;
+	bridge = devm_drm_of_get_bridge(drm->dev, drm->dev->of_node, 0, 0);
+	if (IS_ERR(bridge))
+		return PTR_ERR(bridge);
 
 	ret = drm_bridge_attach(&lcdif->encoder, bridge, NULL, 0);
 	if (ret)
@@ -199,11 +184,11 @@ static void lcdif_unload(struct drm_device *drm)
 	drm->dev_private = NULL;
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(fops);
+DEFINE_DRM_GEM_DMA_FOPS(fops);
 
 static const struct drm_driver lcdif_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 	.fops	= &fops,
 	.name	= "imx-lcdif",
 	.desc	= "i.MX LCDIF Controller DRM",
diff --git a/drivers/gpu/drm/mxsfb/lcdif_drv.h b/drivers/gpu/drm/mxsfb/lcdif_drv.h
index cb916341e845..6cdba6e20c02 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_drv.h
+++ b/drivers/gpu/drm/mxsfb/lcdif_drv.h
@@ -8,6 +8,7 @@
 #ifndef __LCDIF_DRV_H__
 #define __LCDIF_DRV_H__
 
+#include <drm/drm_bridge.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_device.h>
 #include <drm/drm_encoder.h>
diff --git a/drivers/gpu/drm/mxsfb/lcdif_kms.c b/drivers/gpu/drm/mxsfb/lcdif_kms.c
index 1bec1279c8b5..b1092aab1423 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_kms.c
+++ b/drivers/gpu/drm/mxsfb/lcdif_kms.c
@@ -17,13 +17,12 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_encoder.h>
-#include <drm/drm_framebuffer.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
+#include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_plane.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "lcdif_drv.h"
@@ -123,8 +122,8 @@ static void lcdif_set_mode(struct lcdif_drm_private *lcdif, u32 bus_flags)
 
 	writel(ctrl, lcdif->base + LCDC_V8_CTRL);
 
-	writel(DISP_SIZE_DELTA_Y(m->crtc_vdisplay) |
-	       DISP_SIZE_DELTA_X(m->crtc_hdisplay),
+	writel(DISP_SIZE_DELTA_Y(m->vdisplay) |
+	       DISP_SIZE_DELTA_X(m->hdisplay),
 	       lcdif->base + LCDC_V8_DISP_SIZE);
 
 	writel(HSYN_PARA_BP_H(m->htotal - m->hsync_end) |
@@ -139,8 +138,8 @@ static void lcdif_set_mode(struct lcdif_drm_private *lcdif, u32 bus_flags)
 	       VSYN_HSYN_WIDTH_PW_H(m->hsync_end - m->hsync_start),
 	       lcdif->base + LCDC_V8_VSYN_HSYN_WIDTH);
 
-	writel(CTRLDESCL0_1_HEIGHT(m->crtc_vdisplay) |
-	       CTRLDESCL0_1_WIDTH(m->crtc_hdisplay),
+	writel(CTRLDESCL0_1_HEIGHT(m->vdisplay) |
+	       CTRLDESCL0_1_WIDTH(m->hdisplay),
 	       lcdif->base + LCDC_V8_CTRLDESCL0_1);
 
 	writel(CTRLDESCL0_3_PITCH(lcdif->crtc.primary->state->fb->pitches[0]),
@@ -204,7 +203,7 @@ static void lcdif_crtc_mode_set_nofb(struct lcdif_drm_private *lcdif,
 	DRM_DEV_DEBUG_DRIVER(drm->dev, "Pixel clock: %dkHz (actual: %dkHz)\n",
 			     m->crtc_clock,
 			     (int)(clk_get_rate(lcdif->clk) / 1000));
-	DRM_DEV_DEBUG_DRIVER(drm->dev, "Connector bus_flags: 0x%08X\n",
+	DRM_DEV_DEBUG_DRIVER(drm->dev, "Bridge bus_flags: 0x%08X\n",
 			     bus_flags);
 	DRM_DEV_DEBUG_DRIVER(drm->dev, "Mode flags: 0x%08X\n", m->flags);
 
@@ -297,7 +296,7 @@ static void lcdif_crtc_atomic_enable(struct drm_crtc *crtc,
 	lcdif_crtc_mode_set_nofb(lcdif, bridge_state, bus_format);
 
 	/* Write cur_buf as well to avoid an initial corrupt frame */
-	paddr = drm_fb_cma_get_gem_addr(new_pstate->fb, new_pstate, 0);
+	paddr = drm_fb_dma_get_gem_addr(new_pstate->fb, new_pstate, 0);
 	if (paddr) {
 		writel(lower_32_bits(paddr),
 		       lcdif->base + LCDC_V8_CTRLDESCL_LOW0_4);
@@ -393,8 +392,8 @@ static int lcdif_plane_atomic_check(struct drm_plane *plane,
 						   &lcdif->crtc);
 
 	return drm_atomic_helper_check_plane_state(plane_state, crtc_state,
-						   DRM_PLANE_HELPER_NO_SCALING,
-						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
 						   false, true);
 }
 
@@ -406,7 +405,7 @@ static void lcdif_plane_primary_atomic_update(struct drm_plane *plane,
 									    plane);
 	dma_addr_t paddr;
 
-	paddr = drm_fb_cma_get_gem_addr(new_pstate->fb, new_pstate, 0);
+	paddr = drm_fb_dma_get_gem_addr(new_pstate->fb, new_pstate, 0);
 	if (paddr) {
 		writel(lower_32_bits(paddr),
 		       lcdif->base + LCDC_V8_CTRLDESCL_LOW0_4);
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_drv.c b/drivers/gpu/drm/mxsfb/mxsfb_drv.c
index 55aad92e08ba..b29b332ed381 100644
--- a/drivers/gpu/drm/mxsfb/mxsfb_drv.c
+++ b/drivers/gpu/drm/mxsfb/mxsfb_drv.c
@@ -22,7 +22,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_mode_config.h>
 #include <drm/drm_module.h>
@@ -324,11 +324,11 @@ static void mxsfb_unload(struct drm_device *drm)
 	pm_runtime_disable(drm->dev);
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(fops);
+DEFINE_DRM_GEM_DMA_FOPS(fops);
 
 static const struct drm_driver mxsfb_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 	.fops	= &fops,
 	.name	= "mxsfb-drm",
 	.desc	= "MXSFB Controller DRM",
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_kms.c b/drivers/gpu/drm/mxsfb/mxsfb_kms.c
index e38ce5737a5f..3bcc9c0f2019 100644
--- a/drivers/gpu/drm/mxsfb/mxsfb_kms.c
+++ b/drivers/gpu/drm/mxsfb/mxsfb_kms.c
@@ -20,13 +20,12 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_encoder.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_plane.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "mxsfb_drv.h"
@@ -353,7 +352,7 @@ static void mxsfb_crtc_atomic_enable(struct drm_crtc *crtc,
 	struct drm_bridge_state *bridge_state = NULL;
 	struct drm_device *drm = mxsfb->drm;
 	u32 bus_format = 0;
-	dma_addr_t paddr;
+	dma_addr_t dma_addr;
 
 	pm_runtime_get_sync(drm->dev);
 	mxsfb_enable_axi_clk(mxsfb);
@@ -389,10 +388,10 @@ static void mxsfb_crtc_atomic_enable(struct drm_crtc *crtc,
 	mxsfb_crtc_mode_set_nofb(mxsfb, bridge_state, bus_format);
 
 	/* Write cur_buf as well to avoid an initial corrupt frame */
-	paddr = drm_fb_cma_get_gem_addr(new_pstate->fb, new_pstate, 0);
-	if (paddr) {
-		writel(paddr, mxsfb->base + mxsfb->devdata->cur_buf);
-		writel(paddr, mxsfb->base + mxsfb->devdata->next_buf);
+	dma_addr = drm_fb_dma_get_gem_addr(new_pstate->fb, new_pstate, 0);
+	if (dma_addr) {
+		writel(dma_addr, mxsfb->base + mxsfb->devdata->cur_buf);
+		writel(dma_addr, mxsfb->base + mxsfb->devdata->next_buf);
 	}
 
 	mxsfb_enable_controller(mxsfb);
@@ -531,8 +530,8 @@ static int mxsfb_plane_atomic_check(struct drm_plane *plane,
 						   &mxsfb->crtc);
 
 	return drm_atomic_helper_check_plane_state(plane_state, crtc_state,
-						   DRM_PLANE_HELPER_NO_SCALING,
-						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
 						   false, true);
 }
 
@@ -542,11 +541,11 @@ static void mxsfb_plane_primary_atomic_update(struct drm_plane *plane,
 	struct mxsfb_drm_private *mxsfb = to_mxsfb_drm_private(plane->dev);
 	struct drm_plane_state *new_pstate = drm_atomic_get_new_plane_state(state,
 									    plane);
-	dma_addr_t paddr;
+	dma_addr_t dma_addr;
 
-	paddr = drm_fb_cma_get_gem_addr(new_pstate->fb, new_pstate, 0);
-	if (paddr)
-		writel(paddr, mxsfb->base + mxsfb->devdata->next_buf);
+	dma_addr = drm_fb_dma_get_gem_addr(new_pstate->fb, new_pstate, 0);
+	if (dma_addr)
+		writel(dma_addr, mxsfb->base + mxsfb->devdata->next_buf);
 }
 
 static void mxsfb_plane_overlay_atomic_update(struct drm_plane *plane,
@@ -557,11 +556,11 @@ static void mxsfb_plane_overlay_atomic_update(struct drm_plane *plane,
 	struct mxsfb_drm_private *mxsfb = to_mxsfb_drm_private(plane->dev);
 	struct drm_plane_state *new_pstate = drm_atomic_get_new_plane_state(state,
 									    plane);
-	dma_addr_t paddr;
+	dma_addr_t dma_addr;
 	u32 ctrl;
 
-	paddr = drm_fb_cma_get_gem_addr(new_pstate->fb, new_pstate, 0);
-	if (!paddr) {
+	dma_addr = drm_fb_dma_get_gem_addr(new_pstate->fb, new_pstate, 0);
+	if (!dma_addr) {
 		writel(0, mxsfb->base + LCDC_AS_CTRL);
 		return;
 	}
@@ -572,16 +571,16 @@ static void mxsfb_plane_overlay_atomic_update(struct drm_plane *plane,
 	 * is understood, live with the 16 initial invalid pixels on the first
 	 * line and start 64 bytes within the framebuffer.
 	 */
-	paddr += 64;
+	dma_addr += 64;
 
-	writel(paddr, mxsfb->base + LCDC_AS_NEXT_BUF);
+	writel(dma_addr, mxsfb->base + LCDC_AS_NEXT_BUF);
 
 	/*
 	 * If the plane was previously disabled, write LCDC_AS_BUF as well to
 	 * provide the first buffer.
 	 */
 	if (!old_pstate->fb)
-		writel(paddr, mxsfb->base + LCDC_AS_BUF);
+		writel(dma_addr, mxsfb->base + LCDC_AS_BUF);
 
 	ctrl = AS_CTRL_AS_ENABLE | AS_CTRL_ALPHA(255);
 
diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
index f9e962fd94d0..ee92d576d277 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
@@ -1275,31 +1275,9 @@ static const uint32_t modeset_formats[] = {
         DRM_FORMAT_XRGB1555,
 };
 
-static struct drm_plane *
-create_primary_plane(struct drm_device *dev)
-{
-        struct drm_plane *primary;
-        int ret;
-
-        primary = kzalloc(sizeof(*primary), GFP_KERNEL);
-        if (primary == NULL) {
-                DRM_DEBUG_KMS("Failed to allocate primary plane\n");
-                return NULL;
-        }
-
-        /* possible_crtc's will be filled in later by crtc_init */
-        ret = drm_universal_plane_init(dev, primary, 0,
-                                       &drm_primary_helper_funcs,
-                                       modeset_formats,
-                                       ARRAY_SIZE(modeset_formats), NULL,
-                                       DRM_PLANE_TYPE_PRIMARY, NULL);
-        if (ret) {
-                kfree(primary);
-                primary = NULL;
-        }
-
-        return primary;
-}
+static const struct drm_plane_funcs nv04_primary_plane_funcs = {
+	DRM_PLANE_NON_ATOMIC_FUNCS,
+};
 
 static int nv04_crtc_vblank_handler(struct nvif_notify *notify)
 {
@@ -1315,6 +1293,7 @@ nv04_crtc_create(struct drm_device *dev, int crtc_num)
 {
 	struct nouveau_display *disp = nouveau_display(dev);
 	struct nouveau_crtc *nv_crtc;
+	struct drm_plane *primary;
 	int ret;
 
 	nv_crtc = kzalloc(sizeof(*nv_crtc), GFP_KERNEL);
@@ -1329,8 +1308,18 @@ nv04_crtc_create(struct drm_device *dev, int crtc_num)
 	nv_crtc->save = nv_crtc_save;
 	nv_crtc->restore = nv_crtc_restore;
 
-	drm_crtc_init_with_planes(dev, &nv_crtc->base,
-                                  create_primary_plane(dev), NULL,
+	primary = __drm_universal_plane_alloc(dev, sizeof(*primary), 0, 0,
+					      &nv04_primary_plane_funcs,
+					      modeset_formats,
+					      ARRAY_SIZE(modeset_formats), NULL,
+					      DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (IS_ERR(primary)) {
+		ret = PTR_ERR(primary);
+		kfree(nv_crtc);
+		return ret;
+	}
+
+	drm_crtc_init_with_planes(dev, &nv_crtc->base, primary, NULL,
                                   &nv04_crtc_funcs, NULL);
 	drm_crtc_helper_add(&nv_crtc->base, &nv04_crtc_helper_funcs);
 	drm_mode_crtc_set_gamma_size(&nv_crtc->base, 256);
diff --git a/drivers/gpu/drm/nouveau/dispnv04/overlay.c b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
index 37e63e98cd08..33f29736024a 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/overlay.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
@@ -296,9 +296,10 @@ nv10_overlay_init(struct drm_device *device)
 		break;
 	}
 
-	ret = drm_plane_init(device, &plane->base, 3 /* both crtc's */,
-			     &nv10_plane_funcs,
-			     formats, num_formats, false);
+	ret = drm_universal_plane_init(device, &plane->base, 3 /* both crtc's */,
+				       &nv10_plane_funcs,
+				       formats, num_formats, NULL,
+				       DRM_PLANE_TYPE_OVERLAY, NULL);
 	if (ret)
 		goto err;
 
@@ -475,9 +476,9 @@ nv04_overlay_init(struct drm_device *device)
 	if (!plane)
 		return;
 
-	ret = drm_plane_init(device, &plane->base, 1 /* single crtc */,
-			     &nv04_plane_funcs,
-			     formats, 2, false);
+	ret = drm_universal_plane_init(device, &plane->base, 1 /* single crtc */,
+				       &nv04_plane_funcs, formats, 2, NULL,
+				       DRM_PLANE_TYPE_OVERLAY, NULL);
 	if (ret)
 		goto err;
 
diff --git a/drivers/gpu/drm/nouveau/dispnv50/base507c.c b/drivers/gpu/drm/nouveau/dispnv50/base507c.c
index cad5a646983a..70c62b861276 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/base507c.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/base507c.c
@@ -29,7 +29,6 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 
 #include "nouveau_bo.h"
 
@@ -237,8 +236,8 @@ base507c_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
 	int ret;
 
 	ret = drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, true);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/nouveau/dispnv50/curs507a.c b/drivers/gpu/drm/nouveau/dispnv50/curs507a.c
index cd2c79e4b7af..78ee32da01c8 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/curs507a.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/curs507a.c
@@ -29,7 +29,6 @@
 #include <nvhw/class/cl507a.h>
 
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 
 bool
 curs507a_space(struct nv50_wndw *wndw)
@@ -103,8 +102,8 @@ curs507a_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
 	int ret;
 
 	ret = drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  true, true);
 	asyh->curs.visible = asyw->state.visible;
 	if (ret || !asyh->curs.visible)
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c
index a53d685a77eb..33c97d510999 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c
@@ -39,7 +39,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
@@ -933,6 +932,7 @@ struct nv50_msto {
 	struct nv50_head *head;
 	struct nv50_mstc *mstc;
 	bool disabled;
+	bool enabled;
 };
 
 struct nouveau_encoder *nv50_real_outp(struct drm_encoder *encoder)
@@ -948,57 +948,37 @@ struct nouveau_encoder *nv50_real_outp(struct drm_encoder *encoder)
 	return msto->mstc->mstm->outp;
 }
 
-static struct drm_dp_payload *
-nv50_msto_payload(struct nv50_msto *msto)
-{
-	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
-	struct nv50_mstc *mstc = msto->mstc;
-	struct nv50_mstm *mstm = mstc->mstm;
-	int vcpi = mstc->port->vcpi.vcpi, i;
-
-	WARN_ON(!mutex_is_locked(&mstm->mgr.payload_lock));
-
-	NV_ATOMIC(drm, "%s: vcpi %d\n", msto->encoder.name, vcpi);
-	for (i = 0; i < mstm->mgr.max_payloads; i++) {
-		struct drm_dp_payload *payload = &mstm->mgr.payloads[i];
-		NV_ATOMIC(drm, "%s: %d: vcpi %d start 0x%02x slots 0x%02x\n",
-			  mstm->outp->base.base.name, i, payload->vcpi,
-			  payload->start_slot, payload->num_slots);
-	}
-
-	for (i = 0; i < mstm->mgr.max_payloads; i++) {
-		struct drm_dp_payload *payload = &mstm->mgr.payloads[i];
-		if (payload->vcpi == vcpi)
-			return payload;
-	}
-
-	return NULL;
-}
-
 static void
-nv50_msto_cleanup(struct nv50_msto *msto)
+nv50_msto_cleanup(struct drm_atomic_state *state,
+		  struct drm_dp_mst_topology_state *mst_state,
+		  struct drm_dp_mst_topology_mgr *mgr,
+		  struct nv50_msto *msto)
 {
 	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
-	struct nv50_mstc *mstc = msto->mstc;
-	struct nv50_mstm *mstm = mstc->mstm;
-
-	if (!msto->disabled)
-		return;
+	struct drm_dp_mst_atomic_payload *payload =
+		drm_atomic_get_mst_payload_state(mst_state, msto->mstc->port);
 
 	NV_ATOMIC(drm, "%s: msto cleanup\n", msto->encoder.name);
 
-	drm_dp_mst_deallocate_vcpi(&mstm->mgr, mstc->port);
-
-	msto->mstc = NULL;
-	msto->disabled = false;
+	if (msto->disabled) {
+		msto->mstc = NULL;
+		msto->disabled = false;
+	} else if (msto->enabled) {
+		drm_dp_add_payload_part2(mgr, state, payload);
+		msto->enabled = false;
+	}
 }
 
 static void
-nv50_msto_prepare(struct nv50_msto *msto)
+nv50_msto_prepare(struct drm_atomic_state *state,
+		  struct drm_dp_mst_topology_state *mst_state,
+		  struct drm_dp_mst_topology_mgr *mgr,
+		  struct nv50_msto *msto)
 {
 	struct nouveau_drm *drm = nouveau_drm(msto->encoder.dev);
 	struct nv50_mstc *mstc = msto->mstc;
 	struct nv50_mstm *mstm = mstc->mstm;
+	struct drm_dp_mst_atomic_payload *payload;
 	struct {
 		struct nv50_disp_mthd_v1 base;
 		struct nv50_disp_sor_dp_mst_vcpi_v0 vcpi;
@@ -1010,17 +990,21 @@ nv50_msto_prepare(struct nv50_msto *msto)
 			       (0x0100 << msto->head->base.index),
 	};
 
-	mutex_lock(&mstm->mgr.payload_lock);
-
 	NV_ATOMIC(drm, "%s: msto prepare\n", msto->encoder.name);
-	if (mstc->port->vcpi.vcpi > 0) {
-		struct drm_dp_payload *payload = nv50_msto_payload(msto);
-		if (payload) {
-			args.vcpi.start_slot = payload->start_slot;
-			args.vcpi.num_slots = payload->num_slots;
-			args.vcpi.pbn = mstc->port->vcpi.pbn;
-			args.vcpi.aligned_pbn = mstc->port->vcpi.aligned_pbn;
-		}
+
+	payload = drm_atomic_get_mst_payload_state(mst_state, mstc->port);
+
+	// TODO: Figure out if we want to do a better job of handling VCPI allocation failures here?
+	if (msto->disabled) {
+		drm_dp_remove_payload(mgr, mst_state, payload);
+	} else {
+		if (msto->enabled)
+			drm_dp_add_payload_part1(mgr, mst_state, payload);
+
+		args.vcpi.start_slot = payload->vc_start_slot;
+		args.vcpi.num_slots = payload->time_slots;
+		args.vcpi.pbn = payload->pbn;
+		args.vcpi.aligned_pbn = payload->time_slots * mst_state->pbn_div;
 	}
 
 	NV_ATOMIC(drm, "%s: %s: %02x %02x %04x %04x\n",
@@ -1029,7 +1013,6 @@ nv50_msto_prepare(struct nv50_msto *msto)
 		  args.vcpi.pbn, args.vcpi.aligned_pbn);
 
 	nvif_mthd(&drm->display->disp.object, 0, &args, sizeof(args));
-	mutex_unlock(&mstm->mgr.payload_lock);
 }
 
 static int
@@ -1039,6 +1022,7 @@ nv50_msto_atomic_check(struct drm_encoder *encoder,
 {
 	struct drm_atomic_state *state = crtc_state->state;
 	struct drm_connector *connector = conn_state->connector;
+	struct drm_dp_mst_topology_state *mst_state;
 	struct nv50_mstc *mstc = nv50_mstc(connector);
 	struct nv50_mstm *mstm = mstc->mstm;
 	struct nv50_head_atom *asyh = nv50_head_atom(crtc_state);
@@ -1050,7 +1034,7 @@ nv50_msto_atomic_check(struct drm_encoder *encoder,
 	if (ret)
 		return ret;
 
-	if (!crtc_state->mode_changed && !crtc_state->connectors_changed)
+	if (!drm_atomic_crtc_needs_modeset(crtc_state))
 		return 0;
 
 	/*
@@ -1066,8 +1050,18 @@ nv50_msto_atomic_check(struct drm_encoder *encoder,
 						    false);
 	}
 
-	slots = drm_dp_atomic_find_vcpi_slots(state, &mstm->mgr, mstc->port,
-					      asyh->dp.pbn, 0);
+	mst_state = drm_atomic_get_mst_topology_state(state, &mstm->mgr);
+	if (IS_ERR(mst_state))
+		return PTR_ERR(mst_state);
+
+	if (!mst_state->pbn_div) {
+		struct nouveau_encoder *outp = mstc->mstm->outp;
+
+		mst_state->pbn_div = drm_dp_get_vc_payload_bw(&mstm->mgr,
+							      outp->dp.link_bw, outp->dp.link_nr);
+	}
+
+	slots = drm_dp_atomic_find_time_slots(state, &mstm->mgr, mstc->port, asyh->dp.pbn);
 	if (slots < 0)
 		return slots;
 
@@ -1099,7 +1093,6 @@ nv50_msto_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *st
 	struct drm_connector *connector;
 	struct drm_connector_list_iter conn_iter;
 	u8 proto;
-	bool r;
 
 	drm_connector_list_iter_begin(encoder->dev, &conn_iter);
 	drm_for_each_connector_iter(connector, &conn_iter) {
@@ -1114,10 +1107,6 @@ nv50_msto_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *st
 	if (WARN_ON(!mstc))
 		return;
 
-	r = drm_dp_mst_allocate_vcpi(&mstm->mgr, mstc->port, asyh->dp.pbn, asyh->dp.tu);
-	if (!r)
-		DRM_DEBUG_KMS("Failed to allocate VCPI\n");
-
 	if (!mstm->links++)
 		nv50_outp_acquire(mstm->outp, false /*XXX: MST audio.*/);
 
@@ -1130,6 +1119,7 @@ nv50_msto_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *st
 			   nv50_dp_bpc_to_depth(asyh->or.bpc));
 
 	msto->mstc = mstc;
+	msto->enabled = true;
 	mstm->modified = true;
 }
 
@@ -1140,8 +1130,6 @@ nv50_msto_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *s
 	struct nv50_mstc *mstc = msto->mstc;
 	struct nv50_mstm *mstm = mstc->mstm;
 
-	drm_dp_mst_reset_vcpi_slots(&mstm->mgr, mstc->port);
-
 	mstm->outp->update(mstm->outp, msto->head->base.index, NULL, 0, 0);
 	mstm->modified = true;
 	if (!--mstm->links)
@@ -1256,29 +1244,8 @@ nv50_mstc_atomic_check(struct drm_connector *connector,
 {
 	struct nv50_mstc *mstc = nv50_mstc(connector);
 	struct drm_dp_mst_topology_mgr *mgr = &mstc->mstm->mgr;
-	struct drm_connector_state *new_conn_state =
-		drm_atomic_get_new_connector_state(state, connector);
-	struct drm_connector_state *old_conn_state =
-		drm_atomic_get_old_connector_state(state, connector);
-	struct drm_crtc_state *crtc_state;
-	struct drm_crtc *new_crtc = new_conn_state->crtc;
-
-	if (!old_conn_state->crtc)
-		return 0;
-
-	/* We only want to free VCPI if this state disables the CRTC on this
-	 * connector
-	 */
-	if (new_crtc) {
-		crtc_state = drm_atomic_get_new_crtc_state(state, new_crtc);
-
-		if (!crtc_state ||
-		    !drm_atomic_crtc_needs_modeset(crtc_state) ||
-		    crtc_state->enable)
-			return 0;
-	}
 
-	return drm_dp_atomic_release_vcpi_slots(state, mgr, mstc->port);
+	return drm_dp_atomic_release_time_slots(state, mgr, mstc->port);
 }
 
 static int
@@ -1382,7 +1349,9 @@ nv50_mstc_new(struct nv50_mstm *mstm, struct drm_dp_mst_port *port,
 }
 
 static void
-nv50_mstm_cleanup(struct nv50_mstm *mstm)
+nv50_mstm_cleanup(struct drm_atomic_state *state,
+		  struct drm_dp_mst_topology_state *mst_state,
+		  struct nv50_mstm *mstm)
 {
 	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
 	struct drm_encoder *encoder;
@@ -1390,14 +1359,12 @@ nv50_mstm_cleanup(struct nv50_mstm *mstm)
 	NV_ATOMIC(drm, "%s: mstm cleanup\n", mstm->outp->base.base.name);
 	drm_dp_check_act_status(&mstm->mgr);
 
-	drm_dp_update_payload_part2(&mstm->mgr);
-
 	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
 		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
 			struct nv50_msto *msto = nv50_msto(encoder);
 			struct nv50_mstc *mstc = msto->mstc;
 			if (mstc && mstc->mstm == mstm)
-				nv50_msto_cleanup(msto);
+				nv50_msto_cleanup(state, mst_state, &mstm->mgr, msto);
 		}
 	}
 
@@ -1405,20 +1372,34 @@ nv50_mstm_cleanup(struct nv50_mstm *mstm)
 }
 
 static void
-nv50_mstm_prepare(struct nv50_mstm *mstm)
+nv50_mstm_prepare(struct drm_atomic_state *state,
+		  struct drm_dp_mst_topology_state *mst_state,
+		  struct nv50_mstm *mstm)
 {
 	struct nouveau_drm *drm = nouveau_drm(mstm->outp->base.base.dev);
 	struct drm_encoder *encoder;
 
 	NV_ATOMIC(drm, "%s: mstm prepare\n", mstm->outp->base.base.name);
-	drm_dp_update_payload_part1(&mstm->mgr, 1);
 
+	/* Disable payloads first */
 	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
 		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
 			struct nv50_msto *msto = nv50_msto(encoder);
 			struct nv50_mstc *mstc = msto->mstc;
-			if (mstc && mstc->mstm == mstm)
-				nv50_msto_prepare(msto);
+			if (mstc && mstc->mstm == mstm && msto->disabled)
+				nv50_msto_prepare(state, mst_state, &mstm->mgr, msto);
+		}
+	}
+
+	/* Add payloads for new heads, while also updating the start slots of any unmodified (but
+	 * active) heads that may have had their VC slots shifted left after the previous step
+	 */
+	drm_for_each_encoder(encoder, mstm->outp->base.base.dev) {
+		if (encoder->encoder_type == DRM_MODE_ENCODER_DPMST) {
+			struct nv50_msto *msto = nv50_msto(encoder);
+			struct nv50_mstc *mstc = msto->mstc;
+			if (mstc && mstc->mstm == mstm && !msto->disabled)
+				nv50_msto_prepare(state, mst_state, &mstm->mgr, msto);
 		}
 	}
 
@@ -1615,9 +1596,7 @@ nv50_mstm_new(struct nouveau_encoder *outp, struct drm_dp_aux *aux, int aux_max,
 	mstm->mgr.cbs = &nv50_mstm;
 
 	ret = drm_dp_mst_topology_mgr_init(&mstm->mgr, dev, aux, aux_max,
-					   max_payloads, outp->dcb->dpconf.link_nr,
-					   drm_dp_bw_code_to_link_rate(outp->dcb->dpconf.link_bw),
-					   conn_base_id);
+					   max_payloads, conn_base_id);
 	if (ret)
 		return ret;
 
@@ -1835,7 +1814,7 @@ nv50_sor_func = {
 	.destroy = nv50_sor_destroy,
 };
 
-static bool nv50_has_mst(struct nouveau_drm *drm)
+bool nv50_has_mst(struct nouveau_drm *drm)
 {
 	struct nvkm_bios *bios = nvxx_bios(&drm->client.device);
 	u32 data;
@@ -2069,20 +2048,20 @@ nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe)
 static void
 nv50_disp_atomic_commit_core(struct drm_atomic_state *state, u32 *interlock)
 {
+	struct drm_dp_mst_topology_mgr *mgr;
+	struct drm_dp_mst_topology_state *mst_state;
 	struct nouveau_drm *drm = nouveau_drm(state->dev);
 	struct nv50_disp *disp = nv50_disp(drm->dev);
 	struct nv50_core *core = disp->core;
 	struct nv50_mstm *mstm;
-	struct drm_encoder *encoder;
+	int i;
 
 	NV_ATOMIC(drm, "commit core %08x\n", interlock[NV50_DISP_INTERLOCK_BASE]);
 
-	drm_for_each_encoder(encoder, drm->dev) {
-		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
-			mstm = nouveau_encoder(encoder)->dp.mstm;
-			if (mstm && mstm->modified)
-				nv50_mstm_prepare(mstm);
-		}
+	for_each_new_mst_mgr_in_state(state, mgr, mst_state, i) {
+		mstm = nv50_mstm(mgr);
+		if (mstm->modified)
+			nv50_mstm_prepare(state, mst_state, mstm);
 	}
 
 	core->func->ntfy_init(disp->sync, NV50_DISP_CORE_NTFY);
@@ -2091,12 +2070,10 @@ nv50_disp_atomic_commit_core(struct drm_atomic_state *state, u32 *interlock)
 				       disp->core->chan.base.device))
 		NV_ERROR(drm, "core notifier timeout\n");
 
-	drm_for_each_encoder(encoder, drm->dev) {
-		if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
-			mstm = nouveau_encoder(encoder)->dp.mstm;
-			if (mstm && mstm->modified)
-				nv50_mstm_cleanup(mstm);
-		}
+	for_each_new_mst_mgr_in_state(state, mgr, mst_state, i) {
+		mstm = nv50_mstm(mgr);
+		if (mstm->modified)
+			nv50_mstm_cleanup(state, mst_state, mstm);
 	}
 }
 
@@ -2137,6 +2114,7 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 	nv50_crc_atomic_stop_reporting(state);
 	drm_atomic_helper_wait_for_fences(dev, state, false);
 	drm_atomic_helper_wait_for_dependencies(state);
+	drm_dp_mst_atomic_wait_for_dependencies(state);
 	drm_atomic_helper_update_legacy_modeset_state(dev, state);
 	drm_atomic_helper_calc_timestamping_constants(state);
 
@@ -2617,6 +2595,11 @@ nv50_disp_func = {
 	.atomic_state_free = nv50_disp_atomic_state_free,
 };
 
+static const struct drm_mode_config_helper_funcs
+nv50_disp_helper_func = {
+	.atomic_commit_setup = drm_dp_mst_atomic_setup_commit,
+};
+
 /******************************************************************************
  * Init
  *****************************************************************************/
@@ -2700,6 +2683,7 @@ nv50_display_create(struct drm_device *dev)
 	nouveau_display(dev)->fini = nv50_display_fini;
 	disp->disp = &nouveau_display(dev)->disp;
 	dev->mode_config.funcs = &nv50_disp_func;
+	dev->mode_config.helper_private = &nv50_disp_helper_func;
 	dev->mode_config.quirk_addfb_prefer_xbgr_30bpp = true;
 	dev->mode_config.normalize_zpos = true;
 
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.h b/drivers/gpu/drm/nouveau/dispnv50/disp.h
index 38dec11e7dda..9d66c9c726c3 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.h
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.h
@@ -106,6 +106,8 @@ void nv50_dmac_destroy(struct nv50_dmac *);
  */
 struct nouveau_encoder *nv50_real_outp(struct drm_encoder *encoder);
 
+bool nv50_has_mst(struct nouveau_drm *drm);
+
 u32 *evo_wait(struct nv50_dmac *, int nr);
 void evo_kick(u32 *, struct nv50_dmac *);
 
diff --git a/drivers/gpu/drm/nouveau/dispnv50/ovly507e.c b/drivers/gpu/drm/nouveau/dispnv50/ovly507e.c
index d4af69e903ad..797c1e4e0eaa 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/ovly507e.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/ovly507e.c
@@ -24,7 +24,6 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 
 #include <nvif/if0014.h>
 #include <nvif/push507c.h>
@@ -106,8 +105,8 @@ ovly507e_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
 	int ret;
 
 	ret = drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  true, true);
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndwc37e.c b/drivers/gpu/drm/nouveau/dispnv50/wndwc37e.c
index 082a66d59506..b3deea5aca58 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/wndwc37e.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/wndwc37e.c
@@ -23,7 +23,6 @@
 #include "atom.h"
 
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <nouveau_bo.h>
 
 #include <nvif/if0014.h>
@@ -297,8 +296,8 @@ wndwc37e_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
 		 struct nv50_head_atom *asyh)
 {
 	return drm_atomic_helper_check_plane_state(&asyw->state, &asyh->state,
-						   DRM_PLANE_HELPER_NO_SCALING,
-						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
 						   true, true);
 }
 
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndwc57e.c b/drivers/gpu/drm/nouveau/dispnv50/wndwc57e.c
index 31167c398708..1d214a4b960a 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/wndwc57e.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/wndwc57e.c
@@ -23,7 +23,6 @@
 #include "atom.h"
 
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <nouveau_bo.h>
 
 #include <nvif/pushc37b.h>
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c
index 6140db756d06..8cf096f841a9 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c
@@ -386,3 +386,13 @@ nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector)
 
 	return kmemdup(edid, EDID_LENGTH, GFP_KERNEL);
 }
+
+bool nouveau_acpi_video_backlight_use_native(void)
+{
+	return acpi_video_backlight_use_native();
+}
+
+void nouveau_acpi_video_register_backlight(void)
+{
+	acpi_video_register_backlight();
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.h b/drivers/gpu/drm/nouveau/nouveau_acpi.h
index 330f9b837066..e39dd8b94b8b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.h
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.h
@@ -11,6 +11,8 @@ void nouveau_register_dsm_handler(void);
 void nouveau_unregister_dsm_handler(void);
 void nouveau_switcheroo_optimus_dsm(void);
 void *nouveau_acpi_edid(struct drm_device *, struct drm_connector *);
+bool nouveau_acpi_video_backlight_use_native(void);
+void nouveau_acpi_video_register_backlight(void);
 #else
 static inline bool nouveau_is_optimus(void) { return false; };
 static inline bool nouveau_is_v1_dsm(void) { return false; };
@@ -18,6 +20,8 @@ static inline void nouveau_register_dsm_handler(void) {}
 static inline void nouveau_unregister_dsm_handler(void) {}
 static inline void nouveau_switcheroo_optimus_dsm(void) {}
 static inline void *nouveau_acpi_edid(struct drm_device *dev, struct drm_connector *connector) { return NULL; }
+static inline bool nouveau_acpi_video_backlight_use_native(void) { return true; }
+static inline void nouveau_acpi_video_register_backlight(void) {}
 #endif
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c
index a2141d3d9b1d..a614582779ca 100644
--- a/drivers/gpu/drm/nouveau/nouveau_backlight.c
+++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c
@@ -38,6 +38,7 @@
 #include "nouveau_reg.h"
 #include "nouveau_encoder.h"
 #include "nouveau_connector.h"
+#include "nouveau_acpi.h"
 
 static struct ida bl_ida;
 #define BL_NAME_SIZE 15 // 12 for name + 2 for digits + 1 for '\0'
@@ -405,6 +406,11 @@ nouveau_backlight_init(struct drm_connector *connector)
 		goto fail_alloc;
 	}
 
+	if (!nouveau_acpi_video_backlight_use_native()) {
+		NV_INFO(drm, "Skipping nv_backlight registration\n");
+		goto fail_alloc;
+	}
+
 	if (!nouveau_get_backlight_name(backlight_name, bl)) {
 		NV_ERROR(drm, "Failed to retrieve a unique name for the backlight interface\n");
 		goto fail_alloc;
@@ -430,6 +436,13 @@ nouveau_backlight_init(struct drm_connector *connector)
 
 fail_alloc:
 	kfree(bl);
+	/*
+	 * If we get here we have an internal panel, but no nv_backlight,
+	 * try registering an ACPI video backlight device instead.
+	 */
+	if (ret == 0)
+		nouveau_acpi_video_register_backlight();
+
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index e29175e4b44c..126b3c6e12f9 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -281,8 +281,10 @@ nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain,
 			break;
 	}
 
-	if (WARN_ON(pi < 0))
+	if (WARN_ON(pi < 0)) {
+		kfree(nvbo);
 		return ERR_PTR(-EINVAL);
+	}
 
 	/* Disable compression if suitable settings couldn't be found. */
 	if (nvbo->comp && !vmm->page[pi].comp) {
@@ -307,9 +309,9 @@ nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain,
 	nouveau_bo_placement_set(nvbo, domain, 0);
 	INIT_LIST_HEAD(&nvbo->io_reserve_lru);
 
-	ret = ttm_bo_init(nvbo->bo.bdev, &nvbo->bo, size, type,
-			  &nvbo->placement, align >> PAGE_SHIFT, false, sg,
-			  robj, nouveau_bo_del_ttm);
+	ret = ttm_bo_init_validate(nvbo->bo.bdev, &nvbo->bo, type,
+				   &nvbo->placement, align >> PAGE_SHIFT, false,
+				   sg, robj, nouveau_bo_del_ttm);
 	if (ret) {
 		/* ttm will call nouveau_bo_del_ttm if it fails.. */
 		return ret;
@@ -1015,7 +1017,8 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict,
 	}
 
 	/* Fake bo copy. */
-	if (old_reg->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
+	if (!old_reg || (old_reg->mem_type == TTM_PL_SYSTEM &&
+			 !bo->ttm)) {
 		ttm_bo_move_null(bo, new_reg);
 		goto out;
 	}
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index 43a9d1e1cf71..1991bbb1d05c 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -504,7 +504,8 @@ nouveau_connector_set_encoder(struct drm_connector *connector,
 			connector->interlace_allowed =
 				nv_encoder->caps.dp_interlace;
 		else
-			connector->interlace_allowed = true;
+			connector->interlace_allowed =
+				drm->client.device.info.family < NV_DEVICE_INFO_V0_VOLTA;
 		connector->doublescan_allowed = true;
 	} else
 	if (nv_encoder->dcb->type == DCB_OUTPUT_LVDS ||
@@ -1105,11 +1106,25 @@ nouveau_connector_best_encoder(struct drm_connector *connector)
 	return NULL;
 }
 
+static int
+nouveau_connector_atomic_check(struct drm_connector *connector, struct drm_atomic_state *state)
+{
+	struct nouveau_connector *nv_conn = nouveau_connector(connector);
+	struct drm_connector_state *conn_state =
+		drm_atomic_get_new_connector_state(state, connector);
+
+	if (!nv_conn->dp_encoder || !nv50_has_mst(nouveau_drm(connector->dev)))
+		return 0;
+
+	return drm_dp_mst_root_conn_atomic_check(conn_state, &nv_conn->dp_encoder->dp.mstm->mgr);
+}
+
 static const struct drm_connector_helper_funcs
 nouveau_connector_helper_funcs = {
 	.get_modes = nouveau_connector_get_modes,
 	.mode_valid = nouveau_connector_mode_valid,
 	.best_encoder = nouveau_connector_best_encoder,
+	.atomic_check = nouveau_connector_atomic_check,
 };
 
 static const struct drm_connector_funcs
@@ -1367,7 +1382,7 @@ nouveau_connector_create(struct drm_device *dev,
 			return ERR_PTR(-ENOMEM);
 		}
 		drm_dp_aux_init(&nv_connector->aux);
-		fallthrough;
+		break;
 	default:
 		funcs = &nouveau_connector_funcs;
 		break;
@@ -1430,6 +1445,8 @@ nouveau_connector_create(struct drm_device *dev,
 
 	switch (type) {
 	case DRM_MODE_CONNECTOR_DisplayPort:
+		nv_connector->dp_encoder = find_encoder(&nv_connector->base, DCB_OUTPUT_DP);
+		fallthrough;
 	case DRM_MODE_CONNECTOR_eDP:
 		drm_dp_cec_register_connector(&nv_connector->aux, connector);
 		break;
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h
index 4bf0c703eee7..f4e17ff68bf9 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.h
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.h
@@ -128,6 +128,9 @@ struct nouveau_connector {
 
 	struct drm_dp_aux aux;
 
+	/* The fixed DP encoder for this connector, if there is one */
+	struct nouveau_encoder *dp_encoder;
+
 	int dithering_mode;
 	int scaling_mode;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_hwmon.c b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
index 1c3104d20571..a7db7c31064b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hwmon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
@@ -211,75 +211,24 @@ static const struct attribute_group temp1_auto_point_sensor_group = {
 
 #define N_ATTR_GROUPS   3
 
-static const u32 nouveau_config_chip[] = {
-	HWMON_C_UPDATE_INTERVAL,
-	0
-};
-
-static const u32 nouveau_config_in[] = {
-	HWMON_I_INPUT | HWMON_I_MIN | HWMON_I_MAX | HWMON_I_LABEL,
-	0
-};
-
-static const u32 nouveau_config_temp[] = {
-	HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_MAX_HYST |
-	HWMON_T_CRIT | HWMON_T_CRIT_HYST | HWMON_T_EMERGENCY |
-	HWMON_T_EMERGENCY_HYST,
-	0
-};
-
-static const u32 nouveau_config_fan[] = {
-	HWMON_F_INPUT,
-	0
-};
-
-static const u32 nouveau_config_pwm[] = {
-	HWMON_PWM_INPUT | HWMON_PWM_ENABLE,
-	0
-};
-
-static const u32 nouveau_config_power[] = {
-	HWMON_P_INPUT | HWMON_P_CAP_MAX | HWMON_P_CRIT,
-	0
-};
-
-static const struct hwmon_channel_info nouveau_chip = {
-	.type = hwmon_chip,
-	.config = nouveau_config_chip,
-};
-
-static const struct hwmon_channel_info nouveau_temp = {
-	.type = hwmon_temp,
-	.config = nouveau_config_temp,
-};
-
-static const struct hwmon_channel_info nouveau_fan = {
-	.type = hwmon_fan,
-	.config = nouveau_config_fan,
-};
-
-static const struct hwmon_channel_info nouveau_in = {
-	.type = hwmon_in,
-	.config = nouveau_config_in,
-};
-
-static const struct hwmon_channel_info nouveau_pwm = {
-	.type = hwmon_pwm,
-	.config = nouveau_config_pwm,
-};
-
-static const struct hwmon_channel_info nouveau_power = {
-	.type = hwmon_power,
-	.config = nouveau_config_power,
-};
-
 static const struct hwmon_channel_info *nouveau_info[] = {
-	&nouveau_chip,
-	&nouveau_temp,
-	&nouveau_fan,
-	&nouveau_in,
-	&nouveau_pwm,
-	&nouveau_power,
+	HWMON_CHANNEL_INFO(chip,
+			   HWMON_C_UPDATE_INTERVAL),
+	HWMON_CHANNEL_INFO(temp,
+			   HWMON_T_INPUT |
+			   HWMON_T_MAX | HWMON_T_MAX_HYST |
+			   HWMON_T_CRIT | HWMON_T_CRIT_HYST |
+			   HWMON_T_EMERGENCY | HWMON_T_EMERGENCY_HYST),
+	HWMON_CHANNEL_INFO(fan,
+			   HWMON_F_INPUT),
+	HWMON_CHANNEL_INFO(in,
+			   HWMON_I_INPUT |
+			   HWMON_I_MIN | HWMON_I_MAX |
+			   HWMON_I_LABEL),
+	HWMON_CHANNEL_INFO(pwm,
+			   HWMON_PWM_INPUT | HWMON_PWM_ENABLE),
+	HWMON_CHANNEL_INFO(power,
+			   HWMON_P_INPUT | HWMON_P_CAP_MAX | HWMON_P_CRIT),
 	NULL
 };
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c
index 2e517cdc24c9..76f8edefa637 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_mem.c
@@ -187,3 +187,32 @@ nouveau_mem_new(struct nouveau_cli *cli, u8 kind, u8 comp,
 	*res = &mem->base;
 	return 0;
 }
+
+bool
+nouveau_mem_intersects(struct ttm_resource *res,
+		       const struct ttm_place *place,
+		       size_t size)
+{
+	u32 num_pages = PFN_UP(size);
+
+	/* Don't evict BOs outside of the requested placement range */
+	if (place->fpfn >= (res->start + num_pages) ||
+	    (place->lpfn && place->lpfn <= res->start))
+		return false;
+
+	return true;
+}
+
+bool
+nouveau_mem_compatible(struct ttm_resource *res,
+		       const struct ttm_place *place,
+		       size_t size)
+{
+	u32 num_pages = PFN_UP(size);
+
+	if (res->start < place->fpfn ||
+	    (place->lpfn && (res->start + num_pages) > place->lpfn))
+		return false;
+
+	return true;
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.h b/drivers/gpu/drm/nouveau/nouveau_mem.h
index 325551eba5cd..1ee6cdb9ad9b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_mem.h
+++ b/drivers/gpu/drm/nouveau/nouveau_mem.h
@@ -25,6 +25,12 @@ int nouveau_mem_new(struct nouveau_cli *, u8 kind, u8 comp,
 		    struct ttm_resource **);
 void nouveau_mem_del(struct ttm_resource_manager *man,
 		     struct ttm_resource *);
+bool nouveau_mem_intersects(struct ttm_resource *res,
+			    const struct ttm_place *place,
+			    size_t size);
+bool nouveau_mem_compatible(struct ttm_resource *res,
+			    const struct ttm_place *place,
+			    size_t size);
 int nouveau_mem_vram(struct ttm_resource *, bool contig, u8 page);
 int nouveau_mem_host(struct ttm_resource *, struct ttm_tt *);
 void nouveau_mem_fini(struct nouveau_mem *);
diff --git a/drivers/gpu/drm/nouveau/nouveau_prime.c b/drivers/gpu/drm/nouveau/nouveau_prime.c
index 347488685f74..9608121e49b7 100644
--- a/drivers/gpu/drm/nouveau/nouveau_prime.c
+++ b/drivers/gpu/drm/nouveau/nouveau_prime.c
@@ -71,7 +71,6 @@ struct drm_gem_object *nouveau_gem_prime_import_sg_table(struct drm_device *dev,
 	ret = nouveau_bo_init(nvbo, size, align, NOUVEAU_GEM_DOMAIN_GART,
 			      sg, robj);
 	if (ret) {
-		nouveau_bo_ref(NULL, &nvbo);
 		obj = ERR_PTR(ret);
 		goto unlock;
 	}
diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c
index 85f1f5a0fe5d..9602c30928f2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ttm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c
@@ -42,6 +42,24 @@ nouveau_manager_del(struct ttm_resource_manager *man,
 	nouveau_mem_del(man, reg);
 }
 
+static bool
+nouveau_manager_intersects(struct ttm_resource_manager *man,
+			   struct ttm_resource *res,
+			   const struct ttm_place *place,
+			   size_t size)
+{
+	return nouveau_mem_intersects(res, place, size);
+}
+
+static bool
+nouveau_manager_compatible(struct ttm_resource_manager *man,
+			   struct ttm_resource *res,
+			   const struct ttm_place *place,
+			   size_t size)
+{
+	return nouveau_mem_compatible(res, place, size);
+}
+
 static int
 nouveau_vram_manager_new(struct ttm_resource_manager *man,
 			 struct ttm_buffer_object *bo,
@@ -73,6 +91,8 @@ nouveau_vram_manager_new(struct ttm_resource_manager *man,
 const struct ttm_resource_manager_func nouveau_vram_manager = {
 	.alloc = nouveau_vram_manager_new,
 	.free = nouveau_manager_del,
+	.intersects = nouveau_manager_intersects,
+	.compatible = nouveau_manager_compatible,
 };
 
 static int
@@ -97,6 +117,8 @@ nouveau_gart_manager_new(struct ttm_resource_manager *man,
 const struct ttm_resource_manager_func nouveau_gart_manager = {
 	.alloc = nouveau_gart_manager_new,
 	.free = nouveau_manager_del,
+	.intersects = nouveau_manager_intersects,
+	.compatible = nouveau_manager_compatible,
 };
 
 static int
@@ -130,6 +152,8 @@ nv04_gart_manager_new(struct ttm_resource_manager *man,
 const struct ttm_resource_manager_func nv04_gart_manager = {
 	.alloc = nv04_gart_manager_new,
 	.free = nouveau_manager_del,
+	.intersects = nouveau_manager_intersects,
+	.compatible = nouveau_manager_compatible,
 };
 
 static int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pmu.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pmu.c
index b4a308f3cf7b..49e2664a734c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pmu.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pmu.c
@@ -64,12 +64,9 @@ nvbios_pmuEp(struct nvkm_bios *bios, int idx, u8 *ver, u8 *hdr,
 	     struct nvbios_pmuE *info)
 {
 	u32 data = nvbios_pmuEe(bios, idx, ver, hdr);
-	memset(info, 0x00, sizeof(*info));
-	switch (!!data * *ver) {
-	default:
+	if (data) {
 		info->type = nvbios_rd08(bios, data + 0x00);
 		info->data = nvbios_rd32(bios, data + 0x02);
-		break;
 	}
 	return data;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gm20b.c
index a139dafffe06..7c33542f651b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gm20b.c
@@ -581,7 +581,7 @@ gm20b_clk_prog(struct nvkm_clk *base)
 
 	/*
 	 * Interim step for changing DVFS detection settings: low enough
-	 * frequency to be safe at at DVFS coeff = 0.
+	 * frequency to be safe at DVFS coeff = 0.
 	 *
 	 * 1. If voltage is increasing:
 	 * - safe frequency target matches the lowest - old - frequency
diff --git a/drivers/gpu/drm/omapdrm/dss/dispc.c b/drivers/gpu/drm/omapdrm/dss/dispc.c
index c4de142cc85b..0ee344ebcd1c 100644
--- a/drivers/gpu/drm/omapdrm/dss/dispc.c
+++ b/drivers/gpu/drm/omapdrm/dss/dispc.c
@@ -2451,7 +2451,7 @@ static int dispc_ovl_calc_scaling_44xx(struct dispc_device *dispc,
 
 	*decim_x = DIV_ROUND_UP(width, in_width_max);
 
-	*decim_x = *decim_x > decim_x_min ? *decim_x : decim_x_min;
+	*decim_x = max(*decim_x, decim_x_min);
 	if (*decim_x > *x_predecim)
 		return -EINVAL;
 
diff --git a/drivers/gpu/drm/omapdrm/dss/dss.c b/drivers/gpu/drm/omapdrm/dss/dss.c
index 0399f3390a0a..c4febb861910 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss.c
@@ -1176,6 +1176,7 @@ static void __dss_uninit_ports(struct dss_device *dss, unsigned int num_ports)
 		default:
 			break;
 		}
+		of_node_put(port);
 	}
 }
 
@@ -1208,11 +1209,13 @@ static int dss_init_ports(struct dss_device *dss)
 		default:
 			break;
 		}
+		of_node_put(port);
 	}
 
 	return 0;
 
 error:
+	of_node_put(port);
 	__dss_uninit_ports(dss, i);
 	return r;
 }
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index 06a719c104f4..63ddc5127f7b 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -10,7 +10,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_mode.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "omap_drv.h"
diff --git a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
index ac869acf80ea..61a27dd7392e 100644
--- a/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
+++ b/drivers/gpu/drm/omapdrm/omap_dmm_tiler.c
@@ -813,10 +813,8 @@ static int omap_dmm_probe(struct platform_device *dev)
 	}
 
 	omap_dmm->irq = platform_get_irq(dev, 0);
-	if (omap_dmm->irq < 0) {
-		dev_err(&dev->dev, "failed to get IRQ resource\n");
+	if (omap_dmm->irq < 0)
 		goto fail;
-	}
 
 	omap_dmm->dev = &dev->dev;
 
diff --git a/drivers/gpu/drm/omapdrm/omap_overlay.c b/drivers/gpu/drm/omapdrm/omap_overlay.c
index b0bc9ad2ef73..fb97c74386f2 100644
--- a/drivers/gpu/drm/omapdrm/omap_overlay.c
+++ b/drivers/gpu/drm/omapdrm/omap_overlay.c
@@ -6,7 +6,6 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 
 #include "omap_dmm_tiler.h"
 #include "omap_drv.h"
diff --git a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c
index b6cb537f7689..24a2ded08b45 100644
--- a/drivers/gpu/drm/omapdrm/omap_plane.c
+++ b/drivers/gpu/drm/omapdrm/omap_plane.c
@@ -8,7 +8,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index a9043eacce97..a582ddd583c2 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -165,8 +165,8 @@ config DRM_PANEL_ILITEK_IL9322
 config DRM_PANEL_ILITEK_ILI9341
 	tristate "Ilitek ILI9341 240x320 QVGA panels"
 	depends on OF && SPI
-	depends on DRM_KMS_HELPER
-	depends on DRM_GEM_CMA_HELPER
+	select DRM_KMS_HELPER
+	select DRM_GEM_DMA_HELPER
 	depends on BACKLIGHT_CLASS_DEVICE
 	select DRM_MIPI_DBI
 	help
diff --git a/drivers/gpu/drm/panel/panel-asus-z00t-tm5p5-n35596.c b/drivers/gpu/drm/panel/panel-asus-z00t-tm5p5-n35596.c
index 174ff434bd71..b3235781e6ba 100644
--- a/drivers/gpu/drm/panel/panel-asus-z00t-tm5p5-n35596.c
+++ b/drivers/gpu/drm/panel/panel-asus-z00t-tm5p5-n35596.c
@@ -321,7 +321,7 @@ static int tm5p5_nt35596_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int tm5p5_nt35596_remove(struct mipi_dsi_device *dsi)
+static void tm5p5_nt35596_remove(struct mipi_dsi_device *dsi)
 {
 	struct tm5p5_nt35596 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -332,8 +332,6 @@ static int tm5p5_nt35596_remove(struct mipi_dsi_device *dsi)
 			"Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id tm5p5_nt35596_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-boe-bf060y8m-aj0.c b/drivers/gpu/drm/panel/panel-boe-bf060y8m-aj0.c
index ef00cd67dc40..ad58840eda41 100644
--- a/drivers/gpu/drm/panel/panel-boe-bf060y8m-aj0.c
+++ b/drivers/gpu/drm/panel/panel-boe-bf060y8m-aj0.c
@@ -410,7 +410,7 @@ static int boe_bf060y8m_aj0_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int boe_bf060y8m_aj0_remove(struct mipi_dsi_device *dsi)
+static void boe_bf060y8m_aj0_remove(struct mipi_dsi_device *dsi)
 {
 	struct boe_bf060y8m_aj0 *boe = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -420,8 +420,6 @@ static int boe_bf060y8m_aj0_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&boe->panel);
-
-	return 0;
 }
 
 static const struct of_device_id boe_bf060y8m_aj0_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-boe-himax8279d.c b/drivers/gpu/drm/panel/panel-boe-himax8279d.c
index 42854bd37fd5..d879b3b14c48 100644
--- a/drivers/gpu/drm/panel/panel-boe-himax8279d.c
+++ b/drivers/gpu/drm/panel/panel-boe-himax8279d.c
@@ -919,7 +919,7 @@ static int panel_probe(struct mipi_dsi_device *dsi)
 	return err;
 }
 
-static int panel_remove(struct mipi_dsi_device *dsi)
+static void panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct panel_info *pinfo = mipi_dsi_get_drvdata(dsi);
 	int err;
@@ -937,8 +937,6 @@ static int panel_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", err);
 
 	drm_panel_remove(&pinfo->base);
-
-	return 0;
 }
 
 static void panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
index 07f722f33fc5..857a2f0420d7 100644
--- a/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
+++ b/drivers/gpu/drm/panel/panel-boe-tv101wum-nl6.c
@@ -1622,7 +1622,7 @@ static void boe_panel_shutdown(struct mipi_dsi_device *dsi)
 	drm_panel_unprepare(&boe->base);
 }
 
-static int boe_panel_remove(struct mipi_dsi_device *dsi)
+static void boe_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct boe_panel *boe = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -1635,8 +1635,6 @@ static int boe_panel_remove(struct mipi_dsi_device *dsi)
 
 	if (boe->base.dev)
 		drm_panel_remove(&boe->base);
-
-	return 0;
 }
 
 static const struct of_device_id boe_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-dsi-cm.c b/drivers/gpu/drm/panel/panel-dsi-cm.c
index b0213a518f9d..ba17bcc4461c 100644
--- a/drivers/gpu/drm/panel/panel-dsi-cm.c
+++ b/drivers/gpu/drm/panel/panel-dsi-cm.c
@@ -579,7 +579,7 @@ err_bl:
 	return r;
 }
 
-static int dsicm_remove(struct mipi_dsi_device *dsi)
+static void dsicm_remove(struct mipi_dsi_device *dsi)
 {
 	struct panel_drv_data *ddata = mipi_dsi_get_drvdata(dsi);
 
@@ -593,8 +593,6 @@ static int dsicm_remove(struct mipi_dsi_device *dsi)
 
 	if (ddata->extbldev)
 		put_device(&ddata->extbldev->dev);
-
-	return 0;
 }
 
 static const struct dsic_panel_data taal_data = {
diff --git a/drivers/gpu/drm/panel/panel-ebbg-ft8719.c b/drivers/gpu/drm/panel/panel-ebbg-ft8719.c
index 386f8321b930..e85d63a176d0 100644
--- a/drivers/gpu/drm/panel/panel-ebbg-ft8719.c
+++ b/drivers/gpu/drm/panel/panel-ebbg-ft8719.c
@@ -250,7 +250,7 @@ static int ebbg_ft8719_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int ebbg_ft8719_remove(struct mipi_dsi_device *dsi)
+static void ebbg_ft8719_remove(struct mipi_dsi_device *dsi)
 {
 	struct ebbg_ft8719 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -260,8 +260,6 @@ static int ebbg_ft8719_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id ebbg_ft8719_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c
index b75c690bb0cc..4b39d1dd9140 100644
--- a/drivers/gpu/drm/panel/panel-edp.c
+++ b/drivers/gpu/drm/panel/panel-edp.c
@@ -53,7 +53,7 @@ struct panel_delay {
 	 * before the HPD signal is reliable. Ideally this is 0 but some panels,
 	 * board designs, or bad pulldown configs can cause a glitch here.
 	 *
-	 * NOTE: on some old panel data this number appers to be much too big.
+	 * NOTE: on some old panel data this number appears to be much too big.
 	 * Presumably some old panels simply didn't have HPD hooked up and put
 	 * the hpd_absent here because this field predates the
 	 * hpd_absent. While that works, it's non-ideal.
@@ -403,17 +403,10 @@ static int panel_edp_unprepare(struct drm_panel *panel)
 
 static int panel_edp_get_hpd_gpio(struct device *dev, struct panel_edp *p)
 {
-	int err;
-
 	p->hpd_gpio = devm_gpiod_get_optional(dev, "hpd", GPIOD_IN);
-	if (IS_ERR(p->hpd_gpio)) {
-		err = PTR_ERR(p->hpd_gpio);
-
-		if (err != -EPROBE_DEFER)
-			dev_err(dev, "failed to get 'hpd' GPIO: %d\n", err);
-
-		return err;
-	}
+	if (IS_ERR(p->hpd_gpio))
+		return dev_err_probe(dev, PTR_ERR(p->hpd_gpio),
+				     "failed to get 'hpd' GPIO\n");
 
 	return 0;
 }
@@ -832,12 +825,9 @@ static int panel_edp_probe(struct device *dev, const struct panel_desc *desc,
 
 	panel->enable_gpio = devm_gpiod_get_optional(dev, "enable",
 						     GPIOD_OUT_LOW);
-	if (IS_ERR(panel->enable_gpio)) {
-		err = PTR_ERR(panel->enable_gpio);
-		if (err != -EPROBE_DEFER)
-			dev_err(dev, "failed to request GPIO: %d\n", err);
-		return err;
-	}
+	if (IS_ERR(panel->enable_gpio))
+		return dev_err_probe(dev, PTR_ERR(panel->enable_gpio),
+				     "failed to request GPIO\n");
 
 	err = of_drm_get_panel_orientation(dev->of_node, &panel->orientation);
 	if (err) {
@@ -1855,6 +1845,12 @@ static const struct panel_delay delay_100_500_e200 = {
 	.enable = 200,
 };
 
+static const struct panel_delay delay_200_500_e200 = {
+	.hpd_absent = 200,
+	.unprepare = 500,
+	.enable = 200,
+};
+
 #define EDP_PANEL_ENTRY(vend_chr_0, vend_chr_1, vend_chr_2, product_id, _delay, _name) \
 { \
 	.name = _name, \
@@ -1871,6 +1867,9 @@ static const struct panel_delay delay_100_500_e200 = {
  * Sort first by vendor, then by product ID.
  */
 static const struct edp_panel_entry edp_panels[] = {
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x1062, &delay_200_500_e50, "B120XAN01.0"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x1e9b, &delay_200_500_e50, "B133UAN02.1"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x1ea5, &delay_200_500_e50, "B116XAK01.6"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"),
@@ -1878,10 +1877,19 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0786, &delay_200_500_p2e80, "NV116WHM-T01"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x07d1, &boe_nv133fhm_n61.delay, "NV133FHM-N61"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x082d, &boe_nv133fhm_n61.delay, "NV133FHM-N62"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x094b, &delay_200_500_e50, "NT116WHM-N21"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x098d, &boe_nv110wtm_n61.delay, "NV110WTM-N61"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x09dd, &delay_200_500_e50, "NT116WHM-N21"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a5d, &delay_200_500_e50, "NV116WHM-N45"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0ac5, &delay_200_500_e50, "NV116WHM-N4C"),
 
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x114c, &innolux_n116bca_ea1.delay, "N116BCA-EA1"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1152, &delay_200_500_e80_d50, "N116BCN-EA1"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1154, &delay_200_500_e80_d50, "N116BCA-EA2"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1247, &delay_200_500_e80_d50, "N120ACA-EA1"),
+
+	EDP_PANEL_ENTRY('I', 'V', 'O', 0x057d, &delay_200_500_e200, "R140NWF5 RH"),
+	EDP_PANEL_ENTRY('I', 'V', 'O', 0x854b, &delay_200_500_p2e100, "M133NW4J-R3"),
 
 	EDP_PANEL_ENTRY('K', 'D', 'B', 0x0624, &kingdisplay_kd116n21_30nv_a010.delay, "116N21-30NV-A010"),
 	EDP_PANEL_ENTRY('K', 'D', 'B', 0x1120, &delay_200_500_e80_d50, "116N29-30NK-C007"),
diff --git a/drivers/gpu/drm/panel/panel-elida-kd35t133.c b/drivers/gpu/drm/panel/panel-elida-kd35t133.c
index 01dd555a7f26..eee714cf3f49 100644
--- a/drivers/gpu/drm/panel/panel-elida-kd35t133.c
+++ b/drivers/gpu/drm/panel/panel-elida-kd35t133.c
@@ -321,7 +321,7 @@ static void kd35t133_shutdown(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to disable panel: %d\n", ret);
 }
 
-static int kd35t133_remove(struct mipi_dsi_device *dsi)
+static void kd35t133_remove(struct mipi_dsi_device *dsi)
 {
 	struct kd35t133 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -333,8 +333,6 @@ static int kd35t133_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id kd35t133_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-feixin-k101-im2ba02.c b/drivers/gpu/drm/panel/panel-feixin-k101-im2ba02.c
index cb0bb3076099..76572c922983 100644
--- a/drivers/gpu/drm/panel/panel-feixin-k101-im2ba02.c
+++ b/drivers/gpu/drm/panel/panel-feixin-k101-im2ba02.c
@@ -486,14 +486,12 @@ static int k101_im2ba02_dsi_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int k101_im2ba02_dsi_remove(struct mipi_dsi_device *dsi)
+static void k101_im2ba02_dsi_remove(struct mipi_dsi_device *dsi)
 {
 	struct k101_im2ba02 *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id k101_im2ba02_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c b/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
index ee61d60eceae..df493da50afe 100644
--- a/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
+++ b/drivers/gpu/drm/panel/panel-feiyang-fy07024di26a30d.c
@@ -233,14 +233,12 @@ static int feiyang_dsi_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int feiyang_dsi_remove(struct mipi_dsi_device *dsi)
+static void feiyang_dsi_remove(struct mipi_dsi_device *dsi)
 {
 	struct feiyang *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id feiyang_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9341.c b/drivers/gpu/drm/panel/panel-ilitek-ili9341.c
index 6826f4d4826a..39dc40cf681f 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9341.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9341.c
@@ -32,7 +32,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_mipi_dbi.h>
 #include <drm/drm_modes.h>
@@ -576,6 +576,7 @@ out_exit:
 }
 
 static const struct drm_simple_display_pipe_funcs ili9341_dbi_funcs = {
+	.mode_valid = mipi_dbi_pipe_mode_valid,
 	.enable = ili9341_dbi_enable,
 	.disable = mipi_dbi_pipe_disable,
 	.update = mipi_dbi_pipe_update,
@@ -586,12 +587,12 @@ static const struct drm_display_mode ili9341_dbi_mode = {
 	DRM_SIMPLE_MODE(240, 320, 37, 49),
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(ili9341_dbi_fops);
+DEFINE_DRM_GEM_DMA_FOPS(ili9341_dbi_fops);
 
 static struct drm_driver ili9341_dbi_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &ili9341_dbi_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "ili9341",
 	.desc			= "Ilitek ILI9341",
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
index 596861269774..cbb68caa36f2 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
@@ -923,14 +923,12 @@ static int ili9881c_dsi_probe(struct mipi_dsi_device *dsi)
 	return mipi_dsi_attach(dsi);
 }
 
-static int ili9881c_dsi_remove(struct mipi_dsi_device *dsi)
+static void ili9881c_dsi_remove(struct mipi_dsi_device *dsi)
 {
 	struct ili9881c *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct ili9881c_desc lhr050h41_desc = {
diff --git a/drivers/gpu/drm/panel/panel-innolux-p079zca.c b/drivers/gpu/drm/panel/panel-innolux-p079zca.c
index f194b62e290c..9992d0d4c0e5 100644
--- a/drivers/gpu/drm/panel/panel-innolux-p079zca.c
+++ b/drivers/gpu/drm/panel/panel-innolux-p079zca.c
@@ -506,7 +506,7 @@ static int innolux_panel_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int innolux_panel_remove(struct mipi_dsi_device *dsi)
+static void innolux_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct innolux_panel *innolux = mipi_dsi_get_drvdata(dsi);
 	int err;
@@ -524,8 +524,6 @@ static int innolux_panel_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", err);
 
 	innolux_panel_del(innolux);
-
-	return 0;
 }
 
 static void innolux_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c b/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c
index 31eafbc38ec0..d8765b2294fb 100644
--- a/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c
+++ b/drivers/gpu/drm/panel/panel-jdi-fhd-r63452.c
@@ -288,7 +288,7 @@ static int jdi_fhd_r63452_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int jdi_fhd_r63452_remove(struct mipi_dsi_device *dsi)
+static void jdi_fhd_r63452_remove(struct mipi_dsi_device *dsi)
 {
 	struct jdi_fhd_r63452 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -298,8 +298,6 @@ static int jdi_fhd_r63452_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id jdi_fhd_r63452_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c b/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
index 3c86ad262d5e..8f4f137a2af6 100644
--- a/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
+++ b/drivers/gpu/drm/panel/panel-jdi-lt070me05000.c
@@ -482,7 +482,7 @@ static int jdi_panel_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int jdi_panel_remove(struct mipi_dsi_device *dsi)
+static void jdi_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct jdi_panel *jdi = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -497,8 +497,6 @@ static int jdi_panel_remove(struct mipi_dsi_device *dsi)
 			ret);
 
 	jdi_panel_del(jdi);
-
-	return 0;
 }
 
 static void jdi_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-khadas-ts050.c b/drivers/gpu/drm/panel/panel-khadas-ts050.c
index a3ec4cbdbf7a..1ab1ebe30882 100644
--- a/drivers/gpu/drm/panel/panel-khadas-ts050.c
+++ b/drivers/gpu/drm/panel/panel-khadas-ts050.c
@@ -830,7 +830,7 @@ static int khadas_ts050_panel_probe(struct mipi_dsi_device *dsi)
 	return err;
 }
 
-static int khadas_ts050_panel_remove(struct mipi_dsi_device *dsi)
+static void khadas_ts050_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct khadas_ts050_panel *khadas_ts050 = mipi_dsi_get_drvdata(dsi);
 	int err;
@@ -842,8 +842,6 @@ static int khadas_ts050_panel_remove(struct mipi_dsi_device *dsi)
 	drm_panel_remove(&khadas_ts050->base);
 	drm_panel_disable(&khadas_ts050->base);
 	drm_panel_unprepare(&khadas_ts050->base);
-
-	return 0;
 }
 
 static void khadas_ts050_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-kingdisplay-kd097d04.c b/drivers/gpu/drm/panel/panel-kingdisplay-kd097d04.c
index daccb1fd5fda..17f8d80cf2b3 100644
--- a/drivers/gpu/drm/panel/panel-kingdisplay-kd097d04.c
+++ b/drivers/gpu/drm/panel/panel-kingdisplay-kd097d04.c
@@ -415,7 +415,7 @@ static int kingdisplay_panel_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int kingdisplay_panel_remove(struct mipi_dsi_device *dsi)
+static void kingdisplay_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct kingdisplay_panel *kingdisplay = mipi_dsi_get_drvdata(dsi);
 	int err;
@@ -433,8 +433,6 @@ static int kingdisplay_panel_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", err);
 
 	kingdisplay_panel_del(kingdisplay);
-
-	return 0;
 }
 
 static void kingdisplay_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c b/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c
index a5a414920430..5619f186d28c 100644
--- a/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c
+++ b/drivers/gpu/drm/panel/panel-leadtek-ltk050h3146w.c
@@ -628,7 +628,7 @@ static void ltk050h3146w_shutdown(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to disable panel: %d\n", ret);
 }
 
-static int ltk050h3146w_remove(struct mipi_dsi_device *dsi)
+static void ltk050h3146w_remove(struct mipi_dsi_device *dsi)
 {
 	struct ltk050h3146w *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -640,8 +640,6 @@ static int ltk050h3146w_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id ltk050h3146w_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-leadtek-ltk500hd1829.c b/drivers/gpu/drm/panel/panel-leadtek-ltk500hd1829.c
index 21e48923836d..39e408c9f762 100644
--- a/drivers/gpu/drm/panel/panel-leadtek-ltk500hd1829.c
+++ b/drivers/gpu/drm/panel/panel-leadtek-ltk500hd1829.c
@@ -477,7 +477,7 @@ static void ltk500hd1829_shutdown(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to disable panel: %d\n", ret);
 }
 
-static int ltk500hd1829_remove(struct mipi_dsi_device *dsi)
+static void ltk500hd1829_remove(struct mipi_dsi_device *dsi)
 {
 	struct ltk500hd1829 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -489,8 +489,6 @@ static int ltk500hd1829_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id ltk500hd1829_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-mantix-mlaf057we51.c b/drivers/gpu/drm/panel/panel-mantix-mlaf057we51.c
index 31daae1da9c9..772e3b6acece 100644
--- a/drivers/gpu/drm/panel/panel-mantix-mlaf057we51.c
+++ b/drivers/gpu/drm/panel/panel-mantix-mlaf057we51.c
@@ -336,7 +336,7 @@ static void mantix_shutdown(struct mipi_dsi_device *dsi)
 	drm_panel_disable(&ctx->panel);
 }
 
-static int mantix_remove(struct mipi_dsi_device *dsi)
+static void mantix_remove(struct mipi_dsi_device *dsi)
 {
 	struct mantix *ctx = mipi_dsi_get_drvdata(dsi);
 
@@ -344,8 +344,6 @@ static int mantix_remove(struct mipi_dsi_device *dsi)
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id mantix_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt35510.c b/drivers/gpu/drm/panel/panel-novatek-nt35510.c
index 40ea41b0a5dd..493c3c23f0d6 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt35510.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt35510.c
@@ -231,7 +231,7 @@ struct nt35510_config {
 	 * bits 0..2 in the lower nibble controls HCK, the booster clock
 	 * frequency, the values are the same as for PCK in @bt1ctr.
 	 * bits 4..5 in the upper nibble controls BTH, the boosting
-	 * amplification for the the step-up circuit.
+	 * amplification for the step-up circuit.
 	 * 0 = AVDD + VDDB
 	 * 1 = AVDD - AVEE
 	 * 2 = AVDD - AVEE + VDDB
@@ -966,7 +966,7 @@ static int nt35510_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int nt35510_remove(struct mipi_dsi_device *dsi)
+static void nt35510_remove(struct mipi_dsi_device *dsi)
 {
 	struct nt35510 *nt = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -974,9 +974,10 @@ static int nt35510_remove(struct mipi_dsi_device *dsi)
 	mipi_dsi_detach(dsi);
 	/* Power off */
 	ret = nt35510_power_off(nt);
-	drm_panel_remove(&nt->panel);
+	if (ret)
+		dev_err(&dsi->dev, "Failed to power off\n");
 
-	return ret;
+	drm_panel_remove(&nt->panel);
 }
 
 /*
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt35560.c b/drivers/gpu/drm/panel/panel-novatek-nt35560.c
index 1b6042321ea1..cc7f96d70826 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt35560.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt35560.c
@@ -523,14 +523,12 @@ static int nt35560_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int nt35560_remove(struct mipi_dsi_device *dsi)
+static void nt35560_remove(struct mipi_dsi_device *dsi)
 {
 	struct nt35560 *nt = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&nt->panel);
-
-	return 0;
 }
 
 static const struct of_device_id nt35560_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt35950.c b/drivers/gpu/drm/panel/panel-novatek-nt35950.c
index 288c7fa83ecc..3a844917da07 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt35950.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt35950.c
@@ -620,7 +620,7 @@ static int nt35950_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int nt35950_remove(struct mipi_dsi_device *dsi)
+static void nt35950_remove(struct mipi_dsi_device *dsi)
 {
 	struct nt35950 *nt = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -639,8 +639,6 @@ static int nt35950_remove(struct mipi_dsi_device *dsi)
 	}
 
 	drm_panel_remove(&nt->panel);
-
-	return 0;
 }
 
 static const struct nt35950_panel_mode sharp_ls055d1sx04_modes[] = {
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt36672a.c b/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
index 6d6ce42787e2..73bcffa1e0c1 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt36672a.c
@@ -669,7 +669,7 @@ static int nt36672a_panel_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int nt36672a_panel_remove(struct mipi_dsi_device *dsi)
+static void nt36672a_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct nt36672a_panel *pinfo = mipi_dsi_get_drvdata(dsi);
 	int err;
@@ -687,8 +687,6 @@ static int nt36672a_panel_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", err);
 
 	drm_panel_remove(&pinfo->base);
-
-	return 0;
 }
 
 static void nt36672a_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c b/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c
index cb5cb27462df..36a46cb7fe1c 100644
--- a/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c
+++ b/drivers/gpu/drm/panel/panel-olimex-lcd-olinuxino.c
@@ -288,7 +288,7 @@ static int lcd_olinuxino_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lcd_olinuxino_remove(struct i2c_client *client)
+static void lcd_olinuxino_remove(struct i2c_client *client)
 {
 	struct lcd_olinuxino *panel = i2c_get_clientdata(client);
 
@@ -296,8 +296,6 @@ static int lcd_olinuxino_remove(struct i2c_client *client)
 
 	drm_panel_disable(&panel->panel);
 	drm_panel_unprepare(&panel->panel);
-
-	return 0;
 }
 
 static const struct of_device_id lcd_olinuxino_of_ids[] = {
diff --git a/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c b/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
index dfb43b1374e7..b4729a94c34a 100644
--- a/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
+++ b/drivers/gpu/drm/panel/panel-orisetech-otm8009a.c
@@ -497,14 +497,12 @@ static int otm8009a_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int otm8009a_remove(struct mipi_dsi_device *dsi)
+static void otm8009a_remove(struct mipi_dsi_device *dsi)
 {
 	struct otm8009a *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id orisetech_otm8009a_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-osd-osd101t2587-53ts.c b/drivers/gpu/drm/panel/panel-osd-osd101t2587-53ts.c
index 198493a6eb6a..493e0504f6f7 100644
--- a/drivers/gpu/drm/panel/panel-osd-osd101t2587-53ts.c
+++ b/drivers/gpu/drm/panel/panel-osd-osd101t2587-53ts.c
@@ -206,7 +206,7 @@ static int osd101t2587_panel_probe(struct mipi_dsi_device *dsi)
 	return ret;
 }
 
-static int osd101t2587_panel_remove(struct mipi_dsi_device *dsi)
+static void osd101t2587_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct osd101t2587_panel *osd101t2587 = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -221,8 +221,6 @@ static int osd101t2587_panel_remove(struct mipi_dsi_device *dsi)
 	ret = mipi_dsi_detach(dsi);
 	if (ret < 0)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
-
-	return ret;
 }
 
 static void osd101t2587_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c b/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
index 3991f5d950af..8ba6d8287938 100644
--- a/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
+++ b/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
@@ -250,7 +250,7 @@ static int wuxga_nt_panel_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int wuxga_nt_panel_remove(struct mipi_dsi_device *dsi)
+static void wuxga_nt_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct wuxga_nt_panel *wuxga_nt = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -264,8 +264,6 @@ static int wuxga_nt_panel_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
 
 	wuxga_nt_panel_del(wuxga_nt);
-
-	return 0;
 }
 
 static void wuxga_nt_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c b/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
index a6dc5ab182fa..79f852465a84 100644
--- a/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
+++ b/drivers/gpu/drm/panel/panel-raspberrypi-touchscreen.c
@@ -446,7 +446,7 @@ error:
 	return -ENODEV;
 }
 
-static int rpi_touchscreen_remove(struct i2c_client *i2c)
+static void rpi_touchscreen_remove(struct i2c_client *i2c)
 {
 	struct rpi_touchscreen *ts = i2c_get_clientdata(i2c);
 
@@ -455,8 +455,6 @@ static int rpi_touchscreen_remove(struct i2c_client *i2c)
 	drm_panel_remove(&ts->base);
 
 	mipi_dsi_device_unregister(ts->dsi);
-
-	return 0;
 }
 
 static int rpi_touchscreen_dsi_probe(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-raydium-rm67191.c b/drivers/gpu/drm/panel/panel-raydium-rm67191.c
index 4e021a572211..dbb1ed4efbed 100644
--- a/drivers/gpu/drm/panel/panel-raydium-rm67191.c
+++ b/drivers/gpu/drm/panel/panel-raydium-rm67191.c
@@ -616,7 +616,7 @@ static int rad_panel_probe(struct mipi_dsi_device *dsi)
 	return ret;
 }
 
-static int rad_panel_remove(struct mipi_dsi_device *dsi)
+static void rad_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct rad_panel *rad = mipi_dsi_get_drvdata(dsi);
 	struct device *dev = &dsi->dev;
@@ -627,8 +627,6 @@ static int rad_panel_remove(struct mipi_dsi_device *dsi)
 		dev_err(dev, "Failed to detach from host (%d)\n", ret);
 
 	drm_panel_remove(&rad->panel);
-
-	return 0;
 }
 
 static void rad_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-raydium-rm68200.c b/drivers/gpu/drm/panel/panel-raydium-rm68200.c
index 412c0dbcb2b6..5f9b340588fb 100644
--- a/drivers/gpu/drm/panel/panel-raydium-rm68200.c
+++ b/drivers/gpu/drm/panel/panel-raydium-rm68200.c
@@ -412,14 +412,12 @@ static int rm68200_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int rm68200_remove(struct mipi_dsi_device *dsi)
+static void rm68200_remove(struct mipi_dsi_device *dsi)
 {
 	struct rm68200 *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id raydium_rm68200_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c b/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c
index 1fb579a574d9..a8a98c91b13c 100644
--- a/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c
+++ b/drivers/gpu/drm/panel/panel-ronbo-rb070d30.c
@@ -208,14 +208,12 @@ static int rb070d30_panel_dsi_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int rb070d30_panel_dsi_remove(struct mipi_dsi_device *dsi)
+static void rb070d30_panel_dsi_remove(struct mipi_dsi_device *dsi)
 {
 	struct rb070d30_panel *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id rb070d30_panel_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
index 70560cac53a9..008e2b0d6652 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6d16d0.c
@@ -212,14 +212,12 @@ static int s6d16d0_probe(struct mipi_dsi_device *dsi)
 	return ret;
 }
 
-static int s6d16d0_remove(struct mipi_dsi_device *dsi)
+static void s6d16d0_remove(struct mipi_dsi_device *dsi)
 {
 	struct s6d16d0 *s6 = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&s6->panel);
-
-	return 0;
 }
 
 static const struct of_device_id s6d16d0_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e3ha2.c b/drivers/gpu/drm/panel/panel-samsung-s6e3ha2.c
index 0ab1b7ec84cd..5c621b15e84c 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e3ha2.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e3ha2.c
@@ -747,15 +747,13 @@ remove_panel:
 	return ret;
 }
 
-static int s6e3ha2_remove(struct mipi_dsi_device *dsi)
+static void s6e3ha2_remove(struct mipi_dsi_device *dsi)
 {
 	struct s6e3ha2 *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&ctx->panel);
 	backlight_device_unregister(ctx->bl_dev);
-
-	return 0;
 }
 
 static const struct of_device_id s6e3ha2_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e63j0x03.c b/drivers/gpu/drm/panel/panel-samsung-s6e63j0x03.c
index e38262b67ff7..e06fd35de814 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e63j0x03.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e63j0x03.c
@@ -488,7 +488,7 @@ remove_panel:
 	return ret;
 }
 
-static int s6e63j0x03_remove(struct mipi_dsi_device *dsi)
+static void s6e63j0x03_remove(struct mipi_dsi_device *dsi)
 {
 	struct s6e63j0x03 *ctx = mipi_dsi_get_drvdata(dsi);
 
@@ -496,8 +496,6 @@ static int s6e63j0x03_remove(struct mipi_dsi_device *dsi)
 	drm_panel_remove(&ctx->panel);
 
 	backlight_device_unregister(ctx->bl_dev);
-
-	return 0;
 }
 
 static const struct of_device_id s6e63j0x03_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c b/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c
index e0f773678168..ed3895e4ca5e 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e63m0-dsi.c
@@ -113,11 +113,10 @@ static int s6e63m0_dsi_probe(struct mipi_dsi_device *dsi)
 	return ret;
 }
 
-static int s6e63m0_dsi_remove(struct mipi_dsi_device *dsi)
+static void s6e63m0_dsi_remove(struct mipi_dsi_device *dsi)
 {
 	mipi_dsi_detach(dsi);
 	s6e63m0_remove(&dsi->dev);
-	return 0;
 }
 
 static const struct of_device_id s6e63m0_dsi_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c b/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c
index 29fde3823212..97ff7a18545c 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e88a0-ams452ef01.c
@@ -254,7 +254,7 @@ static int s6e88a0_ams452ef01_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int s6e88a0_ams452ef01_remove(struct mipi_dsi_device *dsi)
+static void s6e88a0_ams452ef01_remove(struct mipi_dsi_device *dsi)
 {
 	struct s6e88a0_ams452ef01 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -264,8 +264,6 @@ static int s6e88a0_ams452ef01_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id s6e88a0_ams452ef01_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e8aa0.c b/drivers/gpu/drm/panel/panel-samsung-s6e8aa0.c
index 9b3599d6d2de..54213beafaf5 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e8aa0.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e8aa0.c
@@ -1028,14 +1028,12 @@ static int s6e8aa0_probe(struct mipi_dsi_device *dsi)
 	return ret;
 }
 
-static int s6e8aa0_remove(struct mipi_dsi_device *dsi)
+static void s6e8aa0_remove(struct mipi_dsi_device *dsi)
 {
 	struct s6e8aa0 *ctx = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id s6e8aa0_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-samsung-sofef00.c b/drivers/gpu/drm/panel/panel-samsung-sofef00.c
index 1fb37fda4ba9..1a0d24595faa 100644
--- a/drivers/gpu/drm/panel/panel-samsung-sofef00.c
+++ b/drivers/gpu/drm/panel/panel-samsung-sofef00.c
@@ -305,7 +305,7 @@ static int sofef00_panel_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int sofef00_panel_remove(struct mipi_dsi_device *dsi)
+static void sofef00_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct sofef00_panel *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -315,8 +315,6 @@ static int sofef00_panel_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id sofef00_panel_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-sharp-lq101r1sx01.c b/drivers/gpu/drm/panel/panel-sharp-lq101r1sx01.c
index f8cd2a42ed13..14851408a5e1 100644
--- a/drivers/gpu/drm/panel/panel-sharp-lq101r1sx01.c
+++ b/drivers/gpu/drm/panel/panel-sharp-lq101r1sx01.c
@@ -391,7 +391,7 @@ static int sharp_panel_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int sharp_panel_remove(struct mipi_dsi_device *dsi)
+static void sharp_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct sharp_panel *sharp = mipi_dsi_get_drvdata(dsi);
 	int err;
@@ -399,7 +399,7 @@ static int sharp_panel_remove(struct mipi_dsi_device *dsi)
 	/* only detach from host for the DSI-LINK2 interface */
 	if (!sharp) {
 		mipi_dsi_detach(dsi);
-		return 0;
+		return;
 	}
 
 	err = drm_panel_disable(&sharp->base);
@@ -411,8 +411,6 @@ static int sharp_panel_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", err);
 
 	sharp_panel_del(sharp);
-
-	return 0;
 }
 
 static void sharp_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
index 25829a0a8e80..d1ec80a3e3c7 100644
--- a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
+++ b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
@@ -305,7 +305,7 @@ static int sharp_nt_panel_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int sharp_nt_panel_remove(struct mipi_dsi_device *dsi)
+static void sharp_nt_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct sharp_nt_panel *sharp_nt = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -319,8 +319,6 @@ static int sharp_nt_panel_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", ret);
 
 	sharp_nt_panel_del(sharp_nt);
-
-	return 0;
 }
 
 static void sharp_nt_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-sharp-ls060t1sx01.c b/drivers/gpu/drm/panel/panel-sharp-ls060t1sx01.c
index e12570561629..8a4e0c1fe73f 100644
--- a/drivers/gpu/drm/panel/panel-sharp-ls060t1sx01.c
+++ b/drivers/gpu/drm/panel/panel-sharp-ls060t1sx01.c
@@ -298,7 +298,7 @@ static int sharp_ls060_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int sharp_ls060_remove(struct mipi_dsi_device *dsi)
+static void sharp_ls060_remove(struct mipi_dsi_device *dsi)
 {
 	struct sharp_ls060 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -308,8 +308,6 @@ static int sharp_ls060_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id sharp_ls060t1sx01_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 1e716c23019a..2944228a8e2c 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -575,12 +575,9 @@ static int panel_simple_probe(struct device *dev, const struct panel_desc *desc)
 
 	panel->enable_gpio = devm_gpiod_get_optional(dev, "enable",
 						     GPIOD_OUT_LOW);
-	if (IS_ERR(panel->enable_gpio)) {
-		err = PTR_ERR(panel->enable_gpio);
-		if (err != -EPROBE_DEFER)
-			dev_err(dev, "failed to request GPIO: %d\n", err);
-		return err;
-	}
+	if (IS_ERR(panel->enable_gpio))
+		return dev_err_probe(dev, PTR_ERR(panel->enable_gpio),
+				     "failed to request GPIO\n");
 
 	err = of_drm_get_panel_orientation(dev->of_node, &panel->orientation);
 	if (err) {
@@ -696,7 +693,7 @@ free_ddc:
 	return err;
 }
 
-static int panel_simple_remove(struct device *dev)
+static void panel_simple_remove(struct device *dev)
 {
 	struct panel_simple *panel = dev_get_drvdata(dev);
 
@@ -708,8 +705,6 @@ static int panel_simple_remove(struct device *dev)
 	pm_runtime_disable(dev);
 	if (panel->ddc)
 		put_device(&panel->ddc->dev);
-
-	return 0;
 }
 
 static void panel_simple_shutdown(struct device *dev)
@@ -2703,6 +2698,36 @@ static const struct panel_desc multi_inno_mi0700s4t_6 = {
 	.connector_type = DRM_MODE_CONNECTOR_DPI,
 };
 
+static const struct display_timing multi_inno_mi0800ft_9_timing = {
+	.pixelclock = { 32000000, 40000000, 50000000 },
+	.hactive = { 800, 800, 800 },
+	.hfront_porch = { 16, 210, 354 },
+	.hback_porch = { 6, 26, 45 },
+	.hsync_len = { 1, 20, 40 },
+	.vactive = { 600, 600, 600 },
+	.vfront_porch = { 1, 12, 77 },
+	.vback_porch = { 3, 13, 22 },
+	.vsync_len = { 1, 10, 20 },
+	.flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
+		 DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_PIXDATA_POSEDGE |
+		 DISPLAY_FLAGS_SYNC_POSEDGE,
+};
+
+static const struct panel_desc multi_inno_mi0800ft_9 = {
+	.timings = &multi_inno_mi0800ft_9_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 162,
+		.height = 122,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH |
+		     DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE |
+		     DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+};
+
 static const struct display_timing multi_inno_mi1010ait_1cp_timing = {
 	.pixelclock = { 68900000, 70000000, 73400000 },
 	.hactive = { 1280, 1280, 1280 },
@@ -3220,6 +3245,37 @@ static const struct panel_desc rocktech_rk101ii01d_ct = {
 	.connector_type = DRM_MODE_CONNECTOR_LVDS,
 };
 
+static const struct display_timing samsung_ltl101al01_timing = {
+	.pixelclock = { 66663000, 66663000, 66663000 },
+	.hactive = { 1280, 1280, 1280 },
+	.hfront_porch = { 18, 18, 18 },
+	.hback_porch = { 36, 36, 36 },
+	.hsync_len = { 16, 16, 16 },
+	.vactive = { 800, 800, 800 },
+	.vfront_porch = { 4, 4, 4 },
+	.vback_porch = { 16, 16, 16 },
+	.vsync_len = { 3, 3, 3 },
+	.flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW,
+};
+
+static const struct panel_desc samsung_ltl101al01 = {
+	.timings = &samsung_ltl101al01_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 217,
+		.height = 135,
+	},
+	.delay = {
+		.prepare = 40,
+		.enable = 300,
+		.disable = 200,
+		.unprepare = 600,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
 static const struct drm_display_mode samsung_ltn101nt05_mode = {
 	.clock = 54030,
 	.hdisplay = 1024,
@@ -4104,6 +4160,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "multi-inno,mi0700s4t-6",
 		.data = &multi_inno_mi0700s4t_6,
 	}, {
+		.compatible = "multi-inno,mi0800ft-9",
+		.data = &multi_inno_mi0800ft_9,
+	}, {
 		.compatible = "multi-inno,mi1010ait-1cp",
 		.data = &multi_inno_mi1010ait_1cp,
 	}, {
@@ -4164,6 +4223,9 @@ static const struct of_device_id platform_of_match[] = {
 		.compatible = "rocktech,rk101ii01d-ct",
 		.data = &rocktech_rk101ii01d_ct,
 	}, {
+		.compatible = "samsung,ltl101al01",
+		.data = &samsung_ltl101al01,
+	}, {
 		.compatible = "samsung,ltn101nt05",
 		.data = &samsung_ltn101nt05,
 	}, {
@@ -4273,7 +4335,9 @@ static int panel_simple_platform_probe(struct platform_device *pdev)
 
 static int panel_simple_platform_remove(struct platform_device *pdev)
 {
-	return panel_simple_remove(&pdev->dev);
+	panel_simple_remove(&pdev->dev);
+
+	return 0;
 }
 
 static void panel_simple_platform_shutdown(struct platform_device *pdev)
@@ -4566,7 +4630,7 @@ static int panel_simple_dsi_probe(struct mipi_dsi_device *dsi)
 	return err;
 }
 
-static int panel_simple_dsi_remove(struct mipi_dsi_device *dsi)
+static void panel_simple_dsi_remove(struct mipi_dsi_device *dsi)
 {
 	int err;
 
@@ -4574,7 +4638,7 @@ static int panel_simple_dsi_remove(struct mipi_dsi_device *dsi)
 	if (err < 0)
 		dev_err(&dsi->dev, "failed to detach from DSI host: %d\n", err);
 
-	return panel_simple_remove(&dsi->dev);
+	panel_simple_remove(&dsi->dev);
 }
 
 static void panel_simple_dsi_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-sitronix-st7701.c b/drivers/gpu/drm/panel/panel-sitronix-st7701.c
index 320a2a8fd459..c481daa4bbce 100644
--- a/drivers/gpu/drm/panel/panel-sitronix-st7701.c
+++ b/drivers/gpu/drm/panel/panel-sitronix-st7701.c
@@ -8,6 +8,7 @@
 #include <drm/drm_modes.h>
 #include <drm/drm_panel.h>
 
+#include <linux/bitfield.h>
 #include <linux/gpio/consumer.h>
 #include <linux/delay.h>
 #include <linux/module.h>
@@ -41,59 +42,101 @@
 /*
  * Command2 with BK function selection.
  *
- * BIT[4, 0]: [CN2, BKXSEL]
- * 10 = CMD2BK0, Command2 BK0
- * 11 = CMD2BK1, Command2 BK1
- * 00 = Command2 disable
+ * BIT[4].....CN2
+ * BIT[1:0]...BKXSEL
+ * 1:00 = CMD2BK0, Command2 BK0
+ * 1:01 = CMD2BK1, Command2 BK1
+ * 1:11 = CMD2BK3, Command2 BK3
+ * 0:00 = Command2 disable
  */
-#define DSI_CMD2BK1_SEL			0x11
 #define DSI_CMD2BK0_SEL			0x10
+#define DSI_CMD2BK1_SEL			0x11
+#define DSI_CMD2BK3_SEL			0x13
 #define DSI_CMD2BKX_SEL_NONE		0x00
 
 /* Command2, BK0 bytes */
-#define DSI_LINESET_LINE		0x69
-#define DSI_LINESET_LDE_EN		BIT(7)
-#define DSI_LINESET_LINEDELTA		GENMASK(1, 0)
-#define DSI_CMD2_BK0_LNESET_B1		DSI_LINESET_LINEDELTA
-#define DSI_CMD2_BK0_LNESET_B0		(DSI_LINESET_LDE_EN | DSI_LINESET_LINE)
-#define DSI_INVSEL_DEFAULT		GENMASK(5, 4)
-#define DSI_INVSEL_NLINV		GENMASK(2, 0)
-#define DSI_INVSEL_RTNI			GENMASK(2, 1)
-#define DSI_CMD2_BK0_INVSEL_B1		DSI_INVSEL_RTNI
-#define DSI_CMD2_BK0_INVSEL_B0		(DSI_INVSEL_DEFAULT | DSI_INVSEL_NLINV)
-#define DSI_CMD2_BK0_PORCTRL_B0(m)	((m)->vtotal - (m)->vsync_end)
-#define DSI_CMD2_BK0_PORCTRL_B1(m)	((m)->vsync_start - (m)->vdisplay)
+#define DSI_CMD2_BK0_GAMCTRL_AJ_MASK	GENMASK(7, 6)
+#define DSI_CMD2_BK0_GAMCTRL_VC0_MASK	GENMASK(3, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC4_MASK	GENMASK(5, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC8_MASK	GENMASK(5, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC16_MASK	GENMASK(4, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC24_MASK	GENMASK(4, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC52_MASK	GENMASK(3, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC80_MASK	GENMASK(5, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC108_MASK	GENMASK(3, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC147_MASK	GENMASK(3, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC175_MASK	GENMASK(5, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC203_MASK	GENMASK(3, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC231_MASK	GENMASK(4, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC239_MASK	GENMASK(4, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC247_MASK	GENMASK(5, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC251_MASK	GENMASK(5, 0)
+#define DSI_CMD2_BK0_GAMCTRL_VC255_MASK	GENMASK(4, 0)
+#define DSI_CMD2_BK0_LNESET_LINE_MASK	GENMASK(6, 0)
+#define DSI_CMD2_BK0_LNESET_LDE_EN	BIT(7)
+#define DSI_CMD2_BK0_LNESET_LINEDELTA	GENMASK(1, 0)
+#define DSI_CMD2_BK0_PORCTRL_VBP_MASK	GENMASK(7, 0)
+#define DSI_CMD2_BK0_PORCTRL_VFP_MASK	GENMASK(7, 0)
+#define DSI_CMD2_BK0_INVSEL_ONES_MASK	GENMASK(5, 4)
+#define DSI_CMD2_BK0_INVSEL_NLINV_MASK	GENMASK(2, 0)
+#define DSI_CMD2_BK0_INVSEL_RTNI_MASK	GENMASK(4, 0)
 
 /* Command2, BK1 bytes */
-#define DSI_CMD2_BK1_VRHA_SET		0x45
-#define DSI_CMD2_BK1_VCOM_SET		0x13
-#define DSI_CMD2_BK1_VGHSS_SET		GENMASK(2, 0)
+#define DSI_CMD2_BK1_VRHA_MASK		GENMASK(7, 0)
+#define DSI_CMD2_BK1_VCOM_MASK		GENMASK(7, 0)
+#define DSI_CMD2_BK1_VGHSS_MASK		GENMASK(3, 0)
 #define DSI_CMD2_BK1_TESTCMD_VAL	BIT(7)
-#define DSI_VGLS_DEFAULT		BIT(6)
-#define DSI_VGLS_SEL			GENMASK(2, 0)
-#define DSI_CMD2_BK1_VGLS_SET		(DSI_VGLS_DEFAULT | DSI_VGLS_SEL)
-#define DSI_PWCTLR1_AP			BIT(7) /* Gamma OP bias, max */
-#define DSI_PWCTLR1_APIS		BIT(2) /* Source OP input bias, min */
-#define DSI_PWCTLR1_APOS		BIT(0) /* Source OP output bias, min */
-#define DSI_CMD2_BK1_PWCTLR1_SET	(DSI_PWCTLR1_AP | DSI_PWCTLR1_APIS | \
-					DSI_PWCTLR1_APOS)
-#define DSI_PWCTLR2_AVDD		BIT(5) /* AVDD 6.6v */
-#define DSI_PWCTLR2_AVCL		0x0    /* AVCL -4.4v */
-#define DSI_CMD2_BK1_PWCTLR2_SET	(DSI_PWCTLR2_AVDD | DSI_PWCTLR2_AVCL)
-#define DSI_SPD1_T2D			BIT(3)
-#define DSI_CMD2_BK1_SPD1_SET		(GENMASK(6, 4) | DSI_SPD1_T2D)
-#define DSI_CMD2_BK1_SPD2_SET		DSI_CMD2_BK1_SPD1_SET
-#define DSI_MIPISET1_EOT_EN		BIT(3)
-#define DSI_CMD2_BK1_MIPISET1_SET	(BIT(7) | DSI_MIPISET1_EOT_EN)
+#define DSI_CMD2_BK1_VGLS_ONES		BIT(6)
+#define DSI_CMD2_BK1_VGLS_MASK		GENMASK(3, 0)
+#define DSI_CMD2_BK1_PWRCTRL1_AP_MASK	GENMASK(7, 6)
+#define DSI_CMD2_BK1_PWRCTRL1_APIS_MASK	GENMASK(3, 2)
+#define DSI_CMD2_BK1_PWRCTRL1_APOS_MASK	GENMASK(1, 0)
+#define DSI_CMD2_BK1_PWRCTRL2_AVDD_MASK	GENMASK(5, 4)
+#define DSI_CMD2_BK1_PWRCTRL2_AVCL_MASK	GENMASK(1, 0)
+#define DSI_CMD2_BK1_SPD1_ONES_MASK	GENMASK(6, 4)
+#define DSI_CMD2_BK1_SPD1_T2D_MASK	GENMASK(3, 0)
+#define DSI_CMD2_BK1_SPD2_ONES_MASK	GENMASK(6, 4)
+#define DSI_CMD2_BK1_SPD2_T3D_MASK	GENMASK(3, 0)
+#define DSI_CMD2_BK1_MIPISET1_ONES	BIT(7)
+#define DSI_CMD2_BK1_MIPISET1_EOT_EN	BIT(3)
+
+#define CFIELD_PREP(_mask, _val)					\
+	(((typeof(_mask))(_val) << (__builtin_ffsll(_mask) - 1)) & (_mask))
+
+enum op_bias {
+	OP_BIAS_OFF = 0,
+	OP_BIAS_MIN,
+	OP_BIAS_MIDDLE,
+	OP_BIAS_MAX
+};
+
+struct st7701;
 
 struct st7701_panel_desc {
 	const struct drm_display_mode *mode;
 	unsigned int lanes;
-	unsigned long flags;
 	enum mipi_dsi_pixel_format format;
-	const char *const *supply_names;
-	unsigned int num_supplies;
 	unsigned int panel_sleep_delay;
+
+	/* TFT matrix driver configuration, panel specific. */
+	const u8	pv_gamma[16];	/* Positive voltage gamma control */
+	const u8	nv_gamma[16];	/* Negative voltage gamma control */
+	const u8	nlinv;		/* Inversion selection */
+	const u32	vop_uv;		/* Vop in uV */
+	const u32	vcom_uv;	/* Vcom in uV */
+	const u16	vgh_mv;		/* Vgh in mV */
+	const s16	vgl_mv;		/* Vgl in mV */
+	const u16	avdd_mv;	/* Avdd in mV */
+	const s16	avcl_mv;	/* Avcl in mV */
+	const enum op_bias	gamma_op_bias;
+	const enum op_bias	input_op_bias;
+	const enum op_bias	output_op_bias;
+	const u16	t2d_ns;		/* T2D in ns */
+	const u16	t3d_ns;		/* T3D in ns */
+	const bool	eot_en;
+
+	/* GIP sequence, fully custom and undocumented. */
+	void		(*gip_sequence)(struct st7701 *st7701);
 };
 
 struct st7701 {
@@ -101,7 +144,7 @@ struct st7701 {
 	struct mipi_dsi_device *dsi;
 	const struct st7701_panel_desc *desc;
 
-	struct regulator_bulk_data *supplies;
+	struct regulator_bulk_data supplies[2];
 	struct gpio_desc *reset;
 	unsigned int sleep_delay;
 };
@@ -123,9 +166,37 @@ static inline int st7701_dsi_write(struct st7701 *st7701, const void *seq,
 		st7701_dsi_write(st7701, d, ARRAY_SIZE(d));	\
 	}
 
+static u8 st7701_vgls_map(struct st7701 *st7701)
+{
+	const struct st7701_panel_desc *desc = st7701->desc;
+	struct {
+		s32	vgl;
+		u8	val;
+	} map[16] = {
+		{ -7060, 0x0 }, { -7470, 0x1 },
+		{ -7910, 0x2 }, { -8140, 0x3 },
+		{ -8650, 0x4 }, { -8920, 0x5 },
+		{ -9210, 0x6 }, { -9510, 0x7 },
+		{ -9830, 0x8 }, { -10170, 0x9 },
+		{ -10530, 0xa }, { -10910, 0xb },
+		{ -11310, 0xc }, { -11730, 0xd },
+		{ -12200, 0xe }, { -12690, 0xf }
+	};
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(map); i++)
+		if (desc->vgl_mv == map[i].vgl)
+			return map[i].val;
+
+	return 0;
+}
+
 static void st7701_init_sequence(struct st7701 *st7701)
 {
-	const struct drm_display_mode *mode = st7701->desc->mode;
+	const struct st7701_panel_desc *desc = st7701->desc;
+	const struct drm_display_mode *mode = desc->mode;
+	const u8 linecount8 = mode->vdisplay / 8;
+	const u8 linecountrem2 = (mode->vdisplay % 8) / 2;
 
 	ST7701_DSI(st7701, MIPI_DCS_SOFT_RESET, 0x00);
 
@@ -139,34 +210,105 @@ static void st7701_init_sequence(struct st7701 *st7701)
 	/* Command2, BK0 */
 	ST7701_DSI(st7701, DSI_CMD2BKX_SEL,
 		   0x77, 0x01, 0x00, 0x00, DSI_CMD2BK0_SEL);
-	ST7701_DSI(st7701, DSI_CMD2_BK0_PVGAMCTRL, 0x00, 0x0E, 0x15, 0x0F,
-		   0x11, 0x08, 0x08, 0x08, 0x08, 0x23, 0x04, 0x13, 0x12,
-		   0x2B, 0x34, 0x1F);
-	ST7701_DSI(st7701, DSI_CMD2_BK0_NVGAMCTRL, 0x00, 0x0E, 0x95, 0x0F,
-		   0x13, 0x07, 0x09, 0x08, 0x08, 0x22, 0x04, 0x10, 0x0E,
-		   0x2C, 0x34, 0x1F);
+	mipi_dsi_dcs_write(st7701->dsi, DSI_CMD2_BK0_PVGAMCTRL,
+			   desc->pv_gamma, ARRAY_SIZE(desc->pv_gamma));
+	mipi_dsi_dcs_write(st7701->dsi, DSI_CMD2_BK0_NVGAMCTRL,
+			   desc->nv_gamma, ARRAY_SIZE(desc->nv_gamma));
+	/*
+	 * Vertical line count configuration:
+	 * Line[6:0]: select number of vertical lines of the TFT matrix in
+	 *            multiples of 8 lines
+	 * LDE_EN: enable sub-8-line granularity line count
+	 * Line_delta[1:0]: add 0/2/4/6 extra lines to line count selected
+	 *                  using Line[6:0]
+	 *
+	 * Total number of vertical lines:
+	 * LN = ((Line[6:0] + 1) * 8) + (LDE_EN ? Line_delta[1:0] * 2 : 0)
+	 */
 	ST7701_DSI(st7701, DSI_CMD2_BK0_LNESET,
-		   DSI_CMD2_BK0_LNESET_B0, DSI_CMD2_BK0_LNESET_B1);
+		   FIELD_PREP(DSI_CMD2_BK0_LNESET_LINE_MASK, linecount8 - 1) |
+		   (linecountrem2 ? DSI_CMD2_BK0_LNESET_LDE_EN : 0),
+		   FIELD_PREP(DSI_CMD2_BK0_LNESET_LINEDELTA, linecountrem2));
 	ST7701_DSI(st7701, DSI_CMD2_BK0_PORCTRL,
-		   DSI_CMD2_BK0_PORCTRL_B0(mode),
-		   DSI_CMD2_BK0_PORCTRL_B1(mode));
+		   FIELD_PREP(DSI_CMD2_BK0_PORCTRL_VBP_MASK,
+			      mode->vtotal - mode->vsync_end),
+		   FIELD_PREP(DSI_CMD2_BK0_PORCTRL_VFP_MASK,
+			      mode->vsync_start - mode->vdisplay));
+	/*
+	 * Horizontal pixel count configuration:
+	 * PCLK = 512 + (RTNI[4:0] * 16)
+	 * The PCLK is number of pixel clock per line, which matches
+	 * mode htotal. The minimum is 512 PCLK.
+	 */
 	ST7701_DSI(st7701, DSI_CMD2_BK0_INVSEL,
-		   DSI_CMD2_BK0_INVSEL_B0, DSI_CMD2_BK0_INVSEL_B1);
+		   DSI_CMD2_BK0_INVSEL_ONES_MASK |
+		   FIELD_PREP(DSI_CMD2_BK0_INVSEL_NLINV_MASK, desc->nlinv),
+		   FIELD_PREP(DSI_CMD2_BK0_INVSEL_RTNI_MASK,
+			      DIV_ROUND_UP(mode->htotal, 16)));
 
 	/* Command2, BK1 */
 	ST7701_DSI(st7701, DSI_CMD2BKX_SEL,
 			0x77, 0x01, 0x00, 0x00, DSI_CMD2BK1_SEL);
-	ST7701_DSI(st7701, DSI_CMD2_BK1_VRHS, DSI_CMD2_BK1_VRHA_SET);
-	ST7701_DSI(st7701, DSI_CMD2_BK1_VCOM, DSI_CMD2_BK1_VCOM_SET);
-	ST7701_DSI(st7701, DSI_CMD2_BK1_VGHSS, DSI_CMD2_BK1_VGHSS_SET);
+
+	/* Vop = 3.5375V + (VRHA[7:0] * 0.0125V) */
+	ST7701_DSI(st7701, DSI_CMD2_BK1_VRHS,
+		   FIELD_PREP(DSI_CMD2_BK1_VRHA_MASK,
+			      DIV_ROUND_CLOSEST(desc->vop_uv - 3537500, 12500)));
+
+	/* Vcom = 0.1V + (VCOM[7:0] * 0.0125V) */
+	ST7701_DSI(st7701, DSI_CMD2_BK1_VCOM,
+		   FIELD_PREP(DSI_CMD2_BK1_VCOM_MASK,
+			      DIV_ROUND_CLOSEST(desc->vcom_uv - 100000, 12500)));
+
+	/* Vgh = 11.5V + (VGHSS[7:0] * 0.5V) */
+	ST7701_DSI(st7701, DSI_CMD2_BK1_VGHSS,
+		   FIELD_PREP(DSI_CMD2_BK1_VGHSS_MASK,
+			      DIV_ROUND_CLOSEST(clamp(desc->vgh_mv,
+						      (u16)11500,
+						      (u16)17000) - 11500,
+						500)));
+
 	ST7701_DSI(st7701, DSI_CMD2_BK1_TESTCMD, DSI_CMD2_BK1_TESTCMD_VAL);
-	ST7701_DSI(st7701, DSI_CMD2_BK1_VGLS, DSI_CMD2_BK1_VGLS_SET);
-	ST7701_DSI(st7701, DSI_CMD2_BK1_PWCTLR1, DSI_CMD2_BK1_PWCTLR1_SET);
-	ST7701_DSI(st7701, DSI_CMD2_BK1_PWCTLR2, DSI_CMD2_BK1_PWCTLR2_SET);
-	ST7701_DSI(st7701, DSI_CMD2_BK1_SPD1, DSI_CMD2_BK1_SPD1_SET);
-	ST7701_DSI(st7701, DSI_CMD2_BK1_SPD2, DSI_CMD2_BK1_SPD2_SET);
-	ST7701_DSI(st7701, DSI_CMD2_BK1_MIPISET1, DSI_CMD2_BK1_MIPISET1_SET);
 
+	/* Vgl is non-linear */
+	ST7701_DSI(st7701, DSI_CMD2_BK1_VGLS,
+		   DSI_CMD2_BK1_VGLS_ONES |
+		   FIELD_PREP(DSI_CMD2_BK1_VGLS_MASK, st7701_vgls_map(st7701)));
+
+	ST7701_DSI(st7701, DSI_CMD2_BK1_PWCTLR1,
+		   FIELD_PREP(DSI_CMD2_BK1_PWRCTRL1_AP_MASK,
+			      desc->gamma_op_bias) |
+		   FIELD_PREP(DSI_CMD2_BK1_PWRCTRL1_APIS_MASK,
+			      desc->input_op_bias) |
+		   FIELD_PREP(DSI_CMD2_BK1_PWRCTRL1_APOS_MASK,
+			      desc->output_op_bias));
+
+	/* Avdd = 6.2V + (AVDD[1:0] * 0.2V) , Avcl = -4.4V - (AVCL[1:0] * 0.2V) */
+	ST7701_DSI(st7701, DSI_CMD2_BK1_PWCTLR2,
+		   FIELD_PREP(DSI_CMD2_BK1_PWRCTRL2_AVDD_MASK,
+			      DIV_ROUND_CLOSEST(desc->avdd_mv - 6200, 200)) |
+		   FIELD_PREP(DSI_CMD2_BK1_PWRCTRL2_AVCL_MASK,
+			      DIV_ROUND_CLOSEST(-4400 + desc->avcl_mv, 200)));
+
+	/* T2D = 0.2us * T2D[3:0] */
+	ST7701_DSI(st7701, DSI_CMD2_BK1_SPD1,
+		   DSI_CMD2_BK1_SPD1_ONES_MASK |
+		   FIELD_PREP(DSI_CMD2_BK1_SPD1_T2D_MASK,
+			      DIV_ROUND_CLOSEST(desc->t2d_ns, 200)));
+
+	/* T3D = 4us + (0.8us * T3D[3:0]) */
+	ST7701_DSI(st7701, DSI_CMD2_BK1_SPD2,
+		   DSI_CMD2_BK1_SPD2_ONES_MASK |
+		   FIELD_PREP(DSI_CMD2_BK1_SPD2_T3D_MASK,
+			      DIV_ROUND_CLOSEST(desc->t3d_ns - 4000, 800)));
+
+	ST7701_DSI(st7701, DSI_CMD2_BK1_MIPISET1,
+		   DSI_CMD2_BK1_MIPISET1_ONES |
+		   (desc->eot_en ? DSI_CMD2_BK1_MIPISET1_EOT_EN : 0));
+}
+
+static void ts8550b_gip_sequence(struct st7701 *st7701)
+{
 	/**
 	 * ST7701_SPEC_V1.2 is unable to provide enough information above this
 	 * specific command sequence, so grab the same from vendor BSP driver.
@@ -188,10 +330,78 @@ static void st7701_init_sequence(struct st7701 *st7701)
 	ST7701_DSI(st7701, 0xEC, 0x00, 0x00);
 	ST7701_DSI(st7701, 0xED, 0xFF, 0xF1, 0x04, 0x56, 0x72, 0x3F, 0xFF,
 		   0xFF, 0xFF, 0xFF, 0xF3, 0x27, 0x65, 0x40, 0x1F, 0xFF);
+}
+
+static void dmt028vghmcmi_1a_gip_sequence(struct st7701 *st7701)
+{
+	ST7701_DSI(st7701, 0xEE, 0x42);
+	ST7701_DSI(st7701, 0xE0, 0x00, 0x00, 0x02);
+
+	ST7701_DSI(st7701, 0xE1,
+		   0x04, 0xA0, 0x06, 0xA0,
+			   0x05, 0xA0, 0x07, 0xA0,
+			   0x00, 0x44, 0x44);
+	ST7701_DSI(st7701, 0xE2,
+		   0x00, 0x00, 0x00, 0x00,
+			   0x00, 0x00, 0x00, 0x00,
+			   0x00, 0x00, 0x00, 0x00);
+	ST7701_DSI(st7701, 0xE3,
+		   0x00, 0x00, 0x22, 0x22);
+	ST7701_DSI(st7701, 0xE4, 0x44, 0x44);
+	ST7701_DSI(st7701, 0xE5,
+		   0x0C, 0x90, 0xA0, 0xA0,
+			   0x0E, 0x92, 0xA0, 0xA0,
+			   0x08, 0x8C, 0xA0, 0xA0,
+			   0x0A, 0x8E, 0xA0, 0xA0);
+	ST7701_DSI(st7701, 0xE6,
+		   0x00, 0x00, 0x22, 0x22);
+	ST7701_DSI(st7701, 0xE7, 0x44, 0x44);
+	ST7701_DSI(st7701, 0xE8,
+		   0x0D, 0x91, 0xA0, 0xA0,
+			   0x0F, 0x93, 0xA0, 0xA0,
+			   0x09, 0x8D, 0xA0, 0xA0,
+			   0x0B, 0x8F, 0xA0, 0xA0);
+	ST7701_DSI(st7701, 0xEB,
+		   0x00, 0x00, 0xE4, 0xE4,
+			   0x44, 0x00, 0x00);
+	ST7701_DSI(st7701, 0xED,
+		   0xFF, 0xF5, 0x47, 0x6F,
+			   0x0B, 0xA1, 0xAB, 0xFF,
+			   0xFF, 0xBA, 0x1A, 0xB0,
+			   0xF6, 0x74, 0x5F, 0xFF);
+	ST7701_DSI(st7701, 0xEF,
+		   0x08, 0x08, 0x08, 0x40,
+			   0x3F, 0x64);
+
+	ST7701_DSI(st7701, DSI_CMD2BKX_SEL,
+		   0x77, 0x01, 0x00, 0x00, DSI_CMD2BKX_SEL_NONE);
+
+	ST7701_DSI(st7701, DSI_CMD2BKX_SEL,
+		   0x77, 0x01, 0x00, 0x00, DSI_CMD2BK3_SEL);
+	ST7701_DSI(st7701, 0xE6, 0x7C);
+	ST7701_DSI(st7701, 0xE8, 0x00, 0x0E);
+
+	ST7701_DSI(st7701, DSI_CMD2BKX_SEL,
+		   0x77, 0x01, 0x00, 0x00, DSI_CMD2BKX_SEL_NONE);
+	ST7701_DSI(st7701, 0x11);
+	msleep(120);
+
+	ST7701_DSI(st7701, DSI_CMD2BKX_SEL,
+		   0x77, 0x01, 0x00, 0x00, DSI_CMD2BK3_SEL);
+	ST7701_DSI(st7701, 0xE8, 0x00, 0x0C);
+	msleep(10);
+	ST7701_DSI(st7701, 0xE8, 0x00, 0x00);
+
+	ST7701_DSI(st7701, DSI_CMD2BKX_SEL,
+		   0x77, 0x01, 0x00, 0x00, DSI_CMD2BKX_SEL_NONE);
+	ST7701_DSI(st7701, 0x11);
+	msleep(120);
+	ST7701_DSI(st7701, 0xE8, 0x00, 0x00);
 
-	/* disable Command2 */
 	ST7701_DSI(st7701, DSI_CMD2BKX_SEL,
 		   0x77, 0x01, 0x00, 0x00, DSI_CMD2BKX_SEL_NONE);
+
+	ST7701_DSI(st7701, 0x3A, 0x70);
 }
 
 static int st7701_prepare(struct drm_panel *panel)
@@ -201,7 +411,7 @@ static int st7701_prepare(struct drm_panel *panel)
 
 	gpiod_set_value(st7701->reset, 0);
 
-	ret = regulator_bulk_enable(st7701->desc->num_supplies,
+	ret = regulator_bulk_enable(ARRAY_SIZE(st7701->supplies),
 				    st7701->supplies);
 	if (ret < 0)
 		return ret;
@@ -212,6 +422,13 @@ static int st7701_prepare(struct drm_panel *panel)
 
 	st7701_init_sequence(st7701);
 
+	if (st7701->desc->gip_sequence)
+		st7701->desc->gip_sequence(st7701);
+
+	/* Disable Command2 */
+	ST7701_DSI(st7701, DSI_CMD2BKX_SEL,
+		   0x77, 0x01, 0x00, 0x00, DSI_CMD2BKX_SEL_NONE);
+
 	return 0;
 }
 
@@ -254,7 +471,7 @@ static int st7701_unprepare(struct drm_panel *panel)
 	 */
 	msleep(st7701->sleep_delay);
 
-	regulator_bulk_disable(st7701->desc->num_supplies, st7701->supplies);
+	regulator_bulk_disable(ARRAY_SIZE(st7701->supplies), st7701->supplies);
 
 	return 0;
 }
@@ -310,46 +527,207 @@ static const struct drm_display_mode ts8550b_mode = {
 	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
 };
 
-static const char * const ts8550b_supply_names[] = {
-	"VCC",
-	"IOVCC",
-};
-
 static const struct st7701_panel_desc ts8550b_desc = {
 	.mode = &ts8550b_mode,
 	.lanes = 2,
-	.flags = MIPI_DSI_MODE_VIDEO,
 	.format = MIPI_DSI_FMT_RGB888,
-	.supply_names = ts8550b_supply_names,
-	.num_supplies = ARRAY_SIZE(ts8550b_supply_names),
 	.panel_sleep_delay = 80, /* panel need extra 80ms for sleep out cmd */
+
+	.pv_gamma = {
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC0_MASK, 0),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC4_MASK, 0xe),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC8_MASK, 0x15),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC16_MASK, 0xf),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC24_MASK, 0x11),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC52_MASK, 0x8),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC80_MASK, 0x8),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC108_MASK, 0x8),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC147_MASK, 0x8),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC175_MASK, 0x23),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC203_MASK, 0x4),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC231_MASK, 0x13),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC239_MASK, 0x12),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC247_MASK, 0x2b),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC251_MASK, 0x34),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC255_MASK, 0x1f)
+	},
+	.nv_gamma = {
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC0_MASK, 0),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC4_MASK, 0xe),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0x2) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC8_MASK, 0x15),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC16_MASK, 0xf),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC24_MASK, 0x13),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC52_MASK, 0x7),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC80_MASK, 0x9),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC108_MASK, 0x8),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC147_MASK, 0x8),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC175_MASK, 0x22),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC203_MASK, 0x4),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC231_MASK, 0x10),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC239_MASK, 0xe),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC247_MASK, 0x2c),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC251_MASK, 0x34),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC255_MASK, 0x1f)
+	},
+	.nlinv = 7,
+	.vop_uv = 4400000,
+	.vcom_uv = 337500,
+	.vgh_mv = 15000,
+	.vgl_mv = -9510,
+	.avdd_mv = 6600,
+	.avcl_mv = -4400,
+	.gamma_op_bias = OP_BIAS_MAX,
+	.input_op_bias = OP_BIAS_MIN,
+	.output_op_bias = OP_BIAS_MIN,
+	.t2d_ns = 1600,
+	.t3d_ns = 10400,
+	.eot_en = true,
+	.gip_sequence = ts8550b_gip_sequence,
+};
+
+static const struct drm_display_mode dmt028vghmcmi_1a_mode = {
+	.clock		= 22325,
+
+	.hdisplay	= 480,
+	.hsync_start	= 480 + 40,
+	.hsync_end	= 480 + 40 + 4,
+	.htotal		= 480 + 40 + 4 + 20,
+
+	.vdisplay	= 640,
+	.vsync_start	= 640 + 2,
+	.vsync_end	= 640 + 2 + 40,
+	.vtotal		= 640 + 2 + 40 + 16,
+
+	.width_mm	= 56,
+	.height_mm	= 78,
+
+	.flags		= DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC,
+
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static const struct st7701_panel_desc dmt028vghmcmi_1a_desc = {
+	.mode = &dmt028vghmcmi_1a_mode,
+	.lanes = 2,
+	.format = MIPI_DSI_FMT_RGB888,
+	.panel_sleep_delay = 5, /* panel need extra 5ms for sleep out cmd */
+
+	.pv_gamma = {
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC0_MASK, 0),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC4_MASK, 0x10),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC8_MASK, 0x17),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC16_MASK, 0xd),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC24_MASK, 0x11),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC52_MASK, 0x6),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC80_MASK, 0x5),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC108_MASK, 0x8),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC147_MASK, 0x7),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC175_MASK, 0x1f),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC203_MASK, 0x4),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC231_MASK, 0x11),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC239_MASK, 0xe),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC247_MASK, 0x29),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC251_MASK, 0x30),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC255_MASK, 0x1f)
+	},
+	.nv_gamma = {
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC0_MASK, 0),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC4_MASK, 0xd),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC8_MASK, 0x14),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC16_MASK, 0xe),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC24_MASK, 0x11),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC52_MASK, 0x6),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC80_MASK, 0x4),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC108_MASK, 0x8),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC147_MASK, 0x8),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC175_MASK, 0x20),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC203_MASK, 0x5),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC231_MASK, 0x13),
+
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC239_MASK, 0x13),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC247_MASK, 0x26),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC251_MASK, 0x30),
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_AJ_MASK, 0) |
+		CFIELD_PREP(DSI_CMD2_BK0_GAMCTRL_VC255_MASK, 0x1f)
+	},
+	.nlinv = 1,
+	.vop_uv = 4800000,
+	.vcom_uv = 1650000,
+	.vgh_mv = 15000,
+	.vgl_mv = -10170,
+	.avdd_mv = 6600,
+	.avcl_mv = -4400,
+	.gamma_op_bias = OP_BIAS_MIDDLE,
+	.input_op_bias = OP_BIAS_MIN,
+	.output_op_bias = OP_BIAS_MIN,
+	.t2d_ns = 1600,
+	.t3d_ns = 10400,
+	.eot_en = true,
+	.gip_sequence = dmt028vghmcmi_1a_gip_sequence,
 };
 
 static int st7701_dsi_probe(struct mipi_dsi_device *dsi)
 {
 	const struct st7701_panel_desc *desc;
 	struct st7701 *st7701;
-	int ret, i;
+	int ret;
 
 	st7701 = devm_kzalloc(&dsi->dev, sizeof(*st7701), GFP_KERNEL);
 	if (!st7701)
 		return -ENOMEM;
 
 	desc = of_device_get_match_data(&dsi->dev);
-	dsi->mode_flags = desc->flags;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+			  MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS;
 	dsi->format = desc->format;
 	dsi->lanes = desc->lanes;
 
-	st7701->supplies = devm_kcalloc(&dsi->dev, desc->num_supplies,
-					sizeof(*st7701->supplies),
-					GFP_KERNEL);
-	if (!st7701->supplies)
-		return -ENOMEM;
-
-	for (i = 0; i < desc->num_supplies; i++)
-		st7701->supplies[i].supply = desc->supply_names[i];
+	st7701->supplies[0].supply = "VCC";
+	st7701->supplies[1].supply = "IOVCC";
 
-	ret = devm_regulator_bulk_get(&dsi->dev, desc->num_supplies,
+	ret = devm_regulator_bulk_get(&dsi->dev, ARRAY_SIZE(st7701->supplies),
 				      st7701->supplies);
 	if (ret < 0)
 		return ret;
@@ -387,17 +765,16 @@ static int st7701_dsi_probe(struct mipi_dsi_device *dsi)
 	return mipi_dsi_attach(dsi);
 }
 
-static int st7701_dsi_remove(struct mipi_dsi_device *dsi)
+static void st7701_dsi_remove(struct mipi_dsi_device *dsi)
 {
 	struct st7701 *st7701 = mipi_dsi_get_drvdata(dsi);
 
 	mipi_dsi_detach(dsi);
 	drm_panel_remove(&st7701->panel);
-
-	return 0;
 }
 
 static const struct of_device_id st7701_of_match[] = {
+	{ .compatible = "densitron,dmt028vghmcmi-1a", .data = &dmt028vghmcmi_1a_desc },
 	{ .compatible = "techstar,ts8550b", .data = &ts8550b_desc },
 	{ }
 };
diff --git a/drivers/gpu/drm/panel/panel-sitronix-st7703.c b/drivers/gpu/drm/panel/panel-sitronix-st7703.c
index 73f69c929a75..86a472b01360 100644
--- a/drivers/gpu/drm/panel/panel-sitronix-st7703.c
+++ b/drivers/gpu/drm/panel/panel-sitronix-st7703.c
@@ -598,7 +598,7 @@ static void st7703_shutdown(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to disable panel: %d\n", ret);
 }
 
-static int st7703_remove(struct mipi_dsi_device *dsi)
+static void st7703_remove(struct mipi_dsi_device *dsi)
 {
 	struct st7703 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -612,8 +612,6 @@ static int st7703_remove(struct mipi_dsi_device *dsi)
 	drm_panel_remove(&ctx->panel);
 
 	st7703_debugfs_remove(ctx);
-
-	return 0;
 }
 
 static const struct of_device_id st7703_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-sony-tulip-truly-nt35521.c b/drivers/gpu/drm/panel/panel-sony-tulip-truly-nt35521.c
index 69f07b15fca4..fa9be3c299c0 100644
--- a/drivers/gpu/drm/panel/panel-sony-tulip-truly-nt35521.c
+++ b/drivers/gpu/drm/panel/panel-sony-tulip-truly-nt35521.c
@@ -517,7 +517,7 @@ static int truly_nt35521_probe(struct mipi_dsi_device *dsi)
 	return 0;
 }
 
-static int truly_nt35521_remove(struct mipi_dsi_device *dsi)
+static void truly_nt35521_remove(struct mipi_dsi_device *dsi)
 {
 	struct truly_nt35521 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -527,8 +527,6 @@ static int truly_nt35521_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id truly_nt35521_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c b/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c
index 820731be7147..d8487bc6d611 100644
--- a/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c
+++ b/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c
@@ -210,7 +210,7 @@ static int tdo_tl070wsh30_panel_probe(struct mipi_dsi_device *dsi)
 	return mipi_dsi_attach(dsi);
 }
 
-static int tdo_tl070wsh30_panel_remove(struct mipi_dsi_device *dsi)
+static void tdo_tl070wsh30_panel_remove(struct mipi_dsi_device *dsi)
 {
 	struct tdo_tl070wsh30_panel *tdo_tl070wsh30 = mipi_dsi_get_drvdata(dsi);
 	int err;
@@ -222,8 +222,6 @@ static int tdo_tl070wsh30_panel_remove(struct mipi_dsi_device *dsi)
 	drm_panel_remove(&tdo_tl070wsh30->base);
 	drm_panel_disable(&tdo_tl070wsh30->base);
 	drm_panel_unprepare(&tdo_tl070wsh30->base);
-
-	return 0;
 }
 
 static void tdo_tl070wsh30_panel_shutdown(struct mipi_dsi_device *dsi)
diff --git a/drivers/gpu/drm/panel/panel-truly-nt35597.c b/drivers/gpu/drm/panel/panel-truly-nt35597.c
index 9ca5c7ff41d6..b31cffb660a7 100644
--- a/drivers/gpu/drm/panel/panel-truly-nt35597.c
+++ b/drivers/gpu/drm/panel/panel-truly-nt35597.c
@@ -616,7 +616,7 @@ err_panel_add:
 	return ret;
 }
 
-static int truly_nt35597_remove(struct mipi_dsi_device *dsi)
+static void truly_nt35597_remove(struct mipi_dsi_device *dsi)
 {
 	struct truly_nt35597 *ctx = mipi_dsi_get_drvdata(dsi);
 
@@ -628,7 +628,6 @@ static int truly_nt35597_remove(struct mipi_dsi_device *dsi)
 	}
 
 	drm_panel_remove(&ctx->panel);
-	return 0;
 }
 
 static const struct of_device_id truly_nt35597_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-visionox-rm69299.c b/drivers/gpu/drm/panel/panel-visionox-rm69299.c
index db2443ac81d3..ec228c269146 100644
--- a/drivers/gpu/drm/panel/panel-visionox-rm69299.c
+++ b/drivers/gpu/drm/panel/panel-visionox-rm69299.c
@@ -256,7 +256,7 @@ err_dsi_attach:
 	return ret;
 }
 
-static int visionox_rm69299_remove(struct mipi_dsi_device *dsi)
+static void visionox_rm69299_remove(struct mipi_dsi_device *dsi)
 {
 	struct visionox_rm69299 *ctx = mipi_dsi_get_drvdata(dsi);
 
@@ -264,7 +264,6 @@ static int visionox_rm69299_remove(struct mipi_dsi_device *dsi)
 	mipi_dsi_device_unregister(ctx->dsi);
 
 	drm_panel_remove(&ctx->panel);
-	return 0;
 }
 
 static const struct of_device_id visionox_rm69299_of_match[] = {
diff --git a/drivers/gpu/drm/panel/panel-xinpeng-xpp055c272.c b/drivers/gpu/drm/panel/panel-xinpeng-xpp055c272.c
index 8177f5a360fb..2c54733ee241 100644
--- a/drivers/gpu/drm/panel/panel-xinpeng-xpp055c272.c
+++ b/drivers/gpu/drm/panel/panel-xinpeng-xpp055c272.c
@@ -339,7 +339,7 @@ static void xpp055c272_shutdown(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to disable panel: %d\n", ret);
 }
 
-static int xpp055c272_remove(struct mipi_dsi_device *dsi)
+static void xpp055c272_remove(struct mipi_dsi_device *dsi)
 {
 	struct xpp055c272 *ctx = mipi_dsi_get_drvdata(dsi);
 	int ret;
@@ -351,8 +351,6 @@ static int xpp055c272_remove(struct mipi_dsi_device *dsi)
 		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
 
 	drm_panel_remove(&ctx->panel);
-
-	return 0;
 }
 
 static const struct of_device_id xpp055c272_of_match[] = {
diff --git a/drivers/gpu/drm/panfrost/Kconfig b/drivers/gpu/drm/panfrost/Kconfig
index 86cdc0ce79e6..079600328be1 100644
--- a/drivers/gpu/drm/panfrost/Kconfig
+++ b/drivers/gpu/drm/panfrost/Kconfig
@@ -11,6 +11,7 @@ config DRM_PANFROST
 	select DRM_GEM_SHMEM_HELPER
 	select PM_DEVFREQ
 	select DEVFREQ_GOV_SIMPLE_ONDEMAND
+	select WANT_DEV_COREDUMP
 	help
 	  DRM driver for ARM Mali Midgard (T6xx, T7xx, T8xx) and
 	  Bifrost (G3x, G5x, G7x) GPUs.
diff --git a/drivers/gpu/drm/panfrost/Makefile b/drivers/gpu/drm/panfrost/Makefile
index b71935862417..7da2b3f02ed9 100644
--- a/drivers/gpu/drm/panfrost/Makefile
+++ b/drivers/gpu/drm/panfrost/Makefile
@@ -9,6 +9,7 @@ panfrost-y := \
 	panfrost_gpu.o \
 	panfrost_job.o \
 	panfrost_mmu.o \
-	panfrost_perfcnt.o
+	panfrost_perfcnt.o \
+	panfrost_dump.o
 
 obj-$(CONFIG_DRM_PANFROST) += panfrost.o
diff --git a/drivers/gpu/drm/panfrost/panfrost_dump.c b/drivers/gpu/drm/panfrost/panfrost_dump.c
new file mode 100644
index 000000000000..89056a1aac7d
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_dump.c
@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2021 Collabora ltd. */
+
+#include <linux/err.h>
+#include <linux/device.h>
+#include <linux/devcoredump.h>
+#include <linux/moduleparam.h>
+#include <linux/iosys-map.h>
+#include <drm/panfrost_drm.h>
+#include <drm/drm_device.h>
+
+#include "panfrost_job.h"
+#include "panfrost_gem.h"
+#include "panfrost_regs.h"
+#include "panfrost_dump.h"
+#include "panfrost_device.h"
+
+static bool panfrost_dump_core = true;
+module_param_named(dump_core, panfrost_dump_core, bool, 0600);
+
+struct panfrost_dump_iterator {
+	void *start;
+	struct panfrost_dump_object_header *hdr;
+	void *data;
+};
+
+static const unsigned short panfrost_dump_registers[] = {
+	SHADER_READY_LO,
+	SHADER_READY_HI,
+	TILER_READY_LO,
+	TILER_READY_HI,
+	L2_READY_LO,
+	L2_READY_HI,
+	JOB_INT_MASK,
+	JOB_INT_STAT,
+	JS_HEAD_LO(0),
+	JS_HEAD_HI(0),
+	JS_TAIL_LO(0),
+	JS_TAIL_HI(0),
+	JS_AFFINITY_LO(0),
+	JS_AFFINITY_HI(0),
+	JS_CONFIG(0),
+	JS_STATUS(0),
+	JS_HEAD_NEXT_LO(0),
+	JS_HEAD_NEXT_HI(0),
+	JS_AFFINITY_NEXT_LO(0),
+	JS_AFFINITY_NEXT_HI(0),
+	JS_CONFIG_NEXT(0),
+	MMU_INT_MASK,
+	MMU_INT_STAT,
+	AS_TRANSTAB_LO(0),
+	AS_TRANSTAB_HI(0),
+	AS_MEMATTR_LO(0),
+	AS_MEMATTR_HI(0),
+	AS_FAULTSTATUS(0),
+	AS_FAULTADDRESS_LO(0),
+	AS_FAULTADDRESS_HI(0),
+	AS_STATUS(0),
+};
+
+static void panfrost_core_dump_header(struct panfrost_dump_iterator *iter,
+				      u32 type, void *data_end)
+{
+	struct panfrost_dump_object_header *hdr = iter->hdr;
+
+	hdr->magic = cpu_to_le32(PANFROSTDUMP_MAGIC);
+	hdr->type = cpu_to_le32(type);
+	hdr->file_offset = cpu_to_le32(iter->data - iter->start);
+	hdr->file_size = cpu_to_le32(data_end - iter->data);
+
+	iter->hdr++;
+	iter->data += le32_to_cpu(hdr->file_size);
+}
+
+static void
+panfrost_core_dump_registers(struct panfrost_dump_iterator *iter,
+			     struct panfrost_device *pfdev,
+			     u32 as_nr, int slot)
+{
+	struct panfrost_dump_registers *dumpreg = iter->data;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(panfrost_dump_registers); i++, dumpreg++) {
+		unsigned int js_as_offset = 0;
+		unsigned int reg;
+
+		if (panfrost_dump_registers[i] >= JS_BASE &&
+		    panfrost_dump_registers[i] <= JS_BASE + JS_SLOT_STRIDE)
+			js_as_offset = slot * JS_SLOT_STRIDE;
+		else if (panfrost_dump_registers[i] >= MMU_BASE &&
+			 panfrost_dump_registers[i] <= MMU_BASE + MMU_AS_STRIDE)
+			js_as_offset = (as_nr << MMU_AS_SHIFT);
+
+		reg = panfrost_dump_registers[i] + js_as_offset;
+
+		dumpreg->reg = cpu_to_le32(reg);
+		dumpreg->value = cpu_to_le32(gpu_read(pfdev, reg));
+	}
+
+	panfrost_core_dump_header(iter, PANFROSTDUMP_BUF_REG, dumpreg);
+}
+
+void panfrost_core_dump(struct panfrost_job *job)
+{
+	struct panfrost_device *pfdev = job->pfdev;
+	struct panfrost_dump_iterator iter;
+	struct drm_gem_object *dbo;
+	unsigned int n_obj, n_bomap_pages;
+	__le64 *bomap, *bomap_start;
+	size_t file_size;
+	u32 as_nr;
+	int slot;
+	int ret, i;
+
+	as_nr = job->mmu->as;
+	slot = panfrost_job_get_slot(job);
+
+	/* Only catch the first event, or when manually re-armed */
+	if (!panfrost_dump_core)
+		return;
+	panfrost_dump_core = false;
+
+	/* At least, we dump registers and end marker */
+	n_obj = 2;
+	n_bomap_pages = 0;
+	file_size = ARRAY_SIZE(panfrost_dump_registers) *
+			sizeof(struct panfrost_dump_registers);
+
+	/* Add in the active buffer objects */
+	for (i = 0; i < job->bo_count; i++) {
+		/*
+		 * Even though the CPU could be configured to use 16K or 64K pages, this
+		 * is a very unusual situation for most kernel setups on SoCs that have
+		 * a Panfrost device. Also many places across the driver make the somewhat
+		 * arbitrary assumption that Panfrost's MMU page size is the same as the CPU's,
+		 * so let's have a sanity check to ensure that's always the case
+		 */
+		dbo = job->bos[i];
+		WARN_ON(!IS_ALIGNED(dbo->size, PAGE_SIZE));
+
+		file_size += dbo->size;
+		n_bomap_pages += dbo->size >> PAGE_SHIFT;
+		n_obj++;
+	}
+
+	/* If we have any buffer objects, add a bomap object */
+	if (n_bomap_pages) {
+		file_size += n_bomap_pages * sizeof(*bomap);
+		n_obj++;
+	}
+
+	/* Add the size of the headers */
+	file_size += sizeof(*iter.hdr) * n_obj;
+
+	/*
+	 * Allocate the file in vmalloc memory, it's likely to be big.
+	 * The reason behind these GFP flags is that we don't want to trigger the
+	 * OOM killer in the event that not enough memory could be found for our
+	 * dump file. We also don't want the allocator to do any error reporting,
+	 * as the right behaviour is failing gracefully if a big enough buffer
+	 * could not be allocated.
+	 */
+	iter.start = __vmalloc(file_size, GFP_KERNEL | __GFP_NOWARN |
+			__GFP_NORETRY);
+	if (!iter.start) {
+		dev_warn(pfdev->dev, "failed to allocate devcoredump file\n");
+		return;
+	}
+
+	/* Point the data member after the headers */
+	iter.hdr = iter.start;
+	iter.data = &iter.hdr[n_obj];
+
+	memset(iter.hdr, 0, iter.data - iter.start);
+
+	/*
+	 * For now, we write the job identifier in the register dump header,
+	 * so that we can decode the entire dump later with pandecode
+	 */
+	iter.hdr->reghdr.jc = cpu_to_le64(job->jc);
+	iter.hdr->reghdr.major = cpu_to_le32(PANFROSTDUMP_MAJOR);
+	iter.hdr->reghdr.minor = cpu_to_le32(PANFROSTDUMP_MINOR);
+	iter.hdr->reghdr.gpu_id = cpu_to_le32(pfdev->features.id);
+	iter.hdr->reghdr.nbos = cpu_to_le64(job->bo_count);
+
+	panfrost_core_dump_registers(&iter, pfdev, as_nr, slot);
+
+	/* Reserve space for the bomap */
+	if (job->bo_count) {
+		bomap_start = bomap = iter.data;
+		memset(bomap, 0, sizeof(*bomap) * n_bomap_pages);
+		panfrost_core_dump_header(&iter, PANFROSTDUMP_BUF_BOMAP,
+					  bomap + n_bomap_pages);
+	}
+
+	for (i = 0; i < job->bo_count; i++) {
+		struct iosys_map map;
+		struct panfrost_gem_mapping *mapping;
+		struct panfrost_gem_object *bo;
+		struct sg_page_iter page_iter;
+		void *vaddr;
+
+		bo = to_panfrost_bo(job->bos[i]);
+		mapping = job->mappings[i];
+
+		if (!bo->base.sgt) {
+			dev_err(pfdev->dev, "Panfrost Dump: BO has no sgt, cannot dump\n");
+			iter.hdr->bomap.valid = 0;
+			goto dump_header;
+		}
+
+		ret = drm_gem_shmem_vmap(&bo->base, &map);
+		if (ret) {
+			dev_err(pfdev->dev, "Panfrost Dump: couldn't map Buffer Object\n");
+			iter.hdr->bomap.valid = 0;
+			goto dump_header;
+		}
+
+		WARN_ON(!mapping->active);
+
+		iter.hdr->bomap.data[0] = cpu_to_le32((bomap - bomap_start));
+
+		for_each_sgtable_page(bo->base.sgt, &page_iter, 0) {
+			struct page *page = sg_page_iter_page(&page_iter);
+
+			if (!IS_ERR(page)) {
+				*bomap++ = cpu_to_le64(page_to_phys(page));
+			} else {
+				dev_err(pfdev->dev, "Panfrost Dump: wrong page\n");
+				*bomap++ = ~cpu_to_le64(0);
+			}
+		}
+
+		iter.hdr->bomap.iova = cpu_to_le64(mapping->mmnode.start << PAGE_SHIFT);
+
+		vaddr = map.vaddr;
+		memcpy(iter.data, vaddr, bo->base.base.size);
+
+		drm_gem_shmem_vunmap(&bo->base, &map);
+
+		iter.hdr->bomap.valid = cpu_to_le32(1);
+
+dump_header:	panfrost_core_dump_header(&iter, PANFROSTDUMP_BUF_BO, iter.data +
+					  bo->base.base.size);
+	}
+	panfrost_core_dump_header(&iter, PANFROSTDUMP_BUF_TRAILER, iter.data);
+
+	dev_coredumpv(pfdev->dev, iter.start, iter.data - iter.start, GFP_KERNEL);
+}
diff --git a/drivers/gpu/drm/panfrost/panfrost_dump.h b/drivers/gpu/drm/panfrost/panfrost_dump.h
new file mode 100644
index 000000000000..7d9bcefa5346
--- /dev/null
+++ b/drivers/gpu/drm/panfrost/panfrost_dump.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2021 Collabora ltd.
+ */
+
+#ifndef PANFROST_DUMP_H
+#define PANFROST_DUMP_H
+
+struct panfrost_job;
+void panfrost_core_dump(struct panfrost_job *job);
+
+#endif
diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c
index 7c4208476fbd..dbc597ab46fb 100644
--- a/drivers/gpu/drm/panfrost/panfrost_job.c
+++ b/drivers/gpu/drm/panfrost/panfrost_job.c
@@ -20,6 +20,7 @@
 #include "panfrost_regs.h"
 #include "panfrost_gpu.h"
 #include "panfrost_mmu.h"
+#include "panfrost_dump.h"
 
 #define JOB_TIMEOUT_MS 500
 
@@ -727,6 +728,8 @@ static enum drm_gpu_sched_stat panfrost_job_timedout(struct drm_sched_job
 		job_read(pfdev, JS_TAIL_LO(js)),
 		sched_job);
 
+	panfrost_core_dump(job);
+
 	atomic_set(&pfdev->reset.pending, 1);
 	panfrost_reset(pfdev, sched_job);
 
diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c
index b285a8001b1d..e246d914e7f6 100644
--- a/drivers/gpu/drm/panfrost/panfrost_mmu.c
+++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c
@@ -248,11 +248,15 @@ void panfrost_mmu_reset(struct panfrost_device *pfdev)
 	mmu_write(pfdev, MMU_INT_MASK, ~0);
 }
 
-static size_t get_pgsize(u64 addr, size_t size)
+static size_t get_pgsize(u64 addr, size_t size, size_t *count)
 {
-	if (addr & (SZ_2M - 1) || size < SZ_2M)
-		return SZ_4K;
+	size_t blk_offset = -addr % SZ_2M;
 
+	if (blk_offset || size < SZ_2M) {
+		*count = min_not_zero(blk_offset, size) / SZ_4K;
+		return SZ_4K;
+	}
+	*count = size / SZ_2M;
 	return SZ_2M;
 }
 
@@ -287,12 +291,16 @@ static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu,
 		dev_dbg(pfdev->dev, "map: as=%d, iova=%llx, paddr=%lx, len=%zx", mmu->as, iova, paddr, len);
 
 		while (len) {
-			size_t pgsize = get_pgsize(iova | paddr, len);
-
-			ops->map(ops, iova, paddr, pgsize, prot, GFP_KERNEL);
-			iova += pgsize;
-			paddr += pgsize;
-			len -= pgsize;
+			size_t pgcount, mapped = 0;
+			size_t pgsize = get_pgsize(iova | paddr, len, &pgcount);
+
+			ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot,
+				       GFP_KERNEL, &mapped);
+			/* Don't get stuck if things have gone wrong */
+			mapped = max(mapped, pgsize);
+			iova += mapped;
+			paddr += mapped;
+			len -= mapped;
 		}
 	}
 
@@ -344,15 +352,17 @@ void panfrost_mmu_unmap(struct panfrost_gem_mapping *mapping)
 		mapping->mmu->as, iova, len);
 
 	while (unmapped_len < len) {
-		size_t unmapped_page;
-		size_t pgsize = get_pgsize(iova, len - unmapped_len);
-
-		if (ops->iova_to_phys(ops, iova)) {
-			unmapped_page = ops->unmap(ops, iova, pgsize, NULL);
-			WARN_ON(unmapped_page != pgsize);
+		size_t unmapped_page, pgcount;
+		size_t pgsize = get_pgsize(iova, len - unmapped_len, &pgcount);
+
+		if (bo->is_heap)
+			pgcount = 1;
+		if (!bo->is_heap || ops->iova_to_phys(ops, iova)) {
+			unmapped_page = ops->unmap_pages(ops, iova, pgsize, pgcount, NULL);
+			WARN_ON(unmapped_page != pgsize * pgcount);
 		}
-		iova += pgsize;
-		unmapped_len += pgsize;
+		iova += pgsize * pgcount;
+		unmapped_len += pgsize * pgcount;
 	}
 
 	panfrost_mmu_flush_range(pfdev, mapping->mmu,
diff --git a/drivers/gpu/drm/panfrost/panfrost_regs.h b/drivers/gpu/drm/panfrost/panfrost_regs.h
index accb4fa3adb8..919f44ac853d 100644
--- a/drivers/gpu/drm/panfrost/panfrost_regs.h
+++ b/drivers/gpu/drm/panfrost/panfrost_regs.h
@@ -226,23 +226,25 @@
 #define JOB_INT_MASK_DONE(j)		BIT(j)
 
 #define JS_BASE				0x1800
-#define JS_HEAD_LO(n)			(JS_BASE + ((n) * 0x80) + 0x00)
-#define JS_HEAD_HI(n)			(JS_BASE + ((n) * 0x80) + 0x04)
-#define JS_TAIL_LO(n)			(JS_BASE + ((n) * 0x80) + 0x08)
-#define JS_TAIL_HI(n)			(JS_BASE + ((n) * 0x80) + 0x0c)
-#define JS_AFFINITY_LO(n)		(JS_BASE + ((n) * 0x80) + 0x10)
-#define JS_AFFINITY_HI(n)		(JS_BASE + ((n) * 0x80) + 0x14)
-#define JS_CONFIG(n)			(JS_BASE + ((n) * 0x80) + 0x18)
-#define JS_XAFFINITY(n)			(JS_BASE + ((n) * 0x80) + 0x1c)
-#define JS_COMMAND(n)			(JS_BASE + ((n) * 0x80) + 0x20)
-#define JS_STATUS(n)			(JS_BASE + ((n) * 0x80) + 0x24)
-#define JS_HEAD_NEXT_LO(n)		(JS_BASE + ((n) * 0x80) + 0x40)
-#define JS_HEAD_NEXT_HI(n)		(JS_BASE + ((n) * 0x80) + 0x44)
-#define JS_AFFINITY_NEXT_LO(n)		(JS_BASE + ((n) * 0x80) + 0x50)
-#define JS_AFFINITY_NEXT_HI(n)		(JS_BASE + ((n) * 0x80) + 0x54)
-#define JS_CONFIG_NEXT(n)		(JS_BASE + ((n) * 0x80) + 0x58)
-#define JS_COMMAND_NEXT(n)		(JS_BASE + ((n) * 0x80) + 0x60)
-#define JS_FLUSH_ID_NEXT(n)		(JS_BASE + ((n) * 0x80) + 0x70)
+#define JS_SLOT_STRIDE			0x80
+
+#define JS_HEAD_LO(n)			(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x00)
+#define JS_HEAD_HI(n)			(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x04)
+#define JS_TAIL_LO(n)			(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x08)
+#define JS_TAIL_HI(n)			(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x0c)
+#define JS_AFFINITY_LO(n)		(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x10)
+#define JS_AFFINITY_HI(n)		(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x14)
+#define JS_CONFIG(n)			(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x18)
+#define JS_XAFFINITY(n)			(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x1c)
+#define JS_COMMAND(n)			(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x20)
+#define JS_STATUS(n)			(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x24)
+#define JS_HEAD_NEXT_LO(n)		(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x40)
+#define JS_HEAD_NEXT_HI(n)		(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x44)
+#define JS_AFFINITY_NEXT_LO(n)		(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x50)
+#define JS_AFFINITY_NEXT_HI(n)		(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x54)
+#define JS_CONFIG_NEXT(n)		(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x58)
+#define JS_COMMAND_NEXT(n)		(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x60)
+#define JS_FLUSH_ID_NEXT(n)		(JS_BASE + ((n) * JS_SLOT_STRIDE) + 0x70)
 
 /* Possible values of JS_CONFIG and JS_CONFIG_NEXT registers */
 #define JS_CONFIG_START_FLUSH_CLEAN		BIT(8)
@@ -281,7 +283,9 @@
 #define AS_COMMAND_FLUSH_MEM		0x05	/* Wait for memory accesses to complete, flush all the L1s cache then
 						   flush all L2 caches then issue a flush region command to all MMUs */
 
-#define MMU_AS(as)			(0x2400 + ((as) << 6))
+#define MMU_BASE			0x2400
+#define MMU_AS_SHIFT			0x06
+#define MMU_AS(as)			(MMU_BASE + ((as) << MMU_AS_SHIFT))
 
 #define AS_TRANSTAB_LO(as)		(MMU_AS(as) + 0x00) /* (RW) Translation Table Base Address for address space n, low word */
 #define AS_TRANSTAB_HI(as)		(MMU_AS(as) + 0x04) /* (RW) Translation Table Base Address for address space n, high word */
@@ -300,6 +304,8 @@
 #define AS_FAULTEXTRA_LO(as)		(MMU_AS(as) + 0x38) /* (RO) Secondary fault address for address space n, low word */
 #define AS_FAULTEXTRA_HI(as)		(MMU_AS(as) + 0x3C) /* (RO) Secondary fault address for address space n, high word */
 
+#define MMU_AS_STRIDE			(1 << MMU_AS_SHIFT)
+
 /*
  * Begin LPAE MMU TRANSTAB register values
  */
diff --git a/drivers/gpu/drm/pl111/Kconfig b/drivers/gpu/drm/pl111/Kconfig
index 91ee05b01303..ad24cdf1d992 100644
--- a/drivers/gpu/drm/pl111/Kconfig
+++ b/drivers/gpu/drm/pl111/Kconfig
@@ -6,7 +6,7 @@ config DRM_PL111
 	depends on VEXPRESS_CONFIG || VEXPRESS_CONFIG=n
 	depends on COMMON_CLK
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_BRIDGE
 	select DRM_PANEL_BRIDGE
 	select VT_HW_CONSOLE_BINDING if FRAMEBUFFER_CONSOLE
diff --git a/drivers/gpu/drm/pl111/pl111_display.c b/drivers/gpu/drm/pl111/pl111_display.c
index 6263346f24c6..6afdf260a4e2 100644
--- a/drivers/gpu/drm/pl111/pl111_display.c
+++ b/drivers/gpu/drm/pl111/pl111_display.c
@@ -15,11 +15,11 @@
 #include <linux/media-bus-format.h>
 #include <linux/of_graph.h>
 
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "pl111_drm.h"
@@ -94,7 +94,7 @@ static int pl111_display_check(struct drm_simple_display_pipe *pipe,
 		return -EINVAL;
 
 	if (fb) {
-		u32 offset = drm_fb_cma_get_gem_addr(fb, pstate, 0);
+		u32 offset = drm_fb_dma_get_gem_addr(fb, pstate, 0);
 
 		/* FB base address must be dword aligned. */
 		if (offset & 3)
@@ -398,7 +398,7 @@ static void pl111_display_update(struct drm_simple_display_pipe *pipe,
 	struct drm_framebuffer *fb = pstate->fb;
 
 	if (fb) {
-		u32 addr = drm_fb_cma_get_gem_addr(fb, pstate, 0);
+		u32 addr = drm_fb_dma_get_gem_addr(fb, pstate, 0);
 
 		writel(addr, priv->regs + CLCD_UBAS);
 	}
diff --git a/drivers/gpu/drm/pl111/pl111_drv.c b/drivers/gpu/drm/pl111/pl111_drv.c
index 19a4324bd356..eb25eedb5ee0 100644
--- a/drivers/gpu/drm/pl111/pl111_drv.c
+++ b/drivers/gpu/drm/pl111/pl111_drv.c
@@ -48,10 +48,9 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
@@ -208,10 +207,10 @@ pl111_gem_import_sg_table(struct drm_device *dev,
 	if (priv->use_device_memory)
 		return ERR_PTR(-EINVAL);
 
-	return drm_gem_cma_prime_import_sg_table(dev, attach, sgt);
+	return drm_gem_dma_prime_import_sg_table(dev, attach, sgt);
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(drm_fops);
+DEFINE_DRM_GEM_DMA_FOPS(drm_fops);
 
 static const struct drm_driver pl111_drm_driver = {
 	.driver_features =
@@ -224,7 +223,7 @@ static const struct drm_driver pl111_drm_driver = {
 	.major = 1,
 	.minor = 0,
 	.patchlevel = 0,
-	.dumb_create = drm_gem_cma_dumb_create,
+	.dumb_create = drm_gem_dma_dumb_create,
 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
 	.gem_prime_import_sg_table = pl111_gem_import_sg_table,
diff --git a/drivers/gpu/drm/pl111/pl111_versatile.c b/drivers/gpu/drm/pl111/pl111_versatile.c
index efb01a554574..1b436b75fd39 100644
--- a/drivers/gpu/drm/pl111/pl111_versatile.c
+++ b/drivers/gpu/drm/pl111/pl111_versatile.c
@@ -404,6 +404,7 @@ static int pl111_vexpress_clcd_init(struct device *dev, struct device_node *np,
 		if (of_device_is_compatible(child, "arm,pl111")) {
 			has_coretile_clcd = true;
 			ct_clcd = child;
+			of_node_put(child);
 			break;
 		}
 		if (of_device_is_compatible(child, "arm,hdlcd")) {
diff --git a/drivers/gpu/drm/qxl/qxl_cmd.c b/drivers/gpu/drm/qxl/qxl_cmd.c
index 7b00c955cd82..63aa96a69752 100644
--- a/drivers/gpu/drm/qxl/qxl_cmd.c
+++ b/drivers/gpu/drm/qxl/qxl_cmd.c
@@ -53,17 +53,11 @@ void qxl_ring_free(struct qxl_ring *ring)
 	kfree(ring);
 }
 
-void qxl_ring_init_hdr(struct qxl_ring *ring)
-{
-	ring->ring->header.notify_on_prod = ring->n_elements;
-}
-
 struct qxl_ring *
 qxl_ring_create(struct qxl_ring_header *header,
 		int element_size,
 		int n_elements,
 		int prod_notify,
-		bool set_prod_notify,
 		wait_queue_head_t *push_event)
 {
 	struct qxl_ring *ring;
@@ -77,8 +71,6 @@ qxl_ring_create(struct qxl_ring_header *header,
 	ring->n_elements = n_elements;
 	ring->prod_notify = prod_notify;
 	ring->push_event = push_event;
-	if (set_prod_notify)
-		qxl_ring_init_hdr(ring);
 	spin_lock_init(&ring->lock);
 	return ring;
 }
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 2e8949863d6b..a152a7c6db21 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -902,7 +902,7 @@ static const struct drm_plane_helper_funcs qxl_cursor_helper_funcs = {
 static const struct drm_plane_funcs qxl_cursor_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.destroy	= drm_primary_helper_destroy,
+	.destroy	= drm_plane_helper_destroy,
 	.reset		= drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
@@ -924,7 +924,7 @@ static const struct drm_plane_helper_funcs primary_helper_funcs = {
 static const struct drm_plane_funcs qxl_primary_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.destroy	= drm_primary_helper_destroy,
+	.destroy	= drm_plane_helper_destroy,
 	.reset		= drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c
index 1cb6f0c224bb..3044ca948ce2 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.c
+++ b/drivers/gpu/drm/qxl/qxl_drv.c
@@ -194,7 +194,6 @@ static int qxl_drm_resume(struct drm_device *dev, bool thaw)
 	qdev->ram_header->int_mask = QXL_INTERRUPT_MASK;
 	if (!thaw) {
 		qxl_reinit_memslots(qdev);
-		qxl_ring_init_hdr(qdev->release_ring);
 	}
 
 	qxl_create_monitors_object(qdev);
@@ -220,6 +219,7 @@ static int qxl_pm_resume(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+	struct qxl_device *qdev = to_qxl(drm_dev);
 
 	pci_set_power_state(pdev, PCI_D0);
 	pci_restore_state(pdev);
@@ -227,6 +227,7 @@ static int qxl_pm_resume(struct device *dev)
 		return -EIO;
 	}
 
+	qxl_io_reset(qdev);
 	return qxl_drm_resume(drm_dev, false);
 }
 
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 47c169673088..432758ad39a3 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -277,10 +277,8 @@ struct qxl_ring *qxl_ring_create(struct qxl_ring_header *header,
 				 int element_size,
 				 int n_elements,
 				 int prod_notify,
-				 bool set_prod_notify,
 				 wait_queue_head_t *push_event);
 void qxl_ring_free(struct qxl_ring *ring);
-void qxl_ring_init_hdr(struct qxl_ring *ring);
 int qxl_check_idle(struct qxl_ring *ring);
 
 static inline uint64_t
diff --git a/drivers/gpu/drm/qxl/qxl_kms.c b/drivers/gpu/drm/qxl/qxl_kms.c
index 9bf6d4cc98d4..dc3828db1991 100644
--- a/drivers/gpu/drm/qxl/qxl_kms.c
+++ b/drivers/gpu/drm/qxl/qxl_kms.c
@@ -194,7 +194,6 @@ int qxl_device_init(struct qxl_device *qdev,
 					     sizeof(struct qxl_command),
 					     QXL_COMMAND_RING_SIZE,
 					     qdev->io_base + QXL_IO_NOTIFY_CMD,
-					     false,
 					     &qdev->display_event);
 	if (!qdev->command_ring) {
 		DRM_ERROR("Unable to create command ring\n");
@@ -207,7 +206,6 @@ int qxl_device_init(struct qxl_device *qdev,
 				sizeof(struct qxl_command),
 				QXL_CURSOR_RING_SIZE,
 				qdev->io_base + QXL_IO_NOTIFY_CURSOR,
-				false,
 				&qdev->cursor_event);
 
 	if (!qdev->cursor_ring) {
@@ -219,7 +217,7 @@ int qxl_device_init(struct qxl_device *qdev,
 	qdev->release_ring = qxl_ring_create(
 				&(qdev->ram_header->release_ring_hdr),
 				sizeof(uint64_t),
-				QXL_RELEASE_RING_SIZE, 0, true,
+				QXL_RELEASE_RING_SIZE, 0,
 				NULL);
 
 	if (!qdev->release_ring) {
diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c
index b42a657e4c2f..695d9308d1f0 100644
--- a/drivers/gpu/drm/qxl/qxl_object.c
+++ b/drivers/gpu/drm/qxl/qxl_object.c
@@ -141,7 +141,7 @@ int qxl_bo_create(struct qxl_device *qdev, unsigned long size,
 	qxl_ttm_placement_from_domain(bo, domain);
 
 	bo->tbo.priority = priority;
-	r = ttm_bo_init_reserved(&qdev->mman.bdev, &bo->tbo, size, type,
+	r = ttm_bo_init_reserved(&qdev->mman.bdev, &bo->tbo, type,
 				 &bo->placement, 0, &ctx, NULL, NULL,
 				 &qxl_ttm_bo_destroy);
 	if (unlikely(r != 0)) {
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index e3ab3aca1396..bb4e56f2f170 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -49,7 +49,7 @@ radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \
 	rv770_smc.o cypress_dpm.o btc_dpm.o sumo_dpm.o sumo_smc.o trinity_dpm.o \
 	trinity_smc.o ni_dpm.o si_smc.o si_dpm.o kv_smc.o kv_dpm.o ci_smc.o \
 	ci_dpm.o dce6_afmt.o radeon_vm.o radeon_ucode.o radeon_ib.o \
-	radeon_sync.o radeon_audio.o radeon_dp_auxch.o radeon_dp_mst.o
+	radeon_sync.o radeon_audio.o radeon_dp_auxch.o
 
 radeon-$(CONFIG_MMU_NOTIFIER) += radeon_mn.o
 
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index 69f1bc073902..d28d3acb3ba1 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -617,13 +617,6 @@ static u32 atombios_adjust_pll(struct drm_crtc *crtc,
 		}
 	}
 
-	if (radeon_encoder->is_mst_encoder) {
-		struct radeon_encoder_mst *mst_enc = radeon_encoder->enc_priv;
-		struct radeon_connector_atom_dig *dig_connector = mst_enc->connector->con_priv;
-
-		dp_clock = dig_connector->dp_clock;
-	}
-
 	/* use recommended ref_div for ss */
 	if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) {
 		if (radeon_crtc->ss_enabled) {
@@ -972,9 +965,7 @@ static bool atombios_crtc_prepare_pll(struct drm_crtc *crtc, struct drm_display_
 	radeon_crtc->bpc = 8;
 	radeon_crtc->ss_enabled = false;
 
-	if (radeon_encoder->is_mst_encoder) {
-		radeon_dp_mst_prepare_pll(crtc, mode);
-	} else if ((radeon_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) ||
+	if ((radeon_encoder->active_device & (ATOM_DEVICE_LCD_SUPPORT | ATOM_DEVICE_DFP_SUPPORT)) ||
 	    (radeon_encoder_get_dp_bridge_encoder_id(radeon_crtc->encoder) != ENCODER_OBJECT_ID_NONE)) {
 		struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
 		struct drm_connector *connector =
diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
index c93040e60d04..c841c273222e 100644
--- a/drivers/gpu/drm/radeon/atombios_encoders.c
+++ b/drivers/gpu/drm/radeon/atombios_encoders.c
@@ -32,6 +32,8 @@
 #include <drm/drm_file.h>
 #include <drm/radeon_drm.h>
 
+#include <acpi/video.h>
+
 #include "atom.h"
 #include "radeon_atombios.h"
 #include "radeon.h"
@@ -209,6 +211,11 @@ void radeon_atom_backlight_init(struct radeon_encoder *radeon_encoder,
 	if (!(rdev->mode_info.firmware_flags & ATOM_BIOS_INFO_BL_CONTROLLED_BY_GPU))
 		return;
 
+	if (!acpi_video_backlight_use_native()) {
+		drm_info(dev, "Skipping radeon atom DIG backlight registration\n");
+		return;
+	}
+
 	pdata = kmalloc(sizeof(struct radeon_backlight_privdata), GFP_KERNEL);
 	if (!pdata) {
 		DRM_ERROR("Memory allocation failed\n");
@@ -667,15 +674,7 @@ atombios_get_encoder_mode(struct drm_encoder *encoder)
 	struct drm_connector *connector;
 	struct radeon_connector *radeon_connector;
 	struct radeon_connector_atom_dig *dig_connector;
-	struct radeon_encoder_atom_dig *dig_enc;
 
-	if (radeon_encoder_is_digital(encoder)) {
-		dig_enc = radeon_encoder->enc_priv;
-		if (dig_enc->active_mst_links)
-			return ATOM_ENCODER_MODE_DP_MST;
-	}
-	if (radeon_encoder->is_mst_encoder || radeon_encoder->offset)
-		return ATOM_ENCODER_MODE_DP_MST;
 	/* dp bridges are always DP */
 	if (radeon_encoder_get_dp_bridge_encoder_id(encoder) != ENCODER_OBJECT_ID_NONE)
 		return ATOM_ENCODER_MODE_DP;
@@ -1723,10 +1722,6 @@ radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode)
 	case DRM_MODE_DPMS_SUSPEND:
 	case DRM_MODE_DPMS_OFF:
 
-		/* don't power off encoders with active MST links */
-		if (dig->active_mst_links)
-			return;
-
 		if (ASIC_IS_DCE4(rdev)) {
 			if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && connector)
 				atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_OFF, 0);
@@ -1992,53 +1987,6 @@ atombios_set_encoder_crtc_source(struct drm_encoder *encoder)
 	radeon_atombios_encoder_crtc_scratch_regs(encoder, radeon_crtc->crtc_id);
 }
 
-void
-atombios_set_mst_encoder_crtc_source(struct drm_encoder *encoder, int fe)
-{
-	struct drm_device *dev = encoder->dev;
-	struct radeon_device *rdev = dev->dev_private;
-	struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
-	int index = GetIndexIntoMasterTable(COMMAND, SelectCRTC_Source);
-	uint8_t frev, crev;
-	union crtc_source_param args;
-
-	memset(&args, 0, sizeof(args));
-
-	if (!atom_parse_cmd_header(rdev->mode_info.atom_context, index, &frev, &crev))
-		return;
-
-	if (frev != 1 && crev != 2)
-		DRM_ERROR("Unknown table for MST %d, %d\n", frev, crev);
-
-	args.v2.ucCRTC = radeon_crtc->crtc_id;
-	args.v2.ucEncodeMode = ATOM_ENCODER_MODE_DP_MST;
-
-	switch (fe) {
-	case 0:
-		args.v2.ucEncoderID = ASIC_INT_DIG1_ENCODER_ID;
-		break;
-	case 1:
-		args.v2.ucEncoderID = ASIC_INT_DIG2_ENCODER_ID;
-		break;
-	case 2:
-		args.v2.ucEncoderID = ASIC_INT_DIG3_ENCODER_ID;
-		break;
-	case 3:
-		args.v2.ucEncoderID = ASIC_INT_DIG4_ENCODER_ID;
-		break;
-	case 4:
-		args.v2.ucEncoderID = ASIC_INT_DIG5_ENCODER_ID;
-		break;
-	case 5:
-		args.v2.ucEncoderID = ASIC_INT_DIG6_ENCODER_ID;
-		break;
-	case 6:
-		args.v2.ucEncoderID = ASIC_INT_DIG7_ENCODER_ID;
-		break;
-	}
-	atom_execute_table(rdev->mode_info.atom_context, index, (uint32_t *)&args);
-}
-
 static void
 atombios_apply_encoder_quirks(struct drm_encoder *encoder,
 			      struct drm_display_mode *mode)
diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c
index ac006bed4743..8ef25ab305ae 100644
--- a/drivers/gpu/drm/radeon/ci_dpm.c
+++ b/drivers/gpu/drm/radeon/ci_dpm.c
@@ -2056,7 +2056,7 @@ static void ci_clear_vc(struct radeon_device *rdev)
 static int ci_upload_firmware(struct radeon_device *rdev)
 {
 	struct ci_power_info *pi = ci_get_pi(rdev);
-	int i, ret;
+	int i;
 
 	for (i = 0; i < rdev->usec_timeout; i++) {
 		if (RREG32_SMC(RCU_UC_EVENTS) & BOOT_SEQ_DONE)
@@ -2067,9 +2067,7 @@ static int ci_upload_firmware(struct radeon_device *rdev)
 	ci_stop_smc_clock(rdev);
 	ci_reset_smc(rdev);
 
-	ret = ci_load_smc_ucode(rdev, pi->sram_end);
-
-	return ret;
+	return ci_load_smc_ucode(rdev, pi->sram_end);
 
 }
 
diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c
index 8be4799a98ef..638f861af80f 100644
--- a/drivers/gpu/drm/radeon/r600_cs.c
+++ b/drivers/gpu/drm/radeon/r600_cs.c
@@ -34,8 +34,6 @@
 #include "r600_reg_safe.h"
 
 static int r600_nomm;
-extern void r600_cs_legacy_get_tiling_conf(struct drm_device *dev, u32 *npipes, u32 *nbanks, u32 *group_size);
-
 
 struct r600_cs_track {
 	/* configuration we mirror so that we use same code btw kms/ums */
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 08f83bf2c330..166c18d62f6d 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -116,7 +116,6 @@ extern int radeon_use_pflipirq;
 extern int radeon_bapm;
 extern int radeon_backlight;
 extern int radeon_auxch;
-extern int radeon_mst;
 extern int radeon_uvd;
 extern int radeon_vce;
 extern int radeon_si_support;
@@ -2950,8 +2949,6 @@ struct radeon_hdmi_acr {
 
 };
 
-extern struct radeon_hdmi_acr r600_hdmi_acr(uint32_t clock);
-
 extern u32 r6xx_remap_render_backend(struct radeon_device *rdev,
 				     u32 tiling_pipe_num,
 				     u32 max_rb_num,
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index 28c4413f4dc8..204127bad89c 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -826,8 +826,6 @@ bool radeon_get_atom_connector_info_from_object_table(struct drm_device *dev)
 	}
 
 	radeon_link_encoder_connector(dev);
-
-	radeon_setup_mst_connector(dev);
 	return true;
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 58db79921cd3..f7431d224604 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -37,33 +37,12 @@
 #include <linux/pm_runtime.h>
 #include <linux/vga_switcheroo.h>
 
-static int radeon_dp_handle_hpd(struct drm_connector *connector)
-{
-	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
-	int ret;
-
-	ret = radeon_dp_mst_check_status(radeon_connector);
-	if (ret == -EINVAL)
-		return 1;
-	return 0;
-}
 void radeon_connector_hotplug(struct drm_connector *connector)
 {
 	struct drm_device *dev = connector->dev;
 	struct radeon_device *rdev = dev->dev_private;
 	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
 
-	if (connector->connector_type == DRM_MODE_CONNECTOR_DisplayPort) {
-		struct radeon_connector_atom_dig *dig_connector =
-			radeon_connector->con_priv;
-
-		if (radeon_connector->is_mst_connector)
-			return;
-		if (dig_connector->is_mst) {
-			radeon_dp_handle_hpd(connector);
-			return;
-		}
-	}
 	/* bail if the connector does not have hpd pin, e.g.,
 	 * VGA, TV, etc.
 	 */
@@ -1664,9 +1643,6 @@ radeon_dp_detect(struct drm_connector *connector, bool force)
 	struct drm_encoder *encoder = radeon_best_single_encoder(connector);
 	int r;
 
-	if (radeon_dig_connector->is_mst)
-		return connector_status_disconnected;
-
 	if (!drm_kms_helper_is_poll_worker()) {
 		r = pm_runtime_get_sync(connector->dev->dev);
 		if (r < 0) {
@@ -1729,21 +1705,12 @@ radeon_dp_detect(struct drm_connector *connector, bool force)
 		radeon_dig_connector->dp_sink_type = radeon_dp_getsinktype(radeon_connector);
 		if (radeon_hpd_sense(rdev, radeon_connector->hpd.hpd)) {
 			ret = connector_status_connected;
-			if (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) {
+			if (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT)
 				radeon_dp_getdpcd(radeon_connector);
-				r = radeon_dp_mst_probe(radeon_connector);
-				if (r == 1)
-					ret = connector_status_disconnected;
-			}
 		} else {
 			if (radeon_dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) {
-				if (radeon_dp_getdpcd(radeon_connector)) {
-					r = radeon_dp_mst_probe(radeon_connector);
-					if (r == 1)
-						ret = connector_status_disconnected;
-					else
-						ret = connector_status_connected;
-				}
+				if (radeon_dp_getdpcd(radeon_connector))
+					ret = connector_status_connected;
 			} else {
 				/* try non-aux ddc (DP to DVI/HDMI/etc. adapter) */
 				if (radeon_ddc_probe(radeon_connector, false))
@@ -2561,25 +2528,3 @@ radeon_add_legacy_connector(struct drm_device *dev,
 	connector->display_info.subpixel_order = subpixel_order;
 	drm_connector_register(connector);
 }
-
-void radeon_setup_mst_connector(struct drm_device *dev)
-{
-	struct radeon_device *rdev = dev->dev_private;
-	struct drm_connector *connector;
-	struct radeon_connector *radeon_connector;
-
-	if (!ASIC_IS_DCE5(rdev))
-		return;
-
-	if (radeon_mst == 0)
-		return;
-
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		radeon_connector = to_radeon_connector(connector);
-
-		if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort)
-			continue;
-
-		radeon_dp_mst_init(radeon_connector);
-	}
-}
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index ee0165687239..a556b6be1137 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -1438,7 +1438,6 @@ int radeon_device_init(struct radeon_device *rdev,
 		goto failed;
 
 	radeon_gem_debugfs_init(rdev);
-	radeon_mst_debugfs_init(rdev);
 
 	if (rdev->flags & RADEON_IS_AGP && !rdev->accel_working) {
 		/* Acceleration not working on AGP card try again
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index f12675e3d261..ca5598ae8bfc 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -38,7 +38,6 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_framebuffer_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 #include <drm/radeon_drm.h>
diff --git a/drivers/gpu/drm/radeon/radeon_dp_mst.c b/drivers/gpu/drm/radeon/radeon_dp_mst.c
deleted file mode 100644
index 54ced1f4ff67..000000000000
--- a/drivers/gpu/drm/radeon/radeon_dp_mst.c
+++ /dev/null
@@ -1,778 +0,0 @@
-// SPDX-License-Identifier: MIT
-
-#include <drm/display/drm_dp_mst_helper.h>
-#include <drm/drm_fb_helper.h>
-#include <drm/drm_file.h>
-#include <drm/drm_probe_helper.h>
-
-#include "atom.h"
-#include "ni_reg.h"
-#include "radeon.h"
-
-static struct radeon_encoder *radeon_dp_create_fake_mst_encoder(struct radeon_connector *connector);
-
-static int radeon_atom_set_enc_offset(int id)
-{
-	static const int offsets[] = { EVERGREEN_CRTC0_REGISTER_OFFSET,
-				       EVERGREEN_CRTC1_REGISTER_OFFSET,
-				       EVERGREEN_CRTC2_REGISTER_OFFSET,
-				       EVERGREEN_CRTC3_REGISTER_OFFSET,
-				       EVERGREEN_CRTC4_REGISTER_OFFSET,
-				       EVERGREEN_CRTC5_REGISTER_OFFSET,
-				       0x13830 - 0x7030 };
-
-	return offsets[id];
-}
-
-static int radeon_dp_mst_set_be_cntl(struct radeon_encoder *primary,
-				     struct radeon_encoder_mst *mst_enc,
-				     enum radeon_hpd_id hpd, bool enable)
-{
-	struct drm_device *dev = primary->base.dev;
-	struct radeon_device *rdev = dev->dev_private;
-	uint32_t reg;
-	int retries = 0;
-	uint32_t temp;
-
-	reg = RREG32(NI_DIG_BE_CNTL + primary->offset);
-
-	/* set MST mode */
-	reg &= ~NI_DIG_FE_DIG_MODE(7);
-	reg |= NI_DIG_FE_DIG_MODE(NI_DIG_MODE_DP_MST);
-
-	if (enable)
-		reg |= NI_DIG_FE_SOURCE_SELECT(1 << mst_enc->fe);
-	else
-		reg &= ~NI_DIG_FE_SOURCE_SELECT(1 << mst_enc->fe);
-
-	reg |= NI_DIG_HPD_SELECT(hpd);
-	DRM_DEBUG_KMS("writing 0x%08x 0x%08x\n", NI_DIG_BE_CNTL + primary->offset, reg);
-	WREG32(NI_DIG_BE_CNTL + primary->offset, reg);
-
-	if (enable) {
-		uint32_t offset = radeon_atom_set_enc_offset(mst_enc->fe);
-
-		do {
-			temp = RREG32(NI_DIG_FE_CNTL + offset);
-		} while ((temp & NI_DIG_SYMCLK_FE_ON) && retries++ < 10000);
-		if (retries == 10000)
-			DRM_ERROR("timed out waiting for FE %d %d\n", primary->offset, mst_enc->fe);
-	}
-	return 0;
-}
-
-static int radeon_dp_mst_set_stream_attrib(struct radeon_encoder *primary,
-					   int stream_number,
-					   int fe,
-					   int slots)
-{
-	struct drm_device *dev = primary->base.dev;
-	struct radeon_device *rdev = dev->dev_private;
-	u32 temp, val;
-	int retries  = 0;
-	int satreg, satidx;
-
-	satreg = stream_number >> 1;
-	satidx = stream_number & 1;
-
-	temp = RREG32(NI_DP_MSE_SAT0 + satreg + primary->offset);
-
-	val = NI_DP_MSE_SAT_SLOT_COUNT0(slots) | NI_DP_MSE_SAT_SRC0(fe);
-
-	val <<= (16 * satidx);
-
-	temp &= ~(0xffff << (16 * satidx));
-
-	temp |= val;
-
-	DRM_DEBUG_KMS("writing 0x%08x 0x%08x\n", NI_DP_MSE_SAT0 + satreg + primary->offset, temp);
-	WREG32(NI_DP_MSE_SAT0 + satreg + primary->offset, temp);
-
-	WREG32(NI_DP_MSE_SAT_UPDATE + primary->offset, 1);
-
-	do {
-		unsigned value1, value2;
-		udelay(10);
-		temp = RREG32(NI_DP_MSE_SAT_UPDATE + primary->offset);
-
-		value1 = temp & NI_DP_MSE_SAT_UPDATE_MASK;
-		value2 = temp & NI_DP_MSE_16_MTP_KEEPOUT;
-
-		if (!value1 && !value2)
-			break;
-	} while (retries++ < 50);
-
-	if (retries == 10000)
-		DRM_ERROR("timed out waitin for SAT update %d\n", primary->offset);
-
-	/* MTP 16 ? */
-	return 0;
-}
-
-static int radeon_dp_mst_update_stream_attribs(struct radeon_connector *mst_conn,
-					       struct radeon_encoder *primary)
-{
-	struct drm_device *dev = mst_conn->base.dev;
-	struct stream_attribs new_attribs[6];
-	int i;
-	int idx = 0;
-	struct radeon_connector *radeon_connector;
-	struct drm_connector *connector;
-
-	memset(new_attribs, 0, sizeof(new_attribs));
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		struct radeon_encoder *subenc;
-		struct radeon_encoder_mst *mst_enc;
-
-		radeon_connector = to_radeon_connector(connector);
-		if (!radeon_connector->is_mst_connector)
-			continue;
-
-		if (radeon_connector->mst_port != mst_conn)
-			continue;
-
-		subenc = radeon_connector->mst_encoder;
-		mst_enc = subenc->enc_priv;
-
-		if (!mst_enc->enc_active)
-			continue;
-
-		new_attribs[idx].fe = mst_enc->fe;
-		new_attribs[idx].slots = drm_dp_mst_get_vcpi_slots(&mst_conn->mst_mgr, mst_enc->port);
-		idx++;
-	}
-
-	for (i = 0; i < idx; i++) {
-		if (new_attribs[i].fe != mst_conn->cur_stream_attribs[i].fe ||
-		    new_attribs[i].slots != mst_conn->cur_stream_attribs[i].slots) {
-			radeon_dp_mst_set_stream_attrib(primary, i, new_attribs[i].fe, new_attribs[i].slots);
-			mst_conn->cur_stream_attribs[i].fe = new_attribs[i].fe;
-			mst_conn->cur_stream_attribs[i].slots = new_attribs[i].slots;
-		}
-	}
-
-	for (i = idx; i < mst_conn->enabled_attribs; i++) {
-		radeon_dp_mst_set_stream_attrib(primary, i, 0, 0);
-		mst_conn->cur_stream_attribs[i].fe = 0;
-		mst_conn->cur_stream_attribs[i].slots = 0;
-	}
-	mst_conn->enabled_attribs = idx;
-	return 0;
-}
-
-static int radeon_dp_mst_set_vcp_size(struct radeon_encoder *mst, s64 avg_time_slots_per_mtp)
-{
-	struct drm_device *dev = mst->base.dev;
-	struct radeon_device *rdev = dev->dev_private;
-	struct radeon_encoder_mst *mst_enc = mst->enc_priv;
-	uint32_t val, temp;
-	uint32_t offset = radeon_atom_set_enc_offset(mst_enc->fe);
-	int retries = 0;
-	uint32_t x = drm_fixp2int(avg_time_slots_per_mtp);
-	uint32_t y = drm_fixp2int_ceil((avg_time_slots_per_mtp - x) << 26);
-
-	val = NI_DP_MSE_RATE_X(x) | NI_DP_MSE_RATE_Y(y);
-
-	WREG32(NI_DP_MSE_RATE_CNTL + offset, val);
-
-	do {
-		temp = RREG32(NI_DP_MSE_RATE_UPDATE + offset);
-		udelay(10);
-	} while ((temp & 0x1) && (retries++ < 10000));
-
-	if (retries >= 10000)
-		DRM_ERROR("timed out wait for rate cntl %d\n", mst_enc->fe);
-	return 0;
-}
-
-static int radeon_dp_mst_get_ddc_modes(struct drm_connector *connector)
-{
-	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
-	struct radeon_connector *master = radeon_connector->mst_port;
-	struct edid *edid;
-	int ret = 0;
-
-	edid = drm_dp_mst_get_edid(connector, &master->mst_mgr, radeon_connector->port);
-	radeon_connector->edid = edid;
-	DRM_DEBUG_KMS("edid retrieved %p\n", edid);
-	if (radeon_connector->edid) {
-		drm_connector_update_edid_property(&radeon_connector->base, radeon_connector->edid);
-		ret = drm_add_edid_modes(&radeon_connector->base, radeon_connector->edid);
-		return ret;
-	}
-	drm_connector_update_edid_property(&radeon_connector->base, NULL);
-
-	return ret;
-}
-
-static int radeon_dp_mst_get_modes(struct drm_connector *connector)
-{
-	return radeon_dp_mst_get_ddc_modes(connector);
-}
-
-static enum drm_mode_status
-radeon_dp_mst_mode_valid(struct drm_connector *connector,
-			struct drm_display_mode *mode)
-{
-	/* TODO - validate mode against available PBN for link */
-	if (mode->clock < 10000)
-		return MODE_CLOCK_LOW;
-
-	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
-		return MODE_H_ILLEGAL;
-
-	return MODE_OK;
-}
-
-static struct
-drm_encoder *radeon_mst_best_encoder(struct drm_connector *connector)
-{
-	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
-
-	return &radeon_connector->mst_encoder->base;
-}
-
-static int
-radeon_dp_mst_detect(struct drm_connector *connector,
-		     struct drm_modeset_acquire_ctx *ctx,
-		     bool force)
-{
-	struct radeon_connector *radeon_connector =
-		to_radeon_connector(connector);
-	struct radeon_connector *master = radeon_connector->mst_port;
-
-	if (drm_connector_is_unregistered(connector))
-		return connector_status_disconnected;
-
-	return drm_dp_mst_detect_port(connector, ctx, &master->mst_mgr,
-				      radeon_connector->port);
-}
-
-static const struct drm_connector_helper_funcs radeon_dp_mst_connector_helper_funcs = {
-	.get_modes = radeon_dp_mst_get_modes,
-	.mode_valid = radeon_dp_mst_mode_valid,
-	.best_encoder = radeon_mst_best_encoder,
-	.detect_ctx = radeon_dp_mst_detect,
-};
-
-static void
-radeon_dp_mst_connector_destroy(struct drm_connector *connector)
-{
-	struct radeon_connector *radeon_connector = to_radeon_connector(connector);
-	struct radeon_encoder *radeon_encoder = radeon_connector->mst_encoder;
-
-	drm_encoder_cleanup(&radeon_encoder->base);
-	kfree(radeon_encoder);
-	drm_connector_cleanup(connector);
-	kfree(radeon_connector);
-}
-
-static const struct drm_connector_funcs radeon_dp_mst_connector_funcs = {
-	.dpms = drm_helper_connector_dpms,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = radeon_dp_mst_connector_destroy,
-};
-
-static struct drm_connector *radeon_dp_add_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
-							 struct drm_dp_mst_port *port,
-							 const char *pathprop)
-{
-	struct radeon_connector *master = container_of(mgr, struct radeon_connector, mst_mgr);
-	struct drm_device *dev = master->base.dev;
-	struct radeon_connector *radeon_connector;
-	struct drm_connector *connector;
-
-	radeon_connector = kzalloc(sizeof(*radeon_connector), GFP_KERNEL);
-	if (!radeon_connector)
-		return NULL;
-
-	radeon_connector->is_mst_connector = true;
-	connector = &radeon_connector->base;
-	radeon_connector->port = port;
-	radeon_connector->mst_port = master;
-	DRM_DEBUG_KMS("\n");
-
-	drm_connector_init(dev, connector, &radeon_dp_mst_connector_funcs, DRM_MODE_CONNECTOR_DisplayPort);
-	drm_connector_helper_add(connector, &radeon_dp_mst_connector_helper_funcs);
-	radeon_connector->mst_encoder = radeon_dp_create_fake_mst_encoder(master);
-
-	drm_object_attach_property(&connector->base, dev->mode_config.path_property, 0);
-	drm_object_attach_property(&connector->base, dev->mode_config.tile_property, 0);
-	drm_connector_set_path_property(connector, pathprop);
-
-	return connector;
-}
-
-static const struct drm_dp_mst_topology_cbs mst_cbs = {
-	.add_connector = radeon_dp_add_mst_connector,
-};
-
-static struct
-radeon_connector *radeon_mst_find_connector(struct drm_encoder *encoder)
-{
-	struct drm_device *dev = encoder->dev;
-	struct drm_connector *connector;
-
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		struct radeon_connector *radeon_connector = to_radeon_connector(connector);
-		if (!connector->encoder)
-			continue;
-		if (!radeon_connector->is_mst_connector)
-			continue;
-
-		DRM_DEBUG_KMS("checking %p vs %p\n", connector->encoder, encoder);
-		if (connector->encoder == encoder)
-			return radeon_connector;
-	}
-	return NULL;
-}
-
-void radeon_dp_mst_prepare_pll(struct drm_crtc *crtc, struct drm_display_mode *mode)
-{
-	struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
-	struct drm_device *dev = crtc->dev;
-	struct radeon_device *rdev = dev->dev_private;
-	struct radeon_encoder *radeon_encoder = to_radeon_encoder(radeon_crtc->encoder);
-	struct radeon_encoder_mst *mst_enc = radeon_encoder->enc_priv;
-	struct radeon_connector *radeon_connector = radeon_mst_find_connector(&radeon_encoder->base);
-	int dp_clock;
-	struct radeon_connector_atom_dig *dig_connector = mst_enc->connector->con_priv;
-
-	if (radeon_connector) {
-		radeon_connector->pixelclock_for_modeset = mode->clock;
-		if (radeon_connector->base.display_info.bpc)
-			radeon_crtc->bpc = radeon_connector->base.display_info.bpc;
-		else
-			radeon_crtc->bpc = 8;
-	}
-
-	DRM_DEBUG_KMS("dp_clock %p %d\n", dig_connector, dig_connector->dp_clock);
-	dp_clock = dig_connector->dp_clock;
-	radeon_crtc->ss_enabled =
-		radeon_atombios_get_asic_ss_info(rdev, &radeon_crtc->ss,
-						 ASIC_INTERNAL_SS_ON_DP,
-						 dp_clock);
-}
-
-static void
-radeon_mst_encoder_dpms(struct drm_encoder *encoder, int mode)
-{
-	struct drm_device *dev = encoder->dev;
-	struct radeon_device *rdev = dev->dev_private;
-	struct radeon_encoder *radeon_encoder, *primary;
-	struct radeon_encoder_mst *mst_enc;
-	struct radeon_encoder_atom_dig *dig_enc;
-	struct radeon_connector *radeon_connector;
-	struct drm_crtc *crtc;
-	struct radeon_crtc *radeon_crtc;
-	int slots;
-	s64 fixed_pbn, fixed_pbn_per_slot, avg_time_slots_per_mtp;
-	if (!ASIC_IS_DCE5(rdev)) {
-		DRM_ERROR("got mst dpms on non-DCE5\n");
-		return;
-	}
-
-	radeon_connector = radeon_mst_find_connector(encoder);
-	if (!radeon_connector)
-		return;
-
-	radeon_encoder = to_radeon_encoder(encoder);
-
-	mst_enc = radeon_encoder->enc_priv;
-
-	primary = mst_enc->primary;
-
-	dig_enc = primary->enc_priv;
-
-	crtc = encoder->crtc;
-	DRM_DEBUG_KMS("got connector %d\n", dig_enc->active_mst_links);
-
-	switch (mode) {
-	case DRM_MODE_DPMS_ON:
-		dig_enc->active_mst_links++;
-
-		radeon_crtc = to_radeon_crtc(crtc);
-
-		if (dig_enc->active_mst_links == 1) {
-			mst_enc->fe = dig_enc->dig_encoder;
-			mst_enc->fe_from_be = true;
-			atombios_set_mst_encoder_crtc_source(encoder, mst_enc->fe);
-
-			atombios_dig_encoder_setup(&primary->base, ATOM_ENCODER_CMD_SETUP, 0);
-			atombios_dig_transmitter_setup2(&primary->base, ATOM_TRANSMITTER_ACTION_ENABLE,
-							0, 0, dig_enc->dig_encoder);
-
-			if (radeon_dp_needs_link_train(mst_enc->connector) ||
-			    dig_enc->active_mst_links == 1) {
-				radeon_dp_link_train(&primary->base, &mst_enc->connector->base);
-			}
-
-		} else {
-			mst_enc->fe = radeon_atom_pick_dig_encoder(encoder, radeon_crtc->crtc_id);
-			if (mst_enc->fe == -1)
-				DRM_ERROR("failed to get frontend for dig encoder\n");
-			mst_enc->fe_from_be = false;
-			atombios_set_mst_encoder_crtc_source(encoder, mst_enc->fe);
-		}
-
-		DRM_DEBUG_KMS("dig encoder is %d %d %d\n", dig_enc->dig_encoder,
-			      dig_enc->linkb, radeon_crtc->crtc_id);
-
-		slots = drm_dp_find_vcpi_slots(&radeon_connector->mst_port->mst_mgr,
-					       mst_enc->pbn);
-		drm_dp_mst_allocate_vcpi(&radeon_connector->mst_port->mst_mgr,
-					 radeon_connector->port,
-					 mst_enc->pbn, slots);
-		drm_dp_update_payload_part1(&radeon_connector->mst_port->mst_mgr, 1);
-
-		radeon_dp_mst_set_be_cntl(primary, mst_enc,
-					  radeon_connector->mst_port->hpd.hpd, true);
-
-		mst_enc->enc_active = true;
-		radeon_dp_mst_update_stream_attribs(radeon_connector->mst_port, primary);
-
-		fixed_pbn = drm_int2fixp(mst_enc->pbn);
-		fixed_pbn_per_slot = drm_int2fixp(radeon_connector->mst_port->mst_mgr.pbn_div);
-		avg_time_slots_per_mtp = drm_fixp_div(fixed_pbn, fixed_pbn_per_slot);
-		radeon_dp_mst_set_vcp_size(radeon_encoder, avg_time_slots_per_mtp);
-
-		atombios_dig_encoder_setup2(&primary->base, ATOM_ENCODER_CMD_DP_VIDEO_ON, 0,
-					    mst_enc->fe);
-		drm_dp_check_act_status(&radeon_connector->mst_port->mst_mgr);
-
-		drm_dp_update_payload_part2(&radeon_connector->mst_port->mst_mgr);
-
-		break;
-	case DRM_MODE_DPMS_STANDBY:
-	case DRM_MODE_DPMS_SUSPEND:
-	case DRM_MODE_DPMS_OFF:
-		DRM_ERROR("DPMS OFF %d\n", dig_enc->active_mst_links);
-
-		if (!mst_enc->enc_active)
-			return;
-
-		drm_dp_mst_reset_vcpi_slots(&radeon_connector->mst_port->mst_mgr, mst_enc->port);
-		drm_dp_update_payload_part1(&radeon_connector->mst_port->mst_mgr, 1);
-
-		drm_dp_check_act_status(&radeon_connector->mst_port->mst_mgr);
-		/* and this can also fail */
-		drm_dp_update_payload_part2(&radeon_connector->mst_port->mst_mgr);
-
-		drm_dp_mst_deallocate_vcpi(&radeon_connector->mst_port->mst_mgr, mst_enc->port);
-
-		mst_enc->enc_active = false;
-		radeon_dp_mst_update_stream_attribs(radeon_connector->mst_port, primary);
-
-		radeon_dp_mst_set_be_cntl(primary, mst_enc,
-					  radeon_connector->mst_port->hpd.hpd, false);
-		atombios_dig_encoder_setup2(&primary->base, ATOM_ENCODER_CMD_DP_VIDEO_OFF, 0,
-					    mst_enc->fe);
-
-		if (!mst_enc->fe_from_be)
-			radeon_atom_release_dig_encoder(rdev, mst_enc->fe);
-
-		mst_enc->fe_from_be = false;
-		dig_enc->active_mst_links--;
-		if (dig_enc->active_mst_links == 0) {
-			/* drop link */
-		}
-
-		break;
-	}
-
-}
-
-static bool radeon_mst_mode_fixup(struct drm_encoder *encoder,
-				   const struct drm_display_mode *mode,
-				   struct drm_display_mode *adjusted_mode)
-{
-	struct radeon_encoder_mst *mst_enc;
-	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
-	struct radeon_connector_atom_dig *dig_connector;
-	int bpp = 24;
-
-	mst_enc = radeon_encoder->enc_priv;
-
-	mst_enc->pbn = drm_dp_calc_pbn_mode(adjusted_mode->clock, bpp, false);
-
-	mst_enc->primary->active_device = mst_enc->primary->devices & mst_enc->connector->devices;
-	DRM_DEBUG_KMS("setting active device to %08x from %08x %08x for encoder %d\n",
-		      mst_enc->primary->active_device, mst_enc->primary->devices,
-		      mst_enc->connector->devices, mst_enc->primary->base.encoder_type);
-
-
-	drm_mode_set_crtcinfo(adjusted_mode, 0);
-	dig_connector = mst_enc->connector->con_priv;
-	dig_connector->dp_lane_count = drm_dp_max_lane_count(dig_connector->dpcd);
-	dig_connector->dp_clock = drm_dp_max_link_rate(dig_connector->dpcd);
-	DRM_DEBUG_KMS("dig clock %p %d %d\n", dig_connector,
-		      dig_connector->dp_lane_count, dig_connector->dp_clock);
-	return true;
-}
-
-static void radeon_mst_encoder_prepare(struct drm_encoder *encoder)
-{
-	struct radeon_connector *radeon_connector;
-	struct radeon_encoder *radeon_encoder, *primary;
-	struct radeon_encoder_mst *mst_enc;
-	struct radeon_encoder_atom_dig *dig_enc;
-
-	radeon_connector = radeon_mst_find_connector(encoder);
-	if (!radeon_connector) {
-		DRM_DEBUG_KMS("failed to find connector %p\n", encoder);
-		return;
-	}
-	radeon_encoder = to_radeon_encoder(encoder);
-
-	radeon_mst_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
-
-	mst_enc = radeon_encoder->enc_priv;
-
-	primary = mst_enc->primary;
-
-	dig_enc = primary->enc_priv;
-
-	mst_enc->port = radeon_connector->port;
-
-	if (dig_enc->dig_encoder == -1) {
-		dig_enc->dig_encoder = radeon_atom_pick_dig_encoder(&primary->base, -1);
-		primary->offset = radeon_atom_set_enc_offset(dig_enc->dig_encoder);
-		atombios_set_mst_encoder_crtc_source(encoder, dig_enc->dig_encoder);
-
-
-	}
-	DRM_DEBUG_KMS("%d %d\n", dig_enc->dig_encoder, primary->offset);
-}
-
-static void
-radeon_mst_encoder_mode_set(struct drm_encoder *encoder,
-			     struct drm_display_mode *mode,
-			     struct drm_display_mode *adjusted_mode)
-{
-	DRM_DEBUG_KMS("\n");
-}
-
-static void radeon_mst_encoder_commit(struct drm_encoder *encoder)
-{
-	radeon_mst_encoder_dpms(encoder, DRM_MODE_DPMS_ON);
-	DRM_DEBUG_KMS("\n");
-}
-
-static const struct drm_encoder_helper_funcs radeon_mst_helper_funcs = {
-	.dpms = radeon_mst_encoder_dpms,
-	.mode_fixup = radeon_mst_mode_fixup,
-	.prepare = radeon_mst_encoder_prepare,
-	.mode_set = radeon_mst_encoder_mode_set,
-	.commit = radeon_mst_encoder_commit,
-};
-
-static void radeon_dp_mst_encoder_destroy(struct drm_encoder *encoder)
-{
-	drm_encoder_cleanup(encoder);
-	kfree(encoder);
-}
-
-static const struct drm_encoder_funcs radeon_dp_mst_enc_funcs = {
-	.destroy = radeon_dp_mst_encoder_destroy,
-};
-
-static struct radeon_encoder *
-radeon_dp_create_fake_mst_encoder(struct radeon_connector *connector)
-{
-	struct drm_device *dev = connector->base.dev;
-	struct radeon_device *rdev = dev->dev_private;
-	struct radeon_encoder *radeon_encoder;
-	struct radeon_encoder_mst *mst_enc;
-	struct drm_encoder *encoder;
-	const struct drm_connector_helper_funcs *connector_funcs = connector->base.helper_private;
-	struct drm_encoder *enc_master = connector_funcs->best_encoder(&connector->base);
-
-	DRM_DEBUG_KMS("enc master is %p\n", enc_master);
-	radeon_encoder = kzalloc(sizeof(*radeon_encoder), GFP_KERNEL);
-	if (!radeon_encoder)
-		return NULL;
-
-	radeon_encoder->enc_priv = kzalloc(sizeof(*mst_enc), GFP_KERNEL);
-	if (!radeon_encoder->enc_priv) {
-		kfree(radeon_encoder);
-		return NULL;
-	}
-	encoder = &radeon_encoder->base;
-	switch (rdev->num_crtc) {
-	case 1:
-		encoder->possible_crtcs = 0x1;
-		break;
-	case 2:
-	default:
-		encoder->possible_crtcs = 0x3;
-		break;
-	case 4:
-		encoder->possible_crtcs = 0xf;
-		break;
-	case 6:
-		encoder->possible_crtcs = 0x3f;
-		break;
-	}
-
-	drm_encoder_init(dev, &radeon_encoder->base, &radeon_dp_mst_enc_funcs,
-			 DRM_MODE_ENCODER_DPMST, NULL);
-	drm_encoder_helper_add(encoder, &radeon_mst_helper_funcs);
-
-	mst_enc = radeon_encoder->enc_priv;
-	mst_enc->connector = connector;
-	mst_enc->primary = to_radeon_encoder(enc_master);
-	radeon_encoder->is_mst_encoder = true;
-	return radeon_encoder;
-}
-
-int
-radeon_dp_mst_init(struct radeon_connector *radeon_connector)
-{
-	struct drm_device *dev = radeon_connector->base.dev;
-	int max_link_rate;
-
-	if (!radeon_connector->ddc_bus->has_aux)
-		return 0;
-
-	if (radeon_connector_is_dp12_capable(&radeon_connector->base))
-		max_link_rate = 0x14;
-	else
-		max_link_rate = 0x0a;
-
-	radeon_connector->mst_mgr.cbs = &mst_cbs;
-	return drm_dp_mst_topology_mgr_init(&radeon_connector->mst_mgr, dev,
-					    &radeon_connector->ddc_bus->aux, 16, 6,
-					    4, drm_dp_bw_code_to_link_rate(max_link_rate),
-					    radeon_connector->base.base.id);
-}
-
-int
-radeon_dp_mst_probe(struct radeon_connector *radeon_connector)
-{
-	struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
-	struct drm_device *dev = radeon_connector->base.dev;
-	struct radeon_device *rdev = dev->dev_private;
-	int ret;
-	u8 msg[1];
-
-	if (!radeon_mst)
-		return 0;
-
-	if (!ASIC_IS_DCE5(rdev))
-		return 0;
-
-	if (dig_connector->dpcd[DP_DPCD_REV] < 0x12)
-		return 0;
-
-	ret = drm_dp_dpcd_read(&radeon_connector->ddc_bus->aux, DP_MSTM_CAP, msg,
-			       1);
-	if (ret) {
-		if (msg[0] & DP_MST_CAP) {
-			DRM_DEBUG_KMS("Sink is MST capable\n");
-			dig_connector->is_mst = true;
-		} else {
-			DRM_DEBUG_KMS("Sink is not MST capable\n");
-			dig_connector->is_mst = false;
-		}
-
-	}
-	drm_dp_mst_topology_mgr_set_mst(&radeon_connector->mst_mgr,
-					dig_connector->is_mst);
-	return dig_connector->is_mst;
-}
-
-int
-radeon_dp_mst_check_status(struct radeon_connector *radeon_connector)
-{
-	struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
-	int retry;
-
-	if (dig_connector->is_mst) {
-		u8 esi[16] = { 0 };
-		int dret;
-		int ret = 0;
-		bool handled;
-
-		dret = drm_dp_dpcd_read(&radeon_connector->ddc_bus->aux,
-				       DP_SINK_COUNT_ESI, esi, 8);
-go_again:
-		if (dret == 8) {
-			DRM_DEBUG_KMS("got esi %3ph\n", esi);
-			ret = drm_dp_mst_hpd_irq(&radeon_connector->mst_mgr, esi, &handled);
-
-			if (handled) {
-				for (retry = 0; retry < 3; retry++) {
-					int wret;
-					wret = drm_dp_dpcd_write(&radeon_connector->ddc_bus->aux,
-								 DP_SINK_COUNT_ESI + 1, &esi[1], 3);
-					if (wret == 3)
-						break;
-				}
-
-				dret = drm_dp_dpcd_read(&radeon_connector->ddc_bus->aux,
-							DP_SINK_COUNT_ESI, esi, 8);
-				if (dret == 8) {
-					DRM_DEBUG_KMS("got esi2 %3ph\n", esi);
-					goto go_again;
-				}
-			} else
-				ret = 0;
-
-			return ret;
-		} else {
-			DRM_DEBUG_KMS("failed to get ESI - device may have failed %d\n", ret);
-			dig_connector->is_mst = false;
-			drm_dp_mst_topology_mgr_set_mst(&radeon_connector->mst_mgr,
-							dig_connector->is_mst);
-			/* send a hotplug event */
-		}
-	}
-	return -EINVAL;
-}
-
-#if defined(CONFIG_DEBUG_FS)
-
-static int radeon_debugfs_mst_info_show(struct seq_file *m, void *unused)
-{
-	struct radeon_device *rdev = (struct radeon_device *)m->private;
-	struct drm_device *dev = rdev->ddev;
-	struct drm_connector *connector;
-	struct radeon_connector *radeon_connector;
-	struct radeon_connector_atom_dig *dig_connector;
-	int i;
-
-	drm_modeset_lock_all(dev);
-	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-		if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort)
-			continue;
-
-		radeon_connector = to_radeon_connector(connector);
-		dig_connector = radeon_connector->con_priv;
-		if (radeon_connector->is_mst_connector)
-			continue;
-		if (!dig_connector->is_mst)
-			continue;
-		drm_dp_mst_dump_topology(m, &radeon_connector->mst_mgr);
-
-		for (i = 0; i < radeon_connector->enabled_attribs; i++)
-			seq_printf(m, "attrib %d: %d %d\n", i,
-				   radeon_connector->cur_stream_attribs[i].fe,
-				   radeon_connector->cur_stream_attribs[i].slots);
-	}
-	drm_modeset_unlock_all(dev);
-	return 0;
-}
-
-DEFINE_SHOW_ATTRIBUTE(radeon_debugfs_mst_info);
-#endif
-
-void radeon_mst_debugfs_init(struct radeon_device *rdev)
-{
-#if defined(CONFIG_DEBUG_FS)
-	struct dentry *root = rdev->ddev->primary->debugfs_root;
-
-	debugfs_create_file("radeon_mst_info", 0444, root, rdev,
-			    &radeon_debugfs_mst_info_fops);
-
-#endif
-}
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index 956c72b5aa33..6cbe1ab81aba 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -172,7 +172,6 @@ int radeon_use_pflipirq = 2;
 int radeon_bapm = -1;
 int radeon_backlight = -1;
 int radeon_auxch = -1;
-int radeon_mst = 0;
 int radeon_uvd = 1;
 int radeon_vce = 1;
 
@@ -263,9 +262,6 @@ module_param_named(backlight, radeon_backlight, int, 0444);
 MODULE_PARM_DESC(auxch, "Use native auxch experimental support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(auxch, radeon_auxch, int, 0444);
 
-MODULE_PARM_DESC(mst, "DisplayPort MST experimental support (1 = enable, 0 = disable)");
-module_param_named(mst, radeon_mst, int, 0444);
-
 MODULE_PARM_DESC(uvd, "uvd enable/disable uvd support (1 = enable, 0 = disable)");
 module_param_named(uvd, radeon_uvd, int, 0444);
 
@@ -516,14 +512,11 @@ long radeon_drm_ioctl(struct file *filp,
 static long radeon_kms_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 {
 	unsigned int nr = DRM_IOCTL_NR(cmd);
-	int ret;
 
 	if (nr < DRM_COMMAND_BASE)
 		return drm_compat_ioctl(filp, cmd, arg);
 
-	ret = radeon_drm_ioctl(filp, cmd, arg);
-
-	return ret;
+	return radeon_drm_ioctl(filp, cmd, arg);
 }
 #endif
 
diff --git a/drivers/gpu/drm/radeon/radeon_encoders.c b/drivers/gpu/drm/radeon/radeon_encoders.c
index 46549d5179ee..fbc0a2182318 100644
--- a/drivers/gpu/drm/radeon/radeon_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_encoders.c
@@ -30,6 +30,8 @@
 #include <drm/drm_device.h>
 #include <drm/radeon_drm.h>
 
+#include <acpi/video.h>
+
 #include "radeon.h"
 #include "radeon_atombios.h"
 #include "radeon_legacy_encoders.h"
@@ -167,7 +169,7 @@ static void radeon_encoder_add_backlight(struct radeon_encoder *radeon_encoder,
 		return;
 
 	if (radeon_backlight == 0) {
-		return;
+		use_bl = false;
 	} else if (radeon_backlight == 1) {
 		use_bl = true;
 	} else if (radeon_backlight == -1) {
@@ -193,6 +195,13 @@ static void radeon_encoder_add_backlight(struct radeon_encoder *radeon_encoder,
 		else
 			radeon_legacy_backlight_init(radeon_encoder, connector);
 	}
+
+	/*
+	 * If there is no native backlight device (which may happen even when
+	 * use_bl==true) try registering an ACPI video backlight device instead.
+	 */
+	if (!rdev->mode_info.bl_encoder)
+		acpi_video_register_backlight();
 }
 
 void
@@ -244,16 +253,7 @@ radeon_get_connector_for_encoder(struct drm_encoder *encoder)
 
 	list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
 		radeon_connector = to_radeon_connector(connector);
-		if (radeon_encoder->is_mst_encoder) {
-			struct radeon_encoder_mst *mst_enc;
-
-			if (!radeon_connector->is_mst_connector)
-				continue;
-
-			mst_enc = radeon_encoder->enc_priv;
-			if (mst_enc->connector == radeon_connector->mst_port)
-				return connector;
-		} else if (radeon_encoder->active_device & radeon_connector->devices)
+		if (radeon_encoder->active_device & radeon_connector->devices)
 			return connector;
 	}
 	return NULL;
@@ -399,9 +399,6 @@ bool radeon_dig_monitor_is_duallink(struct drm_encoder *encoder,
 	case DRM_MODE_CONNECTOR_DVID:
 	case DRM_MODE_CONNECTOR_HDMIA:
 	case DRM_MODE_CONNECTOR_DisplayPort:
-		if (radeon_connector->is_mst_connector)
-			return false;
-
 		dig_connector = radeon_connector->con_priv;
 		if ((dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_DISPLAYPORT) ||
 		    (dig_connector->dp_sink_type == CONNECTOR_OBJECT_ID_eDP))
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index 3907785d0798..da2173435edd 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -100,16 +100,8 @@ static void radeon_hotplug_work_func(struct work_struct *work)
 
 static void radeon_dp_work_func(struct work_struct *work)
 {
-	struct radeon_device *rdev = container_of(work, struct radeon_device,
-						  dp_work);
-	struct drm_device *dev = rdev->ddev;
-	struct drm_mode_config *mode_config = &dev->mode_config;
-	struct drm_connector *connector;
-
-	/* this should take a mutex */
-	list_for_each_entry(connector, &mode_config->connector_list, head)
-		radeon_connector_hotplug(connector);
 }
+
 /**
  * radeon_driver_irq_preinstall_kms - drm irq preinstall callback
  *
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
index 1a66fb969ee7..0cd32c65456c 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
@@ -33,6 +33,8 @@
 #include <drm/drm_util.h>
 #include <drm/radeon_drm.h>
 
+#include <acpi/video.h>
+
 #include "radeon.h"
 #include "radeon_asic.h"
 #include "radeon_legacy_encoders.h"
@@ -387,6 +389,11 @@ void radeon_legacy_backlight_init(struct radeon_encoder *radeon_encoder,
 		return;
 #endif
 
+	if (!acpi_video_backlight_use_native()) {
+		drm_info(dev, "Skipping radeon legacy LVDS backlight registration\n");
+		return;
+	}
+
 	pdata = kmalloc(sizeof(struct radeon_backlight_privdata), GFP_KERNEL);
 	if (!pdata) {
 		DRM_ERROR("Memory allocation failed\n");
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index b34cffc162e2..9f5be416454f 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -31,7 +31,6 @@
 #define RADEON_MODE_H
 
 #include <drm/display/drm_dp_helper.h>
-#include <drm/display/drm_dp_mst_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_encoder.h>
@@ -436,24 +435,12 @@ struct radeon_encoder_atom_dig {
 	int panel_mode;
 	struct radeon_afmt *afmt;
 	struct r600_audio_pin *pin;
-	int active_mst_links;
 };
 
 struct radeon_encoder_atom_dac {
 	enum radeon_tv_std tv_std;
 };
 
-struct radeon_encoder_mst {
-	int crtc;
-	struct radeon_encoder *primary;
-	struct radeon_connector *connector;
-	struct drm_dp_mst_port *port;
-	int pbn;
-	int fe;
-	bool fe_from_be;
-	bool enc_active;
-};
-
 struct radeon_encoder {
 	struct drm_encoder base;
 	uint32_t encoder_enum;
@@ -475,8 +462,6 @@ struct radeon_encoder {
 	enum radeon_output_csc output_csc;
 	bool can_mst;
 	uint32_t offset;
-	bool is_mst_encoder;
-	/* front end for this mst encoder */
 };
 
 struct radeon_connector_atom_dig {
@@ -487,7 +472,6 @@ struct radeon_connector_atom_dig {
 	int dp_clock;
 	int dp_lane_count;
 	bool edp_on;
-	bool is_mst;
 };
 
 struct radeon_gpio_rec {
@@ -531,11 +515,6 @@ enum radeon_connector_dither {
 	RADEON_FMT_DITHER_ENABLE = 1,
 };
 
-struct stream_attribs {
-	uint16_t fe;
-	uint16_t slots;
-};
-
 struct radeon_connector {
 	struct drm_connector base;
 	uint32_t connector_id;
@@ -558,14 +537,6 @@ struct radeon_connector {
 	enum radeon_connector_audio audio;
 	enum radeon_connector_dither dither;
 	int pixelclock_for_modeset;
-	bool is_mst_connector;
-	struct radeon_connector *mst_port;
-	struct drm_dp_mst_port *port;
-	struct drm_dp_mst_topology_mgr mst_mgr;
-
-	struct radeon_encoder *mst_encoder;
-	struct stream_attribs cur_stream_attribs[6];
-	int enabled_attribs;
 };
 
 #define ENCODER_MODE_IS_DP(em) (((em) == ATOM_ENCODER_MODE_DP) || \
@@ -767,8 +738,6 @@ extern void atombios_dig_transmitter_setup(struct drm_encoder *encoder,
 extern void atombios_dig_transmitter_setup2(struct drm_encoder *encoder,
 					    int action, uint8_t lane_num,
 					    uint8_t lane_set, int fe);
-extern void atombios_set_mst_encoder_crtc_source(struct drm_encoder *encoder,
-						 int fe);
 extern void radeon_atom_ext_encoder_setup_ddc(struct drm_encoder *encoder);
 extern struct drm_encoder *radeon_get_external_encoder(struct drm_encoder *encoder);
 void radeon_atom_copy_swap(u8 *dst, u8 *src, u8 num_bytes, bool to_le);
@@ -905,7 +874,6 @@ extern struct radeon_encoder_tv_dac *
 radeon_atombios_get_tv_dac_info(struct radeon_encoder *encoder);
 extern struct radeon_encoder_lvds *
 radeon_combios_get_lvds_info(struct radeon_encoder *encoder);
-extern void radeon_combios_get_ext_tmds_info(struct radeon_encoder *encoder);
 extern struct radeon_encoder_tv_dac *
 radeon_combios_get_tv_dac_info(struct radeon_encoder *encoder);
 extern struct radeon_encoder_primary_dac *
@@ -986,15 +954,6 @@ void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id);
 
 int radeon_align_pitch(struct radeon_device *rdev, int width, int bpp, bool tiled);
 
-/* mst */
-int radeon_dp_mst_init(struct radeon_connector *radeon_connector);
-int radeon_dp_mst_probe(struct radeon_connector *radeon_connector);
-int radeon_dp_mst_check_status(struct radeon_connector *radeon_connector);
-void radeon_mst_debugfs_init(struct radeon_device *rdev);
-void radeon_dp_mst_prepare_pll(struct drm_crtc *crtc, struct drm_display_mode *mode);
-
-void radeon_setup_mst_connector(struct drm_device *dev);
-
 int radeon_atom_pick_dig_encoder(struct drm_encoder *encoder, int fe_idx);
 void radeon_atom_release_dig_encoder(struct radeon_device *rdev, int enc_idx);
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index 6c4a6802ca96..00c33b24d5d3 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -202,9 +202,9 @@ int radeon_bo_create(struct radeon_device *rdev,
 	radeon_ttm_placement_from_domain(bo, domain);
 	/* Kernel allocation are uninterruptible */
 	down_read(&rdev->pm.mclk_lock);
-	r = ttm_bo_init(&rdev->mman.bdev, &bo->tbo, size, type,
-			&bo->placement, page_align, !kernel, sg, resv,
-			&radeon_ttm_bo_destroy);
+	r = ttm_bo_init_validate(&rdev->mman.bdev, &bo->tbo, type,
+				 &bo->placement, page_align, !kernel, sg, resv,
+				 &radeon_ttm_bo_destroy);
 	up_read(&rdev->pm.mclk_lock);
 	if (unlikely(r != 0)) {
 		return r;
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index e765abcb3b01..04c693ca419a 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -1899,7 +1899,7 @@ static void radeon_dynpm_idle_work_handler(struct work_struct *work)
 		 * to false since we want to wait for vbl to avoid flicker.
 		 */
 		if (rdev->pm.dynpm_planned_action != DYNPM_ACTION_NONE &&
-		    jiffies > rdev->pm.dynpm_action_timeout) {
+		    time_after(jiffies, rdev->pm.dynpm_action_timeout)) {
 			radeon_pm_get_dynpm_state(rdev);
 			radeon_pm_set_clocks(rdev);
 		}
diff --git a/drivers/gpu/drm/rcar-du/Kconfig b/drivers/gpu/drm/rcar-du/Kconfig
index f6e6a6d5d987..c959e8c6be7d 100644
--- a/drivers/gpu/drm/rcar-du/Kconfig
+++ b/drivers/gpu/drm/rcar-du/Kconfig
@@ -5,7 +5,7 @@ config DRM_RCAR_DU
 	depends on ARM || ARM64
 	depends on ARCH_RENESAS || COMPILE_TEST
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select VIDEOMODE_HELPERS
 	help
 	  Choose this option if you have an R-Car chipset.
diff --git a/drivers/gpu/drm/rcar-du/Makefile b/drivers/gpu/drm/rcar-du/Makefile
index e7275b5e7ec8..6f132325c8b7 100644
--- a/drivers/gpu/drm/rcar-du/Makefile
+++ b/drivers/gpu/drm/rcar-du/Makefile
@@ -14,10 +14,3 @@ obj-$(CONFIG_DRM_RCAR_DU)		+= rcar-du-drm.o
 obj-$(CONFIG_DRM_RCAR_DW_HDMI)		+= rcar_dw_hdmi.o
 obj-$(CONFIG_DRM_RCAR_LVDS)		+= rcar_lvds.o
 obj-$(CONFIG_DRM_RCAR_MIPI_DSI)		+= rcar_mipi_dsi.o
-
-# 'remote-endpoint' is fixed up at run-time
-DTC_FLAGS_rcar_du_of_lvds_r8a7790 += -Wno-graph_endpoint
-DTC_FLAGS_rcar_du_of_lvds_r8a7791 += -Wno-graph_endpoint
-DTC_FLAGS_rcar_du_of_lvds_r8a7793 += -Wno-graph_endpoint
-DTC_FLAGS_rcar_du_of_lvds_r8a7795 += -Wno-graph_endpoint
-DTC_FLAGS_rcar_du_of_lvds_r8a7796 += -Wno-graph_endpoint
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
index 621bbccb95d4..3619e1ddeb62 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -17,9 +17,7 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "rcar_cmm.h"
@@ -31,6 +29,7 @@
 #include "rcar_du_regs.h"
 #include "rcar_du_vsp.h"
 #include "rcar_lvds.h"
+#include "rcar_mipi_dsi.h"
 
 static u32 rcar_du_crtc_read(struct rcar_du_crtc *rcrtc, u32 reg)
 {
@@ -746,7 +745,19 @@ static void rcar_du_crtc_atomic_enable(struct drm_crtc *crtc,
 		const struct drm_display_mode *mode =
 			&crtc->state->adjusted_mode;
 
-		rcar_lvds_clk_enable(bridge, mode->clock * 1000);
+		rcar_lvds_pclk_enable(bridge, mode->clock * 1000);
+	}
+
+	/*
+	 * Similarly to LVDS, on V3U the dot clock is provided by the DSI
+	 * encoder, and we need to enable the DSI clocks before enabling the CRTC.
+	 */
+	if ((rcdu->info->dsi_clk_mask & BIT(rcrtc->index)) &&
+	    (rstate->outputs &
+	     (BIT(RCAR_DU_OUTPUT_DSI0) | BIT(RCAR_DU_OUTPUT_DSI1)))) {
+		struct drm_bridge *bridge = rcdu->dsi[rcrtc->index];
+
+		rcar_mipi_dsi_pclk_enable(bridge, state);
 	}
 
 	rcar_du_crtc_start(rcrtc);
@@ -779,7 +790,20 @@ static void rcar_du_crtc_atomic_disable(struct drm_crtc *crtc,
 		 * Disable the LVDS clock output, see
 		 * rcar_du_crtc_atomic_enable().
 		 */
-		rcar_lvds_clk_disable(bridge);
+		rcar_lvds_pclk_disable(bridge);
+	}
+
+	if ((rcdu->info->dsi_clk_mask & BIT(rcrtc->index)) &&
+	    (rstate->outputs &
+	     (BIT(RCAR_DU_OUTPUT_DSI0) | BIT(RCAR_DU_OUTPUT_DSI1)))) {
+		struct drm_bridge *bridge = rcdu->dsi[rcrtc->index];
+
+		/*
+		 * Disable the DSI clock output, see
+		 * rcar_du_crtc_atomic_enable().
+		 */
+
+		rcar_mipi_dsi_pclk_disable(bridge);
 	}
 
 	spin_lock_irq(&crtc->dev->event_lock);
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
index 70d85610d720..a2776f1d6f2c 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
@@ -20,15 +20,13 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_probe_helper.h>
 
 #include "rcar_du_drv.h"
 #include "rcar_du_kms.h"
-#include "rcar_du_regs.h"
 
 /* -----------------------------------------------------------------------------
  * Device Information
@@ -508,7 +506,8 @@ static const struct rcar_du_device_info rcar_du_r8a7799x_info = {
 static const struct rcar_du_device_info rcar_du_r8a779a0_info = {
 	.gen = 3,
 	.features = RCAR_DU_FEATURE_CRTC_IRQ
-		  | RCAR_DU_FEATURE_VSP1_SOURCE,
+		  | RCAR_DU_FEATURE_VSP1_SOURCE
+		  | RCAR_DU_FEATURE_NO_BLENDING,
 	.channels_mask = BIT(1) | BIT(0),
 	.routes = {
 		/* R8A779A0 has two MIPI DSI outputs. */
@@ -579,7 +578,7 @@ const char *rcar_du_output_name(enum rcar_du_output output)
  * DRM operations
  */
 
-DEFINE_DRM_GEM_CMA_FOPS(rcar_du_fops);
+DEFINE_DRM_GEM_DMA_FOPS(rcar_du_fops);
 
 static const struct drm_driver rcar_du_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.h b/drivers/gpu/drm/rcar-du/rcar_du_drv.h
index bfad7775d9a1..5cfa2bb7ad93 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.h
@@ -31,6 +31,7 @@ struct rcar_du_device;
 #define RCAR_DU_FEATURE_VSP1_SOURCE	BIT(2)	/* Has inputs from VSP1 */
 #define RCAR_DU_FEATURE_INTERLACED	BIT(3)	/* HW supports interlaced */
 #define RCAR_DU_FEATURE_TVM_SYNC	BIT(4)	/* Has TV switch/sync modes */
+#define RCAR_DU_FEATURE_NO_BLENDING	BIT(5)	/* PnMR.SPIM does not have ALP nor EOR bits */
 
 #define RCAR_DU_QUIRK_ALIGN_128B	BIT(0)	/* Align pitches to 128 bytes */
 
@@ -91,6 +92,7 @@ struct rcar_du_device_info {
 #define RCAR_DU_MAX_GROUPS		DIV_ROUND_UP(RCAR_DU_MAX_CRTCS, 2)
 #define RCAR_DU_MAX_VSPS		4
 #define RCAR_DU_MAX_LVDS		2
+#define RCAR_DU_MAX_DSI			2
 
 struct rcar_du_device {
 	struct device *dev;
@@ -107,6 +109,7 @@ struct rcar_du_device {
 	struct platform_device *cmms[RCAR_DU_MAX_CRTCS];
 	struct rcar_du_vsp vsps[RCAR_DU_MAX_VSPS];
 	struct drm_bridge *lvds[RCAR_DU_MAX_LVDS];
+	struct drm_bridge *dsi[RCAR_DU_MAX_DSI];
 
 	struct {
 		struct drm_property *colorkey;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
index 60d6be78323b..b1787be31e92 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
@@ -9,18 +9,13 @@
 
 #include <linux/export.h>
 #include <linux/of.h>
-#include <linux/slab.h>
 
 #include <drm/drm_bridge.h>
 #include <drm/drm_bridge_connector.h>
-#include <drm/drm_crtc.h>
-#include <drm/drm_managed.h>
-#include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_panel.h>
 
 #include "rcar_du_drv.h"
 #include "rcar_du_encoder.h"
-#include "rcar_du_kms.h"
 #include "rcar_lvds.h"
 
 /* -----------------------------------------------------------------------------
@@ -84,6 +79,10 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 		if (output == RCAR_DU_OUTPUT_LVDS0 ||
 		    output == RCAR_DU_OUTPUT_LVDS1)
 			rcdu->lvds[output - RCAR_DU_OUTPUT_LVDS0] = bridge;
+
+		if (output == RCAR_DU_OUTPUT_DSI0 ||
+		    output == RCAR_DU_OUTPUT_DSI1)
+			rcdu->dsi[output - RCAR_DU_OUTPUT_DSI0] = bridge;
 	}
 
 	/*
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
index 761451ee5263..8c2719efda2a 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
@@ -11,9 +11,8 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_probe_helper.h>
@@ -328,12 +327,12 @@ const struct rcar_du_format_info *rcar_du_format_info(u32 fourcc)
  */
 
 static const struct drm_gem_object_funcs rcar_du_gem_funcs = {
-	.free = drm_gem_cma_object_free,
-	.print_info = drm_gem_cma_object_print_info,
-	.get_sg_table = drm_gem_cma_object_get_sg_table,
-	.vmap = drm_gem_cma_object_vmap,
-	.mmap = drm_gem_cma_object_mmap,
-	.vm_ops = &drm_gem_cma_vm_ops,
+	.free = drm_gem_dma_object_free,
+	.print_info = drm_gem_dma_object_print_info,
+	.get_sg_table = drm_gem_dma_object_get_sg_table,
+	.vmap = drm_gem_dma_object_vmap,
+	.mmap = drm_gem_dma_object_mmap,
+	.vm_ops = &drm_gem_dma_vm_ops,
 };
 
 struct drm_gem_object *rcar_du_gem_prime_import_sg_table(struct drm_device *dev,
@@ -341,33 +340,33 @@ struct drm_gem_object *rcar_du_gem_prime_import_sg_table(struct drm_device *dev,
 				struct sg_table *sgt)
 {
 	struct rcar_du_device *rcdu = to_rcar_du_device(dev);
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	struct drm_gem_object *gem_obj;
 	int ret;
 
 	if (!rcar_du_has(rcdu, RCAR_DU_FEATURE_VSP1_SOURCE))
-		return drm_gem_cma_prime_import_sg_table(dev, attach, sgt);
+		return drm_gem_dma_prime_import_sg_table(dev, attach, sgt);
 
-	/* Create a CMA GEM buffer. */
-	cma_obj = kzalloc(sizeof(*cma_obj), GFP_KERNEL);
-	if (!cma_obj)
+	/* Create a DMA GEM buffer. */
+	dma_obj = kzalloc(sizeof(*dma_obj), GFP_KERNEL);
+	if (!dma_obj)
 		return ERR_PTR(-ENOMEM);
 
-	gem_obj = &cma_obj->base;
+	gem_obj = &dma_obj->base;
 	gem_obj->funcs = &rcar_du_gem_funcs;
 
 	drm_gem_private_object_init(dev, gem_obj, attach->dmabuf->size);
-	cma_obj->map_noncoherent = false;
+	dma_obj->map_noncoherent = false;
 
 	ret = drm_gem_create_mmap_offset(gem_obj);
 	if (ret) {
 		drm_gem_object_release(gem_obj);
-		kfree(cma_obj);
+		kfree(dma_obj);
 		return ERR_PTR(ret);
 	}
 
-	cma_obj->paddr = 0;
-	cma_obj->sgt = sgt;
+	dma_obj->dma_addr = 0;
+	dma_obj->sgt = sgt;
 
 	return gem_obj;
 }
@@ -390,7 +389,7 @@ int rcar_du_dumb_create(struct drm_file *file, struct drm_device *dev,
 
 	args->pitch = roundup(min_pitch, align);
 
-	return drm_gem_cma_dumb_create_internal(file, dev, args);
+	return drm_gem_dma_dumb_create_internal(file, dev, args);
 }
 
 static struct drm_framebuffer *
@@ -406,8 +405,8 @@ rcar_du_fb_create(struct drm_device *dev, struct drm_file *file_priv,
 
 	format = rcar_du_format_info(mode_cmd->pixel_format);
 	if (format == NULL) {
-		dev_dbg(dev->dev, "unsupported pixel format %08x\n",
-			mode_cmd->pixel_format);
+		dev_dbg(dev->dev, "unsupported pixel format %p4cc\n",
+			&mode_cmd->pixel_format);
 		return ERR_PTR(-EINVAL);
 	}
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.c b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
index 501d79367e3e..d759e0192181 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_plane.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
@@ -12,11 +12,10 @@
 #include <drm/drm_blend.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 
 #include "rcar_du_drv.h"
 #include "rcar_du_group.h"
@@ -342,7 +341,7 @@ static void rcar_du_plane_setup_scanout(struct rcar_du_group *rgrp,
 
 	if (state->source == RCAR_DU_PLANE_MEMORY) {
 		struct drm_framebuffer *fb = state->state.fb;
-		struct drm_gem_cma_object *gem;
+		struct drm_gem_dma_object *gem;
 		unsigned int i;
 
 		if (state->format->planes == 2)
@@ -351,8 +350,8 @@ static void rcar_du_plane_setup_scanout(struct rcar_du_group *rgrp,
 			pitch = fb->pitches[0] * 8 / state->format->bpp;
 
 		for (i = 0; i < state->format->planes; ++i) {
-			gem = drm_fb_cma_get_gem_obj(fb, i);
-			dma[i] = gem->paddr + fb->offsets[i];
+			gem = drm_fb_dma_get_gem_obj(fb, i);
+			dma[i] = gem->dma_addr + fb->offsets[i];
 		}
 	} else {
 		pitch = drm_rect_width(&state->state.src) >> 16;
@@ -507,8 +506,15 @@ static void rcar_du_plane_setup_format_gen3(struct rcar_du_group *rgrp,
 					    unsigned int index,
 					    const struct rcar_du_plane_state *state)
 {
-	rcar_du_plane_write(rgrp, index, PnMR,
-			    PnMR_SPIM_TP_OFF | state->format->pnmr);
+	struct rcar_du_device *rcdu = rgrp->dev;
+	u32 pnmr = state->format->pnmr | PnMR_SPIM_TP_OFF;
+
+	if (rcdu->info->features & RCAR_DU_FEATURE_NO_BLENDING) {
+		/* No blending. ALP and EOR are not supported. */
+		pnmr &= ~(PnMR_SPIM_ALP | PnMR_SPIM_EOR);
+	}
+
+	rcar_du_plane_write(rgrp, index, PnMR, pnmr);
 
 	rcar_du_plane_write(rgrp, index, PnDDCR4,
 			    state->format->edf | PnDDCR4_CODE);
@@ -522,7 +528,6 @@ static void rcar_du_plane_setup_format_gen3(struct rcar_du_group *rgrp,
 	 * register to 0 to avoid this.
 	 */
 
-	/* TODO: Check if alpha-blending should be disabled in PnMR. */
 	rcar_du_plane_write(rgrp, index, PnALPHAR, 0);
 }
 
@@ -607,8 +612,8 @@ int __rcar_du_plane_atomic_check(struct drm_plane *plane,
 		return PTR_ERR(crtc_state);
 
 	ret = drm_atomic_helper_check_plane_state(state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  true, true);
 	if (ret < 0)
 		return ret;
@@ -620,8 +625,8 @@ int __rcar_du_plane_atomic_check(struct drm_plane *plane,
 
 	*format = rcar_du_format_info(state->fb->format->format);
 	if (*format == NULL) {
-		dev_dbg(dev->dev, "%s: unsupported format %08x\n", __func__,
-			state->fb->format->format);
+		dev_dbg(dev->dev, "%s: unsupported format %p4cc\n", __func__,
+			&state->fb->format->format);
 		return -EINVAL;
 	}
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
index dbc68cdabcff..e465aef41585 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vsp.c
@@ -11,13 +11,12 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_crtc.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_vblank.h>
 
 #include <linux/bitops.h>
@@ -153,6 +152,7 @@ static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane)
 		.alpha = state->state.alpha >> 8,
 		.zpos = state->state.zpos,
 	};
+	u32 fourcc = state->format->fourcc;
 	unsigned int i;
 
 	cfg.src.left = state->state.src.x1 >> 16;
@@ -169,9 +169,27 @@ static void rcar_du_vsp_plane_setup(struct rcar_du_vsp_plane *plane)
 		cfg.mem[i] = sg_dma_address(state->sg_tables[i].sgl)
 			   + fb->offsets[i];
 
-	format = rcar_du_format_info(state->format->fourcc);
+	if (state->state.pixel_blend_mode == DRM_MODE_BLEND_PIXEL_NONE) {
+		switch (fourcc) {
+		case DRM_FORMAT_ARGB1555:
+			fourcc = DRM_FORMAT_XRGB1555;
+			break;
+
+		case DRM_FORMAT_ARGB4444:
+			fourcc = DRM_FORMAT_XRGB4444;
+			break;
+
+		case DRM_FORMAT_ARGB8888:
+			fourcc = DRM_FORMAT_XRGB8888;
+			break;
+		}
+	}
+
+	format = rcar_du_format_info(fourcc);
 	cfg.pixelformat = format->v4l2;
 
+	cfg.premult = state->state.pixel_blend_mode == DRM_MODE_BLEND_PREMULTI;
+
 	vsp1_du_atomic_update(plane->vsp->vsp, crtc->vsp_pipe,
 			      plane->index, &cfg);
 }
@@ -184,7 +202,7 @@ int rcar_du_vsp_map_fb(struct rcar_du_vsp *vsp, struct drm_framebuffer *fb,
 	int ret;
 
 	for (i = 0; i < fb->format->num_planes; ++i) {
-		struct drm_gem_cma_object *gem = drm_fb_cma_get_gem_obj(fb, i);
+		struct drm_gem_dma_object *gem = drm_fb_dma_get_gem_obj(fb, i);
 		struct sg_table *sgt = &sg_tables[i];
 
 		if (gem->sgt) {
@@ -213,7 +231,7 @@ int rcar_du_vsp_map_fb(struct rcar_du_vsp *vsp, struct drm_framebuffer *fb,
 			}
 		} else {
 			ret = dma_get_sgtable(rcdu->dev, sgt, gem->vaddr,
-					      gem->paddr, gem->base.size);
+					      gem->dma_addr, gem->base.size);
 			if (ret)
 				goto fail;
 		}
@@ -437,6 +455,11 @@ int rcar_du_vsp_init(struct rcar_du_vsp *vsp, struct device_node *np,
 		drm_plane_create_zpos_property(&plane->plane, i, 0,
 					       num_planes - 1);
 
+		drm_plane_create_blend_mode_property(&plane->plane,
+					BIT(DRM_MODE_BLEND_PIXEL_NONE) |
+					BIT(DRM_MODE_BLEND_PREMULTI) |
+					BIT(DRM_MODE_BLEND_COVERAGE));
+
 		vsp->num_planes++;
 	}
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_writeback.c b/drivers/gpu/drm/rcar-du/rcar_du_writeback.c
index 25f50a297c11..8cd37d7b8ae2 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_writeback.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_writeback.c
@@ -166,8 +166,8 @@ static int rcar_du_wb_enc_atomic_check(struct drm_encoder *encoder,
 
 	wb_state->format = rcar_du_format_info(fb->format->format);
 	if (wb_state->format == NULL) {
-		dev_dbg(dev->dev, "%s: unsupported format %08x\n", __func__,
-			fb->format->format);
+		dev_dbg(dev->dev, "%s: unsupported format %p4cc\n", __func__,
+			&fb->format->format);
 		return -EINVAL;
 	}
 
diff --git a/drivers/gpu/drm/rcar-du/rcar_lvds.c b/drivers/gpu/drm/rcar-du/rcar_lvds.c
index d85aa4bc7f84..81a060c2fe3f 100644
--- a/drivers/gpu/drm/rcar-du/rcar_lvds.c
+++ b/drivers/gpu/drm/rcar-du/rcar_lvds.c
@@ -306,7 +306,7 @@ static void rcar_lvds_pll_setup_d3_e3(struct rcar_lvds *lvds, unsigned int freq)
  * Clock - D3/E3 only
  */
 
-int rcar_lvds_clk_enable(struct drm_bridge *bridge, unsigned long freq)
+int rcar_lvds_pclk_enable(struct drm_bridge *bridge, unsigned long freq)
 {
 	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
 	int ret;
@@ -324,9 +324,9 @@ int rcar_lvds_clk_enable(struct drm_bridge *bridge, unsigned long freq)
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(rcar_lvds_clk_enable);
+EXPORT_SYMBOL_GPL(rcar_lvds_pclk_enable);
 
-void rcar_lvds_clk_disable(struct drm_bridge *bridge)
+void rcar_lvds_pclk_disable(struct drm_bridge *bridge)
 {
 	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
 
@@ -339,7 +339,7 @@ void rcar_lvds_clk_disable(struct drm_bridge *bridge)
 
 	clk_disable_unprepare(lvds->clocks.mod);
 }
-EXPORT_SYMBOL_GPL(rcar_lvds_clk_disable);
+EXPORT_SYMBOL_GPL(rcar_lvds_pclk_disable);
 
 /* -----------------------------------------------------------------------------
  * Bridge
diff --git a/drivers/gpu/drm/rcar-du/rcar_lvds.h b/drivers/gpu/drm/rcar-du/rcar_lvds.h
index 3097bf749bec..bee7033b60d6 100644
--- a/drivers/gpu/drm/rcar-du/rcar_lvds.h
+++ b/drivers/gpu/drm/rcar-du/rcar_lvds.h
@@ -13,17 +13,17 @@
 struct drm_bridge;
 
 #if IS_ENABLED(CONFIG_DRM_RCAR_LVDS)
-int rcar_lvds_clk_enable(struct drm_bridge *bridge, unsigned long freq);
-void rcar_lvds_clk_disable(struct drm_bridge *bridge);
+int rcar_lvds_pclk_enable(struct drm_bridge *bridge, unsigned long freq);
+void rcar_lvds_pclk_disable(struct drm_bridge *bridge);
 bool rcar_lvds_dual_link(struct drm_bridge *bridge);
 bool rcar_lvds_is_connected(struct drm_bridge *bridge);
 #else
-static inline int rcar_lvds_clk_enable(struct drm_bridge *bridge,
-				       unsigned long freq)
+static inline int rcar_lvds_pclk_enable(struct drm_bridge *bridge,
+					unsigned long freq)
 {
 	return -ENOSYS;
 }
-static inline void rcar_lvds_clk_disable(struct drm_bridge *bridge) { }
+static inline void rcar_lvds_pclk_disable(struct drm_bridge *bridge) { }
 static inline bool rcar_lvds_dual_link(struct drm_bridge *bridge)
 {
 	return false;
diff --git a/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.c b/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.c
index 62f7eb84ab01..a7f2b7f66a17 100644
--- a/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.c
+++ b/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.c
@@ -25,6 +25,7 @@
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
 
+#include "rcar_mipi_dsi.h"
 #include "rcar_mipi_dsi_regs.h"
 
 struct rcar_mipi_dsi {
@@ -414,7 +415,7 @@ static int rcar_mipi_dsi_startup(struct rcar_mipi_dsi *dsi,
 
 	/* Enable DOT clock */
 	vclkset = VCLKSET_CKEN;
-	rcar_mipi_dsi_set(dsi, VCLKSET, vclkset);
+	rcar_mipi_dsi_write(dsi, VCLKSET, vclkset);
 
 	if (dsi_format == 24)
 		vclkset |= VCLKSET_BPP_24;
@@ -429,7 +430,7 @@ static int rcar_mipi_dsi_startup(struct rcar_mipi_dsi *dsi,
 	vclkset |= VCLKSET_COLOR_RGB | VCLKSET_DIV(setup_info.div)
 		|  VCLKSET_LANE(dsi->lanes - 1);
 
-	rcar_mipi_dsi_set(dsi, VCLKSET, vclkset);
+	rcar_mipi_dsi_write(dsi, VCLKSET, vclkset);
 
 	/* After setting VCLKSET register, enable VCLKEN */
 	rcar_mipi_dsi_set(dsi, VCLKEN, VCLKEN_CKEN);
@@ -441,9 +442,21 @@ static int rcar_mipi_dsi_startup(struct rcar_mipi_dsi *dsi,
 
 static void rcar_mipi_dsi_shutdown(struct rcar_mipi_dsi *dsi)
 {
+	/* Disable VCLKEN */
+	rcar_mipi_dsi_write(dsi, VCLKSET, 0);
+
+	/* Disable DOT clock */
+	rcar_mipi_dsi_write(dsi, VCLKSET, 0);
+
 	rcar_mipi_dsi_clr(dsi, PHYSETUP, PHYSETUP_RSTZ);
 	rcar_mipi_dsi_clr(dsi, PHYSETUP, PHYSETUP_SHUTDOWNZ);
 
+	/* CFGCLK disable */
+	rcar_mipi_dsi_clr(dsi, CFGCLKSET, CFGCLKSET_CKEN);
+
+	/* LPCLK disable */
+	rcar_mipi_dsi_clr(dsi, LPCLKSET, LPCLKSET_CKEN);
+
 	dev_dbg(dsi->dev, "DSI device is shutdown\n");
 }
 
@@ -542,6 +555,34 @@ static int rcar_mipi_dsi_start_video(struct rcar_mipi_dsi *dsi)
 	return 0;
 }
 
+static void rcar_mipi_dsi_stop_video(struct rcar_mipi_dsi *dsi)
+{
+	u32 status;
+	int ret;
+
+	/* Disable transmission in video mode. */
+	rcar_mipi_dsi_clr(dsi, TXVMCR, TXVMCR_EN_VIDEO);
+
+	ret = read_poll_timeout(rcar_mipi_dsi_read, status,
+				!(status & TXVMSR_ACT),
+				2000, 100000, false, dsi, TXVMSR);
+	if (ret < 0) {
+		dev_err(dsi->dev, "Failed to disable video transmission\n");
+		return;
+	}
+
+	/* Assert video FIFO clear. */
+	rcar_mipi_dsi_set(dsi, TXVMCR, TXVMCR_VFCLR);
+
+	ret = read_poll_timeout(rcar_mipi_dsi_read, status,
+				!(status & TXVMSR_VFRDY),
+				2000, 100000, false, dsi, TXVMSR);
+	if (ret < 0) {
+		dev_err(dsi->dev, "Failed to assert video FIFO clear\n");
+		return;
+	}
+}
+
 /* -----------------------------------------------------------------------------
  * Bridge
  */
@@ -558,7 +599,22 @@ static int rcar_mipi_dsi_attach(struct drm_bridge *bridge,
 static void rcar_mipi_dsi_atomic_enable(struct drm_bridge *bridge,
 					struct drm_bridge_state *old_bridge_state)
 {
-	struct drm_atomic_state *state = old_bridge_state->base.state;
+	struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge);
+
+	rcar_mipi_dsi_start_video(dsi);
+}
+
+static void rcar_mipi_dsi_atomic_disable(struct drm_bridge *bridge,
+					 struct drm_bridge_state *old_bridge_state)
+{
+	struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge);
+
+	rcar_mipi_dsi_stop_video(dsi);
+}
+
+void rcar_mipi_dsi_pclk_enable(struct drm_bridge *bridge,
+			       struct drm_atomic_state *state)
+{
 	struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge);
 	const struct drm_display_mode *mode;
 	struct drm_connector *connector;
@@ -586,8 +642,6 @@ static void rcar_mipi_dsi_atomic_enable(struct drm_bridge *bridge,
 	if (ret < 0)
 		goto err_dsi_start_hs;
 
-	rcar_mipi_dsi_start_video(dsi);
-
 	return;
 
 err_dsi_start_hs:
@@ -595,15 +649,16 @@ err_dsi_start_hs:
 err_dsi_startup:
 	rcar_mipi_dsi_clk_disable(dsi);
 }
+EXPORT_SYMBOL_GPL(rcar_mipi_dsi_pclk_enable);
 
-static void rcar_mipi_dsi_atomic_disable(struct drm_bridge *bridge,
-					 struct drm_bridge_state *old_bridge_state)
+void rcar_mipi_dsi_pclk_disable(struct drm_bridge *bridge)
 {
 	struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge);
 
 	rcar_mipi_dsi_shutdown(dsi);
 	rcar_mipi_dsi_clk_disable(dsi);
 }
+EXPORT_SYMBOL_GPL(rcar_mipi_dsi_pclk_disable);
 
 static enum drm_mode_status
 rcar_mipi_dsi_bridge_mode_valid(struct drm_bridge *bridge,
diff --git a/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.h b/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.h
new file mode 100644
index 000000000000..528a196e6edd
--- /dev/null
+++ b/drivers/gpu/drm/rcar-du/rcar_mipi_dsi.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * R-Car DSI Encoder
+ *
+ * Copyright (C) 2022 Renesas Electronics Corporation
+ *
+ * Contact: Tomi Valkeinen <tomi.valkeinen@ideasonboard.com>
+ */
+
+#ifndef __RCAR_MIPI_DSI_H__
+#define __RCAR_MIPI_DSI_H__
+
+struct drm_atomic_state;
+struct drm_bridge;
+
+#if IS_ENABLED(CONFIG_DRM_RCAR_MIPI_DSI)
+void rcar_mipi_dsi_pclk_enable(struct drm_bridge *bridge,
+			       struct drm_atomic_state *state);
+void rcar_mipi_dsi_pclk_disable(struct drm_bridge *bridge);
+#else
+static inline void rcar_mipi_dsi_pclk_enable(struct drm_bridge *bridge,
+					     struct drm_atomic_state *state)
+{
+}
+
+static inline void rcar_mipi_dsi_pclk_disable(struct drm_bridge *bridge)
+{
+}
+#endif /* CONFIG_DRM_RCAR_MIPI_DSI */
+
+#endif /* __RCAR_MIPI_DSI_H__ */
diff --git a/drivers/gpu/drm/rockchip/Kconfig b/drivers/gpu/drm/rockchip/Kconfig
index 53c2d9980d48..1bf3e2829cd0 100644
--- a/drivers/gpu/drm/rockchip/Kconfig
+++ b/drivers/gpu/drm/rockchip/Kconfig
@@ -2,7 +2,7 @@
 config DRM_ROCKCHIP
 	tristate "DRM Support for Rockchip"
 	depends on DRM && ROCKCHIP_IOMMU
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_KMS_HELPER
 	select DRM_PANEL
 	select VIDEOMODE_HELPERS
diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
index 110e83aad9bb..bf6948125b84 100644
--- a/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi-rockchip.c
@@ -179,6 +179,23 @@
 #define RK3399_TXRX_SRC_SEL_ISP0	BIT(4)
 #define RK3399_TXRX_TURNREQUEST		GENMASK(3, 0)
 
+#define RK3568_GRF_VO_CON2		0x0368
+#define RK3568_DSI0_SKEWCALHS		(0x1f << 11)
+#define RK3568_DSI0_FORCETXSTOPMODE	(0xf << 4)
+#define RK3568_DSI0_TURNDISABLE		BIT(2)
+#define RK3568_DSI0_FORCERXMODE		BIT(0)
+
+/*
+ * Note these registers do not appear in the datasheet, they are
+ * however present in the BSP driver which is where these values
+ * come from. Name GRF_VO_CON3 is assumed.
+ */
+#define RK3568_GRF_VO_CON3		0x36c
+#define RK3568_DSI1_SKEWCALHS		(0x1f << 11)
+#define RK3568_DSI1_FORCETXSTOPMODE	(0xf << 4)
+#define RK3568_DSI1_TURNDISABLE		BIT(2)
+#define RK3568_DSI1_FORCERXMODE		BIT(0)
+
 #define HIWORD_UPDATE(val, mask)	(val | (mask) << 16)
 
 enum {
@@ -735,8 +752,9 @@ static void dw_mipi_dsi_rockchip_config(struct dw_mipi_dsi_rockchip *dsi)
 static void dw_mipi_dsi_rockchip_set_lcdsel(struct dw_mipi_dsi_rockchip *dsi,
 					    int mux)
 {
-	regmap_write(dsi->grf_regmap, dsi->cdata->lcdsel_grf_reg,
-		mux ? dsi->cdata->lcdsel_lit : dsi->cdata->lcdsel_big);
+	if (dsi->cdata->lcdsel_grf_reg < 0)
+		regmap_write(dsi->grf_regmap, dsi->cdata->lcdsel_grf_reg,
+			mux ? dsi->cdata->lcdsel_lit : dsi->cdata->lcdsel_big);
 }
 
 static int
@@ -963,6 +981,8 @@ static int dw_mipi_dsi_rockchip_bind(struct device *dev,
 		DRM_DEV_ERROR(dev, "Failed to create drm encoder\n");
 		goto out_pll_clk;
 	}
+	rockchip_drm_encoder_set_crtc_endpoint_id(&dsi->encoder,
+						  dev->of_node, 0, 0);
 
 	ret = dw_mipi_dsi_bind(dsi->dmd, &dsi->encoder.encoder);
 	if (ret) {
@@ -1612,6 +1632,30 @@ static const struct rockchip_dw_dsi_chip_data rk3399_chip_data[] = {
 	{ /* sentinel */ }
 };
 
+static const struct rockchip_dw_dsi_chip_data rk3568_chip_data[] = {
+	{
+		.reg = 0xfe060000,
+		.lcdsel_grf_reg = -1,
+		.lanecfg1_grf_reg = RK3568_GRF_VO_CON2,
+		.lanecfg1 = HIWORD_UPDATE(0, RK3568_DSI0_SKEWCALHS |
+					  RK3568_DSI0_FORCETXSTOPMODE |
+					  RK3568_DSI0_TURNDISABLE |
+					  RK3568_DSI0_FORCERXMODE),
+		.max_data_lanes = 4,
+	},
+	{
+		.reg = 0xfe070000,
+		.lcdsel_grf_reg = -1,
+		.lanecfg1_grf_reg = RK3568_GRF_VO_CON3,
+		.lanecfg1 = HIWORD_UPDATE(0, RK3568_DSI1_SKEWCALHS |
+					  RK3568_DSI1_FORCETXSTOPMODE |
+					  RK3568_DSI1_TURNDISABLE |
+					  RK3568_DSI1_FORCERXMODE),
+		.max_data_lanes = 4,
+	},
+	{ /* sentinel */ }
+};
+
 static const struct of_device_id dw_mipi_dsi_rockchip_dt_ids[] = {
 	{
 	 .compatible = "rockchip,px30-mipi-dsi",
@@ -1622,6 +1666,9 @@ static const struct of_device_id dw_mipi_dsi_rockchip_dt_ids[] = {
 	}, {
 	 .compatible = "rockchip,rk3399-mipi-dsi",
 	 .data = &rk3399_chip_data,
+	}, {
+	 .compatible = "rockchip,rk3568-mipi-dsi",
+	 .data = &rk3568_chip_data,
 	},
 	{ /* sentinel */ }
 };
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
index 13ed33e74457..813f9f8c8698 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
@@ -18,7 +18,7 @@
 #include <drm/drm_aperture.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_of.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
index 985584147da1..614e97aaac80 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
@@ -10,7 +10,7 @@
 
 #include <drm/drm.h>
 #include <drm/drm_gem.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_prime.h>
 #include <drm/drm_vma_manager.h>
 
@@ -279,7 +279,7 @@ static const struct drm_gem_object_funcs rockchip_gem_object_funcs = {
 	.vmap = rockchip_gem_prime_vmap,
 	.vunmap	= rockchip_gem_prime_vunmap,
 	.mmap = rockchip_drm_gem_object_mmap,
-	.vm_ops = &drm_gem_cma_vm_ops,
+	.vm_ops = &drm_gem_dma_vm_ops,
 };
 
 static struct rockchip_gem_object *
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index ad3958b6f8bf..c356de5dd220 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -9,6 +9,7 @@
 #include <linux/delay.h>
 #include <linux/iopoll.h>
 #include <linux/kernel.h>
+#include <linux/log2.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -27,7 +28,6 @@
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_self_refresh_helper.h>
 #include <drm/drm_vblank.h>
@@ -68,6 +68,9 @@
 #define VOP_REG_SET(vop, group, name, v) \
 		    vop_reg_set(vop, &vop->data->group->name, 0, ~0, v, #name)
 
+#define VOP_HAS_REG(vop, group, name) \
+		(!!(vop->data->group->name.mask))
+
 #define VOP_INTR_SET_TYPE(vop, name, type, v) \
 	do { \
 		int i, reg = 0, mask = 0; \
@@ -185,12 +188,6 @@ struct vop {
 	struct vop_win win[];
 };
 
-static inline void vop_writel(struct vop *vop, uint32_t offset, uint32_t v)
-{
-	writel(v, vop->regs + offset);
-	vop->regsbak[offset >> 2] = v;
-}
-
 static inline uint32_t vop_readl(struct vop *vop, uint32_t offset)
 {
 	return readl(vop->regs + offset);
@@ -809,9 +806,9 @@ static int vop_plane_atomic_check(struct drm_plane *plane,
 	const struct vop_win_data *win = vop_win->data;
 	int ret;
 	int min_scale = win->phy->scl ? FRAC_16_16(1, 8) :
-					DRM_PLANE_HELPER_NO_SCALING;
+					DRM_PLANE_NO_SCALING;
 	int max_scale = win->phy->scl ? FRAC_16_16(8, 1) :
-					DRM_PLANE_HELPER_NO_SCALING;
+					DRM_PLANE_NO_SCALING;
 
 	if (!crtc || WARN_ON(!fb))
 		return 0;
@@ -1060,9 +1057,9 @@ static int vop_plane_atomic_async_check(struct drm_plane *plane,
 	struct vop_win *vop_win = to_vop_win(plane);
 	const struct vop_win_data *win = vop_win->data;
 	int min_scale = win->phy->scl ? FRAC_16_16(1, 8) :
-					DRM_PLANE_HELPER_NO_SCALING;
+					DRM_PLANE_NO_SCALING;
 	int max_scale = win->phy->scl ? FRAC_16_16(8, 1) :
-					DRM_PLANE_HELPER_NO_SCALING;
+					DRM_PLANE_NO_SCALING;
 	struct drm_crtc_state *crtc_state;
 
 	if (plane != new_plane_state->crtc->cursor)
@@ -1189,7 +1186,7 @@ static bool vop_crtc_mode_fixup(struct drm_crtc *crtc,
 	 *
 	 * Key points:
 	 *
-	 * - DRM works in in kHz.
+	 * - DRM works in kHz.
 	 * - Clock framework works in Hz.
 	 * - Rockchip's clock driver picks the clock rate that is the
 	 *   same _OR LOWER_ than the one requested.
@@ -1224,17 +1221,22 @@ static bool vop_dsp_lut_is_enabled(struct vop *vop)
 	return vop_read_reg(vop, 0, &vop->data->common->dsp_lut_en);
 }
 
+static u32 vop_lut_buffer_index(struct vop *vop)
+{
+	return vop_read_reg(vop, 0, &vop->data->common->lut_buffer_index);
+}
+
 static void vop_crtc_write_gamma_lut(struct vop *vop, struct drm_crtc *crtc)
 {
 	struct drm_color_lut *lut = crtc->state->gamma_lut->data;
-	unsigned int i;
+	unsigned int i, bpc = ilog2(vop->data->lut_size);
 
 	for (i = 0; i < crtc->gamma_size; i++) {
 		u32 word;
 
-		word = (drm_color_lut_extract(lut[i].red, 10) << 20) |
-		       (drm_color_lut_extract(lut[i].green, 10) << 10) |
-			drm_color_lut_extract(lut[i].blue, 10);
+		word = (drm_color_lut_extract(lut[i].red, bpc) << (2 * bpc)) |
+		       (drm_color_lut_extract(lut[i].green, bpc) << bpc) |
+			drm_color_lut_extract(lut[i].blue, bpc);
 		writel(word, vop->lut_regs + i * 4);
 	}
 }
@@ -1244,38 +1246,66 @@ static void vop_crtc_gamma_set(struct vop *vop, struct drm_crtc *crtc,
 {
 	struct drm_crtc_state *state = crtc->state;
 	unsigned int idle;
+	u32 lut_idx, old_idx;
 	int ret;
 
 	if (!vop->lut_regs)
 		return;
-	/*
-	 * To disable gamma (gamma_lut is null) or to write
-	 * an update to the LUT, clear dsp_lut_en.
-	 */
-	spin_lock(&vop->reg_lock);
-	VOP_REG_SET(vop, common, dsp_lut_en, 0);
-	vop_cfg_done(vop);
-	spin_unlock(&vop->reg_lock);
 
-	/*
-	 * In order to write the LUT to the internal memory,
-	 * we need to first make sure the dsp_lut_en bit is cleared.
-	 */
-	ret = readx_poll_timeout(vop_dsp_lut_is_enabled, vop,
-				 idle, !idle, 5, 30 * 1000);
-	if (ret) {
-		DRM_DEV_ERROR(vop->dev, "display LUT RAM enable timeout!\n");
-		return;
-	}
+	if (!state->gamma_lut || !VOP_HAS_REG(vop, common, update_gamma_lut)) {
+		/*
+		 * To disable gamma (gamma_lut is null) or to write
+		 * an update to the LUT, clear dsp_lut_en.
+		 */
+		spin_lock(&vop->reg_lock);
+		VOP_REG_SET(vop, common, dsp_lut_en, 0);
+		vop_cfg_done(vop);
+		spin_unlock(&vop->reg_lock);
 
-	if (!state->gamma_lut)
-		return;
+		/*
+		 * In order to write the LUT to the internal memory,
+		 * we need to first make sure the dsp_lut_en bit is cleared.
+		 */
+		ret = readx_poll_timeout(vop_dsp_lut_is_enabled, vop,
+					 idle, !idle, 5, 30 * 1000);
+		if (ret) {
+			DRM_DEV_ERROR(vop->dev, "display LUT RAM enable timeout!\n");
+			return;
+		}
+
+		if (!state->gamma_lut)
+			return;
+	} else {
+		/*
+		 * On RK3399 the gamma LUT can updated without clearing dsp_lut_en,
+		 * by setting update_gamma_lut then waiting for lut_buffer_index change
+		 */
+		old_idx = vop_lut_buffer_index(vop);
+	}
 
 	spin_lock(&vop->reg_lock);
 	vop_crtc_write_gamma_lut(vop, crtc);
 	VOP_REG_SET(vop, common, dsp_lut_en, 1);
+	VOP_REG_SET(vop, common, update_gamma_lut, 1);
 	vop_cfg_done(vop);
 	spin_unlock(&vop->reg_lock);
+
+	if (VOP_HAS_REG(vop, common, update_gamma_lut)) {
+		ret = readx_poll_timeout(vop_lut_buffer_index, vop,
+					 lut_idx, lut_idx != old_idx, 5, 30 * 1000);
+		if (ret) {
+			DRM_DEV_ERROR(vop->dev, "gamma LUT update timeout!\n");
+			return;
+		}
+
+		/*
+		 * update_gamma_lut is auto cleared by HW, but write 0 to clear the bit
+		 * in our backup of the regs.
+		 */
+		spin_lock(&vop->reg_lock);
+		VOP_REG_SET(vop, common, update_gamma_lut, 0);
+		spin_unlock(&vop->reg_lock);
+	}
 }
 
 static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
@@ -1325,14 +1355,6 @@ static void vop_crtc_atomic_enable(struct drm_crtc *crtc,
 		return;
 	}
 
-	/*
-	 * If we have a GAMMA LUT in the state, then let's make sure
-	 * it's updated. We might be coming out of suspend,
-	 * which means the LUT internal memory needs to be re-written.
-	 */
-	if (crtc->state->gamma_lut)
-		vop_crtc_gamma_set(vop, crtc, old_state);
-
 	mutex_lock(&vop->vop_lock);
 
 	WARN_ON(vop->event);
@@ -1423,6 +1445,14 @@ static void vop_crtc_atomic_enable(struct drm_crtc *crtc,
 
 	VOP_REG_SET(vop, common, standby, 0);
 	mutex_unlock(&vop->vop_lock);
+
+	/*
+	 * If we have a GAMMA LUT in the state, then let's make sure
+	 * it's updated. We might be coming out of suspend,
+	 * which means the LUT internal memory needs to be re-written.
+	 */
+	if (crtc->state->gamma_lut)
+		vop_crtc_gamma_set(vop, crtc, old_state);
 }
 
 static bool vop_fs_irq_is_pending(struct vop *vop)
@@ -2148,8 +2178,8 @@ static int vop_bind(struct device *dev, struct device *master, void *data)
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	if (res) {
-		if (!vop_data->lut_size) {
-			DRM_DEV_ERROR(dev, "no gamma LUT size defined\n");
+		if (vop_data->lut_size != 1024 && vop_data->lut_size != 256) {
+			DRM_DEV_ERROR(dev, "unsupported gamma LUT size %d\n", vop_data->lut_size);
 			return -EINVAL;
 		}
 		vop->lut_regs = devm_ioremap_resource(dev, res);
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
index ba88addc1a75..8502849833d9 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.h
@@ -113,6 +113,8 @@ struct vop_common {
 	struct vop_reg dither_down_en;
 	struct vop_reg dither_up;
 	struct vop_reg dsp_lut_en;
+	struct vop_reg update_gamma_lut;
+	struct vop_reg lut_buffer_index;
 	struct vop_reg gate_en;
 	struct vop_reg mmu_en;
 	struct vop_reg out_mode;
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
index f9aa8b96c695..aac20be5ac08 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
@@ -29,7 +29,6 @@
 #include <drm/drm_debugfs.h>
 #include <drm/drm_flip_work.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
index d03dd0402923..014f99e8928e 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
@@ -875,6 +875,24 @@ static const struct vop_output rk3399_output = {
 	.mipi_dual_channel_en = VOP_REG(RK3288_SYS_CTRL, 0x1, 3),
 };
 
+static const struct vop_common rk3399_common = {
+	.standby = VOP_REG_SYNC(RK3399_SYS_CTRL, 0x1, 22),
+	.gate_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 23),
+	.mmu_en = VOP_REG(RK3399_SYS_CTRL, 0x1, 20),
+	.dither_down_sel = VOP_REG(RK3399_DSP_CTRL1, 0x1, 4),
+	.dither_down_mode = VOP_REG(RK3399_DSP_CTRL1, 0x1, 3),
+	.dither_down_en = VOP_REG(RK3399_DSP_CTRL1, 0x1, 2),
+	.pre_dither_down = VOP_REG(RK3399_DSP_CTRL1, 0x1, 1),
+	.dither_up = VOP_REG(RK3399_DSP_CTRL1, 0x1, 6),
+	.dsp_lut_en = VOP_REG(RK3399_DSP_CTRL1, 0x1, 0),
+	.update_gamma_lut = VOP_REG(RK3399_DSP_CTRL1, 0x1, 7),
+	.lut_buffer_index = VOP_REG(RK3399_DBG_POST_REG1, 0x1, 1),
+	.data_blank = VOP_REG(RK3399_DSP_CTRL0, 0x1, 19),
+	.dsp_blank = VOP_REG(RK3399_DSP_CTRL0, 0x3, 18),
+	.out_mode = VOP_REG(RK3399_DSP_CTRL0, 0xf, 0),
+	.cfg_done = VOP_REG_SYNC(RK3399_REG_CFG_DONE, 0x1, 0),
+};
+
 static const struct vop_yuv2yuv_phy rk3399_yuv2yuv_win01_data = {
 	.y2r_coefficients = {
 		VOP_REG(RK3399_WIN0_YUV2YUV_Y2R + 0, 0xffff, 0),
@@ -957,7 +975,7 @@ static const struct vop_data rk3399_vop_big = {
 	.version = VOP_VERSION(3, 5),
 	.feature = VOP_FEATURE_OUTPUT_RGB10,
 	.intr = &rk3366_vop_intr,
-	.common = &rk3288_common,
+	.common = &rk3399_common,
 	.modeset = &rk3288_modeset,
 	.output = &rk3399_output,
 	.afbc = &rk3399_vop_afbc,
@@ -965,6 +983,7 @@ static const struct vop_data rk3399_vop_big = {
 	.win = rk3399_vop_win_data,
 	.win_size = ARRAY_SIZE(rk3399_vop_win_data),
 	.win_yuv2yuv = rk3399_vop_big_win_yuv2yuv_data,
+	.lut_size = 1024,
 };
 
 static const struct vop_win_data rk3399_vop_lit_win_data[] = {
@@ -983,13 +1002,14 @@ static const struct vop_win_yuv2yuv_data rk3399_vop_lit_win_yuv2yuv_data[] = {
 static const struct vop_data rk3399_vop_lit = {
 	.version = VOP_VERSION(3, 6),
 	.intr = &rk3366_vop_intr,
-	.common = &rk3288_common,
+	.common = &rk3399_common,
 	.modeset = &rk3288_modeset,
 	.output = &rk3399_output,
 	.misc = &rk3368_misc,
 	.win = rk3399_vop_lit_win_data,
 	.win_size = ARRAY_SIZE(rk3399_vop_lit_win_data),
 	.win_yuv2yuv = rk3399_vop_lit_win_yuv2yuv_data,
+	.lut_size = 256,
 };
 
 static const struct vop_win_data rk3228_vop_win_data[] = {
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
index 0b3cd65ba5c1..406e981c75bd 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.h
@@ -628,6 +628,7 @@
 #define RK3399_YUV2YUV_WIN			0x02c0
 #define RK3399_YUV2YUV_POST			0x02c4
 #define RK3399_AUTO_GATING_EN			0x02cc
+#define RK3399_DBG_POST_REG1			0x036c
 #define RK3399_WIN0_CSC_COE			0x03a0
 #define RK3399_WIN1_CSC_COE			0x03c0
 #define RK3399_WIN2_CSC_COE			0x03e0
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 68317d3a7a27..4f2395d1a791 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -198,7 +198,7 @@ static void drm_sched_job_done_cb(struct dma_fence *f, struct dma_fence_cb *cb)
 }
 
 /**
- * drm_sched_dependency_optimized
+ * drm_sched_dependency_optimized - test if the dependency can be optimized
  *
  * @fence: the dependency fence
  * @entity: the entity which depends on the above fence
@@ -592,7 +592,6 @@ int drm_sched_job_init(struct drm_sched_job *job,
 		       struct drm_sched_entity *entity,
 		       void *owner)
 {
-	drm_sched_entity_select_rq(entity);
 	if (!entity->rq)
 		return -ENOENT;
 
@@ -628,7 +627,7 @@ void drm_sched_job_arm(struct drm_sched_job *job)
 	struct drm_sched_entity *entity = job->entity;
 
 	BUG_ON(!entity);
-
+	drm_sched_entity_select_rq(entity);
 	sched = entity->rq->sched;
 
 	job->sched = sched;
@@ -830,7 +829,7 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
 	job = list_first_entry_or_null(&sched->pending_list,
 				       struct drm_sched_job, list);
 
-	if (job && dma_fence_is_signaled(&job->s_fence->finished)) {
+	if (job && dma_fence_is_signaled(job->s_fence->parent)) {
 		/* remove job from pending_list */
 		list_del_init(&job->list);
 
@@ -842,7 +841,7 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched)
 
 		if (next) {
 			next->s_fence->scheduled.timestamp =
-				job->s_fence->finished.timestamp;
+				job->s_fence->parent->timestamp;
 			/* start TO timer for next job */
 			drm_sched_start_timeout(sched);
 		}
@@ -994,6 +993,7 @@ static int drm_sched_main(void *param)
  *		used
  * @score: optional score atomic shared with other schedulers
  * @name: name used for debugging
+ * @dev: target &struct device
  *
  * Return 0 on success, otherwise error code.
  */
diff --git a/drivers/gpu/drm/selftests/Makefile b/drivers/gpu/drm/selftests/Makefile
deleted file mode 100644
index 5ba5f9138c95..000000000000
--- a/drivers/gpu/drm/selftests/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-test-drm_modeset-y := test-drm_modeset_common.o test-drm_plane_helper.o \
-                      test-drm_format.o test-drm_framebuffer.o \
-		      test-drm_damage_helper.o test-drm_dp_mst_helper.o \
-		      test-drm_rect.o
-
-obj-$(CONFIG_DRM_DEBUG_SELFTEST) += test-drm_mm.o test-drm_modeset.o test-drm_cmdline_parser.o \
-				    test-drm_buddy.o
diff --git a/drivers/gpu/drm/selftests/drm_buddy_selftests.h b/drivers/gpu/drm/selftests/drm_buddy_selftests.h
deleted file mode 100644
index 455b756c4ae5..000000000000
--- a/drivers/gpu/drm/selftests/drm_buddy_selftests.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* List each unit test as selftest(name, function)
- *
- * The name is used as both an enum and expanded as igt__name to create
- * a module parameter. It must be unique and legal for a C identifier.
- *
- * Tests are executed in order by igt/drm_buddy
- */
-selftest(sanitycheck, igt_sanitycheck) /* keep first (selfcheck for igt) */
-selftest(buddy_alloc_limit, igt_buddy_alloc_limit)
-selftest(buddy_alloc_range, igt_buddy_alloc_range)
-selftest(buddy_alloc_optimistic, igt_buddy_alloc_optimistic)
-selftest(buddy_alloc_pessimistic, igt_buddy_alloc_pessimistic)
-selftest(buddy_alloc_smoke, igt_buddy_alloc_smoke)
-selftest(buddy_alloc_pathological, igt_buddy_alloc_pathological)
diff --git a/drivers/gpu/drm/selftests/drm_cmdline_selftests.h b/drivers/gpu/drm/selftests/drm_cmdline_selftests.h
deleted file mode 100644
index 29e367db6118..000000000000
--- a/drivers/gpu/drm/selftests/drm_cmdline_selftests.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* List each unit test as selftest(function)
- *
- * The name is used as both an enum and expanded as igt__name to create
- * a module parameter. It must be unique and legal for a C identifier.
- *
- * Tests are executed in order by igt/drm_mm
- */
-
-#define cmdline_test(test)	selftest(test, test)
-
-cmdline_test(drm_cmdline_test_force_d_only)
-cmdline_test(drm_cmdline_test_force_D_only_dvi)
-cmdline_test(drm_cmdline_test_force_D_only_hdmi)
-cmdline_test(drm_cmdline_test_force_D_only_not_digital)
-cmdline_test(drm_cmdline_test_force_e_only)
-cmdline_test(drm_cmdline_test_margin_only)
-cmdline_test(drm_cmdline_test_interlace_only)
-cmdline_test(drm_cmdline_test_res)
-cmdline_test(drm_cmdline_test_res_missing_x)
-cmdline_test(drm_cmdline_test_res_missing_y)
-cmdline_test(drm_cmdline_test_res_bad_y)
-cmdline_test(drm_cmdline_test_res_missing_y_bpp)
-cmdline_test(drm_cmdline_test_res_vesa)
-cmdline_test(drm_cmdline_test_res_vesa_rblank)
-cmdline_test(drm_cmdline_test_res_rblank)
-cmdline_test(drm_cmdline_test_res_bpp)
-cmdline_test(drm_cmdline_test_res_bad_bpp)
-cmdline_test(drm_cmdline_test_res_refresh)
-cmdline_test(drm_cmdline_test_res_bad_refresh)
-cmdline_test(drm_cmdline_test_res_bpp_refresh)
-cmdline_test(drm_cmdline_test_res_bpp_refresh_interlaced)
-cmdline_test(drm_cmdline_test_res_bpp_refresh_margins)
-cmdline_test(drm_cmdline_test_res_bpp_refresh_force_off)
-cmdline_test(drm_cmdline_test_res_bpp_refresh_force_on_off)
-cmdline_test(drm_cmdline_test_res_bpp_refresh_force_on)
-cmdline_test(drm_cmdline_test_res_bpp_refresh_force_on_analog)
-cmdline_test(drm_cmdline_test_res_bpp_refresh_force_on_digital)
-cmdline_test(drm_cmdline_test_res_bpp_refresh_interlaced_margins_force_on)
-cmdline_test(drm_cmdline_test_res_margins_force_on)
-cmdline_test(drm_cmdline_test_res_vesa_margins)
-cmdline_test(drm_cmdline_test_res_invalid_mode)
-cmdline_test(drm_cmdline_test_res_bpp_wrong_place_mode)
-cmdline_test(drm_cmdline_test_name)
-cmdline_test(drm_cmdline_test_name_bpp)
-cmdline_test(drm_cmdline_test_name_refresh)
-cmdline_test(drm_cmdline_test_name_bpp_refresh)
-cmdline_test(drm_cmdline_test_name_refresh_wrong_mode)
-cmdline_test(drm_cmdline_test_name_refresh_invalid_mode)
-cmdline_test(drm_cmdline_test_name_option)
-cmdline_test(drm_cmdline_test_name_bpp_option)
-cmdline_test(drm_cmdline_test_rotate_0)
-cmdline_test(drm_cmdline_test_rotate_90)
-cmdline_test(drm_cmdline_test_rotate_180)
-cmdline_test(drm_cmdline_test_rotate_270)
-cmdline_test(drm_cmdline_test_rotate_multiple)
-cmdline_test(drm_cmdline_test_rotate_invalid_val)
-cmdline_test(drm_cmdline_test_rotate_truncated)
-cmdline_test(drm_cmdline_test_hmirror)
-cmdline_test(drm_cmdline_test_vmirror)
-cmdline_test(drm_cmdline_test_margin_options)
-cmdline_test(drm_cmdline_test_multiple_options)
-cmdline_test(drm_cmdline_test_invalid_option)
-cmdline_test(drm_cmdline_test_bpp_extra_and_option)
-cmdline_test(drm_cmdline_test_extra_and_option)
-cmdline_test(drm_cmdline_test_freestanding_options)
-cmdline_test(drm_cmdline_test_freestanding_force_e_and_options)
-cmdline_test(drm_cmdline_test_panel_orientation)
diff --git a/drivers/gpu/drm/selftests/drm_mm_selftests.h b/drivers/gpu/drm/selftests/drm_mm_selftests.h
deleted file mode 100644
index 8c87c964176b..000000000000
--- a/drivers/gpu/drm/selftests/drm_mm_selftests.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* List each unit test as selftest(name, function)
- *
- * The name is used as both an enum and expanded as igt__name to create
- * a module parameter. It must be unique and legal for a C identifier.
- *
- * Tests are executed in order by igt/drm_mm
- */
-selftest(sanitycheck, igt_sanitycheck) /* keep first (selfcheck for igt) */
-selftest(init, igt_init)
-selftest(debug, igt_debug)
-selftest(reserve, igt_reserve)
-selftest(insert, igt_insert)
-selftest(replace, igt_replace)
-selftest(insert_range, igt_insert_range)
-selftest(align, igt_align)
-selftest(frag, igt_frag)
-selftest(align32, igt_align32)
-selftest(align64, igt_align64)
-selftest(evict, igt_evict)
-selftest(evict_range, igt_evict_range)
-selftest(bottomup, igt_bottomup)
-selftest(lowest, igt_lowest)
-selftest(topdown, igt_topdown)
-selftest(highest, igt_highest)
-selftest(color, igt_color)
-selftest(color_evict, igt_color_evict)
-selftest(color_evict_range, igt_color_evict_range)
diff --git a/drivers/gpu/drm/selftests/drm_modeset_selftests.h b/drivers/gpu/drm/selftests/drm_modeset_selftests.h
deleted file mode 100644
index 782e285ca383..000000000000
--- a/drivers/gpu/drm/selftests/drm_modeset_selftests.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* List each unit test as selftest(name, function)
- *
- * The name is used as both an enum and expanded as igt__name to create
- * a module parameter. It must be unique and legal for a C identifier.
- *
- * Tests are executed in order by igt/drm_selftests_helper
- */
-selftest(drm_rect_clip_scaled_div_by_zero, igt_drm_rect_clip_scaled_div_by_zero)
-selftest(drm_rect_clip_scaled_not_clipped, igt_drm_rect_clip_scaled_not_clipped)
-selftest(drm_rect_clip_scaled_clipped, igt_drm_rect_clip_scaled_clipped)
-selftest(drm_rect_clip_scaled_signed_vs_unsigned, igt_drm_rect_clip_scaled_signed_vs_unsigned)
-selftest(check_plane_state, igt_check_plane_state)
-selftest(check_drm_format_block_width, igt_check_drm_format_block_width)
-selftest(check_drm_format_block_height, igt_check_drm_format_block_height)
-selftest(check_drm_format_min_pitch, igt_check_drm_format_min_pitch)
-selftest(check_drm_framebuffer_create, igt_check_drm_framebuffer_create)
-selftest(damage_iter_no_damage, igt_damage_iter_no_damage)
-selftest(damage_iter_no_damage_fractional_src, igt_damage_iter_no_damage_fractional_src)
-selftest(damage_iter_no_damage_src_moved, igt_damage_iter_no_damage_src_moved)
-selftest(damage_iter_no_damage_fractional_src_moved, igt_damage_iter_no_damage_fractional_src_moved)
-selftest(damage_iter_no_damage_not_visible, igt_damage_iter_no_damage_not_visible)
-selftest(damage_iter_no_damage_no_crtc, igt_damage_iter_no_damage_no_crtc)
-selftest(damage_iter_no_damage_no_fb, igt_damage_iter_no_damage_no_fb)
-selftest(damage_iter_simple_damage, igt_damage_iter_simple_damage)
-selftest(damage_iter_single_damage, igt_damage_iter_single_damage)
-selftest(damage_iter_single_damage_intersect_src, igt_damage_iter_single_damage_intersect_src)
-selftest(damage_iter_single_damage_outside_src, igt_damage_iter_single_damage_outside_src)
-selftest(damage_iter_single_damage_fractional_src, igt_damage_iter_single_damage_fractional_src)
-selftest(damage_iter_single_damage_intersect_fractional_src, igt_damage_iter_single_damage_intersect_fractional_src)
-selftest(damage_iter_single_damage_outside_fractional_src, igt_damage_iter_single_damage_outside_fractional_src)
-selftest(damage_iter_single_damage_src_moved, igt_damage_iter_single_damage_src_moved)
-selftest(damage_iter_single_damage_fractional_src_moved, igt_damage_iter_single_damage_fractional_src_moved)
-selftest(damage_iter_damage, igt_damage_iter_damage)
-selftest(damage_iter_damage_one_intersect, igt_damage_iter_damage_one_intersect)
-selftest(damage_iter_damage_one_outside, igt_damage_iter_damage_one_outside)
-selftest(damage_iter_damage_src_moved, igt_damage_iter_damage_src_moved)
-selftest(damage_iter_damage_not_visible, igt_damage_iter_damage_not_visible)
-selftest(dp_mst_calc_pbn_mode, igt_dp_mst_calc_pbn_mode)
-selftest(dp_mst_sideband_msg_req_decode, igt_dp_mst_sideband_msg_req_decode)
diff --git a/drivers/gpu/drm/selftests/drm_selftest.c b/drivers/gpu/drm/selftests/drm_selftest.c
deleted file mode 100644
index e29ed9faef5b..000000000000
--- a/drivers/gpu/drm/selftests/drm_selftest.c
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Copyright © 2016 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#include <linux/compiler.h>
-
-#define selftest(name, func) __idx_##name,
-enum {
-#include TESTS
-};
-#undef selftest
-
-#define selftest(n, f) [__idx_##n] = { .name = #n, .func = f },
-static struct drm_selftest {
-	bool enabled;
-	const char *name;
-	int (*func)(void *);
-} selftests[] = {
-#include TESTS
-};
-#undef selftest
-
-/* Embed the line number into the parameter name so that we can order tests */
-#define param(n) __PASTE(igt__, __PASTE(__PASTE(__LINE__, __), n))
-#define selftest_0(n, func, id) \
-module_param_named(id, selftests[__idx_##n].enabled, bool, 0400);
-#define selftest(n, func) selftest_0(n, func, param(n))
-#include TESTS
-#undef selftest
-
-static void set_default_test_all(struct drm_selftest *st, unsigned long count)
-{
-	unsigned long i;
-
-	for (i = 0; i < count; i++)
-		if (st[i].enabled)
-			return;
-
-	for (i = 0; i < count; i++)
-		st[i].enabled = true;
-}
-
-static int run_selftests(struct drm_selftest *st,
-			 unsigned long count,
-			 void *data)
-{
-	int err = 0;
-
-	set_default_test_all(st, count);
-
-	/* Tests are listed in natural order in drm_*_selftests.h */
-	for (; count--; st++) {
-		if (!st->enabled)
-			continue;
-
-		pr_debug("drm: Running %s\n", st->name);
-		err = st->func(data);
-		if (err)
-			break;
-	}
-
-	if (WARN(err > 0 || err == -ENOTTY,
-		 "%s returned %d, conflicting with selftest's magic values!\n",
-		 st->name, err))
-		err = -1;
-
-	rcu_barrier();
-	return err;
-}
-
-static int __maybe_unused
-__drm_subtests(const char *caller,
-	       const struct drm_subtest *st,
-	       int count,
-	       void *data)
-{
-	int err;
-
-	for (; count--; st++) {
-		pr_debug("Running %s/%s\n", caller, st->name);
-		err = st->func(data);
-		if (err) {
-			pr_err("%s: %s failed with error %d\n",
-			       caller, st->name, err);
-			return err;
-		}
-	}
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/selftests/drm_selftest.h b/drivers/gpu/drm/selftests/drm_selftest.h
deleted file mode 100644
index c784ec02ff53..000000000000
--- a/drivers/gpu/drm/selftests/drm_selftest.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright © 2016 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
- */
-
-#ifndef __DRM_SELFTEST_H__
-#define __DRM_SELFTEST_H__
-
-struct drm_subtest {
-	int (*func)(void *data);
-	const char *name;
-};
-
-static int __drm_subtests(const char *caller,
-			  const struct drm_subtest *st,
-			  int count,
-			  void *data);
-#define drm_subtests(T, data) \
-	__drm_subtests(__func__, T, ARRAY_SIZE(T), data)
-
-#define SUBTEST(x) { x, #x }
-
-#endif /* __DRM_SELFTEST_H__ */
diff --git a/drivers/gpu/drm/selftests/test-drm_buddy.c b/drivers/gpu/drm/selftests/test-drm_buddy.c
deleted file mode 100644
index aca0c491040f..000000000000
--- a/drivers/gpu/drm/selftests/test-drm_buddy.c
+++ /dev/null
@@ -1,994 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2019 Intel Corporation
- */
-
-#define pr_fmt(fmt) "drm_buddy: " fmt
-
-#include <linux/module.h>
-#include <linux/prime_numbers.h>
-#include <linux/sched/signal.h>
-
-#include <drm/drm_buddy.h>
-
-#include "../lib/drm_random.h"
-
-#define TESTS "drm_buddy_selftests.h"
-#include "drm_selftest.h"
-
-#define IGT_TIMEOUT(name__) \
-	unsigned long name__ = jiffies + MAX_SCHEDULE_TIMEOUT
-
-static unsigned int random_seed;
-
-static inline u64 get_size(int order, u64 chunk_size)
-{
-	return (1 << order) * chunk_size;
-}
-
-__printf(2, 3)
-static bool __igt_timeout(unsigned long timeout, const char *fmt, ...)
-{
-	va_list va;
-
-	if (!signal_pending(current)) {
-		cond_resched();
-		if (time_before(jiffies, timeout))
-			return false;
-	}
-
-	if (fmt) {
-		va_start(va, fmt);
-		vprintk(fmt, va);
-		va_end(va);
-	}
-
-	return true;
-}
-
-static inline const char *yesno(bool v)
-{
-	return v ? "yes" : "no";
-}
-
-static void __igt_dump_block(struct drm_buddy *mm,
-			     struct drm_buddy_block *block,
-			     bool buddy)
-{
-	pr_err("block info: header=%llx, state=%u, order=%d, offset=%llx size=%llx root=%s buddy=%s\n",
-	       block->header,
-	       drm_buddy_block_state(block),
-	       drm_buddy_block_order(block),
-	       drm_buddy_block_offset(block),
-	       drm_buddy_block_size(mm, block),
-	       yesno(!block->parent),
-	       yesno(buddy));
-}
-
-static void igt_dump_block(struct drm_buddy *mm,
-			   struct drm_buddy_block *block)
-{
-	struct drm_buddy_block *buddy;
-
-	__igt_dump_block(mm, block, false);
-
-	buddy = drm_get_buddy(block);
-	if (buddy)
-		__igt_dump_block(mm, buddy, true);
-}
-
-static int igt_check_block(struct drm_buddy *mm,
-			   struct drm_buddy_block *block)
-{
-	struct drm_buddy_block *buddy;
-	unsigned int block_state;
-	u64 block_size;
-	u64 offset;
-	int err = 0;
-
-	block_state = drm_buddy_block_state(block);
-
-	if (block_state != DRM_BUDDY_ALLOCATED &&
-	    block_state != DRM_BUDDY_FREE &&
-	    block_state != DRM_BUDDY_SPLIT) {
-		pr_err("block state mismatch\n");
-		err = -EINVAL;
-	}
-
-	block_size = drm_buddy_block_size(mm, block);
-	offset = drm_buddy_block_offset(block);
-
-	if (block_size < mm->chunk_size) {
-		pr_err("block size smaller than min size\n");
-		err = -EINVAL;
-	}
-
-	if (!is_power_of_2(block_size)) {
-		pr_err("block size not power of two\n");
-		err = -EINVAL;
-	}
-
-	if (!IS_ALIGNED(block_size, mm->chunk_size)) {
-		pr_err("block size not aligned to min size\n");
-		err = -EINVAL;
-	}
-
-	if (!IS_ALIGNED(offset, mm->chunk_size)) {
-		pr_err("block offset not aligned to min size\n");
-		err = -EINVAL;
-	}
-
-	if (!IS_ALIGNED(offset, block_size)) {
-		pr_err("block offset not aligned to block size\n");
-		err = -EINVAL;
-	}
-
-	buddy = drm_get_buddy(block);
-
-	if (!buddy && block->parent) {
-		pr_err("buddy has gone fishing\n");
-		err = -EINVAL;
-	}
-
-	if (buddy) {
-		if (drm_buddy_block_offset(buddy) != (offset ^ block_size)) {
-			pr_err("buddy has wrong offset\n");
-			err = -EINVAL;
-		}
-
-		if (drm_buddy_block_size(mm, buddy) != block_size) {
-			pr_err("buddy size mismatch\n");
-			err = -EINVAL;
-		}
-
-		if (drm_buddy_block_state(buddy) == block_state &&
-		    block_state == DRM_BUDDY_FREE) {
-			pr_err("block and its buddy are free\n");
-			err = -EINVAL;
-		}
-	}
-
-	return err;
-}
-
-static int igt_check_blocks(struct drm_buddy *mm,
-			    struct list_head *blocks,
-			    u64 expected_size,
-			    bool is_contiguous)
-{
-	struct drm_buddy_block *block;
-	struct drm_buddy_block *prev;
-	u64 total;
-	int err = 0;
-
-	block = NULL;
-	prev = NULL;
-	total = 0;
-
-	list_for_each_entry(block, blocks, link) {
-		err = igt_check_block(mm, block);
-
-		if (!drm_buddy_block_is_allocated(block)) {
-			pr_err("block not allocated\n"),
-			       err = -EINVAL;
-		}
-
-		if (is_contiguous && prev) {
-			u64 prev_block_size;
-			u64 prev_offset;
-			u64 offset;
-
-			prev_offset = drm_buddy_block_offset(prev);
-			prev_block_size = drm_buddy_block_size(mm, prev);
-			offset = drm_buddy_block_offset(block);
-
-			if (offset != (prev_offset + prev_block_size)) {
-				pr_err("block offset mismatch\n");
-				err = -EINVAL;
-			}
-		}
-
-		if (err)
-			break;
-
-		total += drm_buddy_block_size(mm, block);
-		prev = block;
-	}
-
-	if (!err) {
-		if (total != expected_size) {
-			pr_err("size mismatch, expected=%llx, found=%llx\n",
-			       expected_size, total);
-			err = -EINVAL;
-		}
-		return err;
-	}
-
-	if (prev) {
-		pr_err("prev block, dump:\n");
-		igt_dump_block(mm, prev);
-	}
-
-	pr_err("bad block, dump:\n");
-	igt_dump_block(mm, block);
-
-	return err;
-}
-
-static int igt_check_mm(struct drm_buddy *mm)
-{
-	struct drm_buddy_block *root;
-	struct drm_buddy_block *prev;
-	unsigned int i;
-	u64 total;
-	int err = 0;
-
-	if (!mm->n_roots) {
-		pr_err("n_roots is zero\n");
-		return -EINVAL;
-	}
-
-	if (mm->n_roots != hweight64(mm->size)) {
-		pr_err("n_roots mismatch, n_roots=%u, expected=%lu\n",
-		       mm->n_roots, hweight64(mm->size));
-		return -EINVAL;
-	}
-
-	root = NULL;
-	prev = NULL;
-	total = 0;
-
-	for (i = 0; i < mm->n_roots; ++i) {
-		struct drm_buddy_block *block;
-		unsigned int order;
-
-		root = mm->roots[i];
-		if (!root) {
-			pr_err("root(%u) is NULL\n", i);
-			err = -EINVAL;
-			break;
-		}
-
-		err = igt_check_block(mm, root);
-
-		if (!drm_buddy_block_is_free(root)) {
-			pr_err("root not free\n");
-			err = -EINVAL;
-		}
-
-		order = drm_buddy_block_order(root);
-
-		if (!i) {
-			if (order != mm->max_order) {
-				pr_err("max order root missing\n");
-				err = -EINVAL;
-			}
-		}
-
-		if (prev) {
-			u64 prev_block_size;
-			u64 prev_offset;
-			u64 offset;
-
-			prev_offset = drm_buddy_block_offset(prev);
-			prev_block_size = drm_buddy_block_size(mm, prev);
-			offset = drm_buddy_block_offset(root);
-
-			if (offset != (prev_offset + prev_block_size)) {
-				pr_err("root offset mismatch\n");
-				err = -EINVAL;
-			}
-		}
-
-		block = list_first_entry_or_null(&mm->free_list[order],
-						 struct drm_buddy_block,
-						 link);
-		if (block != root) {
-			pr_err("root mismatch at order=%u\n", order);
-			err = -EINVAL;
-		}
-
-		if (err)
-			break;
-
-		prev = root;
-		total += drm_buddy_block_size(mm, root);
-	}
-
-	if (!err) {
-		if (total != mm->size) {
-			pr_err("expected mm size=%llx, found=%llx\n", mm->size,
-			       total);
-			err = -EINVAL;
-		}
-		return err;
-	}
-
-	if (prev) {
-		pr_err("prev root(%u), dump:\n", i - 1);
-		igt_dump_block(mm, prev);
-	}
-
-	if (root) {
-		pr_err("bad root(%u), dump:\n", i);
-		igt_dump_block(mm, root);
-	}
-
-	return err;
-}
-
-static void igt_mm_config(u64 *size, u64 *chunk_size)
-{
-	DRM_RND_STATE(prng, random_seed);
-	u32 s, ms;
-
-	/* Nothing fancy, just try to get an interesting bit pattern */
-
-	prandom_seed_state(&prng, random_seed);
-
-	/* Let size be a random number of pages up to 8 GB (2M pages) */
-	s = 1 + drm_prandom_u32_max_state((BIT(33 - 12)) - 1, &prng);
-	/* Let the chunk size be a random power of 2 less than size */
-	ms = BIT(drm_prandom_u32_max_state(ilog2(s), &prng));
-	/* Round size down to the chunk size */
-	s &= -ms;
-
-	/* Convert from pages to bytes */
-	*chunk_size = (u64)ms << 12;
-	*size = (u64)s << 12;
-}
-
-static int igt_buddy_alloc_pathological(void *arg)
-{
-	u64 mm_size, size, min_page_size, start = 0;
-	struct drm_buddy_block *block;
-	const int max_order = 3;
-	unsigned long flags = 0;
-	int order, top, err;
-	struct drm_buddy mm;
-	LIST_HEAD(blocks);
-	LIST_HEAD(holes);
-	LIST_HEAD(tmp);
-
-	/*
-	 * Create a pot-sized mm, then allocate one of each possible
-	 * order within. This should leave the mm with exactly one
-	 * page left. Free the largest block, then whittle down again.
-	 * Eventually we will have a fully 50% fragmented mm.
-	 */
-
-	mm_size = PAGE_SIZE << max_order;
-	err = drm_buddy_init(&mm, mm_size, PAGE_SIZE);
-	if (err) {
-		pr_err("buddy_init failed(%d)\n", err);
-		return err;
-	}
-	BUG_ON(mm.max_order != max_order);
-
-	for (top = max_order; top; top--) {
-		/* Make room by freeing the largest allocated block */
-		block = list_first_entry_or_null(&blocks, typeof(*block), link);
-		if (block) {
-			list_del(&block->link);
-			drm_buddy_free_block(&mm, block);
-		}
-
-		for (order = top; order--; ) {
-			size = min_page_size = get_size(order, PAGE_SIZE);
-			err = drm_buddy_alloc_blocks(&mm, start, mm_size, size,
-						     min_page_size, &tmp, flags);
-			if (err) {
-				pr_info("buddy_alloc hit -ENOMEM with order=%d, top=%d\n",
-					order, top);
-				goto err;
-			}
-
-			block = list_first_entry_or_null(&tmp,
-							 struct drm_buddy_block,
-							 link);
-			if (!block) {
-				pr_err("alloc_blocks has no blocks\n");
-				err = -EINVAL;
-				goto err;
-			}
-
-			list_move_tail(&block->link, &blocks);
-		}
-
-		/* There should be one final page for this sub-allocation */
-		size = min_page_size = get_size(0, PAGE_SIZE);
-		err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, min_page_size, &tmp, flags);
-		if (err) {
-			pr_info("buddy_alloc hit -ENOMEM for hole\n");
-			goto err;
-		}
-
-		block = list_first_entry_or_null(&tmp,
-						 struct drm_buddy_block,
-						 link);
-		if (!block) {
-			pr_err("alloc_blocks has no blocks\n");
-			err = -EINVAL;
-			goto err;
-		}
-
-		list_move_tail(&block->link, &holes);
-
-		size = min_page_size = get_size(top, PAGE_SIZE);
-		err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, min_page_size, &tmp, flags);
-		if (!err) {
-			pr_info("buddy_alloc unexpectedly succeeded at top-order %d/%d, it should be full!",
-				top, max_order);
-			block = list_first_entry_or_null(&tmp,
-							 struct drm_buddy_block,
-							 link);
-			if (!block) {
-				pr_err("alloc_blocks has no blocks\n");
-				err = -EINVAL;
-				goto err;
-			}
-
-			list_move_tail(&block->link, &blocks);
-			err = -EINVAL;
-			goto err;
-		}
-	}
-
-	drm_buddy_free_list(&mm, &holes);
-
-	/* Nothing larger than blocks of chunk_size now available */
-	for (order = 1; order <= max_order; order++) {
-		size = min_page_size = get_size(order, PAGE_SIZE);
-		err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, min_page_size, &tmp, flags);
-		if (!err) {
-			pr_info("buddy_alloc unexpectedly succeeded at order %d, it should be full!",
-				order);
-			block = list_first_entry_or_null(&tmp,
-							 struct drm_buddy_block,
-							 link);
-			if (!block) {
-				pr_err("alloc_blocks has no blocks\n");
-				err = -EINVAL;
-				goto err;
-			}
-
-			list_move_tail(&block->link, &blocks);
-			err = -EINVAL;
-			goto err;
-		}
-	}
-
-	if (err)
-		err = 0;
-
-err:
-	list_splice_tail(&holes, &blocks);
-	drm_buddy_free_list(&mm, &blocks);
-	drm_buddy_fini(&mm);
-	return err;
-}
-
-static int igt_buddy_alloc_smoke(void *arg)
-{
-	u64 mm_size, min_page_size, chunk_size, start = 0;
-	unsigned long flags = 0;
-	struct drm_buddy mm;
-	int *order;
-	int err, i;
-
-	DRM_RND_STATE(prng, random_seed);
-	IGT_TIMEOUT(end_time);
-
-	igt_mm_config(&mm_size, &chunk_size);
-
-	err = drm_buddy_init(&mm, mm_size, chunk_size);
-	if (err) {
-		pr_err("buddy_init failed(%d)\n", err);
-		return err;
-	}
-
-	order = drm_random_order(mm.max_order + 1, &prng);
-	if (!order) {
-		err = -ENOMEM;
-		goto out_fini;
-	}
-
-	for (i = 0; i <= mm.max_order; ++i) {
-		struct drm_buddy_block *block;
-		int max_order = order[i];
-		bool timeout = false;
-		LIST_HEAD(blocks);
-		u64 total, size;
-		LIST_HEAD(tmp);
-		int order;
-
-		err = igt_check_mm(&mm);
-		if (err) {
-			pr_err("pre-mm check failed, abort\n");
-			break;
-		}
-
-		order = max_order;
-		total = 0;
-
-		do {
-retry:
-			size = min_page_size = get_size(order, chunk_size);
-			err = drm_buddy_alloc_blocks(&mm, start, mm_size, size,
-						     min_page_size, &tmp, flags);
-			if (err) {
-				if (err == -ENOMEM) {
-					pr_info("buddy_alloc hit -ENOMEM with order=%d\n",
-						order);
-				} else {
-					if (order--) {
-						err = 0;
-						goto retry;
-					}
-
-					pr_err("buddy_alloc with order=%d failed(%d)\n",
-					       order, err);
-				}
-
-				break;
-			}
-
-			block = list_first_entry_or_null(&tmp,
-							 struct drm_buddy_block,
-							 link);
-			if (!block) {
-				pr_err("alloc_blocks has no blocks\n");
-				err = -EINVAL;
-				break;
-			}
-
-			list_move_tail(&block->link, &blocks);
-
-			if (drm_buddy_block_order(block) != order) {
-				pr_err("buddy_alloc order mismatch\n");
-				err = -EINVAL;
-				break;
-			}
-
-			total += drm_buddy_block_size(&mm, block);
-
-			if (__igt_timeout(end_time, NULL)) {
-				timeout = true;
-				break;
-			}
-		} while (total < mm.size);
-
-		if (!err)
-			err = igt_check_blocks(&mm, &blocks, total, false);
-
-		drm_buddy_free_list(&mm, &blocks);
-
-		if (!err) {
-			err = igt_check_mm(&mm);
-			if (err)
-				pr_err("post-mm check failed\n");
-		}
-
-		if (err || timeout)
-			break;
-
-		cond_resched();
-	}
-
-	if (err == -ENOMEM)
-		err = 0;
-
-	kfree(order);
-out_fini:
-	drm_buddy_fini(&mm);
-
-	return err;
-}
-
-static int igt_buddy_alloc_pessimistic(void *arg)
-{
-	u64 mm_size, size, min_page_size, start = 0;
-	struct drm_buddy_block *block, *bn;
-	const unsigned int max_order = 16;
-	unsigned long flags = 0;
-	struct drm_buddy mm;
-	unsigned int order;
-	LIST_HEAD(blocks);
-	LIST_HEAD(tmp);
-	int err;
-
-	/*
-	 * Create a pot-sized mm, then allocate one of each possible
-	 * order within. This should leave the mm with exactly one
-	 * page left.
-	 */
-
-	mm_size = PAGE_SIZE << max_order;
-	err = drm_buddy_init(&mm, mm_size, PAGE_SIZE);
-	if (err) {
-		pr_err("buddy_init failed(%d)\n", err);
-		return err;
-	}
-	BUG_ON(mm.max_order != max_order);
-
-	for (order = 0; order < max_order; order++) {
-		size = min_page_size = get_size(order, PAGE_SIZE);
-		err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, min_page_size, &tmp, flags);
-		if (err) {
-			pr_info("buddy_alloc hit -ENOMEM with order=%d\n",
-				order);
-			goto err;
-		}
-
-		block = list_first_entry_or_null(&tmp,
-						 struct drm_buddy_block,
-						 link);
-		if (!block) {
-			pr_err("alloc_blocks has no blocks\n");
-			err = -EINVAL;
-			goto err;
-		}
-
-		list_move_tail(&block->link, &blocks);
-	}
-
-	/* And now the last remaining block available */
-	size = min_page_size = get_size(0, PAGE_SIZE);
-	err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, min_page_size, &tmp, flags);
-	if (err) {
-		pr_info("buddy_alloc hit -ENOMEM on final alloc\n");
-		goto err;
-	}
-
-	block = list_first_entry_or_null(&tmp,
-					 struct drm_buddy_block,
-					 link);
-	if (!block) {
-		pr_err("alloc_blocks has no blocks\n");
-		err = -EINVAL;
-		goto err;
-	}
-
-	list_move_tail(&block->link, &blocks);
-
-	/* Should be completely full! */
-	for (order = max_order; order--; ) {
-		size = min_page_size = get_size(order, PAGE_SIZE);
-		err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, min_page_size, &tmp, flags);
-		if (!err) {
-			pr_info("buddy_alloc unexpectedly succeeded at order %d, it should be full!",
-				order);
-			block = list_first_entry_or_null(&tmp,
-							 struct drm_buddy_block,
-							 link);
-			if (!block) {
-				pr_err("alloc_blocks has no blocks\n");
-				err = -EINVAL;
-				goto err;
-			}
-
-			list_move_tail(&block->link, &blocks);
-			err = -EINVAL;
-			goto err;
-		}
-	}
-
-	block = list_last_entry(&blocks, typeof(*block), link);
-	list_del(&block->link);
-	drm_buddy_free_block(&mm, block);
-
-	/* As we free in increasing size, we make available larger blocks */
-	order = 1;
-	list_for_each_entry_safe(block, bn, &blocks, link) {
-		list_del(&block->link);
-		drm_buddy_free_block(&mm, block);
-
-		size = min_page_size = get_size(order, PAGE_SIZE);
-		err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, min_page_size, &tmp, flags);
-		if (err) {
-			pr_info("buddy_alloc (realloc) hit -ENOMEM with order=%d\n",
-				order);
-			goto err;
-		}
-
-		block = list_first_entry_or_null(&tmp,
-						 struct drm_buddy_block,
-						 link);
-		if (!block) {
-			pr_err("alloc_blocks has no blocks\n");
-			err = -EINVAL;
-			goto err;
-		}
-
-		list_del(&block->link);
-		drm_buddy_free_block(&mm, block);
-		order++;
-	}
-
-	/* To confirm, now the whole mm should be available */
-	size = min_page_size = get_size(max_order, PAGE_SIZE);
-	err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, min_page_size, &tmp, flags);
-	if (err) {
-		pr_info("buddy_alloc (realloc) hit -ENOMEM with order=%d\n",
-			max_order);
-		goto err;
-	}
-
-	block = list_first_entry_or_null(&tmp,
-					 struct drm_buddy_block,
-					 link);
-	if (!block) {
-		pr_err("alloc_blocks has no blocks\n");
-		err = -EINVAL;
-		goto err;
-	}
-
-	list_del(&block->link);
-	drm_buddy_free_block(&mm, block);
-
-err:
-	drm_buddy_free_list(&mm, &blocks);
-	drm_buddy_fini(&mm);
-	return err;
-}
-
-static int igt_buddy_alloc_optimistic(void *arg)
-{
-	u64 mm_size, size, min_page_size, start = 0;
-	struct drm_buddy_block *block;
-	unsigned long flags = 0;
-	const int max_order = 16;
-	struct drm_buddy mm;
-	LIST_HEAD(blocks);
-	LIST_HEAD(tmp);
-	int order, err;
-
-	/*
-	 * Create a mm with one block of each order available, and
-	 * try to allocate them all.
-	 */
-
-	mm_size = PAGE_SIZE * ((1 << (max_order + 1)) - 1);
-	err = drm_buddy_init(&mm,
-			     mm_size,
-			     PAGE_SIZE);
-	if (err) {
-		pr_err("buddy_init failed(%d)\n", err);
-		return err;
-	}
-
-	BUG_ON(mm.max_order != max_order);
-
-	for (order = 0; order <= max_order; order++) {
-		size = min_page_size = get_size(order, PAGE_SIZE);
-		err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, min_page_size, &tmp, flags);
-		if (err) {
-			pr_info("buddy_alloc hit -ENOMEM with order=%d\n",
-				order);
-			goto err;
-		}
-
-		block = list_first_entry_or_null(&tmp,
-						 struct drm_buddy_block,
-						 link);
-		if (!block) {
-			pr_err("alloc_blocks has no blocks\n");
-			err = -EINVAL;
-			goto err;
-		}
-
-		list_move_tail(&block->link, &blocks);
-	}
-
-	/* Should be completely full! */
-	size = min_page_size = get_size(0, PAGE_SIZE);
-	err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, min_page_size, &tmp, flags);
-	if (!err) {
-		pr_info("buddy_alloc unexpectedly succeeded, it should be full!");
-		block = list_first_entry_or_null(&tmp,
-						 struct drm_buddy_block,
-						 link);
-		if (!block) {
-			pr_err("alloc_blocks has no blocks\n");
-			err = -EINVAL;
-			goto err;
-		}
-
-		list_move_tail(&block->link, &blocks);
-		err = -EINVAL;
-		goto err;
-	} else {
-		err = 0;
-	}
-
-err:
-	drm_buddy_free_list(&mm, &blocks);
-	drm_buddy_fini(&mm);
-	return err;
-}
-
-static int igt_buddy_alloc_range(void *arg)
-{
-	unsigned long flags = DRM_BUDDY_RANGE_ALLOCATION;
-	u64 offset, size, rem, chunk_size, end;
-	unsigned long page_num;
-	struct drm_buddy mm;
-	LIST_HEAD(blocks);
-	int err;
-
-	igt_mm_config(&size, &chunk_size);
-
-	err = drm_buddy_init(&mm, size, chunk_size);
-	if (err) {
-		pr_err("buddy_init failed(%d)\n", err);
-		return err;
-	}
-
-	err = igt_check_mm(&mm);
-	if (err) {
-		pr_err("pre-mm check failed, abort, abort, abort!\n");
-		goto err_fini;
-	}
-
-	rem = mm.size;
-	offset = 0;
-
-	for_each_prime_number_from(page_num, 1, ULONG_MAX - 1) {
-		struct drm_buddy_block *block;
-		LIST_HEAD(tmp);
-
-		size = min(page_num * mm.chunk_size, rem);
-		end = offset + size;
-
-		err = drm_buddy_alloc_blocks(&mm, offset, end, size, mm.chunk_size, &tmp, flags);
-		if (err) {
-			if (err == -ENOMEM) {
-				pr_info("alloc_range hit -ENOMEM with size=%llx\n",
-					size);
-			} else {
-				pr_err("alloc_range with offset=%llx, size=%llx failed(%d)\n",
-				       offset, size, err);
-			}
-
-			break;
-		}
-
-		block = list_first_entry_or_null(&tmp,
-						 struct drm_buddy_block,
-						 link);
-		if (!block) {
-			pr_err("alloc_range has no blocks\n");
-			err = -EINVAL;
-			break;
-		}
-
-		if (drm_buddy_block_offset(block) != offset) {
-			pr_err("alloc_range start offset mismatch, found=%llx, expected=%llx\n",
-			       drm_buddy_block_offset(block), offset);
-			err = -EINVAL;
-		}
-
-		if (!err)
-			err = igt_check_blocks(&mm, &tmp, size, true);
-
-		list_splice_tail(&tmp, &blocks);
-
-		if (err)
-			break;
-
-		offset += size;
-
-		rem -= size;
-		if (!rem)
-			break;
-
-		cond_resched();
-	}
-
-	if (err == -ENOMEM)
-		err = 0;
-
-	drm_buddy_free_list(&mm, &blocks);
-
-	if (!err) {
-		err = igt_check_mm(&mm);
-		if (err)
-			pr_err("post-mm check failed\n");
-	}
-
-err_fini:
-	drm_buddy_fini(&mm);
-
-	return err;
-}
-
-static int igt_buddy_alloc_limit(void *arg)
-{
-	u64 size = U64_MAX, start = 0;
-	struct drm_buddy_block *block;
-	unsigned long flags = 0;
-	LIST_HEAD(allocated);
-	struct drm_buddy mm;
-	int err;
-
-	err = drm_buddy_init(&mm, size, PAGE_SIZE);
-	if (err)
-		return err;
-
-	if (mm.max_order != DRM_BUDDY_MAX_ORDER) {
-		pr_err("mm.max_order(%d) != %d\n",
-		       mm.max_order, DRM_BUDDY_MAX_ORDER);
-		err = -EINVAL;
-		goto out_fini;
-	}
-
-	size = mm.chunk_size << mm.max_order;
-	err = drm_buddy_alloc_blocks(&mm, start, size, size,
-				     PAGE_SIZE, &allocated, flags);
-
-	if (unlikely(err))
-		goto out_free;
-
-	block = list_first_entry_or_null(&allocated,
-					 struct drm_buddy_block,
-					 link);
-
-	if (!block) {
-		err = -EINVAL;
-		goto out_fini;
-	}
-
-	if (drm_buddy_block_order(block) != mm.max_order) {
-		pr_err("block order(%d) != %d\n",
-		       drm_buddy_block_order(block), mm.max_order);
-		err = -EINVAL;
-		goto out_free;
-	}
-
-	if (drm_buddy_block_size(&mm, block) !=
-	    BIT_ULL(mm.max_order) * PAGE_SIZE) {
-		pr_err("block size(%llu) != %llu\n",
-		       drm_buddy_block_size(&mm, block),
-		       BIT_ULL(mm.max_order) * PAGE_SIZE);
-		err = -EINVAL;
-		goto out_free;
-	}
-
-out_free:
-	drm_buddy_free_list(&mm, &allocated);
-out_fini:
-	drm_buddy_fini(&mm);
-	return err;
-}
-
-static int igt_sanitycheck(void *ignored)
-{
-	pr_info("%s - ok!\n", __func__);
-	return 0;
-}
-
-#include "drm_selftest.c"
-
-static int __init test_drm_buddy_init(void)
-{
-	int err;
-
-	while (!random_seed)
-		random_seed = get_random_int();
-
-	pr_info("Testing DRM buddy manager (struct drm_buddy), with random_seed=0x%x\n",
-		random_seed);
-	err = run_selftests(selftests, ARRAY_SIZE(selftests), NULL);
-
-	return err > 0 ? 0 : err;
-}
-
-static void __exit test_drm_buddy_exit(void)
-{
-}
-
-module_init(test_drm_buddy_init);
-module_exit(test_drm_buddy_exit);
-
-MODULE_AUTHOR("Intel Corporation");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/selftests/test-drm_cmdline_parser.c b/drivers/gpu/drm/selftests/test-drm_cmdline_parser.c
deleted file mode 100644
index d96cd890def6..000000000000
--- a/drivers/gpu/drm/selftests/test-drm_cmdline_parser.c
+++ /dev/null
@@ -1,1141 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (c) 2019 Bootlin
- */
-
-#define pr_fmt(fmt) "drm_cmdline: " fmt
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-#include <drm/drm_connector.h>
-#include <drm/drm_modes.h>
-
-#define TESTS "drm_cmdline_selftests.h"
-#include "drm_selftest.h"
-#include "test-drm_modeset_common.h"
-
-static const struct drm_connector no_connector = {};
-
-static int drm_cmdline_test_force_e_only(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("e",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(mode.specified);
-	FAIL_ON(mode.refresh_specified);
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON);
-
-	return 0;
-}
-
-static int drm_cmdline_test_force_D_only_not_digital(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("D",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(mode.specified);
-	FAIL_ON(mode.refresh_specified);
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON);
-
-	return 0;
-}
-
-static const struct drm_connector connector_hdmi = {
-	.connector_type	= DRM_MODE_CONNECTOR_HDMIB,
-};
-
-static int drm_cmdline_test_force_D_only_hdmi(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("D",
-							   &connector_hdmi,
-							   &mode));
-	FAIL_ON(mode.specified);
-	FAIL_ON(mode.refresh_specified);
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON_DIGITAL);
-
-	return 0;
-}
-
-static const struct drm_connector connector_dvi = {
-	.connector_type	= DRM_MODE_CONNECTOR_DVII,
-};
-
-static int drm_cmdline_test_force_D_only_dvi(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("D",
-							   &connector_dvi,
-							   &mode));
-	FAIL_ON(mode.specified);
-	FAIL_ON(mode.refresh_specified);
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON_DIGITAL);
-
-	return 0;
-}
-
-static int drm_cmdline_test_force_d_only(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("d",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(mode.specified);
-	FAIL_ON(mode.refresh_specified);
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_OFF);
-
-	return 0;
-}
-
-static int drm_cmdline_test_margin_only(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("m",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_interlace_only(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("i",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_res(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_missing_x(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("x480",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_missing_y(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("1024x",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bad_y(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("1024xtest",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_missing_y_bpp(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("1024x-24",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_vesa(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480M",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(!mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_vesa_rblank(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480MR",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(!mode.rb);
-	FAIL_ON(!mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_rblank(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480R",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(!mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480-24",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bad_bpp(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("720x480-test",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_refresh(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480@60",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(!mode.refresh_specified);
-	FAIL_ON(mode.refresh != 60);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bad_refresh(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("720x480@refresh",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp_refresh(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480-24@60",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(!mode.refresh_specified);
-	FAIL_ON(mode.refresh != 60);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp_refresh_interlaced(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480-24@60i",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(!mode.refresh_specified);
-	FAIL_ON(mode.refresh != 60);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(!mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp_refresh_margins(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480-24@60m",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(!mode.refresh_specified);
-	FAIL_ON(mode.refresh != 60);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(!mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp_refresh_force_off(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480-24@60d",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(!mode.refresh_specified);
-	FAIL_ON(mode.refresh != 60);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_OFF);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp_refresh_force_on_off(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("720x480-24@60de",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp_refresh_force_on(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480-24@60e",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(!mode.refresh_specified);
-	FAIL_ON(mode.refresh != 60);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp_refresh_force_on_analog(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480-24@60D",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(!mode.refresh_specified);
-	FAIL_ON(mode.refresh != 60);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp_refresh_force_on_digital(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-	static const struct drm_connector connector = {
-		.connector_type = DRM_MODE_CONNECTOR_DVII,
-	};
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480-24@60D",
-							   &connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(!mode.refresh_specified);
-	FAIL_ON(mode.refresh != 60);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON_DIGITAL);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp_refresh_interlaced_margins_force_on(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480-24@60ime",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(!mode.refresh_specified);
-	FAIL_ON(mode.refresh != 60);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(!mode.interlace);
-	FAIL_ON(!mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_margins_force_on(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480me",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(!mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_vesa_margins(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480Mm",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(!mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(!mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_invalid_mode(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("720x480f",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_res_bpp_wrong_place_mode(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("720x480e-24",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_name(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("NTSC",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(strcmp(mode.name, "NTSC"));
-	FAIL_ON(mode.refresh_specified);
-	FAIL_ON(mode.bpp_specified);
-
-	return 0;
-}
-
-static int drm_cmdline_test_name_bpp(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("NTSC-24",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(strcmp(mode.name, "NTSC"));
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	return 0;
-}
-
-static int drm_cmdline_test_name_bpp_refresh(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("NTSC-24@60",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_name_refresh(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("NTSC@60",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_name_refresh_wrong_mode(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("NTSC@60m",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_name_refresh_invalid_mode(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("NTSC@60f",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_name_option(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("NTSC,rotate=180",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(strcmp(mode.name, "NTSC"));
-	FAIL_ON(mode.rotation_reflection != DRM_MODE_ROTATE_180);
-
-	return 0;
-}
-
-static int drm_cmdline_test_name_bpp_option(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("NTSC-24,rotate=180",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(strcmp(mode.name, "NTSC"));
-	FAIL_ON(mode.rotation_reflection != DRM_MODE_ROTATE_180);
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	return 0;
-}
-
-static int drm_cmdline_test_rotate_0(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480,rotate=0",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-	FAIL_ON(mode.rotation_reflection != DRM_MODE_ROTATE_0);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_rotate_90(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480,rotate=90",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-	FAIL_ON(mode.rotation_reflection != DRM_MODE_ROTATE_90);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_rotate_180(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480,rotate=180",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-	FAIL_ON(mode.rotation_reflection != DRM_MODE_ROTATE_180);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_rotate_270(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480,rotate=270",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-	FAIL_ON(mode.rotation_reflection != DRM_MODE_ROTATE_270);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_rotate_multiple(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("720x480,rotate=0,rotate=90",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_rotate_invalid_val(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("720x480,rotate=42",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_rotate_truncated(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("720x480,rotate=",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_hmirror(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480,reflect_x",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-	FAIL_ON(mode.rotation_reflection != (DRM_MODE_ROTATE_0 | DRM_MODE_REFLECT_X));
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_vmirror(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480,reflect_y",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-	FAIL_ON(mode.rotation_reflection != (DRM_MODE_ROTATE_0 | DRM_MODE_REFLECT_Y));
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_margin_options(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480,margin_right=14,margin_left=24,margin_bottom=36,margin_top=42",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-	FAIL_ON(mode.tv_margins.right != 14);
-	FAIL_ON(mode.tv_margins.left != 24);
-	FAIL_ON(mode.tv_margins.bottom != 36);
-	FAIL_ON(mode.tv_margins.top != 42);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_multiple_options(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480,rotate=270,reflect_x",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-	FAIL_ON(mode.rotation_reflection != (DRM_MODE_ROTATE_270 | DRM_MODE_REFLECT_X));
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_invalid_option(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(drm_mode_parse_command_line_for_connector("720x480,test=42",
-							  &no_connector,
-							  &mode));
-
-	return 0;
-}
-
-static int drm_cmdline_test_bpp_extra_and_option(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480-24e,rotate=180",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-	FAIL_ON(mode.rotation_reflection != DRM_MODE_ROTATE_180);
-
-	FAIL_ON(mode.refresh_specified);
-
-	FAIL_ON(!mode.bpp_specified);
-	FAIL_ON(mode.bpp != 24);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON);
-
-	return 0;
-}
-
-static int drm_cmdline_test_extra_and_option(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("720x480e,rotate=180",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(!mode.specified);
-	FAIL_ON(mode.xres != 720);
-	FAIL_ON(mode.yres != 480);
-	FAIL_ON(mode.rotation_reflection != DRM_MODE_ROTATE_180);
-
-	FAIL_ON(mode.refresh_specified);
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON);
-
-	return 0;
-}
-
-static int drm_cmdline_test_freestanding_options(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("margin_right=14,margin_left=24,margin_bottom=36,margin_top=42",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(mode.specified);
-	FAIL_ON(mode.refresh_specified);
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.tv_margins.right != 14);
-	FAIL_ON(mode.tv_margins.left != 24);
-	FAIL_ON(mode.tv_margins.bottom != 36);
-	FAIL_ON(mode.tv_margins.top != 42);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-static int drm_cmdline_test_freestanding_force_e_and_options(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("e,margin_right=14,margin_left=24,margin_bottom=36,margin_top=42",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(mode.specified);
-	FAIL_ON(mode.refresh_specified);
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.tv_margins.right != 14);
-	FAIL_ON(mode.tv_margins.left != 24);
-	FAIL_ON(mode.tv_margins.bottom != 36);
-	FAIL_ON(mode.tv_margins.top != 42);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_ON);
-
-	return 0;
-}
-
-static int drm_cmdline_test_panel_orientation(void *ignored)
-{
-	struct drm_cmdline_mode mode = { };
-
-	FAIL_ON(!drm_mode_parse_command_line_for_connector("panel_orientation=upside_down",
-							   &no_connector,
-							   &mode));
-	FAIL_ON(mode.specified);
-	FAIL_ON(mode.refresh_specified);
-	FAIL_ON(mode.bpp_specified);
-
-	FAIL_ON(mode.panel_orientation != DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP);
-
-	FAIL_ON(mode.rb);
-	FAIL_ON(mode.cvt);
-	FAIL_ON(mode.interlace);
-	FAIL_ON(mode.margins);
-	FAIL_ON(mode.force != DRM_FORCE_UNSPECIFIED);
-
-	return 0;
-}
-
-#include "drm_selftest.c"
-
-static int __init test_drm_cmdline_init(void)
-{
-	int err;
-
-	err = run_selftests(selftests, ARRAY_SIZE(selftests), NULL);
-
-	return err > 0 ? 0 : err;
-}
-module_init(test_drm_cmdline_init);
-
-MODULE_AUTHOR("Maxime Ripard <maxime.ripard@bootlin.com>");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/selftests/test-drm_damage_helper.c b/drivers/gpu/drm/selftests/test-drm_damage_helper.c
deleted file mode 100644
index 816e1464a98f..000000000000
--- a/drivers/gpu/drm/selftests/test-drm_damage_helper.c
+++ /dev/null
@@ -1,668 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Test case for drm_damage_helper functions
- */
-
-#define pr_fmt(fmt) "drm_damage_helper: " fmt
-
-#include <drm/drm_damage_helper.h>
-#include <drm/drm_framebuffer.h>
-#include <drm/drm_plane.h>
-#include <drm/drm_drv.h>
-
-#include "test-drm_modeset_common.h"
-
-struct drm_driver mock_driver;
-static struct drm_device mock_device;
-static struct drm_object_properties mock_obj_props;
-static struct drm_plane mock_plane;
-static struct drm_property mock_prop;
-
-static void mock_setup(struct drm_plane_state *state)
-{
-	static bool setup_done = false;
-
-	state->plane = &mock_plane;
-
-	if (setup_done)
-		return;
-
-	/* just enough so that drm_plane_enable_fb_damage_clips() works */
-	mock_device.driver = &mock_driver;
-	mock_device.mode_config.prop_fb_damage_clips = &mock_prop;
-	mock_plane.dev = &mock_device;
-	mock_obj_props.count = 0;
-	mock_plane.base.properties = &mock_obj_props;
-	mock_prop.base.id = 1; /* 0 is an invalid id */
-	mock_prop.dev = &mock_device;
-
-	drm_plane_enable_fb_damage_clips(&mock_plane);
-}
-
-static void set_plane_src(struct drm_plane_state *state, int x1, int y1, int x2,
-			  int y2)
-{
-	state->src.x1 = x1;
-	state->src.y1 = y1;
-	state->src.x2 = x2;
-	state->src.y2 = y2;
-}
-
-static void set_damage_clip(struct drm_mode_rect *r, int x1, int y1, int x2,
-			    int y2)
-{
-	r->x1 = x1;
-	r->y1 = y1;
-	r->x2 = x2;
-	r->y2 = y2;
-}
-
-static void set_damage_blob(struct drm_property_blob *damage_blob,
-			    struct drm_mode_rect *r, uint32_t size)
-{
-	damage_blob->length = size;
-	damage_blob->data = r;
-}
-
-static void set_plane_damage(struct drm_plane_state *state,
-			     struct drm_property_blob *damage_blob)
-{
-	state->fb_damage_clips = damage_blob;
-}
-
-static bool check_damage_clip(struct drm_plane_state *state, struct drm_rect *r,
-			      int x1, int y1, int x2, int y2)
-{
-	/*
-	 * Round down x1/y1 and round up x2/y2. This is because damage is not in
-	 * 16.16 fixed point so to catch all pixels.
-	 */
-	int src_x1 = state->src.x1 >> 16;
-	int src_y1 = state->src.y1 >> 16;
-	int src_x2 = (state->src.x2 >> 16) + !!(state->src.x2 & 0xFFFF);
-	int src_y2 = (state->src.y2 >> 16) + !!(state->src.y2 & 0xFFFF);
-
-	if (x1 >= x2 || y1 >= y2) {
-		pr_err("Cannot have damage clip with no dimension.\n");
-		return false;
-	}
-
-	if (x1 < src_x1 || y1 < src_y1 || x2 > src_x2 || y2 > src_y2) {
-		pr_err("Damage cannot be outside rounded plane src.\n");
-		return false;
-	}
-
-	if (r->x1 != x1 || r->y1 != y1 || r->x2 != x2 || r->y2 != y2) {
-		pr_err("Damage = %d %d %d %d\n", r->x1, r->y1, r->x2, r->y2);
-		return false;
-	}
-
-	return true;
-}
-
-const struct drm_framebuffer fb = {
-	.width = 2048,
-	.height = 2048
-};
-
-/* common mocked structs many tests need */
-#define MOCK_VARIABLES() \
-	struct drm_plane_state old_state; \
-	struct drm_plane_state state = { \
-		.crtc = ZERO_SIZE_PTR, \
-		.fb = (struct drm_framebuffer *) &fb, \
-		.visible = true, \
-	}; \
-	mock_setup(&old_state); \
-	mock_setup(&state);
-
-int igt_damage_iter_no_damage(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	/* Plane src same as fb size. */
-	set_plane_src(&old_state, 0, 0, fb.width << 16, fb.height << 16);
-	set_plane_src(&state, 0, 0, fb.width << 16, fb.height << 16);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return plane src as damage.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 0, 0, 2048, 2048));
-
-	return 0;
-}
-
-int igt_damage_iter_no_damage_fractional_src(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	/* Plane src has fractional part. */
-	set_plane_src(&old_state, 0x3fffe, 0x3fffe,
-		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
-	set_plane_src(&state, 0x3fffe, 0x3fffe,
-		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return rounded off plane src as damage.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 3, 3, 1028, 772));
-
-	return 0;
-}
-
-int igt_damage_iter_no_damage_src_moved(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	/* Plane src moved since old plane state. */
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 10 << 16, 10 << 16,
-		      (10 + 1024) << 16, (10 + 768) << 16);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return plane src as damage.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 10, 10, 1034, 778));
-
-	return 0;
-}
-
-int igt_damage_iter_no_damage_fractional_src_moved(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	/* Plane src has fractional part and it moved since old plane state. */
-	set_plane_src(&old_state, 0x3fffe, 0x3fffe,
-		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
-	set_plane_src(&state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return plane src as damage.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 4, 4, 1029, 773));
-
-	return 0;
-}
-
-int igt_damage_iter_no_damage_not_visible(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	state.visible = false;
-
-	mock_setup(&old_state);
-
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 0, 0, 1024 << 16, 768 << 16);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 0, "Should have no damage.");
-
-	return 0;
-}
-
-int igt_damage_iter_no_damage_no_crtc(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	state.crtc = NULL;
-
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 0, 0, 1024 << 16, 768 << 16);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 0, "Should have no damage.");
-
-	return 0;
-}
-
-int igt_damage_iter_no_damage_no_fb(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_plane_state old_state;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	struct drm_plane_state state = {
-		.crtc = ZERO_SIZE_PTR,
-		.fb = 0,
-	};
-
-	mock_setup(&old_state);
-
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 0, 0, 1024 << 16, 768 << 16);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 0, "Should have no damage.");
-
-	return 0;
-}
-
-int igt_damage_iter_simple_damage(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 0, 0, 1024 << 16, 768 << 16);
-	/* Damage set to plane src */
-	set_damage_clip(&damage, 0, 0, 1024, 768);
-	set_damage_blob(&damage_blob, &damage, sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return damage when set.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 0, 0, 1024, 768));
-
-	return 0;
-}
-
-int igt_damage_iter_single_damage(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 0, 0, 1024 << 16, 768 << 16);
-	set_damage_clip(&damage, 256, 192, 768, 576);
-	set_damage_blob(&damage_blob, &damage, sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return damage when set.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 256, 192, 768, 576));
-
-	return 0;
-}
-
-int igt_damage_iter_single_damage_intersect_src(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 0, 0, 1024 << 16, 768 << 16);
-	/* Damage intersect with plane src. */
-	set_damage_clip(&damage, 256, 192, 1360, 768);
-	set_damage_blob(&damage_blob, &damage, sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return damage clipped to src.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 256, 192, 1024, 768));
-
-	return 0;
-}
-
-int igt_damage_iter_single_damage_outside_src(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 0, 0, 1024 << 16, 768 << 16);
-	/* Damage clip outside plane src */
-	set_damage_clip(&damage, 1360, 1360, 1380, 1380);
-	set_damage_blob(&damage_blob, &damage, sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 0, "Should have no damage.");
-
-	return 0;
-}
-
-int igt_damage_iter_single_damage_fractional_src(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	/* Plane src has fractional part. */
-	set_plane_src(&old_state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	set_plane_src(&state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	set_damage_clip(&damage, 10, 10, 256, 330);
-	set_damage_blob(&damage_blob, &damage, sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return damage when set.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 10, 10, 256, 330));
-
-	return 0;
-}
-
-int igt_damage_iter_single_damage_intersect_fractional_src(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	/* Plane src has fractional part. */
-	set_plane_src(&old_state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	set_plane_src(&state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	/* Damage intersect with plane src. */
-	set_damage_clip(&damage, 10, 1, 1360, 330);
-	set_damage_blob(&damage_blob, &damage, sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return damage clipped to rounded off src.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 10, 4, 1029, 330));
-
-	return 0;
-}
-
-int igt_damage_iter_single_damage_outside_fractional_src(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	/* Plane src has fractional part. */
-	set_plane_src(&old_state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	set_plane_src(&state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	/* Damage clip outside plane src */
-	set_damage_clip(&damage, 1360, 1360, 1380, 1380);
-	set_damage_blob(&damage_blob, &damage, sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 0, "Should have no damage.");
-
-	return 0;
-}
-
-int igt_damage_iter_single_damage_src_moved(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	/* Plane src moved since old plane state. */
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 10 << 16, 10 << 16,
-		      (10 + 1024) << 16, (10 + 768) << 16);
-	set_damage_clip(&damage, 20, 30, 256, 256);
-	set_damage_blob(&damage_blob, &damage, sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return plane src as damage.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 10, 10, 1034, 778));
-
-	return 0;
-}
-
-int igt_damage_iter_single_damage_fractional_src_moved(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage;
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	/* Plane src with fractional part moved since old plane state. */
-	set_plane_src(&old_state, 0x3fffe, 0x3fffe,
-		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
-	set_plane_src(&state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	/* Damage intersect with plane src. */
-	set_damage_clip(&damage, 20, 30, 1360, 256);
-	set_damage_blob(&damage_blob, &damage, sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return rounded off plane src as damage.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 4, 4, 1029, 773));
-
-	return 0;
-}
-
-int igt_damage_iter_damage(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage[2];
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 0, 0, 1024 << 16, 768 << 16);
-	/* 2 damage clips. */
-	set_damage_clip(&damage[0], 20, 30, 200, 180);
-	set_damage_clip(&damage[1], 240, 200, 280, 250);
-	set_damage_blob(&damage_blob, &damage[0], sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip) {
-		if (num_hits == 0)
-			FAIL_ON(!check_damage_clip(&state, &clip, 20, 30, 200, 180));
-		if (num_hits == 1)
-			FAIL_ON(!check_damage_clip(&state, &clip, 240, 200, 280, 250));
-		num_hits++;
-	}
-
-	FAIL(num_hits != 2, "Should return damage when set.");
-
-	return 0;
-}
-
-int igt_damage_iter_damage_one_intersect(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage[2];
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	set_plane_src(&old_state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	set_plane_src(&state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	/* 2 damage clips, one intersect plane src. */
-	set_damage_clip(&damage[0], 20, 30, 200, 180);
-	set_damage_clip(&damage[1], 2, 2, 1360, 1360);
-	set_damage_blob(&damage_blob, &damage[0], sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip) {
-		if (num_hits == 0)
-			FAIL_ON(!check_damage_clip(&state, &clip, 20, 30, 200, 180));
-		if (num_hits == 1)
-			FAIL_ON(!check_damage_clip(&state, &clip, 4, 4, 1029, 773));
-		num_hits++;
-	}
-
-	FAIL(num_hits != 2, "Should return damage when set.");
-
-	return 0;
-}
-
-int igt_damage_iter_damage_one_outside(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage[2];
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	set_plane_src(&old_state, 0, 0, 1024 << 16, 768 << 16);
-	set_plane_src(&state, 0, 0, 1024 << 16, 768 << 16);
-	/* 2 damage clips, one outside plane src. */
-	set_damage_clip(&damage[0], 1360, 1360, 1380, 1380);
-	set_damage_clip(&damage[1], 240, 200, 280, 250);
-	set_damage_blob(&damage_blob, &damage[0], sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return damage when set.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 240, 200, 280, 250));
-
-	return 0;
-}
-
-int igt_damage_iter_damage_src_moved(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage[2];
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	set_plane_src(&old_state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	set_plane_src(&state, 0x3fffe, 0x3fffe,
-		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
-	/* 2 damage clips, one outside plane src. */
-	set_damage_clip(&damage[0], 1360, 1360, 1380, 1380);
-	set_damage_clip(&damage[1], 240, 200, 280, 250);
-	set_damage_blob(&damage_blob, &damage[0], sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 1, "Should return round off plane src as damage.");
-	FAIL_ON(!check_damage_clip(&state, &clip, 3, 3, 1028, 772));
-
-	return 0;
-}
-
-int igt_damage_iter_damage_not_visible(void *ignored)
-{
-	struct drm_atomic_helper_damage_iter iter;
-	struct drm_property_blob damage_blob;
-	struct drm_mode_rect damage[2];
-	struct drm_rect clip;
-	uint32_t num_hits = 0;
-
-	MOCK_VARIABLES();
-
-	state.visible = false;
-
-	set_plane_src(&old_state, 0x40002, 0x40002,
-		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
-	set_plane_src(&state, 0x3fffe, 0x3fffe,
-		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
-	/* 2 damage clips, one outside plane src. */
-	set_damage_clip(&damage[0], 1360, 1360, 1380, 1380);
-	set_damage_clip(&damage[1], 240, 200, 280, 250);
-	set_damage_blob(&damage_blob, &damage[0], sizeof(damage));
-	set_plane_damage(&state, &damage_blob);
-	drm_atomic_helper_damage_iter_init(&iter, &old_state, &state);
-	drm_atomic_for_each_plane_damage(&iter, &clip)
-		num_hits++;
-
-	FAIL(num_hits != 0, "Should not return any damage.");
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/selftests/test-drm_format.c b/drivers/gpu/drm/selftests/test-drm_format.c
deleted file mode 100644
index c5e212afa27a..000000000000
--- a/drivers/gpu/drm/selftests/test-drm_format.c
+++ /dev/null
@@ -1,280 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Test cases for the drm_format functions
- */
-
-#define pr_fmt(fmt) "drm_format: " fmt
-
-#include <linux/errno.h>
-#include <linux/kernel.h>
-
-#include <drm/drm_fourcc.h>
-
-#include "test-drm_modeset_common.h"
-
-int igt_check_drm_format_block_width(void *ignored)
-{
-	const struct drm_format_info *info = NULL;
-
-	/* Test invalid arguments */
-	FAIL_ON(drm_format_info_block_width(info, 0) != 0);
-	FAIL_ON(drm_format_info_block_width(info, -1) != 0);
-	FAIL_ON(drm_format_info_block_width(info, 1) != 0);
-
-	/* Test 1 plane format */
-	info = drm_format_info(DRM_FORMAT_XRGB4444);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_block_width(info, 0) != 1);
-	FAIL_ON(drm_format_info_block_width(info, 1) != 0);
-	FAIL_ON(drm_format_info_block_width(info, -1) != 0);
-
-	/* Test 2 planes format */
-	info = drm_format_info(DRM_FORMAT_NV12);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_block_width(info, 0) != 1);
-	FAIL_ON(drm_format_info_block_width(info, 1) != 1);
-	FAIL_ON(drm_format_info_block_width(info, 2) != 0);
-	FAIL_ON(drm_format_info_block_width(info, -1) != 0);
-
-	/* Test 3 planes format */
-	info = drm_format_info(DRM_FORMAT_YUV422);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_block_width(info, 0) != 1);
-	FAIL_ON(drm_format_info_block_width(info, 1) != 1);
-	FAIL_ON(drm_format_info_block_width(info, 2) != 1);
-	FAIL_ON(drm_format_info_block_width(info, 3) != 0);
-	FAIL_ON(drm_format_info_block_width(info, -1) != 0);
-
-	/* Test a tiled format */
-	info = drm_format_info(DRM_FORMAT_X0L0);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_block_width(info, 0) != 2);
-	FAIL_ON(drm_format_info_block_width(info, 1) != 0);
-	FAIL_ON(drm_format_info_block_width(info, -1) != 0);
-
-	return 0;
-}
-
-int igt_check_drm_format_block_height(void *ignored)
-{
-	const struct drm_format_info *info = NULL;
-
-	/* Test invalid arguments */
-	FAIL_ON(drm_format_info_block_height(info, 0) != 0);
-	FAIL_ON(drm_format_info_block_height(info, -1) != 0);
-	FAIL_ON(drm_format_info_block_height(info, 1) != 0);
-
-	/* Test 1 plane format */
-	info = drm_format_info(DRM_FORMAT_XRGB4444);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_block_height(info, 0) != 1);
-	FAIL_ON(drm_format_info_block_height(info, 1) != 0);
-	FAIL_ON(drm_format_info_block_height(info, -1) != 0);
-
-	/* Test 2 planes format */
-	info = drm_format_info(DRM_FORMAT_NV12);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_block_height(info, 0) != 1);
-	FAIL_ON(drm_format_info_block_height(info, 1) != 1);
-	FAIL_ON(drm_format_info_block_height(info, 2) != 0);
-	FAIL_ON(drm_format_info_block_height(info, -1) != 0);
-
-	/* Test 3 planes format */
-	info = drm_format_info(DRM_FORMAT_YUV422);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_block_height(info, 0) != 1);
-	FAIL_ON(drm_format_info_block_height(info, 1) != 1);
-	FAIL_ON(drm_format_info_block_height(info, 2) != 1);
-	FAIL_ON(drm_format_info_block_height(info, 3) != 0);
-	FAIL_ON(drm_format_info_block_height(info, -1) != 0);
-
-	/* Test a tiled format */
-	info = drm_format_info(DRM_FORMAT_X0L0);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_block_height(info, 0) != 2);
-	FAIL_ON(drm_format_info_block_height(info, 1) != 0);
-	FAIL_ON(drm_format_info_block_height(info, -1) != 0);
-
-	return 0;
-}
-
-int igt_check_drm_format_min_pitch(void *ignored)
-{
-	const struct drm_format_info *info = NULL;
-
-	/* Test invalid arguments */
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, -1, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 0) != 0);
-
-	/* Test 1 plane 8 bits per pixel format */
-	info = drm_format_info(DRM_FORMAT_RGB332);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, -1, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 0) != 0);
-
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1) != 1);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 2) != 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 640) != 640);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1024) != 1024);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1920) != 1920);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 4096) != 4096);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 671) != 671);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, UINT_MAX) !=
-			(uint64_t)UINT_MAX);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, (UINT_MAX - 1)) !=
-			(uint64_t)(UINT_MAX - 1));
-
-	/* Test 1 plane 16 bits per pixel format */
-	info = drm_format_info(DRM_FORMAT_XRGB4444);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, -1, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 0) != 0);
-
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1) != 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 2) != 4);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 640) != 1280);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1024) != 2048);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1920) != 3840);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 4096) != 8192);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 671) != 1342);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, UINT_MAX) !=
-			(uint64_t)UINT_MAX * 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, (UINT_MAX - 1)) !=
-			(uint64_t)(UINT_MAX - 1) * 2);
-
-	/* Test 1 plane 24 bits per pixel format */
-	info = drm_format_info(DRM_FORMAT_RGB888);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, -1, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 0) != 0);
-
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1) != 3);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 2) != 6);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 640) != 1920);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1024) != 3072);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1920) != 5760);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 4096) != 12288);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 671) != 2013);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, UINT_MAX) !=
-			(uint64_t)UINT_MAX * 3);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, UINT_MAX - 1) !=
-			(uint64_t)(UINT_MAX - 1) * 3);
-
-	/* Test 1 plane 32 bits per pixel format */
-	info = drm_format_info(DRM_FORMAT_ABGR8888);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, -1, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 0) != 0);
-
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1) != 4);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 2) != 8);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 640) != 2560);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1024) != 4096);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1920) != 7680);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 4096) != 16384);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 671) != 2684);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, UINT_MAX) !=
-			(uint64_t)UINT_MAX * 4);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, UINT_MAX - 1) !=
-			(uint64_t)(UINT_MAX - 1) * 4);
-
-	/* Test 2 planes format */
-	info = drm_format_info(DRM_FORMAT_NV12);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, -1, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, 0) != 0);
-
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1) != 1);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 1) != 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 2) != 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 1) != 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 640) != 640);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 320) != 640);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1024) != 1024);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 512) != 1024);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1920) != 1920);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 960) != 1920);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 4096) != 4096);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 2048) != 4096);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 671) != 671);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 336) != 672);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, UINT_MAX) !=
-			(uint64_t)UINT_MAX);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, UINT_MAX / 2 + 1) !=
-			(uint64_t)UINT_MAX + 1);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, (UINT_MAX - 1)) !=
-			(uint64_t)(UINT_MAX - 1));
-	FAIL_ON(drm_format_info_min_pitch(info, 1, (UINT_MAX - 1) /  2) !=
-			(uint64_t)(UINT_MAX - 1));
-
-	/* Test 3 planes 8 bits per pixel format */
-	info = drm_format_info(DRM_FORMAT_YUV422);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, -1, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 3, 0) != 0);
-
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1) != 1);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 1) != 1);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, 1) != 1);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 2) != 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 2) != 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, 2) != 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 640) != 640);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 320) != 320);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, 320) != 320);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1024) != 1024);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 512) != 512);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, 512) != 512);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1920) != 1920);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 960) != 960);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, 960) != 960);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 4096) != 4096);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 2048) != 2048);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, 2048) != 2048);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 671) != 671);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 336) != 336);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, 336) != 336);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, UINT_MAX) !=
-			(uint64_t)UINT_MAX);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, UINT_MAX / 2 + 1) !=
-			(uint64_t)UINT_MAX / 2 + 1);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, UINT_MAX / 2 + 1) !=
-			(uint64_t)UINT_MAX / 2 + 1);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, (UINT_MAX - 1) / 2) !=
-			(uint64_t)(UINT_MAX - 1) / 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, (UINT_MAX - 1) / 2) !=
-			(uint64_t)(UINT_MAX - 1) / 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 2, (UINT_MAX - 1) / 2) !=
-			(uint64_t)(UINT_MAX - 1) / 2);
-
-	/* Test tiled format */
-	info = drm_format_info(DRM_FORMAT_X0L2);
-	FAIL_ON(!info);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, -1, 0) != 0);
-	FAIL_ON(drm_format_info_min_pitch(info, 1, 0) != 0);
-
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1) != 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 2) != 4);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 640) != 1280);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1024) != 2048);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 1920) != 3840);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 4096) != 8192);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, 671) != 1342);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, UINT_MAX) !=
-			(uint64_t)UINT_MAX * 2);
-	FAIL_ON(drm_format_info_min_pitch(info, 0, UINT_MAX - 1) !=
-			(uint64_t)(UINT_MAX - 1) * 2);
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/selftests/test-drm_modeset_common.c b/drivers/gpu/drm/selftests/test-drm_modeset_common.c
deleted file mode 100644
index 2a7f93774006..000000000000
--- a/drivers/gpu/drm/selftests/test-drm_modeset_common.c
+++ /dev/null
@@ -1,32 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Common file for modeset selftests.
- */
-
-#include <linux/module.h>
-
-#include "test-drm_modeset_common.h"
-
-#define TESTS "drm_modeset_selftests.h"
-#include "drm_selftest.h"
-
-#include "drm_selftest.c"
-
-static int __init test_drm_modeset_init(void)
-{
-	int err;
-
-	err = run_selftests(selftests, ARRAY_SIZE(selftests), NULL);
-
-	return err > 0 ? 0 : err;
-}
-
-static void __exit test_drm_modeset_exit(void)
-{
-}
-
-module_init(test_drm_modeset_init);
-module_exit(test_drm_modeset_exit);
-
-MODULE_AUTHOR("Intel Corporation");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/selftests/test-drm_modeset_common.h b/drivers/gpu/drm/selftests/test-drm_modeset_common.h
deleted file mode 100644
index cfb51d8da2bc..000000000000
--- a/drivers/gpu/drm/selftests/test-drm_modeset_common.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-
-#ifndef __TEST_DRM_MODESET_COMMON_H__
-#define __TEST_DRM_MODESET_COMMON_H__
-
-#include <linux/errno.h>
-#include <linux/printk.h>
-
-#define FAIL(test, msg, ...) \
-	do { \
-		if (test) { \
-			pr_err("%s/%u: " msg, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
-			return -EINVAL; \
-		} \
-	} while (0)
-
-#define FAIL_ON(x) FAIL((x), "%s", "FAIL_ON(" __stringify(x) ")\n")
-
-int igt_drm_rect_clip_scaled_div_by_zero(void *ignored);
-int igt_drm_rect_clip_scaled_not_clipped(void *ignored);
-int igt_drm_rect_clip_scaled_clipped(void *ignored);
-int igt_drm_rect_clip_scaled_signed_vs_unsigned(void *ignored);
-int igt_check_plane_state(void *ignored);
-int igt_check_drm_format_block_width(void *ignored);
-int igt_check_drm_format_block_height(void *ignored);
-int igt_check_drm_format_min_pitch(void *ignored);
-int igt_check_drm_framebuffer_create(void *ignored);
-int igt_damage_iter_no_damage(void *ignored);
-int igt_damage_iter_no_damage_fractional_src(void *ignored);
-int igt_damage_iter_no_damage_src_moved(void *ignored);
-int igt_damage_iter_no_damage_fractional_src_moved(void *ignored);
-int igt_damage_iter_no_damage_not_visible(void *ignored);
-int igt_damage_iter_no_damage_no_crtc(void *ignored);
-int igt_damage_iter_no_damage_no_fb(void *ignored);
-int igt_damage_iter_simple_damage(void *ignored);
-int igt_damage_iter_single_damage(void *ignored);
-int igt_damage_iter_single_damage_intersect_src(void *ignored);
-int igt_damage_iter_single_damage_outside_src(void *ignored);
-int igt_damage_iter_single_damage_fractional_src(void *ignored);
-int igt_damage_iter_single_damage_intersect_fractional_src(void *ignored);
-int igt_damage_iter_single_damage_outside_fractional_src(void *ignored);
-int igt_damage_iter_single_damage_src_moved(void *ignored);
-int igt_damage_iter_single_damage_fractional_src_moved(void *ignored);
-int igt_damage_iter_damage(void *ignored);
-int igt_damage_iter_damage_one_intersect(void *ignored);
-int igt_damage_iter_damage_one_outside(void *ignored);
-int igt_damage_iter_damage_src_moved(void *ignored);
-int igt_damage_iter_damage_not_visible(void *ignored);
-int igt_dp_mst_calc_pbn_mode(void *ignored);
-int igt_dp_mst_sideband_msg_req_decode(void *ignored);
-
-#endif
diff --git a/drivers/gpu/drm/selftests/test-drm_rect.c b/drivers/gpu/drm/selftests/test-drm_rect.c
deleted file mode 100644
index 3a5ff38321f4..000000000000
--- a/drivers/gpu/drm/selftests/test-drm_rect.c
+++ /dev/null
@@ -1,223 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Test cases for the drm_rect functions
- */
-
-#define pr_fmt(fmt) "drm_rect: " fmt
-
-#include <linux/limits.h>
-
-#include <drm/drm_rect.h>
-
-#include "test-drm_modeset_common.h"
-
-int igt_drm_rect_clip_scaled_div_by_zero(void *ignored)
-{
-	struct drm_rect src, dst, clip;
-	bool visible;
-
-	/*
-	 * Make sure we don't divide by zero when dst
-	 * width/height is zero and dst and clip do not intersect.
-	 */
-	drm_rect_init(&src, 0, 0, 0, 0);
-	drm_rect_init(&dst, 0, 0, 0, 0);
-	drm_rect_init(&clip, 1, 1, 1, 1);
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-	FAIL(visible, "Destination not be visible\n");
-	FAIL(drm_rect_visible(&src), "Source should not be visible\n");
-
-	drm_rect_init(&src, 0, 0, 0, 0);
-	drm_rect_init(&dst, 3, 3, 0, 0);
-	drm_rect_init(&clip, 1, 1, 1, 1);
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-	FAIL(visible, "Destination not be visible\n");
-	FAIL(drm_rect_visible(&src), "Source should not be visible\n");
-
-	return 0;
-}
-
-int igt_drm_rect_clip_scaled_not_clipped(void *ignored)
-{
-	struct drm_rect src, dst, clip;
-	bool visible;
-
-	/* 1:1 scaling */
-	drm_rect_init(&src, 0, 0, 1 << 16, 1 << 16);
-	drm_rect_init(&dst, 0, 0, 1, 1);
-	drm_rect_init(&clip, 0, 0, 1, 1);
-
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-
-	FAIL(src.x1 != 0 || src.x2 != 1 << 16 ||
-	     src.y1 != 0 || src.y2 != 1 << 16,
-	     "Source badly clipped\n");
-	FAIL(dst.x1 != 0 || dst.x2 != 1 ||
-	     dst.y1 != 0 || dst.y2 != 1,
-	     "Destination badly clipped\n");
-	FAIL(!visible, "Destination should be visible\n");
-	FAIL(!drm_rect_visible(&src), "Source should be visible\n");
-
-	/* 2:1 scaling */
-	drm_rect_init(&src, 0, 0, 2 << 16, 2 << 16);
-	drm_rect_init(&dst, 0, 0, 1, 1);
-	drm_rect_init(&clip, 0, 0, 1, 1);
-
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-
-	FAIL(src.x1 != 0 || src.x2 != 2 << 16 ||
-	     src.y1 != 0 || src.y2 != 2 << 16,
-	     "Source badly clipped\n");
-	FAIL(dst.x1 != 0 || dst.x2 != 1 ||
-	     dst.y1 != 0 || dst.y2 != 1,
-	     "Destination badly clipped\n");
-	FAIL(!visible, "Destination should be visible\n");
-	FAIL(!drm_rect_visible(&src), "Source should be visible\n");
-
-	/* 1:2 scaling */
-	drm_rect_init(&src, 0, 0, 1 << 16, 1 << 16);
-	drm_rect_init(&dst, 0, 0, 2, 2);
-	drm_rect_init(&clip, 0, 0, 2, 2);
-
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-
-	FAIL(src.x1 != 0 || src.x2 != 1 << 16 ||
-	     src.y1 != 0 || src.y2 != 1 << 16,
-	     "Source badly clipped\n");
-	FAIL(dst.x1 != 0 || dst.x2 != 2 ||
-	     dst.y1 != 0 || dst.y2 != 2,
-	     "Destination badly clipped\n");
-	FAIL(!visible, "Destination should be visible\n");
-	FAIL(!drm_rect_visible(&src), "Source should be visible\n");
-
-	return 0;
-}
-
-int igt_drm_rect_clip_scaled_clipped(void *ignored)
-{
-	struct drm_rect src, dst, clip;
-	bool visible;
-
-	/* 1:1 scaling top/left clip */
-	drm_rect_init(&src, 0, 0, 2 << 16, 2 << 16);
-	drm_rect_init(&dst, 0, 0, 2, 2);
-	drm_rect_init(&clip, 0, 0, 1, 1);
-
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-
-	FAIL(src.x1 != 0 || src.x2 != 1 << 16 ||
-	     src.y1 != 0 || src.y2 != 1 << 16,
-	     "Source badly clipped\n");
-	FAIL(dst.x1 != 0 || dst.x2 != 1 ||
-	     dst.y1 != 0 || dst.y2 != 1,
-	     "Destination badly clipped\n");
-	FAIL(!visible, "Destination should be visible\n");
-	FAIL(!drm_rect_visible(&src), "Source should be visible\n");
-
-	/* 1:1 scaling bottom/right clip */
-	drm_rect_init(&src, 0, 0, 2 << 16, 2 << 16);
-	drm_rect_init(&dst, 0, 0, 2, 2);
-	drm_rect_init(&clip, 1, 1, 1, 1);
-
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-
-	FAIL(src.x1 != 1 << 16 || src.x2 != 2 << 16 ||
-	     src.y1 != 1 << 16 || src.y2 != 2 << 16,
-	     "Source badly clipped\n");
-	FAIL(dst.x1 != 1 || dst.x2 != 2 ||
-	     dst.y1 != 1 || dst.y2 != 2,
-	     "Destination badly clipped\n");
-	FAIL(!visible, "Destination should be visible\n");
-	FAIL(!drm_rect_visible(&src), "Source should be visible\n");
-
-	/* 2:1 scaling top/left clip */
-	drm_rect_init(&src, 0, 0, 4 << 16, 4 << 16);
-	drm_rect_init(&dst, 0, 0, 2, 2);
-	drm_rect_init(&clip, 0, 0, 1, 1);
-
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-
-	FAIL(src.x1 != 0 || src.x2 != 2 << 16 ||
-	     src.y1 != 0 || src.y2 != 2 << 16,
-	     "Source badly clipped\n");
-	FAIL(dst.x1 != 0 || dst.x2 != 1 ||
-	     dst.y1 != 0 || dst.y2 != 1,
-	     "Destination badly clipped\n");
-	FAIL(!visible, "Destination should be visible\n");
-	FAIL(!drm_rect_visible(&src), "Source should be visible\n");
-
-	/* 2:1 scaling bottom/right clip */
-	drm_rect_init(&src, 0, 0, 4 << 16, 4 << 16);
-	drm_rect_init(&dst, 0, 0, 2, 2);
-	drm_rect_init(&clip, 1, 1, 1, 1);
-
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-
-	FAIL(src.x1 != 2 << 16 || src.x2 != 4 << 16 ||
-	     src.y1 != 2 << 16 || src.y2 != 4 << 16,
-	     "Source badly clipped\n");
-	FAIL(dst.x1 != 1 || dst.x2 != 2 ||
-	     dst.y1 != 1 || dst.y2 != 2,
-	     "Destination badly clipped\n");
-	FAIL(!visible, "Destination should be visible\n");
-	FAIL(!drm_rect_visible(&src), "Source should be visible\n");
-
-	/* 1:2 scaling top/left clip */
-	drm_rect_init(&src, 0, 0, 2 << 16, 2 << 16);
-	drm_rect_init(&dst, 0, 0, 4, 4);
-	drm_rect_init(&clip, 0, 0, 2, 2);
-
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-
-	FAIL(src.x1 != 0 || src.x2 != 1 << 16 ||
-	     src.y1 != 0 || src.y2 != 1 << 16,
-	     "Source badly clipped\n");
-	FAIL(dst.x1 != 0 || dst.x2 != 2 ||
-	     dst.y1 != 0 || dst.y2 != 2,
-	     "Destination badly clipped\n");
-	FAIL(!visible, "Destination should be visible\n");
-	FAIL(!drm_rect_visible(&src), "Source should be visible\n");
-
-	/* 1:2 scaling bottom/right clip */
-	drm_rect_init(&src, 0, 0, 2 << 16, 2 << 16);
-	drm_rect_init(&dst, 0, 0, 4, 4);
-	drm_rect_init(&clip, 2, 2, 2, 2);
-
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-
-	FAIL(src.x1 != 1 << 16 || src.x2 != 2 << 16 ||
-	     src.y1 != 1 << 16 || src.y2 != 2 << 16,
-	     "Source badly clipped\n");
-	FAIL(dst.x1 != 2 || dst.x2 != 4 ||
-	     dst.y1 != 2 || dst.y2 != 4,
-	     "Destination badly clipped\n");
-	FAIL(!visible, "Destination should be visible\n");
-	FAIL(!drm_rect_visible(&src), "Source should be visible\n");
-
-	return 0;
-}
-
-int igt_drm_rect_clip_scaled_signed_vs_unsigned(void *ignored)
-{
-	struct drm_rect src, dst, clip;
-	bool visible;
-
-	/*
-	 * 'clip.x2 - dst.x1 >= dst width' could result a negative
-	 * src rectangle width which is no longer expected by the
-	 * code as it's using unsigned types. This could lead to
-	 * the clipped source rectangle appering visible when it
-	 * should have been fully clipped. Make sure both rectangles
-	 * end up invisible.
-	 */
-	drm_rect_init(&src, 0, 0, INT_MAX, INT_MAX);
-	drm_rect_init(&dst, 0, 0, 2, 2);
-	drm_rect_init(&clip, 3, 3, 1, 1);
-
-	visible = drm_rect_clip_scaled(&src, &dst, &clip);
-
-	FAIL(visible, "Destination should not be visible\n");
-	FAIL(drm_rect_visible(&src), "Source should not be visible\n");
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/shmobile/Kconfig b/drivers/gpu/drm/shmobile/Kconfig
index 288b838a904a..4ec5dc74a6b0 100644
--- a/drivers/gpu/drm/shmobile/Kconfig
+++ b/drivers/gpu/drm/shmobile/Kconfig
@@ -5,7 +5,7 @@ config DRM_SHMOBILE
 	depends on ARCH_SHMOBILE || COMPILE_TEST
 	select BACKLIGHT_CLASS_DEVICE
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	help
 	  Choose this option if you have an SH Mobile chipset.
 	  If M is selected the module will be called shmob-drm.
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
index 071a929e9fe3..4624c0aff51f 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
@@ -12,11 +12,10 @@
 
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_vblank.h>
@@ -289,18 +288,18 @@ static void shmob_drm_crtc_compute_base(struct shmob_drm_crtc *scrtc,
 {
 	struct drm_crtc *crtc = &scrtc->crtc;
 	struct drm_framebuffer *fb = crtc->primary->fb;
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 	unsigned int bpp;
 
 	bpp = scrtc->format->yuv ? 8 : scrtc->format->bpp;
-	gem = drm_fb_cma_get_gem_obj(fb, 0);
-	scrtc->dma[0] = gem->paddr + fb->offsets[0]
+	gem = drm_fb_dma_get_gem_obj(fb, 0);
+	scrtc->dma[0] = gem->dma_addr + fb->offsets[0]
 		      + y * fb->pitches[0] + x * bpp / 8;
 
 	if (scrtc->format->yuv) {
 		bpp = scrtc->format->bpp - 8;
-		gem = drm_fb_cma_get_gem_obj(fb, 1);
-		scrtc->dma[1] = gem->paddr + fb->offsets[1]
+		gem = drm_fb_dma_get_gem_obj(fb, 1);
+		scrtc->dma[1] = gem->dma_addr + fb->offsets[1]
 			      + y / (bpp == 4 ? 2 : 1) * fb->pitches[1]
 			      + x * (bpp == 16 ? 2 : 1);
 	}
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_drv.c b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
index 731cbad7520f..3d511fa38913 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_drv.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
@@ -17,7 +17,7 @@
 
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
@@ -126,11 +126,11 @@ static irqreturn_t shmob_drm_irq(int irq, void *arg)
 	return IRQ_HANDLED;
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(shmob_drm_fops);
+DEFINE_DRM_GEM_DMA_FOPS(shmob_drm_fops);
 
 static const struct drm_driver shmob_drm_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 	.fops			= &shmob_drm_fops,
 	.name			= "shmob-drm",
 	.desc			= "Renesas SH Mobile DRM",
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_kms.c b/drivers/gpu/drm/shmobile/shmob_drm_kms.c
index 68d21be784aa..60a2c8d8a0d9 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_kms.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_kms.c
@@ -9,9 +9,8 @@
 
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_probe_helper.h>
 
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_kms.h b/drivers/gpu/drm/shmobile/shmob_drm_kms.h
index 6ec2b732bb94..0347b1fd2338 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_kms.h
+++ b/drivers/gpu/drm/shmobile/shmob_drm_kms.h
@@ -12,7 +12,7 @@
 
 #include <linux/types.h>
 
-struct drm_gem_cma_object;
+struct drm_gem_dma_object;
 struct shmob_drm_device;
 
 struct shmob_drm_format_info {
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_plane.c b/drivers/gpu/drm/shmobile/shmob_drm_plane.c
index 4763ea8e1af0..6c5f0cbe7d95 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_plane.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_plane.c
@@ -9,10 +9,10 @@
 
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 
 #include "shmob_drm_drv.h"
 #include "shmob_drm_kms.h"
@@ -41,18 +41,18 @@ static void shmob_drm_plane_compute_base(struct shmob_drm_plane *splane,
 					 struct drm_framebuffer *fb,
 					 int x, int y)
 {
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 	unsigned int bpp;
 
 	bpp = splane->format->yuv ? 8 : splane->format->bpp;
-	gem = drm_fb_cma_get_gem_obj(fb, 0);
-	splane->dma[0] = gem->paddr + fb->offsets[0]
+	gem = drm_fb_dma_get_gem_obj(fb, 0);
+	splane->dma[0] = gem->dma_addr + fb->offsets[0]
 		       + y * fb->pitches[0] + x * bpp / 8;
 
 	if (splane->format->yuv) {
 		bpp = splane->format->bpp - 8;
-		gem = drm_fb_cma_get_gem_obj(fb, 1);
-		splane->dma[1] = gem->paddr + fb->offsets[1]
+		gem = drm_fb_dma_get_gem_obj(fb, 1);
+		splane->dma[1] = gem->dma_addr + fb->offsets[1]
 			       + y / (bpp == 4 ? 2 : 1) * fb->pitches[1]
 			       + x * (bpp == 16 ? 2 : 1);
 	}
@@ -252,9 +252,10 @@ int shmob_drm_plane_create(struct shmob_drm_device *sdev, unsigned int index)
 	splane->index = index;
 	splane->alpha = 255;
 
-	ret = drm_plane_init(sdev->ddev, &splane->plane, 1,
-			     &shmob_drm_plane_funcs, formats,
-			     ARRAY_SIZE(formats), false);
+	ret = drm_universal_plane_init(sdev->ddev, &splane->plane, 1,
+				       &shmob_drm_plane_funcs,
+				       formats, ARRAY_SIZE(formats), NULL,
+				       DRM_PLANE_TYPE_OVERLAY, NULL);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/sis/sis_drv.c b/drivers/gpu/drm/sis/sis_drv.c
index e35e719cf315..6173020a9bf5 100644
--- a/drivers/gpu/drm/sis/sis_drv.c
+++ b/drivers/gpu/drm/sis/sis_drv.c
@@ -50,7 +50,7 @@ static int sis_driver_load(struct drm_device *dev, unsigned long chipset)
 	if (dev_priv == NULL)
 		return -ENOMEM;
 
-	idr_init(&dev_priv->object_idr);
+	idr_init_base(&dev_priv->object_idr, 1);
 	dev->dev_private = (void *)dev_priv;
 	dev_priv->chipset = chipset;
 
diff --git a/drivers/gpu/drm/solomon/ssd130x-i2c.c b/drivers/gpu/drm/solomon/ssd130x-i2c.c
index 1e0fcec7be47..ddfa0bb5d9c9 100644
--- a/drivers/gpu/drm/solomon/ssd130x-i2c.c
+++ b/drivers/gpu/drm/solomon/ssd130x-i2c.c
@@ -39,13 +39,11 @@ static int ssd130x_i2c_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int ssd130x_i2c_remove(struct i2c_client *client)
+static void ssd130x_i2c_remove(struct i2c_client *client)
 {
 	struct ssd130x_device *ssd130x = i2c_get_clientdata(client);
 
 	ssd130x_remove(ssd130x);
-
-	return 0;
 }
 
 static void ssd130x_i2c_shutdown(struct i2c_client *client)
diff --git a/drivers/gpu/drm/solomon/ssd130x-spi.c b/drivers/gpu/drm/solomon/ssd130x-spi.c
index 07802907e39a..19ab4942cb33 100644
--- a/drivers/gpu/drm/solomon/ssd130x-spi.c
+++ b/drivers/gpu/drm/solomon/ssd130x-spi.c
@@ -18,11 +18,6 @@ struct ssd130x_spi_transport {
 	struct gpio_desc *dc;
 };
 
-static const struct regmap_config ssd130x_spi_regmap_config = {
-	.reg_bits = 8,
-	.val_bits = 8,
-};
-
 /*
  * The regmap bus .write handler, it is just a wrapper around spi_write()
  * but toggling the Data/Command control pin (D/C#). Since for 4-wire SPI
@@ -56,17 +51,12 @@ static int ssd130x_spi_read(void *context, const void *reg, size_t reg_size,
 	return -EOPNOTSUPP;
 }
 
-/*
- * A custom bus is needed due the special write that toggles a D/C# pin,
- * another option could be to just have a .reg_write() callback but that
- * will prevent to do data writes in bulk.
- *
- * Once the regmap API is extended to support defining a bulk write handler
- * in the struct regmap_config, this can be simplified and the bus dropped.
- */
-static struct regmap_bus regmap_ssd130x_spi_bus = {
+static const struct regmap_config ssd130x_spi_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
 	.write = ssd130x_spi_write,
 	.read = ssd130x_spi_read,
+	.can_multi_write = true,
 };
 
 static int ssd130x_spi_probe(struct spi_device *spi)
@@ -90,8 +80,7 @@ static int ssd130x_spi_probe(struct spi_device *spi)
 	t->spi = spi;
 	t->dc = dc;
 
-	regmap = devm_regmap_init(dev, &regmap_ssd130x_spi_bus, t,
-				  &ssd130x_spi_regmap_config);
+	regmap = devm_regmap_init(dev, NULL, t, &ssd130x_spi_regmap_config);
 	if (IS_ERR(regmap))
 		return PTR_ERR(regmap);
 
diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c
index 5a3e3b78cd9e..bc41a5ae810a 100644
--- a/drivers/gpu/drm/solomon/ssd130x.c
+++ b/drivers/gpu/drm/solomon/ssd130x.c
@@ -18,10 +18,10 @@
 #include <linux/pwm.h>
 #include <linux/regulator/consumer.h>
 
+#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_edid.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_format_helper.h>
 #include <drm/drm_framebuffer.h>
@@ -537,11 +537,11 @@ static void ssd130x_clear_screen(struct ssd130x_device *ssd130x)
 	kfree(buf);
 }
 
-static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb, const struct iosys_map *map,
+static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb, const struct iosys_map *vmap,
 				struct drm_rect *rect)
 {
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(fb->dev);
-	void *vmap = map->vaddr; /* TODO: Use mapping abstraction properly */
+	struct iosys_map dst;
 	unsigned int dst_pitch;
 	int ret = 0;
 	u8 *buf = NULL;
@@ -555,127 +555,174 @@ static int ssd130x_fb_blit_rect(struct drm_framebuffer *fb, const struct iosys_m
 	if (!buf)
 		return -ENOMEM;
 
-	drm_fb_xrgb8888_to_mono(buf, dst_pitch, vmap, fb, rect);
+	ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE);
+	if (ret)
+		goto out_free;
+
+	iosys_map_set_vaddr(&dst, buf);
+	drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, vmap, fb, rect);
+
+	drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
 
 	ssd130x_update_rect(ssd130x, buf, rect);
 
+out_free:
 	kfree(buf);
 
 	return ret;
 }
 
-static int ssd130x_display_pipe_mode_valid(struct drm_simple_display_pipe *pipe,
-					   const struct drm_display_mode *mode)
+static void ssd130x_primary_plane_helper_atomic_update(struct drm_plane *plane,
+						       struct drm_atomic_state *state)
 {
-	struct ssd130x_device *ssd130x = drm_to_ssd130x(pipe->crtc.dev);
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
+	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
+	struct drm_device *drm = plane->dev;
+	struct drm_rect src_clip, dst_clip;
+	int idx;
 
-	if (mode->hdisplay != ssd130x->mode.hdisplay &&
-	    mode->vdisplay != ssd130x->mode.vdisplay)
-		return MODE_ONE_SIZE;
+	if (!drm_atomic_helper_damage_merged(old_plane_state, plane_state, &src_clip))
+		return;
 
-	if (mode->hdisplay != ssd130x->mode.hdisplay)
-		return MODE_ONE_WIDTH;
+	dst_clip = plane_state->dst;
+	if (!drm_rect_intersect(&dst_clip, &src_clip))
+		return;
 
-	if (mode->vdisplay != ssd130x->mode.vdisplay)
-		return MODE_ONE_HEIGHT;
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
-	return MODE_OK;
+	ssd130x_fb_blit_rect(plane_state->fb, &shadow_plane_state->data[0], &dst_clip);
+
+	drm_dev_exit(idx);
 }
 
-static void ssd130x_display_pipe_enable(struct drm_simple_display_pipe *pipe,
-					struct drm_crtc_state *crtc_state,
-					struct drm_plane_state *plane_state)
+static void ssd130x_primary_plane_helper_atomic_disable(struct drm_plane *plane,
+							struct drm_atomic_state *state)
 {
-	struct ssd130x_device *ssd130x = drm_to_ssd130x(pipe->crtc.dev);
-	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
-	struct drm_device *drm = &ssd130x->drm;
-	int idx, ret;
+	struct drm_device *drm = plane->dev;
+	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
+	int idx;
 
-	ret = ssd130x_power_on(ssd130x);
-	if (ret)
+	if (!drm_dev_enter(drm, &idx))
 		return;
 
-	ret = ssd130x_init(ssd130x);
-	if (ret)
-		goto out_power_off;
+	ssd130x_clear_screen(ssd130x);
 
-	if (!drm_dev_enter(drm, &idx))
-		goto out_power_off;
+	drm_dev_exit(idx);
+}
 
-	ssd130x_fb_blit_rect(plane_state->fb, &shadow_plane_state->data[0], &plane_state->dst);
+static const struct drm_plane_helper_funcs ssd130x_primary_plane_helper_funcs = {
+	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
+	.atomic_check = drm_plane_helper_atomic_check,
+	.atomic_update = ssd130x_primary_plane_helper_atomic_update,
+	.atomic_disable = ssd130x_primary_plane_helper_atomic_disable,
+};
 
-	ssd130x_write_cmd(ssd130x, 1, SSD130X_DISPLAY_ON);
+static const struct drm_plane_funcs ssd130x_primary_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = drm_plane_cleanup,
+	DRM_GEM_SHADOW_PLANE_FUNCS,
+};
 
-	backlight_enable(ssd130x->bl_dev);
+static enum drm_mode_status ssd130x_crtc_helper_mode_valid(struct drm_crtc *crtc,
+							   const struct drm_display_mode *mode)
+{
+	struct ssd130x_device *ssd130x = drm_to_ssd130x(crtc->dev);
 
-	drm_dev_exit(idx);
+	if (mode->hdisplay != ssd130x->mode.hdisplay &&
+	    mode->vdisplay != ssd130x->mode.vdisplay)
+		return MODE_ONE_SIZE;
+	else if (mode->hdisplay != ssd130x->mode.hdisplay)
+		return MODE_ONE_WIDTH;
+	else if (mode->vdisplay != ssd130x->mode.vdisplay)
+		return MODE_ONE_HEIGHT;
 
-	return;
-out_power_off:
-	ssd130x_power_off(ssd130x);
+	return MODE_OK;
 }
 
-static void ssd130x_display_pipe_disable(struct drm_simple_display_pipe *pipe)
+static int ssd130x_crtc_helper_atomic_check(struct drm_crtc *crtc,
+					    struct drm_atomic_state *new_state)
 {
-	struct ssd130x_device *ssd130x = drm_to_ssd130x(pipe->crtc.dev);
-	struct drm_device *drm = &ssd130x->drm;
-	int idx;
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	int ret;
 
-	if (!drm_dev_enter(drm, &idx))
-		return;
+	ret = drm_atomic_helper_check_crtc_state(new_crtc_state, false);
+	if (ret)
+		return ret;
 
-	ssd130x_clear_screen(ssd130x);
+	return drm_atomic_add_affected_planes(new_state, crtc);
+}
 
-	backlight_disable(ssd130x->bl_dev);
+/*
+ * The CRTC is always enabled. Screen updates are performed by
+ * the primary plane's atomic_update function. Disabling clears
+ * the screen in the primary plane's atomic_disable function.
+ */
+static const struct drm_crtc_helper_funcs ssd130x_crtc_helper_funcs = {
+	.mode_valid = ssd130x_crtc_helper_mode_valid,
+	.atomic_check = ssd130x_crtc_helper_atomic_check,
+};
 
-	ssd130x_write_cmd(ssd130x, 1, SSD130X_DISPLAY_OFF);
+static void ssd130x_crtc_reset(struct drm_crtc *crtc)
+{
+	struct drm_device *drm = crtc->dev;
+	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
 
-	ssd130x_power_off(ssd130x);
+	ssd130x_init(ssd130x);
 
-	drm_dev_exit(idx);
+	drm_atomic_helper_crtc_reset(crtc);
 }
 
-static void ssd130x_display_pipe_update(struct drm_simple_display_pipe *pipe,
-					struct drm_plane_state *old_plane_state)
+static const struct drm_crtc_funcs ssd130x_crtc_funcs = {
+	.reset = ssd130x_crtc_reset,
+	.destroy = drm_crtc_cleanup,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+};
+
+static void ssd130x_encoder_helper_atomic_enable(struct drm_encoder *encoder,
+						 struct drm_atomic_state *state)
 {
-	struct ssd130x_device *ssd130x = drm_to_ssd130x(pipe->crtc.dev);
-	struct drm_plane_state *plane_state = pipe->plane.state;
-	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
-	struct drm_framebuffer *fb = plane_state->fb;
-	struct drm_device *drm = &ssd130x->drm;
-	struct drm_rect src_clip, dst_clip;
-	int idx;
+	struct drm_device *drm = encoder->dev;
+	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
+	int ret;
 
-	if (!fb)
+	ret = ssd130x_power_on(ssd130x);
+	if (ret)
 		return;
 
-	if (!pipe->crtc.state->active)
-		return;
+	ssd130x_write_cmd(ssd130x, 1, SSD130X_DISPLAY_ON);
 
-	if (!drm_atomic_helper_damage_merged(old_plane_state, plane_state, &src_clip))
-		return;
+	backlight_enable(ssd130x->bl_dev);
+}
 
-	dst_clip = plane_state->dst;
-	if (!drm_rect_intersect(&dst_clip, &src_clip))
-		return;
+static void ssd130x_encoder_helper_atomic_disable(struct drm_encoder *encoder,
+						  struct drm_atomic_state *state)
+{
+	struct drm_device *drm = encoder->dev;
+	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
 
-	if (!drm_dev_enter(drm, &idx))
-		return;
+	backlight_disable(ssd130x->bl_dev);
 
-	ssd130x_fb_blit_rect(plane_state->fb, &shadow_plane_state->data[0], &dst_clip);
+	ssd130x_write_cmd(ssd130x, 1, SSD130X_DISPLAY_OFF);
 
-	drm_dev_exit(idx);
+	ssd130x_power_off(ssd130x);
 }
 
-static const struct drm_simple_display_pipe_funcs ssd130x_pipe_funcs = {
-	.mode_valid = ssd130x_display_pipe_mode_valid,
-	.enable = ssd130x_display_pipe_enable,
-	.disable = ssd130x_display_pipe_disable,
-	.update = ssd130x_display_pipe_update,
-	DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS,
+static const struct drm_encoder_helper_funcs ssd130x_encoder_helper_funcs = {
+	.atomic_enable = ssd130x_encoder_helper_atomic_enable,
+	.atomic_disable = ssd130x_encoder_helper_atomic_disable,
+};
+
+static const struct drm_encoder_funcs ssd130x_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
 };
 
-static int ssd130x_connector_get_modes(struct drm_connector *connector)
+static int ssd130x_connector_helper_get_modes(struct drm_connector *connector)
 {
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(connector->dev);
 	struct drm_display_mode *mode;
@@ -695,7 +742,7 @@ static int ssd130x_connector_get_modes(struct drm_connector *connector)
 }
 
 static const struct drm_connector_helper_funcs ssd130x_connector_helper_funcs = {
-	.get_modes = ssd130x_connector_get_modes,
+	.get_modes = ssd130x_connector_helper_get_modes,
 };
 
 static const struct drm_connector_funcs ssd130x_connector_funcs = {
@@ -806,8 +853,16 @@ static int ssd130x_init_modeset(struct ssd130x_device *ssd130x)
 	struct device *dev = ssd130x->dev;
 	struct drm_device *drm = &ssd130x->drm;
 	unsigned long max_width, max_height;
+	struct drm_plane *primary_plane;
+	struct drm_crtc *crtc;
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
 	int ret;
 
+	/*
+	 * Modesetting
+	 */
+
 	ret = drmm_mode_config_init(drm);
 	if (ret) {
 		dev_err(dev, "DRM mode config init failed: %d\n", ret);
@@ -833,25 +888,65 @@ static int ssd130x_init_modeset(struct ssd130x_device *ssd130x)
 	drm->mode_config.preferred_depth = 32;
 	drm->mode_config.funcs = &ssd130x_mode_config_funcs;
 
-	ret = drm_connector_init(drm, &ssd130x->connector, &ssd130x_connector_funcs,
+	/* Primary plane */
+
+	primary_plane = &ssd130x->primary_plane;
+	ret = drm_universal_plane_init(drm, primary_plane, 0, &ssd130x_primary_plane_funcs,
+				       ssd130x_formats, ARRAY_SIZE(ssd130x_formats),
+				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
+	if (ret) {
+		dev_err(dev, "DRM primary plane init failed: %d\n", ret);
+		return ret;
+	}
+
+	drm_plane_helper_add(primary_plane, &ssd130x_primary_plane_helper_funcs);
+
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	/* CRTC */
+
+	crtc = &ssd130x->crtc;
+	ret = drm_crtc_init_with_planes(drm, crtc, primary_plane, NULL,
+					&ssd130x_crtc_funcs, NULL);
+	if (ret) {
+		dev_err(dev, "DRM crtc init failed: %d\n", ret);
+		return ret;
+	}
+
+	drm_crtc_helper_add(crtc, &ssd130x_crtc_helper_funcs);
+
+	/* Encoder */
+
+	encoder = &ssd130x->encoder;
+	ret = drm_encoder_init(drm, encoder, &ssd130x_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
+	if (ret) {
+		dev_err(dev, "DRM encoder init failed: %d\n", ret);
+		return ret;
+	}
+
+	drm_encoder_helper_add(encoder, &ssd130x_encoder_helper_funcs);
+
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+
+	/* Connector */
+
+	connector = &ssd130x->connector;
+	ret = drm_connector_init(drm, connector, &ssd130x_connector_funcs,
 				 DRM_MODE_CONNECTOR_Unknown);
 	if (ret) {
 		dev_err(dev, "DRM connector init failed: %d\n", ret);
 		return ret;
 	}
 
-	drm_connector_helper_add(&ssd130x->connector, &ssd130x_connector_helper_funcs);
+	drm_connector_helper_add(connector, &ssd130x_connector_helper_funcs);
 
-	ret = drm_simple_display_pipe_init(drm, &ssd130x->pipe, &ssd130x_pipe_funcs,
-					   ssd130x_formats, ARRAY_SIZE(ssd130x_formats),
-					   NULL, &ssd130x->connector);
+	ret = drm_connector_attach_encoder(connector, encoder);
 	if (ret) {
-		dev_err(dev, "DRM simple display pipeline init failed: %d\n", ret);
+		dev_err(dev, "DRM attach connector to encoder failed: %d\n", ret);
 		return ret;
 	}
 
-	drm_plane_enable_fb_damage_clips(&ssd130x->pipe.plane);
-
 	drm_mode_config_reset(drm);
 
 	return 0;
diff --git a/drivers/gpu/drm/solomon/ssd130x.h b/drivers/gpu/drm/solomon/ssd130x.h
index 4c4a84e962e7..03038c1b6476 100644
--- a/drivers/gpu/drm/solomon/ssd130x.h
+++ b/drivers/gpu/drm/solomon/ssd130x.h
@@ -13,8 +13,11 @@
 #ifndef __SSD1307X_H__
 #define __SSD1307X_H__
 
+#include <drm/drm_connector.h>
+#include <drm/drm_crtc.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_simple_kms_helper.h>
+#include <drm/drm_encoder.h>
+#include <drm/drm_plane_helper.h>
 
 #include <linux/regmap.h>
 
@@ -42,8 +45,10 @@ struct ssd130x_deviceinfo {
 struct ssd130x_device {
 	struct drm_device drm;
 	struct device *dev;
-	struct drm_simple_display_pipe pipe;
 	struct drm_display_mode mode;
+	struct drm_plane primary_plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
 	struct drm_connector connector;
 	struct i2c_client *client;
 
diff --git a/drivers/gpu/drm/sprd/Kconfig b/drivers/gpu/drm/sprd/Kconfig
index 9a9c7ebfc716..e22b780fe822 100644
--- a/drivers/gpu/drm/sprd/Kconfig
+++ b/drivers/gpu/drm/sprd/Kconfig
@@ -2,7 +2,7 @@ config DRM_SPRD
 	tristate "DRM Support for Unisoc SoCs Platform"
 	depends on ARCH_SPRD || COMPILE_TEST
 	depends on DRM && OF
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DSI
 	select VIDEOMODE_HELPERS
diff --git a/drivers/gpu/drm/sprd/sprd_dpu.c b/drivers/gpu/drm/sprd/sprd_dpu.c
index 3664089b6983..88f4259680f1 100644
--- a/drivers/gpu/drm/sprd/sprd_dpu.c
+++ b/drivers/gpu/drm/sprd/sprd_dpu.c
@@ -19,11 +19,10 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_crtc_helper.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
-#include <drm/drm_plane_helper.h>
 
 #include "sprd_drm.h"
 #include "sprd_dpu.h"
@@ -324,7 +323,7 @@ static u32 drm_blend_to_dpu(struct drm_plane_state *state)
 static void sprd_dpu_layer(struct sprd_dpu *dpu, struct drm_plane_state *state)
 {
 	struct dpu_context *ctx = &dpu->ctx;
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	struct drm_framebuffer *fb = state->fb;
 	u32 addr, size, offset, pitch, blend, format, rotation;
 	u32 src_x = state->src_x >> 16;
@@ -341,8 +340,8 @@ static void sprd_dpu_layer(struct sprd_dpu *dpu, struct drm_plane_state *state)
 	size = (src_w & 0xffff) | (src_h << 16);
 
 	for (i = 0; i < fb->format->num_planes; i++) {
-		cma_obj = drm_fb_cma_get_gem_obj(fb, i);
-		addr = cma_obj->paddr + fb->offsets[i];
+		dma_obj = drm_fb_dma_get_gem_obj(fb, i);
+		addr = dma_obj->dma_addr + fb->offsets[i];
 
 		if (i == 0)
 			layer_reg_wr(ctx, REG_LAY_BASE_ADDR0, addr, index);
@@ -524,8 +523,8 @@ static int sprd_plane_atomic_check(struct drm_plane *plane,
 		return PTR_ERR(crtc_state);
 
 	return drm_atomic_helper_check_plane_state(plane_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  true, true);
 }
 
diff --git a/drivers/gpu/drm/sprd/sprd_drm.c b/drivers/gpu/drm/sprd/sprd_drm.c
index b8fc1c6a0cb8..9d42f17a5734 100644
--- a/drivers/gpu/drm/sprd/sprd_drm.c
+++ b/drivers/gpu/drm/sprd/sprd_drm.c
@@ -13,7 +13,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_of.h>
 #include <drm/drm_probe_helper.h>
@@ -48,14 +48,14 @@ static void sprd_drm_mode_config_init(struct drm_device *drm)
 	drm->mode_config.helper_private = &sprd_drm_mode_config_helper;
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(sprd_drm_fops);
+DEFINE_DRM_GEM_DMA_FOPS(sprd_drm_fops);
 
 static struct drm_driver sprd_drm_drv = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &sprd_drm_fops,
 
 	/* GEM Operations */
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 
 	.name			= DRIVER_NAME,
 	.desc			= DRIVER_DESC,
diff --git a/drivers/gpu/drm/sti/Kconfig b/drivers/gpu/drm/sti/Kconfig
index 246a94afbe74..f2a880c48485 100644
--- a/drivers/gpu/drm/sti/Kconfig
+++ b/drivers/gpu/drm/sti/Kconfig
@@ -4,7 +4,7 @@ config DRM_STI
 	depends on OF && DRM && (ARCH_STI || ARCH_MULTIPLATFORM)
 	select RESET_CONTROLLER
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_PANEL
 	select FW_LOADER
 	select SND_SOC_HDMI_CODEC if SND_SOC
diff --git a/drivers/gpu/drm/sti/sti_crtc.c b/drivers/gpu/drm/sti/sti_crtc.c
index 409795786f03..3c7154f2d5f3 100644
--- a/drivers/gpu/drm/sti/sti_crtc.c
+++ b/drivers/gpu/drm/sti/sti_crtc.c
@@ -11,7 +11,6 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_device.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
diff --git a/drivers/gpu/drm/sti/sti_cursor.c b/drivers/gpu/drm/sti/sti_cursor.c
index 1e9bd4241f10..db0a1eb53532 100644
--- a/drivers/gpu/drm/sti/sti_cursor.c
+++ b/drivers/gpu/drm/sti/sti_cursor.c
@@ -11,9 +11,9 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 
 #include "sti_compositor.h"
 #include "sti_cursor.h"
@@ -243,8 +243,8 @@ static int sti_cursor_atomic_check(struct drm_plane *drm_plane,
 		}
 	}
 
-	if (!drm_fb_cma_get_gem_obj(fb, 0)) {
-		DRM_ERROR("Can't get CMA GEM object for fb\n");
+	if (!drm_fb_dma_get_gem_obj(fb, 0)) {
+		DRM_ERROR("Can't get DMA GEM object for fb\n");
 		return -EINVAL;
 	}
 
@@ -267,7 +267,7 @@ static void sti_cursor_atomic_update(struct drm_plane *drm_plane,
 	struct drm_framebuffer *fb = newstate->fb;
 	struct drm_display_mode *mode;
 	int dst_x, dst_y;
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	u32 y, x;
 	u32 val;
 
@@ -278,10 +278,10 @@ static void sti_cursor_atomic_update(struct drm_plane *drm_plane,
 	dst_x = newstate->crtc_x;
 	dst_y = newstate->crtc_y;
 
-	cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+	dma_obj = drm_fb_dma_get_gem_obj(fb, 0);
 
 	/* Convert ARGB8888 to CLUT8 */
-	sti_cursor_argb8888_to_clut8(cursor, (u32 *)cma_obj->vaddr);
+	sti_cursor_argb8888_to_clut8(cursor, (u32 *)dma_obj->vaddr);
 
 	/* AWS and AWE depend on the mode */
 	y = sti_vtg_get_line_number(*mode, 0);
diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c
index d858209cf8de..7abf010a3293 100644
--- a/drivers/gpu/drm/sti/sti_drv.c
+++ b/drivers/gpu/drm/sti/sti_drv.c
@@ -14,9 +14,8 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_debugfs.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_of.h>
 #include <drm/drm_probe_helper.h>
@@ -128,12 +127,12 @@ static void sti_mode_config_init(struct drm_device *dev)
 	dev->mode_config.normalize_zpos = true;
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(sti_driver_fops);
+DEFINE_DRM_GEM_DMA_FOPS(sti_driver_fops);
 
 static const struct drm_driver sti_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
 	.fops = &sti_driver_fops,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 
 	.debugfs_init = sti_drm_dbg_init,
 
diff --git a/drivers/gpu/drm/sti/sti_gdp.c b/drivers/gpu/drm/sti/sti_gdp.c
index af783f599306..43c72c2604a0 100644
--- a/drivers/gpu/drm/sti/sti_gdp.c
+++ b/drivers/gpu/drm/sti/sti_gdp.c
@@ -12,10 +12,10 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 
 #include "sti_compositor.h"
 #include "sti_gdp.h"
@@ -658,8 +658,8 @@ static int sti_gdp_atomic_check(struct drm_plane *drm_plane,
 		return -EINVAL;
 	}
 
-	if (!drm_fb_cma_get_gem_obj(fb, 0)) {
-		DRM_ERROR("Can't get CMA GEM object for fb\n");
+	if (!drm_fb_dma_get_gem_obj(fb, 0)) {
+		DRM_ERROR("Can't get DMA GEM object for fb\n");
 		return -EINVAL;
 	}
 
@@ -714,7 +714,7 @@ static void sti_gdp_atomic_update(struct drm_plane *drm_plane,
 	struct drm_display_mode *mode;
 	int dst_x, dst_y, dst_w, dst_h;
 	int src_x, src_y, src_w, src_h;
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	struct sti_gdp_node_list *list;
 	struct sti_gdp_node_list *curr_list;
 	struct sti_gdp_node *top_field, *btm_field;
@@ -778,15 +778,15 @@ static void sti_gdp_atomic_update(struct drm_plane *drm_plane,
 	top_field->gam_gdp_ctl |= sti_gdp_get_alpharange(format);
 	top_field->gam_gdp_ppt &= ~GAM_GDP_PPT_IGNORE;
 
-	cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+	dma_obj = drm_fb_dma_get_gem_obj(fb, 0);
 
 	DRM_DEBUG_DRIVER("drm FB:%d format:%.4s phys@:0x%lx\n", fb->base.id,
 			 (char *)&fb->format->format,
-			 (unsigned long)cma_obj->paddr);
+			 (unsigned long) dma_obj->dma_addr);
 
 	/* pixel memory location */
 	bpp = fb->format->cpp[0];
-	top_field->gam_gdp_pml = (u32)cma_obj->paddr + fb->offsets[0];
+	top_field->gam_gdp_pml = (u32) dma_obj->dma_addr + fb->offsets[0];
 	top_field->gam_gdp_pml += src_x * bpp;
 	top_field->gam_gdp_pml += src_y * fb->pitches[0];
 
@@ -831,7 +831,7 @@ static void sti_gdp_atomic_update(struct drm_plane *drm_plane,
 	dev_dbg(gdp->dev, "Current NVN:0x%X\n",
 		readl(gdp->regs + GAM_GDP_NVN_OFFSET));
 	dev_dbg(gdp->dev, "Posted buff: %lx current buff: %x\n",
-		(unsigned long)cma_obj->paddr,
+		(unsigned long) dma_obj->dma_addr,
 		readl(gdp->regs + GAM_GDP_PML_OFFSET));
 
 	if (!curr_list) {
diff --git a/drivers/gpu/drm/sti/sti_hqvdp.c b/drivers/gpu/drm/sti/sti_hqvdp.c
index 271982080437..02b77279f6e4 100644
--- a/drivers/gpu/drm/sti/sti_hqvdp.c
+++ b/drivers/gpu/drm/sti/sti_hqvdp.c
@@ -16,10 +16,10 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 
 #include "sti_compositor.h"
 #include "sti_drv.h"
@@ -1055,8 +1055,8 @@ static int sti_hqvdp_atomic_check(struct drm_plane *drm_plane,
 		return -EINVAL;
 	}
 
-	if (!drm_fb_cma_get_gem_obj(fb, 0)) {
-		DRM_ERROR("Can't get CMA GEM object for fb\n");
+	if (!drm_fb_dma_get_gem_obj(fb, 0)) {
+		DRM_ERROR("Can't get DMA GEM object for fb\n");
 		return -EINVAL;
 	}
 
@@ -1124,7 +1124,7 @@ static void sti_hqvdp_atomic_update(struct drm_plane *drm_plane,
 	struct drm_display_mode *mode;
 	int dst_x, dst_y, dst_w, dst_h;
 	int src_x, src_y, src_w, src_h;
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	struct sti_hqvdp_cmd *cmd;
 	int scale_h, scale_v;
 	int cmd_offset;
@@ -1178,15 +1178,15 @@ static void sti_hqvdp_atomic_update(struct drm_plane *drm_plane,
 	cmd->iqi.sat_gain = IQI_SAT_GAIN_DFLT;
 	cmd->iqi.pxf_conf = IQI_PXF_CONF_DFLT;
 
-	cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+	dma_obj = drm_fb_dma_get_gem_obj(fb, 0);
 
 	DRM_DEBUG_DRIVER("drm FB:%d format:%.4s phys@:0x%lx\n", fb->base.id,
 			 (char *)&fb->format->format,
-			 (unsigned long)cma_obj->paddr);
+			 (unsigned long) dma_obj->dma_addr);
 
 	/* Buffer planes address */
-	cmd->top.current_luma = (u32)cma_obj->paddr + fb->offsets[0];
-	cmd->top.current_chroma = (u32)cma_obj->paddr + fb->offsets[1];
+	cmd->top.current_luma = (u32) dma_obj->dma_addr + fb->offsets[0];
+	cmd->top.current_chroma = (u32) dma_obj->dma_addr + fb->offsets[1];
 
 	/* Pitches */
 	cmd->top.luma_processed_pitch = fb->pitches[0];
diff --git a/drivers/gpu/drm/sti/sti_plane.c b/drivers/gpu/drm/sti/sti_plane.c
index c74b524663ab..29e669ccec5b 100644
--- a/drivers/gpu/drm/sti/sti_plane.c
+++ b/drivers/gpu/drm/sti/sti_plane.c
@@ -9,10 +9,9 @@
 #include <linux/types.h>
 
 #include <drm/drm_blend.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 
 #include "sti_compositor.h"
 #include "sti_drv.h"
diff --git a/drivers/gpu/drm/sti/sti_plane.h b/drivers/gpu/drm/sti/sti_plane.h
index 8e33e629d9b0..2c0156bede9c 100644
--- a/drivers/gpu/drm/sti/sti_plane.h
+++ b/drivers/gpu/drm/sti/sti_plane.h
@@ -8,7 +8,6 @@
 #define _STI_PLANE_H_
 
 #include <drm/drm_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 
 #define to_sti_plane(x) container_of(x, struct sti_plane, drm_plane)
 
diff --git a/drivers/gpu/drm/stm/Kconfig b/drivers/gpu/drm/stm/Kconfig
index e0379488cd0d..ded72f879482 100644
--- a/drivers/gpu/drm/stm/Kconfig
+++ b/drivers/gpu/drm/stm/Kconfig
@@ -3,7 +3,7 @@ config DRM_STM
 	tristate "DRM Support for STMicroelectronics SoC Series"
 	depends on DRM && (ARCH_STM32 || ARCH_MULTIPLATFORM)
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_PANEL_BRIDGE
 	select VIDEOMODE_HELPERS
 	select FB_PROVIDE_GET_FB_UNMAPPED_AREA if FB
diff --git a/drivers/gpu/drm/stm/drv.c b/drivers/gpu/drm/stm/drv.c
index c63945dc2260..d7914f5122df 100644
--- a/drivers/gpu/drm/stm/drv.c
+++ b/drivers/gpu/drm/stm/drv.c
@@ -18,9 +18,8 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_probe_helper.h>
@@ -37,7 +36,7 @@ static const struct drm_mode_config_funcs drv_mode_config_funcs = {
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-static int stm_gem_cma_dumb_create(struct drm_file *file,
+static int stm_gem_dma_dumb_create(struct drm_file *file,
 				   struct drm_device *dev,
 				   struct drm_mode_create_dumb *args)
 {
@@ -50,10 +49,10 @@ static int stm_gem_cma_dumb_create(struct drm_file *file,
 	args->pitch = roundup(min_pitch, 128);
 	args->height = roundup(args->height, 4);
 
-	return drm_gem_cma_dumb_create_internal(file, dev, args);
+	return drm_gem_dma_dumb_create_internal(file, dev, args);
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(drv_driver_fops);
+DEFINE_DRM_GEM_DMA_FOPS(drv_driver_fops);
 
 static const struct drm_driver drv_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
@@ -64,7 +63,7 @@ static const struct drm_driver drv_driver = {
 	.minor = 0,
 	.patchlevel = 0,
 	.fops = &drv_driver_fops,
-	DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(stm_gem_cma_dumb_create),
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(stm_gem_dma_dumb_create),
 };
 
 static int drv_load(struct drm_device *ddev)
diff --git a/drivers/gpu/drm/stm/ltdc.c b/drivers/gpu/drm/stm/ltdc.c
index da7a0a183b27..03c6becda795 100644
--- a/drivers/gpu/drm/stm/ltdc.c
+++ b/drivers/gpu/drm/stm/ltdc.c
@@ -28,13 +28,12 @@
 #include <drm/drm_bridge.h>
 #include <drm/drm_device.h>
 #include <drm/drm_edid.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_of.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_vblank.h>
@@ -1347,7 +1346,7 @@ static void ltdc_plane_atomic_update(struct drm_plane *plane,
 	}
 
 	/* Sets the FB address */
-	paddr = (u32)drm_fb_cma_get_gem_addr(fb, newstate, 0);
+	paddr = (u32)drm_fb_dma_get_gem_addr(fb, newstate, 0);
 
 	if (newstate->rotation & DRM_MODE_REFLECT_X)
 		paddr += (fb->format->cpp[0] * (x1 - x0 + 1)) - 1;
@@ -1381,7 +1380,7 @@ static void ltdc_plane_atomic_update(struct drm_plane *plane,
 			case DRM_FORMAT_NV12:
 			case DRM_FORMAT_NV21:
 			/* Configure the auxiliary frame buffer address 0 */
-			paddr1 = (u32)drm_fb_cma_get_gem_addr(fb, newstate, 1);
+			paddr1 = (u32)drm_fb_dma_get_gem_addr(fb, newstate, 1);
 
 			if (newstate->rotation & DRM_MODE_REFLECT_X)
 				paddr1 += ((fb->format->cpp[1] * (x1 - x0 + 1)) >> 1) - 1;
@@ -1393,8 +1392,8 @@ static void ltdc_plane_atomic_update(struct drm_plane *plane,
 			break;
 			case DRM_FORMAT_YUV420:
 			/* Configure the auxiliary frame buffer address 0 & 1 */
-			paddr1 = (u32)drm_fb_cma_get_gem_addr(fb, newstate, 1);
-			paddr2 = (u32)drm_fb_cma_get_gem_addr(fb, newstate, 2);
+			paddr1 = (u32)drm_fb_dma_get_gem_addr(fb, newstate, 1);
+			paddr2 = (u32)drm_fb_dma_get_gem_addr(fb, newstate, 2);
 
 			if (newstate->rotation & DRM_MODE_REFLECT_X) {
 				paddr1 += ((fb->format->cpp[1] * (x1 - x0 + 1)) >> 1) - 1;
@@ -1411,8 +1410,8 @@ static void ltdc_plane_atomic_update(struct drm_plane *plane,
 			break;
 			case DRM_FORMAT_YVU420:
 			/* Configure the auxiliary frame buffer address 0 & 1 */
-			paddr1 = (u32)drm_fb_cma_get_gem_addr(fb, newstate, 2);
-			paddr2 = (u32)drm_fb_cma_get_gem_addr(fb, newstate, 1);
+			paddr1 = (u32)drm_fb_dma_get_gem_addr(fb, newstate, 2);
+			paddr2 = (u32)drm_fb_dma_get_gem_addr(fb, newstate, 1);
 
 			if (newstate->rotation & DRM_MODE_REFLECT_X) {
 				paddr1 += ((fb->format->cpp[1] * (x1 - x0 + 1)) >> 1) - 1;
diff --git a/drivers/gpu/drm/sun4i/Kconfig b/drivers/gpu/drm/sun4i/Kconfig
index 3a43c436c74a..4741d9f6544c 100644
--- a/drivers/gpu/drm/sun4i/Kconfig
+++ b/drivers/gpu/drm/sun4i/Kconfig
@@ -3,7 +3,7 @@ config DRM_SUN4I
 	tristate "DRM Support for Allwinner A10 Display Engine"
 	depends on DRM && COMMON_CLK
 	depends on ARCH_SUNXI || COMPILE_TEST
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_KMS_HELPER
 	select DRM_PANEL
 	select REGMAP_MMIO
@@ -16,23 +16,25 @@ config DRM_SUN4I
 if DRM_SUN4I
 
 config DRM_SUN4I_HDMI
-	tristate "Allwinner A10 HDMI Controller Support"
+	tristate "Allwinner A10/A10s/A20/A31 HDMI Controller Support"
+	depends on ARM || COMPILE_TEST
 	default DRM_SUN4I
 	help
-	  Choose this option if you have an Allwinner SoC with an HDMI
-	  controller.
+	  Choose this option if you have an Allwinner A10/A10s/A20/A31
+	  SoC with an HDMI controller.
 
 config DRM_SUN4I_HDMI_CEC
-	bool "Allwinner A10 HDMI CEC Support"
+	bool "Allwinner A10/A10s/A20/A31 HDMI CEC Support"
 	depends on DRM_SUN4I_HDMI
 	select CEC_CORE
 	select CEC_PIN
 	help
-	  Choose this option if you have an Allwinner SoC with an HDMI
-	  controller and want to use CEC.
+	  Choose this option if you have an Allwinner A10/A10s/A20/A31
+	  SoC with an HDMI controller and want to use CEC.
 
 config DRM_SUN4I_BACKEND
 	tristate "Support for Allwinner A10 Display Engine Backend"
+	depends on ARM || COMPILE_TEST
 	default DRM_SUN4I
 	help
 	  Choose this option if you have an Allwinner SoC with the
@@ -41,8 +43,8 @@ config DRM_SUN4I_BACKEND
 	  selected the module will be called sun4i-backend.
 
 config DRM_SUN6I_DSI
-	tristate "Allwinner A31 MIPI-DSI Controller Support"
-	default MACH_SUN8I
+	tristate "Allwinner A31/A64 MIPI-DSI Controller Support"
+	default DRM_SUN4I
 	select CRC_CCITT
 	select DRM_MIPI_DSI
 	select RESET_CONTROLLER
@@ -55,15 +57,17 @@ config DRM_SUN6I_DSI
 config DRM_SUN8I_DW_HDMI
 	tristate "Support for Allwinner version of DesignWare HDMI"
 	depends on DRM_SUN4I
+	default DRM_SUN4I
 	select DRM_DW_HDMI
 	help
 	  Choose this option if you have an Allwinner SoC with the
-	  DesignWare HDMI controller with custom HDMI PHY. If M is
+	  DesignWare HDMI controller. SoCs that support HDMI and
+	  have a Display Engine 2.0 contain this controller. If M is
 	  selected the module will be called sun8i_dw_hdmi.
 
 config DRM_SUN8I_MIXER
 	tristate "Support for Allwinner Display Engine 2.0 Mixer"
-	default MACH_SUN8I
+	default DRM_SUN4I
 	help
 	  Choose this option if you have an Allwinner SoC with the
 	  Allwinner Display Engine 2.0, which has a mixer to do some
@@ -75,6 +79,6 @@ config DRM_SUN8I_TCON_TOP
 	default DRM_SUN4I if DRM_SUN8I_MIXER!=n
 	help
 	  TCON TOP is responsible for configuring display pipeline for
-	  HTMI, TVE and LCD.
+	  HDMI, TVE and LCD.
 
 endif
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index 287e8c4bbaea..38070fc261f3 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -19,11 +19,10 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_crtc.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_probe_helper.h>
 
 #include "sun4i_backend.h"
@@ -330,7 +329,7 @@ int sun4i_backend_update_layer_buffer(struct sun4i_backend *backend,
 	struct drm_plane_state *state = plane->state;
 	struct drm_framebuffer *fb = state->fb;
 	u32 lo_paddr, hi_paddr;
-	dma_addr_t paddr;
+	dma_addr_t dma_addr;
 
 	/* Set the line width */
 	DRM_DEBUG_DRIVER("Layer line width: %d bits\n", fb->pitches[0] * 8);
@@ -339,21 +338,21 @@ int sun4i_backend_update_layer_buffer(struct sun4i_backend *backend,
 		     fb->pitches[0] * 8);
 
 	/* Get the start of the displayed memory */
-	paddr = drm_fb_cma_get_gem_addr(fb, state, 0);
-	DRM_DEBUG_DRIVER("Setting buffer address to %pad\n", &paddr);
+	dma_addr = drm_fb_dma_get_gem_addr(fb, state, 0);
+	DRM_DEBUG_DRIVER("Setting buffer address to %pad\n", &dma_addr);
 
 	if (fb->format->is_yuv)
-		return sun4i_backend_update_yuv_buffer(backend, fb, paddr);
+		return sun4i_backend_update_yuv_buffer(backend, fb, dma_addr);
 
 	/* Write the 32 lower bits of the address (in bits) */
-	lo_paddr = paddr << 3;
+	lo_paddr = dma_addr << 3;
 	DRM_DEBUG_DRIVER("Setting address lower bits to 0x%x\n", lo_paddr);
 	regmap_write(backend->engine.regs,
 		     SUN4I_BACKEND_LAYFB_L32ADD_REG(layer),
 		     lo_paddr);
 
 	/* And the upper bits */
-	hi_paddr = paddr >> 29;
+	hi_paddr = dma_addr >> 29;
 	DRM_DEBUG_DRIVER("Setting address high bits to 0x%x\n", hi_paddr);
 	regmap_update_bits(backend->engine.regs, SUN4I_BACKEND_LAYFB_H4ADD_REG,
 			   SUN4I_BACKEND_LAYFB_H4ADD_MSK(layer),
diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c
index 6eb1aabdb161..d06ffd99d86e 100644
--- a/drivers/gpu/drm/sun4i/sun4i_drv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_drv.c
@@ -17,9 +17,8 @@
 #include <drm/drm_aperture.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_of.h>
 #include <drm/drm_probe_helper.h>
@@ -38,10 +37,10 @@ static int drm_sun4i_gem_dumb_create(struct drm_file *file_priv,
 	/* The hardware only allows even pitches for YUV buffers. */
 	args->pitch = ALIGN(DIV_ROUND_UP(args->width * args->bpp, 8), 2);
 
-	return drm_gem_cma_dumb_create_internal(file_priv, drm, args);
+	return drm_gem_dma_dumb_create_internal(file_priv, drm, args);
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(sun4i_drv_fops);
+DEFINE_DRM_GEM_DMA_FOPS(sun4i_drv_fops);
 
 static const struct drm_driver sun4i_drv_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
@@ -55,7 +54,7 @@ static const struct drm_driver sun4i_drv_driver = {
 	.minor			= 0,
 
 	/* GEM Operations */
-	DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(drm_sun4i_gem_dumb_create),
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(drm_sun4i_gem_dumb_create),
 };
 
 static int sun4i_drv_bind(struct device *dev)
diff --git a/drivers/gpu/drm/sun4i/sun4i_frontend.c b/drivers/gpu/drm/sun4i/sun4i_frontend.c
index 462fae73eae9..799ab7460ae5 100644
--- a/drivers/gpu/drm/sun4i/sun4i_frontend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_frontend.c
@@ -14,10 +14,10 @@
 #include <linux/reset.h>
 
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_plane.h>
 
 #include "sun4i_drv.h"
@@ -160,7 +160,7 @@ void sun4i_frontend_update_buffer(struct sun4i_frontend *frontend,
 	struct drm_framebuffer *fb = state->fb;
 	unsigned int strides[3] = {};
 
-	dma_addr_t paddr;
+	dma_addr_t dma_addr;
 	bool swap;
 
 	if (fb->modifier == DRM_FORMAT_MOD_ALLWINNER_TILED) {
@@ -221,22 +221,24 @@ void sun4i_frontend_update_buffer(struct sun4i_frontend *frontend,
 	swap = sun4i_frontend_format_chroma_requires_swap(fb->format->format);
 
 	/* Set the physical address of the buffer in memory */
-	paddr = drm_fb_cma_get_gem_addr(fb, state, 0);
-	DRM_DEBUG_DRIVER("Setting buffer #0 address to %pad\n", &paddr);
-	regmap_write(frontend->regs, SUN4I_FRONTEND_BUF_ADDR0_REG, paddr);
+	dma_addr = drm_fb_dma_get_gem_addr(fb, state, 0);
+	DRM_DEBUG_DRIVER("Setting buffer #0 address to %pad\n", &dma_addr);
+	regmap_write(frontend->regs, SUN4I_FRONTEND_BUF_ADDR0_REG, dma_addr);
 
 	if (fb->format->num_planes > 1) {
-		paddr = drm_fb_cma_get_gem_addr(fb, state, swap ? 2 : 1);
-		DRM_DEBUG_DRIVER("Setting buffer #1 address to %pad\n", &paddr);
+		dma_addr = drm_fb_dma_get_gem_addr(fb, state, swap ? 2 : 1);
+		DRM_DEBUG_DRIVER("Setting buffer #1 address to %pad\n",
+				 &dma_addr);
 		regmap_write(frontend->regs, SUN4I_FRONTEND_BUF_ADDR1_REG,
-			     paddr);
+			     dma_addr);
 	}
 
 	if (fb->format->num_planes > 2) {
-		paddr = drm_fb_cma_get_gem_addr(fb, state, swap ? 1 : 2);
-		DRM_DEBUG_DRIVER("Setting buffer #2 address to %pad\n", &paddr);
+		dma_addr = drm_fb_dma_get_gem_addr(fb, state, swap ? 1 : 2);
+		DRM_DEBUG_DRIVER("Setting buffer #2 address to %pad\n",
+				 &dma_addr);
 		regmap_write(frontend->regs, SUN4I_FRONTEND_BUF_ADDR2_REG,
-			     paddr);
+			     dma_addr);
 	}
 }
 EXPORT_SYMBOL(sun4i_frontend_update_buffer);
diff --git a/drivers/gpu/drm/sun4i/sun4i_layer.c b/drivers/gpu/drm/sun4i/sun4i_layer.c
index 648dd0b5b116..98f3176366c0 100644
--- a/drivers/gpu/drm/sun4i/sun4i_layer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_layer.c
@@ -10,7 +10,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 
 #include "sun4i_backend.h"
 #include "sun4i_frontend.h"
diff --git a/drivers/gpu/drm/sun4i/sun4i_tv.c b/drivers/gpu/drm/sun4i/sun4i_tv.c
index 94883abe0dfd..c65f0a89b6b0 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tv.c
@@ -14,6 +14,7 @@
 #include <linux/regmap.h>
 #include <linux/reset.h>
 
+#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
@@ -275,13 +276,6 @@ drm_encoder_to_sun4i_tv(struct drm_encoder *encoder)
 			    encoder);
 }
 
-static inline struct sun4i_tv *
-drm_connector_to_sun4i_tv(struct drm_connector *connector)
-{
-	return container_of(connector, struct sun4i_tv,
-			    connector);
-}
-
 /*
  * FIXME: If only the drm_display_mode private field was usable, this
  * could go away...
@@ -339,7 +333,8 @@ static void sun4i_tv_mode_to_drm_mode(const struct tv_mode *tv_mode,
 	mode->vtotal = mode->vsync_end  + tv_mode->vback_porch;
 }
 
-static void sun4i_tv_disable(struct drm_encoder *encoder)
+static void sun4i_tv_disable(struct drm_encoder *encoder,
+			    struct drm_atomic_state *state)
 {
 	struct sun4i_tv *tv = drm_encoder_to_sun4i_tv(encoder);
 	struct sun4i_crtc *crtc = drm_crtc_to_sun4i_crtc(encoder->crtc);
@@ -353,27 +348,18 @@ static void sun4i_tv_disable(struct drm_encoder *encoder)
 	sunxi_engine_disable_color_correction(crtc->engine);
 }
 
-static void sun4i_tv_enable(struct drm_encoder *encoder)
+static void sun4i_tv_enable(struct drm_encoder *encoder,
+			    struct drm_atomic_state *state)
 {
 	struct sun4i_tv *tv = drm_encoder_to_sun4i_tv(encoder);
 	struct sun4i_crtc *crtc = drm_crtc_to_sun4i_crtc(encoder->crtc);
+	struct drm_crtc_state *crtc_state =
+		drm_atomic_get_new_crtc_state(state, encoder->crtc);
+	struct drm_display_mode *mode = &crtc_state->mode;
+	const struct tv_mode *tv_mode = sun4i_tv_find_tv_by_mode(mode);
 
 	DRM_DEBUG_DRIVER("Enabling the TV Output\n");
 
-	sunxi_engine_apply_color_correction(crtc->engine);
-
-	regmap_update_bits(tv->regs, SUN4I_TVE_EN_REG,
-			   SUN4I_TVE_EN_ENABLE,
-			   SUN4I_TVE_EN_ENABLE);
-}
-
-static void sun4i_tv_mode_set(struct drm_encoder *encoder,
-			      struct drm_display_mode *mode,
-			      struct drm_display_mode *adjusted_mode)
-{
-	struct sun4i_tv *tv = drm_encoder_to_sun4i_tv(encoder);
-	const struct tv_mode *tv_mode = sun4i_tv_find_tv_by_mode(mode);
-
 	/* Enable and map the DAC to the output */
 	regmap_update_bits(tv->regs, SUN4I_TVE_EN_REG,
 			   SUN4I_TVE_EN_DAC_MAP_MASK,
@@ -466,12 +452,17 @@ static void sun4i_tv_mode_set(struct drm_encoder *encoder,
 		      SUN4I_TVE_RESYNC_FIELD : 0));
 
 	regmap_write(tv->regs, SUN4I_TVE_SLAVE_REG, 0);
+
+	sunxi_engine_apply_color_correction(crtc->engine);
+
+	regmap_update_bits(tv->regs, SUN4I_TVE_EN_REG,
+			   SUN4I_TVE_EN_ENABLE,
+			   SUN4I_TVE_EN_ENABLE);
 }
 
 static const struct drm_encoder_helper_funcs sun4i_tv_helper_funcs = {
-	.disable	= sun4i_tv_disable,
-	.enable		= sun4i_tv_enable,
-	.mode_set	= sun4i_tv_mode_set,
+	.atomic_disable	= sun4i_tv_disable,
+	.atomic_enable	= sun4i_tv_enable,
 };
 
 static int sun4i_tv_comp_get_modes(struct drm_connector *connector)
@@ -497,27 +488,13 @@ static int sun4i_tv_comp_get_modes(struct drm_connector *connector)
 	return i;
 }
 
-static int sun4i_tv_comp_mode_valid(struct drm_connector *connector,
-				    struct drm_display_mode *mode)
-{
-	/* TODO */
-	return MODE_OK;
-}
-
 static const struct drm_connector_helper_funcs sun4i_tv_comp_connector_helper_funcs = {
 	.get_modes	= sun4i_tv_comp_get_modes,
-	.mode_valid	= sun4i_tv_comp_mode_valid,
 };
 
-static void
-sun4i_tv_comp_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_cleanup(connector);
-}
-
 static const struct drm_connector_funcs sun4i_tv_comp_connector_funcs = {
 	.fill_modes		= drm_helper_probe_single_connector_modes,
-	.destroy		= sun4i_tv_comp_connector_destroy,
+	.destroy		= drm_connector_cleanup,
 	.reset			= drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state	= drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
@@ -604,7 +581,7 @@ static int sun4i_tv_bind(struct device *dev, struct device *master,
 	if (ret) {
 		dev_err(dev,
 			"Couldn't initialise the Composite connector\n");
-		goto err_cleanup_connector;
+		goto err_cleanup_encoder;
 	}
 	tv->connector.interlace_allowed = true;
 
@@ -612,7 +589,7 @@ static int sun4i_tv_bind(struct device *dev, struct device *master,
 
 	return 0;
 
-err_cleanup_connector:
+err_cleanup_encoder:
 	drm_encoder_cleanup(&tv->encoder);
 err_disable_clk:
 	clk_disable_unprepare(tv->clk);
@@ -629,6 +606,7 @@ static void sun4i_tv_unbind(struct device *dev, struct device *master,
 	drm_connector_cleanup(&tv->connector);
 	drm_encoder_cleanup(&tv->encoder);
 	clk_disable_unprepare(tv->clk);
+	reset_control_assert(tv->reset);
 }
 
 static const struct component_ops sun4i_tv_ops = {
diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.c b/drivers/gpu/drm/sun4i/sun8i_mixer.c
index 648b38a73066..bafee05f6b24 100644
--- a/drivers/gpu/drm/sun4i/sun8i_mixer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_mixer.c
@@ -16,10 +16,8 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_probe_helper.h>
 
 #include "sun4i_drv.h"
diff --git a/drivers/gpu/drm/sun4i/sun8i_ui_layer.c b/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
index 36da962de394..ca75ca0835a6 100644
--- a/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
@@ -13,12 +13,11 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_crtc.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_probe_helper.h>
 
 #include "sun8i_mixer.h"
@@ -193,25 +192,25 @@ static int sun8i_ui_layer_update_buffer(struct sun8i_mixer *mixer, int channel,
 {
 	struct drm_plane_state *state = plane->state;
 	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_cma_object *gem;
-	dma_addr_t paddr;
+	struct drm_gem_dma_object *gem;
+	dma_addr_t dma_addr;
 	u32 ch_base;
 	int bpp;
 
 	ch_base = sun8i_channel_base(mixer, channel);
 
 	/* Get the physical address of the buffer in memory */
-	gem = drm_fb_cma_get_gem_obj(fb, 0);
+	gem = drm_fb_dma_get_gem_obj(fb, 0);
 
-	DRM_DEBUG_DRIVER("Using GEM @ %pad\n", &gem->paddr);
+	DRM_DEBUG_DRIVER("Using GEM @ %pad\n", &gem->dma_addr);
 
 	/* Compute the start of the displayed memory */
 	bpp = fb->format->cpp[0];
-	paddr = gem->paddr + fb->offsets[0];
+	dma_addr = gem->dma_addr + fb->offsets[0];
 
 	/* Fixup framebuffer address for src coordinates */
-	paddr += (state->src.x1 >> 16) * bpp;
-	paddr += (state->src.y1 >> 16) * fb->pitches[0];
+	dma_addr += (state->src.x1 >> 16) * bpp;
+	dma_addr += (state->src.y1 >> 16) * fb->pitches[0];
 
 	/* Set the line width */
 	DRM_DEBUG_DRIVER("Layer line width: %d bytes\n", fb->pitches[0]);
@@ -219,11 +218,11 @@ static int sun8i_ui_layer_update_buffer(struct sun8i_mixer *mixer, int channel,
 		     SUN8I_MIXER_CHAN_UI_LAYER_PITCH(ch_base, overlay),
 		     fb->pitches[0]);
 
-	DRM_DEBUG_DRIVER("Setting buffer address to %pad\n", &paddr);
+	DRM_DEBUG_DRIVER("Setting buffer address to %pad\n", &dma_addr);
 
 	regmap_write(mixer->engine.regs,
 		     SUN8I_MIXER_CHAN_UI_LAYER_TOP_LADDR(ch_base, overlay),
-		     lower_32_bits(paddr));
+		     lower_32_bits(dma_addr));
 
 	return 0;
 }
@@ -246,8 +245,8 @@ static int sun8i_ui_layer_atomic_check(struct drm_plane *plane,
 	if (WARN_ON(!crtc_state))
 		return -EINVAL;
 
-	min_scale = DRM_PLANE_HELPER_NO_SCALING;
-	max_scale = DRM_PLANE_HELPER_NO_SCALING;
+	min_scale = DRM_PLANE_NO_SCALING;
+	max_scale = DRM_PLANE_NO_SCALING;
 
 	if (layer->mixer->cfg->scaler_mask & BIT(layer->channel)) {
 		min_scale = SUN8I_UI_SCALER_SCALE_MIN;
diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
index 1fee6499bdd3..f9c0a56d3a14 100644
--- a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
@@ -7,11 +7,10 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_blend.h>
 #include <drm/drm_crtc.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_probe_helper.h>
 
 #include "sun8i_csc.h"
@@ -309,9 +308,9 @@ static int sun8i_vi_layer_update_buffer(struct sun8i_mixer *mixer, int channel,
 	struct drm_plane_state *state = plane->state;
 	struct drm_framebuffer *fb = state->fb;
 	const struct drm_format_info *format = fb->format;
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 	u32 dx, dy, src_x, src_y;
-	dma_addr_t paddr;
+	dma_addr_t dma_addr;
 	u32 ch_base;
 	int i;
 
@@ -323,12 +322,12 @@ static int sun8i_vi_layer_update_buffer(struct sun8i_mixer *mixer, int channel,
 
 	for (i = 0; i < format->num_planes; i++) {
 		/* Get the physical address of the buffer in memory */
-		gem = drm_fb_cma_get_gem_obj(fb, i);
+		gem = drm_fb_dma_get_gem_obj(fb, i);
 
-		DRM_DEBUG_DRIVER("Using GEM @ %pad\n", &gem->paddr);
+		DRM_DEBUG_DRIVER("Using GEM @ %pad\n", &gem->dma_addr);
 
 		/* Compute the start of the displayed memory */
-		paddr = gem->paddr + fb->offsets[i];
+		dma_addr = gem->dma_addr + fb->offsets[i];
 
 		dx = src_x;
 		dy = src_y;
@@ -339,8 +338,8 @@ static int sun8i_vi_layer_update_buffer(struct sun8i_mixer *mixer, int channel,
 		}
 
 		/* Fixup framebuffer address for src coordinates */
-		paddr += dx * format->cpp[i];
-		paddr += dy * fb->pitches[i];
+		dma_addr += dx * format->cpp[i];
+		dma_addr += dy * fb->pitches[i];
 
 		/* Set the line width */
 		DRM_DEBUG_DRIVER("Layer %d. line width: %d bytes\n",
@@ -351,12 +350,12 @@ static int sun8i_vi_layer_update_buffer(struct sun8i_mixer *mixer, int channel,
 			     fb->pitches[i]);
 
 		DRM_DEBUG_DRIVER("Setting %d. buffer address to %pad\n",
-				 i + 1, &paddr);
+				 i + 1, &dma_addr);
 
 		regmap_write(mixer->engine.regs,
 			     SUN8I_MIXER_CHAN_VI_LAYER_TOP_LADDR(ch_base,
 								 overlay, i),
-			     lower_32_bits(paddr));
+			     lower_32_bits(dma_addr));
 	}
 
 	return 0;
@@ -380,8 +379,8 @@ static int sun8i_vi_layer_atomic_check(struct drm_plane *plane,
 	if (WARN_ON(!crtc_state))
 		return -EINVAL;
 
-	min_scale = DRM_PLANE_HELPER_NO_SCALING;
-	max_scale = DRM_PLANE_HELPER_NO_SCALING;
+	min_scale = DRM_PLANE_NO_SCALING;
+	max_scale = DRM_PLANE_NO_SCALING;
 
 	if (layer->mixer->cfg->scaler_mask & BIT(layer->channel)) {
 		min_scale = SUN8I_VI_SCALER_SCALE_MIN;
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index 747abafb6a5c..bd0f60704467 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -26,7 +26,6 @@
 #include <drm/drm_debugfs.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "dc.h"
diff --git a/drivers/gpu/drm/tegra/fb.c b/drivers/gpu/drm/tegra/fb.c
index ed828de5ac01..9291209154a7 100644
--- a/drivers/gpu/drm/tegra/fb.c
+++ b/drivers/gpu/drm/tegra/fb.c
@@ -3,7 +3,7 @@
  * Copyright (C) 2012-2013 Avionic Design GmbH
  * Copyright (C) 2012 NVIDIA CORPORATION.  All rights reserved.
  *
- * Based on the KMS/FB CMA helpers
+ * Based on the KMS/FB DMA helpers
  *   Copyright (C) 2012 Analog Devices Inc.
  */
 
diff --git a/drivers/gpu/drm/tegra/plane.c b/drivers/gpu/drm/tegra/plane.c
index ca9f03e3675b..10090116895f 100644
--- a/drivers/gpu/drm/tegra/plane.c
+++ b/drivers/gpu/drm/tegra/plane.c
@@ -12,7 +12,6 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 
 #include "dc.h"
 #include "plane.h"
diff --git a/drivers/gpu/drm/tests/Makefile b/drivers/gpu/drm/tests/Makefile
index 2c8273796d9d..91b70f7d2769 100644
--- a/drivers/gpu/drm/tests/Makefile
+++ b/drivers/gpu/drm/tests/Makefile
@@ -1,3 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-$(CONFIG_DRM_KUNIT_TEST) += drm_format_helper_test.o
+obj-$(CONFIG_DRM_KUNIT_TEST) += drm_format_helper_test.o drm_damage_helper_test.o \
+	drm_cmdline_parser_test.o drm_rect_test.o drm_format_test.o drm_plane_helper_test.o \
+	drm_dp_mst_helper_test.o drm_framebuffer_test.o drm_buddy_test.o drm_mm_test.o
diff --git a/drivers/gpu/drm/tests/drm_buddy_test.c b/drivers/gpu/drm/tests/drm_buddy_test.c
new file mode 100644
index 000000000000..7a2b2d6bc3fe
--- /dev/null
+++ b/drivers/gpu/drm/tests/drm_buddy_test.c
@@ -0,0 +1,756 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2019 Intel Corporation
+ * Copyright © 2022 Maíra Canal <mairacanal@riseup.net>
+ */
+
+#include <kunit/test.h>
+
+#include <linux/prime_numbers.h>
+#include <linux/sched/signal.h>
+
+#include <drm/drm_buddy.h>
+
+#include "../lib/drm_random.h"
+
+#define TIMEOUT(name__)								\
+	unsigned long name__ = jiffies + MAX_SCHEDULE_TIMEOUT
+
+static unsigned int random_seed;
+
+static inline u64 get_size(int order, u64 chunk_size)
+{
+	return (1 << order) * chunk_size;
+}
+
+__printf(2, 3)
+static bool __timeout(unsigned long timeout, const char *fmt, ...)
+{
+	va_list va;
+
+	if (!signal_pending(current)) {
+		cond_resched();
+		if (time_before(jiffies, timeout))
+			return false;
+	}
+
+	if (fmt) {
+		va_start(va, fmt);
+		vprintk(fmt, va);
+		va_end(va);
+	}
+
+	return true;
+}
+
+static void __dump_block(struct kunit *test, struct drm_buddy *mm,
+			 struct drm_buddy_block *block, bool buddy)
+{
+	kunit_err(test, "block info: header=%llx, state=%u, order=%d, offset=%llx size=%llx root=%d buddy=%d\n",
+		  block->header, drm_buddy_block_state(block),
+			  drm_buddy_block_order(block), drm_buddy_block_offset(block),
+			  drm_buddy_block_size(mm, block), !block->parent, buddy);
+}
+
+static void dump_block(struct kunit *test, struct drm_buddy *mm,
+		       struct drm_buddy_block *block)
+{
+	struct drm_buddy_block *buddy;
+
+	__dump_block(test, mm, block, false);
+
+	buddy = drm_get_buddy(block);
+	if (buddy)
+		__dump_block(test, mm, buddy, true);
+}
+
+static int check_block(struct kunit *test, struct drm_buddy *mm,
+		       struct drm_buddy_block *block)
+{
+	struct drm_buddy_block *buddy;
+	unsigned int block_state;
+	u64 block_size;
+	u64 offset;
+	int err = 0;
+
+	block_state = drm_buddy_block_state(block);
+
+	if (block_state != DRM_BUDDY_ALLOCATED &&
+	    block_state != DRM_BUDDY_FREE && block_state != DRM_BUDDY_SPLIT) {
+		kunit_err(test, "block state mismatch\n");
+		err = -EINVAL;
+	}
+
+	block_size = drm_buddy_block_size(mm, block);
+	offset = drm_buddy_block_offset(block);
+
+	if (block_size < mm->chunk_size) {
+		kunit_err(test, "block size smaller than min size\n");
+		err = -EINVAL;
+	}
+
+	if (!is_power_of_2(block_size)) {
+		kunit_err(test, "block size not power of two\n");
+		err = -EINVAL;
+	}
+
+	if (!IS_ALIGNED(block_size, mm->chunk_size)) {
+		kunit_err(test, "block size not aligned to min size\n");
+		err = -EINVAL;
+	}
+
+	if (!IS_ALIGNED(offset, mm->chunk_size)) {
+		kunit_err(test, "block offset not aligned to min size\n");
+		err = -EINVAL;
+	}
+
+	if (!IS_ALIGNED(offset, block_size)) {
+		kunit_err(test, "block offset not aligned to block size\n");
+		err = -EINVAL;
+	}
+
+	buddy = drm_get_buddy(block);
+
+	if (!buddy && block->parent) {
+		kunit_err(test, "buddy has gone fishing\n");
+		err = -EINVAL;
+	}
+
+	if (buddy) {
+		if (drm_buddy_block_offset(buddy) != (offset ^ block_size)) {
+			kunit_err(test, "buddy has wrong offset\n");
+			err = -EINVAL;
+		}
+
+		if (drm_buddy_block_size(mm, buddy) != block_size) {
+			kunit_err(test, "buddy size mismatch\n");
+			err = -EINVAL;
+		}
+
+		if (drm_buddy_block_state(buddy) == block_state &&
+		    block_state == DRM_BUDDY_FREE) {
+			kunit_err(test, "block and its buddy are free\n");
+			err = -EINVAL;
+		}
+	}
+
+	return err;
+}
+
+static int check_blocks(struct kunit *test, struct drm_buddy *mm,
+			struct list_head *blocks, u64 expected_size, bool is_contiguous)
+{
+	struct drm_buddy_block *block;
+	struct drm_buddy_block *prev;
+	u64 total;
+	int err = 0;
+
+	block = NULL;
+	prev = NULL;
+	total = 0;
+
+	list_for_each_entry(block, blocks, link) {
+		err = check_block(test, mm, block);
+
+		if (!drm_buddy_block_is_allocated(block)) {
+			kunit_err(test, "block not allocated\n");
+			err = -EINVAL;
+		}
+
+		if (is_contiguous && prev) {
+			u64 prev_block_size;
+			u64 prev_offset;
+			u64 offset;
+
+			prev_offset = drm_buddy_block_offset(prev);
+			prev_block_size = drm_buddy_block_size(mm, prev);
+			offset = drm_buddy_block_offset(block);
+
+			if (offset != (prev_offset + prev_block_size)) {
+				kunit_err(test, "block offset mismatch\n");
+				err = -EINVAL;
+			}
+		}
+
+		if (err)
+			break;
+
+		total += drm_buddy_block_size(mm, block);
+		prev = block;
+	}
+
+	if (!err) {
+		if (total != expected_size) {
+			kunit_err(test, "size mismatch, expected=%llx, found=%llx\n",
+				  expected_size, total);
+			err = -EINVAL;
+		}
+		return err;
+	}
+
+	if (prev) {
+		kunit_err(test, "prev block, dump:\n");
+		dump_block(test, mm, prev);
+	}
+
+	kunit_err(test, "bad block, dump:\n");
+	dump_block(test, mm, block);
+
+	return err;
+}
+
+static int check_mm(struct kunit *test, struct drm_buddy *mm)
+{
+	struct drm_buddy_block *root;
+	struct drm_buddy_block *prev;
+	unsigned int i;
+	u64 total;
+	int err = 0;
+
+	if (!mm->n_roots) {
+		kunit_err(test, "n_roots is zero\n");
+		return -EINVAL;
+	}
+
+	if (mm->n_roots != hweight64(mm->size)) {
+		kunit_err(test, "n_roots mismatch, n_roots=%u, expected=%lu\n",
+			  mm->n_roots, hweight64(mm->size));
+		return -EINVAL;
+	}
+
+	root = NULL;
+	prev = NULL;
+	total = 0;
+
+	for (i = 0; i < mm->n_roots; ++i) {
+		struct drm_buddy_block *block;
+		unsigned int order;
+
+		root = mm->roots[i];
+		if (!root) {
+			kunit_err(test, "root(%u) is NULL\n", i);
+			err = -EINVAL;
+			break;
+		}
+
+		err = check_block(test, mm, root);
+
+		if (!drm_buddy_block_is_free(root)) {
+			kunit_err(test, "root not free\n");
+			err = -EINVAL;
+		}
+
+		order = drm_buddy_block_order(root);
+
+		if (!i) {
+			if (order != mm->max_order) {
+				kunit_err(test, "max order root missing\n");
+				err = -EINVAL;
+			}
+		}
+
+		if (prev) {
+			u64 prev_block_size;
+			u64 prev_offset;
+			u64 offset;
+
+			prev_offset = drm_buddy_block_offset(prev);
+			prev_block_size = drm_buddy_block_size(mm, prev);
+			offset = drm_buddy_block_offset(root);
+
+			if (offset != (prev_offset + prev_block_size)) {
+				kunit_err(test, "root offset mismatch\n");
+				err = -EINVAL;
+			}
+		}
+
+		block = list_first_entry_or_null(&mm->free_list[order],
+						 struct drm_buddy_block, link);
+		if (block != root) {
+			kunit_err(test, "root mismatch at order=%u\n", order);
+			err = -EINVAL;
+		}
+
+		if (err)
+			break;
+
+		prev = root;
+		total += drm_buddy_block_size(mm, root);
+	}
+
+	if (!err) {
+		if (total != mm->size) {
+			kunit_err(test, "expected mm size=%llx, found=%llx\n",
+				  mm->size, total);
+			err = -EINVAL;
+		}
+		return err;
+	}
+
+	if (prev) {
+		kunit_err(test, "prev root(%u), dump:\n", i - 1);
+		dump_block(test, mm, prev);
+	}
+
+	if (root) {
+		kunit_err(test, "bad root(%u), dump:\n", i);
+		dump_block(test, mm, root);
+	}
+
+	return err;
+}
+
+static void mm_config(u64 *size, u64 *chunk_size)
+{
+	DRM_RND_STATE(prng, random_seed);
+	u32 s, ms;
+
+	/* Nothing fancy, just try to get an interesting bit pattern */
+
+	prandom_seed_state(&prng, random_seed);
+
+	/* Let size be a random number of pages up to 8 GB (2M pages) */
+	s = 1 + drm_prandom_u32_max_state((BIT(33 - 12)) - 1, &prng);
+	/* Let the chunk size be a random power of 2 less than size */
+	ms = BIT(drm_prandom_u32_max_state(ilog2(s), &prng));
+	/* Round size down to the chunk size */
+	s &= -ms;
+
+	/* Convert from pages to bytes */
+	*chunk_size = (u64)ms << 12;
+	*size = (u64)s << 12;
+}
+
+static void drm_test_buddy_alloc_pathological(struct kunit *test)
+{
+	u64 mm_size, size, start = 0;
+	struct drm_buddy_block *block;
+	const int max_order = 3;
+	unsigned long flags = 0;
+	int order, top;
+	struct drm_buddy mm;
+	LIST_HEAD(blocks);
+	LIST_HEAD(holes);
+	LIST_HEAD(tmp);
+
+	/*
+	 * Create a pot-sized mm, then allocate one of each possible
+	 * order within. This should leave the mm with exactly one
+	 * page left. Free the largest block, then whittle down again.
+	 * Eventually we will have a fully 50% fragmented mm.
+	 */
+
+	mm_size = PAGE_SIZE << max_order;
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, PAGE_SIZE),
+			       "buddy_init failed\n");
+
+	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+
+	for (top = max_order; top; top--) {
+		/* Make room by freeing the largest allocated block */
+		block = list_first_entry_or_null(&blocks, typeof(*block), link);
+		if (block) {
+			list_del(&block->link);
+			drm_buddy_free_block(&mm, block);
+		}
+
+		for (order = top; order--;) {
+			size = get_size(order, PAGE_SIZE);
+			KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start,
+									    mm_size, size, size,
+										&tmp, flags),
+					"buddy_alloc hit -ENOMEM with order=%d, top=%d\n",
+					order, top);
+
+			block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+			KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+			list_move_tail(&block->link, &blocks);
+		}
+
+		/* There should be one final page for this sub-allocation */
+		size = get_size(0, PAGE_SIZE);
+		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+								    size, size, &tmp, flags),
+							   "buddy_alloc hit -ENOMEM for hole\n");
+
+		block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+		list_move_tail(&block->link, &holes);
+
+		size = get_size(top, PAGE_SIZE);
+		KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+								   size, size, &tmp, flags),
+							  "buddy_alloc unexpectedly succeeded at top-order %d/%d, it should be full!",
+							  top, max_order);
+	}
+
+	drm_buddy_free_list(&mm, &holes);
+
+	/* Nothing larger than blocks of chunk_size now available */
+	for (order = 1; order <= max_order; order++) {
+		size = get_size(order, PAGE_SIZE);
+		KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+								   size, size, &tmp, flags),
+							  "buddy_alloc unexpectedly succeeded at order %d, it should be full!",
+							  order);
+	}
+
+	list_splice_tail(&holes, &blocks);
+	drm_buddy_free_list(&mm, &blocks);
+	drm_buddy_fini(&mm);
+}
+
+static void drm_test_buddy_alloc_smoke(struct kunit *test)
+{
+	u64 mm_size, chunk_size, start = 0;
+	unsigned long flags = 0;
+	struct drm_buddy mm;
+	int *order;
+	int i;
+
+	DRM_RND_STATE(prng, random_seed);
+	TIMEOUT(end_time);
+
+	mm_config(&mm_size, &chunk_size);
+
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, chunk_size),
+			       "buddy_init failed\n");
+
+	order = drm_random_order(mm.max_order + 1, &prng);
+	KUNIT_ASSERT_TRUE(test, order);
+
+	for (i = 0; i <= mm.max_order; ++i) {
+		struct drm_buddy_block *block;
+		int max_order = order[i];
+		bool timeout = false;
+		LIST_HEAD(blocks);
+		u64 total, size;
+		LIST_HEAD(tmp);
+		int order, err;
+
+		KUNIT_ASSERT_FALSE_MSG(test, check_mm(test, &mm),
+				       "pre-mm check failed, abort\n");
+
+		order = max_order;
+		total = 0;
+
+		do {
+retry:
+			size = get_size(order, chunk_size);
+			err = drm_buddy_alloc_blocks(&mm, start, mm_size, size, size, &tmp, flags);
+			if (err) {
+				if (err == -ENOMEM) {
+					KUNIT_FAIL(test, "buddy_alloc hit -ENOMEM with order=%d\n",
+						   order);
+				} else {
+					if (order--) {
+						err = 0;
+						goto retry;
+					}
+
+					KUNIT_FAIL(test, "buddy_alloc with order=%d failed\n",
+						   order);
+				}
+
+				break;
+			}
+
+			block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+			KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+			list_move_tail(&block->link, &blocks);
+			KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_order(block), order,
+					    "buddy_alloc order mismatch\n");
+
+			total += drm_buddy_block_size(&mm, block);
+
+			if (__timeout(end_time, NULL)) {
+				timeout = true;
+				break;
+			}
+		} while (total < mm.size);
+
+		if (!err)
+			err = check_blocks(test, &mm, &blocks, total, false);
+
+		drm_buddy_free_list(&mm, &blocks);
+
+		if (!err) {
+			KUNIT_EXPECT_FALSE_MSG(test, check_mm(test, &mm),
+					       "post-mm check failed\n");
+		}
+
+		if (err || timeout)
+			break;
+
+		cond_resched();
+	}
+
+	kfree(order);
+	drm_buddy_fini(&mm);
+}
+
+static void drm_test_buddy_alloc_pessimistic(struct kunit *test)
+{
+	u64 mm_size, size, start = 0;
+	struct drm_buddy_block *block, *bn;
+	const unsigned int max_order = 16;
+	unsigned long flags = 0;
+	struct drm_buddy mm;
+	unsigned int order;
+	LIST_HEAD(blocks);
+	LIST_HEAD(tmp);
+
+	/*
+	 * Create a pot-sized mm, then allocate one of each possible
+	 * order within. This should leave the mm with exactly one
+	 * page left.
+	 */
+
+	mm_size = PAGE_SIZE << max_order;
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, PAGE_SIZE),
+			       "buddy_init failed\n");
+
+	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+
+	for (order = 0; order < max_order; order++) {
+		size = get_size(order, PAGE_SIZE);
+		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+								    size, size, &tmp, flags),
+							   "buddy_alloc hit -ENOMEM with order=%d\n",
+							   order);
+
+		block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+		list_move_tail(&block->link, &blocks);
+	}
+
+	/* And now the last remaining block available */
+	size = get_size(0, PAGE_SIZE);
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+							    size, size, &tmp, flags),
+						   "buddy_alloc hit -ENOMEM on final alloc\n");
+
+	block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+	KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+	list_move_tail(&block->link, &blocks);
+
+	/* Should be completely full! */
+	for (order = max_order; order--;) {
+		size = get_size(order, PAGE_SIZE);
+		KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+								   size, size, &tmp, flags),
+							  "buddy_alloc unexpectedly succeeded, it should be full!");
+	}
+
+	block = list_last_entry(&blocks, typeof(*block), link);
+	list_del(&block->link);
+	drm_buddy_free_block(&mm, block);
+
+	/* As we free in increasing size, we make available larger blocks */
+	order = 1;
+	list_for_each_entry_safe(block, bn, &blocks, link) {
+		list_del(&block->link);
+		drm_buddy_free_block(&mm, block);
+
+		size = get_size(order, PAGE_SIZE);
+		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+								    size, size, &tmp, flags),
+							   "buddy_alloc hit -ENOMEM with order=%d\n",
+							   order);
+
+		block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+		list_del(&block->link);
+		drm_buddy_free_block(&mm, block);
+		order++;
+	}
+
+	/* To confirm, now the whole mm should be available */
+	size = get_size(max_order, PAGE_SIZE);
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+							    size, size, &tmp, flags),
+						   "buddy_alloc (realloc) hit -ENOMEM with order=%d\n",
+						   max_order);
+
+	block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+	KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+	list_del(&block->link);
+	drm_buddy_free_block(&mm, block);
+	drm_buddy_free_list(&mm, &blocks);
+	drm_buddy_fini(&mm);
+}
+
+static void drm_test_buddy_alloc_optimistic(struct kunit *test)
+{
+	u64 mm_size, size, start = 0;
+	struct drm_buddy_block *block;
+	unsigned long flags = 0;
+	const int max_order = 16;
+	struct drm_buddy mm;
+	LIST_HEAD(blocks);
+	LIST_HEAD(tmp);
+	int order;
+
+	/*
+	 * Create a mm with one block of each order available, and
+	 * try to allocate them all.
+	 */
+
+	mm_size = PAGE_SIZE * ((1 << (max_order + 1)) - 1);
+
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, mm_size, PAGE_SIZE),
+			       "buddy_init failed\n");
+
+	KUNIT_EXPECT_EQ(test, mm.max_order, max_order);
+
+	for (order = 0; order <= max_order; order++) {
+		size = get_size(order, PAGE_SIZE);
+		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+								    size, size, &tmp, flags),
+							   "buddy_alloc hit -ENOMEM with order=%d\n",
+							   order);
+
+		block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_blocks has no blocks\n");
+
+		list_move_tail(&block->link, &blocks);
+	}
+
+	/* Should be completely full! */
+	size = get_size(0, PAGE_SIZE);
+	KUNIT_ASSERT_TRUE_MSG(test, drm_buddy_alloc_blocks(&mm, start, mm_size,
+							   size, size, &tmp, flags),
+						  "buddy_alloc unexpectedly succeeded, it should be full!");
+
+	drm_buddy_free_list(&mm, &blocks);
+	drm_buddy_fini(&mm);
+}
+
+static void drm_test_buddy_alloc_range(struct kunit *test)
+{
+	unsigned long flags = DRM_BUDDY_RANGE_ALLOCATION;
+	u64 offset, size, rem, chunk_size, end;
+	unsigned long page_num;
+	struct drm_buddy mm;
+	LIST_HEAD(blocks);
+
+	mm_config(&size, &chunk_size);
+
+	KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_init(&mm, size, chunk_size),
+			       "buddy_init failed");
+
+	KUNIT_ASSERT_FALSE_MSG(test, check_mm(test, &mm),
+			       "pre-mm check failed, abort!");
+
+	rem = mm.size;
+	offset = 0;
+
+	for_each_prime_number_from(page_num, 1, ULONG_MAX - 1) {
+		struct drm_buddy_block *block;
+		LIST_HEAD(tmp);
+
+		size = min(page_num * mm.chunk_size, rem);
+		end = offset + size;
+
+		KUNIT_ASSERT_FALSE_MSG(test, drm_buddy_alloc_blocks(&mm, offset, end,
+								    size, mm.chunk_size,
+									&tmp, flags),
+				"alloc_range with offset=%llx, size=%llx failed\n", offset, size);
+
+		block = list_first_entry_or_null(&tmp, struct drm_buddy_block, link);
+		KUNIT_ASSERT_TRUE_MSG(test, block, "alloc_range has no blocks\n");
+
+		KUNIT_ASSERT_EQ_MSG(test, drm_buddy_block_offset(block), offset,
+				    "alloc_range start offset mismatch, found=%llx, expected=%llx\n",
+							drm_buddy_block_offset(block), offset);
+
+		KUNIT_ASSERT_FALSE(test, check_blocks(test, &mm, &tmp, size, true));
+
+		list_splice_tail(&tmp, &blocks);
+
+		offset += size;
+
+		rem -= size;
+		if (!rem)
+			break;
+
+		cond_resched();
+	}
+
+	drm_buddy_free_list(&mm, &blocks);
+
+	KUNIT_EXPECT_FALSE_MSG(test, check_mm(test, &mm), "post-mm check failed\n");
+
+	drm_buddy_fini(&mm);
+}
+
+static void drm_test_buddy_alloc_limit(struct kunit *test)
+{
+	u64 size = U64_MAX, start = 0;
+	struct drm_buddy_block *block;
+	unsigned long flags = 0;
+	LIST_HEAD(allocated);
+	struct drm_buddy mm;
+
+	KUNIT_EXPECT_FALSE(test, drm_buddy_init(&mm, size, PAGE_SIZE));
+
+	KUNIT_EXPECT_EQ_MSG(test, mm.max_order, DRM_BUDDY_MAX_ORDER,
+			    "mm.max_order(%d) != %d\n", mm.max_order,
+						DRM_BUDDY_MAX_ORDER);
+
+	size = mm.chunk_size << mm.max_order;
+	KUNIT_EXPECT_FALSE(test, drm_buddy_alloc_blocks(&mm, start, size, size,
+							PAGE_SIZE, &allocated, flags));
+
+	block = list_first_entry_or_null(&allocated, struct drm_buddy_block, link);
+	KUNIT_EXPECT_TRUE(test, block);
+
+	KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_order(block), mm.max_order,
+			    "block order(%d) != %d\n",
+						drm_buddy_block_order(block), mm.max_order);
+
+	KUNIT_EXPECT_EQ_MSG(test, drm_buddy_block_size(&mm, block),
+			    BIT_ULL(mm.max_order) * PAGE_SIZE,
+						"block size(%llu) != %llu\n",
+						drm_buddy_block_size(&mm, block),
+						BIT_ULL(mm.max_order) * PAGE_SIZE);
+
+	drm_buddy_free_list(&mm, &allocated);
+	drm_buddy_fini(&mm);
+}
+
+static int drm_buddy_init_test(struct kunit *test)
+{
+	while (!random_seed)
+		random_seed = get_random_int();
+
+	return 0;
+}
+
+static struct kunit_case drm_buddy_tests[] = {
+	KUNIT_CASE(drm_test_buddy_alloc_limit),
+	KUNIT_CASE(drm_test_buddy_alloc_range),
+	KUNIT_CASE(drm_test_buddy_alloc_optimistic),
+	KUNIT_CASE(drm_test_buddy_alloc_pessimistic),
+	KUNIT_CASE(drm_test_buddy_alloc_smoke),
+	KUNIT_CASE(drm_test_buddy_alloc_pathological),
+	{}
+};
+
+static struct kunit_suite drm_buddy_test_suite = {
+	.name = "drm_buddy",
+	.init = drm_buddy_init_test,
+	.test_cases = drm_buddy_tests,
+};
+
+kunit_test_suite(drm_buddy_test_suite);
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tests/drm_cmdline_parser_test.c b/drivers/gpu/drm/tests/drm_cmdline_parser_test.c
new file mode 100644
index 000000000000..34790e7a3760
--- /dev/null
+++ b/drivers/gpu/drm/tests/drm_cmdline_parser_test.c
@@ -0,0 +1,991 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2019 Bootlin
+ * Copyright (c) 2022 Maíra Canal <mairacanal@riseup.net>
+ */
+
+#include <kunit/test.h>
+
+#include <drm/drm_connector.h>
+#include <drm/drm_modes.h>
+
+static const struct drm_connector no_connector = {};
+
+static void drm_test_cmdline_force_e_only(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "e";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_FALSE(test, mode.specified);
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON);
+}
+
+static void drm_test_cmdline_force_D_only_not_digital(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "D";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_FALSE(test, mode.specified);
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON);
+}
+
+static const struct drm_connector connector_hdmi = {
+	.connector_type	= DRM_MODE_CONNECTOR_HDMIB,
+};
+
+static void drm_test_cmdline_force_D_only_hdmi(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "D";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &connector_hdmi, &mode));
+	KUNIT_EXPECT_FALSE(test, mode.specified);
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON_DIGITAL);
+}
+
+static const struct drm_connector connector_dvi = {
+	.connector_type	= DRM_MODE_CONNECTOR_DVII,
+};
+
+static void drm_test_cmdline_force_D_only_dvi(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "D";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &connector_dvi, &mode));
+	KUNIT_EXPECT_FALSE(test, mode.specified);
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON_DIGITAL);
+}
+
+static void drm_test_cmdline_force_d_only(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "d";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_FALSE(test, mode.specified);
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_OFF);
+}
+
+static void drm_test_cmdline_res(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_res_vesa(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480M";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_TRUE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_res_vesa_rblank(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480MR";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_TRUE(test, mode.rb);
+	KUNIT_EXPECT_TRUE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_res_rblank(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480R";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_TRUE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_res_bpp(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480-24";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_res_refresh(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480@60";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_TRUE(test, mode.refresh_specified);
+	KUNIT_EXPECT_EQ(test, mode.refresh, 60);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_res_bpp_refresh(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480-24@60";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_TRUE(test, mode.refresh_specified);
+	KUNIT_EXPECT_EQ(test, mode.refresh, 60);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_res_bpp_refresh_interlaced(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480-24@60i";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_TRUE(test, mode.refresh_specified);
+	KUNIT_EXPECT_EQ(test, mode.refresh, 60);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_TRUE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_res_bpp_refresh_margins(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480-24@60m";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_TRUE(test, mode.refresh_specified);
+	KUNIT_EXPECT_EQ(test, mode.refresh, 60);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_TRUE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_res_bpp_refresh_force_off(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480-24@60d";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_TRUE(test, mode.refresh_specified);
+	KUNIT_EXPECT_EQ(test, mode.refresh, 60);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_OFF);
+}
+
+static void drm_test_cmdline_res_bpp_refresh_force_on(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480-24@60e";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_TRUE(test, mode.refresh_specified);
+	KUNIT_EXPECT_EQ(test, mode.refresh, 60);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON);
+}
+
+static void drm_test_cmdline_res_bpp_refresh_force_on_analog(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480-24@60D";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_TRUE(test, mode.refresh_specified);
+	KUNIT_EXPECT_EQ(test, mode.refresh, 60);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON);
+}
+
+static void drm_test_cmdline_res_bpp_refresh_force_on_digital(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	static const struct drm_connector connector = {
+		.connector_type = DRM_MODE_CONNECTOR_DVII,
+	};
+	const char *cmdline = "720x480-24@60D";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_TRUE(test, mode.refresh_specified);
+	KUNIT_EXPECT_EQ(test, mode.refresh, 60);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON_DIGITAL);
+}
+
+static void drm_test_cmdline_res_bpp_refresh_interlaced_margins_force_on(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480-24@60ime";
+
+	KUNIT_EXPECT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_TRUE(test, mode.refresh_specified);
+	KUNIT_EXPECT_EQ(test, mode.refresh, 60);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_TRUE(test, mode.interlace);
+	KUNIT_EXPECT_TRUE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON);
+}
+
+static void drm_test_cmdline_res_margins_force_on(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480me";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_TRUE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON);
+}
+
+static void drm_test_cmdline_res_vesa_margins(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480Mm";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_TRUE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_TRUE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_name(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "NTSC";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_STREQ(test, mode.name, "NTSC");
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+}
+
+static void drm_test_cmdline_name_bpp(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "NTSC-24";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_STREQ(test, mode.name, "NTSC");
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+}
+
+static void drm_test_cmdline_name_option(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "NTSC,rotate=180";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_STREQ(test, mode.name, "NTSC");
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, DRM_MODE_ROTATE_180);
+}
+
+static void drm_test_cmdline_name_bpp_option(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "NTSC-24,rotate=180";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_STREQ(test, mode.name, "NTSC");
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, DRM_MODE_ROTATE_180);
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+}
+
+static void drm_test_cmdline_rotate_0(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480,rotate=0";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, DRM_MODE_ROTATE_0);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_rotate_90(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480,rotate=90";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, DRM_MODE_ROTATE_90);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_rotate_180(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480,rotate=180";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, DRM_MODE_ROTATE_180);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_rotate_270(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480,rotate=270";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, DRM_MODE_ROTATE_270);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_hmirror(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480,reflect_x";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, (DRM_MODE_ROTATE_0 | DRM_MODE_REFLECT_X));
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_vmirror(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480,reflect_y";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, (DRM_MODE_ROTATE_0 | DRM_MODE_REFLECT_Y));
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_margin_options(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline =
+		"720x480,margin_right=14,margin_left=24,margin_bottom=36,margin_top=42";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.right, 14);
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.left, 24);
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.bottom, 36);
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.top, 42);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_multiple_options(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480,rotate=270,reflect_x";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, (DRM_MODE_ROTATE_270 | DRM_MODE_REFLECT_X));
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_bpp_extra_and_option(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480-24e,rotate=180";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, DRM_MODE_ROTATE_180);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+
+	KUNIT_EXPECT_TRUE(test, mode.bpp_specified);
+	KUNIT_EXPECT_EQ(test, mode.bpp, 24);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON);
+}
+
+static void drm_test_cmdline_extra_and_option(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "720x480e,rotate=180";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_TRUE(test, mode.specified);
+	KUNIT_EXPECT_EQ(test, mode.xres, 720);
+	KUNIT_EXPECT_EQ(test, mode.yres, 480);
+	KUNIT_EXPECT_EQ(test, mode.rotation_reflection, DRM_MODE_ROTATE_180);
+
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON);
+}
+
+static void drm_test_cmdline_freestanding_options(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "margin_right=14,margin_left=24,margin_bottom=36,margin_top=42";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_FALSE(test, mode.specified);
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.right, 14);
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.left, 24);
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.bottom, 36);
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.top, 42);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+static void drm_test_cmdline_freestanding_force_e_and_options(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "e,margin_right=14,margin_left=24,margin_bottom=36,margin_top=42";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_FALSE(test, mode.specified);
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.right, 14);
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.left, 24);
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.bottom, 36);
+	KUNIT_EXPECT_EQ(test, mode.tv_margins.top, 42);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_ON);
+}
+
+static void drm_test_cmdline_panel_orientation(struct kunit *test)
+{
+	struct drm_cmdline_mode mode = { };
+	const char *cmdline = "panel_orientation=upside_down";
+
+	KUNIT_ASSERT_TRUE(test, drm_mode_parse_command_line_for_connector(cmdline,
+									  &no_connector, &mode));
+	KUNIT_EXPECT_FALSE(test, mode.specified);
+	KUNIT_EXPECT_FALSE(test, mode.refresh_specified);
+	KUNIT_EXPECT_FALSE(test, mode.bpp_specified);
+
+	KUNIT_EXPECT_EQ(test, mode.panel_orientation, DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP);
+
+	KUNIT_EXPECT_FALSE(test, mode.rb);
+	KUNIT_EXPECT_FALSE(test, mode.cvt);
+	KUNIT_EXPECT_FALSE(test, mode.interlace);
+	KUNIT_EXPECT_FALSE(test, mode.margins);
+	KUNIT_EXPECT_EQ(test, mode.force, DRM_FORCE_UNSPECIFIED);
+}
+
+struct drm_cmdline_invalid_test {
+	const char *name;
+	const char *cmdline;
+};
+
+static void drm_test_cmdline_invalid(struct kunit *test)
+{
+	const struct drm_cmdline_invalid_test *params = test->param_value;
+	struct drm_cmdline_mode mode = { };
+
+	KUNIT_EXPECT_FALSE(test, drm_mode_parse_command_line_for_connector(params->cmdline,
+									   &no_connector,
+									   &mode));
+}
+
+static const struct drm_cmdline_invalid_test drm_cmdline_invalid_tests[] = {
+	{
+		.name = "margin_only",
+		.cmdline = "m",
+	},
+	{
+		.name = "interlace_only",
+		.cmdline = "i",
+	},
+	{
+		.name = "res_missing_x",
+		.cmdline = "x480",
+	},
+	{
+		.name = "res_missing_y",
+		.cmdline = "1024x",
+	},
+	{
+		.name = "res_bad_y",
+		.cmdline = "1024xtest",
+	},
+	{
+		.name = "res_missing_y_bpp",
+		.cmdline = "1024x-24",
+	},
+	{
+		.name = "res_bad_bpp",
+		.cmdline = "720x480-test",
+	},
+	{
+		.name = "res_bad_refresh",
+		.cmdline = "720x480@refresh",
+	},
+	{
+		.name = "res_bpp_refresh_force_on_off",
+		.cmdline = "720x480-24@60de",
+	},
+	{
+		.name = "res_invalid_mode",
+		.cmdline = "720x480f",
+	},
+	{
+		.name = "res_bpp_wrong_place_mode",
+		.cmdline = "720x480e-24",
+	},
+	{
+		.name = "name_bpp_refresh",
+		.cmdline = "NTSC-24@60",
+	},
+	{
+		.name = "name_refresh",
+		.cmdline = "NTSC@60",
+	},
+	{
+		.name = "name_refresh_wrong_mode",
+		.cmdline = "NTSC@60m",
+	},
+	{
+		.name = "name_refresh_invalid_mode",
+		.cmdline = "NTSC@60f",
+	},
+	{
+		.name = "rotate_multiple",
+		.cmdline = "720x480,rotate=0,rotate=90",
+	},
+	{
+		.name = "rotate_invalid_val",
+		.cmdline = "720x480,rotate=42",
+	},
+	{
+		.name = "rotate_truncated",
+		.cmdline = "720x480,rotate=",
+	},
+	{
+		.name = "invalid_option",
+		.cmdline = "720x480,test=42",
+	},
+};
+
+static void drm_cmdline_invalid_desc(const struct drm_cmdline_invalid_test *t,
+				     char *desc)
+{
+	sprintf(desc, "%s", t->name);
+}
+
+KUNIT_ARRAY_PARAM(drm_cmdline_invalid, drm_cmdline_invalid_tests, drm_cmdline_invalid_desc);
+
+static struct kunit_case drm_cmdline_parser_tests[] = {
+	KUNIT_CASE(drm_test_cmdline_force_d_only),
+	KUNIT_CASE(drm_test_cmdline_force_D_only_dvi),
+	KUNIT_CASE(drm_test_cmdline_force_D_only_hdmi),
+	KUNIT_CASE(drm_test_cmdline_force_D_only_not_digital),
+	KUNIT_CASE(drm_test_cmdline_force_e_only),
+	KUNIT_CASE(drm_test_cmdline_res),
+	KUNIT_CASE(drm_test_cmdline_res_vesa),
+	KUNIT_CASE(drm_test_cmdline_res_vesa_rblank),
+	KUNIT_CASE(drm_test_cmdline_res_rblank),
+	KUNIT_CASE(drm_test_cmdline_res_bpp),
+	KUNIT_CASE(drm_test_cmdline_res_refresh),
+	KUNIT_CASE(drm_test_cmdline_res_bpp_refresh),
+	KUNIT_CASE(drm_test_cmdline_res_bpp_refresh_interlaced),
+	KUNIT_CASE(drm_test_cmdline_res_bpp_refresh_margins),
+	KUNIT_CASE(drm_test_cmdline_res_bpp_refresh_force_off),
+	KUNIT_CASE(drm_test_cmdline_res_bpp_refresh_force_on),
+	KUNIT_CASE(drm_test_cmdline_res_bpp_refresh_force_on_analog),
+	KUNIT_CASE(drm_test_cmdline_res_bpp_refresh_force_on_digital),
+	KUNIT_CASE(drm_test_cmdline_res_bpp_refresh_interlaced_margins_force_on),
+	KUNIT_CASE(drm_test_cmdline_res_margins_force_on),
+	KUNIT_CASE(drm_test_cmdline_res_vesa_margins),
+	KUNIT_CASE(drm_test_cmdline_name),
+	KUNIT_CASE(drm_test_cmdline_name_bpp),
+	KUNIT_CASE(drm_test_cmdline_name_option),
+	KUNIT_CASE(drm_test_cmdline_name_bpp_option),
+	KUNIT_CASE(drm_test_cmdline_rotate_0),
+	KUNIT_CASE(drm_test_cmdline_rotate_90),
+	KUNIT_CASE(drm_test_cmdline_rotate_180),
+	KUNIT_CASE(drm_test_cmdline_rotate_270),
+	KUNIT_CASE(drm_test_cmdline_hmirror),
+	KUNIT_CASE(drm_test_cmdline_vmirror),
+	KUNIT_CASE(drm_test_cmdline_margin_options),
+	KUNIT_CASE(drm_test_cmdline_multiple_options),
+	KUNIT_CASE(drm_test_cmdline_bpp_extra_and_option),
+	KUNIT_CASE(drm_test_cmdline_extra_and_option),
+	KUNIT_CASE(drm_test_cmdline_freestanding_options),
+	KUNIT_CASE(drm_test_cmdline_freestanding_force_e_and_options),
+	KUNIT_CASE(drm_test_cmdline_panel_orientation),
+	KUNIT_CASE_PARAM(drm_test_cmdline_invalid, drm_cmdline_invalid_gen_params),
+	{}
+};
+
+static struct kunit_suite drm_cmdline_parser_test_suite = {
+	.name = "drm_cmdline_parser",
+	.test_cases = drm_cmdline_parser_tests
+};
+
+kunit_test_suite(drm_cmdline_parser_test_suite);
+
+MODULE_AUTHOR("Maxime Ripard <maxime.ripard@bootlin.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tests/drm_damage_helper_test.c b/drivers/gpu/drm/tests/drm_damage_helper_test.c
new file mode 100644
index 000000000000..115034fc3421
--- /dev/null
+++ b/drivers/gpu/drm/tests/drm_damage_helper_test.c
@@ -0,0 +1,639 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Test case for drm_damage_helper functions
+ *
+ * Copyright (c) 2022 Maíra Canal <mairacanal@riseup.net>
+ */
+
+#include <kunit/test.h>
+
+#include <drm/drm_damage_helper.h>
+#include <drm/drm_framebuffer.h>
+#include <drm/drm_plane.h>
+#include <drm/drm_drv.h>
+
+struct drm_damage_mock {
+	struct drm_driver driver;
+	struct drm_device device;
+	struct drm_object_properties obj_props;
+	struct drm_plane plane;
+	struct drm_property prop;
+	struct drm_framebuffer fb;
+	struct drm_plane_state state;
+	struct drm_plane_state old_state;
+};
+
+static int drm_damage_helper_init(struct kunit *test)
+{
+	struct drm_damage_mock *mock;
+
+	mock = kunit_kzalloc(test, sizeof(*mock), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mock);
+
+	mock->fb.width = 2048;
+	mock->fb.height = 2048;
+
+	mock->state.crtc = ZERO_SIZE_PTR;
+	mock->state.fb = &mock->fb;
+	mock->state.visible = true;
+
+	mock->old_state.plane = &mock->plane;
+	mock->state.plane = &mock->plane;
+
+	/* just enough so that drm_plane_enable_fb_damage_clips() works */
+	mock->device.driver = &mock->driver;
+	mock->device.mode_config.prop_fb_damage_clips = &mock->prop;
+	mock->plane.dev = &mock->device;
+	mock->obj_props.count = 0;
+	mock->plane.base.properties = &mock->obj_props;
+	mock->prop.base.id = 1; /* 0 is an invalid id */
+	mock->prop.dev = &mock->device;
+
+	drm_plane_enable_fb_damage_clips(&mock->plane);
+
+	test->priv = mock;
+
+	return 0;
+}
+
+static void set_plane_src(struct drm_plane_state *state, int x1, int y1, int x2,
+			  int y2)
+{
+	state->src_x = x1;
+	state->src_y = y1;
+	state->src_w = x2 - x1;
+	state->src_h = y2 - y1;
+
+	state->src.x1 = x1;
+	state->src.y1 = y1;
+	state->src.x2 = x2;
+	state->src.y2 = y2;
+}
+
+static void set_damage_clip(struct drm_mode_rect *r, int x1, int y1, int x2,
+			    int y2)
+{
+	r->x1 = x1;
+	r->y1 = y1;
+	r->x2 = x2;
+	r->y2 = y2;
+}
+
+static void set_damage_blob(struct drm_property_blob *damage_blob,
+			    struct drm_mode_rect *r, u32 size)
+{
+	damage_blob->length = size;
+	damage_blob->data = r;
+}
+
+static void set_plane_damage(struct drm_plane_state *state,
+			     struct drm_property_blob *damage_blob)
+{
+	state->fb_damage_clips = damage_blob;
+}
+
+static void check_damage_clip(struct kunit *test, struct drm_rect *r,
+			      int x1, int y1, int x2, int y2)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_plane_state state = mock->state;
+
+	/*
+	 * Round down x1/y1 and round up x2/y2. This is because damage is not in
+	 * 16.16 fixed point so to catch all pixels.
+	 */
+	int src_x1 = state.src.x1 >> 16;
+	int src_y1 = state.src.y1 >> 16;
+	int src_x2 = (state.src.x2 >> 16) + !!(state.src.x2 & 0xFFFF);
+	int src_y2 = (state.src.y2 >> 16) + !!(state.src.y2 & 0xFFFF);
+
+	if (x1 >= x2 || y1 >= y2)
+		KUNIT_FAIL(test, "Cannot have damage clip with no dimension.");
+	if (x1 < src_x1 || y1 < src_y1 || x2 > src_x2 || y2 > src_y2)
+		KUNIT_FAIL(test, "Damage cannot be outside rounded plane src.");
+	if (r->x1 != x1 || r->y1 != y1 || r->x2 != x2 || r->y2 != y2)
+		KUNIT_FAIL(test, "Damage = %d %d %d %d, want = %d %d %d %d",
+			   r->x1, r->y1, r->x2, r->y2, x1, y1, x2, y2);
+}
+
+static void drm_test_damage_iter_no_damage(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	/* Plane src same as fb size. */
+	set_plane_src(&mock->old_state, 0, 0, mock->fb.width << 16, mock->fb.height << 16);
+	set_plane_src(&mock->state, 0, 0, mock->fb.width << 16, mock->fb.height << 16);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1, "Should return plane src as damage.");
+	check_damage_clip(test, &clip, 0, 0, 2048, 2048);
+}
+
+static void drm_test_damage_iter_no_damage_fractional_src(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	/* Plane src has fractional part. */
+	set_plane_src(&mock->old_state, 0x3fffe, 0x3fffe,
+		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
+	set_plane_src(&mock->state, 0x3fffe, 0x3fffe,
+		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1,
+			    "Should return rounded off plane src as damage.");
+	check_damage_clip(test, &clip, 3, 3, 1028, 772);
+}
+
+static void drm_test_damage_iter_no_damage_src_moved(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	/* Plane src moved since old plane state. */
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 10 << 16, 10 << 16,
+		      (10 + 1024) << 16, (10 + 768) << 16);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1, "Should return plane src as damage.");
+	check_damage_clip(test, &clip, 10, 10, 1034, 778);
+}
+
+static void drm_test_damage_iter_no_damage_fractional_src_moved(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	/* Plane src has fractional part and it moved since old plane state. */
+	set_plane_src(&mock->old_state, 0x3fffe, 0x3fffe,
+		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
+	set_plane_src(&mock->state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1, "Should return plane src as damage.");
+	check_damage_clip(test, &clip, 4, 4, 1029, 773);
+}
+
+static void drm_test_damage_iter_no_damage_not_visible(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	mock->state.visible = false;
+
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 0, 0, 1024 << 16, 768 << 16);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 0, "Should have no damage.");
+}
+
+static void drm_test_damage_iter_no_damage_no_crtc(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	mock->state.crtc = NULL;
+
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 0, 0, 1024 << 16, 768 << 16);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 0, "Should have no damage.");
+}
+
+static void drm_test_damage_iter_no_damage_no_fb(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	mock->state.fb = NULL;
+
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 0, 0, 1024 << 16, 768 << 16);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 0, "Should have no damage.");
+}
+
+static void drm_test_damage_iter_simple_damage(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 0, 0, 1024 << 16, 768 << 16);
+	/* Damage set to plane src */
+	set_damage_clip(&damage, 0, 0, 1024, 768);
+	set_damage_blob(&damage_blob, &damage, sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1, "Should return damage when set.");
+	check_damage_clip(test, &clip, 0, 0, 1024, 768);
+}
+
+static void drm_test_damage_iter_single_damage(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 0, 0, 1024 << 16, 768 << 16);
+	set_damage_clip(&damage, 256, 192, 768, 576);
+	set_damage_blob(&damage_blob, &damage, sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1, "Should return damage when set.");
+	check_damage_clip(test, &clip, 256, 192, 768, 576);
+}
+
+static void drm_test_damage_iter_single_damage_intersect_src(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 0, 0, 1024 << 16, 768 << 16);
+	/* Damage intersect with plane src. */
+	set_damage_clip(&damage, 256, 192, 1360, 768);
+	set_damage_blob(&damage_blob, &damage, sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1, "Should return damage clipped to src.");
+	check_damage_clip(test, &clip, 256, 192, 1024, 768);
+}
+
+static void drm_test_damage_iter_single_damage_outside_src(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 0, 0, 1024 << 16, 768 << 16);
+	/* Damage clip outside plane src */
+	set_damage_clip(&damage, 1360, 1360, 1380, 1380);
+	set_damage_blob(&damage_blob, &damage, sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 0, "Should have no damage.");
+}
+
+static void drm_test_damage_iter_single_damage_fractional_src(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	/* Plane src has fractional part. */
+	set_plane_src(&mock->old_state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	set_plane_src(&mock->state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	set_damage_clip(&damage, 10, 10, 256, 330);
+	set_damage_blob(&damage_blob, &damage, sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1, "Should return damage when set.");
+	check_damage_clip(test, &clip, 10, 10, 256, 330);
+}
+
+static void drm_test_damage_iter_single_damage_intersect_fractional_src(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	/* Plane src has fractional part. */
+	set_plane_src(&mock->old_state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	set_plane_src(&mock->state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	/* Damage intersect with plane src. */
+	set_damage_clip(&damage, 10, 1, 1360, 330);
+	set_damage_blob(&damage_blob, &damage, sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1,
+			    "Should return damage clipped to rounded off src.");
+	check_damage_clip(test, &clip, 10, 4, 1029, 330);
+}
+
+static void drm_test_damage_iter_single_damage_outside_fractional_src(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	/* Plane src has fractional part. */
+	set_plane_src(&mock->old_state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	set_plane_src(&mock->state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	/* Damage clip outside plane src */
+	set_damage_clip(&damage, 1360, 1360, 1380, 1380);
+	set_damage_blob(&damage_blob, &damage, sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 0, "Should have no damage.");
+}
+
+static void drm_test_damage_iter_single_damage_src_moved(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	/* Plane src moved since old plane state. */
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 10 << 16, 10 << 16,
+		      (10 + 1024) << 16, (10 + 768) << 16);
+	set_damage_clip(&damage, 20, 30, 256, 256);
+	set_damage_blob(&damage_blob, &damage, sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1,
+			    "Should return plane src as damage.");
+	check_damage_clip(test, &clip, 10, 10, 1034, 778);
+}
+
+static void drm_test_damage_iter_single_damage_fractional_src_moved(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage;
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	/* Plane src with fractional part moved since old plane state. */
+	set_plane_src(&mock->old_state, 0x3fffe, 0x3fffe,
+		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
+	set_plane_src(&mock->state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	/* Damage intersect with plane src. */
+	set_damage_clip(&damage, 20, 30, 1360, 256);
+	set_damage_blob(&damage_blob, &damage, sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1,
+			    "Should return rounded off plane as damage.");
+	check_damage_clip(test, &clip, 4, 4, 1029, 773);
+}
+
+static void drm_test_damage_iter_damage(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage[2];
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 0, 0, 1024 << 16, 768 << 16);
+	/* 2 damage clips. */
+	set_damage_clip(&damage[0], 20, 30, 200, 180);
+	set_damage_clip(&damage[1], 240, 200, 280, 250);
+	set_damage_blob(&damage_blob, &damage[0], sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip) {
+		if (num_hits == 0)
+			check_damage_clip(test, &clip, 20, 30, 200, 180);
+		if (num_hits == 1)
+			check_damage_clip(test, &clip, 240, 200, 280, 250);
+		num_hits++;
+	}
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 2, "Should return damage when set.");
+}
+
+static void drm_test_damage_iter_damage_one_intersect(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage[2];
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	set_plane_src(&mock->old_state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	set_plane_src(&mock->state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	/* 2 damage clips, one intersect plane src. */
+	set_damage_clip(&damage[0], 20, 30, 200, 180);
+	set_damage_clip(&damage[1], 2, 2, 1360, 1360);
+	set_damage_blob(&damage_blob, &damage[0], sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip) {
+		if (num_hits == 0)
+			check_damage_clip(test, &clip, 20, 30, 200, 180);
+		if (num_hits == 1)
+			check_damage_clip(test, &clip, 4, 4, 1029, 773);
+		num_hits++;
+	}
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 2, "Should return damage when set.");
+}
+
+static void drm_test_damage_iter_damage_one_outside(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage[2];
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	set_plane_src(&mock->old_state, 0, 0, 1024 << 16, 768 << 16);
+	set_plane_src(&mock->state, 0, 0, 1024 << 16, 768 << 16);
+	/* 2 damage clips, one outside plane src. */
+	set_damage_clip(&damage[0], 1360, 1360, 1380, 1380);
+	set_damage_clip(&damage[1], 240, 200, 280, 250);
+	set_damage_blob(&damage_blob, &damage[0], sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1, "Should return damage when set.");
+	check_damage_clip(test, &clip, 240, 200, 280, 250);
+}
+
+static void drm_test_damage_iter_damage_src_moved(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage[2];
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	set_plane_src(&mock->old_state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	set_plane_src(&mock->state, 0x3fffe, 0x3fffe,
+		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
+	/* 2 damage clips, one outside plane src. */
+	set_damage_clip(&damage[0], 1360, 1360, 1380, 1380);
+	set_damage_clip(&damage[1], 240, 200, 280, 250);
+	set_damage_blob(&damage_blob, &damage[0], sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 1,
+			    "Should return round off plane src as damage.");
+	check_damage_clip(test, &clip, 3, 3, 1028, 772);
+}
+
+static void drm_test_damage_iter_damage_not_visible(struct kunit *test)
+{
+	struct drm_damage_mock *mock = test->priv;
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_property_blob damage_blob;
+	struct drm_mode_rect damage[2];
+	struct drm_rect clip;
+	u32 num_hits = 0;
+
+	mock->state.visible = false;
+
+	set_plane_src(&mock->old_state, 0x40002, 0x40002,
+		      0x40002 + (1024 << 16), 0x40002 + (768 << 16));
+	set_plane_src(&mock->state, 0x3fffe, 0x3fffe,
+		      0x3fffe + (1024 << 16), 0x3fffe + (768 << 16));
+	/* 2 damage clips, one outside plane src. */
+	set_damage_clip(&damage[0], 1360, 1360, 1380, 1380);
+	set_damage_clip(&damage[1], 240, 200, 280, 250);
+	set_damage_blob(&damage_blob, &damage[0], sizeof(damage));
+	set_plane_damage(&mock->state, &damage_blob);
+	drm_atomic_helper_damage_iter_init(&iter, &mock->old_state, &mock->state);
+	drm_atomic_for_each_plane_damage(&iter, &clip)
+		num_hits++;
+
+	KUNIT_EXPECT_EQ_MSG(test, num_hits, 0, "Should not return any damage.");
+}
+
+static struct kunit_case drm_damage_helper_tests[] = {
+	KUNIT_CASE(drm_test_damage_iter_no_damage),
+	KUNIT_CASE(drm_test_damage_iter_no_damage_fractional_src),
+	KUNIT_CASE(drm_test_damage_iter_no_damage_src_moved),
+	KUNIT_CASE(drm_test_damage_iter_no_damage_fractional_src_moved),
+	KUNIT_CASE(drm_test_damage_iter_no_damage_not_visible),
+	KUNIT_CASE(drm_test_damage_iter_no_damage_no_crtc),
+	KUNIT_CASE(drm_test_damage_iter_no_damage_no_fb),
+	KUNIT_CASE(drm_test_damage_iter_simple_damage),
+	KUNIT_CASE(drm_test_damage_iter_single_damage),
+	KUNIT_CASE(drm_test_damage_iter_single_damage_intersect_src),
+	KUNIT_CASE(drm_test_damage_iter_single_damage_outside_src),
+	KUNIT_CASE(drm_test_damage_iter_single_damage_fractional_src),
+	KUNIT_CASE(drm_test_damage_iter_single_damage_intersect_fractional_src),
+	KUNIT_CASE(drm_test_damage_iter_single_damage_outside_fractional_src),
+	KUNIT_CASE(drm_test_damage_iter_single_damage_src_moved),
+	KUNIT_CASE(drm_test_damage_iter_single_damage_fractional_src_moved),
+	KUNIT_CASE(drm_test_damage_iter_damage),
+	KUNIT_CASE(drm_test_damage_iter_damage_one_intersect),
+	KUNIT_CASE(drm_test_damage_iter_damage_one_outside),
+	KUNIT_CASE(drm_test_damage_iter_damage_src_moved),
+	KUNIT_CASE(drm_test_damage_iter_damage_not_visible),
+	{ }
+};
+
+static struct kunit_suite drm_damage_helper_test_suite = {
+	.name = "drm_damage_helper",
+	.init = drm_damage_helper_init,
+	.test_cases = drm_damage_helper_tests,
+};
+
+kunit_test_suite(drm_damage_helper_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/selftests/test-drm_dp_mst_helper.c b/drivers/gpu/drm/tests/drm_dp_mst_helper_test.c
index 4caa9be900ac..65c9d225b558 100644
--- a/drivers/gpu/drm/selftests/test-drm_dp_mst_helper.c
+++ b/drivers/gpu/drm/tests/drm_dp_mst_helper_test.c
@@ -1,19 +1,22 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Test cases for the DRM DP MST helpers
+ *
+ * Copyright (c) 2022 Maíra Canal <mairacanal@riseup.net>
  */
 
 #define PREFIX_STR "[drm_dp_mst_helper]"
 
+#include <kunit/test.h>
+
 #include <linux/random.h>
 
 #include <drm/display/drm_dp_mst_helper.h>
 #include <drm/drm_print.h>
 
 #include "../display/drm_dp_mst_topology_internal.h"
-#include "test-drm_modeset_common.h"
 
-int igt_dp_mst_calc_pbn_mode(void *ignored)
+static void drm_test_dp_mst_calc_pbn_mode(struct kunit *test)
 {
 	int pbn, i;
 	const struct {
@@ -33,13 +36,11 @@ int igt_dp_mst_calc_pbn_mode(void *ignored)
 		pbn = drm_dp_calc_pbn_mode(test_params[i].rate,
 					   test_params[i].bpp,
 					   test_params[i].dsc);
-		FAIL(pbn != test_params[i].expected,
-		     "Expected PBN %d for clock %d bpp %d, got %d\n",
+		KUNIT_EXPECT_EQ_MSG(test, pbn, test_params[i].expected,
+				    "Expected PBN %d for clock %d bpp %d, got %d\n",
 		     test_params[i].expected, test_params[i].rate,
 		     test_params[i].bpp, pbn);
 	}
-
-	return 0;
 }
 
 static bool
@@ -176,66 +177,64 @@ out:
 	return result;
 }
 
-int igt_dp_mst_sideband_msg_req_decode(void *unused)
+static void drm_test_dp_mst_sideband_msg_req_decode(struct kunit *test)
 {
 	struct drm_dp_sideband_msg_req_body in = { 0 };
 	u8 data[] = { 0xff, 0x0, 0xdd };
 	int i;
 
-#define DO_TEST() FAIL_ON(!sideband_msg_req_encode_decode(&in))
-
 	in.req_type = DP_ENUM_PATH_RESOURCES;
 	in.u.port_num.port_number = 5;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 
 	in.req_type = DP_POWER_UP_PHY;
 	in.u.port_num.port_number = 5;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 
 	in.req_type = DP_POWER_DOWN_PHY;
 	in.u.port_num.port_number = 5;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 
 	in.req_type = DP_ALLOCATE_PAYLOAD;
 	in.u.allocate_payload.number_sdp_streams = 3;
 	for (i = 0; i < in.u.allocate_payload.number_sdp_streams; i++)
 		in.u.allocate_payload.sdp_stream_sink[i] = i + 1;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.allocate_payload.port_number = 0xf;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.allocate_payload.vcpi = 0x7f;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.allocate_payload.pbn = U16_MAX;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 
 	in.req_type = DP_QUERY_PAYLOAD;
 	in.u.query_payload.port_number = 0xf;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.query_payload.vcpi = 0x7f;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 
 	in.req_type = DP_REMOTE_DPCD_READ;
 	in.u.dpcd_read.port_number = 0xf;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.dpcd_read.dpcd_address = 0xfedcb;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.dpcd_read.num_bytes = U8_MAX;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 
 	in.req_type = DP_REMOTE_DPCD_WRITE;
 	in.u.dpcd_write.port_number = 0xf;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.dpcd_write.dpcd_address = 0xfedcb;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.dpcd_write.num_bytes = ARRAY_SIZE(data);
 	in.u.dpcd_write.bytes = data;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 
 	in.req_type = DP_REMOTE_I2C_READ;
 	in.u.i2c_read.port_number = 0xf;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.i2c_read.read_i2c_device_id = 0x7f;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.i2c_read.num_transactions = 3;
 	in.u.i2c_read.num_bytes_read = ARRAY_SIZE(data) * 3;
 	for (i = 0; i < in.u.i2c_read.num_transactions; i++) {
@@ -244,32 +243,44 @@ int igt_dp_mst_sideband_msg_req_decode(void *unused)
 		in.u.i2c_read.transactions[i].i2c_dev_id = 0x7f & ~i;
 		in.u.i2c_read.transactions[i].i2c_transaction_delay = 0xf & ~i;
 	}
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 
 	in.req_type = DP_REMOTE_I2C_WRITE;
 	in.u.i2c_write.port_number = 0xf;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.i2c_write.write_i2c_device_id = 0x7f;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.i2c_write.num_bytes = ARRAY_SIZE(data);
 	in.u.i2c_write.bytes = data;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 
 	in.req_type = DP_QUERY_STREAM_ENC_STATUS;
 	in.u.enc_status.stream_id = 1;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	get_random_bytes(in.u.enc_status.client_id,
 			 sizeof(in.u.enc_status.client_id));
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.enc_status.stream_event = 3;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.enc_status.valid_stream_event = 0;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.enc_status.stream_behavior = 3;
-	DO_TEST();
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 	in.u.enc_status.valid_stream_behavior = 1;
-	DO_TEST();
-
-#undef DO_TEST
-	return 0;
+	KUNIT_EXPECT_TRUE(test, sideband_msg_req_encode_decode(&in));
 }
+
+static struct kunit_case drm_dp_mst_helper_tests[] = {
+	KUNIT_CASE(drm_test_dp_mst_calc_pbn_mode),
+	KUNIT_CASE(drm_test_dp_mst_sideband_msg_req_decode),
+	{ }
+};
+
+static struct kunit_suite drm_dp_mst_helper_test_suite = {
+	.name = "drm_dp_mst_helper",
+	.test_cases = drm_dp_mst_helper_tests,
+};
+
+kunit_test_suite(drm_dp_mst_helper_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tests/drm_format_helper_test.c b/drivers/gpu/drm/tests/drm_format_helper_test.c
index 98583bf56044..8d86c250c2ec 100644
--- a/drivers/gpu/drm/tests/drm_format_helper_test.c
+++ b/drivers/gpu/drm/tests/drm_format_helper_test.c
@@ -16,34 +16,101 @@
 
 #define TEST_BUF_SIZE 50
 
-struct xrgb8888_to_rgb332_case {
+struct convert_to_gray8_result {
+	unsigned int dst_pitch;
+	const u8 expected[TEST_BUF_SIZE];
+};
+
+struct convert_to_rgb332_result {
+	unsigned int dst_pitch;
+	const u8 expected[TEST_BUF_SIZE];
+};
+
+struct convert_to_rgb565_result {
+	unsigned int dst_pitch;
+	const u16 expected[TEST_BUF_SIZE];
+	const u16 expected_swab[TEST_BUF_SIZE];
+};
+
+struct convert_to_rgb888_result {
+	unsigned int dst_pitch;
+	const u8 expected[TEST_BUF_SIZE];
+};
+
+struct convert_to_xrgb2101010_result {
+	unsigned int dst_pitch;
+	const u32 expected[TEST_BUF_SIZE];
+};
+
+struct convert_xrgb8888_case {
 	const char *name;
 	unsigned int pitch;
-	unsigned int dst_pitch;
 	struct drm_rect clip;
 	const u32 xrgb8888[TEST_BUF_SIZE];
-	const u8 expected[4 * TEST_BUF_SIZE];
+	struct convert_to_gray8_result gray8_result;
+	struct convert_to_rgb332_result rgb332_result;
+	struct convert_to_rgb565_result rgb565_result;
+	struct convert_to_rgb888_result rgb888_result;
+	struct convert_to_xrgb2101010_result xrgb2101010_result;
 };
 
-static struct xrgb8888_to_rgb332_case xrgb8888_to_rgb332_cases[] = {
+static struct convert_xrgb8888_case convert_xrgb8888_cases[] = {
 	{
 		.name = "single_pixel_source_buffer",
 		.pitch = 1 * 4,
-		.dst_pitch = 0,
 		.clip = DRM_RECT_INIT(0, 0, 1, 1),
 		.xrgb8888 = { 0x01FF0000 },
-		.expected = { 0xE0 },
+		.gray8_result = {
+			.dst_pitch = 0,
+			.expected = { 0x4C },
+		},
+		.rgb332_result = {
+			.dst_pitch = 0,
+			.expected = { 0xE0 },
+		},
+		.rgb565_result = {
+			.dst_pitch = 0,
+			.expected = { 0xF800 },
+			.expected_swab = { 0x00F8 },
+		},
+		.rgb888_result = {
+			.dst_pitch = 0,
+			.expected = { 0x00, 0x00, 0xFF },
+		},
+		.xrgb2101010_result = {
+			.dst_pitch = 0,
+			.expected = { 0x3FF00000 },
+		},
 	},
 	{
 		.name = "single_pixel_clip_rectangle",
 		.pitch = 2 * 4,
-		.dst_pitch = 0,
 		.clip = DRM_RECT_INIT(1, 1, 1, 1),
 		.xrgb8888 = {
 			0x00000000, 0x00000000,
 			0x00000000, 0x10FF0000,
 		},
-		.expected = { 0xE0 },
+		.gray8_result = {
+			.dst_pitch = 0,
+			.expected = { 0x4C },
+		},
+		.rgb332_result = {
+			.dst_pitch = 0,
+			.expected = { 0xE0 },
+		},
+		.rgb565_result = {
+			.dst_pitch = 0,
+			.expected = { 0xF800 },
+			.expected_swab = { 0x00F8 },
+		},
+		.rgb888_result = {
+			.dst_pitch = 0,
+			.expected = { 0x00, 0x00, 0xFF },
+		},
+		.xrgb2101010_result = {
+			.dst_pitch = 0,
+			.expected = { 0x3FF00000 },
+		},
 	},
 	{
 		/* Well known colors: White, black, red, green, blue, magenta,
@@ -52,7 +119,6 @@ static struct xrgb8888_to_rgb332_case xrgb8888_to_rgb332_cases[] = {
 		 */
 		.name = "well_known_colors",
 		.pitch = 4 * 4,
-		.dst_pitch = 0,
 		.clip = DRM_RECT_INIT(1, 1, 2, 4),
 		.xrgb8888 = {
 			0x00000000, 0x00000000, 0x00000000, 0x00000000,
@@ -61,28 +127,115 @@ static struct xrgb8888_to_rgb332_case xrgb8888_to_rgb332_cases[] = {
 			0x00000000, 0x550000FF, 0x66FF00FF, 0x00000000,
 			0x00000000, 0x77FFFF00, 0x8800FFFF, 0x00000000,
 		},
-		.expected = {
-			0xFF, 0x00,
-			0xE0, 0x1C,
-			0x03, 0xE3,
-			0xFC, 0x1F,
+		.gray8_result = {
+			.dst_pitch = 0,
+			.expected = {
+				0xFF, 0x00,
+				0x4C, 0x99,
+				0x19, 0x66,
+				0xE5, 0xB2,
+			},
+		},
+		.rgb332_result = {
+			.dst_pitch = 0,
+			.expected = {
+				0xFF, 0x00,
+				0xE0, 0x1C,
+				0x03, 0xE3,
+				0xFC, 0x1F,
+			},
+		},
+		.rgb565_result = {
+			.dst_pitch = 0,
+			.expected = {
+				0xFFFF, 0x0000,
+				0xF800, 0x07E0,
+				0x001F, 0xF81F,
+				0xFFE0, 0x07FF,
+			},
+			.expected_swab = {
+				0xFFFF, 0x0000,
+				0x00F8, 0xE007,
+				0x1F00, 0x1FF8,
+				0xE0FF, 0xFF07,
+			},
+		},
+		.rgb888_result = {
+			.dst_pitch = 0,
+			.expected = {
+				0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00,
+				0x00, 0x00, 0xFF, 0x00, 0xFF, 0x00,
+				0xFF, 0x00, 0x00, 0xFF, 0x00, 0xFF,
+				0x00, 0xFF, 0xFF, 0xFF, 0xFF, 0x00,
+			},
+		},
+		.xrgb2101010_result = {
+			.dst_pitch = 0,
+			.expected = {
+				0x3FFFFFFF, 0x00000000,
+				0x3FF00000, 0x000FFC00,
+				0x000003FF, 0x3FF003FF,
+				0x3FFFFC00, 0x000FFFFF,
+			},
 		},
 	},
 	{
 		/* Randomly picked colors. Full buffer within the clip area. */
 		.name = "destination_pitch",
 		.pitch = 3 * 4,
-		.dst_pitch = 5,
 		.clip = DRM_RECT_INIT(0, 0, 3, 3),
 		.xrgb8888 = {
 			0xA10E449C, 0xB1114D05, 0xC1A80303,
 			0xD16C7073, 0xA20E449C, 0xB2114D05,
 			0xC2A80303, 0xD26C7073, 0xA30E449C,
 		},
-		.expected = {
-			0x0A, 0x08, 0xA0, 0x00, 0x00,
-			0x6D, 0x0A, 0x08, 0x00, 0x00,
-			0xA0, 0x6D, 0x0A, 0x00, 0x00,
+		.gray8_result = {
+			.dst_pitch = 5,
+			.expected = {
+				0x3C, 0x33, 0x34, 0x00, 0x00,
+				0x6F, 0x3C, 0x33, 0x00, 0x00,
+				0x34, 0x6F, 0x3C, 0x00, 0x00,
+			},
+		},
+		.rgb332_result = {
+			.dst_pitch = 5,
+			.expected = {
+				0x0A, 0x08, 0xA0, 0x00, 0x00,
+				0x6D, 0x0A, 0x08, 0x00, 0x00,
+				0xA0, 0x6D, 0x0A, 0x00, 0x00,
+			},
+		},
+		.rgb565_result = {
+			.dst_pitch = 10,
+			.expected = {
+				0x0A33, 0x1260, 0xA800, 0x0000, 0x0000,
+				0x6B8E, 0x0A33, 0x1260, 0x0000, 0x0000,
+				0xA800, 0x6B8E, 0x0A33, 0x0000, 0x0000,
+			},
+			.expected_swab = {
+				0x330A, 0x6012, 0x00A8, 0x0000, 0x0000,
+				0x8E6B, 0x330A, 0x6012, 0x0000, 0x0000,
+				0x00A8, 0x8E6B, 0x330A, 0x0000, 0x0000,
+			},
+		},
+		.rgb888_result = {
+			.dst_pitch = 15,
+			.expected = {
+				0x9C, 0x44, 0x0E, 0x05, 0x4D, 0x11, 0x03, 0x03, 0xA8,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x73, 0x70, 0x6C, 0x9C, 0x44, 0x0E, 0x05, 0x4D, 0x11,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+				0x03, 0x03, 0xA8, 0x73, 0x70, 0x6C, 0x9C, 0x44, 0x0E,
+				0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+			},
+		},
+		.xrgb2101010_result = {
+			.dst_pitch = 20,
+			.expected = {
+				0x03844672, 0x0444D414, 0x2A20300C, 0x00000000, 0x00000000,
+				0x1B1705CD, 0x03844672, 0x0444D414, 0x00000000, 0x00000000,
+				0x2A20300C, 0x1B1705CD, 0x03844672, 0x00000000, 0x00000000,
+			},
 		},
 	},
 };
@@ -111,41 +264,190 @@ static size_t conversion_buf_size(u32 dst_format, unsigned int dst_pitch,
 	return dst_pitch * drm_rect_height(clip);
 }
 
-static void xrgb8888_to_rgb332_case_desc(struct xrgb8888_to_rgb332_case *t,
-					 char *desc)
+static u32 *le32buf_to_cpu(struct kunit *test, const u32 *buf, size_t buf_size)
+{
+	u32 *dst = NULL;
+	int n;
+
+	dst = kunit_kzalloc(test, sizeof(*dst) * buf_size, GFP_KERNEL);
+	if (!dst)
+		return NULL;
+
+	for (n = 0; n < buf_size; n++)
+		dst[n] = le32_to_cpu((__force __le32)buf[n]);
+
+	return dst;
+}
+
+static void convert_xrgb8888_case_desc(struct convert_xrgb8888_case *t,
+				       char *desc)
 {
 	strscpy(desc, t->name, KUNIT_PARAM_DESC_SIZE);
 }
 
-KUNIT_ARRAY_PARAM(xrgb8888_to_rgb332, xrgb8888_to_rgb332_cases,
-		  xrgb8888_to_rgb332_case_desc);
+KUNIT_ARRAY_PARAM(convert_xrgb8888, convert_xrgb8888_cases,
+		  convert_xrgb8888_case_desc);
 
-static void xrgb8888_to_rgb332_test(struct kunit *test)
+static void drm_test_fb_xrgb8888_to_gray8(struct kunit *test)
 {
-	const struct xrgb8888_to_rgb332_case *params = test->param_value;
+	const struct convert_xrgb8888_case *params = test->param_value;
+	const struct convert_to_gray8_result *result = &params->gray8_result;
 	size_t dst_size;
-	__u8 *dst = NULL;
+	__u8 *buf = NULL;
+	__u32 *xrgb8888 = NULL;
+	struct iosys_map dst, src;
 
 	struct drm_framebuffer fb = {
 		.format = drm_format_info(DRM_FORMAT_XRGB8888),
 		.pitches = { params->pitch, 0, 0 },
 	};
 
-	dst_size = conversion_buf_size(DRM_FORMAT_RGB332, params->dst_pitch,
+	dst_size = conversion_buf_size(DRM_FORMAT_R8, result->dst_pitch,
 				       &params->clip);
 	KUNIT_ASSERT_GT(test, dst_size, 0);
 
-	dst = kunit_kzalloc(test, dst_size, GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dst);
+	buf = kunit_kzalloc(test, dst_size, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
+	iosys_map_set_vaddr(&dst, buf);
+
+	xrgb8888 = le32buf_to_cpu(test, params->xrgb8888, TEST_BUF_SIZE);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xrgb8888);
+	iosys_map_set_vaddr(&src, xrgb8888);
+
+	drm_fb_xrgb8888_to_gray8(&dst, &result->dst_pitch, &src, &fb, &params->clip);
+	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected, dst_size), 0);
+}
+
+static void drm_test_fb_xrgb8888_to_rgb332(struct kunit *test)
+{
+	const struct convert_xrgb8888_case *params = test->param_value;
+	const struct convert_to_rgb332_result *result = &params->rgb332_result;
+	size_t dst_size;
+	__u8 *buf = NULL;
+	__u32 *xrgb8888 = NULL;
+	struct iosys_map dst, src;
+
+	struct drm_framebuffer fb = {
+		.format = drm_format_info(DRM_FORMAT_XRGB8888),
+		.pitches = { params->pitch, 0, 0 },
+	};
+
+	dst_size = conversion_buf_size(DRM_FORMAT_RGB332, result->dst_pitch,
+				       &params->clip);
+	KUNIT_ASSERT_GT(test, dst_size, 0);
+
+	buf = kunit_kzalloc(test, dst_size, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
+	iosys_map_set_vaddr(&dst, buf);
+
+	xrgb8888 = le32buf_to_cpu(test, params->xrgb8888, TEST_BUF_SIZE);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xrgb8888);
+	iosys_map_set_vaddr(&src, xrgb8888);
+
+	drm_fb_xrgb8888_to_rgb332(&dst, &result->dst_pitch, &src, &fb, &params->clip);
+	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected, dst_size), 0);
+}
+
+static void drm_test_fb_xrgb8888_to_rgb565(struct kunit *test)
+{
+	const struct convert_xrgb8888_case *params = test->param_value;
+	const struct convert_to_rgb565_result *result = &params->rgb565_result;
+	size_t dst_size;
+	__u16 *buf = NULL;
+	__u32 *xrgb8888 = NULL;
+	struct iosys_map dst, src;
+
+	struct drm_framebuffer fb = {
+		.format = drm_format_info(DRM_FORMAT_XRGB8888),
+		.pitches = { params->pitch, 0, 0 },
+	};
+
+	dst_size = conversion_buf_size(DRM_FORMAT_RGB565, result->dst_pitch,
+				       &params->clip);
+	KUNIT_ASSERT_GT(test, dst_size, 0);
+
+	buf = kunit_kzalloc(test, dst_size, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
+	iosys_map_set_vaddr(&dst, buf);
+
+	xrgb8888 = le32buf_to_cpu(test, params->xrgb8888, TEST_BUF_SIZE);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xrgb8888);
+	iosys_map_set_vaddr(&src, xrgb8888);
+
+	drm_fb_xrgb8888_to_rgb565(&dst, &result->dst_pitch, &src, &fb, &params->clip, false);
+	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected, dst_size), 0);
+
+	drm_fb_xrgb8888_to_rgb565(&dst, &result->dst_pitch, &src, &fb, &params->clip, true);
+	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected_swab, dst_size), 0);
+}
+
+static void drm_test_fb_xrgb8888_to_rgb888(struct kunit *test)
+{
+	const struct convert_xrgb8888_case *params = test->param_value;
+	const struct convert_to_rgb888_result *result = &params->rgb888_result;
+	size_t dst_size;
+	__u8 *buf = NULL;
+	__u32 *xrgb8888 = NULL;
+	struct iosys_map dst, src;
+
+	struct drm_framebuffer fb = {
+		.format = drm_format_info(DRM_FORMAT_XRGB8888),
+		.pitches = { params->pitch, 0, 0 },
+	};
+
+	dst_size = conversion_buf_size(DRM_FORMAT_RGB888, result->dst_pitch,
+				       &params->clip);
+	KUNIT_ASSERT_GT(test, dst_size, 0);
+
+	buf = kunit_kzalloc(test, dst_size, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
+	iosys_map_set_vaddr(&dst, buf);
+
+	xrgb8888 = le32buf_to_cpu(test, params->xrgb8888, TEST_BUF_SIZE);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xrgb8888);
+	iosys_map_set_vaddr(&src, xrgb8888);
+
+	drm_fb_xrgb8888_to_rgb888(&dst, &result->dst_pitch, &src, &fb, &params->clip);
+	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected, dst_size), 0);
+}
+
+static void drm_test_fb_xrgb8888_to_xrgb2101010(struct kunit *test)
+{
+	const struct convert_xrgb8888_case *params = test->param_value;
+	const struct convert_to_xrgb2101010_result *result = &params->xrgb2101010_result;
+	size_t dst_size;
+	__u32 *buf = NULL;
+	__u32 *xrgb8888 = NULL;
+	struct iosys_map dst, src;
+
+	struct drm_framebuffer fb = {
+		.format = drm_format_info(DRM_FORMAT_XRGB8888),
+		.pitches = { params->pitch, 0, 0 },
+	};
+
+	dst_size = conversion_buf_size(DRM_FORMAT_XRGB2101010,
+				       result->dst_pitch, &params->clip);
+	KUNIT_ASSERT_GT(test, dst_size, 0);
+
+	buf = kunit_kzalloc(test, dst_size, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, buf);
+	iosys_map_set_vaddr(&dst, buf);
+
+	xrgb8888 = le32buf_to_cpu(test, params->xrgb8888, TEST_BUF_SIZE);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, xrgb8888);
+	iosys_map_set_vaddr(&src, xrgb8888);
 
-	drm_fb_xrgb8888_to_rgb332(dst, params->dst_pitch, params->xrgb8888,
-				  &fb, &params->clip);
-	KUNIT_EXPECT_EQ(test, memcmp(dst, params->expected, dst_size), 0);
+	drm_fb_xrgb8888_to_xrgb2101010(&dst, &result->dst_pitch, &src, &fb, &params->clip);
+	buf = le32buf_to_cpu(test, buf, TEST_BUF_SIZE);
+	KUNIT_EXPECT_EQ(test, memcmp(buf, result->expected, dst_size), 0);
 }
 
 static struct kunit_case drm_format_helper_test_cases[] = {
-	KUNIT_CASE_PARAM(xrgb8888_to_rgb332_test,
-			 xrgb8888_to_rgb332_gen_params),
+	KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_gray8, convert_xrgb8888_gen_params),
+	KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_rgb332, convert_xrgb8888_gen_params),
+	KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_rgb565, convert_xrgb8888_gen_params),
+	KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_rgb888, convert_xrgb8888_gen_params),
+	KUNIT_CASE_PARAM(drm_test_fb_xrgb8888_to_xrgb2101010, convert_xrgb8888_gen_params),
 	{}
 };
 
diff --git a/drivers/gpu/drm/tests/drm_format_test.c b/drivers/gpu/drm/tests/drm_format_test.c
new file mode 100644
index 000000000000..ec6996ce819a
--- /dev/null
+++ b/drivers/gpu/drm/tests/drm_format_test.c
@@ -0,0 +1,359 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Test cases for the drm_format functions
+ *
+ * Copyright (c) 2022 Maíra Canal <mairacanal@riseup.net>
+ */
+
+#include <kunit/test.h>
+
+#include <drm/drm_fourcc.h>
+
+static void drm_test_format_block_width_invalid(struct kunit *test)
+{
+	const struct drm_format_info *info = NULL;
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, -1), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 1), 0);
+}
+
+static void drm_test_format_block_width_one_plane(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_XRGB4444);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 0), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 1), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, -1), 0);
+}
+
+static void drm_test_format_block_width_two_plane(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_NV12);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 0), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 1), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 2), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, -1), 0);
+}
+
+static void drm_test_format_block_width_three_plane(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_YUV422);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 0), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 1), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 2), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 3), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, -1), 0);
+}
+
+static void drm_test_format_block_width_tiled(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_X0L0);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 0), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, 1), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_width(info, -1), 0);
+}
+
+static void drm_test_format_block_height_invalid(struct kunit *test)
+{
+	const struct drm_format_info *info = NULL;
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, -1), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 1), 0);
+}
+
+static void drm_test_format_block_height_one_plane(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_XRGB4444);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 0), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, -1), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 1), 0);
+}
+
+static void drm_test_format_block_height_two_plane(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_NV12);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 0), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 1), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 2), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, -1), 0);
+}
+
+static void drm_test_format_block_height_three_plane(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_YUV422);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 0), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 1), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 2), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 3), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, -1), 0);
+}
+
+static void drm_test_format_block_height_tiled(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_X0L0);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 0), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, 1), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_block_height(info, -1), 0);
+}
+
+static void drm_test_format_min_pitch_invalid(struct kunit *test)
+{
+	const struct drm_format_info *info = NULL;
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, -1, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 0), 0);
+}
+
+static void drm_test_format_min_pitch_one_plane_8bpp(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_RGB332);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, -1, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 0), 0);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 2), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 640), 640);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1024), 1024);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1920), 1920);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 4096), 4096);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 671), 671);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, UINT_MAX),
+			(uint64_t)UINT_MAX);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, (UINT_MAX - 1)),
+			(uint64_t)(UINT_MAX - 1));
+}
+
+static void drm_test_format_min_pitch_one_plane_16bpp(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_XRGB4444);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, -1, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 0), 0);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 2), 4);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 640), 1280);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1024), 2048);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1920), 3840);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 4096), 8192);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 671), 1342);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, UINT_MAX),
+			(uint64_t)UINT_MAX * 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, (UINT_MAX - 1)),
+			(uint64_t)(UINT_MAX - 1) * 2);
+}
+
+static void drm_test_format_min_pitch_one_plane_24bpp(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_RGB888);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, -1, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 0), 0);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1), 3);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 2), 6);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 640), 1920);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1024), 3072);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1920), 5760);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 4096), 12288);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 671), 2013);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, UINT_MAX),
+			(uint64_t)UINT_MAX * 3);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, UINT_MAX - 1),
+			(uint64_t)(UINT_MAX - 1) * 3);
+}
+
+static void drm_test_format_min_pitch_one_plane_32bpp(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_ABGR8888);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, -1, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 0), 0);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1), 4);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 2), 8);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 640), 2560);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1024), 4096);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1920), 7680);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 4096), 16384);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 671), 2684);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, UINT_MAX),
+			(uint64_t)UINT_MAX * 4);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, UINT_MAX - 1),
+			(uint64_t)(UINT_MAX - 1) * 4);
+}
+
+static void drm_test_format_min_pitch_two_plane(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_NV12);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, -1, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, 0), 0);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 1), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 2), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 1), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 640), 640);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 320), 640);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1024), 1024);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 512), 1024);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1920), 1920);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 960), 1920);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 4096), 4096);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 2048), 4096);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 671), 671);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 336), 672);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, UINT_MAX),
+			(uint64_t)UINT_MAX);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, UINT_MAX / 2 + 1),
+			(uint64_t)UINT_MAX + 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, (UINT_MAX - 1)),
+			(uint64_t)(UINT_MAX - 1));
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, (UINT_MAX - 1) /  2),
+			(uint64_t)(UINT_MAX - 1));
+}
+
+static void drm_test_format_min_pitch_three_plane_8bpp(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_YUV422);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, -1, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 3, 0), 0);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 1), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, 1), 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 2), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 2), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, 2), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 640), 640);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 320), 320);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, 320), 320);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1024), 1024);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 512), 512);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, 512), 512);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1920), 1920);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 960), 960);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, 960), 960);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 4096), 4096);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 2048), 2048);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, 2048), 2048);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 671), 671);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 336), 336);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, 336), 336);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, UINT_MAX),
+			(uint64_t)UINT_MAX);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, UINT_MAX / 2 + 1),
+			(uint64_t)UINT_MAX / 2 + 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, UINT_MAX / 2 + 1),
+			(uint64_t)UINT_MAX / 2 + 1);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, (UINT_MAX - 1) / 2),
+			(uint64_t)(UINT_MAX - 1) / 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, (UINT_MAX - 1) / 2),
+			(uint64_t)(UINT_MAX - 1) / 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 2, (UINT_MAX - 1) / 2),
+			(uint64_t)(UINT_MAX - 1) / 2);
+}
+
+static void drm_test_format_min_pitch_tiled(struct kunit *test)
+{
+	const struct drm_format_info *info = drm_format_info(DRM_FORMAT_X0L2);
+
+	KUNIT_ASSERT_NOT_NULL(test, info);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, -1, 0), 0);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 1, 0), 0);
+
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1), 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 2), 4);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 640), 1280);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1024), 2048);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 1920), 3840);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 4096), 8192);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, 671), 1342);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, UINT_MAX),
+			(uint64_t)UINT_MAX * 2);
+	KUNIT_EXPECT_EQ(test, drm_format_info_min_pitch(info, 0, UINT_MAX - 1),
+			(uint64_t)(UINT_MAX - 1) * 2);
+}
+
+static struct kunit_case drm_format_tests[] = {
+	KUNIT_CASE(drm_test_format_block_width_invalid),
+	KUNIT_CASE(drm_test_format_block_width_one_plane),
+	KUNIT_CASE(drm_test_format_block_width_two_plane),
+	KUNIT_CASE(drm_test_format_block_width_three_plane),
+	KUNIT_CASE(drm_test_format_block_width_tiled),
+	KUNIT_CASE(drm_test_format_block_height_invalid),
+	KUNIT_CASE(drm_test_format_block_height_one_plane),
+	KUNIT_CASE(drm_test_format_block_height_two_plane),
+	KUNIT_CASE(drm_test_format_block_height_three_plane),
+	KUNIT_CASE(drm_test_format_block_height_tiled),
+	KUNIT_CASE(drm_test_format_min_pitch_invalid),
+	KUNIT_CASE(drm_test_format_min_pitch_one_plane_8bpp),
+	KUNIT_CASE(drm_test_format_min_pitch_one_plane_16bpp),
+	KUNIT_CASE(drm_test_format_min_pitch_one_plane_24bpp),
+	KUNIT_CASE(drm_test_format_min_pitch_one_plane_32bpp),
+	KUNIT_CASE(drm_test_format_min_pitch_two_plane),
+	KUNIT_CASE(drm_test_format_min_pitch_three_plane_8bpp),
+	KUNIT_CASE(drm_test_format_min_pitch_tiled),
+	{}
+};
+
+static struct kunit_suite drm_format_test_suite = {
+	.name = "drm_format",
+	.test_cases = drm_format_tests,
+};
+
+kunit_test_suite(drm_format_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/selftests/test-drm_framebuffer.c b/drivers/gpu/drm/tests/drm_framebuffer_test.c
index f6d66285c5fc..df235b7fdaa5 100644
--- a/drivers/gpu/drm/selftests/test-drm_framebuffer.c
+++ b/drivers/gpu/drm/tests/drm_framebuffer_test.c
@@ -1,9 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Test cases for the drm_framebuffer functions
+ *
+ * Copyright (c) 2022 Maíra Canal <mairacanal@riseup.net>
  */
 
-#include <linux/kernel.h>
+#include <kunit/test.h>
 
 #include <drm/drm_device.h>
 #include <drm/drm_mode.h>
@@ -12,8 +14,6 @@
 
 #include "../drm_crtc_internal.h"
 
-#include "test-drm_modeset_common.h"
-
 #define MIN_WIDTH 4
 #define MAX_WIDTH 4096
 #define MIN_HEIGHT 4
@@ -25,7 +25,7 @@ struct drm_framebuffer_test {
 	const char *name;
 };
 
-static struct drm_framebuffer_test createbuffer_tests[] = {
+static const struct drm_framebuffer_test drm_framebuffer_create_cases[] = {
 { .buffer_created = 1, .name = "ABGR8888 normal sizes",
 	.cmd = { .width = 600, .height = 600, .pixel_format = DRM_FORMAT_ABGR8888,
 		 .handles = { 1, 0, 0 }, .pitches = { 4 * 600, 0, 0 },
@@ -73,12 +73,14 @@ static struct drm_framebuffer_test createbuffer_tests[] = {
 },
 { .buffer_created = 0, .name = "ABGR8888 Out of bound height * pitch combination",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_ABGR8888,
-		 .handles = { 1, 0, 0 }, .offsets = { UINT_MAX - 1, 0, 0 }, .pitches = { 4 * MAX_WIDTH, 0, 0 },
+		 .handles = { 1, 0, 0 }, .offsets = { UINT_MAX - 1, 0, 0 },
+		 .pitches = { 4 * MAX_WIDTH, 0, 0 },
 	}
 },
 { .buffer_created = 1, .name = "ABGR8888 Large buffer offset",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_ABGR8888,
-		 .handles = { 1, 0, 0 }, .offsets = { UINT_MAX / 2, 0, 0 }, .pitches = { 4 * MAX_WIDTH, 0, 0 },
+		 .handles = { 1, 0, 0 }, .offsets = { UINT_MAX / 2, 0, 0 },
+		 .pitches = { 4 * MAX_WIDTH, 0, 0 },
 	}
 },
 { .buffer_created = 1, .name = "ABGR8888 Set DRM_MODE_FB_MODIFIERS without modifiers",
@@ -89,11 +91,13 @@ static struct drm_framebuffer_test createbuffer_tests[] = {
 },
 { .buffer_created = 1, .name = "ABGR8888 Valid buffer modifier",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_ABGR8888,
-		 .handles = { 1, 0, 0 }, .offsets = { UINT_MAX / 2, 0, 0 }, .pitches = { 4 * MAX_WIDTH, 0, 0 },
-		 .flags = DRM_MODE_FB_MODIFIERS, .modifier = { AFBC_FORMAT_MOD_YTR, 0, 0 },
+		 .handles = { 1, 0, 0 }, .offsets = { UINT_MAX / 2, 0, 0 },
+		 .pitches = { 4 * MAX_WIDTH, 0, 0 }, .flags = DRM_MODE_FB_MODIFIERS,
+		 .modifier = { AFBC_FORMAT_MOD_YTR, 0, 0 },
 	}
 },
-{ .buffer_created = 0, .name = "ABGR8888 Invalid buffer modifier(DRM_FORMAT_MOD_SAMSUNG_64_32_TILE)",
+{ .buffer_created = 0,
+	.name = "ABGR8888 Invalid buffer modifier(DRM_FORMAT_MOD_SAMSUNG_64_32_TILE)",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_ABGR8888,
 		 .handles = { 1, 0, 0 }, .offsets = { UINT_MAX / 2, 0, 0 },
 		 .pitches = { 4 * MAX_WIDTH, 0, 0 }, .flags = DRM_MODE_FB_MODIFIERS,
@@ -143,7 +147,8 @@ static struct drm_framebuffer_test createbuffer_tests[] = {
 { .buffer_created = 1, .name = "NV12 with DRM_FORMAT_MOD_SAMSUNG_64_32_TILE",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_NV12,
 		 .handles = { 1, 1, 0 }, .flags = DRM_MODE_FB_MODIFIERS,
-		 .modifier = { DRM_FORMAT_MOD_SAMSUNG_64_32_TILE, DRM_FORMAT_MOD_SAMSUNG_64_32_TILE, 0 },
+		 .modifier = { DRM_FORMAT_MOD_SAMSUNG_64_32_TILE,
+			 DRM_FORMAT_MOD_SAMSUNG_64_32_TILE, 0 },
 		 .pitches = { MAX_WIDTH, MAX_WIDTH, 0 },
 	}
 },
@@ -164,7 +169,8 @@ static struct drm_framebuffer_test createbuffer_tests[] = {
 },
 { .buffer_created = 0, .name = "NV12 Handle for inexistent plane",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_NV12,
-		 .handles = { 1, 1, 1 }, .flags = DRM_MODE_FB_MODIFIERS, .pitches = { MAX_WIDTH, MAX_WIDTH, 0 },
+		 .handles = { 1, 1, 1 }, .flags = DRM_MODE_FB_MODIFIERS,
+		 .pitches = { MAX_WIDTH, MAX_WIDTH, 0 },
 	}
 },
 { .buffer_created = 1, .name = "NV12 Handle for inexistent plane without DRM_MODE_FB_MODIFIERS",
@@ -203,24 +209,29 @@ static struct drm_framebuffer_test createbuffer_tests[] = {
 },
 { .buffer_created = 1, .name = "YVU420 Different buffer offsets/pitches",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_YVU420,
-		 .handles = { 1, 1, 1 }, .offsets = { MAX_WIDTH, MAX_WIDTH  + MAX_WIDTH * MAX_HEIGHT,
-						      MAX_WIDTH  + 2 * MAX_WIDTH * MAX_HEIGHT },
-		 .pitches = { MAX_WIDTH, DIV_ROUND_UP(MAX_WIDTH, 2) + 1, DIV_ROUND_UP(MAX_WIDTH, 2) + 7 },
+		 .handles = { 1, 1, 1 }, .offsets = { MAX_WIDTH, MAX_WIDTH  +
+			 MAX_WIDTH * MAX_HEIGHT, MAX_WIDTH  + 2 * MAX_WIDTH * MAX_HEIGHT },
+		 .pitches = { MAX_WIDTH, DIV_ROUND_UP(MAX_WIDTH, 2) + 1,
+			 DIV_ROUND_UP(MAX_WIDTH, 2) + 7 },
 	}
 },
-{ .buffer_created = 0, .name = "YVU420 Modifier set just for plane 0, without DRM_MODE_FB_MODIFIERS",
+{ .buffer_created = 0,
+	.name = "YVU420 Modifier set just for plane 0, without DRM_MODE_FB_MODIFIERS",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_YVU420,
 		 .handles = { 1, 1, 1 }, .modifier = { AFBC_FORMAT_MOD_SPARSE, 0, 0 },
 		 .pitches = { MAX_WIDTH, DIV_ROUND_UP(MAX_WIDTH, 2), DIV_ROUND_UP(MAX_WIDTH, 2) },
 	}
 },
-{ .buffer_created = 0, .name = "YVU420 Modifier set just for planes 0, 1, without DRM_MODE_FB_MODIFIERS",
+{ .buffer_created = 0,
+	.name = "YVU420 Modifier set just for planes 0, 1, without DRM_MODE_FB_MODIFIERS",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_YVU420,
-		 .handles = { 1, 1, 1 }, .modifier = { AFBC_FORMAT_MOD_SPARSE, AFBC_FORMAT_MOD_SPARSE, 0 },
+		 .handles = { 1, 1, 1 },
+		 .modifier = { AFBC_FORMAT_MOD_SPARSE, AFBC_FORMAT_MOD_SPARSE, 0 },
 		 .pitches = { MAX_WIDTH, DIV_ROUND_UP(MAX_WIDTH, 2), DIV_ROUND_UP(MAX_WIDTH, 2) },
 	}
 },
-{ .buffer_created = 0, .name = "YVU420 Modifier set just for plane 0, 1, with DRM_MODE_FB_MODIFIERS",
+{ .buffer_created = 0,
+	.name = "YVU420 Modifier set just for plane 0, 1, with DRM_MODE_FB_MODIFIERS",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_YVU420,
 		 .handles = { 1, 1, 1 }, .flags = DRM_MODE_FB_MODIFIERS,
 		 .modifier = { AFBC_FORMAT_MOD_SPARSE, AFBC_FORMAT_MOD_SPARSE, 0 },
@@ -230,7 +241,8 @@ static struct drm_framebuffer_test createbuffer_tests[] = {
 { .buffer_created = 1, .name = "YVU420 Valid modifier",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_YVU420,
 		 .handles = { 1, 1, 1 }, .flags = DRM_MODE_FB_MODIFIERS,
-		 .modifier = { AFBC_FORMAT_MOD_SPARSE, AFBC_FORMAT_MOD_SPARSE, AFBC_FORMAT_MOD_SPARSE },
+		 .modifier = { AFBC_FORMAT_MOD_SPARSE, AFBC_FORMAT_MOD_SPARSE,
+			 AFBC_FORMAT_MOD_SPARSE },
 		 .pitches = { MAX_WIDTH, DIV_ROUND_UP(MAX_WIDTH, 2), DIV_ROUND_UP(MAX_WIDTH, 2) },
 	}
 },
@@ -245,8 +257,8 @@ static struct drm_framebuffer_test createbuffer_tests[] = {
 { .buffer_created = 0, .name = "YVU420 Modifier for inexistent plane",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_YVU420,
 		 .handles = { 1, 1, 1 }, .flags = DRM_MODE_FB_MODIFIERS,
-		 .modifier = { AFBC_FORMAT_MOD_SPARSE, AFBC_FORMAT_MOD_SPARSE, AFBC_FORMAT_MOD_SPARSE,
-			       AFBC_FORMAT_MOD_SPARSE },
+		 .modifier = { AFBC_FORMAT_MOD_SPARSE, AFBC_FORMAT_MOD_SPARSE,
+			 AFBC_FORMAT_MOD_SPARSE, AFBC_FORMAT_MOD_SPARSE },
 		 .pitches = { MAX_WIDTH, DIV_ROUND_UP(MAX_WIDTH, 2), DIV_ROUND_UP(MAX_WIDTH, 2) },
 	}
 },
@@ -276,7 +288,8 @@ static struct drm_framebuffer_test createbuffer_tests[] = {
 		 .pitches = { 2 * MAX_WIDTH + 1, 0, 0 }
 	}
 },
-{ .buffer_created = 1, .name = "X0L2 Offset for inexistent plane, without DRM_MODE_FB_MODIFIERS set",
+{ .buffer_created = 1,
+	.name = "X0L2 Offset for inexistent plane, without DRM_MODE_FB_MODIFIERS set",
 	.cmd = { .width = MAX_WIDTH, .height = MAX_HEIGHT, .pixel_format = DRM_FORMAT_X0L2,
 		 .handles = { 1, 0, 0 }, .offsets = { 0, 0, 3 },
 		 .pitches = { 2 * MAX_WIDTH + 1, 0, 0 }
@@ -317,34 +330,53 @@ static struct drm_mode_config_funcs mock_config_funcs = {
 	.fb_create = fb_create_mock,
 };
 
-static struct drm_device mock_drm_device = {
-	.mode_config = {
-		.min_width = MIN_WIDTH,
-		.max_width = MAX_WIDTH,
-		.min_height = MIN_HEIGHT,
-		.max_height = MAX_HEIGHT,
-		.funcs = &mock_config_funcs,
-	},
-};
+static int drm_framebuffer_test_init(struct kunit *test)
+{
+	struct drm_device *mock;
+
+	mock = kunit_kzalloc(test, sizeof(*mock), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, mock);
+
+	mock->mode_config.min_width = MIN_WIDTH;
+	mock->mode_config.max_width = MAX_WIDTH;
+	mock->mode_config.min_height = MIN_HEIGHT;
+	mock->mode_config.max_height = MAX_HEIGHT;
+	mock->mode_config.funcs = &mock_config_funcs;
 
-static int execute_drm_mode_fb_cmd2(struct drm_mode_fb_cmd2 *r)
+	test->priv = mock;
+	return 0;
+}
+
+static void drm_test_framebuffer_create(struct kunit *test)
 {
+	const struct drm_framebuffer_test *params = test->param_value;
+	struct drm_device *mock = test->priv;
 	int buffer_created = 0;
 
-	mock_drm_device.dev_private = &buffer_created;
-	drm_internal_framebuffer_create(&mock_drm_device, r, NULL);
-	return buffer_created;
+	mock->dev_private = &buffer_created;
+	drm_internal_framebuffer_create(mock, &params->cmd, NULL);
+	KUNIT_EXPECT_EQ(test, params->buffer_created, buffer_created);
 }
 
-int igt_check_drm_framebuffer_create(void *ignored)
+static void drm_framebuffer_test_to_desc(const struct drm_framebuffer_test *t, char *desc)
 {
-	int i = 0;
+	strcpy(desc, t->name);
+}
 
-	for (i = 0; i < ARRAY_SIZE(createbuffer_tests); i++) {
-		FAIL(createbuffer_tests[i].buffer_created !=
-				execute_drm_mode_fb_cmd2(&createbuffer_tests[i].cmd),
-		     "Test %d: \"%s\" failed\n", i, createbuffer_tests[i].name);
-	}
+KUNIT_ARRAY_PARAM(drm_framebuffer_create, drm_framebuffer_create_cases,
+		  drm_framebuffer_test_to_desc);
 
-	return 0;
-}
+static struct kunit_case drm_framebuffer_tests[] = {
+	KUNIT_CASE_PARAM(drm_test_framebuffer_create, drm_framebuffer_create_gen_params),
+	{ }
+};
+
+static struct kunit_suite drm_framebuffer_test_suite = {
+	.name = "drm_framebuffer",
+	.init = drm_framebuffer_test_init,
+	.test_cases = drm_framebuffer_tests,
+};
+
+kunit_test_suite(drm_framebuffer_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/selftests/test-drm_mm.c b/drivers/gpu/drm/tests/drm_mm_test.c
index b768b53c4aee..659d1af4dca7 100644
--- a/drivers/gpu/drm/selftests/test-drm_mm.c
+++ b/drivers/gpu/drm/tests/drm_mm_test.c
@@ -1,11 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Test cases for the drm_mm range manager
+ *
+ * Copyright (c) 2022 Arthur Grillo <arthur.grillo@usp.br>
  */
 
-#define pr_fmt(fmt) "drm_mm: " fmt
+#include <kunit/test.h>
 
-#include <linux/module.h>
 #include <linux/prime_numbers.h>
 #include <linux/slab.h>
 #include <linux/random.h>
@@ -16,9 +17,6 @@
 
 #include "../lib/drm_random.h"
 
-#define TESTS "drm_mm_selftests.h"
-#include "drm_selftest.h"
-
 static unsigned int random_seed;
 static unsigned int max_iterations = 8192;
 static unsigned int max_prime = 128;
@@ -45,13 +43,7 @@ static const struct insert_mode {
 	{}
 };
 
-static int igt_sanitycheck(void *ignored)
-{
-	pr_info("%s - ok!\n", __func__);
-	return 0;
-}
-
-static bool assert_no_holes(const struct drm_mm *mm)
+static bool assert_no_holes(struct kunit *test, const struct drm_mm *mm)
 {
 	struct drm_mm_node *hole;
 	u64 hole_start, __always_unused hole_end;
@@ -61,13 +53,14 @@ static bool assert_no_holes(const struct drm_mm *mm)
 	drm_mm_for_each_hole(hole, mm, hole_start, hole_end)
 		count++;
 	if (count) {
-		pr_err("Expected to find no holes (after reserve), found %lu instead\n", count);
+		KUNIT_FAIL(test,
+			   "Expected to find no holes (after reserve), found %lu instead\n", count);
 		return false;
 	}
 
 	drm_mm_for_each_node(hole, mm) {
 		if (drm_mm_hole_follows(hole)) {
-			pr_err("Hole follows node, expected none!\n");
+			KUNIT_FAIL(test, "Hole follows node, expected none!\n");
 			return false;
 		}
 	}
@@ -75,7 +68,7 @@ static bool assert_no_holes(const struct drm_mm *mm)
 	return true;
 }
 
-static bool assert_one_hole(const struct drm_mm *mm, u64 start, u64 end)
+static bool assert_one_hole(struct kunit *test, const struct drm_mm *mm, u64 start, u64 end)
 {
 	struct drm_mm_node *hole;
 	u64 hole_start, hole_end;
@@ -89,62 +82,62 @@ static bool assert_one_hole(const struct drm_mm *mm, u64 start, u64 end)
 	drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
 		if (start != hole_start || end != hole_end) {
 			if (ok)
-				pr_err("empty mm has incorrect hole, found (%llx, %llx), expect (%llx, %llx)\n",
-				       hole_start, hole_end,
-				       start, end);
+				KUNIT_FAIL(test,
+					   "empty mm has incorrect hole, found (%llx, %llx), expect (%llx, %llx)\n",
+					   hole_start, hole_end, start, end);
 			ok = false;
 		}
 		count++;
 	}
 	if (count != 1) {
-		pr_err("Expected to find one hole, found %lu instead\n", count);
+		KUNIT_FAIL(test, "Expected to find one hole, found %lu instead\n", count);
 		ok = false;
 	}
 
 	return ok;
 }
 
-static bool assert_continuous(const struct drm_mm *mm, u64 size)
+static bool assert_continuous(struct kunit *test, const struct drm_mm *mm, u64 size)
 {
 	struct drm_mm_node *node, *check, *found;
 	unsigned long n;
 	u64 addr;
 
-	if (!assert_no_holes(mm))
+	if (!assert_no_holes(test, mm))
 		return false;
 
 	n = 0;
 	addr = 0;
 	drm_mm_for_each_node(node, mm) {
 		if (node->start != addr) {
-			pr_err("node[%ld] list out of order, expected %llx found %llx\n",
-			       n, addr, node->start);
+			KUNIT_FAIL(test, "node[%ld] list out of order, expected %llx found %llx\n",
+				   n, addr, node->start);
 			return false;
 		}
 
 		if (node->size != size) {
-			pr_err("node[%ld].size incorrect, expected %llx, found %llx\n",
-			       n, size, node->size);
+			KUNIT_FAIL(test, "node[%ld].size incorrect, expected %llx, found %llx\n",
+				   n, size, node->size);
 			return false;
 		}
 
 		if (drm_mm_hole_follows(node)) {
-			pr_err("node[%ld] is followed by a hole!\n", n);
+			KUNIT_FAIL(test, "node[%ld] is followed by a hole!\n", n);
 			return false;
 		}
 
 		found = NULL;
 		drm_mm_for_each_node_in_range(check, mm, addr, addr + size) {
 			if (node != check) {
-				pr_err("lookup return wrong node, expected start %llx, found %llx\n",
-				       node->start, check->start);
+				KUNIT_FAIL(test,
+					   "lookup return wrong node, expected start %llx, found %llx\n",
+					   node->start, check->start);
 				return false;
 			}
 			found = check;
 		}
 		if (!found) {
-			pr_err("lookup failed for node %llx + %llx\n",
-			       addr, size);
+			KUNIT_FAIL(test, "lookup failed for node %llx + %llx\n", addr, size);
 			return false;
 		}
 
@@ -166,107 +159,96 @@ static u64 misalignment(struct drm_mm_node *node, u64 alignment)
 	return rem;
 }
 
-static bool assert_node(struct drm_mm_node *node, struct drm_mm *mm,
+static bool assert_node(struct kunit *test, struct drm_mm_node *node, struct drm_mm *mm,
 			u64 size, u64 alignment, unsigned long color)
 {
 	bool ok = true;
 
 	if (!drm_mm_node_allocated(node) || node->mm != mm) {
-		pr_err("node not allocated\n");
+		KUNIT_FAIL(test, "node not allocated\n");
 		ok = false;
 	}
 
 	if (node->size != size) {
-		pr_err("node has wrong size, found %llu, expected %llu\n",
-		       node->size, size);
+		KUNIT_FAIL(test, "node has wrong size, found %llu, expected %llu\n",
+			   node->size, size);
 		ok = false;
 	}
 
 	if (misalignment(node, alignment)) {
-		pr_err("node is misaligned, start %llx rem %llu, expected alignment %llu\n",
-		       node->start, misalignment(node, alignment), alignment);
+		KUNIT_FAIL(test,
+			   "node is misaligned, start %llx rem %llu, expected alignment %llu\n",
+			   node->start, misalignment(node, alignment), alignment);
 		ok = false;
 	}
 
 	if (node->color != color) {
-		pr_err("node has wrong color, found %lu, expected %lu\n",
-		       node->color, color);
+		KUNIT_FAIL(test, "node has wrong color, found %lu, expected %lu\n",
+			   node->color, color);
 		ok = false;
 	}
 
 	return ok;
 }
 
-#define show_mm(mm) do { \
-	struct drm_printer __p = drm_debug_printer(__func__); \
-	drm_mm_print((mm), &__p); } while (0)
-
-static int igt_init(void *ignored)
+static void drm_test_mm_init(struct kunit *test)
 {
 	const unsigned int size = 4096;
 	struct drm_mm mm;
 	struct drm_mm_node tmp;
-	int ret = -EINVAL;
 
 	/* Start with some simple checks on initialising the struct drm_mm */
 	memset(&mm, 0, sizeof(mm));
-	if (drm_mm_initialized(&mm)) {
-		pr_err("zeroed mm claims to be initialized\n");
-		return ret;
-	}
+	KUNIT_ASSERT_FALSE_MSG(test, drm_mm_initialized(&mm),
+			       "zeroed mm claims to be initialized\n");
 
 	memset(&mm, 0xff, sizeof(mm));
 	drm_mm_init(&mm, 0, size);
 	if (!drm_mm_initialized(&mm)) {
-		pr_err("mm claims not to be initialized\n");
+		KUNIT_FAIL(test, "mm claims not to be initialized\n");
 		goto out;
 	}
 
 	if (!drm_mm_clean(&mm)) {
-		pr_err("mm not empty on creation\n");
+		KUNIT_FAIL(test, "mm not empty on creation\n");
 		goto out;
 	}
 
 	/* After creation, it should all be one massive hole */
-	if (!assert_one_hole(&mm, 0, size)) {
-		ret = -EINVAL;
+	if (!assert_one_hole(test, &mm, 0, size)) {
+		KUNIT_FAIL(test, "");
 		goto out;
 	}
 
 	memset(&tmp, 0, sizeof(tmp));
 	tmp.start = 0;
 	tmp.size = size;
-	ret = drm_mm_reserve_node(&mm, &tmp);
-	if (ret) {
-		pr_err("failed to reserve whole drm_mm\n");
+	if (drm_mm_reserve_node(&mm, &tmp)) {
+		KUNIT_FAIL(test, "failed to reserve whole drm_mm\n");
 		goto out;
 	}
 
 	/* After filling the range entirely, there should be no holes */
-	if (!assert_no_holes(&mm)) {
-		ret = -EINVAL;
+	if (!assert_no_holes(test, &mm)) {
+		KUNIT_FAIL(test, "");
 		goto out;
 	}
 
 	/* And then after emptying it again, the massive hole should be back */
 	drm_mm_remove_node(&tmp);
-	if (!assert_one_hole(&mm, 0, size)) {
-		ret = -EINVAL;
+	if (!assert_one_hole(test, &mm, 0, size)) {
+		KUNIT_FAIL(test, "");
 		goto out;
 	}
 
 out:
-	if (ret)
-		show_mm(&mm);
 	drm_mm_takedown(&mm);
-	return ret;
 }
 
-static int igt_debug(void *ignored)
+static void drm_test_mm_debug(struct kunit *test)
 {
 	struct drm_mm mm;
 	struct drm_mm_node nodes[2];
-	int ret;
 
 	/* Create a small drm_mm with a couple of nodes and a few holes, and
 	 * check that the debug iterator doesn't explode over a trivial drm_mm.
@@ -277,24 +259,15 @@ static int igt_debug(void *ignored)
 	memset(nodes, 0, sizeof(nodes));
 	nodes[0].start = 512;
 	nodes[0].size = 1024;
-	ret = drm_mm_reserve_node(&mm, &nodes[0]);
-	if (ret) {
-		pr_err("failed to reserve node[0] {start=%lld, size=%lld)\n",
-		       nodes[0].start, nodes[0].size);
-		return ret;
-	}
+	KUNIT_ASSERT_FALSE_MSG(test, drm_mm_reserve_node(&mm, &nodes[0]),
+			       "failed to reserve node[0] {start=%lld, size=%lld)\n",
+			       nodes[0].start, nodes[0].size);
 
 	nodes[1].size = 1024;
 	nodes[1].start = 4096 - 512 - nodes[1].size;
-	ret = drm_mm_reserve_node(&mm, &nodes[1]);
-	if (ret) {
-		pr_err("failed to reserve node[1] {start=%lld, size=%lld)\n",
-		       nodes[1].start, nodes[1].size);
-		return ret;
-	}
-
-	show_mm(&mm);
-	return 0;
+	KUNIT_ASSERT_FALSE_MSG(test, drm_mm_reserve_node(&mm, &nodes[1]),
+			       "failed to reserve node[0] {start=%lld, size=%lld)\n",
+			       nodes[0].start, nodes[0].size);
 }
 
 static struct drm_mm_node *set_node(struct drm_mm_node *node,
@@ -305,7 +278,7 @@ static struct drm_mm_node *set_node(struct drm_mm_node *node,
 	return node;
 }
 
-static bool expect_reserve_fail(struct drm_mm *mm, struct drm_mm_node *node)
+static bool expect_reserve_fail(struct kunit *test, struct drm_mm *mm, struct drm_mm_node *node)
 {
 	int err;
 
@@ -314,17 +287,18 @@ static bool expect_reserve_fail(struct drm_mm *mm, struct drm_mm_node *node)
 		return true;
 
 	if (!err) {
-		pr_err("impossible reserve succeeded, node %llu + %llu\n",
-		       node->start, node->size);
+		KUNIT_FAIL(test, "impossible reserve succeeded, node %llu + %llu\n",
+			   node->start, node->size);
 		drm_mm_remove_node(node);
 	} else {
-		pr_err("impossible reserve failed with wrong error %d [expected %d], node %llu + %llu\n",
+		KUNIT_FAIL(test,
+			   "impossible reserve failed with wrong error %d [expected %d], node %llu + %llu\n",
 		       err, -ENOSPC, node->start, node->size);
 	}
 	return false;
 }
 
-static bool check_reserve_boundaries(struct drm_mm *mm,
+static bool check_reserve_boundaries(struct kunit *test, struct drm_mm *mm,
 				     unsigned int count,
 				     u64 size)
 {
@@ -339,29 +313,27 @@ static bool check_reserve_boundaries(struct drm_mm *mm,
 		B(size * count, 0),
 		B(-size, size),
 		B(-size, -size),
-		B(-size, 2*size),
+		B(-size, 2 * size),
 		B(0, -size),
 		B(size, -size),
-		B(count*size, size),
-		B(count*size, -size),
-		B(count*size, count*size),
-		B(count*size, -count*size),
-		B(count*size, -(count+1)*size),
-		B((count+1)*size, size),
-		B((count+1)*size, -size),
-		B((count+1)*size, -2*size),
+		B(count * size, size),
+		B(count * size, -size),
+		B(count * size, count * size),
+		B(count * size, -count * size),
+		B(count * size, -(count + 1) * size),
+		B((count + 1) * size, size),
+		B((count + 1) * size, -size),
+		B((count + 1) * size, -2 * size),
 #undef B
 	};
 	struct drm_mm_node tmp = {};
 	int n;
 
 	for (n = 0; n < ARRAY_SIZE(boundaries); n++) {
-		if (!expect_reserve_fail(mm,
-					 set_node(&tmp,
-						  boundaries[n].start,
-						  boundaries[n].size))) {
-			pr_err("boundary[%d:%s] failed, count=%u, size=%lld\n",
-			       n, boundaries[n].name, count, size);
+		if (!expect_reserve_fail(test, mm, set_node(&tmp, boundaries[n].start,
+							    boundaries[n].size))) {
+			KUNIT_FAIL(test, "boundary[%d:%s] failed, count=%u, size=%lld\n",
+				   n, boundaries[n].name, count, size);
 			return false;
 		}
 	}
@@ -369,7 +341,7 @@ static bool check_reserve_boundaries(struct drm_mm *mm,
 	return true;
 }
 
-static int __igt_reserve(unsigned int count, u64 size)
+static int __drm_test_mm_reserve(struct kunit *test, unsigned int count, u64 size)
 {
 	DRM_RND_STATE(prng, random_seed);
 	struct drm_mm mm;
@@ -392,13 +364,12 @@ static int __igt_reserve(unsigned int count, u64 size)
 		goto err;
 
 	nodes = vzalloc(array_size(count, sizeof(*nodes)));
-	if (!nodes)
-		goto err_order;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	ret = -EINVAL;
 	drm_mm_init(&mm, 0, count * size);
 
-	if (!check_reserve_boundaries(&mm, count, size))
+	if (!check_reserve_boundaries(test, &mm, count, size))
 		goto out;
 
 	for (n = 0; n < count; n++) {
@@ -407,57 +378,53 @@ static int __igt_reserve(unsigned int count, u64 size)
 
 		err = drm_mm_reserve_node(&mm, &nodes[n]);
 		if (err) {
-			pr_err("reserve failed, step %d, start %llu\n",
-			       n, nodes[n].start);
+			KUNIT_FAIL(test, "reserve failed, step %d, start %llu\n",
+				   n, nodes[n].start);
 			ret = err;
 			goto out;
 		}
 
 		if (!drm_mm_node_allocated(&nodes[n])) {
-			pr_err("reserved node not allocated! step %d, start %llu\n",
-			       n, nodes[n].start);
+			KUNIT_FAIL(test, "reserved node not allocated! step %d, start %llu\n",
+				   n, nodes[n].start);
 			goto out;
 		}
 
-		if (!expect_reserve_fail(&mm, &nodes[n]))
+		if (!expect_reserve_fail(test, &mm, &nodes[n]))
 			goto out;
 	}
 
 	/* After random insertion the nodes should be in order */
-	if (!assert_continuous(&mm, size))
+	if (!assert_continuous(test, &mm, size))
 		goto out;
 
 	/* Repeated use should then fail */
 	drm_random_reorder(order, count, &prng);
 	for (n = 0; n < count; n++) {
-		if (!expect_reserve_fail(&mm,
-					 set_node(&tmp, order[n] * size, 1)))
+		if (!expect_reserve_fail(test, &mm, set_node(&tmp, order[n] * size, 1)))
 			goto out;
 
 		/* Remove and reinsert should work */
 		drm_mm_remove_node(&nodes[order[n]]);
 		err = drm_mm_reserve_node(&mm, &nodes[order[n]]);
 		if (err) {
-			pr_err("reserve failed, step %d, start %llu\n",
-			       n, nodes[n].start);
+			KUNIT_FAIL(test, "reserve failed, step %d, start %llu\n",
+				   n, nodes[n].start);
 			ret = err;
 			goto out;
 		}
 	}
 
-	if (!assert_continuous(&mm, size))
+	if (!assert_continuous(test, &mm, size))
 		goto out;
 
 	/* Overlapping use should then fail */
 	for (n = 0; n < count; n++) {
-		if (!expect_reserve_fail(&mm, set_node(&tmp, 0, size*count)))
+		if (!expect_reserve_fail(test, &mm, set_node(&tmp, 0, size * count)))
 			goto out;
 	}
 	for (n = 0; n < count; n++) {
-		if (!expect_reserve_fail(&mm,
-					 set_node(&tmp,
-						  size * n,
-						  size * (count - n))))
+		if (!expect_reserve_fail(test, &mm, set_node(&tmp, size * n, size * (count - n))))
 			goto out;
 	}
 
@@ -472,8 +439,8 @@ static int __igt_reserve(unsigned int count, u64 size)
 			node = &nodes[order[(o + m) % count]];
 			err = drm_mm_reserve_node(&mm, node);
 			if (err) {
-				pr_err("reserve failed, step %d/%d, start %llu\n",
-				       m, n, node->start);
+				KUNIT_FAIL(test, "reserve failed, step %d/%d, start %llu\n",
+					   m, n, node->start);
 				ret = err;
 				goto out;
 			}
@@ -481,7 +448,7 @@ static int __igt_reserve(unsigned int count, u64 size)
 
 		o += n;
 
-		if (!assert_continuous(&mm, size))
+		if (!assert_continuous(test, &mm, size))
 			goto out;
 	}
 
@@ -491,41 +458,30 @@ out:
 		drm_mm_remove_node(node);
 	drm_mm_takedown(&mm);
 	vfree(nodes);
-err_order:
 	kfree(order);
 err:
 	return ret;
 }
 
-static int igt_reserve(void *ignored)
+static void drm_test_mm_reserve(struct kunit *test)
 {
 	const unsigned int count = min_t(unsigned int, BIT(10), max_iterations);
-	int n, ret;
+	int n;
 
 	for_each_prime_number_from(n, 1, 54) {
 		u64 size = BIT_ULL(n);
 
-		ret = __igt_reserve(count, size - 1);
-		if (ret)
-			return ret;
-
-		ret = __igt_reserve(count, size);
-		if (ret)
-			return ret;
-
-		ret = __igt_reserve(count, size + 1);
-		if (ret)
-			return ret;
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_reserve(test, count, size - 1));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_reserve(test, count, size));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_reserve(test, count, size + 1));
 
 		cond_resched();
 	}
-
-	return 0;
 }
 
-static bool expect_insert(struct drm_mm *mm, struct drm_mm_node *node,
-			  u64 size, u64 alignment, unsigned long color,
-			  const struct insert_mode *mode)
+static bool expect_insert(struct kunit *test, struct drm_mm *mm,
+			  struct drm_mm_node *node, u64 size, u64 alignment, unsigned long color,
+			const struct insert_mode *mode)
 {
 	int err;
 
@@ -533,12 +489,13 @@ static bool expect_insert(struct drm_mm *mm, struct drm_mm_node *node,
 					 size, alignment, color,
 					 mode->mode);
 	if (err) {
-		pr_err("insert (size=%llu, alignment=%llu, color=%lu, mode=%s) failed with err=%d\n",
-		       size, alignment, color, mode->name, err);
+		KUNIT_FAIL(test,
+			   "insert (size=%llu, alignment=%llu, color=%lu, mode=%s) failed with err=%d\n",
+			   size, alignment, color, mode->name, err);
 		return false;
 	}
 
-	if (!assert_node(node, mm, size, alignment, color)) {
+	if (!assert_node(test, node, mm, size, alignment, color)) {
 		drm_mm_remove_node(node);
 		return false;
 	}
@@ -546,7 +503,7 @@ static bool expect_insert(struct drm_mm *mm, struct drm_mm_node *node,
 	return true;
 }
 
-static bool expect_insert_fail(struct drm_mm *mm, u64 size)
+static bool expect_insert_fail(struct kunit *test, struct drm_mm *mm, u64 size)
 {
 	struct drm_mm_node tmp = {};
 	int err;
@@ -556,17 +513,18 @@ static bool expect_insert_fail(struct drm_mm *mm, u64 size)
 		return true;
 
 	if (!err) {
-		pr_err("impossible insert succeeded, node %llu + %llu\n",
-		       tmp.start, tmp.size);
+		KUNIT_FAIL(test, "impossible insert succeeded, node %llu + %llu\n",
+			   tmp.start, tmp.size);
 		drm_mm_remove_node(&tmp);
 	} else {
-		pr_err("impossible insert failed with wrong error %d [expected %d], size %llu\n",
-		       err, -ENOSPC, size);
+		KUNIT_FAIL(test,
+			   "impossible insert failed with wrong error %d [expected %d], size %llu\n",
+			   err, -ENOSPC, size);
 	}
 	return false;
 }
 
-static int __igt_insert(unsigned int count, u64 size, bool replace)
+static int __drm_test_mm_insert(struct kunit *test, unsigned int count, u64 size, bool replace)
 {
 	DRM_RND_STATE(prng, random_seed);
 	const struct insert_mode *mode;
@@ -582,8 +540,7 @@ static int __igt_insert(unsigned int count, u64 size, bool replace)
 
 	ret = -ENOMEM;
 	nodes = vmalloc(array_size(count, sizeof(*nodes)));
-	if (!nodes)
-		goto err;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	order = drm_random_order(count, &prng);
 	if (!order)
@@ -598,41 +555,43 @@ static int __igt_insert(unsigned int count, u64 size, bool replace)
 
 			node = replace ? &tmp : &nodes[n];
 			memset(node, 0, sizeof(*node));
-			if (!expect_insert(&mm, node, size, 0, n, mode)) {
-				pr_err("%s insert failed, size %llu step %d\n",
-				       mode->name, size, n);
+			if (!expect_insert(test, &mm, node, size, 0, n, mode)) {
+				KUNIT_FAIL(test, "%s insert failed, size %llu step %d\n",
+					   mode->name, size, n);
 				goto out;
 			}
 
 			if (replace) {
 				drm_mm_replace_node(&tmp, &nodes[n]);
 				if (drm_mm_node_allocated(&tmp)) {
-					pr_err("replaced old-node still allocated! step %d\n",
-					       n);
+					KUNIT_FAIL(test,
+						   "replaced old-node still allocated! step %d\n",
+						   n);
 					goto out;
 				}
 
-				if (!assert_node(&nodes[n], &mm, size, 0, n)) {
-					pr_err("replaced node did not inherit parameters, size %llu step %d\n",
-					       size, n);
+				if (!assert_node(test, &nodes[n], &mm, size, 0, n)) {
+					KUNIT_FAIL(test,
+						   "replaced node did not inherit parameters, size %llu step %d\n",
+						   size, n);
 					goto out;
 				}
 
 				if (tmp.start != nodes[n].start) {
-					pr_err("replaced node mismatch location expected [%llx + %llx], found [%llx + %llx]\n",
-					       tmp.start, size,
-					       nodes[n].start, nodes[n].size);
+					KUNIT_FAIL(test,
+						   "replaced node mismatch location expected [%llx + %llx], found [%llx + %llx]\n",
+						   tmp.start, size, nodes[n].start, nodes[n].size);
 					goto out;
 				}
 			}
 		}
 
 		/* After random insertion the nodes should be in order */
-		if (!assert_continuous(&mm, size))
+		if (!assert_continuous(test, &mm, size))
 			goto out;
 
 		/* Repeated use should then fail */
-		if (!expect_insert_fail(&mm, size))
+		if (!expect_insert_fail(test, &mm, size))
 			goto out;
 
 		/* Remove one and reinsert, as the only hole it should refill itself */
@@ -640,19 +599,20 @@ static int __igt_insert(unsigned int count, u64 size, bool replace)
 			u64 addr = nodes[n].start;
 
 			drm_mm_remove_node(&nodes[n]);
-			if (!expect_insert(&mm, &nodes[n], size, 0, n, mode)) {
-				pr_err("%s reinsert failed, size %llu step %d\n",
-				       mode->name, size, n);
+			if (!expect_insert(test, &mm, &nodes[n], size, 0, n, mode)) {
+				KUNIT_FAIL(test, "%s reinsert failed, size %llu step %d\n",
+					   mode->name, size, n);
 				goto out;
 			}
 
 			if (nodes[n].start != addr) {
-				pr_err("%s reinsert node moved, step %d, expected %llx, found %llx\n",
-				       mode->name, n, addr, nodes[n].start);
+				KUNIT_FAIL(test,
+					   "%s reinsert node moved, step %d, expected %llx, found %llx\n",
+					   mode->name, n, addr, nodes[n].start);
 				goto out;
 			}
 
-			if (!assert_continuous(&mm, size))
+			if (!assert_continuous(test, &mm, size))
 				goto out;
 		}
 
@@ -665,19 +625,20 @@ static int __igt_insert(unsigned int count, u64 size, bool replace)
 
 			for (m = 0; m < n; m++) {
 				node = &nodes[order[(o + m) % count]];
-				if (!expect_insert(&mm, node, size, 0, n, mode)) {
-					pr_err("%s multiple reinsert failed, size %llu step %d\n",
-					       mode->name, size, n);
+				if (!expect_insert(test, &mm, node, size, 0, n, mode)) {
+					KUNIT_FAIL(test,
+						   "%s multiple reinsert failed, size %llu step %d\n",
+							   mode->name, size, n);
 					goto out;
 				}
 			}
 
 			o += n;
 
-			if (!assert_continuous(&mm, size))
+			if (!assert_continuous(test, &mm, size))
 				goto out;
 
-			if (!expect_insert_fail(&mm, size))
+			if (!expect_insert_fail(test, &mm, size))
 				goto out;
 		}
 
@@ -696,44 +657,31 @@ out:
 	kfree(order);
 err_nodes:
 	vfree(nodes);
-err:
 	return ret;
 }
 
-static int igt_insert(void *ignored)
+static void drm_test_mm_insert(struct kunit *test)
 {
 	const unsigned int count = min_t(unsigned int, BIT(10), max_iterations);
 	unsigned int n;
-	int ret;
 
 	for_each_prime_number_from(n, 1, 54) {
 		u64 size = BIT_ULL(n);
 
-		ret = __igt_insert(count, size - 1, false);
-		if (ret)
-			return ret;
-
-		ret = __igt_insert(count, size, false);
-		if (ret)
-			return ret;
-
-		ret = __igt_insert(count, size + 1, false);
-		if (ret)
-			return ret;
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size - 1, false));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size, false));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size + 1, false));
 
 		cond_resched();
 	}
-
-	return 0;
 }
 
-static int igt_replace(void *ignored)
+static void drm_test_mm_replace(struct kunit *test)
 {
 	const unsigned int count = min_t(unsigned int, BIT(10), max_iterations);
 	unsigned int n;
-	int ret;
 
-	/* Reuse igt_insert to exercise replacement by inserting a dummy node,
+	/* Reuse __drm_test_mm_insert to exercise replacement by inserting a dummy node,
 	 * then replacing it with the intended node. We want to check that
 	 * the tree is intact and all the information we need is carried
 	 * across to the target node.
@@ -742,28 +690,17 @@ static int igt_replace(void *ignored)
 	for_each_prime_number_from(n, 1, 54) {
 		u64 size = BIT_ULL(n);
 
-		ret = __igt_insert(count, size - 1, true);
-		if (ret)
-			return ret;
-
-		ret = __igt_insert(count, size, true);
-		if (ret)
-			return ret;
-
-		ret = __igt_insert(count, size + 1, true);
-		if (ret)
-			return ret;
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size - 1, true));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size, true));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert(test, count, size + 1, true));
 
 		cond_resched();
 	}
-
-	return 0;
 }
 
-static bool expect_insert_in_range(struct drm_mm *mm, struct drm_mm_node *node,
+static bool expect_insert_in_range(struct kunit *test, struct drm_mm *mm, struct drm_mm_node *node,
 				   u64 size, u64 alignment, unsigned long color,
-				   u64 range_start, u64 range_end,
-				   const struct insert_mode *mode)
+				   u64 range_start, u64 range_end, const struct insert_mode *mode)
 {
 	int err;
 
@@ -772,13 +709,14 @@ static bool expect_insert_in_range(struct drm_mm *mm, struct drm_mm_node *node,
 					  range_start, range_end,
 					  mode->mode);
 	if (err) {
-		pr_err("insert (size=%llu, alignment=%llu, color=%lu, mode=%s) nto range [%llx, %llx] failed with err=%d\n",
-		       size, alignment, color, mode->name,
-		       range_start, range_end, err);
+		KUNIT_FAIL(test,
+			   "insert (size=%llu, alignment=%llu, color=%lu, mode=%s) nto range [%llx, %llx] failed with err=%d\n",
+				   size, alignment, color, mode->name,
+				   range_start, range_end, err);
 		return false;
 	}
 
-	if (!assert_node(node, mm, size, alignment, color)) {
+	if (!assert_node(test, node, mm, size, alignment, color)) {
 		drm_mm_remove_node(node);
 		return false;
 	}
@@ -786,67 +724,63 @@ static bool expect_insert_in_range(struct drm_mm *mm, struct drm_mm_node *node,
 	return true;
 }
 
-static bool expect_insert_in_range_fail(struct drm_mm *mm,
-					u64 size,
-					u64 range_start,
-					u64 range_end)
+static bool expect_insert_in_range_fail(struct kunit *test, struct drm_mm *mm,
+					u64 size, u64 range_start, u64 range_end)
 {
 	struct drm_mm_node tmp = {};
 	int err;
 
-	err = drm_mm_insert_node_in_range(mm, &tmp,
-					  size, 0, 0,
-					  range_start, range_end,
+	err = drm_mm_insert_node_in_range(mm, &tmp, size, 0, 0, range_start, range_end,
 					  0);
 	if (likely(err == -ENOSPC))
 		return true;
 
 	if (!err) {
-		pr_err("impossible insert succeeded, node %llx + %llu, range [%llx, %llx]\n",
-		       tmp.start, tmp.size, range_start, range_end);
+		KUNIT_FAIL(test,
+			   "impossible insert succeeded, node %llx + %llu, range [%llx, %llx]\n",
+				   tmp.start, tmp.size, range_start, range_end);
 		drm_mm_remove_node(&tmp);
 	} else {
-		pr_err("impossible insert failed with wrong error %d [expected %d], size %llu, range [%llx, %llx]\n",
-		       err, -ENOSPC, size, range_start, range_end);
+		KUNIT_FAIL(test,
+			   "impossible insert failed with wrong error %d [expected %d], size %llu, range [%llx, %llx]\n",
+				   err, -ENOSPC, size, range_start, range_end);
 	}
 
 	return false;
 }
 
-static bool assert_contiguous_in_range(struct drm_mm *mm,
-				       u64 size,
-				       u64 start,
-				       u64 end)
+static bool assert_contiguous_in_range(struct kunit *test, struct drm_mm *mm,
+				       u64 size, u64 start, u64 end)
 {
 	struct drm_mm_node *node;
 	unsigned int n;
 
-	if (!expect_insert_in_range_fail(mm, size, start, end))
+	if (!expect_insert_in_range_fail(test, mm, size, start, end))
 		return false;
 
 	n = div64_u64(start + size - 1, size);
 	drm_mm_for_each_node(node, mm) {
 		if (node->start < start || node->start + node->size > end) {
-			pr_err("node %d out of range, address [%llx + %llu], range [%llx, %llx]\n",
-			       n, node->start, node->start + node->size, start, end);
+			KUNIT_FAIL(test,
+				   "node %d out of range, address [%llx + %llu], range [%llx, %llx]\n",
+					   n, node->start, node->start + node->size, start, end);
 			return false;
 		}
 
 		if (node->start != n * size) {
-			pr_err("node %d out of order, expected start %llx, found %llx\n",
-			       n, n * size, node->start);
+			KUNIT_FAIL(test, "node %d out of order, expected start %llx, found %llx\n",
+				   n, n * size, node->start);
 			return false;
 		}
 
 		if (node->size != size) {
-			pr_err("node %d has wrong size, expected size %llx, found %llx\n",
-			       n, size, node->size);
+			KUNIT_FAIL(test, "node %d has wrong size, expected size %llx, found %llx\n",
+				   n, size, node->size);
 			return false;
 		}
 
-		if (drm_mm_hole_follows(node) &&
-		    drm_mm_hole_node_end(node) < end) {
-			pr_err("node %d is followed by a hole!\n", n);
+		if (drm_mm_hole_follows(node) && drm_mm_hole_node_end(node) < end) {
+			KUNIT_FAIL(test, "node %d is followed by a hole!\n", n);
 			return false;
 		}
 
@@ -856,8 +790,8 @@ static bool assert_contiguous_in_range(struct drm_mm *mm,
 	if (start > 0) {
 		node = __drm_mm_interval_first(mm, 0, start - 1);
 		if (drm_mm_node_allocated(node)) {
-			pr_err("node before start: node=%llx+%llu, start=%llx\n",
-			       node->start, node->size, start);
+			KUNIT_FAIL(test, "node before start: node=%llx+%llu, start=%llx\n",
+				   node->start, node->size, start);
 			return false;
 		}
 	}
@@ -865,8 +799,8 @@ static bool assert_contiguous_in_range(struct drm_mm *mm,
 	if (end < U64_MAX) {
 		node = __drm_mm_interval_first(mm, end, U64_MAX);
 		if (drm_mm_node_allocated(node)) {
-			pr_err("node after end: node=%llx+%llu, end=%llx\n",
-			       node->start, node->size, end);
+			KUNIT_FAIL(test, "node after end: node=%llx+%llu, end=%llx\n",
+				   node->start, node->size, end);
 			return false;
 		}
 	}
@@ -874,7 +808,8 @@ static bool assert_contiguous_in_range(struct drm_mm *mm,
 	return true;
 }
 
-static int __igt_insert_range(unsigned int count, u64 size, u64 start, u64 end)
+static int __drm_test_mm_insert_range(struct kunit *test, unsigned int count, u64 size,
+				      u64 start, u64 end)
 {
 	const struct insert_mode *mode;
 	struct drm_mm mm;
@@ -886,14 +821,13 @@ static int __igt_insert_range(unsigned int count, u64 size, u64 start, u64 end)
 	DRM_MM_BUG_ON(!size);
 	DRM_MM_BUG_ON(end <= start);
 
-	/* Very similar to __igt_insert(), but now instead of populating the
+	/* Very similar to __drm_test_mm_insert(), but now instead of populating the
 	 * full range of the drm_mm, we try to fill a small portion of it.
 	 */
 
 	ret = -ENOMEM;
 	nodes = vzalloc(array_size(count, sizeof(*nodes)));
-	if (!nodes)
-		goto err;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	ret = -EINVAL;
 	drm_mm_init(&mm, 0, count * size);
@@ -903,20 +837,19 @@ static int __igt_insert_range(unsigned int count, u64 size, u64 start, u64 end)
 
 	for (mode = insert_modes; mode->name; mode++) {
 		for (n = start_n; n <= end_n; n++) {
-			if (!expect_insert_in_range(&mm, &nodes[n],
-						    size, size, n,
+			if (!expect_insert_in_range(test, &mm, &nodes[n], size, size, n,
 						    start, end, mode)) {
-				pr_err("%s insert failed, size %llu, step %d [%d, %d], range [%llx, %llx]\n",
-				       mode->name, size, n,
-				       start_n, end_n,
-				       start, end);
+				KUNIT_FAIL(test,
+					   "%s insert failed, size %llu, step %d [%d, %d], range [%llx, %llx]\n",
+						   mode->name, size, n, start_n, end_n, start, end);
 				goto out;
 			}
 		}
 
-		if (!assert_contiguous_in_range(&mm, size, start, end)) {
-			pr_err("%s: range [%llx, %llx] not full after initialisation, size=%llu\n",
-			       mode->name, start, end, size);
+		if (!assert_contiguous_in_range(test, &mm, size, start, end)) {
+			KUNIT_FAIL(test,
+				   "%s: range [%llx, %llx] not full after initialisation, size=%llu\n",
+				   mode->name, start, end, size);
 			goto out;
 		}
 
@@ -925,23 +858,24 @@ static int __igt_insert_range(unsigned int count, u64 size, u64 start, u64 end)
 			u64 addr = nodes[n].start;
 
 			drm_mm_remove_node(&nodes[n]);
-			if (!expect_insert_in_range(&mm, &nodes[n],
-						    size, size, n,
+			if (!expect_insert_in_range(test, &mm, &nodes[n], size, size, n,
 						    start, end, mode)) {
-				pr_err("%s reinsert failed, step %d\n", mode->name, n);
+				KUNIT_FAIL(test, "%s reinsert failed, step %d\n", mode->name, n);
 				goto out;
 			}
 
 			if (nodes[n].start != addr) {
-				pr_err("%s reinsert node moved, step %d, expected %llx, found %llx\n",
-				       mode->name, n, addr, nodes[n].start);
+				KUNIT_FAIL(test,
+					   "%s reinsert node moved, step %d, expected %llx, found %llx\n",
+					   mode->name, n, addr, nodes[n].start);
 				goto out;
 			}
 		}
 
-		if (!assert_contiguous_in_range(&mm, size, start, end)) {
-			pr_err("%s: range [%llx, %llx] not full after reinsertion, size=%llu\n",
-			       mode->name, start, end, size);
+		if (!assert_contiguous_in_range(test, &mm, size, start, end)) {
+			KUNIT_FAIL(test,
+				   "%s: range [%llx, %llx] not full after reinsertion, size=%llu\n",
+				   mode->name, start, end, size);
 			goto out;
 		}
 
@@ -958,11 +892,10 @@ out:
 		drm_mm_remove_node(node);
 	drm_mm_takedown(&mm);
 	vfree(nodes);
-err:
 	return ret;
 }
 
-static int insert_outside_range(void)
+static int insert_outside_range(struct kunit *test)
 {
 	struct drm_mm mm;
 	const unsigned int start = 1024;
@@ -971,81 +904,58 @@ static int insert_outside_range(void)
 
 	drm_mm_init(&mm, start, size);
 
-	if (!expect_insert_in_range_fail(&mm, 1, 0, start))
+	if (!expect_insert_in_range_fail(test, &mm, 1, 0, start))
 		return -EINVAL;
 
-	if (!expect_insert_in_range_fail(&mm, size,
-					 start - size/2, start + (size+1)/2))
+	if (!expect_insert_in_range_fail(test, &mm, size,
+					 start - size / 2, start + (size + 1) / 2))
 		return -EINVAL;
 
-	if (!expect_insert_in_range_fail(&mm, size,
-					 end - (size+1)/2, end + size/2))
+	if (!expect_insert_in_range_fail(test, &mm, size,
+					 end - (size + 1) / 2, end + size / 2))
 		return -EINVAL;
 
-	if (!expect_insert_in_range_fail(&mm, 1, end, end + size))
+	if (!expect_insert_in_range_fail(test, &mm, 1, end, end + size))
 		return -EINVAL;
 
 	drm_mm_takedown(&mm);
 	return 0;
 }
 
-static int igt_insert_range(void *ignored)
+static void drm_test_mm_insert_range(struct kunit *test)
 {
 	const unsigned int count = min_t(unsigned int, BIT(13), max_iterations);
 	unsigned int n;
-	int ret;
 
 	/* Check that requests outside the bounds of drm_mm are rejected. */
-	ret = insert_outside_range();
-	if (ret)
-		return ret;
+	KUNIT_ASSERT_FALSE(test, insert_outside_range(test));
 
 	for_each_prime_number_from(n, 1, 50) {
 		const u64 size = BIT_ULL(n);
 		const u64 max = count * size;
 
-		ret = __igt_insert_range(count, size, 0, max);
-		if (ret)
-			return ret;
-
-		ret = __igt_insert_range(count, size, 1, max);
-		if (ret)
-			return ret;
-
-		ret = __igt_insert_range(count, size, 0, max - 1);
-		if (ret)
-			return ret;
-
-		ret = __igt_insert_range(count, size, 0, max/2);
-		if (ret)
-			return ret;
-
-		ret = __igt_insert_range(count, size, max/2, max);
-		if (ret)
-			return ret;
-
-		ret = __igt_insert_range(count, size, max/4+1, 3*max/4-1);
-		if (ret)
-			return ret;
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size, 0, max));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size, 1, max));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size, 0, max - 1));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size, 0, max / 2));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size,
+								    max / 2, max / 2));
+		KUNIT_ASSERT_FALSE(test, __drm_test_mm_insert_range(test, count, size,
+								    max / 4 + 1, 3 * max / 4 - 1));
 
 		cond_resched();
 	}
-
-	return 0;
 }
 
-static int prepare_igt_frag(struct drm_mm *mm,
-			    struct drm_mm_node *nodes,
-			    unsigned int num_insert,
-			    const struct insert_mode *mode)
+static int prepare_frag(struct kunit *test, struct drm_mm *mm, struct drm_mm_node *nodes,
+			unsigned int num_insert, const struct insert_mode *mode)
 {
 	unsigned int size = 4096;
 	unsigned int i;
 
 	for (i = 0; i < num_insert; i++) {
-		if (!expect_insert(mm, &nodes[i], size, 0, i,
-				   mode) != 0) {
-			pr_err("%s insert failed\n", mode->name);
+		if (!expect_insert(test, mm, &nodes[i], size, 0, i, mode) != 0) {
+			KUNIT_FAIL(test, "%s insert failed\n", mode->name);
 			return -EINVAL;
 		}
 	}
@@ -1057,12 +967,10 @@ static int prepare_igt_frag(struct drm_mm *mm,
 	}
 
 	return 0;
-
 }
 
-static u64 get_insert_time(struct drm_mm *mm,
-			   unsigned int num_insert,
-			   struct drm_mm_node *nodes,
+static u64 get_insert_time(struct kunit *test, struct drm_mm *mm,
+			   unsigned int num_insert, struct drm_mm_node *nodes,
 			   const struct insert_mode *mode)
 {
 	unsigned int size = 8192;
@@ -1071,8 +979,8 @@ static u64 get_insert_time(struct drm_mm *mm,
 
 	start = ktime_get();
 	for (i = 0; i < num_insert; i++) {
-		if (!expect_insert(mm, &nodes[i], size, 0, i, mode) != 0) {
-			pr_err("%s insert failed\n", mode->name);
+		if (!expect_insert(test, mm, &nodes[i], size, 0, i, mode) != 0) {
+			KUNIT_FAIL(test, "%s insert failed\n", mode->name);
 			return 0;
 		}
 	}
@@ -1080,28 +988,26 @@ static u64 get_insert_time(struct drm_mm *mm,
 	return ktime_to_ns(ktime_sub(ktime_get(), start));
 }
 
-static int igt_frag(void *ignored)
+static void drm_test_mm_frag(struct kunit *test)
 {
 	struct drm_mm mm;
 	const struct insert_mode *mode;
 	struct drm_mm_node *nodes, *node, *next;
 	unsigned int insert_size = 10000;
 	unsigned int scale_factor = 4;
-	int ret = -EINVAL;
 
 	/* We need 4 * insert_size nodes to hold intermediate allocated
 	 * drm_mm nodes.
-	 * 1 times for prepare_igt_frag()
+	 * 1 times for prepare_frag()
 	 * 1 times for get_insert_time()
 	 * 2 times for get_insert_time()
 	 */
 	nodes = vzalloc(array_size(insert_size * 4, sizeof(*nodes)));
-	if (!nodes)
-		return -ENOMEM;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	/* For BOTTOMUP and TOPDOWN, we first fragment the
-	 * address space using prepare_igt_frag() and then try to verify
-	 * that that insertions scale quadratically from 10k to 20k insertions
+	 * address space using prepare_frag() and then try to verify
+	 * that insertions scale quadratically from 10k to 20k insertions
 	 */
 	drm_mm_init(&mm, 1, U64_MAX - 2);
 	for (mode = insert_modes; mode->name; mode++) {
@@ -1111,28 +1017,25 @@ static int igt_frag(void *ignored)
 		    mode->mode != DRM_MM_INSERT_HIGH)
 			continue;
 
-		ret = prepare_igt_frag(&mm, nodes, insert_size, mode);
-		if (ret)
+		if (prepare_frag(test, &mm, nodes, insert_size, mode))
 			goto err;
 
-		insert_time1 = get_insert_time(&mm, insert_size,
+		insert_time1 = get_insert_time(test, &mm, insert_size,
 					       nodes + insert_size, mode);
 		if (insert_time1 == 0)
 			goto err;
 
-		insert_time2 = get_insert_time(&mm, (insert_size * 2),
+		insert_time2 = get_insert_time(test, &mm, (insert_size * 2),
 					       nodes + insert_size * 2, mode);
 		if (insert_time2 == 0)
 			goto err;
 
-		pr_info("%s fragmented insert of %u and %u insertions took %llu and %llu nsecs\n",
-			mode->name, insert_size, insert_size * 2,
-			insert_time1, insert_time2);
+		kunit_info(test, "%s fragmented insert of %u and %u insertions took %llu and %llu nsecs\n",
+			   mode->name, insert_size, insert_size * 2, insert_time1, insert_time2);
 
 		if (insert_time2 > (scale_factor * insert_time1)) {
-			pr_err("%s fragmented insert took %llu nsecs more\n",
-			       mode->name,
-			       insert_time2 - (scale_factor * insert_time1));
+			KUNIT_FAIL(test, "%s fragmented insert took %llu nsecs more\n",
+				   mode->name, insert_time2 - (scale_factor * insert_time1));
 			goto err;
 		}
 
@@ -1140,24 +1043,20 @@ static int igt_frag(void *ignored)
 			drm_mm_remove_node(node);
 	}
 
-	ret = 0;
 err:
 	drm_mm_for_each_node_safe(node, next, &mm)
 		drm_mm_remove_node(node);
 	drm_mm_takedown(&mm);
 	vfree(nodes);
-
-	return ret;
 }
 
-static int igt_align(void *ignored)
+static void drm_test_mm_align(struct kunit *test)
 {
 	const struct insert_mode *mode;
 	const unsigned int max_count = min(8192u, max_prime);
 	struct drm_mm mm;
 	struct drm_mm_node *nodes, *node, *next;
 	unsigned int prime;
-	int ret = -EINVAL;
 
 	/* For each of the possible insertion modes, we pick a few
 	 * arbitrary alignments and check that the inserted node
@@ -1165,8 +1064,7 @@ static int igt_align(void *ignored)
 	 */
 
 	nodes = vzalloc(array_size(max_count, sizeof(*nodes)));
-	if (!nodes)
-		goto err;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	drm_mm_init(&mm, 1, U64_MAX - 2);
 
@@ -1176,11 +1074,9 @@ static int igt_align(void *ignored)
 		for_each_prime_number_from(prime, 1, max_count) {
 			u64 size = next_prime_number(prime);
 
-			if (!expect_insert(&mm, &nodes[i],
-					   size, prime, i,
-					   mode)) {
-				pr_err("%s insert failed with alignment=%d",
-				       mode->name, prime);
+			if (!expect_insert(test, &mm, &nodes[i], size, prime, i, mode)) {
+				KUNIT_FAIL(test, "%s insert failed with alignment=%d",
+					   mode->name, prime);
 				goto out;
 			}
 
@@ -1194,22 +1090,18 @@ static int igt_align(void *ignored)
 		cond_resched();
 	}
 
-	ret = 0;
 out:
 	drm_mm_for_each_node_safe(node, next, &mm)
 		drm_mm_remove_node(node);
 	drm_mm_takedown(&mm);
 	vfree(nodes);
-err:
-	return ret;
 }
 
-static int igt_align_pot(int max)
+static void drm_test_mm_align_pot(struct kunit *test, int max)
 {
 	struct drm_mm mm;
 	struct drm_mm_node *node, *next;
 	int bit;
-	int ret = -EINVAL;
 
 	/* Check that we can align to the full u64 address space */
 
@@ -1220,51 +1112,45 @@ static int igt_align_pot(int max)
 
 		node = kzalloc(sizeof(*node), GFP_KERNEL);
 		if (!node) {
-			ret = -ENOMEM;
+			KUNIT_FAIL(test, "failed to allocate node");
 			goto out;
 		}
 
 		align = BIT_ULL(bit);
-		size = BIT_ULL(bit-1) + 1;
-		if (!expect_insert(&mm, node,
-				   size, align, bit,
-				   &insert_modes[0])) {
-			pr_err("insert failed with alignment=%llx [%d]",
-			       align, bit);
+		size = BIT_ULL(bit - 1) + 1;
+		if (!expect_insert(test, &mm, node, size, align, bit, &insert_modes[0])) {
+			KUNIT_FAIL(test, "insert failed with alignment=%llx [%d]", align, bit);
 			goto out;
 		}
 
 		cond_resched();
 	}
 
-	ret = 0;
 out:
 	drm_mm_for_each_node_safe(node, next, &mm) {
 		drm_mm_remove_node(node);
 		kfree(node);
 	}
 	drm_mm_takedown(&mm);
-	return ret;
 }
 
-static int igt_align32(void *ignored)
+static void drm_test_mm_align32(struct kunit *test)
 {
-	return igt_align_pot(32);
+	drm_test_mm_align_pot(test, 32);
 }
 
-static int igt_align64(void *ignored)
+static void drm_test_mm_align64(struct kunit *test)
 {
-	return igt_align_pot(64);
+	drm_test_mm_align_pot(test, 64);
 }
 
-static void show_scan(const struct drm_mm_scan *scan)
+static void show_scan(struct kunit *test, const struct drm_mm_scan *scan)
 {
-	pr_info("scan: hit [%llx, %llx], size=%lld, align=%lld, color=%ld\n",
-		scan->hit_start, scan->hit_end,
-		scan->size, scan->alignment, scan->color);
+	kunit_info(test, "scan: hit [%llx, %llx], size=%lld, align=%lld, color=%ld\n",
+		   scan->hit_start, scan->hit_end, scan->size, scan->alignment, scan->color);
 }
 
-static void show_holes(const struct drm_mm *mm, int count)
+static void show_holes(struct kunit *test, const struct drm_mm *mm, int count)
 {
 	u64 hole_start, hole_end;
 	struct drm_mm_node *hole;
@@ -1274,19 +1160,15 @@ static void show_holes(const struct drm_mm *mm, int count)
 		const char *node1 = NULL, *node2 = NULL;
 
 		if (drm_mm_node_allocated(hole))
-			node1 = kasprintf(GFP_KERNEL,
-					  "[%llx + %lld, color=%ld], ",
+			node1 = kasprintf(GFP_KERNEL, "[%llx + %lld, color=%ld], ",
 					  hole->start, hole->size, hole->color);
 
 		if (drm_mm_node_allocated(next))
-			node2 = kasprintf(GFP_KERNEL,
-					  ", [%llx + %lld, color=%ld]",
+			node2 = kasprintf(GFP_KERNEL, ", [%llx + %lld, color=%ld]",
 					  next->start, next->size, next->color);
 
-		pr_info("%sHole [%llx - %llx, size %lld]%s\n",
-			node1,
-			hole_start, hole_end, hole_end - hole_start,
-			node2);
+		kunit_info(test, "%sHole [%llx - %llx, size %lld]%s\n", node1,
+			   hole_start, hole_end, hole_end - hole_start, node2);
 
 		kfree(node2);
 		kfree(node1);
@@ -1301,12 +1183,9 @@ struct evict_node {
 	struct list_head link;
 };
 
-static bool evict_nodes(struct drm_mm_scan *scan,
-			struct evict_node *nodes,
-			unsigned int *order,
-			unsigned int count,
-			bool use_color,
-			struct list_head *evict_list)
+static bool evict_nodes(struct kunit *test, struct drm_mm_scan *scan,
+			struct evict_node *nodes, unsigned int *order, unsigned int count,
+			bool use_color, struct list_head *evict_list)
 {
 	struct evict_node *e, *en;
 	unsigned int i;
@@ -1322,8 +1201,9 @@ static bool evict_nodes(struct drm_mm_scan *scan,
 			list_del(&e->link);
 	}
 	if (list_empty(evict_list)) {
-		pr_err("Failed to find eviction: size=%lld [avail=%d], align=%lld (color=%lu)\n",
-		       scan->size, count, scan->alignment, scan->color);
+		KUNIT_FAIL(test,
+			   "Failed to find eviction: size=%lld [avail=%d], align=%lld (color=%lu)\n",
+			   scan->size, count, scan->alignment, scan->color);
 		return false;
 	}
 
@@ -1340,7 +1220,8 @@ static bool evict_nodes(struct drm_mm_scan *scan,
 		}
 	} else {
 		if (drm_mm_scan_color_evict(scan)) {
-			pr_err("drm_mm_scan_color_evict unexpectedly reported overlapping nodes!\n");
+			KUNIT_FAIL(test,
+				   "drm_mm_scan_color_evict unexpectedly reported overlapping nodes!\n");
 			return false;
 		}
 	}
@@ -1348,9 +1229,8 @@ static bool evict_nodes(struct drm_mm_scan *scan,
 	return true;
 }
 
-static bool evict_nothing(struct drm_mm *mm,
-			  unsigned int total_size,
-			  struct evict_node *nodes)
+static bool evict_nothing(struct kunit *test, struct drm_mm *mm,
+			  unsigned int total_size, struct evict_node *nodes)
 {
 	struct drm_mm_scan scan;
 	LIST_HEAD(evict_list);
@@ -1371,7 +1251,7 @@ static bool evict_nothing(struct drm_mm *mm,
 		e = &nodes[n];
 
 		if (!drm_mm_node_allocated(&e->node)) {
-			pr_err("node[%d] no longer allocated!\n", n);
+			KUNIT_FAIL(test, "node[%d] no longer allocated!\n", n);
 			return false;
 		}
 
@@ -1387,17 +1267,16 @@ static bool evict_nothing(struct drm_mm *mm,
 		e = &nodes[n];
 
 		if (!e->link.next) {
-			pr_err("node[%d] no longer connected!\n", n);
+			KUNIT_FAIL(test, "node[%d] no longer connected!\n", n);
 			return false;
 		}
 	}
 
-	return assert_continuous(mm, nodes[0].node.size);
+	return assert_continuous(test, mm, nodes[0].node.size);
 }
 
-static bool evict_everything(struct drm_mm *mm,
-			     unsigned int total_size,
-			     struct evict_node *nodes)
+static bool evict_everything(struct kunit *test, struct drm_mm *mm,
+			     unsigned int total_size, struct evict_node *nodes)
 {
 	struct drm_mm_scan scan;
 	LIST_HEAD(evict_list);
@@ -1417,8 +1296,8 @@ static bool evict_everything(struct drm_mm *mm,
 	list_for_each_entry(e, &evict_list, link) {
 		if (!drm_mm_scan_remove_block(&scan, &e->node)) {
 			if (!err) {
-				pr_err("Node %lld not marked for eviction!\n",
-				       e->node.start);
+				KUNIT_FAIL(test, "Node %lld not marked for eviction!\n",
+					   e->node.start);
 				err = -EINVAL;
 			}
 		}
@@ -1429,29 +1308,25 @@ static bool evict_everything(struct drm_mm *mm,
 	list_for_each_entry(e, &evict_list, link)
 		drm_mm_remove_node(&e->node);
 
-	if (!assert_one_hole(mm, 0, total_size))
+	if (!assert_one_hole(test, mm, 0, total_size))
 		return false;
 
 	list_for_each_entry(e, &evict_list, link) {
 		err = drm_mm_reserve_node(mm, &e->node);
 		if (err) {
-			pr_err("Failed to reinsert node after eviction: start=%llx\n",
-			       e->node.start);
+			KUNIT_FAIL(test, "Failed to reinsert node after eviction: start=%llx\n",
+				   e->node.start);
 			return false;
 		}
 	}
 
-	return assert_continuous(mm, nodes[0].node.size);
+	return assert_continuous(test, mm, nodes[0].node.size);
 }
 
-static int evict_something(struct drm_mm *mm,
-			   u64 range_start, u64 range_end,
-			   struct evict_node *nodes,
-			   unsigned int *order,
-			   unsigned int count,
-			   unsigned int size,
-			   unsigned int alignment,
-			   const struct insert_mode *mode)
+static int evict_something(struct kunit *test, struct drm_mm *mm,
+			   u64 range_start, u64 range_end, struct evict_node *nodes,
+			   unsigned int *order, unsigned int count, unsigned int size,
+			   unsigned int alignment, const struct insert_mode *mode)
 {
 	struct drm_mm_scan scan;
 	LIST_HEAD(evict_list);
@@ -1459,38 +1334,35 @@ static int evict_something(struct drm_mm *mm,
 	struct drm_mm_node tmp;
 	int err;
 
-	drm_mm_scan_init_with_range(&scan, mm,
-				    size, alignment, 0,
-				    range_start, range_end,
-				    mode->mode);
-	if (!evict_nodes(&scan,
-			 nodes, order, count, false,
-			 &evict_list))
+	drm_mm_scan_init_with_range(&scan, mm, size, alignment, 0, range_start,
+				    range_end, mode->mode);
+	if (!evict_nodes(test, &scan, nodes, order, count, false, &evict_list))
 		return -EINVAL;
 
 	memset(&tmp, 0, sizeof(tmp));
 	err = drm_mm_insert_node_generic(mm, &tmp, size, alignment, 0,
 					 DRM_MM_INSERT_EVICT);
 	if (err) {
-		pr_err("Failed to insert into eviction hole: size=%d, align=%d\n",
-		       size, alignment);
-		show_scan(&scan);
-		show_holes(mm, 3);
+		KUNIT_FAIL(test, "Failed to insert into eviction hole: size=%d, align=%d\n",
+			   size, alignment);
+		show_scan(test, &scan);
+		show_holes(test, mm, 3);
 		return err;
 	}
 
 	if (tmp.start < range_start || tmp.start + tmp.size > range_end) {
-		pr_err("Inserted [address=%llu + %llu] did not fit into the request range [%llu, %llu]\n",
-		       tmp.start, tmp.size, range_start, range_end);
+		KUNIT_FAIL(test,
+			   "Inserted [address=%llu + %llu] did not fit into the request range [%llu, %llu]\n",
+			   tmp.start, tmp.size, range_start, range_end);
 		err = -EINVAL;
 	}
 
-	if (!assert_node(&tmp, mm, size, alignment, 0) ||
+	if (!assert_node(test, &tmp, mm, size, alignment, 0) ||
 	    drm_mm_hole_follows(&tmp)) {
-		pr_err("Inserted did not fill the eviction hole: size=%lld [%d], align=%d [rem=%lld], start=%llx, hole-follows?=%d\n",
-		       tmp.size, size,
-		       alignment, misalignment(&tmp, alignment),
-		       tmp.start, drm_mm_hole_follows(&tmp));
+		KUNIT_FAIL(test,
+			   "Inserted did not fill the eviction hole: size=%lld [%d], align=%d [rem=%lld], start=%llx, hole-follows?=%d\n",
+			   tmp.size, size, alignment, misalignment(&tmp, alignment),
+			   tmp.start, drm_mm_hole_follows(&tmp));
 		err = -EINVAL;
 	}
 
@@ -1501,21 +1373,21 @@ static int evict_something(struct drm_mm *mm,
 	list_for_each_entry(e, &evict_list, link) {
 		err = drm_mm_reserve_node(mm, &e->node);
 		if (err) {
-			pr_err("Failed to reinsert node after eviction: start=%llx\n",
-			       e->node.start);
+			KUNIT_FAIL(test, "Failed to reinsert node after eviction: start=%llx\n",
+				   e->node.start);
 			return err;
 		}
 	}
 
-	if (!assert_continuous(mm, nodes[0].node.size)) {
-		pr_err("range is no longer continuous\n");
+	if (!assert_continuous(test, mm, nodes[0].node.size)) {
+		KUNIT_FAIL(test, "range is no longer continuous\n");
 		return -EINVAL;
 	}
 
 	return 0;
 }
 
-static int igt_evict(void *ignored)
+static void drm_test_mm_evict(struct kunit *test)
 {
 	DRM_RND_STATE(prng, random_seed);
 	const unsigned int size = 8192;
@@ -1524,7 +1396,6 @@ static int igt_evict(void *ignored)
 	struct evict_node *nodes;
 	struct drm_mm_node *node, *next;
 	unsigned int *order, n;
-	int ret, err;
 
 	/* Here we populate a full drm_mm and then try and insert a new node
 	 * by evicting other nodes in a random order. The drm_mm_scan should
@@ -1533,61 +1404,49 @@ static int igt_evict(void *ignored)
 	 * sizes to try and stress the hole finder.
 	 */
 
-	ret = -ENOMEM;
 	nodes = vzalloc(array_size(size, sizeof(*nodes)));
-	if (!nodes)
-		goto err;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	order = drm_random_order(size, &prng);
 	if (!order)
 		goto err_nodes;
 
-	ret = -EINVAL;
 	drm_mm_init(&mm, 0, size);
 	for (n = 0; n < size; n++) {
-		err = drm_mm_insert_node(&mm, &nodes[n].node, 1);
-		if (err) {
-			pr_err("insert failed, step %d\n", n);
-			ret = err;
+		if (drm_mm_insert_node(&mm, &nodes[n].node, 1)) {
+			KUNIT_FAIL(test, "insert failed, step %d\n", n);
 			goto out;
 		}
 	}
 
 	/* First check that using the scanner doesn't break the mm */
-	if (!evict_nothing(&mm, size, nodes)) {
-		pr_err("evict_nothing() failed\n");
+	if (!evict_nothing(test, &mm, size, nodes)) {
+		KUNIT_FAIL(test, "evict_nothing() failed\n");
 		goto out;
 	}
-	if (!evict_everything(&mm, size, nodes)) {
-		pr_err("evict_everything() failed\n");
+	if (!evict_everything(test, &mm, size, nodes)) {
+		KUNIT_FAIL(test, "evict_everything() failed\n");
 		goto out;
 	}
 
 	for (mode = evict_modes; mode->name; mode++) {
 		for (n = 1; n <= size; n <<= 1) {
 			drm_random_reorder(order, size, &prng);
-			err = evict_something(&mm, 0, U64_MAX,
-					      nodes, order, size,
-					      n, 1,
-					      mode);
-			if (err) {
-				pr_err("%s evict_something(size=%u) failed\n",
-				       mode->name, n);
-				ret = err;
+			if (evict_something(test, &mm, 0, U64_MAX, nodes, order, size, n, 1,
+					    mode)) {
+				KUNIT_FAIL(test, "%s evict_something(size=%u) failed\n",
+					   mode->name, n);
 				goto out;
 			}
 		}
 
 		for (n = 1; n < size; n <<= 1) {
 			drm_random_reorder(order, size, &prng);
-			err = evict_something(&mm, 0, U64_MAX,
-					      nodes, order, size,
-					      size/2, n,
-					      mode);
-			if (err) {
-				pr_err("%s evict_something(size=%u, alignment=%u) failed\n",
-				       mode->name, size/2, n);
-				ret = err;
+			if (evict_something(test, &mm, 0, U64_MAX, nodes, order, size,
+					    size / 2, n, mode)) {
+				KUNIT_FAIL(test,
+					   "%s evict_something(size=%u, alignment=%u) failed\n",
+					   mode->name, size / 2, n);
 				goto out;
 			}
 		}
@@ -1598,14 +1457,11 @@ static int igt_evict(void *ignored)
 			DRM_MM_BUG_ON(!nsize);
 
 			drm_random_reorder(order, size, &prng);
-			err = evict_something(&mm, 0, U64_MAX,
-					      nodes, order, size,
-					      nsize, n,
-					      mode);
-			if (err) {
-				pr_err("%s evict_something(size=%u, alignment=%u) failed\n",
-				       mode->name, nsize, n);
-				ret = err;
+			if (evict_something(test, &mm, 0, U64_MAX, nodes, order, size,
+					    nsize, n, mode)) {
+				KUNIT_FAIL(test,
+					   "%s evict_something(size=%u, alignment=%u) failed\n",
+					   mode->name, nsize, n);
 				goto out;
 			}
 		}
@@ -1613,7 +1469,6 @@ static int igt_evict(void *ignored)
 		cond_resched();
 	}
 
-	ret = 0;
 out:
 	drm_mm_for_each_node_safe(node, next, &mm)
 		drm_mm_remove_node(node);
@@ -1621,11 +1476,9 @@ out:
 	kfree(order);
 err_nodes:
 	vfree(nodes);
-err:
-	return ret;
 }
 
-static int igt_evict_range(void *ignored)
+static void drm_test_mm_evict_range(struct kunit *test)
 {
 	DRM_RND_STATE(prng, random_seed);
 	const unsigned int size = 8192;
@@ -1637,28 +1490,22 @@ static int igt_evict_range(void *ignored)
 	struct evict_node *nodes;
 	struct drm_mm_node *node, *next;
 	unsigned int *order, n;
-	int ret, err;
 
-	/* Like igt_evict() but now we are limiting the search to a
+	/* Like drm_test_mm_evict() but now we are limiting the search to a
 	 * small portion of the full drm_mm.
 	 */
 
-	ret = -ENOMEM;
 	nodes = vzalloc(array_size(size, sizeof(*nodes)));
-	if (!nodes)
-		goto err;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	order = drm_random_order(size, &prng);
 	if (!order)
 		goto err_nodes;
 
-	ret = -EINVAL;
 	drm_mm_init(&mm, 0, size);
 	for (n = 0; n < size; n++) {
-		err = drm_mm_insert_node(&mm, &nodes[n].node, 1);
-		if (err) {
-			pr_err("insert failed, step %d\n", n);
-			ret = err;
+		if (drm_mm_insert_node(&mm, &nodes[n].node, 1)) {
+			KUNIT_FAIL(test, "insert failed, step %d\n", n);
 			goto out;
 		}
 	}
@@ -1666,26 +1513,22 @@ static int igt_evict_range(void *ignored)
 	for (mode = evict_modes; mode->name; mode++) {
 		for (n = 1; n <= range_size; n <<= 1) {
 			drm_random_reorder(order, size, &prng);
-			err = evict_something(&mm, range_start, range_end,
-					      nodes, order, size,
-					      n, 1,
-					      mode);
-			if (err) {
-				pr_err("%s evict_something(size=%u) failed with range [%u, %u]\n",
-				       mode->name, n, range_start, range_end);
+			if (evict_something(test, &mm, range_start, range_end, nodes,
+					    order, size, n, 1, mode)) {
+				KUNIT_FAIL(test,
+					   "%s evict_something(size=%u) failed with range [%u, %u]\n",
+					   mode->name, n, range_start, range_end);
 				goto out;
 			}
 		}
 
 		for (n = 1; n <= range_size; n <<= 1) {
 			drm_random_reorder(order, size, &prng);
-			err = evict_something(&mm, range_start, range_end,
-					      nodes, order, size,
-					      range_size/2, n,
-					      mode);
-			if (err) {
-				pr_err("%s evict_something(size=%u, alignment=%u) failed with range [%u, %u]\n",
-				       mode->name, range_size/2, n, range_start, range_end);
+			if (evict_something(test, &mm, range_start, range_end, nodes,
+					    order, size, range_size / 2, n, mode)) {
+				KUNIT_FAIL(test,
+					   "%s evict_something(size=%u, alignment=%u) failed with range [%u, %u]\n",
+					   mode->name, range_size / 2, n, range_start, range_end);
 				goto out;
 			}
 		}
@@ -1696,13 +1539,11 @@ static int igt_evict_range(void *ignored)
 			DRM_MM_BUG_ON(!nsize);
 
 			drm_random_reorder(order, size, &prng);
-			err = evict_something(&mm, range_start, range_end,
-					      nodes, order, size,
-					      nsize, n,
-					      mode);
-			if (err) {
-				pr_err("%s evict_something(size=%u, alignment=%u) failed with range [%u, %u]\n",
-				       mode->name, nsize, n, range_start, range_end);
+			if (evict_something(test, &mm, range_start, range_end, nodes,
+					    order, size, nsize, n, mode)) {
+				KUNIT_FAIL(test,
+					   "%s evict_something(size=%u, alignment=%u) failed with range [%u, %u]\n",
+					   mode->name, nsize, n, range_start, range_end);
 				goto out;
 			}
 		}
@@ -1710,7 +1551,6 @@ static int igt_evict_range(void *ignored)
 		cond_resched();
 	}
 
-	ret = 0;
 out:
 	drm_mm_for_each_node_safe(node, next, &mm)
 		drm_mm_remove_node(node);
@@ -1718,8 +1558,6 @@ out:
 	kfree(order);
 err_nodes:
 	vfree(nodes);
-err:
-	return ret;
 }
 
 static unsigned int node_index(const struct drm_mm_node *node)
@@ -1727,9 +1565,10 @@ static unsigned int node_index(const struct drm_mm_node *node)
 	return div64_u64(node->start, node->size);
 }
 
-static int igt_topdown(void *ignored)
+static void drm_test_mm_topdown(struct kunit *test)
 {
 	const struct insert_mode *topdown = &insert_modes[TOPDOWN];
+
 	DRM_RND_STATE(prng, random_seed);
 	const unsigned int count = 8192;
 	unsigned int size;
@@ -1737,17 +1576,14 @@ static int igt_topdown(void *ignored)
 	struct drm_mm mm;
 	struct drm_mm_node *nodes, *node, *next;
 	unsigned int *order, n, m, o = 0;
-	int ret;
 
 	/* When allocating top-down, we expect to be returned a node
 	 * from a suitable hole at the top of the drm_mm. We check that
 	 * the returned node does match the highest available slot.
 	 */
 
-	ret = -ENOMEM;
 	nodes = vzalloc(array_size(count, sizeof(*nodes)));
-	if (!nodes)
-		goto err;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	bitmap = bitmap_zalloc(count, GFP_KERNEL);
 	if (!bitmap)
@@ -1757,28 +1593,26 @@ static int igt_topdown(void *ignored)
 	if (!order)
 		goto err_bitmap;
 
-	ret = -EINVAL;
 	for (size = 1; size <= 64; size <<= 1) {
-		drm_mm_init(&mm, 0, size*count);
+		drm_mm_init(&mm, 0, size * count);
 		for (n = 0; n < count; n++) {
-			if (!expect_insert(&mm, &nodes[n],
-					   size, 0, n,
-					   topdown)) {
-				pr_err("insert failed, size %u step %d\n", size, n);
+			if (!expect_insert(test, &mm, &nodes[n], size, 0, n, topdown)) {
+				KUNIT_FAIL(test, "insert failed, size %u step %d\n", size, n);
 				goto out;
 			}
 
 			if (drm_mm_hole_follows(&nodes[n])) {
-				pr_err("hole after topdown insert %d, start=%llx\n, size=%u",
-				       n, nodes[n].start, size);
+				KUNIT_FAIL(test,
+					   "hole after topdown insert %d, start=%llx\n, size=%u",
+					   n, nodes[n].start, size);
 				goto out;
 			}
 
-			if (!assert_one_hole(&mm, 0, size*(count - n - 1)))
+			if (!assert_one_hole(test, &mm, 0, size * (count - n - 1)))
 				goto out;
 		}
 
-		if (!assert_continuous(&mm, size))
+		if (!assert_continuous(test, &mm, size))
 			goto out;
 
 		drm_random_reorder(order, count, &prng);
@@ -1793,23 +1627,23 @@ static int igt_topdown(void *ignored)
 				unsigned int last;
 
 				node = &nodes[order[(o + m) % count]];
-				if (!expect_insert(&mm, node,
-						   size, 0, 0,
-						   topdown)) {
-					pr_err("insert failed, step %d/%d\n", m, n);
+				if (!expect_insert(test, &mm, node, size, 0, 0, topdown)) {
+					KUNIT_FAIL(test, "insert failed, step %d/%d\n", m, n);
 					goto out;
 				}
 
 				if (drm_mm_hole_follows(node)) {
-					pr_err("hole after topdown insert %d/%d, start=%llx\n",
-					       m, n, node->start);
+					KUNIT_FAIL(test,
+						   "hole after topdown insert %d/%d, start=%llx\n",
+						   m, n, node->start);
 					goto out;
 				}
 
 				last = find_last_bit(bitmap, count);
 				if (node_index(node) != last) {
-					pr_err("node %d/%d, size %d, not inserted into upmost hole, expected %d, found %d\n",
-					       m, n, size, last, node_index(node));
+					KUNIT_FAIL(test,
+						   "node %d/%d, size %d, not inserted into upmost hole, expected %d, found %d\n",
+						   m, n, size, last, node_index(node));
 					goto out;
 				}
 
@@ -1827,7 +1661,6 @@ static int igt_topdown(void *ignored)
 		cond_resched();
 	}
 
-	ret = 0;
 out:
 	drm_mm_for_each_node_safe(node, next, &mm)
 		drm_mm_remove_node(node);
@@ -1837,13 +1670,12 @@ err_bitmap:
 	bitmap_free(bitmap);
 err_nodes:
 	vfree(nodes);
-err:
-	return ret;
 }
 
-static int igt_bottomup(void *ignored)
+static void drm_test_mm_bottomup(struct kunit *test)
 {
 	const struct insert_mode *bottomup = &insert_modes[BOTTOMUP];
+
 	DRM_RND_STATE(prng, random_seed);
 	const unsigned int count = 8192;
 	unsigned int size;
@@ -1851,16 +1683,13 @@ static int igt_bottomup(void *ignored)
 	struct drm_mm mm;
 	struct drm_mm_node *nodes, *node, *next;
 	unsigned int *order, n, m, o = 0;
-	int ret;
 
-	/* Like igt_topdown, but instead of searching for the last hole,
+	/* Like drm_test_mm_topdown, but instead of searching for the last hole,
 	 * we search for the first.
 	 */
 
-	ret = -ENOMEM;
 	nodes = vzalloc(array_size(count, sizeof(*nodes)));
-	if (!nodes)
-		goto err;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	bitmap = bitmap_zalloc(count, GFP_KERNEL);
 	if (!bitmap)
@@ -1870,22 +1699,20 @@ static int igt_bottomup(void *ignored)
 	if (!order)
 		goto err_bitmap;
 
-	ret = -EINVAL;
 	for (size = 1; size <= 64; size <<= 1) {
-		drm_mm_init(&mm, 0, size*count);
+		drm_mm_init(&mm, 0, size * count);
 		for (n = 0; n < count; n++) {
-			if (!expect_insert(&mm, &nodes[n],
-					   size, 0, n,
-					   bottomup)) {
-				pr_err("bottomup insert failed, size %u step %d\n", size, n);
+			if (!expect_insert(test, &mm, &nodes[n], size, 0, n, bottomup)) {
+				KUNIT_FAIL(test,
+					   "bottomup insert failed, size %u step %d\n", size, n);
 				goto out;
 			}
 
-			if (!assert_one_hole(&mm, size*(n + 1), size*count))
+			if (!assert_one_hole(test, &mm, size * (n + 1), size * count))
 				goto out;
 		}
 
-		if (!assert_continuous(&mm, size))
+		if (!assert_continuous(test, &mm, size))
 			goto out;
 
 		drm_random_reorder(order, count, &prng);
@@ -1900,17 +1727,16 @@ static int igt_bottomup(void *ignored)
 				unsigned int first;
 
 				node = &nodes[order[(o + m) % count]];
-				if (!expect_insert(&mm, node,
-						   size, 0, 0,
-						   bottomup)) {
-					pr_err("insert failed, step %d/%d\n", m, n);
+				if (!expect_insert(test, &mm, node, size, 0, 0, bottomup)) {
+					KUNIT_FAIL(test, "insert failed, step %d/%d\n", m, n);
 					goto out;
 				}
 
 				first = find_first_bit(bitmap, count);
 				if (node_index(node) != first) {
-					pr_err("node %d/%d not inserted into bottom hole, expected %d, found %d\n",
-					       m, n, first, node_index(node));
+					KUNIT_FAIL(test,
+						   "node %d/%d not inserted into bottom hole, expected %d, found %d\n",
+						   m, n, first, node_index(node));
 					goto out;
 				}
 				__clear_bit(first, bitmap);
@@ -1927,7 +1753,6 @@ static int igt_bottomup(void *ignored)
 		cond_resched();
 	}
 
-	ret = 0;
 out:
 	drm_mm_for_each_node_safe(node, next, &mm)
 		drm_mm_remove_node(node);
@@ -1937,47 +1762,39 @@ err_bitmap:
 	bitmap_free(bitmap);
 err_nodes:
 	vfree(nodes);
-err:
-	return ret;
 }
 
-static int __igt_once(unsigned int mode)
+static void drm_test_mm_once(struct kunit *test, unsigned int mode)
 {
 	struct drm_mm mm;
 	struct drm_mm_node rsvd_lo, rsvd_hi, node;
-	int err;
 
 	drm_mm_init(&mm, 0, 7);
 
 	memset(&rsvd_lo, 0, sizeof(rsvd_lo));
 	rsvd_lo.start = 1;
 	rsvd_lo.size = 1;
-	err = drm_mm_reserve_node(&mm, &rsvd_lo);
-	if (err) {
-		pr_err("Could not reserve low node\n");
+	if (drm_mm_reserve_node(&mm, &rsvd_lo)) {
+		KUNIT_FAIL(test, "Could not reserve low node\n");
 		goto err;
 	}
 
 	memset(&rsvd_hi, 0, sizeof(rsvd_hi));
 	rsvd_hi.start = 5;
 	rsvd_hi.size = 1;
-	err = drm_mm_reserve_node(&mm, &rsvd_hi);
-	if (err) {
-		pr_err("Could not reserve low node\n");
+	if (drm_mm_reserve_node(&mm, &rsvd_hi)) {
+		KUNIT_FAIL(test, "Could not reserve low node\n");
 		goto err_lo;
 	}
 
 	if (!drm_mm_hole_follows(&rsvd_lo) || !drm_mm_hole_follows(&rsvd_hi)) {
-		pr_err("Expected a hole after lo and high nodes!\n");
-		err = -EINVAL;
+		KUNIT_FAIL(test, "Expected a hole after lo and high nodes!\n");
 		goto err_hi;
 	}
 
 	memset(&node, 0, sizeof(node));
-	err = drm_mm_insert_node_generic(&mm, &node, 2, 0, 0, mode);
-	if (err) {
-		pr_err("Could not insert the node into the available hole!\n");
-		err = -EINVAL;
+	if (drm_mm_insert_node_generic(&mm, &node, 2, 0, 0, mode)) {
+		KUNIT_FAIL(test, "Could not insert the node into the available hole!\n");
 		goto err_hi;
 	}
 
@@ -1988,23 +1805,20 @@ err_lo:
 	drm_mm_remove_node(&rsvd_lo);
 err:
 	drm_mm_takedown(&mm);
-	return err;
 }
 
-static int igt_lowest(void *ignored)
+static void drm_test_mm_lowest(struct kunit *test)
 {
-	return __igt_once(DRM_MM_INSERT_LOW);
+	drm_test_mm_once(test, DRM_MM_INSERT_LOW);
 }
 
-static int igt_highest(void *ignored)
+static void drm_test_mm_highest(struct kunit *test)
 {
-	return __igt_once(DRM_MM_INSERT_HIGH);
+	drm_test_mm_once(test, DRM_MM_INSERT_HIGH);
 }
 
 static void separate_adjacent_colors(const struct drm_mm_node *node,
-				     unsigned long color,
-				     u64 *start,
-				     u64 *end)
+				     unsigned long color, u64 *start, u64 *end)
 {
 	if (drm_mm_node_allocated(node) && node->color != color)
 		++*start;
@@ -2014,12 +1828,12 @@ static void separate_adjacent_colors(const struct drm_mm_node *node,
 		--*end;
 }
 
-static bool colors_abutt(const struct drm_mm_node *node)
+static bool colors_abutt(struct kunit *test, const struct drm_mm_node *node)
 {
 	if (!drm_mm_hole_follows(node) &&
 	    drm_mm_node_allocated(list_next_entry(node, node_list))) {
-		pr_err("colors abutt; %ld [%llx + %llx] is next to %ld [%llx + %llx]!\n",
-		       node->color, node->start, node->size,
+		KUNIT_FAIL(test, "colors abutt; %ld [%llx + %llx] is next to %ld [%llx + %llx]!\n",
+			   node->color, node->start, node->size,
 		       list_next_entry(node, node_list)->color,
 		       list_next_entry(node, node_list)->start,
 		       list_next_entry(node, node_list)->size);
@@ -2029,14 +1843,13 @@ static bool colors_abutt(const struct drm_mm_node *node)
 	return false;
 }
 
-static int igt_color(void *ignored)
+static void drm_test_mm_color(struct kunit *test)
 {
 	const unsigned int count = min(4096u, max_iterations);
 	const struct insert_mode *mode;
 	struct drm_mm mm;
 	struct drm_mm_node *node, *nn;
 	unsigned int n;
-	int ret = -EINVAL, err;
 
 	/* Color adjustment complicates everything. First we just check
 	 * that when we insert a node we apply any color_adjustment callback.
@@ -2049,15 +1862,11 @@ static int igt_color(void *ignored)
 
 	for (n = 1; n <= count; n++) {
 		node = kzalloc(sizeof(*node), GFP_KERNEL);
-		if (!node) {
-			ret = -ENOMEM;
+		if (!node)
 			goto out;
-		}
 
-		if (!expect_insert(&mm, node,
-				   n, 0, n,
-				   &insert_modes[0])) {
-			pr_err("insert failed, step %d\n", n);
+		if (!expect_insert(test, &mm, node, n, 0, n, &insert_modes[0])) {
+			KUNIT_FAIL(test, "insert failed, step %d\n", n);
 			kfree(node);
 			goto out;
 		}
@@ -2065,8 +1874,8 @@ static int igt_color(void *ignored)
 
 	drm_mm_for_each_node_safe(node, nn, &mm) {
 		if (node->color != node->size) {
-			pr_err("invalid color stored: expected %lld, found %ld\n",
-			       node->size, node->color);
+			KUNIT_FAIL(test, "invalid color stored: expected %lld, found %ld\n",
+				   node->size, node->color);
 
 			goto out;
 		}
@@ -2081,18 +1890,14 @@ static int igt_color(void *ignored)
 		u64 last;
 
 		node = kzalloc(sizeof(*node), GFP_KERNEL);
-		if (!node) {
-			ret = -ENOMEM;
+		if (!node)
 			goto out;
-		}
 
-		node->size = 1 + 2*count;
+		node->size = 1 + 2 * count;
 		node->color = node->size;
 
-		err = drm_mm_reserve_node(&mm, node);
-		if (err) {
-			pr_err("initial reserve failed!\n");
-			ret = err;
+		if (drm_mm_reserve_node(&mm, node)) {
+			KUNIT_FAIL(test, "initial reserve failed!\n");
 			goto out;
 		}
 
@@ -2102,19 +1907,15 @@ static int igt_color(void *ignored)
 			int rem;
 
 			node = kzalloc(sizeof(*node), GFP_KERNEL);
-			if (!node) {
-				ret = -ENOMEM;
+			if (!node)
 				goto out;
-			}
 
 			node->start = last;
 			node->size = n + count;
 			node->color = node->size;
 
-			err = drm_mm_reserve_node(&mm, node);
-			if (err != -ENOSPC) {
-				pr_err("reserve %d did not report color overlap! err=%d\n",
-				       n, err);
+			if (drm_mm_reserve_node(&mm, node) != -ENOSPC) {
+				KUNIT_FAIL(test, "reserve %d did not report color overlap!", n);
 				goto out;
 			}
 
@@ -2122,10 +1923,8 @@ static int igt_color(void *ignored)
 			rem = misalignment(node, n + count);
 			node->start += n + count - rem;
 
-			err = drm_mm_reserve_node(&mm, node);
-			if (err) {
-				pr_err("reserve %d failed, err=%d\n", n, err);
-				ret = err;
+			if (drm_mm_reserve_node(&mm, node)) {
+				KUNIT_FAIL(test, "reserve %d failed", n);
 				goto out;
 			}
 
@@ -2134,16 +1933,11 @@ static int igt_color(void *ignored)
 
 		for (n = 1; n <= count; n++) {
 			node = kzalloc(sizeof(*node), GFP_KERNEL);
-			if (!node) {
-				ret = -ENOMEM;
+			if (!node)
 				goto out;
-			}
 
-			if (!expect_insert(&mm, node,
-					   n, n, n,
-					   mode)) {
-				pr_err("%s insert failed, step %d\n",
-				       mode->name, n);
+			if (!expect_insert(test, &mm, node, n, n, n, mode)) {
+				KUNIT_FAIL(test, "%s insert failed, step %d\n", mode->name, n);
 				kfree(node);
 				goto out;
 			}
@@ -2153,19 +1947,21 @@ static int igt_color(void *ignored)
 			u64 rem;
 
 			if (node->color != node->size) {
-				pr_err("%s invalid color stored: expected %lld, found %ld\n",
-				       mode->name, node->size, node->color);
+				KUNIT_FAIL(test,
+					   "%s invalid color stored: expected %lld, found %ld\n",
+					   mode->name, node->size, node->color);
 
 				goto out;
 			}
 
-			if (colors_abutt(node))
+			if (colors_abutt(test, node))
 				goto out;
 
 			div64_u64_rem(node->start, node->size, &rem);
 			if (rem) {
-				pr_err("%s colored node misaligned, start=%llx expected alignment=%lld [rem=%lld]\n",
-				       mode->name, node->start, node->size, rem);
+				KUNIT_FAIL(test,
+					   "%s colored node misaligned, start=%llx expected alignment=%lld [rem=%lld]\n",
+					   mode->name, node->start, node->size, rem);
 				goto out;
 			}
 
@@ -2176,25 +1972,18 @@ static int igt_color(void *ignored)
 		cond_resched();
 	}
 
-	ret = 0;
 out:
 	drm_mm_for_each_node_safe(node, nn, &mm) {
 		drm_mm_remove_node(node);
 		kfree(node);
 	}
 	drm_mm_takedown(&mm);
-	return ret;
 }
 
-static int evict_color(struct drm_mm *mm,
-		       u64 range_start, u64 range_end,
-		       struct evict_node *nodes,
-		       unsigned int *order,
-		       unsigned int count,
-		       unsigned int size,
-		       unsigned int alignment,
-		       unsigned long color,
-		       const struct insert_mode *mode)
+static int evict_color(struct kunit *test, struct drm_mm *mm, u64 range_start,
+		       u64 range_end, struct evict_node *nodes, unsigned int *order,
+		unsigned int count, unsigned int size, unsigned int alignment,
+		unsigned long color, const struct insert_mode *mode)
 {
 	struct drm_mm_scan scan;
 	LIST_HEAD(evict_list);
@@ -2202,39 +1991,37 @@ static int evict_color(struct drm_mm *mm,
 	struct drm_mm_node tmp;
 	int err;
 
-	drm_mm_scan_init_with_range(&scan, mm,
-				    size, alignment, color,
-				    range_start, range_end,
-				    mode->mode);
-	if (!evict_nodes(&scan,
-			 nodes, order, count, true,
-			 &evict_list))
+	drm_mm_scan_init_with_range(&scan, mm, size, alignment, color, range_start,
+				    range_end, mode->mode);
+	if (!evict_nodes(test, &scan, nodes, order, count, true, &evict_list))
 		return -EINVAL;
 
 	memset(&tmp, 0, sizeof(tmp));
 	err = drm_mm_insert_node_generic(mm, &tmp, size, alignment, color,
 					 DRM_MM_INSERT_EVICT);
 	if (err) {
-		pr_err("Failed to insert into eviction hole: size=%d, align=%d, color=%lu, err=%d\n",
-		       size, alignment, color, err);
-		show_scan(&scan);
-		show_holes(mm, 3);
+		KUNIT_FAIL(test,
+			   "Failed to insert into eviction hole: size=%d, align=%d, color=%lu, err=%d\n",
+			   size, alignment, color, err);
+		show_scan(test, &scan);
+		show_holes(test, mm, 3);
 		return err;
 	}
 
 	if (tmp.start < range_start || tmp.start + tmp.size > range_end) {
-		pr_err("Inserted [address=%llu + %llu] did not fit into the request range [%llu, %llu]\n",
-		       tmp.start, tmp.size, range_start, range_end);
+		KUNIT_FAIL(test,
+			   "Inserted [address=%llu + %llu] did not fit into the request range [%llu, %llu]\n",
+			   tmp.start, tmp.size, range_start, range_end);
 		err = -EINVAL;
 	}
 
-	if (colors_abutt(&tmp))
+	if (colors_abutt(test, &tmp))
 		err = -EINVAL;
 
-	if (!assert_node(&tmp, mm, size, alignment, color)) {
-		pr_err("Inserted did not fit the eviction hole: size=%lld [%d], align=%d [rem=%lld], start=%llx\n",
-		       tmp.size, size,
-		       alignment, misalignment(&tmp, alignment), tmp.start);
+	if (!assert_node(test, &tmp, mm, size, alignment, color)) {
+		KUNIT_FAIL(test,
+			   "Inserted did not fit the eviction hole: size=%lld [%d], align=%d [rem=%lld], start=%llx\n",
+			   tmp.size, size, alignment, misalignment(&tmp, alignment), tmp.start);
 		err = -EINVAL;
 	}
 
@@ -2245,8 +2032,8 @@ static int evict_color(struct drm_mm *mm,
 	list_for_each_entry(e, &evict_list, link) {
 		err = drm_mm_reserve_node(mm, &e->node);
 		if (err) {
-			pr_err("Failed to reinsert node after eviction: start=%llx\n",
-			       e->node.start);
+			KUNIT_FAIL(test, "Failed to reinsert node after eviction: start=%llx\n",
+				   e->node.start);
 			return err;
 		}
 	}
@@ -2255,7 +2042,7 @@ static int evict_color(struct drm_mm *mm,
 	return 0;
 }
 
-static int igt_color_evict(void *ignored)
+static void drm_test_mm_color_evict(struct kunit *test)
 {
 	DRM_RND_STATE(prng, random_seed);
 	const unsigned int total_size = min(8192u, max_iterations);
@@ -2265,7 +2052,6 @@ static int igt_color_evict(void *ignored)
 	struct evict_node *nodes;
 	struct drm_mm_node *node, *next;
 	unsigned int *order, n;
-	int ret, err;
 
 	/* Check that the drm_mm_scan also honours color adjustment when
 	 * choosing its victims to create a hole. Our color_adjust does not
@@ -2273,23 +2059,20 @@ static int igt_color_evict(void *ignored)
 	 * enlarging the set of victims that must be evicted.
 	 */
 
-	ret = -ENOMEM;
 	nodes = vzalloc(array_size(total_size, sizeof(*nodes)));
-	if (!nodes)
-		goto err;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	order = drm_random_order(total_size, &prng);
 	if (!order)
 		goto err_nodes;
 
-	ret = -EINVAL;
-	drm_mm_init(&mm, 0, 2*total_size - 1);
+	drm_mm_init(&mm, 0, 2 * total_size - 1);
 	mm.color_adjust = separate_adjacent_colors;
 	for (n = 0; n < total_size; n++) {
-		if (!expect_insert(&mm, &nodes[n].node,
+		if (!expect_insert(test, &mm, &nodes[n].node,
 				   1, 0, color++,
 				   &insert_modes[0])) {
-			pr_err("insert failed, step %d\n", n);
+			KUNIT_FAIL(test, "insert failed, step %d\n", n);
 			goto out;
 		}
 	}
@@ -2297,26 +2080,19 @@ static int igt_color_evict(void *ignored)
 	for (mode = evict_modes; mode->name; mode++) {
 		for (n = 1; n <= total_size; n <<= 1) {
 			drm_random_reorder(order, total_size, &prng);
-			err = evict_color(&mm, 0, U64_MAX,
-					  nodes, order, total_size,
-					  n, 1, color++,
-					  mode);
-			if (err) {
-				pr_err("%s evict_color(size=%u) failed\n",
-				       mode->name, n);
+			if (evict_color(test, &mm, 0, U64_MAX, nodes, order, total_size,
+					n, 1, color++, mode)) {
+				KUNIT_FAIL(test, "%s evict_color(size=%u) failed\n", mode->name, n);
 				goto out;
 			}
 		}
 
 		for (n = 1; n < total_size; n <<= 1) {
 			drm_random_reorder(order, total_size, &prng);
-			err = evict_color(&mm, 0, U64_MAX,
-					  nodes, order, total_size,
-					  total_size/2, n, color++,
-					  mode);
-			if (err) {
-				pr_err("%s evict_color(size=%u, alignment=%u) failed\n",
-				       mode->name, total_size/2, n);
+			if (evict_color(test, &mm, 0, U64_MAX, nodes, order, total_size,
+					total_size / 2, n, color++, mode)) {
+				KUNIT_FAIL(test, "%s evict_color(size=%u, alignment=%u) failed\n",
+					   mode->name, total_size / 2, n);
 				goto out;
 			}
 		}
@@ -2327,13 +2103,10 @@ static int igt_color_evict(void *ignored)
 			DRM_MM_BUG_ON(!nsize);
 
 			drm_random_reorder(order, total_size, &prng);
-			err = evict_color(&mm, 0, U64_MAX,
-					  nodes, order, total_size,
-					  nsize, n, color++,
-					  mode);
-			if (err) {
-				pr_err("%s evict_color(size=%u, alignment=%u) failed\n",
-				       mode->name, nsize, n);
+			if (evict_color(test, &mm, 0, U64_MAX, nodes, order, total_size,
+					nsize, n, color++, mode)) {
+				KUNIT_FAIL(test, "%s evict_color(size=%u, alignment=%u) failed\n",
+					   mode->name, nsize, n);
 				goto out;
 			}
 		}
@@ -2341,21 +2114,16 @@ static int igt_color_evict(void *ignored)
 		cond_resched();
 	}
 
-	ret = 0;
 out:
-	if (ret)
-		show_mm(&mm);
 	drm_mm_for_each_node_safe(node, next, &mm)
 		drm_mm_remove_node(node);
 	drm_mm_takedown(&mm);
 	kfree(order);
 err_nodes:
 	vfree(nodes);
-err:
-	return ret;
 }
 
-static int igt_color_evict_range(void *ignored)
+static void drm_test_mm_color_evict_range(struct kunit *test)
 {
 	DRM_RND_STATE(prng, random_seed);
 	const unsigned int total_size = 8192;
@@ -2368,29 +2136,25 @@ static int igt_color_evict_range(void *ignored)
 	struct evict_node *nodes;
 	struct drm_mm_node *node, *next;
 	unsigned int *order, n;
-	int ret, err;
 
-	/* Like igt_color_evict(), but limited to small portion of the full
+	/* Like drm_test_mm_color_evict(), but limited to small portion of the full
 	 * drm_mm range.
 	 */
 
-	ret = -ENOMEM;
 	nodes = vzalloc(array_size(total_size, sizeof(*nodes)));
-	if (!nodes)
-		goto err;
+	KUNIT_ASSERT_TRUE(test, nodes);
 
 	order = drm_random_order(total_size, &prng);
 	if (!order)
 		goto err_nodes;
 
-	ret = -EINVAL;
-	drm_mm_init(&mm, 0, 2*total_size - 1);
+	drm_mm_init(&mm, 0, 2 * total_size - 1);
 	mm.color_adjust = separate_adjacent_colors;
 	for (n = 0; n < total_size; n++) {
-		if (!expect_insert(&mm, &nodes[n].node,
+		if (!expect_insert(test, &mm, &nodes[n].node,
 				   1, 0, color++,
 				   &insert_modes[0])) {
-			pr_err("insert failed, step %d\n", n);
+			KUNIT_FAIL(test, "insert failed, step %d\n", n);
 			goto out;
 		}
 	}
@@ -2398,26 +2162,22 @@ static int igt_color_evict_range(void *ignored)
 	for (mode = evict_modes; mode->name; mode++) {
 		for (n = 1; n <= range_size; n <<= 1) {
 			drm_random_reorder(order, range_size, &prng);
-			err = evict_color(&mm, range_start, range_end,
-					  nodes, order, total_size,
-					  n, 1, color++,
-					  mode);
-			if (err) {
-				pr_err("%s evict_color(size=%u) failed for range [%x, %x]\n",
-				       mode->name, n, range_start, range_end);
+			if (evict_color(test, &mm, range_start, range_end, nodes, order,
+					total_size, n, 1, color++, mode)) {
+				KUNIT_FAIL(test,
+					   "%s evict_color(size=%u) failed for range [%x, %x]\n",
+						mode->name, n, range_start, range_end);
 				goto out;
 			}
 		}
 
 		for (n = 1; n < range_size; n <<= 1) {
 			drm_random_reorder(order, total_size, &prng);
-			err = evict_color(&mm, range_start, range_end,
-					  nodes, order, total_size,
-					  range_size/2, n, color++,
-					  mode);
-			if (err) {
-				pr_err("%s evict_color(size=%u, alignment=%u) failed for range [%x, %x]\n",
-				       mode->name, total_size/2, n, range_start, range_end);
+			if (evict_color(test, &mm, range_start, range_end, nodes, order,
+					total_size, range_size / 2, n, color++, mode)) {
+				KUNIT_FAIL(test,
+					   "%s evict_color(size=%u, alignment=%u) failed for range [%x, %x]\n",
+					   mode->name, total_size / 2, n, range_start, range_end);
 				goto out;
 			}
 		}
@@ -2428,13 +2188,11 @@ static int igt_color_evict_range(void *ignored)
 			DRM_MM_BUG_ON(!nsize);
 
 			drm_random_reorder(order, total_size, &prng);
-			err = evict_color(&mm, range_start, range_end,
-					  nodes, order, total_size,
-					  nsize, n, color++,
-					  mode);
-			if (err) {
-				pr_err("%s evict_color(size=%u, alignment=%u) failed for range [%x, %x]\n",
-				       mode->name, nsize, n, range_start, range_end);
+			if (evict_color(test, &mm, range_start, range_end, nodes, order,
+					total_size, nsize, n, color++, mode)) {
+				KUNIT_FAIL(test,
+					   "%s evict_color(size=%u, alignment=%u) failed for range [%x, %x]\n",
+					   mode->name, nsize, n, range_start, range_end);
 				goto out;
 			}
 		}
@@ -2442,46 +2200,57 @@ static int igt_color_evict_range(void *ignored)
 		cond_resched();
 	}
 
-	ret = 0;
 out:
-	if (ret)
-		show_mm(&mm);
 	drm_mm_for_each_node_safe(node, next, &mm)
 		drm_mm_remove_node(node);
 	drm_mm_takedown(&mm);
 	kfree(order);
 err_nodes:
 	vfree(nodes);
-err:
-	return ret;
 }
 
-#include "drm_selftest.c"
-
-static int __init test_drm_mm_init(void)
+static int drm_mm_init_test(struct kunit *test)
 {
-	int err;
-
 	while (!random_seed)
 		random_seed = get_random_int();
 
-	pr_info("Testing DRM range manager (struct drm_mm), with random_seed=0x%x max_iterations=%u max_prime=%u\n",
-		random_seed, max_iterations, max_prime);
-	err = run_selftests(selftests, ARRAY_SIZE(selftests), NULL);
-
-	return err > 0 ? 0 : err;
-}
-
-static void __exit test_drm_mm_exit(void)
-{
+	return 0;
 }
 
-module_init(test_drm_mm_init);
-module_exit(test_drm_mm_exit);
-
 module_param(random_seed, uint, 0400);
 module_param(max_iterations, uint, 0400);
 module_param(max_prime, uint, 0400);
 
+static struct kunit_case drm_mm_tests[] = {
+	KUNIT_CASE(drm_test_mm_init),
+	KUNIT_CASE(drm_test_mm_debug),
+	KUNIT_CASE(drm_test_mm_reserve),
+	KUNIT_CASE(drm_test_mm_insert),
+	KUNIT_CASE(drm_test_mm_replace),
+	KUNIT_CASE(drm_test_mm_insert_range),
+	KUNIT_CASE(drm_test_mm_frag),
+	KUNIT_CASE(drm_test_mm_align),
+	KUNIT_CASE(drm_test_mm_align32),
+	KUNIT_CASE(drm_test_mm_align64),
+	KUNIT_CASE(drm_test_mm_evict),
+	KUNIT_CASE(drm_test_mm_evict_range),
+	KUNIT_CASE(drm_test_mm_topdown),
+	KUNIT_CASE(drm_test_mm_bottomup),
+	KUNIT_CASE(drm_test_mm_lowest),
+	KUNIT_CASE(drm_test_mm_highest),
+	KUNIT_CASE(drm_test_mm_color),
+	KUNIT_CASE(drm_test_mm_color_evict),
+	KUNIT_CASE(drm_test_mm_color_evict_range),
+	{}
+};
+
+static struct kunit_suite drm_mm_test_suite = {
+	.name = "drm_mm",
+	.init = drm_mm_init_test,
+	.test_cases = drm_mm_tests,
+};
+
+kunit_test_suite(drm_mm_test_suite);
+
 MODULE_AUTHOR("Intel Corporation");
 MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/selftests/test-drm_plane_helper.c b/drivers/gpu/drm/tests/drm_plane_helper_test.c
index 64e8938ab194..ec71af791f1f 100644
--- a/drivers/gpu/drm/selftests/test-drm_plane_helper.c
+++ b/drivers/gpu/drm/tests/drm_plane_helper_test.c
@@ -1,20 +1,19 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
  * Test cases for the drm_plane_helper functions
+ *
+ * Copyright (c) 2022 Maíra Canal <mairacanal@riseup.net>
  */
 
-#define pr_fmt(fmt) "drm_plane_helper: " fmt
+#include <kunit/test.h>
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_modes.h>
 
-#include "test-drm_modeset_common.h"
-
 static void set_src(struct drm_plane_state *plane_state,
-		    unsigned src_x, unsigned src_y,
-		    unsigned src_w, unsigned src_h)
+		    unsigned int src_x, unsigned int src_y,
+		    unsigned int src_w, unsigned int src_h)
 {
 	plane_state->src_x = src_x;
 	plane_state->src_y = src_y;
@@ -23,8 +22,8 @@ static void set_src(struct drm_plane_state *plane_state,
 }
 
 static bool check_src_eq(struct drm_plane_state *plane_state,
-			 unsigned src_x, unsigned src_y,
-			 unsigned src_w, unsigned src_h)
+			 unsigned int src_x, unsigned int src_y,
+			 unsigned int src_w, unsigned int src_h)
 {
 	if (plane_state->src.x1 < 0) {
 		pr_err("src x coordinate %x should never be below 0.\n", plane_state->src.x1);
@@ -50,7 +49,7 @@ static bool check_src_eq(struct drm_plane_state *plane_state,
 
 static void set_crtc(struct drm_plane_state *plane_state,
 		     int crtc_x, int crtc_y,
-		     unsigned crtc_w, unsigned crtc_h)
+		     unsigned int crtc_w, unsigned int crtc_h)
 {
 	plane_state->crtc_x = crtc_x;
 	plane_state->crtc_y = crtc_y;
@@ -60,7 +59,7 @@ static void set_crtc(struct drm_plane_state *plane_state,
 
 static bool check_crtc_eq(struct drm_plane_state *plane_state,
 			  int crtc_x, int crtc_y,
-			  unsigned crtc_w, unsigned crtc_h)
+			  unsigned int crtc_w, unsigned int crtc_h)
 {
 	if (plane_state->dst.x1 != crtc_x ||
 	    plane_state->dst.y1 != crtc_y ||
@@ -74,7 +73,7 @@ static bool check_crtc_eq(struct drm_plane_state *plane_state,
 	return true;
 }
 
-int igt_check_plane_state(void *ignored)
+static void drm_test_check_plane_state(struct kunit *test)
 {
 	int ret;
 
@@ -83,9 +82,8 @@ int igt_check_plane_state(void *ignored)
 		.enable = true,
 		.active = true,
 		.mode = {
-			DRM_MODE("1024x768", 0, 65000, 1024, 1048,
-				1184, 1344, 0, 768, 771, 777, 806, 0,
-				DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC)
+			DRM_MODE("1024x768", 0, 65000, 1024, 1048, 1184, 1344, 0, 768, 771,
+				 777, 806, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC)
 		},
 	};
 	static struct drm_plane plane = {
@@ -106,119 +104,134 @@ int igt_check_plane_state(void *ignored)
 	set_src(&plane_state, 0, 0, fb.width << 16, fb.height << 16);
 	set_crtc(&plane_state, 0, 0, fb.width, fb.height);
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, false);
-	FAIL(ret < 0, "Simple clipping check should pass\n");
-	FAIL_ON(!plane_state.visible);
-	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 1024 << 16, 768 << 16));
-	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+	KUNIT_EXPECT_FALSE_MSG(test, ret, 0, "Simple clipping check should pass\n");
+	KUNIT_EXPECT_TRUE(test, plane_state.visible);
+	KUNIT_EXPECT_TRUE(test, check_src_eq(&plane_state, 0, 0, 1024 << 16, 768 << 16));
+	KUNIT_EXPECT_TRUE(test, check_crtc_eq(&plane_state, 0, 0, 1024, 768));
 
 	/* Rotated clipping + reflection, no scaling. */
 	plane_state.rotation = DRM_MODE_ROTATE_90 | DRM_MODE_REFLECT_X;
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, false);
-	FAIL(ret < 0, "Rotated clipping check should pass\n");
-	FAIL_ON(!plane_state.visible);
-	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 768 << 16, 1024 << 16));
-	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+	KUNIT_EXPECT_FALSE_MSG(test, ret, 0, "Rotated clipping check should pass\n");
+	KUNIT_EXPECT_TRUE(test, plane_state.visible);
+	KUNIT_EXPECT_TRUE(test, check_src_eq(&plane_state, 0, 0, 768 << 16, 1024 << 16));
+	KUNIT_EXPECT_TRUE(test, check_crtc_eq(&plane_state, 0, 0, 1024, 768));
 	plane_state.rotation = DRM_MODE_ROTATE_0;
 
 	/* Check whether positioning works correctly. */
 	set_src(&plane_state, 0, 0, 1023 << 16, 767 << 16);
 	set_crtc(&plane_state, 0, 0, 1023, 767);
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, false);
-	FAIL(!ret, "Should not be able to position on the crtc with can_position=false\n");
+	KUNIT_EXPECT_TRUE_MSG(test, ret,
+			      "Should not be able to position on the crtc with can_position=false\n");
 
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  true, false);
-	FAIL(ret < 0, "Simple positioning should work\n");
-	FAIL_ON(!plane_state.visible);
-	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 1023 << 16, 767 << 16));
-	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1023, 767));
+	KUNIT_EXPECT_FALSE_MSG(test, ret, 0, "Simple positioning should work\n");
+	KUNIT_EXPECT_TRUE(test, plane_state.visible);
+	KUNIT_EXPECT_TRUE(test, check_src_eq(&plane_state, 0, 0, 1023 << 16, 767 << 16));
+	KUNIT_EXPECT_TRUE(test, check_crtc_eq(&plane_state, 0, 0, 1023, 767));
 
 	/* Simple scaling tests. */
 	set_src(&plane_state, 0, 0, 512 << 16, 384 << 16);
 	set_crtc(&plane_state, 0, 0, 1024, 768);
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
 						  0x8001,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, false);
-	FAIL(!ret, "Upscaling out of range should fail.\n");
+	KUNIT_EXPECT_TRUE_MSG(test, ret, "Upscaling out of range should fail.\n");
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
 						  0x8000,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, false);
-	FAIL(ret < 0, "Upscaling exactly 2x should work\n");
-	FAIL_ON(!plane_state.visible);
-	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 512 << 16, 384 << 16));
-	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+	KUNIT_EXPECT_FALSE_MSG(test, ret, 0, "Upscaling exactly 2x should work\n");
+	KUNIT_EXPECT_TRUE(test, plane_state.visible);
+	KUNIT_EXPECT_TRUE(test, check_src_eq(&plane_state, 0, 0, 512 << 16, 384 << 16));
+	KUNIT_EXPECT_TRUE(test, check_crtc_eq(&plane_state, 0, 0, 1024, 768));
 
 	set_src(&plane_state, 0, 0, 2048 << 16, 1536 << 16);
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  0x1ffff, false, false);
-	FAIL(!ret, "Downscaling out of range should fail.\n");
+	KUNIT_EXPECT_TRUE_MSG(test, ret, "Downscaling out of range should fail.\n");
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  0x20000, false, false);
-	FAIL(ret < 0, "Should succeed with exact scaling limit\n");
-	FAIL_ON(!plane_state.visible);
-	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 2048 << 16, 1536 << 16));
-	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+	KUNIT_EXPECT_FALSE_MSG(test, ret, 0, "Should succeed with exact scaling limit\n");
+	KUNIT_EXPECT_TRUE(test, plane_state.visible);
+	KUNIT_EXPECT_TRUE(test, check_src_eq(&plane_state, 0, 0, 2048 << 16, 1536 << 16));
+	KUNIT_EXPECT_TRUE(test, check_crtc_eq(&plane_state, 0, 0, 1024, 768));
 
 	/* Testing rounding errors. */
 	set_src(&plane_state, 0, 0, 0x40001, 0x40001);
 	set_crtc(&plane_state, 1022, 766, 4, 4);
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  0x10001,
 						  true, false);
-	FAIL(ret < 0, "Should succeed by clipping to exact multiple");
-	FAIL_ON(!plane_state.visible);
-	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 2 << 16, 2 << 16));
-	FAIL_ON(!check_crtc_eq(&plane_state, 1022, 766, 2, 2));
+	KUNIT_EXPECT_FALSE_MSG(test, ret, 0, "Should succeed by clipping to exact multiple");
+	KUNIT_EXPECT_TRUE(test, plane_state.visible);
+	KUNIT_EXPECT_TRUE(test, check_src_eq(&plane_state, 0, 0, 2 << 16, 2 << 16));
+	KUNIT_EXPECT_TRUE(test, check_crtc_eq(&plane_state, 1022, 766, 2, 2));
 
 	set_src(&plane_state, 0x20001, 0x20001, 0x4040001, 0x3040001);
 	set_crtc(&plane_state, -2, -2, 1028, 772);
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  0x10001,
 						  false, false);
-	FAIL(ret < 0, "Should succeed by clipping to exact multiple");
-	FAIL_ON(!plane_state.visible);
-	FAIL_ON(!check_src_eq(&plane_state, 0x40002, 0x40002, 1024 << 16, 768 << 16));
-	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
+	KUNIT_EXPECT_FALSE_MSG(test, ret, 0, "Should succeed by clipping to exact multiple");
+	KUNIT_EXPECT_TRUE(test, plane_state.visible);
+	KUNIT_EXPECT_TRUE(test, check_src_eq(&plane_state, 0x40002, 0x40002,
+					     1024 << 16, 768 << 16));
+	KUNIT_EXPECT_TRUE(test, check_crtc_eq(&plane_state, 0, 0, 1024, 768));
 
 	set_src(&plane_state, 0, 0, 0x3ffff, 0x3ffff);
 	set_crtc(&plane_state, 1022, 766, 4, 4);
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
 						  0xffff,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  true, false);
-	FAIL(ret < 0, "Should succeed by clipping to exact multiple");
-	FAIL_ON(!plane_state.visible);
+	KUNIT_EXPECT_FALSE_MSG(test, ret, 0, "Should succeed by clipping to exact multiple");
+	KUNIT_EXPECT_TRUE(test, plane_state.visible);
 	/* Should not be rounded to 0x20001, which would be upscaling. */
-	FAIL_ON(!check_src_eq(&plane_state, 0, 0, 2 << 16, 2 << 16));
-	FAIL_ON(!check_crtc_eq(&plane_state, 1022, 766, 2, 2));
+	KUNIT_EXPECT_TRUE(test, check_src_eq(&plane_state, 0, 0, 2 << 16, 2 << 16));
+	KUNIT_EXPECT_TRUE(test, check_crtc_eq(&plane_state, 1022, 766, 2, 2));
 
 	set_src(&plane_state, 0x1ffff, 0x1ffff, 0x403ffff, 0x303ffff);
 	set_crtc(&plane_state, -2, -2, 1028, 772);
 	ret = drm_atomic_helper_check_plane_state(&plane_state, &crtc_state,
 						  0xffff,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, false);
-	FAIL(ret < 0, "Should succeed by clipping to exact multiple");
-	FAIL_ON(!plane_state.visible);
-	FAIL_ON(!check_src_eq(&plane_state, 0x3fffe, 0x3fffe, 1024 << 16, 768 << 16));
-	FAIL_ON(!check_crtc_eq(&plane_state, 0, 0, 1024, 768));
-
-	return 0;
+	KUNIT_EXPECT_FALSE_MSG(test, ret, 0, "Should succeed by clipping to exact multiple");
+	KUNIT_EXPECT_TRUE(test, plane_state.visible);
+	KUNIT_EXPECT_TRUE(test, check_src_eq(&plane_state, 0x3fffe, 0x3fffe,
+					     1024 << 16, 768 << 16));
+	KUNIT_EXPECT_TRUE(test, check_crtc_eq(&plane_state, 0, 0, 1024, 768));
 }
+
+static struct kunit_case drm_plane_helper_test[] = {
+	KUNIT_CASE(drm_test_check_plane_state),
+	{}
+};
+
+static struct kunit_suite drm_plane_helper_test_suite = {
+	.name = "drm_plane_helper",
+	.test_cases = drm_plane_helper_test,
+};
+
+kunit_test_suite(drm_plane_helper_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tests/drm_rect_test.c b/drivers/gpu/drm/tests/drm_rect_test.c
new file mode 100644
index 000000000000..e9809ea32696
--- /dev/null
+++ b/drivers/gpu/drm/tests/drm_rect_test.c
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Test cases for the drm_rect functions
+ *
+ * Copyright (c) 2022 Maíra Canal <mairacanal@riseup.net>
+ */
+
+#include <kunit/test.h>
+
+#include <drm/drm_rect.h>
+
+static void drm_test_rect_clip_scaled_div_by_zero(struct kunit *test)
+{
+	struct drm_rect src, dst, clip;
+	bool visible;
+
+	/*
+	 * Make sure we don't divide by zero when dst
+	 * width/height is zero and dst and clip do not intersect.
+	 */
+	drm_rect_init(&src, 0, 0, 0, 0);
+	drm_rect_init(&dst, 0, 0, 0, 0);
+	drm_rect_init(&clip, 1, 1, 1, 1);
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, visible, "Destination not be visible\n");
+	KUNIT_EXPECT_FALSE_MSG(test, drm_rect_visible(&src), "Source should not be visible\n");
+
+	drm_rect_init(&src, 0, 0, 0, 0);
+	drm_rect_init(&dst, 3, 3, 0, 0);
+	drm_rect_init(&clip, 1, 1, 1, 1);
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, visible, "Destination not be visible\n");
+	KUNIT_EXPECT_FALSE_MSG(test, drm_rect_visible(&src), "Source should not be visible\n");
+}
+
+static void drm_test_rect_clip_scaled_not_clipped(struct kunit *test)
+{
+	struct drm_rect src, dst, clip;
+	bool visible;
+
+	/* 1:1 scaling */
+	drm_rect_init(&src, 0, 0, 1 << 16, 1 << 16);
+	drm_rect_init(&dst, 0, 0, 1, 1);
+	drm_rect_init(&clip, 0, 0, 1, 1);
+
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, src.x1 != 0 || src.x2 != 1 << 16 ||
+			       src.y1 != 0 || src.y2 != 1 << 16, "Source badly clipped\n");
+	KUNIT_EXPECT_FALSE_MSG(test, dst.x1 != 0 || dst.x2 != 1 ||
+			       dst.y1 != 0 || dst.y2 != 1, "Destination badly clipped\n");
+	KUNIT_EXPECT_TRUE_MSG(test, visible, "Destination should be visible\n");
+	KUNIT_EXPECT_TRUE_MSG(test, drm_rect_visible(&src), "Source should be visible\n");
+
+	/* 2:1 scaling */
+	drm_rect_init(&src, 0, 0, 2 << 16, 2 << 16);
+	drm_rect_init(&dst, 0, 0, 1, 1);
+	drm_rect_init(&clip, 0, 0, 1, 1);
+
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, src.x1 != 0 || src.x2 != 2 << 16 ||
+			       src.y1 != 0 || src.y2 != 2 << 16, "Source badly clipped\n");
+	KUNIT_EXPECT_FALSE_MSG(test, dst.x1 != 0 || dst.x2 != 1 ||
+			       dst.y1 != 0 || dst.y2 != 1, "Destination badly clipped\n");
+	KUNIT_EXPECT_TRUE_MSG(test, visible, "Destination should be visible\n");
+	KUNIT_EXPECT_TRUE_MSG(test, drm_rect_visible(&src), "Source should be visible\n");
+
+	/* 1:2 scaling */
+	drm_rect_init(&src, 0, 0, 1 << 16, 1 << 16);
+	drm_rect_init(&dst, 0, 0, 2, 2);
+	drm_rect_init(&clip, 0, 0, 2, 2);
+
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, src.x1 != 0 || src.x2 != 1 << 16 ||
+			       src.y1 != 0 || src.y2 != 1 << 16, "Source badly clipped\n");
+	KUNIT_EXPECT_FALSE_MSG(test, dst.x1 != 0 || dst.x2 != 2 ||
+			       dst.y1 != 0 || dst.y2 != 2, "Destination badly clipped\n");
+	KUNIT_EXPECT_TRUE_MSG(test, visible, "Destination should be visible\n");
+	KUNIT_EXPECT_TRUE_MSG(test, drm_rect_visible(&src), "Source should be visible\n");
+}
+
+static void drm_test_rect_clip_scaled_clipped(struct kunit *test)
+{
+	struct drm_rect src, dst, clip;
+	bool visible;
+
+	/* 1:1 scaling top/left clip */
+	drm_rect_init(&src, 0, 0, 2 << 16, 2 << 16);
+	drm_rect_init(&dst, 0, 0, 2, 2);
+	drm_rect_init(&clip, 0, 0, 1, 1);
+
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, src.x1 != 0 || src.x2 != 1 << 16 ||
+			       src.y1 != 0 || src.y2 != 1 << 16, "Source badly clipped\n");
+	KUNIT_EXPECT_FALSE_MSG(test, dst.x1 != 0 || dst.x2 != 1 ||
+			       dst.y1 != 0 || dst.y2 != 1, "Destination badly clipped\n");
+	KUNIT_EXPECT_TRUE_MSG(test, visible, "Destination should be visible\n");
+	KUNIT_EXPECT_TRUE_MSG(test, drm_rect_visible(&src), "Source should be visible\n");
+
+	/* 1:1 scaling bottom/right clip */
+	drm_rect_init(&src, 0, 0, 2 << 16, 2 << 16);
+	drm_rect_init(&dst, 0, 0, 2, 2);
+	drm_rect_init(&clip, 1, 1, 1, 1);
+
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, src.x1 != 1 << 16 || src.x2 != 2 << 16 ||
+			       src.y1 != 1 << 16 || src.y2 != 2 << 16, "Source badly clipped\n");
+	KUNIT_EXPECT_FALSE_MSG(test, dst.x1 != 1 || dst.x2 != 2 || dst.y1 != 1 ||
+			       dst.y2 != 2, "Destination badly clipped\n");
+	KUNIT_EXPECT_TRUE_MSG(test, visible, "Destination should be visible\n");
+	KUNIT_EXPECT_TRUE_MSG(test, drm_rect_visible(&src), "Source should be visible\n");
+
+	/* 2:1 scaling top/left clip */
+	drm_rect_init(&src, 0, 0, 4 << 16, 4 << 16);
+	drm_rect_init(&dst, 0, 0, 2, 2);
+	drm_rect_init(&clip, 0, 0, 1, 1);
+
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, src.x1 != 0 || src.x2 != 2 << 16 ||
+			       src.y1 != 0 || src.y2 != 2 << 16, "Source badly clipped\n");
+	KUNIT_EXPECT_FALSE_MSG(test, dst.x1 != 0 || dst.x2 != 1 || dst.y1 != 0 ||
+			       dst.y2 != 1, "Destination badly clipped\n");
+	KUNIT_EXPECT_TRUE_MSG(test, visible, "Destination should be visible\n");
+	KUNIT_EXPECT_TRUE_MSG(test, drm_rect_visible(&src), "Source should be visible\n");
+
+	/* 2:1 scaling bottom/right clip */
+	drm_rect_init(&src, 0, 0, 4 << 16, 4 << 16);
+	drm_rect_init(&dst, 0, 0, 2, 2);
+	drm_rect_init(&clip, 1, 1, 1, 1);
+
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, src.x1 != 2 << 16 || src.x2 != 4 << 16 ||
+			       src.y1 != 2 << 16 || src.y2 != 4 << 16, "Source badly clipped\n");
+	KUNIT_EXPECT_FALSE_MSG(test, dst.x1 != 1 || dst.x2 != 2 || dst.y1 != 1 ||
+			       dst.y2 != 2, "Destination badly clipped\n");
+	KUNIT_EXPECT_TRUE_MSG(test, visible, "Destination should be visible\n");
+	KUNIT_EXPECT_TRUE_MSG(test, drm_rect_visible(&src), "Source should be visible\n");
+
+	/* 1:2 scaling top/left clip */
+	drm_rect_init(&src, 0, 0, 2 << 16, 2 << 16);
+	drm_rect_init(&dst, 0, 0, 4, 4);
+	drm_rect_init(&clip, 0, 0, 2, 2);
+
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, src.x1 != 0 || src.x2 != 1 << 16 ||
+			       src.y1 != 0 || src.y2 != 1 << 16, "Source badly clipped\n");
+	KUNIT_EXPECT_FALSE_MSG(test, dst.x1 != 0 || dst.x2 != 2 || dst.y1 != 0 ||
+			       dst.y2 != 2, "Destination badly clipped\n");
+	KUNIT_EXPECT_TRUE_MSG(test, visible, "Destination should be visible\n");
+	KUNIT_EXPECT_TRUE_MSG(test, drm_rect_visible(&src), "Source should be visible\n");
+
+	/* 1:2 scaling bottom/right clip */
+	drm_rect_init(&src, 0, 0, 2 << 16, 2 << 16);
+	drm_rect_init(&dst, 0, 0, 4, 4);
+	drm_rect_init(&clip, 2, 2, 2, 2);
+
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, src.x1 != 1 << 16 || src.x2 != 2 << 16 ||
+			       src.y1 != 1 << 16 || src.y2 != 2 << 16, "Source badly clipped\n");
+	KUNIT_EXPECT_FALSE_MSG(test, dst.x1 != 2 || dst.x2 != 4 || dst.y1 != 2 ||
+			       dst.y2 != 4, "Destination badly clipped\n");
+	KUNIT_EXPECT_TRUE_MSG(test, visible, "Destination should be visible\n");
+	KUNIT_EXPECT_TRUE_MSG(test, drm_rect_visible(&src), "Source should be visible\n");
+}
+
+static void drm_test_rect_clip_scaled_signed_vs_unsigned(struct kunit *test)
+{
+	struct drm_rect src, dst, clip;
+	bool visible;
+
+	/*
+	 * 'clip.x2 - dst.x1 >= dst width' could result a negative
+	 * src rectangle width which is no longer expected by the
+	 * code as it's using unsigned types. This could lead to
+	 * the clipped source rectangle appering visible when it
+	 * should have been fully clipped. Make sure both rectangles
+	 * end up invisible.
+	 */
+	drm_rect_init(&src, 0, 0, INT_MAX, INT_MAX);
+	drm_rect_init(&dst, 0, 0, 2, 2);
+	drm_rect_init(&clip, 3, 3, 1, 1);
+
+	visible = drm_rect_clip_scaled(&src, &dst, &clip);
+
+	KUNIT_EXPECT_FALSE_MSG(test, visible, "Destination should not be visible\n");
+	KUNIT_EXPECT_FALSE_MSG(test, drm_rect_visible(&src), "Source should not be visible\n");
+}
+
+static struct kunit_case drm_rect_tests[] = {
+	KUNIT_CASE(drm_test_rect_clip_scaled_div_by_zero),
+	KUNIT_CASE(drm_test_rect_clip_scaled_not_clipped),
+	KUNIT_CASE(drm_test_rect_clip_scaled_clipped),
+	KUNIT_CASE(drm_test_rect_clip_scaled_signed_vs_unsigned),
+	{ }
+};
+
+static struct kunit_suite drm_rect_test_suite = {
+	.name = "drm_rect",
+	.test_cases = drm_rect_tests,
+};
+
+kunit_test_suite(drm_rect_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tidss/Kconfig b/drivers/gpu/drm/tidss/Kconfig
index bc4fa59b6fa9..378600806167 100644
--- a/drivers/gpu/drm/tidss/Kconfig
+++ b/drivers/gpu/drm/tidss/Kconfig
@@ -3,7 +3,7 @@ config DRM_TIDSS
 	depends on DRM && OF
 	depends on ARM || ARM64 || COMPILE_TEST
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	help
 	  The TI Keystone family SoCs introduced a new generation of
 	  Display SubSystem. There is currently three Keystone family
diff --git a/drivers/gpu/drm/tidss/tidss_crtc.c b/drivers/gpu/drm/tidss/tidss_crtc.c
index 2218da3b3ca3..cd3c43a6c806 100644
--- a/drivers/gpu/drm/tidss/tidss_crtc.c
+++ b/drivers/gpu/drm/tidss/tidss_crtc.c
@@ -8,9 +8,7 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
-#include <drm/drm_fb_cma_helper.h>
-#include <drm/drm_gem_cma_helper.h>
-#include <drm/drm_plane_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "tidss_crtc.h"
diff --git a/drivers/gpu/drm/tidss/tidss_dispc.c b/drivers/gpu/drm/tidss/tidss_dispc.c
index dd3c6a606ae2..ad93acc9abd2 100644
--- a/drivers/gpu/drm/tidss/tidss_dispc.c
+++ b/drivers/gpu/drm/tidss/tidss_dispc.c
@@ -24,9 +24,9 @@
 
 #include <drm/drm_blend.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_panel.h>
 
 #include "tidss_crtc.h"
@@ -1954,16 +1954,16 @@ int dispc_plane_check(struct dispc_device *dispc, u32 hw_plane,
 }
 
 static
-dma_addr_t dispc_plane_state_paddr(const struct drm_plane_state *state)
+dma_addr_t dispc_plane_state_dma_addr(const struct drm_plane_state *state)
 {
 	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 	u32 x = state->src_x >> 16;
 	u32 y = state->src_y >> 16;
 
-	gem = drm_fb_cma_get_gem_obj(state->fb, 0);
+	gem = drm_fb_dma_get_gem_obj(state->fb, 0);
 
-	return gem->paddr + fb->offsets[0] + x * fb->format->cpp[0] +
+	return gem->dma_addr + fb->offsets[0] + x * fb->format->cpp[0] +
 		y * fb->pitches[0];
 }
 
@@ -1971,16 +1971,16 @@ static
 dma_addr_t dispc_plane_state_p_uv_addr(const struct drm_plane_state *state)
 {
 	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 	u32 x = state->src_x >> 16;
 	u32 y = state->src_y >> 16;
 
 	if (WARN_ON(state->fb->format->num_planes != 2))
 		return 0;
 
-	gem = drm_fb_cma_get_gem_obj(fb, 1);
+	gem = drm_fb_dma_get_gem_obj(fb, 1);
 
-	return gem->paddr + fb->offsets[1] +
+	return gem->dma_addr + fb->offsets[1] +
 		(x * fb->format->cpp[1] / fb->format->hsub) +
 		(y * fb->pitches[1] / fb->format->vsub);
 }
@@ -1993,17 +1993,17 @@ int dispc_plane_setup(struct dispc_device *dispc, u32 hw_plane,
 	u32 fourcc = state->fb->format->format;
 	u16 cpp = state->fb->format->cpp[0];
 	u32 fb_width = state->fb->pitches[0] / cpp;
-	dma_addr_t paddr = dispc_plane_state_paddr(state);
+	dma_addr_t dma_addr = dispc_plane_state_dma_addr(state);
 	struct dispc_scaling_params scale;
 
 	dispc_vid_calc_scaling(dispc, state, &scale, lite);
 
 	dispc_plane_set_pixel_format(dispc, hw_plane, fourcc);
 
-	dispc_vid_write(dispc, hw_plane, DISPC_VID_BA_0, paddr & 0xffffffff);
-	dispc_vid_write(dispc, hw_plane, DISPC_VID_BA_EXT_0, (u64)paddr >> 32);
-	dispc_vid_write(dispc, hw_plane, DISPC_VID_BA_1, paddr & 0xffffffff);
-	dispc_vid_write(dispc, hw_plane, DISPC_VID_BA_EXT_1, (u64)paddr >> 32);
+	dispc_vid_write(dispc, hw_plane, DISPC_VID_BA_0, dma_addr & 0xffffffff);
+	dispc_vid_write(dispc, hw_plane, DISPC_VID_BA_EXT_0, (u64)dma_addr >> 32);
+	dispc_vid_write(dispc, hw_plane, DISPC_VID_BA_1, dma_addr & 0xffffffff);
+	dispc_vid_write(dispc, hw_plane, DISPC_VID_BA_EXT_1, (u64)dma_addr >> 32);
 
 	dispc_vid_write(dispc, hw_plane, DISPC_VID_PICTURE_SIZE,
 			(scale.in_w - 1) | ((scale.in_h - 1) << 16));
diff --git a/drivers/gpu/drm/tidss/tidss_drv.c b/drivers/gpu/drm/tidss/tidss_drv.c
index 04cfff89ee51..15cd9b91b7e2 100644
--- a/drivers/gpu/drm/tidss/tidss_drv.c
+++ b/drivers/gpu/drm/tidss/tidss_drv.c
@@ -15,7 +15,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_module.h>
 #include <drm/drm_probe_helper.h>
@@ -101,13 +101,13 @@ static void tidss_release(struct drm_device *ddev)
 	drm_kms_helper_poll_fini(ddev);
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(tidss_fops);
+DEFINE_DRM_GEM_DMA_FOPS(tidss_fops);
 
 static const struct drm_driver tidss_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &tidss_fops,
 	.release		= tidss_release,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.name			= "tidss",
 	.desc			= "TI Keystone DSS",
 	.date			= "20180215",
diff --git a/drivers/gpu/drm/tidss/tidss_kms.c b/drivers/gpu/drm/tidss/tidss_kms.c
index 666e527a0acf..afb2879980c6 100644
--- a/drivers/gpu/drm/tidss/tidss_kms.c
+++ b/drivers/gpu/drm/tidss/tidss_kms.c
@@ -10,7 +10,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_crtc_helper.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_of.h>
@@ -71,7 +70,7 @@ static int tidss_atomic_check(struct drm_device *ddev,
 	 * changes. This is needed for updating the plane positions in
 	 * tidss_crtc_position_planes() which is called from
 	 * crtc_atomic_enable() and crtc_atomic_flush(). We have an
-	 * extra flag to to mark x,y-position changes and together
+	 * extra flag to mark x,y-position changes and together
 	 * with zpos_changed the condition recognizes all the above
 	 * cases.
 	 */
diff --git a/drivers/gpu/drm/tidss/tidss_plane.c b/drivers/gpu/drm/tidss/tidss_plane.c
index 68a85a94ffcb..42d50ec5526d 100644
--- a/drivers/gpu/drm/tidss/tidss_plane.c
+++ b/drivers/gpu/drm/tidss/tidss_plane.c
@@ -11,7 +11,6 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
 
 #include "tidss_crtc.h"
diff --git a/drivers/gpu/drm/tilcdc/Kconfig b/drivers/gpu/drm/tilcdc/Kconfig
index e315591eb36b..d3bd2d7a181e 100644
--- a/drivers/gpu/drm/tilcdc/Kconfig
+++ b/drivers/gpu/drm/tilcdc/Kconfig
@@ -3,7 +3,7 @@ config DRM_TILCDC
 	tristate "DRM Support for TI LCDC Display Controller"
 	depends on DRM && OF && ARM
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_BRIDGE
 	select DRM_PANEL_BRIDGE
 	select VIDEOMODE_HELPERS
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index 509fbae8c9a6..b5f60b2b2d0e 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -12,10 +12,10 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
@@ -64,13 +64,13 @@ static void set_scanout(struct drm_crtc *crtc, struct drm_framebuffer *fb)
 {
 	struct drm_device *dev = crtc->dev;
 	struct tilcdc_drm_private *priv = dev->dev_private;
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 	dma_addr_t start, end;
 	u64 dma_base_and_ceiling;
 
-	gem = drm_fb_cma_get_gem_obj(fb, 0);
+	gem = drm_fb_dma_get_gem_obj(fb, 0);
 
-	start = gem->paddr + fb->offsets[0] +
+	start = gem->dma_addr + fb->offsets[0] +
 		crtc->y * fb->pitches[0] +
 		crtc->x * fb->format->cpp[0];
 
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index eee3c447fbac..f72755b8ea14 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -18,7 +18,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_mm.h>
 #include <drm/drm_probe_helper.h>
@@ -476,11 +476,11 @@ static void tilcdc_debugfs_init(struct drm_minor *minor)
 }
 #endif
 
-DEFINE_DRM_GEM_CMA_FOPS(fops);
+DEFINE_DRM_GEM_DMA_FOPS(fops);
 
 static const struct drm_driver tilcdc_driver = {
 	.driver_features    = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init       = tilcdc_debugfs_init,
 #endif
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_plane.c b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
index 960136518814..cf77a8ce7398 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_plane.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
@@ -5,7 +5,6 @@
  */
 
 #include <drm/drm_atomic.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
@@ -106,11 +105,10 @@ int tilcdc_plane_init(struct drm_device *dev,
 	struct tilcdc_drm_private *priv = dev->dev_private;
 	int ret;
 
-	ret = drm_plane_init(dev, plane, 1,
-			     &tilcdc_plane_funcs,
-			     priv->pixelformats,
-			     priv->num_pixelformats,
-			     true);
+	ret = drm_universal_plane_init(dev, plane, 1, &tilcdc_plane_funcs,
+				       priv->pixelformats,
+				       priv->num_pixelformats,
+				       NULL, DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret) {
 		dev_err(dev->dev, "Failed to initialize plane: %d\n", ret);
 		return ret;
diff --git a/drivers/gpu/drm/tiny/Kconfig b/drivers/gpu/drm/tiny/Kconfig
index 027cd87c3d0d..565957264875 100644
--- a/drivers/gpu/drm/tiny/Kconfig
+++ b/drivers/gpu/drm/tiny/Kconfig
@@ -3,7 +3,7 @@
 config DRM_ARCPGU
 	tristate "ARC PGU"
 	depends on DRM && OF
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_KMS_HELPER
 	help
 	  Choose this option if you have an ARC PGU controller.
@@ -55,7 +55,7 @@ config DRM_PANEL_MIPI_DBI
 	tristate "DRM support for MIPI DBI compatible panels"
 	depends on DRM && SPI
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_MIPI_DBI
 	select BACKLIGHT_CLASS_DEVICE
 	select VIDEOMODE_HELPERS
@@ -87,7 +87,7 @@ config TINYDRM_HX8357D
 	tristate "DRM support for HX8357D display panels"
 	depends on DRM && SPI
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_MIPI_DBI
 	select BACKLIGHT_CLASS_DEVICE
 	help
@@ -100,7 +100,7 @@ config TINYDRM_ILI9163
 	tristate "DRM support for ILI9163 display panels"
 	depends on DRM && SPI
 	select BACKLIGHT_CLASS_DEVICE
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_KMS_HELPER
 	select DRM_MIPI_DBI
 	help
@@ -113,7 +113,7 @@ config TINYDRM_ILI9225
 	tristate "DRM support for ILI9225 display panels"
 	depends on DRM && SPI
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_MIPI_DBI
 	help
 	  DRM driver for the following Ilitek ILI9225 panels:
@@ -125,7 +125,7 @@ config TINYDRM_ILI9341
 	tristate "DRM support for ILI9341 display panels"
 	depends on DRM && SPI
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_MIPI_DBI
 	select BACKLIGHT_CLASS_DEVICE
 	help
@@ -138,7 +138,7 @@ config TINYDRM_ILI9486
 	tristate "DRM support for ILI9486 display panels"
 	depends on DRM && SPI
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_MIPI_DBI
 	select BACKLIGHT_CLASS_DEVICE
 	help
@@ -152,7 +152,7 @@ config TINYDRM_MI0283QT
 	tristate "DRM support for MI0283QT"
 	depends on DRM && SPI
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_MIPI_DBI
 	select BACKLIGHT_CLASS_DEVICE
 	help
@@ -163,7 +163,7 @@ config TINYDRM_REPAPER
 	tristate "DRM support for Pervasive Displays RePaper panels (V231)"
 	depends on DRM && SPI
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	help
 	  DRM driver for the following Pervasive Displays panels:
 	  1.44" TFT EPD Panel (E1144CS021)
@@ -177,7 +177,7 @@ config TINYDRM_ST7586
 	tristate "DRM support for Sitronix ST7586 display panels"
 	depends on DRM && SPI
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_MIPI_DBI
 	help
 	  DRM driver for the following Sitronix ST7586 panels:
@@ -189,7 +189,7 @@ config TINYDRM_ST7735R
 	tristate "DRM support for Sitronix ST7715R/ST7735R display panels"
 	depends on DRM && SPI
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_MIPI_DBI
 	select BACKLIGHT_CLASS_DEVICE
 	help
diff --git a/drivers/gpu/drm/tiny/arcpgu.c b/drivers/gpu/drm/tiny/arcpgu.c
index 7461cb401407..bb302a3fd6b5 100644
--- a/drivers/gpu/drm/tiny/arcpgu.c
+++ b/drivers/gpu/drm/tiny/arcpgu.c
@@ -11,11 +11,11 @@
 #include <drm/drm_device.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_edid.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_of.h>
@@ -220,14 +220,14 @@ static void arc_pgu_update(struct drm_simple_display_pipe *pipe,
 			   struct drm_plane_state *state)
 {
 	struct arcpgu_drm_private *arcpgu;
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 
 	if (!pipe->plane.state->fb)
 		return;
 
 	arcpgu = pipe_to_arcpgu_priv(pipe);
-	gem = drm_fb_cma_get_gem_obj(pipe->plane.state->fb, 0);
-	arc_pgu_write(arcpgu, ARCPGU_REG_BUF0_ADDR, gem->paddr);
+	gem = drm_fb_dma_get_gem_obj(pipe->plane.state->fb, 0);
+	arc_pgu_write(arcpgu, ARCPGU_REG_BUF0_ADDR, gem->dma_addr);
 }
 
 static const struct drm_simple_display_pipe_funcs arc_pgu_pipe_funcs = {
@@ -243,7 +243,7 @@ static const struct drm_mode_config_funcs arcpgu_drm_modecfg_funcs = {
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(arcpgu_drm_ops);
+DEFINE_DRM_GEM_DMA_FOPS(arcpgu_drm_ops);
 
 static int arcpgu_load(struct arcpgu_drm_private *arcpgu)
 {
@@ -370,7 +370,7 @@ static const struct drm_driver arcpgu_drm_driver = {
 	.minor = 0,
 	.patchlevel = 0,
 	.fops = &arcpgu_drm_ops,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init = arcpgu_debugfs_init,
 #endif
diff --git a/drivers/gpu/drm/tiny/bochs.c b/drivers/gpu/drm/tiny/bochs.c
index 82364a0a7b18..a51262289aef 100644
--- a/drivers/gpu/drm/tiny/bochs.c
+++ b/drivers/gpu/drm/tiny/bochs.c
@@ -309,6 +309,8 @@ static void bochs_hw_fini(struct drm_device *dev)
 static void bochs_hw_blank(struct bochs_device *bochs, bool blank)
 {
 	DRM_DEBUG_DRIVER("hw_blank %d\n", blank);
+	/* enable color bit (so VGA_IS1_RC access works) */
+	bochs_vga_writeb(bochs, VGA_MIS_W, VGA_MIS_COLOR);
 	/* discard ar_flip_flop */
 	(void)bochs_vga_readb(bochs, VGA_IS1_RC);
 	/* blank or unblank; we need only update index and set 0x20 */
@@ -583,13 +585,17 @@ static int bochs_load(struct drm_device *dev)
 
 	ret = drmm_vram_helper_init(dev, bochs->fb_base, bochs->fb_size);
 	if (ret)
-		return ret;
+		goto err_hw_fini;
 
 	ret = bochs_kms_init(bochs);
 	if (ret)
-		return ret;
+		goto err_hw_fini;
 
 	return 0;
+
+err_hw_fini:
+	bochs_hw_fini(dev);
+	return ret;
 }
 
 DEFINE_DRM_GEM_FOPS(bochs_fops);
@@ -664,11 +670,13 @@ static int bochs_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent
 
 	ret = drm_dev_register(dev, 0);
 	if (ret)
-		goto err_free_dev;
+		goto err_hw_fini;
 
 	drm_fbdev_generic_setup(dev, 32);
 	return ret;
 
+err_hw_fini:
+	bochs_hw_fini(dev);
 err_free_dev:
 	drm_dev_put(dev);
 	return ret;
diff --git a/drivers/gpu/drm/tiny/cirrus.c b/drivers/gpu/drm/tiny/cirrus.c
index c4f5beea1f90..354d5e854a6f 100644
--- a/drivers/gpu/drm/tiny/cirrus.c
+++ b/drivers/gpu/drm/tiny/cirrus.c
@@ -316,28 +316,29 @@ static int cirrus_mode_set(struct cirrus_device *cirrus,
 }
 
 static int cirrus_fb_blit_rect(struct drm_framebuffer *fb,
-			       const struct iosys_map *map,
+			       const struct iosys_map *vmap,
 			       struct drm_rect *rect)
 {
 	struct cirrus_device *cirrus = to_cirrus(fb->dev);
-	void __iomem *dst = cirrus->vram;
-	void *vmap = map->vaddr; /* TODO: Use mapping abstraction properly */
+	struct iosys_map dst;
 	int idx;
 
 	if (!drm_dev_enter(&cirrus->dev, &idx))
 		return -ENODEV;
 
+	iosys_map_set_vaddr_iomem(&dst, cirrus->vram);
+
 	if (cirrus->cpp == fb->format->cpp[0]) {
-		dst += drm_fb_clip_offset(fb->pitches[0], fb->format, rect);
-		drm_fb_memcpy_toio(dst, fb->pitches[0], vmap, fb, rect);
+		iosys_map_incr(&dst, drm_fb_clip_offset(fb->pitches[0], fb->format, rect));
+		drm_fb_memcpy(&dst, fb->pitches, vmap, fb, rect);
 
 	} else if (fb->format->cpp[0] == 4 && cirrus->cpp == 2) {
-		dst += drm_fb_clip_offset(cirrus->pitch, fb->format, rect);
-		drm_fb_xrgb8888_to_rgb565_toio(dst, cirrus->pitch, vmap, fb, rect, false);
+		iosys_map_incr(&dst, drm_fb_clip_offset(cirrus->pitch, fb->format, rect));
+		drm_fb_xrgb8888_to_rgb565(&dst, &cirrus->pitch, vmap, fb, rect, false);
 
 	} else if (fb->format->cpp[0] == 4 && cirrus->cpp == 3) {
-		dst += drm_fb_clip_offset(cirrus->pitch, fb->format, rect);
-		drm_fb_xrgb8888_to_rgb888_toio(dst, cirrus->pitch, vmap, fb, rect);
+		iosys_map_incr(&dst, drm_fb_clip_offset(cirrus->pitch, fb->format, rect));
+		drm_fb_xrgb8888_to_rgb888(&dst, &cirrus->pitch, vmap, fb, rect);
 
 	} else {
 		WARN_ON_ONCE("cpp mismatch");
diff --git a/drivers/gpu/drm/tiny/hx8357d.c b/drivers/gpu/drm/tiny/hx8357d.c
index ebb025543f8d..48c24aa8c28a 100644
--- a/drivers/gpu/drm/tiny/hx8357d.c
+++ b/drivers/gpu/drm/tiny/hx8357d.c
@@ -20,7 +20,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mipi_dbi.h>
 #include <drm/drm_modeset_helper.h>
@@ -181,6 +181,7 @@ out_exit:
 }
 
 static const struct drm_simple_display_pipe_funcs hx8357d_pipe_funcs = {
+	.mode_valid = mipi_dbi_pipe_mode_valid,
 	.enable = yx240qv29_enable,
 	.disable = mipi_dbi_pipe_disable,
 	.update = mipi_dbi_pipe_update,
@@ -190,12 +191,12 @@ static const struct drm_display_mode yx350hv15_mode = {
 	DRM_SIMPLE_MODE(320, 480, 60, 75),
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(hx8357d_fops);
+DEFINE_DRM_GEM_DMA_FOPS(hx8357d_fops);
 
 static const struct drm_driver hx8357d_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &hx8357d_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "hx8357d",
 	.desc			= "HX8357D",
diff --git a/drivers/gpu/drm/tiny/ili9163.c b/drivers/gpu/drm/tiny/ili9163.c
index fc8ed245b0bc..9a1a5943bee0 100644
--- a/drivers/gpu/drm/tiny/ili9163.c
+++ b/drivers/gpu/drm/tiny/ili9163.c
@@ -11,7 +11,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_mipi_dbi.h>
 #include <drm/drm_modeset_helper.h>
 
@@ -100,6 +100,7 @@ out_exit:
 }
 
 static const struct drm_simple_display_pipe_funcs ili9163_pipe_funcs = {
+	.mode_valid = mipi_dbi_pipe_mode_valid,
 	.enable = yx240qv29_enable,
 	.disable = mipi_dbi_pipe_disable,
 	.update = mipi_dbi_pipe_update,
@@ -110,12 +111,12 @@ static const struct drm_display_mode yx240qv29_mode = {
 	DRM_SIMPLE_MODE(128, 160, 28, 35),
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(ili9163_fops);
+DEFINE_DRM_GEM_DMA_FOPS(ili9163_fops);
 
 static struct drm_driver ili9163_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &ili9163_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "ili9163",
 	.desc			= "Ilitek ILI9163",
diff --git a/drivers/gpu/drm/tiny/ili9225.c b/drivers/gpu/drm/tiny/ili9225.c
index 8d686eecd5f4..a79da2b4af64 100644
--- a/drivers/gpu/drm/tiny/ili9225.c
+++ b/drivers/gpu/drm/tiny/ili9225.c
@@ -19,12 +19,12 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mipi_dbi.h>
 #include <drm/drm_rect.h>
@@ -78,7 +78,7 @@ static inline int ili9225_command(struct mipi_dbi *dbi, u8 cmd, u16 data)
 
 static void ili9225_fb_dirty(struct drm_framebuffer *fb, struct drm_rect *rect)
 {
-	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+	struct drm_gem_dma_object *dma_obj = drm_fb_dma_get_gem_obj(fb, 0);
 	struct mipi_dbi_dev *dbidev = drm_to_mipi_dbi_dev(fb->dev);
 	unsigned int height = rect->y2 - rect->y1;
 	unsigned int width = rect->x2 - rect->x1;
@@ -104,7 +104,7 @@ static void ili9225_fb_dirty(struct drm_framebuffer *fb, struct drm_rect *rect)
 		if (ret)
 			goto err_msg;
 	} else {
-		tr = cma_obj->vaddr;
+		tr = dma_obj->vaddr;
 	}
 
 	switch (dbidev->rotation) {
@@ -335,12 +335,12 @@ static const struct drm_display_mode ili9225_mode = {
 	DRM_SIMPLE_MODE(176, 220, 35, 44),
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(ili9225_fops);
+DEFINE_DRM_GEM_DMA_FOPS(ili9225_fops);
 
 static const struct drm_driver ili9225_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &ili9225_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.name			= "ili9225",
 	.desc			= "Ilitek ILI9225",
 	.date			= "20171106",
diff --git a/drivers/gpu/drm/tiny/ili9341.c b/drivers/gpu/drm/tiny/ili9341.c
index 5b8cc770ee7b..69b265e78096 100644
--- a/drivers/gpu/drm/tiny/ili9341.c
+++ b/drivers/gpu/drm/tiny/ili9341.c
@@ -19,7 +19,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mipi_dbi.h>
 #include <drm/drm_modeset_helper.h>
@@ -137,6 +137,7 @@ out_exit:
 }
 
 static const struct drm_simple_display_pipe_funcs ili9341_pipe_funcs = {
+	.mode_valid = mipi_dbi_pipe_mode_valid,
 	.enable = yx240qv29_enable,
 	.disable = mipi_dbi_pipe_disable,
 	.update = mipi_dbi_pipe_update,
@@ -146,12 +147,12 @@ static const struct drm_display_mode yx240qv29_mode = {
 	DRM_SIMPLE_MODE(240, 320, 37, 49),
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(ili9341_fops);
+DEFINE_DRM_GEM_DMA_FOPS(ili9341_fops);
 
 static const struct drm_driver ili9341_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &ili9341_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "ili9341",
 	.desc			= "Ilitek ILI9341",
diff --git a/drivers/gpu/drm/tiny/ili9486.c b/drivers/gpu/drm/tiny/ili9486.c
index 6d655e18e0aa..c80028bb1d11 100644
--- a/drivers/gpu/drm/tiny/ili9486.c
+++ b/drivers/gpu/drm/tiny/ili9486.c
@@ -18,7 +18,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mipi_dbi.h>
 #include <drm/drm_modeset_helper.h>
@@ -150,6 +150,7 @@ static void waveshare_enable(struct drm_simple_display_pipe *pipe,
 }
 
 static const struct drm_simple_display_pipe_funcs waveshare_pipe_funcs = {
+	.mode_valid = mipi_dbi_pipe_mode_valid,
 	.enable = waveshare_enable,
 	.disable = mipi_dbi_pipe_disable,
 	.update = mipi_dbi_pipe_update,
@@ -159,12 +160,12 @@ static const struct drm_display_mode waveshare_mode = {
 	DRM_SIMPLE_MODE(480, 320, 73, 49),
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(ili9486_fops);
+DEFINE_DRM_GEM_DMA_FOPS(ili9486_fops);
 
 static const struct drm_driver ili9486_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &ili9486_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "ili9486",
 	.desc			= "Ilitek ILI9486",
diff --git a/drivers/gpu/drm/tiny/mi0283qt.c b/drivers/gpu/drm/tiny/mi0283qt.c
index 5e060f6910bb..bc522fb3d94d 100644
--- a/drivers/gpu/drm/tiny/mi0283qt.c
+++ b/drivers/gpu/drm/tiny/mi0283qt.c
@@ -17,7 +17,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mipi_dbi.h>
 #include <drm/drm_modeset_helper.h>
@@ -141,6 +141,7 @@ out_exit:
 }
 
 static const struct drm_simple_display_pipe_funcs mi0283qt_pipe_funcs = {
+	.mode_valid = mipi_dbi_pipe_mode_valid,
 	.enable = mi0283qt_enable,
 	.disable = mipi_dbi_pipe_disable,
 	.update = mipi_dbi_pipe_update,
@@ -150,12 +151,12 @@ static const struct drm_display_mode mi0283qt_mode = {
 	DRM_SIMPLE_MODE(320, 240, 58, 43),
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(mi0283qt_fops);
+DEFINE_DRM_GEM_DMA_FOPS(mi0283qt_fops);
 
 static const struct drm_driver mi0283qt_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &mi0283qt_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "mi0283qt",
 	.desc			= "Multi-Inno MI0283QT",
diff --git a/drivers/gpu/drm/tiny/panel-mipi-dbi.c b/drivers/gpu/drm/tiny/panel-mipi-dbi.c
index c759ff9c2c87..955a61d628e7 100644
--- a/drivers/gpu/drm/tiny/panel-mipi-dbi.c
+++ b/drivers/gpu/drm/tiny/panel-mipi-dbi.c
@@ -18,7 +18,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mipi_dbi.h>
 #include <drm/drm_modes.h>
@@ -212,17 +212,18 @@ out_exit:
 }
 
 static const struct drm_simple_display_pipe_funcs panel_mipi_dbi_pipe_funcs = {
+	.mode_valid = mipi_dbi_pipe_mode_valid,
 	.enable = panel_mipi_dbi_enable,
 	.disable = mipi_dbi_pipe_disable,
 	.update = mipi_dbi_pipe_update,
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(panel_mipi_dbi_fops);
+DEFINE_DRM_GEM_DMA_FOPS(panel_mipi_dbi_fops);
 
 static const struct drm_driver panel_mipi_dbi_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &panel_mipi_dbi_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "panel-mipi-dbi",
 	.desc			= "MIPI DBI compatible display panel",
diff --git a/drivers/gpu/drm/tiny/repaper.c b/drivers/gpu/drm/tiny/repaper.c
index 013790c45d0a..e62f4d16b2c6 100644
--- a/drivers/gpu/drm/tiny/repaper.c
+++ b/drivers/gpu/drm/tiny/repaper.c
@@ -25,12 +25,12 @@
 #include <drm/drm_connector.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_format_helper.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_modes.h>
@@ -511,8 +511,10 @@ static void repaper_get_temperature(struct repaper_epd *epd)
 
 static int repaper_fb_dirty(struct drm_framebuffer *fb)
 {
-	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+	struct drm_gem_dma_object *dma_obj = drm_fb_dma_get_gem_obj(fb, 0);
 	struct repaper_epd *epd = drm_to_epd(fb->dev);
+	unsigned int dst_pitch = 0;
+	struct iosys_map dst, vmap;
 	struct drm_rect clip;
 	int idx, ret = 0;
 	u8 *buf = NULL;
@@ -541,7 +543,9 @@ static int repaper_fb_dirty(struct drm_framebuffer *fb)
 	if (ret)
 		goto out_free;
 
-	drm_fb_xrgb8888_to_mono(buf, 0, cma_obj->vaddr, fb, &clip);
+	iosys_map_set_vaddr(&dst, buf);
+	iosys_map_set_vaddr(&vmap, dma_obj->vaddr);
+	drm_fb_xrgb8888_to_mono(&dst, &dst_pitch, &vmap, fb, &clip);
 
 	drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
 
@@ -617,6 +621,15 @@ static void power_off(struct repaper_epd *epd)
 	gpiod_set_value_cansleep(epd->discharge, 0);
 }
 
+static enum drm_mode_status repaper_pipe_mode_valid(struct drm_simple_display_pipe *pipe,
+						    const struct drm_display_mode *mode)
+{
+	struct drm_crtc *crtc = &pipe->crtc;
+	struct repaper_epd *epd = drm_to_epd(crtc->dev);
+
+	return drm_crtc_helper_mode_valid_fixed(crtc, mode, epd->mode);
+}
+
 static void repaper_pipe_enable(struct drm_simple_display_pipe *pipe,
 				struct drm_crtc_state *crtc_state,
 				struct drm_plane_state *plane_state)
@@ -827,6 +840,7 @@ static void repaper_pipe_update(struct drm_simple_display_pipe *pipe,
 }
 
 static const struct drm_simple_display_pipe_funcs repaper_pipe_funcs = {
+	.mode_valid = repaper_pipe_mode_valid,
 	.enable = repaper_pipe_enable,
 	.disable = repaper_pipe_disable,
 	.update = repaper_pipe_update,
@@ -835,22 +849,8 @@ static const struct drm_simple_display_pipe_funcs repaper_pipe_funcs = {
 static int repaper_connector_get_modes(struct drm_connector *connector)
 {
 	struct repaper_epd *epd = drm_to_epd(connector->dev);
-	struct drm_display_mode *mode;
-
-	mode = drm_mode_duplicate(connector->dev, epd->mode);
-	if (!mode) {
-		DRM_ERROR("Failed to duplicate mode\n");
-		return 0;
-	}
-
-	drm_mode_set_name(mode);
-	mode->type |= DRM_MODE_TYPE_PREFERRED;
-	drm_mode_probed_add(connector, mode);
-
-	connector->display_info.width_mm = mode->width_mm;
-	connector->display_info.height_mm = mode->height_mm;
 
-	return 1;
+	return drm_connector_helper_get_modes_fixed(connector, epd->mode);
 }
 
 static const struct drm_connector_helper_funcs repaper_connector_hfuncs = {
@@ -903,12 +903,12 @@ static const struct drm_display_mode repaper_e2271cs021_mode = {
 static const u8 repaper_e2271cs021_cs[] = { 0x00, 0x00, 0x00, 0x7f,
 					    0xff, 0xfe, 0x00, 0x00 };
 
-DEFINE_DRM_GEM_CMA_FOPS(repaper_fops);
+DEFINE_DRM_GEM_DMA_FOPS(repaper_fops);
 
 static const struct drm_driver repaper_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &repaper_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.name			= "repaper",
 	.desc			= "Pervasive Displays RePaper e-ink panels",
 	.date			= "20170405",
diff --git a/drivers/gpu/drm/tiny/simpledrm.c b/drivers/gpu/drm/tiny/simpledrm.c
index 5422363690e7..18489779fb8a 100644
--- a/drivers/gpu/drm/tiny/simpledrm.c
+++ b/drivers/gpu/drm/tiny/simpledrm.c
@@ -8,6 +8,7 @@
 #include <linux/regulator/consumer.h>
 
 #include <drm/drm_aperture.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_connector.h>
 #include <drm/drm_damage_helper.h>
@@ -20,8 +21,8 @@
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_modeset_helper_vtables.h>
+#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
-#include <drm/drm_simple_kms_helper.h>
 
 #define DRIVER_NAME	"simpledrm"
 #define DRIVER_DESC	"DRM driver for simple-framebuffer platform devices"
@@ -30,16 +31,6 @@
 #define DRIVER_MINOR	0
 
 /*
- * Assume a monitor resolution of 96 dpi to
- * get a somewhat reasonable screen size.
- */
-#define RES_MM(d)	\
-	(((d) * 254ul) / (96ul * 10ul))
-
-#define SIMPLEDRM_MODE(hd, vd)	\
-	DRM_SIMPLE_MODE(hd, vd, RES_MM(hd), RES_MM(vd))
-
-/*
  * Helpers for simplefb
  */
 
@@ -198,7 +189,6 @@ simplefb_get_format_of(struct drm_device *dev, struct device_node *of_node)
 
 struct simpledrm_device {
 	struct drm_device dev;
-	struct platform_device *pdev;
 
 	/* clocks */
 #if defined CONFIG_OF && defined CONFIG_COMMON_CLK
@@ -217,14 +207,15 @@ struct simpledrm_device {
 	unsigned int pitch;
 
 	/* memory management */
-	struct resource *mem;
 	void __iomem *screen_base;
 
 	/* modesetting */
 	uint32_t formats[8];
 	size_t nformats;
+	struct drm_plane primary_plane;
+	struct drm_crtc crtc;
+	struct drm_encoder encoder;
 	struct drm_connector connector;
-	struct drm_simple_display_pipe pipe;
 };
 
 static struct simpledrm_device *simpledrm_device_of_dev(struct drm_device *dev)
@@ -272,7 +263,7 @@ static void simpledrm_device_release_clocks(void *res)
 static int simpledrm_device_init_clocks(struct simpledrm_device *sdev)
 {
 	struct drm_device *dev = &sdev->dev;
-	struct platform_device *pdev = sdev->pdev;
+	struct platform_device *pdev = to_platform_device(dev->dev);
 	struct device_node *of_node = pdev->dev.of_node;
 	struct clk *clock;
 	unsigned int i;
@@ -370,7 +361,7 @@ static void simpledrm_device_release_regulators(void *res)
 static int simpledrm_device_init_regulators(struct simpledrm_device *sdev)
 {
 	struct drm_device *dev = &sdev->dev;
-	struct platform_device *pdev = sdev->pdev;
+	struct platform_device *pdev = to_platform_device(dev->dev);
 	struct device_node *of_node = pdev->dev.of_node;
 	struct property *prop;
 	struct regulator *regulator;
@@ -451,120 +442,6 @@ static int simpledrm_device_init_regulators(struct simpledrm_device *sdev)
 #endif
 
 /*
- *  Simplefb settings
- */
-
-static struct drm_display_mode simpledrm_mode(unsigned int width,
-					      unsigned int height)
-{
-	struct drm_display_mode mode = { SIMPLEDRM_MODE(width, height) };
-
-	mode.clock = mode.hdisplay * mode.vdisplay * 60 / 1000 /* kHz */;
-	drm_mode_set_name(&mode);
-
-	return mode;
-}
-
-static int simpledrm_device_init_fb(struct simpledrm_device *sdev)
-{
-	int width, height, stride;
-	const struct drm_format_info *format;
-	struct drm_device *dev = &sdev->dev;
-	struct platform_device *pdev = sdev->pdev;
-	const struct simplefb_platform_data *pd = dev_get_platdata(&pdev->dev);
-	struct device_node *of_node = pdev->dev.of_node;
-
-	if (pd) {
-		width = simplefb_get_width_pd(dev, pd);
-		if (width < 0)
-			return width;
-		height = simplefb_get_height_pd(dev, pd);
-		if (height < 0)
-			return height;
-		stride = simplefb_get_stride_pd(dev, pd);
-		if (stride < 0)
-			return stride;
-		format = simplefb_get_format_pd(dev, pd);
-		if (IS_ERR(format))
-			return PTR_ERR(format);
-	} else if (of_node) {
-		width = simplefb_get_width_of(dev, of_node);
-		if (width < 0)
-			return width;
-		height = simplefb_get_height_of(dev, of_node);
-		if (height < 0)
-			return height;
-		stride = simplefb_get_stride_of(dev, of_node);
-		if (stride < 0)
-			return stride;
-		format = simplefb_get_format_of(dev, of_node);
-		if (IS_ERR(format))
-			return PTR_ERR(format);
-	} else {
-		drm_err(dev, "no simplefb configuration found\n");
-		return -ENODEV;
-	}
-
-	sdev->mode = simpledrm_mode(width, height);
-	sdev->format = format;
-	sdev->pitch = stride;
-
-	drm_dbg_kms(dev, "display mode={" DRM_MODE_FMT "}\n",
-		    DRM_MODE_ARG(&sdev->mode));
-	drm_dbg_kms(dev,
-		    "framebuffer format=%p4cc, size=%dx%d, stride=%d byte\n",
-		    &format->format, width, height, stride);
-
-	return 0;
-}
-
-/*
- * Memory management
- */
-
-static int simpledrm_device_init_mm(struct simpledrm_device *sdev)
-{
-	struct drm_device *dev = &sdev->dev;
-	struct platform_device *pdev = sdev->pdev;
-	struct resource *res, *mem;
-	void __iomem *screen_base;
-	int ret;
-
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -EINVAL;
-
-	ret = devm_aperture_acquire_from_firmware(dev, res->start, resource_size(res));
-	if (ret) {
-		drm_err(dev, "could not acquire memory range %pr: error %d\n",
-			res, ret);
-		return ret;
-	}
-
-	mem = devm_request_mem_region(&pdev->dev, res->start, resource_size(res),
-				      sdev->dev.driver->name);
-	if (!mem) {
-		/*
-		 * We cannot make this fatal. Sometimes this comes from magic
-		 * spaces our resource handlers simply don't know about. Use
-		 * the I/O-memory resource as-is and try to map that instead.
-		 */
-		drm_warn(dev, "could not acquire memory region %pr\n", res);
-		mem = res;
-	}
-
-	screen_base = devm_ioremap_wc(&pdev->dev, mem->start,
-				      resource_size(mem));
-	if (!screen_base)
-		return -ENOMEM;
-
-	sdev->mem = mem;
-	sdev->screen_base = screen_base;
-
-	return 0;
-}
-
-/*
  * Modesetting
  */
 
@@ -576,7 +453,7 @@ static int simpledrm_device_init_mm(struct simpledrm_device *sdev)
  * TODO: Add blit helpers for remaining formats and uncomment
  *       constants.
  */
-static const uint32_t simpledrm_default_formats[] = {
+static const uint32_t simpledrm_primary_plane_formats[] = {
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_ARGB8888,
 	DRM_FORMAT_RGB565,
@@ -587,100 +464,54 @@ static const uint32_t simpledrm_default_formats[] = {
 	DRM_FORMAT_ARGB2101010,
 };
 
-static const uint64_t simpledrm_format_modifiers[] = {
+static const uint64_t simpledrm_primary_plane_format_modifiers[] = {
 	DRM_FORMAT_MOD_LINEAR,
 	DRM_FORMAT_MOD_INVALID
 };
 
-static int simpledrm_connector_helper_get_modes(struct drm_connector *connector)
-{
-	struct simpledrm_device *sdev = simpledrm_device_of_dev(connector->dev);
-	struct drm_display_mode *mode;
-
-	mode = drm_mode_duplicate(connector->dev, &sdev->mode);
-	if (!mode)
-		return 0;
-
-	if (mode->name[0] == '\0')
-		drm_mode_set_name(mode);
-
-	mode->type |= DRM_MODE_TYPE_PREFERRED;
-	drm_mode_probed_add(connector, mode);
-
-	if (mode->width_mm)
-		connector->display_info.width_mm = mode->width_mm;
-	if (mode->height_mm)
-		connector->display_info.height_mm = mode->height_mm;
-
-	return 1;
-}
-
-static const struct drm_connector_helper_funcs simpledrm_connector_helper_funcs = {
-	.get_modes = simpledrm_connector_helper_get_modes,
-};
-
-static const struct drm_connector_funcs simpledrm_connector_funcs = {
-	.reset = drm_atomic_helper_connector_reset,
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = drm_connector_cleanup,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-};
-
-static enum drm_mode_status
-simpledrm_simple_display_pipe_mode_valid(struct drm_simple_display_pipe *pipe,
-				    const struct drm_display_mode *mode)
+static void simpledrm_primary_plane_helper_atomic_update(struct drm_plane *plane,
+							 struct drm_atomic_state *state)
 {
-	struct simpledrm_device *sdev = simpledrm_device_of_dev(pipe->crtc.dev);
-
-	if (mode->hdisplay != sdev->mode.hdisplay &&
-	    mode->vdisplay != sdev->mode.vdisplay)
-		return MODE_ONE_SIZE;
-	else if (mode->hdisplay != sdev->mode.hdisplay)
-		return MODE_ONE_WIDTH;
-	else if (mode->vdisplay != sdev->mode.vdisplay)
-		return MODE_ONE_HEIGHT;
-
-	return MODE_OK;
-}
-
-static void
-simpledrm_simple_display_pipe_enable(struct drm_simple_display_pipe *pipe,
-				     struct drm_crtc_state *crtc_state,
-				     struct drm_plane_state *plane_state)
-{
-	struct simpledrm_device *sdev = simpledrm_device_of_dev(pipe->crtc.dev);
+	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
+	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
 	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
 	struct drm_framebuffer *fb = plane_state->fb;
-	void *vmap = shadow_plane_state->data[0].vaddr; /* TODO: Use mapping abstraction */
-	struct drm_device *dev = &sdev->dev;
-	void __iomem *dst = sdev->screen_base;
-	struct drm_rect src_clip, dst_clip;
-	int idx;
+	struct drm_device *dev = plane->dev;
+	struct simpledrm_device *sdev = simpledrm_device_of_dev(dev);
+	struct drm_atomic_helper_damage_iter iter;
+	struct drm_rect damage;
+	int ret, idx;
 
-	if (!fb)
+	ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE);
+	if (ret)
 		return;
 
-	drm_rect_fp_to_int(&src_clip, &plane_state->src);
+	if (!drm_dev_enter(dev, &idx))
+		goto out_drm_gem_fb_end_cpu_access;
 
-	dst_clip = plane_state->dst;
-	if (!drm_rect_intersect(&dst_clip, &src_clip))
-		return;
+	drm_atomic_helper_damage_iter_init(&iter, old_plane_state, plane_state);
+	drm_atomic_for_each_plane_damage(&iter, &damage) {
+		struct iosys_map dst = IOSYS_MAP_INIT_VADDR(sdev->screen_base);
+		struct drm_rect dst_clip = plane_state->dst;
 
-	if (!drm_dev_enter(dev, &idx))
-		return;
+		if (!drm_rect_intersect(&dst_clip, &damage))
+			continue;
 
-	dst += drm_fb_clip_offset(sdev->pitch, sdev->format, &dst_clip);
-	drm_fb_blit_toio(dst, sdev->pitch, sdev->format->format, vmap, fb, &src_clip);
+		iosys_map_incr(&dst, drm_fb_clip_offset(sdev->pitch, sdev->format, &dst_clip));
+		drm_fb_blit(&dst, &sdev->pitch, sdev->format->format, shadow_plane_state->data, fb,
+			    &damage);
+	}
 
 	drm_dev_exit(idx);
+out_drm_gem_fb_end_cpu_access:
+	drm_gem_fb_end_cpu_access(fb, DMA_FROM_DEVICE);
 }
 
-static void
-simpledrm_simple_display_pipe_disable(struct drm_simple_display_pipe *pipe)
+static void simpledrm_primary_plane_helper_atomic_disable(struct drm_plane *plane,
+							  struct drm_atomic_state *state)
 {
-	struct simpledrm_device *sdev = simpledrm_device_of_dev(pipe->crtc.dev);
-	struct drm_device *dev = &sdev->dev;
+	struct drm_device *dev = plane->dev;
+	struct simpledrm_device *sdev = simpledrm_device_of_dev(dev);
 	int idx;
 
 	if (!drm_dev_enter(dev, &idx))
@@ -692,46 +523,81 @@ simpledrm_simple_display_pipe_disable(struct drm_simple_display_pipe *pipe)
 	drm_dev_exit(idx);
 }
 
-static void
-simpledrm_simple_display_pipe_update(struct drm_simple_display_pipe *pipe,
-				     struct drm_plane_state *old_plane_state)
+static const struct drm_plane_helper_funcs simpledrm_primary_plane_helper_funcs = {
+	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS,
+	.atomic_check = drm_plane_helper_atomic_check,
+	.atomic_update = simpledrm_primary_plane_helper_atomic_update,
+	.atomic_disable = simpledrm_primary_plane_helper_atomic_disable,
+};
+
+static const struct drm_plane_funcs simpledrm_primary_plane_funcs = {
+	.update_plane = drm_atomic_helper_update_plane,
+	.disable_plane = drm_atomic_helper_disable_plane,
+	.destroy = drm_plane_cleanup,
+	DRM_GEM_SHADOW_PLANE_FUNCS,
+};
+
+static enum drm_mode_status simpledrm_crtc_helper_mode_valid(struct drm_crtc *crtc,
+							     const struct drm_display_mode *mode)
 {
-	struct simpledrm_device *sdev = simpledrm_device_of_dev(pipe->crtc.dev);
-	struct drm_plane_state *plane_state = pipe->plane.state;
-	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
-	void *vmap = shadow_plane_state->data[0].vaddr; /* TODO: Use mapping abstraction */
-	struct drm_framebuffer *fb = plane_state->fb;
-	struct drm_device *dev = &sdev->dev;
-	void __iomem *dst = sdev->screen_base;
-	struct drm_rect src_clip, dst_clip;
-	int idx;
+	struct simpledrm_device *sdev = simpledrm_device_of_dev(crtc->dev);
 
-	if (!fb)
-		return;
+	return drm_crtc_helper_mode_valid_fixed(crtc, mode, &sdev->mode);
+}
 
-	if (!drm_atomic_helper_damage_merged(old_plane_state, plane_state, &src_clip))
-		return;
+static int simpledrm_crtc_helper_atomic_check(struct drm_crtc *crtc,
+					      struct drm_atomic_state *new_state)
+{
+	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(new_state, crtc);
+	int ret;
 
-	dst_clip = plane_state->dst;
-	if (!drm_rect_intersect(&dst_clip, &src_clip))
-		return;
+	ret = drm_atomic_helper_check_crtc_state(new_crtc_state, false);
+	if (ret)
+		return ret;
 
-	if (!drm_dev_enter(dev, &idx))
-		return;
+	return drm_atomic_add_affected_planes(new_state, crtc);
+}
 
-	dst += drm_fb_clip_offset(sdev->pitch, sdev->format, &dst_clip);
-	drm_fb_blit_toio(dst, sdev->pitch, sdev->format->format, vmap, fb, &src_clip);
+/*
+ * The CRTC is always enabled. Screen updates are performed by
+ * the primary plane's atomic_update function. Disabling clears
+ * the screen in the primary plane's atomic_disable function.
+ */
+static const struct drm_crtc_helper_funcs simpledrm_crtc_helper_funcs = {
+	.mode_valid = simpledrm_crtc_helper_mode_valid,
+	.atomic_check = simpledrm_crtc_helper_atomic_check,
+};
 
-	drm_dev_exit(idx);
+static const struct drm_crtc_funcs simpledrm_crtc_funcs = {
+	.reset = drm_atomic_helper_crtc_reset,
+	.destroy = drm_crtc_cleanup,
+	.set_config = drm_atomic_helper_set_config,
+	.page_flip = drm_atomic_helper_page_flip,
+	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+};
+
+static const struct drm_encoder_funcs simpledrm_encoder_funcs = {
+	.destroy = drm_encoder_cleanup,
+};
+
+static int simpledrm_connector_helper_get_modes(struct drm_connector *connector)
+{
+	struct simpledrm_device *sdev = simpledrm_device_of_dev(connector->dev);
+
+	return drm_connector_helper_get_modes_fixed(connector, &sdev->mode);
 }
 
-static const struct drm_simple_display_pipe_funcs
-simpledrm_simple_display_pipe_funcs = {
-	.mode_valid = simpledrm_simple_display_pipe_mode_valid,
-	.enable = simpledrm_simple_display_pipe_enable,
-	.disable = simpledrm_simple_display_pipe_disable,
-	.update = simpledrm_simple_display_pipe_update,
-	DRM_GEM_SIMPLE_DISPLAY_PIPE_SHADOW_PLANE_FUNCS,
+static const struct drm_connector_helper_funcs simpledrm_connector_helper_funcs = {
+	.get_modes = simpledrm_connector_helper_get_modes,
+};
+
+static const struct drm_connector_funcs simpledrm_connector_funcs = {
+	.reset = drm_atomic_helper_connector_reset,
+	.fill_modes = drm_helper_probe_single_connector_modes,
+	.destroy = drm_connector_cleanup,
+	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
 static const struct drm_mode_config_funcs simpledrm_mode_config_funcs = {
@@ -740,127 +606,207 @@ static const struct drm_mode_config_funcs simpledrm_mode_config_funcs = {
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-static const uint32_t *simpledrm_device_formats(struct simpledrm_device *sdev,
-						size_t *nformats_out)
-{
-	struct drm_device *dev = &sdev->dev;
-	size_t i;
-
-	if (sdev->nformats)
-		goto out; /* don't rebuild list on recurring calls */
-
-	/* native format goes first */
-	sdev->formats[0] = sdev->format->format;
-	sdev->nformats = 1;
-
-	/* default formats go second */
-	for (i = 0; i < ARRAY_SIZE(simpledrm_default_formats); ++i) {
-		if (simpledrm_default_formats[i] == sdev->format->format)
-			continue; /* native format already went first */
-		sdev->formats[sdev->nformats] = simpledrm_default_formats[i];
-		sdev->nformats++;
-	}
+/*
+ * Init / Cleanup
+ */
 
+static struct drm_display_mode simpledrm_mode(unsigned int width,
+					      unsigned int height)
+{
 	/*
-	 * TODO: The simpledrm driver converts framebuffers to the native
-	 * format when copying them to device memory. If there are more
-	 * formats listed than supported by the driver, the native format
-	 * is not supported by the conversion helpers. Therefore *only*
-	 * support the native format and add a conversion helper ASAP.
+	 * Assume a monitor resolution of 96 dpi to
+	 * get a somewhat reasonable screen size.
 	 */
-	if (drm_WARN_ONCE(dev, i != sdev->nformats,
-			  "format conversion helpers required for %p4cc",
-			  &sdev->format->format)) {
-		sdev->nformats = 1;
-	}
+	const struct drm_display_mode mode = {
+		DRM_MODE_INIT(60, width, height,
+			      DRM_MODE_RES_MM(width, 96ul),
+			      DRM_MODE_RES_MM(height, 96ul))
+	};
 
-out:
-	*nformats_out = sdev->nformats;
-	return sdev->formats;
+	return mode;
 }
 
-static int simpledrm_device_init_modeset(struct simpledrm_device *sdev)
+static struct simpledrm_device *simpledrm_device_create(struct drm_driver *drv,
+							struct platform_device *pdev)
 {
-	struct drm_device *dev = &sdev->dev;
-	struct drm_display_mode *mode = &sdev->mode;
-	struct drm_connector *connector = &sdev->connector;
-	struct drm_simple_display_pipe *pipe = &sdev->pipe;
+	const struct simplefb_platform_data *pd = dev_get_platdata(&pdev->dev);
+	struct device_node *of_node = pdev->dev.of_node;
+	struct simpledrm_device *sdev;
+	struct drm_device *dev;
+	int width, height, stride;
+	const struct drm_format_info *format;
+	struct resource *res, *mem;
+	void __iomem *screen_base;
+	struct drm_plane *primary_plane;
+	struct drm_crtc *crtc;
+	struct drm_encoder *encoder;
+	struct drm_connector *connector;
 	unsigned long max_width, max_height;
-	const uint32_t *formats;
 	size_t nformats;
 	int ret;
 
-	ret = drmm_mode_config_init(dev);
+	sdev = devm_drm_dev_alloc(&pdev->dev, drv, struct simpledrm_device, dev);
+	if (IS_ERR(sdev))
+		return ERR_CAST(sdev);
+	dev = &sdev->dev;
+	platform_set_drvdata(pdev, sdev);
+
+	/*
+	 * Hardware settings
+	 */
+
+	ret = simpledrm_device_init_clocks(sdev);
 	if (ret)
-		return ret;
+		return ERR_PTR(ret);
+	ret = simpledrm_device_init_regulators(sdev);
+	if (ret)
+		return ERR_PTR(ret);
+
+	if (pd) {
+		width = simplefb_get_width_pd(dev, pd);
+		if (width < 0)
+			return ERR_PTR(width);
+		height = simplefb_get_height_pd(dev, pd);
+		if (height < 0)
+			return ERR_PTR(height);
+		stride = simplefb_get_stride_pd(dev, pd);
+		if (stride < 0)
+			return ERR_PTR(stride);
+		format = simplefb_get_format_pd(dev, pd);
+		if (IS_ERR(format))
+			return ERR_CAST(format);
+	} else if (of_node) {
+		width = simplefb_get_width_of(dev, of_node);
+		if (width < 0)
+			return ERR_PTR(width);
+		height = simplefb_get_height_of(dev, of_node);
+		if (height < 0)
+			return ERR_PTR(height);
+		stride = simplefb_get_stride_of(dev, of_node);
+		if (stride < 0)
+			return ERR_PTR(stride);
+		format = simplefb_get_format_of(dev, of_node);
+		if (IS_ERR(format))
+			return ERR_CAST(format);
+	} else {
+		drm_err(dev, "no simplefb configuration found\n");
+		return ERR_PTR(-ENODEV);
+	}
+	if (!stride) {
+		stride = drm_format_info_min_pitch(format, 0, width);
+		if (drm_WARN_ON(dev, !stride))
+			return ERR_PTR(-EINVAL);
+	}
 
-	max_width = max_t(unsigned long, mode->hdisplay, DRM_SHADOW_PLANE_MAX_WIDTH);
-	max_height = max_t(unsigned long, mode->vdisplay, DRM_SHADOW_PLANE_MAX_HEIGHT);
+	sdev->mode = simpledrm_mode(width, height);
+	sdev->format = format;
+	sdev->pitch = stride;
 
-	dev->mode_config.min_width = mode->hdisplay;
-	dev->mode_config.max_width = max_width;
-	dev->mode_config.min_height = mode->vdisplay;
-	dev->mode_config.max_height = max_height;
-	dev->mode_config.preferred_depth = sdev->format->cpp[0] * 8;
-	dev->mode_config.funcs = &simpledrm_mode_config_funcs;
+	drm_dbg(dev, "display mode={" DRM_MODE_FMT "}\n", DRM_MODE_ARG(&sdev->mode));
+	drm_dbg(dev, "framebuffer format=%p4cc, size=%dx%d, stride=%d byte\n",
+		&format->format, width, height, stride);
 
-	ret = drm_connector_init(dev, connector, &simpledrm_connector_funcs,
-				 DRM_MODE_CONNECTOR_Unknown);
-	if (ret)
-		return ret;
-	drm_connector_helper_add(connector, &simpledrm_connector_helper_funcs);
-	drm_connector_set_panel_orientation_with_quirk(connector,
-						       DRM_MODE_PANEL_ORIENTATION_UNKNOWN,
-						       mode->hdisplay, mode->vdisplay);
+	/*
+	 * Memory management
+	 */
 
-	formats = simpledrm_device_formats(sdev, &nformats);
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return ERR_PTR(-EINVAL);
 
-	ret = drm_simple_display_pipe_init(dev, pipe, &simpledrm_simple_display_pipe_funcs,
-					   formats, nformats, simpledrm_format_modifiers,
-					   connector);
-	if (ret)
-		return ret;
+	ret = devm_aperture_acquire_from_firmware(dev, res->start, resource_size(res));
+	if (ret) {
+		drm_err(dev, "could not acquire memory range %pr: error %d\n", res, ret);
+		return ERR_PTR(ret);
+	}
 
-	drm_plane_enable_fb_damage_clips(&pipe->plane);
+	mem = devm_request_mem_region(&pdev->dev, res->start, resource_size(res), drv->name);
+	if (!mem) {
+		/*
+		 * We cannot make this fatal. Sometimes this comes from magic
+		 * spaces our resource handlers simply don't know about. Use
+		 * the I/O-memory resource as-is and try to map that instead.
+		 */
+		drm_warn(dev, "could not acquire memory region %pr\n", res);
+		mem = res;
+	}
 
-	drm_mode_config_reset(dev);
+	screen_base = devm_ioremap_wc(&pdev->dev, mem->start, resource_size(mem));
+	if (!screen_base)
+		return ERR_PTR(-ENOMEM);
+	sdev->screen_base = screen_base;
 
-	return 0;
-}
+	/*
+	 * Modesetting
+	 */
 
-/*
- * Init / Cleanup
- */
+	ret = drmm_mode_config_init(dev);
+	if (ret)
+		return ERR_PTR(ret);
 
-static struct simpledrm_device *
-simpledrm_device_create(struct drm_driver *drv, struct platform_device *pdev)
-{
-	struct simpledrm_device *sdev;
-	int ret;
+	max_width = max_t(unsigned long, width, DRM_SHADOW_PLANE_MAX_WIDTH);
+	max_height = max_t(unsigned long, height, DRM_SHADOW_PLANE_MAX_HEIGHT);
 
-	sdev = devm_drm_dev_alloc(&pdev->dev, drv, struct simpledrm_device,
-				  dev);
-	if (IS_ERR(sdev))
-		return ERR_CAST(sdev);
-	sdev->pdev = pdev;
-	platform_set_drvdata(pdev, sdev);
+	dev->mode_config.min_width = width;
+	dev->mode_config.max_width = max_width;
+	dev->mode_config.min_height = height;
+	dev->mode_config.max_height = max_height;
+	dev->mode_config.preferred_depth = format->cpp[0] * 8;
+	dev->mode_config.funcs = &simpledrm_mode_config_funcs;
 
-	ret = simpledrm_device_init_clocks(sdev);
+	/* Primary plane */
+
+	nformats = drm_fb_build_fourcc_list(dev, &format->format, 1,
+					    simpledrm_primary_plane_formats,
+					    ARRAY_SIZE(simpledrm_primary_plane_formats),
+					    sdev->formats, ARRAY_SIZE(sdev->formats));
+
+	primary_plane = &sdev->primary_plane;
+	ret = drm_universal_plane_init(dev, primary_plane, 0, &simpledrm_primary_plane_funcs,
+				       sdev->formats, nformats,
+				       simpledrm_primary_plane_format_modifiers,
+				       DRM_PLANE_TYPE_PRIMARY, NULL);
 	if (ret)
 		return ERR_PTR(ret);
-	ret = simpledrm_device_init_regulators(sdev);
+	drm_plane_helper_add(primary_plane, &simpledrm_primary_plane_helper_funcs);
+	drm_plane_enable_fb_damage_clips(primary_plane);
+
+	/* CRTC */
+
+	crtc = &sdev->crtc;
+	ret = drm_crtc_init_with_planes(dev, crtc, primary_plane, NULL,
+					&simpledrm_crtc_funcs, NULL);
 	if (ret)
 		return ERR_PTR(ret);
-	ret = simpledrm_device_init_fb(sdev);
+	drm_crtc_helper_add(crtc, &simpledrm_crtc_helper_funcs);
+
+	/* Encoder */
+
+	encoder = &sdev->encoder;
+	ret = drm_encoder_init(dev, encoder, &simpledrm_encoder_funcs,
+			       DRM_MODE_ENCODER_NONE, NULL);
 	if (ret)
 		return ERR_PTR(ret);
-	ret = simpledrm_device_init_mm(sdev);
+	encoder->possible_crtcs = drm_crtc_mask(crtc);
+
+	/* Connector */
+
+	connector = &sdev->connector;
+	ret = drm_connector_init(dev, connector, &simpledrm_connector_funcs,
+				 DRM_MODE_CONNECTOR_Unknown);
 	if (ret)
 		return ERR_PTR(ret);
-	ret = simpledrm_device_init_modeset(sdev);
+	drm_connector_helper_add(connector, &simpledrm_connector_helper_funcs);
+	drm_connector_set_panel_orientation_with_quirk(connector,
+						       DRM_MODE_PANEL_ORIENTATION_UNKNOWN,
+						       width, height);
+
+	ret = drm_connector_attach_encoder(connector, encoder);
 	if (ret)
 		return ERR_PTR(ret);
 
+	drm_mode_config_reset(dev);
+
 	return sdev;
 }
 
diff --git a/drivers/gpu/drm/tiny/st7586.c b/drivers/gpu/drm/tiny/st7586.c
index 8eddb020c43e..b6f620b902e6 100644
--- a/drivers/gpu/drm/tiny/st7586.c
+++ b/drivers/gpu/drm/tiny/st7586.c
@@ -15,12 +15,12 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_format_helper.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mipi_dbi.h>
@@ -69,12 +69,15 @@ static void st7586_xrgb8888_to_gray332(u8 *dst, void *vaddr,
 	size_t len = (clip->x2 - clip->x1) * (clip->y2 - clip->y1);
 	unsigned int x, y;
 	u8 *src, *buf, val;
+	struct iosys_map dst_map, vmap;
 
 	buf = kmalloc(len, GFP_KERNEL);
 	if (!buf)
 		return;
 
-	drm_fb_xrgb8888_to_gray8(buf, 0, vaddr, fb, clip);
+	iosys_map_set_vaddr(&dst_map, buf);
+	iosys_map_set_vaddr(&vmap, vaddr);
+	drm_fb_xrgb8888_to_gray8(&dst_map, NULL, &vmap, fb, clip);
 	src = buf;
 
 	for (y = clip->y1; y < clip->y2; y++) {
@@ -92,8 +95,8 @@ static void st7586_xrgb8888_to_gray332(u8 *dst, void *vaddr,
 static int st7586_buf_copy(void *dst, struct drm_framebuffer *fb,
 			   struct drm_rect *clip)
 {
-	struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
-	void *src = cma_obj->vaddr;
+	struct drm_gem_dma_object *dma_obj = drm_fb_dma_get_gem_obj(fb, 0);
+	void *src = dma_obj->vaddr;
 	int ret = 0;
 
 	ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE);
@@ -269,12 +272,12 @@ static const struct drm_display_mode st7586_mode = {
 	DRM_SIMPLE_MODE(178, 128, 37, 27),
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(st7586_fops);
+DEFINE_DRM_GEM_DMA_FOPS(st7586_fops);
 
 static const struct drm_driver st7586_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &st7586_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "st7586",
 	.desc			= "Sitronix ST7586",
diff --git a/drivers/gpu/drm/tiny/st7735r.c b/drivers/gpu/drm/tiny/st7735r.c
index e0f02d367d88..c36ba08acda1 100644
--- a/drivers/gpu/drm/tiny/st7735r.c
+++ b/drivers/gpu/drm/tiny/st7735r.c
@@ -20,7 +20,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mipi_dbi.h>
 
@@ -133,6 +133,7 @@ out_exit:
 }
 
 static const struct drm_simple_display_pipe_funcs st7735r_pipe_funcs = {
+	.mode_valid	= mipi_dbi_pipe_mode_valid,
 	.enable		= st7735r_pipe_enable,
 	.disable	= mipi_dbi_pipe_disable,
 	.update		= mipi_dbi_pipe_update,
@@ -151,12 +152,12 @@ static const struct st7735r_cfg rh128128t_cfg = {
 	.rgb		= true,
 };
 
-DEFINE_DRM_GEM_CMA_FOPS(st7735r_fops);
+DEFINE_DRM_GEM_DMA_FOPS(st7735r_fops);
 
 static const struct drm_driver st7735r_driver = {
 	.driver_features	= DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC,
 	.fops			= &st7735r_fops,
-	DRM_GEM_CMA_DRIVER_OPS_VMAP,
+	DRM_GEM_DMA_DRIVER_OPS_VMAP,
 	.debugfs_init		= mipi_dbi_debugfs_init,
 	.name			= "st7735r",
 	.desc			= "Sitronix ST7735R",
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 97184c333526..7c8e8be774f1 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -117,12 +117,13 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 				  struct ttm_operation_ctx *ctx,
 				  struct ttm_place *hop)
 {
-	struct ttm_resource_manager *old_man, *new_man;
 	struct ttm_device *bdev = bo->bdev;
+	bool old_use_tt, new_use_tt;
 	int ret;
 
-	old_man = ttm_manager_type(bdev, bo->resource->mem_type);
-	new_man = ttm_manager_type(bdev, mem->mem_type);
+	old_use_tt = bo->resource &&
+		ttm_manager_type(bdev, bo->resource->mem_type)->use_tt;
+	new_use_tt = ttm_manager_type(bdev, mem->mem_type)->use_tt;
 
 	ttm_bo_unmap_virtual(bo);
 
@@ -130,11 +131,11 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 	 * Create and bind a ttm if required.
 	 */
 
-	if (new_man->use_tt) {
+	if (new_use_tt) {
 		/* Zero init the new TTM structure if the old location should
 		 * have used one as well.
 		 */
-		ret = ttm_tt_create(bo, old_man->use_tt);
+		ret = ttm_tt_create(bo, old_use_tt);
 		if (ret)
 			goto out_err;
 
@@ -160,8 +161,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
 	return 0;
 
 out_err:
-	new_man = ttm_manager_type(bdev, bo->resource->mem_type);
-	if (!new_man->use_tt)
+	if (!old_use_tt)
 		ttm_bo_tt_destroy(bo);
 
 	return ret;
@@ -518,6 +518,9 @@ out:
 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 			      const struct ttm_place *place)
 {
+	struct ttm_resource *res = bo->resource;
+	struct ttm_device *bdev = bo->bdev;
+
 	dma_resv_assert_held(bo->base.resv);
 	if (bo->resource->mem_type == TTM_PL_SYSTEM)
 		return true;
@@ -525,11 +528,7 @@ bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 	/* Don't evict this BO if it's outside of the
 	 * requested placement range
 	 */
-	if (place->fpfn >= (bo->resource->start + bo->resource->num_pages) ||
-	    (place->lpfn && place->lpfn <= bo->resource->start))
-		return false;
-
-	return true;
+	return ttm_resource_intersects(bdev, res, place, bo->base.size);
 }
 EXPORT_SYMBOL(ttm_bo_eviction_valuable);
 
@@ -904,7 +903,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,
 	/*
 	 * Check whether we need to move buffer.
 	 */
-	if (!ttm_resource_compat(bo->resource, placement)) {
+	if (!bo->resource || !ttm_resource_compat(bo->resource, placement)) {
 		ret = ttm_bo_move_buffer(bo, placement, ctx);
 		if (ret)
 			return ret;
@@ -921,36 +920,61 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,
 }
 EXPORT_SYMBOL(ttm_bo_validate);
 
-int ttm_bo_init_reserved(struct ttm_device *bdev,
-			 struct ttm_buffer_object *bo,
-			 size_t size,
-			 enum ttm_bo_type type,
-			 struct ttm_placement *placement,
-			 uint32_t page_alignment,
-			 struct ttm_operation_ctx *ctx,
-			 struct sg_table *sg,
-			 struct dma_resv *resv,
+/**
+ * ttm_bo_init_reserved
+ *
+ * @bdev: Pointer to a ttm_device struct.
+ * @bo: Pointer to a ttm_buffer_object to be initialized.
+ * @type: Requested type of buffer object.
+ * @placement: Initial placement for buffer object.
+ * @alignment: Data alignment in pages.
+ * @ctx: TTM operation context for memory allocation.
+ * @sg: Scatter-gather table.
+ * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
+ * @destroy: Destroy function. Use NULL for kfree().
+ *
+ * This function initializes a pre-allocated struct ttm_buffer_object.
+ * As this object may be part of a larger structure, this function,
+ * together with the @destroy function, enables driver-specific objects
+ * derived from a ttm_buffer_object.
+ *
+ * On successful return, the caller owns an object kref to @bo. The kref and
+ * list_kref are usually set to 1, but note that in some situations, other
+ * tasks may already be holding references to @bo as well.
+ * Furthermore, if resv == NULL, the buffer's reservation lock will be held,
+ * and it is the caller's responsibility to call ttm_bo_unreserve.
+ *
+ * If a failure occurs, the function will call the @destroy function. Thus,
+ * after a failure, dereferencing @bo is illegal and will likely cause memory
+ * corruption.
+ *
+ * Returns
+ * -ENOMEM: Out of memory.
+ * -EINVAL: Invalid placement flags.
+ * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
+ */
+int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo,
+			 enum ttm_bo_type type, struct ttm_placement *placement,
+			 uint32_t alignment, struct ttm_operation_ctx *ctx,
+			 struct sg_table *sg, struct dma_resv *resv,
 			 void (*destroy) (struct ttm_buffer_object *))
 {
 	static const struct ttm_place sys_mem = { .mem_type = TTM_PL_SYSTEM };
-	bool locked;
 	int ret;
 
-	bo->destroy = destroy;
 	kref_init(&bo->kref);
 	INIT_LIST_HEAD(&bo->ddestroy);
 	bo->bdev = bdev;
 	bo->type = type;
-	bo->page_alignment = page_alignment;
+	bo->page_alignment = alignment;
+	bo->destroy = destroy;
 	bo->pin_count = 0;
 	bo->sg = sg;
 	bo->bulk_move = NULL;
-	if (resv) {
+	if (resv)
 		bo->base.resv = resv;
-		dma_resv_assert_held(bo->base.resv);
-	} else {
+	else
 		bo->base.resv = &bo->base._resv;
-	}
 	atomic_inc(&ttm_glob.bo_count);
 
 	ret = ttm_resource_alloc(bo, &sys_mem, &bo->resource);
@@ -963,50 +987,84 @@ int ttm_bo_init_reserved(struct ttm_device *bdev,
 	 * For ttm_bo_type_device buffers, allocate
 	 * address space from the device.
 	 */
-	if (bo->type == ttm_bo_type_device ||
-	    bo->type == ttm_bo_type_sg)
+	if (bo->type == ttm_bo_type_device || bo->type == ttm_bo_type_sg) {
 		ret = drm_vma_offset_add(bdev->vma_manager, &bo->base.vma_node,
-					 bo->resource->num_pages);
+					 PFN_UP(bo->base.size));
+		if (ret)
+			goto err_put;
+	}
 
 	/* passed reservation objects should already be locked,
 	 * since otherwise lockdep will be angered in radeon.
 	 */
-	if (!resv) {
-		locked = dma_resv_trylock(bo->base.resv);
-		WARN_ON(!locked);
-	}
+	if (!resv)
+		WARN_ON(!dma_resv_trylock(bo->base.resv));
+	else
+		dma_resv_assert_held(resv);
 
-	if (likely(!ret))
-		ret = ttm_bo_validate(bo, placement, ctx);
+	ret = ttm_bo_validate(bo, placement, ctx);
+	if (unlikely(ret))
+		goto err_unlock;
 
-	if (unlikely(ret)) {
-		if (!resv)
-			ttm_bo_unreserve(bo);
+	return 0;
 
-		ttm_bo_put(bo);
-		return ret;
-	}
+err_unlock:
+	if (!resv)
+		dma_resv_unlock(bo->base.resv);
 
+err_put:
+	ttm_bo_put(bo);
 	return ret;
 }
 EXPORT_SYMBOL(ttm_bo_init_reserved);
 
-int ttm_bo_init(struct ttm_device *bdev,
-		struct ttm_buffer_object *bo,
-		size_t size,
-		enum ttm_bo_type type,
-		struct ttm_placement *placement,
-		uint32_t page_alignment,
-		bool interruptible,
-		struct sg_table *sg,
-		struct dma_resv *resv,
-		void (*destroy) (struct ttm_buffer_object *))
+/**
+ * ttm_bo_init_validate
+ *
+ * @bdev: Pointer to a ttm_device struct.
+ * @bo: Pointer to a ttm_buffer_object to be initialized.
+ * @type: Requested type of buffer object.
+ * @placement: Initial placement for buffer object.
+ * @alignment: Data alignment in pages.
+ * @interruptible: If needing to sleep to wait for GPU resources,
+ * sleep interruptible.
+ * pinned in physical memory. If this behaviour is not desired, this member
+ * holds a pointer to a persistent shmem object. Typically, this would
+ * point to the shmem object backing a GEM object if TTM is used to back a
+ * GEM user interface.
+ * @sg: Scatter-gather table.
+ * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
+ * @destroy: Destroy function. Use NULL for kfree().
+ *
+ * This function initializes a pre-allocated struct ttm_buffer_object.
+ * As this object may be part of a larger structure, this function,
+ * together with the @destroy function,
+ * enables driver-specific objects derived from a ttm_buffer_object.
+ *
+ * On successful return, the caller owns an object kref to @bo. The kref and
+ * list_kref are usually set to 1, but note that in some situations, other
+ * tasks may already be holding references to @bo as well.
+ *
+ * If a failure occurs, the function will call the @destroy function, Thus,
+ * after a failure, dereferencing @bo is illegal and will likely cause memory
+ * corruption.
+ *
+ * Returns
+ * -ENOMEM: Out of memory.
+ * -EINVAL: Invalid placement flags.
+ * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
+ */
+int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo,
+			 enum ttm_bo_type type, struct ttm_placement *placement,
+			 uint32_t alignment, bool interruptible,
+			 struct sg_table *sg, struct dma_resv *resv,
+			 void (*destroy) (struct ttm_buffer_object *))
 {
 	struct ttm_operation_ctx ctx = { interruptible, false };
 	int ret;
 
-	ret = ttm_bo_init_reserved(bdev, bo, size, type, placement,
-				   page_alignment, &ctx, sg, resv, destroy);
+	ret = ttm_bo_init_reserved(bdev, bo, type, placement, alignment, &ctx,
+				   sg, resv, destroy);
 	if (ret)
 		return ret;
 
@@ -1015,7 +1073,7 @@ int ttm_bo_init(struct ttm_device *bdev,
 
 	return 0;
 }
-EXPORT_SYMBOL(ttm_bo_init);
+EXPORT_SYMBOL(ttm_bo_init_validate);
 
 /*
  * buffer object vm functions.
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 911141d16e95..fa04e62202c1 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -137,8 +137,7 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 		ttm_manager_type(bo->bdev, dst_mem->mem_type);
 	struct ttm_tt *ttm = bo->ttm;
 	struct ttm_resource *src_mem = bo->resource;
-	struct ttm_resource_manager *src_man =
-		ttm_manager_type(bdev, src_mem->mem_type);
+	struct ttm_resource_manager *src_man;
 	union {
 		struct ttm_kmap_iter_tt tt;
 		struct ttm_kmap_iter_linear_io io;
@@ -147,6 +146,10 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
 	bool clear;
 	int ret = 0;
 
+	if (!src_mem)
+		return 0;
+
+	src_man = ttm_manager_type(bdev, src_mem->mem_type);
 	if (ttm && ((ttm->page_flags & TTM_TT_FLAG_SWAPPED) ||
 		    dst_man->use_tt)) {
 		ret = ttm_tt_populate(bdev, ttm, ctx);
@@ -402,6 +405,8 @@ int ttm_bo_vmap(struct ttm_buffer_object *bo, struct iosys_map *map)
 	struct ttm_resource *mem = bo->resource;
 	int ret;
 
+	dma_resv_assert_held(bo->base.resv);
+
 	ret = ttm_mem_io_reserve(bo->bdev, mem);
 	if (ret)
 		return ret;
@@ -460,6 +465,8 @@ void ttm_bo_vunmap(struct ttm_buffer_object *bo, struct iosys_map *map)
 {
 	struct ttm_resource *mem = bo->resource;
 
+	dma_resv_assert_held(bo->base.resv);
+
 	if (iosys_map_is_null(map))
 		return;
 
diff --git a/drivers/gpu/drm/ttm/ttm_range_manager.c b/drivers/gpu/drm/ttm/ttm_range_manager.c
index d91666721dc6..4cfef2b3514d 100644
--- a/drivers/gpu/drm/ttm/ttm_range_manager.c
+++ b/drivers/gpu/drm/ttm/ttm_range_manager.c
@@ -113,6 +113,37 @@ static void ttm_range_man_free(struct ttm_resource_manager *man,
 	kfree(node);
 }
 
+static bool ttm_range_man_intersects(struct ttm_resource_manager *man,
+				     struct ttm_resource *res,
+				     const struct ttm_place *place,
+				     size_t size)
+{
+	struct drm_mm_node *node = &to_ttm_range_mgr_node(res)->mm_nodes[0];
+	u32 num_pages = PFN_UP(size);
+
+	/* Don't evict BOs outside of the requested placement range */
+	if (place->fpfn >= (node->start + num_pages) ||
+	    (place->lpfn && place->lpfn <= node->start))
+		return false;
+
+	return true;
+}
+
+static bool ttm_range_man_compatible(struct ttm_resource_manager *man,
+				     struct ttm_resource *res,
+				     const struct ttm_place *place,
+				     size_t size)
+{
+	struct drm_mm_node *node = &to_ttm_range_mgr_node(res)->mm_nodes[0];
+	u32 num_pages = PFN_UP(size);
+
+	if (node->start < place->fpfn ||
+	    (place->lpfn && (node->start + num_pages) > place->lpfn))
+		return false;
+
+	return true;
+}
+
 static void ttm_range_man_debug(struct ttm_resource_manager *man,
 				struct drm_printer *printer)
 {
@@ -126,6 +157,8 @@ static void ttm_range_man_debug(struct ttm_resource_manager *man,
 static const struct ttm_resource_manager_func ttm_range_manager_func = {
 	.alloc = ttm_range_man_alloc,
 	.free = ttm_range_man_free,
+	.intersects = ttm_range_man_intersects,
+	.compatible = ttm_range_man_compatible,
 	.debug = ttm_range_man_debug
 };
 
diff --git a/drivers/gpu/drm/ttm/ttm_resource.c b/drivers/gpu/drm/ttm/ttm_resource.c
index 20f9adcc3235..a729c32a1e48 100644
--- a/drivers/gpu/drm/ttm/ttm_resource.c
+++ b/drivers/gpu/drm/ttm/ttm_resource.c
@@ -253,10 +253,71 @@ void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res)
 }
 EXPORT_SYMBOL(ttm_resource_free);
 
+/**
+ * ttm_resource_intersects - test for intersection
+ *
+ * @bdev: TTM device structure
+ * @res: The resource to test
+ * @place: The placement to test
+ * @size: How many bytes the new allocation needs.
+ *
+ * Test if @res intersects with @place and @size. Used for testing if evictions
+ * are valueable or not.
+ *
+ * Returns true if the res placement intersects with @place and @size.
+ */
+bool ttm_resource_intersects(struct ttm_device *bdev,
+			     struct ttm_resource *res,
+			     const struct ttm_place *place,
+			     size_t size)
+{
+	struct ttm_resource_manager *man;
+
+	if (!res)
+		return false;
+
+	man = ttm_manager_type(bdev, res->mem_type);
+	if (!place || !man->func->intersects)
+		return true;
+
+	return man->func->intersects(man, res, place, size);
+}
+
+/**
+ * ttm_resource_compatible - test for compatibility
+ *
+ * @bdev: TTM device structure
+ * @res: The resource to test
+ * @place: The placement to test
+ * @size: How many bytes the new allocation needs.
+ *
+ * Test if @res compatible with @place and @size.
+ *
+ * Returns true if the res placement compatible with @place and @size.
+ */
+bool ttm_resource_compatible(struct ttm_device *bdev,
+			     struct ttm_resource *res,
+			     const struct ttm_place *place,
+			     size_t size)
+{
+	struct ttm_resource_manager *man;
+
+	if (!res || !place)
+		return false;
+
+	man = ttm_manager_type(bdev, res->mem_type);
+	if (!man->func->compatible)
+		return true;
+
+	return man->func->compatible(man, res, place, size);
+}
+
 static bool ttm_resource_places_compat(struct ttm_resource *res,
 				       const struct ttm_place *places,
 				       unsigned num_placement)
 {
+	struct ttm_buffer_object *bo = res->bo;
+	struct ttm_device *bdev = bo->bdev;
 	unsigned i;
 
 	if (res->placement & TTM_PL_FLAG_TEMPORARY)
@@ -265,8 +326,7 @@ static bool ttm_resource_places_compat(struct ttm_resource *res,
 	for (i = 0; i < num_placement; i++) {
 		const struct ttm_place *heap = &places[i];
 
-		if (res->start < heap->fpfn || (heap->lpfn &&
-		    (res->start + res->num_pages) > heap->lpfn))
+		if (!ttm_resource_compatible(bdev, res, heap, bo->base.size))
 			continue;
 
 		if ((res->mem_type == heap->mem_type) &&
diff --git a/drivers/gpu/drm/tve200/Kconfig b/drivers/gpu/drm/tve200/Kconfig
index 47a7dbe6c114..11e865be81c6 100644
--- a/drivers/gpu/drm/tve200/Kconfig
+++ b/drivers/gpu/drm/tve200/Kconfig
@@ -8,7 +8,7 @@ config DRM_TVE200
 	select DRM_BRIDGE
 	select DRM_PANEL_BRIDGE
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select VT_HW_CONSOLE_BINDING if FRAMEBUFFER_CONSOLE
 	help
 	  Choose this option for DRM support for the Faraday TV Encoder
diff --git a/drivers/gpu/drm/tve200/tve200_display.c b/drivers/gpu/drm/tve200/tve200_display.c
index 771bad881714..37bdd976ae59 100644
--- a/drivers/gpu/drm/tve200/tve200_display.c
+++ b/drivers/gpu/drm/tve200/tve200_display.c
@@ -15,11 +15,11 @@
 #include <linux/of_graph.h>
 #include <linux/delay.h>
 
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_vblank.h>
 
@@ -90,7 +90,7 @@ static int tve200_display_check(struct drm_simple_display_pipe *pipe,
 	}
 
 	if (fb) {
-		u32 offset = drm_fb_cma_get_gem_addr(fb, pstate, 0);
+		u32 offset = drm_fb_dma_get_gem_addr(fb, pstate, 0);
 
 		/* FB base address must be dword aligned. */
 		if (offset & 3) {
@@ -267,14 +267,14 @@ static void tve200_display_update(struct drm_simple_display_pipe *pipe,
 
 	if (fb) {
 		/* For RGB, the Y component is used as base address */
-		writel(drm_fb_cma_get_gem_addr(fb, pstate, 0),
+		writel(drm_fb_dma_get_gem_addr(fb, pstate, 0),
 		       priv->regs + TVE200_Y_FRAME_BASE_ADDR);
 
 		/* For three plane YUV we need two more addresses */
 		if (fb->format->format == DRM_FORMAT_YUV420) {
-			writel(drm_fb_cma_get_gem_addr(fb, pstate, 1),
+			writel(drm_fb_dma_get_gem_addr(fb, pstate, 1),
 			       priv->regs + TVE200_U_FRAME_BASE_ADDR);
-			writel(drm_fb_cma_get_gem_addr(fb, pstate, 2),
+			writel(drm_fb_dma_get_gem_addr(fb, pstate, 2),
 			       priv->regs + TVE200_V_FRAME_BASE_ADDR);
 		}
 	}
diff --git a/drivers/gpu/drm/tve200/tve200_drv.c b/drivers/gpu/drm/tve200/tve200_drv.c
index 6d9d2921abf4..04db72e3fa9c 100644
--- a/drivers/gpu/drm/tve200/tve200_drv.c
+++ b/drivers/gpu/drm/tve200/tve200_drv.c
@@ -39,9 +39,8 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_module.h>
 #include <drm/drm_of.h>
@@ -65,7 +64,7 @@ static int tve200_modeset_init(struct drm_device *dev)
 	struct tve200_drm_dev_private *priv = dev->dev_private;
 	struct drm_panel *panel;
 	struct drm_bridge *bridge;
-	int ret = 0;
+	int ret;
 
 	drm_mode_config_init(dev);
 	mode_config = &dev->mode_config;
@@ -93,6 +92,7 @@ static int tve200_modeset_init(struct drm_device *dev)
 		 * method to get the connector out of the bridge.
 		 */
 		dev_err(dev->dev, "the bridge is not a panel\n");
+		ret = -EINVAL;
 		goto out_bridge;
 	}
 
@@ -135,7 +135,7 @@ finish:
 	return ret;
 }
 
-DEFINE_DRM_GEM_CMA_FOPS(drm_fops);
+DEFINE_DRM_GEM_DMA_FOPS(drm_fops);
 
 static const struct drm_driver tve200_drm_driver = {
 	.driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_ATOMIC,
@@ -147,7 +147,7 @@ static const struct drm_driver tve200_drm_driver = {
 	.major = 1,
 	.minor = 0,
 	.patchlevel = 0,
-	DRM_GEM_CMA_DRIVER_OPS,
+	DRM_GEM_DMA_DRIVER_OPS,
 };
 
 static int tve200_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/udl/udl_drv.c b/drivers/gpu/drm/udl/udl_drv.c
index 5703277c6f52..91effdcefb6d 100644
--- a/drivers/gpu/drm/udl/udl_drv.c
+++ b/drivers/gpu/drm/udl/udl_drv.c
@@ -21,8 +21,14 @@ static int udl_usb_suspend(struct usb_interface *interface,
 			   pm_message_t message)
 {
 	struct drm_device *dev = usb_get_intfdata(interface);
+	int ret;
 
-	return drm_mode_config_helper_suspend(dev);
+	ret = drm_mode_config_helper_suspend(dev);
+	if (ret)
+		return ret;
+
+	udl_sync_pending_urbs(dev);
+	return 0;
 }
 
 static int udl_usb_resume(struct usb_interface *interface)
@@ -32,6 +38,16 @@ static int udl_usb_resume(struct usb_interface *interface)
 	return drm_mode_config_helper_resume(dev);
 }
 
+static int udl_usb_reset_resume(struct usb_interface *interface)
+{
+	struct drm_device *dev = usb_get_intfdata(interface);
+	struct udl_device *udl = to_udl(dev);
+
+	udl_select_std_channel(udl);
+
+	return drm_mode_config_helper_resume(dev);
+}
+
 /*
  * FIXME: Dma-buf sharing requires DMA support by the importing device.
  *        This function is a workaround to make USB devices work as well.
@@ -140,6 +156,7 @@ static struct usb_driver udl_driver = {
 	.disconnect = udl_usb_disconnect,
 	.suspend = udl_usb_suspend,
 	.resume = udl_usb_resume,
+	.reset_resume = udl_usb_reset_resume,
 	.id_table = id_table,
 };
 module_usb_driver(udl_driver);
diff --git a/drivers/gpu/drm/udl/udl_drv.h b/drivers/gpu/drm/udl/udl_drv.h
index cc16a13316e4..b4cc7cc568c7 100644
--- a/drivers/gpu/drm/udl/udl_drv.h
+++ b/drivers/gpu/drm/udl/udl_drv.h
@@ -34,14 +34,13 @@ struct udl_device;
 struct urb_node {
 	struct list_head entry;
 	struct udl_device *dev;
-	struct delayed_work release_urb_work;
 	struct urb *urb;
 };
 
 struct urb_list {
 	struct list_head list;
 	spinlock_t lock;
-	struct semaphore limit_sem;
+	wait_queue_head_t sleep;
 	int available;
 	int count;
 	size_t size;
@@ -78,6 +77,7 @@ struct drm_connector *udl_connector_init(struct drm_device *dev);
 struct urb *udl_get_urb(struct drm_device *dev);
 
 int udl_submit_urb(struct drm_device *dev, struct urb *urb, size_t len);
+void udl_sync_pending_urbs(struct drm_device *dev);
 void udl_urb_completion(struct urb *urb);
 
 int udl_init(struct udl_device *udl);
@@ -87,6 +87,7 @@ int udl_render_hline(struct drm_device *dev, int log_bpp, struct urb **urb_ptr,
 		     u32 byte_offset, u32 device_byte_offset, u32 byte_width);
 
 int udl_drop_usb(struct drm_device *dev);
+int udl_select_std_channel(struct udl_device *udl);
 
 #define CMD_WRITE_RAW8   "\xAF\x60" /**< 8 bit raw write command. */
 #define CMD_WRITE_RL8    "\xAF\x61" /**< 8 bit run length command. */
diff --git a/drivers/gpu/drm/udl/udl_main.c b/drivers/gpu/drm/udl/udl_main.c
index 853f147036f6..061cb88c08a2 100644
--- a/drivers/gpu/drm/udl/udl_main.c
+++ b/drivers/gpu/drm/udl/udl_main.c
@@ -20,11 +20,10 @@
 #define NR_USB_REQUEST_CHANNEL 0x12
 
 #define MAX_TRANSFER (PAGE_SIZE*16 - BULK_SIZE)
-#define WRITES_IN_FLIGHT (4)
+#define WRITES_IN_FLIGHT (20)
 #define MAX_VENDOR_DESCRIPTOR_SIZE 256
 
-#define GET_URB_TIMEOUT	HZ
-#define FREE_URB_TIMEOUT (HZ*2)
+static struct urb *udl_get_urb_locked(struct udl_device *udl, long timeout);
 
 static int udl_parse_vendor_descriptor(struct udl_device *udl)
 {
@@ -95,7 +94,7 @@ success:
 /*
  * Need to ensure a channel is selected before submitting URBs
  */
-static int udl_select_std_channel(struct udl_device *udl)
+int udl_select_std_channel(struct udl_device *udl)
 {
 	static const u8 set_def_chn[] = {0x57, 0xCD, 0xDC, 0xA7,
 					 0x1C, 0x88, 0x5E, 0x15,
@@ -119,14 +118,6 @@ static int udl_select_std_channel(struct udl_device *udl)
 	return ret < 0 ? ret : 0;
 }
 
-static void udl_release_urb_work(struct work_struct *work)
-{
-	struct urb_node *unode = container_of(work, struct urb_node,
-					      release_urb_work.work);
-
-	up(&unode->dev->urbs.limit_sem);
-}
-
 void udl_urb_completion(struct urb *urb)
 {
 	struct urb_node *unode = urb->context;
@@ -137,6 +128,7 @@ void udl_urb_completion(struct urb *urb)
 	if (urb->status) {
 		if (!(urb->status == -ENOENT ||
 		    urb->status == -ECONNRESET ||
+		    urb->status == -EPROTO ||
 		    urb->status == -ESHUTDOWN)) {
 			DRM_ERROR("%s - nonzero write bulk status received: %d\n",
 				__func__, urb->status);
@@ -150,49 +142,34 @@ void udl_urb_completion(struct urb *urb)
 	udl->urbs.available++;
 	spin_unlock_irqrestore(&udl->urbs.lock, flags);
 
-#if 0
-	/*
-	 * When using fb_defio, we deadlock if up() is called
-	 * while another is waiting. So queue to another process.
-	 */
-	if (fb_defio)
-		schedule_delayed_work(&unode->release_urb_work, 0);
-	else
-#endif
-		up(&udl->urbs.limit_sem);
+	wake_up(&udl->urbs.sleep);
 }
 
 static void udl_free_urb_list(struct drm_device *dev)
 {
 	struct udl_device *udl = to_udl(dev);
-	int count = udl->urbs.count;
-	struct list_head *node;
 	struct urb_node *unode;
 	struct urb *urb;
 
 	DRM_DEBUG("Waiting for completes and freeing all render urbs\n");
 
 	/* keep waiting and freeing, until we've got 'em all */
-	while (count--) {
-		down(&udl->urbs.limit_sem);
-
+	while (udl->urbs.count) {
 		spin_lock_irq(&udl->urbs.lock);
-
-		node = udl->urbs.list.next; /* have reserved one with sem */
-		list_del_init(node);
-
+		urb = udl_get_urb_locked(udl, MAX_SCHEDULE_TIMEOUT);
+		udl->urbs.count--;
 		spin_unlock_irq(&udl->urbs.lock);
-
-		unode = list_entry(node, struct urb_node, entry);
-		urb = unode->urb;
-
+		if (WARN_ON(!urb))
+			break;
+		unode = urb->context;
 		/* Free each separately allocated piece */
 		usb_free_coherent(urb->dev, udl->urbs.size,
 				  urb->transfer_buffer, urb->transfer_dma);
 		usb_free_urb(urb);
-		kfree(node);
+		kfree(unode);
 	}
-	udl->urbs.count = 0;
+
+	wake_up_all(&udl->urbs.sleep);
 }
 
 static int udl_alloc_urb_list(struct drm_device *dev, int count, size_t size)
@@ -205,24 +182,20 @@ static int udl_alloc_urb_list(struct drm_device *dev, int count, size_t size)
 	struct usb_device *udev = udl_to_usb_device(udl);
 
 	spin_lock_init(&udl->urbs.lock);
-
-retry:
-	udl->urbs.size = size;
 	INIT_LIST_HEAD(&udl->urbs.list);
-
-	sema_init(&udl->urbs.limit_sem, 0);
+	init_waitqueue_head(&udl->urbs.sleep);
 	udl->urbs.count = 0;
 	udl->urbs.available = 0;
 
+retry:
+	udl->urbs.size = size;
+
 	while (udl->urbs.count * size < wanted_size) {
 		unode = kzalloc(sizeof(struct urb_node), GFP_KERNEL);
 		if (!unode)
 			break;
 		unode->dev = udl;
 
-		INIT_DELAYED_WORK(&unode->release_urb_work,
-			  udl_release_urb_work);
-
 		urb = usb_alloc_urb(0, GFP_KERNEL);
 		if (!urb) {
 			kfree(unode);
@@ -250,7 +223,6 @@ retry:
 
 		list_add_tail(&unode->entry, &udl->urbs.list);
 
-		up(&udl->urbs.limit_sem);
 		udl->urbs.count++;
 		udl->urbs.available++;
 	}
@@ -260,35 +232,41 @@ retry:
 	return udl->urbs.count;
 }
 
-struct urb *udl_get_urb(struct drm_device *dev)
+static struct urb *udl_get_urb_locked(struct udl_device *udl, long timeout)
 {
-	struct udl_device *udl = to_udl(dev);
-	int ret = 0;
-	struct list_head *entry;
 	struct urb_node *unode;
-	struct urb *urb = NULL;
+
+	assert_spin_locked(&udl->urbs.lock);
 
 	/* Wait for an in-flight buffer to complete and get re-queued */
-	ret = down_timeout(&udl->urbs.limit_sem, GET_URB_TIMEOUT);
-	if (ret) {
-		DRM_INFO("wait for urb interrupted: %x available: %d\n",
-		       ret, udl->urbs.available);
-		goto error;
+	if (!wait_event_lock_irq_timeout(udl->urbs.sleep,
+					 !udl->urbs.count ||
+					 !list_empty(&udl->urbs.list),
+					 udl->urbs.lock, timeout)) {
+		DRM_INFO("wait for urb interrupted: available: %d\n",
+			 udl->urbs.available);
+		return NULL;
 	}
 
-	spin_lock_irq(&udl->urbs.lock);
+	if (!udl->urbs.count)
+		return NULL;
 
-	BUG_ON(list_empty(&udl->urbs.list)); /* reserved one with limit_sem */
-	entry = udl->urbs.list.next;
-	list_del_init(entry);
+	unode = list_first_entry(&udl->urbs.list, struct urb_node, entry);
+	list_del_init(&unode->entry);
 	udl->urbs.available--;
 
-	spin_unlock_irq(&udl->urbs.lock);
+	return unode ? unode->urb : NULL;
+}
 
-	unode = list_entry(entry, struct urb_node, entry);
-	urb = unode->urb;
+#define GET_URB_TIMEOUT	HZ
+struct urb *udl_get_urb(struct drm_device *dev)
+{
+	struct udl_device *udl = to_udl(dev);
+	struct urb *urb;
 
-error:
+	spin_lock_irq(&udl->urbs.lock);
+	urb = udl_get_urb_locked(udl, GET_URB_TIMEOUT);
+	spin_unlock_irq(&udl->urbs.lock);
 	return urb;
 }
 
@@ -297,10 +275,13 @@ int udl_submit_urb(struct drm_device *dev, struct urb *urb, size_t len)
 	struct udl_device *udl = to_udl(dev);
 	int ret;
 
-	BUG_ON(len > udl->urbs.size);
-
+	if (WARN_ON(len > udl->urbs.size)) {
+		ret = -EINVAL;
+		goto error;
+	}
 	urb->transfer_buffer_length = len; /* set to actual payload len */
 	ret = usb_submit_urb(urb, GFP_ATOMIC);
+ error:
 	if (ret) {
 		udl_urb_completion(urb); /* because no one else will */
 		DRM_ERROR("usb_submit_urb error %x\n", ret);
@@ -308,6 +289,21 @@ int udl_submit_urb(struct drm_device *dev, struct urb *urb, size_t len)
 	return ret;
 }
 
+/* wait until all pending URBs have been processed */
+void udl_sync_pending_urbs(struct drm_device *dev)
+{
+	struct udl_device *udl = to_udl(dev);
+
+	spin_lock_irq(&udl->urbs.lock);
+	/* 2 seconds as a sane timeout */
+	if (!wait_event_lock_irq_timeout(udl->urbs.sleep,
+					 udl->urbs.available == udl->urbs.count,
+					 udl->urbs.lock,
+					 msecs_to_jiffies(2000)))
+		drm_err(dev, "Timeout for syncing pending URBs\n");
+	spin_unlock_irq(&udl->urbs.lock);
+}
+
 int udl_init(struct udl_device *udl)
 {
 	struct drm_device *dev = &udl->drm;
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index e67c40a48fb4..ec6876f449f3 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -242,38 +242,15 @@ static long udl_log_cpp(unsigned int cpp)
 	return __ffs(cpp);
 }
 
-static int udl_aligned_damage_clip(struct drm_rect *clip, int x, int y,
-				   int width, int height)
-{
-	int x1, x2;
-
-	if (WARN_ON_ONCE(x < 0) ||
-	    WARN_ON_ONCE(y < 0) ||
-	    WARN_ON_ONCE(width < 0) ||
-	    WARN_ON_ONCE(height < 0))
-		return -EINVAL;
-
-	x1 = ALIGN_DOWN(x, sizeof(unsigned long));
-	x2 = ALIGN(width + (x - x1), sizeof(unsigned long)) + x1;
-
-	clip->x1 = x1;
-	clip->y1 = y;
-	clip->x2 = x2;
-	clip->y2 = y + height;
-
-	return 0;
-}
-
 static int udl_handle_damage(struct drm_framebuffer *fb,
 			     const struct iosys_map *map,
-			     int x, int y, int width, int height)
+			     const struct drm_rect *clip)
 {
 	struct drm_device *dev = fb->dev;
 	void *vaddr = map->vaddr; /* TODO: Use mapping abstraction properly */
 	int i, ret;
 	char *cmd;
 	struct urb *urb;
-	struct drm_rect clip;
 	int log_bpp;
 
 	ret = udl_log_cpp(fb->format->cpp[0]);
@@ -281,12 +258,6 @@ static int udl_handle_damage(struct drm_framebuffer *fb,
 		return ret;
 	log_bpp = ret;
 
-	ret = udl_aligned_damage_clip(&clip, x, y, width, height);
-	if (ret)
-		return ret;
-	else if ((clip.x2 > fb->width) || (clip.y2 > fb->height))
-		return -EINVAL;
-
 	ret = drm_gem_fb_begin_cpu_access(fb, DMA_FROM_DEVICE);
 	if (ret)
 		return ret;
@@ -298,11 +269,11 @@ static int udl_handle_damage(struct drm_framebuffer *fb,
 	}
 	cmd = urb->transfer_buffer;
 
-	for (i = clip.y1; i < clip.y2; i++) {
+	for (i = clip->y1; i < clip->y2; i++) {
 		const int line_offset = fb->pitches[0] * i;
-		const int byte_offset = line_offset + (clip.x1 << log_bpp);
-		const int dev_byte_offset = (fb->width * i + clip.x1) << log_bpp;
-		const int byte_width = (clip.x2 - clip.x1) << log_bpp;
+		const int byte_offset = line_offset + (clip->x1 << log_bpp);
+		const int dev_byte_offset = (fb->width * i + clip->x1) << log_bpp;
+		const int byte_width = drm_rect_width(clip) << log_bpp;
 		ret = udl_render_hline(dev, log_bpp, &urb, (char *)vaddr,
 				       &cmd, byte_offset, dev_byte_offset,
 				       byte_width);
@@ -355,6 +326,7 @@ udl_simple_display_pipe_enable(struct drm_simple_display_pipe *pipe,
 	struct udl_device *udl = to_udl(dev);
 	struct drm_display_mode *mode = &crtc_state->mode;
 	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
+	struct drm_rect clip = DRM_RECT_INIT(0, 0, fb->width, fb->height);
 	char *buf;
 	char *wrptr;
 	int color_depth = UDL_COLOR_DEPTH_16BPP;
@@ -380,10 +352,7 @@ udl_simple_display_pipe_enable(struct drm_simple_display_pipe *pipe,
 
 	udl->mode_buf_len = wrptr - buf;
 
-	udl_handle_damage(fb, &shadow_plane_state->data[0], 0, 0, fb->width, fb->height);
-
-	if (!crtc_state->mode_changed)
-		return;
+	udl_handle_damage(fb, &shadow_plane_state->data[0], &clip);
 
 	/* enable display */
 	udl_crtc_write_mode_to_hw(crtc);
@@ -423,8 +392,7 @@ udl_simple_display_pipe_update(struct drm_simple_display_pipe *pipe,
 		return;
 
 	if (drm_atomic_helper_damage_merged(old_plane_state, state, &rect))
-		udl_handle_damage(fb, &shadow_plane_state->data[0], rect.x1, rect.y1,
-				  rect.x2 - rect.x1, rect.y2 - rect.y1);
+		udl_handle_damage(fb, &shadow_plane_state->data[0], &rect);
 }
 
 static const struct drm_simple_display_pipe_funcs udl_simple_display_pipe_funcs = {
@@ -479,6 +447,7 @@ int udl_modeset_init(struct drm_device *dev)
 					   format_count, NULL, connector);
 	if (ret)
 		return ret;
+	drm_plane_enable_fb_damage_clips(&udl->display_pipe.plane);
 
 	drm_mode_config_reset(dev);
 
diff --git a/drivers/gpu/drm/udl/udl_transfer.c b/drivers/gpu/drm/udl/udl_transfer.c
index 971927669d6b..b57844632dbd 100644
--- a/drivers/gpu/drm/udl/udl_transfer.c
+++ b/drivers/gpu/drm/udl/udl_transfer.c
@@ -25,46 +25,6 @@
 #define MIN_RAW_PIX_BYTES	2
 #define MIN_RAW_CMD_BYTES	(RAW_HEADER_BYTES + MIN_RAW_PIX_BYTES)
 
-/*
- * Trims identical data from front and back of line
- * Sets new front buffer address and width
- * And returns byte count of identical pixels
- * Assumes CPU natural alignment (unsigned long)
- * for back and front buffer ptrs and width
- */
-#if 0
-static int udl_trim_hline(const u8 *bback, const u8 **bfront, int *width_bytes)
-{
-	int j, k;
-	const unsigned long *back = (const unsigned long *) bback;
-	const unsigned long *front = (const unsigned long *) *bfront;
-	const int width = *width_bytes / sizeof(unsigned long);
-	int identical = width;
-	int start = width;
-	int end = width;
-
-	for (j = 0; j < width; j++) {
-		if (back[j] != front[j]) {
-			start = j;
-			break;
-		}
-	}
-
-	for (k = width - 1; k > j; k--) {
-		if (back[k] != front[k]) {
-			end = k+1;
-			break;
-		}
-	}
-
-	identical = start + (width - end);
-	*bfront = (u8 *) &front[start];
-	*width_bytes = (end - start) * sizeof(unsigned long);
-
-	return identical * sizeof(unsigned long);
-}
-#endif
-
 static inline u16 pixel32_to_be16(const uint32_t pixel)
 {
 	return (((pixel >> 3) & 0x001f) |
@@ -220,7 +180,11 @@ int udl_render_hline(struct drm_device *dev, int log_bpp, struct urb **urb_ptr,
 	u8 *cmd = *urb_buf_ptr;
 	u8 *cmd_end = (u8 *) urb->transfer_buffer + urb->transfer_buffer_length;
 
-	BUG_ON(!(log_bpp == 1 || log_bpp == 2));
+	if (WARN_ON(!(log_bpp == 1 || log_bpp == 2))) {
+		/* need to finish URB at error from this function */
+		udl_urb_completion(urb);
+		return -EINVAL;
+	}
 
 	line_start = (u8 *) (front + byte_offset);
 	next_pixel = line_start;
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index 8c7f910daa28..e8c975b81585 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -22,7 +22,6 @@
 #include <linux/reset.h>
 
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_managed.h>
 #include <uapi/drm/v3d_drm.h>
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index 725a252e837b..b8980440d137 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -313,7 +313,7 @@ v3d_lookup_bos(struct drm_device *dev,
 	}
 
 	job->bo = kvmalloc_array(job->bo_count,
-				 sizeof(struct drm_gem_cma_object *),
+				 sizeof(struct drm_gem_dma_object *),
 				 GFP_KERNEL | __GFP_ZERO);
 	if (!job->bo) {
 		DRM_DEBUG("Failed to allocate validated BO pointers\n");
@@ -1092,7 +1092,7 @@ v3d_gem_init(struct drm_device *dev)
 	if (!v3d->pt) {
 		drm_mm_takedown(&v3d->mm);
 		dev_err(v3d->drm.dev,
-			"Failed to allocate page tables. Please ensure you have CMA enabled.\n");
+			"Failed to allocate page tables. Please ensure you have DMA enabled.\n");
 		return -ENOMEM;
 	}
 
diff --git a/drivers/gpu/drm/v3d/v3d_perfmon.c b/drivers/gpu/drm/v3d/v3d_perfmon.c
index f6a88abccc7d..48aaaa972c49 100644
--- a/drivers/gpu/drm/v3d/v3d_perfmon.c
+++ b/drivers/gpu/drm/v3d/v3d_perfmon.c
@@ -95,7 +95,7 @@ struct v3d_perfmon *v3d_perfmon_find(struct v3d_file_priv *v3d_priv, int id)
 void v3d_perfmon_open_file(struct v3d_file_priv *v3d_priv)
 {
 	mutex_init(&v3d_priv->perfmon.lock);
-	idr_init(&v3d_priv->perfmon.idr);
+	idr_init_base(&v3d_priv->perfmon.idr, 1);
 }
 
 static int v3d_perfmon_idr_del(int id, void *elem, void *data)
diff --git a/drivers/gpu/drm/vboxvideo/vbox_mode.c b/drivers/gpu/drm/vboxvideo/vbox_mode.c
index fa0d73ce07bc..341edd982cb3 100644
--- a/drivers/gpu/drm/vboxvideo/vbox_mode.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_mode.c
@@ -269,8 +269,8 @@ static int vbox_primary_atomic_check(struct drm_plane *plane,
 	}
 
 	return drm_atomic_helper_check_plane_state(new_state, crtc_state,
-						   DRM_PLANE_HELPER_NO_SCALING,
-						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
 						   false, true);
 }
 
@@ -351,8 +351,8 @@ static int vbox_cursor_atomic_check(struct drm_plane *plane,
 	}
 
 	ret = drm_atomic_helper_check_plane_state(new_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  true, true);
 	if (ret)
 		return ret;
@@ -477,7 +477,7 @@ static const struct drm_plane_helper_funcs vbox_cursor_helper_funcs = {
 static const struct drm_plane_funcs vbox_cursor_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.destroy	= drm_primary_helper_destroy,
+	.destroy	= drm_plane_helper_destroy,
 	DRM_GEM_SHADOW_PLANE_FUNCS,
 };
 
@@ -496,7 +496,7 @@ static const struct drm_plane_helper_funcs vbox_primary_helper_funcs = {
 static const struct drm_plane_funcs vbox_primary_plane_funcs = {
 	.update_plane	= drm_atomic_helper_update_plane,
 	.disable_plane	= drm_atomic_helper_disable_plane,
-	.destroy	= drm_primary_helper_destroy,
+	.destroy	= drm_plane_helper_destroy,
 	.reset		= drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
diff --git a/drivers/gpu/drm/vboxvideo/vboxvideo.h b/drivers/gpu/drm/vboxvideo/vboxvideo.h
index a5de40fe1a76..f60d82504da0 100644
--- a/drivers/gpu/drm/vboxvideo/vboxvideo.h
+++ b/drivers/gpu/drm/vboxvideo/vboxvideo.h
@@ -43,7 +43,7 @@
  * VBE_DISPI_INDEX_VBOX_VIDEO is used to read the configuration information
  * from the host and issue commands to the host.
  *
- * The guest writes the VBE_DISPI_INDEX_VBOX_VIDEO index register, the the
+ * The guest writes the VBE_DISPI_INDEX_VBOX_VIDEO index register, the
  * following operations with the VBE data register can be performed:
  *
  * Operation            Result
diff --git a/drivers/gpu/drm/vc4/Kconfig b/drivers/gpu/drm/vc4/Kconfig
index b0f3117102ca..246305d17a52 100644
--- a/drivers/gpu/drm/vc4/Kconfig
+++ b/drivers/gpu/drm/vc4/Kconfig
@@ -12,7 +12,7 @@ config DRM_VC4
 	select DRM_DISPLAY_HDMI_HELPER
 	select DRM_DISPLAY_HELPER
 	select DRM_KMS_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_PANEL_BRIDGE
 	select SND_PCM
 	select SND_PCM_ELD
diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c
index 0846d56f74f2..231add8b8e12 100644
--- a/drivers/gpu/drm/vc4/vc4_bo.c
+++ b/drivers/gpu/drm/vc4/vc4_bo.c
@@ -8,10 +8,10 @@
  *
  * The VC4 GPU architecture (both scanout and rendering) has direct
  * access to system memory with no MMU in between.  To support it, we
- * use the GEM CMA helper functions to allocate contiguous ranges of
+ * use the GEM DMA helper functions to allocate contiguous ranges of
  * physical memory for our BOs.
  *
- * Since the CMA allocator is very slow, we keep a cache of recently
+ * Since the DMA allocator is very slow, we keep a cache of recently
  * freed BOs around so that the kernel's allocation of objects for 3D
  * rendering can return quickly.
  */
@@ -179,7 +179,7 @@ static void vc4_bo_destroy(struct vc4_bo *bo)
 		bo->validated_shader = NULL;
 	}
 
-	drm_gem_cma_free(&bo->base);
+	drm_gem_dma_free(&bo->base);
 }
 
 static void vc4_bo_remove_from_cache(struct vc4_bo *bo)
@@ -303,7 +303,7 @@ static void vc4_bo_purge(struct drm_gem_object *obj)
 
 	drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping);
 
-	dma_free_wc(dev->dev, obj->size, bo->base.vaddr, bo->base.paddr);
+	dma_free_wc(dev->dev, obj->size, bo->base.vaddr, bo->base.dma_addr);
 	bo->base.vaddr = NULL;
 	bo->madv = __VC4_MADV_PURGED;
 }
@@ -387,13 +387,14 @@ out:
  * @dev: DRM device
  * @size: Size in bytes of the memory the object will reference
  *
- * This lets the CMA helpers allocate object structs for us, and keep
+ * This lets the DMA helpers allocate object structs for us, and keep
  * our BO stats correct.
  */
 struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct vc4_bo *bo;
+	int ret;
 
 	if (WARN_ON_ONCE(vc4->is_vc5))
 		return ERR_PTR(-ENODEV);
@@ -404,7 +405,11 @@ struct drm_gem_object *vc4_create_object(struct drm_device *dev, size_t size)
 
 	bo->madv = VC4_MADV_WILLNEED;
 	refcount_set(&bo->usecnt, 0);
-	mutex_init(&bo->madv_lock);
+
+	ret = drmm_mutex_init(dev, &bo->madv_lock);
+	if (ret)
+		return ERR_PTR(ret);
+
 	mutex_lock(&vc4->bo_lock);
 	bo->label = VC4_BO_TYPE_KERNEL;
 	vc4->bo_labels[VC4_BO_TYPE_KERNEL].num_allocated++;
@@ -421,7 +426,7 @@ struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size,
 {
 	size_t size = roundup(unaligned_size, PAGE_SIZE);
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	struct drm_gem_cma_object *cma_obj;
+	struct drm_gem_dma_object *dma_obj;
 	struct vc4_bo *bo;
 
 	if (WARN_ON_ONCE(vc4->is_vc5))
@@ -438,39 +443,39 @@ struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size,
 		return bo;
 	}
 
-	cma_obj = drm_gem_cma_create(dev, size);
-	if (IS_ERR(cma_obj)) {
+	dma_obj = drm_gem_dma_create(dev, size);
+	if (IS_ERR(dma_obj)) {
 		/*
-		 * If we've run out of CMA memory, kill the cache of
-		 * CMA allocations we've got laying around and try again.
+		 * If we've run out of DMA memory, kill the cache of
+		 * DMA allocations we've got laying around and try again.
 		 */
 		vc4_bo_cache_purge(dev);
-		cma_obj = drm_gem_cma_create(dev, size);
+		dma_obj = drm_gem_dma_create(dev, size);
 	}
 
-	if (IS_ERR(cma_obj)) {
+	if (IS_ERR(dma_obj)) {
 		/*
-		 * Still not enough CMA memory, purge the userspace BO
+		 * Still not enough DMA memory, purge the userspace BO
 		 * cache and retry.
 		 * This is sub-optimal since we purge the whole userspace
 		 * BO cache which forces user that want to re-use the BO to
 		 * restore its initial content.
 		 * Ideally, we should purge entries one by one and retry
-		 * after each to see if CMA allocation succeeds. Or even
+		 * after each to see if DMA allocation succeeds. Or even
 		 * better, try to find an entry with at least the same
 		 * size.
 		 */
 		vc4_bo_userspace_cache_purge(dev);
-		cma_obj = drm_gem_cma_create(dev, size);
+		dma_obj = drm_gem_dma_create(dev, size);
 	}
 
-	if (IS_ERR(cma_obj)) {
+	if (IS_ERR(dma_obj)) {
 		struct drm_printer p = drm_info_printer(vc4->base.dev);
-		DRM_ERROR("Failed to allocate from CMA:\n");
+		DRM_ERROR("Failed to allocate from GEM DMA helper:\n");
 		vc4_bo_stats_print(&p, vc4);
 		return ERR_PTR(-ENOMEM);
 	}
-	bo = to_vc4_bo(&cma_obj->base);
+	bo = to_vc4_bo(&dma_obj->base);
 
 	/* By default, BOs do not support the MADV ioctl. This will be enabled
 	 * only on BOs that are exposed to userspace (V3D, V3D_SHADER and DUMB
@@ -479,7 +484,7 @@ struct vc4_bo *vc4_bo_create(struct drm_device *dev, size_t unaligned_size,
 	bo->madv = __VC4_MADV_NOTSUPP;
 
 	mutex_lock(&vc4->bo_lock);
-	vc4_bo_set_label(&cma_obj->base, type);
+	vc4_bo_set_label(&dma_obj->base, type);
 	mutex_unlock(&vc4->bo_lock);
 
 	return bo;
@@ -564,7 +569,7 @@ static void vc4_free_object(struct drm_gem_object *gem_bo)
 		goto out;
 	}
 
-	/* If this object was partially constructed but CMA allocation
+	/* If this object was partially constructed but DMA allocation
 	 * had failed, just free it. Can also happen when the BO has been
 	 * purged.
 	 */
@@ -742,7 +747,7 @@ static int vc4_gem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct
 		return -EINVAL;
 	}
 
-	return drm_gem_cma_mmap(&bo->base, vma);
+	return drm_gem_dma_mmap(&bo->base, vma);
 }
 
 static const struct vm_operations_struct vc4_vm_ops = {
@@ -754,8 +759,8 @@ static const struct vm_operations_struct vc4_vm_ops = {
 static const struct drm_gem_object_funcs vc4_gem_object_funcs = {
 	.free = vc4_free_object,
 	.export = vc4_prime_export,
-	.get_sg_table = drm_gem_cma_object_get_sg_table,
-	.vmap = drm_gem_cma_object_vmap,
+	.get_sg_table = drm_gem_dma_object_get_sg_table,
+	.vmap = drm_gem_dma_object_vmap,
 	.mmap = vc4_gem_object_mmap,
 	.vm_ops = &vc4_vm_ops,
 };
@@ -984,10 +989,28 @@ int vc4_get_tiling_ioctl(struct drm_device *dev, void *data,
 	return 0;
 }
 
+int vc4_bo_debugfs_init(struct drm_minor *minor)
+{
+	struct drm_device *drm = minor->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(drm);
+	int ret;
+
+	if (!vc4->v3d)
+		return -ENODEV;
+
+	ret = vc4_debugfs_add_file(minor, "bo_stats",
+				   vc4_bo_stats_debugfs, NULL);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
 static void vc4_bo_cache_destroy(struct drm_device *dev, void *unused);
 int vc4_bo_cache_init(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	int ret;
 	int i;
 
 	if (WARN_ON_ONCE(vc4->is_vc5))
@@ -1007,9 +1030,11 @@ int vc4_bo_cache_init(struct drm_device *dev)
 	for (i = 0; i < VC4_BO_TYPE_COUNT; i++)
 		vc4->bo_labels[i].name = bo_type_names[i];
 
-	mutex_init(&vc4->bo_lock);
-
-	vc4_debugfs_add_file(dev, "bo_stats", vc4_bo_stats_debugfs, NULL);
+	ret = drmm_mutex_init(dev, &vc4->bo_lock);
+	if (ret) {
+		kfree(vc4->bo_labels);
+		return ret;
+	}
 
 	INIT_LIST_HEAD(&vc4->bo_cache.time_list);
 
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index 029be98660b3..0108613e79d5 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -37,8 +37,9 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_atomic_uapi.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_framebuffer.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
@@ -206,11 +207,6 @@ static bool vc4_crtc_get_scanout_position(struct drm_crtc *crtc,
 	return ret;
 }
 
-void vc4_crtc_destroy(struct drm_crtc *crtc)
-{
-	drm_crtc_cleanup(crtc);
-}
-
 static u32 vc4_get_fifo_full_level(struct vc4_crtc *vc4_crtc, u32 format)
 {
 	const struct vc4_crtc_data *crtc_data = vc4_crtc_to_vc4_crtc_data(vc4_crtc);
@@ -300,10 +296,17 @@ struct drm_encoder *vc4_get_crtc_encoder(struct drm_crtc *crtc,
 static void vc4_crtc_pixelvalve_reset(struct drm_crtc *crtc)
 {
 	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	int idx;
+
+	if (!drm_dev_enter(dev, &idx))
+		return;
 
 	/* The PV needs to be disabled before it can be flushed */
 	CRTC_WRITE(PV_CONTROL, CRTC_READ(PV_CONTROL) & ~PV_CONTROL_EN);
 	CRTC_WRITE(PV_CONTROL, CRTC_READ(PV_CONTROL) | PV_CONTROL_FIFO_CLR);
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_crtc_config_pv(struct drm_crtc *crtc, struct drm_encoder *encoder,
@@ -326,6 +329,10 @@ static void vc4_crtc_config_pv(struct drm_crtc *crtc, struct drm_encoder *encode
 	u32 format = is_dsi1 ? PV_CONTROL_FORMAT_DSIV_24 : PV_CONTROL_FORMAT_24;
 	u8 ppc = pv_data->pixels_per_clock;
 	bool debug_dump_regs = false;
+	int idx;
+
+	if (!drm_dev_enter(dev, &idx))
+		return;
 
 	if (debug_dump_regs) {
 		struct drm_printer p = drm_info_printer(&vc4_crtc->pdev->dev);
@@ -415,6 +422,8 @@ static void vc4_crtc_config_pv(struct drm_crtc *crtc, struct drm_encoder *encode
 			 drm_crtc_index(crtc));
 		drm_print_regset32(&p, &vc4_crtc->regset);
 	}
+
+	drm_dev_exit(idx);
 }
 
 static void require_hvs_enabled(struct drm_device *dev)
@@ -435,7 +444,10 @@ static int vc4_crtc_disable(struct drm_crtc *crtc,
 	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	int ret;
+	int idx, ret;
+
+	if (!drm_dev_enter(dev, &idx))
+		return -ENODEV;
 
 	CRTC_WRITE(PV_V_CONTROL,
 		   CRTC_READ(PV_V_CONTROL) & ~PV_VCONTROL_VIDEN);
@@ -469,6 +481,8 @@ static int vc4_crtc_disable(struct drm_crtc *crtc,
 	if (vc4_encoder && vc4_encoder->post_crtc_powerdown)
 		vc4_encoder->post_crtc_powerdown(encoder, state);
 
+	drm_dev_exit(idx);
+
 	return 0;
 }
 
@@ -544,6 +558,20 @@ int vc4_crtc_disable_at_boot(struct drm_crtc *crtc)
 	return 0;
 }
 
+void vc4_crtc_send_vblank(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	unsigned long flags;
+
+	if (!crtc->state || !crtc->state->event)
+		return;
+
+	spin_lock_irqsave(&dev->event_lock, flags);
+	drm_crtc_send_vblank_event(crtc, crtc->state->event);
+	crtc->state->event = NULL;
+	spin_unlock_irqrestore(&dev->event_lock, flags);
+}
+
 static void vc4_crtc_atomic_disable(struct drm_crtc *crtc,
 				    struct drm_atomic_state *state)
 {
@@ -567,14 +595,7 @@ static void vc4_crtc_atomic_disable(struct drm_crtc *crtc,
 	 * Make sure we issue a vblank event after disabling the CRTC if
 	 * someone was waiting it.
 	 */
-	if (crtc->state->event) {
-		unsigned long flags;
-
-		spin_lock_irqsave(&dev->event_lock, flags);
-		drm_crtc_send_vblank_event(crtc, crtc->state->event);
-		crtc->state->event = NULL;
-		spin_unlock_irqrestore(&dev->event_lock, flags);
-	}
+	vc4_crtc_send_vblank(crtc);
 }
 
 static void vc4_crtc_atomic_enable(struct drm_crtc *crtc,
@@ -586,10 +607,14 @@ static void vc4_crtc_atomic_enable(struct drm_crtc *crtc,
 	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 	struct drm_encoder *encoder = vc4_get_crtc_encoder(crtc, new_state);
 	struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder);
+	int idx;
 
 	drm_dbg(dev, "Enabling CRTC %s (%u) connected to Encoder %s (%u)",
 		crtc->name, crtc->base.id, encoder->name, encoder->base.id);
 
+	if (!drm_dev_enter(dev, &idx))
+		return;
+
 	require_hvs_enabled(dev);
 
 	/* Enable vblank irq handling before crtc is started otherwise
@@ -617,6 +642,8 @@ static void vc4_crtc_atomic_enable(struct drm_crtc *crtc,
 
 	if (vc4_encoder->post_crtc_enable)
 		vc4_encoder->post_crtc_enable(encoder, state);
+
+	drm_dev_exit(idx);
 }
 
 static enum drm_mode_status vc4_crtc_mode_valid(struct drm_crtc *crtc,
@@ -709,17 +736,31 @@ static int vc4_crtc_atomic_check(struct drm_crtc *crtc,
 static int vc4_enable_vblank(struct drm_crtc *crtc)
 {
 	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	int idx;
+
+	if (!drm_dev_enter(dev, &idx))
+		return -ENODEV;
 
 	CRTC_WRITE(PV_INTEN, PV_INT_VFP_START);
 
+	drm_dev_exit(idx);
+
 	return 0;
 }
 
 static void vc4_disable_vblank(struct drm_crtc *crtc)
 {
 	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+	struct drm_device *dev = crtc->dev;
+	int idx;
+
+	if (!drm_dev_enter(dev, &idx))
+		return;
 
 	CRTC_WRITE(PV_INTEN, 0);
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_crtc_handle_page_flip(struct vc4_crtc *vc4_crtc)
@@ -821,9 +862,9 @@ static void vc4_async_page_flip_seqno_complete(struct vc4_seqno_cb *cb)
 	struct vc4_bo *bo = NULL;
 
 	if (flip_state->old_fb) {
-		struct drm_gem_cma_object *cma_bo =
-			drm_fb_cma_get_gem_obj(flip_state->old_fb, 0);
-		bo = to_vc4_bo(&cma_bo->base);
+		struct drm_gem_dma_object *dma_bo =
+			drm_fb_dma_get_gem_obj(flip_state->old_fb, 0);
+		bo = to_vc4_bo(&dma_bo->base);
 	}
 
 	vc4_async_page_flip_complete(flip_state);
@@ -855,19 +896,19 @@ static int vc4_async_set_fence_cb(struct drm_device *dev,
 				  struct vc4_async_flip_state *flip_state)
 {
 	struct drm_framebuffer *fb = flip_state->fb;
-	struct drm_gem_cma_object *cma_bo = drm_fb_cma_get_gem_obj(fb, 0);
+	struct drm_gem_dma_object *dma_bo = drm_fb_dma_get_gem_obj(fb, 0);
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct dma_fence *fence;
 	int ret;
 
 	if (!vc4->is_vc5) {
-		struct vc4_bo *bo = to_vc4_bo(&cma_bo->base);
+		struct vc4_bo *bo = to_vc4_bo(&dma_bo->base);
 
 		return vc4_queue_seqno_cb(dev, &flip_state->cb.seqno, bo->seqno,
 					  vc4_async_page_flip_seqno_complete);
 	}
 
-	ret = dma_resv_get_singleton(cma_bo->base.resv, DMA_RESV_USAGE_READ, &fence);
+	ret = dma_resv_get_singleton(dma_bo->base.resv, DMA_RESV_USAGE_READ, &fence);
 	if (ret)
 		return ret;
 
@@ -943,8 +984,8 @@ static int vc4_async_page_flip(struct drm_crtc *crtc,
 {
 	struct drm_device *dev = crtc->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	struct drm_gem_cma_object *cma_bo = drm_fb_cma_get_gem_obj(fb, 0);
-	struct vc4_bo *bo = to_vc4_bo(&cma_bo->base);
+	struct drm_gem_dma_object *dma_bo = drm_fb_dma_get_gem_obj(fb, 0);
+	struct vc4_bo *bo = to_vc4_bo(&dma_bo->base);
 	int ret;
 
 	if (WARN_ON_ONCE(vc4->is_vc5))
@@ -1050,9 +1091,23 @@ void vc4_crtc_reset(struct drm_crtc *crtc)
 	__drm_atomic_helper_crtc_reset(crtc, &vc4_crtc_state->base);
 }
 
+int vc4_crtc_late_register(struct drm_crtc *crtc)
+{
+	struct drm_device *drm = crtc->dev;
+	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
+	const struct vc4_crtc_data *crtc_data = vc4_crtc_to_vc4_crtc_data(vc4_crtc);
+	int ret;
+
+	ret = vc4_debugfs_add_regset32(drm->primary, crtc_data->debugfs_name,
+				       &vc4_crtc->regset);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
 static const struct drm_crtc_funcs vc4_crtc_funcs = {
 	.set_config = drm_atomic_helper_set_config,
-	.destroy = vc4_crtc_destroy,
 	.page_flip = vc4_page_flip,
 	.set_property = NULL,
 	.cursor_set = NULL, /* handled by drm_mode_cursor_universal */
@@ -1063,6 +1118,7 @@ static const struct drm_crtc_funcs vc4_crtc_funcs = {
 	.enable_vblank = vc4_enable_vblank,
 	.disable_vblank = vc4_disable_vblank,
 	.get_vblank_timestamp = drm_crtc_vblank_helper_get_vblank_timestamp,
+	.late_register = vc4_crtc_late_register,
 };
 
 static const struct drm_crtc_helper_funcs vc4_crtc_helper_funcs = {
@@ -1077,10 +1133,10 @@ static const struct drm_crtc_helper_funcs vc4_crtc_helper_funcs = {
 
 static const struct vc4_pv_data bcm2835_pv0_data = {
 	.base = {
+		.debugfs_name = "crtc0_regs",
 		.hvs_available_channels = BIT(0),
 		.hvs_output = 0,
 	},
-	.debugfs_name = "crtc0_regs",
 	.fifo_depth = 64,
 	.pixels_per_clock = 1,
 	.encoder_types = {
@@ -1091,10 +1147,10 @@ static const struct vc4_pv_data bcm2835_pv0_data = {
 
 static const struct vc4_pv_data bcm2835_pv1_data = {
 	.base = {
+		.debugfs_name = "crtc1_regs",
 		.hvs_available_channels = BIT(2),
 		.hvs_output = 2,
 	},
-	.debugfs_name = "crtc1_regs",
 	.fifo_depth = 64,
 	.pixels_per_clock = 1,
 	.encoder_types = {
@@ -1105,10 +1161,10 @@ static const struct vc4_pv_data bcm2835_pv1_data = {
 
 static const struct vc4_pv_data bcm2835_pv2_data = {
 	.base = {
+		.debugfs_name = "crtc2_regs",
 		.hvs_available_channels = BIT(1),
 		.hvs_output = 1,
 	},
-	.debugfs_name = "crtc2_regs",
 	.fifo_depth = 64,
 	.pixels_per_clock = 1,
 	.encoder_types = {
@@ -1119,10 +1175,10 @@ static const struct vc4_pv_data bcm2835_pv2_data = {
 
 static const struct vc4_pv_data bcm2711_pv0_data = {
 	.base = {
+		.debugfs_name = "crtc0_regs",
 		.hvs_available_channels = BIT(0),
 		.hvs_output = 0,
 	},
-	.debugfs_name = "crtc0_regs",
 	.fifo_depth = 64,
 	.pixels_per_clock = 1,
 	.encoder_types = {
@@ -1133,10 +1189,10 @@ static const struct vc4_pv_data bcm2711_pv0_data = {
 
 static const struct vc4_pv_data bcm2711_pv1_data = {
 	.base = {
+		.debugfs_name = "crtc1_regs",
 		.hvs_available_channels = BIT(0) | BIT(1) | BIT(2),
 		.hvs_output = 3,
 	},
-	.debugfs_name = "crtc1_regs",
 	.fifo_depth = 64,
 	.pixels_per_clock = 1,
 	.encoder_types = {
@@ -1147,10 +1203,10 @@ static const struct vc4_pv_data bcm2711_pv1_data = {
 
 static const struct vc4_pv_data bcm2711_pv2_data = {
 	.base = {
+		.debugfs_name = "crtc2_regs",
 		.hvs_available_channels = BIT(0) | BIT(1) | BIT(2),
 		.hvs_output = 4,
 	},
-	.debugfs_name = "crtc2_regs",
 	.fifo_depth = 256,
 	.pixels_per_clock = 2,
 	.encoder_types = {
@@ -1160,10 +1216,10 @@ static const struct vc4_pv_data bcm2711_pv2_data = {
 
 static const struct vc4_pv_data bcm2711_pv3_data = {
 	.base = {
+		.debugfs_name = "crtc3_regs",
 		.hvs_available_channels = BIT(1),
 		.hvs_output = 1,
 	},
-	.debugfs_name = "crtc3_regs",
 	.fifo_depth = 64,
 	.pixels_per_clock = 1,
 	.encoder_types = {
@@ -1173,10 +1229,10 @@ static const struct vc4_pv_data bcm2711_pv3_data = {
 
 static const struct vc4_pv_data bcm2711_pv4_data = {
 	.base = {
+		.debugfs_name = "crtc4_regs",
 		.hvs_available_channels = BIT(0) | BIT(1) | BIT(2),
 		.hvs_output = 5,
 	},
-	.debugfs_name = "crtc4_regs",
 	.fifo_depth = 64,
 	.pixels_per_clock = 2,
 	.encoder_types = {
@@ -1230,6 +1286,7 @@ int vc4_crtc_init(struct drm_device *drm, struct vc4_crtc *vc4_crtc,
 	struct drm_crtc *crtc = &vc4_crtc->base;
 	struct drm_plane *primary_plane;
 	unsigned int i;
+	int ret;
 
 	/* For now, we create just the primary and the legacy cursor
 	 * planes.  We should be able to stack more planes on easily,
@@ -1237,15 +1294,18 @@ int vc4_crtc_init(struct drm_device *drm, struct vc4_crtc *vc4_crtc,
 	 * requirement of the plane configuration, and reject ones
 	 * that will take too much.
 	 */
-	primary_plane = vc4_plane_init(drm, DRM_PLANE_TYPE_PRIMARY);
+	primary_plane = vc4_plane_init(drm, DRM_PLANE_TYPE_PRIMARY, 0);
 	if (IS_ERR(primary_plane)) {
 		dev_err(drm->dev, "failed to construct primary plane\n");
 		return PTR_ERR(primary_plane);
 	}
 
 	spin_lock_init(&vc4_crtc->irq_lock);
-	drm_crtc_init_with_planes(drm, crtc, primary_plane, NULL,
-				  crtc_funcs, NULL);
+	ret = drmm_crtc_init_with_planes(drm, crtc, primary_plane, NULL,
+					 crtc_funcs, NULL);
+	if (ret)
+		return ret;
+
 	drm_crtc_helper_add(crtc, crtc_helper_funcs);
 
 	if (!vc4->is_vc5) {
@@ -1275,10 +1335,9 @@ static int vc4_crtc_bind(struct device *dev, struct device *master, void *data)
 	const struct vc4_pv_data *pv_data;
 	struct vc4_crtc *vc4_crtc;
 	struct drm_crtc *crtc;
-	struct drm_plane *destroy_plane, *temp;
 	int ret;
 
-	vc4_crtc = devm_kzalloc(dev, sizeof(*vc4_crtc), GFP_KERNEL);
+	vc4_crtc = drmm_kzalloc(drm, sizeof(*vc4_crtc), GFP_KERNEL);
 	if (!vc4_crtc)
 		return -ENOMEM;
 	crtc = &vc4_crtc->base;
@@ -1310,23 +1369,11 @@ static int vc4_crtc_bind(struct device *dev, struct device *master, void *data)
 			       IRQF_SHARED,
 			       "vc4 crtc", vc4_crtc);
 	if (ret)
-		goto err_destroy_planes;
+		return ret;
 
 	platform_set_drvdata(pdev, vc4_crtc);
 
-	vc4_debugfs_add_regset32(drm, pv_data->debugfs_name,
-				 &vc4_crtc->regset);
-
 	return 0;
-
-err_destroy_planes:
-	list_for_each_entry_safe(destroy_plane, temp,
-				 &drm->mode_config.plane_list, head) {
-		if (destroy_plane->possible_crtcs == drm_crtc_mask(crtc))
-		    destroy_plane->funcs->destroy(destroy_plane);
-	}
-
-	return ret;
 }
 
 static void vc4_crtc_unbind(struct device *dev, struct device *master,
@@ -1335,8 +1382,6 @@ static void vc4_crtc_unbind(struct device *dev, struct device *master,
 	struct platform_device *pdev = to_platform_device(dev);
 	struct vc4_crtc *vc4_crtc = dev_get_drvdata(dev);
 
-	vc4_crtc_destroy(&vc4_crtc->base);
-
 	CRTC_WRITE(PV_INTEN, 0);
 
 	platform_set_drvdata(pdev, NULL);
diff --git a/drivers/gpu/drm/vc4/vc4_debugfs.c b/drivers/gpu/drm/vc4/vc4_debugfs.c
index ba2d8ea562af..19cda4f91a82 100644
--- a/drivers/gpu/drm/vc4/vc4_debugfs.c
+++ b/drivers/gpu/drm/vc4/vc4_debugfs.c
@@ -3,6 +3,8 @@
  *  Copyright © 2014 Broadcom
  */
 
+#include <drm/drm_drv.h>
+
 #include <linux/seq_file.h>
 #include <linux/circ_buf.h>
 #include <linux/ctype.h>
@@ -12,11 +14,6 @@
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 
-struct vc4_debugfs_info_entry {
-	struct list_head link;
-	struct drm_info_list info;
-};
-
 /*
  * Called at drm_dev_register() time on each of the minors registered
  * by the DRM device, to attach the debugfs files.
@@ -25,62 +22,59 @@ void
 vc4_debugfs_init(struct drm_minor *minor)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(minor->dev);
-	struct vc4_debugfs_info_entry *entry;
+	struct drm_device *drm = &vc4->base;
 
-	if (!of_device_is_compatible(vc4->hvs->pdev->dev.of_node,
-				     "brcm,bcm2711-vc5"))
-		debugfs_create_bool("hvs_load_tracker", S_IRUGO | S_IWUSR,
-				    minor->debugfs_root, &vc4->load_tracker_enabled);
+	drm_WARN_ON(drm, vc4_hvs_debugfs_init(minor));
 
-	list_for_each_entry(entry, &vc4->debugfs_list, link) {
-		drm_debugfs_create_files(&entry->info, 1,
-					 minor->debugfs_root, minor);
+	if (vc4->v3d) {
+		drm_WARN_ON(drm, vc4_bo_debugfs_init(minor));
+		drm_WARN_ON(drm, vc4_v3d_debugfs_init(minor));
 	}
 }
 
 static int vc4_debugfs_regset32(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *)m->private;
+	struct drm_device *drm = node->minor->dev;
 	struct debugfs_regset32 *regset = node->info_ent->data;
 	struct drm_printer p = drm_seq_file_printer(m);
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return -ENODEV;
 
 	drm_print_regset32(&p, regset);
 
+	drm_dev_exit(idx);
+
 	return 0;
 }
 
-/*
- * Registers a debugfs file with a callback function for a vc4 component.
- *
- * This is like drm_debugfs_create_files(), but that can only be
- * called a given DRM minor, while the various VC4 components want to
- * register their debugfs files during the component bind process.  We
- * track the request and delay it to be called on each minor during
- * vc4_debugfs_init().
- */
-void vc4_debugfs_add_file(struct drm_device *dev,
-			  const char *name,
-			  int (*show)(struct seq_file*, void*),
-			  void *data)
+int vc4_debugfs_add_file(struct drm_minor *minor,
+			 const char *name,
+			 int (*show)(struct seq_file*, void*),
+			 void *data)
 {
-	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	struct drm_device *dev = minor->dev;
+	struct dentry *root = minor->debugfs_root;
+	struct drm_info_list *file;
 
-	struct vc4_debugfs_info_entry *entry =
-		devm_kzalloc(dev->dev, sizeof(*entry), GFP_KERNEL);
+	file = drmm_kzalloc(dev, sizeof(*file), GFP_KERNEL);
+	if (!file)
+		return -ENOMEM;
 
-	if (!entry)
-		return;
+	file->name = name;
+	file->show = show;
+	file->data = data;
 
-	entry->info.name = name;
-	entry->info.show = show;
-	entry->info.data = data;
+	drm_debugfs_create_files(file, 1, root, minor);
 
-	list_add(&entry->link, &vc4->debugfs_list);
+	return 0;
 }
 
-void vc4_debugfs_add_regset32(struct drm_device *drm,
-			      const char *name,
-			      struct debugfs_regset32 *regset)
+int vc4_debugfs_add_regset32(struct drm_minor *minor,
+			     const char *name,
+			     struct debugfs_regset32 *regset)
 {
-	vc4_debugfs_add_file(drm, name, vc4_debugfs_regset32, regset);
+	return vc4_debugfs_add_file(minor, name, vc4_debugfs_regset32, regset);
 }
diff --git a/drivers/gpu/drm/vc4/vc4_dpi.c b/drivers/gpu/drm/vc4/vc4_dpi.c
index ef5e3921062c..1f8f44b7b5a5 100644
--- a/drivers/gpu/drm/vc4/vc4_dpi.c
+++ b/drivers/gpu/drm/vc4/vc4_dpi.c
@@ -13,6 +13,7 @@
 
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
@@ -84,9 +85,9 @@
 
 /* General DPI hardware state. */
 struct vc4_dpi {
-	struct platform_device *pdev;
+	struct vc4_encoder encoder;
 
-	struct drm_encoder *encoder;
+	struct platform_device *pdev;
 
 	void __iomem *regs;
 
@@ -96,21 +97,15 @@ struct vc4_dpi {
 	struct debugfs_regset32 regset;
 };
 
-#define DPI_READ(offset) readl(dpi->regs + (offset))
-#define DPI_WRITE(offset, val) writel(val, dpi->regs + (offset))
-
-/* VC4 DPI encoder KMS struct */
-struct vc4_dpi_encoder {
-	struct vc4_encoder base;
-	struct vc4_dpi *dpi;
-};
-
-static inline struct vc4_dpi_encoder *
-to_vc4_dpi_encoder(struct drm_encoder *encoder)
+static inline struct vc4_dpi *
+to_vc4_dpi(struct drm_encoder *encoder)
 {
-	return container_of(encoder, struct vc4_dpi_encoder, base.base);
+	return container_of(encoder, struct vc4_dpi, encoder.base);
 }
 
+#define DPI_READ(offset) readl(dpi->regs + (offset))
+#define DPI_WRITE(offset, val) writel(val, dpi->regs + (offset))
+
 static const struct debugfs_reg32 dpi_regs[] = {
 	VC4_REG32(DPI_C),
 	VC4_REG32(DPI_ID),
@@ -118,21 +113,27 @@ static const struct debugfs_reg32 dpi_regs[] = {
 
 static void vc4_dpi_encoder_disable(struct drm_encoder *encoder)
 {
-	struct vc4_dpi_encoder *vc4_encoder = to_vc4_dpi_encoder(encoder);
-	struct vc4_dpi *dpi = vc4_encoder->dpi;
+	struct drm_device *dev = encoder->dev;
+	struct vc4_dpi *dpi = to_vc4_dpi(encoder);
+	int idx;
+
+	if (!drm_dev_enter(dev, &idx))
+		return;
 
 	clk_disable_unprepare(dpi->pixel_clock);
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_dpi_encoder_enable(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
 	struct drm_display_mode *mode = &encoder->crtc->mode;
-	struct vc4_dpi_encoder *vc4_encoder = to_vc4_dpi_encoder(encoder);
-	struct vc4_dpi *dpi = vc4_encoder->dpi;
+	struct vc4_dpi *dpi = to_vc4_dpi(encoder);
 	struct drm_connector_list_iter conn_iter;
 	struct drm_connector *connector = NULL, *connector_scan;
 	u32 dpi_c = DPI_ENABLE;
+	int idx;
 	int ret;
 
 	/* Look up the connector attached to DPI so we can get the
@@ -212,6 +213,9 @@ static void vc4_dpi_encoder_enable(struct drm_encoder *encoder)
 			dpi_c |= DPI_VSYNC_DISABLE;
 	}
 
+	if (!drm_dev_enter(dev, &idx))
+		return;
+
 	DPI_WRITE(DPI_C, dpi_c);
 
 	ret = clk_set_rate(dpi->pixel_clock, mode->clock * 1000);
@@ -221,6 +225,8 @@ static void vc4_dpi_encoder_enable(struct drm_encoder *encoder)
 	ret = clk_prepare_enable(dpi->pixel_clock);
 	if (ret)
 		DRM_ERROR("Failed to set clock rate: %d\n", ret);
+
+	drm_dev_exit(idx);
 }
 
 static enum drm_mode_status vc4_dpi_encoder_mode_valid(struct drm_encoder *encoder,
@@ -238,6 +244,23 @@ static const struct drm_encoder_helper_funcs vc4_dpi_encoder_helper_funcs = {
 	.mode_valid = vc4_dpi_encoder_mode_valid,
 };
 
+static int vc4_dpi_late_register(struct drm_encoder *encoder)
+{
+	struct drm_device *drm = encoder->dev;
+	struct vc4_dpi *dpi = to_vc4_dpi(encoder);
+	int ret;
+
+	ret = vc4_debugfs_add_regset32(drm->primary, "dpi_regs", &dpi->regset);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct drm_encoder_funcs vc4_dpi_encoder_funcs = {
+	.late_register = vc4_dpi_late_register,
+};
+
 static const struct of_device_id vc4_dpi_dt_match[] = {
 	{ .compatible = "brcm,bcm2835-dpi", .data = NULL },
 	{}
@@ -248,10 +271,11 @@ static const struct of_device_id vc4_dpi_dt_match[] = {
  */
 static int vc4_dpi_init_bridge(struct vc4_dpi *dpi)
 {
+	struct drm_device *drm = dpi->encoder.base.dev;
 	struct device *dev = &dpi->pdev->dev;
 	struct drm_bridge *bridge;
 
-	bridge = devm_drm_of_get_bridge(dev, dev->of_node, 0, 0);
+	bridge = drmm_of_get_bridge(drm, dev->of_node, 0, 0);
 	if (IS_ERR(bridge)) {
 		/* If nothing was connected in the DT, that's not an
 		 * error.
@@ -262,30 +286,28 @@ static int vc4_dpi_init_bridge(struct vc4_dpi *dpi)
 			return PTR_ERR(bridge);
 	}
 
-	return drm_bridge_attach(dpi->encoder, bridge, NULL, 0);
+	return drm_bridge_attach(&dpi->encoder.base, bridge, NULL, 0);
+}
+
+static void vc4_dpi_disable_clock(void *ptr)
+{
+	struct vc4_dpi *dpi = ptr;
+
+	clk_disable_unprepare(dpi->core_clock);
 }
 
 static int vc4_dpi_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct drm_device *drm = dev_get_drvdata(master);
-	struct vc4_dev *vc4 = to_vc4_dev(drm);
 	struct vc4_dpi *dpi;
-	struct vc4_dpi_encoder *vc4_dpi_encoder;
 	int ret;
 
-	dpi = devm_kzalloc(dev, sizeof(*dpi), GFP_KERNEL);
+	dpi = drmm_kzalloc(drm, sizeof(*dpi), GFP_KERNEL);
 	if (!dpi)
 		return -ENOMEM;
 
-	vc4_dpi_encoder = devm_kzalloc(dev, sizeof(*vc4_dpi_encoder),
-				       GFP_KERNEL);
-	if (!vc4_dpi_encoder)
-		return -ENOMEM;
-	vc4_dpi_encoder->base.type = VC4_ENCODER_TYPE_DPI;
-	vc4_dpi_encoder->dpi = dpi;
-	dpi->encoder = &vc4_dpi_encoder->base.base;
-
+	dpi->encoder.type = VC4_ENCODER_TYPE_DPI;
 	dpi->pdev = pdev;
 	dpi->regs = vc4_ioremap_regs(pdev, 0);
 	if (IS_ERR(dpi->regs))
@@ -307,6 +329,7 @@ static int vc4_dpi_bind(struct device *dev, struct device *master, void *data)
 			DRM_ERROR("Failed to get core clock: %d\n", ret);
 		return ret;
 	}
+
 	dpi->pixel_clock = devm_clk_get(dev, "pixel");
 	if (IS_ERR(dpi->pixel_clock)) {
 		ret = PTR_ERR(dpi->pixel_clock);
@@ -316,49 +339,35 @@ static int vc4_dpi_bind(struct device *dev, struct device *master, void *data)
 	}
 
 	ret = clk_prepare_enable(dpi->core_clock);
-	if (ret)
+	if (ret) {
 		DRM_ERROR("Failed to turn on core clock: %d\n", ret);
+		return ret;
+	}
 
-	drm_simple_encoder_init(drm, dpi->encoder, DRM_MODE_ENCODER_DPI);
-	drm_encoder_helper_add(dpi->encoder, &vc4_dpi_encoder_helper_funcs);
+	ret = devm_add_action_or_reset(dev, vc4_dpi_disable_clock, dpi);
+	if (ret)
+		return ret;
 
-	ret = vc4_dpi_init_bridge(dpi);
+	ret = drmm_encoder_init(drm, &dpi->encoder.base,
+				&vc4_dpi_encoder_funcs,
+				DRM_MODE_ENCODER_DPI,
+				NULL);
 	if (ret)
-		goto err_destroy_encoder;
+		return ret;
 
-	dev_set_drvdata(dev, dpi);
+	drm_encoder_helper_add(&dpi->encoder.base, &vc4_dpi_encoder_helper_funcs);
 
-	vc4->dpi = dpi;
+	ret = vc4_dpi_init_bridge(dpi);
+	if (ret)
+		return ret;
 
-	vc4_debugfs_add_regset32(drm, "dpi_regs", &dpi->regset);
+	dev_set_drvdata(dev, dpi);
 
 	return 0;
-
-err_destroy_encoder:
-	drm_encoder_cleanup(dpi->encoder);
-	clk_disable_unprepare(dpi->core_clock);
-	return ret;
-}
-
-static void vc4_dpi_unbind(struct device *dev, struct device *master,
-			   void *data)
-{
-	struct drm_device *drm = dev_get_drvdata(master);
-	struct vc4_dev *vc4 = to_vc4_dev(drm);
-	struct vc4_dpi *dpi = dev_get_drvdata(dev);
-
-	drm_of_panel_bridge_remove(dev->of_node, 0, 0);
-
-	drm_encoder_cleanup(dpi->encoder);
-
-	clk_disable_unprepare(dpi->core_clock);
-
-	vc4->dpi = NULL;
 }
 
 static const struct component_ops vc4_dpi_ops = {
 	.bind   = vc4_dpi_bind,
-	.unbind = vc4_dpi_unbind,
 };
 
 static int vc4_dpi_dev_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index 292d1b6a01b6..ffbbb454c9e8 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -33,7 +33,6 @@
 #include <drm/drm_aperture.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_drv.h>
-#include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_vblank.h>
 
@@ -86,7 +85,7 @@ static int vc5_dumb_create(struct drm_file *file_priv,
 	if (ret)
 		return ret;
 
-	return drm_gem_cma_dumb_create_internal(file_priv, dev, args);
+	return drm_gem_dma_dumb_create_internal(file_priv, dev, args);
 }
 
 static int vc4_get_param_ioctl(struct drm_device *dev, void *data,
@@ -212,7 +211,7 @@ static const struct drm_driver vc4_drm_driver = {
 
 	.gem_create_object = vc4_create_object,
 
-	DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(vc4_bo_dumb_create),
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(vc4_bo_dumb_create),
 
 	.ioctls = vc4_drm_ioctls,
 	.num_ioctls = ARRAY_SIZE(vc4_drm_ioctls),
@@ -235,7 +234,7 @@ static const struct drm_driver vc5_drm_driver = {
 	.debugfs_init = vc4_debugfs_init,
 #endif
 
-	DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(vc5_dumb_create),
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(vc5_dumb_create),
 
 	.fops = &vc4_drm_fops,
 
@@ -267,6 +266,13 @@ static void vc4_match_add_drivers(struct device *dev,
 	}
 }
 
+static void vc4_component_unbind_all(void *ptr)
+{
+	struct vc4_dev *vc4 = ptr;
+
+	component_unbind_all(vc4->dev, &vc4->base);
+}
+
 static const struct of_device_id vc4_dma_range_matches[] = {
 	{ .compatible = "brcm,bcm2711-hvs" },
 	{ .compatible = "brcm,bcm2835-hvs" },
@@ -310,13 +316,16 @@ static int vc4_drm_bind(struct device *dev)
 	if (IS_ERR(vc4))
 		return PTR_ERR(vc4);
 	vc4->is_vc5 = is_vc5;
+	vc4->dev = dev;
 
 	drm = &vc4->base;
 	platform_set_drvdata(pdev, drm);
 	INIT_LIST_HEAD(&vc4->debugfs_list);
 
 	if (!is_vc5) {
-		mutex_init(&vc4->bin_bo_lock);
+		ret = drmm_mutex_init(drm, &vc4->bin_bo_lock);
+		if (ret)
+			return ret;
 
 		ret = vc4_bo_cache_init(drm);
 		if (ret)
@@ -360,6 +369,10 @@ static int vc4_drm_bind(struct device *dev)
 	if (ret)
 		return ret;
 
+	ret = devm_add_action_or_reset(dev, vc4_component_unbind_all, vc4);
+	if (ret)
+		return ret;
+
 	ret = vc4_plane_create_additional_planes(drm);
 	if (ret)
 		goto unbind_all;
@@ -380,8 +393,6 @@ static int vc4_drm_bind(struct device *dev)
 	return 0;
 
 unbind_all:
-	component_unbind_all(dev, drm);
-
 	return ret;
 }
 
@@ -389,8 +400,7 @@ static void vc4_drm_unbind(struct device *dev)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
 
-	drm_dev_unregister(drm);
-
+	drm_dev_unplug(drm);
 	drm_atomic_helper_shutdown(drm);
 }
 
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 1beb96b77b8c..418a8242691f 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -14,7 +14,7 @@
 #include <drm/drm_debugfs.h>
 #include <drm/drm_device.h>
 #include <drm/drm_encoder.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mm.h>
 #include <drm/drm_modeset_lock.h>
@@ -76,6 +76,7 @@ struct vc4_perfmon {
 
 struct vc4_dev {
 	struct drm_device base;
+	struct device *dev;
 
 	bool is_vc5;
 
@@ -83,9 +84,6 @@ struct vc4_dev {
 
 	struct vc4_hvs *hvs;
 	struct vc4_v3d *v3d;
-	struct vc4_dpi *dpi;
-	struct vc4_vec *vec;
-	struct vc4_txp *txp;
 
 	struct vc4_hang_state *hang_state;
 
@@ -241,7 +239,7 @@ to_vc4_dev(struct drm_device *dev)
 }
 
 struct vc4_bo {
-	struct drm_gem_cma_object base;
+	struct drm_gem_dma_object base;
 
 	/* seqno of the last job to render using this BO. */
 	uint64_t seqno;
@@ -290,7 +288,7 @@ struct vc4_bo {
 static inline struct vc4_bo *
 to_vc4_bo(struct drm_gem_object *bo)
 {
-	return container_of(to_drm_gem_cma_obj(bo), struct vc4_bo, base);
+	return container_of(to_drm_gem_dma_obj(bo), struct vc4_bo, base);
 }
 
 struct vc4_fence {
@@ -458,6 +456,8 @@ to_vc4_encoder(struct drm_encoder *encoder)
 }
 
 struct vc4_crtc_data {
+	const char *debugfs_name;
+
 	/* Bitmask of channels (FIFOs) of the HVS that the output can source from */
 	unsigned int hvs_available_channels;
 
@@ -475,8 +475,6 @@ struct vc4_pv_data {
 	u8 pixels_per_clock;
 
 	enum vc4_encoder_type encoder_types[4];
-	const char *debugfs_name;
-
 };
 
 struct vc4_crtc {
@@ -604,14 +602,14 @@ struct vc4_exec_info {
 	/* This is the array of BOs that were looked up at the start of exec.
 	 * Command validation will use indices into this array.
 	 */
-	struct drm_gem_cma_object **bo;
+	struct drm_gem_dma_object **bo;
 	uint32_t bo_count;
 
 	/* List of BOs that are being written by the RCL.  Other than
 	 * the binner temporary storage, this is all the BOs written
 	 * by the job.
 	 */
-	struct drm_gem_cma_object *rcl_write_bo[4];
+	struct drm_gem_dma_object *rcl_write_bo[4];
 	uint32_t rcl_write_bo_count;
 
 	/* Pointers for our position in vc4->job_list */
@@ -630,7 +628,7 @@ struct vc4_exec_info {
 	/* This is the BO where we store the validated command lists, shader
 	 * records, and uniforms.
 	 */
-	struct drm_gem_cma_object *exec_bo;
+	struct drm_gem_dma_object *exec_bo;
 
 	/**
 	 * This tracks the per-shader-record state (packet 64) that
@@ -843,6 +841,7 @@ int vc4_bo_inc_usecnt(struct vc4_bo *bo);
 void vc4_bo_dec_usecnt(struct vc4_bo *bo);
 void vc4_bo_add_to_purgeable_pool(struct vc4_bo *bo);
 void vc4_bo_remove_from_purgeable_pool(struct vc4_bo *bo);
+int vc4_bo_debugfs_init(struct drm_minor *minor);
 
 /* vc4_crtc.c */
 extern struct platform_driver vc4_crtc_driver;
@@ -850,7 +849,6 @@ int vc4_crtc_disable_at_boot(struct drm_crtc *crtc);
 int vc4_crtc_init(struct drm_device *drm, struct vc4_crtc *vc4_crtc,
 		  const struct drm_crtc_funcs *crtc_funcs,
 		  const struct drm_crtc_helper_funcs *crtc_helper_funcs);
-void vc4_crtc_destroy(struct drm_crtc *crtc);
 int vc4_page_flip(struct drm_crtc *crtc,
 		  struct drm_framebuffer *fb,
 		  struct drm_pending_vblank_event *event,
@@ -861,6 +859,8 @@ void vc4_crtc_destroy_state(struct drm_crtc *crtc,
 			    struct drm_crtc_state *state);
 void vc4_crtc_reset(struct drm_crtc *crtc);
 void vc4_crtc_handle_vblank(struct vc4_crtc *crtc);
+void vc4_crtc_send_vblank(struct drm_crtc *crtc);
+int vc4_crtc_late_register(struct drm_crtc *crtc);
 void vc4_crtc_get_margins(struct drm_crtc_state *state,
 			  unsigned int *left, unsigned int *right,
 			  unsigned int *top, unsigned int *bottom);
@@ -868,25 +868,27 @@ void vc4_crtc_get_margins(struct drm_crtc_state *state,
 /* vc4_debugfs.c */
 void vc4_debugfs_init(struct drm_minor *minor);
 #ifdef CONFIG_DEBUG_FS
-void vc4_debugfs_add_file(struct drm_device *drm,
-			  const char *filename,
-			  int (*show)(struct seq_file*, void*),
-			  void *data);
-void vc4_debugfs_add_regset32(struct drm_device *drm,
-			      const char *filename,
-			      struct debugfs_regset32 *regset);
+int vc4_debugfs_add_file(struct drm_minor *minor,
+			 const char *filename,
+			 int (*show)(struct seq_file*, void*),
+			 void *data);
+int vc4_debugfs_add_regset32(struct drm_minor *minor,
+			     const char *filename,
+			     struct debugfs_regset32 *regset);
 #else
-static inline void vc4_debugfs_add_file(struct drm_device *drm,
-					const char *filename,
-					int (*show)(struct seq_file*, void*),
-					void *data)
+static inline int vc4_debugfs_add_file(struct drm_minor *minor,
+				       const char *filename,
+				       int (*show)(struct seq_file*, void*),
+				       void *data)
 {
+	return 0;
 }
 
-static inline void vc4_debugfs_add_regset32(struct drm_device *drm,
-					    const char *filename,
-					    struct debugfs_regset32 *regset)
+static inline int vc4_debugfs_add_regset32(struct drm_minor *minor,
+					   const char *filename,
+					   struct debugfs_regset32 *regset)
 {
+	return 0;
 }
 #endif
 
@@ -952,13 +954,15 @@ void vc4_hvs_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *state)
 void vc4_hvs_dump_state(struct vc4_hvs *hvs);
 void vc4_hvs_unmask_underrun(struct vc4_hvs *hvs, int channel);
 void vc4_hvs_mask_underrun(struct vc4_hvs *hvs, int channel);
+int vc4_hvs_debugfs_init(struct drm_minor *minor);
 
 /* vc4_kms.c */
 int vc4_kms_load(struct drm_device *dev);
 
 /* vc4_plane.c */
 struct drm_plane *vc4_plane_init(struct drm_device *dev,
-				 enum drm_plane_type type);
+				 enum drm_plane_type type,
+				 uint32_t possible_crtcs);
 int vc4_plane_create_additional_planes(struct drm_device *dev);
 u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist);
 u32 vc4_plane_dlist_size(const struct drm_plane_state *state);
@@ -973,6 +977,7 @@ int vc4_v3d_bin_bo_get(struct vc4_dev *vc4, bool *used);
 void vc4_v3d_bin_bo_put(struct vc4_dev *vc4);
 int vc4_v3d_pm_get(struct vc4_dev *vc4);
 void vc4_v3d_pm_put(struct vc4_dev *vc4);
+int vc4_v3d_debugfs_init(struct drm_minor *minor);
 
 /* vc4_validate.c */
 int
@@ -984,19 +989,19 @@ vc4_validate_bin_cl(struct drm_device *dev,
 int
 vc4_validate_shader_recs(struct drm_device *dev, struct vc4_exec_info *exec);
 
-struct drm_gem_cma_object *vc4_use_bo(struct vc4_exec_info *exec,
+struct drm_gem_dma_object *vc4_use_bo(struct vc4_exec_info *exec,
 				      uint32_t hindex);
 
 int vc4_get_rcl(struct drm_device *dev, struct vc4_exec_info *exec);
 
 bool vc4_check_tex_size(struct vc4_exec_info *exec,
-			struct drm_gem_cma_object *fbo,
+			struct drm_gem_dma_object *fbo,
 			uint32_t offset, uint8_t tiling_format,
 			uint32_t width, uint32_t height, uint8_t cpp);
 
 /* vc4_validate_shader.c */
 struct vc4_validated_shader_info *
-vc4_validate_shader(struct drm_gem_cma_object *shader_obj);
+vc4_validate_shader(struct drm_gem_dma_object *shader_obj);
 
 /* vc4_perfmon.c */
 void vc4_perfmon_get(struct vc4_perfmon *perfmon);
diff --git a/drivers/gpu/drm/vc4/vc4_dsi.c b/drivers/gpu/drm/vc4/vc4_dsi.c
index b7b2c76770dc..878e05d79e81 100644
--- a/drivers/gpu/drm/vc4/vc4_dsi.c
+++ b/drivers/gpu/drm/vc4/vc4_dsi.c
@@ -549,10 +549,13 @@ struct vc4_dsi_variant {
 
 /* General DSI hardware state. */
 struct vc4_dsi {
+	struct vc4_encoder encoder;
+	struct mipi_dsi_host dsi_host;
+
+	struct kref kref;
+
 	struct platform_device *pdev;
 
-	struct mipi_dsi_host dsi_host;
-	struct drm_encoder *encoder;
 	struct drm_bridge *bridge;
 	struct list_head bridge_chain;
 
@@ -600,6 +603,12 @@ struct vc4_dsi {
 
 #define host_to_dsi(host) container_of(host, struct vc4_dsi, dsi_host)
 
+static inline struct vc4_dsi *
+to_vc4_dsi(struct drm_encoder *encoder)
+{
+	return container_of(encoder, struct vc4_dsi, encoder.base);
+}
+
 static inline void
 dsi_dma_workaround_write(struct vc4_dsi *dsi, u32 offset, u32 val)
 {
@@ -644,18 +653,6 @@ dsi_dma_workaround_write(struct vc4_dsi *dsi, u32 offset, u32 val)
 	DSI_WRITE(dsi->variant->port ? DSI1_##offset : DSI0_##offset, val)
 #define DSI_PORT_BIT(bit) (dsi->variant->port ? DSI1_##bit : DSI0_##bit)
 
-/* VC4 DSI encoder KMS struct */
-struct vc4_dsi_encoder {
-	struct vc4_encoder base;
-	struct vc4_dsi *dsi;
-};
-
-static inline struct vc4_dsi_encoder *
-to_vc4_dsi_encoder(struct drm_encoder *encoder)
-{
-	return container_of(encoder, struct vc4_dsi_encoder, base.base);
-}
-
 static const struct debugfs_reg32 dsi0_regs[] = {
 	VC4_REG32(DSI0_CTRL),
 	VC4_REG32(DSI0_STAT),
@@ -795,8 +792,7 @@ dsi_esc_timing(u32 ns)
 
 static void vc4_dsi_encoder_disable(struct drm_encoder *encoder)
 {
-	struct vc4_dsi_encoder *vc4_encoder = to_vc4_dsi_encoder(encoder);
-	struct vc4_dsi *dsi = vc4_encoder->dsi;
+	struct vc4_dsi *dsi = to_vc4_dsi(encoder);
 	struct device *dev = &dsi->pdev->dev;
 	struct drm_bridge *iter;
 
@@ -839,8 +835,7 @@ static bool vc4_dsi_encoder_mode_fixup(struct drm_encoder *encoder,
 				       const struct drm_display_mode *mode,
 				       struct drm_display_mode *adjusted_mode)
 {
-	struct vc4_dsi_encoder *vc4_encoder = to_vc4_dsi_encoder(encoder);
-	struct vc4_dsi *dsi = vc4_encoder->dsi;
+	struct vc4_dsi *dsi = to_vc4_dsi(encoder);
 	struct clk *phy_parent = clk_get_parent(dsi->pll_phy_clock);
 	unsigned long parent_rate = clk_get_rate(phy_parent);
 	unsigned long pixel_clock_hz = mode->clock * 1000;
@@ -875,8 +870,7 @@ static bool vc4_dsi_encoder_mode_fixup(struct drm_encoder *encoder,
 static void vc4_dsi_encoder_enable(struct drm_encoder *encoder)
 {
 	struct drm_display_mode *mode = &encoder->crtc->state->adjusted_mode;
-	struct vc4_dsi_encoder *vc4_encoder = to_vc4_dsi_encoder(encoder);
-	struct vc4_dsi *dsi = vc4_encoder->dsi;
+	struct vc4_dsi *dsi = to_vc4_dsi(encoder);
 	struct device *dev = &dsi->pdev->dev;
 	bool debug_dump_regs = false;
 	struct drm_bridge *iter;
@@ -1378,6 +1372,24 @@ static const struct drm_encoder_helper_funcs vc4_dsi_encoder_helper_funcs = {
 	.mode_fixup = vc4_dsi_encoder_mode_fixup,
 };
 
+static int vc4_dsi_late_register(struct drm_encoder *encoder)
+{
+	struct drm_device *drm = encoder->dev;
+	struct vc4_dsi *dsi = to_vc4_dsi(encoder);
+	int ret;
+
+	ret = vc4_debugfs_add_regset32(drm->primary, dsi->variant->debugfs_name,
+				       &dsi->regset);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct drm_encoder_funcs vc4_dsi_encoder_funcs = {
+	.late_register = vc4_dsi_late_register,
+};
+
 static const struct vc4_dsi_variant bcm2711_dsi1_variant = {
 	.port			= 1,
 	.debugfs_name		= "dsi1_regs",
@@ -1564,26 +1576,50 @@ static void vc4_dsi_dma_chan_release(void *ptr)
 	dsi->reg_dma_chan = NULL;
 }
 
+static void vc4_dsi_release(struct kref *kref)
+{
+	struct vc4_dsi *dsi =
+		container_of(kref, struct vc4_dsi, kref);
+
+	kfree(dsi);
+}
+
+static void vc4_dsi_get(struct vc4_dsi *dsi)
+{
+	kref_get(&dsi->kref);
+}
+
+static void vc4_dsi_put(struct vc4_dsi *dsi)
+{
+	kref_put(&dsi->kref, &vc4_dsi_release);
+}
+
+static void vc4_dsi_release_action(struct drm_device *drm, void *ptr)
+{
+	struct vc4_dsi *dsi = ptr;
+
+	vc4_dsi_put(dsi);
+}
+
 static int vc4_dsi_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct drm_device *drm = dev_get_drvdata(master);
 	struct vc4_dsi *dsi = dev_get_drvdata(dev);
-	struct vc4_dsi_encoder *vc4_dsi_encoder;
+	struct drm_encoder *encoder = &dsi->encoder.base;
 	int ret;
 
-	dsi->variant = of_device_get_match_data(dev);
+	vc4_dsi_get(dsi);
 
-	vc4_dsi_encoder = devm_kzalloc(dev, sizeof(*vc4_dsi_encoder),
-				       GFP_KERNEL);
-	if (!vc4_dsi_encoder)
-		return -ENOMEM;
+	ret = drmm_add_action_or_reset(drm, vc4_dsi_release_action, dsi);
+	if (ret)
+		return ret;
+
+	dsi->variant = of_device_get_match_data(dev);
 
 	INIT_LIST_HEAD(&dsi->bridge_chain);
-	vc4_dsi_encoder->base.type = dsi->variant->port ?
-			VC4_ENCODER_TYPE_DSI1 : VC4_ENCODER_TYPE_DSI0;
-	vc4_dsi_encoder->dsi = dsi;
-	dsi->encoder = &vc4_dsi_encoder->base.base;
+	dsi->encoder.type = dsi->variant->port ?
+		VC4_ENCODER_TYPE_DSI1 : VC4_ENCODER_TYPE_DSI0;
 
 	dsi->regs = vc4_ioremap_regs(pdev, 0);
 	if (IS_ERR(dsi->regs))
@@ -1687,7 +1723,7 @@ static int vc4_dsi_bind(struct device *dev, struct device *master, void *data)
 		return ret;
 	}
 
-	dsi->bridge = devm_drm_of_get_bridge(dev, dev->of_node, 0, 0);
+	dsi->bridge = drmm_of_get_bridge(drm, dev->of_node, 0, 0);
 	if (IS_ERR(dsi->bridge))
 		return PTR_ERR(dsi->bridge);
 
@@ -1702,10 +1738,20 @@ static int vc4_dsi_bind(struct device *dev, struct device *master, void *data)
 	if (ret)
 		return ret;
 
-	drm_simple_encoder_init(drm, dsi->encoder, DRM_MODE_ENCODER_DSI);
-	drm_encoder_helper_add(dsi->encoder, &vc4_dsi_encoder_helper_funcs);
+	ret = drmm_encoder_init(drm, encoder,
+				&vc4_dsi_encoder_funcs,
+				DRM_MODE_ENCODER_DSI,
+				NULL);
+	if (ret)
+		return ret;
+
+	drm_encoder_helper_add(encoder, &vc4_dsi_encoder_helper_funcs);
 
-	ret = drm_bridge_attach(dsi->encoder, dsi->bridge, NULL, 0);
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return ret;
+
+	ret = drm_bridge_attach(encoder, dsi->bridge, NULL, 0);
 	if (ret)
 		return ret;
 	/* Disable the atomic helper calls into the bridge.  We
@@ -1713,11 +1759,7 @@ static int vc4_dsi_bind(struct device *dev, struct device *master, void *data)
 	 * from our driver, since we need to sequence them within the
 	 * encoder's enable/disable paths.
 	 */
-	list_splice_init(&dsi->encoder->bridge_chain, &dsi->bridge_chain);
-
-	vc4_debugfs_add_regset32(drm, dsi->variant->debugfs_name, &dsi->regset);
-
-	pm_runtime_enable(dev);
+	list_splice_init(&encoder->bridge_chain, &dsi->bridge_chain);
 
 	return 0;
 }
@@ -1726,15 +1768,13 @@ static void vc4_dsi_unbind(struct device *dev, struct device *master,
 			   void *data)
 {
 	struct vc4_dsi *dsi = dev_get_drvdata(dev);
-
-	pm_runtime_disable(dev);
+	struct drm_encoder *encoder = &dsi->encoder.base;
 
 	/*
 	 * Restore the bridge_chain so the bridge detach procedure can happen
 	 * normally.
 	 */
-	list_splice_init(&dsi->bridge_chain, &dsi->encoder->bridge_chain);
-	drm_encoder_cleanup(dsi->encoder);
+	list_splice_init(&dsi->bridge_chain, &encoder->bridge_chain);
 }
 
 static const struct component_ops vc4_dsi_ops = {
@@ -1747,11 +1787,12 @@ static int vc4_dsi_dev_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct vc4_dsi *dsi;
 
-	dsi = devm_kzalloc(dev, sizeof(*dsi), GFP_KERNEL);
+	dsi = kzalloc(sizeof(*dsi), GFP_KERNEL);
 	if (!dsi)
 		return -ENOMEM;
 	dev_set_drvdata(dev, dsi);
 
+	kref_init(&dsi->kref);
 	dsi->pdev = pdev;
 	dsi->dsi_host.ops = &vc4_dsi_host_ops;
 	dsi->dsi_host.dev = dev;
@@ -1766,6 +1807,8 @@ static int vc4_dsi_dev_remove(struct platform_device *pdev)
 	struct vc4_dsi *dsi = dev_get_drvdata(dev);
 
 	mipi_dsi_host_unregister(&dsi->dsi_host);
+	vc4_dsi_put(dsi);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index fe10d9c3fff8..628d40ff3aa1 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -126,7 +126,7 @@ vc4_get_hang_state_ioctl(struct drm_device *dev, void *data,
 			goto err_delete_handle;
 		}
 		bo_state[i].handle = handle;
-		bo_state[i].paddr = vc4_bo->base.paddr;
+		bo_state[i].paddr = vc4_bo->base.dma_addr;
 		bo_state[i].size = vc4_bo->base.base.size;
 	}
 
@@ -764,7 +764,7 @@ vc4_cl_lookup_bos(struct drm_device *dev,
 	}
 
 	exec->bo = kvmalloc_array(exec->bo_count,
-				    sizeof(struct drm_gem_cma_object *),
+				    sizeof(struct drm_gem_dma_object *),
 				    GFP_KERNEL | __GFP_ZERO);
 	if (!exec->bo) {
 		DRM_ERROR("Failed to allocate validated BO pointers\n");
@@ -797,7 +797,7 @@ vc4_cl_lookup_bos(struct drm_device *dev,
 		}
 
 		drm_gem_object_get(bo);
-		exec->bo[i] = (struct drm_gem_cma_object *)bo;
+		exec->bo[i] = (struct drm_gem_dma_object *)bo;
 	}
 	spin_unlock(&file_priv->table_lock);
 
@@ -917,16 +917,16 @@ vc4_get_bcl(struct drm_device *dev, struct vc4_exec_info *exec)
 	list_add_tail(&to_vc4_bo(&exec->exec_bo->base)->unref_head,
 		      &exec->unref_list);
 
-	exec->ct0ca = exec->exec_bo->paddr + bin_offset;
+	exec->ct0ca = exec->exec_bo->dma_addr + bin_offset;
 
 	exec->bin_u = bin;
 
 	exec->shader_rec_v = exec->exec_bo->vaddr + shader_rec_offset;
-	exec->shader_rec_p = exec->exec_bo->paddr + shader_rec_offset;
+	exec->shader_rec_p = exec->exec_bo->dma_addr + shader_rec_offset;
 	exec->shader_rec_size = args->shader_rec_size;
 
 	exec->uniforms_v = exec->exec_bo->vaddr + uniforms_offset;
-	exec->uniforms_p = exec->exec_bo->paddr + uniforms_offset;
+	exec->uniforms_p = exec->exec_bo->dma_addr + uniforms_offset;
 	exec->uniforms_size = args->uniforms_size;
 
 	ret = vc4_validate_bin_cl(dev,
@@ -1308,6 +1308,7 @@ static void vc4_gem_destroy(struct drm_device *dev, void *unused);
 int vc4_gem_init(struct drm_device *dev)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
+	int ret;
 
 	if (WARN_ON_ONCE(vc4->is_vc5))
 		return -ENODEV;
@@ -1325,10 +1326,15 @@ int vc4_gem_init(struct drm_device *dev)
 
 	INIT_WORK(&vc4->job_done_work, vc4_job_done_work);
 
-	mutex_init(&vc4->power_lock);
+	ret = drmm_mutex_init(dev, &vc4->power_lock);
+	if (ret)
+		return ret;
 
 	INIT_LIST_HEAD(&vc4->purgeable.list);
-	mutex_init(&vc4->purgeable.lock);
+
+	ret = drmm_mutex_init(dev, &vc4->purgeable.lock);
+	if (ret)
+		return ret;
 
 	return drmm_add_action_or_reset(dev, vc4_gem_destroy, NULL);
 }
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index 1e5f68704d7d..64f9feabf43e 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -34,6 +34,7 @@
 #include <drm/display/drm_hdmi_helper.h>
 #include <drm/display/drm_scdc_helper.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
 #include <linux/clk.h>
@@ -41,7 +42,6 @@
 #include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/of_address.h>
-#include <linux/of_gpio.h>
 #include <linux/of_platform.h>
 #include <linux/pm_runtime.h>
 #include <linux/rational.h>
@@ -124,6 +124,23 @@ static unsigned long long
 vc4_hdmi_encoder_compute_mode_clock(const struct drm_display_mode *mode,
 				    unsigned int bpc, enum vc4_hdmi_output_format fmt);
 
+static bool vc4_hdmi_supports_scrambling(struct drm_encoder *encoder)
+{
+	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+	struct drm_display_info *display = &vc4_hdmi->connector.display_info;
+
+	lockdep_assert_held(&vc4_hdmi->mutex);
+
+	if (!display->is_hdmi)
+		return false;
+
+	if (!display->hdmi.scdc.supported ||
+	    !display->hdmi.scdc.scrambling.supported)
+		return false;
+
+	return true;
+}
+
 static bool vc4_hdmi_mode_needs_scrambling(const struct drm_display_mode *mode,
 					   unsigned int bpc,
 					   enum vc4_hdmi_output_format fmt)
@@ -146,7 +163,12 @@ static int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused)
 {
 	struct drm_info_node *node = (struct drm_info_node *)m->private;
 	struct vc4_hdmi *vc4_hdmi = node->info_ent->data;
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	struct drm_printer p = drm_seq_file_printer(m);
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return -ENODEV;
 
 	drm_print_regset32(&p, &vc4_hdmi->hdmi_regset);
 	drm_print_regset32(&p, &vc4_hdmi->hd_regset);
@@ -157,12 +179,23 @@ static int vc4_hdmi_debugfs_regs(struct seq_file *m, void *unused)
 	drm_print_regset32(&p, &vc4_hdmi->ram_regset);
 	drm_print_regset32(&p, &vc4_hdmi->rm_regset);
 
+	drm_dev_exit(idx);
+
 	return 0;
 }
 
 static void vc4_hdmi_reset(struct vc4_hdmi *vc4_hdmi)
 {
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
+	int idx;
+
+	/*
+	 * We can be called by our bind callback, when the
+	 * connector->dev pointer might not be initialised yet.
+	 */
+	if (drm && !drm_dev_enter(drm, &idx))
+		return;
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
@@ -179,11 +212,23 @@ static void vc4_hdmi_reset(struct vc4_hdmi *vc4_hdmi)
 	HDMI_WRITE(HDMI_SW_RESET_CONTROL, 0);
 
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
+
+	if (drm)
+		drm_dev_exit(idx);
 }
 
 static void vc5_hdmi_reset(struct vc4_hdmi *vc4_hdmi)
 {
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
+	int idx;
+
+	/*
+	 * We can be called by our bind callback, when the
+	 * connector->dev pointer might not be initialised yet.
+	 */
+	if (drm && !drm_dev_enter(drm, &idx))
+		return;
 
 	reset_control_reset(vc4_hdmi->reset);
 
@@ -195,15 +240,31 @@ static void vc5_hdmi_reset(struct vc4_hdmi *vc4_hdmi)
 		   HDMI_READ(HDMI_CLOCK_STOP) | VC4_DVP_HT_CLOCK_STOP_PIXEL);
 
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
+
+	if (drm)
+		drm_dev_exit(idx);
 }
 
 #ifdef CONFIG_DRM_VC4_HDMI_CEC
 static void vc4_hdmi_cec_update_clk_div(struct vc4_hdmi *vc4_hdmi)
 {
-	unsigned long cec_rate = clk_get_rate(vc4_hdmi->cec_clock);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
+	unsigned long cec_rate;
 	unsigned long flags;
 	u16 clk_cnt;
 	u32 value;
+	int idx;
+
+	/*
+	 * This function is called by our runtime_resume implementation
+	 * and thus at bind time, when we haven't registered our
+	 * connector yet and thus don't have a pointer to the DRM
+	 * device.
+	 */
+	if (drm && !drm_dev_enter(drm, &idx))
+		return;
+
+	cec_rate = clk_get_rate(vc4_hdmi->cec_clock);
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
@@ -219,58 +280,180 @@ static void vc4_hdmi_cec_update_clk_div(struct vc4_hdmi *vc4_hdmi)
 	HDMI_WRITE(HDMI_CEC_CNTRL_1, value);
 
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
+
+	if (drm)
+		drm_dev_exit(idx);
 }
 #else
 static void vc4_hdmi_cec_update_clk_div(struct vc4_hdmi *vc4_hdmi) {}
 #endif
 
-static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder);
+static int reset_pipe(struct drm_crtc *crtc,
+			struct drm_modeset_acquire_ctx *ctx)
+{
+	struct drm_atomic_state *state;
+	struct drm_crtc_state *crtc_state;
+	int ret;
+
+	state = drm_atomic_state_alloc(crtc->dev);
+	if (!state)
+		return -ENOMEM;
+
+	state->acquire_ctx = ctx;
+
+	crtc_state = drm_atomic_get_crtc_state(state, crtc);
+	if (IS_ERR(crtc_state)) {
+		ret = PTR_ERR(crtc_state);
+		goto out;
+	}
+
+	crtc_state->connectors_changed = true;
+
+	ret = drm_atomic_commit(state);
+out:
+	drm_atomic_state_put(state);
+
+	return ret;
+}
 
-static enum drm_connector_status
-vc4_hdmi_connector_detect(struct drm_connector *connector, bool force)
+static int vc4_hdmi_reset_link(struct drm_connector *connector,
+			       struct drm_modeset_acquire_ctx *ctx)
 {
+	struct drm_device *drm = connector->dev;
 	struct vc4_hdmi *vc4_hdmi = connector_to_vc4_hdmi(connector);
-	bool connected = false;
+	struct drm_encoder *encoder = &vc4_hdmi->encoder.base;
+	struct drm_connector_state *conn_state;
+	struct drm_crtc_state *crtc_state;
+	struct drm_crtc *crtc;
+	bool scrambling_needed;
+	u8 config;
+	int ret;
 
-	mutex_lock(&vc4_hdmi->mutex);
+	if (!connector)
+		return 0;
+
+	ret = drm_modeset_lock(&drm->mode_config.connection_mutex, ctx);
+	if (ret)
+		return ret;
+
+	conn_state = connector->state;
+	crtc = conn_state->crtc;
+	if (!crtc)
+		return 0;
+
+	ret = drm_modeset_lock(&crtc->mutex, ctx);
+	if (ret)
+		return ret;
+
+	crtc_state = crtc->state;
+	if (!crtc_state->active)
+		return 0;
+
+	if (!vc4_hdmi_supports_scrambling(encoder))
+		return 0;
+
+	scrambling_needed = vc4_hdmi_mode_needs_scrambling(&vc4_hdmi->saved_adjusted_mode,
+							   vc4_hdmi->output_bpc,
+							   vc4_hdmi->output_format);
+	if (!scrambling_needed)
+		return 0;
+
+	if (conn_state->commit &&
+	    !try_wait_for_completion(&conn_state->commit->hw_done))
+		return 0;
+
+	ret = drm_scdc_readb(connector->ddc, SCDC_TMDS_CONFIG, &config);
+	if (ret < 0) {
+		drm_err(drm, "Failed to read TMDS config: %d\n", ret);
+		return 0;
+	}
+
+	if (!!(config & SCDC_SCRAMBLING_ENABLE) == scrambling_needed)
+		return 0;
+
+	/*
+	 * HDMI 2.0 says that one should not send scrambled data
+	 * prior to configuring the sink scrambling, and that
+	 * TMDS clock/data transmission should be suspended when
+	 * changing the TMDS clock rate in the sink. So let's
+	 * just do a full modeset here, even though some sinks
+	 * would be perfectly happy if were to just reconfigure
+	 * the SCDC settings on the fly.
+	 */
+	return reset_pipe(crtc, ctx);
+}
+
+static void vc4_hdmi_handle_hotplug(struct vc4_hdmi *vc4_hdmi,
+				    struct drm_modeset_acquire_ctx *ctx,
+				    enum drm_connector_status status)
+{
+	struct drm_connector *connector = &vc4_hdmi->connector;
+	struct edid *edid;
+
+	/*
+	 * NOTE: This function should really be called with
+	 * vc4_hdmi->mutex held, but doing so results in reentrancy
+	 * issues since cec_s_phys_addr_from_edid might call
+	 * .adap_enable, which leads to that funtion being called with
+	 * our mutex held.
+	 *
+	 * A similar situation occurs with
+	 * drm_atomic_helper_connector_hdmi_reset_link() that will call
+	 * into our KMS hooks if the scrambling was enabled.
+	 *
+	 * Concurrency isn't an issue at the moment since we don't share
+	 * any state with any of the other frameworks so we can ignore
+	 * the lock for now.
+	 */
+
+	if (status == connector_status_disconnected) {
+		cec_phys_addr_invalidate(vc4_hdmi->cec_adap);
+		return;
+	}
+
+	edid = drm_get_edid(connector, vc4_hdmi->ddc);
+	if (!edid)
+		return;
+
+	cec_s_phys_addr_from_edid(vc4_hdmi->cec_adap, edid);
+	kfree(edid);
+
+	vc4_hdmi_reset_link(connector, ctx);
+}
+
+static int vc4_hdmi_connector_detect_ctx(struct drm_connector *connector,
+					 struct drm_modeset_acquire_ctx *ctx,
+					 bool force)
+{
+	struct vc4_hdmi *vc4_hdmi = connector_to_vc4_hdmi(connector);
+	enum drm_connector_status status = connector_status_disconnected;
+
+	/*
+	 * NOTE: This function should really take vc4_hdmi->mutex, but
+	 * doing so results in reentrancy issues since
+	 * vc4_hdmi_handle_hotplug() can call into other functions that
+	 * would take the mutex while it's held here.
+	 *
+	 * Concurrency isn't an issue at the moment since we don't share
+	 * any state with any of the other frameworks so we can ignore
+	 * the lock for now.
+	 */
 
 	WARN_ON(pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev));
 
 	if (vc4_hdmi->hpd_gpio) {
 		if (gpiod_get_value_cansleep(vc4_hdmi->hpd_gpio))
-			connected = true;
+			status = connector_status_connected;
 	} else {
 		if (vc4_hdmi->variant->hp_detect &&
 		    vc4_hdmi->variant->hp_detect(vc4_hdmi))
-			connected = true;
+			status = connector_status_connected;
 	}
 
-	if (connected) {
-		if (connector->status != connector_status_connected) {
-			struct edid *edid = drm_get_edid(connector, vc4_hdmi->ddc);
-
-			if (edid) {
-				cec_s_phys_addr_from_edid(vc4_hdmi->cec_adap, edid);
-				kfree(edid);
-			}
-		}
-
-		vc4_hdmi_enable_scrambling(&vc4_hdmi->encoder.base);
-		pm_runtime_put(&vc4_hdmi->pdev->dev);
-		mutex_unlock(&vc4_hdmi->mutex);
-		return connector_status_connected;
-	}
-
-	cec_phys_addr_invalidate(vc4_hdmi->cec_adap);
+	vc4_hdmi_handle_hotplug(vc4_hdmi, ctx, status);
 	pm_runtime_put(&vc4_hdmi->pdev->dev);
-	mutex_unlock(&vc4_hdmi->mutex);
-	return connector_status_disconnected;
-}
 
-static void vc4_hdmi_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_unregister(connector);
-	drm_connector_cleanup(connector);
+	return status;
 }
 
 static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
@@ -279,14 +462,21 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
 	int ret = 0;
 	struct edid *edid;
 
-	mutex_lock(&vc4_hdmi->mutex);
+	/*
+	 * NOTE: This function should really take vc4_hdmi->mutex, but
+	 * doing so results in reentrancy issues since
+	 * cec_s_phys_addr_from_edid might call .adap_enable, which
+	 * leads to that funtion being called with our mutex held.
+	 *
+	 * Concurrency isn't an issue at the moment since we don't share
+	 * any state with any of the other frameworks so we can ignore
+	 * the lock for now.
+	 */
 
 	edid = drm_get_edid(connector, vc4_hdmi->ddc);
 	cec_s_phys_addr_from_edid(vc4_hdmi->cec_adap, edid);
-	if (!edid) {
-		ret = -ENODEV;
-		goto out;
-	}
+	if (!edid)
+		return -ENODEV;
 
 	drm_connector_update_edid_property(connector, edid);
 	ret = drm_add_edid_modes(connector, edid);
@@ -294,7 +484,7 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
 
 	if (vc4_hdmi->disable_4kp60) {
 		struct drm_device *drm = connector->dev;
-		struct drm_display_mode *mode;
+		const struct drm_display_mode *mode;
 
 		list_for_each_entry(mode, &connector->probed_modes, head) {
 			if (vc4_hdmi_mode_needs_scrambling(mode, 8, VC4_HDMI_OUTPUT_RGB)) {
@@ -304,9 +494,6 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
 		}
 	}
 
-out:
-	mutex_unlock(&vc4_hdmi->mutex);
-
 	return ret;
 }
 
@@ -378,15 +565,14 @@ vc4_hdmi_connector_duplicate_state(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs vc4_hdmi_connector_funcs = {
-	.detect = vc4_hdmi_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = vc4_hdmi_connector_destroy,
 	.reset = vc4_hdmi_connector_reset,
 	.atomic_duplicate_state = vc4_hdmi_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
 static const struct drm_connector_helper_funcs vc4_hdmi_connector_helper_funcs = {
+	.detect_ctx = vc4_hdmi_connector_detect_ctx,
 	.get_modes = vc4_hdmi_connector_get_modes,
 	.atomic_check = vc4_hdmi_connector_atomic_check,
 };
@@ -398,10 +584,13 @@ static int vc4_hdmi_connector_init(struct drm_device *dev,
 	struct drm_encoder *encoder = &vc4_hdmi->encoder.base;
 	int ret;
 
-	drm_connector_init_with_ddc(dev, connector,
-				    &vc4_hdmi_connector_funcs,
-				    DRM_MODE_CONNECTOR_HDMIA,
-				    vc4_hdmi->ddc);
+	ret = drmm_connector_init(dev, connector,
+				  &vc4_hdmi_connector_funcs,
+				  DRM_MODE_CONNECTOR_HDMIA,
+				  vc4_hdmi->ddc);
+	if (ret)
+		return ret;
+
 	drm_connector_helper_add(connector, &vc4_hdmi_connector_helper_funcs);
 
 	/*
@@ -444,25 +633,34 @@ static int vc4_hdmi_stop_packet(struct drm_encoder *encoder,
 				bool poll)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	u32 packet_id = type - 0x80;
 	unsigned long flags;
+	int ret = 0;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return -ENODEV;
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 	HDMI_WRITE(HDMI_RAM_PACKET_CONFIG,
 		   HDMI_READ(HDMI_RAM_PACKET_CONFIG) & ~BIT(packet_id));
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
 
-	if (!poll)
-		return 0;
+	if (poll) {
+		ret = wait_for(!(HDMI_READ(HDMI_RAM_PACKET_STATUS) &
+				 BIT(packet_id)), 100);
+	}
 
-	return wait_for(!(HDMI_READ(HDMI_RAM_PACKET_STATUS) &
-			  BIT(packet_id)), 100);
+	drm_dev_exit(idx);
+	return ret;
 }
 
 static void vc4_hdmi_write_infoframe(struct drm_encoder *encoder,
 				     union hdmi_infoframe *frame)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	u32 packet_id = frame->any.type - 0x80;
 	const struct vc4_hdmi_register *ram_packet_start =
 		&vc4_hdmi->variant->registers[HDMI_RAM_PACKET_START];
@@ -475,6 +673,10 @@ static void vc4_hdmi_write_infoframe(struct drm_encoder *encoder,
 	unsigned long flags;
 	ssize_t len, i;
 	int ret;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	WARN_ONCE(!(HDMI_READ(HDMI_RAM_PACKET_CONFIG) &
 		    VC4_HDMI_RAM_PACKET_ENABLE),
@@ -482,12 +684,12 @@ static void vc4_hdmi_write_infoframe(struct drm_encoder *encoder,
 
 	len = hdmi_infoframe_pack(frame, buffer, sizeof(buffer));
 	if (len < 0)
-		return;
+		goto out;
 
 	ret = vc4_hdmi_stop_packet(encoder, frame->any.type, true);
 	if (ret) {
 		DRM_ERROR("Failed to wait for infoframe to go idle: %d\n", ret);
-		return;
+		goto out;
 	}
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
@@ -523,6 +725,9 @@ static void vc4_hdmi_write_infoframe(struct drm_encoder *encoder,
 			BIT(packet_id)), 100);
 	if (ret)
 		DRM_ERROR("Failed to wait for infoframe to start: %d\n", ret);
+
+out:
+	drm_dev_exit(idx);
 }
 
 static void vc4_hdmi_avi_infoframe_colorspace(struct hdmi_avi_infoframe *frame,
@@ -649,35 +854,19 @@ static void vc4_hdmi_set_infoframes(struct drm_encoder *encoder)
 	vc4_hdmi_set_hdr_infoframe(encoder);
 }
 
-static bool vc4_hdmi_supports_scrambling(struct drm_encoder *encoder,
-					 struct drm_display_mode *mode)
-{
-	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
-	struct drm_display_info *display = &vc4_hdmi->connector.display_info;
-
-	lockdep_assert_held(&vc4_hdmi->mutex);
-
-	if (!display->is_hdmi)
-		return false;
-
-	if (!display->hdmi.scdc.supported ||
-	    !display->hdmi.scdc.scrambling.supported)
-		return false;
-
-	return true;
-}
-
 #define SCRAMBLING_POLLING_DELAY_MS	1000
 
 static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
-	struct drm_display_mode *mode = &vc4_hdmi->saved_adjusted_mode;
+	struct drm_device *drm = vc4_hdmi->connector.dev;
+	const struct drm_display_mode *mode = &vc4_hdmi->saved_adjusted_mode;
 	unsigned long flags;
+	int idx;
 
 	lockdep_assert_held(&vc4_hdmi->mutex);
 
-	if (!vc4_hdmi_supports_scrambling(encoder, mode))
+	if (!vc4_hdmi_supports_scrambling(encoder))
 		return;
 
 	if (!vc4_hdmi_mode_needs_scrambling(mode,
@@ -685,6 +874,9 @@ static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder)
 					    vc4_hdmi->output_format))
 		return;
 
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
 	drm_scdc_set_high_tmds_clock_ratio(vc4_hdmi->ddc, true);
 	drm_scdc_set_scrambling(vc4_hdmi->ddc, true);
 
@@ -693,6 +885,8 @@ static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder)
 		   VC5_HDMI_SCRAMBLER_CTL_ENABLE);
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
 
+	drm_dev_exit(idx);
+
 	vc4_hdmi->scdc_enabled = true;
 
 	queue_delayed_work(system_wq, &vc4_hdmi->scrambling_work,
@@ -702,7 +896,9 @@ static void vc4_hdmi_enable_scrambling(struct drm_encoder *encoder)
 static void vc4_hdmi_disable_scrambling(struct drm_encoder *encoder)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
+	int idx;
 
 	lockdep_assert_held(&vc4_hdmi->mutex);
 
@@ -714,6 +910,9 @@ static void vc4_hdmi_disable_scrambling(struct drm_encoder *encoder)
 	if (delayed_work_pending(&vc4_hdmi->scrambling_work))
 		cancel_delayed_work_sync(&vc4_hdmi->scrambling_work);
 
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 	HDMI_WRITE(HDMI_SCRAMBLER_CTL, HDMI_READ(HDMI_SCRAMBLER_CTL) &
 		   ~VC5_HDMI_SCRAMBLER_CTL_ENABLE);
@@ -721,6 +920,8 @@ static void vc4_hdmi_disable_scrambling(struct drm_encoder *encoder)
 
 	drm_scdc_set_scrambling(vc4_hdmi->ddc, false);
 	drm_scdc_set_high_tmds_clock_ratio(vc4_hdmi->ddc, false);
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_hdmi_scrambling_wq(struct work_struct *work)
@@ -743,12 +944,17 @@ static void vc4_hdmi_encoder_post_crtc_disable(struct drm_encoder *encoder,
 					       struct drm_atomic_state *state)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
+	int idx;
 
 	mutex_lock(&vc4_hdmi->mutex);
 
 	vc4_hdmi->packet_ram_enabled = false;
 
+	if (!drm_dev_enter(drm, &idx))
+		goto out;
+
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
 	HDMI_WRITE(HDMI_RAM_PACKET_CONFIG, 0);
@@ -766,6 +972,9 @@ static void vc4_hdmi_encoder_post_crtc_disable(struct drm_encoder *encoder,
 
 	vc4_hdmi_disable_scrambling(encoder);
 
+	drm_dev_exit(idx);
+
+out:
 	mutex_unlock(&vc4_hdmi->mutex);
 }
 
@@ -773,11 +982,16 @@ static void vc4_hdmi_encoder_post_crtc_powerdown(struct drm_encoder *encoder,
 						 struct drm_atomic_state *state)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
 	int ret;
+	int idx;
 
 	mutex_lock(&vc4_hdmi->mutex);
 
+	if (!drm_dev_enter(drm, &idx))
+		goto out;
+
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 	HDMI_WRITE(HDMI_VID_CTL,
 		   HDMI_READ(HDMI_VID_CTL) | VC4_HD_VID_CTL_BLANKPIX);
@@ -793,6 +1007,9 @@ static void vc4_hdmi_encoder_post_crtc_powerdown(struct drm_encoder *encoder,
 	if (ret < 0)
 		DRM_ERROR("Failed to release power domain: %d\n", ret);
 
+	drm_dev_exit(idx);
+
+out:
 	mutex_unlock(&vc4_hdmi->mutex);
 }
 
@@ -800,8 +1017,13 @@ static void vc4_hdmi_csc_setup(struct vc4_hdmi *vc4_hdmi,
 			       struct drm_connector_state *state,
 			       const struct drm_display_mode *mode)
 {
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
 	u32 csc_ctl;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
@@ -836,6 +1058,8 @@ static void vc4_hdmi_csc_setup(struct vc4_hdmi *vc4_hdmi,
 	HDMI_WRITE(HDMI_CSC_CTL, csc_ctl);
 
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
+
+	drm_dev_exit(idx);
 }
 
 /*
@@ -920,6 +1144,7 @@ static void vc5_hdmi_csc_setup(struct vc4_hdmi *vc4_hdmi,
 			       struct drm_connector_state *state,
 			       const struct drm_display_mode *mode)
 {
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	struct vc4_hdmi_connector_state *vc4_state =
 		conn_state_to_vc4_hdmi_conn_state(state);
 	unsigned long flags;
@@ -928,6 +1153,10 @@ static void vc5_hdmi_csc_setup(struct vc4_hdmi *vc4_hdmi,
 	u32 csc_chan_ctl = 0;
 	u32 csc_ctl = VC5_MT_CP_CSC_CTL_ENABLE | VC4_SET_FIELD(VC4_HD_CSC_CTL_MODE_CUSTOM,
 							       VC5_MT_CP_CSC_CTL_MODE);
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
@@ -970,12 +1199,15 @@ static void vc5_hdmi_csc_setup(struct vc4_hdmi *vc4_hdmi,
 	HDMI_WRITE(HDMI_CSC_CTL, csc_ctl);
 
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_hdmi_set_timings(struct vc4_hdmi *vc4_hdmi,
 				 struct drm_connector_state *state,
-				 struct drm_display_mode *mode)
+				 const struct drm_display_mode *mode)
 {
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	bool hsync_pos = mode->flags & DRM_MODE_FLAG_PHSYNC;
 	bool vsync_pos = mode->flags & DRM_MODE_FLAG_PVSYNC;
 	bool interlaced = mode->flags & DRM_MODE_FLAG_INTERLACE;
@@ -995,6 +1227,10 @@ static void vc4_hdmi_set_timings(struct vc4_hdmi *vc4_hdmi,
 					VC4_HDMI_VERTB_VBP));
 	unsigned long flags;
 	u32 reg;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
@@ -1027,12 +1263,15 @@ static void vc4_hdmi_set_timings(struct vc4_hdmi *vc4_hdmi,
 	HDMI_WRITE(HDMI_MISC_CONTROL, reg);
 
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
+
+	drm_dev_exit(idx);
 }
 
 static void vc5_hdmi_set_timings(struct vc4_hdmi *vc4_hdmi,
 				 struct drm_connector_state *state,
-				 struct drm_display_mode *mode)
+				 const struct drm_display_mode *mode)
 {
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	const struct vc4_hdmi_connector_state *vc4_state =
 		conn_state_to_vc4_hdmi_conn_state(state);
 	bool hsync_pos = mode->flags & DRM_MODE_FLAG_PHSYNC;
@@ -1056,6 +1295,10 @@ static void vc5_hdmi_set_timings(struct vc4_hdmi *vc4_hdmi,
 	unsigned char gcp;
 	bool gcp_en;
 	u32 reg;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
@@ -1132,13 +1375,20 @@ static void vc5_hdmi_set_timings(struct vc4_hdmi *vc4_hdmi,
 	HDMI_WRITE(HDMI_CLOCK_STOP, 0);
 
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_hdmi_recenter_fifo(struct vc4_hdmi *vc4_hdmi)
 {
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
 	u32 drift;
 	int ret;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
@@ -1167,25 +1417,32 @@ static void vc4_hdmi_recenter_fifo(struct vc4_hdmi *vc4_hdmi)
 		       VC4_HDMI_FIFO_CTL_RECENTER_DONE, 1);
 	WARN_ONCE(ret, "Timeout waiting for "
 		  "VC4_HDMI_FIFO_CTL_RECENTER_DONE");
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
 						struct drm_atomic_state *state)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	struct drm_connector *connector = &vc4_hdmi->connector;
 	struct drm_connector_state *conn_state =
 		drm_atomic_get_new_connector_state(state, connector);
 	struct vc4_hdmi_connector_state *vc4_conn_state =
 		conn_state_to_vc4_hdmi_conn_state(conn_state);
-	struct drm_display_mode *mode = &vc4_hdmi->saved_adjusted_mode;
+	const struct drm_display_mode *mode = &vc4_hdmi->saved_adjusted_mode;
 	unsigned long tmds_char_rate = vc4_conn_state->tmds_char_rate;
 	unsigned long bvb_rate, hsm_rate;
 	unsigned long flags;
 	int ret;
+	int idx;
 
 	mutex_lock(&vc4_hdmi->mutex);
 
+	if (!drm_dev_enter(drm, &idx))
+		goto out;
+
 	/*
 	 * As stated in RPi's vc4 firmware "HDMI state machine (HSM) clock must
 	 * be faster than pixel clock, infinitesimally faster, tested in
@@ -1206,13 +1463,13 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
 	ret = clk_set_min_rate(vc4_hdmi->hsm_clock, hsm_rate);
 	if (ret) {
 		DRM_ERROR("Failed to set HSM clock rate: %d\n", ret);
-		goto out;
+		goto err_dev_exit;
 	}
 
 	ret = pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev);
 	if (ret < 0) {
 		DRM_ERROR("Failed to retain power domain: %d\n", ret);
-		goto out;
+		goto err_dev_exit;
 	}
 
 	ret = clk_set_rate(vc4_hdmi->pixel_clock, tmds_char_rate);
@@ -1264,6 +1521,8 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
 	if (vc4_hdmi->variant->set_timings)
 		vc4_hdmi->variant->set_timings(vc4_hdmi, conn_state, mode);
 
+	drm_dev_exit(idx);
+
 	mutex_unlock(&vc4_hdmi->mutex);
 
 	return;
@@ -1272,6 +1531,8 @@ err_disable_pixel_clock:
 	clk_disable_unprepare(vc4_hdmi->pixel_clock);
 err_put_runtime_pm:
 	pm_runtime_put(&vc4_hdmi->pdev->dev);
+err_dev_exit:
+	drm_dev_exit(idx);
 out:
 	mutex_unlock(&vc4_hdmi->mutex);
 	return;
@@ -1281,14 +1542,19 @@ static void vc4_hdmi_encoder_pre_crtc_enable(struct drm_encoder *encoder,
 					     struct drm_atomic_state *state)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	struct drm_connector *connector = &vc4_hdmi->connector;
-	struct drm_display_mode *mode = &vc4_hdmi->saved_adjusted_mode;
+	const struct drm_display_mode *mode = &vc4_hdmi->saved_adjusted_mode;
 	struct drm_connector_state *conn_state =
 		drm_atomic_get_new_connector_state(state, connector);
 	unsigned long flags;
+	int idx;
 
 	mutex_lock(&vc4_hdmi->mutex);
 
+	if (!drm_dev_enter(drm, &idx))
+		goto out;
+
 	if (vc4_hdmi->variant->csc_setup)
 		vc4_hdmi->variant->csc_setup(vc4_hdmi, conn_state, mode);
 
@@ -1296,6 +1562,9 @@ static void vc4_hdmi_encoder_pre_crtc_enable(struct drm_encoder *encoder,
 	HDMI_WRITE(HDMI_FIFO_CTL, VC4_HDMI_FIFO_CTL_MASTER_SLAVE_N);
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
 
+	drm_dev_exit(idx);
+
+out:
 	mutex_unlock(&vc4_hdmi->mutex);
 }
 
@@ -1303,15 +1572,20 @@ static void vc4_hdmi_encoder_post_crtc_enable(struct drm_encoder *encoder,
 					      struct drm_atomic_state *state)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
-	struct drm_display_mode *mode = &vc4_hdmi->saved_adjusted_mode;
+	struct drm_device *drm = vc4_hdmi->connector.dev;
+	const struct drm_display_mode *mode = &vc4_hdmi->saved_adjusted_mode;
 	struct drm_display_info *display = &vc4_hdmi->connector.display_info;
 	bool hsync_pos = mode->flags & DRM_MODE_FLAG_PHSYNC;
 	bool vsync_pos = mode->flags & DRM_MODE_FLAG_PVSYNC;
 	unsigned long flags;
 	int ret;
+	int idx;
 
 	mutex_lock(&vc4_hdmi->mutex);
 
+	if (!drm_dev_enter(drm, &idx))
+		goto out;
+
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
 	HDMI_WRITE(HDMI_VID_CTL,
@@ -1370,6 +1644,9 @@ static void vc4_hdmi_encoder_post_crtc_enable(struct drm_encoder *encoder,
 	vc4_hdmi_recenter_fifo(vc4_hdmi);
 	vc4_hdmi_enable_scrambling(encoder);
 
+	drm_dev_exit(idx);
+
+out:
 	mutex_unlock(&vc4_hdmi->mutex);
 }
 
@@ -1692,6 +1969,26 @@ static const struct drm_encoder_helper_funcs vc4_hdmi_encoder_helper_funcs = {
 	.mode_valid = vc4_hdmi_encoder_mode_valid,
 };
 
+static int vc4_hdmi_late_register(struct drm_encoder *encoder)
+{
+	struct drm_device *drm = encoder->dev;
+	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
+	const struct vc4_hdmi_variant *variant = vc4_hdmi->variant;
+	int ret;
+
+	ret = vc4_debugfs_add_file(drm->primary, variant->debugfs_name,
+				   vc4_hdmi_debugfs_regs,
+				   vc4_hdmi);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct drm_encoder_funcs vc4_hdmi_encoder_funcs = {
+	.late_register = vc4_hdmi_late_register,
+};
+
 static u32 vc4_hdmi_channel_map(struct vc4_hdmi *vc4_hdmi, u32 channel_mask)
 {
 	int i;
@@ -1718,13 +2015,20 @@ static u32 vc5_hdmi_channel_map(struct vc4_hdmi *vc4_hdmi, u32 channel_mask)
 
 static bool vc5_hdmi_hp_detect(struct vc4_hdmi *vc4_hdmi)
 {
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
 	u32 hotplug;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return false;
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 	hotplug = HDMI_READ(HDMI_HOTPLUG);
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
 
+	drm_dev_exit(idx);
+
 	return !!(hotplug & VC4_HDMI_HOTPLUG_CONNECTED);
 }
 
@@ -1732,10 +2036,16 @@ static bool vc5_hdmi_hp_detect(struct vc4_hdmi *vc4_hdmi)
 static void vc4_hdmi_audio_set_mai_clock(struct vc4_hdmi *vc4_hdmi,
 					 unsigned int samplerate)
 {
-	u32 hsm_clock = clk_get_rate(vc4_hdmi->audio_clock);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
+	u32 hsm_clock;
 	unsigned long flags;
 	unsigned long n, m;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
+	hsm_clock = clk_get_rate(vc4_hdmi->audio_clock);
 	rational_best_approximation(hsm_clock, samplerate,
 				    VC4_HD_MAI_SMP_N_MASK >>
 				    VC4_HD_MAI_SMP_N_SHIFT,
@@ -1748,6 +2058,8 @@ static void vc4_hdmi_audio_set_mai_clock(struct vc4_hdmi *vc4_hdmi,
 		   VC4_SET_FIELD(n, VC4_HD_MAI_SMP_N) |
 		   VC4_SET_FIELD(m - 1, VC4_HD_MAI_SMP_M));
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_hdmi_set_n_cts(struct vc4_hdmi *vc4_hdmi, unsigned int samplerate)
@@ -1803,13 +2115,21 @@ static bool vc4_hdmi_audio_can_stream(struct vc4_hdmi *vc4_hdmi)
 static int vc4_hdmi_audio_startup(struct device *dev, void *data)
 {
 	struct vc4_hdmi *vc4_hdmi = dev_get_drvdata(dev);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
+	int ret = 0;
+	int idx;
 
 	mutex_lock(&vc4_hdmi->mutex);
 
+	if (!drm_dev_enter(drm, &idx)) {
+		ret = -ENODEV;
+		goto out;
+	}
+
 	if (!vc4_hdmi_audio_can_stream(vc4_hdmi)) {
-		mutex_unlock(&vc4_hdmi->mutex);
-		return -ENODEV;
+		ret = -ENODEV;
+		goto out_dev_exit;
 	}
 
 	vc4_hdmi->audio.streaming = true;
@@ -1826,9 +2146,12 @@ static int vc4_hdmi_audio_startup(struct device *dev, void *data)
 	if (vc4_hdmi->variant->phy_rng_enable)
 		vc4_hdmi->variant->phy_rng_enable(vc4_hdmi);
 
+out_dev_exit:
+	drm_dev_exit(idx);
+out:
 	mutex_unlock(&vc4_hdmi->mutex);
 
-	return 0;
+	return ret;
 }
 
 static void vc4_hdmi_audio_reset(struct vc4_hdmi *vc4_hdmi)
@@ -1857,10 +2180,15 @@ static void vc4_hdmi_audio_reset(struct vc4_hdmi *vc4_hdmi)
 static void vc4_hdmi_audio_shutdown(struct device *dev, void *data)
 {
 	struct vc4_hdmi *vc4_hdmi = dev_get_drvdata(dev);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
+	int idx;
 
 	mutex_lock(&vc4_hdmi->mutex);
 
+	if (!drm_dev_enter(drm, &idx))
+		goto out;
+
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
 	HDMI_WRITE(HDMI_MAI_CTL,
@@ -1876,6 +2204,9 @@ static void vc4_hdmi_audio_shutdown(struct device *dev, void *data)
 	vc4_hdmi->audio.streaming = false;
 	vc4_hdmi_audio_reset(vc4_hdmi);
 
+	drm_dev_exit(idx);
+
+out:
 	mutex_unlock(&vc4_hdmi->mutex);
 }
 
@@ -1923,6 +2254,7 @@ static int vc4_hdmi_audio_prepare(struct device *dev, void *data,
 				  struct hdmi_codec_params *params)
 {
 	struct vc4_hdmi *vc4_hdmi = dev_get_drvdata(dev);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	struct drm_encoder *encoder = &vc4_hdmi->encoder.base;
 	unsigned int sample_rate = params->sample_rate;
 	unsigned int channels = params->channels;
@@ -1931,15 +2263,22 @@ static int vc4_hdmi_audio_prepare(struct device *dev, void *data,
 	u32 channel_map;
 	u32 mai_audio_format;
 	u32 mai_sample_rate;
+	int ret = 0;
+	int idx;
 
 	dev_dbg(dev, "%s: %u Hz, %d bit, %d channels\n", __func__,
 		sample_rate, params->sample_width, channels);
 
 	mutex_lock(&vc4_hdmi->mutex);
 
+	if (!drm_dev_enter(drm, &idx)) {
+		ret = -ENODEV;
+		goto out;
+	}
+
 	if (!vc4_hdmi_audio_can_stream(vc4_hdmi)) {
-		mutex_unlock(&vc4_hdmi->mutex);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out_dev_exit;
 	}
 
 	vc4_hdmi_audio_set_mai_clock(vc4_hdmi, sample_rate);
@@ -1996,9 +2335,12 @@ static int vc4_hdmi_audio_prepare(struct device *dev, void *data,
 	memcpy(&vc4_hdmi->audio.infoframe, &params->cea, sizeof(params->cea));
 	vc4_hdmi_set_audio_infoframe(encoder);
 
+out_dev_exit:
+	drm_dev_exit(idx);
+out:
 	mutex_unlock(&vc4_hdmi->mutex);
 
-	return 0;
+	return ret;
 }
 
 static const struct snd_soc_component_driver vc4_hdmi_audio_cpu_dai_comp = {
@@ -2061,6 +2403,14 @@ static struct hdmi_codec_pdata vc4_hdmi_codec_pdata = {
 	.i2s = 1,
 };
 
+static void vc4_hdmi_audio_codec_release(void *ptr)
+{
+	struct vc4_hdmi *vc4_hdmi = ptr;
+
+	platform_device_unregister(vc4_hdmi->audio.codec_pdev);
+	vc4_hdmi->audio.codec_pdev = NULL;
+}
+
 static int vc4_hdmi_audio_init(struct vc4_hdmi *vc4_hdmi)
 {
 	const struct vc4_hdmi_register *mai_data =
@@ -2073,6 +2423,26 @@ static int vc4_hdmi_audio_init(struct vc4_hdmi *vc4_hdmi)
 	int index, len;
 	int ret;
 
+	/*
+	 * ASoC makes it a bit hard to retrieve a pointer to the
+	 * vc4_hdmi structure. Registering the card will overwrite our
+	 * device drvdata with a pointer to the snd_soc_card structure,
+	 * which can then be used to retrieve whatever drvdata we want
+	 * to associate.
+	 *
+	 * However, that doesn't fly in the case where we wouldn't
+	 * register an ASoC card (because of an old DT that is missing
+	 * the dmas properties for example), then the card isn't
+	 * registered and the device drvdata wouldn't be set.
+	 *
+	 * We can deal with both cases by making sure a snd_soc_card
+	 * pointer and a vc4_hdmi structure are pointing to the same
+	 * memory address, so we can treat them indistinctly without any
+	 * issue.
+	 */
+	BUILD_BUG_ON(offsetof(struct vc4_hdmi_audio, card) != 0);
+	BUILD_BUG_ON(offsetof(struct vc4_hdmi, audio) != 0);
+
 	if (!of_find_property(dev->of_node, "dmas", &len) || !len) {
 		dev_warn(dev,
 			 "'dmas' DT property is missing or empty, no HDMI audio\n");
@@ -2102,6 +2472,30 @@ static int vc4_hdmi_audio_init(struct vc4_hdmi *vc4_hdmi)
 	vc4_hdmi->audio.dma_data.addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
 	vc4_hdmi->audio.dma_data.maxburst = 2;
 
+	/*
+	 * NOTE: Strictly speaking, we should probably use a DRM-managed
+	 * registration there to avoid removing all the audio components
+	 * by the time the driver doesn't have any user anymore.
+	 *
+	 * However, the ASoC core uses a number of devm_kzalloc calls
+	 * when registering, even when using non-device-managed
+	 * functions (such as in snd_soc_register_component()).
+	 *
+	 * If we call snd_soc_unregister_component() in a DRM-managed
+	 * action, the device-managed actions have already been executed
+	 * and thus we would access memory that has been freed.
+	 *
+	 * Using device-managed hooks here probably leaves us open to a
+	 * bunch of issues if userspace still has a handle on the ALSA
+	 * device when the device is removed. However, this is mitigated
+	 * by the use of drm_dev_enter()/drm_dev_exit() in the audio
+	 * path to prevent the access to the device resources if it
+	 * isn't there anymore.
+	 *
+	 * Then, the vc4_hdmi structure is DRM-managed and thus only
+	 * freed whenever the last user has closed the DRM device file.
+	 * It should thus outlive ALSA in most situations.
+	 */
 	ret = devm_snd_dmaengine_pcm_register(dev, &pcm_conf, 0);
 	if (ret) {
 		dev_err(dev, "Could not register PCM component: %d\n", ret);
@@ -2125,6 +2519,10 @@ static int vc4_hdmi_audio_init(struct vc4_hdmi *vc4_hdmi)
 	}
 	vc4_hdmi->audio.codec_pdev = codec_pdev;
 
+	ret = devm_add_action_or_reset(dev, vc4_hdmi_audio_codec_release, vc4_hdmi);
+	if (ret)
+		return ret;
+
 	dai_link->cpus		= &vc4_hdmi->audio.cpu;
 	dai_link->codecs	= &vc4_hdmi->audio.codec;
 	dai_link->platforms	= &vc4_hdmi->audio.platform;
@@ -2163,12 +2561,6 @@ static int vc4_hdmi_audio_init(struct vc4_hdmi *vc4_hdmi)
 
 }
 
-static void vc4_hdmi_audio_exit(struct vc4_hdmi *vc4_hdmi)
-{
-	platform_device_unregister(vc4_hdmi->audio.codec_pdev);
-	vc4_hdmi->audio.codec_pdev = NULL;
-}
-
 static irqreturn_t vc4_hdmi_hpd_irq_thread(int irq, void *priv)
 {
 	struct vc4_hdmi *vc4_hdmi = priv;
@@ -2191,21 +2583,19 @@ static int vc4_hdmi_hotplug_init(struct vc4_hdmi *vc4_hdmi)
 		unsigned int hpd_con = platform_get_irq_byname(pdev, "hpd-connected");
 		unsigned int hpd_rm = platform_get_irq_byname(pdev, "hpd-removed");
 
-		ret = request_threaded_irq(hpd_con,
-					   NULL,
-					   vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT,
-					   "vc4 hdmi hpd connected", vc4_hdmi);
+		ret = devm_request_threaded_irq(&pdev->dev, hpd_con,
+						NULL,
+						vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT,
+						"vc4 hdmi hpd connected", vc4_hdmi);
 		if (ret)
 			return ret;
 
-		ret = request_threaded_irq(hpd_rm,
-					   NULL,
-					   vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT,
-					   "vc4 hdmi hpd disconnected", vc4_hdmi);
-		if (ret) {
-			free_irq(hpd_con, vc4_hdmi);
+		ret = devm_request_threaded_irq(&pdev->dev, hpd_rm,
+						NULL,
+						vc4_hdmi_hpd_irq_thread, IRQF_ONESHOT,
+						"vc4 hdmi hpd disconnected", vc4_hdmi);
+		if (ret)
 			return ret;
-		}
 
 		connector->polled = DRM_CONNECTOR_POLL_HPD;
 	}
@@ -2213,16 +2603,6 @@ static int vc4_hdmi_hotplug_init(struct vc4_hdmi *vc4_hdmi)
 	return 0;
 }
 
-static void vc4_hdmi_hotplug_exit(struct vc4_hdmi *vc4_hdmi)
-{
-	struct platform_device *pdev = vc4_hdmi->pdev;
-
-	if (vc4_hdmi->variant->external_irq_controller) {
-		free_irq(platform_get_irq_byname(pdev, "hpd-connected"), vc4_hdmi);
-		free_irq(platform_get_irq_byname(pdev, "hpd-removed"), vc4_hdmi);
-	}
-}
-
 #ifdef CONFIG_DRM_VC4_HDMI_CEC
 static irqreturn_t vc4_cec_irq_handler_rx_thread(int irq, void *priv)
 {
@@ -2296,6 +2676,17 @@ static irqreturn_t vc4_cec_irq_handler_tx_bare_locked(struct vc4_hdmi *vc4_hdmi)
 {
 	u32 cntrl1;
 
+	/*
+	 * We don't need to protect the register access using
+	 * drm_dev_enter() there because the interrupt handler lifetime
+	 * is tied to the device itself, and not to the DRM device.
+	 *
+	 * So when the device will be gone, one of the first thing we
+	 * will be doing will be to unregister the interrupt handler,
+	 * and then unregister the DRM device. drm_dev_enter() would
+	 * thus always succeed if we are here.
+	 */
+
 	lockdep_assert_held(&vc4_hdmi->hw_lock);
 
 	cntrl1 = HDMI_READ(HDMI_CEC_CNTRL_1);
@@ -2324,6 +2715,17 @@ static irqreturn_t vc4_cec_irq_handler_rx_bare_locked(struct vc4_hdmi *vc4_hdmi)
 
 	lockdep_assert_held(&vc4_hdmi->hw_lock);
 
+	/*
+	 * We don't need to protect the register access using
+	 * drm_dev_enter() there because the interrupt handler lifetime
+	 * is tied to the device itself, and not to the DRM device.
+	 *
+	 * So when the device will be gone, one of the first thing we
+	 * will be doing will be to unregister the interrupt handler,
+	 * and then unregister the DRM device. drm_dev_enter() would
+	 * thus always succeed if we are here.
+	 */
+
 	vc4_hdmi->cec_rx_msg.len = 0;
 	cntrl1 = HDMI_READ(HDMI_CEC_CNTRL_1);
 	vc4_cec_read_msg(vc4_hdmi, cntrl1);
@@ -2355,6 +2757,17 @@ static irqreturn_t vc4_cec_irq_handler(int irq, void *priv)
 	irqreturn_t ret;
 	u32 cntrl5;
 
+	/*
+	 * We don't need to protect the register access using
+	 * drm_dev_enter() there because the interrupt handler lifetime
+	 * is tied to the device itself, and not to the DRM device.
+	 *
+	 * So when the device will be gone, one of the first thing we
+	 * will be doing will be to unregister the interrupt handler,
+	 * and then unregister the DRM device. drm_dev_enter() would
+	 * thus always succeed if we are here.
+	 */
+
 	if (!(stat & VC4_HDMI_CPU_CEC))
 		return IRQ_NONE;
 
@@ -2375,26 +2788,29 @@ static irqreturn_t vc4_cec_irq_handler(int irq, void *priv)
 static int vc4_hdmi_cec_enable(struct cec_adapter *adap)
 {
 	struct vc4_hdmi *vc4_hdmi = cec_get_drvdata(adap);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	/* clock period in microseconds */
 	const u32 usecs = 1000000 / CEC_CLOCK_FREQ;
 	unsigned long flags;
 	u32 val;
 	int ret;
+	int idx;
 
-	/*
-	 * NOTE: This function should really take vc4_hdmi->mutex, but doing so
-	 * results in a reentrancy since cec_s_phys_addr_from_edid() called in
-	 * .detect or .get_modes might call .adap_enable, which leads to this
-	 * function being called with that mutex held.
-	 *
-	 * Concurrency is not an issue for the moment since we don't share any
-	 * state with KMS, so we can ignore the lock for now, but we need to
-	 * keep it in mind if we were to change that assumption.
-	 */
+	if (!drm_dev_enter(drm, &idx))
+		/*
+		 * We can't return an error code, because the CEC
+		 * framework will emit WARN_ON messages at unbind
+		 * otherwise.
+		 */
+		return 0;
 
 	ret = pm_runtime_resume_and_get(&vc4_hdmi->pdev->dev);
-	if (ret)
+	if (ret) {
+		drm_dev_exit(idx);
 		return ret;
+	}
+
+	mutex_lock(&vc4_hdmi->mutex);
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
@@ -2430,24 +2846,28 @@ static int vc4_hdmi_cec_enable(struct cec_adapter *adap)
 
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
 
+	mutex_unlock(&vc4_hdmi->mutex);
+	drm_dev_exit(idx);
+
 	return 0;
 }
 
 static int vc4_hdmi_cec_disable(struct cec_adapter *adap)
 {
 	struct vc4_hdmi *vc4_hdmi = cec_get_drvdata(adap);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
+	int idx;
 
-	/*
-	 * NOTE: This function should really take vc4_hdmi->mutex, but doing so
-	 * results in a reentrancy since cec_s_phys_addr_from_edid() called in
-	 * .detect or .get_modes might call .adap_enable, which leads to this
-	 * function being called with that mutex held.
-	 *
-	 * Concurrency is not an issue for the moment since we don't share any
-	 * state with KMS, so we can ignore the lock for now, but we need to
-	 * keep it in mind if we were to change that assumption.
-	 */
+	if (!drm_dev_enter(drm, &idx))
+		/*
+		 * We can't return an error code, because the CEC
+		 * framework will emit WARN_ON messages at unbind
+		 * otherwise.
+		 */
+		return 0;
+
+	mutex_lock(&vc4_hdmi->mutex);
 
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
@@ -2459,8 +2879,12 @@ static int vc4_hdmi_cec_disable(struct cec_adapter *adap)
 
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
 
+	mutex_unlock(&vc4_hdmi->mutex);
+
 	pm_runtime_put(&vc4_hdmi->pdev->dev);
 
+	drm_dev_exit(idx);
+
 	return 0;
 }
 
@@ -2475,24 +2899,27 @@ static int vc4_hdmi_cec_adap_enable(struct cec_adapter *adap, bool enable)
 static int vc4_hdmi_cec_adap_log_addr(struct cec_adapter *adap, u8 log_addr)
 {
 	struct vc4_hdmi *vc4_hdmi = cec_get_drvdata(adap);
+	struct drm_device *drm = vc4_hdmi->connector.dev;
 	unsigned long flags;
+	int idx;
 
-	/*
-	 * NOTE: This function should really take vc4_hdmi->mutex, but doing so
-	 * results in a reentrancy since cec_s_phys_addr_from_edid() called in
-	 * .detect or .get_modes might call .adap_enable, which leads to this
-	 * function being called with that mutex held.
-	 *
-	 * Concurrency is not an issue for the moment since we don't share any
-	 * state with KMS, so we can ignore the lock for now, but we need to
-	 * keep it in mind if we were to change that assumption.
-	 */
+	if (!drm_dev_enter(drm, &idx))
+		/*
+		 * We can't return an error code, because the CEC
+		 * framework will emit WARN_ON messages at unbind
+		 * otherwise.
+		 */
+		return 0;
 
+	mutex_lock(&vc4_hdmi->mutex);
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 	HDMI_WRITE(HDMI_CEC_CNTRL_1,
 		   (HDMI_READ(HDMI_CEC_CNTRL_1) & ~VC4_HDMI_CEC_ADDR_MASK) |
 		   (log_addr & 0xf) << VC4_HDMI_CEC_ADDR_SHIFT);
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
+	mutex_unlock(&vc4_hdmi->mutex);
+
+	drm_dev_exit(idx);
 
 	return 0;
 }
@@ -2505,23 +2932,19 @@ static int vc4_hdmi_cec_adap_transmit(struct cec_adapter *adap, u8 attempts,
 	unsigned long flags;
 	u32 val;
 	unsigned int i;
+	int idx;
 
-	/*
-	 * NOTE: This function should really take vc4_hdmi->mutex, but doing so
-	 * results in a reentrancy since cec_s_phys_addr_from_edid() called in
-	 * .detect or .get_modes might call .adap_enable, which leads to this
-	 * function being called with that mutex held.
-	 *
-	 * Concurrency is not an issue for the moment since we don't share any
-	 * state with KMS, so we can ignore the lock for now, but we need to
-	 * keep it in mind if we were to change that assumption.
-	 */
+	if (!drm_dev_enter(dev, &idx))
+		return -ENODEV;
 
 	if (msg->len > 16) {
 		drm_err(dev, "Attempting to transmit too much data (%d)\n", msg->len);
+		drm_dev_exit(idx);
 		return -ENOMEM;
 	}
 
+	mutex_lock(&vc4_hdmi->mutex);
+
 	spin_lock_irqsave(&vc4_hdmi->hw_lock, flags);
 
 	for (i = 0; i < msg->len; i += 4)
@@ -2541,6 +2964,8 @@ static int vc4_hdmi_cec_adap_transmit(struct cec_adapter *adap, u8 attempts,
 	HDMI_WRITE(HDMI_CEC_CNTRL_1, val);
 
 	spin_unlock_irqrestore(&vc4_hdmi->hw_lock, flags);
+	mutex_unlock(&vc4_hdmi->mutex);
+	drm_dev_exit(idx);
 
 	return 0;
 }
@@ -2551,6 +2976,14 @@ static const struct cec_adap_ops vc4_hdmi_cec_adap_ops = {
 	.adap_transmit = vc4_hdmi_cec_adap_transmit,
 };
 
+static void vc4_hdmi_cec_release(void *ptr)
+{
+	struct vc4_hdmi *vc4_hdmi = ptr;
+
+	cec_unregister_adapter(vc4_hdmi->cec_adap);
+	vc4_hdmi->cec_adap = NULL;
+}
+
 static int vc4_hdmi_cec_init(struct vc4_hdmi *vc4_hdmi)
 {
 	struct cec_connector_info conn_info;
@@ -2575,73 +3008,82 @@ static int vc4_hdmi_cec_init(struct vc4_hdmi *vc4_hdmi)
 	cec_s_conn_info(vc4_hdmi->cec_adap, &conn_info);
 
 	if (vc4_hdmi->variant->external_irq_controller) {
-		ret = request_threaded_irq(platform_get_irq_byname(pdev, "cec-rx"),
-					   vc4_cec_irq_handler_rx_bare,
-					   vc4_cec_irq_handler_rx_thread, 0,
-					   "vc4 hdmi cec rx", vc4_hdmi);
+		ret = devm_request_threaded_irq(dev, platform_get_irq_byname(pdev, "cec-rx"),
+						vc4_cec_irq_handler_rx_bare,
+						vc4_cec_irq_handler_rx_thread, 0,
+						"vc4 hdmi cec rx", vc4_hdmi);
 		if (ret)
 			goto err_delete_cec_adap;
 
-		ret = request_threaded_irq(platform_get_irq_byname(pdev, "cec-tx"),
-					   vc4_cec_irq_handler_tx_bare,
-					   vc4_cec_irq_handler_tx_thread, 0,
-					   "vc4 hdmi cec tx", vc4_hdmi);
+		ret = devm_request_threaded_irq(dev, platform_get_irq_byname(pdev, "cec-tx"),
+						vc4_cec_irq_handler_tx_bare,
+						vc4_cec_irq_handler_tx_thread, 0,
+						"vc4 hdmi cec tx", vc4_hdmi);
 		if (ret)
-			goto err_remove_cec_rx_handler;
+			goto err_delete_cec_adap;
 	} else {
-		ret = request_threaded_irq(platform_get_irq(pdev, 0),
-					   vc4_cec_irq_handler,
-					   vc4_cec_irq_handler_thread, 0,
-					   "vc4 hdmi cec", vc4_hdmi);
+		ret = devm_request_threaded_irq(dev, platform_get_irq(pdev, 0),
+						vc4_cec_irq_handler,
+						vc4_cec_irq_handler_thread, 0,
+						"vc4 hdmi cec", vc4_hdmi);
 		if (ret)
 			goto err_delete_cec_adap;
 	}
 
 	ret = cec_register_adapter(vc4_hdmi->cec_adap, &pdev->dev);
 	if (ret < 0)
-		goto err_remove_handlers;
+		goto err_delete_cec_adap;
 
-	return 0;
-
-err_remove_handlers:
-	if (vc4_hdmi->variant->external_irq_controller)
-		free_irq(platform_get_irq_byname(pdev, "cec-tx"), vc4_hdmi);
-	else
-		free_irq(platform_get_irq(pdev, 0), vc4_hdmi);
+	/*
+	 * NOTE: Strictly speaking, we should probably use a DRM-managed
+	 * registration there to avoid removing the CEC adapter by the
+	 * time the DRM driver doesn't have any user anymore.
+	 *
+	 * However, the CEC framework already cleans up the CEC adapter
+	 * only when the last user has closed its file descriptor, so we
+	 * don't need to handle it in DRM.
+	 *
+	 * By the time the device-managed hook is executed, we will give
+	 * up our reference to the CEC adapter and therefore don't
+	 * really care when it's actually freed.
+	 *
+	 * There's still a problematic sequence: if we unregister our
+	 * CEC adapter, but the userspace keeps a handle on the CEC
+	 * adapter but not the DRM device for some reason. In such a
+	 * case, our vc4_hdmi structure will be freed, but the
+	 * cec_adapter structure will have a dangling pointer to what
+	 * used to be our HDMI controller. If we get a CEC call at that
+	 * moment, we could end up with a use-after-free. Fortunately,
+	 * the CEC framework already handles this too, by calling
+	 * cec_is_registered() in cec_ioctl() and cec_poll().
+	 */
+	ret = devm_add_action_or_reset(dev, vc4_hdmi_cec_release, vc4_hdmi);
+	if (ret)
+		return ret;
 
-err_remove_cec_rx_handler:
-	if (vc4_hdmi->variant->external_irq_controller)
-		free_irq(platform_get_irq_byname(pdev, "cec-rx"), vc4_hdmi);
+	return 0;
 
 err_delete_cec_adap:
 	cec_delete_adapter(vc4_hdmi->cec_adap);
 
 	return ret;
 }
-
-static void vc4_hdmi_cec_exit(struct vc4_hdmi *vc4_hdmi)
-{
-	struct platform_device *pdev = vc4_hdmi->pdev;
-
-	if (vc4_hdmi->variant->external_irq_controller) {
-		free_irq(platform_get_irq_byname(pdev, "cec-rx"), vc4_hdmi);
-		free_irq(platform_get_irq_byname(pdev, "cec-tx"), vc4_hdmi);
-	} else {
-		free_irq(platform_get_irq(pdev, 0), vc4_hdmi);
-	}
-
-	cec_unregister_adapter(vc4_hdmi->cec_adap);
-}
 #else
 static int vc4_hdmi_cec_init(struct vc4_hdmi *vc4_hdmi)
 {
 	return 0;
 }
-
-static void vc4_hdmi_cec_exit(struct vc4_hdmi *vc4_hdmi) {};
 #endif
 
-static int vc4_hdmi_build_regset(struct vc4_hdmi *vc4_hdmi,
+static void vc4_hdmi_free_regset(struct drm_device *drm, void *ptr)
+{
+	struct debugfs_reg32 *regs = ptr;
+
+	kfree(regs);
+}
+
+static int vc4_hdmi_build_regset(struct drm_device *drm,
+				 struct vc4_hdmi *vc4_hdmi,
 				 struct debugfs_regset32 *regset,
 				 enum vc4_hdmi_regs reg)
 {
@@ -2649,6 +3091,7 @@ static int vc4_hdmi_build_regset(struct vc4_hdmi *vc4_hdmi,
 	struct debugfs_reg32 *regs, *new_regs;
 	unsigned int count = 0;
 	unsigned int i;
+	int ret;
 
 	regs = kcalloc(variant->num_registers, sizeof(*regs),
 		       GFP_KERNEL);
@@ -2674,10 +3117,15 @@ static int vc4_hdmi_build_regset(struct vc4_hdmi *vc4_hdmi,
 	regset->regs = new_regs;
 	regset->nregs = count;
 
+	ret = drmm_add_action_or_reset(drm, vc4_hdmi_free_regset, new_regs);
+	if (ret)
+		return ret;
+
 	return 0;
 }
 
-static int vc4_hdmi_init_resources(struct vc4_hdmi *vc4_hdmi)
+static int vc4_hdmi_init_resources(struct drm_device *drm,
+				   struct vc4_hdmi *vc4_hdmi)
 {
 	struct platform_device *pdev = vc4_hdmi->pdev;
 	struct device *dev = &pdev->dev;
@@ -2691,11 +3139,11 @@ static int vc4_hdmi_init_resources(struct vc4_hdmi *vc4_hdmi)
 	if (IS_ERR(vc4_hdmi->hd_regs))
 		return PTR_ERR(vc4_hdmi->hd_regs);
 
-	ret = vc4_hdmi_build_regset(vc4_hdmi, &vc4_hdmi->hd_regset, VC4_HD);
+	ret = vc4_hdmi_build_regset(drm, vc4_hdmi, &vc4_hdmi->hd_regset, VC4_HD);
 	if (ret)
 		return ret;
 
-	ret = vc4_hdmi_build_regset(vc4_hdmi, &vc4_hdmi->hdmi_regset, VC4_HDMI);
+	ret = vc4_hdmi_build_regset(drm, vc4_hdmi, &vc4_hdmi->hdmi_regset, VC4_HDMI);
 	if (ret)
 		return ret;
 
@@ -2718,7 +3166,8 @@ static int vc4_hdmi_init_resources(struct vc4_hdmi *vc4_hdmi)
 	return 0;
 }
 
-static int vc5_hdmi_init_resources(struct vc4_hdmi *vc4_hdmi)
+static int vc5_hdmi_init_resources(struct drm_device *drm,
+				   struct vc4_hdmi *vc4_hdmi)
 {
 	struct platform_device *pdev = vc4_hdmi->pdev;
 	struct device *dev = &pdev->dev;
@@ -2820,35 +3269,35 @@ static int vc5_hdmi_init_resources(struct vc4_hdmi *vc4_hdmi)
 		return PTR_ERR(vc4_hdmi->reset);
 	}
 
-	ret = vc4_hdmi_build_regset(vc4_hdmi, &vc4_hdmi->hdmi_regset, VC4_HDMI);
+	ret = vc4_hdmi_build_regset(drm, vc4_hdmi, &vc4_hdmi->hdmi_regset, VC4_HDMI);
 	if (ret)
 		return ret;
 
-	ret = vc4_hdmi_build_regset(vc4_hdmi, &vc4_hdmi->hd_regset, VC4_HD);
+	ret = vc4_hdmi_build_regset(drm, vc4_hdmi, &vc4_hdmi->hd_regset, VC4_HD);
 	if (ret)
 		return ret;
 
-	ret = vc4_hdmi_build_regset(vc4_hdmi, &vc4_hdmi->cec_regset, VC5_CEC);
+	ret = vc4_hdmi_build_regset(drm, vc4_hdmi, &vc4_hdmi->cec_regset, VC5_CEC);
 	if (ret)
 		return ret;
 
-	ret = vc4_hdmi_build_regset(vc4_hdmi, &vc4_hdmi->csc_regset, VC5_CSC);
+	ret = vc4_hdmi_build_regset(drm, vc4_hdmi, &vc4_hdmi->csc_regset, VC5_CSC);
 	if (ret)
 		return ret;
 
-	ret = vc4_hdmi_build_regset(vc4_hdmi, &vc4_hdmi->dvp_regset, VC5_DVP);
+	ret = vc4_hdmi_build_regset(drm, vc4_hdmi, &vc4_hdmi->dvp_regset, VC5_DVP);
 	if (ret)
 		return ret;
 
-	ret = vc4_hdmi_build_regset(vc4_hdmi, &vc4_hdmi->phy_regset, VC5_PHY);
+	ret = vc4_hdmi_build_regset(drm, vc4_hdmi, &vc4_hdmi->phy_regset, VC5_PHY);
 	if (ret)
 		return ret;
 
-	ret = vc4_hdmi_build_regset(vc4_hdmi, &vc4_hdmi->ram_regset, VC5_RAM);
+	ret = vc4_hdmi_build_regset(drm, vc4_hdmi, &vc4_hdmi->ram_regset, VC5_RAM);
 	if (ret)
 		return ret;
 
-	ret = vc4_hdmi_build_regset(vc4_hdmi, &vc4_hdmi->rm_regset, VC5_RM);
+	ret = vc4_hdmi_build_regset(drm, vc4_hdmi, &vc4_hdmi->rm_regset, VC5_RM);
 	if (ret)
 		return ret;
 
@@ -2898,6 +3347,13 @@ static int vc4_hdmi_runtime_resume(struct device *dev)
 	return 0;
 }
 
+static void vc4_hdmi_put_ddc_device(void *ptr)
+{
+	struct vc4_hdmi *vc4_hdmi = ptr;
+
+	put_device(&vc4_hdmi->ddc->dev);
+}
+
 static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 {
 	const struct vc4_hdmi_variant *variant = of_device_get_match_data(dev);
@@ -2908,10 +3364,14 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 	struct device_node *ddc_node;
 	int ret;
 
-	vc4_hdmi = devm_kzalloc(dev, sizeof(*vc4_hdmi), GFP_KERNEL);
+	vc4_hdmi = drmm_kzalloc(drm, sizeof(*vc4_hdmi), GFP_KERNEL);
 	if (!vc4_hdmi)
 		return -ENOMEM;
-	mutex_init(&vc4_hdmi->mutex);
+
+	ret = drmm_mutex_init(drm, &vc4_hdmi->mutex);
+	if (ret)
+		return ret;
+
 	spin_lock_init(&vc4_hdmi->hw_lock);
 	INIT_DELAYED_WORK(&vc4_hdmi->scrambling_work, vc4_hdmi_scrambling_wq);
 
@@ -2935,7 +3395,7 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 	if (variant->max_pixel_clock > HDMI_14_MAX_TMDS_CLK)
 		vc4_hdmi->scdc_enabled = true;
 
-	ret = variant->init_resources(vc4_hdmi);
+	ret = variant->init_resources(drm, vc4_hdmi);
 	if (ret)
 		return ret;
 
@@ -2952,13 +3412,16 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 		return -EPROBE_DEFER;
 	}
 
+	ret = devm_add_action_or_reset(dev, vc4_hdmi_put_ddc_device, vc4_hdmi);
+	if (ret)
+		return ret;
+
 	/* Only use the GPIO HPD pin if present in the DT, otherwise
 	 * we'll use the HDMI core's register.
 	 */
 	vc4_hdmi->hpd_gpio = devm_gpiod_get_optional(dev, "hpd", GPIOD_IN);
 	if (IS_ERR(vc4_hdmi->hpd_gpio)) {
-		ret = PTR_ERR(vc4_hdmi->hpd_gpio);
-		goto err_put_ddc;
+		return PTR_ERR(vc4_hdmi->hpd_gpio);
 	}
 
 	vc4_hdmi->disable_wifi_frequencies =
@@ -2972,7 +3435,9 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 			vc4_hdmi->disable_4kp60 = true;
 	}
 
-	pm_runtime_enable(dev);
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return ret;
 
 	/*
 	 *  We need to have the device powered up at this point to call
@@ -2980,7 +3445,7 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 	 */
 	ret = pm_runtime_resume_and_get(dev);
 	if (ret)
-		goto err_disable_runtime_pm;
+		return ret;
 
 	if ((of_device_is_compatible(dev->of_node, "brcm,bcm2711-hdmi0") ||
 	     of_device_is_compatible(dev->of_node, "brcm,bcm2711-hdmi1")) &&
@@ -2990,93 +3455,43 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
 		clk_prepare_enable(vc4_hdmi->pixel_bvb_clock);
 	}
 
-	drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS);
+	ret = drmm_encoder_init(drm, encoder,
+				&vc4_hdmi_encoder_funcs,
+				DRM_MODE_ENCODER_TMDS,
+				NULL);
+	if (ret)
+		goto err_put_runtime_pm;
+
 	drm_encoder_helper_add(encoder, &vc4_hdmi_encoder_helper_funcs);
 
 	ret = vc4_hdmi_connector_init(drm, vc4_hdmi);
 	if (ret)
-		goto err_destroy_encoder;
+		goto err_put_runtime_pm;
 
 	ret = vc4_hdmi_hotplug_init(vc4_hdmi);
 	if (ret)
-		goto err_destroy_conn;
+		goto err_put_runtime_pm;
 
 	ret = vc4_hdmi_cec_init(vc4_hdmi);
 	if (ret)
-		goto err_free_hotplug;
+		goto err_put_runtime_pm;
 
 	ret = vc4_hdmi_audio_init(vc4_hdmi);
 	if (ret)
-		goto err_free_cec;
-
-	vc4_debugfs_add_file(drm, variant->debugfs_name,
-			     vc4_hdmi_debugfs_regs,
-			     vc4_hdmi);
+		goto err_put_runtime_pm;
 
 	pm_runtime_put_sync(dev);
 
 	return 0;
 
-err_free_cec:
-	vc4_hdmi_cec_exit(vc4_hdmi);
-err_free_hotplug:
-	vc4_hdmi_hotplug_exit(vc4_hdmi);
-err_destroy_conn:
-	vc4_hdmi_connector_destroy(&vc4_hdmi->connector);
-err_destroy_encoder:
-	drm_encoder_cleanup(encoder);
+err_put_runtime_pm:
 	pm_runtime_put_sync(dev);
-err_disable_runtime_pm:
-	pm_runtime_disable(dev);
-err_put_ddc:
-	put_device(&vc4_hdmi->ddc->dev);
 
 	return ret;
 }
 
-static void vc4_hdmi_unbind(struct device *dev, struct device *master,
-			    void *data)
-{
-	struct vc4_hdmi *vc4_hdmi;
-
-	/*
-	 * ASoC makes it a bit hard to retrieve a pointer to the
-	 * vc4_hdmi structure. Registering the card will overwrite our
-	 * device drvdata with a pointer to the snd_soc_card structure,
-	 * which can then be used to retrieve whatever drvdata we want
-	 * to associate.
-	 *
-	 * However, that doesn't fly in the case where we wouldn't
-	 * register an ASoC card (because of an old DT that is missing
-	 * the dmas properties for example), then the card isn't
-	 * registered and the device drvdata wouldn't be set.
-	 *
-	 * We can deal with both cases by making sure a snd_soc_card
-	 * pointer and a vc4_hdmi structure are pointing to the same
-	 * memory address, so we can treat them indistinctly without any
-	 * issue.
-	 */
-	BUILD_BUG_ON(offsetof(struct vc4_hdmi_audio, card) != 0);
-	BUILD_BUG_ON(offsetof(struct vc4_hdmi, audio) != 0);
-	vc4_hdmi = dev_get_drvdata(dev);
-
-	kfree(vc4_hdmi->hdmi_regset.regs);
-	kfree(vc4_hdmi->hd_regset.regs);
-
-	vc4_hdmi_audio_exit(vc4_hdmi);
-	vc4_hdmi_cec_exit(vc4_hdmi);
-	vc4_hdmi_hotplug_exit(vc4_hdmi);
-	vc4_hdmi_connector_destroy(&vc4_hdmi->connector);
-	drm_encoder_cleanup(&vc4_hdmi->encoder.base);
-
-	pm_runtime_disable(dev);
-
-	put_device(&vc4_hdmi->ddc->dev);
-}
-
 static const struct component_ops vc4_hdmi_ops = {
 	.bind   = vc4_hdmi_bind,
-	.unbind = vc4_hdmi_unbind,
 };
 
 static int vc4_hdmi_dev_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.h b/drivers/gpu/drm/vc4/vc4_hdmi.h
index c3ed2b07df23..db823efb2563 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.h
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.h
@@ -58,7 +58,8 @@ struct vc4_hdmi_variant {
 	/* Callback to get the resources (memory region, interrupts,
 	 * clocks, etc) for that variant.
 	 */
-	int (*init_resources)(struct vc4_hdmi *vc4_hdmi);
+	int (*init_resources)(struct drm_device *drm,
+			      struct vc4_hdmi *vc4_hdmi);
 
 	/* Callback to reset the HDMI block */
 	void (*reset)(struct vc4_hdmi *vc4_hdmi);
@@ -71,7 +72,7 @@ struct vc4_hdmi_variant {
 	/* Callback to configure the video timings in the HDMI block */
 	void (*set_timings)(struct vc4_hdmi *vc4_hdmi,
 			    struct drm_connector_state *state,
-			    struct drm_display_mode *mode);
+			    const struct drm_display_mode *mode);
 
 	/* Callback to initialize the PHY according to the connector state */
 	void (*phy_init)(struct vc4_hdmi *vc4_hdmi,
@@ -194,15 +195,7 @@ struct vc4_hdmi {
 
 	/**
 	 * @mutex: Mutex protecting the driver access across multiple
-	 * frameworks (KMS, ALSA).
-	 *
-	 * NOTE: While supported, CEC has been left out since
-	 * cec_s_phys_addr_from_edid() might call .adap_enable and lead to a
-	 * reentrancy issue between .get_modes (or .detect) and .adap_enable.
-	 * Since we don't share any state between the CEC hooks and KMS', it's
-	 * not a big deal. The only trouble might come from updating the CEC
-	 * clock divider which might be affected by a modeset, but CEC should
-	 * be resilient to that.
+	 * frameworks (KMS, ALSA, CEC).
 	 */
 	struct mutex mutex;
 
diff --git a/drivers/gpu/drm/vc4/vc4_hvs.c b/drivers/gpu/drm/vc4/vc4_hvs.c
index fbaa741dda5f..4ac9f5a2d5f9 100644
--- a/drivers/gpu/drm/vc4/vc4_hvs.c
+++ b/drivers/gpu/drm/vc4/vc4_hvs.c
@@ -25,6 +25,7 @@
 #include <linux/platform_device.h>
 
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_vblank.h>
 
 #include "vc4_drv.h"
@@ -66,8 +67,12 @@ static const struct debugfs_reg32 hvs_regs[] = {
 
 void vc4_hvs_dump_state(struct vc4_hvs *hvs)
 {
+	struct drm_device *drm = &hvs->vc4->base;
 	struct drm_printer p = drm_info_printer(&hvs->pdev->dev);
-	int i;
+	int idx, i;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	drm_print_regset32(&p, &hvs->regset);
 
@@ -80,6 +85,8 @@ void vc4_hvs_dump_state(struct vc4_hvs *hvs)
 			 readl((u32 __iomem *)hvs->dlist + i + 2),
 			 readl((u32 __iomem *)hvs->dlist + i + 3));
 	}
+
+	drm_dev_exit(idx);
 }
 
 static int vc4_hvs_debugfs_underrun(struct seq_file *m, void *data)
@@ -175,6 +182,11 @@ static int vc4_hvs_upload_linear_kernel(struct vc4_hvs *hvs,
 	int ret, i;
 	u32 __iomem *dst_kernel;
 
+	/*
+	 * NOTE: We don't need a call to drm_dev_enter()/drm_dev_exit()
+	 * here since that function is only called from vc4_hvs_bind().
+	 */
+
 	ret = drm_mm_insert_node(&hvs->dlist_mm, space, VC4_KERNEL_DWORDS);
 	if (ret) {
 		DRM_ERROR("Failed to allocate space for filter kernel: %d\n",
@@ -199,10 +211,15 @@ static int vc4_hvs_upload_linear_kernel(struct vc4_hvs *hvs,
 static void vc4_hvs_lut_load(struct vc4_hvs *hvs,
 			     struct vc4_crtc *vc4_crtc)
 {
+	struct drm_device *drm = &hvs->vc4->base;
 	struct drm_crtc *crtc = &vc4_crtc->base;
 	struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
+	int idx;
 	u32 i;
 
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
 	/* The LUT memory is laid out with each HVS channel in order,
 	 * each of which takes 256 writes for R, 256 for G, then 256
 	 * for B.
@@ -217,6 +234,8 @@ static void vc4_hvs_lut_load(struct vc4_hvs *hvs,
 		HVS_WRITE(SCALER_GAMDATA, vc4_crtc->lut_g[i]);
 	for (i = 0; i < crtc->gamma_size; i++)
 		HVS_WRITE(SCALER_GAMDATA, vc4_crtc->lut_b[i]);
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_hvs_update_gamma_lut(struct vc4_hvs *hvs,
@@ -238,7 +257,12 @@ static void vc4_hvs_update_gamma_lut(struct vc4_hvs *hvs,
 
 u8 vc4_hvs_get_fifo_frame_count(struct vc4_hvs *hvs, unsigned int fifo)
 {
+	struct drm_device *drm = &hvs->vc4->base;
 	u8 field = 0;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return 0;
 
 	switch (fifo) {
 	case 0:
@@ -255,6 +279,7 @@ u8 vc4_hvs_get_fifo_frame_count(struct vc4_hvs *hvs, unsigned int fifo)
 		break;
 	}
 
+	drm_dev_exit(idx);
 	return field;
 }
 
@@ -267,6 +292,12 @@ int vc4_hvs_get_fifo_from_output(struct vc4_hvs *hvs, unsigned int output)
 	if (!vc4->is_vc5)
 		return output;
 
+	/*
+	 * NOTE: We should probably use drm_dev_enter()/drm_dev_exit()
+	 * here, but this function is only used during the DRM device
+	 * initialization, so we should be fine.
+	 */
+
 	switch (output) {
 	case 0:
 		return 0;
@@ -315,12 +346,17 @@ static int vc4_hvs_init_channel(struct vc4_hvs *hvs, struct drm_crtc *crtc,
 				struct drm_display_mode *mode, bool oneshot)
 {
 	struct vc4_dev *vc4 = hvs->vc4;
+	struct drm_device *drm = &vc4->base;
 	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 	struct vc4_crtc_state *vc4_crtc_state = to_vc4_crtc_state(crtc->state);
 	unsigned int chan = vc4_crtc_state->assigned_channel;
 	bool interlace = mode->flags & DRM_MODE_FLAG_INTERLACE;
 	u32 dispbkgndx;
 	u32 dispctrl;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return -ENODEV;
 
 	HVS_WRITE(SCALER_DISPCTRLX(chan), 0);
 	HVS_WRITE(SCALER_DISPCTRLX(chan), SCALER_DISPCTRLX_RESET);
@@ -362,14 +398,22 @@ static int vc4_hvs_init_channel(struct vc4_hvs *hvs, struct drm_crtc *crtc,
 	 */
 	vc4_hvs_lut_load(hvs, vc4_crtc);
 
+	drm_dev_exit(idx);
+
 	return 0;
 }
 
 void vc4_hvs_stop_channel(struct vc4_hvs *hvs, unsigned int chan)
 {
-	if (HVS_READ(SCALER_DISPCTRLX(chan)) & SCALER_DISPCTRLX_ENABLE)
+	struct drm_device *drm = &hvs->vc4->base;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
 		return;
 
+	if (HVS_READ(SCALER_DISPCTRLX(chan)) & SCALER_DISPCTRLX_ENABLE)
+		goto out;
+
 	HVS_WRITE(SCALER_DISPCTRLX(chan),
 		  HVS_READ(SCALER_DISPCTRLX(chan)) | SCALER_DISPCTRLX_RESET);
 	HVS_WRITE(SCALER_DISPCTRLX(chan),
@@ -385,6 +429,9 @@ void vc4_hvs_stop_channel(struct vc4_hvs *hvs, unsigned int chan)
 	WARN_ON_ONCE((HVS_READ(SCALER_DISPSTATX(chan)) &
 		      (SCALER_DISPSTATX_FULL | SCALER_DISPSTATX_EMPTY)) !=
 		     SCALER_DISPSTATX_EMPTY);
+
+out:
+	drm_dev_exit(idx);
 }
 
 int vc4_hvs_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state)
@@ -426,9 +473,15 @@ static void vc4_hvs_install_dlist(struct drm_crtc *crtc)
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct vc4_hvs *hvs = vc4->hvs;
 	struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
+	int idx;
+
+	if (!drm_dev_enter(dev, &idx))
+		return;
 
 	HVS_WRITE(SCALER_DISPLISTX(vc4_state->assigned_channel),
 		  vc4_state->mm.start);
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_hvs_update_dlist(struct drm_crtc *crtc)
@@ -513,6 +566,12 @@ void vc4_hvs_atomic_flush(struct drm_crtc *crtc,
 	bool enable_bg_fill = false;
 	u32 __iomem *dlist_start = vc4->hvs->dlist + vc4_state->mm.start;
 	u32 __iomem *dlist_next = dlist_start;
+	int idx;
+
+	if (!drm_dev_enter(dev, &idx)) {
+		vc4_crtc_send_vblank(crtc);
+		return;
+	}
 
 	if (debug_dump_regs) {
 		DRM_INFO("CRTC %d HVS before:\n", drm_crtc_index(crtc));
@@ -583,26 +642,44 @@ void vc4_hvs_atomic_flush(struct drm_crtc *crtc,
 		DRM_INFO("CRTC %d HVS after:\n", drm_crtc_index(crtc));
 		vc4_hvs_dump_state(hvs);
 	}
+
+	drm_dev_exit(idx);
 }
 
 void vc4_hvs_mask_underrun(struct vc4_hvs *hvs, int channel)
 {
-	u32 dispctrl = HVS_READ(SCALER_DISPCTRL);
+	struct drm_device *drm = &hvs->vc4->base;
+	u32 dispctrl;
+	int idx;
 
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
+	dispctrl = HVS_READ(SCALER_DISPCTRL);
 	dispctrl &= ~SCALER_DISPCTRL_DSPEISLUR(channel);
 
 	HVS_WRITE(SCALER_DISPCTRL, dispctrl);
+
+	drm_dev_exit(idx);
 }
 
 void vc4_hvs_unmask_underrun(struct vc4_hvs *hvs, int channel)
 {
-	u32 dispctrl = HVS_READ(SCALER_DISPCTRL);
+	struct drm_device *drm = &hvs->vc4->base;
+	u32 dispctrl;
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
+	dispctrl = HVS_READ(SCALER_DISPCTRL);
 	dispctrl |= SCALER_DISPCTRL_DSPEISLUR(channel);
 
 	HVS_WRITE(SCALER_DISPSTAT,
 		  SCALER_DISPSTAT_EUFLOW(channel));
 	HVS_WRITE(SCALER_DISPCTRL, dispctrl);
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_hvs_report_underrun(struct drm_device *dev)
@@ -623,6 +700,17 @@ static irqreturn_t vc4_hvs_irq_handler(int irq, void *data)
 	u32 control;
 	u32 status;
 
+	/*
+	 * NOTE: We don't need to protect the register access using
+	 * drm_dev_enter() there because the interrupt handler lifetime
+	 * is tied to the device itself, and not to the DRM device.
+	 *
+	 * So when the device will be gone, one of the first thing we
+	 * will be doing will be to unregister the interrupt handler,
+	 * and then unregister the DRM device. drm_dev_enter() would
+	 * thus always succeed if we are here.
+	 */
+
 	status = HVS_READ(SCALER_DISPSTAT);
 	control = HVS_READ(SCALER_DISPCTRL);
 
@@ -645,6 +733,39 @@ static irqreturn_t vc4_hvs_irq_handler(int irq, void *data)
 	return irqret;
 }
 
+int vc4_hvs_debugfs_init(struct drm_minor *minor)
+{
+	struct drm_device *drm = minor->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(drm);
+	struct vc4_hvs *hvs = vc4->hvs;
+	int ret;
+
+	if (!vc4->hvs)
+		return -ENODEV;
+
+	if (!vc4->is_vc5)
+		debugfs_create_bool("hvs_load_tracker", S_IRUGO | S_IWUSR,
+				    minor->debugfs_root,
+				    &vc4->load_tracker_enabled);
+
+	ret = vc4_debugfs_add_file(minor, "hvs_dlists",
+				   vc4_hvs_debugfs_dlist, NULL);
+	if (ret)
+		return ret;
+
+	ret = vc4_debugfs_add_file(minor, "hvs_underrun",
+				   vc4_hvs_debugfs_underrun, NULL);
+	if (ret)
+		return ret;
+
+	ret = vc4_debugfs_add_regset32(minor, "hvs_regs",
+				       &hvs->regset);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
 static int vc4_hvs_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
@@ -655,10 +776,9 @@ static int vc4_hvs_bind(struct device *dev, struct device *master, void *data)
 	u32 dispctrl;
 	u32 reg;
 
-	hvs = devm_kzalloc(&pdev->dev, sizeof(*hvs), GFP_KERNEL);
+	hvs = drmm_kzalloc(drm, sizeof(*hvs), GFP_KERNEL);
 	if (!hvs)
 		return -ENOMEM;
-
 	hvs->vc4 = vc4;
 	hvs->pdev = pdev;
 
@@ -771,12 +891,6 @@ static int vc4_hvs_bind(struct device *dev, struct device *master, void *data)
 	if (ret)
 		return ret;
 
-	vc4_debugfs_add_regset32(drm, "hvs_regs", &hvs->regset);
-	vc4_debugfs_add_file(drm, "hvs_underrun", vc4_hvs_debugfs_underrun,
-			     NULL);
-	vc4_debugfs_add_file(drm, "hvs_dlists", vc4_hvs_debugfs_dlist,
-			     NULL);
-
 	return 0;
 }
 
@@ -786,11 +900,18 @@ static void vc4_hvs_unbind(struct device *dev, struct device *master,
 	struct drm_device *drm = dev_get_drvdata(master);
 	struct vc4_dev *vc4 = to_vc4_dev(drm);
 	struct vc4_hvs *hvs = vc4->hvs;
+	struct drm_mm_node *node, *next;
 
 	if (drm_mm_node_allocated(&vc4->hvs->mitchell_netravali_filter))
 		drm_mm_remove_node(&vc4->hvs->mitchell_netravali_filter);
 
+	drm_mm_for_each_node_safe(node, next, &vc4->hvs->dlist_mm)
+		drm_mm_remove_node(node);
+
 	drm_mm_takedown(&vc4->hvs->dlist_mm);
+
+	drm_mm_for_each_node_safe(node, next, &vc4->hvs->lbm_mm)
+		drm_mm_remove_node(node);
 	drm_mm_takedown(&vc4->hvs->lbm_mm);
 
 	clk_disable_unprepare(hvs->core_clk);
diff --git a/drivers/gpu/drm/vc4/vc4_irq.c b/drivers/gpu/drm/vc4/vc4_irq.c
index 2eacfb6773d2..1e6db0121ccd 100644
--- a/drivers/gpu/drm/vc4/vc4_irq.c
+++ b/drivers/gpu/drm/vc4/vc4_irq.c
@@ -105,7 +105,7 @@ vc4_overflow_mem_work(struct work_struct *work)
 	}
 	vc4->bin_alloc_overflow = BIT(bin_bo_slot);
 
-	V3D_WRITE(V3D_BPOA, bo->base.paddr + bin_bo_slot * vc4->bin_alloc_size);
+	V3D_WRITE(V3D_BPOA, bo->base.dma_addr + bin_bo_slot * vc4->bin_alloc_size);
 	V3D_WRITE(V3D_BPOS, bo->base.base.size);
 	V3D_WRITE(V3D_INTCTL, V3D_INT_OUTOMEM);
 	V3D_WRITE(V3D_INTENA, V3D_INT_OUTOMEM);
@@ -295,7 +295,7 @@ vc4_irq_disable(struct drm_device *dev)
 	V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS);
 
 	/* Finish any interrupt handler still in flight. */
-	disable_irq(vc4->irq);
+	synchronize_irq(vc4->irq);
 
 	cancel_work_sync(&vc4->overflow_mem_work);
 }
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index b45dcdfd7306..4419e810103d 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -18,7 +18,6 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_framebuffer_helper.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_vblank.h>
 
diff --git a/drivers/gpu/drm/vc4/vc4_perfmon.c b/drivers/gpu/drm/vc4/vc4_perfmon.c
index 79a74184d732..c4ac2c946238 100644
--- a/drivers/gpu/drm/vc4/vc4_perfmon.c
+++ b/drivers/gpu/drm/vc4/vc4_perfmon.c
@@ -133,6 +133,7 @@ void vc4_perfmon_close_file(struct vc4_file *vc4file)
 	idr_for_each(&vc4file->perfmon.idr, vc4_perfmon_idr_del, NULL);
 	idr_destroy(&vc4file->perfmon.idr);
 	mutex_unlock(&vc4file->perfmon.lock);
+	mutex_destroy(&vc4file->perfmon.lock);
 }
 
 int vc4_perfmon_create_ioctl(struct drm_device *dev, void *data,
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index f27e87a23df7..8b92a45a3c89 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -19,11 +19,11 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_atomic_uapi.h>
 #include <drm/drm_blend.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_drv.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_gem_atomic_helper.h>
-#include <drm/drm_plane_helper.h>
 
 #include "uapi/drm/vc4_drm.h"
 
@@ -340,7 +340,7 @@ static int vc4_plane_setup_clipping_and_scaling(struct drm_plane_state *state)
 {
 	struct vc4_plane_state *vc4_state = to_vc4_plane_state(state);
 	struct drm_framebuffer *fb = state->fb;
-	struct drm_gem_cma_object *bo = drm_fb_cma_get_gem_obj(fb, 0);
+	struct drm_gem_dma_object *bo = drm_fb_dma_get_gem_obj(fb, 0);
 	int num_planes = fb->format->num_planes;
 	struct drm_crtc_state *crtc_state;
 	u32 h_subsample = fb->format->hsub;
@@ -360,7 +360,7 @@ static int vc4_plane_setup_clipping_and_scaling(struct drm_plane_state *state)
 		return ret;
 
 	for (i = 0; i < num_planes; i++)
-		vc4_state->offsets[i] = bo->paddr + fb->offsets[i];
+		vc4_state->offsets[i] = bo->dma_addr + fb->offsets[i];
 
 	/*
 	 * We don't support subpixel source positioning for scaling,
@@ -1220,6 +1220,10 @@ u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist)
 {
 	struct vc4_plane_state *vc4_state = to_vc4_plane_state(plane->state);
 	int i;
+	int idx;
+
+	if (!drm_dev_enter(plane->dev, &idx))
+		goto out;
 
 	vc4_state->hw_dlist = dlist;
 
@@ -1227,6 +1231,9 @@ u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist)
 	for (i = 0; i < vc4_state->dlist_count; i++)
 		writel(vc4_state->dlist[i], &dlist[i]);
 
+	drm_dev_exit(idx);
+
+out:
 	return vc4_state->dlist_count;
 }
 
@@ -1244,14 +1251,18 @@ u32 vc4_plane_dlist_size(const struct drm_plane_state *state)
 void vc4_plane_async_set_fb(struct drm_plane *plane, struct drm_framebuffer *fb)
 {
 	struct vc4_plane_state *vc4_state = to_vc4_plane_state(plane->state);
-	struct drm_gem_cma_object *bo = drm_fb_cma_get_gem_obj(fb, 0);
+	struct drm_gem_dma_object *bo = drm_fb_dma_get_gem_obj(fb, 0);
 	uint32_t addr;
+	int idx;
+
+	if (!drm_dev_enter(plane->dev, &idx))
+		return;
 
 	/* We're skipping the address adjustment for negative origin,
 	 * because this is only called on the primary plane.
 	 */
 	WARN_ON_ONCE(plane->state->crtc_x < 0 || plane->state->crtc_y < 0);
-	addr = bo->paddr + fb->offsets[0];
+	addr = bo->dma_addr + fb->offsets[0];
 
 	/* Write the new address into the hardware immediately.  The
 	 * scanout will start from this address as soon as the FIFO
@@ -1264,6 +1275,8 @@ void vc4_plane_async_set_fb(struct drm_plane *plane, struct drm_framebuffer *fb)
 	 * also use our updated address.
 	 */
 	vc4_state->dlist[vc4_state->ptr0_offset] = addr;
+
+	drm_dev_exit(idx);
 }
 
 static void vc4_plane_atomic_async_update(struct drm_plane *plane,
@@ -1272,6 +1285,10 @@ static void vc4_plane_atomic_async_update(struct drm_plane *plane,
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
 	struct vc4_plane_state *vc4_state, *new_vc4_state;
+	int idx;
+
+	if (!drm_dev_enter(plane->dev, &idx))
+		return;
 
 	swap(plane->state->fb, new_plane_state->fb);
 	plane->state->crtc_x = new_plane_state->crtc_x;
@@ -1334,6 +1351,8 @@ static void vc4_plane_atomic_async_update(struct drm_plane *plane,
 	       &vc4_state->hw_dlist[vc4_state->pos2_offset]);
 	writel(vc4_state->dlist[vc4_state->ptr0_offset],
 	       &vc4_state->hw_dlist[vc4_state->ptr0_offset]);
+
+	drm_dev_exit(idx);
 }
 
 static int vc4_plane_atomic_async_check(struct drm_plane *plane,
@@ -1388,7 +1407,7 @@ static int vc4_prepare_fb(struct drm_plane *plane,
 	if (!state->fb)
 		return 0;
 
-	bo = to_vc4_bo(&drm_fb_cma_get_gem_obj(state->fb, 0)->base);
+	bo = to_vc4_bo(&drm_fb_dma_get_gem_obj(state->fb, 0)->base);
 
 	drm_gem_plane_helper_prepare_fb(plane, state);
 
@@ -1406,7 +1425,7 @@ static void vc4_cleanup_fb(struct drm_plane *plane,
 	if (plane->state->fb == state->fb || !state->fb)
 		return;
 
-	bo = to_vc4_bo(&drm_fb_cma_get_gem_obj(state->fb, 0)->base);
+	bo = to_vc4_bo(&drm_fb_dma_get_gem_obj(state->fb, 0)->base);
 	vc4_bo_dec_usecnt(bo);
 }
 
@@ -1483,8 +1502,6 @@ static bool vc4_format_mod_supported(struct drm_plane *plane,
 static const struct drm_plane_funcs vc4_plane_funcs = {
 	.update_plane = drm_atomic_helper_update_plane,
 	.disable_plane = drm_atomic_helper_disable_plane,
-	.destroy = drm_plane_cleanup,
-	.set_property = NULL,
 	.reset = vc4_plane_reset,
 	.atomic_duplicate_state = vc4_plane_duplicate_state,
 	.atomic_destroy_state = vc4_plane_destroy_state,
@@ -1492,14 +1509,14 @@ static const struct drm_plane_funcs vc4_plane_funcs = {
 };
 
 struct drm_plane *vc4_plane_init(struct drm_device *dev,
-				 enum drm_plane_type type)
+				 enum drm_plane_type type,
+				 uint32_t possible_crtcs)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
-	struct drm_plane *plane = NULL;
+	struct drm_plane *plane;
 	struct vc4_plane *vc4_plane;
 	u32 formats[ARRAY_SIZE(hvs_formats)];
 	int num_formats = 0;
-	int ret = 0;
 	unsigned i;
 	static const uint64_t modifiers[] = {
 		DRM_FORMAT_MOD_BROADCOM_VC4_T_TILED,
@@ -1510,11 +1527,6 @@ struct drm_plane *vc4_plane_init(struct drm_device *dev,
 		DRM_FORMAT_MOD_INVALID
 	};
 
-	vc4_plane = devm_kzalloc(dev->dev, sizeof(*vc4_plane),
-				 GFP_KERNEL);
-	if (!vc4_plane)
-		return ERR_PTR(-ENOMEM);
-
 	for (i = 0; i < ARRAY_SIZE(hvs_formats); i++) {
 		if (!hvs_formats[i].hvs5_only || vc4->is_vc5) {
 			formats[num_formats] = hvs_formats[i].drm;
@@ -1522,13 +1534,14 @@ struct drm_plane *vc4_plane_init(struct drm_device *dev,
 		}
 	}
 
+	vc4_plane = drmm_universal_plane_alloc(dev, struct vc4_plane, base,
+					       possible_crtcs,
+					       &vc4_plane_funcs,
+					       formats, num_formats,
+					       modifiers, type, NULL);
+	if (IS_ERR(vc4_plane))
+		return ERR_CAST(vc4_plane);
 	plane = &vc4_plane->base;
-	ret = drm_universal_plane_init(dev, plane, 0,
-				       &vc4_plane_funcs,
-				       formats, num_formats,
-				       modifiers, type, NULL);
-	if (ret)
-		return ERR_PTR(ret);
 
 	if (vc4->is_vc5)
 		drm_plane_helper_add(plane, &vc5_plane_helper_funcs);
@@ -1575,13 +1588,11 @@ int vc4_plane_create_additional_planes(struct drm_device *drm)
 	 */
 	for (i = 0; i < 16; i++) {
 		struct drm_plane *plane =
-			vc4_plane_init(drm, DRM_PLANE_TYPE_OVERLAY);
+			vc4_plane_init(drm, DRM_PLANE_TYPE_OVERLAY,
+				       GENMASK(drm->mode_config.num_crtc - 1, 0));
 
 		if (IS_ERR(plane))
 			continue;
-
-		plane->possible_crtcs =
-			GENMASK(drm->mode_config.num_crtc - 1, 0);
 	}
 
 	drm_for_each_crtc(crtc, drm) {
@@ -1589,9 +1600,9 @@ int vc4_plane_create_additional_planes(struct drm_device *drm)
 		 * since we overlay planes on the CRTC in the order they were
 		 * initialized.
 		 */
-		cursor_plane = vc4_plane_init(drm, DRM_PLANE_TYPE_CURSOR);
+		cursor_plane = vc4_plane_init(drm, DRM_PLANE_TYPE_CURSOR,
+					      drm_crtc_mask(crtc));
 		if (!IS_ERR(cursor_plane)) {
-			cursor_plane->possible_crtcs = drm_crtc_mask(crtc);
 			crtc->cursor = cursor_plane;
 		}
 	}
diff --git a/drivers/gpu/drm/vc4/vc4_render_cl.c b/drivers/gpu/drm/vc4/vc4_render_cl.c
index f6b7dc3df08c..1bda5010f15a 100644
--- a/drivers/gpu/drm/vc4/vc4_render_cl.c
+++ b/drivers/gpu/drm/vc4/vc4_render_cl.c
@@ -40,14 +40,14 @@
 #include "vc4_packet.h"
 
 struct vc4_rcl_setup {
-	struct drm_gem_cma_object *color_read;
-	struct drm_gem_cma_object *color_write;
-	struct drm_gem_cma_object *zs_read;
-	struct drm_gem_cma_object *zs_write;
-	struct drm_gem_cma_object *msaa_color_write;
-	struct drm_gem_cma_object *msaa_zs_write;
-
-	struct drm_gem_cma_object *rcl;
+	struct drm_gem_dma_object *color_read;
+	struct drm_gem_dma_object *color_write;
+	struct drm_gem_dma_object *zs_read;
+	struct drm_gem_dma_object *zs_write;
+	struct drm_gem_dma_object *msaa_color_write;
+	struct drm_gem_dma_object *msaa_zs_write;
+
+	struct drm_gem_dma_object *rcl;
 	u32 next_offset;
 
 	u32 next_write_bo_index;
@@ -97,11 +97,11 @@ static void vc4_store_before_load(struct vc4_rcl_setup *setup)
  * coordinates packet, and instead just store to the address given.
  */
 static uint32_t vc4_full_res_offset(struct vc4_exec_info *exec,
-				    struct drm_gem_cma_object *bo,
+				    struct drm_gem_dma_object *bo,
 				    struct drm_vc4_submit_rcl_surface *surf,
 				    uint8_t x, uint8_t y)
 {
-	return bo->paddr + surf->offset + VC4_TILE_BUFFER_SIZE *
+	return bo->dma_addr + surf->offset + VC4_TILE_BUFFER_SIZE *
 		(DIV_ROUND_UP(exec->args->width, 32) * y + x);
 }
 
@@ -142,7 +142,7 @@ static void emit_tile(struct vc4_exec_info *exec,
 		} else {
 			rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
 			rcl_u16(setup, args->color_read.bits);
-			rcl_u32(setup, setup->color_read->paddr +
+			rcl_u32(setup, setup->color_read->dma_addr +
 				args->color_read.offset);
 		}
 	}
@@ -164,7 +164,7 @@ static void emit_tile(struct vc4_exec_info *exec,
 		} else {
 			rcl_u8(setup, VC4_PACKET_LOAD_TILE_BUFFER_GENERAL);
 			rcl_u16(setup, args->zs_read.bits);
-			rcl_u32(setup, setup->zs_read->paddr +
+			rcl_u32(setup, setup->zs_read->dma_addr +
 				args->zs_read.offset);
 		}
 	}
@@ -232,7 +232,7 @@ static void emit_tile(struct vc4_exec_info *exec,
 			(last_tile_write ?
 			 0 : VC4_STORE_TILE_BUFFER_DISABLE_COLOR_CLEAR));
 		rcl_u32(setup,
-			(setup->zs_write->paddr + args->zs_write.offset) |
+			(setup->zs_write->dma_addr + args->zs_write.offset) |
 			((last && last_tile_write) ?
 			 VC4_LOADSTORE_TILE_BUFFER_EOF : 0));
 	}
@@ -355,7 +355,7 @@ static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
 
 	rcl_u8(setup, VC4_PACKET_TILE_RENDERING_MODE_CONFIG);
 	rcl_u32(setup,
-		(setup->color_write ? (setup->color_write->paddr +
+		(setup->color_write ? (setup->color_write->dma_addr +
 				       args->color_write.offset) :
 		 0));
 	rcl_u16(setup, args->width);
@@ -374,14 +374,14 @@ static int vc4_create_rcl_bo(struct drm_device *dev, struct vc4_exec_info *exec,
 	}
 
 	BUG_ON(setup->next_offset != size);
-	exec->ct1ca = setup->rcl->paddr;
-	exec->ct1ea = setup->rcl->paddr + setup->next_offset;
+	exec->ct1ca = setup->rcl->dma_addr;
+	exec->ct1ea = setup->rcl->dma_addr + setup->next_offset;
 
 	return 0;
 }
 
 static int vc4_full_res_bounds_check(struct vc4_exec_info *exec,
-				     struct drm_gem_cma_object *obj,
+				     struct drm_gem_dma_object *obj,
 				     struct drm_vc4_submit_rcl_surface *surf)
 {
 	struct drm_vc4_submit_cl *args = exec->args;
@@ -407,7 +407,7 @@ static int vc4_full_res_bounds_check(struct vc4_exec_info *exec,
 }
 
 static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
-				      struct drm_gem_cma_object **obj,
+				      struct drm_gem_dma_object **obj,
 				      struct drm_vc4_submit_rcl_surface *surf)
 {
 	if (surf->flags != 0 || surf->bits != 0) {
@@ -433,7 +433,7 @@ static int vc4_rcl_msaa_surface_setup(struct vc4_exec_info *exec,
 }
 
 static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
-				 struct drm_gem_cma_object **obj,
+				 struct drm_gem_dma_object **obj,
 				 struct drm_vc4_submit_rcl_surface *surf,
 				 bool is_write)
 {
@@ -533,7 +533,7 @@ static int vc4_rcl_surface_setup(struct vc4_exec_info *exec,
 static int
 vc4_rcl_render_config_surface_setup(struct vc4_exec_info *exec,
 				    struct vc4_rcl_setup *setup,
-				    struct drm_gem_cma_object **obj,
+				    struct drm_gem_dma_object **obj,
 				    struct drm_vc4_submit_rcl_surface *surf)
 {
 	uint8_t tiling = VC4_GET_FIELD(surf->bits,
diff --git a/drivers/gpu/drm/vc4/vc4_txp.c b/drivers/gpu/drm/vc4/vc4_txp.c
index d20b0bc51a18..bd181b5a7b52 100644
--- a/drivers/gpu/drm/vc4/vc4_txp.c
+++ b/drivers/gpu/drm/vc4/vc4_txp.c
@@ -15,8 +15,9 @@
 
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_edid.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_panel.h>
@@ -155,7 +156,6 @@ struct vc4_txp {
 	struct drm_writeback_connector connector;
 
 	void __iomem *regs;
-	struct debugfs_regset32 regset;
 };
 
 static inline struct vc4_txp *encoder_to_vc4_txp(struct drm_encoder *encoder)
@@ -276,13 +276,15 @@ static int vc4_txp_connector_atomic_check(struct drm_connector *conn,
 static void vc4_txp_connector_atomic_commit(struct drm_connector *conn,
 					struct drm_atomic_state *state)
 {
+	struct drm_device *drm = conn->dev;
 	struct drm_connector_state *conn_state = drm_atomic_get_new_connector_state(state,
 										    conn);
 	struct vc4_txp *txp = connector_to_vc4_txp(conn);
-	struct drm_gem_cma_object *gem;
+	struct drm_gem_dma_object *gem;
 	struct drm_display_mode *mode;
 	struct drm_framebuffer *fb;
 	u32 ctrl;
+	int idx;
 	int i;
 
 	if (WARN_ON(!conn_state->writeback_job))
@@ -312,8 +314,11 @@ static void vc4_txp_connector_atomic_commit(struct drm_connector *conn,
 		 */
 		ctrl |= TXP_ALPHA_INVERT;
 
-	gem = drm_fb_cma_get_gem_obj(fb, 0);
-	TXP_WRITE(TXP_DST_PTR, gem->paddr + fb->offsets[0]);
+	if (!drm_dev_enter(drm, &idx))
+		return;
+
+	gem = drm_fb_dma_get_gem_obj(fb, 0);
+	TXP_WRITE(TXP_DST_PTR, gem->dma_addr + fb->offsets[0]);
 	TXP_WRITE(TXP_DST_PITCH, fb->pitches[0]);
 	TXP_WRITE(TXP_DIM,
 		  VC4_SET_FIELD(mode->hdisplay, TXP_WIDTH) |
@@ -322,6 +327,8 @@ static void vc4_txp_connector_atomic_commit(struct drm_connector *conn,
 	TXP_WRITE(TXP_DST_CTRL, ctrl);
 
 	drm_writeback_queue_job(&txp->connector, conn_state);
+
+	drm_dev_exit(idx);
 }
 
 static const struct drm_connector_helper_funcs vc4_txp_connector_helper_funcs = {
@@ -337,16 +344,10 @@ vc4_txp_connector_detect(struct drm_connector *connector, bool force)
 	return connector_status_connected;
 }
 
-static void vc4_txp_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_unregister(connector);
-	drm_connector_cleanup(connector);
-}
-
 static const struct drm_connector_funcs vc4_txp_connector_funcs = {
 	.detect = vc4_txp_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = vc4_txp_connector_destroy,
+	.destroy = drm_connector_cleanup,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
@@ -354,7 +355,12 @@ static const struct drm_connector_funcs vc4_txp_connector_funcs = {
 
 static void vc4_txp_encoder_disable(struct drm_encoder *encoder)
 {
+	struct drm_device *drm = encoder->dev;
 	struct vc4_txp *txp = encoder_to_vc4_txp(encoder);
+	int idx;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	if (TXP_READ(TXP_DST_CTRL) & TXP_BUSY) {
 		unsigned long timeout = jiffies + msecs_to_jiffies(1000);
@@ -369,6 +375,8 @@ static void vc4_txp_encoder_disable(struct drm_encoder *encoder)
 	}
 
 	TXP_WRITE(TXP_DST_CTRL, TXP_POWERDOWN);
+
+	drm_dev_exit(idx);
 }
 
 static const struct drm_encoder_helper_funcs vc4_txp_encoder_helper_funcs = {
@@ -384,13 +392,13 @@ static void vc4_txp_disable_vblank(struct drm_crtc *crtc) {}
 
 static const struct drm_crtc_funcs vc4_txp_crtc_funcs = {
 	.set_config		= drm_atomic_helper_set_config,
-	.destroy		= vc4_crtc_destroy,
 	.page_flip		= vc4_page_flip,
 	.reset			= vc4_crtc_reset,
 	.atomic_duplicate_state	= vc4_crtc_duplicate_state,
 	.atomic_destroy_state	= vc4_crtc_destroy_state,
 	.enable_vblank		= vc4_txp_enable_vblank,
 	.disable_vblank		= vc4_txp_disable_vblank,
+	.late_register		= vc4_crtc_late_register,
 };
 
 static int vc4_txp_atomic_check(struct drm_crtc *crtc,
@@ -453,6 +461,16 @@ static irqreturn_t vc4_txp_interrupt(int irq, void *data)
 	struct vc4_txp *txp = data;
 	struct vc4_crtc *vc4_crtc = &txp->base;
 
+	/*
+	 * We don't need to protect the register access using
+	 * drm_dev_enter() there because the interrupt handler lifetime
+	 * is tied to the device itself, and not to the DRM device.
+	 *
+	 * So when the device will be gone, one of the first thing we
+	 * will be doing will be to unregister the interrupt handler,
+	 * and then unregister the DRM device. drm_dev_enter() would
+	 * thus always succeed if we are here.
+	 */
 	TXP_WRITE(TXP_DST_CTRL, TXP_READ(TXP_DST_CTRL) & ~TXP_EI);
 	vc4_crtc_handle_vblank(vc4_crtc);
 	drm_writeback_signal_completion(&txp->connector, 0);
@@ -461,6 +479,7 @@ static irqreturn_t vc4_txp_interrupt(int irq, void *data)
 }
 
 static const struct vc4_crtc_data vc4_txp_crtc_data = {
+	.debugfs_name = "txp_regs",
 	.hvs_available_channels = BIT(2),
 	.hvs_output = 2,
 };
@@ -469,7 +488,6 @@ static int vc4_txp_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct drm_device *drm = dev_get_drvdata(master);
-	struct vc4_dev *vc4 = to_vc4_dev(drm);
 	struct vc4_crtc *vc4_crtc;
 	struct vc4_txp *txp;
 	struct drm_crtc *crtc;
@@ -480,7 +498,7 @@ static int vc4_txp_bind(struct device *dev, struct device *master, void *data)
 	if (irq < 0)
 		return irq;
 
-	txp = devm_kzalloc(dev, sizeof(*txp), GFP_KERNEL);
+	txp = drmm_kzalloc(drm, sizeof(*txp), GFP_KERNEL);
 	if (!txp)
 		return -ENOMEM;
 	vc4_crtc = &txp->base;
@@ -495,9 +513,9 @@ static int vc4_txp_bind(struct device *dev, struct device *master, void *data)
 	txp->regs = vc4_ioremap_regs(pdev, 0);
 	if (IS_ERR(txp->regs))
 		return PTR_ERR(txp->regs);
-	txp->regset.base = txp->regs;
-	txp->regset.regs = txp_regs;
-	txp->regset.nregs = ARRAY_SIZE(txp_regs);
+	vc4_crtc->regset.base = txp->regs;
+	vc4_crtc->regset.regs = txp_regs;
+	vc4_crtc->regset.nregs = ARRAY_SIZE(txp_regs);
 
 	drm_connector_helper_add(&txp->connector.base,
 				 &vc4_txp_connector_helper_funcs);
@@ -523,9 +541,6 @@ static int vc4_txp_bind(struct device *dev, struct device *master, void *data)
 		return ret;
 
 	dev_set_drvdata(dev, txp);
-	vc4->txp = txp;
-
-	vc4_debugfs_add_regset32(drm, "txp_regs", &txp->regset);
 
 	return 0;
 }
@@ -533,13 +548,9 @@ static int vc4_txp_bind(struct device *dev, struct device *master, void *data)
 static void vc4_txp_unbind(struct device *dev, struct device *master,
 			   void *data)
 {
-	struct drm_device *drm = dev_get_drvdata(master);
-	struct vc4_dev *vc4 = to_vc4_dev(drm);
 	struct vc4_txp *txp = dev_get_drvdata(dev);
 
-	vc4_txp_connector_destroy(&txp->connector.base);
-
-	vc4->txp = NULL;
+	drm_connector_cleanup(&txp->connector.base);
 }
 
 static const struct component_ops vc4_txp_ops = {
diff --git a/drivers/gpu/drm/vc4/vc4_v3d.c b/drivers/gpu/drm/vc4/vc4_v3d.c
index cc714dcfe1f2..56abb0d6bc39 100644
--- a/drivers/gpu/drm/vc4/vc4_v3d.c
+++ b/drivers/gpu/drm/vc4/vc4_v3d.c
@@ -231,7 +231,7 @@ try_again:
  * if it doesn't fit within the buffer that we allocated up front.
  * However, it turns out that 16MB is "enough for anybody", and
  * real-world applications run into allocation failures from the
- * overall CMA pool before they make scenes complicated enough to run
+ * overall DMA pool before they make scenes complicated enough to run
  * out of bin space.
  */
 static int bin_bo_alloc(struct vc4_dev *vc4)
@@ -261,15 +261,15 @@ static int bin_bo_alloc(struct vc4_dev *vc4)
 
 			dev_err(&v3d->pdev->dev,
 				"Failed to allocate memory for tile binning: "
-				"%d. You may need to enable CMA or give it "
+				"%d. You may need to enable DMA or give it "
 				"more memory.",
 				ret);
 			break;
 		}
 
 		/* Check if this BO won't trigger the addressing bug. */
-		if ((bo->base.paddr & 0xf0000000) ==
-		    ((bo->base.paddr + bo->base.base.size - 1) & 0xf0000000)) {
+		if ((bo->base.dma_addr & 0xf0000000) ==
+		    ((bo->base.dma_addr + bo->base.base.size - 1) & 0xf0000000)) {
 			vc4->bin_bo = bo;
 
 			/* Set up for allocating 512KB chunks of
@@ -393,14 +393,34 @@ static int vc4_v3d_runtime_resume(struct device *dev)
 
 	vc4_v3d_init_hw(&vc4->base);
 
-	/* We disabled the IRQ as part of vc4_irq_uninstall in suspend. */
-	enable_irq(vc4->irq);
 	vc4_irq_enable(&vc4->base);
 
 	return 0;
 }
 #endif
 
+int vc4_v3d_debugfs_init(struct drm_minor *minor)
+{
+	struct drm_device *drm = minor->dev;
+	struct vc4_dev *vc4 = to_vc4_dev(drm);
+	struct vc4_v3d *v3d = vc4->v3d;
+	int ret;
+
+	if (!vc4->v3d)
+		return -ENODEV;
+
+	ret = vc4_debugfs_add_file(minor, "v3d_ident",
+				   vc4_v3d_debugfs_ident, NULL);
+	if (ret)
+		return ret;
+
+	ret = vc4_debugfs_add_regset32(minor, "v3d_regs", &v3d->regset);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
 static int vc4_v3d_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
@@ -443,44 +463,47 @@ static int vc4_v3d_bind(struct device *dev, struct device *master, void *data)
 		}
 	}
 
+	ret = platform_get_irq(pdev, 0);
+	if (ret < 0)
+		return ret;
+	vc4->irq = ret;
+
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return ret;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret)
+		return ret;
+
 	if (V3D_READ(V3D_IDENT0) != V3D_EXPECTED_IDENT0) {
 		DRM_ERROR("V3D_IDENT0 read 0x%08x instead of 0x%08x\n",
 			  V3D_READ(V3D_IDENT0), V3D_EXPECTED_IDENT0);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto err_put_runtime_pm;
 	}
 
-	ret = clk_prepare_enable(v3d->clk);
-	if (ret != 0)
-		return ret;
-
 	/* Reset the binner overflow address/size at setup, to be sure
 	 * we don't reuse an old one.
 	 */
 	V3D_WRITE(V3D_BPOA, 0);
 	V3D_WRITE(V3D_BPOS, 0);
 
-	vc4_v3d_init_hw(drm);
-
-	ret = platform_get_irq(pdev, 0);
-	if (ret < 0)
-		return ret;
-	vc4->irq = ret;
-
 	ret = vc4_irq_install(drm, vc4->irq);
 	if (ret) {
 		DRM_ERROR("Failed to install IRQ handler\n");
-		return ret;
+		goto err_put_runtime_pm;
 	}
 
-	pm_runtime_set_active(dev);
 	pm_runtime_use_autosuspend(dev);
 	pm_runtime_set_autosuspend_delay(dev, 40); /* a little over 2 frames. */
-	pm_runtime_enable(dev);
-
-	vc4_debugfs_add_file(drm, "v3d_ident", vc4_v3d_debugfs_ident, NULL);
-	vc4_debugfs_add_regset32(drm, "v3d_regs", &v3d->regset);
 
 	return 0;
+
+err_put_runtime_pm:
+	pm_runtime_put(dev);
+
+	return ret;
 }
 
 static void vc4_v3d_unbind(struct device *dev, struct device *master,
@@ -489,8 +512,6 @@ static void vc4_v3d_unbind(struct device *dev, struct device *master,
 	struct drm_device *drm = dev_get_drvdata(master);
 	struct vc4_dev *vc4 = to_vc4_dev(drm);
 
-	pm_runtime_disable(dev);
-
 	vc4_irq_uninstall(drm);
 
 	/* Disable the binner's overflow memory address, so the next
diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c
index 2feba55bcef7..520231af4df9 100644
--- a/drivers/gpu/drm/vc4/vc4_validate.c
+++ b/drivers/gpu/drm/vc4/vc4_validate.c
@@ -102,11 +102,11 @@ size_is_lt(uint32_t width, uint32_t height, int cpp)
 		height <= 4 * utile_height(cpp));
 }
 
-struct drm_gem_cma_object *
+struct drm_gem_dma_object *
 vc4_use_bo(struct vc4_exec_info *exec, uint32_t hindex)
 {
 	struct vc4_dev *vc4 = exec->dev;
-	struct drm_gem_cma_object *obj;
+	struct drm_gem_dma_object *obj;
 	struct vc4_bo *bo;
 
 	if (WARN_ON_ONCE(vc4->is_vc5))
@@ -129,7 +129,7 @@ vc4_use_bo(struct vc4_exec_info *exec, uint32_t hindex)
 	return obj;
 }
 
-static struct drm_gem_cma_object *
+static struct drm_gem_dma_object *
 vc4_use_handle(struct vc4_exec_info *exec, uint32_t gem_handles_packet_index)
 {
 	return vc4_use_bo(exec, exec->bo_index[gem_handles_packet_index]);
@@ -160,7 +160,7 @@ gl_shader_rec_size(uint32_t pointer_bits)
 }
 
 bool
-vc4_check_tex_size(struct vc4_exec_info *exec, struct drm_gem_cma_object *fbo,
+vc4_check_tex_size(struct vc4_exec_info *exec, struct drm_gem_dma_object *fbo,
 		   uint32_t offset, uint8_t tiling_format,
 		   uint32_t width, uint32_t height, uint8_t cpp)
 {
@@ -263,7 +263,7 @@ validate_increment_semaphore(VALIDATE_ARGS)
 static int
 validate_indexed_prim_list(VALIDATE_ARGS)
 {
-	struct drm_gem_cma_object *ib;
+	struct drm_gem_dma_object *ib;
 	uint32_t length = *(uint32_t *)(untrusted + 1);
 	uint32_t offset = *(uint32_t *)(untrusted + 5);
 	uint32_t max_index = *(uint32_t *)(untrusted + 9);
@@ -294,7 +294,7 @@ validate_indexed_prim_list(VALIDATE_ARGS)
 		return -EINVAL;
 	}
 
-	*(uint32_t *)(validated + 5) = ib->paddr + offset;
+	*(uint32_t *)(validated + 5) = ib->dma_addr + offset;
 
 	return 0;
 }
@@ -400,7 +400,7 @@ validate_tile_binning_config(VALIDATE_ARGS)
 	 * free when the job completes rendering.
 	 */
 	exec->bin_slots |= BIT(bin_slot);
-	bin_addr = vc4->bin_bo->base.paddr + bin_slot * vc4->bin_alloc_size;
+	bin_addr = vc4->bin_bo->base.dma_addr + bin_slot * vc4->bin_alloc_size;
 
 	/* The tile state data array is 48 bytes per tile, and we put it at
 	 * the start of a BO containing both it and the tile alloc.
@@ -575,7 +575,7 @@ reloc_tex(struct vc4_exec_info *exec,
 	  struct vc4_texture_sample_info *sample,
 	  uint32_t texture_handle_index, bool is_cs)
 {
-	struct drm_gem_cma_object *tex;
+	struct drm_gem_dma_object *tex;
 	uint32_t p0 = *(uint32_t *)(uniform_data_u + sample->p_offset[0]);
 	uint32_t p1 = *(uint32_t *)(uniform_data_u + sample->p_offset[1]);
 	uint32_t p2 = (sample->p_offset[2] != ~0 ?
@@ -608,7 +608,7 @@ reloc_tex(struct vc4_exec_info *exec,
 				  "outside of UBO\n");
 			goto fail;
 		}
-		*validated_p0 = tex->paddr + p0;
+		*validated_p0 = tex->dma_addr + p0;
 		return true;
 	}
 
@@ -736,7 +736,7 @@ reloc_tex(struct vc4_exec_info *exec,
 		offset -= level_size;
 	}
 
-	*validated_p0 = tex->paddr + p0;
+	*validated_p0 = tex->dma_addr + p0;
 
 	if (is_cs) {
 		exec->bin_dep_seqno = max(exec->bin_dep_seqno,
@@ -765,7 +765,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 		28, /* cs */
 	};
 	uint32_t shader_reloc_count = ARRAY_SIZE(shader_reloc_offsets);
-	struct drm_gem_cma_object *bo[ARRAY_SIZE(shader_reloc_offsets) + 8];
+	struct drm_gem_dma_object *bo[ARRAY_SIZE(shader_reloc_offsets) + 8];
 	uint32_t nr_attributes, nr_relocs, packet_size;
 	int i;
 
@@ -840,7 +840,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 		void *uniform_data_u;
 		uint32_t tex, uni;
 
-		*(uint32_t *)(pkt_v + o) = bo[i]->paddr + src_offset;
+		*(uint32_t *)(pkt_v + o) = bo[i]->dma_addr + src_offset;
 
 		if (src_offset != 0) {
 			DRM_DEBUG("Shaders must be at offset 0 of "
@@ -896,7 +896,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 	}
 
 	for (i = 0; i < nr_attributes; i++) {
-		struct drm_gem_cma_object *vbo =
+		struct drm_gem_dma_object *vbo =
 			bo[ARRAY_SIZE(shader_reloc_offsets) + i];
 		uint32_t o = 36 + i * 8;
 		uint32_t offset = *(uint32_t *)(pkt_u + o + 0);
@@ -928,7 +928,7 @@ validate_gl_shader_rec(struct drm_device *dev,
 			}
 		}
 
-		*(uint32_t *)(pkt_v + o) = vbo->paddr + offset;
+		*(uint32_t *)(pkt_v + o) = vbo->dma_addr + offset;
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/vc4/vc4_validate_shaders.c b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
index e315aeb5fef5..9745f8810eca 100644
--- a/drivers/gpu/drm/vc4/vc4_validate_shaders.c
+++ b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
@@ -776,7 +776,7 @@ vc4_handle_branch_target(struct vc4_shader_validation_state *validation_state)
 }
 
 struct vc4_validated_shader_info *
-vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
+vc4_validate_shader(struct drm_gem_dma_object *shader_obj)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(shader_obj->base.dev);
 	bool found_shader_end = false;
diff --git a/drivers/gpu/drm/vc4/vc4_vec.c b/drivers/gpu/drm/vc4/vc4_vec.c
index 11fc3d6f66b1..0b3333865702 100644
--- a/drivers/gpu/drm/vc4/vc4_vec.c
+++ b/drivers/gpu/drm/vc4/vc4_vec.c
@@ -14,6 +14,7 @@
  */
 
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_probe_helper.h>
@@ -160,48 +161,28 @@ struct vc4_vec_variant {
 
 /* General VEC hardware state. */
 struct vc4_vec {
+	struct vc4_encoder encoder;
+	struct drm_connector connector;
+
 	struct platform_device *pdev;
 	const struct vc4_vec_variant *variant;
 
-	struct drm_encoder *encoder;
-	struct drm_connector *connector;
-
 	void __iomem *regs;
 
 	struct clk *clock;
 
-	const struct vc4_vec_tv_mode *tv_mode;
-
 	struct debugfs_regset32 regset;
 };
 
 #define VEC_READ(offset) readl(vec->regs + (offset))
 #define VEC_WRITE(offset, val) writel(val, vec->regs + (offset))
 
-/* VC4 VEC encoder KMS struct */
-struct vc4_vec_encoder {
-	struct vc4_encoder base;
-	struct vc4_vec *vec;
-};
-
-static inline struct vc4_vec_encoder *
-to_vc4_vec_encoder(struct drm_encoder *encoder)
+static inline struct vc4_vec *
+encoder_to_vc4_vec(struct drm_encoder *encoder)
 {
-	return container_of(encoder, struct vc4_vec_encoder, base.base);
+	return container_of(encoder, struct vc4_vec, encoder.base);
 }
 
-/* VC4 VEC connector KMS struct */
-struct vc4_vec_connector {
-	struct drm_connector base;
-	struct vc4_vec *vec;
-
-	/* Since the connector is attached to just the one encoder,
-	 * this is the reference to it so we can do the best_encoder()
-	 * hook.
-	 */
-	struct drm_encoder *encoder;
-};
-
 enum vc4_vec_tv_mode_id {
 	VC4_VEC_TV_MODE_NTSC,
 	VC4_VEC_TV_MODE_NTSC_J,
@@ -211,7 +192,9 @@ enum vc4_vec_tv_mode_id {
 
 struct vc4_vec_tv_mode {
 	const struct drm_display_mode *mode;
-	void (*mode_set)(struct vc4_vec *vec);
+	u32 config0;
+	u32 config1;
+	u32 custom_freq;
 };
 
 static const struct debugfs_reg32 vec_regs[] = {
@@ -241,63 +224,41 @@ static const struct debugfs_reg32 vec_regs[] = {
 	VC4_REG32(VEC_DAC_MISC),
 };
 
-static void vc4_vec_ntsc_mode_set(struct vc4_vec *vec)
-{
-	VEC_WRITE(VEC_CONFIG0, VEC_CONFIG0_NTSC_STD | VEC_CONFIG0_PDEN);
-	VEC_WRITE(VEC_CONFIG1, VEC_CONFIG1_C_CVBS_CVBS);
-}
-
-static void vc4_vec_ntsc_j_mode_set(struct vc4_vec *vec)
-{
-	VEC_WRITE(VEC_CONFIG0, VEC_CONFIG0_NTSC_STD);
-	VEC_WRITE(VEC_CONFIG1, VEC_CONFIG1_C_CVBS_CVBS);
-}
-
 static const struct drm_display_mode ntsc_mode = {
 	DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 13500,
 		 720, 720 + 14, 720 + 14 + 64, 720 + 14 + 64 + 60, 0,
-		 480, 480 + 3, 480 + 3 + 3, 480 + 3 + 3 + 16, 0,
+		 480, 480 + 7, 480 + 7 + 6, 525, 0,
 		 DRM_MODE_FLAG_INTERLACE)
 };
 
-static void vc4_vec_pal_mode_set(struct vc4_vec *vec)
-{
-	VEC_WRITE(VEC_CONFIG0, VEC_CONFIG0_PAL_BDGHI_STD);
-	VEC_WRITE(VEC_CONFIG1, VEC_CONFIG1_C_CVBS_CVBS);
-}
-
-static void vc4_vec_pal_m_mode_set(struct vc4_vec *vec)
-{
-	VEC_WRITE(VEC_CONFIG0, VEC_CONFIG0_PAL_BDGHI_STD);
-	VEC_WRITE(VEC_CONFIG1,
-		  VEC_CONFIG1_C_CVBS_CVBS | VEC_CONFIG1_CUSTOM_FREQ);
-	VEC_WRITE(VEC_FREQ3_2, 0x223b);
-	VEC_WRITE(VEC_FREQ1_0, 0x61d1);
-}
-
 static const struct drm_display_mode pal_mode = {
 	DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 13500,
 		 720, 720 + 20, 720 + 20 + 64, 720 + 20 + 64 + 60, 0,
-		 576, 576 + 2, 576 + 2 + 3, 576 + 2 + 3 + 20, 0,
+		 576, 576 + 4, 576 + 4 + 6, 625, 0,
 		 DRM_MODE_FLAG_INTERLACE)
 };
 
 static const struct vc4_vec_tv_mode vc4_vec_tv_modes[] = {
 	[VC4_VEC_TV_MODE_NTSC] = {
 		.mode = &ntsc_mode,
-		.mode_set = vc4_vec_ntsc_mode_set,
+		.config0 = VEC_CONFIG0_NTSC_STD | VEC_CONFIG0_PDEN,
+		.config1 = VEC_CONFIG1_C_CVBS_CVBS,
 	},
 	[VC4_VEC_TV_MODE_NTSC_J] = {
 		.mode = &ntsc_mode,
-		.mode_set = vc4_vec_ntsc_j_mode_set,
+		.config0 = VEC_CONFIG0_NTSC_STD,
+		.config1 = VEC_CONFIG1_C_CVBS_CVBS,
 	},
 	[VC4_VEC_TV_MODE_PAL] = {
 		.mode = &pal_mode,
-		.mode_set = vc4_vec_pal_mode_set,
+		.config0 = VEC_CONFIG0_PAL_BDGHI_STD,
+		.config1 = VEC_CONFIG1_C_CVBS_CVBS,
 	},
 	[VC4_VEC_TV_MODE_PAL_M] = {
 		.mode = &pal_mode,
-		.mode_set = vc4_vec_pal_m_mode_set,
+		.config0 = VEC_CONFIG0_PAL_BDGHI_STD,
+		.config1 = VEC_CONFIG1_C_CVBS_CVBS | VEC_CONFIG1_CUSTOM_FREQ,
+		.custom_freq = 0x223b61d1,
 	},
 };
 
@@ -307,12 +268,6 @@ vc4_vec_connector_detect(struct drm_connector *connector, bool force)
 	return connector_status_unknown;
 }
 
-static void vc4_vec_connector_destroy(struct drm_connector *connector)
-{
-	drm_connector_unregister(connector);
-	drm_connector_cleanup(connector);
-}
-
 static int vc4_vec_connector_get_modes(struct drm_connector *connector)
 {
 	struct drm_connector_state *state = connector->state;
@@ -333,7 +288,6 @@ static int vc4_vec_connector_get_modes(struct drm_connector *connector)
 static const struct drm_connector_funcs vc4_vec_connector_funcs = {
 	.detect = vc4_vec_connector_detect,
 	.fill_modes = drm_helper_probe_single_connector_modes,
-	.destroy = vc4_vec_connector_destroy,
 	.reset = drm_atomic_helper_connector_reset,
 	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
@@ -343,42 +297,38 @@ static const struct drm_connector_helper_funcs vc4_vec_connector_helper_funcs =
 	.get_modes = vc4_vec_connector_get_modes,
 };
 
-static struct drm_connector *vc4_vec_connector_init(struct drm_device *dev,
-						    struct vc4_vec *vec)
+static int vc4_vec_connector_init(struct drm_device *dev, struct vc4_vec *vec)
 {
-	struct drm_connector *connector = NULL;
-	struct vc4_vec_connector *vec_connector;
-
-	vec_connector = devm_kzalloc(dev->dev, sizeof(*vec_connector),
-				     GFP_KERNEL);
-	if (!vec_connector)
-		return ERR_PTR(-ENOMEM);
+	struct drm_connector *connector = &vec->connector;
+	int ret;
 
-	connector = &vec_connector->base;
 	connector->interlace_allowed = true;
 
-	vec_connector->encoder = vec->encoder;
-	vec_connector->vec = vec;
+	ret = drmm_connector_init(dev, connector, &vc4_vec_connector_funcs,
+				 DRM_MODE_CONNECTOR_Composite, NULL);
+	if (ret)
+		return ret;
 
-	drm_connector_init(dev, connector, &vc4_vec_connector_funcs,
-			   DRM_MODE_CONNECTOR_Composite);
 	drm_connector_helper_add(connector, &vc4_vec_connector_helper_funcs);
 
 	drm_object_attach_property(&connector->base,
 				   dev->mode_config.tv_mode_property,
 				   VC4_VEC_TV_MODE_NTSC);
-	vec->tv_mode = &vc4_vec_tv_modes[VC4_VEC_TV_MODE_NTSC];
 
-	drm_connector_attach_encoder(connector, vec->encoder);
+	drm_connector_attach_encoder(connector, &vec->encoder.base);
 
-	return connector;
+	return 0;
 }
 
-static void vc4_vec_encoder_disable(struct drm_encoder *encoder)
+static void vc4_vec_encoder_disable(struct drm_encoder *encoder,
+				    struct drm_atomic_state *state)
 {
-	struct vc4_vec_encoder *vc4_vec_encoder = to_vc4_vec_encoder(encoder);
-	struct vc4_vec *vec = vc4_vec_encoder->vec;
-	int ret;
+	struct drm_device *drm = encoder->dev;
+	struct vc4_vec *vec = encoder_to_vc4_vec(encoder);
+	int idx, ret;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	VEC_WRITE(VEC_CFG, 0);
 	VEC_WRITE(VEC_DAC_MISC,
@@ -392,20 +342,35 @@ static void vc4_vec_encoder_disable(struct drm_encoder *encoder)
 	ret = pm_runtime_put(&vec->pdev->dev);
 	if (ret < 0) {
 		DRM_ERROR("Failed to release power domain: %d\n", ret);
-		return;
+		goto err_dev_exit;
 	}
+
+	drm_dev_exit(idx);
+	return;
+
+err_dev_exit:
+	drm_dev_exit(idx);
 }
 
-static void vc4_vec_encoder_enable(struct drm_encoder *encoder)
+static void vc4_vec_encoder_enable(struct drm_encoder *encoder,
+				   struct drm_atomic_state *state)
 {
-	struct vc4_vec_encoder *vc4_vec_encoder = to_vc4_vec_encoder(encoder);
-	struct vc4_vec *vec = vc4_vec_encoder->vec;
-	int ret;
+	struct drm_device *drm = encoder->dev;
+	struct vc4_vec *vec = encoder_to_vc4_vec(encoder);
+	struct drm_connector *connector = &vec->connector;
+	struct drm_connector_state *conn_state =
+		drm_atomic_get_new_connector_state(state, connector);
+	const struct vc4_vec_tv_mode *tv_mode =
+		&vc4_vec_tv_modes[conn_state->tv.mode];
+	int idx, ret;
+
+	if (!drm_dev_enter(drm, &idx))
+		return;
 
 	ret = pm_runtime_get_sync(&vec->pdev->dev);
 	if (ret < 0) {
 		DRM_ERROR("Failed to retain power domain: %d\n", ret);
-		return;
+		goto err_dev_exit;
 	}
 
 	/*
@@ -418,13 +383,13 @@ static void vc4_vec_encoder_enable(struct drm_encoder *encoder)
 	ret = clk_set_rate(vec->clock, 108000000);
 	if (ret) {
 		DRM_ERROR("Failed to set clock rate: %d\n", ret);
-		return;
+		goto err_put_runtime_pm;
 	}
 
 	ret = clk_prepare_enable(vec->clock);
 	if (ret) {
 		DRM_ERROR("Failed to turn on core clock: %d\n", ret);
-		return;
+		goto err_put_runtime_pm;
 	}
 
 	/* Reset the different blocks */
@@ -455,29 +420,27 @@ static void vc4_vec_encoder_enable(struct drm_encoder *encoder)
 	/* Mask all interrupts. */
 	VEC_WRITE(VEC_MASK0, 0);
 
-	vec->tv_mode->mode_set(vec);
+	VEC_WRITE(VEC_CONFIG0, tv_mode->config0);
+	VEC_WRITE(VEC_CONFIG1, tv_mode->config1);
+
+	if (tv_mode->custom_freq) {
+		VEC_WRITE(VEC_FREQ3_2,
+			  (tv_mode->custom_freq >> 16) & 0xffff);
+		VEC_WRITE(VEC_FREQ1_0,
+			  tv_mode->custom_freq & 0xffff);
+	}
 
 	VEC_WRITE(VEC_DAC_MISC,
 		  VEC_DAC_MISC_VID_ACT | VEC_DAC_MISC_DAC_RST_N);
 	VEC_WRITE(VEC_CFG, VEC_CFG_VEC_EN);
-}
 
+	drm_dev_exit(idx);
+	return;
 
-static bool vc4_vec_encoder_mode_fixup(struct drm_encoder *encoder,
-				       const struct drm_display_mode *mode,
-				       struct drm_display_mode *adjusted_mode)
-{
-	return true;
-}
-
-static void vc4_vec_encoder_atomic_mode_set(struct drm_encoder *encoder,
-					struct drm_crtc_state *crtc_state,
-					struct drm_connector_state *conn_state)
-{
-	struct vc4_vec_encoder *vc4_vec_encoder = to_vc4_vec_encoder(encoder);
-	struct vc4_vec *vec = vc4_vec_encoder->vec;
-
-	vec->tv_mode = &vc4_vec_tv_modes[conn_state->tv.mode];
+err_put_runtime_pm:
+	pm_runtime_put(&vec->pdev->dev);
+err_dev_exit:
+	drm_dev_exit(idx);
 }
 
 static int vc4_vec_encoder_atomic_check(struct drm_encoder *encoder,
@@ -496,11 +459,27 @@ static int vc4_vec_encoder_atomic_check(struct drm_encoder *encoder,
 }
 
 static const struct drm_encoder_helper_funcs vc4_vec_encoder_helper_funcs = {
-	.disable = vc4_vec_encoder_disable,
-	.enable = vc4_vec_encoder_enable,
-	.mode_fixup = vc4_vec_encoder_mode_fixup,
 	.atomic_check = vc4_vec_encoder_atomic_check,
-	.atomic_mode_set = vc4_vec_encoder_atomic_mode_set,
+	.atomic_disable = vc4_vec_encoder_disable,
+	.atomic_enable = vc4_vec_encoder_enable,
+};
+
+static int vc4_vec_late_register(struct drm_encoder *encoder)
+{
+	struct drm_device *drm = encoder->dev;
+	struct vc4_vec *vec = encoder_to_vc4_vec(encoder);
+	int ret;
+
+	ret = vc4_debugfs_add_regset32(drm->primary, "vec_regs",
+				       &vec->regset);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static const struct drm_encoder_funcs vc4_vec_encoder_funcs = {
+	.late_register = vc4_vec_late_register,
 };
 
 static const struct vc4_vec_variant bcm2835_vec_variant = {
@@ -532,9 +511,7 @@ static int vc4_vec_bind(struct device *dev, struct device *master, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct drm_device *drm = dev_get_drvdata(master);
-	struct vc4_dev *vc4 = to_vc4_dev(drm);
 	struct vc4_vec *vec;
-	struct vc4_vec_encoder *vc4_vec_encoder;
 	int ret;
 
 	ret = drm_mode_create_tv_properties(drm, ARRAY_SIZE(tv_mode_names),
@@ -542,18 +519,11 @@ static int vc4_vec_bind(struct device *dev, struct device *master, void *data)
 	if (ret)
 		return ret;
 
-	vec = devm_kzalloc(dev, sizeof(*vec), GFP_KERNEL);
+	vec = drmm_kzalloc(drm, sizeof(*vec), GFP_KERNEL);
 	if (!vec)
 		return -ENOMEM;
 
-	vc4_vec_encoder = devm_kzalloc(dev, sizeof(*vc4_vec_encoder),
-				       GFP_KERNEL);
-	if (!vc4_vec_encoder)
-		return -ENOMEM;
-	vc4_vec_encoder->base.type = VC4_ENCODER_TYPE_VEC;
-	vc4_vec_encoder->vec = vec;
-	vec->encoder = &vc4_vec_encoder->base.base;
-
+	vec->encoder.type = VC4_ENCODER_TYPE_VEC;
 	vec->pdev = pdev;
 	vec->variant = (const struct vc4_vec_variant *)
 		of_device_get_match_data(dev);
@@ -572,49 +542,30 @@ static int vc4_vec_bind(struct device *dev, struct device *master, void *data)
 		return ret;
 	}
 
-	pm_runtime_enable(dev);
-
-	drm_simple_encoder_init(drm, vec->encoder, DRM_MODE_ENCODER_TVDAC);
-	drm_encoder_helper_add(vec->encoder, &vc4_vec_encoder_helper_funcs);
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return ret;
 
-	vec->connector = vc4_vec_connector_init(drm, vec);
-	if (IS_ERR(vec->connector)) {
-		ret = PTR_ERR(vec->connector);
-		goto err_destroy_encoder;
-	}
+	ret = drmm_encoder_init(drm, &vec->encoder.base,
+				&vc4_vec_encoder_funcs,
+				DRM_MODE_ENCODER_TVDAC,
+				NULL);
+	if (ret)
+		return ret;
 
-	dev_set_drvdata(dev, vec);
+	drm_encoder_helper_add(&vec->encoder.base, &vc4_vec_encoder_helper_funcs);
 
-	vc4->vec = vec;
+	ret = vc4_vec_connector_init(drm, vec);
+	if (ret)
+		return ret;
 
-	vc4_debugfs_add_regset32(drm, "vec_regs", &vec->regset);
+	dev_set_drvdata(dev, vec);
 
 	return 0;
-
-err_destroy_encoder:
-	drm_encoder_cleanup(vec->encoder);
-	pm_runtime_disable(dev);
-
-	return ret;
-}
-
-static void vc4_vec_unbind(struct device *dev, struct device *master,
-			   void *data)
-{
-	struct drm_device *drm = dev_get_drvdata(master);
-	struct vc4_dev *vc4 = to_vc4_dev(drm);
-	struct vc4_vec *vec = dev_get_drvdata(dev);
-
-	vc4_vec_connector_destroy(vec->connector);
-	drm_encoder_cleanup(vec->encoder);
-	pm_runtime_disable(dev);
-
-	vc4->vec = NULL;
 }
 
 static const struct component_ops vc4_vec_ops = {
 	.bind   = vc4_vec_bind,
-	.unbind = vc4_vec_unbind,
 };
 
 static int vc4_vec_dev_probe(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/via/Makefile b/drivers/gpu/drm/via/Makefile
index 84db4eee7828..8b978dd51a25 100644
--- a/drivers/gpu/drm/via/Makefile
+++ b/drivers/gpu/drm/via/Makefile
@@ -3,6 +3,6 @@
 # Makefile for the drm device driver.  This driver provides support for the
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
-via-y    := via_irq.o via_drv.o via_map.o via_mm.o via_dma.o via_verifier.o via_video.o via_dmablit.o
+via-y    := via_dri1.o
 
 obj-$(CONFIG_DRM_VIA)	+=via.o
diff --git a/drivers/gpu/drm/via/via_3d_reg.h b/drivers/gpu/drm/via/via_3d_reg.h
index 462375d543b9..eb848508b12b 100644
--- a/drivers/gpu/drm/via/via_3d_reg.h
+++ b/drivers/gpu/drm/via/via_3d_reg.h
@@ -1,25 +1,7 @@
+/* SPDX-License-Identifier: MIT */
 /*
- * Copyright 1998-2003 VIA Technologies, Inc. All Rights Reserved.
- * Copyright 2001-2003 S3 Graphics, Inc. All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * VIA, S3 GRAPHICS, AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
+ * Copyright 1998-2011 VIA Technologies, Inc. All Rights Reserved.
+ * Copyright 2001-2011 S3 Graphics, Inc. All Rights Reserved.
  */
 
 #ifndef VIA_3D_REG_H
@@ -50,6 +32,7 @@
 #define HC_ParaType_Palette     0x0003
 #define HC_ParaType_PreCR       0x0010
 #define HC_ParaType_Auto        0x00fe
+#define INV_ParaType_Dummy          0x00300000
 
 /* Transmission Space
  */
@@ -173,10 +156,10 @@
 #define HC_HSPXOS_SHIFT         12
 #define HC_HSPYOS_MASK          0x00000fff
 
-/* Command
+/*
  * Command A
  */
-#define HC_HCmdHeader_MASK      0xfe000000	/*0xffe00000 */
+#define HC_HCmdHeader_MASK      0xfe000000  /*0xffe00000 */
 #define HC_HE3Fire_MASK         0x00100000
 #define HC_HPMType_MASK         0x000f0000
 #define HC_HEFlag_MASK          0x0000e000
@@ -236,6 +219,8 @@
 /* Enable Setting
  */
 #define HC_SubA_HEnable         0x0000
+#define HC_HenForce1P_MASK      0x00800000  /* [Force 1 Pipe] */
+#define HC_HenZDCheck_MASK      0x00400000  /* [Z dirty bit settings] */
 #define HC_HenTXEnvMap_MASK     0x00200000
 #define HC_HenVertexCNT_MASK    0x00100000
 #define HC_HenCPUDAZ_MASK       0x00080000
@@ -684,6 +669,12 @@
 
 /* Texture subtype definitions
  */
+#define HC_SubType_Samp0        0x00000020
+#define HC_SubType_Samp1        0x00000021
+
+
+/* Texture subtype definitions
+ */
 #define HC_SubType_Tex0         0x00000000
 #define HC_SubType_Tex1         0x00000001
 #define HC_SubType_TexGeneral   0x000000fe
@@ -762,7 +753,13 @@
 #define HC_SubA_HTXnBumpM10     0x0092
 #define HC_SubA_HTXnBumpM11     0x0093
 #define HC_SubA_HTXnLScale      0x0094
-#define HC_SubA_HTXSMD          0x0000
+
+#define HC_SubA_HTXSMD             0x0000
+#define HC_SubA_HTXYUV2RGB1        0x0001
+#define HC_SubA_HTXYUV2RGB2        0x0002
+#define HC_SubA_HTXYUV2RGB3        0x0003
+#define HTXYUV2RGB4BT601           (1<<23)
+#define HTXYUV2RGB4BT709           (1<<22)
 /* HC_SubA_HTXnL012BasH    0x0020
  */
 #define HC_HTXnL0BasH_MASK      0x000000ff
@@ -965,6 +962,7 @@
 #define HC_HTXnFM_Lum           0x00100000
 #define HC_HTXnFM_Alpha         0x00180000
 #define HC_HTXnFM_DX            0x00280000
+#define HC_HTXnFM_YUV           0x00300000
 #define HC_HTXnFM_ARGB16        0x00880000
 #define HC_HTXnFM_ARGB32        0x00980000
 #define HC_HTXnFM_ABGR16        0x00a80000
@@ -995,6 +993,12 @@
 #define HC_HTXnFM_DX1           (HC_HTXnFM_DX        | 0x00010000)
 #define HC_HTXnFM_DX23          (HC_HTXnFM_DX        | 0x00020000)
 #define HC_HTXnFM_DX45          (HC_HTXnFM_DX        | 0x00030000)
+/* YUV package mode */
+#define HC_HTXnFM_YUY2          (HC_HTXnFM_YUV           | 0x00000000)
+/* YUV planner mode */
+#define HC_HTXnFM_YV12          (HC_HTXnFM_YUV           | 0x00040000)
+/* YUV planner mode */
+#define HC_HTXnFM_IYUV          (HC_HTXnFM_YUV           | 0x00040000)
 #define HC_HTXnFM_RGB555        (HC_HTXnFM_ARGB16    | 0x00000000)
 #define HC_HTXnFM_RGB565        (HC_HTXnFM_ARGB16    | 0x00010000)
 #define HC_HTXnFM_ARGB1555      (HC_HTXnFM_ARGB16    | 0x00020000)
@@ -1023,6 +1027,13 @@
 #define HC_HTXnLoc_Local        0x00000000
 #define HC_HTXnLoc_Sys          0x00000002
 #define HC_HTXnLoc_AGP          0x00000003
+
+/* Video Texture */
+#define HC_HTXnYUV2RGBMode_RGB          0x00000000
+#define HC_HTXnYUV2RGBMode_SDTV         0x00000001
+#define HC_HTXnYUV2RGBMode_HDTV         0x00000002
+#define HC_HTXnYUV2RGBMode_TABLE        0x00000003
+
 /* HC_SubA_HTXnTRAH        0x007f
  */
 #define HC_HTXnTRAH_MASK        0x00ff0000
@@ -1330,9 +1341,9 @@
  */
 #define HC_HFthRTXA_MASK        0x000000ff
 
-/******************************************************************************
-** Define the Halcyon Internal register access constants. For simulator only.
-******************************************************************************/
+/****************************************************************************
+ * Define the Halcyon Internal register access constants. For simulator only.
+ ***************************************************************************/
 #define HC_SIMA_HAGPBstL        0x0000
 #define HC_SIMA_HAGPBendL       0x0001
 #define HC_SIMA_HAGPCMNT        0x0002
@@ -1477,80 +1488,80 @@
 #define HC_SIMA_TX0TX1_OFF      0x0050
 /*---- start of texture 1 setting ----
  */
-#define HC_SIMA_HTX1L0BasL      (HC_SIMA_HTX0L0BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L1BasL      (HC_SIMA_HTX0L1BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L2BasL      (HC_SIMA_HTX0L2BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L3BasL      (HC_SIMA_HTX0L3BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L4BasL      (HC_SIMA_HTX0L4BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L5BasL      (HC_SIMA_HTX0L5BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L6BasL      (HC_SIMA_HTX0L6BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L7BasL      (HC_SIMA_HTX0L7BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L8BasL      (HC_SIMA_HTX0L8BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L9BasL      (HC_SIMA_HTX0L9BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LaBasL      (HC_SIMA_HTX0LaBasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LbBasL      (HC_SIMA_HTX0LbBasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LcBasL      (HC_SIMA_HTX0LcBasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LdBasL      (HC_SIMA_HTX0LdBasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LeBasL      (HC_SIMA_HTX0LeBasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LfBasL      (HC_SIMA_HTX0LfBasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L10BasL     (HC_SIMA_HTX0L10BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L11BasL     (HC_SIMA_HTX0L11BasL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L012BasH    (HC_SIMA_HTX0L012BasH + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L345BasH    (HC_SIMA_HTX0L345BasH + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L678BasH    (HC_SIMA_HTX0L678BasH + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L9abBasH    (HC_SIMA_HTX0L9abBasH + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LcdeBasH    (HC_SIMA_HTX0LcdeBasH + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1Lf1011BasH  (HC_SIMA_HTX0Lf1011BasH + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L0Pit       (HC_SIMA_HTX0L0Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L1Pit       (HC_SIMA_HTX0L1Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L2Pit       (HC_SIMA_HTX0L2Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L3Pit       (HC_SIMA_HTX0L3Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L4Pit       (HC_SIMA_HTX0L4Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L5Pit       (HC_SIMA_HTX0L5Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L6Pit       (HC_SIMA_HTX0L6Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L7Pit       (HC_SIMA_HTX0L7Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L8Pit       (HC_SIMA_HTX0L8Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L9Pit       (HC_SIMA_HTX0L9Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LaPit       (HC_SIMA_HTX0LaPit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LbPit       (HC_SIMA_HTX0LbPit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LcPit       (HC_SIMA_HTX0LcPit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LdPit       (HC_SIMA_HTX0LdPit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LePit       (HC_SIMA_HTX0LePit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LfPit       (HC_SIMA_HTX0LfPit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L10Pit      (HC_SIMA_HTX0L10Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L11Pit      (HC_SIMA_HTX0L11Pit + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L0_5WE      (HC_SIMA_HTX0L0_5WE + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L6_bWE      (HC_SIMA_HTX0L6_bWE + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1Lc_11WE     (HC_SIMA_HTX0Lc_11WE + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L0_5HE      (HC_SIMA_HTX0L0_5HE + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L6_bHE      (HC_SIMA_HTX0L6_bHE + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L0BasL       (HC_SIMA_HTX0L0BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L1BasL       (HC_SIMA_HTX0L1BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L2BasL       (HC_SIMA_HTX0L2BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L3BasL       (HC_SIMA_HTX0L3BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L4BasL       (HC_SIMA_HTX0L4BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L5BasL       (HC_SIMA_HTX0L5BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L6BasL       (HC_SIMA_HTX0L6BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L7BasL       (HC_SIMA_HTX0L7BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L8BasL       (HC_SIMA_HTX0L8BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L9BasL       (HC_SIMA_HTX0L9BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LaBasL       (HC_SIMA_HTX0LaBasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LbBasL       (HC_SIMA_HTX0LbBasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LcBasL       (HC_SIMA_HTX0LcBasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LdBasL       (HC_SIMA_HTX0LdBasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LeBasL       (HC_SIMA_HTX0LeBasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LfBasL       (HC_SIMA_HTX0LfBasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L10BasL      (HC_SIMA_HTX0L10BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L11BasL      (HC_SIMA_HTX0L11BasL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L012BasH     (HC_SIMA_HTX0L012BasH + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L345BasH     (HC_SIMA_HTX0L345BasH + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L678BasH     (HC_SIMA_HTX0L678BasH + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L9abBasH     (HC_SIMA_HTX0L9abBasH + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LcdeBasH     (HC_SIMA_HTX0LcdeBasH + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1Lf1011BasH   (HC_SIMA_HTX0Lf1011BasH + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L0Pit        (HC_SIMA_HTX0L0Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L1Pit        (HC_SIMA_HTX0L1Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L2Pit        (HC_SIMA_HTX0L2Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L3Pit        (HC_SIMA_HTX0L3Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L4Pit        (HC_SIMA_HTX0L4Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L5Pit        (HC_SIMA_HTX0L5Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L6Pit        (HC_SIMA_HTX0L6Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L7Pit        (HC_SIMA_HTX0L7Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L8Pit        (HC_SIMA_HTX0L8Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L9Pit        (HC_SIMA_HTX0L9Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LaPit        (HC_SIMA_HTX0LaPit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LbPit        (HC_SIMA_HTX0LbPit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LcPit        (HC_SIMA_HTX0LcPit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LdPit        (HC_SIMA_HTX0LdPit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LePit        (HC_SIMA_HTX0LePit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LfPit        (HC_SIMA_HTX0LfPit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L10Pit       (HC_SIMA_HTX0L10Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L11Pit       (HC_SIMA_HTX0L11Pit + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L0_5WE       (HC_SIMA_HTX0L0_5WE + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L6_bWE       (HC_SIMA_HTX0L6_bWE + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1Lc_11WE      (HC_SIMA_HTX0Lc_11WE + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L0_5HE       (HC_SIMA_HTX0L0_5HE + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L6_bHE       (HC_SIMA_HTX0L6_bHE + HC_SIMA_TX0TX1_OFF)
 #define HC_SIMA_HTX1Lc_11HE      (HC_SIMA_HTX0Lc_11HE + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1L0OS        (HC_SIMA_HTX0L0OS + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TB          (HC_SIMA_HTX0TB + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1MPMD        (HC_SIMA_HTX0MPMD + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1CLODu       (HC_SIMA_HTX0CLODu + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1FM          (HC_SIMA_HTX0FM + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TRCH        (HC_SIMA_HTX0TRCH + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TRCL        (HC_SIMA_HTX0TRCL + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBC         (HC_SIMA_HTX0TBC + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TRAH        (HC_SIMA_HTX0TRAH + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LTC         (HC_SIMA_HTX0LTC + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LTA         (HC_SIMA_HTX0LTA + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBLCsat     (HC_SIMA_HTX0TBLCsat + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBLCop      (HC_SIMA_HTX0TBLCop + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBLMPfog    (HC_SIMA_HTX0TBLMPfog + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBLAsat     (HC_SIMA_HTX0TBLAsat + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBLRCa      (HC_SIMA_HTX0TBLRCa + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBLRCb      (HC_SIMA_HTX0TBLRCb + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBLRCc      (HC_SIMA_HTX0TBLRCc + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBLRCbias   (HC_SIMA_HTX0TBLRCbias + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBLRAa      (HC_SIMA_HTX0TBLRAa + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1TBLRFog     (HC_SIMA_HTX0TBLRFog + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1BumpM00     (HC_SIMA_HTX0BumpM00 + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1BumpM01     (HC_SIMA_HTX0BumpM01 + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1BumpM10     (HC_SIMA_HTX0BumpM10 + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1BumpM11     (HC_SIMA_HTX0BumpM11 + HC_SIMA_TX0TX1_OFF)
-#define HC_SIMA_HTX1LScale      (HC_SIMA_HTX0LScale + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1L0OS         (HC_SIMA_HTX0L0OS + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TB           (HC_SIMA_HTX0TB + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1MPMD         (HC_SIMA_HTX0MPMD + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1CLODu        (HC_SIMA_HTX0CLODu + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1FM           (HC_SIMA_HTX0FM + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TRCH         (HC_SIMA_HTX0TRCH + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TRCL         (HC_SIMA_HTX0TRCL + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBC          (HC_SIMA_HTX0TBC + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TRAH         (HC_SIMA_HTX0TRAH + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LTC          (HC_SIMA_HTX0LTC + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LTA          (HC_SIMA_HTX0LTA + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBLCsat      (HC_SIMA_HTX0TBLCsat + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBLCop       (HC_SIMA_HTX0TBLCop + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBLMPfog     (HC_SIMA_HTX0TBLMPfog + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBLAsat      (HC_SIMA_HTX0TBLAsat + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBLRCa       (HC_SIMA_HTX0TBLRCa + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBLRCb       (HC_SIMA_HTX0TBLRCb + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBLRCc       (HC_SIMA_HTX0TBLRCc + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBLRCbias    (HC_SIMA_HTX0TBLRCbias + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBLRAa       (HC_SIMA_HTX0TBLRAa + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1TBLRFog      (HC_SIMA_HTX0TBLRFog + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1BumpM00      (HC_SIMA_HTX0BumpM00 + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1BumpM01      (HC_SIMA_HTX0BumpM01 + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1BumpM10      (HC_SIMA_HTX0BumpM10 + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1BumpM11      (HC_SIMA_HTX0BumpM11 + HC_SIMA_TX0TX1_OFF)
+#define HC_SIMA_HTX1LScale       (HC_SIMA_HTX0LScale + HC_SIMA_TX0TX1_OFF)
 /*---- end of texture 1 setting ---- 0xaf
  */
 #define HC_SIMA_HTXSMD          0x00b0
@@ -1580,9 +1591,9 @@
 #define HC_SIMA_HRErr           0x0445
 #define HC_SIMA_FIFOstatus      0x0446
 
-/******************************************************************************
-** Define the AGP command header.
-******************************************************************************/
+/****************************************************************************
+ * Define the AGP command header.
+ ***************************************************************************/
 #define HC_ACMD_MASK            0xfe000000
 #define HC_ACMD_SUB_MASK        0x0c000000
 #define HC_ACMD_HCmdA           0xee000000
@@ -1605,18 +1616,18 @@
 #define HC_ACMD_H4COUNT_MASK    0x01fffe00
 #define HC_ACMD_H4COUNT_SHIFT   9
 
-/********************************************************************************
-** Define Header
-********************************************************************************/
-#define HC_HEADER2		0xF210F110
+/*****************************************************************************
+ * Define Header
+ ****************************************************************************/
+#define HC_HEADER2        0xF210F110
 
-/********************************************************************************
-** Define Dummy Value
-********************************************************************************/
-#define HC_DUMMY		0xCCCCCCCC
-/********************************************************************************
-** Define for DMA use
-********************************************************************************/
+/*****************************************************************************
+ * Define Dummy Value
+ ****************************************************************************/
+#define HC_DUMMY        0xCCCCCCCC
+/*****************************************************************************
+ * Define for DMA use
+ ****************************************************************************/
 #define HALCYON_HEADER2     0XF210F110
 #define HALCYON_FIRECMD     0XEE100000
 #define HALCYON_FIREMASK    0XFFF00000
@@ -1643,8 +1654,118 @@
 #define HC_HAGPBpID_STOP        0x00000002
 #define HC_HAGPBpH_MASK         0x00ffffff
 
+
 #define VIA_VIDEO_HEADER5       0xFE040000
 #define VIA_VIDEO_HEADER6       0xFE050000
 #define VIA_VIDEO_HEADER7       0xFE060000
 #define VIA_VIDEOMASK           0xFFFF0000
+
+/*****************************************************************************
+ * Define for H5 DMA use
+ ****************************************************************************/
+#define H5_HC_DUMMY                    0xCC000000
+
+/* Command Header Type */
+#define INV_DUMMY_MASK		0xFF000000
+#define INV_AGPHeader0              0xFE000000
+#define INV_AGPHeader1              0xFE010000
+#define INV_AGPHeader2              0xFE020000
+#define INV_AGPHeader3              0xFE030000
+#define INV_AGPHeader4              0xFE040000
+#define INV_AGPHeader5              0xFE050000
+#define INV_AGPHeader6              0xFE060000
+#define INV_AGPHeader7              0xFE070000
+#define INV_AGPHeader9              0xFE090000
+#define INV_AGPHeaderA              0xFE0A0000
+#define INV_AGPHeader40             0xFE400000
+#define INV_AGPHeader41             0xFE410000
+#define INV_AGPHeader43             0xFE430000
+#define INV_AGPHeader45             0xFE450000
+#define INV_AGPHeader47             0xFE470000
+#define INV_AGPHeader4A             0xFE4A0000
+#define INV_AGPHeader82             0xFE820000
+#define INV_AGPHeader83             0xFE830000
+#define INV_AGPHeader_MASK          0xFFFF0000
+#define INV_AGPHeader2A             0xFE2A0000
+#define INV_AGPHeader25             0xFE250000
+#define INV_AGPHeader20             0xFE200000
+#define INV_AGPHeader23             0xFE230000
+#define INV_AGPHeaderE2             0xFEE20000
+#define INV_AGPHeaderE3             0xFEE30000
+
+/*Transmission IO Space*/
+#define INV_REG_CR_TRANS            0x041C
+#define INV_REG_CR_BEGIN            0x0420
+#define INV_REG_CR_END              0x0438
+
+#define INV_REG_3D_TRANS            0x043C
+#define INV_REG_3D_BEGIN            0x0440
+#define INV_REG_3D_END              0x06FC
+
+#define INV_ParaType_CmdVdata        0x0000
+
+/* H5 Enable Setting
+ */
+#define INV_HC_SubA_HEnable1        0x00
+
+#define INV_HC_HenAT4ALLRT_MASK     0x00100000
+#define INV_HC_HenATMRT3_MASK       0x00080000
+#define INV_HC_HenATMRT2_MASK       0x00040000
+#define INV_HC_HenATMRT1_MASK       0x00020000
+#define INV_HC_HenATMRT0_MASK        0x00010000
+#define INV_HC_HenSCMRT3_MASK        0x00008000
+#define INV_HC_HenSCMRT2_MASK        0x00004000
+#define INV_HC_HenSCMRT1_MASK        0x00002000
+#define INV_HC_HenSCMRT0_MASK        0x00001000
+#define INV_HC_HenFOGMRT3_MASK        0x00000800
+#define INV_HC_HenFOGMRT2_MASK        0x00000400
+#define INV_HC_HenFOGMRT1_MASK        0x00000200
+#define INV_HC_HenFOGMRT0_MASK        0x00000100
+#define INV_HC_HenABLMRT3_MASK        0x00000080
+#define INV_HC_HenABLMRT2_MASK        0x00000040
+#define INV_HC_HenABLMRT1_MASK        0x00000020
+#define INV_HC_HenABLMRT0_MASK        0x00000010
+#define INV_HC_HenDTMRT3_MASK        0x00000008
+#define INV_HC_HenDTMRT2_MASK        0x00000004
+#define INV_HC_HenDTMRT1_MASK        0x00000002
+#define INV_HC_HenDTMRT0_MASK        0x00000001
+
+#define INV_HC_SubA_HEnable2        0x01
+
+#define INV_HC_HenLUL2DR_MASK         0x00800000
+#define INV_HC_HenLDIAMOND_MASK     0x00400000
+#define INV_HC_HenPSPRITE_MASK         0x00200000
+#define INV_HC_HenC2S_MASK             0x00100000
+#define INV_HC_HenFOGPP_MASK           0x00080000
+#define INV_HC_HenSCPP_MASK           0x00040000
+#define INV_HC_HenCPP_MASK           0x00020000
+#define INV_HC_HenCZ_MASK            0x00002000
+#define INV_HC_HenVC_MASK            0x00001000
+#define INV_HC_HenCL_MASK            0x00000800
+#define INV_HC_HenPS_MASK            0x00000400
+#define INV_HC_HenWCZ_MASK            0x00000200
+#define INV_HC_HenTXCH_MASK            0x00000100
+#define INV_HC_HenBFCULL_MASK        0x00000080
+#define INV_HC_HenCW_MASK            0x00000040
+#define INV_HC_HenAA_MASK            0x00000020
+#define INV_HC_HenST_MASK            0x00000010
+#define INV_HC_HenZT_MASK            0x00000008
+#define INV_HC_HenZW_MASK            0x00000004
+#define INV_HC_HenSP_MASK            0x00000002
+#define INV_HC_HenLP_MASK            0x00000001
+
+/* H5 Miscellaneous Settings
+ */
+#define INV_HC_SubA_HCClipTL           0x0080
+#define INV_HC_SubA_HCClipBL           0x0081
+#define INV_HC_SubA_HSClipTL           0x0082
+#define INV_HC_SubA_HSClipBL           0x0083
+#define INV_HC_SubA_HSolidCL        0x0086
+#define INV_HC_SubA_HSolidCH        0x0087
+#define INV_HC_SubA_HGBClipGL       0x0088
+#define INV_HC_SubA_HGBClipGR       0x0089
+
+
+#define INV_HC_ParaType_Vetex        0x00040000
+
 #endif
diff --git a/drivers/gpu/drm/via/via_dma.c b/drivers/gpu/drm/via/via_dma.c
deleted file mode 100644
index 177b0499abf1..000000000000
--- a/drivers/gpu/drm/via/via_dma.c
+++ /dev/null
@@ -1,744 +0,0 @@
-/* via_dma.c -- DMA support for the VIA Unichrome/Pro
- *
- * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
- * All Rights Reserved.
- *
- * Copyright 2004 Digeo, Inc., Palo Alto, CA, U.S.A.
- * All Rights Reserved.
- *
- * Copyright 2004 The Unichrome project.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
- * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Tungsten Graphics,
- *    Erdi Chen,
- *    Thomas Hellstrom.
- */
-
-#include <linux/delay.h>
-#include <linux/uaccess.h>
-
-#include <drm/drm.h>
-#include <drm/drm_device.h>
-#include <drm/drm_file.h>
-#include <drm/via_drm.h>
-
-#include "via_drv.h"
-#include "via_3d_reg.h"
-
-#define CMDBUF_ALIGNMENT_SIZE   (0x100)
-#define CMDBUF_ALIGNMENT_MASK   (0x0ff)
-
-/* defines for VIA 3D registers */
-#define VIA_REG_STATUS          0x400
-#define VIA_REG_TRANSET         0x43C
-#define VIA_REG_TRANSPACE       0x440
-
-/* VIA_REG_STATUS(0x400): Engine Status */
-#define VIA_CMD_RGTR_BUSY       0x00000080	/* Command Regulator is busy */
-#define VIA_2D_ENG_BUSY         0x00000001	/* 2D Engine is busy */
-#define VIA_3D_ENG_BUSY         0x00000002	/* 3D Engine is busy */
-#define VIA_VR_QUEUE_BUSY       0x00020000	/* Virtual Queue is busy */
-
-#define SetReg2DAGP(nReg, nData) {				\
-	*((uint32_t *)(vb)) = ((nReg) >> 2) | HALCYON_HEADER1;	\
-	*((uint32_t *)(vb) + 1) = (nData);			\
-	vb = ((uint32_t *)vb) + 2;				\
-	dev_priv->dma_low += 8;					\
-}
-
-#define via_flush_write_combine() mb()
-
-#define VIA_OUT_RING_QW(w1, w2)	do {		\
-	*vb++ = (w1);				\
-	*vb++ = (w2);				\
-	dev_priv->dma_low += 8;			\
-} while (0)
-
-static void via_cmdbuf_start(drm_via_private_t *dev_priv);
-static void via_cmdbuf_pause(drm_via_private_t *dev_priv);
-static void via_cmdbuf_reset(drm_via_private_t *dev_priv);
-static void via_cmdbuf_rewind(drm_via_private_t *dev_priv);
-static int via_wait_idle(drm_via_private_t *dev_priv);
-static void via_pad_cache(drm_via_private_t *dev_priv, int qwords);
-
-/*
- * Free space in command buffer.
- */
-
-static uint32_t via_cmdbuf_space(drm_via_private_t *dev_priv)
-{
-	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;
-	uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base;
-
-	return ((hw_addr <= dev_priv->dma_low) ?
-		(dev_priv->dma_high + hw_addr - dev_priv->dma_low) :
-		(hw_addr - dev_priv->dma_low));
-}
-
-/*
- * How much does the command regulator lag behind?
- */
-
-static uint32_t via_cmdbuf_lag(drm_via_private_t *dev_priv)
-{
-	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;
-	uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base;
-
-	return ((hw_addr <= dev_priv->dma_low) ?
-		(dev_priv->dma_low - hw_addr) :
-		(dev_priv->dma_wrap + dev_priv->dma_low - hw_addr));
-}
-
-/*
- * Check that the given size fits in the buffer, otherwise wait.
- */
-
-static inline int
-via_cmdbuf_wait(drm_via_private_t *dev_priv, unsigned int size)
-{
-	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;
-	uint32_t cur_addr, hw_addr, next_addr;
-	volatile uint32_t *hw_addr_ptr;
-	uint32_t count;
-	hw_addr_ptr = dev_priv->hw_addr_ptr;
-	cur_addr = dev_priv->dma_low;
-	next_addr = cur_addr + size + 512 * 1024;
-	count = 1000000;
-	do {
-		hw_addr = *hw_addr_ptr - agp_base;
-		if (count-- == 0) {
-			DRM_ERROR
-			    ("via_cmdbuf_wait timed out hw %x cur_addr %x next_addr %x\n",
-			     hw_addr, cur_addr, next_addr);
-			return -1;
-		}
-		if  ((cur_addr < hw_addr) && (next_addr >= hw_addr))
-			msleep(1);
-	} while ((cur_addr < hw_addr) && (next_addr >= hw_addr));
-	return 0;
-}
-
-/*
- * Checks whether buffer head has reach the end. Rewind the ring buffer
- * when necessary.
- *
- * Returns virtual pointer to ring buffer.
- */
-
-static inline uint32_t *via_check_dma(drm_via_private_t * dev_priv,
-				      unsigned int size)
-{
-	if ((dev_priv->dma_low + size + 4 * CMDBUF_ALIGNMENT_SIZE) >
-	    dev_priv->dma_high) {
-		via_cmdbuf_rewind(dev_priv);
-	}
-	if (via_cmdbuf_wait(dev_priv, size) != 0)
-		return NULL;
-
-	return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low);
-}
-
-int via_dma_cleanup(struct drm_device *dev)
-{
-	if (dev->dev_private) {
-		drm_via_private_t *dev_priv =
-		    (drm_via_private_t *) dev->dev_private;
-
-		if (dev_priv->ring.virtual_start) {
-			via_cmdbuf_reset(dev_priv);
-
-			drm_legacy_ioremapfree(&dev_priv->ring.map, dev);
-			dev_priv->ring.virtual_start = NULL;
-		}
-
-	}
-
-	return 0;
-}
-
-static int via_initialize(struct drm_device *dev,
-			  drm_via_private_t *dev_priv,
-			  drm_via_dma_init_t *init)
-{
-	if (!dev_priv || !dev_priv->mmio) {
-		DRM_ERROR("via_dma_init called before via_map_init\n");
-		return -EFAULT;
-	}
-
-	if (dev_priv->ring.virtual_start != NULL) {
-		DRM_ERROR("called again without calling cleanup\n");
-		return -EFAULT;
-	}
-
-	if (!dev->agp || !dev->agp->base) {
-		DRM_ERROR("called with no agp memory available\n");
-		return -EFAULT;
-	}
-
-	if (dev_priv->chipset == VIA_DX9_0) {
-		DRM_ERROR("AGP DMA is not supported on this chip\n");
-		return -EINVAL;
-	}
-
-	dev_priv->ring.map.offset = dev->agp->base + init->offset;
-	dev_priv->ring.map.size = init->size;
-	dev_priv->ring.map.type = 0;
-	dev_priv->ring.map.flags = 0;
-	dev_priv->ring.map.mtrr = 0;
-
-	drm_legacy_ioremap(&dev_priv->ring.map, dev);
-
-	if (dev_priv->ring.map.handle == NULL) {
-		via_dma_cleanup(dev);
-		DRM_ERROR("can not ioremap virtual address for"
-			  " ring buffer\n");
-		return -ENOMEM;
-	}
-
-	dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
-
-	dev_priv->dma_ptr = dev_priv->ring.virtual_start;
-	dev_priv->dma_low = 0;
-	dev_priv->dma_high = init->size;
-	dev_priv->dma_wrap = init->size;
-	dev_priv->dma_offset = init->offset;
-	dev_priv->last_pause_ptr = NULL;
-	dev_priv->hw_addr_ptr =
-		(volatile uint32_t *)((char *)dev_priv->mmio->handle +
-		init->reg_pause_addr);
-
-	via_cmdbuf_start(dev_priv);
-
-	return 0;
-}
-
-static int via_dma_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	drm_via_dma_init_t *init = data;
-	int retcode = 0;
-
-	switch (init->func) {
-	case VIA_INIT_DMA:
-		if (!capable(CAP_SYS_ADMIN))
-			retcode = -EPERM;
-		else
-			retcode = via_initialize(dev, dev_priv, init);
-		break;
-	case VIA_CLEANUP_DMA:
-		if (!capable(CAP_SYS_ADMIN))
-			retcode = -EPERM;
-		else
-			retcode = via_dma_cleanup(dev);
-		break;
-	case VIA_DMA_INITIALIZED:
-		retcode = (dev_priv->ring.virtual_start != NULL) ?
-			0 : -EFAULT;
-		break;
-	default:
-		retcode = -EINVAL;
-		break;
-	}
-
-	return retcode;
-}
-
-static int via_dispatch_cmdbuffer(struct drm_device *dev, drm_via_cmdbuffer_t *cmd)
-{
-	drm_via_private_t *dev_priv;
-	uint32_t *vb;
-	int ret;
-
-	dev_priv = (drm_via_private_t *) dev->dev_private;
-
-	if (dev_priv->ring.virtual_start == NULL) {
-		DRM_ERROR("called without initializing AGP ring buffer.\n");
-		return -EFAULT;
-	}
-
-	if (cmd->size > VIA_PCI_BUF_SIZE)
-		return -ENOMEM;
-
-	if (copy_from_user(dev_priv->pci_buf, cmd->buf, cmd->size))
-		return -EFAULT;
-
-	/*
-	 * Running this function on AGP memory is dead slow. Therefore
-	 * we run it on a temporary cacheable system memory buffer and
-	 * copy it to AGP memory when ready.
-	 */
-
-	if ((ret =
-	     via_verify_command_stream((uint32_t *) dev_priv->pci_buf,
-				       cmd->size, dev, 1))) {
-		return ret;
-	}
-
-	vb = via_check_dma(dev_priv, (cmd->size < 0x100) ? 0x102 : cmd->size);
-	if (vb == NULL)
-		return -EAGAIN;
-
-	memcpy(vb, dev_priv->pci_buf, cmd->size);
-
-	dev_priv->dma_low += cmd->size;
-
-	/*
-	 * Small submissions somehow stalls the CPU. (AGP cache effects?)
-	 * pad to greater size.
-	 */
-
-	if (cmd->size < 0x100)
-		via_pad_cache(dev_priv, (0x100 - cmd->size) >> 3);
-	via_cmdbuf_pause(dev_priv);
-
-	return 0;
-}
-
-int via_driver_dma_quiescent(struct drm_device *dev)
-{
-	drm_via_private_t *dev_priv = dev->dev_private;
-
-	if (!via_wait_idle(dev_priv))
-		return -EBUSY;
-	return 0;
-}
-
-static int via_flush_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	return via_driver_dma_quiescent(dev);
-}
-
-static int via_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_cmdbuffer_t *cmdbuf = data;
-	int ret;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	DRM_DEBUG("buf %p size %lu\n", cmdbuf->buf, cmdbuf->size);
-
-	ret = via_dispatch_cmdbuffer(dev, cmdbuf);
-	return ret;
-}
-
-static int via_dispatch_pci_cmdbuffer(struct drm_device *dev,
-				      drm_via_cmdbuffer_t *cmd)
-{
-	drm_via_private_t *dev_priv = dev->dev_private;
-	int ret;
-
-	if (cmd->size > VIA_PCI_BUF_SIZE)
-		return -ENOMEM;
-	if (copy_from_user(dev_priv->pci_buf, cmd->buf, cmd->size))
-		return -EFAULT;
-
-	if ((ret =
-	     via_verify_command_stream((uint32_t *) dev_priv->pci_buf,
-				       cmd->size, dev, 0))) {
-		return ret;
-	}
-
-	ret =
-	    via_parse_command_stream(dev, (const uint32_t *)dev_priv->pci_buf,
-				     cmd->size);
-	return ret;
-}
-
-static int via_pci_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_cmdbuffer_t *cmdbuf = data;
-	int ret;
-
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	DRM_DEBUG("buf %p size %lu\n", cmdbuf->buf, cmdbuf->size);
-
-	ret = via_dispatch_pci_cmdbuffer(dev, cmdbuf);
-	return ret;
-}
-
-static inline uint32_t *via_align_buffer(drm_via_private_t *dev_priv,
-					 uint32_t * vb, int qw_count)
-{
-	for (; qw_count > 0; --qw_count)
-		VIA_OUT_RING_QW(HC_DUMMY, HC_DUMMY);
-	return vb;
-}
-
-/*
- * This function is used internally by ring buffer management code.
- *
- * Returns virtual pointer to ring buffer.
- */
-static inline uint32_t *via_get_dma(drm_via_private_t *dev_priv)
-{
-	return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low);
-}
-
-/*
- * Hooks a segment of data into the tail of the ring-buffer by
- * modifying the pause address stored in the buffer itself. If
- * the regulator has already paused, restart it.
- */
-static int via_hook_segment(drm_via_private_t *dev_priv,
-			    uint32_t pause_addr_hi, uint32_t pause_addr_lo,
-			    int no_pci_fire)
-{
-	int paused, count;
-	volatile uint32_t *paused_at = dev_priv->last_pause_ptr;
-	uint32_t reader, ptr;
-	uint32_t diff;
-
-	paused = 0;
-	via_flush_write_combine();
-	(void) *(volatile uint32_t *)(via_get_dma(dev_priv) - 1);
-
-	*paused_at = pause_addr_lo;
-	via_flush_write_combine();
-	(void) *paused_at;
-
-	reader = *(dev_priv->hw_addr_ptr);
-	ptr = ((volatile char *)paused_at - dev_priv->dma_ptr) +
-		dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4;
-
-	dev_priv->last_pause_ptr = via_get_dma(dev_priv) - 1;
-
-	/*
-	 * If there is a possibility that the command reader will
-	 * miss the new pause address and pause on the old one,
-	 * In that case we need to program the new start address
-	 * using PCI.
-	 */
-
-	diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;
-	count = 10000000;
-	while (diff == 0 && count--) {
-		paused = (via_read(dev_priv, 0x41c) & 0x80000000);
-		if (paused)
-			break;
-		reader = *(dev_priv->hw_addr_ptr);
-		diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;
-	}
-
-	paused = via_read(dev_priv, 0x41c) & 0x80000000;
-
-	if (paused && !no_pci_fire) {
-		reader = *(dev_priv->hw_addr_ptr);
-		diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;
-		diff &= (dev_priv->dma_high - 1);
-		if (diff != 0 && diff < (dev_priv->dma_high >> 1)) {
-			DRM_ERROR("Paused at incorrect address. "
-				  "0x%08x, 0x%08x 0x%08x\n",
-				  ptr, reader, dev_priv->dma_diff);
-		} else if (diff == 0) {
-			/*
-			 * There is a concern that these writes may stall the PCI bus
-			 * if the GPU is not idle. However, idling the GPU first
-			 * doesn't make a difference.
-			 */
-
-			via_write(dev_priv, VIA_REG_TRANSET, (HC_ParaType_PreCR << 16));
-			via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_hi);
-			via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_lo);
-			via_read(dev_priv, VIA_REG_TRANSPACE);
-		}
-	}
-	return paused;
-}
-
-static int via_wait_idle(drm_via_private_t *dev_priv)
-{
-	int count = 10000000;
-
-	while (!(via_read(dev_priv, VIA_REG_STATUS) & VIA_VR_QUEUE_BUSY) && --count)
-		;
-
-	while (count && (via_read(dev_priv, VIA_REG_STATUS) &
-			   (VIA_CMD_RGTR_BUSY | VIA_2D_ENG_BUSY |
-			    VIA_3D_ENG_BUSY)))
-		--count;
-	return count;
-}
-
-static uint32_t *via_align_cmd(drm_via_private_t *dev_priv, uint32_t cmd_type,
-			       uint32_t addr, uint32_t *cmd_addr_hi,
-			       uint32_t *cmd_addr_lo, int skip_wait)
-{
-	uint32_t agp_base;
-	uint32_t cmd_addr, addr_lo, addr_hi;
-	uint32_t *vb;
-	uint32_t qw_pad_count;
-
-	if (!skip_wait)
-		via_cmdbuf_wait(dev_priv, 2 * CMDBUF_ALIGNMENT_SIZE);
-
-	vb = via_get_dma(dev_priv);
-	VIA_OUT_RING_QW(HC_HEADER2 | ((VIA_REG_TRANSET >> 2) << 12) |
-			(VIA_REG_TRANSPACE >> 2), HC_ParaType_PreCR << 16);
-	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;
-	qw_pad_count = (CMDBUF_ALIGNMENT_SIZE >> 3) -
-	    ((dev_priv->dma_low & CMDBUF_ALIGNMENT_MASK) >> 3);
-
-	cmd_addr = (addr) ? addr :
-	    agp_base + dev_priv->dma_low - 8 + (qw_pad_count << 3);
-	addr_lo = ((HC_SubA_HAGPBpL << 24) | (cmd_type & HC_HAGPBpID_MASK) |
-		   (cmd_addr & HC_HAGPBpL_MASK));
-	addr_hi = ((HC_SubA_HAGPBpH << 24) | (cmd_addr >> 24));
-
-	vb = via_align_buffer(dev_priv, vb, qw_pad_count - 1);
-	VIA_OUT_RING_QW(*cmd_addr_hi = addr_hi, *cmd_addr_lo = addr_lo);
-	return vb;
-}
-
-static void via_cmdbuf_start(drm_via_private_t *dev_priv)
-{
-	uint32_t pause_addr_lo, pause_addr_hi;
-	uint32_t start_addr, start_addr_lo;
-	uint32_t end_addr, end_addr_lo;
-	uint32_t command;
-	uint32_t agp_base;
-	uint32_t ptr;
-	uint32_t reader;
-	int count;
-
-	dev_priv->dma_low = 0;
-
-	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;
-	start_addr = agp_base;
-	end_addr = agp_base + dev_priv->dma_high;
-
-	start_addr_lo = ((HC_SubA_HAGPBstL << 24) | (start_addr & 0xFFFFFF));
-	end_addr_lo = ((HC_SubA_HAGPBendL << 24) | (end_addr & 0xFFFFFF));
-	command = ((HC_SubA_HAGPCMNT << 24) | (start_addr >> 24) |
-		   ((end_addr & 0xff000000) >> 16));
-
-	dev_priv->last_pause_ptr =
-	    via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0,
-			  &pause_addr_hi, &pause_addr_lo, 1) - 1;
-
-	via_flush_write_combine();
-	(void) *(volatile uint32_t *)dev_priv->last_pause_ptr;
-
-	via_write(dev_priv, VIA_REG_TRANSET, (HC_ParaType_PreCR << 16));
-	via_write(dev_priv, VIA_REG_TRANSPACE, command);
-	via_write(dev_priv, VIA_REG_TRANSPACE, start_addr_lo);
-	via_write(dev_priv, VIA_REG_TRANSPACE, end_addr_lo);
-
-	via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_hi);
-	via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_lo);
-	wmb();
-	via_write(dev_priv, VIA_REG_TRANSPACE, command | HC_HAGPCMNT_MASK);
-	via_read(dev_priv, VIA_REG_TRANSPACE);
-
-	dev_priv->dma_diff = 0;
-
-	count = 10000000;
-	while (!(via_read(dev_priv, 0x41c) & 0x80000000) && count--);
-
-	reader = *(dev_priv->hw_addr_ptr);
-	ptr = ((volatile char *)dev_priv->last_pause_ptr - dev_priv->dma_ptr) +
-	    dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4;
-
-	/*
-	 * This is the difference between where we tell the
-	 * command reader to pause and where it actually pauses.
-	 * This differs between hw implementation so we need to
-	 * detect it.
-	 */
-
-	dev_priv->dma_diff = ptr - reader;
-}
-
-static void via_pad_cache(drm_via_private_t *dev_priv, int qwords)
-{
-	uint32_t *vb;
-
-	via_cmdbuf_wait(dev_priv, qwords + 2);
-	vb = via_get_dma(dev_priv);
-	VIA_OUT_RING_QW(HC_HEADER2, HC_ParaType_NotTex << 16);
-	via_align_buffer(dev_priv, vb, qwords);
-}
-
-static inline void via_dummy_bitblt(drm_via_private_t *dev_priv)
-{
-	uint32_t *vb = via_get_dma(dev_priv);
-	SetReg2DAGP(0x0C, (0 | (0 << 16)));
-	SetReg2DAGP(0x10, 0 | (0 << 16));
-	SetReg2DAGP(0x0, 0x1 | 0x2000 | 0xAA000000);
-}
-
-static void via_cmdbuf_jump(drm_via_private_t *dev_priv)
-{
-	uint32_t pause_addr_lo, pause_addr_hi;
-	uint32_t jump_addr_lo, jump_addr_hi;
-	volatile uint32_t *last_pause_ptr;
-	uint32_t dma_low_save1, dma_low_save2;
-
-	via_align_cmd(dev_priv, HC_HAGPBpID_JUMP, 0, &jump_addr_hi,
-		      &jump_addr_lo, 0);
-
-	dev_priv->dma_wrap = dev_priv->dma_low;
-
-	/*
-	 * Wrap command buffer to the beginning.
-	 */
-
-	dev_priv->dma_low = 0;
-	if (via_cmdbuf_wait(dev_priv, CMDBUF_ALIGNMENT_SIZE) != 0)
-		DRM_ERROR("via_cmdbuf_jump failed\n");
-
-	via_dummy_bitblt(dev_priv);
-	via_dummy_bitblt(dev_priv);
-
-	last_pause_ptr =
-	    via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,
-			  &pause_addr_lo, 0) - 1;
-	via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,
-		      &pause_addr_lo, 0);
-
-	*last_pause_ptr = pause_addr_lo;
-	dma_low_save1 = dev_priv->dma_low;
-
-	/*
-	 * Now, set a trap that will pause the regulator if it tries to rerun the old
-	 * command buffer. (Which may happen if via_hook_segment detecs a command regulator pause
-	 * and reissues the jump command over PCI, while the regulator has already taken the jump
-	 * and actually paused at the current buffer end).
-	 * There appears to be no other way to detect this condition, since the hw_addr_pointer
-	 * does not seem to get updated immediately when a jump occurs.
-	 */
-
-	last_pause_ptr =
-		via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,
-			      &pause_addr_lo, 0) - 1;
-	via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,
-		      &pause_addr_lo, 0);
-	*last_pause_ptr = pause_addr_lo;
-
-	dma_low_save2 = dev_priv->dma_low;
-	dev_priv->dma_low = dma_low_save1;
-	via_hook_segment(dev_priv, jump_addr_hi, jump_addr_lo, 0);
-	dev_priv->dma_low = dma_low_save2;
-	via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0);
-}
-
-
-static void via_cmdbuf_rewind(drm_via_private_t *dev_priv)
-{
-	via_cmdbuf_jump(dev_priv);
-}
-
-static void via_cmdbuf_flush(drm_via_private_t *dev_priv, uint32_t cmd_type)
-{
-	uint32_t pause_addr_lo, pause_addr_hi;
-
-	via_align_cmd(dev_priv, cmd_type, 0, &pause_addr_hi, &pause_addr_lo, 0);
-	via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0);
-}
-
-static void via_cmdbuf_pause(drm_via_private_t *dev_priv)
-{
-	via_cmdbuf_flush(dev_priv, HC_HAGPBpID_PAUSE);
-}
-
-static void via_cmdbuf_reset(drm_via_private_t *dev_priv)
-{
-	via_cmdbuf_flush(dev_priv, HC_HAGPBpID_STOP);
-	via_wait_idle(dev_priv);
-}
-
-/*
- * User interface to the space and lag functions.
- */
-
-static int via_cmdbuf_size(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_cmdbuf_size_t *d_siz = data;
-	int ret = 0;
-	uint32_t tmp_size, count;
-	drm_via_private_t *dev_priv;
-
-	DRM_DEBUG("\n");
-	LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	dev_priv = (drm_via_private_t *) dev->dev_private;
-
-	if (dev_priv->ring.virtual_start == NULL) {
-		DRM_ERROR("called without initializing AGP ring buffer.\n");
-		return -EFAULT;
-	}
-
-	count = 1000000;
-	tmp_size = d_siz->size;
-	switch (d_siz->func) {
-	case VIA_CMDBUF_SPACE:
-		while (((tmp_size = via_cmdbuf_space(dev_priv)) < d_siz->size)
-		       && --count) {
-			if (!d_siz->wait)
-				break;
-		}
-		if (!count) {
-			DRM_ERROR("VIA_CMDBUF_SPACE timed out.\n");
-			ret = -EAGAIN;
-		}
-		break;
-	case VIA_CMDBUF_LAG:
-		while (((tmp_size = via_cmdbuf_lag(dev_priv)) > d_siz->size)
-		       && --count) {
-			if (!d_siz->wait)
-				break;
-		}
-		if (!count) {
-			DRM_ERROR("VIA_CMDBUF_LAG timed out.\n");
-			ret = -EAGAIN;
-		}
-		break;
-	default:
-		ret = -EFAULT;
-	}
-	d_siz->size = tmp_size;
-
-	return ret;
-}
-
-const struct drm_ioctl_desc via_ioctls[] = {
-	DRM_IOCTL_DEF_DRV(VIA_ALLOCMEM, via_mem_alloc, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(VIA_FREEMEM, via_mem_free, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(VIA_AGP_INIT, via_agp_init, DRM_AUTH|DRM_MASTER),
-	DRM_IOCTL_DEF_DRV(VIA_FB_INIT, via_fb_init, DRM_AUTH|DRM_MASTER),
-	DRM_IOCTL_DEF_DRV(VIA_MAP_INIT, via_map_init, DRM_AUTH|DRM_MASTER),
-	DRM_IOCTL_DEF_DRV(VIA_DEC_FUTEX, via_decoder_futex, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(VIA_DMA_INIT, via_dma_init, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(VIA_CMDBUFFER, via_cmdbuffer, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(VIA_FLUSH, via_flush_ioctl, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(VIA_PCICMD, via_pci_cmdbuffer, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(VIA_CMDBUF_SIZE, via_cmdbuf_size, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(VIA_WAIT_IRQ, via_wait_irq, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(VIA_DMA_BLIT, via_dma_blit, DRM_AUTH),
-	DRM_IOCTL_DEF_DRV(VIA_BLIT_SYNC, via_dma_blit_sync, DRM_AUTH)
-};
-
-int via_max_ioctl = ARRAY_SIZE(via_ioctls);
diff --git a/drivers/gpu/drm/via/via_dmablit.c b/drivers/gpu/drm/via/via_dmablit.c
deleted file mode 100644
index e016a4d62090..000000000000
--- a/drivers/gpu/drm/via/via_dmablit.c
+++ /dev/null
@@ -1,807 +0,0 @@
-/* via_dmablit.c -- PCI DMA BitBlt support for the VIA Unichrome/Pro
- *
- * Copyright (C) 2005 Thomas Hellstrom, All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
- * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Thomas Hellstrom.
- *    Partially based on code obtained from Digeo Inc.
- */
-
-
-/*
- * Unmaps the DMA mappings.
- * FIXME: Is this a NoOp on x86? Also
- * FIXME: What happens if this one is called and a pending blit has previously done
- * the same DMA mappings?
- */
-
-#include <linux/pagemap.h>
-#include <linux/pci.h>
-#include <linux/slab.h>
-#include <linux/vmalloc.h>
-
-#include <drm/drm_device.h>
-#include <drm/via_drm.h>
-
-#include "via_dmablit.h"
-#include "via_drv.h"
-
-#define VIA_PGDN(x)	     (((unsigned long)(x)) & PAGE_MASK)
-#define VIA_PGOFF(x)	    (((unsigned long)(x)) & ~PAGE_MASK)
-#define VIA_PFN(x)	      ((unsigned long)(x) >> PAGE_SHIFT)
-
-typedef struct _drm_via_descriptor {
-	uint32_t mem_addr;
-	uint32_t dev_addr;
-	uint32_t size;
-	uint32_t next;
-} drm_via_descriptor_t;
-
-
-/*
- * Unmap a DMA mapping.
- */
-
-
-
-static void
-via_unmap_blit_from_device(struct pci_dev *pdev, drm_via_sg_info_t *vsg)
-{
-	int num_desc = vsg->num_desc;
-	unsigned cur_descriptor_page = num_desc / vsg->descriptors_per_page;
-	unsigned descriptor_this_page = num_desc % vsg->descriptors_per_page;
-	drm_via_descriptor_t *desc_ptr = vsg->desc_pages[cur_descriptor_page] +
-		descriptor_this_page;
-	dma_addr_t next = vsg->chain_start;
-
-	while (num_desc--) {
-		if (descriptor_this_page-- == 0) {
-			cur_descriptor_page--;
-			descriptor_this_page = vsg->descriptors_per_page - 1;
-			desc_ptr = vsg->desc_pages[cur_descriptor_page] +
-				descriptor_this_page;
-		}
-		dma_unmap_single(&pdev->dev, next, sizeof(*desc_ptr), DMA_TO_DEVICE);
-		dma_unmap_page(&pdev->dev, desc_ptr->mem_addr, desc_ptr->size, vsg->direction);
-		next = (dma_addr_t) desc_ptr->next;
-		desc_ptr--;
-	}
-}
-
-/*
- * If mode = 0, count how many descriptors are needed.
- * If mode = 1, Map the DMA pages for the device, put together and map also the descriptors.
- * Descriptors are run in reverse order by the hardware because we are not allowed to update the
- * 'next' field without syncing calls when the descriptor is already mapped.
- */
-
-static void
-via_map_blit_for_device(struct pci_dev *pdev,
-		   const drm_via_dmablit_t *xfer,
-		   drm_via_sg_info_t *vsg,
-		   int mode)
-{
-	unsigned cur_descriptor_page = 0;
-	unsigned num_descriptors_this_page = 0;
-	unsigned char *mem_addr = xfer->mem_addr;
-	unsigned char *cur_mem;
-	unsigned char *first_addr = (unsigned char *)VIA_PGDN(mem_addr);
-	uint32_t fb_addr = xfer->fb_addr;
-	uint32_t cur_fb;
-	unsigned long line_len;
-	unsigned remaining_len;
-	int num_desc = 0;
-	int cur_line;
-	dma_addr_t next = 0 | VIA_DMA_DPR_EC;
-	drm_via_descriptor_t *desc_ptr = NULL;
-
-	if (mode == 1)
-		desc_ptr = vsg->desc_pages[cur_descriptor_page];
-
-	for (cur_line = 0; cur_line < xfer->num_lines; ++cur_line) {
-
-		line_len = xfer->line_length;
-		cur_fb = fb_addr;
-		cur_mem = mem_addr;
-
-		while (line_len > 0) {
-
-			remaining_len = min(PAGE_SIZE-VIA_PGOFF(cur_mem), line_len);
-			line_len -= remaining_len;
-
-			if (mode == 1) {
-				desc_ptr->mem_addr =
-					dma_map_page(&pdev->dev,
-						     vsg->pages[VIA_PFN(cur_mem) -
-								VIA_PFN(first_addr)],
-						     VIA_PGOFF(cur_mem), remaining_len,
-						     vsg->direction);
-				desc_ptr->dev_addr = cur_fb;
-
-				desc_ptr->size = remaining_len;
-				desc_ptr->next = (uint32_t) next;
-				next = dma_map_single(&pdev->dev, desc_ptr, sizeof(*desc_ptr),
-						      DMA_TO_DEVICE);
-				desc_ptr++;
-				if (++num_descriptors_this_page >= vsg->descriptors_per_page) {
-					num_descriptors_this_page = 0;
-					desc_ptr = vsg->desc_pages[++cur_descriptor_page];
-				}
-			}
-
-			num_desc++;
-			cur_mem += remaining_len;
-			cur_fb += remaining_len;
-		}
-
-		mem_addr += xfer->mem_stride;
-		fb_addr += xfer->fb_stride;
-	}
-
-	if (mode == 1) {
-		vsg->chain_start = next;
-		vsg->state = dr_via_device_mapped;
-	}
-	vsg->num_desc = num_desc;
-}
-
-/*
- * Function that frees up all resources for a blit. It is usable even if the
- * blit info has only been partially built as long as the status enum is consistent
- * with the actual status of the used resources.
- */
-
-
-static void
-via_free_sg_info(struct pci_dev *pdev, drm_via_sg_info_t *vsg)
-{
-	int i;
-
-	switch (vsg->state) {
-	case dr_via_device_mapped:
-		via_unmap_blit_from_device(pdev, vsg);
-		fallthrough;
-	case dr_via_desc_pages_alloc:
-		for (i = 0; i < vsg->num_desc_pages; ++i) {
-			if (vsg->desc_pages[i] != NULL)
-				free_page((unsigned long)vsg->desc_pages[i]);
-		}
-		kfree(vsg->desc_pages);
-		fallthrough;
-	case dr_via_pages_locked:
-		unpin_user_pages_dirty_lock(vsg->pages, vsg->num_pages,
-					   (vsg->direction == DMA_FROM_DEVICE));
-		fallthrough;
-	case dr_via_pages_alloc:
-		vfree(vsg->pages);
-		fallthrough;
-	default:
-		vsg->state = dr_via_sg_init;
-	}
-	vfree(vsg->bounce_buffer);
-	vsg->bounce_buffer = NULL;
-	vsg->free_on_sequence = 0;
-}
-
-/*
- * Fire a blit engine.
- */
-
-static void
-via_fire_dmablit(struct drm_device *dev, drm_via_sg_info_t *vsg, int engine)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
-
-	via_write(dev_priv, VIA_PCI_DMA_MAR0 + engine*0x10, 0);
-	via_write(dev_priv, VIA_PCI_DMA_DAR0 + engine*0x10, 0);
-	via_write(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_DD | VIA_DMA_CSR_TD |
-		  VIA_DMA_CSR_DE);
-	via_write(dev_priv, VIA_PCI_DMA_MR0  + engine*0x04, VIA_DMA_MR_CM | VIA_DMA_MR_TDIE);
-	via_write(dev_priv, VIA_PCI_DMA_BCR0 + engine*0x10, 0);
-	via_write(dev_priv, VIA_PCI_DMA_DPR0 + engine*0x10, vsg->chain_start);
-	wmb();
-	via_write(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_DE | VIA_DMA_CSR_TS);
-	via_read(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04);
-}
-
-/*
- * Obtain a page pointer array and lock all pages into system memory. A segmentation violation will
- * occur here if the calling user does not have access to the submitted address.
- */
-
-static int
-via_lock_all_dma_pages(drm_via_sg_info_t *vsg,  drm_via_dmablit_t *xfer)
-{
-	int ret;
-	unsigned long first_pfn = VIA_PFN(xfer->mem_addr);
-	vsg->num_pages = VIA_PFN(xfer->mem_addr + (xfer->num_lines * xfer->mem_stride - 1)) -
-		first_pfn + 1;
-
-	vsg->pages = vzalloc(array_size(sizeof(struct page *), vsg->num_pages));
-	if (NULL == vsg->pages)
-		return -ENOMEM;
-	ret = pin_user_pages_fast((unsigned long)xfer->mem_addr,
-			vsg->num_pages,
-			vsg->direction == DMA_FROM_DEVICE ? FOLL_WRITE : 0,
-			vsg->pages);
-	if (ret != vsg->num_pages) {
-		if (ret < 0)
-			return ret;
-		vsg->state = dr_via_pages_locked;
-		return -EINVAL;
-	}
-	vsg->state = dr_via_pages_locked;
-	DRM_DEBUG("DMA pages locked\n");
-	return 0;
-}
-
-/*
- * Allocate DMA capable memory for the blit descriptor chain, and an array that keeps track of the
- * pages we allocate. We don't want to use kmalloc for the descriptor chain because it may be
- * quite large for some blits, and pages don't need to be contiguous.
- */
-
-static int
-via_alloc_desc_pages(drm_via_sg_info_t *vsg)
-{
-	int i;
-
-	vsg->descriptors_per_page = PAGE_SIZE / sizeof(drm_via_descriptor_t);
-	vsg->num_desc_pages = (vsg->num_desc + vsg->descriptors_per_page - 1) /
-		vsg->descriptors_per_page;
-
-	if (NULL ==  (vsg->desc_pages = kcalloc(vsg->num_desc_pages, sizeof(void *), GFP_KERNEL)))
-		return -ENOMEM;
-
-	vsg->state = dr_via_desc_pages_alloc;
-	for (i = 0; i < vsg->num_desc_pages; ++i) {
-		if (NULL == (vsg->desc_pages[i] =
-			     (drm_via_descriptor_t *) __get_free_page(GFP_KERNEL)))
-			return -ENOMEM;
-	}
-	DRM_DEBUG("Allocated %d pages for %d descriptors.\n", vsg->num_desc_pages,
-		  vsg->num_desc);
-	return 0;
-}
-
-static void
-via_abort_dmablit(struct drm_device *dev, int engine)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
-
-	via_write(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TA);
-}
-
-static void
-via_dmablit_engine_off(struct drm_device *dev, int engine)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
-
-	via_write(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TD | VIA_DMA_CSR_DD);
-}
-
-
-
-/*
- * The dmablit part of the IRQ handler. Trying to do only reasonably fast things here.
- * The rest, like unmapping and freeing memory for done blits is done in a separate workqueue
- * task. Basically the task of the interrupt handler is to submit a new blit to the engine, while
- * the workqueue task takes care of processing associated with the old blit.
- */
-
-void
-via_dmablit_handler(struct drm_device *dev, int engine, int from_irq)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
-	drm_via_blitq_t *blitq = dev_priv->blit_queues + engine;
-	int cur;
-	int done_transfer;
-	unsigned long irqsave = 0;
-	uint32_t status = 0;
-
-	DRM_DEBUG("DMA blit handler called. engine = %d, from_irq = %d, blitq = 0x%lx\n",
-		  engine, from_irq, (unsigned long) blitq);
-
-	if (from_irq)
-		spin_lock(&blitq->blit_lock);
-	else
-		spin_lock_irqsave(&blitq->blit_lock, irqsave);
-
-	done_transfer = blitq->is_active &&
-	  ((status = via_read(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04)) & VIA_DMA_CSR_TD);
-	done_transfer = done_transfer || (blitq->aborting && !(status & VIA_DMA_CSR_DE));
-
-	cur = blitq->cur;
-	if (done_transfer) {
-
-		blitq->blits[cur]->aborted = blitq->aborting;
-		blitq->done_blit_handle++;
-		wake_up(blitq->blit_queue + cur);
-
-		cur++;
-		if (cur >= VIA_NUM_BLIT_SLOTS)
-			cur = 0;
-		blitq->cur = cur;
-
-		/*
-		 * Clear transfer done flag.
-		 */
-
-		via_write(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04,  VIA_DMA_CSR_TD);
-
-		blitq->is_active = 0;
-		blitq->aborting = 0;
-		schedule_work(&blitq->wq);
-
-	} else if (blitq->is_active && time_after_eq(jiffies, blitq->end)) {
-
-		/*
-		 * Abort transfer after one second.
-		 */
-
-		via_abort_dmablit(dev, engine);
-		blitq->aborting = 1;
-		blitq->end = jiffies + HZ;
-	}
-
-	if (!blitq->is_active) {
-		if (blitq->num_outstanding) {
-			via_fire_dmablit(dev, blitq->blits[cur], engine);
-			blitq->is_active = 1;
-			blitq->cur = cur;
-			blitq->num_outstanding--;
-			blitq->end = jiffies + HZ;
-			if (!timer_pending(&blitq->poll_timer))
-				mod_timer(&blitq->poll_timer, jiffies + 1);
-		} else {
-			if (timer_pending(&blitq->poll_timer))
-				del_timer(&blitq->poll_timer);
-			via_dmablit_engine_off(dev, engine);
-		}
-	}
-
-	if (from_irq)
-		spin_unlock(&blitq->blit_lock);
-	else
-		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
-}
-
-
-
-/*
- * Check whether this blit is still active, performing necessary locking.
- */
-
-static int
-via_dmablit_active(drm_via_blitq_t *blitq, int engine, uint32_t handle, wait_queue_head_t **queue)
-{
-	unsigned long irqsave;
-	uint32_t slot;
-	int active;
-
-	spin_lock_irqsave(&blitq->blit_lock, irqsave);
-
-	/*
-	 * Allow for handle wraparounds.
-	 */
-
-	active = ((blitq->done_blit_handle - handle) > (1 << 23)) &&
-		((blitq->cur_blit_handle - handle) <= (1 << 23));
-
-	if (queue && active) {
-		slot = handle - blitq->done_blit_handle + blitq->cur - 1;
-		if (slot >= VIA_NUM_BLIT_SLOTS)
-			slot -= VIA_NUM_BLIT_SLOTS;
-		*queue = blitq->blit_queue + slot;
-	}
-
-	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
-
-	return active;
-}
-
-/*
- * Sync. Wait for at least three seconds for the blit to be performed.
- */
-
-static int
-via_dmablit_sync(struct drm_device *dev, uint32_t handle, int engine)
-{
-
-	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
-	drm_via_blitq_t *blitq = dev_priv->blit_queues + engine;
-	wait_queue_head_t *queue;
-	int ret = 0;
-
-	if (via_dmablit_active(blitq, engine, handle, &queue)) {
-		VIA_WAIT_ON(ret, *queue, 3 * HZ,
-			    !via_dmablit_active(blitq, engine, handle, NULL));
-	}
-	DRM_DEBUG("DMA blit sync handle 0x%x engine %d returned %d\n",
-		  handle, engine, ret);
-
-	return ret;
-}
-
-
-/*
- * A timer that regularly polls the blit engine in cases where we don't have interrupts:
- * a) Broken hardware (typically those that don't have any video capture facility).
- * b) Blit abort. The hardware doesn't send an interrupt when a blit is aborted.
- * The timer and hardware IRQ's can and do work in parallel. If the hardware has
- * irqs, it will shorten the latency somewhat.
- */
-
-
-
-static void
-via_dmablit_timer(struct timer_list *t)
-{
-	drm_via_blitq_t *blitq = from_timer(blitq, t, poll_timer);
-	struct drm_device *dev = blitq->dev;
-	int engine = (int)
-		(blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues);
-
-	DRM_DEBUG("Polling timer called for engine %d, jiffies %lu\n", engine,
-		  (unsigned long) jiffies);
-
-	via_dmablit_handler(dev, engine, 0);
-
-	if (!timer_pending(&blitq->poll_timer)) {
-		mod_timer(&blitq->poll_timer, jiffies + 1);
-
-	       /*
-		* Rerun handler to delete timer if engines are off, and
-		* to shorten abort latency. This is a little nasty.
-		*/
-
-	       via_dmablit_handler(dev, engine, 0);
-
-	}
-}
-
-
-
-
-/*
- * Workqueue task that frees data and mappings associated with a blit.
- * Also wakes up waiting processes. Each of these tasks handles one
- * blit engine only and may not be called on each interrupt.
- */
-
-
-static void
-via_dmablit_workqueue(struct work_struct *work)
-{
-	drm_via_blitq_t *blitq = container_of(work, drm_via_blitq_t, wq);
-	struct drm_device *dev = blitq->dev;
-	struct pci_dev *pdev = to_pci_dev(dev->dev);
-	unsigned long irqsave;
-	drm_via_sg_info_t *cur_sg;
-	int cur_released;
-
-
-	DRM_DEBUG("Workqueue task called for blit engine %ld\n", (unsigned long)
-		  (blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues));
-
-	spin_lock_irqsave(&blitq->blit_lock, irqsave);
-
-	while (blitq->serviced != blitq->cur) {
-
-		cur_released = blitq->serviced++;
-
-		DRM_DEBUG("Releasing blit slot %d\n", cur_released);
-
-		if (blitq->serviced >= VIA_NUM_BLIT_SLOTS)
-			blitq->serviced = 0;
-
-		cur_sg = blitq->blits[cur_released];
-		blitq->num_free++;
-
-		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
-
-		wake_up(&blitq->busy_queue);
-
-		via_free_sg_info(pdev, cur_sg);
-		kfree(cur_sg);
-
-		spin_lock_irqsave(&blitq->blit_lock, irqsave);
-	}
-
-	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
-}
-
-
-/*
- * Init all blit engines. Currently we use two, but some hardware have 4.
- */
-
-
-void
-via_init_dmablit(struct drm_device *dev)
-{
-	int i, j;
-	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
-	struct pci_dev *pdev = to_pci_dev(dev->dev);
-	drm_via_blitq_t *blitq;
-
-	pci_set_master(pdev);
-
-	for (i = 0; i < VIA_NUM_BLIT_ENGINES; ++i) {
-		blitq = dev_priv->blit_queues + i;
-		blitq->dev = dev;
-		blitq->cur_blit_handle = 0;
-		blitq->done_blit_handle = 0;
-		blitq->head = 0;
-		blitq->cur = 0;
-		blitq->serviced = 0;
-		blitq->num_free = VIA_NUM_BLIT_SLOTS - 1;
-		blitq->num_outstanding = 0;
-		blitq->is_active = 0;
-		blitq->aborting = 0;
-		spin_lock_init(&blitq->blit_lock);
-		for (j = 0; j < VIA_NUM_BLIT_SLOTS; ++j)
-			init_waitqueue_head(blitq->blit_queue + j);
-		init_waitqueue_head(&blitq->busy_queue);
-		INIT_WORK(&blitq->wq, via_dmablit_workqueue);
-		timer_setup(&blitq->poll_timer, via_dmablit_timer, 0);
-	}
-}
-
-/*
- * Build all info and do all mappings required for a blit.
- */
-
-
-static int
-via_build_sg_info(struct drm_device *dev, drm_via_sg_info_t *vsg, drm_via_dmablit_t *xfer)
-{
-	struct pci_dev *pdev = to_pci_dev(dev->dev);
-	int draw = xfer->to_fb;
-	int ret = 0;
-
-	vsg->direction = (draw) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
-	vsg->bounce_buffer = NULL;
-
-	vsg->state = dr_via_sg_init;
-
-	if (xfer->num_lines <= 0 || xfer->line_length <= 0) {
-		DRM_ERROR("Zero size bitblt.\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * Below check is a driver limitation, not a hardware one. We
-	 * don't want to lock unused pages, and don't want to incoporate the
-	 * extra logic of avoiding them. Make sure there are no.
-	 * (Not a big limitation anyway.)
-	 */
-
-	if ((xfer->mem_stride - xfer->line_length) > 2*PAGE_SIZE) {
-		DRM_ERROR("Too large system memory stride. Stride: %d, "
-			  "Length: %d\n", xfer->mem_stride, xfer->line_length);
-		return -EINVAL;
-	}
-
-	if ((xfer->mem_stride == xfer->line_length) &&
-	   (xfer->fb_stride == xfer->line_length)) {
-		xfer->mem_stride *= xfer->num_lines;
-		xfer->line_length = xfer->mem_stride;
-		xfer->fb_stride = xfer->mem_stride;
-		xfer->num_lines = 1;
-	}
-
-	/*
-	 * Don't lock an arbitrary large number of pages, since that causes a
-	 * DOS security hole.
-	 */
-
-	if (xfer->num_lines > 2048 || (xfer->num_lines*xfer->mem_stride > (2048*2048*4))) {
-		DRM_ERROR("Too large PCI DMA bitblt.\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * we allow a negative fb stride to allow flipping of images in
-	 * transfer.
-	 */
-
-	if (xfer->mem_stride < xfer->line_length ||
-		abs(xfer->fb_stride) < xfer->line_length) {
-		DRM_ERROR("Invalid frame-buffer / memory stride.\n");
-		return -EINVAL;
-	}
-
-	/*
-	 * A hardware bug seems to be worked around if system memory addresses start on
-	 * 16 byte boundaries. This seems a bit restrictive however. VIA is contacted
-	 * about this. Meanwhile, impose the following restrictions:
-	 */
-
-#ifdef VIA_BUGFREE
-	if ((((unsigned long)xfer->mem_addr & 3) != ((unsigned long)xfer->fb_addr & 3)) ||
-	    ((xfer->num_lines > 1) && ((xfer->mem_stride & 3) != (xfer->fb_stride & 3)))) {
-		DRM_ERROR("Invalid DRM bitblt alignment.\n");
-		return -EINVAL;
-	}
-#else
-	if ((((unsigned long)xfer->mem_addr & 15) ||
-	      ((unsigned long)xfer->fb_addr & 3)) ||
-	   ((xfer->num_lines > 1) &&
-	   ((xfer->mem_stride & 15) || (xfer->fb_stride & 3)))) {
-		DRM_ERROR("Invalid DRM bitblt alignment.\n");
-		return -EINVAL;
-	}
-#endif
-
-	if (0 != (ret = via_lock_all_dma_pages(vsg, xfer))) {
-		DRM_ERROR("Could not lock DMA pages.\n");
-		via_free_sg_info(pdev, vsg);
-		return ret;
-	}
-
-	via_map_blit_for_device(pdev, xfer, vsg, 0);
-	if (0 != (ret = via_alloc_desc_pages(vsg))) {
-		DRM_ERROR("Could not allocate DMA descriptor pages.\n");
-		via_free_sg_info(pdev, vsg);
-		return ret;
-	}
-	via_map_blit_for_device(pdev, xfer, vsg, 1);
-
-	return 0;
-}
-
-
-/*
- * Reserve one free slot in the blit queue. Will wait for one second for one
- * to become available. Otherwise -EBUSY is returned.
- */
-
-static int
-via_dmablit_grab_slot(drm_via_blitq_t *blitq, int engine)
-{
-	int ret = 0;
-	unsigned long irqsave;
-
-	DRM_DEBUG("Num free is %d\n", blitq->num_free);
-	spin_lock_irqsave(&blitq->blit_lock, irqsave);
-	while (blitq->num_free == 0) {
-		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
-
-		VIA_WAIT_ON(ret, blitq->busy_queue, HZ, blitq->num_free > 0);
-		if (ret)
-			return (-EINTR == ret) ? -EAGAIN : ret;
-
-		spin_lock_irqsave(&blitq->blit_lock, irqsave);
-	}
-
-	blitq->num_free--;
-	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
-
-	return 0;
-}
-
-/*
- * Hand back a free slot if we changed our mind.
- */
-
-static void
-via_dmablit_release_slot(drm_via_blitq_t *blitq)
-{
-	unsigned long irqsave;
-
-	spin_lock_irqsave(&blitq->blit_lock, irqsave);
-	blitq->num_free++;
-	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
-	wake_up(&blitq->busy_queue);
-}
-
-/*
- * Grab a free slot. Build blit info and queue a blit.
- */
-
-
-static int
-via_dmablit(struct drm_device *dev, drm_via_dmablit_t *xfer)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
-	drm_via_sg_info_t *vsg;
-	drm_via_blitq_t *blitq;
-	int ret;
-	int engine;
-	unsigned long irqsave;
-
-	if (dev_priv == NULL) {
-		DRM_ERROR("Called without initialization.\n");
-		return -EINVAL;
-	}
-
-	engine = (xfer->to_fb) ? 0 : 1;
-	blitq = dev_priv->blit_queues + engine;
-	if (0 != (ret = via_dmablit_grab_slot(blitq, engine)))
-		return ret;
-	if (NULL == (vsg = kmalloc(sizeof(*vsg), GFP_KERNEL))) {
-		via_dmablit_release_slot(blitq);
-		return -ENOMEM;
-	}
-	if (0 != (ret = via_build_sg_info(dev, vsg, xfer))) {
-		via_dmablit_release_slot(blitq);
-		kfree(vsg);
-		return ret;
-	}
-	spin_lock_irqsave(&blitq->blit_lock, irqsave);
-
-	blitq->blits[blitq->head++] = vsg;
-	if (blitq->head >= VIA_NUM_BLIT_SLOTS)
-		blitq->head = 0;
-	blitq->num_outstanding++;
-	xfer->sync.sync_handle = ++blitq->cur_blit_handle;
-
-	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
-	xfer->sync.engine = engine;
-
-	via_dmablit_handler(dev, engine, 0);
-
-	return 0;
-}
-
-/*
- * Sync on a previously submitted blit. Note that the X server use signals extensively, and
- * that there is a very big probability that this IOCTL will be interrupted by a signal. In that
- * case it returns with -EAGAIN for the signal to be delivered.
- * The caller should then reissue the IOCTL. This is similar to what is being done for drmGetLock().
- */
-
-int
-via_dma_blit_sync(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_blitsync_t *sync = data;
-	int err;
-
-	if (sync->engine >= VIA_NUM_BLIT_ENGINES)
-		return -EINVAL;
-
-	err = via_dmablit_sync(dev, sync->sync_handle, sync->engine);
-
-	if (-EINTR == err)
-		err = -EAGAIN;
-
-	return err;
-}
-
-
-/*
- * Queue a blit and hand back a handle to be used for sync. This IOCTL may be interrupted by a signal
- * while waiting for a free slot in the blit queue. In that case it returns with -EAGAIN and should
- * be reissued. See the above IOCTL code.
- */
-
-int
-via_dma_blit(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_dmablit_t *xfer = data;
-	int err;
-
-	err = via_dmablit(dev, xfer);
-
-	return err;
-}
diff --git a/drivers/gpu/drm/via/via_dmablit.h b/drivers/gpu/drm/via/via_dmablit.h
deleted file mode 100644
index 9b662a327cef..000000000000
--- a/drivers/gpu/drm/via/via_dmablit.h
+++ /dev/null
@@ -1,140 +0,0 @@
-/* via_dmablit.h -- PCI DMA BitBlt support for the VIA Unichrome/Pro
- *
- * Copyright 2005 Thomas Hellstrom.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
- * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
- * USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Thomas Hellstrom.
- *    Register info from Digeo Inc.
- */
-
-#ifndef _VIA_DMABLIT_H
-#define _VIA_DMABLIT_H
-
-#include <linux/dma-mapping.h>
-
-#define VIA_NUM_BLIT_ENGINES 2
-#define VIA_NUM_BLIT_SLOTS 8
-
-struct _drm_via_descriptor;
-
-typedef struct _drm_via_sg_info {
-	struct page **pages;
-	unsigned long num_pages;
-	struct _drm_via_descriptor **desc_pages;
-	int num_desc_pages;
-	int num_desc;
-	enum dma_data_direction direction;
-	unsigned char *bounce_buffer;
-	dma_addr_t chain_start;
-	uint32_t free_on_sequence;
-	unsigned int descriptors_per_page;
-	int aborted;
-	enum {
-		dr_via_device_mapped,
-		dr_via_desc_pages_alloc,
-		dr_via_pages_locked,
-		dr_via_pages_alloc,
-		dr_via_sg_init
-	} state;
-} drm_via_sg_info_t;
-
-typedef struct _drm_via_blitq {
-	struct drm_device *dev;
-	uint32_t cur_blit_handle;
-	uint32_t done_blit_handle;
-	unsigned serviced;
-	unsigned head;
-	unsigned cur;
-	unsigned num_free;
-	unsigned num_outstanding;
-	unsigned long end;
-	int aborting;
-	int is_active;
-	drm_via_sg_info_t *blits[VIA_NUM_BLIT_SLOTS];
-	spinlock_t blit_lock;
-	wait_queue_head_t blit_queue[VIA_NUM_BLIT_SLOTS];
-	wait_queue_head_t busy_queue;
-	struct work_struct wq;
-	struct timer_list poll_timer;
-} drm_via_blitq_t;
-
-
-/*
- *  PCI DMA Registers
- *  Channels 2 & 3 don't seem to be implemented in hardware.
- */
-
-#define VIA_PCI_DMA_MAR0            0xE40   /* Memory Address Register of Channel 0 */
-#define VIA_PCI_DMA_DAR0            0xE44   /* Device Address Register of Channel 0 */
-#define VIA_PCI_DMA_BCR0            0xE48   /* Byte Count Register of Channel 0 */
-#define VIA_PCI_DMA_DPR0            0xE4C   /* Descriptor Pointer Register of Channel 0 */
-
-#define VIA_PCI_DMA_MAR1            0xE50   /* Memory Address Register of Channel 1 */
-#define VIA_PCI_DMA_DAR1            0xE54   /* Device Address Register of Channel 1 */
-#define VIA_PCI_DMA_BCR1            0xE58   /* Byte Count Register of Channel 1 */
-#define VIA_PCI_DMA_DPR1            0xE5C   /* Descriptor Pointer Register of Channel 1 */
-
-#define VIA_PCI_DMA_MAR2            0xE60   /* Memory Address Register of Channel 2 */
-#define VIA_PCI_DMA_DAR2            0xE64   /* Device Address Register of Channel 2 */
-#define VIA_PCI_DMA_BCR2            0xE68   /* Byte Count Register of Channel 2 */
-#define VIA_PCI_DMA_DPR2            0xE6C   /* Descriptor Pointer Register of Channel 2 */
-
-#define VIA_PCI_DMA_MAR3            0xE70   /* Memory Address Register of Channel 3 */
-#define VIA_PCI_DMA_DAR3            0xE74   /* Device Address Register of Channel 3 */
-#define VIA_PCI_DMA_BCR3            0xE78   /* Byte Count Register of Channel 3 */
-#define VIA_PCI_DMA_DPR3            0xE7C   /* Descriptor Pointer Register of Channel 3 */
-
-#define VIA_PCI_DMA_MR0             0xE80   /* Mode Register of Channel 0 */
-#define VIA_PCI_DMA_MR1             0xE84   /* Mode Register of Channel 1 */
-#define VIA_PCI_DMA_MR2             0xE88   /* Mode Register of Channel 2 */
-#define VIA_PCI_DMA_MR3             0xE8C   /* Mode Register of Channel 3 */
-
-#define VIA_PCI_DMA_CSR0            0xE90   /* Command/Status Register of Channel 0 */
-#define VIA_PCI_DMA_CSR1            0xE94   /* Command/Status Register of Channel 1 */
-#define VIA_PCI_DMA_CSR2            0xE98   /* Command/Status Register of Channel 2 */
-#define VIA_PCI_DMA_CSR3            0xE9C   /* Command/Status Register of Channel 3 */
-
-#define VIA_PCI_DMA_PTR             0xEA0   /* Priority Type Register */
-
-/* Define for DMA engine */
-/* DPR */
-#define VIA_DMA_DPR_EC		(1<<1)	/* end of chain */
-#define VIA_DMA_DPR_DDIE	(1<<2)	/* descriptor done interrupt enable */
-#define VIA_DMA_DPR_DT		(1<<3)	/* direction of transfer (RO) */
-
-/* MR */
-#define VIA_DMA_MR_CM		(1<<0)	/* chaining mode */
-#define VIA_DMA_MR_TDIE		(1<<1)	/* transfer done interrupt enable */
-#define VIA_DMA_MR_HENDMACMD		(1<<7) /* ? */
-
-/* CSR */
-#define VIA_DMA_CSR_DE		(1<<0)	/* DMA enable */
-#define VIA_DMA_CSR_TS		(1<<1)	/* transfer start */
-#define VIA_DMA_CSR_TA		(1<<2)	/* transfer abort */
-#define VIA_DMA_CSR_TD		(1<<3)	/* transfer done */
-#define VIA_DMA_CSR_DD		(1<<4)	/* descriptor done */
-#define VIA_DMA_DPR_EC          (1<<1)  /* end of chain */
-
-
-
-#endif
diff --git a/drivers/gpu/drm/via/via_dri1.c b/drivers/gpu/drm/via/via_dri1.c
new file mode 100644
index 000000000000..217d1e84b0ea
--- /dev/null
+++ b/drivers/gpu/drm/via/via_dri1.c
@@ -0,0 +1,3630 @@
+/*
+ * Copyright 1998-2003 VIA Technologies, Inc. All Rights Reserved.
+ * Copyright 2001-2003 S3 Graphics, Inc. All Rights Reserved.
+ * Copyright 2002 Tungsten Graphics, Inc.
+ * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas. All Rights Reserved.
+ * Copyright 2006 Tungsten Graphics Inc., Bismarck, ND., USA.
+ * Copyright 2004 Digeo, Inc., Palo Alto, CA, U.S.A. All Rights Reserved.
+ * Copyright 2004 The Unichrome project. All Rights Reserved.
+ * Copyright 2004 BEAM Ltd.
+ * Copyright 2005 Thomas Hellstrom. All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sub license,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * VIA, S3 GRAPHICS, AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/vmalloc.h>
+
+#include <drm/drm_drv.h>
+#include <drm/drm_file.h>
+#include <drm/drm_ioctl.h>
+#include <drm/drm_legacy.h>
+#include <drm/drm_mm.h>
+#include <drm/drm_pciids.h>
+#include <drm/drm_print.h>
+#include <drm/drm_vblank.h>
+#include <drm/via_drm.h>
+
+#include "via_3d_reg.h"
+
+#define DRIVER_AUTHOR	"Various"
+
+#define DRIVER_NAME		"via"
+#define DRIVER_DESC		"VIA Unichrome / Pro"
+#define DRIVER_DATE		"20070202"
+
+#define DRIVER_MAJOR		2
+#define DRIVER_MINOR		11
+#define DRIVER_PATCHLEVEL	1
+
+typedef enum {
+	no_sequence = 0,
+	z_address,
+	dest_address,
+	tex_address
+} drm_via_sequence_t;
+
+typedef struct {
+	unsigned texture;
+	uint32_t z_addr;
+	uint32_t d_addr;
+	uint32_t t_addr[2][10];
+	uint32_t pitch[2][10];
+	uint32_t height[2][10];
+	uint32_t tex_level_lo[2];
+	uint32_t tex_level_hi[2];
+	uint32_t tex_palette_size[2];
+	uint32_t tex_npot[2];
+	drm_via_sequence_t unfinished;
+	int agp_texture;
+	int multitex;
+	struct drm_device *dev;
+	drm_local_map_t *map_cache;
+	uint32_t vertex_count;
+	int agp;
+	const uint32_t *buf_start;
+} drm_via_state_t;
+
+#define VIA_PCI_BUF_SIZE 60000
+#define VIA_FIRE_BUF_SIZE  1024
+#define VIA_NUM_IRQS 4
+
+
+#define VIA_NUM_BLIT_ENGINES 2
+#define VIA_NUM_BLIT_SLOTS 8
+
+struct _drm_via_descriptor;
+
+typedef struct _drm_via_sg_info {
+	struct page **pages;
+	unsigned long num_pages;
+	struct _drm_via_descriptor **desc_pages;
+	int num_desc_pages;
+	int num_desc;
+	enum dma_data_direction direction;
+	unsigned char *bounce_buffer;
+	dma_addr_t chain_start;
+	uint32_t free_on_sequence;
+	unsigned int descriptors_per_page;
+	int aborted;
+	enum {
+		dr_via_device_mapped,
+		dr_via_desc_pages_alloc,
+		dr_via_pages_locked,
+		dr_via_pages_alloc,
+		dr_via_sg_init
+	} state;
+} drm_via_sg_info_t;
+
+typedef struct _drm_via_blitq {
+	struct drm_device *dev;
+	uint32_t cur_blit_handle;
+	uint32_t done_blit_handle;
+	unsigned serviced;
+	unsigned head;
+	unsigned cur;
+	unsigned num_free;
+	unsigned num_outstanding;
+	unsigned long end;
+	int aborting;
+	int is_active;
+	drm_via_sg_info_t *blits[VIA_NUM_BLIT_SLOTS];
+	spinlock_t blit_lock;
+	wait_queue_head_t blit_queue[VIA_NUM_BLIT_SLOTS];
+	wait_queue_head_t busy_queue;
+	struct work_struct wq;
+	struct timer_list poll_timer;
+} drm_via_blitq_t;
+
+typedef struct drm_via_ring_buffer {
+	drm_local_map_t map;
+	char *virtual_start;
+} drm_via_ring_buffer_t;
+
+typedef uint32_t maskarray_t[5];
+
+typedef struct drm_via_irq {
+	atomic_t irq_received;
+	uint32_t pending_mask;
+	uint32_t enable_mask;
+	wait_queue_head_t irq_queue;
+} drm_via_irq_t;
+
+typedef struct drm_via_private {
+	drm_via_sarea_t *sarea_priv;
+	drm_local_map_t *sarea;
+	drm_local_map_t *fb;
+	drm_local_map_t *mmio;
+	unsigned long agpAddr;
+	wait_queue_head_t decoder_queue[VIA_NR_XVMC_LOCKS];
+	char *dma_ptr;
+	unsigned int dma_low;
+	unsigned int dma_high;
+	unsigned int dma_offset;
+	uint32_t dma_wrap;
+	volatile uint32_t *last_pause_ptr;
+	volatile uint32_t *hw_addr_ptr;
+	drm_via_ring_buffer_t ring;
+	ktime_t last_vblank;
+	int last_vblank_valid;
+	ktime_t nsec_per_vblank;
+	atomic_t vbl_received;
+	drm_via_state_t hc_state;
+	char pci_buf[VIA_PCI_BUF_SIZE];
+	const uint32_t *fire_offsets[VIA_FIRE_BUF_SIZE];
+	uint32_t num_fire_offsets;
+	int chipset;
+	drm_via_irq_t via_irqs[VIA_NUM_IRQS];
+	unsigned num_irqs;
+	maskarray_t *irq_masks;
+	uint32_t irq_enable_mask;
+	uint32_t irq_pending_mask;
+	int *irq_map;
+	unsigned int idle_fault;
+	int vram_initialized;
+	struct drm_mm vram_mm;
+	int agp_initialized;
+	struct drm_mm agp_mm;
+	/** Mapping of userspace keys to mm objects */
+	struct idr object_idr;
+	unsigned long vram_offset;
+	unsigned long agp_offset;
+	drm_via_blitq_t blit_queues[VIA_NUM_BLIT_ENGINES];
+	uint32_t dma_diff;
+} drm_via_private_t;
+
+struct via_file_private {
+	struct list_head obj_list;
+};
+
+enum via_family {
+  VIA_OTHER = 0,     /* Baseline */
+  VIA_PRO_GROUP_A,   /* Another video engine and DMA commands */
+  VIA_DX9_0          /* Same video as pro_group_a, but 3D is unsupported */
+};
+
+/* VIA MMIO register access */
+static inline u32 via_read(struct drm_via_private *dev_priv, u32 reg)
+{
+	return readl((void __iomem *)(dev_priv->mmio->handle + reg));
+}
+
+static inline void via_write(struct drm_via_private *dev_priv, u32 reg,
+			     u32 val)
+{
+	writel(val, (void __iomem *)(dev_priv->mmio->handle + reg));
+}
+
+static inline void via_write8(struct drm_via_private *dev_priv, u32 reg,
+			      u32 val)
+{
+	writeb(val, (void __iomem *)(dev_priv->mmio->handle + reg));
+}
+
+static inline void via_write8_mask(struct drm_via_private *dev_priv,
+				   u32 reg, u32 mask, u32 val)
+{
+	u32 tmp;
+
+	tmp = readb((void __iomem *)(dev_priv->mmio->handle + reg));
+	tmp = (tmp & ~mask) | (val & mask);
+	writeb(tmp, (void __iomem *)(dev_priv->mmio->handle + reg));
+}
+
+/*
+ * Poll in a loop waiting for 'contidition' to be true.
+ * Note: A direct replacement with wait_event_interruptible_timeout()
+ *       will not work unless driver is updated to emit wake_up()
+ *       in relevant places that can impact the 'condition'
+ *
+ * Returns:
+ *   ret keeps current value if 'condition' becomes true
+ *   ret = -BUSY if timeout happens
+ *   ret = -EINTR if a signal interrupted the waiting period
+ */
+#define VIA_WAIT_ON( ret, queue, timeout, condition )		\
+do {								\
+	DECLARE_WAITQUEUE(entry, current);			\
+	unsigned long end = jiffies + (timeout);		\
+	add_wait_queue(&(queue), &entry);			\
+								\
+	for (;;) {						\
+		__set_current_state(TASK_INTERRUPTIBLE);	\
+		if (condition)					\
+			break;					\
+		if (time_after_eq(jiffies, end)) {		\
+			ret = -EBUSY;				\
+			break;					\
+		}						\
+		schedule_timeout((HZ/100 > 1) ? HZ/100 : 1);	\
+		if (signal_pending(current)) {			\
+			ret = -EINTR;				\
+			break;					\
+		}						\
+	}							\
+	__set_current_state(TASK_RUNNING);			\
+	remove_wait_queue(&(queue), &entry);			\
+} while (0)
+
+int via_do_cleanup_map(struct drm_device *dev);
+
+int via_dma_cleanup(struct drm_device *dev);
+int via_driver_dma_quiescent(struct drm_device *dev);
+
+#define CMDBUF_ALIGNMENT_SIZE   (0x100)
+#define CMDBUF_ALIGNMENT_MASK   (0x0ff)
+
+/* defines for VIA 3D registers */
+#define VIA_REG_STATUS          0x400
+#define VIA_REG_TRANSET         0x43C
+#define VIA_REG_TRANSPACE       0x440
+
+/* VIA_REG_STATUS(0x400): Engine Status */
+#define VIA_CMD_RGTR_BUSY       0x00000080	/* Command Regulator is busy */
+#define VIA_2D_ENG_BUSY         0x00000001	/* 2D Engine is busy */
+#define VIA_3D_ENG_BUSY         0x00000002	/* 3D Engine is busy */
+#define VIA_VR_QUEUE_BUSY       0x00020000	/* Virtual Queue is busy */
+
+#define SetReg2DAGP(nReg, nData) {				\
+	*((uint32_t *)(vb)) = ((nReg) >> 2) | HALCYON_HEADER1;	\
+	*((uint32_t *)(vb) + 1) = (nData);			\
+	vb = ((uint32_t *)vb) + 2;				\
+	dev_priv->dma_low += 8;					\
+}
+
+#define via_flush_write_combine() mb()
+
+#define VIA_OUT_RING_QW(w1, w2)	do {		\
+	*vb++ = (w1);				\
+	*vb++ = (w2);				\
+	dev_priv->dma_low += 8;			\
+} while (0)
+
+#define VIA_MM_ALIGN_SHIFT 4
+#define VIA_MM_ALIGN_MASK ((1 << VIA_MM_ALIGN_SHIFT) - 1)
+
+struct via_memblock {
+	struct drm_mm_node mm_node;
+	struct list_head owner_list;
+};
+
+#define VIA_REG_INTERRUPT       0x200
+
+/* VIA_REG_INTERRUPT */
+#define VIA_IRQ_GLOBAL	  (1 << 31)
+#define VIA_IRQ_VBLANK_ENABLE   (1 << 19)
+#define VIA_IRQ_VBLANK_PENDING  (1 << 3)
+#define VIA_IRQ_HQV0_ENABLE     (1 << 11)
+#define VIA_IRQ_HQV1_ENABLE     (1 << 25)
+#define VIA_IRQ_HQV0_PENDING    (1 << 9)
+#define VIA_IRQ_HQV1_PENDING    (1 << 10)
+#define VIA_IRQ_DMA0_DD_ENABLE  (1 << 20)
+#define VIA_IRQ_DMA0_TD_ENABLE  (1 << 21)
+#define VIA_IRQ_DMA1_DD_ENABLE  (1 << 22)
+#define VIA_IRQ_DMA1_TD_ENABLE  (1 << 23)
+#define VIA_IRQ_DMA0_DD_PENDING (1 << 4)
+#define VIA_IRQ_DMA0_TD_PENDING (1 << 5)
+#define VIA_IRQ_DMA1_DD_PENDING (1 << 6)
+#define VIA_IRQ_DMA1_TD_PENDING (1 << 7)
+
+/*
+ *  PCI DMA Registers
+ *  Channels 2 & 3 don't seem to be implemented in hardware.
+ */
+
+#define VIA_PCI_DMA_MAR0            0xE40   /* Memory Address Register of Channel 0 */
+#define VIA_PCI_DMA_DAR0            0xE44   /* Device Address Register of Channel 0 */
+#define VIA_PCI_DMA_BCR0            0xE48   /* Byte Count Register of Channel 0 */
+#define VIA_PCI_DMA_DPR0            0xE4C   /* Descriptor Pointer Register of Channel 0 */
+
+#define VIA_PCI_DMA_MAR1            0xE50   /* Memory Address Register of Channel 1 */
+#define VIA_PCI_DMA_DAR1            0xE54   /* Device Address Register of Channel 1 */
+#define VIA_PCI_DMA_BCR1            0xE58   /* Byte Count Register of Channel 1 */
+#define VIA_PCI_DMA_DPR1            0xE5C   /* Descriptor Pointer Register of Channel 1 */
+
+#define VIA_PCI_DMA_MAR2            0xE60   /* Memory Address Register of Channel 2 */
+#define VIA_PCI_DMA_DAR2            0xE64   /* Device Address Register of Channel 2 */
+#define VIA_PCI_DMA_BCR2            0xE68   /* Byte Count Register of Channel 2 */
+#define VIA_PCI_DMA_DPR2            0xE6C   /* Descriptor Pointer Register of Channel 2 */
+
+#define VIA_PCI_DMA_MAR3            0xE70   /* Memory Address Register of Channel 3 */
+#define VIA_PCI_DMA_DAR3            0xE74   /* Device Address Register of Channel 3 */
+#define VIA_PCI_DMA_BCR3            0xE78   /* Byte Count Register of Channel 3 */
+#define VIA_PCI_DMA_DPR3            0xE7C   /* Descriptor Pointer Register of Channel 3 */
+
+#define VIA_PCI_DMA_MR0             0xE80   /* Mode Register of Channel 0 */
+#define VIA_PCI_DMA_MR1             0xE84   /* Mode Register of Channel 1 */
+#define VIA_PCI_DMA_MR2             0xE88   /* Mode Register of Channel 2 */
+#define VIA_PCI_DMA_MR3             0xE8C   /* Mode Register of Channel 3 */
+
+#define VIA_PCI_DMA_CSR0            0xE90   /* Command/Status Register of Channel 0 */
+#define VIA_PCI_DMA_CSR1            0xE94   /* Command/Status Register of Channel 1 */
+#define VIA_PCI_DMA_CSR2            0xE98   /* Command/Status Register of Channel 2 */
+#define VIA_PCI_DMA_CSR3            0xE9C   /* Command/Status Register of Channel 3 */
+
+#define VIA_PCI_DMA_PTR             0xEA0   /* Priority Type Register */
+
+/* Define for DMA engine */
+/* DPR */
+#define VIA_DMA_DPR_EC		(1<<1)	/* end of chain */
+#define VIA_DMA_DPR_DDIE	(1<<2)	/* descriptor done interrupt enable */
+#define VIA_DMA_DPR_DT		(1<<3)	/* direction of transfer (RO) */
+
+/* MR */
+#define VIA_DMA_MR_CM		(1<<0)	/* chaining mode */
+#define VIA_DMA_MR_TDIE		(1<<1)	/* transfer done interrupt enable */
+#define VIA_DMA_MR_HENDMACMD		(1<<7) /* ? */
+
+/* CSR */
+#define VIA_DMA_CSR_DE		(1<<0)	/* DMA enable */
+#define VIA_DMA_CSR_TS		(1<<1)	/* transfer start */
+#define VIA_DMA_CSR_TA		(1<<2)	/* transfer abort */
+#define VIA_DMA_CSR_TD		(1<<3)	/* transfer done */
+#define VIA_DMA_CSR_DD		(1<<4)	/* descriptor done */
+#define VIA_DMA_DPR_EC          (1<<1)  /* end of chain */
+
+/*
+ * Device-specific IRQs go here. This type might need to be extended with
+ * the register if there are multiple IRQ control registers.
+ * Currently we activate the HQV interrupts of  Unichrome Pro group A.
+ */
+
+static maskarray_t via_pro_group_a_irqs[] = {
+	{VIA_IRQ_HQV0_ENABLE, VIA_IRQ_HQV0_PENDING, 0x000003D0, 0x00008010,
+	 0x00000000 },
+	{VIA_IRQ_HQV1_ENABLE, VIA_IRQ_HQV1_PENDING, 0x000013D0, 0x00008010,
+	 0x00000000 },
+	{VIA_IRQ_DMA0_TD_ENABLE, VIA_IRQ_DMA0_TD_PENDING, VIA_PCI_DMA_CSR0,
+	 VIA_DMA_CSR_TA | VIA_DMA_CSR_TD, 0x00000008},
+	{VIA_IRQ_DMA1_TD_ENABLE, VIA_IRQ_DMA1_TD_PENDING, VIA_PCI_DMA_CSR1,
+	 VIA_DMA_CSR_TA | VIA_DMA_CSR_TD, 0x00000008},
+};
+static int via_num_pro_group_a = ARRAY_SIZE(via_pro_group_a_irqs);
+static int via_irqmap_pro_group_a[] = {0, 1, -1, 2, -1, 3};
+
+static maskarray_t via_unichrome_irqs[] = {
+	{VIA_IRQ_DMA0_TD_ENABLE, VIA_IRQ_DMA0_TD_PENDING, VIA_PCI_DMA_CSR0,
+	 VIA_DMA_CSR_TA | VIA_DMA_CSR_TD, 0x00000008},
+	{VIA_IRQ_DMA1_TD_ENABLE, VIA_IRQ_DMA1_TD_PENDING, VIA_PCI_DMA_CSR1,
+	 VIA_DMA_CSR_TA | VIA_DMA_CSR_TD, 0x00000008}
+};
+static int via_num_unichrome = ARRAY_SIZE(via_unichrome_irqs);
+static int via_irqmap_unichrome[] = {-1, -1, -1, 0, -1, 1};
+
+
+/*
+ * Unmaps the DMA mappings.
+ * FIXME: Is this a NoOp on x86? Also
+ * FIXME: What happens if this one is called and a pending blit has previously done
+ * the same DMA mappings?
+ */
+#define VIA_PGDN(x)	     (((unsigned long)(x)) & PAGE_MASK)
+#define VIA_PGOFF(x)	    (((unsigned long)(x)) & ~PAGE_MASK)
+#define VIA_PFN(x)	      ((unsigned long)(x) >> PAGE_SHIFT)
+
+typedef struct _drm_via_descriptor {
+	uint32_t mem_addr;
+	uint32_t dev_addr;
+	uint32_t size;
+	uint32_t next;
+} drm_via_descriptor_t;
+
+typedef enum {
+	state_command,
+	state_header2,
+	state_header1,
+	state_vheader5,
+	state_vheader6,
+	state_error
+} verifier_state_t;
+
+typedef enum {
+	no_check = 0,
+	check_for_header2,
+	check_for_header1,
+	check_for_header2_err,
+	check_for_header1_err,
+	check_for_fire,
+	check_z_buffer_addr0,
+	check_z_buffer_addr1,
+	check_z_buffer_addr_mode,
+	check_destination_addr0,
+	check_destination_addr1,
+	check_destination_addr_mode,
+	check_for_dummy,
+	check_for_dd,
+	check_texture_addr0,
+	check_texture_addr1,
+	check_texture_addr2,
+	check_texture_addr3,
+	check_texture_addr4,
+	check_texture_addr5,
+	check_texture_addr6,
+	check_texture_addr7,
+	check_texture_addr8,
+	check_texture_addr_mode,
+	check_for_vertex_count,
+	check_number_texunits,
+	forbidden_command
+} hazard_t;
+
+/*
+ * Associates each hazard above with a possible multi-command
+ * sequence. For example an address that is split over multiple
+ * commands and that needs to be checked at the first command
+ * that does not include any part of the address.
+ */
+
+static drm_via_sequence_t seqs[] = {
+	no_sequence,
+	no_sequence,
+	no_sequence,
+	no_sequence,
+	no_sequence,
+	no_sequence,
+	z_address,
+	z_address,
+	z_address,
+	dest_address,
+	dest_address,
+	dest_address,
+	no_sequence,
+	no_sequence,
+	tex_address,
+	tex_address,
+	tex_address,
+	tex_address,
+	tex_address,
+	tex_address,
+	tex_address,
+	tex_address,
+	tex_address,
+	tex_address,
+	no_sequence
+};
+
+typedef struct {
+	unsigned int code;
+	hazard_t hz;
+} hz_init_t;
+
+static hz_init_t init_table1[] = {
+	{0xf2, check_for_header2_err},
+	{0xf0, check_for_header1_err},
+	{0xee, check_for_fire},
+	{0xcc, check_for_dummy},
+	{0xdd, check_for_dd},
+	{0x00, no_check},
+	{0x10, check_z_buffer_addr0},
+	{0x11, check_z_buffer_addr1},
+	{0x12, check_z_buffer_addr_mode},
+	{0x13, no_check},
+	{0x14, no_check},
+	{0x15, no_check},
+	{0x23, no_check},
+	{0x24, no_check},
+	{0x33, no_check},
+	{0x34, no_check},
+	{0x35, no_check},
+	{0x36, no_check},
+	{0x37, no_check},
+	{0x38, no_check},
+	{0x39, no_check},
+	{0x3A, no_check},
+	{0x3B, no_check},
+	{0x3C, no_check},
+	{0x3D, no_check},
+	{0x3E, no_check},
+	{0x40, check_destination_addr0},
+	{0x41, check_destination_addr1},
+	{0x42, check_destination_addr_mode},
+	{0x43, no_check},
+	{0x44, no_check},
+	{0x50, no_check},
+	{0x51, no_check},
+	{0x52, no_check},
+	{0x53, no_check},
+	{0x54, no_check},
+	{0x55, no_check},
+	{0x56, no_check},
+	{0x57, no_check},
+	{0x58, no_check},
+	{0x70, no_check},
+	{0x71, no_check},
+	{0x78, no_check},
+	{0x79, no_check},
+	{0x7A, no_check},
+	{0x7B, no_check},
+	{0x7C, no_check},
+	{0x7D, check_for_vertex_count}
+};
+
+static hz_init_t init_table2[] = {
+	{0xf2, check_for_header2_err},
+	{0xf0, check_for_header1_err},
+	{0xee, check_for_fire},
+	{0xcc, check_for_dummy},
+	{0x00, check_texture_addr0},
+	{0x01, check_texture_addr0},
+	{0x02, check_texture_addr0},
+	{0x03, check_texture_addr0},
+	{0x04, check_texture_addr0},
+	{0x05, check_texture_addr0},
+	{0x06, check_texture_addr0},
+	{0x07, check_texture_addr0},
+	{0x08, check_texture_addr0},
+	{0x09, check_texture_addr0},
+	{0x20, check_texture_addr1},
+	{0x21, check_texture_addr1},
+	{0x22, check_texture_addr1},
+	{0x23, check_texture_addr4},
+	{0x2B, check_texture_addr3},
+	{0x2C, check_texture_addr3},
+	{0x2D, check_texture_addr3},
+	{0x2E, check_texture_addr3},
+	{0x2F, check_texture_addr3},
+	{0x30, check_texture_addr3},
+	{0x31, check_texture_addr3},
+	{0x32, check_texture_addr3},
+	{0x33, check_texture_addr3},
+	{0x34, check_texture_addr3},
+	{0x4B, check_texture_addr5},
+	{0x4C, check_texture_addr6},
+	{0x51, check_texture_addr7},
+	{0x52, check_texture_addr8},
+	{0x77, check_texture_addr2},
+	{0x78, no_check},
+	{0x79, no_check},
+	{0x7A, no_check},
+	{0x7B, check_texture_addr_mode},
+	{0x7C, no_check},
+	{0x7D, no_check},
+	{0x7E, no_check},
+	{0x7F, no_check},
+	{0x80, no_check},
+	{0x81, no_check},
+	{0x82, no_check},
+	{0x83, no_check},
+	{0x85, no_check},
+	{0x86, no_check},
+	{0x87, no_check},
+	{0x88, no_check},
+	{0x89, no_check},
+	{0x8A, no_check},
+	{0x90, no_check},
+	{0x91, no_check},
+	{0x92, no_check},
+	{0x93, no_check}
+};
+
+static hz_init_t init_table3[] = {
+	{0xf2, check_for_header2_err},
+	{0xf0, check_for_header1_err},
+	{0xcc, check_for_dummy},
+	{0x00, check_number_texunits}
+};
+
+static hazard_t table1[256];
+static hazard_t table2[256];
+static hazard_t table3[256];
+
+static __inline__ int
+eat_words(const uint32_t **buf, const uint32_t *buf_end, unsigned num_words)
+{
+	if ((buf_end - *buf) >= num_words) {
+		*buf += num_words;
+		return 0;
+	}
+	DRM_ERROR("Illegal termination of DMA command buffer\n");
+	return 1;
+}
+
+/*
+ * Partially stolen from drm_memory.h
+ */
+
+static __inline__ drm_local_map_t *via_drm_lookup_agp_map(drm_via_state_t *seq,
+						    unsigned long offset,
+						    unsigned long size,
+						    struct drm_device *dev)
+{
+	struct drm_map_list *r_list;
+	drm_local_map_t *map = seq->map_cache;
+
+	if (map && map->offset <= offset
+	    && (offset + size) <= (map->offset + map->size)) {
+		return map;
+	}
+
+	list_for_each_entry(r_list, &dev->maplist, head) {
+		map = r_list->map;
+		if (!map)
+			continue;
+		if (map->offset <= offset
+		    && (offset + size) <= (map->offset + map->size)
+		    && !(map->flags & _DRM_RESTRICTED)
+		    && (map->type == _DRM_AGP)) {
+			seq->map_cache = map;
+			return map;
+		}
+	}
+	return NULL;
+}
+
+/*
+ * Require that all AGP texture levels reside in the same AGP map which should
+ * be mappable by the client. This is not a big restriction.
+ * FIXME: To actually enforce this security policy strictly, drm_rmmap
+ * would have to wait for dma quiescent before removing an AGP map.
+ * The via_drm_lookup_agp_map call in reality seems to take
+ * very little CPU time.
+ */
+
+static __inline__ int finish_current_sequence(drm_via_state_t * cur_seq)
+{
+	switch (cur_seq->unfinished) {
+	case z_address:
+		DRM_DEBUG("Z Buffer start address is 0x%x\n", cur_seq->z_addr);
+		break;
+	case dest_address:
+		DRM_DEBUG("Destination start address is 0x%x\n",
+			  cur_seq->d_addr);
+		break;
+	case tex_address:
+		if (cur_seq->agp_texture) {
+			unsigned start =
+			    cur_seq->tex_level_lo[cur_seq->texture];
+			unsigned end = cur_seq->tex_level_hi[cur_seq->texture];
+			unsigned long lo = ~0, hi = 0, tmp;
+			uint32_t *addr, *pitch, *height, tex;
+			unsigned i;
+			int npot;
+
+			if (end > 9)
+				end = 9;
+			if (start > 9)
+				start = 9;
+
+			addr =
+			    &(cur_seq->t_addr[tex = cur_seq->texture][start]);
+			pitch = &(cur_seq->pitch[tex][start]);
+			height = &(cur_seq->height[tex][start]);
+			npot = cur_seq->tex_npot[tex];
+			for (i = start; i <= end; ++i) {
+				tmp = *addr++;
+				if (tmp < lo)
+					lo = tmp;
+				if (i == 0 && npot)
+					tmp += (*height++ * *pitch++);
+				else
+					tmp += (*height++ << *pitch++);
+				if (tmp > hi)
+					hi = tmp;
+			}
+
+			if (!via_drm_lookup_agp_map
+			    (cur_seq, lo, hi - lo, cur_seq->dev)) {
+				DRM_ERROR
+				    ("AGP texture is not in allowed map\n");
+				return 2;
+			}
+		}
+		break;
+	default:
+		break;
+	}
+	cur_seq->unfinished = no_sequence;
+	return 0;
+}
+
+static __inline__ int
+investigate_hazard(uint32_t cmd, hazard_t hz, drm_via_state_t *cur_seq)
+{
+	register uint32_t tmp, *tmp_addr;
+
+	if (cur_seq->unfinished && (cur_seq->unfinished != seqs[hz])) {
+		int ret;
+		if ((ret = finish_current_sequence(cur_seq)))
+			return ret;
+	}
+
+	switch (hz) {
+	case check_for_header2:
+		if (cmd == HALCYON_HEADER2)
+			return 1;
+		return 0;
+	case check_for_header1:
+		if ((cmd & HALCYON_HEADER1MASK) == HALCYON_HEADER1)
+			return 1;
+		return 0;
+	case check_for_header2_err:
+		if (cmd == HALCYON_HEADER2)
+			return 1;
+		DRM_ERROR("Illegal DMA HALCYON_HEADER2 command\n");
+		break;
+	case check_for_header1_err:
+		if ((cmd & HALCYON_HEADER1MASK) == HALCYON_HEADER1)
+			return 1;
+		DRM_ERROR("Illegal DMA HALCYON_HEADER1 command\n");
+		break;
+	case check_for_fire:
+		if ((cmd & HALCYON_FIREMASK) == HALCYON_FIRECMD)
+			return 1;
+		DRM_ERROR("Illegal DMA HALCYON_FIRECMD command\n");
+		break;
+	case check_for_dummy:
+		if (HC_DUMMY == cmd)
+			return 0;
+		DRM_ERROR("Illegal DMA HC_DUMMY command\n");
+		break;
+	case check_for_dd:
+		if (0xdddddddd == cmd)
+			return 0;
+		DRM_ERROR("Illegal DMA 0xdddddddd command\n");
+		break;
+	case check_z_buffer_addr0:
+		cur_seq->unfinished = z_address;
+		cur_seq->z_addr = (cur_seq->z_addr & 0xFF000000) |
+		    (cmd & 0x00FFFFFF);
+		return 0;
+	case check_z_buffer_addr1:
+		cur_seq->unfinished = z_address;
+		cur_seq->z_addr = (cur_seq->z_addr & 0x00FFFFFF) |
+		    ((cmd & 0xFF) << 24);
+		return 0;
+	case check_z_buffer_addr_mode:
+		cur_seq->unfinished = z_address;
+		if ((cmd & 0x0000C000) == 0)
+			return 0;
+		DRM_ERROR("Attempt to place Z buffer in system memory\n");
+		return 2;
+	case check_destination_addr0:
+		cur_seq->unfinished = dest_address;
+		cur_seq->d_addr = (cur_seq->d_addr & 0xFF000000) |
+		    (cmd & 0x00FFFFFF);
+		return 0;
+	case check_destination_addr1:
+		cur_seq->unfinished = dest_address;
+		cur_seq->d_addr = (cur_seq->d_addr & 0x00FFFFFF) |
+		    ((cmd & 0xFF) << 24);
+		return 0;
+	case check_destination_addr_mode:
+		cur_seq->unfinished = dest_address;
+		if ((cmd & 0x0000C000) == 0)
+			return 0;
+		DRM_ERROR
+		    ("Attempt to place 3D drawing buffer in system memory\n");
+		return 2;
+	case check_texture_addr0:
+		cur_seq->unfinished = tex_address;
+		tmp = (cmd >> 24);
+		tmp_addr = &cur_seq->t_addr[cur_seq->texture][tmp];
+		*tmp_addr = (*tmp_addr & 0xFF000000) | (cmd & 0x00FFFFFF);
+		return 0;
+	case check_texture_addr1:
+		cur_seq->unfinished = tex_address;
+		tmp = ((cmd >> 24) - 0x20);
+		tmp += tmp << 1;
+		tmp_addr = &cur_seq->t_addr[cur_seq->texture][tmp];
+		*tmp_addr = (*tmp_addr & 0x00FFFFFF) | ((cmd & 0xFF) << 24);
+		tmp_addr++;
+		*tmp_addr = (*tmp_addr & 0x00FFFFFF) | ((cmd & 0xFF00) << 16);
+		tmp_addr++;
+		*tmp_addr = (*tmp_addr & 0x00FFFFFF) | ((cmd & 0xFF0000) << 8);
+		return 0;
+	case check_texture_addr2:
+		cur_seq->unfinished = tex_address;
+		cur_seq->tex_level_lo[tmp = cur_seq->texture] = cmd & 0x3F;
+		cur_seq->tex_level_hi[tmp] = (cmd & 0xFC0) >> 6;
+		return 0;
+	case check_texture_addr3:
+		cur_seq->unfinished = tex_address;
+		tmp = ((cmd >> 24) - HC_SubA_HTXnL0Pit);
+		if (tmp == 0 &&
+		    (cmd & HC_HTXnEnPit_MASK)) {
+			cur_seq->pitch[cur_seq->texture][tmp] =
+				(cmd & HC_HTXnLnPit_MASK);
+			cur_seq->tex_npot[cur_seq->texture] = 1;
+		} else {
+			cur_seq->pitch[cur_seq->texture][tmp] =
+				(cmd & HC_HTXnLnPitE_MASK) >> HC_HTXnLnPitE_SHIFT;
+			cur_seq->tex_npot[cur_seq->texture] = 0;
+			if (cmd & 0x000FFFFF) {
+				DRM_ERROR
+					("Unimplemented texture level 0 pitch mode.\n");
+				return 2;
+			}
+		}
+		return 0;
+	case check_texture_addr4:
+		cur_seq->unfinished = tex_address;
+		tmp_addr = &cur_seq->t_addr[cur_seq->texture][9];
+		*tmp_addr = (*tmp_addr & 0x00FFFFFF) | ((cmd & 0xFF) << 24);
+		return 0;
+	case check_texture_addr5:
+	case check_texture_addr6:
+		cur_seq->unfinished = tex_address;
+		/*
+		 * Texture width. We don't care since we have the pitch.
+		 */
+		return 0;
+	case check_texture_addr7:
+		cur_seq->unfinished = tex_address;
+		tmp_addr = &(cur_seq->height[cur_seq->texture][0]);
+		tmp_addr[5] = 1 << ((cmd & 0x00F00000) >> 20);
+		tmp_addr[4] = 1 << ((cmd & 0x000F0000) >> 16);
+		tmp_addr[3] = 1 << ((cmd & 0x0000F000) >> 12);
+		tmp_addr[2] = 1 << ((cmd & 0x00000F00) >> 8);
+		tmp_addr[1] = 1 << ((cmd & 0x000000F0) >> 4);
+		tmp_addr[0] = 1 << (cmd & 0x0000000F);
+		return 0;
+	case check_texture_addr8:
+		cur_seq->unfinished = tex_address;
+		tmp_addr = &(cur_seq->height[cur_seq->texture][0]);
+		tmp_addr[9] = 1 << ((cmd & 0x0000F000) >> 12);
+		tmp_addr[8] = 1 << ((cmd & 0x00000F00) >> 8);
+		tmp_addr[7] = 1 << ((cmd & 0x000000F0) >> 4);
+		tmp_addr[6] = 1 << (cmd & 0x0000000F);
+		return 0;
+	case check_texture_addr_mode:
+		cur_seq->unfinished = tex_address;
+		if (2 == (tmp = cmd & 0x00000003)) {
+			DRM_ERROR
+			    ("Attempt to fetch texture from system memory.\n");
+			return 2;
+		}
+		cur_seq->agp_texture = (tmp == 3);
+		cur_seq->tex_palette_size[cur_seq->texture] =
+		    (cmd >> 16) & 0x000000007;
+		return 0;
+	case check_for_vertex_count:
+		cur_seq->vertex_count = cmd & 0x0000FFFF;
+		return 0;
+	case check_number_texunits:
+		cur_seq->multitex = (cmd >> 3) & 1;
+		return 0;
+	default:
+		DRM_ERROR("Illegal DMA data: 0x%x\n", cmd);
+		return 2;
+	}
+	return 2;
+}
+
+static __inline__ int
+via_check_prim_list(uint32_t const **buffer, const uint32_t * buf_end,
+		    drm_via_state_t *cur_seq)
+{
+	drm_via_private_t *dev_priv =
+	    (drm_via_private_t *) cur_seq->dev->dev_private;
+	uint32_t a_fire, bcmd, dw_count;
+	int ret = 0;
+	int have_fire;
+	const uint32_t *buf = *buffer;
+
+	while (buf < buf_end) {
+		have_fire = 0;
+		if ((buf_end - buf) < 2) {
+			DRM_ERROR
+			    ("Unexpected termination of primitive list.\n");
+			ret = 1;
+			break;
+		}
+		if ((*buf & HC_ACMD_MASK) != HC_ACMD_HCmdB)
+			break;
+		bcmd = *buf++;
+		if ((*buf & HC_ACMD_MASK) != HC_ACMD_HCmdA) {
+			DRM_ERROR("Expected Vertex List A command, got 0x%x\n",
+				  *buf);
+			ret = 1;
+			break;
+		}
+		a_fire =
+		    *buf++ | HC_HPLEND_MASK | HC_HPMValidN_MASK |
+		    HC_HE3Fire_MASK;
+
+		/*
+		 * How many dwords per vertex ?
+		 */
+
+		if (cur_seq->agp && ((bcmd & (0xF << 11)) == 0)) {
+			DRM_ERROR("Illegal B command vertex data for AGP.\n");
+			ret = 1;
+			break;
+		}
+
+		dw_count = 0;
+		if (bcmd & (1 << 7))
+			dw_count += (cur_seq->multitex) ? 2 : 1;
+		if (bcmd & (1 << 8))
+			dw_count += (cur_seq->multitex) ? 2 : 1;
+		if (bcmd & (1 << 9))
+			dw_count++;
+		if (bcmd & (1 << 10))
+			dw_count++;
+		if (bcmd & (1 << 11))
+			dw_count++;
+		if (bcmd & (1 << 12))
+			dw_count++;
+		if (bcmd & (1 << 13))
+			dw_count++;
+		if (bcmd & (1 << 14))
+			dw_count++;
+
+		while (buf < buf_end) {
+			if (*buf == a_fire) {
+				if (dev_priv->num_fire_offsets >=
+				    VIA_FIRE_BUF_SIZE) {
+					DRM_ERROR("Fire offset buffer full.\n");
+					ret = 1;
+					break;
+				}
+				dev_priv->fire_offsets[dev_priv->
+						       num_fire_offsets++] =
+				    buf;
+				have_fire = 1;
+				buf++;
+				if (buf < buf_end && *buf == a_fire)
+					buf++;
+				break;
+			}
+			if ((*buf == HALCYON_HEADER2) ||
+			    ((*buf & HALCYON_FIREMASK) == HALCYON_FIRECMD)) {
+				DRM_ERROR("Missing Vertex Fire command, "
+					  "Stray Vertex Fire command  or verifier "
+					  "lost sync.\n");
+				ret = 1;
+				break;
+			}
+			if ((ret = eat_words(&buf, buf_end, dw_count)))
+				break;
+		}
+		if (buf >= buf_end && !have_fire) {
+			DRM_ERROR("Missing Vertex Fire command or verifier "
+				  "lost sync.\n");
+			ret = 1;
+			break;
+		}
+		if (cur_seq->agp && ((buf - cur_seq->buf_start) & 0x01)) {
+			DRM_ERROR("AGP Primitive list end misaligned.\n");
+			ret = 1;
+			break;
+		}
+	}
+	*buffer = buf;
+	return ret;
+}
+
+static __inline__ verifier_state_t
+via_check_header2(uint32_t const **buffer, const uint32_t *buf_end,
+		  drm_via_state_t *hc_state)
+{
+	uint32_t cmd;
+	int hz_mode;
+	hazard_t hz;
+	const uint32_t *buf = *buffer;
+	const hazard_t *hz_table;
+
+	if ((buf_end - buf) < 2) {
+		DRM_ERROR
+		    ("Illegal termination of DMA HALCYON_HEADER2 sequence.\n");
+		return state_error;
+	}
+	buf++;
+	cmd = (*buf++ & 0xFFFF0000) >> 16;
+
+	switch (cmd) {
+	case HC_ParaType_CmdVdata:
+		if (via_check_prim_list(&buf, buf_end, hc_state))
+			return state_error;
+		*buffer = buf;
+		return state_command;
+	case HC_ParaType_NotTex:
+		hz_table = table1;
+		break;
+	case HC_ParaType_Tex:
+		hc_state->texture = 0;
+		hz_table = table2;
+		break;
+	case (HC_ParaType_Tex | (HC_SubType_Tex1 << 8)):
+		hc_state->texture = 1;
+		hz_table = table2;
+		break;
+	case (HC_ParaType_Tex | (HC_SubType_TexGeneral << 8)):
+		hz_table = table3;
+		break;
+	case HC_ParaType_Auto:
+		if (eat_words(&buf, buf_end, 2))
+			return state_error;
+		*buffer = buf;
+		return state_command;
+	case (HC_ParaType_Palette | (HC_SubType_Stipple << 8)):
+		if (eat_words(&buf, buf_end, 32))
+			return state_error;
+		*buffer = buf;
+		return state_command;
+	case (HC_ParaType_Palette | (HC_SubType_TexPalette0 << 8)):
+	case (HC_ParaType_Palette | (HC_SubType_TexPalette1 << 8)):
+		DRM_ERROR("Texture palettes are rejected because of "
+			  "lack of info how to determine their size.\n");
+		return state_error;
+	case (HC_ParaType_Palette | (HC_SubType_FogTable << 8)):
+		DRM_ERROR("Fog factor palettes are rejected because of "
+			  "lack of info how to determine their size.\n");
+		return state_error;
+	default:
+
+		/*
+		 * There are some unimplemented HC_ParaTypes here, that
+		 * need to be implemented if the Mesa driver is extended.
+		 */
+
+		DRM_ERROR("Invalid or unimplemented HALCYON_HEADER2 "
+			  "DMA subcommand: 0x%x. Previous dword: 0x%x\n",
+			  cmd, *(buf - 2));
+		*buffer = buf;
+		return state_error;
+	}
+
+	while (buf < buf_end) {
+		cmd = *buf++;
+		if ((hz = hz_table[cmd >> 24])) {
+			if ((hz_mode = investigate_hazard(cmd, hz, hc_state))) {
+				if (hz_mode == 1) {
+					buf--;
+					break;
+				}
+				return state_error;
+			}
+		} else if (hc_state->unfinished &&
+			   finish_current_sequence(hc_state)) {
+			return state_error;
+		}
+	}
+	if (hc_state->unfinished && finish_current_sequence(hc_state))
+		return state_error;
+	*buffer = buf;
+	return state_command;
+}
+
+static __inline__ verifier_state_t
+via_parse_header2(drm_via_private_t *dev_priv, uint32_t const **buffer,
+		  const uint32_t *buf_end, int *fire_count)
+{
+	uint32_t cmd;
+	const uint32_t *buf = *buffer;
+	const uint32_t *next_fire;
+	int burst = 0;
+
+	next_fire = dev_priv->fire_offsets[*fire_count];
+	buf++;
+	cmd = (*buf & 0xFFFF0000) >> 16;
+	via_write(dev_priv, HC_REG_TRANS_SET + HC_REG_BASE, *buf++);
+	switch (cmd) {
+	case HC_ParaType_CmdVdata:
+		while ((buf < buf_end) &&
+		       (*fire_count < dev_priv->num_fire_offsets) &&
+		       (*buf & HC_ACMD_MASK) == HC_ACMD_HCmdB) {
+			while (buf <= next_fire) {
+				via_write(dev_priv, HC_REG_TRANS_SPACE + HC_REG_BASE +
+					  (burst & 63), *buf++);
+				burst += 4;
+			}
+			if ((buf < buf_end)
+			    && ((*buf & HALCYON_FIREMASK) == HALCYON_FIRECMD))
+				buf++;
+
+			if (++(*fire_count) < dev_priv->num_fire_offsets)
+				next_fire = dev_priv->fire_offsets[*fire_count];
+		}
+		break;
+	default:
+		while (buf < buf_end) {
+
+			if (*buf == HC_HEADER2 ||
+			    (*buf & HALCYON_HEADER1MASK) == HALCYON_HEADER1 ||
+			    (*buf & VIA_VIDEOMASK) == VIA_VIDEO_HEADER5 ||
+			    (*buf & VIA_VIDEOMASK) == VIA_VIDEO_HEADER6)
+				break;
+
+			via_write(dev_priv, HC_REG_TRANS_SPACE + HC_REG_BASE +
+				  (burst & 63), *buf++);
+			burst += 4;
+		}
+	}
+	*buffer = buf;
+	return state_command;
+}
+
+static __inline__ int verify_mmio_address(uint32_t address)
+{
+	if ((address > 0x3FF) && (address < 0xC00)) {
+		DRM_ERROR("Invalid VIDEO DMA command. "
+			  "Attempt to access 3D- or command burst area.\n");
+		return 1;
+	} else if ((address > 0xCFF) && (address < 0x1300)) {
+		DRM_ERROR("Invalid VIDEO DMA command. "
+			  "Attempt to access PCI DMA area.\n");
+		return 1;
+	} else if (address > 0x13FF) {
+		DRM_ERROR("Invalid VIDEO DMA command. "
+			  "Attempt to access VGA registers.\n");
+		return 1;
+	}
+	return 0;
+}
+
+static __inline__ int
+verify_video_tail(uint32_t const **buffer, const uint32_t * buf_end,
+		  uint32_t dwords)
+{
+	const uint32_t *buf = *buffer;
+
+	if (buf_end - buf < dwords) {
+		DRM_ERROR("Illegal termination of video command.\n");
+		return 1;
+	}
+	while (dwords--) {
+		if (*buf++) {
+			DRM_ERROR("Illegal video command tail.\n");
+			return 1;
+		}
+	}
+	*buffer = buf;
+	return 0;
+}
+
+static __inline__ verifier_state_t
+via_check_header1(uint32_t const **buffer, const uint32_t * buf_end)
+{
+	uint32_t cmd;
+	const uint32_t *buf = *buffer;
+	verifier_state_t ret = state_command;
+
+	while (buf < buf_end) {
+		cmd = *buf;
+		if ((cmd > ((0x3FF >> 2) | HALCYON_HEADER1)) &&
+		    (cmd < ((0xC00 >> 2) | HALCYON_HEADER1))) {
+			if ((cmd & HALCYON_HEADER1MASK) != HALCYON_HEADER1)
+				break;
+			DRM_ERROR("Invalid HALCYON_HEADER1 command. "
+				  "Attempt to access 3D- or command burst area.\n");
+			ret = state_error;
+			break;
+		} else if (cmd > ((0xCFF >> 2) | HALCYON_HEADER1)) {
+			if ((cmd & HALCYON_HEADER1MASK) != HALCYON_HEADER1)
+				break;
+			DRM_ERROR("Invalid HALCYON_HEADER1 command. "
+				  "Attempt to access VGA registers.\n");
+			ret = state_error;
+			break;
+		} else {
+			buf += 2;
+		}
+	}
+	*buffer = buf;
+	return ret;
+}
+
+static __inline__ verifier_state_t
+via_parse_header1(drm_via_private_t *dev_priv, uint32_t const **buffer,
+		  const uint32_t *buf_end)
+{
+	register uint32_t cmd;
+	const uint32_t *buf = *buffer;
+
+	while (buf < buf_end) {
+		cmd = *buf;
+		if ((cmd & HALCYON_HEADER1MASK) != HALCYON_HEADER1)
+			break;
+		via_write(dev_priv, (cmd & ~HALCYON_HEADER1MASK) << 2, *++buf);
+		buf++;
+	}
+	*buffer = buf;
+	return state_command;
+}
+
+static __inline__ verifier_state_t
+via_check_vheader5(uint32_t const **buffer, const uint32_t *buf_end)
+{
+	uint32_t data;
+	const uint32_t *buf = *buffer;
+
+	if (buf_end - buf < 4) {
+		DRM_ERROR("Illegal termination of video header5 command\n");
+		return state_error;
+	}
+
+	data = *buf++ & ~VIA_VIDEOMASK;
+	if (verify_mmio_address(data))
+		return state_error;
+
+	data = *buf++;
+	if (*buf++ != 0x00F50000) {
+		DRM_ERROR("Illegal header5 header data\n");
+		return state_error;
+	}
+	if (*buf++ != 0x00000000) {
+		DRM_ERROR("Illegal header5 header data\n");
+		return state_error;
+	}
+	if (eat_words(&buf, buf_end, data))
+		return state_error;
+	if ((data & 3) && verify_video_tail(&buf, buf_end, 4 - (data & 3)))
+		return state_error;
+	*buffer = buf;
+	return state_command;
+
+}
+
+static __inline__ verifier_state_t
+via_parse_vheader5(drm_via_private_t *dev_priv, uint32_t const **buffer,
+		   const uint32_t *buf_end)
+{
+	uint32_t addr, count, i;
+	const uint32_t *buf = *buffer;
+
+	addr = *buf++ & ~VIA_VIDEOMASK;
+	i = count = *buf;
+	buf += 3;
+	while (i--)
+		via_write(dev_priv, addr, *buf++);
+	if (count & 3)
+		buf += 4 - (count & 3);
+	*buffer = buf;
+	return state_command;
+}
+
+static __inline__ verifier_state_t
+via_check_vheader6(uint32_t const **buffer, const uint32_t * buf_end)
+{
+	uint32_t data;
+	const uint32_t *buf = *buffer;
+	uint32_t i;
+
+	if (buf_end - buf < 4) {
+		DRM_ERROR("Illegal termination of video header6 command\n");
+		return state_error;
+	}
+	buf++;
+	data = *buf++;
+	if (*buf++ != 0x00F60000) {
+		DRM_ERROR("Illegal header6 header data\n");
+		return state_error;
+	}
+	if (*buf++ != 0x00000000) {
+		DRM_ERROR("Illegal header6 header data\n");
+		return state_error;
+	}
+	if ((buf_end - buf) < (data << 1)) {
+		DRM_ERROR("Illegal termination of video header6 command\n");
+		return state_error;
+	}
+	for (i = 0; i < data; ++i) {
+		if (verify_mmio_address(*buf++))
+			return state_error;
+		buf++;
+	}
+	data <<= 1;
+	if ((data & 3) && verify_video_tail(&buf, buf_end, 4 - (data & 3)))
+		return state_error;
+	*buffer = buf;
+	return state_command;
+}
+
+static __inline__ verifier_state_t
+via_parse_vheader6(drm_via_private_t *dev_priv, uint32_t const **buffer,
+		   const uint32_t *buf_end)
+{
+
+	uint32_t addr, count, i;
+	const uint32_t *buf = *buffer;
+
+	i = count = *++buf;
+	buf += 3;
+	while (i--) {
+		addr = *buf++;
+		via_write(dev_priv, addr, *buf++);
+	}
+	count <<= 1;
+	if (count & 3)
+		buf += 4 - (count & 3);
+	*buffer = buf;
+	return state_command;
+}
+
+static int
+via_verify_command_stream(const uint32_t * buf, unsigned int size,
+			  struct drm_device * dev, int agp)
+{
+
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	drm_via_state_t *hc_state = &dev_priv->hc_state;
+	drm_via_state_t saved_state = *hc_state;
+	uint32_t cmd;
+	const uint32_t *buf_end = buf + (size >> 2);
+	verifier_state_t state = state_command;
+	int cme_video;
+	int supported_3d;
+
+	cme_video = (dev_priv->chipset == VIA_PRO_GROUP_A ||
+		     dev_priv->chipset == VIA_DX9_0);
+
+	supported_3d = dev_priv->chipset != VIA_DX9_0;
+
+	hc_state->dev = dev;
+	hc_state->unfinished = no_sequence;
+	hc_state->map_cache = NULL;
+	hc_state->agp = agp;
+	hc_state->buf_start = buf;
+	dev_priv->num_fire_offsets = 0;
+
+	while (buf < buf_end) {
+
+		switch (state) {
+		case state_header2:
+			state = via_check_header2(&buf, buf_end, hc_state);
+			break;
+		case state_header1:
+			state = via_check_header1(&buf, buf_end);
+			break;
+		case state_vheader5:
+			state = via_check_vheader5(&buf, buf_end);
+			break;
+		case state_vheader6:
+			state = via_check_vheader6(&buf, buf_end);
+			break;
+		case state_command:
+			cmd = *buf;
+			if ((cmd == HALCYON_HEADER2) && supported_3d)
+				state = state_header2;
+			else if ((cmd & HALCYON_HEADER1MASK) == HALCYON_HEADER1)
+				state = state_header1;
+			else if (cme_video
+				 && (cmd & VIA_VIDEOMASK) == VIA_VIDEO_HEADER5)
+				state = state_vheader5;
+			else if (cme_video
+				 && (cmd & VIA_VIDEOMASK) == VIA_VIDEO_HEADER6)
+				state = state_vheader6;
+			else if ((cmd == HALCYON_HEADER2) && !supported_3d) {
+				DRM_ERROR("Accelerated 3D is not supported on this chipset yet.\n");
+				state = state_error;
+			} else {
+				DRM_ERROR
+				    ("Invalid / Unimplemented DMA HEADER command. 0x%x\n",
+				     cmd);
+				state = state_error;
+			}
+			break;
+		case state_error:
+		default:
+			*hc_state = saved_state;
+			return -EINVAL;
+		}
+	}
+	if (state == state_error) {
+		*hc_state = saved_state;
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int
+via_parse_command_stream(struct drm_device *dev, const uint32_t *buf,
+			 unsigned int size)
+{
+
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	uint32_t cmd;
+	const uint32_t *buf_end = buf + (size >> 2);
+	verifier_state_t state = state_command;
+	int fire_count = 0;
+
+	while (buf < buf_end) {
+
+		switch (state) {
+		case state_header2:
+			state =
+			    via_parse_header2(dev_priv, &buf, buf_end,
+					      &fire_count);
+			break;
+		case state_header1:
+			state = via_parse_header1(dev_priv, &buf, buf_end);
+			break;
+		case state_vheader5:
+			state = via_parse_vheader5(dev_priv, &buf, buf_end);
+			break;
+		case state_vheader6:
+			state = via_parse_vheader6(dev_priv, &buf, buf_end);
+			break;
+		case state_command:
+			cmd = *buf;
+			if (cmd == HALCYON_HEADER2)
+				state = state_header2;
+			else if ((cmd & HALCYON_HEADER1MASK) == HALCYON_HEADER1)
+				state = state_header1;
+			else if ((cmd & VIA_VIDEOMASK) == VIA_VIDEO_HEADER5)
+				state = state_vheader5;
+			else if ((cmd & VIA_VIDEOMASK) == VIA_VIDEO_HEADER6)
+				state = state_vheader6;
+			else {
+				DRM_ERROR
+				    ("Invalid / Unimplemented DMA HEADER command. 0x%x\n",
+				     cmd);
+				state = state_error;
+			}
+			break;
+		case state_error:
+		default:
+			return -EINVAL;
+		}
+	}
+	if (state == state_error)
+		return -EINVAL;
+	return 0;
+}
+
+static void
+setup_hazard_table(hz_init_t init_table[], hazard_t table[], int size)
+{
+	int i;
+
+	for (i = 0; i < 256; ++i)
+		table[i] = forbidden_command;
+
+	for (i = 0; i < size; ++i)
+		table[init_table[i].code] = init_table[i].hz;
+}
+
+static void via_init_command_verifier(void)
+{
+	setup_hazard_table(init_table1, table1, ARRAY_SIZE(init_table1));
+	setup_hazard_table(init_table2, table2, ARRAY_SIZE(init_table2));
+	setup_hazard_table(init_table3, table3, ARRAY_SIZE(init_table3));
+}
+/*
+ * Unmap a DMA mapping.
+ */
+static void
+via_unmap_blit_from_device(struct pci_dev *pdev, drm_via_sg_info_t *vsg)
+{
+	int num_desc = vsg->num_desc;
+	unsigned cur_descriptor_page = num_desc / vsg->descriptors_per_page;
+	unsigned descriptor_this_page = num_desc % vsg->descriptors_per_page;
+	drm_via_descriptor_t *desc_ptr = vsg->desc_pages[cur_descriptor_page] +
+		descriptor_this_page;
+	dma_addr_t next = vsg->chain_start;
+
+	while (num_desc--) {
+		if (descriptor_this_page-- == 0) {
+			cur_descriptor_page--;
+			descriptor_this_page = vsg->descriptors_per_page - 1;
+			desc_ptr = vsg->desc_pages[cur_descriptor_page] +
+				descriptor_this_page;
+		}
+		dma_unmap_single(&pdev->dev, next, sizeof(*desc_ptr), DMA_TO_DEVICE);
+		dma_unmap_page(&pdev->dev, desc_ptr->mem_addr, desc_ptr->size, vsg->direction);
+		next = (dma_addr_t) desc_ptr->next;
+		desc_ptr--;
+	}
+}
+
+/*
+ * If mode = 0, count how many descriptors are needed.
+ * If mode = 1, Map the DMA pages for the device, put together and map also the descriptors.
+ * Descriptors are run in reverse order by the hardware because we are not allowed to update the
+ * 'next' field without syncing calls when the descriptor is already mapped.
+ */
+static void
+via_map_blit_for_device(struct pci_dev *pdev,
+		   const drm_via_dmablit_t *xfer,
+		   drm_via_sg_info_t *vsg,
+		   int mode)
+{
+	unsigned cur_descriptor_page = 0;
+	unsigned num_descriptors_this_page = 0;
+	unsigned char *mem_addr = xfer->mem_addr;
+	unsigned char *cur_mem;
+	unsigned char *first_addr = (unsigned char *)VIA_PGDN(mem_addr);
+	uint32_t fb_addr = xfer->fb_addr;
+	uint32_t cur_fb;
+	unsigned long line_len;
+	unsigned remaining_len;
+	int num_desc = 0;
+	int cur_line;
+	dma_addr_t next = 0 | VIA_DMA_DPR_EC;
+	drm_via_descriptor_t *desc_ptr = NULL;
+
+	if (mode == 1)
+		desc_ptr = vsg->desc_pages[cur_descriptor_page];
+
+	for (cur_line = 0; cur_line < xfer->num_lines; ++cur_line) {
+
+		line_len = xfer->line_length;
+		cur_fb = fb_addr;
+		cur_mem = mem_addr;
+
+		while (line_len > 0) {
+
+			remaining_len = min(PAGE_SIZE-VIA_PGOFF(cur_mem), line_len);
+			line_len -= remaining_len;
+
+			if (mode == 1) {
+				desc_ptr->mem_addr =
+					dma_map_page(&pdev->dev,
+						     vsg->pages[VIA_PFN(cur_mem) -
+								VIA_PFN(first_addr)],
+						     VIA_PGOFF(cur_mem), remaining_len,
+						     vsg->direction);
+				desc_ptr->dev_addr = cur_fb;
+
+				desc_ptr->size = remaining_len;
+				desc_ptr->next = (uint32_t) next;
+				next = dma_map_single(&pdev->dev, desc_ptr, sizeof(*desc_ptr),
+						      DMA_TO_DEVICE);
+				desc_ptr++;
+				if (++num_descriptors_this_page >= vsg->descriptors_per_page) {
+					num_descriptors_this_page = 0;
+					desc_ptr = vsg->desc_pages[++cur_descriptor_page];
+				}
+			}
+
+			num_desc++;
+			cur_mem += remaining_len;
+			cur_fb += remaining_len;
+		}
+
+		mem_addr += xfer->mem_stride;
+		fb_addr += xfer->fb_stride;
+	}
+
+	if (mode == 1) {
+		vsg->chain_start = next;
+		vsg->state = dr_via_device_mapped;
+	}
+	vsg->num_desc = num_desc;
+}
+
+/*
+ * Function that frees up all resources for a blit. It is usable even if the
+ * blit info has only been partially built as long as the status enum is consistent
+ * with the actual status of the used resources.
+ */
+static void
+via_free_sg_info(struct pci_dev *pdev, drm_via_sg_info_t *vsg)
+{
+	int i;
+
+	switch (vsg->state) {
+	case dr_via_device_mapped:
+		via_unmap_blit_from_device(pdev, vsg);
+		fallthrough;
+	case dr_via_desc_pages_alloc:
+		for (i = 0; i < vsg->num_desc_pages; ++i) {
+			if (vsg->desc_pages[i] != NULL)
+				free_page((unsigned long)vsg->desc_pages[i]);
+		}
+		kfree(vsg->desc_pages);
+		fallthrough;
+	case dr_via_pages_locked:
+		unpin_user_pages_dirty_lock(vsg->pages, vsg->num_pages,
+					   (vsg->direction == DMA_FROM_DEVICE));
+		fallthrough;
+	case dr_via_pages_alloc:
+		vfree(vsg->pages);
+		fallthrough;
+	default:
+		vsg->state = dr_via_sg_init;
+	}
+	vfree(vsg->bounce_buffer);
+	vsg->bounce_buffer = NULL;
+	vsg->free_on_sequence = 0;
+}
+
+/*
+ * Fire a blit engine.
+ */
+static void
+via_fire_dmablit(struct drm_device *dev, drm_via_sg_info_t *vsg, int engine)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
+
+	via_write(dev_priv, VIA_PCI_DMA_MAR0 + engine*0x10, 0);
+	via_write(dev_priv, VIA_PCI_DMA_DAR0 + engine*0x10, 0);
+	via_write(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_DD | VIA_DMA_CSR_TD |
+		  VIA_DMA_CSR_DE);
+	via_write(dev_priv, VIA_PCI_DMA_MR0  + engine*0x04, VIA_DMA_MR_CM | VIA_DMA_MR_TDIE);
+	via_write(dev_priv, VIA_PCI_DMA_BCR0 + engine*0x10, 0);
+	via_write(dev_priv, VIA_PCI_DMA_DPR0 + engine*0x10, vsg->chain_start);
+	wmb();
+	via_write(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_DE | VIA_DMA_CSR_TS);
+	via_read(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04);
+}
+
+/*
+ * Obtain a page pointer array and lock all pages into system memory. A segmentation violation will
+ * occur here if the calling user does not have access to the submitted address.
+ */
+static int
+via_lock_all_dma_pages(drm_via_sg_info_t *vsg,  drm_via_dmablit_t *xfer)
+{
+	int ret;
+	unsigned long first_pfn = VIA_PFN(xfer->mem_addr);
+	vsg->num_pages = VIA_PFN(xfer->mem_addr + (xfer->num_lines * xfer->mem_stride - 1)) -
+		first_pfn + 1;
+
+	vsg->pages = vzalloc(array_size(sizeof(struct page *), vsg->num_pages));
+	if (NULL == vsg->pages)
+		return -ENOMEM;
+	ret = pin_user_pages_fast((unsigned long)xfer->mem_addr,
+			vsg->num_pages,
+			vsg->direction == DMA_FROM_DEVICE ? FOLL_WRITE : 0,
+			vsg->pages);
+	if (ret != vsg->num_pages) {
+		if (ret < 0)
+			return ret;
+		vsg->state = dr_via_pages_locked;
+		return -EINVAL;
+	}
+	vsg->state = dr_via_pages_locked;
+	DRM_DEBUG("DMA pages locked\n");
+	return 0;
+}
+
+/*
+ * Allocate DMA capable memory for the blit descriptor chain, and an array that keeps track of the
+ * pages we allocate. We don't want to use kmalloc for the descriptor chain because it may be
+ * quite large for some blits, and pages don't need to be contiguous.
+ */
+static int
+via_alloc_desc_pages(drm_via_sg_info_t *vsg)
+{
+	int i;
+
+	vsg->descriptors_per_page = PAGE_SIZE / sizeof(drm_via_descriptor_t);
+	vsg->num_desc_pages = (vsg->num_desc + vsg->descriptors_per_page - 1) /
+		vsg->descriptors_per_page;
+
+	if (NULL ==  (vsg->desc_pages = kcalloc(vsg->num_desc_pages, sizeof(void *), GFP_KERNEL)))
+		return -ENOMEM;
+
+	vsg->state = dr_via_desc_pages_alloc;
+	for (i = 0; i < vsg->num_desc_pages; ++i) {
+		if (NULL == (vsg->desc_pages[i] =
+			     (drm_via_descriptor_t *) __get_free_page(GFP_KERNEL)))
+			return -ENOMEM;
+	}
+	DRM_DEBUG("Allocated %d pages for %d descriptors.\n", vsg->num_desc_pages,
+		  vsg->num_desc);
+	return 0;
+}
+
+static void
+via_abort_dmablit(struct drm_device *dev, int engine)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
+
+	via_write(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TA);
+}
+
+static void
+via_dmablit_engine_off(struct drm_device *dev, int engine)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
+
+	via_write(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04, VIA_DMA_CSR_TD | VIA_DMA_CSR_DD);
+}
+
+/*
+ * The dmablit part of the IRQ handler. Trying to do only reasonably fast things here.
+ * The rest, like unmapping and freeing memory for done blits is done in a separate workqueue
+ * task. Basically the task of the interrupt handler is to submit a new blit to the engine, while
+ * the workqueue task takes care of processing associated with the old blit.
+ */
+static void
+via_dmablit_handler(struct drm_device *dev, int engine, int from_irq)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
+	drm_via_blitq_t *blitq = dev_priv->blit_queues + engine;
+	int cur;
+	int done_transfer;
+	unsigned long irqsave = 0;
+	uint32_t status = 0;
+
+	DRM_DEBUG("DMA blit handler called. engine = %d, from_irq = %d, blitq = 0x%lx\n",
+		  engine, from_irq, (unsigned long) blitq);
+
+	if (from_irq)
+		spin_lock(&blitq->blit_lock);
+	else
+		spin_lock_irqsave(&blitq->blit_lock, irqsave);
+
+	done_transfer = blitq->is_active &&
+	  ((status = via_read(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04)) & VIA_DMA_CSR_TD);
+	done_transfer = done_transfer || (blitq->aborting && !(status & VIA_DMA_CSR_DE));
+
+	cur = blitq->cur;
+	if (done_transfer) {
+
+		blitq->blits[cur]->aborted = blitq->aborting;
+		blitq->done_blit_handle++;
+		wake_up(blitq->blit_queue + cur);
+
+		cur++;
+		if (cur >= VIA_NUM_BLIT_SLOTS)
+			cur = 0;
+		blitq->cur = cur;
+
+		/*
+		 * Clear transfer done flag.
+		 */
+
+		via_write(dev_priv, VIA_PCI_DMA_CSR0 + engine*0x04,  VIA_DMA_CSR_TD);
+
+		blitq->is_active = 0;
+		blitq->aborting = 0;
+		schedule_work(&blitq->wq);
+
+	} else if (blitq->is_active && time_after_eq(jiffies, blitq->end)) {
+
+		/*
+		 * Abort transfer after one second.
+		 */
+
+		via_abort_dmablit(dev, engine);
+		blitq->aborting = 1;
+		blitq->end = jiffies + HZ;
+	}
+
+	if (!blitq->is_active) {
+		if (blitq->num_outstanding) {
+			via_fire_dmablit(dev, blitq->blits[cur], engine);
+			blitq->is_active = 1;
+			blitq->cur = cur;
+			blitq->num_outstanding--;
+			blitq->end = jiffies + HZ;
+			if (!timer_pending(&blitq->poll_timer))
+				mod_timer(&blitq->poll_timer, jiffies + 1);
+		} else {
+			if (timer_pending(&blitq->poll_timer))
+				del_timer(&blitq->poll_timer);
+			via_dmablit_engine_off(dev, engine);
+		}
+	}
+
+	if (from_irq)
+		spin_unlock(&blitq->blit_lock);
+	else
+		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
+}
+
+/*
+ * Check whether this blit is still active, performing necessary locking.
+ */
+static int
+via_dmablit_active(drm_via_blitq_t *blitq, int engine, uint32_t handle, wait_queue_head_t **queue)
+{
+	unsigned long irqsave;
+	uint32_t slot;
+	int active;
+
+	spin_lock_irqsave(&blitq->blit_lock, irqsave);
+
+	/*
+	 * Allow for handle wraparounds.
+	 */
+
+	active = ((blitq->done_blit_handle - handle) > (1 << 23)) &&
+		((blitq->cur_blit_handle - handle) <= (1 << 23));
+
+	if (queue && active) {
+		slot = handle - blitq->done_blit_handle + blitq->cur - 1;
+		if (slot >= VIA_NUM_BLIT_SLOTS)
+			slot -= VIA_NUM_BLIT_SLOTS;
+		*queue = blitq->blit_queue + slot;
+	}
+
+	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
+
+	return active;
+}
+
+/*
+ * Sync. Wait for at least three seconds for the blit to be performed.
+ */
+static int
+via_dmablit_sync(struct drm_device *dev, uint32_t handle, int engine)
+{
+
+	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
+	drm_via_blitq_t *blitq = dev_priv->blit_queues + engine;
+	wait_queue_head_t *queue;
+	int ret = 0;
+
+	if (via_dmablit_active(blitq, engine, handle, &queue)) {
+		VIA_WAIT_ON(ret, *queue, 3 * HZ,
+			    !via_dmablit_active(blitq, engine, handle, NULL));
+	}
+	DRM_DEBUG("DMA blit sync handle 0x%x engine %d returned %d\n",
+		  handle, engine, ret);
+
+	return ret;
+}
+
+/*
+ * A timer that regularly polls the blit engine in cases where we don't have interrupts:
+ * a) Broken hardware (typically those that don't have any video capture facility).
+ * b) Blit abort. The hardware doesn't send an interrupt when a blit is aborted.
+ * The timer and hardware IRQ's can and do work in parallel. If the hardware has
+ * irqs, it will shorten the latency somewhat.
+ */
+static void
+via_dmablit_timer(struct timer_list *t)
+{
+	drm_via_blitq_t *blitq = from_timer(blitq, t, poll_timer);
+	struct drm_device *dev = blitq->dev;
+	int engine = (int)
+		(blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues);
+
+	DRM_DEBUG("Polling timer called for engine %d, jiffies %lu\n", engine,
+		  (unsigned long) jiffies);
+
+	via_dmablit_handler(dev, engine, 0);
+
+	if (!timer_pending(&blitq->poll_timer)) {
+		mod_timer(&blitq->poll_timer, jiffies + 1);
+
+	       /*
+		* Rerun handler to delete timer if engines are off, and
+		* to shorten abort latency. This is a little nasty.
+		*/
+
+	       via_dmablit_handler(dev, engine, 0);
+
+	}
+}
+
+/*
+ * Workqueue task that frees data and mappings associated with a blit.
+ * Also wakes up waiting processes. Each of these tasks handles one
+ * blit engine only and may not be called on each interrupt.
+ */
+static void
+via_dmablit_workqueue(struct work_struct *work)
+{
+	drm_via_blitq_t *blitq = container_of(work, drm_via_blitq_t, wq);
+	struct drm_device *dev = blitq->dev;
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+	unsigned long irqsave;
+	drm_via_sg_info_t *cur_sg;
+	int cur_released;
+
+
+	DRM_DEBUG("Workqueue task called for blit engine %ld\n", (unsigned long)
+		  (blitq - ((drm_via_private_t *)dev->dev_private)->blit_queues));
+
+	spin_lock_irqsave(&blitq->blit_lock, irqsave);
+
+	while (blitq->serviced != blitq->cur) {
+
+		cur_released = blitq->serviced++;
+
+		DRM_DEBUG("Releasing blit slot %d\n", cur_released);
+
+		if (blitq->serviced >= VIA_NUM_BLIT_SLOTS)
+			blitq->serviced = 0;
+
+		cur_sg = blitq->blits[cur_released];
+		blitq->num_free++;
+
+		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
+
+		wake_up(&blitq->busy_queue);
+
+		via_free_sg_info(pdev, cur_sg);
+		kfree(cur_sg);
+
+		spin_lock_irqsave(&blitq->blit_lock, irqsave);
+	}
+
+	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
+}
+
+/*
+ * Init all blit engines. Currently we use two, but some hardware have 4.
+ */
+static void
+via_init_dmablit(struct drm_device *dev)
+{
+	int i, j;
+	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+	drm_via_blitq_t *blitq;
+
+	pci_set_master(pdev);
+
+	for (i = 0; i < VIA_NUM_BLIT_ENGINES; ++i) {
+		blitq = dev_priv->blit_queues + i;
+		blitq->dev = dev;
+		blitq->cur_blit_handle = 0;
+		blitq->done_blit_handle = 0;
+		blitq->head = 0;
+		blitq->cur = 0;
+		blitq->serviced = 0;
+		blitq->num_free = VIA_NUM_BLIT_SLOTS - 1;
+		blitq->num_outstanding = 0;
+		blitq->is_active = 0;
+		blitq->aborting = 0;
+		spin_lock_init(&blitq->blit_lock);
+		for (j = 0; j < VIA_NUM_BLIT_SLOTS; ++j)
+			init_waitqueue_head(blitq->blit_queue + j);
+		init_waitqueue_head(&blitq->busy_queue);
+		INIT_WORK(&blitq->wq, via_dmablit_workqueue);
+		timer_setup(&blitq->poll_timer, via_dmablit_timer, 0);
+	}
+}
+
+/*
+ * Build all info and do all mappings required for a blit.
+ */
+static int
+via_build_sg_info(struct drm_device *dev, drm_via_sg_info_t *vsg, drm_via_dmablit_t *xfer)
+{
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+	int draw = xfer->to_fb;
+	int ret = 0;
+
+	vsg->direction = (draw) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;
+	vsg->bounce_buffer = NULL;
+
+	vsg->state = dr_via_sg_init;
+
+	if (xfer->num_lines <= 0 || xfer->line_length <= 0) {
+		DRM_ERROR("Zero size bitblt.\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * Below check is a driver limitation, not a hardware one. We
+	 * don't want to lock unused pages, and don't want to incoporate the
+	 * extra logic of avoiding them. Make sure there are no.
+	 * (Not a big limitation anyway.)
+	 */
+
+	if ((xfer->mem_stride - xfer->line_length) > 2*PAGE_SIZE) {
+		DRM_ERROR("Too large system memory stride. Stride: %d, "
+			  "Length: %d\n", xfer->mem_stride, xfer->line_length);
+		return -EINVAL;
+	}
+
+	if ((xfer->mem_stride == xfer->line_length) &&
+	   (xfer->fb_stride == xfer->line_length)) {
+		xfer->mem_stride *= xfer->num_lines;
+		xfer->line_length = xfer->mem_stride;
+		xfer->fb_stride = xfer->mem_stride;
+		xfer->num_lines = 1;
+	}
+
+	/*
+	 * Don't lock an arbitrary large number of pages, since that causes a
+	 * DOS security hole.
+	 */
+
+	if (xfer->num_lines > 2048 || (xfer->num_lines*xfer->mem_stride > (2048*2048*4))) {
+		DRM_ERROR("Too large PCI DMA bitblt.\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * we allow a negative fb stride to allow flipping of images in
+	 * transfer.
+	 */
+
+	if (xfer->mem_stride < xfer->line_length ||
+		abs(xfer->fb_stride) < xfer->line_length) {
+		DRM_ERROR("Invalid frame-buffer / memory stride.\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * A hardware bug seems to be worked around if system memory addresses start on
+	 * 16 byte boundaries. This seems a bit restrictive however. VIA is contacted
+	 * about this. Meanwhile, impose the following restrictions:
+	 */
+
+#ifdef VIA_BUGFREE
+	if ((((unsigned long)xfer->mem_addr & 3) != ((unsigned long)xfer->fb_addr & 3)) ||
+	    ((xfer->num_lines > 1) && ((xfer->mem_stride & 3) != (xfer->fb_stride & 3)))) {
+		DRM_ERROR("Invalid DRM bitblt alignment.\n");
+		return -EINVAL;
+	}
+#else
+	if ((((unsigned long)xfer->mem_addr & 15) ||
+	      ((unsigned long)xfer->fb_addr & 3)) ||
+	   ((xfer->num_lines > 1) &&
+	   ((xfer->mem_stride & 15) || (xfer->fb_stride & 3)))) {
+		DRM_ERROR("Invalid DRM bitblt alignment.\n");
+		return -EINVAL;
+	}
+#endif
+
+	if (0 != (ret = via_lock_all_dma_pages(vsg, xfer))) {
+		DRM_ERROR("Could not lock DMA pages.\n");
+		via_free_sg_info(pdev, vsg);
+		return ret;
+	}
+
+	via_map_blit_for_device(pdev, xfer, vsg, 0);
+	if (0 != (ret = via_alloc_desc_pages(vsg))) {
+		DRM_ERROR("Could not allocate DMA descriptor pages.\n");
+		via_free_sg_info(pdev, vsg);
+		return ret;
+	}
+	via_map_blit_for_device(pdev, xfer, vsg, 1);
+
+	return 0;
+}
+
+/*
+ * Reserve one free slot in the blit queue. Will wait for one second for one
+ * to become available. Otherwise -EBUSY is returned.
+ */
+static int
+via_dmablit_grab_slot(drm_via_blitq_t *blitq, int engine)
+{
+	int ret = 0;
+	unsigned long irqsave;
+
+	DRM_DEBUG("Num free is %d\n", blitq->num_free);
+	spin_lock_irqsave(&blitq->blit_lock, irqsave);
+	while (blitq->num_free == 0) {
+		spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
+
+		VIA_WAIT_ON(ret, blitq->busy_queue, HZ, blitq->num_free > 0);
+		if (ret)
+			return (-EINTR == ret) ? -EAGAIN : ret;
+
+		spin_lock_irqsave(&blitq->blit_lock, irqsave);
+	}
+
+	blitq->num_free--;
+	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
+
+	return 0;
+}
+
+/*
+ * Hand back a free slot if we changed our mind.
+ */
+static void
+via_dmablit_release_slot(drm_via_blitq_t *blitq)
+{
+	unsigned long irqsave;
+
+	spin_lock_irqsave(&blitq->blit_lock, irqsave);
+	blitq->num_free++;
+	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
+	wake_up(&blitq->busy_queue);
+}
+
+/*
+ * Grab a free slot. Build blit info and queue a blit.
+ */
+static int
+via_dmablit(struct drm_device *dev, drm_via_dmablit_t *xfer)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *)dev->dev_private;
+	drm_via_sg_info_t *vsg;
+	drm_via_blitq_t *blitq;
+	int ret;
+	int engine;
+	unsigned long irqsave;
+
+	if (dev_priv == NULL) {
+		DRM_ERROR("Called without initialization.\n");
+		return -EINVAL;
+	}
+
+	engine = (xfer->to_fb) ? 0 : 1;
+	blitq = dev_priv->blit_queues + engine;
+	if (0 != (ret = via_dmablit_grab_slot(blitq, engine)))
+		return ret;
+	if (NULL == (vsg = kmalloc(sizeof(*vsg), GFP_KERNEL))) {
+		via_dmablit_release_slot(blitq);
+		return -ENOMEM;
+	}
+	if (0 != (ret = via_build_sg_info(dev, vsg, xfer))) {
+		via_dmablit_release_slot(blitq);
+		kfree(vsg);
+		return ret;
+	}
+	spin_lock_irqsave(&blitq->blit_lock, irqsave);
+
+	blitq->blits[blitq->head++] = vsg;
+	if (blitq->head >= VIA_NUM_BLIT_SLOTS)
+		blitq->head = 0;
+	blitq->num_outstanding++;
+	xfer->sync.sync_handle = ++blitq->cur_blit_handle;
+
+	spin_unlock_irqrestore(&blitq->blit_lock, irqsave);
+	xfer->sync.engine = engine;
+
+	via_dmablit_handler(dev, engine, 0);
+
+	return 0;
+}
+
+/*
+ * Sync on a previously submitted blit. Note that the X server use signals extensively, and
+ * that there is a very big probability that this IOCTL will be interrupted by a signal. In that
+ * case it returns with -EAGAIN for the signal to be delivered.
+ * The caller should then reissue the IOCTL. This is similar to what is being done for drmGetLock().
+ */
+static int
+via_dma_blit_sync(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_blitsync_t *sync = data;
+	int err;
+
+	if (sync->engine >= VIA_NUM_BLIT_ENGINES)
+		return -EINVAL;
+
+	err = via_dmablit_sync(dev, sync->sync_handle, sync->engine);
+
+	if (-EINTR == err)
+		err = -EAGAIN;
+
+	return err;
+}
+
+/*
+ * Queue a blit and hand back a handle to be used for sync. This IOCTL may be interrupted by a signal
+ * while waiting for a free slot in the blit queue. In that case it returns with -EAGAIN and should
+ * be reissued. See the above IOCTL code.
+ */
+static int
+via_dma_blit(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_dmablit_t *xfer = data;
+	int err;
+
+	err = via_dmablit(dev, xfer);
+
+	return err;
+}
+
+static u32 via_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
+{
+	drm_via_private_t *dev_priv = dev->dev_private;
+
+	if (pipe != 0)
+		return 0;
+
+	return atomic_read(&dev_priv->vbl_received);
+}
+
+static irqreturn_t via_driver_irq_handler(int irq, void *arg)
+{
+	struct drm_device *dev = (struct drm_device *) arg;
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	u32 status;
+	int handled = 0;
+	ktime_t cur_vblank;
+	drm_via_irq_t *cur_irq = dev_priv->via_irqs;
+	int i;
+
+	status = via_read(dev_priv, VIA_REG_INTERRUPT);
+	if (status & VIA_IRQ_VBLANK_PENDING) {
+		atomic_inc(&dev_priv->vbl_received);
+		if (!(atomic_read(&dev_priv->vbl_received) & 0x0F)) {
+			cur_vblank = ktime_get();
+			if (dev_priv->last_vblank_valid) {
+				dev_priv->nsec_per_vblank =
+					ktime_sub(cur_vblank,
+						dev_priv->last_vblank) >> 4;
+			}
+			dev_priv->last_vblank = cur_vblank;
+			dev_priv->last_vblank_valid = 1;
+		}
+		if (!(atomic_read(&dev_priv->vbl_received) & 0xFF)) {
+			DRM_DEBUG("nsec per vblank is: %llu\n",
+				  ktime_to_ns(dev_priv->nsec_per_vblank));
+		}
+		drm_handle_vblank(dev, 0);
+		handled = 1;
+	}
+
+	for (i = 0; i < dev_priv->num_irqs; ++i) {
+		if (status & cur_irq->pending_mask) {
+			atomic_inc(&cur_irq->irq_received);
+			wake_up(&cur_irq->irq_queue);
+			handled = 1;
+			if (dev_priv->irq_map[drm_via_irq_dma0_td] == i)
+				via_dmablit_handler(dev, 0, 1);
+			else if (dev_priv->irq_map[drm_via_irq_dma1_td] == i)
+				via_dmablit_handler(dev, 1, 1);
+		}
+		cur_irq++;
+	}
+
+	/* Acknowledge interrupts */
+	via_write(dev_priv, VIA_REG_INTERRUPT, status);
+
+
+	if (handled)
+		return IRQ_HANDLED;
+	else
+		return IRQ_NONE;
+}
+
+static __inline__ void viadrv_acknowledge_irqs(drm_via_private_t *dev_priv)
+{
+	u32 status;
+
+	if (dev_priv) {
+		/* Acknowledge interrupts */
+		status = via_read(dev_priv, VIA_REG_INTERRUPT);
+		via_write(dev_priv, VIA_REG_INTERRUPT, status |
+			  dev_priv->irq_pending_mask);
+	}
+}
+
+static int via_enable_vblank(struct drm_device *dev, unsigned int pipe)
+{
+	drm_via_private_t *dev_priv = dev->dev_private;
+	u32 status;
+
+	if (pipe != 0) {
+		DRM_ERROR("%s:  bad crtc %u\n", __func__, pipe);
+		return -EINVAL;
+	}
+
+	status = via_read(dev_priv, VIA_REG_INTERRUPT);
+	via_write(dev_priv, VIA_REG_INTERRUPT, status | VIA_IRQ_VBLANK_ENABLE);
+
+	via_write8(dev_priv, 0x83d4, 0x11);
+	via_write8_mask(dev_priv, 0x83d5, 0x30, 0x30);
+
+	return 0;
+}
+
+static void via_disable_vblank(struct drm_device *dev, unsigned int pipe)
+{
+	drm_via_private_t *dev_priv = dev->dev_private;
+	u32 status;
+
+	status = via_read(dev_priv, VIA_REG_INTERRUPT);
+	via_write(dev_priv, VIA_REG_INTERRUPT, status & ~VIA_IRQ_VBLANK_ENABLE);
+
+	via_write8(dev_priv, 0x83d4, 0x11);
+	via_write8_mask(dev_priv, 0x83d5, 0x30, 0);
+
+	if (pipe != 0)
+		DRM_ERROR("%s:  bad crtc %u\n", __func__, pipe);
+}
+
+static int
+via_driver_irq_wait(struct drm_device *dev, unsigned int irq, int force_sequence,
+		    unsigned int *sequence)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	unsigned int cur_irq_sequence;
+	drm_via_irq_t *cur_irq;
+	int ret = 0;
+	maskarray_t *masks;
+	int real_irq;
+
+	DRM_DEBUG("\n");
+
+	if (!dev_priv) {
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
+	}
+
+	if (irq >= drm_via_irq_num) {
+		DRM_ERROR("Trying to wait on unknown irq %d\n", irq);
+		return -EINVAL;
+	}
+
+	real_irq = dev_priv->irq_map[irq];
+
+	if (real_irq < 0) {
+		DRM_ERROR("Video IRQ %d not available on this hardware.\n",
+			  irq);
+		return -EINVAL;
+	}
+
+	masks = dev_priv->irq_masks;
+	cur_irq = dev_priv->via_irqs + real_irq;
+
+	if (masks[real_irq][2] && !force_sequence) {
+		VIA_WAIT_ON(ret, cur_irq->irq_queue, 3 * HZ,
+			    ((via_read(dev_priv, masks[irq][2]) & masks[irq][3]) ==
+			     masks[irq][4]));
+		cur_irq_sequence = atomic_read(&cur_irq->irq_received);
+	} else {
+		VIA_WAIT_ON(ret, cur_irq->irq_queue, 3 * HZ,
+			    (((cur_irq_sequence =
+			       atomic_read(&cur_irq->irq_received)) -
+			      *sequence) <= (1 << 23)));
+	}
+	*sequence = cur_irq_sequence;
+	return ret;
+}
+
+
+/*
+ * drm_dma.h hooks
+ */
+
+static void via_driver_irq_preinstall(struct drm_device *dev)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	u32 status;
+	drm_via_irq_t *cur_irq;
+	int i;
+
+	DRM_DEBUG("dev_priv: %p\n", dev_priv);
+	if (dev_priv) {
+		cur_irq = dev_priv->via_irqs;
+
+		dev_priv->irq_enable_mask = VIA_IRQ_VBLANK_ENABLE;
+		dev_priv->irq_pending_mask = VIA_IRQ_VBLANK_PENDING;
+
+		if (dev_priv->chipset == VIA_PRO_GROUP_A ||
+		    dev_priv->chipset == VIA_DX9_0) {
+			dev_priv->irq_masks = via_pro_group_a_irqs;
+			dev_priv->num_irqs = via_num_pro_group_a;
+			dev_priv->irq_map = via_irqmap_pro_group_a;
+		} else {
+			dev_priv->irq_masks = via_unichrome_irqs;
+			dev_priv->num_irqs = via_num_unichrome;
+			dev_priv->irq_map = via_irqmap_unichrome;
+		}
+
+		for (i = 0; i < dev_priv->num_irqs; ++i) {
+			atomic_set(&cur_irq->irq_received, 0);
+			cur_irq->enable_mask = dev_priv->irq_masks[i][0];
+			cur_irq->pending_mask = dev_priv->irq_masks[i][1];
+			init_waitqueue_head(&cur_irq->irq_queue);
+			dev_priv->irq_enable_mask |= cur_irq->enable_mask;
+			dev_priv->irq_pending_mask |= cur_irq->pending_mask;
+			cur_irq++;
+
+			DRM_DEBUG("Initializing IRQ %d\n", i);
+		}
+
+		dev_priv->last_vblank_valid = 0;
+
+		/* Clear VSync interrupt regs */
+		status = via_read(dev_priv, VIA_REG_INTERRUPT);
+		via_write(dev_priv, VIA_REG_INTERRUPT, status &
+			  ~(dev_priv->irq_enable_mask));
+
+		/* Clear bits if they're already high */
+		viadrv_acknowledge_irqs(dev_priv);
+	}
+}
+
+static int via_driver_irq_postinstall(struct drm_device *dev)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	u32 status;
+
+	DRM_DEBUG("fun: %s\n", __func__);
+	if (!dev_priv)
+		return -EINVAL;
+
+	status = via_read(dev_priv, VIA_REG_INTERRUPT);
+	via_write(dev_priv, VIA_REG_INTERRUPT, status | VIA_IRQ_GLOBAL
+		  | dev_priv->irq_enable_mask);
+
+	/* Some magic, oh for some data sheets ! */
+	via_write8(dev_priv, 0x83d4, 0x11);
+	via_write8_mask(dev_priv, 0x83d5, 0x30, 0x30);
+
+	return 0;
+}
+
+static void via_driver_irq_uninstall(struct drm_device *dev)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	u32 status;
+
+	DRM_DEBUG("\n");
+	if (dev_priv) {
+
+		/* Some more magic, oh for some data sheets ! */
+
+		via_write8(dev_priv, 0x83d4, 0x11);
+		via_write8_mask(dev_priv, 0x83d5, 0x30, 0);
+
+		status = via_read(dev_priv, VIA_REG_INTERRUPT);
+		via_write(dev_priv, VIA_REG_INTERRUPT, status &
+			  ~(VIA_IRQ_VBLANK_ENABLE | dev_priv->irq_enable_mask));
+	}
+}
+
+static int via_wait_irq(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_irqwait_t *irqwait = data;
+	struct timespec64 now;
+	int ret = 0;
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	drm_via_irq_t *cur_irq = dev_priv->via_irqs;
+	int force_sequence;
+
+	if (irqwait->request.irq >= dev_priv->num_irqs) {
+		DRM_ERROR("Trying to wait on unknown irq %d\n",
+			  irqwait->request.irq);
+		return -EINVAL;
+	}
+
+	cur_irq += irqwait->request.irq;
+
+	switch (irqwait->request.type & ~VIA_IRQ_FLAGS_MASK) {
+	case VIA_IRQ_RELATIVE:
+		irqwait->request.sequence +=
+			atomic_read(&cur_irq->irq_received);
+		irqwait->request.type &= ~_DRM_VBLANK_RELATIVE;
+		break;
+	case VIA_IRQ_ABSOLUTE:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (irqwait->request.type & VIA_IRQ_SIGNAL) {
+		DRM_ERROR("Signals on Via IRQs not implemented yet.\n");
+		return -EINVAL;
+	}
+
+	force_sequence = (irqwait->request.type & VIA_IRQ_FORCE_SEQUENCE);
+
+	ret = via_driver_irq_wait(dev, irqwait->request.irq, force_sequence,
+				  &irqwait->request.sequence);
+	ktime_get_ts64(&now);
+	irqwait->reply.tval_sec = now.tv_sec;
+	irqwait->reply.tval_usec = now.tv_nsec / NSEC_PER_USEC;
+
+	return ret;
+}
+
+static void via_init_futex(drm_via_private_t *dev_priv)
+{
+	unsigned int i;
+
+	DRM_DEBUG("\n");
+
+	for (i = 0; i < VIA_NR_XVMC_LOCKS; ++i) {
+		init_waitqueue_head(&(dev_priv->decoder_queue[i]));
+		XVMCLOCKPTR(dev_priv->sarea_priv, i)->lock = 0;
+	}
+}
+
+static void via_cleanup_futex(drm_via_private_t *dev_priv)
+{
+}
+
+static void via_release_futex(drm_via_private_t *dev_priv, int context)
+{
+	unsigned int i;
+	volatile int *lock;
+
+	if (!dev_priv->sarea_priv)
+		return;
+
+	for (i = 0; i < VIA_NR_XVMC_LOCKS; ++i) {
+		lock = (volatile int *)XVMCLOCKPTR(dev_priv->sarea_priv, i);
+		if ((_DRM_LOCKING_CONTEXT(*lock) == context)) {
+			if (_DRM_LOCK_IS_HELD(*lock)
+			    && (*lock & _DRM_LOCK_CONT)) {
+				wake_up(&(dev_priv->decoder_queue[i]));
+			}
+			*lock = 0;
+		}
+	}
+}
+
+static int via_decoder_futex(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_futex_t *fx = data;
+	volatile int *lock;
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	drm_via_sarea_t *sAPriv = dev_priv->sarea_priv;
+	int ret = 0;
+
+	DRM_DEBUG("\n");
+
+	if (fx->lock >= VIA_NR_XVMC_LOCKS)
+		return -EFAULT;
+
+	lock = (volatile int *)XVMCLOCKPTR(sAPriv, fx->lock);
+
+	switch (fx->func) {
+	case VIA_FUTEX_WAIT:
+		VIA_WAIT_ON(ret, dev_priv->decoder_queue[fx->lock],
+			    (fx->ms / 10) * (HZ / 100), *lock != fx->val);
+		return ret;
+	case VIA_FUTEX_WAKE:
+		wake_up(&(dev_priv->decoder_queue[fx->lock]));
+		return 0;
+	}
+	return 0;
+}
+
+static int via_agp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_agp_t *agp = data;
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+
+	mutex_lock(&dev->struct_mutex);
+	drm_mm_init(&dev_priv->agp_mm, 0, agp->size >> VIA_MM_ALIGN_SHIFT);
+
+	dev_priv->agp_initialized = 1;
+	dev_priv->agp_offset = agp->offset;
+	mutex_unlock(&dev->struct_mutex);
+
+	DRM_DEBUG("offset = %u, size = %u\n", agp->offset, agp->size);
+	return 0;
+}
+
+static int via_fb_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_fb_t *fb = data;
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+
+	mutex_lock(&dev->struct_mutex);
+	drm_mm_init(&dev_priv->vram_mm, 0, fb->size >> VIA_MM_ALIGN_SHIFT);
+
+	dev_priv->vram_initialized = 1;
+	dev_priv->vram_offset = fb->offset;
+
+	mutex_unlock(&dev->struct_mutex);
+	DRM_DEBUG("offset = %u, size = %u\n", fb->offset, fb->size);
+
+	return 0;
+
+}
+
+static int via_final_context(struct drm_device *dev, int context)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+
+	via_release_futex(dev_priv, context);
+
+	/* Linux specific until context tracking code gets ported to BSD */
+	/* Last context, perform cleanup */
+	if (list_is_singular(&dev->ctxlist)) {
+		DRM_DEBUG("Last Context\n");
+		drm_legacy_irq_uninstall(dev);
+		via_cleanup_futex(dev_priv);
+		via_do_cleanup_map(dev);
+	}
+	return 1;
+}
+
+static void via_lastclose(struct drm_device *dev)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+
+	if (!dev_priv)
+		return;
+
+	mutex_lock(&dev->struct_mutex);
+	if (dev_priv->vram_initialized) {
+		drm_mm_takedown(&dev_priv->vram_mm);
+		dev_priv->vram_initialized = 0;
+	}
+	if (dev_priv->agp_initialized) {
+		drm_mm_takedown(&dev_priv->agp_mm);
+		dev_priv->agp_initialized = 0;
+	}
+	mutex_unlock(&dev->struct_mutex);
+}
+
+static int via_mem_alloc(struct drm_device *dev, void *data,
+		  struct drm_file *file)
+{
+	drm_via_mem_t *mem = data;
+	int retval = 0, user_key;
+	struct via_memblock *item;
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	struct via_file_private *file_priv = file->driver_priv;
+	unsigned long tmpSize;
+
+	if (mem->type > VIA_MEM_AGP) {
+		DRM_ERROR("Unknown memory type allocation\n");
+		return -EINVAL;
+	}
+	mutex_lock(&dev->struct_mutex);
+	if (0 == ((mem->type == VIA_MEM_VIDEO) ? dev_priv->vram_initialized :
+		      dev_priv->agp_initialized)) {
+		mutex_unlock(&dev->struct_mutex);
+		DRM_ERROR
+		    ("Attempt to allocate from uninitialized memory manager.\n");
+		return -EINVAL;
+	}
+
+	item = kzalloc(sizeof(*item), GFP_KERNEL);
+	if (!item) {
+		retval = -ENOMEM;
+		goto fail_alloc;
+	}
+
+	tmpSize = (mem->size + VIA_MM_ALIGN_MASK) >> VIA_MM_ALIGN_SHIFT;
+	if (mem->type == VIA_MEM_AGP)
+		retval = drm_mm_insert_node(&dev_priv->agp_mm,
+					    &item->mm_node,
+					    tmpSize);
+	else
+		retval = drm_mm_insert_node(&dev_priv->vram_mm,
+					    &item->mm_node,
+					    tmpSize);
+	if (retval)
+		goto fail_alloc;
+
+	retval = idr_alloc(&dev_priv->object_idr, item, 1, 0, GFP_KERNEL);
+	if (retval < 0)
+		goto fail_idr;
+	user_key = retval;
+
+	list_add(&item->owner_list, &file_priv->obj_list);
+	mutex_unlock(&dev->struct_mutex);
+
+	mem->offset = ((mem->type == VIA_MEM_VIDEO) ?
+		      dev_priv->vram_offset : dev_priv->agp_offset) +
+	    ((item->mm_node.start) << VIA_MM_ALIGN_SHIFT);
+	mem->index = user_key;
+
+	return 0;
+
+fail_idr:
+	drm_mm_remove_node(&item->mm_node);
+fail_alloc:
+	kfree(item);
+	mutex_unlock(&dev->struct_mutex);
+
+	mem->offset = 0;
+	mem->size = 0;
+	mem->index = 0;
+	DRM_DEBUG("Video memory allocation failed\n");
+
+	return retval;
+}
+
+static int via_mem_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_private_t *dev_priv = dev->dev_private;
+	drm_via_mem_t *mem = data;
+	struct via_memblock *obj;
+
+	mutex_lock(&dev->struct_mutex);
+	obj = idr_find(&dev_priv->object_idr, mem->index);
+	if (obj == NULL) {
+		mutex_unlock(&dev->struct_mutex);
+		return -EINVAL;
+	}
+
+	idr_remove(&dev_priv->object_idr, mem->index);
+	list_del(&obj->owner_list);
+	drm_mm_remove_node(&obj->mm_node);
+	kfree(obj);
+	mutex_unlock(&dev->struct_mutex);
+
+	DRM_DEBUG("free = 0x%lx\n", mem->index);
+
+	return 0;
+}
+
+
+static void via_reclaim_buffers_locked(struct drm_device *dev,
+				struct drm_file *file)
+{
+	struct via_file_private *file_priv = file->driver_priv;
+	struct via_memblock *entry, *next;
+
+	if (!(dev->master && file->master->lock.hw_lock))
+		return;
+
+	drm_legacy_idlelock_take(&file->master->lock);
+
+	mutex_lock(&dev->struct_mutex);
+	if (list_empty(&file_priv->obj_list)) {
+		mutex_unlock(&dev->struct_mutex);
+		drm_legacy_idlelock_release(&file->master->lock);
+
+		return;
+	}
+
+	via_driver_dma_quiescent(dev);
+
+	list_for_each_entry_safe(entry, next, &file_priv->obj_list,
+				 owner_list) {
+		list_del(&entry->owner_list);
+		drm_mm_remove_node(&entry->mm_node);
+		kfree(entry);
+	}
+	mutex_unlock(&dev->struct_mutex);
+
+	drm_legacy_idlelock_release(&file->master->lock);
+
+	return;
+}
+
+static int via_do_init_map(struct drm_device *dev, drm_via_init_t *init)
+{
+	drm_via_private_t *dev_priv = dev->dev_private;
+
+	DRM_DEBUG("\n");
+
+	dev_priv->sarea = drm_legacy_getsarea(dev);
+	if (!dev_priv->sarea) {
+		DRM_ERROR("could not find sarea!\n");
+		dev->dev_private = (void *)dev_priv;
+		via_do_cleanup_map(dev);
+		return -EINVAL;
+	}
+
+	dev_priv->fb = drm_legacy_findmap(dev, init->fb_offset);
+	if (!dev_priv->fb) {
+		DRM_ERROR("could not find framebuffer!\n");
+		dev->dev_private = (void *)dev_priv;
+		via_do_cleanup_map(dev);
+		return -EINVAL;
+	}
+	dev_priv->mmio = drm_legacy_findmap(dev, init->mmio_offset);
+	if (!dev_priv->mmio) {
+		DRM_ERROR("could not find mmio region!\n");
+		dev->dev_private = (void *)dev_priv;
+		via_do_cleanup_map(dev);
+		return -EINVAL;
+	}
+
+	dev_priv->sarea_priv =
+	    (drm_via_sarea_t *) ((u8 *) dev_priv->sarea->handle +
+				 init->sarea_priv_offset);
+
+	dev_priv->agpAddr = init->agpAddr;
+
+	via_init_futex(dev_priv);
+
+	via_init_dmablit(dev);
+
+	dev->dev_private = (void *)dev_priv;
+	return 0;
+}
+
+int via_do_cleanup_map(struct drm_device *dev)
+{
+	via_dma_cleanup(dev);
+
+	return 0;
+}
+
+static int via_map_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_init_t *init = data;
+
+	DRM_DEBUG("\n");
+
+	switch (init->func) {
+	case VIA_INIT_MAP:
+		return via_do_init_map(dev, init);
+	case VIA_CLEANUP_MAP:
+		return via_do_cleanup_map(dev);
+	}
+
+	return -EINVAL;
+}
+
+static int via_driver_load(struct drm_device *dev, unsigned long chipset)
+{
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
+	drm_via_private_t *dev_priv;
+	int ret = 0;
+
+	dev_priv = kzalloc(sizeof(drm_via_private_t), GFP_KERNEL);
+	if (dev_priv == NULL)
+		return -ENOMEM;
+
+	idr_init_base(&dev_priv->object_idr, 1);
+	dev->dev_private = (void *)dev_priv;
+
+	dev_priv->chipset = chipset;
+
+	pci_set_master(pdev);
+
+	ret = drm_vblank_init(dev, 1);
+	if (ret) {
+		kfree(dev_priv);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void via_driver_unload(struct drm_device *dev)
+{
+	drm_via_private_t *dev_priv = dev->dev_private;
+
+	idr_destroy(&dev_priv->object_idr);
+
+	kfree(dev_priv);
+}
+
+static void via_cmdbuf_start(drm_via_private_t *dev_priv);
+static void via_cmdbuf_pause(drm_via_private_t *dev_priv);
+static void via_cmdbuf_reset(drm_via_private_t *dev_priv);
+static void via_cmdbuf_rewind(drm_via_private_t *dev_priv);
+static int via_wait_idle(drm_via_private_t *dev_priv);
+static void via_pad_cache(drm_via_private_t *dev_priv, int qwords);
+
+/*
+ * Free space in command buffer.
+ */
+
+static uint32_t via_cmdbuf_space(drm_via_private_t *dev_priv)
+{
+	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;
+	uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base;
+
+	return ((hw_addr <= dev_priv->dma_low) ?
+		(dev_priv->dma_high + hw_addr - dev_priv->dma_low) :
+		(hw_addr - dev_priv->dma_low));
+}
+
+/*
+ * How much does the command regulator lag behind?
+ */
+
+static uint32_t via_cmdbuf_lag(drm_via_private_t *dev_priv)
+{
+	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;
+	uint32_t hw_addr = *(dev_priv->hw_addr_ptr) - agp_base;
+
+	return ((hw_addr <= dev_priv->dma_low) ?
+		(dev_priv->dma_low - hw_addr) :
+		(dev_priv->dma_wrap + dev_priv->dma_low - hw_addr));
+}
+
+/*
+ * Check that the given size fits in the buffer, otherwise wait.
+ */
+
+static inline int
+via_cmdbuf_wait(drm_via_private_t *dev_priv, unsigned int size)
+{
+	uint32_t agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;
+	uint32_t cur_addr, hw_addr, next_addr;
+	volatile uint32_t *hw_addr_ptr;
+	uint32_t count;
+	hw_addr_ptr = dev_priv->hw_addr_ptr;
+	cur_addr = dev_priv->dma_low;
+	next_addr = cur_addr + size + 512 * 1024;
+	count = 1000000;
+	do {
+		hw_addr = *hw_addr_ptr - agp_base;
+		if (count-- == 0) {
+			DRM_ERROR
+			    ("via_cmdbuf_wait timed out hw %x cur_addr %x next_addr %x\n",
+			     hw_addr, cur_addr, next_addr);
+			return -1;
+		}
+		if  ((cur_addr < hw_addr) && (next_addr >= hw_addr))
+			msleep(1);
+	} while ((cur_addr < hw_addr) && (next_addr >= hw_addr));
+	return 0;
+}
+
+/*
+ * Checks whether buffer head has reach the end. Rewind the ring buffer
+ * when necessary.
+ *
+ * Returns virtual pointer to ring buffer.
+ */
+
+static inline uint32_t *via_check_dma(drm_via_private_t * dev_priv,
+				      unsigned int size)
+{
+	if ((dev_priv->dma_low + size + 4 * CMDBUF_ALIGNMENT_SIZE) >
+	    dev_priv->dma_high) {
+		via_cmdbuf_rewind(dev_priv);
+	}
+	if (via_cmdbuf_wait(dev_priv, size) != 0)
+		return NULL;
+
+	return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low);
+}
+
+int via_dma_cleanup(struct drm_device *dev)
+{
+	if (dev->dev_private) {
+		drm_via_private_t *dev_priv =
+		    (drm_via_private_t *) dev->dev_private;
+
+		if (dev_priv->ring.virtual_start && dev_priv->mmio) {
+			via_cmdbuf_reset(dev_priv);
+
+			drm_legacy_ioremapfree(&dev_priv->ring.map, dev);
+			dev_priv->ring.virtual_start = NULL;
+		}
+
+	}
+
+	return 0;
+}
+
+static int via_initialize(struct drm_device *dev,
+			  drm_via_private_t *dev_priv,
+			  drm_via_dma_init_t *init)
+{
+	if (!dev_priv || !dev_priv->mmio) {
+		DRM_ERROR("via_dma_init called before via_map_init\n");
+		return -EFAULT;
+	}
+
+	if (dev_priv->ring.virtual_start != NULL) {
+		DRM_ERROR("called again without calling cleanup\n");
+		return -EFAULT;
+	}
+
+	if (!dev->agp || !dev->agp->base) {
+		DRM_ERROR("called with no agp memory available\n");
+		return -EFAULT;
+	}
+
+	if (dev_priv->chipset == VIA_DX9_0) {
+		DRM_ERROR("AGP DMA is not supported on this chip\n");
+		return -EINVAL;
+	}
+
+	dev_priv->ring.map.offset = dev->agp->base + init->offset;
+	dev_priv->ring.map.size = init->size;
+	dev_priv->ring.map.type = 0;
+	dev_priv->ring.map.flags = 0;
+	dev_priv->ring.map.mtrr = 0;
+
+	drm_legacy_ioremap(&dev_priv->ring.map, dev);
+
+	if (dev_priv->ring.map.handle == NULL) {
+		via_dma_cleanup(dev);
+		DRM_ERROR("can not ioremap virtual address for"
+			  " ring buffer\n");
+		return -ENOMEM;
+	}
+
+	dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
+
+	dev_priv->dma_ptr = dev_priv->ring.virtual_start;
+	dev_priv->dma_low = 0;
+	dev_priv->dma_high = init->size;
+	dev_priv->dma_wrap = init->size;
+	dev_priv->dma_offset = init->offset;
+	dev_priv->last_pause_ptr = NULL;
+	dev_priv->hw_addr_ptr =
+		(volatile uint32_t *)((char *)dev_priv->mmio->handle +
+		init->reg_pause_addr);
+
+	via_cmdbuf_start(dev_priv);
+
+	return 0;
+}
+
+static int via_dma_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
+	drm_via_dma_init_t *init = data;
+	int retcode = 0;
+
+	switch (init->func) {
+	case VIA_INIT_DMA:
+		if (!capable(CAP_SYS_ADMIN))
+			retcode = -EPERM;
+		else
+			retcode = via_initialize(dev, dev_priv, init);
+		break;
+	case VIA_CLEANUP_DMA:
+		if (!capable(CAP_SYS_ADMIN))
+			retcode = -EPERM;
+		else
+			retcode = via_dma_cleanup(dev);
+		break;
+	case VIA_DMA_INITIALIZED:
+		retcode = (dev_priv->ring.virtual_start != NULL) ?
+			0 : -EFAULT;
+		break;
+	default:
+		retcode = -EINVAL;
+		break;
+	}
+
+	return retcode;
+}
+
+static int via_dispatch_cmdbuffer(struct drm_device *dev, drm_via_cmdbuffer_t *cmd)
+{
+	drm_via_private_t *dev_priv;
+	uint32_t *vb;
+	int ret;
+
+	dev_priv = (drm_via_private_t *) dev->dev_private;
+
+	if (dev_priv->ring.virtual_start == NULL) {
+		DRM_ERROR("called without initializing AGP ring buffer.\n");
+		return -EFAULT;
+	}
+
+	if (cmd->size > VIA_PCI_BUF_SIZE)
+		return -ENOMEM;
+
+	if (copy_from_user(dev_priv->pci_buf, cmd->buf, cmd->size))
+		return -EFAULT;
+
+	/*
+	 * Running this function on AGP memory is dead slow. Therefore
+	 * we run it on a temporary cacheable system memory buffer and
+	 * copy it to AGP memory when ready.
+	 */
+
+	if ((ret =
+	     via_verify_command_stream((uint32_t *) dev_priv->pci_buf,
+				       cmd->size, dev, 1))) {
+		return ret;
+	}
+
+	vb = via_check_dma(dev_priv, (cmd->size < 0x100) ? 0x102 : cmd->size);
+	if (vb == NULL)
+		return -EAGAIN;
+
+	memcpy(vb, dev_priv->pci_buf, cmd->size);
+
+	dev_priv->dma_low += cmd->size;
+
+	/*
+	 * Small submissions somehow stalls the CPU. (AGP cache effects?)
+	 * pad to greater size.
+	 */
+
+	if (cmd->size < 0x100)
+		via_pad_cache(dev_priv, (0x100 - cmd->size) >> 3);
+	via_cmdbuf_pause(dev_priv);
+
+	return 0;
+}
+
+int via_driver_dma_quiescent(struct drm_device *dev)
+{
+	drm_via_private_t *dev_priv = dev->dev_private;
+
+	if (!via_wait_idle(dev_priv))
+		return -EBUSY;
+	return 0;
+}
+
+static int via_flush_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
+
+	return via_driver_dma_quiescent(dev);
+}
+
+static int via_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_cmdbuffer_t *cmdbuf = data;
+	int ret;
+
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
+
+	DRM_DEBUG("buf %p size %lu\n", cmdbuf->buf, cmdbuf->size);
+
+	ret = via_dispatch_cmdbuffer(dev, cmdbuf);
+	return ret;
+}
+
+static int via_dispatch_pci_cmdbuffer(struct drm_device *dev,
+				      drm_via_cmdbuffer_t *cmd)
+{
+	drm_via_private_t *dev_priv = dev->dev_private;
+	int ret;
+
+	if (cmd->size > VIA_PCI_BUF_SIZE)
+		return -ENOMEM;
+	if (copy_from_user(dev_priv->pci_buf, cmd->buf, cmd->size))
+		return -EFAULT;
+
+	if ((ret =
+	     via_verify_command_stream((uint32_t *) dev_priv->pci_buf,
+				       cmd->size, dev, 0))) {
+		return ret;
+	}
+
+	ret =
+	    via_parse_command_stream(dev, (const uint32_t *)dev_priv->pci_buf,
+				     cmd->size);
+	return ret;
+}
+
+static int via_pci_cmdbuffer(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_cmdbuffer_t *cmdbuf = data;
+	int ret;
+
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
+
+	DRM_DEBUG("buf %p size %lu\n", cmdbuf->buf, cmdbuf->size);
+
+	ret = via_dispatch_pci_cmdbuffer(dev, cmdbuf);
+	return ret;
+}
+
+static inline uint32_t *via_align_buffer(drm_via_private_t *dev_priv,
+					 uint32_t * vb, int qw_count)
+{
+	for (; qw_count > 0; --qw_count)
+		VIA_OUT_RING_QW(HC_DUMMY, HC_DUMMY);
+	return vb;
+}
+
+/*
+ * This function is used internally by ring buffer management code.
+ *
+ * Returns virtual pointer to ring buffer.
+ */
+static inline uint32_t *via_get_dma(drm_via_private_t *dev_priv)
+{
+	return (uint32_t *) (dev_priv->dma_ptr + dev_priv->dma_low);
+}
+
+/*
+ * Hooks a segment of data into the tail of the ring-buffer by
+ * modifying the pause address stored in the buffer itself. If
+ * the regulator has already paused, restart it.
+ */
+static int via_hook_segment(drm_via_private_t *dev_priv,
+			    uint32_t pause_addr_hi, uint32_t pause_addr_lo,
+			    int no_pci_fire)
+{
+	int paused, count;
+	volatile uint32_t *paused_at = dev_priv->last_pause_ptr;
+	uint32_t reader, ptr;
+	uint32_t diff;
+
+	paused = 0;
+	via_flush_write_combine();
+	(void) *(volatile uint32_t *)(via_get_dma(dev_priv) - 1);
+
+	*paused_at = pause_addr_lo;
+	via_flush_write_combine();
+	(void) *paused_at;
+
+	reader = *(dev_priv->hw_addr_ptr);
+	ptr = ((volatile char *)paused_at - dev_priv->dma_ptr) +
+		dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4;
+
+	dev_priv->last_pause_ptr = via_get_dma(dev_priv) - 1;
+
+	/*
+	 * If there is a possibility that the command reader will
+	 * miss the new pause address and pause on the old one,
+	 * In that case we need to program the new start address
+	 * using PCI.
+	 */
+
+	diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;
+	count = 10000000;
+	while (diff == 0 && count--) {
+		paused = (via_read(dev_priv, 0x41c) & 0x80000000);
+		if (paused)
+			break;
+		reader = *(dev_priv->hw_addr_ptr);
+		diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;
+	}
+
+	paused = via_read(dev_priv, 0x41c) & 0x80000000;
+
+	if (paused && !no_pci_fire) {
+		reader = *(dev_priv->hw_addr_ptr);
+		diff = (uint32_t) (ptr - reader) - dev_priv->dma_diff;
+		diff &= (dev_priv->dma_high - 1);
+		if (diff != 0 && diff < (dev_priv->dma_high >> 1)) {
+			DRM_ERROR("Paused at incorrect address. "
+				  "0x%08x, 0x%08x 0x%08x\n",
+				  ptr, reader, dev_priv->dma_diff);
+		} else if (diff == 0) {
+			/*
+			 * There is a concern that these writes may stall the PCI bus
+			 * if the GPU is not idle. However, idling the GPU first
+			 * doesn't make a difference.
+			 */
+
+			via_write(dev_priv, VIA_REG_TRANSET, (HC_ParaType_PreCR << 16));
+			via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_hi);
+			via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_lo);
+			via_read(dev_priv, VIA_REG_TRANSPACE);
+		}
+	}
+	return paused;
+}
+
+static int via_wait_idle(drm_via_private_t *dev_priv)
+{
+	int count = 10000000;
+
+	while (!(via_read(dev_priv, VIA_REG_STATUS) & VIA_VR_QUEUE_BUSY) && --count)
+		;
+
+	while (count && (via_read(dev_priv, VIA_REG_STATUS) &
+			   (VIA_CMD_RGTR_BUSY | VIA_2D_ENG_BUSY |
+			    VIA_3D_ENG_BUSY)))
+		--count;
+	return count;
+}
+
+static uint32_t *via_align_cmd(drm_via_private_t *dev_priv, uint32_t cmd_type,
+			       uint32_t addr, uint32_t *cmd_addr_hi,
+			       uint32_t *cmd_addr_lo, int skip_wait)
+{
+	uint32_t agp_base;
+	uint32_t cmd_addr, addr_lo, addr_hi;
+	uint32_t *vb;
+	uint32_t qw_pad_count;
+
+	if (!skip_wait)
+		via_cmdbuf_wait(dev_priv, 2 * CMDBUF_ALIGNMENT_SIZE);
+
+	vb = via_get_dma(dev_priv);
+	VIA_OUT_RING_QW(HC_HEADER2 | ((VIA_REG_TRANSET >> 2) << 12) |
+			(VIA_REG_TRANSPACE >> 2), HC_ParaType_PreCR << 16);
+	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;
+	qw_pad_count = (CMDBUF_ALIGNMENT_SIZE >> 3) -
+	    ((dev_priv->dma_low & CMDBUF_ALIGNMENT_MASK) >> 3);
+
+	cmd_addr = (addr) ? addr :
+	    agp_base + dev_priv->dma_low - 8 + (qw_pad_count << 3);
+	addr_lo = ((HC_SubA_HAGPBpL << 24) | (cmd_type & HC_HAGPBpID_MASK) |
+		   (cmd_addr & HC_HAGPBpL_MASK));
+	addr_hi = ((HC_SubA_HAGPBpH << 24) | (cmd_addr >> 24));
+
+	vb = via_align_buffer(dev_priv, vb, qw_pad_count - 1);
+	VIA_OUT_RING_QW(*cmd_addr_hi = addr_hi, *cmd_addr_lo = addr_lo);
+	return vb;
+}
+
+static void via_cmdbuf_start(drm_via_private_t *dev_priv)
+{
+	uint32_t pause_addr_lo, pause_addr_hi;
+	uint32_t start_addr, start_addr_lo;
+	uint32_t end_addr, end_addr_lo;
+	uint32_t command;
+	uint32_t agp_base;
+	uint32_t ptr;
+	uint32_t reader;
+	int count;
+
+	dev_priv->dma_low = 0;
+
+	agp_base = dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr;
+	start_addr = agp_base;
+	end_addr = agp_base + dev_priv->dma_high;
+
+	start_addr_lo = ((HC_SubA_HAGPBstL << 24) | (start_addr & 0xFFFFFF));
+	end_addr_lo = ((HC_SubA_HAGPBendL << 24) | (end_addr & 0xFFFFFF));
+	command = ((HC_SubA_HAGPCMNT << 24) | (start_addr >> 24) |
+		   ((end_addr & 0xff000000) >> 16));
+
+	dev_priv->last_pause_ptr =
+	    via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0,
+			  &pause_addr_hi, &pause_addr_lo, 1) - 1;
+
+	via_flush_write_combine();
+	(void) *(volatile uint32_t *)dev_priv->last_pause_ptr;
+
+	via_write(dev_priv, VIA_REG_TRANSET, (HC_ParaType_PreCR << 16));
+	via_write(dev_priv, VIA_REG_TRANSPACE, command);
+	via_write(dev_priv, VIA_REG_TRANSPACE, start_addr_lo);
+	via_write(dev_priv, VIA_REG_TRANSPACE, end_addr_lo);
+
+	via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_hi);
+	via_write(dev_priv, VIA_REG_TRANSPACE, pause_addr_lo);
+	wmb();
+	via_write(dev_priv, VIA_REG_TRANSPACE, command | HC_HAGPCMNT_MASK);
+	via_read(dev_priv, VIA_REG_TRANSPACE);
+
+	dev_priv->dma_diff = 0;
+
+	count = 10000000;
+	while (!(via_read(dev_priv, 0x41c) & 0x80000000) && count--);
+
+	reader = *(dev_priv->hw_addr_ptr);
+	ptr = ((volatile char *)dev_priv->last_pause_ptr - dev_priv->dma_ptr) +
+	    dev_priv->dma_offset + (uint32_t) dev_priv->agpAddr + 4;
+
+	/*
+	 * This is the difference between where we tell the
+	 * command reader to pause and where it actually pauses.
+	 * This differs between hw implementation so we need to
+	 * detect it.
+	 */
+
+	dev_priv->dma_diff = ptr - reader;
+}
+
+static void via_pad_cache(drm_via_private_t *dev_priv, int qwords)
+{
+	uint32_t *vb;
+
+	via_cmdbuf_wait(dev_priv, qwords + 2);
+	vb = via_get_dma(dev_priv);
+	VIA_OUT_RING_QW(HC_HEADER2, HC_ParaType_NotTex << 16);
+	via_align_buffer(dev_priv, vb, qwords);
+}
+
+static inline void via_dummy_bitblt(drm_via_private_t *dev_priv)
+{
+	uint32_t *vb = via_get_dma(dev_priv);
+	SetReg2DAGP(0x0C, (0 | (0 << 16)));
+	SetReg2DAGP(0x10, 0 | (0 << 16));
+	SetReg2DAGP(0x0, 0x1 | 0x2000 | 0xAA000000);
+}
+
+static void via_cmdbuf_jump(drm_via_private_t *dev_priv)
+{
+	uint32_t pause_addr_lo, pause_addr_hi;
+	uint32_t jump_addr_lo, jump_addr_hi;
+	volatile uint32_t *last_pause_ptr;
+	uint32_t dma_low_save1, dma_low_save2;
+
+	via_align_cmd(dev_priv, HC_HAGPBpID_JUMP, 0, &jump_addr_hi,
+		      &jump_addr_lo, 0);
+
+	dev_priv->dma_wrap = dev_priv->dma_low;
+
+	/*
+	 * Wrap command buffer to the beginning.
+	 */
+
+	dev_priv->dma_low = 0;
+	if (via_cmdbuf_wait(dev_priv, CMDBUF_ALIGNMENT_SIZE) != 0)
+		DRM_ERROR("via_cmdbuf_jump failed\n");
+
+	via_dummy_bitblt(dev_priv);
+	via_dummy_bitblt(dev_priv);
+
+	last_pause_ptr =
+	    via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,
+			  &pause_addr_lo, 0) - 1;
+	via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,
+		      &pause_addr_lo, 0);
+
+	*last_pause_ptr = pause_addr_lo;
+	dma_low_save1 = dev_priv->dma_low;
+
+	/*
+	 * Now, set a trap that will pause the regulator if it tries to rerun the old
+	 * command buffer. (Which may happen if via_hook_segment detecs a command regulator pause
+	 * and reissues the jump command over PCI, while the regulator has already taken the jump
+	 * and actually paused at the current buffer end).
+	 * There appears to be no other way to detect this condition, since the hw_addr_pointer
+	 * does not seem to get updated immediately when a jump occurs.
+	 */
+
+	last_pause_ptr =
+		via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,
+			      &pause_addr_lo, 0) - 1;
+	via_align_cmd(dev_priv, HC_HAGPBpID_PAUSE, 0, &pause_addr_hi,
+		      &pause_addr_lo, 0);
+	*last_pause_ptr = pause_addr_lo;
+
+	dma_low_save2 = dev_priv->dma_low;
+	dev_priv->dma_low = dma_low_save1;
+	via_hook_segment(dev_priv, jump_addr_hi, jump_addr_lo, 0);
+	dev_priv->dma_low = dma_low_save2;
+	via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0);
+}
+
+
+static void via_cmdbuf_rewind(drm_via_private_t *dev_priv)
+{
+	via_cmdbuf_jump(dev_priv);
+}
+
+static void via_cmdbuf_flush(drm_via_private_t *dev_priv, uint32_t cmd_type)
+{
+	uint32_t pause_addr_lo, pause_addr_hi;
+
+	via_align_cmd(dev_priv, cmd_type, 0, &pause_addr_hi, &pause_addr_lo, 0);
+	via_hook_segment(dev_priv, pause_addr_hi, pause_addr_lo, 0);
+}
+
+static void via_cmdbuf_pause(drm_via_private_t *dev_priv)
+{
+	via_cmdbuf_flush(dev_priv, HC_HAGPBpID_PAUSE);
+}
+
+static void via_cmdbuf_reset(drm_via_private_t *dev_priv)
+{
+	via_cmdbuf_flush(dev_priv, HC_HAGPBpID_STOP);
+	via_wait_idle(dev_priv);
+}
+
+/*
+ * User interface to the space and lag functions.
+ */
+
+static int via_cmdbuf_size(struct drm_device *dev, void *data, struct drm_file *file_priv)
+{
+	drm_via_cmdbuf_size_t *d_siz = data;
+	int ret = 0;
+	uint32_t tmp_size, count;
+	drm_via_private_t *dev_priv;
+
+	DRM_DEBUG("\n");
+	LOCK_TEST_WITH_RETURN(dev, file_priv);
+
+	dev_priv = (drm_via_private_t *) dev->dev_private;
+
+	if (dev_priv->ring.virtual_start == NULL) {
+		DRM_ERROR("called without initializing AGP ring buffer.\n");
+		return -EFAULT;
+	}
+
+	count = 1000000;
+	tmp_size = d_siz->size;
+	switch (d_siz->func) {
+	case VIA_CMDBUF_SPACE:
+		while (((tmp_size = via_cmdbuf_space(dev_priv)) < d_siz->size)
+		       && --count) {
+			if (!d_siz->wait)
+				break;
+		}
+		if (!count) {
+			DRM_ERROR("VIA_CMDBUF_SPACE timed out.\n");
+			ret = -EAGAIN;
+		}
+		break;
+	case VIA_CMDBUF_LAG:
+		while (((tmp_size = via_cmdbuf_lag(dev_priv)) > d_siz->size)
+		       && --count) {
+			if (!d_siz->wait)
+				break;
+		}
+		if (!count) {
+			DRM_ERROR("VIA_CMDBUF_LAG timed out.\n");
+			ret = -EAGAIN;
+		}
+		break;
+	default:
+		ret = -EFAULT;
+	}
+	d_siz->size = tmp_size;
+
+	return ret;
+}
+
+static const struct drm_ioctl_desc via_ioctls[] = {
+	DRM_IOCTL_DEF_DRV(VIA_ALLOCMEM, via_mem_alloc, DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(VIA_FREEMEM, via_mem_free, DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(VIA_AGP_INIT, via_agp_init, DRM_AUTH|DRM_MASTER),
+	DRM_IOCTL_DEF_DRV(VIA_FB_INIT, via_fb_init, DRM_AUTH|DRM_MASTER),
+	DRM_IOCTL_DEF_DRV(VIA_MAP_INIT, via_map_init, DRM_AUTH|DRM_MASTER),
+	DRM_IOCTL_DEF_DRV(VIA_DEC_FUTEX, via_decoder_futex, DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(VIA_DMA_INIT, via_dma_init, DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(VIA_CMDBUFFER, via_cmdbuffer, DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(VIA_FLUSH, via_flush_ioctl, DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(VIA_PCICMD, via_pci_cmdbuffer, DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(VIA_CMDBUF_SIZE, via_cmdbuf_size, DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(VIA_WAIT_IRQ, via_wait_irq, DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(VIA_DMA_BLIT, via_dma_blit, DRM_AUTH),
+	DRM_IOCTL_DEF_DRV(VIA_BLIT_SYNC, via_dma_blit_sync, DRM_AUTH)
+};
+
+static int via_max_ioctl = ARRAY_SIZE(via_ioctls);
+static int via_driver_open(struct drm_device *dev, struct drm_file *file)
+{
+	struct via_file_private *file_priv;
+
+	DRM_DEBUG_DRIVER("\n");
+	file_priv = kmalloc(sizeof(*file_priv), GFP_KERNEL);
+	if (!file_priv)
+		return -ENOMEM;
+
+	file->driver_priv = file_priv;
+
+	INIT_LIST_HEAD(&file_priv->obj_list);
+
+	return 0;
+}
+
+static void via_driver_postclose(struct drm_device *dev, struct drm_file *file)
+{
+	struct via_file_private *file_priv = file->driver_priv;
+
+	kfree(file_priv);
+}
+
+static struct pci_device_id pciidlist[] = {
+	viadrv_PCI_IDS
+};
+
+static const struct file_operations via_driver_fops = {
+	.owner = THIS_MODULE,
+	.open = drm_open,
+	.release = drm_release,
+	.unlocked_ioctl = drm_ioctl,
+	.mmap = drm_legacy_mmap,
+	.poll = drm_poll,
+	.compat_ioctl = drm_compat_ioctl,
+	.llseek = noop_llseek,
+};
+
+static struct drm_driver driver = {
+	.driver_features =
+	    DRIVER_USE_AGP | DRIVER_HAVE_IRQ | DRIVER_LEGACY,
+	.load = via_driver_load,
+	.unload = via_driver_unload,
+	.open = via_driver_open,
+	.preclose = via_reclaim_buffers_locked,
+	.postclose = via_driver_postclose,
+	.context_dtor = via_final_context,
+	.get_vblank_counter = via_get_vblank_counter,
+	.enable_vblank = via_enable_vblank,
+	.disable_vblank = via_disable_vblank,
+	.irq_preinstall = via_driver_irq_preinstall,
+	.irq_postinstall = via_driver_irq_postinstall,
+	.irq_uninstall = via_driver_irq_uninstall,
+	.irq_handler = via_driver_irq_handler,
+	.dma_quiescent = via_driver_dma_quiescent,
+	.lastclose = via_lastclose,
+	.ioctls = via_ioctls,
+	.fops = &via_driver_fops,
+	.name = DRIVER_NAME,
+	.desc = DRIVER_DESC,
+	.date = DRIVER_DATE,
+	.major = DRIVER_MAJOR,
+	.minor = DRIVER_MINOR,
+	.patchlevel = DRIVER_PATCHLEVEL,
+};
+
+static struct pci_driver via_pci_driver = {
+	.name = DRIVER_NAME,
+	.id_table = pciidlist,
+};
+
+static int __init via_init(void)
+{
+	driver.num_ioctls = via_max_ioctl;
+	via_init_command_verifier();
+	return drm_legacy_pci_init(&driver, &via_pci_driver);
+}
+
+static void __exit via_exit(void)
+{
+	drm_legacy_pci_exit(&driver, &via_pci_driver);
+}
+
+module_init(via_init);
+module_exit(via_exit);
+
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/via/via_drv.c b/drivers/gpu/drm/via/via_drv.c
deleted file mode 100644
index 5da38082821f..000000000000
--- a/drivers/gpu/drm/via/via_drv.c
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Copyright 1998-2003 VIA Technologies, Inc. All Rights Reserved.
- * Copyright 2001-2003 S3 Graphics, Inc. All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * VIA, S3 GRAPHICS, AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- */
-
-#include <linux/module.h>
-#include <linux/pci.h>
-
-#include <drm/drm_drv.h>
-#include <drm/drm_file.h>
-#include <drm/drm_pciids.h>
-#include <drm/via_drm.h>
-
-#include "via_drv.h"
-
-
-static int via_driver_open(struct drm_device *dev, struct drm_file *file)
-{
-	struct via_file_private *file_priv;
-
-	DRM_DEBUG_DRIVER("\n");
-	file_priv = kmalloc(sizeof(*file_priv), GFP_KERNEL);
-	if (!file_priv)
-		return -ENOMEM;
-
-	file->driver_priv = file_priv;
-
-	INIT_LIST_HEAD(&file_priv->obj_list);
-
-	return 0;
-}
-
-static void via_driver_postclose(struct drm_device *dev, struct drm_file *file)
-{
-	struct via_file_private *file_priv = file->driver_priv;
-
-	kfree(file_priv);
-}
-
-static struct pci_device_id pciidlist[] = {
-	viadrv_PCI_IDS
-};
-
-static const struct file_operations via_driver_fops = {
-	.owner = THIS_MODULE,
-	.open = drm_open,
-	.release = drm_release,
-	.unlocked_ioctl = drm_ioctl,
-	.mmap = drm_legacy_mmap,
-	.poll = drm_poll,
-	.compat_ioctl = drm_compat_ioctl,
-	.llseek = noop_llseek,
-};
-
-static struct drm_driver driver = {
-	.driver_features =
-	    DRIVER_USE_AGP | DRIVER_HAVE_IRQ | DRIVER_LEGACY,
-	.load = via_driver_load,
-	.unload = via_driver_unload,
-	.open = via_driver_open,
-	.preclose = via_reclaim_buffers_locked,
-	.postclose = via_driver_postclose,
-	.context_dtor = via_final_context,
-	.get_vblank_counter = via_get_vblank_counter,
-	.enable_vblank = via_enable_vblank,
-	.disable_vblank = via_disable_vblank,
-	.irq_preinstall = via_driver_irq_preinstall,
-	.irq_postinstall = via_driver_irq_postinstall,
-	.irq_uninstall = via_driver_irq_uninstall,
-	.irq_handler = via_driver_irq_handler,
-	.dma_quiescent = via_driver_dma_quiescent,
-	.lastclose = via_lastclose,
-	.ioctls = via_ioctls,
-	.fops = &via_driver_fops,
-	.name = DRIVER_NAME,
-	.desc = DRIVER_DESC,
-	.date = DRIVER_DATE,
-	.major = DRIVER_MAJOR,
-	.minor = DRIVER_MINOR,
-	.patchlevel = DRIVER_PATCHLEVEL,
-};
-
-static struct pci_driver via_pci_driver = {
-	.name = DRIVER_NAME,
-	.id_table = pciidlist,
-};
-
-static int __init via_init(void)
-{
-	driver.num_ioctls = via_max_ioctl;
-	via_init_command_verifier();
-	return drm_legacy_pci_init(&driver, &via_pci_driver);
-}
-
-static void __exit via_exit(void)
-{
-	drm_legacy_pci_exit(&driver, &via_pci_driver);
-}
-
-module_init(via_init);
-module_exit(via_exit);
-
-MODULE_AUTHOR(DRIVER_AUTHOR);
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/via/via_drv.h b/drivers/gpu/drm/via/via_drv.h
deleted file mode 100644
index d5ad1b05bf77..000000000000
--- a/drivers/gpu/drm/via/via_drv.h
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Copyright 1998-2003 VIA Technologies, Inc. All Rights Reserved.
- * Copyright 2001-2003 S3 Graphics, Inc. All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * VIA, S3 GRAPHICS, AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- */
-#ifndef _VIA_DRV_H_
-#define _VIA_DRV_H_
-
-#include <linux/irqreturn.h>
-#include <linux/jiffies.h>
-#include <linux/sched.h>
-#include <linux/sched/signal.h>
-#include <linux/wait.h>
-
-#include <drm/drm_ioctl.h>
-#include <drm/drm_legacy.h>
-#include <drm/drm_mm.h>
-#include <drm/via_drm.h>
-
-#define DRIVER_AUTHOR	"Various"
-
-#define DRIVER_NAME		"via"
-#define DRIVER_DESC		"VIA Unichrome / Pro"
-#define DRIVER_DATE		"20070202"
-
-#define DRIVER_MAJOR		2
-#define DRIVER_MINOR		11
-#define DRIVER_PATCHLEVEL	1
-
-#include "via_verifier.h"
-
-#include "via_dmablit.h"
-
-#define VIA_PCI_BUF_SIZE 60000
-#define VIA_FIRE_BUF_SIZE  1024
-#define VIA_NUM_IRQS 4
-
-typedef struct drm_via_ring_buffer {
-	drm_local_map_t map;
-	char *virtual_start;
-} drm_via_ring_buffer_t;
-
-typedef uint32_t maskarray_t[5];
-
-typedef struct drm_via_irq {
-	atomic_t irq_received;
-	uint32_t pending_mask;
-	uint32_t enable_mask;
-	wait_queue_head_t irq_queue;
-} drm_via_irq_t;
-
-typedef struct drm_via_private {
-	drm_via_sarea_t *sarea_priv;
-	drm_local_map_t *sarea;
-	drm_local_map_t *fb;
-	drm_local_map_t *mmio;
-	unsigned long agpAddr;
-	wait_queue_head_t decoder_queue[VIA_NR_XVMC_LOCKS];
-	char *dma_ptr;
-	unsigned int dma_low;
-	unsigned int dma_high;
-	unsigned int dma_offset;
-	uint32_t dma_wrap;
-	volatile uint32_t *last_pause_ptr;
-	volatile uint32_t *hw_addr_ptr;
-	drm_via_ring_buffer_t ring;
-	ktime_t last_vblank;
-	int last_vblank_valid;
-	ktime_t nsec_per_vblank;
-	atomic_t vbl_received;
-	drm_via_state_t hc_state;
-	char pci_buf[VIA_PCI_BUF_SIZE];
-	const uint32_t *fire_offsets[VIA_FIRE_BUF_SIZE];
-	uint32_t num_fire_offsets;
-	int chipset;
-	drm_via_irq_t via_irqs[VIA_NUM_IRQS];
-	unsigned num_irqs;
-	maskarray_t *irq_masks;
-	uint32_t irq_enable_mask;
-	uint32_t irq_pending_mask;
-	int *irq_map;
-	unsigned int idle_fault;
-	int vram_initialized;
-	struct drm_mm vram_mm;
-	int agp_initialized;
-	struct drm_mm agp_mm;
-	/** Mapping of userspace keys to mm objects */
-	struct idr object_idr;
-	unsigned long vram_offset;
-	unsigned long agp_offset;
-	drm_via_blitq_t blit_queues[VIA_NUM_BLIT_ENGINES];
-	uint32_t dma_diff;
-} drm_via_private_t;
-
-struct via_file_private {
-	struct list_head obj_list;
-};
-
-enum via_family {
-  VIA_OTHER = 0,     /* Baseline */
-  VIA_PRO_GROUP_A,   /* Another video engine and DMA commands */
-  VIA_DX9_0          /* Same video as pro_group_a, but 3D is unsupported */
-};
-
-/* VIA MMIO register access */
-static inline u32 via_read(struct drm_via_private *dev_priv, u32 reg)
-{
-	return readl((void __iomem *)(dev_priv->mmio->handle + reg));
-}
-
-static inline void via_write(struct drm_via_private *dev_priv, u32 reg,
-			     u32 val)
-{
-	writel(val, (void __iomem *)(dev_priv->mmio->handle + reg));
-}
-
-static inline void via_write8(struct drm_via_private *dev_priv, u32 reg,
-			      u32 val)
-{
-	writeb(val, (void __iomem *)(dev_priv->mmio->handle + reg));
-}
-
-static inline void via_write8_mask(struct drm_via_private *dev_priv,
-				   u32 reg, u32 mask, u32 val)
-{
-	u32 tmp;
-
-	tmp = readb((void __iomem *)(dev_priv->mmio->handle + reg));
-	tmp = (tmp & ~mask) | (val & mask);
-	writeb(tmp, (void __iomem *)(dev_priv->mmio->handle + reg));
-}
-
-/*
- * Poll in a loop waiting for 'contidition' to be true.
- * Note: A direct replacement with wait_event_interruptible_timeout()
- *       will not work unless driver is updated to emit wake_up()
- *       in relevant places that can impact the 'condition'
- *
- * Returns:
- *   ret keeps current value if 'condition' becomes true
- *   ret = -BUSY if timeout happens
- *   ret = -EINTR if a signal interrupted the waiting period
- */
-#define VIA_WAIT_ON( ret, queue, timeout, condition )		\
-do {								\
-	DECLARE_WAITQUEUE(entry, current);			\
-	unsigned long end = jiffies + (timeout);		\
-	add_wait_queue(&(queue), &entry);			\
-								\
-	for (;;) {						\
-		__set_current_state(TASK_INTERRUPTIBLE);	\
-		if (condition)					\
-			break;					\
-		if (time_after_eq(jiffies, end)) {		\
-			ret = -EBUSY;				\
-			break;					\
-		}						\
-		schedule_timeout((HZ/100 > 1) ? HZ/100 : 1);	\
-		if (signal_pending(current)) {			\
-			ret = -EINTR;				\
-			break;					\
-		}						\
-	}							\
-	__set_current_state(TASK_RUNNING);			\
-	remove_wait_queue(&(queue), &entry);			\
-} while (0)
-
-extern const struct drm_ioctl_desc via_ioctls[];
-extern int via_max_ioctl;
-
-extern int via_fb_init(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int via_mem_alloc(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int via_mem_free(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int via_agp_init(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int via_map_init(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int via_decoder_futex(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int via_wait_irq(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int via_dma_blit_sync(struct drm_device *dev, void *data, struct drm_file *file_priv);
-extern int via_dma_blit(struct drm_device *dev, void *data, struct drm_file *file_priv);
-
-extern int via_driver_load(struct drm_device *dev, unsigned long chipset);
-extern void via_driver_unload(struct drm_device *dev);
-
-extern int via_init_context(struct drm_device *dev, int context);
-extern int via_final_context(struct drm_device *dev, int context);
-
-extern int via_do_cleanup_map(struct drm_device *dev);
-extern u32 via_get_vblank_counter(struct drm_device *dev, unsigned int pipe);
-extern int via_enable_vblank(struct drm_device *dev, unsigned int pipe);
-extern void via_disable_vblank(struct drm_device *dev, unsigned int pipe);
-
-extern irqreturn_t via_driver_irq_handler(int irq, void *arg);
-extern void via_driver_irq_preinstall(struct drm_device *dev);
-extern int via_driver_irq_postinstall(struct drm_device *dev);
-extern void via_driver_irq_uninstall(struct drm_device *dev);
-
-extern int via_dma_cleanup(struct drm_device *dev);
-extern void via_init_command_verifier(void);
-extern int via_driver_dma_quiescent(struct drm_device *dev);
-extern void via_init_futex(drm_via_private_t *dev_priv);
-extern void via_cleanup_futex(drm_via_private_t *dev_priv);
-extern void via_release_futex(drm_via_private_t *dev_priv, int context);
-
-extern void via_reclaim_buffers_locked(struct drm_device *dev,
-				       struct drm_file *file_priv);
-extern void via_lastclose(struct drm_device *dev);
-
-extern void via_dmablit_handler(struct drm_device *dev, int engine, int from_irq);
-extern void via_init_dmablit(struct drm_device *dev);
-
-#endif
diff --git a/drivers/gpu/drm/via/via_irq.c b/drivers/gpu/drm/via/via_irq.c
deleted file mode 100644
index faeae5d881fb..000000000000
--- a/drivers/gpu/drm/via/via_irq.c
+++ /dev/null
@@ -1,388 +0,0 @@
-/* via_irq.c
- *
- * Copyright 2004 BEAM Ltd.
- * Copyright 2002 Tungsten Graphics, Inc.
- * Copyright 2005 Thomas Hellstrom.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * BEAM LTD, TUNGSTEN GRAPHICS  AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Terry Barnaby <terry1@beam.ltd.uk>
- *    Keith Whitwell <keith@tungstengraphics.com>
- *    Thomas Hellstrom <unichrome@shipmail.org>
- *
- * This code provides standard DRM access to the Via Unichrome / Pro Vertical blank
- * interrupt, as well as an infrastructure to handle other interrupts of the chip.
- * The refresh rate is also calculated for video playback sync purposes.
- */
-
-#include <drm/drm_device.h>
-#include <drm/drm_vblank.h>
-#include <drm/via_drm.h>
-
-#include "via_drv.h"
-
-#define VIA_REG_INTERRUPT       0x200
-
-/* VIA_REG_INTERRUPT */
-#define VIA_IRQ_GLOBAL	  (1 << 31)
-#define VIA_IRQ_VBLANK_ENABLE   (1 << 19)
-#define VIA_IRQ_VBLANK_PENDING  (1 << 3)
-#define VIA_IRQ_HQV0_ENABLE     (1 << 11)
-#define VIA_IRQ_HQV1_ENABLE     (1 << 25)
-#define VIA_IRQ_HQV0_PENDING    (1 << 9)
-#define VIA_IRQ_HQV1_PENDING    (1 << 10)
-#define VIA_IRQ_DMA0_DD_ENABLE  (1 << 20)
-#define VIA_IRQ_DMA0_TD_ENABLE  (1 << 21)
-#define VIA_IRQ_DMA1_DD_ENABLE  (1 << 22)
-#define VIA_IRQ_DMA1_TD_ENABLE  (1 << 23)
-#define VIA_IRQ_DMA0_DD_PENDING (1 << 4)
-#define VIA_IRQ_DMA0_TD_PENDING (1 << 5)
-#define VIA_IRQ_DMA1_DD_PENDING (1 << 6)
-#define VIA_IRQ_DMA1_TD_PENDING (1 << 7)
-
-
-/*
- * Device-specific IRQs go here. This type might need to be extended with
- * the register if there are multiple IRQ control registers.
- * Currently we activate the HQV interrupts of  Unichrome Pro group A.
- */
-
-static maskarray_t via_pro_group_a_irqs[] = {
-	{VIA_IRQ_HQV0_ENABLE, VIA_IRQ_HQV0_PENDING, 0x000003D0, 0x00008010,
-	 0x00000000 },
-	{VIA_IRQ_HQV1_ENABLE, VIA_IRQ_HQV1_PENDING, 0x000013D0, 0x00008010,
-	 0x00000000 },
-	{VIA_IRQ_DMA0_TD_ENABLE, VIA_IRQ_DMA0_TD_PENDING, VIA_PCI_DMA_CSR0,
-	 VIA_DMA_CSR_TA | VIA_DMA_CSR_TD, 0x00000008},
-	{VIA_IRQ_DMA1_TD_ENABLE, VIA_IRQ_DMA1_TD_PENDING, VIA_PCI_DMA_CSR1,
-	 VIA_DMA_CSR_TA | VIA_DMA_CSR_TD, 0x00000008},
-};
-static int via_num_pro_group_a = ARRAY_SIZE(via_pro_group_a_irqs);
-static int via_irqmap_pro_group_a[] = {0, 1, -1, 2, -1, 3};
-
-static maskarray_t via_unichrome_irqs[] = {
-	{VIA_IRQ_DMA0_TD_ENABLE, VIA_IRQ_DMA0_TD_PENDING, VIA_PCI_DMA_CSR0,
-	 VIA_DMA_CSR_TA | VIA_DMA_CSR_TD, 0x00000008},
-	{VIA_IRQ_DMA1_TD_ENABLE, VIA_IRQ_DMA1_TD_PENDING, VIA_PCI_DMA_CSR1,
-	 VIA_DMA_CSR_TA | VIA_DMA_CSR_TD, 0x00000008}
-};
-static int via_num_unichrome = ARRAY_SIZE(via_unichrome_irqs);
-static int via_irqmap_unichrome[] = {-1, -1, -1, 0, -1, 1};
-
-
-u32 via_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
-{
-	drm_via_private_t *dev_priv = dev->dev_private;
-
-	if (pipe != 0)
-		return 0;
-
-	return atomic_read(&dev_priv->vbl_received);
-}
-
-irqreturn_t via_driver_irq_handler(int irq, void *arg)
-{
-	struct drm_device *dev = (struct drm_device *) arg;
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	u32 status;
-	int handled = 0;
-	ktime_t cur_vblank;
-	drm_via_irq_t *cur_irq = dev_priv->via_irqs;
-	int i;
-
-	status = via_read(dev_priv, VIA_REG_INTERRUPT);
-	if (status & VIA_IRQ_VBLANK_PENDING) {
-		atomic_inc(&dev_priv->vbl_received);
-		if (!(atomic_read(&dev_priv->vbl_received) & 0x0F)) {
-			cur_vblank = ktime_get();
-			if (dev_priv->last_vblank_valid) {
-				dev_priv->nsec_per_vblank =
-					ktime_sub(cur_vblank,
-						dev_priv->last_vblank) >> 4;
-			}
-			dev_priv->last_vblank = cur_vblank;
-			dev_priv->last_vblank_valid = 1;
-		}
-		if (!(atomic_read(&dev_priv->vbl_received) & 0xFF)) {
-			DRM_DEBUG("nsec per vblank is: %llu\n",
-				  ktime_to_ns(dev_priv->nsec_per_vblank));
-		}
-		drm_handle_vblank(dev, 0);
-		handled = 1;
-	}
-
-	for (i = 0; i < dev_priv->num_irqs; ++i) {
-		if (status & cur_irq->pending_mask) {
-			atomic_inc(&cur_irq->irq_received);
-			wake_up(&cur_irq->irq_queue);
-			handled = 1;
-			if (dev_priv->irq_map[drm_via_irq_dma0_td] == i)
-				via_dmablit_handler(dev, 0, 1);
-			else if (dev_priv->irq_map[drm_via_irq_dma1_td] == i)
-				via_dmablit_handler(dev, 1, 1);
-		}
-		cur_irq++;
-	}
-
-	/* Acknowledge interrupts */
-	via_write(dev_priv, VIA_REG_INTERRUPT, status);
-
-
-	if (handled)
-		return IRQ_HANDLED;
-	else
-		return IRQ_NONE;
-}
-
-static __inline__ void viadrv_acknowledge_irqs(drm_via_private_t *dev_priv)
-{
-	u32 status;
-
-	if (dev_priv) {
-		/* Acknowledge interrupts */
-		status = via_read(dev_priv, VIA_REG_INTERRUPT);
-		via_write(dev_priv, VIA_REG_INTERRUPT, status |
-			  dev_priv->irq_pending_mask);
-	}
-}
-
-int via_enable_vblank(struct drm_device *dev, unsigned int pipe)
-{
-	drm_via_private_t *dev_priv = dev->dev_private;
-	u32 status;
-
-	if (pipe != 0) {
-		DRM_ERROR("%s:  bad crtc %u\n", __func__, pipe);
-		return -EINVAL;
-	}
-
-	status = via_read(dev_priv, VIA_REG_INTERRUPT);
-	via_write(dev_priv, VIA_REG_INTERRUPT, status | VIA_IRQ_VBLANK_ENABLE);
-
-	via_write8(dev_priv, 0x83d4, 0x11);
-	via_write8_mask(dev_priv, 0x83d5, 0x30, 0x30);
-
-	return 0;
-}
-
-void via_disable_vblank(struct drm_device *dev, unsigned int pipe)
-{
-	drm_via_private_t *dev_priv = dev->dev_private;
-	u32 status;
-
-	status = via_read(dev_priv, VIA_REG_INTERRUPT);
-	via_write(dev_priv, VIA_REG_INTERRUPT, status & ~VIA_IRQ_VBLANK_ENABLE);
-
-	via_write8(dev_priv, 0x83d4, 0x11);
-	via_write8_mask(dev_priv, 0x83d5, 0x30, 0);
-
-	if (pipe != 0)
-		DRM_ERROR("%s:  bad crtc %u\n", __func__, pipe);
-}
-
-static int
-via_driver_irq_wait(struct drm_device *dev, unsigned int irq, int force_sequence,
-		    unsigned int *sequence)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	unsigned int cur_irq_sequence;
-	drm_via_irq_t *cur_irq;
-	int ret = 0;
-	maskarray_t *masks;
-	int real_irq;
-
-	DRM_DEBUG("\n");
-
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
-	if (irq >= drm_via_irq_num) {
-		DRM_ERROR("Trying to wait on unknown irq %d\n", irq);
-		return -EINVAL;
-	}
-
-	real_irq = dev_priv->irq_map[irq];
-
-	if (real_irq < 0) {
-		DRM_ERROR("Video IRQ %d not available on this hardware.\n",
-			  irq);
-		return -EINVAL;
-	}
-
-	masks = dev_priv->irq_masks;
-	cur_irq = dev_priv->via_irqs + real_irq;
-
-	if (masks[real_irq][2] && !force_sequence) {
-		VIA_WAIT_ON(ret, cur_irq->irq_queue, 3 * HZ,
-			    ((via_read(dev_priv, masks[irq][2]) & masks[irq][3]) ==
-			     masks[irq][4]));
-		cur_irq_sequence = atomic_read(&cur_irq->irq_received);
-	} else {
-		VIA_WAIT_ON(ret, cur_irq->irq_queue, 3 * HZ,
-			    (((cur_irq_sequence =
-			       atomic_read(&cur_irq->irq_received)) -
-			      *sequence) <= (1 << 23)));
-	}
-	*sequence = cur_irq_sequence;
-	return ret;
-}
-
-
-/*
- * drm_dma.h hooks
- */
-
-void via_driver_irq_preinstall(struct drm_device *dev)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	u32 status;
-	drm_via_irq_t *cur_irq;
-	int i;
-
-	DRM_DEBUG("dev_priv: %p\n", dev_priv);
-	if (dev_priv) {
-		cur_irq = dev_priv->via_irqs;
-
-		dev_priv->irq_enable_mask = VIA_IRQ_VBLANK_ENABLE;
-		dev_priv->irq_pending_mask = VIA_IRQ_VBLANK_PENDING;
-
-		if (dev_priv->chipset == VIA_PRO_GROUP_A ||
-		    dev_priv->chipset == VIA_DX9_0) {
-			dev_priv->irq_masks = via_pro_group_a_irqs;
-			dev_priv->num_irqs = via_num_pro_group_a;
-			dev_priv->irq_map = via_irqmap_pro_group_a;
-		} else {
-			dev_priv->irq_masks = via_unichrome_irqs;
-			dev_priv->num_irqs = via_num_unichrome;
-			dev_priv->irq_map = via_irqmap_unichrome;
-		}
-
-		for (i = 0; i < dev_priv->num_irqs; ++i) {
-			atomic_set(&cur_irq->irq_received, 0);
-			cur_irq->enable_mask = dev_priv->irq_masks[i][0];
-			cur_irq->pending_mask = dev_priv->irq_masks[i][1];
-			init_waitqueue_head(&cur_irq->irq_queue);
-			dev_priv->irq_enable_mask |= cur_irq->enable_mask;
-			dev_priv->irq_pending_mask |= cur_irq->pending_mask;
-			cur_irq++;
-
-			DRM_DEBUG("Initializing IRQ %d\n", i);
-		}
-
-		dev_priv->last_vblank_valid = 0;
-
-		/* Clear VSync interrupt regs */
-		status = via_read(dev_priv, VIA_REG_INTERRUPT);
-		via_write(dev_priv, VIA_REG_INTERRUPT, status &
-			  ~(dev_priv->irq_enable_mask));
-
-		/* Clear bits if they're already high */
-		viadrv_acknowledge_irqs(dev_priv);
-	}
-}
-
-int via_driver_irq_postinstall(struct drm_device *dev)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	u32 status;
-
-	DRM_DEBUG("fun: %s\n", __func__);
-	if (!dev_priv)
-		return -EINVAL;
-
-	status = via_read(dev_priv, VIA_REG_INTERRUPT);
-	via_write(dev_priv, VIA_REG_INTERRUPT, status | VIA_IRQ_GLOBAL
-		  | dev_priv->irq_enable_mask);
-
-	/* Some magic, oh for some data sheets ! */
-	via_write8(dev_priv, 0x83d4, 0x11);
-	via_write8_mask(dev_priv, 0x83d5, 0x30, 0x30);
-
-	return 0;
-}
-
-void via_driver_irq_uninstall(struct drm_device *dev)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	u32 status;
-
-	DRM_DEBUG("\n");
-	if (dev_priv) {
-
-		/* Some more magic, oh for some data sheets ! */
-
-		via_write8(dev_priv, 0x83d4, 0x11);
-		via_write8_mask(dev_priv, 0x83d5, 0x30, 0);
-
-		status = via_read(dev_priv, VIA_REG_INTERRUPT);
-		via_write(dev_priv, VIA_REG_INTERRUPT, status &
-			  ~(VIA_IRQ_VBLANK_ENABLE | dev_priv->irq_enable_mask));
-	}
-}
-
-int via_wait_irq(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_irqwait_t *irqwait = data;
-	struct timespec64 now;
-	int ret = 0;
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	drm_via_irq_t *cur_irq = dev_priv->via_irqs;
-	int force_sequence;
-
-	if (irqwait->request.irq >= dev_priv->num_irqs) {
-		DRM_ERROR("Trying to wait on unknown irq %d\n",
-			  irqwait->request.irq);
-		return -EINVAL;
-	}
-
-	cur_irq += irqwait->request.irq;
-
-	switch (irqwait->request.type & ~VIA_IRQ_FLAGS_MASK) {
-	case VIA_IRQ_RELATIVE:
-		irqwait->request.sequence +=
-			atomic_read(&cur_irq->irq_received);
-		irqwait->request.type &= ~_DRM_VBLANK_RELATIVE;
-		break;
-	case VIA_IRQ_ABSOLUTE:
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	if (irqwait->request.type & VIA_IRQ_SIGNAL) {
-		DRM_ERROR("Signals on Via IRQs not implemented yet.\n");
-		return -EINVAL;
-	}
-
-	force_sequence = (irqwait->request.type & VIA_IRQ_FORCE_SEQUENCE);
-
-	ret = via_driver_irq_wait(dev, irqwait->request.irq, force_sequence,
-				  &irqwait->request.sequence);
-	ktime_get_ts64(&now);
-	irqwait->reply.tval_sec = now.tv_sec;
-	irqwait->reply.tval_usec = now.tv_nsec / NSEC_PER_USEC;
-
-	return ret;
-}
diff --git a/drivers/gpu/drm/via/via_map.c b/drivers/gpu/drm/via/via_map.c
deleted file mode 100644
index a9f6b0c11966..000000000000
--- a/drivers/gpu/drm/via/via_map.c
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Copyright 1998-2003 VIA Technologies, Inc. All Rights Reserved.
- * Copyright 2001-2003 S3 Graphics, Inc. All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * VIA, S3 GRAPHICS, AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- */
-
-#include <linux/pci.h>
-
-#include <drm/drm_device.h>
-#include <drm/drm_vblank.h>
-#include <drm/via_drm.h>
-
-#include "via_drv.h"
-
-static int via_do_init_map(struct drm_device *dev, drm_via_init_t *init)
-{
-	drm_via_private_t *dev_priv = dev->dev_private;
-
-	DRM_DEBUG("\n");
-
-	dev_priv->sarea = drm_legacy_getsarea(dev);
-	if (!dev_priv->sarea) {
-		DRM_ERROR("could not find sarea!\n");
-		dev->dev_private = (void *)dev_priv;
-		via_do_cleanup_map(dev);
-		return -EINVAL;
-	}
-
-	dev_priv->fb = drm_legacy_findmap(dev, init->fb_offset);
-	if (!dev_priv->fb) {
-		DRM_ERROR("could not find framebuffer!\n");
-		dev->dev_private = (void *)dev_priv;
-		via_do_cleanup_map(dev);
-		return -EINVAL;
-	}
-	dev_priv->mmio = drm_legacy_findmap(dev, init->mmio_offset);
-	if (!dev_priv->mmio) {
-		DRM_ERROR("could not find mmio region!\n");
-		dev->dev_private = (void *)dev_priv;
-		via_do_cleanup_map(dev);
-		return -EINVAL;
-	}
-
-	dev_priv->sarea_priv =
-	    (drm_via_sarea_t *) ((u8 *) dev_priv->sarea->handle +
-				 init->sarea_priv_offset);
-
-	dev_priv->agpAddr = init->agpAddr;
-
-	via_init_futex(dev_priv);
-
-	via_init_dmablit(dev);
-
-	dev->dev_private = (void *)dev_priv;
-	return 0;
-}
-
-int via_do_cleanup_map(struct drm_device *dev)
-{
-	via_dma_cleanup(dev);
-
-	return 0;
-}
-
-int via_map_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_init_t *init = data;
-
-	DRM_DEBUG("\n");
-
-	switch (init->func) {
-	case VIA_INIT_MAP:
-		return via_do_init_map(dev, init);
-	case VIA_CLEANUP_MAP:
-		return via_do_cleanup_map(dev);
-	}
-
-	return -EINVAL;
-}
-
-int via_driver_load(struct drm_device *dev, unsigned long chipset)
-{
-	struct pci_dev *pdev = to_pci_dev(dev->dev);
-	drm_via_private_t *dev_priv;
-	int ret = 0;
-
-	dev_priv = kzalloc(sizeof(drm_via_private_t), GFP_KERNEL);
-	if (dev_priv == NULL)
-		return -ENOMEM;
-
-	idr_init(&dev_priv->object_idr);
-	dev->dev_private = (void *)dev_priv;
-
-	dev_priv->chipset = chipset;
-
-	pci_set_master(pdev);
-
-	ret = drm_vblank_init(dev, 1);
-	if (ret) {
-		kfree(dev_priv);
-		return ret;
-	}
-
-	return 0;
-}
-
-void via_driver_unload(struct drm_device *dev)
-{
-	drm_via_private_t *dev_priv = dev->dev_private;
-
-	idr_destroy(&dev_priv->object_idr);
-
-	kfree(dev_priv);
-}
diff --git a/drivers/gpu/drm/via/via_mm.c b/drivers/gpu/drm/via/via_mm.c
deleted file mode 100644
index c9afa1a51f23..000000000000
--- a/drivers/gpu/drm/via/via_mm.c
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Copyright 2006 Tungsten Graphics Inc., Bismarck, ND., USA.
- * All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
- * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- */
-/*
- * Authors: Thomas Hellström <thomas-at-tungstengraphics-dot-com>
- */
-
-#include <linux/slab.h>
-
-#include <drm/drm_device.h>
-#include <drm/drm_file.h>
-#include <drm/via_drm.h>
-
-#include "via_drv.h"
-
-#define VIA_MM_ALIGN_SHIFT 4
-#define VIA_MM_ALIGN_MASK ((1 << VIA_MM_ALIGN_SHIFT) - 1)
-
-struct via_memblock {
-	struct drm_mm_node mm_node;
-	struct list_head owner_list;
-};
-
-int via_agp_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_agp_t *agp = data;
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-
-	mutex_lock(&dev->struct_mutex);
-	drm_mm_init(&dev_priv->agp_mm, 0, agp->size >> VIA_MM_ALIGN_SHIFT);
-
-	dev_priv->agp_initialized = 1;
-	dev_priv->agp_offset = agp->offset;
-	mutex_unlock(&dev->struct_mutex);
-
-	DRM_DEBUG("offset = %u, size = %u\n", agp->offset, agp->size);
-	return 0;
-}
-
-int via_fb_init(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_fb_t *fb = data;
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-
-	mutex_lock(&dev->struct_mutex);
-	drm_mm_init(&dev_priv->vram_mm, 0, fb->size >> VIA_MM_ALIGN_SHIFT);
-
-	dev_priv->vram_initialized = 1;
-	dev_priv->vram_offset = fb->offset;
-
-	mutex_unlock(&dev->struct_mutex);
-	DRM_DEBUG("offset = %u, size = %u\n", fb->offset, fb->size);
-
-	return 0;
-
-}
-
-int via_final_context(struct drm_device *dev, int context)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-
-	via_release_futex(dev_priv, context);
-
-	/* Linux specific until context tracking code gets ported to BSD */
-	/* Last context, perform cleanup */
-	if (list_is_singular(&dev->ctxlist)) {
-		DRM_DEBUG("Last Context\n");
-		drm_legacy_irq_uninstall(dev);
-		via_cleanup_futex(dev_priv);
-		via_do_cleanup_map(dev);
-	}
-	return 1;
-}
-
-void via_lastclose(struct drm_device *dev)
-{
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-
-	if (!dev_priv)
-		return;
-
-	mutex_lock(&dev->struct_mutex);
-	if (dev_priv->vram_initialized) {
-		drm_mm_takedown(&dev_priv->vram_mm);
-		dev_priv->vram_initialized = 0;
-	}
-	if (dev_priv->agp_initialized) {
-		drm_mm_takedown(&dev_priv->agp_mm);
-		dev_priv->agp_initialized = 0;
-	}
-	mutex_unlock(&dev->struct_mutex);
-}
-
-int via_mem_alloc(struct drm_device *dev, void *data,
-		  struct drm_file *file)
-{
-	drm_via_mem_t *mem = data;
-	int retval = 0, user_key;
-	struct via_memblock *item;
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	struct via_file_private *file_priv = file->driver_priv;
-	unsigned long tmpSize;
-
-	if (mem->type > VIA_MEM_AGP) {
-		DRM_ERROR("Unknown memory type allocation\n");
-		return -EINVAL;
-	}
-	mutex_lock(&dev->struct_mutex);
-	if (0 == ((mem->type == VIA_MEM_VIDEO) ? dev_priv->vram_initialized :
-		      dev_priv->agp_initialized)) {
-		mutex_unlock(&dev->struct_mutex);
-		DRM_ERROR
-		    ("Attempt to allocate from uninitialized memory manager.\n");
-		return -EINVAL;
-	}
-
-	item = kzalloc(sizeof(*item), GFP_KERNEL);
-	if (!item) {
-		retval = -ENOMEM;
-		goto fail_alloc;
-	}
-
-	tmpSize = (mem->size + VIA_MM_ALIGN_MASK) >> VIA_MM_ALIGN_SHIFT;
-	if (mem->type == VIA_MEM_AGP)
-		retval = drm_mm_insert_node(&dev_priv->agp_mm,
-					    &item->mm_node,
-					    tmpSize);
-	else
-		retval = drm_mm_insert_node(&dev_priv->vram_mm,
-					    &item->mm_node,
-					    tmpSize);
-	if (retval)
-		goto fail_alloc;
-
-	retval = idr_alloc(&dev_priv->object_idr, item, 1, 0, GFP_KERNEL);
-	if (retval < 0)
-		goto fail_idr;
-	user_key = retval;
-
-	list_add(&item->owner_list, &file_priv->obj_list);
-	mutex_unlock(&dev->struct_mutex);
-
-	mem->offset = ((mem->type == VIA_MEM_VIDEO) ?
-		      dev_priv->vram_offset : dev_priv->agp_offset) +
-	    ((item->mm_node.start) << VIA_MM_ALIGN_SHIFT);
-	mem->index = user_key;
-
-	return 0;
-
-fail_idr:
-	drm_mm_remove_node(&item->mm_node);
-fail_alloc:
-	kfree(item);
-	mutex_unlock(&dev->struct_mutex);
-
-	mem->offset = 0;
-	mem->size = 0;
-	mem->index = 0;
-	DRM_DEBUG("Video memory allocation failed\n");
-
-	return retval;
-}
-
-int via_mem_free(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_private_t *dev_priv = dev->dev_private;
-	drm_via_mem_t *mem = data;
-	struct via_memblock *obj;
-
-	mutex_lock(&dev->struct_mutex);
-	obj = idr_find(&dev_priv->object_idr, mem->index);
-	if (obj == NULL) {
-		mutex_unlock(&dev->struct_mutex);
-		return -EINVAL;
-	}
-
-	idr_remove(&dev_priv->object_idr, mem->index);
-	list_del(&obj->owner_list);
-	drm_mm_remove_node(&obj->mm_node);
-	kfree(obj);
-	mutex_unlock(&dev->struct_mutex);
-
-	DRM_DEBUG("free = 0x%lx\n", mem->index);
-
-	return 0;
-}
-
-
-void via_reclaim_buffers_locked(struct drm_device *dev,
-				struct drm_file *file)
-{
-	struct via_file_private *file_priv = file->driver_priv;
-	struct via_memblock *entry, *next;
-
-	if (!(dev->master && file->master->lock.hw_lock))
-		return;
-
-	drm_legacy_idlelock_take(&file->master->lock);
-
-	mutex_lock(&dev->struct_mutex);
-	if (list_empty(&file_priv->obj_list)) {
-		mutex_unlock(&dev->struct_mutex);
-		drm_legacy_idlelock_release(&file->master->lock);
-
-		return;
-	}
-
-	via_driver_dma_quiescent(dev);
-
-	list_for_each_entry_safe(entry, next, &file_priv->obj_list,
-				 owner_list) {
-		list_del(&entry->owner_list);
-		drm_mm_remove_node(&entry->mm_node);
-		kfree(entry);
-	}
-	mutex_unlock(&dev->struct_mutex);
-
-	drm_legacy_idlelock_release(&file->master->lock);
-
-	return;
-}
diff --git a/drivers/gpu/drm/via/via_verifier.c b/drivers/gpu/drm/via/via_verifier.c
deleted file mode 100644
index 3d6e3a70f318..000000000000
--- a/drivers/gpu/drm/via/via_verifier.c
+++ /dev/null
@@ -1,1110 +0,0 @@
-/*
- * Copyright 2004 The Unichrome Project. All Rights Reserved.
- * Copyright 2005 Thomas Hellstrom. All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHOR(S), AND/OR THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Author: Thomas Hellstrom 2004, 2005.
- * This code was written using docs obtained under NDA from VIA Inc.
- *
- * Don't run this code directly on an AGP buffer. Due to cache problems it will
- * be very slow.
- */
-
-#include <drm/drm_device.h>
-#include <drm/drm_legacy.h>
-#include <drm/via_drm.h>
-
-#include "via_3d_reg.h"
-#include "via_drv.h"
-#include "via_verifier.h"
-
-typedef enum {
-	state_command,
-	state_header2,
-	state_header1,
-	state_vheader5,
-	state_vheader6,
-	state_error
-} verifier_state_t;
-
-typedef enum {
-	no_check = 0,
-	check_for_header2,
-	check_for_header1,
-	check_for_header2_err,
-	check_for_header1_err,
-	check_for_fire,
-	check_z_buffer_addr0,
-	check_z_buffer_addr1,
-	check_z_buffer_addr_mode,
-	check_destination_addr0,
-	check_destination_addr1,
-	check_destination_addr_mode,
-	check_for_dummy,
-	check_for_dd,
-	check_texture_addr0,
-	check_texture_addr1,
-	check_texture_addr2,
-	check_texture_addr3,
-	check_texture_addr4,
-	check_texture_addr5,
-	check_texture_addr6,
-	check_texture_addr7,
-	check_texture_addr8,
-	check_texture_addr_mode,
-	check_for_vertex_count,
-	check_number_texunits,
-	forbidden_command
-} hazard_t;
-
-/*
- * Associates each hazard above with a possible multi-command
- * sequence. For example an address that is split over multiple
- * commands and that needs to be checked at the first command
- * that does not include any part of the address.
- */
-
-static drm_via_sequence_t seqs[] = {
-	no_sequence,
-	no_sequence,
-	no_sequence,
-	no_sequence,
-	no_sequence,
-	no_sequence,
-	z_address,
-	z_address,
-	z_address,
-	dest_address,
-	dest_address,
-	dest_address,
-	no_sequence,
-	no_sequence,
-	tex_address,
-	tex_address,
-	tex_address,
-	tex_address,
-	tex_address,
-	tex_address,
-	tex_address,
-	tex_address,
-	tex_address,
-	tex_address,
-	no_sequence
-};
-
-typedef struct {
-	unsigned int code;
-	hazard_t hz;
-} hz_init_t;
-
-static hz_init_t init_table1[] = {
-	{0xf2, check_for_header2_err},
-	{0xf0, check_for_header1_err},
-	{0xee, check_for_fire},
-	{0xcc, check_for_dummy},
-	{0xdd, check_for_dd},
-	{0x00, no_check},
-	{0x10, check_z_buffer_addr0},
-	{0x11, check_z_buffer_addr1},
-	{0x12, check_z_buffer_addr_mode},
-	{0x13, no_check},
-	{0x14, no_check},
-	{0x15, no_check},
-	{0x23, no_check},
-	{0x24, no_check},
-	{0x33, no_check},
-	{0x34, no_check},
-	{0x35, no_check},
-	{0x36, no_check},
-	{0x37, no_check},
-	{0x38, no_check},
-	{0x39, no_check},
-	{0x3A, no_check},
-	{0x3B, no_check},
-	{0x3C, no_check},
-	{0x3D, no_check},
-	{0x3E, no_check},
-	{0x40, check_destination_addr0},
-	{0x41, check_destination_addr1},
-	{0x42, check_destination_addr_mode},
-	{0x43, no_check},
-	{0x44, no_check},
-	{0x50, no_check},
-	{0x51, no_check},
-	{0x52, no_check},
-	{0x53, no_check},
-	{0x54, no_check},
-	{0x55, no_check},
-	{0x56, no_check},
-	{0x57, no_check},
-	{0x58, no_check},
-	{0x70, no_check},
-	{0x71, no_check},
-	{0x78, no_check},
-	{0x79, no_check},
-	{0x7A, no_check},
-	{0x7B, no_check},
-	{0x7C, no_check},
-	{0x7D, check_for_vertex_count}
-};
-
-static hz_init_t init_table2[] = {
-	{0xf2, check_for_header2_err},
-	{0xf0, check_for_header1_err},
-	{0xee, check_for_fire},
-	{0xcc, check_for_dummy},
-	{0x00, check_texture_addr0},
-	{0x01, check_texture_addr0},
-	{0x02, check_texture_addr0},
-	{0x03, check_texture_addr0},
-	{0x04, check_texture_addr0},
-	{0x05, check_texture_addr0},
-	{0x06, check_texture_addr0},
-	{0x07, check_texture_addr0},
-	{0x08, check_texture_addr0},
-	{0x09, check_texture_addr0},
-	{0x20, check_texture_addr1},
-	{0x21, check_texture_addr1},
-	{0x22, check_texture_addr1},
-	{0x23, check_texture_addr4},
-	{0x2B, check_texture_addr3},
-	{0x2C, check_texture_addr3},
-	{0x2D, check_texture_addr3},
-	{0x2E, check_texture_addr3},
-	{0x2F, check_texture_addr3},
-	{0x30, check_texture_addr3},
-	{0x31, check_texture_addr3},
-	{0x32, check_texture_addr3},
-	{0x33, check_texture_addr3},
-	{0x34, check_texture_addr3},
-	{0x4B, check_texture_addr5},
-	{0x4C, check_texture_addr6},
-	{0x51, check_texture_addr7},
-	{0x52, check_texture_addr8},
-	{0x77, check_texture_addr2},
-	{0x78, no_check},
-	{0x79, no_check},
-	{0x7A, no_check},
-	{0x7B, check_texture_addr_mode},
-	{0x7C, no_check},
-	{0x7D, no_check},
-	{0x7E, no_check},
-	{0x7F, no_check},
-	{0x80, no_check},
-	{0x81, no_check},
-	{0x82, no_check},
-	{0x83, no_check},
-	{0x85, no_check},
-	{0x86, no_check},
-	{0x87, no_check},
-	{0x88, no_check},
-	{0x89, no_check},
-	{0x8A, no_check},
-	{0x90, no_check},
-	{0x91, no_check},
-	{0x92, no_check},
-	{0x93, no_check}
-};
-
-static hz_init_t init_table3[] = {
-	{0xf2, check_for_header2_err},
-	{0xf0, check_for_header1_err},
-	{0xcc, check_for_dummy},
-	{0x00, check_number_texunits}
-};
-
-static hazard_t table1[256];
-static hazard_t table2[256];
-static hazard_t table3[256];
-
-static __inline__ int
-eat_words(const uint32_t **buf, const uint32_t *buf_end, unsigned num_words)
-{
-	if ((buf_end - *buf) >= num_words) {
-		*buf += num_words;
-		return 0;
-	}
-	DRM_ERROR("Illegal termination of DMA command buffer\n");
-	return 1;
-}
-
-/*
- * Partially stolen from drm_memory.h
- */
-
-static __inline__ drm_local_map_t *via_drm_lookup_agp_map(drm_via_state_t *seq,
-						    unsigned long offset,
-						    unsigned long size,
-						    struct drm_device *dev)
-{
-	struct drm_map_list *r_list;
-	drm_local_map_t *map = seq->map_cache;
-
-	if (map && map->offset <= offset
-	    && (offset + size) <= (map->offset + map->size)) {
-		return map;
-	}
-
-	list_for_each_entry(r_list, &dev->maplist, head) {
-		map = r_list->map;
-		if (!map)
-			continue;
-		if (map->offset <= offset
-		    && (offset + size) <= (map->offset + map->size)
-		    && !(map->flags & _DRM_RESTRICTED)
-		    && (map->type == _DRM_AGP)) {
-			seq->map_cache = map;
-			return map;
-		}
-	}
-	return NULL;
-}
-
-/*
- * Require that all AGP texture levels reside in the same AGP map which should
- * be mappable by the client. This is not a big restriction.
- * FIXME: To actually enforce this security policy strictly, drm_rmmap
- * would have to wait for dma quiescent before removing an AGP map.
- * The via_drm_lookup_agp_map call in reality seems to take
- * very little CPU time.
- */
-
-static __inline__ int finish_current_sequence(drm_via_state_t * cur_seq)
-{
-	switch (cur_seq->unfinished) {
-	case z_address:
-		DRM_DEBUG("Z Buffer start address is 0x%x\n", cur_seq->z_addr);
-		break;
-	case dest_address:
-		DRM_DEBUG("Destination start address is 0x%x\n",
-			  cur_seq->d_addr);
-		break;
-	case tex_address:
-		if (cur_seq->agp_texture) {
-			unsigned start =
-			    cur_seq->tex_level_lo[cur_seq->texture];
-			unsigned end = cur_seq->tex_level_hi[cur_seq->texture];
-			unsigned long lo = ~0, hi = 0, tmp;
-			uint32_t *addr, *pitch, *height, tex;
-			unsigned i;
-			int npot;
-
-			if (end > 9)
-				end = 9;
-			if (start > 9)
-				start = 9;
-
-			addr =
-			    &(cur_seq->t_addr[tex = cur_seq->texture][start]);
-			pitch = &(cur_seq->pitch[tex][start]);
-			height = &(cur_seq->height[tex][start]);
-			npot = cur_seq->tex_npot[tex];
-			for (i = start; i <= end; ++i) {
-				tmp = *addr++;
-				if (tmp < lo)
-					lo = tmp;
-				if (i == 0 && npot)
-					tmp += (*height++ * *pitch++);
-				else
-					tmp += (*height++ << *pitch++);
-				if (tmp > hi)
-					hi = tmp;
-			}
-
-			if (!via_drm_lookup_agp_map
-			    (cur_seq, lo, hi - lo, cur_seq->dev)) {
-				DRM_ERROR
-				    ("AGP texture is not in allowed map\n");
-				return 2;
-			}
-		}
-		break;
-	default:
-		break;
-	}
-	cur_seq->unfinished = no_sequence;
-	return 0;
-}
-
-static __inline__ int
-investigate_hazard(uint32_t cmd, hazard_t hz, drm_via_state_t *cur_seq)
-{
-	register uint32_t tmp, *tmp_addr;
-
-	if (cur_seq->unfinished && (cur_seq->unfinished != seqs[hz])) {
-		int ret;
-		if ((ret = finish_current_sequence(cur_seq)))
-			return ret;
-	}
-
-	switch (hz) {
-	case check_for_header2:
-		if (cmd == HALCYON_HEADER2)
-			return 1;
-		return 0;
-	case check_for_header1:
-		if ((cmd & HALCYON_HEADER1MASK) == HALCYON_HEADER1)
-			return 1;
-		return 0;
-	case check_for_header2_err:
-		if (cmd == HALCYON_HEADER2)
-			return 1;
-		DRM_ERROR("Illegal DMA HALCYON_HEADER2 command\n");
-		break;
-	case check_for_header1_err:
-		if ((cmd & HALCYON_HEADER1MASK) == HALCYON_HEADER1)
-			return 1;
-		DRM_ERROR("Illegal DMA HALCYON_HEADER1 command\n");
-		break;
-	case check_for_fire:
-		if ((cmd & HALCYON_FIREMASK) == HALCYON_FIRECMD)
-			return 1;
-		DRM_ERROR("Illegal DMA HALCYON_FIRECMD command\n");
-		break;
-	case check_for_dummy:
-		if (HC_DUMMY == cmd)
-			return 0;
-		DRM_ERROR("Illegal DMA HC_DUMMY command\n");
-		break;
-	case check_for_dd:
-		if (0xdddddddd == cmd)
-			return 0;
-		DRM_ERROR("Illegal DMA 0xdddddddd command\n");
-		break;
-	case check_z_buffer_addr0:
-		cur_seq->unfinished = z_address;
-		cur_seq->z_addr = (cur_seq->z_addr & 0xFF000000) |
-		    (cmd & 0x00FFFFFF);
-		return 0;
-	case check_z_buffer_addr1:
-		cur_seq->unfinished = z_address;
-		cur_seq->z_addr = (cur_seq->z_addr & 0x00FFFFFF) |
-		    ((cmd & 0xFF) << 24);
-		return 0;
-	case check_z_buffer_addr_mode:
-		cur_seq->unfinished = z_address;
-		if ((cmd & 0x0000C000) == 0)
-			return 0;
-		DRM_ERROR("Attempt to place Z buffer in system memory\n");
-		return 2;
-	case check_destination_addr0:
-		cur_seq->unfinished = dest_address;
-		cur_seq->d_addr = (cur_seq->d_addr & 0xFF000000) |
-		    (cmd & 0x00FFFFFF);
-		return 0;
-	case check_destination_addr1:
-		cur_seq->unfinished = dest_address;
-		cur_seq->d_addr = (cur_seq->d_addr & 0x00FFFFFF) |
-		    ((cmd & 0xFF) << 24);
-		return 0;
-	case check_destination_addr_mode:
-		cur_seq->unfinished = dest_address;
-		if ((cmd & 0x0000C000) == 0)
-			return 0;
-		DRM_ERROR
-		    ("Attempt to place 3D drawing buffer in system memory\n");
-		return 2;
-	case check_texture_addr0:
-		cur_seq->unfinished = tex_address;
-		tmp = (cmd >> 24);
-		tmp_addr = &cur_seq->t_addr[cur_seq->texture][tmp];
-		*tmp_addr = (*tmp_addr & 0xFF000000) | (cmd & 0x00FFFFFF);
-		return 0;
-	case check_texture_addr1:
-		cur_seq->unfinished = tex_address;
-		tmp = ((cmd >> 24) - 0x20);
-		tmp += tmp << 1;
-		tmp_addr = &cur_seq->t_addr[cur_seq->texture][tmp];
-		*tmp_addr = (*tmp_addr & 0x00FFFFFF) | ((cmd & 0xFF) << 24);
-		tmp_addr++;
-		*tmp_addr = (*tmp_addr & 0x00FFFFFF) | ((cmd & 0xFF00) << 16);
-		tmp_addr++;
-		*tmp_addr = (*tmp_addr & 0x00FFFFFF) | ((cmd & 0xFF0000) << 8);
-		return 0;
-	case check_texture_addr2:
-		cur_seq->unfinished = tex_address;
-		cur_seq->tex_level_lo[tmp = cur_seq->texture] = cmd & 0x3F;
-		cur_seq->tex_level_hi[tmp] = (cmd & 0xFC0) >> 6;
-		return 0;
-	case check_texture_addr3:
-		cur_seq->unfinished = tex_address;
-		tmp = ((cmd >> 24) - HC_SubA_HTXnL0Pit);
-		if (tmp == 0 &&
-		    (cmd & HC_HTXnEnPit_MASK)) {
-			cur_seq->pitch[cur_seq->texture][tmp] =
-				(cmd & HC_HTXnLnPit_MASK);
-			cur_seq->tex_npot[cur_seq->texture] = 1;
-		} else {
-			cur_seq->pitch[cur_seq->texture][tmp] =
-				(cmd & HC_HTXnLnPitE_MASK) >> HC_HTXnLnPitE_SHIFT;
-			cur_seq->tex_npot[cur_seq->texture] = 0;
-			if (cmd & 0x000FFFFF) {
-				DRM_ERROR
-					("Unimplemented texture level 0 pitch mode.\n");
-				return 2;
-			}
-		}
-		return 0;
-	case check_texture_addr4:
-		cur_seq->unfinished = tex_address;
-		tmp_addr = &cur_seq->t_addr[cur_seq->texture][9];
-		*tmp_addr = (*tmp_addr & 0x00FFFFFF) | ((cmd & 0xFF) << 24);
-		return 0;
-	case check_texture_addr5:
-	case check_texture_addr6:
-		cur_seq->unfinished = tex_address;
-		/*
-		 * Texture width. We don't care since we have the pitch.
-		 */
-		return 0;
-	case check_texture_addr7:
-		cur_seq->unfinished = tex_address;
-		tmp_addr = &(cur_seq->height[cur_seq->texture][0]);
-		tmp_addr[5] = 1 << ((cmd & 0x00F00000) >> 20);
-		tmp_addr[4] = 1 << ((cmd & 0x000F0000) >> 16);
-		tmp_addr[3] = 1 << ((cmd & 0x0000F000) >> 12);
-		tmp_addr[2] = 1 << ((cmd & 0x00000F00) >> 8);
-		tmp_addr[1] = 1 << ((cmd & 0x000000F0) >> 4);
-		tmp_addr[0] = 1 << (cmd & 0x0000000F);
-		return 0;
-	case check_texture_addr8:
-		cur_seq->unfinished = tex_address;
-		tmp_addr = &(cur_seq->height[cur_seq->texture][0]);
-		tmp_addr[9] = 1 << ((cmd & 0x0000F000) >> 12);
-		tmp_addr[8] = 1 << ((cmd & 0x00000F00) >> 8);
-		tmp_addr[7] = 1 << ((cmd & 0x000000F0) >> 4);
-		tmp_addr[6] = 1 << (cmd & 0x0000000F);
-		return 0;
-	case check_texture_addr_mode:
-		cur_seq->unfinished = tex_address;
-		if (2 == (tmp = cmd & 0x00000003)) {
-			DRM_ERROR
-			    ("Attempt to fetch texture from system memory.\n");
-			return 2;
-		}
-		cur_seq->agp_texture = (tmp == 3);
-		cur_seq->tex_palette_size[cur_seq->texture] =
-		    (cmd >> 16) & 0x000000007;
-		return 0;
-	case check_for_vertex_count:
-		cur_seq->vertex_count = cmd & 0x0000FFFF;
-		return 0;
-	case check_number_texunits:
-		cur_seq->multitex = (cmd >> 3) & 1;
-		return 0;
-	default:
-		DRM_ERROR("Illegal DMA data: 0x%x\n", cmd);
-		return 2;
-	}
-	return 2;
-}
-
-static __inline__ int
-via_check_prim_list(uint32_t const **buffer, const uint32_t * buf_end,
-		    drm_via_state_t *cur_seq)
-{
-	drm_via_private_t *dev_priv =
-	    (drm_via_private_t *) cur_seq->dev->dev_private;
-	uint32_t a_fire, bcmd, dw_count;
-	int ret = 0;
-	int have_fire;
-	const uint32_t *buf = *buffer;
-
-	while (buf < buf_end) {
-		have_fire = 0;
-		if ((buf_end - buf) < 2) {
-			DRM_ERROR
-			    ("Unexpected termination of primitive list.\n");
-			ret = 1;
-			break;
-		}
-		if ((*buf & HC_ACMD_MASK) != HC_ACMD_HCmdB)
-			break;
-		bcmd = *buf++;
-		if ((*buf & HC_ACMD_MASK) != HC_ACMD_HCmdA) {
-			DRM_ERROR("Expected Vertex List A command, got 0x%x\n",
-				  *buf);
-			ret = 1;
-			break;
-		}
-		a_fire =
-		    *buf++ | HC_HPLEND_MASK | HC_HPMValidN_MASK |
-		    HC_HE3Fire_MASK;
-
-		/*
-		 * How many dwords per vertex ?
-		 */
-
-		if (cur_seq->agp && ((bcmd & (0xF << 11)) == 0)) {
-			DRM_ERROR("Illegal B command vertex data for AGP.\n");
-			ret = 1;
-			break;
-		}
-
-		dw_count = 0;
-		if (bcmd & (1 << 7))
-			dw_count += (cur_seq->multitex) ? 2 : 1;
-		if (bcmd & (1 << 8))
-			dw_count += (cur_seq->multitex) ? 2 : 1;
-		if (bcmd & (1 << 9))
-			dw_count++;
-		if (bcmd & (1 << 10))
-			dw_count++;
-		if (bcmd & (1 << 11))
-			dw_count++;
-		if (bcmd & (1 << 12))
-			dw_count++;
-		if (bcmd & (1 << 13))
-			dw_count++;
-		if (bcmd & (1 << 14))
-			dw_count++;
-
-		while (buf < buf_end) {
-			if (*buf == a_fire) {
-				if (dev_priv->num_fire_offsets >=
-				    VIA_FIRE_BUF_SIZE) {
-					DRM_ERROR("Fire offset buffer full.\n");
-					ret = 1;
-					break;
-				}
-				dev_priv->fire_offsets[dev_priv->
-						       num_fire_offsets++] =
-				    buf;
-				have_fire = 1;
-				buf++;
-				if (buf < buf_end && *buf == a_fire)
-					buf++;
-				break;
-			}
-			if ((*buf == HALCYON_HEADER2) ||
-			    ((*buf & HALCYON_FIREMASK) == HALCYON_FIRECMD)) {
-				DRM_ERROR("Missing Vertex Fire command, "
-					  "Stray Vertex Fire command  or verifier "
-					  "lost sync.\n");
-				ret = 1;
-				break;
-			}
-			if ((ret = eat_words(&buf, buf_end, dw_count)))
-				break;
-		}
-		if (buf >= buf_end && !have_fire) {
-			DRM_ERROR("Missing Vertex Fire command or verifier "
-				  "lost sync.\n");
-			ret = 1;
-			break;
-		}
-		if (cur_seq->agp && ((buf - cur_seq->buf_start) & 0x01)) {
-			DRM_ERROR("AGP Primitive list end misaligned.\n");
-			ret = 1;
-			break;
-		}
-	}
-	*buffer = buf;
-	return ret;
-}
-
-static __inline__ verifier_state_t
-via_check_header2(uint32_t const **buffer, const uint32_t *buf_end,
-		  drm_via_state_t *hc_state)
-{
-	uint32_t cmd;
-	int hz_mode;
-	hazard_t hz;
-	const uint32_t *buf = *buffer;
-	const hazard_t *hz_table;
-
-	if ((buf_end - buf) < 2) {
-		DRM_ERROR
-		    ("Illegal termination of DMA HALCYON_HEADER2 sequence.\n");
-		return state_error;
-	}
-	buf++;
-	cmd = (*buf++ & 0xFFFF0000) >> 16;
-
-	switch (cmd) {
-	case HC_ParaType_CmdVdata:
-		if (via_check_prim_list(&buf, buf_end, hc_state))
-			return state_error;
-		*buffer = buf;
-		return state_command;
-	case HC_ParaType_NotTex:
-		hz_table = table1;
-		break;
-	case HC_ParaType_Tex:
-		hc_state->texture = 0;
-		hz_table = table2;
-		break;
-	case (HC_ParaType_Tex | (HC_SubType_Tex1 << 8)):
-		hc_state->texture = 1;
-		hz_table = table2;
-		break;
-	case (HC_ParaType_Tex | (HC_SubType_TexGeneral << 8)):
-		hz_table = table3;
-		break;
-	case HC_ParaType_Auto:
-		if (eat_words(&buf, buf_end, 2))
-			return state_error;
-		*buffer = buf;
-		return state_command;
-	case (HC_ParaType_Palette | (HC_SubType_Stipple << 8)):
-		if (eat_words(&buf, buf_end, 32))
-			return state_error;
-		*buffer = buf;
-		return state_command;
-	case (HC_ParaType_Palette | (HC_SubType_TexPalette0 << 8)):
-	case (HC_ParaType_Palette | (HC_SubType_TexPalette1 << 8)):
-		DRM_ERROR("Texture palettes are rejected because of "
-			  "lack of info how to determine their size.\n");
-		return state_error;
-	case (HC_ParaType_Palette | (HC_SubType_FogTable << 8)):
-		DRM_ERROR("Fog factor palettes are rejected because of "
-			  "lack of info how to determine their size.\n");
-		return state_error;
-	default:
-
-		/*
-		 * There are some unimplemented HC_ParaTypes here, that
-		 * need to be implemented if the Mesa driver is extended.
-		 */
-
-		DRM_ERROR("Invalid or unimplemented HALCYON_HEADER2 "
-			  "DMA subcommand: 0x%x. Previous dword: 0x%x\n",
-			  cmd, *(buf - 2));
-		*buffer = buf;
-		return state_error;
-	}
-
-	while (buf < buf_end) {
-		cmd = *buf++;
-		if ((hz = hz_table[cmd >> 24])) {
-			if ((hz_mode = investigate_hazard(cmd, hz, hc_state))) {
-				if (hz_mode == 1) {
-					buf--;
-					break;
-				}
-				return state_error;
-			}
-		} else if (hc_state->unfinished &&
-			   finish_current_sequence(hc_state)) {
-			return state_error;
-		}
-	}
-	if (hc_state->unfinished && finish_current_sequence(hc_state))
-		return state_error;
-	*buffer = buf;
-	return state_command;
-}
-
-static __inline__ verifier_state_t
-via_parse_header2(drm_via_private_t *dev_priv, uint32_t const **buffer,
-		  const uint32_t *buf_end, int *fire_count)
-{
-	uint32_t cmd;
-	const uint32_t *buf = *buffer;
-	const uint32_t *next_fire;
-	int burst = 0;
-
-	next_fire = dev_priv->fire_offsets[*fire_count];
-	buf++;
-	cmd = (*buf & 0xFFFF0000) >> 16;
-	via_write(dev_priv, HC_REG_TRANS_SET + HC_REG_BASE, *buf++);
-	switch (cmd) {
-	case HC_ParaType_CmdVdata:
-		while ((buf < buf_end) &&
-		       (*fire_count < dev_priv->num_fire_offsets) &&
-		       (*buf & HC_ACMD_MASK) == HC_ACMD_HCmdB) {
-			while (buf <= next_fire) {
-				via_write(dev_priv, HC_REG_TRANS_SPACE + HC_REG_BASE +
-					  (burst & 63), *buf++);
-				burst += 4;
-			}
-			if ((buf < buf_end)
-			    && ((*buf & HALCYON_FIREMASK) == HALCYON_FIRECMD))
-				buf++;
-
-			if (++(*fire_count) < dev_priv->num_fire_offsets)
-				next_fire = dev_priv->fire_offsets[*fire_count];
-		}
-		break;
-	default:
-		while (buf < buf_end) {
-
-			if (*buf == HC_HEADER2 ||
-			    (*buf & HALCYON_HEADER1MASK) == HALCYON_HEADER1 ||
-			    (*buf & VIA_VIDEOMASK) == VIA_VIDEO_HEADER5 ||
-			    (*buf & VIA_VIDEOMASK) == VIA_VIDEO_HEADER6)
-				break;
-
-			via_write(dev_priv, HC_REG_TRANS_SPACE + HC_REG_BASE +
-				  (burst & 63), *buf++);
-			burst += 4;
-		}
-	}
-	*buffer = buf;
-	return state_command;
-}
-
-static __inline__ int verify_mmio_address(uint32_t address)
-{
-	if ((address > 0x3FF) && (address < 0xC00)) {
-		DRM_ERROR("Invalid VIDEO DMA command. "
-			  "Attempt to access 3D- or command burst area.\n");
-		return 1;
-	} else if ((address > 0xCFF) && (address < 0x1300)) {
-		DRM_ERROR("Invalid VIDEO DMA command. "
-			  "Attempt to access PCI DMA area.\n");
-		return 1;
-	} else if (address > 0x13FF) {
-		DRM_ERROR("Invalid VIDEO DMA command. "
-			  "Attempt to access VGA registers.\n");
-		return 1;
-	}
-	return 0;
-}
-
-static __inline__ int
-verify_video_tail(uint32_t const **buffer, const uint32_t * buf_end,
-		  uint32_t dwords)
-{
-	const uint32_t *buf = *buffer;
-
-	if (buf_end - buf < dwords) {
-		DRM_ERROR("Illegal termination of video command.\n");
-		return 1;
-	}
-	while (dwords--) {
-		if (*buf++) {
-			DRM_ERROR("Illegal video command tail.\n");
-			return 1;
-		}
-	}
-	*buffer = buf;
-	return 0;
-}
-
-static __inline__ verifier_state_t
-via_check_header1(uint32_t const **buffer, const uint32_t * buf_end)
-{
-	uint32_t cmd;
-	const uint32_t *buf = *buffer;
-	verifier_state_t ret = state_command;
-
-	while (buf < buf_end) {
-		cmd = *buf;
-		if ((cmd > ((0x3FF >> 2) | HALCYON_HEADER1)) &&
-		    (cmd < ((0xC00 >> 2) | HALCYON_HEADER1))) {
-			if ((cmd & HALCYON_HEADER1MASK) != HALCYON_HEADER1)
-				break;
-			DRM_ERROR("Invalid HALCYON_HEADER1 command. "
-				  "Attempt to access 3D- or command burst area.\n");
-			ret = state_error;
-			break;
-		} else if (cmd > ((0xCFF >> 2) | HALCYON_HEADER1)) {
-			if ((cmd & HALCYON_HEADER1MASK) != HALCYON_HEADER1)
-				break;
-			DRM_ERROR("Invalid HALCYON_HEADER1 command. "
-				  "Attempt to access VGA registers.\n");
-			ret = state_error;
-			break;
-		} else {
-			buf += 2;
-		}
-	}
-	*buffer = buf;
-	return ret;
-}
-
-static __inline__ verifier_state_t
-via_parse_header1(drm_via_private_t *dev_priv, uint32_t const **buffer,
-		  const uint32_t *buf_end)
-{
-	register uint32_t cmd;
-	const uint32_t *buf = *buffer;
-
-	while (buf < buf_end) {
-		cmd = *buf;
-		if ((cmd & HALCYON_HEADER1MASK) != HALCYON_HEADER1)
-			break;
-		via_write(dev_priv, (cmd & ~HALCYON_HEADER1MASK) << 2, *++buf);
-		buf++;
-	}
-	*buffer = buf;
-	return state_command;
-}
-
-static __inline__ verifier_state_t
-via_check_vheader5(uint32_t const **buffer, const uint32_t *buf_end)
-{
-	uint32_t data;
-	const uint32_t *buf = *buffer;
-
-	if (buf_end - buf < 4) {
-		DRM_ERROR("Illegal termination of video header5 command\n");
-		return state_error;
-	}
-
-	data = *buf++ & ~VIA_VIDEOMASK;
-	if (verify_mmio_address(data))
-		return state_error;
-
-	data = *buf++;
-	if (*buf++ != 0x00F50000) {
-		DRM_ERROR("Illegal header5 header data\n");
-		return state_error;
-	}
-	if (*buf++ != 0x00000000) {
-		DRM_ERROR("Illegal header5 header data\n");
-		return state_error;
-	}
-	if (eat_words(&buf, buf_end, data))
-		return state_error;
-	if ((data & 3) && verify_video_tail(&buf, buf_end, 4 - (data & 3)))
-		return state_error;
-	*buffer = buf;
-	return state_command;
-
-}
-
-static __inline__ verifier_state_t
-via_parse_vheader5(drm_via_private_t *dev_priv, uint32_t const **buffer,
-		   const uint32_t *buf_end)
-{
-	uint32_t addr, count, i;
-	const uint32_t *buf = *buffer;
-
-	addr = *buf++ & ~VIA_VIDEOMASK;
-	i = count = *buf;
-	buf += 3;
-	while (i--)
-		via_write(dev_priv, addr, *buf++);
-	if (count & 3)
-		buf += 4 - (count & 3);
-	*buffer = buf;
-	return state_command;
-}
-
-static __inline__ verifier_state_t
-via_check_vheader6(uint32_t const **buffer, const uint32_t * buf_end)
-{
-	uint32_t data;
-	const uint32_t *buf = *buffer;
-	uint32_t i;
-
-	if (buf_end - buf < 4) {
-		DRM_ERROR("Illegal termination of video header6 command\n");
-		return state_error;
-	}
-	buf++;
-	data = *buf++;
-	if (*buf++ != 0x00F60000) {
-		DRM_ERROR("Illegal header6 header data\n");
-		return state_error;
-	}
-	if (*buf++ != 0x00000000) {
-		DRM_ERROR("Illegal header6 header data\n");
-		return state_error;
-	}
-	if ((buf_end - buf) < (data << 1)) {
-		DRM_ERROR("Illegal termination of video header6 command\n");
-		return state_error;
-	}
-	for (i = 0; i < data; ++i) {
-		if (verify_mmio_address(*buf++))
-			return state_error;
-		buf++;
-	}
-	data <<= 1;
-	if ((data & 3) && verify_video_tail(&buf, buf_end, 4 - (data & 3)))
-		return state_error;
-	*buffer = buf;
-	return state_command;
-}
-
-static __inline__ verifier_state_t
-via_parse_vheader6(drm_via_private_t *dev_priv, uint32_t const **buffer,
-		   const uint32_t *buf_end)
-{
-
-	uint32_t addr, count, i;
-	const uint32_t *buf = *buffer;
-
-	i = count = *++buf;
-	buf += 3;
-	while (i--) {
-		addr = *buf++;
-		via_write(dev_priv, addr, *buf++);
-	}
-	count <<= 1;
-	if (count & 3)
-		buf += 4 - (count & 3);
-	*buffer = buf;
-	return state_command;
-}
-
-int
-via_verify_command_stream(const uint32_t * buf, unsigned int size,
-			  struct drm_device * dev, int agp)
-{
-
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	drm_via_state_t *hc_state = &dev_priv->hc_state;
-	drm_via_state_t saved_state = *hc_state;
-	uint32_t cmd;
-	const uint32_t *buf_end = buf + (size >> 2);
-	verifier_state_t state = state_command;
-	int cme_video;
-	int supported_3d;
-
-	cme_video = (dev_priv->chipset == VIA_PRO_GROUP_A ||
-		     dev_priv->chipset == VIA_DX9_0);
-
-	supported_3d = dev_priv->chipset != VIA_DX9_0;
-
-	hc_state->dev = dev;
-	hc_state->unfinished = no_sequence;
-	hc_state->map_cache = NULL;
-	hc_state->agp = agp;
-	hc_state->buf_start = buf;
-	dev_priv->num_fire_offsets = 0;
-
-	while (buf < buf_end) {
-
-		switch (state) {
-		case state_header2:
-			state = via_check_header2(&buf, buf_end, hc_state);
-			break;
-		case state_header1:
-			state = via_check_header1(&buf, buf_end);
-			break;
-		case state_vheader5:
-			state = via_check_vheader5(&buf, buf_end);
-			break;
-		case state_vheader6:
-			state = via_check_vheader6(&buf, buf_end);
-			break;
-		case state_command:
-			cmd = *buf;
-			if ((cmd == HALCYON_HEADER2) && supported_3d)
-				state = state_header2;
-			else if ((cmd & HALCYON_HEADER1MASK) == HALCYON_HEADER1)
-				state = state_header1;
-			else if (cme_video
-				 && (cmd & VIA_VIDEOMASK) == VIA_VIDEO_HEADER5)
-				state = state_vheader5;
-			else if (cme_video
-				 && (cmd & VIA_VIDEOMASK) == VIA_VIDEO_HEADER6)
-				state = state_vheader6;
-			else if ((cmd == HALCYON_HEADER2) && !supported_3d) {
-				DRM_ERROR("Accelerated 3D is not supported on this chipset yet.\n");
-				state = state_error;
-			} else {
-				DRM_ERROR
-				    ("Invalid / Unimplemented DMA HEADER command. 0x%x\n",
-				     cmd);
-				state = state_error;
-			}
-			break;
-		case state_error:
-		default:
-			*hc_state = saved_state;
-			return -EINVAL;
-		}
-	}
-	if (state == state_error) {
-		*hc_state = saved_state;
-		return -EINVAL;
-	}
-	return 0;
-}
-
-int
-via_parse_command_stream(struct drm_device *dev, const uint32_t *buf,
-			 unsigned int size)
-{
-
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	uint32_t cmd;
-	const uint32_t *buf_end = buf + (size >> 2);
-	verifier_state_t state = state_command;
-	int fire_count = 0;
-
-	while (buf < buf_end) {
-
-		switch (state) {
-		case state_header2:
-			state =
-			    via_parse_header2(dev_priv, &buf, buf_end,
-					      &fire_count);
-			break;
-		case state_header1:
-			state = via_parse_header1(dev_priv, &buf, buf_end);
-			break;
-		case state_vheader5:
-			state = via_parse_vheader5(dev_priv, &buf, buf_end);
-			break;
-		case state_vheader6:
-			state = via_parse_vheader6(dev_priv, &buf, buf_end);
-			break;
-		case state_command:
-			cmd = *buf;
-			if (cmd == HALCYON_HEADER2)
-				state = state_header2;
-			else if ((cmd & HALCYON_HEADER1MASK) == HALCYON_HEADER1)
-				state = state_header1;
-			else if ((cmd & VIA_VIDEOMASK) == VIA_VIDEO_HEADER5)
-				state = state_vheader5;
-			else if ((cmd & VIA_VIDEOMASK) == VIA_VIDEO_HEADER6)
-				state = state_vheader6;
-			else {
-				DRM_ERROR
-				    ("Invalid / Unimplemented DMA HEADER command. 0x%x\n",
-				     cmd);
-				state = state_error;
-			}
-			break;
-		case state_error:
-		default:
-			return -EINVAL;
-		}
-	}
-	if (state == state_error)
-		return -EINVAL;
-	return 0;
-}
-
-static void
-setup_hazard_table(hz_init_t init_table[], hazard_t table[], int size)
-{
-	int i;
-
-	for (i = 0; i < 256; ++i)
-		table[i] = forbidden_command;
-
-	for (i = 0; i < size; ++i)
-		table[init_table[i].code] = init_table[i].hz;
-}
-
-void via_init_command_verifier(void)
-{
-	setup_hazard_table(init_table1, table1, ARRAY_SIZE(init_table1));
-	setup_hazard_table(init_table2, table2, ARRAY_SIZE(init_table2));
-	setup_hazard_table(init_table3, table3, ARRAY_SIZE(init_table3));
-}
diff --git a/drivers/gpu/drm/via/via_verifier.h b/drivers/gpu/drm/via/via_verifier.h
deleted file mode 100644
index 26b6d361ab95..000000000000
--- a/drivers/gpu/drm/via/via_verifier.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2004 The Unichrome Project. All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE UNICHROME PROJECT, AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Author: Thomas Hellström 2004.
- */
-
-#ifndef _VIA_VERIFIER_H_
-#define _VIA_VERIFIER_H_
-
-typedef enum {
-	no_sequence = 0,
-	z_address,
-	dest_address,
-	tex_address
-} drm_via_sequence_t;
-
-typedef struct {
-	unsigned texture;
-	uint32_t z_addr;
-	uint32_t d_addr;
-	uint32_t t_addr[2][10];
-	uint32_t pitch[2][10];
-	uint32_t height[2][10];
-	uint32_t tex_level_lo[2];
-	uint32_t tex_level_hi[2];
-	uint32_t tex_palette_size[2];
-	uint32_t tex_npot[2];
-	drm_via_sequence_t unfinished;
-	int agp_texture;
-	int multitex;
-	struct drm_device *dev;
-	drm_local_map_t *map_cache;
-	uint32_t vertex_count;
-	int agp;
-	const uint32_t *buf_start;
-} drm_via_state_t;
-
-extern int via_verify_command_stream(const uint32_t *buf, unsigned int size,
-				     struct drm_device *dev, int agp);
-extern int via_parse_command_stream(struct drm_device *dev, const uint32_t *buf,
-				    unsigned int size);
-
-#endif
diff --git a/drivers/gpu/drm/via/via_video.c b/drivers/gpu/drm/via/via_video.c
deleted file mode 100644
index 53b1f58f99b4..000000000000
--- a/drivers/gpu/drm/via/via_video.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2005 Thomas Hellstrom. All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sub license,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHOR(S), AND/OR THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Author: Thomas Hellstrom 2005.
- *
- * Video and XvMC related functions.
- */
-
-#include <drm/drm_device.h>
-#include <drm/via_drm.h>
-
-#include "via_drv.h"
-
-void via_init_futex(drm_via_private_t *dev_priv)
-{
-	unsigned int i;
-
-	DRM_DEBUG("\n");
-
-	for (i = 0; i < VIA_NR_XVMC_LOCKS; ++i) {
-		init_waitqueue_head(&(dev_priv->decoder_queue[i]));
-		XVMCLOCKPTR(dev_priv->sarea_priv, i)->lock = 0;
-	}
-}
-
-void via_cleanup_futex(drm_via_private_t *dev_priv)
-{
-}
-
-void via_release_futex(drm_via_private_t *dev_priv, int context)
-{
-	unsigned int i;
-	volatile int *lock;
-
-	if (!dev_priv->sarea_priv)
-		return;
-
-	for (i = 0; i < VIA_NR_XVMC_LOCKS; ++i) {
-		lock = (volatile int *)XVMCLOCKPTR(dev_priv->sarea_priv, i);
-		if ((_DRM_LOCKING_CONTEXT(*lock) == context)) {
-			if (_DRM_LOCK_IS_HELD(*lock)
-			    && (*lock & _DRM_LOCK_CONT)) {
-				wake_up(&(dev_priv->decoder_queue[i]));
-			}
-			*lock = 0;
-		}
-	}
-}
-
-int via_decoder_futex(struct drm_device *dev, void *data, struct drm_file *file_priv)
-{
-	drm_via_futex_t *fx = data;
-	volatile int *lock;
-	drm_via_private_t *dev_priv = (drm_via_private_t *) dev->dev_private;
-	drm_via_sarea_t *sAPriv = dev_priv->sarea_priv;
-	int ret = 0;
-
-	DRM_DEBUG("\n");
-
-	if (fx->lock >= VIA_NR_XVMC_LOCKS)
-		return -EFAULT;
-
-	lock = (volatile int *)XVMCLOCKPTR(sAPriv, fx->lock);
-
-	switch (fx->func) {
-	case VIA_FUTEX_WAIT:
-		VIA_WAIT_ON(ret, dev_priv->decoder_queue[fx->lock],
-			    (fx->ms / 10) * (HZ / 100), *lock != fx->val);
-		return ret;
-	case VIA_FUTEX_WAKE:
-		wake_up(&(dev_priv->decoder_queue[fx->lock]));
-		return 0;
-	}
-	return 0;
-}
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index 5c7f198c0712..9ea7611a9e0f 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -349,6 +349,8 @@ int virtio_gpu_modeset_init(struct virtio_gpu_device *vgdev)
 	vgdev->ddev->mode_config.max_width = XRES_MAX;
 	vgdev->ddev->mode_config.max_height = YRES_MAX;
 
+	vgdev->ddev->mode_config.fb_modifiers_not_supported = true;
+
 	for (i = 0 ; i < vgdev->num_scanouts; ++i)
 		vgdev_output_init(vgdev, i);
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c
index 5f25a8d15464..0035affc3e59 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.c
@@ -46,12 +46,11 @@ static int virtio_gpu_modeset = -1;
 MODULE_PARM_DESC(modeset, "Disable/Enable modesetting");
 module_param_named(modeset, virtio_gpu_modeset, int, 0400);
 
-static int virtio_gpu_pci_quirk(struct drm_device *dev, struct virtio_device *vdev)
+static int virtio_gpu_pci_quirk(struct drm_device *dev)
 {
-	struct pci_dev *pdev = to_pci_dev(vdev->dev.parent);
+	struct pci_dev *pdev = to_pci_dev(dev->dev);
 	const char *pname = dev_name(&pdev->dev);
 	bool vga = (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
-	char unique[20];
 	int ret;
 
 	DRM_INFO("pci: %s detected at %s\n",
@@ -63,39 +62,7 @@ static int virtio_gpu_pci_quirk(struct drm_device *dev, struct virtio_device *vd
 			return ret;
 	}
 
-	/*
-	 * Normally the drm_dev_set_unique() call is done by core DRM.
-	 * The following comment covers, why virtio cannot rely on it.
-	 *
-	 * Unlike the other virtual GPU drivers, virtio abstracts the
-	 * underlying bus type by using struct virtio_device.
-	 *
-	 * Hence the dev_is_pci() check, used in core DRM, will fail
-	 * and the unique returned will be the virtio_device "virtio0",
-	 * while a "pci:..." one is required.
-	 *
-	 * A few other ideas were considered:
-	 * - Extend the dev_is_pci() check [in drm_set_busid] to
-	 *   consider virtio.
-	 *   Seems like a bigger hack than what we have already.
-	 *
-	 * - Point drm_device::dev to the parent of the virtio_device
-	 *   Semantic changes:
-	 *   * Using the wrong device for i2c, framebuffer_alloc and
-	 *     prime import.
-	 *   Visual changes:
-	 *   * Helpers such as DRM_DEV_ERROR, dev_info, drm_printer,
-	 *     will print the wrong information.
-	 *
-	 * We could address the latter issues, by introducing
-	 * drm_device::bus_dev, ... which would be used solely for this.
-	 *
-	 * So for the moment keep things as-is, with a bulky comment
-	 * for the next person who feels like removing this
-	 * drm_dev_set_unique() quirk.
-	 */
-	snprintf(unique, sizeof(unique), "pci:%s", pname);
-	return drm_dev_set_unique(dev, unique);
+	return 0;
 }
 
 static int virtio_gpu_probe(struct virtio_device *vdev)
@@ -109,18 +76,24 @@ static int virtio_gpu_probe(struct virtio_device *vdev)
 	if (virtio_gpu_modeset == 0)
 		return -EINVAL;
 
-	dev = drm_dev_alloc(&driver, &vdev->dev);
+	/*
+	 * The virtio-gpu device is a virtual device that doesn't have DMA
+	 * ops assigned to it, nor DMA mask set and etc. Its parent device
+	 * is actual GPU device we want to use it for the DRM's device in
+	 * order to benefit from using generic DRM APIs.
+	 */
+	dev = drm_dev_alloc(&driver, vdev->dev.parent);
 	if (IS_ERR(dev))
 		return PTR_ERR(dev);
 	vdev->priv = dev;
 
-	if (!strcmp(vdev->dev.parent->bus->name, "pci")) {
-		ret = virtio_gpu_pci_quirk(dev, vdev);
+	if (dev_is_pci(vdev->dev.parent)) {
+		ret = virtio_gpu_pci_quirk(dev);
 		if (ret)
 			goto err_free;
 	}
 
-	ret = virtio_gpu_init(dev);
+	ret = virtio_gpu_init(vdev, dev);
 	if (ret)
 		goto err_free;
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index f80664cf98d0..9b98470593b0 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -101,8 +101,6 @@ struct virtio_gpu_object {
 
 struct virtio_gpu_object_shmem {
 	struct virtio_gpu_object base;
-	struct sg_table *pages;
-	uint32_t mapped;
 };
 
 struct virtio_gpu_object_vram {
@@ -215,7 +213,6 @@ struct virtio_gpu_drv_cap_cache {
 };
 
 struct virtio_gpu_device {
-	struct device *dev;
 	struct drm_device *ddev;
 
 	struct virtio_device *vdev;
@@ -283,7 +280,7 @@ extern struct drm_ioctl_desc virtio_gpu_ioctls[DRM_VIRTIO_NUM_IOCTLS];
 void virtio_gpu_create_context(struct drm_device *dev, struct drm_file *file);
 
 /* virtgpu_kms.c */
-int virtio_gpu_init(struct drm_device *dev);
+int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev);
 void virtio_gpu_deinit(struct drm_device *dev);
 void virtio_gpu_release(struct drm_device *dev);
 int virtio_gpu_driver_open(struct drm_device *dev, struct drm_file *file);
diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c
index 580a78809836..7db48d17ee3a 100644
--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
+++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
@@ -228,8 +228,10 @@ int virtio_gpu_array_lock_resv(struct virtio_gpu_object_array *objs)
 
 	for (i = 0; i < objs->nents; ++i) {
 		ret = dma_resv_reserve_fences(objs->objs[i]->resv, 1);
-		if (ret)
+		if (ret) {
+			virtio_gpu_array_unlock_resv(objs);
 			return ret;
+		}
 	}
 	return ret;
 }
diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
index 9b2702116f93..5d05093014ac 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
@@ -47,7 +47,7 @@ static int virtio_gpu_fence_event_create(struct drm_device *dev,
 	struct virtio_gpu_fence_event *e = NULL;
 	int ret;
 
-	if (!(vfpriv->ring_idx_mask & (1 << ring_idx)))
+	if (!(vfpriv->ring_idx_mask & BIT_ULL(ring_idx)))
 		return 0;
 
 	e = kzalloc(sizeof(*e), GFP_KERNEL);
@@ -168,7 +168,7 @@ static int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
 		 * array contains any fence from a foreign context.
 		 */
 		ret = 0;
-		if (!dma_fence_match_context(in_fence, vgdev->fence_drv.context))
+		if (!dma_fence_match_context(in_fence, fence_ctx + ring_idx))
 			ret = dma_fence_wait(in_fence, true);
 
 		dma_fence_put(in_fence);
diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c
index 3313b92db531..27b7f14dae89 100644
--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
+++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
@@ -28,6 +28,7 @@
 #include <linux/virtio_ring.h>
 
 #include <drm/drm_file.h>
+#include <drm/drm_managed.h>
 
 #include "virtgpu_drv.h"
 
@@ -66,10 +67,11 @@ static void virtio_gpu_get_capsets(struct virtio_gpu_device *vgdev,
 {
 	int i, ret;
 	bool invalid_capset_id = false;
+	struct drm_device *drm = vgdev->ddev;
 
-	vgdev->capsets = kcalloc(num_capsets,
-				 sizeof(struct virtio_gpu_drv_capset),
-				 GFP_KERNEL);
+	vgdev->capsets = drmm_kcalloc(drm, num_capsets,
+				      sizeof(struct virtio_gpu_drv_capset),
+				      GFP_KERNEL);
 	if (!vgdev->capsets) {
 		DRM_ERROR("failed to allocate cap sets\n");
 		return;
@@ -94,7 +96,7 @@ static void virtio_gpu_get_capsets(struct virtio_gpu_device *vgdev,
 
 		if (ret == 0 || invalid_capset_id) {
 			spin_lock(&vgdev->display_info_lock);
-			kfree(vgdev->capsets);
+			drmm_kfree(drm, vgdev->capsets);
 			vgdev->capsets = NULL;
 			spin_unlock(&vgdev->display_info_lock);
 			return;
@@ -110,7 +112,7 @@ static void virtio_gpu_get_capsets(struct virtio_gpu_device *vgdev,
 	vgdev->num_capsets = num_capsets;
 }
 
-int virtio_gpu_init(struct drm_device *dev)
+int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
 {
 	static vq_callback_t *callbacks[] = {
 		virtio_gpu_ctrl_ack, virtio_gpu_cursor_ack
@@ -123,17 +125,16 @@ int virtio_gpu_init(struct drm_device *dev)
 	u32 num_scanouts, num_capsets;
 	int ret = 0;
 
-	if (!virtio_has_feature(dev_to_virtio(dev->dev), VIRTIO_F_VERSION_1))
+	if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
 		return -ENODEV;
 
-	vgdev = kzalloc(sizeof(struct virtio_gpu_device), GFP_KERNEL);
+	vgdev = drmm_kzalloc(dev, sizeof(struct virtio_gpu_device), GFP_KERNEL);
 	if (!vgdev)
 		return -ENOMEM;
 
 	vgdev->ddev = dev;
 	dev->dev_private = vgdev;
-	vgdev->vdev = dev_to_virtio(dev->dev);
-	vgdev->dev = dev->dev;
+	vgdev->vdev = vdev;
 
 	spin_lock_init(&vgdev->display_info_lock);
 	spin_lock_init(&vgdev->resource_export_lock);
@@ -257,7 +258,6 @@ err_vbufs:
 	vgdev->vdev->config->del_vqs(vgdev->vdev);
 err_vqs:
 	dev->dev_private = NULL;
-	kfree(vgdev);
 	return ret;
 }
 
@@ -296,9 +296,6 @@ void virtio_gpu_release(struct drm_device *dev)
 
 	if (vgdev->has_host_visible)
 		drm_mm_takedown(&vgdev->host_visible_mm);
-
-	kfree(vgdev->capsets);
-	kfree(vgdev);
 }
 
 int virtio_gpu_driver_open(struct drm_device *dev, struct drm_file *file)
diff --git a/drivers/gpu/drm/virtio/virtgpu_object.c b/drivers/gpu/drm/virtio/virtgpu_object.c
index 1cc8f3fc8e4b..8d7728181de0 100644
--- a/drivers/gpu/drm/virtio/virtgpu_object.c
+++ b/drivers/gpu/drm/virtio/virtgpu_object.c
@@ -67,21 +67,6 @@ void virtio_gpu_cleanup_object(struct virtio_gpu_object *bo)
 
 	virtio_gpu_resource_id_put(vgdev, bo->hw_res_handle);
 	if (virtio_gpu_is_shmem(bo)) {
-		struct virtio_gpu_object_shmem *shmem = to_virtio_gpu_shmem(bo);
-
-		if (shmem->pages) {
-			if (shmem->mapped) {
-				dma_unmap_sgtable(vgdev->vdev->dev.parent,
-					     shmem->pages, DMA_TO_DEVICE, 0);
-				shmem->mapped = 0;
-			}
-
-			sg_free_table(shmem->pages);
-			kfree(shmem->pages);
-			shmem->pages = NULL;
-			drm_gem_shmem_unpin(&bo->base);
-		}
-
 		drm_gem_shmem_free(&bo->base);
 	} else if (virtio_gpu_is_vram(bo)) {
 		struct virtio_gpu_object_vram *vram = to_virtio_gpu_vram(bo);
@@ -153,35 +138,18 @@ static int virtio_gpu_object_shmem_init(struct virtio_gpu_device *vgdev,
 					unsigned int *nents)
 {
 	bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev);
-	struct virtio_gpu_object_shmem *shmem = to_virtio_gpu_shmem(bo);
 	struct scatterlist *sg;
-	int si, ret;
+	struct sg_table *pages;
+	int si;
 
-	ret = drm_gem_shmem_pin(&bo->base);
-	if (ret < 0)
-		return -EINVAL;
-
-	/*
-	 * virtio_gpu uses drm_gem_shmem_get_sg_table instead of
-	 * drm_gem_shmem_get_pages_sgt because virtio has it's own set of
-	 * dma-ops. This is discouraged for other drivers, but should be fine
-	 * since virtio_gpu doesn't support dma-buf import from other devices.
-	 */
-	shmem->pages = drm_gem_shmem_get_sg_table(&bo->base);
-	if (IS_ERR(shmem->pages)) {
-		drm_gem_shmem_unpin(&bo->base);
-		return PTR_ERR(shmem->pages);
-	}
+	pages = drm_gem_shmem_get_pages_sgt(&bo->base);
+	if (IS_ERR(pages))
+		return PTR_ERR(pages);
 
-	if (use_dma_api) {
-		ret = dma_map_sgtable(vgdev->vdev->dev.parent,
-				      shmem->pages, DMA_TO_DEVICE, 0);
-		if (ret)
-			return ret;
-		*nents = shmem->mapped = shmem->pages->nents;
-	} else {
-		*nents = shmem->pages->orig_nents;
-	}
+	if (use_dma_api)
+		*nents = pages->nents;
+	else
+		*nents = pages->orig_nents;
 
 	*ents = kvmalloc_array(*nents,
 			       sizeof(struct virtio_gpu_mem_entry),
@@ -192,13 +160,13 @@ static int virtio_gpu_object_shmem_init(struct virtio_gpu_device *vgdev,
 	}
 
 	if (use_dma_api) {
-		for_each_sgtable_dma_sg(shmem->pages, sg, si) {
+		for_each_sgtable_dma_sg(pages, sg, si) {
 			(*ents)[si].addr = cpu_to_le64(sg_dma_address(sg));
 			(*ents)[si].length = cpu_to_le32(sg_dma_len(sg));
 			(*ents)[si].padding = 0;
 		}
 	} else {
-		for_each_sgtable_sg(shmem->pages, sg, si) {
+		for_each_sgtable_sg(pages, sg, si) {
 			(*ents)[si].addr = cpu_to_le64(sg_phys(sg));
 			(*ents)[si].length = cpu_to_le32(sg->length);
 			(*ents)[si].padding = 0;
@@ -234,6 +202,10 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
 
 	bo->dumb = params->dumb;
 
+	ret = virtio_gpu_object_shmem_init(vgdev, bo, &ents, &nents);
+	if (ret != 0)
+		goto err_put_id;
+
 	if (fence) {
 		ret = -ENOMEM;
 		objs = virtio_gpu_array_alloc(1);
@@ -246,13 +218,6 @@ int virtio_gpu_object_create(struct virtio_gpu_device *vgdev,
 			goto err_put_objs;
 	}
 
-	ret = virtio_gpu_object_shmem_init(vgdev, bo, &ents, &nents);
-	if (ret != 0) {
-		virtio_gpu_array_put_free(objs);
-		virtio_gpu_free_object(&shmem_obj->base);
-		return ret;
-	}
-
 	if (params->blob) {
 		if (params->blob_mem == VIRTGPU_BLOB_MEM_GUEST)
 			bo->guest_blob = true;
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index 6d3cc9e238a4..4c09e313bebc 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -26,7 +26,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 
 #include "virtgpu_drv.h"
 
@@ -67,16 +66,9 @@ uint32_t virtio_gpu_translate_format(uint32_t drm_fourcc)
 	return format;
 }
 
-static void virtio_gpu_plane_destroy(struct drm_plane *plane)
-{
-	drm_plane_cleanup(plane);
-	kfree(plane);
-}
-
 static const struct drm_plane_funcs virtio_gpu_plane_funcs = {
 	.update_plane		= drm_atomic_helper_update_plane,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.destroy		= virtio_gpu_plane_destroy,
 	.reset			= drm_atomic_helper_plane_reset,
 	.atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
 	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
@@ -100,8 +92,8 @@ static int virtio_gpu_plane_atomic_check(struct drm_plane *plane,
                 return PTR_ERR(crtc_state);
 
 	ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  is_cursor, true);
 	return ret;
 }
@@ -266,14 +258,14 @@ static int virtio_gpu_plane_prepare_fb(struct drm_plane *plane,
 }
 
 static void virtio_gpu_plane_cleanup_fb(struct drm_plane *plane,
-					struct drm_plane_state *old_state)
+					struct drm_plane_state *state)
 {
 	struct virtio_gpu_framebuffer *vgfb;
 
-	if (!plane->state->fb)
+	if (!state->fb)
 		return;
 
-	vgfb = to_virtio_gpu_framebuffer(plane->state->fb);
+	vgfb = to_virtio_gpu_framebuffer(state->fb);
 	if (vgfb->fence) {
 		dma_fence_put(&vgfb->fence->f);
 		vgfb->fence = NULL;
@@ -379,11 +371,7 @@ struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev,
 	const struct drm_plane_helper_funcs *funcs;
 	struct drm_plane *plane;
 	const uint32_t *formats;
-	int ret, nformats;
-
-	plane = kzalloc(sizeof(*plane), GFP_KERNEL);
-	if (!plane)
-		return ERR_PTR(-ENOMEM);
+	int nformats;
 
 	if (type == DRM_PLANE_TYPE_CURSOR) {
 		formats = virtio_gpu_cursor_formats;
@@ -394,17 +382,13 @@ struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev,
 		nformats = ARRAY_SIZE(virtio_gpu_formats);
 		funcs = &virtio_gpu_primary_helper_funcs;
 	}
-	ret = drm_universal_plane_init(dev, plane, 1 << index,
-				       &virtio_gpu_plane_funcs,
-				       formats, nformats,
-				       NULL, type, NULL);
-	if (ret)
-		goto err_plane_init;
+
+	plane = drmm_universal_plane_alloc(dev, struct drm_plane, dev,
+					   1 << index, &virtio_gpu_plane_funcs,
+					   formats, nformats, NULL, type, NULL);
+	if (IS_ERR(plane))
+		return plane;
 
 	drm_plane_helper_add(plane, funcs);
 	return plane;
-
-err_plane_init:
-	kfree(plane);
-	return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/virtio/virtgpu_vq.c b/drivers/gpu/drm/virtio/virtgpu_vq.c
index b7529b2b9883..9ff8660b50ad 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vq.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vq.c
@@ -322,7 +322,7 @@ static int virtio_gpu_queue_ctrl_sgs(struct virtio_gpu_device *vgdev,
 		if (fence && vbuf->objs)
 			virtio_gpu_array_unlock_resv(vbuf->objs);
 		free_vbuf(vgdev, vbuf);
-		return -1;
+		return -ENODEV;
 	}
 
 	if (vgdev->has_indirect)
@@ -386,7 +386,7 @@ static int virtio_gpu_queue_fenced_ctrl_buffer(struct virtio_gpu_device *vgdev,
 			if (!sgt) {
 				if (fence && vbuf->objs)
 					virtio_gpu_array_unlock_resv(vbuf->objs);
-				return -1;
+				return -ENOMEM;
 			}
 
 			elemcnt += sg_ents;
@@ -595,11 +595,10 @@ void virtio_gpu_cmd_transfer_to_host_2d(struct virtio_gpu_device *vgdev,
 	struct virtio_gpu_transfer_to_host_2d *cmd_p;
 	struct virtio_gpu_vbuffer *vbuf;
 	bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev);
-	struct virtio_gpu_object_shmem *shmem = to_virtio_gpu_shmem(bo);
 
-	if (use_dma_api)
-		dma_sync_sgtable_for_device(vgdev->vdev->dev.parent,
-					    shmem->pages, DMA_TO_DEVICE);
+	if (virtio_gpu_is_shmem(bo) && use_dma_api)
+		dma_sync_sgtable_for_device(&vgdev->vdev->dev,
+					    bo->base.sgt, DMA_TO_DEVICE);
 
 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
 	memset(cmd_p, 0, sizeof(*cmd_p));
@@ -721,7 +720,7 @@ static int virtio_get_edid_block(void *data, u8 *buf,
 	size_t start = block * EDID_LENGTH;
 
 	if (start + len > le32_to_cpu(resp->size))
-		return -1;
+		return -EINVAL;
 	memcpy(buf, resp->edid + start, len);
 	return 0;
 }
@@ -1019,11 +1018,9 @@ void virtio_gpu_cmd_transfer_to_host_3d(struct virtio_gpu_device *vgdev,
 	struct virtio_gpu_vbuffer *vbuf;
 	bool use_dma_api = !virtio_has_dma_quirk(vgdev->vdev);
 
-	if (virtio_gpu_is_shmem(bo) && use_dma_api) {
-		struct virtio_gpu_object_shmem *shmem = to_virtio_gpu_shmem(bo);
-		dma_sync_sgtable_for_device(vgdev->vdev->dev.parent,
-					    shmem->pages, DMA_TO_DEVICE);
-	}
+	if (virtio_gpu_is_shmem(bo) && use_dma_api)
+		dma_sync_sgtable_for_device(&vgdev->vdev->dev,
+					    bo->base.sgt, DMA_TO_DEVICE);
 
 	cmd_p = virtio_gpu_alloc_cmd(vgdev, &vbuf, sizeof(*cmd_p));
 	memset(cmd_p, 0, sizeof(*cmd_p));
diff --git a/drivers/gpu/drm/vkms/Makefile b/drivers/gpu/drm/vkms/Makefile
index 72f779cbfedd..1b28a6a32948 100644
--- a/drivers/gpu/drm/vkms/Makefile
+++ b/drivers/gpu/drm/vkms/Makefile
@@ -3,6 +3,7 @@ vkms-y := \
 	vkms_drv.o \
 	vkms_plane.o \
 	vkms_output.o \
+	vkms_formats.o \
 	vkms_crtc.o \
 	vkms_composer.o \
 	vkms_writeback.o
diff --git a/drivers/gpu/drm/vkms/vkms_composer.c b/drivers/gpu/drm/vkms/vkms_composer.c
index 775b97766e08..8e53fa80742b 100644
--- a/drivers/gpu/drm/vkms/vkms_composer.c
+++ b/drivers/gpu/drm/vkms/vkms_composer.c
@@ -7,203 +7,185 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_vblank.h>
+#include <linux/minmax.h>
 
 #include "vkms_drv.h"
 
-static u32 get_pixel_from_buffer(int x, int y, const u8 *buffer,
-				 const struct vkms_composer *composer)
+static u16 pre_mul_blend_channel(u16 src, u16 dst, u16 alpha)
 {
-	u32 pixel;
-	int src_offset = composer->offset + (y * composer->pitch)
-				      + (x * composer->cpp);
+	u32 new_color;
 
-	pixel = *(u32 *)&buffer[src_offset];
+	new_color = (src * 0xffff + dst * (0xffff - alpha));
 
-	return pixel;
+	return DIV_ROUND_CLOSEST(new_color, 0xffff);
 }
 
 /**
- * compute_crc - Compute CRC value on output frame
+ * pre_mul_alpha_blend - alpha blending equation
+ * @src_frame_info: source framebuffer's metadata
+ * @stage_buffer: The line with the pixels from src_plane
+ * @output_buffer: A line buffer that receives all the blends output
  *
- * @vaddr: address to final framebuffer
- * @composer: framebuffer's metadata
+ * Using the information from the `frame_info`, this blends only the
+ * necessary pixels from the `stage_buffer` to the `output_buffer`
+ * using premultiplied blend formula.
  *
- * returns CRC value computed using crc32 on the visible portion of
- * the final framebuffer at vaddr_out
+ * The current DRM assumption is that pixel color values have been already
+ * pre-multiplied with the alpha channel values. See more
+ * drm_plane_create_blend_mode_property(). Also, this formula assumes a
+ * completely opaque background.
  */
-static uint32_t compute_crc(const u8 *vaddr,
-			    const struct vkms_composer *composer)
+static void pre_mul_alpha_blend(struct vkms_frame_info *frame_info,
+				struct line_buffer *stage_buffer,
+				struct line_buffer *output_buffer)
 {
-	int x, y;
-	u32 crc = 0, pixel = 0;
-	int x_src = composer->src.x1 >> 16;
-	int y_src = composer->src.y1 >> 16;
-	int h_src = drm_rect_height(&composer->src) >> 16;
-	int w_src = drm_rect_width(&composer->src) >> 16;
-
-	for (y = y_src; y < y_src + h_src; ++y) {
-		for (x = x_src; x < x_src + w_src; ++x) {
-			pixel = get_pixel_from_buffer(x, y, vaddr, composer);
-			crc = crc32_le(crc, (void *)&pixel, sizeof(u32));
-		}
+	int x_dst = frame_info->dst.x1;
+	struct pixel_argb_u16 *out = output_buffer->pixels + x_dst;
+	struct pixel_argb_u16 *in = stage_buffer->pixels;
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			    stage_buffer->n_pixels);
+
+	for (int x = 0; x < x_limit; x++) {
+		out[x].a = (u16)0xffff;
+		out[x].r = pre_mul_blend_channel(in[x].r, out[x].r, in[x].a);
+		out[x].g = pre_mul_blend_channel(in[x].g, out[x].g, in[x].a);
+		out[x].b = pre_mul_blend_channel(in[x].b, out[x].b, in[x].a);
 	}
-
-	return crc;
 }
 
-static u8 blend_channel(u8 src, u8 dst, u8 alpha)
+static bool check_y_limit(struct vkms_frame_info *frame_info, int y)
 {
-	u32 pre_blend;
-	u8 new_color;
-
-	pre_blend = (src * 255 + dst * (255 - alpha));
-
-	/* Faster div by 255 */
-	new_color = ((pre_blend + ((pre_blend + 257) >> 8)) >> 8);
+	if (y >= frame_info->dst.y1 && y < frame_info->dst.y2)
+		return true;
 
-	return new_color;
+	return false;
 }
 
-/**
- * alpha_blend - alpha blending equation
- * @argb_src: src pixel on premultiplied alpha mode
- * @argb_dst: dst pixel completely opaque
- *
- * blend pixels using premultiplied blend formula. The current DRM assumption
- * is that pixel color values have been already pre-multiplied with the alpha
- * channel values. See more drm_plane_create_blend_mode_property(). Also, this
- * formula assumes a completely opaque background.
- */
-static void alpha_blend(const u8 *argb_src, u8 *argb_dst)
+static void fill_background(const struct pixel_argb_u16 *background_color,
+			    struct line_buffer *output_buffer)
 {
-	u8 alpha;
-
-	alpha = argb_src[3];
-	argb_dst[0] = blend_channel(argb_src[0], argb_dst[0], alpha);
-	argb_dst[1] = blend_channel(argb_src[1], argb_dst[1], alpha);
-	argb_dst[2] = blend_channel(argb_src[2], argb_dst[2], alpha);
+	for (size_t i = 0; i < output_buffer->n_pixels; i++)
+		output_buffer->pixels[i] = *background_color;
 }
 
 /**
- * x_blend - blending equation that ignores the pixel alpha
+ * @wb_frame_info: The writeback frame buffer metadata
+ * @crtc_state: The crtc state
+ * @crc32: The crc output of the final frame
+ * @output_buffer: A buffer of a row that will receive the result of the blend(s)
+ * @stage_buffer: The line with the pixels from plane being blend to the output
  *
- * overwrites RGB color value from src pixel to dst pixel.
+ * This function blends the pixels (Using the `pre_mul_alpha_blend`)
+ * from all planes, calculates the crc32 of the output from the former step,
+ * and, if necessary, convert and store the output to the writeback buffer.
  */
-static void x_blend(const u8 *xrgb_src, u8 *xrgb_dst)
+static void blend(struct vkms_writeback_job *wb,
+		  struct vkms_crtc_state *crtc_state,
+		  u32 *crc32, struct line_buffer *stage_buffer,
+		  struct line_buffer *output_buffer, size_t row_size)
 {
-	memcpy(xrgb_dst, xrgb_src, sizeof(u8) * 3);
-}
+	struct vkms_plane_state **plane = crtc_state->active_planes;
+	u32 n_active_planes = crtc_state->num_active_planes;
 
-/**
- * blend - blend value at vaddr_src with value at vaddr_dst
- * @vaddr_dst: destination address
- * @vaddr_src: source address
- * @dst_composer: destination framebuffer's metadata
- * @src_composer: source framebuffer's metadata
- * @pixel_blend: blending equation based on plane format
- *
- * Blend the vaddr_src value with the vaddr_dst value using a pixel blend
- * equation according to the supported plane formats DRM_FORMAT_(A/XRGB8888)
- * and clearing alpha channel to an completely opaque background. This function
- * uses buffer's metadata to locate the new composite values at vaddr_dst.
- *
- * TODO: completely clear the primary plane (a = 0xff) before starting to blend
- * pixel color values
- */
-static void blend(void *vaddr_dst, void *vaddr_src,
-		  struct vkms_composer *dst_composer,
-		  struct vkms_composer *src_composer,
-		  void (*pixel_blend)(const u8 *, u8 *))
-{
-	int i, j, j_dst, i_dst;
-	int offset_src, offset_dst;
-	u8 *pixel_dst, *pixel_src;
-
-	int x_src = src_composer->src.x1 >> 16;
-	int y_src = src_composer->src.y1 >> 16;
-
-	int x_dst = src_composer->dst.x1;
-	int y_dst = src_composer->dst.y1;
-	int h_dst = drm_rect_height(&src_composer->dst);
-	int w_dst = drm_rect_width(&src_composer->dst);
-
-	int y_limit = y_src + h_dst;
-	int x_limit = x_src + w_dst;
-
-	for (i = y_src, i_dst = y_dst; i < y_limit; ++i) {
-		for (j = x_src, j_dst = x_dst; j < x_limit; ++j) {
-			offset_dst = dst_composer->offset
-				     + (i_dst * dst_composer->pitch)
-				     + (j_dst++ * dst_composer->cpp);
-			offset_src = src_composer->offset
-				     + (i * src_composer->pitch)
-				     + (j * src_composer->cpp);
-
-			pixel_src = (u8 *)(vaddr_src + offset_src);
-			pixel_dst = (u8 *)(vaddr_dst + offset_dst);
-			pixel_blend(pixel_src, pixel_dst);
-			/* clearing alpha channel (0xff)*/
-			pixel_dst[3] = 0xff;
+	const struct pixel_argb_u16 background_color = { .a = 0xffff };
+
+	size_t crtc_y_limit = crtc_state->base.crtc->mode.vdisplay;
+
+	for (size_t y = 0; y < crtc_y_limit; y++) {
+		fill_background(&background_color, output_buffer);
+
+		/* The active planes are composed associatively in z-order. */
+		for (size_t i = 0; i < n_active_planes; i++) {
+			if (!check_y_limit(plane[i]->frame_info, y))
+				continue;
+
+			plane[i]->plane_read(stage_buffer, plane[i]->frame_info, y);
+			pre_mul_alpha_blend(plane[i]->frame_info, stage_buffer,
+					    output_buffer);
 		}
-		i_dst++;
+
+		*crc32 = crc32_le(*crc32, (void *)output_buffer->pixels, row_size);
+
+		if (wb)
+			wb->wb_write(&wb->wb_frame_info, output_buffer, y);
 	}
 }
 
-static void compose_plane(struct vkms_composer *primary_composer,
-			  struct vkms_composer *plane_composer,
-			  void *vaddr_out)
+static int check_format_funcs(struct vkms_crtc_state *crtc_state,
+			      struct vkms_writeback_job *active_wb)
 {
-	struct drm_framebuffer *fb = &plane_composer->fb;
-	void *vaddr;
-	void (*pixel_blend)(const u8 *p_src, u8 *p_dst);
+	struct vkms_plane_state **planes = crtc_state->active_planes;
+	u32 n_active_planes = crtc_state->num_active_planes;
 
-	if (WARN_ON(iosys_map_is_null(&plane_composer->map[0])))
-		return;
+	for (size_t i = 0; i < n_active_planes; i++)
+		if (!planes[i]->plane_read)
+			return -1;
 
-	vaddr = plane_composer->map[0].vaddr;
+	if (active_wb && !active_wb->wb_write)
+		return -1;
 
-	if (fb->format->format == DRM_FORMAT_ARGB8888)
-		pixel_blend = &alpha_blend;
-	else
-		pixel_blend = &x_blend;
+	return 0;
+}
+
+static int check_iosys_map(struct vkms_crtc_state *crtc_state)
+{
+	struct vkms_plane_state **plane_state = crtc_state->active_planes;
+	u32 n_active_planes = crtc_state->num_active_planes;
+
+	for (size_t i = 0; i < n_active_planes; i++)
+		if (iosys_map_is_null(&plane_state[i]->frame_info->map[0]))
+			return -1;
 
-	blend(vaddr_out, vaddr, primary_composer, plane_composer, pixel_blend);
+	return 0;
 }
 
-static int compose_active_planes(void **vaddr_out,
-				 struct vkms_composer *primary_composer,
-				 struct vkms_crtc_state *crtc_state)
+static int compose_active_planes(struct vkms_writeback_job *active_wb,
+				 struct vkms_crtc_state *crtc_state,
+				 u32 *crc32)
 {
-	struct drm_framebuffer *fb = &primary_composer->fb;
-	struct drm_gem_object *gem_obj = drm_gem_fb_get_obj(fb, 0);
-	const void *vaddr;
-	int i;
-
-	if (!*vaddr_out) {
-		*vaddr_out = kvzalloc(gem_obj->size, GFP_KERNEL);
-		if (!*vaddr_out) {
-			DRM_ERROR("Cannot allocate memory for output frame.");
-			return -ENOMEM;
-		}
-	}
+	size_t line_width, pixel_size = sizeof(struct pixel_argb_u16);
+	struct line_buffer output_buffer, stage_buffer;
+	int ret = 0;
+
+	/*
+	 * This check exists so we can call `crc32_le` for the entire line
+	 * instead doing it for each channel of each pixel in case
+	 * `struct `pixel_argb_u16` had any gap added by the compiler
+	 * between the struct fields.
+	 */
+	static_assert(sizeof(struct pixel_argb_u16) == 8);
 
-	if (WARN_ON(iosys_map_is_null(&primary_composer->map[0])))
+	if (WARN_ON(check_iosys_map(crtc_state)))
 		return -EINVAL;
 
-	vaddr = primary_composer->map[0].vaddr;
+	if (WARN_ON(check_format_funcs(crtc_state, active_wb)))
+		return -EINVAL;
 
-	memcpy(*vaddr_out, vaddr, gem_obj->size);
+	line_width = crtc_state->base.crtc->mode.hdisplay;
+	stage_buffer.n_pixels = line_width;
+	output_buffer.n_pixels = line_width;
 
-	/* If there are other planes besides primary, we consider the active
-	 * planes should be in z-order and compose them associatively:
-	 * ((primary <- overlay) <- cursor)
-	 */
-	for (i = 1; i < crtc_state->num_active_planes; i++)
-		compose_plane(primary_composer,
-			      crtc_state->active_planes[i]->composer,
-			      *vaddr_out);
+	stage_buffer.pixels = kvmalloc(line_width * pixel_size, GFP_KERNEL);
+	if (!stage_buffer.pixels) {
+		DRM_ERROR("Cannot allocate memory for the output line buffer");
+		return -ENOMEM;
+	}
 
-	return 0;
+	output_buffer.pixels = kvmalloc(line_width * pixel_size, GFP_KERNEL);
+	if (!output_buffer.pixels) {
+		DRM_ERROR("Cannot allocate memory for intermediate line buffer");
+		ret = -ENOMEM;
+		goto free_stage_buffer;
+	}
+
+	blend(active_wb, crtc_state, crc32, &stage_buffer,
+	      &output_buffer, line_width * pixel_size);
+
+	kvfree(output_buffer.pixels);
+free_stage_buffer:
+	kvfree(stage_buffer.pixels);
+
+	return ret;
 }
 
 /**
@@ -221,13 +203,11 @@ void vkms_composer_worker(struct work_struct *work)
 						struct vkms_crtc_state,
 						composer_work);
 	struct drm_crtc *crtc = crtc_state->base.crtc;
+	struct vkms_writeback_job *active_wb = crtc_state->active_writeback;
 	struct vkms_output *out = drm_crtc_to_vkms_output(crtc);
-	struct vkms_composer *primary_composer = NULL;
-	struct vkms_plane_state *act_plane = NULL;
 	bool crc_pending, wb_pending;
-	void *vaddr_out = NULL;
-	u32 crc32 = 0;
 	u64 frame_start, frame_end;
+	u32 crc32 = 0;
 	int ret;
 
 	spin_lock_irq(&out->composer_lock);
@@ -247,35 +227,19 @@ void vkms_composer_worker(struct work_struct *work)
 	if (!crc_pending)
 		return;
 
-	if (crtc_state->num_active_planes >= 1) {
-		act_plane = crtc_state->active_planes[0];
-		if (act_plane->base.base.plane->type == DRM_PLANE_TYPE_PRIMARY)
-			primary_composer = act_plane->composer;
-	}
-
-	if (!primary_composer)
-		return;
-
 	if (wb_pending)
-		vaddr_out = crtc_state->active_writeback->data[0].vaddr;
+		ret = compose_active_planes(active_wb, crtc_state, &crc32);
+	else
+		ret = compose_active_planes(NULL, crtc_state, &crc32);
 
-	ret = compose_active_planes(&vaddr_out, primary_composer,
-				    crtc_state);
-	if (ret) {
-		if (ret == -EINVAL && !wb_pending)
-			kvfree(vaddr_out);
+	if (ret)
 		return;
-	}
-
-	crc32 = compute_crc(vaddr_out, primary_composer);
 
 	if (wb_pending) {
 		drm_writeback_signal_completion(&out->wb_connector, 0);
 		spin_lock_irq(&out->composer_lock);
 		crtc_state->wb_pending = false;
 		spin_unlock_irq(&out->composer_lock);
-	} else {
-		kvfree(vaddr_out);
 	}
 
 	/*
diff --git a/drivers/gpu/drm/vkms/vkms_drv.h b/drivers/gpu/drm/vkms/vkms_drv.h
index 1d60654b553b..0a67b8073f7e 100644
--- a/drivers/gpu/drm/vkms/vkms_drv.h
+++ b/drivers/gpu/drm/vkms/vkms_drv.h
@@ -23,28 +23,41 @@
 
 #define NUM_OVERLAY_PLANES 8
 
-struct vkms_writeback_job {
-	struct iosys_map map[DRM_FORMAT_MAX_PLANES];
-	struct iosys_map data[DRM_FORMAT_MAX_PLANES];
-};
-
-struct vkms_composer {
-	struct drm_framebuffer fb;
+struct vkms_frame_info {
+	struct drm_framebuffer *fb;
 	struct drm_rect src, dst;
-	struct iosys_map map[4];
+	struct iosys_map map[DRM_FORMAT_MAX_PLANES];
 	unsigned int offset;
 	unsigned int pitch;
 	unsigned int cpp;
 };
 
+struct pixel_argb_u16 {
+	u16 a, r, g, b;
+};
+
+struct line_buffer {
+	size_t n_pixels;
+	struct pixel_argb_u16 *pixels;
+};
+
+struct vkms_writeback_job {
+	struct iosys_map data[DRM_FORMAT_MAX_PLANES];
+	struct vkms_frame_info wb_frame_info;
+	void (*wb_write)(struct vkms_frame_info *frame_info,
+			 const struct line_buffer *buffer, int y);
+};
+
 /**
  * vkms_plane_state - Driver specific plane state
  * @base: base plane state
- * @composer: data required for composing computation
+ * @frame_info: data required for composing computation
  */
 struct vkms_plane_state {
 	struct drm_shadow_plane_state base;
-	struct vkms_composer *composer;
+	struct vkms_frame_info *frame_info;
+	void (*plane_read)(struct line_buffer *buffer,
+			   const struct vkms_frame_info *frame_info, int y);
 };
 
 struct vkms_plane {
diff --git a/drivers/gpu/drm/vkms/vkms_formats.c b/drivers/gpu/drm/vkms/vkms_formats.c
new file mode 100644
index 000000000000..d4950688b3f1
--- /dev/null
+++ b/drivers/gpu/drm/vkms/vkms_formats.c
@@ -0,0 +1,286 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/kernel.h>
+#include <linux/minmax.h>
+#include <drm/drm_rect.h>
+#include <drm/drm_fixed.h>
+
+#include "vkms_formats.h"
+
+static size_t pixel_offset(const struct vkms_frame_info *frame_info, int x, int y)
+{
+	return frame_info->offset + (y * frame_info->pitch)
+				  + (x * frame_info->cpp);
+}
+
+/*
+ * packed_pixels_addr - Get the pointer to pixel of a given pair of coordinates
+ *
+ * @frame_info: Buffer metadata
+ * @x: The x(width) coordinate of the 2D buffer
+ * @y: The y(Heigth) coordinate of the 2D buffer
+ *
+ * Takes the information stored in the frame_info, a pair of coordinates, and
+ * returns the address of the first color channel.
+ * This function assumes the channels are packed together, i.e. a color channel
+ * comes immediately after another in the memory. And therefore, this function
+ * doesn't work for YUV with chroma subsampling (e.g. YUV420 and NV21).
+ */
+static void *packed_pixels_addr(const struct vkms_frame_info *frame_info,
+				int x, int y)
+{
+	size_t offset = pixel_offset(frame_info, x, y);
+
+	return (u8 *)frame_info->map[0].vaddr + offset;
+}
+
+static void *get_packed_src_addr(const struct vkms_frame_info *frame_info, int y)
+{
+	int x_src = frame_info->src.x1 >> 16;
+	int y_src = y - frame_info->dst.y1 + (frame_info->src.y1 >> 16);
+
+	return packed_pixels_addr(frame_info, x_src, y_src);
+}
+
+static void ARGB8888_to_argb_u16(struct line_buffer *stage_buffer,
+				 const struct vkms_frame_info *frame_info, int y)
+{
+	struct pixel_argb_u16 *out_pixels = stage_buffer->pixels;
+	u8 *src_pixels = get_packed_src_addr(frame_info, y);
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			    stage_buffer->n_pixels);
+
+	for (size_t x = 0; x < x_limit; x++, src_pixels += 4) {
+		/*
+		 * The 257 is the "conversion ratio". This number is obtained by the
+		 * (2^16 - 1) / (2^8 - 1) division. Which, in this case, tries to get
+		 * the best color value in a pixel format with more possibilities.
+		 * A similar idea applies to others RGB color conversions.
+		 */
+		out_pixels[x].a = (u16)src_pixels[3] * 257;
+		out_pixels[x].r = (u16)src_pixels[2] * 257;
+		out_pixels[x].g = (u16)src_pixels[1] * 257;
+		out_pixels[x].b = (u16)src_pixels[0] * 257;
+	}
+}
+
+static void XRGB8888_to_argb_u16(struct line_buffer *stage_buffer,
+				 const struct vkms_frame_info *frame_info, int y)
+{
+	struct pixel_argb_u16 *out_pixels = stage_buffer->pixels;
+	u8 *src_pixels = get_packed_src_addr(frame_info, y);
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			    stage_buffer->n_pixels);
+
+	for (size_t x = 0; x < x_limit; x++, src_pixels += 4) {
+		out_pixels[x].a = (u16)0xffff;
+		out_pixels[x].r = (u16)src_pixels[2] * 257;
+		out_pixels[x].g = (u16)src_pixels[1] * 257;
+		out_pixels[x].b = (u16)src_pixels[0] * 257;
+	}
+}
+
+static void ARGB16161616_to_argb_u16(struct line_buffer *stage_buffer,
+				     const struct vkms_frame_info *frame_info,
+				     int y)
+{
+	struct pixel_argb_u16 *out_pixels = stage_buffer->pixels;
+	u16 *src_pixels = get_packed_src_addr(frame_info, y);
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			    stage_buffer->n_pixels);
+
+	for (size_t x = 0; x < x_limit; x++, src_pixels += 4) {
+		out_pixels[x].a = le16_to_cpu(src_pixels[3]);
+		out_pixels[x].r = le16_to_cpu(src_pixels[2]);
+		out_pixels[x].g = le16_to_cpu(src_pixels[1]);
+		out_pixels[x].b = le16_to_cpu(src_pixels[0]);
+	}
+}
+
+static void XRGB16161616_to_argb_u16(struct line_buffer *stage_buffer,
+				     const struct vkms_frame_info *frame_info,
+				     int y)
+{
+	struct pixel_argb_u16 *out_pixels = stage_buffer->pixels;
+	u16 *src_pixels = get_packed_src_addr(frame_info, y);
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			    stage_buffer->n_pixels);
+
+	for (size_t x = 0; x < x_limit; x++, src_pixels += 4) {
+		out_pixels[x].a = (u16)0xffff;
+		out_pixels[x].r = le16_to_cpu(src_pixels[2]);
+		out_pixels[x].g = le16_to_cpu(src_pixels[1]);
+		out_pixels[x].b = le16_to_cpu(src_pixels[0]);
+	}
+}
+
+static void RGB565_to_argb_u16(struct line_buffer *stage_buffer,
+			       const struct vkms_frame_info *frame_info, int y)
+{
+	struct pixel_argb_u16 *out_pixels = stage_buffer->pixels;
+	u16 *src_pixels = get_packed_src_addr(frame_info, y);
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			       stage_buffer->n_pixels);
+
+	s64 fp_rb_ratio = drm_fixp_div(drm_int2fixp(65535), drm_int2fixp(31));
+	s64 fp_g_ratio = drm_fixp_div(drm_int2fixp(65535), drm_int2fixp(63));
+
+	for (size_t x = 0; x < x_limit; x++, src_pixels++) {
+		u16 rgb_565 = le16_to_cpu(*src_pixels);
+		s64 fp_r = drm_int2fixp((rgb_565 >> 11) & 0x1f);
+		s64 fp_g = drm_int2fixp((rgb_565 >> 5) & 0x3f);
+		s64 fp_b = drm_int2fixp(rgb_565 & 0x1f);
+
+		out_pixels[x].a = (u16)0xffff;
+		out_pixels[x].r = drm_fixp2int(drm_fixp_mul(fp_r, fp_rb_ratio));
+		out_pixels[x].g = drm_fixp2int(drm_fixp_mul(fp_g, fp_g_ratio));
+		out_pixels[x].b = drm_fixp2int(drm_fixp_mul(fp_b, fp_rb_ratio));
+	}
+}
+
+/*
+ * The following  functions take an line of argb_u16 pixels from the
+ * src_buffer, convert them to a specific format, and store them in the
+ * destination.
+ *
+ * They are used in the `compose_active_planes` to convert and store a line
+ * from the src_buffer to the writeback buffer.
+ */
+static void argb_u16_to_ARGB8888(struct vkms_frame_info *frame_info,
+				 const struct line_buffer *src_buffer, int y)
+{
+	int x_dst = frame_info->dst.x1;
+	u8 *dst_pixels = packed_pixels_addr(frame_info, x_dst, y);
+	struct pixel_argb_u16 *in_pixels = src_buffer->pixels;
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			    src_buffer->n_pixels);
+
+	for (size_t x = 0; x < x_limit; x++, dst_pixels += 4) {
+		/*
+		 * This sequence below is important because the format's byte order is
+		 * in little-endian. In the case of the ARGB8888 the memory is
+		 * organized this way:
+		 *
+		 * | Addr     | = blue channel
+		 * | Addr + 1 | = green channel
+		 * | Addr + 2 | = Red channel
+		 * | Addr + 3 | = Alpha channel
+		 */
+		dst_pixels[3] = DIV_ROUND_CLOSEST(in_pixels[x].a, 257);
+		dst_pixels[2] = DIV_ROUND_CLOSEST(in_pixels[x].r, 257);
+		dst_pixels[1] = DIV_ROUND_CLOSEST(in_pixels[x].g, 257);
+		dst_pixels[0] = DIV_ROUND_CLOSEST(in_pixels[x].b, 257);
+	}
+}
+
+static void argb_u16_to_XRGB8888(struct vkms_frame_info *frame_info,
+				 const struct line_buffer *src_buffer, int y)
+{
+	int x_dst = frame_info->dst.x1;
+	u8 *dst_pixels = packed_pixels_addr(frame_info, x_dst, y);
+	struct pixel_argb_u16 *in_pixels = src_buffer->pixels;
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			    src_buffer->n_pixels);
+
+	for (size_t x = 0; x < x_limit; x++, dst_pixels += 4) {
+		dst_pixels[3] = 0xff;
+		dst_pixels[2] = DIV_ROUND_CLOSEST(in_pixels[x].r, 257);
+		dst_pixels[1] = DIV_ROUND_CLOSEST(in_pixels[x].g, 257);
+		dst_pixels[0] = DIV_ROUND_CLOSEST(in_pixels[x].b, 257);
+	}
+}
+
+static void argb_u16_to_ARGB16161616(struct vkms_frame_info *frame_info,
+				     const struct line_buffer *src_buffer, int y)
+{
+	int x_dst = frame_info->dst.x1;
+	u16 *dst_pixels = packed_pixels_addr(frame_info, x_dst, y);
+	struct pixel_argb_u16 *in_pixels = src_buffer->pixels;
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			    src_buffer->n_pixels);
+
+	for (size_t x = 0; x < x_limit; x++, dst_pixels += 4) {
+		dst_pixels[3] = cpu_to_le16(in_pixels[x].a);
+		dst_pixels[2] = cpu_to_le16(in_pixels[x].r);
+		dst_pixels[1] = cpu_to_le16(in_pixels[x].g);
+		dst_pixels[0] = cpu_to_le16(in_pixels[x].b);
+	}
+}
+
+static void argb_u16_to_XRGB16161616(struct vkms_frame_info *frame_info,
+				     const struct line_buffer *src_buffer, int y)
+{
+	int x_dst = frame_info->dst.x1;
+	u16 *dst_pixels = packed_pixels_addr(frame_info, x_dst, y);
+	struct pixel_argb_u16 *in_pixels = src_buffer->pixels;
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			    src_buffer->n_pixels);
+
+	for (size_t x = 0; x < x_limit; x++, dst_pixels += 4) {
+		dst_pixels[3] = 0xffff;
+		dst_pixels[2] = cpu_to_le16(in_pixels[x].r);
+		dst_pixels[1] = cpu_to_le16(in_pixels[x].g);
+		dst_pixels[0] = cpu_to_le16(in_pixels[x].b);
+	}
+}
+
+static void argb_u16_to_RGB565(struct vkms_frame_info *frame_info,
+			       const struct line_buffer *src_buffer, int y)
+{
+	int x_dst = frame_info->dst.x1;
+	u16 *dst_pixels = packed_pixels_addr(frame_info, x_dst, y);
+	struct pixel_argb_u16 *in_pixels = src_buffer->pixels;
+	int x_limit = min_t(size_t, drm_rect_width(&frame_info->dst),
+			    src_buffer->n_pixels);
+
+	s64 fp_rb_ratio = drm_fixp_div(drm_int2fixp(65535), drm_int2fixp(31));
+	s64 fp_g_ratio = drm_fixp_div(drm_int2fixp(65535), drm_int2fixp(63));
+
+	for (size_t x = 0; x < x_limit; x++, dst_pixels++) {
+		s64 fp_r = drm_int2fixp(in_pixels[x].r);
+		s64 fp_g = drm_int2fixp(in_pixels[x].g);
+		s64 fp_b = drm_int2fixp(in_pixels[x].b);
+
+		u16 r = drm_fixp2int(drm_fixp_div(fp_r, fp_rb_ratio));
+		u16 g = drm_fixp2int(drm_fixp_div(fp_g, fp_g_ratio));
+		u16 b = drm_fixp2int(drm_fixp_div(fp_b, fp_rb_ratio));
+
+		*dst_pixels = cpu_to_le16(r << 11 | g << 5 | b);
+	}
+}
+
+void *get_frame_to_line_function(u32 format)
+{
+	switch (format) {
+	case DRM_FORMAT_ARGB8888:
+		return &ARGB8888_to_argb_u16;
+	case DRM_FORMAT_XRGB8888:
+		return &XRGB8888_to_argb_u16;
+	case DRM_FORMAT_ARGB16161616:
+		return &ARGB16161616_to_argb_u16;
+	case DRM_FORMAT_XRGB16161616:
+		return &XRGB16161616_to_argb_u16;
+	case DRM_FORMAT_RGB565:
+		return &RGB565_to_argb_u16;
+	default:
+		return NULL;
+	}
+}
+
+void *get_line_to_frame_function(u32 format)
+{
+	switch (format) {
+	case DRM_FORMAT_ARGB8888:
+		return &argb_u16_to_ARGB8888;
+	case DRM_FORMAT_XRGB8888:
+		return &argb_u16_to_XRGB8888;
+	case DRM_FORMAT_ARGB16161616:
+		return &argb_u16_to_ARGB16161616;
+	case DRM_FORMAT_XRGB16161616:
+		return &argb_u16_to_XRGB16161616;
+	case DRM_FORMAT_RGB565:
+		return &argb_u16_to_RGB565;
+	default:
+		return NULL;
+	}
+}
diff --git a/drivers/gpu/drm/vkms/vkms_formats.h b/drivers/gpu/drm/vkms/vkms_formats.h
new file mode 100644
index 000000000000..43b7c1979018
--- /dev/null
+++ b/drivers/gpu/drm/vkms/vkms_formats.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+
+#ifndef _VKMS_FORMATS_H_
+#define _VKMS_FORMATS_H_
+
+#include "vkms_drv.h"
+
+void *get_frame_to_line_function(u32 format);
+
+void *get_line_to_frame_function(u32 format);
+
+#endif /* _VKMS_FORMATS_H_ */
diff --git a/drivers/gpu/drm/vkms/vkms_plane.c b/drivers/gpu/drm/vkms/vkms_plane.c
index d8eb674b49a6..c3a845220e10 100644
--- a/drivers/gpu/drm/vkms/vkms_plane.c
+++ b/drivers/gpu/drm/vkms/vkms_plane.c
@@ -7,37 +7,42 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
-#include <drm/drm_plane_helper.h>
 
 #include "vkms_drv.h"
+#include "vkms_formats.h"
 
 static const u32 vkms_formats[] = {
 	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XRGB16161616,
+	DRM_FORMAT_RGB565
 };
 
 static const u32 vkms_plane_formats[] = {
 	DRM_FORMAT_ARGB8888,
-	DRM_FORMAT_XRGB8888
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XRGB16161616,
+	DRM_FORMAT_ARGB16161616,
+	DRM_FORMAT_RGB565
 };
 
 static struct drm_plane_state *
 vkms_plane_duplicate_state(struct drm_plane *plane)
 {
 	struct vkms_plane_state *vkms_state;
-	struct vkms_composer *composer;
+	struct vkms_frame_info *frame_info;
 
 	vkms_state = kzalloc(sizeof(*vkms_state), GFP_KERNEL);
 	if (!vkms_state)
 		return NULL;
 
-	composer = kzalloc(sizeof(*composer), GFP_KERNEL);
-	if (!composer) {
-		DRM_DEBUG_KMS("Couldn't allocate composer\n");
+	frame_info = kzalloc(sizeof(*frame_info), GFP_KERNEL);
+	if (!frame_info) {
+		DRM_DEBUG_KMS("Couldn't allocate frame_info\n");
 		kfree(vkms_state);
 		return NULL;
 	}
 
-	vkms_state->composer = composer;
+	vkms_state->frame_info = frame_info;
 
 	__drm_gem_duplicate_shadow_plane_state(plane, &vkms_state->base);
 
@@ -50,16 +55,16 @@ static void vkms_plane_destroy_state(struct drm_plane *plane,
 	struct vkms_plane_state *vkms_state = to_vkms_plane_state(old_state);
 	struct drm_crtc *crtc = vkms_state->base.base.crtc;
 
-	if (crtc) {
+	if (crtc && vkms_state->frame_info->fb) {
 		/* dropping the reference we acquired in
 		 * vkms_primary_plane_update()
 		 */
-		if (drm_framebuffer_read_refcount(&vkms_state->composer->fb))
-			drm_framebuffer_put(&vkms_state->composer->fb);
+		if (drm_framebuffer_read_refcount(vkms_state->frame_info->fb))
+			drm_framebuffer_put(vkms_state->frame_info->fb);
 	}
 
-	kfree(vkms_state->composer);
-	vkms_state->composer = NULL;
+	kfree(vkms_state->frame_info);
+	vkms_state->frame_info = NULL;
 
 	__drm_gem_destroy_shadow_plane_state(&vkms_state->base);
 	kfree(vkms_state);
@@ -99,23 +104,26 @@ static void vkms_plane_atomic_update(struct drm_plane *plane,
 	struct vkms_plane_state *vkms_plane_state;
 	struct drm_shadow_plane_state *shadow_plane_state;
 	struct drm_framebuffer *fb = new_state->fb;
-	struct vkms_composer *composer;
+	struct vkms_frame_info *frame_info;
+	u32 fmt;
 
 	if (!new_state->crtc || !fb)
 		return;
 
+	fmt = fb->format->format;
 	vkms_plane_state = to_vkms_plane_state(new_state);
 	shadow_plane_state = &vkms_plane_state->base;
 
-	composer = vkms_plane_state->composer;
-	memcpy(&composer->src, &new_state->src, sizeof(struct drm_rect));
-	memcpy(&composer->dst, &new_state->dst, sizeof(struct drm_rect));
-	memcpy(&composer->fb, fb, sizeof(struct drm_framebuffer));
-	memcpy(&composer->map, &shadow_plane_state->data, sizeof(composer->map));
-	drm_framebuffer_get(&composer->fb);
-	composer->offset = fb->offsets[0];
-	composer->pitch = fb->pitches[0];
-	composer->cpp = fb->format->cpp[0];
+	frame_info = vkms_plane_state->frame_info;
+	memcpy(&frame_info->src, &new_state->src, sizeof(struct drm_rect));
+	memcpy(&frame_info->dst, &new_state->dst, sizeof(struct drm_rect));
+	frame_info->fb = fb;
+	memcpy(&frame_info->map, &shadow_plane_state->data, sizeof(frame_info->map));
+	drm_framebuffer_get(frame_info->fb);
+	frame_info->offset = fb->offsets[0];
+	frame_info->pitch = fb->pitches[0];
+	frame_info->cpp = fb->format->cpp[0];
+	vkms_plane_state->plane_read = get_frame_to_line_function(fmt);
 }
 
 static int vkms_plane_atomic_check(struct drm_plane *plane,
@@ -139,8 +147,8 @@ static int vkms_plane_atomic_check(struct drm_plane *plane,
 		can_position = true;
 
 	ret = drm_atomic_helper_check_plane_state(new_plane_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  can_position, true);
 	if (ret != 0)
 		return ret;
diff --git a/drivers/gpu/drm/vkms/vkms_writeback.c b/drivers/gpu/drm/vkms/vkms_writeback.c
index 3b3c1e757ab4..84a51cd281b9 100644
--- a/drivers/gpu/drm/vkms/vkms_writeback.c
+++ b/drivers/gpu/drm/vkms/vkms_writeback.c
@@ -12,9 +12,13 @@
 #include <drm/drm_gem_shmem_helper.h>
 
 #include "vkms_drv.h"
+#include "vkms_formats.h"
 
 static const u32 vkms_wb_formats[] = {
 	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XRGB16161616,
+	DRM_FORMAT_ARGB16161616,
+	DRM_FORMAT_RGB565
 };
 
 static const struct drm_connector_funcs vkms_wb_connector_funcs = {
@@ -31,6 +35,7 @@ static int vkms_wb_encoder_atomic_check(struct drm_encoder *encoder,
 {
 	struct drm_framebuffer *fb;
 	const struct drm_display_mode *mode = &crtc_state->mode;
+	int ret;
 
 	if (!conn_state->writeback_job || !conn_state->writeback_job->fb)
 		return 0;
@@ -42,11 +47,9 @@ static int vkms_wb_encoder_atomic_check(struct drm_encoder *encoder,
 		return -EINVAL;
 	}
 
-	if (fb->format->format != vkms_wb_formats[0]) {
-		DRM_DEBUG_KMS("Invalid pixel format %p4cc\n",
-			      &fb->format->format);
-		return -EINVAL;
-	}
+	ret = drm_atomic_helper_check_wb_encoder_state(encoder, conn_state);
+	if (ret < 0)
+		return ret;
 
 	return 0;
 }
@@ -76,12 +79,15 @@ static int vkms_wb_prepare_job(struct drm_writeback_connector *wb_connector,
 	if (!vkmsjob)
 		return -ENOMEM;
 
-	ret = drm_gem_fb_vmap(job->fb, vkmsjob->map, vkmsjob->data);
+	ret = drm_gem_fb_vmap(job->fb, vkmsjob->wb_frame_info.map, vkmsjob->data);
 	if (ret) {
 		DRM_ERROR("vmap failed: %d\n", ret);
 		goto err_kfree;
 	}
 
+	vkmsjob->wb_frame_info.fb = job->fb;
+	drm_framebuffer_get(vkmsjob->wb_frame_info.fb);
+
 	job->priv = vkmsjob;
 
 	return 0;
@@ -100,7 +106,9 @@ static void vkms_wb_cleanup_job(struct drm_writeback_connector *connector,
 	if (!job->fb)
 		return;
 
-	drm_gem_fb_vunmap(job->fb, vkmsjob->map);
+	drm_gem_fb_vunmap(job->fb, vkmsjob->wb_frame_info.map);
+
+	drm_framebuffer_put(vkmsjob->wb_frame_info.fb);
 
 	vkmsdev = drm_device_to_vkms_device(job->fb->dev);
 	vkms_set_composer(&vkmsdev->output, false);
@@ -117,17 +125,32 @@ static void vkms_wb_atomic_commit(struct drm_connector *conn,
 	struct drm_writeback_connector *wb_conn = &output->wb_connector;
 	struct drm_connector_state *conn_state = wb_conn->base.state;
 	struct vkms_crtc_state *crtc_state = output->composer_state;
+	struct drm_framebuffer *fb = connector_state->writeback_job->fb;
+	u16 crtc_height = crtc_state->base.crtc->mode.vdisplay;
+	u16 crtc_width = crtc_state->base.crtc->mode.hdisplay;
+	struct vkms_writeback_job *active_wb;
+	struct vkms_frame_info *wb_frame_info;
+	u32 wb_format = fb->format->format;
 
 	if (!conn_state)
 		return;
 
 	vkms_set_composer(&vkmsdev->output, true);
 
+	active_wb = conn_state->writeback_job->priv;
+	wb_frame_info = &active_wb->wb_frame_info;
+
 	spin_lock_irq(&output->composer_lock);
-	crtc_state->active_writeback = conn_state->writeback_job->priv;
+	crtc_state->active_writeback = active_wb;
+	wb_frame_info->offset = fb->offsets[0];
+	wb_frame_info->pitch = fb->pitches[0];
+	wb_frame_info->cpp = fb->format->cpp[0];
 	crtc_state->wb_pending = true;
 	spin_unlock_irq(&output->composer_lock);
 	drm_writeback_queue_job(wb_conn, connector_state);
+	active_wb->wb_write = get_line_to_frame_function(wb_format);
+	drm_rect_init(&wb_frame_info->src, 0, 0, crtc_width, crtc_height);
+	drm_rect_init(&wb_frame_info->dst, 0, 0, crtc_width, crtc_height);
 }
 
 static const struct drm_connector_helper_funcs vkms_wb_conn_helper_funcs = {
diff --git a/drivers/gpu/drm/vmwgfx/device_include/vm_basic_types.h b/drivers/gpu/drm/vmwgfx/device_include/vm_basic_types.h
index 1f6e3bbc6605..f84376718086 100644
--- a/drivers/gpu/drm/vmwgfx/device_include/vm_basic_types.h
+++ b/drivers/gpu/drm/vmwgfx/device_include/vm_basic_types.h
@@ -121,7 +121,7 @@ typedef __attribute__((aligned(32))) struct MKSGuestStatInfoEntry {
  *
  * Since the MKSGuestStatInfoEntry structures contain userlevel
  * pointers, the InstanceDescriptor also contains pointers to the
- * begining of these sections allowing the host side code to correctly
+ * beginning of these sections allowing the host side code to correctly
  * interpret the pointers.
  *
  * Because the host side code never acknowledges anything back to the
diff --git a/drivers/gpu/drm/vmwgfx/ttm_object.h b/drivers/gpu/drm/vmwgfx/ttm_object.h
index 4c8700027c6d..1a2fa0f83f5f 100644
--- a/drivers/gpu/drm/vmwgfx/ttm_object.h
+++ b/drivers/gpu/drm/vmwgfx/ttm_object.h
@@ -96,7 +96,7 @@ struct ttm_object_device;
  *
  * This struct is intended to be used as a base struct for objects that
  * are visible to user-space. It provides a global name, race-safe
- * access and refcounting, minimal access contol and hooks for unref actions.
+ * access and refcounting, minimal access control and hooks for unref actions.
  */
 
 struct ttm_base_object {
@@ -138,7 +138,7 @@ struct ttm_prime_object {
  *
  * @tfile: Pointer to a struct ttm_object_file.
  * @base: The struct ttm_base_object to initialize.
- * @shareable: This object is shareable with other applcations.
+ * @shareable: This object is shareable with other applications.
  * (different @tfile pointers.)
  * @type: The object type.
  * @refcount_release: See the struct ttm_base_object description.
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
index 85a66014c2b6..822251aaab0a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c
@@ -429,9 +429,9 @@ int vmw_bo_create_kernel(struct vmw_private *dev_priv, unsigned long size,
 
 	drm_gem_private_object_init(vdev, &bo->base, size);
 
-	ret = ttm_bo_init_reserved(&dev_priv->bdev, bo, size,
-				   ttm_bo_type_kernel, placement, 0,
-				   &ctx, NULL, NULL, vmw_bo_default_destroy);
+	ret = ttm_bo_init_reserved(&dev_priv->bdev, bo, ttm_bo_type_kernel,
+				   placement, 0, &ctx, NULL, NULL,
+				   vmw_bo_default_destroy);
 	if (unlikely(ret))
 		goto error_free;
 
@@ -512,10 +512,8 @@ int vmw_bo_init(struct vmw_private *dev_priv,
 	size = ALIGN(size, PAGE_SIZE);
 	drm_gem_private_object_init(vdev, &vmw_bo->base.base, size);
 
-	ret = ttm_bo_init_reserved(bdev, &vmw_bo->base, size,
-				   ttm_bo_type_device,
-				   placement,
-				   0, &ctx, NULL, NULL, bo_free);
+	ret = ttm_bo_init_reserved(bdev, &vmw_bo->base, ttm_bo_type_device,
+				   placement, 0, &ctx, NULL, NULL, bo_free);
 	if (unlikely(ret)) {
 		return ret;
 	}
@@ -729,7 +727,7 @@ int vmw_user_bo_lookup(struct drm_file *filp,
  * Any persistent usage of the object requires a refcount to be taken using
  * ttm_bo_reference_unless_doomed(). Iff this function returns successfully it
  * needs to be paired with vmw_user_bo_noref_release() and no sleeping-
- * or scheduling functions may be called inbetween these function calls.
+ * or scheduling functions may be called in between these function calls.
  *
  * Return: A struct vmw_buffer_object pointer if successful or negative
  * error pointer on failure.
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf_res.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf_res.c
index 415774fde796..82ef58ccdd42 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf_res.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf_res.c
@@ -36,7 +36,7 @@
  * @res: Refcounted pointer to a struct vmw_resource.
  * @hash: Hash entry for the manager hash table.
  * @head: List head used either by the staging list or the manager list
- * of commited resources.
+ * of committed resources.
  * @state: Staging state of this resource entry.
  * @man: Pointer to a resource manager for this entry.
  */
@@ -51,9 +51,9 @@ struct vmw_cmdbuf_res {
 /**
  * struct vmw_cmdbuf_res_manager - Command buffer resource manager.
  *
- * @resources: Hash table containing staged and commited command buffer
+ * @resources: Hash table containing staged and committed command buffer
  * resources
- * @list: List of commited command buffer resources.
+ * @list: List of committed command buffer resources.
  * @dev_priv: Pointer to a device private structure.
  *
  * @resources and @list are protected by the cmdbuf mutex for now.
@@ -118,7 +118,7 @@ static void vmw_cmdbuf_res_free(struct vmw_cmdbuf_res_manager *man,
  * This function commits a list of command buffer resource
  * additions or removals.
  * It is typically called when the execbuf ioctl call triggering these
- * actions has commited the fifo contents to the device.
+ * actions has committed the fifo contents to the device.
  */
 void vmw_cmdbuf_res_commit(struct list_head *list)
 {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 01a5b47e95f9..d7bd5eb1d3ac 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -1398,18 +1398,6 @@ static void vmw_debugfs_resource_managers_init(struct vmw_private *vmw)
 					    root, "system_mob_ttm");
 }
 
-static unsigned long
-vmw_get_unmapped_area(struct file *file, unsigned long uaddr,
-		      unsigned long len, unsigned long pgoff,
-		      unsigned long flags)
-{
-	struct drm_file *file_priv = file->private_data;
-	struct vmw_private *dev_priv = vmw_priv(file_priv->minor->dev);
-
-	return drm_get_unmapped_area(file, uaddr, len, pgoff, flags,
-				     dev_priv->drm.vma_offset_manager);
-}
-
 static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
 			      void *ptr)
 {
@@ -1576,7 +1564,6 @@ static const struct file_operations vmwgfx_driver_fops = {
 	.compat_ioctl = vmw_compat_ioctl,
 #endif
 	.llseek = noop_llseek,
-	.get_unmapped_area = vmw_get_unmapped_area,
 };
 
 static const struct drm_driver driver = {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index be19aa6e1f13..09e2d738aa87 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -877,7 +877,6 @@ static inline void vmw_user_resource_noref_release(void)
 /**
  * Buffer object helper functions - vmwgfx_bo.c
  */
-extern bool vmw_bo_is_vmw_bo(struct ttm_buffer_object *bo);
 extern int vmw_bo_pin_in_placement(struct vmw_private *vmw_priv,
 				   struct vmw_buffer_object *bo,
 				   struct ttm_placement *placement,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index d49de4905efa..f085dbd4736d 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -1172,7 +1172,7 @@ static int vmw_translate_mob_ptr(struct vmw_private *dev_priv,
 
 	vmw_validation_preload_bo(sw_context->ctx);
 	vmw_bo = vmw_user_bo_noref_lookup(sw_context->filp, handle);
-	if (IS_ERR_OR_NULL(vmw_bo)) {
+	if (IS_ERR(vmw_bo)) {
 		VMW_DEBUG_USER("Could not find or use MOB buffer.\n");
 		return PTR_ERR(vmw_bo);
 	}
@@ -1226,7 +1226,7 @@ static int vmw_translate_guest_ptr(struct vmw_private *dev_priv,
 
 	vmw_validation_preload_bo(sw_context->ctx);
 	vmw_bo = vmw_user_bo_noref_lookup(sw_context->filp, handle);
-	if (IS_ERR_OR_NULL(vmw_bo)) {
+	if (IS_ERR(vmw_bo)) {
 		VMW_DEBUG_USER("Could not find or use GMR region.\n");
 		return PTR_ERR(vmw_bo);
 	}
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index ff2f735bbe7a..214829c32ed8 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -29,7 +29,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_rect.h>
 #include <drm/drm_sysfs.h>
 #include <drm/drm_vblank.h>
@@ -720,8 +719,8 @@ int vmw_du_primary_plane_atomic_check(struct drm_plane *plane,
 							   new_state->crtc);
 
 	ret = drm_atomic_helper_check_plane_state(new_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  false, true);
 
 	if (!ret && new_fb) {
@@ -762,8 +761,8 @@ int vmw_du_cursor_plane_atomic_check(struct drm_plane *plane,
 							   new_state->crtc);
 
 	ret = drm_atomic_helper_check_plane_state(new_state, crtc_state,
-						  DRM_PLANE_HELPER_NO_SCALING,
-						  DRM_PLANE_HELPER_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
 						  true, true);
 	if (ret)
 		return ret;
@@ -2257,7 +2256,7 @@ out_fini:
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
 	mutex_unlock(&dev->mode_config.mutex);
- 
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index 7046dfd0d1c6..85f86faa3243 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -70,7 +70,7 @@ struct vmw_du_update_plane {
 	 *
 	 * Some surface resource or buffer object need some extra cmd submission
 	 * like update GB image for proxy surface and define a GMRFB for screen
-	 * object. That should should be done here as this callback will be
+	 * object. That should be done here as this callback will be
 	 * called after FIFO allocation with the address of command buufer.
 	 *
 	 * This callback is optional.
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index e4347faccee0..b8761f16dd78 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -28,7 +28,6 @@
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "vmwgfx_kms.h"
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c b/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c
index 2aceac7856e2..089046fa21be 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_msg.c
@@ -1076,6 +1076,7 @@ int vmw_mksstat_add_ioctl(struct drm_device *dev, void *data,
 
 	if (desc_len < 0) {
 		atomic_set(&dev_priv->mksstat_user_pids[slot], 0);
+		__free_page(page);
 		return -EFAULT;
 	}
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index a7d62a4eb47b..f66caa540e14 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -525,7 +525,7 @@ void vmw_resource_unreserve(struct vmw_resource *res,
  *                             for a resource and in that case, allocate
  *                             one, reserve and validate it.
  *
- * @ticket:         The ww aqcquire context to use, or NULL if trylocking.
+ * @ticket:         The ww acquire context to use, or NULL if trylocking.
  * @res:            The resource for which to allocate a backup buffer.
  * @interruptible:  Whether any sleeps during allocation should be
  *                  performed while interruptible.
@@ -686,7 +686,7 @@ out_no_unbind:
  * @intr: Perform waits interruptible if possible.
  * @dirtying: Pending GPU operation will dirty the resource
  *
- * On succesful return, any backup DMA buffer pointed to by @res->backup will
+ * On successful return, any backup DMA buffer pointed to by @res->backup will
  * be reserved and validated.
  * On hardware resource shortage, this function will repeatedly evict
  * resources of the same type until the validation succeeds.
@@ -804,7 +804,7 @@ void vmw_resource_unbind_list(struct vmw_buffer_object *vbo)
  * @dx_query_mob: Buffer containing the DX query MOB
  *
  * Read back cached states from the device if they exist.  This function
- * assumings binding_mutex is held.
+ * assumes binding_mutex is held.
  */
 int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob)
 {
@@ -1125,7 +1125,7 @@ int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start,
 	}
 
 	/*
-	 * In order of increasing backup_offset, clean dirty resorces
+	 * In order of increasing backup_offset, clean dirty resources
 	 * intersecting the range.
 	 */
 	while (found) {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index c89ad3a2d141..ecd3c2fc978b 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -29,7 +29,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "vmwgfx_kms.h"
@@ -1383,6 +1382,6 @@ out_revert:
 	vmw_validation_revert(&val_ctx);
 out_unref:
 	vmw_validation_unref_lists(&val_ctx);
-	
+
 	return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c
index 483ad544ea54..0d51b4542269 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_simple_resource.c
@@ -196,7 +196,7 @@ out_ret:
  * type.
  *
  * Returns: Refcounted pointer to the embedded struct vmw_resource if
- * successfule. Error pointer otherwise.
+ * successful. Error pointer otherwise.
  */
 struct vmw_resource *
 vmw_simple_resource_lookup(struct ttm_object_file *tfile,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index eb014b97d156..8650c3aea8f0 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -29,7 +29,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_vblank.h>
 
 #include "vmwgfx_kms.h"
diff --git a/drivers/gpu/drm/xlnx/Kconfig b/drivers/gpu/drm/xlnx/Kconfig
index f9cf93c9e7e3..68ee897de9d7 100644
--- a/drivers/gpu/drm/xlnx/Kconfig
+++ b/drivers/gpu/drm/xlnx/Kconfig
@@ -8,7 +8,7 @@ config DRM_ZYNQMP_DPSUB
 	select DMA_ENGINE
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HELPER
-	select DRM_GEM_CMA_HELPER
+	select DRM_GEM_DMA_HELPER
 	select DRM_KMS_HELPER
 	select GENERIC_PHY
 	help
diff --git a/drivers/gpu/drm/xlnx/zynqmp_disp.c b/drivers/gpu/drm/xlnx/zynqmp_disp.c
index cc32aa89cf8f..bbb365f2d087 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_disp.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_disp.c
@@ -15,12 +15,11 @@
 #include <drm/drm_blend.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_device.h>
-#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_framebuffer.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_plane.h>
-#include <drm/drm_plane_helper.h>
 #include <drm/drm_vblank.h>
 
 #include <linux/clk.h>
@@ -1099,14 +1098,14 @@ static int zynqmp_disp_layer_update(struct zynqmp_disp_layer *layer,
 		unsigned int height = state->crtc_h / (i ? info->vsub : 1);
 		struct zynqmp_disp_layer_dma *dma = &layer->dmas[i];
 		struct dma_async_tx_descriptor *desc;
-		dma_addr_t paddr;
+		dma_addr_t dma_addr;
 
-		paddr = drm_fb_cma_get_gem_addr(state->fb, state, i);
+		dma_addr = drm_fb_dma_get_gem_addr(state->fb, state, i);
 
 		dma->xt.numf = height;
 		dma->sgl.size = width * info->cpp[i];
 		dma->sgl.icg = state->fb->pitches[i] - dma->sgl.size;
-		dma->xt.src_start = paddr;
+		dma->xt.src_start = dma_addr;
 		dma->xt.frame_size = 1;
 		dma->xt.dir = DMA_MEM_TO_DEV;
 		dma->xt.src_sgl = true;
@@ -1151,8 +1150,8 @@ zynqmp_disp_plane_atomic_check(struct drm_plane *plane,
 
 	return drm_atomic_helper_check_plane_state(new_plane_state,
 						   crtc_state,
-						   DRM_PLANE_HELPER_NO_SCALING,
-						   DRM_PLANE_HELPER_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
+						   DRM_PLANE_NO_SCALING,
 						   false, false);
 }
 
diff --git a/drivers/gpu/drm/xlnx/zynqmp_dpsub.c b/drivers/gpu/drm/xlnx/zynqmp_dpsub.c
index 824b510e337b..1de2d927c32b 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_dpsub.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_dpsub.c
@@ -21,7 +21,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fourcc.h>
-#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_gem_dma_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_mode_config.h>
@@ -47,7 +47,7 @@ static int zynqmp_dpsub_dumb_create(struct drm_file *file_priv,
 	/* Enforce the alignment constraints of the DMA engine. */
 	args->pitch = ALIGN(pitch, dpsub->dma_align);
 
-	return drm_gem_cma_dumb_create_internal(file_priv, drm, args);
+	return drm_gem_dma_dumb_create_internal(file_priv, drm, args);
 }
 
 static struct drm_framebuffer *
@@ -75,13 +75,13 @@ static const struct drm_mode_config_funcs zynqmp_dpsub_mode_config_funcs = {
  * DRM/KMS Driver
  */
 
-DEFINE_DRM_GEM_CMA_FOPS(zynqmp_dpsub_drm_fops);
+DEFINE_DRM_GEM_DMA_FOPS(zynqmp_dpsub_drm_fops);
 
 static const struct drm_driver zynqmp_dpsub_drm_driver = {
 	.driver_features		= DRIVER_MODESET | DRIVER_GEM |
 					  DRIVER_ATOMIC,
 
-	DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(zynqmp_dpsub_dumb_create),
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(zynqmp_dpsub_dumb_create),
 
 	.fops				= &zynqmp_dpsub_drm_fops,
 
diff --git a/drivers/hid/i2c-hid/i2c-hid-core.c b/drivers/hid/i2c-hid/i2c-hid-core.c
index c078f09a2318..95cefae47adf 100644
--- a/drivers/hid/i2c-hid/i2c-hid-core.c
+++ b/drivers/hid/i2c-hid/i2c-hid-core.c
@@ -1064,7 +1064,7 @@ err_powered:
 }
 EXPORT_SYMBOL_GPL(i2c_hid_core_probe);
 
-int i2c_hid_core_remove(struct i2c_client *client)
+void i2c_hid_core_remove(struct i2c_client *client)
 {
 	struct i2c_hid *ihid = i2c_get_clientdata(client);
 	struct hid_device *hid;
@@ -1078,8 +1078,6 @@ int i2c_hid_core_remove(struct i2c_client *client)
 		i2c_hid_free_buffers(ihid);
 
 	i2c_hid_core_power_down(ihid);
-
-	return 0;
 }
 EXPORT_SYMBOL_GPL(i2c_hid_core_remove);
 
diff --git a/drivers/hid/i2c-hid/i2c-hid.h b/drivers/hid/i2c-hid/i2c-hid.h
index 236cc062d5ef..96c75510ad3f 100644
--- a/drivers/hid/i2c-hid/i2c-hid.h
+++ b/drivers/hid/i2c-hid/i2c-hid.h
@@ -33,7 +33,7 @@ struct i2chid_ops {
 
 int i2c_hid_core_probe(struct i2c_client *client, struct i2chid_ops *ops,
 		       u16 hid_descriptor_address, u32 quirks);
-int i2c_hid_core_remove(struct i2c_client *client);
+void i2c_hid_core_remove(struct i2c_client *client);
 
 void i2c_hid_core_shutdown(struct i2c_client *client);
 
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index 3c833ea60db6..7b9f3fc3adf7 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -2453,7 +2453,8 @@ static int vmbus_acpi_add(struct acpi_device *device)
 	 * Some ancestor of the vmbus acpi device (Gen1 or Gen2
 	 * firmware) is the VMOD that has the mmio ranges. Get that.
 	 */
-	for (ancestor = device->parent; ancestor; ancestor = ancestor->parent) {
+	for (ancestor = acpi_dev_parent(device); ancestor;
+	     ancestor = acpi_dev_parent(ancestor)) {
 		result = acpi_walk_resources(ancestor->handle, METHOD_NAME__CRS,
 					     vmbus_walk_resources, NULL);
 
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index e70d9614bec2..5695b266abcf 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -257,14 +257,14 @@ config SENSORS_AHT10
 	  will be called aht10.
 
 config SENSORS_AQUACOMPUTER_D5NEXT
-	tristate "Aquacomputer D5 Next, Octo, Quadro, Farbwerk, and Farbwerk 360"
+	tristate "Aquacomputer D5 Next, Octo, Quadro, Farbwerk, Farbwerk 360, High Flow Next"
 	depends on USB_HID
 	select CRC16
 	help
 	  If you say yes here you get support for sensors and fans of
 	  the Aquacomputer D5 Next watercooling pump, Octo and Quadro fan
-	  controllers, Farbwerk and Farbwerk 360 RGB controllers, where
-	  available.
+	  controllers, Farbwerk and Farbwerk 360 RGB controllers, High Flow
+	  Next sensor, where available.
 
 	  This driver can also be built as a module. If so, the module
 	  will be called aquacomputer_d5next.
@@ -393,6 +393,7 @@ config SENSORS_ASB100
 
 config SENSORS_ASPEED
 	tristate "ASPEED AST2400/AST2500 PWM and Fan tach driver"
+	depends on ARCH_ASPEED || COMPILE_TEST
 	depends on THERMAL || THERMAL=n
 	select REGMAP
 	help
@@ -1066,6 +1067,18 @@ config SENSORS_MAX31730
 	  This driver can also be built as a module. If so, the module
 	  will be called max31730.
 
+config SENSORS_MAX31760
+	tristate "MAX31760 fan speed controller"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Support for the Analog Devices MAX31760 Precision Fan-Speed
+	  Controller. MAX31760 integrates temperature sensing along with
+	  precision PWM fan control.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called max31760.
+
 config SENSORS_MAX6620
 	tristate "Maxim MAX6620 fan controller"
 	depends on I2C
@@ -1785,6 +1798,19 @@ config SENSORS_EMC2103
 	  This driver can also be built as a module. If so, the module
 	  will be called emc2103.
 
+config SENSORS_EMC2305
+	tristate "Microchip EMC2305 and compatible EMC2301/2/3"
+	depends on I2C
+	imply THERMAL
+	help
+	  If you say yes here you get support for the Microchip EMC2305
+	  fan controller chips.
+	  The Microchip EMC2305 is a fan controller for up to 5 fans.
+	  Fan rotation speeds are reported in RPM.
+
+	  This driver can also be built as a module.  If so, the module
+	  will be called emc2305.
+
 config SENSORS_EMC6W201
 	tristate "SMSC EMC6W201"
 	depends on I2C
@@ -2341,21 +2367,6 @@ config SENSORS_ASUS_WMI
 	  This driver can also be built as a module. If so, the module
 	  will be called asus_wmi_sensors.
 
-config SENSORS_ASUS_WMI_EC
-	tristate "ASUS WMI B550/X570"
-	depends on ACPI_WMI && SENSORS_ASUS_EC=n
-	help
-	  If you say yes here you get support for the ACPI embedded controller
-	  hardware monitoring interface found in B550/X570 ASUS motherboards.
-	  This driver will provide readings of fans, voltages and temperatures
-	  through the system firmware.
-
-	  This driver is deprecated in favor of the ASUS EC Sensors driver
-	  which provides fully compatible output.
-
-	  This driver can also be built as a module. If so, the module
-	  will be called asus_wmi_sensors_ec.
-
 config SENSORS_ASUS_EC
 	tristate "ASUS EC Sensors"
 	depends on X86
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 007e829d1d0d..11d076cad8a2 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -11,7 +11,6 @@ obj-$(CONFIG_SENSORS_ACPI_POWER) += acpi_power_meter.o
 obj-$(CONFIG_SENSORS_ATK0110)	+= asus_atk0110.o
 obj-$(CONFIG_SENSORS_ASUS_EC)	+= asus-ec-sensors.o
 obj-$(CONFIG_SENSORS_ASUS_WMI)	+= asus_wmi_sensors.o
-obj-$(CONFIG_SENSORS_ASUS_WMI_EC)	+= asus_wmi_ec_sensors.o
 
 # Native drivers
 # asb100, then w83781d go first, as they can override other drivers' addresses.
@@ -70,6 +69,7 @@ obj-$(CONFIG_SENSORS_DS620)	+= ds620.o
 obj-$(CONFIG_SENSORS_DS1621)	+= ds1621.o
 obj-$(CONFIG_SENSORS_EMC1403)	+= emc1403.o
 obj-$(CONFIG_SENSORS_EMC2103)	+= emc2103.o
+obj-$(CONFIG_SENSORS_EMC2305)	+= emc2305.o
 obj-$(CONFIG_SENSORS_EMC6W201)	+= emc6w201.o
 obj-$(CONFIG_SENSORS_F71805F)	+= f71805f.o
 obj-$(CONFIG_SENSORS_F71882FG)	+= f71882fg.o
@@ -140,6 +140,7 @@ obj-$(CONFIG_SENSORS_MAX1668)	+= max1668.o
 obj-$(CONFIG_SENSORS_MAX197)	+= max197.o
 obj-$(CONFIG_SENSORS_MAX31722)	+= max31722.o
 obj-$(CONFIG_SENSORS_MAX31730)	+= max31730.o
+obj-$(CONFIG_SENSORS_MAX31760)  += max31760.o
 obj-$(CONFIG_SENSORS_MAX6620)	+= max6620.o
 obj-$(CONFIG_SENSORS_MAX6621)	+= max6621.o
 obj-$(CONFIG_SENSORS_MAX6639)	+= max6639.o
diff --git a/drivers/hwmon/abituguru.c b/drivers/hwmon/abituguru.c
index 681f0623868f..a7cae6568155 100644
--- a/drivers/hwmon/abituguru.c
+++ b/drivers/hwmon/abituguru.c
@@ -1504,7 +1504,6 @@ LEAVE_UPDATE:
 		return NULL;
 }
 
-#ifdef CONFIG_PM_SLEEP
 static int abituguru_suspend(struct device *dev)
 {
 	struct abituguru_data *data = dev_get_drvdata(dev);
@@ -1526,16 +1525,12 @@ static int abituguru_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(abituguru_pm, abituguru_suspend, abituguru_resume);
-#define ABIT_UGURU_PM	(&abituguru_pm)
-#else
-#define ABIT_UGURU_PM	NULL
-#endif /* CONFIG_PM */
+static DEFINE_SIMPLE_DEV_PM_OPS(abituguru_pm, abituguru_suspend, abituguru_resume);
 
 static struct platform_driver abituguru_driver = {
 	.driver = {
 		.name	= ABIT_UGURU_NAME,
-		.pm	= ABIT_UGURU_PM,
+		.pm	= pm_sleep_ptr(&abituguru_pm),
 	},
 	.probe		= abituguru_probe,
 	.remove		= abituguru_remove,
diff --git a/drivers/hwmon/abituguru3.c b/drivers/hwmon/abituguru3.c
index 8229ad30c909..afb21f73032d 100644
--- a/drivers/hwmon/abituguru3.c
+++ b/drivers/hwmon/abituguru3.c
@@ -1127,7 +1127,6 @@ LEAVE_UPDATE:
 		return NULL;
 }
 
-#ifdef CONFIG_PM_SLEEP
 static int abituguru3_suspend(struct device *dev)
 {
 	struct abituguru3_data *data = dev_get_drvdata(dev);
@@ -1146,16 +1145,12 @@ static int abituguru3_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(abituguru3_pm, abituguru3_suspend, abituguru3_resume);
-#define ABIT_UGURU3_PM	(&abituguru3_pm)
-#else
-#define ABIT_UGURU3_PM	NULL
-#endif /* CONFIG_PM */
+static DEFINE_SIMPLE_DEV_PM_OPS(abituguru3_pm, abituguru3_suspend, abituguru3_resume);
 
 static struct platform_driver abituguru3_driver = {
 	.driver = {
 		.name	= ABIT_UGURU3_NAME,
-		.pm	= ABIT_UGURU3_PM
+		.pm	= pm_sleep_ptr(&abituguru3_pm),
 	},
 	.probe	= abituguru3_probe,
 	.remove	= abituguru3_remove,
diff --git a/drivers/hwmon/acpi_power_meter.c b/drivers/hwmon/acpi_power_meter.c
index d2545a1be9fc..0962c12eba5a 100644
--- a/drivers/hwmon/acpi_power_meter.c
+++ b/drivers/hwmon/acpi_power_meter.c
@@ -598,7 +598,7 @@ static int read_domain_devices(struct acpi_power_meter_resource *resource)
 			continue;
 
 		/* Create a symlink to domain objects */
-		obj = acpi_bus_get_acpi_device(element->reference.handle);
+		obj = acpi_get_acpi_dev(element->reference.handle);
 		resource->domain_devices[i] = obj;
 		if (!obj)
 			continue;
@@ -927,8 +927,6 @@ static int acpi_power_meter_remove(struct acpi_device *device)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-
 static int acpi_power_meter_resume(struct device *dev)
 {
 	struct acpi_power_meter_resource *resource;
@@ -946,9 +944,8 @@ static int acpi_power_meter_resume(struct device *dev)
 	return 0;
 }
 
-#endif /* CONFIG_PM_SLEEP */
-
-static SIMPLE_DEV_PM_OPS(acpi_power_meter_pm, NULL, acpi_power_meter_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(acpi_power_meter_pm, NULL,
+				acpi_power_meter_resume);
 
 static struct acpi_driver acpi_power_meter_driver = {
 	.name = "power_meter",
@@ -959,7 +956,7 @@ static struct acpi_driver acpi_power_meter_driver = {
 		.remove = acpi_power_meter_remove,
 		.notify = acpi_power_meter_notify,
 		},
-	.drv.pm = &acpi_power_meter_pm,
+	.drv.pm = pm_sleep_ptr(&acpi_power_meter_pm),
 };
 
 /* Module init/exit routines */
diff --git a/drivers/hwmon/adc128d818.c b/drivers/hwmon/adc128d818.c
index fd938c70293f..97b330b6c165 100644
--- a/drivers/hwmon/adc128d818.c
+++ b/drivers/hwmon/adc128d818.c
@@ -384,7 +384,7 @@ static int adc128_detect(struct i2c_client *client, struct i2c_board_info *info)
 	if (i2c_smbus_read_byte_data(client, ADC128_REG_BUSY_STATUS) & 0xfc)
 		return -ENODEV;
 
-	strlcpy(info->type, "adc128d818", I2C_NAME_SIZE);
+	strscpy(info->type, "adc128d818", I2C_NAME_SIZE);
 
 	return 0;
 }
@@ -495,14 +495,12 @@ error:
 	return err;
 }
 
-static int adc128_remove(struct i2c_client *client)
+static void adc128_remove(struct i2c_client *client)
 {
 	struct adc128_data *data = i2c_get_clientdata(client);
 
 	if (data->regulator)
 		regulator_disable(data->regulator);
-
-	return 0;
 }
 
 static const struct i2c_device_id adc128_id[] = {
diff --git a/drivers/hwmon/adm1021.c b/drivers/hwmon/adm1021.c
index 91ecfee243bf..2dc45e958730 100644
--- a/drivers/hwmon/adm1021.c
+++ b/drivers/hwmon/adm1021.c
@@ -426,7 +426,7 @@ static int adm1021_detect(struct i2c_client *client,
 
 	pr_debug("Detected chip %s at adapter %d, address 0x%02x.\n",
 		 type_name, i2c_adapter_id(adapter), client->addr);
-	strlcpy(info->type, type_name, I2C_NAME_SIZE);
+	strscpy(info->type, type_name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/adm1025.c b/drivers/hwmon/adm1025.c
index 4352f6a884e8..2984c4f98496 100644
--- a/drivers/hwmon/adm1025.c
+++ b/drivers/hwmon/adm1025.c
@@ -470,7 +470,7 @@ static int adm1025_detect(struct i2c_client *client,
 	else
 		return -ENODEV;
 
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/adm1026.c b/drivers/hwmon/adm1026.c
index 69b3ec752944..1f084f708743 100644
--- a/drivers/hwmon/adm1026.c
+++ b/drivers/hwmon/adm1026.c
@@ -1610,7 +1610,7 @@ static int adm1026_detect(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, "adm1026", I2C_NAME_SIZE);
+	strscpy(info->type, "adm1026", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/adm1029.c b/drivers/hwmon/adm1029.c
index 3e1999413f32..eaf6e5e04aac 100644
--- a/drivers/hwmon/adm1029.c
+++ b/drivers/hwmon/adm1029.c
@@ -329,7 +329,7 @@ static int adm1029_detect(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, "adm1029", I2C_NAME_SIZE);
+	strscpy(info->type, "adm1029", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/adm1031.c b/drivers/hwmon/adm1031.c
index ac841fa3a369..b42797bcb5b4 100644
--- a/drivers/hwmon/adm1031.c
+++ b/drivers/hwmon/adm1031.c
@@ -985,7 +985,7 @@ static int adm1031_detect(struct i2c_client *client,
 		return -ENODEV;
 	name = (id == 0x30) ? "adm1030" : "adm1031";
 
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/adm9240.c b/drivers/hwmon/adm9240.c
index 483cd757abd3..40e3558d3709 100644
--- a/drivers/hwmon/adm9240.c
+++ b/drivers/hwmon/adm9240.c
@@ -501,17 +501,23 @@ static int adm9240_fan_read(struct device *dev, u32 attr, int channel, long *val
 
 	switch (attr) {
 	case hwmon_fan_input:
+		mutex_lock(&data->update_lock);
 		err = regmap_read(data->regmap, ADM9240_REG_FAN(channel), &regval);
-		if (err < 0)
+		if (err < 0) {
+			mutex_unlock(&data->update_lock);
 			return err;
+		}
 		if (regval == 255 && data->fan_div[channel] < 3) {
 			/* adjust fan clock divider on overflow */
 			err = adm9240_write_fan_div(data, channel,
 						    ++data->fan_div[channel]);
-			if (err)
+			if (err) {
+				mutex_unlock(&data->update_lock);
 				return err;
+			}
 		}
 		*val = FAN_FROM_REG(regval, BIT(data->fan_div[channel]));
+		mutex_unlock(&data->update_lock);
 		break;
 	case hwmon_fan_div:
 		*val = BIT(data->fan_div[channel]);
diff --git a/drivers/hwmon/adt7310.c b/drivers/hwmon/adt7310.c
index 1efc0bdcceab..067865f4887a 100644
--- a/drivers/hwmon/adt7310.c
+++ b/drivers/hwmon/adt7310.c
@@ -152,7 +152,7 @@ MODULE_DEVICE_TABLE(spi, adt7310_id);
 static struct spi_driver adt7310_driver = {
 	.driver = {
 		.name	= "adt7310",
-		.pm	= ADT7X10_DEV_PM_OPS,
+		.pm	= pm_sleep_ptr(&adt7x10_dev_pm_ops),
 	},
 	.probe		= adt7310_spi_probe,
 	.id_table	= adt7310_id,
diff --git a/drivers/hwmon/adt7410.c b/drivers/hwmon/adt7410.c
index aede5baca7b9..0cebf6777239 100644
--- a/drivers/hwmon/adt7410.c
+++ b/drivers/hwmon/adt7410.c
@@ -98,7 +98,7 @@ static struct i2c_driver adt7410_driver = {
 	.class		= I2C_CLASS_HWMON,
 	.driver = {
 		.name	= "adt7410",
-		.pm	= ADT7X10_DEV_PM_OPS,
+		.pm	= pm_sleep_ptr(&adt7x10_dev_pm_ops),
 	},
 	.probe_new	= adt7410_i2c_probe,
 	.id_table	= adt7410_ids,
diff --git a/drivers/hwmon/adt7411.c b/drivers/hwmon/adt7411.c
index fad74aa62b64..bf5c5618f8d0 100644
--- a/drivers/hwmon/adt7411.c
+++ b/drivers/hwmon/adt7411.c
@@ -590,7 +590,7 @@ static int adt7411_detect(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, "adt7411", I2C_NAME_SIZE);
+	strscpy(info->type, "adt7411", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/adt7462.c b/drivers/hwmon/adt7462.c
index e75bbd87ad09..9c0235849d4b 100644
--- a/drivers/hwmon/adt7462.c
+++ b/drivers/hwmon/adt7462.c
@@ -1782,7 +1782,7 @@ static int adt7462_detect(struct i2c_client *client,
 	if (revision != ADT7462_REVISION)
 		return -ENODEV;
 
-	strlcpy(info->type, "adt7462", I2C_NAME_SIZE);
+	strscpy(info->type, "adt7462", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c
index c67cd037a93f..927f8df05b7c 100644
--- a/drivers/hwmon/adt7470.c
+++ b/drivers/hwmon/adt7470.c
@@ -1296,12 +1296,11 @@ static int adt7470_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int adt7470_remove(struct i2c_client *client)
+static void adt7470_remove(struct i2c_client *client)
 {
 	struct adt7470_data *data = i2c_get_clientdata(client);
 
 	kthread_stop(data->auto_update);
-	return 0;
 }
 
 static const struct i2c_device_id adt7470_id[] = {
diff --git a/drivers/hwmon/adt7475.c b/drivers/hwmon/adt7475.c
index ac480e6e4818..51b3d16c3223 100644
--- a/drivers/hwmon/adt7475.c
+++ b/drivers/hwmon/adt7475.c
@@ -1342,7 +1342,7 @@ static int adt7475_detect(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/adt7x10.c b/drivers/hwmon/adt7x10.c
index ce54bffab2ec..da67734edafd 100644
--- a/drivers/hwmon/adt7x10.c
+++ b/drivers/hwmon/adt7x10.c
@@ -397,8 +397,6 @@ int adt7x10_probe(struct device *dev, const char *name, int irq,
 }
 EXPORT_SYMBOL_GPL(adt7x10_probe);
 
-#ifdef CONFIG_PM_SLEEP
-
 static int adt7x10_suspend(struct device *dev)
 {
 	struct adt7x10_data *data = dev_get_drvdata(dev);
@@ -414,10 +412,7 @@ static int adt7x10_resume(struct device *dev)
 	return regmap_write(data->regmap, ADT7X10_CONFIG, data->config);
 }
 
-SIMPLE_DEV_PM_OPS(adt7x10_dev_pm_ops, adt7x10_suspend, adt7x10_resume);
-EXPORT_SYMBOL_GPL(adt7x10_dev_pm_ops);
-
-#endif /* CONFIG_PM_SLEEP */
+EXPORT_SIMPLE_DEV_PM_OPS(adt7x10_dev_pm_ops, adt7x10_suspend, adt7x10_resume);
 
 MODULE_AUTHOR("Hartmut Knaack");
 MODULE_DESCRIPTION("ADT7410/ADT7420, ADT7310/ADT7320 common code");
diff --git a/drivers/hwmon/adt7x10.h b/drivers/hwmon/adt7x10.h
index ba22c32c8355..46caf3e21978 100644
--- a/drivers/hwmon/adt7x10.h
+++ b/drivers/hwmon/adt7x10.h
@@ -20,11 +20,6 @@ struct device;
 int adt7x10_probe(struct device *dev, const char *name, int irq,
 		  struct regmap *regmap);
 
-#ifdef CONFIG_PM_SLEEP
 extern const struct dev_pm_ops adt7x10_dev_pm_ops;
-#define ADT7X10_DEV_PM_OPS (&adt7x10_dev_pm_ops)
-#else
-#define ADT7X10_DEV_PM_OPS NULL
-#endif
 
 #endif
diff --git a/drivers/hwmon/amc6821.c b/drivers/hwmon/amc6821.c
index 0c16face3fd3..3bfd12ff4b3c 100644
--- a/drivers/hwmon/amc6821.c
+++ b/drivers/hwmon/amc6821.c
@@ -809,7 +809,7 @@ static int amc6821_detect(
 	}
 
 	dev_info(&adapter->dev, "amc6821: chip found at 0x%02x.\n", address);
-	strlcpy(info->type, "amc6821", I2C_NAME_SIZE);
+	strscpy(info->type, "amc6821", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/aquacomputer_d5next.c b/drivers/hwmon/aquacomputer_d5next.c
index 66430553cc45..c51a2678f0eb 100644
--- a/drivers/hwmon/aquacomputer_d5next.c
+++ b/drivers/hwmon/aquacomputer_d5next.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0+
 /*
  * hwmon driver for Aquacomputer devices (D5 Next, Farbwerk, Farbwerk 360, Octo,
- * Quadro)
+ * Quadro, High Flow Next)
  *
  * Aquacomputer devices send HID reports (with ID 0x01) every second to report
  * sensor values.
@@ -26,15 +26,17 @@
 #define USB_PRODUCT_ID_D5NEXT		0xf00e
 #define USB_PRODUCT_ID_FARBWERK360	0xf010
 #define USB_PRODUCT_ID_OCTO		0xf011
+#define USB_PRODUCT_ID_HIGHFLOWNEXT	0xf012
 
-enum kinds { d5next, farbwerk, farbwerk360, octo, quadro };
+enum kinds { d5next, farbwerk, farbwerk360, octo, quadro, highflownext };
 
 static const char *const aqc_device_names[] = {
 	[d5next] = "d5next",
 	[farbwerk] = "farbwerk",
 	[farbwerk360] = "farbwerk360",
 	[octo] = "octo",
-	[quadro] = "quadro"
+	[quadro] = "quadro",
+	[highflownext] = "highflownext"
 };
 
 #define DRIVER_NAME			"aquacomputer_d5next"
@@ -71,6 +73,8 @@ static u8 secondary_ctrl_report[] = {
 #define D5NEXT_COOLANT_TEMP		0x57
 #define D5NEXT_NUM_FANS			2
 #define D5NEXT_NUM_SENSORS		1
+#define D5NEXT_NUM_VIRTUAL_SENSORS	8
+#define D5NEXT_VIRTUAL_SENSORS_START	0x3f
 #define D5NEXT_PUMP_OFFSET		0x6c
 #define D5NEXT_FAN_OFFSET		0x5f
 #define D5NEXT_5V_VOLTAGE		0x39
@@ -86,14 +90,18 @@ static u16 d5next_ctrl_fan_offsets[] = { 0x97, 0x42 };
 #define FARBWERK_SENSOR_START		0x2f
 
 /* Register offsets for the Farbwerk 360 RGB controller */
-#define FARBWERK360_NUM_SENSORS		4
-#define FARBWERK360_SENSOR_START	0x32
+#define FARBWERK360_NUM_SENSORS			4
+#define FARBWERK360_SENSOR_START		0x32
+#define FARBWERK360_NUM_VIRTUAL_SENSORS		16
+#define FARBWERK360_VIRTUAL_SENSORS_START	0x3a
 
 /* Register offsets for the Octo fan controller */
 #define OCTO_POWER_CYCLES		0x18
 #define OCTO_NUM_FANS			8
 #define OCTO_NUM_SENSORS		4
 #define OCTO_SENSOR_START		0x3D
+#define OCTO_NUM_VIRTUAL_SENSORS	16
+#define OCTO_VIRTUAL_SENSORS_START	0x45
 #define OCTO_CTRL_REPORT_SIZE		0x65F
 static u8 octo_sensor_fan_offsets[] = { 0x7D, 0x8A, 0x97, 0xA4, 0xB1, 0xBE, 0xCB, 0xD8 };
 
@@ -105,12 +113,24 @@ static u16 octo_ctrl_fan_offsets[] = { 0x5B, 0xB0, 0x105, 0x15A, 0x1AF, 0x204, 0
 #define QUADRO_NUM_FANS			4
 #define QUADRO_NUM_SENSORS		4
 #define QUADRO_SENSOR_START		0x34
+#define QUADRO_NUM_VIRTUAL_SENSORS	16
+#define QUADRO_VIRTUAL_SENSORS_START	0x3c
 #define QUADRO_CTRL_REPORT_SIZE		0x3c1
 #define QUADRO_FLOW_SENSOR_OFFSET	0x6e
 static u8 quadro_sensor_fan_offsets[] = { 0x70, 0x7D, 0x8A, 0x97 };
 
 /* Fan speed registers in Quadro control report (from 0-100%) */
-static u16 quadro_ctrl_fan_offsets[] = { 0x36, 0x8b, 0xe0, 0x135 };
+static u16 quadro_ctrl_fan_offsets[] = { 0x37, 0x8c, 0xe1, 0x136 };
+
+/* Register offsets for the High Flow Next */
+#define HIGHFLOWNEXT_NUM_SENSORS	2
+#define HIGHFLOWNEXT_SENSOR_START	85
+#define HIGHFLOWNEXT_FLOW		81
+#define HIGHFLOWNEXT_WATER_QUALITY	89
+#define HIGHFLOWNEXT_POWER		91
+#define HIGHFLOWNEXT_CONDUCTIVITY	95
+#define HIGHFLOWNEXT_5V_VOLTAGE		97
+#define HIGHFLOWNEXT_5V_VOLTAGE_USB	99
 
 /* Labels for D5 Next */
 static const char *const label_d5next_temp[] = {
@@ -147,6 +167,25 @@ static const char *const label_temp_sensors[] = {
 	"Sensor 4"
 };
 
+static const char *const label_virtual_temp_sensors[] = {
+	"Virtual sensor 1",
+	"Virtual sensor 2",
+	"Virtual sensor 3",
+	"Virtual sensor 4",
+	"Virtual sensor 5",
+	"Virtual sensor 6",
+	"Virtual sensor 7",
+	"Virtual sensor 8",
+	"Virtual sensor 9",
+	"Virtual sensor 10",
+	"Virtual sensor 11",
+	"Virtual sensor 12",
+	"Virtual sensor 13",
+	"Virtual sensor 14",
+	"Virtual sensor 15",
+	"Virtual sensor 16",
+};
+
 /* Labels for Octo and Quadro (except speed) */
 static const char *const label_fan_speed[] = {
 	"Fan 1 speed",
@@ -201,6 +240,27 @@ static const char *const label_quadro_speeds[] = {
 	"Flow speed [dL/h]"
 };
 
+/* Labels for High Flow Next */
+static const char *const label_highflownext_temp_sensors[] = {
+	"Coolant temp",
+	"External sensor"
+};
+
+static const char *const label_highflownext_fan_speed[] = {
+	"Flow [dL/h]",
+	"Water quality [%]",
+	"Conductivity [nS/cm]",
+};
+
+static const char *const label_highflownext_power[] = {
+	"Dissipated power",
+};
+
+static const char *const label_highflownext_voltage[] = {
+	"+5V voltage",
+	"+5V USB voltage"
+};
+
 struct aqc_data {
 	struct hid_device *hdev;
 	struct device *hwmon_dev;
@@ -220,6 +280,8 @@ struct aqc_data {
 	u16 *fan_ctrl_offsets;
 	int num_temp_sensors;
 	int temp_sensor_start_offset;
+	int num_virtual_temp_sensors;
+	int virtual_temp_sensor_start_offset;
 	u16 power_cycle_count_offset;
 	u8 flow_sensor_offset;
 
@@ -231,7 +293,7 @@ struct aqc_data {
 	u32 power_cycles;
 
 	/* Sensor values */
-	s32 temp_input[4];
+	s32 temp_input[20];	/* Max 4 physical and 16 virtual */
 	u16 speed_input[8];
 	u32 power_input[8];
 	u16 voltage_input[8];
@@ -239,6 +301,7 @@ struct aqc_data {
 
 	/* Label values */
 	const char *const *temp_label;
+	const char *const *virtual_temp_label;
 	const char *const *speed_label;
 	const char *const *power_label;
 	const char *const *voltage_label;
@@ -345,7 +408,7 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3
 
 	switch (type) {
 	case hwmon_temp:
-		if (channel < priv->num_temp_sensors)
+		if (channel < priv->num_temp_sensors + priv->num_virtual_temp_sensors)
 			return 0444;
 		break;
 	case hwmon_pwm:
@@ -360,6 +423,11 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3
 		break;
 	case hwmon_fan:
 		switch (priv->kind) {
+		case highflownext:
+			/* Special case to support flow sensor, water quality and conductivity */
+			if (channel < 3)
+				return 0444;
+			break;
 		case quadro:
 			/* Special case to support flow sensor */
 			if (channel < priv->num_fans + 1)
@@ -372,6 +440,18 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3
 		}
 		break;
 	case hwmon_power:
+		switch (priv->kind) {
+		case highflownext:
+			/* Special case to support one power sensor */
+			if (channel == 0)
+				return 0444;
+			break;
+		default:
+			if (channel < priv->num_fans)
+				return 0444;
+			break;
+		}
+		break;
 	case hwmon_curr:
 		if (channel < priv->num_fans)
 			return 0444;
@@ -383,6 +463,11 @@ static umode_t aqc_is_visible(const void *data, enum hwmon_sensor_types type, u3
 			if (channel < priv->num_fans + 2)
 				return 0444;
 			break;
+		case highflownext:
+			/* Special case to support two voltage sensors */
+			if (channel < 2)
+				return 0444;
+			break;
 		default:
 			if (channel < priv->num_fans)
 				return 0444;
@@ -447,7 +532,10 @@ static int aqc_read_string(struct device *dev, enum hwmon_sensor_types type, u32
 
 	switch (type) {
 	case hwmon_temp:
-		*str = priv->temp_label[channel];
+		if (channel < priv->num_temp_sensors)
+			*str = priv->temp_label[channel];
+		else
+			*str = priv->virtual_temp_label[channel - priv->num_temp_sensors];
 		break;
 	case hwmon_fan:
 		*str = priv->speed_label[channel];
@@ -512,6 +600,22 @@ static const struct hwmon_channel_info *aqc_info[] = {
 			   HWMON_T_INPUT | HWMON_T_LABEL,
 			   HWMON_T_INPUT | HWMON_T_LABEL,
 			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_LABEL,
 			   HWMON_T_INPUT | HWMON_T_LABEL),
 	HWMON_CHANNEL_INFO(fan,
 			   HWMON_F_INPUT | HWMON_F_LABEL,
@@ -568,7 +672,7 @@ static const struct hwmon_chip_info aqc_chip_info = {
 
 static int aqc_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *data, int size)
 {
-	int i, sensor_value;
+	int i, j, sensor_value;
 	struct aqc_data *priv;
 
 	if (report->id != STATUS_REPORT_ID)
@@ -581,7 +685,7 @@ static int aqc_raw_event(struct hid_device *hdev, struct hid_report *report, u8
 	priv->serial_number[1] = get_unaligned_be16(data + SERIAL_SECOND_PART);
 	priv->firmware_version = get_unaligned_be16(data + FIRMWARE_VERSION);
 
-	/* Temperature sensor readings */
+	/* Physical temperature sensor readings */
 	for (i = 0; i < priv->num_temp_sensors; i++) {
 		sensor_value = get_unaligned_be16(data +
 						  priv->temp_sensor_start_offset +
@@ -592,6 +696,18 @@ static int aqc_raw_event(struct hid_device *hdev, struct hid_report *report, u8
 			priv->temp_input[i] = sensor_value * 10;
 	}
 
+	/* Virtual temperature sensor readings */
+	for (j = 0; j < priv->num_virtual_temp_sensors; j++) {
+		sensor_value = get_unaligned_be16(data +
+						  priv->virtual_temp_sensor_start_offset +
+						  j * AQC_TEMP_SENSOR_SIZE);
+		if (sensor_value == AQC_TEMP_SENSOR_DISCONNECTED)
+			priv->temp_input[i] = -ENODATA;
+		else
+			priv->temp_input[i] = sensor_value * 10;
+		i++;
+	}
+
 	/* Fan speed and related readings */
 	for (i = 0; i < priv->num_fans; i++) {
 		priv->speed_input[i] =
@@ -618,6 +734,22 @@ static int aqc_raw_event(struct hid_device *hdev, struct hid_report *report, u8
 	case quadro:
 		priv->speed_input[4] = get_unaligned_be16(data + priv->flow_sensor_offset);
 		break;
+	case highflownext:
+		/* If external temp sensor is not connected, its power reading is also N/A */
+		if (priv->temp_input[1] == -ENODATA)
+			priv->power_input[0] = -ENODATA;
+		else
+			priv->power_input[0] =
+			    get_unaligned_be16(data + HIGHFLOWNEXT_POWER) * 1000000;
+
+		priv->voltage_input[0] = get_unaligned_be16(data + HIGHFLOWNEXT_5V_VOLTAGE) * 10;
+		priv->voltage_input[1] =
+		    get_unaligned_be16(data + HIGHFLOWNEXT_5V_VOLTAGE_USB) * 10;
+
+		priv->speed_input[0] = get_unaligned_be16(data + HIGHFLOWNEXT_FLOW);
+		priv->speed_input[1] = get_unaligned_be16(data + HIGHFLOWNEXT_WATER_QUALITY);
+		priv->speed_input[2] = get_unaligned_be16(data + HIGHFLOWNEXT_CONDUCTIVITY);
+		break;
 	default:
 		break;
 	}
@@ -717,10 +849,13 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		priv->fan_ctrl_offsets = d5next_ctrl_fan_offsets;
 		priv->num_temp_sensors = D5NEXT_NUM_SENSORS;
 		priv->temp_sensor_start_offset = D5NEXT_COOLANT_TEMP;
+		priv->num_virtual_temp_sensors = D5NEXT_NUM_VIRTUAL_SENSORS;
+		priv->virtual_temp_sensor_start_offset = D5NEXT_VIRTUAL_SENSORS_START;
 		priv->power_cycle_count_offset = D5NEXT_POWER_CYCLES;
 		priv->buffer_size = D5NEXT_CTRL_REPORT_SIZE;
 
 		priv->temp_label = label_d5next_temp;
+		priv->virtual_temp_label = label_virtual_temp_sensors;
 		priv->speed_label = label_d5next_speeds;
 		priv->power_label = label_d5next_power;
 		priv->voltage_label = label_d5next_voltages;
@@ -740,7 +875,11 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		priv->num_fans = 0;
 		priv->num_temp_sensors = FARBWERK360_NUM_SENSORS;
 		priv->temp_sensor_start_offset = FARBWERK360_SENSOR_START;
+		priv->num_virtual_temp_sensors = FARBWERK360_NUM_VIRTUAL_SENSORS;
+		priv->virtual_temp_sensor_start_offset = FARBWERK360_VIRTUAL_SENSORS_START;
+
 		priv->temp_label = label_temp_sensors;
+		priv->virtual_temp_label = label_virtual_temp_sensors;
 		break;
 	case USB_PRODUCT_ID_OCTO:
 		priv->kind = octo;
@@ -750,10 +889,13 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		priv->fan_ctrl_offsets = octo_ctrl_fan_offsets;
 		priv->num_temp_sensors = OCTO_NUM_SENSORS;
 		priv->temp_sensor_start_offset = OCTO_SENSOR_START;
+		priv->num_virtual_temp_sensors = OCTO_NUM_VIRTUAL_SENSORS;
+		priv->virtual_temp_sensor_start_offset = OCTO_VIRTUAL_SENSORS_START;
 		priv->power_cycle_count_offset = OCTO_POWER_CYCLES;
 		priv->buffer_size = OCTO_CTRL_REPORT_SIZE;
 
 		priv->temp_label = label_temp_sensors;
+		priv->virtual_temp_label = label_virtual_temp_sensors;
 		priv->speed_label = label_fan_speed;
 		priv->power_label = label_fan_power;
 		priv->voltage_label = label_fan_voltage;
@@ -767,16 +909,32 @@ static int aqc_probe(struct hid_device *hdev, const struct hid_device_id *id)
 		priv->fan_ctrl_offsets = quadro_ctrl_fan_offsets;
 		priv->num_temp_sensors = QUADRO_NUM_SENSORS;
 		priv->temp_sensor_start_offset = QUADRO_SENSOR_START;
+		priv->num_virtual_temp_sensors = QUADRO_NUM_VIRTUAL_SENSORS;
+		priv->virtual_temp_sensor_start_offset = QUADRO_VIRTUAL_SENSORS_START;
 		priv->power_cycle_count_offset = QUADRO_POWER_CYCLES;
 		priv->buffer_size = QUADRO_CTRL_REPORT_SIZE;
 		priv->flow_sensor_offset = QUADRO_FLOW_SENSOR_OFFSET;
 
 		priv->temp_label = label_temp_sensors;
+		priv->virtual_temp_label = label_virtual_temp_sensors;
 		priv->speed_label = label_quadro_speeds;
 		priv->power_label = label_fan_power;
 		priv->voltage_label = label_fan_voltage;
 		priv->current_label = label_fan_current;
 		break;
+	case USB_PRODUCT_ID_HIGHFLOWNEXT:
+		priv->kind = highflownext;
+
+		priv->num_fans = 0;
+		priv->num_temp_sensors = HIGHFLOWNEXT_NUM_SENSORS;
+		priv->temp_sensor_start_offset = HIGHFLOWNEXT_SENSOR_START;
+		priv->power_cycle_count_offset = QUADRO_POWER_CYCLES;
+
+		priv->temp_label = label_highflownext_temp_sensors;
+		priv->speed_label = label_highflownext_fan_speed;
+		priv->power_label = label_highflownext_power;
+		priv->voltage_label = label_highflownext_voltage;
+		break;
 	default:
 		break;
 	}
@@ -833,6 +991,7 @@ static const struct hid_device_id aqc_table[] = {
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_FARBWERK360) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_OCTO) },
 	{ HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_QUADRO) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_AQUACOMPUTER, USB_PRODUCT_ID_HIGHFLOWNEXT) },
 	{ }
 };
 
diff --git a/drivers/hwmon/asb100.c b/drivers/hwmon/asb100.c
index 8cf0bcb85eb4..ce4da836765c 100644
--- a/drivers/hwmon/asb100.c
+++ b/drivers/hwmon/asb100.c
@@ -208,7 +208,7 @@ static void asb100_write_value(struct i2c_client *client, u16 reg, u16 val);
 static int asb100_probe(struct i2c_client *client);
 static int asb100_detect(struct i2c_client *client,
 			 struct i2c_board_info *info);
-static int asb100_remove(struct i2c_client *client);
+static void asb100_remove(struct i2c_client *client);
 static struct asb100_data *asb100_update_device(struct device *dev);
 static void asb100_init_client(struct i2c_client *client);
 
@@ -769,7 +769,7 @@ static int asb100_detect(struct i2c_client *client,
 	if (val1 != 0x31 || val2 != 0x06)
 		return -ENODEV;
 
-	strlcpy(info->type, "asb100", I2C_NAME_SIZE);
+	strscpy(info->type, "asb100", I2C_NAME_SIZE);
 
 	return 0;
 }
@@ -822,7 +822,7 @@ ERROR3:
 	return err;
 }
 
-static int asb100_remove(struct i2c_client *client)
+static void asb100_remove(struct i2c_client *client)
 {
 	struct asb100_data *data = i2c_get_clientdata(client);
 
@@ -831,8 +831,6 @@ static int asb100_remove(struct i2c_client *client)
 
 	i2c_unregister_device(data->lm75[1]);
 	i2c_unregister_device(data->lm75[0]);
-
-	return 0;
 }
 
 /*
diff --git a/drivers/hwmon/asc7621.c b/drivers/hwmon/asc7621.c
index e835605a7456..54595454537b 100644
--- a/drivers/hwmon/asc7621.c
+++ b/drivers/hwmon/asc7621.c
@@ -1153,7 +1153,7 @@ static int asc7621_detect(struct i2c_client *client,
 
 		if (company == asc7621_chips[chip_index].company_id &&
 		    verstep == asc7621_chips[chip_index].verstep_id) {
-			strlcpy(info->type, asc7621_chips[chip_index].name,
+			strscpy(info->type, asc7621_chips[chip_index].name,
 				I2C_NAME_SIZE);
 
 			dev_info(&adapter->dev, "Matched %s at 0x%02x\n",
@@ -1165,7 +1165,7 @@ static int asc7621_detect(struct i2c_client *client,
 	return -ENODEV;
 }
 
-static int asc7621_remove(struct i2c_client *client)
+static void asc7621_remove(struct i2c_client *client)
 {
 	struct asc7621_data *data = i2c_get_clientdata(client);
 	int i;
@@ -1176,8 +1176,6 @@ static int asc7621_remove(struct i2c_client *client)
 		device_remove_file(&client->dev,
 				   &(asc7621_params[i].sda.dev_attr));
 	}
-
-	return 0;
 }
 
 static const struct i2c_device_id asc7621_id[] = {
diff --git a/drivers/hwmon/asus_wmi_ec_sensors.c b/drivers/hwmon/asus_wmi_ec_sensors.c
deleted file mode 100644
index a3a2f014dec0..000000000000
--- a/drivers/hwmon/asus_wmi_ec_sensors.c
+++ /dev/null
@@ -1,622 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * HWMON driver for ASUS B550/X570 motherboards that publish sensor
- * values via the embedded controller registers.
- *
- * Copyright (C) 2021 Eugene Shalygin <eugene.shalygin@gmail.com>
- * Copyright (C) 2018-2019 Ed Brindley <kernel@maidavale.org>
- *
- * EC provides:
- * - Chipset temperature
- * - CPU temperature
- * - Motherboard temperature
- * - T_Sensor temperature
- * - VRM temperature
- * - Water In temperature
- * - Water Out temperature
- * - CPU Optional Fan RPM
- * - Chipset Fan RPM
- * - Water Flow Fan RPM
- * - CPU current
- */
-
-#include <linux/acpi.h>
-#include <linux/dmi.h>
-#include <linux/hwmon.h>
-#include <linux/init.h>
-#include <linux/jiffies.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/nls.h>
-#include <linux/units.h>
-#include <linux/wmi.h>
-
-#include <asm/unaligned.h>
-
-#define ASUSWMI_MONITORING_GUID		"466747A0-70EC-11DE-8A39-0800200C9A66"
-#define ASUSWMI_METHODID_BLOCK_READ_EC	0x42524543 /* BREC */
-/* From the ASUS DSDT source */
-#define ASUSWMI_BREC_REGISTERS_MAX	16
-#define ASUSWMI_MAX_BUF_LEN		128
-#define SENSOR_LABEL_LEN		16
-
-static u32 hwmon_attributes[hwmon_max] = {
-	[hwmon_chip]	= HWMON_C_REGISTER_TZ,
-	[hwmon_temp]	= HWMON_T_INPUT | HWMON_T_LABEL,
-	[hwmon_in]	= HWMON_I_INPUT | HWMON_I_LABEL,
-	[hwmon_curr]	= HWMON_C_INPUT | HWMON_C_LABEL,
-	[hwmon_fan]	= HWMON_F_INPUT | HWMON_F_LABEL,
-};
-
-struct asus_wmi_ec_sensor_address {
-	u8 index;
-	u8 bank;
-	u8 size;
-};
-
-#define MAKE_SENSOR_ADDRESS(size_i, bank_i, index_i) {	\
-	.size = size_i,					\
-	.bank = bank_i,					\
-	.index = index_i,				\
-}
-
-struct ec_sensor_info {
-	struct asus_wmi_ec_sensor_address addr;
-	char label[SENSOR_LABEL_LEN];
-	enum hwmon_sensor_types type;
-};
-
-#define EC_SENSOR(sensor_label, sensor_type, size, bank, index) {	\
-	.addr = MAKE_SENSOR_ADDRESS(size, bank, index),			\
-	.label = sensor_label,						\
-	.type = sensor_type,						\
-}
-
-enum known_ec_sensor {
-	SENSOR_TEMP_CHIPSET,
-	SENSOR_TEMP_CPU,
-	SENSOR_TEMP_MB,
-	SENSOR_TEMP_T_SENSOR,
-	SENSOR_TEMP_VRM,
-	SENSOR_FAN_CPU_OPT,
-	SENSOR_FAN_CHIPSET,
-	SENSOR_FAN_VRM_HS,
-	SENSOR_FAN_WATER_FLOW,
-	SENSOR_CURR_CPU,
-	SENSOR_TEMP_WATER_IN,
-	SENSOR_TEMP_WATER_OUT,
-	SENSOR_MAX
-};
-
-/* All known sensors for ASUS EC controllers */
-static const struct ec_sensor_info known_ec_sensors[] = {
-	[SENSOR_TEMP_CHIPSET]	= EC_SENSOR("Chipset", hwmon_temp, 1, 0x00, 0x3a),
-	[SENSOR_TEMP_CPU]	= EC_SENSOR("CPU", hwmon_temp, 1, 0x00, 0x3b),
-	[SENSOR_TEMP_MB]	= EC_SENSOR("Motherboard", hwmon_temp, 1, 0x00, 0x3c),
-	[SENSOR_TEMP_T_SENSOR]	= EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x3d),
-	[SENSOR_TEMP_VRM]	= EC_SENSOR("VRM", hwmon_temp, 1, 0x00, 0x3e),
-	[SENSOR_FAN_CPU_OPT]	= EC_SENSOR("CPU_Opt", hwmon_fan, 2, 0x00, 0xb0),
-	[SENSOR_FAN_VRM_HS]	= EC_SENSOR("VRM HS", hwmon_fan, 2, 0x00, 0xb2),
-	[SENSOR_FAN_CHIPSET]	= EC_SENSOR("Chipset", hwmon_fan, 2, 0x00, 0xb4),
-	[SENSOR_FAN_WATER_FLOW]	= EC_SENSOR("Water_Flow", hwmon_fan, 2, 0x00, 0xbc),
-	[SENSOR_CURR_CPU]	= EC_SENSOR("CPU", hwmon_curr, 1, 0x00, 0xf4),
-	[SENSOR_TEMP_WATER_IN]	= EC_SENSOR("Water_In", hwmon_temp, 1, 0x01, 0x00),
-	[SENSOR_TEMP_WATER_OUT]	= EC_SENSOR("Water_Out", hwmon_temp, 1, 0x01, 0x01),
-};
-
-struct asus_wmi_data {
-	const enum known_ec_sensor known_board_sensors[SENSOR_MAX + 1];
-};
-
-/* boards with EC support */
-static struct asus_wmi_data sensors_board_PW_X570_P = {
-	.known_board_sensors = {
-		SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB,
-		SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM,
-		SENSOR_FAN_CHIPSET,
-		SENSOR_MAX
-	},
-};
-
-static struct asus_wmi_data sensors_board_PW_X570_A = {
-	.known_board_sensors = {
-		SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB, SENSOR_TEMP_VRM,
-		SENSOR_FAN_CHIPSET,
-		SENSOR_CURR_CPU,
-		SENSOR_MAX
-	},
-};
-
-static struct asus_wmi_data sensors_board_R_C8H = {
-	.known_board_sensors = {
-		SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB,
-		SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM,
-		SENSOR_TEMP_WATER_IN, SENSOR_TEMP_WATER_OUT,
-		SENSOR_FAN_CPU_OPT, SENSOR_FAN_CHIPSET, SENSOR_FAN_WATER_FLOW,
-		SENSOR_CURR_CPU,
-		SENSOR_MAX
-	},
-};
-
-/* Same as Hero but without chipset fan */
-static struct asus_wmi_data sensors_board_R_C8DH = {
-	.known_board_sensors = {
-		SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB,
-		SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM,
-		SENSOR_TEMP_WATER_IN, SENSOR_TEMP_WATER_OUT,
-		SENSOR_FAN_CPU_OPT, SENSOR_FAN_WATER_FLOW,
-		SENSOR_CURR_CPU,
-		SENSOR_MAX
-	},
-};
-
-/* Same as Hero but without water */
-static struct asus_wmi_data sensors_board_R_C8F = {
-	.known_board_sensors = {
-		SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB,
-		SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM,
-		SENSOR_FAN_CPU_OPT, SENSOR_FAN_CHIPSET,
-		SENSOR_CURR_CPU,
-		SENSOR_MAX
-	},
-};
-
-static struct asus_wmi_data sensors_board_RS_B550_E_G = {
-	.known_board_sensors = {
-		SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB,
-		SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM,
-		SENSOR_FAN_CPU_OPT,
-		SENSOR_MAX
-	},
-};
-
-static struct asus_wmi_data sensors_board_RS_B550_I_G = {
-	.known_board_sensors = {
-		SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB,
-		SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM,
-		SENSOR_FAN_VRM_HS,
-		SENSOR_CURR_CPU,
-		SENSOR_MAX
-	},
-};
-
-static struct asus_wmi_data sensors_board_RS_X570_E_G = {
-	.known_board_sensors = {
-		SENSOR_TEMP_CHIPSET, SENSOR_TEMP_CPU, SENSOR_TEMP_MB,
-		SENSOR_TEMP_T_SENSOR, SENSOR_TEMP_VRM,
-		SENSOR_FAN_CHIPSET,
-		SENSOR_CURR_CPU,
-		SENSOR_MAX
-	},
-};
-
-#define DMI_EXACT_MATCH_ASUS_BOARD_NAME(name, sensors) {			\
-	.matches = {								\
-		DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."),	\
-		DMI_EXACT_MATCH(DMI_BOARD_NAME, name),				\
-	},									\
-	.driver_data = sensors,							\
-}
-
-static const struct dmi_system_id asus_wmi_ec_dmi_table[] = {
-	DMI_EXACT_MATCH_ASUS_BOARD_NAME("PRIME X570-PRO", &sensors_board_PW_X570_P),
-	DMI_EXACT_MATCH_ASUS_BOARD_NAME("Pro WS X570-ACE", &sensors_board_PW_X570_A),
-	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII DARK HERO", &sensors_board_R_C8DH),
-	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII FORMULA", &sensors_board_R_C8F),
-	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG CROSSHAIR VIII HERO", &sensors_board_R_C8H),
-	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX B550-E GAMING", &sensors_board_RS_B550_E_G),
-	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX B550-I GAMING", &sensors_board_RS_B550_I_G),
-	DMI_EXACT_MATCH_ASUS_BOARD_NAME("ROG STRIX X570-E GAMING", &sensors_board_RS_X570_E_G),
-	{}
-};
-MODULE_DEVICE_TABLE(dmi, asus_wmi_ec_dmi_table);
-
-struct ec_sensor {
-	enum known_ec_sensor info_index;
-	long cached_value;
-};
-
-/**
- * struct asus_wmi_ec_info - sensor info.
- * @sensors: list of sensors.
- * @read_arg: UTF-16LE string to pass to BRxx() WMI function.
- * @read_buffer: decoded output from WMI result.
- * @nr_sensors: number of board EC sensors.
- * @nr_registers: number of EC registers to read (sensor might span more than 1 register).
- * @last_updated: in jiffies.
- */
-struct asus_wmi_ec_info {
-	struct ec_sensor sensors[SENSOR_MAX];
-	char read_arg[(ASUSWMI_BREC_REGISTERS_MAX * 4 + 1) * 2];
-	u8 read_buffer[ASUSWMI_BREC_REGISTERS_MAX];
-	unsigned int nr_sensors;
-	unsigned int nr_registers;
-	unsigned long last_updated;
-};
-
-struct asus_wmi_sensors {
-	struct asus_wmi_ec_info ec;
-	/* lock access to internal cache */
-	struct mutex lock;
-};
-
-static int asus_wmi_ec_fill_board_sensors(struct asus_wmi_ec_info *ec,
-					  const enum known_ec_sensor *bsi)
-{
-	struct ec_sensor *s = ec->sensors;
-	int i;
-
-	ec->nr_sensors = 0;
-	ec->nr_registers = 0;
-
-	for (i = 0; bsi[i] != SENSOR_MAX; i++) {
-		s[i].info_index = bsi[i];
-		ec->nr_sensors++;
-		ec->nr_registers += known_ec_sensors[bsi[i]].addr.size;
-	}
-
-	return 0;
-}
-
-/*
- * The next four functions convert to or from BRxx string argument format.
- * The format of the string is as follows:
- * - The string consists of two-byte UTF-16LE characters.
- * - The value of the very first byte in the string is equal to the total
- *   length of the next string in bytes, thus excluding the first two-byte
- *   character.
- * - The rest of the string encodes the pairs of (bank, index) pairs, where
- *   both values are byte-long (0x00 to 0xFF).
- * - Numbers are encoded as UTF-16LE hex values.
- */
-static int asus_wmi_ec_decode_reply_buffer(const u8 *in, u32 length, u8 *out)
-{
-	char buffer[ASUSWMI_MAX_BUF_LEN * 2];
-	u32 len = min_t(u32, get_unaligned_le16(in), length - 2);
-
-	utf16s_to_utf8s((wchar_t *)(in + 2), len / 2, UTF16_LITTLE_ENDIAN, buffer, sizeof(buffer));
-
-	return hex2bin(out, buffer, len / 4);
-}
-
-static void asus_wmi_ec_encode_registers(const u8 *in, u32 len, char *out)
-{
-	char buffer[ASUSWMI_MAX_BUF_LEN * 2];
-
-	bin2hex(buffer, in, len);
-
-	utf8s_to_utf16s(buffer, len * 2, UTF16_LITTLE_ENDIAN, (wchar_t *)(out + 2), len * 2);
-
-	put_unaligned_le16(len * 4, out);
-}
-
-static void asus_wmi_ec_make_block_read_query(struct asus_wmi_ec_info *ec)
-{
-	u8 registers[ASUSWMI_BREC_REGISTERS_MAX * 2];
-	const struct ec_sensor_info *si;
-	int i, j, offset;
-
-	offset = 0;
-	for (i = 0; i < ec->nr_sensors; i++) {
-		si = &known_ec_sensors[ec->sensors[i].info_index];
-		for (j = 0; j < si->addr.size; j++) {
-			registers[offset++] = si->addr.bank;
-			registers[offset++] = si->addr.index + j;
-		}
-	}
-
-	asus_wmi_ec_encode_registers(registers, offset, ec->read_arg);
-}
-
-static int asus_wmi_ec_block_read(u32 method_id, char *query, u8 *out)
-{
-	struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
-	struct acpi_buffer input;
-	union acpi_object *obj;
-	acpi_status status;
-	int ret;
-
-	/* The first byte of the BRxx() argument string has to be the string size. */
-	input.length = query[0] + 2;
-	input.pointer = query;
-	status = wmi_evaluate_method(ASUSWMI_MONITORING_GUID, 0, method_id, &input, &output);
-	if (ACPI_FAILURE(status))
-		return -EIO;
-
-	obj = output.pointer;
-	if (!obj)
-		return -EIO;
-
-	if (obj->type != ACPI_TYPE_BUFFER || obj->buffer.length < 2) {
-		ret = -EIO;
-		goto out_free_obj;
-	}
-
-	ret = asus_wmi_ec_decode_reply_buffer(obj->buffer.pointer, obj->buffer.length, out);
-
-out_free_obj:
-	ACPI_FREE(obj);
-	return ret;
-}
-
-static inline long get_sensor_value(const struct ec_sensor_info *si, u8 *data)
-{
-	switch (si->addr.size) {
-	case 1:
-		return *data;
-	case 2:
-		return get_unaligned_be16(data);
-	case 4:
-		return get_unaligned_be32(data);
-	default:
-		return 0;
-	}
-}
-
-static void asus_wmi_ec_update_ec_sensors(struct asus_wmi_ec_info *ec)
-{
-	const struct ec_sensor_info *si;
-	struct ec_sensor *s;
-	u8 i_sensor;
-	u8 *data;
-
-	data = ec->read_buffer;
-	for (i_sensor = 0; i_sensor < ec->nr_sensors; i_sensor++) {
-		s = &ec->sensors[i_sensor];
-		si = &known_ec_sensors[s->info_index];
-		s->cached_value = get_sensor_value(si, data);
-		data += si->addr.size;
-	}
-}
-
-static long asus_wmi_ec_scale_sensor_value(long value, int data_type)
-{
-	switch (data_type) {
-	case hwmon_curr:
-	case hwmon_temp:
-	case hwmon_in:
-		return value * MILLI;
-	default:
-		return value;
-	}
-}
-
-static int asus_wmi_ec_find_sensor_index(const struct asus_wmi_ec_info *ec,
-					 enum hwmon_sensor_types type, int channel)
-{
-	int i;
-
-	for (i = 0; i < ec->nr_sensors; i++) {
-		if (known_ec_sensors[ec->sensors[i].info_index].type == type) {
-			if (channel == 0)
-				return i;
-
-			channel--;
-		}
-	}
-	return -EINVAL;
-}
-
-static int asus_wmi_ec_get_cached_value_or_update(struct asus_wmi_sensors *sensor_data,
-						  int sensor_index,
-						  long *value)
-{
-	struct asus_wmi_ec_info *ec = &sensor_data->ec;
-	int ret = 0;
-
-	mutex_lock(&sensor_data->lock);
-
-	if (time_after(jiffies, ec->last_updated + HZ)) {
-		ret = asus_wmi_ec_block_read(ASUSWMI_METHODID_BLOCK_READ_EC,
-					     ec->read_arg, ec->read_buffer);
-		if (ret)
-			goto unlock;
-
-		asus_wmi_ec_update_ec_sensors(ec);
-		ec->last_updated = jiffies;
-	}
-
-	*value = ec->sensors[sensor_index].cached_value;
-
-unlock:
-	mutex_unlock(&sensor_data->lock);
-
-	return ret;
-}
-
-/* Now follow the functions that implement the hwmon interface */
-
-static int asus_wmi_ec_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
-				  u32 attr, int channel, long *val)
-{
-	struct asus_wmi_sensors *sensor_data = dev_get_drvdata(dev);
-	struct asus_wmi_ec_info *ec = &sensor_data->ec;
-	int ret, sidx, info_index;
-	long value = 0;
-
-	sidx = asus_wmi_ec_find_sensor_index(ec, type, channel);
-	if (sidx < 0)
-		return sidx;
-
-	ret = asus_wmi_ec_get_cached_value_or_update(sensor_data, sidx, &value);
-	if (ret)
-		return ret;
-
-	info_index = ec->sensors[sidx].info_index;
-	*val = asus_wmi_ec_scale_sensor_value(value, known_ec_sensors[info_index].type);
-
-	return ret;
-}
-
-static int asus_wmi_ec_hwmon_read_string(struct device *dev,
-					 enum hwmon_sensor_types type, u32 attr,
-					 int channel, const char **str)
-{
-	struct asus_wmi_sensors *sensor_data = dev_get_drvdata(dev);
-	struct asus_wmi_ec_info *ec = &sensor_data->ec;
-	int sensor_index;
-
-	sensor_index = asus_wmi_ec_find_sensor_index(ec, type, channel);
-	*str = known_ec_sensors[ec->sensors[sensor_index].info_index].label;
-
-	return 0;
-}
-
-static umode_t asus_wmi_ec_hwmon_is_visible(const void *drvdata,
-					    enum hwmon_sensor_types type, u32 attr,
-					    int channel)
-{
-	const struct asus_wmi_sensors *sensor_data = drvdata;
-	const struct asus_wmi_ec_info *ec = &sensor_data->ec;
-	int index;
-
-	index = asus_wmi_ec_find_sensor_index(ec, type, channel);
-
-	return index < 0 ? 0 : 0444;
-}
-
-static int asus_wmi_hwmon_add_chan_info(struct hwmon_channel_info *asus_wmi_hwmon_chan,
-					struct device *dev, int num,
-					enum hwmon_sensor_types type, u32 config)
-{
-	u32 *cfg;
-
-	cfg = devm_kcalloc(dev, num + 1, sizeof(*cfg), GFP_KERNEL);
-	if (!cfg)
-		return -ENOMEM;
-
-	asus_wmi_hwmon_chan->type = type;
-	asus_wmi_hwmon_chan->config = cfg;
-	memset32(cfg, config, num);
-
-	return 0;
-}
-
-static const struct hwmon_ops asus_wmi_ec_hwmon_ops = {
-	.is_visible = asus_wmi_ec_hwmon_is_visible,
-	.read = asus_wmi_ec_hwmon_read,
-	.read_string = asus_wmi_ec_hwmon_read_string,
-};
-
-static struct hwmon_chip_info asus_wmi_ec_chip_info = {
-	.ops = &asus_wmi_ec_hwmon_ops,
-};
-
-static int asus_wmi_ec_configure_sensor_setup(struct device *dev,
-					      const enum known_ec_sensor *bsi)
-{
-	struct asus_wmi_sensors *sensor_data = dev_get_drvdata(dev);
-	struct asus_wmi_ec_info *ec = &sensor_data->ec;
-	struct hwmon_channel_info *asus_wmi_hwmon_chan;
-	const struct hwmon_channel_info **asus_wmi_ci;
-	int nr_count[hwmon_max] = {}, nr_types = 0;
-	const struct hwmon_chip_info *chip_info;
-	const struct ec_sensor_info *si;
-	enum hwmon_sensor_types type;
-	struct device *hwdev;
-	int i, ret;
-
-	ret = asus_wmi_ec_fill_board_sensors(ec, bsi);
-	if (ret)
-		return ret;
-
-	if (!sensor_data->ec.nr_sensors)
-		return -ENODEV;
-
-	for (i = 0; i < ec->nr_sensors; i++) {
-		si = &known_ec_sensors[ec->sensors[i].info_index];
-		if (!nr_count[si->type])
-			nr_types++;
-		nr_count[si->type]++;
-	}
-
-	if (nr_count[hwmon_temp]) {
-		nr_count[hwmon_chip]++;
-		nr_types++;
-	}
-
-	/*
-	 * If we can get values for all the registers in a single query,
-	 * the query will not change from call to call.
-	 */
-	asus_wmi_ec_make_block_read_query(ec);
-
-	asus_wmi_hwmon_chan = devm_kcalloc(dev, nr_types, sizeof(*asus_wmi_hwmon_chan),
-					   GFP_KERNEL);
-	if (!asus_wmi_hwmon_chan)
-		return -ENOMEM;
-
-	asus_wmi_ci = devm_kcalloc(dev, nr_types + 1, sizeof(*asus_wmi_ci), GFP_KERNEL);
-	if (!asus_wmi_ci)
-		return -ENOMEM;
-
-	asus_wmi_ec_chip_info.info = asus_wmi_ci;
-	chip_info = &asus_wmi_ec_chip_info;
-
-	for (type = 0; type < hwmon_max; type++) {
-		if (!nr_count[type])
-			continue;
-
-		ret = asus_wmi_hwmon_add_chan_info(asus_wmi_hwmon_chan, dev,
-						   nr_count[type], type,
-						   hwmon_attributes[type]);
-		if (ret)
-			return ret;
-
-		*asus_wmi_ci++ = asus_wmi_hwmon_chan++;
-	}
-
-	dev_dbg(dev, "board has %d EC sensors that span %d registers",
-		ec->nr_sensors, ec->nr_registers);
-
-	hwdev = devm_hwmon_device_register_with_info(dev, "asus_wmi_ec_sensors",
-						     sensor_data, chip_info, NULL);
-
-	return PTR_ERR_OR_ZERO(hwdev);
-}
-
-static int asus_wmi_probe(struct wmi_device *wdev, const void *context)
-{
-	struct asus_wmi_sensors *sensor_data;
-	struct asus_wmi_data *board_sensors;
-	const struct dmi_system_id *dmi_id;
-	const enum known_ec_sensor *bsi;
-	struct device *dev = &wdev->dev;
-
-	dmi_id = dmi_first_match(asus_wmi_ec_dmi_table);
-	if (!dmi_id)
-		return -ENODEV;
-
-	board_sensors = dmi_id->driver_data;
-	bsi = board_sensors->known_board_sensors;
-
-	sensor_data = devm_kzalloc(dev, sizeof(*sensor_data), GFP_KERNEL);
-	if (!sensor_data)
-		return -ENOMEM;
-
-	mutex_init(&sensor_data->lock);
-
-	dev_set_drvdata(dev, sensor_data);
-
-	return asus_wmi_ec_configure_sensor_setup(dev, bsi);
-}
-
-static const struct wmi_device_id asus_ec_wmi_id_table[] = {
-	{ ASUSWMI_MONITORING_GUID, NULL },
-	{ }
-};
-
-static struct wmi_driver asus_sensors_wmi_driver = {
-	.driver = {
-		.name = "asus_wmi_ec_sensors",
-	},
-	.id_table = asus_ec_wmi_id_table,
-	.probe = asus_wmi_probe,
-};
-module_wmi_driver(asus_sensors_wmi_driver);
-
-MODULE_AUTHOR("Ed Brindley <kernel@maidavale.org>");
-MODULE_AUTHOR("Eugene Shalygin <eugene.shalygin@gmail.com>");
-MODULE_DESCRIPTION("Asus WMI Sensors Driver");
-MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/axi-fan-control.c b/drivers/hwmon/axi-fan-control.c
index 96c4a5c45291..6724e0dd3088 100644
--- a/drivers/hwmon/axi-fan-control.c
+++ b/drivers/hwmon/axi-fan-control.c
@@ -394,11 +394,6 @@ static int axi_fan_control_init(struct axi_fan_control_data *ctl,
 	return ret;
 }
 
-static void axi_fan_control_clk_disable(void *clk)
-{
-	clk_disable_unprepare(clk);
-}
-
 static const struct hwmon_channel_info *axi_fan_control_info[] = {
 	HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT),
 	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT | HWMON_F_FAULT | HWMON_F_LABEL),
@@ -478,20 +473,12 @@ static int axi_fan_control_probe(struct platform_device *pdev)
 	if (IS_ERR(ctl->base))
 		return PTR_ERR(ctl->base);
 
-	clk = devm_clk_get(&pdev->dev, NULL);
+	clk = devm_clk_get_enabled(&pdev->dev, NULL);
 	if (IS_ERR(clk)) {
 		dev_err(&pdev->dev, "clk_get failed with %ld\n", PTR_ERR(clk));
 		return PTR_ERR(clk);
 	}
 
-	ret = clk_prepare_enable(clk);
-	if (ret)
-		return ret;
-
-	ret = devm_add_action_or_reset(&pdev->dev, axi_fan_control_clk_disable, clk);
-	if (ret)
-		return ret;
-
 	ctl->clk_rate = clk_get_rate(clk);
 	if (!ctl->clk_rate)
 		return -EINVAL;
diff --git a/drivers/hwmon/corsair-psu.c b/drivers/hwmon/corsair-psu.c
index 14389fd7afb8..345d883ab044 100644
--- a/drivers/hwmon/corsair-psu.c
+++ b/drivers/hwmon/corsair-psu.c
@@ -55,6 +55,7 @@
 #define SECONDS_PER_DAY		(SECONDS_PER_HOUR * 24)
 #define RAIL_COUNT		3 /* 3v3 + 5v + 12v */
 #define TEMP_COUNT		2
+#define OCP_MULTI_RAIL		0x02
 
 #define PSU_CMD_SELECT_RAIL	0x00 /* expects length 2 */
 #define PSU_CMD_RAIL_VOLTS_HCRIT 0x40 /* the rest of the commands expect length 3 */
@@ -71,9 +72,10 @@
 #define PSU_CMD_RAIL_WATTS	0x96
 #define PSU_CMD_VEND_STR	0x99
 #define PSU_CMD_PROD_STR	0x9A
-#define PSU_CMD_TOTAL_WATTS	0xEE
 #define PSU_CMD_TOTAL_UPTIME	0xD1
 #define PSU_CMD_UPTIME		0xD2
+#define PSU_CMD_OCPMODE		0xD8
+#define PSU_CMD_TOTAL_WATTS	0xEE
 #define PSU_CMD_INIT		0xFE
 
 #define L_IN_VOLTS		"v_in"
@@ -268,6 +270,7 @@ static int corsairpsu_get_value(struct corsairpsu_data *priv, u8 cmd, u8 rail, l
 		break;
 	case PSU_CMD_TOTAL_UPTIME:
 	case PSU_CMD_UPTIME:
+	case PSU_CMD_OCPMODE:
 		*val = tmp;
 		break;
 	default:
@@ -660,6 +663,29 @@ static int product_show(struct seq_file *seqf, void *unused)
 }
 DEFINE_SHOW_ATTRIBUTE(product);
 
+static int ocpmode_show(struct seq_file *seqf, void *unused)
+{
+	struct corsairpsu_data *priv = seqf->private;
+	long val;
+	int ret;
+
+	/*
+	 * The rail mode is switchable on the fly. The RAW interface can be used for this. But it
+	 * will not be included here, because I consider it somewhat dangerous for the health of the
+	 * PSU. The returned value can be a bogus one, if the PSU is in the process of switching and
+	 * getting of the value itself can also fail during this. Because of this every other value
+	 * than OCP_MULTI_RAIL can be considered as "single rail".
+	 */
+	ret = corsairpsu_get_value(priv, PSU_CMD_OCPMODE, 0, &val);
+	if (ret < 0)
+		seq_puts(seqf, "N/A\n");
+	else
+		seq_printf(seqf, "%s\n", (val == OCP_MULTI_RAIL) ? "multi rail" : "single rail");
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(ocpmode);
+
 static void corsairpsu_debugfs_init(struct corsairpsu_data *priv)
 {
 	char name[32];
@@ -671,6 +697,7 @@ static void corsairpsu_debugfs_init(struct corsairpsu_data *priv)
 	debugfs_create_file("uptime_total", 0444, priv->debugfs, priv, &uptime_total_fops);
 	debugfs_create_file("vendor", 0444, priv->debugfs, priv, &vendor_fops);
 	debugfs_create_file("product", 0444, priv->debugfs, priv, &product_fops);
+	debugfs_create_file("ocpmode", 0444, priv->debugfs, priv, &ocpmode_fops);
 }
 
 #else
@@ -786,13 +813,14 @@ static const struct hid_device_id corsairpsu_idtable[] = {
 	{ HID_USB_DEVICE(0x1b1c, 0x1c04) }, /* Corsair HX650i */
 	{ HID_USB_DEVICE(0x1b1c, 0x1c05) }, /* Corsair HX750i */
 	{ HID_USB_DEVICE(0x1b1c, 0x1c06) }, /* Corsair HX850i */
-	{ HID_USB_DEVICE(0x1b1c, 0x1c07) }, /* Corsair HX1000i */
+	{ HID_USB_DEVICE(0x1b1c, 0x1c07) }, /* Corsair HX1000i revision 1 */
 	{ HID_USB_DEVICE(0x1b1c, 0x1c08) }, /* Corsair HX1200i */
 	{ HID_USB_DEVICE(0x1b1c, 0x1c09) }, /* Corsair RM550i */
 	{ HID_USB_DEVICE(0x1b1c, 0x1c0a) }, /* Corsair RM650i */
 	{ HID_USB_DEVICE(0x1b1c, 0x1c0b) }, /* Corsair RM750i */
 	{ HID_USB_DEVICE(0x1b1c, 0x1c0c) }, /* Corsair RM850i */
 	{ HID_USB_DEVICE(0x1b1c, 0x1c0d) }, /* Corsair RM1000i */
+	{ HID_USB_DEVICE(0x1b1c, 0x1c1e) }, /* Corsaur HX1000i revision 2 */
 	{ },
 };
 MODULE_DEVICE_TABLE(hid, corsairpsu_idtable);
diff --git a/drivers/hwmon/dell-smm-hwmon.c b/drivers/hwmon/dell-smm-hwmon.c
index 7f8d95dd2717..1572b5416015 100644
--- a/drivers/hwmon/dell-smm-hwmon.c
+++ b/drivers/hwmon/dell-smm-hwmon.c
@@ -1355,15 +1355,21 @@ static int __init dell_smm_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, data);
 
 	if (dmi_check_system(i8k_blacklist_fan_support_dmi_table)) {
-		dev_warn(&pdev->dev, "broken Dell BIOS detected, disallow fan support\n");
-		if (!force)
+		if (!force) {
+			dev_notice(&pdev->dev, "Disabling fan support due to BIOS bugs\n");
 			data->disallow_fan_support = true;
+		} else {
+			dev_warn(&pdev->dev, "Enabling fan support despite BIOS bugs\n");
+		}
 	}
 
 	if (dmi_check_system(i8k_blacklist_fan_type_dmi_table)) {
-		dev_warn(&pdev->dev, "broken Dell BIOS detected, disallow fan type call\n");
-		if (!force)
+		if (!force) {
+			dev_notice(&pdev->dev, "Disabling fan type call due to BIOS bugs\n");
 			data->disallow_fan_type_call = true;
+		} else {
+			dev_warn(&pdev->dev, "Enabling fan type call despite BIOS bugs\n");
+		}
 	}
 
 	strscpy(data->bios_version, i8k_get_dmi_data(DMI_BIOS_VERSION),
diff --git a/drivers/hwmon/dme1737.c b/drivers/hwmon/dme1737.c
index e3ad4c2d0038..66c48f70fae7 100644
--- a/drivers/hwmon/dme1737.c
+++ b/drivers/hwmon/dme1737.c
@@ -2456,7 +2456,7 @@ static int dme1737_i2c_detect(struct i2c_client *client,
 	dev_info(dev, "Found a %s chip at 0x%02x (rev 0x%02x).\n",
 		 verstep == SCH5027_VERSTEP ? "SCH5027" : "DME1737",
 		 client->addr, verstep);
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 
 	return 0;
 }
@@ -2508,14 +2508,12 @@ exit_remove:
 	return err;
 }
 
-static int dme1737_i2c_remove(struct i2c_client *client)
+static void dme1737_i2c_remove(struct i2c_client *client)
 {
 	struct dme1737_data *data = i2c_get_clientdata(client);
 
 	hwmon_device_unregister(data->hwmon_dev);
 	dme1737_remove_files(&client->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id dme1737_id[] = {
diff --git a/drivers/hwmon/emc1403.c b/drivers/hwmon/emc1403.c
index 314838272049..61d59189a6d1 100644
--- a/drivers/hwmon/emc1403.c
+++ b/drivers/hwmon/emc1403.c
@@ -329,22 +329,22 @@ static int emc1403_detect(struct i2c_client *client,
 	id = i2c_smbus_read_byte_data(client, THERMAL_PID_REG);
 	switch (id) {
 	case 0x20:
-		strlcpy(info->type, "emc1402", I2C_NAME_SIZE);
+		strscpy(info->type, "emc1402", I2C_NAME_SIZE);
 		break;
 	case 0x21:
-		strlcpy(info->type, "emc1403", I2C_NAME_SIZE);
+		strscpy(info->type, "emc1403", I2C_NAME_SIZE);
 		break;
 	case 0x22:
-		strlcpy(info->type, "emc1422", I2C_NAME_SIZE);
+		strscpy(info->type, "emc1422", I2C_NAME_SIZE);
 		break;
 	case 0x23:
-		strlcpy(info->type, "emc1423", I2C_NAME_SIZE);
+		strscpy(info->type, "emc1423", I2C_NAME_SIZE);
 		break;
 	case 0x25:
-		strlcpy(info->type, "emc1404", I2C_NAME_SIZE);
+		strscpy(info->type, "emc1404", I2C_NAME_SIZE);
 		break;
 	case 0x27:
-		strlcpy(info->type, "emc1424", I2C_NAME_SIZE);
+		strscpy(info->type, "emc1424", I2C_NAME_SIZE);
 		break;
 	default:
 		return -ENODEV;
diff --git a/drivers/hwmon/emc2103.c b/drivers/hwmon/emc2103.c
index e4c95ca9e19f..361cf9292456 100644
--- a/drivers/hwmon/emc2103.c
+++ b/drivers/hwmon/emc2103.c
@@ -643,7 +643,7 @@ emc2103_detect(struct i2c_client *new_client, struct i2c_board_info *info)
 	if ((product != 0x24) && (product != 0x26))
 		return -ENODEV;
 
-	strlcpy(info->type, "emc2103", I2C_NAME_SIZE);
+	strscpy(info->type, "emc2103", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/emc2305.c b/drivers/hwmon/emc2305.c
new file mode 100644
index 000000000000..aa1f25add0b6
--- /dev/null
+++ b/drivers/hwmon/emc2305.c
@@ -0,0 +1,620 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Hardware monitoring driver for EMC2305 fan controller
+ *
+ * Copyright (C) 2022 Nvidia Technologies Ltd.
+ */
+
+#include <linux/err.h>
+#include <linux/hwmon.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/platform_data/emc2305.h>
+#include <linux/thermal.h>
+
+static const unsigned short
+emc2305_normal_i2c[] = { 0x27, 0x2c, 0x2d, 0x2e, 0x2f, 0x4c, 0x4d, I2C_CLIENT_END };
+
+#define EMC2305_REG_DRIVE_FAIL_STATUS	0x27
+#define EMC2305_REG_DEVICE		0xfd
+#define EMC2305_REG_VENDOR		0xfe
+#define EMC2305_FAN_MAX			0xff
+#define EMC2305_FAN_MIN			0x00
+#define EMC2305_FAN_MAX_STATE		10
+#define EMC2305_DEVICE			0x34
+#define EMC2305_VENDOR			0x5d
+#define EMC2305_REG_PRODUCT_ID		0xfd
+#define EMC2305_TACH_REGS_UNUSE_BITS	3
+#define EMC2305_TACH_CNT_MULTIPLIER	0x02
+#define EMC2305_TACH_RANGE_MIN		480
+
+#define EMC2305_PWM_DUTY2STATE(duty, max_state, pwm_max) \
+	DIV_ROUND_CLOSEST((duty) * (max_state), (pwm_max))
+#define EMC2305_PWM_STATE2DUTY(state, max_state, pwm_max) \
+	DIV_ROUND_CLOSEST((state) * (pwm_max), (max_state))
+
+/*
+ * Factor by equations [2] and [3] from data sheet; valid for fans where the number of edges
+ * equal (poles * 2 + 1).
+ */
+#define EMC2305_RPM_FACTOR		3932160
+
+#define EMC2305_REG_FAN_DRIVE(n)	(0x30 + 0x10 * (n))
+#define EMC2305_REG_FAN_MIN_DRIVE(n)	(0x38 + 0x10 * (n))
+#define EMC2305_REG_FAN_TACH(n)		(0x3e + 0x10 * (n))
+
+enum emc230x_product_id {
+	EMC2305 = 0x34,
+	EMC2303 = 0x35,
+	EMC2302 = 0x36,
+	EMC2301 = 0x37,
+};
+
+static const struct i2c_device_id emc2305_ids[] = {
+	{ "emc2305", 0 },
+	{ "emc2303", 0 },
+	{ "emc2302", 0 },
+	{ "emc2301", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, emc2305_ids);
+
+/**
+ * @cdev: cooling device;
+ * @curr_state: cooling current state;
+ * @last_hwmon_state: last cooling state updated by hwmon subsystem;
+ * @last_thermal_state: last cooling state updated by thermal subsystem;
+ *
+ * The 'last_hwmon_state' and 'last_thermal_state' fields are provided to support fan low limit
+ * speed feature. The purpose of this feature is to provides ability to limit fan speed
+ * according to some system wise considerations, like absence of some replaceable units (PSU or
+ * line cards), high system ambient temperature, unreliable transceivers temperature sensing or
+ * some other factors which indirectly impacts system's airflow
+ * Fan low limit feature is supported through 'hwmon' interface: 'hwmon' 'pwm' attribute is
+ * used for setting low limit for fan speed in case 'thermal' subsystem is configured in
+ * kernel. In this case setting fan speed through 'hwmon' will never let the 'thermal'
+ * subsystem to select a lower duty cycle than the duty cycle selected with the 'pwm'
+ * attribute.
+ * From other side, fan speed is to be updated in hardware through 'pwm' only in case the
+ * requested fan speed is above last speed set by 'thermal' subsystem, otherwise requested fan
+ * speed will be just stored with no PWM update.
+ */
+struct emc2305_cdev_data {
+	struct thermal_cooling_device *cdev;
+	unsigned int cur_state;
+	unsigned long last_hwmon_state;
+	unsigned long last_thermal_state;
+};
+
+/**
+ * @client: i2c client;
+ * @hwmon_dev: hwmon device;
+ * @max_state: maximum cooling state of the cooling device;
+ * @pwm_num: number of PWM channels;
+ * @pwm_separate: separate PWM settings for every channel;
+ * @pwm_min: array of minimum PWM per channel;
+ * @cdev_data: array of cooling devices data;
+ */
+struct emc2305_data {
+	struct i2c_client *client;
+	struct device *hwmon_dev;
+	u8 max_state;
+	u8 pwm_num;
+	bool pwm_separate;
+	u8 pwm_min[EMC2305_PWM_MAX];
+	struct emc2305_cdev_data cdev_data[EMC2305_PWM_MAX];
+};
+
+static char *emc2305_fan_name[] = {
+	"emc2305_fan",
+	"emc2305_fan1",
+	"emc2305_fan2",
+	"emc2305_fan3",
+	"emc2305_fan4",
+	"emc2305_fan5",
+};
+
+static void emc2305_unset_tz(struct device *dev);
+
+static int emc2305_get_max_channel(const struct emc2305_data *data)
+{
+	return data->pwm_num;
+}
+
+static int emc2305_get_cdev_idx(struct thermal_cooling_device *cdev)
+{
+	struct emc2305_data *data = cdev->devdata;
+	size_t len = strlen(cdev->type);
+	int ret;
+
+	if (len <= 0)
+		return -EINVAL;
+
+	/*
+	 * Returns index of cooling device 0..4 in case of separate PWM setting.
+	 * Zero index is used in case of one common PWM setting.
+	 * If the mode is not set as pwm_separate, all PWMs are to be bound
+	 * to the common thermal zone and should work at the same speed
+	 * to perform cooling for the same thermal junction.
+	 * Otherwise, return specific channel that will be used in bound
+	 * related PWM to the thermal zone.
+	 */
+	if (!data->pwm_separate)
+		return 0;
+
+	ret = cdev->type[len - 1];
+	switch (ret) {
+	case '1' ... '5':
+		return ret - '1';
+	default:
+		break;
+	}
+	return -EINVAL;
+}
+
+static int emc2305_get_cur_state(struct thermal_cooling_device *cdev, unsigned long *state)
+{
+	int cdev_idx;
+	struct emc2305_data *data = cdev->devdata;
+
+	cdev_idx = emc2305_get_cdev_idx(cdev);
+	if (cdev_idx < 0)
+		return cdev_idx;
+
+	*state = data->cdev_data[cdev_idx].cur_state;
+	return 0;
+}
+
+static int emc2305_get_max_state(struct thermal_cooling_device *cdev, unsigned long *state)
+{
+	struct emc2305_data *data = cdev->devdata;
+	*state = data->max_state;
+	return 0;
+}
+
+static int emc2305_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state)
+{
+	int cdev_idx, ret;
+	struct emc2305_data *data = cdev->devdata;
+	struct i2c_client *client = data->client;
+	u8 val, i;
+
+	if (state > data->max_state)
+		return -EINVAL;
+
+	cdev_idx =  emc2305_get_cdev_idx(cdev);
+	if (cdev_idx < 0)
+		return cdev_idx;
+
+	/* Save thermal state. */
+	data->cdev_data[cdev_idx].last_thermal_state = state;
+	state = max_t(unsigned long, state, data->cdev_data[cdev_idx].last_hwmon_state);
+
+	val = EMC2305_PWM_STATE2DUTY(state, data->max_state, EMC2305_FAN_MAX);
+
+	data->cdev_data[cdev_idx].cur_state = state;
+	if (data->pwm_separate) {
+		ret = i2c_smbus_write_byte_data(client, EMC2305_REG_FAN_DRIVE(cdev_idx), val);
+		if (ret < 0)
+			return ret;
+	} else {
+		/*
+		 * Set the same PWM value in all channels
+		 * if common PWM channel is used.
+		 */
+		for (i = 0; i < data->pwm_num; i++) {
+			ret = i2c_smbus_write_byte_data(client, EMC2305_REG_FAN_DRIVE(i), val);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+static const struct thermal_cooling_device_ops emc2305_cooling_ops = {
+	.get_max_state = emc2305_get_max_state,
+	.get_cur_state = emc2305_get_cur_state,
+	.set_cur_state = emc2305_set_cur_state,
+};
+
+static int emc2305_show_fault(struct device *dev, int channel)
+{
+	struct emc2305_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	int status_reg;
+
+	status_reg = i2c_smbus_read_byte_data(client, EMC2305_REG_DRIVE_FAIL_STATUS);
+	if (status_reg < 0)
+		return status_reg;
+
+	return status_reg & (1 << channel) ? 1 : 0;
+}
+
+static int emc2305_show_fan(struct device *dev, int channel)
+{
+	struct emc2305_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	int ret;
+
+	ret = i2c_smbus_read_word_swapped(client, EMC2305_REG_FAN_TACH(channel));
+	if (ret <= 0)
+		return ret;
+
+	ret = ret >> EMC2305_TACH_REGS_UNUSE_BITS;
+	ret = EMC2305_RPM_FACTOR / ret;
+	if (ret <= EMC2305_TACH_RANGE_MIN)
+		return 0;
+
+	return ret * EMC2305_TACH_CNT_MULTIPLIER;
+}
+
+static int emc2305_show_pwm(struct device *dev, int channel)
+{
+	struct emc2305_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+
+	return i2c_smbus_read_byte_data(client, EMC2305_REG_FAN_DRIVE(channel));
+}
+
+static int emc2305_set_pwm(struct device *dev, long val, int channel)
+{
+	struct emc2305_data *data = dev_get_drvdata(dev);
+	struct i2c_client *client = data->client;
+	int ret;
+
+	if (val < data->pwm_min[channel] || val > EMC2305_FAN_MAX)
+		return -EINVAL;
+
+	ret = i2c_smbus_write_byte_data(client, EMC2305_REG_FAN_DRIVE(channel), val);
+	if (ret < 0)
+		return ret;
+	data->cdev_data[channel].cur_state = EMC2305_PWM_DUTY2STATE(val, data->max_state,
+								    EMC2305_FAN_MAX);
+	return 0;
+}
+
+static int emc2305_set_single_tz(struct device *dev, int idx)
+{
+	struct emc2305_data *data = dev_get_drvdata(dev);
+	long pwm;
+	int i, cdev_idx, ret;
+
+	cdev_idx = (idx) ? idx - 1 : 0;
+	pwm = data->pwm_min[cdev_idx];
+
+	data->cdev_data[cdev_idx].cdev =
+		thermal_cooling_device_register(emc2305_fan_name[idx], data,
+						&emc2305_cooling_ops);
+
+	if (IS_ERR(data->cdev_data[cdev_idx].cdev)) {
+		dev_err(dev, "Failed to register cooling device %s\n", emc2305_fan_name[idx]);
+		return PTR_ERR(data->cdev_data[cdev_idx].cdev);
+	}
+	/* Set minimal PWM speed. */
+	if (data->pwm_separate) {
+		ret = emc2305_set_pwm(dev, pwm, cdev_idx);
+		if (ret < 0)
+			return ret;
+	} else {
+		for (i = 0; i < data->pwm_num; i++) {
+			ret = emc2305_set_pwm(dev, pwm, i);
+			if (ret < 0)
+				return ret;
+		}
+	}
+	data->cdev_data[cdev_idx].cur_state =
+		EMC2305_PWM_DUTY2STATE(data->pwm_min[cdev_idx], data->max_state,
+				       EMC2305_FAN_MAX);
+	data->cdev_data[cdev_idx].last_hwmon_state =
+		EMC2305_PWM_DUTY2STATE(data->pwm_min[cdev_idx], data->max_state,
+				       EMC2305_FAN_MAX);
+	return 0;
+}
+
+static int emc2305_set_tz(struct device *dev)
+{
+	struct emc2305_data *data = dev_get_drvdata(dev);
+	int i, ret;
+
+	if (!data->pwm_separate)
+		return emc2305_set_single_tz(dev, 0);
+
+	for (i = 0; i < data->pwm_num; i++) {
+		ret = emc2305_set_single_tz(dev, i + 1);
+		if (ret)
+			goto thermal_cooling_device_register_fail;
+	}
+	return 0;
+
+thermal_cooling_device_register_fail:
+	emc2305_unset_tz(dev);
+	return ret;
+}
+
+static void emc2305_unset_tz(struct device *dev)
+{
+	struct emc2305_data *data = dev_get_drvdata(dev);
+	int i;
+
+	/* Unregister cooling device. */
+	for (i = 0; i < EMC2305_PWM_MAX; i++)
+		if (data->cdev_data[i].cdev)
+			thermal_cooling_device_unregister(data->cdev_data[i].cdev);
+}
+
+static umode_t
+emc2305_is_visible(const void *data, enum hwmon_sensor_types type, u32 attr, int channel)
+{
+	int max_channel = emc2305_get_max_channel(data);
+
+	/* Don't show channels which are not physically connected. */
+	if (channel >= max_channel)
+		return 0;
+	switch (type) {
+	case hwmon_fan:
+		switch (attr) {
+		case hwmon_fan_input:
+			return 0444;
+		case hwmon_fan_fault:
+			return 0444;
+		default:
+			break;
+		}
+		break;
+	case hwmon_pwm:
+		switch (attr) {
+		case hwmon_pwm_input:
+			return 0644;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+};
+
+static int
+emc2305_write(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long val)
+{
+	struct emc2305_data *data = dev_get_drvdata(dev);
+	int cdev_idx;
+
+	switch (type) {
+	case hwmon_pwm:
+		switch (attr) {
+		case hwmon_pwm_input:
+			/* If thermal is configured - handle PWM limit setting. */
+			if (IS_REACHABLE(CONFIG_THERMAL)) {
+				if (data->pwm_separate)
+					cdev_idx = channel;
+				else
+					cdev_idx = 0;
+				data->cdev_data[cdev_idx].last_hwmon_state =
+					EMC2305_PWM_DUTY2STATE(val, data->max_state,
+							       EMC2305_FAN_MAX);
+				/*
+				 * Update PWM only in case requested state is not less than the
+				 * last thermal state.
+				 */
+				if (data->cdev_data[cdev_idx].last_hwmon_state >=
+				    data->cdev_data[cdev_idx].last_thermal_state)
+					return emc2305_set_cur_state(data->cdev_data[cdev_idx].cdev,
+							data->cdev_data[cdev_idx].last_hwmon_state);
+				return 0;
+			}
+			return emc2305_set_pwm(dev, val, channel);
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+};
+
+static int
+emc2305_read(struct device *dev, enum hwmon_sensor_types type, u32 attr, int channel, long *val)
+{
+	int ret;
+
+	switch (type) {
+	case hwmon_fan:
+		switch (attr) {
+		case hwmon_fan_input:
+			ret = emc2305_show_fan(dev, channel);
+			if (ret < 0)
+				return ret;
+			*val = ret;
+			return 0;
+		case hwmon_fan_fault:
+			ret = emc2305_show_fault(dev, channel);
+			if (ret < 0)
+				return ret;
+			*val = ret;
+			return 0;
+		default:
+			break;
+		}
+		break;
+	case hwmon_pwm:
+		switch (attr) {
+		case hwmon_pwm_input:
+			ret = emc2305_show_pwm(dev, channel);
+			if (ret < 0)
+				return ret;
+			*val = ret;
+			return 0;
+		default:
+			break;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return -EOPNOTSUPP;
+};
+
+static const struct hwmon_ops emc2305_ops = {
+	.is_visible = emc2305_is_visible,
+	.read = emc2305_read,
+	.write = emc2305_write,
+};
+
+static const struct hwmon_channel_info *emc2305_info[] = {
+	HWMON_CHANNEL_INFO(fan,
+			   HWMON_F_INPUT | HWMON_F_FAULT,
+			   HWMON_F_INPUT | HWMON_F_FAULT,
+			   HWMON_F_INPUT | HWMON_F_FAULT,
+			   HWMON_F_INPUT | HWMON_F_FAULT,
+			   HWMON_F_INPUT | HWMON_F_FAULT),
+	HWMON_CHANNEL_INFO(pwm,
+			   HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT),
+	NULL
+};
+
+static const struct hwmon_chip_info emc2305_chip_info = {
+	.ops = &emc2305_ops,
+	.info = emc2305_info,
+};
+
+static int emc2305_identify(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct emc2305_data *data = i2c_get_clientdata(client);
+	int ret;
+
+	ret = i2c_smbus_read_byte_data(client, EMC2305_REG_PRODUCT_ID);
+	if (ret < 0)
+		return ret;
+
+	switch (ret) {
+	case EMC2305:
+		data->pwm_num = 5;
+		break;
+	case EMC2303:
+		data->pwm_num = 3;
+		break;
+	case EMC2302:
+		data->pwm_num = 2;
+		break;
+	case EMC2301:
+		data->pwm_num = 1;
+		break;
+	default:
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static int emc2305_probe(struct i2c_client *client, const struct i2c_device_id *id)
+{
+	struct i2c_adapter *adapter = client->adapter;
+	struct device *dev = &client->dev;
+	struct emc2305_data *data;
+	struct emc2305_platform_data *pdata;
+	int vendor, device;
+	int ret;
+	int i;
+
+	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA))
+		return -ENODEV;
+
+	vendor = i2c_smbus_read_byte_data(client, EMC2305_REG_VENDOR);
+	if (vendor != EMC2305_VENDOR)
+		return -ENODEV;
+
+	device = i2c_smbus_read_byte_data(client, EMC2305_REG_DEVICE);
+	if (device != EMC2305_DEVICE)
+		return -ENODEV;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	i2c_set_clientdata(client, data);
+	data->client = client;
+
+	ret = emc2305_identify(dev);
+	if (ret)
+		return ret;
+
+	pdata = dev_get_platdata(&client->dev);
+	if (pdata) {
+		if (!pdata->max_state || pdata->max_state > EMC2305_FAN_MAX_STATE)
+			return -EINVAL;
+		data->max_state = pdata->max_state;
+		/*
+		 * Validate a number of active PWM channels. Note that
+		 * configured number can be less than the actual maximum
+		 * supported by the device.
+		 */
+		if (!pdata->pwm_num || pdata->pwm_num > EMC2305_PWM_MAX)
+			return -EINVAL;
+		data->pwm_num = pdata->pwm_num;
+		data->pwm_separate = pdata->pwm_separate;
+		for (i = 0; i < EMC2305_PWM_MAX; i++)
+			data->pwm_min[i] = pdata->pwm_min[i];
+	} else {
+		data->max_state = EMC2305_FAN_MAX_STATE;
+		data->pwm_separate = false;
+		for (i = 0; i < EMC2305_PWM_MAX; i++)
+			data->pwm_min[i] = EMC2305_FAN_MIN;
+	}
+
+	data->hwmon_dev = devm_hwmon_device_register_with_info(dev, "emc2305", data,
+							       &emc2305_chip_info, NULL);
+	if (IS_ERR(data->hwmon_dev))
+		return PTR_ERR(data->hwmon_dev);
+
+	if (IS_REACHABLE(CONFIG_THERMAL)) {
+		ret = emc2305_set_tz(dev);
+		if (ret != 0)
+			return ret;
+	}
+
+	for (i = 0; i < data->pwm_num; i++) {
+		ret = i2c_smbus_write_byte_data(client, EMC2305_REG_FAN_MIN_DRIVE(i),
+						data->pwm_min[i]);
+		if (ret < 0)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void emc2305_remove(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+
+	if (IS_REACHABLE(CONFIG_THERMAL))
+		emc2305_unset_tz(dev);
+}
+
+static struct i2c_driver emc2305_driver = {
+	.class  = I2C_CLASS_HWMON,
+	.driver = {
+		.name = "emc2305",
+	},
+	.probe    = emc2305_probe,
+	.remove	  = emc2305_remove,
+	.id_table = emc2305_ids,
+	.address_list = emc2305_normal_i2c,
+};
+
+module_i2c_driver(emc2305_driver);
+
+MODULE_AUTHOR("Nvidia");
+MODULE_DESCRIPTION("Microchip EMC2305 fan controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/emc6w201.c b/drivers/hwmon/emc6w201.c
index 29082c8463f4..bcd93f0fe982 100644
--- a/drivers/hwmon/emc6w201.c
+++ b/drivers/hwmon/emc6w201.c
@@ -439,7 +439,7 @@ static int emc6w201_detect(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, "emc6w201", I2C_NAME_SIZE);
+	strscpy(info->type, "emc6w201", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/f71882fg.c b/drivers/hwmon/f71882fg.c
index 19b6c643059a..70121482a617 100644
--- a/drivers/hwmon/f71882fg.c
+++ b/drivers/hwmon/f71882fg.c
@@ -237,13 +237,6 @@ static const char f71882fg_nr_temps[] = {
 
 static struct platform_device *f71882fg_pdev;
 
-/* Super-I/O Function prototypes */
-static inline int superio_inb(int base, int reg);
-static inline int superio_inw(int base, int reg);
-static inline int superio_enter(int base);
-static inline void superio_select(int base, int ld);
-static inline void superio_exit(int base);
-
 struct f71882fg_sio_data {
 	enum chips type;
 };
@@ -292,108 +285,422 @@ struct f71882fg_data {
 	s8	pwm_auto_point_temp[4][4];
 };
 
-/* Sysfs in */
-static ssize_t show_in(struct device *dev, struct device_attribute *devattr,
-	char *buf);
-static ssize_t show_in_max(struct device *dev, struct device_attribute
-	*devattr, char *buf);
-static ssize_t store_in_max(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count);
-static ssize_t show_in_beep(struct device *dev, struct device_attribute
-	*devattr, char *buf);
-static ssize_t store_in_beep(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count);
-static ssize_t show_in_alarm(struct device *dev, struct device_attribute
-	*devattr, char *buf);
-/* Sysfs Fan */
-static ssize_t show_fan(struct device *dev, struct device_attribute *devattr,
-	char *buf);
-static ssize_t show_fan_full_speed(struct device *dev,
-	struct device_attribute *devattr, char *buf);
-static ssize_t store_fan_full_speed(struct device *dev,
-	struct device_attribute *devattr, const char *buf, size_t count);
-static ssize_t show_fan_beep(struct device *dev, struct device_attribute
-	*devattr, char *buf);
-static ssize_t store_fan_beep(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count);
-static ssize_t show_fan_alarm(struct device *dev, struct device_attribute
-	*devattr, char *buf);
-/* Sysfs Temp */
-static ssize_t show_temp(struct device *dev, struct device_attribute
-	*devattr, char *buf);
+static u8 f71882fg_read8(struct f71882fg_data *data, u8 reg)
+{
+	u8 val;
+
+	outb(reg, data->addr + ADDR_REG_OFFSET);
+	val = inb(data->addr + DATA_REG_OFFSET);
+
+	return val;
+}
+
+static u16 f71882fg_read16(struct f71882fg_data *data, u8 reg)
+{
+	u16 val;
+
+	val  = f71882fg_read8(data, reg) << 8;
+	val |= f71882fg_read8(data, reg + 1);
+
+	return val;
+}
+
+static inline int fan_from_reg(u16 reg)
+{
+	return reg ? (1500000 / reg) : 0;
+}
+
+static inline u16 fan_to_reg(int fan)
+{
+	return fan ? (1500000 / fan) : 0;
+}
+
+static void f71882fg_write8(struct f71882fg_data *data, u8 reg, u8 val)
+{
+	outb(reg, data->addr + ADDR_REG_OFFSET);
+	outb(val, data->addr + DATA_REG_OFFSET);
+}
+
+static void f71882fg_write16(struct f71882fg_data *data, u8 reg, u16 val)
+{
+	f71882fg_write8(data, reg,     val >> 8);
+	f71882fg_write8(data, reg + 1, val & 0xff);
+}
+
+static u16 f71882fg_read_temp(struct f71882fg_data *data, int nr)
+{
+	if (data->type == f71858fg)
+		return f71882fg_read16(data, F71882FG_REG_TEMP(nr));
+	else
+		return f71882fg_read8(data, F71882FG_REG_TEMP(nr));
+}
+
+static struct f71882fg_data *f71882fg_update_device(struct device *dev)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int nr_fans = f71882fg_nr_fans[data->type];
+	int nr_temps = f71882fg_nr_temps[data->type];
+	int nr, reg, point;
+
+	mutex_lock(&data->update_lock);
+
+	/* Update once every 60 seconds */
+	if (time_after(jiffies, data->last_limits + 60 * HZ) ||
+			!data->valid) {
+		if (f71882fg_has_in1_alarm[data->type]) {
+			if (data->type == f81866a) {
+				data->in1_max =
+					f71882fg_read8(data,
+						       F81866_REG_IN1_HIGH);
+				data->in_beep =
+					f71882fg_read8(data,
+						       F81866_REG_IN_BEEP);
+			} else {
+				data->in1_max =
+					f71882fg_read8(data,
+						       F71882FG_REG_IN1_HIGH);
+				data->in_beep =
+					f71882fg_read8(data,
+						       F71882FG_REG_IN_BEEP);
+			}
+		}
+
+		/* Get High & boundary temps*/
+		for (nr = data->temp_start; nr < nr_temps + data->temp_start;
+									nr++) {
+			data->temp_ovt[nr] = f71882fg_read8(data,
+						F71882FG_REG_TEMP_OVT(nr));
+			data->temp_high[nr] = f71882fg_read8(data,
+						F71882FG_REG_TEMP_HIGH(nr));
+		}
+
+		if (data->type != f8000) {
+			data->temp_hyst[0] = f71882fg_read8(data,
+						F71882FG_REG_TEMP_HYST(0));
+			data->temp_hyst[1] = f71882fg_read8(data,
+						F71882FG_REG_TEMP_HYST(1));
+		}
+		/* All but the f71858fg / f8000 have this register */
+		if ((data->type != f71858fg) && (data->type != f8000)) {
+			reg  = f71882fg_read8(data, F71882FG_REG_TEMP_TYPE);
+			data->temp_type[1] = (reg & 0x02) ? 2 : 4;
+			data->temp_type[2] = (reg & 0x04) ? 2 : 4;
+			data->temp_type[3] = (reg & 0x08) ? 2 : 4;
+		}
+
+		if (f71882fg_fan_has_beep[data->type])
+			data->fan_beep = f71882fg_read8(data,
+						F71882FG_REG_FAN_BEEP);
+
+		if (f71882fg_temp_has_beep[data->type])
+			data->temp_beep = f71882fg_read8(data,
+						F71882FG_REG_TEMP_BEEP);
+
+		data->pwm_enable = f71882fg_read8(data,
+						  F71882FG_REG_PWM_ENABLE);
+		data->pwm_auto_point_hyst[0] =
+			f71882fg_read8(data, F71882FG_REG_FAN_HYST(0));
+		data->pwm_auto_point_hyst[1] =
+			f71882fg_read8(data, F71882FG_REG_FAN_HYST(1));
+
+		for (nr = 0; nr < nr_fans; nr++) {
+			data->pwm_auto_point_mapping[nr] =
+			    f71882fg_read8(data,
+					   F71882FG_REG_POINT_MAPPING(nr));
+
+			switch (data->type) {
+			default:
+				for (point = 0; point < 5; point++) {
+					data->pwm_auto_point_pwm[nr][point] =
+						f71882fg_read8(data,
+							F71882FG_REG_POINT_PWM
+							(nr, point));
+				}
+				for (point = 0; point < 4; point++) {
+					data->pwm_auto_point_temp[nr][point] =
+						f71882fg_read8(data,
+							F71882FG_REG_POINT_TEMP
+							(nr, point));
+				}
+				break;
+			case f71808e:
+			case f71869:
+				data->pwm_auto_point_pwm[nr][0] =
+					f71882fg_read8(data,
+						F71882FG_REG_POINT_PWM(nr, 0));
+				fallthrough;
+			case f71862fg:
+				data->pwm_auto_point_pwm[nr][1] =
+					f71882fg_read8(data,
+						F71882FG_REG_POINT_PWM
+						(nr, 1));
+				data->pwm_auto_point_pwm[nr][4] =
+					f71882fg_read8(data,
+						F71882FG_REG_POINT_PWM
+						(nr, 4));
+				data->pwm_auto_point_temp[nr][0] =
+					f71882fg_read8(data,
+						F71882FG_REG_POINT_TEMP
+						(nr, 0));
+				data->pwm_auto_point_temp[nr][3] =
+					f71882fg_read8(data,
+						F71882FG_REG_POINT_TEMP
+						(nr, 3));
+				break;
+			}
+		}
+		data->last_limits = jiffies;
+	}
+
+	/* Update every second */
+	if (time_after(jiffies, data->last_updated + HZ) || !data->valid) {
+		data->temp_status = f71882fg_read8(data,
+						F71882FG_REG_TEMP_STATUS);
+		data->temp_diode_open = f71882fg_read8(data,
+						F71882FG_REG_TEMP_DIODE_OPEN);
+		for (nr = data->temp_start; nr < nr_temps + data->temp_start;
+									nr++)
+			data->temp[nr] = f71882fg_read_temp(data, nr);
+
+		data->fan_status = f71882fg_read8(data,
+						F71882FG_REG_FAN_STATUS);
+		for (nr = 0; nr < nr_fans; nr++) {
+			data->fan[nr] = f71882fg_read16(data,
+						F71882FG_REG_FAN(nr));
+			data->fan_target[nr] =
+			    f71882fg_read16(data, F71882FG_REG_FAN_TARGET(nr));
+			data->fan_full_speed[nr] =
+			    f71882fg_read16(data,
+					    F71882FG_REG_FAN_FULL_SPEED(nr));
+			data->pwm[nr] =
+			    f71882fg_read8(data, F71882FG_REG_PWM(nr));
+		}
+		/* Some models have 1 more fan with limited capabilities */
+		if (data->type == f71808a) {
+			data->fan[2] = f71882fg_read16(data,
+						F71882FG_REG_FAN(2));
+			data->pwm[2] = f71882fg_read8(data,
+							F71882FG_REG_PWM(2));
+		}
+		if (data->type == f8000)
+			data->fan[3] = f71882fg_read16(data,
+						F71882FG_REG_FAN(3));
+
+		if (f71882fg_has_in1_alarm[data->type]) {
+			if (data->type == f81866a)
+				data->in_status = f71882fg_read8(data,
+						F81866_REG_IN_STATUS);
+
+			else
+				data->in_status = f71882fg_read8(data,
+						F71882FG_REG_IN_STATUS);
+		}
+
+		for (nr = 0; nr < F71882FG_MAX_INS; nr++)
+			if (f71882fg_has_in[data->type][nr])
+				data->in[nr] = f71882fg_read8(data,
+							F71882FG_REG_IN(nr));
+
+		data->last_updated = jiffies;
+		data->valid = true;
+	}
+
+	mutex_unlock(&data->update_lock);
+
+	return data;
+}
+
+static ssize_t name_show(struct device *dev, struct device_attribute *devattr,
+	char *buf)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	return sprintf(buf, "%s\n", f71882fg_names[data->type]);
+}
+
+static DEVICE_ATTR_RO(name);
+
+static ssize_t show_temp(struct device *dev, struct device_attribute *devattr,
+	char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+	int sign, temp;
+
+	if (data->type == f71858fg) {
+		/* TEMP_TABLE_SEL 1 or 3 ? */
+		if (data->temp_config & 1) {
+			sign = data->temp[nr] & 0x0001;
+			temp = (data->temp[nr] >> 5) & 0x7ff;
+		} else {
+			sign = data->temp[nr] & 0x8000;
+			temp = (data->temp[nr] >> 5) & 0x3ff;
+		}
+		temp *= 125;
+		if (sign)
+			temp -= 128000;
+	} else {
+		temp = ((s8)data->temp[nr]) * 1000;
+	}
+
+	return sprintf(buf, "%d\n", temp);
+}
+
 static ssize_t show_temp_max(struct device *dev, struct device_attribute
-	*devattr, char *buf);
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	return sprintf(buf, "%d\n", data->temp_high[nr] * 1000);
+}
+
 static ssize_t store_temp_max(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count);
+	*devattr, const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	val /= 1000;
+	val = clamp_val(val, 0, 255);
+
+	mutex_lock(&data->update_lock);
+	f71882fg_write8(data, F71882FG_REG_TEMP_HIGH(nr), val);
+	data->temp_high[nr] = val;
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
 static ssize_t show_temp_max_hyst(struct device *dev, struct device_attribute
-	*devattr, char *buf);
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+	int temp_max_hyst;
+
+	mutex_lock(&data->update_lock);
+	if (nr & 1)
+		temp_max_hyst = data->temp_hyst[nr / 2] >> 4;
+	else
+		temp_max_hyst = data->temp_hyst[nr / 2] & 0x0f;
+	temp_max_hyst = (data->temp_high[nr] - temp_max_hyst) * 1000;
+	mutex_unlock(&data->update_lock);
+
+	return sprintf(buf, "%d\n", temp_max_hyst);
+}
+
 static ssize_t store_temp_max_hyst(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count);
+	*devattr, const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	ssize_t ret = count;
+	u8 reg;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	val /= 1000;
+
+	mutex_lock(&data->update_lock);
+
+	/* convert abs to relative and check */
+	data->temp_high[nr] = f71882fg_read8(data, F71882FG_REG_TEMP_HIGH(nr));
+	val = clamp_val(val, data->temp_high[nr] - 15, data->temp_high[nr]);
+	val = data->temp_high[nr] - val;
+
+	/* convert value to register contents */
+	reg = f71882fg_read8(data, F71882FG_REG_TEMP_HYST(nr / 2));
+	if (nr & 1)
+		reg = (reg & 0x0f) | (val << 4);
+	else
+		reg = (reg & 0xf0) | val;
+	f71882fg_write8(data, F71882FG_REG_TEMP_HYST(nr / 2), reg);
+	data->temp_hyst[nr / 2] = reg;
+
+	mutex_unlock(&data->update_lock);
+	return ret;
+}
+
+static ssize_t show_temp_alarm(struct device *dev, struct device_attribute
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	if (data->temp_status & (1 << nr))
+		return sprintf(buf, "1\n");
+	else
+		return sprintf(buf, "0\n");
+}
+
 static ssize_t show_temp_crit(struct device *dev, struct device_attribute
-	*devattr, char *buf);
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	return sprintf(buf, "%d\n", data->temp_ovt[nr] * 1000);
+}
+
 static ssize_t store_temp_crit(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count);
+	*devattr, const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	val /= 1000;
+	val = clamp_val(val, 0, 255);
+
+	mutex_lock(&data->update_lock);
+	f71882fg_write8(data, F71882FG_REG_TEMP_OVT(nr), val);
+	data->temp_ovt[nr] = val;
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
 static ssize_t show_temp_crit_hyst(struct device *dev, struct device_attribute
-	*devattr, char *buf);
-static ssize_t show_temp_type(struct device *dev, struct device_attribute
-	*devattr, char *buf);
-static ssize_t show_temp_beep(struct device *dev, struct device_attribute
-	*devattr, char *buf);
-static ssize_t store_temp_beep(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count);
-static ssize_t show_temp_alarm(struct device *dev, struct device_attribute
-	*devattr, char *buf);
-static ssize_t show_temp_fault(struct device *dev, struct device_attribute
-	*devattr, char *buf);
-/* PWM and Auto point control */
-static ssize_t show_pwm(struct device *dev, struct device_attribute *devattr,
-	char *buf);
-static ssize_t store_pwm(struct device *dev, struct device_attribute *devattr,
-	const char *buf, size_t count);
-static ssize_t show_simple_pwm(struct device *dev,
-	struct device_attribute *devattr, char *buf);
-static ssize_t store_simple_pwm(struct device *dev,
-	struct device_attribute *devattr, const char *buf, size_t count);
-static ssize_t show_pwm_enable(struct device *dev,
-	struct device_attribute *devattr, char *buf);
-static ssize_t store_pwm_enable(struct device *dev,
-	struct device_attribute	*devattr, const char *buf, size_t count);
-static ssize_t show_pwm_interpolate(struct device *dev,
-	struct device_attribute *devattr, char *buf);
-static ssize_t store_pwm_interpolate(struct device *dev,
-	struct device_attribute *devattr, const char *buf, size_t count);
-static ssize_t show_pwm_auto_point_channel(struct device *dev,
-	struct device_attribute *devattr, char *buf);
-static ssize_t store_pwm_auto_point_channel(struct device *dev,
-	struct device_attribute *devattr, const char *buf, size_t count);
-static ssize_t show_pwm_auto_point_temp_hyst(struct device *dev,
-	struct device_attribute *devattr, char *buf);
-static ssize_t store_pwm_auto_point_temp_hyst(struct device *dev,
-	struct device_attribute *devattr, const char *buf, size_t count);
-static ssize_t show_pwm_auto_point_pwm(struct device *dev,
-	struct device_attribute *devattr, char *buf);
-static ssize_t store_pwm_auto_point_pwm(struct device *dev,
-	struct device_attribute *devattr, const char *buf, size_t count);
-static ssize_t show_pwm_auto_point_temp(struct device *dev,
-	struct device_attribute *devattr, char *buf);
-static ssize_t store_pwm_auto_point_temp(struct device *dev,
-	struct device_attribute *devattr, const char *buf, size_t count);
-/* Sysfs misc */
-static ssize_t name_show(struct device *dev, struct device_attribute *devattr,
-	char *buf);
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+	int temp_crit_hyst;
 
-static int f71882fg_probe(struct platform_device *pdev);
-static int f71882fg_remove(struct platform_device *pdev);
+	mutex_lock(&data->update_lock);
+	if (nr & 1)
+		temp_crit_hyst = data->temp_hyst[nr / 2] >> 4;
+	else
+		temp_crit_hyst = data->temp_hyst[nr / 2] & 0x0f;
+	temp_crit_hyst = (data->temp_ovt[nr] - temp_crit_hyst) * 1000;
+	mutex_unlock(&data->update_lock);
 
-static struct platform_driver f71882fg_driver = {
-	.driver = {
-		.name	= DRVNAME,
-	},
-	.probe		= f71882fg_probe,
-	.remove		= f71882fg_remove,
-};
+	return sprintf(buf, "%d\n", temp_crit_hyst);
+}
 
-static DEVICE_ATTR_RO(name);
+static ssize_t show_temp_fault(struct device *dev, struct device_attribute
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	if (data->temp_diode_open & (1 << nr))
+		return sprintf(buf, "1\n");
+	else
+		return sprintf(buf, "0\n");
+}
 
 /*
  * Temp attr for the f71858fg, the f71858fg is special as it has its
@@ -438,6 +745,15 @@ static struct sensor_device_attribute_2 f71858fg_temp_attr[] = {
 	SENSOR_ATTR_2(temp3_fault, S_IRUGO, show_temp_fault, NULL, 0, 2),
 };
 
+static ssize_t show_temp_type(struct device *dev, struct device_attribute
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	return sprintf(buf, "%d\n", data->temp_type[nr]);
+}
+
 /* Temp attr for the standard models */
 static struct sensor_device_attribute_2 fxxxx_temp_attr[3][9] = { {
 	SENSOR_ATTR_2(temp1_input, S_IRUGO, show_temp, NULL, 0, 1),
@@ -490,6 +806,42 @@ static struct sensor_device_attribute_2 fxxxx_temp_attr[3][9] = { {
 	SENSOR_ATTR_2(temp3_fault, S_IRUGO, show_temp_fault, NULL, 0, 3),
 } };
 
+static ssize_t show_temp_beep(struct device *dev, struct device_attribute
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	if (data->temp_beep & (1 << nr))
+		return sprintf(buf, "1\n");
+	else
+		return sprintf(buf, "0\n");
+}
+
+static ssize_t store_temp_beep(struct device *dev, struct device_attribute
+	*devattr, const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	unsigned long val;
+
+	err = kstrtoul(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+	data->temp_beep = f71882fg_read8(data, F71882FG_REG_TEMP_BEEP);
+	if (val)
+		data->temp_beep |= 1 << nr;
+	else
+		data->temp_beep &= ~(1 << nr);
+
+	f71882fg_write8(data, F71882FG_REG_TEMP_BEEP, data->temp_beep);
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
 /* Temp attr for models which can beep on temp alarm */
 static struct sensor_device_attribute_2 fxxxx_temp_beep_attr[3][2] = { {
 	SENSOR_ATTR_2(temp1_max_beep, S_IRUGO|S_IWUSR, show_temp_beep,
@@ -555,6 +907,15 @@ static struct sensor_device_attribute_2 f8000_temp_attr[] = {
 	SENSOR_ATTR_2(temp3_fault, S_IRUGO, show_temp_fault, NULL, 0, 2),
 };
 
+static ssize_t show_in(struct device *dev, struct device_attribute *devattr,
+	char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	return sprintf(buf, "%d\n", data->in[nr] * 8);
+}
+
 /* in attr for all models */
 static struct sensor_device_attribute_2 fxxxx_in_attr[] = {
 	SENSOR_ATTR_2(in0_input, S_IRUGO, show_in, NULL, 0, 0),
@@ -570,6 +931,94 @@ static struct sensor_device_attribute_2 fxxxx_in_attr[] = {
 	SENSOR_ATTR_2(in10_input, S_IRUGO, show_in, NULL, 0, 10),
 };
 
+static ssize_t show_in_max(struct device *dev, struct device_attribute
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+
+	return sprintf(buf, "%d\n", data->in1_max * 8);
+}
+
+static ssize_t store_in_max(struct device *dev, struct device_attribute
+	*devattr, const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	val /= 8;
+	val = clamp_val(val, 0, 255);
+
+	mutex_lock(&data->update_lock);
+	if (data->type == f81866a)
+		f71882fg_write8(data, F81866_REG_IN1_HIGH, val);
+	else
+		f71882fg_write8(data, F71882FG_REG_IN1_HIGH, val);
+	data->in1_max = val;
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
+static ssize_t show_in_beep(struct device *dev, struct device_attribute
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	if (data->in_beep & (1 << nr))
+		return sprintf(buf, "1\n");
+	else
+		return sprintf(buf, "0\n");
+}
+
+static ssize_t store_in_beep(struct device *dev, struct device_attribute
+	*devattr, const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	unsigned long val;
+
+	err = kstrtoul(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+	if (data->type == f81866a)
+		data->in_beep = f71882fg_read8(data, F81866_REG_IN_BEEP);
+	else
+		data->in_beep = f71882fg_read8(data, F71882FG_REG_IN_BEEP);
+
+	if (val)
+		data->in_beep |= 1 << nr;
+	else
+		data->in_beep &= ~(1 << nr);
+
+	if (data->type == f81866a)
+		f71882fg_write8(data, F81866_REG_IN_BEEP, data->in_beep);
+	else
+		f71882fg_write8(data, F71882FG_REG_IN_BEEP, data->in_beep);
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
+static ssize_t show_in_alarm(struct device *dev, struct device_attribute
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	if (data->in_status & (1 << nr))
+		return sprintf(buf, "1\n");
+	else
+		return sprintf(buf, "0\n");
+}
+
 /* For models with in1 alarm capability */
 static struct sensor_device_attribute_2 fxxxx_in1_alarm_attr[] = {
 	SENSOR_ATTR_2(in1_max, S_IRUGO|S_IWUSR, show_in_max, store_in_max,
@@ -579,6 +1028,242 @@ static struct sensor_device_attribute_2 fxxxx_in1_alarm_attr[] = {
 	SENSOR_ATTR_2(in1_alarm, S_IRUGO, show_in_alarm, NULL, 0, 1),
 };
 
+static ssize_t show_fan(struct device *dev, struct device_attribute *devattr,
+	char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+	int speed = fan_from_reg(data->fan[nr]);
+
+	if (speed == FAN_MIN_DETECT)
+		speed = 0;
+
+	return sprintf(buf, "%d\n", speed);
+}
+
+static ssize_t show_fan_full_speed(struct device *dev,
+				   struct device_attribute *devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+	int speed = fan_from_reg(data->fan_full_speed[nr]);
+	return sprintf(buf, "%d\n", speed);
+}
+
+static ssize_t store_fan_full_speed(struct device *dev,
+				    struct device_attribute *devattr,
+				    const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	val = clamp_val(val, 23, 1500000);
+	val = fan_to_reg(val);
+
+	mutex_lock(&data->update_lock);
+	f71882fg_write16(data, F71882FG_REG_FAN_FULL_SPEED(nr), val);
+	data->fan_full_speed[nr] = val;
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
+static ssize_t show_fan_alarm(struct device *dev, struct device_attribute
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	if (data->fan_status & (1 << nr))
+		return sprintf(buf, "1\n");
+	else
+		return sprintf(buf, "0\n");
+}
+
+static ssize_t show_pwm(struct device *dev,
+			struct device_attribute *devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int val, nr = to_sensor_dev_attr_2(devattr)->index;
+	mutex_lock(&data->update_lock);
+	if (data->pwm_enable & (1 << (2 * nr)))
+		/* PWM mode */
+		val = data->pwm[nr];
+	else {
+		/* RPM mode */
+		val = 255 * fan_from_reg(data->fan_target[nr])
+			/ fan_from_reg(data->fan_full_speed[nr]);
+	}
+	mutex_unlock(&data->update_lock);
+	return sprintf(buf, "%d\n", val);
+}
+
+static ssize_t store_pwm(struct device *dev,
+			 struct device_attribute *devattr, const char *buf,
+			 size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	val = clamp_val(val, 0, 255);
+
+	mutex_lock(&data->update_lock);
+	data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE);
+	if ((data->type == f8000 && ((data->pwm_enable >> 2 * nr) & 3) != 2) ||
+	    (data->type != f8000 && !((data->pwm_enable >> 2 * nr) & 2))) {
+		count = -EROFS;
+		goto leave;
+	}
+	if (data->pwm_enable & (1 << (2 * nr))) {
+		/* PWM mode */
+		f71882fg_write8(data, F71882FG_REG_PWM(nr), val);
+		data->pwm[nr] = val;
+	} else {
+		/* RPM mode */
+		int target, full_speed;
+		full_speed = f71882fg_read16(data,
+					     F71882FG_REG_FAN_FULL_SPEED(nr));
+		target = fan_to_reg(val * fan_from_reg(full_speed) / 255);
+		f71882fg_write16(data, F71882FG_REG_FAN_TARGET(nr), target);
+		data->fan_target[nr] = target;
+		data->fan_full_speed[nr] = full_speed;
+	}
+leave:
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
+static ssize_t show_pwm_enable(struct device *dev,
+			       struct device_attribute *devattr, char *buf)
+{
+	int result = 0;
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	switch ((data->pwm_enable >> 2 * nr) & 3) {
+	case 0:
+	case 1:
+		result = 2; /* Normal auto mode */
+		break;
+	case 2:
+		result = 1; /* Manual mode */
+		break;
+	case 3:
+		if (data->type == f8000)
+			result = 3; /* Thermostat mode */
+		else
+			result = 1; /* Manual mode */
+		break;
+	}
+
+	return sprintf(buf, "%d\n", result);
+}
+
+static ssize_t store_pwm_enable(struct device *dev, struct device_attribute
+				*devattr, const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	/* Special case for F8000 pwm channel 3 which only does auto mode */
+	if (data->type == f8000 && nr == 2 && val != 2)
+		return -EINVAL;
+
+	mutex_lock(&data->update_lock);
+	data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE);
+	/* Special case for F8000 auto PWM mode / Thermostat mode */
+	if (data->type == f8000 && ((data->pwm_enable >> 2 * nr) & 1)) {
+		switch (val) {
+		case 2:
+			data->pwm_enable &= ~(2 << (2 * nr));
+			break;		/* Normal auto mode */
+		case 3:
+			data->pwm_enable |= 2 << (2 * nr);
+			break;		/* Thermostat mode */
+		default:
+			count = -EINVAL;
+			goto leave;
+		}
+	} else {
+		switch (val) {
+		case 1:
+			/* The f71858fg does not support manual RPM mode */
+			if (data->type == f71858fg &&
+			    ((data->pwm_enable >> (2 * nr)) & 1)) {
+				count = -EINVAL;
+				goto leave;
+			}
+			data->pwm_enable |= 2 << (2 * nr);
+			break;		/* Manual */
+		case 2:
+			data->pwm_enable &= ~(2 << (2 * nr));
+			break;		/* Normal auto mode */
+		default:
+			count = -EINVAL;
+			goto leave;
+		}
+	}
+	f71882fg_write8(data, F71882FG_REG_PWM_ENABLE, data->pwm_enable);
+leave:
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
+static ssize_t show_pwm_interpolate(struct device *dev,
+				    struct device_attribute *devattr, char *buf)
+{
+	int result;
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	result = (data->pwm_auto_point_mapping[nr] >> 4) & 1;
+
+	return sprintf(buf, "%d\n", result);
+}
+
+static ssize_t store_pwm_interpolate(struct device *dev,
+				     struct device_attribute *devattr,
+				     const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	unsigned long val;
+
+	err = kstrtoul(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+	data->pwm_auto_point_mapping[nr] =
+		f71882fg_read8(data, F71882FG_REG_POINT_MAPPING(nr));
+	if (val)
+		val = data->pwm_auto_point_mapping[nr] | (1 << 4);
+	else
+		val = data->pwm_auto_point_mapping[nr] & (~(1 << 4));
+	f71882fg_write8(data, F71882FG_REG_POINT_MAPPING(nr), val);
+	data->pwm_auto_point_mapping[nr] = val;
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
 /* Fan / PWM attr common to all models */
 static struct sensor_device_attribute_2 fxxxx_fan_attr[4][6] = { {
 	SENSOR_ATTR_2(fan1_input, S_IRUGO, show_fan, NULL, 0, 0),
@@ -626,6 +1311,38 @@ static struct sensor_device_attribute_2 fxxxx_fan_attr[4][6] = { {
 		      show_pwm_interpolate, store_pwm_interpolate, 0, 3),
 } };
 
+static ssize_t show_simple_pwm(struct device *dev,
+			       struct device_attribute *devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int val, nr = to_sensor_dev_attr_2(devattr)->index;
+
+	val = data->pwm[nr];
+	return sprintf(buf, "%d\n", val);
+}
+
+static ssize_t store_simple_pwm(struct device *dev,
+				struct device_attribute *devattr,
+				const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	val = clamp_val(val, 0, 255);
+
+	mutex_lock(&data->update_lock);
+	f71882fg_write8(data, F71882FG_REG_PWM(nr), val);
+	data->pwm[nr] = val;
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
 /* Attr for the third fan of the f71808a, which only has manual pwm */
 static struct sensor_device_attribute_2 f71808a_fan3_attr[] = {
 	SENSOR_ATTR_2(fan3_input, S_IRUGO, show_fan, NULL, 0, 2),
@@ -634,6 +1351,42 @@ static struct sensor_device_attribute_2 f71808a_fan3_attr[] = {
 		      show_simple_pwm, store_simple_pwm, 0, 2),
 };
 
+static ssize_t show_fan_beep(struct device *dev, struct device_attribute
+	*devattr, char *buf)
+{
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	if (data->fan_beep & (1 << nr))
+		return sprintf(buf, "1\n");
+	else
+		return sprintf(buf, "0\n");
+}
+
+static ssize_t store_fan_beep(struct device *dev, struct device_attribute
+	*devattr, const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	unsigned long val;
+
+	err = kstrtoul(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+	data->fan_beep = f71882fg_read8(data, F71882FG_REG_FAN_BEEP);
+	if (val)
+		data->fan_beep |= 1 << nr;
+	else
+		data->fan_beep &= ~(1 << nr);
+
+	f71882fg_write8(data, F71882FG_REG_FAN_BEEP, data->fan_beep);
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
 /* Attr for models which can beep on Fan alarm */
 static struct sensor_device_attribute_2 fxxxx_fan_beep_attr[] = {
 	SENSOR_ATTR_2(fan1_beep, S_IRUGO|S_IWUSR, show_fan_beep,
@@ -646,6 +1399,209 @@ static struct sensor_device_attribute_2 fxxxx_fan_beep_attr[] = {
 		store_fan_beep, 0, 3),
 };
 
+static ssize_t show_pwm_auto_point_channel(struct device *dev,
+					   struct device_attribute *devattr,
+					   char *buf)
+{
+	int result;
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+
+	result = 1 << ((data->pwm_auto_point_mapping[nr] & 3) -
+		       data->temp_start);
+
+	return sprintf(buf, "%d\n", result);
+}
+
+static ssize_t store_pwm_auto_point_channel(struct device *dev,
+					    struct device_attribute *devattr,
+					    const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	switch (val) {
+	case 1:
+		val = 0;
+		break;
+	case 2:
+		val = 1;
+		break;
+	case 4:
+		val = 2;
+		break;
+	default:
+		return -EINVAL;
+	}
+	val += data->temp_start;
+	mutex_lock(&data->update_lock);
+	data->pwm_auto_point_mapping[nr] =
+		f71882fg_read8(data, F71882FG_REG_POINT_MAPPING(nr));
+	val = (data->pwm_auto_point_mapping[nr] & 0xfc) | val;
+	f71882fg_write8(data, F71882FG_REG_POINT_MAPPING(nr), val);
+	data->pwm_auto_point_mapping[nr] = val;
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
+static ssize_t show_pwm_auto_point_pwm(struct device *dev,
+				       struct device_attribute *devattr,
+				       char *buf)
+{
+	int result;
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int pwm = to_sensor_dev_attr_2(devattr)->index;
+	int point = to_sensor_dev_attr_2(devattr)->nr;
+
+	mutex_lock(&data->update_lock);
+	if (data->pwm_enable & (1 << (2 * pwm))) {
+		/* PWM mode */
+		result = data->pwm_auto_point_pwm[pwm][point];
+	} else {
+		/* RPM mode */
+		result = 32 * 255 / (32 + data->pwm_auto_point_pwm[pwm][point]);
+	}
+	mutex_unlock(&data->update_lock);
+
+	return sprintf(buf, "%d\n", result);
+}
+
+static ssize_t store_pwm_auto_point_pwm(struct device *dev,
+					struct device_attribute *devattr,
+					const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, pwm = to_sensor_dev_attr_2(devattr)->index;
+	int point = to_sensor_dev_attr_2(devattr)->nr;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	val = clamp_val(val, 0, 255);
+
+	mutex_lock(&data->update_lock);
+	data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE);
+	if (data->pwm_enable & (1 << (2 * pwm))) {
+		/* PWM mode */
+	} else {
+		/* RPM mode */
+		if (val < 29)	/* Prevent negative numbers */
+			val = 255;
+		else
+			val = (255 - val) * 32 / val;
+	}
+	f71882fg_write8(data, F71882FG_REG_POINT_PWM(pwm, point), val);
+	data->pwm_auto_point_pwm[pwm][point] = val;
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
+static ssize_t show_pwm_auto_point_temp(struct device *dev,
+					struct device_attribute *devattr,
+					char *buf)
+{
+	int result;
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int pwm = to_sensor_dev_attr_2(devattr)->index;
+	int point = to_sensor_dev_attr_2(devattr)->nr;
+
+	result = data->pwm_auto_point_temp[pwm][point];
+	return sprintf(buf, "%d\n", 1000 * result);
+}
+
+static ssize_t store_pwm_auto_point_temp(struct device *dev,
+					 struct device_attribute *devattr,
+					 const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, pwm = to_sensor_dev_attr_2(devattr)->index;
+	int point = to_sensor_dev_attr_2(devattr)->nr;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	val /= 1000;
+
+	if (data->auto_point_temp_signed)
+		val = clamp_val(val, -128, 127);
+	else
+		val = clamp_val(val, 0, 127);
+
+	mutex_lock(&data->update_lock);
+	f71882fg_write8(data, F71882FG_REG_POINT_TEMP(pwm, point), val);
+	data->pwm_auto_point_temp[pwm][point] = val;
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
+static ssize_t show_pwm_auto_point_temp_hyst(struct device *dev,
+					     struct device_attribute *devattr,
+					     char *buf)
+{
+	int result = 0;
+	struct f71882fg_data *data = f71882fg_update_device(dev);
+	int nr = to_sensor_dev_attr_2(devattr)->index;
+	int point = to_sensor_dev_attr_2(devattr)->nr;
+
+	mutex_lock(&data->update_lock);
+	if (nr & 1)
+		result = data->pwm_auto_point_hyst[nr / 2] >> 4;
+	else
+		result = data->pwm_auto_point_hyst[nr / 2] & 0x0f;
+	result = 1000 * (data->pwm_auto_point_temp[nr][point] - result);
+	mutex_unlock(&data->update_lock);
+
+	return sprintf(buf, "%d\n", result);
+}
+
+static ssize_t store_pwm_auto_point_temp_hyst(struct device *dev,
+					      struct device_attribute *devattr,
+					      const char *buf, size_t count)
+{
+	struct f71882fg_data *data = dev_get_drvdata(dev);
+	int err, nr = to_sensor_dev_attr_2(devattr)->index;
+	int point = to_sensor_dev_attr_2(devattr)->nr;
+	u8 reg;
+	long val;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	val /= 1000;
+
+	mutex_lock(&data->update_lock);
+	data->pwm_auto_point_temp[nr][point] =
+		f71882fg_read8(data, F71882FG_REG_POINT_TEMP(nr, point));
+	val = clamp_val(val, data->pwm_auto_point_temp[nr][point] - 15,
+			data->pwm_auto_point_temp[nr][point]);
+	val = data->pwm_auto_point_temp[nr][point] - val;
+
+	reg = f71882fg_read8(data, F71882FG_REG_FAN_HYST(nr / 2));
+	if (nr & 1)
+		reg = (reg & 0x0f) | (val << 4);
+	else
+		reg = (reg & 0xf0) | val;
+
+	f71882fg_write8(data, F71882FG_REG_FAN_HYST(nr / 2), reg);
+	data->pwm_auto_point_hyst[nr / 2] = reg;
+	mutex_unlock(&data->update_lock);
+
+	return count;
+}
+
 /*
  * PWM attr for the f71862fg, fewer pwms and fewer zones per pwm than the
  * standard models
@@ -1144,1071 +2100,6 @@ static inline void superio_exit(int base)
 	release_region(base, 2);
 }
 
-static inline int fan_from_reg(u16 reg)
-{
-	return reg ? (1500000 / reg) : 0;
-}
-
-static inline u16 fan_to_reg(int fan)
-{
-	return fan ? (1500000 / fan) : 0;
-}
-
-static u8 f71882fg_read8(struct f71882fg_data *data, u8 reg)
-{
-	u8 val;
-
-	outb(reg, data->addr + ADDR_REG_OFFSET);
-	val = inb(data->addr + DATA_REG_OFFSET);
-
-	return val;
-}
-
-static u16 f71882fg_read16(struct f71882fg_data *data, u8 reg)
-{
-	u16 val;
-
-	val  = f71882fg_read8(data, reg) << 8;
-	val |= f71882fg_read8(data, reg + 1);
-
-	return val;
-}
-
-static void f71882fg_write8(struct f71882fg_data *data, u8 reg, u8 val)
-{
-	outb(reg, data->addr + ADDR_REG_OFFSET);
-	outb(val, data->addr + DATA_REG_OFFSET);
-}
-
-static void f71882fg_write16(struct f71882fg_data *data, u8 reg, u16 val)
-{
-	f71882fg_write8(data, reg,     val >> 8);
-	f71882fg_write8(data, reg + 1, val & 0xff);
-}
-
-static u16 f71882fg_read_temp(struct f71882fg_data *data, int nr)
-{
-	if (data->type == f71858fg)
-		return f71882fg_read16(data, F71882FG_REG_TEMP(nr));
-	else
-		return f71882fg_read8(data, F71882FG_REG_TEMP(nr));
-}
-
-static struct f71882fg_data *f71882fg_update_device(struct device *dev)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int nr_fans = f71882fg_nr_fans[data->type];
-	int nr_temps = f71882fg_nr_temps[data->type];
-	int nr, reg, point;
-
-	mutex_lock(&data->update_lock);
-
-	/* Update once every 60 seconds */
-	if (time_after(jiffies, data->last_limits + 60 * HZ) ||
-			!data->valid) {
-		if (f71882fg_has_in1_alarm[data->type]) {
-			if (data->type == f81866a) {
-				data->in1_max =
-					f71882fg_read8(data,
-						       F81866_REG_IN1_HIGH);
-				data->in_beep =
-					f71882fg_read8(data,
-						       F81866_REG_IN_BEEP);
-			} else {
-				data->in1_max =
-					f71882fg_read8(data,
-						       F71882FG_REG_IN1_HIGH);
-				data->in_beep =
-					f71882fg_read8(data,
-						       F71882FG_REG_IN_BEEP);
-			}
-		}
-
-		/* Get High & boundary temps*/
-		for (nr = data->temp_start; nr < nr_temps + data->temp_start;
-									nr++) {
-			data->temp_ovt[nr] = f71882fg_read8(data,
-						F71882FG_REG_TEMP_OVT(nr));
-			data->temp_high[nr] = f71882fg_read8(data,
-						F71882FG_REG_TEMP_HIGH(nr));
-		}
-
-		if (data->type != f8000) {
-			data->temp_hyst[0] = f71882fg_read8(data,
-						F71882FG_REG_TEMP_HYST(0));
-			data->temp_hyst[1] = f71882fg_read8(data,
-						F71882FG_REG_TEMP_HYST(1));
-		}
-		/* All but the f71858fg / f8000 have this register */
-		if ((data->type != f71858fg) && (data->type != f8000)) {
-			reg  = f71882fg_read8(data, F71882FG_REG_TEMP_TYPE);
-			data->temp_type[1] = (reg & 0x02) ? 2 : 4;
-			data->temp_type[2] = (reg & 0x04) ? 2 : 4;
-			data->temp_type[3] = (reg & 0x08) ? 2 : 4;
-		}
-
-		if (f71882fg_fan_has_beep[data->type])
-			data->fan_beep = f71882fg_read8(data,
-						F71882FG_REG_FAN_BEEP);
-
-		if (f71882fg_temp_has_beep[data->type])
-			data->temp_beep = f71882fg_read8(data,
-						F71882FG_REG_TEMP_BEEP);
-
-		data->pwm_enable = f71882fg_read8(data,
-						  F71882FG_REG_PWM_ENABLE);
-		data->pwm_auto_point_hyst[0] =
-			f71882fg_read8(data, F71882FG_REG_FAN_HYST(0));
-		data->pwm_auto_point_hyst[1] =
-			f71882fg_read8(data, F71882FG_REG_FAN_HYST(1));
-
-		for (nr = 0; nr < nr_fans; nr++) {
-			data->pwm_auto_point_mapping[nr] =
-			    f71882fg_read8(data,
-					   F71882FG_REG_POINT_MAPPING(nr));
-
-			switch (data->type) {
-			default:
-				for (point = 0; point < 5; point++) {
-					data->pwm_auto_point_pwm[nr][point] =
-						f71882fg_read8(data,
-							F71882FG_REG_POINT_PWM
-							(nr, point));
-				}
-				for (point = 0; point < 4; point++) {
-					data->pwm_auto_point_temp[nr][point] =
-						f71882fg_read8(data,
-							F71882FG_REG_POINT_TEMP
-							(nr, point));
-				}
-				break;
-			case f71808e:
-			case f71869:
-				data->pwm_auto_point_pwm[nr][0] =
-					f71882fg_read8(data,
-						F71882FG_REG_POINT_PWM(nr, 0));
-				fallthrough;
-			case f71862fg:
-				data->pwm_auto_point_pwm[nr][1] =
-					f71882fg_read8(data,
-						F71882FG_REG_POINT_PWM
-						(nr, 1));
-				data->pwm_auto_point_pwm[nr][4] =
-					f71882fg_read8(data,
-						F71882FG_REG_POINT_PWM
-						(nr, 4));
-				data->pwm_auto_point_temp[nr][0] =
-					f71882fg_read8(data,
-						F71882FG_REG_POINT_TEMP
-						(nr, 0));
-				data->pwm_auto_point_temp[nr][3] =
-					f71882fg_read8(data,
-						F71882FG_REG_POINT_TEMP
-						(nr, 3));
-				break;
-			}
-		}
-		data->last_limits = jiffies;
-	}
-
-	/* Update every second */
-	if (time_after(jiffies, data->last_updated + HZ) || !data->valid) {
-		data->temp_status = f71882fg_read8(data,
-						F71882FG_REG_TEMP_STATUS);
-		data->temp_diode_open = f71882fg_read8(data,
-						F71882FG_REG_TEMP_DIODE_OPEN);
-		for (nr = data->temp_start; nr < nr_temps + data->temp_start;
-									nr++)
-			data->temp[nr] = f71882fg_read_temp(data, nr);
-
-		data->fan_status = f71882fg_read8(data,
-						F71882FG_REG_FAN_STATUS);
-		for (nr = 0; nr < nr_fans; nr++) {
-			data->fan[nr] = f71882fg_read16(data,
-						F71882FG_REG_FAN(nr));
-			data->fan_target[nr] =
-			    f71882fg_read16(data, F71882FG_REG_FAN_TARGET(nr));
-			data->fan_full_speed[nr] =
-			    f71882fg_read16(data,
-					    F71882FG_REG_FAN_FULL_SPEED(nr));
-			data->pwm[nr] =
-			    f71882fg_read8(data, F71882FG_REG_PWM(nr));
-		}
-		/* Some models have 1 more fan with limited capabilities */
-		if (data->type == f71808a) {
-			data->fan[2] = f71882fg_read16(data,
-						F71882FG_REG_FAN(2));
-			data->pwm[2] = f71882fg_read8(data,
-							F71882FG_REG_PWM(2));
-		}
-		if (data->type == f8000)
-			data->fan[3] = f71882fg_read16(data,
-						F71882FG_REG_FAN(3));
-
-		if (f71882fg_has_in1_alarm[data->type]) {
-			if (data->type == f81866a)
-				data->in_status = f71882fg_read8(data,
-						F81866_REG_IN_STATUS);
-
-			else
-				data->in_status = f71882fg_read8(data,
-						F71882FG_REG_IN_STATUS);
-		}
-
-		for (nr = 0; nr < F71882FG_MAX_INS; nr++)
-			if (f71882fg_has_in[data->type][nr])
-				data->in[nr] = f71882fg_read8(data,
-							F71882FG_REG_IN(nr));
-
-		data->last_updated = jiffies;
-		data->valid = true;
-	}
-
-	mutex_unlock(&data->update_lock);
-
-	return data;
-}
-
-/* Sysfs Interface */
-static ssize_t show_fan(struct device *dev, struct device_attribute *devattr,
-	char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-	int speed = fan_from_reg(data->fan[nr]);
-
-	if (speed == FAN_MIN_DETECT)
-		speed = 0;
-
-	return sprintf(buf, "%d\n", speed);
-}
-
-static ssize_t show_fan_full_speed(struct device *dev,
-				   struct device_attribute *devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-	int speed = fan_from_reg(data->fan_full_speed[nr]);
-	return sprintf(buf, "%d\n", speed);
-}
-
-static ssize_t store_fan_full_speed(struct device *dev,
-				    struct device_attribute *devattr,
-				    const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	val = clamp_val(val, 23, 1500000);
-	val = fan_to_reg(val);
-
-	mutex_lock(&data->update_lock);
-	f71882fg_write16(data, F71882FG_REG_FAN_FULL_SPEED(nr), val);
-	data->fan_full_speed[nr] = val;
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_fan_beep(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	if (data->fan_beep & (1 << nr))
-		return sprintf(buf, "1\n");
-	else
-		return sprintf(buf, "0\n");
-}
-
-static ssize_t store_fan_beep(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	unsigned long val;
-
-	err = kstrtoul(buf, 10, &val);
-	if (err)
-		return err;
-
-	mutex_lock(&data->update_lock);
-	data->fan_beep = f71882fg_read8(data, F71882FG_REG_FAN_BEEP);
-	if (val)
-		data->fan_beep |= 1 << nr;
-	else
-		data->fan_beep &= ~(1 << nr);
-
-	f71882fg_write8(data, F71882FG_REG_FAN_BEEP, data->fan_beep);
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_fan_alarm(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	if (data->fan_status & (1 << nr))
-		return sprintf(buf, "1\n");
-	else
-		return sprintf(buf, "0\n");
-}
-
-static ssize_t show_in(struct device *dev, struct device_attribute *devattr,
-	char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	return sprintf(buf, "%d\n", data->in[nr] * 8);
-}
-
-static ssize_t show_in_max(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-
-	return sprintf(buf, "%d\n", data->in1_max * 8);
-}
-
-static ssize_t store_in_max(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	val /= 8;
-	val = clamp_val(val, 0, 255);
-
-	mutex_lock(&data->update_lock);
-	if (data->type == f81866a)
-		f71882fg_write8(data, F81866_REG_IN1_HIGH, val);
-	else
-		f71882fg_write8(data, F71882FG_REG_IN1_HIGH, val);
-	data->in1_max = val;
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_in_beep(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	if (data->in_beep & (1 << nr))
-		return sprintf(buf, "1\n");
-	else
-		return sprintf(buf, "0\n");
-}
-
-static ssize_t store_in_beep(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	unsigned long val;
-
-	err = kstrtoul(buf, 10, &val);
-	if (err)
-		return err;
-
-	mutex_lock(&data->update_lock);
-	if (data->type == f81866a)
-		data->in_beep = f71882fg_read8(data, F81866_REG_IN_BEEP);
-	else
-		data->in_beep = f71882fg_read8(data, F71882FG_REG_IN_BEEP);
-
-	if (val)
-		data->in_beep |= 1 << nr;
-	else
-		data->in_beep &= ~(1 << nr);
-
-	if (data->type == f81866a)
-		f71882fg_write8(data, F81866_REG_IN_BEEP, data->in_beep);
-	else
-		f71882fg_write8(data, F71882FG_REG_IN_BEEP, data->in_beep);
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_in_alarm(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	if (data->in_status & (1 << nr))
-		return sprintf(buf, "1\n");
-	else
-		return sprintf(buf, "0\n");
-}
-
-static ssize_t show_temp(struct device *dev, struct device_attribute *devattr,
-	char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-	int sign, temp;
-
-	if (data->type == f71858fg) {
-		/* TEMP_TABLE_SEL 1 or 3 ? */
-		if (data->temp_config & 1) {
-			sign = data->temp[nr] & 0x0001;
-			temp = (data->temp[nr] >> 5) & 0x7ff;
-		} else {
-			sign = data->temp[nr] & 0x8000;
-			temp = (data->temp[nr] >> 5) & 0x3ff;
-		}
-		temp *= 125;
-		if (sign)
-			temp -= 128000;
-	} else {
-		temp = ((s8)data->temp[nr]) * 1000;
-	}
-
-	return sprintf(buf, "%d\n", temp);
-}
-
-static ssize_t show_temp_max(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	return sprintf(buf, "%d\n", data->temp_high[nr] * 1000);
-}
-
-static ssize_t store_temp_max(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	val /= 1000;
-	val = clamp_val(val, 0, 255);
-
-	mutex_lock(&data->update_lock);
-	f71882fg_write8(data, F71882FG_REG_TEMP_HIGH(nr), val);
-	data->temp_high[nr] = val;
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_temp_max_hyst(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-	int temp_max_hyst;
-
-	mutex_lock(&data->update_lock);
-	if (nr & 1)
-		temp_max_hyst = data->temp_hyst[nr / 2] >> 4;
-	else
-		temp_max_hyst = data->temp_hyst[nr / 2] & 0x0f;
-	temp_max_hyst = (data->temp_high[nr] - temp_max_hyst) * 1000;
-	mutex_unlock(&data->update_lock);
-
-	return sprintf(buf, "%d\n", temp_max_hyst);
-}
-
-static ssize_t store_temp_max_hyst(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	ssize_t ret = count;
-	u8 reg;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	val /= 1000;
-
-	mutex_lock(&data->update_lock);
-
-	/* convert abs to relative and check */
-	data->temp_high[nr] = f71882fg_read8(data, F71882FG_REG_TEMP_HIGH(nr));
-	val = clamp_val(val, data->temp_high[nr] - 15, data->temp_high[nr]);
-	val = data->temp_high[nr] - val;
-
-	/* convert value to register contents */
-	reg = f71882fg_read8(data, F71882FG_REG_TEMP_HYST(nr / 2));
-	if (nr & 1)
-		reg = (reg & 0x0f) | (val << 4);
-	else
-		reg = (reg & 0xf0) | val;
-	f71882fg_write8(data, F71882FG_REG_TEMP_HYST(nr / 2), reg);
-	data->temp_hyst[nr / 2] = reg;
-
-	mutex_unlock(&data->update_lock);
-	return ret;
-}
-
-static ssize_t show_temp_crit(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	return sprintf(buf, "%d\n", data->temp_ovt[nr] * 1000);
-}
-
-static ssize_t store_temp_crit(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	val /= 1000;
-	val = clamp_val(val, 0, 255);
-
-	mutex_lock(&data->update_lock);
-	f71882fg_write8(data, F71882FG_REG_TEMP_OVT(nr), val);
-	data->temp_ovt[nr] = val;
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_temp_crit_hyst(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-	int temp_crit_hyst;
-
-	mutex_lock(&data->update_lock);
-	if (nr & 1)
-		temp_crit_hyst = data->temp_hyst[nr / 2] >> 4;
-	else
-		temp_crit_hyst = data->temp_hyst[nr / 2] & 0x0f;
-	temp_crit_hyst = (data->temp_ovt[nr] - temp_crit_hyst) * 1000;
-	mutex_unlock(&data->update_lock);
-
-	return sprintf(buf, "%d\n", temp_crit_hyst);
-}
-
-static ssize_t show_temp_type(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	return sprintf(buf, "%d\n", data->temp_type[nr]);
-}
-
-static ssize_t show_temp_beep(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	if (data->temp_beep & (1 << nr))
-		return sprintf(buf, "1\n");
-	else
-		return sprintf(buf, "0\n");
-}
-
-static ssize_t store_temp_beep(struct device *dev, struct device_attribute
-	*devattr, const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	unsigned long val;
-
-	err = kstrtoul(buf, 10, &val);
-	if (err)
-		return err;
-
-	mutex_lock(&data->update_lock);
-	data->temp_beep = f71882fg_read8(data, F71882FG_REG_TEMP_BEEP);
-	if (val)
-		data->temp_beep |= 1 << nr;
-	else
-		data->temp_beep &= ~(1 << nr);
-
-	f71882fg_write8(data, F71882FG_REG_TEMP_BEEP, data->temp_beep);
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_temp_alarm(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	if (data->temp_status & (1 << nr))
-		return sprintf(buf, "1\n");
-	else
-		return sprintf(buf, "0\n");
-}
-
-static ssize_t show_temp_fault(struct device *dev, struct device_attribute
-	*devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	if (data->temp_diode_open & (1 << nr))
-		return sprintf(buf, "1\n");
-	else
-		return sprintf(buf, "0\n");
-}
-
-static ssize_t show_pwm(struct device *dev,
-			struct device_attribute *devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int val, nr = to_sensor_dev_attr_2(devattr)->index;
-	mutex_lock(&data->update_lock);
-	if (data->pwm_enable & (1 << (2 * nr)))
-		/* PWM mode */
-		val = data->pwm[nr];
-	else {
-		/* RPM mode */
-		val = 255 * fan_from_reg(data->fan_target[nr])
-			/ fan_from_reg(data->fan_full_speed[nr]);
-	}
-	mutex_unlock(&data->update_lock);
-	return sprintf(buf, "%d\n", val);
-}
-
-static ssize_t store_pwm(struct device *dev,
-			 struct device_attribute *devattr, const char *buf,
-			 size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	val = clamp_val(val, 0, 255);
-
-	mutex_lock(&data->update_lock);
-	data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE);
-	if ((data->type == f8000 && ((data->pwm_enable >> 2 * nr) & 3) != 2) ||
-	    (data->type != f8000 && !((data->pwm_enable >> 2 * nr) & 2))) {
-		count = -EROFS;
-		goto leave;
-	}
-	if (data->pwm_enable & (1 << (2 * nr))) {
-		/* PWM mode */
-		f71882fg_write8(data, F71882FG_REG_PWM(nr), val);
-		data->pwm[nr] = val;
-	} else {
-		/* RPM mode */
-		int target, full_speed;
-		full_speed = f71882fg_read16(data,
-					     F71882FG_REG_FAN_FULL_SPEED(nr));
-		target = fan_to_reg(val * fan_from_reg(full_speed) / 255);
-		f71882fg_write16(data, F71882FG_REG_FAN_TARGET(nr), target);
-		data->fan_target[nr] = target;
-		data->fan_full_speed[nr] = full_speed;
-	}
-leave:
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_simple_pwm(struct device *dev,
-			       struct device_attribute *devattr, char *buf)
-{
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int val, nr = to_sensor_dev_attr_2(devattr)->index;
-
-	val = data->pwm[nr];
-	return sprintf(buf, "%d\n", val);
-}
-
-static ssize_t store_simple_pwm(struct device *dev,
-				struct device_attribute *devattr,
-				const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	val = clamp_val(val, 0, 255);
-
-	mutex_lock(&data->update_lock);
-	f71882fg_write8(data, F71882FG_REG_PWM(nr), val);
-	data->pwm[nr] = val;
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_pwm_enable(struct device *dev,
-			       struct device_attribute *devattr, char *buf)
-{
-	int result = 0;
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	switch ((data->pwm_enable >> 2 * nr) & 3) {
-	case 0:
-	case 1:
-		result = 2; /* Normal auto mode */
-		break;
-	case 2:
-		result = 1; /* Manual mode */
-		break;
-	case 3:
-		if (data->type == f8000)
-			result = 3; /* Thermostat mode */
-		else
-			result = 1; /* Manual mode */
-		break;
-	}
-
-	return sprintf(buf, "%d\n", result);
-}
-
-static ssize_t store_pwm_enable(struct device *dev, struct device_attribute
-				*devattr, const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	/* Special case for F8000 pwm channel 3 which only does auto mode */
-	if (data->type == f8000 && nr == 2 && val != 2)
-		return -EINVAL;
-
-	mutex_lock(&data->update_lock);
-	data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE);
-	/* Special case for F8000 auto PWM mode / Thermostat mode */
-	if (data->type == f8000 && ((data->pwm_enable >> 2 * nr) & 1)) {
-		switch (val) {
-		case 2:
-			data->pwm_enable &= ~(2 << (2 * nr));
-			break;		/* Normal auto mode */
-		case 3:
-			data->pwm_enable |= 2 << (2 * nr);
-			break;		/* Thermostat mode */
-		default:
-			count = -EINVAL;
-			goto leave;
-		}
-	} else {
-		switch (val) {
-		case 1:
-			/* The f71858fg does not support manual RPM mode */
-			if (data->type == f71858fg &&
-			    ((data->pwm_enable >> (2 * nr)) & 1)) {
-				count = -EINVAL;
-				goto leave;
-			}
-			data->pwm_enable |= 2 << (2 * nr);
-			break;		/* Manual */
-		case 2:
-			data->pwm_enable &= ~(2 << (2 * nr));
-			break;		/* Normal auto mode */
-		default:
-			count = -EINVAL;
-			goto leave;
-		}
-	}
-	f71882fg_write8(data, F71882FG_REG_PWM_ENABLE, data->pwm_enable);
-leave:
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_pwm_auto_point_pwm(struct device *dev,
-				       struct device_attribute *devattr,
-				       char *buf)
-{
-	int result;
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int pwm = to_sensor_dev_attr_2(devattr)->index;
-	int point = to_sensor_dev_attr_2(devattr)->nr;
-
-	mutex_lock(&data->update_lock);
-	if (data->pwm_enable & (1 << (2 * pwm))) {
-		/* PWM mode */
-		result = data->pwm_auto_point_pwm[pwm][point];
-	} else {
-		/* RPM mode */
-		result = 32 * 255 / (32 + data->pwm_auto_point_pwm[pwm][point]);
-	}
-	mutex_unlock(&data->update_lock);
-
-	return sprintf(buf, "%d\n", result);
-}
-
-static ssize_t store_pwm_auto_point_pwm(struct device *dev,
-					struct device_attribute *devattr,
-					const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, pwm = to_sensor_dev_attr_2(devattr)->index;
-	int point = to_sensor_dev_attr_2(devattr)->nr;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	val = clamp_val(val, 0, 255);
-
-	mutex_lock(&data->update_lock);
-	data->pwm_enable = f71882fg_read8(data, F71882FG_REG_PWM_ENABLE);
-	if (data->pwm_enable & (1 << (2 * pwm))) {
-		/* PWM mode */
-	} else {
-		/* RPM mode */
-		if (val < 29)	/* Prevent negative numbers */
-			val = 255;
-		else
-			val = (255 - val) * 32 / val;
-	}
-	f71882fg_write8(data, F71882FG_REG_POINT_PWM(pwm, point), val);
-	data->pwm_auto_point_pwm[pwm][point] = val;
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_pwm_auto_point_temp_hyst(struct device *dev,
-					     struct device_attribute *devattr,
-					     char *buf)
-{
-	int result = 0;
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-	int point = to_sensor_dev_attr_2(devattr)->nr;
-
-	mutex_lock(&data->update_lock);
-	if (nr & 1)
-		result = data->pwm_auto_point_hyst[nr / 2] >> 4;
-	else
-		result = data->pwm_auto_point_hyst[nr / 2] & 0x0f;
-	result = 1000 * (data->pwm_auto_point_temp[nr][point] - result);
-	mutex_unlock(&data->update_lock);
-
-	return sprintf(buf, "%d\n", result);
-}
-
-static ssize_t store_pwm_auto_point_temp_hyst(struct device *dev,
-					      struct device_attribute *devattr,
-					      const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	int point = to_sensor_dev_attr_2(devattr)->nr;
-	u8 reg;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	val /= 1000;
-
-	mutex_lock(&data->update_lock);
-	data->pwm_auto_point_temp[nr][point] =
-		f71882fg_read8(data, F71882FG_REG_POINT_TEMP(nr, point));
-	val = clamp_val(val, data->pwm_auto_point_temp[nr][point] - 15,
-			data->pwm_auto_point_temp[nr][point]);
-	val = data->pwm_auto_point_temp[nr][point] - val;
-
-	reg = f71882fg_read8(data, F71882FG_REG_FAN_HYST(nr / 2));
-	if (nr & 1)
-		reg = (reg & 0x0f) | (val << 4);
-	else
-		reg = (reg & 0xf0) | val;
-
-	f71882fg_write8(data, F71882FG_REG_FAN_HYST(nr / 2), reg);
-	data->pwm_auto_point_hyst[nr / 2] = reg;
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_pwm_interpolate(struct device *dev,
-				    struct device_attribute *devattr, char *buf)
-{
-	int result;
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	result = (data->pwm_auto_point_mapping[nr] >> 4) & 1;
-
-	return sprintf(buf, "%d\n", result);
-}
-
-static ssize_t store_pwm_interpolate(struct device *dev,
-				     struct device_attribute *devattr,
-				     const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	unsigned long val;
-
-	err = kstrtoul(buf, 10, &val);
-	if (err)
-		return err;
-
-	mutex_lock(&data->update_lock);
-	data->pwm_auto_point_mapping[nr] =
-		f71882fg_read8(data, F71882FG_REG_POINT_MAPPING(nr));
-	if (val)
-		val = data->pwm_auto_point_mapping[nr] | (1 << 4);
-	else
-		val = data->pwm_auto_point_mapping[nr] & (~(1 << 4));
-	f71882fg_write8(data, F71882FG_REG_POINT_MAPPING(nr), val);
-	data->pwm_auto_point_mapping[nr] = val;
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_pwm_auto_point_channel(struct device *dev,
-					   struct device_attribute *devattr,
-					   char *buf)
-{
-	int result;
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int nr = to_sensor_dev_attr_2(devattr)->index;
-
-	result = 1 << ((data->pwm_auto_point_mapping[nr] & 3) -
-		       data->temp_start);
-
-	return sprintf(buf, "%d\n", result);
-}
-
-static ssize_t store_pwm_auto_point_channel(struct device *dev,
-					    struct device_attribute *devattr,
-					    const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, nr = to_sensor_dev_attr_2(devattr)->index;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	switch (val) {
-	case 1:
-		val = 0;
-		break;
-	case 2:
-		val = 1;
-		break;
-	case 4:
-		val = 2;
-		break;
-	default:
-		return -EINVAL;
-	}
-	val += data->temp_start;
-	mutex_lock(&data->update_lock);
-	data->pwm_auto_point_mapping[nr] =
-		f71882fg_read8(data, F71882FG_REG_POINT_MAPPING(nr));
-	val = (data->pwm_auto_point_mapping[nr] & 0xfc) | val;
-	f71882fg_write8(data, F71882FG_REG_POINT_MAPPING(nr), val);
-	data->pwm_auto_point_mapping[nr] = val;
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t show_pwm_auto_point_temp(struct device *dev,
-					struct device_attribute *devattr,
-					char *buf)
-{
-	int result;
-	struct f71882fg_data *data = f71882fg_update_device(dev);
-	int pwm = to_sensor_dev_attr_2(devattr)->index;
-	int point = to_sensor_dev_attr_2(devattr)->nr;
-
-	result = data->pwm_auto_point_temp[pwm][point];
-	return sprintf(buf, "%d\n", 1000 * result);
-}
-
-static ssize_t store_pwm_auto_point_temp(struct device *dev,
-					 struct device_attribute *devattr,
-					 const char *buf, size_t count)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	int err, pwm = to_sensor_dev_attr_2(devattr)->index;
-	int point = to_sensor_dev_attr_2(devattr)->nr;
-	long val;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	val /= 1000;
-
-	if (data->auto_point_temp_signed)
-		val = clamp_val(val, -128, 127);
-	else
-		val = clamp_val(val, 0, 127);
-
-	mutex_lock(&data->update_lock);
-	f71882fg_write8(data, F71882FG_REG_POINT_TEMP(pwm, point), val);
-	data->pwm_auto_point_temp[pwm][point] = val;
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
-
-static ssize_t name_show(struct device *dev, struct device_attribute *devattr,
-	char *buf)
-{
-	struct f71882fg_data *data = dev_get_drvdata(dev);
-	return sprintf(buf, "%s\n", f71882fg_names[data->type]);
-}
-
 static int f71882fg_create_sysfs_files(struct platform_device *pdev,
 	struct sensor_device_attribute_2 *attr, int count)
 {
@@ -2329,6 +2220,119 @@ static int f71882fg_create_fan_sysfs_files(
 	return err;
 }
 
+static int f71882fg_remove(struct platform_device *pdev)
+{
+	struct f71882fg_data *data = platform_get_drvdata(pdev);
+	int nr_fans = f71882fg_nr_fans[data->type];
+	int nr_temps = f71882fg_nr_temps[data->type];
+	int i;
+	u8 start_reg = f71882fg_read8(data, F71882FG_REG_START);
+
+	if (data->hwmon_dev)
+		hwmon_device_unregister(data->hwmon_dev);
+
+	device_remove_file(&pdev->dev, &dev_attr_name);
+
+	if (start_reg & 0x01) {
+		switch (data->type) {
+		case f71858fg:
+			if (data->temp_config & 0x10)
+				f71882fg_remove_sysfs_files(pdev,
+					f8000_temp_attr,
+					ARRAY_SIZE(f8000_temp_attr));
+			else
+				f71882fg_remove_sysfs_files(pdev,
+					f71858fg_temp_attr,
+					ARRAY_SIZE(f71858fg_temp_attr));
+			break;
+		case f8000:
+			f71882fg_remove_sysfs_files(pdev,
+					f8000_temp_attr,
+					ARRAY_SIZE(f8000_temp_attr));
+			break;
+		case f81866a:
+			f71882fg_remove_sysfs_files(pdev,
+					f71858fg_temp_attr,
+					ARRAY_SIZE(f71858fg_temp_attr));
+			break;
+		default:
+			f71882fg_remove_sysfs_files(pdev,
+				&fxxxx_temp_attr[0][0],
+				ARRAY_SIZE(fxxxx_temp_attr[0]) * nr_temps);
+		}
+		if (f71882fg_temp_has_beep[data->type]) {
+			if (data->type == f81866a)
+				f71882fg_remove_sysfs_files(pdev,
+					&f81866_temp_beep_attr[0][0],
+					ARRAY_SIZE(f81866_temp_beep_attr[0])
+						* nr_temps);
+			else
+				f71882fg_remove_sysfs_files(pdev,
+					&fxxxx_temp_beep_attr[0][0],
+					ARRAY_SIZE(fxxxx_temp_beep_attr[0])
+						* nr_temps);
+		}
+
+		for (i = 0; i < F71882FG_MAX_INS; i++) {
+			if (f71882fg_has_in[data->type][i]) {
+				device_remove_file(&pdev->dev,
+						&fxxxx_in_attr[i].dev_attr);
+			}
+		}
+		if (f71882fg_has_in1_alarm[data->type]) {
+			f71882fg_remove_sysfs_files(pdev,
+					fxxxx_in1_alarm_attr,
+					ARRAY_SIZE(fxxxx_in1_alarm_attr));
+		}
+	}
+
+	if (start_reg & 0x02) {
+		f71882fg_remove_sysfs_files(pdev, &fxxxx_fan_attr[0][0],
+				ARRAY_SIZE(fxxxx_fan_attr[0]) * nr_fans);
+
+		if (f71882fg_fan_has_beep[data->type]) {
+			f71882fg_remove_sysfs_files(pdev,
+					fxxxx_fan_beep_attr, nr_fans);
+		}
+
+		switch (data->type) {
+		case f71808a:
+			f71882fg_remove_sysfs_files(pdev,
+				&fxxxx_auto_pwm_attr[0][0],
+				ARRAY_SIZE(fxxxx_auto_pwm_attr[0]) * nr_fans);
+			f71882fg_remove_sysfs_files(pdev,
+					f71808a_fan3_attr,
+					ARRAY_SIZE(f71808a_fan3_attr));
+			break;
+		case f71862fg:
+			f71882fg_remove_sysfs_files(pdev,
+				&f71862fg_auto_pwm_attr[0][0],
+				ARRAY_SIZE(f71862fg_auto_pwm_attr[0]) *
+					nr_fans);
+			break;
+		case f71808e:
+		case f71869:
+			f71882fg_remove_sysfs_files(pdev,
+				&f71869_auto_pwm_attr[0][0],
+				ARRAY_SIZE(f71869_auto_pwm_attr[0]) * nr_fans);
+			break;
+		case f8000:
+			f71882fg_remove_sysfs_files(pdev,
+					f8000_fan_attr,
+					ARRAY_SIZE(f8000_fan_attr));
+			f71882fg_remove_sysfs_files(pdev,
+				&f8000_auto_pwm_attr[0][0],
+				ARRAY_SIZE(f8000_auto_pwm_attr[0]) * nr_fans);
+			break;
+		default:
+			f71882fg_remove_sysfs_files(pdev,
+				&fxxxx_auto_pwm_attr[0][0],
+				ARRAY_SIZE(fxxxx_auto_pwm_attr[0]) * nr_fans);
+		}
+	}
+	return 0;
+}
+
 static int f71882fg_probe(struct platform_device *pdev)
 {
 	struct f71882fg_data *data;
@@ -2502,119 +2506,6 @@ exit_unregister_sysfs:
 	return err; /* f71882fg_remove() also frees our data */
 }
 
-static int f71882fg_remove(struct platform_device *pdev)
-{
-	struct f71882fg_data *data = platform_get_drvdata(pdev);
-	int nr_fans = f71882fg_nr_fans[data->type];
-	int nr_temps = f71882fg_nr_temps[data->type];
-	int i;
-	u8 start_reg = f71882fg_read8(data, F71882FG_REG_START);
-
-	if (data->hwmon_dev)
-		hwmon_device_unregister(data->hwmon_dev);
-
-	device_remove_file(&pdev->dev, &dev_attr_name);
-
-	if (start_reg & 0x01) {
-		switch (data->type) {
-		case f71858fg:
-			if (data->temp_config & 0x10)
-				f71882fg_remove_sysfs_files(pdev,
-					f8000_temp_attr,
-					ARRAY_SIZE(f8000_temp_attr));
-			else
-				f71882fg_remove_sysfs_files(pdev,
-					f71858fg_temp_attr,
-					ARRAY_SIZE(f71858fg_temp_attr));
-			break;
-		case f8000:
-			f71882fg_remove_sysfs_files(pdev,
-					f8000_temp_attr,
-					ARRAY_SIZE(f8000_temp_attr));
-			break;
-		case f81866a:
-			f71882fg_remove_sysfs_files(pdev,
-					f71858fg_temp_attr,
-					ARRAY_SIZE(f71858fg_temp_attr));
-			break;
-		default:
-			f71882fg_remove_sysfs_files(pdev,
-				&fxxxx_temp_attr[0][0],
-				ARRAY_SIZE(fxxxx_temp_attr[0]) * nr_temps);
-		}
-		if (f71882fg_temp_has_beep[data->type]) {
-			if (data->type == f81866a)
-				f71882fg_remove_sysfs_files(pdev,
-					&f81866_temp_beep_attr[0][0],
-					ARRAY_SIZE(f81866_temp_beep_attr[0])
-						* nr_temps);
-			else
-				f71882fg_remove_sysfs_files(pdev,
-					&fxxxx_temp_beep_attr[0][0],
-					ARRAY_SIZE(fxxxx_temp_beep_attr[0])
-						* nr_temps);
-		}
-
-		for (i = 0; i < F71882FG_MAX_INS; i++) {
-			if (f71882fg_has_in[data->type][i]) {
-				device_remove_file(&pdev->dev,
-						&fxxxx_in_attr[i].dev_attr);
-			}
-		}
-		if (f71882fg_has_in1_alarm[data->type]) {
-			f71882fg_remove_sysfs_files(pdev,
-					fxxxx_in1_alarm_attr,
-					ARRAY_SIZE(fxxxx_in1_alarm_attr));
-		}
-	}
-
-	if (start_reg & 0x02) {
-		f71882fg_remove_sysfs_files(pdev, &fxxxx_fan_attr[0][0],
-				ARRAY_SIZE(fxxxx_fan_attr[0]) * nr_fans);
-
-		if (f71882fg_fan_has_beep[data->type]) {
-			f71882fg_remove_sysfs_files(pdev,
-					fxxxx_fan_beep_attr, nr_fans);
-		}
-
-		switch (data->type) {
-		case f71808a:
-			f71882fg_remove_sysfs_files(pdev,
-				&fxxxx_auto_pwm_attr[0][0],
-				ARRAY_SIZE(fxxxx_auto_pwm_attr[0]) * nr_fans);
-			f71882fg_remove_sysfs_files(pdev,
-					f71808a_fan3_attr,
-					ARRAY_SIZE(f71808a_fan3_attr));
-			break;
-		case f71862fg:
-			f71882fg_remove_sysfs_files(pdev,
-				&f71862fg_auto_pwm_attr[0][0],
-				ARRAY_SIZE(f71862fg_auto_pwm_attr[0]) *
-					nr_fans);
-			break;
-		case f71808e:
-		case f71869:
-			f71882fg_remove_sysfs_files(pdev,
-				&f71869_auto_pwm_attr[0][0],
-				ARRAY_SIZE(f71869_auto_pwm_attr[0]) * nr_fans);
-			break;
-		case f8000:
-			f71882fg_remove_sysfs_files(pdev,
-					f8000_fan_attr,
-					ARRAY_SIZE(f8000_fan_attr));
-			f71882fg_remove_sysfs_files(pdev,
-				&f8000_auto_pwm_attr[0][0],
-				ARRAY_SIZE(f8000_auto_pwm_attr[0]) * nr_fans);
-			break;
-		default:
-			f71882fg_remove_sysfs_files(pdev,
-				&fxxxx_auto_pwm_attr[0][0],
-				ARRAY_SIZE(fxxxx_auto_pwm_attr[0]) * nr_fans);
-		}
-	}
-	return 0;
-}
-
 static int __init f71882fg_find(int sioaddr, struct f71882fg_sio_data *sio_data)
 {
 	u16 devid;
@@ -2760,6 +2651,14 @@ exit_device_put:
 	return err;
 }
 
+static struct platform_driver f71882fg_driver = {
+	.driver = {
+		.name	= DRVNAME,
+	},
+	.probe		= f71882fg_probe,
+	.remove		= f71882fg_remove,
+};
+
 static int __init f71882fg_init(void)
 {
 	int err;
diff --git a/drivers/hwmon/f75375s.c b/drivers/hwmon/f75375s.c
index 57c8a473698d..64fbb8cf687c 100644
--- a/drivers/hwmon/f75375s.c
+++ b/drivers/hwmon/f75375s.c
@@ -114,7 +114,7 @@ struct f75375_data {
 static int f75375_detect(struct i2c_client *client,
 			 struct i2c_board_info *info);
 static int f75375_probe(struct i2c_client *client);
-static int f75375_remove(struct i2c_client *client);
+static void f75375_remove(struct i2c_client *client);
 
 static const struct i2c_device_id f75375_id[] = {
 	{ "f75373", f75373 },
@@ -864,12 +864,11 @@ exit_remove:
 	return err;
 }
 
-static int f75375_remove(struct i2c_client *client)
+static void f75375_remove(struct i2c_client *client)
 {
 	struct f75375_data *data = i2c_get_clientdata(client);
 	hwmon_device_unregister(data->hwmon_dev);
 	sysfs_remove_group(&client->dev.kobj, &f75375_group);
-	return 0;
 }
 
 /* Return 0 if detection is successful, -ENODEV otherwise */
@@ -897,7 +896,7 @@ static int f75375_detect(struct i2c_client *client,
 
 	version = f75375_read8(client, F75375_REG_VERSION);
 	dev_info(&adapter->dev, "found %s version: %02X\n", name, version);
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/fschmd.c b/drivers/hwmon/fschmd.c
index c26195e3aad7..0a77d6161928 100644
--- a/drivers/hwmon/fschmd.c
+++ b/drivers/hwmon/fschmd.c
@@ -217,7 +217,7 @@ static const int FSCHMD_NO_TEMP_SENSORS[7] = { 3, 3, 4, 3, 5, 5, 11 };
 static int fschmd_probe(struct i2c_client *client);
 static int fschmd_detect(struct i2c_client *client,
 			 struct i2c_board_info *info);
-static int fschmd_remove(struct i2c_client *client);
+static void fschmd_remove(struct i2c_client *client);
 static struct fschmd_data *fschmd_update_device(struct device *dev);
 
 /*
@@ -1075,7 +1075,7 @@ static int fschmd_detect(struct i2c_client *client,
 	else
 		return -ENODEV;
 
-	strlcpy(info->type, fschmd_id[kind].name, I2C_NAME_SIZE);
+	strscpy(info->type, fschmd_id[kind].name, I2C_NAME_SIZE);
 
 	return 0;
 }
@@ -1248,7 +1248,7 @@ exit_detach:
 	return err;
 }
 
-static int fschmd_remove(struct i2c_client *client)
+static void fschmd_remove(struct i2c_client *client)
 {
 	struct fschmd_data *data = i2c_get_clientdata(client);
 	int i;
@@ -1291,8 +1291,6 @@ static int fschmd_remove(struct i2c_client *client)
 	mutex_lock(&watchdog_data_mutex);
 	kref_put(&data->kref, fschmd_release_resources);
 	mutex_unlock(&watchdog_data_mutex);
-
-	return 0;
 }
 
 static struct fschmd_data *fschmd_update_device(struct device *dev)
diff --git a/drivers/hwmon/ftsteutates.c b/drivers/hwmon/ftsteutates.c
index ceffc76a0c51..f5b8e724a8ca 100644
--- a/drivers/hwmon/ftsteutates.c
+++ b/drivers/hwmon/ftsteutates.c
@@ -739,17 +739,16 @@ static int fts_detect(struct i2c_client *client,
 	if (val != 0x11)
 		return -ENODEV;
 
-	strlcpy(info->type, fts_id[0].name, I2C_NAME_SIZE);
+	strscpy(info->type, fts_id[0].name, I2C_NAME_SIZE);
 	info->flags = 0;
 	return 0;
 }
 
-static int fts_remove(struct i2c_client *client)
+static void fts_remove(struct i2c_client *client)
 {
 	struct fts_data *data = dev_get_drvdata(&client->dev);
 
 	watchdog_unregister_device(&data->wdd);
-	return 0;
 }
 
 static int fts_probe(struct i2c_client *client)
diff --git a/drivers/hwmon/gl518sm.c b/drivers/hwmon/gl518sm.c
index dd683b0a648f..95286c40f55a 100644
--- a/drivers/hwmon/gl518sm.c
+++ b/drivers/hwmon/gl518sm.c
@@ -586,7 +586,7 @@ static int gl518_detect(struct i2c_client *client, struct i2c_board_info *info)
 	if (rev != 0x00 && rev != 0x80)
 		return -ENODEV;
 
-	strlcpy(info->type, "gl518sm", I2C_NAME_SIZE);
+	strscpy(info->type, "gl518sm", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/gl520sm.c b/drivers/hwmon/gl520sm.c
index 096ba9797211..394da4ac977c 100644
--- a/drivers/hwmon/gl520sm.c
+++ b/drivers/hwmon/gl520sm.c
@@ -811,7 +811,7 @@ static int gl520_detect(struct i2c_client *client, struct i2c_board_info *info)
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, "gl520sm", I2C_NAME_SIZE);
+	strscpy(info->type, "gl520sm", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/gpio-fan.c b/drivers/hwmon/gpio-fan.c
index fbf3f5a4ecb6..ba408942dbe7 100644
--- a/drivers/hwmon/gpio-fan.c
+++ b/drivers/hwmon/gpio-fan.c
@@ -37,9 +37,7 @@ struct gpio_fan_data {
 	int			num_speed;
 	struct gpio_fan_speed	*speed;
 	int			speed_index;
-#ifdef CONFIG_PM_SLEEP
 	int			resume_speed;
-#endif
 	bool			pwm_enable;
 	struct gpio_desc	*alarm_gpio;
 	struct work_struct	alarm_work;
@@ -557,7 +555,6 @@ static void gpio_fan_shutdown(struct platform_device *pdev)
 		set_fan_speed(fan_data, 0);
 }
 
-#ifdef CONFIG_PM_SLEEP
 static int gpio_fan_suspend(struct device *dev)
 {
 	struct gpio_fan_data *fan_data = dev_get_drvdata(dev);
@@ -580,18 +577,14 @@ static int gpio_fan_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(gpio_fan_pm, gpio_fan_suspend, gpio_fan_resume);
-#define GPIO_FAN_PM	(&gpio_fan_pm)
-#else
-#define GPIO_FAN_PM	NULL
-#endif
+static DEFINE_SIMPLE_DEV_PM_OPS(gpio_fan_pm, gpio_fan_suspend, gpio_fan_resume);
 
 static struct platform_driver gpio_fan_driver = {
 	.probe		= gpio_fan_probe,
 	.shutdown	= gpio_fan_shutdown,
 	.driver	= {
 		.name	= "gpio-fan",
-		.pm	= GPIO_FAN_PM,
+		.pm	= pm_sleep_ptr(&gpio_fan_pm),
 		.of_match_table = of_match_ptr(of_gpio_fan_match),
 	},
 };
diff --git a/drivers/hwmon/gsc-hwmon.c b/drivers/hwmon/gsc-hwmon.c
index d64be48f1ef6..b60ec95b5edb 100644
--- a/drivers/hwmon/gsc-hwmon.c
+++ b/drivers/hwmon/gsc-hwmon.c
@@ -267,6 +267,7 @@ gsc_hwmon_get_devtree_pdata(struct device *dev)
 	pdata->nchannels = nchannels;
 
 	/* fan controller base address */
+	of_node_get(dev->parent->of_node);
 	fan = of_find_compatible_node(dev->parent->of_node, NULL, "gw,gsc-fan");
 	if (fan && of_property_read_u32(fan, "reg", &pdata->fan_base)) {
 		of_node_put(fan);
diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c
index 2e2cd79d89eb..4218750d5a66 100644
--- a/drivers/hwmon/hwmon.c
+++ b/drivers/hwmon/hwmon.c
@@ -151,9 +151,9 @@ static DEFINE_IDA(hwmon_ida);
  * between hwmon and thermal_sys modules.
  */
 #ifdef CONFIG_THERMAL_OF
-static int hwmon_thermal_get_temp(void *data, int *temp)
+static int hwmon_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct hwmon_thermal_data *tdata = data;
+	struct hwmon_thermal_data *tdata = tz->devdata;
 	struct hwmon_device *hwdev = to_hwmon_device(tdata->dev);
 	int ret;
 	long t;
@@ -168,9 +168,9 @@ static int hwmon_thermal_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static int hwmon_thermal_set_trips(void *data, int low, int high)
+static int hwmon_thermal_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct hwmon_thermal_data *tdata = data;
+	struct hwmon_thermal_data *tdata = tz->devdata;
 	struct hwmon_device *hwdev = to_hwmon_device(tdata->dev);
 	const struct hwmon_chip_info *chip = hwdev->chip;
 	const struct hwmon_channel_info **info = chip->info;
@@ -203,7 +203,7 @@ static int hwmon_thermal_set_trips(void *data, int low, int high)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops hwmon_thermal_ops = {
+static const struct thermal_zone_device_ops hwmon_thermal_ops = {
 	.get_temp = hwmon_thermal_get_temp,
 	.set_trips = hwmon_thermal_set_trips,
 };
@@ -227,8 +227,8 @@ static int hwmon_thermal_add_sensor(struct device *dev, int index)
 	tdata->dev = dev;
 	tdata->index = index;
 
-	tzd = devm_thermal_zone_of_sensor_register(dev, index, tdata,
-						   &hwmon_thermal_ops);
+	tzd = devm_thermal_of_zone_register(dev, index, tdata,
+					    &hwmon_thermal_ops);
 	if (IS_ERR(tzd)) {
 		if (PTR_ERR(tzd) != -ENODEV)
 			return PTR_ERR(tzd);
diff --git a/drivers/hwmon/iio_hwmon.c b/drivers/hwmon/iio_hwmon.c
index 580a7d125b88..3aa40893fc09 100644
--- a/drivers/hwmon/iio_hwmon.c
+++ b/drivers/hwmon/iio_hwmon.c
@@ -6,11 +6,13 @@
 
 #include <linux/kernel.h>
 #include <linux/slab.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/err.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
+
 #include <linux/hwmon.h>
-#include <linux/of.h>
 #include <linux/hwmon-sysfs.h>
 #include <linux/iio/consumer.h>
 #include <linux/iio/types.h>
@@ -149,8 +151,8 @@ static int iio_hwmon_probe(struct platform_device *pdev)
 	st->attr_group.attrs = st->attrs;
 	st->groups[0] = &st->attr_group;
 
-	if (dev->of_node) {
-		sname = devm_kasprintf(dev, GFP_KERNEL, "%pOFn", dev->of_node);
+	if (dev_fwnode(dev)) {
+		sname = devm_kasprintf(dev, GFP_KERNEL, "%pfwP", dev_fwnode(dev));
 		if (!sname)
 			return -ENOMEM;
 		strreplace(sname, '-', '_');
diff --git a/drivers/hwmon/ina209.c b/drivers/hwmon/ina209.c
index fc3007c3e85c..9b58655d2de4 100644
--- a/drivers/hwmon/ina209.c
+++ b/drivers/hwmon/ina209.c
@@ -568,13 +568,11 @@ out_restore_conf:
 	return ret;
 }
 
-static int ina209_remove(struct i2c_client *client)
+static void ina209_remove(struct i2c_client *client)
 {
 	struct ina209_data *data = i2c_get_clientdata(client);
 
 	ina209_restore_conf(client, data);
-
-	return 0;
 }
 
 static const struct i2c_device_id ina209_id[] = {
diff --git a/drivers/hwmon/ina3221.c b/drivers/hwmon/ina3221.c
index 58d3828e2ec0..2a57f4b60c29 100644
--- a/drivers/hwmon/ina3221.c
+++ b/drivers/hwmon/ina3221.c
@@ -913,7 +913,7 @@ fail:
 	return ret;
 }
 
-static int ina3221_remove(struct i2c_client *client)
+static void ina3221_remove(struct i2c_client *client)
 {
 	struct ina3221_data *ina = dev_get_drvdata(&client->dev);
 	int i;
@@ -926,11 +926,9 @@ static int ina3221_remove(struct i2c_client *client)
 		pm_runtime_put_noidle(ina->pm_dev);
 
 	mutex_destroy(&ina->lock);
-
-	return 0;
 }
 
-static int __maybe_unused ina3221_suspend(struct device *dev)
+static int ina3221_suspend(struct device *dev)
 {
 	struct ina3221_data *ina = dev_get_drvdata(dev);
 	int ret;
@@ -953,7 +951,7 @@ static int __maybe_unused ina3221_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused ina3221_resume(struct device *dev)
+static int ina3221_resume(struct device *dev)
 {
 	struct ina3221_data *ina = dev_get_drvdata(dev);
 	int ret;
@@ -996,11 +994,8 @@ static int __maybe_unused ina3221_resume(struct device *dev)
 	return 0;
 }
 
-static const struct dev_pm_ops ina3221_pm = {
-	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
-				pm_runtime_force_resume)
-	SET_RUNTIME_PM_OPS(ina3221_suspend, ina3221_resume, NULL)
-};
+static DEFINE_RUNTIME_DEV_PM_OPS(ina3221_pm, ina3221_suspend, ina3221_resume,
+				 NULL);
 
 static const struct of_device_id ina3221_of_match_table[] = {
 	{ .compatible = "ti,ina3221", },
@@ -1020,7 +1015,7 @@ static struct i2c_driver ina3221_i2c_driver = {
 	.driver = {
 		.name = INA3221_DRIVER_NAME,
 		.of_match_table = ina3221_of_match_table,
-		.pm = &ina3221_pm,
+		.pm = pm_ptr(&ina3221_pm),
 	},
 	.id_table = ina3221_ids,
 };
diff --git a/drivers/hwmon/it87.c b/drivers/hwmon/it87.c
index 0e543dbe0a6b..7bd154ba351b 100644
--- a/drivers/hwmon/it87.c
+++ b/drivers/hwmon/it87.c
@@ -3179,7 +3179,7 @@ static int it87_probe(struct platform_device *pdev)
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
-static void __maybe_unused it87_resume_sio(struct platform_device *pdev)
+static void it87_resume_sio(struct platform_device *pdev)
 {
 	struct it87_data *data = dev_get_drvdata(&pdev->dev);
 	int err;
@@ -3211,7 +3211,7 @@ static void __maybe_unused it87_resume_sio(struct platform_device *pdev)
 	superio_exit(data->sioaddr);
 }
 
-static int __maybe_unused it87_resume(struct device *dev)
+static int it87_resume(struct device *dev)
 {
 	struct platform_device *pdev = to_platform_device(dev);
 	struct it87_data *data = dev_get_drvdata(dev);
@@ -3238,12 +3238,12 @@ static int __maybe_unused it87_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(it87_dev_pm_ops, NULL, it87_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(it87_dev_pm_ops, NULL, it87_resume);
 
 static struct platform_driver it87_driver = {
 	.driver = {
 		.name	= DRVNAME,
-		.pm     = &it87_dev_pm_ops,
+		.pm     = pm_sleep_ptr(&it87_dev_pm_ops),
 	},
 	.probe	= it87_probe,
 };
diff --git a/drivers/hwmon/jc42.c b/drivers/hwmon/jc42.c
index 07f7f8b5b73d..30888feaf589 100644
--- a/drivers/hwmon/jc42.c
+++ b/drivers/hwmon/jc42.c
@@ -441,7 +441,7 @@ static int jc42_detect(struct i2c_client *client, struct i2c_board_info *info)
 		struct jc42_chips *chip = &jc42_chips[i];
 		if (manid == chip->manid &&
 		    (devid & chip->devid_mask) == chip->devid) {
-			strlcpy(info->type, "jc42", I2C_NAME_SIZE);
+			strscpy(info->type, "jc42", I2C_NAME_SIZE);
 			return 0;
 		}
 	}
@@ -524,7 +524,7 @@ static int jc42_probe(struct i2c_client *client)
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
-static int jc42_remove(struct i2c_client *client)
+static void jc42_remove(struct i2c_client *client)
 {
 	struct jc42_data *data = i2c_get_clientdata(client);
 
@@ -537,7 +537,6 @@ static int jc42_remove(struct i2c_client *client)
 		  | (data->config & JC42_CFG_HYST_MASK);
 		i2c_smbus_write_word_swapped(client, JC42_REG_CONFIG, config);
 	}
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/hwmon/lm63.c b/drivers/hwmon/lm63.c
index 339a145afc09..9ab2cab4c710 100644
--- a/drivers/hwmon/lm63.c
+++ b/drivers/hwmon/lm63.c
@@ -996,11 +996,11 @@ static int lm63_detect(struct i2c_client *client,
 	}
 
 	if (chip_id == 0x41 && address == 0x4c)
-		strlcpy(info->type, "lm63", I2C_NAME_SIZE);
+		strscpy(info->type, "lm63", I2C_NAME_SIZE);
 	else if (chip_id == 0x51 && (address == 0x18 || address == 0x4e))
-		strlcpy(info->type, "lm64", I2C_NAME_SIZE);
+		strscpy(info->type, "lm64", I2C_NAME_SIZE);
 	else if (chip_id == 0x49 && address == 0x4c)
-		strlcpy(info->type, "lm96163", I2C_NAME_SIZE);
+		strscpy(info->type, "lm96163", I2C_NAME_SIZE);
 	else
 		return -ENODEV;
 
diff --git a/drivers/hwmon/lm73.c b/drivers/hwmon/lm73.c
index beb0d61bcd82..1346b3b3f463 100644
--- a/drivers/hwmon/lm73.c
+++ b/drivers/hwmon/lm73.c
@@ -257,7 +257,7 @@ static int lm73_detect(struct i2c_client *new_client,
 	if (id < 0 || id != LM73_ID)
 		return -ENODEV;
 
-	strlcpy(info->type, "lm73", I2C_NAME_SIZE);
+	strscpy(info->type, "lm73", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/lm75.c b/drivers/hwmon/lm75.c
index 66dc826f7962..bcc3adcb3af1 100644
--- a/drivers/hwmon/lm75.c
+++ b/drivers/hwmon/lm75.c
@@ -893,7 +893,7 @@ static int lm75_detect(struct i2c_client *new_client,
 			return -ENODEV;
 	}
 
-	strlcpy(info->type, is_lm75a ? "lm75a" : "lm75", I2C_NAME_SIZE);
+	strscpy(info->type, is_lm75a ? "lm75a" : "lm75", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/lm77.c b/drivers/hwmon/lm77.c
index df6af85e170a..645cb2191abe 100644
--- a/drivers/hwmon/lm77.c
+++ b/drivers/hwmon/lm77.c
@@ -302,7 +302,7 @@ static int lm77_detect(struct i2c_client *client, struct i2c_board_info *info)
 	 || i2c_smbus_read_word_data(client, 7) != min)
 		return -ENODEV;
 
-	strlcpy(info->type, "lm77", I2C_NAME_SIZE);
+	strscpy(info->type, "lm77", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/lm78.c b/drivers/hwmon/lm78.c
index 5e129cbec1cb..694e171cab7f 100644
--- a/drivers/hwmon/lm78.c
+++ b/drivers/hwmon/lm78.c
@@ -617,7 +617,7 @@ static int lm78_i2c_detect(struct i2c_client *client,
 	if (isa)
 		mutex_unlock(&isa->update_lock);
 
-	strlcpy(info->type, client_name, I2C_NAME_SIZE);
+	strscpy(info->type, client_name, I2C_NAME_SIZE);
 
 	return 0;
 
diff --git a/drivers/hwmon/lm80.c b/drivers/hwmon/lm80.c
index e85e062bbf32..35db0b97f912 100644
--- a/drivers/hwmon/lm80.c
+++ b/drivers/hwmon/lm80.c
@@ -586,7 +586,7 @@ static int lm80_detect(struct i2c_client *client, struct i2c_board_info *info)
 		name = "lm80";
 	}
 
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/lm83.c b/drivers/hwmon/lm83.c
index 905f5689f907..616449f2cc50 100644
--- a/drivers/hwmon/lm83.c
+++ b/drivers/hwmon/lm83.c
@@ -412,7 +412,7 @@ static int lm83_detect(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/lm85.c b/drivers/hwmon/lm85.c
index 88cf2012d34b..8d33c2484755 100644
--- a/drivers/hwmon/lm85.c
+++ b/drivers/hwmon/lm85.c
@@ -1539,7 +1539,7 @@ static int lm85_detect(struct i2c_client *client, struct i2c_board_info *info)
 	if (!type_name)
 		return -ENODEV;
 
-	strlcpy(info->type, type_name, I2C_NAME_SIZE);
+	strscpy(info->type, type_name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/lm87.c b/drivers/hwmon/lm87.c
index 1750bc588856..818fb6195245 100644
--- a/drivers/hwmon/lm87.c
+++ b/drivers/hwmon/lm87.c
@@ -833,7 +833,7 @@ static int lm87_detect(struct i2c_client *client, struct i2c_board_info *info)
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c
index 221de01a327a..db595f7d01f8 100644
--- a/drivers/hwmon/lm90.c
+++ b/drivers/hwmon/lm90.c
@@ -2547,7 +2547,7 @@ static int lm90_detect(struct i2c_client *client, struct i2c_board_info *info)
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 
 	return 0;
 }
@@ -2956,7 +2956,7 @@ static void lm90_alert(struct i2c_client *client, enum i2c_alert_protocol type,
 	}
 }
 
-static int __maybe_unused lm90_suspend(struct device *dev)
+static int lm90_suspend(struct device *dev)
 {
 	struct lm90_data *data = dev_get_drvdata(dev);
 	struct i2c_client *client = data->client;
@@ -2967,7 +2967,7 @@ static int __maybe_unused lm90_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused lm90_resume(struct device *dev)
+static int lm90_resume(struct device *dev)
 {
 	struct lm90_data *data = dev_get_drvdata(dev);
 	struct i2c_client *client = data->client;
@@ -2978,14 +2978,14 @@ static int __maybe_unused lm90_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(lm90_pm_ops, lm90_suspend, lm90_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(lm90_pm_ops, lm90_suspend, lm90_resume);
 
 static struct i2c_driver lm90_driver = {
 	.class		= I2C_CLASS_HWMON,
 	.driver = {
 		.name	= "lm90",
 		.of_match_table = of_match_ptr(lm90_of_match),
-		.pm	= &lm90_pm_ops,
+		.pm	= pm_sleep_ptr(&lm90_pm_ops),
 	},
 	.probe_new	= lm90_probe,
 	.alert		= lm90_alert,
diff --git a/drivers/hwmon/lm92.c b/drivers/hwmon/lm92.c
index 5bae6eedcaf1..2ff3044a677d 100644
--- a/drivers/hwmon/lm92.c
+++ b/drivers/hwmon/lm92.c
@@ -287,7 +287,7 @@ static int lm92_detect(struct i2c_client *new_client,
 	else
 		return -ENODEV;
 
-	strlcpy(info->type, "lm92", I2C_NAME_SIZE);
+	strscpy(info->type, "lm92", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/lm93.c b/drivers/hwmon/lm93.c
index dc67bf954b21..4cf50d5f4f59 100644
--- a/drivers/hwmon/lm93.c
+++ b/drivers/hwmon/lm93.c
@@ -2575,7 +2575,7 @@ static int lm93_detect(struct i2c_client *client, struct i2c_board_info *info)
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 	dev_dbg(&adapter->dev, "loading %s at %d, 0x%02x\n",
 		client->name, i2c_adapter_id(client->adapter),
 		client->addr);
diff --git a/drivers/hwmon/lm95234.c b/drivers/hwmon/lm95234.c
index ac169a994ae0..b4a9d0c223c4 100644
--- a/drivers/hwmon/lm95234.c
+++ b/drivers/hwmon/lm95234.c
@@ -644,7 +644,7 @@ static int lm95234_detect(struct i2c_client *client,
 	if (val & model_mask)
 		return -ENODEV;
 
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 	return 0;
 }
 
diff --git a/drivers/hwmon/lm95241.c b/drivers/hwmon/lm95241.c
index 8ea46ff20be5..f1ed777a8735 100644
--- a/drivers/hwmon/lm95241.c
+++ b/drivers/hwmon/lm95241.c
@@ -389,7 +389,7 @@ static int lm95241_detect(struct i2c_client *new_client,
 	}
 
 	/* Fill the i2c board info */
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 	return 0;
 }
 
diff --git a/drivers/hwmon/lm95245.c b/drivers/hwmon/lm95245.c
index 29388fcf5f74..c433f0af2d31 100644
--- a/drivers/hwmon/lm95245.c
+++ b/drivers/hwmon/lm95245.c
@@ -461,7 +461,7 @@ static int lm95245_detect(struct i2c_client *new_client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, name, I2C_NAME_SIZE);
+	strscpy(info->type, name, I2C_NAME_SIZE);
 	return 0;
 }
 
diff --git a/drivers/hwmon/ltc2947-core.c b/drivers/hwmon/ltc2947-core.c
index 5423466de697..7404e974762f 100644
--- a/drivers/hwmon/ltc2947-core.c
+++ b/drivers/hwmon/ltc2947-core.c
@@ -956,13 +956,6 @@ static struct attribute *ltc2947_attrs[] = {
 };
 ATTRIBUTE_GROUPS(ltc2947);
 
-static void ltc2947_clk_disable(void *data)
-{
-	struct clk *extclk = data;
-
-	clk_disable_unprepare(extclk);
-}
-
 static int ltc2947_setup(struct ltc2947_data *st)
 {
 	int ret;
@@ -989,7 +982,7 @@ static int ltc2947_setup(struct ltc2947_data *st)
 		return ret;
 
 	/* check external clock presence */
-	extclk = devm_clk_get_optional(st->dev, NULL);
+	extclk = devm_clk_get_optional_enabled(st->dev, NULL);
 	if (IS_ERR(extclk))
 		return dev_err_probe(st->dev, PTR_ERR(extclk),
 				     "Failed to get external clock\n");
@@ -1007,14 +1000,6 @@ static int ltc2947_setup(struct ltc2947_data *st)
 			return -EINVAL;
 		}
 
-		ret = clk_prepare_enable(extclk);
-		if (ret)
-			return ret;
-
-		ret = devm_add_action_or_reset(st->dev, ltc2947_clk_disable,
-					       extclk);
-		if (ret)
-			return ret;
 		/* as in table 1 of the datasheet */
 		if (rate_hz >= LTC2947_CLK_MIN && rate_hz <= 1000000)
 			pre = 0;
@@ -1135,7 +1120,7 @@ int ltc2947_core_probe(struct regmap *map, const char *name)
 }
 EXPORT_SYMBOL_GPL(ltc2947_core_probe);
 
-static int __maybe_unused ltc2947_resume(struct device *dev)
+static int ltc2947_resume(struct device *dev)
 {
 	struct ltc2947_data *st = dev_get_drvdata(dev);
 	u32 ctrl = 0;
@@ -1164,7 +1149,7 @@ static int __maybe_unused ltc2947_resume(struct device *dev)
 				  LTC2947_CONT_MODE_MASK, LTC2947_CONT_MODE(1));
 }
 
-static int __maybe_unused ltc2947_suspend(struct device *dev)
+static int ltc2947_suspend(struct device *dev)
 {
 	struct ltc2947_data *st = dev_get_drvdata(dev);
 
@@ -1172,8 +1157,7 @@ static int __maybe_unused ltc2947_suspend(struct device *dev)
 				  LTC2947_SHUTDOWN_MASK, 1);
 }
 
-SIMPLE_DEV_PM_OPS(ltc2947_pm_ops, ltc2947_suspend, ltc2947_resume);
-EXPORT_SYMBOL_GPL(ltc2947_pm_ops);
+EXPORT_SIMPLE_DEV_PM_OPS(ltc2947_pm_ops, ltc2947_suspend, ltc2947_resume);
 
 const struct of_device_id ltc2947_of_match[] = {
 	{ .compatible = "adi,ltc2947" },
diff --git a/drivers/hwmon/ltc2947-i2c.c b/drivers/hwmon/ltc2947-i2c.c
index ad0dfd3efbf8..96852bc8a964 100644
--- a/drivers/hwmon/ltc2947-i2c.c
+++ b/drivers/hwmon/ltc2947-i2c.c
@@ -36,7 +36,7 @@ static struct i2c_driver ltc2947_driver = {
 	.driver = {
 		.name = "ltc2947",
 		.of_match_table = ltc2947_of_match,
-		.pm = &ltc2947_pm_ops,
+		.pm = pm_sleep_ptr(&ltc2947_pm_ops),
 	},
 	.probe_new = ltc2947_probe,
 	.id_table = ltc2947_id,
diff --git a/drivers/hwmon/ltc2947-spi.c b/drivers/hwmon/ltc2947-spi.c
index c24ca569db1b..a33be110098c 100644
--- a/drivers/hwmon/ltc2947-spi.c
+++ b/drivers/hwmon/ltc2947-spi.c
@@ -38,7 +38,7 @@ static struct spi_driver ltc2947_driver = {
 	.driver = {
 		.name = "ltc2947",
 		.of_match_table = ltc2947_of_match,
-		.pm = &ltc2947_pm_ops,
+		.pm = pm_sleep_ptr(&ltc2947_pm_ops),
 	},
 	.probe = ltc2947_probe,
 	.id_table = ltc2947_id,
diff --git a/drivers/hwmon/max1619.c b/drivers/hwmon/max1619.c
index eae9e68027bc..445c77197f69 100644
--- a/drivers/hwmon/max1619.c
+++ b/drivers/hwmon/max1619.c
@@ -241,7 +241,7 @@ static int max1619_detect(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, "max1619", I2C_NAME_SIZE);
+	strscpy(info->type, "max1619", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/max1668.c b/drivers/hwmon/max1668.c
index 78688e6cb87d..9f748973d6a3 100644
--- a/drivers/hwmon/max1668.c
+++ b/drivers/hwmon/max1668.c
@@ -386,7 +386,7 @@ static int max1668_detect(struct i2c_client *client,
 	if (!type_name)
 		return -ENODEV;
 
-	strlcpy(info->type, type_name, I2C_NAME_SIZE);
+	strscpy(info->type, type_name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/max31722.c b/drivers/hwmon/max31722.c
index 93e048ee4955..9a31ef388396 100644
--- a/drivers/hwmon/max31722.c
+++ b/drivers/hwmon/max31722.c
@@ -113,7 +113,7 @@ static void max31722_remove(struct spi_device *spi)
 		dev_warn(&spi->dev, "Failed to put device in stand-by mode\n");
 }
 
-static int __maybe_unused max31722_suspend(struct device *dev)
+static int max31722_suspend(struct device *dev)
 {
 	struct spi_device *spi_device = to_spi_device(dev);
 	struct max31722_data *data = spi_get_drvdata(spi_device);
@@ -121,7 +121,7 @@ static int __maybe_unused max31722_suspend(struct device *dev)
 	return max31722_set_mode(data, MAX31722_MODE_STANDBY);
 }
 
-static int __maybe_unused max31722_resume(struct device *dev)
+static int max31722_resume(struct device *dev)
 {
 	struct spi_device *spi_device = to_spi_device(dev);
 	struct max31722_data *data = spi_get_drvdata(spi_device);
@@ -129,7 +129,7 @@ static int __maybe_unused max31722_resume(struct device *dev)
 	return max31722_set_mode(data, MAX31722_MODE_CONTINUOUS);
 }
 
-static SIMPLE_DEV_PM_OPS(max31722_pm_ops, max31722_suspend, max31722_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(max31722_pm_ops, max31722_suspend, max31722_resume);
 
 static const struct spi_device_id max31722_spi_id[] = {
 	{"max31722", 0},
@@ -141,7 +141,7 @@ MODULE_DEVICE_TABLE(spi, max31722_spi_id);
 static struct spi_driver max31722_driver = {
 	.driver = {
 		.name = "max31722",
-		.pm = &max31722_pm_ops,
+		.pm = pm_sleep_ptr(&max31722_pm_ops),
 	},
 	.probe =            max31722_probe,
 	.remove =           max31722_remove,
diff --git a/drivers/hwmon/max31730.c b/drivers/hwmon/max31730.c
index 23598b8b8793..746a767c9fc6 100644
--- a/drivers/hwmon/max31730.c
+++ b/drivers/hwmon/max31730.c
@@ -399,33 +399,33 @@ static int max31730_detect(struct i2c_client *client,
 			return -ENODEV;
 	}
 
-	strlcpy(info->type, "max31730", I2C_NAME_SIZE);
+	strscpy(info->type, "max31730", I2C_NAME_SIZE);
 
 	return 0;
 }
 
-static int __maybe_unused max31730_suspend(struct device *dev)
+static int max31730_suspend(struct device *dev)
 {
 	struct max31730_data *data = dev_get_drvdata(dev);
 
 	return max31730_write_config(data, MAX31730_STOP, 0);
 }
 
-static int __maybe_unused max31730_resume(struct device *dev)
+static int max31730_resume(struct device *dev)
 {
 	struct max31730_data *data = dev_get_drvdata(dev);
 
 	return max31730_write_config(data, 0, MAX31730_STOP);
 }
 
-static SIMPLE_DEV_PM_OPS(max31730_pm_ops, max31730_suspend, max31730_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(max31730_pm_ops, max31730_suspend, max31730_resume);
 
 static struct i2c_driver max31730_driver = {
 	.class		= I2C_CLASS_HWMON,
 	.driver = {
 		.name	= "max31730",
 		.of_match_table = of_match_ptr(max31730_of_match),
-		.pm	= &max31730_pm_ops,
+		.pm	= pm_sleep_ptr(&max31730_pm_ops),
 	},
 	.probe_new	= max31730_probe,
 	.id_table	= max31730_ids,
diff --git a/drivers/hwmon/max31760.c b/drivers/hwmon/max31760.c
new file mode 100644
index 000000000000..06d5f39dc33d
--- /dev/null
+++ b/drivers/hwmon/max31760.c
@@ -0,0 +1,596 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/err.h>
+#include <linux/hwmon.h>
+#include <linux/hwmon-sysfs.h>
+#include <linux/i2c.h>
+#include <linux/regmap.h>
+#include <linux/util_macros.h>
+
+#define REG_CR1		0x00
+#define CR1_HYST	BIT(5)
+#define CR1_DRV		GENMASK(4, 3)
+#define CR1_TEMP_SRC	GENMASK(1, 0)
+#define REG_CR2		0x01
+#define CR2_STBY	BIT(7)
+#define CR2_ALERTS	BIT(6)
+#define CR2_DFC		BIT(0)
+#define REG_CR3		0x02
+#define REG_PWMR	0x50
+#define REG_PWMV	0x51
+#define REG_STATUS	0x5A
+#define STATUS_ALARM_CRIT(ch)	BIT(2 + 2 * (ch))
+#define STATUS_ALARM_MAX(ch)	BIT(3 + 2 * (ch))
+#define STATUS_RDFA		BIT(6)
+
+#define REG_TACH(ch)		(0x52 + (ch) * 2)
+#define REG_TEMP_INPUT(ch)	(0x56 + (ch) * 2)
+#define REG_TEMP_MAX(ch)	(0x06 + (ch) * 2)
+#define REG_TEMP_CRIT(ch)	(0x0A + (ch) * 2)
+
+#define TEMP11_FROM_REG(reg)	((reg) / 32 * 125)
+#define TEMP11_TO_REG(val)	(DIV_ROUND_CLOSEST(clamp_val((val), -128000, \
+							     127875), 125) * 32)
+
+#define LUT_SIZE	48
+
+#define REG_LUT(index)		(0x20 + (index))
+
+struct max31760_state {
+	struct regmap *regmap;
+
+	struct lut_attribute {
+		char name[24];
+		struct sensor_device_attribute sda;
+	} lut[LUT_SIZE];
+
+	struct attribute *attrs[LUT_SIZE + 2];
+	struct attribute_group group;
+	const struct attribute_group *groups[2];
+};
+
+static bool max31760_volatile_reg(struct device *dev, unsigned int reg)
+{
+	return reg > 0x50;
+}
+
+static const struct regmap_config regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = 0x5B,
+	.cache_type = REGCACHE_RBTREE,
+	.volatile_reg = max31760_volatile_reg,
+};
+
+static const int max31760_pwm_freq[] = {33, 150, 1500, 25000};
+
+static int tach_to_rpm(u16 tach)
+{
+	if (tach == 0)
+		tach = 1;
+
+	return 60 * 100000 / tach / 2;
+}
+
+static int max31760_read(struct device *dev, enum hwmon_sensor_types type,
+			 u32 attr, int channel, long *val)
+{
+	struct max31760_state *state = dev_get_drvdata(dev);
+	unsigned int regval;
+	unsigned int reg_temp;
+	s16 temp;
+	u8 reg[2];
+	int ret;
+
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_fault:
+			ret = regmap_read(state->regmap, REG_STATUS, &regval);
+			if (ret)
+				return ret;
+
+			*val = FIELD_GET(STATUS_RDFA, regval);
+
+			return 0;
+		case hwmon_temp_max_alarm:
+			ret = regmap_read(state->regmap, REG_STATUS, &regval);
+			if (ret)
+				return ret;
+
+			if (channel)
+				*val = FIELD_GET(STATUS_ALARM_MAX(1), regval);
+			else
+				*val = FIELD_GET(STATUS_ALARM_MAX(0), regval);
+
+			return 0;
+		case hwmon_temp_crit_alarm:
+			ret = regmap_read(state->regmap, REG_STATUS, &regval);
+			if (ret)
+				return ret;
+
+			if (channel)
+				*val = FIELD_GET(STATUS_ALARM_CRIT(1), regval);
+			else
+				*val = FIELD_GET(STATUS_ALARM_CRIT(0), regval);
+
+			return 0;
+		case hwmon_temp_input:
+			reg_temp = REG_TEMP_INPUT(channel);
+			break;
+		case hwmon_temp_max:
+			reg_temp = REG_TEMP_MAX(channel);
+			break;
+		case hwmon_temp_crit:
+			reg_temp = REG_TEMP_CRIT(channel);
+			break;
+		default:
+			return -EOPNOTSUPP;
+		}
+
+		ret = regmap_bulk_read(state->regmap, reg_temp, reg, 2);
+		if (ret)
+			return ret;
+
+		temp = (reg[0] << 8) | reg[1];
+
+		*val = TEMP11_FROM_REG(temp);
+
+		return 0;
+	case hwmon_fan:
+		switch (attr) {
+		case hwmon_fan_input:
+			ret = regmap_bulk_read(state->regmap, REG_TACH(channel), reg, 2);
+			if (ret)
+				return ret;
+
+			*val = tach_to_rpm(reg[0] * 256 + reg[1]);
+
+			return 0;
+		case hwmon_fan_fault:
+			ret = regmap_read(state->regmap, REG_STATUS, &regval);
+			if (ret)
+				return ret;
+
+			if (channel)
+				*val = FIELD_GET(BIT(1), regval);
+			else
+				*val = FIELD_GET(BIT(0), regval);
+
+			return 0;
+		case hwmon_fan_enable:
+			ret = regmap_read(state->regmap, REG_CR3, &regval);
+			if (ret)
+				return ret;
+
+			if (channel)
+				*val = FIELD_GET(BIT(1), regval);
+			else
+				*val = FIELD_GET(BIT(0), regval);
+
+			return 0;
+		default:
+			return -EOPNOTSUPP;
+		}
+	case hwmon_pwm:
+		switch (attr) {
+		case hwmon_pwm_input:
+			ret = regmap_read(state->regmap, REG_PWMV, &regval);
+			if (ret)
+				return ret;
+
+			*val = regval;
+
+			return 0;
+		case hwmon_pwm_freq:
+			ret = regmap_read(state->regmap, REG_CR1, &regval);
+			if (ret)
+				return ret;
+
+			regval = FIELD_GET(CR1_DRV, regval);
+			if (regval >= ARRAY_SIZE(max31760_pwm_freq))
+				return -EINVAL;
+
+			*val = max31760_pwm_freq[regval];
+
+			return 0;
+		case hwmon_pwm_enable:
+			ret = regmap_read(state->regmap, REG_CR2, &regval);
+			if (ret)
+				return ret;
+
+			*val = 2 - FIELD_GET(CR2_DFC, regval);
+
+			return 0;
+		case hwmon_pwm_auto_channels_temp:
+			ret = regmap_read(state->regmap, REG_CR1, &regval);
+			if (ret)
+				return ret;
+
+			switch (FIELD_GET(CR1_TEMP_SRC, regval)) {
+			case 0:
+				*val = 2;
+				break;
+			case 1:
+				*val = 1;
+				break;
+			case 2:
+			case 3:
+				*val = 3;
+				break;
+			default:
+				return -EINVAL;
+			}
+
+			return 0;
+		default:
+			return -EOPNOTSUPP;
+		}
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int max31760_write(struct device *dev, enum hwmon_sensor_types type,
+			  u32 attr, int channel, long val)
+{
+	struct max31760_state *state = dev_get_drvdata(dev);
+	unsigned int pwm_index;
+	unsigned int reg_temp;
+	int temp;
+	u8 reg_val[2];
+
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_max:
+			reg_temp = REG_TEMP_MAX(channel);
+			break;
+		case hwmon_temp_crit:
+			reg_temp = REG_TEMP_CRIT(channel);
+			break;
+		default:
+			return -EOPNOTSUPP;
+		}
+
+		temp = TEMP11_TO_REG(val);
+		reg_val[0] = temp >> 8;
+		reg_val[1] = temp & 0xFF;
+
+		return regmap_bulk_write(state->regmap, reg_temp, reg_val, 2);
+	case hwmon_fan:
+		switch (attr) {
+		case hwmon_fan_enable:
+			if (val == 0)
+				return regmap_clear_bits(state->regmap, REG_CR3, BIT(channel));
+
+			if (val == 1)
+				return regmap_set_bits(state->regmap, REG_CR3, BIT(channel));
+
+			return -EINVAL;
+		default:
+			return -EOPNOTSUPP;
+		}
+	case hwmon_pwm:
+		switch (attr) {
+		case hwmon_pwm_input:
+			if (val < 0 || val > 255)
+				return -EINVAL;
+
+			return regmap_write(state->regmap, REG_PWMR, val);
+		case hwmon_pwm_enable:
+			if (val == 1)
+				return regmap_set_bits(state->regmap, REG_CR2, CR2_DFC);
+
+			if (val == 2)
+				return regmap_clear_bits(state->regmap, REG_CR2, CR2_DFC);
+
+			return -EINVAL;
+		case hwmon_pwm_freq:
+			pwm_index = find_closest(val, max31760_pwm_freq,
+						 ARRAY_SIZE(max31760_pwm_freq));
+
+			return regmap_update_bits(state->regmap,
+						  REG_CR1, CR1_DRV,
+						  FIELD_PREP(CR1_DRV, pwm_index));
+		case hwmon_pwm_auto_channels_temp:
+			switch (val) {
+			case 1:
+				break;
+			case 2:
+				val = 0;
+				break;
+			case 3:
+				val = 2;
+				break;
+			default:
+				return -EINVAL;
+			}
+
+			return regmap_update_bits(state->regmap, REG_CR1, CR1_TEMP_SRC, val);
+		default:
+			return -EOPNOTSUPP;
+		}
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static const struct hwmon_channel_info *max31760_info[] = {
+	HWMON_CHANNEL_INFO(chip,
+			   HWMON_C_REGISTER_TZ),
+	HWMON_CHANNEL_INFO(fan,
+			   HWMON_F_INPUT | HWMON_F_FAULT | HWMON_F_ENABLE,
+			   HWMON_F_INPUT | HWMON_F_FAULT | HWMON_F_ENABLE),
+	HWMON_CHANNEL_INFO(temp,
+			   HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT | HWMON_T_FAULT |
+			   HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_LABEL,
+			   HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT |
+			   HWMON_T_MAX_ALARM | HWMON_T_CRIT_ALARM | HWMON_T_LABEL),
+	HWMON_CHANNEL_INFO(pwm,
+			   HWMON_PWM_ENABLE | HWMON_PWM_FREQ | HWMON_PWM_INPUT |
+			   HWMON_PWM_AUTO_CHANNELS_TEMP),
+	NULL
+};
+
+static umode_t max31760_is_visible(const void *data,
+				   enum hwmon_sensor_types type,
+				   u32 attr, int channel)
+{
+	switch (type) {
+	case hwmon_temp:
+		switch (attr) {
+		case hwmon_temp_input:
+		case hwmon_temp_max_alarm:
+		case hwmon_temp_crit_alarm:
+		case hwmon_temp_fault:
+		case hwmon_temp_label:
+			return 0444;
+		case hwmon_temp_max:
+		case hwmon_temp_crit:
+			return 0644;
+		default:
+			return 0;
+		}
+	case hwmon_fan:
+		switch (attr) {
+		case hwmon_fan_input:
+		case hwmon_fan_fault:
+			return 0444;
+		case hwmon_fan_enable:
+			return 0644;
+		default:
+			return 0;
+		}
+	case hwmon_pwm:
+		switch (attr) {
+		case hwmon_pwm_enable:
+		case hwmon_pwm_input:
+		case hwmon_pwm_freq:
+		case hwmon_pwm_auto_channels_temp:
+			return 0644;
+		default:
+			return 0;
+		}
+	default:
+		return 0;
+	}
+}
+
+static int max31760_read_string(struct device *dev,
+				enum hwmon_sensor_types type,
+				u32 attr, int channel, const char **str)
+{
+	switch (type) {
+	case hwmon_temp:
+		if (attr != hwmon_temp_label)
+			return -EOPNOTSUPP;
+
+		*str = channel ? "local" : "remote";
+
+		return 0;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static const struct hwmon_ops max31760_hwmon_ops = {
+	.is_visible = max31760_is_visible,
+	.read = max31760_read,
+	.write = max31760_write,
+	.read_string = max31760_read_string
+};
+
+static const struct hwmon_chip_info max31760_chip_info = {
+	.ops = &max31760_hwmon_ops,
+	.info = max31760_info,
+};
+
+static ssize_t lut_show(struct device *dev,
+			struct device_attribute *devattr, char *buf)
+{
+	struct sensor_device_attribute *sda = to_sensor_dev_attr(devattr);
+	struct max31760_state *state = dev_get_drvdata(dev);
+	int ret;
+	unsigned int regval;
+
+	ret = regmap_read(state->regmap, REG_LUT(sda->index), &regval);
+	if (ret)
+		return ret;
+
+	return sysfs_emit(buf, "%d\n", regval);
+}
+
+static ssize_t lut_store(struct device *dev,
+			 struct device_attribute *devattr,
+			 const char *buf, size_t count)
+{
+	struct sensor_device_attribute *sda = to_sensor_dev_attr(devattr);
+	struct max31760_state *state = dev_get_drvdata(dev);
+	int ret;
+	u8 pwm;
+
+	ret = kstrtou8(buf, 10, &pwm);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(state->regmap, REG_LUT(sda->index), pwm);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static ssize_t pwm1_auto_point_temp_hyst_show(struct device *dev,
+					      struct device_attribute *attr,
+					      char *buf)
+{
+	struct max31760_state *state = dev_get_drvdata(dev);
+	unsigned int regval;
+	int ret;
+
+	ret = regmap_read(state->regmap, REG_CR1, &regval);
+	if (ret)
+		return ret;
+
+	return sysfs_emit(buf, "%d\n", (1 + (int)FIELD_GET(CR1_HYST, regval)) * 2000);
+}
+
+static ssize_t pwm1_auto_point_temp_hyst_store(struct device *dev,
+					       struct device_attribute *attr,
+					       const char *buf,
+					       size_t count)
+{
+	struct max31760_state *state = dev_get_drvdata(dev);
+	unsigned int hyst;
+	int ret;
+
+	ret = kstrtou32(buf, 10, &hyst);
+	if (ret)
+		return ret;
+
+	if (hyst < 3000)
+		ret = regmap_clear_bits(state->regmap, REG_CR1, CR1_HYST);
+	else
+		ret = regmap_set_bits(state->regmap, REG_CR1, CR1_HYST);
+
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(pwm1_auto_point_temp_hyst);
+
+static void max31760_create_lut_nodes(struct max31760_state *state)
+{
+	int i;
+	struct sensor_device_attribute *sda;
+	struct lut_attribute *lut;
+
+	for (i = 0; i < LUT_SIZE; ++i) {
+		lut = &state->lut[i];
+		sda = &lut->sda;
+
+		snprintf(lut->name, sizeof(lut->name),
+			 "pwm1_auto_point%d_pwm", i + 1);
+
+		sda->dev_attr.attr.mode = 0644;
+		sda->index = i;
+		sda->dev_attr.show = lut_show;
+		sda->dev_attr.store = lut_store;
+		sda->dev_attr.attr.name = lut->name;
+
+		sysfs_attr_init(&sda->dev_attr.attr);
+
+		state->attrs[i] = &sda->dev_attr.attr;
+	}
+
+	state->attrs[i] = &dev_attr_pwm1_auto_point_temp_hyst.attr;
+
+	state->group.attrs = state->attrs;
+	state->groups[0] = &state->group;
+}
+
+static int max31760_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct max31760_state *state;
+	struct device *hwmon_dev;
+	int ret;
+
+	state = devm_kzalloc(dev, sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return -ENOMEM;
+
+	state->regmap = devm_regmap_init_i2c(client, &regmap_config);
+	if (IS_ERR(state->regmap))
+		return dev_err_probe(dev,
+				     PTR_ERR(state->regmap),
+				     "regmap initialization failed\n");
+
+	dev_set_drvdata(dev, state);
+
+	/* Set alert output to comparator mode */
+	ret = regmap_set_bits(state->regmap, REG_CR2, CR2_ALERTS);
+	if (ret)
+		return dev_err_probe(dev, ret, "cannot write register\n");
+
+	max31760_create_lut_nodes(state);
+
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, client->name,
+							 state,
+							 &max31760_chip_info,
+							 state->groups);
+
+	return PTR_ERR_OR_ZERO(hwmon_dev);
+}
+
+static const struct of_device_id max31760_of_match[] = {
+	{.compatible = "adi,max31760"},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, max31760_of_match);
+
+static const struct i2c_device_id max31760_id[] = {
+	{"max31760"},
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, max31760_id);
+
+static int max31760_suspend(struct device *dev)
+{
+	struct max31760_state *state = dev_get_drvdata(dev);
+
+	return regmap_set_bits(state->regmap, REG_CR2, CR2_STBY);
+}
+
+static int max31760_resume(struct device *dev)
+{
+	struct max31760_state *state = dev_get_drvdata(dev);
+
+	return regmap_clear_bits(state->regmap, REG_CR2, CR2_STBY);
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(max31760_pm_ops, max31760_suspend,
+				max31760_resume);
+
+static struct i2c_driver max31760_driver = {
+	.class		= I2C_CLASS_HWMON,
+	.driver = {
+		.name	= "max31760",
+		.of_match_table = max31760_of_match,
+		.pm = pm_ptr(&max31760_pm_ops)
+	},
+	.probe_new	= max31760_probe,
+	.id_table	= max31760_id
+};
+module_i2c_driver(max31760_driver);
+
+MODULE_AUTHOR("Ibrahim Tilki <Ibrahim.Tilki@analog.com>");
+MODULE_DESCRIPTION("Analog Devices MAX31760 Fan Speed Controller");
+MODULE_SOFTDEP("pre: regmap_i2c");
+MODULE_VERSION("1.0");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/max31790.c b/drivers/hwmon/max31790.c
index 7e9362f6dc29..20bf5ffadefe 100644
--- a/drivers/hwmon/max31790.c
+++ b/drivers/hwmon/max31790.c
@@ -202,6 +202,9 @@ static int max31790_read_fan(struct device *dev, u32 attr, int channel,
 		}
 		mutex_unlock(&data->update_lock);
 		return 0;
+	case hwmon_fan_enable:
+		*val = !!(data->fan_config[channel] & MAX31790_FAN_CFG_TACH_INPUT_EN);
+		return 0;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -214,7 +217,7 @@ static int max31790_write_fan(struct device *dev, u32 attr, int channel,
 	struct i2c_client *client = data->client;
 	int target_count;
 	int err = 0;
-	u8 bits;
+	u8 bits, fan_config;
 	int sr;
 
 	mutex_lock(&data->update_lock);
@@ -243,6 +246,23 @@ static int max31790_write_fan(struct device *dev, u32 attr, int channel,
 					MAX31790_REG_TARGET_COUNT(channel),
 					data->target_count[channel]);
 		break;
+	case hwmon_fan_enable:
+		fan_config = data->fan_config[channel];
+		if (val == 0) {
+			fan_config &= ~MAX31790_FAN_CFG_TACH_INPUT_EN;
+		} else if (val == 1) {
+			fan_config |= MAX31790_FAN_CFG_TACH_INPUT_EN;
+		} else {
+			err = -EINVAL;
+			break;
+		}
+		if (fan_config != data->fan_config[channel]) {
+			err = i2c_smbus_write_byte_data(client, MAX31790_REG_FAN_CONFIG(channel),
+							fan_config);
+			if (!err)
+				data->fan_config[channel] = fan_config;
+		}
+		break;
 	default:
 		err = -EOPNOTSUPP;
 		break;
@@ -270,6 +290,10 @@ static umode_t max31790_fan_is_visible(const void *_data, u32 attr, int channel)
 		    !(fan_config & MAX31790_FAN_CFG_TACH_INPUT))
 			return 0644;
 		return 0;
+	case hwmon_fan_enable:
+		if (channel < NR_CHANNEL)
+			return 0644;
+		return 0;
 	default:
 		return 0;
 	}
@@ -423,12 +447,12 @@ static umode_t max31790_is_visible(const void *data,
 
 static const struct hwmon_channel_info *max31790_info[] = {
 	HWMON_CHANNEL_INFO(fan,
-			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
-			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
-			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
-			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
-			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
-			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT,
+			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE,
+			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE,
+			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE,
+			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE,
+			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE,
+			   HWMON_F_INPUT | HWMON_F_TARGET | HWMON_F_FAULT | HWMON_F_ENABLE,
 			   HWMON_F_INPUT | HWMON_F_FAULT,
 			   HWMON_F_INPUT | HWMON_F_FAULT,
 			   HWMON_F_INPUT | HWMON_F_FAULT,
diff --git a/drivers/hwmon/max6639.c b/drivers/hwmon/max6639.c
index 14bb7726f8d7..9b895402c80d 100644
--- a/drivers/hwmon/max6639.c
+++ b/drivers/hwmon/max6639.c
@@ -514,7 +514,7 @@ static int max6639_detect(struct i2c_client *client,
 	if (dev_id != 0x58 || manu_id != 0x4D)
 		return -ENODEV;
 
-	strlcpy(info->type, "max6639", I2C_NAME_SIZE);
+	strscpy(info->type, "max6639", I2C_NAME_SIZE);
 
 	return 0;
 }
@@ -571,7 +571,6 @@ static int max6639_probe(struct i2c_client *client)
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
-#ifdef CONFIG_PM_SLEEP
 static int max6639_suspend(struct device *dev)
 {
 	struct i2c_client *client = to_i2c_client(dev);
@@ -609,7 +608,6 @@ static int max6639_resume(struct device *dev)
 	return i2c_smbus_write_byte_data(client,
 			MAX6639_REG_GCONFIG, ret & ~MAX6639_GCONFIG_STANDBY);
 }
-#endif /* CONFIG_PM_SLEEP */
 
 static const struct i2c_device_id max6639_id[] = {
 	{"max6639", 0},
@@ -618,13 +616,13 @@ static const struct i2c_device_id max6639_id[] = {
 
 MODULE_DEVICE_TABLE(i2c, max6639_id);
 
-static SIMPLE_DEV_PM_OPS(max6639_pm_ops, max6639_suspend, max6639_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(max6639_pm_ops, max6639_suspend, max6639_resume);
 
 static struct i2c_driver max6639_driver = {
 	.class = I2C_CLASS_HWMON,
 	.driver = {
 		   .name = "max6639",
-		   .pm = &max6639_pm_ops,
+		   .pm = pm_sleep_ptr(&max6639_pm_ops),
 		   },
 	.probe_new = max6639_probe,
 	.id_table = max6639_id,
diff --git a/drivers/hwmon/max6642.c b/drivers/hwmon/max6642.c
index 699d265aae2e..47ea34ff78f3 100644
--- a/drivers/hwmon/max6642.c
+++ b/drivers/hwmon/max6642.c
@@ -148,7 +148,7 @@ static int max6642_detect(struct i2c_client *client,
 	if ((reg_status & 0x2b) != 0x00)
 		return -ENODEV;
 
-	strlcpy(info->type, "max6642", I2C_NAME_SIZE);
+	strscpy(info->type, "max6642", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/mr75203.c b/drivers/hwmon/mr75203.c
index 9259779cc2df..394a4c7e46ab 100644
--- a/drivers/hwmon/mr75203.c
+++ b/drivers/hwmon/mr75203.c
@@ -9,6 +9,7 @@
  */
 #include <linux/bits.h>
 #include <linux/clk.h>
+#include <linux/debugfs.h>
 #include <linux/hwmon.h>
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
@@ -17,6 +18,7 @@
 #include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/reset.h>
+#include <linux/slab.h>
 #include <linux/units.h>
 
 /* PVT Common register */
@@ -30,6 +32,8 @@
 #define CH_NUM_MSK	GENMASK(31, 24)
 #define CH_NUM_SFT	24
 
+#define VM_NUM_MAX	(VM_NUM_MSK >> VM_NUM_SFT)
+
 /* Macro Common Register */
 #define CLK_SYNTH		0x00
 #define CLK_SYNTH_LO_SFT	0
@@ -99,13 +103,67 @@
 
 #define PVT_POLL_DELAY_US	20
 #define PVT_POLL_TIMEOUT_US	20000
-#define PVT_H_CONST		100000
-#define PVT_CAL5_CONST		2047
-#define PVT_G_CONST		40000
 #define PVT_CONV_BITS		10
 #define PVT_N_CONST		90
 #define PVT_R_CONST		245805
 
+#define PVT_TEMP_MIN_mC		-40000
+#define PVT_TEMP_MAX_mC		125000
+
+/* Temperature coefficients for series 5 */
+#define PVT_SERIES5_H_CONST	200000
+#define PVT_SERIES5_G_CONST	60000
+#define PVT_SERIES5_J_CONST	-100
+#define PVT_SERIES5_CAL5_CONST	4094
+
+/* Temperature coefficients for series 6 */
+#define PVT_SERIES6_H_CONST	249400
+#define PVT_SERIES6_G_CONST	57400
+#define PVT_SERIES6_J_CONST	0
+#define PVT_SERIES6_CAL5_CONST	4096
+
+#define TEMPERATURE_SENSOR_SERIES_5	5
+#define TEMPERATURE_SENSOR_SERIES_6	6
+
+#define PRE_SCALER_X1	1
+#define PRE_SCALER_X2	2
+
+/**
+ * struct voltage_device - VM single input parameters.
+ * @vm_map: Map channel number to VM index.
+ * @ch_map: Map channel number to channel index.
+ * @pre_scaler: Pre scaler value (1 or 2) used to normalize the voltage output
+ *              result.
+ *
+ * The structure provides mapping between channel-number (0..N-1) to VM-index
+ * (0..num_vm-1) and channel-index (0..ch_num-1) where N = num_vm * ch_num.
+ * It also provides normalization factor for the VM equation.
+ */
+struct voltage_device {
+	u32 vm_map;
+	u32 ch_map;
+	u32 pre_scaler;
+};
+
+/**
+ * struct voltage_channels - VM channel count.
+ * @total: Total number of channels in all VMs.
+ * @max: Maximum number of channels among all VMs.
+ *
+ * The structure provides channel count information across all VMs.
+ */
+struct voltage_channels {
+	u32 total;
+	u8 max;
+};
+
+struct temp_coeff {
+	u32 h;
+	u32 g;
+	u32 cal5;
+	s32 j;
+};
+
 struct pvt_device {
 	struct regmap		*c_map;
 	struct regmap		*t_map;
@@ -113,14 +171,74 @@ struct pvt_device {
 	struct regmap		*v_map;
 	struct clk		*clk;
 	struct reset_control	*rst;
+	struct dentry		*dbgfs_dir;
+	struct voltage_device	*vd;
+	struct voltage_channels	vm_channels;
+	struct temp_coeff	ts_coeff;
 	u32			t_num;
 	u32			p_num;
 	u32			v_num;
-	u32			c_num;
 	u32			ip_freq;
-	u8			*vm_idx;
 };
 
+static ssize_t pvt_ts_coeff_j_read(struct file *file, char __user *user_buf,
+				   size_t count, loff_t *ppos)
+{
+	struct pvt_device *pvt = file->private_data;
+	unsigned int len;
+	char buf[13];
+
+	len = scnprintf(buf, sizeof(buf), "%d\n", pvt->ts_coeff.j);
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
+}
+
+static ssize_t pvt_ts_coeff_j_write(struct file *file,
+				    const char __user *user_buf,
+				    size_t count, loff_t *ppos)
+{
+	struct pvt_device *pvt = file->private_data;
+	int ret;
+
+	ret = kstrtos32_from_user(user_buf, count, 0, &pvt->ts_coeff.j);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static const struct file_operations pvt_ts_coeff_j_fops = {
+	.read = pvt_ts_coeff_j_read,
+	.write = pvt_ts_coeff_j_write,
+	.open = simple_open,
+	.owner = THIS_MODULE,
+	.llseek = default_llseek,
+};
+
+static void devm_pvt_ts_dbgfs_remove(void *data)
+{
+	struct pvt_device *pvt = (struct pvt_device *)data;
+
+	debugfs_remove_recursive(pvt->dbgfs_dir);
+	pvt->dbgfs_dir = NULL;
+}
+
+static int pvt_ts_dbgfs_create(struct pvt_device *pvt, struct device *dev)
+{
+	pvt->dbgfs_dir = debugfs_create_dir(dev_name(dev), NULL);
+
+	debugfs_create_u32("ts_coeff_h", 0644, pvt->dbgfs_dir,
+			   &pvt->ts_coeff.h);
+	debugfs_create_u32("ts_coeff_g", 0644, pvt->dbgfs_dir,
+			   &pvt->ts_coeff.g);
+	debugfs_create_u32("ts_coeff_cal5", 0644, pvt->dbgfs_dir,
+			   &pvt->ts_coeff.cal5);
+	debugfs_create_file("ts_coeff_j", 0644, pvt->dbgfs_dir, pvt,
+			    &pvt_ts_coeff_j_fops);
+
+	return devm_add_action_or_reset(dev, devm_pvt_ts_dbgfs_remove, pvt);
+}
+
 static umode_t pvt_is_visible(const void *data, enum hwmon_sensor_types type,
 			      u32 attr, int channel)
 {
@@ -139,13 +257,28 @@ static umode_t pvt_is_visible(const void *data, enum hwmon_sensor_types type,
 	return 0;
 }
 
+static long pvt_calc_temp(struct pvt_device *pvt, u32 nbs)
+{
+	/*
+	 * Convert the register value to degrees centigrade temperature:
+	 * T = G + H * (n / cal5 - 0.5) + J * F
+	 */
+	struct temp_coeff *ts_coeff = &pvt->ts_coeff;
+
+	s64 tmp = ts_coeff->g +
+		div_s64(ts_coeff->h * (s64)nbs, ts_coeff->cal5) -
+		ts_coeff->h / 2 +
+		div_s64(ts_coeff->j * (s64)pvt->ip_freq, HZ_PER_MHZ);
+
+	return clamp_val(tmp, PVT_TEMP_MIN_mC, PVT_TEMP_MAX_mC);
+}
+
 static int pvt_read_temp(struct device *dev, u32 attr, int channel, long *val)
 {
 	struct pvt_device *pvt = dev_get_drvdata(dev);
 	struct regmap *t_map = pvt->t_map;
 	u32 stat, nbs;
 	int ret;
-	u64 tmp;
 
 	switch (attr) {
 	case hwmon_temp_input:
@@ -157,7 +290,7 @@ static int pvt_read_temp(struct device *dev, u32 attr, int channel, long *val)
 			return ret;
 
 		ret = regmap_read(t_map, SDIF_DATA(channel), &nbs);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		nbs &= SAMPLE_DATA_MSK;
@@ -166,9 +299,7 @@ static int pvt_read_temp(struct device *dev, u32 attr, int channel, long *val)
 		 * Convert the register value to
 		 * degrees centigrade temperature
 		 */
-		tmp = nbs * PVT_H_CONST;
-		do_div(tmp, PVT_CAL5_CONST);
-		*val = tmp - PVT_G_CONST - pvt->ip_freq;
+		*val = pvt_calc_temp(pvt, nbs);
 
 		return 0;
 	default:
@@ -180,15 +311,15 @@ static int pvt_read_in(struct device *dev, u32 attr, int channel, long *val)
 {
 	struct pvt_device *pvt = dev_get_drvdata(dev);
 	struct regmap *v_map = pvt->v_map;
+	u32 n, stat, pre_scaler;
 	u8 vm_idx, ch_idx;
-	u32 n, stat;
 	int ret;
 
-	if (channel >= pvt->v_num * pvt->c_num)
+	if (channel >= pvt->vm_channels.total)
 		return -EINVAL;
 
-	vm_idx = pvt->vm_idx[channel / pvt->c_num];
-	ch_idx = channel % pvt->c_num;
+	vm_idx = pvt->vd[channel].vm_map;
+	ch_idx = pvt->vd[channel].ch_map;
 
 	switch (attr) {
 	case hwmon_in_input:
@@ -200,10 +331,11 @@ static int pvt_read_in(struct device *dev, u32 attr, int channel, long *val)
 			return ret;
 
 		ret = regmap_read(v_map, VM_SDIF_DATA(vm_idx, ch_idx), &n);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		n &= SAMPLE_DATA_MSK;
+		pre_scaler = pvt->vd[channel].pre_scaler;
 		/*
 		 * Convert the N bitstream count into voltage.
 		 * To support negative voltage calculation for 64bit machines
@@ -215,7 +347,8 @@ static int pvt_read_in(struct device *dev, u32 attr, int channel, long *val)
 		 * BIT(x) may not be used instead of (1 << x) because it's
 		 * unsigned.
 		 */
-		*val = (PVT_N_CONST * (long)n - PVT_R_CONST) / (1 << PVT_CONV_BITS);
+		*val = pre_scaler * (PVT_N_CONST * (long)n - PVT_R_CONST) /
+			(1 << PVT_CONV_BITS);
 
 		return 0;
 	default:
@@ -290,23 +423,23 @@ static int pvt_init(struct pvt_device *pvt)
 		    (key >> 1) << CLK_SYNTH_HI_SFT |
 		    (key >> 1) << CLK_SYNTH_HOLD_SFT | CLK_SYNTH_EN;
 
-	pvt->ip_freq = sys_freq * 100 / (key + 2);
+	pvt->ip_freq = clk_get_rate(pvt->clk) / (key + 2);
 
 	if (t_num) {
 		ret = regmap_write(t_map, SDIF_SMPL_CTRL, 0x0);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_write(t_map, SDIF_HALT, 0x0);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_write(t_map, CLK_SYNTH, clk_synth);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_write(t_map, SDIF_DISABLE, 0x0);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_read_poll_timeout(t_map, SDIF_STAT,
@@ -319,7 +452,7 @@ static int pvt_init(struct pvt_device *pvt)
 		val = CFG0_MODE_2 | CFG0_PARALLEL_OUT | CFG0_12_BIT |
 		      IP_CFG << SDIF_ADDR_SFT | SDIF_WRN_W | SDIF_PROG;
 		ret = regmap_write(t_map, SDIF_W, val);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_read_poll_timeout(t_map, SDIF_STAT,
@@ -332,7 +465,7 @@ static int pvt_init(struct pvt_device *pvt)
 		val = POWER_DELAY_CYCLE_256 | IP_TMR << SDIF_ADDR_SFT |
 			      SDIF_WRN_W | SDIF_PROG;
 		ret = regmap_write(t_map, SDIF_W, val);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_read_poll_timeout(t_map, SDIF_STAT,
@@ -346,39 +479,39 @@ static int pvt_init(struct pvt_device *pvt)
 		      IP_CTRL << SDIF_ADDR_SFT |
 		      SDIF_WRN_W | SDIF_PROG;
 		ret = regmap_write(t_map, SDIF_W, val);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 	}
 
 	if (p_num) {
 		ret = regmap_write(p_map, SDIF_HALT, 0x0);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_write(p_map, SDIF_DISABLE, BIT(p_num) - 1);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_write(p_map, CLK_SYNTH, clk_synth);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 	}
 
 	if (v_num) {
 		ret = regmap_write(v_map, SDIF_SMPL_CTRL, 0x0);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_write(v_map, SDIF_HALT, 0x0);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_write(v_map, CLK_SYNTH, clk_synth);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_write(v_map, SDIF_DISABLE, 0x0);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_read_poll_timeout(v_map, SDIF_STAT,
@@ -388,7 +521,7 @@ static int pvt_init(struct pvt_device *pvt)
 		if (ret)
 			return ret;
 
-		val = (BIT(pvt->c_num) - 1) | VM_CH_INIT |
+		val = (BIT(pvt->vm_channels.max) - 1) | VM_CH_INIT |
 		      IP_POLL << SDIF_ADDR_SFT | SDIF_WRN_W | SDIF_PROG;
 		ret = regmap_write(v_map, SDIF_W, val);
 		if (ret < 0)
@@ -405,7 +538,7 @@ static int pvt_init(struct pvt_device *pvt)
 		      CFG1_14_BIT | IP_CFG << SDIF_ADDR_SFT |
 		      SDIF_WRN_W | SDIF_PROG;
 		ret = regmap_write(v_map, SDIF_W, val);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_read_poll_timeout(v_map, SDIF_STAT,
@@ -418,7 +551,7 @@ static int pvt_init(struct pvt_device *pvt)
 		val = POWER_DELAY_CYCLE_64 | IP_TMR << SDIF_ADDR_SFT |
 		      SDIF_WRN_W | SDIF_PROG;
 		ret = regmap_write(v_map, SDIF_W, val);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 
 		ret = regmap_read_poll_timeout(v_map, SDIF_STAT,
@@ -432,7 +565,7 @@ static int pvt_init(struct pvt_device *pvt)
 		      IP_CTRL << SDIF_ADDR_SFT |
 		      SDIF_WRN_W | SDIF_PROG;
 		ret = regmap_write(v_map, SDIF_W, val);
-		if(ret < 0)
+		if (ret < 0)
 			return ret;
 	}
 
@@ -477,40 +610,157 @@ static int pvt_get_regmap(struct platform_device *pdev, char *reg_name,
 	return 0;
 }
 
-static void pvt_clk_disable(void *data)
+static void pvt_reset_control_assert(void *data)
 {
 	struct pvt_device *pvt = data;
 
-	clk_disable_unprepare(pvt->clk);
+	reset_control_assert(pvt->rst);
 }
 
-static int pvt_clk_enable(struct device *dev, struct pvt_device *pvt)
+static int pvt_reset_control_deassert(struct device *dev, struct pvt_device *pvt)
 {
 	int ret;
 
-	ret = clk_prepare_enable(pvt->clk);
+	ret = reset_control_deassert(pvt->rst);
 	if (ret)
 		return ret;
 
-	return devm_add_action_or_reset(dev, pvt_clk_disable, pvt);
+	return devm_add_action_or_reset(dev, pvt_reset_control_assert, pvt);
 }
 
-static void pvt_reset_control_assert(void *data)
+static int pvt_get_active_channel(struct device *dev, struct pvt_device *pvt,
+				  u32 vm_num, u32 ch_num, u8 *vm_idx)
 {
-	struct pvt_device *pvt = data;
+	u8 vm_active_ch[VM_NUM_MAX];
+	int ret, i, j, k;
 
-	reset_control_assert(pvt->rst);
+	ret = device_property_read_u8_array(dev, "moortec,vm-active-channels",
+					    vm_active_ch, vm_num);
+	if (ret) {
+		/*
+		 * Incase "moortec,vm-active-channels" property is not defined,
+		 * we assume each VM sensor has all of its channels active.
+		 */
+		memset(vm_active_ch, ch_num, vm_num);
+		pvt->vm_channels.max = ch_num;
+		pvt->vm_channels.total = ch_num * vm_num;
+	} else {
+		for (i = 0; i < vm_num; i++) {
+			if (vm_active_ch[i] > ch_num) {
+				dev_err(dev, "invalid active channels: %u\n",
+					vm_active_ch[i]);
+				return -EINVAL;
+			}
+
+			pvt->vm_channels.total += vm_active_ch[i];
+
+			if (vm_active_ch[i] > pvt->vm_channels.max)
+				pvt->vm_channels.max = vm_active_ch[i];
+		}
+	}
+
+	/*
+	 * Map between the channel-number to VM-index and channel-index.
+	 * Example - 3 VMs, "moortec,vm_active_ch" = <5 2 4>:
+	 * vm_map = [0 0 0 0 0 1 1 2 2 2 2]
+	 * ch_map = [0 1 2 3 4 0 1 0 1 2 3]
+	 */
+	pvt->vd = devm_kcalloc(dev, pvt->vm_channels.total, sizeof(*pvt->vd),
+			       GFP_KERNEL);
+	if (!pvt->vd)
+		return -ENOMEM;
+
+	k = 0;
+	for (i = 0; i < vm_num; i++) {
+		for (j = 0; j < vm_active_ch[i]; j++) {
+			pvt->vd[k].vm_map = vm_idx[i];
+			pvt->vd[k].ch_map = j;
+			k++;
+		}
+	}
+
+	return 0;
 }
 
-static int pvt_reset_control_deassert(struct device *dev, struct pvt_device *pvt)
+static int pvt_get_pre_scaler(struct device *dev, struct pvt_device *pvt)
+{
+	u8 *pre_scaler_ch_list;
+	int i, ret, num_ch;
+	u32 channel;
+
+	/* Set default pre-scaler value to be 1. */
+	for (i = 0; i < pvt->vm_channels.total; i++)
+		pvt->vd[i].pre_scaler = PRE_SCALER_X1;
+
+	/* Get number of channels configured in "moortec,vm-pre-scaler-x2". */
+	num_ch = device_property_count_u8(dev, "moortec,vm-pre-scaler-x2");
+	if (num_ch <= 0)
+		return 0;
+
+	pre_scaler_ch_list = kcalloc(num_ch, sizeof(*pre_scaler_ch_list),
+				     GFP_KERNEL);
+	if (!pre_scaler_ch_list)
+		return -ENOMEM;
+
+	/* Get list of all channels that have pre-scaler of 2. */
+	ret = device_property_read_u8_array(dev, "moortec,vm-pre-scaler-x2",
+					    pre_scaler_ch_list, num_ch);
+	if (ret)
+		goto out;
+
+	for (i = 0; i < num_ch; i++) {
+		channel = pre_scaler_ch_list[i];
+		pvt->vd[channel].pre_scaler = PRE_SCALER_X2;
+	}
+
+out:
+	kfree(pre_scaler_ch_list);
+
+	return ret;
+}
+
+static int pvt_set_temp_coeff(struct device *dev, struct pvt_device *pvt)
 {
+	struct temp_coeff *ts_coeff = &pvt->ts_coeff;
+	u32 series;
 	int ret;
 
-	ret = reset_control_deassert(pvt->rst);
+	/* Incase ts-series property is not defined, use default 5. */
+	ret = device_property_read_u32(dev, "moortec,ts-series", &series);
 	if (ret)
-		return ret;
+		series = TEMPERATURE_SENSOR_SERIES_5;
+
+	switch (series) {
+	case TEMPERATURE_SENSOR_SERIES_5:
+		ts_coeff->h = PVT_SERIES5_H_CONST;
+		ts_coeff->g = PVT_SERIES5_G_CONST;
+		ts_coeff->j = PVT_SERIES5_J_CONST;
+		ts_coeff->cal5 = PVT_SERIES5_CAL5_CONST;
+		break;
+	case TEMPERATURE_SENSOR_SERIES_6:
+		ts_coeff->h = PVT_SERIES6_H_CONST;
+		ts_coeff->g = PVT_SERIES6_G_CONST;
+		ts_coeff->j = PVT_SERIES6_J_CONST;
+		ts_coeff->cal5 = PVT_SERIES6_CAL5_CONST;
+		break;
+	default:
+		dev_err(dev, "invalid temperature sensor series (%u)\n",
+			series);
+		return -EINVAL;
+	}
 
-	return devm_add_action_or_reset(dev, pvt_reset_control_assert, pvt);
+	dev_dbg(dev, "temperature sensor series = %u\n", series);
+
+	/* Override ts-coeff-h/g/j/cal5 if they are defined. */
+	device_property_read_u32(dev, "moortec,ts-coeff-h", &ts_coeff->h);
+	device_property_read_u32(dev, "moortec,ts-coeff-g", &ts_coeff->g);
+	device_property_read_u32(dev, "moortec,ts-coeff-j", &ts_coeff->j);
+	device_property_read_u32(dev, "moortec,ts-coeff-cal5", &ts_coeff->cal5);
+
+	dev_dbg(dev, "ts-coeff: h = %u, g = %u, j = %d, cal5 = %u\n",
+		ts_coeff->h, ts_coeff->g, ts_coeff->j, ts_coeff->cal5);
+
+	return 0;
 }
 
 static int mr75203_probe(struct platform_device *pdev)
@@ -531,27 +781,24 @@ static int mr75203_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	pvt->clk = devm_clk_get(dev, NULL);
+	pvt->clk = devm_clk_get_enabled(dev, NULL);
 	if (IS_ERR(pvt->clk))
 		return dev_err_probe(dev, PTR_ERR(pvt->clk), "failed to get clock\n");
 
-	ret = pvt_clk_enable(dev, pvt);
-	if (ret) {
-		dev_err(dev, "failed to enable clock\n");
-		return ret;
-	}
-
-	pvt->rst = devm_reset_control_get_exclusive(dev, NULL);
+	pvt->rst = devm_reset_control_get_optional_exclusive(dev, NULL);
 	if (IS_ERR(pvt->rst))
 		return dev_err_probe(dev, PTR_ERR(pvt->rst),
 				     "failed to get reset control\n");
 
-	ret = pvt_reset_control_deassert(dev, pvt);
-	if (ret)
-		return dev_err_probe(dev, ret, "cannot deassert reset control\n");
+	if (pvt->rst) {
+		ret = pvt_reset_control_deassert(dev, pvt);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "cannot deassert reset control\n");
+	}
 
 	ret = regmap_read(pvt->c_map, PVT_IP_CONFIG, &val);
-	if(ret < 0)
+	if (ret < 0)
 		return ret;
 
 	ts_num = (val & TS_NUM_MSK) >> TS_NUM_SFT;
@@ -561,7 +808,6 @@ static int mr75203_probe(struct platform_device *pdev)
 	pvt->t_num = ts_num;
 	pvt->p_num = pd_num;
 	pvt->v_num = vm_num;
-	pvt->c_num = ch_num;
 	val = 0;
 	if (ts_num)
 		val++;
@@ -581,6 +827,10 @@ static int mr75203_probe(struct platform_device *pdev)
 		if (ret)
 			return ret;
 
+		ret = pvt_set_temp_coeff(dev, pvt);
+		if (ret)
+			return ret;
+
 		temp_config = devm_kcalloc(dev, ts_num + 1,
 					   sizeof(*temp_config), GFP_KERNEL);
 		if (!temp_config)
@@ -589,6 +839,8 @@ static int mr75203_probe(struct platform_device *pdev)
 		memset32(temp_config, HWMON_T_INPUT, ts_num);
 		pvt_temp.config = temp_config;
 		pvt_info[index++] = &pvt_temp;
+
+		pvt_ts_dbgfs_create(pvt, dev);
 	}
 
 	if (pd_num) {
@@ -598,44 +850,45 @@ static int mr75203_probe(struct platform_device *pdev)
 	}
 
 	if (vm_num) {
-		u32 total_ch;
+		u8 vm_idx[VM_NUM_MAX];
 
 		ret = pvt_get_regmap(pdev, "vm", pvt);
 		if (ret)
 			return ret;
 
-		pvt->vm_idx = devm_kcalloc(dev, vm_num, sizeof(*pvt->vm_idx),
-					   GFP_KERNEL);
-		if (!pvt->vm_idx)
-			return -ENOMEM;
-
-		ret = device_property_read_u8_array(dev, "intel,vm-map",
-						    pvt->vm_idx, vm_num);
+		ret = device_property_read_u8_array(dev, "intel,vm-map", vm_idx,
+						    vm_num);
 		if (ret) {
 			/*
 			 * Incase intel,vm-map property is not defined, we
 			 * assume incremental channel numbers.
 			 */
 			for (i = 0; i < vm_num; i++)
-				pvt->vm_idx[i] = i;
+				vm_idx[i] = i;
 		} else {
 			for (i = 0; i < vm_num; i++)
-				if (pvt->vm_idx[i] >= vm_num ||
-				    pvt->vm_idx[i] == 0xff) {
+				if (vm_idx[i] >= vm_num || vm_idx[i] == 0xff) {
 					pvt->v_num = i;
 					vm_num = i;
 					break;
 				}
 		}
 
-		total_ch = ch_num * vm_num;
-		in_config = devm_kcalloc(dev, total_ch + 1,
+		ret = pvt_get_active_channel(dev, pvt, vm_num, ch_num, vm_idx);
+		if (ret)
+			return ret;
+
+		ret = pvt_get_pre_scaler(dev, pvt);
+		if (ret)
+			return ret;
+
+		in_config = devm_kcalloc(dev, pvt->vm_channels.total + 1,
 					 sizeof(*in_config), GFP_KERNEL);
 		if (!in_config)
 			return -ENOMEM;
 
-		memset32(in_config, HWMON_I_INPUT, total_ch);
-		in_config[total_ch] = 0;
+		memset32(in_config, HWMON_I_INPUT, pvt->vm_channels.total);
+		in_config[pvt->vm_channels.total] = 0;
 		pvt_in.config = in_config;
 
 		pvt_info[index++] = &pvt_in;
diff --git a/drivers/hwmon/nct6683.c b/drivers/hwmon/nct6683.c
index 6a9f420e7d32..a872f783e9cc 100644
--- a/drivers/hwmon/nct6683.c
+++ b/drivers/hwmon/nct6683.c
@@ -412,7 +412,7 @@ nct6683_create_attr_group(struct device *dev,
 	struct sensor_device_attr_u *su;
 	struct attribute_group *group;
 	struct attribute **attrs;
-	int i, j, count;
+	int i, count;
 
 	if (repeat <= 0)
 		return ERR_PTR(-EINVAL);
@@ -443,7 +443,7 @@ nct6683_create_attr_group(struct device *dev,
 
 	for (i = 0; i < repeat; i++) {
 		t = tg->templates;
-		for (j = 0; *t != NULL; j++) {
+		while (*t) {
 			snprintf(su->name, sizeof(su->name),
 				 (*t)->dev_attr.attr.name, tg->base + i);
 			if ((*t)->s2) {
diff --git a/drivers/hwmon/nct6775-platform.c b/drivers/hwmon/nct6775-platform.c
index 41c97cfacfb8..b34783784213 100644
--- a/drivers/hwmon/nct6775-platform.c
+++ b/drivers/hwmon/nct6775-platform.c
@@ -355,7 +355,7 @@ static void nct6791_enable_io_mapping(struct nct6775_sio_data *sio_data)
 	}
 }
 
-static int __maybe_unused nct6775_suspend(struct device *dev)
+static int nct6775_suspend(struct device *dev)
 {
 	int err;
 	u16 tmp;
@@ -386,7 +386,7 @@ out:
 	return err;
 }
 
-static int __maybe_unused nct6775_resume(struct device *dev)
+static int nct6775_resume(struct device *dev)
 {
 	struct nct6775_data *data = dev_get_drvdata(dev);
 	struct nct6775_sio_data *sio_data = dev_get_platdata(dev);
@@ -467,7 +467,7 @@ abort:
 	return err;
 }
 
-static SIMPLE_DEV_PM_OPS(nct6775_dev_pm_ops, nct6775_suspend, nct6775_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(nct6775_dev_pm_ops, nct6775_suspend, nct6775_resume);
 
 static void
 nct6775_check_fan_inputs(struct nct6775_data *data, struct nct6775_sio_data *sio_data)
@@ -934,7 +934,7 @@ static int nct6775_platform_probe(struct platform_device *pdev)
 static struct platform_driver nct6775_driver = {
 	.driver = {
 		.name	= DRVNAME,
-		.pm	= &nct6775_dev_pm_ops,
+		.pm	= pm_sleep_ptr(&nct6775_dev_pm_ops),
 	},
 	.probe		= nct6775_platform_probe,
 };
diff --git a/drivers/hwmon/nct7802.c b/drivers/hwmon/nct7802.c
index d1eeef02b6dc..a175f8283695 100644
--- a/drivers/hwmon/nct7802.c
+++ b/drivers/hwmon/nct7802.c
@@ -1038,7 +1038,7 @@ static int nct7802_detect(struct i2c_client *client,
 	if (reg < 0 || (reg & 0x3f))
 		return -ENODEV;
 
-	strlcpy(info->type, "nct7802", I2C_NAME_SIZE);
+	strscpy(info->type, "nct7802", I2C_NAME_SIZE);
 	return 0;
 }
 
diff --git a/drivers/hwmon/nct7904.c b/drivers/hwmon/nct7904.c
index b1c837fc407a..ecc5db0011a3 100644
--- a/drivers/hwmon/nct7904.c
+++ b/drivers/hwmon/nct7904.c
@@ -798,7 +798,7 @@ static int nct7904_detect(struct i2c_client *client,
 	    (i2c_smbus_read_byte_data(client, BANK_SEL_REG) & 0xf8) != 0x00)
 		return -ENODEV;
 
-	strlcpy(info->type, "nct7904", I2C_NAME_SIZE);
+	strscpy(info->type, "nct7904", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/nzxt-smart2.c b/drivers/hwmon/nzxt-smart2.c
index dd892ff5a3e8..533f38b0b4e9 100644
--- a/drivers/hwmon/nzxt-smart2.c
+++ b/drivers/hwmon/nzxt-smart2.c
@@ -787,6 +787,7 @@ static void nzxt_smart2_hid_remove(struct hid_device *hdev)
 static const struct hid_device_id nzxt_smart2_hid_id_table[] = {
 	{ HID_USB_DEVICE(0x1e71, 0x2006) }, /* NZXT Smart Device V2 */
 	{ HID_USB_DEVICE(0x1e71, 0x200d) }, /* NZXT Smart Device V2 */
+	{ HID_USB_DEVICE(0x1e71, 0x200f) }, /* NZXT Smart Device V2 */
 	{ HID_USB_DEVICE(0x1e71, 0x2009) }, /* NZXT RGB & Fan Controller */
 	{ HID_USB_DEVICE(0x1e71, 0x200e) }, /* NZXT RGB & Fan Controller */
 	{ HID_USB_DEVICE(0x1e71, 0x2010) }, /* NZXT RGB & Fan Controller */
diff --git a/drivers/hwmon/occ/p8_i2c.c b/drivers/hwmon/occ/p8_i2c.c
index b221be1f35f3..9e1744fccb35 100644
--- a/drivers/hwmon/occ/p8_i2c.c
+++ b/drivers/hwmon/occ/p8_i2c.c
@@ -227,13 +227,11 @@ static int p8_i2c_occ_probe(struct i2c_client *client)
 	return occ_setup(occ);
 }
 
-static int p8_i2c_occ_remove(struct i2c_client *client)
+static void p8_i2c_occ_remove(struct i2c_client *client)
 {
 	struct occ *occ = dev_get_drvdata(&client->dev);
 
 	occ_shutdown(occ);
-
-	return 0;
 }
 
 static const struct of_device_id p8_i2c_occ_of_match[] = {
diff --git a/drivers/hwmon/pc87360.c b/drivers/hwmon/pc87360.c
index 0828436a1f6c..a4adc8bd531f 100644
--- a/drivers/hwmon/pc87360.c
+++ b/drivers/hwmon/pc87360.c
@@ -35,6 +35,18 @@
 #include <linux/acpi.h>
 #include <linux/io.h>
 
+#define DRIVER_NAME "pc87360"
+
+/* (temp & vin) channel conversion status register flags (pdf sec.11.5.12) */
+#define CHAN_CNVRTD	0x80	/* new data ready */
+#define CHAN_ENA	0x01	/* enabled channel (temp or vin) */
+#define CHAN_ALM_ENA	0x10	/* propagate to alarms-reg ?? (chk val!) */
+#define CHAN_READY	(CHAN_ENA|CHAN_CNVRTD) /* sample ready mask */
+
+#define TEMP_OTS_OE	0x20	/* OTS Output Enable */
+#define VIN_RW1C_MASK	(CHAN_READY|CHAN_ALM_MAX|CHAN_ALM_MIN)   /* 0x87 */
+#define TEMP_RW1C_MASK	(VIN_RW1C_MASK|TEMP_ALM_CRIT|TEMP_FAULT) /* 0xCF */
+
 static u8 devid;
 static struct platform_device *pdev;
 static unsigned short extra_isa[3];
@@ -211,183 +223,181 @@ struct pc87360_data {
 };
 
 /*
- * Functions declaration
+ * ldi is the logical device index
+ * bank is for voltages and temperatures only
  */
-
-static int pc87360_probe(struct platform_device *pdev);
-static int pc87360_remove(struct platform_device *pdev);
-
 static int pc87360_read_value(struct pc87360_data *data, u8 ldi, u8 bank,
-			      u8 reg);
-static void pc87360_write_value(struct pc87360_data *data, u8 ldi, u8 bank,
-				u8 reg, u8 value);
-static void pc87360_init_device(struct platform_device *pdev,
-				int use_thermistors);
-static struct pc87360_data *pc87360_update_device(struct device *dev);
-
-/*
- * Driver data
- */
-
-static struct platform_driver pc87360_driver = {
-	.driver = {
-		.name	= "pc87360",
-	},
-	.probe		= pc87360_probe,
-	.remove		= pc87360_remove,
-};
+			      u8 reg)
+{
+	int res;
 
-/*
- * Sysfs stuff
- */
+	mutex_lock(&(data->lock));
+	if (bank != NO_BANK)
+		outb_p(bank, data->address[ldi] + PC87365_REG_BANK);
+	res = inb_p(data->address[ldi] + reg);
+	mutex_unlock(&(data->lock));
 
-static ssize_t fan_input_show(struct device *dev,
-			      struct device_attribute *devattr, char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n", FAN_FROM_REG(data->fan[attr->index],
-		       FAN_DIV_FROM_REG(data->fan_status[attr->index])));
-}
-static ssize_t fan_min_show(struct device *dev,
-			    struct device_attribute *devattr, char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n", FAN_FROM_REG(data->fan_min[attr->index],
-		       FAN_DIV_FROM_REG(data->fan_status[attr->index])));
-}
-static ssize_t fan_div_show(struct device *dev,
-			    struct device_attribute *devattr, char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n",
-		       FAN_DIV_FROM_REG(data->fan_status[attr->index]));
+	return res;
 }
-static ssize_t fan_status_show(struct device *dev,
-			       struct device_attribute *devattr, char *buf)
+
+static void pc87360_write_value(struct pc87360_data *data, u8 ldi, u8 bank,
+				u8 reg, u8 value)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n",
-		       FAN_STATUS_FROM_REG(data->fan_status[attr->index]));
+	mutex_lock(&(data->lock));
+	if (bank != NO_BANK)
+		outb_p(bank, data->address[ldi] + PC87365_REG_BANK);
+	outb_p(value, data->address[ldi] + reg);
+	mutex_unlock(&(data->lock));
 }
-static ssize_t fan_min_store(struct device *dev,
-			     struct device_attribute *devattr,
-			     const char *buf, size_t count)
+
+static void pc87360_autodiv(struct device *dev, int nr)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct pc87360_data *data = dev_get_drvdata(dev);
-	long fan_min;
-	int err;
-
-	err = kstrtol(buf, 10, &fan_min);
-	if (err)
-		return err;
-
-	mutex_lock(&data->update_lock);
-	fan_min = FAN_TO_REG(fan_min,
-			     FAN_DIV_FROM_REG(data->fan_status[attr->index]));
+	u8 old_min = data->fan_min[nr];
 
-	/* If it wouldn't fit, change clock divisor */
-	while (fan_min > 255
-	    && (data->fan_status[attr->index] & 0x60) != 0x60) {
-		fan_min >>= 1;
-		data->fan[attr->index] >>= 1;
-		data->fan_status[attr->index] += 0x20;
+	/* Increase clock divider if needed and possible */
+	if ((data->fan_status[nr] & 0x04) /* overflow flag */
+	 || (data->fan[nr] >= 224)) { /* next to overflow */
+		if ((data->fan_status[nr] & 0x60) != 0x60) {
+			data->fan_status[nr] += 0x20;
+			data->fan_min[nr] >>= 1;
+			data->fan[nr] >>= 1;
+			dev_dbg(dev,
+				"Increasing clock divider to %d for fan %d\n",
+				FAN_DIV_FROM_REG(data->fan_status[nr]), nr + 1);
+		}
+	} else {
+		/* Decrease clock divider if possible */
+		while (!(data->fan_min[nr] & 0x80) /* min "nails" divider */
+		 && data->fan[nr] < 85 /* bad accuracy */
+		 && (data->fan_status[nr] & 0x60) != 0x00) {
+			data->fan_status[nr] -= 0x20;
+			data->fan_min[nr] <<= 1;
+			data->fan[nr] <<= 1;
+			dev_dbg(dev,
+				"Decreasing clock divider to %d for fan %d\n",
+				FAN_DIV_FROM_REG(data->fan_status[nr]),
+				nr + 1);
+		}
 	}
-	data->fan_min[attr->index] = fan_min > 255 ? 255 : fan_min;
-	pc87360_write_value(data, LD_FAN, NO_BANK,
-			    PC87360_REG_FAN_MIN(attr->index),
-			    data->fan_min[attr->index]);
-
-	/* Write new divider, preserve alarm bits */
-	pc87360_write_value(data, LD_FAN, NO_BANK,
-			    PC87360_REG_FAN_STATUS(attr->index),
-			    data->fan_status[attr->index] & 0xF9);
-	mutex_unlock(&data->update_lock);
-
-	return count;
-}
 
-static struct sensor_device_attribute fan_input[] = {
-	SENSOR_ATTR_RO(fan1_input, fan_input, 0),
-	SENSOR_ATTR_RO(fan2_input, fan_input, 1),
-	SENSOR_ATTR_RO(fan3_input, fan_input, 2),
-};
-static struct sensor_device_attribute fan_status[] = {
-	SENSOR_ATTR_RO(fan1_status, fan_status, 0),
-	SENSOR_ATTR_RO(fan2_status, fan_status, 1),
-	SENSOR_ATTR_RO(fan3_status, fan_status, 2),
-};
-static struct sensor_device_attribute fan_div[] = {
-	SENSOR_ATTR_RO(fan1_div, fan_div, 0),
-	SENSOR_ATTR_RO(fan2_div, fan_div, 1),
-	SENSOR_ATTR_RO(fan3_div, fan_div, 2),
-};
-static struct sensor_device_attribute fan_min[] = {
-	SENSOR_ATTR_RW(fan1_min, fan_min, 0),
-	SENSOR_ATTR_RW(fan2_min, fan_min, 1),
-	SENSOR_ATTR_RW(fan3_min, fan_min, 2),
-};
-
-#define FAN_UNIT_ATTRS(X)		\
-{	&fan_input[X].dev_attr.attr,	\
-	&fan_status[X].dev_attr.attr,	\
-	&fan_div[X].dev_attr.attr,	\
-	&fan_min[X].dev_attr.attr,	\
-	NULL				\
+	/* Write new fan min if it changed */
+	if (old_min != data->fan_min[nr]) {
+		pc87360_write_value(data, LD_FAN, NO_BANK,
+				    PC87360_REG_FAN_MIN(nr),
+				    data->fan_min[nr]);
+	}
 }
 
-static ssize_t pwm_show(struct device *dev, struct device_attribute *devattr,
-			char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n",
-		       PWM_FROM_REG(data->pwm[attr->index],
-				    FAN_CONFIG_INVERT(data->fan_conf,
-						      attr->index)));
-}
-static ssize_t pwm_store(struct device *dev, struct device_attribute *devattr,
-			 const char *buf, size_t count)
+static struct pc87360_data *pc87360_update_device(struct device *dev)
 {
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct pc87360_data *data = dev_get_drvdata(dev);
-	long val;
-	int err;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
+	u8 i;
 
 	mutex_lock(&data->update_lock);
-	data->pwm[attr->index] = PWM_TO_REG(val,
-			      FAN_CONFIG_INVERT(data->fan_conf, attr->index));
-	pc87360_write_value(data, LD_FAN, NO_BANK, PC87360_REG_PWM(attr->index),
-			    data->pwm[attr->index]);
-	mutex_unlock(&data->update_lock);
-	return count;
-}
 
-static struct sensor_device_attribute pwm[] = {
-	SENSOR_ATTR_RW(pwm1, pwm, 0),
-	SENSOR_ATTR_RW(pwm2, pwm, 1),
-	SENSOR_ATTR_RW(pwm3, pwm, 2),
-};
+	if (time_after(jiffies, data->last_updated + HZ * 2) || !data->valid) {
+		dev_dbg(dev, "Data update\n");
 
-static struct attribute *pc8736x_fan_attr[][5] = {
-	FAN_UNIT_ATTRS(0),
-	FAN_UNIT_ATTRS(1),
-	FAN_UNIT_ATTRS(2)
-};
+		/* Fans */
+		for (i = 0; i < data->fannr; i++) {
+			if (FAN_CONFIG_MONITOR(data->fan_conf, i)) {
+				data->fan_status[i] =
+					pc87360_read_value(data, LD_FAN,
+					NO_BANK, PC87360_REG_FAN_STATUS(i));
+				data->fan[i] = pc87360_read_value(data, LD_FAN,
+					       NO_BANK, PC87360_REG_FAN(i));
+				data->fan_min[i] = pc87360_read_value(data,
+						   LD_FAN, NO_BANK,
+						   PC87360_REG_FAN_MIN(i));
+				/* Change clock divider if needed */
+				pc87360_autodiv(dev, i);
+				/* Clear bits and write new divider */
+				pc87360_write_value(data, LD_FAN, NO_BANK,
+						    PC87360_REG_FAN_STATUS(i),
+						    data->fan_status[i]);
+			}
+			if (FAN_CONFIG_CONTROL(data->fan_conf, i))
+				data->pwm[i] = pc87360_read_value(data, LD_FAN,
+					       NO_BANK, PC87360_REG_PWM(i));
+		}
 
-static const struct attribute_group pc8736x_fan_attr_group[] = {
-	{ .attrs = pc8736x_fan_attr[0], },
-	{ .attrs = pc8736x_fan_attr[1], },
-	{ .attrs = pc8736x_fan_attr[2], },
-};
+		/* Voltages */
+		for (i = 0; i < data->innr; i++) {
+			data->in_status[i] = pc87360_read_value(data, LD_IN, i,
+					     PC87365_REG_IN_STATUS);
+			/* Clear bits */
+			pc87360_write_value(data, LD_IN, i,
+					    PC87365_REG_IN_STATUS,
+					    data->in_status[i]);
+			if ((data->in_status[i] & CHAN_READY) == CHAN_READY) {
+				data->in[i] = pc87360_read_value(data, LD_IN,
+					      i, PC87365_REG_IN);
+			}
+			if (data->in_status[i] & CHAN_ENA) {
+				data->in_min[i] = pc87360_read_value(data,
+						  LD_IN, i,
+						  PC87365_REG_IN_MIN);
+				data->in_max[i] = pc87360_read_value(data,
+						  LD_IN, i,
+						  PC87365_REG_IN_MAX);
+				if (i >= 11)
+					data->in_crit[i-11] =
+						pc87360_read_value(data, LD_IN,
+						i, PC87365_REG_TEMP_CRIT);
+			}
+		}
+		if (data->innr) {
+			data->in_alarms = pc87360_read_value(data, LD_IN,
+					  NO_BANK, PC87365_REG_IN_ALARMS1)
+					| ((pc87360_read_value(data, LD_IN,
+					    NO_BANK, PC87365_REG_IN_ALARMS2)
+					    & 0x07) << 8);
+			data->vid = (data->vid_conf & 0xE0) ?
+				    pc87360_read_value(data, LD_IN,
+				    NO_BANK, PC87365_REG_VID) : 0x1F;
+		}
+
+		/* Temperatures */
+		for (i = 0; i < data->tempnr; i++) {
+			data->temp_status[i] = pc87360_read_value(data,
+					       LD_TEMP, i,
+					       PC87365_REG_TEMP_STATUS);
+			/* Clear bits */
+			pc87360_write_value(data, LD_TEMP, i,
+					    PC87365_REG_TEMP_STATUS,
+					    data->temp_status[i]);
+			if ((data->temp_status[i] & CHAN_READY) == CHAN_READY) {
+				data->temp[i] = pc87360_read_value(data,
+						LD_TEMP, i,
+						PC87365_REG_TEMP);
+			}
+			if (data->temp_status[i] & CHAN_ENA) {
+				data->temp_min[i] = pc87360_read_value(data,
+						    LD_TEMP, i,
+						    PC87365_REG_TEMP_MIN);
+				data->temp_max[i] = pc87360_read_value(data,
+						    LD_TEMP, i,
+						    PC87365_REG_TEMP_MAX);
+				data->temp_crit[i] = pc87360_read_value(data,
+						     LD_TEMP, i,
+						     PC87365_REG_TEMP_CRIT);
+			}
+		}
+		if (data->tempnr) {
+			data->temp_alarms = pc87360_read_value(data, LD_TEMP,
+					    NO_BANK, PC87365_REG_TEMP_ALARMS)
+					    & 0x3F;
+		}
+
+		data->last_updated = jiffies;
+		data->valid = true;
+	}
+
+	mutex_unlock(&data->update_lock);
+
+	return data;
+}
 
 static ssize_t in_input_show(struct device *dev,
 			     struct device_attribute *devattr, char *buf)
@@ -397,29 +407,52 @@ static ssize_t in_input_show(struct device *dev,
 	return sprintf(buf, "%u\n", IN_FROM_REG(data->in[attr->index],
 		       data->in_vref));
 }
-static ssize_t in_min_show(struct device *dev,
-			   struct device_attribute *devattr, char *buf)
+
+static struct sensor_device_attribute in_input[] = {
+	SENSOR_ATTR_RO(in0_input, in_input, 0),
+	SENSOR_ATTR_RO(in1_input, in_input, 1),
+	SENSOR_ATTR_RO(in2_input, in_input, 2),
+	SENSOR_ATTR_RO(in3_input, in_input, 3),
+	SENSOR_ATTR_RO(in4_input, in_input, 4),
+	SENSOR_ATTR_RO(in5_input, in_input, 5),
+	SENSOR_ATTR_RO(in6_input, in_input, 6),
+	SENSOR_ATTR_RO(in7_input, in_input, 7),
+	SENSOR_ATTR_RO(in8_input, in_input, 8),
+	SENSOR_ATTR_RO(in9_input, in_input, 9),
+	SENSOR_ATTR_RO(in10_input, in_input, 10),
+};
+
+static ssize_t in_status_show(struct device *dev,
+			      struct device_attribute *devattr, char *buf)
 {
 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n", IN_FROM_REG(data->in_min[attr->index],
-		       data->in_vref));
+	return sprintf(buf, "%u\n", data->in_status[attr->index]);
 }
-static ssize_t in_max_show(struct device *dev,
+
+static struct sensor_device_attribute in_status[] = {
+	SENSOR_ATTR_RO(in0_status, in_status, 0),
+	SENSOR_ATTR_RO(in1_status, in_status, 1),
+	SENSOR_ATTR_RO(in2_status, in_status, 2),
+	SENSOR_ATTR_RO(in3_status, in_status, 3),
+	SENSOR_ATTR_RO(in4_status, in_status, 4),
+	SENSOR_ATTR_RO(in5_status, in_status, 5),
+	SENSOR_ATTR_RO(in6_status, in_status, 6),
+	SENSOR_ATTR_RO(in7_status, in_status, 7),
+	SENSOR_ATTR_RO(in8_status, in_status, 8),
+	SENSOR_ATTR_RO(in9_status, in_status, 9),
+	SENSOR_ATTR_RO(in10_status, in_status, 10),
+};
+
+static ssize_t in_min_show(struct device *dev,
 			   struct device_attribute *devattr, char *buf)
 {
 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n", IN_FROM_REG(data->in_max[attr->index],
+	return sprintf(buf, "%u\n", IN_FROM_REG(data->in_min[attr->index],
 		       data->in_vref));
 }
-static ssize_t in_status_show(struct device *dev,
-			      struct device_attribute *devattr, char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n", data->in_status[attr->index]);
-}
+
 static ssize_t in_min_store(struct device *dev,
 			    struct device_attribute *devattr, const char *buf,
 			    size_t count)
@@ -440,6 +473,30 @@ static ssize_t in_min_store(struct device *dev,
 	mutex_unlock(&data->update_lock);
 	return count;
 }
+
+static struct sensor_device_attribute in_min[] = {
+	SENSOR_ATTR_RW(in0_min, in_min, 0),
+	SENSOR_ATTR_RW(in1_min, in_min, 1),
+	SENSOR_ATTR_RW(in2_min, in_min, 2),
+	SENSOR_ATTR_RW(in3_min, in_min, 3),
+	SENSOR_ATTR_RW(in4_min, in_min, 4),
+	SENSOR_ATTR_RW(in5_min, in_min, 5),
+	SENSOR_ATTR_RW(in6_min, in_min, 6),
+	SENSOR_ATTR_RW(in7_min, in_min, 7),
+	SENSOR_ATTR_RW(in8_min, in_min, 8),
+	SENSOR_ATTR_RW(in9_min, in_min, 9),
+	SENSOR_ATTR_RW(in10_min, in_min, 10),
+};
+
+static ssize_t in_max_show(struct device *dev,
+			   struct device_attribute *devattr, char *buf)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%u\n", IN_FROM_REG(data->in_max[attr->index],
+		       data->in_vref));
+}
+
 static ssize_t in_max_store(struct device *dev,
 			    struct device_attribute *devattr, const char *buf,
 			    size_t count)
@@ -462,45 +519,6 @@ static ssize_t in_max_store(struct device *dev,
 	return count;
 }
 
-static struct sensor_device_attribute in_input[] = {
-	SENSOR_ATTR_RO(in0_input, in_input, 0),
-	SENSOR_ATTR_RO(in1_input, in_input, 1),
-	SENSOR_ATTR_RO(in2_input, in_input, 2),
-	SENSOR_ATTR_RO(in3_input, in_input, 3),
-	SENSOR_ATTR_RO(in4_input, in_input, 4),
-	SENSOR_ATTR_RO(in5_input, in_input, 5),
-	SENSOR_ATTR_RO(in6_input, in_input, 6),
-	SENSOR_ATTR_RO(in7_input, in_input, 7),
-	SENSOR_ATTR_RO(in8_input, in_input, 8),
-	SENSOR_ATTR_RO(in9_input, in_input, 9),
-	SENSOR_ATTR_RO(in10_input, in_input, 10),
-};
-static struct sensor_device_attribute in_status[] = {
-	SENSOR_ATTR_RO(in0_status, in_status, 0),
-	SENSOR_ATTR_RO(in1_status, in_status, 1),
-	SENSOR_ATTR_RO(in2_status, in_status, 2),
-	SENSOR_ATTR_RO(in3_status, in_status, 3),
-	SENSOR_ATTR_RO(in4_status, in_status, 4),
-	SENSOR_ATTR_RO(in5_status, in_status, 5),
-	SENSOR_ATTR_RO(in6_status, in_status, 6),
-	SENSOR_ATTR_RO(in7_status, in_status, 7),
-	SENSOR_ATTR_RO(in8_status, in_status, 8),
-	SENSOR_ATTR_RO(in9_status, in_status, 9),
-	SENSOR_ATTR_RO(in10_status, in_status, 10),
-};
-static struct sensor_device_attribute in_min[] = {
-	SENSOR_ATTR_RW(in0_min, in_min, 0),
-	SENSOR_ATTR_RW(in1_min, in_min, 1),
-	SENSOR_ATTR_RW(in2_min, in_min, 2),
-	SENSOR_ATTR_RW(in3_min, in_min, 3),
-	SENSOR_ATTR_RW(in4_min, in_min, 4),
-	SENSOR_ATTR_RW(in5_min, in_min, 5),
-	SENSOR_ATTR_RW(in6_min, in_min, 6),
-	SENSOR_ATTR_RW(in7_min, in_min, 7),
-	SENSOR_ATTR_RW(in8_min, in_min, 8),
-	SENSOR_ATTR_RW(in9_min, in_min, 9),
-	SENSOR_ATTR_RW(in10_min, in_min, 10),
-};
 static struct sensor_device_attribute in_max[] = {
 	SENSOR_ATTR_RW(in0_max, in_max, 0),
 	SENSOR_ATTR_RW(in1_max, in_max, 1),
@@ -534,14 +552,6 @@ static ssize_t in_min_alarm_show(struct device *dev,
 
 	return sprintf(buf, "%u\n", !!(data->in_status[nr] & CHAN_ALM_MIN));
 }
-static ssize_t in_max_alarm_show(struct device *dev,
-				 struct device_attribute *devattr, char *buf)
-{
-	struct pc87360_data *data = pc87360_update_device(dev);
-	unsigned nr = to_sensor_dev_attr(devattr)->index;
-
-	return sprintf(buf, "%u\n", !!(data->in_status[nr] & CHAN_ALM_MAX));
-}
 
 static struct sensor_device_attribute in_min_alarm[] = {
 	SENSOR_ATTR_RO(in0_min_alarm, in_min_alarm, 0),
@@ -556,6 +566,16 @@ static struct sensor_device_attribute in_min_alarm[] = {
 	SENSOR_ATTR_RO(in9_min_alarm, in_min_alarm, 9),
 	SENSOR_ATTR_RO(in10_min_alarm, in_min_alarm, 10),
 };
+
+static ssize_t in_max_alarm_show(struct device *dev,
+				 struct device_attribute *devattr, char *buf)
+{
+	struct pc87360_data *data = pc87360_update_device(dev);
+	unsigned nr = to_sensor_dev_attr(devattr)->index;
+
+	return sprintf(buf, "%u\n", !!(data->in_status[nr] & CHAN_ALM_MAX));
+}
+
 static struct sensor_device_attribute in_max_alarm[] = {
 	SENSOR_ATTR_RO(in0_max_alarm, in_max_alarm, 0),
 	SENSOR_ATTR_RO(in1_max_alarm, in_max_alarm, 1),
@@ -592,6 +612,7 @@ static ssize_t vrm_show(struct device *dev, struct device_attribute *attr,
 	struct pc87360_data *data = dev_get_drvdata(dev);
 	return sprintf(buf, "%u\n", data->vrm);
 }
+
 static ssize_t vrm_store(struct device *dev, struct device_attribute *attr,
 			 const char *buf, size_t count)
 {
@@ -648,37 +669,39 @@ static ssize_t therm_input_show(struct device *dev,
 	return sprintf(buf, "%u\n", IN_FROM_REG(data->in[attr->index],
 		       data->in_vref));
 }
-static ssize_t therm_min_show(struct device *dev,
-			      struct device_attribute *devattr, char *buf)
+
+/*
+ * the +11 term below reflects the fact that VLM units 11,12,13 are
+ * used in the chip to measure voltage across the thermistors
+ */
+static struct sensor_device_attribute therm_input[] = {
+	SENSOR_ATTR_RO(temp4_input, therm_input, 0 + 11),
+	SENSOR_ATTR_RO(temp5_input, therm_input, 1 + 11),
+	SENSOR_ATTR_RO(temp6_input, therm_input, 2 + 11),
+};
+
+static ssize_t therm_status_show(struct device *dev,
+				 struct device_attribute *devattr, char *buf)
 {
 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n", IN_FROM_REG(data->in_min[attr->index],
-		       data->in_vref));
+	return sprintf(buf, "%u\n", data->in_status[attr->index]);
 }
-static ssize_t therm_max_show(struct device *dev,
+
+static struct sensor_device_attribute therm_status[] = {
+	SENSOR_ATTR_RO(temp4_status, therm_status, 0 + 11),
+	SENSOR_ATTR_RO(temp5_status, therm_status, 1 + 11),
+	SENSOR_ATTR_RO(temp6_status, therm_status, 2 + 11),
+};
+
+static ssize_t therm_min_show(struct device *dev,
 			      struct device_attribute *devattr, char *buf)
 {
 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n", IN_FROM_REG(data->in_max[attr->index],
-		       data->in_vref));
-}
-static ssize_t therm_crit_show(struct device *dev,
-			       struct device_attribute *devattr, char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n", IN_FROM_REG(data->in_crit[attr->index-11],
+	return sprintf(buf, "%u\n", IN_FROM_REG(data->in_min[attr->index],
 		       data->in_vref));
 }
-static ssize_t therm_status_show(struct device *dev,
-				 struct device_attribute *devattr, char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n", data->in_status[attr->index]);
-}
 
 static ssize_t therm_min_store(struct device *dev,
 			       struct device_attribute *devattr,
@@ -701,6 +724,21 @@ static ssize_t therm_min_store(struct device *dev,
 	return count;
 }
 
+static struct sensor_device_attribute therm_min[] = {
+	SENSOR_ATTR_RW(temp4_min, therm_min, 0 + 11),
+	SENSOR_ATTR_RW(temp5_min, therm_min, 1 + 11),
+	SENSOR_ATTR_RW(temp6_min, therm_min, 2 + 11),
+};
+
+static ssize_t therm_max_show(struct device *dev,
+			      struct device_attribute *devattr, char *buf)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%u\n", IN_FROM_REG(data->in_max[attr->index],
+		       data->in_vref));
+}
+
 static ssize_t therm_max_store(struct device *dev,
 			       struct device_attribute *devattr,
 			       const char *buf, size_t count)
@@ -721,6 +759,22 @@ static ssize_t therm_max_store(struct device *dev,
 	mutex_unlock(&data->update_lock);
 	return count;
 }
+
+static struct sensor_device_attribute therm_max[] = {
+	SENSOR_ATTR_RW(temp4_max, therm_max, 0 + 11),
+	SENSOR_ATTR_RW(temp5_max, therm_max, 1 + 11),
+	SENSOR_ATTR_RW(temp6_max, therm_max, 2 + 11),
+};
+
+static ssize_t therm_crit_show(struct device *dev,
+			       struct device_attribute *devattr, char *buf)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%u\n", IN_FROM_REG(data->in_crit[attr->index-11],
+		       data->in_vref));
+}
+
 static ssize_t therm_crit_store(struct device *dev,
 				struct device_attribute *devattr,
 				const char *buf, size_t count)
@@ -742,30 +796,6 @@ static ssize_t therm_crit_store(struct device *dev,
 	return count;
 }
 
-/*
- * the +11 term below reflects the fact that VLM units 11,12,13 are
- * used in the chip to measure voltage across the thermistors
- */
-static struct sensor_device_attribute therm_input[] = {
-	SENSOR_ATTR_RO(temp4_input, therm_input, 0 + 11),
-	SENSOR_ATTR_RO(temp5_input, therm_input, 1 + 11),
-	SENSOR_ATTR_RO(temp6_input, therm_input, 2 + 11),
-};
-static struct sensor_device_attribute therm_status[] = {
-	SENSOR_ATTR_RO(temp4_status, therm_status, 0 + 11),
-	SENSOR_ATTR_RO(temp5_status, therm_status, 1 + 11),
-	SENSOR_ATTR_RO(temp6_status, therm_status, 2 + 11),
-};
-static struct sensor_device_attribute therm_min[] = {
-	SENSOR_ATTR_RW(temp4_min, therm_min, 0 + 11),
-	SENSOR_ATTR_RW(temp5_min, therm_min, 1 + 11),
-	SENSOR_ATTR_RW(temp6_min, therm_min, 2 + 11),
-};
-static struct sensor_device_attribute therm_max[] = {
-	SENSOR_ATTR_RW(temp4_max, therm_max, 0 + 11),
-	SENSOR_ATTR_RW(temp5_max, therm_max, 1 + 11),
-	SENSOR_ATTR_RW(temp6_max, therm_max, 2 + 11),
-};
 static struct sensor_device_attribute therm_crit[] = {
 	SENSOR_ATTR_RW(temp4_crit, therm_crit, 0 + 11),
 	SENSOR_ATTR_RW(temp5_crit, therm_crit, 1 + 11),
@@ -776,7 +806,6 @@ static struct sensor_device_attribute therm_crit[] = {
  * show_therm_min/max_alarm() reads data from the per-channel voltage
  * status register (sec 11.5.12)
  */
-
 static ssize_t therm_min_alarm_show(struct device *dev,
 				    struct device_attribute *devattr,
 				    char *buf)
@@ -786,6 +815,13 @@ static ssize_t therm_min_alarm_show(struct device *dev,
 
 	return sprintf(buf, "%u\n", !!(data->in_status[nr] & CHAN_ALM_MIN));
 }
+
+static struct sensor_device_attribute therm_min_alarm[] = {
+	SENSOR_ATTR_RO(temp4_min_alarm, therm_min_alarm, 0 + 11),
+	SENSOR_ATTR_RO(temp5_min_alarm, therm_min_alarm, 1 + 11),
+	SENSOR_ATTR_RO(temp6_min_alarm, therm_min_alarm, 2 + 11),
+};
+
 static ssize_t therm_max_alarm_show(struct device *dev,
 				    struct device_attribute *devattr,
 				    char *buf)
@@ -795,6 +831,13 @@ static ssize_t therm_max_alarm_show(struct device *dev,
 
 	return sprintf(buf, "%u\n", !!(data->in_status[nr] & CHAN_ALM_MAX));
 }
+
+static struct sensor_device_attribute therm_max_alarm[] = {
+	SENSOR_ATTR_RO(temp4_max_alarm, therm_max_alarm, 0 + 11),
+	SENSOR_ATTR_RO(temp5_max_alarm, therm_max_alarm, 1 + 11),
+	SENSOR_ATTR_RO(temp6_max_alarm, therm_max_alarm, 2 + 11),
+};
+
 static ssize_t therm_crit_alarm_show(struct device *dev,
 				     struct device_attribute *devattr,
 				     char *buf)
@@ -805,16 +848,6 @@ static ssize_t therm_crit_alarm_show(struct device *dev,
 	return sprintf(buf, "%u\n", !!(data->in_status[nr] & TEMP_ALM_CRIT));
 }
 
-static struct sensor_device_attribute therm_min_alarm[] = {
-	SENSOR_ATTR_RO(temp4_min_alarm, therm_min_alarm, 0 + 11),
-	SENSOR_ATTR_RO(temp5_min_alarm, therm_min_alarm, 1 + 11),
-	SENSOR_ATTR_RO(temp6_min_alarm, therm_min_alarm, 2 + 11),
-};
-static struct sensor_device_attribute therm_max_alarm[] = {
-	SENSOR_ATTR_RO(temp4_max_alarm, therm_max_alarm, 0 + 11),
-	SENSOR_ATTR_RO(temp5_max_alarm, therm_max_alarm, 1 + 11),
-	SENSOR_ATTR_RO(temp6_max_alarm, therm_max_alarm, 2 + 11),
-};
 static struct sensor_device_attribute therm_crit_alarm[] = {
 	SENSOR_ATTR_RO(temp4_crit_alarm, therm_crit_alarm, 0 + 11),
 	SENSOR_ATTR_RO(temp5_crit_alarm, therm_crit_alarm, 1 + 11),
@@ -849,37 +882,32 @@ static ssize_t temp_input_show(struct device *dev,
 	return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp[attr->index]));
 }
 
-static ssize_t temp_min_show(struct device *dev,
-			     struct device_attribute *devattr, char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp_min[attr->index]));
-}
+static struct sensor_device_attribute temp_input[] = {
+	SENSOR_ATTR_RO(temp1_input, temp_input, 0),
+	SENSOR_ATTR_RO(temp2_input, temp_input, 1),
+	SENSOR_ATTR_RO(temp3_input, temp_input, 2),
+};
 
-static ssize_t temp_max_show(struct device *dev,
-			     struct device_attribute *devattr, char *buf)
+static ssize_t temp_status_show(struct device *dev,
+				struct device_attribute *devattr, char *buf)
 {
 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp_max[attr->index]));
+	return sprintf(buf, "%d\n", data->temp_status[attr->index]);
 }
 
-static ssize_t temp_crit_show(struct device *dev,
-			      struct device_attribute *devattr, char *buf)
-{
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%d\n",
-		       TEMP_FROM_REG(data->temp_crit[attr->index]));
-}
+static struct sensor_device_attribute temp_status[] = {
+	SENSOR_ATTR_RO(temp1_status, temp_status, 0),
+	SENSOR_ATTR_RO(temp2_status, temp_status, 1),
+	SENSOR_ATTR_RO(temp3_status, temp_status, 2),
+};
 
-static ssize_t temp_status_show(struct device *dev,
-				struct device_attribute *devattr, char *buf)
+static ssize_t temp_min_show(struct device *dev,
+			     struct device_attribute *devattr, char *buf)
 {
 	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
 	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%d\n", data->temp_status[attr->index]);
+	return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp_min[attr->index]));
 }
 
 static ssize_t temp_min_store(struct device *dev,
@@ -903,6 +931,20 @@ static ssize_t temp_min_store(struct device *dev,
 	return count;
 }
 
+static struct sensor_device_attribute temp_min[] = {
+	SENSOR_ATTR_RW(temp1_min, temp_min, 0),
+	SENSOR_ATTR_RW(temp2_min, temp_min, 1),
+	SENSOR_ATTR_RW(temp3_min, temp_min, 2),
+};
+
+static ssize_t temp_max_show(struct device *dev,
+			     struct device_attribute *devattr, char *buf)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%d\n", TEMP_FROM_REG(data->temp_max[attr->index]));
+}
+
 static ssize_t temp_max_store(struct device *dev,
 			      struct device_attribute *devattr,
 			      const char *buf, size_t count)
@@ -924,6 +966,21 @@ static ssize_t temp_max_store(struct device *dev,
 	return count;
 }
 
+static struct sensor_device_attribute temp_max[] = {
+	SENSOR_ATTR_RW(temp1_max, temp_max, 0),
+	SENSOR_ATTR_RW(temp2_max, temp_max, 1),
+	SENSOR_ATTR_RW(temp3_max, temp_max, 2),
+};
+
+static ssize_t temp_crit_show(struct device *dev,
+			      struct device_attribute *devattr, char *buf)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%d\n",
+		       TEMP_FROM_REG(data->temp_crit[attr->index]));
+}
+
 static ssize_t temp_crit_store(struct device *dev,
 			       struct device_attribute *devattr,
 			       const char *buf, size_t count)
@@ -945,47 +1002,17 @@ static ssize_t temp_crit_store(struct device *dev,
 	return count;
 }
 
-static struct sensor_device_attribute temp_input[] = {
-	SENSOR_ATTR_RO(temp1_input, temp_input, 0),
-	SENSOR_ATTR_RO(temp2_input, temp_input, 1),
-	SENSOR_ATTR_RO(temp3_input, temp_input, 2),
-};
-static struct sensor_device_attribute temp_status[] = {
-	SENSOR_ATTR_RO(temp1_status, temp_status, 0),
-	SENSOR_ATTR_RO(temp2_status, temp_status, 1),
-	SENSOR_ATTR_RO(temp3_status, temp_status, 2),
-};
-static struct sensor_device_attribute temp_min[] = {
-	SENSOR_ATTR_RW(temp1_min, temp_min, 0),
-	SENSOR_ATTR_RW(temp2_min, temp_min, 1),
-	SENSOR_ATTR_RW(temp3_min, temp_min, 2),
-};
-static struct sensor_device_attribute temp_max[] = {
-	SENSOR_ATTR_RW(temp1_max, temp_max, 0),
-	SENSOR_ATTR_RW(temp2_max, temp_max, 1),
-	SENSOR_ATTR_RW(temp3_max, temp_max, 2),
-};
 static struct sensor_device_attribute temp_crit[] = {
 	SENSOR_ATTR_RW(temp1_crit, temp_crit, 0),
 	SENSOR_ATTR_RW(temp2_crit, temp_crit, 1),
 	SENSOR_ATTR_RW(temp3_crit, temp_crit, 2),
 };
 
-static ssize_t alarms_temp_show(struct device *dev,
-				struct device_attribute *attr, char *buf)
-{
-	struct pc87360_data *data = pc87360_update_device(dev);
-	return sprintf(buf, "%u\n", data->temp_alarms);
-}
-
-static DEVICE_ATTR_RO(alarms_temp);
-
 /*
- * show_temp_min/max_alarm() reads data from the per-channel status
+ * temp_min/max_alarm_show() reads data from the per-channel status
  * register (sec 12.3.7), not the temp event status registers (sec
  * 12.3.2) that show_temp_alarm() reads (via data->temp_alarms)
  */
-
 static ssize_t temp_min_alarm_show(struct device *dev,
 				   struct device_attribute *devattr,
 				   char *buf)
@@ -996,6 +1023,12 @@ static ssize_t temp_min_alarm_show(struct device *dev,
 	return sprintf(buf, "%u\n", !!(data->temp_status[nr] & CHAN_ALM_MIN));
 }
 
+static struct sensor_device_attribute temp_min_alarm[] = {
+	SENSOR_ATTR_RO(temp1_min_alarm, temp_min_alarm, 0),
+	SENSOR_ATTR_RO(temp2_min_alarm, temp_min_alarm, 1),
+	SENSOR_ATTR_RO(temp3_min_alarm, temp_min_alarm, 2),
+};
+
 static ssize_t temp_max_alarm_show(struct device *dev,
 				   struct device_attribute *devattr,
 				   char *buf)
@@ -1006,6 +1039,12 @@ static ssize_t temp_max_alarm_show(struct device *dev,
 	return sprintf(buf, "%u\n", !!(data->temp_status[nr] & CHAN_ALM_MAX));
 }
 
+static struct sensor_device_attribute temp_max_alarm[] = {
+	SENSOR_ATTR_RO(temp1_max_alarm, temp_max_alarm, 0),
+	SENSOR_ATTR_RO(temp2_max_alarm, temp_max_alarm, 1),
+	SENSOR_ATTR_RO(temp3_max_alarm, temp_max_alarm, 2),
+};
+
 static ssize_t temp_crit_alarm_show(struct device *dev,
 				    struct device_attribute *devattr,
 				    char *buf)
@@ -1016,18 +1055,6 @@ static ssize_t temp_crit_alarm_show(struct device *dev,
 	return sprintf(buf, "%u\n", !!(data->temp_status[nr] & TEMP_ALM_CRIT));
 }
 
-static struct sensor_device_attribute temp_min_alarm[] = {
-	SENSOR_ATTR_RO(temp1_min_alarm, temp_min_alarm, 0),
-	SENSOR_ATTR_RO(temp2_min_alarm, temp_min_alarm, 1),
-	SENSOR_ATTR_RO(temp3_min_alarm, temp_min_alarm, 2),
-};
-
-static struct sensor_device_attribute temp_max_alarm[] = {
-	SENSOR_ATTR_RO(temp1_max_alarm, temp_max_alarm, 0),
-	SENSOR_ATTR_RO(temp2_max_alarm, temp_max_alarm, 1),
-	SENSOR_ATTR_RO(temp3_max_alarm, temp_max_alarm, 2),
-};
-
 static struct sensor_device_attribute temp_crit_alarm[] = {
 	SENSOR_ATTR_RO(temp1_crit_alarm, temp_crit_alarm, 0),
 	SENSOR_ATTR_RO(temp2_crit_alarm, temp_crit_alarm, 1),
@@ -1043,6 +1070,7 @@ static ssize_t temp_fault_show(struct device *dev,
 
 	return sprintf(buf, "%u\n", !!(data->temp_status[nr] & TEMP_FAULT));
 }
+
 static struct sensor_device_attribute temp_fault[] = {
 	SENSOR_ATTR_RO(temp1_fault, temp_fault, 0),
 	SENSOR_ATTR_RO(temp2_fault, temp_fault, 1),
@@ -1074,106 +1102,180 @@ static const struct attribute_group pc8736x_temp_attr_group[] = {
 	{ .attrs = pc8736x_temp_attr[2] }
 };
 
-static ssize_t name_show(struct device *dev,
-			struct device_attribute *devattr, char *buf)
+static ssize_t alarms_temp_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
 {
-	struct pc87360_data *data = dev_get_drvdata(dev);
-	return sprintf(buf, "%s\n", data->name);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%u\n", data->temp_alarms);
 }
 
-static DEVICE_ATTR_RO(name);
+static DEVICE_ATTR_RO(alarms_temp);
 
-/*
- * Device detection, registration and update
- */
+static ssize_t fan_input_show(struct device *dev,
+			      struct device_attribute *devattr, char *buf)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%u\n", FAN_FROM_REG(data->fan[attr->index],
+		       FAN_DIV_FROM_REG(data->fan_status[attr->index])));
+}
 
-static int __init pc87360_find(int sioaddr, u8 *devid,
-			       unsigned short *addresses)
+static struct sensor_device_attribute fan_input[] = {
+	SENSOR_ATTR_RO(fan1_input, fan_input, 0),
+	SENSOR_ATTR_RO(fan2_input, fan_input, 1),
+	SENSOR_ATTR_RO(fan3_input, fan_input, 2),
+};
+
+static ssize_t fan_status_show(struct device *dev,
+			       struct device_attribute *devattr, char *buf)
 {
-	u16 val;
-	int i;
-	int nrdev; /* logical device count */
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%u\n",
+		       FAN_STATUS_FROM_REG(data->fan_status[attr->index]));
+}
 
-	/* No superio_enter */
+static struct sensor_device_attribute fan_status[] = {
+	SENSOR_ATTR_RO(fan1_status, fan_status, 0),
+	SENSOR_ATTR_RO(fan2_status, fan_status, 1),
+	SENSOR_ATTR_RO(fan3_status, fan_status, 2),
+};
 
-	/* Identify device */
-	val = force_id ? force_id : superio_inb(sioaddr, DEVID);
-	switch (val) {
-	case 0xE1: /* PC87360 */
-	case 0xE8: /* PC87363 */
-	case 0xE4: /* PC87364 */
-		nrdev = 1;
-		break;
-	case 0xE5: /* PC87365 */
-	case 0xE9: /* PC87366 */
-		nrdev = 3;
-		break;
-	default:
-		superio_exit(sioaddr);
-		return -ENODEV;
+static ssize_t fan_div_show(struct device *dev,
+			    struct device_attribute *devattr, char *buf)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%u\n",
+		       FAN_DIV_FROM_REG(data->fan_status[attr->index]));
+}
+
+static struct sensor_device_attribute fan_div[] = {
+	SENSOR_ATTR_RO(fan1_div, fan_div, 0),
+	SENSOR_ATTR_RO(fan2_div, fan_div, 1),
+	SENSOR_ATTR_RO(fan3_div, fan_div, 2),
+};
+
+static ssize_t fan_min_show(struct device *dev,
+			    struct device_attribute *devattr, char *buf)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%u\n", FAN_FROM_REG(data->fan_min[attr->index],
+		       FAN_DIV_FROM_REG(data->fan_status[attr->index])));
+}
+
+static ssize_t fan_min_store(struct device *dev,
+			     struct device_attribute *devattr,
+			     const char *buf, size_t count)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = dev_get_drvdata(dev);
+	long fan_min;
+	int err;
+
+	err = kstrtol(buf, 10, &fan_min);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+	fan_min = FAN_TO_REG(fan_min,
+			     FAN_DIV_FROM_REG(data->fan_status[attr->index]));
+
+	/* If it wouldn't fit, change clock divisor */
+	while (fan_min > 255
+	    && (data->fan_status[attr->index] & 0x60) != 0x60) {
+		fan_min >>= 1;
+		data->fan[attr->index] >>= 1;
+		data->fan_status[attr->index] += 0x20;
 	}
-	/* Remember the device id */
-	*devid = val;
+	data->fan_min[attr->index] = fan_min > 255 ? 255 : fan_min;
+	pc87360_write_value(data, LD_FAN, NO_BANK,
+			    PC87360_REG_FAN_MIN(attr->index),
+			    data->fan_min[attr->index]);
 
-	for (i = 0; i < nrdev; i++) {
-		/* select logical device */
-		superio_outb(sioaddr, DEV, logdev[i]);
+	/* Write new divider, preserve alarm bits */
+	pc87360_write_value(data, LD_FAN, NO_BANK,
+			    PC87360_REG_FAN_STATUS(attr->index),
+			    data->fan_status[attr->index] & 0xF9);
+	mutex_unlock(&data->update_lock);
 
-		val = superio_inb(sioaddr, ACT);
-		if (!(val & 0x01)) {
-			pr_info("Device 0x%02x not activated\n", logdev[i]);
-			continue;
-		}
+	return count;
+}
 
-		val = (superio_inb(sioaddr, BASE) << 8)
-		    | superio_inb(sioaddr, BASE + 1);
-		if (!val) {
-			pr_info("Base address not set for device 0x%02x\n",
-				logdev[i]);
-			continue;
-		}
+static struct sensor_device_attribute fan_min[] = {
+	SENSOR_ATTR_RW(fan1_min, fan_min, 0),
+	SENSOR_ATTR_RW(fan2_min, fan_min, 1),
+	SENSOR_ATTR_RW(fan3_min, fan_min, 2),
+};
 
-		addresses[i] = val;
+#define FAN_UNIT_ATTRS(X)		\
+{	&fan_input[X].dev_attr.attr,	\
+	&fan_status[X].dev_attr.attr,	\
+	&fan_div[X].dev_attr.attr,	\
+	&fan_min[X].dev_attr.attr,	\
+	NULL				\
+}
 
-		if (i == 0) { /* Fans */
-			confreg[0] = superio_inb(sioaddr, 0xF0);
-			confreg[1] = superio_inb(sioaddr, 0xF1);
+static struct attribute *pc8736x_fan_attr[][5] = {
+	FAN_UNIT_ATTRS(0),
+	FAN_UNIT_ATTRS(1),
+	FAN_UNIT_ATTRS(2)
+};
 
-			pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 1,
-				 (confreg[0] >> 2) & 1, (confreg[0] >> 3) & 1,
-				 (confreg[0] >> 4) & 1);
-			pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 2,
-				 (confreg[0] >> 5) & 1, (confreg[0] >> 6) & 1,
-				 (confreg[0] >> 7) & 1);
-			pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 3,
-				 confreg[1] & 1, (confreg[1] >> 1) & 1,
-				 (confreg[1] >> 2) & 1);
-		} else if (i == 1) { /* Voltages */
-			/* Are we using thermistors? */
-			if (*devid == 0xE9) { /* PC87366 */
-				/*
-				 * These registers are not logical-device
-				 * specific, just that we won't need them if
-				 * we don't use the VLM device
-				 */
-				confreg[2] = superio_inb(sioaddr, 0x2B);
-				confreg[3] = superio_inb(sioaddr, 0x25);
+static const struct attribute_group pc8736x_fan_attr_group[] = {
+	{ .attrs = pc8736x_fan_attr[0], },
+	{ .attrs = pc8736x_fan_attr[1], },
+	{ .attrs = pc8736x_fan_attr[2], },
+};
 
-				if (confreg[2] & 0x40) {
-					pr_info("Using thermistors for temperature monitoring\n");
-				}
-				if (confreg[3] & 0xE0) {
-					pr_info("VID inputs routed (mode %u)\n",
-						confreg[3] >> 5);
-				}
-			}
-		}
-	}
+static ssize_t pwm_show(struct device *dev, struct device_attribute *devattr,
+			char *buf)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = pc87360_update_device(dev);
+	return sprintf(buf, "%u\n",
+		       PWM_FROM_REG(data->pwm[attr->index],
+				    FAN_CONFIG_INVERT(data->fan_conf,
+						      attr->index)));
+}
 
-	superio_exit(sioaddr);
-	return 0;
+static ssize_t pwm_store(struct device *dev, struct device_attribute *devattr,
+			 const char *buf, size_t count)
+{
+	struct sensor_device_attribute *attr = to_sensor_dev_attr(devattr);
+	struct pc87360_data *data = dev_get_drvdata(dev);
+	long val;
+	int err;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+	data->pwm[attr->index] = PWM_TO_REG(val,
+			      FAN_CONFIG_INVERT(data->fan_conf, attr->index));
+	pc87360_write_value(data, LD_FAN, NO_BANK, PC87360_REG_PWM(attr->index),
+			    data->pwm[attr->index]);
+	mutex_unlock(&data->update_lock);
+	return count;
 }
 
+static struct sensor_device_attribute pwm[] = {
+	SENSOR_ATTR_RW(pwm1, pwm, 0),
+	SENSOR_ATTR_RW(pwm2, pwm, 1),
+	SENSOR_ATTR_RW(pwm3, pwm, 2),
+};
+
+static ssize_t name_show(struct device *dev,
+			struct device_attribute *devattr, char *buf)
+{
+	struct pc87360_data *data = dev_get_drvdata(dev);
+	return sprintf(buf, "%s\n", data->name);
+}
+
+static DEVICE_ATTR_RO(name);
+
 static void pc87360_remove_files(struct device *dev)
 {
 	int i;
@@ -1190,6 +1292,146 @@ static void pc87360_remove_files(struct device *dev)
 	sysfs_remove_group(&dev->kobj, &pc8736x_vin_group);
 }
 
+static void pc87360_init_device(struct platform_device *pdev,
+				int use_thermistors)
+{
+	struct pc87360_data *data = platform_get_drvdata(pdev);
+	int i, nr;
+	const u8 init_in[14] = { 2, 2, 2, 2, 2, 2, 2, 1, 1, 3, 1, 2, 2, 2 };
+	const u8 init_temp[3] = { 2, 2, 1 };
+	u8 reg;
+
+	if (init >= 2 && data->innr) {
+		reg = pc87360_read_value(data, LD_IN, NO_BANK,
+					 PC87365_REG_IN_CONVRATE);
+		dev_info(&pdev->dev,
+			 "VLM conversion set to 1s period, 160us delay\n");
+		pc87360_write_value(data, LD_IN, NO_BANK,
+				    PC87365_REG_IN_CONVRATE,
+				    (reg & 0xC0) | 0x11);
+	}
+
+	nr = data->innr < 11 ? data->innr : 11;
+	for (i = 0; i < nr; i++) {
+		reg = pc87360_read_value(data, LD_IN, i,
+					 PC87365_REG_IN_STATUS);
+		dev_dbg(&pdev->dev, "bios in%d status:0x%02x\n", i, reg);
+		if (init >= init_in[i]) {
+			/* Forcibly enable voltage channel */
+			if (!(reg & CHAN_ENA)) {
+				dev_dbg(&pdev->dev, "Forcibly enabling in%d\n",
+					i);
+				pc87360_write_value(data, LD_IN, i,
+						    PC87365_REG_IN_STATUS,
+						    (reg & 0x68) | 0x87);
+			}
+		}
+	}
+
+	/*
+	 * We can't blindly trust the Super-I/O space configuration bit,
+	 * most BIOS won't set it properly
+	 */
+	dev_dbg(&pdev->dev, "bios thermistors:%d\n", use_thermistors);
+	for (i = 11; i < data->innr; i++) {
+		reg = pc87360_read_value(data, LD_IN, i,
+					 PC87365_REG_TEMP_STATUS);
+		use_thermistors = use_thermistors || (reg & CHAN_ENA);
+		/* thermistors are temp[4-6], measured on vin[11-14] */
+		dev_dbg(&pdev->dev, "bios temp%d_status:0x%02x\n", i-7, reg);
+	}
+	dev_dbg(&pdev->dev, "using thermistors:%d\n", use_thermistors);
+
+	i = use_thermistors ? 2 : 0;
+	for (; i < data->tempnr; i++) {
+		reg = pc87360_read_value(data, LD_TEMP, i,
+					 PC87365_REG_TEMP_STATUS);
+		dev_dbg(&pdev->dev, "bios temp%d_status:0x%02x\n", i + 1, reg);
+		if (init >= init_temp[i]) {
+			/* Forcibly enable temperature channel */
+			if (!(reg & CHAN_ENA)) {
+				dev_dbg(&pdev->dev,
+					"Forcibly enabling temp%d\n", i + 1);
+				pc87360_write_value(data, LD_TEMP, i,
+						    PC87365_REG_TEMP_STATUS,
+						    0xCF);
+			}
+		}
+	}
+
+	if (use_thermistors) {
+		for (i = 11; i < data->innr; i++) {
+			if (init >= init_in[i]) {
+				/*
+				 * The pin may already be used by thermal
+				 * diodes
+				 */
+				reg = pc87360_read_value(data, LD_TEMP,
+				      (i - 11) / 2, PC87365_REG_TEMP_STATUS);
+				if (reg & CHAN_ENA) {
+					dev_dbg(&pdev->dev,
+			"Skipping temp%d, pin already in use by temp%d\n",
+						i - 7, (i - 11) / 2);
+					continue;
+				}
+
+				/* Forcibly enable thermistor channel */
+				reg = pc87360_read_value(data, LD_IN, i,
+							 PC87365_REG_IN_STATUS);
+				if (!(reg & CHAN_ENA)) {
+					dev_dbg(&pdev->dev,
+						"Forcibly enabling temp%d\n",
+						i - 7);
+					pc87360_write_value(data, LD_IN, i,
+						PC87365_REG_TEMP_STATUS,
+						(reg & 0x60) | 0x8F);
+				}
+			}
+		}
+	}
+
+	if (data->innr) {
+		reg = pc87360_read_value(data, LD_IN, NO_BANK,
+					 PC87365_REG_IN_CONFIG);
+		dev_dbg(&pdev->dev, "bios vin-cfg:0x%02x\n", reg);
+		if (reg & CHAN_ENA) {
+			dev_dbg(&pdev->dev,
+				"Forcibly enabling monitoring (VLM)\n");
+			pc87360_write_value(data, LD_IN, NO_BANK,
+					    PC87365_REG_IN_CONFIG,
+					    reg & 0xFE);
+		}
+	}
+
+	if (data->tempnr) {
+		reg = pc87360_read_value(data, LD_TEMP, NO_BANK,
+					 PC87365_REG_TEMP_CONFIG);
+		dev_dbg(&pdev->dev, "bios temp-cfg:0x%02x\n", reg);
+		if (reg & CHAN_ENA) {
+			dev_dbg(&pdev->dev,
+				"Forcibly enabling monitoring (TMS)\n");
+			pc87360_write_value(data, LD_TEMP, NO_BANK,
+					    PC87365_REG_TEMP_CONFIG,
+					    reg & 0xFE);
+		}
+
+		if (init >= 2) {
+			/* Chip config as documented by National Semi. */
+			pc87360_write_value(data, LD_TEMP, 0xF, 0xA, 0x08);
+			/*
+			 * We voluntarily omit the bank here, in case the
+			 * sequence itself matters. It shouldn't be a problem,
+			 * since nobody else is supposed to access the
+			 * device at that point.
+			 */
+			pc87360_write_value(data, LD_TEMP, NO_BANK, 0xB, 0x04);
+			pc87360_write_value(data, LD_TEMP, NO_BANK, 0xC, 0x35);
+			pc87360_write_value(data, LD_TEMP, NO_BANK, 0xD, 0x05);
+			pc87360_write_value(data, LD_TEMP, NO_BANK, 0xE, 0x05);
+		}
+	}
+}
+
 static int pc87360_probe(struct platform_device *pdev)
 {
 	int i;
@@ -1239,7 +1481,7 @@ static int pc87360_probe(struct platform_device *pdev)
 		data->address[i] = extra_isa[i];
 		if (data->address[i]
 		 && !devm_request_region(dev, extra_isa[i], PC87360_EXTENT,
-					 pc87360_driver.driver.name)) {
+					 DRIVER_NAME)) {
 			dev_err(dev,
 				"Region 0x%x-0x%x already in use!\n",
 				extra_isa[i], extra_isa[i]+PC87360_EXTENT-1);
@@ -1355,330 +1597,105 @@ static int pc87360_remove(struct platform_device *pdev)
 }
 
 /*
- * ldi is the logical device index
- * bank is for voltages and temperatures only
+ * Driver data
  */
-static int pc87360_read_value(struct pc87360_data *data, u8 ldi, u8 bank,
-			      u8 reg)
-{
-	int res;
-
-	mutex_lock(&(data->lock));
-	if (bank != NO_BANK)
-		outb_p(bank, data->address[ldi] + PC87365_REG_BANK);
-	res = inb_p(data->address[ldi] + reg);
-	mutex_unlock(&(data->lock));
+static struct platform_driver pc87360_driver = {
+	.driver = {
+		.name	= DRIVER_NAME,
+	},
+	.probe		= pc87360_probe,
+	.remove		= pc87360_remove,
+};
 
-	return res;
-}
+/*
+ * Device detection, registration and update
+ */
 
-static void pc87360_write_value(struct pc87360_data *data, u8 ldi, u8 bank,
-				u8 reg, u8 value)
+static int __init pc87360_find(int sioaddr, u8 *devid,
+			       unsigned short *addresses)
 {
-	mutex_lock(&(data->lock));
-	if (bank != NO_BANK)
-		outb_p(bank, data->address[ldi] + PC87365_REG_BANK);
-	outb_p(value, data->address[ldi] + reg);
-	mutex_unlock(&(data->lock));
-}
+	u16 val;
+	int i;
+	int nrdev; /* logical device count */
 
-/* (temp & vin) channel conversion status register flags (pdf sec.11.5.12) */
-#define CHAN_CNVRTD	0x80	/* new data ready */
-#define CHAN_ENA	0x01	/* enabled channel (temp or vin) */
-#define CHAN_ALM_ENA	0x10	/* propagate to alarms-reg ?? (chk val!) */
-#define CHAN_READY	(CHAN_ENA|CHAN_CNVRTD) /* sample ready mask */
+	/* No superio_enter */
 
-#define TEMP_OTS_OE	0x20	/* OTS Output Enable */
-#define VIN_RW1C_MASK	(CHAN_READY|CHAN_ALM_MAX|CHAN_ALM_MIN)   /* 0x87 */
-#define TEMP_RW1C_MASK	(VIN_RW1C_MASK|TEMP_ALM_CRIT|TEMP_FAULT) /* 0xCF */
+	/* Identify device */
+	val = force_id ? force_id : superio_inb(sioaddr, DEVID);
+	switch (val) {
+	case 0xE1: /* PC87360 */
+	case 0xE8: /* PC87363 */
+	case 0xE4: /* PC87364 */
+		nrdev = 1;
+		break;
+	case 0xE5: /* PC87365 */
+	case 0xE9: /* PC87366 */
+		nrdev = 3;
+		break;
+	default:
+		superio_exit(sioaddr);
+		return -ENODEV;
+	}
+	/* Remember the device id */
+	*devid = val;
 
-static void pc87360_init_device(struct platform_device *pdev,
-				int use_thermistors)
-{
-	struct pc87360_data *data = platform_get_drvdata(pdev);
-	int i, nr;
-	const u8 init_in[14] = { 2, 2, 2, 2, 2, 2, 2, 1, 1, 3, 1, 2, 2, 2 };
-	const u8 init_temp[3] = { 2, 2, 1 };
-	u8 reg;
+	for (i = 0; i < nrdev; i++) {
+		/* select logical device */
+		superio_outb(sioaddr, DEV, logdev[i]);
 
-	if (init >= 2 && data->innr) {
-		reg = pc87360_read_value(data, LD_IN, NO_BANK,
-					 PC87365_REG_IN_CONVRATE);
-		dev_info(&pdev->dev,
-			 "VLM conversion set to 1s period, 160us delay\n");
-		pc87360_write_value(data, LD_IN, NO_BANK,
-				    PC87365_REG_IN_CONVRATE,
-				    (reg & 0xC0) | 0x11);
-	}
+		val = superio_inb(sioaddr, ACT);
+		if (!(val & 0x01)) {
+			pr_info("Device 0x%02x not activated\n", logdev[i]);
+			continue;
+		}
 
-	nr = data->innr < 11 ? data->innr : 11;
-	for (i = 0; i < nr; i++) {
-		reg = pc87360_read_value(data, LD_IN, i,
-					 PC87365_REG_IN_STATUS);
-		dev_dbg(&pdev->dev, "bios in%d status:0x%02x\n", i, reg);
-		if (init >= init_in[i]) {
-			/* Forcibly enable voltage channel */
-			if (!(reg & CHAN_ENA)) {
-				dev_dbg(&pdev->dev, "Forcibly enabling in%d\n",
-					i);
-				pc87360_write_value(data, LD_IN, i,
-						    PC87365_REG_IN_STATUS,
-						    (reg & 0x68) | 0x87);
-			}
+		val = (superio_inb(sioaddr, BASE) << 8)
+		    | superio_inb(sioaddr, BASE + 1);
+		if (!val) {
+			pr_info("Base address not set for device 0x%02x\n",
+				logdev[i]);
+			continue;
 		}
-	}
 
-	/*
-	 * We can't blindly trust the Super-I/O space configuration bit,
-	 * most BIOS won't set it properly
-	 */
-	dev_dbg(&pdev->dev, "bios thermistors:%d\n", use_thermistors);
-	for (i = 11; i < data->innr; i++) {
-		reg = pc87360_read_value(data, LD_IN, i,
-					 PC87365_REG_TEMP_STATUS);
-		use_thermistors = use_thermistors || (reg & CHAN_ENA);
-		/* thermistors are temp[4-6], measured on vin[11-14] */
-		dev_dbg(&pdev->dev, "bios temp%d_status:0x%02x\n", i-7, reg);
-	}
-	dev_dbg(&pdev->dev, "using thermistors:%d\n", use_thermistors);
+		addresses[i] = val;
 
-	i = use_thermistors ? 2 : 0;
-	for (; i < data->tempnr; i++) {
-		reg = pc87360_read_value(data, LD_TEMP, i,
-					 PC87365_REG_TEMP_STATUS);
-		dev_dbg(&pdev->dev, "bios temp%d_status:0x%02x\n", i + 1, reg);
-		if (init >= init_temp[i]) {
-			/* Forcibly enable temperature channel */
-			if (!(reg & CHAN_ENA)) {
-				dev_dbg(&pdev->dev,
-					"Forcibly enabling temp%d\n", i + 1);
-				pc87360_write_value(data, LD_TEMP, i,
-						    PC87365_REG_TEMP_STATUS,
-						    0xCF);
-			}
-		}
-	}
+		if (i == 0) { /* Fans */
+			confreg[0] = superio_inb(sioaddr, 0xF0);
+			confreg[1] = superio_inb(sioaddr, 0xF1);
 
-	if (use_thermistors) {
-		for (i = 11; i < data->innr; i++) {
-			if (init >= init_in[i]) {
+			pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 1,
+				 (confreg[0] >> 2) & 1, (confreg[0] >> 3) & 1,
+				 (confreg[0] >> 4) & 1);
+			pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 2,
+				 (confreg[0] >> 5) & 1, (confreg[0] >> 6) & 1,
+				 (confreg[0] >> 7) & 1);
+			pr_debug("Fan %d: mon=%d ctrl=%d inv=%d\n", 3,
+				 confreg[1] & 1, (confreg[1] >> 1) & 1,
+				 (confreg[1] >> 2) & 1);
+		} else if (i == 1) { /* Voltages */
+			/* Are we using thermistors? */
+			if (*devid == 0xE9) { /* PC87366 */
 				/*
-				 * The pin may already be used by thermal
-				 * diodes
+				 * These registers are not logical-device
+				 * specific, just that we won't need them if
+				 * we don't use the VLM device
 				 */
-				reg = pc87360_read_value(data, LD_TEMP,
-				      (i - 11) / 2, PC87365_REG_TEMP_STATUS);
-				if (reg & CHAN_ENA) {
-					dev_dbg(&pdev->dev,
-			"Skipping temp%d, pin already in use by temp%d\n",
-						i - 7, (i - 11) / 2);
-					continue;
-				}
+				confreg[2] = superio_inb(sioaddr, 0x2B);
+				confreg[3] = superio_inb(sioaddr, 0x25);
 
-				/* Forcibly enable thermistor channel */
-				reg = pc87360_read_value(data, LD_IN, i,
-							 PC87365_REG_IN_STATUS);
-				if (!(reg & CHAN_ENA)) {
-					dev_dbg(&pdev->dev,
-						"Forcibly enabling temp%d\n",
-						i - 7);
-					pc87360_write_value(data, LD_IN, i,
-						PC87365_REG_TEMP_STATUS,
-						(reg & 0x60) | 0x8F);
+				if (confreg[2] & 0x40) {
+					pr_info("Using thermistors for temperature monitoring\n");
+				}
+				if (confreg[3] & 0xE0) {
+					pr_info("VID inputs routed (mode %u)\n",
+						confreg[3] >> 5);
 				}
 			}
 		}
 	}
 
-	if (data->innr) {
-		reg = pc87360_read_value(data, LD_IN, NO_BANK,
-					 PC87365_REG_IN_CONFIG);
-		dev_dbg(&pdev->dev, "bios vin-cfg:0x%02x\n", reg);
-		if (reg & CHAN_ENA) {
-			dev_dbg(&pdev->dev,
-				"Forcibly enabling monitoring (VLM)\n");
-			pc87360_write_value(data, LD_IN, NO_BANK,
-					    PC87365_REG_IN_CONFIG,
-					    reg & 0xFE);
-		}
-	}
-
-	if (data->tempnr) {
-		reg = pc87360_read_value(data, LD_TEMP, NO_BANK,
-					 PC87365_REG_TEMP_CONFIG);
-		dev_dbg(&pdev->dev, "bios temp-cfg:0x%02x\n", reg);
-		if (reg & CHAN_ENA) {
-			dev_dbg(&pdev->dev,
-				"Forcibly enabling monitoring (TMS)\n");
-			pc87360_write_value(data, LD_TEMP, NO_BANK,
-					    PC87365_REG_TEMP_CONFIG,
-					    reg & 0xFE);
-		}
-
-		if (init >= 2) {
-			/* Chip config as documented by National Semi. */
-			pc87360_write_value(data, LD_TEMP, 0xF, 0xA, 0x08);
-			/*
-			 * We voluntarily omit the bank here, in case the
-			 * sequence itself matters. It shouldn't be a problem,
-			 * since nobody else is supposed to access the
-			 * device at that point.
-			 */
-			pc87360_write_value(data, LD_TEMP, NO_BANK, 0xB, 0x04);
-			pc87360_write_value(data, LD_TEMP, NO_BANK, 0xC, 0x35);
-			pc87360_write_value(data, LD_TEMP, NO_BANK, 0xD, 0x05);
-			pc87360_write_value(data, LD_TEMP, NO_BANK, 0xE, 0x05);
-		}
-	}
-}
-
-static void pc87360_autodiv(struct device *dev, int nr)
-{
-	struct pc87360_data *data = dev_get_drvdata(dev);
-	u8 old_min = data->fan_min[nr];
-
-	/* Increase clock divider if needed and possible */
-	if ((data->fan_status[nr] & 0x04) /* overflow flag */
-	 || (data->fan[nr] >= 224)) { /* next to overflow */
-		if ((data->fan_status[nr] & 0x60) != 0x60) {
-			data->fan_status[nr] += 0x20;
-			data->fan_min[nr] >>= 1;
-			data->fan[nr] >>= 1;
-			dev_dbg(dev,
-				"Increasing clock divider to %d for fan %d\n",
-				FAN_DIV_FROM_REG(data->fan_status[nr]), nr + 1);
-		}
-	} else {
-		/* Decrease clock divider if possible */
-		while (!(data->fan_min[nr] & 0x80) /* min "nails" divider */
-		 && data->fan[nr] < 85 /* bad accuracy */
-		 && (data->fan_status[nr] & 0x60) != 0x00) {
-			data->fan_status[nr] -= 0x20;
-			data->fan_min[nr] <<= 1;
-			data->fan[nr] <<= 1;
-			dev_dbg(dev,
-				"Decreasing clock divider to %d for fan %d\n",
-				FAN_DIV_FROM_REG(data->fan_status[nr]),
-				nr + 1);
-		}
-	}
-
-	/* Write new fan min if it changed */
-	if (old_min != data->fan_min[nr]) {
-		pc87360_write_value(data, LD_FAN, NO_BANK,
-				    PC87360_REG_FAN_MIN(nr),
-				    data->fan_min[nr]);
-	}
-}
-
-static struct pc87360_data *pc87360_update_device(struct device *dev)
-{
-	struct pc87360_data *data = dev_get_drvdata(dev);
-	u8 i;
-
-	mutex_lock(&data->update_lock);
-
-	if (time_after(jiffies, data->last_updated + HZ * 2) || !data->valid) {
-		dev_dbg(dev, "Data update\n");
-
-		/* Fans */
-		for (i = 0; i < data->fannr; i++) {
-			if (FAN_CONFIG_MONITOR(data->fan_conf, i)) {
-				data->fan_status[i] =
-					pc87360_read_value(data, LD_FAN,
-					NO_BANK, PC87360_REG_FAN_STATUS(i));
-				data->fan[i] = pc87360_read_value(data, LD_FAN,
-					       NO_BANK, PC87360_REG_FAN(i));
-				data->fan_min[i] = pc87360_read_value(data,
-						   LD_FAN, NO_BANK,
-						   PC87360_REG_FAN_MIN(i));
-				/* Change clock divider if needed */
-				pc87360_autodiv(dev, i);
-				/* Clear bits and write new divider */
-				pc87360_write_value(data, LD_FAN, NO_BANK,
-						    PC87360_REG_FAN_STATUS(i),
-						    data->fan_status[i]);
-			}
-			if (FAN_CONFIG_CONTROL(data->fan_conf, i))
-				data->pwm[i] = pc87360_read_value(data, LD_FAN,
-					       NO_BANK, PC87360_REG_PWM(i));
-		}
-
-		/* Voltages */
-		for (i = 0; i < data->innr; i++) {
-			data->in_status[i] = pc87360_read_value(data, LD_IN, i,
-					     PC87365_REG_IN_STATUS);
-			/* Clear bits */
-			pc87360_write_value(data, LD_IN, i,
-					    PC87365_REG_IN_STATUS,
-					    data->in_status[i]);
-			if ((data->in_status[i] & CHAN_READY) == CHAN_READY) {
-				data->in[i] = pc87360_read_value(data, LD_IN,
-					      i, PC87365_REG_IN);
-			}
-			if (data->in_status[i] & CHAN_ENA) {
-				data->in_min[i] = pc87360_read_value(data,
-						  LD_IN, i,
-						  PC87365_REG_IN_MIN);
-				data->in_max[i] = pc87360_read_value(data,
-						  LD_IN, i,
-						  PC87365_REG_IN_MAX);
-				if (i >= 11)
-					data->in_crit[i-11] =
-						pc87360_read_value(data, LD_IN,
-						i, PC87365_REG_TEMP_CRIT);
-			}
-		}
-		if (data->innr) {
-			data->in_alarms = pc87360_read_value(data, LD_IN,
-					  NO_BANK, PC87365_REG_IN_ALARMS1)
-					| ((pc87360_read_value(data, LD_IN,
-					    NO_BANK, PC87365_REG_IN_ALARMS2)
-					    & 0x07) << 8);
-			data->vid = (data->vid_conf & 0xE0) ?
-				    pc87360_read_value(data, LD_IN,
-				    NO_BANK, PC87365_REG_VID) : 0x1F;
-		}
-
-		/* Temperatures */
-		for (i = 0; i < data->tempnr; i++) {
-			data->temp_status[i] = pc87360_read_value(data,
-					       LD_TEMP, i,
-					       PC87365_REG_TEMP_STATUS);
-			/* Clear bits */
-			pc87360_write_value(data, LD_TEMP, i,
-					    PC87365_REG_TEMP_STATUS,
-					    data->temp_status[i]);
-			if ((data->temp_status[i] & CHAN_READY) == CHAN_READY) {
-				data->temp[i] = pc87360_read_value(data,
-						LD_TEMP, i,
-						PC87365_REG_TEMP);
-			}
-			if (data->temp_status[i] & CHAN_ENA) {
-				data->temp_min[i] = pc87360_read_value(data,
-						    LD_TEMP, i,
-						    PC87365_REG_TEMP_MIN);
-				data->temp_max[i] = pc87360_read_value(data,
-						    LD_TEMP, i,
-						    PC87365_REG_TEMP_MAX);
-				data->temp_crit[i] = pc87360_read_value(data,
-						     LD_TEMP, i,
-						     PC87365_REG_TEMP_CRIT);
-			}
-		}
-		if (data->tempnr) {
-			data->temp_alarms = pc87360_read_value(data, LD_TEMP,
-					    NO_BANK, PC87365_REG_TEMP_ALARMS)
-					    & 0x3F;
-		}
-
-		data->last_updated = jiffies;
-		data->valid = true;
-	}
-
-	mutex_unlock(&data->update_lock);
-
-	return data;
+	superio_exit(sioaddr);
+	return 0;
 }
 
 static int __init pc87360_device_add(unsigned short address)
@@ -1777,10 +1794,10 @@ static void __exit pc87360_exit(void)
 	platform_driver_unregister(&pc87360_driver);
 }
 
-
 MODULE_AUTHOR("Jean Delvare <jdelvare@suse.de>");
 MODULE_DESCRIPTION("PC8736x hardware monitor");
 MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:" DRIVER_NAME);
 
 module_init(pc87360_init);
 module_exit(pc87360_exit);
diff --git a/drivers/hwmon/pcf8591.c b/drivers/hwmon/pcf8591.c
index a97a51005c61..af9614e918a4 100644
--- a/drivers/hwmon/pcf8591.c
+++ b/drivers/hwmon/pcf8591.c
@@ -228,14 +228,13 @@ exit_sysfs_remove:
 	return err;
 }
 
-static int pcf8591_remove(struct i2c_client *client)
+static void pcf8591_remove(struct i2c_client *client)
 {
 	struct pcf8591_data *data = i2c_get_clientdata(client);
 
 	hwmon_device_unregister(data->hwmon_dev);
 	sysfs_remove_group(&client->dev.kobj, &pcf8591_attr_group_opt);
 	sysfs_remove_group(&client->dev.kobj, &pcf8591_attr_group);
-	return 0;
 }
 
 /* Called when we have found a new PCF8591. */
diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig
index 951e4a9ff2d6..89668af67206 100644
--- a/drivers/hwmon/pmbus/Kconfig
+++ b/drivers/hwmon/pmbus/Kconfig
@@ -397,6 +397,15 @@ config SENSORS_TPS53679
 	  This driver can also be built as a module. If so, the module will
 	  be called tps53679.
 
+config SENSORS_TPS546D24
+	tristate "TPS546D24"
+	help
+	  If you say yes here you get hardware monitoring support for TEXAS
+	  TPS546D24.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called tps546d24
+
 config SENSORS_UCD9000
 	tristate "TI UCD90120, UCD90124, UCD90160, UCD90320, UCD9090, UCD90910"
 	help
diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile
index e2fe86f98965..0002dbe22d52 100644
--- a/drivers/hwmon/pmbus/Makefile
+++ b/drivers/hwmon/pmbus/Makefile
@@ -41,6 +41,7 @@ obj-$(CONFIG_SENSORS_Q54SJ108A2)	+= q54sj108a2.o
 obj-$(CONFIG_SENSORS_STPDDC60)	+= stpddc60.o
 obj-$(CONFIG_SENSORS_TPS40422)	+= tps40422.o
 obj-$(CONFIG_SENSORS_TPS53679)	+= tps53679.o
+obj-$(CONFIG_SENSORS_TPS546D24)	+= tps546d24.o
 obj-$(CONFIG_SENSORS_UCD9000)	+= ucd9000.o
 obj-$(CONFIG_SENSORS_UCD9200)	+= ucd9200.o
 obj-$(CONFIG_SENSORS_XDPE122)	+= xdpe12284.o
diff --git a/drivers/hwmon/pmbus/mp2888.c b/drivers/hwmon/pmbus/mp2888.c
index 8ecd4adfef40..24e5194706cf 100644
--- a/drivers/hwmon/pmbus/mp2888.c
+++ b/drivers/hwmon/pmbus/mp2888.c
@@ -34,7 +34,7 @@ struct mp2888_data {
 	int curr_sense_gain;
 };
 
-#define to_mp2888_data(x)  container_of(x, struct mp2888_data, info)
+#define to_mp2888_data(x)	container_of(x, struct mp2888_data, info)
 
 static int mp2888_read_byte_data(struct i2c_client *client, int page, int reg)
 {
@@ -109,7 +109,7 @@ mp2888_read_phase(struct i2c_client *client, struct mp2888_data *data, int page,
 	 * - Kcs is the DrMOS current sense gain of power stage, which is obtained from the
 	 *   register MP2888_MFR_VR_CONFIG1, bits 13-12 with the following selection of DrMOS
 	 *   (data->curr_sense_gain):
-	 *   00b - 5µA/A, 01b - 8.5µA/A, 10b - 9.7µA/A, 11b - 10µA/A.
+	 *   00b - 8.5µA/A, 01b - 9.7µA/A, 1b - 10µA/A, 11b - 5µA/A.
 	 * - Rcs is the internal phase current sense resistor. This parameter depends on hardware
 	 *   assembly. By default it is set to 1kΩ. In case of different assembly, user should
 	 *   scale this parameter by dividing it by Rcs.
@@ -118,10 +118,9 @@ mp2888_read_phase(struct i2c_client *client, struct mp2888_data *data, int page,
 	 * because sampling of current occurrence of bit weight has a big deviation, especially for
 	 * light load.
 	 */
-	ret = DIV_ROUND_CLOSEST(ret * 100 - 9800, data->curr_sense_gain);
-	ret = (data->phase_curr_resolution) ? ret * 2 : ret;
+	ret = DIV_ROUND_CLOSEST(ret * 200 - 19600, data->curr_sense_gain);
 	/* Scale according to total current resolution. */
-	ret = (data->total_curr_resolution) ? ret * 8 : ret * 4;
+	ret = (data->total_curr_resolution) ? ret * 2 : ret;
 	return ret;
 }
 
@@ -212,7 +211,7 @@ static int mp2888_read_word_data(struct i2c_client *client, int page, int phase,
 		ret = pmbus_read_word_data(client, page, phase, reg);
 		if (ret < 0)
 			return ret;
-		ret = data->total_curr_resolution ? ret * 2 : ret;
+		ret = data->total_curr_resolution ? ret : DIV_ROUND_CLOSEST(ret, 2);
 		break;
 	case PMBUS_POUT_OP_WARN_LIMIT:
 		ret = pmbus_read_word_data(client, page, phase, reg);
@@ -223,7 +222,7 @@ static int mp2888_read_word_data(struct i2c_client *client, int page, int phase,
 		 * set 1. Actual power is reported with 0.5W or 1W respectively resolution. Scaling
 		 * is needed to match both.
 		 */
-		ret = data->total_curr_resolution ? ret * 4 : ret * 2;
+		ret = data->total_curr_resolution ? ret * 2 : ret;
 		break;
 	/*
 	 * The below registers are not implemented by device or implemented not according to the
diff --git a/drivers/hwmon/pmbus/pmbus_core.c b/drivers/hwmon/pmbus/pmbus_core.c
index 81d3f91dd204..7ec04934747e 100644
--- a/drivers/hwmon/pmbus/pmbus_core.c
+++ b/drivers/hwmon/pmbus/pmbus_core.c
@@ -1270,9 +1270,9 @@ struct pmbus_thermal_data {
 	struct pmbus_sensor *sensor;
 };
 
-static int pmbus_thermal_get_temp(void *data, int *temp)
+static int pmbus_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct pmbus_thermal_data *tdata = data;
+	struct pmbus_thermal_data *tdata = tz->devdata;
 	struct pmbus_sensor *sensor = tdata->sensor;
 	struct pmbus_data *pmbus_data = tdata->pmbus_data;
 	struct i2c_client *client = to_i2c_client(pmbus_data->dev);
@@ -1296,7 +1296,7 @@ static int pmbus_thermal_get_temp(void *data, int *temp)
 	return ret;
 }
 
-static const struct thermal_zone_of_device_ops pmbus_thermal_ops = {
+static const struct thermal_zone_device_ops pmbus_thermal_ops = {
 	.get_temp = pmbus_thermal_get_temp,
 };
 
@@ -1314,8 +1314,8 @@ static int pmbus_thermal_add_sensor(struct pmbus_data *pmbus_data,
 	tdata->sensor = sensor;
 	tdata->pmbus_data = pmbus_data;
 
-	tzd = devm_thermal_zone_of_sensor_register(dev, index, tdata,
-						   &pmbus_thermal_ops);
+	tzd = devm_thermal_of_zone_register(dev, index, tdata,
+					    &pmbus_thermal_ops);
 	/*
 	 * If CONFIG_THERMAL_OF is disabled, this returns -ENODEV,
 	 * so ignore that error but forward any other error.
diff --git a/drivers/hwmon/pmbus/tps546d24.c b/drivers/hwmon/pmbus/tps546d24.c
new file mode 100644
index 000000000000..435f94304ad8
--- /dev/null
+++ b/drivers/hwmon/pmbus/tps546d24.c
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Hardware monitoring driver for TEXAS TPS546D24 buck converter
+ */
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pmbus.h>
+#include "pmbus.h"
+
+static struct pmbus_driver_info tps546d24_info = {
+	.pages = 1,
+	.format[PSC_VOLTAGE_IN] = linear,
+	.format[PSC_VOLTAGE_OUT] = linear,
+	.format[PSC_TEMPERATURE] = linear,
+	.format[PSC_CURRENT_OUT] = linear,
+	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_IIN
+			| PMBUS_HAVE_IOUT | PMBUS_HAVE_VOUT
+			| PMBUS_HAVE_STATUS_IOUT | PMBUS_HAVE_STATUS_VOUT
+			| PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
+};
+
+static int tps546d24_probe(struct i2c_client *client)
+{
+	int reg;
+
+	reg = i2c_smbus_read_byte_data(client, PMBUS_VOUT_MODE);
+	if (reg < 0)
+		return reg;
+
+	if (reg & 0x80) {
+		int err;
+
+		err = i2c_smbus_write_byte_data(client, PMBUS_VOUT_MODE, reg & 0x7f);
+		if (err < 0)
+			return err;
+	}
+	return pmbus_do_probe(client, &tps546d24_info);
+}
+
+static const struct i2c_device_id tps546d24_id[] = {
+	{"tps546d24", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, tps546d24_id);
+
+static const struct of_device_id __maybe_unused tps546d24_of_match[] = {
+	{.compatible = "ti,tps546d24"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, tps546d24_of_match);
+
+/* This is the driver that will be inserted */
+static struct i2c_driver tps546d24_driver = {
+	.driver = {
+		   .name = "tps546d24",
+		   .of_match_table = of_match_ptr(tps546d24_of_match),
+	   },
+	.probe_new = tps546d24_probe,
+	.id_table = tps546d24_id,
+};
+
+module_i2c_driver(tps546d24_driver);
+
+MODULE_AUTHOR("Duke Du <dukedu83@gmail.com>");
+MODULE_DESCRIPTION("PMBus driver for TI tps546d24");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS(PMBUS);
diff --git a/drivers/hwmon/pwm-fan.c b/drivers/hwmon/pwm-fan.c
index 6c08551d8d14..dc3d9a22d917 100644
--- a/drivers/hwmon/pwm-fan.c
+++ b/drivers/hwmon/pwm-fan.c
@@ -28,11 +28,23 @@ struct pwm_fan_tach {
 	u8 pulses_per_revolution;
 };
 
+enum pwm_fan_enable_mode {
+	pwm_off_reg_off,
+	pwm_disable_reg_enable,
+	pwm_enable_reg_enable,
+	pwm_disable_reg_disable,
+};
+
 struct pwm_fan_ctx {
+	struct device *dev;
+
 	struct mutex lock;
 	struct pwm_device *pwm;
 	struct pwm_state pwm_state;
 	struct regulator *reg_en;
+	enum pwm_fan_enable_mode enable_mode;
+	bool regulator_enabled;
+	bool enabled;
 
 	int tach_count;
 	struct pwm_fan_tach *tachs;
@@ -82,25 +94,140 @@ static void sample_timer(struct timer_list *t)
 	mod_timer(&ctx->rpm_timer, jiffies + HZ);
 }
 
-static int  __set_pwm(struct pwm_fan_ctx *ctx, unsigned long pwm)
+static void pwm_fan_enable_mode_2_state(int enable_mode,
+					struct pwm_state *state,
+					bool *enable_regulator)
+{
+	switch (enable_mode) {
+	case pwm_disable_reg_enable:
+		/* disable pwm, keep regulator enabled */
+		state->enabled = false;
+		*enable_regulator = true;
+		break;
+	case pwm_enable_reg_enable:
+		/* keep pwm and regulator enabled */
+		state->enabled = true;
+		*enable_regulator = true;
+		break;
+	case pwm_off_reg_off:
+	case pwm_disable_reg_disable:
+		/* disable pwm and regulator */
+		state->enabled = false;
+		*enable_regulator = false;
+	}
+}
+
+static int pwm_fan_switch_power(struct pwm_fan_ctx *ctx, bool on)
 {
-	unsigned long period;
 	int ret = 0;
+
+	if (!ctx->reg_en)
+		return ret;
+
+	if (!ctx->regulator_enabled && on) {
+		ret = regulator_enable(ctx->reg_en);
+		if (ret == 0)
+			ctx->regulator_enabled = true;
+	} else if (ctx->regulator_enabled && !on) {
+		ret = regulator_disable(ctx->reg_en);
+		if (ret == 0)
+			ctx->regulator_enabled = false;
+	}
+	return ret;
+}
+
+static int pwm_fan_power_on(struct pwm_fan_ctx *ctx)
+{
 	struct pwm_state *state = &ctx->pwm_state;
+	int ret;
 
-	mutex_lock(&ctx->lock);
-	if (ctx->pwm_value == pwm)
-		goto exit_set_pwm_err;
+	if (ctx->enabled)
+		return 0;
+
+	ret = pwm_fan_switch_power(ctx, true);
+	if (ret < 0) {
+		dev_err(ctx->dev, "failed to enable power supply\n");
+		return ret;
+	}
+
+	state->enabled = true;
+	ret = pwm_apply_state(ctx->pwm, state);
+	if (ret) {
+		dev_err(ctx->dev, "failed to enable PWM\n");
+		goto disable_regulator;
+	}
+
+	ctx->enabled = true;
+
+	return 0;
+
+disable_regulator:
+	pwm_fan_switch_power(ctx, false);
+	return ret;
+}
 
-	period = state->period;
-	state->duty_cycle = DIV_ROUND_UP(pwm * (period - 1), MAX_PWM);
-	state->enabled = pwm ? true : false;
+static int pwm_fan_power_off(struct pwm_fan_ctx *ctx)
+{
+	struct pwm_state *state = &ctx->pwm_state;
+	bool enable_regulator = false;
+	int ret;
 
+	if (!ctx->enabled)
+		return 0;
+
+	pwm_fan_enable_mode_2_state(ctx->enable_mode,
+				    state,
+				    &enable_regulator);
+
+	state->enabled = false;
+	state->duty_cycle = 0;
 	ret = pwm_apply_state(ctx->pwm, state);
+	if (ret) {
+		dev_err(ctx->dev, "failed to disable PWM\n");
+		return ret;
+	}
+
+	pwm_fan_switch_power(ctx, enable_regulator);
+
+	ctx->enabled = false;
+
+	return 0;
+}
+
+static int  __set_pwm(struct pwm_fan_ctx *ctx, unsigned long pwm)
+{
+	struct pwm_state *state = &ctx->pwm_state;
+	unsigned long period;
+	int ret = 0;
+
+	if (pwm > 0) {
+		if (ctx->enable_mode == pwm_off_reg_off)
+			/* pwm-fan hard disabled */
+			return 0;
+
+		period = state->period;
+		state->duty_cycle = DIV_ROUND_UP(pwm * (period - 1), MAX_PWM);
+		ret = pwm_apply_state(ctx->pwm, state);
+		if (ret)
+			return ret;
+		ret = pwm_fan_power_on(ctx);
+	} else {
+		ret = pwm_fan_power_off(ctx);
+	}
 	if (!ret)
 		ctx->pwm_value = pwm;
-exit_set_pwm_err:
+
+	return ret;
+}
+
+static int set_pwm(struct pwm_fan_ctx *ctx, unsigned long pwm)
+{
+	int ret;
+
+	mutex_lock(&ctx->lock);
+	ret = __set_pwm(ctx, pwm);
 	mutex_unlock(&ctx->lock);
+
 	return ret;
 }
 
@@ -115,20 +242,76 @@ static void pwm_fan_update_state(struct pwm_fan_ctx *ctx, unsigned long pwm)
 	ctx->pwm_fan_state = i;
 }
 
+static int pwm_fan_update_enable(struct pwm_fan_ctx *ctx, long val)
+{
+	int ret = 0;
+	int old_val;
+
+	mutex_lock(&ctx->lock);
+
+	if (ctx->enable_mode == val)
+		goto out;
+
+	old_val = ctx->enable_mode;
+	ctx->enable_mode = val;
+
+	if (val == 0) {
+		/* Disable pwm-fan unconditionally */
+		ret = __set_pwm(ctx, 0);
+		if (ret)
+			ctx->enable_mode = old_val;
+		pwm_fan_update_state(ctx, 0);
+	} else {
+		/*
+		 * Change PWM and/or regulator state if currently disabled
+		 * Nothing to do if currently enabled
+		 */
+		if (!ctx->enabled) {
+			struct pwm_state *state = &ctx->pwm_state;
+			bool enable_regulator = false;
+
+			state->duty_cycle = 0;
+			pwm_fan_enable_mode_2_state(val,
+						    state,
+						    &enable_regulator);
+
+			pwm_apply_state(ctx->pwm, state);
+			pwm_fan_switch_power(ctx, enable_regulator);
+			pwm_fan_update_state(ctx, 0);
+		}
+	}
+out:
+	mutex_unlock(&ctx->lock);
+
+	return ret;
+}
+
 static int pwm_fan_write(struct device *dev, enum hwmon_sensor_types type,
 			 u32 attr, int channel, long val)
 {
 	struct pwm_fan_ctx *ctx = dev_get_drvdata(dev);
 	int ret;
 
-	if (val < 0 || val > MAX_PWM)
-		return -EINVAL;
+	switch (attr) {
+	case hwmon_pwm_input:
+		if (val < 0 || val > MAX_PWM)
+			return -EINVAL;
+		ret = set_pwm(ctx, val);
+		if (ret)
+			return ret;
+		pwm_fan_update_state(ctx, val);
+		break;
+	case hwmon_pwm_enable:
+		if (val < 0 || val > 3)
+			ret = -EINVAL;
+		else
+			ret = pwm_fan_update_enable(ctx, val);
 
-	ret = __set_pwm(ctx, val);
-	if (ret)
 		return ret;
+	default:
+		return -EOPNOTSUPP;
+	}
 
-	pwm_fan_update_state(ctx, val);
 	return 0;
 }
 
@@ -139,9 +322,15 @@ static int pwm_fan_read(struct device *dev, enum hwmon_sensor_types type,
 
 	switch (type) {
 	case hwmon_pwm:
-		*val = ctx->pwm_value;
-		return 0;
-
+		switch (attr) {
+		case hwmon_pwm_input:
+			*val = ctx->pwm_value;
+			return 0;
+		case hwmon_pwm_enable:
+			*val = ctx->enable_mode;
+			return 0;
+		}
+		return -EOPNOTSUPP;
 	case hwmon_fan:
 		*val = ctx->tachs[channel].rpm;
 		return 0;
@@ -212,7 +401,7 @@ pwm_fan_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state)
 	if (state == ctx->pwm_fan_state)
 		return 0;
 
-	ret = __set_pwm(ctx, ctx->pwm_fan_cooling_levels[state]);
+	ret = set_pwm(ctx, ctx->pwm_fan_cooling_levels[state]);
 	if (ret) {
 		dev_err(&cdev->device, "Cannot set pwm!\n");
 		return ret;
@@ -270,18 +459,14 @@ static int pwm_fan_of_get_cooling_data(struct device *dev,
 	return 0;
 }
 
-static void pwm_fan_regulator_disable(void *data)
-{
-	regulator_disable(data);
-}
-
-static void pwm_fan_pwm_disable(void *__ctx)
+static void pwm_fan_cleanup(void *__ctx)
 {
 	struct pwm_fan_ctx *ctx = __ctx;
 
-	ctx->pwm_state.enabled = false;
-	pwm_apply_state(ctx->pwm, &ctx->pwm_state);
 	del_timer_sync(&ctx->rpm_timer);
+	/* Switch off everything */
+	ctx->enable_mode = pwm_disable_reg_disable;
+	pwm_fan_power_off(ctx);
 }
 
 static int pwm_fan_probe(struct platform_device *pdev)
@@ -302,7 +487,8 @@ static int pwm_fan_probe(struct platform_device *pdev)
 
 	mutex_init(&ctx->lock);
 
-	ctx->pwm = devm_of_pwm_get(dev, dev->of_node, NULL);
+	ctx->dev = &pdev->dev;
+	ctx->pwm = devm_pwm_get(dev, NULL);
 	if (IS_ERR(ctx->pwm))
 		return dev_err_probe(dev, PTR_ERR(ctx->pwm), "Could not get PWM\n");
 
@@ -314,22 +500,12 @@ static int pwm_fan_probe(struct platform_device *pdev)
 			return PTR_ERR(ctx->reg_en);
 
 		ctx->reg_en = NULL;
-	} else {
-		ret = regulator_enable(ctx->reg_en);
-		if (ret) {
-			dev_err(dev, "Failed to enable fan supply: %d\n", ret);
-			return ret;
-		}
-		ret = devm_add_action_or_reset(dev, pwm_fan_regulator_disable,
-					       ctx->reg_en);
-		if (ret)
-			return ret;
 	}
 
 	pwm_init_state(ctx->pwm, &ctx->pwm_state);
 
 	/*
-	 * __set_pwm assumes that MAX_PWM * (period - 1) fits into an unsigned
+	 * set_pwm assumes that MAX_PWM * (period - 1) fits into an unsigned
 	 * long. Check this here to prevent the fan running at a too low
 	 * frequency.
 	 */
@@ -338,14 +514,19 @@ static int pwm_fan_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	/* Set duty cycle to maximum allowed and enable PWM output */
-	ret = __set_pwm(ctx, MAX_PWM);
+	ctx->enable_mode = pwm_disable_reg_enable;
+
+	/*
+	 * Set duty cycle to maximum allowed and enable PWM output as well as
+	 * the regulator. In case of error nothing is changed
+	 */
+	ret = set_pwm(ctx, MAX_PWM);
 	if (ret) {
 		dev_err(dev, "Failed to configure PWM: %d\n", ret);
 		return ret;
 	}
 	timer_setup(&ctx->rpm_timer, sample_timer, 0);
-	ret = devm_add_action_or_reset(dev, pwm_fan_pwm_disable, ctx);
+	ret = devm_add_action_or_reset(dev, pwm_fan_cleanup, ctx);
 	if (ret)
 		return ret;
 
@@ -377,7 +558,7 @@ static int pwm_fan_probe(struct platform_device *pdev)
 	if (!channels)
 		return -ENOMEM;
 
-	channels[0] = HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT);
+	channels[0] = HWMON_CHANNEL_INFO(pwm, HWMON_PWM_INPUT | HWMON_PWM_ENABLE);
 
 	for (i = 0; i < ctx->tach_count; i++) {
 		struct pwm_fan_tach *tach = &ctx->tachs[i];
@@ -451,65 +632,28 @@ static int pwm_fan_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int pwm_fan_disable(struct device *dev)
-{
-	struct pwm_fan_ctx *ctx = dev_get_drvdata(dev);
-	int ret;
-
-	if (ctx->pwm_value) {
-		/* keep ctx->pwm_state unmodified for pwm_fan_resume() */
-		struct pwm_state state = ctx->pwm_state;
-
-		state.duty_cycle = 0;
-		state.enabled = false;
-		ret = pwm_apply_state(ctx->pwm, &state);
-		if (ret < 0)
-			return ret;
-	}
-
-	if (ctx->reg_en) {
-		ret = regulator_disable(ctx->reg_en);
-		if (ret) {
-			dev_err(dev, "Failed to disable fan supply: %d\n", ret);
-			return ret;
-		}
-	}
-
-	return 0;
-}
-
 static void pwm_fan_shutdown(struct platform_device *pdev)
 {
-	pwm_fan_disable(&pdev->dev);
+	struct pwm_fan_ctx *ctx = platform_get_drvdata(pdev);
+
+	pwm_fan_cleanup(ctx);
 }
 
-#ifdef CONFIG_PM_SLEEP
 static int pwm_fan_suspend(struct device *dev)
 {
-	return pwm_fan_disable(dev);
+	struct pwm_fan_ctx *ctx = dev_get_drvdata(dev);
+
+	return pwm_fan_power_off(ctx);
 }
 
 static int pwm_fan_resume(struct device *dev)
 {
 	struct pwm_fan_ctx *ctx = dev_get_drvdata(dev);
-	int ret;
-
-	if (ctx->reg_en) {
-		ret = regulator_enable(ctx->reg_en);
-		if (ret) {
-			dev_err(dev, "Failed to enable fan supply: %d\n", ret);
-			return ret;
-		}
-	}
-
-	if (ctx->pwm_value == 0)
-		return 0;
 
-	return pwm_apply_state(ctx->pwm, &ctx->pwm_state);
+	return set_pwm(ctx, ctx->pwm_value);
 }
-#endif
 
-static SIMPLE_DEV_PM_OPS(pwm_fan_pm, pwm_fan_suspend, pwm_fan_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(pwm_fan_pm, pwm_fan_suspend, pwm_fan_resume);
 
 static const struct of_device_id of_pwm_fan_match[] = {
 	{ .compatible = "pwm-fan", },
@@ -522,7 +666,7 @@ static struct platform_driver pwm_fan_driver = {
 	.shutdown	= pwm_fan_shutdown,
 	.driver	= {
 		.name		= "pwm-fan",
-		.pm		= &pwm_fan_pm,
+		.pm		= pm_sleep_ptr(&pwm_fan_pm),
 		.of_match_table	= of_pwm_fan_match,
 	},
 };
diff --git a/drivers/hwmon/scpi-hwmon.c b/drivers/hwmon/scpi-hwmon.c
index 5187c6dd5a4f..4d75385f7d5e 100644
--- a/drivers/hwmon/scpi-hwmon.c
+++ b/drivers/hwmon/scpi-hwmon.c
@@ -62,9 +62,9 @@ static void scpi_scale_reading(u64 *value, struct sensor_data *sensor)
 	}
 }
 
-static int scpi_read_temp(void *dev, int *temp)
+static int scpi_read_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct scpi_thermal_zone *zone = dev;
+	struct scpi_thermal_zone *zone = tz->devdata;
 	struct scpi_sensors *scpi_sensors = zone->scpi_sensors;
 	struct scpi_ops *scpi_ops = scpi_sensors->scpi_ops;
 	struct sensor_data *sensor = &scpi_sensors->data[zone->sensor_id];
@@ -121,7 +121,7 @@ scpi_show_label(struct device *dev, struct device_attribute *attr, char *buf)
 	return sprintf(buf, "%s\n", sensor->info.name);
 }
 
-static const struct thermal_zone_of_device_ops scpi_sensor_ops = {
+static const struct thermal_zone_device_ops scpi_sensor_ops = {
 	.get_temp = scpi_read_temp,
 };
 
@@ -275,10 +275,10 @@ static int scpi_hwmon_probe(struct platform_device *pdev)
 
 		zone->sensor_id = i;
 		zone->scpi_sensors = scpi_sensors;
-		z = devm_thermal_zone_of_sensor_register(dev,
-							 sensor->info.sensor_id,
-							 zone,
-							 &scpi_sensor_ops);
+		z = devm_thermal_of_zone_register(dev,
+						  sensor->info.sensor_id,
+						  zone,
+						  &scpi_sensor_ops);
 		/*
 		 * The call to thermal_zone_of_sensor_register returns
 		 * an error for sensors that are not associated with
diff --git a/drivers/hwmon/sht4x.c b/drivers/hwmon/sht4x.c
index c19df3ade48e..13ac2d8f22c7 100644
--- a/drivers/hwmon/sht4x.c
+++ b/drivers/hwmon/sht4x.c
@@ -129,7 +129,7 @@ unlock:
 
 static ssize_t sht4x_interval_write(struct sht4x_data *data, long val)
 {
-	data->update_interval = clamp_val(val, SHT4X_MIN_POLL_INTERVAL, UINT_MAX);
+	data->update_interval = clamp_val(val, SHT4X_MIN_POLL_INTERVAL, INT_MAX);
 
 	return 0;
 }
diff --git a/drivers/hwmon/sis5595.c b/drivers/hwmon/sis5595.c
index 018cb5a7651f..b0b05fd12221 100644
--- a/drivers/hwmon/sis5595.c
+++ b/drivers/hwmon/sis5595.c
@@ -37,6 +37,7 @@
  *	 735		0008		0735
  */
 
+#define DRIVER_NAME "sis5595"
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/module.h>
@@ -191,21 +192,75 @@ struct sis5595_data {
 
 static struct pci_dev *s_bridge;	/* pointer to the (only) sis5595 */
 
-static int sis5595_probe(struct platform_device *pdev);
-static int sis5595_remove(struct platform_device *pdev);
+/* ISA access must be locked explicitly. */
+static int sis5595_read_value(struct sis5595_data *data, u8 reg)
+{
+	int res;
 
-static int sis5595_read_value(struct sis5595_data *data, u8 reg);
-static void sis5595_write_value(struct sis5595_data *data, u8 reg, u8 value);
-static struct sis5595_data *sis5595_update_device(struct device *dev);
-static void sis5595_init_device(struct sis5595_data *data);
+	mutex_lock(&data->lock);
+	outb_p(reg, data->addr + SIS5595_ADDR_REG_OFFSET);
+	res = inb_p(data->addr + SIS5595_DATA_REG_OFFSET);
+	mutex_unlock(&data->lock);
+	return res;
+}
 
-static struct platform_driver sis5595_driver = {
-	.driver = {
-		.name	= "sis5595",
-	},
-	.probe		= sis5595_probe,
-	.remove		= sis5595_remove,
-};
+static void sis5595_write_value(struct sis5595_data *data, u8 reg, u8 value)
+{
+	mutex_lock(&data->lock);
+	outb_p(reg, data->addr + SIS5595_ADDR_REG_OFFSET);
+	outb_p(value, data->addr + SIS5595_DATA_REG_OFFSET);
+	mutex_unlock(&data->lock);
+}
+
+static struct sis5595_data *sis5595_update_device(struct device *dev)
+{
+	struct sis5595_data *data = dev_get_drvdata(dev);
+	int i;
+
+	mutex_lock(&data->update_lock);
+
+	if (time_after(jiffies, data->last_updated + HZ + HZ / 2)
+	    || !data->valid) {
+
+		for (i = 0; i <= data->maxins; i++) {
+			data->in[i] =
+			    sis5595_read_value(data, SIS5595_REG_IN(i));
+			data->in_min[i] =
+			    sis5595_read_value(data,
+					       SIS5595_REG_IN_MIN(i));
+			data->in_max[i] =
+			    sis5595_read_value(data,
+					       SIS5595_REG_IN_MAX(i));
+		}
+		for (i = 0; i < 2; i++) {
+			data->fan[i] =
+			    sis5595_read_value(data, SIS5595_REG_FAN(i));
+			data->fan_min[i] =
+			    sis5595_read_value(data,
+					       SIS5595_REG_FAN_MIN(i));
+		}
+		if (data->maxins == 3) {
+			data->temp =
+			    sis5595_read_value(data, SIS5595_REG_TEMP);
+			data->temp_over =
+			    sis5595_read_value(data, SIS5595_REG_TEMP_OVER);
+			data->temp_hyst =
+			    sis5595_read_value(data, SIS5595_REG_TEMP_HYST);
+		}
+		i = sis5595_read_value(data, SIS5595_REG_FANDIV);
+		data->fan_div[0] = (i >> 4) & 0x03;
+		data->fan_div[1] = i >> 6;
+		data->alarms =
+		    sis5595_read_value(data, SIS5595_REG_ALARM1) |
+		    (sis5595_read_value(data, SIS5595_REG_ALARM2) << 8);
+		data->last_updated = jiffies;
+		data->valid = true;
+	}
+
+	mutex_unlock(&data->update_lock);
+
+	return data;
+}
 
 /* 4 Voltages */
 static ssize_t in_show(struct device *dev, struct device_attribute *da,
@@ -568,6 +623,15 @@ static const struct attribute_group sis5595_group_temp1 = {
 	.attrs = sis5595_attributes_temp1,
 };
 
+/* Called when we have found a new SIS5595. */
+static void sis5595_init_device(struct sis5595_data *data)
+{
+	u8 config = sis5595_read_value(data, SIS5595_REG_CONFIG);
+	if (!(config & 0x01))
+		sis5595_write_value(data, SIS5595_REG_CONFIG,
+				(config & 0xf7) | 0x01);
+}
+
 /* This is called when the module is loaded */
 static int sis5595_probe(struct platform_device *pdev)
 {
@@ -580,7 +644,7 @@ static int sis5595_probe(struct platform_device *pdev)
 	/* Reserve the ISA region */
 	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
 	if (!devm_request_region(&pdev->dev, res->start, SIS5595_EXTENT,
-				 sis5595_driver.driver.name))
+				 DRIVER_NAME))
 		return -EBUSY;
 
 	data = devm_kzalloc(&pdev->dev, sizeof(struct sis5595_data),
@@ -591,7 +655,7 @@ static int sis5595_probe(struct platform_device *pdev)
 	mutex_init(&data->lock);
 	mutex_init(&data->update_lock);
 	data->addr = res->start;
-	data->name = "sis5595";
+	data->name = DRIVER_NAME;
 	platform_set_drvdata(pdev, data);
 
 	/*
@@ -657,85 +721,6 @@ static int sis5595_remove(struct platform_device *pdev)
 	return 0;
 }
 
-/* ISA access must be locked explicitly. */
-static int sis5595_read_value(struct sis5595_data *data, u8 reg)
-{
-	int res;
-
-	mutex_lock(&data->lock);
-	outb_p(reg, data->addr + SIS5595_ADDR_REG_OFFSET);
-	res = inb_p(data->addr + SIS5595_DATA_REG_OFFSET);
-	mutex_unlock(&data->lock);
-	return res;
-}
-
-static void sis5595_write_value(struct sis5595_data *data, u8 reg, u8 value)
-{
-	mutex_lock(&data->lock);
-	outb_p(reg, data->addr + SIS5595_ADDR_REG_OFFSET);
-	outb_p(value, data->addr + SIS5595_DATA_REG_OFFSET);
-	mutex_unlock(&data->lock);
-}
-
-/* Called when we have found a new SIS5595. */
-static void sis5595_init_device(struct sis5595_data *data)
-{
-	u8 config = sis5595_read_value(data, SIS5595_REG_CONFIG);
-	if (!(config & 0x01))
-		sis5595_write_value(data, SIS5595_REG_CONFIG,
-				(config & 0xf7) | 0x01);
-}
-
-static struct sis5595_data *sis5595_update_device(struct device *dev)
-{
-	struct sis5595_data *data = dev_get_drvdata(dev);
-	int i;
-
-	mutex_lock(&data->update_lock);
-
-	if (time_after(jiffies, data->last_updated + HZ + HZ / 2)
-	    || !data->valid) {
-
-		for (i = 0; i <= data->maxins; i++) {
-			data->in[i] =
-			    sis5595_read_value(data, SIS5595_REG_IN(i));
-			data->in_min[i] =
-			    sis5595_read_value(data,
-					       SIS5595_REG_IN_MIN(i));
-			data->in_max[i] =
-			    sis5595_read_value(data,
-					       SIS5595_REG_IN_MAX(i));
-		}
-		for (i = 0; i < 2; i++) {
-			data->fan[i] =
-			    sis5595_read_value(data, SIS5595_REG_FAN(i));
-			data->fan_min[i] =
-			    sis5595_read_value(data,
-					       SIS5595_REG_FAN_MIN(i));
-		}
-		if (data->maxins == 3) {
-			data->temp =
-			    sis5595_read_value(data, SIS5595_REG_TEMP);
-			data->temp_over =
-			    sis5595_read_value(data, SIS5595_REG_TEMP_OVER);
-			data->temp_hyst =
-			    sis5595_read_value(data, SIS5595_REG_TEMP_HYST);
-		}
-		i = sis5595_read_value(data, SIS5595_REG_FANDIV);
-		data->fan_div[0] = (i >> 4) & 0x03;
-		data->fan_div[1] = i >> 6;
-		data->alarms =
-		    sis5595_read_value(data, SIS5595_REG_ALARM1) |
-		    (sis5595_read_value(data, SIS5595_REG_ALARM2) << 8);
-		data->last_updated = jiffies;
-		data->valid = true;
-	}
-
-	mutex_unlock(&data->update_lock);
-
-	return data;
-}
-
 static const struct pci_device_id sis5595_pci_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_503) },
 	{ 0, }
@@ -764,7 +749,7 @@ static int sis5595_device_add(unsigned short address)
 	struct resource res = {
 		.start	= address,
 		.end	= address + SIS5595_EXTENT - 1,
-		.name	= "sis5595",
+		.name	= DRIVER_NAME,
 		.flags	= IORESOURCE_IO,
 	};
 	int err;
@@ -773,7 +758,7 @@ static int sis5595_device_add(unsigned short address)
 	if (err)
 		goto exit;
 
-	pdev = platform_device_alloc("sis5595", address);
+	pdev = platform_device_alloc(DRIVER_NAME, address);
 	if (!pdev) {
 		err = -ENOMEM;
 		pr_err("Device allocation failed\n");
@@ -800,6 +785,14 @@ exit:
 	return err;
 }
 
+static struct platform_driver sis5595_driver = {
+	.driver = {
+		.name	= DRIVER_NAME,
+	},
+	.probe		= sis5595_probe,
+	.remove		= sis5595_remove,
+};
+
 static int sis5595_pci_probe(struct pci_dev *dev,
 				       const struct pci_device_id *id)
 {
@@ -886,7 +879,7 @@ exit:
 }
 
 static struct pci_driver sis5595_pci_driver = {
-	.name            = "sis5595",
+	.name            = DRIVER_NAME,
 	.id_table        = sis5595_pci_ids,
 	.probe           = sis5595_pci_probe,
 };
diff --git a/drivers/hwmon/smm665.c b/drivers/hwmon/smm665.c
index 8c4ed72e5d68..c36bdbe423de 100644
--- a/drivers/hwmon/smm665.c
+++ b/drivers/hwmon/smm665.c
@@ -671,12 +671,11 @@ out_unregister:
 	return ret;
 }
 
-static int smm665_remove(struct i2c_client *client)
+static void smm665_remove(struct i2c_client *client)
 {
 	struct smm665_data *data = i2c_get_clientdata(client);
 
 	i2c_unregister_device(data->cmdreg);
-	return 0;
 }
 
 static const struct i2c_device_id smm665_id[] = {
diff --git a/drivers/hwmon/smsc47m192.c b/drivers/hwmon/smsc47m192.c
index a5db15c087ae..70d2152234e2 100644
--- a/drivers/hwmon/smsc47m192.c
+++ b/drivers/hwmon/smsc47m192.c
@@ -582,7 +582,7 @@ static int smsc47m192_detect(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, "smsc47m192", I2C_NAME_SIZE);
+	strscpy(info->type, "smsc47m192", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/sparx5-temp.c b/drivers/hwmon/sparx5-temp.c
index 98be48e3a22a..04fd8505e5d6 100644
--- a/drivers/hwmon/sparx5-temp.c
+++ b/drivers/hwmon/sparx5-temp.c
@@ -26,13 +26,6 @@ struct s5_hwmon {
 	struct clk *clk;
 };
 
-static void s5_temp_clk_disable(void *data)
-{
-	struct clk *clk = data;
-
-	clk_disable_unprepare(clk);
-}
-
 static void s5_temp_enable(struct s5_hwmon *hwmon)
 {
 	u32 val = readl(hwmon->base + TEMP_CFG);
@@ -113,7 +106,6 @@ static int s5_temp_probe(struct platform_device *pdev)
 {
 	struct device *hwmon_dev;
 	struct s5_hwmon *hwmon;
-	int ret;
 
 	hwmon = devm_kzalloc(&pdev->dev, sizeof(*hwmon), GFP_KERNEL);
 	if (!hwmon)
@@ -123,19 +115,10 @@ static int s5_temp_probe(struct platform_device *pdev)
 	if (IS_ERR(hwmon->base))
 		return PTR_ERR(hwmon->base);
 
-	hwmon->clk = devm_clk_get(&pdev->dev, NULL);
+	hwmon->clk = devm_clk_get_enabled(&pdev->dev, NULL);
 	if (IS_ERR(hwmon->clk))
 		return PTR_ERR(hwmon->clk);
 
-	ret = clk_prepare_enable(hwmon->clk);
-	if (ret)
-		return ret;
-
-	ret = devm_add_action_or_reset(&pdev->dev, s5_temp_clk_disable,
-				       hwmon->clk);
-	if (ret)
-		return ret;
-
 	s5_temp_enable(hwmon);
 
 	hwmon_dev = devm_hwmon_device_register_with_info(&pdev->dev,
diff --git a/drivers/hwmon/stts751.c b/drivers/hwmon/stts751.c
index 0ed28408aa07..2f67c6747ead 100644
--- a/drivers/hwmon/stts751.c
+++ b/drivers/hwmon/stts751.c
@@ -692,7 +692,7 @@ static int stts751_detect(struct i2c_client *new_client,
 	}
 	dev_dbg(&new_client->dev, "Chip %s detected", name);
 
-	strlcpy(info->type, stts751_id[0].name, I2C_NAME_SIZE);
+	strscpy(info->type, stts751_id[0].name, I2C_NAME_SIZE);
 	return 0;
 }
 
diff --git a/drivers/hwmon/thmc50.c b/drivers/hwmon/thmc50.c
index 6a804f5036f4..81cdb012993c 100644
--- a/drivers/hwmon/thmc50.c
+++ b/drivers/hwmon/thmc50.c
@@ -352,7 +352,7 @@ static int thmc50_detect(struct i2c_client *client,
 	pr_debug("thmc50: Detected %s (version %x, revision %x)\n",
 		 type_name, (revision >> 4) - 0xc, revision & 0xf);
 
-	strlcpy(info->type, type_name, I2C_NAME_SIZE);
+	strscpy(info->type, type_name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/tmp102.c b/drivers/hwmon/tmp102.c
index e867a0c2e539..2bf496a62206 100644
--- a/drivers/hwmon/tmp102.c
+++ b/drivers/hwmon/tmp102.c
@@ -260,7 +260,6 @@ static int tmp102_probe(struct i2c_client *client)
 	return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
 static int tmp102_suspend(struct device *dev)
 {
 	struct i2c_client *client = to_i2c_client(dev);
@@ -283,9 +282,8 @@ static int tmp102_resume(struct device *dev)
 
 	return err;
 }
-#endif /* CONFIG_PM */
 
-static SIMPLE_DEV_PM_OPS(tmp102_dev_pm_ops, tmp102_suspend, tmp102_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(tmp102_dev_pm_ops, tmp102_suspend, tmp102_resume);
 
 static const struct i2c_device_id tmp102_id[] = {
 	{ "tmp102", 0 },
@@ -302,7 +300,7 @@ MODULE_DEVICE_TABLE(of, tmp102_of_match);
 static struct i2c_driver tmp102_driver = {
 	.driver.name	= DRIVER_NAME,
 	.driver.of_match_table = of_match_ptr(tmp102_of_match),
-	.driver.pm	= &tmp102_dev_pm_ops,
+	.driver.pm	= pm_sleep_ptr(&tmp102_dev_pm_ops),
 	.probe_new	= tmp102_probe,
 	.id_table	= tmp102_id,
 };
diff --git a/drivers/hwmon/tmp103.c b/drivers/hwmon/tmp103.c
index 5cab4436aa77..56d5cbf36a45 100644
--- a/drivers/hwmon/tmp103.c
+++ b/drivers/hwmon/tmp103.c
@@ -178,7 +178,7 @@ static int tmp103_probe(struct i2c_client *client)
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
-static int __maybe_unused tmp103_suspend(struct device *dev)
+static int tmp103_suspend(struct device *dev)
 {
 	struct regmap *regmap = dev_get_drvdata(dev);
 
@@ -186,7 +186,7 @@ static int __maybe_unused tmp103_suspend(struct device *dev)
 				  TMP103_CONF_SD_MASK, 0);
 }
 
-static int __maybe_unused tmp103_resume(struct device *dev)
+static int tmp103_resume(struct device *dev)
 {
 	struct regmap *regmap = dev_get_drvdata(dev);
 
@@ -194,7 +194,7 @@ static int __maybe_unused tmp103_resume(struct device *dev)
 				  TMP103_CONF_SD_MASK, TMP103_CONF_SD);
 }
 
-static SIMPLE_DEV_PM_OPS(tmp103_dev_pm_ops, tmp103_suspend, tmp103_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(tmp103_dev_pm_ops, tmp103_suspend, tmp103_resume);
 
 static const struct i2c_device_id tmp103_id[] = {
 	{ "tmp103", 0 },
@@ -212,7 +212,7 @@ static struct i2c_driver tmp103_driver = {
 	.driver = {
 		.name	= "tmp103",
 		.of_match_table = of_match_ptr(tmp103_of_match),
-		.pm	= &tmp103_dev_pm_ops,
+		.pm	= pm_sleep_ptr(&tmp103_dev_pm_ops),
 	},
 	.probe_new	= tmp103_probe,
 	.id_table	= tmp103_id,
diff --git a/drivers/hwmon/tmp108.c b/drivers/hwmon/tmp108.c
index 5435664c3f6e..acb4ba750b09 100644
--- a/drivers/hwmon/tmp108.c
+++ b/drivers/hwmon/tmp108.c
@@ -390,7 +390,7 @@ static int tmp108_probe(struct i2c_client *client)
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
-static int __maybe_unused tmp108_suspend(struct device *dev)
+static int tmp108_suspend(struct device *dev)
 {
 	struct tmp108 *tmp108 = dev_get_drvdata(dev);
 
@@ -398,7 +398,7 @@ static int __maybe_unused tmp108_suspend(struct device *dev)
 				  TMP108_CONF_MODE_MASK, TMP108_MODE_SHUTDOWN);
 }
 
-static int __maybe_unused tmp108_resume(struct device *dev)
+static int tmp108_resume(struct device *dev)
 {
 	struct tmp108 *tmp108 = dev_get_drvdata(dev);
 	int err;
@@ -410,7 +410,7 @@ static int __maybe_unused tmp108_resume(struct device *dev)
 	return err;
 }
 
-static SIMPLE_DEV_PM_OPS(tmp108_dev_pm_ops, tmp108_suspend, tmp108_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(tmp108_dev_pm_ops, tmp108_suspend, tmp108_resume);
 
 static const struct i2c_device_id tmp108_i2c_ids[] = {
 	{ "tmp108", 0 },
@@ -429,7 +429,7 @@ MODULE_DEVICE_TABLE(of, tmp108_of_ids);
 static struct i2c_driver tmp108_driver = {
 	.driver = {
 		.name	= DRIVER_NAME,
-		.pm	= &tmp108_dev_pm_ops,
+		.pm	= pm_sleep_ptr(&tmp108_dev_pm_ops),
 		.of_match_table = of_match_ptr(tmp108_of_ids),
 	},
 	.probe_new	= tmp108_probe,
diff --git a/drivers/hwmon/tmp401.c b/drivers/hwmon/tmp401.c
index cc0a1c219b1f..f358ba679626 100644
--- a/drivers/hwmon/tmp401.c
+++ b/drivers/hwmon/tmp401.c
@@ -671,7 +671,7 @@ static int tmp401_detect(struct i2c_client *client,
 	if (reg > 15)
 		return -ENODEV;
 
-	strlcpy(info->type, tmp401_id[kind].name, I2C_NAME_SIZE);
+	strscpy(info->type, tmp401_id[kind].name, I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/tmp421.c b/drivers/hwmon/tmp421.c
index 1fd8d41d90c8..45fd7fb5ee01 100644
--- a/drivers/hwmon/tmp421.c
+++ b/drivers/hwmon/tmp421.c
@@ -353,7 +353,7 @@ static int tmp421_detect(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, tmp421_id[kind].name, I2C_NAME_SIZE);
+	strscpy(info->type, tmp421_id[kind].name, I2C_NAME_SIZE);
 	dev_info(&adapter->dev, "Detected TI %s chip at 0x%02x\n",
 		 names[kind], client->addr);
 
diff --git a/drivers/hwmon/tps23861.c b/drivers/hwmon/tps23861.c
index f7c59ff7ae8e..68c77c493270 100644
--- a/drivers/hwmon/tps23861.c
+++ b/drivers/hwmon/tps23861.c
@@ -372,29 +372,12 @@ static const struct hwmon_chip_info tps23861_chip_info = {
 	.info = tps23861_info,
 };
 
-static char *tps23861_port_operating_mode(struct tps23861_data *data, int port)
+static char *port_operating_mode_string(uint8_t mode_reg, unsigned int port)
 {
-	unsigned int regval;
-	int mode;
+	unsigned int mode = ~0;
 
-	regmap_read(data->regmap, OPERATING_MODE, &regval);
-
-	switch (port) {
-	case 1:
-		mode = FIELD_GET(OPERATING_MODE_PORT_1_MASK, regval);
-		break;
-	case 2:
-		mode = FIELD_GET(OPERATING_MODE_PORT_2_MASK, regval);
-		break;
-	case 3:
-		mode = FIELD_GET(OPERATING_MODE_PORT_3_MASK, regval);
-		break;
-	case 4:
-		mode = FIELD_GET(OPERATING_MODE_PORT_4_MASK, regval);
-		break;
-	default:
-		mode = -EINVAL;
-	}
+	if (port < TPS23861_NUM_PORTS)
+		mode = (mode_reg >> (2 * port)) & OPERATING_MODE_PORT_1_MASK;
 
 	switch (mode) {
 	case OPERATING_MODE_OFF:
@@ -410,15 +393,9 @@ static char *tps23861_port_operating_mode(struct tps23861_data *data, int port)
 	}
 }
 
-static char *tps23861_port_detect_status(struct tps23861_data *data, int port)
+static char *port_detect_status_string(uint8_t status_reg)
 {
-	unsigned int regval;
-
-	regmap_read(data->regmap,
-		    PORT_1_STATUS + (port - 1),
-		    &regval);
-
-	switch (FIELD_GET(PORT_STATUS_DETECT_MASK, regval)) {
+	switch (FIELD_GET(PORT_STATUS_DETECT_MASK, status_reg)) {
 	case PORT_DETECT_UNKNOWN:
 		return "Unknown device";
 	case PORT_DETECT_SHORT:
@@ -448,15 +425,9 @@ static char *tps23861_port_detect_status(struct tps23861_data *data, int port)
 	}
 }
 
-static char *tps23861_port_class_status(struct tps23861_data *data, int port)
+static char *port_class_status_string(uint8_t status_reg)
 {
-	unsigned int regval;
-
-	regmap_read(data->regmap,
-		    PORT_1_STATUS + (port - 1),
-		    &regval);
-
-	switch (FIELD_GET(PORT_STATUS_CLASS_MASK, regval)) {
+	switch (FIELD_GET(PORT_STATUS_CLASS_MASK, status_reg)) {
 	case PORT_CLASS_UNKNOWN:
 		return "Unknown";
 	case PORT_CLASS_RESERVED:
@@ -479,16 +450,9 @@ static char *tps23861_port_class_status(struct tps23861_data *data, int port)
 	}
 }
 
-static char *tps23861_port_poe_plus_status(struct tps23861_data *data, int port)
+static char *port_poe_plus_status_string(uint8_t poe_plus, unsigned int port)
 {
-	unsigned int regval;
-
-	regmap_read(data->regmap, POE_PLUS, &regval);
-
-	if (BIT(port + 3) & regval)
-		return "Yes";
-	else
-		return "No";
+	return (BIT(port + 4) & poe_plus) ? "Yes" : "No";
 }
 
 static int tps23861_port_resistance(struct tps23861_data *data, int port)
@@ -497,7 +461,7 @@ static int tps23861_port_resistance(struct tps23861_data *data, int port)
 	__le16 regval;
 
 	regmap_bulk_read(data->regmap,
-			 PORT_1_RESISTANCE_LSB + PORT_N_RESISTANCE_LSB_OFFSET * (port - 1),
+			 PORT_1_RESISTANCE_LSB + PORT_N_RESISTANCE_LSB_OFFSET * port,
 			 &regval,
 			 2);
 
@@ -517,14 +481,19 @@ static int tps23861_port_resistance(struct tps23861_data *data, int port)
 static int tps23861_port_status_show(struct seq_file *s, void *data)
 {
 	struct tps23861_data *priv = s->private;
-	int i;
-
-	for (i = 1; i < TPS23861_NUM_PORTS + 1; i++) {
-		seq_printf(s, "Port: \t\t%d\n", i);
-		seq_printf(s, "Operating mode: %s\n", tps23861_port_operating_mode(priv, i));
-		seq_printf(s, "Detected: \t%s\n", tps23861_port_detect_status(priv, i));
-		seq_printf(s, "Class: \t\t%s\n", tps23861_port_class_status(priv, i));
-		seq_printf(s, "PoE Plus: \t%s\n", tps23861_port_poe_plus_status(priv, i));
+	unsigned int i, mode, poe_plus, status;
+
+	regmap_read(priv->regmap, OPERATING_MODE, &mode);
+	regmap_read(priv->regmap, POE_PLUS, &poe_plus);
+
+	for (i = 0; i < TPS23861_NUM_PORTS; i++) {
+		regmap_read(priv->regmap, PORT_1_STATUS + i, &status);
+
+		seq_printf(s, "Port: \t\t%d\n", i + 1);
+		seq_printf(s, "Operating mode: %s\n", port_operating_mode_string(mode, i));
+		seq_printf(s, "Detected: \t%s\n", port_detect_status_string(status));
+		seq_printf(s, "Class: \t\t%s\n", port_class_status_string(status));
+		seq_printf(s, "PoE Plus: \t%s\n", port_poe_plus_status_string(poe_plus, i));
 		seq_printf(s, "Resistance: \t%d\n", tps23861_port_resistance(priv, i));
 		seq_putc(s, '\n');
 	}
@@ -534,9 +503,17 @@ static int tps23861_port_status_show(struct seq_file *s, void *data)
 
 DEFINE_SHOW_ATTRIBUTE(tps23861_port_status);
 
-static void tps23861_init_debugfs(struct tps23861_data *data)
+static void tps23861_init_debugfs(struct tps23861_data *data,
+				  struct device *hwmon_dev)
 {
-	data->debugfs_dir = debugfs_create_dir(data->client->name, NULL);
+	const char *debugfs_name;
+
+	debugfs_name = devm_kasprintf(&data->client->dev, GFP_KERNEL, "%s-%s",
+				      data->client->name, dev_name(hwmon_dev));
+	if (!debugfs_name)
+		return;
+
+	data->debugfs_dir = debugfs_create_dir(debugfs_name, NULL);
 
 	debugfs_create_file("port_status",
 			    0400,
@@ -585,18 +562,16 @@ static int tps23861_probe(struct i2c_client *client)
 	if (IS_ERR(hwmon_dev))
 		return PTR_ERR(hwmon_dev);
 
-	tps23861_init_debugfs(data);
+	tps23861_init_debugfs(data, hwmon_dev);
 
 	return 0;
 }
 
-static int tps23861_remove(struct i2c_client *client)
+static void tps23861_remove(struct i2c_client *client)
 {
 	struct tps23861_data *data = i2c_get_clientdata(client);
 
 	debugfs_remove_recursive(data->debugfs_dir);
-
-	return 0;
 }
 
 static const struct of_device_id __maybe_unused tps23861_of_match[] = {
diff --git a/drivers/hwmon/via686a.c b/drivers/hwmon/via686a.c
index 55634110c2f9..37d7374896f6 100644
--- a/drivers/hwmon/via686a.c
+++ b/drivers/hwmon/via686a.c
@@ -34,6 +34,8 @@
 #include <linux/acpi.h>
 #include <linux/io.h>
 
+#define DRIVER_NAME "via686a"
+
 /*
  * If force_addr is set to anything different from 0, we forcibly enable
  * the device at the given address.
@@ -321,9 +323,6 @@ struct via686a_data {
 
 static struct pci_dev *s_bridge;	/* pointer to the (only) via686a */
 
-static int via686a_probe(struct platform_device *pdev);
-static int via686a_remove(struct platform_device *pdev);
-
 static inline int via686a_read_value(struct via686a_data *data, u8 reg)
 {
 	return inb_p(data->addr + reg);
@@ -335,8 +334,76 @@ static inline void via686a_write_value(struct via686a_data *data, u8 reg,
 	outb_p(value, data->addr + reg);
 }
 
-static struct via686a_data *via686a_update_device(struct device *dev);
-static void via686a_init_device(struct via686a_data *data);
+static void via686a_update_fan_div(struct via686a_data *data)
+{
+	int reg = via686a_read_value(data, VIA686A_REG_FANDIV);
+	data->fan_div[0] = (reg >> 4) & 0x03;
+	data->fan_div[1] = reg >> 6;
+}
+
+static struct via686a_data *via686a_update_device(struct device *dev)
+{
+	struct via686a_data *data = dev_get_drvdata(dev);
+	int i;
+
+	mutex_lock(&data->update_lock);
+
+	if (time_after(jiffies, data->last_updated + HZ + HZ / 2)
+	    || !data->valid) {
+		for (i = 0; i <= 4; i++) {
+			data->in[i] =
+			    via686a_read_value(data, VIA686A_REG_IN(i));
+			data->in_min[i] = via686a_read_value(data,
+							     VIA686A_REG_IN_MIN
+							     (i));
+			data->in_max[i] =
+			    via686a_read_value(data, VIA686A_REG_IN_MAX(i));
+		}
+		for (i = 1; i <= 2; i++) {
+			data->fan[i - 1] =
+			    via686a_read_value(data, VIA686A_REG_FAN(i));
+			data->fan_min[i - 1] = via686a_read_value(data,
+						     VIA686A_REG_FAN_MIN(i));
+		}
+		for (i = 0; i <= 2; i++) {
+			data->temp[i] = via686a_read_value(data,
+						 VIA686A_REG_TEMP[i]) << 2;
+			data->temp_over[i] =
+			    via686a_read_value(data,
+					       VIA686A_REG_TEMP_OVER[i]);
+			data->temp_hyst[i] =
+			    via686a_read_value(data,
+					       VIA686A_REG_TEMP_HYST[i]);
+		}
+		/*
+		 * add in lower 2 bits
+		 * temp1 uses bits 7-6 of VIA686A_REG_TEMP_LOW1
+		 * temp2 uses bits 5-4 of VIA686A_REG_TEMP_LOW23
+		 * temp3 uses bits 7-6 of VIA686A_REG_TEMP_LOW23
+		 */
+		data->temp[0] |= (via686a_read_value(data,
+						     VIA686A_REG_TEMP_LOW1)
+				  & 0xc0) >> 6;
+		data->temp[1] |=
+		    (via686a_read_value(data, VIA686A_REG_TEMP_LOW23) &
+		     0x30) >> 4;
+		data->temp[2] |=
+		    (via686a_read_value(data, VIA686A_REG_TEMP_LOW23) &
+		     0xc0) >> 6;
+
+		via686a_update_fan_div(data);
+		data->alarms =
+		    via686a_read_value(data,
+				       VIA686A_REG_ALARM1) |
+		    (via686a_read_value(data, VIA686A_REG_ALARM2) << 8);
+		data->last_updated = jiffies;
+		data->valid = true;
+	}
+
+	mutex_unlock(&data->update_lock);
+
+	return data;
+}
 
 /* following are the sysfs callback functions */
 
@@ -654,13 +721,23 @@ static const struct attribute_group via686a_group = {
 	.attrs = via686a_attributes,
 };
 
-static struct platform_driver via686a_driver = {
-	.driver = {
-		.name	= "via686a",
-	},
-	.probe		= via686a_probe,
-	.remove		= via686a_remove,
-};
+static void via686a_init_device(struct via686a_data *data)
+{
+	u8 reg;
+
+	/* Start monitoring */
+	reg = via686a_read_value(data, VIA686A_REG_CONFIG);
+	via686a_write_value(data, VIA686A_REG_CONFIG, (reg | 0x01) & 0x7F);
+
+	/* Configure temp interrupt mode for continuous-interrupt operation */
+	reg = via686a_read_value(data, VIA686A_REG_TEMP_MODE);
+	via686a_write_value(data, VIA686A_REG_TEMP_MODE,
+			    (reg & ~VIA686A_TEMP_MODE_MASK)
+			    | VIA686A_TEMP_MODE_CONTINUOUS);
+
+	/* Pre-read fan clock divisor values */
+	via686a_update_fan_div(data);
+}
 
 /* This is called when the module is loaded */
 static int via686a_probe(struct platform_device *pdev)
@@ -672,7 +749,7 @@ static int via686a_probe(struct platform_device *pdev)
 	/* Reserve the ISA region */
 	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
 	if (!devm_request_region(&pdev->dev, res->start, VIA686A_EXTENT,
-				 via686a_driver.driver.name)) {
+				 DRIVER_NAME)) {
 		dev_err(&pdev->dev, "Region 0x%lx-0x%lx already in use!\n",
 			(unsigned long)res->start, (unsigned long)res->end);
 		return -ENODEV;
@@ -685,7 +762,7 @@ static int via686a_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, data);
 	data->addr = res->start;
-	data->name = "via686a";
+	data->name = DRIVER_NAME;
 	mutex_init(&data->update_lock);
 
 	/* Initialize the VIA686A chip */
@@ -719,94 +796,13 @@ static int via686a_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static void via686a_update_fan_div(struct via686a_data *data)
-{
-	int reg = via686a_read_value(data, VIA686A_REG_FANDIV);
-	data->fan_div[0] = (reg >> 4) & 0x03;
-	data->fan_div[1] = reg >> 6;
-}
-
-static void via686a_init_device(struct via686a_data *data)
-{
-	u8 reg;
-
-	/* Start monitoring */
-	reg = via686a_read_value(data, VIA686A_REG_CONFIG);
-	via686a_write_value(data, VIA686A_REG_CONFIG, (reg | 0x01) & 0x7F);
-
-	/* Configure temp interrupt mode for continuous-interrupt operation */
-	reg = via686a_read_value(data, VIA686A_REG_TEMP_MODE);
-	via686a_write_value(data, VIA686A_REG_TEMP_MODE,
-			    (reg & ~VIA686A_TEMP_MODE_MASK)
-			    | VIA686A_TEMP_MODE_CONTINUOUS);
-
-	/* Pre-read fan clock divisor values */
-	via686a_update_fan_div(data);
-}
-
-static struct via686a_data *via686a_update_device(struct device *dev)
-{
-	struct via686a_data *data = dev_get_drvdata(dev);
-	int i;
-
-	mutex_lock(&data->update_lock);
-
-	if (time_after(jiffies, data->last_updated + HZ + HZ / 2)
-	    || !data->valid) {
-		for (i = 0; i <= 4; i++) {
-			data->in[i] =
-			    via686a_read_value(data, VIA686A_REG_IN(i));
-			data->in_min[i] = via686a_read_value(data,
-							     VIA686A_REG_IN_MIN
-							     (i));
-			data->in_max[i] =
-			    via686a_read_value(data, VIA686A_REG_IN_MAX(i));
-		}
-		for (i = 1; i <= 2; i++) {
-			data->fan[i - 1] =
-			    via686a_read_value(data, VIA686A_REG_FAN(i));
-			data->fan_min[i - 1] = via686a_read_value(data,
-						     VIA686A_REG_FAN_MIN(i));
-		}
-		for (i = 0; i <= 2; i++) {
-			data->temp[i] = via686a_read_value(data,
-						 VIA686A_REG_TEMP[i]) << 2;
-			data->temp_over[i] =
-			    via686a_read_value(data,
-					       VIA686A_REG_TEMP_OVER[i]);
-			data->temp_hyst[i] =
-			    via686a_read_value(data,
-					       VIA686A_REG_TEMP_HYST[i]);
-		}
-		/*
-		 * add in lower 2 bits
-		 * temp1 uses bits 7-6 of VIA686A_REG_TEMP_LOW1
-		 * temp2 uses bits 5-4 of VIA686A_REG_TEMP_LOW23
-		 * temp3 uses bits 7-6 of VIA686A_REG_TEMP_LOW23
-		 */
-		data->temp[0] |= (via686a_read_value(data,
-						     VIA686A_REG_TEMP_LOW1)
-				  & 0xc0) >> 6;
-		data->temp[1] |=
-		    (via686a_read_value(data, VIA686A_REG_TEMP_LOW23) &
-		     0x30) >> 4;
-		data->temp[2] |=
-		    (via686a_read_value(data, VIA686A_REG_TEMP_LOW23) &
-		     0xc0) >> 6;
-
-		via686a_update_fan_div(data);
-		data->alarms =
-		    via686a_read_value(data,
-				       VIA686A_REG_ALARM1) |
-		    (via686a_read_value(data, VIA686A_REG_ALARM2) << 8);
-		data->last_updated = jiffies;
-		data->valid = true;
-	}
-
-	mutex_unlock(&data->update_lock);
-
-	return data;
-}
+static struct platform_driver via686a_driver = {
+	.driver = {
+		.name	= DRIVER_NAME,
+	},
+	.probe		= via686a_probe,
+	.remove		= via686a_remove,
+};
 
 static const struct pci_device_id via686a_pci_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_82C686_4) },
@@ -819,7 +815,7 @@ static int via686a_device_add(unsigned short address)
 	struct resource res = {
 		.start	= address,
 		.end	= address + VIA686A_EXTENT - 1,
-		.name	= "via686a",
+		.name	= DRIVER_NAME,
 		.flags	= IORESOURCE_IO,
 	};
 	int err;
@@ -828,7 +824,7 @@ static int via686a_device_add(unsigned short address)
 	if (err)
 		goto exit;
 
-	pdev = platform_device_alloc("via686a", address);
+	pdev = platform_device_alloc(DRIVER_NAME, address);
 	if (!pdev) {
 		err = -ENOMEM;
 		pr_err("Device allocation failed\n");
@@ -918,7 +914,7 @@ exit:
 }
 
 static struct pci_driver via686a_pci_driver = {
-	.name		= "via686a",
+	.name		= DRIVER_NAME,
 	.id_table	= via686a_pci_ids,
 	.probe		= via686a_pci_probe,
 };
diff --git a/drivers/hwmon/vt8231.c b/drivers/hwmon/vt8231.c
index 03275ac8ba72..3b7f8922b0d5 100644
--- a/drivers/hwmon/vt8231.c
+++ b/drivers/hwmon/vt8231.c
@@ -38,6 +38,8 @@ static struct platform_device *pdev;
 #define VT8231_BASE_REG 0x70
 #define VT8231_ENABLE_REG 0x74
 
+#define DRIVER_NAME "vt8231"
+
 /*
  * The VT8231 registers
  *
@@ -162,10 +164,6 @@ struct vt8231_data {
 };
 
 static struct pci_dev *s_bridge;
-static int vt8231_probe(struct platform_device *pdev);
-static int vt8231_remove(struct platform_device *pdev);
-static struct vt8231_data *vt8231_update_device(struct device *dev);
-static void vt8231_init_device(struct vt8231_data *data);
 
 static inline int vt8231_read_value(struct vt8231_data *data, u8 reg)
 {
@@ -178,6 +176,74 @@ static inline void vt8231_write_value(struct vt8231_data *data, u8 reg,
 	outb_p(value, data->addr + reg);
 }
 
+static struct vt8231_data *vt8231_update_device(struct device *dev)
+{
+	struct vt8231_data *data = dev_get_drvdata(dev);
+	int i;
+	u16 low;
+
+	mutex_lock(&data->update_lock);
+
+	if (time_after(jiffies, data->last_updated + HZ + HZ / 2)
+	    || !data->valid) {
+		for (i = 0; i < 6; i++) {
+			if (ISVOLT(i, data->uch_config)) {
+				data->in[i] = vt8231_read_value(data,
+						regvolt[i]);
+				data->in_min[i] = vt8231_read_value(data,
+						regvoltmin[i]);
+				data->in_max[i] = vt8231_read_value(data,
+						regvoltmax[i]);
+			}
+		}
+		for (i = 0; i < 2; i++) {
+			data->fan[i] = vt8231_read_value(data,
+						VT8231_REG_FAN(i));
+			data->fan_min[i] = vt8231_read_value(data,
+						VT8231_REG_FAN_MIN(i));
+		}
+
+		low = vt8231_read_value(data, VT8231_REG_TEMP_LOW01);
+		low = (low >> 6) | ((low & 0x30) >> 2)
+		    | (vt8231_read_value(data, VT8231_REG_TEMP_LOW25) << 4);
+		for (i = 0; i < 6; i++) {
+			if (ISTEMP(i, data->uch_config)) {
+				data->temp[i] = (vt8231_read_value(data,
+						       regtemp[i]) << 2)
+						| ((low >> (2 * i)) & 0x03);
+				data->temp_max[i] = vt8231_read_value(data,
+						      regtempmax[i]);
+				data->temp_min[i] = vt8231_read_value(data,
+						      regtempmin[i]);
+			}
+		}
+
+		i = vt8231_read_value(data, VT8231_REG_FANDIV);
+		data->fan_div[0] = (i >> 4) & 0x03;
+		data->fan_div[1] = i >> 6;
+		data->alarms = vt8231_read_value(data, VT8231_REG_ALARM1) |
+			(vt8231_read_value(data, VT8231_REG_ALARM2) << 8);
+
+		/* Set alarm flags correctly */
+		if (!data->fan[0] && data->fan_min[0])
+			data->alarms |= 0x40;
+		else if (data->fan[0] && !data->fan_min[0])
+			data->alarms &= ~0x40;
+
+		if (!data->fan[1] && data->fan_min[1])
+			data->alarms |= 0x80;
+		else if (data->fan[1] && !data->fan_min[1])
+			data->alarms &= ~0x80;
+
+		data->last_updated = jiffies;
+		data->valid = true;
+	}
+
+	mutex_unlock(&data->update_lock);
+
+	return data;
+}
+
 /* following are the sysfs callback functions */
 static ssize_t in_show(struct device *dev, struct device_attribute *attr,
 		       char *buf)
@@ -751,29 +817,11 @@ static const struct attribute_group vt8231_group = {
 	.attrs = vt8231_attributes,
 };
 
-static struct platform_driver vt8231_driver = {
-	.driver = {
-		.name	= "vt8231",
-	},
-	.probe	= vt8231_probe,
-	.remove	= vt8231_remove,
-};
-
-static const struct pci_device_id vt8231_pci_ids[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8231_4) },
-	{ 0, }
-};
-
-MODULE_DEVICE_TABLE(pci, vt8231_pci_ids);
-
-static int vt8231_pci_probe(struct pci_dev *dev,
-				      const struct pci_device_id *id);
-
-static struct pci_driver vt8231_pci_driver = {
-	.name		= "vt8231",
-	.id_table	= vt8231_pci_ids,
-	.probe		= vt8231_pci_probe,
-};
+static void vt8231_init_device(struct vt8231_data *data)
+{
+	vt8231_write_value(data, VT8231_REG_TEMP1_CONFIG, 0);
+	vt8231_write_value(data, VT8231_REG_TEMP2_CONFIG, 0);
+}
 
 static int vt8231_probe(struct platform_device *pdev)
 {
@@ -784,7 +832,7 @@ static int vt8231_probe(struct platform_device *pdev)
 	/* Reserve the ISA region */
 	res = platform_get_resource(pdev, IORESOURCE_IO, 0);
 	if (!devm_request_region(&pdev->dev, res->start, VT8231_EXTENT,
-				 vt8231_driver.driver.name)) {
+				 DRIVER_NAME)) {
 		dev_err(&pdev->dev, "Region 0x%lx-0x%lx already in use!\n",
 			(unsigned long)res->start, (unsigned long)res->end);
 		return -ENODEV;
@@ -796,7 +844,7 @@ static int vt8231_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, data);
 	data->addr = res->start;
-	data->name = "vt8231";
+	data->name = DRIVER_NAME;
 
 	mutex_init(&data->update_lock);
 	vt8231_init_device(data);
@@ -863,86 +911,28 @@ static int vt8231_remove(struct platform_device *pdev)
 	return 0;
 }
 
-static void vt8231_init_device(struct vt8231_data *data)
-{
-	vt8231_write_value(data, VT8231_REG_TEMP1_CONFIG, 0);
-	vt8231_write_value(data, VT8231_REG_TEMP2_CONFIG, 0);
-}
-
-static struct vt8231_data *vt8231_update_device(struct device *dev)
-{
-	struct vt8231_data *data = dev_get_drvdata(dev);
-	int i;
-	u16 low;
-
-	mutex_lock(&data->update_lock);
-
-	if (time_after(jiffies, data->last_updated + HZ + HZ / 2)
-	    || !data->valid) {
-		for (i = 0; i < 6; i++) {
-			if (ISVOLT(i, data->uch_config)) {
-				data->in[i] = vt8231_read_value(data,
-						regvolt[i]);
-				data->in_min[i] = vt8231_read_value(data,
-						regvoltmin[i]);
-				data->in_max[i] = vt8231_read_value(data,
-						regvoltmax[i]);
-			}
-		}
-		for (i = 0; i < 2; i++) {
-			data->fan[i] = vt8231_read_value(data,
-						VT8231_REG_FAN(i));
-			data->fan_min[i] = vt8231_read_value(data,
-						VT8231_REG_FAN_MIN(i));
-		}
-
-		low = vt8231_read_value(data, VT8231_REG_TEMP_LOW01);
-		low = (low >> 6) | ((low & 0x30) >> 2)
-		    | (vt8231_read_value(data, VT8231_REG_TEMP_LOW25) << 4);
-		for (i = 0; i < 6; i++) {
-			if (ISTEMP(i, data->uch_config)) {
-				data->temp[i] = (vt8231_read_value(data,
-						       regtemp[i]) << 2)
-						| ((low >> (2 * i)) & 0x03);
-				data->temp_max[i] = vt8231_read_value(data,
-						      regtempmax[i]);
-				data->temp_min[i] = vt8231_read_value(data,
-						      regtempmin[i]);
-			}
-		}
-
-		i = vt8231_read_value(data, VT8231_REG_FANDIV);
-		data->fan_div[0] = (i >> 4) & 0x03;
-		data->fan_div[1] = i >> 6;
-		data->alarms = vt8231_read_value(data, VT8231_REG_ALARM1) |
-			(vt8231_read_value(data, VT8231_REG_ALARM2) << 8);
-
-		/* Set alarm flags correctly */
-		if (!data->fan[0] && data->fan_min[0])
-			data->alarms |= 0x40;
-		else if (data->fan[0] && !data->fan_min[0])
-			data->alarms &= ~0x40;
-
-		if (!data->fan[1] && data->fan_min[1])
-			data->alarms |= 0x80;
-		else if (data->fan[1] && !data->fan_min[1])
-			data->alarms &= ~0x80;
 
-		data->last_updated = jiffies;
-		data->valid = true;
-	}
+static struct platform_driver vt8231_driver = {
+	.driver = {
+		.name	= DRIVER_NAME,
+	},
+	.probe	= vt8231_probe,
+	.remove	= vt8231_remove,
+};
 
-	mutex_unlock(&data->update_lock);
+static const struct pci_device_id vt8231_pci_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8231_4) },
+	{ 0, }
+};
 
-	return data;
-}
+MODULE_DEVICE_TABLE(pci, vt8231_pci_ids);
 
 static int vt8231_device_add(unsigned short address)
 {
 	struct resource res = {
 		.start	= address,
 		.end	= address + VT8231_EXTENT - 1,
-		.name	= "vt8231",
+		.name	= DRIVER_NAME,
 		.flags	= IORESOURCE_IO,
 	};
 	int err;
@@ -951,7 +941,7 @@ static int vt8231_device_add(unsigned short address)
 	if (err)
 		goto exit;
 
-	pdev = platform_device_alloc("vt8231", address);
+	pdev = platform_device_alloc(DRIVER_NAME, address);
 	if (!pdev) {
 		err = -ENOMEM;
 		pr_err("Device allocation failed\n");
@@ -1040,6 +1030,12 @@ exit:
 	return -ENODEV;
 }
 
+static struct pci_driver vt8231_pci_driver = {
+	.name		= DRIVER_NAME,
+	.id_table	= vt8231_pci_ids,
+	.probe		= vt8231_pci_probe,
+};
+
 static int __init sm_vt8231_init(void)
 {
 	return pci_register_driver(&vt8231_pci_driver);
diff --git a/drivers/hwmon/w83627ehf.c b/drivers/hwmon/w83627ehf.c
index af89b32a93a5..939d4c35e713 100644
--- a/drivers/hwmon/w83627ehf.c
+++ b/drivers/hwmon/w83627ehf.c
@@ -1944,7 +1944,7 @@ static int __init w83627ehf_probe(struct platform_device *pdev)
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
-static int __maybe_unused w83627ehf_suspend(struct device *dev)
+static int w83627ehf_suspend(struct device *dev)
 {
 	struct w83627ehf_data *data = w83627ehf_update_device(dev);
 
@@ -1955,7 +1955,7 @@ static int __maybe_unused w83627ehf_suspend(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused w83627ehf_resume(struct device *dev)
+static int w83627ehf_resume(struct device *dev)
 {
 	struct w83627ehf_data *data = dev_get_drvdata(dev);
 	int i;
@@ -2010,12 +2010,12 @@ static int __maybe_unused w83627ehf_resume(struct device *dev)
 	return 0;
 }
 
-static SIMPLE_DEV_PM_OPS(w83627ehf_dev_pm_ops, w83627ehf_suspend, w83627ehf_resume);
+static DEFINE_SIMPLE_DEV_PM_OPS(w83627ehf_dev_pm_ops, w83627ehf_suspend, w83627ehf_resume);
 
 static struct platform_driver w83627ehf_driver = {
 	.driver = {
 		.name	= DRVNAME,
-		.pm	= &w83627ehf_dev_pm_ops,
+		.pm	= pm_sleep_ptr(&w83627ehf_dev_pm_ops),
 	},
 };
 
diff --git a/drivers/hwmon/w83627hf.c b/drivers/hwmon/w83627hf.c
index 9be277156ed2..b638d672ac45 100644
--- a/drivers/hwmon/w83627hf.c
+++ b/drivers/hwmon/w83627hf.c
@@ -389,14 +389,184 @@ struct w83627hf_data {
 #endif
 };
 
-static int w83627hf_probe(struct platform_device *pdev);
-static int w83627hf_remove(struct platform_device *pdev);
+/* Registers 0x50-0x5f are banked */
+static inline void w83627hf_set_bank(struct w83627hf_data *data, u16 reg)
+{
+	if ((reg & 0x00f0) == 0x50) {
+		outb_p(W83781D_REG_BANK, data->addr + W83781D_ADDR_REG_OFFSET);
+		outb_p(reg >> 8, data->addr + W83781D_DATA_REG_OFFSET);
+	}
+}
+
+/* Not strictly necessary, but play it safe for now */
+static inline void w83627hf_reset_bank(struct w83627hf_data *data, u16 reg)
+{
+	if (reg & 0xff00) {
+		outb_p(W83781D_REG_BANK, data->addr + W83781D_ADDR_REG_OFFSET);
+		outb_p(0, data->addr + W83781D_DATA_REG_OFFSET);
+	}
+}
+
+static int w83627hf_read_value(struct w83627hf_data *data, u16 reg)
+{
+	int res, word_sized;
+
+	mutex_lock(&data->lock);
+	word_sized = (((reg & 0xff00) == 0x100)
+		   || ((reg & 0xff00) == 0x200))
+		  && (((reg & 0x00ff) == 0x50)
+		   || ((reg & 0x00ff) == 0x53)
+		   || ((reg & 0x00ff) == 0x55));
+	w83627hf_set_bank(data, reg);
+	outb_p(reg & 0xff, data->addr + W83781D_ADDR_REG_OFFSET);
+	res = inb_p(data->addr + W83781D_DATA_REG_OFFSET);
+	if (word_sized) {
+		outb_p((reg & 0xff) + 1,
+		       data->addr + W83781D_ADDR_REG_OFFSET);
+		res =
+		    (res << 8) + inb_p(data->addr +
+				       W83781D_DATA_REG_OFFSET);
+	}
+	w83627hf_reset_bank(data, reg);
+	mutex_unlock(&data->lock);
+	return res;
+}
+
+static int w83627hf_write_value(struct w83627hf_data *data, u16 reg, u16 value)
+{
+	int word_sized;
+
+	mutex_lock(&data->lock);
+	word_sized = (((reg & 0xff00) == 0x100)
+		   || ((reg & 0xff00) == 0x200))
+		  && (((reg & 0x00ff) == 0x53)
+		   || ((reg & 0x00ff) == 0x55));
+	w83627hf_set_bank(data, reg);
+	outb_p(reg & 0xff, data->addr + W83781D_ADDR_REG_OFFSET);
+	if (word_sized) {
+		outb_p(value >> 8,
+		       data->addr + W83781D_DATA_REG_OFFSET);
+		outb_p((reg & 0xff) + 1,
+		       data->addr + W83781D_ADDR_REG_OFFSET);
+	}
+	outb_p(value & 0xff,
+	       data->addr + W83781D_DATA_REG_OFFSET);
+	w83627hf_reset_bank(data, reg);
+	mutex_unlock(&data->lock);
+	return 0;
+}
+
+static void w83627hf_update_fan_div(struct w83627hf_data *data)
+{
+	int reg;
+
+	reg = w83627hf_read_value(data, W83781D_REG_VID_FANDIV);
+	data->fan_div[0] = (reg >> 4) & 0x03;
+	data->fan_div[1] = (reg >> 6) & 0x03;
+	if (data->type != w83697hf) {
+		data->fan_div[2] = (w83627hf_read_value(data,
+				       W83781D_REG_PIN) >> 6) & 0x03;
+	}
+	reg = w83627hf_read_value(data, W83781D_REG_VBAT);
+	data->fan_div[0] |= (reg >> 3) & 0x04;
+	data->fan_div[1] |= (reg >> 4) & 0x04;
+	if (data->type != w83697hf)
+		data->fan_div[2] |= (reg >> 5) & 0x04;
+}
 
-static int w83627hf_read_value(struct w83627hf_data *data, u16 reg);
-static int w83627hf_write_value(struct w83627hf_data *data, u16 reg, u16 value);
-static void w83627hf_update_fan_div(struct w83627hf_data *data);
-static struct w83627hf_data *w83627hf_update_device(struct device *dev);
-static void w83627hf_init_device(struct platform_device *pdev);
+static struct w83627hf_data *w83627hf_update_device(struct device *dev)
+{
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+	int i, num_temps = (data->type == w83697hf) ? 2 : 3;
+	int num_pwms = (data->type == w83697hf) ? 2 : 3;
+
+	mutex_lock(&data->update_lock);
+
+	if (time_after(jiffies, data->last_updated + HZ + HZ / 2)
+	    || !data->valid) {
+		for (i = 0; i <= 8; i++) {
+			/* skip missing sensors */
+			if (((data->type == w83697hf) && (i == 1)) ||
+			    ((data->type != w83627hf && data->type != w83697hf)
+			    && (i == 5 || i == 6)))
+				continue;
+			data->in[i] =
+			    w83627hf_read_value(data, W83781D_REG_IN(i));
+			data->in_min[i] =
+			    w83627hf_read_value(data,
+					       W83781D_REG_IN_MIN(i));
+			data->in_max[i] =
+			    w83627hf_read_value(data,
+					       W83781D_REG_IN_MAX(i));
+		}
+		for (i = 0; i <= 2; i++) {
+			data->fan[i] =
+			    w83627hf_read_value(data, W83627HF_REG_FAN(i));
+			data->fan_min[i] =
+			    w83627hf_read_value(data,
+					       W83627HF_REG_FAN_MIN(i));
+		}
+		for (i = 0; i <= 2; i++) {
+			u8 tmp = w83627hf_read_value(data,
+				W836X7HF_REG_PWM(data->type, i));
+			/* bits 0-3 are reserved  in 627THF */
+			if (data->type == w83627thf)
+				tmp &= 0xf0;
+			data->pwm[i] = tmp;
+			if (i == 1 &&
+			    (data->type == w83627hf || data->type == w83697hf))
+				break;
+		}
+		if (data->type == w83627hf) {
+				u8 tmp = w83627hf_read_value(data,
+						W83627HF_REG_PWM_FREQ);
+				data->pwm_freq[0] = tmp & 0x07;
+				data->pwm_freq[1] = (tmp >> 4) & 0x07;
+		} else if (data->type != w83627thf) {
+			for (i = 1; i <= 3; i++) {
+				data->pwm_freq[i - 1] =
+					w83627hf_read_value(data,
+						W83637HF_REG_PWM_FREQ[i - 1]);
+				if (i == 2 && (data->type == w83697hf))
+					break;
+			}
+		}
+		if (data->type != w83627hf) {
+			for (i = 0; i < num_pwms; i++) {
+				u8 tmp = w83627hf_read_value(data,
+					W83627THF_REG_PWM_ENABLE[i]);
+				data->pwm_enable[i] =
+					((tmp >> W83627THF_PWM_ENABLE_SHIFT[i])
+					& 0x03) + 1;
+			}
+		}
+		for (i = 0; i < num_temps; i++) {
+			data->temp[i] = w83627hf_read_value(
+						data, w83627hf_reg_temp[i]);
+			data->temp_max[i] = w83627hf_read_value(
+						data, w83627hf_reg_temp_over[i]);
+			data->temp_max_hyst[i] = w83627hf_read_value(
+						data, w83627hf_reg_temp_hyst[i]);
+		}
+
+		w83627hf_update_fan_div(data);
+
+		data->alarms =
+		    w83627hf_read_value(data, W83781D_REG_ALARM1) |
+		    (w83627hf_read_value(data, W83781D_REG_ALARM2) << 8) |
+		    (w83627hf_read_value(data, W83781D_REG_ALARM3) << 16);
+		i = w83627hf_read_value(data, W83781D_REG_BEEP_INTS2);
+		data->beep_mask = (i << 8) |
+		    w83627hf_read_value(data, W83781D_REG_BEEP_INTS1) |
+		    w83627hf_read_value(data, W83781D_REG_BEEP_INTS3) << 16;
+		data->last_updated = jiffies;
+		data->valid = true;
+	}
+
+	mutex_unlock(&data->update_lock);
+
+	return data;
+}
 
 #ifdef CONFIG_PM
 static int w83627hf_suspend(struct device *dev)
@@ -464,99 +634,171 @@ static const struct dev_pm_ops w83627hf_dev_pm_ops = {
 #define W83627HF_DEV_PM_OPS	NULL
 #endif /* CONFIG_PM */
 
-static struct platform_driver w83627hf_driver = {
-	.driver = {
-		.name	= DRVNAME,
-		.pm	= W83627HF_DEV_PM_OPS,
-	},
-	.probe		= w83627hf_probe,
-	.remove		= w83627hf_remove,
-};
-
-static ssize_t
-in_input_show(struct device *dev, struct device_attribute *devattr, char *buf)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in[nr]));
-}
-static ssize_t
-in_min_show(struct device *dev, struct device_attribute *devattr, char *buf)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in_min[nr]));
-}
-static ssize_t
-in_max_show(struct device *dev, struct device_attribute *devattr, char *buf)
+static int w83627thf_read_gpio5(struct platform_device *pdev)
 {
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in_max[nr]));
+	struct w83627hf_sio_data *sio_data = dev_get_platdata(&pdev->dev);
+	int res = 0xff, sel;
+
+	if (superio_enter(sio_data)) {
+		/*
+		 * Some other driver reserved the address space for itself.
+		 * We don't want to fail driver instantiation because of that,
+		 * so display a warning and keep going.
+		 */
+		dev_warn(&pdev->dev,
+			 "Can not read VID data: Failed to enable SuperIO access\n");
+		return res;
+	}
+
+	superio_select(sio_data, W83627HF_LD_GPIO5);
+
+	res = 0xff;
+
+	/* Make sure these GPIO pins are enabled */
+	if (!(superio_inb(sio_data, W83627THF_GPIO5_EN) & (1<<3))) {
+		dev_dbg(&pdev->dev, "GPIO5 disabled, no VID function\n");
+		goto exit;
+	}
+
+	/*
+	 * Make sure the pins are configured for input
+	 * There must be at least five (VRM 9), and possibly 6 (VRM 10)
+	 */
+	sel = superio_inb(sio_data, W83627THF_GPIO5_IOSR) & 0x3f;
+	if ((sel & 0x1f) != 0x1f) {
+		dev_dbg(&pdev->dev, "GPIO5 not configured for VID "
+			"function\n");
+		goto exit;
+	}
+
+	dev_info(&pdev->dev, "Reading VID from GPIO5\n");
+	res = superio_inb(sio_data, W83627THF_GPIO5_DR) & sel;
+
+exit:
+	superio_exit(sio_data);
+	return res;
 }
-static ssize_t
-in_min_store(struct device *dev, struct device_attribute *devattr,
-	     const char *buf, size_t count)
+
+static int w83687thf_read_vid(struct platform_device *pdev)
 {
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-	long val;
-	int err;
+	struct w83627hf_sio_data *sio_data = dev_get_platdata(&pdev->dev);
+	int res = 0xff;
 
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
+	if (superio_enter(sio_data)) {
+		/*
+		 * Some other driver reserved the address space for itself.
+		 * We don't want to fail driver instantiation because of that,
+		 * so display a warning and keep going.
+		 */
+		dev_warn(&pdev->dev,
+			 "Can not read VID data: Failed to enable SuperIO access\n");
+		return res;
+	}
 
-	mutex_lock(&data->update_lock);
-	data->in_min[nr] = IN_TO_REG(val);
-	w83627hf_write_value(data, W83781D_REG_IN_MIN(nr), data->in_min[nr]);
-	mutex_unlock(&data->update_lock);
-	return count;
+	superio_select(sio_data, W83627HF_LD_HWM);
+
+	/* Make sure these GPIO pins are enabled */
+	if (!(superio_inb(sio_data, W83687THF_VID_EN) & (1 << 2))) {
+		dev_dbg(&pdev->dev, "VID disabled, no VID function\n");
+		goto exit;
+	}
+
+	/* Make sure the pins are configured for input */
+	if (!(superio_inb(sio_data, W83687THF_VID_CFG) & (1 << 4))) {
+		dev_dbg(&pdev->dev, "VID configured as output, "
+			"no VID function\n");
+		goto exit;
+	}
+
+	res = superio_inb(sio_data, W83687THF_VID_DATA) & 0x3f;
+
+exit:
+	superio_exit(sio_data);
+	return res;
 }
-static ssize_t
-in_max_store(struct device *dev, struct device_attribute *devattr,
-	     const char *buf, size_t count)
+
+static void w83627hf_init_device(struct platform_device *pdev)
 {
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-	long val;
-	int err;
+	struct w83627hf_data *data = platform_get_drvdata(pdev);
+	int i;
+	enum chips type = data->type;
+	u8 tmp;
 
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
+	/* Minimize conflicts with other winbond i2c-only clients...  */
+	/* disable i2c subclients... how to disable main i2c client?? */
+	/* force i2c address to relatively uncommon address */
+	if (type == w83627hf) {
+		w83627hf_write_value(data, W83781D_REG_I2C_SUBADDR, 0x89);
+		w83627hf_write_value(data, W83781D_REG_I2C_ADDR, force_i2c);
+	}
 
-	mutex_lock(&data->update_lock);
-	data->in_max[nr] = IN_TO_REG(val);
-	w83627hf_write_value(data, W83781D_REG_IN_MAX(nr), data->in_max[nr]);
-	mutex_unlock(&data->update_lock);
-	return count;
-}
+	/* Read VID only once */
+	if (type == w83627hf || type == w83637hf) {
+		int lo = w83627hf_read_value(data, W83781D_REG_VID_FANDIV);
+		int hi = w83627hf_read_value(data, W83781D_REG_CHIPID);
+		data->vid = (lo & 0x0f) | ((hi & 0x01) << 4);
+	} else if (type == w83627thf) {
+		data->vid = w83627thf_read_gpio5(pdev);
+	} else if (type == w83687thf) {
+		data->vid = w83687thf_read_vid(pdev);
+	}
 
-static SENSOR_DEVICE_ATTR_RO(in1_input, in_input, 1);
-static SENSOR_DEVICE_ATTR_RW(in1_min, in_min, 1);
-static SENSOR_DEVICE_ATTR_RW(in1_max, in_max, 1);
-static SENSOR_DEVICE_ATTR_RO(in2_input, in_input, 2);
-static SENSOR_DEVICE_ATTR_RW(in2_min, in_min, 2);
-static SENSOR_DEVICE_ATTR_RW(in2_max, in_max, 2);
-static SENSOR_DEVICE_ATTR_RO(in3_input, in_input, 3);
-static SENSOR_DEVICE_ATTR_RW(in3_min, in_min, 3);
-static SENSOR_DEVICE_ATTR_RW(in3_max, in_max, 3);
-static SENSOR_DEVICE_ATTR_RO(in4_input, in_input, 4);
-static SENSOR_DEVICE_ATTR_RW(in4_min, in_min, 4);
-static SENSOR_DEVICE_ATTR_RW(in4_max, in_max, 4);
-static SENSOR_DEVICE_ATTR_RO(in5_input, in_input, 5);
-static SENSOR_DEVICE_ATTR_RW(in5_min, in_min, 5);
-static SENSOR_DEVICE_ATTR_RW(in5_max, in_max, 5);
-static SENSOR_DEVICE_ATTR_RO(in6_input, in_input, 6);
-static SENSOR_DEVICE_ATTR_RW(in6_min, in_min, 6);
-static SENSOR_DEVICE_ATTR_RW(in6_max, in_max, 6);
-static SENSOR_DEVICE_ATTR_RO(in7_input, in_input, 7);
-static SENSOR_DEVICE_ATTR_RW(in7_min, in_min, 7);
-static SENSOR_DEVICE_ATTR_RW(in7_max, in_max, 7);
-static SENSOR_DEVICE_ATTR_RO(in8_input, in_input, 8);
-static SENSOR_DEVICE_ATTR_RW(in8_min, in_min, 8);
-static SENSOR_DEVICE_ATTR_RW(in8_max, in_max, 8);
+	/* Read VRM & OVT Config only once */
+	if (type == w83627thf || type == w83637hf || type == w83687thf) {
+		data->vrm_ovt =
+			w83627hf_read_value(data, W83627THF_REG_VRM_OVT_CFG);
+	}
+
+	tmp = w83627hf_read_value(data, W83781D_REG_SCFG1);
+	for (i = 1; i <= 3; i++) {
+		if (!(tmp & BIT_SCFG1[i - 1])) {
+			data->sens[i - 1] = 4;
+		} else {
+			if (w83627hf_read_value
+			    (data,
+			     W83781D_REG_SCFG2) & BIT_SCFG2[i - 1])
+				data->sens[i - 1] = 1;
+			else
+				data->sens[i - 1] = 2;
+		}
+		if ((type == w83697hf) && (i == 2))
+			break;
+	}
+
+	if(init) {
+		/* Enable temp2 */
+		tmp = w83627hf_read_value(data, W83627HF_REG_TEMP2_CONFIG);
+		if (tmp & 0x01) {
+			dev_warn(&pdev->dev, "Enabling temp2, readings "
+				 "might not make sense\n");
+			w83627hf_write_value(data, W83627HF_REG_TEMP2_CONFIG,
+				tmp & 0xfe);
+		}
+
+		/* Enable temp3 */
+		if (type != w83697hf) {
+			tmp = w83627hf_read_value(data,
+				W83627HF_REG_TEMP3_CONFIG);
+			if (tmp & 0x01) {
+				dev_warn(&pdev->dev, "Enabling temp3, "
+					 "readings might not make sense\n");
+				w83627hf_write_value(data,
+					W83627HF_REG_TEMP3_CONFIG, tmp & 0xfe);
+			}
+		}
+	}
+
+	/* Start monitoring */
+	w83627hf_write_value(data, W83781D_REG_CONFIG,
+			    (w83627hf_read_value(data,
+						W83781D_REG_CONFIG) & 0xf7)
+			    | 0x01);
+
+	/* Enable VBAT monitoring if needed */
+	tmp = w83627hf_read_value(data, W83781D_REG_VBAT);
+	if (!(tmp & 0x01))
+		w83627hf_write_value(data, W83781D_REG_VBAT, tmp | 0x01);
+}
 
 /* use a different set of functions for in0 */
 static ssize_t show_in_0(struct w83627hf_data *data, char *buf, u8 reg)
@@ -582,6 +824,7 @@ static ssize_t in0_input_show(struct device *dev,
 	struct w83627hf_data *data = w83627hf_update_device(dev);
 	return show_in_0(data, buf, data->in[0]);
 }
+static DEVICE_ATTR_RO(in0_input);
 
 static ssize_t in0_min_show(struct device *dev, struct device_attribute *attr,
 			    char *buf)
@@ -590,13 +833,6 @@ static ssize_t in0_min_show(struct device *dev, struct device_attribute *attr,
 	return show_in_0(data, buf, data->in_min[0]);
 }
 
-static ssize_t in0_max_show(struct device *dev, struct device_attribute *attr,
-			    char *buf)
-{
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return show_in_0(data, buf, data->in_max[0]);
-}
-
 static ssize_t in0_min_store(struct device *dev,
 			     struct device_attribute *attr, const char *buf,
 			     size_t count)
@@ -627,6 +863,15 @@ static ssize_t in0_min_store(struct device *dev,
 	return count;
 }
 
+static DEVICE_ATTR_RW(in0_min);
+
+static ssize_t in0_max_show(struct device *dev, struct device_attribute *attr,
+			    char *buf)
+{
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return show_in_0(data, buf, data->in_max[0]);
+}
+
 static ssize_t in0_max_store(struct device *dev,
 			     struct device_attribute *attr, const char *buf,
 			     size_t count)
@@ -657,193 +902,16 @@ static ssize_t in0_max_store(struct device *dev,
 	return count;
 }
 
-static DEVICE_ATTR_RO(in0_input);
-static DEVICE_ATTR_RW(in0_min);
 static DEVICE_ATTR_RW(in0_max);
 
 static ssize_t
-fan_input_show(struct device *dev, struct device_attribute *devattr,
-	       char *buf)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return sprintf(buf, "%ld\n", FAN_FROM_REG(data->fan[nr],
-				(long)DIV_FROM_REG(data->fan_div[nr])));
-}
-static ssize_t
-fan_min_show(struct device *dev, struct device_attribute *devattr, char *buf)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return sprintf(buf, "%ld\n", FAN_FROM_REG(data->fan_min[nr],
-				(long)DIV_FROM_REG(data->fan_div[nr])));
-}
-static ssize_t
-fan_min_store(struct device *dev, struct device_attribute *devattr,
-	      const char *buf, size_t count)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-	unsigned long val;
-	int err;
-
-	err = kstrtoul(buf, 10, &val);
-	if (err)
-		return err;
-
-	mutex_lock(&data->update_lock);
-	data->fan_min[nr] = FAN_TO_REG(val, DIV_FROM_REG(data->fan_div[nr]));
-	w83627hf_write_value(data, W83627HF_REG_FAN_MIN(nr),
-			     data->fan_min[nr]);
-
-	mutex_unlock(&data->update_lock);
-	return count;
-}
-
-static SENSOR_DEVICE_ATTR_RO(fan1_input, fan_input, 0);
-static SENSOR_DEVICE_ATTR_RW(fan1_min, fan_min, 0);
-static SENSOR_DEVICE_ATTR_RO(fan2_input, fan_input, 1);
-static SENSOR_DEVICE_ATTR_RW(fan2_min, fan_min, 1);
-static SENSOR_DEVICE_ATTR_RO(fan3_input, fan_input, 2);
-static SENSOR_DEVICE_ATTR_RW(fan3_min, fan_min, 2);
-
-static ssize_t
-temp_show(struct device *dev, struct device_attribute *devattr, char *buf)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-
-	u16 tmp = data->temp[nr];
-	return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp)
-					  : (long) TEMP_FROM_REG(tmp));
-}
-
-static ssize_t
-temp_max_show(struct device *dev, struct device_attribute *devattr, char *buf)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-
-	u16 tmp = data->temp_max[nr];
-	return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp)
-					  : (long) TEMP_FROM_REG(tmp));
-}
-
-static ssize_t
-temp_max_hyst_show(struct device *dev, struct device_attribute *devattr,
-		   char *buf)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-
-	u16 tmp = data->temp_max_hyst[nr];
-	return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp)
-					  : (long) TEMP_FROM_REG(tmp));
-}
-
-static ssize_t
-temp_max_store(struct device *dev, struct device_attribute *devattr,
-	       const char *buf, size_t count)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-	u16 tmp;
-	long val;
-	int err;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	tmp = (nr) ? LM75_TEMP_TO_REG(val) : TEMP_TO_REG(val);
-	mutex_lock(&data->update_lock);
-	data->temp_max[nr] = tmp;
-	w83627hf_write_value(data, w83627hf_reg_temp_over[nr], tmp);
-	mutex_unlock(&data->update_lock);
-	return count;
-}
-
-static ssize_t
-temp_max_hyst_store(struct device *dev, struct device_attribute *devattr,
-		    const char *buf, size_t count)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-	u16 tmp;
-	long val;
-	int err;
-
-	err = kstrtol(buf, 10, &val);
-	if (err)
-		return err;
-
-	tmp = (nr) ? LM75_TEMP_TO_REG(val) : TEMP_TO_REG(val);
-	mutex_lock(&data->update_lock);
-	data->temp_max_hyst[nr] = tmp;
-	w83627hf_write_value(data, w83627hf_reg_temp_hyst[nr], tmp);
-	mutex_unlock(&data->update_lock);
-	return count;
-}
-
-static SENSOR_DEVICE_ATTR_RO(temp1_input, temp, 0);
-static SENSOR_DEVICE_ATTR_RW(temp1_max, temp_max, 0);
-static SENSOR_DEVICE_ATTR_RW(temp1_max_hyst, temp_max_hyst, 0);
-static SENSOR_DEVICE_ATTR_RO(temp2_input, temp, 1);
-static SENSOR_DEVICE_ATTR_RW(temp2_max, temp_max, 1);
-static SENSOR_DEVICE_ATTR_RW(temp2_max_hyst, temp_max_hyst, 1);
-static SENSOR_DEVICE_ATTR_RO(temp3_input, temp, 2);
-static SENSOR_DEVICE_ATTR_RW(temp3_max, temp_max, 2);
-static SENSOR_DEVICE_ATTR_RW(temp3_max_hyst, temp_max_hyst, 2);
-
-static ssize_t
-cpu0_vid_show(struct device *dev, struct device_attribute *attr, char *buf)
-{
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return sprintf(buf, "%ld\n", (long) vid_from_reg(data->vid, data->vrm));
-}
-static DEVICE_ATTR_RO(cpu0_vid);
-
-static ssize_t
-vrm_show(struct device *dev, struct device_attribute *attr, char *buf)
-{
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-	return sprintf(buf, "%ld\n", (long) data->vrm);
-}
-static ssize_t
-vrm_store(struct device *dev, struct device_attribute *attr, const char *buf,
-	  size_t count)
-{
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-	unsigned long val;
-	int err;
-
-	err = kstrtoul(buf, 10, &val);
-	if (err)
-		return err;
-
-	if (val > 255)
-		return -EINVAL;
-	data->vrm = val;
-
-	return count;
-}
-static DEVICE_ATTR_RW(vrm);
-
-static ssize_t
-alarms_show(struct device *dev, struct device_attribute *attr, char *buf)
-{
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return sprintf(buf, "%ld\n", (long) data->alarms);
-}
-static DEVICE_ATTR_RO(alarms);
-
-static ssize_t
 alarm_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct w83627hf_data *data = w83627hf_update_device(dev);
 	int bitnr = to_sensor_dev_attr(attr)->index;
 	return sprintf(buf, "%u\n", (data->alarms >> bitnr) & 1);
 }
+
 static SENSOR_DEVICE_ATTR_RO(in0_alarm, alarm, 0);
 static SENSOR_DEVICE_ATTR_RO(in1_alarm, alarm, 1);
 static SENSOR_DEVICE_ATTR_RO(in2_alarm, alarm, 2);
@@ -861,44 +929,6 @@ static SENSOR_DEVICE_ATTR_RO(temp2_alarm, alarm, 5);
 static SENSOR_DEVICE_ATTR_RO(temp3_alarm, alarm, 13);
 
 static ssize_t
-beep_mask_show(struct device *dev, struct device_attribute *attr, char *buf)
-{
-	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return sprintf(buf, "%ld\n",
-		      (long)BEEP_MASK_FROM_REG(data->beep_mask));
-}
-
-static ssize_t
-beep_mask_store(struct device *dev, struct device_attribute *attr,
-		const char *buf, size_t count)
-{
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-	unsigned long val;
-	int err;
-
-	err = kstrtoul(buf, 10, &val);
-	if (err)
-		return err;
-
-	mutex_lock(&data->update_lock);
-
-	/* preserve beep enable */
-	data->beep_mask = (data->beep_mask & 0x8000)
-			| BEEP_MASK_TO_REG(val);
-	w83627hf_write_value(data, W83781D_REG_BEEP_INTS1,
-			    data->beep_mask & 0xff);
-	w83627hf_write_value(data, W83781D_REG_BEEP_INTS3,
-			    ((data->beep_mask) >> 16) & 0xff);
-	w83627hf_write_value(data, W83781D_REG_BEEP_INTS2,
-			    (data->beep_mask >> 8) & 0xff);
-
-	mutex_unlock(&data->update_lock);
-	return count;
-}
-
-static DEVICE_ATTR_RW(beep_mask);
-
-static ssize_t
 beep_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct w83627hf_data *data = w83627hf_update_device(dev);
@@ -974,6 +1004,143 @@ static SENSOR_DEVICE_ATTR_RW(temp3_beep, beep, 13);
 static SENSOR_DEVICE_ATTR_RW(beep_enable, beep, 15);
 
 static ssize_t
+in_input_show(struct device *dev, struct device_attribute *devattr, char *buf)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in[nr]));
+}
+
+static ssize_t
+in_min_show(struct device *dev, struct device_attribute *devattr, char *buf)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in_min[nr]));
+}
+
+static ssize_t
+in_min_store(struct device *dev, struct device_attribute *devattr,
+	     const char *buf, size_t count)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+	long val;
+	int err;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+	data->in_min[nr] = IN_TO_REG(val);
+	w83627hf_write_value(data, W83781D_REG_IN_MIN(nr), data->in_min[nr]);
+	mutex_unlock(&data->update_lock);
+	return count;
+}
+
+static ssize_t
+in_max_show(struct device *dev, struct device_attribute *devattr, char *buf)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return sprintf(buf, "%ld\n", (long)IN_FROM_REG(data->in_max[nr]));
+}
+
+static ssize_t
+in_max_store(struct device *dev, struct device_attribute *devattr,
+	     const char *buf, size_t count)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+	long val;
+	int err;
+
+	err = kstrtol(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+	data->in_max[nr] = IN_TO_REG(val);
+	w83627hf_write_value(data, W83781D_REG_IN_MAX(nr), data->in_max[nr]);
+	mutex_unlock(&data->update_lock);
+	return count;
+}
+
+static SENSOR_DEVICE_ATTR_RO(in1_input, in_input, 1);
+static SENSOR_DEVICE_ATTR_RW(in1_min, in_min, 1);
+static SENSOR_DEVICE_ATTR_RW(in1_max, in_max, 1);
+static SENSOR_DEVICE_ATTR_RO(in2_input, in_input, 2);
+static SENSOR_DEVICE_ATTR_RW(in2_min, in_min, 2);
+static SENSOR_DEVICE_ATTR_RW(in2_max, in_max, 2);
+static SENSOR_DEVICE_ATTR_RO(in3_input, in_input, 3);
+static SENSOR_DEVICE_ATTR_RW(in3_min, in_min, 3);
+static SENSOR_DEVICE_ATTR_RW(in3_max, in_max, 3);
+static SENSOR_DEVICE_ATTR_RO(in4_input, in_input, 4);
+static SENSOR_DEVICE_ATTR_RW(in4_min, in_min, 4);
+static SENSOR_DEVICE_ATTR_RW(in4_max, in_max, 4);
+static SENSOR_DEVICE_ATTR_RO(in5_input, in_input, 5);
+static SENSOR_DEVICE_ATTR_RW(in5_min, in_min, 5);
+static SENSOR_DEVICE_ATTR_RW(in5_max, in_max, 5);
+static SENSOR_DEVICE_ATTR_RO(in6_input, in_input, 6);
+static SENSOR_DEVICE_ATTR_RW(in6_min, in_min, 6);
+static SENSOR_DEVICE_ATTR_RW(in6_max, in_max, 6);
+static SENSOR_DEVICE_ATTR_RO(in7_input, in_input, 7);
+static SENSOR_DEVICE_ATTR_RW(in7_min, in_min, 7);
+static SENSOR_DEVICE_ATTR_RW(in7_max, in_max, 7);
+static SENSOR_DEVICE_ATTR_RO(in8_input, in_input, 8);
+static SENSOR_DEVICE_ATTR_RW(in8_min, in_min, 8);
+static SENSOR_DEVICE_ATTR_RW(in8_max, in_max, 8);
+
+static ssize_t
+fan_input_show(struct device *dev, struct device_attribute *devattr,
+	       char *buf)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return sprintf(buf, "%ld\n", FAN_FROM_REG(data->fan[nr],
+				(long)DIV_FROM_REG(data->fan_div[nr])));
+}
+
+static ssize_t
+fan_min_show(struct device *dev, struct device_attribute *devattr, char *buf)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return sprintf(buf, "%ld\n", FAN_FROM_REG(data->fan_min[nr],
+				(long)DIV_FROM_REG(data->fan_div[nr])));
+}
+
+static ssize_t
+fan_min_store(struct device *dev, struct device_attribute *devattr,
+	      const char *buf, size_t count)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+	unsigned long val;
+	int err;
+
+	err = kstrtoul(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+	data->fan_min[nr] = FAN_TO_REG(val, DIV_FROM_REG(data->fan_div[nr]));
+	w83627hf_write_value(data, W83627HF_REG_FAN_MIN(nr),
+			     data->fan_min[nr]);
+
+	mutex_unlock(&data->update_lock);
+	return count;
+}
+
+static SENSOR_DEVICE_ATTR_RO(fan1_input, fan_input, 0);
+static SENSOR_DEVICE_ATTR_RW(fan1_min, fan_min, 0);
+static SENSOR_DEVICE_ATTR_RO(fan2_input, fan_input, 1);
+static SENSOR_DEVICE_ATTR_RW(fan2_min, fan_min, 1);
+static SENSOR_DEVICE_ATTR_RO(fan3_input, fan_input, 2);
+static SENSOR_DEVICE_ATTR_RW(fan3_min, fan_min, 2);
+
+static ssize_t
 fan_div_show(struct device *dev, struct device_attribute *devattr, char *buf)
 {
 	int nr = to_sensor_dev_attr(devattr)->index;
@@ -981,6 +1148,7 @@ fan_div_show(struct device *dev, struct device_attribute *devattr, char *buf)
 	return sprintf(buf, "%ld\n",
 		       (long) DIV_FROM_REG(data->fan_div[nr]));
 }
+
 /*
  * Note: we save and restore the fan minimum here, because its value is
  * determined in part by the fan divisor.  This follows the principle of
@@ -1033,138 +1201,92 @@ static SENSOR_DEVICE_ATTR_RW(fan2_div, fan_div, 1);
 static SENSOR_DEVICE_ATTR_RW(fan3_div, fan_div, 2);
 
 static ssize_t
-pwm_show(struct device *dev, struct device_attribute *devattr, char *buf)
+temp_show(struct device *dev, struct device_attribute *devattr, char *buf)
 {
 	int nr = to_sensor_dev_attr(devattr)->index;
 	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return sprintf(buf, "%ld\n", (long) data->pwm[nr]);
-}
 
-static ssize_t
-pwm_store(struct device *dev, struct device_attribute *devattr,
-	  const char *buf, size_t count)
-{
-	int nr = to_sensor_dev_attr(devattr)->index;
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-	unsigned long val;
-	int err;
-
-	err = kstrtoul(buf, 10, &val);
-	if (err)
-		return err;
-
-	mutex_lock(&data->update_lock);
-
-	if (data->type == w83627thf) {
-		/* bits 0-3 are reserved  in 627THF */
-		data->pwm[nr] = PWM_TO_REG(val) & 0xf0;
-		w83627hf_write_value(data,
-				     W836X7HF_REG_PWM(data->type, nr),
-				     data->pwm[nr] |
-				     (w83627hf_read_value(data,
-				     W836X7HF_REG_PWM(data->type, nr)) & 0x0f));
-	} else {
-		data->pwm[nr] = PWM_TO_REG(val);
-		w83627hf_write_value(data,
-				     W836X7HF_REG_PWM(data->type, nr),
-				     data->pwm[nr]);
-	}
-
-	mutex_unlock(&data->update_lock);
-	return count;
+	u16 tmp = data->temp[nr];
+	return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp)
+					  : (long) TEMP_FROM_REG(tmp));
 }
 
-static SENSOR_DEVICE_ATTR_RW(pwm1, pwm, 0);
-static SENSOR_DEVICE_ATTR_RW(pwm2, pwm, 1);
-static SENSOR_DEVICE_ATTR_RW(pwm3, pwm, 2);
-
 static ssize_t
-pwm_enable_show(struct device *dev, struct device_attribute *devattr,
-		char *buf)
+temp_max_show(struct device *dev, struct device_attribute *devattr, char *buf)
 {
 	int nr = to_sensor_dev_attr(devattr)->index;
 	struct w83627hf_data *data = w83627hf_update_device(dev);
-	return sprintf(buf, "%d\n", data->pwm_enable[nr]);
+
+	u16 tmp = data->temp_max[nr];
+	return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp)
+					  : (long) TEMP_FROM_REG(tmp));
 }
 
 static ssize_t
-pwm_enable_store(struct device *dev, struct device_attribute *devattr,
-		 const char *buf, size_t count)
+temp_max_store(struct device *dev, struct device_attribute *devattr,
+	       const char *buf, size_t count)
 {
 	int nr = to_sensor_dev_attr(devattr)->index;
 	struct w83627hf_data *data = dev_get_drvdata(dev);
-	u8 reg;
-	unsigned long val;
+	u16 tmp;
+	long val;
 	int err;
 
-	err = kstrtoul(buf, 10, &val);
+	err = kstrtol(buf, 10, &val);
 	if (err)
 		return err;
 
-	if (!val || val > 3)	/* modes 1, 2 and 3 are supported */
-		return -EINVAL;
+	tmp = (nr) ? LM75_TEMP_TO_REG(val) : TEMP_TO_REG(val);
 	mutex_lock(&data->update_lock);
-	data->pwm_enable[nr] = val;
-	reg = w83627hf_read_value(data, W83627THF_REG_PWM_ENABLE[nr]);
-	reg &= ~(0x03 << W83627THF_PWM_ENABLE_SHIFT[nr]);
-	reg |= (val - 1) << W83627THF_PWM_ENABLE_SHIFT[nr];
-	w83627hf_write_value(data, W83627THF_REG_PWM_ENABLE[nr], reg);
+	data->temp_max[nr] = tmp;
+	w83627hf_write_value(data, w83627hf_reg_temp_over[nr], tmp);
 	mutex_unlock(&data->update_lock);
 	return count;
 }
 
-static SENSOR_DEVICE_ATTR_RW(pwm1_enable, pwm_enable, 0);
-static SENSOR_DEVICE_ATTR_RW(pwm2_enable, pwm_enable, 1);
-static SENSOR_DEVICE_ATTR_RW(pwm3_enable, pwm_enable, 2);
-
 static ssize_t
-pwm_freq_show(struct device *dev, struct device_attribute *devattr, char *buf)
+temp_max_hyst_show(struct device *dev, struct device_attribute *devattr,
+		   char *buf)
 {
 	int nr = to_sensor_dev_attr(devattr)->index;
 	struct w83627hf_data *data = w83627hf_update_device(dev);
-	if (data->type == w83627hf)
-		return sprintf(buf, "%ld\n",
-			pwm_freq_from_reg_627hf(data->pwm_freq[nr]));
-	else
-		return sprintf(buf, "%ld\n",
-			pwm_freq_from_reg(data->pwm_freq[nr]));
+
+	u16 tmp = data->temp_max_hyst[nr];
+	return sprintf(buf, "%ld\n", (nr) ? (long) LM75_TEMP_FROM_REG(tmp)
+					  : (long) TEMP_FROM_REG(tmp));
 }
 
 static ssize_t
-pwm_freq_store(struct device *dev, struct device_attribute *devattr,
-	       const char *buf, size_t count)
+temp_max_hyst_store(struct device *dev, struct device_attribute *devattr,
+		    const char *buf, size_t count)
 {
 	int nr = to_sensor_dev_attr(devattr)->index;
 	struct w83627hf_data *data = dev_get_drvdata(dev);
-	static const u8 mask[]={0xF8, 0x8F};
-	unsigned long val;
+	u16 tmp;
+	long val;
 	int err;
 
-	err = kstrtoul(buf, 10, &val);
+	err = kstrtol(buf, 10, &val);
 	if (err)
 		return err;
 
+	tmp = (nr) ? LM75_TEMP_TO_REG(val) : TEMP_TO_REG(val);
 	mutex_lock(&data->update_lock);
-
-	if (data->type == w83627hf) {
-		data->pwm_freq[nr] = pwm_freq_to_reg_627hf(val);
-		w83627hf_write_value(data, W83627HF_REG_PWM_FREQ,
-				(data->pwm_freq[nr] << (nr*4)) |
-				(w83627hf_read_value(data,
-				W83627HF_REG_PWM_FREQ) & mask[nr]));
-	} else {
-		data->pwm_freq[nr] = pwm_freq_to_reg(val);
-		w83627hf_write_value(data, W83637HF_REG_PWM_FREQ[nr],
-				data->pwm_freq[nr]);
-	}
-
+	data->temp_max_hyst[nr] = tmp;
+	w83627hf_write_value(data, w83627hf_reg_temp_hyst[nr], tmp);
 	mutex_unlock(&data->update_lock);
 	return count;
 }
 
-static SENSOR_DEVICE_ATTR_RW(pwm1_freq, pwm_freq, 0);
-static SENSOR_DEVICE_ATTR_RW(pwm2_freq, pwm_freq, 1);
-static SENSOR_DEVICE_ATTR_RW(pwm3_freq, pwm_freq, 2);
+static SENSOR_DEVICE_ATTR_RO(temp1_input, temp, 0);
+static SENSOR_DEVICE_ATTR_RW(temp1_max, temp_max, 0);
+static SENSOR_DEVICE_ATTR_RW(temp1_max_hyst, temp_max_hyst, 0);
+static SENSOR_DEVICE_ATTR_RO(temp2_input, temp, 1);
+static SENSOR_DEVICE_ATTR_RW(temp2_max, temp_max, 1);
+static SENSOR_DEVICE_ATTR_RW(temp2_max_hyst, temp_max_hyst, 1);
+static SENSOR_DEVICE_ATTR_RO(temp3_input, temp, 2);
+static SENSOR_DEVICE_ATTR_RW(temp3_max, temp_max, 2);
+static SENSOR_DEVICE_ATTR_RW(temp3_max_hyst, temp_max_hyst, 2);
 
 static ssize_t
 temp_type_show(struct device *dev, struct device_attribute *devattr,
@@ -1236,81 +1358,12 @@ static SENSOR_DEVICE_ATTR_RW(temp2_type, temp_type, 1);
 static SENSOR_DEVICE_ATTR_RW(temp3_type, temp_type, 2);
 
 static ssize_t
-name_show(struct device *dev, struct device_attribute *devattr, char *buf)
-{
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-
-	return sprintf(buf, "%s\n", data->name);
-}
-static DEVICE_ATTR_RO(name);
-
-static int __init w83627hf_find(int sioaddr, unsigned short *addr,
-				struct w83627hf_sio_data *sio_data)
+alarms_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
-	int err;
-	u16 val;
-
-	static __initconst char *const names[] = {
-		"W83627HF",
-		"W83627THF",
-		"W83697HF",
-		"W83637HF",
-		"W83687THF",
-	};
-
-	sio_data->sioaddr = sioaddr;
-	err = superio_enter(sio_data);
-	if (err)
-		return err;
-
-	err = -ENODEV;
-	val = force_id ? force_id : superio_inb(sio_data, DEVID);
-	switch (val) {
-	case W627_DEVID:
-		sio_data->type = w83627hf;
-		break;
-	case W627THF_DEVID:
-		sio_data->type = w83627thf;
-		break;
-	case W697_DEVID:
-		sio_data->type = w83697hf;
-		break;
-	case W637_DEVID:
-		sio_data->type = w83637hf;
-		break;
-	case W687THF_DEVID:
-		sio_data->type = w83687thf;
-		break;
-	case 0xff:	/* No device at all */
-		goto exit;
-	default:
-		pr_debug(DRVNAME ": Unsupported chip (DEVID=0x%02x)\n", val);
-		goto exit;
-	}
-
-	superio_select(sio_data, W83627HF_LD_HWM);
-	val = (superio_inb(sio_data, WINB_BASE_REG) << 8) |
-	       superio_inb(sio_data, WINB_BASE_REG + 1);
-	*addr = val & WINB_ALIGNMENT;
-	if (*addr == 0) {
-		pr_warn("Base address not set, skipping\n");
-		goto exit;
-	}
-
-	val = superio_inb(sio_data, WINB_ACT_REG);
-	if (!(val & 0x01)) {
-		pr_warn("Enabling HWM logical device\n");
-		superio_outb(sio_data, WINB_ACT_REG, val | 0x01);
-	}
-
-	err = 0;
-	pr_info(DRVNAME ": Found %s chip at %#x\n",
-		names[sio_data->type], *addr);
-
- exit:
-	superio_exit(sio_data);
-	return err;
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return sprintf(buf, "%ld\n", (long) data->alarms);
 }
+static DEVICE_ATTR_RO(alarms);
 
 #define VIN_UNIT_ATTRS(_X_)	\
 	&sensor_dev_attr_in##_X_##_input.dev_attr.attr,		\
@@ -1334,6 +1387,100 @@ static int __init w83627hf_find(int sioaddr, unsigned short *addr,
 	&sensor_dev_attr_temp##_X_##_alarm.dev_attr.attr,	\
 	&sensor_dev_attr_temp##_X_##_beep.dev_attr.attr
 
+static ssize_t
+beep_mask_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return sprintf(buf, "%ld\n",
+		      (long)BEEP_MASK_FROM_REG(data->beep_mask));
+}
+
+static ssize_t
+beep_mask_store(struct device *dev, struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+	unsigned long val;
+	int err;
+
+	err = kstrtoul(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+
+	/* preserve beep enable */
+	data->beep_mask = (data->beep_mask & 0x8000)
+			| BEEP_MASK_TO_REG(val);
+	w83627hf_write_value(data, W83781D_REG_BEEP_INTS1,
+			    data->beep_mask & 0xff);
+	w83627hf_write_value(data, W83781D_REG_BEEP_INTS3,
+			    ((data->beep_mask) >> 16) & 0xff);
+	w83627hf_write_value(data, W83781D_REG_BEEP_INTS2,
+			    (data->beep_mask >> 8) & 0xff);
+
+	mutex_unlock(&data->update_lock);
+	return count;
+}
+
+static DEVICE_ATTR_RW(beep_mask);
+
+static ssize_t
+pwm_show(struct device *dev, struct device_attribute *devattr, char *buf)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return sprintf(buf, "%ld\n", (long) data->pwm[nr]);
+}
+
+static ssize_t
+pwm_store(struct device *dev, struct device_attribute *devattr,
+	  const char *buf, size_t count)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+	unsigned long val;
+	int err;
+
+	err = kstrtoul(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+
+	if (data->type == w83627thf) {
+		/* bits 0-3 are reserved  in 627THF */
+		data->pwm[nr] = PWM_TO_REG(val) & 0xf0;
+		w83627hf_write_value(data,
+				     W836X7HF_REG_PWM(data->type, nr),
+				     data->pwm[nr] |
+				     (w83627hf_read_value(data,
+				     W836X7HF_REG_PWM(data->type, nr)) & 0x0f));
+	} else {
+		data->pwm[nr] = PWM_TO_REG(val);
+		w83627hf_write_value(data,
+				     W836X7HF_REG_PWM(data->type, nr),
+				     data->pwm[nr]);
+	}
+
+	mutex_unlock(&data->update_lock);
+	return count;
+}
+
+static SENSOR_DEVICE_ATTR_RW(pwm1, pwm, 0);
+static SENSOR_DEVICE_ATTR_RW(pwm2, pwm, 1);
+static SENSOR_DEVICE_ATTR_RW(pwm3, pwm, 2);
+
+static ssize_t
+name_show(struct device *dev, struct device_attribute *devattr, char *buf)
+{
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+
+	return sprintf(buf, "%s\n", data->name);
+}
+
+static DEVICE_ATTR_RO(name);
+
 static struct attribute *w83627hf_attributes[] = {
 	&dev_attr_in0_input.attr,
 	&dev_attr_in0_min.attr,
@@ -1366,6 +1513,131 @@ static const struct attribute_group w83627hf_group = {
 	.attrs = w83627hf_attributes,
 };
 
+static ssize_t
+pwm_freq_show(struct device *dev, struct device_attribute *devattr, char *buf)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	if (data->type == w83627hf)
+		return sprintf(buf, "%ld\n",
+			pwm_freq_from_reg_627hf(data->pwm_freq[nr]));
+	else
+		return sprintf(buf, "%ld\n",
+			pwm_freq_from_reg(data->pwm_freq[nr]));
+}
+
+static ssize_t
+pwm_freq_store(struct device *dev, struct device_attribute *devattr,
+	       const char *buf, size_t count)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+	static const u8 mask[]={0xF8, 0x8F};
+	unsigned long val;
+	int err;
+
+	err = kstrtoul(buf, 10, &val);
+	if (err)
+		return err;
+
+	mutex_lock(&data->update_lock);
+
+	if (data->type == w83627hf) {
+		data->pwm_freq[nr] = pwm_freq_to_reg_627hf(val);
+		w83627hf_write_value(data, W83627HF_REG_PWM_FREQ,
+				(data->pwm_freq[nr] << (nr*4)) |
+				(w83627hf_read_value(data,
+				W83627HF_REG_PWM_FREQ) & mask[nr]));
+	} else {
+		data->pwm_freq[nr] = pwm_freq_to_reg(val);
+		w83627hf_write_value(data, W83637HF_REG_PWM_FREQ[nr],
+				data->pwm_freq[nr]);
+	}
+
+	mutex_unlock(&data->update_lock);
+	return count;
+}
+
+static SENSOR_DEVICE_ATTR_RW(pwm1_freq, pwm_freq, 0);
+static SENSOR_DEVICE_ATTR_RW(pwm2_freq, pwm_freq, 1);
+static SENSOR_DEVICE_ATTR_RW(pwm3_freq, pwm_freq, 2);
+
+static ssize_t
+cpu0_vid_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return sprintf(buf, "%ld\n", (long) vid_from_reg(data->vid, data->vrm));
+}
+
+static DEVICE_ATTR_RO(cpu0_vid);
+
+static ssize_t
+vrm_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+	return sprintf(buf, "%ld\n", (long) data->vrm);
+}
+
+static ssize_t
+vrm_store(struct device *dev, struct device_attribute *attr, const char *buf,
+	  size_t count)
+{
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+	unsigned long val;
+	int err;
+
+	err = kstrtoul(buf, 10, &val);
+	if (err)
+		return err;
+
+	if (val > 255)
+		return -EINVAL;
+	data->vrm = val;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(vrm);
+
+static ssize_t
+pwm_enable_show(struct device *dev, struct device_attribute *devattr,
+		char *buf)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = w83627hf_update_device(dev);
+	return sprintf(buf, "%d\n", data->pwm_enable[nr]);
+}
+
+static ssize_t
+pwm_enable_store(struct device *dev, struct device_attribute *devattr,
+		 const char *buf, size_t count)
+{
+	int nr = to_sensor_dev_attr(devattr)->index;
+	struct w83627hf_data *data = dev_get_drvdata(dev);
+	u8 reg;
+	unsigned long val;
+	int err;
+
+	err = kstrtoul(buf, 10, &val);
+	if (err)
+		return err;
+
+	if (!val || val > 3)	/* modes 1, 2 and 3 are supported */
+		return -EINVAL;
+	mutex_lock(&data->update_lock);
+	data->pwm_enable[nr] = val;
+	reg = w83627hf_read_value(data, W83627THF_REG_PWM_ENABLE[nr]);
+	reg &= ~(0x03 << W83627THF_PWM_ENABLE_SHIFT[nr]);
+	reg |= (val - 1) << W83627THF_PWM_ENABLE_SHIFT[nr];
+	w83627hf_write_value(data, W83627THF_REG_PWM_ENABLE[nr], reg);
+	mutex_unlock(&data->update_lock);
+	return count;
+}
+
+static SENSOR_DEVICE_ATTR_RW(pwm1_enable, pwm_enable, 0);
+static SENSOR_DEVICE_ATTR_RW(pwm2_enable, pwm_enable, 1);
+static SENSOR_DEVICE_ATTR_RW(pwm3_enable, pwm_enable, 2);
+
 static struct attribute *w83627hf_attributes_opt[] = {
 	VIN_UNIT_ATTRS(1),
 	VIN_UNIT_ATTRS(5),
@@ -1568,349 +1840,81 @@ static int w83627hf_remove(struct platform_device *pdev)
 	return 0;
 }
 
-/* Registers 0x50-0x5f are banked */
-static inline void w83627hf_set_bank(struct w83627hf_data *data, u16 reg)
-{
-	if ((reg & 0x00f0) == 0x50) {
-		outb_p(W83781D_REG_BANK, data->addr + W83781D_ADDR_REG_OFFSET);
-		outb_p(reg >> 8, data->addr + W83781D_DATA_REG_OFFSET);
-	}
-}
-
-/* Not strictly necessary, but play it safe for now */
-static inline void w83627hf_reset_bank(struct w83627hf_data *data, u16 reg)
-{
-	if (reg & 0xff00) {
-		outb_p(W83781D_REG_BANK, data->addr + W83781D_ADDR_REG_OFFSET);
-		outb_p(0, data->addr + W83781D_DATA_REG_OFFSET);
-	}
-}
-
-static int w83627hf_read_value(struct w83627hf_data *data, u16 reg)
-{
-	int res, word_sized;
-
-	mutex_lock(&data->lock);
-	word_sized = (((reg & 0xff00) == 0x100)
-		   || ((reg & 0xff00) == 0x200))
-		  && (((reg & 0x00ff) == 0x50)
-		   || ((reg & 0x00ff) == 0x53)
-		   || ((reg & 0x00ff) == 0x55));
-	w83627hf_set_bank(data, reg);
-	outb_p(reg & 0xff, data->addr + W83781D_ADDR_REG_OFFSET);
-	res = inb_p(data->addr + W83781D_DATA_REG_OFFSET);
-	if (word_sized) {
-		outb_p((reg & 0xff) + 1,
-		       data->addr + W83781D_ADDR_REG_OFFSET);
-		res =
-		    (res << 8) + inb_p(data->addr +
-				       W83781D_DATA_REG_OFFSET);
-	}
-	w83627hf_reset_bank(data, reg);
-	mutex_unlock(&data->lock);
-	return res;
-}
+static struct platform_driver w83627hf_driver = {
+	.driver = {
+		.name	= DRVNAME,
+		.pm	= W83627HF_DEV_PM_OPS,
+	},
+	.probe		= w83627hf_probe,
+	.remove		= w83627hf_remove,
+};
 
-static int w83627thf_read_gpio5(struct platform_device *pdev)
+static int __init w83627hf_find(int sioaddr, unsigned short *addr,
+				struct w83627hf_sio_data *sio_data)
 {
-	struct w83627hf_sio_data *sio_data = dev_get_platdata(&pdev->dev);
-	int res = 0xff, sel;
-
-	if (superio_enter(sio_data)) {
-		/*
-		 * Some other driver reserved the address space for itself.
-		 * We don't want to fail driver instantiation because of that,
-		 * so display a warning and keep going.
-		 */
-		dev_warn(&pdev->dev,
-			 "Can not read VID data: Failed to enable SuperIO access\n");
-		return res;
-	}
+	int err;
+	u16 val;
 
-	superio_select(sio_data, W83627HF_LD_GPIO5);
+	static __initconst char *const names[] = {
+		"W83627HF",
+		"W83627THF",
+		"W83697HF",
+		"W83637HF",
+		"W83687THF",
+	};
 
-	res = 0xff;
+	sio_data->sioaddr = sioaddr;
+	err = superio_enter(sio_data);
+	if (err)
+		return err;
 
-	/* Make sure these GPIO pins are enabled */
-	if (!(superio_inb(sio_data, W83627THF_GPIO5_EN) & (1<<3))) {
-		dev_dbg(&pdev->dev, "GPIO5 disabled, no VID function\n");
+	err = -ENODEV;
+	val = force_id ? force_id : superio_inb(sio_data, DEVID);
+	switch (val) {
+	case W627_DEVID:
+		sio_data->type = w83627hf;
+		break;
+	case W627THF_DEVID:
+		sio_data->type = w83627thf;
+		break;
+	case W697_DEVID:
+		sio_data->type = w83697hf;
+		break;
+	case W637_DEVID:
+		sio_data->type = w83637hf;
+		break;
+	case W687THF_DEVID:
+		sio_data->type = w83687thf;
+		break;
+	case 0xff:	/* No device at all */
 		goto exit;
-	}
-
-	/*
-	 * Make sure the pins are configured for input
-	 * There must be at least five (VRM 9), and possibly 6 (VRM 10)
-	 */
-	sel = superio_inb(sio_data, W83627THF_GPIO5_IOSR) & 0x3f;
-	if ((sel & 0x1f) != 0x1f) {
-		dev_dbg(&pdev->dev, "GPIO5 not configured for VID "
-			"function\n");
+	default:
+		pr_debug(DRVNAME ": Unsupported chip (DEVID=0x%02x)\n", val);
 		goto exit;
 	}
 
-	dev_info(&pdev->dev, "Reading VID from GPIO5\n");
-	res = superio_inb(sio_data, W83627THF_GPIO5_DR) & sel;
-
-exit:
-	superio_exit(sio_data);
-	return res;
-}
-
-static int w83687thf_read_vid(struct platform_device *pdev)
-{
-	struct w83627hf_sio_data *sio_data = dev_get_platdata(&pdev->dev);
-	int res = 0xff;
-
-	if (superio_enter(sio_data)) {
-		/*
-		 * Some other driver reserved the address space for itself.
-		 * We don't want to fail driver instantiation because of that,
-		 * so display a warning and keep going.
-		 */
-		dev_warn(&pdev->dev,
-			 "Can not read VID data: Failed to enable SuperIO access\n");
-		return res;
-	}
-
 	superio_select(sio_data, W83627HF_LD_HWM);
-
-	/* Make sure these GPIO pins are enabled */
-	if (!(superio_inb(sio_data, W83687THF_VID_EN) & (1 << 2))) {
-		dev_dbg(&pdev->dev, "VID disabled, no VID function\n");
+	val = (superio_inb(sio_data, WINB_BASE_REG) << 8) |
+	       superio_inb(sio_data, WINB_BASE_REG + 1);
+	*addr = val & WINB_ALIGNMENT;
+	if (*addr == 0) {
+		pr_warn("Base address not set, skipping\n");
 		goto exit;
 	}
 
-	/* Make sure the pins are configured for input */
-	if (!(superio_inb(sio_data, W83687THF_VID_CFG) & (1 << 4))) {
-		dev_dbg(&pdev->dev, "VID configured as output, "
-			"no VID function\n");
-		goto exit;
+	val = superio_inb(sio_data, WINB_ACT_REG);
+	if (!(val & 0x01)) {
+		pr_warn("Enabling HWM logical device\n");
+		superio_outb(sio_data, WINB_ACT_REG, val | 0x01);
 	}
 
-	res = superio_inb(sio_data, W83687THF_VID_DATA) & 0x3f;
+	err = 0;
+	pr_info(DRVNAME ": Found %s chip at %#x\n",
+		names[sio_data->type], *addr);
 
-exit:
+ exit:
 	superio_exit(sio_data);
-	return res;
-}
-
-static int w83627hf_write_value(struct w83627hf_data *data, u16 reg, u16 value)
-{
-	int word_sized;
-
-	mutex_lock(&data->lock);
-	word_sized = (((reg & 0xff00) == 0x100)
-		   || ((reg & 0xff00) == 0x200))
-		  && (((reg & 0x00ff) == 0x53)
-		   || ((reg & 0x00ff) == 0x55));
-	w83627hf_set_bank(data, reg);
-	outb_p(reg & 0xff, data->addr + W83781D_ADDR_REG_OFFSET);
-	if (word_sized) {
-		outb_p(value >> 8,
-		       data->addr + W83781D_DATA_REG_OFFSET);
-		outb_p((reg & 0xff) + 1,
-		       data->addr + W83781D_ADDR_REG_OFFSET);
-	}
-	outb_p(value & 0xff,
-	       data->addr + W83781D_DATA_REG_OFFSET);
-	w83627hf_reset_bank(data, reg);
-	mutex_unlock(&data->lock);
-	return 0;
-}
-
-static void w83627hf_init_device(struct platform_device *pdev)
-{
-	struct w83627hf_data *data = platform_get_drvdata(pdev);
-	int i;
-	enum chips type = data->type;
-	u8 tmp;
-
-	/* Minimize conflicts with other winbond i2c-only clients...  */
-	/* disable i2c subclients... how to disable main i2c client?? */
-	/* force i2c address to relatively uncommon address */
-	if (type == w83627hf) {
-		w83627hf_write_value(data, W83781D_REG_I2C_SUBADDR, 0x89);
-		w83627hf_write_value(data, W83781D_REG_I2C_ADDR, force_i2c);
-	}
-
-	/* Read VID only once */
-	if (type == w83627hf || type == w83637hf) {
-		int lo = w83627hf_read_value(data, W83781D_REG_VID_FANDIV);
-		int hi = w83627hf_read_value(data, W83781D_REG_CHIPID);
-		data->vid = (lo & 0x0f) | ((hi & 0x01) << 4);
-	} else if (type == w83627thf) {
-		data->vid = w83627thf_read_gpio5(pdev);
-	} else if (type == w83687thf) {
-		data->vid = w83687thf_read_vid(pdev);
-	}
-
-	/* Read VRM & OVT Config only once */
-	if (type == w83627thf || type == w83637hf || type == w83687thf) {
-		data->vrm_ovt = 
-			w83627hf_read_value(data, W83627THF_REG_VRM_OVT_CFG);
-	}
-
-	tmp = w83627hf_read_value(data, W83781D_REG_SCFG1);
-	for (i = 1; i <= 3; i++) {
-		if (!(tmp & BIT_SCFG1[i - 1])) {
-			data->sens[i - 1] = 4;
-		} else {
-			if (w83627hf_read_value
-			    (data,
-			     W83781D_REG_SCFG2) & BIT_SCFG2[i - 1])
-				data->sens[i - 1] = 1;
-			else
-				data->sens[i - 1] = 2;
-		}
-		if ((type == w83697hf) && (i == 2))
-			break;
-	}
-
-	if(init) {
-		/* Enable temp2 */
-		tmp = w83627hf_read_value(data, W83627HF_REG_TEMP2_CONFIG);
-		if (tmp & 0x01) {
-			dev_warn(&pdev->dev, "Enabling temp2, readings "
-				 "might not make sense\n");
-			w83627hf_write_value(data, W83627HF_REG_TEMP2_CONFIG,
-				tmp & 0xfe);
-		}
-
-		/* Enable temp3 */
-		if (type != w83697hf) {
-			tmp = w83627hf_read_value(data,
-				W83627HF_REG_TEMP3_CONFIG);
-			if (tmp & 0x01) {
-				dev_warn(&pdev->dev, "Enabling temp3, "
-					 "readings might not make sense\n");
-				w83627hf_write_value(data,
-					W83627HF_REG_TEMP3_CONFIG, tmp & 0xfe);
-			}
-		}
-	}
-
-	/* Start monitoring */
-	w83627hf_write_value(data, W83781D_REG_CONFIG,
-			    (w83627hf_read_value(data,
-						W83781D_REG_CONFIG) & 0xf7)
-			    | 0x01);
-
-	/* Enable VBAT monitoring if needed */
-	tmp = w83627hf_read_value(data, W83781D_REG_VBAT);
-	if (!(tmp & 0x01))
-		w83627hf_write_value(data, W83781D_REG_VBAT, tmp | 0x01);
-}
-
-static void w83627hf_update_fan_div(struct w83627hf_data *data)
-{
-	int reg;
-
-	reg = w83627hf_read_value(data, W83781D_REG_VID_FANDIV);
-	data->fan_div[0] = (reg >> 4) & 0x03;
-	data->fan_div[1] = (reg >> 6) & 0x03;
-	if (data->type != w83697hf) {
-		data->fan_div[2] = (w83627hf_read_value(data,
-				       W83781D_REG_PIN) >> 6) & 0x03;
-	}
-	reg = w83627hf_read_value(data, W83781D_REG_VBAT);
-	data->fan_div[0] |= (reg >> 3) & 0x04;
-	data->fan_div[1] |= (reg >> 4) & 0x04;
-	if (data->type != w83697hf)
-		data->fan_div[2] |= (reg >> 5) & 0x04;
-}
-
-static struct w83627hf_data *w83627hf_update_device(struct device *dev)
-{
-	struct w83627hf_data *data = dev_get_drvdata(dev);
-	int i, num_temps = (data->type == w83697hf) ? 2 : 3;
-	int num_pwms = (data->type == w83697hf) ? 2 : 3;
-
-	mutex_lock(&data->update_lock);
-
-	if (time_after(jiffies, data->last_updated + HZ + HZ / 2)
-	    || !data->valid) {
-		for (i = 0; i <= 8; i++) {
-			/* skip missing sensors */
-			if (((data->type == w83697hf) && (i == 1)) ||
-			    ((data->type != w83627hf && data->type != w83697hf)
-			    && (i == 5 || i == 6)))
-				continue;
-			data->in[i] =
-			    w83627hf_read_value(data, W83781D_REG_IN(i));
-			data->in_min[i] =
-			    w83627hf_read_value(data,
-					       W83781D_REG_IN_MIN(i));
-			data->in_max[i] =
-			    w83627hf_read_value(data,
-					       W83781D_REG_IN_MAX(i));
-		}
-		for (i = 0; i <= 2; i++) {
-			data->fan[i] =
-			    w83627hf_read_value(data, W83627HF_REG_FAN(i));
-			data->fan_min[i] =
-			    w83627hf_read_value(data,
-					       W83627HF_REG_FAN_MIN(i));
-		}
-		for (i = 0; i <= 2; i++) {
-			u8 tmp = w83627hf_read_value(data,
-				W836X7HF_REG_PWM(data->type, i));
- 			/* bits 0-3 are reserved  in 627THF */
- 			if (data->type == w83627thf)
-				tmp &= 0xf0;
-			data->pwm[i] = tmp;
-			if (i == 1 &&
-			    (data->type == w83627hf || data->type == w83697hf))
-				break;
-		}
-		if (data->type == w83627hf) {
-				u8 tmp = w83627hf_read_value(data,
-						W83627HF_REG_PWM_FREQ);
-				data->pwm_freq[0] = tmp & 0x07;
-				data->pwm_freq[1] = (tmp >> 4) & 0x07;
-		} else if (data->type != w83627thf) {
-			for (i = 1; i <= 3; i++) {
-				data->pwm_freq[i - 1] =
-					w83627hf_read_value(data,
-						W83637HF_REG_PWM_FREQ[i - 1]);
-				if (i == 2 && (data->type == w83697hf))
-					break;
-			}
-		}
-		if (data->type != w83627hf) {
-			for (i = 0; i < num_pwms; i++) {
-				u8 tmp = w83627hf_read_value(data,
-					W83627THF_REG_PWM_ENABLE[i]);
-				data->pwm_enable[i] =
-					((tmp >> W83627THF_PWM_ENABLE_SHIFT[i])
-					& 0x03) + 1;
-			}
-		}
-		for (i = 0; i < num_temps; i++) {
-			data->temp[i] = w83627hf_read_value(
-						data, w83627hf_reg_temp[i]);
-			data->temp_max[i] = w83627hf_read_value(
-						data, w83627hf_reg_temp_over[i]);
-			data->temp_max_hyst[i] = w83627hf_read_value(
-						data, w83627hf_reg_temp_hyst[i]);
-		}
-
-		w83627hf_update_fan_div(data);
-
-		data->alarms =
-		    w83627hf_read_value(data, W83781D_REG_ALARM1) |
-		    (w83627hf_read_value(data, W83781D_REG_ALARM2) << 8) |
-		    (w83627hf_read_value(data, W83781D_REG_ALARM3) << 16);
-		i = w83627hf_read_value(data, W83781D_REG_BEEP_INTS2);
-		data->beep_mask = (i << 8) |
-		    w83627hf_read_value(data, W83781D_REG_BEEP_INTS1) |
-		    w83627hf_read_value(data, W83781D_REG_BEEP_INTS3) << 16;
-		data->last_updated = jiffies;
-		data->valid = true;
-	}
-
-	mutex_unlock(&data->update_lock);
-
-	return data;
+	return err;
 }
 
 static int __init w83627hf_device_add(unsigned short address,
diff --git a/drivers/hwmon/w83781d.c b/drivers/hwmon/w83781d.c
index b3579721265f..dacabf25e83f 100644
--- a/drivers/hwmon/w83781d.c
+++ b/drivers/hwmon/w83781d.c
@@ -1171,7 +1171,7 @@ w83781d_detect(struct i2c_client *client, struct i2c_board_info *info)
 	if (isa)
 		mutex_unlock(&isa->update_lock);
 
-	strlcpy(info->type, client_name, I2C_NAME_SIZE);
+	strscpy(info->type, client_name, I2C_NAME_SIZE);
 
 	return 0;
 
@@ -1239,7 +1239,7 @@ static int w83781d_probe(struct i2c_client *client)
 	return err;
 }
 
-static int
+static void
 w83781d_remove(struct i2c_client *client)
 {
 	struct w83781d_data *data = i2c_get_clientdata(client);
@@ -1250,8 +1250,6 @@ w83781d_remove(struct i2c_client *client)
 
 	i2c_unregister_device(data->lm75[0]);
 	i2c_unregister_device(data->lm75[1]);
-
-	return 0;
 }
 
 static int
diff --git a/drivers/hwmon/w83791d.c b/drivers/hwmon/w83791d.c
index 80a9a78d7ce9..eaf691365023 100644
--- a/drivers/hwmon/w83791d.c
+++ b/drivers/hwmon/w83791d.c
@@ -315,7 +315,7 @@ struct w83791d_data {
 static int w83791d_probe(struct i2c_client *client);
 static int w83791d_detect(struct i2c_client *client,
 			  struct i2c_board_info *info);
-static int w83791d_remove(struct i2c_client *client);
+static void w83791d_remove(struct i2c_client *client);
 
 static int w83791d_read(struct i2c_client *client, u8 reg);
 static int w83791d_write(struct i2c_client *client, u8 reg, u8 value);
@@ -1333,7 +1333,7 @@ static int w83791d_detect(struct i2c_client *client,
 	if (val1 != 0x71 || val2 != 0x5c)
 		return -ENODEV;
 
-	strlcpy(info->type, "w83791d", I2C_NAME_SIZE);
+	strscpy(info->type, "w83791d", I2C_NAME_SIZE);
 
 	return 0;
 }
@@ -1405,14 +1405,12 @@ error4:
 	return err;
 }
 
-static int w83791d_remove(struct i2c_client *client)
+static void w83791d_remove(struct i2c_client *client)
 {
 	struct w83791d_data *data = i2c_get_clientdata(client);
 
 	hwmon_device_unregister(data->hwmon_dev);
 	sysfs_remove_group(&client->dev.kobj, &w83791d_group);
-
-	return 0;
 }
 
 static void w83791d_init_client(struct i2c_client *client)
diff --git a/drivers/hwmon/w83792d.c b/drivers/hwmon/w83792d.c
index 31a1cdc30877..6d160eee1446 100644
--- a/drivers/hwmon/w83792d.c
+++ b/drivers/hwmon/w83792d.c
@@ -286,7 +286,7 @@ struct w83792d_data {
 static int w83792d_probe(struct i2c_client *client);
 static int w83792d_detect(struct i2c_client *client,
 			  struct i2c_board_info *info);
-static int w83792d_remove(struct i2c_client *client);
+static void w83792d_remove(struct i2c_client *client);
 static struct w83792d_data *w83792d_update_device(struct device *dev);
 
 #ifdef DEBUG
@@ -1346,7 +1346,7 @@ w83792d_detect(struct i2c_client *client, struct i2c_board_info *info)
 	if (val1 != 0x7a || val2 != 0x5c)
 		return -ENODEV;
 
-	strlcpy(info->type, "w83792d", I2C_NAME_SIZE);
+	strscpy(info->type, "w83792d", I2C_NAME_SIZE);
 
 	return 0;
 }
@@ -1429,7 +1429,7 @@ exit_remove_files:
 	return err;
 }
 
-static int
+static void
 w83792d_remove(struct i2c_client *client)
 {
 	struct w83792d_data *data = i2c_get_clientdata(client);
@@ -1440,8 +1440,6 @@ w83792d_remove(struct i2c_client *client)
 	for (i = 0; i < ARRAY_SIZE(w83792d_group_fan); i++)
 		sysfs_remove_group(&client->dev.kobj,
 				   &w83792d_group_fan[i]);
-
-	return 0;
 }
 
 static void
diff --git a/drivers/hwmon/w83793.c b/drivers/hwmon/w83793.c
index 0a65d164c8f0..a4926d907198 100644
--- a/drivers/hwmon/w83793.c
+++ b/drivers/hwmon/w83793.c
@@ -285,7 +285,7 @@ static int w83793_write_value(struct i2c_client *client, u16 reg, u8 value);
 static int w83793_probe(struct i2c_client *client);
 static int w83793_detect(struct i2c_client *client,
 			 struct i2c_board_info *info);
-static int w83793_remove(struct i2c_client *client);
+static void w83793_remove(struct i2c_client *client);
 static void w83793_init_client(struct i2c_client *client);
 static void w83793_update_nonvolatile(struct device *dev);
 static struct w83793_data *w83793_update_device(struct device *dev);
@@ -1495,7 +1495,7 @@ static struct notifier_block watchdog_notifier = {
  * Init / remove routines
  */
 
-static int w83793_remove(struct i2c_client *client)
+static void w83793_remove(struct i2c_client *client)
 {
 	struct w83793_data *data = i2c_get_clientdata(client);
 	struct device *dev = &client->dev;
@@ -1554,8 +1554,6 @@ static int w83793_remove(struct i2c_client *client)
 	mutex_lock(&watchdog_data_mutex);
 	kref_put(&data->kref, w83793_release_resources);
 	mutex_unlock(&watchdog_data_mutex);
-
-	return 0;
 }
 
 static int
@@ -1636,7 +1634,7 @@ static int w83793_detect(struct i2c_client *client,
 	if (chip_id != 0x7b)
 		return -ENODEV;
 
-	strlcpy(info->type, "w83793", I2C_NAME_SIZE);
+	strscpy(info->type, "w83793", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwmon/w83795.c b/drivers/hwmon/w83795.c
index 45b12c4287df..84ff5c57e98c 100644
--- a/drivers/hwmon/w83795.c
+++ b/drivers/hwmon/w83795.c
@@ -1967,7 +1967,7 @@ static int w83795_detect(struct i2c_client *client,
 	else
 		chip_name = "w83795g";
 
-	strlcpy(info->type, chip_name, I2C_NAME_SIZE);
+	strscpy(info->type, chip_name, I2C_NAME_SIZE);
 	dev_info(&adapter->dev, "Found %s rev. %c at 0x%02hx\n", chip_name,
 		 'A' + (device_id & 0xf), address);
 
@@ -2235,14 +2235,12 @@ exit_remove:
 	return err;
 }
 
-static int w83795_remove(struct i2c_client *client)
+static void w83795_remove(struct i2c_client *client)
 {
 	struct w83795_data *data = i2c_get_clientdata(client);
 
 	hwmon_device_unregister(data->hwmon_dev);
 	w83795_handle_files(&client->dev, device_remove_file_wrapper);
-
-	return 0;
 }
 
 
diff --git a/drivers/hwmon/w83l785ts.c b/drivers/hwmon/w83l785ts.c
index a41f989d66e2..f3622de0d96f 100644
--- a/drivers/hwmon/w83l785ts.c
+++ b/drivers/hwmon/w83l785ts.c
@@ -65,7 +65,7 @@ static const unsigned short normal_i2c[] = { 0x2e, I2C_CLIENT_END };
 static int w83l785ts_probe(struct i2c_client *client);
 static int w83l785ts_detect(struct i2c_client *client,
 			    struct i2c_board_info *info);
-static int w83l785ts_remove(struct i2c_client *client);
+static void w83l785ts_remove(struct i2c_client *client);
 static u8 w83l785ts_read_value(struct i2c_client *client, u8 reg, u8 defval);
 static struct w83l785ts_data *w83l785ts_update_device(struct device *dev);
 
@@ -157,7 +157,7 @@ static int w83l785ts_detect(struct i2c_client *client,
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, "w83l785ts", I2C_NAME_SIZE);
+	strscpy(info->type, "w83l785ts", I2C_NAME_SIZE);
 
 	return 0;
 }
@@ -203,7 +203,7 @@ exit_remove:
 	return err;
 }
 
-static int w83l785ts_remove(struct i2c_client *client)
+static void w83l785ts_remove(struct i2c_client *client)
 {
 	struct w83l785ts_data *data = i2c_get_clientdata(client);
 
@@ -212,8 +212,6 @@ static int w83l785ts_remove(struct i2c_client *client)
 			   &sensor_dev_attr_temp1_input.dev_attr);
 	device_remove_file(&client->dev,
 			   &sensor_dev_attr_temp1_max.dev_attr);
-
-	return 0;
 }
 
 static u8 w83l785ts_read_value(struct i2c_client *client, u8 reg, u8 defval)
diff --git a/drivers/hwmon/w83l786ng.c b/drivers/hwmon/w83l786ng.c
index 11ba23c1af85..2c4646fa8426 100644
--- a/drivers/hwmon/w83l786ng.c
+++ b/drivers/hwmon/w83l786ng.c
@@ -687,7 +687,7 @@ w83l786ng_detect(struct i2c_client *client, struct i2c_board_info *info)
 		return -ENODEV;
 	}
 
-	strlcpy(info->type, "w83l786ng", I2C_NAME_SIZE);
+	strscpy(info->type, "w83l786ng", I2C_NAME_SIZE);
 
 	return 0;
 }
diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c
index d39660a3e50c..80fefaba58ee 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x-core.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c
@@ -966,7 +966,7 @@ static inline bool cpu_supports_sysreg_trace(void)
 {
 	u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);
 
-	return ((dfr0 >> ID_AA64DFR0_TRACEVER_SHIFT) & 0xfUL) > 0;
+	return ((dfr0 >> ID_AA64DFR0_EL1_TraceVer_SHIFT) & 0xfUL) > 0;
 }
 
 static bool etm4_init_sysreg_access(struct etmv4_drvdata *drvdata,
@@ -1054,7 +1054,7 @@ static void cpu_detect_trace_filtering(struct etmv4_drvdata *drvdata)
 	u64 trfcr;
 
 	drvdata->trfcr = 0;
-	if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_TRACE_FILT_SHIFT))
+	if (!cpuid_feature_extract_unsigned_field(dfr0, ID_AA64DFR0_EL1_TraceFilt_SHIFT))
 		return;
 
 	/*
diff --git a/drivers/hwtracing/coresight/coresight-trbe.h b/drivers/hwtracing/coresight/coresight-trbe.h
index 30e4d7db4f8e..98ff1b17ad07 100644
--- a/drivers/hwtracing/coresight/coresight-trbe.h
+++ b/drivers/hwtracing/coresight/coresight-trbe.h
@@ -20,7 +20,8 @@
 static inline bool is_trbe_available(void)
 {
 	u64 aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);
-	unsigned int trbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_TRBE_SHIFT);
+	unsigned int trbe = cpuid_feature_extract_unsigned_field(aa64dfr0,
+								 ID_AA64DFR0_EL1_TraceBuffer_SHIFT);
 
 	return trbe >= 0b0001;
 }
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index 7284206b278b..264e780ae32e 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -488,8 +488,8 @@ config I2C_BCM_KONA
 
 config I2C_BRCMSTB
 	tristate "BRCM Settop/DSL I2C controller"
-	depends on ARCH_BCM2835 || ARCH_BCM4908 || ARCH_BCMBCA || \
-		   ARCH_BRCMSTB || BMIPS_GENERIC || COMPILE_TEST
+	depends on ARCH_BCM2835 || ARCH_BCMBCA || ARCH_BRCMSTB || \
+		   BMIPS_GENERIC || COMPILE_TEST
 	default y
 	help
 	  If you say yes to this option, support will be included for the
@@ -1267,6 +1267,16 @@ config I2C_PARPORT
 	  This support is also available as a module.  If so, the module
 	  will be called i2c-parport.
 
+config I2C_PCI1XXXX
+	tristate "PCI1XXXX I2C Host Adapter"
+	depends on PCI
+	help
+	  If you say yes to this option, support will be included for
+	  Microchip PCI1XXXX's I2C interface.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called i2c-mchp-pci1xxxx.
+
 config I2C_ROBOTFUZZ_OSIF
 	tristate "RobotFuzz Open Source InterFace USB adapter"
 	depends on USB
diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile
index c5cac15f075c..e73cdb1d2b5a 100644
--- a/drivers/i2c/busses/Makefile
+++ b/drivers/i2c/busses/Makefile
@@ -133,6 +133,7 @@ obj-$(CONFIG_I2C_DIOLAN_U2C)	+= i2c-diolan-u2c.o
 obj-$(CONFIG_I2C_DLN2)		+= i2c-dln2.o
 obj-$(CONFIG_I2C_CP2615) += i2c-cp2615.o
 obj-$(CONFIG_I2C_PARPORT)	+= i2c-parport.o
+obj-$(CONFIG_I2C_PCI1XXXX)	+= i2c-mchp-pci1xxxx.o
 obj-$(CONFIG_I2C_ROBOTFUZZ_OSIF)	+= i2c-robotfuzz-osif.o
 obj-$(CONFIG_I2C_TAOS_EVM)	+= i2c-taos-evm.o
 obj-$(CONFIG_I2C_TINY_USB)	+= i2c-tiny-usb.o
diff --git a/drivers/i2c/busses/i2c-amd-mp2-plat.c b/drivers/i2c/busses/i2c-amd-mp2-plat.c
index 84b7e6cbc67b..423fe0c8a471 100644
--- a/drivers/i2c/busses/i2c-amd-mp2-plat.c
+++ b/drivers/i2c/busses/i2c-amd-mp2-plat.c
@@ -244,14 +244,18 @@ static const struct i2c_adapter_quirks amd_i2c_dev_quirks = {
 
 static int i2c_amd_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
 	int ret;
 	struct amd_i2c_dev *i2c_dev;
-	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
 	struct amd_mp2_dev *mp2_dev;
-	const char *uid;
+	u64 uid;
 
-	if (!adev)
-		return -ENODEV;
+	ret = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &uid);
+	if (ret)
+		return dev_err_probe(dev, ret, "missing UID/bus id!\n");
+	if (uid >= 2)
+		return dev_err_probe(dev, -EINVAL, "incorrect UID/bus id \"%llu\"!\n", uid);
+	dev_dbg(dev, "bus id is %llu\n", uid);
 
 	/* The ACPI namespace doesn't contain information about which MP2 PCI
 	 * device an AMDI0011 ACPI device is related to, so assume that there's
@@ -266,6 +270,7 @@ static int i2c_amd_probe(struct platform_device *pdev)
 	if (!i2c_dev)
 		return -ENOMEM;
 
+	i2c_dev->common.bus_id = uid;
 	i2c_dev->common.mp2_dev = mp2_dev;
 	i2c_dev->pdev = pdev;
 	platform_set_drvdata(pdev, i2c_dev);
@@ -276,20 +281,6 @@ static int i2c_amd_probe(struct platform_device *pdev)
 	i2c_dev->common.resume = &i2c_amd_resume;
 #endif
 
-	uid = adev->pnp.unique_id;
-	if (!uid) {
-		dev_err(&pdev->dev, "missing UID/bus id!\n");
-		return -EINVAL;
-	} else if (strcmp(uid, "0") == 0) {
-		i2c_dev->common.bus_id = 0;
-	} else if (strcmp(uid, "1") == 0) {
-		i2c_dev->common.bus_id = 1;
-	} else {
-		dev_err(&pdev->dev, "incorrect UID/bus id \"%s\"!\n", uid);
-		return -EINVAL;
-	}
-	dev_dbg(&pdev->dev, "bus id is %u\n", i2c_dev->common.bus_id);
-
 	/* Register the adapter */
 	amd_mp2_pm_runtime_get(mp2_dev);
 
diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c
index 33f5588a50c0..fe0cd205502d 100644
--- a/drivers/i2c/busses/i2c-cadence.c
+++ b/drivers/i2c/busses/i2c-cadence.c
@@ -10,10 +10,12 @@
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/of.h>
 #include <linux/pm_runtime.h>
+#include <linux/pinctrl/consumer.h>
 
 /* Register offsets for the I2C device. */
 #define CDNS_I2C_CR_OFFSET		0x00 /* Control Register, RW */
@@ -127,6 +129,8 @@
 #define CDNS_I2C_TIMEOUT_MAX	0xFF
 
 #define CDNS_I2C_BROKEN_HOLD_BIT	BIT(0)
+#define CDNS_I2C_POLL_US	100000
+#define CDNS_I2C_TIMEOUT_US	500000
 
 #define cdns_i2c_readreg(offset)       readl_relaxed(id->membase + offset)
 #define cdns_i2c_writereg(val, offset) writel_relaxed(val, id->membase + offset)
@@ -204,6 +208,7 @@ struct cdns_i2c {
 	struct notifier_block clk_rate_change_nb;
 	u32 quirks;
 	u32 ctrl_reg;
+	struct i2c_bus_recovery_info rinfo;
 #if IS_ENABLED(CONFIG_I2C_SLAVE)
 	u16 ctrl_reg_diva_divb;
 	struct i2c_client *slave;
@@ -840,8 +845,14 @@ static int cdns_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs,
 #endif
 
 	/* Check if the bus is free */
-	if (cdns_i2c_readreg(CDNS_I2C_SR_OFFSET) & CDNS_I2C_SR_BA) {
+
+	ret = readl_relaxed_poll_timeout(id->membase + CDNS_I2C_SR_OFFSET,
+					 reg,
+					 !(reg & CDNS_I2C_SR_BA),
+					 CDNS_I2C_POLL_US, CDNS_I2C_TIMEOUT_US);
+	if (ret) {
 		ret = -EAGAIN;
+		i2c_recover_bus(adap);
 		goto out;
 	}
 
@@ -1250,6 +1261,12 @@ static int cdns_i2c_probe(struct platform_device *pdev)
 		id->quirks = data->quirks;
 	}
 
+	id->rinfo.pinctrl = devm_pinctrl_get(&pdev->dev);
+	if (IS_ERR(id->rinfo.pinctrl)) {
+		dev_info(&pdev->dev, "can't get pinctrl, bus recovery not supported\n");
+		return PTR_ERR(id->rinfo.pinctrl);
+	}
+
 	id->membase = devm_platform_get_and_ioremap_resource(pdev, 0, &r_mem);
 	if (IS_ERR(id->membase))
 		return PTR_ERR(id->membase);
@@ -1266,6 +1283,7 @@ static int cdns_i2c_probe(struct platform_device *pdev)
 	id->adap.retries = 3;		/* Default retry value. */
 	id->adap.algo_data = id;
 	id->adap.dev.parent = &pdev->dev;
+	id->adap.bus_recovery_info = &id->rinfo;
 	init_completion(&id->xfer_done);
 	snprintf(id->adap.name, sizeof(id->adap.name),
 		 "Cadence I2C at %08lx", (unsigned long)r_mem->start);
diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c
index 471c47db546b..c836cf884185 100644
--- a/drivers/i2c/busses/i2c-davinci.c
+++ b/drivers/i2c/busses/i2c-davinci.c
@@ -823,7 +823,7 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 	r = pm_runtime_resume_and_get(dev->dev);
 	if (r < 0) {
 		dev_err(dev->dev, "failed to runtime_get device: %d\n", r);
-		return r;
+		goto err_pm;
 	}
 
 	i2c_davinci_init(dev);
@@ -882,6 +882,7 @@ static int davinci_i2c_probe(struct platform_device *pdev)
 err_unuse_clocks:
 	pm_runtime_dont_use_autosuspend(dev->dev);
 	pm_runtime_put_sync(dev->dev);
+err_pm:
 	pm_runtime_disable(dev->dev);
 
 	return r;
diff --git a/drivers/i2c/busses/i2c-designware-amdpsp.c b/drivers/i2c/busses/i2c-designware-amdpsp.c
index b624356c945f..8f36167bce62 100644
--- a/drivers/i2c/busses/i2c-designware-amdpsp.c
+++ b/drivers/i2c/busses/i2c-designware-amdpsp.c
@@ -6,6 +6,7 @@
 #include <linux/io-64-nonatomic-lo-hi.h>
 #include <linux/psp-sev.h>
 #include <linux/types.h>
+#include <linux/workqueue.h>
 
 #include <asm/msr.h>
 
@@ -15,6 +16,8 @@
 #define PSP_MBOX_OFFSET		0x10570
 #define PSP_CMD_TIMEOUT_US	(500 * USEC_PER_MSEC)
 
+#define PSP_I2C_RESERVATION_TIME_MS 100
+
 #define PSP_I2C_REQ_BUS_CMD		0x64
 #define PSP_I2C_REQ_RETRY_CNT		400
 #define PSP_I2C_REQ_RETRY_DELAY_US	(25 * USEC_PER_MSEC)
@@ -240,6 +243,41 @@ cleanup:
 	return ret;
 }
 
+static void release_bus(void)
+{
+	int status;
+
+	if (!psp_i2c_sem_acquired)
+		return;
+
+	status = psp_send_i2c_req(PSP_I2C_REQ_RELEASE);
+	if (status)
+		return;
+
+	dev_dbg(psp_i2c_dev, "PSP semaphore held for %ums\n",
+		jiffies_to_msecs(jiffies - psp_i2c_sem_acquired));
+
+	psp_i2c_sem_acquired = 0;
+}
+
+static void psp_release_i2c_bus_deferred(struct work_struct *work)
+{
+	mutex_lock(&psp_i2c_access_mutex);
+
+	/*
+	 * If there is any pending transaction, cannot release the bus here.
+	 * psp_release_i2c_bus will take care of this later.
+	 */
+	if (psp_i2c_access_count)
+		goto cleanup;
+
+	release_bus();
+
+cleanup:
+	mutex_unlock(&psp_i2c_access_mutex);
+}
+static DECLARE_DELAYED_WORK(release_queue, psp_release_i2c_bus_deferred);
+
 static int psp_acquire_i2c_bus(void)
 {
 	int status;
@@ -250,21 +288,23 @@ static int psp_acquire_i2c_bus(void)
 	if (psp_i2c_mbox_fail)
 		goto cleanup;
 
+	psp_i2c_access_count++;
+
 	/*
-	 * Simply increment usage counter and return if PSP semaphore was
-	 * already taken by kernel.
+	 * No need to request bus arbitration once we are inside semaphore
+	 * reservation period.
 	 */
-	if (psp_i2c_access_count) {
-		psp_i2c_access_count++;
+	if (psp_i2c_sem_acquired)
 		goto cleanup;
-	}
 
 	status = psp_send_i2c_req(PSP_I2C_REQ_ACQUIRE);
 	if (status)
 		goto cleanup;
 
 	psp_i2c_sem_acquired = jiffies;
-	psp_i2c_access_count++;
+
+	schedule_delayed_work(&release_queue,
+			      msecs_to_jiffies(PSP_I2C_RESERVATION_TIME_MS));
 
 	/*
 	 * In case of errors with PSP arbitrator psp_i2c_mbox_fail variable is
@@ -279,8 +319,6 @@ cleanup:
 
 static void psp_release_i2c_bus(void)
 {
-	int status;
-
 	mutex_lock(&psp_i2c_access_mutex);
 
 	/* Return early if mailbox was malfunctional */
@@ -295,13 +333,12 @@ static void psp_release_i2c_bus(void)
 	if (psp_i2c_access_count)
 		goto cleanup;
 
-	/* Send a release command to PSP */
-	status = psp_send_i2c_req(PSP_I2C_REQ_RELEASE);
-	if (status)
-		goto cleanup;
-
-	dev_dbg(psp_i2c_dev, "PSP semaphore held for %ums\n",
-		jiffies_to_msecs(jiffies - psp_i2c_sem_acquired));
+	/*
+	 * Send a release command to PSP if the semaphore reservation timeout
+	 * elapsed but x86 still owns the controller.
+	 */
+	if (!delayed_work_pending(&release_queue))
+		release_bus();
 
 cleanup:
 	mutex_unlock(&psp_i2c_access_mutex);
diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c
index 608e61209455..e499f96506c5 100644
--- a/drivers/i2c/busses/i2c-designware-pcidrv.c
+++ b/drivers/i2c/busses/i2c-designware-pcidrv.c
@@ -27,7 +27,6 @@
 #include "i2c-ccgx-ucsi.h"
 
 #define DRIVER_NAME "i2c-designware-pci"
-#define AMD_CLK_RATE_HZ	100000
 
 enum dw_pci_ctl_id_t {
 	medfield,
@@ -100,11 +99,6 @@ static u32 mfld_get_clk_rate_khz(struct dw_i2c_dev *dev)
 	return 25000;
 }
 
-static u32 navi_amd_get_clk_rate_khz(struct dw_i2c_dev *dev)
-{
-	return AMD_CLK_RATE_HZ;
-}
-
 static int mfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c)
 {
 	struct dw_i2c_dev *dev = dev_get_drvdata(&pdev->dev);
@@ -126,15 +120,6 @@ static int mfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c)
 	return -ENODEV;
 }
 
-static int navi_amd_setup(struct pci_dev *pdev, struct dw_pci_controller *c)
-{
-	struct dw_i2c_dev *dev = dev_get_drvdata(&pdev->dev);
-
-	dev->flags |= MODEL_AMD_NAVI_GPU;
-	dev->timings.bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ;
-	return 0;
-}
-
 static int mrfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c)
 {
 	/*
@@ -159,6 +144,20 @@ static u32 ehl_get_clk_rate_khz(struct dw_i2c_dev *dev)
 	return 100000;
 }
 
+static u32 navi_amd_get_clk_rate_khz(struct dw_i2c_dev *dev)
+{
+	return 100000;
+}
+
+static int navi_amd_setup(struct pci_dev *pdev, struct dw_pci_controller *c)
+{
+	struct dw_i2c_dev *dev = dev_get_drvdata(&pdev->dev);
+
+	dev->flags |= MODEL_AMD_NAVI_GPU;
+	dev->timings.bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ;
+	return 0;
+}
+
 static struct dw_pci_controller dw_pci_controllers[] = {
 	[medfield] = {
 		.bus_num = -1,
@@ -243,6 +242,7 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev,
 	int r;
 	struct dw_pci_controller *controller;
 	struct dw_scl_sda_cfg *cfg;
+	struct i2c_timings *t;
 
 	if (id->driver_data >= ARRAY_SIZE(dw_pci_controllers))
 		return dev_err_probe(&pdev->dev, -EINVAL,
@@ -263,7 +263,7 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev,
 		return dev_err_probe(&pdev->dev, r,
 				     "I/O memory remapping failed\n");
 
-	dev = devm_kzalloc(&pdev->dev, sizeof(struct dw_i2c_dev), GFP_KERNEL);
+	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
 	if (!dev)
 		return -ENOMEM;
 
@@ -272,12 +272,14 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev,
 		return r;
 
 	dev->get_clk_rate_khz = controller->get_clk_rate_khz;
-	dev->timings.bus_freq_hz = I2C_MAX_FAST_MODE_FREQ;
 	dev->base = pcim_iomap_table(pdev)[0];
 	dev->dev = &pdev->dev;
 	dev->irq = pci_irq_vector(pdev, 0);
 	dev->flags |= controller->flags;
 
+	t = &dev->timings;
+	i2c_parse_fw_timings(&pdev->dev, t, false);
+
 	pci_set_drvdata(pdev, dev);
 
 	if (controller->setup) {
@@ -389,6 +391,7 @@ static const struct pci_device_id i2_designware_pci_ids[] = {
 	{ PCI_VDEVICE(INTEL, 0x4bbe), elkhartlake },
 	{ PCI_VDEVICE(INTEL, 0x4bbf), elkhartlake },
 	{ PCI_VDEVICE(INTEL, 0x4bc0), elkhartlake },
+	/* AMD NAVI */
 	{ PCI_VDEVICE(ATI,  0x7314), navi_amd },
 	{ PCI_VDEVICE(ATI,  0x73a4), navi_amd },
 	{ PCI_VDEVICE(ATI,  0x73e4), navi_amd },
diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c
index a176296f4fff..e06509edc5f3 100644
--- a/drivers/i2c/busses/i2c-i801.c
+++ b/drivers/i2c/busses/i2c-i801.c
@@ -1838,6 +1838,7 @@ static struct pci_driver i801_driver = {
 	.shutdown	= i801_shutdown,
 	.driver		= {
 		.pm	= &i801_pm_ops,
+		.probe_type = PROBE_PREFER_ASYNCHRONOUS,
 	},
 };
 
diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c
index b51ab3cad2b1..188f2a36d2fd 100644
--- a/drivers/i2c/busses/i2c-imx-lpi2c.c
+++ b/drivers/i2c/busses/i2c-imx-lpi2c.c
@@ -94,7 +94,8 @@ enum lpi2c_imx_pincfg {
 
 struct lpi2c_imx_struct {
 	struct i2c_adapter	adapter;
-	struct clk		*clk;
+	int			num_clks;
+	struct clk_bulk_data	*clks;
 	void __iomem		*base;
 	__u8			*rx_buf;
 	__u8			*tx_buf;
@@ -207,7 +208,7 @@ static int lpi2c_imx_config(struct lpi2c_imx_struct *lpi2c_imx)
 
 	lpi2c_imx_set_mode(lpi2c_imx);
 
-	clk_rate = clk_get_rate(lpi2c_imx->clk);
+	clk_rate = clk_get_rate(lpi2c_imx->clks[0].clk);
 	if (lpi2c_imx->mode == HS || lpi2c_imx->mode == ULTRA_FAST)
 		filt = 0;
 	else
@@ -561,11 +562,12 @@ static int lpi2c_imx_probe(struct platform_device *pdev)
 	strscpy(lpi2c_imx->adapter.name, pdev->name,
 		sizeof(lpi2c_imx->adapter.name));
 
-	lpi2c_imx->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(lpi2c_imx->clk)) {
-		dev_err(&pdev->dev, "can't get I2C peripheral clock\n");
-		return PTR_ERR(lpi2c_imx->clk);
+	ret = devm_clk_bulk_get_all(&pdev->dev, &lpi2c_imx->clks);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "can't get I2C peripheral clock, ret=%d\n", ret);
+		return ret;
 	}
+	lpi2c_imx->num_clks = ret;
 
 	ret = of_property_read_u32(pdev->dev.of_node,
 				   "clock-frequency", &lpi2c_imx->bitrate);
@@ -582,11 +584,9 @@ static int lpi2c_imx_probe(struct platform_device *pdev)
 	i2c_set_adapdata(&lpi2c_imx->adapter, lpi2c_imx);
 	platform_set_drvdata(pdev, lpi2c_imx);
 
-	ret = clk_prepare_enable(lpi2c_imx->clk);
-	if (ret) {
-		dev_err(&pdev->dev, "clk enable failed %d\n", ret);
+	ret = clk_bulk_prepare_enable(lpi2c_imx->num_clks, lpi2c_imx->clks);
+	if (ret)
 		return ret;
-	}
 
 	pm_runtime_set_autosuspend_delay(&pdev->dev, I2C_PM_TIMEOUT);
 	pm_runtime_use_autosuspend(&pdev->dev);
@@ -633,7 +633,7 @@ static int __maybe_unused lpi2c_runtime_suspend(struct device *dev)
 {
 	struct lpi2c_imx_struct *lpi2c_imx = dev_get_drvdata(dev);
 
-	clk_disable_unprepare(lpi2c_imx->clk);
+	clk_bulk_disable_unprepare(lpi2c_imx->num_clks, lpi2c_imx->clks);
 	pinctrl_pm_select_sleep_state(dev);
 
 	return 0;
@@ -645,7 +645,7 @@ static int __maybe_unused lpi2c_runtime_resume(struct device *dev)
 	int ret;
 
 	pinctrl_pm_select_default_state(dev);
-	ret = clk_prepare_enable(lpi2c_imx->clk);
+	ret = clk_bulk_prepare_enable(lpi2c_imx->num_clks, lpi2c_imx->clks);
 	if (ret) {
 		dev_err(dev, "failed to enable I2C clock, ret=%d\n", ret);
 		return ret;
diff --git a/drivers/i2c/busses/i2c-ismt.c b/drivers/i2c/busses/i2c-ismt.c
index 6078fa0c0d48..fe2349590f75 100644
--- a/drivers/i2c/busses/i2c-ismt.c
+++ b/drivers/i2c/busses/i2c-ismt.c
@@ -937,11 +937,8 @@ ismt_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
 	if (err) {
-		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
-		if (err) {
-			dev_err(&pdev->dev, "dma_set_mask fail\n");
-			return -ENODEV;
-		}
+		dev_err(&pdev->dev, "dma_set_mask fail\n");
+		return -ENODEV;
 	}
 
 	err = ismt_dev_init(priv);
diff --git a/drivers/i2c/busses/i2c-mchp-pci1xxxx.c b/drivers/i2c/busses/i2c-mchp-pci1xxxx.c
new file mode 100644
index 000000000000..f5342201eb6b
--- /dev/null
+++ b/drivers/i2c/busses/i2c-mchp-pci1xxxx.c
@@ -0,0 +1,1210 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Microchip PCI1XXXX I2C adapter driver for PCIe Switch
+ * which has I2C controller in one of its downstream functions
+ *
+ * Copyright (C) 2021 - 2022 Microchip Technology Inc.
+ *
+ * Authors: Tharun Kumar P <tharunkumar.pasumarthi@microchip.com>
+ *          Kumaravel Thiagarajan <kumaravel.thiagarajan@microchip.com>
+ */
+
+#include <linux/bits.h>
+#include <linux/delay.h>
+#include <linux/i2c.h>
+#include <linux/i2c-smbus.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/types.h>
+
+#define SMBUS_MAST_CORE_ADDR_BASE		0x00000
+#define SMBUS_MAST_SYS_REG_ADDR_BASE		0x01000
+
+/* SMB register space. */
+#define SMB_CORE_CTRL_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x00)
+
+#define SMB_CORE_CTRL_ESO		BIT(6)
+#define SMB_CORE_CTRL_FW_ACK		BIT(4)
+#define SMB_CORE_CTRL_ACK		BIT(0)
+
+#define SMB_CORE_CMD_REG_OFF3	(SMBUS_MAST_CORE_ADDR_BASE + 0x0F)
+#define SMB_CORE_CMD_REG_OFF2	(SMBUS_MAST_CORE_ADDR_BASE + 0x0E)
+#define SMB_CORE_CMD_REG_OFF1	(SMBUS_MAST_CORE_ADDR_BASE + 0x0D)
+
+#define SMB_CORE_CMD_READM		BIT(4)
+#define SMB_CORE_CMD_STOP		BIT(2)
+#define SMB_CORE_CMD_START		BIT(0)
+
+#define SMB_CORE_CMD_REG_OFF0	(SMBUS_MAST_CORE_ADDR_BASE + 0x0C)
+
+#define SMB_CORE_CMD_M_PROCEED		BIT(1)
+#define SMB_CORE_CMD_M_RUN		BIT(0)
+
+#define SMB_CORE_SR_HOLD_TIME_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x18)
+
+/*
+ * SR_HOLD_TIME_XK_TICKS field will indicate the number of ticks of the
+ * baud clock required to program 'Hold Time' at X KHz.
+ */
+#define SR_HOLD_TIME_100K_TICKS	133
+#define SR_HOLD_TIME_400K_TICKS	20
+#define SR_HOLD_TIME_1000K_TICKS	11
+
+#define SMB_CORE_COMPLETION_REG_OFF3	(SMBUS_MAST_CORE_ADDR_BASE + 0x23)
+
+#define COMPLETION_MDONE		BIT(6)
+#define COMPLETION_IDLE			BIT(5)
+#define COMPLETION_MNAKX		BIT(0)
+
+#define SMB_CORE_IDLE_SCALING_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x24)
+
+/*
+ * FAIR_BUS_IDLE_MIN_XK_TICKS field will indicate the number of ticks of
+ * the baud clock required to program 'fair idle delay' at X KHz. Fair idle
+ * delay establishes the MCTP T(IDLE_DELAY) period.
+ */
+#define FAIR_BUS_IDLE_MIN_100K_TICKS		969
+#define FAIR_BUS_IDLE_MIN_400K_TICKS		157
+#define FAIR_BUS_IDLE_MIN_1000K_TICKS		157
+
+/*
+ * FAIR_IDLE_DELAY_XK_TICKS field will indicate the number of ticks of the
+ * baud clock required to satisfy the fairness protocol at X KHz.
+ */
+#define FAIR_IDLE_DELAY_100K_TICKS	1000
+#define FAIR_IDLE_DELAY_400K_TICKS	500
+#define FAIR_IDLE_DELAY_1000K_TICKS	500
+
+#define SMB_IDLE_SCALING_100K		\
+	((FAIR_IDLE_DELAY_100K_TICKS << 16) | FAIR_BUS_IDLE_MIN_100K_TICKS)
+#define SMB_IDLE_SCALING_400K		\
+	((FAIR_IDLE_DELAY_400K_TICKS << 16) | FAIR_BUS_IDLE_MIN_400K_TICKS)
+#define SMB_IDLE_SCALING_1000K	\
+	((FAIR_IDLE_DELAY_1000K_TICKS << 16) | FAIR_BUS_IDLE_MIN_1000K_TICKS)
+
+#define SMB_CORE_CONFIG_REG3		(SMBUS_MAST_CORE_ADDR_BASE + 0x2B)
+
+#define SMB_CONFIG3_ENMI		BIT(6)
+#define SMB_CONFIG3_ENIDI		BIT(5)
+
+#define SMB_CORE_CONFIG_REG2		(SMBUS_MAST_CORE_ADDR_BASE + 0x2A)
+#define SMB_CORE_CONFIG_REG1		(SMBUS_MAST_CORE_ADDR_BASE + 0x29)
+
+#define SMB_CONFIG1_ASR			BIT(7)
+#define SMB_CONFIG1_ENAB		BIT(2)
+#define SMB_CONFIG1_RESET		BIT(1)
+#define SMB_CONFIG1_FEN			BIT(0)
+
+#define SMB_CORE_BUS_CLK_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x2C)
+
+/*
+ * BUS_CLK_XK_LOW_PERIOD_TICKS field defines the number of I2C Baud Clock
+ * periods that make up the low phase of the I2C/SMBus bus clock at X KHz.
+ */
+#define BUS_CLK_100K_LOW_PERIOD_TICKS		156
+#define BUS_CLK_400K_LOW_PERIOD_TICKS		41
+#define BUS_CLK_1000K_LOW_PERIOD_TICKS	15
+
+/*
+ * BUS_CLK_XK_HIGH_PERIOD_TICKS field defines the number of I2C Baud Clock
+ * periods that make up the high phase of the I2C/SMBus bus clock at X KHz.
+ */
+#define BUS_CLK_100K_HIGH_PERIOD_TICKS	154
+#define BUS_CLK_400K_HIGH_PERIOD_TICKS	35
+#define BUS_CLK_1000K_HIGH_PERIOD_TICKS	14
+
+#define BUS_CLK_100K			\
+	((BUS_CLK_100K_HIGH_PERIOD_TICKS << 8) | BUS_CLK_100K_LOW_PERIOD_TICKS)
+#define BUS_CLK_400K			\
+	((BUS_CLK_400K_HIGH_PERIOD_TICKS << 8) | BUS_CLK_400K_LOW_PERIOD_TICKS)
+#define BUS_CLK_1000K			\
+	((BUS_CLK_1000K_HIGH_PERIOD_TICKS << 8) | BUS_CLK_1000K_LOW_PERIOD_TICKS)
+
+#define SMB_CORE_CLK_SYNC_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x3C)
+
+/*
+ * CLK_SYNC_XK defines the number of clock cycles to sync up to the external
+ * clock before comparing the internal and external clocks for clock stretching
+ * at X KHz.
+ */
+#define CLK_SYNC_100K			4
+#define CLK_SYNC_400K			4
+#define CLK_SYNC_1000K		4
+
+#define SMB_CORE_DATA_TIMING_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x40)
+
+/*
+ *
+ * FIRST_START_HOLD_XK_TICKS will indicate the number of ticks of the baud
+ * clock required to program 'FIRST_START_HOLD' timer at X KHz. This timer
+ * determines the SCLK hold time following SDAT driven low during the first
+ * START bit in a transfer.
+ */
+#define FIRST_START_HOLD_100K_TICKS	22
+#define FIRST_START_HOLD_400K_TICKS	16
+#define FIRST_START_HOLD_1000K_TICKS	6
+
+/*
+ * STOP_SETUP_XK_TICKS will indicate the number of ticks of the baud clock
+ * required to program 'STOP_SETUP' timer at X KHz. This timer determines the
+ * SDAT setup time from the rising edge of SCLK for a STOP condition.
+ */
+#define STOP_SETUP_100K_TICKS		157
+#define STOP_SETUP_400K_TICKS		20
+#define STOP_SETUP_1000K_TICKS	12
+
+/*
+ * RESTART_SETUP_XK_TICKS will indicate the number of ticks of the baud clock
+ * required to program 'RESTART_SETUP' timer at X KHz. This timer determines the
+ * SDAT setup time from the rising edge of SCLK for a repeated START condition.
+ */
+#define RESTART_SETUP_100K_TICKS	157
+#define RESTART_SETUP_400K_TICKS	20
+#define RESTART_SETUP_1000K_TICKS	12
+
+/*
+ * DATA_HOLD_XK_TICKS will indicate the number of ticks of the baud clock
+ * required to program 'DATA_HOLD' timer at X KHz. This timer determines the
+ * SDAT hold time following SCLK driven low.
+ */
+#define DATA_HOLD_100K_TICKS		2
+#define DATA_HOLD_400K_TICKS		2
+#define DATA_HOLD_1000K_TICKS		2
+
+#define DATA_TIMING_100K		\
+	((FIRST_START_HOLD_100K_TICKS << 24) | (STOP_SETUP_100K_TICKS << 16) | \
+	(RESTART_SETUP_100K_TICKS << 8) | DATA_HOLD_100K_TICKS)
+#define DATA_TIMING_400K		\
+	((FIRST_START_HOLD_400K_TICKS << 24) | (STOP_SETUP_400K_TICKS << 16) | \
+	(RESTART_SETUP_400K_TICKS << 8) | DATA_HOLD_400K_TICKS)
+#define DATA_TIMING_1000K		\
+	((FIRST_START_HOLD_1000K_TICKS << 24) | (STOP_SETUP_1000K_TICKS << 16) | \
+	(RESTART_SETUP_1000K_TICKS << 8) | DATA_HOLD_1000K_TICKS)
+
+#define SMB_CORE_TO_SCALING_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x44)
+
+/*
+ * BUS_IDLE_MIN_XK_TICKS defines Bus Idle Minimum Time.
+ * Bus Idle Minimum time = BUS_IDLE_MIN[7:0] x Baud_Clock_Period x
+ * (BUS_IDLE_MIN_XK_TICKS[7] ? 4,1)
+ */
+#define BUS_IDLE_MIN_100K_TICKS		167UL
+#define BUS_IDLE_MIN_400K_TICKS		139UL
+#define BUS_IDLE_MIN_1000K_TICKS		133UL
+
+/*
+ * CTRL_CUM_TIME_OUT_XK_TICKS defines SMBus Controller Cumulative Time-Out.
+ * SMBus Controller Cumulative Time-Out duration =
+ * CTRL_CUM_TIME_OUT_XK_TICKS[7:0] x Baud_Clock_Period x 2048
+ */
+#define CTRL_CUM_TIME_OUT_100K_TICKS		159
+#define CTRL_CUM_TIME_OUT_400K_TICKS		159
+#define CTRL_CUM_TIME_OUT_1000K_TICKS		159
+
+/*
+ * TARGET_CUM_TIME_OUT_XK_TICKS defines SMBus Target Cumulative Time-Out duration.
+ * SMBus Target Cumulative Time-Out duration = TARGET_CUM_TIME_OUT_XK_TICKS[7:0] x
+ * Baud_Clock_Period x 4096
+ */
+#define TARGET_CUM_TIME_OUT_100K_TICKS	199
+#define TARGET_CUM_TIME_OUT_400K_TICKS	199
+#define TARGET_CUM_TIME_OUT_1000K_TICKS	199
+
+/*
+ * CLOCK_HIGH_TIME_OUT_XK defines Clock High time out period.
+ * Clock High time out period = CLOCK_HIGH_TIME_OUT_XK[7:0] x Baud_Clock_Period x 8
+ */
+#define CLOCK_HIGH_TIME_OUT_100K_TICKS	204
+#define CLOCK_HIGH_TIME_OUT_400K_TICKS	204
+#define CLOCK_HIGH_TIME_OUT_1000K_TICKS	204
+
+#define TO_SCALING_100K		\
+	((BUS_IDLE_MIN_100K_TICKS << 24) | (CTRL_CUM_TIME_OUT_100K_TICKS << 16) | \
+	(TARGET_CUM_TIME_OUT_100K_TICKS << 8) | CLOCK_HIGH_TIME_OUT_100K_TICKS)
+#define TO_SCALING_400K		\
+	((BUS_IDLE_MIN_400K_TICKS << 24) | (CTRL_CUM_TIME_OUT_400K_TICKS << 16) | \
+	(TARGET_CUM_TIME_OUT_400K_TICKS << 8) | CLOCK_HIGH_TIME_OUT_400K_TICKS)
+#define TO_SCALING_1000K		\
+	((BUS_IDLE_MIN_1000K_TICKS << 24) | (CTRL_CUM_TIME_OUT_1000K_TICKS << 16) | \
+	(TARGET_CUM_TIME_OUT_1000K_TICKS << 8) | CLOCK_HIGH_TIME_OUT_1000K_TICKS)
+
+#define I2C_SCL_PAD_CTRL_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x100)
+#define I2C_SDA_PAD_CTRL_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x101)
+
+#define I2C_FOD_EN			BIT(4)
+#define I2C_PULL_UP_EN			BIT(3)
+#define I2C_PULL_DOWN_EN		BIT(2)
+#define I2C_INPUT_EN			BIT(1)
+#define I2C_OUTPUT_EN			BIT(0)
+
+#define SMBUS_CONTROL_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x200)
+
+#define CTL_RESET_COUNTERS		BIT(3)
+#define CTL_TRANSFER_DIR		BIT(2)
+#define CTL_HOST_FIFO_ENTRY		BIT(1)
+#define CTL_RUN				BIT(0)
+
+#define I2C_DIRN_WRITE			0
+#define I2C_DIRN_READ			1
+
+#define SMBUS_STATUS_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x204)
+
+#define STA_DMA_TERM			BIT(7)
+#define STA_DMA_REQ			BIT(6)
+#define STA_THRESHOLD			BIT(2)
+#define STA_BUF_FULL			BIT(1)
+#define STA_BUF_EMPTY			BIT(0)
+
+#define SMBUS_INTR_STAT_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x208)
+
+#define INTR_STAT_DMA_TERM		BIT(7)
+#define INTR_STAT_THRESHOLD		BIT(2)
+#define INTR_STAT_BUF_FULL		BIT(1)
+#define INTR_STAT_BUF_EMPTY		BIT(0)
+
+#define SMBUS_INTR_MSK_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x20C)
+
+#define INTR_MSK_DMA_TERM		BIT(7)
+#define INTR_MSK_THRESHOLD		BIT(2)
+#define INTR_MSK_BUF_FULL		BIT(1)
+#define INTR_MSK_BUF_EMPTY		BIT(0)
+
+#define ALL_NW_LAYER_INTERRUPTS  \
+	(INTR_MSK_DMA_TERM | INTR_MSK_THRESHOLD | INTR_MSK_BUF_FULL | \
+	 INTR_MSK_BUF_EMPTY)
+
+#define SMBUS_MCU_COUNTER_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x214)
+
+#define SMBALERT_MST_PAD_CTRL_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x230)
+
+#define SMBALERT_MST_PU			BIT(0)
+
+#define SMBUS_GEN_INT_STAT_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x23C)
+
+#define SMBUS_GEN_INT_MASK_REG_OFF	(SMBUS_MAST_CORE_ADDR_BASE + 0x240)
+
+#define SMBALERT_INTR_MASK		BIT(10)
+#define I2C_BUF_MSTR_INTR_MASK		BIT(9)
+#define I2C_INTR_MASK			BIT(8)
+#define SMBALERT_WAKE_INTR_MASK		BIT(2)
+#define I2C_BUF_MSTR_WAKE_INTR_MASK	BIT(1)
+#define I2C_WAKE_INTR_MASK		BIT(0)
+
+#define ALL_HIGH_LAYER_INTR     \
+	(SMBALERT_INTR_MASK | I2C_BUF_MSTR_INTR_MASK | I2C_INTR_MASK | \
+	SMBALERT_WAKE_INTR_MASK | I2C_BUF_MSTR_WAKE_INTR_MASK | \
+	I2C_WAKE_INTR_MASK)
+
+#define SMBUS_RESET_REG		(SMBUS_MAST_CORE_ADDR_BASE + 0x248)
+
+#define PERI_SMBUS_D3_RESET_DIS		BIT(16)
+
+#define SMBUS_MST_BUF		(SMBUS_MAST_CORE_ADDR_BASE + 0x280)
+
+#define SMBUS_BUF_MAX_SIZE		0x80
+
+#define I2C_FLAGS_DIRECT_MODE		BIT(7)
+#define I2C_FLAGS_POLLING_MODE		BIT(6)
+#define I2C_FLAGS_STOP			BIT(5)
+#define I2C_FLAGS_SMB_BLK_READ		BIT(4)
+
+#define PCI1XXXX_I2C_TIMEOUT_MS		1000
+
+/* General Purpose Register. */
+#define SMB_GPR_REG		(SMBUS_MAST_CORE_ADDR_BASE + 0x1000 + 0x0c00 + \
+				0x00)
+
+/* Lock Register. */
+#define SMB_GPR_LOCK_REG	(SMBUS_MAST_CORE_ADDR_BASE + 0x1000 + 0x0000 + \
+				0x00A0)
+
+#define SMBUS_PERI_LOCK		BIT(3)
+
+struct pci1xxxx_i2c {
+	struct completion i2c_xfer_done;
+	bool i2c_xfer_in_progress;
+	struct i2c_adapter adap;
+	void __iomem *i2c_base;
+	u32 freq;
+	u32 flags;
+};
+
+static int set_sys_lock(struct pci1xxxx_i2c *i2c)
+{
+	void __iomem *p = i2c->i2c_base + SMB_GPR_LOCK_REG;
+	u8 data;
+
+	writel(SMBUS_PERI_LOCK, p);
+	data = readl(p);
+	if (data != SMBUS_PERI_LOCK)
+		return -EPERM;
+
+	return 0;
+}
+
+static int release_sys_lock(struct pci1xxxx_i2c *i2c)
+{
+	void __iomem *p = i2c->i2c_base + SMB_GPR_LOCK_REG;
+	u8 data;
+
+	data = readl(p);
+	if (data != SMBUS_PERI_LOCK)
+		return 0;
+
+	writel(0, p);
+	data = readl(p);
+	if (data & SMBUS_PERI_LOCK)
+		return -EPERM;
+
+	return 0;
+}
+
+static void pci1xxxx_ack_high_level_intr(struct pci1xxxx_i2c *i2c, u16 intr_msk)
+{
+	writew(intr_msk, i2c->i2c_base + SMBUS_GEN_INT_STAT_REG_OFF);
+}
+
+static void pci1xxxx_i2c_configure_smbalert_pin(struct pci1xxxx_i2c *i2c,
+						bool enable)
+{
+	void __iomem *p = i2c->i2c_base + SMBALERT_MST_PAD_CTRL_REG_OFF;
+	u8 regval;
+
+	regval = readb(p);
+
+	if (enable)
+		regval |= SMBALERT_MST_PU;
+	else
+		regval &= ~SMBALERT_MST_PU;
+
+	writeb(regval, p);
+}
+
+static void pci1xxxx_i2c_send_start_stop(struct pci1xxxx_i2c *i2c, bool start)
+{
+	void __iomem *p = i2c->i2c_base + SMB_CORE_CMD_REG_OFF1;
+	u8 regval;
+
+	regval = readb(p);
+
+	if (start)
+		regval |= SMB_CORE_CMD_START;
+	else
+		regval |= SMB_CORE_CMD_STOP;
+
+	writeb(regval, p);
+}
+
+/*
+ * When accessing the core control reg, we should not do a read modified write
+ * as they are write '1' to clear bits. Instead we need to write with the
+ * specific bits that needs to be set.
+ */
+static void pci1xxxx_i2c_set_clear_FW_ACK(struct pci1xxxx_i2c *i2c, bool set)
+{
+	u8 regval;
+
+	if (set)
+		regval = SMB_CORE_CTRL_FW_ACK | SMB_CORE_CTRL_ESO | SMB_CORE_CTRL_ACK;
+	else
+		regval = SMB_CORE_CTRL_ESO | SMB_CORE_CTRL_ACK;
+
+	writeb(regval, i2c->i2c_base + SMB_CORE_CTRL_REG_OFF);
+}
+
+static void pci1xxxx_i2c_buffer_write(struct pci1xxxx_i2c *i2c, u8 slaveaddr,
+				      u8 transferlen, unsigned char *buf)
+{
+	void __iomem *p = i2c->i2c_base + SMBUS_MST_BUF;
+
+	if (slaveaddr)
+		writeb(slaveaddr, p++);
+
+	if (buf)
+		memcpy_toio(p, buf, transferlen);
+}
+
+/*
+ * When accessing the core control reg, we should not do a read modified write
+ * as there are write '1' to clear bits. Instead we need to write with the
+ * specific bits that needs to be set.
+ */
+static void pci1xxxx_i2c_enable_ESO(struct pci1xxxx_i2c *i2c)
+{
+	writeb(SMB_CORE_CTRL_ESO, i2c->i2c_base + SMB_CORE_CTRL_REG_OFF);
+}
+
+static void pci1xxxx_i2c_reset_counters(struct pci1xxxx_i2c *i2c)
+{
+	void __iomem *p = i2c->i2c_base + SMBUS_CONTROL_REG_OFF;
+	u8 regval;
+
+	regval = readb(p);
+	regval |= CTL_RESET_COUNTERS;
+	writeb(regval, p);
+}
+
+static void pci1xxxx_i2c_set_transfer_dir(struct pci1xxxx_i2c *i2c, u8 direction)
+{
+	void __iomem *p = i2c->i2c_base + SMBUS_CONTROL_REG_OFF;
+	u8 regval;
+
+	regval = readb(p);
+	if (direction == I2C_DIRN_WRITE)
+		regval &= ~CTL_TRANSFER_DIR;
+	else
+		regval |= CTL_TRANSFER_DIR;
+
+	writeb(regval, p);
+}
+
+static void pci1xxxx_i2c_set_mcu_count(struct pci1xxxx_i2c *i2c, u8 count)
+{
+	writeb(count, i2c->i2c_base + SMBUS_MCU_COUNTER_REG_OFF);
+}
+
+static void pci1xxxx_i2c_set_read_count(struct pci1xxxx_i2c *i2c, u8 readcount)
+{
+	writeb(readcount, i2c->i2c_base + SMB_CORE_CMD_REG_OFF3);
+}
+
+static void pci1xxxx_i2c_set_write_count(struct pci1xxxx_i2c *i2c, u8 writecount)
+{
+	writeb(writecount, i2c->i2c_base + SMB_CORE_CMD_REG_OFF2);
+}
+
+static void pci1xxxx_i2c_set_DMA_run(struct pci1xxxx_i2c *i2c)
+{
+	void __iomem *p = i2c->i2c_base + SMBUS_CONTROL_REG_OFF;
+	u8 regval;
+
+	regval = readb(p);
+	regval |= CTL_RUN;
+	writeb(regval, p);
+}
+
+static void pci1xxxx_i2c_set_mrun_proceed(struct pci1xxxx_i2c *i2c)
+{
+	void __iomem *p = i2c->i2c_base + SMB_CORE_CMD_REG_OFF0;
+	u8 regval;
+
+	regval = readb(p);
+	regval |= SMB_CORE_CMD_M_RUN;
+	regval |= SMB_CORE_CMD_M_PROCEED;
+	writeb(regval, p);
+}
+
+static void pci1xxxx_i2c_start_DMA(struct pci1xxxx_i2c *i2c)
+{
+	pci1xxxx_i2c_set_DMA_run(i2c);
+	pci1xxxx_i2c_set_mrun_proceed(i2c);
+}
+
+static void pci1xxxx_i2c_config_asr(struct pci1xxxx_i2c *i2c, bool enable)
+{
+	void __iomem *p = i2c->i2c_base + SMB_CORE_CONFIG_REG1;
+	u8 regval;
+
+	regval = readb(p);
+	if (enable)
+		regval |= SMB_CONFIG1_ASR;
+	else
+		regval &= ~SMB_CONFIG1_ASR;
+	writeb(regval, p);
+}
+
+static irqreturn_t pci1xxxx_i2c_isr(int irq, void *dev)
+{
+	struct pci1xxxx_i2c *i2c = dev;
+	void __iomem *p1 = i2c->i2c_base + SMBUS_GEN_INT_STAT_REG_OFF;
+	void __iomem *p2 = i2c->i2c_base + SMBUS_INTR_STAT_REG_OFF;
+	irqreturn_t intr_handled = IRQ_NONE;
+	u16 reg1;
+	u8 reg3;
+
+	/*
+	 *  Read the SMBus interrupt status register to see if the
+	 *  DMA_TERM interrupt has caused this callback.
+	 */
+	reg1 = readw(p1);
+
+	if (reg1 & I2C_BUF_MSTR_INTR_MASK) {
+		reg3 = readb(p2);
+		if (reg3 & INTR_STAT_DMA_TERM) {
+			complete(&i2c->i2c_xfer_done);
+			intr_handled = IRQ_HANDLED;
+			writeb(INTR_STAT_DMA_TERM, p2);
+		}
+		pci1xxxx_ack_high_level_intr(i2c, I2C_BUF_MSTR_INTR_MASK);
+	}
+
+	if (reg1 & SMBALERT_INTR_MASK) {
+		intr_handled = IRQ_HANDLED;
+		pci1xxxx_ack_high_level_intr(i2c, SMBALERT_INTR_MASK);
+	}
+
+	return intr_handled;
+}
+
+static void pci1xxxx_i2c_set_count(struct pci1xxxx_i2c *i2c, u8 mcucount,
+				   u8 writecount, u8 readcount)
+{
+	pci1xxxx_i2c_set_mcu_count(i2c, mcucount);
+	pci1xxxx_i2c_set_write_count(i2c, writecount);
+	pci1xxxx_i2c_set_read_count(i2c, readcount);
+}
+
+static void pci1xxxx_i2c_set_readm(struct pci1xxxx_i2c *i2c, bool enable)
+{
+	void __iomem *p = i2c->i2c_base + SMB_CORE_CMD_REG_OFF1;
+	u8 regval;
+
+	regval = readb(p);
+	if (enable)
+		regval |= SMB_CORE_CMD_READM;
+	else
+		regval &= ~SMB_CORE_CMD_READM;
+
+	writeb(regval, p);
+}
+
+static void pci1xxxx_ack_nw_layer_intr(struct pci1xxxx_i2c *i2c, u8 ack_intr_msk)
+{
+	writeb(ack_intr_msk, i2c->i2c_base + SMBUS_INTR_STAT_REG_OFF);
+}
+
+static void pci1xxxx_config_nw_layer_intr(struct pci1xxxx_i2c *i2c,
+					  u8 intr_msk, bool enable)
+{
+	void __iomem *p = i2c->i2c_base + SMBUS_INTR_MSK_REG_OFF;
+	u8 regval;
+
+	regval = readb(p);
+	if (enable)
+		regval &= ~intr_msk;
+	else
+		regval |= intr_msk;
+
+	writeb(regval, p);
+}
+
+static void pci1xxxx_i2c_config_padctrl(struct pci1xxxx_i2c *i2c, bool enable)
+{
+	void __iomem *p1 = i2c->i2c_base + I2C_SCL_PAD_CTRL_REG_OFF;
+	void __iomem *p2 = i2c->i2c_base + I2C_SDA_PAD_CTRL_REG_OFF;
+	u8 regval;
+
+	regval = readb(p1);
+	if (enable)
+		regval |= I2C_INPUT_EN | I2C_OUTPUT_EN;
+	else
+		regval &= ~(I2C_INPUT_EN | I2C_OUTPUT_EN);
+
+	writeb(regval, p1);
+
+	regval = readb(p2);
+	if (enable)
+		regval |= I2C_INPUT_EN | I2C_OUTPUT_EN;
+	else
+		regval &= ~(I2C_INPUT_EN | I2C_OUTPUT_EN);
+
+	writeb(regval, p2);
+}
+
+static void pci1xxxx_i2c_set_mode(struct pci1xxxx_i2c *i2c)
+{
+	void __iomem *p = i2c->i2c_base + SMBUS_CONTROL_REG_OFF;
+	u8 regval;
+
+	regval = readb(p);
+	if (i2c->flags & I2C_FLAGS_DIRECT_MODE)
+		regval &= ~CTL_HOST_FIFO_ENTRY;
+	else
+		regval |= CTL_HOST_FIFO_ENTRY;
+
+	writeb(regval, p);
+}
+
+static void pci1xxxx_i2c_config_high_level_intr(struct pci1xxxx_i2c *i2c,
+						u16 intr_msk, bool enable)
+{
+	void __iomem *p = i2c->i2c_base + SMBUS_GEN_INT_MASK_REG_OFF;
+	u16 regval;
+
+	regval = readw(p);
+	if (enable)
+		regval &= ~intr_msk;
+	else
+		regval |= intr_msk;
+	writew(regval, p);
+}
+
+static void pci1xxxx_i2c_configure_core_reg(struct pci1xxxx_i2c *i2c, bool enable)
+{
+	void __iomem *p1 = i2c->i2c_base + SMB_CORE_CONFIG_REG1;
+	void __iomem *p3 = i2c->i2c_base + SMB_CORE_CONFIG_REG3;
+	u8 reg1;
+	u8 reg3;
+
+	reg1 = readb(p1);
+	reg3 = readb(p3);
+	if (enable) {
+		reg1 |= SMB_CONFIG1_ENAB | SMB_CONFIG1_FEN;
+		reg3 |= SMB_CONFIG3_ENMI | SMB_CONFIG3_ENIDI;
+	} else {
+		reg1 &= ~(SMB_CONFIG1_ENAB | SMB_CONFIG1_FEN);
+		reg3 &= ~(SMB_CONFIG3_ENMI | SMB_CONFIG3_ENIDI);
+	}
+
+	writeb(reg1, p1);
+	writeb(reg3, p3);
+}
+
+static void pci1xxxx_i2c_set_freq(struct pci1xxxx_i2c *i2c)
+{
+	void __iomem *bp = i2c->i2c_base;
+	void __iomem *p_idle_scaling = bp + SMB_CORE_IDLE_SCALING_REG_OFF;
+	void __iomem *p_data_timing = bp + SMB_CORE_DATA_TIMING_REG_OFF;
+	void __iomem *p_hold_time = bp + SMB_CORE_SR_HOLD_TIME_REG_OFF;
+	void __iomem *p_to_scaling = bp + SMB_CORE_TO_SCALING_REG_OFF;
+	void __iomem *p_clk_sync = bp + SMB_CORE_CLK_SYNC_REG_OFF;
+	void __iomem *p_clk_reg = bp + SMB_CORE_BUS_CLK_REG_OFF;
+
+	switch (i2c->freq) {
+	case I2C_MAX_STANDARD_MODE_FREQ:
+		writeb(SR_HOLD_TIME_100K_TICKS, p_hold_time);
+		writel(SMB_IDLE_SCALING_100K, p_idle_scaling);
+		writew(BUS_CLK_100K, p_clk_reg);
+		writel(CLK_SYNC_100K, p_clk_sync);
+		writel(DATA_TIMING_100K, p_data_timing);
+		writel(TO_SCALING_100K, p_to_scaling);
+		break;
+
+	case I2C_MAX_FAST_MODE_PLUS_FREQ:
+		writeb(SR_HOLD_TIME_1000K_TICKS, p_hold_time);
+		writel(SMB_IDLE_SCALING_1000K, p_idle_scaling);
+		writew(BUS_CLK_1000K, p_clk_reg);
+		writel(CLK_SYNC_1000K, p_clk_sync);
+		writel(DATA_TIMING_1000K, p_data_timing);
+		writel(TO_SCALING_1000K, p_to_scaling);
+		break;
+
+	case I2C_MAX_FAST_MODE_FREQ:
+	default:
+		writeb(SR_HOLD_TIME_400K_TICKS, p_hold_time);
+		writel(SMB_IDLE_SCALING_400K, p_idle_scaling);
+		writew(BUS_CLK_400K, p_clk_reg);
+		writel(CLK_SYNC_400K, p_clk_sync);
+		writel(DATA_TIMING_400K, p_data_timing);
+		writel(TO_SCALING_400K, p_to_scaling);
+		break;
+	}
+}
+
+static void pci1xxxx_i2c_init(struct pci1xxxx_i2c *i2c)
+{
+	void __iomem *p2 = i2c->i2c_base + SMBUS_STATUS_REG_OFF;
+	void __iomem *p1 = i2c->i2c_base + SMB_GPR_REG;
+	u8 regval;
+	u8 ret;
+
+	ret = set_sys_lock(i2c);
+	if (ret == -EPERM) {
+		/*
+		 * Configure I2C Fast Mode as default frequency if unable
+		 * to acquire sys lock.
+		 */
+		regval = 0;
+	} else {
+		regval = readl(p1);
+		release_sys_lock(i2c);
+	}
+
+	switch (regval) {
+	case 0:
+		i2c->freq = I2C_MAX_FAST_MODE_FREQ;
+		pci1xxxx_i2c_set_freq(i2c);
+		break;
+	case 1:
+		i2c->freq = I2C_MAX_STANDARD_MODE_FREQ;
+		pci1xxxx_i2c_set_freq(i2c);
+		break;
+	case 2:
+		i2c->freq = I2C_MAX_FAST_MODE_PLUS_FREQ;
+		pci1xxxx_i2c_set_freq(i2c);
+		break;
+	case 3:
+	default:
+		break;
+	}
+
+	pci1xxxx_i2c_config_padctrl(i2c, true);
+	i2c->flags |= I2C_FLAGS_DIRECT_MODE;
+	pci1xxxx_i2c_set_mode(i2c);
+
+	/*
+	 * Added as a precaution since BUF_EMPTY in status register
+	 * also trigered an Interrupt.
+	 */
+	writeb(STA_BUF_EMPTY, p2);
+
+	/* Configure core I2c control registers. */
+	pci1xxxx_i2c_configure_core_reg(i2c, true);
+
+	/*
+	 * Enable pull-up for the SMB alert pin which is just used for
+	 * wakeup right now.
+	 */
+	pci1xxxx_i2c_configure_smbalert_pin(i2c, true);
+}
+
+static void pci1xxxx_i2c_clear_flags(struct pci1xxxx_i2c *i2c)
+{
+	u8 regval;
+
+	/* Reset the internal buffer counters. */
+	pci1xxxx_i2c_reset_counters(i2c);
+
+	/* Clear low level interrupts. */
+	regval = COMPLETION_MNAKX | COMPLETION_IDLE | COMPLETION_MDONE;
+	writeb(regval, i2c->i2c_base + SMB_CORE_COMPLETION_REG_OFF3);
+	reinit_completion(&i2c->i2c_xfer_done);
+	pci1xxxx_ack_nw_layer_intr(i2c, ALL_NW_LAYER_INTERRUPTS);
+	pci1xxxx_ack_high_level_intr(i2c, ALL_HIGH_LAYER_INTR);
+}
+
+static int pci1xxxx_i2c_read(struct pci1xxxx_i2c *i2c, u8 slaveaddr,
+			     unsigned char *buf, u16 total_len)
+{
+	void __iomem *p2 = i2c->i2c_base + SMB_CORE_COMPLETION_REG_OFF3;
+	void __iomem *p1 = i2c->i2c_base + SMB_CORE_CMD_REG_OFF1;
+	void __iomem *p3 = i2c->i2c_base + SMBUS_MST_BUF;
+	unsigned long time_left;
+	u16 remainingbytes;
+	u8 transferlen;
+	int retval = 0;
+	u8 read_count;
+	u32 regval;
+	u16 count;
+
+	/* Enable I2C host controller by setting the ESO bit in the CONTROL REG. */
+	pci1xxxx_i2c_enable_ESO(i2c);
+	pci1xxxx_i2c_clear_flags(i2c);
+	pci1xxxx_config_nw_layer_intr(i2c, INTR_MSK_DMA_TERM, true);
+	pci1xxxx_i2c_config_high_level_intr(i2c, I2C_BUF_MSTR_INTR_MASK, true);
+
+	/*
+	 * The I2C transfer could be more than 128 bytes. Our Core is
+	 * capable of only sending 128 at a time.
+	 * As far as the I2C read is concerned, initailly send the
+	 * read slave address along with the number of bytes to read in
+	 * ReadCount. After sending the slave address the interrupt
+	 * is generated. On seeing the ACK for the slave address, reverse the
+	 * buffer direction and run the DMA to initiate Read from slave.
+	 */
+	for (count = 0; count < total_len; count += transferlen) {
+
+		/*
+		 * Before start of any transaction clear the existing
+		 * START/STOP conditions.
+		 */
+		writeb(0, p1);
+		remainingbytes = total_len - count;
+		transferlen = min_t(u16, remainingbytes, SMBUS_BUF_MAX_SIZE);
+
+		/*
+		 * Send STOP bit for the last chunk in the transaction.
+		 * For I2C read transaction of more than BUF_SIZE, NACK should
+		 * only be sent for the last read.
+		 * Hence a bit FW_ACK is set for all the read chunks except for
+		 * the last chunk. For the last chunk NACK should be sent and
+		 * FW_ACK is cleared Send STOP only when I2C_FLAGS_STOP bit is
+		 * set in the flags and only for the last transaction.
+		 */
+		if ((count + transferlen >= total_len) &&
+		    (i2c->flags & I2C_FLAGS_STOP)) {
+			pci1xxxx_i2c_set_clear_FW_ACK(i2c, false);
+			pci1xxxx_i2c_send_start_stop(i2c, 0);
+		} else {
+			pci1xxxx_i2c_set_clear_FW_ACK(i2c, true);
+		}
+
+		/* Send START bit for the first transaction. */
+		if (count == 0) {
+			pci1xxxx_i2c_set_transfer_dir(i2c, I2C_DIRN_WRITE);
+			pci1xxxx_i2c_send_start_stop(i2c, 1);
+
+			/* Write I2c buffer with just the slave addr. */
+			pci1xxxx_i2c_buffer_write(i2c, slaveaddr, 0, NULL);
+
+			/* Set the count. Readcount is the transfer bytes. */
+			pci1xxxx_i2c_set_count(i2c, 1, 1, transferlen);
+
+			/*
+			 * Set the Auto_start_read bit so that the HW itself
+			 * will take care of the read phase.
+			 */
+			pci1xxxx_i2c_config_asr(i2c, true);
+			if (i2c->flags & I2C_FLAGS_SMB_BLK_READ)
+				pci1xxxx_i2c_set_readm(i2c, true);
+		} else {
+			pci1xxxx_i2c_set_count(i2c, 0, 0, transferlen);
+			pci1xxxx_i2c_config_asr(i2c, false);
+			pci1xxxx_i2c_clear_flags(i2c);
+			pci1xxxx_i2c_set_transfer_dir(i2c, I2C_DIRN_READ);
+		}
+
+		/* Start the DMA. */
+		pci1xxxx_i2c_start_DMA(i2c);
+
+		/* Wait for the DMA_TERM interrupt. */
+		time_left = wait_for_completion_timeout(&i2c->i2c_xfer_done,
+			    msecs_to_jiffies(PCI1XXXX_I2C_TIMEOUT_MS));
+		if (time_left == 0) {
+			/* Reset the I2C core to release the bus lock. */
+			pci1xxxx_i2c_init(i2c);
+			retval = -ETIMEDOUT;
+			goto cleanup;
+		}
+
+		/* Read the completion reg to know the reason for DMA_TERM. */
+		regval = readb(p2);
+
+		/* Slave did not respond. */
+		if (regval & COMPLETION_MNAKX) {
+			writeb(COMPLETION_MNAKX, p2);
+			retval = -ETIMEDOUT;
+			goto cleanup;
+		}
+
+		if (i2c->flags & I2C_FLAGS_SMB_BLK_READ) {
+			buf[0] = readb(p3);
+			read_count = buf[0];
+			memcpy_fromio(&buf[1], p3 + 1, read_count);
+		} else {
+			memcpy_fromio(&buf[count], p3, transferlen);
+		}
+	}
+
+cleanup:
+	/* Disable all the interrupts. */
+	pci1xxxx_config_nw_layer_intr(i2c, INTR_MSK_DMA_TERM, false);
+	pci1xxxx_i2c_config_high_level_intr(i2c, I2C_BUF_MSTR_INTR_MASK, false);
+	pci1xxxx_i2c_config_asr(i2c, false);
+	return retval;
+}
+
+static int pci1xxxx_i2c_write(struct pci1xxxx_i2c *i2c, u8 slaveaddr,
+			      unsigned char *buf, u16 total_len)
+{
+	void __iomem *p2 = i2c->i2c_base + SMB_CORE_COMPLETION_REG_OFF3;
+	void __iomem *p1 = i2c->i2c_base + SMB_CORE_CMD_REG_OFF1;
+	unsigned long time_left;
+	u16 remainingbytes;
+	u8 actualwritelen;
+	u8 transferlen;
+	int retval = 0;
+	u32 regval;
+	u16 count;
+
+	/* Enable I2C host controller by setting the ESO bit in the CONTROL REG. */
+	pci1xxxx_i2c_enable_ESO(i2c);
+
+	/* Set the Buffer direction. */
+	pci1xxxx_i2c_set_transfer_dir(i2c, I2C_DIRN_WRITE);
+	pci1xxxx_config_nw_layer_intr(i2c, INTR_MSK_DMA_TERM, true);
+	pci1xxxx_i2c_config_high_level_intr(i2c, I2C_BUF_MSTR_INTR_MASK, true);
+
+	/*
+	 * The i2c transfer could be more than 128 bytes. Our Core is
+	 * capable of only sending 128 at a time.
+	 */
+	for (count = 0; count < total_len; count += transferlen) {
+		/*
+		 * Before start of any transaction clear the existing
+		 * START/STOP conditions.
+		 */
+		writeb(0, p1);
+		pci1xxxx_i2c_clear_flags(i2c);
+		remainingbytes = total_len - count;
+
+		/* If it is the starting of the transaction send START. */
+		if (count == 0) {
+			pci1xxxx_i2c_send_start_stop(i2c, 1);
+
+			/* -1 for the slave address. */
+			transferlen = min_t(u16, SMBUS_BUF_MAX_SIZE - 1,
+					    remainingbytes);
+			pci1xxxx_i2c_buffer_write(i2c, slaveaddr,
+						  transferlen, &buf[count]);
+			/*
+			 * The actual number of bytes written on the I2C bus
+			 * is including the slave address.
+			 */
+			actualwritelen = transferlen + 1;
+		} else {
+			transferlen = min_t(u16, SMBUS_BUF_MAX_SIZE, remainingbytes);
+			pci1xxxx_i2c_buffer_write(i2c, 0, transferlen, &buf[count]);
+			actualwritelen = transferlen;
+		}
+
+		pci1xxxx_i2c_set_count(i2c, actualwritelen, actualwritelen, 0);
+
+		/*
+		 * Send STOP only when I2C_FLAGS_STOP bit is set in the flags and
+		 * only for the last transaction.
+		 */
+		if (remainingbytes <= transferlen &&
+		   (i2c->flags & I2C_FLAGS_STOP))
+			pci1xxxx_i2c_send_start_stop(i2c, 0);
+
+		pci1xxxx_i2c_start_DMA(i2c);
+
+		/*
+		 * Wait for the DMA_TERM interrupt.
+		 */
+		time_left = wait_for_completion_timeout(&i2c->i2c_xfer_done,
+			    msecs_to_jiffies(PCI1XXXX_I2C_TIMEOUT_MS));
+		if (time_left == 0) {
+			/* Reset the I2C core to release the bus lock. */
+			pci1xxxx_i2c_init(i2c);
+			retval = -ETIMEDOUT;
+			goto cleanup;
+		}
+
+		regval = readb(p2);
+		if (regval & COMPLETION_MNAKX) {
+			writeb(COMPLETION_MNAKX, p2);
+			retval = -ETIMEDOUT;
+			goto cleanup;
+		}
+	}
+cleanup:
+	/* Disable all the interrupts. */
+	pci1xxxx_config_nw_layer_intr(i2c, INTR_MSK_DMA_TERM, false);
+	pci1xxxx_i2c_config_high_level_intr(i2c, I2C_BUF_MSTR_INTR_MASK, false);
+
+	return retval;
+}
+
+static int pci1xxxx_i2c_xfer(struct i2c_adapter *adap,
+			     struct i2c_msg *msgs, int num)
+{
+	struct pci1xxxx_i2c *i2c = i2c_get_adapdata(adap);
+	u8 slaveaddr;
+	int retval;
+	u32 i;
+
+	i2c->i2c_xfer_in_progress = true;
+	for (i = 0; i < num; i++) {
+		slaveaddr = i2c_8bit_addr_from_msg(&msgs[i]);
+
+		/*
+		 * Send the STOP bit if the transfer is the final one or
+		 * if the I2C_M_STOP flag is set.
+		 */
+		if ((i == num - 1) || (msgs[i].flags & I2C_M_STOP))
+			i2c->flags |= I2C_FLAGS_STOP;
+		else
+			i2c->flags &= ~I2C_FLAGS_STOP;
+
+		if (msgs[i].flags & I2C_M_RECV_LEN)
+			i2c->flags |= I2C_FLAGS_SMB_BLK_READ;
+		else
+			i2c->flags &= ~I2C_FLAGS_SMB_BLK_READ;
+
+		if (msgs[i].flags & I2C_M_RD)
+			retval = pci1xxxx_i2c_read(i2c, slaveaddr,
+						   msgs[i].buf, msgs[i].len);
+		else
+			retval = pci1xxxx_i2c_write(i2c, slaveaddr,
+						    msgs[i].buf, msgs[i].len);
+
+		if (retval < 0)
+			break;
+	}
+	i2c->i2c_xfer_in_progress = false;
+
+	if (retval < 0)
+		return retval;
+
+	return num;
+}
+
+/*
+ * List of supported functions by the driver.
+ */
+static u32 pci1xxxx_i2c_get_funcs(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_PROTOCOL_MANGLING |
+		I2C_FUNC_SMBUS_BLOCK_PROC_CALL |
+		I2C_FUNC_SMBUS_BYTE |
+		I2C_FUNC_SMBUS_BYTE_DATA |
+		I2C_FUNC_SMBUS_WORD_DATA |
+		I2C_FUNC_SMBUS_PROC_CALL |
+		I2C_FUNC_SMBUS_BLOCK_DATA;
+}
+
+static const struct i2c_algorithm pci1xxxx_i2c_algo = {
+	.master_xfer = pci1xxxx_i2c_xfer,
+	.functionality = pci1xxxx_i2c_get_funcs,
+};
+
+static const struct i2c_adapter_quirks pci1xxxx_i2c_quirks = {
+	.flags = I2C_AQ_NO_ZERO_LEN,
+};
+
+static const struct i2c_adapter pci1xxxx_i2c_ops = {
+	.owner	= THIS_MODULE,
+	.name	= "PCI1xxxx I2C Adapter",
+	.algo	= &pci1xxxx_i2c_algo,
+	.quirks = &pci1xxxx_i2c_quirks,
+};
+
+static int pci1xxxx_i2c_suspend(struct device *dev)
+{
+	struct pci1xxxx_i2c *i2c = dev_get_drvdata(dev);
+	void __iomem *p = i2c->i2c_base + SMBUS_RESET_REG;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	u32 regval;
+
+	i2c_mark_adapter_suspended(&i2c->adap);
+
+	/*
+	 * If the system is put into 'suspend' state when the I2C transfer is in
+	 * progress, wait until the transfer completes.
+	 */
+	while (i2c->i2c_xfer_in_progress)
+		msleep(20);
+
+	pci1xxxx_i2c_config_high_level_intr(i2c, SMBALERT_WAKE_INTR_MASK, true);
+
+	/*
+	 * Enable the PERST_DIS bit to mask the PERST from resetting the core
+	 * registers.
+	 */
+	regval = readl(p);
+	regval |= PERI_SMBUS_D3_RESET_DIS;
+	writel(regval, p);
+
+	/* Enable PCI wake in the PMCSR register. */
+	device_set_wakeup_enable(dev, true);
+	pci_wake_from_d3(pdev, true);
+
+	return 0;
+}
+
+static int pci1xxxx_i2c_resume(struct device *dev)
+{
+	struct pci1xxxx_i2c *i2c = dev_get_drvdata(dev);
+	void __iomem *p1 = i2c->i2c_base + SMBUS_GEN_INT_STAT_REG_OFF;
+	void __iomem *p2 = i2c->i2c_base + SMBUS_RESET_REG;
+	struct pci_dev *pdev = to_pci_dev(dev);
+	u32 regval;
+
+	regval = readw(p1);
+	writew(regval, p1);
+	pci1xxxx_i2c_config_high_level_intr(i2c, SMBALERT_WAKE_INTR_MASK, false);
+	regval = readl(p2);
+	regval &= ~PERI_SMBUS_D3_RESET_DIS;
+	writel(regval, p2);
+	i2c_mark_adapter_resumed(&i2c->adap);
+	pci_wake_from_d3(pdev, false);
+	return 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(pci1xxxx_i2c_pm_ops, pci1xxxx_i2c_suspend,
+			 pci1xxxx_i2c_resume);
+
+static void pci1xxxx_i2c_shutdown(struct pci1xxxx_i2c *i2c)
+{
+	pci1xxxx_i2c_config_padctrl(i2c, false);
+	pci1xxxx_i2c_configure_core_reg(i2c, false);
+}
+
+static int pci1xxxx_i2c_probe_pci(struct pci_dev *pdev,
+				  const struct pci_device_id *ent)
+{
+	struct device *dev = &pdev->dev;
+	struct pci1xxxx_i2c *i2c;
+	int ret;
+
+	i2c = devm_kzalloc(dev, sizeof(*i2c), GFP_KERNEL);
+	if (!i2c)
+		return -ENOMEM;
+
+	pci_set_drvdata(pdev, i2c);
+	i2c->i2c_xfer_in_progress = false;
+
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return ret;
+
+	pci_set_master(pdev);
+
+	/*
+	 * We are getting the base address of the SMB core. SMB core uses
+	 * BAR0 and size is 32K.
+	 */
+	ret = pcim_iomap_regions(pdev, BIT(0), pci_name(pdev));
+	if (ret < 0)
+		return ret;
+
+	i2c->i2c_base =	pcim_iomap_table(pdev)[0];
+	init_completion(&i2c->i2c_xfer_done);
+	pci1xxxx_i2c_init(i2c);
+
+	ret = devm_add_action(dev, (void (*)(void *))pci1xxxx_i2c_shutdown, i2c);
+	if (ret)
+		return ret;
+
+	ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
+	if (ret < 0)
+		return ret;
+
+	ret = devm_request_irq(dev, pci_irq_vector(pdev, 0), pci1xxxx_i2c_isr,
+			       0, pci_name(pdev), i2c);
+	if (ret)
+		return ret;
+
+	i2c->adap = pci1xxxx_i2c_ops;
+	i2c->adap.dev.parent = dev;
+
+	snprintf(i2c->adap.name, sizeof(i2c->adap.name),
+		 "MCHP PCI1xxxx i2c adapter at %s", pci_name(pdev));
+
+	i2c_set_adapdata(&i2c->adap, i2c);
+
+	ret = devm_i2c_add_adapter(dev, &i2c->adap);
+	if (ret)
+		return dev_err_probe(dev, ret, "i2c add adapter failed\n");
+
+	return 0;
+}
+
+static const struct pci_device_id pci1xxxx_i2c_pci_id_table[] = {
+	{ PCI_VDEVICE(EFAR, 0xA003) },
+	{ PCI_VDEVICE(EFAR, 0xA013) },
+	{ PCI_VDEVICE(EFAR, 0xA023) },
+	{ PCI_VDEVICE(EFAR, 0xA033) },
+	{ PCI_VDEVICE(EFAR, 0xA043) },
+	{ }
+};
+MODULE_DEVICE_TABLE(pci, pci1xxxx_i2c_pci_id_table);
+
+static struct pci_driver pci1xxxx_i2c_pci_driver = {
+	.name		= "i2c-mchp-pci1xxxx",
+	.id_table	= pci1xxxx_i2c_pci_id_table,
+	.probe		= pci1xxxx_i2c_probe_pci,
+	.driver = {
+		.pm = pm_sleep_ptr(&pci1xxxx_i2c_pm_ops),
+	},
+};
+module_pci_driver(pci1xxxx_i2c_pci_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Tharun Kumar P<tharunkumar.pasumarthi@microchip.com>");
+MODULE_AUTHOR("Kumaravel Thiagarajan <kumaravel.thiagarajan@microchip.com>");
+MODULE_DESCRIPTION("Microchip Technology Inc. pci1xxxx I2C bus driver");
diff --git a/drivers/i2c/busses/i2c-mlxbf.c b/drivers/i2c/busses/i2c-mlxbf.c
index ad5efd7497d1..e68e775f187e 100644
--- a/drivers/i2c/busses/i2c-mlxbf.c
+++ b/drivers/i2c/busses/i2c-mlxbf.c
@@ -32,8 +32,6 @@
 	(MLXBF_I2C_FUNC_SMBUS_DEFAULT | MLXBF_I2C_FUNC_SMBUS_BLOCK | \
 	 I2C_FUNC_SMBUS_QUICK | I2C_FUNC_SLAVE)
 
-#define MLXBF_I2C_SMBUS_MAX        3
-
 /* Shared resources info in BlueField platforms. */
 
 #define MLXBF_I2C_COALESCE_TYU_ADDR    0x02801300
@@ -48,6 +46,9 @@
 #define MLXBF_I2C_COREPLL_YU_ADDR      0x02800c30
 #define MLXBF_I2C_COREPLL_YU_SIZE      0x00c
 
+#define MLXBF_I2C_COREPLL_RSH_YU_ADDR  0x13409824
+#define MLXBF_I2C_COREPLL_RSH_YU_SIZE  0x00c
+
 #define MLXBF_I2C_SHARED_RES_MAX       3
 
 /*
@@ -131,14 +132,10 @@
 /* Slave busy bit reset. */
 #define MLXBF_I2C_CAUSE_S_GW_BUSY_FALL        BIT(18)
 
-#define MLXBF_I2C_CAUSE_SLAVE_ARBITER_BITS_MASK     GENMASK(20, 0)
-
 /* Cause coalesce registers. */
 #define MLXBF_I2C_CAUSE_COALESCE_0        0x00
-#define MLXBF_I2C_CAUSE_COALESCE_1        0x04
-#define MLXBF_I2C_CAUSE_COALESCE_2        0x08
 
-#define MLXBF_I2C_CAUSE_TYU_SLAVE_BIT   MLXBF_I2C_SMBUS_MAX
+#define MLXBF_I2C_CAUSE_TYU_SLAVE_BIT   3
 #define MLXBF_I2C_CAUSE_YU_SLAVE_BIT    1
 
 /* Functional enable register. */
@@ -165,15 +162,6 @@
 #define MLXBF_I2C_GPIO_SMBUS_GW_ASSERT_PINS(num, val) \
 	((val) | (0x3 << MLXBF_I2C_GPIO_SMBUS_GW_PINS(num)))
 
-/* SMBus timing parameters. */
-#define MLXBF_I2C_SMBUS_TIMER_SCL_LOW_SCL_HIGH    0x00
-#define MLXBF_I2C_SMBUS_TIMER_FALL_RISE_SPIKE     0x04
-#define MLXBF_I2C_SMBUS_TIMER_THOLD               0x08
-#define MLXBF_I2C_SMBUS_TIMER_TSETUP_START_STOP   0x0c
-#define MLXBF_I2C_SMBUS_TIMER_TSETUP_DATA         0x10
-#define MLXBF_I2C_SMBUS_THIGH_MAX_TBUF            0x14
-#define MLXBF_I2C_SMBUS_SCL_LOW_TIMEOUT           0x18
-
 /*
  * Defines SMBus operating frequency and core clock frequency.
  * According to ADB files, default values are compliant to 100KHz SMBus
@@ -192,26 +180,37 @@
 #define MLXBF_I2C_COREPLL_CORE_OD_YU_MASK   GENMASK(3, 0)
 #define MLXBF_I2C_COREPLL_CORE_R_YU_MASK    GENMASK(31, 26)
 
+/* SMBus timing parameters. */
+#define MLXBF_I2C_SMBUS_TIMER_SCL_LOW_SCL_HIGH    0x00
+#define MLXBF_I2C_SMBUS_TIMER_FALL_RISE_SPIKE     0x04
+#define MLXBF_I2C_SMBUS_TIMER_THOLD               0x08
+#define MLXBF_I2C_SMBUS_TIMER_TSETUP_START_STOP   0x0c
+#define MLXBF_I2C_SMBUS_TIMER_TSETUP_DATA         0x10
+#define MLXBF_I2C_SMBUS_THIGH_MAX_TBUF            0x14
+#define MLXBF_I2C_SMBUS_SCL_LOW_TIMEOUT           0x18
+
+#define MLXBF_I2C_SHIFT_0   0
+#define MLXBF_I2C_SHIFT_8   8
+#define MLXBF_I2C_SHIFT_16  16
+#define MLXBF_I2C_SHIFT_24  24
+
+#define MLXBF_I2C_MASK_8    GENMASK(7, 0)
+#define MLXBF_I2C_MASK_16   GENMASK(15, 0)
 
-/* Core PLL frequency. */
-static u64 mlxbf_i2c_corepll_frequency;
+#define MLXBF_I2C_MST_ADDR_OFFSET         0x200
 
 /* SMBus Master GW. */
-#define MLXBF_I2C_SMBUS_MASTER_GW     0x200
+#define MLXBF_I2C_SMBUS_MASTER_GW         0x0
 /* Number of bytes received and sent. */
-#define MLXBF_I2C_SMBUS_RS_BYTES      0x300
+#define MLXBF_I2C_YU_SMBUS_RS_BYTES       0x100
+#define MLXBF_I2C_RSH_YU_SMBUS_RS_BYTES   0x10c
 /* Packet error check (PEC) value. */
-#define MLXBF_I2C_SMBUS_MASTER_PEC    0x304
+#define MLXBF_I2C_SMBUS_MASTER_PEC        0x104
 /* Status bits (ACK/NACK/FW Timeout). */
-#define MLXBF_I2C_SMBUS_MASTER_STATUS 0x308
+#define MLXBF_I2C_SMBUS_MASTER_STATUS     0x108
 /* SMbus Master Finite State Machine. */
-#define MLXBF_I2C_SMBUS_MASTER_FSM    0x310
-
-/*
- * When enabled, the master will issue a stop condition in case of
- * timeout while waiting for FW response.
- */
-#define MLXBF_I2C_SMBUS_EN_FW_TIMEOUT 0x31c
+#define MLXBF_I2C_YU_SMBUS_MASTER_FSM     0x110
+#define MLXBF_I2C_RSH_YU_SMBUS_MASTER_FSM 0x100
 
 /* SMBus master GW control bits offset in MLXBF_I2C_SMBUS_MASTER_GW[31:3]. */
 #define MLXBF_I2C_MASTER_LOCK_BIT         BIT(31) /* Lock bit. */
@@ -231,14 +230,14 @@ static u64 mlxbf_i2c_corepll_frequency;
 #define MLXBF_I2C_MASTER_ENABLE_READ \
 	(MLXBF_I2C_MASTER_ENABLE | MLXBF_I2C_MASTER_CTL_READ_BIT)
 
-#define MLXBF_I2C_MASTER_SLV_ADDR_SHIFT   12 /* Slave address shift. */
-#define MLXBF_I2C_MASTER_WRITE_SHIFT      21 /* Control write bytes shift. */
-#define MLXBF_I2C_MASTER_SEND_PEC_SHIFT   20 /* Send PEC byte shift. */
-#define MLXBF_I2C_MASTER_PARSE_EXP_SHIFT  11 /* Parse expected bytes shift. */
-#define MLXBF_I2C_MASTER_READ_SHIFT       4  /* Control read bytes shift. */
+#define MLXBF_I2C_MASTER_WRITE_SHIFT      21 /* Control write bytes */
+#define MLXBF_I2C_MASTER_SEND_PEC_SHIFT   20 /* Send PEC byte when set to 1 */
+#define MLXBF_I2C_MASTER_PARSE_EXP_SHIFT  11 /* Control parse expected bytes */
+#define MLXBF_I2C_MASTER_SLV_ADDR_SHIFT   12 /* Slave address */
+#define MLXBF_I2C_MASTER_READ_SHIFT       4  /* Control read bytes */
 
 /* SMBus master GW Data descriptor. */
-#define MLXBF_I2C_MASTER_DATA_DESC_ADDR   0x280
+#define MLXBF_I2C_MASTER_DATA_DESC_ADDR   0x80
 #define MLXBF_I2C_MASTER_DATA_DESC_SIZE   0x80 /* Size in bytes. */
 
 /* Maximum bytes to read/write per SMBus transaction. */
@@ -264,19 +263,21 @@ static u64 mlxbf_i2c_corepll_frequency;
 #define MLXBF_I2C_SMBUS_MASTER_FSM_STOP_MASK      BIT(31)
 #define MLXBF_I2C_SMBUS_MASTER_FSM_PS_STATE_MASK  BIT(15)
 
+#define MLXBF_I2C_SLV_ADDR_OFFSET             0x400
+
 /* SMBus slave GW. */
-#define MLXBF_I2C_SMBUS_SLAVE_GW              0x400
+#define MLXBF_I2C_SMBUS_SLAVE_GW              0x0
 /* Number of bytes received and sent from/to master. */
-#define MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES 0x500
+#define MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES 0x100
 /* Packet error check (PEC) value. */
-#define MLXBF_I2C_SMBUS_SLAVE_PEC             0x504
+#define MLXBF_I2C_SMBUS_SLAVE_PEC             0x104
 /* SMBus slave Finite State Machine (FSM). */
-#define MLXBF_I2C_SMBUS_SLAVE_FSM             0x510
+#define MLXBF_I2C_SMBUS_SLAVE_FSM             0x110
 /*
  * Should be set when all raised causes handled, and cleared by HW on
  * every new cause.
  */
-#define MLXBF_I2C_SMBUS_SLAVE_READY           0x52c
+#define MLXBF_I2C_SMBUS_SLAVE_READY           0x12c
 
 /* SMBus slave GW control bits offset in MLXBF_I2C_SMBUS_SLAVE_GW[31:19]. */
 #define MLXBF_I2C_SLAVE_BUSY_BIT         BIT(30) /* Busy bit. */
@@ -289,23 +290,74 @@ static u64 mlxbf_i2c_corepll_frequency;
 #define MLXBF_I2C_SLAVE_SEND_PEC_SHIFT    21 /* Send PEC byte shift. */
 
 /* SMBus slave GW Data descriptor. */
-#define MLXBF_I2C_SLAVE_DATA_DESC_ADDR   0x480
+#define MLXBF_I2C_SLAVE_DATA_DESC_ADDR   0x80
 #define MLXBF_I2C_SLAVE_DATA_DESC_SIZE   0x80 /* Size in bytes. */
 
 /* SMbus slave configuration registers. */
-#define MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG        0x514
+#define MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG        0x114
 #define MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT        16
-#define MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT     7
+#define MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT     BIT(7)
 #define MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK       GENMASK(6, 0)
 
-#define MLXBF_I2C_SLAVE_ADDR_ENABLED(addr) \
-	((addr) & (1 << MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT))
-
 /*
  * Timeout is given in microsends. Note also that timeout handling is not
  * exact.
  */
 #define MLXBF_I2C_SMBUS_TIMEOUT   (300 * 1000) /* 300ms */
+#define MLXBF_I2C_SMBUS_LOCK_POLL_TIMEOUT (300 * 1000) /* 300ms */
+
+/* Polling frequency in microseconds. */
+#define MLXBF_I2C_POLL_FREQ_IN_USEC        200
+
+#define MLXBF_I2C_SMBUS_OP_CNT_1   1
+#define MLXBF_I2C_SMBUS_OP_CNT_2   2
+#define MLXBF_I2C_SMBUS_OP_CNT_3   3
+#define MLXBF_I2C_SMBUS_MAX_OP_CNT MLXBF_I2C_SMBUS_OP_CNT_3
+
+/* Helper macro to define an I2C resource parameters. */
+#define MLXBF_I2C_RES_PARAMS(addr, size, str) \
+	{ \
+		.start = (addr), \
+		.end = (addr) + (size) - 1, \
+		.name = (str) \
+	}
+
+enum {
+	MLXBF_I2C_TIMING_100KHZ = 100000,
+	MLXBF_I2C_TIMING_400KHZ = 400000,
+	MLXBF_I2C_TIMING_1000KHZ = 1000000,
+};
+
+enum {
+	MLXBF_I2C_F_READ = BIT(0),
+	MLXBF_I2C_F_WRITE = BIT(1),
+	MLXBF_I2C_F_NORESTART = BIT(3),
+	MLXBF_I2C_F_SMBUS_OPERATION = BIT(4),
+	MLXBF_I2C_F_SMBUS_BLOCK = BIT(5),
+	MLXBF_I2C_F_SMBUS_PEC = BIT(6),
+	MLXBF_I2C_F_SMBUS_PROCESS_CALL = BIT(7),
+};
+
+/* Mellanox BlueField chip type. */
+enum mlxbf_i2c_chip_type {
+	MLXBF_I2C_CHIP_TYPE_1, /* Mellanox BlueField-1 chip. */
+	MLXBF_I2C_CHIP_TYPE_2, /* Mellanox BlueField-2 chip. */
+	MLXBF_I2C_CHIP_TYPE_3 /* Mellanox BlueField-3 chip. */
+};
+
+/* List of chip resources that are being accessed by the driver. */
+enum {
+	MLXBF_I2C_SMBUS_RES,
+	MLXBF_I2C_MST_CAUSE_RES,
+	MLXBF_I2C_SLV_CAUSE_RES,
+	MLXBF_I2C_COALESCE_RES,
+	MLXBF_I2C_SMBUS_TIMER_RES,
+	MLXBF_I2C_SMBUS_MST_RES,
+	MLXBF_I2C_SMBUS_SLV_RES,
+	MLXBF_I2C_COREPLL_RES,
+	MLXBF_I2C_GPIO_RES,
+	MLXBF_I2C_END_RES
+};
 
 /* Encapsulates timing parameters. */
 struct mlxbf_i2c_timings {
@@ -326,27 +378,12 @@ struct mlxbf_i2c_timings {
 	u32 timeout;		/* Detect clock low timeout. */
 };
 
-enum {
-	MLXBF_I2C_F_READ = BIT(0),
-	MLXBF_I2C_F_WRITE = BIT(1),
-	MLXBF_I2C_F_NORESTART = BIT(3),
-	MLXBF_I2C_F_SMBUS_OPERATION = BIT(4),
-	MLXBF_I2C_F_SMBUS_BLOCK = BIT(5),
-	MLXBF_I2C_F_SMBUS_PEC = BIT(6),
-	MLXBF_I2C_F_SMBUS_PROCESS_CALL = BIT(7),
-};
-
 struct mlxbf_i2c_smbus_operation {
 	u32 flags;
 	u32 length; /* Buffer length in bytes. */
 	u8 *buffer;
 };
 
-#define MLXBF_I2C_SMBUS_OP_CNT_1	1
-#define MLXBF_I2C_SMBUS_OP_CNT_2	2
-#define MLXBF_I2C_SMBUS_OP_CNT_3	3
-#define MLXBF_I2C_SMBUS_MAX_OP_CNT	MLXBF_I2C_SMBUS_OP_CNT_3
-
 struct mlxbf_i2c_smbus_request {
 	u8 slave;
 	u8 operation_cnt;
@@ -360,24 +397,38 @@ struct mlxbf_i2c_resource {
 	u8 type;
 };
 
-/* List of chip resources that are being accessed by the driver. */
-enum {
-	MLXBF_I2C_SMBUS_RES,
-	MLXBF_I2C_MST_CAUSE_RES,
-	MLXBF_I2C_SLV_CAUSE_RES,
-	MLXBF_I2C_COALESCE_RES,
-	MLXBF_I2C_COREPLL_RES,
-	MLXBF_I2C_GPIO_RES,
-	MLXBF_I2C_END_RES,
+struct mlxbf_i2c_chip_info {
+	enum mlxbf_i2c_chip_type type;
+	/* Chip shared resources that are being used by the I2C controller. */
+	struct mlxbf_i2c_resource *shared_res[MLXBF_I2C_SHARED_RES_MAX];
+
+	/* Callback to calculate the core PLL frequency. */
+	u64 (*calculate_freq)(struct mlxbf_i2c_resource *corepll_res);
+
+	/* Registers' address offset */
+	u32 smbus_master_rs_bytes_off;
+	u32 smbus_master_fsm_off;
 };
 
-/* Helper macro to define an I2C resource parameters. */
-#define MLXBF_I2C_RES_PARAMS(addr, size, str) \
-	{ \
-		.start = (addr), \
-		.end = (addr) + (size) - 1, \
-		.name = (str) \
-	}
+struct mlxbf_i2c_priv {
+	const struct mlxbf_i2c_chip_info *chip;
+	struct i2c_adapter adap;
+	struct mlxbf_i2c_resource *smbus;
+	struct mlxbf_i2c_resource *timer;
+	struct mlxbf_i2c_resource *mst;
+	struct mlxbf_i2c_resource *slv;
+	struct mlxbf_i2c_resource *mst_cause;
+	struct mlxbf_i2c_resource *slv_cause;
+	struct mlxbf_i2c_resource *coalesce;
+	u64 frequency; /* Core frequency in Hz. */
+	int bus; /* Physical bus identifier. */
+	int irq;
+	struct i2c_client *slave[MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT];
+	u32 resource_version;
+};
+
+/* Core PLL frequency. */
+static u64 mlxbf_i2c_corepll_frequency;
 
 static struct resource mlxbf_i2c_coalesce_tyu_params =
 		MLXBF_I2C_RES_PARAMS(MLXBF_I2C_COALESCE_TYU_ADDR,
@@ -391,6 +442,10 @@ static struct resource mlxbf_i2c_corepll_yu_params =
 		MLXBF_I2C_RES_PARAMS(MLXBF_I2C_COREPLL_YU_ADDR,
 				     MLXBF_I2C_COREPLL_YU_SIZE,
 				     "COREPLL_MEM");
+static struct resource mlxbf_i2c_corepll_rsh_yu_params =
+		MLXBF_I2C_RES_PARAMS(MLXBF_I2C_COREPLL_RSH_YU_ADDR,
+				     MLXBF_I2C_COREPLL_RSH_YU_SIZE,
+				     "COREPLL_MEM");
 static struct resource mlxbf_i2c_gpio_tyu_params =
 		MLXBF_I2C_RES_PARAMS(MLXBF_I2C_GPIO_TYU_ADDR,
 				     MLXBF_I2C_GPIO_TYU_SIZE,
@@ -400,34 +455,6 @@ static struct mutex mlxbf_i2c_coalesce_lock;
 static struct mutex mlxbf_i2c_corepll_lock;
 static struct mutex mlxbf_i2c_gpio_lock;
 
-/* Mellanox BlueField chip type. */
-enum mlxbf_i2c_chip_type {
-	MLXBF_I2C_CHIP_TYPE_1, /* Mellanox BlueField-1 chip. */
-	MLXBF_I2C_CHIP_TYPE_2, /* Mallanox BlueField-2 chip. */
-};
-
-struct mlxbf_i2c_chip_info {
-	enum mlxbf_i2c_chip_type type;
-	/* Chip shared resources that are being used by the I2C controller. */
-	struct mlxbf_i2c_resource *shared_res[MLXBF_I2C_SHARED_RES_MAX];
-
-	/* Callback to calculate the core PLL frequency. */
-	u64 (*calculate_freq)(struct mlxbf_i2c_resource *corepll_res);
-};
-
-struct mlxbf_i2c_priv {
-	const struct mlxbf_i2c_chip_info *chip;
-	struct i2c_adapter adap;
-	struct mlxbf_i2c_resource *smbus;
-	struct mlxbf_i2c_resource *mst_cause;
-	struct mlxbf_i2c_resource *slv_cause;
-	struct mlxbf_i2c_resource *coalesce;
-	u64 frequency; /* Core frequency in Hz. */
-	int bus; /* Physical bus identifier. */
-	int irq;
-	struct i2c_client *slave;
-};
-
 static struct mlxbf_i2c_resource mlxbf_i2c_coalesce_res[] = {
 	[MLXBF_I2C_CHIP_TYPE_1] = {
 		.params = &mlxbf_i2c_coalesce_tyu_params,
@@ -447,6 +474,11 @@ static struct mlxbf_i2c_resource mlxbf_i2c_corepll_res[] = {
 		.params = &mlxbf_i2c_corepll_yu_params,
 		.lock = &mlxbf_i2c_corepll_lock,
 		.type = MLXBF_I2C_COREPLL_RES,
+	},
+	[MLXBF_I2C_CHIP_TYPE_3] = {
+		.params = &mlxbf_i2c_corepll_rsh_yu_params,
+		.lock = &mlxbf_i2c_corepll_lock,
+		.type = MLXBF_I2C_COREPLL_RES,
 	}
 };
 
@@ -463,24 +495,13 @@ static u8 mlxbf_i2c_bus_count;
 
 static struct mutex mlxbf_i2c_bus_lock;
 
-/* Polling frequency in microseconds. */
-#define MLXBF_I2C_POLL_FREQ_IN_USEC        200
-
-#define MLXBF_I2C_SHIFT_0   0
-#define MLXBF_I2C_SHIFT_8   8
-#define MLXBF_I2C_SHIFT_16  16
-#define MLXBF_I2C_SHIFT_24  24
-
-#define MLXBF_I2C_MASK_8    GENMASK(7, 0)
-#define MLXBF_I2C_MASK_16   GENMASK(15, 0)
-
 /*
  * Function to poll a set of bits at a specific address; it checks whether
  * the bits are equal to zero when eq_zero is set to 'true', and not equal
  * to zero when eq_zero is set to 'false'.
  * Note that the timeout is given in microseconds.
  */
-static u32 mlxbf_smbus_poll(void __iomem *io, u32 addr, u32 mask,
+static u32 mlxbf_i2c_poll(void __iomem *io, u32 addr, u32 mask,
 			    bool eq_zero, u32  timeout)
 {
 	u32 bits;
@@ -502,18 +523,37 @@ static u32 mlxbf_smbus_poll(void __iomem *io, u32 addr, u32 mask,
  * a transaction. Accordingly, this function polls the Master FSM stop
  * bit; it returns false when the bit is asserted, true if not.
  */
-static bool mlxbf_smbus_master_wait_for_idle(struct mlxbf_i2c_priv *priv)
+static bool mlxbf_i2c_smbus_master_wait_for_idle(struct mlxbf_i2c_priv *priv)
 {
 	u32 mask = MLXBF_I2C_SMBUS_MASTER_FSM_STOP_MASK;
-	u32 addr = MLXBF_I2C_SMBUS_MASTER_FSM;
+	u32 addr = priv->chip->smbus_master_fsm_off;
 	u32 timeout = MLXBF_I2C_SMBUS_TIMEOUT;
 
-	if (mlxbf_smbus_poll(priv->smbus->io, addr, mask, true, timeout))
+	if (mlxbf_i2c_poll(priv->mst->io, addr, mask, true, timeout))
+		return true;
+
+	return false;
+}
+
+/*
+ * wait for the lock to be released before acquiring it.
+ */
+static bool mlxbf_i2c_smbus_master_lock(struct mlxbf_i2c_priv *priv)
+{
+	if (mlxbf_i2c_poll(priv->mst->io, MLXBF_I2C_SMBUS_MASTER_GW,
+			   MLXBF_I2C_MASTER_LOCK_BIT, true,
+			   MLXBF_I2C_SMBUS_LOCK_POLL_TIMEOUT))
 		return true;
 
 	return false;
 }
 
+static void mlxbf_i2c_smbus_master_unlock(struct mlxbf_i2c_priv *priv)
+{
+	/* Clear the gw to clear the lock */
+	writel(0, priv->mst->io + MLXBF_I2C_SMBUS_MASTER_GW);
+}
+
 static bool mlxbf_i2c_smbus_transaction_success(u32 master_status,
 						u32 cause_status)
 {
@@ -551,7 +591,7 @@ static int mlxbf_i2c_smbus_check_status(struct mlxbf_i2c_priv *priv)
 	 * then read the cause and master status bits to determine if
 	 * errors occurred during the transaction.
 	 */
-	mlxbf_smbus_poll(priv->smbus->io, MLXBF_I2C_SMBUS_MASTER_GW,
+	mlxbf_i2c_poll(priv->mst->io, MLXBF_I2C_SMBUS_MASTER_GW,
 			 MLXBF_I2C_MASTER_BUSY_BIT, true,
 			 MLXBF_I2C_SMBUS_TIMEOUT);
 
@@ -564,7 +604,7 @@ static int mlxbf_i2c_smbus_check_status(struct mlxbf_i2c_priv *priv)
 	 * Parse both Cause and Master GW bits, then return transaction status.
 	 */
 
-	master_status_bits = readl(priv->smbus->io +
+	master_status_bits = readl(priv->mst->io +
 					MLXBF_I2C_SMBUS_MASTER_STATUS);
 	master_status_bits &= MLXBF_I2C_SMBUS_MASTER_STATUS_MASK;
 
@@ -589,7 +629,8 @@ static int mlxbf_i2c_smbus_check_status(struct mlxbf_i2c_priv *priv)
 }
 
 static void mlxbf_i2c_smbus_write_data(struct mlxbf_i2c_priv *priv,
-				       const u8 *data, u8 length, u32 addr)
+				       const u8 *data, u8 length, u32 addr,
+				       bool is_master)
 {
 	u8 offset, aligned_length;
 	u32 data32;
@@ -606,12 +647,16 @@ static void mlxbf_i2c_smbus_write_data(struct mlxbf_i2c_priv *priv,
 	 */
 	for (offset = 0; offset < aligned_length; offset += sizeof(u32)) {
 		data32 = *((u32 *)(data + offset));
-		iowrite32be(data32, priv->smbus->io + addr + offset);
+		if (is_master)
+			iowrite32be(data32, priv->mst->io + addr + offset);
+		else
+			iowrite32be(data32, priv->slv->io + addr + offset);
 	}
 }
 
 static void mlxbf_i2c_smbus_read_data(struct mlxbf_i2c_priv *priv,
-				      u8 *data, u8 length, u32 addr)
+				      u8 *data, u8 length, u32 addr,
+				      bool is_master)
 {
 	u32 data32, mask;
 	u8 byte, offset;
@@ -627,14 +672,20 @@ static void mlxbf_i2c_smbus_read_data(struct mlxbf_i2c_priv *priv,
 	 */
 
 	for (offset = 0; offset < (length & ~mask); offset += sizeof(u32)) {
-		data32 = ioread32be(priv->smbus->io + addr + offset);
+		if (is_master)
+			data32 = ioread32be(priv->mst->io + addr + offset);
+		else
+			data32 = ioread32be(priv->slv->io + addr + offset);
 		*((u32 *)(data + offset)) = data32;
 	}
 
 	if (!(length & mask))
 		return;
 
-	data32 = ioread32be(priv->smbus->io + addr + offset);
+	if (is_master)
+		data32 = ioread32be(priv->mst->io + addr + offset);
+	else
+		data32 = ioread32be(priv->slv->io + addr + offset);
 
 	for (byte = 0; byte < (length & mask); byte++) {
 		data[offset + byte] = data32 & GENMASK(7, 0);
@@ -660,16 +711,16 @@ static int mlxbf_i2c_smbus_enable(struct mlxbf_i2c_priv *priv, u8 slave,
 	command |= rol32(pec_en, MLXBF_I2C_MASTER_SEND_PEC_SHIFT);
 
 	/* Clear status bits. */
-	writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_MASTER_STATUS);
+	writel(0x0, priv->mst->io + MLXBF_I2C_SMBUS_MASTER_STATUS);
 	/* Set the cause data. */
 	writel(~0x0, priv->mst_cause->io + MLXBF_I2C_CAUSE_OR_CLEAR);
 	/* Zero PEC byte. */
-	writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_MASTER_PEC);
+	writel(0x0, priv->mst->io + MLXBF_I2C_SMBUS_MASTER_PEC);
 	/* Zero byte count. */
-	writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_RS_BYTES);
+	writel(0x0, priv->mst->io + priv->chip->smbus_master_rs_bytes_off);
 
 	/* GW activation. */
-	writel(command, priv->smbus->io + MLXBF_I2C_SMBUS_MASTER_GW);
+	writel(command, priv->mst->io + MLXBF_I2C_SMBUS_MASTER_GW);
 
 	/*
 	 * Poll master status and check status bits. An ACK is sent when
@@ -705,10 +756,19 @@ mlxbf_i2c_smbus_start_transaction(struct mlxbf_i2c_priv *priv,
 	slave = request->slave & GENMASK(6, 0);
 	addr = slave << 1;
 
-	/* First of all, check whether the HW is idle. */
-	if (WARN_ON(!mlxbf_smbus_master_wait_for_idle(priv)))
+	/*
+	 * Try to acquire the smbus gw lock before any reads of the GW register since
+	 * a read sets the lock.
+	 */
+	if (WARN_ON(!mlxbf_i2c_smbus_master_lock(priv)))
 		return -EBUSY;
 
+	/* Check whether the HW is idle */
+	if (WARN_ON(!mlxbf_i2c_smbus_master_wait_for_idle(priv))) {
+		ret = -EBUSY;
+		goto out_unlock;
+	}
+
 	/* Set first byte. */
 	data_desc[data_idx++] = addr;
 
@@ -732,8 +792,10 @@ mlxbf_i2c_smbus_start_transaction(struct mlxbf_i2c_priv *priv,
 			write_en = 1;
 			write_len += operation->length;
 			if (data_idx + operation->length >
-					MLXBF_I2C_MASTER_DATA_DESC_SIZE)
-				return -ENOBUFS;
+					MLXBF_I2C_MASTER_DATA_DESC_SIZE) {
+				ret = -ENOBUFS;
+				goto out_unlock;
+			}
 			memcpy(data_desc + data_idx,
 			       operation->buffer, operation->length);
 			data_idx += operation->length;
@@ -759,25 +821,25 @@ mlxbf_i2c_smbus_start_transaction(struct mlxbf_i2c_priv *priv,
 	 * must be written to the data registers.
 	 */
 	mlxbf_i2c_smbus_write_data(priv, (const u8 *)data_desc, data_len,
-				   MLXBF_I2C_MASTER_DATA_DESC_ADDR);
+				   MLXBF_I2C_MASTER_DATA_DESC_ADDR, true);
 
 	if (write_en) {
 		ret = mlxbf_i2c_smbus_enable(priv, slave, write_len, block_en,
 					 pec_en, 0);
 		if (ret)
-			return ret;
+			goto out_unlock;
 	}
 
 	if (read_en) {
 		/* Write slave address to Master GW data descriptor. */
 		mlxbf_i2c_smbus_write_data(priv, (const u8 *)&addr, 1,
-					   MLXBF_I2C_MASTER_DATA_DESC_ADDR);
+					   MLXBF_I2C_MASTER_DATA_DESC_ADDR, true);
 		ret = mlxbf_i2c_smbus_enable(priv, slave, read_len, block_en,
 					 pec_en, 1);
 		if (!ret) {
 			/* Get Master GW data descriptor. */
 			mlxbf_i2c_smbus_read_data(priv, data_desc, read_len + 1,
-					     MLXBF_I2C_MASTER_DATA_DESC_ADDR);
+					     MLXBF_I2C_MASTER_DATA_DESC_ADDR, true);
 
 			/* Get data from Master GW data descriptor. */
 			memcpy(read_buf, data_desc, read_len + 1);
@@ -789,9 +851,12 @@ mlxbf_i2c_smbus_start_transaction(struct mlxbf_i2c_priv *priv,
 		 * next tag integration.
 		 */
 		writel(MLXBF_I2C_SMBUS_MASTER_FSM_PS_STATE_MASK,
-			priv->smbus->io + MLXBF_I2C_SMBUS_MASTER_FSM);
+			priv->mst->io + priv->chip->smbus_master_fsm_off);
 	}
 
+out_unlock:
+	mlxbf_i2c_smbus_master_unlock(priv);
+
 	return ret;
 }
 
@@ -1078,7 +1143,7 @@ static void mlxbf_i2c_set_timings(struct mlxbf_i2c_priv *priv,
 	timer |= mlxbf_i2c_set_timer(priv, timings->scl_low,
 				     false, MLXBF_I2C_MASK_16,
 				     MLXBF_I2C_SHIFT_16);
-	writel(timer, priv->smbus->io +
+	writel(timer, priv->timer->io +
 		MLXBF_I2C_SMBUS_TIMER_SCL_LOW_SCL_HIGH);
 
 	timer = mlxbf_i2c_set_timer(priv, timings->sda_rise, false,
@@ -1089,34 +1154,34 @@ static void mlxbf_i2c_set_timings(struct mlxbf_i2c_priv *priv,
 				     MLXBF_I2C_MASK_8, MLXBF_I2C_SHIFT_16);
 	timer |= mlxbf_i2c_set_timer(priv, timings->scl_fall, false,
 				     MLXBF_I2C_MASK_8, MLXBF_I2C_SHIFT_24);
-	writel(timer, priv->smbus->io +
+	writel(timer, priv->timer->io +
 		MLXBF_I2C_SMBUS_TIMER_FALL_RISE_SPIKE);
 
 	timer = mlxbf_i2c_set_timer(priv, timings->hold_start, true,
 				    MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0);
 	timer |= mlxbf_i2c_set_timer(priv, timings->hold_data, true,
 				     MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_16);
-	writel(timer, priv->smbus->io + MLXBF_I2C_SMBUS_TIMER_THOLD);
+	writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_TIMER_THOLD);
 
 	timer = mlxbf_i2c_set_timer(priv, timings->setup_start, true,
 				    MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0);
 	timer |= mlxbf_i2c_set_timer(priv, timings->setup_stop, true,
 				     MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_16);
-	writel(timer, priv->smbus->io +
+	writel(timer, priv->timer->io +
 		MLXBF_I2C_SMBUS_TIMER_TSETUP_START_STOP);
 
 	timer = mlxbf_i2c_set_timer(priv, timings->setup_data, true,
 				    MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0);
-	writel(timer, priv->smbus->io + MLXBF_I2C_SMBUS_TIMER_TSETUP_DATA);
+	writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_TIMER_TSETUP_DATA);
 
 	timer = mlxbf_i2c_set_timer(priv, timings->buf, false,
 				    MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_0);
 	timer |= mlxbf_i2c_set_timer(priv, timings->thigh_max, false,
 				     MLXBF_I2C_MASK_16, MLXBF_I2C_SHIFT_16);
-	writel(timer, priv->smbus->io + MLXBF_I2C_SMBUS_THIGH_MAX_TBUF);
+	writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_THIGH_MAX_TBUF);
 
 	timer = timings->timeout;
-	writel(timer, priv->smbus->io + MLXBF_I2C_SMBUS_SCL_LOW_TIMEOUT);
+	writel(timer, priv->timer->io + MLXBF_I2C_SMBUS_SCL_LOW_TIMEOUT);
 }
 
 enum mlxbf_i2c_timings_config {
@@ -1509,28 +1574,26 @@ static int mlxbf_i2c_calculate_corepll_freq(struct platform_device *pdev,
 	return 0;
 }
 
-static int mlxbf_slave_enable(struct mlxbf_i2c_priv *priv, u8 addr)
+static int mlxbf_i2c_slave_enable(struct mlxbf_i2c_priv *priv,
+			      struct i2c_client *slave)
 {
-	u32 slave_reg, slave_reg_tmp, slave_reg_avail, slave_addr_mask;
-	u8 reg, reg_cnt, byte, addr_tmp, reg_avail, byte_avail;
-	bool avail, disabled;
-
-	disabled = false;
-	avail = false;
+	u8 reg, reg_cnt, byte, addr_tmp;
+	u32 slave_reg, slave_reg_tmp;
 
 	if (!priv)
 		return -EPERM;
 
 	reg_cnt = MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT >> 2;
-	slave_addr_mask = MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK;
 
 	/*
 	 * Read the slave registers. There are 4 * 32-bit slave registers.
-	 * Each slave register can hold up to 4 * 8-bit slave configuration
-	 * (7-bit address, 1 status bit (1 if enabled, 0 if not)).
+	 * Each slave register can hold up to 4 * 8-bit slave configuration:
+	 * 1) A 7-bit address
+	 * 2) And a status bit (1 if enabled, 0 if not).
+	 * Look for the next available slave register slot.
 	 */
 	for (reg = 0; reg < reg_cnt; reg++) {
-		slave_reg = readl(priv->smbus->io +
+		slave_reg = readl(priv->slv->io +
 				MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + reg * 0x4);
 		/*
 		 * Each register holds 4 slave addresses. So, we have to keep
@@ -1542,121 +1605,87 @@ static int mlxbf_slave_enable(struct mlxbf_i2c_priv *priv, u8 addr)
 			addr_tmp = slave_reg_tmp & GENMASK(7, 0);
 
 			/*
-			 * Mark the first available slave address slot, i.e. its
-			 * enabled bit should be unset. This slot might be used
-			 * later on to register our slave.
-			 */
-			if (!avail && !MLXBF_I2C_SLAVE_ADDR_ENABLED(addr_tmp)) {
-				avail = true;
-				reg_avail = reg;
-				byte_avail = byte;
-				slave_reg_avail = slave_reg;
-			}
-
-			/*
-			 * Parse slave address bytes and check whether the
-			 * slave address already exists and it's enabled,
-			 * i.e. most significant bit is set.
+			 * If an enable bit is not set in the
+			 * MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG register, then the
+			 * slave address slot associated with that bit is
+			 * free. So set the enable bit and write the
+			 * slave address bits.
 			 */
-			if ((addr_tmp & slave_addr_mask) == addr) {
-				if (MLXBF_I2C_SLAVE_ADDR_ENABLED(addr_tmp))
-					return 0;
-				disabled = true;
-				break;
+			if (!(addr_tmp & MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT)) {
+				slave_reg &= ~(MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK << (byte * 8));
+				slave_reg |= (slave->addr << (byte * 8));
+				slave_reg |= MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT << (byte * 8);
+				writel(slave_reg, priv->slv->io +
+					MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG +
+					(reg * 0x4));
+
+				/*
+				 * Set the slave at the corresponding index.
+				 */
+				priv->slave[(reg * 4) + byte] = slave;
+
+				return 0;
 			}
 
 			/* Parse next byte. */
 			slave_reg_tmp >>= 8;
 		}
-
-		/* Exit the loop if the slave address is found. */
-		if (disabled)
-			break;
-	}
-
-	if (!avail && !disabled)
-		return -EINVAL; /* No room for a new slave address. */
-
-	if (avail && !disabled) {
-		reg = reg_avail;
-		byte = byte_avail;
-		/* Set the slave address. */
-		slave_reg_avail &= ~(slave_addr_mask << (byte * 8));
-		slave_reg_avail |= addr << (byte * 8);
-		slave_reg = slave_reg_avail;
 	}
 
-	/* Enable the slave address and update the register. */
-	slave_reg |= (1 << MLXBF_I2C_SMBUS_SLAVE_ADDR_EN_BIT) << (byte * 8);
-	writel(slave_reg, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG +
-		reg * 0x4);
-
-	return 0;
+	return -EBUSY;
 }
 
-static int mlxbf_slave_disable(struct mlxbf_i2c_priv *priv)
+static int mlxbf_i2c_slave_disable(struct mlxbf_i2c_priv *priv, u8 addr)
 {
-	u32 slave_reg, slave_reg_tmp, slave_addr_mask;
-	u8 addr, addr_tmp, reg, reg_cnt, slave_byte;
-	struct i2c_client *client = priv->slave;
-	bool exist;
+	u8 addr_tmp, reg, reg_cnt, byte;
+	u32 slave_reg, slave_reg_tmp;
 
-	exist = false;
-
-	addr = client->addr;
 	reg_cnt = MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT >> 2;
-	slave_addr_mask = MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK;
 
 	/*
 	 * Read the slave registers. There are 4 * 32-bit slave registers.
-	 * Each slave register can hold up to 4 * 8-bit slave configuration
-	 * (7-bit address, 1 status bit (1 if enabled, 0 if not)).
+	 * Each slave register can hold up to 4 * 8-bit slave configuration:
+	 * 1) A 7-bit address
+	 * 2) And a status bit (1 if enabled, 0 if not).
+	 * Check if addr is present in the registers.
 	 */
 	for (reg = 0; reg < reg_cnt; reg++) {
-		slave_reg = readl(priv->smbus->io +
+		slave_reg = readl(priv->slv->io +
 				MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG + reg * 0x4);
 
 		/* Check whether the address slots are empty. */
-		if (slave_reg == 0)
+		if (!slave_reg)
 			continue;
 
 		/*
-		 * Each register holds 4 slave addresses. So, we have to keep
-		 * the byte order consistent with the value read in order to
-		 * update the register correctly, if needed.
+		 * Check if addr matches any of the 4 slave addresses
+		 * in the register.
 		 */
 		slave_reg_tmp = slave_reg;
-		slave_byte = 0;
-		while (slave_reg_tmp != 0) {
-			addr_tmp = slave_reg_tmp & slave_addr_mask;
+		for (byte = 0; byte < 4; byte++) {
+			addr_tmp = slave_reg_tmp & MLXBF_I2C_SMBUS_SLAVE_ADDR_MASK;
 			/*
 			 * Parse slave address bytes and check whether the
 			 * slave address already exists.
 			 */
 			if (addr_tmp == addr) {
-				exist = true;
-				break;
+				/* Clear the slave address slot. */
+				slave_reg &= ~(GENMASK(7, 0) << (byte * 8));
+				writel(slave_reg, priv->slv->io +
+					MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG +
+					(reg * 0x4));
+				/* Free slave at the corresponding index */
+				priv->slave[(reg * 4) + byte] = NULL;
+
+				return 0;
 			}
 
 			/* Parse next byte. */
 			slave_reg_tmp >>= 8;
-			slave_byte += 1;
 		}
-
-		/* Exit the loop if the slave address is found. */
-		if (exist)
-			break;
 	}
 
-	if (!exist)
-		return 0; /* Slave is not registered, nothing to do. */
-
-	/* Cleanup the slave address slot. */
-	slave_reg &= ~(GENMASK(7, 0) << (slave_byte * 8));
-	writel(slave_reg, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_ADDR_CFG +
-		reg * 0x4);
-
-	return 0;
+	return -ENXIO;
 }
 
 static int mlxbf_i2c_init_coalesce(struct platform_device *pdev,
@@ -1746,7 +1775,7 @@ static int mlxbf_i2c_init_slave(struct platform_device *pdev,
 	int ret;
 
 	/* Reset FSM. */
-	writel(0, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_FSM);
+	writel(0, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_FSM);
 
 	/*
 	 * Enable slave cause interrupt bits. Drive
@@ -1761,7 +1790,7 @@ static int mlxbf_i2c_init_slave(struct platform_device *pdev,
 	writel(int_reg, priv->slv_cause->io + MLXBF_I2C_CAUSE_OR_EVTEN0);
 
 	/* Finally, set the 'ready' bit to start handling transactions. */
-	writel(0x1, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_READY);
+	writel(0x1, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_READY);
 
 	/* Initialize the cause coalesce resource. */
 	ret = mlxbf_i2c_init_coalesce(pdev, priv);
@@ -1806,84 +1835,93 @@ static bool mlxbf_i2c_has_coalesce(struct mlxbf_i2c_priv *priv, bool *read,
 	return true;
 }
 
-static bool mlxbf_smbus_slave_wait_for_idle(struct mlxbf_i2c_priv *priv,
+static bool mlxbf_i2c_slave_wait_for_idle(struct mlxbf_i2c_priv *priv,
 					    u32 timeout)
 {
 	u32 mask = MLXBF_I2C_CAUSE_S_GW_BUSY_FALL;
 	u32 addr = MLXBF_I2C_CAUSE_ARBITER;
 
-	if (mlxbf_smbus_poll(priv->slv_cause->io, addr, mask, false, timeout))
+	if (mlxbf_i2c_poll(priv->slv_cause->io, addr, mask, false, timeout))
 		return true;
 
 	return false;
 }
 
-/* Send byte to 'external' smbus master. */
-static int mlxbf_smbus_irq_send(struct mlxbf_i2c_priv *priv, u8 recv_bytes)
+static struct i2c_client *mlxbf_i2c_get_slave_from_addr(
+			struct mlxbf_i2c_priv *priv, u8 addr)
 {
-	u8 data_desc[MLXBF_I2C_SLAVE_DATA_DESC_SIZE] = { 0 };
-	u8 write_size, pec_en, addr, byte, value, byte_cnt, desc_size;
-	struct i2c_client *slave = priv->slave;
-	u32 control32, data32;
-	int ret;
+	int i;
 
-	if (!slave)
-		return -EINVAL;
+	for (i = 0; i < MLXBF_I2C_SMBUS_SLAVE_ADDR_CNT; i++) {
+		if (!priv->slave[i])
+			continue;
+
+		if (priv->slave[i]->addr == addr)
+			return priv->slave[i];
+	}
 
-	addr = 0;
-	byte = 0;
-	desc_size = MLXBF_I2C_SLAVE_DATA_DESC_SIZE;
+	return NULL;
+}
+
+/*
+ * Send byte to 'external' smbus master. This function is executed when
+ * an external smbus master wants to read data from the BlueField.
+ */
+static int mlxbf_i2c_irq_send(struct mlxbf_i2c_priv *priv, u8 recv_bytes)
+{
+	u8 data_desc[MLXBF_I2C_SLAVE_DATA_DESC_SIZE] = { 0 };
+	u8 write_size, pec_en, addr, value, byte_cnt;
+	struct i2c_client *slave;
+	u32 control32, data32;
+	int ret = 0;
 
 	/*
-	 * Read bytes received from the external master. These bytes should
-	 * be located in the first data descriptor register of the slave GW.
-	 * These bytes are the slave address byte and the internal register
-	 * address, if supplied.
+	 * Read the first byte received from the external master to
+	 * determine the slave address. This byte is located in the
+	 * first data descriptor register of the slave GW.
 	 */
-	if (recv_bytes > 0) {
-		data32 = ioread32be(priv->smbus->io +
-					MLXBF_I2C_SLAVE_DATA_DESC_ADDR);
-
-		/* Parse the received bytes. */
-		switch (recv_bytes) {
-		case 2:
-			byte = (data32 >> 8) & GENMASK(7, 0);
-			fallthrough;
-		case 1:
-			addr = (data32 & GENMASK(7, 0)) >> 1;
-		}
+	data32 = ioread32be(priv->slv->io +
+				MLXBF_I2C_SLAVE_DATA_DESC_ADDR);
+	addr = (data32 & GENMASK(7, 0)) >> 1;
 
-		/* Check whether it's our slave address. */
-		if (slave->addr != addr)
-			return -EINVAL;
+	/*
+	 * Check if the slave address received in the data descriptor register
+	 * matches any of the slave addresses registered. If there is a match,
+	 * set the slave.
+	 */
+	slave = mlxbf_i2c_get_slave_from_addr(priv, addr);
+	if (!slave) {
+		ret = -ENXIO;
+		goto clear_csr;
 	}
 
 	/*
-	 * I2C read transactions may start by a WRITE followed by a READ.
-	 * Indeed, most slave devices would expect the internal address
-	 * following the slave address byte. So, write that byte first,
-	 * and then, send the requested data bytes to the master.
+	 * An I2C read can consist of a WRITE bit transaction followed by
+	 * a READ bit transaction. Indeed, slave devices often expect
+	 * the slave address to be followed by the internal address.
+	 * So, write the internal address byte first, and then, send the
+	 * requested data to the master.
 	 */
 	if (recv_bytes > 1) {
 		i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value);
-		value = byte;
+		value = (data32 >> 8) & GENMASK(7, 0);
 		ret = i2c_slave_event(slave, I2C_SLAVE_WRITE_RECEIVED,
 				      &value);
 		i2c_slave_event(slave, I2C_SLAVE_STOP, &value);
 
 		if (ret < 0)
-			return ret;
+			goto clear_csr;
 	}
 
 	/*
-	 * Now, send data to the master; currently, the driver supports
-	 * READ_BYTE, READ_WORD and BLOCK READ protocols. Note that the
-	 * hardware can send up to 128 bytes per transfer. That is the
-	 * size of its data registers.
+	 * Send data to the master. Currently, the driver supports
+	 * READ_BYTE, READ_WORD and BLOCK READ protocols. The
+	 * hardware can send up to 128 bytes per transfer which is
+	 * the total size of the data registers.
 	 */
 	i2c_slave_event(slave, I2C_SLAVE_READ_REQUESTED, &value);
 
-	for (byte_cnt = 0; byte_cnt < desc_size; byte_cnt++) {
+	for (byte_cnt = 0; byte_cnt < MLXBF_I2C_SLAVE_DATA_DESC_SIZE; byte_cnt++) {
 		data_desc[byte_cnt] = value;
 		i2c_slave_event(slave, I2C_SLAVE_READ_PROCESSED, &value);
 	}
@@ -1891,14 +1929,12 @@ static int mlxbf_smbus_irq_send(struct mlxbf_i2c_priv *priv, u8 recv_bytes)
 	/* Send a stop condition to the backend. */
 	i2c_slave_event(slave, I2C_SLAVE_STOP, &value);
 
-	/* Handle the actual transfer. */
-
 	/* Set the number of bytes to write to master. */
 	write_size = (byte_cnt - 1) & 0x7f;
 
 	/* Write data to Slave GW data descriptor. */
 	mlxbf_i2c_smbus_write_data(priv, data_desc, byte_cnt,
-				   MLXBF_I2C_SLAVE_DATA_DESC_ADDR);
+				   MLXBF_I2C_SLAVE_DATA_DESC_ADDR, false);
 
 	pec_en = 0; /* Disable PEC since it is not supported. */
 
@@ -1907,46 +1943,52 @@ static int mlxbf_smbus_irq_send(struct mlxbf_i2c_priv *priv, u8 recv_bytes)
 	control32 |= rol32(write_size, MLXBF_I2C_SLAVE_WRITE_BYTES_SHIFT);
 	control32 |= rol32(pec_en, MLXBF_I2C_SLAVE_SEND_PEC_SHIFT);
 
-	writel(control32, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_GW);
+	writel(control32, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_GW);
 
 	/*
 	 * Wait until the transfer is completed; the driver will wait
 	 * until the GW is idle, a cause will rise on fall of GW busy.
 	 */
-	mlxbf_smbus_slave_wait_for_idle(priv, MLXBF_I2C_SMBUS_TIMEOUT);
+	mlxbf_i2c_slave_wait_for_idle(priv, MLXBF_I2C_SMBUS_TIMEOUT);
 
+clear_csr:
 	/* Release the Slave GW. */
-	writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES);
-	writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_PEC);
-	writel(0x1, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_READY);
+	writel(0x0, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES);
+	writel(0x0, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_PEC);
+	writel(0x1, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_READY);
 
-	return 0;
+	return ret;
 }
 
-/* Receive bytes from 'external' smbus master. */
-static int mlxbf_smbus_irq_recv(struct mlxbf_i2c_priv *priv, u8 recv_bytes)
+/*
+ * Receive bytes from 'external' smbus master. This function is executed when
+ * an external smbus master wants to write data to the BlueField.
+ */
+static int mlxbf_i2c_irq_recv(struct mlxbf_i2c_priv *priv, u8 recv_bytes)
 {
 	u8 data_desc[MLXBF_I2C_SLAVE_DATA_DESC_SIZE] = { 0 };
-	struct i2c_client *slave = priv->slave;
+	struct i2c_client *slave;
 	u8 value, byte, addr;
 	int ret = 0;
 
-	if (!slave)
-		return -EINVAL;
-
 	/* Read data from Slave GW data descriptor. */
 	mlxbf_i2c_smbus_read_data(priv, data_desc, recv_bytes,
-				  MLXBF_I2C_SLAVE_DATA_DESC_ADDR);
-
-	/* Check whether its our slave address. */
+				  MLXBF_I2C_SLAVE_DATA_DESC_ADDR, false);
 	addr = data_desc[0] >> 1;
-	if (slave->addr != addr)
-		return -EINVAL;
 
 	/*
-	 * Notify the slave backend; another I2C master wants to write data
-	 * to us. This event is sent once the slave address and the write bit
-	 * is detected.
+	 * Check if the slave address received in the data descriptor register
+	 * matches any of the slave addresses registered.
+	 */
+	slave = mlxbf_i2c_get_slave_from_addr(priv, addr);
+	if (!slave) {
+		ret = -EINVAL;
+		goto clear_csr;
+	}
+
+	/*
+	 * Notify the slave backend that an smbus master wants to write data
+	 * to the BlueField.
 	 */
 	i2c_slave_event(slave, I2C_SLAVE_WRITE_REQUESTED, &value);
 
@@ -1959,18 +2001,22 @@ static int mlxbf_smbus_irq_recv(struct mlxbf_i2c_priv *priv, u8 recv_bytes)
 			break;
 	}
 
-	/* Send a stop condition to the backend. */
+	/*
+	 * Send a stop event to the slave backend, to signal
+	 * the end of the write transactions.
+	 */
 	i2c_slave_event(slave, I2C_SLAVE_STOP, &value);
 
+clear_csr:
 	/* Release the Slave GW. */
-	writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES);
-	writel(0x0, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_PEC);
-	writel(0x1, priv->smbus->io + MLXBF_I2C_SMBUS_SLAVE_READY);
+	writel(0x0, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES);
+	writel(0x0, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_PEC);
+	writel(0x1, priv->slv->io + MLXBF_I2C_SMBUS_SLAVE_READY);
 
 	return ret;
 }
 
-static irqreturn_t mlxbf_smbus_irq(int irq, void *ptr)
+static irqreturn_t mlxbf_i2c_irq(int irq, void *ptr)
 {
 	struct mlxbf_i2c_priv *priv = ptr;
 	bool read, write, irq_is_set;
@@ -2000,7 +2046,7 @@ static irqreturn_t mlxbf_smbus_irq(int irq, void *ptr)
 	 * slave, if the higher 8 bits are sent then the slave expect N bytes
 	 * from the master.
 	 */
-	rw_bytes_reg = readl(priv->smbus->io +
+	rw_bytes_reg = readl(priv->slv->io +
 				MLXBF_I2C_SMBUS_SLAVE_RS_MASTER_BYTES);
 	recv_bytes = (rw_bytes_reg >> 8) & GENMASK(7, 0);
 
@@ -2018,9 +2064,9 @@ static irqreturn_t mlxbf_smbus_irq(int irq, void *ptr)
 		MLXBF_I2C_SLAVE_DATA_DESC_SIZE : recv_bytes;
 
 	if (read)
-		mlxbf_smbus_irq_send(priv, recv_bytes);
+		mlxbf_i2c_irq_send(priv, recv_bytes);
 	else
-		mlxbf_smbus_irq_recv(priv, recv_bytes);
+		mlxbf_i2c_irq_recv(priv, recv_bytes);
 
 	return IRQ_HANDLED;
 }
@@ -2115,23 +2161,21 @@ static s32 mlxbf_i2c_smbus_xfer(struct i2c_adapter *adap, u16 addr,
 static int mlxbf_i2c_reg_slave(struct i2c_client *slave)
 {
 	struct mlxbf_i2c_priv *priv = i2c_get_adapdata(slave->adapter);
+	struct device *dev = &slave->dev;
 	int ret;
 
-	if (priv->slave)
-		return -EBUSY;
-
 	/*
 	 * Do not support ten bit chip address and do not use Packet Error
 	 * Checking (PEC).
 	 */
-	if (slave->flags & (I2C_CLIENT_TEN | I2C_CLIENT_PEC))
+	if (slave->flags & (I2C_CLIENT_TEN | I2C_CLIENT_PEC)) {
+		dev_err(dev, "SMBus PEC and 10 bit address not supported\n");
 		return -EAFNOSUPPORT;
+	}
 
-	ret = mlxbf_slave_enable(priv, slave->addr);
-	if (ret < 0)
-		return ret;
-
-	priv->slave = slave;
+	ret = mlxbf_i2c_slave_enable(priv, slave);
+	if (ret)
+		dev_err(dev, "Surpassed max number of registered slaves allowed\n");
 
 	return 0;
 }
@@ -2139,18 +2183,19 @@ static int mlxbf_i2c_reg_slave(struct i2c_client *slave)
 static int mlxbf_i2c_unreg_slave(struct i2c_client *slave)
 {
 	struct mlxbf_i2c_priv *priv = i2c_get_adapdata(slave->adapter);
+	struct device *dev = &slave->dev;
 	int ret;
 
-	WARN_ON(!priv->slave);
-
-	/* Unregister slave, i.e. disable the slave address in hardware. */
-	ret = mlxbf_slave_disable(priv);
-	if (ret < 0)
-		return ret;
-
-	priv->slave = NULL;
+	/*
+	 * Unregister slave by:
+	 * 1) Disabling the slave address in hardware
+	 * 2) Freeing priv->slave at the corresponding index
+	 */
+	ret = mlxbf_i2c_slave_disable(priv, slave->addr);
+	if (ret)
+		dev_err(dev, "Unable to find slave 0x%x\n", slave->addr);
 
-	return 0;
+	return ret;
 }
 
 static u32 mlxbf_i2c_functionality(struct i2c_adapter *adap)
@@ -2166,14 +2211,27 @@ static struct mlxbf_i2c_chip_info mlxbf_i2c_chip[] = {
 			[1] = &mlxbf_i2c_corepll_res[MLXBF_I2C_CHIP_TYPE_1],
 			[2] = &mlxbf_i2c_gpio_res[MLXBF_I2C_CHIP_TYPE_1]
 		},
-		.calculate_freq = mlxbf_i2c_calculate_freq_from_tyu
+		.calculate_freq = mlxbf_i2c_calculate_freq_from_tyu,
+		.smbus_master_rs_bytes_off = MLXBF_I2C_YU_SMBUS_RS_BYTES,
+		.smbus_master_fsm_off = MLXBF_I2C_YU_SMBUS_MASTER_FSM
 	},
 	[MLXBF_I2C_CHIP_TYPE_2] = {
 		.type = MLXBF_I2C_CHIP_TYPE_2,
 		.shared_res = {
 			[0] = &mlxbf_i2c_corepll_res[MLXBF_I2C_CHIP_TYPE_2]
 		},
-		.calculate_freq = mlxbf_i2c_calculate_freq_from_yu
+		.calculate_freq = mlxbf_i2c_calculate_freq_from_yu,
+		.smbus_master_rs_bytes_off = MLXBF_I2C_YU_SMBUS_RS_BYTES,
+		.smbus_master_fsm_off = MLXBF_I2C_YU_SMBUS_MASTER_FSM
+	},
+	[MLXBF_I2C_CHIP_TYPE_3] = {
+		.type = MLXBF_I2C_CHIP_TYPE_3,
+		.shared_res = {
+			[0] = &mlxbf_i2c_corepll_res[MLXBF_I2C_CHIP_TYPE_3]
+		},
+		.calculate_freq = mlxbf_i2c_calculate_freq_from_yu,
+		.smbus_master_rs_bytes_off = MLXBF_I2C_RSH_YU_SMBUS_RS_BYTES,
+		.smbus_master_fsm_off = MLXBF_I2C_RSH_YU_SMBUS_MASTER_FSM
 	}
 };
 
@@ -2189,24 +2247,11 @@ static struct i2c_adapter_quirks mlxbf_i2c_quirks = {
 	.max_write_len = MLXBF_I2C_MASTER_DATA_W_LENGTH,
 };
 
-static const struct of_device_id mlxbf_i2c_dt_ids[] = {
-	{
-		.compatible = "mellanox,i2c-mlxbf1",
-		.data = &mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_1]
-	},
-	{
-		.compatible = "mellanox,i2c-mlxbf2",
-		.data = &mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_2]
-	},
-	{},
-};
-
-MODULE_DEVICE_TABLE(of, mlxbf_i2c_dt_ids);
-
 #ifdef CONFIG_ACPI
 static const struct acpi_device_id mlxbf_i2c_acpi_ids[] = {
 	{ "MLNXBF03", (kernel_ulong_t)&mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_1] },
 	{ "MLNXBF23", (kernel_ulong_t)&mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_2] },
+	{ "MLNXBF31", (kernel_ulong_t)&mlxbf_i2c_chip[MLXBF_I2C_CHIP_TYPE_3] },
 	{},
 };
 
@@ -2215,35 +2260,27 @@ MODULE_DEVICE_TABLE(acpi, mlxbf_i2c_acpi_ids);
 static int mlxbf_i2c_acpi_probe(struct device *dev, struct mlxbf_i2c_priv *priv)
 {
 	const struct acpi_device_id *aid;
-	struct acpi_device *adev;
-	unsigned long bus_id = 0;
-	const char *uid;
+	u64 bus_id;
 	int ret;
 
 	if (acpi_disabled)
 		return -ENOENT;
 
-	adev = ACPI_COMPANION(dev);
-	if (!adev)
-		return -ENXIO;
-
 	aid = acpi_match_device(mlxbf_i2c_acpi_ids, dev);
 	if (!aid)
 		return -ENODEV;
 
 	priv->chip = (struct mlxbf_i2c_chip_info *)aid->driver_data;
 
-	uid = acpi_device_uid(adev);
-	if (!uid || !(*uid)) {
+	ret = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &bus_id);
+	if (ret) {
 		dev_err(dev, "Cannot retrieve UID\n");
-		return -ENODEV;
+		return ret;
 	}
 
-	ret = kstrtoul(uid, 0, &bus_id);
-	if (!ret)
-		priv->bus = bus_id;
+	priv->bus = bus_id;
 
-	return ret;
+	return 0;
 }
 #else
 static int mlxbf_i2c_acpi_probe(struct device *dev, struct mlxbf_i2c_priv *priv)
@@ -2252,36 +2289,12 @@ static int mlxbf_i2c_acpi_probe(struct device *dev, struct mlxbf_i2c_priv *priv)
 }
 #endif /* CONFIG_ACPI */
 
-static int mlxbf_i2c_of_probe(struct device *dev, struct mlxbf_i2c_priv *priv)
-{
-	const struct of_device_id *oid;
-	int bus_id = -1;
-
-	if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
-		oid = of_match_node(mlxbf_i2c_dt_ids, dev->of_node);
-		if (!oid)
-			return -ENODEV;
-
-		priv->chip = oid->data;
-
-		bus_id = of_alias_get_id(dev->of_node, "i2c");
-		if (bus_id >= 0)
-			priv->bus = bus_id;
-	}
-
-	if (bus_id < 0) {
-		dev_err(dev, "Cannot get bus id");
-		return bus_id;
-	}
-
-	return 0;
-}
-
 static int mlxbf_i2c_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct mlxbf_i2c_priv *priv;
 	struct i2c_adapter *adap;
+	u32 resource_version;
 	int irq, ret;
 
 	priv = devm_kzalloc(dev, sizeof(struct mlxbf_i2c_priv), GFP_KERNEL);
@@ -2289,17 +2302,63 @@ static int mlxbf_i2c_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	ret = mlxbf_i2c_acpi_probe(dev, priv);
-	if (ret < 0 && ret != -ENOENT && ret != -ENXIO)
-		ret = mlxbf_i2c_of_probe(dev, priv);
-
 	if (ret < 0)
 		return ret;
 
-	ret = mlxbf_i2c_init_resource(pdev, &priv->smbus,
-				      MLXBF_I2C_SMBUS_RES);
-	if (ret < 0) {
-		dev_err(dev, "Cannot fetch smbus resource info");
-		return ret;
+	/* This property allows the driver to stay backward compatible with older
+	 * ACPI tables.
+	 * Starting BlueField-3 SoC, the "smbus" resource was broken down into 3
+	 * separate resources "timer", "master" and "slave".
+	 */
+	if (device_property_read_u32(dev, "resource_version", &resource_version))
+		resource_version = 0;
+
+	priv->resource_version = resource_version;
+
+	if (priv->chip->type < MLXBF_I2C_CHIP_TYPE_3 && resource_version == 0) {
+		priv->timer = devm_kzalloc(dev, sizeof(struct mlxbf_i2c_resource), GFP_KERNEL);
+		if (!priv->timer)
+			return -ENOMEM;
+
+		priv->mst = devm_kzalloc(dev, sizeof(struct mlxbf_i2c_resource), GFP_KERNEL);
+		if (!priv->mst)
+			return -ENOMEM;
+
+		priv->slv = devm_kzalloc(dev, sizeof(struct mlxbf_i2c_resource), GFP_KERNEL);
+		if (!priv->slv)
+			return -ENOMEM;
+
+		ret = mlxbf_i2c_init_resource(pdev, &priv->smbus,
+					      MLXBF_I2C_SMBUS_RES);
+		if (ret < 0) {
+			dev_err(dev, "Cannot fetch smbus resource info");
+			return ret;
+		}
+
+		priv->timer->io = priv->smbus->io;
+		priv->mst->io = priv->smbus->io + MLXBF_I2C_MST_ADDR_OFFSET;
+		priv->slv->io = priv->smbus->io + MLXBF_I2C_SLV_ADDR_OFFSET;
+	} else {
+		ret = mlxbf_i2c_init_resource(pdev, &priv->timer,
+					      MLXBF_I2C_SMBUS_TIMER_RES);
+		if (ret < 0) {
+			dev_err(dev, "Cannot fetch timer resource info");
+			return ret;
+		}
+
+		ret = mlxbf_i2c_init_resource(pdev, &priv->mst,
+					      MLXBF_I2C_SMBUS_MST_RES);
+		if (ret < 0) {
+			dev_err(dev, "Cannot fetch master resource info");
+			return ret;
+		}
+
+		ret = mlxbf_i2c_init_resource(pdev, &priv->slv,
+					      MLXBF_I2C_SMBUS_SLV_RES);
+		if (ret < 0) {
+			dev_err(dev, "Cannot fetch slave resource info");
+			return ret;
+		}
 	}
 
 	ret = mlxbf_i2c_init_resource(pdev, &priv->mst_cause,
@@ -2358,8 +2417,8 @@ static int mlxbf_i2c_probe(struct platform_device *pdev)
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
 		return irq;
-	ret = devm_request_irq(dev, irq, mlxbf_smbus_irq,
-			       IRQF_ONESHOT | IRQF_SHARED | IRQF_PROBE_SHARED,
+	ret = devm_request_irq(dev, irq, mlxbf_i2c_irq,
+			       IRQF_SHARED | IRQF_PROBE_SHARED,
 			       dev_name(dev), priv);
 	if (ret < 0) {
 		dev_err(dev, "Cannot get irq %d\n", irq);
@@ -2387,8 +2446,19 @@ static int mlxbf_i2c_remove(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct resource *params;
 
-	params = priv->smbus->params;
-	devm_release_mem_region(dev, params->start, resource_size(params));
+	if (priv->chip->type < MLXBF_I2C_CHIP_TYPE_3 && priv->resource_version == 0) {
+		params = priv->smbus->params;
+		devm_release_mem_region(dev, params->start, resource_size(params));
+	} else {
+		params = priv->timer->params;
+		devm_release_mem_region(dev, params->start, resource_size(params));
+
+		params = priv->mst->params;
+		devm_release_mem_region(dev, params->start, resource_size(params));
+
+		params = priv->slv->params;
+		devm_release_mem_region(dev, params->start, resource_size(params));
+	}
 
 	params = priv->mst_cause->params;
 	devm_release_mem_region(dev, params->start, resource_size(params));
@@ -2420,7 +2490,6 @@ static struct platform_driver mlxbf_i2c_driver = {
 	.remove = mlxbf_i2c_remove,
 	.driver = {
 		.name = "i2c-mlxbf",
-		.of_match_table = mlxbf_i2c_dt_ids,
 #ifdef CONFIG_ACPI
 		.acpi_match_table = ACPI_PTR(mlxbf_i2c_acpi_ids),
 #endif /* CONFIG_ACPI  */
@@ -2453,4 +2522,5 @@ module_exit(mlxbf_i2c_exit);
 
 MODULE_DESCRIPTION("Mellanox BlueField I2C bus driver");
 MODULE_AUTHOR("Khalil Blaiech <kblaiech@nvidia.com>");
+MODULE_AUTHOR("Asmaa Mnebhi <asmaa@nvidia.com>");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/i2c/busses/i2c-riic.c b/drivers/i2c/busses/i2c-riic.c
index ecba1dfc1278..849848ccb080 100644
--- a/drivers/i2c/busses/i2c-riic.c
+++ b/drivers/i2c/busses/i2c-riic.c
@@ -400,7 +400,6 @@ static int riic_i2c_probe(struct platform_device *pdev)
 {
 	struct riic_dev *riic;
 	struct i2c_adapter *adap;
-	struct resource *res;
 	struct i2c_timings i2c_t;
 	struct reset_control *rstc;
 	int i, ret;
@@ -409,8 +408,7 @@ static int riic_i2c_probe(struct platform_device *pdev)
 	if (!riic)
 		return -ENOMEM;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	riic->base = devm_ioremap_resource(&pdev->dev, res);
+	riic->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(riic->base))
 		return PTR_ERR(riic->base);
 
diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c
index 2e98e7793bba..d1658ed76562 100644
--- a/drivers/i2c/busses/i2c-rk3x.c
+++ b/drivers/i2c/busses/i2c-rk3x.c
@@ -1165,6 +1165,11 @@ static const struct rk3x_i2c_soc_data rv1108_soc_data = {
 	.calc_timings = rk3x_i2c_v1_calc_timings,
 };
 
+static const struct rk3x_i2c_soc_data rv1126_soc_data = {
+	.grf_offset = 0x118,
+	.calc_timings = rk3x_i2c_v1_calc_timings,
+};
+
 static const struct rk3x_i2c_soc_data rk3066_soc_data = {
 	.grf_offset = 0x154,
 	.calc_timings = rk3x_i2c_v0_calc_timings,
@@ -1196,6 +1201,10 @@ static const struct of_device_id rk3x_i2c_match[] = {
 		.data = &rv1108_soc_data
 	},
 	{
+		.compatible = "rockchip,rv1126-i2c",
+		.data = &rv1126_soc_data
+	},
+	{
 		.compatible = "rockchip,rk3066-i2c",
 		.data = &rk3066_soc_data
 	},
diff --git a/drivers/i2c/busses/i2c-scmi.c b/drivers/i2c/busses/i2c-scmi.c
index 6746aa46d96c..0239e134b90f 100644
--- a/drivers/i2c/busses/i2c-scmi.c
+++ b/drivers/i2c/busses/i2c-scmi.c
@@ -6,15 +6,13 @@
  */
 
 #include <linux/module.h>
+#include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/kernel.h>
 #include <linux/stddef.h>
 #include <linux/i2c.h>
 #include <linux/acpi.h>
 
-#define ACPI_SMBUS_HC_CLASS		"smbus"
-#define ACPI_SMBUS_HC_DEVICE_NAME	"cmi"
-
 /* SMBUS HID definition as supported by Microsoft Windows */
 #define ACPI_SMBUS_MS_HID		"SMB0001"
 
@@ -30,7 +28,7 @@ struct acpi_smbus_cmi {
 	u8 cap_info:1;
 	u8 cap_read:1;
 	u8 cap_write:1;
-	struct smbus_methods_t *methods;
+	const struct smbus_methods_t *methods;
 };
 
 static const struct smbus_methods_t smbus_methods = {
@@ -358,29 +356,25 @@ static acpi_status acpi_smbus_cmi_query_methods(acpi_handle handle, u32 level,
 	return AE_OK;
 }
 
-static int acpi_smbus_cmi_add(struct acpi_device *device)
+static int smbus_cmi_probe(struct platform_device *device)
 {
+	struct device *dev = &device->dev;
 	struct acpi_smbus_cmi *smbus_cmi;
-	const struct acpi_device_id *id;
 	int ret;
 
 	smbus_cmi = kzalloc(sizeof(struct acpi_smbus_cmi), GFP_KERNEL);
 	if (!smbus_cmi)
 		return -ENOMEM;
 
-	smbus_cmi->handle = device->handle;
-	strcpy(acpi_device_name(device), ACPI_SMBUS_HC_DEVICE_NAME);
-	strcpy(acpi_device_class(device), ACPI_SMBUS_HC_CLASS);
-	device->driver_data = smbus_cmi;
+	smbus_cmi->handle = ACPI_HANDLE(dev);
+	smbus_cmi->methods = device_get_match_data(dev);
+
+	platform_set_drvdata(device, smbus_cmi);
+
 	smbus_cmi->cap_info = 0;
 	smbus_cmi->cap_read = 0;
 	smbus_cmi->cap_write = 0;
 
-	for (id = acpi_smbus_cmi_ids; id->id[0]; id++)
-		if (!strcmp(id->id, acpi_device_hid(device)))
-			smbus_cmi->methods =
-				(struct smbus_methods_t *) id->driver_data;
-
 	acpi_walk_namespace(ACPI_TYPE_METHOD, smbus_cmi->handle, 1,
 			    acpi_smbus_cmi_query_methods, NULL, smbus_cmi, NULL);
 
@@ -390,8 +384,7 @@ static int acpi_smbus_cmi_add(struct acpi_device *device)
 	}
 
 	snprintf(smbus_cmi->adapter.name, sizeof(smbus_cmi->adapter.name),
-		"SMBus CMI adapter %s",
-		acpi_device_name(device));
+		 "SMBus CMI adapter %s", dev_name(dev));
 	smbus_cmi->adapter.owner = THIS_MODULE;
 	smbus_cmi->adapter.algo = &acpi_smbus_cmi_algorithm;
 	smbus_cmi->adapter.algo_data = smbus_cmi;
@@ -408,31 +401,28 @@ static int acpi_smbus_cmi_add(struct acpi_device *device)
 
 err:
 	kfree(smbus_cmi);
-	device->driver_data = NULL;
 	return ret;
 }
 
-static int acpi_smbus_cmi_remove(struct acpi_device *device)
+static int smbus_cmi_remove(struct platform_device *device)
 {
-	struct acpi_smbus_cmi *smbus_cmi = acpi_driver_data(device);
+	struct acpi_smbus_cmi *smbus_cmi = platform_get_drvdata(device);
 
 	i2c_del_adapter(&smbus_cmi->adapter);
 	kfree(smbus_cmi);
-	device->driver_data = NULL;
 
 	return 0;
 }
 
-static struct acpi_driver acpi_smbus_cmi_driver = {
-	.name = ACPI_SMBUS_HC_DEVICE_NAME,
-	.class = ACPI_SMBUS_HC_CLASS,
-	.ids = acpi_smbus_cmi_ids,
-	.ops = {
-		.add = acpi_smbus_cmi_add,
-		.remove = acpi_smbus_cmi_remove,
+static struct platform_driver smbus_cmi_driver = {
+	.probe = smbus_cmi_probe,
+	.remove = smbus_cmi_remove,
+	.driver = {
+		.name   = "smbus_cmi",
+		.acpi_match_table = acpi_smbus_cmi_ids,
 	},
 };
-module_acpi_driver(acpi_smbus_cmi_driver);
+module_platform_driver(smbus_cmi_driver);
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Crane Cai <crane.cai@amd.com>");
diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c
index 031c78ac42e6..954022c04cc4 100644
--- a/drivers/i2c/busses/i2c-tegra.c
+++ b/drivers/i2c/busses/i2c-tegra.c
@@ -443,11 +443,16 @@ static int tegra_i2c_init_dma(struct tegra_i2c_dev *i2c_dev)
 	u32 *dma_buf;
 	int err;
 
-	if (!i2c_dev->hw->has_apb_dma || i2c_dev->is_vi)
+	if (i2c_dev->is_vi)
 		return 0;
 
-	if (!IS_ENABLED(CONFIG_TEGRA20_APB_DMA)) {
-		dev_dbg(i2c_dev->dev, "DMA support not enabled\n");
+	if (!i2c_dev->hw->has_apb_dma) {
+		if (!IS_ENABLED(CONFIG_TEGRA20_APB_DMA)) {
+			dev_dbg(i2c_dev->dev, "APB DMA support not enabled\n");
+			return 0;
+		}
+	} else if (!IS_ENABLED(CONFIG_TEGRA186_GPC_DMA)) {
+		dev_dbg(i2c_dev->dev, "GPC DMA support not enabled\n");
 		return 0;
 	}
 
diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c
index 08b561f0709d..da6568a20177 100644
--- a/drivers/i2c/i2c-core-acpi.c
+++ b/drivers/i2c/i2c-core-acpi.c
@@ -26,7 +26,7 @@ struct gsb_buffer {
 	union {
 		u16	wdata;
 		u8	bdata;
-		u8	data[0];
+		DECLARE_FLEX_ARRAY(u8, data);
 	};
 } __packed;
 
diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
index 91007558bcb2..8c7e3494ca5f 100644
--- a/drivers/i2c/i2c-core-base.c
+++ b/drivers/i2c/i2c-core-base.c
@@ -599,13 +599,9 @@ static void i2c_device_remove(struct device *dev)
 
 	driver = to_i2c_driver(dev->driver);
 	if (driver->remove) {
-		int status;
-
 		dev_dbg(dev, "remove\n");
 
-		status = driver->remove(client);
-		if (status)
-			dev_warn(dev, "remove failed (%pe), will be ignored\n", ERR_PTR(status));
+		driver->remove(client);
 	}
 
 	devres_release_group(&client->dev, client->devres_group_id);
diff --git a/drivers/i2c/i2c-slave-eeprom.c b/drivers/i2c/i2c-slave-eeprom.c
index 5c7ae421cacf..4abc2d919881 100644
--- a/drivers/i2c/i2c-slave-eeprom.c
+++ b/drivers/i2c/i2c-slave-eeprom.c
@@ -181,14 +181,12 @@ static int i2c_slave_eeprom_probe(struct i2c_client *client, const struct i2c_de
 	return 0;
 };
 
-static int i2c_slave_eeprom_remove(struct i2c_client *client)
+static void i2c_slave_eeprom_remove(struct i2c_client *client)
 {
 	struct eeprom_data *eeprom = i2c_get_clientdata(client);
 
 	i2c_slave_unregister(client);
 	sysfs_remove_bin_file(&client->dev.kobj, &eeprom->bin);
-
-	return 0;
 }
 
 static const struct i2c_device_id i2c_slave_eeprom_id[] = {
diff --git a/drivers/i2c/i2c-slave-testunit.c b/drivers/i2c/i2c-slave-testunit.c
index 56dae08dfd48..75ee7ebdb614 100644
--- a/drivers/i2c/i2c-slave-testunit.c
+++ b/drivers/i2c/i2c-slave-testunit.c
@@ -153,13 +153,12 @@ static int i2c_slave_testunit_probe(struct i2c_client *client)
 	return i2c_slave_register(client, i2c_slave_testunit_slave_cb);
 };
 
-static int i2c_slave_testunit_remove(struct i2c_client *client)
+static void i2c_slave_testunit_remove(struct i2c_client *client)
 {
 	struct testunit_data *tu = i2c_get_clientdata(client);
 
 	cancel_delayed_work_sync(&tu->worker);
 	i2c_slave_unregister(client);
-	return 0;
 }
 
 static const struct i2c_device_id i2c_slave_testunit_id[] = {
diff --git a/drivers/i2c/i2c-smbus.c b/drivers/i2c/i2c-smbus.c
index 8ba9b59a3c40..07c92c8495a3 100644
--- a/drivers/i2c/i2c-smbus.c
+++ b/drivers/i2c/i2c-smbus.c
@@ -153,12 +153,11 @@ static int smbalert_probe(struct i2c_client *ara,
 }
 
 /* IRQ and memory resources are managed so they are freed automatically */
-static int smbalert_remove(struct i2c_client *ara)
+static void smbalert_remove(struct i2c_client *ara)
 {
 	struct i2c_smbus_alert *alert = i2c_get_clientdata(ara);
 
 	cancel_work_sync(&alert->alert);
-	return 0;
 }
 
 static const struct i2c_device_id smbalert_ids[] = {
diff --git a/drivers/i2c/muxes/Kconfig b/drivers/i2c/muxes/Kconfig
index 1708b1a82da2..ea838dbae32e 100644
--- a/drivers/i2c/muxes/Kconfig
+++ b/drivers/i2c/muxes/Kconfig
@@ -9,7 +9,7 @@ menu "Multiplexer I2C Chip support"
 config I2C_ARB_GPIO_CHALLENGE
 	tristate "GPIO-based I2C arbitration"
 	depends on GPIOLIB || COMPILE_TEST
-	depends on OF || COMPILE_TEST
+	depends on OF
 	help
 	  If you say yes to this option, support will be included for an
 	  I2C multimaster arbitration scheme using GPIOs and a challenge &
@@ -34,7 +34,7 @@ config I2C_MUX_GPIO
 config I2C_MUX_GPMUX
 	tristate "General Purpose I2C multiplexer"
 	select MULTIPLEXER
-	depends on OF || COMPILE_TEST
+	depends on OF
 	help
 	  If you say yes to this option, support will be included for a
 	  general purpose I2C multiplexer. This driver provides access to
@@ -77,7 +77,7 @@ config I2C_MUX_PCA954x
 config I2C_MUX_PINCTRL
 	tristate "pinctrl-based I2C multiplexer"
 	depends on PINCTRL
-	depends on OF || COMPILE_TEST
+	depends on OF
 	help
 	  If you say yes to this option, support will be included for an I2C
 	  multiplexer that uses the pinctrl subsystem, i.e. pin multiplexing.
diff --git a/drivers/i2c/muxes/i2c-mux-ltc4306.c b/drivers/i2c/muxes/i2c-mux-ltc4306.c
index 704f1e50f6f4..70835825083f 100644
--- a/drivers/i2c/muxes/i2c-mux-ltc4306.c
+++ b/drivers/i2c/muxes/i2c-mux-ltc4306.c
@@ -294,13 +294,11 @@ static int ltc4306_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int ltc4306_remove(struct i2c_client *client)
+static void ltc4306_remove(struct i2c_client *client)
 {
 	struct i2c_mux_core *muxc = i2c_get_clientdata(client);
 
 	i2c_mux_del_adapters(muxc);
-
-	return 0;
 }
 
 static struct i2c_driver ltc4306_driver = {
diff --git a/drivers/i2c/muxes/i2c-mux-pca9541.c b/drivers/i2c/muxes/i2c-mux-pca9541.c
index 6daec8d3d331..ea83de78f52d 100644
--- a/drivers/i2c/muxes/i2c-mux-pca9541.c
+++ b/drivers/i2c/muxes/i2c-mux-pca9541.c
@@ -325,12 +325,11 @@ static int pca9541_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int pca9541_remove(struct i2c_client *client)
+static void pca9541_remove(struct i2c_client *client)
 {
 	struct i2c_mux_core *muxc = i2c_get_clientdata(client);
 
 	i2c_mux_del_adapters(muxc);
-	return 0;
 }
 
 static struct i2c_driver pca9541_driver = {
diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c
index 4ad665757dd8..a5f458b635df 100644
--- a/drivers/i2c/muxes/i2c-mux-pca954x.c
+++ b/drivers/i2c/muxes/i2c-mux-pca954x.c
@@ -521,14 +521,13 @@ fail_cleanup:
 	return ret;
 }
 
-static int pca954x_remove(struct i2c_client *client)
+static void pca954x_remove(struct i2c_client *client)
 {
 	struct i2c_mux_core *muxc = i2c_get_clientdata(client);
 
 	device_remove_file(&client->dev, &dev_attr_idle_state);
 
 	pca954x_cleanup(muxc);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/i2c/muxes/i2c-mux-pinctrl.c b/drivers/i2c/muxes/i2c-mux-pinctrl.c
index f1bb00a11ad6..d5ad904756fd 100644
--- a/drivers/i2c/muxes/i2c-mux-pinctrl.c
+++ b/drivers/i2c/muxes/i2c-mux-pinctrl.c
@@ -185,7 +185,7 @@ MODULE_DEVICE_TABLE(of, i2c_mux_pinctrl_of_match);
 static struct platform_driver i2c_mux_pinctrl_driver = {
 	.driver	= {
 		.name	= "i2c-mux-pinctrl",
-		.of_match_table = of_match_ptr(i2c_mux_pinctrl_of_match),
+		.of_match_table = i2c_mux_pinctrl_of_match,
 	},
 	.probe	= i2c_mux_pinctrl_probe,
 	.remove	= i2c_mux_pinctrl_remove,
diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
index 3e101719689a..cfeb24d40d37 100644
--- a/drivers/idle/intel_idle.c
+++ b/drivers/idle/intel_idle.c
@@ -928,6 +928,51 @@ static struct cpuidle_state adl_l_cstates[] __initdata = {
 		.enter = NULL }
 };
 
+static struct cpuidle_state adl_n_cstates[] __initdata = {
+	{
+		.name = "C1",
+		.desc = "MWAIT 0x00",
+		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
+		.exit_latency = 1,
+		.target_residency = 1,
+		.enter = &intel_idle,
+		.enter_s2idle = intel_idle_s2idle, },
+	{
+		.name = "C1E",
+		.desc = "MWAIT 0x01",
+		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
+		.exit_latency = 2,
+		.target_residency = 4,
+		.enter = &intel_idle,
+		.enter_s2idle = intel_idle_s2idle, },
+	{
+		.name = "C6",
+		.desc = "MWAIT 0x20",
+		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
+		.exit_latency = 195,
+		.target_residency = 585,
+		.enter = &intel_idle,
+		.enter_s2idle = intel_idle_s2idle, },
+	{
+		.name = "C8",
+		.desc = "MWAIT 0x40",
+		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
+		.exit_latency = 260,
+		.target_residency = 1040,
+		.enter = &intel_idle,
+		.enter_s2idle = intel_idle_s2idle, },
+	{
+		.name = "C10",
+		.desc = "MWAIT 0x60",
+		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
+		.exit_latency = 660,
+		.target_residency = 1980,
+		.enter = &intel_idle,
+		.enter_s2idle = intel_idle_s2idle, },
+	{
+		.enter = NULL }
+};
+
 static struct cpuidle_state spr_cstates[] __initdata = {
 	{
 		.name = "C1",
@@ -1309,6 +1354,10 @@ static const struct idle_cpu idle_cpu_adl_l __initconst = {
 	.state_table = adl_l_cstates,
 };
 
+static const struct idle_cpu idle_cpu_adl_n __initconst = {
+	.state_table = adl_n_cstates,
+};
+
 static const struct idle_cpu idle_cpu_spr __initconst = {
 	.state_table = spr_cstates,
 	.disable_promotion_to_c1e = true,
@@ -1379,6 +1428,7 @@ static const struct x86_cpu_id intel_idle_ids[] __initconst = {
 	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D,		&idle_cpu_icx),
 	X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE,		&idle_cpu_adl),
 	X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L,		&idle_cpu_adl_l),
+	X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N,		&idle_cpu_adl_n),
 	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X,	&idle_cpu_spr),
 	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL,	&idle_cpu_knl),
 	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM,	&idle_cpu_knl),
@@ -1507,7 +1557,7 @@ static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
 		state = &drv->states[drv->state_count++];
 
 		snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
-		strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
+		strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
 		state->exit_latency = cx->latency;
 		/*
 		 * For C1-type C-states use the same number for both the exit
@@ -1816,6 +1866,7 @@ static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
 		break;
 	case INTEL_FAM6_ALDERLAKE:
 	case INTEL_FAM6_ALDERLAKE_L:
+	case INTEL_FAM6_ALDERLAKE_N:
 		adl_idle_state_table_update();
 		break;
 	}
diff --git a/drivers/iio/accel/bma180.c b/drivers/iio/accel/bma180.c
index 9c9e98578667..d03fc3400f94 100644
--- a/drivers/iio/accel/bma180.c
+++ b/drivers/iio/accel/bma180.c
@@ -1045,7 +1045,7 @@ err_disable_vdd:
 	return ret;
 }
 
-static int bma180_remove(struct i2c_client *client)
+static void bma180_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct bma180_data *data = iio_priv(indio_dev);
@@ -1062,8 +1062,6 @@ static int bma180_remove(struct i2c_client *client)
 	mutex_unlock(&data->mutex);
 	regulator_disable(data->vddio_supply);
 	regulator_disable(data->vdd_supply);
-
-	return 0;
 }
 
 static int bma180_suspend(struct device *dev)
diff --git a/drivers/iio/accel/bmc150-accel-i2c.c b/drivers/iio/accel/bmc150-accel-i2c.c
index dff4d7dd101c..be8cc598b88e 100644
--- a/drivers/iio/accel/bmc150-accel-i2c.c
+++ b/drivers/iio/accel/bmc150-accel-i2c.c
@@ -209,13 +209,11 @@ static int bmc150_accel_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int bmc150_accel_remove(struct i2c_client *client)
+static void bmc150_accel_remove(struct i2c_client *client)
 {
 	bmc150_acpi_dual_accel_remove(client);
 
 	bmc150_accel_core_remove(&client->dev);
-
-	return 0;
 }
 
 static const struct acpi_device_id bmc150_accel_acpi_match[] = {
diff --git a/drivers/iio/accel/kxcjk-1013.c b/drivers/iio/accel/kxcjk-1013.c
index 748b35c2f0c3..94f7b6ac5c87 100644
--- a/drivers/iio/accel/kxcjk-1013.c
+++ b/drivers/iio/accel/kxcjk-1013.c
@@ -1611,7 +1611,7 @@ err_poweroff:
 	return ret;
 }
 
-static int kxcjk1013_remove(struct i2c_client *client)
+static void kxcjk1013_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct kxcjk1013_data *data = iio_priv(indio_dev);
@@ -1630,8 +1630,6 @@ static int kxcjk1013_remove(struct i2c_client *client)
 	mutex_lock(&data->mutex);
 	kxcjk1013_set_mode(data, STANDBY);
 	mutex_unlock(&data->mutex);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/iio/accel/kxsd9-i2c.c b/drivers/iio/accel/kxsd9-i2c.c
index d57f264bd6c8..61346ea8ef19 100644
--- a/drivers/iio/accel/kxsd9-i2c.c
+++ b/drivers/iio/accel/kxsd9-i2c.c
@@ -32,11 +32,9 @@ static int kxsd9_i2c_probe(struct i2c_client *i2c,
 				  i2c->name);
 }
 
-static int kxsd9_i2c_remove(struct i2c_client *client)
+static void kxsd9_i2c_remove(struct i2c_client *client)
 {
 	kxsd9_common_remove(&client->dev);
-
-	return 0;
 }
 
 static const struct of_device_id kxsd9_of_match[] = {
diff --git a/drivers/iio/accel/mc3230.c b/drivers/iio/accel/mc3230.c
index c15d16e7f1da..2462000e0519 100644
--- a/drivers/iio/accel/mc3230.c
+++ b/drivers/iio/accel/mc3230.c
@@ -151,15 +151,13 @@ static int mc3230_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int mc3230_remove(struct i2c_client *client)
+static void mc3230_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
 	iio_device_unregister(indio_dev);
 
 	mc3230_set_opcon(iio_priv(indio_dev), MC3230_MODE_OPCON_STANDBY);
-
-	return 0;
 }
 
 static int mc3230_suspend(struct device *dev)
diff --git a/drivers/iio/accel/mma7455_i2c.c b/drivers/iio/accel/mma7455_i2c.c
index a3b84e8a3ea8..c63b321b01cd 100644
--- a/drivers/iio/accel/mma7455_i2c.c
+++ b/drivers/iio/accel/mma7455_i2c.c
@@ -26,11 +26,9 @@ static int mma7455_i2c_probe(struct i2c_client *i2c,
 	return mma7455_core_probe(&i2c->dev, regmap, name);
 }
 
-static int mma7455_i2c_remove(struct i2c_client *i2c)
+static void mma7455_i2c_remove(struct i2c_client *i2c)
 {
 	mma7455_core_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id mma7455_i2c_ids[] = {
diff --git a/drivers/iio/accel/mma7660.c b/drivers/iio/accel/mma7660.c
index 794f2f383303..85829990bbad 100644
--- a/drivers/iio/accel/mma7660.c
+++ b/drivers/iio/accel/mma7660.c
@@ -207,7 +207,7 @@ static int mma7660_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int mma7660_remove(struct i2c_client *client)
+static void mma7660_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	int ret;
@@ -218,8 +218,6 @@ static int mma7660_remove(struct i2c_client *client)
 	if (ret)
 		dev_warn(&client->dev, "Failed to put device in stand-by mode (%pe), ignoring\n",
 			 ERR_PTR(ret));
-
-	return 0;
 }
 
 static int mma7660_suspend(struct device *dev)
diff --git a/drivers/iio/accel/mma8452.c b/drivers/iio/accel/mma8452.c
index c7d9ca96dbaa..3ba28c2ff68a 100644
--- a/drivers/iio/accel/mma8452.c
+++ b/drivers/iio/accel/mma8452.c
@@ -1735,7 +1735,7 @@ disable_regulator_vdd:
 	return ret;
 }
 
-static int mma8452_remove(struct i2c_client *client)
+static void mma8452_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct mma8452_data *data = iio_priv(indio_dev);
@@ -1751,8 +1751,6 @@ static int mma8452_remove(struct i2c_client *client)
 
 	regulator_disable(data->vddio_reg);
 	regulator_disable(data->vdd_reg);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/iio/accel/mma9551.c b/drivers/iio/accel/mma9551.c
index 123cdbbb265c..f7a793f4a8e3 100644
--- a/drivers/iio/accel/mma9551.c
+++ b/drivers/iio/accel/mma9551.c
@@ -509,7 +509,7 @@ out_poweroff:
 	return ret;
 }
 
-static int mma9551_remove(struct i2c_client *client)
+static void mma9551_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct mma9551_data *data = iio_priv(indio_dev);
@@ -522,8 +522,6 @@ static int mma9551_remove(struct i2c_client *client)
 	mutex_lock(&data->mutex);
 	mma9551_set_device_state(data->client, false);
 	mutex_unlock(&data->mutex);
-
-	return 0;
 }
 
 static int mma9551_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/accel/mma9553.c b/drivers/iio/accel/mma9553.c
index 09df58d4be33..2da0e005b13e 100644
--- a/drivers/iio/accel/mma9553.c
+++ b/drivers/iio/accel/mma9553.c
@@ -1148,7 +1148,7 @@ out_poweroff:
 	return ret;
 }
 
-static int mma9553_remove(struct i2c_client *client)
+static void mma9553_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct mma9553_data *data = iio_priv(indio_dev);
@@ -1161,8 +1161,6 @@ static int mma9553_remove(struct i2c_client *client)
 	mutex_lock(&data->mutex);
 	mma9551_set_device_state(data->client, false);
 	mutex_unlock(&data->mutex);
-
-	return 0;
 }
 
 static int mma9553_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/accel/stk8312.c b/drivers/iio/accel/stk8312.c
index ceca28913355..7b1d6fb692b3 100644
--- a/drivers/iio/accel/stk8312.c
+++ b/drivers/iio/accel/stk8312.c
@@ -597,7 +597,7 @@ err_power_off:
 	return ret;
 }
 
-static int stk8312_remove(struct i2c_client *client)
+static void stk8312_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct stk8312_data *data = iio_priv(indio_dev);
@@ -609,8 +609,6 @@ static int stk8312_remove(struct i2c_client *client)
 		iio_trigger_unregister(data->dready_trig);
 
 	stk8312_set_mode(data, STK8312_MODE_STANDBY);
-
-	return 0;
 }
 
 static int stk8312_suspend(struct device *dev)
diff --git a/drivers/iio/accel/stk8ba50.c b/drivers/iio/accel/stk8ba50.c
index 7d59efb41e22..2f5e4ab2a6e7 100644
--- a/drivers/iio/accel/stk8ba50.c
+++ b/drivers/iio/accel/stk8ba50.c
@@ -490,7 +490,7 @@ err_power_off:
 	return ret;
 }
 
-static int stk8ba50_remove(struct i2c_client *client)
+static void stk8ba50_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct stk8ba50_data *data = iio_priv(indio_dev);
@@ -502,8 +502,6 @@ static int stk8ba50_remove(struct i2c_client *client)
 		iio_trigger_unregister(data->dready_trig);
 
 	stk8ba50_set_power(data, STK8BA50_MODE_SUSPEND);
-
-	return 0;
 }
 
 static int stk8ba50_suspend(struct device *dev)
diff --git a/drivers/iio/adc/ad799x.c b/drivers/iio/adc/ad799x.c
index 262bd7665b33..6dbe9d5e08a2 100644
--- a/drivers/iio/adc/ad799x.c
+++ b/drivers/iio/adc/ad799x.c
@@ -880,7 +880,7 @@ error_disable_reg:
 	return ret;
 }
 
-static int ad799x_remove(struct i2c_client *client)
+static void ad799x_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct ad799x_state *st = iio_priv(indio_dev);
@@ -892,8 +892,6 @@ static int ad799x_remove(struct i2c_client *client)
 		regulator_disable(st->vref);
 	regulator_disable(st->reg);
 	kfree(st->rx_buf);
-
-	return 0;
 }
 
 static int ad799x_suspend(struct device *dev)
diff --git a/drivers/iio/adc/ina2xx-adc.c b/drivers/iio/adc/ina2xx-adc.c
index 240e6c420701..910e7e965fc4 100644
--- a/drivers/iio/adc/ina2xx-adc.c
+++ b/drivers/iio/adc/ina2xx-adc.c
@@ -1034,7 +1034,7 @@ static int ina2xx_probe(struct i2c_client *client,
 	return iio_device_register(indio_dev);
 }
 
-static int ina2xx_remove(struct i2c_client *client)
+static void ina2xx_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct ina2xx_chip_info *chip = iio_priv(indio_dev);
@@ -1048,8 +1048,6 @@ static int ina2xx_remove(struct i2c_client *client)
 	if (ret)
 		dev_warn(&client->dev, "Failed to power down device (%pe)\n",
 			 ERR_PTR(ret));
-
-	return 0;
 }
 
 static const struct i2c_device_id ina2xx_id[] = {
diff --git a/drivers/iio/adc/ltc2497.c b/drivers/iio/adc/ltc2497.c
index f7c786f37ceb..d58a432bafe1 100644
--- a/drivers/iio/adc/ltc2497.c
+++ b/drivers/iio/adc/ltc2497.c
@@ -74,13 +74,11 @@ static int ltc2497_probe(struct i2c_client *client,
 	return ltc2497core_probe(dev, indio_dev);
 }
 
-static int ltc2497_remove(struct i2c_client *client)
+static void ltc2497_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
 	ltc2497core_remove(indio_dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id ltc2497_id[] = {
diff --git a/drivers/iio/adc/sun4i-gpadc-iio.c b/drivers/iio/adc/sun4i-gpadc-iio.c
index 2d393a4dfff6..a6ade70dedf8 100644
--- a/drivers/iio/adc/sun4i-gpadc-iio.c
+++ b/drivers/iio/adc/sun4i-gpadc-iio.c
@@ -412,9 +412,9 @@ static int sun4i_gpadc_runtime_resume(struct device *dev)
 	return 0;
 }
 
-static int sun4i_gpadc_get_temp(void *data, int *temp)
+static int sun4i_gpadc_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct sun4i_gpadc_iio *info = data;
+	struct sun4i_gpadc_iio *info = tz->devdata;
 	int val, scale, offset;
 
 	if (sun4i_gpadc_temp_read(info->indio_dev, &val))
@@ -428,7 +428,7 @@ static int sun4i_gpadc_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops sun4i_ts_tz_ops = {
+static const struct thermal_zone_device_ops sun4i_ts_tz_ops = {
 	.get_temp = &sun4i_gpadc_get_temp,
 };
 
@@ -637,9 +637,9 @@ static int sun4i_gpadc_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 
 	if (IS_ENABLED(CONFIG_THERMAL_OF)) {
-		info->tzd = thermal_zone_of_sensor_register(info->sensor_device,
-							    0, info,
-							    &sun4i_ts_tz_ops);
+		info->tzd = devm_thermal_of_zone_register(info->sensor_device,
+							  0, info,
+							  &sun4i_ts_tz_ops);
 		/*
 		 * Do not fail driver probing when failing to register in
 		 * thermal because no thermal DT node is found.
@@ -681,8 +681,6 @@ static int sun4i_gpadc_remove(struct platform_device *pdev)
 	if (!IS_ENABLED(CONFIG_THERMAL_OF))
 		return 0;
 
-	thermal_zone_of_sensor_unregister(info->sensor_device, info->tzd);
-
 	if (!info->no_irq)
 		iio_map_array_unregister(indio_dev);
 
diff --git a/drivers/iio/adc/ti-ads1015.c b/drivers/iio/adc/ti-ads1015.c
index e3dfc155fbe2..8bceba694026 100644
--- a/drivers/iio/adc/ti-ads1015.c
+++ b/drivers/iio/adc/ti-ads1015.c
@@ -1094,7 +1094,7 @@ static int ads1015_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int ads1015_remove(struct i2c_client *client)
+static void ads1015_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct ads1015_data *data = iio_priv(indio_dev);
@@ -1110,8 +1110,6 @@ static int ads1015_remove(struct i2c_client *client)
 	if (ret)
 		dev_warn(&client->dev, "Failed to power down (%pe)\n",
 			 ERR_PTR(ret));
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/iio/chemical/atlas-sensor.c b/drivers/iio/chemical/atlas-sensor.c
index 8378c00fa2ff..7cac77a931c7 100644
--- a/drivers/iio/chemical/atlas-sensor.c
+++ b/drivers/iio/chemical/atlas-sensor.c
@@ -722,7 +722,7 @@ unregister_trigger:
 	return ret;
 }
 
-static int atlas_remove(struct i2c_client *client)
+static void atlas_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct atlas_data *data = iio_priv(indio_dev);
@@ -739,8 +739,6 @@ static int atlas_remove(struct i2c_client *client)
 	if (ret)
 		dev_err(&client->dev, "Failed to power down device (%pe)\n",
 			ERR_PTR(ret));
-
-	return 0;
 }
 
 static int atlas_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/chemical/ccs811.c b/drivers/iio/chemical/ccs811.c
index 560183efb36f..ba4045e20303 100644
--- a/drivers/iio/chemical/ccs811.c
+++ b/drivers/iio/chemical/ccs811.c
@@ -532,7 +532,7 @@ err_poweroff:
 	return ret;
 }
 
-static int ccs811_remove(struct i2c_client *client)
+static void ccs811_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct ccs811_data *data = iio_priv(indio_dev);
@@ -548,8 +548,6 @@ static int ccs811_remove(struct i2c_client *client)
 	if (ret)
 		dev_warn(&client->dev, "Failed to power down device (%pe)\n",
 			 ERR_PTR(ret));
-
-	return 0;
 }
 
 static const struct i2c_device_id ccs811_id[] = {
diff --git a/drivers/iio/chemical/sgp30.c b/drivers/iio/chemical/sgp30.c
index 2343d444604d..e2c13c78c7e0 100644
--- a/drivers/iio/chemical/sgp30.c
+++ b/drivers/iio/chemical/sgp30.c
@@ -552,15 +552,13 @@ static int sgp_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int sgp_remove(struct i2c_client *client)
+static void sgp_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct sgp_data *data = iio_priv(indio_dev);
 
 	if (data->iaq_thread)
 		kthread_stop(data->iaq_thread);
-
-	return 0;
 }
 
 static const struct i2c_device_id sgp_id[] = {
diff --git a/drivers/iio/dac/ad5380.c b/drivers/iio/dac/ad5380.c
index 81775152aac6..a81bfa47a221 100644
--- a/drivers/iio/dac/ad5380.c
+++ b/drivers/iio/dac/ad5380.c
@@ -559,11 +559,9 @@ static int ad5380_i2c_probe(struct i2c_client *i2c,
 	return ad5380_probe(&i2c->dev, regmap, id->driver_data, id->name);
 }
 
-static int ad5380_i2c_remove(struct i2c_client *i2c)
+static void ad5380_i2c_remove(struct i2c_client *i2c)
 {
 	ad5380_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id ad5380_i2c_ids[] = {
diff --git a/drivers/iio/dac/ad5446.c b/drivers/iio/dac/ad5446.c
index 09e242949cd0..7324065d3782 100644
--- a/drivers/iio/dac/ad5446.c
+++ b/drivers/iio/dac/ad5446.c
@@ -575,11 +575,9 @@ static int ad5446_i2c_probe(struct i2c_client *i2c,
 		&ad5446_i2c_chip_info[id->driver_data]);
 }
 
-static int ad5446_i2c_remove(struct i2c_client *i2c)
+static void ad5446_i2c_remove(struct i2c_client *i2c)
 {
 	ad5446_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id ad5446_i2c_ids[] = {
diff --git a/drivers/iio/dac/ad5593r.c b/drivers/iio/dac/ad5593r.c
index 34e1319a9712..92be661034a6 100644
--- a/drivers/iio/dac/ad5593r.c
+++ b/drivers/iio/dac/ad5593r.c
@@ -97,11 +97,9 @@ static int ad5593r_i2c_probe(struct i2c_client *i2c,
 	return ad5592r_probe(&i2c->dev, id->name, &ad5593r_rw_ops);
 }
 
-static int ad5593r_i2c_remove(struct i2c_client *i2c)
+static void ad5593r_i2c_remove(struct i2c_client *i2c)
 {
 	ad5592r_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id ad5593r_i2c_ids[] = {
diff --git a/drivers/iio/dac/ad5696-i2c.c b/drivers/iio/dac/ad5696-i2c.c
index 762503c1901b..aa36cbf0137c 100644
--- a/drivers/iio/dac/ad5696-i2c.c
+++ b/drivers/iio/dac/ad5696-i2c.c
@@ -65,11 +65,9 @@ static int ad5686_i2c_probe(struct i2c_client *i2c,
 			    ad5686_i2c_write, ad5686_i2c_read);
 }
 
-static int ad5686_i2c_remove(struct i2c_client *i2c)
+static void ad5686_i2c_remove(struct i2c_client *i2c)
 {
 	ad5686_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id ad5686_i2c_id[] = {
diff --git a/drivers/iio/dac/ds4424.c b/drivers/iio/dac/ds4424.c
index 509394690bcc..3e17a681174e 100644
--- a/drivers/iio/dac/ds4424.c
+++ b/drivers/iio/dac/ds4424.c
@@ -281,15 +281,13 @@ fail:
 	return ret;
 }
 
-static int ds4424_remove(struct i2c_client *client)
+static void ds4424_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct ds4424_data *data = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
 	regulator_disable(data->vcc_reg);
-
-	return 0;
 }
 
 static const struct i2c_device_id ds4424_id[] = {
diff --git a/drivers/iio/dac/m62332.c b/drivers/iio/dac/m62332.c
index 22b02f50fe41..5a812f87970c 100644
--- a/drivers/iio/dac/m62332.c
+++ b/drivers/iio/dac/m62332.c
@@ -218,7 +218,7 @@ err:
 	return ret;
 }
 
-static int m62332_remove(struct i2c_client *client)
+static void m62332_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
@@ -226,8 +226,6 @@ static int m62332_remove(struct i2c_client *client)
 	iio_map_array_unregister(indio_dev);
 	m62332_set_value(indio_dev, 0, 0);
 	m62332_set_value(indio_dev, 0, 1);
-
-	return 0;
 }
 
 static const struct i2c_device_id m62332_id[] = {
diff --git a/drivers/iio/dac/mcp4725.c b/drivers/iio/dac/mcp4725.c
index bb4b85a7b95b..446d1a8fe4be 100644
--- a/drivers/iio/dac/mcp4725.c
+++ b/drivers/iio/dac/mcp4725.c
@@ -486,7 +486,7 @@ err_disable_vdd_reg:
 	return err;
 }
 
-static int mcp4725_remove(struct i2c_client *client)
+static void mcp4725_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct mcp4725_data *data = iio_priv(indio_dev);
@@ -496,8 +496,6 @@ static int mcp4725_remove(struct i2c_client *client)
 	if (data->vref_reg)
 		regulator_disable(data->vref_reg);
 	regulator_disable(data->vdd_reg);
-
-	return 0;
 }
 
 static const struct i2c_device_id mcp4725_id[] = {
diff --git a/drivers/iio/dac/ti-dac5571.c b/drivers/iio/dac/ti-dac5571.c
index f91f8a504989..3210e3098f9a 100644
--- a/drivers/iio/dac/ti-dac5571.c
+++ b/drivers/iio/dac/ti-dac5571.c
@@ -382,15 +382,13 @@ static int dac5571_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int dac5571_remove(struct i2c_client *i2c)
+static void dac5571_remove(struct i2c_client *i2c)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(i2c);
 	struct dac5571_data *data = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
 	regulator_disable(data->vref);
-
-	return 0;
 }
 
 static const struct of_device_id dac5571_of_id[] = {
diff --git a/drivers/iio/gyro/bmg160_i2c.c b/drivers/iio/gyro/bmg160_i2c.c
index b3fa46bd02cb..908ccc385254 100644
--- a/drivers/iio/gyro/bmg160_i2c.c
+++ b/drivers/iio/gyro/bmg160_i2c.c
@@ -32,11 +32,9 @@ static int bmg160_i2c_probe(struct i2c_client *client,
 	return bmg160_core_probe(&client->dev, regmap, client->irq, name);
 }
 
-static int bmg160_i2c_remove(struct i2c_client *client)
+static void bmg160_i2c_remove(struct i2c_client *client)
 {
 	bmg160_core_remove(&client->dev);
-
-	return 0;
 }
 
 static const struct acpi_device_id bmg160_acpi_match[] = {
diff --git a/drivers/iio/gyro/fxas21002c_i2c.c b/drivers/iio/gyro/fxas21002c_i2c.c
index a7807fd97483..13bb52c594d1 100644
--- a/drivers/iio/gyro/fxas21002c_i2c.c
+++ b/drivers/iio/gyro/fxas21002c_i2c.c
@@ -33,11 +33,9 @@ static int fxas21002c_i2c_probe(struct i2c_client *i2c)
 	return fxas21002c_core_probe(&i2c->dev, regmap, i2c->irq, i2c->name);
 }
 
-static int fxas21002c_i2c_remove(struct i2c_client *i2c)
+static void fxas21002c_i2c_remove(struct i2c_client *i2c)
 {
 	fxas21002c_core_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id fxas21002c_i2c_id[] = {
diff --git a/drivers/iio/gyro/itg3200_core.c b/drivers/iio/gyro/itg3200_core.c
index 0491c64e1b32..421501584587 100644
--- a/drivers/iio/gyro/itg3200_core.c
+++ b/drivers/iio/gyro/itg3200_core.c
@@ -350,7 +350,7 @@ error_unconfigure_buffer:
 	return ret;
 }
 
-static int itg3200_remove(struct i2c_client *client)
+static void itg3200_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
@@ -360,8 +360,6 @@ static int itg3200_remove(struct i2c_client *client)
 		itg3200_remove_trigger(indio_dev);
 
 	itg3200_buffer_unconfigure(indio_dev);
-
-	return 0;
 }
 
 static int itg3200_suspend(struct device *dev)
diff --git a/drivers/iio/gyro/mpu3050-i2c.c b/drivers/iio/gyro/mpu3050-i2c.c
index 78f4a0102986..12e3afa9dd11 100644
--- a/drivers/iio/gyro/mpu3050-i2c.c
+++ b/drivers/iio/gyro/mpu3050-i2c.c
@@ -78,7 +78,7 @@ static int mpu3050_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int mpu3050_i2c_remove(struct i2c_client *client)
+static void mpu3050_i2c_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(&client->dev);
 	struct mpu3050 *mpu3050 = iio_priv(indio_dev);
@@ -87,8 +87,6 @@ static int mpu3050_i2c_remove(struct i2c_client *client)
 		i2c_mux_del_adapters(mpu3050->i2cmux);
 
 	mpu3050_common_remove(&client->dev);
-
-	return 0;
 }
 
 /*
diff --git a/drivers/iio/health/afe4404.c b/drivers/iio/health/afe4404.c
index dd7800159051..8fca787b2524 100644
--- a/drivers/iio/health/afe4404.c
+++ b/drivers/iio/health/afe4404.c
@@ -578,7 +578,7 @@ disable_reg:
 	return ret;
 }
 
-static int afe4404_remove(struct i2c_client *client)
+static void afe4404_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct afe4404_data *afe = iio_priv(indio_dev);
@@ -594,8 +594,6 @@ static int afe4404_remove(struct i2c_client *client)
 	ret = regulator_disable(afe->regulator);
 	if (ret)
 		dev_err(afe->dev, "Unable to disable regulator\n");
-
-	return 0;
 }
 
 static const struct i2c_device_id afe4404_ids[] = {
diff --git a/drivers/iio/health/max30100.c b/drivers/iio/health/max30100.c
index ad5717965223..2cca5e0519f8 100644
--- a/drivers/iio/health/max30100.c
+++ b/drivers/iio/health/max30100.c
@@ -471,15 +471,13 @@ static int max30100_probe(struct i2c_client *client,
 	return iio_device_register(indio_dev);
 }
 
-static int max30100_remove(struct i2c_client *client)
+static void max30100_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct max30100_data *data = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
 	max30100_set_powermode(data, false);
-
-	return 0;
 }
 
 static const struct i2c_device_id max30100_id[] = {
diff --git a/drivers/iio/health/max30102.c b/drivers/iio/health/max30102.c
index abbcef563807..437298a29f2d 100644
--- a/drivers/iio/health/max30102.c
+++ b/drivers/iio/health/max30102.c
@@ -592,15 +592,13 @@ static int max30102_probe(struct i2c_client *client,
 	return iio_device_register(indio_dev);
 }
 
-static int max30102_remove(struct i2c_client *client)
+static void max30102_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct max30102_data *data = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
 	max30102_set_power(data, false);
-
-	return 0;
 }
 
 static const struct i2c_device_id max30102_id[] = {
diff --git a/drivers/iio/humidity/hdc2010.c b/drivers/iio/humidity/hdc2010.c
index 1381df46187c..d6858ccb056e 100644
--- a/drivers/iio/humidity/hdc2010.c
+++ b/drivers/iio/humidity/hdc2010.c
@@ -308,7 +308,7 @@ static int hdc2010_probe(struct i2c_client *client,
 	return iio_device_register(indio_dev);
 }
 
-static int hdc2010_remove(struct i2c_client *client)
+static void hdc2010_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct hdc2010_data *data = iio_priv(indio_dev);
@@ -318,8 +318,6 @@ static int hdc2010_remove(struct i2c_client *client)
 	/* Disable Automatic Measurement Mode */
 	if (hdc2010_update_drdy_config(data, HDC2010_AMM, 0))
 		dev_warn(&client->dev, "Unable to restore default AMM\n");
-
-	return 0;
 }
 
 static const struct i2c_device_id hdc2010_id[] = {
diff --git a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
index 2aa647704a79..14255a918eb1 100644
--- a/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
+++ b/drivers/iio/imu/inv_mpu6050/inv_mpu_i2c.c
@@ -157,7 +157,7 @@ out_del_mux:
 	return result;
 }
 
-static int inv_mpu_remove(struct i2c_client *client)
+static void inv_mpu_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct inv_mpu6050_state *st = iio_priv(indio_dev);
@@ -166,8 +166,6 @@ static int inv_mpu_remove(struct i2c_client *client)
 		inv_mpu_acpi_delete_mux_client(client);
 		i2c_mux_del_adapters(st->muxc);
 	}
-
-	return 0;
 }
 
 /*
diff --git a/drivers/iio/imu/kmx61.c b/drivers/iio/imu/kmx61.c
index ec23b1ee472b..b10c0dcac0bb 100644
--- a/drivers/iio/imu/kmx61.c
+++ b/drivers/iio/imu/kmx61.c
@@ -1418,7 +1418,7 @@ err_chip_uninit:
 	return ret;
 }
 
-static int kmx61_remove(struct i2c_client *client)
+static void kmx61_remove(struct i2c_client *client)
 {
 	struct kmx61_data *data = i2c_get_clientdata(client);
 
@@ -1439,8 +1439,6 @@ static int kmx61_remove(struct i2c_client *client)
 	mutex_lock(&data->lock);
 	kmx61_set_mode(data, KMX61_ALL_STBY, KMX61_ACC | KMX61_MAG, true);
 	mutex_unlock(&data->lock);
-
-	return 0;
 }
 
 static int kmx61_suspend(struct device *dev)
diff --git a/drivers/iio/light/apds9300.c b/drivers/iio/light/apds9300.c
index 0f9d77598997..b70f2681bcb3 100644
--- a/drivers/iio/light/apds9300.c
+++ b/drivers/iio/light/apds9300.c
@@ -452,7 +452,7 @@ err:
 	return ret;
 }
 
-static int apds9300_remove(struct i2c_client *client)
+static void apds9300_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct apds9300_data *data = iio_priv(indio_dev);
@@ -462,8 +462,6 @@ static int apds9300_remove(struct i2c_client *client)
 	/* Ensure that power off and interrupts are disabled */
 	apds9300_set_intr_state(data, 0);
 	apds9300_set_power_state(data, 0);
-
-	return 0;
 }
 
 static int apds9300_suspend(struct device *dev)
diff --git a/drivers/iio/light/apds9960.c b/drivers/iio/light/apds9960.c
index 09b831f9f40b..b62c139baf41 100644
--- a/drivers/iio/light/apds9960.c
+++ b/drivers/iio/light/apds9960.c
@@ -1067,7 +1067,7 @@ error_power_down:
 	return ret;
 }
 
-static int apds9960_remove(struct i2c_client *client)
+static void apds9960_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct apds9960_data *data = iio_priv(indio_dev);
@@ -1076,8 +1076,6 @@ static int apds9960_remove(struct i2c_client *client)
 	pm_runtime_disable(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
 	apds9960_set_powermode(data, 0);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/iio/light/bh1750.c b/drivers/iio/light/bh1750.c
index 471985c220bb..3e92820bc820 100644
--- a/drivers/iio/light/bh1750.c
+++ b/drivers/iio/light/bh1750.c
@@ -263,7 +263,7 @@ static int bh1750_probe(struct i2c_client *client,
 	return iio_device_register(indio_dev);
 }
 
-static int bh1750_remove(struct i2c_client *client)
+static void bh1750_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct bh1750_data *data = iio_priv(indio_dev);
@@ -273,8 +273,6 @@ static int bh1750_remove(struct i2c_client *client)
 	mutex_lock(&data->lock);
 	i2c_smbus_write_byte(client, BH1750_POWER_DOWN);
 	mutex_unlock(&data->lock);
-
-	return 0;
 }
 
 static int bh1750_suspend(struct device *dev)
diff --git a/drivers/iio/light/bh1780.c b/drivers/iio/light/bh1780.c
index fc7141390117..90bca392b262 100644
--- a/drivers/iio/light/bh1780.c
+++ b/drivers/iio/light/bh1780.c
@@ -202,7 +202,7 @@ out_disable_pm:
 	return ret;
 }
 
-static int bh1780_remove(struct i2c_client *client)
+static void bh1780_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct bh1780_data *bh1780 = iio_priv(indio_dev);
@@ -216,8 +216,6 @@ static int bh1780_remove(struct i2c_client *client)
 	if (ret < 0)
 		dev_err(&client->dev, "failed to power off (%pe)\n",
 			ERR_PTR(ret));
-
-	return 0;
 }
 
 static int bh1780_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/light/cm3232.c b/drivers/iio/light/cm3232.c
index 2c80a0535d2c..5214cd014cf8 100644
--- a/drivers/iio/light/cm3232.c
+++ b/drivers/iio/light/cm3232.c
@@ -357,7 +357,7 @@ static int cm3232_probe(struct i2c_client *client,
 	return iio_device_register(indio_dev);
 }
 
-static int cm3232_remove(struct i2c_client *client)
+static void cm3232_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
@@ -365,8 +365,6 @@ static int cm3232_remove(struct i2c_client *client)
 		CM3232_CMD_ALS_DISABLE);
 
 	iio_device_unregister(indio_dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id cm3232_id[] = {
diff --git a/drivers/iio/light/cm36651.c b/drivers/iio/light/cm36651.c
index 89f5e48a6642..6615c98b601c 100644
--- a/drivers/iio/light/cm36651.c
+++ b/drivers/iio/light/cm36651.c
@@ -700,7 +700,7 @@ error_disable_reg:
 	return ret;
 }
 
-static int cm36651_remove(struct i2c_client *client)
+static void cm36651_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct cm36651_data *cm36651 = iio_priv(indio_dev);
@@ -710,8 +710,6 @@ static int cm36651_remove(struct i2c_client *client)
 	free_irq(client->irq, indio_dev);
 	i2c_unregister_device(cm36651->ps_client);
 	i2c_unregister_device(cm36651->ara_client);
-
-	return 0;
 }
 
 static const struct i2c_device_id cm36651_id[] = {
diff --git a/drivers/iio/light/gp2ap002.c b/drivers/iio/light/gp2ap002.c
index e2707416f9a8..8000fa347344 100644
--- a/drivers/iio/light/gp2ap002.c
+++ b/drivers/iio/light/gp2ap002.c
@@ -619,7 +619,7 @@ out_disable_vdd:
 	return ret;
 }
 
-static int gp2ap002_remove(struct i2c_client *client)
+static void gp2ap002_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct gp2ap002 *gp2ap002 = iio_priv(indio_dev);
@@ -631,8 +631,6 @@ static int gp2ap002_remove(struct i2c_client *client)
 	iio_device_unregister(indio_dev);
 	regulator_disable(gp2ap002->vio);
 	regulator_disable(gp2ap002->vdd);
-
-	return 0;
 }
 
 static int gp2ap002_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/light/gp2ap020a00f.c b/drivers/iio/light/gp2ap020a00f.c
index b820041159f7..826439299e8b 100644
--- a/drivers/iio/light/gp2ap020a00f.c
+++ b/drivers/iio/light/gp2ap020a00f.c
@@ -1573,7 +1573,7 @@ error_regulator_disable:
 	return err;
 }
 
-static int gp2ap020a00f_remove(struct i2c_client *client)
+static void gp2ap020a00f_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct gp2ap020a00f_data *data = iio_priv(indio_dev);
@@ -1589,8 +1589,6 @@ static int gp2ap020a00f_remove(struct i2c_client *client)
 	free_irq(client->irq, indio_dev);
 	iio_triggered_buffer_cleanup(indio_dev);
 	regulator_disable(data->vled_reg);
-
-	return 0;
 }
 
 static const struct i2c_device_id gp2ap020a00f_id[] = {
diff --git a/drivers/iio/light/isl29028.c b/drivers/iio/light/isl29028.c
index ff5996d77818..32d58e18f26d 100644
--- a/drivers/iio/light/isl29028.c
+++ b/drivers/iio/light/isl29028.c
@@ -636,7 +636,7 @@ static int isl29028_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int isl29028_remove(struct i2c_client *client)
+static void isl29028_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct isl29028_chip *chip = iio_priv(indio_dev);
@@ -647,8 +647,6 @@ static int isl29028_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	isl29028_clear_configure_reg(chip);
-
-	return 0;
 }
 
 static int isl29028_suspend(struct device *dev)
diff --git a/drivers/iio/light/isl29125.c b/drivers/iio/light/isl29125.c
index eb68a52aab82..c199e63cce82 100644
--- a/drivers/iio/light/isl29125.c
+++ b/drivers/iio/light/isl29125.c
@@ -300,15 +300,13 @@ static int isl29125_powerdown(struct isl29125_data *data)
 		(data->conf1 & ~ISL29125_MODE_MASK) | ISL29125_MODE_PD);
 }
 
-static int isl29125_remove(struct i2c_client *client)
+static void isl29125_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
 	iio_device_unregister(indio_dev);
 	iio_triggered_buffer_cleanup(indio_dev);
 	isl29125_powerdown(iio_priv(indio_dev));
-
-	return 0;
 }
 
 static int isl29125_suspend(struct device *dev)
diff --git a/drivers/iio/light/jsa1212.c b/drivers/iio/light/jsa1212.c
index 5387c12231cf..57ce6d75966c 100644
--- a/drivers/iio/light/jsa1212.c
+++ b/drivers/iio/light/jsa1212.c
@@ -373,7 +373,7 @@ static int jsa1212_power_off(struct jsa1212_data *data)
 	return ret;
 }
 
-static int jsa1212_remove(struct i2c_client *client)
+static void jsa1212_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct jsa1212_data *data = iio_priv(indio_dev);
@@ -381,8 +381,6 @@ static int jsa1212_remove(struct i2c_client *client)
 	iio_device_unregister(indio_dev);
 
 	jsa1212_power_off(data);
-
-	return 0;
 }
 
 static int jsa1212_suspend(struct device *dev)
diff --git a/drivers/iio/light/ltr501.c b/drivers/iio/light/ltr501.c
index 679a1e1086ae..74a1ccda8b9c 100644
--- a/drivers/iio/light/ltr501.c
+++ b/drivers/iio/light/ltr501.c
@@ -1600,15 +1600,13 @@ powerdown_on_error:
 	return ret;
 }
 
-static int ltr501_remove(struct i2c_client *client)
+static void ltr501_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
 	iio_device_unregister(indio_dev);
 	iio_triggered_buffer_cleanup(indio_dev);
 	ltr501_powerdown(iio_priv(indio_dev));
-
-	return 0;
 }
 
 static int ltr501_suspend(struct device *dev)
diff --git a/drivers/iio/light/opt3001.c b/drivers/iio/light/opt3001.c
index a326d47afc9b..a26d1c3f9543 100644
--- a/drivers/iio/light/opt3001.c
+++ b/drivers/iio/light/opt3001.c
@@ -794,7 +794,7 @@ static int opt3001_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int opt3001_remove(struct i2c_client *client)
+static void opt3001_remove(struct i2c_client *client)
 {
 	struct iio_dev *iio = i2c_get_clientdata(client);
 	struct opt3001 *opt = iio_priv(iio);
@@ -808,7 +808,7 @@ static int opt3001_remove(struct i2c_client *client)
 	if (ret < 0) {
 		dev_err(opt->dev, "failed to read register %02x\n",
 				OPT3001_CONFIGURATION);
-		return 0;
+		return;
 	}
 
 	reg = ret;
@@ -820,8 +820,6 @@ static int opt3001_remove(struct i2c_client *client)
 		dev_err(opt->dev, "failed to write register %02x\n",
 				OPT3001_CONFIGURATION);
 	}
-
-	return 0;
 }
 
 static const struct i2c_device_id opt3001_id[] = {
diff --git a/drivers/iio/light/pa12203001.c b/drivers/iio/light/pa12203001.c
index 772874e707ae..3cb2de51f4aa 100644
--- a/drivers/iio/light/pa12203001.c
+++ b/drivers/iio/light/pa12203001.c
@@ -394,7 +394,7 @@ out_err:
 	return ret;
 }
 
-static int pa12203001_remove(struct i2c_client *client)
+static void pa12203001_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	int ret;
@@ -408,8 +408,6 @@ static int pa12203001_remove(struct i2c_client *client)
 	if (ret)
 		dev_warn(&client->dev, "Failed to power down (%pe)\n",
 			 ERR_PTR(ret));
-
-	return 0;
 }
 
 #if defined(CONFIG_PM_SLEEP) || defined(CONFIG_PM)
diff --git a/drivers/iio/light/rpr0521.c b/drivers/iio/light/rpr0521.c
index dabdd05f0e2c..d1c16dd76058 100644
--- a/drivers/iio/light/rpr0521.c
+++ b/drivers/iio/light/rpr0521.c
@@ -1041,7 +1041,7 @@ err_poweroff:
 	return ret;
 }
 
-static int rpr0521_remove(struct i2c_client *client)
+static void rpr0521_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
@@ -1051,8 +1051,6 @@ static int rpr0521_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	rpr0521_poweroff(iio_priv(indio_dev));
-
-	return 0;
 }
 
 static int rpr0521_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/light/stk3310.c b/drivers/iio/light/stk3310.c
index f7cc7a6c0c8d..7b8e0da6aabc 100644
--- a/drivers/iio/light/stk3310.c
+++ b/drivers/iio/light/stk3310.c
@@ -649,14 +649,12 @@ err_standby:
 	return ret;
 }
 
-static int stk3310_remove(struct i2c_client *client)
+static void stk3310_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
 	iio_device_unregister(indio_dev);
 	stk3310_set_state(iio_priv(indio_dev), STK3310_STATE_STANDBY);
-
-	return 0;
 }
 
 static int stk3310_suspend(struct device *dev)
diff --git a/drivers/iio/light/tcs3472.c b/drivers/iio/light/tcs3472.c
index 823435f59bb6..db17fec634be 100644
--- a/drivers/iio/light/tcs3472.c
+++ b/drivers/iio/light/tcs3472.c
@@ -559,7 +559,7 @@ static int tcs3472_powerdown(struct tcs3472_data *data)
 	return ret;
 }
 
-static int tcs3472_remove(struct i2c_client *client)
+static void tcs3472_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
@@ -568,8 +568,6 @@ static int tcs3472_remove(struct i2c_client *client)
 		free_irq(client->irq, indio_dev);
 	iio_triggered_buffer_cleanup(indio_dev);
 	tcs3472_powerdown(iio_priv(indio_dev));
-
-	return 0;
 }
 
 static int tcs3472_suspend(struct device *dev)
diff --git a/drivers/iio/light/tsl2563.c b/drivers/iio/light/tsl2563.c
index bbb577459fb9..951f35ef3f41 100644
--- a/drivers/iio/light/tsl2563.c
+++ b/drivers/iio/light/tsl2563.c
@@ -796,7 +796,7 @@ fail:
 	return err;
 }
 
-static int tsl2563_remove(struct i2c_client *client)
+static void tsl2563_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct tsl2563_chip *chip = iio_priv(indio_dev);
@@ -809,8 +809,6 @@ static int tsl2563_remove(struct i2c_client *client)
 	i2c_smbus_write_byte_data(chip->client, TSL2563_CMD | TSL2563_REG_INT,
 				  chip->intr);
 	tsl2563_set_power(chip, 0);
-
-	return 0;
 }
 
 static int tsl2563_suspend(struct device *dev)
diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c
index 82662dab87c0..0a2ca1a8146d 100644
--- a/drivers/iio/light/tsl2583.c
+++ b/drivers/iio/light/tsl2583.c
@@ -873,7 +873,7 @@ static int tsl2583_probe(struct i2c_client *clientp,
 	return 0;
 }
 
-static int tsl2583_remove(struct i2c_client *client)
+static void tsl2583_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct tsl2583_chip *chip = iio_priv(indio_dev);
@@ -884,8 +884,6 @@ static int tsl2583_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	tsl2583_set_power_state(chip, TSL2583_CNTL_PWR_OFF);
-
-	return 0;
 }
 
 static int tsl2583_suspend(struct device *dev)
diff --git a/drivers/iio/light/tsl4531.c b/drivers/iio/light/tsl4531.c
index 6ae1b27e50b6..090038fed889 100644
--- a/drivers/iio/light/tsl4531.c
+++ b/drivers/iio/light/tsl4531.c
@@ -207,12 +207,10 @@ static int tsl4531_powerdown(struct i2c_client *client)
 		TSL4531_MODE_POWERDOWN);
 }
 
-static int tsl4531_remove(struct i2c_client *client)
+static void tsl4531_remove(struct i2c_client *client)
 {
 	iio_device_unregister(i2c_get_clientdata(client));
 	tsl4531_powerdown(client);
-
-	return 0;
 }
 
 static int tsl4531_suspend(struct device *dev)
diff --git a/drivers/iio/light/us5182d.c b/drivers/iio/light/us5182d.c
index 80d2299da561..3e652d7f3b0e 100644
--- a/drivers/iio/light/us5182d.c
+++ b/drivers/iio/light/us5182d.c
@@ -904,7 +904,7 @@ out_err:
 
 }
 
-static int us5182d_remove(struct i2c_client *client)
+static void us5182d_remove(struct i2c_client *client)
 {
 	struct us5182d_data *data = iio_priv(i2c_get_clientdata(client));
 	int ret;
@@ -918,8 +918,6 @@ static int us5182d_remove(struct i2c_client *client)
 	if (ret)
 		dev_warn(&client->dev, "Failed to shut down (%pe)\n",
 			 ERR_PTR(ret));
-
-	return 0;
 }
 
 static int us5182d_suspend(struct device *dev)
diff --git a/drivers/iio/light/vcnl4000.c b/drivers/iio/light/vcnl4000.c
index 3db4e26731bb..f6c83ecaad8b 100644
--- a/drivers/iio/light/vcnl4000.c
+++ b/drivers/iio/light/vcnl4000.c
@@ -1111,7 +1111,7 @@ static const struct of_device_id vcnl_4000_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, vcnl_4000_of_match);
 
-static int vcnl4000_remove(struct i2c_client *client)
+static void vcnl4000_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct vcnl4000_data *data = iio_priv(indio_dev);
@@ -1126,8 +1126,6 @@ static int vcnl4000_remove(struct i2c_client *client)
 	if (ret)
 		dev_warn(&client->dev, "Failed to power down (%pe)\n",
 			 ERR_PTR(ret));
-
-	return 0;
 }
 
 static int vcnl4000_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/light/vcnl4035.c b/drivers/iio/light/vcnl4035.c
index 6a196cf2270b..3ed37f6057fb 100644
--- a/drivers/iio/light/vcnl4035.c
+++ b/drivers/iio/light/vcnl4035.c
@@ -601,7 +601,7 @@ fail_poweroff:
 	return ret;
 }
 
-static int vcnl4035_remove(struct i2c_client *client)
+static void vcnl4035_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	int ret;
@@ -616,8 +616,6 @@ static int vcnl4035_remove(struct i2c_client *client)
 	if (ret)
 		dev_warn(&client->dev, "Failed to put device into standby (%pe)\n",
 			 ERR_PTR(ret));
-
-	return 0;
 }
 
 static int vcnl4035_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/light/veml6070.c b/drivers/iio/light/veml6070.c
index 1e55e09a8d16..cfa4e9e7c803 100644
--- a/drivers/iio/light/veml6070.c
+++ b/drivers/iio/light/veml6070.c
@@ -180,15 +180,13 @@ fail:
 	return ret;
 }
 
-static int veml6070_remove(struct i2c_client *client)
+static void veml6070_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct veml6070_data *data = iio_priv(indio_dev);
 
 	iio_device_unregister(indio_dev);
 	i2c_unregister_device(data->client2);
-
-	return 0;
 }
 
 static const struct i2c_device_id veml6070_id[] = {
diff --git a/drivers/iio/magnetometer/ak8974.c b/drivers/iio/magnetometer/ak8974.c
index c89a91db0690..7ec9ab3beb45 100644
--- a/drivers/iio/magnetometer/ak8974.c
+++ b/drivers/iio/magnetometer/ak8974.c
@@ -969,7 +969,7 @@ disable_pm:
 	return ret;
 }
 
-static int ak8974_remove(struct i2c_client *i2c)
+static void ak8974_remove(struct i2c_client *i2c)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(i2c);
 	struct ak8974 *ak8974 = iio_priv(indio_dev);
@@ -981,8 +981,6 @@ static int ak8974_remove(struct i2c_client *i2c)
 	pm_runtime_disable(&i2c->dev);
 	ak8974_set_power(ak8974, AK8974_PWR_OFF);
 	regulator_bulk_disable(ARRAY_SIZE(ak8974->regs), ak8974->regs);
-
-	return 0;
 }
 
 static int ak8974_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c
index 2432e697150c..caf03a2a98a5 100644
--- a/drivers/iio/magnetometer/ak8975.c
+++ b/drivers/iio/magnetometer/ak8975.c
@@ -1018,7 +1018,7 @@ power_off:
 	return err;
 }
 
-static int ak8975_remove(struct i2c_client *client)
+static void ak8975_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct ak8975_data *data = iio_priv(indio_dev);
@@ -1030,8 +1030,6 @@ static int ak8975_remove(struct i2c_client *client)
 	iio_triggered_buffer_cleanup(indio_dev);
 	ak8975_set_mode(data, POWER_DOWN);
 	ak8975_power_off(data);
-
-	return 0;
 }
 
 static int ak8975_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/magnetometer/bmc150_magn_i2c.c b/drivers/iio/magnetometer/bmc150_magn_i2c.c
index 65c004411d0f..570deaa87836 100644
--- a/drivers/iio/magnetometer/bmc150_magn_i2c.c
+++ b/drivers/iio/magnetometer/bmc150_magn_i2c.c
@@ -34,11 +34,9 @@ static int bmc150_magn_i2c_probe(struct i2c_client *client,
 	return bmc150_magn_probe(&client->dev, regmap, client->irq, name);
 }
 
-static int bmc150_magn_i2c_remove(struct i2c_client *client)
+static void bmc150_magn_i2c_remove(struct i2c_client *client)
 {
 	bmc150_magn_remove(&client->dev);
-
-	return 0;
 }
 
 static const struct acpi_device_id bmc150_magn_acpi_match[] = {
diff --git a/drivers/iio/magnetometer/hmc5843_i2c.c b/drivers/iio/magnetometer/hmc5843_i2c.c
index 8d2ff8fc204d..fe5e8415b2f2 100644
--- a/drivers/iio/magnetometer/hmc5843_i2c.c
+++ b/drivers/iio/magnetometer/hmc5843_i2c.c
@@ -65,11 +65,9 @@ static int hmc5843_i2c_probe(struct i2c_client *cli,
 			id->driver_data, id->name);
 }
 
-static int hmc5843_i2c_remove(struct i2c_client *client)
+static void hmc5843_i2c_remove(struct i2c_client *client)
 {
 	hmc5843_common_remove(&client->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id hmc5843_id[] = {
diff --git a/drivers/iio/magnetometer/mag3110.c b/drivers/iio/magnetometer/mag3110.c
index 226439d0bfb5..b870ad803862 100644
--- a/drivers/iio/magnetometer/mag3110.c
+++ b/drivers/iio/magnetometer/mag3110.c
@@ -559,7 +559,7 @@ disable_regulator_vdd:
 	return ret;
 }
 
-static int mag3110_remove(struct i2c_client *client)
+static void mag3110_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct mag3110_data *data = iio_priv(indio_dev);
@@ -569,8 +569,6 @@ static int mag3110_remove(struct i2c_client *client)
 	mag3110_standby(iio_priv(indio_dev));
 	regulator_disable(data->vddio_reg);
 	regulator_disable(data->vdd_reg);
-
-	return 0;
 }
 
 static int mag3110_suspend(struct device *dev)
diff --git a/drivers/iio/magnetometer/yamaha-yas530.c b/drivers/iio/magnetometer/yamaha-yas530.c
index aeaa4da6923b..c3a10942654e 100644
--- a/drivers/iio/magnetometer/yamaha-yas530.c
+++ b/drivers/iio/magnetometer/yamaha-yas530.c
@@ -943,7 +943,7 @@ reg_off:
 	return ret;
 }
 
-static int yas5xx_remove(struct i2c_client *i2c)
+static void yas5xx_remove(struct i2c_client *i2c)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(i2c);
 	struct yas5xx *yas5xx = iio_priv(indio_dev);
@@ -961,8 +961,6 @@ static int yas5xx_remove(struct i2c_client *i2c)
 	pm_runtime_disable(dev);
 	gpiod_set_value_cansleep(yas5xx->reset, 1);
 	regulator_bulk_disable(ARRAY_SIZE(yas5xx->regs), yas5xx->regs);
-
-	return 0;
 }
 
 static int yas5xx_runtime_suspend(struct device *dev)
diff --git a/drivers/iio/potentiostat/lmp91000.c b/drivers/iio/potentiostat/lmp91000.c
index fe514f0b5506..5ec7060d31d9 100644
--- a/drivers/iio/potentiostat/lmp91000.c
+++ b/drivers/iio/potentiostat/lmp91000.c
@@ -384,7 +384,7 @@ error_unreg_trigger:
 	return ret;
 }
 
-static int lmp91000_remove(struct i2c_client *client)
+static void lmp91000_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct lmp91000_data *data = iio_priv(indio_dev);
@@ -396,8 +396,6 @@ static int lmp91000_remove(struct i2c_client *client)
 
 	iio_triggered_buffer_cleanup(indio_dev);
 	iio_trigger_unregister(data->trig);
-
-	return 0;
 }
 
 static const struct of_device_id lmp91000_of_match[] = {
diff --git a/drivers/iio/pressure/mpl3115.c b/drivers/iio/pressure/mpl3115.c
index d4f89e4babed..2f22aba61e4d 100644
--- a/drivers/iio/pressure/mpl3115.c
+++ b/drivers/iio/pressure/mpl3115.c
@@ -290,15 +290,13 @@ static int mpl3115_standby(struct mpl3115_data *data)
 		data->ctrl_reg1 & ~MPL3115_CTRL_ACTIVE);
 }
 
-static int mpl3115_remove(struct i2c_client *client)
+static void mpl3115_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
 	iio_device_unregister(indio_dev);
 	iio_triggered_buffer_cleanup(indio_dev);
 	mpl3115_standby(iio_priv(indio_dev));
-
-	return 0;
 }
 
 static int mpl3115_suspend(struct device *dev)
diff --git a/drivers/iio/pressure/ms5611_i2c.c b/drivers/iio/pressure/ms5611_i2c.c
index 3b1de71e0d15..b681a4183909 100644
--- a/drivers/iio/pressure/ms5611_i2c.c
+++ b/drivers/iio/pressure/ms5611_i2c.c
@@ -105,11 +105,9 @@ static int ms5611_i2c_probe(struct i2c_client *client,
 	return ms5611_probe(indio_dev, &client->dev, id->name, id->driver_data);
 }
 
-static int ms5611_i2c_remove(struct i2c_client *client)
+static void ms5611_i2c_remove(struct i2c_client *client)
 {
 	ms5611_remove(i2c_get_clientdata(client));
-
-	return 0;
 }
 
 static const struct of_device_id ms5611_i2c_matches[] = {
diff --git a/drivers/iio/pressure/zpa2326_i2c.c b/drivers/iio/pressure/zpa2326_i2c.c
index 0db0860d386b..f26dd8cbb387 100644
--- a/drivers/iio/pressure/zpa2326_i2c.c
+++ b/drivers/iio/pressure/zpa2326_i2c.c
@@ -53,11 +53,9 @@ static int zpa2326_probe_i2c(struct i2c_client          *client,
 			     zpa2326_i2c_hwid(client), regmap);
 }
 
-static int zpa2326_remove_i2c(struct i2c_client *client)
+static void zpa2326_remove_i2c(struct i2c_client *client)
 {
 	zpa2326_remove(&client->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id zpa2326_i2c_ids[] = {
diff --git a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c
index 648ae576d6fa..791a33d5286c 100644
--- a/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c
+++ b/drivers/iio/proximity/pulsedlight-lidar-lite-v2.c
@@ -311,7 +311,7 @@ error_unreg_buffer:
 	return ret;
 }
 
-static int lidar_remove(struct i2c_client *client)
+static void lidar_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 
@@ -320,8 +320,6 @@ static int lidar_remove(struct i2c_client *client)
 
 	pm_runtime_disable(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id lidar_id[] = {
diff --git a/drivers/iio/proximity/sx9500.c b/drivers/iio/proximity/sx9500.c
index 42589d6200ad..d4670864ddc7 100644
--- a/drivers/iio/proximity/sx9500.c
+++ b/drivers/iio/proximity/sx9500.c
@@ -979,7 +979,7 @@ out_trigger_unregister:
 	return ret;
 }
 
-static int sx9500_remove(struct i2c_client *client)
+static void sx9500_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct sx9500_data *data = iio_priv(indio_dev);
@@ -989,8 +989,6 @@ static int sx9500_remove(struct i2c_client *client)
 	if (client->irq > 0)
 		iio_trigger_unregister(data->trig);
 	kfree(data->buffer);
-
-	return 0;
 }
 
 static int sx9500_suspend(struct device *dev)
diff --git a/drivers/iio/temperature/mlx90614.c b/drivers/iio/temperature/mlx90614.c
index c253a5315988..0808bb865928 100644
--- a/drivers/iio/temperature/mlx90614.c
+++ b/drivers/iio/temperature/mlx90614.c
@@ -571,7 +571,7 @@ static int mlx90614_probe(struct i2c_client *client,
 	return iio_device_register(indio_dev);
 }
 
-static int mlx90614_remove(struct i2c_client *client)
+static void mlx90614_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct mlx90614_data *data = iio_priv(indio_dev);
@@ -584,8 +584,6 @@ static int mlx90614_remove(struct i2c_client *client)
 			mlx90614_sleep(data);
 		pm_runtime_set_suspended(&client->dev);
 	}
-
-	return 0;
 }
 
 static const struct i2c_device_id mlx90614_id[] = {
diff --git a/drivers/iio/temperature/mlx90632.c b/drivers/iio/temperature/mlx90632.c
index 7ee7ff8047a4..e8ef47147e2b 100644
--- a/drivers/iio/temperature/mlx90632.c
+++ b/drivers/iio/temperature/mlx90632.c
@@ -924,7 +924,7 @@ static int mlx90632_probe(struct i2c_client *client,
 	return iio_device_register(indio_dev);
 }
 
-static int mlx90632_remove(struct i2c_client *client)
+static void mlx90632_remove(struct i2c_client *client)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(client);
 	struct mlx90632_data *data = iio_priv(indio_dev);
@@ -936,8 +936,6 @@ static int mlx90632_remove(struct i2c_client *client)
 	pm_runtime_put_noidle(&client->dev);
 
 	mlx90632_sleep(data);
-
-	return 0;
 }
 
 static const struct i2c_device_id mlx90632_id[] = {
diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c
index b4dc52392275..9c8a7b206dcf 100644
--- a/drivers/infiniband/hw/mlx5/mad.c
+++ b/drivers/infiniband/hw/mlx5/mad.c
@@ -147,6 +147,28 @@ static void pma_cnt_assign(struct ib_pma_portcounters *pma_cnt,
 			     vl_15_dropped);
 }
 
+static int query_ib_ppcnt(struct mlx5_core_dev *dev, u8 port_num, void *out,
+			  size_t sz)
+{
+	u32 *in;
+	int err;
+
+	in  = kvzalloc(sz, GFP_KERNEL);
+	if (!in) {
+		err = -ENOMEM;
+		return err;
+	}
+
+	MLX5_SET(ppcnt_reg, in, local_port, port_num);
+
+	MLX5_SET(ppcnt_reg, in, grp, MLX5_INFINIBAND_PORT_COUNTERS_GROUP);
+	err = mlx5_core_access_reg(dev, in, sz, out,
+				   sz, MLX5_REG_PPCNT, 0, 0);
+
+	kvfree(in);
+	return err;
+}
+
 static int process_pma_cmd(struct mlx5_ib_dev *dev, u32 port_num,
 			   const struct ib_mad *in_mad, struct ib_mad *out_mad)
 {
@@ -208,8 +230,7 @@ static int process_pma_cmd(struct mlx5_ib_dev *dev, u32 port_num,
 			goto done;
 		}
 
-		err = mlx5_core_query_ib_ppcnt(mdev, mdev_port_num,
-					       out_cnt, sz);
+		err = query_ib_ppcnt(mdev, mdev_port_num, out_cnt, sz);
 		if (!err)
 			pma_cnt_assign(pma_cnt, out_cnt);
 	}
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index e66bf72f1f04..62338f44a30e 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1541,6 +1541,18 @@ int mlx5_ib_test_wc(struct mlx5_ib_dev *dev);
 
 static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev)
 {
+	/*
+	 * If the driver is in hash mode and the port_select_flow_table_bypass cap
+	 * is supported, it means that the driver no longer needs to assign the port
+	 * affinity by default. If a user wants to set the port affinity explicitly,
+	 * the user has a dedicated API to do that, so there is no need to assign
+	 * the port affinity by default.
+	 */
+	if (dev->lag_active &&
+	    mlx5_lag_mode_is_hash(dev->mdev) &&
+	    MLX5_CAP_PORT_SELECTION(dev->mdev, port_select_flow_table_bypass))
+		return 0;
+
 	return dev->lag_active ||
 		(MLX5_CAP_GEN(dev->mdev, num_lag_ports) > 1 &&
 		 MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity));
diff --git a/drivers/input/joystick/as5011.c b/drivers/input/joystick/as5011.c
index 34bcd99a46f5..2beda29021a3 100644
--- a/drivers/input/joystick/as5011.c
+++ b/drivers/input/joystick/as5011.c
@@ -327,7 +327,7 @@ err_free_mem:
 	return error;
 }
 
-static int as5011_remove(struct i2c_client *client)
+static void as5011_remove(struct i2c_client *client)
 {
 	struct as5011_device *as5011 = i2c_get_clientdata(client);
 
@@ -337,8 +337,6 @@ static int as5011_remove(struct i2c_client *client)
 
 	input_unregister_device(as5011->input_dev);
 	kfree(as5011);
-
-	return 0;
 }
 
 static const struct i2c_device_id as5011_id[] = {
diff --git a/drivers/input/keyboard/adp5588-keys.c b/drivers/input/keyboard/adp5588-keys.c
index 1a1a05d7cd42..e2719737360a 100644
--- a/drivers/input/keyboard/adp5588-keys.c
+++ b/drivers/input/keyboard/adp5588-keys.c
@@ -592,12 +592,11 @@ static int adp5588_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int adp5588_remove(struct i2c_client *client)
+static void adp5588_remove(struct i2c_client *client)
 {
 	adp5588_write(client, CFG, 0);
 
 	/* all resources will be freed by devm */
-	return 0;
 }
 
 static int __maybe_unused adp5588_suspend(struct device *dev)
diff --git a/drivers/input/keyboard/iqs62x-keys.c b/drivers/input/keyboard/iqs62x-keys.c
index 93446b21f98f..db793a550c25 100644
--- a/drivers/input/keyboard/iqs62x-keys.c
+++ b/drivers/input/keyboard/iqs62x-keys.c
@@ -77,6 +77,7 @@ static int iqs62x_keys_parse_prop(struct platform_device *pdev,
 		if (ret) {
 			dev_err(&pdev->dev, "Failed to read switch code: %d\n",
 				ret);
+			fwnode_handle_put(child);
 			return ret;
 		}
 		iqs62x_keys->switches[i].code = val;
@@ -90,6 +91,8 @@ static int iqs62x_keys_parse_prop(struct platform_device *pdev,
 			iqs62x_keys->switches[i].flag = (i == IQS62X_SW_HALL_N ?
 							 IQS62X_EVENT_HALL_N_T :
 							 IQS62X_EVENT_HALL_S_T);
+
+		fwnode_handle_put(child);
 	}
 
 	return 0;
diff --git a/drivers/input/keyboard/lm8323.c b/drivers/input/keyboard/lm8323.c
index 6c38d034ec6e..407dd2ad6302 100644
--- a/drivers/input/keyboard/lm8323.c
+++ b/drivers/input/keyboard/lm8323.c
@@ -752,7 +752,7 @@ fail1:
 	return err;
 }
 
-static int lm8323_remove(struct i2c_client *client)
+static void lm8323_remove(struct i2c_client *client)
 {
 	struct lm8323_chip *lm = i2c_get_clientdata(client);
 	int i;
@@ -769,8 +769,6 @@ static int lm8323_remove(struct i2c_client *client)
 			led_classdev_unregister(&lm->pwm[i].cdev);
 
 	kfree(lm);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/input/keyboard/lm8333.c b/drivers/input/keyboard/lm8333.c
index 7c5f8c6bb957..9dac22c14125 100644
--- a/drivers/input/keyboard/lm8333.c
+++ b/drivers/input/keyboard/lm8333.c
@@ -200,15 +200,13 @@ static int lm8333_probe(struct i2c_client *client,
 	return err;
 }
 
-static int lm8333_remove(struct i2c_client *client)
+static void lm8333_remove(struct i2c_client *client)
 {
 	struct lm8333 *lm8333 = i2c_get_clientdata(client);
 
 	free_irq(client->irq, lm8333);
 	input_unregister_device(lm8333->input);
 	kfree(lm8333);
-
-	return 0;
 }
 
 static const struct i2c_device_id lm8333_id[] = {
diff --git a/drivers/input/keyboard/mcs_touchkey.c b/drivers/input/keyboard/mcs_touchkey.c
index 8cb0062b98e4..ac1637a3389e 100644
--- a/drivers/input/keyboard/mcs_touchkey.c
+++ b/drivers/input/keyboard/mcs_touchkey.c
@@ -194,7 +194,7 @@ err_free_mem:
 	return error;
 }
 
-static int mcs_touchkey_remove(struct i2c_client *client)
+static void mcs_touchkey_remove(struct i2c_client *client)
 {
 	struct mcs_touchkey_data *data = i2c_get_clientdata(client);
 
@@ -203,8 +203,6 @@ static int mcs_touchkey_remove(struct i2c_client *client)
 		data->poweron(false);
 	input_unregister_device(data->input_dev);
 	kfree(data);
-
-	return 0;
 }
 
 static void mcs_touchkey_shutdown(struct i2c_client *client)
diff --git a/drivers/input/keyboard/qt1070.c b/drivers/input/keyboard/qt1070.c
index 7174e1df1ee3..9fcce18b1d65 100644
--- a/drivers/input/keyboard/qt1070.c
+++ b/drivers/input/keyboard/qt1070.c
@@ -216,7 +216,7 @@ err_free_mem:
 	return err;
 }
 
-static int qt1070_remove(struct i2c_client *client)
+static void qt1070_remove(struct i2c_client *client)
 {
 	struct qt1070_data *data = i2c_get_clientdata(client);
 
@@ -225,8 +225,6 @@ static int qt1070_remove(struct i2c_client *client)
 
 	input_unregister_device(data->input);
 	kfree(data);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/input/keyboard/qt2160.c b/drivers/input/keyboard/qt2160.c
index 32d4a076eaa3..382b1519218c 100644
--- a/drivers/input/keyboard/qt2160.c
+++ b/drivers/input/keyboard/qt2160.c
@@ -432,7 +432,7 @@ err_free_mem:
 	return error;
 }
 
-static int qt2160_remove(struct i2c_client *client)
+static void qt2160_remove(struct i2c_client *client)
 {
 	struct qt2160_data *qt2160 = i2c_get_clientdata(client);
 
@@ -446,8 +446,6 @@ static int qt2160_remove(struct i2c_client *client)
 
 	input_unregister_device(qt2160->input);
 	kfree(qt2160);
-
-	return 0;
 }
 
 static const struct i2c_device_id qt2160_idtable[] = {
diff --git a/drivers/input/keyboard/snvs_pwrkey.c b/drivers/input/keyboard/snvs_pwrkey.c
index 65286762b02a..ad8660be0127 100644
--- a/drivers/input/keyboard/snvs_pwrkey.c
+++ b/drivers/input/keyboard/snvs_pwrkey.c
@@ -20,7 +20,7 @@
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
 
-#define SNVS_HPVIDR1_REG	0xF8
+#define SNVS_HPVIDR1_REG	0xBF8
 #define SNVS_LPSR_REG		0x4C	/* LP Status Register */
 #define SNVS_LPCR_REG		0x38	/* LP Control Register */
 #define SNVS_HPSR_REG		0x14
diff --git a/drivers/input/keyboard/tca6416-keypad.c b/drivers/input/keyboard/tca6416-keypad.c
index 2a9755910065..afcdfbb002ff 100644
--- a/drivers/input/keyboard/tca6416-keypad.c
+++ b/drivers/input/keyboard/tca6416-keypad.c
@@ -307,7 +307,7 @@ fail1:
 	return error;
 }
 
-static int tca6416_keypad_remove(struct i2c_client *client)
+static void tca6416_keypad_remove(struct i2c_client *client)
 {
 	struct tca6416_keypad_chip *chip = i2c_get_clientdata(client);
 
@@ -318,8 +318,6 @@ static int tca6416_keypad_remove(struct i2c_client *client)
 
 	input_unregister_device(chip->input);
 	kfree(chip);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/input/misc/adxl34x-i2c.c b/drivers/input/misc/adxl34x-i2c.c
index a3b5f88d2bd1..5be636aaa94f 100644
--- a/drivers/input/misc/adxl34x-i2c.c
+++ b/drivers/input/misc/adxl34x-i2c.c
@@ -99,13 +99,11 @@ static int adxl34x_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int adxl34x_i2c_remove(struct i2c_client *client)
+static void adxl34x_i2c_remove(struct i2c_client *client)
 {
 	struct adxl34x *ac = i2c_get_clientdata(client);
 
 	adxl34x_remove(ac);
-
-	return 0;
 }
 
 static int __maybe_unused adxl34x_i2c_suspend(struct device *dev)
diff --git a/drivers/input/misc/bma150.c b/drivers/input/misc/bma150.c
index a9d984da95f3..84fe394da7a6 100644
--- a/drivers/input/misc/bma150.c
+++ b/drivers/input/misc/bma150.c
@@ -513,11 +513,9 @@ static int bma150_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int bma150_remove(struct i2c_client *client)
+static void bma150_remove(struct i2c_client *client)
 {
 	pm_runtime_disable(&client->dev);
-
-	return 0;
 }
 
 static int __maybe_unused bma150_suspend(struct device *dev)
diff --git a/drivers/input/misc/cma3000_d0x_i2c.c b/drivers/input/misc/cma3000_d0x_i2c.c
index 03fb49127c3a..3b23210c46b7 100644
--- a/drivers/input/misc/cma3000_d0x_i2c.c
+++ b/drivers/input/misc/cma3000_d0x_i2c.c
@@ -58,13 +58,11 @@ static int cma3000_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int cma3000_i2c_remove(struct i2c_client *client)
+static void cma3000_i2c_remove(struct i2c_client *client)
 {
 	struct cma3000_accl_data *data = i2c_get_clientdata(client);
 
 	cma3000_exit(data);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/input/misc/pcf8574_keypad.c b/drivers/input/misc/pcf8574_keypad.c
index abc423165522..cfd6640e4f82 100644
--- a/drivers/input/misc/pcf8574_keypad.c
+++ b/drivers/input/misc/pcf8574_keypad.c
@@ -157,7 +157,7 @@ static int pcf8574_kp_probe(struct i2c_client *client, const struct i2c_device_i
 	return ret;
 }
 
-static int pcf8574_kp_remove(struct i2c_client *client)
+static void pcf8574_kp_remove(struct i2c_client *client)
 {
 	struct kp_data *lp = i2c_get_clientdata(client);
 
@@ -165,8 +165,6 @@ static int pcf8574_kp_remove(struct i2c_client *client)
 
 	input_unregister_device(lp->idev);
 	kfree(lp);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c
index 434d48ae4b12..ffad142801b3 100644
--- a/drivers/input/mouse/synaptics.c
+++ b/drivers/input/mouse/synaptics.c
@@ -186,7 +186,6 @@ static const char * const smbus_pnp_ids[] = {
 	"LEN2044", /* L470  */
 	"LEN2054", /* E480 */
 	"LEN2055", /* E580 */
-	"LEN2064", /* T14 Gen 1 AMD / P14s Gen 1 AMD */
 	"LEN2068", /* T14 Gen 1 */
 	"SYN3052", /* HP EliteBook 840 G4 */
 	"SYN3221", /* HP 15-ay000 */
diff --git a/drivers/input/mouse/synaptics_i2c.c b/drivers/input/mouse/synaptics_i2c.c
index fa304648d611..987ee67a1045 100644
--- a/drivers/input/mouse/synaptics_i2c.c
+++ b/drivers/input/mouse/synaptics_i2c.c
@@ -587,7 +587,7 @@ err_mem_free:
 	return ret;
 }
 
-static int synaptics_i2c_remove(struct i2c_client *client)
+static void synaptics_i2c_remove(struct i2c_client *client)
 {
 	struct synaptics_i2c *touch = i2c_get_clientdata(client);
 
@@ -596,8 +596,6 @@ static int synaptics_i2c_remove(struct i2c_client *client)
 
 	input_unregister_device(touch->input);
 	kfree(touch);
-
-	return 0;
 }
 
 static int __maybe_unused synaptics_i2c_suspend(struct device *dev)
diff --git a/drivers/input/rmi4/rmi_smbus.c b/drivers/input/rmi4/rmi_smbus.c
index 2407ea43de59..c130468541b7 100644
--- a/drivers/input/rmi4/rmi_smbus.c
+++ b/drivers/input/rmi4/rmi_smbus.c
@@ -338,13 +338,11 @@ static int rmi_smb_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int rmi_smb_remove(struct i2c_client *client)
+static void rmi_smb_remove(struct i2c_client *client)
 {
 	struct rmi_smb_xport *rmi_smb = i2c_get_clientdata(client);
 
 	rmi_unregister_transport_device(&rmi_smb->xport);
-
-	return 0;
 }
 
 static int __maybe_unused rmi_smb_suspend(struct device *dev)
diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c
index eb66cd2689b7..4eedea08b0b5 100644
--- a/drivers/input/touchscreen/atmel_mxt_ts.c
+++ b/drivers/input/touchscreen/atmel_mxt_ts.c
@@ -3284,7 +3284,7 @@ err_disable_regulators:
 	return error;
 }
 
-static int mxt_remove(struct i2c_client *client)
+static void mxt_remove(struct i2c_client *client)
 {
 	struct mxt_data *data = i2c_get_clientdata(client);
 
@@ -3294,8 +3294,6 @@ static int mxt_remove(struct i2c_client *client)
 	mxt_free_object_table(data);
 	regulator_bulk_disable(ARRAY_SIZE(data->regulators),
 			       data->regulators);
-
-	return 0;
 }
 
 static int __maybe_unused mxt_suspend(struct device *dev)
diff --git a/drivers/input/touchscreen/bu21013_ts.c b/drivers/input/touchscreen/bu21013_ts.c
index 2f1f0d7607f8..34f422e246ef 100644
--- a/drivers/input/touchscreen/bu21013_ts.c
+++ b/drivers/input/touchscreen/bu21013_ts.c
@@ -552,15 +552,13 @@ static int bu21013_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int bu21013_remove(struct i2c_client *client)
+static void bu21013_remove(struct i2c_client *client)
 {
 	struct bu21013_ts *ts = i2c_get_clientdata(client);
 
 	/* Make sure IRQ will exit quickly even if there is contact */
 	ts->touch_stopped = true;
 	/* The resources will be freed by devm */
-
-	return 0;
 }
 
 static int __maybe_unused bu21013_suspend(struct device *dev)
diff --git a/drivers/input/touchscreen/cyttsp4_i2c.c b/drivers/input/touchscreen/cyttsp4_i2c.c
index c65ccb2f4716..28ae7c15397a 100644
--- a/drivers/input/touchscreen/cyttsp4_i2c.c
+++ b/drivers/input/touchscreen/cyttsp4_i2c.c
@@ -43,13 +43,11 @@ static int cyttsp4_i2c_probe(struct i2c_client *client,
 	return PTR_ERR_OR_ZERO(ts);
 }
 
-static int cyttsp4_i2c_remove(struct i2c_client *client)
+static void cyttsp4_i2c_remove(struct i2c_client *client)
 {
 	struct cyttsp4 *ts = i2c_get_clientdata(client);
 
 	cyttsp4_remove(ts);
-
-	return 0;
 }
 
 static const struct i2c_device_id cyttsp4_i2c_id[] = {
diff --git a/drivers/input/touchscreen/edt-ft5x06.c b/drivers/input/touchscreen/edt-ft5x06.c
index 82beddb28761..5fb441387fe5 100644
--- a/drivers/input/touchscreen/edt-ft5x06.c
+++ b/drivers/input/touchscreen/edt-ft5x06.c
@@ -1346,13 +1346,11 @@ static int edt_ft5x06_ts_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int edt_ft5x06_ts_remove(struct i2c_client *client)
+static void edt_ft5x06_ts_remove(struct i2c_client *client)
 {
 	struct edt_ft5x06_ts_data *tsdata = i2c_get_clientdata(client);
 
 	edt_ft5x06_ts_teardown_debugfs(tsdata);
-
-	return 0;
 }
 
 static int __maybe_unused edt_ft5x06_ts_suspend(struct device *dev)
diff --git a/drivers/input/touchscreen/goodix.c b/drivers/input/touchscreen/goodix.c
index 21c0dddbe41d..a33cc7950cf5 100644
--- a/drivers/input/touchscreen/goodix.c
+++ b/drivers/input/touchscreen/goodix.c
@@ -1383,14 +1383,12 @@ reset:
 	return 0;
 }
 
-static int goodix_ts_remove(struct i2c_client *client)
+static void goodix_ts_remove(struct i2c_client *client)
 {
 	struct goodix_ts_data *ts = i2c_get_clientdata(client);
 
 	if (ts->load_cfg_from_disk)
 		wait_for_completion(&ts->firmware_loading_complete);
-
-	return 0;
 }
 
 static int __maybe_unused goodix_suspend(struct device *dev)
diff --git a/drivers/input/touchscreen/melfas_mip4.c b/drivers/input/touchscreen/melfas_mip4.c
index 2745bf1aee38..83f4be05e27b 100644
--- a/drivers/input/touchscreen/melfas_mip4.c
+++ b/drivers/input/touchscreen/melfas_mip4.c
@@ -1453,7 +1453,7 @@ static int mip4_probe(struct i2c_client *client, const struct i2c_device_id *id)
 					      "ce", GPIOD_OUT_LOW);
 	if (IS_ERR(ts->gpio_ce)) {
 		error = PTR_ERR(ts->gpio_ce);
-		if (error != EPROBE_DEFER)
+		if (error != -EPROBE_DEFER)
 			dev_err(&client->dev,
 				"Failed to get gpio: %d\n", error);
 		return error;
diff --git a/drivers/input/touchscreen/migor_ts.c b/drivers/input/touchscreen/migor_ts.c
index 42d3fd7e04d7..79cd660d879e 100644
--- a/drivers/input/touchscreen/migor_ts.c
+++ b/drivers/input/touchscreen/migor_ts.c
@@ -176,7 +176,7 @@ static int migor_ts_probe(struct i2c_client *client,
 	return error;
 }
 
-static int migor_ts_remove(struct i2c_client *client)
+static void migor_ts_remove(struct i2c_client *client)
 {
 	struct migor_ts_priv *priv = i2c_get_clientdata(client);
 
@@ -185,8 +185,6 @@ static int migor_ts_remove(struct i2c_client *client)
 	kfree(priv);
 
 	dev_set_drvdata(&client->dev, NULL);
-
-	return 0;
 }
 
 static int __maybe_unused migor_ts_suspend(struct device *dev)
diff --git a/drivers/input/touchscreen/s6sy761.c b/drivers/input/touchscreen/s6sy761.c
index 85a1f465c097..1a7d00289b4c 100644
--- a/drivers/input/touchscreen/s6sy761.c
+++ b/drivers/input/touchscreen/s6sy761.c
@@ -475,11 +475,9 @@ static int s6sy761_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int s6sy761_remove(struct i2c_client *client)
+static void s6sy761_remove(struct i2c_client *client)
 {
 	pm_runtime_disable(&client->dev);
-
-	return 0;
 }
 
 static int __maybe_unused s6sy761_runtime_suspend(struct device *dev)
diff --git a/drivers/input/touchscreen/stmfts.c b/drivers/input/touchscreen/stmfts.c
index c175d44c52f3..d5bd170808fb 100644
--- a/drivers/input/touchscreen/stmfts.c
+++ b/drivers/input/touchscreen/stmfts.c
@@ -738,11 +738,9 @@ static int stmfts_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int stmfts_remove(struct i2c_client *client)
+static void stmfts_remove(struct i2c_client *client)
 {
 	pm_runtime_disable(&client->dev);
-
-	return 0;
 }
 
 static int __maybe_unused stmfts_runtime_suspend(struct device *dev)
diff --git a/drivers/input/touchscreen/sun4i-ts.c b/drivers/input/touchscreen/sun4i-ts.c
index 742a7e96c1b5..73eb8f80be6e 100644
--- a/drivers/input/touchscreen/sun4i-ts.c
+++ b/drivers/input/touchscreen/sun4i-ts.c
@@ -192,12 +192,12 @@ static int sun4i_get_temp(const struct sun4i_ts_data *ts, int *temp)
 	return 0;
 }
 
-static int sun4i_get_tz_temp(void *data, int *temp)
+static int sun4i_get_tz_temp(struct thermal_zone_device *tz, int *temp)
 {
-	return sun4i_get_temp(data, temp);
+	return sun4i_get_temp(tz->devdata, temp);
 }
 
-static const struct thermal_zone_of_device_ops sun4i_ts_tz_ops = {
+static const struct thermal_zone_device_ops sun4i_ts_tz_ops = {
 	.get_temp = sun4i_get_tz_temp,
 };
 
@@ -356,8 +356,8 @@ static int sun4i_ts_probe(struct platform_device *pdev)
 	if (IS_ERR(hwmon))
 		return PTR_ERR(hwmon);
 
-	thermal = devm_thermal_zone_of_sensor_register(ts->dev, 0, ts,
-						       &sun4i_ts_tz_ops);
+	thermal = devm_thermal_of_zone_register(ts->dev, 0, ts,
+						&sun4i_ts_tz_ops);
 	if (IS_ERR(thermal))
 		return PTR_ERR(thermal);
 
diff --git a/drivers/input/touchscreen/tsc2004.c b/drivers/input/touchscreen/tsc2004.c
index 9fdd870c4c0b..a9565353ee98 100644
--- a/drivers/input/touchscreen/tsc2004.c
+++ b/drivers/input/touchscreen/tsc2004.c
@@ -43,11 +43,9 @@ static int tsc2004_probe(struct i2c_client *i2c,
 			     tsc2004_cmd);
 }
 
-static int tsc2004_remove(struct i2c_client *i2c)
+static void tsc2004_remove(struct i2c_client *i2c)
 {
 	tsc200x_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id tsc2004_idtable[] = {
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
index 1ef7bbb4acf3..5968a568aae2 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
@@ -150,7 +150,7 @@ static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm)
 	}
 
 	reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
-	par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_PARANGE_SHIFT);
+	par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT);
 	tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par);
 
 	cd->ttbr = virt_to_phys(mm->pgd);
@@ -425,13 +425,13 @@ bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
 	 * addresses larger than what we support.
 	 */
 	reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
-	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_PARANGE_SHIFT);
+	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT);
 	oas = id_aa64mmfr0_parange_to_phys_shift(fld);
 	if (smmu->oas < oas)
 		return false;
 
 	/* We can support bigger ASIDs than the CPU, but not smaller */
-	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_ASID_SHIFT);
+	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT);
 	asid_bits = fld ? 16 : 8;
 	if (smmu->asid_bits < asid_bits)
 		return false;
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index 66b9fa408bf2..eb5ea5b69cfa 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -561,6 +561,11 @@ config IRQ_LOONGARCH_CPU
 	select GENERIC_IRQ_CHIP
 	select IRQ_DOMAIN
 	select GENERIC_IRQ_EFFECTIVE_AFF_MASK
+	select LOONGSON_LIOINTC
+	select LOONGSON_EIOINTC
+	select LOONGSON_PCH_PIC
+	select LOONGSON_PCH_MSI
+	select LOONGSON_PCH_LPC
 	help
 	  Support for the LoongArch CPU Interrupt Controller. For details of
 	  irq chip hierarchy on LoongArch platforms please read the document
@@ -623,8 +628,9 @@ config LOONGSON_PCH_MSI
 
 config LOONGSON_PCH_LPC
 	bool "Loongson PCH LPC Controller"
+	depends on LOONGARCH
 	depends on MACH_LOONGSON64
-	default (MACH_LOONGSON64 && LOONGARCH)
+	default MACH_LOONGSON64
 	select IRQ_DOMAIN_HIERARCHY
 	help
 	  Support for the Loongson PCH LPC Controller.
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 5ff09de6c48f..beead1a0191c 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -1574,13 +1574,15 @@ static int its_select_cpu(struct irq_data *d,
 			  const struct cpumask *aff_mask)
 {
 	struct its_device *its_dev = irq_data_get_irq_chip_data(d);
-	cpumask_var_t tmpmask;
+	static DEFINE_RAW_SPINLOCK(tmpmask_lock);
+	static struct cpumask __tmpmask;
+	struct cpumask *tmpmask;
+	unsigned long flags;
 	int cpu, node;
-
-	if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
-		return -ENOMEM;
-
 	node = its_dev->its->numa_node;
+	tmpmask = &__tmpmask;
+
+	raw_spin_lock_irqsave(&tmpmask_lock, flags);
 
 	if (!irqd_affinity_is_managed(d)) {
 		/* First try the NUMA node */
@@ -1634,7 +1636,7 @@ static int its_select_cpu(struct irq_data *d,
 		cpu = cpumask_pick_least_loaded(d, tmpmask);
 	}
 out:
-	free_cpumask_var(tmpmask);
+	raw_spin_unlock_irqrestore(&tmpmask_lock, flags);
 
 	pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu);
 	return cpu;
diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c
index 4ea71b28f9f5..a6277dea4c7a 100644
--- a/drivers/irqchip/irq-gic-v4.c
+++ b/drivers/irqchip/irq-gic-v4.c
@@ -94,7 +94,7 @@ bool gic_cpuif_has_vsgi(void)
 {
 	unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
 
-	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_GIC_SHIFT);
+	fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_EL1_GIC_SHIFT);
 
 	return fld >= 0x3;
 }
diff --git a/drivers/irqchip/irq-stm32-exti.c b/drivers/irqchip/irq-stm32-exti.c
index a73763d475f0..6a3f7498ea8e 100644
--- a/drivers/irqchip/irq-stm32-exti.c
+++ b/drivers/irqchip/irq-stm32-exti.c
@@ -716,7 +716,7 @@ static int stm32_exti_h_domain_alloc(struct irq_domain *dm,
 
 	irq_domain_set_hwirq_and_chip(dm, virq, hwirq, chip, chip_data);
 
-	if (!host_data->drv_data || !host_data->drv_data->desc_irqs)
+	if (!host_data->drv_data->desc_irqs)
 		return -EINVAL;
 
 	desc_irq = host_data->drv_data->desc_irqs[hwirq];
diff --git a/drivers/isdn/capi/kcapi.c b/drivers/isdn/capi/kcapi.c
index e69c4bf557bf..ae24848af233 100644
--- a/drivers/isdn/capi/kcapi.c
+++ b/drivers/isdn/capi/kcapi.c
@@ -798,7 +798,7 @@ u16 capi20_get_serial(u32 contr, u8 serial[CAPI_SERIAL_LEN])
 	u16 ret;
 
 	if (contr == 0) {
-		strlcpy(serial, driver_serial, CAPI_SERIAL_LEN);
+		strscpy(serial, driver_serial, CAPI_SERIAL_LEN);
 		return CAPI_NOERROR;
 	}
 
@@ -806,7 +806,7 @@ u16 capi20_get_serial(u32 contr, u8 serial[CAPI_SERIAL_LEN])
 
 	ctr = get_capi_ctr_by_nr(contr);
 	if (ctr && ctr->state == CAPI_CTR_RUNNING) {
-		strlcpy(serial, ctr->serial, CAPI_SERIAL_LEN);
+		strscpy(serial, ctr->serial, CAPI_SERIAL_LEN);
 		ret = CAPI_NOERROR;
 	} else
 		ret = CAPI_REGNOTINSTALLED;
diff --git a/drivers/isdn/mISDN/l1oip.h b/drivers/isdn/mISDN/l1oip.h
index 7ea10db20e3a..48133d022812 100644
--- a/drivers/isdn/mISDN/l1oip.h
+++ b/drivers/isdn/mISDN/l1oip.h
@@ -59,6 +59,7 @@ struct l1oip {
 	int			bundle;		/* bundle channels in one frm */
 	int			codec;		/* codec to use for transmis. */
 	int			limit;		/* limit number of bchannels */
+	bool			shutdown;	/* if card is released */
 
 	/* timer */
 	struct timer_list	keep_tl;
diff --git a/drivers/isdn/mISDN/l1oip_core.c b/drivers/isdn/mISDN/l1oip_core.c
index 2c40412466e6..a77195e378b7 100644
--- a/drivers/isdn/mISDN/l1oip_core.c
+++ b/drivers/isdn/mISDN/l1oip_core.c
@@ -275,7 +275,7 @@ l1oip_socket_send(struct l1oip *hc, u8 localcodec, u8 channel, u32 chanmask,
 	p = frame;
 
 	/* restart timer */
-	if (time_before(hc->keep_tl.expires, jiffies + 5 * HZ))
+	if (time_before(hc->keep_tl.expires, jiffies + 5 * HZ) && !hc->shutdown)
 		mod_timer(&hc->keep_tl, jiffies + L1OIP_KEEPALIVE * HZ);
 	else
 		hc->keep_tl.expires = jiffies + L1OIP_KEEPALIVE * HZ;
@@ -601,7 +601,9 @@ multiframe:
 		goto multiframe;
 
 	/* restart timer */
-	if (time_before(hc->timeout_tl.expires, jiffies + 5 * HZ) || !hc->timeout_on) {
+	if ((time_before(hc->timeout_tl.expires, jiffies + 5 * HZ) ||
+	     !hc->timeout_on) &&
+	    !hc->shutdown) {
 		hc->timeout_on = 1;
 		mod_timer(&hc->timeout_tl, jiffies + L1OIP_TIMEOUT * HZ);
 	} else /* only adjust timer */
@@ -1232,11 +1234,10 @@ release_card(struct l1oip *hc)
 {
 	int	ch;
 
-	if (timer_pending(&hc->keep_tl))
-		del_timer(&hc->keep_tl);
+	hc->shutdown = true;
 
-	if (timer_pending(&hc->timeout_tl))
-		del_timer(&hc->timeout_tl);
+	del_timer_sync(&hc->keep_tl);
+	del_timer_sync(&hc->timeout_tl);
 
 	cancel_work_sync(&hc->workq);
 
diff --git a/drivers/leds/flash/leds-as3645a.c b/drivers/leds/flash/leds-as3645a.c
index aa3f82be0a9c..bb2249771acb 100644
--- a/drivers/leds/flash/leds-as3645a.c
+++ b/drivers/leds/flash/leds-as3645a.c
@@ -724,7 +724,7 @@ out_put_nodes:
 	return rval;
 }
 
-static int as3645a_remove(struct i2c_client *client)
+static void as3645a_remove(struct i2c_client *client)
 {
 	struct as3645a *flash = i2c_get_clientdata(client);
 
@@ -740,8 +740,6 @@ static int as3645a_remove(struct i2c_client *client)
 
 	fwnode_handle_put(flash->flash_node);
 	fwnode_handle_put(flash->indicator_node);
-
-	return 0;
 }
 
 static const struct i2c_device_id as3645a_id_table[] = {
diff --git a/drivers/leds/flash/leds-lm3601x.c b/drivers/leds/flash/leds-lm3601x.c
index d0e1d4814042..78730e066a73 100644
--- a/drivers/leds/flash/leds-lm3601x.c
+++ b/drivers/leds/flash/leds-lm3601x.c
@@ -440,15 +440,16 @@ static int lm3601x_probe(struct i2c_client *client)
 	return lm3601x_register_leds(led, fwnode);
 }
 
-static int lm3601x_remove(struct i2c_client *client)
+static void lm3601x_remove(struct i2c_client *client)
 {
 	struct lm3601x_led *led = i2c_get_clientdata(client);
+	int ret;
 
-	mutex_destroy(&led->lock);
-
-	return regmap_update_bits(led->regmap, LM3601X_ENABLE_REG,
-			   LM3601X_ENABLE_MASK,
-			   LM3601X_MODE_STANDBY);
+	ret = regmap_update_bits(led->regmap, LM3601X_ENABLE_REG,
+				 LM3601X_ENABLE_MASK, LM3601X_MODE_STANDBY);
+	if (ret)
+		dev_warn(&client->dev,
+			 "Failed to put into standby (%pe)\n", ERR_PTR(ret));
 }
 
 static const struct i2c_device_id lm3601x_id[] = {
diff --git a/drivers/leds/flash/leds-rt4505.c b/drivers/leds/flash/leds-rt4505.c
index ee129ab7255d..e404fe8b0314 100644
--- a/drivers/leds/flash/leds-rt4505.c
+++ b/drivers/leds/flash/leds-rt4505.c
@@ -393,12 +393,11 @@ static int rt4505_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int rt4505_remove(struct i2c_client *client)
+static void rt4505_remove(struct i2c_client *client)
 {
 	struct rt4505_priv *priv = i2c_get_clientdata(client);
 
 	v4l2_flash_release(priv->v4l2_flash);
-	return 0;
 }
 
 static void rt4505_shutdown(struct i2c_client *client)
diff --git a/drivers/leds/leds-an30259a.c b/drivers/leds/leds-an30259a.c
index a0df1fb28774..e072ee5409f7 100644
--- a/drivers/leds/leds-an30259a.c
+++ b/drivers/leds/leds-an30259a.c
@@ -334,13 +334,11 @@ exit:
 	return err;
 }
 
-static int an30259a_remove(struct i2c_client *client)
+static void an30259a_remove(struct i2c_client *client)
 {
 	struct an30259a *chip = i2c_get_clientdata(client);
 
 	mutex_destroy(&chip->mutex);
-
-	return 0;
 }
 
 static const struct of_device_id an30259a_match_table[] = {
diff --git a/drivers/leds/leds-aw2013.c b/drivers/leds/leds-aw2013.c
index 80d937454aee..0b52fc9097c6 100644
--- a/drivers/leds/leds-aw2013.c
+++ b/drivers/leds/leds-aw2013.c
@@ -401,15 +401,13 @@ error:
 	return ret;
 }
 
-static int aw2013_remove(struct i2c_client *client)
+static void aw2013_remove(struct i2c_client *client)
 {
 	struct aw2013 *chip = i2c_get_clientdata(client);
 
 	aw2013_chip_disable(chip);
 
 	mutex_destroy(&chip->mutex);
-
-	return 0;
 }
 
 static const struct of_device_id aw2013_match_table[] = {
diff --git a/drivers/leds/leds-bd2802.c b/drivers/leds/leds-bd2802.c
index 8bbaef5a2986..2b6678f6bd56 100644
--- a/drivers/leds/leds-bd2802.c
+++ b/drivers/leds/leds-bd2802.c
@@ -722,7 +722,7 @@ failed_unregister_dev_file:
 	return ret;
 }
 
-static int bd2802_remove(struct i2c_client *client)
+static void bd2802_remove(struct i2c_client *client)
 {
 	struct bd2802_led *led = i2c_get_clientdata(client);
 	int i;
@@ -733,8 +733,6 @@ static int bd2802_remove(struct i2c_client *client)
 		bd2802_disable_adv_conf(led);
 	for (i = 0; i < ARRAY_SIZE(bd2802_attributes); i++)
 		device_remove_file(&led->client->dev, bd2802_attributes[i]);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/leds/leds-blinkm.c b/drivers/leds/leds-blinkm.c
index bd7d0d5cf3b6..3fb6a2fdaefa 100644
--- a/drivers/leds/leds-blinkm.c
+++ b/drivers/leds/leds-blinkm.c
@@ -677,7 +677,7 @@ exit:
 	return err;
 }
 
-static int blinkm_remove(struct i2c_client *client)
+static void blinkm_remove(struct i2c_client *client)
 {
 	struct blinkm_data *data = i2c_get_clientdata(client);
 	int ret = 0;
@@ -716,7 +716,6 @@ static int blinkm_remove(struct i2c_client *client)
 		dev_err(&client->dev, "Failure in blinkm_remove ignored. Continuing.\n");
 
 	sysfs_remove_group(&client->dev.kobj, &blinkm_group);
-	return 0;
 }
 
 static const struct i2c_device_id blinkm_id[] = {
diff --git a/drivers/leds/leds-is31fl32xx.c b/drivers/leds/leds-is31fl32xx.c
index fc63fce38c19..0d219c1ac3b5 100644
--- a/drivers/leds/leds-is31fl32xx.c
+++ b/drivers/leds/leds-is31fl32xx.c
@@ -457,7 +457,7 @@ static int is31fl32xx_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int is31fl32xx_remove(struct i2c_client *client)
+static void is31fl32xx_remove(struct i2c_client *client)
 {
 	struct is31fl32xx_priv *priv = i2c_get_clientdata(client);
 	int ret;
@@ -466,8 +466,6 @@ static int is31fl32xx_remove(struct i2c_client *client)
 	if (ret)
 		dev_err(&client->dev, "Failed to reset registers on removal (%pe)\n",
 			ERR_PTR(ret));
-
-	return 0;
 }
 
 /*
diff --git a/drivers/leds/leds-lm3530.c b/drivers/leds/leds-lm3530.c
index e72393534b72..ba906c253c7f 100644
--- a/drivers/leds/leds-lm3530.c
+++ b/drivers/leds/leds-lm3530.c
@@ -470,13 +470,12 @@ static int lm3530_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lm3530_remove(struct i2c_client *client)
+static void lm3530_remove(struct i2c_client *client)
 {
 	struct lm3530_data *drvdata = i2c_get_clientdata(client);
 
 	lm3530_led_disable(drvdata);
 	led_classdev_unregister(&drvdata->led_dev);
-	return 0;
 }
 
 static const struct i2c_device_id lm3530_id[] = {
diff --git a/drivers/leds/leds-lm3532.c b/drivers/leds/leds-lm3532.c
index beb53040e09e..db64d44bcbbf 100644
--- a/drivers/leds/leds-lm3532.c
+++ b/drivers/leds/leds-lm3532.c
@@ -704,7 +704,7 @@ static int lm3532_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int lm3532_remove(struct i2c_client *client)
+static void lm3532_remove(struct i2c_client *client)
 {
 	struct lm3532_data *drvdata = i2c_get_clientdata(client);
 
@@ -712,8 +712,6 @@ static int lm3532_remove(struct i2c_client *client)
 
 	if (drvdata->enable_gpio)
 		gpiod_direction_output(drvdata->enable_gpio, 0);
-
-	return 0;
 }
 
 static const struct of_device_id of_lm3532_leds_match[] = {
diff --git a/drivers/leds/leds-lm355x.c b/drivers/leds/leds-lm355x.c
index 2d3e11845ba5..daa35927b301 100644
--- a/drivers/leds/leds-lm355x.c
+++ b/drivers/leds/leds-lm355x.c
@@ -491,7 +491,7 @@ err_out:
 	return err;
 }
 
-static int lm355x_remove(struct i2c_client *client)
+static void lm355x_remove(struct i2c_client *client)
 {
 	struct lm355x_chip_data *chip = i2c_get_clientdata(client);
 	struct lm355x_reg_data *preg = chip->regs;
@@ -501,8 +501,6 @@ static int lm355x_remove(struct i2c_client *client)
 	led_classdev_unregister(&chip->cdev_torch);
 	led_classdev_unregister(&chip->cdev_flash);
 	dev_info(&client->dev, "%s is removed\n", lm355x_name[chip->type]);
-
-	return 0;
 }
 
 static const struct i2c_device_id lm355x_id[] = {
diff --git a/drivers/leds/leds-lm3642.c b/drivers/leds/leds-lm3642.c
index 435309154e6b..428a5d928150 100644
--- a/drivers/leds/leds-lm3642.c
+++ b/drivers/leds/leds-lm3642.c
@@ -380,7 +380,7 @@ err_out:
 	return err;
 }
 
-static int lm3642_remove(struct i2c_client *client)
+static void lm3642_remove(struct i2c_client *client)
 {
 	struct lm3642_chip_data *chip = i2c_get_clientdata(client);
 
@@ -388,7 +388,6 @@ static int lm3642_remove(struct i2c_client *client)
 	led_classdev_unregister(&chip->cdev_torch);
 	led_classdev_unregister(&chip->cdev_flash);
 	regmap_write(chip->regmap, REG_ENABLE, 0);
-	return 0;
 }
 
 static const struct i2c_device_id lm3642_id[] = {
diff --git a/drivers/leds/leds-lm3692x.c b/drivers/leds/leds-lm3692x.c
index 87cd24ce3f95..54b4662bff41 100644
--- a/drivers/leds/leds-lm3692x.c
+++ b/drivers/leds/leds-lm3692x.c
@@ -491,14 +491,12 @@ static int lm3692x_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lm3692x_remove(struct i2c_client *client)
+static void lm3692x_remove(struct i2c_client *client)
 {
 	struct lm3692x_led *led = i2c_get_clientdata(client);
 
 	lm3692x_leds_disable(led);
 	mutex_destroy(&led->lock);
-
-	return 0;
 }
 
 static const struct i2c_device_id lm3692x_id[] = {
diff --git a/drivers/leds/leds-lm3697.c b/drivers/leds/leds-lm3697.c
index a8c9322558cc..71231a60eebc 100644
--- a/drivers/leds/leds-lm3697.c
+++ b/drivers/leds/leds-lm3697.c
@@ -337,7 +337,7 @@ static int lm3697_probe(struct i2c_client *client,
 	return lm3697_init(led);
 }
 
-static int lm3697_remove(struct i2c_client *client)
+static void lm3697_remove(struct i2c_client *client)
 {
 	struct lm3697 *led = i2c_get_clientdata(client);
 	struct device *dev = &led->client->dev;
@@ -345,10 +345,8 @@ static int lm3697_remove(struct i2c_client *client)
 
 	ret = regmap_update_bits(led->regmap, LM3697_CTRL_ENABLE,
 				 LM3697_CTRL_A_B_EN, 0);
-	if (ret) {
+	if (ret)
 		dev_err(dev, "Failed to disable the device\n");
-		return ret;
-	}
 
 	if (led->enable_gpio)
 		gpiod_direction_output(led->enable_gpio, 0);
@@ -360,8 +358,6 @@ static int lm3697_remove(struct i2c_client *client)
 	}
 
 	mutex_destroy(&led->lock);
-
-	return 0;
 }
 
 static const struct i2c_device_id lm3697_id[] = {
diff --git a/drivers/leds/leds-lp3944.c b/drivers/leds/leds-lp3944.c
index 437c711b2a27..673ad8c04f41 100644
--- a/drivers/leds/leds-lp3944.c
+++ b/drivers/leds/leds-lp3944.c
@@ -397,7 +397,7 @@ static int lp3944_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lp3944_remove(struct i2c_client *client)
+static void lp3944_remove(struct i2c_client *client)
 {
 	struct lp3944_platform_data *pdata = dev_get_platdata(&client->dev);
 	struct lp3944_data *data = i2c_get_clientdata(client);
@@ -414,8 +414,6 @@ static int lp3944_remove(struct i2c_client *client)
 		default:
 			break;
 		}
-
-	return 0;
 }
 
 /* lp3944 i2c driver struct */
diff --git a/drivers/leds/leds-lp3952.c b/drivers/leds/leds-lp3952.c
index 6ee9131fbf25..bf0ad1b5ce24 100644
--- a/drivers/leds/leds-lp3952.c
+++ b/drivers/leds/leds-lp3952.c
@@ -255,15 +255,13 @@ static int lp3952_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lp3952_remove(struct i2c_client *client)
+static void lp3952_remove(struct i2c_client *client)
 {
 	struct lp3952_led_array *priv;
 
 	priv = i2c_get_clientdata(client);
 	lp3952_on_off(priv, LP3952_LED_ALL, false);
 	gpiod_set_value(priv->enable_gpio, 0);
-
-	return 0;
 }
 
 static const struct i2c_device_id lp3952_id[] = {
diff --git a/drivers/leds/leds-lp50xx.c b/drivers/leds/leds-lp50xx.c
index e129dcc656b8..28d6b39fa72d 100644
--- a/drivers/leds/leds-lp50xx.c
+++ b/drivers/leds/leds-lp50xx.c
@@ -563,7 +563,7 @@ static int lp50xx_probe(struct i2c_client *client)
 	return lp50xx_probe_dt(led);
 }
 
-static int lp50xx_remove(struct i2c_client *client)
+static void lp50xx_remove(struct i2c_client *client)
 {
 	struct lp50xx *led = i2c_get_clientdata(client);
 	int ret;
@@ -579,8 +579,6 @@ static int lp50xx_remove(struct i2c_client *client)
 	}
 
 	mutex_destroy(&led->lock);
-
-	return 0;
 }
 
 static const struct i2c_device_id lp50xx_id[] = {
diff --git a/drivers/leds/leds-lp5521.c b/drivers/leds/leds-lp5521.c
index a9e7507c998c..7ff20c260504 100644
--- a/drivers/leds/leds-lp5521.c
+++ b/drivers/leds/leds-lp5521.c
@@ -579,7 +579,7 @@ err_init:
 	return ret;
 }
 
-static int lp5521_remove(struct i2c_client *client)
+static void lp5521_remove(struct i2c_client *client)
 {
 	struct lp55xx_led *led = i2c_get_clientdata(client);
 	struct lp55xx_chip *chip = led->chip;
@@ -587,8 +587,6 @@ static int lp5521_remove(struct i2c_client *client)
 	lp5521_stop_all_engines(chip);
 	lp55xx_unregister_sysfs(chip);
 	lp55xx_deinit_device(chip);
-
-	return 0;
 }
 
 static const struct i2c_device_id lp5521_id[] = {
diff --git a/drivers/leds/leds-lp5523.c b/drivers/leds/leds-lp5523.c
index b1590cb4a188..369d40b0b65b 100644
--- a/drivers/leds/leds-lp5523.c
+++ b/drivers/leds/leds-lp5523.c
@@ -947,7 +947,7 @@ err_init:
 	return ret;
 }
 
-static int lp5523_remove(struct i2c_client *client)
+static void lp5523_remove(struct i2c_client *client)
 {
 	struct lp55xx_led *led = i2c_get_clientdata(client);
 	struct lp55xx_chip *chip = led->chip;
@@ -955,8 +955,6 @@ static int lp5523_remove(struct i2c_client *client)
 	lp5523_stop_all_engines(chip);
 	lp55xx_unregister_sysfs(chip);
 	lp55xx_deinit_device(chip);
-
-	return 0;
 }
 
 static const struct i2c_device_id lp5523_id[] = {
diff --git a/drivers/leds/leds-lp5562.c b/drivers/leds/leds-lp5562.c
index 31c14016d289..0e490085ff35 100644
--- a/drivers/leds/leds-lp5562.c
+++ b/drivers/leds/leds-lp5562.c
@@ -573,7 +573,7 @@ err_init:
 	return ret;
 }
 
-static int lp5562_remove(struct i2c_client *client)
+static void lp5562_remove(struct i2c_client *client)
 {
 	struct lp55xx_led *led = i2c_get_clientdata(client);
 	struct lp55xx_chip *chip = led->chip;
@@ -582,8 +582,6 @@ static int lp5562_remove(struct i2c_client *client)
 
 	lp55xx_unregister_sysfs(chip);
 	lp55xx_deinit_device(chip);
-
-	return 0;
 }
 
 static const struct i2c_device_id lp5562_id[] = {
diff --git a/drivers/leds/leds-lp8501.c b/drivers/leds/leds-lp8501.c
index 2d2fda2ab104..ae11a02c0ab2 100644
--- a/drivers/leds/leds-lp8501.c
+++ b/drivers/leds/leds-lp8501.c
@@ -362,7 +362,7 @@ err_init:
 	return ret;
 }
 
-static int lp8501_remove(struct i2c_client *client)
+static void lp8501_remove(struct i2c_client *client)
 {
 	struct lp55xx_led *led = i2c_get_clientdata(client);
 	struct lp55xx_chip *chip = led->chip;
@@ -370,8 +370,6 @@ static int lp8501_remove(struct i2c_client *client)
 	lp8501_stop_engine(chip);
 	lp55xx_unregister_sysfs(chip);
 	lp55xx_deinit_device(chip);
-
-	return 0;
 }
 
 static const struct i2c_device_id lp8501_id[] = {
diff --git a/drivers/leds/leds-lp8860.c b/drivers/leds/leds-lp8860.c
index 3c693d5e3b44..e2b36d3187eb 100644
--- a/drivers/leds/leds-lp8860.c
+++ b/drivers/leds/leds-lp8860.c
@@ -445,7 +445,7 @@ static int lp8860_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lp8860_remove(struct i2c_client *client)
+static void lp8860_remove(struct i2c_client *client)
 {
 	struct lp8860_led *led = i2c_get_clientdata(client);
 	int ret;
@@ -461,8 +461,6 @@ static int lp8860_remove(struct i2c_client *client)
 	}
 
 	mutex_destroy(&led->lock);
-
-	return 0;
 }
 
 static const struct i2c_device_id lp8860_id[] = {
diff --git a/drivers/leds/leds-pca9532.c b/drivers/leds/leds-pca9532.c
index f72b5d1be3a6..df83d97cb479 100644
--- a/drivers/leds/leds-pca9532.c
+++ b/drivers/leds/leds-pca9532.c
@@ -52,7 +52,7 @@ struct pca9532_data {
 
 static int pca9532_probe(struct i2c_client *client,
 	const struct i2c_device_id *id);
-static int pca9532_remove(struct i2c_client *client);
+static void pca9532_remove(struct i2c_client *client);
 
 enum {
 	pca9530,
@@ -546,13 +546,11 @@ static int pca9532_probe(struct i2c_client *client,
 	return pca9532_configure(client, data, pca9532_pdata);
 }
 
-static int pca9532_remove(struct i2c_client *client)
+static void pca9532_remove(struct i2c_client *client)
 {
 	struct pca9532_data *data = i2c_get_clientdata(client);
 
 	pca9532_destroy_devices(data, data->chip_info->num_leds);
-
-	return 0;
 }
 
 module_i2c_driver(pca9532_driver);
diff --git a/drivers/leds/leds-tca6507.c b/drivers/leds/leds-tca6507.c
index 1473ced8664c..161bef65c6b7 100644
--- a/drivers/leds/leds-tca6507.c
+++ b/drivers/leds/leds-tca6507.c
@@ -790,7 +790,7 @@ exit:
 	return err;
 }
 
-static int tca6507_remove(struct i2c_client *client)
+static void tca6507_remove(struct i2c_client *client)
 {
 	int i;
 	struct tca6507_chip *tca = i2c_get_clientdata(client);
@@ -802,8 +802,6 @@ static int tca6507_remove(struct i2c_client *client)
 	}
 	tca6507_remove_gpio(tca);
 	cancel_work_sync(&tca->work);
-
-	return 0;
 }
 
 static struct i2c_driver tca6507_driver = {
diff --git a/drivers/leds/leds-turris-omnia.c b/drivers/leds/leds-turris-omnia.c
index eac6f4a573b2..c7c9851c894a 100644
--- a/drivers/leds/leds-turris-omnia.c
+++ b/drivers/leds/leds-turris-omnia.c
@@ -242,7 +242,7 @@ static int omnia_leds_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int omnia_leds_remove(struct i2c_client *client)
+static void omnia_leds_remove(struct i2c_client *client)
 {
 	u8 buf[5];
 
@@ -258,8 +258,6 @@ static int omnia_leds_remove(struct i2c_client *client)
 	buf[4] = 255;
 
 	i2c_master_send(client, buf, 5);
-
-	return 0;
 }
 
 static const struct of_device_id of_omnia_leds_match[] = {
diff --git a/drivers/leds/simple/simatic-ipc-leds-gpio.c b/drivers/leds/simple/simatic-ipc-leds-gpio.c
index 4c9e663a90ba..b9eeb8702df0 100644
--- a/drivers/leds/simple/simatic-ipc-leds-gpio.c
+++ b/drivers/leds/simple/simatic-ipc-leds-gpio.c
@@ -13,28 +13,45 @@
 #include <linux/leds.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/platform_data/x86/simatic-ipc-base.h>
 
-static struct gpiod_lookup_table simatic_ipc_led_gpio_table = {
+static struct gpiod_lookup_table *simatic_ipc_led_gpio_table;
+
+static struct gpiod_lookup_table simatic_ipc_led_gpio_table_127e = {
 	.dev_id = "leds-gpio",
 	.table = {
-		GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 51, NULL, 0, GPIO_ACTIVE_LOW),
 		GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 52, NULL, 1, GPIO_ACTIVE_LOW),
 		GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 53, NULL, 2, GPIO_ACTIVE_LOW),
 		GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 57, NULL, 3, GPIO_ACTIVE_LOW),
 		GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 58, NULL, 4, GPIO_ACTIVE_LOW),
 		GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 60, NULL, 5, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 51, NULL, 0, GPIO_ACTIVE_LOW),
 		GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 56, NULL, 6, GPIO_ACTIVE_LOW),
 		GPIO_LOOKUP_IDX("apollolake-pinctrl.0", 59, NULL, 7, GPIO_ACTIVE_HIGH),
 	},
 };
 
+static struct gpiod_lookup_table simatic_ipc_led_gpio_table_227g = {
+	.dev_id = "leds-gpio",
+	.table = {
+		GPIO_LOOKUP_IDX("gpio-f7188x-2", 0, NULL, 0, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("gpio-f7188x-2", 1, NULL, 1, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("gpio-f7188x-2", 2, NULL, 2, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("gpio-f7188x-2", 3, NULL, 3, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("gpio-f7188x-2", 4, NULL, 4, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("gpio-f7188x-2", 5, NULL, 5, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("gpio-f7188x-3", 6, NULL, 6, GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP_IDX("gpio-f7188x-3", 7, NULL, 7, GPIO_ACTIVE_HIGH),
+	}
+};
+
 static const struct gpio_led simatic_ipc_gpio_leds[] = {
-	{ .name = "green:" LED_FUNCTION_STATUS "-3" },
 	{ .name = "red:" LED_FUNCTION_STATUS "-1" },
 	{ .name = "green:" LED_FUNCTION_STATUS "-1" },
 	{ .name = "red:" LED_FUNCTION_STATUS "-2" },
 	{ .name = "green:" LED_FUNCTION_STATUS "-2" },
 	{ .name = "red:" LED_FUNCTION_STATUS "-3" },
+	{ .name = "green:" LED_FUNCTION_STATUS "-3" },
 };
 
 static const struct gpio_led_platform_data simatic_ipc_gpio_leds_pdata = {
@@ -46,7 +63,7 @@ static struct platform_device *simatic_leds_pdev;
 
 static int simatic_ipc_leds_gpio_remove(struct platform_device *pdev)
 {
-	gpiod_remove_lookup_table(&simatic_ipc_led_gpio_table);
+	gpiod_remove_lookup_table(simatic_ipc_led_gpio_table);
 	platform_device_unregister(simatic_leds_pdev);
 
 	return 0;
@@ -54,10 +71,25 @@ static int simatic_ipc_leds_gpio_remove(struct platform_device *pdev)
 
 static int simatic_ipc_leds_gpio_probe(struct platform_device *pdev)
 {
+	const struct simatic_ipc_platform *plat = pdev->dev.platform_data;
 	struct gpio_desc *gpiod;
 	int err;
 
-	gpiod_add_lookup_table(&simatic_ipc_led_gpio_table);
+	switch (plat->devmode) {
+	case SIMATIC_IPC_DEVICE_127E:
+		simatic_ipc_led_gpio_table = &simatic_ipc_led_gpio_table_127e;
+		break;
+	case SIMATIC_IPC_DEVICE_227G:
+		if (!IS_ENABLED(CONFIG_GPIO_F7188X))
+			return -ENODEV;
+		request_module("gpio-f7188x");
+		simatic_ipc_led_gpio_table = &simatic_ipc_led_gpio_table_227g;
+		break;
+	default:
+		return -ENODEV;
+	}
+
+	gpiod_add_lookup_table(simatic_ipc_led_gpio_table);
 	simatic_leds_pdev = platform_device_register_resndata(NULL,
 		"leds-gpio", PLATFORM_DEVID_NONE, NULL, 0,
 		&simatic_ipc_gpio_leds_pdata,
diff --git a/drivers/macintosh/ams/ams-i2c.c b/drivers/macintosh/ams/ams-i2c.c
index d2f0cde6f9c7..3ded340699fb 100644
--- a/drivers/macintosh/ams/ams-i2c.c
+++ b/drivers/macintosh/ams/ams-i2c.c
@@ -58,7 +58,7 @@ enum ams_i2c_cmd {
 
 static int ams_i2c_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id);
-static int ams_i2c_remove(struct i2c_client *client);
+static void ams_i2c_remove(struct i2c_client *client);
 
 static const struct i2c_device_id ams_id[] = {
 	{ "MAC,accelerometer_1", 0 },
@@ -230,7 +230,7 @@ static int ams_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int ams_i2c_remove(struct i2c_client *client)
+static void ams_i2c_remove(struct i2c_client *client)
 {
 	if (ams_info.has_device) {
 		ams_sensor_detach();
@@ -245,8 +245,6 @@ static int ams_i2c_remove(struct i2c_client *client)
 
 		ams_info.has_device = 0;
 	}
-
-	return 0;
 }
 
 static void ams_i2c_exit(void)
diff --git a/drivers/macintosh/therm_adt746x.c b/drivers/macintosh/therm_adt746x.c
index e604cbc91763..b004ea2a1102 100644
--- a/drivers/macintosh/therm_adt746x.c
+++ b/drivers/macintosh/therm_adt746x.c
@@ -563,7 +563,7 @@ static int probe_thermostat(struct i2c_client *client,
 	return 0;
 }
 
-static int remove_thermostat(struct i2c_client *client)
+static void remove_thermostat(struct i2c_client *client)
 {
 	struct thermostat *th = i2c_get_clientdata(client);
 	int i;
@@ -585,8 +585,6 @@ static int remove_thermostat(struct i2c_client *client)
 	write_both_fan_speed(th, -1);
 
 	kfree(th);
-
-	return 0;
 }
 
 static const struct i2c_device_id therm_adt746x_id[] = {
diff --git a/drivers/macintosh/therm_windtunnel.c b/drivers/macintosh/therm_windtunnel.c
index 9226b74fa08f..61fe2ab910b8 100644
--- a/drivers/macintosh/therm_windtunnel.c
+++ b/drivers/macintosh/therm_windtunnel.c
@@ -334,7 +334,7 @@ static void do_attach(struct i2c_adapter *adapter)
 	}
 }
 
-static int
+static void
 do_remove(struct i2c_client *client)
 {
 	if (x.running) {
@@ -348,8 +348,6 @@ do_remove(struct i2c_client *client)
 		x.fan = NULL;
 	else
 		printk(KERN_ERR "g4fan: bad client\n");
-
-	return 0;
 }
 
 static int
diff --git a/drivers/macintosh/windfarm_ad7417_sensor.c b/drivers/macintosh/windfarm_ad7417_sensor.c
index 6ad6441abcbc..c5c54a4ce91f 100644
--- a/drivers/macintosh/windfarm_ad7417_sensor.c
+++ b/drivers/macintosh/windfarm_ad7417_sensor.c
@@ -289,7 +289,7 @@ static int wf_ad7417_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int wf_ad7417_remove(struct i2c_client *client)
+static void wf_ad7417_remove(struct i2c_client *client)
 {
 	struct wf_ad7417_priv *pv = dev_get_drvdata(&client->dev);
 	int i;
@@ -302,8 +302,6 @@ static int wf_ad7417_remove(struct i2c_client *client)
 		wf_unregister_sensor(&pv->sensors[i]);
 
 	kref_put(&pv->ref, wf_ad7417_release);
-
-	return 0;
 }
 
 static const struct i2c_device_id wf_ad7417_id[] = {
diff --git a/drivers/macintosh/windfarm_fcu_controls.c b/drivers/macintosh/windfarm_fcu_controls.c
index 82e7b2005ae7..c5b1ca5bcd73 100644
--- a/drivers/macintosh/windfarm_fcu_controls.c
+++ b/drivers/macintosh/windfarm_fcu_controls.c
@@ -560,7 +560,7 @@ static int wf_fcu_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int wf_fcu_remove(struct i2c_client *client)
+static void wf_fcu_remove(struct i2c_client *client)
 {
 	struct wf_fcu_priv *pv = dev_get_drvdata(&client->dev);
 	struct wf_fcu_fan *fan;
@@ -571,7 +571,6 @@ static int wf_fcu_remove(struct i2c_client *client)
 		wf_unregister_control(&fan->ctrl);
 	}
 	kref_put(&pv->ref, wf_fcu_release);
-	return 0;
 }
 
 static const struct i2c_device_id wf_fcu_id[] = {
diff --git a/drivers/macintosh/windfarm_lm75_sensor.c b/drivers/macintosh/windfarm_lm75_sensor.c
index eb7e7f0bd219..204661c8e918 100644
--- a/drivers/macintosh/windfarm_lm75_sensor.c
+++ b/drivers/macintosh/windfarm_lm75_sensor.c
@@ -147,7 +147,7 @@ static int wf_lm75_probe(struct i2c_client *client,
 	return rc;
 }
 
-static int wf_lm75_remove(struct i2c_client *client)
+static void wf_lm75_remove(struct i2c_client *client)
 {
 	struct wf_lm75_sensor *lm = i2c_get_clientdata(client);
 
@@ -156,8 +156,6 @@ static int wf_lm75_remove(struct i2c_client *client)
 
 	/* release sensor */
 	wf_unregister_sensor(&lm->sens);
-
-	return 0;
 }
 
 static const struct i2c_device_id wf_lm75_id[] = {
diff --git a/drivers/macintosh/windfarm_lm87_sensor.c b/drivers/macintosh/windfarm_lm87_sensor.c
index 807efdde86bc..40d25463346e 100644
--- a/drivers/macintosh/windfarm_lm87_sensor.c
+++ b/drivers/macintosh/windfarm_lm87_sensor.c
@@ -145,7 +145,7 @@ static int wf_lm87_probe(struct i2c_client *client,
 	return rc;
 }
 
-static int wf_lm87_remove(struct i2c_client *client)
+static void wf_lm87_remove(struct i2c_client *client)
 {
 	struct wf_lm87_sensor *lm = i2c_get_clientdata(client);
 
@@ -154,8 +154,6 @@ static int wf_lm87_remove(struct i2c_client *client)
 
 	/* release sensor */
 	wf_unregister_sensor(&lm->sens);
-
-	return 0;
 }
 
 static const struct i2c_device_id wf_lm87_id[] = {
diff --git a/drivers/macintosh/windfarm_max6690_sensor.c b/drivers/macintosh/windfarm_max6690_sensor.c
index 55ee417fb878..c0d404ebc792 100644
--- a/drivers/macintosh/windfarm_max6690_sensor.c
+++ b/drivers/macintosh/windfarm_max6690_sensor.c
@@ -104,14 +104,12 @@ static int wf_max6690_probe(struct i2c_client *client,
 	return rc;
 }
 
-static int wf_max6690_remove(struct i2c_client *client)
+static void wf_max6690_remove(struct i2c_client *client)
 {
 	struct wf_6690_sensor *max = i2c_get_clientdata(client);
 
 	max->i2c = NULL;
 	wf_unregister_sensor(&max->sens);
-
-	return 0;
 }
 
 static const struct i2c_device_id wf_max6690_id[] = {
diff --git a/drivers/macintosh/windfarm_smu_sat.c b/drivers/macintosh/windfarm_smu_sat.c
index 5ade627eaa78..be5d4593db93 100644
--- a/drivers/macintosh/windfarm_smu_sat.c
+++ b/drivers/macintosh/windfarm_smu_sat.c
@@ -316,7 +316,7 @@ static int wf_sat_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int wf_sat_remove(struct i2c_client *client)
+static void wf_sat_remove(struct i2c_client *client)
 {
 	struct wf_sat *sat = i2c_get_clientdata(client);
 	struct wf_sat_sensor *sens;
@@ -330,8 +330,6 @@ static int wf_sat_remove(struct i2c_client *client)
 	}
 	sat->i2c = NULL;
 	kref_put(&sat->ref, wf_sat_release);
-
-	return 0;
 }
 
 static const struct i2c_device_id wf_sat_id[] = {
diff --git a/drivers/md/dm-verity-loadpin.c b/drivers/md/dm-verity-loadpin.c
index 387ec43aef72..4f78cc55c251 100644
--- a/drivers/md/dm-verity-loadpin.c
+++ b/drivers/md/dm-verity-loadpin.c
@@ -14,6 +14,7 @@ LIST_HEAD(dm_verity_loadpin_trusted_root_digests);
 
 static bool is_trusted_verity_target(struct dm_target *ti)
 {
+	int verity_mode;
 	u8 *root_digest;
 	unsigned int digest_size;
 	struct dm_verity_loadpin_trusted_root_digest *trd;
@@ -22,6 +23,13 @@ static bool is_trusted_verity_target(struct dm_target *ti)
 	if (!dm_is_verity_target(ti))
 		return false;
 
+	verity_mode = dm_verity_get_mode(ti);
+
+	if ((verity_mode != DM_VERITY_MODE_EIO) &&
+	    (verity_mode != DM_VERITY_MODE_RESTART) &&
+	    (verity_mode != DM_VERITY_MODE_PANIC))
+		return false;
+
 	if (dm_verity_get_root_digest(ti, &root_digest, &digest_size))
 		return false;
 
diff --git a/drivers/md/dm-verity-target.c b/drivers/md/dm-verity-target.c
index 94b6cb599db4..8a00cc42e498 100644
--- a/drivers/md/dm-verity-target.c
+++ b/drivers/md/dm-verity-target.c
@@ -1447,6 +1447,22 @@ bool dm_is_verity_target(struct dm_target *ti)
 }
 
 /*
+ * Get the verity mode (error behavior) of a verity target.
+ *
+ * Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity
+ * target.
+ */
+int dm_verity_get_mode(struct dm_target *ti)
+{
+	struct dm_verity *v = ti->private;
+
+	if (!dm_is_verity_target(ti))
+		return -EINVAL;
+
+	return v->mode;
+}
+
+/*
  * Get the root digest of a verity target.
  *
  * Returns a copy of the root digest, the caller is responsible for
diff --git a/drivers/md/dm-verity.h b/drivers/md/dm-verity.h
index 45455de1b4bc..98f306ec6a33 100644
--- a/drivers/md/dm-verity.h
+++ b/drivers/md/dm-verity.h
@@ -134,6 +134,7 @@ extern int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,
 				 sector_t block, u8 *digest, bool *is_zero);
 
 extern bool dm_is_verity_target(struct dm_target *ti);
+extern int dm_verity_get_mode(struct dm_target *ti);
 extern int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest,
 				     unsigned int *digest_size);
 
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 742b2349fea3..10e0c5381d01 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -876,8 +876,8 @@ static int join(struct mddev *mddev, int nodes)
 	memset(str, 0, 64);
 	sprintf(str, "%pU", mddev->uuid);
 	ret = dlm_new_lockspace(str, mddev->bitmap_info.cluster_name,
-				DLM_LSFL_FS, LVB_SIZE,
-				&md_ls_ops, mddev, &ops_rv, &cinfo->lockspace);
+				0, LVB_SIZE, &md_ls_ops, mddev,
+				&ops_rv, &cinfo->lockspace);
 	if (ret)
 		goto err;
 	wait_for_completion(&cinfo->completion);
diff --git a/drivers/media/cec/i2c/ch7322.c b/drivers/media/cec/i2c/ch7322.c
index 0814338c43e4..34fad7123704 100644
--- a/drivers/media/cec/i2c/ch7322.c
+++ b/drivers/media/cec/i2c/ch7322.c
@@ -565,7 +565,7 @@ err_mutex:
 	return ret;
 }
 
-static int ch7322_remove(struct i2c_client *client)
+static void ch7322_remove(struct i2c_client *client)
 {
 	struct ch7322 *ch7322 = i2c_get_clientdata(client);
 
@@ -578,8 +578,6 @@ static int ch7322_remove(struct i2c_client *client)
 	mutex_destroy(&ch7322->mutex);
 
 	dev_info(&client->dev, "device unregistered\n");
-
-	return 0;
 }
 
 static const struct of_device_id ch7322_of_match[] = {
diff --git a/drivers/media/dvb-core/dvb_vb2.c b/drivers/media/dvb-core/dvb_vb2.c
index a1bd6d9c9223..909df82fed33 100644
--- a/drivers/media/dvb-core/dvb_vb2.c
+++ b/drivers/media/dvb-core/dvb_vb2.c
@@ -354,6 +354,12 @@ int dvb_vb2_reqbufs(struct dvb_vb2_ctx *ctx, struct dmx_requestbuffers *req)
 
 int dvb_vb2_querybuf(struct dvb_vb2_ctx *ctx, struct dmx_buffer *b)
 {
+	struct vb2_queue *q = &ctx->vb_q;
+
+	if (b->index >= q->num_buffers) {
+		dprintk(1, "[%s] buffer index out of range\n", ctx->name);
+		return -EINVAL;
+	}
 	vb2_core_querybuf(&ctx->vb_q, b->index, b);
 	dprintk(3, "[%s] index=%d\n", ctx->name, b->index);
 	return 0;
@@ -378,8 +384,13 @@ int dvb_vb2_expbuf(struct dvb_vb2_ctx *ctx, struct dmx_exportbuffer *exp)
 
 int dvb_vb2_qbuf(struct dvb_vb2_ctx *ctx, struct dmx_buffer *b)
 {
+	struct vb2_queue *q = &ctx->vb_q;
 	int ret;
 
+	if (b->index >= q->num_buffers) {
+		dprintk(1, "[%s] buffer index out of range\n", ctx->name);
+		return -EINVAL;
+	}
 	ret = vb2_core_qbuf(&ctx->vb_q, b->index, b, NULL);
 	if (ret) {
 		dprintk(1, "[%s] index=%d errno=%d\n", ctx->name,
diff --git a/drivers/media/dvb-frontends/a8293.c b/drivers/media/dvb-frontends/a8293.c
index 57f52c004a23..ba38783b2b4f 100644
--- a/drivers/media/dvb-frontends/a8293.c
+++ b/drivers/media/dvb-frontends/a8293.c
@@ -98,14 +98,13 @@ err:
 	return ret;
 }
 
-static int a8293_remove(struct i2c_client *client)
+static void a8293_remove(struct i2c_client *client)
 {
 	struct a8293_dev *dev = i2c_get_clientdata(client);
 
 	dev_dbg(&client->dev, "\n");
 
 	kfree(dev);
-	return 0;
 }
 
 static const struct i2c_device_id a8293_id_table[] = {
diff --git a/drivers/media/dvb-frontends/af9013.c b/drivers/media/dvb-frontends/af9013.c
index 7d7c341b2bd8..d85929582c3f 100644
--- a/drivers/media/dvb-frontends/af9013.c
+++ b/drivers/media/dvb-frontends/af9013.c
@@ -1540,7 +1540,7 @@ err:
 	return ret;
 }
 
-static int af9013_remove(struct i2c_client *client)
+static void af9013_remove(struct i2c_client *client)
 {
 	struct af9013_state *state = i2c_get_clientdata(client);
 
@@ -1551,8 +1551,6 @@ static int af9013_remove(struct i2c_client *client)
 	regmap_exit(state->regmap);
 
 	kfree(state);
-
-	return 0;
 }
 
 static const struct i2c_device_id af9013_id_table[] = {
diff --git a/drivers/media/dvb-frontends/af9033.c b/drivers/media/dvb-frontends/af9033.c
index 785c49b3d307..808da7a9ffe7 100644
--- a/drivers/media/dvb-frontends/af9033.c
+++ b/drivers/media/dvb-frontends/af9033.c
@@ -1163,7 +1163,7 @@ err:
 	return ret;
 }
 
-static int af9033_remove(struct i2c_client *client)
+static void af9033_remove(struct i2c_client *client)
 {
 	struct af9033_dev *dev = i2c_get_clientdata(client);
 
@@ -1171,8 +1171,6 @@ static int af9033_remove(struct i2c_client *client)
 
 	regmap_exit(dev->regmap);
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id af9033_id_table[] = {
diff --git a/drivers/media/dvb-frontends/au8522_decoder.c b/drivers/media/dvb-frontends/au8522_decoder.c
index 8cdca051e51b..e4f99bd468cb 100644
--- a/drivers/media/dvb-frontends/au8522_decoder.c
+++ b/drivers/media/dvb-frontends/au8522_decoder.c
@@ -758,13 +758,12 @@ static int au8522_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int au8522_remove(struct i2c_client *client)
+static void au8522_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	au8522_release_state(to_state(sd));
-	return 0;
 }
 
 static const struct i2c_device_id au8522_id[] = {
diff --git a/drivers/media/dvb-frontends/cxd2099.c b/drivers/media/dvb-frontends/cxd2099.c
index 1c8207ab8988..fbc666fa04ec 100644
--- a/drivers/media/dvb-frontends/cxd2099.c
+++ b/drivers/media/dvb-frontends/cxd2099.c
@@ -664,14 +664,12 @@ err:
 	return ret;
 }
 
-static int cxd2099_remove(struct i2c_client *client)
+static void cxd2099_remove(struct i2c_client *client)
 {
 	struct cxd *ci = i2c_get_clientdata(client);
 
 	regmap_exit(ci->regmap);
 	kfree(ci);
-
-	return 0;
 }
 
 static const struct i2c_device_id cxd2099_id[] = {
diff --git a/drivers/media/dvb-frontends/cxd2820r_core.c b/drivers/media/dvb-frontends/cxd2820r_core.c
index b1618339eec0..5d98222f9df0 100644
--- a/drivers/media/dvb-frontends/cxd2820r_core.c
+++ b/drivers/media/dvb-frontends/cxd2820r_core.c
@@ -705,7 +705,7 @@ err:
 	return ret;
 }
 
-static int cxd2820r_remove(struct i2c_client *client)
+static void cxd2820r_remove(struct i2c_client *client)
 {
 	struct cxd2820r_priv *priv = i2c_get_clientdata(client);
 
@@ -721,8 +721,6 @@ static int cxd2820r_remove(struct i2c_client *client)
 	regmap_exit(priv->regmap[0]);
 
 	kfree(priv);
-
-	return 0;
 }
 
 static const struct i2c_device_id cxd2820r_id_table[] = {
diff --git a/drivers/media/dvb-frontends/dvb-pll.c b/drivers/media/dvb-frontends/dvb-pll.c
index d45b4ddc8f91..baf2a378e565 100644
--- a/drivers/media/dvb-frontends/dvb-pll.c
+++ b/drivers/media/dvb-frontends/dvb-pll.c
@@ -899,14 +899,13 @@ dvb_pll_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	return 0;
 }
 
-static int dvb_pll_remove(struct i2c_client *client)
+static void dvb_pll_remove(struct i2c_client *client)
 {
 	struct dvb_frontend *fe = i2c_get_clientdata(client);
 	struct dvb_pll_priv *priv = fe->tuner_priv;
 
 	ida_simple_remove(&pll_ida, priv->nr);
 	dvb_pll_release(fe);
-	return 0;
 }
 
 
diff --git a/drivers/media/dvb-frontends/lgdt3306a.c b/drivers/media/dvb-frontends/lgdt3306a.c
index 136b76cb4807..424311afb2bf 100644
--- a/drivers/media/dvb-frontends/lgdt3306a.c
+++ b/drivers/media/dvb-frontends/lgdt3306a.c
@@ -2226,7 +2226,7 @@ fail:
 	return ret;
 }
 
-static int lgdt3306a_remove(struct i2c_client *client)
+static void lgdt3306a_remove(struct i2c_client *client)
 {
 	struct lgdt3306a_state *state = i2c_get_clientdata(client);
 
@@ -2237,8 +2237,6 @@ static int lgdt3306a_remove(struct i2c_client *client)
 
 	kfree(state->cfg);
 	kfree(state);
-
-	return 0;
 }
 
 static const struct i2c_device_id lgdt3306a_id_table[] = {
diff --git a/drivers/media/dvb-frontends/lgdt330x.c b/drivers/media/dvb-frontends/lgdt330x.c
index da3a8c5e18d8..ea9ae22fd201 100644
--- a/drivers/media/dvb-frontends/lgdt330x.c
+++ b/drivers/media/dvb-frontends/lgdt330x.c
@@ -974,15 +974,13 @@ static const struct dvb_frontend_ops lgdt3303_ops = {
 	.release              = lgdt330x_release,
 };
 
-static int lgdt330x_remove(struct i2c_client *client)
+static void lgdt330x_remove(struct i2c_client *client)
 {
 	struct lgdt330x_state *state = i2c_get_clientdata(client);
 
 	dev_dbg(&client->dev, "\n");
 
 	kfree(state);
-
-	return 0;
 }
 
 static const struct i2c_device_id lgdt330x_id_table[] = {
diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c
index bce0f42f3d19..4e844b2ef597 100644
--- a/drivers/media/dvb-frontends/m88ds3103.c
+++ b/drivers/media/dvb-frontends/m88ds3103.c
@@ -1914,7 +1914,7 @@ err:
 	return ret;
 }
 
-static int m88ds3103_remove(struct i2c_client *client)
+static void m88ds3103_remove(struct i2c_client *client)
 {
 	struct m88ds3103_dev *dev = i2c_get_clientdata(client);
 
@@ -1926,7 +1926,6 @@ static int m88ds3103_remove(struct i2c_client *client)
 	i2c_mux_del_adapters(dev->muxc);
 
 	kfree(dev);
-	return 0;
 }
 
 static const struct i2c_device_id m88ds3103_id_table[] = {
diff --git a/drivers/media/dvb-frontends/mn88443x.c b/drivers/media/dvb-frontends/mn88443x.c
index fff212c0bf3b..452571b380b7 100644
--- a/drivers/media/dvb-frontends/mn88443x.c
+++ b/drivers/media/dvb-frontends/mn88443x.c
@@ -762,15 +762,13 @@ err_i2c_t:
 	return ret;
 }
 
-static int mn88443x_remove(struct i2c_client *client)
+static void mn88443x_remove(struct i2c_client *client)
 {
 	struct mn88443x_priv *chip = i2c_get_clientdata(client);
 
 	mn88443x_cmn_power_off(chip);
 
 	i2c_unregister_device(chip->client_t);
-
-	return 0;
 }
 
 static const struct mn88443x_spec mn88443x_spec_pri = {
diff --git a/drivers/media/dvb-frontends/mn88472.c b/drivers/media/dvb-frontends/mn88472.c
index 73922fc8f39c..2b01cc678f7e 100644
--- a/drivers/media/dvb-frontends/mn88472.c
+++ b/drivers/media/dvb-frontends/mn88472.c
@@ -691,7 +691,7 @@ err:
 	return ret;
 }
 
-static int mn88472_remove(struct i2c_client *client)
+static void mn88472_remove(struct i2c_client *client)
 {
 	struct mn88472_dev *dev = i2c_get_clientdata(client);
 
@@ -706,8 +706,6 @@ static int mn88472_remove(struct i2c_client *client)
 	regmap_exit(dev->regmap[0]);
 
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id mn88472_id_table[] = {
diff --git a/drivers/media/dvb-frontends/mn88473.c b/drivers/media/dvb-frontends/mn88473.c
index 4838969ef735..f0ecf5910c02 100644
--- a/drivers/media/dvb-frontends/mn88473.c
+++ b/drivers/media/dvb-frontends/mn88473.c
@@ -726,7 +726,7 @@ err:
 	return ret;
 }
 
-static int mn88473_remove(struct i2c_client *client)
+static void mn88473_remove(struct i2c_client *client)
 {
 	struct mn88473_dev *dev = i2c_get_clientdata(client);
 
@@ -741,8 +741,6 @@ static int mn88473_remove(struct i2c_client *client)
 	regmap_exit(dev->regmap[0]);
 
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id mn88473_id_table[] = {
diff --git a/drivers/media/dvb-frontends/mxl692.c b/drivers/media/dvb-frontends/mxl692.c
index dd7954e8f553..129630cbffff 100644
--- a/drivers/media/dvb-frontends/mxl692.c
+++ b/drivers/media/dvb-frontends/mxl692.c
@@ -1337,15 +1337,13 @@ err:
 	return -ENODEV;
 }
 
-static int mxl692_remove(struct i2c_client *client)
+static void mxl692_remove(struct i2c_client *client)
 {
 	struct mxl692_dev *dev = i2c_get_clientdata(client);
 
 	dev->fe.demodulator_priv = NULL;
 	i2c_set_clientdata(client, NULL);
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id mxl692_id_table[] = {
diff --git a/drivers/media/dvb-frontends/rtl2830.c b/drivers/media/dvb-frontends/rtl2830.c
index e6b8367c8cce..e0fbf41316ae 100644
--- a/drivers/media/dvb-frontends/rtl2830.c
+++ b/drivers/media/dvb-frontends/rtl2830.c
@@ -865,7 +865,7 @@ err:
 	return ret;
 }
 
-static int rtl2830_remove(struct i2c_client *client)
+static void rtl2830_remove(struct i2c_client *client)
 {
 	struct rtl2830_dev *dev = i2c_get_clientdata(client);
 
@@ -874,8 +874,6 @@ static int rtl2830_remove(struct i2c_client *client)
 	i2c_mux_del_adapters(dev->muxc);
 	regmap_exit(dev->regmap);
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id rtl2830_id_table[] = {
diff --git a/drivers/media/dvb-frontends/rtl2832.c b/drivers/media/dvb-frontends/rtl2832.c
index dcbeb9f5e12a..4fa884eda5d5 100644
--- a/drivers/media/dvb-frontends/rtl2832.c
+++ b/drivers/media/dvb-frontends/rtl2832.c
@@ -1110,7 +1110,7 @@ err:
 	return ret;
 }
 
-static int rtl2832_remove(struct i2c_client *client)
+static void rtl2832_remove(struct i2c_client *client)
 {
 	struct rtl2832_dev *dev = i2c_get_clientdata(client);
 
@@ -1123,8 +1123,6 @@ static int rtl2832_remove(struct i2c_client *client)
 	regmap_exit(dev->regmap);
 
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id rtl2832_id_table[] = {
diff --git a/drivers/media/dvb-frontends/si2165.c b/drivers/media/dvb-frontends/si2165.c
index ebee230afb7b..86b0d59169dd 100644
--- a/drivers/media/dvb-frontends/si2165.c
+++ b/drivers/media/dvb-frontends/si2165.c
@@ -1274,14 +1274,13 @@ error:
 	return ret;
 }
 
-static int si2165_remove(struct i2c_client *client)
+static void si2165_remove(struct i2c_client *client)
 {
 	struct si2165_state *state = i2c_get_clientdata(client);
 
 	dev_dbg(&client->dev, "\n");
 
 	kfree(state);
-	return 0;
 }
 
 static const struct i2c_device_id si2165_id_table[] = {
diff --git a/drivers/media/dvb-frontends/si2168.c b/drivers/media/dvb-frontends/si2168.c
index 196e028a6617..8157df4570d1 100644
--- a/drivers/media/dvb-frontends/si2168.c
+++ b/drivers/media/dvb-frontends/si2168.c
@@ -774,7 +774,7 @@ err:
 	return ret;
 }
 
-static int si2168_remove(struct i2c_client *client)
+static void si2168_remove(struct i2c_client *client)
 {
 	struct si2168_dev *dev = i2c_get_clientdata(client);
 
@@ -786,8 +786,6 @@ static int si2168_remove(struct i2c_client *client)
 	dev->fe.demodulator_priv = NULL;
 
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id si2168_id_table[] = {
diff --git a/drivers/media/dvb-frontends/sp2.c b/drivers/media/dvb-frontends/sp2.c
index 992f22167fbe..27e7037e130e 100644
--- a/drivers/media/dvb-frontends/sp2.c
+++ b/drivers/media/dvb-frontends/sp2.c
@@ -398,14 +398,13 @@ err:
 	return ret;
 }
 
-static int sp2_remove(struct i2c_client *client)
+static void sp2_remove(struct i2c_client *client)
 {
 	struct sp2 *s = i2c_get_clientdata(client);
 
 	dev_dbg(&client->dev, "\n");
 	sp2_exit(client);
 	kfree(s);
-	return 0;
 }
 
 static const struct i2c_device_id sp2_id[] = {
diff --git a/drivers/media/dvb-frontends/stv090x.c b/drivers/media/dvb-frontends/stv090x.c
index 90d24131d335..0a600c1d7d1b 100644
--- a/drivers/media/dvb-frontends/stv090x.c
+++ b/drivers/media/dvb-frontends/stv090x.c
@@ -5032,12 +5032,11 @@ error:
 	return ret;
 }
 
-static int stv090x_remove(struct i2c_client *client)
+static void stv090x_remove(struct i2c_client *client)
 {
 	struct stv090x_state *state = i2c_get_clientdata(client);
 
 	stv090x_release(&state->frontend);
-	return 0;
 }
 
 struct dvb_frontend *stv090x_attach(struct stv090x_config *config,
diff --git a/drivers/media/dvb-frontends/stv6110x.c b/drivers/media/dvb-frontends/stv6110x.c
index 5012d0231652..fbc4dbd62151 100644
--- a/drivers/media/dvb-frontends/stv6110x.c
+++ b/drivers/media/dvb-frontends/stv6110x.c
@@ -436,12 +436,11 @@ static int stv6110x_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int stv6110x_remove(struct i2c_client *client)
+static void stv6110x_remove(struct i2c_client *client)
 {
 	struct stv6110x_state *stv6110x = i2c_get_clientdata(client);
 
 	stv6110x_release(stv6110x->frontend);
-	return 0;
 }
 
 const struct stv6110x_devctl *stv6110x_attach(struct dvb_frontend *fe,
diff --git a/drivers/media/dvb-frontends/tc90522.c b/drivers/media/dvb-frontends/tc90522.c
index e83836b29715..c22d2a2b2a45 100644
--- a/drivers/media/dvb-frontends/tc90522.c
+++ b/drivers/media/dvb-frontends/tc90522.c
@@ -819,14 +819,13 @@ free_state:
 	return ret;
 }
 
-static int tc90522_remove(struct i2c_client *client)
+static void tc90522_remove(struct i2c_client *client)
 {
 	struct tc90522_state *state;
 
 	state = cfg_to_state(i2c_get_clientdata(client));
 	i2c_del_adapter(&state->tuner_i2c);
 	kfree(state);
-	return 0;
 }
 
 
diff --git a/drivers/media/dvb-frontends/tda10071.c b/drivers/media/dvb-frontends/tda10071.c
index 685c0ac71819..d1098ef20a8b 100644
--- a/drivers/media/dvb-frontends/tda10071.c
+++ b/drivers/media/dvb-frontends/tda10071.c
@@ -1221,14 +1221,13 @@ err:
 	return ret;
 }
 
-static int tda10071_remove(struct i2c_client *client)
+static void tda10071_remove(struct i2c_client *client)
 {
 	struct tda10071_dev *dev = i2c_get_clientdata(client);
 
 	dev_dbg(&client->dev, "\n");
 
 	kfree(dev);
-	return 0;
 }
 
 static const struct i2c_device_id tda10071_id_table[] = {
diff --git a/drivers/media/dvb-frontends/ts2020.c b/drivers/media/dvb-frontends/ts2020.c
index 3e383912bcfd..02338256b974 100644
--- a/drivers/media/dvb-frontends/ts2020.c
+++ b/drivers/media/dvb-frontends/ts2020.c
@@ -696,7 +696,7 @@ err:
 	return ret;
 }
 
-static int ts2020_remove(struct i2c_client *client)
+static void ts2020_remove(struct i2c_client *client)
 {
 	struct ts2020_priv *dev = i2c_get_clientdata(client);
 
@@ -708,7 +708,6 @@ static int ts2020_remove(struct i2c_client *client)
 
 	regmap_exit(dev->regmap);
 	kfree(dev);
-	return 0;
 }
 
 static const struct i2c_device_id ts2020_id_table[] = {
diff --git a/drivers/media/i2c/ad5820.c b/drivers/media/i2c/ad5820.c
index 2958a4694461..516de278cc49 100644
--- a/drivers/media/i2c/ad5820.c
+++ b/drivers/media/i2c/ad5820.c
@@ -342,7 +342,7 @@ cleanup:
 	return ret;
 }
 
-static int ad5820_remove(struct i2c_client *client)
+static void ad5820_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
 	struct ad5820_device *coil = to_ad5820_device(subdev);
@@ -351,7 +351,6 @@ static int ad5820_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(&coil->ctrls);
 	media_entity_cleanup(&coil->subdev.entity);
 	mutex_destroy(&coil->power_lock);
-	return 0;
 }
 
 static const struct i2c_device_id ad5820_id_table[] = {
diff --git a/drivers/media/i2c/ad9389b.c b/drivers/media/i2c/ad9389b.c
index 8679a44e6413..4a255a492918 100644
--- a/drivers/media/i2c/ad9389b.c
+++ b/drivers/media/i2c/ad9389b.c
@@ -1174,7 +1174,7 @@ err_hdl:
 
 /* ----------------------------------------------------------------------- */
 
-static int ad9389b_remove(struct i2c_client *client)
+static void ad9389b_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ad9389b_state *state = get_ad9389b_state(sd);
@@ -1192,7 +1192,6 @@ static int ad9389b_remove(struct i2c_client *client)
 	v4l2_device_unregister_subdev(sd);
 	media_entity_cleanup(&sd->entity);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/adp1653.c b/drivers/media/i2c/adp1653.c
index 522a0b10e415..1f353157df07 100644
--- a/drivers/media/i2c/adp1653.c
+++ b/drivers/media/i2c/adp1653.c
@@ -510,7 +510,7 @@ free_and_quit:
 	return ret;
 }
 
-static int adp1653_remove(struct i2c_client *client)
+static void adp1653_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
 	struct adp1653_flash *flash = to_adp1653_flash(subdev);
@@ -518,8 +518,6 @@ static int adp1653_remove(struct i2c_client *client)
 	v4l2_device_unregister_subdev(&flash->subdev);
 	v4l2_ctrl_handler_free(&flash->ctrls);
 	media_entity_cleanup(&flash->subdev.entity);
-
-	return 0;
 }
 
 static const struct i2c_device_id adp1653_id_table[] = {
diff --git a/drivers/media/i2c/adv7170.c b/drivers/media/i2c/adv7170.c
index 714e31f993e1..61a2f87d3c62 100644
--- a/drivers/media/i2c/adv7170.c
+++ b/drivers/media/i2c/adv7170.c
@@ -368,12 +368,11 @@ static int adv7170_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int adv7170_remove(struct i2c_client *client)
+static void adv7170_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/adv7175.c b/drivers/media/i2c/adv7175.c
index 1813f67f0fe1..b58689728243 100644
--- a/drivers/media/i2c/adv7175.c
+++ b/drivers/media/i2c/adv7175.c
@@ -423,12 +423,11 @@ static int adv7175_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int adv7175_remove(struct i2c_client *client)
+static void adv7175_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c
index 5fde5243722d..216fe396973f 100644
--- a/drivers/media/i2c/adv7180.c
+++ b/drivers/media/i2c/adv7180.c
@@ -1514,7 +1514,7 @@ err_unregister_csi_client:
 	return ret;
 }
 
-static int adv7180_remove(struct i2c_client *client)
+static void adv7180_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct adv7180_state *state = to_state(sd);
@@ -1534,8 +1534,6 @@ static int adv7180_remove(struct i2c_client *client)
 	adv7180_set_power_pin(state, false);
 
 	mutex_destroy(&state->mutex);
-
-	return 0;
 }
 
 static const struct i2c_device_id adv7180_id[] = {
diff --git a/drivers/media/i2c/adv7183.c b/drivers/media/i2c/adv7183.c
index ba746a19fd39..313c706e8335 100644
--- a/drivers/media/i2c/adv7183.c
+++ b/drivers/media/i2c/adv7183.c
@@ -613,13 +613,12 @@ static int adv7183_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int adv7183_remove(struct i2c_client *client)
+static void adv7183_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	return 0;
 }
 
 static const struct i2c_device_id adv7183_id[] = {
diff --git a/drivers/media/i2c/adv7343.c b/drivers/media/i2c/adv7343.c
index 63e94dfcb5d3..7e84869d2434 100644
--- a/drivers/media/i2c/adv7343.c
+++ b/drivers/media/i2c/adv7343.c
@@ -492,15 +492,13 @@ done:
 	return err;
 }
 
-static int adv7343_remove(struct i2c_client *client)
+static void adv7343_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct adv7343_state *state = to_state(sd);
 
 	v4l2_async_unregister_subdev(&state->sd);
 	v4l2_ctrl_handler_free(&state->hdl);
-
-	return 0;
 }
 
 static const struct i2c_device_id adv7343_id[] = {
diff --git a/drivers/media/i2c/adv7393.c b/drivers/media/i2c/adv7393.c
index b6234c8231c9..fb5fefa83b18 100644
--- a/drivers/media/i2c/adv7393.c
+++ b/drivers/media/i2c/adv7393.c
@@ -437,15 +437,13 @@ static int adv7393_probe(struct i2c_client *client,
 	return err;
 }
 
-static int adv7393_remove(struct i2c_client *client)
+static void adv7393_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct adv7393_state *state = to_state(sd);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&state->hdl);
-
-	return 0;
 }
 
 static const struct i2c_device_id adv7393_id[] = {
diff --git a/drivers/media/i2c/adv748x/adv748x-core.c b/drivers/media/i2c/adv748x/adv748x-core.c
index 4e54148147b9..4498d78a2357 100644
--- a/drivers/media/i2c/adv748x/adv748x-core.c
+++ b/drivers/media/i2c/adv748x/adv748x-core.c
@@ -815,7 +815,7 @@ err_free_mutex:
 	return ret;
 }
 
-static int adv748x_remove(struct i2c_client *client)
+static void adv748x_remove(struct i2c_client *client)
 {
 	struct adv748x_state *state = i2c_get_clientdata(client);
 
@@ -828,8 +828,6 @@ static int adv748x_remove(struct i2c_client *client)
 	adv748x_unregister_clients(state);
 	adv748x_dt_cleanup(state);
 	mutex_destroy(&state->mutex);
-
-	return 0;
 }
 
 static const struct of_device_id adv748x_of_table[] = {
diff --git a/drivers/media/i2c/adv7511-v4l2.c b/drivers/media/i2c/adv7511-v4l2.c
index 202e0cd83f90..49aca579576a 100644
--- a/drivers/media/i2c/adv7511-v4l2.c
+++ b/drivers/media/i2c/adv7511-v4l2.c
@@ -1923,7 +1923,7 @@ err_hdl:
 
 /* ----------------------------------------------------------------------- */
 
-static int adv7511_remove(struct i2c_client *client)
+static void adv7511_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct adv7511_state *state = get_adv7511_state(sd);
@@ -1943,7 +1943,6 @@ static int adv7511_remove(struct i2c_client *client)
 	v4l2_device_unregister_subdev(sd);
 	media_entity_cleanup(&sd->entity);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c
index 497419a5cfdd..5988a4fa0c46 100644
--- a/drivers/media/i2c/adv7604.c
+++ b/drivers/media/i2c/adv7604.c
@@ -3660,7 +3660,7 @@ err_hdl:
 
 /* ----------------------------------------------------------------------- */
 
-static int adv76xx_remove(struct i2c_client *client)
+static void adv76xx_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct adv76xx_state *state = to_state(sd);
@@ -3677,7 +3677,6 @@ static int adv76xx_remove(struct i2c_client *client)
 	media_entity_cleanup(&sd->entity);
 	adv76xx_unregister_clients(to_state(sd));
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/adv7842.c b/drivers/media/i2c/adv7842.c
index 22caa070273b..a8dd92948df0 100644
--- a/drivers/media/i2c/adv7842.c
+++ b/drivers/media/i2c/adv7842.c
@@ -3593,7 +3593,7 @@ err_hdl:
 
 /* ----------------------------------------------------------------------- */
 
-static int adv7842_remove(struct i2c_client *client)
+static void adv7842_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct adv7842_state *state = to_state(sd);
@@ -3604,7 +3604,6 @@ static int adv7842_remove(struct i2c_client *client)
 	media_entity_cleanup(&sd->entity);
 	adv7842_unregister_clients(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/ak7375.c b/drivers/media/i2c/ak7375.c
index 40b1a4aa846c..1af9f698eecf 100644
--- a/drivers/media/i2c/ak7375.c
+++ b/drivers/media/i2c/ak7375.c
@@ -169,7 +169,7 @@ err_cleanup:
 	return ret;
 }
 
-static int ak7375_remove(struct i2c_client *client)
+static void ak7375_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ak7375_device *ak7375_dev = sd_to_ak7375_vcm(sd);
@@ -177,8 +177,6 @@ static int ak7375_remove(struct i2c_client *client)
 	ak7375_subdev_cleanup(ak7375_dev);
 	pm_runtime_disable(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
-
-	return 0;
 }
 
 /*
diff --git a/drivers/media/i2c/ak881x.c b/drivers/media/i2c/ak881x.c
index dc569d5a4d9d..0370ad6b6811 100644
--- a/drivers/media/i2c/ak881x.c
+++ b/drivers/media/i2c/ak881x.c
@@ -297,13 +297,11 @@ static int ak881x_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int ak881x_remove(struct i2c_client *client)
+static void ak881x_remove(struct i2c_client *client)
 {
 	struct ak881x *ak881x = to_ak881x(client);
 
 	v4l2_device_unregister_subdev(&ak881x->subdev);
-
-	return 0;
 }
 
 static const struct i2c_device_id ak881x_id[] = {
diff --git a/drivers/media/i2c/ar0521.c b/drivers/media/i2c/ar0521.c
index c7bdfc69b9be..c6ab531532be 100644
--- a/drivers/media/i2c/ar0521.c
+++ b/drivers/media/i2c/ar0521.c
@@ -1018,7 +1018,7 @@ entity_cleanup:
 	return ret;
 }
 
-static int ar0521_remove(struct i2c_client *client)
+static void ar0521_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ar0521_dev *sensor = to_ar0521_dev(sd);
@@ -1031,7 +1031,6 @@ static int ar0521_remove(struct i2c_client *client)
 		ar0521_power_off(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
 	mutex_destroy(&sensor->lock);
-	return 0;
 }
 
 static const struct dev_pm_ops ar0521_pm_ops = {
diff --git a/drivers/media/i2c/bt819.c b/drivers/media/i2c/bt819.c
index 73bc50c919d7..4d9bb6eb7d65 100644
--- a/drivers/media/i2c/bt819.c
+++ b/drivers/media/i2c/bt819.c
@@ -446,14 +446,13 @@ static int bt819_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int bt819_remove(struct i2c_client *client)
+static void bt819_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct bt819 *decoder = to_bt819(sd);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&decoder->hdl);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/bt856.c b/drivers/media/i2c/bt856.c
index c134fda270a1..70443ef1ac46 100644
--- a/drivers/media/i2c/bt856.c
+++ b/drivers/media/i2c/bt856.c
@@ -223,12 +223,11 @@ static int bt856_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int bt856_remove(struct i2c_client *client)
+static void bt856_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 static const struct i2c_device_id bt856_id[] = {
diff --git a/drivers/media/i2c/bt866.c b/drivers/media/i2c/bt866.c
index 1a8df9f18ffb..c2508cbafd02 100644
--- a/drivers/media/i2c/bt866.c
+++ b/drivers/media/i2c/bt866.c
@@ -190,12 +190,11 @@ static int bt866_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int bt866_remove(struct i2c_client *client)
+static void bt866_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 static const struct i2c_device_id bt866_id[] = {
diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c
index 7609add2aff4..4a14d7e5d9f2 100644
--- a/drivers/media/i2c/ccs/ccs-core.c
+++ b/drivers/media/i2c/ccs/ccs-core.c
@@ -3665,7 +3665,7 @@ out_power_off:
 	return rval;
 }
 
-static int ccs_remove(struct i2c_client *client)
+static void ccs_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
 	struct ccs_sensor *sensor = to_ccs_sensor(subdev);
@@ -3687,8 +3687,6 @@ static int ccs_remove(struct i2c_client *client)
 	kfree(sensor->ccs_limits);
 	kvfree(sensor->sdata.backing);
 	kvfree(sensor->mdata.backing);
-
-	return 0;
 }
 
 static const struct ccs_device smia_device = {
diff --git a/drivers/media/i2c/cs3308.c b/drivers/media/i2c/cs3308.c
index ebe55e261bff..d901a59883a9 100644
--- a/drivers/media/i2c/cs3308.c
+++ b/drivers/media/i2c/cs3308.c
@@ -99,13 +99,12 @@ static int cs3308_probe(struct i2c_client *client,
 
 /* ----------------------------------------------------------------------- */
 
-static int cs3308_remove(struct i2c_client *client)
+static void cs3308_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	kfree(sd);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/cs5345.c b/drivers/media/i2c/cs5345.c
index f6dd5edf77dd..591b1e7b24ee 100644
--- a/drivers/media/i2c/cs5345.c
+++ b/drivers/media/i2c/cs5345.c
@@ -178,14 +178,13 @@ static int cs5345_probe(struct i2c_client *client,
 
 /* ----------------------------------------------------------------------- */
 
-static int cs5345_remove(struct i2c_client *client)
+static void cs5345_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct cs5345_state *state = to_state(sd);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&state->hdl);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/cs53l32a.c b/drivers/media/i2c/cs53l32a.c
index 9a411106cfb3..9461589aea30 100644
--- a/drivers/media/i2c/cs53l32a.c
+++ b/drivers/media/i2c/cs53l32a.c
@@ -190,14 +190,13 @@ static int cs53l32a_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int cs53l32a_remove(struct i2c_client *client)
+static void cs53l32a_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct cs53l32a_state *state = to_state(sd);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&state->hdl);
-	return 0;
 }
 
 static const struct i2c_device_id cs53l32a_id[] = {
diff --git a/drivers/media/i2c/cx25840/cx25840-core.c b/drivers/media/i2c/cx25840/cx25840-core.c
index dc31944c7d5b..f1a978af82ef 100644
--- a/drivers/media/i2c/cx25840/cx25840-core.c
+++ b/drivers/media/i2c/cx25840/cx25840-core.c
@@ -6026,7 +6026,7 @@ static int cx25840_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int cx25840_remove(struct i2c_client *client)
+static void cx25840_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct cx25840_state *state = to_state(sd);
@@ -6034,7 +6034,6 @@ static int cx25840_remove(struct i2c_client *client)
 	cx25840_ir_remove(sd);
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&state->hdl);
-	return 0;
 }
 
 static const struct i2c_device_id cx25840_id[] = {
diff --git a/drivers/media/i2c/dw9714.c b/drivers/media/i2c/dw9714.c
index 206d74338b9c..af59687383aa 100644
--- a/drivers/media/i2c/dw9714.c
+++ b/drivers/media/i2c/dw9714.c
@@ -190,7 +190,7 @@ err_cleanup:
 	return rval;
 }
 
-static int dw9714_remove(struct i2c_client *client)
+static void dw9714_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct dw9714_device *dw9714_dev = sd_to_dw9714_vcm(sd);
@@ -206,8 +206,6 @@ static int dw9714_remove(struct i2c_client *client)
 	}
 	pm_runtime_set_suspended(&client->dev);
 	dw9714_subdev_cleanup(dw9714_dev);
-
-	return 0;
 }
 
 /*
diff --git a/drivers/media/i2c/dw9768.c b/drivers/media/i2c/dw9768.c
index c086580efac7..0f47ef015a1d 100644
--- a/drivers/media/i2c/dw9768.c
+++ b/drivers/media/i2c/dw9768.c
@@ -499,7 +499,7 @@ err_free_handler:
 	return ret;
 }
 
-static int dw9768_remove(struct i2c_client *client)
+static void dw9768_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct dw9768 *dw9768 = sd_to_dw9768(sd);
@@ -511,8 +511,6 @@ static int dw9768_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(&client->dev))
 		dw9768_runtime_suspend(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
-
-	return 0;
 }
 
 static const struct of_device_id dw9768_of_table[] = {
diff --git a/drivers/media/i2c/dw9807-vcm.c b/drivers/media/i2c/dw9807-vcm.c
index 01c372925a80..3599720db7e9 100644
--- a/drivers/media/i2c/dw9807-vcm.c
+++ b/drivers/media/i2c/dw9807-vcm.c
@@ -216,7 +216,7 @@ err_cleanup:
 	return rval;
 }
 
-static int dw9807_remove(struct i2c_client *client)
+static void dw9807_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct dw9807_device *dw9807_dev = sd_to_dw9807_vcm(sd);
@@ -224,8 +224,6 @@ static int dw9807_remove(struct i2c_client *client)
 	pm_runtime_disable(&client->dev);
 
 	dw9807_subdev_cleanup(dw9807_dev);
-
-	return 0;
 }
 
 /*
diff --git a/drivers/media/i2c/et8ek8/et8ek8_driver.c b/drivers/media/i2c/et8ek8/et8ek8_driver.c
index 873d614339bb..ff9bb9fc97dd 100644
--- a/drivers/media/i2c/et8ek8/et8ek8_driver.c
+++ b/drivers/media/i2c/et8ek8/et8ek8_driver.c
@@ -1460,7 +1460,7 @@ err_mutex:
 	return ret;
 }
 
-static int __exit et8ek8_remove(struct i2c_client *client)
+static void __exit et8ek8_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
 	struct et8ek8_sensor *sensor = to_et8ek8_sensor(subdev);
@@ -1477,8 +1477,6 @@ static int __exit et8ek8_remove(struct i2c_client *client)
 	v4l2_async_unregister_subdev(&sensor->subdev);
 	media_entity_cleanup(&sensor->subdev.entity);
 	mutex_destroy(&sensor->power_lock);
-
-	return 0;
 }
 
 static const struct of_device_id et8ek8_of_table[] = {
diff --git a/drivers/media/i2c/hi556.c b/drivers/media/i2c/hi556.c
index 055d1aa8410e..e422ac7609b5 100644
--- a/drivers/media/i2c/hi556.c
+++ b/drivers/media/i2c/hi556.c
@@ -1101,7 +1101,7 @@ check_hwcfg_error:
 	return ret;
 }
 
-static int hi556_remove(struct i2c_client *client)
+static void hi556_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct hi556 *hi556 = to_hi556(sd);
@@ -1111,8 +1111,6 @@ static int hi556_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	pm_runtime_disable(&client->dev);
 	mutex_destroy(&hi556->mutex);
-
-	return 0;
 }
 
 static int hi556_probe(struct i2c_client *client)
diff --git a/drivers/media/i2c/hi846.c b/drivers/media/i2c/hi846.c
index ad35c3ff3611..c5b69823f257 100644
--- a/drivers/media/i2c/hi846.c
+++ b/drivers/media/i2c/hi846.c
@@ -2143,7 +2143,7 @@ err_mutex:
 	return ret;
 }
 
-static int hi846_remove(struct i2c_client *client)
+static void hi846_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct hi846 *hi846 = to_hi846(sd);
@@ -2158,8 +2158,6 @@ static int hi846_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	mutex_destroy(&hi846->mutex);
-
-	return 0;
 }
 
 static const struct dev_pm_ops hi846_pm_ops = {
diff --git a/drivers/media/i2c/hi847.c b/drivers/media/i2c/hi847.c
index 7e85349e1852..5a82b15a9513 100644
--- a/drivers/media/i2c/hi847.c
+++ b/drivers/media/i2c/hi847.c
@@ -2903,7 +2903,7 @@ check_hwcfg_error:
 	return ret;
 }
 
-static int hi847_remove(struct i2c_client *client)
+static void hi847_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct hi847 *hi847 = to_hi847(sd);
@@ -2913,8 +2913,6 @@ static int hi847_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	pm_runtime_disable(&client->dev);
 	mutex_destroy(&hi847->mutex);
-
-	return 0;
 }
 
 static int hi847_probe(struct i2c_client *client)
diff --git a/drivers/media/i2c/imx208.c b/drivers/media/i2c/imx208.c
index b9516b2f1c15..a0e17bb9d4ca 100644
--- a/drivers/media/i2c/imx208.c
+++ b/drivers/media/i2c/imx208.c
@@ -1061,7 +1061,7 @@ error_probe:
 	return ret;
 }
 
-static int imx208_remove(struct i2c_client *client)
+static void imx208_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct imx208 *imx208 = to_imx208(sd);
@@ -1075,8 +1075,6 @@ static int imx208_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	mutex_destroy(&imx208->imx208_mx);
-
-	return 0;
 }
 
 static const struct dev_pm_ops imx208_pm_ops = {
diff --git a/drivers/media/i2c/imx214.c b/drivers/media/i2c/imx214.c
index 83c1737abeec..710c9fb515fd 100644
--- a/drivers/media/i2c/imx214.c
+++ b/drivers/media/i2c/imx214.c
@@ -1080,7 +1080,7 @@ free_ctrl:
 	return ret;
 }
 
-static int imx214_remove(struct i2c_client *client)
+static void imx214_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct imx214 *imx214 = to_imx214(sd);
@@ -1093,8 +1093,6 @@ static int imx214_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	mutex_destroy(&imx214->mutex);
-
-	return 0;
 }
 
 static const struct of_device_id imx214_of_match[] = {
diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c
index e10af3f74b38..77bd79a5954e 100644
--- a/drivers/media/i2c/imx219.c
+++ b/drivers/media/i2c/imx219.c
@@ -1562,7 +1562,7 @@ error_power_off:
 	return ret;
 }
 
-static int imx219_remove(struct i2c_client *client)
+static void imx219_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct imx219 *imx219 = to_imx219(sd);
@@ -1575,8 +1575,6 @@ static int imx219_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(&client->dev))
 		imx219_power_off(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
-
-	return 0;
 }
 
 static const struct of_device_id imx219_dt_ids[] = {
diff --git a/drivers/media/i2c/imx258.c b/drivers/media/i2c/imx258.c
index c249507aa2db..eab5fc1ee2f7 100644
--- a/drivers/media/i2c/imx258.c
+++ b/drivers/media/i2c/imx258.c
@@ -1338,7 +1338,7 @@ error_identify:
 	return ret;
 }
 
-static int imx258_remove(struct i2c_client *client)
+static void imx258_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct imx258 *imx258 = to_imx258(sd);
@@ -1351,8 +1351,6 @@ static int imx258_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(&client->dev))
 		imx258_power_off(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
-
-	return 0;
 }
 
 static const struct dev_pm_ops imx258_pm_ops = {
diff --git a/drivers/media/i2c/imx274.c b/drivers/media/i2c/imx274.c
index 7de1f2948e53..a00761b1e18c 100644
--- a/drivers/media/i2c/imx274.c
+++ b/drivers/media/i2c/imx274.c
@@ -2142,7 +2142,7 @@ err_regmap:
 	return ret;
 }
 
-static int imx274_remove(struct i2c_client *client)
+static void imx274_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct stimx274 *imx274 = to_imx274(sd);
@@ -2157,7 +2157,6 @@ static int imx274_remove(struct i2c_client *client)
 
 	media_entity_cleanup(&sd->entity);
 	mutex_destroy(&imx274->lock);
-	return 0;
 }
 
 static const struct dev_pm_ops imx274_pm_ops = {
diff --git a/drivers/media/i2c/imx290.c b/drivers/media/i2c/imx290.c
index 99f2a50d39a4..1ce64dcdf7f0 100644
--- a/drivers/media/i2c/imx290.c
+++ b/drivers/media/i2c/imx290.c
@@ -1119,7 +1119,7 @@ free_err:
 	return ret;
 }
 
-static int imx290_remove(struct i2c_client *client)
+static void imx290_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct imx290 *imx290 = to_imx290(sd);
@@ -1134,8 +1134,6 @@ static int imx290_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(imx290->dev))
 		imx290_power_off(imx290->dev);
 	pm_runtime_set_suspended(imx290->dev);
-
-	return 0;
 }
 
 static const struct of_device_id imx290_of_match[] = {
diff --git a/drivers/media/i2c/imx319.c b/drivers/media/i2c/imx319.c
index a2b5a34de76b..245a18fb40ad 100644
--- a/drivers/media/i2c/imx319.c
+++ b/drivers/media/i2c/imx319.c
@@ -2523,7 +2523,7 @@ error_probe:
 	return ret;
 }
 
-static int imx319_remove(struct i2c_client *client)
+static void imx319_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct imx319 *imx319 = to_imx319(sd);
@@ -2536,8 +2536,6 @@ static int imx319_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	mutex_destroy(&imx319->mutex);
-
-	return 0;
 }
 
 static const struct dev_pm_ops imx319_pm_ops = {
diff --git a/drivers/media/i2c/imx334.c b/drivers/media/i2c/imx334.c
index 062125501788..7b0a9086447d 100644
--- a/drivers/media/i2c/imx334.c
+++ b/drivers/media/i2c/imx334.c
@@ -1089,7 +1089,7 @@ error_mutex_destroy:
  *
  * Return: 0 if successful, error code otherwise.
  */
-static int imx334_remove(struct i2c_client *client)
+static void imx334_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct imx334 *imx334 = to_imx334(sd);
@@ -1102,8 +1102,6 @@ static int imx334_remove(struct i2c_client *client)
 	pm_runtime_suspended(&client->dev);
 
 	mutex_destroy(&imx334->mutex);
-
-	return 0;
 }
 
 static const struct dev_pm_ops imx334_pm_ops = {
diff --git a/drivers/media/i2c/imx335.c b/drivers/media/i2c/imx335.c
index 410d6b86feb5..078ede2b7a00 100644
--- a/drivers/media/i2c/imx335.c
+++ b/drivers/media/i2c/imx335.c
@@ -1083,7 +1083,7 @@ error_mutex_destroy:
  *
  * Return: 0 if successful, error code otherwise.
  */
-static int imx335_remove(struct i2c_client *client)
+static void imx335_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct imx335 *imx335 = to_imx335(sd);
@@ -1098,8 +1098,6 @@ static int imx335_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	mutex_destroy(&imx335->mutex);
-
-	return 0;
 }
 
 static const struct dev_pm_ops imx335_pm_ops = {
diff --git a/drivers/media/i2c/imx355.c b/drivers/media/i2c/imx355.c
index 3922b9305978..b46178681c05 100644
--- a/drivers/media/i2c/imx355.c
+++ b/drivers/media/i2c/imx355.c
@@ -1810,7 +1810,7 @@ error_probe:
 	return ret;
 }
 
-static int imx355_remove(struct i2c_client *client)
+static void imx355_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct imx355 *imx355 = to_imx355(sd);
@@ -1823,8 +1823,6 @@ static int imx355_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	mutex_destroy(&imx355->mutex);
-
-	return 0;
 }
 
 static const struct dev_pm_ops imx355_pm_ops = {
diff --git a/drivers/media/i2c/imx412.c b/drivers/media/i2c/imx412.c
index a1394d6c1432..7f6d29e0e7c4 100644
--- a/drivers/media/i2c/imx412.c
+++ b/drivers/media/i2c/imx412.c
@@ -1257,7 +1257,7 @@ error_mutex_destroy:
  *
  * Return: 0 if successful, error code otherwise.
  */
-static int imx412_remove(struct i2c_client *client)
+static void imx412_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct imx412 *imx412 = to_imx412(sd);
@@ -1272,8 +1272,6 @@ static int imx412_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	mutex_destroy(&imx412->mutex);
-
-	return 0;
 }
 
 static const struct dev_pm_ops imx412_pm_ops = {
diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c
index 56674173524f..ee6bbbb977f7 100644
--- a/drivers/media/i2c/ir-kbd-i2c.c
+++ b/drivers/media/i2c/ir-kbd-i2c.c
@@ -915,7 +915,7 @@ static int ir_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	return err;
 }
 
-static int ir_remove(struct i2c_client *client)
+static void ir_remove(struct i2c_client *client)
 {
 	struct IR_i2c *ir = i2c_get_clientdata(client);
 
@@ -924,8 +924,6 @@ static int ir_remove(struct i2c_client *client)
 	i2c_unregister_device(ir->tx_c);
 
 	rc_unregister_device(ir->rc);
-
-	return 0;
 }
 
 static const struct i2c_device_id ir_kbd_id[] = {
diff --git a/drivers/media/i2c/isl7998x.c b/drivers/media/i2c/isl7998x.c
index dc3068549dfa..246d8d182a8e 100644
--- a/drivers/media/i2c/isl7998x.c
+++ b/drivers/media/i2c/isl7998x.c
@@ -1544,7 +1544,7 @@ err_entity_cleanup:
 	return ret;
 }
 
-static int isl7998x_remove(struct i2c_client *client)
+static void isl7998x_remove(struct i2c_client *client)
 {
 	struct isl7998x *isl7998x = i2c_to_isl7998x(client);
 
@@ -1552,8 +1552,6 @@ static int isl7998x_remove(struct i2c_client *client)
 	v4l2_async_unregister_subdev(&isl7998x->subdev);
 	isl7998x_remove_controls(isl7998x);
 	media_entity_cleanup(&isl7998x->subdev.entity);
-
-	return 0;
 }
 
 static const struct of_device_id isl7998x_of_match[] = {
diff --git a/drivers/media/i2c/ks0127.c b/drivers/media/i2c/ks0127.c
index c077f53b9c30..215d9a43b0b9 100644
--- a/drivers/media/i2c/ks0127.c
+++ b/drivers/media/i2c/ks0127.c
@@ -675,14 +675,13 @@ static int ks0127_probe(struct i2c_client *client, const struct i2c_device_id *i
 	return 0;
 }
 
-static int ks0127_remove(struct i2c_client *client)
+static void ks0127_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	ks0127_write(sd, KS_OFMTA, 0x20); /* tristate */
 	ks0127_write(sd, KS_CMDA, 0x2c | 0x80); /* power down */
-	return 0;
 }
 
 static const struct i2c_device_id ks0127_id[] = {
diff --git a/drivers/media/i2c/lm3560.c b/drivers/media/i2c/lm3560.c
index 9e34ccce4fc3..edad3138cb07 100644
--- a/drivers/media/i2c/lm3560.c
+++ b/drivers/media/i2c/lm3560.c
@@ -443,7 +443,7 @@ static int lm3560_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lm3560_remove(struct i2c_client *client)
+static void lm3560_remove(struct i2c_client *client)
 {
 	struct lm3560_flash *flash = i2c_get_clientdata(client);
 	unsigned int i;
@@ -453,8 +453,6 @@ static int lm3560_remove(struct i2c_client *client)
 		v4l2_ctrl_handler_free(&flash->ctrls_led[i]);
 		media_entity_cleanup(&flash->subdev_led[i].entity);
 	}
-
-	return 0;
 }
 
 static const struct i2c_device_id lm3560_id_table[] = {
diff --git a/drivers/media/i2c/lm3646.c b/drivers/media/i2c/lm3646.c
index c76ccf67a909..0aaa963917d8 100644
--- a/drivers/media/i2c/lm3646.c
+++ b/drivers/media/i2c/lm3646.c
@@ -377,15 +377,13 @@ static int lm3646_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lm3646_remove(struct i2c_client *client)
+static void lm3646_remove(struct i2c_client *client)
 {
 	struct lm3646_flash *flash = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(&flash->subdev_led);
 	v4l2_ctrl_handler_free(&flash->ctrls_led);
 	media_entity_cleanup(&flash->subdev_led.entity);
-
-	return 0;
 }
 
 static const struct i2c_device_id lm3646_id_table[] = {
diff --git a/drivers/media/i2c/m52790.c b/drivers/media/i2c/m52790.c
index 0a1efc1417bc..2ab91b993c33 100644
--- a/drivers/media/i2c/m52790.c
+++ b/drivers/media/i2c/m52790.c
@@ -154,12 +154,11 @@ static int m52790_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int m52790_remove(struct i2c_client *client)
+static void m52790_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/m5mols/m5mols_core.c b/drivers/media/i2c/m5mols/m5mols_core.c
index c19590389bfe..2201d2a26353 100644
--- a/drivers/media/i2c/m5mols/m5mols_core.c
+++ b/drivers/media/i2c/m5mols/m5mols_core.c
@@ -1020,15 +1020,13 @@ error:
 	return ret;
 }
 
-static int m5mols_remove(struct i2c_client *client)
+static void m5mols_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	media_entity_cleanup(&sd->entity);
-
-	return 0;
 }
 
 static const struct i2c_device_id m5mols_id[] = {
diff --git a/drivers/media/i2c/max2175.c b/drivers/media/i2c/max2175.c
index 0eea200124d2..1019020f3a37 100644
--- a/drivers/media/i2c/max2175.c
+++ b/drivers/media/i2c/max2175.c
@@ -1403,15 +1403,13 @@ err_reg:
 	return ret;
 }
 
-static int max2175_remove(struct i2c_client *client)
+static void max2175_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct max2175 *ctx = max2175_from_sd(sd);
 
 	v4l2_ctrl_handler_free(&ctx->ctrl_hdl);
 	v4l2_async_unregister_subdev(sd);
-
-	return 0;
 }
 
 static const struct i2c_device_id max2175_id[] = {
diff --git a/drivers/media/i2c/max9286.c b/drivers/media/i2c/max9286.c
index 3684faa72253..9c083cf14231 100644
--- a/drivers/media/i2c/max9286.c
+++ b/drivers/media/i2c/max9286.c
@@ -1378,7 +1378,7 @@ err_powerdown:
 	return ret;
 }
 
-static int max9286_remove(struct i2c_client *client)
+static void max9286_remove(struct i2c_client *client)
 {
 	struct max9286_priv *priv = sd_to_max9286(i2c_get_clientdata(client));
 
@@ -1391,8 +1391,6 @@ static int max9286_remove(struct i2c_client *client)
 	gpiod_set_value_cansleep(priv->gpiod_pwdn, 0);
 
 	max9286_cleanup_dt(priv);
-
-	return 0;
 }
 
 static const struct of_device_id max9286_dt_ids[] = {
diff --git a/drivers/media/i2c/ml86v7667.c b/drivers/media/i2c/ml86v7667.c
index 48cc0b0922f4..49ec59b0ca43 100644
--- a/drivers/media/i2c/ml86v7667.c
+++ b/drivers/media/i2c/ml86v7667.c
@@ -415,15 +415,13 @@ cleanup:
 	return ret;
 }
 
-static int ml86v7667_remove(struct i2c_client *client)
+static void ml86v7667_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ml86v7667_priv *priv = to_ml86v7667(sd);
 
 	v4l2_ctrl_handler_free(&priv->hdl);
 	v4l2_device_unregister_subdev(&priv->sd);
-
-	return 0;
 }
 
 static const struct i2c_device_id ml86v7667_id[] = {
diff --git a/drivers/media/i2c/msp3400-driver.c b/drivers/media/i2c/msp3400-driver.c
index 39530d43590e..4ce7a15a9884 100644
--- a/drivers/media/i2c/msp3400-driver.c
+++ b/drivers/media/i2c/msp3400-driver.c
@@ -859,7 +859,7 @@ static int msp_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	return 0;
 }
 
-static int msp_remove(struct i2c_client *client)
+static void msp_remove(struct i2c_client *client)
 {
 	struct msp_state *state = to_state(i2c_get_clientdata(client));
 
@@ -872,7 +872,6 @@ static int msp_remove(struct i2c_client *client)
 	msp_reset(client);
 
 	v4l2_ctrl_handler_free(&state->hdl);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/mt9m001.c b/drivers/media/i2c/mt9m001.c
index ad13b0c890c0..ebf9cf1e1bce 100644
--- a/drivers/media/i2c/mt9m001.c
+++ b/drivers/media/i2c/mt9m001.c
@@ -833,7 +833,7 @@ error_hdl_free:
 	return ret;
 }
 
-static int mt9m001_remove(struct i2c_client *client)
+static void mt9m001_remove(struct i2c_client *client)
 {
 	struct mt9m001 *mt9m001 = to_mt9m001(client);
 
@@ -853,8 +853,6 @@ static int mt9m001_remove(struct i2c_client *client)
 
 	v4l2_ctrl_handler_free(&mt9m001->hdl);
 	mutex_destroy(&mt9m001->mutex);
-
-	return 0;
 }
 
 static const struct i2c_device_id mt9m001_id[] = {
diff --git a/drivers/media/i2c/mt9m032.c b/drivers/media/i2c/mt9m032.c
index ba0c0ea91c95..76b8c9c08c82 100644
--- a/drivers/media/i2c/mt9m032.c
+++ b/drivers/media/i2c/mt9m032.c
@@ -858,7 +858,7 @@ error_sensor:
 	return ret;
 }
 
-static int mt9m032_remove(struct i2c_client *client)
+static void mt9m032_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
 	struct mt9m032 *sensor = to_mt9m032(subdev);
@@ -867,7 +867,6 @@ static int mt9m032_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(&sensor->ctrls);
 	media_entity_cleanup(&subdev->entity);
 	mutex_destroy(&sensor->lock);
-	return 0;
 }
 
 static const struct i2c_device_id mt9m032_id_table[] = {
diff --git a/drivers/media/i2c/mt9m111.c b/drivers/media/i2c/mt9m111.c
index afc86efa9e3e..f5fe272d1205 100644
--- a/drivers/media/i2c/mt9m111.c
+++ b/drivers/media/i2c/mt9m111.c
@@ -1359,15 +1359,13 @@ out_hdlfree:
 	return ret;
 }
 
-static int mt9m111_remove(struct i2c_client *client)
+static void mt9m111_remove(struct i2c_client *client)
 {
 	struct mt9m111 *mt9m111 = to_mt9m111(client);
 
 	v4l2_async_unregister_subdev(&mt9m111->subdev);
 	media_entity_cleanup(&mt9m111->subdev.entity);
 	v4l2_ctrl_handler_free(&mt9m111->hdl);
-
-	return 0;
 }
 static const struct of_device_id mt9m111_of_match[] = {
 	{ .compatible = "micron,mt9m111", },
diff --git a/drivers/media/i2c/mt9p031.c b/drivers/media/i2c/mt9p031.c
index 1fd4dc6e4726..45f7b5e52bc3 100644
--- a/drivers/media/i2c/mt9p031.c
+++ b/drivers/media/i2c/mt9p031.c
@@ -1209,7 +1209,7 @@ done:
 	return ret;
 }
 
-static int mt9p031_remove(struct i2c_client *client)
+static void mt9p031_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
 	struct mt9p031 *mt9p031 = to_mt9p031(subdev);
@@ -1218,8 +1218,6 @@ static int mt9p031_remove(struct i2c_client *client)
 	v4l2_async_unregister_subdev(subdev);
 	media_entity_cleanup(&subdev->entity);
 	mutex_destroy(&mt9p031->power_lock);
-
-	return 0;
 }
 
 static const struct i2c_device_id mt9p031_id[] = {
diff --git a/drivers/media/i2c/mt9t001.c b/drivers/media/i2c/mt9t001.c
index b651ee4a26e8..d5abe4a7ef07 100644
--- a/drivers/media/i2c/mt9t001.c
+++ b/drivers/media/i2c/mt9t001.c
@@ -961,7 +961,7 @@ done:
 	return ret;
 }
 
-static int mt9t001_remove(struct i2c_client *client)
+static void mt9t001_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
 	struct mt9t001 *mt9t001 = to_mt9t001(subdev);
@@ -969,7 +969,6 @@ static int mt9t001_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(&mt9t001->ctrls);
 	v4l2_device_unregister_subdev(subdev);
 	media_entity_cleanup(&subdev->entity);
-	return 0;
 }
 
 static const struct i2c_device_id mt9t001_id[] = {
diff --git a/drivers/media/i2c/mt9t112.c b/drivers/media/i2c/mt9t112.c
index 8d2e3caa9b28..ad564095d0cf 100644
--- a/drivers/media/i2c/mt9t112.c
+++ b/drivers/media/i2c/mt9t112.c
@@ -1102,14 +1102,12 @@ static int mt9t112_probe(struct i2c_client *client,
 	return v4l2_async_register_subdev(&priv->subdev);
 }
 
-static int mt9t112_remove(struct i2c_client *client)
+static void mt9t112_remove(struct i2c_client *client)
 {
 	struct mt9t112_priv *priv = to_mt9t112(client);
 
 	clk_disable_unprepare(priv->clk);
 	v4l2_async_unregister_subdev(&priv->subdev);
-
-	return 0;
 }
 
 static const struct i2c_device_id mt9t112_id[] = {
diff --git a/drivers/media/i2c/mt9v011.c b/drivers/media/i2c/mt9v011.c
index 7699e64e1127..9952ce06ebb2 100644
--- a/drivers/media/i2c/mt9v011.c
+++ b/drivers/media/i2c/mt9v011.c
@@ -561,7 +561,7 @@ static int mt9v011_probe(struct i2c_client *c,
 	return 0;
 }
 
-static int mt9v011_remove(struct i2c_client *c)
+static void mt9v011_remove(struct i2c_client *c)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(c);
 	struct mt9v011 *core = to_mt9v011(sd);
@@ -572,8 +572,6 @@ static int mt9v011_remove(struct i2c_client *c)
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&core->ctrls);
-
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/mt9v032.c b/drivers/media/i2c/mt9v032.c
index 4cfdd3dfbd42..bc4388ccc2a8 100644
--- a/drivers/media/i2c/mt9v032.c
+++ b/drivers/media/i2c/mt9v032.c
@@ -1192,7 +1192,7 @@ err:
 	return ret;
 }
 
-static int mt9v032_remove(struct i2c_client *client)
+static void mt9v032_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
 	struct mt9v032 *mt9v032 = to_mt9v032(subdev);
@@ -1200,8 +1200,6 @@ static int mt9v032_remove(struct i2c_client *client)
 	v4l2_async_unregister_subdev(subdev);
 	v4l2_ctrl_handler_free(&mt9v032->ctrls);
 	media_entity_cleanup(&subdev->entity);
-
-	return 0;
 }
 
 static const struct mt9v032_model_data mt9v032_model_data[] = {
diff --git a/drivers/media/i2c/mt9v111.c b/drivers/media/i2c/mt9v111.c
index 2dc4a0f24ce8..fe18e5258d7a 100644
--- a/drivers/media/i2c/mt9v111.c
+++ b/drivers/media/i2c/mt9v111.c
@@ -1238,7 +1238,7 @@ error_free_ctrls:
 	return ret;
 }
 
-static int mt9v111_remove(struct i2c_client *client)
+static void mt9v111_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct mt9v111_dev *mt9v111 = sd_to_mt9v111(sd);
@@ -1253,8 +1253,6 @@ static int mt9v111_remove(struct i2c_client *client)
 
 	mutex_destroy(&mt9v111->pwr_mutex);
 	mutex_destroy(&mt9v111->stream_mutex);
-
-	return 0;
 }
 
 static const struct of_device_id mt9v111_of_match[] = {
diff --git a/drivers/media/i2c/noon010pc30.c b/drivers/media/i2c/noon010pc30.c
index bc5187f46365..ecaf5e9057f1 100644
--- a/drivers/media/i2c/noon010pc30.c
+++ b/drivers/media/i2c/noon010pc30.c
@@ -789,7 +789,7 @@ np_err:
 	return ret;
 }
 
-static int noon010_remove(struct i2c_client *client)
+static void noon010_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct noon010_info *info = to_noon010(sd);
@@ -797,8 +797,6 @@ static int noon010_remove(struct i2c_client *client)
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&info->hdl);
 	media_entity_cleanup(&sd->entity);
-
-	return 0;
 }
 
 static const struct i2c_device_id noon010_id[] = {
diff --git a/drivers/media/i2c/og01a1b.c b/drivers/media/i2c/og01a1b.c
index 87179fc04e00..35663c10fcd9 100644
--- a/drivers/media/i2c/og01a1b.c
+++ b/drivers/media/i2c/og01a1b.c
@@ -1015,7 +1015,7 @@ check_hwcfg_error:
 	return ret;
 }
 
-static int og01a1b_remove(struct i2c_client *client)
+static void og01a1b_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct og01a1b *og01a1b = to_og01a1b(sd);
@@ -1025,8 +1025,6 @@ static int og01a1b_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	pm_runtime_disable(&client->dev);
 	mutex_destroy(&og01a1b->mutex);
-
-	return 0;
 }
 
 static int og01a1b_probe(struct i2c_client *client)
diff --git a/drivers/media/i2c/ov02a10.c b/drivers/media/i2c/ov02a10.c
index 0f08c05333ea..2c1eb724d8e5 100644
--- a/drivers/media/i2c/ov02a10.c
+++ b/drivers/media/i2c/ov02a10.c
@@ -975,7 +975,7 @@ err_destroy_mutex:
 	return ret;
 }
 
-static int ov02a10_remove(struct i2c_client *client)
+static void ov02a10_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov02a10 *ov02a10 = to_ov02a10(sd);
@@ -988,8 +988,6 @@ static int ov02a10_remove(struct i2c_client *client)
 		ov02a10_power_off(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
 	mutex_destroy(&ov02a10->mutex);
-
-	return 0;
 }
 
 static const struct of_device_id ov02a10_of_match[] = {
diff --git a/drivers/media/i2c/ov08d10.c b/drivers/media/i2c/ov08d10.c
index e5ef6466a3ec..c1703596c3dc 100644
--- a/drivers/media/i2c/ov08d10.c
+++ b/drivers/media/i2c/ov08d10.c
@@ -1415,7 +1415,7 @@ check_hwcfg_error:
 	return ret;
 }
 
-static int ov08d10_remove(struct i2c_client *client)
+static void ov08d10_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov08d10 *ov08d10 = to_ov08d10(sd);
@@ -1425,8 +1425,6 @@ static int ov08d10_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	pm_runtime_disable(&client->dev);
 	mutex_destroy(&ov08d10->mutex);
-
-	return 0;
 }
 
 static int ov08d10_probe(struct i2c_client *client)
diff --git a/drivers/media/i2c/ov13858.c b/drivers/media/i2c/ov13858.c
index d5fe67c763f7..e618b613e078 100644
--- a/drivers/media/i2c/ov13858.c
+++ b/drivers/media/i2c/ov13858.c
@@ -1769,7 +1769,7 @@ error_handler_free:
 	return ret;
 }
 
-static int ov13858_remove(struct i2c_client *client)
+static void ov13858_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov13858 *ov13858 = to_ov13858(sd);
@@ -1779,8 +1779,6 @@ static int ov13858_remove(struct i2c_client *client)
 	ov13858_free_controls(ov13858);
 
 	pm_runtime_disable(&client->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id ov13858_id_table[] = {
diff --git a/drivers/media/i2c/ov13b10.c b/drivers/media/i2c/ov13b10.c
index 7caeae641051..549e5d93e568 100644
--- a/drivers/media/i2c/ov13b10.c
+++ b/drivers/media/i2c/ov13b10.c
@@ -1447,7 +1447,7 @@ error_handler_free:
 	return ret;
 }
 
-static int ov13b10_remove(struct i2c_client *client)
+static void ov13b10_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov13b10 *ov13b = to_ov13b10(sd);
@@ -1457,8 +1457,6 @@ static int ov13b10_remove(struct i2c_client *client)
 	ov13b10_free_controls(ov13b);
 
 	pm_runtime_disable(&client->dev);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ov13b10_pm_ops = {
diff --git a/drivers/media/i2c/ov2640.c b/drivers/media/i2c/ov2640.c
index 4b75da55b260..29ed0ef8c033 100644
--- a/drivers/media/i2c/ov2640.c
+++ b/drivers/media/i2c/ov2640.c
@@ -1271,7 +1271,7 @@ err_clk:
 	return ret;
 }
 
-static int ov2640_remove(struct i2c_client *client)
+static void ov2640_remove(struct i2c_client *client)
 {
 	struct ov2640_priv       *priv = to_ov2640(client);
 
@@ -1281,7 +1281,6 @@ static int ov2640_remove(struct i2c_client *client)
 	media_entity_cleanup(&priv->subdev.entity);
 	v4l2_device_unregister_subdev(&priv->subdev);
 	clk_disable_unprepare(priv->clk);
-	return 0;
 }
 
 static const struct i2c_device_id ov2640_id[] = {
diff --git a/drivers/media/i2c/ov2659.c b/drivers/media/i2c/ov2659.c
index 13ded5b2aa66..42fc64ada08c 100644
--- a/drivers/media/i2c/ov2659.c
+++ b/drivers/media/i2c/ov2659.c
@@ -1544,7 +1544,7 @@ error:
 	return ret;
 }
 
-static int ov2659_remove(struct i2c_client *client)
+static void ov2659_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov2659 *ov2659 = to_ov2659(sd);
@@ -1558,8 +1558,6 @@ static int ov2659_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(&client->dev))
 		ov2659_power_off(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ov2659_pm_ops = {
diff --git a/drivers/media/i2c/ov2680.c b/drivers/media/i2c/ov2680.c
index 906c711f6821..de66d3395a4d 100644
--- a/drivers/media/i2c/ov2680.c
+++ b/drivers/media/i2c/ov2680.c
@@ -1097,7 +1097,7 @@ lock_destroy:
 	return ret;
 }
 
-static int ov2680_remove(struct i2c_client *client)
+static void ov2680_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov2680_dev *sensor = to_ov2680_dev(sd);
@@ -1106,8 +1106,6 @@ static int ov2680_remove(struct i2c_client *client)
 	mutex_destroy(&sensor->lock);
 	media_entity_cleanup(&sensor->sd.entity);
 	v4l2_ctrl_handler_free(&sensor->ctrls.handler);
-
-	return 0;
 }
 
 static int __maybe_unused ov2680_suspend(struct device *dev)
diff --git a/drivers/media/i2c/ov2685.c b/drivers/media/i2c/ov2685.c
index b6e010ea3249..a3b524f15d89 100644
--- a/drivers/media/i2c/ov2685.c
+++ b/drivers/media/i2c/ov2685.c
@@ -798,7 +798,7 @@ err_destroy_mutex:
 	return ret;
 }
 
-static int ov2685_remove(struct i2c_client *client)
+static void ov2685_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov2685 *ov2685 = to_ov2685(sd);
@@ -814,8 +814,6 @@ static int ov2685_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(&client->dev))
 		__ov2685_power_off(ov2685);
 	pm_runtime_set_suspended(&client->dev);
-
-	return 0;
 }
 
 #if IS_ENABLED(CONFIG_OF)
diff --git a/drivers/media/i2c/ov2740.c b/drivers/media/i2c/ov2740.c
index d5f0eabf20c6..5d74ad479214 100644
--- a/drivers/media/i2c/ov2740.c
+++ b/drivers/media/i2c/ov2740.c
@@ -1053,7 +1053,7 @@ check_hwcfg_error:
 	return ret;
 }
 
-static int ov2740_remove(struct i2c_client *client)
+static void ov2740_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov2740 *ov2740 = to_ov2740(sd);
@@ -1063,8 +1063,6 @@ static int ov2740_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	pm_runtime_disable(&client->dev);
 	mutex_destroy(&ov2740->mutex);
-
-	return 0;
 }
 
 static int ov2740_nvmem_read(void *priv, unsigned int off, void *val,
diff --git a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c
index 502f0b62e950..1852e1cfc7df 100644
--- a/drivers/media/i2c/ov5640.c
+++ b/drivers/media/i2c/ov5640.c
@@ -3906,7 +3906,7 @@ entity_cleanup:
 	return ret;
 }
 
-static int ov5640_remove(struct i2c_client *client)
+static void ov5640_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov5640_dev *sensor = to_ov5640_dev(sd);
@@ -3915,8 +3915,6 @@ static int ov5640_remove(struct i2c_client *client)
 	media_entity_cleanup(&sensor->sd.entity);
 	v4l2_ctrl_handler_free(&sensor->ctrls.handler);
 	mutex_destroy(&sensor->lock);
-
-	return 0;
 }
 
 static const struct i2c_device_id ov5640_id[] = {
diff --git a/drivers/media/i2c/ov5645.c b/drivers/media/i2c/ov5645.c
index 562c62f192c4..81e4e87e1821 100644
--- a/drivers/media/i2c/ov5645.c
+++ b/drivers/media/i2c/ov5645.c
@@ -1256,7 +1256,7 @@ free_ctrl:
 	return ret;
 }
 
-static int ov5645_remove(struct i2c_client *client)
+static void ov5645_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov5645 *ov5645 = to_ov5645(sd);
@@ -1265,8 +1265,6 @@ static int ov5645_remove(struct i2c_client *client)
 	media_entity_cleanup(&ov5645->sd.entity);
 	v4l2_ctrl_handler_free(&ov5645->ctrls);
 	mutex_destroy(&ov5645->power_lock);
-
-	return 0;
 }
 
 static const struct i2c_device_id ov5645_id[] = {
diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c
index d346d18ce629..847a7bbb69c5 100644
--- a/drivers/media/i2c/ov5647.c
+++ b/drivers/media/i2c/ov5647.c
@@ -1448,7 +1448,7 @@ mutex_destroy:
 	return ret;
 }
 
-static int ov5647_remove(struct i2c_client *client)
+static void ov5647_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov5647 *sensor = to_sensor(sd);
@@ -1459,8 +1459,6 @@ static int ov5647_remove(struct i2c_client *client)
 	v4l2_device_unregister_subdev(sd);
 	pm_runtime_disable(&client->dev);
 	mutex_destroy(&sensor->lock);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ov5647_pm_ops = {
diff --git a/drivers/media/i2c/ov5648.c b/drivers/media/i2c/ov5648.c
index dfcd33e9ee13..84604ea7bdf9 100644
--- a/drivers/media/i2c/ov5648.c
+++ b/drivers/media/i2c/ov5648.c
@@ -2587,7 +2587,7 @@ error_endpoint:
 	return ret;
 }
 
-static int ov5648_remove(struct i2c_client *client)
+static void ov5648_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
 	struct ov5648_sensor *sensor = ov5648_subdev_sensor(subdev);
@@ -2597,8 +2597,6 @@ static int ov5648_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(&sensor->ctrls.handler);
 	mutex_destroy(&sensor->mutex);
 	media_entity_cleanup(&subdev->entity);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ov5648_pm_ops = {
diff --git a/drivers/media/i2c/ov5670.c b/drivers/media/i2c/ov5670.c
index 02f75c18e480..bc9fc3bc90c2 100644
--- a/drivers/media/i2c/ov5670.c
+++ b/drivers/media/i2c/ov5670.c
@@ -2557,7 +2557,7 @@ error_print:
 	return ret;
 }
 
-static int ov5670_remove(struct i2c_client *client)
+static void ov5670_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov5670 *ov5670 = to_ov5670(sd);
@@ -2568,8 +2568,6 @@ static int ov5670_remove(struct i2c_client *client)
 	mutex_destroy(&ov5670->mutex);
 
 	pm_runtime_disable(&client->dev);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ov5670_pm_ops = {
diff --git a/drivers/media/i2c/ov5675.c b/drivers/media/i2c/ov5675.c
index 82ba9f56baec..94dc8cb7a7c0 100644
--- a/drivers/media/i2c/ov5675.c
+++ b/drivers/media/i2c/ov5675.c
@@ -1175,7 +1175,7 @@ check_hwcfg_error:
 	return ret;
 }
 
-static int ov5675_remove(struct i2c_client *client)
+static void ov5675_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov5675 *ov5675 = to_ov5675(sd);
@@ -1185,8 +1185,6 @@ static int ov5675_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	pm_runtime_disable(&client->dev);
 	mutex_destroy(&ov5675->mutex);
-
-	return 0;
 }
 
 static int ov5675_probe(struct i2c_client *client)
diff --git a/drivers/media/i2c/ov5693.c b/drivers/media/i2c/ov5693.c
index 82a9b2de7735..a97ec132ba3a 100644
--- a/drivers/media/i2c/ov5693.c
+++ b/drivers/media/i2c/ov5693.c
@@ -1501,7 +1501,7 @@ err_ctrl_handler_free:
 	return ret;
 }
 
-static int ov5693_remove(struct i2c_client *client)
+static void ov5693_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov5693_device *ov5693 = to_ov5693_sensor(sd);
@@ -1519,8 +1519,6 @@ static int ov5693_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(&client->dev))
 		ov5693_sensor_powerdown(ov5693);
 	pm_runtime_set_suspended(&client->dev);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ov5693_pm_ops = {
diff --git a/drivers/media/i2c/ov5695.c b/drivers/media/i2c/ov5695.c
index 910309783885..61906fc54e37 100644
--- a/drivers/media/i2c/ov5695.c
+++ b/drivers/media/i2c/ov5695.c
@@ -1361,7 +1361,7 @@ err_destroy_mutex:
 	return ret;
 }
 
-static int ov5695_remove(struct i2c_client *client)
+static void ov5695_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov5695 *ov5695 = to_ov5695(sd);
@@ -1377,8 +1377,6 @@ static int ov5695_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(&client->dev))
 		__ov5695_power_off(ov5695);
 	pm_runtime_set_suspended(&client->dev);
-
-	return 0;
 }
 
 #if IS_ENABLED(CONFIG_OF)
diff --git a/drivers/media/i2c/ov6650.c b/drivers/media/i2c/ov6650.c
index 6458e96d9091..18f041e985b7 100644
--- a/drivers/media/i2c/ov6650.c
+++ b/drivers/media/i2c/ov6650.c
@@ -1096,13 +1096,12 @@ ectlhdlfree:
 	return ret;
 }
 
-static int ov6650_remove(struct i2c_client *client)
+static void ov6650_remove(struct i2c_client *client)
 {
 	struct ov6650 *priv = to_ov6650(client);
 
 	v4l2_async_unregister_subdev(&priv->subdev);
 	v4l2_ctrl_handler_free(&priv->hdl);
-	return 0;
 }
 
 static const struct i2c_device_id ov6650_id[] = {
diff --git a/drivers/media/i2c/ov7251.c b/drivers/media/i2c/ov7251.c
index 1bd797c7926b..88e987435285 100644
--- a/drivers/media/i2c/ov7251.c
+++ b/drivers/media/i2c/ov7251.c
@@ -1767,7 +1767,7 @@ destroy_mutex:
 	return ret;
 }
 
-static int ov7251_remove(struct i2c_client *client)
+static void ov7251_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov7251 *ov7251 = to_ov7251(sd);
@@ -1781,8 +1781,6 @@ static int ov7251_remove(struct i2c_client *client)
 	if (!pm_runtime_status_suspended(ov7251->dev))
 		ov7251_set_power_off(ov7251->dev);
 	pm_runtime_set_suspended(ov7251->dev);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ov7251_pm_ops = {
diff --git a/drivers/media/i2c/ov7640.c b/drivers/media/i2c/ov7640.c
index 977cd2d8ad33..5e2d67f0f9f2 100644
--- a/drivers/media/i2c/ov7640.c
+++ b/drivers/media/i2c/ov7640.c
@@ -70,13 +70,11 @@ static int ov7640_probe(struct i2c_client *client,
 }
 
 
-static int ov7640_remove(struct i2c_client *client)
+static void ov7640_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-
-	return 0;
 }
 
 static const struct i2c_device_id ov7640_id[] = {
diff --git a/drivers/media/i2c/ov7670.c b/drivers/media/i2c/ov7670.c
index 1be2c0e5bdc1..4b9b156b53c7 100644
--- a/drivers/media/i2c/ov7670.c
+++ b/drivers/media/i2c/ov7670.c
@@ -2009,7 +2009,7 @@ power_off:
 	return ret;
 }
 
-static int ov7670_remove(struct i2c_client *client)
+static void ov7670_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov7670_info *info = to_state(sd);
@@ -2017,7 +2017,6 @@ static int ov7670_remove(struct i2c_client *client)
 	v4l2_async_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&info->hdl);
 	media_entity_cleanup(&info->sd.entity);
-	return 0;
 }
 
 static const struct i2c_device_id ov7670_id[] = {
diff --git a/drivers/media/i2c/ov772x.c b/drivers/media/i2c/ov772x.c
index 78602a2f70b0..4189e3fc3d53 100644
--- a/drivers/media/i2c/ov772x.c
+++ b/drivers/media/i2c/ov772x.c
@@ -1521,7 +1521,7 @@ error_mutex_destroy:
 	return ret;
 }
 
-static int ov772x_remove(struct i2c_client *client)
+static void ov772x_remove(struct i2c_client *client)
 {
 	struct ov772x_priv *priv = to_ov772x(i2c_get_clientdata(client));
 
@@ -1532,8 +1532,6 @@ static int ov772x_remove(struct i2c_client *client)
 	v4l2_async_unregister_subdev(&priv->subdev);
 	v4l2_ctrl_handler_free(&priv->hdl);
 	mutex_destroy(&priv->lock);
-
-	return 0;
 }
 
 static const struct i2c_device_id ov772x_id[] = {
diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c
index 2539cfee85c8..c9fd9b0bc54a 100644
--- a/drivers/media/i2c/ov7740.c
+++ b/drivers/media/i2c/ov7740.c
@@ -1153,7 +1153,7 @@ error_detect:
 	return ret;
 }
 
-static int ov7740_remove(struct i2c_client *client)
+static void ov7740_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev);
@@ -1170,7 +1170,6 @@ static int ov7740_remove(struct i2c_client *client)
 	pm_runtime_put_noidle(&client->dev);
 
 	ov7740_set_power(ov7740, 0);
-	return 0;
 }
 
 static int __maybe_unused ov7740_runtime_suspend(struct device *dev)
diff --git a/drivers/media/i2c/ov8856.c b/drivers/media/i2c/ov8856.c
index a9728afc81d4..efa18d026ac3 100644
--- a/drivers/media/i2c/ov8856.c
+++ b/drivers/media/i2c/ov8856.c
@@ -2440,7 +2440,7 @@ check_hwcfg_error:
 	return ret;
 }
 
-static int ov8856_remove(struct i2c_client *client)
+static void ov8856_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov8856 *ov8856 = to_ov8856(sd);
@@ -2452,8 +2452,6 @@ static int ov8856_remove(struct i2c_client *client)
 	mutex_destroy(&ov8856->mutex);
 
 	__ov8856_power_off(ov8856);
-
-	return 0;
 }
 
 static int ov8856_probe(struct i2c_client *client)
diff --git a/drivers/media/i2c/ov8865.c b/drivers/media/i2c/ov8865.c
index b8f4f0d3e33d..a233c34b168e 100644
--- a/drivers/media/i2c/ov8865.c
+++ b/drivers/media/i2c/ov8865.c
@@ -3119,7 +3119,7 @@ error_endpoint:
 	return ret;
 }
 
-static int ov8865_remove(struct i2c_client *client)
+static void ov8865_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *subdev = i2c_get_clientdata(client);
 	struct ov8865_sensor *sensor = ov8865_subdev_sensor(subdev);
@@ -3131,8 +3131,6 @@ static int ov8865_remove(struct i2c_client *client)
 	media_entity_cleanup(&subdev->entity);
 
 	v4l2_fwnode_endpoint_free(&sensor->endpoint);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ov8865_pm_ops = {
diff --git a/drivers/media/i2c/ov9282.c b/drivers/media/i2c/ov9282.c
index 2e0b315801e5..df144a2f6eda 100644
--- a/drivers/media/i2c/ov9282.c
+++ b/drivers/media/i2c/ov9282.c
@@ -1091,7 +1091,7 @@ error_mutex_destroy:
  *
  * Return: 0 if successful, error code otherwise.
  */
-static int ov9282_remove(struct i2c_client *client)
+static void ov9282_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov9282 *ov9282 = to_ov9282(sd);
@@ -1106,8 +1106,6 @@ static int ov9282_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 
 	mutex_destroy(&ov9282->mutex);
-
-	return 0;
 }
 
 static const struct dev_pm_ops ov9282_pm_ops = {
diff --git a/drivers/media/i2c/ov9640.c b/drivers/media/i2c/ov9640.c
index 9f44ed52d164..8b80be33c5f4 100644
--- a/drivers/media/i2c/ov9640.c
+++ b/drivers/media/i2c/ov9640.c
@@ -744,15 +744,13 @@ ectrlinit:
 	return ret;
 }
 
-static int ov9640_remove(struct i2c_client *client)
+static void ov9640_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov9640_priv *priv = to_ov9640_sensor(sd);
 
 	v4l2_async_unregister_subdev(&priv->subdev);
 	v4l2_ctrl_handler_free(&priv->hdl);
-
-	return 0;
 }
 
 static const struct i2c_device_id ov9640_id[] = {
diff --git a/drivers/media/i2c/ov9650.c b/drivers/media/i2c/ov9650.c
index c313e11a9754..4d458993e6d6 100644
--- a/drivers/media/i2c/ov9650.c
+++ b/drivers/media/i2c/ov9650.c
@@ -1584,7 +1584,7 @@ err_mutex:
 	return ret;
 }
 
-static int ov965x_remove(struct i2c_client *client)
+static void ov965x_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov965x *ov965x = to_ov965x(sd);
@@ -1593,8 +1593,6 @@ static int ov965x_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	media_entity_cleanup(&sd->entity);
 	mutex_destroy(&ov965x->lock);
-
-	return 0;
 }
 
 static const struct i2c_device_id ov965x_id[] = {
diff --git a/drivers/media/i2c/ov9734.c b/drivers/media/i2c/ov9734.c
index df538ceb71c3..8b0a158cb297 100644
--- a/drivers/media/i2c/ov9734.c
+++ b/drivers/media/i2c/ov9734.c
@@ -930,7 +930,7 @@ check_hwcfg_error:
 	return ret;
 }
 
-static int ov9734_remove(struct i2c_client *client)
+static void ov9734_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov9734 *ov9734 = to_ov9734(sd);
@@ -940,8 +940,6 @@ static int ov9734_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	pm_runtime_disable(&client->dev);
 	mutex_destroy(&ov9734->mutex);
-
-	return 0;
 }
 
 static int ov9734_probe(struct i2c_client *client)
diff --git a/drivers/media/i2c/rdacm20.c b/drivers/media/i2c/rdacm20.c
index 2615ad154f49..a2263fa825b5 100644
--- a/drivers/media/i2c/rdacm20.c
+++ b/drivers/media/i2c/rdacm20.c
@@ -646,7 +646,7 @@ error:
 	return ret;
 }
 
-static int rdacm20_remove(struct i2c_client *client)
+static void rdacm20_remove(struct i2c_client *client)
 {
 	struct rdacm20_device *dev = i2c_to_rdacm20(client);
 
@@ -655,8 +655,6 @@ static int rdacm20_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(&dev->ctrls);
 	media_entity_cleanup(&dev->sd.entity);
 	i2c_unregister_device(dev->sensor);
-
-	return 0;
 }
 
 static void rdacm20_shutdown(struct i2c_client *client)
diff --git a/drivers/media/i2c/rdacm21.c b/drivers/media/i2c/rdacm21.c
index ef31cf5f23ca..9ccc56c30d3b 100644
--- a/drivers/media/i2c/rdacm21.c
+++ b/drivers/media/i2c/rdacm21.c
@@ -614,7 +614,7 @@ error:
 	return ret;
 }
 
-static int rdacm21_remove(struct i2c_client *client)
+static void rdacm21_remove(struct i2c_client *client)
 {
 	struct rdacm21_device *dev = sd_to_rdacm21(i2c_get_clientdata(client));
 
@@ -622,8 +622,6 @@ static int rdacm21_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(&dev->ctrls);
 	i2c_unregister_device(dev->isp);
 	fwnode_handle_put(dev->sd.fwnode);
-
-	return 0;
 }
 
 static const struct of_device_id rdacm21_of_ids[] = {
diff --git a/drivers/media/i2c/rj54n1cb0c.c b/drivers/media/i2c/rj54n1cb0c.c
index 2e4018c26912..1c3502f34cd3 100644
--- a/drivers/media/i2c/rj54n1cb0c.c
+++ b/drivers/media/i2c/rj54n1cb0c.c
@@ -1398,7 +1398,7 @@ err_free_ctrl:
 	return ret;
 }
 
-static int rj54n1_remove(struct i2c_client *client)
+static void rj54n1_remove(struct i2c_client *client)
 {
 	struct rj54n1 *rj54n1 = to_rj54n1(client);
 
@@ -1410,8 +1410,6 @@ static int rj54n1_remove(struct i2c_client *client)
 	clk_put(rj54n1->clk);
 	v4l2_ctrl_handler_free(&rj54n1->hdl);
 	v4l2_async_unregister_subdev(&rj54n1->subdev);
-
-	return 0;
 }
 
 static const struct i2c_device_id rj54n1_id[] = {
diff --git a/drivers/media/i2c/s5c73m3/s5c73m3-core.c b/drivers/media/i2c/s5c73m3/s5c73m3-core.c
index e2b88c5e4f98..d96ba58ce1e5 100644
--- a/drivers/media/i2c/s5c73m3/s5c73m3-core.c
+++ b/drivers/media/i2c/s5c73m3/s5c73m3-core.c
@@ -1770,7 +1770,7 @@ out_err:
 	return ret;
 }
 
-static int s5c73m3_remove(struct i2c_client *client)
+static void s5c73m3_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *oif_sd = i2c_get_clientdata(client);
 	struct s5c73m3 *state = oif_sd_to_s5c73m3(oif_sd);
@@ -1785,8 +1785,6 @@ static int s5c73m3_remove(struct i2c_client *client)
 	media_entity_cleanup(&sensor_sd->entity);
 
 	s5c73m3_unregister_spi_driver(state);
-
-	return 0;
 }
 
 static const struct i2c_device_id s5c73m3_id[] = {
diff --git a/drivers/media/i2c/s5k4ecgx.c b/drivers/media/i2c/s5k4ecgx.c
index af9a305242cd..3dddcd9dd351 100644
--- a/drivers/media/i2c/s5k4ecgx.c
+++ b/drivers/media/i2c/s5k4ecgx.c
@@ -996,7 +996,7 @@ out_err1:
 	return ret;
 }
 
-static int s5k4ecgx_remove(struct i2c_client *client)
+static void s5k4ecgx_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct s5k4ecgx *priv = to_s5k4ecgx(sd);
@@ -1006,8 +1006,6 @@ static int s5k4ecgx_remove(struct i2c_client *client)
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&priv->handler);
 	media_entity_cleanup(&sd->entity);
-
-	return 0;
 }
 
 static const struct i2c_device_id s5k4ecgx_id[] = {
diff --git a/drivers/media/i2c/s5k5baf.c b/drivers/media/i2c/s5k5baf.c
index 6a5dceb699a8..5c2253ab3b6f 100644
--- a/drivers/media/i2c/s5k5baf.c
+++ b/drivers/media/i2c/s5k5baf.c
@@ -2018,7 +2018,7 @@ err_me:
 	return ret;
 }
 
-static int s5k5baf_remove(struct i2c_client *c)
+static void s5k5baf_remove(struct i2c_client *c)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(c);
 	struct s5k5baf *state = to_s5k5baf(sd);
@@ -2030,8 +2030,6 @@ static int s5k5baf_remove(struct i2c_client *c)
 	sd = &state->cis_sd;
 	v4l2_device_unregister_subdev(sd);
 	media_entity_cleanup(&sd->entity);
-
-	return 0;
 }
 
 static const struct i2c_device_id s5k5baf_id[] = {
diff --git a/drivers/media/i2c/s5k6a3.c b/drivers/media/i2c/s5k6a3.c
index f6ecf6f92bb2..a4efd6d10b43 100644
--- a/drivers/media/i2c/s5k6a3.c
+++ b/drivers/media/i2c/s5k6a3.c
@@ -354,14 +354,13 @@ static int s5k6a3_probe(struct i2c_client *client)
 	return ret;
 }
 
-static int s5k6a3_remove(struct i2c_client *client)
+static void s5k6a3_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	pm_runtime_disable(&client->dev);
 	v4l2_async_unregister_subdev(sd);
 	media_entity_cleanup(&sd->entity);
-	return 0;
 }
 
 static const struct i2c_device_id s5k6a3_ids[] = {
diff --git a/drivers/media/i2c/s5k6aa.c b/drivers/media/i2c/s5k6aa.c
index 105a4b7d8354..059211788a65 100644
--- a/drivers/media/i2c/s5k6aa.c
+++ b/drivers/media/i2c/s5k6aa.c
@@ -1621,15 +1621,13 @@ out_err:
 	return ret;
 }
 
-static int s5k6aa_remove(struct i2c_client *client)
+static void s5k6aa_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
 	media_entity_cleanup(&sd->entity);
-
-	return 0;
 }
 
 static const struct i2c_device_id s5k6aa_id[] = {
diff --git a/drivers/media/i2c/saa6588.c b/drivers/media/i2c/saa6588.c
index d1e0716bdfff..d6a51beabd02 100644
--- a/drivers/media/i2c/saa6588.c
+++ b/drivers/media/i2c/saa6588.c
@@ -484,7 +484,7 @@ static int saa6588_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int saa6588_remove(struct i2c_client *client)
+static void saa6588_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct saa6588 *s = to_saa6588(sd);
@@ -492,8 +492,6 @@ static int saa6588_remove(struct i2c_client *client)
 	v4l2_device_unregister_subdev(sd);
 
 	cancel_delayed_work_sync(&s->work);
-
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/saa6752hs.c b/drivers/media/i2c/saa6752hs.c
index a7f043cad149..5928cc6f4595 100644
--- a/drivers/media/i2c/saa6752hs.c
+++ b/drivers/media/i2c/saa6752hs.c
@@ -764,13 +764,12 @@ static int saa6752hs_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int saa6752hs_remove(struct i2c_client *client)
+static void saa6752hs_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&to_state(sd)->hdl);
-	return 0;
 }
 
 static const struct i2c_device_id saa6752hs_id[] = {
diff --git a/drivers/media/i2c/saa7110.c b/drivers/media/i2c/saa7110.c
index 0c7a9ce0a693..5067525d8b11 100644
--- a/drivers/media/i2c/saa7110.c
+++ b/drivers/media/i2c/saa7110.c
@@ -428,14 +428,13 @@ static int saa7110_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int saa7110_remove(struct i2c_client *client)
+static void saa7110_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct saa7110 *decoder = to_saa7110(sd);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&decoder->hdl);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/saa7115.c b/drivers/media/i2c/saa7115.c
index 15ff80e6301e..86e70a980218 100644
--- a/drivers/media/i2c/saa7115.c
+++ b/drivers/media/i2c/saa7115.c
@@ -1927,13 +1927,12 @@ static int saa711x_probe(struct i2c_client *client,
 
 /* ----------------------------------------------------------------------- */
 
-static int saa711x_remove(struct i2c_client *client)
+static void saa711x_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	return 0;
 }
 
 static const struct i2c_device_id saa711x_id[] = {
diff --git a/drivers/media/i2c/saa7127.c b/drivers/media/i2c/saa7127.c
index 891192f6412a..78c9388c2ea1 100644
--- a/drivers/media/i2c/saa7127.c
+++ b/drivers/media/i2c/saa7127.c
@@ -785,14 +785,13 @@ static int saa7127_probe(struct i2c_client *client,
 
 /* ----------------------------------------------------------------------- */
 
-static int saa7127_remove(struct i2c_client *client)
+static void saa7127_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	/* Turn off TV output */
 	saa7127_set_video_enable(sd, 0);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/saa717x.c b/drivers/media/i2c/saa717x.c
index adf905360171..4f3d1b432a4e 100644
--- a/drivers/media/i2c/saa717x.c
+++ b/drivers/media/i2c/saa717x.c
@@ -1324,13 +1324,12 @@ static int saa717x_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int saa717x_remove(struct i2c_client *client)
+static void saa717x_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/saa7185.c b/drivers/media/i2c/saa7185.c
index 7a04422df8c8..266462325d30 100644
--- a/drivers/media/i2c/saa7185.c
+++ b/drivers/media/i2c/saa7185.c
@@ -322,7 +322,7 @@ static int saa7185_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int saa7185_remove(struct i2c_client *client)
+static void saa7185_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct saa7185 *encoder = to_saa7185(sd);
@@ -330,7 +330,6 @@ static int saa7185_remove(struct i2c_client *client)
 	v4l2_device_unregister_subdev(sd);
 	/* SW: output off is active */
 	saa7185_write(sd, 0x61, (encoder->reg[0x61]) | 0x40);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/sony-btf-mpx.c b/drivers/media/i2c/sony-btf-mpx.c
index ad239280c42e..927a9ec41463 100644
--- a/drivers/media/i2c/sony-btf-mpx.c
+++ b/drivers/media/i2c/sony-btf-mpx.c
@@ -357,13 +357,11 @@ static int sony_btf_mpx_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int sony_btf_mpx_remove(struct i2c_client *client)
+static void sony_btf_mpx_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/sr030pc30.c b/drivers/media/i2c/sr030pc30.c
index 19c0252df2f1..ff18693beb5c 100644
--- a/drivers/media/i2c/sr030pc30.c
+++ b/drivers/media/i2c/sr030pc30.c
@@ -732,13 +732,12 @@ static int sr030pc30_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int sr030pc30_remove(struct i2c_client *client)
+static void sr030pc30_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	return 0;
 }
 
 static const struct i2c_device_id sr030pc30_id[] = {
diff --git a/drivers/media/i2c/st-mipid02.c b/drivers/media/i2c/st-mipid02.c
index 16cc547976dd..31b89aff0e86 100644
--- a/drivers/media/i2c/st-mipid02.c
+++ b/drivers/media/i2c/st-mipid02.c
@@ -1067,7 +1067,7 @@ mutex_cleanup:
 	return ret;
 }
 
-static int mipid02_remove(struct i2c_client *client)
+static void mipid02_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct mipid02_dev *bridge = to_mipid02_dev(sd);
@@ -1078,8 +1078,6 @@ static int mipid02_remove(struct i2c_client *client)
 	mipid02_set_power_off(bridge);
 	media_entity_cleanup(&bridge->sd.entity);
 	mutex_destroy(&bridge->lock);
-
-	return 0;
 }
 
 static const struct of_device_id mipid02_dt_ids[] = {
diff --git a/drivers/media/i2c/tc358743.c b/drivers/media/i2c/tc358743.c
index e18b8947ad7e..d99eedbdf011 100644
--- a/drivers/media/i2c/tc358743.c
+++ b/drivers/media/i2c/tc358743.c
@@ -2169,7 +2169,7 @@ err_hdl:
 	return err;
 }
 
-static int tc358743_remove(struct i2c_client *client)
+static void tc358743_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct tc358743_state *state = to_state(sd);
@@ -2185,8 +2185,6 @@ static int tc358743_remove(struct i2c_client *client)
 	mutex_destroy(&state->confctl_mutex);
 	media_entity_cleanup(&sd->entity);
 	v4l2_ctrl_handler_free(&state->hdl);
-
-	return 0;
 }
 
 static const struct i2c_device_id tc358743_id[] = {
diff --git a/drivers/media/i2c/tda1997x.c b/drivers/media/i2c/tda1997x.c
index f66ac14cffad..83931826cf6f 100644
--- a/drivers/media/i2c/tda1997x.c
+++ b/drivers/media/i2c/tda1997x.c
@@ -2805,7 +2805,7 @@ err_free_state:
 	return ret;
 }
 
-static int tda1997x_remove(struct i2c_client *client)
+static void tda1997x_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct tda1997x_state *state = to_state(sd);
@@ -2827,8 +2827,6 @@ static int tda1997x_remove(struct i2c_client *client)
 	mutex_destroy(&state->lock);
 
 	kfree(state);
-
-	return 0;
 }
 
 static struct i2c_driver tda1997x_i2c_driver = {
diff --git a/drivers/media/i2c/tda7432.c b/drivers/media/i2c/tda7432.c
index cbdc9be0a597..11e918311b13 100644
--- a/drivers/media/i2c/tda7432.c
+++ b/drivers/media/i2c/tda7432.c
@@ -390,7 +390,7 @@ static int tda7432_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tda7432_remove(struct i2c_client *client)
+static void tda7432_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct tda7432 *t = to_state(sd);
@@ -398,7 +398,6 @@ static int tda7432_remove(struct i2c_client *client)
 	tda7432_set(sd);
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&t->hdl);
-	return 0;
 }
 
 static const struct i2c_device_id tda7432_id[] = {
diff --git a/drivers/media/i2c/tda9840.c b/drivers/media/i2c/tda9840.c
index 8c6dfe746b20..aaa74944fc7c 100644
--- a/drivers/media/i2c/tda9840.c
+++ b/drivers/media/i2c/tda9840.c
@@ -175,12 +175,11 @@ static int tda9840_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tda9840_remove(struct i2c_client *client)
+static void tda9840_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 static const struct i2c_device_id tda9840_id[] = {
diff --git a/drivers/media/i2c/tea6415c.c b/drivers/media/i2c/tea6415c.c
index 67378dbcc74b..50e74314f315 100644
--- a/drivers/media/i2c/tea6415c.c
+++ b/drivers/media/i2c/tea6415c.c
@@ -134,12 +134,11 @@ static int tea6415c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tea6415c_remove(struct i2c_client *client)
+static void tea6415c_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 static const struct i2c_device_id tea6415c_id[] = {
diff --git a/drivers/media/i2c/tea6420.c b/drivers/media/i2c/tea6420.c
index 712141b261ed..246f2b10ccc7 100644
--- a/drivers/media/i2c/tea6420.c
+++ b/drivers/media/i2c/tea6420.c
@@ -116,12 +116,11 @@ static int tea6420_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tea6420_remove(struct i2c_client *client)
+static void tea6420_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 static const struct i2c_device_id tea6420_id[] = {
diff --git a/drivers/media/i2c/ths7303.c b/drivers/media/i2c/ths7303.c
index 8206bf7a5a8f..2a0f9a3d1a66 100644
--- a/drivers/media/i2c/ths7303.c
+++ b/drivers/media/i2c/ths7303.c
@@ -358,13 +358,11 @@ static int ths7303_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int ths7303_remove(struct i2c_client *client)
+static void ths7303_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-
-	return 0;
 }
 
 static const struct i2c_device_id ths7303_id[] = {
diff --git a/drivers/media/i2c/ths8200.c b/drivers/media/i2c/ths8200.c
index c52fe84cba1b..081ef5a4b950 100644
--- a/drivers/media/i2c/ths8200.c
+++ b/drivers/media/i2c/ths8200.c
@@ -468,7 +468,7 @@ static int ths8200_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int ths8200_remove(struct i2c_client *client)
+static void ths8200_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ths8200_state *decoder = to_state(sd);
@@ -478,8 +478,6 @@ static int ths8200_remove(struct i2c_client *client)
 
 	ths8200_s_power(sd, false);
 	v4l2_async_unregister_subdev(&decoder->sd);
-
-	return 0;
 }
 
 static const struct i2c_device_id ths8200_id[] = {
diff --git a/drivers/media/i2c/tlv320aic23b.c b/drivers/media/i2c/tlv320aic23b.c
index e4c21990fea9..937fa1dbaecb 100644
--- a/drivers/media/i2c/tlv320aic23b.c
+++ b/drivers/media/i2c/tlv320aic23b.c
@@ -177,14 +177,13 @@ static int tlv320aic23b_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tlv320aic23b_remove(struct i2c_client *client)
+static void tlv320aic23b_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct tlv320aic23b_state *state = to_state(sd);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&state->hdl);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/tvaudio.c b/drivers/media/i2c/tvaudio.c
index e6796e94dadf..9f1ed078b661 100644
--- a/drivers/media/i2c/tvaudio.c
+++ b/drivers/media/i2c/tvaudio.c
@@ -2065,7 +2065,7 @@ static int tvaudio_probe(struct i2c_client *client, const struct i2c_device_id *
 	return 0;
 }
 
-static int tvaudio_remove(struct i2c_client *client)
+static void tvaudio_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct CHIPSTATE *chip = to_state(sd);
@@ -2079,7 +2079,6 @@ static int tvaudio_remove(struct i2c_client *client)
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&chip->hdl);
-	return 0;
 }
 
 /* This driver supports many devices and the idea is to let the driver
diff --git a/drivers/media/i2c/tvp514x.c b/drivers/media/i2c/tvp514x.c
index cee60f945036..a746d96875f9 100644
--- a/drivers/media/i2c/tvp514x.c
+++ b/drivers/media/i2c/tvp514x.c
@@ -1121,7 +1121,7 @@ done:
  * Unregister decoder as an i2c client device and V4L2
  * device. Complement of tvp514x_probe().
  */
-static int tvp514x_remove(struct i2c_client *client)
+static void tvp514x_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct tvp514x_decoder *decoder = to_decoder(sd);
@@ -1129,7 +1129,6 @@ static int tvp514x_remove(struct i2c_client *client)
 	v4l2_async_unregister_subdev(&decoder->sd);
 	media_entity_cleanup(&decoder->sd.entity);
 	v4l2_ctrl_handler_free(&decoder->hdl);
-	return 0;
 }
 /* TVP5146 Init/Power on Sequence */
 static const struct tvp514x_reg tvp5146_init_reg_seq[] = {
diff --git a/drivers/media/i2c/tvp5150.c b/drivers/media/i2c/tvp5150.c
index 93a980c4e899..859f1cb2fa74 100644
--- a/drivers/media/i2c/tvp5150.c
+++ b/drivers/media/i2c/tvp5150.c
@@ -2230,7 +2230,7 @@ err:
 	return res;
 }
 
-static int tvp5150_remove(struct i2c_client *c)
+static void tvp5150_remove(struct i2c_client *c)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(c);
 	struct tvp5150 *decoder = to_tvp5150(sd);
@@ -2250,8 +2250,6 @@ static int tvp5150_remove(struct i2c_client *c)
 	v4l2_ctrl_handler_free(&decoder->hdl);
 	pm_runtime_disable(&c->dev);
 	pm_runtime_set_suspended(&c->dev);
-
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/tvp7002.c b/drivers/media/i2c/tvp7002.c
index 2de18833b07b..4ccd218f5584 100644
--- a/drivers/media/i2c/tvp7002.c
+++ b/drivers/media/i2c/tvp7002.c
@@ -1044,7 +1044,7 @@ error:
  * Reset the TVP7002 device
  * Returns zero.
  */
-static int tvp7002_remove(struct i2c_client *c)
+static void tvp7002_remove(struct i2c_client *c)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(c);
 	struct tvp7002 *device = to_tvp7002(sd);
@@ -1056,7 +1056,6 @@ static int tvp7002_remove(struct i2c_client *c)
 	media_entity_cleanup(&device->sd.entity);
 #endif
 	v4l2_ctrl_handler_free(&device->hdl);
-	return 0;
 }
 
 /* I2C Device ID table */
diff --git a/drivers/media/i2c/tw2804.c b/drivers/media/i2c/tw2804.c
index cd05f1ff504d..c7c8dfe8a8a8 100644
--- a/drivers/media/i2c/tw2804.c
+++ b/drivers/media/i2c/tw2804.c
@@ -405,14 +405,13 @@ static int tw2804_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tw2804_remove(struct i2c_client *client)
+static void tw2804_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct tw2804 *state = to_state(sd);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&state->hdl);
-	return 0;
 }
 
 static const struct i2c_device_id tw2804_id[] = {
diff --git a/drivers/media/i2c/tw9903.c b/drivers/media/i2c/tw9903.c
index f8e3ab4909d8..d7eef7986b75 100644
--- a/drivers/media/i2c/tw9903.c
+++ b/drivers/media/i2c/tw9903.c
@@ -235,13 +235,12 @@ static int tw9903_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tw9903_remove(struct i2c_client *client)
+static void tw9903_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&to_state(sd)->hdl);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/tw9906.c b/drivers/media/i2c/tw9906.c
index c528eb01fed0..549ad8f72f12 100644
--- a/drivers/media/i2c/tw9906.c
+++ b/drivers/media/i2c/tw9906.c
@@ -203,13 +203,12 @@ static int tw9906_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tw9906_remove(struct i2c_client *client)
+static void tw9906_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&to_state(sd)->hdl);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/tw9910.c b/drivers/media/i2c/tw9910.c
index 09f5b3986928..853b5acead32 100644
--- a/drivers/media/i2c/tw9910.c
+++ b/drivers/media/i2c/tw9910.c
@@ -993,7 +993,7 @@ error_clk_put:
 	return ret;
 }
 
-static int tw9910_remove(struct i2c_client *client)
+static void tw9910_remove(struct i2c_client *client)
 {
 	struct tw9910_priv *priv = to_tw9910(client);
 
@@ -1001,8 +1001,6 @@ static int tw9910_remove(struct i2c_client *client)
 		gpiod_put(priv->pdn_gpio);
 	clk_put(priv->clk);
 	v4l2_async_unregister_subdev(&priv->subdev);
-
-	return 0;
 }
 
 static const struct i2c_device_id tw9910_id[] = {
diff --git a/drivers/media/i2c/uda1342.c b/drivers/media/i2c/uda1342.c
index b0a9c6d7163f..d0659c4392f2 100644
--- a/drivers/media/i2c/uda1342.c
+++ b/drivers/media/i2c/uda1342.c
@@ -72,12 +72,11 @@ static int uda1342_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int uda1342_remove(struct i2c_client *client)
+static void uda1342_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 static const struct i2c_device_id uda1342_id[] = {
diff --git a/drivers/media/i2c/upd64031a.c b/drivers/media/i2c/upd64031a.c
index ef35c6574785..4de26ed2ba00 100644
--- a/drivers/media/i2c/upd64031a.c
+++ b/drivers/media/i2c/upd64031a.c
@@ -210,12 +210,11 @@ static int upd64031a_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int upd64031a_remove(struct i2c_client *client)
+static void upd64031a_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/upd64083.c b/drivers/media/i2c/upd64083.c
index d6a1698caa2a..2bfd5443d406 100644
--- a/drivers/media/i2c/upd64083.c
+++ b/drivers/media/i2c/upd64083.c
@@ -181,12 +181,11 @@ static int upd64083_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int upd64083_remove(struct i2c_client *client)
+static void upd64083_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/video-i2c.c b/drivers/media/i2c/video-i2c.c
index e08e3579c0a1..f15ef2d13059 100644
--- a/drivers/media/i2c/video-i2c.c
+++ b/drivers/media/i2c/video-i2c.c
@@ -895,7 +895,7 @@ error_free_device:
 	return ret;
 }
 
-static int video_i2c_remove(struct i2c_client *client)
+static void video_i2c_remove(struct i2c_client *client)
 {
 	struct video_i2c_data *data = i2c_get_clientdata(client);
 
@@ -908,8 +908,6 @@ static int video_i2c_remove(struct i2c_client *client)
 		data->chip->set_power(data, false);
 
 	video_unregister_device(&data->vdev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/media/i2c/vp27smpx.c b/drivers/media/i2c/vp27smpx.c
index 492af8749fca..c832edad5fa7 100644
--- a/drivers/media/i2c/vp27smpx.c
+++ b/drivers/media/i2c/vp27smpx.c
@@ -163,12 +163,11 @@ static int vp27smpx_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int vp27smpx_remove(struct i2c_client *client)
+static void vp27smpx_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
-	return 0;
 }
 
 /* ----------------------------------------------------------------------- */
diff --git a/drivers/media/i2c/vpx3220.c b/drivers/media/i2c/vpx3220.c
index 8be03fe5928c..b481ec196b88 100644
--- a/drivers/media/i2c/vpx3220.c
+++ b/drivers/media/i2c/vpx3220.c
@@ -526,15 +526,13 @@ static int vpx3220_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int vpx3220_remove(struct i2c_client *client)
+static void vpx3220_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct vpx3220 *decoder = to_vpx3220(sd);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&decoder->hdl);
-
-	return 0;
 }
 
 static const struct i2c_device_id vpx3220_id[] = {
diff --git a/drivers/media/i2c/vs6624.c b/drivers/media/i2c/vs6624.c
index 29003dec6f2d..d496bb45f201 100644
--- a/drivers/media/i2c/vs6624.c
+++ b/drivers/media/i2c/vs6624.c
@@ -824,13 +824,12 @@ static int vs6624_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int vs6624_remove(struct i2c_client *client)
+static void vs6624_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
-	return 0;
 }
 
 static const struct i2c_device_id vs6624_id[] = {
diff --git a/drivers/media/i2c/wm8739.c b/drivers/media/i2c/wm8739.c
index ed533834db54..180b35347521 100644
--- a/drivers/media/i2c/wm8739.c
+++ b/drivers/media/i2c/wm8739.c
@@ -234,14 +234,13 @@ static int wm8739_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int wm8739_remove(struct i2c_client *client)
+static void wm8739_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct wm8739_state *state = to_state(sd);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&state->hdl);
-	return 0;
 }
 
 static const struct i2c_device_id wm8739_id[] = {
diff --git a/drivers/media/i2c/wm8775.c b/drivers/media/i2c/wm8775.c
index d4c83c39892a..8ff97867d3cd 100644
--- a/drivers/media/i2c/wm8775.c
+++ b/drivers/media/i2c/wm8775.c
@@ -280,14 +280,13 @@ static int wm8775_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int wm8775_remove(struct i2c_client *client)
+static void wm8775_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct wm8775_state *state = to_state(sd);
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&state->hdl);
-	return 0;
 }
 
 static const struct i2c_device_id wm8775_id[] = {
diff --git a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c
index 95e8c29ccc65..d2f5f30582a9 100644
--- a/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c
+++ b/drivers/media/platform/mediatek/vcodec/mtk_vcodec_enc_drv.c
@@ -228,7 +228,6 @@ static int mtk_vcodec_probe(struct platform_device *pdev)
 {
 	struct mtk_vcodec_dev *dev;
 	struct video_device *vfd_enc;
-	struct resource *res;
 	phandle rproc_phandle;
 	enum mtk_vcodec_fw_type fw_type;
 	int ret;
@@ -272,14 +271,12 @@ static int mtk_vcodec_probe(struct platform_device *pdev)
 		goto err_res;
 	}
 
-	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-	if (res == NULL) {
-		dev_err(&pdev->dev, "failed to get irq resource");
-		ret = -ENOENT;
+	dev->enc_irq = platform_get_irq(pdev, 0);
+	if (dev->enc_irq < 0) {
+		ret = dev->enc_irq;
 		goto err_res;
 	}
 
-	dev->enc_irq = platform_get_irq(pdev, 0);
 	irq_set_status_flags(dev->enc_irq, IRQ_NOAUTOEN);
 	ret = devm_request_irq(&pdev->dev, dev->enc_irq,
 			       mtk_vcodec_enc_irq_handler,
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_drm.c b/drivers/media/platform/renesas/vsp1/vsp1_drm.c
index 0c2507dc03d6..c6f25200982c 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_drm.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_drm.c
@@ -856,6 +856,8 @@ int vsp1_du_atomic_update(struct device *dev, unsigned int pipe_index,
 	rpf->mem.addr[1] = cfg->mem[1];
 	rpf->mem.addr[2] = cfg->mem[2];
 
+	rpf->format.flags = cfg->premult ? V4L2_PIX_FMT_FLAG_PREMUL_ALPHA : 0;
+
 	vsp1->drm->inputs[rpf_index].crop = cfg->src;
 	vsp1->drm->inputs[rpf_index].compose = cfg->dst;
 	vsp1->drm->inputs[rpf_index].zpos = cfg->zpos;
diff --git a/drivers/media/radio/radio-tea5764.c b/drivers/media/radio/radio-tea5764.c
index 877a24e5c577..abda40e81612 100644
--- a/drivers/media/radio/radio-tea5764.c
+++ b/drivers/media/radio/radio-tea5764.c
@@ -487,7 +487,7 @@ errfr:
 	return ret;
 }
 
-static int tea5764_i2c_remove(struct i2c_client *client)
+static void tea5764_i2c_remove(struct i2c_client *client)
 {
 	struct tea5764_device *radio = i2c_get_clientdata(client);
 
@@ -499,7 +499,6 @@ static int tea5764_i2c_remove(struct i2c_client *client)
 		v4l2_device_unregister(&radio->v4l2_dev);
 		kfree(radio);
 	}
-	return 0;
 }
 
 /* I2C subsystem interface */
diff --git a/drivers/media/radio/saa7706h.c b/drivers/media/radio/saa7706h.c
index adb66f869dd2..f9e990a9c3ef 100644
--- a/drivers/media/radio/saa7706h.c
+++ b/drivers/media/radio/saa7706h.c
@@ -384,7 +384,7 @@ err:
 	return err;
 }
 
-static int saa7706h_remove(struct i2c_client *client)
+static void saa7706h_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct saa7706h_state *state = to_state(sd);
@@ -393,7 +393,6 @@ static int saa7706h_remove(struct i2c_client *client)
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(&state->hdl);
 	kfree(to_state(sd));
-	return 0;
 }
 
 static const struct i2c_device_id saa7706h_id[] = {
diff --git a/drivers/media/radio/si470x/radio-si470x-i2c.c b/drivers/media/radio/si470x/radio-si470x-i2c.c
index 59b3d77e282d..a6ad926c2b4e 100644
--- a/drivers/media/radio/si470x/radio-si470x-i2c.c
+++ b/drivers/media/radio/si470x/radio-si470x-i2c.c
@@ -461,7 +461,7 @@ err_initial:
 /*
  * si470x_i2c_remove - remove the device
  */
-static int si470x_i2c_remove(struct i2c_client *client)
+static void si470x_i2c_remove(struct i2c_client *client)
 {
 	struct si470x_device *radio = i2c_get_clientdata(client);
 
@@ -472,7 +472,6 @@ static int si470x_i2c_remove(struct i2c_client *client)
 
 	v4l2_ctrl_handler_free(&radio->hdl);
 	v4l2_device_unregister(&radio->v4l2_dev);
-	return 0;
 }
 
 
diff --git a/drivers/media/radio/si4713/si4713.c b/drivers/media/radio/si4713/si4713.c
index adbf43ff6a21..2aec642133a1 100644
--- a/drivers/media/radio/si4713/si4713.c
+++ b/drivers/media/radio/si4713/si4713.c
@@ -1623,7 +1623,7 @@ exit:
 }
 
 /* si4713_remove - remove the device */
-static int si4713_remove(struct i2c_client *client)
+static void si4713_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct si4713_device *sdev = to_si4713_device(sd);
@@ -1635,8 +1635,6 @@ static int si4713_remove(struct i2c_client *client)
 
 	v4l2_device_unregister_subdev(sd);
 	v4l2_ctrl_handler_free(sd->ctrl_handler);
-
-	return 0;
 }
 
 /* si4713_i2c_driver - i2c driver interface */
diff --git a/drivers/media/radio/tef6862.c b/drivers/media/radio/tef6862.c
index d8810492db4f..7b0870a9785b 100644
--- a/drivers/media/radio/tef6862.c
+++ b/drivers/media/radio/tef6862.c
@@ -165,13 +165,12 @@ static int tef6862_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tef6862_remove(struct i2c_client *client)
+static void tef6862_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 
 	v4l2_device_unregister_subdev(sd);
 	kfree(to_state(sd));
-	return 0;
 }
 
 static const struct i2c_device_id tef6862_id[] = {
diff --git a/drivers/media/test-drivers/vidtv/vidtv_demod.c b/drivers/media/test-drivers/vidtv/vidtv_demod.c
index b7823d97b30d..e7959ab1add8 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_demod.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_demod.c
@@ -438,13 +438,11 @@ static int vidtv_demod_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int vidtv_demod_i2c_remove(struct i2c_client *client)
+static void vidtv_demod_i2c_remove(struct i2c_client *client)
 {
 	struct vidtv_demod_state *state = i2c_get_clientdata(client);
 
 	kfree(state);
-
-	return 0;
 }
 
 static struct i2c_driver vidtv_demod_i2c_driver = {
diff --git a/drivers/media/test-drivers/vidtv/vidtv_tuner.c b/drivers/media/test-drivers/vidtv/vidtv_tuner.c
index 14b6bc902ee1..aabc97ed736b 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_tuner.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_tuner.c
@@ -414,13 +414,11 @@ static int vidtv_tuner_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int vidtv_tuner_i2c_remove(struct i2c_client *client)
+static void vidtv_tuner_i2c_remove(struct i2c_client *client)
 {
 	struct vidtv_tuner_dev *tuner_dev = i2c_get_clientdata(client);
 
 	kfree(tuner_dev);
-
-	return 0;
 }
 
 static struct i2c_driver vidtv_tuner_i2c_driver = {
diff --git a/drivers/media/tuners/e4000.c b/drivers/media/tuners/e4000.c
index a3a8d051dc6c..61ae884ea59a 100644
--- a/drivers/media/tuners/e4000.c
+++ b/drivers/media/tuners/e4000.c
@@ -706,7 +706,7 @@ err:
 	return ret;
 }
 
-static int e4000_remove(struct i2c_client *client)
+static void e4000_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct e4000_dev *dev = container_of(sd, struct e4000_dev, sd);
@@ -717,8 +717,6 @@ static int e4000_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(&dev->hdl);
 #endif
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id e4000_id_table[] = {
diff --git a/drivers/media/tuners/fc2580.c b/drivers/media/tuners/fc2580.c
index 1b5961bdf2d5..f30932e1a0f3 100644
--- a/drivers/media/tuners/fc2580.c
+++ b/drivers/media/tuners/fc2580.c
@@ -588,7 +588,7 @@ err:
 	return ret;
 }
 
-static int fc2580_remove(struct i2c_client *client)
+static void fc2580_remove(struct i2c_client *client)
 {
 	struct fc2580_dev *dev = i2c_get_clientdata(client);
 
@@ -598,7 +598,6 @@ static int fc2580_remove(struct i2c_client *client)
 	v4l2_ctrl_handler_free(&dev->hdl);
 #endif
 	kfree(dev);
-	return 0;
 }
 
 static const struct i2c_device_id fc2580_id_table[] = {
diff --git a/drivers/media/tuners/m88rs6000t.c b/drivers/media/tuners/m88rs6000t.c
index 8647c50b66e5..e32e3e9daa15 100644
--- a/drivers/media/tuners/m88rs6000t.c
+++ b/drivers/media/tuners/m88rs6000t.c
@@ -697,7 +697,7 @@ err:
 	return ret;
 }
 
-static int m88rs6000t_remove(struct i2c_client *client)
+static void m88rs6000t_remove(struct i2c_client *client)
 {
 	struct m88rs6000t_dev *dev = i2c_get_clientdata(client);
 	struct dvb_frontend *fe = dev->cfg.fe;
@@ -707,8 +707,6 @@ static int m88rs6000t_remove(struct i2c_client *client)
 	memset(&fe->ops.tuner_ops, 0, sizeof(struct dvb_tuner_ops));
 	fe->tuner_priv = NULL;
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id m88rs6000t_id[] = {
diff --git a/drivers/media/tuners/mt2060.c b/drivers/media/tuners/mt2060.c
index 204e6186bf71..322c806228a5 100644
--- a/drivers/media/tuners/mt2060.c
+++ b/drivers/media/tuners/mt2060.c
@@ -509,11 +509,9 @@ err:
 	return ret;
 }
 
-static int mt2060_remove(struct i2c_client *client)
+static void mt2060_remove(struct i2c_client *client)
 {
 	dev_dbg(&client->dev, "\n");
-
-	return 0;
 }
 
 static const struct i2c_device_id mt2060_id_table[] = {
diff --git a/drivers/media/tuners/mxl301rf.c b/drivers/media/tuners/mxl301rf.c
index c628435a1b06..6422056185a9 100644
--- a/drivers/media/tuners/mxl301rf.c
+++ b/drivers/media/tuners/mxl301rf.c
@@ -307,14 +307,13 @@ static int mxl301rf_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int mxl301rf_remove(struct i2c_client *client)
+static void mxl301rf_remove(struct i2c_client *client)
 {
 	struct mxl301rf_state *state;
 
 	state = cfg_to_state(i2c_get_clientdata(client));
 	state->cfg.fe->tuner_priv = NULL;
 	kfree(state);
-	return 0;
 }
 
 
diff --git a/drivers/media/tuners/qm1d1b0004.c b/drivers/media/tuners/qm1d1b0004.c
index 008ad870c00f..9cba0893207c 100644
--- a/drivers/media/tuners/qm1d1b0004.c
+++ b/drivers/media/tuners/qm1d1b0004.c
@@ -232,14 +232,13 @@ err_mem:
 	return ret;
 }
 
-static int qm1d1b0004_remove(struct i2c_client *client)
+static void qm1d1b0004_remove(struct i2c_client *client)
 {
 	struct dvb_frontend *fe;
 
 	fe = i2c_get_clientdata(client);
 	kfree(fe->tuner_priv);
 	fe->tuner_priv = NULL;
-	return 0;
 }
 
 
diff --git a/drivers/media/tuners/qm1d1c0042.c b/drivers/media/tuners/qm1d1c0042.c
index 53aa2558f71e..2d60bf501fb5 100644
--- a/drivers/media/tuners/qm1d1c0042.c
+++ b/drivers/media/tuners/qm1d1c0042.c
@@ -424,14 +424,13 @@ static int qm1d1c0042_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int qm1d1c0042_remove(struct i2c_client *client)
+static void qm1d1c0042_remove(struct i2c_client *client)
 {
 	struct qm1d1c0042_state *state;
 
 	state = cfg_to_state(i2c_get_clientdata(client));
 	state->cfg.fe->tuner_priv = NULL;
 	kfree(state);
-	return 0;
 }
 
 
diff --git a/drivers/media/tuners/si2157.c b/drivers/media/tuners/si2157.c
index 0de587b412d4..476b32c04c20 100644
--- a/drivers/media/tuners/si2157.c
+++ b/drivers/media/tuners/si2157.c
@@ -951,7 +951,7 @@ err:
 	return ret;
 }
 
-static int si2157_remove(struct i2c_client *client)
+static void si2157_remove(struct i2c_client *client)
 {
 	struct si2157_dev *dev = i2c_get_clientdata(client);
 	struct dvb_frontend *fe = dev->fe;
@@ -969,8 +969,6 @@ static int si2157_remove(struct i2c_client *client)
 	memset(&fe->ops.tuner_ops, 0, sizeof(struct dvb_tuner_ops));
 	fe->tuner_priv = NULL;
 	kfree(dev);
-
-	return 0;
 }
 
 /*
diff --git a/drivers/media/tuners/tda18212.c b/drivers/media/tuners/tda18212.c
index bf48f1cd83d2..eb97711c9c68 100644
--- a/drivers/media/tuners/tda18212.c
+++ b/drivers/media/tuners/tda18212.c
@@ -242,7 +242,7 @@ err:
 	return ret;
 }
 
-static int tda18212_remove(struct i2c_client *client)
+static void tda18212_remove(struct i2c_client *client)
 {
 	struct tda18212_dev *dev = i2c_get_clientdata(client);
 	struct dvb_frontend *fe = dev->cfg.fe;
@@ -252,8 +252,6 @@ static int tda18212_remove(struct i2c_client *client)
 	memset(&fe->ops.tuner_ops, 0, sizeof(struct dvb_tuner_ops));
 	fe->tuner_priv = NULL;
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id tda18212_id[] = {
diff --git a/drivers/media/tuners/tda18250.c b/drivers/media/tuners/tda18250.c
index 8a5781b966ee..e404a5afad4c 100644
--- a/drivers/media/tuners/tda18250.c
+++ b/drivers/media/tuners/tda18250.c
@@ -856,7 +856,7 @@ err:
 	return ret;
 }
 
-static int tda18250_remove(struct i2c_client *client)
+static void tda18250_remove(struct i2c_client *client)
 {
 	struct tda18250_dev *dev = i2c_get_clientdata(client);
 	struct dvb_frontend *fe = dev->fe;
@@ -866,8 +866,6 @@ static int tda18250_remove(struct i2c_client *client)
 	memset(&fe->ops.tuner_ops, 0, sizeof(struct dvb_tuner_ops));
 	fe->tuner_priv = NULL;
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id tda18250_id_table[] = {
diff --git a/drivers/media/tuners/tua9001.c b/drivers/media/tuners/tua9001.c
index af7d5ea1f77e..d141d000b819 100644
--- a/drivers/media/tuners/tua9001.c
+++ b/drivers/media/tuners/tua9001.c
@@ -227,7 +227,7 @@ err:
 	return ret;
 }
 
-static int tua9001_remove(struct i2c_client *client)
+static void tua9001_remove(struct i2c_client *client)
 {
 	struct tua9001_dev *dev = i2c_get_clientdata(client);
 	struct dvb_frontend *fe = dev->fe;
@@ -243,7 +243,6 @@ static int tua9001_remove(struct i2c_client *client)
 			dev_err(&client->dev, "Tuner disable failed (%pe)\n", ERR_PTR(ret));
 	}
 	kfree(dev);
-	return 0;
 }
 
 static const struct i2c_device_id tua9001_id_table[] = {
diff --git a/drivers/media/usb/go7007/s2250-board.c b/drivers/media/usb/go7007/s2250-board.c
index 1fa6f10ee157..2f45188bf9d4 100644
--- a/drivers/media/usb/go7007/s2250-board.c
+++ b/drivers/media/usb/go7007/s2250-board.c
@@ -601,7 +601,7 @@ fail:
 	return err;
 }
 
-static int s2250_remove(struct i2c_client *client)
+static void s2250_remove(struct i2c_client *client)
 {
 	struct s2250 *state = to_state(i2c_get_clientdata(client));
 
@@ -609,7 +609,6 @@ static int s2250_remove(struct i2c_client *client)
 	v4l2_device_unregister_subdev(&state->sd);
 	v4l2_ctrl_handler_free(&state->hdl);
 	kfree(state);
-	return 0;
 }
 
 static const struct i2c_device_id s2250_id[] = {
diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c
index 9c05776f11d1..d509a4a2f08e 100644
--- a/drivers/media/usb/uvc/uvc_driver.c
+++ b/drivers/media/usb/uvc/uvc_driver.c
@@ -2740,7 +2740,7 @@ static const struct usb_device_id uvc_ids[] = {
 	  .idProduct		= 0x4034,
 	  .bInterfaceClass	= USB_CLASS_VIDEO,
 	  .bInterfaceSubClass	= 1,
-	  .bInterfaceProtocol	= 0,
+	  .bInterfaceProtocol	= UVC_PC_PROTOCOL_15,
 	  .driver_info		= (kernel_ulong_t)&uvc_ctrl_power_line_limited },
 	/* LogiLink Wireless Webcam */
 	{ .match_flags		= USB_DEVICE_ID_MATCH_DEVICE
diff --git a/drivers/media/v4l2-core/tuner-core.c b/drivers/media/v4l2-core/tuner-core.c
index 2d47c10de062..33162dc1daf6 100644
--- a/drivers/media/v4l2-core/tuner-core.c
+++ b/drivers/media/v4l2-core/tuner-core.c
@@ -779,7 +779,7 @@ register_client:
  * @client:	i2c_client descriptor
  */
 
-static int tuner_remove(struct i2c_client *client)
+static void tuner_remove(struct i2c_client *client)
 {
 	struct tuner *t = to_tuner(i2c_get_clientdata(client));
 
@@ -789,7 +789,6 @@ static int tuner_remove(struct i2c_client *client)
 
 	list_del(&t->list);
 	kfree(t);
-	return 0;
 }
 
 /*
diff --git a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
index 0f3d6b5667b0..55c26e7d370e 100644
--- a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
+++ b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c
@@ -1040,6 +1040,8 @@ int v4l2_compat_get_array_args(struct file *file, void *mbuf,
 {
 	int err = 0;
 
+	memset(mbuf, 0, array_size);
+
 	switch (cmd) {
 	case VIDIOC_G_FMT32:
 	case VIDIOC_S_FMT32:
diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c
index c314025d977e..e6fd355a2e92 100644
--- a/drivers/media/v4l2-core/v4l2-ioctl.c
+++ b/drivers/media/v4l2-core/v4l2-ioctl.c
@@ -2872,9 +2872,9 @@ static const struct v4l2_ioctl_info v4l2_ioctls[] = {
 	IOCTL_INFO(VIDIOC_S_PRIORITY, v4l_s_priority, v4l_print_u32, INFO_FL_PRIO),
 	IOCTL_INFO(VIDIOC_G_SLICED_VBI_CAP, v4l_g_sliced_vbi_cap, v4l_print_sliced_vbi_cap, INFO_FL_CLEAR(v4l2_sliced_vbi_cap, type)),
 	IOCTL_INFO(VIDIOC_LOG_STATUS, v4l_log_status, v4l_print_newline, 0),
-	IOCTL_INFO(VIDIOC_G_EXT_CTRLS, v4l_g_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL),
-	IOCTL_INFO(VIDIOC_S_EXT_CTRLS, v4l_s_ext_ctrls, v4l_print_ext_controls, INFO_FL_PRIO | INFO_FL_CTRL),
-	IOCTL_INFO(VIDIOC_TRY_EXT_CTRLS, v4l_try_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL),
+	IOCTL_INFO(VIDIOC_G_EXT_CTRLS, v4l_g_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL | INFO_FL_ALWAYS_COPY),
+	IOCTL_INFO(VIDIOC_S_EXT_CTRLS, v4l_s_ext_ctrls, v4l_print_ext_controls, INFO_FL_PRIO | INFO_FL_CTRL | INFO_FL_ALWAYS_COPY),
+	IOCTL_INFO(VIDIOC_TRY_EXT_CTRLS, v4l_try_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL | INFO_FL_ALWAYS_COPY),
 	IOCTL_INFO(VIDIOC_ENUM_FRAMESIZES, v4l_stub_enum_framesizes, v4l_print_frmsizeenum, INFO_FL_CLEAR(v4l2_frmsizeenum, pixel_format)),
 	IOCTL_INFO(VIDIOC_ENUM_FRAMEINTERVALS, v4l_stub_enum_frameintervals, v4l_print_frmivalenum, INFO_FL_CLEAR(v4l2_frmivalenum, height)),
 	IOCTL_INFO(VIDIOC_G_ENC_INDEX, v4l_stub_g_enc_index, v4l_print_enc_idx, 0),
@@ -3367,8 +3367,7 @@ video_usercopy(struct file *file, unsigned int orig_cmd, unsigned long arg,
 		array_buf = kvmalloc(array_size, GFP_KERNEL);
 		err = -ENOMEM;
 		if (array_buf == NULL)
-			goto out_array_args;
-		err = -EFAULT;
+			goto out;
 		if (in_compat_syscall())
 			err = v4l2_compat_get_array_args(file, array_buf,
 							 user_ptr, array_size,
@@ -3377,7 +3376,7 @@ video_usercopy(struct file *file, unsigned int orig_cmd, unsigned long arg,
 			err = copy_from_user(array_buf, user_ptr, array_size) ?
 								-EFAULT : 0;
 		if (err)
-			goto out_array_args;
+			goto out;
 		*kernel_ptr = array_buf;
 	}
 
@@ -3395,6 +3394,13 @@ video_usercopy(struct file *file, unsigned int orig_cmd, unsigned long arg,
 			trace_v4l2_qbuf(video_devdata(file)->minor, parg);
 	}
 
+	/*
+	 * Some ioctls can return an error, but still have valid
+	 * results that must be returned.
+	 */
+	if (err < 0 && !always_copy)
+		goto out;
+
 	if (has_array_args) {
 		*kernel_ptr = (void __force *)user_ptr;
 		if (in_compat_syscall()) {
@@ -3409,16 +3415,8 @@ video_usercopy(struct file *file, unsigned int orig_cmd, unsigned long arg,
 		} else if (copy_to_user(user_ptr, array_buf, array_size)) {
 			err = -EFAULT;
 		}
-		goto out_array_args;
 	}
-	/*
-	 * Some ioctls can return an error, but still have valid
-	 * results that must be returned.
-	 */
-	if (err < 0 && !always_copy)
-		goto out;
 
-out_array_args:
 	if (video_put_user((void __user *)arg, parg, cmd, orig_cmd))
 		err = -EFAULT;
 out:
diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig
index ac1a411648d8..fac290e48e0b 100644
--- a/drivers/memory/Kconfig
+++ b/drivers/memory/Kconfig
@@ -66,6 +66,15 @@ config BRCMSTB_DPFE
 	  for the DRAM's temperature. Slower refresh rate means cooler RAM,
 	  higher refresh rate means hotter RAM.
 
+config BRCMSTB_MEMC
+	tristate "Broadcom STB MEMC driver"
+	default ARCH_BRCMSTB
+	depends on ARCH_BRCMSTB || COMPILE_TEST
+	help
+	  This driver provides a way to configure the Broadcom STB memory
+	  controller and specifically control the Self Refresh Power Down
+	  (SRPD) inactivity timeout.
+
 config BT1_L2_CTL
 	bool "Baikal-T1 CM2 L2-RAM Cache Control Block"
 	depends on MIPS_BAIKAL_T1 || COMPILE_TEST
diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile
index bc7663ed1c25..e148f636c082 100644
--- a/drivers/memory/Makefile
+++ b/drivers/memory/Makefile
@@ -11,6 +11,7 @@ obj-$(CONFIG_ARM_PL172_MPMC)	+= pl172.o
 obj-$(CONFIG_ATMEL_SDRAMC)	+= atmel-sdramc.o
 obj-$(CONFIG_ATMEL_EBI)		+= atmel-ebi.o
 obj-$(CONFIG_BRCMSTB_DPFE)	+= brcmstb_dpfe.o
+obj-$(CONFIG_BRCMSTB_MEMC)	+= brcmstb_memc.o
 obj-$(CONFIG_BT1_L2_CTL)	+= bt1-l2-ctl.o
 obj-$(CONFIG_TI_AEMIF)		+= ti-aemif.o
 obj-$(CONFIG_TI_EMIF)		+= emif.o
diff --git a/drivers/memory/brcmstb_memc.c b/drivers/memory/brcmstb_memc.c
new file mode 100644
index 000000000000..233a53f5bce1
--- /dev/null
+++ b/drivers/memory/brcmstb_memc.c
@@ -0,0 +1,301 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * DDR Self-Refresh Power Down (SRPD) support for Broadcom STB SoCs
+ *
+ */
+
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+
+#define REG_MEMC_CNTRLR_CONFIG		0x00
+#define  CNTRLR_CONFIG_LPDDR4_SHIFT	5
+#define  CNTRLR_CONFIG_MASK		0xf
+#define REG_MEMC_SRPD_CFG_21		0x20
+#define REG_MEMC_SRPD_CFG_20		0x34
+#define REG_MEMC_SRPD_CFG_1x		0x3c
+#define INACT_COUNT_SHIFT		0
+#define INACT_COUNT_MASK		0xffff
+#define SRPD_EN_SHIFT			16
+
+struct brcmstb_memc_data {
+	u32 srpd_offset;
+};
+
+struct brcmstb_memc {
+	struct device *dev;
+	void __iomem *ddr_ctrl;
+	unsigned int timeout_cycles;
+	u32 frequency;
+	u32 srpd_offset;
+};
+
+static int brcmstb_memc_uses_lpddr4(struct brcmstb_memc *memc)
+{
+	void __iomem *config = memc->ddr_ctrl + REG_MEMC_CNTRLR_CONFIG;
+	u32 reg;
+
+	reg = readl_relaxed(config) & CNTRLR_CONFIG_MASK;
+
+	return reg == CNTRLR_CONFIG_LPDDR4_SHIFT;
+}
+
+static int brcmstb_memc_srpd_config(struct brcmstb_memc *memc,
+				    unsigned int cycles)
+{
+	void __iomem *cfg = memc->ddr_ctrl + memc->srpd_offset;
+	u32 val;
+
+	/* Max timeout supported in HW */
+	if (cycles > INACT_COUNT_MASK)
+		return -EINVAL;
+
+	memc->timeout_cycles = cycles;
+
+	val = (cycles << INACT_COUNT_SHIFT) & INACT_COUNT_MASK;
+	if (cycles)
+		val |= BIT(SRPD_EN_SHIFT);
+
+	writel_relaxed(val, cfg);
+	/* Ensure the write is committed to the controller */
+	(void)readl_relaxed(cfg);
+
+	return 0;
+}
+
+static ssize_t frequency_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct brcmstb_memc *memc = dev_get_drvdata(dev);
+
+	return sprintf(buf, "%d\n", memc->frequency);
+}
+
+static ssize_t srpd_show(struct device *dev,
+			 struct device_attribute *attr, char *buf)
+{
+	struct brcmstb_memc *memc = dev_get_drvdata(dev);
+
+	return sprintf(buf, "%d\n", memc->timeout_cycles);
+}
+
+static ssize_t srpd_store(struct device *dev, struct device_attribute *attr,
+			  const char *buf, size_t count)
+{
+	struct brcmstb_memc *memc = dev_get_drvdata(dev);
+	unsigned int val;
+	int ret;
+
+	/*
+	 * Cannot change the inactivity timeout on LPDDR4 chips because the
+	 * dynamic tuning process will also get affected by the inactivity
+	 * timeout, thus making it non functional.
+	 */
+	if (brcmstb_memc_uses_lpddr4(memc))
+		return -EOPNOTSUPP;
+
+	ret = kstrtouint(buf, 10, &val);
+	if (ret < 0)
+		return ret;
+
+	ret = brcmstb_memc_srpd_config(memc, val);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static DEVICE_ATTR_RO(frequency);
+static DEVICE_ATTR_RW(srpd);
+
+static struct attribute *dev_attrs[] = {
+	&dev_attr_frequency.attr,
+	&dev_attr_srpd.attr,
+	NULL,
+};
+
+static struct attribute_group dev_attr_group = {
+	.attrs = dev_attrs,
+};
+
+static const struct of_device_id brcmstb_memc_of_match[];
+
+static int brcmstb_memc_probe(struct platform_device *pdev)
+{
+	const struct brcmstb_memc_data *memc_data;
+	const struct of_device_id *of_id;
+	struct device *dev = &pdev->dev;
+	struct brcmstb_memc *memc;
+	int ret;
+
+	memc = devm_kzalloc(dev, sizeof(*memc), GFP_KERNEL);
+	if (!memc)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, memc);
+
+	of_id = of_match_device(brcmstb_memc_of_match, dev);
+	memc_data = of_id->data;
+	memc->srpd_offset = memc_data->srpd_offset;
+
+	memc->ddr_ctrl = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(memc->ddr_ctrl))
+		return PTR_ERR(memc->ddr_ctrl);
+
+	of_property_read_u32(pdev->dev.of_node, "clock-frequency",
+			     &memc->frequency);
+
+	ret = sysfs_create_group(&dev->kobj, &dev_attr_group);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int brcmstb_memc_remove(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+
+	sysfs_remove_group(&dev->kobj, &dev_attr_group);
+
+	return 0;
+}
+
+enum brcmstb_memc_hwtype {
+	BRCMSTB_MEMC_V21,
+	BRCMSTB_MEMC_V20,
+	BRCMSTB_MEMC_V1X,
+};
+
+static const struct brcmstb_memc_data brcmstb_memc_versions[] = {
+	{ .srpd_offset = REG_MEMC_SRPD_CFG_21 },
+	{ .srpd_offset = REG_MEMC_SRPD_CFG_20 },
+	{ .srpd_offset = REG_MEMC_SRPD_CFG_1x },
+};
+
+static const struct of_device_id brcmstb_memc_of_match[] = {
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.1.x",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V1X]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.0",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V20]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.1",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.2",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.3",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.5",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.6",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.7",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.2.8",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.3.0",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-b.3.1",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-c.1.0",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-c.1.1",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-c.1.2",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-c.1.3",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	{
+		.compatible = "brcm,brcmstb-memc-ddr-rev-c.1.4",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V21]
+	},
+	/* default to the original offset */
+	{
+		.compatible = "brcm,brcmstb-memc-ddr",
+		.data = &brcmstb_memc_versions[BRCMSTB_MEMC_V1X]
+	},
+	{}
+};
+
+static int brcmstb_memc_suspend(struct device *dev)
+{
+	struct brcmstb_memc *memc = dev_get_drvdata(dev);
+	void __iomem *cfg = memc->ddr_ctrl + memc->srpd_offset;
+	u32 val;
+
+	if (memc->timeout_cycles == 0)
+		return 0;
+
+	/*
+	 * Disable SRPD prior to suspending the system since that can
+	 * cause issues with other memory clients managed by the ARM
+	 * trusted firmware to access memory.
+	 */
+	val = readl_relaxed(cfg);
+	val &= ~BIT(SRPD_EN_SHIFT);
+	writel_relaxed(val, cfg);
+	/* Ensure the write is committed to the controller */
+	(void)readl_relaxed(cfg);
+
+	return 0;
+}
+
+static int brcmstb_memc_resume(struct device *dev)
+{
+	struct brcmstb_memc *memc = dev_get_drvdata(dev);
+
+	if (memc->timeout_cycles == 0)
+		return 0;
+
+	return brcmstb_memc_srpd_config(memc, memc->timeout_cycles);
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(brcmstb_memc_pm_ops, brcmstb_memc_suspend,
+				brcmstb_memc_resume);
+
+static struct platform_driver brcmstb_memc_driver = {
+	.probe = brcmstb_memc_probe,
+	.remove = brcmstb_memc_remove,
+	.driver = {
+		.name		= "brcmstb_memc",
+		.of_match_table	= brcmstb_memc_of_match,
+		.pm		= pm_ptr(&brcmstb_memc_pm_ops),
+	},
+};
+module_platform_driver(brcmstb_memc_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Broadcom");
+MODULE_DESCRIPTION("DDR SRPD driver for Broadcom STB chips");
diff --git a/drivers/memory/dfl-emif.c b/drivers/memory/dfl-emif.c
index 3f719816771d..da06cd30a016 100644
--- a/drivers/memory/dfl-emif.c
+++ b/drivers/memory/dfl-emif.c
@@ -24,11 +24,24 @@
 #define EMIF_STAT_CLEAR_BUSY_SFT	16
 #define EMIF_CTRL			0x10
 #define EMIF_CTRL_CLEAR_EN_SFT		0
-#define EMIF_CTRL_CLEAR_EN_MSK		GENMASK_ULL(3, 0)
+#define EMIF_CTRL_CLEAR_EN_MSK		GENMASK_ULL(7, 0)
 
 #define EMIF_POLL_INVL			10000 /* us */
 #define EMIF_POLL_TIMEOUT		5000000 /* us */
 
+/*
+ * The Capability Register replaces the Control Register (at the same
+ * offset) for EMIF feature revisions > 0. The bitmask that indicates
+ * the presence of memory channels exists in both the Capability Register
+ * and Control Register definitions. These can be thought of as a C union.
+ * The Capability Register definitions are used to check for the existence
+ * of a memory channel, and the Control Register definitions are used for
+ * managing the memory-clear functionality in revision 0.
+ */
+#define EMIF_CAPABILITY_BASE		0x10
+#define EMIF_CAPABILITY_CHN_MSK_V0	GENMASK_ULL(3, 0)
+#define EMIF_CAPABILITY_CHN_MSK		GENMASK_ULL(7, 0)
+
 struct dfl_emif {
 	struct device *dev;
 	void __iomem *base;
@@ -106,16 +119,30 @@ emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 0);
 emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 1);
 emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 2);
 emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 3);
+emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 4);
+emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 5);
+emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 6);
+emif_state_attr(init_done, EMIF_STAT_INIT_DONE_SFT, 7);
 
 emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 0);
 emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 1);
 emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 2);
 emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 3);
+emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 4);
+emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 5);
+emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 6);
+emif_state_attr(cal_fail, EMIF_STAT_CALC_FAIL_SFT, 7);
+
 
 emif_clear_attr(0);
 emif_clear_attr(1);
 emif_clear_attr(2);
 emif_clear_attr(3);
+emif_clear_attr(4);
+emif_clear_attr(5);
+emif_clear_attr(6);
+emif_clear_attr(7);
+
 
 static struct attribute *dfl_emif_attrs[] = {
 	&emif_attr_inf0_init_done.attr.attr,
@@ -134,6 +161,22 @@ static struct attribute *dfl_emif_attrs[] = {
 	&emif_attr_inf3_cal_fail.attr.attr,
 	&emif_attr_inf3_clear.attr.attr,
 
+	&emif_attr_inf4_init_done.attr.attr,
+	&emif_attr_inf4_cal_fail.attr.attr,
+	&emif_attr_inf4_clear.attr.attr,
+
+	&emif_attr_inf5_init_done.attr.attr,
+	&emif_attr_inf5_cal_fail.attr.attr,
+	&emif_attr_inf5_clear.attr.attr,
+
+	&emif_attr_inf6_init_done.attr.attr,
+	&emif_attr_inf6_cal_fail.attr.attr,
+	&emif_attr_inf6_clear.attr.attr,
+
+	&emif_attr_inf7_init_done.attr.attr,
+	&emif_attr_inf7_cal_fail.attr.attr,
+	&emif_attr_inf7_clear.attr.attr,
+
 	NULL,
 };
 
@@ -143,15 +186,24 @@ static umode_t dfl_emif_visible(struct kobject *kobj,
 	struct dfl_emif *de = dev_get_drvdata(kobj_to_dev(kobj));
 	struct emif_attr *eattr = container_of(attr, struct emif_attr,
 					       attr.attr);
+	struct dfl_device *ddev = to_dfl_dev(de->dev);
 	u64 val;
 
 	/*
-	 * This device supports upto 4 memory interfaces, but not all
+	 * This device supports up to 8 memory interfaces, but not all
 	 * interfaces are used on different platforms. The read out value of
-	 * CLEAN_EN field (which is a bitmap) could tell how many interfaces
-	 * are available.
+	 * CAPABILITY_CHN_MSK field (which is a bitmap) indicates which
+	 * interfaces are available.
 	 */
-	val = FIELD_GET(EMIF_CTRL_CLEAR_EN_MSK, readq(de->base + EMIF_CTRL));
+	if (ddev->revision > 0 && strstr(attr->name, "_clear"))
+		return 0;
+
+	if (ddev->revision == 0)
+		val = FIELD_GET(EMIF_CAPABILITY_CHN_MSK_V0,
+				readq(de->base + EMIF_CAPABILITY_BASE));
+	else
+		val = FIELD_GET(EMIF_CAPABILITY_CHN_MSK,
+				readq(de->base + EMIF_CAPABILITY_BASE));
 
 	return (val & BIT_ULL(eattr->index)) ? attr->mode : 0;
 }
diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c
index d7cb7ead2ac7..5a9754442bc7 100644
--- a/drivers/memory/mtk-smi.c
+++ b/drivers/memory/mtk-smi.c
@@ -3,6 +3,7 @@
  * Copyright (c) 2015-2016 MediaTek Inc.
  * Author: Yong Wu <yong.wu@mediatek.com>
  */
+#include <linux/arm-smccc.h>
 #include <linux/clk.h>
 #include <linux/component.h>
 #include <linux/device.h>
@@ -14,6 +15,7 @@
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/soc/mediatek/mtk_sip_svc.h>
 #include <soc/mediatek/smi.h>
 #include <dt-bindings/memory/mt2701-larb-port.h>
 #include <dt-bindings/memory/mtk-memory-port.h>
@@ -89,6 +91,7 @@
 #define MTK_SMI_FLAG_THRT_UPDATE	BIT(0)
 #define MTK_SMI_FLAG_SW_FLAG		BIT(1)
 #define MTK_SMI_FLAG_SLEEP_CTL		BIT(2)
+#define MTK_SMI_FLAG_CFG_PORT_SEC_CTL	BIT(3)
 #define MTK_SMI_CAPS(flags, _x)		(!!((flags) & (_x)))
 
 struct mtk_smi_reg_pair {
@@ -127,7 +130,7 @@ struct mtk_smi_common_plat {
 
 struct mtk_smi_larb_gen {
 	int port_in_larb[MTK_LARB_NR_MAX + 1];
-	void (*config_port)(struct device *dev);
+	int				(*config_port)(struct device *dev);
 	unsigned int			larb_direct_to_common_mask;
 	unsigned int			flags_general;
 	const u8			(*ostd)[SMI_LARB_PORT_NR_MAX];
@@ -185,7 +188,7 @@ static const struct component_ops mtk_smi_larb_component_ops = {
 	.unbind = mtk_smi_larb_unbind,
 };
 
-static void mtk_smi_larb_config_port_gen1(struct device *dev)
+static int mtk_smi_larb_config_port_gen1(struct device *dev)
 {
 	struct mtk_smi_larb *larb = dev_get_drvdata(dev);
 	const struct mtk_smi_larb_gen *larb_gen = larb->larb_gen;
@@ -214,31 +217,35 @@ static void mtk_smi_larb_config_port_gen1(struct device *dev)
 			common->smi_ao_base
 			+ REG_SMI_SECUR_CON_ADDR(m4u_port_id));
 	}
+	return 0;
 }
 
-static void mtk_smi_larb_config_port_mt8167(struct device *dev)
+static int mtk_smi_larb_config_port_mt8167(struct device *dev)
 {
 	struct mtk_smi_larb *larb = dev_get_drvdata(dev);
 
 	writel(*larb->mmu, larb->base + MT8167_SMI_LARB_MMU_EN);
+	return 0;
 }
 
-static void mtk_smi_larb_config_port_mt8173(struct device *dev)
+static int mtk_smi_larb_config_port_mt8173(struct device *dev)
 {
 	struct mtk_smi_larb *larb = dev_get_drvdata(dev);
 
 	writel(*larb->mmu, larb->base + MT8173_SMI_LARB_MMU_EN);
+	return 0;
 }
 
-static void mtk_smi_larb_config_port_gen2_general(struct device *dev)
+static int mtk_smi_larb_config_port_gen2_general(struct device *dev)
 {
 	struct mtk_smi_larb *larb = dev_get_drvdata(dev);
 	u32 reg, flags_general = larb->larb_gen->flags_general;
 	const u8 *larbostd = larb->larb_gen->ostd ? larb->larb_gen->ostd[larb->larbid] : NULL;
+	struct arm_smccc_res res;
 	int i;
 
 	if (BIT(larb->larbid) & larb->larb_gen->larb_direct_to_common_mask)
-		return;
+		return 0;
 
 	if (MTK_SMI_CAPS(flags_general, MTK_SMI_FLAG_THRT_UPDATE)) {
 		reg = readl_relaxed(larb->base + SMI_LARB_CMD_THRT_CON);
@@ -253,14 +260,78 @@ static void mtk_smi_larb_config_port_gen2_general(struct device *dev)
 	for (i = 0; i < SMI_LARB_PORT_NR_MAX && larbostd && !!larbostd[i]; i++)
 		writel_relaxed(larbostd[i], larb->base + SMI_LARB_OSTDL_PORTx(i));
 
+	/*
+	 * When mmu_en bits are in security world, the bank_sel still is in the
+	 * LARB_NONSEC_CON below. And the mmu_en bits of LARB_NONSEC_CON have no
+	 * effect in this case.
+	 */
+	if (MTK_SMI_CAPS(flags_general, MTK_SMI_FLAG_CFG_PORT_SEC_CTL)) {
+		arm_smccc_smc(MTK_SIP_KERNEL_IOMMU_CONTROL, IOMMU_ATF_CMD_CONFIG_SMI_LARB,
+			      larb->larbid, *larb->mmu, 0, 0, 0, 0, &res);
+		if (res.a0 != 0) {
+			dev_err(dev, "Enable iommu fail, ret %ld\n", res.a0);
+			return -EINVAL;
+		}
+	}
+
 	for_each_set_bit(i, (unsigned long *)larb->mmu, 32) {
 		reg = readl_relaxed(larb->base + SMI_LARB_NONSEC_CON(i));
 		reg |= F_MMU_EN;
 		reg |= BANK_SEL(larb->bank[i]);
 		writel(reg, larb->base + SMI_LARB_NONSEC_CON(i));
 	}
+	return 0;
 }
 
+static const u8 mtk_smi_larb_mt8188_ostd[][SMI_LARB_PORT_NR_MAX] = {
+	[0] = {0x02, 0x18, 0x22, 0x22, 0x01, 0x02, 0x0a,},
+	[1] = {0x12, 0x02, 0x14, 0x14, 0x01, 0x18, 0x0a,},
+	[2] = {0x12, 0x12, 0x12, 0x12, 0x0a,},
+	[3] = {0x12, 0x12, 0x12, 0x12, 0x28, 0x28, 0x0a,},
+	[4] = {0x06, 0x01, 0x17, 0x06, 0x0a, 0x07, 0x07,},
+	[5] = {0x02, 0x01, 0x04, 0x02, 0x06, 0x01, 0x06, 0x0a,},
+	[6] = {0x06, 0x01, 0x06, 0x0a,},
+	[7] = {0x0c, 0x0c, 0x12,},
+	[8] = {0x0c, 0x01, 0x0a, 0x05, 0x02, 0x03, 0x01, 0x01, 0x14, 0x14,
+	       0x0a, 0x14, 0x1e, 0x01, 0x0c, 0x0a, 0x05, 0x02, 0x02, 0x05,
+	       0x03, 0x01, 0x1e, 0x01, 0x05,},
+	[9] = {0x1e, 0x01, 0x0a, 0x0a, 0x01, 0x01, 0x03, 0x1e, 0x1e, 0x10,
+	       0x07, 0x01, 0x0a, 0x06, 0x03, 0x03, 0x0e, 0x01, 0x04, 0x28,},
+	[10] = {0x03, 0x20, 0x01, 0x20, 0x01, 0x01, 0x14, 0x0a, 0x0a, 0x0c,
+		0x0a, 0x05, 0x02, 0x03, 0x02, 0x14, 0x0a, 0x0a, 0x14, 0x14,
+		0x14, 0x01, 0x01, 0x14, 0x1e, 0x01, 0x05, 0x03, 0x02, 0x28,},
+	[11] = {0x03, 0x20, 0x01, 0x20, 0x01, 0x01, 0x14, 0x0a, 0x0a, 0x0c,
+		0x0a, 0x05, 0x02, 0x03, 0x02, 0x14, 0x0a, 0x0a, 0x14, 0x14,
+		0x14, 0x01, 0x01, 0x14, 0x1e, 0x01, 0x05, 0x03, 0x02, 0x28,},
+	[12] = {0x03, 0x20, 0x01, 0x20, 0x01, 0x01, 0x14, 0x0a, 0x0a, 0x0c,
+		0x0a, 0x05, 0x02, 0x03, 0x02, 0x14, 0x0a, 0x0a, 0x14, 0x14,
+		0x14, 0x01, 0x01, 0x14, 0x1e, 0x01, 0x05, 0x03, 0x02, 0x28,},
+	[13] = {0x07, 0x02, 0x04, 0x02, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05,
+		0x07, 0x02, 0x04, 0x02, 0x05, 0x05,},
+	[14] = {0x02, 0x02, 0x0c, 0x0c, 0x0c, 0x0c, 0x01, 0x01, 0x02, 0x02,
+		0x02, 0x02, 0x0c, 0x0c, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+		0x02, 0x02, 0x01, 0x01,},
+	[15] = {0x0c, 0x0c, 0x02, 0x02, 0x02, 0x02, 0x01, 0x01, 0x0c, 0x0c,
+		0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x01, 0x02,
+		0x0c, 0x01, 0x01,},
+	[16] = {0x28, 0x28, 0x03, 0x01, 0x01, 0x03, 0x14, 0x14, 0x0a, 0x0d,
+		0x03, 0x05, 0x0e, 0x01, 0x01, 0x05, 0x06, 0x0d, 0x01,},
+	[17] = {0x28, 0x02, 0x02, 0x12, 0x02, 0x12, 0x10, 0x02, 0x02, 0x0a,
+		0x12, 0x02, 0x02, 0x0a, 0x16, 0x02, 0x04,},
+	[18] = {0x28, 0x02, 0x02, 0x12, 0x02, 0x12, 0x10, 0x02, 0x02, 0x0a,
+		0x12, 0x02, 0x02, 0x0a, 0x16, 0x02, 0x04,},
+	[19] = {0x1a, 0x0e, 0x0a, 0x0a, 0x0c, 0x0e, 0x10,},
+	[20] = {0x1a, 0x0e, 0x0a, 0x0a, 0x0c, 0x0e, 0x10,},
+	[21] = {0x01, 0x04, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04, 0x04, 0x01,
+		0x01, 0x01, 0x04, 0x0a, 0x06, 0x01, 0x01, 0x01, 0x0a, 0x06,
+		0x01, 0x01, 0x05, 0x03, 0x03, 0x04, 0x01,},
+	[22] = {0x28, 0x19, 0x0c, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x04,
+		0x01,},
+	[23] = {0x01, 0x01, 0x04, 0x01, 0x01, 0x01, 0x18, 0x01, 0x01,},
+	[24] = {0x12, 0x06, 0x12, 0x06,},
+	[25] = {0x01},
+};
+
 static const u8 mtk_smi_larb_mt8195_ostd[][SMI_LARB_PORT_NR_MAX] = {
 	[0] = {0x0a, 0xc, 0x22, 0x22, 0x01, 0x0a,}, /* larb0 */
 	[1] = {0x0a, 0xc, 0x22, 0x22, 0x01, 0x0a,}, /* larb1 */
@@ -347,6 +418,13 @@ static const struct mtk_smi_larb_gen mtk_smi_larb_mt8186 = {
 	.flags_general	            = MTK_SMI_FLAG_SLEEP_CTL,
 };
 
+static const struct mtk_smi_larb_gen mtk_smi_larb_mt8188 = {
+	.config_port                = mtk_smi_larb_config_port_gen2_general,
+	.flags_general	            = MTK_SMI_FLAG_THRT_UPDATE | MTK_SMI_FLAG_SW_FLAG |
+				      MTK_SMI_FLAG_SLEEP_CTL | MTK_SMI_FLAG_CFG_PORT_SEC_CTL,
+	.ostd		            = mtk_smi_larb_mt8188_ostd,
+};
+
 static const struct mtk_smi_larb_gen mtk_smi_larb_mt8192 = {
 	.config_port                = mtk_smi_larb_config_port_gen2_general,
 };
@@ -367,6 +445,7 @@ static const struct of_device_id mtk_smi_larb_of_ids[] = {
 	{.compatible = "mediatek,mt8173-smi-larb", .data = &mtk_smi_larb_mt8173},
 	{.compatible = "mediatek,mt8183-smi-larb", .data = &mtk_smi_larb_mt8183},
 	{.compatible = "mediatek,mt8186-smi-larb", .data = &mtk_smi_larb_mt8186},
+	{.compatible = "mediatek,mt8188-smi-larb", .data = &mtk_smi_larb_mt8188},
 	{.compatible = "mediatek,mt8192-smi-larb", .data = &mtk_smi_larb_mt8192},
 	{.compatible = "mediatek,mt8195-smi-larb", .data = &mtk_smi_larb_mt8195},
 	{}
@@ -511,9 +590,7 @@ static int __maybe_unused mtk_smi_larb_resume(struct device *dev)
 		mtk_smi_larb_sleep_ctrl_disable(larb);
 
 	/* Configure the basic setting for this larb */
-	larb_gen->config_port(dev);
-
-	return 0;
+	return larb_gen->config_port(dev);
 }
 
 static int __maybe_unused mtk_smi_larb_suspend(struct device *dev)
@@ -597,6 +674,18 @@ static const struct mtk_smi_common_plat mtk_smi_common_mt8186 = {
 	.bus_sel  = F_MMU1_LARB(1) | F_MMU1_LARB(4) | F_MMU1_LARB(7),
 };
 
+static const struct mtk_smi_common_plat mtk_smi_common_mt8188_vdo = {
+	.type     = MTK_SMI_GEN2,
+	.bus_sel  = F_MMU1_LARB(1) | F_MMU1_LARB(5) | F_MMU1_LARB(7),
+	.init     = mtk_smi_common_mt8195_init,
+};
+
+static const struct mtk_smi_common_plat mtk_smi_common_mt8188_vpp = {
+	.type     = MTK_SMI_GEN2,
+	.bus_sel  = F_MMU1_LARB(1) | F_MMU1_LARB(2) | F_MMU1_LARB(7),
+	.init     = mtk_smi_common_mt8195_init,
+};
+
 static const struct mtk_smi_common_plat mtk_smi_common_mt8192 = {
 	.type     = MTK_SMI_GEN2,
 	.has_gals = true,
@@ -633,6 +722,8 @@ static const struct of_device_id mtk_smi_common_of_ids[] = {
 	{.compatible = "mediatek,mt8173-smi-common", .data = &mtk_smi_common_gen2},
 	{.compatible = "mediatek,mt8183-smi-common", .data = &mtk_smi_common_mt8183},
 	{.compatible = "mediatek,mt8186-smi-common", .data = &mtk_smi_common_mt8186},
+	{.compatible = "mediatek,mt8188-smi-common-vdo", .data = &mtk_smi_common_mt8188_vdo},
+	{.compatible = "mediatek,mt8188-smi-common-vpp", .data = &mtk_smi_common_mt8188_vpp},
 	{.compatible = "mediatek,mt8192-smi-common", .data = &mtk_smi_common_mt8192},
 	{.compatible = "mediatek,mt8195-smi-common-vdo", .data = &mtk_smi_common_mt8195_vdo},
 	{.compatible = "mediatek,mt8195-smi-common-vpp", .data = &mtk_smi_common_mt8195_vpp},
diff --git a/drivers/memory/of_memory.c b/drivers/memory/of_memory.c
index dbdf87bc0b78..fcd20d85d385 100644
--- a/drivers/memory/of_memory.c
+++ b/drivers/memory/of_memory.c
@@ -134,6 +134,7 @@ const struct lpddr2_timings *of_get_ddr_timings(struct device_node *np_ddr,
 	for_each_child_of_node(np_ddr, np_tim) {
 		if (of_device_is_compatible(np_tim, tim_compat)) {
 			if (of_do_get_timings(np_tim, &timings[i])) {
+				of_node_put(np_tim);
 				devm_kfree(dev, timings);
 				goto default_timings;
 			}
@@ -284,6 +285,7 @@ const struct lpddr3_timings
 		if (of_device_is_compatible(np_tim, tim_compat)) {
 			if (of_lpddr3_do_get_timings(np_tim, &timings[i])) {
 				devm_kfree(dev, timings);
+				of_node_put(np_tim);
 				goto default_timings;
 			}
 			i++;
diff --git a/drivers/memory/pl353-smc.c b/drivers/memory/pl353-smc.c
index f84b98278745..d39ee7d06665 100644
--- a/drivers/memory/pl353-smc.c
+++ b/drivers/memory/pl353-smc.c
@@ -122,6 +122,7 @@ static int pl353_smc_probe(struct amba_device *adev, const struct amba_id *id)
 	}
 
 	of_platform_device_create(child, NULL, &adev->dev);
+	of_node_put(child);
 
 	return 0;
 
diff --git a/drivers/mfd/88pm800.c b/drivers/mfd/88pm800.c
index eaf9845633b4..a30e47b74327 100644
--- a/drivers/mfd/88pm800.c
+++ b/drivers/mfd/88pm800.c
@@ -583,7 +583,7 @@ out_init:
 	return ret;
 }
 
-static int pm800_remove(struct i2c_client *client)
+static void pm800_remove(struct i2c_client *client)
 {
 	struct pm80x_chip *chip = i2c_get_clientdata(client);
 
@@ -592,8 +592,6 @@ static int pm800_remove(struct i2c_client *client)
 
 	pm800_pages_exit(chip);
 	pm80x_deinit();
-
-	return 0;
 }
 
 static struct i2c_driver pm800_driver = {
diff --git a/drivers/mfd/88pm805.c b/drivers/mfd/88pm805.c
index ada6c513302b..10d3637840c8 100644
--- a/drivers/mfd/88pm805.c
+++ b/drivers/mfd/88pm805.c
@@ -239,7 +239,7 @@ out_init:
 	return ret;
 }
 
-static int pm805_remove(struct i2c_client *client)
+static void pm805_remove(struct i2c_client *client)
 {
 	struct pm80x_chip *chip = i2c_get_clientdata(client);
 
@@ -247,8 +247,6 @@ static int pm805_remove(struct i2c_client *client)
 	device_irq_exit_805(chip);
 
 	pm80x_deinit();
-
-	return 0;
 }
 
 static struct i2c_driver pm805_driver = {
diff --git a/drivers/mfd/88pm860x-core.c b/drivers/mfd/88pm860x-core.c
index b1e829ea909b..5dc86dd66202 100644
--- a/drivers/mfd/88pm860x-core.c
+++ b/drivers/mfd/88pm860x-core.c
@@ -1201,7 +1201,7 @@ static int pm860x_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int pm860x_remove(struct i2c_client *client)
+static void pm860x_remove(struct i2c_client *client)
 {
 	struct pm860x_chip *chip = i2c_get_clientdata(client);
 
@@ -1210,7 +1210,6 @@ static int pm860x_remove(struct i2c_client *client)
 		regmap_exit(chip->regmap_companion);
 		i2c_unregister_device(chip->companion);
 	}
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
index abb58ab1a1a4..c3dd1fe8d8c9 100644
--- a/drivers/mfd/Kconfig
+++ b/drivers/mfd/Kconfig
@@ -963,6 +963,27 @@ config MFD_MENF21BMC
 	  This driver can also be built as a module. If so the module
 	  will be called menf21bmc.
 
+config MFD_OCELOT
+	tristate "Microsemi Ocelot External Control Support"
+	depends on SPI_MASTER
+	select MFD_CORE
+	select REGMAP_SPI
+	help
+	  Ocelot is a family of networking chips that support multiple ethernet
+	  and fibre interfaces. In addition to networking, they contain several
+	  other functions, including pinctrl, MDIO, and communication with
+	  external chips. While some chips have an internal processor capable of
+	  running an OS, others don't. All chips can be controlled externally
+	  through different interfaces, including SPI, I2C, and PCIe.
+
+	  Say yes here to add support for Ocelot chips (VSC7511, VSC7512,
+	  VSC7513, VSC7514) controlled externally.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called ocelot-soc.
+
+	  If unsure, say N.
+
 config EZX_PCAP
 	bool "Motorola EZXPCAP Support"
 	depends on SPI_MASTER
diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile
index 858cacf659d6..0004b7e86220 100644
--- a/drivers/mfd/Makefile
+++ b/drivers/mfd/Makefile
@@ -120,6 +120,9 @@ obj-$(CONFIG_MFD_MC13XXX_I2C)	+= mc13xxx-i2c.o
 
 obj-$(CONFIG_MFD_CORE)		+= mfd-core.o
 
+ocelot-soc-objs			:= ocelot-core.o ocelot-spi.o
+obj-$(CONFIG_MFD_OCELOT)	+= ocelot-soc.o
+
 obj-$(CONFIG_EZX_PCAP)		+= ezx-pcap.o
 obj-$(CONFIG_MFD_CPCAP)		+= motorola-cpcap.o
 
diff --git a/drivers/mfd/acer-ec-a500.c b/drivers/mfd/acer-ec-a500.c
index 80c2fdd14fc4..7fd8b9988075 100644
--- a/drivers/mfd/acer-ec-a500.c
+++ b/drivers/mfd/acer-ec-a500.c
@@ -169,7 +169,7 @@ static int a500_ec_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int a500_ec_remove(struct i2c_client *client)
+static void a500_ec_remove(struct i2c_client *client)
 {
 	if (of_device_is_system_power_controller(client->dev.of_node)) {
 		if (pm_power_off == a500_ec_poweroff)
@@ -177,8 +177,6 @@ static int a500_ec_remove(struct i2c_client *client)
 
 		unregister_restart_handler(&a500_ec_restart_handler);
 	}
-
-	return 0;
 }
 
 static const struct of_device_id a500_ec_match[] = {
diff --git a/drivers/mfd/arizona-i2c.c b/drivers/mfd/arizona-i2c.c
index 6d83e6b9a692..bfc7cf56ff2c 100644
--- a/drivers/mfd/arizona-i2c.c
+++ b/drivers/mfd/arizona-i2c.c
@@ -84,13 +84,11 @@ static int arizona_i2c_probe(struct i2c_client *i2c,
 	return arizona_dev_init(arizona);
 }
 
-static int arizona_i2c_remove(struct i2c_client *i2c)
+static void arizona_i2c_remove(struct i2c_client *i2c)
 {
 	struct arizona *arizona = dev_get_drvdata(&i2c->dev);
 
 	arizona_dev_exit(arizona);
-
-	return 0;
 }
 
 static const struct i2c_device_id arizona_i2c_id[] = {
diff --git a/drivers/mfd/axp20x-i2c.c b/drivers/mfd/axp20x-i2c.c
index 00ab48018d8d..8fd6727dc30a 100644
--- a/drivers/mfd/axp20x-i2c.c
+++ b/drivers/mfd/axp20x-i2c.c
@@ -50,13 +50,11 @@ static int axp20x_i2c_probe(struct i2c_client *i2c,
 	return axp20x_device_probe(axp20x);
 }
 
-static int axp20x_i2c_remove(struct i2c_client *i2c)
+static void axp20x_i2c_remove(struct i2c_client *i2c)
 {
 	struct axp20x_dev *axp20x = i2c_get_clientdata(i2c);
 
 	axp20x_device_remove(axp20x);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
diff --git a/drivers/mfd/da903x.c b/drivers/mfd/da903x.c
index a818fbb55988..3f8f6ad3a98c 100644
--- a/drivers/mfd/da903x.c
+++ b/drivers/mfd/da903x.c
@@ -532,12 +532,11 @@ static int da903x_probe(struct i2c_client *client,
 	return da903x_add_subdevs(chip, pdata);
 }
 
-static int da903x_remove(struct i2c_client *client)
+static void da903x_remove(struct i2c_client *client)
 {
 	struct da903x_chip *chip = i2c_get_clientdata(client);
 
 	da903x_remove_subdevs(chip);
-	return 0;
 }
 
 static struct i2c_driver da903x_driver = {
diff --git a/drivers/mfd/da9052-i2c.c b/drivers/mfd/da9052-i2c.c
index 8de93db35f3a..5a74696c8704 100644
--- a/drivers/mfd/da9052-i2c.c
+++ b/drivers/mfd/da9052-i2c.c
@@ -168,12 +168,11 @@ static int da9052_i2c_probe(struct i2c_client *client,
 	return da9052_device_init(da9052, id->driver_data);
 }
 
-static int da9052_i2c_remove(struct i2c_client *client)
+static void da9052_i2c_remove(struct i2c_client *client)
 {
 	struct da9052 *da9052 = i2c_get_clientdata(client);
 
 	da9052_device_exit(da9052);
-	return 0;
 }
 
 static struct i2c_driver da9052_i2c_driver = {
diff --git a/drivers/mfd/da9055-i2c.c b/drivers/mfd/da9055-i2c.c
index bc60433b68db..276c7d1c509e 100644
--- a/drivers/mfd/da9055-i2c.c
+++ b/drivers/mfd/da9055-i2c.c
@@ -41,13 +41,11 @@ static int da9055_i2c_probe(struct i2c_client *i2c,
 	return da9055_device_init(da9055);
 }
 
-static int da9055_i2c_remove(struct i2c_client *i2c)
+static void da9055_i2c_remove(struct i2c_client *i2c)
 {
 	struct da9055 *da9055 = i2c_get_clientdata(i2c);
 
 	da9055_device_exit(da9055);
-
-	return 0;
 }
 
 /*
diff --git a/drivers/mfd/da9062-core.c b/drivers/mfd/da9062-core.c
index 2774b2cbaea6..0a80d82c6858 100644
--- a/drivers/mfd/da9062-core.c
+++ b/drivers/mfd/da9062-core.c
@@ -723,14 +723,12 @@ static int da9062_i2c_probe(struct i2c_client *i2c,
 	return ret;
 }
 
-static int da9062_i2c_remove(struct i2c_client *i2c)
+static void da9062_i2c_remove(struct i2c_client *i2c)
 {
 	struct da9062 *chip = i2c_get_clientdata(i2c);
 
 	mfd_remove_devices(chip->dev);
 	regmap_del_irq_chip(i2c->irq, chip->regmap_irq);
-
-	return 0;
 }
 
 static const struct i2c_device_id da9062_i2c_id[] = {
diff --git a/drivers/mfd/da9150-core.c b/drivers/mfd/da9150-core.c
index 58009c8cb870..6ae56e46d24e 100644
--- a/drivers/mfd/da9150-core.c
+++ b/drivers/mfd/da9150-core.c
@@ -471,15 +471,13 @@ regmap_irq_fail:
 	return ret;
 }
 
-static int da9150_remove(struct i2c_client *client)
+static void da9150_remove(struct i2c_client *client)
 {
 	struct da9150 *da9150 = i2c_get_clientdata(client);
 
 	regmap_del_irq_chip(da9150->irq, da9150->regmap_irq_data);
 	mfd_remove_devices(da9150->dev);
 	i2c_unregister_device(da9150->core_qif);
-
-	return 0;
 }
 
 static void da9150_shutdown(struct i2c_client *client)
diff --git a/drivers/mfd/dm355evm_msp.c b/drivers/mfd/dm355evm_msp.c
index 54fb6cbd2aa0..759c59690680 100644
--- a/drivers/mfd/dm355evm_msp.c
+++ b/drivers/mfd/dm355evm_msp.c
@@ -375,11 +375,10 @@ static void dm355evm_power_off(void)
 	dm355evm_command(MSP_COMMAND_POWEROFF);
 }
 
-static int dm355evm_msp_remove(struct i2c_client *client)
+static void dm355evm_msp_remove(struct i2c_client *client)
 {
 	pm_power_off = NULL;
 	msp430 = NULL;
-	return 0;
 }
 
 static int
diff --git a/drivers/mfd/ene-kb3930.c b/drivers/mfd/ene-kb3930.c
index 1b73318d1f1f..3eff98e26bea 100644
--- a/drivers/mfd/ene-kb3930.c
+++ b/drivers/mfd/ene-kb3930.c
@@ -177,7 +177,7 @@ static int kb3930_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int kb3930_remove(struct i2c_client *client)
+static void kb3930_remove(struct i2c_client *client)
 {
 	struct kb3930 *ddata = i2c_get_clientdata(client);
 
@@ -187,8 +187,6 @@ static int kb3930_remove(struct i2c_client *client)
 		unregister_restart_handler(&kb3930_restart_nb);
 	}
 	kb3930_power_off = NULL;
-
-	return 0;
 }
 
 static const struct of_device_id kb3930_dt_ids[] = {
diff --git a/drivers/mfd/gateworks-gsc.c b/drivers/mfd/gateworks-gsc.c
index d87876747b91..9d7d870c44a8 100644
--- a/drivers/mfd/gateworks-gsc.c
+++ b/drivers/mfd/gateworks-gsc.c
@@ -255,11 +255,9 @@ static int gsc_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int gsc_remove(struct i2c_client *client)
+static void gsc_remove(struct i2c_client *client)
 {
 	sysfs_remove_group(&client->dev.kobj, &attr_group);
-
-	return 0;
 }
 
 static struct i2c_driver gsc_driver = {
diff --git a/drivers/mfd/intel_soc_pmic_core.c b/drivers/mfd/intel_soc_pmic_core.c
index 5e8c94e008ed..b824e15f4d22 100644
--- a/drivers/mfd/intel_soc_pmic_core.c
+++ b/drivers/mfd/intel_soc_pmic_core.c
@@ -81,7 +81,7 @@ err_del_irq_chip:
 	return ret;
 }
 
-static int intel_soc_pmic_i2c_remove(struct i2c_client *i2c)
+static void intel_soc_pmic_i2c_remove(struct i2c_client *i2c)
 {
 	struct intel_soc_pmic *pmic = dev_get_drvdata(&i2c->dev);
 
@@ -91,8 +91,6 @@ static int intel_soc_pmic_i2c_remove(struct i2c_client *i2c)
 	pwm_remove_table(crc_pwm_lookup, ARRAY_SIZE(crc_pwm_lookup));
 
 	mfd_remove_devices(&i2c->dev);
-
-	return 0;
 }
 
 static void intel_soc_pmic_shutdown(struct i2c_client *i2c)
diff --git a/drivers/mfd/iqs62x.c b/drivers/mfd/iqs62x.c
index 575ab67e243d..1895fce25b06 100644
--- a/drivers/mfd/iqs62x.c
+++ b/drivers/mfd/iqs62x.c
@@ -1008,13 +1008,11 @@ static int iqs62x_probe(struct i2c_client *client)
 	return ret;
 }
 
-static int iqs62x_remove(struct i2c_client *client)
+static void iqs62x_remove(struct i2c_client *client)
 {
 	struct iqs62x_core *iqs62x = i2c_get_clientdata(client);
 
 	wait_for_completion(&iqs62x->fw_done);
-
-	return 0;
 }
 
 static int __maybe_unused iqs62x_suspend(struct device *dev)
diff --git a/drivers/mfd/lm3533-core.c b/drivers/mfd/lm3533-core.c
index 5690768f3e63..be32ffc5af38 100644
--- a/drivers/mfd/lm3533-core.c
+++ b/drivers/mfd/lm3533-core.c
@@ -607,15 +607,13 @@ static int lm3533_i2c_probe(struct i2c_client *i2c,
 	return lm3533_device_init(lm3533);
 }
 
-static int lm3533_i2c_remove(struct i2c_client *i2c)
+static void lm3533_i2c_remove(struct i2c_client *i2c)
 {
 	struct lm3533 *lm3533 = i2c_get_clientdata(i2c);
 
 	dev_dbg(&i2c->dev, "%s\n", __func__);
 
 	lm3533_device_exit(lm3533);
-
-	return 0;
 }
 
 static const struct i2c_device_id lm3533_i2c_ids[] = {
diff --git a/drivers/mfd/lp8788.c b/drivers/mfd/lp8788.c
index c223d2c6a363..e7c601bca9ef 100644
--- a/drivers/mfd/lp8788.c
+++ b/drivers/mfd/lp8788.c
@@ -199,13 +199,12 @@ static int lp8788_probe(struct i2c_client *cl, const struct i2c_device_id *id)
 			       ARRAY_SIZE(lp8788_devs), NULL, 0, NULL);
 }
 
-static int lp8788_remove(struct i2c_client *cl)
+static void lp8788_remove(struct i2c_client *cl)
 {
 	struct lp8788 *lp = i2c_get_clientdata(cl);
 
 	mfd_remove_devices(lp->dev);
 	lp8788_irq_exit(lp);
-	return 0;
 }
 
 static const struct i2c_device_id lp8788_ids[] = {
diff --git a/drivers/mfd/madera-i2c.c b/drivers/mfd/madera-i2c.c
index 7df5b9ba5855..915d2f95bad3 100644
--- a/drivers/mfd/madera-i2c.c
+++ b/drivers/mfd/madera-i2c.c
@@ -112,13 +112,11 @@ static int madera_i2c_probe(struct i2c_client *i2c,
 	return madera_dev_init(madera);
 }
 
-static int madera_i2c_remove(struct i2c_client *i2c)
+static void madera_i2c_remove(struct i2c_client *i2c)
 {
 	struct madera *madera = dev_get_drvdata(&i2c->dev);
 
 	madera_dev_exit(madera);
-
-	return 0;
 }
 
 static const struct i2c_device_id madera_i2c_id[] = {
diff --git a/drivers/mfd/max14577.c b/drivers/mfd/max14577.c
index 6c487fa14e9c..d44ad6f33742 100644
--- a/drivers/mfd/max14577.c
+++ b/drivers/mfd/max14577.c
@@ -463,7 +463,7 @@ err_max77836:
 	return ret;
 }
 
-static int max14577_i2c_remove(struct i2c_client *i2c)
+static void max14577_i2c_remove(struct i2c_client *i2c)
 {
 	struct max14577 *max14577 = i2c_get_clientdata(i2c);
 
@@ -471,8 +471,6 @@ static int max14577_i2c_remove(struct i2c_client *i2c)
 	regmap_del_irq_chip(max14577->irq, max14577->irq_data);
 	if (max14577->dev_type == MAXIM_DEVICE_TYPE_MAX77836)
 		max77836_remove(max14577);
-
-	return 0;
 }
 
 static const struct i2c_device_id max14577_i2c_id[] = {
diff --git a/drivers/mfd/max77693.c b/drivers/mfd/max77693.c
index 4e6244e17559..7088cb6f9174 100644
--- a/drivers/mfd/max77693.c
+++ b/drivers/mfd/max77693.c
@@ -294,7 +294,7 @@ err_i2c_haptic:
 	return ret;
 }
 
-static int max77693_i2c_remove(struct i2c_client *i2c)
+static void max77693_i2c_remove(struct i2c_client *i2c)
 {
 	struct max77693_dev *max77693 = i2c_get_clientdata(i2c);
 
@@ -307,8 +307,6 @@ static int max77693_i2c_remove(struct i2c_client *i2c)
 
 	i2c_unregister_device(max77693->i2c_muic);
 	i2c_unregister_device(max77693->i2c_haptic);
-
-	return 0;
 }
 
 static const struct i2c_device_id max77693_i2c_id[] = {
diff --git a/drivers/mfd/max8907.c b/drivers/mfd/max8907.c
index 41f566e6a096..c340080971ce 100644
--- a/drivers/mfd/max8907.c
+++ b/drivers/mfd/max8907.c
@@ -282,7 +282,7 @@ err_alloc_drvdata:
 	return ret;
 }
 
-static int max8907_i2c_remove(struct i2c_client *i2c)
+static void max8907_i2c_remove(struct i2c_client *i2c)
 {
 	struct max8907 *max8907 = i2c_get_clientdata(i2c);
 
@@ -293,8 +293,6 @@ static int max8907_i2c_remove(struct i2c_client *i2c)
 	regmap_del_irq_chip(max8907->i2c_gen->irq, max8907->irqc_chg);
 
 	i2c_unregister_device(max8907->i2c_rtc);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
diff --git a/drivers/mfd/max8925-i2c.c b/drivers/mfd/max8925-i2c.c
index 114e905bef25..04101da42bd3 100644
--- a/drivers/mfd/max8925-i2c.c
+++ b/drivers/mfd/max8925-i2c.c
@@ -198,14 +198,13 @@ static int max8925_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int max8925_remove(struct i2c_client *client)
+static void max8925_remove(struct i2c_client *client)
 {
 	struct max8925_chip *chip = i2c_get_clientdata(client);
 
 	max8925_device_exit(chip);
 	i2c_unregister_device(chip->adc);
 	i2c_unregister_device(chip->rtc);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/mfd/mc13xxx-i2c.c b/drivers/mfd/mc13xxx-i2c.c
index fb937f66277e..eb94f3004cf3 100644
--- a/drivers/mfd/mc13xxx-i2c.c
+++ b/drivers/mfd/mc13xxx-i2c.c
@@ -85,10 +85,9 @@ static int mc13xxx_i2c_probe(struct i2c_client *client,
 	return mc13xxx_common_init(&client->dev);
 }
 
-static int mc13xxx_i2c_remove(struct i2c_client *client)
+static void mc13xxx_i2c_remove(struct i2c_client *client)
 {
 	mc13xxx_common_exit(&client->dev);
-	return 0;
 }
 
 static struct i2c_driver mc13xxx_i2c_driver = {
diff --git a/drivers/mfd/menelaus.c b/drivers/mfd/menelaus.c
index 07e0ca2e467c..eb08f69001f9 100644
--- a/drivers/mfd/menelaus.c
+++ b/drivers/mfd/menelaus.c
@@ -1222,14 +1222,13 @@ fail:
 	return err;
 }
 
-static int menelaus_remove(struct i2c_client *client)
+static void menelaus_remove(struct i2c_client *client)
 {
 	struct menelaus_chip	*menelaus = i2c_get_clientdata(client);
 
 	free_irq(client->irq, menelaus);
 	flush_work(&menelaus->work);
 	the_menelaus = NULL;
-	return 0;
 }
 
 static const struct i2c_device_id menelaus_id[] = {
diff --git a/drivers/mfd/ntxec.c b/drivers/mfd/ntxec.c
index b711e73eedcb..e16a7a82a929 100644
--- a/drivers/mfd/ntxec.c
+++ b/drivers/mfd/ntxec.c
@@ -239,15 +239,13 @@ static int ntxec_probe(struct i2c_client *client)
 	return res;
 }
 
-static int ntxec_remove(struct i2c_client *client)
+static void ntxec_remove(struct i2c_client *client)
 {
 	if (client == poweroff_restart_client) {
 		poweroff_restart_client = NULL;
 		pm_power_off = NULL;
 		unregister_restart_handler(&ntxec_restart_handler);
 	}
-
-	return 0;
 }
 
 static const struct of_device_id of_ntxec_match_table[] = {
diff --git a/drivers/mfd/ocelot-core.c b/drivers/mfd/ocelot-core.c
new file mode 100644
index 000000000000..1816d52c65c5
--- /dev/null
+++ b/drivers/mfd/ocelot-core.c
@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Core driver for the Ocelot chip family.
+ *
+ * The VSC7511, 7512, 7513, and 7514 can be controlled internally via an
+ * on-chip MIPS processor, or externally via SPI, I2C, PCIe. This core driver is
+ * intended to be the bus-agnostic glue between, for example, the SPI bus and
+ * the child devices.
+ *
+ * Copyright 2021-2022 Innovative Advantage Inc.
+ *
+ * Author: Colin Foster <colin.foster@in-advantage.com>
+ */
+
+#include <linux/bits.h>
+#include <linux/device.h>
+#include <linux/export.h>
+#include <linux/iopoll.h>
+#include <linux/ioport.h>
+#include <linux/kernel.h>
+#include <linux/mfd/core.h>
+#include <linux/mfd/ocelot.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+#include <soc/mscc/ocelot.h>
+
+#include "ocelot.h"
+
+#define REG_GCB_SOFT_RST		0x0008
+
+#define BIT_SOFT_CHIP_RST		BIT(0)
+
+#define VSC7512_MIIM0_RES_START		0x7107009c
+#define VSC7512_MIIM1_RES_START		0x710700c0
+#define VSC7512_MIIM_RES_SIZE		0x024
+
+#define VSC7512_PHY_RES_START		0x710700f0
+#define VSC7512_PHY_RES_SIZE		0x004
+
+#define VSC7512_GPIO_RES_START		0x71070034
+#define VSC7512_GPIO_RES_SIZE		0x06c
+
+#define VSC7512_SIO_CTRL_RES_START	0x710700f8
+#define VSC7512_SIO_CTRL_RES_SIZE	0x100
+
+#define VSC7512_GCB_RST_SLEEP_US	100
+#define VSC7512_GCB_RST_TIMEOUT_US	100000
+
+static int ocelot_gcb_chip_rst_status(struct ocelot_ddata *ddata)
+{
+	int val, err;
+
+	err = regmap_read(ddata->gcb_regmap, REG_GCB_SOFT_RST, &val);
+	if (err)
+		return err;
+
+	return val;
+}
+
+int ocelot_chip_reset(struct device *dev)
+{
+	struct ocelot_ddata *ddata = dev_get_drvdata(dev);
+	int ret, val;
+
+	/*
+	 * Reset the entire chip here to put it into a completely known state.
+	 * Other drivers may want to reset their own subsystems. The register
+	 * self-clears, so one write is all that is needed and wait for it to
+	 * clear.
+	 */
+	ret = regmap_write(ddata->gcb_regmap, REG_GCB_SOFT_RST, BIT_SOFT_CHIP_RST);
+	if (ret)
+		return ret;
+
+	return readx_poll_timeout(ocelot_gcb_chip_rst_status, ddata, val, !val,
+				  VSC7512_GCB_RST_SLEEP_US, VSC7512_GCB_RST_TIMEOUT_US);
+}
+EXPORT_SYMBOL_NS(ocelot_chip_reset, MFD_OCELOT);
+
+static const struct resource vsc7512_miim0_resources[] = {
+	DEFINE_RES_REG_NAMED(VSC7512_MIIM0_RES_START, VSC7512_MIIM_RES_SIZE, "gcb_miim0"),
+	DEFINE_RES_REG_NAMED(VSC7512_PHY_RES_START, VSC7512_PHY_RES_SIZE, "gcb_phy"),
+};
+
+static const struct resource vsc7512_miim1_resources[] = {
+	DEFINE_RES_REG_NAMED(VSC7512_MIIM1_RES_START, VSC7512_MIIM_RES_SIZE, "gcb_miim1"),
+};
+
+static const struct resource vsc7512_pinctrl_resources[] = {
+	DEFINE_RES_REG_NAMED(VSC7512_GPIO_RES_START, VSC7512_GPIO_RES_SIZE, "gcb_gpio"),
+};
+
+static const struct resource vsc7512_sgpio_resources[] = {
+	DEFINE_RES_REG_NAMED(VSC7512_SIO_CTRL_RES_START, VSC7512_SIO_CTRL_RES_SIZE, "gcb_sio"),
+};
+
+static const struct mfd_cell vsc7512_devs[] = {
+	{
+		.name = "ocelot-pinctrl",
+		.of_compatible = "mscc,ocelot-pinctrl",
+		.num_resources = ARRAY_SIZE(vsc7512_pinctrl_resources),
+		.resources = vsc7512_pinctrl_resources,
+	}, {
+		.name = "ocelot-sgpio",
+		.of_compatible = "mscc,ocelot-sgpio",
+		.num_resources = ARRAY_SIZE(vsc7512_sgpio_resources),
+		.resources = vsc7512_sgpio_resources,
+	}, {
+		.name = "ocelot-miim0",
+		.of_compatible = "mscc,ocelot-miim",
+		.of_reg = VSC7512_MIIM0_RES_START,
+		.use_of_reg = true,
+		.num_resources = ARRAY_SIZE(vsc7512_miim0_resources),
+		.resources = vsc7512_miim0_resources,
+	}, {
+		.name = "ocelot-miim1",
+		.of_compatible = "mscc,ocelot-miim",
+		.of_reg = VSC7512_MIIM1_RES_START,
+		.use_of_reg = true,
+		.num_resources = ARRAY_SIZE(vsc7512_miim1_resources),
+		.resources = vsc7512_miim1_resources,
+	},
+};
+
+static void ocelot_core_try_add_regmap(struct device *dev,
+				       const struct resource *res)
+{
+	if (dev_get_regmap(dev, res->name))
+		return;
+
+	ocelot_spi_init_regmap(dev, res);
+}
+
+static void ocelot_core_try_add_regmaps(struct device *dev,
+					const struct mfd_cell *cell)
+{
+	int i;
+
+	for (i = 0; i < cell->num_resources; i++)
+		ocelot_core_try_add_regmap(dev, &cell->resources[i]);
+}
+
+int ocelot_core_init(struct device *dev)
+{
+	int i, ndevs;
+
+	ndevs = ARRAY_SIZE(vsc7512_devs);
+
+	for (i = 0; i < ndevs; i++)
+		ocelot_core_try_add_regmaps(dev, &vsc7512_devs[i]);
+
+	return devm_mfd_add_devices(dev, PLATFORM_DEVID_AUTO, vsc7512_devs, ndevs, NULL, 0, NULL);
+}
+EXPORT_SYMBOL_NS(ocelot_core_init, MFD_OCELOT);
+
+MODULE_DESCRIPTION("Externally Controlled Ocelot Chip Driver");
+MODULE_AUTHOR("Colin Foster <colin.foster@in-advantage.com>");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS(MFD_OCELOT_SPI);
diff --git a/drivers/mfd/ocelot-spi.c b/drivers/mfd/ocelot-spi.c
new file mode 100644
index 000000000000..0f097f4829d1
--- /dev/null
+++ b/drivers/mfd/ocelot-spi.c
@@ -0,0 +1,299 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * SPI core driver for the Ocelot chip family.
+ *
+ * This driver will handle everything necessary to allow for communication over
+ * SPI to the VSC7511, VSC7512, VSC7513 and VSC7514 chips. The main functions
+ * are to prepare the chip's SPI interface for a specific bus speed, and a host
+ * processor's endianness. This will create and distribute regmaps for any
+ * children.
+ *
+ * Copyright 2021-2022 Innovative Advantage Inc.
+ *
+ * Author: Colin Foster <colin.foster@in-advantage.com>
+ */
+
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/export.h>
+#include <linux/ioport.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+#include "ocelot.h"
+
+#define REG_DEV_CPUORG_IF_CTRL		0x0000
+#define REG_DEV_CPUORG_IF_CFGSTAT	0x0004
+
+#define CFGSTAT_IF_NUM_VCORE		(0 << 24)
+#define CFGSTAT_IF_NUM_VRAP		(1 << 24)
+#define CFGSTAT_IF_NUM_SI		(2 << 24)
+#define CFGSTAT_IF_NUM_MIIM		(3 << 24)
+
+#define VSC7512_DEVCPU_ORG_RES_START	0x71000000
+#define VSC7512_DEVCPU_ORG_RES_SIZE	0x38
+
+#define VSC7512_CHIP_REGS_RES_START	0x71070000
+#define VSC7512_CHIP_REGS_RES_SIZE	0x14
+
+static const struct resource vsc7512_dev_cpuorg_resource =
+	DEFINE_RES_REG_NAMED(VSC7512_DEVCPU_ORG_RES_START,
+			     VSC7512_DEVCPU_ORG_RES_SIZE,
+			     "devcpu_org");
+
+static const struct resource vsc7512_gcb_resource =
+	DEFINE_RES_REG_NAMED(VSC7512_CHIP_REGS_RES_START,
+			     VSC7512_CHIP_REGS_RES_SIZE,
+			     "devcpu_gcb_chip_regs");
+
+static int ocelot_spi_initialize(struct device *dev)
+{
+	struct ocelot_ddata *ddata = dev_get_drvdata(dev);
+	u32 val, check;
+	int err;
+
+	val = OCELOT_SPI_BYTE_ORDER;
+
+	/*
+	 * The SPI address must be big-endian, but we want the payload to match
+	 * our CPU. These are two bits (0 and 1) but they're repeated such that
+	 * the write from any configuration will be valid. The four
+	 * configurations are:
+	 *
+	 * 0b00: little-endian, MSB first
+	 * |            111111   | 22221111 | 33222222 |
+	 * | 76543210 | 54321098 | 32109876 | 10987654 |
+	 *
+	 * 0b01: big-endian, MSB first
+	 * | 33222222 | 22221111 | 111111   |          |
+	 * | 10987654 | 32109876 | 54321098 | 76543210 |
+	 *
+	 * 0b10: little-endian, LSB first
+	 * |              111111 | 11112222 | 22222233 |
+	 * | 01234567 | 89012345 | 67890123 | 45678901 |
+	 *
+	 * 0b11: big-endian, LSB first
+	 * | 22222233 | 11112222 |   111111 |          |
+	 * | 45678901 | 67890123 | 89012345 | 01234567 |
+	 */
+	err = regmap_write(ddata->cpuorg_regmap, REG_DEV_CPUORG_IF_CTRL, val);
+	if (err)
+		return err;
+
+	/*
+	 * Apply the number of padding bytes between a read request and the data
+	 * payload. Some registers have access times of up to 1us, so if the
+	 * first payload bit is shifted out too quickly, the read will fail.
+	 */
+	val = ddata->spi_padding_bytes;
+	err = regmap_write(ddata->cpuorg_regmap, REG_DEV_CPUORG_IF_CFGSTAT, val);
+	if (err)
+		return err;
+
+	/*
+	 * After we write the interface configuration, read it back here. This
+	 * will verify several different things. The first is that the number of
+	 * padding bytes actually got written correctly. These are found in bits
+	 * 0:3.
+	 *
+	 * The second is that bit 16 is cleared. Bit 16 is IF_CFGSTAT:IF_STAT,
+	 * and will be set if the register access is too fast. This would be in
+	 * the condition that the number of padding bytes is insufficient for
+	 * the SPI bus frequency.
+	 *
+	 * The last check is for bits 31:24, which define the interface by which
+	 * the registers are being accessed. Since we're accessing them via the
+	 * serial interface, it must return IF_NUM_SI.
+	 */
+	check = val | CFGSTAT_IF_NUM_SI;
+
+	err = regmap_read(ddata->cpuorg_regmap, REG_DEV_CPUORG_IF_CFGSTAT, &val);
+	if (err)
+		return err;
+
+	if (check != val)
+		return -ENODEV;
+
+	return 0;
+}
+
+static const struct regmap_config ocelot_spi_regmap_config = {
+	.reg_bits = 24,
+	.reg_stride = 4,
+	.reg_downshift = 2,
+	.val_bits = 32,
+
+	.write_flag_mask = 0x80,
+
+	.use_single_write = true,
+	.can_multi_write = false,
+
+	.reg_format_endian = REGMAP_ENDIAN_BIG,
+	.val_format_endian = REGMAP_ENDIAN_NATIVE,
+};
+
+static int ocelot_spi_regmap_bus_read(void *context, const void *reg, size_t reg_size,
+				      void *val, size_t val_size)
+{
+	struct spi_transfer xfers[3] = {0};
+	struct device *dev = context;
+	struct ocelot_ddata *ddata;
+	struct spi_device *spi;
+	struct spi_message msg;
+	unsigned int index = 0;
+
+	ddata = dev_get_drvdata(dev);
+	spi = to_spi_device(dev);
+
+	xfers[index].tx_buf = reg;
+	xfers[index].len = reg_size;
+	index++;
+
+	if (ddata->spi_padding_bytes) {
+		xfers[index].len = ddata->spi_padding_bytes;
+		xfers[index].tx_buf = ddata->dummy_buf;
+		xfers[index].dummy_data = 1;
+		index++;
+	}
+
+	xfers[index].rx_buf = val;
+	xfers[index].len = val_size;
+	index++;
+
+	spi_message_init_with_transfers(&msg, xfers, index);
+
+	return spi_sync(spi, &msg);
+}
+
+static int ocelot_spi_regmap_bus_write(void *context, const void *data, size_t count)
+{
+	struct device *dev = context;
+	struct spi_device *spi = to_spi_device(dev);
+
+	return spi_write(spi, data, count);
+}
+
+static const struct regmap_bus ocelot_spi_regmap_bus = {
+	.write = ocelot_spi_regmap_bus_write,
+	.read = ocelot_spi_regmap_bus_read,
+};
+
+struct regmap *ocelot_spi_init_regmap(struct device *dev, const struct resource *res)
+{
+	struct regmap_config regmap_config;
+
+	memcpy(&regmap_config, &ocelot_spi_regmap_config, sizeof(regmap_config));
+
+	regmap_config.name = res->name;
+	regmap_config.max_register = resource_size(res) - 1;
+	regmap_config.reg_base = res->start;
+
+	return devm_regmap_init(dev, &ocelot_spi_regmap_bus, dev, &regmap_config);
+}
+EXPORT_SYMBOL_NS(ocelot_spi_init_regmap, MFD_OCELOT_SPI);
+
+static int ocelot_spi_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct ocelot_ddata *ddata;
+	struct regmap *r;
+	int err;
+
+	ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL);
+	if (!ddata)
+		return -ENOMEM;
+
+	spi_set_drvdata(spi, ddata);
+
+	if (spi->max_speed_hz <= 500000) {
+		ddata->spi_padding_bytes = 0;
+	} else {
+		/*
+		 * Calculation taken from the manual for IF_CFGSTAT:IF_CFG.
+		 * Register access time is 1us, so we need to configure and send
+		 * out enough padding bytes between the read request and data
+		 * transmission that lasts at least 1 microsecond.
+		 */
+		ddata->spi_padding_bytes = 1 + (spi->max_speed_hz / HZ_PER_MHZ + 2) / 8;
+
+		ddata->dummy_buf = devm_kzalloc(dev, ddata->spi_padding_bytes, GFP_KERNEL);
+		if (!ddata->dummy_buf)
+			return -ENOMEM;
+	}
+
+	spi->bits_per_word = 8;
+
+	err = spi_setup(spi);
+	if (err)
+		return dev_err_probe(&spi->dev, err, "Error performing SPI setup\n");
+
+	r = ocelot_spi_init_regmap(dev, &vsc7512_dev_cpuorg_resource);
+	if (IS_ERR(r))
+		return PTR_ERR(r);
+
+	ddata->cpuorg_regmap = r;
+
+	r = ocelot_spi_init_regmap(dev, &vsc7512_gcb_resource);
+	if (IS_ERR(r))
+		return PTR_ERR(r);
+
+	ddata->gcb_regmap = r;
+
+	/*
+	 * The chip must be set up for SPI before it gets initialized and reset.
+	 * This must be done before calling init, and after a chip reset is
+	 * performed.
+	 */
+	err = ocelot_spi_initialize(dev);
+	if (err)
+		return dev_err_probe(dev, err, "Error initializing SPI bus\n");
+
+	err = ocelot_chip_reset(dev);
+	if (err)
+		return dev_err_probe(dev, err, "Error resetting device\n");
+
+	/*
+	 * A chip reset will clear the SPI configuration, so it needs to be done
+	 * again before we can access any registers.
+	 */
+	err = ocelot_spi_initialize(dev);
+	if (err)
+		return dev_err_probe(dev, err, "Error initializing SPI bus after reset\n");
+
+	err = ocelot_core_init(dev);
+	if (err)
+		return dev_err_probe(dev, err, "Error initializing Ocelot core\n");
+
+	return 0;
+}
+
+static const struct spi_device_id ocelot_spi_ids[] = {
+	{ "vsc7512", 0 },
+	{ }
+};
+
+static const struct of_device_id ocelot_spi_of_match[] = {
+	{ .compatible = "mscc,vsc7512" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ocelot_spi_of_match);
+
+static struct spi_driver ocelot_spi_driver = {
+	.driver = {
+		.name = "ocelot-soc",
+		.of_match_table = ocelot_spi_of_match,
+	},
+	.id_table = ocelot_spi_ids,
+	.probe = ocelot_spi_probe,
+};
+module_spi_driver(ocelot_spi_driver);
+
+MODULE_DESCRIPTION("SPI Controlled Ocelot Chip Driver");
+MODULE_AUTHOR("Colin Foster <colin.foster@in-advantage.com>");
+MODULE_LICENSE("Dual MIT/GPL");
+MODULE_IMPORT_NS(MFD_OCELOT);
diff --git a/drivers/mfd/ocelot.h b/drivers/mfd/ocelot.h
new file mode 100644
index 000000000000..b8bc2f1486e2
--- /dev/null
+++ b/drivers/mfd/ocelot.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2021, 2022 Innovative Advantage Inc. */
+
+#ifndef _MFD_OCELOT_H
+#define _MFD_OCELOT_H
+
+#include <linux/kconfig.h>
+
+struct device;
+struct regmap;
+struct resource;
+
+/**
+ * struct ocelot_ddata - Private data for an external Ocelot chip
+ * @gcb_regmap:		General Configuration Block regmap. Used for
+ *			operations like chip reset.
+ * @cpuorg_regmap:	CPU Device Origin Block regmap. Used for operations
+ *			like SPI bus configuration.
+ * @spi_padding_bytes:	Number of padding bytes that must be thrown out before
+ *			read data gets returned. This is calculated during
+ *			initialization based on bus speed.
+ * @dummy_buf:		Zero-filled buffer of spi_padding_bytes size. The dummy
+ *			bytes that will be sent out between the address and
+ *			data of a SPI read operation.
+ */
+struct ocelot_ddata {
+	struct regmap *gcb_regmap;
+	struct regmap *cpuorg_regmap;
+	int spi_padding_bytes;
+	void *dummy_buf;
+};
+
+int ocelot_chip_reset(struct device *dev);
+int ocelot_core_init(struct device *dev);
+
+/* SPI-specific routines that won't be necessary for other interfaces */
+struct regmap *ocelot_spi_init_regmap(struct device *dev,
+				      const struct resource *res);
+
+#define OCELOT_SPI_BYTE_ORDER_LE 0x00000000
+#define OCELOT_SPI_BYTE_ORDER_BE 0x81818181
+
+#ifdef __LITTLE_ENDIAN
+#define OCELOT_SPI_BYTE_ORDER OCELOT_SPI_BYTE_ORDER_LE
+#else
+#define OCELOT_SPI_BYTE_ORDER OCELOT_SPI_BYTE_ORDER_BE
+#endif
+
+#endif
diff --git a/drivers/mfd/palmas.c b/drivers/mfd/palmas.c
index f5b3fa973b13..8b7429bd2e3e 100644
--- a/drivers/mfd/palmas.c
+++ b/drivers/mfd/palmas.c
@@ -700,7 +700,7 @@ err_i2c:
 	return ret;
 }
 
-static int palmas_i2c_remove(struct i2c_client *i2c)
+static void palmas_i2c_remove(struct i2c_client *i2c)
 {
 	struct palmas *palmas = i2c_get_clientdata(i2c);
 	int i;
@@ -716,8 +716,6 @@ static int palmas_i2c_remove(struct i2c_client *i2c)
 		pm_power_off = NULL;
 		palmas_dev = NULL;
 	}
-
-	return 0;
 }
 
 static const struct i2c_device_id palmas_i2c_id[] = {
diff --git a/drivers/mfd/pcf50633-core.c b/drivers/mfd/pcf50633-core.c
index e9c565cf0f54..4ccc2c3e7681 100644
--- a/drivers/mfd/pcf50633-core.c
+++ b/drivers/mfd/pcf50633-core.c
@@ -273,7 +273,7 @@ err2:
 	return ret;
 }
 
-static int pcf50633_remove(struct i2c_client *client)
+static void pcf50633_remove(struct i2c_client *client)
 {
 	struct pcf50633 *pcf = i2c_get_clientdata(client);
 	int i;
@@ -289,8 +289,6 @@ static int pcf50633_remove(struct i2c_client *client)
 
 	for (i = 0; i < PCF50633_NUM_REGULATORS; i++)
 		platform_device_unregister(pcf->regulator_pdev[i]);
-
-	return 0;
 }
 
 static const struct i2c_device_id pcf50633_id_table[] = {
diff --git a/drivers/mfd/retu-mfd.c b/drivers/mfd/retu-mfd.c
index c748fd29a220..3b5acf7ca39c 100644
--- a/drivers/mfd/retu-mfd.c
+++ b/drivers/mfd/retu-mfd.c
@@ -287,7 +287,7 @@ static int retu_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 	return 0;
 }
 
-static int retu_remove(struct i2c_client *i2c)
+static void retu_remove(struct i2c_client *i2c)
 {
 	struct retu_dev *rdev = i2c_get_clientdata(i2c);
 
@@ -297,8 +297,6 @@ static int retu_remove(struct i2c_client *i2c)
 	}
 	mfd_remove_devices(rdev->dev);
 	regmap_del_irq_chip(i2c->irq, rdev->irq_data);
-
-	return 0;
 }
 
 static const struct i2c_device_id retu_id[] = {
diff --git a/drivers/mfd/rk808.c b/drivers/mfd/rk808.c
index 4142b638e5fa..d5d641efa077 100644
--- a/drivers/mfd/rk808.c
+++ b/drivers/mfd/rk808.c
@@ -778,7 +778,7 @@ err_irq:
 	return ret;
 }
 
-static int rk808_remove(struct i2c_client *client)
+static void rk808_remove(struct i2c_client *client)
 {
 	struct rk808 *rk808 = i2c_get_clientdata(client);
 
@@ -792,8 +792,6 @@ static int rk808_remove(struct i2c_client *client)
 		pm_power_off = NULL;
 
 	unregister_restart_handler(&rk808_restart_handler);
-
-	return 0;
 }
 
 static int __maybe_unused rk8xx_suspend(struct device *dev)
diff --git a/drivers/mfd/rn5t618.c b/drivers/mfd/rn5t618.c
index 384acb459427..eb8005b4e58d 100644
--- a/drivers/mfd/rn5t618.c
+++ b/drivers/mfd/rn5t618.c
@@ -241,7 +241,7 @@ static int rn5t618_i2c_probe(struct i2c_client *i2c)
 	return rn5t618_irq_init(priv);
 }
 
-static int rn5t618_i2c_remove(struct i2c_client *i2c)
+static void rn5t618_i2c_remove(struct i2c_client *i2c)
 {
 	if (i2c == rn5t618_pm_power_off) {
 		rn5t618_pm_power_off = NULL;
@@ -249,8 +249,6 @@ static int rn5t618_i2c_remove(struct i2c_client *i2c)
 	}
 
 	unregister_restart_handler(&rn5t618_restart_handler);
-
-	return 0;
 }
 
 static int __maybe_unused rn5t618_i2c_suspend(struct device *dev)
diff --git a/drivers/mfd/rsmu_i2c.c b/drivers/mfd/rsmu_i2c.c
index dc001c9791c1..f716ab8039a0 100644
--- a/drivers/mfd/rsmu_i2c.c
+++ b/drivers/mfd/rsmu_i2c.c
@@ -146,13 +146,11 @@ static int rsmu_i2c_probe(struct i2c_client *client,
 	return rsmu_core_init(rsmu);
 }
 
-static int rsmu_i2c_remove(struct i2c_client *client)
+static void rsmu_i2c_remove(struct i2c_client *client)
 {
 	struct rsmu_ddata *rsmu = i2c_get_clientdata(client);
 
 	rsmu_core_exit(rsmu);
-
-	return 0;
 }
 
 static const struct i2c_device_id rsmu_i2c_id[] = {
diff --git a/drivers/mfd/rt4831.c b/drivers/mfd/rt4831.c
index fb3bd788a3eb..c6d34dc2b520 100644
--- a/drivers/mfd/rt4831.c
+++ b/drivers/mfd/rt4831.c
@@ -87,7 +87,7 @@ static int rt4831_probe(struct i2c_client *client)
 				    ARRAY_SIZE(rt4831_subdevs), NULL, 0, NULL);
 }
 
-static int rt4831_remove(struct i2c_client *client)
+static void rt4831_remove(struct i2c_client *client)
 {
 	struct regmap *regmap = dev_get_regmap(&client->dev, NULL);
 	int ret;
@@ -96,8 +96,6 @@ static int rt4831_remove(struct i2c_client *client)
 	ret = regmap_update_bits(regmap, RT4831_REG_ENABLE, RT4831_RESET_MASK, RT4831_RESET_MASK);
 	if (ret)
 		dev_warn(&client->dev, "Failed to disable outputs (%pe)\n", ERR_PTR(ret));
-
-	return 0;
 }
 
 static const struct of_device_id __maybe_unused rt4831_of_match[] = {
diff --git a/drivers/mfd/si476x-i2c.c b/drivers/mfd/si476x-i2c.c
index a2635c2d9d1a..8166949b725c 100644
--- a/drivers/mfd/si476x-i2c.c
+++ b/drivers/mfd/si476x-i2c.c
@@ -835,7 +835,7 @@ free_gpio:
 	return rval;
 }
 
-static int si476x_core_remove(struct i2c_client *client)
+static void si476x_core_remove(struct i2c_client *client)
 {
 	struct si476x_core *core = i2c_get_clientdata(client);
 
@@ -851,8 +851,6 @@ static int si476x_core_remove(struct i2c_client *client)
 
 	if (gpio_is_valid(core->gpio_reset))
 		gpio_free(core->gpio_reset);
-
-	return 0;
 }
 
 
diff --git a/drivers/mfd/stmfx.c b/drivers/mfd/stmfx.c
index 122f96094410..5dd7d9688459 100644
--- a/drivers/mfd/stmfx.c
+++ b/drivers/mfd/stmfx.c
@@ -467,13 +467,11 @@ err_chip_exit:
 	return ret;
 }
 
-static int stmfx_remove(struct i2c_client *client)
+static void stmfx_remove(struct i2c_client *client)
 {
 	stmfx_irq_exit(client);
 
 	stmfx_chip_exit(client);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/mfd/stmpe-i2c.c b/drivers/mfd/stmpe-i2c.c
index d3eedf3d607e..4d55494a97c4 100644
--- a/drivers/mfd/stmpe-i2c.c
+++ b/drivers/mfd/stmpe-i2c.c
@@ -91,13 +91,11 @@ stmpe_i2c_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 	return stmpe_probe(&i2c_ci, partnum);
 }
 
-static int stmpe_i2c_remove(struct i2c_client *i2c)
+static void stmpe_i2c_remove(struct i2c_client *i2c)
 {
 	struct stmpe *stmpe = dev_get_drvdata(&i2c->dev);
 
 	stmpe_remove(stmpe);
-
-	return 0;
 }
 
 static const struct i2c_device_id stmpe_i2c_id[] = {
diff --git a/drivers/mfd/tc3589x.c b/drivers/mfd/tc3589x.c
index 13583cdb93b6..d5d0ec117acb 100644
--- a/drivers/mfd/tc3589x.c
+++ b/drivers/mfd/tc3589x.c
@@ -429,13 +429,11 @@ static int tc3589x_probe(struct i2c_client *i2c,
 	return 0;
 }
 
-static int tc3589x_remove(struct i2c_client *client)
+static void tc3589x_remove(struct i2c_client *client)
 {
 	struct tc3589x *tc3589x = i2c_get_clientdata(client);
 
 	mfd_remove_devices(tc3589x->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/mfd/tps6105x.c b/drivers/mfd/tps6105x.c
index c906324d293e..b360568ea675 100644
--- a/drivers/mfd/tps6105x.c
+++ b/drivers/mfd/tps6105x.c
@@ -179,7 +179,7 @@ static int tps6105x_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int tps6105x_remove(struct i2c_client *client)
+static void tps6105x_remove(struct i2c_client *client)
 {
 	struct tps6105x *tps6105x = i2c_get_clientdata(client);
 
@@ -189,8 +189,6 @@ static int tps6105x_remove(struct i2c_client *client)
 	regmap_update_bits(tps6105x->regmap, TPS6105X_REG_0,
 		TPS6105X_REG0_MODE_MASK,
 		TPS6105X_MODE_SHUTDOWN << TPS6105X_REG0_MODE_SHIFT);
-
-	return 0;
 }
 
 static const struct i2c_device_id tps6105x_id[] = {
diff --git a/drivers/mfd/tps65010.c b/drivers/mfd/tps65010.c
index 7e7dbee58ca9..c2afa2e69f42 100644
--- a/drivers/mfd/tps65010.c
+++ b/drivers/mfd/tps65010.c
@@ -501,7 +501,7 @@ static int tps65010_gpio_get(struct gpio_chip *chip, unsigned offset)
 
 static struct tps65010 *the_tps;
 
-static int tps65010_remove(struct i2c_client *client)
+static void tps65010_remove(struct i2c_client *client)
 {
 	struct tps65010		*tps = i2c_get_clientdata(client);
 	struct tps65010_board	*board = dev_get_platdata(&client->dev);
@@ -517,7 +517,6 @@ static int tps65010_remove(struct i2c_client *client)
 	cancel_delayed_work_sync(&tps->work);
 	debugfs_remove(tps->file);
 	the_tps = NULL;
-	return 0;
 }
 
 static int tps65010_probe(struct i2c_client *client,
diff --git a/drivers/mfd/tps65086.c b/drivers/mfd/tps65086.c
index cbae9777a24e..81a7360a87bb 100644
--- a/drivers/mfd/tps65086.c
+++ b/drivers/mfd/tps65086.c
@@ -111,14 +111,12 @@ static int tps65086_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int tps65086_remove(struct i2c_client *client)
+static void tps65086_remove(struct i2c_client *client)
 {
 	struct tps65086 *tps = i2c_get_clientdata(client);
 
 	if (tps->irq > 0)
 		regmap_del_irq_chip(tps->irq, tps->irq_data);
-
-	return 0;
 }
 
 static const struct i2c_device_id tps65086_id_table[] = {
diff --git a/drivers/mfd/tps65217.c b/drivers/mfd/tps65217.c
index 8e8da204a02e..eebd60601b01 100644
--- a/drivers/mfd/tps65217.c
+++ b/drivers/mfd/tps65217.c
@@ -374,7 +374,7 @@ static int tps65217_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int tps65217_remove(struct i2c_client *client)
+static void tps65217_remove(struct i2c_client *client)
 {
 	struct tps65217 *tps = i2c_get_clientdata(client);
 	unsigned int virq;
@@ -388,8 +388,6 @@ static int tps65217_remove(struct i2c_client *client)
 
 	irq_domain_remove(tps->irq_domain);
 	tps->irq_domain = NULL;
-
-	return 0;
 }
 
 static const struct i2c_device_id tps65217_id_table[] = {
diff --git a/drivers/mfd/tps6586x.c b/drivers/mfd/tps6586x.c
index c9303d3d6602..fb340da64bbc 100644
--- a/drivers/mfd/tps6586x.c
+++ b/drivers/mfd/tps6586x.c
@@ -579,7 +579,7 @@ err_mfd_add:
 	return ret;
 }
 
-static int tps6586x_i2c_remove(struct i2c_client *client)
+static void tps6586x_i2c_remove(struct i2c_client *client)
 {
 	struct tps6586x *tps6586x = i2c_get_clientdata(client);
 
@@ -587,7 +587,6 @@ static int tps6586x_i2c_remove(struct i2c_client *client)
 	mfd_remove_devices(tps6586x->dev);
 	if (client->irq)
 		free_irq(client->irq, tps6586x);
-	return 0;
 }
 
 static int __maybe_unused tps6586x_i2c_suspend(struct device *dev)
diff --git a/drivers/mfd/tps65912-i2c.c b/drivers/mfd/tps65912-i2c.c
index afb7f7d97dc0..7e2b19efe867 100644
--- a/drivers/mfd/tps65912-i2c.c
+++ b/drivers/mfd/tps65912-i2c.c
@@ -43,13 +43,11 @@ static int tps65912_i2c_probe(struct i2c_client *client,
 	return tps65912_device_init(tps);
 }
 
-static int tps65912_i2c_remove(struct i2c_client *client)
+static void tps65912_i2c_remove(struct i2c_client *client)
 {
 	struct tps65912 *tps = i2c_get_clientdata(client);
 
 	tps65912_device_exit(tps);
-
-	return 0;
 }
 
 static const struct i2c_device_id tps65912_i2c_id_table[] = {
diff --git a/drivers/mfd/twl-core.c b/drivers/mfd/twl-core.c
index 2cb9326f3e61..2679c41232e6 100644
--- a/drivers/mfd/twl-core.c
+++ b/drivers/mfd/twl-core.c
@@ -727,7 +727,7 @@ static void clocks_init(struct device *dev)
 /*----------------------------------------------------------------------*/
 
 
-static int twl_remove(struct i2c_client *client)
+static void twl_remove(struct i2c_client *client)
 {
 	unsigned i, num_slaves;
 
@@ -745,7 +745,6 @@ static int twl_remove(struct i2c_client *client)
 		twl->client = NULL;
 	}
 	twl_priv->ready = false;
-	return 0;
 }
 
 static struct of_dev_auxdata twl_auxdata_lookup[] = {
diff --git a/drivers/mfd/twl6040.c b/drivers/mfd/twl6040.c
index b9c6d94b4002..f429b8f00db6 100644
--- a/drivers/mfd/twl6040.c
+++ b/drivers/mfd/twl6040.c
@@ -808,7 +808,7 @@ gpio_err:
 	return ret;
 }
 
-static int twl6040_remove(struct i2c_client *client)
+static void twl6040_remove(struct i2c_client *client)
 {
 	struct twl6040 *twl6040 = i2c_get_clientdata(client);
 
@@ -820,8 +820,6 @@ static int twl6040_remove(struct i2c_client *client)
 	mfd_remove_devices(&client->dev);
 
 	regulator_bulk_disable(TWL6040_NUM_SUPPLIES, twl6040->supplies);
-
-	return 0;
 }
 
 static const struct i2c_device_id twl6040_i2c_id[] = {
diff --git a/drivers/mfd/wm8994-core.c b/drivers/mfd/wm8994-core.c
index 7b1d270722ba..7e88f5b0abe6 100644
--- a/drivers/mfd/wm8994-core.c
+++ b/drivers/mfd/wm8994-core.c
@@ -657,13 +657,11 @@ static int wm8994_i2c_probe(struct i2c_client *i2c,
 	return wm8994_device_init(wm8994, i2c->irq);
 }
 
-static int wm8994_i2c_remove(struct i2c_client *i2c)
+static void wm8994_i2c_remove(struct i2c_client *i2c)
 {
 	struct wm8994 *wm8994 = i2c_get_clientdata(i2c);
 
 	wm8994_device_exit(wm8994);
-
-	return 0;
 }
 
 static const struct i2c_device_id wm8994_i2c_id[] = {
diff --git a/drivers/misc/ad525x_dpot-i2c.c b/drivers/misc/ad525x_dpot-i2c.c
index 0ee0c6d808c3..28ffb4377d98 100644
--- a/drivers/misc/ad525x_dpot-i2c.c
+++ b/drivers/misc/ad525x_dpot-i2c.c
@@ -67,10 +67,9 @@ static int ad_dpot_i2c_probe(struct i2c_client *client,
 	return ad_dpot_probe(&client->dev, &bdata, id->driver_data, id->name);
 }
 
-static int ad_dpot_i2c_remove(struct i2c_client *client)
+static void ad_dpot_i2c_remove(struct i2c_client *client)
 {
 	ad_dpot_remove(&client->dev);
-	return 0;
 }
 
 static const struct i2c_device_id ad_dpot_id[] = {
diff --git a/drivers/misc/apds9802als.c b/drivers/misc/apds9802als.c
index 6fff44b952bd..a32431f4b370 100644
--- a/drivers/misc/apds9802als.c
+++ b/drivers/misc/apds9802als.c
@@ -242,7 +242,7 @@ als_error1:
 	return res;
 }
 
-static int apds9802als_remove(struct i2c_client *client)
+static void apds9802als_remove(struct i2c_client *client)
 {
 	struct als_data *data = i2c_get_clientdata(client);
 
@@ -256,7 +256,6 @@ static int apds9802als_remove(struct i2c_client *client)
 	pm_runtime_put_noidle(&client->dev);
 
 	kfree(data);
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/misc/apds990x.c b/drivers/misc/apds990x.c
index 45f5b997a0e1..e2100cc42ce8 100644
--- a/drivers/misc/apds990x.c
+++ b/drivers/misc/apds990x.c
@@ -1185,7 +1185,7 @@ fail1:
 	return err;
 }
 
-static int apds990x_remove(struct i2c_client *client)
+static void apds990x_remove(struct i2c_client *client)
 {
 	struct apds990x_chip *chip = i2c_get_clientdata(client);
 
@@ -1205,7 +1205,6 @@ static int apds990x_remove(struct i2c_client *client)
 	regulator_bulk_free(ARRAY_SIZE(chip->regs), chip->regs);
 
 	kfree(chip);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/misc/bh1770glc.c b/drivers/misc/bh1770glc.c
index 0581bb9cef2e..d0dfa674414c 100644
--- a/drivers/misc/bh1770glc.c
+++ b/drivers/misc/bh1770glc.c
@@ -1280,7 +1280,7 @@ fail0:
 	return err;
 }
 
-static int bh1770_remove(struct i2c_client *client)
+static void bh1770_remove(struct i2c_client *client)
 {
 	struct bh1770_chip *chip = i2c_get_clientdata(client);
 
@@ -1299,8 +1299,6 @@ static int bh1770_remove(struct i2c_client *client)
 
 	pm_runtime_disable(&client->dev);
 	pm_runtime_set_suspended(&client->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/misc/ds1682.c b/drivers/misc/ds1682.c
index 42f316c2d719..0698ddc5f4d5 100644
--- a/drivers/misc/ds1682.c
+++ b/drivers/misc/ds1682.c
@@ -228,11 +228,10 @@ static int ds1682_probe(struct i2c_client *client,
 	return rc;
 }
 
-static int ds1682_remove(struct i2c_client *client)
+static void ds1682_remove(struct i2c_client *client)
 {
 	sysfs_remove_bin_file(&client->dev.kobj, &ds1682_eeprom_attr);
 	sysfs_remove_group(&client->dev.kobj, &ds1682_group);
-	return 0;
 }
 
 static const struct i2c_device_id ds1682_id[] = {
diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c
index 633e1cf08d6e..938c4f41b98c 100644
--- a/drivers/misc/eeprom/at24.c
+++ b/drivers/misc/eeprom/at24.c
@@ -791,7 +791,7 @@ static int at24_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int at24_remove(struct i2c_client *client)
+static void at24_remove(struct i2c_client *client)
 {
 	struct at24_data *at24 = i2c_get_clientdata(client);
 
@@ -801,8 +801,6 @@ static int at24_remove(struct i2c_client *client)
 			regulator_disable(at24->vcc_reg);
 		pm_runtime_set_suspended(&client->dev);
 	}
-
-	return 0;
 }
 
 static int __maybe_unused at24_suspend(struct device *dev)
diff --git a/drivers/misc/eeprom/ee1004.c b/drivers/misc/eeprom/ee1004.c
index 9fbfe784d710..c8c6deb7ed89 100644
--- a/drivers/misc/eeprom/ee1004.c
+++ b/drivers/misc/eeprom/ee1004.c
@@ -219,14 +219,12 @@ static int ee1004_probe(struct i2c_client *client)
 	return err;
 }
 
-static int ee1004_remove(struct i2c_client *client)
+static void ee1004_remove(struct i2c_client *client)
 {
 	/* Remove page select clients if this is the last device */
 	mutex_lock(&ee1004_bus_lock);
 	ee1004_cleanup(EE1004_NUM_PAGES);
 	mutex_unlock(&ee1004_bus_lock);
-
-	return 0;
 }
 
 /*-------------------------------------------------------------------------*/
diff --git a/drivers/misc/eeprom/eeprom.c b/drivers/misc/eeprom/eeprom.c
index 34fa385dfd4b..4a9445fea93d 100644
--- a/drivers/misc/eeprom/eeprom.c
+++ b/drivers/misc/eeprom/eeprom.c
@@ -183,11 +183,9 @@ static int eeprom_probe(struct i2c_client *client,
 	return sysfs_create_bin_file(&client->dev.kobj, &eeprom_attr);
 }
 
-static int eeprom_remove(struct i2c_client *client)
+static void eeprom_remove(struct i2c_client *client)
 {
 	sysfs_remove_bin_file(&client->dev.kobj, &eeprom_attr);
-
-	return 0;
 }
 
 static const struct i2c_device_id eeprom_id[] = {
diff --git a/drivers/misc/eeprom/idt_89hpesx.c b/drivers/misc/eeprom/idt_89hpesx.c
index 9aec3338e37d..ada2a3af36d7 100644
--- a/drivers/misc/eeprom/idt_89hpesx.c
+++ b/drivers/misc/eeprom/idt_89hpesx.c
@@ -1405,7 +1405,7 @@ err_free_pdev:
 /*
  * idt_remove() - IDT 89HPESx driver remove() callback method
  */
-static int idt_remove(struct i2c_client *client)
+static void idt_remove(struct i2c_client *client)
 {
 	struct idt_89hpesx_dev *pdev = i2c_get_clientdata(client);
 
@@ -1417,8 +1417,6 @@ static int idt_remove(struct i2c_client *client)
 
 	/* Discard driver data structure */
 	idt_free_pdev(pdev);
-
-	return 0;
 }
 
 /*
diff --git a/drivers/misc/eeprom/max6875.c b/drivers/misc/eeprom/max6875.c
index 9da81f6d4a1c..6bd4f4339af4 100644
--- a/drivers/misc/eeprom/max6875.c
+++ b/drivers/misc/eeprom/max6875.c
@@ -173,7 +173,7 @@ exit_kfree:
 	return err;
 }
 
-static int max6875_remove(struct i2c_client *client)
+static void max6875_remove(struct i2c_client *client)
 {
 	struct max6875_data *data = i2c_get_clientdata(client);
 
@@ -181,8 +181,6 @@ static int max6875_remove(struct i2c_client *client)
 
 	sysfs_remove_bin_file(&client->dev.kobj, &user_eeprom_attr);
 	kfree(data);
-
-	return 0;
 }
 
 static const struct i2c_device_id max6875_id[] = {
diff --git a/drivers/misc/hmc6352.c b/drivers/misc/hmc6352.c
index 572a2ff10f00..42b9adef28a3 100644
--- a/drivers/misc/hmc6352.c
+++ b/drivers/misc/hmc6352.c
@@ -116,10 +116,9 @@ static int hmc6352_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int hmc6352_remove(struct i2c_client *client)
+static void hmc6352_remove(struct i2c_client *client)
 {
 	sysfs_remove_group(&client->dev.kobj, &m_compass_gr);
-	return 0;
 }
 
 static const struct i2c_device_id hmc6352_id[] = {
diff --git a/drivers/misc/ibmvmc.c b/drivers/misc/ibmvmc.c
index c0fe3295c330..cbaf6d35e854 100644
--- a/drivers/misc/ibmvmc.c
+++ b/drivers/misc/ibmvmc.c
@@ -1039,6 +1039,7 @@ static unsigned int ibmvmc_poll(struct file *file, poll_table *wait)
 static ssize_t ibmvmc_write(struct file *file, const char *buffer,
 			    size_t count, loff_t *ppos)
 {
+	struct inode *inode;
 	struct ibmvmc_buffer *vmc_buffer;
 	struct ibmvmc_file_session *session;
 	struct crq_server_adapter *adapter;
@@ -1122,8 +1123,9 @@ static ssize_t ibmvmc_write(struct file *file, const char *buffer,
 	if (p == buffer)
 		goto out;
 
-	file->f_path.dentry->d_inode->i_mtime = current_time(file_inode(file));
-	mark_inode_dirty(file->f_path.dentry->d_inode);
+	inode = file_inode(file);
+	inode->i_mtime = current_time(inode);
+	mark_inode_dirty(inode);
 
 	dev_dbg(adapter->dev, "write: file = 0x%lx, count = 0x%lx\n",
 		(unsigned long)file, (unsigned long)count);
diff --git a/drivers/misc/ics932s401.c b/drivers/misc/ics932s401.c
index 0f9ea75b0b18..2c4bb6d6e1a0 100644
--- a/drivers/misc/ics932s401.c
+++ b/drivers/misc/ics932s401.c
@@ -93,7 +93,7 @@ static int ics932s401_probe(struct i2c_client *client,
 			 const struct i2c_device_id *id);
 static int ics932s401_detect(struct i2c_client *client,
 			  struct i2c_board_info *info);
-static int ics932s401_remove(struct i2c_client *client);
+static void ics932s401_remove(struct i2c_client *client);
 
 static const struct i2c_device_id ics932s401_id[] = {
 	{ "ics932s401", 0 },
@@ -460,13 +460,12 @@ exit:
 	return err;
 }
 
-static int ics932s401_remove(struct i2c_client *client)
+static void ics932s401_remove(struct i2c_client *client)
 {
 	struct ics932s401_data *data = i2c_get_clientdata(client);
 
 	sysfs_remove_group(&client->dev.kobj, &data->attrs);
 	kfree(data);
-	return 0;
 }
 
 module_i2c_driver(ics932s401_driver);
diff --git a/drivers/misc/isl29003.c b/drivers/misc/isl29003.c
index 703d20e83ebd..8ab61be79c76 100644
--- a/drivers/misc/isl29003.c
+++ b/drivers/misc/isl29003.c
@@ -410,12 +410,11 @@ exit_kfree:
 	return err;
 }
 
-static int isl29003_remove(struct i2c_client *client)
+static void isl29003_remove(struct i2c_client *client)
 {
 	sysfs_remove_group(&client->dev.kobj, &isl29003_attr_group);
 	isl29003_set_power_state(client, 0);
 	kfree(i2c_get_clientdata(client));
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/misc/isl29020.c b/drivers/misc/isl29020.c
index fc5ff2805b94..c6f2a94f501a 100644
--- a/drivers/misc/isl29020.c
+++ b/drivers/misc/isl29020.c
@@ -171,11 +171,10 @@ static int  isl29020_probe(struct i2c_client *client,
 	return res;
 }
 
-static int isl29020_remove(struct i2c_client *client)
+static void isl29020_remove(struct i2c_client *client)
 {
 	pm_runtime_disable(&client->dev);
 	sysfs_remove_group(&client->dev.kobj, &m_als_gr);
-	return 0;
 }
 
 static const struct i2c_device_id isl29020_id[] = {
diff --git a/drivers/misc/lis3lv02d/lis3lv02d_i2c.c b/drivers/misc/lis3lv02d/lis3lv02d_i2c.c
index 52555d2e824b..d7daa01fe7ca 100644
--- a/drivers/misc/lis3lv02d/lis3lv02d_i2c.c
+++ b/drivers/misc/lis3lv02d/lis3lv02d_i2c.c
@@ -177,7 +177,7 @@ fail:
 	return ret;
 }
 
-static int lis3lv02d_i2c_remove(struct i2c_client *client)
+static void lis3lv02d_i2c_remove(struct i2c_client *client)
 {
 	struct lis3lv02d *lis3 = i2c_get_clientdata(client);
 	struct lis3lv02d_platform_data *pdata = client->dev.platform_data;
@@ -190,7 +190,6 @@ static int lis3lv02d_i2c_remove(struct i2c_client *client)
 
 	regulator_bulk_free(ARRAY_SIZE(lis3->regulators),
 			    lis3_dev.regulators);
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/misc/lkdtm/cfi.c b/drivers/misc/lkdtm/cfi.c
index 71483cb1e422..5245cf6013c9 100644
--- a/drivers/misc/lkdtm/cfi.c
+++ b/drivers/misc/lkdtm/cfi.c
@@ -20,6 +20,13 @@ static noinline int lkdtm_increment_int(int *counter)
 
 	return *counter;
 }
+
+/* Don't allow the compiler to inline the calls. */
+static noinline void lkdtm_indirect_call(void (*func)(int *))
+{
+	func(&called_count);
+}
+
 /*
  * This tries to call an indirect function with a mismatched prototype.
  */
@@ -29,15 +36,11 @@ static void lkdtm_CFI_FORWARD_PROTO(void)
 	 * Matches lkdtm_increment_void()'s prototype, but not
 	 * lkdtm_increment_int()'s prototype.
 	 */
-	void (*func)(int *);
-
 	pr_info("Calling matched prototype ...\n");
-	func = lkdtm_increment_void;
-	func(&called_count);
+	lkdtm_indirect_call(lkdtm_increment_void);
 
 	pr_info("Calling mismatched prototype ...\n");
-	func = (void *)lkdtm_increment_int;
-	func(&called_count);
+	lkdtm_indirect_call((void *)lkdtm_increment_int);
 
 	pr_err("FAIL: survived mismatched prototype function call!\n");
 	pr_expected_config(CONFIG_CFI_CLANG);
diff --git a/drivers/misc/lkdtm/fortify.c b/drivers/misc/lkdtm/fortify.c
index 080293fa3c52..015927665678 100644
--- a/drivers/misc/lkdtm/fortify.c
+++ b/drivers/misc/lkdtm/fortify.c
@@ -10,28 +10,31 @@
 
 static volatile int fortify_scratch_space;
 
-static void lkdtm_FORTIFIED_OBJECT(void)
+static void lkdtm_FORTIFY_STR_OBJECT(void)
 {
 	struct target {
 		char a[10];
-	} target[2] = {};
+		int foo;
+	} target[3] = {};
 	/*
 	 * Using volatile prevents the compiler from determining the value of
 	 * 'size' at compile time. Without that, we would get a compile error
 	 * rather than a runtime error.
 	 */
-	volatile int size = 11;
+	volatile int size = 20;
+
+	pr_info("trying to strcmp() past the end of a struct\n");
 
-	pr_info("trying to read past the end of a struct\n");
+	strncpy(target[0].a, target[1].a, size);
 
 	/* Store result to global to prevent the code from being eliminated */
-	fortify_scratch_space = memcmp(&target[0], &target[1], size);
+	fortify_scratch_space = target[0].a[3];
 
-	pr_err("FAIL: fortify did not block an object overread!\n");
+	pr_err("FAIL: fortify did not block a strncpy() object write overflow!\n");
 	pr_expected_config(CONFIG_FORTIFY_SOURCE);
 }
 
-static void lkdtm_FORTIFIED_SUBOBJECT(void)
+static void lkdtm_FORTIFY_STR_MEMBER(void)
 {
 	struct target {
 		char a[10];
@@ -44,7 +47,7 @@ static void lkdtm_FORTIFIED_SUBOBJECT(void)
 	strscpy(src, "over ten bytes", size);
 	size = strlen(src) + 1;
 
-	pr_info("trying to strncpy past the end of a member of a struct\n");
+	pr_info("trying to strncpy() past the end of a struct member...\n");
 
 	/*
 	 * strncpy(target.a, src, 20); will hit a compile error because the
@@ -56,7 +59,72 @@ static void lkdtm_FORTIFIED_SUBOBJECT(void)
 	/* Store result to global to prevent the code from being eliminated */
 	fortify_scratch_space = target.a[3];
 
-	pr_err("FAIL: fortify did not block an sub-object overrun!\n");
+	pr_err("FAIL: fortify did not block a strncpy() struct member write overflow!\n");
+	pr_expected_config(CONFIG_FORTIFY_SOURCE);
+
+	kfree(src);
+}
+
+static void lkdtm_FORTIFY_MEM_OBJECT(void)
+{
+	int before[10];
+	struct target {
+		char a[10];
+		int foo;
+	} target = {};
+	int after[10];
+	/*
+	 * Using volatile prevents the compiler from determining the value of
+	 * 'size' at compile time. Without that, we would get a compile error
+	 * rather than a runtime error.
+	 */
+	volatile int size = 20;
+
+	memset(before, 0, sizeof(before));
+	memset(after, 0, sizeof(after));
+	fortify_scratch_space = before[5];
+	fortify_scratch_space = after[5];
+
+	pr_info("trying to memcpy() past the end of a struct\n");
+
+	pr_info("0: %zu\n", __builtin_object_size(&target, 0));
+	pr_info("1: %zu\n", __builtin_object_size(&target, 1));
+	pr_info("s: %d\n", size);
+	memcpy(&target, &before, size);
+
+	/* Store result to global to prevent the code from being eliminated */
+	fortify_scratch_space = target.a[3];
+
+	pr_err("FAIL: fortify did not block a memcpy() object write overflow!\n");
+	pr_expected_config(CONFIG_FORTIFY_SOURCE);
+}
+
+static void lkdtm_FORTIFY_MEM_MEMBER(void)
+{
+	struct target {
+		char a[10];
+		char b[10];
+	} target;
+	volatile int size = 20;
+	char *src;
+
+	src = kmalloc(size, GFP_KERNEL);
+	strscpy(src, "over ten bytes", size);
+	size = strlen(src) + 1;
+
+	pr_info("trying to memcpy() past the end of a struct member...\n");
+
+	/*
+	 * strncpy(target.a, src, 20); will hit a compile error because the
+	 * compiler knows at build time that target.a < 20 bytes. Use a
+	 * volatile to force a runtime error.
+	 */
+	memcpy(target.a, src, size);
+
+	/* Store result to global to prevent the code from being eliminated */
+	fortify_scratch_space = target.a[3];
+
+	pr_err("FAIL: fortify did not block a memcpy() struct member write overflow!\n");
 	pr_expected_config(CONFIG_FORTIFY_SOURCE);
 
 	kfree(src);
@@ -67,7 +135,7 @@ static void lkdtm_FORTIFIED_SUBOBJECT(void)
  * strscpy and generate a panic because there is a write overflow (i.e. src
  * length is greater than dst length).
  */
-static void lkdtm_FORTIFIED_STRSCPY(void)
+static void lkdtm_FORTIFY_STRSCPY(void)
 {
 	char *src;
 	char dst[5];
@@ -136,9 +204,11 @@ static void lkdtm_FORTIFIED_STRSCPY(void)
 }
 
 static struct crashtype crashtypes[] = {
-	CRASHTYPE(FORTIFIED_OBJECT),
-	CRASHTYPE(FORTIFIED_SUBOBJECT),
-	CRASHTYPE(FORTIFIED_STRSCPY),
+	CRASHTYPE(FORTIFY_STR_OBJECT),
+	CRASHTYPE(FORTIFY_STR_MEMBER),
+	CRASHTYPE(FORTIFY_MEM_OBJECT),
+	CRASHTYPE(FORTIFY_MEM_MEMBER),
+	CRASHTYPE(FORTIFY_STRSCPY),
 };
 
 struct crashtype_category fortify_crashtypes = {
diff --git a/drivers/misc/lkdtm/usercopy.c b/drivers/misc/lkdtm/usercopy.c
index 6215ec995cd3..67db57249a34 100644
--- a/drivers/misc/lkdtm/usercopy.c
+++ b/drivers/misc/lkdtm/usercopy.c
@@ -330,7 +330,7 @@ static void lkdtm_USERCOPY_KERNEL(void)
 
 	pr_info("attempting bad copy_to_user from kernel text: %px\n",
 		vm_mmap);
-	if (copy_to_user((void __user *)user_addr, function_nocfi(vm_mmap),
+	if (copy_to_user((void __user *)user_addr, vm_mmap,
 			 unconst + PAGE_SIZE)) {
 		pr_warn("copy_to_user failed, but lacked Oops\n");
 		goto free_user;
diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c
index 59506ba6fc48..79305e4acce2 100644
--- a/drivers/misc/mei/bus-fixup.c
+++ b/drivers/misc/mei/bus-fixup.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2013-2020, Intel Corporation. All rights reserved.
+ * Copyright (c) 2013-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -15,6 +15,7 @@
 
 #include "mei_dev.h"
 #include "client.h"
+#include "mkhi.h"
 
 #define MEI_UUID_NFC_INFO UUID_LE(0xd2de1625, 0x382d, 0x417d, \
 			0x48, 0xa4, 0xef, 0xab, 0xba, 0x8a, 0x12, 0x06)
@@ -89,20 +90,6 @@ struct mei_os_ver {
 	u8  reserved2;
 } __packed;
 
-#define MKHI_FEATURE_PTT 0x10
-
-struct mkhi_rule_id {
-	__le16 rule_type;
-	u8 feature_id;
-	u8 reserved;
-} __packed;
-
-struct mkhi_fwcaps {
-	struct mkhi_rule_id id;
-	u8 len;
-	u8 data[];
-} __packed;
-
 struct mkhi_fw_ver_block {
 	u16 minor;
 	u8 major;
@@ -115,22 +102,6 @@ struct mkhi_fw_ver {
 	struct mkhi_fw_ver_block ver[MEI_MAX_FW_VER_BLOCKS];
 } __packed;
 
-#define MKHI_FWCAPS_GROUP_ID 0x3
-#define MKHI_FWCAPS_SET_OS_VER_APP_RULE_CMD 6
-#define MKHI_GEN_GROUP_ID 0xFF
-#define MKHI_GEN_GET_FW_VERSION_CMD 0x2
-struct mkhi_msg_hdr {
-	u8  group_id;
-	u8  command;
-	u8  reserved;
-	u8  result;
-} __packed;
-
-struct mkhi_msg {
-	struct mkhi_msg_hdr hdr;
-	u8 data[];
-} __packed;
-
 #define MKHI_OSVER_BUF_LEN (sizeof(struct mkhi_msg_hdr) + \
 			    sizeof(struct mkhi_fwcaps) + \
 			    sizeof(struct mei_os_ver))
@@ -164,7 +135,6 @@ static int mei_osver(struct mei_cl_device *cldev)
 			    sizeof(struct mkhi_fw_ver))
 #define MKHI_FWVER_LEN(__num) (sizeof(struct mkhi_msg_hdr) + \
 			       sizeof(struct mkhi_fw_ver_block) * (__num))
-#define MKHI_RCV_TIMEOUT 500 /* receive timeout in msec */
 static int mei_fwver(struct mei_cl_device *cldev)
 {
 	char buf[MKHI_FWVER_BUF_LEN];
@@ -187,7 +157,7 @@ static int mei_fwver(struct mei_cl_device *cldev)
 
 	ret = 0;
 	bytes_recv = __mei_cl_recv(cldev->cl, buf, sizeof(buf), NULL, 0,
-				   MKHI_RCV_TIMEOUT);
+				   cldev->bus->timeouts.mkhi_recv);
 	if (bytes_recv < 0 || (size_t)bytes_recv < MKHI_FWVER_LEN(1)) {
 		/*
 		 * Should be at least one version block,
@@ -218,6 +188,19 @@ static int mei_fwver(struct mei_cl_device *cldev)
 	return ret;
 }
 
+static int mei_gfx_memory_ready(struct mei_cl_device *cldev)
+{
+	struct mkhi_gfx_mem_ready req = {0};
+	unsigned int mode = MEI_CL_IO_TX_INTERNAL;
+
+	req.hdr.group_id = MKHI_GROUP_ID_GFX;
+	req.hdr.command = MKHI_GFX_MEMORY_READY_CMD_REQ;
+	req.flags = MKHI_GFX_MEM_READY_PXP_ALLOWED;
+
+	dev_dbg(&cldev->dev, "Sending memory ready command\n");
+	return __mei_cl_send(cldev->cl, (u8 *)&req, sizeof(req), 0, mode);
+}
+
 static void mei_mkhi_fix(struct mei_cl_device *cldev)
 {
 	int ret;
@@ -264,6 +247,39 @@ static void mei_gsc_mkhi_ver(struct mei_cl_device *cldev)
 		dev_err(&cldev->dev, "FW version command failed %d\n", ret);
 	mei_cldev_disable(cldev);
 }
+
+static void mei_gsc_mkhi_fix_ver(struct mei_cl_device *cldev)
+{
+	int ret;
+
+	/* No need to enable the client if nothing is needed from it */
+	if (!cldev->bus->fw_f_fw_ver_supported &&
+	    cldev->bus->pxp_mode != MEI_DEV_PXP_INIT)
+		return;
+
+	ret = mei_cldev_enable(cldev);
+	if (ret)
+		return;
+
+	if (cldev->bus->pxp_mode == MEI_DEV_PXP_INIT) {
+		ret = mei_gfx_memory_ready(cldev);
+		if (ret < 0)
+			dev_err(&cldev->dev, "memory ready command failed %d\n", ret);
+		else
+			dev_dbg(&cldev->dev, "memory ready command sent\n");
+		/* we go to reset after that */
+		cldev->bus->pxp_mode = MEI_DEV_PXP_SETUP;
+		goto out;
+	}
+
+	ret = mei_fwver(cldev);
+	if (ret < 0)
+		dev_err(&cldev->dev, "FW version command failed %d\n",
+			ret);
+out:
+	mei_cldev_disable(cldev);
+}
+
 /**
  * mei_wd - wd client on the bus, change protocol version
  *   as the API has changed.
@@ -503,6 +519,26 @@ static void vt_support(struct mei_cl_device *cldev)
 		cldev->do_match = 1;
 }
 
+/**
+ * pxp_is_ready - enable bus client if pxp is ready
+ *
+ * @cldev: me clients device
+ */
+static void pxp_is_ready(struct mei_cl_device *cldev)
+{
+	struct mei_device *bus = cldev->bus;
+
+	switch (bus->pxp_mode) {
+	case MEI_DEV_PXP_READY:
+	case MEI_DEV_PXP_DEFAULT:
+		cldev->do_match = 1;
+	break;
+	default:
+		cldev->do_match = 0;
+	break;
+	}
+}
+
 #define MEI_FIXUP(_uuid, _hook) { _uuid, _hook }
 
 static struct mei_fixup {
@@ -516,10 +552,10 @@ static struct mei_fixup {
 	MEI_FIXUP(MEI_UUID_WD, mei_wd),
 	MEI_FIXUP(MEI_UUID_MKHIF_FIX, mei_mkhi_fix),
 	MEI_FIXUP(MEI_UUID_IGSC_MKHI, mei_gsc_mkhi_ver),
-	MEI_FIXUP(MEI_UUID_IGSC_MKHI_FIX, mei_gsc_mkhi_ver),
+	MEI_FIXUP(MEI_UUID_IGSC_MKHI_FIX, mei_gsc_mkhi_fix_ver),
 	MEI_FIXUP(MEI_UUID_HDCP, whitelist),
 	MEI_FIXUP(MEI_UUID_ANY, vt_support),
-	MEI_FIXUP(MEI_UUID_PAVP, whitelist),
+	MEI_FIXUP(MEI_UUID_PAVP, pxp_is_ready),
 };
 
 /**
diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c
index 31264ab2eb13..0b2fbe1335a7 100644
--- a/drivers/misc/mei/client.c
+++ b/drivers/misc/mei/client.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2003-2020, Intel Corporation. All rights reserved.
+ * Copyright (c) 2003-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -870,7 +870,7 @@ static int mei_cl_send_disconnect(struct mei_cl *cl, struct mei_cl_cb *cb)
 	}
 
 	list_move_tail(&cb->list, &dev->ctrl_rd_list);
-	cl->timer_count = MEI_CONNECT_TIMEOUT;
+	cl->timer_count = dev->timeouts.connect;
 	mei_schedule_stall_timer(dev);
 
 	return 0;
@@ -945,7 +945,7 @@ static int __mei_cl_disconnect(struct mei_cl *cl)
 	wait_event_timeout(cl->wait,
 			   cl->state == MEI_FILE_DISCONNECT_REPLY ||
 			   cl->state == MEI_FILE_DISCONNECTED,
-			   mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT));
+			   dev->timeouts.cl_connect);
 	mutex_lock(&dev->device_lock);
 
 	rets = cl->status;
@@ -1065,7 +1065,7 @@ static int mei_cl_send_connect(struct mei_cl *cl, struct mei_cl_cb *cb)
 	}
 
 	list_move_tail(&cb->list, &dev->ctrl_rd_list);
-	cl->timer_count = MEI_CONNECT_TIMEOUT;
+	cl->timer_count = dev->timeouts.connect;
 	mei_schedule_stall_timer(dev);
 	return 0;
 }
@@ -1164,7 +1164,7 @@ int mei_cl_connect(struct mei_cl *cl, struct mei_me_client *me_cl,
 			 cl->state == MEI_FILE_DISCONNECTED ||
 			 cl->state == MEI_FILE_DISCONNECT_REQUIRED ||
 			 cl->state == MEI_FILE_DISCONNECT_REPLY),
-			mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT));
+			dev->timeouts.cl_connect);
 	mutex_lock(&dev->device_lock);
 
 	if (!mei_cl_is_connected(cl)) {
@@ -1562,7 +1562,7 @@ int mei_cl_notify_request(struct mei_cl *cl,
 			   cl->notify_en == request ||
 			   cl->status ||
 			   !mei_cl_is_connected(cl),
-			   mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT));
+			   dev->timeouts.cl_connect);
 	mutex_lock(&dev->device_lock);
 
 	if (cl->notify_en != request && !cl->status)
@@ -2336,7 +2336,7 @@ int mei_cl_dma_alloc_and_map(struct mei_cl *cl, const struct file *fp,
 	mutex_unlock(&dev->device_lock);
 	wait_event_timeout(cl->wait,
 			   cl->dma_mapped || cl->status,
-			   mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT));
+			   dev->timeouts.cl_connect);
 	mutex_lock(&dev->device_lock);
 
 	if (!cl->dma_mapped && !cl->status)
@@ -2415,7 +2415,7 @@ int mei_cl_dma_unmap(struct mei_cl *cl, const struct file *fp)
 	mutex_unlock(&dev->device_lock);
 	wait_event_timeout(cl->wait,
 			   !cl->dma_mapped || cl->status,
-			   mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT));
+			   dev->timeouts.cl_connect);
 	mutex_lock(&dev->device_lock);
 
 	if (cl->dma_mapped && !cl->status)
diff --git a/drivers/misc/mei/debugfs.c b/drivers/misc/mei/debugfs.c
index 1ce61e9e24fc..3b098d4c8e3d 100644
--- a/drivers/misc/mei/debugfs.c
+++ b/drivers/misc/mei/debugfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2012-2016, Intel Corporation. All rights reserved
+ * Copyright (c) 2012-2022, Intel Corporation. All rights reserved
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -86,6 +86,20 @@ out:
 }
 DEFINE_SHOW_ATTRIBUTE(mei_dbgfs_active);
 
+static const char *mei_dev_pxp_mode_str(enum mei_dev_pxp_mode state)
+{
+#define MEI_PXP_MODE(state) case MEI_DEV_PXP_##state: return #state
+	switch (state) {
+	MEI_PXP_MODE(DEFAULT);
+	MEI_PXP_MODE(INIT);
+	MEI_PXP_MODE(SETUP);
+	MEI_PXP_MODE(READY);
+	default:
+		return "unknown";
+	}
+#undef MEI_PXP_MODE
+}
+
 static int mei_dbgfs_devstate_show(struct seq_file *m, void *unused)
 {
 	struct mei_device *dev = m->private;
@@ -112,6 +126,9 @@ static int mei_dbgfs_devstate_show(struct seq_file *m, void *unused)
 	seq_printf(m, "pg:  %s, %s\n",
 		   mei_pg_is_enabled(dev) ? "ENABLED" : "DISABLED",
 		   mei_pg_state_str(mei_pg_state(dev)));
+
+	seq_printf(m, "pxp: %s\n", mei_dev_pxp_mode_str(dev->pxp_mode));
+
 	return 0;
 }
 DEFINE_SHOW_ATTRIBUTE(mei_dbgfs_devstate);
diff --git a/drivers/misc/mei/gsc-me.c b/drivers/misc/mei/gsc-me.c
index c8145e9b62b6..75765e4df4ed 100644
--- a/drivers/misc/mei/gsc-me.c
+++ b/drivers/misc/mei/gsc-me.c
@@ -13,6 +13,7 @@
 #include <linux/ktime.h>
 #include <linux/delay.h>
 #include <linux/pm_runtime.h>
+#include <linux/kthread.h>
 
 #include "mei_dev.h"
 #include "hw-me.h"
@@ -31,6 +32,17 @@ static int mei_gsc_read_hfs(const struct mei_device *dev, int where, u32 *val)
 	return 0;
 }
 
+static void mei_gsc_set_ext_op_mem(const struct mei_me_hw *hw, struct resource *mem)
+{
+	u32 low = lower_32_bits(mem->start);
+	u32 hi  = upper_32_bits(mem->start);
+	u32 limit = (resource_size(mem) / SZ_4K) | GSC_EXT_OP_MEM_VALID;
+
+	iowrite32(low, hw->mem_addr + H_GSC_EXT_OP_MEM_BASE_ADDR_LO_REG);
+	iowrite32(hi, hw->mem_addr + H_GSC_EXT_OP_MEM_BASE_ADDR_HI_REG);
+	iowrite32(limit, hw->mem_addr + H_GSC_EXT_OP_MEM_LIMIT_REG);
+}
+
 static int mei_gsc_probe(struct auxiliary_device *aux_dev,
 			 const struct auxiliary_device_id *aux_dev_id)
 {
@@ -47,7 +59,7 @@ static int mei_gsc_probe(struct auxiliary_device *aux_dev,
 
 	device = &aux_dev->dev;
 
-	dev = mei_me_dev_init(device, cfg);
+	dev = mei_me_dev_init(device, cfg, adev->slow_firmware);
 	if (!dev) {
 		ret = -ENOMEM;
 		goto err;
@@ -66,13 +78,33 @@ static int mei_gsc_probe(struct auxiliary_device *aux_dev,
 
 	dev_set_drvdata(device, dev);
 
-	ret = devm_request_threaded_irq(device, hw->irq,
-					mei_me_irq_quick_handler,
-					mei_me_irq_thread_handler,
-					IRQF_ONESHOT, KBUILD_MODNAME, dev);
-	if (ret) {
-		dev_err(device, "irq register failed %d\n", ret);
-		goto err;
+	if (adev->ext_op_mem.start) {
+		mei_gsc_set_ext_op_mem(hw, &adev->ext_op_mem);
+		dev->pxp_mode = MEI_DEV_PXP_INIT;
+	}
+
+	/* use polling */
+	if (mei_me_hw_use_polling(hw)) {
+		mei_disable_interrupts(dev);
+		mei_clear_interrupts(dev);
+		init_waitqueue_head(&hw->wait_active);
+		hw->is_active = true; /* start in active mode for initialization */
+		hw->polling_thread = kthread_run(mei_me_polling_thread, dev,
+						 "kmegscirqd/%s", dev_name(device));
+		if (IS_ERR(hw->polling_thread)) {
+			ret = PTR_ERR(hw->polling_thread);
+			dev_err(device, "unable to create kernel thread: %d\n", ret);
+			goto err;
+		}
+	} else {
+		ret = devm_request_threaded_irq(device, hw->irq,
+						mei_me_irq_quick_handler,
+						mei_me_irq_thread_handler,
+						IRQF_ONESHOT, KBUILD_MODNAME, dev);
+		if (ret) {
+			dev_err(device, "irq register failed %d\n", ret);
+			goto err;
+		}
 	}
 
 	pm_runtime_get_noresume(device);
@@ -98,7 +130,8 @@ static int mei_gsc_probe(struct auxiliary_device *aux_dev,
 
 register_err:
 	mei_stop(dev);
-	devm_free_irq(device, hw->irq, dev);
+	if (!mei_me_hw_use_polling(hw))
+		devm_free_irq(device, hw->irq, dev);
 
 err:
 	dev_err(device, "probe failed: %d\n", ret);
@@ -119,12 +152,17 @@ static void mei_gsc_remove(struct auxiliary_device *aux_dev)
 
 	mei_stop(dev);
 
+	hw = to_me_hw(dev);
+	if (mei_me_hw_use_polling(hw))
+		kthread_stop(hw->polling_thread);
+
 	mei_deregister(dev);
 
 	pm_runtime_disable(&aux_dev->dev);
 
 	mei_disable_interrupts(dev);
-	devm_free_irq(&aux_dev->dev, hw->irq, dev);
+	if (!mei_me_hw_use_polling(hw))
+		devm_free_irq(&aux_dev->dev, hw->irq, dev);
 }
 
 static int __maybe_unused mei_gsc_pm_suspend(struct device *device)
@@ -144,11 +182,22 @@ static int __maybe_unused mei_gsc_pm_suspend(struct device *device)
 static int __maybe_unused mei_gsc_pm_resume(struct device *device)
 {
 	struct mei_device *dev = dev_get_drvdata(device);
+	struct auxiliary_device *aux_dev;
+	struct mei_aux_device *adev;
 	int err;
+	struct mei_me_hw *hw;
 
 	if (!dev)
 		return -ENODEV;
 
+	hw = to_me_hw(dev);
+	aux_dev = to_auxiliary_dev(device);
+	adev = auxiliary_dev_to_mei_aux_dev(aux_dev);
+	if (adev->ext_op_mem.start) {
+		mei_gsc_set_ext_op_mem(hw, &adev->ext_op_mem);
+		dev->pxp_mode = MEI_DEV_PXP_INIT;
+	}
+
 	err = mei_restart(dev);
 	if (err)
 		return err;
@@ -185,6 +234,9 @@ static int  __maybe_unused mei_gsc_pm_runtime_suspend(struct device *device)
 	if (mei_write_is_idle(dev)) {
 		hw = to_me_hw(dev);
 		hw->pg_state = MEI_PG_ON;
+
+		if (mei_me_hw_use_polling(hw))
+			hw->is_active = false;
 		ret = 0;
 	} else {
 		ret = -EAGAIN;
@@ -209,6 +261,11 @@ static int __maybe_unused mei_gsc_pm_runtime_resume(struct device *device)
 	hw = to_me_hw(dev);
 	hw->pg_state = MEI_PG_OFF;
 
+	if (mei_me_hw_use_polling(hw)) {
+		hw->is_active = true;
+		wake_up(&hw->wait_active);
+	}
+
 	mutex_unlock(&dev->device_lock);
 
 	irq_ret = mei_me_irq_thread_handler(1, dev);
diff --git a/drivers/misc/mei/hbm.c b/drivers/misc/mei/hbm.c
index cf2b8261da14..de712cbf5d07 100644
--- a/drivers/misc/mei/hbm.c
+++ b/drivers/misc/mei/hbm.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2003-2020, Intel Corporation. All rights reserved.
+ * Copyright (c) 2003-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 #include <linux/export.h>
@@ -232,7 +232,7 @@ int mei_hbm_start_wait(struct mei_device *dev)
 	mutex_unlock(&dev->device_lock);
 	ret = wait_event_timeout(dev->wait_hbm_start,
 			dev->hbm_state != MEI_HBM_STARTING,
-			mei_secs_to_jiffies(MEI_HBM_TIMEOUT));
+			dev->timeouts.hbm);
 	mutex_lock(&dev->device_lock);
 
 	if (ret == 0 && (dev->hbm_state <= MEI_HBM_STARTING)) {
@@ -275,7 +275,7 @@ int mei_hbm_start_req(struct mei_device *dev)
 	}
 
 	dev->hbm_state = MEI_HBM_STARTING;
-	dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT;
+	dev->init_clients_timer = dev->timeouts.client_init;
 	mei_schedule_stall_timer(dev);
 	return 0;
 }
@@ -316,7 +316,7 @@ static int mei_hbm_dma_setup_req(struct mei_device *dev)
 	}
 
 	dev->hbm_state = MEI_HBM_DR_SETUP;
-	dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT;
+	dev->init_clients_timer = dev->timeouts.client_init;
 	mei_schedule_stall_timer(dev);
 	return 0;
 }
@@ -351,7 +351,7 @@ static int mei_hbm_capabilities_req(struct mei_device *dev)
 	}
 
 	dev->hbm_state = MEI_HBM_CAP_SETUP;
-	dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT;
+	dev->init_clients_timer = dev->timeouts.client_init;
 	mei_schedule_stall_timer(dev);
 	return 0;
 }
@@ -385,7 +385,7 @@ static int mei_hbm_enum_clients_req(struct mei_device *dev)
 		return ret;
 	}
 	dev->hbm_state = MEI_HBM_ENUM_CLIENTS;
-	dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT;
+	dev->init_clients_timer = dev->timeouts.client_init;
 	mei_schedule_stall_timer(dev);
 	return 0;
 }
@@ -751,7 +751,7 @@ static int mei_hbm_prop_req(struct mei_device *dev, unsigned long start_idx)
 		return ret;
 	}
 
-	dev->init_clients_timer = MEI_CLIENTS_INIT_TIMEOUT;
+	dev->init_clients_timer = dev->timeouts.client_init;
 	mei_schedule_stall_timer(dev);
 
 	return 0;
diff --git a/drivers/misc/mei/hw-me-regs.h b/drivers/misc/mei/hw-me-regs.h
index 15e8e2b322b1..99966cd3e7d8 100644
--- a/drivers/misc/mei/hw-me-regs.h
+++ b/drivers/misc/mei/hw-me-regs.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (c) 2003-2019, Intel Corporation. All rights reserved.
+ * Copyright (c) 2003-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 #ifndef _MEI_HW_MEI_REGS_H_
@@ -127,6 +127,8 @@
 #  define PCI_CFG_HFS_3_FW_SKU_SPS   0x00000060
 #define PCI_CFG_HFS_4         0x64
 #define PCI_CFG_HFS_5         0x68
+#  define GSC_CFG_HFS_5_BOOT_TYPE_MSK      0x00000003
+#  define GSC_CFG_HFS_5_BOOT_TYPE_PXP               3
 #define PCI_CFG_HFS_6         0x6C
 
 /* MEI registers */
@@ -143,6 +145,11 @@
 /* H_D0I3C - D0I3 Control  */
 #define H_D0I3C    0x800
 
+#define H_GSC_EXT_OP_MEM_BASE_ADDR_LO_REG 0x100
+#define H_GSC_EXT_OP_MEM_BASE_ADDR_HI_REG 0x104
+#define H_GSC_EXT_OP_MEM_LIMIT_REG        0x108
+#define GSC_EXT_OP_MEM_VALID              BIT(31)
+
 /* register bits of H_CSR (Host Control Status register) */
 /* Host Circular Buffer Depth - maximum number of 32-bit entries in CB */
 #define H_CBD             0xFF000000
diff --git a/drivers/misc/mei/hw-me.c b/drivers/misc/mei/hw-me.c
index 3a95fe7d4e33..9e2f781c6ed5 100644
--- a/drivers/misc/mei/hw-me.c
+++ b/drivers/misc/mei/hw-me.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2003-2020, Intel Corporation. All rights reserved.
+ * Copyright (c) 2003-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -10,6 +10,7 @@
 #include <linux/interrupt.h>
 #include <linux/pm_runtime.h>
 #include <linux/sizes.h>
+#include <linux/delay.h>
 
 #include "mei_dev.h"
 #include "hbm.h"
@@ -327,9 +328,12 @@ static void mei_me_intr_clear(struct mei_device *dev)
  */
 static void mei_me_intr_enable(struct mei_device *dev)
 {
-	u32 hcsr = mei_hcsr_read(dev);
+	u32 hcsr;
+
+	if (mei_me_hw_use_polling(to_me_hw(dev)))
+		return;
 
-	hcsr |= H_CSR_IE_MASK;
+	hcsr = mei_hcsr_read(dev) | H_CSR_IE_MASK;
 	mei_hcsr_set(dev, hcsr);
 }
 
@@ -354,6 +358,9 @@ static void mei_me_synchronize_irq(struct mei_device *dev)
 {
 	struct mei_me_hw *hw = to_me_hw(dev);
 
+	if (mei_me_hw_use_polling(hw))
+		return;
+
 	synchronize_irq(hw->irq);
 }
 
@@ -380,7 +387,10 @@ static void mei_me_host_set_ready(struct mei_device *dev)
 {
 	u32 hcsr = mei_hcsr_read(dev);
 
-	hcsr |= H_CSR_IE_MASK | H_IG | H_RDY;
+	if (!mei_me_hw_use_polling(to_me_hw(dev)))
+		hcsr |= H_CSR_IE_MASK;
+
+	hcsr |=  H_IG | H_RDY;
 	mei_hcsr_set(dev, hcsr);
 }
 
@@ -424,6 +434,29 @@ static bool mei_me_hw_is_resetting(struct mei_device *dev)
 }
 
 /**
+ * mei_gsc_pxp_check - check for gsc firmware entering pxp mode
+ *
+ * @dev: the device structure
+ */
+static void mei_gsc_pxp_check(struct mei_device *dev)
+{
+	struct mei_me_hw *hw = to_me_hw(dev);
+	u32 fwsts5 = 0;
+
+	if (dev->pxp_mode == MEI_DEV_PXP_DEFAULT)
+		return;
+
+	hw->read_fws(dev, PCI_CFG_HFS_5, &fwsts5);
+	trace_mei_pci_cfg_read(dev->dev, "PCI_CFG_HFS_5", PCI_CFG_HFS_5, fwsts5);
+	if ((fwsts5 & GSC_CFG_HFS_5_BOOT_TYPE_MSK) == GSC_CFG_HFS_5_BOOT_TYPE_PXP) {
+		dev_dbg(dev->dev, "pxp mode is ready 0x%08x\n", fwsts5);
+		dev->pxp_mode = MEI_DEV_PXP_READY;
+	} else {
+		dev_dbg(dev->dev, "pxp mode is not ready 0x%08x\n", fwsts5);
+	}
+}
+
+/**
  * mei_me_hw_ready_wait - wait until the me(hw) has turned ready
  *  or timeout is reached
  *
@@ -435,13 +468,15 @@ static int mei_me_hw_ready_wait(struct mei_device *dev)
 	mutex_unlock(&dev->device_lock);
 	wait_event_timeout(dev->wait_hw_ready,
 			dev->recvd_hw_ready,
-			mei_secs_to_jiffies(MEI_HW_READY_TIMEOUT));
+			dev->timeouts.hw_ready);
 	mutex_lock(&dev->device_lock);
 	if (!dev->recvd_hw_ready) {
 		dev_err(dev->dev, "wait hw ready failed\n");
 		return -ETIME;
 	}
 
+	mei_gsc_pxp_check(dev);
+
 	mei_me_hw_reset_release(dev);
 	dev->recvd_hw_ready = false;
 	return 0;
@@ -697,7 +732,6 @@ static void mei_me_pg_unset(struct mei_device *dev)
 static int mei_me_pg_legacy_enter_sync(struct mei_device *dev)
 {
 	struct mei_me_hw *hw = to_me_hw(dev);
-	unsigned long timeout = mei_secs_to_jiffies(MEI_PGI_TIMEOUT);
 	int ret;
 
 	dev->pg_event = MEI_PG_EVENT_WAIT;
@@ -708,7 +742,8 @@ static int mei_me_pg_legacy_enter_sync(struct mei_device *dev)
 
 	mutex_unlock(&dev->device_lock);
 	wait_event_timeout(dev->wait_pg,
-		dev->pg_event == MEI_PG_EVENT_RECEIVED, timeout);
+		dev->pg_event == MEI_PG_EVENT_RECEIVED,
+		dev->timeouts.pgi);
 	mutex_lock(&dev->device_lock);
 
 	if (dev->pg_event == MEI_PG_EVENT_RECEIVED) {
@@ -734,7 +769,6 @@ static int mei_me_pg_legacy_enter_sync(struct mei_device *dev)
 static int mei_me_pg_legacy_exit_sync(struct mei_device *dev)
 {
 	struct mei_me_hw *hw = to_me_hw(dev);
-	unsigned long timeout = mei_secs_to_jiffies(MEI_PGI_TIMEOUT);
 	int ret;
 
 	if (dev->pg_event == MEI_PG_EVENT_RECEIVED)
@@ -746,7 +780,8 @@ static int mei_me_pg_legacy_exit_sync(struct mei_device *dev)
 
 	mutex_unlock(&dev->device_lock);
 	wait_event_timeout(dev->wait_pg,
-		dev->pg_event == MEI_PG_EVENT_RECEIVED, timeout);
+		dev->pg_event == MEI_PG_EVENT_RECEIVED,
+		dev->timeouts.pgi);
 	mutex_lock(&dev->device_lock);
 
 reply:
@@ -762,7 +797,8 @@ reply:
 
 	mutex_unlock(&dev->device_lock);
 	wait_event_timeout(dev->wait_pg,
-		dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED, timeout);
+		dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED,
+		dev->timeouts.pgi);
 	mutex_lock(&dev->device_lock);
 
 	if (dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED)
@@ -877,8 +913,6 @@ static u32 mei_me_d0i3_unset(struct mei_device *dev)
 static int mei_me_d0i3_enter_sync(struct mei_device *dev)
 {
 	struct mei_me_hw *hw = to_me_hw(dev);
-	unsigned long d0i3_timeout = mei_secs_to_jiffies(MEI_D0I3_TIMEOUT);
-	unsigned long pgi_timeout = mei_secs_to_jiffies(MEI_PGI_TIMEOUT);
 	int ret;
 	u32 reg;
 
@@ -900,7 +934,8 @@ static int mei_me_d0i3_enter_sync(struct mei_device *dev)
 
 	mutex_unlock(&dev->device_lock);
 	wait_event_timeout(dev->wait_pg,
-		dev->pg_event == MEI_PG_EVENT_RECEIVED, pgi_timeout);
+		dev->pg_event == MEI_PG_EVENT_RECEIVED,
+		dev->timeouts.pgi);
 	mutex_lock(&dev->device_lock);
 
 	if (dev->pg_event != MEI_PG_EVENT_RECEIVED) {
@@ -920,7 +955,8 @@ static int mei_me_d0i3_enter_sync(struct mei_device *dev)
 
 	mutex_unlock(&dev->device_lock);
 	wait_event_timeout(dev->wait_pg,
-		dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED, d0i3_timeout);
+		dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED,
+		dev->timeouts.d0i3);
 	mutex_lock(&dev->device_lock);
 
 	if (dev->pg_event != MEI_PG_EVENT_INTR_RECEIVED) {
@@ -980,7 +1016,6 @@ on:
 static int mei_me_d0i3_exit_sync(struct mei_device *dev)
 {
 	struct mei_me_hw *hw = to_me_hw(dev);
-	unsigned long timeout = mei_secs_to_jiffies(MEI_D0I3_TIMEOUT);
 	int ret;
 	u32 reg;
 
@@ -1003,7 +1038,8 @@ static int mei_me_d0i3_exit_sync(struct mei_device *dev)
 
 	mutex_unlock(&dev->device_lock);
 	wait_event_timeout(dev->wait_pg,
-		dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED, timeout);
+		dev->pg_event == MEI_PG_EVENT_INTR_RECEIVED,
+		dev->timeouts.d0i3);
 	mutex_lock(&dev->device_lock);
 
 	if (dev->pg_event != MEI_PG_EVENT_INTR_RECEIVED) {
@@ -1176,7 +1212,7 @@ static int mei_me_hw_reset(struct mei_device *dev, bool intr_enable)
 
 	hcsr |= H_RST | H_IG | H_CSR_IS_MASK;
 
-	if (!intr_enable)
+	if (!intr_enable || mei_me_hw_use_polling(to_me_hw(dev)))
 		hcsr &= ~H_CSR_IE_MASK;
 
 	dev->recvd_hw_ready = false;
@@ -1259,7 +1295,8 @@ irqreturn_t mei_me_irq_thread_handler(int irq, void *dev_id)
 
 	/* check if ME wants a reset */
 	if (!mei_hw_is_ready(dev) && dev->dev_state != MEI_DEV_RESETTING) {
-		dev_warn(dev->dev, "FW not ready: resetting.\n");
+		dev_warn(dev->dev, "FW not ready: resetting: dev_state = %d pxp = %d\n",
+			 dev->dev_state, dev->pxp_mode);
 		if (dev->dev_state == MEI_DEV_POWERING_DOWN ||
 		    dev->dev_state == MEI_DEV_POWER_DOWN)
 			mei_cl_all_disconnect(dev);
@@ -1331,6 +1368,66 @@ end:
 }
 EXPORT_SYMBOL_GPL(mei_me_irq_thread_handler);
 
+#define MEI_POLLING_TIMEOUT_ACTIVE 100
+#define MEI_POLLING_TIMEOUT_IDLE   500
+
+/**
+ * mei_me_polling_thread - interrupt register polling thread
+ *
+ * The thread monitors the interrupt source register and calls
+ * mei_me_irq_thread_handler() to handle the firmware
+ * input.
+ *
+ * The function polls in MEI_POLLING_TIMEOUT_ACTIVE timeout
+ * in case there was an event, in idle case the polling
+ * time increases yet again by MEI_POLLING_TIMEOUT_ACTIVE
+ * up to MEI_POLLING_TIMEOUT_IDLE.
+ *
+ * @_dev: mei device
+ *
+ * Return: always 0
+ */
+int mei_me_polling_thread(void *_dev)
+{
+	struct mei_device *dev = _dev;
+	irqreturn_t irq_ret;
+	long polling_timeout = MEI_POLLING_TIMEOUT_ACTIVE;
+
+	dev_dbg(dev->dev, "kernel thread is running\n");
+	while (!kthread_should_stop()) {
+		struct mei_me_hw *hw = to_me_hw(dev);
+		u32 hcsr;
+
+		wait_event_timeout(hw->wait_active,
+				   hw->is_active || kthread_should_stop(),
+				   msecs_to_jiffies(MEI_POLLING_TIMEOUT_IDLE));
+
+		if (kthread_should_stop())
+			break;
+
+		hcsr = mei_hcsr_read(dev);
+		if (me_intr_src(hcsr)) {
+			polling_timeout = MEI_POLLING_TIMEOUT_ACTIVE;
+			irq_ret = mei_me_irq_thread_handler(1, dev);
+			if (irq_ret != IRQ_HANDLED)
+				dev_err(dev->dev, "irq_ret %d\n", irq_ret);
+		} else {
+			/*
+			 * Increase timeout by MEI_POLLING_TIMEOUT_ACTIVE
+			 * up to MEI_POLLING_TIMEOUT_IDLE
+			 */
+			polling_timeout = clamp_val(polling_timeout + MEI_POLLING_TIMEOUT_ACTIVE,
+						    MEI_POLLING_TIMEOUT_ACTIVE,
+						    MEI_POLLING_TIMEOUT_IDLE);
+		}
+
+		schedule_timeout_interruptible(msecs_to_jiffies(polling_timeout));
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mei_me_polling_thread);
+
 static const struct mei_hw_ops mei_me_hw_ops = {
 
 	.trc_status = mei_me_trc_status,
@@ -1636,11 +1733,12 @@ EXPORT_SYMBOL_GPL(mei_me_get_cfg);
  *
  * @parent: device associated with physical device (pci/platform)
  * @cfg: per device generation config
+ * @slow_fw: configure longer timeouts as FW is slow
  *
  * Return: The mei_device pointer on success, NULL on failure.
  */
 struct mei_device *mei_me_dev_init(struct device *parent,
-				   const struct mei_cfg *cfg)
+				   const struct mei_cfg *cfg, bool slow_fw)
 {
 	struct mei_device *dev;
 	struct mei_me_hw *hw;
@@ -1655,7 +1753,7 @@ struct mei_device *mei_me_dev_init(struct device *parent,
 	for (i = 0; i < DMA_DSCR_NUM; i++)
 		dev->dr_dscr[i].size = cfg->dma_size[i];
 
-	mei_device_init(dev, parent, &mei_me_hw_ops);
+	mei_device_init(dev, parent, slow_fw, &mei_me_hw_ops);
 	hw->cfg = cfg;
 
 	dev->fw_f_fw_ver_supported = cfg->fw_ver_supported;
diff --git a/drivers/misc/mei/hw-me.h b/drivers/misc/mei/hw-me.h
index a071c645e905..95cf830b7c7b 100644
--- a/drivers/misc/mei/hw-me.h
+++ b/drivers/misc/mei/hw-me.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (c) 2012-2020, Intel Corporation. All rights reserved.
+ * Copyright (c) 2012-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -51,6 +51,9 @@ struct mei_cfg {
  * @d0i3_supported: di03 support
  * @hbuf_depth: depth of hardware host/write buffer in slots
  * @read_fws: read FW status register handler
+ * @polling_thread: interrupt polling thread
+ * @wait_active: the polling thread activity wait queue
+ * @is_active: the device is active
  */
 struct mei_me_hw {
 	const struct mei_cfg *cfg;
@@ -60,10 +63,19 @@ struct mei_me_hw {
 	bool d0i3_supported;
 	u8 hbuf_depth;
 	int (*read_fws)(const struct mei_device *dev, int where, u32 *val);
+	/* polling */
+	struct task_struct *polling_thread;
+	wait_queue_head_t wait_active;
+	bool is_active;
 };
 
 #define to_me_hw(dev) (struct mei_me_hw *)((dev)->hw)
 
+static inline bool mei_me_hw_use_polling(const struct mei_me_hw *hw)
+{
+	return hw->irq < 0;
+}
+
 /**
  * enum mei_cfg_idx - indices to platform specific configurations.
  *
@@ -120,12 +132,13 @@ enum mei_cfg_idx {
 const struct mei_cfg *mei_me_get_cfg(kernel_ulong_t idx);
 
 struct mei_device *mei_me_dev_init(struct device *parent,
-				   const struct mei_cfg *cfg);
+				   const struct mei_cfg *cfg, bool slow_fw);
 
 int mei_me_pg_enter_sync(struct mei_device *dev);
 int mei_me_pg_exit_sync(struct mei_device *dev);
 
 irqreturn_t mei_me_irq_quick_handler(int irq, void *dev_id);
 irqreturn_t mei_me_irq_thread_handler(int irq, void *dev_id);
+int mei_me_polling_thread(void *_dev);
 
 #endif /* _MEI_INTERFACE_H_ */
diff --git a/drivers/misc/mei/hw-txe.c b/drivers/misc/mei/hw-txe.c
index 00652c137cc7..9862c6cd3e32 100644
--- a/drivers/misc/mei/hw-txe.c
+++ b/drivers/misc/mei/hw-txe.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2013-2020, Intel Corporation. All rights reserved.
+ * Copyright (c) 2013-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -1201,7 +1201,7 @@ struct mei_device *mei_txe_dev_init(struct pci_dev *pdev)
 	if (!dev)
 		return NULL;
 
-	mei_device_init(dev, &pdev->dev, &mei_txe_hw_ops);
+	mei_device_init(dev, &pdev->dev, false, &mei_txe_hw_ops);
 
 	hw = to_txe_hw(dev);
 
diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h
index b46077b17114..e7e020dba6b1 100644
--- a/drivers/misc/mei/hw.h
+++ b/drivers/misc/mei/hw.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (c) 2003-2020, Intel Corporation. All rights reserved
+ * Copyright (c) 2003-2022, Intel Corporation. All rights reserved
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -16,11 +16,16 @@
 #define MEI_CONNECT_TIMEOUT         3  /* HPS: at least 2 seconds */
 
 #define MEI_CL_CONNECT_TIMEOUT     15  /* HPS: Client Connect Timeout */
+#define MEI_CL_CONNECT_TIMEOUT_SLOW 30 /* HPS: Client Connect Timeout, slow FW */
 #define MEI_CLIENTS_INIT_TIMEOUT   15  /* HPS: Clients Enumeration Timeout */
 
 #define MEI_PGI_TIMEOUT             1  /* PG Isolation time response 1 sec */
 #define MEI_D0I3_TIMEOUT            5  /* D0i3 set/unset max response time */
 #define MEI_HBM_TIMEOUT             1  /* 1 second */
+#define MEI_HBM_TIMEOUT_SLOW        5  /* 5 second, slow FW */
+
+#define MKHI_RCV_TIMEOUT 500 /* receive timeout in msec */
+#define MKHI_RCV_TIMEOUT_SLOW 10000 /* receive timeout in msec, slow FW */
 
 /*
  * FW page size for DMA allocations
diff --git a/drivers/misc/mei/init.c b/drivers/misc/mei/init.c
index eb052005ca86..bac8852aad51 100644
--- a/drivers/misc/mei/init.c
+++ b/drivers/misc/mei/init.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2012-2019, Intel Corporation. All rights reserved.
+ * Copyright (c) 2012-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -218,16 +218,6 @@ int mei_start(struct mei_device *dev)
 		goto err;
 	}
 
-	if (!mei_host_is_ready(dev)) {
-		dev_err(dev->dev, "host is not ready.\n");
-		goto err;
-	}
-
-	if (!mei_hw_is_ready(dev)) {
-		dev_err(dev->dev, "ME is not ready.\n");
-		goto err;
-	}
-
 	if (!mei_hbm_version_is_supported(dev)) {
 		dev_dbg(dev->dev, "MEI start failed.\n");
 		goto err;
@@ -320,6 +310,8 @@ void mei_stop(struct mei_device *dev)
 
 	mei_clear_interrupts(dev);
 	mei_synchronize_irq(dev);
+	/* to catch HW-initiated reset */
+	mei_cancel_work(dev);
 
 	mutex_lock(&dev->device_lock);
 
@@ -357,14 +349,16 @@ bool mei_write_is_idle(struct mei_device *dev)
 EXPORT_SYMBOL_GPL(mei_write_is_idle);
 
 /**
- * mei_device_init  -- initialize mei_device structure
+ * mei_device_init - initialize mei_device structure
  *
  * @dev: the mei device
  * @device: the device structure
+ * @slow_fw: configure longer timeouts as FW is slow
  * @hw_ops: hw operations
  */
 void mei_device_init(struct mei_device *dev,
 		     struct device *device,
+		     bool slow_fw,
 		     const struct mei_hw_ops *hw_ops)
 {
 	/* setup our list array */
@@ -393,6 +387,8 @@ void mei_device_init(struct mei_device *dev,
 	bitmap_zero(dev->host_clients_map, MEI_CLIENTS_MAX);
 	dev->open_handle_count = 0;
 
+	dev->pxp_mode = MEI_DEV_PXP_DEFAULT;
+
 	/*
 	 * Reserving the first client ID
 	 * 0: Reserved for MEI Bus Message communications
@@ -402,6 +398,21 @@ void mei_device_init(struct mei_device *dev,
 	dev->pg_event = MEI_PG_EVENT_IDLE;
 	dev->ops      = hw_ops;
 	dev->dev      = device;
+
+	dev->timeouts.hw_ready = mei_secs_to_jiffies(MEI_HW_READY_TIMEOUT);
+	dev->timeouts.connect = MEI_CONNECT_TIMEOUT;
+	dev->timeouts.client_init = MEI_CLIENTS_INIT_TIMEOUT;
+	dev->timeouts.pgi = mei_secs_to_jiffies(MEI_PGI_TIMEOUT);
+	dev->timeouts.d0i3 = mei_secs_to_jiffies(MEI_D0I3_TIMEOUT);
+	if (slow_fw) {
+		dev->timeouts.cl_connect = mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT_SLOW);
+		dev->timeouts.hbm = mei_secs_to_jiffies(MEI_HBM_TIMEOUT_SLOW);
+		dev->timeouts.mkhi_recv = msecs_to_jiffies(MKHI_RCV_TIMEOUT_SLOW);
+	} else {
+		dev->timeouts.cl_connect = mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT);
+		dev->timeouts.hbm = mei_secs_to_jiffies(MEI_HBM_TIMEOUT);
+		dev->timeouts.mkhi_recv = msecs_to_jiffies(MKHI_RCV_TIMEOUT);
+	}
 }
 EXPORT_SYMBOL_GPL(mei_device_init);
 
diff --git a/drivers/misc/mei/main.c b/drivers/misc/mei/main.c
index 786f7c8f7f61..930887e7e38d 100644
--- a/drivers/misc/mei/main.c
+++ b/drivers/misc/mei/main.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2003-2020, Intel Corporation. All rights reserved.
+ * Copyright (c) 2003-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -571,7 +571,7 @@ static int mei_ioctl_connect_vtag(struct file *file,
 				    cl->state == MEI_FILE_DISCONNECTED ||
 				    cl->state == MEI_FILE_DISCONNECT_REQUIRED ||
 				    cl->state == MEI_FILE_DISCONNECT_REPLY),
-				   mei_secs_to_jiffies(MEI_CL_CONNECT_TIMEOUT));
+				   dev->timeouts.cl_connect);
 		mutex_lock(&dev->device_lock);
 	}
 
diff --git a/drivers/misc/mei/mei_dev.h b/drivers/misc/mei/mei_dev.h
index 694f866f87ef..6bb3e1ba9ded 100644
--- a/drivers/misc/mei/mei_dev.h
+++ b/drivers/misc/mei/mei_dev.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (c) 2003-2019, Intel Corporation. All rights reserved.
+ * Copyright (c) 2003-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -62,6 +62,21 @@ enum mei_dev_state {
 	MEI_DEV_POWER_UP
 };
 
+/**
+ * enum mei_dev_pxp_mode - MEI PXP mode state
+ *
+ * @MEI_DEV_PXP_DEFAULT: PCH based device, no initailization required
+ * @MEI_DEV_PXP_INIT:    device requires initialization, send setup message to firmware
+ * @MEI_DEV_PXP_SETUP:   device is in setup stage, waiting for firmware repsonse
+ * @MEI_DEV_PXP_READY:   device initialized
+ */
+enum mei_dev_pxp_mode {
+	MEI_DEV_PXP_DEFAULT = 0,
+	MEI_DEV_PXP_INIT    = 1,
+	MEI_DEV_PXP_SETUP   = 2,
+	MEI_DEV_PXP_READY   = 3,
+};
+
 const char *mei_dev_state_str(int state);
 
 enum mei_file_transaction_states {
@@ -415,6 +430,17 @@ struct mei_fw_version {
 
 #define MEI_MAX_FW_VER_BLOCKS 3
 
+struct mei_dev_timeouts {
+	unsigned long hw_ready; /* Timeout on ready message, in jiffies */
+	int connect; /* HPS: at least 2 seconds, in seconds */
+	unsigned long cl_connect; /* HPS: Client Connect Timeout, in jiffies */
+	int client_init; /* HPS: Clients Enumeration Timeout, in seconds */
+	unsigned long pgi; /* PG Isolation time response, in jiffies */
+	unsigned int d0i3; /* D0i3 set/unset max response time, in jiffies */
+	unsigned long hbm; /* HBM operation timeout, in jiffies */
+	unsigned long mkhi_recv; /* receive timeout, in jiffies */
+};
+
 /**
  * struct mei_device -  MEI private device struct
  *
@@ -443,6 +469,7 @@ struct mei_fw_version {
  * @reset_count : number of consecutive resets
  * @dev_state   : device state
  * @hbm_state   : state of host bus message protocol
+ * @pxp_mode    : PXP device mode
  * @init_clients_timer : HBM init handshake timeout
  *
  * @pg_event    : power gating event
@@ -480,6 +507,8 @@ struct mei_fw_version {
  * @allow_fixed_address: allow user space to connect a fixed client
  * @override_fixed_address: force allow fixed address behavior
  *
+ * @timeouts: actual timeout values
+ *
  * @reset_work  : work item for the device reset
  * @bus_rescan_work : work item for the bus rescan
  *
@@ -524,6 +553,7 @@ struct mei_device {
 	unsigned long reset_count;
 	enum mei_dev_state dev_state;
 	enum mei_hbm_state hbm_state;
+	enum mei_dev_pxp_mode pxp_mode;
 	u16 init_clients_timer;
 
 	/*
@@ -568,6 +598,8 @@ struct mei_device {
 	bool allow_fixed_address;
 	bool override_fixed_address;
 
+	struct mei_dev_timeouts timeouts;
+
 	struct work_struct reset_work;
 	struct work_struct bus_rescan_work;
 
@@ -632,6 +664,7 @@ static inline u32 mei_slots2data(int slots)
  */
 void mei_device_init(struct mei_device *dev,
 		     struct device *device,
+		     bool slow_fw,
 		     const struct mei_hw_ops *hw_ops);
 int mei_reset(struct mei_device *dev);
 int mei_start(struct mei_device *dev);
diff --git a/drivers/misc/mei/mkhi.h b/drivers/misc/mei/mkhi.h
new file mode 100644
index 000000000000..1473ea489666
--- /dev/null
+++ b/drivers/misc/mei/mkhi.h
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2003-2022, Intel Corporation. All rights reserved.
+ * Intel Management Engine Interface (Intel MEI) Linux driver
+ */
+
+#ifndef _MEI_MKHI_H_
+#define _MEI_MKHI_H_
+
+#include <linux/types.h>
+
+#define MKHI_FEATURE_PTT 0x10
+
+#define MKHI_FWCAPS_GROUP_ID 0x3
+#define MKHI_FWCAPS_SET_OS_VER_APP_RULE_CMD 6
+#define MKHI_GEN_GROUP_ID 0xFF
+#define MKHI_GEN_GET_FW_VERSION_CMD 0x2
+
+#define MKHI_GROUP_ID_GFX              0x30
+#define MKHI_GFX_RESET_WARN_CMD_REQ    0x0
+#define MKHI_GFX_MEMORY_READY_CMD_REQ  0x1
+
+/* Allow transition to PXP mode without approval */
+#define MKHI_GFX_MEM_READY_PXP_ALLOWED  0x1
+
+struct mkhi_rule_id {
+	__le16 rule_type;
+	u8 feature_id;
+	u8 reserved;
+} __packed;
+
+struct mkhi_fwcaps {
+	struct mkhi_rule_id id;
+	u8 len;
+	u8 data[];
+} __packed;
+
+struct mkhi_msg_hdr {
+	u8  group_id;
+	u8  command;
+	u8  reserved;
+	u8  result;
+} __packed;
+
+struct mkhi_msg {
+	struct mkhi_msg_hdr hdr;
+	u8 data[];
+} __packed;
+
+struct mkhi_gfx_mem_ready {
+	struct mkhi_msg_hdr hdr;
+	u32    flags;
+} __packed;
+
+#endif /* _MEI_MKHI_H_ */
diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c
index 5435604327a7..704cd0caa172 100644
--- a/drivers/misc/mei/pci-me.c
+++ b/drivers/misc/mei/pci-me.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Copyright (c) 2003-2020, Intel Corporation. All rights reserved.
+ * Copyright (c) 2003-2022, Intel Corporation. All rights reserved.
  * Intel Management Engine Interface (Intel MEI) Linux driver
  */
 
@@ -203,7 +203,7 @@ static int mei_me_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	}
 
 	/* allocates and initializes the mei dev structure */
-	dev = mei_me_dev_init(&pdev->dev, cfg);
+	dev = mei_me_dev_init(&pdev->dev, cfg, false);
 	if (!dev) {
 		err = -ENOMEM;
 		goto end;
diff --git a/drivers/misc/tsl2550.c b/drivers/misc/tsl2550.c
index 6d71865c8042..1652fb9b3856 100644
--- a/drivers/misc/tsl2550.c
+++ b/drivers/misc/tsl2550.c
@@ -389,7 +389,7 @@ exit:
 	return err;
 }
 
-static int tsl2550_remove(struct i2c_client *client)
+static void tsl2550_remove(struct i2c_client *client)
 {
 	sysfs_remove_group(&client->dev.kobj, &tsl2550_attr_group);
 
@@ -397,8 +397,6 @@ static int tsl2550_remove(struct i2c_client *client)
 	tsl2550_set_power_state(client, 0);
 
 	kfree(i2c_get_clientdata(client));
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/mmc/core/host.c b/drivers/mmc/core/host.c
index 0fd91f749b3a..b89dca1f15e9 100644
--- a/drivers/mmc/core/host.c
+++ b/drivers/mmc/core/host.c
@@ -565,7 +565,7 @@ struct mmc_host *mmc_alloc_host(int extra, struct device *dev)
 	spin_lock_init(&host->lock);
 	init_waitqueue_head(&host->wq);
 	INIT_DELAYED_WORK(&host->detect, mmc_rescan);
-	INIT_DELAYED_WORK(&host->sdio_irq_work, sdio_irq_work);
+	INIT_WORK(&host->sdio_irq_work, sdio_irq_work);
 	timer_setup(&host->retune_timer, mmc_retune_timer, 0);
 
 	/*
diff --git a/drivers/mmc/core/sd.c b/drivers/mmc/core/sd.c
index 06aa62ce0ed1..3662bf5320ce 100644
--- a/drivers/mmc/core/sd.c
+++ b/drivers/mmc/core/sd.c
@@ -870,7 +870,8 @@ try_again:
 	 * the CCS bit is set as well. We deliberately deviate from the spec in
 	 * regards to this, which allows UHS-I to be supported for SDSC cards.
 	 */
-	if (!mmc_host_is_spi(host) && rocr && (*rocr & SD_ROCR_S18A)) {
+	if (!mmc_host_is_spi(host) && (ocr & SD_OCR_S18R) &&
+	    rocr && (*rocr & SD_ROCR_S18A)) {
 		err = mmc_set_uhs_voltage(host, pocr);
 		if (err == -EAGAIN) {
 			retries--;
diff --git a/drivers/mmc/core/sdio.c b/drivers/mmc/core/sdio.c
index 0b682a31cd3e..f64b9ac76a5c 100644
--- a/drivers/mmc/core/sdio.c
+++ b/drivers/mmc/core/sdio.c
@@ -1043,7 +1043,7 @@ static int mmc_sdio_suspend(struct mmc_host *host)
 
 	/* Prevent processing of SDIO IRQs in suspended state. */
 	mmc_card_set_suspended(host->card);
-	cancel_delayed_work_sync(&host->sdio_irq_work);
+	cancel_work_sync(&host->sdio_irq_work);
 
 	mmc_claim_host(host);
 
@@ -1103,7 +1103,7 @@ static int mmc_sdio_resume(struct mmc_host *host)
 		if (!(host->caps2 & MMC_CAP2_SDIO_IRQ_NOTHREAD))
 			wake_up_process(host->sdio_irq_thread);
 		else if (host->caps & MMC_CAP_SDIO_IRQ)
-			queue_delayed_work(system_wq, &host->sdio_irq_work, 0);
+			schedule_work(&host->sdio_irq_work);
 	}
 
 out:
diff --git a/drivers/mmc/core/sdio_irq.c b/drivers/mmc/core/sdio_irq.c
index 4b1f7c966ec8..2b24bdf38296 100644
--- a/drivers/mmc/core/sdio_irq.c
+++ b/drivers/mmc/core/sdio_irq.c
@@ -124,7 +124,7 @@ static void sdio_run_irqs(struct mmc_host *host)
 void sdio_irq_work(struct work_struct *work)
 {
 	struct mmc_host *host =
-		container_of(work, struct mmc_host, sdio_irq_work.work);
+		container_of(work, struct mmc_host, sdio_irq_work);
 
 	sdio_run_irqs(host);
 }
@@ -132,7 +132,7 @@ void sdio_irq_work(struct work_struct *work)
 void sdio_signal_irq(struct mmc_host *host)
 {
 	host->sdio_irq_pending = true;
-	queue_delayed_work(system_wq, &host->sdio_irq_work, 0);
+	schedule_work(&host->sdio_irq_work);
 }
 EXPORT_SYMBOL_GPL(sdio_signal_irq);
 
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index e63608834411..f324daadaf70 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -157,6 +157,7 @@ config MMC_SDHCI_OF_ARASAN
 
 config MMC_SDHCI_OF_ASPEED
 	tristate "SDHCI OF support for the ASPEED SDHCI controller"
+	depends on ARCH_ASPEED || COMPILE_TEST
 	depends on MMC_SDHCI_PLTFM
 	depends on OF && OF_ADDRESS
 	select MMC_SDHCI_IO_ACCESSORS
diff --git a/drivers/mmc/host/au1xmmc.c b/drivers/mmc/host/au1xmmc.c
index a9a0837153d8..c88b039dc9fb 100644
--- a/drivers/mmc/host/au1xmmc.c
+++ b/drivers/mmc/host/au1xmmc.c
@@ -1097,8 +1097,9 @@ out5:
 	if (host->platdata && host->platdata->cd_setup &&
 	    !(mmc->caps & MMC_CAP_NEEDS_POLL))
 		host->platdata->cd_setup(mmc, 0);
-out_clk:
+
 	clk_disable_unprepare(host->clk);
+out_clk:
 	clk_put(host->clk);
 out_irq:
 	free_irq(host->irq, host);
diff --git a/drivers/mmc/host/jz4740_mmc.c b/drivers/mmc/host/jz4740_mmc.c
index b1d563b2ed1b..dc2db9c185ea 100644
--- a/drivers/mmc/host/jz4740_mmc.c
+++ b/drivers/mmc/host/jz4740_mmc.c
@@ -298,7 +298,7 @@ static int jz4740_mmc_prepare_dma_data(struct jz4740_mmc_host *host,
 {
 	struct dma_chan *chan = jz4740_mmc_get_dma_chan(host, data);
 	enum dma_data_direction dir = mmc_get_dma_dir(data);
-	int sg_count;
+	unsigned int sg_count;
 
 	if (data->host_cookie == COOKIE_PREMAPPED)
 		return data->sg_count;
@@ -308,7 +308,7 @@ static int jz4740_mmc_prepare_dma_data(struct jz4740_mmc_host *host,
 			data->sg_len,
 			dir);
 
-	if (sg_count <= 0) {
+	if (!sg_count) {
 		dev_err(mmc_dev(host->mmc),
 			"Failed to map scatterlist for DMA operation\n");
 		return -EINVAL;
diff --git a/drivers/mmc/host/meson-gx-mmc.c b/drivers/mmc/host/meson-gx-mmc.c
index fc462995cf94..df05e60bed9a 100644
--- a/drivers/mmc/host/meson-gx-mmc.c
+++ b/drivers/mmc/host/meson-gx-mmc.c
@@ -41,14 +41,17 @@
 #define   CLK_V2_TX_DELAY_MASK GENMASK(19, 16)
 #define   CLK_V2_RX_DELAY_MASK GENMASK(23, 20)
 #define   CLK_V2_ALWAYS_ON BIT(24)
+#define   CLK_V2_IRQ_SDIO_SLEEP BIT(25)
 
 #define   CLK_V3_TX_DELAY_MASK GENMASK(21, 16)
 #define   CLK_V3_RX_DELAY_MASK GENMASK(27, 22)
 #define   CLK_V3_ALWAYS_ON BIT(28)
+#define   CLK_V3_IRQ_SDIO_SLEEP BIT(29)
 
 #define   CLK_TX_DELAY_MASK(h)		(h->data->tx_delay_mask)
 #define   CLK_RX_DELAY_MASK(h)		(h->data->rx_delay_mask)
 #define   CLK_ALWAYS_ON(h)		(h->data->always_on)
+#define   CLK_IRQ_SDIO_SLEEP(h)		(h->data->irq_sdio_sleep)
 
 #define SD_EMMC_DELAY 0x4
 #define SD_EMMC_ADJUST 0x8
@@ -101,8 +104,7 @@
 #define   IRQ_RESP_STATUS BIT(14)
 #define   IRQ_SDIO BIT(15)
 #define   IRQ_EN_MASK \
-	(IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN | IRQ_RESP_STATUS |\
-	 IRQ_SDIO)
+	(IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN)
 
 #define SD_EMMC_CMD_CFG 0x50
 #define SD_EMMC_CMD_ARG 0x54
@@ -136,6 +138,7 @@ struct meson_mmc_data {
 	unsigned int rx_delay_mask;
 	unsigned int always_on;
 	unsigned int adjust;
+	unsigned int irq_sdio_sleep;
 };
 
 struct sd_emmc_desc {
@@ -175,6 +178,7 @@ struct meson_host {
 	bool vqmmc_enabled;
 	bool needs_pre_post_req;
 
+	spinlock_t lock;
 };
 
 #define CMD_CFG_LENGTH_MASK GENMASK(8, 0)
@@ -431,6 +435,7 @@ static int meson_mmc_clk_init(struct meson_host *host)
 	clk_reg |= FIELD_PREP(CLK_CORE_PHASE_MASK, CLK_PHASE_180);
 	clk_reg |= FIELD_PREP(CLK_TX_PHASE_MASK, CLK_PHASE_0);
 	clk_reg |= FIELD_PREP(CLK_RX_PHASE_MASK, CLK_PHASE_0);
+	clk_reg |= CLK_IRQ_SDIO_SLEEP(host);
 	writel(clk_reg, host->regs + SD_EMMC_CLOCK);
 
 	/* get the mux parents */
@@ -929,33 +934,54 @@ static void meson_mmc_read_resp(struct mmc_host *mmc, struct mmc_command *cmd)
 	}
 }
 
+static void __meson_mmc_enable_sdio_irq(struct mmc_host *mmc, int enable)
+{
+	struct meson_host *host = mmc_priv(mmc);
+	u32 reg_irqen = IRQ_EN_MASK;
+
+	if (enable)
+		reg_irqen |= IRQ_SDIO;
+	writel(reg_irqen, host->regs + SD_EMMC_IRQ_EN);
+}
+
 static irqreturn_t meson_mmc_irq(int irq, void *dev_id)
 {
 	struct meson_host *host = dev_id;
 	struct mmc_command *cmd;
-	struct mmc_data *data;
-	u32 irq_en, status, raw_status;
+	u32 status, raw_status;
 	irqreturn_t ret = IRQ_NONE;
 
-	irq_en = readl(host->regs + SD_EMMC_IRQ_EN);
 	raw_status = readl(host->regs + SD_EMMC_STATUS);
-	status = raw_status & irq_en;
+	status = raw_status & (IRQ_EN_MASK | IRQ_SDIO);
 
 	if (!status) {
 		dev_dbg(host->dev,
-			"Unexpected IRQ! irq_en 0x%08x - status 0x%08x\n",
-			 irq_en, raw_status);
+			"Unexpected IRQ! irq_en 0x%08lx - status 0x%08x\n",
+			 IRQ_EN_MASK | IRQ_SDIO, raw_status);
 		return IRQ_NONE;
 	}
 
-	if (WARN_ON(!host) || WARN_ON(!host->cmd))
+	if (WARN_ON(!host))
 		return IRQ_NONE;
 
 	/* ack all raised interrupts */
 	writel(status, host->regs + SD_EMMC_STATUS);
 
 	cmd = host->cmd;
-	data = cmd->data;
+
+	if (status & IRQ_SDIO) {
+		spin_lock(&host->lock);
+		__meson_mmc_enable_sdio_irq(host->mmc, 0);
+		sdio_signal_irq(host->mmc);
+		spin_unlock(&host->lock);
+		status &= ~IRQ_SDIO;
+		if (!status)
+			return IRQ_HANDLED;
+	}
+
+	if (WARN_ON(!cmd))
+		return IRQ_NONE;
+
 	cmd->error = 0;
 	if (status & IRQ_CRC_ERR) {
 		dev_dbg(host->dev, "CRC Error - status 0x%08x\n", status);
@@ -973,12 +999,9 @@ static irqreturn_t meson_mmc_irq(int irq, void *dev_id)
 
 	meson_mmc_read_resp(host->mmc, cmd);
 
-	if (status & IRQ_SDIO) {
-		dev_dbg(host->dev, "IRQ: SDIO TODO.\n");
-		ret = IRQ_HANDLED;
-	}
-
 	if (status & (IRQ_END_OF_CHAIN | IRQ_RESP_STATUS)) {
+		struct mmc_data *data = cmd->data;
+
 		if (data && !cmd->error)
 			data->bytes_xfered = data->blksz * data->blocks;
 		if (meson_mmc_bounce_buf_read(data) ||
@@ -1121,6 +1144,21 @@ static int meson_mmc_voltage_switch(struct mmc_host *mmc, struct mmc_ios *ios)
 	return -EINVAL;
 }
 
+static void meson_mmc_enable_sdio_irq(struct mmc_host *mmc, int enable)
+{
+	struct meson_host *host = mmc_priv(mmc);
+	unsigned long flags;
+
+	spin_lock_irqsave(&host->lock, flags);
+	__meson_mmc_enable_sdio_irq(mmc, enable);
+	spin_unlock_irqrestore(&host->lock, flags);
+}
+
+static void meson_mmc_ack_sdio_irq(struct mmc_host *mmc)
+{
+	meson_mmc_enable_sdio_irq(mmc, 1);
+}
+
 static const struct mmc_host_ops meson_mmc_ops = {
 	.request	= meson_mmc_request,
 	.set_ios	= meson_mmc_set_ios,
@@ -1130,6 +1168,8 @@ static const struct mmc_host_ops meson_mmc_ops = {
 	.execute_tuning = meson_mmc_resampling_tuning,
 	.card_busy	= meson_mmc_card_busy,
 	.start_signal_voltage_switch = meson_mmc_voltage_switch,
+	.enable_sdio_irq = meson_mmc_enable_sdio_irq,
+	.ack_sdio_irq	= meson_mmc_ack_sdio_irq,
 };
 
 static int meson_mmc_probe(struct platform_device *pdev)
@@ -1226,10 +1266,8 @@ static int meson_mmc_probe(struct platform_device *pdev)
 
 	/* clear, ack and enable interrupts */
 	writel(0, host->regs + SD_EMMC_IRQ_EN);
-	writel(IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN,
-	       host->regs + SD_EMMC_STATUS);
-	writel(IRQ_CRC_ERR | IRQ_TIMEOUTS | IRQ_END_OF_CHAIN,
-	       host->regs + SD_EMMC_IRQ_EN);
+	writel(IRQ_EN_MASK, host->regs + SD_EMMC_STATUS);
+	writel(IRQ_EN_MASK, host->regs + SD_EMMC_IRQ_EN);
 
 	ret = request_threaded_irq(host->irq, meson_mmc_irq,
 				   meson_mmc_irq_thread, IRQF_ONESHOT,
@@ -1237,7 +1275,13 @@ static int meson_mmc_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_init_clk;
 
+	spin_lock_init(&host->lock);
+
 	mmc->caps |= MMC_CAP_CMD23;
+
+	if (mmc->caps & MMC_CAP_SDIO_IRQ)
+		mmc->caps2 |= MMC_CAP2_SDIO_IRQ_NOTHREAD;
+
 	if (host->dram_access_quirk) {
 		/* Limit segments to 1 due to low available sram memory */
 		mmc->max_segs = 1;
@@ -1328,6 +1372,7 @@ static const struct meson_mmc_data meson_gx_data = {
 	.rx_delay_mask	= CLK_V2_RX_DELAY_MASK,
 	.always_on	= CLK_V2_ALWAYS_ON,
 	.adjust		= SD_EMMC_ADJUST,
+	.irq_sdio_sleep	= CLK_V2_IRQ_SDIO_SLEEP,
 };
 
 static const struct meson_mmc_data meson_axg_data = {
@@ -1335,6 +1380,7 @@ static const struct meson_mmc_data meson_axg_data = {
 	.rx_delay_mask	= CLK_V3_RX_DELAY_MASK,
 	.always_on	= CLK_V3_ALWAYS_ON,
 	.adjust		= SD_EMMC_V3_ADJUST,
+	.irq_sdio_sleep	= CLK_V3_IRQ_SDIO_SLEEP,
 };
 
 static const struct of_device_id meson_mmc_of_match[] = {
diff --git a/drivers/mmc/host/meson-mx-sdhc-mmc.c b/drivers/mmc/host/meson-mx-sdhc-mmc.c
index e92e63cb5641..da85c2f2acb8 100644
--- a/drivers/mmc/host/meson-mx-sdhc-mmc.c
+++ b/drivers/mmc/host/meson-mx-sdhc-mmc.c
@@ -381,14 +381,14 @@ static void meson_mx_sdhc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 static int meson_mx_sdhc_map_dma(struct mmc_host *mmc, struct mmc_request *mrq)
 {
 	struct mmc_data *data = mrq->data;
-	int dma_len;
+	unsigned int dma_len;
 
 	if (!data)
 		return 0;
 
 	dma_len = dma_map_sg(mmc_dev(mmc), data->sg, data->sg_len,
 			     mmc_get_dma_dir(data));
-	if (dma_len <= 0) {
+	if (!dma_len) {
 		dev_err(mmc_dev(mmc), "dma_map_sg failed\n");
 		return -ENOMEM;
 	}
diff --git a/drivers/mmc/host/mmc_hsq.c b/drivers/mmc/host/mmc_hsq.c
index a5e05ed0fda3..9d35453e7371 100644
--- a/drivers/mmc/host/mmc_hsq.c
+++ b/drivers/mmc/host/mmc_hsq.c
@@ -34,7 +34,7 @@ static void mmc_hsq_pump_requests(struct mmc_hsq *hsq)
 	spin_lock_irqsave(&hsq->lock, flags);
 
 	/* Make sure we are not already running a request now */
-	if (hsq->mrq) {
+	if (hsq->mrq || hsq->recovery_halt) {
 		spin_unlock_irqrestore(&hsq->lock, flags);
 		return;
 	}
diff --git a/drivers/mmc/host/moxart-mmc.c b/drivers/mmc/host/moxart-mmc.c
index b6eb75f4bbfc..dfc3ffd5b1f8 100644
--- a/drivers/mmc/host/moxart-mmc.c
+++ b/drivers/mmc/host/moxart-mmc.c
@@ -111,8 +111,8 @@
 #define CLK_DIV_MASK		0x7f
 
 /* REG_BUS_WIDTH */
-#define BUS_WIDTH_8		BIT(2)
-#define BUS_WIDTH_4		BIT(1)
+#define BUS_WIDTH_4_SUPPORT	BIT(3)
+#define BUS_WIDTH_4		BIT(2)
 #define BUS_WIDTH_1		BIT(0)
 
 #define MMC_VDD_360		23
@@ -524,9 +524,6 @@ static void moxart_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 	case MMC_BUS_WIDTH_4:
 		writel(BUS_WIDTH_4, host->base + REG_BUS_WIDTH);
 		break;
-	case MMC_BUS_WIDTH_8:
-		writel(BUS_WIDTH_8, host->base + REG_BUS_WIDTH);
-		break;
 	default:
 		writel(BUS_WIDTH_1, host->base + REG_BUS_WIDTH);
 		break;
@@ -651,16 +648,8 @@ static int moxart_probe(struct platform_device *pdev)
 		dmaengine_slave_config(host->dma_chan_rx, &cfg);
 	}
 
-	switch ((readl(host->base + REG_BUS_WIDTH) >> 3) & 3) {
-	case 1:
+	if (readl(host->base + REG_BUS_WIDTH) & BUS_WIDTH_4_SUPPORT)
 		mmc->caps |= MMC_CAP_4_BIT_DATA;
-		break;
-	case 2:
-		mmc->caps |= MMC_CAP_4_BIT_DATA | MMC_CAP_8_BIT_DATA;
-		break;
-	default:
-		break;
-	}
 
 	writel(0, host->base + REG_INTERRUPT_MASK);
 
diff --git a/drivers/mmc/host/mtk-sd.c b/drivers/mmc/host/mtk-sd.c
index 69d78604d1fc..df941438aef5 100644
--- a/drivers/mmc/host/mtk-sd.c
+++ b/drivers/mmc/host/mtk-sd.c
@@ -474,33 +474,20 @@ struct msdc_host {
 	struct cqhci_host *cq_host;
 };
 
-static const struct mtk_mmc_compatible mt8135_compat = {
-	.clk_div_bits = 8,
+static const struct mtk_mmc_compatible mt2701_compat = {
+	.clk_div_bits = 12,
 	.recheck_sdio_irq = true,
 	.hs400_tune = false,
-	.pad_tune_reg = MSDC_PAD_TUNE,
-	.async_fifo = false,
-	.data_tune = false,
-	.busy_check = false,
-	.stop_clk_fix = false,
-	.enhance_rx = false,
-	.support_64g = false,
-};
-
-static const struct mtk_mmc_compatible mt8173_compat = {
-	.clk_div_bits = 8,
-	.recheck_sdio_irq = true,
-	.hs400_tune = true,
-	.pad_tune_reg = MSDC_PAD_TUNE,
-	.async_fifo = false,
-	.data_tune = false,
+	.pad_tune_reg = MSDC_PAD_TUNE0,
+	.async_fifo = true,
+	.data_tune = true,
 	.busy_check = false,
 	.stop_clk_fix = false,
 	.enhance_rx = false,
 	.support_64g = false,
 };
 
-static const struct mtk_mmc_compatible mt8183_compat = {
+static const struct mtk_mmc_compatible mt2712_compat = {
 	.clk_div_bits = 12,
 	.recheck_sdio_irq = false,
 	.hs400_tune = false,
@@ -513,30 +500,43 @@ static const struct mtk_mmc_compatible mt8183_compat = {
 	.support_64g = true,
 };
 
-static const struct mtk_mmc_compatible mt2701_compat = {
+static const struct mtk_mmc_compatible mt6779_compat = {
 	.clk_div_bits = 12,
-	.recheck_sdio_irq = true,
+	.recheck_sdio_irq = false,
 	.hs400_tune = false,
 	.pad_tune_reg = MSDC_PAD_TUNE0,
 	.async_fifo = true,
 	.data_tune = true,
+	.busy_check = true,
+	.stop_clk_fix = true,
+	.enhance_rx = true,
+	.support_64g = true,
+};
+
+static const struct mtk_mmc_compatible mt6795_compat = {
+	.clk_div_bits = 8,
+	.recheck_sdio_irq = false,
+	.hs400_tune = true,
+	.pad_tune_reg = MSDC_PAD_TUNE,
+	.async_fifo = false,
+	.data_tune = false,
 	.busy_check = false,
 	.stop_clk_fix = false,
 	.enhance_rx = false,
 	.support_64g = false,
 };
 
-static const struct mtk_mmc_compatible mt2712_compat = {
-	.clk_div_bits = 12,
-	.recheck_sdio_irq = false,
+static const struct mtk_mmc_compatible mt7620_compat = {
+	.clk_div_bits = 8,
+	.recheck_sdio_irq = true,
 	.hs400_tune = false,
-	.pad_tune_reg = MSDC_PAD_TUNE0,
-	.async_fifo = true,
-	.data_tune = true,
-	.busy_check = true,
-	.stop_clk_fix = true,
-	.enhance_rx = true,
-	.support_64g = true,
+	.pad_tune_reg = MSDC_PAD_TUNE,
+	.async_fifo = false,
+	.data_tune = false,
+	.busy_check = false,
+	.stop_clk_fix = false,
+	.enhance_rx = false,
+	.use_internal_cd = true,
 };
 
 static const struct mtk_mmc_compatible mt7622_compat = {
@@ -552,31 +552,33 @@ static const struct mtk_mmc_compatible mt7622_compat = {
 	.support_64g = false,
 };
 
-static const struct mtk_mmc_compatible mt8516_compat = {
-	.clk_div_bits = 12,
+static const struct mtk_mmc_compatible mt8135_compat = {
+	.clk_div_bits = 8,
 	.recheck_sdio_irq = true,
 	.hs400_tune = false,
-	.pad_tune_reg = MSDC_PAD_TUNE0,
-	.async_fifo = true,
-	.data_tune = true,
-	.busy_check = true,
-	.stop_clk_fix = true,
+	.pad_tune_reg = MSDC_PAD_TUNE,
+	.async_fifo = false,
+	.data_tune = false,
+	.busy_check = false,
+	.stop_clk_fix = false,
+	.enhance_rx = false,
+	.support_64g = false,
 };
 
-static const struct mtk_mmc_compatible mt7620_compat = {
+static const struct mtk_mmc_compatible mt8173_compat = {
 	.clk_div_bits = 8,
 	.recheck_sdio_irq = true,
-	.hs400_tune = false,
+	.hs400_tune = true,
 	.pad_tune_reg = MSDC_PAD_TUNE,
 	.async_fifo = false,
 	.data_tune = false,
 	.busy_check = false,
 	.stop_clk_fix = false,
 	.enhance_rx = false,
-	.use_internal_cd = true,
+	.support_64g = false,
 };
 
-static const struct mtk_mmc_compatible mt6779_compat = {
+static const struct mtk_mmc_compatible mt8183_compat = {
 	.clk_div_bits = 12,
 	.recheck_sdio_irq = false,
 	.hs400_tune = false,
@@ -589,16 +591,29 @@ static const struct mtk_mmc_compatible mt6779_compat = {
 	.support_64g = true,
 };
 
+static const struct mtk_mmc_compatible mt8516_compat = {
+	.clk_div_bits = 12,
+	.recheck_sdio_irq = true,
+	.hs400_tune = false,
+	.pad_tune_reg = MSDC_PAD_TUNE0,
+	.async_fifo = true,
+	.data_tune = true,
+	.busy_check = true,
+	.stop_clk_fix = true,
+};
+
 static const struct of_device_id msdc_of_ids[] = {
-	{ .compatible = "mediatek,mt8135-mmc", .data = &mt8135_compat},
-	{ .compatible = "mediatek,mt8173-mmc", .data = &mt8173_compat},
-	{ .compatible = "mediatek,mt8183-mmc", .data = &mt8183_compat},
 	{ .compatible = "mediatek,mt2701-mmc", .data = &mt2701_compat},
 	{ .compatible = "mediatek,mt2712-mmc", .data = &mt2712_compat},
+	{ .compatible = "mediatek,mt6779-mmc", .data = &mt6779_compat},
+	{ .compatible = "mediatek,mt6795-mmc", .data = &mt6795_compat},
+	{ .compatible = "mediatek,mt7620-mmc", .data = &mt7620_compat},
 	{ .compatible = "mediatek,mt7622-mmc", .data = &mt7622_compat},
+	{ .compatible = "mediatek,mt8135-mmc", .data = &mt8135_compat},
+	{ .compatible = "mediatek,mt8173-mmc", .data = &mt8173_compat},
+	{ .compatible = "mediatek,mt8183-mmc", .data = &mt8183_compat},
 	{ .compatible = "mediatek,mt8516-mmc", .data = &mt8516_compat},
-	{ .compatible = "mediatek,mt7620-mmc", .data = &mt7620_compat},
-	{ .compatible = "mediatek,mt6779-mmc", .data = &mt6779_compat},
+
 	{}
 };
 MODULE_DEVICE_TABLE(of, msdc_of_ids);
diff --git a/drivers/mmc/host/rtsx_usb_sdmmc.c b/drivers/mmc/host/rtsx_usb_sdmmc.c
index 5fe4528e296e..5798aee06653 100644
--- a/drivers/mmc/host/rtsx_usb_sdmmc.c
+++ b/drivers/mmc/host/rtsx_usb_sdmmc.c
@@ -1042,7 +1042,6 @@ static int sd_set_timing(struct rtsx_usb_sdmmc *host,
 		unsigned char timing, bool *ddr_mode)
 {
 	struct rtsx_ucr *ucr = host->ucr;
-	int err;
 
 	*ddr_mode = false;
 
@@ -1097,9 +1096,7 @@ static int sd_set_timing(struct rtsx_usb_sdmmc *host,
 		break;
 	}
 
-	err = rtsx_usb_send_cmd(ucr, MODE_C, 100);
-
-	return err;
+	return rtsx_usb_send_cmd(ucr, MODE_C, 100);
 }
 
 static void sdmmc_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
index dc2991422a87..3a091a387ecb 100644
--- a/drivers/mmc/host/sdhci-msm.c
+++ b/drivers/mmc/host/sdhci-msm.c
@@ -2441,6 +2441,7 @@ static const struct of_device_id sdhci_msm_dt_match[] = {
 	 */
 	{.compatible = "qcom,sdhci-msm-v4", .data = &sdhci_msm_mci_var},
 	{.compatible = "qcom,sdhci-msm-v5", .data = &sdhci_msm_v5_var},
+	{.compatible = "qcom,sdm670-sdhci", .data = &sdm845_sdhci_var},
 	{.compatible = "qcom,sdm845-sdhci", .data = &sdm845_sdhci_var},
 	{.compatible = "qcom,sc7180-sdhci", .data = &sdm845_sdhci_var},
 	{},
diff --git a/drivers/mmc/host/sdhci-pci-core.c b/drivers/mmc/host/sdhci-pci-core.c
index 622b7de96c7f..169b84761041 100644
--- a/drivers/mmc/host/sdhci-pci-core.c
+++ b/drivers/mmc/host/sdhci-pci-core.c
@@ -297,6 +297,27 @@ static const struct sdhci_pci_fixes sdhci_ricoh_mmc = {
 			  SDHCI_QUIRK_MISSING_CAPS
 };
 
+static void ene_714_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+
+	sdhci_set_ios(mmc, ios);
+
+	/*
+	 * Some (ENE) controllers misbehave on some ios operations,
+	 * signalling timeout and CRC errors even on CMD0. Resetting
+	 * it on each ios seems to solve the problem.
+	 */
+	if (!(host->flags & SDHCI_DEVICE_DEAD))
+		sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+}
+
+static int ene_714_probe_slot(struct sdhci_pci_slot *slot)
+{
+	slot->host->mmc_host_ops.set_ios = ene_714_set_ios;
+	return 0;
+}
+
 static const struct sdhci_pci_fixes sdhci_ene_712 = {
 	.quirks		= SDHCI_QUIRK_SINGLE_POWER_WRITE |
 			  SDHCI_QUIRK_BROKEN_DMA,
@@ -304,8 +325,8 @@ static const struct sdhci_pci_fixes sdhci_ene_712 = {
 
 static const struct sdhci_pci_fixes sdhci_ene_714 = {
 	.quirks		= SDHCI_QUIRK_SINGLE_POWER_WRITE |
-			  SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS |
 			  SDHCI_QUIRK_BROKEN_DMA,
+	.probe_slot	= ene_714_probe_slot,
 };
 
 static const struct sdhci_pci_fixes sdhci_cafe = {
diff --git a/drivers/mmc/host/sdhci-pci-o2micro.c b/drivers/mmc/host/sdhci-pci-o2micro.c
index 0d4d343dbb77..ad457cd9cbaa 100644
--- a/drivers/mmc/host/sdhci-pci-o2micro.c
+++ b/drivers/mmc/host/sdhci-pci-o2micro.c
@@ -317,11 +317,12 @@ static int sdhci_o2_execute_tuning(struct mmc_host *mmc, u32 opcode)
 	u32 reg_val;
 
 	/*
-	 * This handler only implements the eMMC tuning that is specific to
+	 * This handler implements the hardware tuning that is specific to
 	 * this controller.  Fall back to the standard method for other TIMING.
 	 */
 	if ((host->timing != MMC_TIMING_MMC_HS200) &&
-		(host->timing != MMC_TIMING_UHS_SDR104))
+		(host->timing != MMC_TIMING_UHS_SDR104) &&
+		(host->timing != MMC_TIMING_UHS_SDR50))
 		return sdhci_execute_tuning(mmc, opcode);
 
 	if (WARN_ON((opcode != MMC_SEND_TUNING_BLOCK_HS200) &&
@@ -631,6 +632,8 @@ static int sdhci_pci_o2_probe_slot(struct sdhci_pci_slot *slot)
 		if (reg & 0x1)
 			host->quirks |= SDHCI_QUIRK_MULTIBLOCK_READ_ACMD12;
 
+		host->quirks2 |= SDHCI_QUIRK2_BROKEN_DDR50;
+
 		sdhci_pci_o2_enable_msi(chip, host);
 
 		if (chip->pdev->device == PCI_DEVICE_ID_O2_SEABIRD0) {
diff --git a/drivers/mmc/host/sdhci-sprd.c b/drivers/mmc/host/sdhci-sprd.c
index f33e9349e4e6..46c55ab4884c 100644
--- a/drivers/mmc/host/sdhci-sprd.c
+++ b/drivers/mmc/host/sdhci-sprd.c
@@ -205,14 +205,14 @@ static inline u32 sdhci_sprd_calc_div(u32 base_clk, u32 clk)
 	if ((base_clk / div) > (clk * 2))
 		div++;
 
-	if (div > SDHCI_SPRD_CLK_MAX_DIV)
-		div = SDHCI_SPRD_CLK_MAX_DIV;
-
 	if (div % 2)
 		div = (div + 1) / 2;
 	else
 		div = div / 2;
 
+	if (div > SDHCI_SPRD_CLK_MAX_DIV)
+		div = SDHCI_SPRD_CLK_MAX_DIV;
+
 	return div;
 }
 
diff --git a/drivers/mmc/host/sdhci.c b/drivers/mmc/host/sdhci.c
index 7689ffec5ad1..fef03de85b99 100644
--- a/drivers/mmc/host/sdhci.c
+++ b/drivers/mmc/host/sdhci.c
@@ -233,28 +233,62 @@ void sdhci_reset(struct sdhci_host *host, u8 mask)
 }
 EXPORT_SYMBOL_GPL(sdhci_reset);
 
-static void sdhci_do_reset(struct sdhci_host *host, u8 mask)
+static bool sdhci_do_reset(struct sdhci_host *host, u8 mask)
 {
 	if (host->quirks & SDHCI_QUIRK_NO_CARD_NO_RESET) {
 		struct mmc_host *mmc = host->mmc;
 
 		if (!mmc->ops->get_cd(mmc))
-			return;
+			return false;
 	}
 
 	host->ops->reset(host, mask);
 
-	if (mask & SDHCI_RESET_ALL) {
+	return true;
+}
+
+static void sdhci_reset_for_all(struct sdhci_host *host)
+{
+	if (sdhci_do_reset(host, SDHCI_RESET_ALL)) {
 		if (host->flags & (SDHCI_USE_SDMA | SDHCI_USE_ADMA)) {
 			if (host->ops->enable_dma)
 				host->ops->enable_dma(host);
 		}
-
 		/* Resetting the controller clears many */
 		host->preset_enabled = false;
 	}
 }
 
+enum sdhci_reset_reason {
+	SDHCI_RESET_FOR_INIT,
+	SDHCI_RESET_FOR_REQUEST_ERROR,
+	SDHCI_RESET_FOR_REQUEST_ERROR_DATA_ONLY,
+	SDHCI_RESET_FOR_TUNING_ABORT,
+	SDHCI_RESET_FOR_CARD_REMOVED,
+	SDHCI_RESET_FOR_CQE_RECOVERY,
+};
+
+static void sdhci_reset_for_reason(struct sdhci_host *host, enum sdhci_reset_reason reason)
+{
+	switch (reason) {
+	case SDHCI_RESET_FOR_INIT:
+		sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+		break;
+	case SDHCI_RESET_FOR_REQUEST_ERROR:
+	case SDHCI_RESET_FOR_TUNING_ABORT:
+	case SDHCI_RESET_FOR_CARD_REMOVED:
+	case SDHCI_RESET_FOR_CQE_RECOVERY:
+		sdhci_do_reset(host, SDHCI_RESET_CMD);
+		sdhci_do_reset(host, SDHCI_RESET_DATA);
+		break;
+	case SDHCI_RESET_FOR_REQUEST_ERROR_DATA_ONLY:
+		sdhci_do_reset(host, SDHCI_RESET_DATA);
+		break;
+	}
+}
+
+#define sdhci_reset_for(h, r) sdhci_reset_for_reason((h), SDHCI_RESET_FOR_##r)
+
 static void sdhci_set_default_irqs(struct sdhci_host *host)
 {
 	host->ier = SDHCI_INT_BUS_POWER | SDHCI_INT_DATA_END_BIT |
@@ -323,9 +357,9 @@ static void sdhci_init(struct sdhci_host *host, int soft)
 	unsigned long flags;
 
 	if (soft)
-		sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
+		sdhci_reset_for(host, INIT);
 	else
-		sdhci_do_reset(host, SDHCI_RESET_ALL);
+		sdhci_reset_for_all(host);
 
 	if (host->v4_mode)
 		sdhci_do_enable_v4_mode(host);
@@ -1538,8 +1572,9 @@ static void __sdhci_finish_data(struct sdhci_host *host, bool sw_data_timeout)
 	 */
 	if (data->error) {
 		if (!host->cmd || host->cmd == data_cmd)
-			sdhci_do_reset(host, SDHCI_RESET_CMD);
-		sdhci_do_reset(host, SDHCI_RESET_DATA);
+			sdhci_reset_for(host, REQUEST_ERROR);
+		else
+			sdhci_reset_for(host, REQUEST_ERROR_DATA_ONLY);
 	}
 
 	if ((host->flags & (SDHCI_REQ_USE_DMA | SDHCI_USE_ADMA)) ==
@@ -2403,14 +2438,6 @@ void sdhci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
 		host->ops->set_clock(host, host->clock);
 	} else
 		sdhci_writeb(host, ctrl, SDHCI_HOST_CONTROL);
-
-	/*
-	 * Some (ENE) controllers go apeshit on some ios operation,
-	 * signalling timeout and CRC errors even on CMD0. Resetting
-	 * it on each ios seems to solve the problem.
-	 */
-	if (host->quirks & SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS)
-		sdhci_do_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA);
 }
 EXPORT_SYMBOL_GPL(sdhci_set_ios);
 
@@ -2718,8 +2745,7 @@ void sdhci_abort_tuning(struct sdhci_host *host, u32 opcode)
 {
 	sdhci_reset_tuning(host);
 
-	sdhci_do_reset(host, SDHCI_RESET_CMD);
-	sdhci_do_reset(host, SDHCI_RESET_DATA);
+	sdhci_reset_for(host, TUNING_ABORT);
 
 	sdhci_end_tuning(host);
 
@@ -2987,8 +3013,7 @@ static void sdhci_card_event(struct mmc_host *mmc)
 		pr_err("%s: Resetting controller.\n",
 			mmc_hostname(mmc));
 
-		sdhci_do_reset(host, SDHCI_RESET_CMD);
-		sdhci_do_reset(host, SDHCI_RESET_DATA);
+		sdhci_reset_for(host, CARD_REMOVED);
 
 		sdhci_error_out_mrqs(host, -ENOMEDIUM);
 	}
@@ -3059,12 +3084,7 @@ static bool sdhci_request_done(struct sdhci_host *host)
 			/* This is to force an update */
 			host->ops->set_clock(host, host->clock);
 
-		/*
-		 * Spec says we should do both at the same time, but Ricoh
-		 * controllers do not like that.
-		 */
-		sdhci_do_reset(host, SDHCI_RESET_CMD);
-		sdhci_do_reset(host, SDHCI_RESET_DATA);
+		sdhci_reset_for(host, REQUEST_ERROR);
 
 		host->pending_reset = false;
 	}
@@ -3905,10 +3925,8 @@ void sdhci_cqe_disable(struct mmc_host *mmc, bool recovery)
 
 	host->cqe_on = false;
 
-	if (recovery) {
-		sdhci_do_reset(host, SDHCI_RESET_CMD);
-		sdhci_do_reset(host, SDHCI_RESET_DATA);
-	}
+	if (recovery)
+		sdhci_reset_for(host, CQE_RECOVERY);
 
 	pr_debug("%s: sdhci: CQE off, IRQ mask %#x, IRQ status %#x\n",
 		 mmc_hostname(mmc), host->ier,
@@ -3928,7 +3946,7 @@ bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error,
 
 	if (intmask & (SDHCI_INT_INDEX | SDHCI_INT_END_BIT | SDHCI_INT_CRC)) {
 		*cmd_error = -EILSEQ;
-		if (!mmc_op_tuning(host->cmd->opcode))
+		if (!mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND))))
 			sdhci_err_stats_inc(host, CMD_CRC);
 	} else if (intmask & SDHCI_INT_TIMEOUT) {
 		*cmd_error = -ETIMEDOUT;
@@ -3938,7 +3956,7 @@ bool sdhci_cqe_irq(struct sdhci_host *host, u32 intmask, int *cmd_error,
 
 	if (intmask & (SDHCI_INT_DATA_END_BIT | SDHCI_INT_DATA_CRC)) {
 		*data_error = -EILSEQ;
-		if (!mmc_op_tuning(host->cmd->opcode))
+		if (!mmc_op_tuning(SDHCI_GET_CMD(sdhci_readw(host, SDHCI_COMMAND))))
 			sdhci_err_stats_inc(host, DAT_CRC);
 	} else if (intmask & SDHCI_INT_DATA_TIMEOUT) {
 		*data_error = -ETIMEDOUT;
@@ -4066,7 +4084,7 @@ void __sdhci_read_caps(struct sdhci_host *host, const u16 *ver,
 	if (debug_quirks2)
 		host->quirks2 = debug_quirks2;
 
-	sdhci_do_reset(host, SDHCI_RESET_ALL);
+	sdhci_reset_for_all(host);
 
 	if (host->v4_mode)
 		sdhci_do_enable_v4_mode(host);
@@ -4807,7 +4825,7 @@ int __sdhci_add_host(struct sdhci_host *host)
 unled:
 	sdhci_led_unregister(host);
 unirq:
-	sdhci_do_reset(host, SDHCI_RESET_ALL);
+	sdhci_reset_for_all(host);
 	sdhci_writel(host, 0, SDHCI_INT_ENABLE);
 	sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
 	free_irq(host->irq, host);
@@ -4865,7 +4883,7 @@ void sdhci_remove_host(struct sdhci_host *host, int dead)
 	sdhci_led_unregister(host);
 
 	if (!dead)
-		sdhci_do_reset(host, SDHCI_RESET_ALL);
+		sdhci_reset_for_all(host);
 
 	sdhci_writel(host, 0, SDHCI_INT_ENABLE);
 	sdhci_writel(host, 0, SDHCI_SIGNAL_ENABLE);
diff --git a/drivers/mmc/host/sdhci.h b/drivers/mmc/host/sdhci.h
index 95a08f09df30..d750c464bd1e 100644
--- a/drivers/mmc/host/sdhci.h
+++ b/drivers/mmc/host/sdhci.h
@@ -379,8 +379,6 @@ struct sdhci_host {
 #define SDHCI_QUIRK_NO_CARD_NO_RESET			(1<<2)
 /* Controller doesn't like clearing the power reg before a change */
 #define SDHCI_QUIRK_SINGLE_POWER_WRITE			(1<<3)
-/* Controller has flaky internal state so reset it on each ios change */
-#define SDHCI_QUIRK_RESET_CMD_DATA_ON_IOS		(1<<4)
 /* Controller has an unusable DMA engine */
 #define SDHCI_QUIRK_BROKEN_DMA				(1<<5)
 /* Controller has an unusable ADMA engine */
diff --git a/drivers/mmc/host/sdhci_am654.c b/drivers/mmc/host/sdhci_am654.c
index e7ced1496a07..8f1023480e12 100644
--- a/drivers/mmc/host/sdhci_am654.c
+++ b/drivers/mmc/host/sdhci_am654.c
@@ -554,7 +554,6 @@ static const struct cqhci_host_ops sdhci_am654_cqhci_ops = {
 static int sdhci_am654_cqe_add_host(struct sdhci_host *host)
 {
 	struct cqhci_host *cq_host;
-	int ret;
 
 	cq_host = devm_kzalloc(mmc_dev(host->mmc), sizeof(struct cqhci_host),
 			       GFP_KERNEL);
@@ -568,9 +567,7 @@ static int sdhci_am654_cqe_add_host(struct sdhci_host *host)
 
 	host->mmc->caps2 |= MMC_CAP2_CQE;
 
-	ret = cqhci_init(cq_host, host->mmc, 1);
-
-	return ret;
+	return cqhci_init(cq_host, host->mmc, 1);
 }
 
 static int sdhci_am654_get_otap_delay(struct sdhci_host *host,
diff --git a/drivers/mmc/host/wmt-sdmmc.c b/drivers/mmc/host/wmt-sdmmc.c
index 163ac9df8cca..9b5c503e3a3f 100644
--- a/drivers/mmc/host/wmt-sdmmc.c
+++ b/drivers/mmc/host/wmt-sdmmc.c
@@ -846,7 +846,7 @@ static int wmt_mci_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->clk_sdmmc)) {
 		dev_err(&pdev->dev, "Error getting clock\n");
 		ret = PTR_ERR(priv->clk_sdmmc);
-		goto fail5;
+		goto fail5_and_a_half;
 	}
 
 	ret = clk_prepare_enable(priv->clk_sdmmc);
@@ -863,6 +863,9 @@ static int wmt_mci_probe(struct platform_device *pdev)
 	return 0;
 fail6:
 	clk_put(priv->clk_sdmmc);
+fail5_and_a_half:
+	dma_free_coherent(&pdev->dev, mmc->max_blk_count * 16,
+			  priv->dma_desc_buffer, priv->dma_desc_device_addr);
 fail5:
 	free_irq(dma_irq, priv);
 fail4:
diff --git a/drivers/mtd/maps/pismo.c b/drivers/mtd/maps/pismo.c
index 946ba80f9758..5fcefcd0baca 100644
--- a/drivers/mtd/maps/pismo.c
+++ b/drivers/mtd/maps/pismo.c
@@ -195,7 +195,7 @@ static void pismo_add_one(struct pismo_data *pismo, int i,
 	}
 }
 
-static int pismo_remove(struct i2c_client *client)
+static void pismo_remove(struct i2c_client *client)
 {
 	struct pismo_data *pismo = i2c_get_clientdata(client);
 	int i;
@@ -204,8 +204,6 @@ static int pismo_remove(struct i2c_client *client)
 		platform_device_unregister(pismo->dev[i]);
 
 	kfree(pismo);
-
-	return 0;
 }
 
 static int pismo_probe(struct i2c_client *client,
diff --git a/drivers/mtd/parsers/Kconfig b/drivers/mtd/parsers/Kconfig
index b43df73927a0..d6db655a1d24 100644
--- a/drivers/mtd/parsers/Kconfig
+++ b/drivers/mtd/parsers/Kconfig
@@ -69,8 +69,8 @@ config MTD_OF_PARTS
 
 config MTD_OF_PARTS_BCM4908
 	bool "BCM4908 partitioning support"
-	depends on MTD_OF_PARTS && (ARCH_BCM4908 || COMPILE_TEST)
-	default ARCH_BCM4908
+	depends on MTD_OF_PARTS && (ARCH_BCMBCA || COMPILE_TEST)
+	default ARCH_BCMBCA
 	help
 	  This provides partitions parser for BCM4908 family devices
 	  that can have multiple "firmware" partitions. It takes care of
@@ -78,7 +78,7 @@ config MTD_OF_PARTS_BCM4908
 
 config MTD_OF_PARTS_LINKSYS_NS
 	bool "Linksys Northstar partitioning support"
-	depends on MTD_OF_PARTS && (ARCH_BCM_5301X || ARCH_BCM4908 || COMPILE_TEST)
+	depends on MTD_OF_PARTS && (ARCH_BCM_5301X || ARCH_BCMBCA || COMPILE_TEST)
 	default ARCH_BCM_5301X
 	help
 	  This provides partitions parser for Linksys devices based on Broadcom
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index 94c889802566..15d4a38b1351 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -500,6 +500,8 @@ config NET_SB1000
 
 source "drivers/net/phy/Kconfig"
 
+source "drivers/net/pse-pd/Kconfig"
+
 source "drivers/net/can/Kconfig"
 
 source "drivers/net/mctp/Kconfig"
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index 3f1192d3c52d..6ce076462dbf 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -23,6 +23,7 @@ obj-$(CONFIG_NET) += loopback.o
 obj-$(CONFIG_NETDEV_LEGACY_INIT) += Space.o
 obj-$(CONFIG_NETCONSOLE) += netconsole.o
 obj-y += phy/
+obj-y += pse-pd/
 obj-y += mdio/
 obj-y += pcs/
 obj-$(CONFIG_RIONET) += rionet.o
diff --git a/drivers/net/Space.c b/drivers/net/Space.c
index f475eef14390..83214e2e70ab 100644
--- a/drivers/net/Space.c
+++ b/drivers/net/Space.c
@@ -68,7 +68,7 @@ static int netdev_boot_setup_add(char *name, struct ifmap *map)
 	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
 		if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
 			memset(s[i].name, 0, sizeof(s[i].name));
-			strlcpy(s[i].name, name, IFNAMSIZ);
+			strscpy(s[i].name, name, IFNAMSIZ);
 			memcpy(&s[i].map, map, sizeof(s[i].map));
 			break;
 		}
diff --git a/drivers/net/amt.c b/drivers/net/amt.c
index 9a247eb7679c..2d20be6ffb7e 100644
--- a/drivers/net/amt.c
+++ b/drivers/net/amt.c
@@ -2894,8 +2894,7 @@ static void amt_event_work(struct work_struct *work)
 			amt_event_send_request(amt);
 			break;
 		default:
-			if (skb)
-				kfree_skb(skb);
+			kfree_skb(skb);
 			break;
 		}
 	}
@@ -3033,8 +3032,7 @@ static int amt_dev_stop(struct net_device *dev)
 	cancel_work_sync(&amt->event_wq);
 	for (i = 0; i < AMT_MAX_EVENTS; i++) {
 		skb = amt->events[i].skb;
-		if (skb)
-			kfree_skb(skb);
+		kfree_skb(skb);
 		amt->events[i].event = AMT_EVENT_NONE;
 		amt->events[i].skb = NULL;
 	}
diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c
index 86d42306aa5e..24bb50dfd362 100644
--- a/drivers/net/bonding/bond_main.c
+++ b/drivers/net/bonding/bond_main.c
@@ -5650,7 +5650,7 @@ static int bond_ethtool_get_link_ksettings(struct net_device *bond_dev,
 static void bond_ethtool_get_drvinfo(struct net_device *bond_dev,
 				     struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d",
 		 BOND_ABI_VERSION);
 }
diff --git a/drivers/net/bonding/bond_sysfs.c b/drivers/net/bonding/bond_sysfs.c
index 9b5a5df23d21..8996bd0a194a 100644
--- a/drivers/net/bonding/bond_sysfs.c
+++ b/drivers/net/bonding/bond_sysfs.c
@@ -47,10 +47,10 @@ static ssize_t bonding_show_bonds(struct class *cls,
 			/* not enough space for another interface name */
 			if ((PAGE_SIZE - res) > 10)
 				res = PAGE_SIZE - 10;
-			res += sprintf(buf + res, "++more++ ");
+			res += sysfs_emit_at(buf, res, "++more++ ");
 			break;
 		}
-		res += sprintf(buf + res, "%s ", bond->dev->name);
+		res += sysfs_emit_at(buf, res, "%s ", bond->dev->name);
 	}
 	if (res)
 		buf[res-1] = '\n'; /* eat the leftover space */
@@ -178,10 +178,10 @@ static ssize_t bonding_show_slaves(struct device *d,
 			/* not enough space for another interface name */
 			if ((PAGE_SIZE - res) > 10)
 				res = PAGE_SIZE - 10;
-			res += sprintf(buf + res, "++more++ ");
+			res += sysfs_emit_at(buf, res, "++more++ ");
 			break;
 		}
-		res += sprintf(buf + res, "%s ", slave->dev->name);
+		res += sysfs_emit_at(buf, res, "%s ", slave->dev->name);
 	}
 
 	rtnl_unlock();
@@ -203,7 +203,7 @@ static ssize_t bonding_show_mode(struct device *d,
 
 	val = bond_opt_get_val(BOND_OPT_MODE, BOND_MODE(bond));
 
-	return sprintf(buf, "%s %d\n", val->string, BOND_MODE(bond));
+	return sysfs_emit(buf, "%s %d\n", val->string, BOND_MODE(bond));
 }
 static DEVICE_ATTR(mode, 0644, bonding_show_mode, bonding_sysfs_store_option);
 
@@ -217,7 +217,7 @@ static ssize_t bonding_show_xmit_hash(struct device *d,
 
 	val = bond_opt_get_val(BOND_OPT_XMIT_HASH, bond->params.xmit_policy);
 
-	return sprintf(buf, "%s %d\n", val->string, bond->params.xmit_policy);
+	return sysfs_emit(buf, "%s %d\n", val->string, bond->params.xmit_policy);
 }
 static DEVICE_ATTR(xmit_hash_policy, 0644,
 		   bonding_show_xmit_hash, bonding_sysfs_store_option);
@@ -233,7 +233,7 @@ static ssize_t bonding_show_arp_validate(struct device *d,
 	val = bond_opt_get_val(BOND_OPT_ARP_VALIDATE,
 			       bond->params.arp_validate);
 
-	return sprintf(buf, "%s %d\n", val->string, bond->params.arp_validate);
+	return sysfs_emit(buf, "%s %d\n", val->string, bond->params.arp_validate);
 }
 static DEVICE_ATTR(arp_validate, 0644, bonding_show_arp_validate,
 		   bonding_sysfs_store_option);
@@ -248,7 +248,7 @@ static ssize_t bonding_show_arp_all_targets(struct device *d,
 
 	val = bond_opt_get_val(BOND_OPT_ARP_ALL_TARGETS,
 			       bond->params.arp_all_targets);
-	return sprintf(buf, "%s %d\n",
+	return sysfs_emit(buf, "%s %d\n",
 		       val->string, bond->params.arp_all_targets);
 }
 static DEVICE_ATTR(arp_all_targets, 0644,
@@ -265,7 +265,7 @@ static ssize_t bonding_show_fail_over_mac(struct device *d,
 	val = bond_opt_get_val(BOND_OPT_FAIL_OVER_MAC,
 			       bond->params.fail_over_mac);
 
-	return sprintf(buf, "%s %d\n", val->string, bond->params.fail_over_mac);
+	return sysfs_emit(buf, "%s %d\n", val->string, bond->params.fail_over_mac);
 }
 static DEVICE_ATTR(fail_over_mac, 0644,
 		   bonding_show_fail_over_mac, bonding_sysfs_store_option);
@@ -277,7 +277,7 @@ static ssize_t bonding_show_arp_interval(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n", bond->params.arp_interval);
+	return sysfs_emit(buf, "%d\n", bond->params.arp_interval);
 }
 static DEVICE_ATTR(arp_interval, 0644,
 		   bonding_show_arp_interval, bonding_sysfs_store_option);
@@ -292,8 +292,8 @@ static ssize_t bonding_show_arp_targets(struct device *d,
 
 	for (i = 0; i < BOND_MAX_ARP_TARGETS; i++) {
 		if (bond->params.arp_targets[i])
-			res += sprintf(buf + res, "%pI4 ",
-				       &bond->params.arp_targets[i]);
+			res += sysfs_emit_at(buf, res, "%pI4 ",
+					     &bond->params.arp_targets[i]);
 	}
 	if (res)
 		buf[res-1] = '\n'; /* eat the leftover space */
@@ -310,7 +310,7 @@ static ssize_t bonding_show_missed_max(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%u\n", bond->params.missed_max);
+	return sysfs_emit(buf, "%u\n", bond->params.missed_max);
 }
 static DEVICE_ATTR(arp_missed_max, 0644,
 		   bonding_show_missed_max, bonding_sysfs_store_option);
@@ -322,7 +322,7 @@ static ssize_t bonding_show_downdelay(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n", bond->params.downdelay * bond->params.miimon);
+	return sysfs_emit(buf, "%d\n", bond->params.downdelay * bond->params.miimon);
 }
 static DEVICE_ATTR(downdelay, 0644,
 		   bonding_show_downdelay, bonding_sysfs_store_option);
@@ -333,7 +333,7 @@ static ssize_t bonding_show_updelay(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n", bond->params.updelay * bond->params.miimon);
+	return sysfs_emit(buf, "%d\n", bond->params.updelay * bond->params.miimon);
 
 }
 static DEVICE_ATTR(updelay, 0644,
@@ -345,8 +345,8 @@ static ssize_t bonding_show_peer_notif_delay(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n",
-		       bond->params.peer_notif_delay * bond->params.miimon);
+	return sysfs_emit(buf, "%d\n",
+			  bond->params.peer_notif_delay * bond->params.miimon);
 }
 static DEVICE_ATTR(peer_notif_delay, 0644,
 		   bonding_show_peer_notif_delay, bonding_sysfs_store_option);
@@ -361,7 +361,7 @@ static ssize_t bonding_show_lacp_active(struct device *d,
 
 	val = bond_opt_get_val(BOND_OPT_LACP_ACTIVE, bond->params.lacp_active);
 
-	return sprintf(buf, "%s %d\n", val->string, bond->params.lacp_active);
+	return sysfs_emit(buf, "%s %d\n", val->string, bond->params.lacp_active);
 }
 static DEVICE_ATTR(lacp_active, 0644,
 		   bonding_show_lacp_active, bonding_sysfs_store_option);
@@ -375,7 +375,7 @@ static ssize_t bonding_show_lacp_rate(struct device *d,
 
 	val = bond_opt_get_val(BOND_OPT_LACP_RATE, bond->params.lacp_fast);
 
-	return sprintf(buf, "%s %d\n", val->string, bond->params.lacp_fast);
+	return sysfs_emit(buf, "%s %d\n", val->string, bond->params.lacp_fast);
 }
 static DEVICE_ATTR(lacp_rate, 0644,
 		   bonding_show_lacp_rate, bonding_sysfs_store_option);
@@ -386,7 +386,7 @@ static ssize_t bonding_show_min_links(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%u\n", bond->params.min_links);
+	return sysfs_emit(buf, "%u\n", bond->params.min_links);
 }
 static DEVICE_ATTR(min_links, 0644,
 		   bonding_show_min_links, bonding_sysfs_store_option);
@@ -400,7 +400,7 @@ static ssize_t bonding_show_ad_select(struct device *d,
 
 	val = bond_opt_get_val(BOND_OPT_AD_SELECT, bond->params.ad_select);
 
-	return sprintf(buf, "%s %d\n", val->string, bond->params.ad_select);
+	return sysfs_emit(buf, "%s %d\n", val->string, bond->params.ad_select);
 }
 static DEVICE_ATTR(ad_select, 0644,
 		   bonding_show_ad_select, bonding_sysfs_store_option);
@@ -412,7 +412,7 @@ static ssize_t bonding_show_num_peer_notif(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n", bond->params.num_peer_notif);
+	return sysfs_emit(buf, "%d\n", bond->params.num_peer_notif);
 }
 static DEVICE_ATTR(num_grat_arp, 0644,
 		   bonding_show_num_peer_notif, bonding_sysfs_store_option);
@@ -426,7 +426,7 @@ static ssize_t bonding_show_miimon(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n", bond->params.miimon);
+	return sysfs_emit(buf, "%d\n", bond->params.miimon);
 }
 static DEVICE_ATTR(miimon, 0644,
 		   bonding_show_miimon, bonding_sysfs_store_option);
@@ -443,7 +443,7 @@ static ssize_t bonding_show_primary(struct device *d,
 	rcu_read_lock();
 	primary = rcu_dereference(bond->primary_slave);
 	if (primary)
-		count = sprintf(buf, "%s\n", primary->dev->name);
+		count = sysfs_emit(buf, "%s\n", primary->dev->name);
 	rcu_read_unlock();
 
 	return count;
@@ -462,8 +462,8 @@ static ssize_t bonding_show_primary_reselect(struct device *d,
 	val = bond_opt_get_val(BOND_OPT_PRIMARY_RESELECT,
 			       bond->params.primary_reselect);
 
-	return sprintf(buf, "%s %d\n",
-		       val->string, bond->params.primary_reselect);
+	return sysfs_emit(buf, "%s %d\n",
+			  val->string, bond->params.primary_reselect);
 }
 static DEVICE_ATTR(primary_reselect, 0644,
 		   bonding_show_primary_reselect, bonding_sysfs_store_option);
@@ -475,7 +475,7 @@ static ssize_t bonding_show_carrier(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n", bond->params.use_carrier);
+	return sysfs_emit(buf, "%d\n", bond->params.use_carrier);
 }
 static DEVICE_ATTR(use_carrier, 0644,
 		   bonding_show_carrier, bonding_sysfs_store_option);
@@ -493,7 +493,7 @@ static ssize_t bonding_show_active_slave(struct device *d,
 	rcu_read_lock();
 	slave_dev = bond_option_active_slave_get_rcu(bond);
 	if (slave_dev)
-		count = sprintf(buf, "%s\n", slave_dev->name);
+		count = sysfs_emit(buf, "%s\n", slave_dev->name);
 	rcu_read_unlock();
 
 	return count;
@@ -509,7 +509,7 @@ static ssize_t bonding_show_mii_status(struct device *d,
 	struct bonding *bond = to_bond(d);
 	bool active = netif_carrier_ok(bond->dev);
 
-	return sprintf(buf, "%s\n", active ? "up" : "down");
+	return sysfs_emit(buf, "%s\n", active ? "up" : "down");
 }
 static DEVICE_ATTR(mii_status, 0444, bonding_show_mii_status, NULL);
 
@@ -524,9 +524,9 @@ static ssize_t bonding_show_ad_aggregator(struct device *d,
 	if (BOND_MODE(bond) == BOND_MODE_8023AD) {
 		struct ad_info ad_info;
 
-		count = sprintf(buf, "%d\n",
-				bond_3ad_get_active_agg_info(bond, &ad_info)
-				?  0 : ad_info.aggregator_id);
+		count = sysfs_emit(buf, "%d\n",
+				   bond_3ad_get_active_agg_info(bond, &ad_info)
+				   ?  0 : ad_info.aggregator_id);
 	}
 
 	return count;
@@ -545,9 +545,9 @@ static ssize_t bonding_show_ad_num_ports(struct device *d,
 	if (BOND_MODE(bond) == BOND_MODE_8023AD) {
 		struct ad_info ad_info;
 
-		count = sprintf(buf, "%d\n",
-				bond_3ad_get_active_agg_info(bond, &ad_info)
-				?  0 : ad_info.ports);
+		count = sysfs_emit(buf, "%d\n",
+				   bond_3ad_get_active_agg_info(bond, &ad_info)
+				   ?  0 : ad_info.ports);
 	}
 
 	return count;
@@ -566,9 +566,9 @@ static ssize_t bonding_show_ad_actor_key(struct device *d,
 	if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) {
 		struct ad_info ad_info;
 
-		count = sprintf(buf, "%d\n",
-				bond_3ad_get_active_agg_info(bond, &ad_info)
-				?  0 : ad_info.actor_key);
+		count = sysfs_emit(buf, "%d\n",
+				   bond_3ad_get_active_agg_info(bond, &ad_info)
+				   ?  0 : ad_info.actor_key);
 	}
 
 	return count;
@@ -587,9 +587,9 @@ static ssize_t bonding_show_ad_partner_key(struct device *d,
 	if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN)) {
 		struct ad_info ad_info;
 
-		count = sprintf(buf, "%d\n",
-				bond_3ad_get_active_agg_info(bond, &ad_info)
-				?  0 : ad_info.partner_key);
+		count = sysfs_emit(buf, "%d\n",
+				   bond_3ad_get_active_agg_info(bond, &ad_info)
+				   ?  0 : ad_info.partner_key);
 	}
 
 	return count;
@@ -609,7 +609,7 @@ static ssize_t bonding_show_ad_partner_mac(struct device *d,
 		struct ad_info ad_info;
 
 		if (!bond_3ad_get_active_agg_info(bond, &ad_info))
-			count = sprintf(buf, "%pM\n", ad_info.partner_system);
+			count = sysfs_emit(buf, "%pM\n", ad_info.partner_system);
 	}
 
 	return count;
@@ -634,11 +634,11 @@ static ssize_t bonding_show_queue_id(struct device *d,
 			/* not enough space for another interface_name:queue_id pair */
 			if ((PAGE_SIZE - res) > 10)
 				res = PAGE_SIZE - 10;
-			res += sprintf(buf + res, "++more++ ");
+			res += sysfs_emit_at(buf, res, "++more++ ");
 			break;
 		}
-		res += sprintf(buf + res, "%s:%d ",
-			       slave->dev->name, slave->queue_id);
+		res += sysfs_emit_at(buf, res, "%s:%d ",
+				     slave->dev->name, slave->queue_id);
 	}
 	if (res)
 		buf[res-1] = '\n'; /* eat the leftover space */
@@ -658,7 +658,7 @@ static ssize_t bonding_show_slaves_active(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n", bond->params.all_slaves_active);
+	return sysfs_emit(buf, "%d\n", bond->params.all_slaves_active);
 }
 static DEVICE_ATTR(all_slaves_active, 0644,
 		   bonding_show_slaves_active, bonding_sysfs_store_option);
@@ -670,7 +670,7 @@ static ssize_t bonding_show_resend_igmp(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n", bond->params.resend_igmp);
+	return sysfs_emit(buf, "%d\n", bond->params.resend_igmp);
 }
 static DEVICE_ATTR(resend_igmp, 0644,
 		   bonding_show_resend_igmp, bonding_sysfs_store_option);
@@ -682,7 +682,7 @@ static ssize_t bonding_show_lp_interval(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n", bond->params.lp_interval);
+	return sysfs_emit(buf, "%d\n", bond->params.lp_interval);
 }
 static DEVICE_ATTR(lp_interval, 0644,
 		   bonding_show_lp_interval, bonding_sysfs_store_option);
@@ -693,7 +693,7 @@ static ssize_t bonding_show_tlb_dynamic_lb(struct device *d,
 {
 	struct bonding *bond = to_bond(d);
 
-	return sprintf(buf, "%d\n", bond->params.tlb_dynamic_lb);
+	return sysfs_emit(buf, "%d\n", bond->params.tlb_dynamic_lb);
 }
 static DEVICE_ATTR(tlb_dynamic_lb, 0644,
 		   bonding_show_tlb_dynamic_lb, bonding_sysfs_store_option);
@@ -705,7 +705,7 @@ static ssize_t bonding_show_packets_per_slave(struct device *d,
 	struct bonding *bond = to_bond(d);
 	unsigned int packets_per_slave = bond->params.packets_per_slave;
 
-	return sprintf(buf, "%u\n", packets_per_slave);
+	return sysfs_emit(buf, "%u\n", packets_per_slave);
 }
 static DEVICE_ATTR(packets_per_slave, 0644,
 		   bonding_show_packets_per_slave, bonding_sysfs_store_option);
@@ -717,7 +717,7 @@ static ssize_t bonding_show_ad_actor_sys_prio(struct device *d,
 	struct bonding *bond = to_bond(d);
 
 	if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN))
-		return sprintf(buf, "%hu\n", bond->params.ad_actor_sys_prio);
+		return sysfs_emit(buf, "%hu\n", bond->params.ad_actor_sys_prio);
 
 	return 0;
 }
@@ -731,7 +731,7 @@ static ssize_t bonding_show_ad_actor_system(struct device *d,
 	struct bonding *bond = to_bond(d);
 
 	if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN))
-		return sprintf(buf, "%pM\n", bond->params.ad_actor_system);
+		return sysfs_emit(buf, "%pM\n", bond->params.ad_actor_system);
 
 	return 0;
 }
@@ -746,7 +746,7 @@ static ssize_t bonding_show_ad_user_port_key(struct device *d,
 	struct bonding *bond = to_bond(d);
 
 	if (BOND_MODE(bond) == BOND_MODE_8023AD && capable(CAP_NET_ADMIN))
-		return sprintf(buf, "%hu\n", bond->params.ad_user_port_key);
+		return sysfs_emit(buf, "%hu\n", bond->params.ad_user_port_key);
 
 	return 0;
 }
diff --git a/drivers/net/bonding/bond_sysfs_slave.c b/drivers/net/bonding/bond_sysfs_slave.c
index 69b0a3751dff..313866f2c0e4 100644
--- a/drivers/net/bonding/bond_sysfs_slave.c
+++ b/drivers/net/bonding/bond_sysfs_slave.c
@@ -22,30 +22,30 @@ static ssize_t state_show(struct slave *slave, char *buf)
 {
 	switch (bond_slave_state(slave)) {
 	case BOND_STATE_ACTIVE:
-		return sprintf(buf, "active\n");
+		return sysfs_emit(buf, "active\n");
 	case BOND_STATE_BACKUP:
-		return sprintf(buf, "backup\n");
+		return sysfs_emit(buf, "backup\n");
 	default:
-		return sprintf(buf, "UNKNOWN\n");
+		return sysfs_emit(buf, "UNKNOWN\n");
 	}
 }
 static SLAVE_ATTR_RO(state);
 
 static ssize_t mii_status_show(struct slave *slave, char *buf)
 {
-	return sprintf(buf, "%s\n", bond_slave_link_status(slave->link));
+	return sysfs_emit(buf, "%s\n", bond_slave_link_status(slave->link));
 }
 static SLAVE_ATTR_RO(mii_status);
 
 static ssize_t link_failure_count_show(struct slave *slave, char *buf)
 {
-	return sprintf(buf, "%d\n", slave->link_failure_count);
+	return sysfs_emit(buf, "%d\n", slave->link_failure_count);
 }
 static SLAVE_ATTR_RO(link_failure_count);
 
 static ssize_t perm_hwaddr_show(struct slave *slave, char *buf)
 {
-	return sprintf(buf, "%*phC\n",
+	return sysfs_emit(buf, "%*phC\n",
 		       slave->dev->addr_len,
 		       slave->perm_hwaddr);
 }
@@ -53,7 +53,7 @@ static SLAVE_ATTR_RO(perm_hwaddr);
 
 static ssize_t queue_id_show(struct slave *slave, char *buf)
 {
-	return sprintf(buf, "%d\n", slave->queue_id);
+	return sysfs_emit(buf, "%d\n", slave->queue_id);
 }
 static SLAVE_ATTR_RO(queue_id);
 
@@ -64,11 +64,11 @@ static ssize_t ad_aggregator_id_show(struct slave *slave, char *buf)
 	if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) {
 		agg = SLAVE_AD_INFO(slave)->port.aggregator;
 		if (agg)
-			return sprintf(buf, "%d\n",
-				       agg->aggregator_identifier);
+			return sysfs_emit(buf, "%d\n",
+					  agg->aggregator_identifier);
 	}
 
-	return sprintf(buf, "N/A\n");
+	return sysfs_emit(buf, "N/A\n");
 }
 static SLAVE_ATTR_RO(ad_aggregator_id);
 
@@ -79,11 +79,11 @@ static ssize_t ad_actor_oper_port_state_show(struct slave *slave, char *buf)
 	if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) {
 		ad_port = &SLAVE_AD_INFO(slave)->port;
 		if (ad_port->aggregator)
-			return sprintf(buf, "%u\n",
+			return sysfs_emit(buf, "%u\n",
 				       ad_port->actor_oper_port_state);
 	}
 
-	return sprintf(buf, "N/A\n");
+	return sysfs_emit(buf, "N/A\n");
 }
 static SLAVE_ATTR_RO(ad_actor_oper_port_state);
 
@@ -94,11 +94,11 @@ static ssize_t ad_partner_oper_port_state_show(struct slave *slave, char *buf)
 	if (BOND_MODE(slave->bond) == BOND_MODE_8023AD) {
 		ad_port = &SLAVE_AD_INFO(slave)->port;
 		if (ad_port->aggregator)
-			return sprintf(buf, "%u\n",
+			return sysfs_emit(buf, "%u\n",
 				       ad_port->partner_oper.port_state);
 	}
 
-	return sprintf(buf, "N/A\n");
+	return sysfs_emit(buf, "N/A\n");
 }
 static SLAVE_ATTR_RO(ad_partner_oper_port_state);
 
diff --git a/drivers/net/can/c_can/c_can.h b/drivers/net/can/c_can/c_can.h
index f23a03300a81..029cd8194ed5 100644
--- a/drivers/net/can/c_can/c_can.h
+++ b/drivers/net/can/c_can/c_can.h
@@ -235,9 +235,22 @@ static inline u8 c_can_get_tx_tail(const struct c_can_tx_ring *ring)
 	return ring->tail & (ring->obj_num - 1);
 }
 
-static inline u8 c_can_get_tx_free(const struct c_can_tx_ring *ring)
+static inline u8 c_can_get_tx_free(const struct c_can_priv *priv,
+				   const struct c_can_tx_ring *ring)
 {
-	return ring->obj_num - (ring->head - ring->tail);
+	u8 head = c_can_get_tx_head(ring);
+	u8 tail = c_can_get_tx_tail(ring);
+
+	if (priv->type == BOSCH_D_CAN)
+		return ring->obj_num - (ring->head - ring->tail);
+
+	/* This is not a FIFO. C/D_CAN sends out the buffers
+	 * prioritized. The lowest buffer number wins.
+	 */
+	if (head < tail)
+		return 0;
+
+	return ring->obj_num - head;
 }
 
 #endif /* C_CAN_H */
diff --git a/drivers/net/can/c_can/c_can_main.c b/drivers/net/can/c_can/c_can_main.c
index dc8132862f33..d6605dbb7737 100644
--- a/drivers/net/can/c_can/c_can_main.c
+++ b/drivers/net/can/c_can/c_can_main.c
@@ -429,7 +429,7 @@ static void c_can_setup_receive_object(struct net_device *dev, int iface,
 static bool c_can_tx_busy(const struct c_can_priv *priv,
 			  const struct c_can_tx_ring *tx_ring)
 {
-	if (c_can_get_tx_free(tx_ring) > 0)
+	if (c_can_get_tx_free(priv, tx_ring) > 0)
 		return false;
 
 	netif_stop_queue(priv->dev);
@@ -437,7 +437,7 @@ static bool c_can_tx_busy(const struct c_can_priv *priv,
 	/* Memory barrier before checking tx_free (head and tail) */
 	smp_mb();
 
-	if (c_can_get_tx_free(tx_ring) == 0) {
+	if (c_can_get_tx_free(priv, tx_ring) == 0) {
 		netdev_dbg(priv->dev,
 			   "Stopping tx-queue (tx_head=0x%08x, tx_tail=0x%08x, len=%d).\n",
 			   tx_ring->head, tx_ring->tail,
@@ -465,7 +465,7 @@ static netdev_tx_t c_can_start_xmit(struct sk_buff *skb,
 
 	idx = c_can_get_tx_head(tx_ring);
 	tx_ring->head++;
-	if (c_can_get_tx_free(tx_ring) == 0)
+	if (c_can_get_tx_free(priv, tx_ring) == 0)
 		netif_stop_queue(dev);
 
 	if (idx < c_can_get_tx_tail(tx_ring))
@@ -748,7 +748,7 @@ static void c_can_do_tx(struct net_device *dev)
 		return;
 
 	tx_ring->tail += pkts;
-	if (c_can_get_tx_free(tx_ring)) {
+	if (c_can_get_tx_free(priv, tx_ring)) {
 		/* Make sure that anybody stopping the queue after
 		 * this sees the new tx_ring->tail.
 		 */
@@ -760,8 +760,7 @@ static void c_can_do_tx(struct net_device *dev)
 	stats->tx_packets += pkts;
 
 	tail = c_can_get_tx_tail(tx_ring);
-
-	if (tail == 0) {
+	if (priv->type == BOSCH_D_CAN && tail == 0) {
 		u8 head = c_can_get_tx_head(tx_ring);
 
 		/* Start transmission for all cached messages */
diff --git a/drivers/net/can/ctucanfd/ctucanfd_base.c b/drivers/net/can/ctucanfd/ctucanfd_base.c
index 3c18d028bd8c..b8da15ea6ad9 100644
--- a/drivers/net/can/ctucanfd/ctucanfd_base.c
+++ b/drivers/net/can/ctucanfd/ctucanfd_base.c
@@ -657,7 +657,6 @@ static void ctucan_read_rx_frame(struct ctucan_priv *priv, struct canfd_frame *c
 		cf->can_id = (idw >> 18) & CAN_SFF_MASK;
 
 	/* BRS, ESI, RTR Flags */
-	cf->flags = 0;
 	if (FIELD_GET(REG_FRAME_FORMAT_W_FDF, ffw)) {
 		if (FIELD_GET(REG_FRAME_FORMAT_W_BRS, ffw))
 			cf->flags |= CANFD_BRS;
@@ -1425,7 +1424,7 @@ int ctucan_probe_common(struct device *dev, void __iomem *addr, int irq, unsigne
 
 	priv->can.clock.freq = can_clk_rate;
 
-	netif_napi_add(ndev, &priv->napi, ctucan_rx_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &priv->napi, ctucan_rx_poll);
 
 	ret = register_candev(ndev);
 	if (ret) {
diff --git a/drivers/net/can/ctucanfd/ctucanfd_platform.c b/drivers/net/can/ctucanfd/ctucanfd_platform.c
index 89d54c2151e1..f83684f006ea 100644
--- a/drivers/net/can/ctucanfd/ctucanfd_platform.c
+++ b/drivers/net/can/ctucanfd/ctucanfd_platform.c
@@ -58,7 +58,6 @@ static int ctucan_platform_probe(struct platform_device *pdev)
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	addr = devm_ioremap_resource(dev, res);
 	if (IS_ERR(addr)) {
-		dev_err(dev, "Cannot remap address.\n");
 		ret = PTR_ERR(addr);
 		goto err;
 	}
diff --git a/drivers/net/can/dev/rx-offload.c b/drivers/net/can/dev/rx-offload.c
index a32a01c172d4..81ebf0562c89 100644
--- a/drivers/net/can/dev/rx-offload.c
+++ b/drivers/net/can/dev/rx-offload.c
@@ -247,7 +247,7 @@ unsigned int can_rx_offload_get_echo_skb(struct can_rx_offload *offload,
 	struct net_device *dev = offload->dev;
 	struct net_device_stats *stats = &dev->stats;
 	struct sk_buff *skb;
-	u8 len;
+	unsigned int len;
 	int err;
 
 	skb = __can_get_echo_skb(dev, idx, &len, frame_len_ptr);
@@ -329,7 +329,7 @@ static int can_rx_offload_init_queue(struct net_device *dev,
 {
 	offload->dev = dev;
 
-	/* Limit queue len to 4x the weight (rounted to next power of two) */
+	/* Limit queue len to 4x the weight (rounded to next power of two) */
 	offload->skb_queue_len_max = 2 << fls(weight);
 	offload->skb_queue_len_max *= 4;
 	skb_queue_head_init(&offload->skb_queue);
diff --git a/drivers/net/can/dev/skb.c b/drivers/net/can/dev/skb.c
index 07e0feac8629..791a51e2f5d6 100644
--- a/drivers/net/can/dev/skb.c
+++ b/drivers/net/can/dev/skb.c
@@ -91,8 +91,8 @@ int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev,
 EXPORT_SYMBOL_GPL(can_put_echo_skb);
 
 struct sk_buff *
-__can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr,
-		   unsigned int *frame_len_ptr)
+__can_get_echo_skb(struct net_device *dev, unsigned int idx,
+		   unsigned int *len_ptr, unsigned int *frame_len_ptr)
 {
 	struct can_priv *priv = netdev_priv(dev);
 
@@ -108,16 +108,12 @@ __can_get_echo_skb(struct net_device *dev, unsigned int idx, u8 *len_ptr,
 		 */
 		struct sk_buff *skb = priv->echo_skb[idx];
 		struct can_skb_priv *can_skb_priv = can_skb_prv(skb);
-		struct canfd_frame *cf = (struct canfd_frame *)skb->data;
 
 		if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)
 			skb_tstamp_tx(skb, skb_hwtstamps(skb));
 
 		/* get the real payload length for netdev statistics */
-		if (cf->can_id & CAN_RTR_FLAG)
-			*len_ptr = 0;
-		else
-			*len_ptr = cf->len;
+		*len_ptr = can_skb_get_data_len(skb);
 
 		if (frame_len_ptr)
 			*frame_len_ptr = can_skb_priv->frame_len;
@@ -147,7 +143,7 @@ unsigned int can_get_echo_skb(struct net_device *dev, unsigned int idx,
 			      unsigned int *frame_len_ptr)
 {
 	struct sk_buff *skb;
-	u8 len;
+	unsigned int len;
 
 	skb = __can_get_echo_skb(dev, idx, &len, frame_len_ptr);
 	if (!skb)
@@ -191,6 +187,20 @@ void can_free_echo_skb(struct net_device *dev, unsigned int idx,
 }
 EXPORT_SYMBOL_GPL(can_free_echo_skb);
 
+/* fill common values for CAN sk_buffs */
+static void init_can_skb_reserve(struct sk_buff *skb)
+{
+	skb->pkt_type = PACKET_BROADCAST;
+	skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+	skb_reset_mac_header(skb);
+	skb_reset_network_header(skb);
+	skb_reset_transport_header(skb);
+
+	can_skb_reserve(skb);
+	can_skb_prv(skb)->skbcnt = 0;
+}
+
 struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf)
 {
 	struct sk_buff *skb;
@@ -204,16 +214,8 @@ struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf)
 	}
 
 	skb->protocol = htons(ETH_P_CAN);
-	skb->pkt_type = PACKET_BROADCAST;
-	skb->ip_summed = CHECKSUM_UNNECESSARY;
-
-	skb_reset_mac_header(skb);
-	skb_reset_network_header(skb);
-	skb_reset_transport_header(skb);
-
-	can_skb_reserve(skb);
+	init_can_skb_reserve(skb);
 	can_skb_prv(skb)->ifindex = dev->ifindex;
-	can_skb_prv(skb)->skbcnt = 0;
 
 	*cf = skb_put_zero(skb, sizeof(struct can_frame));
 
@@ -235,23 +237,51 @@ struct sk_buff *alloc_canfd_skb(struct net_device *dev,
 	}
 
 	skb->protocol = htons(ETH_P_CANFD);
-	skb->pkt_type = PACKET_BROADCAST;
-	skb->ip_summed = CHECKSUM_UNNECESSARY;
-
-	skb_reset_mac_header(skb);
-	skb_reset_network_header(skb);
-	skb_reset_transport_header(skb);
-
-	can_skb_reserve(skb);
+	init_can_skb_reserve(skb);
 	can_skb_prv(skb)->ifindex = dev->ifindex;
-	can_skb_prv(skb)->skbcnt = 0;
 
 	*cfd = skb_put_zero(skb, sizeof(struct canfd_frame));
 
+	/* set CAN FD flag by default */
+	(*cfd)->flags = CANFD_FDF;
+
 	return skb;
 }
 EXPORT_SYMBOL_GPL(alloc_canfd_skb);
 
+struct sk_buff *alloc_canxl_skb(struct net_device *dev,
+				struct canxl_frame **cxl,
+				unsigned int data_len)
+{
+	struct sk_buff *skb;
+
+	if (data_len < CANXL_MIN_DLEN || data_len > CANXL_MAX_DLEN)
+		goto out_error;
+
+	skb = netdev_alloc_skb(dev, sizeof(struct can_skb_priv) +
+			       CANXL_HDR_SIZE + data_len);
+	if (unlikely(!skb))
+		goto out_error;
+
+	skb->protocol = htons(ETH_P_CANXL);
+	init_can_skb_reserve(skb);
+	can_skb_prv(skb)->ifindex = dev->ifindex;
+
+	*cxl = skb_put_zero(skb, CANXL_HDR_SIZE + data_len);
+
+	/* set CAN XL flag and length information by default */
+	(*cxl)->flags = CANXL_XLF;
+	(*cxl)->len = data_len;
+
+	return skb;
+
+out_error:
+	*cxl = NULL;
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(alloc_canxl_skb);
+
 struct sk_buff *alloc_can_err_skb(struct net_device *dev, struct can_frame **cf)
 {
 	struct sk_buff *skb;
@@ -291,6 +321,14 @@ static bool can_skb_headroom_valid(struct net_device *dev, struct sk_buff *skb)
 		skb_reset_mac_header(skb);
 		skb_reset_network_header(skb);
 		skb_reset_transport_header(skb);
+
+		/* set CANFD_FDF flag for CAN FD frames */
+		if (can_is_canfd_skb(skb)) {
+			struct canfd_frame *cfd;
+
+			cfd = (struct canfd_frame *)skb->data;
+			cfd->flags |= CANFD_FDF;
+		}
 	}
 
 	return true;
@@ -299,18 +337,25 @@ static bool can_skb_headroom_valid(struct net_device *dev, struct sk_buff *skb)
 /* Drop a given socketbuffer if it does not contain a valid CAN frame. */
 bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb)
 {
-	const struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
 	struct can_priv *priv = netdev_priv(dev);
 
-	if (skb->protocol == htons(ETH_P_CAN)) {
-		if (unlikely(skb->len != CAN_MTU ||
-			     cfd->len > CAN_MAX_DLEN))
+	switch (ntohs(skb->protocol)) {
+	case ETH_P_CAN:
+		if (!can_is_can_skb(skb))
 			goto inval_skb;
-	} else if (skb->protocol == htons(ETH_P_CANFD)) {
-		if (unlikely(skb->len != CANFD_MTU ||
-			     cfd->len > CANFD_MAX_DLEN))
+		break;
+
+	case ETH_P_CANFD:
+		if (!can_is_canfd_skb(skb))
 			goto inval_skb;
-	} else {
+		break;
+
+	case ETH_P_CANXL:
+		if (!can_is_canxl_skb(skb))
+			goto inval_skb;
+		break;
+
+	default:
 		goto inval_skb;
 	}
 
diff --git a/drivers/net/can/flexcan/flexcan-core.c b/drivers/net/can/flexcan/flexcan-core.c
index ccb438eca517..5ee38e586fd8 100644
--- a/drivers/net/can/flexcan/flexcan-core.c
+++ b/drivers/net/can/flexcan/flexcan-core.c
@@ -295,45 +295,45 @@ static_assert(sizeof(struct flexcan_regs) ==  0x4 * 18 + 0xfb8);
 static const struct flexcan_devtype_data fsl_mcf5441x_devtype_data = {
 	.quirks = FLEXCAN_QUIRK_BROKEN_PERR_STATE |
 		FLEXCAN_QUIRK_NR_IRQ_3 | FLEXCAN_QUIRK_NR_MB_16 |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_FIFO,
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_FIFO,
 };
 
 static const struct flexcan_devtype_data fsl_p1010_devtype_data = {
 	.quirks = FLEXCAN_QUIRK_BROKEN_WERR_STATE |
 		FLEXCAN_QUIRK_BROKEN_PERR_STATE |
 		FLEXCAN_QUIRK_DEFAULT_BIG_ENDIAN |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_FIFO,
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_FIFO,
 };
 
 static const struct flexcan_devtype_data fsl_imx25_devtype_data = {
 	.quirks = FLEXCAN_QUIRK_BROKEN_WERR_STATE |
 		FLEXCAN_QUIRK_BROKEN_PERR_STATE |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_FIFO,
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_FIFO,
 };
 
 static const struct flexcan_devtype_data fsl_imx28_devtype_data = {
 	.quirks = FLEXCAN_QUIRK_BROKEN_PERR_STATE |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_FIFO,
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_FIFO,
 };
 
 static const struct flexcan_devtype_data fsl_imx6q_devtype_data = {
 	.quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
 		FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
 		FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR,
 };
 
 static const struct flexcan_devtype_data fsl_imx8qm_devtype_data = {
 	.quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
 		FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
 		FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SETUP_STOP_MODE_SCFW |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR,
 };
 
 static struct flexcan_devtype_data fsl_imx8mp_devtype_data = {
@@ -341,23 +341,23 @@ static struct flexcan_devtype_data fsl_imx8mp_devtype_data = {
 		FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_RX_MAILBOX |
 		FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_SETUP_STOP_MODE_GPR |
 		FLEXCAN_QUIRK_SUPPORT_FD | FLEXCAN_QUIRK_SUPPORT_ECC |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR,
 };
 
 static const struct flexcan_devtype_data fsl_vf610_devtype_data = {
 	.quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
 		FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_USE_RX_MAILBOX |
 		FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_SUPPORT_ECC |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR,
 };
 
 static const struct flexcan_devtype_data fsl_ls1021a_r2_devtype_data = {
 	.quirks = FLEXCAN_QUIRK_DISABLE_RXFG | FLEXCAN_QUIRK_ENABLE_EACEN_RRS |
 		FLEXCAN_QUIRK_BROKEN_PERR_STATE | FLEXCAN_QUIRK_USE_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR,
 };
 
 static const struct flexcan_devtype_data fsl_lx2160a_r1_devtype_data = {
@@ -365,8 +365,8 @@ static const struct flexcan_devtype_data fsl_lx2160a_r1_devtype_data = {
 		FLEXCAN_QUIRK_DISABLE_MECR | FLEXCAN_QUIRK_BROKEN_PERR_STATE |
 		FLEXCAN_QUIRK_USE_RX_MAILBOX | FLEXCAN_QUIRK_SUPPORT_FD |
 		FLEXCAN_QUIRK_SUPPORT_ECC |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR,
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR,
 };
 
 static const struct can_bittiming_const flexcan_bittiming_const = {
@@ -2085,20 +2085,20 @@ static int flexcan_probe(struct platform_device *pdev)
 	if ((devtype_data->quirks & FLEXCAN_QUIRK_SUPPORT_FD) &&
 	    !((devtype_data->quirks &
 	       (FLEXCAN_QUIRK_USE_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR |
-		FLEXCAN_QUIRK_SUPPPORT_RX_FIFO)) ==
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR |
+		FLEXCAN_QUIRK_SUPPORT_RX_FIFO)) ==
 	      (FLEXCAN_QUIRK_USE_RX_MAILBOX |
-	       FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-	       FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR))) {
+	       FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+	       FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR))) {
 		dev_err(&pdev->dev, "CAN-FD mode doesn't work in RX-FIFO mode!\n");
 		return -EINVAL;
 	}
 
 	if ((devtype_data->quirks &
-	     (FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-	      FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR)) ==
-	    FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR) {
+	     (FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+	      FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR)) ==
+	    FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR) {
 		dev_err(&pdev->dev,
 			"Quirks (0x%08x) inconsistent: RX_MAILBOX_RX supported but not RX_MAILBOX\n",
 			devtype_data->quirks);
@@ -2177,8 +2177,7 @@ static int flexcan_probe(struct platform_device *pdev)
 
 	err = flexcan_setup_stop_mode(pdev);
 	if (err < 0) {
-		if (err != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "setup stop mode failed\n");
+		dev_err_probe(&pdev->dev, err, "setup stop mode failed\n");
 		goto failed_setup_stop_mode;
 	}
 
diff --git a/drivers/net/can/flexcan/flexcan.h b/drivers/net/can/flexcan/flexcan.h
index 8621a8ea1dea..025c3417031f 100644
--- a/drivers/net/can/flexcan/flexcan.h
+++ b/drivers/net/can/flexcan/flexcan.h
@@ -63,11 +63,11 @@
 /* Setup 16 mailboxes */
 #define FLEXCAN_QUIRK_NR_MB_16 BIT(13)
 /* Device supports RX via mailboxes */
-#define FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX BIT(14)
+#define FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX BIT(14)
 /* Device supports RTR reception via mailboxes */
-#define FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR BIT(15)
+#define FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR BIT(15)
 /* Device supports RX via FIFO */
-#define FLEXCAN_QUIRK_SUPPPORT_RX_FIFO BIT(16)
+#define FLEXCAN_QUIRK_SUPPORT_RX_FIFO BIT(16)
 
 struct flexcan_devtype_data {
 	u32 quirks;		/* quirks needed for different IP cores */
@@ -121,7 +121,7 @@ flexcan_supports_rx_mailbox(const struct flexcan_priv *priv)
 {
 	const u32 quirks = priv->devtype_data.quirks;
 
-	return quirks & FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX;
+	return quirks & FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX;
 }
 
 static inline bool
@@ -129,10 +129,10 @@ flexcan_supports_rx_mailbox_rtr(const struct flexcan_priv *priv)
 {
 	const u32 quirks = priv->devtype_data.quirks;
 
-	return (quirks & (FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-			  FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR)) ==
-		(FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX |
-		 FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR);
+	return (quirks & (FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+			  FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR)) ==
+		(FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX |
+		 FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR);
 }
 
 static inline bool
@@ -140,7 +140,7 @@ flexcan_supports_rx_fifo(const struct flexcan_priv *priv)
 {
 	const u32 quirks = priv->devtype_data.quirks;
 
-	return quirks & FLEXCAN_QUIRK_SUPPPORT_RX_FIFO;
+	return quirks & FLEXCAN_QUIRK_SUPPORT_RX_FIFO;
 }
 
 static inline bool
@@ -149,7 +149,7 @@ flexcan_active_rx_rtr(const struct flexcan_priv *priv)
 	const u32 quirks = priv->devtype_data.quirks;
 
 	if (quirks & FLEXCAN_QUIRK_USE_RX_MAILBOX) {
-		if (quirks & FLEXCAN_QUIRK_SUPPPORT_RX_MAILBOX_RTR)
+		if (quirks & FLEXCAN_QUIRK_SUPPORT_RX_MAILBOX_RTR)
 			return true;
 	} else {
 		/*  RX-FIFO is always RTR capable */
diff --git a/drivers/net/can/ifi_canfd/ifi_canfd.c b/drivers/net/can/ifi_canfd/ifi_canfd.c
index ad7a89b95da7..8d42b7e6661f 100644
--- a/drivers/net/can/ifi_canfd/ifi_canfd.c
+++ b/drivers/net/can/ifi_canfd/ifi_canfd.c
@@ -973,7 +973,7 @@ static int ifi_canfd_plat_probe(struct platform_device *pdev)
 	priv->ndev = ndev;
 	priv->base = addr;
 
-	netif_napi_add(ndev, &priv->napi, ifi_canfd_poll, 64);
+	netif_napi_add(ndev, &priv->napi, ifi_canfd_poll);
 
 	priv->can.state = CAN_STATE_STOPPED;
 
diff --git a/drivers/net/can/kvaser_pciefd.c b/drivers/net/can/kvaser_pciefd.c
index ed54c0b3c7d4..4e9680c8eb34 100644
--- a/drivers/net/can/kvaser_pciefd.c
+++ b/drivers/net/can/kvaser_pciefd.c
@@ -329,12 +329,9 @@ MODULE_DEVICE_TABLE(pci, kvaser_pciefd_id_table);
 static int kvaser_pciefd_spi_wait_loop(struct kvaser_pciefd *pcie, int msk)
 {
 	u32 res;
-	int ret;
-
-	ret = readl_poll_timeout(pcie->reg_base + KVASER_PCIEFD_SPI_STATUS_REG,
-				 res, res & msk, 0, 10);
 
-	return ret;
+	return readl_poll_timeout(pcie->reg_base + KVASER_PCIEFD_SPI_STATUS_REG,
+			res, res & msk, 0, 10);
 }
 
 static int kvaser_pciefd_spi_cmd(struct kvaser_pciefd *pcie, const u8 *tx,
diff --git a/drivers/net/can/m_can/m_can.c b/drivers/net/can/m_can/m_can.c
index 4709c012b1dc..dcb582563d5e 100644
--- a/drivers/net/can/m_can/m_can.c
+++ b/drivers/net/can/m_can/m_can.c
@@ -1467,8 +1467,7 @@ static int m_can_dev_setup(struct m_can_classdev *cdev)
 	}
 
 	if (!cdev->is_peripheral)
-		netif_napi_add(dev, &cdev->napi,
-			       m_can_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(dev, &cdev->napi, m_can_poll);
 
 	/* Shared properties of all M_CAN versions */
 	cdev->version = m_can_version;
diff --git a/drivers/net/can/rcar/rcar_canfd.c b/drivers/net/can/rcar/rcar_canfd.c
index 27085b796e75..567620d215f8 100644
--- a/drivers/net/can/rcar/rcar_canfd.c
+++ b/drivers/net/can/rcar/rcar_canfd.c
@@ -1880,10 +1880,9 @@ static int rcar_canfd_probe(struct platform_device *pdev)
 
 	/* Global controller context */
 	gpriv = devm_kzalloc(&pdev->dev, sizeof(*gpriv), GFP_KERNEL);
-	if (!gpriv) {
-		err = -ENOMEM;
-		goto fail_dev;
-	}
+	if (!gpriv)
+		return -ENOMEM;
+
 	gpriv->pdev = pdev;
 	gpriv->channels_mask = channels_mask;
 	gpriv->fdmode = fdmode;
@@ -1904,12 +1903,9 @@ static int rcar_canfd_probe(struct platform_device *pdev)
 
 	/* Peripheral clock */
 	gpriv->clkp = devm_clk_get(&pdev->dev, "fck");
-	if (IS_ERR(gpriv->clkp)) {
-		err = PTR_ERR(gpriv->clkp);
-		dev_err(&pdev->dev, "cannot get peripheral clock, error %d\n",
-			err);
-		goto fail_dev;
-	}
+	if (IS_ERR(gpriv->clkp))
+		return dev_err_probe(&pdev->dev, PTR_ERR(gpriv->clkp),
+				     "cannot get peripheral clock\n");
 
 	/* fCAN clock: Pick External clock. If not available fallback to
 	 * CANFD clock
@@ -1917,12 +1913,10 @@ static int rcar_canfd_probe(struct platform_device *pdev)
 	gpriv->can_clk = devm_clk_get(&pdev->dev, "can_clk");
 	if (IS_ERR(gpriv->can_clk) || (clk_get_rate(gpriv->can_clk) == 0)) {
 		gpriv->can_clk = devm_clk_get(&pdev->dev, "canfd");
-		if (IS_ERR(gpriv->can_clk)) {
-			err = PTR_ERR(gpriv->can_clk);
-			dev_err(&pdev->dev,
-				"cannot get canfd clock, error %d\n", err);
-			goto fail_dev;
-		}
+		if (IS_ERR(gpriv->can_clk))
+			return dev_err_probe(&pdev->dev, PTR_ERR(gpriv->can_clk),
+					     "cannot get canfd clock\n");
+
 		gpriv->fcan = RCANFD_CANFDCLK;
 
 	} else {
diff --git a/drivers/net/can/sja1000/peak_pcmcia.c b/drivers/net/can/sja1000/peak_pcmcia.c
index 131a084c3535..ebd5941c3f53 100644
--- a/drivers/net/can/sja1000/peak_pcmcia.c
+++ b/drivers/net/can/sja1000/peak_pcmcia.c
@@ -478,7 +478,7 @@ static void pcan_free_channels(struct pcan_pccard *card)
 		if (!netdev)
 			continue;
 
-		strlcpy(name, netdev->name, IFNAMSIZ);
+		strscpy(name, netdev->name, IFNAMSIZ);
 
 		unregister_sja1000dev(netdev);
 
diff --git a/drivers/net/can/sja1000/sja1000.c b/drivers/net/can/sja1000/sja1000.c
index 98dfd5f295a7..1bb1129b0450 100644
--- a/drivers/net/can/sja1000/sja1000.c
+++ b/drivers/net/can/sja1000/sja1000.c
@@ -661,8 +661,6 @@ static const struct ethtool_ops sja1000_ethtool_ops = {
 
 int register_sja1000dev(struct net_device *dev)
 {
-	int ret;
-
 	if (!sja1000_probe_chip(dev))
 		return -ENODEV;
 
@@ -673,9 +671,7 @@ int register_sja1000dev(struct net_device *dev)
 	set_reset_mode(dev);
 	chipset_init(dev);
 
-	ret =  register_candev(dev);
-
-	return ret;
+	return register_candev(dev);
 }
 EXPORT_SYMBOL_GPL(register_sja1000dev);
 
diff --git a/drivers/net/can/sja1000/sja1000_platform.c b/drivers/net/can/sja1000/sja1000_platform.c
index 81bc741905fd..6779d5357069 100644
--- a/drivers/net/can/sja1000/sja1000_platform.c
+++ b/drivers/net/can/sja1000/sja1000_platform.c
@@ -14,6 +14,7 @@
 #include <linux/irq.h>
 #include <linux/can/dev.h>
 #include <linux/can/platform/sja1000.h>
+#include <linux/clk.h>
 #include <linux/io.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
@@ -103,6 +104,11 @@ static void sp_technologic_init(struct sja1000_priv *priv, struct device_node *o
 	spin_lock_init(&tp->io_lock);
 }
 
+static void sp_rzn1_init(struct sja1000_priv *priv, struct device_node *of)
+{
+	priv->flags = SJA1000_QUIRK_NO_CDR_REG;
+}
+
 static void sp_populate(struct sja1000_priv *priv,
 			struct sja1000_platform_data *pdata,
 			unsigned long resource_mem_flags)
@@ -153,11 +159,13 @@ static void sp_populate_of(struct sja1000_priv *priv, struct device_node *of)
 		priv->write_reg = sp_write_reg8;
 	}
 
-	err = of_property_read_u32(of, "nxp,external-clock-frequency", &prop);
-	if (!err)
-		priv->can.clock.freq = prop / 2;
-	else
-		priv->can.clock.freq = SP_CAN_CLOCK; /* default */
+	if (!priv->can.clock.freq) {
+		err = of_property_read_u32(of, "nxp,external-clock-frequency", &prop);
+		if (!err)
+			priv->can.clock.freq = prop / 2;
+		else
+			priv->can.clock.freq = SP_CAN_CLOCK; /* default */
+	}
 
 	err = of_property_read_u32(of, "nxp,tx-output-mode", &prop);
 	if (!err)
@@ -192,8 +200,13 @@ static struct sja1000_of_data technologic_data = {
 	.init = sp_technologic_init,
 };
 
+static struct sja1000_of_data renesas_data = {
+	.init = sp_rzn1_init,
+};
+
 static const struct of_device_id sp_of_table[] = {
 	{ .compatible = "nxp,sja1000", .data = NULL, },
+	{ .compatible = "renesas,rzn1-sja1000", .data = &renesas_data, },
 	{ .compatible = "technologic,sja1000", .data = &technologic_data, },
 	{ /* sentinel */ },
 };
@@ -210,6 +223,7 @@ static int sp_probe(struct platform_device *pdev)
 	struct device_node *of = pdev->dev.of_node;
 	const struct sja1000_of_data *of_data = NULL;
 	size_t priv_sz = 0;
+	struct clk *clk;
 
 	pdata = dev_get_platdata(&pdev->dev);
 	if (!pdata && !of) {
@@ -234,6 +248,11 @@ static int sp_probe(struct platform_device *pdev)
 		irq = platform_get_irq(pdev, 0);
 		if (irq < 0)
 			return irq;
+
+		clk = devm_clk_get_optional_enabled(&pdev->dev, NULL);
+		if (IS_ERR(clk))
+			return dev_err_probe(&pdev->dev, PTR_ERR(clk),
+					     "CAN clk operation failed");
 	} else {
 		res_irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
 		if (!res_irq)
@@ -262,6 +281,15 @@ static int sp_probe(struct platform_device *pdev)
 	priv->reg_base = addr;
 
 	if (of) {
+		if (clk) {
+			priv->can.clock.freq  = clk_get_rate(clk) / 2;
+			if (!priv->can.clock.freq) {
+				err = -EINVAL;
+				dev_err(&pdev->dev, "Zero CAN clk rate");
+				goto exit_free;
+			}
+		}
+
 		sp_populate_of(priv, of);
 
 		if (of_data && of_data->init)
diff --git a/drivers/net/can/usb/etas_es58x/es58x_core.h b/drivers/net/can/usb/etas_es58x/es58x_core.h
index d769bdf740b7..640fe0a1df63 100644
--- a/drivers/net/can/usb/etas_es58x/es58x_core.h
+++ b/drivers/net/can/usb/etas_es58x/es58x_core.h
@@ -222,7 +222,7 @@ union es58x_urb_cmd {
 		u8 cmd_type;
 		u8 cmd_id;
 	} __packed;
-	u8 raw_cmd[0];
+	DECLARE_FLEX_ARRAY(u8, raw_cmd);
 };
 
 /**
diff --git a/drivers/net/can/usb/gs_usb.c b/drivers/net/can/usb/gs_usb.c
index c1ff3c046d62..f0065d40eb24 100644
--- a/drivers/net/can/usb/gs_usb.c
+++ b/drivers/net/can/usb/gs_usb.c
@@ -10,20 +10,24 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/clocksource.h>
 #include <linux/ethtool.h>
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/netdevice.h>
 #include <linux/signal.h>
+#include <linux/timecounter.h>
+#include <linux/units.h>
 #include <linux/usb.h>
+#include <linux/workqueue.h>
 
 #include <linux/can.h>
 #include <linux/can/dev.h>
 #include <linux/can/error.h>
 
 /* Device specific constants */
-#define USB_GSUSB_1_VENDOR_ID 0x1d50
-#define USB_GSUSB_1_PRODUCT_ID 0x606f
+#define USB_GS_USB_1_VENDOR_ID 0x1d50
+#define USB_GS_USB_1_PRODUCT_ID 0x606f
 
 #define USB_CANDLELIGHT_VENDOR_ID 0x1209
 #define USB_CANDLELIGHT_PRODUCT_ID 0x2323
@@ -34,8 +38,16 @@
 #define USB_ABE_CANDEBUGGER_FD_VENDOR_ID 0x16d0
 #define USB_ABE_CANDEBUGGER_FD_PRODUCT_ID 0x10b8
 
-#define GSUSB_ENDPOINT_IN 1
-#define GSUSB_ENDPOINT_OUT 2
+#define GS_USB_ENDPOINT_IN 1
+#define GS_USB_ENDPOINT_OUT 2
+
+/* Timestamp 32 bit timer runs at 1 MHz (1 µs tick). Worker accounts
+ * for timer overflow (will be after ~71 minutes)
+ */
+#define GS_USB_TIMESTAMP_TIMER_HZ (1 * HZ_PER_MHZ)
+#define GS_USB_TIMESTAMP_WORK_DELAY_SEC 1800
+static_assert(GS_USB_TIMESTAMP_WORK_DELAY_SEC <
+	      CYCLECOUNTER_MASK(32) / GS_USB_TIMESTAMP_TIMER_HZ / 2);
 
 /* Device specific constants */
 enum gs_usb_breq {
@@ -52,6 +64,8 @@ enum gs_usb_breq {
 	GS_USB_BREQ_SET_USER_ID,
 	GS_USB_BREQ_DATA_BITTIMING,
 	GS_USB_BREQ_BT_CONST_EXT,
+	GS_USB_BREQ_SET_TERMINATION,
+	GS_USB_BREQ_GET_TERMINATION,
 };
 
 enum gs_can_mode {
@@ -75,6 +89,14 @@ enum gs_can_identify_mode {
 	GS_CAN_IDENTIFY_ON
 };
 
+enum gs_can_termination_state {
+	GS_CAN_TERMINATION_STATE_OFF = 0,
+	GS_CAN_TERMINATION_STATE_ON
+};
+
+#define GS_USB_TERMINATION_DISABLED CAN_TERMINATION_DISABLED
+#define GS_USB_TERMINATION_ENABLED 120
+
 /* data types passed between host and device */
 
 /* The firmware on the original USB2CAN by Geschwister Schneider
@@ -111,6 +133,7 @@ struct gs_device_config {
 #define GS_CAN_MODE_FD BIT(8)
 /* GS_CAN_FEATURE_REQ_USB_QUIRK_LPC546XX BIT(9) */
 /* GS_CAN_FEATURE_BT_CONST_EXT BIT(10) */
+/* GS_CAN_FEATURE_TERMINATION BIT(11) */
 
 struct gs_device_mode {
 	__le32 mode;
@@ -135,6 +158,10 @@ struct gs_identify_mode {
 	__le32 mode;
 } __packed;
 
+struct gs_device_termination_state {
+	__le32 state;
+} __packed;
+
 #define GS_CAN_FEATURE_LISTEN_ONLY BIT(0)
 #define GS_CAN_FEATURE_LOOP_BACK BIT(1)
 #define GS_CAN_FEATURE_TRIPLE_SAMPLE BIT(2)
@@ -146,7 +173,8 @@ struct gs_identify_mode {
 #define GS_CAN_FEATURE_FD BIT(8)
 #define GS_CAN_FEATURE_REQ_USB_QUIRK_LPC546XX BIT(9)
 #define GS_CAN_FEATURE_BT_CONST_EXT BIT(10)
-#define GS_CAN_FEATURE_MASK GENMASK(10, 0)
+#define GS_CAN_FEATURE_TERMINATION BIT(11)
+#define GS_CAN_FEATURE_MASK GENMASK(11, 0)
 
 /* internal quirks - keep in GS_CAN_FEATURE space for now */
 
@@ -199,6 +227,11 @@ struct classic_can {
 	u8 data[8];
 } __packed;
 
+struct classic_can_ts {
+	u8 data[8];
+	__le32 timestamp_us;
+} __packed;
+
 struct classic_can_quirk {
 	u8 data[8];
 	u8 quirk;
@@ -208,6 +241,11 @@ struct canfd {
 	u8 data[64];
 } __packed;
 
+struct canfd_ts {
+	u8 data[64];
+	__le32 timestamp_us;
+} __packed;
+
 struct canfd_quirk {
 	u8 data[64];
 	u8 quirk;
@@ -224,8 +262,10 @@ struct gs_host_frame {
 
 	union {
 		DECLARE_FLEX_ARRAY(struct classic_can, classic_can);
+		DECLARE_FLEX_ARRAY(struct classic_can_ts, classic_can_ts);
 		DECLARE_FLEX_ARRAY(struct classic_can_quirk, classic_can_quirk);
 		DECLARE_FLEX_ARRAY(struct canfd, canfd);
+		DECLARE_FLEX_ARRAY(struct canfd_ts, canfd_ts);
 		DECLARE_FLEX_ARRAY(struct canfd_quirk, canfd_quirk);
 	};
 } __packed;
@@ -259,6 +299,12 @@ struct gs_can {
 	struct can_bittiming_const bt_const, data_bt_const;
 	unsigned int channel;	/* channel number */
 
+	/* time counter for hardware timestamps */
+	struct cyclecounter cc;
+	struct timecounter tc;
+	spinlock_t tc_lock; /* spinlock to guard access tc->cycle_last */
+	struct delayed_work timestamp;
+
 	u32 feature;
 	unsigned int hf_size_tx;
 
@@ -268,8 +314,6 @@ struct gs_can {
 
 	struct usb_anchor tx_submitted;
 	atomic_t active_tx_urbs;
-	void *rxbuf[GS_MAX_RX_URBS];
-	dma_addr_t rxbuf_dma[GS_MAX_RX_URBS];
 };
 
 /* usb interface struct */
@@ -328,27 +372,109 @@ static struct gs_tx_context *gs_get_tx_context(struct gs_can *dev,
 	return NULL;
 }
 
-static int gs_cmd_reset(struct gs_can *gsdev)
+static int gs_cmd_reset(struct gs_can *dev)
+{
+	struct gs_device_mode dm = {
+		.mode = GS_CAN_MODE_RESET,
+	};
+
+	return usb_control_msg_send(interface_to_usbdev(dev->iface), 0,
+				    GS_USB_BREQ_MODE,
+				    USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				    dev->channel, 0, &dm, sizeof(dm), 1000,
+				    GFP_KERNEL);
+}
+
+static inline int gs_usb_get_timestamp(const struct gs_can *dev,
+				       u32 *timestamp_p)
 {
-	struct gs_device_mode *dm;
-	struct usb_interface *intf = gsdev->iface;
+	__le32 timestamp;
 	int rc;
 
-	dm = kzalloc(sizeof(*dm), GFP_KERNEL);
-	if (!dm)
-		return -ENOMEM;
+	rc = usb_control_msg_recv(interface_to_usbdev(dev->iface), 0,
+				  GS_USB_BREQ_TIMESTAMP,
+				  USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				  dev->channel, 0,
+				  &timestamp, sizeof(timestamp),
+				  USB_CTRL_GET_TIMEOUT,
+				  GFP_KERNEL);
+	if (rc)
+		return rc;
+
+	*timestamp_p = le32_to_cpu(timestamp);
 
-	dm->mode = GS_CAN_MODE_RESET;
+	return 0;
+}
 
-	rc = usb_control_msg(interface_to_usbdev(intf),
-			     usb_sndctrlpipe(interface_to_usbdev(intf), 0),
-			     GS_USB_BREQ_MODE,
-			     USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
-			     gsdev->channel, 0, dm, sizeof(*dm), 1000);
+static u64 gs_usb_timestamp_read(const struct cyclecounter *cc) __must_hold(&dev->tc_lock)
+{
+	struct gs_can *dev = container_of(cc, struct gs_can, cc);
+	u32 timestamp = 0;
+	int err;
+
+	lockdep_assert_held(&dev->tc_lock);
+
+	/* drop lock for synchronous USB transfer */
+	spin_unlock_bh(&dev->tc_lock);
+	err = gs_usb_get_timestamp(dev, &timestamp);
+	spin_lock_bh(&dev->tc_lock);
+	if (err)
+		netdev_err(dev->netdev,
+			   "Error %d while reading timestamp. HW timestamps may be inaccurate.",
+			   err);
+
+	return timestamp;
+}
 
-	kfree(dm);
+static void gs_usb_timestamp_work(struct work_struct *work)
+{
+	struct delayed_work *delayed_work = to_delayed_work(work);
+	struct gs_can *dev;
 
-	return rc;
+	dev = container_of(delayed_work, struct gs_can, timestamp);
+	spin_lock_bh(&dev->tc_lock);
+	timecounter_read(&dev->tc);
+	spin_unlock_bh(&dev->tc_lock);
+
+	schedule_delayed_work(&dev->timestamp,
+			      GS_USB_TIMESTAMP_WORK_DELAY_SEC * HZ);
+}
+
+static void gs_usb_skb_set_timestamp(struct gs_can *dev,
+				     struct sk_buff *skb, u32 timestamp)
+{
+	struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
+	u64 ns;
+
+	spin_lock_bh(&dev->tc_lock);
+	ns = timecounter_cyc2time(&dev->tc, timestamp);
+	spin_unlock_bh(&dev->tc_lock);
+
+	hwtstamps->hwtstamp = ns_to_ktime(ns);
+}
+
+static void gs_usb_timestamp_init(struct gs_can *dev)
+{
+	struct cyclecounter *cc = &dev->cc;
+
+	cc->read = gs_usb_timestamp_read;
+	cc->mask = CYCLECOUNTER_MASK(32);
+	cc->shift = 32 - bits_per(NSEC_PER_SEC / GS_USB_TIMESTAMP_TIMER_HZ);
+	cc->mult = clocksource_hz2mult(GS_USB_TIMESTAMP_TIMER_HZ, cc->shift);
+
+	spin_lock_init(&dev->tc_lock);
+	spin_lock_bh(&dev->tc_lock);
+	timecounter_init(&dev->tc, &dev->cc, ktime_get_real_ns());
+	spin_unlock_bh(&dev->tc_lock);
+
+	INIT_DELAYED_WORK(&dev->timestamp, gs_usb_timestamp_work);
+	schedule_delayed_work(&dev->timestamp,
+			      GS_USB_TIMESTAMP_WORK_DELAY_SEC * HZ);
+}
+
+static void gs_usb_timestamp_stop(struct gs_can *dev)
+{
+	cancel_delayed_work_sync(&dev->timestamp);
 }
 
 static void gs_update_state(struct gs_can *dev, struct can_frame *cf)
@@ -376,6 +502,24 @@ static void gs_update_state(struct gs_can *dev, struct can_frame *cf)
 	}
 }
 
+static void gs_usb_set_timestamp(struct gs_can *dev, struct sk_buff *skb,
+				 const struct gs_host_frame *hf)
+{
+	u32 timestamp;
+
+	if (!(dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP))
+		return;
+
+	if (hf->flags & GS_CAN_FLAG_FD)
+		timestamp = le32_to_cpu(hf->canfd_ts->timestamp_us);
+	else
+		timestamp = le32_to_cpu(hf->classic_can_ts->timestamp_us);
+
+	gs_usb_skb_set_timestamp(dev, skb, timestamp);
+
+	return;
+}
+
 static void gs_usb_receive_bulk_callback(struct urb *urb)
 {
 	struct gs_usb *usbcan = urb->context;
@@ -443,6 +587,8 @@ static void gs_usb_receive_bulk_callback(struct urb *urb)
 				gs_update_state(dev, cf);
 		}
 
+		gs_usb_set_timestamp(dev, skb, hf);
+
 		netdev->stats.rx_packets++;
 		netdev->stats.rx_bytes += hf->can_dlc;
 
@@ -465,6 +611,9 @@ static void gs_usb_receive_bulk_callback(struct urb *urb)
 			goto resubmit_urb;
 		}
 
+		skb = dev->can.echo_skb[hf->echo_id];
+		gs_usb_set_timestamp(dev, skb, hf);
+
 		netdev->stats.tx_packets++;
 		netdev->stats.tx_bytes += can_get_echo_skb(netdev, hf->echo_id,
 							   NULL);
@@ -491,7 +640,7 @@ static void gs_usb_receive_bulk_callback(struct urb *urb)
 
  resubmit_urb:
 	usb_fill_bulk_urb(urb, usbcan->udev,
-			  usb_rcvbulkpipe(usbcan->udev, GSUSB_ENDPOINT_IN),
+			  usb_rcvbulkpipe(usbcan->udev, GS_USB_ENDPOINT_IN),
 			  hf, dev->parent->hf_size_rx,
 			  gs_usb_receive_bulk_callback, usbcan);
 
@@ -511,72 +660,44 @@ static int gs_usb_set_bittiming(struct net_device *netdev)
 {
 	struct gs_can *dev = netdev_priv(netdev);
 	struct can_bittiming *bt = &dev->can.bittiming;
-	struct usb_interface *intf = dev->iface;
-	int rc;
-	struct gs_device_bittiming *dbt;
-
-	dbt = kmalloc(sizeof(*dbt), GFP_KERNEL);
-	if (!dbt)
-		return -ENOMEM;
-
-	dbt->prop_seg = cpu_to_le32(bt->prop_seg);
-	dbt->phase_seg1 = cpu_to_le32(bt->phase_seg1);
-	dbt->phase_seg2 = cpu_to_le32(bt->phase_seg2);
-	dbt->sjw = cpu_to_le32(bt->sjw);
-	dbt->brp = cpu_to_le32(bt->brp);
+	struct gs_device_bittiming dbt = {
+		.prop_seg = cpu_to_le32(bt->prop_seg),
+		.phase_seg1 = cpu_to_le32(bt->phase_seg1),
+		.phase_seg2 = cpu_to_le32(bt->phase_seg2),
+		.sjw = cpu_to_le32(bt->sjw),
+		.brp = cpu_to_le32(bt->brp),
+	};
 
 	/* request bit timings */
-	rc = usb_control_msg(interface_to_usbdev(intf),
-			     usb_sndctrlpipe(interface_to_usbdev(intf), 0),
-			     GS_USB_BREQ_BITTIMING,
-			     USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
-			     dev->channel, 0, dbt, sizeof(*dbt), 1000);
-
-	kfree(dbt);
-
-	if (rc < 0)
-		dev_err(netdev->dev.parent, "Couldn't set bittimings (err=%d)",
-			rc);
-
-	return (rc > 0) ? 0 : rc;
+	return usb_control_msg_send(interface_to_usbdev(dev->iface), 0,
+				    GS_USB_BREQ_BITTIMING,
+				    USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				    dev->channel, 0, &dbt, sizeof(dbt), 1000,
+				    GFP_KERNEL);
 }
 
 static int gs_usb_set_data_bittiming(struct net_device *netdev)
 {
 	struct gs_can *dev = netdev_priv(netdev);
 	struct can_bittiming *bt = &dev->can.data_bittiming;
-	struct usb_interface *intf = dev->iface;
-	struct gs_device_bittiming *dbt;
+	struct gs_device_bittiming dbt = {
+		.prop_seg = cpu_to_le32(bt->prop_seg),
+		.phase_seg1 = cpu_to_le32(bt->phase_seg1),
+		.phase_seg2 = cpu_to_le32(bt->phase_seg2),
+		.sjw = cpu_to_le32(bt->sjw),
+		.brp = cpu_to_le32(bt->brp),
+	};
 	u8 request = GS_USB_BREQ_DATA_BITTIMING;
-	int rc;
-
-	dbt = kmalloc(sizeof(*dbt), GFP_KERNEL);
-	if (!dbt)
-		return -ENOMEM;
-
-	dbt->prop_seg = cpu_to_le32(bt->prop_seg);
-	dbt->phase_seg1 = cpu_to_le32(bt->phase_seg1);
-	dbt->phase_seg2 = cpu_to_le32(bt->phase_seg2);
-	dbt->sjw = cpu_to_le32(bt->sjw);
-	dbt->brp = cpu_to_le32(bt->brp);
 
 	if (dev->feature & GS_CAN_FEATURE_QUIRK_BREQ_CANTACT_PRO)
 		request = GS_USB_BREQ_QUIRK_CANTACT_PRO_DATA_BITTIMING;
 
-	/* request bit timings */
-	rc = usb_control_msg(interface_to_usbdev(intf),
-			     usb_sndctrlpipe(interface_to_usbdev(intf), 0),
-			     request,
-			     USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
-			     dev->channel, 0, dbt, sizeof(*dbt), 1000);
-
-	kfree(dbt);
-
-	if (rc < 0)
-		dev_err(netdev->dev.parent,
-			"Couldn't set data bittimings (err=%d)", rc);
-
-	return (rc > 0) ? 0 : rc;
+	/* request data bit timings */
+	return usb_control_msg_send(interface_to_usbdev(dev->iface), 0,
+				    request,
+				    USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				    dev->channel, 0, &dbt, sizeof(dbt), 1000,
+				    GFP_KERNEL);
 }
 
 static void gs_usb_xmit_callback(struct urb *urb)
@@ -587,9 +708,6 @@ static void gs_usb_xmit_callback(struct urb *urb)
 
 	if (urb->status)
 		netdev_info(netdev, "usb xmit fail %u\n", txc->echo_id);
-
-	usb_free_coherent(urb->dev, urb->transfer_buffer_length,
-			  urb->transfer_buffer, urb->transfer_dma);
 }
 
 static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb,
@@ -618,8 +736,7 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb,
 	if (!urb)
 		goto nomem_urb;
 
-	hf = usb_alloc_coherent(dev->udev, dev->hf_size_tx, GFP_ATOMIC,
-				&urb->transfer_dma);
+	hf = kmalloc(dev->hf_size_tx, GFP_ATOMIC);
 	if (!hf) {
 		netdev_err(netdev, "No memory left for USB buffer\n");
 		goto nomem_hf;
@@ -659,11 +776,11 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb,
 	}
 
 	usb_fill_bulk_urb(urb, dev->udev,
-			  usb_sndbulkpipe(dev->udev, GSUSB_ENDPOINT_OUT),
+			  usb_sndbulkpipe(dev->udev, GS_USB_ENDPOINT_OUT),
 			  hf, dev->hf_size_tx,
 			  gs_usb_xmit_callback, txc);
 
-	urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
+	urb->transfer_flags |= URB_FREE_BUFFER;
 	usb_anchor_urb(urb, &dev->tx_submitted);
 
 	can_put_echo_skb(skb, netdev, idx, 0);
@@ -678,8 +795,6 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb,
 		gs_free_tx_context(txc);
 
 		usb_unanchor_urb(urb);
-		usb_free_coherent(dev->udev, urb->transfer_buffer_length,
-				  urb->transfer_buffer, urb->transfer_dma);
 
 		if (rc == -ENODEV) {
 			netif_device_detach(netdev);
@@ -699,8 +814,7 @@ static netdev_tx_t gs_can_start_xmit(struct sk_buff *skb,
 	return NETDEV_TX_OK;
 
  badidx:
-	usb_free_coherent(dev->udev, urb->transfer_buffer_length,
-			  urb->transfer_buffer, urb->transfer_dma);
+	kfree(hf);
  nomem_hf:
 	usb_free_urb(urb);
 
@@ -715,11 +829,13 @@ static int gs_can_open(struct net_device *netdev)
 {
 	struct gs_can *dev = netdev_priv(netdev);
 	struct gs_usb *parent = dev->parent;
-	int rc, i;
-	struct gs_device_mode *dm;
+	struct gs_device_mode dm = {
+		.mode = cpu_to_le32(GS_CAN_MODE_START),
+	};
 	struct gs_host_frame *hf;
 	u32 ctrlmode;
 	u32 flags = 0;
+	int rc, i;
 
 	rc = open_candev(netdev);
 	if (rc)
@@ -744,7 +860,6 @@ static int gs_can_open(struct net_device *netdev)
 		for (i = 0; i < GS_MAX_RX_URBS; i++) {
 			struct urb *urb;
 			u8 *buf;
-			dma_addr_t buf_dma;
 
 			/* alloc rx urb */
 			urb = usb_alloc_urb(0, GFP_KERNEL);
@@ -752,10 +867,8 @@ static int gs_can_open(struct net_device *netdev)
 				return -ENOMEM;
 
 			/* alloc rx buffer */
-			buf = usb_alloc_coherent(dev->udev,
-						 dev->parent->hf_size_rx,
-						 GFP_KERNEL,
-						 &buf_dma);
+			buf = kmalloc(dev->parent->hf_size_rx,
+				      GFP_KERNEL);
 			if (!buf) {
 				netdev_err(netdev,
 					   "No memory left for USB buffer\n");
@@ -763,17 +876,15 @@ static int gs_can_open(struct net_device *netdev)
 				return -ENOMEM;
 			}
 
-			urb->transfer_dma = buf_dma;
-
 			/* fill, anchor, and submit rx urb */
 			usb_fill_bulk_urb(urb,
 					  dev->udev,
 					  usb_rcvbulkpipe(dev->udev,
-							  GSUSB_ENDPOINT_IN),
+							  GS_USB_ENDPOINT_IN),
 					  buf,
 					  dev->parent->hf_size_rx,
 					  gs_usb_receive_bulk_callback, parent);
-			urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
+			urb->transfer_flags |= URB_FREE_BUFFER;
 
 			usb_anchor_urb(urb, &parent->rx_submitted);
 
@@ -786,17 +897,10 @@ static int gs_can_open(struct net_device *netdev)
 					   "usb_submit failed (err=%d)\n", rc);
 
 				usb_unanchor_urb(urb);
-				usb_free_coherent(dev->udev,
-						  sizeof(struct gs_host_frame),
-						  buf,
-						  buf_dma);
 				usb_free_urb(urb);
 				break;
 			}
 
-			dev->rxbuf[i] = buf;
-			dev->rxbuf_dma[i] = buf_dma;
-
 			/* Drop reference,
 			 * USB core will take care of freeing it
 			 */
@@ -804,10 +908,6 @@ static int gs_can_open(struct net_device *netdev)
 		}
 	}
 
-	dm = kmalloc(sizeof(*dm), GFP_KERNEL);
-	if (!dm)
-		return -ENOMEM;
-
 	/* flags */
 	if (ctrlmode & CAN_CTRLMODE_LOOPBACK)
 		flags |= GS_CAN_MODE_LOOP_BACK;
@@ -823,25 +923,30 @@ static int gs_can_open(struct net_device *netdev)
 	if (ctrlmode & CAN_CTRLMODE_3_SAMPLES)
 		flags |= GS_CAN_MODE_TRIPLE_SAMPLE;
 
+	/* if hardware supports timestamps, enable it */
+	if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
+		flags |= GS_CAN_MODE_HW_TIMESTAMP;
+
+	/* start polling timestamp */
+	if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
+		gs_usb_timestamp_init(dev);
+
 	/* finally start device */
 	dev->can.state = CAN_STATE_ERROR_ACTIVE;
-	dm->mode = cpu_to_le32(GS_CAN_MODE_START);
-	dm->flags = cpu_to_le32(flags);
-	rc = usb_control_msg(interface_to_usbdev(dev->iface),
-			     usb_sndctrlpipe(interface_to_usbdev(dev->iface), 0),
-			     GS_USB_BREQ_MODE,
-			     USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
-			     dev->channel, 0, dm, sizeof(*dm), 1000);
-
-	if (rc < 0) {
+	dm.flags = cpu_to_le32(flags);
+	rc = usb_control_msg_send(interface_to_usbdev(dev->iface), 0,
+				  GS_USB_BREQ_MODE,
+				  USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				  dev->channel, 0, &dm, sizeof(dm), 1000,
+				  GFP_KERNEL);
+	if (rc) {
 		netdev_err(netdev, "Couldn't start device (err=%d)\n", rc);
-		kfree(dm);
+		if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
+			gs_usb_timestamp_stop(dev);
 		dev->can.state = CAN_STATE_STOPPED;
 		return rc;
 	}
 
-	kfree(dm);
-
 	parent->active_channels++;
 	if (!(dev->can.ctrlmode & CAN_CTRLMODE_LISTENONLY))
 		netif_start_queue(netdev);
@@ -854,19 +959,17 @@ static int gs_can_close(struct net_device *netdev)
 	int rc;
 	struct gs_can *dev = netdev_priv(netdev);
 	struct gs_usb *parent = dev->parent;
-	unsigned int i;
 
 	netif_stop_queue(netdev);
 
+	/* stop polling timestamp */
+	if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
+		gs_usb_timestamp_stop(dev);
+
 	/* Stop polling */
 	parent->active_channels--;
 	if (!parent->active_channels) {
 		usb_kill_anchored_urbs(&parent->rx_submitted);
-		for (i = 0; i < GS_MAX_RX_URBS; i++)
-			usb_free_coherent(dev->udev,
-					  sizeof(struct gs_host_frame),
-					  dev->rxbuf[i],
-					  dev->rxbuf_dma[i]);
 	}
 
 	/* Stop sending URBs */
@@ -890,38 +993,39 @@ static int gs_can_close(struct net_device *netdev)
 	return 0;
 }
 
+static int gs_can_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
+{
+	const struct gs_can *dev = netdev_priv(netdev);
+
+	if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
+		return can_eth_ioctl_hwts(netdev, ifr, cmd);
+
+	return -EOPNOTSUPP;
+}
+
 static const struct net_device_ops gs_usb_netdev_ops = {
 	.ndo_open = gs_can_open,
 	.ndo_stop = gs_can_close,
 	.ndo_start_xmit = gs_can_start_xmit,
 	.ndo_change_mtu = can_change_mtu,
+	.ndo_eth_ioctl = gs_can_eth_ioctl,
 };
 
 static int gs_usb_set_identify(struct net_device *netdev, bool do_identify)
 {
 	struct gs_can *dev = netdev_priv(netdev);
-	struct gs_identify_mode *imode;
-	int rc;
-
-	imode = kmalloc(sizeof(*imode), GFP_KERNEL);
-
-	if (!imode)
-		return -ENOMEM;
+	struct gs_identify_mode imode;
 
 	if (do_identify)
-		imode->mode = cpu_to_le32(GS_CAN_IDENTIFY_ON);
+		imode.mode = cpu_to_le32(GS_CAN_IDENTIFY_ON);
 	else
-		imode->mode = cpu_to_le32(GS_CAN_IDENTIFY_OFF);
-
-	rc = usb_control_msg(interface_to_usbdev(dev->iface),
-			     usb_sndctrlpipe(interface_to_usbdev(dev->iface), 0),
-			     GS_USB_BREQ_IDENTIFY,
-			     USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
-			     dev->channel, 0, imode, sizeof(*imode), 100);
+		imode.mode = cpu_to_le32(GS_CAN_IDENTIFY_OFF);
 
-	kfree(imode);
-
-	return (rc > 0) ? 0 : rc;
+	return usb_control_msg_send(interface_to_usbdev(dev->iface), 0,
+				    GS_USB_BREQ_IDENTIFY,
+				    USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				    dev->channel, 0, &imode, sizeof(imode), 100,
+				    GFP_KERNEL);
 }
 
 /* blink LED's for finding the this interface */
@@ -948,9 +1052,67 @@ static int gs_usb_set_phys_id(struct net_device *netdev,
 	return rc;
 }
 
+static int gs_usb_get_ts_info(struct net_device *netdev,
+			      struct ethtool_ts_info *info)
+{
+	struct gs_can *dev = netdev_priv(netdev);
+
+	/* report if device supports HW timestamps */
+	if (dev->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
+		return can_ethtool_op_get_ts_info_hwts(netdev, info);
+
+	return ethtool_op_get_ts_info(netdev, info);
+}
+
 static const struct ethtool_ops gs_usb_ethtool_ops = {
 	.set_phys_id = gs_usb_set_phys_id,
-	.get_ts_info = ethtool_op_get_ts_info,
+	.get_ts_info = gs_usb_get_ts_info,
+};
+
+static int gs_usb_get_termination(struct net_device *netdev, u16 *term)
+{
+	struct gs_can *dev = netdev_priv(netdev);
+	struct gs_device_termination_state term_state;
+	int rc;
+
+	rc = usb_control_msg_recv(interface_to_usbdev(dev->iface), 0,
+				  GS_USB_BREQ_GET_TERMINATION,
+				  USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				  dev->channel, 0,
+				  &term_state, sizeof(term_state), 1000,
+				  GFP_KERNEL);
+	if (rc)
+		return rc;
+
+	if (term_state.state == cpu_to_le32(GS_CAN_TERMINATION_STATE_ON))
+		*term = GS_USB_TERMINATION_ENABLED;
+	else
+		*term = GS_USB_TERMINATION_DISABLED;
+
+	return 0;
+}
+
+static int gs_usb_set_termination(struct net_device *netdev, u16 term)
+{
+	struct gs_can *dev = netdev_priv(netdev);
+	struct gs_device_termination_state term_state;
+
+	if (term == GS_USB_TERMINATION_ENABLED)
+		term_state.state = cpu_to_le32(GS_CAN_TERMINATION_STATE_ON);
+	else
+		term_state.state = cpu_to_le32(GS_CAN_TERMINATION_STATE_OFF);
+
+	return usb_control_msg_send(interface_to_usbdev(dev->iface), 0,
+				    GS_USB_BREQ_SET_TERMINATION,
+				    USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				    dev->channel, 0,
+				    &term_state, sizeof(term_state), 1000,
+				    GFP_KERNEL);
+}
+
+static const u16 gs_usb_termination_const[] = {
+	GS_USB_TERMINATION_DISABLED,
+	GS_USB_TERMINATION_ENABLED
 };
 
 static struct gs_can *gs_make_candev(unsigned int channel,
@@ -960,26 +1122,21 @@ static struct gs_can *gs_make_candev(unsigned int channel,
 	struct gs_can *dev;
 	struct net_device *netdev;
 	int rc;
-	struct gs_device_bt_const *bt_const;
-	struct gs_device_bt_const_extended *bt_const_extended;
+	struct gs_device_bt_const_extended bt_const_extended;
+	struct gs_device_bt_const bt_const;
 	u32 feature;
 
-	bt_const = kmalloc(sizeof(*bt_const), GFP_KERNEL);
-	if (!bt_const)
-		return ERR_PTR(-ENOMEM);
-
 	/* fetch bit timing constants */
-	rc = usb_control_msg(interface_to_usbdev(intf),
-			     usb_rcvctrlpipe(interface_to_usbdev(intf), 0),
-			     GS_USB_BREQ_BT_CONST,
-			     USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
-			     channel, 0, bt_const, sizeof(*bt_const), 1000);
+	rc = usb_control_msg_recv(interface_to_usbdev(intf), 0,
+				  GS_USB_BREQ_BT_CONST,
+				  USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				  channel, 0, &bt_const, sizeof(bt_const), 1000,
+				  GFP_KERNEL);
 
-	if (rc < 0) {
+	if (rc) {
 		dev_err(&intf->dev,
-			"Couldn't get bit timing const for channel (err=%d)\n",
-			rc);
-		kfree(bt_const);
+			"Couldn't get bit timing const for channel %d (%pe)\n",
+			channel, ERR_PTR(rc));
 		return ERR_PTR(rc);
 	}
 
@@ -987,7 +1144,6 @@ static struct gs_can *gs_make_candev(unsigned int channel,
 	netdev = alloc_candev(sizeof(struct gs_can), GS_MAX_TX_URBS);
 	if (!netdev) {
 		dev_err(&intf->dev, "Couldn't allocate candev\n");
-		kfree(bt_const);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -1000,14 +1156,14 @@ static struct gs_can *gs_make_candev(unsigned int channel,
 
 	/* dev setup */
 	strcpy(dev->bt_const.name, KBUILD_MODNAME);
-	dev->bt_const.tseg1_min = le32_to_cpu(bt_const->tseg1_min);
-	dev->bt_const.tseg1_max = le32_to_cpu(bt_const->tseg1_max);
-	dev->bt_const.tseg2_min = le32_to_cpu(bt_const->tseg2_min);
-	dev->bt_const.tseg2_max = le32_to_cpu(bt_const->tseg2_max);
-	dev->bt_const.sjw_max = le32_to_cpu(bt_const->sjw_max);
-	dev->bt_const.brp_min = le32_to_cpu(bt_const->brp_min);
-	dev->bt_const.brp_max = le32_to_cpu(bt_const->brp_max);
-	dev->bt_const.brp_inc = le32_to_cpu(bt_const->brp_inc);
+	dev->bt_const.tseg1_min = le32_to_cpu(bt_const.tseg1_min);
+	dev->bt_const.tseg1_max = le32_to_cpu(bt_const.tseg1_max);
+	dev->bt_const.tseg2_min = le32_to_cpu(bt_const.tseg2_min);
+	dev->bt_const.tseg2_max = le32_to_cpu(bt_const.tseg2_max);
+	dev->bt_const.sjw_max = le32_to_cpu(bt_const.sjw_max);
+	dev->bt_const.brp_min = le32_to_cpu(bt_const.brp_min);
+	dev->bt_const.brp_max = le32_to_cpu(bt_const.brp_max);
+	dev->bt_const.brp_inc = le32_to_cpu(bt_const.brp_inc);
 
 	dev->udev = interface_to_usbdev(intf);
 	dev->iface = intf;
@@ -1024,13 +1180,13 @@ static struct gs_can *gs_make_candev(unsigned int channel,
 
 	/* can setup */
 	dev->can.state = CAN_STATE_STOPPED;
-	dev->can.clock.freq = le32_to_cpu(bt_const->fclk_can);
+	dev->can.clock.freq = le32_to_cpu(bt_const.fclk_can);
 	dev->can.bittiming_const = &dev->bt_const;
 	dev->can.do_set_bittiming = gs_usb_set_bittiming;
 
 	dev->can.ctrlmode_supported = CAN_CTRLMODE_CC_LEN8_DLC;
 
-	feature = le32_to_cpu(bt_const->feature);
+	feature = le32_to_cpu(bt_const.feature);
 	dev->feature = FIELD_GET(GS_CAN_FEATURE_MASK, feature);
 	if (feature & GS_CAN_FEATURE_LISTEN_ONLY)
 		dev->can.ctrlmode_supported |= CAN_CTRLMODE_LISTENONLY;
@@ -1053,6 +1209,21 @@ static struct gs_can *gs_make_candev(unsigned int channel,
 		dev->can.do_set_data_bittiming = gs_usb_set_data_bittiming;
 	}
 
+	if (feature & GS_CAN_FEATURE_TERMINATION) {
+		rc = gs_usb_get_termination(netdev, &dev->can.termination);
+		if (rc) {
+			dev->feature &= ~GS_CAN_FEATURE_TERMINATION;
+
+			dev_info(&intf->dev,
+				 "Disabling termination support for channel %d (%pe)\n",
+				 channel, ERR_PTR(rc));
+		} else {
+			dev->can.termination_const = gs_usb_termination_const;
+			dev->can.termination_const_cnt = ARRAY_SIZE(gs_usb_termination_const);
+			dev->can.do_set_termination = gs_usb_set_termination;
+		}
+	}
+
 	/* The CANtact Pro from LinkLayer Labs is based on the
 	 * LPC54616 µC, which is affected by the NXP LPC USB transfer
 	 * erratum. However, the current firmware (version 2) doesn't
@@ -1067,8 +1238,8 @@ static struct gs_can *gs_make_candev(unsigned int channel,
 	 * GS_CAN_FEATURE_QUIRK_BREQ_CANTACT_PRO to workaround this
 	 * issue.
 	 */
-	if (dev->udev->descriptor.idVendor == cpu_to_le16(USB_GSUSB_1_VENDOR_ID) &&
-	    dev->udev->descriptor.idProduct == cpu_to_le16(USB_GSUSB_1_PRODUCT_ID) &&
+	if (dev->udev->descriptor.idVendor == cpu_to_le16(USB_GS_USB_1_VENDOR_ID) &&
+	    dev->udev->descriptor.idProduct == cpu_to_le16(USB_GS_USB_1_PRODUCT_ID) &&
 	    dev->udev->manufacturer && dev->udev->product &&
 	    !strcmp(dev->udev->manufacturer, "LinkLayer Labs") &&
 	    !strcmp(dev->udev->product, "CANtact Pro") &&
@@ -1081,57 +1252,52 @@ static struct gs_can *gs_make_candev(unsigned int channel,
 	      feature & GS_CAN_FEATURE_IDENTIFY))
 		dev->feature &= ~GS_CAN_FEATURE_IDENTIFY;
 
-	kfree(bt_const);
-
 	/* fetch extended bit timing constants if device has feature
 	 * GS_CAN_FEATURE_FD and GS_CAN_FEATURE_BT_CONST_EXT
 	 */
 	if (feature & GS_CAN_FEATURE_FD &&
 	    feature & GS_CAN_FEATURE_BT_CONST_EXT) {
-		bt_const_extended = kmalloc(sizeof(*bt_const_extended), GFP_KERNEL);
-		if (!bt_const_extended)
-			return ERR_PTR(-ENOMEM);
-
-		rc = usb_control_msg(interface_to_usbdev(intf),
-				     usb_rcvctrlpipe(interface_to_usbdev(intf), 0),
-				     GS_USB_BREQ_BT_CONST_EXT,
-				     USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
-				     channel, 0, bt_const_extended,
-				     sizeof(*bt_const_extended),
-				     1000);
-		if (rc < 0) {
+		rc = usb_control_msg_recv(interface_to_usbdev(intf), 0,
+					  GS_USB_BREQ_BT_CONST_EXT,
+					  USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+					  channel, 0, &bt_const_extended,
+					  sizeof(bt_const_extended),
+					  1000, GFP_KERNEL);
+		if (rc) {
 			dev_err(&intf->dev,
-				"Couldn't get extended bit timing const for channel (err=%d)\n",
-				rc);
-			kfree(bt_const_extended);
-			return ERR_PTR(rc);
+				"Couldn't get extended bit timing const for channel %d (%pe)\n",
+				channel, ERR_PTR(rc));
+			goto out_free_candev;
 		}
 
 		strcpy(dev->data_bt_const.name, KBUILD_MODNAME);
-		dev->data_bt_const.tseg1_min = le32_to_cpu(bt_const_extended->dtseg1_min);
-		dev->data_bt_const.tseg1_max = le32_to_cpu(bt_const_extended->dtseg1_max);
-		dev->data_bt_const.tseg2_min = le32_to_cpu(bt_const_extended->dtseg2_min);
-		dev->data_bt_const.tseg2_max = le32_to_cpu(bt_const_extended->dtseg2_max);
-		dev->data_bt_const.sjw_max = le32_to_cpu(bt_const_extended->dsjw_max);
-		dev->data_bt_const.brp_min = le32_to_cpu(bt_const_extended->dbrp_min);
-		dev->data_bt_const.brp_max = le32_to_cpu(bt_const_extended->dbrp_max);
-		dev->data_bt_const.brp_inc = le32_to_cpu(bt_const_extended->dbrp_inc);
+		dev->data_bt_const.tseg1_min = le32_to_cpu(bt_const_extended.dtseg1_min);
+		dev->data_bt_const.tseg1_max = le32_to_cpu(bt_const_extended.dtseg1_max);
+		dev->data_bt_const.tseg2_min = le32_to_cpu(bt_const_extended.dtseg2_min);
+		dev->data_bt_const.tseg2_max = le32_to_cpu(bt_const_extended.dtseg2_max);
+		dev->data_bt_const.sjw_max = le32_to_cpu(bt_const_extended.dsjw_max);
+		dev->data_bt_const.brp_min = le32_to_cpu(bt_const_extended.dbrp_min);
+		dev->data_bt_const.brp_max = le32_to_cpu(bt_const_extended.dbrp_max);
+		dev->data_bt_const.brp_inc = le32_to_cpu(bt_const_extended.dbrp_inc);
 
 		dev->can.data_bittiming_const = &dev->data_bt_const;
-
-		kfree(bt_const_extended);
 	}
 
 	SET_NETDEV_DEV(netdev, &intf->dev);
 
 	rc = register_candev(dev->netdev);
 	if (rc) {
-		free_candev(dev->netdev);
-		dev_err(&intf->dev, "Couldn't register candev (err=%d)\n", rc);
-		return ERR_PTR(rc);
+		dev_err(&intf->dev,
+			"Couldn't register candev for channel %d (%pe)\n",
+			channel, ERR_PTR(rc));
+		goto out_free_candev;
 	}
 
 	return dev;
+
+ out_free_candev:
+	free_candev(dev->netdev);
+	return ERR_PTR(rc);
 }
 
 static void gs_destroy_candev(struct gs_can *dev)
@@ -1147,76 +1313,61 @@ static int gs_usb_probe(struct usb_interface *intf,
 	struct usb_device *udev = interface_to_usbdev(intf);
 	struct gs_host_frame *hf;
 	struct gs_usb *dev;
-	int rc = -ENOMEM;
+	struct gs_host_config hconf = {
+		.byte_order = cpu_to_le32(0x0000beef),
+	};
+	struct gs_device_config dconf;
 	unsigned int icount, i;
-	struct gs_host_config *hconf;
-	struct gs_device_config *dconf;
-
-	hconf = kmalloc(sizeof(*hconf), GFP_KERNEL);
-	if (!hconf)
-		return -ENOMEM;
-
-	hconf->byte_order = cpu_to_le32(0x0000beef);
+	int rc;
 
 	/* send host config */
-	rc = usb_control_msg(udev, usb_sndctrlpipe(udev, 0),
-			     GS_USB_BREQ_HOST_FORMAT,
-			     USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
-			     1, intf->cur_altsetting->desc.bInterfaceNumber,
-			     hconf, sizeof(*hconf), 1000);
-
-	kfree(hconf);
-
-	if (rc < 0) {
+	rc = usb_control_msg_send(udev, 0,
+				  GS_USB_BREQ_HOST_FORMAT,
+				  USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				  1, intf->cur_altsetting->desc.bInterfaceNumber,
+				  &hconf, sizeof(hconf), 1000,
+				  GFP_KERNEL);
+	if (rc) {
 		dev_err(&intf->dev, "Couldn't send data format (err=%d)\n", rc);
 		return rc;
 	}
 
-	dconf = kmalloc(sizeof(*dconf), GFP_KERNEL);
-	if (!dconf)
-		return -ENOMEM;
-
 	/* read device config */
-	rc = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0),
-			     GS_USB_BREQ_DEVICE_CONFIG,
-			     USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
-			     1, intf->cur_altsetting->desc.bInterfaceNumber,
-			     dconf, sizeof(*dconf), 1000);
-	if (rc < 0) {
+	rc = usb_control_msg_recv(udev, 0,
+				  GS_USB_BREQ_DEVICE_CONFIG,
+				  USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_INTERFACE,
+				  1, intf->cur_altsetting->desc.bInterfaceNumber,
+				  &dconf, sizeof(dconf), 1000,
+				  GFP_KERNEL);
+	if (rc) {
 		dev_err(&intf->dev, "Couldn't get device config: (err=%d)\n",
 			rc);
-		kfree(dconf);
 		return rc;
 	}
 
-	icount = dconf->icount + 1;
+	icount = dconf.icount + 1;
 	dev_info(&intf->dev, "Configuring for %u interfaces\n", icount);
 
 	if (icount > GS_MAX_INTF) {
 		dev_err(&intf->dev,
 			"Driver cannot handle more that %u CAN interfaces\n",
 			GS_MAX_INTF);
-		kfree(dconf);
 		return -EINVAL;
 	}
 
 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-	if (!dev) {
-		kfree(dconf);
+	if (!dev)
 		return -ENOMEM;
-	}
 
 	init_usb_anchor(&dev->rx_submitted);
-	/* default to classic CAN, switch to CAN-FD if at least one of
-	 * our channels support CAN-FD.
-	 */
-	dev->hf_size_rx = struct_size(hf, classic_can, 1);
 
 	usb_set_intfdata(intf, dev);
 	dev->udev = udev;
 
 	for (i = 0; i < icount; i++) {
-		dev->canch[i] = gs_make_candev(i, intf, dconf);
+		unsigned int hf_size_rx = 0;
+
+		dev->canch[i] = gs_make_candev(i, intf, &dconf);
 		if (IS_ERR_OR_NULL(dev->canch[i])) {
 			/* save error code to return later */
 			rc = PTR_ERR(dev->canch[i]);
@@ -1227,18 +1378,28 @@ static int gs_usb_probe(struct usb_interface *intf,
 				gs_destroy_candev(dev->canch[i]);
 
 			usb_kill_anchored_urbs(&dev->rx_submitted);
-			kfree(dconf);
 			kfree(dev);
 			return rc;
 		}
 		dev->canch[i]->parent = dev;
 
-		if (dev->canch[i]->can.ctrlmode_supported & CAN_CTRLMODE_FD)
-			dev->hf_size_rx = struct_size(hf, canfd, 1);
+		/* set RX packet size based on FD and if hardware
+                * timestamps are supported.
+		*/
+		if (dev->canch[i]->can.ctrlmode_supported & CAN_CTRLMODE_FD) {
+			if (dev->canch[i]->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
+				hf_size_rx = struct_size(hf, canfd_ts, 1);
+			else
+				hf_size_rx = struct_size(hf, canfd, 1);
+		} else {
+			if (dev->canch[i]->feature & GS_CAN_FEATURE_HW_TIMESTAMP)
+				hf_size_rx = struct_size(hf, classic_can_ts, 1);
+			else
+				hf_size_rx = struct_size(hf, classic_can, 1);
+		}
+		dev->hf_size_rx = max(dev->hf_size_rx, hf_size_rx);
 	}
 
-	kfree(dconf);
-
 	return 0;
 }
 
@@ -1263,8 +1424,8 @@ static void gs_usb_disconnect(struct usb_interface *intf)
 }
 
 static const struct usb_device_id gs_usb_table[] = {
-	{ USB_DEVICE_INTERFACE_NUMBER(USB_GSUSB_1_VENDOR_ID,
-				      USB_GSUSB_1_PRODUCT_ID, 0) },
+	{ USB_DEVICE_INTERFACE_NUMBER(USB_GS_USB_1_VENDOR_ID,
+				      USB_GS_USB_1_PRODUCT_ID, 0) },
 	{ USB_DEVICE_INTERFACE_NUMBER(USB_CANDLELIGHT_VENDOR_ID,
 				      USB_CANDLELIGHT_PRODUCT_ID, 0) },
 	{ USB_DEVICE_INTERFACE_NUMBER(USB_CES_CANEXT_FD_VENDOR_ID,
diff --git a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c
index dd65c101bfb8..6871d474dabf 100644
--- a/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c
+++ b/drivers/net/can/usb/kvaser_usb/kvaser_usb_hydra.c
@@ -534,7 +534,7 @@ static int kvaser_usb_hydra_send_simple_cmd(struct kvaser_usb *dev,
 	struct kvaser_cmd *cmd;
 	int err;
 
-	cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL);
+	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 	if (!cmd)
 		return -ENOMEM;
 
@@ -573,7 +573,7 @@ kvaser_usb_hydra_send_simple_cmd_async(struct kvaser_usb_net_priv *priv,
 	struct kvaser_usb *dev = priv->dev;
 	int err;
 
-	cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_ATOMIC);
+	cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC);
 	if (!cmd)
 		return -ENOMEM;
 
@@ -694,7 +694,7 @@ static int kvaser_usb_hydra_map_channel(struct kvaser_usb *dev, u16 transid,
 	struct kvaser_cmd *cmd;
 	int err;
 
-	cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL);
+	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 	if (!cmd)
 		return -ENOMEM;
 
@@ -735,7 +735,7 @@ static int kvaser_usb_hydra_get_single_capability(struct kvaser_usb *dev,
 	int err;
 	int i;
 
-	cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL);
+	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 	if (!cmd)
 		return -ENOMEM;
 
@@ -1394,7 +1394,7 @@ kvaser_usb_hydra_frame_to_cmd_ext(const struct kvaser_usb_net_priv *priv,
 	u32 kcan_id;
 	u32 kcan_header;
 
-	cmd = kcalloc(1, sizeof(struct kvaser_cmd_ext), GFP_ATOMIC);
+	cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC);
 	if (!cmd)
 		return NULL;
 
@@ -1468,7 +1468,7 @@ kvaser_usb_hydra_frame_to_cmd_std(const struct kvaser_usb_net_priv *priv,
 	u32 flags;
 	u32 id;
 
-	cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_ATOMIC);
+	cmd = kzalloc(sizeof(*cmd), GFP_ATOMIC);
 	if (!cmd)
 		return NULL;
 
@@ -1533,7 +1533,7 @@ static int kvaser_usb_hydra_set_bittiming(struct net_device *netdev)
 	int sjw = bt->sjw;
 	int err;
 
-	cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL);
+	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 	if (!cmd)
 		return -ENOMEM;
 
@@ -1567,7 +1567,7 @@ static int kvaser_usb_hydra_set_data_bittiming(struct net_device *netdev)
 	int sjw = dbt->sjw;
 	int err;
 
-	cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL);
+	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 	if (!cmd)
 		return -ENOMEM;
 
@@ -1711,7 +1711,7 @@ static int kvaser_usb_hydra_get_software_details(struct kvaser_usb *dev)
 	u32 flags;
 	struct kvaser_usb_dev_card_data *card_data = &dev->card_data;
 
-	cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL);
+	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 	if (!cmd)
 		return -ENOMEM;
 
@@ -1851,7 +1851,7 @@ static int kvaser_usb_hydra_set_opt_mode(const struct kvaser_usb_net_priv *priv)
 		return -EINVAL;
 	}
 
-	cmd = kcalloc(1, sizeof(struct kvaser_cmd), GFP_KERNEL);
+	cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 	if (!cmd)
 		return -ENOMEM;
 
diff --git a/drivers/net/can/usb/peak_usb/pcan_usb_core.c b/drivers/net/can/usb/peak_usb/pcan_usb_core.c
index 8c9d53f6e24c..225697d70a9a 100644
--- a/drivers/net/can/usb/peak_usb/pcan_usb_core.c
+++ b/drivers/net/can/usb/peak_usb/pcan_usb_core.c
@@ -962,7 +962,7 @@ static void peak_usb_disconnect(struct usb_interface *intf)
 
 		dev_prev_siblings = dev->prev_siblings;
 		dev->state &= ~PCAN_USB_STATE_CONNECTED;
-		strlcpy(name, netdev->name, IFNAMSIZ);
+		strscpy(name, netdev->name, IFNAMSIZ);
 
 		unregister_netdev(netdev);
 
diff --git a/drivers/net/can/vcan.c b/drivers/net/can/vcan.c
index 36b6310a2e5b..285635c23443 100644
--- a/drivers/net/can/vcan.c
+++ b/drivers/net/can/vcan.c
@@ -71,11 +71,10 @@ MODULE_PARM_DESC(echo, "Echo sent frames (for testing). Default: 0 (Off)");
 
 static void vcan_rx(struct sk_buff *skb, struct net_device *dev)
 {
-	struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
 	struct net_device_stats *stats = &dev->stats;
 
 	stats->rx_packets++;
-	stats->rx_bytes += cfd->len;
+	stats->rx_bytes += can_skb_get_data_len(skb);
 
 	skb->pkt_type  = PACKET_BROADCAST;
 	skb->dev       = dev;
@@ -86,14 +85,14 @@ static void vcan_rx(struct sk_buff *skb, struct net_device *dev)
 
 static netdev_tx_t vcan_tx(struct sk_buff *skb, struct net_device *dev)
 {
-	struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
 	struct net_device_stats *stats = &dev->stats;
-	int loop, len;
+	unsigned int len;
+	int loop;
 
 	if (can_dropped_invalid_skb(dev, skb))
 		return NETDEV_TX_OK;
 
-	len = cfd->can_id & CAN_RTR_FLAG ? 0 : cfd->len;
+	len = can_skb_get_data_len(skb);
 	stats->tx_packets++;
 	stats->tx_bytes += len;
 
@@ -137,7 +136,8 @@ static int vcan_change_mtu(struct net_device *dev, int new_mtu)
 	if (dev->flags & IFF_UP)
 		return -EBUSY;
 
-	if (new_mtu != CAN_MTU && new_mtu != CANFD_MTU)
+	if (new_mtu != CAN_MTU && new_mtu != CANFD_MTU &&
+	    !can_is_canxl_dev_mtu(new_mtu))
 		return -EINVAL;
 
 	dev->mtu = new_mtu;
diff --git a/drivers/net/can/vxcan.c b/drivers/net/can/vxcan.c
index cffd107d8b28..26a472d2ea58 100644
--- a/drivers/net/can/vxcan.c
+++ b/drivers/net/can/vxcan.c
@@ -38,10 +38,9 @@ static netdev_tx_t vxcan_xmit(struct sk_buff *oskb, struct net_device *dev)
 {
 	struct vxcan_priv *priv = netdev_priv(dev);
 	struct net_device *peer;
-	struct canfd_frame *cfd = (struct canfd_frame *)oskb->data;
 	struct net_device_stats *peerstats, *srcstats = &dev->stats;
 	struct sk_buff *skb;
-	u8 len;
+	unsigned int len;
 
 	if (can_dropped_invalid_skb(dev, oskb))
 		return NETDEV_TX_OK;
@@ -70,7 +69,7 @@ static netdev_tx_t vxcan_xmit(struct sk_buff *oskb, struct net_device *dev)
 	skb->dev        = peer;
 	skb->ip_summed  = CHECKSUM_UNNECESSARY;
 
-	len = cfd->can_id & CAN_RTR_FLAG ? 0 : cfd->len;
+	len = can_skb_get_data_len(skb);
 	if (netif_rx(skb) == NET_RX_SUCCESS) {
 		srcstats->tx_packets++;
 		srcstats->tx_bytes += len;
@@ -132,7 +131,8 @@ static int vxcan_change_mtu(struct net_device *dev, int new_mtu)
 	if (dev->flags & IFF_UP)
 		return -EBUSY;
 
-	if (new_mtu != CAN_MTU && new_mtu != CANFD_MTU)
+	if (new_mtu != CAN_MTU && new_mtu != CANFD_MTU &&
+	    !can_is_canxl_dev_mtu(new_mtu))
 		return -EINVAL;
 
 	dev->mtu = new_mtu;
diff --git a/drivers/net/dsa/Kconfig b/drivers/net/dsa/Kconfig
index d8ae0e8af2a0..07507b4820d7 100644
--- a/drivers/net/dsa/Kconfig
+++ b/drivers/net/dsa/Kconfig
@@ -76,7 +76,7 @@ config NET_DSA_SMSC_LAN9303
 	select NET_DSA_TAG_LAN9303
 	select REGMAP
 	help
-	  This enables support for the SMSC/Microchip LAN9303 3 port ethernet
+	  This enables support for the Microchip LAN9303/LAN9354 3 port ethernet
 	  switch chips.
 
 config NET_DSA_SMSC_LAN9303_I2C
@@ -90,11 +90,11 @@ config NET_DSA_SMSC_LAN9303_I2C
 	  for I2C managed mode.
 
 config NET_DSA_SMSC_LAN9303_MDIO
-	tristate "SMSC/Microchip LAN9303 3-ports 10/100 ethernet switch in MDIO managed mode"
+	tristate "Microchip LAN9303/LAN9354 3-ports 10/100 ethernet switch in MDIO managed mode"
 	select NET_DSA_SMSC_LAN9303
 	depends on VLAN_8021Q || VLAN_8021Q=n
 	help
-	  Enable access functions if the SMSC/Microchip LAN9303 is configured
+	  Enable access functions if the Microchip LAN9303/LAN9354 is configured
 	  for MDIO managed mode.
 
 config NET_DSA_VITESSE_VSC73XX
diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c
index 48cf344750ff..59cdfc51ce06 100644
--- a/drivers/net/dsa/b53/b53_common.c
+++ b/drivers/net/dsa/b53/b53_common.c
@@ -972,7 +972,7 @@ void b53_get_strings(struct dsa_switch *ds, int port, u32 stringset,
 
 	if (stringset == ETH_SS_STATS) {
 		for (i = 0; i < mib_size; i++)
-			strlcpy(data + i * ETH_GSTRING_LEN,
+			strscpy(data + i * ETH_GSTRING_LEN,
 				mibs[i].name, ETH_GSTRING_LEN);
 	} else if (stringset == ETH_SS_PHY_STATS) {
 		phydev = b53_get_phy_device(ds, port);
diff --git a/drivers/net/dsa/b53/b53_mdio.c b/drivers/net/dsa/b53/b53_mdio.c
index a7aeb3c132c9..6ddc03b58b28 100644
--- a/drivers/net/dsa/b53/b53_mdio.c
+++ b/drivers/net/dsa/b53/b53_mdio.c
@@ -356,8 +356,6 @@ static void b53_mdio_remove(struct mdio_device *mdiodev)
 		return;
 
 	b53_switch_remove(dev);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void b53_mdio_shutdown(struct mdio_device *mdiodev)
diff --git a/drivers/net/dsa/b53/b53_mmap.c b/drivers/net/dsa/b53/b53_mmap.c
index ae4c79d39bc0..e968322dfbf0 100644
--- a/drivers/net/dsa/b53/b53_mmap.c
+++ b/drivers/net/dsa/b53/b53_mmap.c
@@ -316,8 +316,6 @@ static int b53_mmap_remove(struct platform_device *pdev)
 	if (dev)
 		b53_switch_remove(dev);
 
-	platform_set_drvdata(pdev, NULL);
-
 	return 0;
 }
 
diff --git a/drivers/net/dsa/b53/b53_srab.c b/drivers/net/dsa/b53/b53_srab.c
index da0b889880f6..bcb44034404d 100644
--- a/drivers/net/dsa/b53/b53_srab.c
+++ b/drivers/net/dsa/b53/b53_srab.c
@@ -667,8 +667,6 @@ static int b53_srab_remove(struct platform_device *pdev)
 	b53_srab_intr_set(dev->priv, false);
 	b53_switch_remove(dev);
 
-	platform_set_drvdata(pdev, NULL);
-
 	return 0;
 }
 
diff --git a/drivers/net/dsa/bcm_sf2.c b/drivers/net/dsa/bcm_sf2.c
index be0edfa093d0..cde253d27bd0 100644
--- a/drivers/net/dsa/bcm_sf2.c
+++ b/drivers/net/dsa/bcm_sf2.c
@@ -94,6 +94,24 @@ static u16 bcm_sf2_reg_led_base(struct bcm_sf2_priv *priv, int port)
 	return REG_SWITCH_STATUS;
 }
 
+static u32 bcm_sf2_port_override_offset(struct bcm_sf2_priv *priv, int port)
+{
+	switch (priv->type) {
+	case BCM4908_DEVICE_ID:
+	case BCM7445_DEVICE_ID:
+		return port == 8 ? CORE_STS_OVERRIDE_IMP :
+				   CORE_STS_OVERRIDE_GMIIP_PORT(port);
+	case BCM7278_DEVICE_ID:
+		return port == 8 ? CORE_STS_OVERRIDE_IMP2 :
+				   CORE_STS_OVERRIDE_GMIIP2_PORT(port);
+	default:
+		WARN_ONCE(1, "Unsupported device: %d\n", priv->type);
+	}
+
+	/* RO fallback register */
+	return REG_SWITCH_STATUS;
+}
+
 /* Return the number of active ports, not counting the IMP (CPU) port */
 static unsigned int bcm_sf2_num_active_ports(struct dsa_switch *ds)
 {
@@ -141,7 +159,7 @@ static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port)
 {
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	unsigned int i;
-	u32 reg, offset;
+	u32 reg;
 
 	/* Enable the port memories */
 	reg = core_readl(priv, CORE_MEM_PSM_VDD_CTRL);
@@ -167,21 +185,6 @@ static void bcm_sf2_imp_setup(struct dsa_switch *ds, int port)
 	b53_brcm_hdr_setup(ds, port);
 
 	if (port == 8) {
-		if (priv->type == BCM4908_DEVICE_ID ||
-		    priv->type == BCM7445_DEVICE_ID)
-			offset = CORE_STS_OVERRIDE_IMP;
-		else
-			offset = CORE_STS_OVERRIDE_IMP2;
-
-		/* Force link status for IMP port */
-		reg = core_readl(priv, offset);
-		reg |= (MII_SW_OR | LINK_STS);
-		if (priv->type == BCM4908_DEVICE_ID)
-			reg |= GMII_SPEED_UP_2G;
-		else
-			reg &= ~GMII_SPEED_UP_2G;
-		core_writel(priv, reg, offset);
-
 		/* Enable Broadcast, Multicast, Unicast forwarding to IMP port */
 		reg = core_readl(priv, CORE_IMP_CTL);
 		reg |= (RX_BCST_EN | RX_MCST_EN | RX_UCST_EN);
@@ -812,17 +815,10 @@ static void bcm_sf2_sw_mac_link_down(struct dsa_switch *ds, int port,
 	if (priv->wol_ports_mask & BIT(port))
 		return;
 
-	if (port != core_readl(priv, CORE_IMP0_PRT_ID)) {
-		if (priv->type == BCM4908_DEVICE_ID ||
-		    priv->type == BCM7445_DEVICE_ID)
-			offset = CORE_STS_OVERRIDE_GMIIP_PORT(port);
-		else
-			offset = CORE_STS_OVERRIDE_GMIIP2_PORT(port);
-
-		reg = core_readl(priv, offset);
-		reg &= ~LINK_STS;
-		core_writel(priv, reg, offset);
-	}
+	offset = bcm_sf2_port_override_offset(priv, port);
+	reg = core_readl(priv, offset);
+	reg &= ~LINK_STS;
+	core_writel(priv, reg, offset);
 
 	bcm_sf2_sw_mac_link_set(ds, port, interface, false);
 }
@@ -836,56 +832,56 @@ static void bcm_sf2_sw_mac_link_up(struct dsa_switch *ds, int port,
 {
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	struct ethtool_eee *p = &priv->dev->ports[port].eee;
+	u32 reg_rgmii_ctrl = 0;
+	u32 reg, offset;
 
 	bcm_sf2_sw_mac_link_set(ds, port, interface, true);
 
-	if (port != core_readl(priv, CORE_IMP0_PRT_ID)) {
-		u32 reg_rgmii_ctrl = 0;
-		u32 reg, offset;
+	offset = bcm_sf2_port_override_offset(priv, port);
 
-		if (priv->type == BCM4908_DEVICE_ID ||
-		    priv->type == BCM7445_DEVICE_ID)
-			offset = CORE_STS_OVERRIDE_GMIIP_PORT(port);
-		else
-			offset = CORE_STS_OVERRIDE_GMIIP2_PORT(port);
-
-		if (interface == PHY_INTERFACE_MODE_RGMII ||
-		    interface == PHY_INTERFACE_MODE_RGMII_TXID ||
-		    interface == PHY_INTERFACE_MODE_MII ||
-		    interface == PHY_INTERFACE_MODE_REVMII) {
-			reg_rgmii_ctrl = bcm_sf2_reg_rgmii_cntrl(priv, port);
-			reg = reg_readl(priv, reg_rgmii_ctrl);
-			reg &= ~(RX_PAUSE_EN | TX_PAUSE_EN);
-
-			if (tx_pause)
-				reg |= TX_PAUSE_EN;
-			if (rx_pause)
-				reg |= RX_PAUSE_EN;
-
-			reg_writel(priv, reg, reg_rgmii_ctrl);
-		}
-
-		reg = SW_OVERRIDE | LINK_STS;
-		switch (speed) {
-		case SPEED_1000:
-			reg |= SPDSTS_1000 << SPEED_SHIFT;
-			break;
-		case SPEED_100:
-			reg |= SPDSTS_100 << SPEED_SHIFT;
-			break;
-		}
-
-		if (duplex == DUPLEX_FULL)
-			reg |= DUPLX_MODE;
+	if (phy_interface_mode_is_rgmii(interface) ||
+	    interface == PHY_INTERFACE_MODE_MII ||
+	    interface == PHY_INTERFACE_MODE_REVMII) {
+		reg_rgmii_ctrl = bcm_sf2_reg_rgmii_cntrl(priv, port);
+		reg = reg_readl(priv, reg_rgmii_ctrl);
+		reg &= ~(RX_PAUSE_EN | TX_PAUSE_EN);
 
 		if (tx_pause)
-			reg |= TXFLOW_CNTL;
+			reg |= TX_PAUSE_EN;
 		if (rx_pause)
-			reg |= RXFLOW_CNTL;
+			reg |= RX_PAUSE_EN;
 
-		core_writel(priv, reg, offset);
+		reg_writel(priv, reg, reg_rgmii_ctrl);
 	}
 
+	reg = LINK_STS;
+	if (port == 8) {
+		if (priv->type == BCM4908_DEVICE_ID)
+			reg |= GMII_SPEED_UP_2G;
+		reg |= MII_SW_OR;
+	} else {
+		reg |= SW_OVERRIDE;
+	}
+
+	switch (speed) {
+	case SPEED_1000:
+		reg |= SPDSTS_1000 << SPEED_SHIFT;
+		break;
+	case SPEED_100:
+		reg |= SPDSTS_100 << SPEED_SHIFT;
+		break;
+	}
+
+	if (duplex == DUPLEX_FULL)
+		reg |= DUPLX_MODE;
+
+	if (tx_pause)
+		reg |= TXFLOW_CNTL;
+	if (rx_pause)
+		reg |= RXFLOW_CNTL;
+
+	core_writel(priv, reg, offset);
+
 	if (mode == MLO_AN_PHY && phydev)
 		p->eee_enabled = b53_eee_init(ds, port, phydev);
 }
@@ -987,7 +983,7 @@ static int bcm_sf2_sw_resume(struct dsa_switch *ds)
 static void bcm_sf2_sw_get_wol(struct dsa_switch *ds, int port,
 			       struct ethtool_wolinfo *wol)
 {
-	struct net_device *p = dsa_to_port(ds, port)->cpu_dp->master;
+	struct net_device *p = dsa_port_to_master(dsa_to_port(ds, port));
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	struct ethtool_wolinfo pwol = { };
 
@@ -1011,7 +1007,7 @@ static void bcm_sf2_sw_get_wol(struct dsa_switch *ds, int port,
 static int bcm_sf2_sw_set_wol(struct dsa_switch *ds, int port,
 			      struct ethtool_wolinfo *wol)
 {
-	struct net_device *p = dsa_to_port(ds, port)->cpu_dp->master;
+	struct net_device *p = dsa_port_to_master(dsa_to_port(ds, port));
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	s8 cpu_port = dsa_to_port(ds, port)->cpu_dp->index;
 	struct ethtool_wolinfo pwol =  { };
@@ -1555,8 +1551,6 @@ static int bcm_sf2_sw_remove(struct platform_device *pdev)
 	if (priv->type == BCM7278_DEVICE_ID)
 		reset_control_assert(priv->rcdev);
 
-	platform_set_drvdata(pdev, NULL);
-
 	return 0;
 }
 
diff --git a/drivers/net/dsa/bcm_sf2_cfp.c b/drivers/net/dsa/bcm_sf2_cfp.c
index edbe5e7f1cb6..c4010b7bf089 100644
--- a/drivers/net/dsa/bcm_sf2_cfp.c
+++ b/drivers/net/dsa/bcm_sf2_cfp.c
@@ -1102,7 +1102,7 @@ static int bcm_sf2_cfp_rule_get_all(struct bcm_sf2_priv *priv,
 int bcm_sf2_get_rxnfc(struct dsa_switch *ds, int port,
 		      struct ethtool_rxnfc *nfc, u32 *rule_locs)
 {
-	struct net_device *p = dsa_to_port(ds, port)->cpu_dp->master;
+	struct net_device *p = dsa_port_to_master(dsa_to_port(ds, port));
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	int ret = 0;
 
@@ -1145,7 +1145,7 @@ int bcm_sf2_get_rxnfc(struct dsa_switch *ds, int port,
 int bcm_sf2_set_rxnfc(struct dsa_switch *ds, int port,
 		      struct ethtool_rxnfc *nfc)
 {
-	struct net_device *p = dsa_to_port(ds, port)->cpu_dp->master;
+	struct net_device *p = dsa_port_to_master(dsa_to_port(ds, port));
 	struct bcm_sf2_priv *priv = bcm_sf2_to_priv(ds);
 	int ret = 0;
 
@@ -1296,7 +1296,7 @@ void bcm_sf2_cfp_get_strings(struct dsa_switch *ds, int port,
 				 "CFP%03d_%sCntr",
 				 i, bcm_sf2_cfp_stats[j].name);
 			iter = (i - 1) * s + j;
-			strlcpy(data + iter * ETH_GSTRING_LEN,
+			strscpy(data + iter * ETH_GSTRING_LEN,
 				buf, ETH_GSTRING_LEN);
 		}
 	}
diff --git a/drivers/net/dsa/dsa_loop.c b/drivers/net/dsa/dsa_loop.c
index 263e41191c29..b9107fe40023 100644
--- a/drivers/net/dsa/dsa_loop.c
+++ b/drivers/net/dsa/dsa_loop.c
@@ -351,8 +351,6 @@ static void dsa_loop_drv_remove(struct mdio_device *mdiodev)
 
 	dsa_unregister_switch(ds);
 	dev_put(ps->netdev);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void dsa_loop_drv_shutdown(struct mdio_device *mdiodev)
diff --git a/drivers/net/dsa/hirschmann/hellcreek.c b/drivers/net/dsa/hirschmann/hellcreek.c
index 01f90994dedd..951f7935c872 100644
--- a/drivers/net/dsa/hirschmann/hellcreek.c
+++ b/drivers/net/dsa/hirschmann/hellcreek.c
@@ -128,6 +128,16 @@ static void hellcreek_select_prio(struct hellcreek *hellcreek, int prio)
 	hellcreek_write(hellcreek, val, HR_PSEL);
 }
 
+static void hellcreek_select_port_prio(struct hellcreek *hellcreek, int port,
+				       int prio)
+{
+	u16 val = port << HR_PSEL_PTWSEL_SHIFT;
+
+	val |= prio << HR_PSEL_PRTCWSEL_SHIFT;
+
+	hellcreek_write(hellcreek, val, HR_PSEL);
+}
+
 static void hellcreek_select_counter(struct hellcreek *hellcreek, int counter)
 {
 	u16 val = counter << HR_CSEL_SHIFT;
@@ -288,7 +298,7 @@ static void hellcreek_get_strings(struct dsa_switch *ds, int port,
 	for (i = 0; i < ARRAY_SIZE(hellcreek_counter); ++i) {
 		const struct hellcreek_counter *counter = &hellcreek_counter[i];
 
-		strlcpy(data + i * ETH_GSTRING_LEN,
+		strscpy(data + i * ETH_GSTRING_LEN,
 			counter->name, ETH_GSTRING_LEN);
 	}
 }
@@ -1537,6 +1547,45 @@ out:
 	return ret;
 }
 
+static void hellcreek_setup_maxsdu(struct hellcreek *hellcreek, int port,
+				   const struct tc_taprio_qopt_offload *schedule)
+{
+	int tc;
+
+	for (tc = 0; tc < 8; ++tc) {
+		u32 max_sdu = schedule->max_sdu[tc] + VLAN_ETH_HLEN - ETH_FCS_LEN;
+		u16 val;
+
+		if (!schedule->max_sdu[tc])
+			continue;
+
+		dev_dbg(hellcreek->dev, "Configure max-sdu %u for tc %d on port %d\n",
+			max_sdu, tc, port);
+
+		hellcreek_select_port_prio(hellcreek, port, tc);
+
+		val = (max_sdu & HR_PTPRTCCFG_MAXSDU_MASK) << HR_PTPRTCCFG_MAXSDU_SHIFT;
+
+		hellcreek_write(hellcreek, val, HR_PTPRTCCFG);
+	}
+}
+
+static void hellcreek_reset_maxsdu(struct hellcreek *hellcreek, int port)
+{
+	int tc;
+
+	for (tc = 0; tc < 8; ++tc) {
+		u16 val;
+
+		hellcreek_select_port_prio(hellcreek, port, tc);
+
+		val = (HELLCREEK_DEFAULT_MAX_SDU & HR_PTPRTCCFG_MAXSDU_MASK)
+			<< HR_PTPRTCCFG_MAXSDU_SHIFT;
+
+		hellcreek_write(hellcreek, val, HR_PTPRTCCFG);
+	}
+}
+
 static void hellcreek_setup_gcl(struct hellcreek *hellcreek, int port,
 				const struct tc_taprio_qopt_offload *schedule)
 {
@@ -1720,7 +1769,10 @@ static int hellcreek_port_set_schedule(struct dsa_switch *ds, int port,
 	}
 	hellcreek_port->current_schedule = taprio_offload_get(taprio);
 
-	/* Then select port */
+	/* Configure max sdu */
+	hellcreek_setup_maxsdu(hellcreek, port, hellcreek_port->current_schedule);
+
+	/* Select tdg */
 	hellcreek_select_tgd(hellcreek, port);
 
 	/* Enable gating and keep defaults */
@@ -1772,7 +1824,10 @@ static int hellcreek_port_del_schedule(struct dsa_switch *ds, int port)
 		hellcreek_port->current_schedule = NULL;
 	}
 
-	/* Then select port */
+	/* Reset max sdu */
+	hellcreek_reset_maxsdu(hellcreek, port);
+
+	/* Select tgd */
 	hellcreek_select_tgd(hellcreek, port);
 
 	/* Disable gating and return to regular switching flow */
@@ -1809,22 +1864,43 @@ static bool hellcreek_validate_schedule(struct hellcreek *hellcreek,
 	return true;
 }
 
+static int hellcreek_tc_query_caps(struct tc_query_caps_base *base)
+{
+	switch (base->type) {
+	case TC_SETUP_QDISC_TAPRIO: {
+		struct tc_taprio_caps *caps = base->caps;
+
+		caps->supports_queue_max_sdu = true;
+
+		return 0;
+	}
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 static int hellcreek_port_setup_tc(struct dsa_switch *ds, int port,
 				   enum tc_setup_type type, void *type_data)
 {
-	struct tc_taprio_qopt_offload *taprio = type_data;
 	struct hellcreek *hellcreek = ds->priv;
 
-	if (type != TC_SETUP_QDISC_TAPRIO)
-		return -EOPNOTSUPP;
+	switch (type) {
+	case TC_QUERY_CAPS:
+		return hellcreek_tc_query_caps(type_data);
+	case TC_SETUP_QDISC_TAPRIO: {
+		struct tc_taprio_qopt_offload *taprio = type_data;
 
-	if (!hellcreek_validate_schedule(hellcreek, taprio))
-		return -EOPNOTSUPP;
+		if (!hellcreek_validate_schedule(hellcreek, taprio))
+			return -EOPNOTSUPP;
 
-	if (taprio->enable)
-		return hellcreek_port_set_schedule(ds, port, taprio);
+		if (taprio->enable)
+			return hellcreek_port_set_schedule(ds, port, taprio);
 
-	return hellcreek_port_del_schedule(ds, port);
+		return hellcreek_port_del_schedule(ds, port);
+	}
+	default:
+		return -EOPNOTSUPP;
+	}
 }
 
 static const struct dsa_switch_ops hellcreek_ds_ops = {
@@ -1996,7 +2072,6 @@ static int hellcreek_remove(struct platform_device *pdev)
 	hellcreek_hwtstamp_free(hellcreek);
 	hellcreek_ptp_free(hellcreek);
 	dsa_unregister_switch(hellcreek->ds);
-	platform_set_drvdata(pdev, NULL);
 
 	return 0;
 }
diff --git a/drivers/net/dsa/hirschmann/hellcreek.h b/drivers/net/dsa/hirschmann/hellcreek.h
index 9e303b8ab13c..4a678f7d61ae 100644
--- a/drivers/net/dsa/hirschmann/hellcreek.h
+++ b/drivers/net/dsa/hirschmann/hellcreek.h
@@ -37,6 +37,7 @@
 #define HELLCREEK_VLAN_UNTAGGED_MEMBER	0x1
 #define HELLCREEK_VLAN_TAGGED_MEMBER	0x3
 #define HELLCREEK_NUM_EGRESS_QUEUES	8
+#define HELLCREEK_DEFAULT_MAX_SDU	1536
 
 /* Register definitions */
 #define HR_MODID_C			(0 * 2)
@@ -72,6 +73,12 @@
 #define HR_PRTCCFG_PCP_TC_MAP_SHIFT	0
 #define HR_PRTCCFG_PCP_TC_MAP_MASK	GENMASK(2, 0)
 
+#define HR_PTPRTCCFG			(0xa9 * 2)
+#define HR_PTPRTCCFG_SET_QTRACK		BIT(15)
+#define HR_PTPRTCCFG_REJECT		BIT(14)
+#define HR_PTPRTCCFG_MAXSDU_SHIFT	0
+#define HR_PTPRTCCFG_MAXSDU_MASK	GENMASK(10, 0)
+
 #define HR_CSEL				(0x8d * 2)
 #define HR_CSEL_SHIFT			0
 #define HR_CSEL_MASK			GENMASK(7, 0)
diff --git a/drivers/net/dsa/lan9303-core.c b/drivers/net/dsa/lan9303-core.c
index e03ff1f267bb..438e46af03e9 100644
--- a/drivers/net/dsa/lan9303-core.c
+++ b/drivers/net/dsa/lan9303-core.c
@@ -22,6 +22,10 @@
  */
 #define LAN9303_CHIP_REV 0x14
 # define LAN9303_CHIP_ID 0x9303
+# define LAN9352_CHIP_ID 0x9352
+# define LAN9353_CHIP_ID 0x9353
+# define LAN9354_CHIP_ID 0x9354
+# define LAN9355_CHIP_ID 0x9355
 #define LAN9303_IRQ_CFG 0x15
 # define LAN9303_IRQ_CFG_IRQ_ENABLE BIT(8)
 # define LAN9303_IRQ_CFG_IRQ_POL BIT(4)
@@ -32,6 +36,7 @@
 #define LAN9303_INT_EN 0x17
 # define LAN9303_INT_EN_PHY_INT2_EN BIT(27)
 # define LAN9303_INT_EN_PHY_INT1_EN BIT(26)
+#define LAN9303_BYTE_ORDER 0x19
 #define LAN9303_HW_CFG 0x1D
 # define LAN9303_HW_CFG_READY BIT(27)
 # define LAN9303_HW_CFG_AMDX_EN_PORT2 BIT(26)
@@ -851,15 +856,12 @@ static int lan9303_check_device(struct lan9303 *chip)
 	if (ret) {
 		dev_err(chip->dev, "failed to read chip revision register: %d\n",
 			ret);
-		if (!chip->reset_gpio) {
-			dev_dbg(chip->dev,
-				"hint: maybe failed due to missing reset GPIO\n");
-		}
 		return ret;
 	}
 
-	if ((reg >> 16) != LAN9303_CHIP_ID) {
-		dev_err(chip->dev, "expecting LAN9303 chip, but found: %X\n",
+	if (((reg >> 16) != LAN9303_CHIP_ID) &&
+	    ((reg >> 16) != LAN9354_CHIP_ID)) {
+		dev_err(chip->dev, "unexpected device found: LAN%4.4X\n",
 			reg >> 16);
 		return -ENODEV;
 	}
@@ -875,7 +877,7 @@ static int lan9303_check_device(struct lan9303 *chip)
 	if (ret)
 		dev_warn(chip->dev, "failed to disable switching %d\n", ret);
 
-	dev_info(chip->dev, "Found LAN9303 rev. %u\n", reg & 0xffff);
+	dev_info(chip->dev, "Found LAN%4.4X rev. %u\n", (reg >> 16), reg & 0xffff);
 
 	ret = lan9303_detect_phy_setup(chip);
 	if (ret) {
@@ -1090,7 +1092,7 @@ static int lan9303_port_enable(struct dsa_switch *ds, int port,
 	if (!dsa_port_is_user(dp))
 		return 0;
 
-	vlan_vid_add(dp->cpu_dp->master, htons(ETH_P_8021Q), port);
+	vlan_vid_add(dsa_port_to_master(dp), htons(ETH_P_8021Q), port);
 
 	return lan9303_enable_processing_port(chip, port);
 }
@@ -1103,7 +1105,7 @@ static void lan9303_port_disable(struct dsa_switch *ds, int port)
 	if (!dsa_port_is_user(dp))
 		return;
 
-	vlan_vid_del(dp->cpu_dp->master, htons(ETH_P_8021Q), port);
+	vlan_vid_del(dsa_port_to_master(dp), htons(ETH_P_8021Q), port);
 
 	lan9303_disable_processing_port(chip, port);
 	lan9303_phy_write(ds, chip->phy_addr_base + port, MII_BMCR, BMCR_PDOWN);
@@ -1349,6 +1351,7 @@ static int lan9303_probe_reset_gpio(struct lan9303 *chip,
 int lan9303_probe(struct lan9303 *chip, struct device_node *np)
 {
 	int ret;
+	u32 reg;
 
 	mutex_init(&chip->indirect_mutex);
 	mutex_init(&chip->alr_mutex);
@@ -1359,6 +1362,19 @@ int lan9303_probe(struct lan9303 *chip, struct device_node *np)
 
 	lan9303_handle_reset(chip);
 
+	/* First read to the device.  This is a Dummy read to ensure MDIO */
+	/* access is in 32-bit sync. */
+	ret = lan9303_read(chip->regmap, LAN9303_BYTE_ORDER, &reg);
+	if (ret) {
+		dev_err(chip->dev, "failed to access the device: %d\n",
+			ret);
+		if (!chip->reset_gpio) {
+			dev_dbg(chip->dev,
+				"hint: maybe failed due to missing reset GPIO\n");
+		}
+		return ret;
+	}
+
 	ret = lan9303_check_device(chip);
 	if (ret)
 		return ret;
diff --git a/drivers/net/dsa/lan9303_i2c.c b/drivers/net/dsa/lan9303_i2c.c
index 8ca4713310fa..7d746cd9ca1b 100644
--- a/drivers/net/dsa/lan9303_i2c.c
+++ b/drivers/net/dsa/lan9303_i2c.c
@@ -65,18 +65,14 @@ static int lan9303_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lan9303_i2c_remove(struct i2c_client *client)
+static void lan9303_i2c_remove(struct i2c_client *client)
 {
 	struct lan9303_i2c *sw_dev = i2c_get_clientdata(client);
 
 	if (!sw_dev)
-		return 0;
+		return;
 
 	lan9303_remove(&sw_dev->chip);
-
-	i2c_set_clientdata(client, NULL);
-
-	return 0;
 }
 
 static void lan9303_i2c_shutdown(struct i2c_client *client)
diff --git a/drivers/net/dsa/lan9303_mdio.c b/drivers/net/dsa/lan9303_mdio.c
index bbb7032409ba..4f33369a2de5 100644
--- a/drivers/net/dsa/lan9303_mdio.c
+++ b/drivers/net/dsa/lan9303_mdio.c
@@ -138,8 +138,6 @@ static void lan9303_mdio_remove(struct mdio_device *mdiodev)
 		return;
 
 	lan9303_remove(&sw_dev->chip);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void lan9303_mdio_shutdown(struct mdio_device *mdiodev)
@@ -158,6 +156,7 @@ static void lan9303_mdio_shutdown(struct mdio_device *mdiodev)
 
 static const struct of_device_id lan9303_mdio_of_match[] = {
 	{ .compatible = "smsc,lan9303-mdio" },
+	{ .compatible = "microchip,lan9354-mdio" },
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, lan9303_mdio_of_match);
diff --git a/drivers/net/dsa/lantiq_gswip.c b/drivers/net/dsa/lantiq_gswip.c
index e531b93f3cb2..05ecaa007ab1 100644
--- a/drivers/net/dsa/lantiq_gswip.c
+++ b/drivers/net/dsa/lantiq_gswip.c
@@ -1989,11 +1989,9 @@ static int gswip_gphy_fw_probe(struct gswip_priv *priv,
 	}
 
 	gphy_fw->reset = of_reset_control_array_get_exclusive(gphy_fw_np);
-	if (IS_ERR(gphy_fw->reset)) {
-		if (PTR_ERR(gphy_fw->reset) != -EPROBE_DEFER)
-			dev_err(dev, "Failed to lookup gphy reset\n");
-		return PTR_ERR(gphy_fw->reset);
-	}
+	if (IS_ERR(gphy_fw->reset))
+		return dev_err_probe(dev, PTR_ERR(gphy_fw->reset),
+				     "Failed to lookup gphy reset\n");
 
 	return gswip_gphy_fw_load(priv, gphy_fw);
 }
@@ -2231,8 +2229,6 @@ static int gswip_remove(struct platform_device *pdev)
 	for (i = 0; i < priv->num_gphy_fw; i++)
 		gswip_gphy_fw_remove(priv, &priv->gphy_fw[i]);
 
-	platform_set_drvdata(pdev, NULL);
-
 	return 0;
 }
 
diff --git a/drivers/net/dsa/microchip/ksz8.h b/drivers/net/dsa/microchip/ksz8.h
index 42c50cc4d853..8582b4b67d98 100644
--- a/drivers/net/dsa/microchip/ksz8.h
+++ b/drivers/net/dsa/microchip/ksz8.h
@@ -17,8 +17,8 @@ u32 ksz8_get_port_addr(int port, int offset);
 void ksz8_cfg_port_member(struct ksz_device *dev, int port, u8 member);
 void ksz8_flush_dyn_mac_table(struct ksz_device *dev, int port);
 void ksz8_port_setup(struct ksz_device *dev, int port, bool cpu_port);
-void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val);
-void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val);
+int ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val);
+int ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val);
 int ksz8_r_dyn_mac_table(struct ksz_device *dev, u16 addr, u8 *mac_addr,
 			 u8 *fid, u8 *src_port, u8 *timestamp, u16 *entries);
 int ksz8_r_sta_mac_table(struct ksz_device *dev, u16 addr,
diff --git a/drivers/net/dsa/microchip/ksz8795.c b/drivers/net/dsa/microchip/ksz8795.c
index c79a5128235f..bd3b133e7085 100644
--- a/drivers/net/dsa/microchip/ksz8795.c
+++ b/drivers/net/dsa/microchip/ksz8795.c
@@ -552,7 +552,7 @@ static void ksz8_w_vlan_table(struct ksz_device *dev, u16 vid, u16 vlan)
 	ksz8_w_table(dev, TABLE_VLAN, addr, buf);
 }
 
-void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
+int ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 {
 	u8 restart, speed, ctrl, link;
 	int processed = true;
@@ -560,14 +560,24 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 	u8 val1, val2;
 	u16 data = 0;
 	u8 p = phy;
+	int ret;
 
 	regs = dev->info->regs;
 
 	switch (reg) {
 	case MII_BMCR:
-		ksz_pread8(dev, p, regs[P_NEG_RESTART_CTRL], &restart);
-		ksz_pread8(dev, p, regs[P_SPEED_STATUS], &speed);
-		ksz_pread8(dev, p, regs[P_FORCE_CTRL], &ctrl);
+		ret = ksz_pread8(dev, p, regs[P_NEG_RESTART_CTRL], &restart);
+		if (ret)
+			return ret;
+
+		ret = ksz_pread8(dev, p, regs[P_SPEED_STATUS], &speed);
+		if (ret)
+			return ret;
+
+		ret = ksz_pread8(dev, p, regs[P_FORCE_CTRL], &ctrl);
+		if (ret)
+			return ret;
+
 		if (restart & PORT_PHY_LOOPBACK)
 			data |= BMCR_LOOPBACK;
 		if (ctrl & PORT_FORCE_100_MBIT)
@@ -597,7 +607,10 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 			data |= KSZ886X_BMCR_DISABLE_LED;
 		break;
 	case MII_BMSR:
-		ksz_pread8(dev, p, regs[P_LINK_STATUS], &link);
+		ret = ksz_pread8(dev, p, regs[P_LINK_STATUS], &link);
+		if (ret)
+			return ret;
+
 		data = BMSR_100FULL |
 		       BMSR_100HALF |
 		       BMSR_10FULL |
@@ -618,7 +631,10 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 			data = KSZ8795_ID_LO;
 		break;
 	case MII_ADVERTISE:
-		ksz_pread8(dev, p, regs[P_LOCAL_CTRL], &ctrl);
+		ret = ksz_pread8(dev, p, regs[P_LOCAL_CTRL], &ctrl);
+		if (ret)
+			return ret;
+
 		data = ADVERTISE_CSMA;
 		if (ctrl & PORT_AUTO_NEG_SYM_PAUSE)
 			data |= ADVERTISE_PAUSE_CAP;
@@ -632,7 +648,10 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 			data |= ADVERTISE_10HALF;
 		break;
 	case MII_LPA:
-		ksz_pread8(dev, p, regs[P_REMOTE_STATUS], &link);
+		ret = ksz_pread8(dev, p, regs[P_REMOTE_STATUS], &link);
+		if (ret)
+			return ret;
+
 		data = LPA_SLCT;
 		if (link & PORT_REMOTE_SYM_PAUSE)
 			data |= LPA_PAUSE_CAP;
@@ -648,8 +667,14 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 			data |= LPA_LPACK;
 		break;
 	case PHY_REG_LINK_MD:
-		ksz_pread8(dev, p, REG_PORT_LINK_MD_CTRL, &val1);
-		ksz_pread8(dev, p, REG_PORT_LINK_MD_RESULT, &val2);
+		ret = ksz_pread8(dev, p, REG_PORT_LINK_MD_CTRL, &val1);
+		if (ret)
+			return ret;
+
+		ret = ksz_pread8(dev, p, REG_PORT_LINK_MD_RESULT, &val2);
+		if (ret)
+			return ret;
+
 		if (val1 & PORT_START_CABLE_DIAG)
 			data |= PHY_START_CABLE_DIAG;
 
@@ -664,7 +689,10 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 				FIELD_GET(PORT_CABLE_FAULT_COUNTER_L, val2));
 		break;
 	case PHY_REG_PHY_CTRL:
-		ksz_pread8(dev, p, regs[P_LINK_STATUS], &link);
+		ret = ksz_pread8(dev, p, regs[P_LINK_STATUS], &link);
+		if (ret)
+			return ret;
+
 		if (link & PORT_MDIX_STATUS)
 			data |= KSZ886X_CTRL_MDIX_STAT;
 		break;
@@ -674,13 +702,16 @@ void ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 	}
 	if (processed)
 		*val = data;
+
+	return 0;
 }
 
-void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
+int ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 {
 	u8 restart, speed, ctrl, data;
 	const u16 *regs;
 	u8 p = phy;
+	int ret;
 
 	regs = dev->info->regs;
 
@@ -690,15 +721,26 @@ void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 		/* Do not support PHY reset function. */
 		if (val & BMCR_RESET)
 			break;
-		ksz_pread8(dev, p, regs[P_SPEED_STATUS], &speed);
+		ret = ksz_pread8(dev, p, regs[P_SPEED_STATUS], &speed);
+		if (ret)
+			return ret;
+
 		data = speed;
 		if (val & KSZ886X_BMCR_HP_MDIX)
 			data |= PORT_HP_MDIX;
 		else
 			data &= ~PORT_HP_MDIX;
-		if (data != speed)
-			ksz_pwrite8(dev, p, regs[P_SPEED_STATUS], data);
-		ksz_pread8(dev, p, regs[P_FORCE_CTRL], &ctrl);
+
+		if (data != speed) {
+			ret = ksz_pwrite8(dev, p, regs[P_SPEED_STATUS], data);
+			if (ret)
+				return ret;
+		}
+
+		ret = ksz_pread8(dev, p, regs[P_FORCE_CTRL], &ctrl);
+		if (ret)
+			return ret;
+
 		data = ctrl;
 		if (ksz_is_ksz88x3(dev)) {
 			if ((val & BMCR_ANENABLE))
@@ -724,9 +766,17 @@ void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 			data |= PORT_FORCE_FULL_DUPLEX;
 		else
 			data &= ~PORT_FORCE_FULL_DUPLEX;
-		if (data != ctrl)
-			ksz_pwrite8(dev, p, regs[P_FORCE_CTRL], data);
-		ksz_pread8(dev, p, regs[P_NEG_RESTART_CTRL], &restart);
+
+		if (data != ctrl) {
+			ret = ksz_pwrite8(dev, p, regs[P_FORCE_CTRL], data);
+			if (ret)
+				return ret;
+		}
+
+		ret = ksz_pread8(dev, p, regs[P_NEG_RESTART_CTRL], &restart);
+		if (ret)
+			return ret;
+
 		data = restart;
 		if (val & KSZ886X_BMCR_DISABLE_LED)
 			data |= PORT_LED_OFF;
@@ -756,11 +806,19 @@ void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 			data |= PORT_PHY_LOOPBACK;
 		else
 			data &= ~PORT_PHY_LOOPBACK;
-		if (data != restart)
-			ksz_pwrite8(dev, p, regs[P_NEG_RESTART_CTRL], data);
+
+		if (data != restart) {
+			ret = ksz_pwrite8(dev, p, regs[P_NEG_RESTART_CTRL],
+					  data);
+			if (ret)
+				return ret;
+		}
 		break;
 	case MII_ADVERTISE:
-		ksz_pread8(dev, p, regs[P_LOCAL_CTRL], &ctrl);
+		ret = ksz_pread8(dev, p, regs[P_LOCAL_CTRL], &ctrl);
+		if (ret)
+			return ret;
+
 		data = ctrl;
 		data &= ~(PORT_AUTO_NEG_SYM_PAUSE |
 			  PORT_AUTO_NEG_100BTX_FD |
@@ -777,8 +835,12 @@ void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 			data |= PORT_AUTO_NEG_10BT_FD;
 		if (val & ADVERTISE_10HALF)
 			data |= PORT_AUTO_NEG_10BT;
-		if (data != ctrl)
-			ksz_pwrite8(dev, p, regs[P_LOCAL_CTRL], data);
+
+		if (data != ctrl) {
+			ret = ksz_pwrite8(dev, p, regs[P_LOCAL_CTRL], data);
+			if (ret)
+				return ret;
+		}
 		break;
 	case PHY_REG_LINK_MD:
 		if (val & PHY_START_CABLE_DIAG)
@@ -787,6 +849,8 @@ void ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 	default:
 		break;
 	}
+
+	return 0;
 }
 
 void ksz8_cfg_port_member(struct ksz_device *dev, int port, u8 member)
@@ -1187,7 +1251,6 @@ void ksz8_config_cpu_port(struct dsa_switch *ds)
 		if (i == dev->phy_port_cnt)
 			break;
 		p->on = 1;
-		p->phy = 1;
 	}
 	for (i = 0; i < dev->phy_port_cnt; i++) {
 		p = &dev->ports[i];
diff --git a/drivers/net/dsa/microchip/ksz8863_smi.c b/drivers/net/dsa/microchip/ksz8863_smi.c
index 5247fdfb964d..ddb40838181e 100644
--- a/drivers/net/dsa/microchip/ksz8863_smi.c
+++ b/drivers/net/dsa/microchip/ksz8863_smi.c
@@ -180,8 +180,6 @@ static void ksz8863_smi_remove(struct mdio_device *mdiodev)
 
 	if (dev)
 		ksz_switch_remove(dev);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void ksz8863_smi_shutdown(struct mdio_device *mdiodev)
diff --git a/drivers/net/dsa/microchip/ksz9477.c b/drivers/net/dsa/microchip/ksz9477.c
index e4f446db0ca1..a6a0321a8931 100644
--- a/drivers/net/dsa/microchip/ksz9477.c
+++ b/drivers/net/dsa/microchip/ksz9477.c
@@ -193,6 +193,11 @@ int ksz9477_reset_switch(struct ksz_device *dev)
 	ksz_write32(dev, REG_SW_PORT_INT_MASK__4, 0x7F);
 	ksz_read32(dev, REG_SW_PORT_INT_STATUS__4, &data32);
 
+	/* KSZ9893 compatible chips do not support refclk configuration */
+	if (dev->chip_id == KSZ9893_CHIP_ID ||
+	    dev->chip_id == KSZ8563_CHIP_ID)
+		return 0;
+
 	data8 = SW_ENABLE_REFCLKO;
 	if (dev->synclko_disable)
 		data8 = 0;
@@ -264,9 +269,20 @@ void ksz9477_port_init_cnt(struct ksz_device *dev, int port)
 	mutex_unlock(&mib->cnt_mutex);
 }
 
-void ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
+static void ksz9477_r_phy_quirks(struct ksz_device *dev, u16 addr, u16 reg,
+				 u16 *data)
+{
+	/* KSZ8563R do not have extended registers but BMSR_ESTATEN and
+	 * BMSR_ERCAP bits are set.
+	 */
+	if (dev->chip_id == KSZ8563_CHIP_ID && reg == MII_BMSR)
+		*data &= ~(BMSR_ESTATEN | BMSR_ERCAP);
+}
+
+int ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
 {
 	u16 val = 0xffff;
+	int ret;
 
 	/* No real PHY after this. Simulate the PHY.
 	 * A fixed PHY can be setup in the device tree, but this function is
@@ -274,7 +290,7 @@ void ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
 	 * For RGMII PHY there is no way to access it so the fixed PHY should
 	 * be used.  For SGMII PHY the supporting code will be added later.
 	 */
-	if (addr >= dev->phy_port_cnt) {
+	if (!dev->info->internal_phy[addr]) {
 		struct ksz_port *p = &dev->ports[addr];
 
 		switch (reg) {
@@ -307,23 +323,25 @@ void ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
 			break;
 		}
 	} else {
-		ksz_pread16(dev, addr, 0x100 + (reg << 1), &val);
+		ret = ksz_pread16(dev, addr, 0x100 + (reg << 1), &val);
+		if (ret)
+			return ret;
+
+		ksz9477_r_phy_quirks(dev, addr, reg, &val);
 	}
 
 	*data = val;
+
+	return 0;
 }
 
-void ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val)
+int ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val)
 {
 	/* No real PHY after this. */
-	if (addr >= dev->phy_port_cnt)
-		return;
+	if (!dev->info->internal_phy[addr])
+		return 0;
 
-	/* No gigabit support.  Do not write to this register. */
-	if (!(dev->features & GBIT_SUPPORT) && reg == MII_CTRL1000)
-		return;
-
-	ksz_pwrite16(dev, addr, 0x100 + (reg << 1), val);
+	return ksz_pwrite16(dev, addr, 0x100 + (reg << 1), val);
 }
 
 void ksz9477_cfg_port_member(struct ksz_device *dev, int port, u8 member)
@@ -869,7 +887,7 @@ static phy_interface_t ksz9477_get_interface(struct ksz_device *dev, int port)
 	phy_interface_t interface;
 	bool gbit;
 
-	if (port < dev->phy_port_cnt)
+	if (dev->info->internal_phy[port])
 		return PHY_INTERFACE_MODE_NA;
 
 	gbit = ksz_get_gbit(dev, port);
@@ -914,7 +932,7 @@ static void ksz9477_phy_errata_setup(struct ksz_device *dev, int port)
 	/* Energy Efficient Ethernet (EEE) feature select must
 	 * be manually disabled (except on KSZ8565 which is 100Mbit)
 	 */
-	if (dev->features & GBIT_SUPPORT)
+	if (dev->info->gbit_capable[port])
 		ksz9477_port_mmd_write(dev, port, 0x07, 0x3c, 0x0000);
 
 	/* Register settings are required to meet data sheet
@@ -941,10 +959,35 @@ void ksz9477_get_caps(struct ksz_device *dev, int port,
 	config->mac_capabilities = MAC_10 | MAC_100 | MAC_ASYM_PAUSE |
 				   MAC_SYM_PAUSE;
 
-	if (dev->features & GBIT_SUPPORT)
+	if (dev->info->gbit_capable[port])
 		config->mac_capabilities |= MAC_1000FD;
 }
 
+int ksz9477_set_ageing_time(struct ksz_device *dev, unsigned int msecs)
+{
+	u32 secs = msecs / 1000;
+	u8 value;
+	u8 data;
+	int ret;
+
+	value = FIELD_GET(SW_AGE_PERIOD_7_0_M, secs);
+
+	ret = ksz_write8(dev, REG_SW_LUE_CTRL_3, value);
+	if (ret < 0)
+		return ret;
+
+	data = FIELD_GET(SW_AGE_PERIOD_10_8_M, secs);
+
+	ret = ksz_read8(dev, REG_SW_LUE_CTRL_0, &value);
+	if (ret < 0)
+		return ret;
+
+	value &= ~SW_AGE_CNT_M;
+	value |= FIELD_PREP(SW_AGE_CNT_M, data);
+
+	return ksz_write8(dev, REG_SW_LUE_CTRL_0, value);
+}
+
 void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 {
 	struct dsa_switch *ds = dev->ds;
@@ -976,7 +1019,7 @@ void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 	/* enable 802.1p priority */
 	ksz_port_cfg(dev, port, P_PRIO_CTRL, PORT_802_1P_PRIO_ENABLE, true);
 
-	if (port < dev->phy_port_cnt) {
+	if (dev->info->internal_phy[port]) {
 		/* do not force flow control */
 		ksz_port_cfg(dev, port, REG_PORT_CTRL_0,
 			     PORT_FORCE_TX_FLOW_CTRL | PORT_FORCE_RX_FLOW_CTRL,
@@ -999,7 +1042,7 @@ void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 	ksz9477_cfg_port_member(dev, port, member);
 
 	/* clear pending interrupts */
-	if (port < dev->phy_port_cnt)
+	if (dev->info->internal_phy[port])
 		ksz_pread16(dev, port, REG_PORT_PHY_INT_ENABLE, &data16);
 }
 
@@ -1051,25 +1094,13 @@ void ksz9477_config_cpu_port(struct dsa_switch *ds)
 
 			/* enable cpu port */
 			ksz9477_port_setup(dev, i, true);
-			p->on = 1;
 		}
 	}
 
 	for (i = 0; i < dev->info->port_cnt; i++) {
 		if (i == dev->cpu_port)
 			continue;
-		p = &dev->ports[i];
-
 		ksz_port_stp_state_set(ds, i, BR_STATE_DISABLED);
-		p->on = 1;
-		if (i < dev->phy_port_cnt)
-			p->phy = 1;
-		if (dev->chip_id == 0x00947700 && i == 6) {
-			p->sgmii = 1;
-
-			/* SGMII PHY detection code is not implemented yet. */
-			p->phy = 0;
-		}
 	}
 }
 
@@ -1158,29 +1189,6 @@ int ksz9477_switch_init(struct ksz_device *dev)
 	if (ret)
 		return ret;
 
-	ret = ksz_read8(dev, REG_GLOBAL_OPTIONS, &data8);
-	if (ret)
-		return ret;
-
-	/* Number of ports can be reduced depending on chip. */
-	dev->phy_port_cnt = 5;
-
-	/* Default capability is gigabit capable. */
-	dev->features = GBIT_SUPPORT;
-
-	if (dev->chip_id == KSZ9893_CHIP_ID) {
-		dev->features |= IS_9893;
-
-		/* Chip does not support gigabit. */
-		if (data8 & SW_QW_ABLE)
-			dev->features &= ~GBIT_SUPPORT;
-		dev->phy_port_cnt = 2;
-	} else {
-		/* Chip does not support gigabit. */
-		if (!(data8 & SW_GIGABIT_ABLE))
-			dev->features &= ~GBIT_SUPPORT;
-	}
-
 	return 0;
 }
 
diff --git a/drivers/net/dsa/microchip/ksz9477.h b/drivers/net/dsa/microchip/ksz9477.h
index cd278b307b3c..00862c4cfb7f 100644
--- a/drivers/net/dsa/microchip/ksz9477.h
+++ b/drivers/net/dsa/microchip/ksz9477.h
@@ -16,8 +16,9 @@ u32 ksz9477_get_port_addr(int port, int offset);
 void ksz9477_cfg_port_member(struct ksz_device *dev, int port, u8 member);
 void ksz9477_flush_dyn_mac_table(struct ksz_device *dev, int port);
 void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port);
-void ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data);
-void ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val);
+int ksz9477_set_ageing_time(struct ksz_device *dev, unsigned int msecs);
+int ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data);
+int ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val);
 void ksz9477_r_mib_cnt(struct ksz_device *dev, int port, u16 addr, u64 *cnt);
 void ksz9477_r_mib_pkt(struct ksz_device *dev, int port, u16 addr,
 		       u64 *dropped, u64 *cnt);
diff --git a/drivers/net/dsa/microchip/ksz9477_i2c.c b/drivers/net/dsa/microchip/ksz9477_i2c.c
index 99966514d444..3763930dc6fc 100644
--- a/drivers/net/dsa/microchip/ksz9477_i2c.c
+++ b/drivers/net/dsa/microchip/ksz9477_i2c.c
@@ -52,16 +52,12 @@ static int ksz9477_i2c_probe(struct i2c_client *i2c,
 	return 0;
 }
 
-static int ksz9477_i2c_remove(struct i2c_client *i2c)
+static void ksz9477_i2c_remove(struct i2c_client *i2c)
 {
 	struct ksz_device *dev = i2c_get_clientdata(i2c);
 
 	if (dev)
 		ksz_switch_remove(dev);
-
-	i2c_set_clientdata(i2c, NULL);
-
-	return 0;
 }
 
 static void ksz9477_i2c_shutdown(struct i2c_client *i2c)
@@ -92,6 +88,10 @@ static const struct of_device_id ksz9477_dt_ids[] = {
 		.data = &ksz_switch_chips[KSZ9477]
 	},
 	{
+		.compatible = "microchip,ksz9896",
+		.data = &ksz_switch_chips[KSZ9896]
+	},
+	{
 		.compatible = "microchip,ksz9897",
 		.data = &ksz_switch_chips[KSZ9897]
 	},
diff --git a/drivers/net/dsa/microchip/ksz9477_reg.h b/drivers/net/dsa/microchip/ksz9477_reg.h
index ddf99d1e4bbd..53c68d286dd3 100644
--- a/drivers/net/dsa/microchip/ksz9477_reg.h
+++ b/drivers/net/dsa/microchip/ksz9477_reg.h
@@ -189,8 +189,9 @@
 
 #define SW_VLAN_ENABLE			BIT(7)
 #define SW_DROP_INVALID_VID		BIT(6)
-#define SW_AGE_CNT_M			0x7
+#define SW_AGE_CNT_M			GENMASK(5, 3)
 #define SW_AGE_CNT_S			3
+#define SW_AGE_PERIOD_10_8_M		GENMASK(10, 8)
 #define SW_RESV_MCAST_ENABLE		BIT(2)
 #define SW_HASH_OPTION_M		0x03
 #define SW_HASH_OPTION_CRC		1
@@ -225,6 +226,7 @@
 #define SW_PRIO_LOWEST_DA_SA		3
 
 #define REG_SW_LUE_CTRL_3		0x0313
+#define SW_AGE_PERIOD_7_0_M		GENMASK(7, 0)
 
 #define REG_SW_LUE_INT_STATUS		0x0314
 #define REG_SW_LUE_INT_ENABLE		0x0315
diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c
index 872aba63e7d4..d612181b3226 100644
--- a/drivers/net/dsa/microchip/ksz_common.c
+++ b/drivers/net/dsa/microchip/ksz_common.c
@@ -14,6 +14,9 @@
 #include <linux/phy.h>
 #include <linux/etherdevice.h>
 #include <linux/if_bridge.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of_mdio.h>
 #include <linux/of_device.h>
 #include <linux/of_net.h>
 #include <linux/micrel_phy.h>
@@ -183,6 +186,7 @@ static const struct ksz_dev_ops ksz9477_dev_ops = {
 	.cfg_port_member = ksz9477_cfg_port_member,
 	.flush_dyn_mac_table = ksz9477_flush_dyn_mac_table,
 	.port_setup = ksz9477_port_setup,
+	.set_ageing_time = ksz9477_set_ageing_time,
 	.r_phy = ksz9477_r_phy,
 	.w_phy = ksz9477_w_phy,
 	.r_mib_cnt = ksz9477_r_mib_cnt,
@@ -213,10 +217,12 @@ static const struct ksz_dev_ops ksz9477_dev_ops = {
 
 static const struct ksz_dev_ops lan937x_dev_ops = {
 	.setup = lan937x_setup,
+	.teardown = lan937x_teardown,
 	.get_port_addr = ksz9477_get_port_addr,
 	.cfg_port_member = ksz9477_cfg_port_member,
 	.flush_dyn_mac_table = ksz9477_flush_dyn_mac_table,
 	.port_setup = lan937x_port_setup,
+	.set_ageing_time = lan937x_set_ageing_time,
 	.r_phy = lan937x_r_phy,
 	.w_phy = lan937x_w_phy,
 	.r_mib_cnt = ksz9477_r_mib_cnt,
@@ -421,7 +427,636 @@ static const u8 lan937x_shifts[] = {
 	[ALU_STAT_INDEX]		= 8,
 };
 
+static const struct regmap_range ksz8563_valid_regs[] = {
+	regmap_reg_range(0x0000, 0x0003),
+	regmap_reg_range(0x0006, 0x0006),
+	regmap_reg_range(0x000f, 0x001f),
+	regmap_reg_range(0x0100, 0x0100),
+	regmap_reg_range(0x0104, 0x0107),
+	regmap_reg_range(0x010d, 0x010d),
+	regmap_reg_range(0x0110, 0x0113),
+	regmap_reg_range(0x0120, 0x012b),
+	regmap_reg_range(0x0201, 0x0201),
+	regmap_reg_range(0x0210, 0x0213),
+	regmap_reg_range(0x0300, 0x0300),
+	regmap_reg_range(0x0302, 0x031b),
+	regmap_reg_range(0x0320, 0x032b),
+	regmap_reg_range(0x0330, 0x0336),
+	regmap_reg_range(0x0338, 0x033e),
+	regmap_reg_range(0x0340, 0x035f),
+	regmap_reg_range(0x0370, 0x0370),
+	regmap_reg_range(0x0378, 0x0378),
+	regmap_reg_range(0x037c, 0x037d),
+	regmap_reg_range(0x0390, 0x0393),
+	regmap_reg_range(0x0400, 0x040e),
+	regmap_reg_range(0x0410, 0x042f),
+	regmap_reg_range(0x0500, 0x0519),
+	regmap_reg_range(0x0520, 0x054b),
+	regmap_reg_range(0x0550, 0x05b3),
+
+	/* port 1 */
+	regmap_reg_range(0x1000, 0x1001),
+	regmap_reg_range(0x1004, 0x100b),
+	regmap_reg_range(0x1013, 0x1013),
+	regmap_reg_range(0x1017, 0x1017),
+	regmap_reg_range(0x101b, 0x101b),
+	regmap_reg_range(0x101f, 0x1021),
+	regmap_reg_range(0x1030, 0x1030),
+	regmap_reg_range(0x1100, 0x1111),
+	regmap_reg_range(0x111a, 0x111d),
+	regmap_reg_range(0x1122, 0x1127),
+	regmap_reg_range(0x112a, 0x112b),
+	regmap_reg_range(0x1136, 0x1139),
+	regmap_reg_range(0x113e, 0x113f),
+	regmap_reg_range(0x1400, 0x1401),
+	regmap_reg_range(0x1403, 0x1403),
+	regmap_reg_range(0x1410, 0x1417),
+	regmap_reg_range(0x1420, 0x1423),
+	regmap_reg_range(0x1500, 0x1507),
+	regmap_reg_range(0x1600, 0x1612),
+	regmap_reg_range(0x1800, 0x180f),
+	regmap_reg_range(0x1900, 0x1907),
+	regmap_reg_range(0x1914, 0x191b),
+	regmap_reg_range(0x1a00, 0x1a03),
+	regmap_reg_range(0x1a04, 0x1a08),
+	regmap_reg_range(0x1b00, 0x1b01),
+	regmap_reg_range(0x1b04, 0x1b04),
+	regmap_reg_range(0x1c00, 0x1c05),
+	regmap_reg_range(0x1c08, 0x1c1b),
+
+	/* port 2 */
+	regmap_reg_range(0x2000, 0x2001),
+	regmap_reg_range(0x2004, 0x200b),
+	regmap_reg_range(0x2013, 0x2013),
+	regmap_reg_range(0x2017, 0x2017),
+	regmap_reg_range(0x201b, 0x201b),
+	regmap_reg_range(0x201f, 0x2021),
+	regmap_reg_range(0x2030, 0x2030),
+	regmap_reg_range(0x2100, 0x2111),
+	regmap_reg_range(0x211a, 0x211d),
+	regmap_reg_range(0x2122, 0x2127),
+	regmap_reg_range(0x212a, 0x212b),
+	regmap_reg_range(0x2136, 0x2139),
+	regmap_reg_range(0x213e, 0x213f),
+	regmap_reg_range(0x2400, 0x2401),
+	regmap_reg_range(0x2403, 0x2403),
+	regmap_reg_range(0x2410, 0x2417),
+	regmap_reg_range(0x2420, 0x2423),
+	regmap_reg_range(0x2500, 0x2507),
+	regmap_reg_range(0x2600, 0x2612),
+	regmap_reg_range(0x2800, 0x280f),
+	regmap_reg_range(0x2900, 0x2907),
+	regmap_reg_range(0x2914, 0x291b),
+	regmap_reg_range(0x2a00, 0x2a03),
+	regmap_reg_range(0x2a04, 0x2a08),
+	regmap_reg_range(0x2b00, 0x2b01),
+	regmap_reg_range(0x2b04, 0x2b04),
+	regmap_reg_range(0x2c00, 0x2c05),
+	regmap_reg_range(0x2c08, 0x2c1b),
+
+	/* port 3 */
+	regmap_reg_range(0x3000, 0x3001),
+	regmap_reg_range(0x3004, 0x300b),
+	regmap_reg_range(0x3013, 0x3013),
+	regmap_reg_range(0x3017, 0x3017),
+	regmap_reg_range(0x301b, 0x301b),
+	regmap_reg_range(0x301f, 0x3021),
+	regmap_reg_range(0x3030, 0x3030),
+	regmap_reg_range(0x3300, 0x3301),
+	regmap_reg_range(0x3303, 0x3303),
+	regmap_reg_range(0x3400, 0x3401),
+	regmap_reg_range(0x3403, 0x3403),
+	regmap_reg_range(0x3410, 0x3417),
+	regmap_reg_range(0x3420, 0x3423),
+	regmap_reg_range(0x3500, 0x3507),
+	regmap_reg_range(0x3600, 0x3612),
+	regmap_reg_range(0x3800, 0x380f),
+	regmap_reg_range(0x3900, 0x3907),
+	regmap_reg_range(0x3914, 0x391b),
+	regmap_reg_range(0x3a00, 0x3a03),
+	regmap_reg_range(0x3a04, 0x3a08),
+	regmap_reg_range(0x3b00, 0x3b01),
+	regmap_reg_range(0x3b04, 0x3b04),
+	regmap_reg_range(0x3c00, 0x3c05),
+	regmap_reg_range(0x3c08, 0x3c1b),
+};
+
+static const struct regmap_access_table ksz8563_register_set = {
+	.yes_ranges = ksz8563_valid_regs,
+	.n_yes_ranges = ARRAY_SIZE(ksz8563_valid_regs),
+};
+
+static const struct regmap_range ksz9477_valid_regs[] = {
+	regmap_reg_range(0x0000, 0x0003),
+	regmap_reg_range(0x0006, 0x0006),
+	regmap_reg_range(0x0010, 0x001f),
+	regmap_reg_range(0x0100, 0x0100),
+	regmap_reg_range(0x0103, 0x0107),
+	regmap_reg_range(0x010d, 0x010d),
+	regmap_reg_range(0x0110, 0x0113),
+	regmap_reg_range(0x0120, 0x012b),
+	regmap_reg_range(0x0201, 0x0201),
+	regmap_reg_range(0x0210, 0x0213),
+	regmap_reg_range(0x0300, 0x0300),
+	regmap_reg_range(0x0302, 0x031b),
+	regmap_reg_range(0x0320, 0x032b),
+	regmap_reg_range(0x0330, 0x0336),
+	regmap_reg_range(0x0338, 0x033b),
+	regmap_reg_range(0x033e, 0x033e),
+	regmap_reg_range(0x0340, 0x035f),
+	regmap_reg_range(0x0370, 0x0370),
+	regmap_reg_range(0x0378, 0x0378),
+	regmap_reg_range(0x037c, 0x037d),
+	regmap_reg_range(0x0390, 0x0393),
+	regmap_reg_range(0x0400, 0x040e),
+	regmap_reg_range(0x0410, 0x042f),
+	regmap_reg_range(0x0444, 0x044b),
+	regmap_reg_range(0x0450, 0x046f),
+	regmap_reg_range(0x0500, 0x0519),
+	regmap_reg_range(0x0520, 0x054b),
+	regmap_reg_range(0x0550, 0x05b3),
+	regmap_reg_range(0x0604, 0x060b),
+	regmap_reg_range(0x0610, 0x0612),
+	regmap_reg_range(0x0614, 0x062c),
+	regmap_reg_range(0x0640, 0x0645),
+	regmap_reg_range(0x0648, 0x064d),
+
+	/* port 1 */
+	regmap_reg_range(0x1000, 0x1001),
+	regmap_reg_range(0x1013, 0x1013),
+	regmap_reg_range(0x1017, 0x1017),
+	regmap_reg_range(0x101b, 0x101b),
+	regmap_reg_range(0x101f, 0x1020),
+	regmap_reg_range(0x1030, 0x1030),
+	regmap_reg_range(0x1100, 0x1115),
+	regmap_reg_range(0x111a, 0x111f),
+	regmap_reg_range(0x1122, 0x1127),
+	regmap_reg_range(0x112a, 0x112b),
+	regmap_reg_range(0x1136, 0x1139),
+	regmap_reg_range(0x113e, 0x113f),
+	regmap_reg_range(0x1400, 0x1401),
+	regmap_reg_range(0x1403, 0x1403),
+	regmap_reg_range(0x1410, 0x1417),
+	regmap_reg_range(0x1420, 0x1423),
+	regmap_reg_range(0x1500, 0x1507),
+	regmap_reg_range(0x1600, 0x1613),
+	regmap_reg_range(0x1800, 0x180f),
+	regmap_reg_range(0x1820, 0x1827),
+	regmap_reg_range(0x1830, 0x1837),
+	regmap_reg_range(0x1840, 0x184b),
+	regmap_reg_range(0x1900, 0x1907),
+	regmap_reg_range(0x1914, 0x191b),
+	regmap_reg_range(0x1920, 0x1920),
+	regmap_reg_range(0x1923, 0x1927),
+	regmap_reg_range(0x1a00, 0x1a03),
+	regmap_reg_range(0x1a04, 0x1a07),
+	regmap_reg_range(0x1b00, 0x1b01),
+	regmap_reg_range(0x1b04, 0x1b04),
+	regmap_reg_range(0x1c00, 0x1c05),
+	regmap_reg_range(0x1c08, 0x1c1b),
+
+	/* port 2 */
+	regmap_reg_range(0x2000, 0x2001),
+	regmap_reg_range(0x2013, 0x2013),
+	regmap_reg_range(0x2017, 0x2017),
+	regmap_reg_range(0x201b, 0x201b),
+	regmap_reg_range(0x201f, 0x2020),
+	regmap_reg_range(0x2030, 0x2030),
+	regmap_reg_range(0x2100, 0x2115),
+	regmap_reg_range(0x211a, 0x211f),
+	regmap_reg_range(0x2122, 0x2127),
+	regmap_reg_range(0x212a, 0x212b),
+	regmap_reg_range(0x2136, 0x2139),
+	regmap_reg_range(0x213e, 0x213f),
+	regmap_reg_range(0x2400, 0x2401),
+	regmap_reg_range(0x2403, 0x2403),
+	regmap_reg_range(0x2410, 0x2417),
+	regmap_reg_range(0x2420, 0x2423),
+	regmap_reg_range(0x2500, 0x2507),
+	regmap_reg_range(0x2600, 0x2613),
+	regmap_reg_range(0x2800, 0x280f),
+	regmap_reg_range(0x2820, 0x2827),
+	regmap_reg_range(0x2830, 0x2837),
+	regmap_reg_range(0x2840, 0x284b),
+	regmap_reg_range(0x2900, 0x2907),
+	regmap_reg_range(0x2914, 0x291b),
+	regmap_reg_range(0x2920, 0x2920),
+	regmap_reg_range(0x2923, 0x2927),
+	regmap_reg_range(0x2a00, 0x2a03),
+	regmap_reg_range(0x2a04, 0x2a07),
+	regmap_reg_range(0x2b00, 0x2b01),
+	regmap_reg_range(0x2b04, 0x2b04),
+	regmap_reg_range(0x2c00, 0x2c05),
+	regmap_reg_range(0x2c08, 0x2c1b),
+
+	/* port 3 */
+	regmap_reg_range(0x3000, 0x3001),
+	regmap_reg_range(0x3013, 0x3013),
+	regmap_reg_range(0x3017, 0x3017),
+	regmap_reg_range(0x301b, 0x301b),
+	regmap_reg_range(0x301f, 0x3020),
+	regmap_reg_range(0x3030, 0x3030),
+	regmap_reg_range(0x3100, 0x3115),
+	regmap_reg_range(0x311a, 0x311f),
+	regmap_reg_range(0x3122, 0x3127),
+	regmap_reg_range(0x312a, 0x312b),
+	regmap_reg_range(0x3136, 0x3139),
+	regmap_reg_range(0x313e, 0x313f),
+	regmap_reg_range(0x3400, 0x3401),
+	regmap_reg_range(0x3403, 0x3403),
+	regmap_reg_range(0x3410, 0x3417),
+	regmap_reg_range(0x3420, 0x3423),
+	regmap_reg_range(0x3500, 0x3507),
+	regmap_reg_range(0x3600, 0x3613),
+	regmap_reg_range(0x3800, 0x380f),
+	regmap_reg_range(0x3820, 0x3827),
+	regmap_reg_range(0x3830, 0x3837),
+	regmap_reg_range(0x3840, 0x384b),
+	regmap_reg_range(0x3900, 0x3907),
+	regmap_reg_range(0x3914, 0x391b),
+	regmap_reg_range(0x3920, 0x3920),
+	regmap_reg_range(0x3923, 0x3927),
+	regmap_reg_range(0x3a00, 0x3a03),
+	regmap_reg_range(0x3a04, 0x3a07),
+	regmap_reg_range(0x3b00, 0x3b01),
+	regmap_reg_range(0x3b04, 0x3b04),
+	regmap_reg_range(0x3c00, 0x3c05),
+	regmap_reg_range(0x3c08, 0x3c1b),
+
+	/* port 4 */
+	regmap_reg_range(0x4000, 0x4001),
+	regmap_reg_range(0x4013, 0x4013),
+	regmap_reg_range(0x4017, 0x4017),
+	regmap_reg_range(0x401b, 0x401b),
+	regmap_reg_range(0x401f, 0x4020),
+	regmap_reg_range(0x4030, 0x4030),
+	regmap_reg_range(0x4100, 0x4115),
+	regmap_reg_range(0x411a, 0x411f),
+	regmap_reg_range(0x4122, 0x4127),
+	regmap_reg_range(0x412a, 0x412b),
+	regmap_reg_range(0x4136, 0x4139),
+	regmap_reg_range(0x413e, 0x413f),
+	regmap_reg_range(0x4400, 0x4401),
+	regmap_reg_range(0x4403, 0x4403),
+	regmap_reg_range(0x4410, 0x4417),
+	regmap_reg_range(0x4420, 0x4423),
+	regmap_reg_range(0x4500, 0x4507),
+	regmap_reg_range(0x4600, 0x4613),
+	regmap_reg_range(0x4800, 0x480f),
+	regmap_reg_range(0x4820, 0x4827),
+	regmap_reg_range(0x4830, 0x4837),
+	regmap_reg_range(0x4840, 0x484b),
+	regmap_reg_range(0x4900, 0x4907),
+	regmap_reg_range(0x4914, 0x491b),
+	regmap_reg_range(0x4920, 0x4920),
+	regmap_reg_range(0x4923, 0x4927),
+	regmap_reg_range(0x4a00, 0x4a03),
+	regmap_reg_range(0x4a04, 0x4a07),
+	regmap_reg_range(0x4b00, 0x4b01),
+	regmap_reg_range(0x4b04, 0x4b04),
+	regmap_reg_range(0x4c00, 0x4c05),
+	regmap_reg_range(0x4c08, 0x4c1b),
+
+	/* port 5 */
+	regmap_reg_range(0x5000, 0x5001),
+	regmap_reg_range(0x5013, 0x5013),
+	regmap_reg_range(0x5017, 0x5017),
+	regmap_reg_range(0x501b, 0x501b),
+	regmap_reg_range(0x501f, 0x5020),
+	regmap_reg_range(0x5030, 0x5030),
+	regmap_reg_range(0x5100, 0x5115),
+	regmap_reg_range(0x511a, 0x511f),
+	regmap_reg_range(0x5122, 0x5127),
+	regmap_reg_range(0x512a, 0x512b),
+	regmap_reg_range(0x5136, 0x5139),
+	regmap_reg_range(0x513e, 0x513f),
+	regmap_reg_range(0x5400, 0x5401),
+	regmap_reg_range(0x5403, 0x5403),
+	regmap_reg_range(0x5410, 0x5417),
+	regmap_reg_range(0x5420, 0x5423),
+	regmap_reg_range(0x5500, 0x5507),
+	regmap_reg_range(0x5600, 0x5613),
+	regmap_reg_range(0x5800, 0x580f),
+	regmap_reg_range(0x5820, 0x5827),
+	regmap_reg_range(0x5830, 0x5837),
+	regmap_reg_range(0x5840, 0x584b),
+	regmap_reg_range(0x5900, 0x5907),
+	regmap_reg_range(0x5914, 0x591b),
+	regmap_reg_range(0x5920, 0x5920),
+	regmap_reg_range(0x5923, 0x5927),
+	regmap_reg_range(0x5a00, 0x5a03),
+	regmap_reg_range(0x5a04, 0x5a07),
+	regmap_reg_range(0x5b00, 0x5b01),
+	regmap_reg_range(0x5b04, 0x5b04),
+	regmap_reg_range(0x5c00, 0x5c05),
+	regmap_reg_range(0x5c08, 0x5c1b),
+
+	/* port 6 */
+	regmap_reg_range(0x6000, 0x6001),
+	regmap_reg_range(0x6013, 0x6013),
+	regmap_reg_range(0x6017, 0x6017),
+	regmap_reg_range(0x601b, 0x601b),
+	regmap_reg_range(0x601f, 0x6020),
+	regmap_reg_range(0x6030, 0x6030),
+	regmap_reg_range(0x6300, 0x6301),
+	regmap_reg_range(0x6400, 0x6401),
+	regmap_reg_range(0x6403, 0x6403),
+	regmap_reg_range(0x6410, 0x6417),
+	regmap_reg_range(0x6420, 0x6423),
+	regmap_reg_range(0x6500, 0x6507),
+	regmap_reg_range(0x6600, 0x6613),
+	regmap_reg_range(0x6800, 0x680f),
+	regmap_reg_range(0x6820, 0x6827),
+	regmap_reg_range(0x6830, 0x6837),
+	regmap_reg_range(0x6840, 0x684b),
+	regmap_reg_range(0x6900, 0x6907),
+	regmap_reg_range(0x6914, 0x691b),
+	regmap_reg_range(0x6920, 0x6920),
+	regmap_reg_range(0x6923, 0x6927),
+	regmap_reg_range(0x6a00, 0x6a03),
+	regmap_reg_range(0x6a04, 0x6a07),
+	regmap_reg_range(0x6b00, 0x6b01),
+	regmap_reg_range(0x6b04, 0x6b04),
+	regmap_reg_range(0x6c00, 0x6c05),
+	regmap_reg_range(0x6c08, 0x6c1b),
+
+	/* port 7 */
+	regmap_reg_range(0x7000, 0x7001),
+	regmap_reg_range(0x7013, 0x7013),
+	regmap_reg_range(0x7017, 0x7017),
+	regmap_reg_range(0x701b, 0x701b),
+	regmap_reg_range(0x701f, 0x7020),
+	regmap_reg_range(0x7030, 0x7030),
+	regmap_reg_range(0x7200, 0x7203),
+	regmap_reg_range(0x7206, 0x7207),
+	regmap_reg_range(0x7300, 0x7301),
+	regmap_reg_range(0x7400, 0x7401),
+	regmap_reg_range(0x7403, 0x7403),
+	regmap_reg_range(0x7410, 0x7417),
+	regmap_reg_range(0x7420, 0x7423),
+	regmap_reg_range(0x7500, 0x7507),
+	regmap_reg_range(0x7600, 0x7613),
+	regmap_reg_range(0x7800, 0x780f),
+	regmap_reg_range(0x7820, 0x7827),
+	regmap_reg_range(0x7830, 0x7837),
+	regmap_reg_range(0x7840, 0x784b),
+	regmap_reg_range(0x7900, 0x7907),
+	regmap_reg_range(0x7914, 0x791b),
+	regmap_reg_range(0x7920, 0x7920),
+	regmap_reg_range(0x7923, 0x7927),
+	regmap_reg_range(0x7a00, 0x7a03),
+	regmap_reg_range(0x7a04, 0x7a07),
+	regmap_reg_range(0x7b00, 0x7b01),
+	regmap_reg_range(0x7b04, 0x7b04),
+	regmap_reg_range(0x7c00, 0x7c05),
+	regmap_reg_range(0x7c08, 0x7c1b),
+};
+
+static const struct regmap_access_table ksz9477_register_set = {
+	.yes_ranges = ksz9477_valid_regs,
+	.n_yes_ranges = ARRAY_SIZE(ksz9477_valid_regs),
+};
+
+static const struct regmap_range ksz9896_valid_regs[] = {
+	regmap_reg_range(0x0000, 0x0003),
+	regmap_reg_range(0x0006, 0x0006),
+	regmap_reg_range(0x0010, 0x001f),
+	regmap_reg_range(0x0100, 0x0100),
+	regmap_reg_range(0x0103, 0x0107),
+	regmap_reg_range(0x010d, 0x010d),
+	regmap_reg_range(0x0110, 0x0113),
+	regmap_reg_range(0x0120, 0x0127),
+	regmap_reg_range(0x0201, 0x0201),
+	regmap_reg_range(0x0210, 0x0213),
+	regmap_reg_range(0x0300, 0x0300),
+	regmap_reg_range(0x0302, 0x030b),
+	regmap_reg_range(0x0310, 0x031b),
+	regmap_reg_range(0x0320, 0x032b),
+	regmap_reg_range(0x0330, 0x0336),
+	regmap_reg_range(0x0338, 0x033b),
+	regmap_reg_range(0x033e, 0x033e),
+	regmap_reg_range(0x0340, 0x035f),
+	regmap_reg_range(0x0370, 0x0370),
+	regmap_reg_range(0x0378, 0x0378),
+	regmap_reg_range(0x037c, 0x037d),
+	regmap_reg_range(0x0390, 0x0393),
+	regmap_reg_range(0x0400, 0x040e),
+	regmap_reg_range(0x0410, 0x042f),
+
+	/* port 1 */
+	regmap_reg_range(0x1000, 0x1001),
+	regmap_reg_range(0x1013, 0x1013),
+	regmap_reg_range(0x1017, 0x1017),
+	regmap_reg_range(0x101b, 0x101b),
+	regmap_reg_range(0x101f, 0x1020),
+	regmap_reg_range(0x1030, 0x1030),
+	regmap_reg_range(0x1100, 0x1115),
+	regmap_reg_range(0x111a, 0x111f),
+	regmap_reg_range(0x1122, 0x1127),
+	regmap_reg_range(0x112a, 0x112b),
+	regmap_reg_range(0x1136, 0x1139),
+	regmap_reg_range(0x113e, 0x113f),
+	regmap_reg_range(0x1400, 0x1401),
+	regmap_reg_range(0x1403, 0x1403),
+	regmap_reg_range(0x1410, 0x1417),
+	regmap_reg_range(0x1420, 0x1423),
+	regmap_reg_range(0x1500, 0x1507),
+	regmap_reg_range(0x1600, 0x1612),
+	regmap_reg_range(0x1800, 0x180f),
+	regmap_reg_range(0x1820, 0x1827),
+	regmap_reg_range(0x1830, 0x1837),
+	regmap_reg_range(0x1840, 0x184b),
+	regmap_reg_range(0x1900, 0x1907),
+	regmap_reg_range(0x1914, 0x1915),
+	regmap_reg_range(0x1a00, 0x1a03),
+	regmap_reg_range(0x1a04, 0x1a07),
+	regmap_reg_range(0x1b00, 0x1b01),
+	regmap_reg_range(0x1b04, 0x1b04),
+
+	/* port 2 */
+	regmap_reg_range(0x2000, 0x2001),
+	regmap_reg_range(0x2013, 0x2013),
+	regmap_reg_range(0x2017, 0x2017),
+	regmap_reg_range(0x201b, 0x201b),
+	regmap_reg_range(0x201f, 0x2020),
+	regmap_reg_range(0x2030, 0x2030),
+	regmap_reg_range(0x2100, 0x2115),
+	regmap_reg_range(0x211a, 0x211f),
+	regmap_reg_range(0x2122, 0x2127),
+	regmap_reg_range(0x212a, 0x212b),
+	regmap_reg_range(0x2136, 0x2139),
+	regmap_reg_range(0x213e, 0x213f),
+	regmap_reg_range(0x2400, 0x2401),
+	regmap_reg_range(0x2403, 0x2403),
+	regmap_reg_range(0x2410, 0x2417),
+	regmap_reg_range(0x2420, 0x2423),
+	regmap_reg_range(0x2500, 0x2507),
+	regmap_reg_range(0x2600, 0x2612),
+	regmap_reg_range(0x2800, 0x280f),
+	regmap_reg_range(0x2820, 0x2827),
+	regmap_reg_range(0x2830, 0x2837),
+	regmap_reg_range(0x2840, 0x284b),
+	regmap_reg_range(0x2900, 0x2907),
+	regmap_reg_range(0x2914, 0x2915),
+	regmap_reg_range(0x2a00, 0x2a03),
+	regmap_reg_range(0x2a04, 0x2a07),
+	regmap_reg_range(0x2b00, 0x2b01),
+	regmap_reg_range(0x2b04, 0x2b04),
+
+	/* port 3 */
+	regmap_reg_range(0x3000, 0x3001),
+	regmap_reg_range(0x3013, 0x3013),
+	regmap_reg_range(0x3017, 0x3017),
+	regmap_reg_range(0x301b, 0x301b),
+	regmap_reg_range(0x301f, 0x3020),
+	regmap_reg_range(0x3030, 0x3030),
+	regmap_reg_range(0x3100, 0x3115),
+	regmap_reg_range(0x311a, 0x311f),
+	regmap_reg_range(0x3122, 0x3127),
+	regmap_reg_range(0x312a, 0x312b),
+	regmap_reg_range(0x3136, 0x3139),
+	regmap_reg_range(0x313e, 0x313f),
+	regmap_reg_range(0x3400, 0x3401),
+	regmap_reg_range(0x3403, 0x3403),
+	regmap_reg_range(0x3410, 0x3417),
+	regmap_reg_range(0x3420, 0x3423),
+	regmap_reg_range(0x3500, 0x3507),
+	regmap_reg_range(0x3600, 0x3612),
+	regmap_reg_range(0x3800, 0x380f),
+	regmap_reg_range(0x3820, 0x3827),
+	regmap_reg_range(0x3830, 0x3837),
+	regmap_reg_range(0x3840, 0x384b),
+	regmap_reg_range(0x3900, 0x3907),
+	regmap_reg_range(0x3914, 0x3915),
+	regmap_reg_range(0x3a00, 0x3a03),
+	regmap_reg_range(0x3a04, 0x3a07),
+	regmap_reg_range(0x3b00, 0x3b01),
+	regmap_reg_range(0x3b04, 0x3b04),
+
+	/* port 4 */
+	regmap_reg_range(0x4000, 0x4001),
+	regmap_reg_range(0x4013, 0x4013),
+	regmap_reg_range(0x4017, 0x4017),
+	regmap_reg_range(0x401b, 0x401b),
+	regmap_reg_range(0x401f, 0x4020),
+	regmap_reg_range(0x4030, 0x4030),
+	regmap_reg_range(0x4100, 0x4115),
+	regmap_reg_range(0x411a, 0x411f),
+	regmap_reg_range(0x4122, 0x4127),
+	regmap_reg_range(0x412a, 0x412b),
+	regmap_reg_range(0x4136, 0x4139),
+	regmap_reg_range(0x413e, 0x413f),
+	regmap_reg_range(0x4400, 0x4401),
+	regmap_reg_range(0x4403, 0x4403),
+	regmap_reg_range(0x4410, 0x4417),
+	regmap_reg_range(0x4420, 0x4423),
+	regmap_reg_range(0x4500, 0x4507),
+	regmap_reg_range(0x4600, 0x4612),
+	regmap_reg_range(0x4800, 0x480f),
+	regmap_reg_range(0x4820, 0x4827),
+	regmap_reg_range(0x4830, 0x4837),
+	regmap_reg_range(0x4840, 0x484b),
+	regmap_reg_range(0x4900, 0x4907),
+	regmap_reg_range(0x4914, 0x4915),
+	regmap_reg_range(0x4a00, 0x4a03),
+	regmap_reg_range(0x4a04, 0x4a07),
+	regmap_reg_range(0x4b00, 0x4b01),
+	regmap_reg_range(0x4b04, 0x4b04),
+
+	/* port 5 */
+	regmap_reg_range(0x5000, 0x5001),
+	regmap_reg_range(0x5013, 0x5013),
+	regmap_reg_range(0x5017, 0x5017),
+	regmap_reg_range(0x501b, 0x501b),
+	regmap_reg_range(0x501f, 0x5020),
+	regmap_reg_range(0x5030, 0x5030),
+	regmap_reg_range(0x5100, 0x5115),
+	regmap_reg_range(0x511a, 0x511f),
+	regmap_reg_range(0x5122, 0x5127),
+	regmap_reg_range(0x512a, 0x512b),
+	regmap_reg_range(0x5136, 0x5139),
+	regmap_reg_range(0x513e, 0x513f),
+	regmap_reg_range(0x5400, 0x5401),
+	regmap_reg_range(0x5403, 0x5403),
+	regmap_reg_range(0x5410, 0x5417),
+	regmap_reg_range(0x5420, 0x5423),
+	regmap_reg_range(0x5500, 0x5507),
+	regmap_reg_range(0x5600, 0x5612),
+	regmap_reg_range(0x5800, 0x580f),
+	regmap_reg_range(0x5820, 0x5827),
+	regmap_reg_range(0x5830, 0x5837),
+	regmap_reg_range(0x5840, 0x584b),
+	regmap_reg_range(0x5900, 0x5907),
+	regmap_reg_range(0x5914, 0x5915),
+	regmap_reg_range(0x5a00, 0x5a03),
+	regmap_reg_range(0x5a04, 0x5a07),
+	regmap_reg_range(0x5b00, 0x5b01),
+	regmap_reg_range(0x5b04, 0x5b04),
+
+	/* port 6 */
+	regmap_reg_range(0x6000, 0x6001),
+	regmap_reg_range(0x6013, 0x6013),
+	regmap_reg_range(0x6017, 0x6017),
+	regmap_reg_range(0x601b, 0x601b),
+	regmap_reg_range(0x601f, 0x6020),
+	regmap_reg_range(0x6030, 0x6030),
+	regmap_reg_range(0x6100, 0x6115),
+	regmap_reg_range(0x611a, 0x611f),
+	regmap_reg_range(0x6122, 0x6127),
+	regmap_reg_range(0x612a, 0x612b),
+	regmap_reg_range(0x6136, 0x6139),
+	regmap_reg_range(0x613e, 0x613f),
+	regmap_reg_range(0x6300, 0x6301),
+	regmap_reg_range(0x6400, 0x6401),
+	regmap_reg_range(0x6403, 0x6403),
+	regmap_reg_range(0x6410, 0x6417),
+	regmap_reg_range(0x6420, 0x6423),
+	regmap_reg_range(0x6500, 0x6507),
+	regmap_reg_range(0x6600, 0x6612),
+	regmap_reg_range(0x6800, 0x680f),
+	regmap_reg_range(0x6820, 0x6827),
+	regmap_reg_range(0x6830, 0x6837),
+	regmap_reg_range(0x6840, 0x684b),
+	regmap_reg_range(0x6900, 0x6907),
+	regmap_reg_range(0x6914, 0x6915),
+	regmap_reg_range(0x6a00, 0x6a03),
+	regmap_reg_range(0x6a04, 0x6a07),
+	regmap_reg_range(0x6b00, 0x6b01),
+	regmap_reg_range(0x6b04, 0x6b04),
+};
+
+static const struct regmap_access_table ksz9896_register_set = {
+	.yes_ranges = ksz9896_valid_regs,
+	.n_yes_ranges = ARRAY_SIZE(ksz9896_valid_regs),
+};
+
 const struct ksz_chip_data ksz_switch_chips[] = {
+	[KSZ8563] = {
+		.chip_id = KSZ8563_CHIP_ID,
+		.dev_name = "KSZ8563",
+		.num_vlans = 4096,
+		.num_alus = 4096,
+		.num_statics = 16,
+		.cpu_ports = 0x07,	/* can be configured as cpu port */
+		.port_cnt = 3,		/* total port count */
+		.ops = &ksz9477_dev_ops,
+		.mib_names = ksz9477_mib_names,
+		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
+		.reg_mib_cnt = MIB_COUNTER_NUM,
+		.regs = ksz9477_regs,
+		.masks = ksz9477_masks,
+		.shifts = ksz9477_shifts,
+		.xmii_ctrl0 = ksz9477_xmii_ctrl0,
+		.xmii_ctrl1 = ksz8795_xmii_ctrl1, /* Same as ksz8795 */
+		.supports_mii = {false, false, true},
+		.supports_rmii = {false, false, true},
+		.supports_rgmii = {false, false, true},
+		.internal_phy = {true, true, false},
+		.gbit_capable = {false, false, true},
+		.wr_table = &ksz8563_register_set,
+		.rd_table = &ksz8563_register_set,
+	},
+
 	[KSZ8795] = {
 		.chip_id = KSZ8795_CHIP_ID,
 		.dev_name = "KSZ8795",
@@ -536,6 +1171,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_statics = 16,
 		.cpu_ports = 0x7F,	/* can be configured as cpu port */
 		.port_cnt = 7,		/* total physical port count */
+		.port_nirqs = 4,
 		.ops = &ksz9477_dev_ops,
 		.phy_errata_9477 = true,
 		.mib_names = ksz9477_mib_names,
@@ -554,6 +1190,41 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 				   false, true, false},
 		.internal_phy	= {true, true, true, true,
 				   true, false, false},
+		.gbit_capable	= {true, true, true, true, true, true, true},
+		.wr_table = &ksz9477_register_set,
+		.rd_table = &ksz9477_register_set,
+	},
+
+	[KSZ9896] = {
+		.chip_id = KSZ9896_CHIP_ID,
+		.dev_name = "KSZ9896",
+		.num_vlans = 4096,
+		.num_alus = 4096,
+		.num_statics = 16,
+		.cpu_ports = 0x3F,	/* can be configured as cpu port */
+		.port_cnt = 6,		/* total physical port count */
+		.port_nirqs = 2,
+		.ops = &ksz9477_dev_ops,
+		.phy_errata_9477 = true,
+		.mib_names = ksz9477_mib_names,
+		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
+		.reg_mib_cnt = MIB_COUNTER_NUM,
+		.regs = ksz9477_regs,
+		.masks = ksz9477_masks,
+		.shifts = ksz9477_shifts,
+		.xmii_ctrl0 = ksz9477_xmii_ctrl0,
+		.xmii_ctrl1 = ksz9477_xmii_ctrl1,
+		.supports_mii	= {false, false, false, false,
+				   false, true},
+		.supports_rmii	= {false, false, false, false,
+				   false, true},
+		.supports_rgmii = {false, false, false, false,
+				   false, true},
+		.internal_phy	= {true, true, true, true,
+				   true, false},
+		.gbit_capable	= {true, true, true, true, true, true},
+		.wr_table = &ksz9896_register_set,
+		.rd_table = &ksz9896_register_set,
 	},
 
 	[KSZ9897] = {
@@ -564,6 +1235,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_statics = 16,
 		.cpu_ports = 0x7F,	/* can be configured as cpu port */
 		.port_cnt = 7,		/* total physical port count */
+		.port_nirqs = 2,
 		.ops = &ksz9477_dev_ops,
 		.phy_errata_9477 = true,
 		.mib_names = ksz9477_mib_names,
@@ -582,6 +1254,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 				   false, true, true},
 		.internal_phy	= {true, true, true, true,
 				   true, false, false},
+		.gbit_capable	= {true, true, true, true, true, true, true},
 	},
 
 	[KSZ9893] = {
@@ -592,6 +1265,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_statics = 16,
 		.cpu_ports = 0x07,	/* can be configured as cpu port */
 		.port_cnt = 3,		/* total port count */
+		.port_nirqs = 2,
 		.ops = &ksz9477_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -605,6 +1279,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.supports_rmii = {false, false, true},
 		.supports_rgmii = {false, false, true},
 		.internal_phy = {true, true, false},
+		.gbit_capable = {true, true, true},
 	},
 
 	[KSZ9567] = {
@@ -615,6 +1290,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_statics = 16,
 		.cpu_ports = 0x7F,	/* can be configured as cpu port */
 		.port_cnt = 7,		/* total physical port count */
+		.port_nirqs = 3,
 		.ops = &ksz9477_dev_ops,
 		.phy_errata_9477 = true,
 		.mib_names = ksz9477_mib_names,
@@ -633,6 +1309,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 				   false, true, true},
 		.internal_phy	= {true, true, true, true,
 				   true, false, false},
+		.gbit_capable	= {true, true, true, true, true, true, true},
 	},
 
 	[LAN9370] = {
@@ -643,6 +1320,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_statics = 256,
 		.cpu_ports = 0x10,	/* can be configured as cpu port */
 		.port_cnt = 5,		/* total physical port count */
+		.port_nirqs = 6,
 		.ops = &lan937x_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -666,6 +1344,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_statics = 256,
 		.cpu_ports = 0x30,	/* can be configured as cpu port */
 		.port_cnt = 6,		/* total physical port count */
+		.port_nirqs = 6,
 		.ops = &lan937x_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -689,6 +1368,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_statics = 256,
 		.cpu_ports = 0x30,	/* can be configured as cpu port */
 		.port_cnt = 8,		/* total physical port count */
+		.port_nirqs = 6,
 		.ops = &lan937x_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -716,6 +1396,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_statics = 256,
 		.cpu_ports = 0x38,	/* can be configured as cpu port */
 		.port_cnt = 5,		/* total physical port count */
+		.port_nirqs = 6,
 		.ops = &lan937x_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -743,6 +1424,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_statics = 256,
 		.cpu_ports = 0x30,	/* can be configured as cpu port */
 		.port_cnt = 8,		/* total physical port count */
+		.port_nirqs = 6,
 		.ops = &lan937x_dev_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -974,9 +1656,280 @@ static void ksz_update_port_member(struct ksz_device *dev, int port)
 	dev->dev_ops->cfg_port_member(dev, port, port_member | cpu_port);
 }
 
+static int ksz_sw_mdio_read(struct mii_bus *bus, int addr, int regnum)
+{
+	struct ksz_device *dev = bus->priv;
+	u16 val;
+	int ret;
+
+	if (regnum & MII_ADDR_C45)
+		return -EOPNOTSUPP;
+
+	ret = dev->dev_ops->r_phy(dev, addr, regnum, &val);
+	if (ret < 0)
+		return ret;
+
+	return val;
+}
+
+static int ksz_sw_mdio_write(struct mii_bus *bus, int addr, int regnum,
+			     u16 val)
+{
+	struct ksz_device *dev = bus->priv;
+
+	if (regnum & MII_ADDR_C45)
+		return -EOPNOTSUPP;
+
+	return dev->dev_ops->w_phy(dev, addr, regnum, val);
+}
+
+static int ksz_irq_phy_setup(struct ksz_device *dev)
+{
+	struct dsa_switch *ds = dev->ds;
+	int phy;
+	int irq;
+	int ret;
+
+	for (phy = 0; phy < KSZ_MAX_NUM_PORTS; phy++) {
+		if (BIT(phy) & ds->phys_mii_mask) {
+			irq = irq_find_mapping(dev->ports[phy].pirq.domain,
+					       PORT_SRC_PHY_INT);
+			if (irq < 0) {
+				ret = irq;
+				goto out;
+			}
+			ds->slave_mii_bus->irq[phy] = irq;
+		}
+	}
+	return 0;
+out:
+	while (phy--)
+		if (BIT(phy) & ds->phys_mii_mask)
+			irq_dispose_mapping(ds->slave_mii_bus->irq[phy]);
+
+	return ret;
+}
+
+static void ksz_irq_phy_free(struct ksz_device *dev)
+{
+	struct dsa_switch *ds = dev->ds;
+	int phy;
+
+	for (phy = 0; phy < KSZ_MAX_NUM_PORTS; phy++)
+		if (BIT(phy) & ds->phys_mii_mask)
+			irq_dispose_mapping(ds->slave_mii_bus->irq[phy]);
+}
+
+static int ksz_mdio_register(struct ksz_device *dev)
+{
+	struct dsa_switch *ds = dev->ds;
+	struct device_node *mdio_np;
+	struct mii_bus *bus;
+	int ret;
+
+	mdio_np = of_get_child_by_name(dev->dev->of_node, "mdio");
+	if (!mdio_np)
+		return 0;
+
+	bus = devm_mdiobus_alloc(ds->dev);
+	if (!bus) {
+		of_node_put(mdio_np);
+		return -ENOMEM;
+	}
+
+	bus->priv = dev;
+	bus->read = ksz_sw_mdio_read;
+	bus->write = ksz_sw_mdio_write;
+	bus->name = "ksz slave smi";
+	snprintf(bus->id, MII_BUS_ID_SIZE, "SMI-%d", ds->index);
+	bus->parent = ds->dev;
+	bus->phy_mask = ~ds->phys_mii_mask;
+
+	ds->slave_mii_bus = bus;
+
+	if (dev->irq > 0) {
+		ret = ksz_irq_phy_setup(dev);
+		if (ret) {
+			of_node_put(mdio_np);
+			return ret;
+		}
+	}
+
+	ret = devm_of_mdiobus_register(ds->dev, bus, mdio_np);
+	if (ret) {
+		dev_err(ds->dev, "unable to register MDIO bus %s\n",
+			bus->id);
+		if (dev->irq > 0)
+			ksz_irq_phy_free(dev);
+	}
+
+	of_node_put(mdio_np);
+
+	return ret;
+}
+
+static void ksz_irq_mask(struct irq_data *d)
+{
+	struct ksz_irq *kirq = irq_data_get_irq_chip_data(d);
+
+	kirq->masked |= BIT(d->hwirq);
+}
+
+static void ksz_irq_unmask(struct irq_data *d)
+{
+	struct ksz_irq *kirq = irq_data_get_irq_chip_data(d);
+
+	kirq->masked &= ~BIT(d->hwirq);
+}
+
+static void ksz_irq_bus_lock(struct irq_data *d)
+{
+	struct ksz_irq *kirq  = irq_data_get_irq_chip_data(d);
+
+	mutex_lock(&kirq->dev->lock_irq);
+}
+
+static void ksz_irq_bus_sync_unlock(struct irq_data *d)
+{
+	struct ksz_irq *kirq  = irq_data_get_irq_chip_data(d);
+	struct ksz_device *dev = kirq->dev;
+	int ret;
+
+	ret = ksz_write32(dev, kirq->reg_mask, kirq->masked);
+	if (ret)
+		dev_err(dev->dev, "failed to change IRQ mask\n");
+
+	mutex_unlock(&dev->lock_irq);
+}
+
+static const struct irq_chip ksz_irq_chip = {
+	.name			= "ksz-irq",
+	.irq_mask		= ksz_irq_mask,
+	.irq_unmask		= ksz_irq_unmask,
+	.irq_bus_lock		= ksz_irq_bus_lock,
+	.irq_bus_sync_unlock	= ksz_irq_bus_sync_unlock,
+};
+
+static int ksz_irq_domain_map(struct irq_domain *d,
+			      unsigned int irq, irq_hw_number_t hwirq)
+{
+	irq_set_chip_data(irq, d->host_data);
+	irq_set_chip_and_handler(irq, &ksz_irq_chip, handle_level_irq);
+	irq_set_noprobe(irq);
+
+	return 0;
+}
+
+static const struct irq_domain_ops ksz_irq_domain_ops = {
+	.map	= ksz_irq_domain_map,
+	.xlate	= irq_domain_xlate_twocell,
+};
+
+static void ksz_irq_free(struct ksz_irq *kirq)
+{
+	int irq, virq;
+
+	free_irq(kirq->irq_num, kirq);
+
+	for (irq = 0; irq < kirq->nirqs; irq++) {
+		virq = irq_find_mapping(kirq->domain, irq);
+		irq_dispose_mapping(virq);
+	}
+
+	irq_domain_remove(kirq->domain);
+}
+
+static irqreturn_t ksz_irq_thread_fn(int irq, void *dev_id)
+{
+	struct ksz_irq *kirq = dev_id;
+	unsigned int nhandled = 0;
+	struct ksz_device *dev;
+	unsigned int sub_irq;
+	u8 data;
+	int ret;
+	u8 n;
+
+	dev = kirq->dev;
+
+	/* Read interrupt status register */
+	ret = ksz_read8(dev, kirq->reg_status, &data);
+	if (ret)
+		goto out;
+
+	for (n = 0; n < kirq->nirqs; ++n) {
+		if (data & BIT(n)) {
+			sub_irq = irq_find_mapping(kirq->domain, n);
+			handle_nested_irq(sub_irq);
+			++nhandled;
+		}
+	}
+out:
+	return (nhandled > 0 ? IRQ_HANDLED : IRQ_NONE);
+}
+
+static int ksz_irq_common_setup(struct ksz_device *dev, struct ksz_irq *kirq)
+{
+	int ret, n;
+
+	kirq->dev = dev;
+	kirq->masked = ~0;
+
+	kirq->domain = irq_domain_add_simple(dev->dev->of_node, kirq->nirqs, 0,
+					     &ksz_irq_domain_ops, kirq);
+	if (!kirq->domain)
+		return -ENOMEM;
+
+	for (n = 0; n < kirq->nirqs; n++)
+		irq_create_mapping(kirq->domain, n);
+
+	ret = request_threaded_irq(kirq->irq_num, NULL, ksz_irq_thread_fn,
+				   IRQF_ONESHOT | IRQF_TRIGGER_FALLING,
+				   kirq->name, kirq);
+	if (ret)
+		goto out;
+
+	return 0;
+
+out:
+	ksz_irq_free(kirq);
+
+	return ret;
+}
+
+static int ksz_girq_setup(struct ksz_device *dev)
+{
+	struct ksz_irq *girq = &dev->girq;
+
+	girq->nirqs = dev->info->port_cnt;
+	girq->reg_mask = REG_SW_PORT_INT_MASK__1;
+	girq->reg_status = REG_SW_PORT_INT_STATUS__1;
+	snprintf(girq->name, sizeof(girq->name), "global_port_irq");
+
+	girq->irq_num = dev->irq;
+
+	return ksz_irq_common_setup(dev, girq);
+}
+
+static int ksz_pirq_setup(struct ksz_device *dev, u8 p)
+{
+	struct ksz_irq *pirq = &dev->ports[p].pirq;
+
+	pirq->nirqs = dev->info->port_nirqs;
+	pirq->reg_mask = dev->dev_ops->get_port_addr(p, REG_PORT_INT_MASK);
+	pirq->reg_status = dev->dev_ops->get_port_addr(p, REG_PORT_INT_STATUS);
+	snprintf(pirq->name, sizeof(pirq->name), "port_irq-%d", p);
+
+	pirq->irq_num = irq_find_mapping(dev->girq.domain, p);
+	if (pirq->irq_num < 0)
+		return pirq->irq_num;
+
+	return ksz_irq_common_setup(dev, pirq);
+}
+
 static int ksz_setup(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
+	struct dsa_port *dp;
 	struct ksz_port *p;
 	const u16 *regs;
 	int ret;
@@ -1025,11 +1978,55 @@ static int ksz_setup(struct dsa_switch *ds)
 	p = &dev->ports[dev->cpu_port];
 	p->learning = true;
 
+	if (dev->irq > 0) {
+		ret = ksz_girq_setup(dev);
+		if (ret)
+			return ret;
+
+		dsa_switch_for_each_user_port(dp, dev->ds) {
+			ret = ksz_pirq_setup(dev, dp->index);
+			if (ret)
+				goto out_girq;
+		}
+	}
+
+	ret = ksz_mdio_register(dev);
+	if (ret < 0) {
+		dev_err(dev->dev, "failed to register the mdio");
+		goto out_pirq;
+	}
+
 	/* start switch */
 	regmap_update_bits(dev->regmap[0], regs[S_START_CTRL],
 			   SW_START, SW_START);
 
 	return 0;
+
+out_pirq:
+	if (dev->irq > 0)
+		dsa_switch_for_each_user_port(dp, dev->ds)
+			ksz_irq_free(&dev->ports[dp->index].pirq);
+out_girq:
+	if (dev->irq > 0)
+		ksz_irq_free(&dev->girq);
+
+	return ret;
+}
+
+static void ksz_teardown(struct dsa_switch *ds)
+{
+	struct ksz_device *dev = ds->priv;
+	struct dsa_port *dp;
+
+	if (dev->irq > 0) {
+		dsa_switch_for_each_user_port(dp, dev->ds)
+			ksz_irq_free(&dev->ports[dp->index].pirq);
+
+		ksz_irq_free(&dev->girq);
+	}
+
+	if (dev->dev_ops->teardown)
+		dev->dev_ops->teardown(ds);
 }
 
 static void port_r_cnt(struct ksz_device *dev, int port)
@@ -1113,8 +2110,11 @@ static int ksz_phy_read16(struct dsa_switch *ds, int addr, int reg)
 {
 	struct ksz_device *dev = ds->priv;
 	u16 val = 0xffff;
+	int ret;
 
-	dev->dev_ops->r_phy(dev, addr, reg, &val);
+	ret = dev->dev_ops->r_phy(dev, addr, reg, &val);
+	if (ret)
+		return ret;
 
 	return val;
 }
@@ -1122,8 +2122,11 @@ static int ksz_phy_read16(struct dsa_switch *ds, int addr, int reg)
 static int ksz_phy_write16(struct dsa_switch *ds, int addr, int reg, u16 val)
 {
 	struct ksz_device *dev = ds->priv;
+	int ret;
 
-	dev->dev_ops->w_phy(dev, addr, reg, val);
+	ret = dev->dev_ops->w_phy(dev, addr, reg, val);
+	if (ret)
+		return ret;
 
 	return 0;
 }
@@ -1212,6 +2215,16 @@ static void ksz_port_fast_age(struct dsa_switch *ds, int port)
 	dev->dev_ops->flush_dyn_mac_table(dev, port);
 }
 
+static int ksz_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
+{
+	struct ksz_device *dev = ds->priv;
+
+	if (!dev->dev_ops->set_ageing_time)
+		return -EOPNOTSUPP;
+
+	return dev->dev_ops->set_ageing_time(dev, msecs);
+}
+
 static int ksz_port_fdb_add(struct dsa_switch *ds, int port,
 			    const unsigned char *addr, u16 vid,
 			    struct dsa_db db)
@@ -1375,10 +2388,12 @@ static enum dsa_tag_protocol ksz_get_tag_protocol(struct dsa_switch *ds,
 		proto = DSA_TAG_PROTO_KSZ8795;
 
 	if (dev->chip_id == KSZ8830_CHIP_ID ||
+	    dev->chip_id == KSZ8563_CHIP_ID ||
 	    dev->chip_id == KSZ9893_CHIP_ID)
 		proto = DSA_TAG_PROTO_KSZ9893;
 
 	if (dev->chip_id == KSZ9477_CHIP_ID ||
+	    dev->chip_id == KSZ9896_CHIP_ID ||
 	    dev->chip_id == KSZ9897_CHIP_ID ||
 	    dev->chip_id == KSZ9567_CHIP_ID)
 		proto = DSA_TAG_PROTO_KSZ9477;
@@ -1493,7 +2508,8 @@ static void ksz_set_xmii(struct ksz_device *dev, int port,
 	case PHY_INTERFACE_MODE_RGMII_RXID:
 		data8 |= bitval[P_RGMII_SEL];
 		/* On KSZ9893, disable RGMII in-band status support */
-		if (dev->features & IS_9893)
+		if (dev->chip_id == KSZ9893_CHIP_ID ||
+		    dev->chip_id == KSZ8563_CHIP_ID)
 			data8 &= ~P_MII_MAC_MODE;
 		break;
 	default:
@@ -1703,7 +2719,7 @@ static void ksz_phylink_mac_link_up(struct dsa_switch *ds, int port,
 
 static int ksz_switch_detect(struct ksz_device *dev)
 {
-	u8 id1, id2;
+	u8 id1, id2, id4;
 	u16 id16;
 	u32 id32;
 	int ret;
@@ -1748,8 +2764,8 @@ static int ksz_switch_detect(struct ksz_device *dev)
 
 		switch (id32) {
 		case KSZ9477_CHIP_ID:
+		case KSZ9896_CHIP_ID:
 		case KSZ9897_CHIP_ID:
-		case KSZ9893_CHIP_ID:
 		case KSZ9567_CHIP_ID:
 		case LAN9370_CHIP_ID:
 		case LAN9371_CHIP_ID:
@@ -1758,6 +2774,18 @@ static int ksz_switch_detect(struct ksz_device *dev)
 		case LAN9374_CHIP_ID:
 			dev->chip_id = id32;
 			break;
+		case KSZ9893_CHIP_ID:
+			ret = ksz_read8(dev, REG_CHIP_ID4,
+					&id4);
+			if (ret)
+				return ret;
+
+			if (id4 == SKU_ID_KSZ8563)
+				dev->chip_id = KSZ8563_CHIP_ID;
+			else
+				dev->chip_id = KSZ9893_CHIP_ID;
+
+			break;
 		default:
 			dev_err(dev->dev,
 				"unsupported switch detected %x)\n", id32);
@@ -1771,6 +2799,7 @@ static const struct dsa_switch_ops ksz_switch_ops = {
 	.get_tag_protocol	= ksz_get_tag_protocol,
 	.get_phy_flags		= ksz_get_phy_flags,
 	.setup			= ksz_setup,
+	.teardown		= ksz_teardown,
 	.phy_read		= ksz_phy_read16,
 	.phy_write		= ksz_phy_write16,
 	.phylink_get_caps	= ksz_phylink_get_caps,
@@ -1778,6 +2807,7 @@ static const struct dsa_switch_ops ksz_switch_ops = {
 	.phylink_mac_link_up	= ksz_phylink_mac_link_up,
 	.phylink_mac_link_down	= ksz_mac_link_down,
 	.port_enable		= ksz_enable_port,
+	.set_ageing_time	= ksz_set_ageing_time,
 	.get_strings		= ksz_get_strings,
 	.get_ethtool_stats	= ksz_get_ethtool_stats,
 	.get_sset_count		= ksz_sset_count,
@@ -1935,6 +2965,9 @@ int ksz_switch_register(struct ksz_device *dev)
 				     GFP_KERNEL);
 		if (!dev->ports[i].mib.counters)
 			return -ENOMEM;
+
+		dev->ports[i].ksz_dev = dev;
+		dev->ports[i].num = i;
 	}
 
 	/* set the real number of ports */
diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h
index 0d9520dc6d2d..9cfa179575ce 100644
--- a/drivers/net/dsa/microchip/ksz_common.h
+++ b/drivers/net/dsa/microchip/ksz_common.h
@@ -13,9 +13,12 @@
 #include <linux/phy.h>
 #include <linux/regmap.h>
 #include <net/dsa.h>
+#include <linux/irq.h>
 
 #define KSZ_MAX_NUM_PORTS 8
 
+struct ksz_device;
+
 struct vlan_table {
 	u32 table[3];
 };
@@ -42,6 +45,7 @@ struct ksz_chip_data {
 	int num_statics;
 	int cpu_ports;
 	int port_cnt;
+	u8 port_nirqs;
 	const struct ksz_dev_ops *ops;
 	bool phy_errata_9477;
 	bool ksz87xx_eee_link_erratum;
@@ -61,6 +65,20 @@ struct ksz_chip_data {
 	bool supports_rmii[KSZ_MAX_NUM_PORTS];
 	bool supports_rgmii[KSZ_MAX_NUM_PORTS];
 	bool internal_phy[KSZ_MAX_NUM_PORTS];
+	bool gbit_capable[KSZ_MAX_NUM_PORTS];
+	const struct regmap_access_table *wr_table;
+	const struct regmap_access_table *rd_table;
+};
+
+struct ksz_irq {
+	u16 masked;
+	u16 reg_mask;
+	u16 reg_status;
+	struct irq_domain *domain;
+	int nirqs;
+	int irq_num;
+	char name[16];
+	struct ksz_device *dev;
 };
 
 struct ksz_port {
@@ -70,9 +88,7 @@ struct ksz_port {
 	struct phy_device phydev;
 
 	u32 on:1;			/* port is not disabled by hardware */
-	u32 phy:1;			/* port has a PHY */
 	u32 fiber:1;			/* port is fiber */
-	u32 sgmii:1;			/* port is SGMII */
 	u32 force:1;
 	u32 read:1;			/* read MIB counters in background */
 	u32 freeze:1;			/* MIB counter freeze is enabled */
@@ -82,6 +98,9 @@ struct ksz_port {
 	u16 max_frame;
 	u32 rgmii_tx_val;
 	u32 rgmii_rx_val;
+	struct ksz_device *ksz_dev;
+	struct ksz_irq pirq;
+	u8 num;
 };
 
 struct ksz_device {
@@ -99,6 +118,7 @@ struct ksz_device {
 	struct regmap *regmap[3];
 
 	void *priv;
+	int irq;
 
 	struct gpio_desc *reset_gpio;	/* Optional reset GPIO */
 
@@ -118,17 +138,20 @@ struct ksz_device {
 	unsigned long mib_read_interval;
 	u16 mirror_rx;
 	u16 mirror_tx;
-	u32 features;			/* chip specific features */
 	u16 port_mask;
+	struct mutex lock_irq;		/* IRQ Access */
+	struct ksz_irq girq;
 };
 
 /* List of supported models */
 enum ksz_model {
+	KSZ8563,
 	KSZ8795,
 	KSZ8794,
 	KSZ8765,
 	KSZ8830,
 	KSZ9477,
+	KSZ9896,
 	KSZ9897,
 	KSZ9893,
 	KSZ9567,
@@ -140,11 +163,13 @@ enum ksz_model {
 };
 
 enum ksz_chip_id {
+	KSZ8563_CHIP_ID = 0x8563,
 	KSZ8795_CHIP_ID = 0x8795,
 	KSZ8794_CHIP_ID = 0x8794,
 	KSZ8765_CHIP_ID = 0x8765,
 	KSZ8830_CHIP_ID = 0x8830,
 	KSZ9477_CHIP_ID = 0x00947700,
+	KSZ9896_CHIP_ID = 0x00989600,
 	KSZ9897_CHIP_ID = 0x00989700,
 	KSZ9893_CHIP_ID = 0x00989300,
 	KSZ9567_CHIP_ID = 0x00956700,
@@ -254,13 +279,15 @@ struct alu_struct {
 
 struct ksz_dev_ops {
 	int (*setup)(struct dsa_switch *ds);
+	void (*teardown)(struct dsa_switch *ds);
 	u32 (*get_port_addr)(int port, int offset);
 	void (*cfg_port_member)(struct ksz_device *dev, int port, u8 member);
 	void (*flush_dyn_mac_table)(struct ksz_device *dev, int port);
 	void (*port_cleanup)(struct ksz_device *dev, int port);
 	void (*port_setup)(struct ksz_device *dev, int port, bool cpu_port);
-	void (*r_phy)(struct ksz_device *dev, u16 phy, u16 reg, u16 *val);
-	void (*w_phy)(struct ksz_device *dev, u16 phy, u16 reg, u16 val);
+	int (*set_ageing_time)(struct ksz_device *dev, unsigned int msecs);
+	int (*r_phy)(struct ksz_device *dev, u16 phy, u16 reg, u16 *val);
+	int (*w_phy)(struct ksz_device *dev, u16 phy, u16 reg, u16 val);
 	void (*r_mib_cnt)(struct ksz_device *dev, int port, u16 addr,
 			  u64 *cnt);
 	void (*r_mib_pkt)(struct ksz_device *dev, int port, u16 addr,
@@ -330,6 +357,10 @@ static inline int ksz_read8(struct ksz_device *dev, u32 reg, u8 *val)
 	unsigned int value;
 	int ret = regmap_read(dev->regmap[0], reg, &value);
 
+	if (ret)
+		dev_err(dev->dev, "can't read 8bit reg: 0x%x %pe\n", reg,
+			ERR_PTR(ret));
+
 	*val = value;
 	return ret;
 }
@@ -339,6 +370,10 @@ static inline int ksz_read16(struct ksz_device *dev, u32 reg, u16 *val)
 	unsigned int value;
 	int ret = regmap_read(dev->regmap[1], reg, &value);
 
+	if (ret)
+		dev_err(dev->dev, "can't read 16bit reg: 0x%x %pe\n", reg,
+			ERR_PTR(ret));
+
 	*val = value;
 	return ret;
 }
@@ -348,6 +383,10 @@ static inline int ksz_read32(struct ksz_device *dev, u32 reg, u32 *val)
 	unsigned int value;
 	int ret = regmap_read(dev->regmap[2], reg, &value);
 
+	if (ret)
+		dev_err(dev->dev, "can't read 32bit reg: 0x%x %pe\n", reg,
+			ERR_PTR(ret));
+
 	*val = value;
 	return ret;
 }
@@ -358,7 +397,10 @@ static inline int ksz_read64(struct ksz_device *dev, u32 reg, u64 *val)
 	int ret;
 
 	ret = regmap_bulk_read(dev->regmap[2], reg, value, 2);
-	if (!ret)
+	if (ret)
+		dev_err(dev->dev, "can't read 64bit reg: 0x%x %pe\n", reg,
+			ERR_PTR(ret));
+	else
 		*val = (u64)value[0] << 32 | value[1];
 
 	return ret;
@@ -366,17 +408,38 @@ static inline int ksz_read64(struct ksz_device *dev, u32 reg, u64 *val)
 
 static inline int ksz_write8(struct ksz_device *dev, u32 reg, u8 value)
 {
-	return regmap_write(dev->regmap[0], reg, value);
+	int ret;
+
+	ret = regmap_write(dev->regmap[0], reg, value);
+	if (ret)
+		dev_err(dev->dev, "can't write 8bit reg: 0x%x %pe\n", reg,
+			ERR_PTR(ret));
+
+	return ret;
 }
 
 static inline int ksz_write16(struct ksz_device *dev, u32 reg, u16 value)
 {
-	return regmap_write(dev->regmap[1], reg, value);
+	int ret;
+
+	ret = regmap_write(dev->regmap[1], reg, value);
+	if (ret)
+		dev_err(dev->dev, "can't write 16bit reg: 0x%x %pe\n", reg,
+			ERR_PTR(ret));
+
+	return ret;
 }
 
 static inline int ksz_write32(struct ksz_device *dev, u32 reg, u32 value)
 {
-	return regmap_write(dev->regmap[2], reg, value);
+	int ret;
+
+	ret = regmap_write(dev->regmap[2], reg, value);
+	if (ret)
+		dev_err(dev->dev, "can't write 32bit reg: 0x%x %pe\n", reg,
+			ERR_PTR(ret));
+
+	return ret;
 }
 
 static inline int ksz_write64(struct ksz_device *dev, u32 reg, u64 value)
@@ -391,40 +454,42 @@ static inline int ksz_write64(struct ksz_device *dev, u32 reg, u64 value)
 	return regmap_bulk_write(dev->regmap[2], reg, val, 2);
 }
 
-static inline void ksz_pread8(struct ksz_device *dev, int port, int offset,
-			      u8 *data)
+static inline int ksz_pread8(struct ksz_device *dev, int port, int offset,
+			     u8 *data)
 {
-	ksz_read8(dev, dev->dev_ops->get_port_addr(port, offset), data);
+	return ksz_read8(dev, dev->dev_ops->get_port_addr(port, offset), data);
 }
 
-static inline void ksz_pread16(struct ksz_device *dev, int port, int offset,
-			       u16 *data)
+static inline int ksz_pread16(struct ksz_device *dev, int port, int offset,
+			      u16 *data)
 {
-	ksz_read16(dev, dev->dev_ops->get_port_addr(port, offset), data);
+	return ksz_read16(dev, dev->dev_ops->get_port_addr(port, offset), data);
 }
 
-static inline void ksz_pread32(struct ksz_device *dev, int port, int offset,
-			       u32 *data)
+static inline int ksz_pread32(struct ksz_device *dev, int port, int offset,
+			      u32 *data)
 {
-	ksz_read32(dev, dev->dev_ops->get_port_addr(port, offset), data);
+	return ksz_read32(dev, dev->dev_ops->get_port_addr(port, offset), data);
 }
 
-static inline void ksz_pwrite8(struct ksz_device *dev, int port, int offset,
-			       u8 data)
+static inline int ksz_pwrite8(struct ksz_device *dev, int port, int offset,
+			      u8 data)
 {
-	ksz_write8(dev, dev->dev_ops->get_port_addr(port, offset), data);
+	return ksz_write8(dev, dev->dev_ops->get_port_addr(port, offset), data);
 }
 
-static inline void ksz_pwrite16(struct ksz_device *dev, int port, int offset,
-				u16 data)
+static inline int ksz_pwrite16(struct ksz_device *dev, int port, int offset,
+			       u16 data)
 {
-	ksz_write16(dev, dev->dev_ops->get_port_addr(port, offset), data);
+	return ksz_write16(dev, dev->dev_ops->get_port_addr(port, offset),
+			   data);
 }
 
-static inline void ksz_pwrite32(struct ksz_device *dev, int port, int offset,
-				u32 data)
+static inline int ksz_pwrite32(struct ksz_device *dev, int port, int offset,
+			       u32 data)
 {
-	ksz_write32(dev, dev->dev_ops->get_port_addr(port, offset), data);
+	return ksz_write32(dev, dev->dev_ops->get_port_addr(port, offset),
+			   data);
 }
 
 static inline void ksz_prmw8(struct ksz_device *dev, int port, int offset,
@@ -483,6 +548,10 @@ static inline int is_lan937x(struct ksz_device *dev)
 
 #define SW_REV_ID_M			GENMASK(7, 4)
 
+/* KSZ9893, KSZ9563, KSZ8563 specific register  */
+#define REG_CHIP_ID4			0x0f
+#define SKU_ID_KSZ8563			0x3c
+
 /* Driver set switch broadcast storm protection at 10% rate. */
 #define BROADCAST_STORM_PROT_RATE	10
 
@@ -497,10 +566,6 @@ static inline int is_lan937x(struct ksz_device *dev)
 
 #define SW_START			0x01
 
-/* Used with variable features to indicate capabilities. */
-#define GBIT_SUPPORT			BIT(0)
-#define IS_9893				BIT(2)
-
 /* xMII configuration */
 #define P_MII_DUPLEX_M			BIT(6)
 #define P_MII_100MBIT_M			BIT(4)
@@ -511,6 +576,15 @@ static inline int is_lan937x(struct ksz_device *dev)
 #define P_MII_MAC_MODE			BIT(2)
 #define P_MII_SEL_M			0x3
 
+/* Interrupt */
+#define REG_SW_PORT_INT_STATUS__1	0x001B
+#define REG_SW_PORT_INT_MASK__1		0x001F
+
+#define REG_PORT_INT_STATUS		0x001B
+#define REG_PORT_INT_MASK		0x001F
+
+#define PORT_SRC_PHY_INT		1
+
 /* Regmap tables generation */
 #define KSZ_SPI_OP_RD		3
 #define KSZ_SPI_OP_WR		2
diff --git a/drivers/net/dsa/microchip/ksz_spi.c b/drivers/net/dsa/microchip/ksz_spi.c
index 05bd089795f8..1b6ab891b986 100644
--- a/drivers/net/dsa/microchip/ksz_spi.c
+++ b/drivers/net/dsa/microchip/ksz_spi.c
@@ -66,7 +66,10 @@ static int ksz_spi_probe(struct spi_device *spi)
 	for (i = 0; i < ARRAY_SIZE(ksz8795_regmap_config); i++) {
 		rc = regmap_config[i];
 		rc.lock_arg = &dev->regmap_mutex;
+		rc.wr_table = chip->wr_table;
+		rc.rd_table = chip->rd_table;
 		dev->regmap[i] = devm_regmap_init_spi(spi, &rc);
+
 		if (IS_ERR(dev->regmap[i])) {
 			ret = PTR_ERR(dev->regmap[i]);
 			dev_err(&spi->dev,
@@ -85,6 +88,8 @@ static int ksz_spi_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
+	dev->irq = spi->irq;
+
 	ret = ksz_switch_register(dev);
 
 	/* Main DSA driver may not be started yet. */
@@ -102,8 +107,6 @@ static void ksz_spi_remove(struct spi_device *spi)
 
 	if (dev)
 		ksz_switch_remove(dev);
-
-	spi_set_drvdata(spi, NULL);
 }
 
 static void ksz_spi_shutdown(struct spi_device *spi)
@@ -147,6 +150,10 @@ static const struct of_device_id ksz_dt_ids[] = {
 		.data = &ksz_switch_chips[KSZ9477]
 	},
 	{
+		.compatible = "microchip,ksz9896",
+		.data = &ksz_switch_chips[KSZ9896]
+	},
+	{
 		.compatible = "microchip,ksz9897",
 		.data = &ksz_switch_chips[KSZ9897]
 	},
@@ -160,7 +167,7 @@ static const struct of_device_id ksz_dt_ids[] = {
 	},
 	{
 		.compatible = "microchip,ksz8563",
-		.data = &ksz_switch_chips[KSZ9893]
+		.data = &ksz_switch_chips[KSZ8563]
 	},
 	{
 		.compatible = "microchip,ksz9567",
@@ -197,6 +204,7 @@ static const struct spi_device_id ksz_spi_ids[] = {
 	{ "ksz8863" },
 	{ "ksz8873" },
 	{ "ksz9477" },
+	{ "ksz9896" },
 	{ "ksz9897" },
 	{ "ksz9893" },
 	{ "ksz9563" },
@@ -226,6 +234,7 @@ static struct spi_driver ksz_spi_driver = {
 module_spi_driver(ksz_spi_driver);
 
 MODULE_ALIAS("spi:ksz9477");
+MODULE_ALIAS("spi:ksz9896");
 MODULE_ALIAS("spi:ksz9897");
 MODULE_ALIAS("spi:ksz9893");
 MODULE_ALIAS("spi:ksz9563");
diff --git a/drivers/net/dsa/microchip/lan937x.h b/drivers/net/dsa/microchip/lan937x.h
index 4e0b1dccec27..8e9e66d6728d 100644
--- a/drivers/net/dsa/microchip/lan937x.h
+++ b/drivers/net/dsa/microchip/lan937x.h
@@ -8,14 +8,16 @@
 
 int lan937x_reset_switch(struct ksz_device *dev);
 int lan937x_setup(struct dsa_switch *ds);
+void lan937x_teardown(struct dsa_switch *ds);
 void lan937x_port_setup(struct ksz_device *dev, int port, bool cpu_port);
 void lan937x_config_cpu_port(struct dsa_switch *ds);
 int lan937x_switch_init(struct ksz_device *dev);
 void lan937x_switch_exit(struct ksz_device *dev);
-void lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data);
-void lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val);
+int lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data);
+int lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val);
 int lan937x_change_mtu(struct ksz_device *dev, int port, int new_mtu);
 void lan937x_phylink_get_caps(struct ksz_device *dev, int port,
 			      struct phylink_config *config);
 void lan937x_setup_rgmii_delay(struct ksz_device *dev, int port);
+int lan937x_set_ageing_time(struct ksz_device *dev, unsigned int msecs);
 #endif
diff --git a/drivers/net/dsa/microchip/lan937x_main.c b/drivers/net/dsa/microchip/lan937x_main.c
index 5579644e8fde..7e4f307a0387 100644
--- a/drivers/net/dsa/microchip/lan937x_main.c
+++ b/drivers/net/dsa/microchip/lan937x_main.c
@@ -7,7 +7,6 @@
 #include <linux/iopoll.h>
 #include <linux/phy.h>
 #include <linux/of_net.h>
-#include <linux/of_mdio.h>
 #include <linux/if_bridge.h>
 #include <linux/if_vlan.h>
 #include <linux/math.h>
@@ -128,81 +127,14 @@ static int lan937x_internal_phy_read(struct ksz_device *dev, int addr, int reg,
 	return ksz_read16(dev, REG_VPHY_IND_DATA__2, val);
 }
 
-void lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
+int lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
 {
-	lan937x_internal_phy_read(dev, addr, reg, data);
+	return lan937x_internal_phy_read(dev, addr, reg, data);
 }
 
-void lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val)
+int lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val)
 {
-	lan937x_internal_phy_write(dev, addr, reg, val);
-}
-
-static int lan937x_sw_mdio_read(struct mii_bus *bus, int addr, int regnum)
-{
-	struct ksz_device *dev = bus->priv;
-	u16 val;
-	int ret;
-
-	if (regnum & MII_ADDR_C45)
-		return -EOPNOTSUPP;
-
-	ret = lan937x_internal_phy_read(dev, addr, regnum, &val);
-	if (ret < 0)
-		return ret;
-
-	return val;
-}
-
-static int lan937x_sw_mdio_write(struct mii_bus *bus, int addr, int regnum,
-				 u16 val)
-{
-	struct ksz_device *dev = bus->priv;
-
-	if (regnum & MII_ADDR_C45)
-		return -EOPNOTSUPP;
-
-	return lan937x_internal_phy_write(dev, addr, regnum, val);
-}
-
-static int lan937x_mdio_register(struct ksz_device *dev)
-{
-	struct dsa_switch *ds = dev->ds;
-	struct device_node *mdio_np;
-	struct mii_bus *bus;
-	int ret;
-
-	mdio_np = of_get_child_by_name(dev->dev->of_node, "mdio");
-	if (!mdio_np) {
-		dev_err(ds->dev, "no MDIO bus node\n");
-		return -ENODEV;
-	}
-
-	bus = devm_mdiobus_alloc(ds->dev);
-	if (!bus) {
-		of_node_put(mdio_np);
-		return -ENOMEM;
-	}
-
-	bus->priv = dev;
-	bus->read = lan937x_sw_mdio_read;
-	bus->write = lan937x_sw_mdio_write;
-	bus->name = "lan937x slave smi";
-	snprintf(bus->id, MII_BUS_ID_SIZE, "SMI-%d", ds->index);
-	bus->parent = ds->dev;
-	bus->phy_mask = ~ds->phys_mii_mask;
-
-	ds->slave_mii_bus = bus;
-
-	ret = devm_of_mdiobus_register(ds->dev, bus, mdio_np);
-	if (ret) {
-		dev_err(ds->dev, "unable to register MDIO bus %s\n",
-			bus->id);
-	}
-
-	of_node_put(mdio_np);
-
-	return ret;
+	return lan937x_internal_phy_write(dev, addr, reg, val);
 }
 
 int lan937x_reset_switch(struct ksz_device *dev)
@@ -225,6 +157,10 @@ int lan937x_reset_switch(struct ksz_device *dev)
 	if (ret < 0)
 		return ret;
 
+	ret = ksz_write32(dev, REG_SW_INT_STATUS__4, POR_READY_INT);
+	if (ret < 0)
+		return ret;
+
 	ret = ksz_write32(dev, REG_SW_PORT_INT_MASK__4, 0xFF);
 	if (ret < 0)
 		return ret;
@@ -311,6 +247,23 @@ int lan937x_change_mtu(struct ksz_device *dev, int port, int new_mtu)
 	return 0;
 }
 
+int lan937x_set_ageing_time(struct ksz_device *dev, unsigned int msecs)
+{
+	u32 secs = msecs / 1000;
+	u32 value;
+	int ret;
+
+	value = FIELD_GET(SW_AGE_PERIOD_7_0_M, secs);
+
+	ret = ksz_write8(dev, REG_SW_AGE_PERIOD__1, value);
+	if (ret < 0)
+		return ret;
+
+	value = FIELD_GET(SW_AGE_PERIOD_19_8_M, secs);
+
+	return ksz_write16(dev, REG_SW_AGE_PERIOD__2, value);
+}
+
 static void lan937x_set_tune_adj(struct ksz_device *dev, int port,
 				 u16 reg, u8 val)
 {
@@ -379,6 +332,13 @@ void lan937x_setup_rgmii_delay(struct ksz_device *dev, int port)
 	}
 }
 
+int lan937x_switch_init(struct ksz_device *dev)
+{
+	dev->port_mask = (1 << dev->info->port_cnt) - 1;
+
+	return 0;
+}
+
 int lan937x_setup(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
@@ -391,12 +351,6 @@ int lan937x_setup(struct dsa_switch *ds)
 		return ret;
 	}
 
-	ret = lan937x_mdio_register(dev);
-	if (ret < 0) {
-		dev_err(dev->dev, "failed to register the mdio");
-		return ret;
-	}
-
 	/* The VLAN aware is a global setting. Mixed vlan
 	 * filterings are not supported.
 	 */
@@ -422,11 +376,9 @@ int lan937x_setup(struct dsa_switch *ds)
 	return 0;
 }
 
-int lan937x_switch_init(struct ksz_device *dev)
+void lan937x_teardown(struct dsa_switch *ds)
 {
-	dev->port_mask = (1 << dev->info->port_cnt) - 1;
 
-	return 0;
 }
 
 void lan937x_switch_exit(struct ksz_device *dev)
diff --git a/drivers/net/dsa/microchip/lan937x_reg.h b/drivers/net/dsa/microchip/lan937x_reg.h
index ba4adaddb3ec..5bc16a4c4441 100644
--- a/drivers/net/dsa/microchip/lan937x_reg.h
+++ b/drivers/net/dsa/microchip/lan937x_reg.h
@@ -62,6 +62,12 @@
 #define SW_FAST_AGING			BIT(1)
 #define SW_LINK_AUTO_AGING		BIT(0)
 
+#define REG_SW_AGE_PERIOD__1		0x0313
+#define SW_AGE_PERIOD_7_0_M		GENMASK(7, 0)
+
+#define REG_SW_AGE_PERIOD__2		0x0320
+#define SW_AGE_PERIOD_19_8_M		GENMASK(19, 8)
+
 #define REG_SW_MAC_CTRL_0		0x0330
 #define SW_NEW_BACKOFF			BIT(7)
 #define SW_PAUSE_UNH_MODE		BIT(1)
@@ -118,6 +124,18 @@
 /* Port Registers */
 
 /* 0 - Operation */
+#define REG_PORT_INT_STATUS		0x001B
+#define REG_PORT_INT_MASK		0x001F
+
+#define PORT_TAS_INT			BIT(5)
+#define PORT_QCI_INT			BIT(4)
+#define PORT_SGMII_INT			BIT(3)
+#define PORT_PTP_INT			BIT(2)
+#define PORT_PHY_INT			BIT(1)
+#define PORT_ACL_INT			BIT(0)
+
+#define PORT_SRC_PHY_INT		1
+
 #define REG_PORT_CTRL_0			0x0020
 
 #define PORT_MAC_LOOPBACK		BIT(7)
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 835807911be0..e74c6b406172 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -506,14 +506,19 @@ static bool mt7531_dual_sgmii_supported(struct mt7530_priv *priv)
 static int
 mt7531_pad_setup(struct dsa_switch *ds, phy_interface_t interface)
 {
-	struct mt7530_priv *priv = ds->priv;
+	return 0;
+}
+
+static void
+mt7531_pll_setup(struct mt7530_priv *priv)
+{
 	u32 top_sig;
 	u32 hwstrap;
 	u32 xtal;
 	u32 val;
 
 	if (mt7531_dual_sgmii_supported(priv))
-		return 0;
+		return;
 
 	val = mt7530_read(priv, MT7531_CREV);
 	top_sig = mt7530_read(priv, MT7531_TOP_SIG_SR);
@@ -592,8 +597,6 @@ mt7531_pad_setup(struct dsa_switch *ds, phy_interface_t interface)
 	val |= EN_COREPLL;
 	mt7530_write(priv, MT7531_PLLGP_EN, val);
 	usleep_range(25, 35);
-
-	return 0;
 }
 
 static void
@@ -2326,11 +2329,17 @@ mt7531_setup(struct dsa_switch *ds)
 		return -ENODEV;
 	}
 
+	/* all MACs must be forced link-down before sw reset */
+	for (i = 0; i < MT7530_NUM_PORTS; i++)
+		mt7530_write(priv, MT7530_PMCR_P(i), MT7531_FORCE_LNK);
+
 	/* Reset the switch through internal reset */
 	mt7530_write(priv, MT7530_SYS_CTRL,
 		     SYS_CTRL_PHY_RST | SYS_CTRL_SW_RST |
 		     SYS_CTRL_REG_RST);
 
+	mt7531_pll_setup(priv);
+
 	if (mt7531_dual_sgmii_supported(priv)) {
 		priv->p5_intf_sel = P5_INTF_SEL_GMAC5_SGMII;
 
@@ -2699,9 +2708,6 @@ mt7531_mac_config(struct dsa_switch *ds, int port, unsigned int mode,
 	case PHY_INTERFACE_MODE_NA:
 	case PHY_INTERFACE_MODE_1000BASEX:
 	case PHY_INTERFACE_MODE_2500BASEX:
-		if (phylink_autoneg_inband(mode))
-			return -EINVAL;
-
 		return mt7531_sgmii_setup_mode_force(priv, port, interface);
 	default:
 		return -EINVAL;
@@ -2777,13 +2783,6 @@ unsupported:
 		return;
 	}
 
-	if (phylink_autoneg_inband(mode) &&
-	    state->interface != PHY_INTERFACE_MODE_SGMII) {
-		dev_err(ds->dev, "%s: in-band negotiation unsupported\n",
-			__func__);
-		return;
-	}
-
 	mcr_cur = mt7530_read(priv, MT7530_PMCR_P(port));
 	mcr_new = mcr_cur;
 	mcr_new &= ~PMCR_LINK_SETTINGS_MASK;
@@ -2887,8 +2886,6 @@ mt7531_cpu_port_config(struct dsa_switch *ds, int port)
 	case 6:
 		interface = PHY_INTERFACE_MODE_2500BASEX;
 
-		mt7531_pad_setup(ds, interface);
-
 		priv->p6_interface = interface;
 		break;
 	default:
@@ -2922,6 +2919,9 @@ static void mt753x_phylink_get_caps(struct dsa_switch *ds, int port,
 	config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
 				   MAC_10 | MAC_100 | MAC_1000FD;
 
+	if ((priv->id == ID_MT7531) && mt753x_is_mac_port(port))
+		config->mac_capabilities |= MAC_2500FD;
+
 	/* This driver does not make use of the speed, duplex, pause or the
 	 * advertisement in its mac_config, so it is safe to mark this driver
 	 * as non-legacy.
@@ -2987,6 +2987,7 @@ mt7531_sgmii_pcs_get_state_an(struct mt7530_priv *priv, int port,
 
 	status = mt7530_read(priv, MT7531_PCS_CONTROL_1(port));
 	state->link = !!(status & MT7531_SGMII_LINK_STATUS);
+	state->an_complete = !!(status & MT7531_SGMII_AN_COMPLETE);
 	if (state->interface == PHY_INTERFACE_MODE_SGMII &&
 	    (status & MT7531_SGMII_AN_ENABLE)) {
 		val = mt7530_read(priv, MT7531_PCS_SPEED_ABILITY(port));
@@ -3017,16 +3018,44 @@ mt7531_sgmii_pcs_get_state_an(struct mt7530_priv *priv, int port,
 	return 0;
 }
 
+static void
+mt7531_sgmii_pcs_get_state_inband(struct mt7530_priv *priv, int port,
+				  struct phylink_link_state *state)
+{
+	unsigned int val;
+
+	val = mt7530_read(priv, MT7531_PCS_CONTROL_1(port));
+	state->link = !!(val & MT7531_SGMII_LINK_STATUS);
+	if (!state->link)
+		return;
+
+	state->an_complete = state->link;
+
+	if (state->interface == PHY_INTERFACE_MODE_2500BASEX)
+		state->speed = SPEED_2500;
+	else
+		state->speed = SPEED_1000;
+
+	state->duplex = DUPLEX_FULL;
+	state->pause = MLO_PAUSE_NONE;
+}
+
 static void mt7531_pcs_get_state(struct phylink_pcs *pcs,
 				 struct phylink_link_state *state)
 {
 	struct mt7530_priv *priv = pcs_to_mt753x_pcs(pcs)->priv;
 	int port = pcs_to_mt753x_pcs(pcs)->port;
 
-	if (state->interface == PHY_INTERFACE_MODE_SGMII)
+	if (state->interface == PHY_INTERFACE_MODE_SGMII) {
 		mt7531_sgmii_pcs_get_state_an(priv, port, state);
-	else
-		state->link = false;
+		return;
+	} else if ((state->interface == PHY_INTERFACE_MODE_1000BASEX) ||
+		   (state->interface == PHY_INTERFACE_MODE_2500BASEX)) {
+		mt7531_sgmii_pcs_get_state_inband(priv, port, state);
+		return;
+	}
+
+	state->link = false;
 }
 
 static int mt753x_pcs_config(struct phylink_pcs *pcs, unsigned int mode,
@@ -3067,6 +3096,8 @@ mt753x_setup(struct dsa_switch *ds)
 		priv->pcs[i].pcs.ops = priv->info->pcs_ops;
 		priv->pcs[i].priv = priv;
 		priv->pcs[i].port = i;
+		if (mt753x_is_mac_port(i))
+			priv->pcs[i].pcs.poll = 1;
 	}
 
 	ret = priv->info->sw_setup(ds);
@@ -3300,8 +3331,6 @@ mt7530_remove(struct mdio_device *mdiodev)
 
 	dsa_unregister_switch(priv->ds);
 	mutex_destroy(&priv->reg_mutex);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void mt7530_shutdown(struct mdio_device *mdiodev)
diff --git a/drivers/net/dsa/mt7530.h b/drivers/net/dsa/mt7530.h
index e509af95c354..e8d966435350 100644
--- a/drivers/net/dsa/mt7530.h
+++ b/drivers/net/dsa/mt7530.h
@@ -373,6 +373,7 @@ enum mt7530_vlan_port_acc_frm {
 #define  MT7531_SGMII_LINK_STATUS	BIT(18)
 #define  MT7531_SGMII_AN_ENABLE		BIT(12)
 #define  MT7531_SGMII_AN_RESTART	BIT(9)
+#define  MT7531_SGMII_AN_COMPLETE	BIT(21)
 
 /* Register for SGMII PCS_SPPED_ABILITY */
 #define MT7531_PCS_SPEED_ABILITY(p)	MT7531_SGMII_REG(p, 0x08)
diff --git a/drivers/net/dsa/mv88e6060.c b/drivers/net/dsa/mv88e6060.c
index 83dca9179aa0..fdda62d6eb16 100644
--- a/drivers/net/dsa/mv88e6060.c
+++ b/drivers/net/dsa/mv88e6060.c
@@ -297,8 +297,6 @@ static void mv88e6060_remove(struct mdio_device *mdiodev)
 		return;
 
 	dsa_unregister_switch(ds);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void mv88e6060_shutdown(struct mdio_device *mdiodev)
diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
index 07e9a4da924c..2479be3a1e35 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -816,6 +816,14 @@ static void mv88e6393x_phylink_get_caps(struct mv88e6xxx_chip *chip, int port,
 				MAC_10000FD;
 		}
 	}
+
+	if (port == 0) {
+		__set_bit(PHY_INTERFACE_MODE_RMII, supported);
+		__set_bit(PHY_INTERFACE_MODE_RGMII, supported);
+		__set_bit(PHY_INTERFACE_MODE_RGMII_ID, supported);
+		__set_bit(PHY_INTERFACE_MODE_RGMII_RXID, supported);
+		__set_bit(PHY_INTERFACE_MODE_RGMII_TXID, supported);
+	}
 }
 
 static void mv88e6xxx_get_caps(struct dsa_switch *ds, int port,
@@ -1128,7 +1136,7 @@ static void mv88e6xxx_atu_vtu_get_strings(uint8_t *data)
 	unsigned int i;
 
 	for (i = 0; i < ARRAY_SIZE(mv88e6xxx_atu_vtu_stats_strings); i++)
-		strlcpy(data + i * ETH_GSTRING_LEN,
+		strscpy(data + i * ETH_GSTRING_LEN,
 			mv88e6xxx_atu_vtu_stats_strings[i],
 			ETH_GSTRING_LEN);
 }
@@ -6585,14 +6593,17 @@ out:
 
 static bool mv88e6xxx_lag_can_offload(struct dsa_switch *ds,
 				      struct dsa_lag lag,
-				      struct netdev_lag_upper_info *info)
+				      struct netdev_lag_upper_info *info,
+				      struct netlink_ext_ack *extack)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
 	struct dsa_port *dp;
 	int members = 0;
 
-	if (!mv88e6xxx_has_lag(chip))
+	if (!mv88e6xxx_has_lag(chip)) {
+		NL_SET_ERR_MSG_MOD(extack, "Chip does not support LAG offload");
 		return false;
+	}
 
 	if (!lag.id)
 		return false;
@@ -6601,14 +6612,20 @@ static bool mv88e6xxx_lag_can_offload(struct dsa_switch *ds,
 		/* Includes the port joining the LAG */
 		members++;
 
-	if (members > 8)
+	if (members > 8) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot offload more than 8 LAG ports");
 		return false;
+	}
 
 	/* We could potentially relax this to include active
 	 * backup in the future.
 	 */
-	if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH)
+	if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Can only offload LAG using hash TX type");
 		return false;
+	}
 
 	/* Ideally we would also validate that the hash type matches
 	 * the hardware. Alas, this is always set to unknown on team
@@ -6761,12 +6778,13 @@ static int mv88e6xxx_port_lag_change(struct dsa_switch *ds, int port)
 
 static int mv88e6xxx_port_lag_join(struct dsa_switch *ds, int port,
 				   struct dsa_lag lag,
-				   struct netdev_lag_upper_info *info)
+				   struct netdev_lag_upper_info *info,
+				   struct netlink_ext_ack *extack)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
 	int err, id;
 
-	if (!mv88e6xxx_lag_can_offload(ds, lag, info))
+	if (!mv88e6xxx_lag_can_offload(ds, lag, info, extack))
 		return -EOPNOTSUPP;
 
 	/* DSA LAG IDs are one-based */
@@ -6819,12 +6837,13 @@ static int mv88e6xxx_crosschip_lag_change(struct dsa_switch *ds, int sw_index,
 
 static int mv88e6xxx_crosschip_lag_join(struct dsa_switch *ds, int sw_index,
 					int port, struct dsa_lag lag,
-					struct netdev_lag_upper_info *info)
+					struct netdev_lag_upper_info *info,
+					struct netlink_ext_ack *extack)
 {
 	struct mv88e6xxx_chip *chip = ds->priv;
 	int err;
 
-	if (!mv88e6xxx_lag_can_offload(ds, lag, info))
+	if (!mv88e6xxx_lag_can_offload(ds, lag, info, extack))
 		return -EOPNOTSUPP;
 
 	mv88e6xxx_reg_lock(chip);
@@ -7166,8 +7185,6 @@ static void mv88e6xxx_remove(struct mdio_device *mdiodev)
 		mv88e6xxx_g1_irq_free(chip);
 	else
 		mv88e6xxx_irq_poll_free(chip);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void mv88e6xxx_shutdown(struct mdio_device *mdiodev)
diff --git a/drivers/net/dsa/mv88e6xxx/global2.h b/drivers/net/dsa/mv88e6xxx/global2.h
index 807aeaad9830..7536b8b0ad01 100644
--- a/drivers/net/dsa/mv88e6xxx/global2.h
+++ b/drivers/net/dsa/mv88e6xxx/global2.h
@@ -298,7 +298,7 @@
 #define MV88E6352_G2_SCRATCH_CONFIG_DATA1	0x71
 #define MV88E6352_G2_SCRATCH_CONFIG_DATA1_NO_CPU	BIT(2)
 #define MV88E6352_G2_SCRATCH_CONFIG_DATA2	0x72
-#define MV88E6352_G2_SCRATCH_CONFIG_DATA2_P0_MODE_MASK	0x3
+#define MV88E6352_G2_SCRATCH_CONFIG_DATA2_P0_MODE_MASK	0xf
 #define MV88E6352_G2_SCRATCH_CONFIG_DATA3	0x73
 #define MV88E6352_G2_SCRATCH_CONFIG_DATA3_S_SEL		BIT(1)
 
diff --git a/drivers/net/dsa/mv88e6xxx/port.c b/drivers/net/dsa/mv88e6xxx/port.c
index 90c55f23b7c9..5c4195c635b0 100644
--- a/drivers/net/dsa/mv88e6xxx/port.c
+++ b/drivers/net/dsa/mv88e6xxx/port.c
@@ -517,6 +517,12 @@ static int mv88e6xxx_port_set_cmode(struct mv88e6xxx_chip *chip, int port,
 	case PHY_INTERFACE_MODE_RMII:
 		cmode = MV88E6XXX_PORT_STS_CMODE_RMII;
 		break;
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+		cmode = MV88E6XXX_PORT_STS_CMODE_RGMII;
+		break;
 	case PHY_INTERFACE_MODE_1000BASEX:
 		cmode = MV88E6XXX_PORT_STS_CMODE_1000BASEX;
 		break;
@@ -634,6 +640,19 @@ int mv88e6393x_port_set_cmode(struct mv88e6xxx_chip *chip, int port,
 	if (port != 0 && port != 9 && port != 10)
 		return -EOPNOTSUPP;
 
+	if (port == 9 || port == 10) {
+		switch (mode) {
+		case PHY_INTERFACE_MODE_RMII:
+		case PHY_INTERFACE_MODE_RGMII:
+		case PHY_INTERFACE_MODE_RGMII_ID:
+		case PHY_INTERFACE_MODE_RGMII_RXID:
+		case PHY_INTERFACE_MODE_RGMII_TXID:
+			return -EINVAL;
+		default:
+			break;
+		}
+	}
+
 	/* mv88e6393x errata 4.5: EEE should be disabled on SERDES ports */
 	err = mv88e6xxx_port_read(chip, port, MV88E6XXX_PORT_MAC_CTL, &reg);
 	if (err)
diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c
index aadb0bd7c24f..dd3a18cc89dd 100644
--- a/drivers/net/dsa/ocelot/felix.c
+++ b/drivers/net/dsa/ocelot/felix.c
@@ -42,6 +42,25 @@ static struct net_device *felix_classify_db(struct dsa_db db)
 	}
 }
 
+static int felix_cpu_port_for_master(struct dsa_switch *ds,
+				     struct net_device *master)
+{
+	struct ocelot *ocelot = ds->priv;
+	struct dsa_port *cpu_dp;
+	int lag;
+
+	if (netif_is_lag_master(master)) {
+		mutex_lock(&ocelot->fwd_domain_lock);
+		lag = ocelot_bond_get_id(ocelot, master);
+		mutex_unlock(&ocelot->fwd_domain_lock);
+
+		return lag;
+	}
+
+	cpu_dp = master->dsa_ptr;
+	return cpu_dp->index;
+}
+
 /* Set up VCAP ES0 rules for pushing a tag_8021q VLAN towards the CPU such that
  * the tagger can perform RX source port identification.
  */
@@ -422,6 +441,40 @@ static unsigned long felix_tag_npi_get_host_fwd_mask(struct dsa_switch *ds)
 	return BIT(ocelot->num_phys_ports);
 }
 
+static int felix_tag_npi_change_master(struct dsa_switch *ds, int port,
+				       struct net_device *master,
+				       struct netlink_ext_ack *extack)
+{
+	struct dsa_port *dp = dsa_to_port(ds, port), *other_dp;
+	struct ocelot *ocelot = ds->priv;
+
+	if (netif_is_lag_master(master)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "LAG DSA master only supported using ocelot-8021q");
+		return -EOPNOTSUPP;
+	}
+
+	/* Changing the NPI port breaks user ports still assigned to the old
+	 * one, so only allow it while they're down, and don't allow them to
+	 * come back up until they're all changed to the new one.
+	 */
+	dsa_switch_for_each_user_port(other_dp, ds) {
+		struct net_device *slave = other_dp->slave;
+
+		if (other_dp != dp && (slave->flags & IFF_UP) &&
+		    dsa_port_to_master(other_dp) != master) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Cannot change while old master still has users");
+			return -EOPNOTSUPP;
+		}
+	}
+
+	felix_npi_port_deinit(ocelot, ocelot->npi);
+	felix_npi_port_init(ocelot, felix_cpu_port_for_master(ds, master));
+
+	return 0;
+}
+
 /* Alternatively to using the NPI functionality, that same hardware MAC
  * connected internally to the enetc or fman DSA master can be configured to
  * use the software-defined tag_8021q frame format. As far as the hardware is
@@ -433,6 +486,7 @@ static const struct felix_tag_proto_ops felix_tag_npi_proto_ops = {
 	.setup			= felix_tag_npi_setup,
 	.teardown		= felix_tag_npi_teardown,
 	.get_host_fwd_mask	= felix_tag_npi_get_host_fwd_mask,
+	.change_master		= felix_tag_npi_change_master,
 };
 
 static int felix_tag_8021q_setup(struct dsa_switch *ds)
@@ -445,6 +499,9 @@ static int felix_tag_8021q_setup(struct dsa_switch *ds)
 	if (err)
 		return err;
 
+	dsa_switch_for_each_cpu_port(dp, ds)
+		ocelot_port_setup_dsa_8021q_cpu(ocelot, dp->index);
+
 	dsa_switch_for_each_user_port(dp, ds)
 		ocelot_port_assign_dsa_8021q_cpu(ocelot, dp->index,
 						 dp->cpu_dp->index);
@@ -493,6 +550,9 @@ static void felix_tag_8021q_teardown(struct dsa_switch *ds)
 	dsa_switch_for_each_user_port(dp, ds)
 		ocelot_port_unassign_dsa_8021q_cpu(ocelot, dp->index);
 
+	dsa_switch_for_each_cpu_port(dp, ds)
+		ocelot_port_teardown_dsa_8021q_cpu(ocelot, dp->index);
+
 	dsa_tag_8021q_unregister(ds);
 }
 
@@ -501,10 +561,24 @@ static unsigned long felix_tag_8021q_get_host_fwd_mask(struct dsa_switch *ds)
 	return dsa_cpu_ports(ds);
 }
 
+static int felix_tag_8021q_change_master(struct dsa_switch *ds, int port,
+					 struct net_device *master,
+					 struct netlink_ext_ack *extack)
+{
+	int cpu = felix_cpu_port_for_master(ds, master);
+	struct ocelot *ocelot = ds->priv;
+
+	ocelot_port_unassign_dsa_8021q_cpu(ocelot, port);
+	ocelot_port_assign_dsa_8021q_cpu(ocelot, port, cpu);
+
+	return felix_update_trapping_destinations(ds, true);
+}
+
 static const struct felix_tag_proto_ops felix_tag_8021q_proto_ops = {
 	.setup			= felix_tag_8021q_setup,
 	.teardown		= felix_tag_8021q_teardown,
 	.get_host_fwd_mask	= felix_tag_8021q_get_host_fwd_mask,
+	.change_master		= felix_tag_8021q_change_master,
 };
 
 static void felix_set_host_flood(struct dsa_switch *ds, unsigned long mask,
@@ -667,6 +741,16 @@ static void felix_port_set_host_flood(struct dsa_switch *ds, int port,
 			     !!felix->host_flood_mc_mask, true);
 }
 
+static int felix_port_change_master(struct dsa_switch *ds, int port,
+				    struct net_device *master,
+				    struct netlink_ext_ack *extack)
+{
+	struct ocelot *ocelot = ds->priv;
+	struct felix *felix = ocelot_to_felix(ocelot);
+
+	return felix->tag_proto_ops->change_master(ds, port, master, extack);
+}
+
 static int felix_set_ageing_time(struct dsa_switch *ds,
 				 unsigned int ageing_time)
 {
@@ -855,11 +939,21 @@ static void felix_bridge_leave(struct dsa_switch *ds, int port,
 
 static int felix_lag_join(struct dsa_switch *ds, int port,
 			  struct dsa_lag lag,
-			  struct netdev_lag_upper_info *info)
+			  struct netdev_lag_upper_info *info,
+			  struct netlink_ext_ack *extack)
 {
 	struct ocelot *ocelot = ds->priv;
+	int err;
+
+	err = ocelot_port_lag_join(ocelot, port, lag.dev, info, extack);
+	if (err)
+		return err;
+
+	/* Update the logical LAG port that serves as tag_8021q CPU port */
+	if (!dsa_is_cpu_port(ds, port))
+		return 0;
 
-	return ocelot_port_lag_join(ocelot, port, lag.dev, info);
+	return felix_port_change_master(ds, port, lag.dev, extack);
 }
 
 static int felix_lag_leave(struct dsa_switch *ds, int port,
@@ -869,7 +963,11 @@ static int felix_lag_leave(struct dsa_switch *ds, int port,
 
 	ocelot_port_lag_leave(ocelot, port, lag.dev);
 
-	return 0;
+	/* Update the logical LAG port that serves as tag_8021q CPU port */
+	if (!dsa_is_cpu_port(ds, port))
+		return 0;
+
+	return felix_port_change_master(ds, port, lag.dev, NULL);
 }
 
 static int felix_lag_change(struct dsa_switch *ds, int port)
@@ -1007,6 +1105,27 @@ static void felix_phylink_mac_link_up(struct dsa_switch *ds, int port,
 		felix->info->port_sched_speed_set(ocelot, port, speed);
 }
 
+static int felix_port_enable(struct dsa_switch *ds, int port,
+			     struct phy_device *phydev)
+{
+	struct dsa_port *dp = dsa_to_port(ds, port);
+	struct ocelot *ocelot = ds->priv;
+
+	if (!dsa_port_is_user(dp))
+		return 0;
+
+	if (ocelot->npi >= 0) {
+		struct net_device *master = dsa_port_to_master(dp);
+
+		if (felix_cpu_port_for_master(ds, master) != ocelot->npi) {
+			dev_err(ds->dev, "Multiple masters are not allowed\n");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
 static void felix_port_qos_map_init(struct ocelot *ocelot, int port)
 {
 	int i;
@@ -1028,6 +1147,55 @@ static void felix_port_qos_map_init(struct ocelot *ocelot, int port)
 	}
 }
 
+static void felix_get_stats64(struct dsa_switch *ds, int port,
+			      struct rtnl_link_stats64 *stats)
+{
+	struct ocelot *ocelot = ds->priv;
+
+	ocelot_port_get_stats64(ocelot, port, stats);
+}
+
+static void felix_get_pause_stats(struct dsa_switch *ds, int port,
+				  struct ethtool_pause_stats *pause_stats)
+{
+	struct ocelot *ocelot = ds->priv;
+
+	ocelot_port_get_pause_stats(ocelot, port, pause_stats);
+}
+
+static void felix_get_rmon_stats(struct dsa_switch *ds, int port,
+				 struct ethtool_rmon_stats *rmon_stats,
+				 const struct ethtool_rmon_hist_range **ranges)
+{
+	struct ocelot *ocelot = ds->priv;
+
+	ocelot_port_get_rmon_stats(ocelot, port, rmon_stats, ranges);
+}
+
+static void felix_get_eth_ctrl_stats(struct dsa_switch *ds, int port,
+				     struct ethtool_eth_ctrl_stats *ctrl_stats)
+{
+	struct ocelot *ocelot = ds->priv;
+
+	ocelot_port_get_eth_ctrl_stats(ocelot, port, ctrl_stats);
+}
+
+static void felix_get_eth_mac_stats(struct dsa_switch *ds, int port,
+				    struct ethtool_eth_mac_stats *mac_stats)
+{
+	struct ocelot *ocelot = ds->priv;
+
+	ocelot_port_get_eth_mac_stats(ocelot, port, mac_stats);
+}
+
+static void felix_get_eth_phy_stats(struct dsa_switch *ds, int port,
+				    struct ethtool_eth_phy_stats *phy_stats)
+{
+	struct ocelot *ocelot = ds->priv;
+
+	ocelot_port_get_eth_phy_stats(ocelot, port, phy_stats);
+}
+
 static void felix_get_strings(struct dsa_switch *ds, int port,
 			      u32 stringset, u8 *data)
 {
@@ -1144,11 +1312,55 @@ static int felix_parse_dt(struct felix *felix, phy_interface_t *port_phy_modes)
 	return err;
 }
 
+static struct regmap *felix_request_regmap_by_name(struct felix *felix,
+						   const char *resource_name)
+{
+	struct ocelot *ocelot = &felix->ocelot;
+	struct resource res;
+	int i;
+
+	for (i = 0; i < felix->info->num_resources; i++) {
+		if (strcmp(resource_name, felix->info->resources[i].name))
+			continue;
+
+		memcpy(&res, &felix->info->resources[i], sizeof(res));
+		res.start += felix->switch_base;
+		res.end += felix->switch_base;
+
+		return ocelot_regmap_init(ocelot, &res);
+	}
+
+	return ERR_PTR(-ENOENT);
+}
+
+static struct regmap *felix_request_regmap(struct felix *felix,
+					   enum ocelot_target target)
+{
+	const char *resource_name = felix->info->resource_names[target];
+
+	/* If the driver didn't provide a resource name for the target,
+	 * the resource is optional.
+	 */
+	if (!resource_name)
+		return NULL;
+
+	return felix_request_regmap_by_name(felix, resource_name);
+}
+
+static struct regmap *felix_request_port_regmap(struct felix *felix, int port)
+{
+	char resource_name[32];
+
+	sprintf(resource_name, "port%d", port);
+
+	return felix_request_regmap_by_name(felix, resource_name);
+}
+
 static int felix_init_structs(struct felix *felix, int num_phys_ports)
 {
 	struct ocelot *ocelot = &felix->ocelot;
 	phy_interface_t *port_phy_modes;
-	struct resource res;
+	struct regmap *target;
 	int port, i, err;
 
 	ocelot->num_phys_ports = num_phys_ports;
@@ -1182,20 +1394,11 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports)
 	}
 
 	for (i = 0; i < TARGET_MAX; i++) {
-		struct regmap *target;
-
-		if (!felix->info->target_io_res[i].name)
-			continue;
-
-		memcpy(&res, &felix->info->target_io_res[i], sizeof(res));
-		res.flags = IORESOURCE_MEM;
-		res.start += felix->switch_base;
-		res.end += felix->switch_base;
-
-		target = felix->info->init_regmap(ocelot, &res);
+		target = felix_request_regmap(felix, i);
 		if (IS_ERR(target)) {
 			dev_err(ocelot->dev,
-				"Failed to map device memory space\n");
+				"Failed to map device memory space: %pe\n",
+				target);
 			kfree(port_phy_modes);
 			return PTR_ERR(target);
 		}
@@ -1212,7 +1415,6 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports)
 
 	for (port = 0; port < num_phys_ports; port++) {
 		struct ocelot_port *ocelot_port;
-		struct regmap *target;
 
 		ocelot_port = devm_kzalloc(ocelot->dev,
 					   sizeof(struct ocelot_port),
@@ -1224,16 +1426,11 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports)
 			return -ENOMEM;
 		}
 
-		memcpy(&res, &felix->info->port_io_res[port], sizeof(res));
-		res.flags = IORESOURCE_MEM;
-		res.start += felix->switch_base;
-		res.end += felix->switch_base;
-
-		target = felix->info->init_regmap(ocelot, &res);
+		target = felix_request_port_regmap(felix, port);
 		if (IS_ERR(target)) {
 			dev_err(ocelot->dev,
-				"Failed to map memory space for port %d\n",
-				port);
+				"Failed to map memory space for port %d: %pe\n",
+				port, target);
 			kfree(port_phy_modes);
 			return PTR_ERR(target);
 		}
@@ -1842,6 +2039,12 @@ const struct dsa_switch_ops felix_switch_ops = {
 	.setup				= felix_setup,
 	.teardown			= felix_teardown,
 	.set_ageing_time		= felix_set_ageing_time,
+	.get_stats64			= felix_get_stats64,
+	.get_pause_stats		= felix_get_pause_stats,
+	.get_rmon_stats			= felix_get_rmon_stats,
+	.get_eth_ctrl_stats		= felix_get_eth_ctrl_stats,
+	.get_eth_mac_stats		= felix_get_eth_mac_stats,
+	.get_eth_phy_stats		= felix_get_eth_phy_stats,
 	.get_strings			= felix_get_strings,
 	.get_ethtool_stats		= felix_get_ethtool_stats,
 	.get_sset_count			= felix_get_sset_count,
@@ -1851,6 +2054,7 @@ const struct dsa_switch_ops felix_switch_ops = {
 	.phylink_mac_select_pcs		= felix_phylink_mac_select_pcs,
 	.phylink_mac_link_down		= felix_phylink_mac_link_down,
 	.phylink_mac_link_up		= felix_phylink_mac_link_up,
+	.port_enable			= felix_port_enable,
 	.port_fast_age			= felix_port_fast_age,
 	.port_fdb_dump			= felix_fdb_dump,
 	.port_fdb_add			= felix_fdb_add,
@@ -1906,6 +2110,7 @@ const struct dsa_switch_ops felix_switch_ops = {
 	.port_add_dscp_prio		= felix_port_add_dscp_prio,
 	.port_del_dscp_prio		= felix_port_del_dscp_prio,
 	.port_set_host_flood		= felix_port_set_host_flood,
+	.port_change_master		= felix_port_change_master,
 };
 
 struct net_device *felix_port_to_netdev(struct ocelot *ocelot, int port)
diff --git a/drivers/net/dsa/ocelot/felix.h b/drivers/net/dsa/ocelot/felix.h
index deb8dde1fc19..c9c29999c336 100644
--- a/drivers/net/dsa/ocelot/felix.h
+++ b/drivers/net/dsa/ocelot/felix.h
@@ -16,9 +16,13 @@
 
 /* Platform-specific information */
 struct felix_info {
-	const struct resource		*target_io_res;
-	const struct resource		*port_io_res;
-	const struct resource		*imdio_res;
+	/* Hardcoded resources provided by the hardware instantiation. */
+	const struct resource		*resources;
+	size_t				num_resources;
+	/* Names of the mandatory resources that will be requested during
+	 * probe. Must have TARGET_MAX elements, since it is indexed by target.
+	 */
+	const char *const		*resource_names;
 	const struct reg_field		*regfields;
 	const u32 *const		*map;
 	const struct ocelot_ops		*ops;
@@ -56,8 +60,6 @@ struct felix_info {
 	void	(*tas_guard_bands_update)(struct ocelot *ocelot, int port);
 	void	(*port_sched_speed_set)(struct ocelot *ocelot, int port,
 					u32 speed);
-	struct regmap *(*init_regmap)(struct ocelot *ocelot,
-				      struct resource *res);
 };
 
 /* Methods for initializing the hardware resources specific to a tagging
@@ -71,6 +73,9 @@ struct felix_tag_proto_ops {
 	int (*setup)(struct dsa_switch *ds);
 	void (*teardown)(struct dsa_switch *ds);
 	unsigned long (*get_host_fwd_mask)(struct dsa_switch *ds);
+	int (*change_master)(struct dsa_switch *ds, int port,
+			     struct net_device *master,
+			     struct netlink_ext_ack *extack);
 };
 
 extern const struct dsa_switch_ops felix_switch_ops;
@@ -83,7 +88,6 @@ struct felix {
 	struct mii_bus			*imdio;
 	struct phylink_pcs		**pcs;
 	resource_size_t			switch_base;
-	resource_size_t			imdio_base;
 	enum dsa_tag_protocol		tag_proto;
 	const struct felix_tag_proto_ops *tag_proto_ops;
 	struct kthread_worker		*xmit_worker;
diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c
index f8f19a85744c..26a35ae322d1 100644
--- a/drivers/net/dsa/ocelot/felix_vsc9959.c
+++ b/drivers/net/dsa/ocelot/felix_vsc9959.c
@@ -348,7 +348,7 @@ static const u32 vsc9959_sys_regmap[] = {
 	REG(SYS_COUNT_TX_GREEN_PRIO_5,		0x00026c),
 	REG(SYS_COUNT_TX_GREEN_PRIO_6,		0x000270),
 	REG(SYS_COUNT_TX_GREEN_PRIO_7,		0x000274),
-	REG(SYS_COUNT_TX_AGING,			0x000278),
+	REG(SYS_COUNT_TX_AGED,			0x000278),
 	REG(SYS_COUNT_DROP_LOCAL,		0x000400),
 	REG(SYS_COUNT_DROP_TAIL,		0x000404),
 	REG(SYS_COUNT_DROP_YELLOW_PRIO_0,	0x000408),
@@ -367,6 +367,10 @@ static const u32 vsc9959_sys_regmap[] = {
 	REG(SYS_COUNT_DROP_GREEN_PRIO_5,	0x00043c),
 	REG(SYS_COUNT_DROP_GREEN_PRIO_6,	0x000440),
 	REG(SYS_COUNT_DROP_GREEN_PRIO_7,	0x000444),
+	REG(SYS_COUNT_SF_MATCHING_FRAMES,	0x000800),
+	REG(SYS_COUNT_SF_NOT_PASSING_FRAMES,	0x000804),
+	REG(SYS_COUNT_SF_NOT_PASSING_SDU,	0x000808),
+	REG(SYS_COUNT_SF_RED_FRAMES,		0x00080c),
 	REG(SYS_RESET_CFG,			0x000e00),
 	REG(SYS_SR_ETYPE_CFG,			0x000e04),
 	REG(SYS_VLAN_ETYPE_CFG,			0x000e08),
@@ -388,7 +392,6 @@ static const u32 vsc9959_sys_regmap[] = {
 	REG_RESERVED(SYS_MMGT_FAST),
 	REG_RESERVED(SYS_EVENTS_DIF),
 	REG_RESERVED(SYS_EVENTS_CORE),
-	REG(SYS_CNT,				0x000000),
 	REG(SYS_PTP_STATUS,			0x000f14),
 	REG(SYS_PTP_TXSTAMP,			0x000f18),
 	REG(SYS_PTP_NXT,			0x000f1c),
@@ -474,100 +477,43 @@ static const u32 *vsc9959_regmap[TARGET_MAX] = {
 };
 
 /* Addresses are relative to the PCI device's base address */
-static const struct resource vsc9959_target_io_res[TARGET_MAX] = {
-	[ANA] = {
-		.start	= 0x0280000,
-		.end	= 0x028ffff,
-		.name	= "ana",
-	},
-	[QS] = {
-		.start	= 0x0080000,
-		.end	= 0x00800ff,
-		.name	= "qs",
-	},
-	[QSYS] = {
-		.start	= 0x0200000,
-		.end	= 0x021ffff,
-		.name	= "qsys",
-	},
-	[REW] = {
-		.start	= 0x0030000,
-		.end	= 0x003ffff,
-		.name	= "rew",
-	},
-	[SYS] = {
-		.start	= 0x0010000,
-		.end	= 0x001ffff,
-		.name	= "sys",
-	},
-	[S0] = {
-		.start	= 0x0040000,
-		.end	= 0x00403ff,
-		.name	= "s0",
-	},
-	[S1] = {
-		.start	= 0x0050000,
-		.end	= 0x00503ff,
-		.name	= "s1",
-	},
-	[S2] = {
-		.start	= 0x0060000,
-		.end	= 0x00603ff,
-		.name	= "s2",
-	},
-	[PTP] = {
-		.start	= 0x0090000,
-		.end	= 0x00900cb,
-		.name	= "ptp",
-	},
-	[GCB] = {
-		.start	= 0x0070000,
-		.end	= 0x00701ff,
-		.name	= "devcpu_gcb",
-	},
+static const struct resource vsc9959_resources[] = {
+	DEFINE_RES_MEM_NAMED(0x0010000, 0x0010000, "sys"),
+	DEFINE_RES_MEM_NAMED(0x0030000, 0x0010000, "rew"),
+	DEFINE_RES_MEM_NAMED(0x0040000, 0x0000400, "s0"),
+	DEFINE_RES_MEM_NAMED(0x0050000, 0x0000400, "s1"),
+	DEFINE_RES_MEM_NAMED(0x0060000, 0x0000400, "s2"),
+	DEFINE_RES_MEM_NAMED(0x0070000, 0x0000200, "devcpu_gcb"),
+	DEFINE_RES_MEM_NAMED(0x0080000, 0x0000100, "qs"),
+	DEFINE_RES_MEM_NAMED(0x0090000, 0x00000cc, "ptp"),
+	DEFINE_RES_MEM_NAMED(0x0100000, 0x0010000, "port0"),
+	DEFINE_RES_MEM_NAMED(0x0110000, 0x0010000, "port1"),
+	DEFINE_RES_MEM_NAMED(0x0120000, 0x0010000, "port2"),
+	DEFINE_RES_MEM_NAMED(0x0130000, 0x0010000, "port3"),
+	DEFINE_RES_MEM_NAMED(0x0140000, 0x0010000, "port4"),
+	DEFINE_RES_MEM_NAMED(0x0150000, 0x0010000, "port5"),
+	DEFINE_RES_MEM_NAMED(0x0200000, 0x0020000, "qsys"),
+	DEFINE_RES_MEM_NAMED(0x0280000, 0x0010000, "ana"),
 };
 
-static const struct resource vsc9959_port_io_res[] = {
-	{
-		.start	= 0x0100000,
-		.end	= 0x010ffff,
-		.name	= "port0",
-	},
-	{
-		.start	= 0x0110000,
-		.end	= 0x011ffff,
-		.name	= "port1",
-	},
-	{
-		.start	= 0x0120000,
-		.end	= 0x012ffff,
-		.name	= "port2",
-	},
-	{
-		.start	= 0x0130000,
-		.end	= 0x013ffff,
-		.name	= "port3",
-	},
-	{
-		.start	= 0x0140000,
-		.end	= 0x014ffff,
-		.name	= "port4",
-	},
-	{
-		.start	= 0x0150000,
-		.end	= 0x015ffff,
-		.name	= "port5",
-	},
+static const char * const vsc9959_resource_names[TARGET_MAX] = {
+	[SYS] = "sys",
+	[REW] = "rew",
+	[S0] = "s0",
+	[S1] = "s1",
+	[S2] = "s2",
+	[GCB] = "devcpu_gcb",
+	[QS] = "qs",
+	[PTP] = "ptp",
+	[QSYS] = "qsys",
+	[ANA] = "ana",
 };
 
 /* Port MAC 0 Internal MDIO bus through which the SerDes acting as an
  * SGMII/QSGMII MAC PCS can be found.
  */
-static const struct resource vsc9959_imdio_res = {
-	.start		= 0x8030,
-	.end		= 0x8040,
-	.name		= "imdio",
-};
+static const struct resource vsc9959_imdio_res =
+	DEFINE_RES_MEM_NAMED(0x8030, 0x8040, "imdio");
 
 static const struct reg_field vsc9959_regfields[REGFIELD_MAX] = {
 	[ANA_ADVLEARN_VLAN_CHK] = REG_FIELD(ANA_ADVLEARN, 6, 6),
@@ -620,378 +566,7 @@ static const struct reg_field vsc9959_regfields[REGFIELD_MAX] = {
 };
 
 static const struct ocelot_stat_layout vsc9959_stats_layout[OCELOT_NUM_STATS] = {
-	[OCELOT_STAT_RX_OCTETS] = {
-		.name = "rx_octets",
-		.reg = SYS_COUNT_RX_OCTETS,
-	},
-	[OCELOT_STAT_RX_UNICAST] = {
-		.name = "rx_unicast",
-		.reg = SYS_COUNT_RX_UNICAST,
-	},
-	[OCELOT_STAT_RX_MULTICAST] = {
-		.name = "rx_multicast",
-		.reg = SYS_COUNT_RX_MULTICAST,
-	},
-	[OCELOT_STAT_RX_BROADCAST] = {
-		.name = "rx_broadcast",
-		.reg = SYS_COUNT_RX_BROADCAST,
-	},
-	[OCELOT_STAT_RX_SHORTS] = {
-		.name = "rx_shorts",
-		.reg = SYS_COUNT_RX_SHORTS,
-	},
-	[OCELOT_STAT_RX_FRAGMENTS] = {
-		.name = "rx_fragments",
-		.reg = SYS_COUNT_RX_FRAGMENTS,
-	},
-	[OCELOT_STAT_RX_JABBERS] = {
-		.name = "rx_jabbers",
-		.reg = SYS_COUNT_RX_JABBERS,
-	},
-	[OCELOT_STAT_RX_CRC_ALIGN_ERRS] = {
-		.name = "rx_crc_align_errs",
-		.reg = SYS_COUNT_RX_CRC_ALIGN_ERRS,
-	},
-	[OCELOT_STAT_RX_SYM_ERRS] = {
-		.name = "rx_sym_errs",
-		.reg = SYS_COUNT_RX_SYM_ERRS,
-	},
-	[OCELOT_STAT_RX_64] = {
-		.name = "rx_frames_below_65_octets",
-		.reg = SYS_COUNT_RX_64,
-	},
-	[OCELOT_STAT_RX_65_127] = {
-		.name = "rx_frames_65_to_127_octets",
-		.reg = SYS_COUNT_RX_65_127,
-	},
-	[OCELOT_STAT_RX_128_255] = {
-		.name = "rx_frames_128_to_255_octets",
-		.reg = SYS_COUNT_RX_128_255,
-	},
-	[OCELOT_STAT_RX_256_511] = {
-		.name = "rx_frames_256_to_511_octets",
-		.reg = SYS_COUNT_RX_256_511,
-	},
-	[OCELOT_STAT_RX_512_1023] = {
-		.name = "rx_frames_512_to_1023_octets",
-		.reg = SYS_COUNT_RX_512_1023,
-	},
-	[OCELOT_STAT_RX_1024_1526] = {
-		.name = "rx_frames_1024_to_1526_octets",
-		.reg = SYS_COUNT_RX_1024_1526,
-	},
-	[OCELOT_STAT_RX_1527_MAX] = {
-		.name = "rx_frames_over_1526_octets",
-		.reg = SYS_COUNT_RX_1527_MAX,
-	},
-	[OCELOT_STAT_RX_PAUSE] = {
-		.name = "rx_pause",
-		.reg = SYS_COUNT_RX_PAUSE,
-	},
-	[OCELOT_STAT_RX_CONTROL] = {
-		.name = "rx_control",
-		.reg = SYS_COUNT_RX_CONTROL,
-	},
-	[OCELOT_STAT_RX_LONGS] = {
-		.name = "rx_longs",
-		.reg = SYS_COUNT_RX_LONGS,
-	},
-	[OCELOT_STAT_RX_CLASSIFIED_DROPS] = {
-		.name = "rx_classified_drops",
-		.reg = SYS_COUNT_RX_CLASSIFIED_DROPS,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_0] = {
-		.name = "rx_red_prio_0",
-		.reg = SYS_COUNT_RX_RED_PRIO_0,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_1] = {
-		.name = "rx_red_prio_1",
-		.reg = SYS_COUNT_RX_RED_PRIO_1,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_2] = {
-		.name = "rx_red_prio_2",
-		.reg = SYS_COUNT_RX_RED_PRIO_2,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_3] = {
-		.name = "rx_red_prio_3",
-		.reg = SYS_COUNT_RX_RED_PRIO_3,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_4] = {
-		.name = "rx_red_prio_4",
-		.reg = SYS_COUNT_RX_RED_PRIO_4,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_5] = {
-		.name = "rx_red_prio_5",
-		.reg = SYS_COUNT_RX_RED_PRIO_5,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_6] = {
-		.name = "rx_red_prio_6",
-		.reg = SYS_COUNT_RX_RED_PRIO_6,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_7] = {
-		.name = "rx_red_prio_7",
-		.reg = SYS_COUNT_RX_RED_PRIO_7,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_0] = {
-		.name = "rx_yellow_prio_0",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_0,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_1] = {
-		.name = "rx_yellow_prio_1",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_1,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_2] = {
-		.name = "rx_yellow_prio_2",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_2,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_3] = {
-		.name = "rx_yellow_prio_3",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_3,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_4] = {
-		.name = "rx_yellow_prio_4",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_4,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_5] = {
-		.name = "rx_yellow_prio_5",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_5,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_6] = {
-		.name = "rx_yellow_prio_6",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_6,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_7] = {
-		.name = "rx_yellow_prio_7",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_7,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_0] = {
-		.name = "rx_green_prio_0",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_0,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_1] = {
-		.name = "rx_green_prio_1",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_1,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_2] = {
-		.name = "rx_green_prio_2",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_2,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_3] = {
-		.name = "rx_green_prio_3",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_3,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_4] = {
-		.name = "rx_green_prio_4",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_4,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_5] = {
-		.name = "rx_green_prio_5",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_5,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_6] = {
-		.name = "rx_green_prio_6",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_6,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_7] = {
-		.name = "rx_green_prio_7",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_7,
-	},
-	[OCELOT_STAT_TX_OCTETS] = {
-		.name = "tx_octets",
-		.reg = SYS_COUNT_TX_OCTETS,
-	},
-	[OCELOT_STAT_TX_UNICAST] = {
-		.name = "tx_unicast",
-		.reg = SYS_COUNT_TX_UNICAST,
-	},
-	[OCELOT_STAT_TX_MULTICAST] = {
-		.name = "tx_multicast",
-		.reg = SYS_COUNT_TX_MULTICAST,
-	},
-	[OCELOT_STAT_TX_BROADCAST] = {
-		.name = "tx_broadcast",
-		.reg = SYS_COUNT_TX_BROADCAST,
-	},
-	[OCELOT_STAT_TX_COLLISION] = {
-		.name = "tx_collision",
-		.reg = SYS_COUNT_TX_COLLISION,
-	},
-	[OCELOT_STAT_TX_DROPS] = {
-		.name = "tx_drops",
-		.reg = SYS_COUNT_TX_DROPS,
-	},
-	[OCELOT_STAT_TX_PAUSE] = {
-		.name = "tx_pause",
-		.reg = SYS_COUNT_TX_PAUSE,
-	},
-	[OCELOT_STAT_TX_64] = {
-		.name = "tx_frames_below_65_octets",
-		.reg = SYS_COUNT_TX_64,
-	},
-	[OCELOT_STAT_TX_65_127] = {
-		.name = "tx_frames_65_to_127_octets",
-		.reg = SYS_COUNT_TX_65_127,
-	},
-	[OCELOT_STAT_TX_128_255] = {
-		.name = "tx_frames_128_255_octets",
-		.reg = SYS_COUNT_TX_128_255,
-	},
-	[OCELOT_STAT_TX_256_511] = {
-		.name = "tx_frames_256_511_octets",
-		.reg = SYS_COUNT_TX_256_511,
-	},
-	[OCELOT_STAT_TX_512_1023] = {
-		.name = "tx_frames_512_1023_octets",
-		.reg = SYS_COUNT_TX_512_1023,
-	},
-	[OCELOT_STAT_TX_1024_1526] = {
-		.name = "tx_frames_1024_1526_octets",
-		.reg = SYS_COUNT_TX_1024_1526,
-	},
-	[OCELOT_STAT_TX_1527_MAX] = {
-		.name = "tx_frames_over_1526_octets",
-		.reg = SYS_COUNT_TX_1527_MAX,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_0] = {
-		.name = "tx_yellow_prio_0",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_0,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_1] = {
-		.name = "tx_yellow_prio_1",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_1,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_2] = {
-		.name = "tx_yellow_prio_2",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_2,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_3] = {
-		.name = "tx_yellow_prio_3",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_3,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_4] = {
-		.name = "tx_yellow_prio_4",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_4,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_5] = {
-		.name = "tx_yellow_prio_5",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_5,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_6] = {
-		.name = "tx_yellow_prio_6",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_6,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_7] = {
-		.name = "tx_yellow_prio_7",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_7,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_0] = {
-		.name = "tx_green_prio_0",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_0,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_1] = {
-		.name = "tx_green_prio_1",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_1,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_2] = {
-		.name = "tx_green_prio_2",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_2,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_3] = {
-		.name = "tx_green_prio_3",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_3,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_4] = {
-		.name = "tx_green_prio_4",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_4,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_5] = {
-		.name = "tx_green_prio_5",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_5,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_6] = {
-		.name = "tx_green_prio_6",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_6,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_7] = {
-		.name = "tx_green_prio_7",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_7,
-	},
-	[OCELOT_STAT_TX_AGED] = {
-		.name = "tx_aged",
-		.reg = SYS_COUNT_TX_AGING,
-	},
-	[OCELOT_STAT_DROP_LOCAL] = {
-		.name = "drop_local",
-		.reg = SYS_COUNT_DROP_LOCAL,
-	},
-	[OCELOT_STAT_DROP_TAIL] = {
-		.name = "drop_tail",
-		.reg = SYS_COUNT_DROP_TAIL,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_0] = {
-		.name = "drop_yellow_prio_0",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_0,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_1] = {
-		.name = "drop_yellow_prio_1",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_1,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_2] = {
-		.name = "drop_yellow_prio_2",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_2,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_3] = {
-		.name = "drop_yellow_prio_3",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_3,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_4] = {
-		.name = "drop_yellow_prio_4",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_4,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_5] = {
-		.name = "drop_yellow_prio_5",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_5,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_6] = {
-		.name = "drop_yellow_prio_6",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_6,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_7] = {
-		.name = "drop_yellow_prio_7",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_7,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_0] = {
-		.name = "drop_green_prio_0",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_0,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_1] = {
-		.name = "drop_green_prio_1",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_1,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_2] = {
-		.name = "drop_green_prio_2",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_2,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_3] = {
-		.name = "drop_green_prio_3",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_3,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_4] = {
-		.name = "drop_green_prio_4",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_4,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_5] = {
-		.name = "drop_green_prio_5",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_5,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_6] = {
-		.name = "drop_green_prio_6",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_6,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_7] = {
-		.name = "drop_green_prio_7",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_7,
-	},
+	OCELOT_COMMON_STATS,
 };
 
 static const struct vcap_field vsc9959_vcap_es0_keys[] = {
@@ -1371,9 +946,11 @@ static void vsc9959_wm_stat(u32 val, u32 *inuse, u32 *maxuse)
 
 static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot)
 {
+	struct pci_dev *pdev = to_pci_dev(ocelot->dev);
 	struct felix *felix = ocelot_to_felix(ocelot);
 	struct enetc_mdio_priv *mdio_priv;
 	struct device *dev = ocelot->dev;
+	resource_size_t imdio_base;
 	void __iomem *imdio_regs;
 	struct resource res;
 	struct enetc_hw *hw;
@@ -1389,10 +966,11 @@ static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot)
 		return -ENOMEM;
 	}
 
-	memcpy(&res, felix->info->imdio_res, sizeof(res));
-	res.flags = IORESOURCE_MEM;
-	res.start += felix->imdio_base;
-	res.end += felix->imdio_base;
+	imdio_base = pci_resource_start(pdev, VSC9959_IMDIO_PCI_BAR);
+
+	memcpy(&res, &vsc9959_imdio_res, sizeof(res));
+	res.start += imdio_base;
+	res.end += imdio_base;
 
 	imdio_regs = devm_ioremap_resource(dev, &res);
 	if (IS_ERR(imdio_regs))
@@ -1616,6 +1194,14 @@ static u32 vsc9959_port_qmaxsdu_get(struct ocelot *ocelot, int port, int tc)
 	}
 }
 
+static u32 vsc9959_tas_tc_max_sdu(struct tc_taprio_qopt_offload *taprio, int tc)
+{
+	if (!taprio || !taprio->max_sdu[tc])
+		return 0;
+
+	return taprio->max_sdu[tc] + ETH_HLEN + 2 * VLAN_HLEN + ETH_FCS_LEN;
+}
+
 /* Update QSYS_PORT_MAX_SDU to make sure the static guard bands added by the
  * switch (see the ALWAYS_GUARD_BAND_SCH_Q comment) are correct at all MTU
  * values (the default value is 1518). Also, for traffic class windows smaller
@@ -1625,6 +1211,7 @@ static u32 vsc9959_port_qmaxsdu_get(struct ocelot *ocelot, int port, int tc)
 static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port)
 {
 	struct ocelot_port *ocelot_port = ocelot->ports[port];
+	struct tc_taprio_qopt_offload *taprio;
 	u64 min_gate_len[OCELOT_NUM_TC];
 	int speed, picos_per_byte;
 	u64 needed_bit_time_ps;
@@ -1634,6 +1221,8 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port)
 
 	lockdep_assert_held(&ocelot->tas_lock);
 
+	taprio = ocelot_port->taprio;
+
 	val = ocelot_read_rix(ocelot, QSYS_TAG_CONFIG, port);
 	tas_speed = QSYS_TAG_CONFIG_LINK_SPEED_X(val);
 
@@ -1670,11 +1259,12 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port)
 		"port %d: max frame size %d needs %llu ps at speed %d\n",
 		port, maxlen, needed_bit_time_ps, speed);
 
-	vsc9959_tas_min_gate_lengths(ocelot_port->taprio, min_gate_len);
+	vsc9959_tas_min_gate_lengths(taprio, min_gate_len);
 
 	mutex_lock(&ocelot->fwd_domain_lock);
 
 	for (tc = 0; tc < OCELOT_NUM_TC; tc++) {
+		u32 requested_max_sdu = vsc9959_tas_tc_max_sdu(taprio, tc);
 		u64 remaining_gate_len_ps;
 		u32 max_sdu;
 
@@ -1685,7 +1275,7 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port)
 			/* Setting QMAXSDU_CFG to 0 disables oversized frame
 			 * dropping.
 			 */
-			max_sdu = 0;
+			max_sdu = requested_max_sdu;
 			dev_dbg(ocelot->dev,
 				"port %d tc %d min gate len %llu"
 				", sending all frames\n",
@@ -1716,6 +1306,10 @@ static void vsc9959_tas_guard_bands_update(struct ocelot *ocelot, int port)
 			 */
 			if (max_sdu > 20)
 				max_sdu -= 20;
+
+			if (requested_max_sdu && requested_max_sdu < max_sdu)
+				max_sdu = requested_max_sdu;
+
 			dev_info(ocelot->dev,
 				 "port %d tc %d min gate length %llu"
 				 " ns not enough for max frame size %d at %d"
@@ -2005,6 +1599,21 @@ static int vsc9959_qos_port_cbs_set(struct dsa_switch *ds, int port,
 	return 0;
 }
 
+static int vsc9959_qos_query_caps(struct tc_query_caps_base *base)
+{
+	switch (base->type) {
+	case TC_SETUP_QDISC_TAPRIO: {
+		struct tc_taprio_caps *caps = base->caps;
+
+		caps->supports_queue_max_sdu = true;
+
+		return 0;
+	}
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 static int vsc9959_port_setup_tc(struct dsa_switch *ds, int port,
 				 enum tc_setup_type type,
 				 void *type_data)
@@ -2012,6 +1621,8 @@ static int vsc9959_port_setup_tc(struct dsa_switch *ds, int port,
 	struct ocelot *ocelot = ds->priv;
 
 	switch (type) {
+	case TC_QUERY_CAPS:
+		return vsc9959_qos_query_caps(type_data);
 	case TC_SETUP_QDISC_TAPRIO:
 		return vsc9959_qos_port_tas_set(ocelot, port, type_data);
 	case TC_SETUP_QDISC_CBS:
@@ -2043,7 +1654,15 @@ struct felix_stream {
 	u32 ssid;
 };
 
+struct felix_stream_filter_counters {
+	u64 match;
+	u64 not_pass_gate;
+	u64 not_pass_sdu;
+	u64 red;
+};
+
 struct felix_stream_filter {
+	struct felix_stream_filter_counters stats;
 	struct list_head list;
 	refcount_t refcount;
 	u32 index;
@@ -2058,13 +1677,6 @@ struct felix_stream_filter {
 	u32 maxsdu;
 };
 
-struct felix_stream_filter_counters {
-	u32 match;
-	u32 not_pass_gate;
-	u32 not_pass_sdu;
-	u32 red;
-};
-
 struct felix_stream_gate {
 	u32 index;
 	u8 enable;
@@ -2568,29 +2180,6 @@ static void vsc9959_psfp_sgi_table_del(struct ocelot *ocelot,
 		}
 }
 
-static void vsc9959_psfp_counters_get(struct ocelot *ocelot, u32 index,
-				      struct felix_stream_filter_counters *counters)
-{
-	spin_lock(&ocelot->stats_lock);
-
-	ocelot_rmw(ocelot, SYS_STAT_CFG_STAT_VIEW(index),
-		   SYS_STAT_CFG_STAT_VIEW_M,
-		   SYS_STAT_CFG);
-
-	counters->match = ocelot_read_gix(ocelot, SYS_CNT, 0x200);
-	counters->not_pass_gate = ocelot_read_gix(ocelot, SYS_CNT, 0x201);
-	counters->not_pass_sdu = ocelot_read_gix(ocelot, SYS_CNT, 0x202);
-	counters->red = ocelot_read_gix(ocelot, SYS_CNT, 0x203);
-
-	/* Clear the PSFP counter. */
-	ocelot_write(ocelot,
-		     SYS_STAT_CFG_STAT_VIEW(index) |
-		     SYS_STAT_CFG_STAT_CLEAR_SHOT(0x10),
-		     SYS_STAT_CFG);
-
-	spin_unlock(&ocelot->stats_lock);
-}
-
 static int vsc9959_psfp_filter_add(struct ocelot *ocelot, int port,
 				   struct flow_cls_offload *f)
 {
@@ -2615,6 +2204,8 @@ static int vsc9959_psfp_filter_add(struct ocelot *ocelot, int port,
 		return ret;
 	}
 
+	mutex_lock(&psfp->lock);
+
 	flow_action_for_each(i, a, &f->rule->action) {
 		switch (a->id) {
 		case FLOW_ACTION_GATE:
@@ -2656,6 +2247,7 @@ static int vsc9959_psfp_filter_add(struct ocelot *ocelot, int port,
 			sfi.maxsdu = a->police.mtu;
 			break;
 		default:
+			mutex_unlock(&psfp->lock);
 			return -EOPNOTSUPP;
 		}
 	}
@@ -2725,6 +2317,8 @@ static int vsc9959_psfp_filter_add(struct ocelot *ocelot, int port,
 		goto err;
 	}
 
+	mutex_unlock(&psfp->lock);
+
 	return 0;
 
 err:
@@ -2734,6 +2328,8 @@ err:
 	if (sfi.fm_valid)
 		ocelot_vcap_policer_del(ocelot, sfi.fmid);
 
+	mutex_unlock(&psfp->lock);
+
 	return ret;
 }
 
@@ -2741,18 +2337,22 @@ static int vsc9959_psfp_filter_del(struct ocelot *ocelot,
 				   struct flow_cls_offload *f)
 {
 	struct felix_stream *stream, tmp, *stream_entry;
+	struct ocelot_psfp_list *psfp = &ocelot->psfp;
 	static struct felix_stream_filter *sfi;
-	struct ocelot_psfp_list *psfp;
 
-	psfp = &ocelot->psfp;
+	mutex_lock(&psfp->lock);
 
 	stream = vsc9959_stream_table_get(&psfp->stream_list, f->cookie);
-	if (!stream)
+	if (!stream) {
+		mutex_unlock(&psfp->lock);
 		return -ENOMEM;
+	}
 
 	sfi = vsc9959_psfp_sfi_table_get(&psfp->sfi_list, stream->sfid);
-	if (!sfi)
+	if (!sfi) {
+		mutex_unlock(&psfp->lock);
 		return -ENOMEM;
+	}
 
 	if (sfi->sg_valid)
 		vsc9959_psfp_sgi_table_del(ocelot, sfi->sgid);
@@ -2778,27 +2378,83 @@ static int vsc9959_psfp_filter_del(struct ocelot *ocelot,
 					  stream_entry->ports);
 	}
 
+	mutex_unlock(&psfp->lock);
+
 	return 0;
 }
 
+static void vsc9959_update_sfid_stats(struct ocelot *ocelot,
+				      struct felix_stream_filter *sfi)
+{
+	struct felix_stream_filter_counters *s = &sfi->stats;
+	u32 match, not_pass_gate, not_pass_sdu, red;
+	u32 sfid = sfi->index;
+
+	lockdep_assert_held(&ocelot->stat_view_lock);
+
+	ocelot_rmw(ocelot, SYS_STAT_CFG_STAT_VIEW(sfid),
+		   SYS_STAT_CFG_STAT_VIEW_M,
+		   SYS_STAT_CFG);
+
+	match = ocelot_read(ocelot, SYS_COUNT_SF_MATCHING_FRAMES);
+	not_pass_gate = ocelot_read(ocelot, SYS_COUNT_SF_NOT_PASSING_FRAMES);
+	not_pass_sdu = ocelot_read(ocelot, SYS_COUNT_SF_NOT_PASSING_SDU);
+	red = ocelot_read(ocelot, SYS_COUNT_SF_RED_FRAMES);
+
+	/* Clear the PSFP counter. */
+	ocelot_write(ocelot,
+		     SYS_STAT_CFG_STAT_VIEW(sfid) |
+		     SYS_STAT_CFG_STAT_CLEAR_SHOT(0x10),
+		     SYS_STAT_CFG);
+
+	s->match += match;
+	s->not_pass_gate += not_pass_gate;
+	s->not_pass_sdu += not_pass_sdu;
+	s->red += red;
+}
+
+/* Caller must hold &ocelot->stat_view_lock */
+static void vsc9959_update_stats(struct ocelot *ocelot)
+{
+	struct ocelot_psfp_list *psfp = &ocelot->psfp;
+	struct felix_stream_filter *sfi;
+
+	mutex_lock(&psfp->lock);
+
+	list_for_each_entry(sfi, &psfp->sfi_list, list)
+		vsc9959_update_sfid_stats(ocelot, sfi);
+
+	mutex_unlock(&psfp->lock);
+}
+
 static int vsc9959_psfp_stats_get(struct ocelot *ocelot,
 				  struct flow_cls_offload *f,
 				  struct flow_stats *stats)
 {
-	struct felix_stream_filter_counters counters;
-	struct ocelot_psfp_list *psfp;
+	struct ocelot_psfp_list *psfp = &ocelot->psfp;
+	struct felix_stream_filter_counters *s;
+	static struct felix_stream_filter *sfi;
 	struct felix_stream *stream;
 
-	psfp = &ocelot->psfp;
 	stream = vsc9959_stream_table_get(&psfp->stream_list, f->cookie);
 	if (!stream)
 		return -ENOMEM;
 
-	vsc9959_psfp_counters_get(ocelot, stream->sfid, &counters);
+	sfi = vsc9959_psfp_sfi_table_get(&psfp->sfi_list, stream->sfid);
+	if (!sfi)
+		return -EINVAL;
+
+	mutex_lock(&ocelot->stat_view_lock);
+
+	vsc9959_update_sfid_stats(ocelot, sfi);
+
+	s = &sfi->stats;
+	stats->pkts = s->match;
+	stats->drops = s->not_pass_gate + s->not_pass_sdu + s->red;
 
-	stats->pkts = counters.match;
-	stats->drops = counters.not_pass_gate + counters.not_pass_sdu +
-		       counters.red;
+	memset(s, 0, sizeof(*s));
+
+	mutex_unlock(&ocelot->stat_view_lock);
 
 	return 0;
 }
@@ -2810,6 +2466,7 @@ static void vsc9959_psfp_init(struct ocelot *ocelot)
 	INIT_LIST_HEAD(&psfp->stream_list);
 	INIT_LIST_HEAD(&psfp->sfi_list);
 	INIT_LIST_HEAD(&psfp->sgi_list);
+	mutex_init(&psfp->lock);
 }
 
 /* When using cut-through forwarding and the egress port runs at a higher data
@@ -2908,12 +2565,13 @@ static const struct ocelot_ops vsc9959_ops = {
 	.psfp_stats_get		= vsc9959_psfp_stats_get,
 	.cut_through_fwd	= vsc9959_cut_through_fwd,
 	.tas_clock_adjust	= vsc9959_tas_clock_adjust,
+	.update_stats		= vsc9959_update_stats,
 };
 
 static const struct felix_info felix_info_vsc9959 = {
-	.target_io_res		= vsc9959_target_io_res,
-	.port_io_res		= vsc9959_port_io_res,
-	.imdio_res		= &vsc9959_imdio_res,
+	.resources		= vsc9959_resources,
+	.num_resources		= ARRAY_SIZE(vsc9959_resources),
+	.resource_names		= vsc9959_resource_names,
 	.regfields		= vsc9959_regfields,
 	.map			= vsc9959_regmap,
 	.ops			= &vsc9959_ops,
@@ -2935,7 +2593,6 @@ static const struct felix_info felix_info_vsc9959 = {
 	.port_setup_tc		= vsc9959_port_setup_tc,
 	.port_sched_speed_set	= vsc9959_sched_speed_set,
 	.tas_guard_bands_update	= vsc9959_tas_guard_bands_update,
-	.init_regmap		= ocelot_regmap_init,
 };
 
 static irqreturn_t felix_irq_handler(int irq, void *data)
@@ -2987,7 +2644,6 @@ static int felix_pci_probe(struct pci_dev *pdev,
 	ocelot->num_flooding_pgids = OCELOT_NUM_TC;
 	felix->info = &felix_info_vsc9959;
 	felix->switch_base = pci_resource_start(pdev, VSC9959_SWITCH_PCI_BAR);
-	felix->imdio_base = pci_resource_start(pdev, VSC9959_IMDIO_PCI_BAR);
 
 	pci_set_master(pdev);
 
@@ -3048,8 +2704,6 @@ static void felix_pci_remove(struct pci_dev *pdev)
 	kfree(felix);
 
 	pci_disable_device(pdev);
-
-	pci_set_drvdata(pdev, NULL);
 }
 
 static void felix_pci_shutdown(struct pci_dev *pdev)
diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c
index b34f4cdfe814..7af33b2c685d 100644
--- a/drivers/net/dsa/ocelot/seville_vsc9953.c
+++ b/drivers/net/dsa/ocelot/seville_vsc9953.c
@@ -343,7 +343,7 @@ static const u32 vsc9953_sys_regmap[] = {
 	REG(SYS_COUNT_TX_GREEN_PRIO_5,		0x00016c),
 	REG(SYS_COUNT_TX_GREEN_PRIO_6,		0x000170),
 	REG(SYS_COUNT_TX_GREEN_PRIO_7,		0x000174),
-	REG(SYS_COUNT_TX_AGING,			0x000178),
+	REG(SYS_COUNT_TX_AGED,			0x000178),
 	REG(SYS_COUNT_DROP_LOCAL,		0x000200),
 	REG(SYS_COUNT_DROP_TAIL,		0x000204),
 	REG(SYS_COUNT_DROP_YELLOW_PRIO_0,	0x000208),
@@ -383,7 +383,6 @@ static const u32 vsc9953_sys_regmap[] = {
 	REG_RESERVED(SYS_MMGT_FAST),
 	REG_RESERVED(SYS_EVENTS_DIF),
 	REG_RESERVED(SYS_EVENTS_CORE),
-	REG_RESERVED(SYS_CNT),
 	REG_RESERVED(SYS_PTP_STATUS),
 	REG_RESERVED(SYS_PTP_TXSTAMP),
 	REG_RESERVED(SYS_PTP_NXT),
@@ -459,110 +458,40 @@ static const u32 *vsc9953_regmap[TARGET_MAX] = {
 };
 
 /* Addresses are relative to the device's base address */
-static const struct resource vsc9953_target_io_res[TARGET_MAX] = {
-	[ANA] = {
-		.start	= 0x0280000,
-		.end	= 0x028ffff,
-		.name	= "ana",
-	},
-	[QS] = {
-		.start	= 0x0080000,
-		.end	= 0x00800ff,
-		.name	= "qs",
-	},
-	[QSYS] = {
-		.start	= 0x0200000,
-		.end	= 0x021ffff,
-		.name	= "qsys",
-	},
-	[REW] = {
-		.start	= 0x0030000,
-		.end	= 0x003ffff,
-		.name	= "rew",
-	},
-	[SYS] = {
-		.start	= 0x0010000,
-		.end	= 0x001ffff,
-		.name	= "sys",
-	},
-	[S0] = {
-		.start	= 0x0040000,
-		.end	= 0x00403ff,
-		.name	= "s0",
-	},
-	[S1] = {
-		.start	= 0x0050000,
-		.end	= 0x00503ff,
-		.name	= "s1",
-	},
-	[S2] = {
-		.start	= 0x0060000,
-		.end	= 0x00603ff,
-		.name	= "s2",
-	},
-	[PTP] = {
-		.start	= 0x0090000,
-		.end	= 0x00900cb,
-		.name	= "ptp",
-	},
-	[GCB] = {
-		.start	= 0x0070000,
-		.end	= 0x00701ff,
-		.name	= "devcpu_gcb",
-	},
+static const struct resource vsc9953_resources[] = {
+	DEFINE_RES_MEM_NAMED(0x0010000, 0x0010000, "sys"),
+	DEFINE_RES_MEM_NAMED(0x0030000, 0x0010000, "rew"),
+	DEFINE_RES_MEM_NAMED(0x0040000, 0x0000400, "s0"),
+	DEFINE_RES_MEM_NAMED(0x0050000, 0x0000400, "s1"),
+	DEFINE_RES_MEM_NAMED(0x0060000, 0x0000400, "s2"),
+	DEFINE_RES_MEM_NAMED(0x0070000, 0x0000200, "devcpu_gcb"),
+	DEFINE_RES_MEM_NAMED(0x0080000, 0x0000100, "qs"),
+	DEFINE_RES_MEM_NAMED(0x0090000, 0x00000cc, "ptp"),
+	DEFINE_RES_MEM_NAMED(0x0100000, 0x0010000, "port0"),
+	DEFINE_RES_MEM_NAMED(0x0110000, 0x0010000, "port1"),
+	DEFINE_RES_MEM_NAMED(0x0120000, 0x0010000, "port2"),
+	DEFINE_RES_MEM_NAMED(0x0130000, 0x0010000, "port3"),
+	DEFINE_RES_MEM_NAMED(0x0140000, 0x0010000, "port4"),
+	DEFINE_RES_MEM_NAMED(0x0150000, 0x0010000, "port5"),
+	DEFINE_RES_MEM_NAMED(0x0160000, 0x0010000, "port6"),
+	DEFINE_RES_MEM_NAMED(0x0170000, 0x0010000, "port7"),
+	DEFINE_RES_MEM_NAMED(0x0180000, 0x0010000, "port8"),
+	DEFINE_RES_MEM_NAMED(0x0190000, 0x0010000, "port9"),
+	DEFINE_RES_MEM_NAMED(0x0200000, 0x0020000, "qsys"),
+	DEFINE_RES_MEM_NAMED(0x0280000, 0x0010000, "ana"),
 };
 
-static const struct resource vsc9953_port_io_res[] = {
-	{
-		.start	= 0x0100000,
-		.end	= 0x010ffff,
-		.name	= "port0",
-	},
-	{
-		.start	= 0x0110000,
-		.end	= 0x011ffff,
-		.name	= "port1",
-	},
-	{
-		.start	= 0x0120000,
-		.end	= 0x012ffff,
-		.name	= "port2",
-	},
-	{
-		.start	= 0x0130000,
-		.end	= 0x013ffff,
-		.name	= "port3",
-	},
-	{
-		.start	= 0x0140000,
-		.end	= 0x014ffff,
-		.name	= "port4",
-	},
-	{
-		.start	= 0x0150000,
-		.end	= 0x015ffff,
-		.name	= "port5",
-	},
-	{
-		.start	= 0x0160000,
-		.end	= 0x016ffff,
-		.name	= "port6",
-	},
-	{
-		.start	= 0x0170000,
-		.end	= 0x017ffff,
-		.name	= "port7",
-	},
-	{
-		.start	= 0x0180000,
-		.end	= 0x018ffff,
-		.name	= "port8",
-	},
-	{
-		.start	= 0x0190000,
-		.end	= 0x019ffff,
-		.name	= "port9",
-	},
+static const char * const vsc9953_resource_names[TARGET_MAX] = {
+	[SYS] = "sys",
+	[REW] = "rew",
+	[S0] = "s0",
+	[S1] = "s1",
+	[S2] = "s2",
+	[GCB] = "devcpu_gcb",
+	[QS] = "qs",
+	[PTP] = "ptp",
+	[QSYS] = "qsys",
+	[ANA] = "ana",
 };
 
 static const struct reg_field vsc9953_regfields[REGFIELD_MAX] = {
@@ -615,378 +544,7 @@ static const struct reg_field vsc9953_regfields[REGFIELD_MAX] = {
 };
 
 static const struct ocelot_stat_layout vsc9953_stats_layout[OCELOT_NUM_STATS] = {
-	[OCELOT_STAT_RX_OCTETS] = {
-		.name = "rx_octets",
-		.reg = SYS_COUNT_RX_OCTETS,
-	},
-	[OCELOT_STAT_RX_UNICAST] = {
-		.name = "rx_unicast",
-		.reg = SYS_COUNT_RX_UNICAST,
-	},
-	[OCELOT_STAT_RX_MULTICAST] = {
-		.name = "rx_multicast",
-		.reg = SYS_COUNT_RX_MULTICAST,
-	},
-	[OCELOT_STAT_RX_BROADCAST] = {
-		.name = "rx_broadcast",
-		.reg = SYS_COUNT_RX_BROADCAST,
-	},
-	[OCELOT_STAT_RX_SHORTS] = {
-		.name = "rx_shorts",
-		.reg = SYS_COUNT_RX_SHORTS,
-	},
-	[OCELOT_STAT_RX_FRAGMENTS] = {
-		.name = "rx_fragments",
-		.reg = SYS_COUNT_RX_FRAGMENTS,
-	},
-	[OCELOT_STAT_RX_JABBERS] = {
-		.name = "rx_jabbers",
-		.reg = SYS_COUNT_RX_JABBERS,
-	},
-	[OCELOT_STAT_RX_CRC_ALIGN_ERRS] = {
-		.name = "rx_crc_align_errs",
-		.reg = SYS_COUNT_RX_CRC_ALIGN_ERRS,
-	},
-	[OCELOT_STAT_RX_SYM_ERRS] = {
-		.name = "rx_sym_errs",
-		.reg = SYS_COUNT_RX_SYM_ERRS,
-	},
-	[OCELOT_STAT_RX_64] = {
-		.name = "rx_frames_below_65_octets",
-		.reg = SYS_COUNT_RX_64,
-	},
-	[OCELOT_STAT_RX_65_127] = {
-		.name = "rx_frames_65_to_127_octets",
-		.reg = SYS_COUNT_RX_65_127,
-	},
-	[OCELOT_STAT_RX_128_255] = {
-		.name = "rx_frames_128_to_255_octets",
-		.reg = SYS_COUNT_RX_128_255,
-	},
-	[OCELOT_STAT_RX_256_511] = {
-		.name = "rx_frames_256_to_511_octets",
-		.reg = SYS_COUNT_RX_256_511,
-	},
-	[OCELOT_STAT_RX_512_1023] = {
-		.name = "rx_frames_512_to_1023_octets",
-		.reg = SYS_COUNT_RX_512_1023,
-	},
-	[OCELOT_STAT_RX_1024_1526] = {
-		.name = "rx_frames_1024_to_1526_octets",
-		.reg = SYS_COUNT_RX_1024_1526,
-	},
-	[OCELOT_STAT_RX_1527_MAX] = {
-		.name = "rx_frames_over_1526_octets",
-		.reg = SYS_COUNT_RX_1527_MAX,
-	},
-	[OCELOT_STAT_RX_PAUSE] = {
-		.name = "rx_pause",
-		.reg = SYS_COUNT_RX_PAUSE,
-	},
-	[OCELOT_STAT_RX_CONTROL] = {
-		.name = "rx_control",
-		.reg = SYS_COUNT_RX_CONTROL,
-	},
-	[OCELOT_STAT_RX_LONGS] = {
-		.name = "rx_longs",
-		.reg = SYS_COUNT_RX_LONGS,
-	},
-	[OCELOT_STAT_RX_CLASSIFIED_DROPS] = {
-		.name = "rx_classified_drops",
-		.reg = SYS_COUNT_RX_CLASSIFIED_DROPS,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_0] = {
-		.name = "rx_red_prio_0",
-		.reg = SYS_COUNT_RX_RED_PRIO_0,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_1] = {
-		.name = "rx_red_prio_1",
-		.reg = SYS_COUNT_RX_RED_PRIO_1,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_2] = {
-		.name = "rx_red_prio_2",
-		.reg = SYS_COUNT_RX_RED_PRIO_2,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_3] = {
-		.name = "rx_red_prio_3",
-		.reg = SYS_COUNT_RX_RED_PRIO_3,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_4] = {
-		.name = "rx_red_prio_4",
-		.reg = SYS_COUNT_RX_RED_PRIO_4,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_5] = {
-		.name = "rx_red_prio_5",
-		.reg = SYS_COUNT_RX_RED_PRIO_5,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_6] = {
-		.name = "rx_red_prio_6",
-		.reg = SYS_COUNT_RX_RED_PRIO_6,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_7] = {
-		.name = "rx_red_prio_7",
-		.reg = SYS_COUNT_RX_RED_PRIO_7,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_0] = {
-		.name = "rx_yellow_prio_0",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_0,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_1] = {
-		.name = "rx_yellow_prio_1",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_1,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_2] = {
-		.name = "rx_yellow_prio_2",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_2,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_3] = {
-		.name = "rx_yellow_prio_3",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_3,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_4] = {
-		.name = "rx_yellow_prio_4",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_4,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_5] = {
-		.name = "rx_yellow_prio_5",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_5,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_6] = {
-		.name = "rx_yellow_prio_6",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_6,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_7] = {
-		.name = "rx_yellow_prio_7",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_7,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_0] = {
-		.name = "rx_green_prio_0",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_0,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_1] = {
-		.name = "rx_green_prio_1",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_1,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_2] = {
-		.name = "rx_green_prio_2",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_2,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_3] = {
-		.name = "rx_green_prio_3",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_3,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_4] = {
-		.name = "rx_green_prio_4",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_4,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_5] = {
-		.name = "rx_green_prio_5",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_5,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_6] = {
-		.name = "rx_green_prio_6",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_6,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_7] = {
-		.name = "rx_green_prio_7",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_7,
-	},
-	[OCELOT_STAT_TX_OCTETS] = {
-		.name = "tx_octets",
-		.reg = SYS_COUNT_TX_OCTETS,
-	},
-	[OCELOT_STAT_TX_UNICAST] = {
-		.name = "tx_unicast",
-		.reg = SYS_COUNT_TX_UNICAST,
-	},
-	[OCELOT_STAT_TX_MULTICAST] = {
-		.name = "tx_multicast",
-		.reg = SYS_COUNT_TX_MULTICAST,
-	},
-	[OCELOT_STAT_TX_BROADCAST] = {
-		.name = "tx_broadcast",
-		.reg = SYS_COUNT_TX_BROADCAST,
-	},
-	[OCELOT_STAT_TX_COLLISION] = {
-		.name = "tx_collision",
-		.reg = SYS_COUNT_TX_COLLISION,
-	},
-	[OCELOT_STAT_TX_DROPS] = {
-		.name = "tx_drops",
-		.reg = SYS_COUNT_TX_DROPS,
-	},
-	[OCELOT_STAT_TX_PAUSE] = {
-		.name = "tx_pause",
-		.reg = SYS_COUNT_TX_PAUSE,
-	},
-	[OCELOT_STAT_TX_64] = {
-		.name = "tx_frames_below_65_octets",
-		.reg = SYS_COUNT_TX_64,
-	},
-	[OCELOT_STAT_TX_65_127] = {
-		.name = "tx_frames_65_to_127_octets",
-		.reg = SYS_COUNT_TX_65_127,
-	},
-	[OCELOT_STAT_TX_128_255] = {
-		.name = "tx_frames_128_255_octets",
-		.reg = SYS_COUNT_TX_128_255,
-	},
-	[OCELOT_STAT_TX_256_511] = {
-		.name = "tx_frames_256_511_octets",
-		.reg = SYS_COUNT_TX_256_511,
-	},
-	[OCELOT_STAT_TX_512_1023] = {
-		.name = "tx_frames_512_1023_octets",
-		.reg = SYS_COUNT_TX_512_1023,
-	},
-	[OCELOT_STAT_TX_1024_1526] = {
-		.name = "tx_frames_1024_1526_octets",
-		.reg = SYS_COUNT_TX_1024_1526,
-	},
-	[OCELOT_STAT_TX_1527_MAX] = {
-		.name = "tx_frames_over_1526_octets",
-		.reg = SYS_COUNT_TX_1527_MAX,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_0] = {
-		.name = "tx_yellow_prio_0",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_0,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_1] = {
-		.name = "tx_yellow_prio_1",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_1,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_2] = {
-		.name = "tx_yellow_prio_2",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_2,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_3] = {
-		.name = "tx_yellow_prio_3",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_3,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_4] = {
-		.name = "tx_yellow_prio_4",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_4,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_5] = {
-		.name = "tx_yellow_prio_5",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_5,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_6] = {
-		.name = "tx_yellow_prio_6",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_6,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_7] = {
-		.name = "tx_yellow_prio_7",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_7,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_0] = {
-		.name = "tx_green_prio_0",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_0,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_1] = {
-		.name = "tx_green_prio_1",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_1,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_2] = {
-		.name = "tx_green_prio_2",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_2,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_3] = {
-		.name = "tx_green_prio_3",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_3,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_4] = {
-		.name = "tx_green_prio_4",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_4,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_5] = {
-		.name = "tx_green_prio_5",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_5,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_6] = {
-		.name = "tx_green_prio_6",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_6,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_7] = {
-		.name = "tx_green_prio_7",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_7,
-	},
-	[OCELOT_STAT_TX_AGED] = {
-		.name = "tx_aged",
-		.reg = SYS_COUNT_TX_AGING,
-	},
-	[OCELOT_STAT_DROP_LOCAL] = {
-		.name = "drop_local",
-		.reg = SYS_COUNT_DROP_LOCAL,
-	},
-	[OCELOT_STAT_DROP_TAIL] = {
-		.name = "drop_tail",
-		.reg = SYS_COUNT_DROP_TAIL,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_0] = {
-		.name = "drop_yellow_prio_0",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_0,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_1] = {
-		.name = "drop_yellow_prio_1",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_1,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_2] = {
-		.name = "drop_yellow_prio_2",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_2,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_3] = {
-		.name = "drop_yellow_prio_3",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_3,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_4] = {
-		.name = "drop_yellow_prio_4",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_4,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_5] = {
-		.name = "drop_yellow_prio_5",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_5,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_6] = {
-		.name = "drop_yellow_prio_6",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_6,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_7] = {
-		.name = "drop_yellow_prio_7",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_7,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_0] = {
-		.name = "drop_green_prio_0",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_0,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_1] = {
-		.name = "drop_green_prio_1",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_1,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_2] = {
-		.name = "drop_green_prio_2",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_2,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_3] = {
-		.name = "drop_green_prio_3",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_3,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_4] = {
-		.name = "drop_green_prio_4",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_4,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_5] = {
-		.name = "drop_green_prio_5",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_5,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_6] = {
-		.name = "drop_green_prio_6",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_6,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_7] = {
-		.name = "drop_green_prio_7",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_7,
-	},
+	OCELOT_COMMON_STATS,
 };
 
 static const struct vcap_field vsc9953_vcap_es0_keys[] = {
@@ -1432,8 +990,9 @@ static void vsc9953_mdio_bus_free(struct ocelot *ocelot)
 }
 
 static const struct felix_info seville_info_vsc9953 = {
-	.target_io_res		= vsc9953_target_io_res,
-	.port_io_res		= vsc9953_port_io_res,
+	.resources		= vsc9953_resources,
+	.num_resources		= ARRAY_SIZE(vsc9953_resources),
+	.resource_names		= vsc9953_resource_names,
 	.regfields		= vsc9953_regfields,
 	.map			= vsc9953_regmap,
 	.ops			= &vsc9953_ops,
@@ -1450,7 +1009,6 @@ static const struct felix_info seville_info_vsc9953 = {
 	.mdio_bus_free		= vsc9953_mdio_bus_free,
 	.phylink_validate	= vsc9953_phylink_validate,
 	.port_modes		= vsc9953_port_modes,
-	.init_regmap		= ocelot_regmap_init,
 };
 
 static int seville_probe(struct platform_device *pdev)
@@ -1525,8 +1083,6 @@ static int seville_remove(struct platform_device *pdev)
 	kfree(felix->ds);
 	kfree(felix);
 
-	platform_set_drvdata(pdev, NULL);
-
 	return 0;
 }
 
diff --git a/drivers/net/dsa/qca/ar9331.c b/drivers/net/dsa/qca/ar9331.c
index 0796b7cf8cae..e7b98b864fa1 100644
--- a/drivers/net/dsa/qca/ar9331.c
+++ b/drivers/net/dsa/qca/ar9331.c
@@ -1099,8 +1099,6 @@ static void ar9331_sw_remove(struct mdio_device *mdiodev)
 	dsa_unregister_switch(&priv->ds);
 
 	reset_control_assert(priv->sw_reset);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void ar9331_sw_shutdown(struct mdio_device *mdiodev)
diff --git a/drivers/net/dsa/qca/qca8k-8xxx.c b/drivers/net/dsa/qca/qca8k-8xxx.c
index c181346388a4..5669c92c93f7 100644
--- a/drivers/net/dsa/qca/qca8k-8xxx.c
+++ b/drivers/net/dsa/qca/qca8k-8xxx.c
@@ -1957,8 +1957,6 @@ qca8k_sw_remove(struct mdio_device *mdiodev)
 		qca8k_port_set_status(priv, i, 0);
 
 	dsa_unregister_switch(priv->ds);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void qca8k_sw_shutdown(struct mdio_device *mdiodev)
diff --git a/drivers/net/dsa/qca/qca8k-common.c b/drivers/net/dsa/qca/qca8k-common.c
index bba95613e218..fb45b598847b 100644
--- a/drivers/net/dsa/qca/qca8k-common.c
+++ b/drivers/net/dsa/qca/qca8k-common.c
@@ -1017,7 +1017,8 @@ int qca8k_port_vlan_del(struct dsa_switch *ds, int port,
 
 static bool qca8k_lag_can_offload(struct dsa_switch *ds,
 				  struct dsa_lag lag,
-				  struct netdev_lag_upper_info *info)
+				  struct netdev_lag_upper_info *info,
+				  struct netlink_ext_ack *extack)
 {
 	struct dsa_port *dp;
 	int members = 0;
@@ -1029,15 +1030,24 @@ static bool qca8k_lag_can_offload(struct dsa_switch *ds,
 		/* Includes the port joining the LAG */
 		members++;
 
-	if (members > QCA8K_NUM_PORTS_FOR_LAG)
+	if (members > QCA8K_NUM_PORTS_FOR_LAG) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot offload more than 4 LAG ports");
 		return false;
+	}
 
-	if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH)
+	if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Can only offload LAG using hash TX type");
 		return false;
+	}
 
 	if (info->hash_type != NETDEV_LAG_HASH_L2 &&
-	    info->hash_type != NETDEV_LAG_HASH_L23)
+	    info->hash_type != NETDEV_LAG_HASH_L23) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Can only offload L2 or L2+L3 TX hash");
 		return false;
+	}
 
 	return true;
 }
@@ -1160,11 +1170,12 @@ static int qca8k_lag_refresh_portmap(struct dsa_switch *ds, int port,
 }
 
 int qca8k_port_lag_join(struct dsa_switch *ds, int port, struct dsa_lag lag,
-			struct netdev_lag_upper_info *info)
+			struct netdev_lag_upper_info *info,
+			struct netlink_ext_ack *extack)
 {
 	int ret;
 
-	if (!qca8k_lag_can_offload(ds, lag, info))
+	if (!qca8k_lag_can_offload(ds, lag, info, extack))
 		return -EOPNOTSUPP;
 
 	ret = qca8k_lag_setup_hash(ds, lag, info);
diff --git a/drivers/net/dsa/qca/qca8k.h b/drivers/net/dsa/qca/qca8k.h
index e36ecc9777f4..0b7a5cb12321 100644
--- a/drivers/net/dsa/qca/qca8k.h
+++ b/drivers/net/dsa/qca/qca8k.h
@@ -512,7 +512,8 @@ int qca8k_port_vlan_del(struct dsa_switch *ds, int port,
 
 /* Common port LAG function */
 int qca8k_port_lag_join(struct dsa_switch *ds, int port, struct dsa_lag lag,
-			struct netdev_lag_upper_info *info);
+			struct netdev_lag_upper_info *info,
+			struct netlink_ext_ack *extack);
 int qca8k_port_lag_leave(struct dsa_switch *ds, int port,
 			 struct dsa_lag lag);
 
diff --git a/drivers/net/dsa/realtek/realtek-mdio.c b/drivers/net/dsa/realtek/realtek-mdio.c
index c58f49d558d2..3e54fac5f902 100644
--- a/drivers/net/dsa/realtek/realtek-mdio.c
+++ b/drivers/net/dsa/realtek/realtek-mdio.c
@@ -245,8 +245,6 @@ static void realtek_mdio_remove(struct mdio_device *mdiodev)
 	/* leave the device reset asserted */
 	if (priv->reset)
 		gpiod_set_value(priv->reset, 1);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void realtek_mdio_shutdown(struct mdio_device *mdiodev)
diff --git a/drivers/net/dsa/realtek/realtek-smi.c b/drivers/net/dsa/realtek/realtek-smi.c
index 45992f79ec8d..1b447d96b9c4 100644
--- a/drivers/net/dsa/realtek/realtek-smi.c
+++ b/drivers/net/dsa/realtek/realtek-smi.c
@@ -522,8 +522,6 @@ static int realtek_smi_remove(struct platform_device *pdev)
 	if (priv->reset)
 		gpiod_set_value(priv->reset, 1);
 
-	platform_set_drvdata(pdev, NULL);
-
 	return 0;
 }
 
diff --git a/drivers/net/dsa/rzn1_a5psw.c b/drivers/net/dsa/rzn1_a5psw.c
index 0744e8162e1d..ed413d555bec 100644
--- a/drivers/net/dsa/rzn1_a5psw.c
+++ b/drivers/net/dsa/rzn1_a5psw.c
@@ -1025,8 +1025,6 @@ static int a5psw_remove(struct platform_device *pdev)
 	clk_disable_unprepare(a5psw->hclk);
 	clk_disable_unprepare(a5psw->clk);
 
-	platform_set_drvdata(pdev, NULL);
-
 	return 0;
 }
 
diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c
index b03d0d0c3dbf..412666111b0c 100644
--- a/drivers/net/dsa/sja1105/sja1105_main.c
+++ b/drivers/net/dsa/sja1105/sja1105_main.c
@@ -3351,8 +3351,6 @@ static void sja1105_remove(struct spi_device *spi)
 		return;
 
 	dsa_unregister_switch(priv->ds);
-
-	spi_set_drvdata(spi, NULL);
 }
 
 static void sja1105_shutdown(struct spi_device *spi)
diff --git a/drivers/net/dsa/vitesse-vsc73xx-platform.c b/drivers/net/dsa/vitesse-vsc73xx-platform.c
index fe4b154a0a57..bd4206e8f9af 100644
--- a/drivers/net/dsa/vitesse-vsc73xx-platform.c
+++ b/drivers/net/dsa/vitesse-vsc73xx-platform.c
@@ -121,8 +121,6 @@ static int vsc73xx_platform_remove(struct platform_device *pdev)
 
 	vsc73xx_remove(&vsc_platform->vsc);
 
-	platform_set_drvdata(pdev, NULL);
-
 	return 0;
 }
 
diff --git a/drivers/net/dsa/vitesse-vsc73xx-spi.c b/drivers/net/dsa/vitesse-vsc73xx-spi.c
index 97a92e6da60d..85b9a0f51dd8 100644
--- a/drivers/net/dsa/vitesse-vsc73xx-spi.c
+++ b/drivers/net/dsa/vitesse-vsc73xx-spi.c
@@ -167,8 +167,6 @@ static void vsc73xx_spi_remove(struct spi_device *spi)
 		return;
 
 	vsc73xx_remove(&vsc_spi->vsc);
-
-	spi_set_drvdata(spi, NULL);
 }
 
 static void vsc73xx_spi_shutdown(struct spi_device *spi)
diff --git a/drivers/net/dsa/xrs700x/xrs700x_i2c.c b/drivers/net/dsa/xrs700x/xrs700x_i2c.c
index 6deae388a0d6..54065cdedd35 100644
--- a/drivers/net/dsa/xrs700x/xrs700x_i2c.c
+++ b/drivers/net/dsa/xrs700x/xrs700x_i2c.c
@@ -105,18 +105,14 @@ static int xrs700x_i2c_probe(struct i2c_client *i2c,
 	return 0;
 }
 
-static int xrs700x_i2c_remove(struct i2c_client *i2c)
+static void xrs700x_i2c_remove(struct i2c_client *i2c)
 {
 	struct xrs700x *priv = i2c_get_clientdata(i2c);
 
 	if (!priv)
-		return 0;
+		return;
 
 	xrs700x_switch_remove(priv);
-
-	i2c_set_clientdata(i2c, NULL);
-
-	return 0;
 }
 
 static void xrs700x_i2c_shutdown(struct i2c_client *i2c)
diff --git a/drivers/net/dsa/xrs700x/xrs700x_mdio.c b/drivers/net/dsa/xrs700x/xrs700x_mdio.c
index 127a677d1f39..5f7d344b5d73 100644
--- a/drivers/net/dsa/xrs700x/xrs700x_mdio.c
+++ b/drivers/net/dsa/xrs700x/xrs700x_mdio.c
@@ -140,8 +140,6 @@ static void xrs700x_mdio_remove(struct mdio_device *mdiodev)
 		return;
 
 	xrs700x_switch_remove(priv);
-
-	dev_set_drvdata(&mdiodev->dev, NULL);
 }
 
 static void xrs700x_mdio_shutdown(struct mdio_device *mdiodev)
diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c
index f82ad7419508..aa0fc00faecb 100644
--- a/drivers/net/dummy.c
+++ b/drivers/net/dummy.c
@@ -102,7 +102,7 @@ static const struct net_device_ops dummy_netdev_ops = {
 static void dummy_get_drvinfo(struct net_device *dev,
 			      struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 }
 
 static const struct ethtool_ops dummy_ethtool_ops = {
diff --git a/drivers/net/ethernet/3com/3c509.c b/drivers/net/ethernet/3com/3c509.c
index 846fa3af4504..fb68339e1511 100644
--- a/drivers/net/ethernet/3com/3c509.c
+++ b/drivers/net/ethernet/3com/3c509.c
@@ -1135,7 +1135,7 @@ el3_netdev_set_ecmd(struct net_device *dev,
 
 static void el3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 }
 
 static int el3_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/3com/3c515.c b/drivers/net/ethernet/3com/3c515.c
index 1d124b0f65e7..d2f4358cc550 100644
--- a/drivers/net/ethernet/3com/3c515.c
+++ b/drivers/net/ethernet/3com/3c515.c
@@ -1527,7 +1527,7 @@ static void set_rx_mode(struct net_device *dev)
 static void netdev_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 	snprintf(info->bus_info, sizeof(info->bus_info), "ISA 0x%lx",
 		 dev->base_addr);
 }
diff --git a/drivers/net/ethernet/3com/3c589_cs.c b/drivers/net/ethernet/3com/3c589_cs.c
index 4673bc1604e7..82f94b1635bf 100644
--- a/drivers/net/ethernet/3com/3c589_cs.c
+++ b/drivers/net/ethernet/3com/3c589_cs.c
@@ -480,7 +480,7 @@ static void tc589_reset(struct net_device *dev)
 static void netdev_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 	snprintf(info->bus_info, sizeof(info->bus_info),
 		"PCMCIA 0x%lx", dev->base_addr);
 }
diff --git a/drivers/net/ethernet/3com/3c59x.c b/drivers/net/ethernet/3com/3c59x.c
index ccf07667aa5e..082388bb6169 100644
--- a/drivers/net/ethernet/3com/3c59x.c
+++ b/drivers/net/ethernet/3com/3c59x.c
@@ -2959,13 +2959,13 @@ static void vortex_get_drvinfo(struct net_device *dev,
 {
 	struct vortex_private *vp = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 	if (VORTEX_PCI(vp)) {
-		strlcpy(info->bus_info, pci_name(VORTEX_PCI(vp)),
+		strscpy(info->bus_info, pci_name(VORTEX_PCI(vp)),
 			sizeof(info->bus_info));
 	} else {
 		if (VORTEX_EISA(vp))
-			strlcpy(info->bus_info, dev_name(vp->gendev),
+			strscpy(info->bus_info, dev_name(vp->gendev),
 				sizeof(info->bus_info));
 		else
 			snprintf(info->bus_info, sizeof(info->bus_info),
diff --git a/drivers/net/ethernet/3com/typhoon.c b/drivers/net/ethernet/3com/typhoon.c
index cad4f354cc76..aaaff3ba43ef 100644
--- a/drivers/net/ethernet/3com/typhoon.c
+++ b/drivers/net/ethernet/3com/typhoon.c
@@ -969,12 +969,12 @@ typhoon_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 
 	smp_rmb();
 	if (tp->card_state == Sleeping) {
-		strlcpy(info->fw_version, "Sleep image",
+		strscpy(info->fw_version, "Sleep image",
 			sizeof(info->fw_version));
 	} else {
 		INIT_COMMAND_WITH_RESPONSE(&xp_cmd, TYPHOON_CMD_READ_VERSIONS);
 		if (typhoon_issue_command(tp, 1, &xp_cmd, 3, xp_resp) < 0) {
-			strlcpy(info->fw_version, "Unknown runtime",
+			strscpy(info->fw_version, "Unknown runtime",
 				sizeof(info->fw_version));
 		} else {
 			u32 sleep_ver = le32_to_cpu(xp_resp[0].parm2);
@@ -984,8 +984,8 @@ typhoon_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 		}
 	}
 
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(pci_dev), sizeof(info->bus_info));
 }
 
 static int
diff --git a/drivers/net/ethernet/8390/ax88796.c b/drivers/net/ethernet/8390/ax88796.c
index 1f8acbba5b6b..af603256b724 100644
--- a/drivers/net/ethernet/8390/ax88796.c
+++ b/drivers/net/ethernet/8390/ax88796.c
@@ -579,9 +579,9 @@ static void ax_get_drvinfo(struct net_device *dev,
 {
 	struct platform_device *pdev = to_platform_device(dev->dev.parent);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pdev->name, sizeof(info->bus_info));
 }
 
 static u32 ax_get_msglevel(struct net_device *dev)
diff --git a/drivers/net/ethernet/8390/etherh.c b/drivers/net/ethernet/8390/etherh.c
index e7b879123bb1..05d39ecb97ff 100644
--- a/drivers/net/ethernet/8390/etherh.c
+++ b/drivers/net/ethernet/8390/etherh.c
@@ -555,9 +555,9 @@ static int __init etherm_addr(char *addr)
 
 static void etherh_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, dev_name(dev->dev.parent),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, dev_name(dev->dev.parent),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/8390/mcf8390.c b/drivers/net/ethernet/8390/mcf8390.c
index 21047ae1bc3d..8a7918d33419 100644
--- a/drivers/net/ethernet/8390/mcf8390.c
+++ b/drivers/net/ethernet/8390/mcf8390.c
@@ -450,8 +450,7 @@ static int mcf8390_remove(struct platform_device *pdev)
 
 	unregister_netdev(dev);
 	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (mem)
-		release_mem_region(mem->start, resource_size(mem));
+	release_mem_region(mem->start, resource_size(mem));
 	free_netdev(dev);
 	return 0;
 }
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
index 9a55c1d5a0a1..1917da784191 100644
--- a/drivers/net/ethernet/Kconfig
+++ b/drivers/net/ethernet/Kconfig
@@ -121,6 +121,7 @@ config LANTIQ_XRX200
 	  Support for the PMAC of the Gigabit switch (GSWIP) inside the
 	  Lantiq / Intel VRX200 VDSL SoC
 
+source "drivers/net/ethernet/adi/Kconfig"
 source "drivers/net/ethernet/litex/Kconfig"
 source "drivers/net/ethernet/marvell/Kconfig"
 source "drivers/net/ethernet/mediatek/Kconfig"
diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile
index c06e75ed4231..0d872d4efcd1 100644
--- a/drivers/net/ethernet/Makefile
+++ b/drivers/net/ethernet/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_NET_VENDOR_8390) += 8390/
 obj-$(CONFIG_NET_VENDOR_ACTIONS) += actions/
 obj-$(CONFIG_NET_VENDOR_ADAPTEC) += adaptec/
 obj-$(CONFIG_GRETH) += aeroflex/
+obj-$(CONFIG_NET_VENDOR_ADI) += adi/
 obj-$(CONFIG_NET_VENDOR_AGERE) += agere/
 obj-$(CONFIG_NET_VENDOR_ALACRITECH) += alacritech/
 obj-$(CONFIG_NET_VENDOR_ALLWINNER) += allwinner/
diff --git a/drivers/net/ethernet/actions/owl-emac.c b/drivers/net/ethernet/actions/owl-emac.c
index 1cfdd01b4c2e..cd4d71b83c33 100644
--- a/drivers/net/ethernet/actions/owl-emac.c
+++ b/drivers/net/ethernet/actions/owl-emac.c
@@ -1576,7 +1576,7 @@ static int owl_emac_probe(struct platform_device *pdev)
 	netdev->watchdog_timeo = OWL_EMAC_TX_TIMEOUT;
 	netdev->netdev_ops = &owl_emac_netdev_ops;
 	netdev->ethtool_ops = &owl_emac_ethtool_ops;
-	netif_napi_add(netdev, &priv->napi, owl_emac_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, &priv->napi, owl_emac_poll);
 
 	ret = devm_register_netdev(dev, netdev);
 	if (ret) {
diff --git a/drivers/net/ethernet/adaptec/starfire.c b/drivers/net/ethernet/adaptec/starfire.c
index 8f0a6b9c518e..857361c74f5d 100644
--- a/drivers/net/ethernet/adaptec/starfire.c
+++ b/drivers/net/ethernet/adaptec/starfire.c
@@ -1844,8 +1844,8 @@ static int check_if_running(struct net_device *dev)
 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
 	struct netdev_private *np = netdev_priv(dev);
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
 }
 
 static int get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/adi/Kconfig b/drivers/net/ethernet/adi/Kconfig
new file mode 100644
index 000000000000..da3bdd302502
--- /dev/null
+++ b/drivers/net/ethernet/adi/Kconfig
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+#
+# Analog Devices device configuration
+#
+
+config NET_VENDOR_ADI
+	bool "Analog Devices devices"
+	default y
+	depends on SPI
+	help
+	  If you have a network (Ethernet) card belonging to this class, say Y.
+
+	  Note that the answer to this question doesn't directly affect the
+	  kernel: saying N will just cause the configurator to skip all
+	  the questions about ADI devices. If you say Y, you will be asked
+	  for your specific card in the following questions.
+
+if NET_VENDOR_ADI
+
+config ADIN1110
+	tristate "Analog Devices ADIN1110 MAC-PHY"
+	depends on SPI && NET_SWITCHDEV
+	select CRC8
+	help
+	  Say yes here to build support for Analog Devices ADIN1110
+	  Low Power 10BASE-T1L Ethernet MAC-PHY.
+
+endif # NET_VENDOR_ADI
diff --git a/drivers/net/ethernet/adi/Makefile b/drivers/net/ethernet/adi/Makefile
new file mode 100644
index 000000000000..d0383d94303c
--- /dev/null
+++ b/drivers/net/ethernet/adi/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+#
+# Makefile for the Analog Devices network device drivers.
+#
+
+obj-$(CONFIG_ADIN1110) += adin1110.o
diff --git a/drivers/net/ethernet/adi/adin1110.c b/drivers/net/ethernet/adi/adin1110.c
new file mode 100644
index 000000000000..aaee7c4248e6
--- /dev/null
+++ b/drivers/net/ethernet/adi/adin1110.c
@@ -0,0 +1,1697 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+/* ADIN1110 Low Power 10BASE-T1L Ethernet MAC-PHY
+ * ADIN2111 2-Port Ethernet Switch with Integrated 10BASE-T1L PHY
+ *
+ * Copyright 2021 Analog Devices Inc.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/cache.h>
+#include <linux/crc8.h>
+#include <linux/etherdevice.h>
+#include <linux/ethtool.h>
+#include <linux/if_bridge.h>
+#include <linux/interrupt.h>
+#include <linux/iopoll.h>
+#include <linux/gpio.h>
+#include <linux/kernel.h>
+#include <linux/mii.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/regulator/consumer.h>
+#include <linux/phy.h>
+#include <linux/property.h>
+#include <linux/spi/spi.h>
+
+#include <net/switchdev.h>
+
+#include <asm/unaligned.h>
+
+#define ADIN1110_PHY_ID				0x1
+
+#define ADIN1110_RESET				0x03
+#define   ADIN1110_SWRESET			BIT(0)
+
+#define ADIN1110_CONFIG1			0x04
+#define   ADIN1110_CONFIG1_SYNC			BIT(15)
+
+#define ADIN1110_CONFIG2			0x06
+#define   ADIN2111_P2_FWD_UNK2HOST		BIT(12)
+#define   ADIN2111_PORT_CUT_THRU_EN		BIT(11)
+#define   ADIN1110_CRC_APPEND			BIT(5)
+#define   ADIN1110_FWD_UNK2HOST			BIT(2)
+
+#define ADIN1110_STATUS0			0x08
+
+#define ADIN1110_STATUS1			0x09
+#define   ADIN2111_P2_RX_RDY			BIT(17)
+#define   ADIN1110_SPI_ERR			BIT(10)
+#define   ADIN1110_RX_RDY			BIT(4)
+
+#define ADIN1110_IMASK1				0x0D
+#define   ADIN2111_RX_RDY_IRQ			BIT(17)
+#define   ADIN1110_SPI_ERR_IRQ			BIT(10)
+#define   ADIN1110_RX_RDY_IRQ			BIT(4)
+#define   ADIN1110_TX_RDY_IRQ			BIT(3)
+
+#define ADIN1110_MDIOACC			0x20
+#define   ADIN1110_MDIO_TRDONE			BIT(31)
+#define   ADIN1110_MDIO_ST			GENMASK(29, 28)
+#define   ADIN1110_MDIO_OP			GENMASK(27, 26)
+#define   ADIN1110_MDIO_PRTAD			GENMASK(25, 21)
+#define   ADIN1110_MDIO_DEVAD			GENMASK(20, 16)
+#define   ADIN1110_MDIO_DATA			GENMASK(15, 0)
+
+#define ADIN1110_TX_FSIZE			0x30
+#define ADIN1110_TX				0x31
+#define ADIN1110_TX_SPACE			0x32
+
+#define ADIN1110_MAC_ADDR_FILTER_UPR		0x50
+#define   ADIN2111_MAC_ADDR_APPLY2PORT2		BIT(31)
+#define   ADIN1110_MAC_ADDR_APPLY2PORT		BIT(30)
+#define   ADIN2111_MAC_ADDR_TO_OTHER_PORT	BIT(17)
+#define   ADIN1110_MAC_ADDR_TO_HOST		BIT(16)
+
+#define ADIN1110_MAC_ADDR_FILTER_LWR		0x51
+
+#define ADIN1110_MAC_ADDR_MASK_UPR		0x70
+#define ADIN1110_MAC_ADDR_MASK_LWR		0x71
+
+#define ADIN1110_RX_FSIZE			0x90
+#define ADIN1110_RX				0x91
+
+#define ADIN2111_RX_P2_FSIZE			0xC0
+#define ADIN2111_RX_P2				0xC1
+
+#define ADIN1110_CLEAR_STATUS0			0xFFF
+
+/* MDIO_OP codes */
+#define ADIN1110_MDIO_OP_WR			0x1
+#define ADIN1110_MDIO_OP_RD			0x3
+
+#define ADIN1110_CD				BIT(7)
+#define ADIN1110_WRITE				BIT(5)
+
+#define ADIN1110_MAX_BUFF			2048
+#define ADIN1110_MAX_FRAMES_READ		64
+#define ADIN1110_WR_HEADER_LEN			2
+#define ADIN1110_FRAME_HEADER_LEN		2
+#define ADIN1110_INTERNAL_SIZE_HEADER_LEN	2
+#define ADIN1110_RD_HEADER_LEN			3
+#define ADIN1110_REG_LEN			4
+#define ADIN1110_FEC_LEN			4
+
+#define ADIN1110_PHY_ID_VAL			0x0283BC91
+#define ADIN2111_PHY_ID_VAL			0x0283BCA1
+
+#define ADIN_MAC_MAX_PORTS			2
+#define ADIN_MAC_MAX_ADDR_SLOTS			16
+
+#define ADIN_MAC_MULTICAST_ADDR_SLOT		0
+#define ADIN_MAC_BROADCAST_ADDR_SLOT		1
+#define ADIN_MAC_P1_ADDR_SLOT			2
+#define ADIN_MAC_P2_ADDR_SLOT			3
+#define ADIN_MAC_FDB_ADDR_SLOT			4
+
+DECLARE_CRC8_TABLE(adin1110_crc_table);
+
+enum adin1110_chips_id {
+	ADIN1110_MAC = 0,
+	ADIN2111_MAC,
+};
+
+struct adin1110_cfg {
+	enum adin1110_chips_id	id;
+	char			name[MDIO_NAME_SIZE];
+	u32			phy_ids[PHY_MAX_ADDR];
+	u32			ports_nr;
+	u32			phy_id_val;
+};
+
+struct adin1110_port_priv {
+	struct adin1110_priv		*priv;
+	struct net_device		*netdev;
+	struct net_device		*bridge;
+	struct phy_device		*phydev;
+	struct work_struct		tx_work;
+	u64				rx_packets;
+	u64				tx_packets;
+	u64				rx_bytes;
+	u64				tx_bytes;
+	struct work_struct		rx_mode_work;
+	u32				flags;
+	struct sk_buff_head		txq;
+	u32				nr;
+	u32				state;
+	struct adin1110_cfg		*cfg;
+};
+
+struct adin1110_priv {
+	struct mutex			lock; /* protect spi */
+	spinlock_t			state_lock; /* protect RX mode */
+	struct mii_bus			*mii_bus;
+	struct spi_device		*spidev;
+	bool				append_crc;
+	struct adin1110_cfg		*cfg;
+	u32				tx_space;
+	u32				irq_mask;
+	bool				forwarding;
+	int				irq;
+	struct adin1110_port_priv	*ports[ADIN_MAC_MAX_PORTS];
+	char				mii_bus_name[MII_BUS_ID_SIZE];
+	u8				data[ADIN1110_MAX_BUFF] ____cacheline_aligned;
+};
+
+struct adin1110_switchdev_event_work {
+	struct work_struct work;
+	struct switchdev_notifier_fdb_info fdb_info;
+	struct adin1110_port_priv *port_priv;
+	unsigned long event;
+};
+
+static struct adin1110_cfg adin1110_cfgs[] = {
+	{
+		.id = ADIN1110_MAC,
+		.name = "adin1110",
+		.phy_ids = {1},
+		.ports_nr = 1,
+		.phy_id_val = ADIN1110_PHY_ID_VAL,
+	},
+	{
+		.id = ADIN2111_MAC,
+		.name = "adin2111",
+		.phy_ids = {1, 2},
+		.ports_nr = 2,
+		.phy_id_val = ADIN2111_PHY_ID_VAL,
+	},
+};
+
+static u8 adin1110_crc_data(u8 *data, u32 len)
+{
+	return crc8(adin1110_crc_table, data, len, 0);
+}
+
+static int adin1110_read_reg(struct adin1110_priv *priv, u16 reg, u32 *val)
+{
+	u32 header_len = ADIN1110_RD_HEADER_LEN;
+	u32 read_len = ADIN1110_REG_LEN;
+	struct spi_transfer t[2] = {0};
+	int ret;
+
+	priv->data[0] = ADIN1110_CD | FIELD_GET(GENMASK(12, 8), reg);
+	priv->data[1] = FIELD_GET(GENMASK(7, 0), reg);
+	priv->data[2] = 0x00;
+
+	if (priv->append_crc) {
+		priv->data[2] = adin1110_crc_data(&priv->data[0], 2);
+		priv->data[3] = 0x00;
+		header_len++;
+	}
+
+	t[0].tx_buf = &priv->data[0];
+	t[0].len = header_len;
+
+	if (priv->append_crc)
+		read_len++;
+
+	memset(&priv->data[header_len], 0, read_len);
+	t[1].rx_buf = &priv->data[header_len];
+	t[1].len = read_len;
+
+	ret = spi_sync_transfer(priv->spidev, t, 2);
+	if (ret)
+		return ret;
+
+	if (priv->append_crc) {
+		u8 recv_crc;
+		u8 crc;
+
+		crc = adin1110_crc_data(&priv->data[header_len],
+					ADIN1110_REG_LEN);
+		recv_crc = priv->data[header_len + ADIN1110_REG_LEN];
+
+		if (crc != recv_crc) {
+			dev_err_ratelimited(&priv->spidev->dev, "CRC error.");
+			return -EBADMSG;
+		}
+	}
+
+	*val = get_unaligned_be32(&priv->data[header_len]);
+
+	return ret;
+}
+
+static int adin1110_write_reg(struct adin1110_priv *priv, u16 reg, u32 val)
+{
+	u32 header_len = ADIN1110_WR_HEADER_LEN;
+	u32 write_len = ADIN1110_REG_LEN;
+
+	priv->data[0] = ADIN1110_CD | ADIN1110_WRITE | FIELD_GET(GENMASK(12, 8), reg);
+	priv->data[1] = FIELD_GET(GENMASK(7, 0), reg);
+
+	if (priv->append_crc) {
+		priv->data[2] = adin1110_crc_data(&priv->data[0], header_len);
+		header_len++;
+	}
+
+	put_unaligned_be32(val, &priv->data[header_len]);
+	if (priv->append_crc) {
+		priv->data[header_len + write_len] = adin1110_crc_data(&priv->data[header_len],
+								       write_len);
+		write_len++;
+	}
+
+	return spi_write(priv->spidev, &priv->data[0], header_len + write_len);
+}
+
+static int adin1110_set_bits(struct adin1110_priv *priv, u16 reg,
+			     unsigned long mask, unsigned long val)
+{
+	u32 write_val;
+	int ret;
+
+	ret = adin1110_read_reg(priv, reg, &write_val);
+	if (ret < 0)
+		return ret;
+
+	set_mask_bits(&write_val, mask, val);
+
+	return adin1110_write_reg(priv, reg, write_val);
+}
+
+static int adin1110_round_len(int len)
+{
+	/* can read/write only mutiples of 4 bytes of payload */
+	len = ALIGN(len, 4);
+
+	/* NOTE: ADIN1110_WR_HEADER_LEN should be used for write ops. */
+	if (len + ADIN1110_RD_HEADER_LEN > ADIN1110_MAX_BUFF)
+		return -EINVAL;
+
+	return len;
+}
+
+static int adin1110_read_fifo(struct adin1110_port_priv *port_priv)
+{
+	struct adin1110_priv *priv = port_priv->priv;
+	u32 header_len = ADIN1110_RD_HEADER_LEN;
+	struct spi_transfer t[2] = {0};
+	u32 frame_size_no_fcs;
+	struct sk_buff *rxb;
+	u32 frame_size;
+	int round_len;
+	u16 reg;
+	int ret;
+
+	if (!port_priv->nr) {
+		reg = ADIN1110_RX;
+		ret = adin1110_read_reg(priv, ADIN1110_RX_FSIZE, &frame_size);
+	} else {
+		reg = ADIN2111_RX_P2;
+		ret = adin1110_read_reg(priv, ADIN2111_RX_P2_FSIZE,
+					&frame_size);
+	}
+
+	if (ret < 0)
+		return ret;
+
+	/* The read frame size includes the extra 2 bytes
+	 * from the  ADIN1110 frame header.
+	 */
+	if (frame_size < ADIN1110_FRAME_HEADER_LEN + ADIN1110_FEC_LEN)
+		return ret;
+
+	round_len = adin1110_round_len(frame_size);
+	if (round_len < 0)
+		return ret;
+
+	frame_size_no_fcs = frame_size - ADIN1110_FRAME_HEADER_LEN - ADIN1110_FEC_LEN;
+
+	rxb = netdev_alloc_skb(port_priv->netdev, round_len);
+	if (!rxb)
+		return -ENOMEM;
+
+	memset(priv->data, 0, round_len + ADIN1110_RD_HEADER_LEN);
+
+	priv->data[0] = ADIN1110_CD | FIELD_GET(GENMASK(12, 8), reg);
+	priv->data[1] = FIELD_GET(GENMASK(7, 0), reg);
+
+	if (priv->append_crc) {
+		priv->data[2] = adin1110_crc_data(&priv->data[0], 2);
+		header_len++;
+	}
+
+	skb_put(rxb, frame_size_no_fcs + ADIN1110_FRAME_HEADER_LEN);
+
+	t[0].tx_buf = &priv->data[0];
+	t[0].len = header_len;
+
+	t[1].rx_buf = &rxb->data[0];
+	t[1].len = round_len;
+
+	ret = spi_sync_transfer(priv->spidev, t, 2);
+	if (ret) {
+		kfree_skb(rxb);
+		return ret;
+	}
+
+	skb_pull(rxb, ADIN1110_FRAME_HEADER_LEN);
+	rxb->protocol = eth_type_trans(rxb, port_priv->netdev);
+
+	if ((port_priv->flags & IFF_ALLMULTI && rxb->pkt_type == PACKET_MULTICAST) ||
+	    (port_priv->flags & IFF_BROADCAST && rxb->pkt_type == PACKET_BROADCAST))
+		rxb->offload_fwd_mark = 1;
+
+	netif_rx(rxb);
+
+	port_priv->rx_bytes += frame_size - ADIN1110_FRAME_HEADER_LEN;
+	port_priv->rx_packets++;
+
+	return 0;
+}
+
+static int adin1110_write_fifo(struct adin1110_port_priv *port_priv,
+			       struct sk_buff *txb)
+{
+	struct adin1110_priv *priv = port_priv->priv;
+	u32 header_len = ADIN1110_WR_HEADER_LEN;
+	__be16 frame_header;
+	int padding = 0;
+	int padded_len;
+	int round_len;
+	int ret;
+
+	/* Pad frame to 64 byte length,
+	 * MAC nor PHY will otherwise add the
+	 * required padding.
+	 * The FEC will be added by the MAC internally.
+	 */
+	if (txb->len + ADIN1110_FEC_LEN < 64)
+		padding = 64 - (txb->len + ADIN1110_FEC_LEN);
+
+	padded_len = txb->len + padding + ADIN1110_FRAME_HEADER_LEN;
+
+	round_len = adin1110_round_len(padded_len);
+	if (round_len < 0)
+		return round_len;
+
+	ret = adin1110_write_reg(priv, ADIN1110_TX_FSIZE, padded_len);
+	if (ret < 0)
+		return ret;
+
+	memset(priv->data, 0, round_len + ADIN1110_WR_HEADER_LEN);
+
+	priv->data[0] = ADIN1110_CD | ADIN1110_WRITE;
+	priv->data[0] |= FIELD_GET(GENMASK(12, 8), ADIN1110_TX);
+	priv->data[1] = FIELD_GET(GENMASK(7, 0), ADIN1110_TX);
+	if (priv->append_crc) {
+		priv->data[2] = adin1110_crc_data(&priv->data[0], 2);
+		header_len++;
+	}
+
+	/* mention the port on which to send the frame in the frame header */
+	frame_header = cpu_to_be16(port_priv->nr);
+	memcpy(&priv->data[header_len], &frame_header,
+	       ADIN1110_FRAME_HEADER_LEN);
+
+	memcpy(&priv->data[header_len + ADIN1110_FRAME_HEADER_LEN],
+	       txb->data, txb->len);
+
+	ret = spi_write(priv->spidev, &priv->data[0], round_len + header_len);
+	if (ret < 0)
+		return ret;
+
+	port_priv->tx_bytes += txb->len;
+	port_priv->tx_packets++;
+
+	return 0;
+}
+
+static int adin1110_read_mdio_acc(struct adin1110_priv *priv)
+{
+	u32 val;
+	int ret;
+
+	mutex_lock(&priv->lock);
+	ret = adin1110_read_reg(priv, ADIN1110_MDIOACC, &val);
+	mutex_unlock(&priv->lock);
+	if (ret < 0)
+		return 0;
+
+	return val;
+}
+
+static int adin1110_mdio_read(struct mii_bus *bus, int phy_id, int reg)
+{
+	struct adin1110_priv *priv = bus->priv;
+	u32 val = 0;
+	int ret;
+
+	if (mdio_phy_id_is_c45(phy_id))
+		return -EOPNOTSUPP;
+
+	val |= FIELD_PREP(ADIN1110_MDIO_OP, ADIN1110_MDIO_OP_RD);
+	val |= FIELD_PREP(ADIN1110_MDIO_ST, 0x1);
+	val |= FIELD_PREP(ADIN1110_MDIO_PRTAD, phy_id);
+	val |= FIELD_PREP(ADIN1110_MDIO_DEVAD, reg);
+
+	/* write the clause 22 read command to the chip */
+	mutex_lock(&priv->lock);
+	ret = adin1110_write_reg(priv, ADIN1110_MDIOACC, val);
+	mutex_unlock(&priv->lock);
+	if (ret < 0)
+		return ret;
+
+	/* ADIN1110_MDIO_TRDONE BIT of the ADIN1110_MDIOACC
+	 * register is set when the read is done.
+	 * After the transaction is done, ADIN1110_MDIO_DATA
+	 * bitfield of ADIN1110_MDIOACC register will contain
+	 * the requested register value.
+	 */
+	ret = readx_poll_timeout(adin1110_read_mdio_acc, priv, val,
+				 (val & ADIN1110_MDIO_TRDONE), 10000, 30000);
+	if (ret < 0)
+		return ret;
+
+	return (val & ADIN1110_MDIO_DATA);
+}
+
+static int adin1110_mdio_write(struct mii_bus *bus, int phy_id,
+			       int reg, u16 reg_val)
+{
+	struct adin1110_priv *priv = bus->priv;
+	u32 val = 0;
+	int ret;
+
+	if (mdio_phy_id_is_c45(phy_id))
+		return -EOPNOTSUPP;
+
+	val |= FIELD_PREP(ADIN1110_MDIO_OP, ADIN1110_MDIO_OP_WR);
+	val |= FIELD_PREP(ADIN1110_MDIO_ST, 0x1);
+	val |= FIELD_PREP(ADIN1110_MDIO_PRTAD, phy_id);
+	val |= FIELD_PREP(ADIN1110_MDIO_DEVAD, reg);
+	val |= FIELD_PREP(ADIN1110_MDIO_DATA, reg_val);
+
+	/* write the clause 22 write command to the chip */
+	mutex_lock(&priv->lock);
+	ret = adin1110_write_reg(priv, ADIN1110_MDIOACC, val);
+	mutex_unlock(&priv->lock);
+	if (ret < 0)
+		return ret;
+
+	return readx_poll_timeout(adin1110_read_mdio_acc, priv, val,
+				  (val & ADIN1110_MDIO_TRDONE), 10000, 30000);
+}
+
+/* ADIN1110 MAC-PHY contains an ADIN1100 PHY.
+ * ADIN2111 MAC-PHY contains two ADIN1100 PHYs.
+ * By registering a new MDIO bus we allow the PAL to discover
+ * the encapsulated PHY and probe the ADIN1100 driver.
+ */
+static int adin1110_register_mdiobus(struct adin1110_priv *priv,
+				     struct device *dev)
+{
+	struct mii_bus *mii_bus;
+	int ret;
+
+	mii_bus = devm_mdiobus_alloc(dev);
+	if (!mii_bus)
+		return -ENOMEM;
+
+	snprintf(priv->mii_bus_name, MII_BUS_ID_SIZE, "%s-%u",
+		 priv->cfg->name, priv->spidev->chip_select);
+
+	mii_bus->name = priv->mii_bus_name;
+	mii_bus->read = adin1110_mdio_read;
+	mii_bus->write = adin1110_mdio_write;
+	mii_bus->priv = priv;
+	mii_bus->parent = dev;
+	mii_bus->phy_mask = ~((u32)GENMASK(2, 0));
+	mii_bus->probe_capabilities = MDIOBUS_C22;
+	snprintf(mii_bus->id, MII_BUS_ID_SIZE, "%s", dev_name(dev));
+
+	ret = devm_mdiobus_register(dev, mii_bus);
+	if (ret)
+		return ret;
+
+	priv->mii_bus = mii_bus;
+
+	return 0;
+}
+
+static bool adin1110_port_rx_ready(struct adin1110_port_priv *port_priv,
+				   u32 status)
+{
+	if (!netif_oper_up(port_priv->netdev))
+		return false;
+
+	if (!port_priv->nr)
+		return !!(status & ADIN1110_RX_RDY);
+	else
+		return !!(status & ADIN2111_P2_RX_RDY);
+}
+
+static void adin1110_read_frames(struct adin1110_port_priv *port_priv,
+				 unsigned int budget)
+{
+	struct adin1110_priv *priv = port_priv->priv;
+	u32 status1;
+	int ret;
+
+	while (budget) {
+		ret = adin1110_read_reg(priv, ADIN1110_STATUS1, &status1);
+		if (ret < 0)
+			return;
+
+		if (!adin1110_port_rx_ready(port_priv, status1))
+			break;
+
+		ret = adin1110_read_fifo(port_priv);
+		if (ret < 0)
+			return;
+
+		budget--;
+	}
+}
+
+static void adin1110_wake_queues(struct adin1110_priv *priv)
+{
+	int i;
+
+	for (i = 0; i < priv->cfg->ports_nr; i++)
+		netif_wake_queue(priv->ports[i]->netdev);
+}
+
+static irqreturn_t adin1110_irq(int irq, void *p)
+{
+	struct adin1110_priv *priv = p;
+	u32 status1;
+	u32 val;
+	int ret;
+	int i;
+
+	mutex_lock(&priv->lock);
+
+	ret = adin1110_read_reg(priv, ADIN1110_STATUS1, &status1);
+	if (ret < 0)
+		goto out;
+
+	if (priv->append_crc && (status1 & ADIN1110_SPI_ERR))
+		dev_warn_ratelimited(&priv->spidev->dev,
+				     "SPI CRC error on write.\n");
+
+	ret = adin1110_read_reg(priv, ADIN1110_TX_SPACE, &val);
+	if (ret < 0)
+		goto out;
+
+	/* TX FIFO space is expressed in half-words */
+	priv->tx_space = 2 * val;
+
+	for (i = 0; i < priv->cfg->ports_nr; i++) {
+		if (adin1110_port_rx_ready(priv->ports[i], status1))
+			adin1110_read_frames(priv->ports[i],
+					     ADIN1110_MAX_FRAMES_READ);
+	}
+
+	/* clear IRQ sources */
+	adin1110_write_reg(priv, ADIN1110_STATUS0, ADIN1110_CLEAR_STATUS0);
+	adin1110_write_reg(priv, ADIN1110_STATUS1, priv->irq_mask);
+
+out:
+	mutex_unlock(&priv->lock);
+
+	if (priv->tx_space > 0 && ret >= 0)
+		adin1110_wake_queues(priv);
+
+	return IRQ_HANDLED;
+}
+
+/* ADIN1110 can filter up to 16 MAC addresses, mac_nr here is the slot used */
+static int adin1110_write_mac_address(struct adin1110_port_priv *port_priv,
+				      int mac_nr, const u8 *addr,
+				      u8 *mask, u32 port_rules)
+{
+	struct adin1110_priv *priv = port_priv->priv;
+	u32 offset = mac_nr * 2;
+	u32 port_rules_mask;
+	int ret;
+	u32 val;
+
+	if (!port_priv->nr)
+		port_rules_mask = ADIN1110_MAC_ADDR_APPLY2PORT;
+	else
+		port_rules_mask = ADIN2111_MAC_ADDR_APPLY2PORT2;
+
+	if (port_rules & port_rules_mask)
+		port_rules_mask |= ADIN1110_MAC_ADDR_TO_HOST | ADIN2111_MAC_ADDR_TO_OTHER_PORT;
+
+	port_rules_mask |= GENMASK(15, 0);
+	val = port_rules | get_unaligned_be16(&addr[0]);
+	ret = adin1110_set_bits(priv, ADIN1110_MAC_ADDR_FILTER_UPR + offset,
+				port_rules_mask, val);
+	if (ret < 0)
+		return ret;
+
+	val = get_unaligned_be32(&addr[2]);
+	ret =  adin1110_write_reg(priv,
+				  ADIN1110_MAC_ADDR_FILTER_LWR + offset, val);
+	if (ret < 0)
+		return ret;
+
+	/* Only the first two MAC address slots support masking. */
+	if (mac_nr < ADIN_MAC_P1_ADDR_SLOT) {
+		val = get_unaligned_be16(&mask[0]);
+		ret = adin1110_write_reg(priv,
+					 ADIN1110_MAC_ADDR_MASK_UPR + offset,
+					 val);
+		if (ret < 0)
+			return ret;
+
+		val = get_unaligned_be32(&mask[2]);
+		return adin1110_write_reg(priv,
+					  ADIN1110_MAC_ADDR_MASK_LWR + offset,
+					  val);
+	}
+
+	return 0;
+}
+
+static int adin1110_clear_mac_address(struct adin1110_priv *priv, int mac_nr)
+{
+	u32 offset = mac_nr * 2;
+	int ret;
+
+	ret = adin1110_write_reg(priv, ADIN1110_MAC_ADDR_FILTER_UPR + offset, 0);
+	if (ret < 0)
+		return ret;
+
+	ret =  adin1110_write_reg(priv, ADIN1110_MAC_ADDR_FILTER_LWR + offset, 0);
+	if (ret < 0)
+		return ret;
+
+	/* only the first two MAC address slots are maskable */
+	if (mac_nr <= 1) {
+		ret = adin1110_write_reg(priv, ADIN1110_MAC_ADDR_MASK_UPR + offset, 0);
+		if (ret < 0)
+			return ret;
+
+		ret = adin1110_write_reg(priv, ADIN1110_MAC_ADDR_MASK_LWR + offset, 0);
+	}
+
+	return ret;
+}
+
+static u32 adin1110_port_rules(struct adin1110_port_priv *port_priv,
+			       bool fw_to_host,
+			       bool fw_to_other_port)
+{
+	u32 port_rules = 0;
+
+	if (!port_priv->nr)
+		port_rules |= ADIN1110_MAC_ADDR_APPLY2PORT;
+	else
+		port_rules |= ADIN2111_MAC_ADDR_APPLY2PORT2;
+
+	if (fw_to_host)
+		port_rules |= ADIN1110_MAC_ADDR_TO_HOST;
+
+	if (fw_to_other_port && port_priv->priv->forwarding)
+		port_rules |= ADIN2111_MAC_ADDR_TO_OTHER_PORT;
+
+	return port_rules;
+}
+
+static int adin1110_multicast_filter(struct adin1110_port_priv *port_priv,
+				     int mac_nr, bool accept_multicast)
+{
+	u8 mask[ETH_ALEN] = {0};
+	u8 mac[ETH_ALEN] = {0};
+	u32 port_rules = 0;
+
+	mask[0] = BIT(0);
+	mac[0] = BIT(0);
+
+	if (accept_multicast && port_priv->state == BR_STATE_FORWARDING)
+		port_rules = adin1110_port_rules(port_priv, true, true);
+
+	return adin1110_write_mac_address(port_priv, mac_nr, mac,
+					  mask, port_rules);
+}
+
+static int adin1110_broadcasts_filter(struct adin1110_port_priv *port_priv,
+				      int mac_nr, bool accept_broadcast)
+{
+	u32 port_rules = 0;
+	u8 mask[ETH_ALEN];
+
+	memset(mask, 0xFF, ETH_ALEN);
+
+	if (accept_broadcast && port_priv->state == BR_STATE_FORWARDING)
+		port_rules = adin1110_port_rules(port_priv, true, true);
+
+	return adin1110_write_mac_address(port_priv, mac_nr, mask,
+					  mask, port_rules);
+}
+
+static int adin1110_set_mac_address(struct net_device *netdev,
+				    const unsigned char *dev_addr)
+{
+	struct adin1110_port_priv *port_priv = netdev_priv(netdev);
+	u8 mask[ETH_ALEN];
+	u32 port_rules;
+	u32 mac_slot;
+
+	if (!is_valid_ether_addr(dev_addr))
+		return -EADDRNOTAVAIL;
+
+	eth_hw_addr_set(netdev, dev_addr);
+	memset(mask, 0xFF, ETH_ALEN);
+
+	mac_slot = (!port_priv->nr) ?  ADIN_MAC_P1_ADDR_SLOT : ADIN_MAC_P2_ADDR_SLOT;
+	port_rules = adin1110_port_rules(port_priv, true, false);
+
+	return adin1110_write_mac_address(port_priv, mac_slot, netdev->dev_addr,
+					  mask, port_rules);
+}
+
+static int adin1110_ndo_set_mac_address(struct net_device *netdev, void *addr)
+{
+	struct sockaddr *sa = addr;
+	int ret;
+
+	ret = eth_prepare_mac_addr_change(netdev, addr);
+	if (ret < 0)
+		return ret;
+
+	return adin1110_set_mac_address(netdev, sa->sa_data);
+}
+
+static int adin1110_ioctl(struct net_device *netdev, struct ifreq *rq, int cmd)
+{
+	if (!netif_running(netdev))
+		return -EINVAL;
+
+	return phy_do_ioctl(netdev, rq, cmd);
+}
+
+static int adin1110_set_promisc_mode(struct adin1110_port_priv *port_priv,
+				     bool promisc)
+{
+	struct adin1110_priv *priv = port_priv->priv;
+	u32 mask;
+
+	if (port_priv->state != BR_STATE_FORWARDING)
+		promisc = false;
+
+	if (!port_priv->nr)
+		mask = ADIN1110_FWD_UNK2HOST;
+	else
+		mask = ADIN2111_P2_FWD_UNK2HOST;
+
+	return adin1110_set_bits(priv, ADIN1110_CONFIG2,
+				 mask, promisc ? mask : 0);
+}
+
+static int adin1110_setup_rx_mode(struct adin1110_port_priv *port_priv)
+{
+	int ret;
+
+	ret = adin1110_set_promisc_mode(port_priv,
+					!!(port_priv->flags & IFF_PROMISC));
+	if (ret < 0)
+		return ret;
+
+	ret = adin1110_multicast_filter(port_priv, ADIN_MAC_MULTICAST_ADDR_SLOT,
+					!!(port_priv->flags & IFF_ALLMULTI));
+	if (ret < 0)
+		return ret;
+
+	ret = adin1110_broadcasts_filter(port_priv,
+					 ADIN_MAC_BROADCAST_ADDR_SLOT,
+					 !!(port_priv->flags & IFF_BROADCAST));
+	if (ret < 0)
+		return ret;
+
+	return adin1110_set_bits(port_priv->priv, ADIN1110_CONFIG1,
+				 ADIN1110_CONFIG1_SYNC, ADIN1110_CONFIG1_SYNC);
+}
+
+static bool adin1110_can_offload_forwarding(struct adin1110_priv *priv)
+{
+	int i;
+
+	if (priv->cfg->id != ADIN2111_MAC)
+		return false;
+
+	/* Can't enable forwarding if ports do not belong to the same bridge */
+	if (priv->ports[0]->bridge != priv->ports[1]->bridge || !priv->ports[0]->bridge)
+		return false;
+
+	/* Can't enable forwarding if there is a port
+	 * that has been blocked by STP.
+	 */
+	for (i = 0; i < priv->cfg->ports_nr; i++) {
+		if (priv->ports[i]->state != BR_STATE_FORWARDING)
+			return false;
+	}
+
+	return true;
+}
+
+static void adin1110_rx_mode_work(struct work_struct *work)
+{
+	struct adin1110_port_priv *port_priv;
+	struct adin1110_priv *priv;
+
+	port_priv = container_of(work, struct adin1110_port_priv, rx_mode_work);
+	priv = port_priv->priv;
+
+	mutex_lock(&priv->lock);
+	adin1110_setup_rx_mode(port_priv);
+	mutex_unlock(&priv->lock);
+}
+
+static void adin1110_set_rx_mode(struct net_device *dev)
+{
+	struct adin1110_port_priv *port_priv = netdev_priv(dev);
+	struct adin1110_priv *priv = port_priv->priv;
+
+	spin_lock(&priv->state_lock);
+
+	port_priv->flags = dev->flags;
+	schedule_work(&port_priv->rx_mode_work);
+
+	spin_unlock(&priv->state_lock);
+}
+
+static int adin1110_net_open(struct net_device *net_dev)
+{
+	struct adin1110_port_priv *port_priv = netdev_priv(net_dev);
+	struct adin1110_priv *priv = port_priv->priv;
+	u32 val;
+	int ret;
+
+	mutex_lock(&priv->lock);
+
+	/* Configure MAC to compute and append the FCS itself. */
+	ret = adin1110_write_reg(priv, ADIN1110_CONFIG2, ADIN1110_CRC_APPEND);
+	if (ret < 0)
+		goto out;
+
+	val = ADIN1110_TX_RDY_IRQ | ADIN1110_RX_RDY_IRQ | ADIN1110_SPI_ERR_IRQ;
+	if (priv->cfg->id == ADIN2111_MAC)
+		val |= ADIN2111_RX_RDY_IRQ;
+
+	priv->irq_mask = val;
+	ret = adin1110_write_reg(priv, ADIN1110_IMASK1, ~val);
+	if (ret < 0) {
+		netdev_err(net_dev, "Failed to enable chip IRQs: %d\n", ret);
+		goto out;
+	}
+
+	ret = adin1110_read_reg(priv, ADIN1110_TX_SPACE, &val);
+	if (ret < 0) {
+		netdev_err(net_dev, "Failed to read TX FIFO space: %d\n", ret);
+		goto out;
+	}
+
+	priv->tx_space = 2 * val;
+
+	port_priv->state = BR_STATE_FORWARDING;
+	ret = adin1110_set_mac_address(net_dev, net_dev->dev_addr);
+	if (ret < 0) {
+		netdev_err(net_dev, "Could not set MAC address: %pM, %d\n",
+			   net_dev->dev_addr, ret);
+		goto out;
+	}
+
+	ret = adin1110_set_bits(priv, ADIN1110_CONFIG1, ADIN1110_CONFIG1_SYNC,
+				ADIN1110_CONFIG1_SYNC);
+
+out:
+	mutex_unlock(&priv->lock);
+
+	if (ret < 0)
+		return ret;
+
+	phy_start(port_priv->phydev);
+
+	netif_start_queue(net_dev);
+
+	return 0;
+}
+
+static int adin1110_net_stop(struct net_device *net_dev)
+{
+	struct adin1110_port_priv *port_priv = netdev_priv(net_dev);
+	struct adin1110_priv *priv = port_priv->priv;
+	u32 mask;
+	int ret;
+
+	mask = !port_priv->nr ? ADIN2111_RX_RDY_IRQ : ADIN1110_RX_RDY_IRQ;
+
+	/* Disable RX RDY IRQs */
+	mutex_lock(&priv->lock);
+	ret = adin1110_set_bits(priv, ADIN1110_IMASK1, mask, mask);
+	mutex_unlock(&priv->lock);
+	if (ret < 0)
+		return ret;
+
+	netif_stop_queue(port_priv->netdev);
+	flush_work(&port_priv->tx_work);
+	phy_stop(port_priv->phydev);
+
+	return 0;
+}
+
+static void adin1110_tx_work(struct work_struct *work)
+{
+	struct adin1110_port_priv *port_priv;
+	struct adin1110_priv *priv;
+	struct sk_buff *txb;
+	int ret;
+
+	port_priv = container_of(work, struct adin1110_port_priv, tx_work);
+	priv = port_priv->priv;
+
+	mutex_lock(&priv->lock);
+
+	while ((txb = skb_dequeue(&port_priv->txq))) {
+		ret = adin1110_write_fifo(port_priv, txb);
+		if (ret < 0)
+			dev_err_ratelimited(&priv->spidev->dev,
+					    "Frame write error: %d\n", ret);
+
+		dev_kfree_skb(txb);
+	}
+
+	mutex_unlock(&priv->lock);
+}
+
+static netdev_tx_t adin1110_start_xmit(struct sk_buff *skb, struct net_device *dev)
+{
+	struct adin1110_port_priv *port_priv = netdev_priv(dev);
+	struct adin1110_priv *priv = port_priv->priv;
+	netdev_tx_t netdev_ret = NETDEV_TX_OK;
+	u32 tx_space_needed;
+
+	tx_space_needed = skb->len + ADIN1110_FRAME_HEADER_LEN + ADIN1110_INTERNAL_SIZE_HEADER_LEN;
+	if (tx_space_needed > priv->tx_space) {
+		netif_stop_queue(dev);
+		netdev_ret = NETDEV_TX_BUSY;
+	} else {
+		priv->tx_space -= tx_space_needed;
+		skb_queue_tail(&port_priv->txq, skb);
+	}
+
+	schedule_work(&port_priv->tx_work);
+
+	return netdev_ret;
+}
+
+static void adin1110_ndo_get_stats64(struct net_device *dev,
+				     struct rtnl_link_stats64 *storage)
+{
+	struct adin1110_port_priv *port_priv = netdev_priv(dev);
+
+	storage->rx_packets = port_priv->rx_packets;
+	storage->tx_packets = port_priv->tx_packets;
+
+	storage->rx_bytes = port_priv->rx_bytes;
+	storage->tx_bytes = port_priv->tx_bytes;
+}
+
+static int adin1110_port_get_port_parent_id(struct net_device *dev,
+					    struct netdev_phys_item_id *ppid)
+{
+	struct adin1110_port_priv *port_priv = netdev_priv(dev);
+	struct adin1110_priv *priv = port_priv->priv;
+
+	ppid->id_len = strnlen(priv->mii_bus_name, MAX_PHYS_ITEM_ID_LEN);
+	memcpy(ppid->id, priv->mii_bus_name, ppid->id_len);
+
+	return 0;
+}
+
+static int adin1110_ndo_get_phys_port_name(struct net_device *dev,
+					   char *name, size_t len)
+{
+	struct adin1110_port_priv *port_priv = netdev_priv(dev);
+	int err;
+
+	err = snprintf(name, len, "p%d", port_priv->nr);
+	if (err >= len)
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct net_device_ops adin1110_netdev_ops = {
+	.ndo_open		= adin1110_net_open,
+	.ndo_stop		= adin1110_net_stop,
+	.ndo_eth_ioctl		= adin1110_ioctl,
+	.ndo_start_xmit		= adin1110_start_xmit,
+	.ndo_set_mac_address	= adin1110_ndo_set_mac_address,
+	.ndo_set_rx_mode	= adin1110_set_rx_mode,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_get_stats64	= adin1110_ndo_get_stats64,
+	.ndo_get_port_parent_id	= adin1110_port_get_port_parent_id,
+	.ndo_get_phys_port_name	= adin1110_ndo_get_phys_port_name,
+};
+
+static void adin1110_get_drvinfo(struct net_device *dev,
+				 struct ethtool_drvinfo *di)
+{
+	strscpy(di->driver, "ADIN1110", sizeof(di->driver));
+	strscpy(di->bus_info, dev_name(dev->dev.parent), sizeof(di->bus_info));
+}
+
+static const struct ethtool_ops adin1110_ethtool_ops = {
+	.get_drvinfo		= adin1110_get_drvinfo,
+	.get_link		= ethtool_op_get_link,
+	.get_link_ksettings	= phy_ethtool_get_link_ksettings,
+	.set_link_ksettings	= phy_ethtool_set_link_ksettings,
+};
+
+static void adin1110_adjust_link(struct net_device *dev)
+{
+	struct phy_device *phydev = dev->phydev;
+
+	if (!phydev->link)
+		phy_print_status(phydev);
+}
+
+/* PHY ID is stored in the MAC registers too,
+ * check spi connection by reading it.
+ */
+static int adin1110_check_spi(struct adin1110_priv *priv)
+{
+	int ret;
+	u32 val;
+
+	ret = adin1110_read_reg(priv, ADIN1110_PHY_ID, &val);
+	if (ret < 0)
+		return ret;
+
+	if (val != priv->cfg->phy_id_val) {
+		dev_err(&priv->spidev->dev, "PHY ID expected: %x, read: %x\n",
+			priv->cfg->phy_id_val, val);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+static int adin1110_hw_forwarding(struct adin1110_priv *priv, bool enable)
+{
+	int ret;
+	int i;
+
+	priv->forwarding = enable;
+
+	if (!priv->forwarding) {
+		for (i = ADIN_MAC_FDB_ADDR_SLOT; i < ADIN_MAC_MAX_ADDR_SLOTS; i++) {
+			ret = adin1110_clear_mac_address(priv, i);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	/* Forwarding is optimised when MAC runs in Cut Through mode. */
+	ret = adin1110_set_bits(priv, ADIN1110_CONFIG2,
+				ADIN2111_PORT_CUT_THRU_EN,
+				priv->forwarding ? ADIN2111_PORT_CUT_THRU_EN : 0);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < priv->cfg->ports_nr; i++) {
+		ret = adin1110_setup_rx_mode(priv->ports[i]);
+		if (ret < 0)
+			return ret;
+	}
+
+	return ret;
+}
+
+static int adin1110_port_bridge_join(struct adin1110_port_priv *port_priv,
+				     struct net_device *bridge)
+{
+	struct adin1110_priv *priv = port_priv->priv;
+	int ret;
+
+	port_priv->bridge = bridge;
+
+	if (adin1110_can_offload_forwarding(priv)) {
+		mutex_lock(&priv->lock);
+		ret = adin1110_hw_forwarding(priv, true);
+		mutex_unlock(&priv->lock);
+
+		if (ret < 0)
+			return ret;
+	}
+
+	return adin1110_set_mac_address(port_priv->netdev, bridge->dev_addr);
+}
+
+static int adin1110_port_bridge_leave(struct adin1110_port_priv *port_priv,
+				      struct net_device *bridge)
+{
+	struct adin1110_priv *priv = port_priv->priv;
+	int ret;
+
+	port_priv->bridge = NULL;
+
+	mutex_lock(&priv->lock);
+	ret = adin1110_hw_forwarding(priv, false);
+	mutex_unlock(&priv->lock);
+
+	return ret;
+}
+
+static int adin1110_netdevice_event(struct notifier_block *unused,
+				    unsigned long event, void *ptr)
+{
+	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+	struct adin1110_port_priv *port_priv = netdev_priv(dev);
+	struct netdev_notifier_changeupper_info *info = ptr;
+	int ret = 0;
+
+	switch (event) {
+	case NETDEV_CHANGEUPPER:
+		if (netif_is_bridge_master(info->upper_dev)) {
+			if (info->linking)
+				ret = adin1110_port_bridge_join(port_priv, info->upper_dev);
+			else
+				ret = adin1110_port_bridge_leave(port_priv, info->upper_dev);
+		}
+		break;
+	default:
+		break;
+	}
+
+	return notifier_from_errno(ret);
+}
+
+static struct notifier_block adin1110_netdevice_nb = {
+	.notifier_call = adin1110_netdevice_event,
+};
+
+static void adin1110_disconnect_phy(void *data)
+{
+	phy_disconnect(data);
+}
+
+static bool adin1110_port_dev_check(const struct net_device *dev)
+{
+	return dev->netdev_ops == &adin1110_netdev_ops;
+}
+
+static int adin1110_port_set_forwarding_state(struct adin1110_port_priv *port_priv)
+{
+	struct adin1110_priv *priv = port_priv->priv;
+	int ret;
+
+	port_priv->state = BR_STATE_FORWARDING;
+
+	mutex_lock(&priv->lock);
+	ret = adin1110_set_mac_address(port_priv->netdev,
+				       port_priv->netdev->dev_addr);
+	if (ret < 0)
+		goto out;
+
+	if (adin1110_can_offload_forwarding(priv))
+		ret = adin1110_hw_forwarding(priv, true);
+	else
+		ret = adin1110_setup_rx_mode(port_priv);
+out:
+	mutex_unlock(&priv->lock);
+
+	return ret;
+}
+
+static int adin1110_port_set_blocking_state(struct adin1110_port_priv *port_priv)
+{
+	u8 mac[ETH_ALEN] = {0x01, 0x80, 0xC2, 0x00, 0x00, 0x00};
+	struct adin1110_priv *priv = port_priv->priv;
+	u8 mask[ETH_ALEN];
+	u32 port_rules;
+	int mac_slot;
+	int ret;
+
+	port_priv->state = BR_STATE_BLOCKING;
+
+	mutex_lock(&priv->lock);
+
+	mac_slot = (!port_priv->nr) ?  ADIN_MAC_P1_ADDR_SLOT : ADIN_MAC_P2_ADDR_SLOT;
+	ret = adin1110_clear_mac_address(priv, mac_slot);
+	if (ret < 0)
+		goto out;
+
+	ret = adin1110_hw_forwarding(priv, false);
+	if (ret < 0)
+		goto out;
+
+	/* Allow only BPDUs to be passed to the CPU */
+	memset(mask, 0xFF, ETH_ALEN);
+	port_rules = adin1110_port_rules(port_priv, true, false);
+	ret = adin1110_write_mac_address(port_priv, mac_slot, mac,
+					 mask, port_rules);
+out:
+	mutex_unlock(&priv->lock);
+
+	return ret;
+}
+
+/* ADIN1110/2111 does not have any native STP support.
+ * Listen for bridge core state changes and
+ * allow all frames to pass or only the BPDUs.
+ */
+static int adin1110_port_attr_stp_state_set(struct adin1110_port_priv *port_priv,
+					    u8 state)
+{
+	switch (state) {
+	case BR_STATE_FORWARDING:
+		return adin1110_port_set_forwarding_state(port_priv);
+	case BR_STATE_LEARNING:
+	case BR_STATE_LISTENING:
+	case BR_STATE_DISABLED:
+	case BR_STATE_BLOCKING:
+		return adin1110_port_set_blocking_state(port_priv);
+	default:
+		return -EINVAL;
+	}
+}
+
+static int adin1110_port_attr_set(struct net_device *dev, const void *ctx,
+				  const struct switchdev_attr *attr,
+				  struct netlink_ext_ack *extack)
+{
+	struct adin1110_port_priv *port_priv = netdev_priv(dev);
+
+	switch (attr->id) {
+	case SWITCHDEV_ATTR_ID_PORT_STP_STATE:
+		return adin1110_port_attr_stp_state_set(port_priv,
+							attr->u.stp_state);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int adin1110_switchdev_blocking_event(struct notifier_block *unused,
+					     unsigned long event,
+					     void *ptr)
+{
+	struct net_device *netdev = switchdev_notifier_info_to_dev(ptr);
+	int ret;
+
+	if (event == SWITCHDEV_PORT_ATTR_SET) {
+		ret = switchdev_handle_port_attr_set(netdev, ptr,
+						     adin1110_port_dev_check,
+						     adin1110_port_attr_set);
+
+		return notifier_from_errno(ret);
+	}
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block adin1110_switchdev_blocking_notifier = {
+	.notifier_call = adin1110_switchdev_blocking_event,
+};
+
+static void adin1110_fdb_offload_notify(struct net_device *netdev,
+					struct switchdev_notifier_fdb_info *rcv)
+{
+	struct switchdev_notifier_fdb_info info = {};
+
+	info.addr = rcv->addr;
+	info.vid = rcv->vid;
+	info.offloaded = true;
+	call_switchdev_notifiers(SWITCHDEV_FDB_OFFLOADED,
+				 netdev, &info.info, NULL);
+}
+
+static int adin1110_fdb_add(struct adin1110_port_priv *port_priv,
+			    struct switchdev_notifier_fdb_info *fdb)
+{
+	struct adin1110_priv *priv = port_priv->priv;
+	struct adin1110_port_priv *other_port;
+	u8 mask[ETH_ALEN];
+	u32 port_rules;
+	int mac_nr;
+	u32 val;
+	int ret;
+
+	netdev_dbg(port_priv->netdev,
+		   "DEBUG: %s: MACID = %pM vid = %u flags = %u %u -- port %d\n",
+		    __func__, fdb->addr, fdb->vid, fdb->added_by_user,
+		    fdb->offloaded, port_priv->nr);
+
+	if (!priv->forwarding)
+		return 0;
+
+	if (fdb->is_local)
+		return -EINVAL;
+
+	/* Find free FDB slot on device. */
+	for (mac_nr = ADIN_MAC_FDB_ADDR_SLOT; mac_nr < ADIN_MAC_MAX_ADDR_SLOTS; mac_nr++) {
+		ret = adin1110_read_reg(priv, ADIN1110_MAC_ADDR_FILTER_UPR + (mac_nr * 2), &val);
+		if (ret < 0)
+			return ret;
+		if (!val)
+			break;
+	}
+
+	if (mac_nr == ADIN_MAC_MAX_ADDR_SLOTS)
+		return -ENOMEM;
+
+	other_port = priv->ports[!port_priv->nr];
+	port_rules = adin1110_port_rules(port_priv, false, true);
+	memset(mask, 0xFF, ETH_ALEN);
+
+	return adin1110_write_mac_address(other_port, mac_nr, (u8 *)fdb->addr,
+					  mask, port_rules);
+}
+
+static int adin1110_read_mac(struct adin1110_priv *priv, int mac_nr, u8 *addr)
+{
+	u32 val;
+	int ret;
+
+	ret = adin1110_read_reg(priv, ADIN1110_MAC_ADDR_FILTER_UPR + (mac_nr * 2), &val);
+	if (ret < 0)
+		return ret;
+
+	put_unaligned_be16(val, addr);
+
+	ret = adin1110_read_reg(priv, ADIN1110_MAC_ADDR_FILTER_LWR + (mac_nr * 2), &val);
+	if (ret < 0)
+		return ret;
+
+	put_unaligned_be32(val, addr + 2);
+
+	return 0;
+}
+
+static int adin1110_fdb_del(struct adin1110_port_priv *port_priv,
+			    struct switchdev_notifier_fdb_info *fdb)
+{
+	struct adin1110_priv *priv = port_priv->priv;
+	u8 addr[ETH_ALEN];
+	int mac_nr;
+	int ret;
+
+	netdev_dbg(port_priv->netdev,
+		   "DEBUG: %s: MACID = %pM vid = %u flags = %u %u -- port %d\n",
+		   __func__, fdb->addr, fdb->vid, fdb->added_by_user,
+		   fdb->offloaded, port_priv->nr);
+
+	if (fdb->is_local)
+		return -EINVAL;
+
+	for (mac_nr = ADIN_MAC_FDB_ADDR_SLOT; mac_nr < ADIN_MAC_MAX_ADDR_SLOTS; mac_nr++) {
+		ret = adin1110_read_mac(priv, mac_nr, addr);
+		if (ret < 0)
+			return ret;
+
+		if (ether_addr_equal(addr, fdb->addr)) {
+			ret = adin1110_clear_mac_address(priv, mac_nr);
+			if (ret < 0)
+				return ret;
+		}
+	}
+
+	return 0;
+}
+
+static void adin1110_switchdev_event_work(struct work_struct *work)
+{
+	struct adin1110_switchdev_event_work *switchdev_work;
+	struct adin1110_port_priv *port_priv;
+	int ret;
+
+	switchdev_work = container_of(work, struct adin1110_switchdev_event_work, work);
+	port_priv = switchdev_work->port_priv;
+
+	mutex_lock(&port_priv->priv->lock);
+
+	switch (switchdev_work->event) {
+	case SWITCHDEV_FDB_ADD_TO_DEVICE:
+		ret = adin1110_fdb_add(port_priv, &switchdev_work->fdb_info);
+		if (!ret)
+			adin1110_fdb_offload_notify(port_priv->netdev,
+						    &switchdev_work->fdb_info);
+		break;
+	case SWITCHDEV_FDB_DEL_TO_DEVICE:
+		adin1110_fdb_del(port_priv, &switchdev_work->fdb_info);
+		break;
+	default:
+		break;
+	}
+
+	mutex_unlock(&port_priv->priv->lock);
+
+	kfree(switchdev_work->fdb_info.addr);
+	kfree(switchdev_work);
+	dev_put(port_priv->netdev);
+}
+
+/* called under rcu_read_lock() */
+static int adin1110_switchdev_event(struct notifier_block *unused,
+				    unsigned long event, void *ptr)
+{
+	struct net_device *netdev = switchdev_notifier_info_to_dev(ptr);
+	struct adin1110_port_priv *port_priv = netdev_priv(netdev);
+	struct adin1110_switchdev_event_work *switchdev_work;
+	struct switchdev_notifier_fdb_info *fdb_info = ptr;
+
+	if (!adin1110_port_dev_check(netdev))
+		return NOTIFY_DONE;
+
+	switchdev_work = kzalloc(sizeof(*switchdev_work), GFP_ATOMIC);
+	if (WARN_ON(!switchdev_work))
+		return NOTIFY_BAD;
+
+	INIT_WORK(&switchdev_work->work, adin1110_switchdev_event_work);
+	switchdev_work->port_priv = port_priv;
+	switchdev_work->event = event;
+
+	switch (event) {
+	case SWITCHDEV_FDB_ADD_TO_DEVICE:
+	case SWITCHDEV_FDB_DEL_TO_DEVICE:
+		memcpy(&switchdev_work->fdb_info, ptr,
+		       sizeof(switchdev_work->fdb_info));
+		switchdev_work->fdb_info.addr = kzalloc(ETH_ALEN, GFP_ATOMIC);
+
+		if (!switchdev_work->fdb_info.addr)
+			goto err_addr_alloc;
+
+		ether_addr_copy((u8 *)switchdev_work->fdb_info.addr,
+				fdb_info->addr);
+		dev_hold(netdev);
+		break;
+	default:
+		kfree(switchdev_work);
+		return NOTIFY_DONE;
+	}
+
+	queue_work(system_long_wq, &switchdev_work->work);
+
+	return NOTIFY_DONE;
+
+err_addr_alloc:
+	kfree(switchdev_work);
+	return NOTIFY_BAD;
+}
+
+static struct notifier_block adin1110_switchdev_notifier = {
+	.notifier_call = adin1110_switchdev_event,
+};
+
+static void adin1110_unregister_notifiers(void *data)
+{
+	unregister_switchdev_blocking_notifier(&adin1110_switchdev_blocking_notifier);
+	unregister_switchdev_notifier(&adin1110_switchdev_notifier);
+	unregister_netdevice_notifier(&adin1110_netdevice_nb);
+}
+
+static int adin1110_setup_notifiers(struct adin1110_priv *priv)
+{
+	struct device *dev = &priv->spidev->dev;
+	int ret;
+
+	ret = register_netdevice_notifier(&adin1110_netdevice_nb);
+	if (ret < 0)
+		return ret;
+
+	ret = register_switchdev_notifier(&adin1110_switchdev_notifier);
+	if (ret < 0)
+		goto err_netdev;
+
+	ret = register_switchdev_blocking_notifier(&adin1110_switchdev_blocking_notifier);
+	if (ret < 0)
+		goto err_sdev;
+
+	return devm_add_action_or_reset(dev, adin1110_unregister_notifiers, NULL);
+
+err_sdev:
+	unregister_switchdev_notifier(&adin1110_switchdev_notifier);
+
+err_netdev:
+	unregister_netdevice_notifier(&adin1110_netdevice_nb);
+	return ret;
+}
+
+static int adin1110_probe_netdevs(struct adin1110_priv *priv)
+{
+	struct device *dev = &priv->spidev->dev;
+	struct adin1110_port_priv *port_priv;
+	struct net_device *netdev;
+	int ret;
+	int i;
+
+	for (i = 0; i < priv->cfg->ports_nr; i++) {
+		netdev = devm_alloc_etherdev(dev, sizeof(*port_priv));
+		if (!netdev)
+			return -ENOMEM;
+
+		port_priv = netdev_priv(netdev);
+		port_priv->netdev = netdev;
+		port_priv->priv = priv;
+		port_priv->cfg = priv->cfg;
+		port_priv->nr = i;
+		priv->ports[i] = port_priv;
+		SET_NETDEV_DEV(netdev, dev);
+
+		ret = device_get_ethdev_address(dev, netdev);
+		if (ret < 0)
+			return ret;
+
+		netdev->irq = priv->spidev->irq;
+		INIT_WORK(&port_priv->tx_work, adin1110_tx_work);
+		INIT_WORK(&port_priv->rx_mode_work, adin1110_rx_mode_work);
+		skb_queue_head_init(&port_priv->txq);
+
+		netif_carrier_off(netdev);
+
+		netdev->if_port = IF_PORT_10BASET;
+		netdev->netdev_ops = &adin1110_netdev_ops;
+		netdev->ethtool_ops = &adin1110_ethtool_ops;
+		netdev->priv_flags |= IFF_UNICAST_FLT;
+		netdev->features |= NETIF_F_NETNS_LOCAL;
+
+		port_priv->phydev = get_phy_device(priv->mii_bus, i + 1, false);
+		if (IS_ERR(port_priv->phydev)) {
+			netdev_err(netdev, "Could not find PHY with device address: %d.\n", i);
+			return PTR_ERR(port_priv->phydev);
+		}
+
+		port_priv->phydev = phy_connect(netdev,
+						phydev_name(port_priv->phydev),
+						adin1110_adjust_link,
+						PHY_INTERFACE_MODE_INTERNAL);
+		if (IS_ERR(port_priv->phydev)) {
+			netdev_err(netdev, "Could not connect PHY with device address: %d.\n", i);
+			return PTR_ERR(port_priv->phydev);
+		}
+
+		ret = devm_add_action_or_reset(dev, adin1110_disconnect_phy,
+					       port_priv->phydev);
+		if (ret < 0)
+			return ret;
+	}
+
+	/* ADIN1110 INT_N pin will be used to signal the host */
+	ret = devm_request_threaded_irq(dev, priv->spidev->irq, NULL,
+					adin1110_irq,
+					IRQF_TRIGGER_LOW | IRQF_ONESHOT,
+					dev_name(dev), priv);
+	if (ret < 0)
+		return ret;
+
+	ret = adin1110_setup_notifiers(priv);
+	if (ret < 0)
+		return ret;
+
+	for (i = 0; i < priv->cfg->ports_nr; i++) {
+		ret = devm_register_netdev(dev, priv->ports[i]->netdev);
+		if (ret < 0) {
+			dev_err(dev, "Failed to register network device.\n");
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static int adin1110_probe(struct spi_device *spi)
+{
+	const struct spi_device_id *dev_id = spi_get_device_id(spi);
+	struct device *dev = &spi->dev;
+	struct adin1110_priv *priv;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(struct adin1110_priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->spidev = spi;
+	priv->cfg = &adin1110_cfgs[dev_id->driver_data];
+	spi->bits_per_word = 8;
+	spi->mode = SPI_MODE_0;
+
+	mutex_init(&priv->lock);
+	spin_lock_init(&priv->state_lock);
+
+	/* use of CRC on control and data transactions is pin dependent */
+	priv->append_crc = device_property_read_bool(dev, "adi,spi-crc");
+	if (priv->append_crc)
+		crc8_populate_msb(adin1110_crc_table, 0x7);
+
+	ret = adin1110_check_spi(priv);
+	if (ret < 0) {
+		dev_err(dev, "Probe SPI Read check failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = adin1110_write_reg(priv, ADIN1110_RESET, ADIN1110_SWRESET);
+	if (ret < 0)
+		return ret;
+
+	ret = adin1110_register_mdiobus(priv, dev);
+	if (ret < 0) {
+		dev_err(dev, "Could not register MDIO bus %d\n", ret);
+		return ret;
+	}
+
+	return adin1110_probe_netdevs(priv);
+}
+
+static const struct of_device_id adin1110_match_table[] = {
+	{ .compatible = "adi,adin1110" },
+	{ .compatible = "adi,adin2111" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, adin1110_match_table);
+
+static const struct spi_device_id adin1110_spi_id[] = {
+	{ .name = "adin1110", .driver_data = ADIN1110_MAC },
+	{ .name = "adin2111", .driver_data = ADIN2111_MAC },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, adin1110_spi_id);
+
+static struct spi_driver adin1110_driver = {
+	.driver = {
+		.name = "adin1110",
+		.of_match_table = adin1110_match_table,
+	},
+	.probe = adin1110_probe,
+	.id_table = adin1110_spi_id,
+};
+module_spi_driver(adin1110_driver);
+
+MODULE_DESCRIPTION("ADIN1110 Network driver");
+MODULE_AUTHOR("Alexandru Tachici <alexandru.tachici@analog.com>");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/ethernet/aeroflex/greth.c b/drivers/net/ethernet/aeroflex/greth.c
index 447dc64a17e5..e104fb02817d 100644
--- a/drivers/net/ethernet/aeroflex/greth.c
+++ b/drivers/net/ethernet/aeroflex/greth.c
@@ -1112,9 +1112,9 @@ static void greth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *in
 {
 	struct greth_private *greth = netdev_priv(dev);
 
-	strlcpy(info->driver, dev_driver_string(greth->dev),
+	strscpy(info->driver, dev_driver_string(greth->dev),
 		sizeof(info->driver));
-	strlcpy(info->bus_info, greth->dev->bus->name, sizeof(info->bus_info));
+	strscpy(info->bus_info, greth->dev->bus->name, sizeof(info->bus_info));
 }
 
 static void greth_get_regs(struct net_device *dev, struct ethtool_regs *regs, void *p)
@@ -1507,7 +1507,7 @@ static int greth_of_probe(struct platform_device *ofdev)
 	}
 
 	/* setup NAPI */
-	netif_napi_add(dev, &greth->napi, greth_poll, 64);
+	netif_napi_add(dev, &greth->napi, greth_poll);
 
 	return 0;
 
diff --git a/drivers/net/ethernet/agere/et131x.c b/drivers/net/ethernet/agere/et131x.c
index d19d1579c415..5fab589b3ddf 100644
--- a/drivers/net/ethernet/agere/et131x.c
+++ b/drivers/net/ethernet/agere/et131x.c
@@ -2952,8 +2952,8 @@ static void et131x_get_drvinfo(struct net_device *netdev,
 {
 	struct et131x_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(adapter->pdev),
+	strscpy(info->driver, DRIVER_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(adapter->pdev),
 		sizeof(info->bus_info));
 }
 
@@ -3969,7 +3969,7 @@ static int et131x_pci_setup(struct pci_dev *pdev,
 
 	et131x_init_send(adapter);
 
-	netif_napi_add(netdev, &adapter->napi, et131x_poll, 64);
+	netif_napi_add(netdev, &adapter->napi, et131x_poll);
 
 	eth_hw_addr_set(netdev, adapter->addr);
 
diff --git a/drivers/net/ethernet/alacritech/slicoss.c b/drivers/net/ethernet/alacritech/slicoss.c
index ce353b0c02a3..a30d0f172986 100644
--- a/drivers/net/ethernet/alacritech/slicoss.c
+++ b/drivers/net/ethernet/alacritech/slicoss.c
@@ -1531,8 +1531,8 @@ static void slic_get_drvinfo(struct net_device *dev,
 {
 	struct slic_device *sdev = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(sdev->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(sdev->pdev), sizeof(info->bus_info));
 }
 
 static const struct ethtool_ops slic_ethtool_ops = {
@@ -1803,7 +1803,7 @@ static int slic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto unmap;
 	}
 
-	netif_napi_add(dev, &sdev->napi, slic_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &sdev->napi, slic_poll);
 	netif_carrier_off(dev);
 
 	err = register_netdev(dev);
diff --git a/drivers/net/ethernet/allwinner/sun4i-emac.c b/drivers/net/ethernet/allwinner/sun4i-emac.c
index 621ce742ad21..a94c62956eed 100644
--- a/drivers/net/ethernet/allwinner/sun4i-emac.c
+++ b/drivers/net/ethernet/allwinner/sun4i-emac.c
@@ -331,8 +331,8 @@ prepare_err:
 static void emac_get_drvinfo(struct net_device *dev,
 			      struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, dev_name(&dev->dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, dev_name(&dev->dev), sizeof(info->bus_info));
 }
 
 static u32 emac_get_msglevel(struct net_device *dev)
diff --git a/drivers/net/ethernet/alteon/acenic.c b/drivers/net/ethernet/alteon/acenic.c
index 22fe98555b24..d7762da8b2c0 100644
--- a/drivers/net/ethernet/alteon/acenic.c
+++ b/drivers/net/ethernet/alteon/acenic.c
@@ -2691,12 +2691,12 @@ static void ace_get_drvinfo(struct net_device *dev,
 {
 	struct ace_private *ap = netdev_priv(dev);
 
-	strlcpy(info->driver, "acenic", sizeof(info->driver));
+	strscpy(info->driver, "acenic", sizeof(info->driver));
 	snprintf(info->fw_version, sizeof(info->version), "%i.%i.%i",
 		 ap->firmware_major, ap->firmware_minor, ap->firmware_fix);
 
 	if (ap->pdev)
-		strlcpy(info->bus_info, pci_name(ap->pdev),
+		strscpy(info->bus_info, pci_name(ap->pdev),
 			sizeof(info->bus_info));
 
 }
diff --git a/drivers/net/ethernet/altera/Kconfig b/drivers/net/ethernet/altera/Kconfig
index 914e56b91467..dd7fd41ccde5 100644
--- a/drivers/net/ethernet/altera/Kconfig
+++ b/drivers/net/ethernet/altera/Kconfig
@@ -3,6 +3,8 @@ config ALTERA_TSE
 	tristate "Altera Triple-Speed Ethernet MAC support"
 	depends on HAS_DMA
 	select PHYLIB
+	select PHYLINK
+	select PCS_ALTERA_TSE
 	help
 	  This driver supports the Altera Triple-Speed (TSE) Ethernet MAC.
 
diff --git a/drivers/net/ethernet/altera/altera_tse.h b/drivers/net/ethernet/altera/altera_tse.h
index f17acfb579a0..db5eed06e92d 100644
--- a/drivers/net/ethernet/altera/altera_tse.h
+++ b/drivers/net/ethernet/altera/altera_tse.h
@@ -27,6 +27,7 @@
 #include <linux/list.h>
 #include <linux/netdevice.h>
 #include <linux/phy.h>
+#include <linux/phylink.h>
 
 #define ALTERA_TSE_SW_RESET_WATCHDOG_CNTR	10000
 #define ALTERA_TSE_MAC_FIFO_WIDTH		4	/* TX/RX FIFO width in
@@ -109,17 +110,6 @@
 #define MAC_CMDCFG_DISABLE_READ_TIMEOUT_GET(v)	GET_BIT_VALUE(v, 27)
 #define MAC_CMDCFG_CNT_RESET_GET(v)		GET_BIT_VALUE(v, 31)
 
-/* SGMII PCS register addresses
- */
-#define SGMII_PCS_SCRATCH	0x10
-#define SGMII_PCS_REV		0x11
-#define SGMII_PCS_LINK_TIMER_0	0x12
-#define SGMII_PCS_LINK_TIMER_1	0x13
-#define SGMII_PCS_IF_MODE	0x14
-#define SGMII_PCS_DIS_READ_TO	0x15
-#define SGMII_PCS_READ_TO	0x16
-#define SGMII_PCS_SW_RESET_TIMEOUT 100 /* usecs */
-
 /* MDIO registers within MAC register Space
  */
 struct altera_tse_mdio {
@@ -423,6 +413,9 @@ struct altera_tse_private {
 	void __iomem *tx_dma_csr;
 	void __iomem *tx_dma_desc;
 
+	/* SGMII PCS address space */
+	void __iomem *pcs_base;
+
 	/* Rx buffers queue */
 	struct tse_buffer *rx_ring;
 	u32 rx_cons;
@@ -480,6 +473,10 @@ struct altera_tse_private {
 	u32 msg_enable;
 
 	struct altera_dmaops *dmaops;
+
+	struct phylink *phylink;
+	struct phylink_config phylink_config;
+	struct phylink_pcs *pcs;
 };
 
 /* Function prototypes
diff --git a/drivers/net/ethernet/altera/altera_tse_ethtool.c b/drivers/net/ethernet/altera/altera_tse_ethtool.c
index 4299f1301149..81313c85833e 100644
--- a/drivers/net/ethernet/altera/altera_tse_ethtool.c
+++ b/drivers/net/ethernet/altera/altera_tse_ethtool.c
@@ -199,9 +199,9 @@ static int tse_reglen(struct net_device *dev)
 static void tse_get_regs(struct net_device *dev, struct ethtool_regs *regs,
 			 void *regbuf)
 {
-	int i;
 	struct altera_tse_private *priv = netdev_priv(dev);
 	u32 *buf = regbuf;
+	int i;
 
 	/* Set version to a known value, so ethtool knows
 	 * how to do any special formatting of this data.
@@ -221,6 +221,22 @@ static void tse_get_regs(struct net_device *dev, struct ethtool_regs *regs,
 		buf[i] = csrrd32(priv->mac_dev, i * 4);
 }
 
+static int tse_ethtool_set_link_ksettings(struct net_device *dev,
+					  const struct ethtool_link_ksettings *cmd)
+{
+	struct altera_tse_private *priv = netdev_priv(dev);
+
+	return phylink_ethtool_ksettings_set(priv->phylink, cmd);
+}
+
+static int tse_ethtool_get_link_ksettings(struct net_device *dev,
+					  struct ethtool_link_ksettings *cmd)
+{
+	struct altera_tse_private *priv = netdev_priv(dev);
+
+	return phylink_ethtool_ksettings_get(priv->phylink, cmd);
+}
+
 static const struct ethtool_ops tse_ethtool_ops = {
 	.get_drvinfo = tse_get_drvinfo,
 	.get_regs_len = tse_reglen,
@@ -231,8 +247,9 @@ static const struct ethtool_ops tse_ethtool_ops = {
 	.get_ethtool_stats = tse_fill_stats,
 	.get_msglevel = tse_get_msglevel,
 	.set_msglevel = tse_set_msglevel,
-	.get_link_ksettings = phy_ethtool_get_link_ksettings,
-	.set_link_ksettings = phy_ethtool_set_link_ksettings,
+	.get_link_ksettings = tse_ethtool_get_link_ksettings,
+	.set_link_ksettings = tse_ethtool_set_link_ksettings,
+	.get_ts_info = ethtool_op_get_ts_info,
 };
 
 void altera_tse_set_ethtool_ops(struct net_device *netdev)
diff --git a/drivers/net/ethernet/altera/altera_tse_main.c b/drivers/net/ethernet/altera/altera_tse_main.c
index 8c5828582c21..7633b227b2ca 100644
--- a/drivers/net/ethernet/altera/altera_tse_main.c
+++ b/drivers/net/ethernet/altera/altera_tse_main.c
@@ -32,6 +32,7 @@
 #include <linux/of_mdio.h>
 #include <linux/of_net.h>
 #include <linux/of_platform.h>
+#include <linux/pcs-altera-tse.h>
 #include <linux/phy.h>
 #include <linux/platform_device.h>
 #include <linux/skbuff.h>
@@ -86,27 +87,6 @@ static inline u32 tse_tx_avail(struct altera_tse_private *priv)
 	return priv->tx_cons + priv->tx_ring_size - priv->tx_prod - 1;
 }
 
-/* PCS Register read/write functions
- */
-static u16 sgmii_pcs_read(struct altera_tse_private *priv, int regnum)
-{
-	return csrrd32(priv->mac_dev,
-		       tse_csroffs(mdio_phy0) + regnum * 4) & 0xffff;
-}
-
-static void sgmii_pcs_write(struct altera_tse_private *priv, int regnum,
-				u16 value)
-{
-	csrwr32(value, priv->mac_dev, tse_csroffs(mdio_phy0) + regnum * 4);
-}
-
-/* Check PCS scratch memory */
-static int sgmii_pcs_scratch_test(struct altera_tse_private *priv, u16 value)
-{
-	sgmii_pcs_write(priv, SGMII_PCS_SCRATCH, value);
-	return (sgmii_pcs_read(priv, SGMII_PCS_SCRATCH) == value);
-}
-
 /* MDIO specific functions
  */
 static int altera_tse_mdio_read(struct mii_bus *bus, int mii_id, int regnum)
@@ -141,10 +121,10 @@ static int altera_tse_mdio_write(struct mii_bus *bus, int mii_id, int regnum,
 static int altera_tse_mdio_create(struct net_device *dev, unsigned int id)
 {
 	struct altera_tse_private *priv = netdev_priv(dev);
-	int ret;
 	struct device_node *mdio_node = NULL;
-	struct mii_bus *mdio = NULL;
 	struct device_node *child_node = NULL;
+	struct mii_bus *mdio = NULL;
+	int ret;
 
 	for_each_child_of_node(priv->device->of_node, child_node) {
 		if (of_device_is_compatible(child_node, "altr,tse-mdio")) {
@@ -236,8 +216,8 @@ static int tse_init_rx_buffer(struct altera_tse_private *priv,
 static void tse_free_rx_buffer(struct altera_tse_private *priv,
 			       struct tse_buffer *rxbuffer)
 {
-	struct sk_buff *skb = rxbuffer->skb;
 	dma_addr_t dma_addr = rxbuffer->dma_addr;
+	struct sk_buff *skb = rxbuffer->skb;
 
 	if (skb != NULL) {
 		if (dma_addr)
@@ -358,6 +338,7 @@ static inline void tse_rx_vlan(struct net_device *dev, struct sk_buff *skb)
 {
 	struct ethhdr *eth_hdr;
 	u16 vid;
+
 	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
 	    !__vlan_get_tag(skb, &vid)) {
 		eth_hdr = (struct ethhdr *)skb->data;
@@ -371,10 +352,10 @@ static inline void tse_rx_vlan(struct net_device *dev, struct sk_buff *skb)
  */
 static int tse_rx(struct altera_tse_private *priv, int limit)
 {
-	unsigned int count = 0;
+	unsigned int entry = priv->rx_cons % priv->rx_ring_size;
 	unsigned int next_entry;
+	unsigned int count = 0;
 	struct sk_buff *skb;
-	unsigned int entry = priv->rx_cons % priv->rx_ring_size;
 	u32 rxstatus;
 	u16 pktlength;
 	u16 pktstatus;
@@ -448,10 +429,10 @@ static int tse_rx(struct altera_tse_private *priv, int limit)
 static int tse_tx_complete(struct altera_tse_private *priv)
 {
 	unsigned int txsize = priv->tx_ring_size;
-	u32 ready;
-	unsigned int entry;
 	struct tse_buffer *tx_buff;
+	unsigned int entry;
 	int txcomplete = 0;
+	u32 ready;
 
 	spin_lock(&priv->tx_lock);
 
@@ -497,8 +478,8 @@ static int tse_poll(struct napi_struct *napi, int budget)
 {
 	struct altera_tse_private *priv =
 			container_of(napi, struct altera_tse_private, napi);
-	int rxcomplete = 0;
 	unsigned long int flags;
+	int rxcomplete = 0;
 
 	tse_tx_complete(priv);
 
@@ -561,13 +542,13 @@ static irqreturn_t altera_isr(int irq, void *dev_id)
 static netdev_tx_t tse_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct altera_tse_private *priv = netdev_priv(dev);
+	unsigned int nopaged_len = skb_headlen(skb);
 	unsigned int txsize = priv->tx_ring_size;
-	unsigned int entry;
-	struct tse_buffer *buffer = NULL;
 	int nfrags = skb_shinfo(skb)->nr_frags;
-	unsigned int nopaged_len = skb_headlen(skb);
+	struct tse_buffer *buffer = NULL;
 	netdev_tx_t ret = NETDEV_TX_OK;
 	dma_addr_t dma_addr;
+	unsigned int entry;
 
 	spin_lock_bh(&priv->tx_lock);
 
@@ -619,117 +600,6 @@ out:
 	return ret;
 }
 
-/* Called every time the controller might need to be made
- * aware of new link state.  The PHY code conveys this
- * information through variables in the phydev structure, and this
- * function converts those variables into the appropriate
- * register values, and can bring down the device if needed.
- */
-static void altera_tse_adjust_link(struct net_device *dev)
-{
-	struct altera_tse_private *priv = netdev_priv(dev);
-	struct phy_device *phydev = dev->phydev;
-	int new_state = 0;
-
-	/* only change config if there is a link */
-	spin_lock(&priv->mac_cfg_lock);
-	if (phydev->link) {
-		/* Read old config */
-		u32 cfg_reg = ioread32(&priv->mac_dev->command_config);
-
-		/* Check duplex */
-		if (phydev->duplex != priv->oldduplex) {
-			new_state = 1;
-			if (!(phydev->duplex))
-				cfg_reg |= MAC_CMDCFG_HD_ENA;
-			else
-				cfg_reg &= ~MAC_CMDCFG_HD_ENA;
-
-			netdev_dbg(priv->dev, "%s: Link duplex = 0x%x\n",
-				   dev->name, phydev->duplex);
-
-			priv->oldduplex = phydev->duplex;
-		}
-
-		/* Check speed */
-		if (phydev->speed != priv->oldspeed) {
-			new_state = 1;
-			switch (phydev->speed) {
-			case 1000:
-				cfg_reg |= MAC_CMDCFG_ETH_SPEED;
-				cfg_reg &= ~MAC_CMDCFG_ENA_10;
-				break;
-			case 100:
-				cfg_reg &= ~MAC_CMDCFG_ETH_SPEED;
-				cfg_reg &= ~MAC_CMDCFG_ENA_10;
-				break;
-			case 10:
-				cfg_reg &= ~MAC_CMDCFG_ETH_SPEED;
-				cfg_reg |= MAC_CMDCFG_ENA_10;
-				break;
-			default:
-				if (netif_msg_link(priv))
-					netdev_warn(dev, "Speed (%d) is not 10/100/1000!\n",
-						    phydev->speed);
-				break;
-			}
-			priv->oldspeed = phydev->speed;
-		}
-		iowrite32(cfg_reg, &priv->mac_dev->command_config);
-
-		if (!priv->oldlink) {
-			new_state = 1;
-			priv->oldlink = 1;
-		}
-	} else if (priv->oldlink) {
-		new_state = 1;
-		priv->oldlink = 0;
-		priv->oldspeed = 0;
-		priv->oldduplex = -1;
-	}
-
-	if (new_state && netif_msg_link(priv))
-		phy_print_status(phydev);
-
-	spin_unlock(&priv->mac_cfg_lock);
-}
-static struct phy_device *connect_local_phy(struct net_device *dev)
-{
-	struct altera_tse_private *priv = netdev_priv(dev);
-	struct phy_device *phydev = NULL;
-	char phy_id_fmt[MII_BUS_ID_SIZE + 3];
-
-	if (priv->phy_addr != POLL_PHY) {
-		snprintf(phy_id_fmt, MII_BUS_ID_SIZE + 3, PHY_ID_FMT,
-			 priv->mdio->id, priv->phy_addr);
-
-		netdev_dbg(dev, "trying to attach to %s\n", phy_id_fmt);
-
-		phydev = phy_connect(dev, phy_id_fmt, &altera_tse_adjust_link,
-				     priv->phy_iface);
-		if (IS_ERR(phydev)) {
-			netdev_err(dev, "Could not attach to PHY\n");
-			phydev = NULL;
-		}
-
-	} else {
-		int ret;
-		phydev = phy_find_first(priv->mdio);
-		if (phydev == NULL) {
-			netdev_err(dev, "No PHY found\n");
-			return phydev;
-		}
-
-		ret = phy_connect_direct(dev, phydev, &altera_tse_adjust_link,
-				priv->phy_iface);
-		if (ret != 0) {
-			netdev_err(dev, "Could not attach to PHY\n");
-			phydev = NULL;
-		}
-	}
-	return phydev;
-}
-
 static int altera_tse_phy_get_addr_mdio_create(struct net_device *dev)
 {
 	struct altera_tse_private *priv = netdev_priv(dev);
@@ -768,91 +638,6 @@ static int altera_tse_phy_get_addr_mdio_create(struct net_device *dev)
 	return 0;
 }
 
-/* Initialize driver's PHY state, and attach to the PHY
- */
-static int init_phy(struct net_device *dev)
-{
-	struct altera_tse_private *priv = netdev_priv(dev);
-	struct phy_device *phydev;
-	struct device_node *phynode;
-	bool fixed_link = false;
-	int rc = 0;
-
-	/* Avoid init phy in case of no phy present */
-	if (!priv->phy_iface)
-		return 0;
-
-	priv->oldlink = 0;
-	priv->oldspeed = 0;
-	priv->oldduplex = -1;
-
-	phynode = of_parse_phandle(priv->device->of_node, "phy-handle", 0);
-
-	if (!phynode) {
-		/* check if a fixed-link is defined in device-tree */
-		if (of_phy_is_fixed_link(priv->device->of_node)) {
-			rc = of_phy_register_fixed_link(priv->device->of_node);
-			if (rc < 0) {
-				netdev_err(dev, "cannot register fixed PHY\n");
-				return rc;
-			}
-
-			/* In the case of a fixed PHY, the DT node associated
-			 * to the PHY is the Ethernet MAC DT node.
-			 */
-			phynode = of_node_get(priv->device->of_node);
-			fixed_link = true;
-
-			netdev_dbg(dev, "fixed-link detected\n");
-			phydev = of_phy_connect(dev, phynode,
-						&altera_tse_adjust_link,
-						0, priv->phy_iface);
-		} else {
-			netdev_dbg(dev, "no phy-handle found\n");
-			if (!priv->mdio) {
-				netdev_err(dev, "No phy-handle nor local mdio specified\n");
-				return -ENODEV;
-			}
-			phydev = connect_local_phy(dev);
-		}
-	} else {
-		netdev_dbg(dev, "phy-handle found\n");
-		phydev = of_phy_connect(dev, phynode,
-			&altera_tse_adjust_link, 0, priv->phy_iface);
-	}
-	of_node_put(phynode);
-
-	if (!phydev) {
-		netdev_err(dev, "Could not find the PHY\n");
-		if (fixed_link)
-			of_phy_deregister_fixed_link(priv->device->of_node);
-		return -ENODEV;
-	}
-
-	/* Stop Advertising 1000BASE Capability if interface is not GMII
-	 */
-	if ((priv->phy_iface == PHY_INTERFACE_MODE_MII) ||
-	    (priv->phy_iface == PHY_INTERFACE_MODE_RMII))
-		phy_set_max_speed(phydev, SPEED_100);
-
-	/* Broken HW is sometimes missing the pull-up resistor on the
-	 * MDIO line, which results in reads to non-existent devices returning
-	 * 0 rather than 0xffff. Catch this here and treat 0 as a non-existent
-	 * device as well. If a fixed-link is used the phy_id is always 0.
-	 * Note: phydev->phy_id is the result of reading the UID PHY registers.
-	 */
-	if ((phydev->phy_id == 0) && !fixed_link) {
-		netdev_err(dev, "Bad PHY UID 0x%08x\n", phydev->phy_id);
-		phy_disconnect(phydev);
-		return -ENODEV;
-	}
-
-	netdev_dbg(dev, "attached to PHY %d UID 0x%08x Link = %d\n",
-		   phydev->mdio.addr, phydev->phy_id, phydev->link);
-
-	return 0;
-}
-
 static void tse_update_mac_addr(struct altera_tse_private *priv, const u8 *addr)
 {
 	u32 msb;
@@ -1012,8 +797,8 @@ static int tse_change_mtu(struct net_device *dev, int new_mtu)
 static void altera_tse_set_mcfilter(struct net_device *dev)
 {
 	struct altera_tse_private *priv = netdev_priv(dev);
-	int i;
 	struct netdev_hw_addr *ha;
+	int i;
 
 	/* clear the hash filter */
 	for (i = 0; i < 64; i++)
@@ -1087,74 +872,14 @@ static void tse_set_rx_mode(struct net_device *dev)
 	spin_unlock(&priv->mac_cfg_lock);
 }
 
-/* Initialise (if necessary) the SGMII PCS component
- */
-static int init_sgmii_pcs(struct net_device *dev)
-{
-	struct altera_tse_private *priv = netdev_priv(dev);
-	int n;
-	unsigned int tmp_reg = 0;
-
-	if (priv->phy_iface != PHY_INTERFACE_MODE_SGMII)
-		return 0; /* Nothing to do, not in SGMII mode */
-
-	/* The TSE SGMII PCS block looks a little like a PHY, it is
-	 * mapped into the zeroth MDIO space of the MAC and it has
-	 * ID registers like a PHY would.  Sadly this is often
-	 * configured to zeroes, so don't be surprised if it does
-	 * show 0x00000000.
-	 */
-
-	if (sgmii_pcs_scratch_test(priv, 0x0000) &&
-		sgmii_pcs_scratch_test(priv, 0xffff) &&
-		sgmii_pcs_scratch_test(priv, 0xa5a5) &&
-		sgmii_pcs_scratch_test(priv, 0x5a5a)) {
-		netdev_info(dev, "PCS PHY ID: 0x%04x%04x\n",
-				sgmii_pcs_read(priv, MII_PHYSID1),
-				sgmii_pcs_read(priv, MII_PHYSID2));
-	} else {
-		netdev_err(dev, "SGMII PCS Scratch memory test failed.\n");
-		return -ENOMEM;
-	}
-
-	/* Starting on page 5-29 of the MegaCore Function User Guide
-	 * Set SGMII Link timer to 1.6ms
-	 */
-	sgmii_pcs_write(priv, SGMII_PCS_LINK_TIMER_0, 0x0D40);
-	sgmii_pcs_write(priv, SGMII_PCS_LINK_TIMER_1, 0x03);
-
-	/* Enable SGMII Interface and Enable SGMII Auto Negotiation */
-	sgmii_pcs_write(priv, SGMII_PCS_IF_MODE, 0x3);
-
-	/* Enable Autonegotiation */
-	tmp_reg = sgmii_pcs_read(priv, MII_BMCR);
-	tmp_reg |= (BMCR_SPEED1000 | BMCR_FULLDPLX | BMCR_ANENABLE);
-	sgmii_pcs_write(priv, MII_BMCR, tmp_reg);
-
-	/* Reset PCS block */
-	tmp_reg |= BMCR_RESET;
-	sgmii_pcs_write(priv, MII_BMCR, tmp_reg);
-	for (n = 0; n < SGMII_PCS_SW_RESET_TIMEOUT; n++) {
-		if (!(sgmii_pcs_read(priv, MII_BMCR) & BMCR_RESET)) {
-			netdev_info(dev, "SGMII PCS block initialised OK\n");
-			return 0;
-		}
-		udelay(1);
-	}
-
-	/* We failed to reset the block, return a timeout */
-	netdev_err(dev, "SGMII PCS block reset failed.\n");
-	return -ETIMEDOUT;
-}
-
 /* Open and initialize the interface
  */
 static int tse_open(struct net_device *dev)
 {
 	struct altera_tse_private *priv = netdev_priv(dev);
+	unsigned long flags;
 	int ret = 0;
 	int i;
-	unsigned long int flags;
 
 	/* Reset and configure TSE MAC and probe associated PHY */
 	ret = priv->dmaops->init_dma(priv);
@@ -1171,14 +896,6 @@ static int tse_open(struct net_device *dev)
 		netdev_warn(dev, "TSE revision %x\n", priv->revision);
 
 	spin_lock(&priv->mac_cfg_lock);
-	/* no-op if MAC not operating in SGMII mode*/
-	ret = init_sgmii_pcs(dev);
-	if (ret) {
-		netdev_err(dev,
-			   "Cannot init the SGMII PCS (error: %d)\n", ret);
-		spin_unlock(&priv->mac_cfg_lock);
-		goto phy_error;
-	}
 
 	ret = reset_mac(priv);
 	/* Note that reset_mac will fail if the clocks are gated by the PHY
@@ -1236,8 +953,12 @@ static int tse_open(struct net_device *dev)
 
 	spin_unlock_irqrestore(&priv->rxdma_irq_lock, flags);
 
-	if (dev->phydev)
-		phy_start(dev->phydev);
+	ret = phylink_of_phy_connect(priv->phylink, priv->device->of_node, 0);
+	if (ret) {
+		netdev_err(dev, "could not connect phylink (%d)\n", ret);
+		goto tx_request_irq_error;
+	}
+	phylink_start(priv->phylink);
 
 	napi_enable(&priv->napi);
 	netif_start_queue(dev);
@@ -1265,13 +986,10 @@ phy_error:
 static int tse_shutdown(struct net_device *dev)
 {
 	struct altera_tse_private *priv = netdev_priv(dev);
-	int ret;
 	unsigned long int flags;
+	int ret;
 
-	/* Stop the PHY */
-	if (dev->phydev)
-		phy_stop(dev->phydev);
-
+	phylink_stop(priv->phylink);
 	netif_stop_queue(dev);
 	napi_disable(&priv->napi);
 
@@ -1317,11 +1035,79 @@ static struct net_device_ops altera_tse_netdev_ops = {
 	.ndo_validate_addr	= eth_validate_addr,
 };
 
+static void alt_tse_mac_an_restart(struct phylink_config *config)
+{
+}
+
+static void alt_tse_mac_config(struct phylink_config *config, unsigned int mode,
+			       const struct phylink_link_state *state)
+{
+	struct net_device *ndev = to_net_dev(config->dev);
+	struct altera_tse_private *priv = netdev_priv(ndev);
+
+	spin_lock(&priv->mac_cfg_lock);
+	reset_mac(priv);
+	tse_set_mac(priv, true);
+	spin_unlock(&priv->mac_cfg_lock);
+}
+
+static void alt_tse_mac_link_down(struct phylink_config *config,
+				  unsigned int mode, phy_interface_t interface)
+{
+}
+
+static void alt_tse_mac_link_up(struct phylink_config *config,
+				struct phy_device *phy, unsigned int mode,
+				phy_interface_t interface, int speed,
+				int duplex, bool tx_pause, bool rx_pause)
+{
+	struct net_device *ndev = to_net_dev(config->dev);
+	struct altera_tse_private *priv = netdev_priv(ndev);
+	u32 ctrl;
+
+	ctrl = csrrd32(priv->mac_dev, tse_csroffs(command_config));
+	ctrl &= ~(MAC_CMDCFG_ENA_10 | MAC_CMDCFG_ETH_SPEED | MAC_CMDCFG_HD_ENA);
+
+	if (duplex == DUPLEX_HALF)
+		ctrl |= MAC_CMDCFG_HD_ENA;
+
+	if (speed == SPEED_1000)
+		ctrl |= MAC_CMDCFG_ETH_SPEED;
+	else if (speed == SPEED_10)
+		ctrl |= MAC_CMDCFG_ENA_10;
+
+	spin_lock(&priv->mac_cfg_lock);
+	csrwr32(ctrl, priv->mac_dev, tse_csroffs(command_config));
+	spin_unlock(&priv->mac_cfg_lock);
+}
+
+static struct phylink_pcs *alt_tse_select_pcs(struct phylink_config *config,
+					      phy_interface_t interface)
+{
+	struct net_device *ndev = to_net_dev(config->dev);
+	struct altera_tse_private *priv = netdev_priv(ndev);
+
+	if (interface == PHY_INTERFACE_MODE_SGMII ||
+	    interface == PHY_INTERFACE_MODE_1000BASEX)
+		return priv->pcs;
+	else
+		return NULL;
+}
+
+static const struct phylink_mac_ops alt_tse_phylink_ops = {
+	.validate = phylink_generic_validate,
+	.mac_an_restart = alt_tse_mac_an_restart,
+	.mac_config = alt_tse_mac_config,
+	.mac_link_down = alt_tse_mac_link_down,
+	.mac_link_up = alt_tse_mac_link_up,
+	.mac_select_pcs = alt_tse_select_pcs,
+};
+
 static int request_and_map(struct platform_device *pdev, const char *name,
 			   struct resource **res, void __iomem **ptr)
 {
-	struct resource *region;
 	struct device *device = &pdev->dev;
+	struct resource *region;
 
 	*res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
 	if (*res == NULL) {
@@ -1350,13 +1136,15 @@ static int request_and_map(struct platform_device *pdev, const char *name,
  */
 static int altera_tse_probe(struct platform_device *pdev)
 {
-	struct net_device *ndev;
-	int ret = -ENODEV;
+	const struct of_device_id *of_id = NULL;
+	struct altera_tse_private *priv;
 	struct resource *control_port;
 	struct resource *dma_res;
-	struct altera_tse_private *priv;
+	struct resource *pcs_res;
+	struct net_device *ndev;
 	void __iomem *descmap;
-	const struct of_device_id *of_id = NULL;
+	int pcs_reg_width = 2;
+	int ret = -ENODEV;
 
 	ndev = alloc_etherdev(sizeof(struct altera_tse_private));
 	if (!ndev) {
@@ -1467,6 +1255,17 @@ static int altera_tse_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_free_netdev;
 
+	/* SGMII PCS address space. The location can vary depending on how the
+	 * IP is integrated. We can have a resource dedicated to it at a specific
+	 * address space, but if it's not the case, we fallback to the mdiophy0
+	 * from the MAC's address space
+	 */
+	ret = request_and_map(pdev, "pcs", &pcs_res,
+			      &priv->pcs_base);
+	if (ret) {
+		priv->pcs_base = priv->mac_dev + tse_csroffs(mdio_phy0);
+		pcs_reg_width = 4;
+	}
 
 	/* Rx IRQ */
 	priv->rx_irq = platform_get_irq_byname(pdev, "rx_irq");
@@ -1566,7 +1365,7 @@ static int altera_tse_probe(struct platform_device *pdev)
 	ndev->features |= NETIF_F_HW_VLAN_CTAG_RX;
 
 	/* setup NAPI interface */
-	netif_napi_add(ndev, &priv->napi, tse_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &priv->napi, tse_poll);
 
 	spin_lock_init(&priv->mac_cfg_lock);
 	spin_lock_init(&priv->tx_lock);
@@ -1590,11 +1389,32 @@ static int altera_tse_probe(struct platform_device *pdev)
 			 (unsigned long) control_port->start, priv->rx_irq,
 			 priv->tx_irq);
 
-	ret = init_phy(ndev);
-	if (ret != 0) {
-		netdev_err(ndev, "Cannot attach to PHY (error: %d)\n", ret);
+	priv->pcs = alt_tse_pcs_create(ndev, priv->pcs_base, pcs_reg_width);
+
+	priv->phylink_config.dev = &ndev->dev;
+	priv->phylink_config.type = PHYLINK_NETDEV;
+	priv->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 |
+						MAC_100 | MAC_1000FD;
+
+	phy_interface_set_rgmii(priv->phylink_config.supported_interfaces);
+	__set_bit(PHY_INTERFACE_MODE_MII,
+		  priv->phylink_config.supported_interfaces);
+	__set_bit(PHY_INTERFACE_MODE_GMII,
+		  priv->phylink_config.supported_interfaces);
+	__set_bit(PHY_INTERFACE_MODE_SGMII,
+		  priv->phylink_config.supported_interfaces);
+	__set_bit(PHY_INTERFACE_MODE_1000BASEX,
+		  priv->phylink_config.supported_interfaces);
+
+	priv->phylink = phylink_create(&priv->phylink_config,
+				       of_fwnode_handle(priv->device->of_node),
+				       priv->phy_iface, &alt_tse_phylink_ops);
+	if (IS_ERR(priv->phylink)) {
+		dev_err(&pdev->dev, "failed to create phylink\n");
+		ret = PTR_ERR(priv->phylink);
 		goto err_init_phy;
 	}
+
 	return 0;
 
 err_init_phy:
@@ -1614,16 +1434,10 @@ static int altera_tse_remove(struct platform_device *pdev)
 	struct net_device *ndev = platform_get_drvdata(pdev);
 	struct altera_tse_private *priv = netdev_priv(ndev);
 
-	if (ndev->phydev) {
-		phy_disconnect(ndev->phydev);
-
-		if (of_phy_is_fixed_link(priv->device->of_node))
-			of_phy_deregister_fixed_link(priv->device->of_node);
-	}
-
 	platform_set_drvdata(pdev, NULL);
 	altera_tse_mdio_destroy(ndev);
 	unregister_netdev(ndev);
+	phylink_destroy(priv->phylink);
 	free_netdev(ndev);
 
 	return 0;
diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c
index 39242c5a1729..98d6386b7f39 100644
--- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c
+++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c
@@ -462,8 +462,8 @@ static void ena_get_drvinfo(struct net_device *dev,
 {
 	struct ena_adapter *adapter = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(adapter->pdev),
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(adapter->pdev),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c
index 6a356a6cee15..d350eeec8bad 100644
--- a/drivers/net/ethernet/amazon/ena/ena_netdev.c
+++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c
@@ -2265,10 +2265,8 @@ static void ena_init_napi_in_range(struct ena_adapter *adapter,
 	for (i = first_index; i < first_index + count; i++) {
 		struct ena_napi *napi = &adapter->ena_napi[i];
 
-		netif_napi_add(adapter->netdev,
-			       &napi->napi,
-			       ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(adapter->netdev, &napi->napi,
+			       ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll);
 
 		if (!ENA_IS_XDP_INDEX(adapter, i)) {
 			napi->rx_ring = &adapter->rx_ring[i];
@@ -3166,7 +3164,7 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pd
 	host_info->bdf = (pdev->bus->number << 8) | pdev->devfn;
 	host_info->os_type = ENA_ADMIN_OS_LINUX;
 	host_info->kernel_ver = LINUX_VERSION_CODE;
-	strlcpy(host_info->kernel_ver_str, utsname()->version,
+	strscpy(host_info->kernel_ver_str, utsname()->version,
 		sizeof(host_info->kernel_ver_str) - 1);
 	host_info->os_dist = 0;
 	strncpy(host_info->os_dist_str, utsname()->release,
diff --git a/drivers/net/ethernet/amd/a2065.c b/drivers/net/ethernet/amd/a2065.c
index 3a351d3396bf..68983b717145 100644
--- a/drivers/net/ethernet/amd/a2065.c
+++ b/drivers/net/ethernet/amd/a2065.c
@@ -695,7 +695,7 @@ static int a2065_init_one(struct zorro_dev *z,
 	}
 
 	dev = alloc_etherdev(sizeof(struct lance_private));
-	if (dev == NULL) {
+	if (!dev) {
 		release_mem_region(base_addr, sizeof(struct lance_regs));
 		release_mem_region(mem_start, A2065_RAM_SIZE);
 		return -ENOMEM;
diff --git a/drivers/net/ethernet/amd/amd8111e.c b/drivers/net/ethernet/amd/amd8111e.c
index 5d1baa01360f..ea6cfc2095e1 100644
--- a/drivers/net/ethernet/amd/amd8111e.c
+++ b/drivers/net/ethernet/amd/amd8111e.c
@@ -43,7 +43,7 @@ Revision History:
 	3.0.4 12/09/2003
 	 1. Added set_mac_address routine for bonding driver support.
 	 2. Tested the driver for bonding support
-	 3. Bug fix: Fixed mismach in actual receive buffer lenth and lenth
+	 3. Bug fix: Fixed mismach in actual receive buffer length and length
 	    indicated to the h/w.
 	 4. Modified amd8111e_rx() routine to receive all the received packets
 	    in the first interrupt.
@@ -185,24 +185,23 @@ static void amd8111e_set_ext_phy(struct net_device *dev)
 	advert = amd8111e_mdio_read(dev, lp->ext_phy_addr, MII_ADVERTISE);
 	tmp = advert & ~(ADVERTISE_ALL | ADVERTISE_100BASE4);
 	switch (lp->ext_phy_option) {
-
-		default:
-		case SPEED_AUTONEG: /* advertise all values */
-			tmp |= (ADVERTISE_10HALF | ADVERTISE_10FULL |
-				ADVERTISE_100HALF | ADVERTISE_100FULL);
-			break;
-		case SPEED10_HALF:
-			tmp |= ADVERTISE_10HALF;
-			break;
-		case SPEED10_FULL:
-			tmp |= ADVERTISE_10FULL;
-			break;
-		case SPEED100_HALF:
-			tmp |= ADVERTISE_100HALF;
-			break;
-		case SPEED100_FULL:
-			tmp |= ADVERTISE_100FULL;
-			break;
+	default:
+	case SPEED_AUTONEG: /* advertise all values */
+		tmp |= (ADVERTISE_10HALF | ADVERTISE_10FULL |
+			ADVERTISE_100HALF | ADVERTISE_100FULL);
+		break;
+	case SPEED10_HALF:
+		tmp |= ADVERTISE_10HALF;
+		break;
+	case SPEED10_FULL:
+		tmp |= ADVERTISE_10FULL;
+		break;
+	case SPEED100_HALF:
+		tmp |= ADVERTISE_100HALF;
+		break;
+	case SPEED100_FULL:
+		tmp |= ADVERTISE_100FULL;
+		break;
 	}
 
 	if(advert != tmp)
@@ -237,7 +236,7 @@ static int amd8111e_free_skbs(struct net_device *dev)
 	/* Freeing previously allocated receive buffers */
 	for (i = 0; i < NUM_RX_BUFFERS; i++) {
 		rx_skbuff = lp->rx_skbuff[i];
-		if (rx_skbuff != NULL) {
+		if (rx_skbuff) {
 			dma_unmap_single(&lp->pci_dev->dev,
 					 lp->rx_dma_addr[i],
 					 lp->rx_buff_len - 2, DMA_FROM_DEVICE);
@@ -1084,7 +1083,7 @@ static irqreturn_t amd8111e_interrupt(int irq, void *dev_id)
 	unsigned int intr0, intren0;
 	unsigned int handled = 1;
 
-	if (unlikely(dev == NULL))
+	if (unlikely(!dev))
 		return IRQ_NONE;
 
 	spin_lock(&lp->lock);
@@ -1109,7 +1108,7 @@ static irqreturn_t amd8111e_interrupt(int irq, void *dev_id)
 	/* Check if Receive Interrupt has occurred. */
 	if (intr0 & RINT0) {
 		if (napi_schedule_prep(&lp->napi)) {
-			/* Disable receive interupts */
+			/* Disable receive interrupts */
 			writel(RINTEN0, mmio + INTEN0);
 			/* Schedule a polling routine */
 			__napi_schedule(&lp->napi);
@@ -1364,10 +1363,10 @@ static void amd8111e_get_drvinfo(struct net_device *dev,
 {
 	struct amd8111e_priv *lp = netdev_priv(dev);
 	struct pci_dev *pci_dev = lp->pci_dev;
-	strlcpy(info->driver, MODULE_NAME, sizeof(info->driver));
+	strscpy(info->driver, MODULE_NAME, sizeof(info->driver));
 	snprintf(info->fw_version, sizeof(info->fw_version),
 		"%u", chip_version);
-	strlcpy(info->bus_info, pci_name(pci_dev), sizeof(info->bus_info));
+	strscpy(info->bus_info, pci_name(pci_dev), sizeof(info->bus_info));
 }
 
 static int amd8111e_get_regs_len(struct net_device *dev)
@@ -1554,7 +1553,7 @@ static int amd8111e_enable_magicpkt(struct amd8111e_priv *lp)
 static int amd8111e_enable_link_change(struct amd8111e_priv *lp)
 {
 
-	/* Adapter is already stoped/suspended/interrupt-disabled */
+	/* Adapter is already stopped/suspended/interrupt-disabled */
 	writel(VAL0 | LCMODE_SW, lp->mmio + CMD7);
 
 	/* To eliminate PCI posting bug */
diff --git a/drivers/net/ethernet/amd/amd8111e.h b/drivers/net/ethernet/amd/amd8111e.h
index 37da79da5f5e..9d570adb295b 100644
--- a/drivers/net/ethernet/amd/amd8111e.h
+++ b/drivers/net/ethernet/amd/amd8111e.h
@@ -600,7 +600,7 @@ typedef enum {
 #define CSTATE  1
 #define SSTATE  2
 
-/* Assume contoller gets data 10 times the maximum processing time */
+/* Assume controller gets data 10 times the maximum processing time */
 #define  REPEAT_CNT			10
 
 /* amd8111e descriptor flag definitions */
diff --git a/drivers/net/ethernet/amd/ariadne.c b/drivers/net/ethernet/amd/ariadne.c
index 4ea7b9f3c424..38153e633231 100644
--- a/drivers/net/ethernet/amd/ariadne.c
+++ b/drivers/net/ethernet/amd/ariadne.c
@@ -193,7 +193,7 @@ static int ariadne_rx(struct net_device *dev)
 			struct sk_buff *skb;
 
 			skb = netdev_alloc_skb(dev, pkt_len + 2);
-			if (skb == NULL) {
+			if (!skb) {
 				for (i = 0; i < RX_RING_SIZE; i++)
 					if (lowb(priv->rx_ring[(entry + i) % RX_RING_SIZE]->RMD1) & RF_OWN)
 						break;
@@ -731,7 +731,7 @@ static int ariadne_init_one(struct zorro_dev *z,
 	}
 
 	dev = alloc_etherdev(sizeof(struct ariadne_private));
-	if (dev == NULL) {
+	if (!dev) {
 		release_mem_region(base_addr, sizeof(struct Am79C960));
 		release_mem_region(mem_start, ARIADNE_RAM_SIZE);
 		return -ENOMEM;
diff --git a/drivers/net/ethernet/amd/atarilance.c b/drivers/net/ethernet/amd/atarilance.c
index 27869164c6e6..3222c48ce6ae 100644
--- a/drivers/net/ethernet/amd/atarilance.c
+++ b/drivers/net/ethernet/amd/atarilance.c
@@ -581,15 +581,15 @@ static unsigned long __init lance_probe1( struct net_device *dev,
 
 	/* Get the ethernet address */
 	switch( lp->cardtype ) {
-	  case OLD_RIEBL:
+	case OLD_RIEBL:
 		/* No ethernet address! (Set some default address) */
 		eth_hw_addr_set(dev, OldRieblDefHwaddr);
 		break;
-	  case NEW_RIEBL:
+	case NEW_RIEBL:
 		lp->memcpy_f(addr, RIEBL_HWADDR_ADDR, ETH_ALEN);
 		eth_hw_addr_set(dev, addr);
 		break;
-	  case PAM_CARD:
+	case PAM_CARD:
 		i = IO->eeprom;
 		for( i = 0; i < 6; ++i )
 			addr[i] =
@@ -854,7 +854,7 @@ static irqreturn_t lance_interrupt( int irq, void *dev_id )
 	int csr0, boguscnt = 10;
 	int handled = 0;
 
-	if (dev == NULL) {
+	if (!dev) {
 		DPRINTK( 1, ( "lance_interrupt(): interrupt for unknown device.\n" ));
 		return IRQ_NONE;
 	}
@@ -995,7 +995,7 @@ static int lance_rx( struct net_device *dev )
 			}
 			else {
 				skb = netdev_alloc_skb(dev, pkt_len + 2);
-				if (skb == NULL) {
+				if (!skb) {
 					for( i = 0; i < RX_RING_SIZE; i++ )
 						if (MEM->rx_head[(entry+i) & RX_RING_MOD_MASK].flag &
 							RMD1_OWN_CHIP)
diff --git a/drivers/net/ethernet/amd/au1000_eth.c b/drivers/net/ethernet/amd/au1000_eth.c
index d5f2c6989221..c5cec4e79489 100644
--- a/drivers/net/ethernet/amd/au1000_eth.c
+++ b/drivers/net/ethernet/amd/au1000_eth.c
@@ -650,7 +650,7 @@ au1000_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
 	struct au1000_private *aup = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 	snprintf(info->bus_info, sizeof(info->bus_info), "%s %d", DRV_NAME,
 		 aup->mac_id);
 }
@@ -786,7 +786,7 @@ static int au1000_rx(struct net_device *dev)
 			frmlen = (status & RX_FRAME_LEN_MASK);
 			frmlen -= 4; /* Remove FCS */
 			skb = netdev_alloc_skb(dev, frmlen + 2);
-			if (skb == NULL) {
+			if (!skb) {
 				dev->stats.rx_dropped++;
 				continue;
 			}
@@ -1199,7 +1199,7 @@ static int au1000_probe(struct platform_device *pdev)
 	}
 
 	aup->mii_bus = mdiobus_alloc();
-	if (aup->mii_bus == NULL) {
+	if (!aup->mii_bus) {
 		dev_err(&pdev->dev, "failed to allocate mdiobus structure\n");
 		err = -ENOMEM;
 		goto err_mdiobus_alloc;
@@ -1284,7 +1284,7 @@ static int au1000_probe(struct platform_device *pdev)
 	return 0;
 
 err_out:
-	if (aup->mii_bus != NULL)
+	if (aup->mii_bus)
 		mdiobus_unregister(aup->mii_bus);
 
 	/* here we should have a valid dev plus aup-> register addresses
diff --git a/drivers/net/ethernet/amd/lance.c b/drivers/net/ethernet/amd/lance.c
index 462016666752..fb8686214a32 100644
--- a/drivers/net/ethernet/amd/lance.c
+++ b/drivers/net/ethernet/amd/lance.c
@@ -880,7 +880,7 @@ lance_init_ring(struct net_device *dev, gfp_t gfp)
 			rx_buff = skb->data;
 		else
 			rx_buff = kmalloc(PKT_BUF_SZ, GFP_DMA | gfp);
-		if (rx_buff == NULL)
+		if (!rx_buff)
 			lp->rx_ring[i].base = 0;
 		else
 			lp->rx_ring[i].base = (u32)isa_virt_to_bus(rx_buff) | 0x80000000;
@@ -1186,7 +1186,7 @@ lance_rx(struct net_device *dev)
 			else
 			{
 				skb = dev_alloc_skb(pkt_len+2);
-				if (skb == NULL)
+				if (!skb)
 				{
 					printk("%s: Memory squeeze, deferring packet.\n", dev->name);
 					for (i=0; i < RX_RING_SIZE; i++)
diff --git a/drivers/net/ethernet/amd/nmclan_cs.c b/drivers/net/ethernet/amd/nmclan_cs.c
index 30ee5329bd7c..823a329a921f 100644
--- a/drivers/net/ethernet/amd/nmclan_cs.c
+++ b/drivers/net/ethernet/amd/nmclan_cs.c
@@ -485,10 +485,10 @@ static int mace_read(mace_private *lp, unsigned int ioaddr, int reg)
   unsigned long flags;
 
   switch (reg >> 4) {
-    case 0: /* register 0-15 */
+  case 0: /* register 0-15 */
       data = inb(ioaddr + AM2150_MACE_BASE + reg);
       break;
-    case 1: /* register 16-31 */
+  case 1: /* register 16-31 */
       spin_lock_irqsave(&lp->bank_lock, flags);
       MACEBANK(1);
       data = inb(ioaddr + AM2150_MACE_BASE + (reg & 0x0F));
@@ -512,10 +512,10 @@ static void mace_write(mace_private *lp, unsigned int ioaddr, int reg,
   unsigned long flags;
 
   switch (reg >> 4) {
-    case 0: /* register 0-15 */
+  case 0: /* register 0-15 */
       outb(data & 0xFF, ioaddr + AM2150_MACE_BASE + reg);
       break;
-    case 1: /* register 16-31 */
+  case 1: /* register 16-31 */
       spin_lock_irqsave(&lp->bank_lock, flags);
       MACEBANK(1);
       outb(data & 0xFF, ioaddr + AM2150_MACE_BASE + (reg & 0x0F));
@@ -567,13 +567,13 @@ static int mace_init(mace_private *lp, unsigned int ioaddr,
    * Or just set ASEL in PHYCC below!
    */
   switch (if_port) {
-    case 1:
+  case 1:
       mace_write(lp, ioaddr, MACE_PLSCC, 0x02);
       break;
-    case 2:
+  case 2:
       mace_write(lp, ioaddr, MACE_PLSCC, 0x00);
       break;
-    default:
+  default:
       mace_write(lp, ioaddr, MACE_PHYCC, /* ASEL */ 4);
       /* ASEL Auto Select.  When set, the PORTSEL[1-0] bits are overridden,
 	 and the MACE device will automatically select the operating media
@@ -815,7 +815,7 @@ static int mace_close(struct net_device *dev)
 static void netdev_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 	snprintf(info->bus_info, sizeof(info->bus_info),
 		"PCMCIA 0x%lx", dev->base_addr);
 }
@@ -918,7 +918,7 @@ static irqreturn_t mace_interrupt(int irq, void *dev_id)
   int status;
   int IntrCnt = MACE_MAX_IR_ITERATIONS;
 
-  if (dev == NULL) {
+  if (!dev) {
     pr_debug("mace_interrupt(): irq 0x%X for unknown device.\n",
 	  irq);
     return IRQ_NONE;
@@ -1102,7 +1102,7 @@ static int mace_rx(struct net_device *dev, unsigned char RxCnt)
 
       skb = netdev_alloc_skb(dev, pkt_len + 2);
 
-      if (skb != NULL) {
+      if (skb) {
 	skb_reserve(skb, 2);
 	insw(ioaddr + AM2150_RCV, skb_put(skb, pkt_len), pkt_len>>1);
 	if (pkt_len & 1)
diff --git a/drivers/net/ethernet/amd/pcnet32.c b/drivers/net/ethernet/amd/pcnet32.c
index b5ff47283cfe..72db9f9e7bee 100644
--- a/drivers/net/ethernet/amd/pcnet32.c
+++ b/drivers/net/ethernet/amd/pcnet32.c
@@ -488,7 +488,7 @@ static void pcnet32_realloc_tx_ring(struct net_device *dev,
 		dma_alloc_coherent(&lp->pci_dev->dev,
 				   sizeof(struct pcnet32_tx_head) * entries,
 				   &new_ring_dma_addr, GFP_ATOMIC);
-	if (new_tx_ring == NULL)
+	if (!new_tx_ring)
 		return;
 
 	new_dma_addr_list = kcalloc(entries, sizeof(dma_addr_t), GFP_ATOMIC);
@@ -547,7 +547,7 @@ static void pcnet32_realloc_rx_ring(struct net_device *dev,
 		dma_alloc_coherent(&lp->pci_dev->dev,
 				   sizeof(struct pcnet32_rx_head) * entries,
 				   &new_ring_dma_addr, GFP_ATOMIC);
-	if (new_rx_ring == NULL)
+	if (!new_rx_ring)
 		return;
 
 	new_dma_addr_list = kcalloc(entries, sizeof(dma_addr_t), GFP_ATOMIC);
@@ -797,9 +797,9 @@ static void pcnet32_get_drvinfo(struct net_device *dev,
 {
 	struct pcnet32_private *lp = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 	if (lp->pci_dev)
-		strlcpy(info->bus_info, pci_name(lp->pci_dev),
+		strscpy(info->bus_info, pci_name(lp->pci_dev),
 			sizeof(info->bus_info));
 	else
 		snprintf(info->bus_info, sizeof(info->bus_info),
@@ -1249,7 +1249,7 @@ static void pcnet32_rx_entry(struct net_device *dev,
 	} else
 		skb = netdev_alloc_skb(dev, pkt_len + NET_IP_ALIGN);
 
-	if (skb == NULL) {
+	if (!skb) {
 		dev->stats.rx_dropped++;
 		return;
 	}
@@ -2018,7 +2018,7 @@ static int pcnet32_alloc_ring(struct net_device *dev, const char *name)
 	lp->tx_ring = dma_alloc_coherent(&lp->pci_dev->dev,
 					 sizeof(struct pcnet32_tx_head) * lp->tx_ring_size,
 					 &lp->tx_ring_dma_addr, GFP_KERNEL);
-	if (lp->tx_ring == NULL) {
+	if (!lp->tx_ring) {
 		netif_err(lp, drv, dev, "Coherent memory allocation failed\n");
 		return -ENOMEM;
 	}
@@ -2026,7 +2026,7 @@ static int pcnet32_alloc_ring(struct net_device *dev, const char *name)
 	lp->rx_ring = dma_alloc_coherent(&lp->pci_dev->dev,
 					 sizeof(struct pcnet32_rx_head) * lp->rx_ring_size,
 					 &lp->rx_ring_dma_addr, GFP_KERNEL);
-	if (lp->rx_ring == NULL) {
+	if (!lp->rx_ring) {
 		netif_err(lp, drv, dev, "Coherent memory allocation failed\n");
 		return -ENOMEM;
 	}
@@ -2365,7 +2365,7 @@ static int pcnet32_init_ring(struct net_device *dev)
 
 	for (i = 0; i < lp->rx_ring_size; i++) {
 		struct sk_buff *rx_skbuff = lp->rx_skbuff[i];
-		if (rx_skbuff == NULL) {
+		if (!rx_skbuff) {
 			lp->rx_skbuff[i] = netdev_alloc_skb(dev, PKT_BUF_SKB);
 			rx_skbuff = lp->rx_skbuff[i];
 			if (!rx_skbuff) {
diff --git a/drivers/net/ethernet/amd/sun3lance.c b/drivers/net/ethernet/amd/sun3lance.c
index 007bd7787291..246f34c43765 100644
--- a/drivers/net/ethernet/amd/sun3lance.c
+++ b/drivers/net/ethernet/amd/sun3lance.c
@@ -341,7 +341,7 @@ static int __init lance_probe( struct net_device *dev)
 
 	/* XXX - leak? */
 	MEM = dvma_malloc_align(sizeof(struct lance_memory), 0x10000);
-	if (MEM == NULL) {
+	if (!MEM) {
 #ifdef CONFIG_SUN3
 		iounmap((void __iomem *)ioaddr);
 #endif
@@ -796,7 +796,7 @@ static int lance_rx( struct net_device *dev )
 			}
 			else {
 				skb = netdev_alloc_skb(dev, pkt_len + 2);
-				if (skb == NULL) {
+				if (!skb) {
 					dev->stats.rx_dropped++;
 					head->msg_length = 0;
 					head->flag |= RMD1_OWN_CHIP;
diff --git a/drivers/net/ethernet/amd/sunlance.c b/drivers/net/ethernet/amd/sunlance.c
index 22d609563af8..68ca1225eedc 100644
--- a/drivers/net/ethernet/amd/sunlance.c
+++ b/drivers/net/ethernet/amd/sunlance.c
@@ -530,7 +530,7 @@ static void lance_rx_dvma(struct net_device *dev)
 			len = (rd->mblength & 0xfff) - 4;
 			skb = netdev_alloc_skb(dev, len + 2);
 
-			if (skb == NULL) {
+			if (!skb) {
 				dev->stats.rx_dropped++;
 				rd->mblength = 0;
 				rd->rmd1_bits = LE_R1_OWN;
@@ -700,7 +700,7 @@ static void lance_rx_pio(struct net_device *dev)
 			len = (sbus_readw(&rd->mblength) & 0xfff) - 4;
 			skb = netdev_alloc_skb(dev, len + 2);
 
-			if (skb == NULL) {
+			if (!skb) {
 				dev->stats.rx_dropped++;
 				sbus_writew(0, &rd->mblength);
 				sbus_writeb(LE_R1_OWN, &rd->rmd1_bits);
@@ -1276,7 +1276,7 @@ static void lance_free_hwresources(struct lance_private *lp)
 /* Ethtool support... */
 static void sparc_lance_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, "sunlance", sizeof(info->driver));
+	strscpy(info->driver, "sunlance", sizeof(info->driver));
 }
 
 static const struct ethtool_ops sparc_lance_ethtool_ops = {
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
index f342bb853189..7b666106feee 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c
@@ -952,14 +952,14 @@ static void xgbe_napi_enable(struct xgbe_prv_data *pdata, unsigned int add)
 			channel = pdata->channel[i];
 			if (add)
 				netif_napi_add(pdata->netdev, &channel->napi,
-					       xgbe_one_poll, NAPI_POLL_WEIGHT);
+					       xgbe_one_poll);
 
 			napi_enable(&channel->napi);
 		}
 	} else {
 		if (add)
 			netif_napi_add(pdata->netdev, &pdata->napi,
-				       xgbe_all_poll, NAPI_POLL_WEIGHT);
+				       xgbe_all_poll);
 
 		napi_enable(&pdata->napi);
 	}
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
index 6ceb1cdf6eba..6e83ff59172a 100644
--- a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
+++ b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c
@@ -402,8 +402,8 @@ static void xgbe_get_drvinfo(struct net_device *netdev,
 	struct xgbe_prv_data *pdata = netdev_priv(netdev);
 	struct xgbe_hw_features *hw_feat = &pdata->hw_feat;
 
-	strlcpy(drvinfo->driver, XGBE_DRV_NAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, dev_name(pdata->dev),
+	strscpy(drvinfo->driver, XGBE_DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->bus_info, dev_name(pdata->dev),
 		sizeof(drvinfo->bus_info));
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), "%d.%d.%d",
 		 XGMAC_GET_BITS(hw_feat->version, MAC_VR, USERVER),
diff --git a/drivers/net/ethernet/apm/xgene-v2/main.c b/drivers/net/ethernet/apm/xgene-v2/main.c
index d022b6db9e06..379d19d18dbe 100644
--- a/drivers/net/ethernet/apm/xgene-v2/main.c
+++ b/drivers/net/ethernet/apm/xgene-v2/main.c
@@ -672,7 +672,7 @@ static int xge_probe(struct platform_device *pdev)
 	if (ret)
 		goto err;
 
-	netif_napi_add(ndev, &pdata->napi, xge_napi, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &pdata->napi, xge_napi);
 
 	ret = register_netdev(ndev);
 	if (ret) {
diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
index 53dc8d5fede8..d6cfea65a714 100644
--- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
+++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c
@@ -1977,14 +1977,12 @@ static void xgene_enet_napi_add(struct xgene_enet_pdata *pdata)
 
 	for (i = 0; i < pdata->rxq_cnt; i++) {
 		napi = &pdata->rx_ring[i]->napi;
-		netif_napi_add(pdata->ndev, napi, xgene_enet_napi,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(pdata->ndev, napi, xgene_enet_napi);
 	}
 
 	for (i = 0; i < pdata->cq_cnt; i++) {
 		napi = &pdata->tx_ring[i]->cp_ring->napi;
-		netif_napi_add(pdata->ndev, napi, xgene_enet_napi,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(pdata->ndev, napi, xgene_enet_napi);
 	}
 }
 
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c
index 1daecd483b8d..a08f221e30d4 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c
@@ -238,7 +238,7 @@ static void aq_ethtool_get_drvinfo(struct net_device *ndev,
 		 "%u.%u.%u", firmware_version >> 24,
 		 (firmware_version >> 16) & 0xFFU, firmware_version & 0xFFFFU);
 
-	strlcpy(drvinfo->bus_info, pdev ? pci_name(pdev) : "",
+	strscpy(drvinfo->bus_info, pdev ? pci_name(pdev) : "",
 		sizeof(drvinfo->bus_info));
 	drvinfo->n_stats = aq_ethtool_n_stats(ndev);
 	drvinfo->testinfo_len = 0;
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c b/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c
index 02058fe79f52..3d0e16791e1c 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_macsec.c
@@ -292,9 +292,6 @@ static int aq_mdo_dev_open(struct macsec_context *ctx)
 	struct aq_nic_s *nic = netdev_priv(ctx->netdev);
 	int ret = 0;
 
-	if (ctx->prepare)
-		return 0;
-
 	if (netif_carrier_ok(nic->ndev))
 		ret = aq_apply_secy_cfg(nic, ctx->secy);
 
@@ -306,9 +303,6 @@ static int aq_mdo_dev_stop(struct macsec_context *ctx)
 	struct aq_nic_s *nic = netdev_priv(ctx->netdev);
 	int i;
 
-	if (ctx->prepare)
-		return 0;
-
 	for (i = 0; i < AQ_MACSEC_MAX_SC; i++) {
 		if (nic->macsec_cfg->txsc_idx_busy & BIT(i))
 			aq_clear_secy(nic, nic->macsec_cfg->aq_txsc[i].sw_secy,
@@ -466,9 +460,6 @@ static int aq_mdo_add_secy(struct macsec_context *ctx)
 	if (txsc_idx == AQ_MACSEC_MAX_SC)
 		return -ENOSPC;
 
-	if (ctx->prepare)
-		return 0;
-
 	cfg->sc_sa = sc_sa;
 	cfg->aq_txsc[txsc_idx].hw_sc_idx = aq_to_hw_sc_idx(txsc_idx, sc_sa);
 	cfg->aq_txsc[txsc_idx].sw_secy = secy;
@@ -492,9 +483,6 @@ static int aq_mdo_upd_secy(struct macsec_context *ctx)
 	if (txsc_idx < 0)
 		return -ENOENT;
 
-	if (ctx->prepare)
-		return 0;
-
 	if (netif_carrier_ok(nic->ndev) && netif_running(secy->netdev))
 		ret = aq_set_txsc(nic, txsc_idx);
 
@@ -543,9 +531,6 @@ static int aq_mdo_del_secy(struct macsec_context *ctx)
 	struct aq_nic_s *nic = netdev_priv(ctx->netdev);
 	int ret = 0;
 
-	if (ctx->prepare)
-		return 0;
-
 	if (!nic->macsec_cfg)
 		return 0;
 
@@ -601,9 +586,6 @@ static int aq_mdo_add_txsa(struct macsec_context *ctx)
 	if (txsc_idx < 0)
 		return -EINVAL;
 
-	if (ctx->prepare)
-		return 0;
-
 	aq_txsc = &cfg->aq_txsc[txsc_idx];
 	set_bit(ctx->sa.assoc_num, &aq_txsc->tx_sa_idx_busy);
 
@@ -631,9 +613,6 @@ static int aq_mdo_upd_txsa(struct macsec_context *ctx)
 	if (txsc_idx < 0)
 		return -EINVAL;
 
-	if (ctx->prepare)
-		return 0;
-
 	aq_txsc = &cfg->aq_txsc[txsc_idx];
 	if (netif_carrier_ok(nic->ndev) && netif_running(secy->netdev))
 		ret = aq_update_txsa(nic, aq_txsc->hw_sc_idx, secy,
@@ -681,9 +660,6 @@ static int aq_mdo_del_txsa(struct macsec_context *ctx)
 	if (txsc_idx < 0)
 		return -EINVAL;
 
-	if (ctx->prepare)
-		return 0;
-
 	ret = aq_clear_txsa(nic, &cfg->aq_txsc[txsc_idx], ctx->sa.assoc_num,
 			    AQ_CLEAR_ALL);
 
@@ -780,9 +756,6 @@ static int aq_mdo_add_rxsc(struct macsec_context *ctx)
 	if (rxsc_idx >= rxsc_idx_max)
 		return -ENOSPC;
 
-	if (ctx->prepare)
-		return 0;
-
 	cfg->aq_rxsc[rxsc_idx].hw_sc_idx = aq_to_hw_sc_idx(rxsc_idx,
 							   cfg->sc_sa);
 	cfg->aq_rxsc[rxsc_idx].sw_secy = ctx->secy;
@@ -809,9 +782,6 @@ static int aq_mdo_upd_rxsc(struct macsec_context *ctx)
 	if (rxsc_idx < 0)
 		return -ENOENT;
 
-	if (ctx->prepare)
-		return 0;
-
 	if (netif_carrier_ok(nic->ndev) && netif_running(ctx->secy->netdev))
 		ret = aq_set_rxsc(nic, rxsc_idx);
 
@@ -876,9 +846,6 @@ static int aq_mdo_del_rxsc(struct macsec_context *ctx)
 	if (rxsc_idx < 0)
 		return -ENOENT;
 
-	if (ctx->prepare)
-		return 0;
-
 	if (netif_carrier_ok(nic->ndev))
 		clear_type = AQ_CLEAR_ALL;
 
@@ -948,9 +915,6 @@ static int aq_mdo_add_rxsa(struct macsec_context *ctx)
 	if (rxsc_idx < 0)
 		return -EINVAL;
 
-	if (ctx->prepare)
-		return 0;
-
 	aq_rxsc = &nic->macsec_cfg->aq_rxsc[rxsc_idx];
 	set_bit(ctx->sa.assoc_num, &aq_rxsc->rx_sa_idx_busy);
 
@@ -978,9 +942,6 @@ static int aq_mdo_upd_rxsa(struct macsec_context *ctx)
 	if (rxsc_idx < 0)
 		return -EINVAL;
 
-	if (ctx->prepare)
-		return 0;
-
 	if (netif_carrier_ok(nic->ndev) && netif_running(secy->netdev))
 		ret = aq_update_rxsa(nic, cfg->aq_rxsc[rxsc_idx].hw_sc_idx,
 				     secy, ctx->sa.rx_sa, NULL,
@@ -1029,9 +990,6 @@ static int aq_mdo_del_rxsa(struct macsec_context *ctx)
 	if (rxsc_idx < 0)
 		return -EINVAL;
 
-	if (ctx->prepare)
-		return 0;
-
 	ret = aq_clear_rxsa(nic, &cfg->aq_rxsc[rxsc_idx], ctx->sa.assoc_num,
 			    AQ_CLEAR_ALL);
 
@@ -1044,9 +1002,6 @@ static int aq_mdo_get_dev_stats(struct macsec_context *ctx)
 	struct aq_macsec_common_stats *stats = &nic->macsec_cfg->stats;
 	struct aq_hw_s *hw = nic->aq_hw;
 
-	if (ctx->prepare)
-		return 0;
-
 	aq_get_macsec_common_stats(hw, stats);
 
 	ctx->stats.dev_stats->OutPktsUntagged = stats->out.untagged_pkts;
@@ -1073,9 +1028,6 @@ static int aq_mdo_get_tx_sc_stats(struct macsec_context *ctx)
 	if (txsc_idx < 0)
 		return -ENOENT;
 
-	if (ctx->prepare)
-		return 0;
-
 	aq_txsc = &nic->macsec_cfg->aq_txsc[txsc_idx];
 	stats = &aq_txsc->stats;
 	aq_get_txsc_stats(hw, aq_txsc->hw_sc_idx, stats);
@@ -1106,9 +1058,6 @@ static int aq_mdo_get_tx_sa_stats(struct macsec_context *ctx)
 	if (txsc_idx < 0)
 		return -EINVAL;
 
-	if (ctx->prepare)
-		return 0;
-
 	aq_txsc = &cfg->aq_txsc[txsc_idx];
 	sa_idx = aq_txsc->hw_sc_idx | ctx->sa.assoc_num;
 	stats = &aq_txsc->tx_sa_stats[ctx->sa.assoc_num];
@@ -1147,9 +1096,6 @@ static int aq_mdo_get_rx_sc_stats(struct macsec_context *ctx)
 	if (rxsc_idx < 0)
 		return -ENOENT;
 
-	if (ctx->prepare)
-		return 0;
-
 	aq_rxsc = &cfg->aq_rxsc[rxsc_idx];
 	for (i = 0; i < MACSEC_NUM_AN; i++) {
 		if (!test_bit(i, &aq_rxsc->rx_sa_idx_busy))
@@ -1196,9 +1142,6 @@ static int aq_mdo_get_rx_sa_stats(struct macsec_context *ctx)
 	if (rxsc_idx < 0)
 		return -EINVAL;
 
-	if (ctx->prepare)
-		return 0;
-
 	aq_rxsc = &cfg->aq_rxsc[rxsc_idx];
 	stats = &aq_rxsc->rx_sa_stats[ctx->sa.assoc_num];
 	sa_idx = aq_rxsc->hw_sc_idx | ctx->sa.assoc_num;
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ptp.c b/drivers/net/ethernet/aquantia/atlantic/aq_ptp.c
index 275324c9e51e..80b44043e6c5 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_ptp.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_ptp.c
@@ -1217,8 +1217,7 @@ int aq_ptp_init(struct aq_nic_s *aq_nic, unsigned int idx_vec)
 	atomic_set(&aq_ptp->offset_egress, 0);
 	atomic_set(&aq_ptp->offset_ingress, 0);
 
-	netif_napi_add(aq_nic_get_ndev(aq_nic), &aq_ptp->napi,
-		       aq_ptp_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(aq_nic_get_ndev(aq_nic), &aq_ptp->napi, aq_ptp_poll);
 
 	aq_ptp->idx_vector = idx_vec;
 
diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
index f0fdf20f01c1..f5db1c44e9b9 100644
--- a/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
+++ b/drivers/net/ethernet/aquantia/atlantic/aq_vec.c
@@ -119,8 +119,7 @@ struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx,
 	self->tx_rings = 0;
 	self->rx_rings = 0;
 
-	netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi,
-		       aq_vec_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi, aq_vec_poll);
 
 err_exit:
 	return self;
diff --git a/drivers/net/ethernet/arc/emac_main.c b/drivers/net/ethernet/arc/emac_main.c
index 288e2961823e..ba0646b3b122 100644
--- a/drivers/net/ethernet/arc/emac_main.c
+++ b/drivers/net/ethernet/arc/emac_main.c
@@ -91,7 +91,7 @@ static void arc_emac_get_drvinfo(struct net_device *ndev,
 {
 	struct arc_emac_priv *priv = netdev_priv(ndev);
 
-	strlcpy(info->driver, priv->drv_name, sizeof(info->driver));
+	strscpy(info->driver, priv->drv_name, sizeof(info->driver));
 }
 
 static const struct ethtool_ops arc_emac_ethtool_ops = {
diff --git a/drivers/net/ethernet/asix/ax88796c_main.c b/drivers/net/ethernet/asix/ax88796c_main.c
index 6ba5b024a7be..8b7cdf015a16 100644
--- a/drivers/net/ethernet/asix/ax88796c_main.c
+++ b/drivers/net/ethernet/asix/ax88796c_main.c
@@ -293,7 +293,7 @@ ax88796c_tx_fixup(struct net_device *ndev, struct sk_buff_head *q)
 	skb_put(skb, padlen);
 
 	/* EOP header */
-	memcpy(skb_put(skb, TX_EOP_SIZE), &info.eop, TX_EOP_SIZE);
+	skb_put_data(skb, &info.eop, TX_EOP_SIZE);
 
 	skb_unlink(skb, q);
 
@@ -381,7 +381,7 @@ static int ax88796c_hard_xmit(struct ax88796c_device *ax_local)
 	return 1;
 }
 
-static int
+static netdev_tx_t
 ax88796c_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 {
 	struct ax88796c_device *ax_local = to_ax88796c_device(ndev);
diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c
index e461f4764066..cc932b3cf873 100644
--- a/drivers/net/ethernet/atheros/ag71xx.c
+++ b/drivers/net/ethernet/atheros/ag71xx.c
@@ -451,8 +451,8 @@ static void ag71xx_get_drvinfo(struct net_device *ndev,
 {
 	struct ag71xx *ag = netdev_priv(ndev);
 
-	strlcpy(info->driver, "ag71xx", sizeof(info->driver));
-	strlcpy(info->bus_info, of_node_full_name(ag->pdev->dev.of_node),
+	strscpy(info->driver, "ag71xx", sizeof(info->driver));
+	strscpy(info->bus_info, of_node_full_name(ag->pdev->dev.of_node),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/atheros/alx/main.c b/drivers/net/ethernet/atheros/alx/main.c
index a89b93cb4e26..d30d11872719 100644
--- a/drivers/net/ethernet/atheros/alx/main.c
+++ b/drivers/net/ethernet/atheros/alx/main.c
@@ -752,7 +752,7 @@ static int alx_alloc_napis(struct alx_priv *alx)
 			goto err_out;
 
 		np->alx = alx;
-		netif_napi_add(alx->dev, &np->napi, alx_poll, 64);
+		netif_napi_add(alx->dev, &np->napi, alx_poll);
 		alx->qnapi[i] = np;
 	}
 
@@ -1912,11 +1912,14 @@ static int alx_suspend(struct device *dev)
 
 	if (!netif_running(alx->dev))
 		return 0;
+
+	rtnl_lock();
 	netif_device_detach(alx->dev);
 
 	mutex_lock(&alx->mtx);
 	__alx_stop(alx);
 	mutex_unlock(&alx->mtx);
+	rtnl_unlock();
 
 	return 0;
 }
@@ -1927,6 +1930,7 @@ static int alx_resume(struct device *dev)
 	struct alx_hw *hw = &alx->hw;
 	int err;
 
+	rtnl_lock();
 	mutex_lock(&alx->mtx);
 	alx_reset_phy(hw);
 
@@ -1943,6 +1947,7 @@ static int alx_resume(struct device *dev)
 
 unlock:
 	mutex_unlock(&alx->mtx);
+	rtnl_unlock();
 	return err;
 }
 
diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_ethtool.c b/drivers/net/ethernet/atheros/atl1c/atl1c_ethtool.c
index e2eb7b8c63a0..0bce122c68f1 100644
--- a/drivers/net/ethernet/atheros/atl1c/atl1c_ethtool.c
+++ b/drivers/net/ethernet/atheros/atl1c/atl1c_ethtool.c
@@ -220,8 +220,8 @@ static void atl1c_get_drvinfo(struct net_device *netdev,
 {
 	struct atl1c_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver,  atl1c_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->driver,  atl1c_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
index be4b1f8eef29..40c781695d58 100644
--- a/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
+++ b/drivers/net/ethernet/atheros/atl1c/atl1c_main.c
@@ -2732,7 +2732,7 @@ static int atl1c_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	dev_set_threaded(netdev, true);
 	for (i = 0; i < adapter->rx_queue_count; ++i)
 		netif_napi_add(netdev, &adapter->rrd_ring[i].napi,
-			       atl1c_clean_rx, 64);
+			       atl1c_clean_rx);
 	for (i = 0; i < adapter->tx_queue_count; ++i)
 		netif_napi_add_tx(netdev, &adapter->tpd_ring[i].napi,
 				  atl1c_clean_tx);
diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_ethtool.c b/drivers/net/ethernet/atheros/atl1e/atl1e_ethtool.c
index 0cbde352d1ba..68f1832a198d 100644
--- a/drivers/net/ethernet/atheros/atl1e/atl1e_ethtool.c
+++ b/drivers/net/ethernet/atheros/atl1e/atl1e_ethtool.c
@@ -306,9 +306,9 @@ static void atl1e_get_drvinfo(struct net_device *netdev,
 {
 	struct atl1e_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver,  atl1e_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->fw_version, "L1e", sizeof(drvinfo->fw_version));
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->driver,  atl1e_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->fw_version, "L1e", sizeof(drvinfo->fw_version));
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c
index 57a51fb7746c..5db0f3495a32 100644
--- a/drivers/net/ethernet/atheros/atl1e/atl1e_main.c
+++ b/drivers/net/ethernet/atheros/atl1e/atl1e_main.c
@@ -2354,7 +2354,7 @@ static int atl1e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	adapter->mii.phy_id_mask = 0x1f;
 	adapter->mii.reg_num_mask = MDIO_REG_ADDR_MASK;
 
-	netif_napi_add(netdev, &adapter->napi, atl1e_clean, 64);
+	netif_napi_add(netdev, &adapter->napi, atl1e_clean);
 
 	timer_setup(&adapter->phy_config_timer, atl1e_phy_config, 0);
 
diff --git a/drivers/net/ethernet/atheros/atlx/atl1.c b/drivers/net/ethernet/atheros/atlx/atl1.c
index ff1fe09abf9f..c8444bcdf527 100644
--- a/drivers/net/ethernet/atheros/atlx/atl1.c
+++ b/drivers/net/ethernet/atheros/atlx/atl1.c
@@ -2977,7 +2977,7 @@ static int atl1_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	netdev->netdev_ops = &atl1_netdev_ops;
 	netdev->watchdog_timeo = 5 * HZ;
-	netif_napi_add(netdev, &adapter->napi, atl1_rings_clean, 64);
+	netif_napi_add(netdev, &adapter->napi, atl1_rings_clean);
 
 	netdev->ethtool_ops = &atl1_ethtool_ops;
 	adapter->bd_number = cards_found;
@@ -3340,8 +3340,8 @@ static void atl1_get_drvinfo(struct net_device *netdev,
 {
 	struct atl1_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, ATLX_DRIVER_NAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->driver, ATLX_DRIVER_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/atheros/atlx/atl2.c b/drivers/net/ethernet/atheros/atlx/atl2.c
index bbc4d7b08a49..1b487c071cb6 100644
--- a/drivers/net/ethernet/atheros/atlx/atl2.c
+++ b/drivers/net/ethernet/atheros/atlx/atl2.c
@@ -1980,9 +1980,9 @@ static void atl2_get_drvinfo(struct net_device *netdev,
 {
 	struct atl2_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver,  atl2_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->fw_version, "L2", sizeof(drvinfo->fw_version));
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->driver,  atl2_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->fw_version, "L2", sizeof(drvinfo->fw_version));
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig
index 56e0fb07aec7..f4e1ca68d831 100644
--- a/drivers/net/ethernet/broadcom/Kconfig
+++ b/drivers/net/ethernet/broadcom/Kconfig
@@ -53,8 +53,8 @@ config B44_PCI
 
 config BCM4908_ENET
 	tristate "Broadcom BCM4908 internal mac support"
-	depends on ARCH_BCM4908 || COMPILE_TEST
-	default y if ARCH_BCM4908
+	depends on ARCH_BCMBCA || COMPILE_TEST
+	default y if ARCH_BCMBCA
 	help
 	  This driver supports Ethernet controller integrated into Broadcom
 	  BCM4908 family SoCs.
diff --git a/drivers/net/ethernet/broadcom/b44.c b/drivers/net/ethernet/broadcom/b44.c
index e5857e88c207..7f876721596c 100644
--- a/drivers/net/ethernet/broadcom/b44.c
+++ b/drivers/net/ethernet/broadcom/b44.c
@@ -1790,13 +1790,13 @@ static void b44_get_drvinfo (struct net_device *dev, struct ethtool_drvinfo *inf
 	struct b44 *bp = netdev_priv(dev);
 	struct ssb_bus *bus = bp->sdev->bus;
 
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
 	switch (bus->bustype) {
 	case SSB_BUSTYPE_PCI:
-		strlcpy(info->bus_info, pci_name(bus->host_pci), sizeof(info->bus_info));
+		strscpy(info->bus_info, pci_name(bus->host_pci), sizeof(info->bus_info));
 		break;
 	case SSB_BUSTYPE_SSB:
-		strlcpy(info->bus_info, "SSB", sizeof(info->bus_info));
+		strscpy(info->bus_info, "SSB", sizeof(info->bus_info));
 		break;
 	case SSB_BUSTYPE_PCMCIA:
 	case SSB_BUSTYPE_SDIO:
@@ -2375,7 +2375,7 @@ static int b44_init_one(struct ssb_device *sdev,
 	bp->tx_pending = B44_DEF_TX_RING_PENDING;
 
 	dev->netdev_ops = &b44_netdev_ops;
-	netif_napi_add(dev, &bp->napi, b44_poll, 64);
+	netif_napi_add(dev, &bp->napi, b44_poll);
 	dev->watchdog_timeo = B44_TX_TIMEOUT;
 	dev->min_mtu = B44_MIN_MTU;
 	dev->max_mtu = B44_MAX_MTU;
diff --git a/drivers/net/ethernet/broadcom/bcm4908_enet.c b/drivers/net/ethernet/broadcom/bcm4908_enet.c
index c131d8118489..93ccf549e2ed 100644
--- a/drivers/net/ethernet/broadcom/bcm4908_enet.c
+++ b/drivers/net/ethernet/broadcom/bcm4908_enet.c
@@ -507,7 +507,7 @@ static int bcm4908_enet_stop(struct net_device *netdev)
 	return 0;
 }
 
-static int bcm4908_enet_start_xmit(struct sk_buff *skb, struct net_device *netdev)
+static netdev_tx_t bcm4908_enet_start_xmit(struct sk_buff *skb, struct net_device *netdev)
 {
 	struct bcm4908_enet *enet = netdev_priv(netdev);
 	struct bcm4908_enet_dma_ring *ring = &enet->tx_ring;
@@ -716,6 +716,8 @@ static int bcm4908_enet_probe(struct platform_device *pdev)
 
 	SET_NETDEV_DEV(netdev, &pdev->dev);
 	err = of_get_ethdev_address(dev->of_node, netdev);
+	if (err == -EPROBE_DEFER)
+		goto err_dma_free;
 	if (err)
 		eth_hw_addr_random(netdev);
 	netdev->netdev_ops = &bcm4908_enet_netdev_ops;
@@ -723,17 +725,20 @@ static int bcm4908_enet_probe(struct platform_device *pdev)
 	netdev->mtu = ETH_DATA_LEN;
 	netdev->max_mtu = ENET_MTU_MAX;
 	netif_napi_add_tx(netdev, &enet->tx_ring.napi, bcm4908_enet_poll_tx);
-	netif_napi_add(netdev, &enet->rx_ring.napi, bcm4908_enet_poll_rx, NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, &enet->rx_ring.napi, bcm4908_enet_poll_rx);
 
 	err = register_netdev(netdev);
-	if (err) {
-		bcm4908_enet_dma_free(enet);
-		return err;
-	}
+	if (err)
+		goto err_dma_free;
 
 	platform_set_drvdata(pdev, enet);
 
 	return 0;
+
+err_dma_free:
+	bcm4908_enet_dma_free(enet);
+
+	return err;
 }
 
 static int bcm4908_enet_remove(struct platform_device *pdev)
diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
index 1c6aea12db72..d91fdb0c2649 100644
--- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c
+++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c
@@ -1321,8 +1321,8 @@ static const u32 unused_mib_regs[] = {
 static void bcm_enet_get_drvinfo(struct net_device *netdev,
 				 struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, bcm_enet_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, "bcm63xx", sizeof(drvinfo->bus_info));
+	strscpy(drvinfo->driver, bcm_enet_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->bus_info, "bcm63xx", sizeof(drvinfo->bus_info));
 }
 
 static int bcm_enet_get_sset_count(struct net_device *netdev,
diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c
index 47fc8e6963d5..867f14c30e09 100644
--- a/drivers/net/ethernet/broadcom/bcmsysport.c
+++ b/drivers/net/ethernet/broadcom/bcmsysport.c
@@ -308,8 +308,8 @@ static const struct bcm_sysport_stats bcm_sysport_gstrings_stats[] = {
 static void bcm_sysport_get_drvinfo(struct net_device *dev,
 				    struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
-	strlcpy(info->bus_info, "platform", sizeof(info->bus_info));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->bus_info, "platform", sizeof(info->bus_info));
 }
 
 static u32 bcm_sysport_get_msglvl(struct net_device *dev)
@@ -2564,7 +2564,7 @@ static int bcm_sysport_probe(struct platform_device *pdev)
 	dev_set_drvdata(&pdev->dev, dev);
 	dev->ethtool_ops = &bcm_sysport_ethtool_ops;
 	dev->netdev_ops = &bcm_sysport_netdev_ops;
-	netif_napi_add(dev, &priv->napi, bcm_sysport_poll, 64);
+	netif_napi_add(dev, &priv->napi, bcm_sysport_poll);
 
 	dev->features |= NETIF_F_RXCSUM | NETIF_F_HIGHDMA |
 			 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
diff --git a/drivers/net/ethernet/broadcom/bgmac.c b/drivers/net/ethernet/broadcom/bgmac.c
index 93580484a3f4..5fb3af5670ec 100644
--- a/drivers/net/ethernet/broadcom/bgmac.c
+++ b/drivers/net/ethernet/broadcom/bgmac.c
@@ -1367,7 +1367,7 @@ static void bgmac_get_strings(struct net_device *dev, u32 stringset,
 		return;
 
 	for (i = 0; i < BGMAC_STATS_LEN; i++)
-		strlcpy(data + i * ETH_GSTRING_LEN,
+		strscpy(data + i * ETH_GSTRING_LEN,
 			bgmac_get_strings_stats[i].name, ETH_GSTRING_LEN);
 }
 
@@ -1395,8 +1395,8 @@ static void bgmac_get_ethtool_stats(struct net_device *dev,
 static void bgmac_get_drvinfo(struct net_device *net_dev,
 			      struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
-	strlcpy(info->bus_info, "AXI", sizeof(info->bus_info));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->bus_info, "AXI", sizeof(info->bus_info));
 }
 
 static const struct ethtool_ops bgmac_ethtool_ops = {
@@ -1527,7 +1527,7 @@ int bgmac_enet_probe(struct bgmac *bgmac)
 	if (bcm47xx_nvram_getenv("et0_no_txint", NULL, 0) == 0)
 		bgmac->int_mask &= ~BGMAC_IS_TX_MASK;
 
-	netif_napi_add(net_dev, &bgmac->napi, bgmac_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(net_dev, &bgmac->napi, bgmac_poll);
 
 	err = bgmac_phy_connect(bgmac);
 	if (err) {
diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c
index b97ed9b5f685..fec57f1982c8 100644
--- a/drivers/net/ethernet/broadcom/bnx2.c
+++ b/drivers/net/ethernet/broadcom/bnx2.c
@@ -176,12 +176,12 @@ static const struct flash_spec flash_table[] =
 	{0x19000002, 0x5b808201, 0x000500db, 0x03840253, 0xaf020406,
 	 NONBUFFERED_FLAGS, ST_MICRO_FLASH_PAGE_BITS, ST_MICRO_FLASH_PAGE_SIZE,
 	 ST_MICRO_FLASH_BYTE_ADDR_MASK, ST_MICRO_FLASH_BASE_TOTAL_SIZE*2,
-	 "Entry 0101: ST M45PE10 (128kB non-bufferred)"},
+	 "Entry 0101: ST M45PE10 (128kB non-buffered)"},
 	/* Entry 0110: ST M45PE20 (non-buffered flash)*/
 	{0x15000001, 0x57808201, 0x000500db, 0x03840253, 0xaf020406,
 	 NONBUFFERED_FLAGS, ST_MICRO_FLASH_PAGE_BITS, ST_MICRO_FLASH_PAGE_SIZE,
 	 ST_MICRO_FLASH_BYTE_ADDR_MASK, ST_MICRO_FLASH_BASE_TOTAL_SIZE*4,
-	 "Entry 0110: ST M45PE20 (256kB non-bufferred)"},
+	 "Entry 0110: ST M45PE20 (256kB non-buffered)"},
 	/* Saifun SA25F005 (non-buffered flash) */
 	/* strap, cfg1, & write1 need updates */
 	{0x1d000003, 0x5f808201, 0x00050081, 0x03840253, 0xaf020406,
@@ -7042,9 +7042,9 @@ bnx2_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
 	struct bnx2 *bp = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info));
-	strlcpy(info->fw_version, bp->fw_version, sizeof(info->fw_version));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info));
+	strscpy(info->fw_version, bp->fw_version, sizeof(info->fw_version));
 }
 
 #define BNX2_REGDUMP_LEN		(32 * 1024)
@@ -8522,7 +8522,7 @@ bnx2_init_napi(struct bnx2 *bp)
 		else
 			poll = bnx2_poll_msix;
 
-		netif_napi_add(bp->dev, &bp->bnx2_napi[i].napi, poll, 64);
+		netif_napi_add(bp->dev, &bp->bnx2_napi[i].napi, poll);
 		bnapi->bp = bp;
 	}
 }
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
index 712b5595bc39..16c490692f42 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c
@@ -44,8 +44,7 @@ static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
 
 	/* Add NAPI objects */
 	for_each_rx_queue_cnic(bp, i) {
-		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
-			       bnx2x_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), bnx2x_poll);
 	}
 }
 
@@ -55,8 +54,7 @@ static void bnx2x_add_all_napi(struct bnx2x *bp)
 
 	/* Add NAPI objects */
 	for_each_eth_queue(bp, i) {
-		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
-			       bnx2x_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi), bnx2x_poll);
 	}
 }
 
@@ -150,7 +148,7 @@ void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
 		phy_fw_ver[0] = '\0';
 		bnx2x_get_ext_phy_fw_version(&bp->link_params,
 					     phy_fw_ver, PHY_FW_VER_LEN);
-		strlcpy(buf, bp->fw_ver, buf_len);
+		strscpy(buf, bp->fw_ver, buf_len);
 		snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
 			 "bc %d.%d.%d%s%s",
 			 (bp->common.bc_ver & 0xff0000) >> 16,
@@ -789,6 +787,7 @@ static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
 			BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
 				  pad, len, fp->rx_buf_size);
 			bnx2x_panic();
+			bnx2x_frag_free(fp, new_data);
 			return;
 		}
 #endif
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c
index 0e319ac7799f..bda3ccc28eca 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c
@@ -1112,7 +1112,7 @@ static void bnx2x_get_drvinfo(struct net_device *dev,
 	int ext_dev_info_offset;
 	u32 mbi;
 
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
 
 	if (SHMEM2_HAS(bp, extended_dev_info_shared_addr)) {
 		ext_dev_info_offset = SHMEM2_RD(bp,
@@ -1126,7 +1126,7 @@ static void bnx2x_get_drvinfo(struct net_device *dev,
 				 (mbi & 0xff000000) >> 24,
 				 (mbi & 0x00ff0000) >> 16,
 				 (mbi & 0x0000ff00) >> 8);
-			strlcpy(info->fw_version, version,
+			strscpy(info->fw_version, version,
 				sizeof(info->fw_version));
 		}
 	}
@@ -1135,7 +1135,7 @@ static void bnx2x_get_drvinfo(struct net_device *dev,
 	bnx2x_fill_fw_str(bp, version, ETHTOOL_FWVERS_LEN);
 	strlcat(info->fw_version, version, sizeof(info->fw_version));
 
-	strlcpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info));
+	strscpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info));
 }
 
 static void bnx2x_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
index 962253db25b8..51b1690fd045 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
@@ -3385,7 +3385,7 @@ static void bnx2x_drv_info_ether_stat(struct bnx2x *bp)
 		&bp->sp_objs->mac_obj;
 	int i;
 
-	strlcpy(ether_stat->version, DRV_MODULE_VERSION,
+	strscpy(ether_stat->version, DRV_MODULE_VERSION,
 		ETH_STAT_INFO_VERSION_LEN);
 
 	/* get DRV_INFO_ETH_STAT_NUM_MACS_REQUIRED macs, placing them in the
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h
index 2dac704dc346..02a4e557e176 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h
@@ -518,7 +518,7 @@ int bnx2x_vfpf_storm_rx_mode(struct bnx2x *bp);
 static inline void bnx2x_vf_fill_fw_str(struct bnx2x *bp, char *buf,
 					size_t buf_len)
 {
-	strlcpy(buf, bp->acquire_resp.pfdev_info.fw_ver, buf_len);
+	strscpy(buf, bp->acquire_resp.pfdev_info.fw_ver, buf_len);
 }
 
 static inline int bnx2x_vf_ustorm_prods_offset(struct bnx2x *bp,
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_vfpf.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_vfpf.c
index c9129b9ba446..0657a0f5170f 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_vfpf.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_vfpf.c
@@ -380,7 +380,7 @@ int bnx2x_vfpf_acquire(struct bnx2x *bp, u8 tx_count, u8 rx_count)
 	bp->igu_base_sb = bp->acquire_resp.resc.hw_sbs[0].hw_sb_id;
 	bp->vlan_credit = bp->acquire_resp.resc.num_vlan_filters;
 
-	strlcpy(bp->fw_ver, bp->acquire_resp.pfdev_info.fw_ver,
+	strscpy(bp->fw_ver, bp->acquire_resp.pfdev_info.fw_ver,
 		sizeof(bp->fw_ver));
 
 	if (is_valid_ether_addr(bp->acquire_resp.resc.current_mac_addr))
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index 96da0ba3d507..eed98c10ca9d 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -9366,16 +9366,16 @@ static void bnxt_init_napi(struct bnxt *bp)
 			cp_nr_rings--;
 		for (i = 0; i < cp_nr_rings; i++) {
 			bnapi = bp->bnapi[i];
-			netif_napi_add(bp->dev, &bnapi->napi, poll_fn, 64);
+			netif_napi_add(bp->dev, &bnapi->napi, poll_fn);
 		}
 		if (BNXT_CHIP_TYPE_NITRO_A0(bp)) {
 			bnapi = bp->bnapi[cp_nr_rings];
 			netif_napi_add(bp->dev, &bnapi->napi,
-				       bnxt_poll_nitroa0, 64);
+				       bnxt_poll_nitroa0);
 		}
 	} else {
 		bnapi = bp->bnapi[0];
-		netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll, 64);
+		netif_napi_add(bp->dev, &bnapi->napi, bnxt_poll);
 	}
 }
 
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
index 87eb5362ad70..f57e524c7e30 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c
@@ -1371,9 +1371,9 @@ static void bnxt_get_drvinfo(struct net_device *dev,
 {
 	struct bnxt *bp = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
-	strlcpy(info->fw_version, bp->fw_ver_str, sizeof(info->fw_version));
-	strlcpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->fw_version, bp->fw_ver_str, sizeof(info->fw_version));
+	strscpy(info->bus_info, pci_name(bp->pdev), sizeof(info->bus_info));
 	info->n_stats = bnxt_get_num_stats(bp);
 	info->testinfo_len = bp->num_tests;
 	/* TODO CHIMP_FW: eeprom dump details */
@@ -3876,7 +3876,7 @@ void bnxt_ethtool_init(struct bnxt *bp)
 		} else if (i == BNXT_IRQ_TEST_IDX) {
 			strcpy(str, "Interrupt_test (offline)");
 		} else {
-			strlcpy(str, fw_str, ETH_GSTRING_LEN);
+			strscpy(str, fw_str, ETH_GSTRING_LEN);
 			strncat(str, " test", ETH_GSTRING_LEN - strlen(str));
 			if (test_info->offline_mask & (1 << i))
 				strncat(str, " (offline)",
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
index 8e316367f6ce..2132ce63193c 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ptp.c
@@ -505,9 +505,13 @@ static int bnxt_hwrm_ptp_cfg(struct bnxt *bp)
 	ptp->tstamp_filters = flags;
 
 	if (netif_running(bp->dev)) {
-		rc = bnxt_close_nic(bp, false, false);
-		if (!rc)
-			rc = bnxt_open_nic(bp, false, false);
+		if (ptp->rx_filter == HWTSTAMP_FILTER_ALL) {
+			rc = bnxt_close_nic(bp, false, false);
+			if (!rc)
+				rc = bnxt_open_nic(bp, false, false);
+		} else {
+			bnxt_ptp_cfg_tstamp_filters(bp);
+		}
 		if (!rc && !ptp->tstamp_filters)
 			rc = -EIO;
 	}
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
index eb4803b11c0e..fcc65890820a 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c
@@ -222,7 +222,7 @@ static int bnxt_vf_rep_get_phys_port_name(struct net_device *dev, char *buf,
 static void bnxt_vf_rep_get_drvinfo(struct net_device *dev,
 				    struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
 }
 
 static int bnxt_vf_rep_get_port_parent_id(struct net_device *dev,
diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
index 8309fb993cdb..25c450606985 100644
--- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c
+++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c
@@ -1146,7 +1146,7 @@ static const struct bcmgenet_stats bcmgenet_gstrings_stats[] = {
 static void bcmgenet_get_drvinfo(struct net_device *dev,
 				 struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, "bcmgenet", sizeof(info->driver));
+	strscpy(info->driver, "bcmgenet", sizeof(info->driver));
 }
 
 static int bcmgenet_get_sset_count(struct net_device *dev, int string_set)
@@ -2707,8 +2707,7 @@ static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv,
 	bcmgenet_init_rx_coalesce(ring);
 
 	/* Initialize Rx NAPI */
-	netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(priv->dev, &ring->napi, bcmgenet_rx_poll);
 
 	bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_PROD_INDEX);
 	bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX);
diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c
index 89889d8150da..4179a12fc881 100644
--- a/drivers/net/ethernet/broadcom/tg3.c
+++ b/drivers/net/ethernet/broadcom/tg3.c
@@ -7380,9 +7380,9 @@ static void tg3_napi_init(struct tg3 *tp)
 {
 	int i;
 
-	netif_napi_add(tp->dev, &tp->napi[0].napi, tg3_poll, 64);
+	netif_napi_add(tp->dev, &tp->napi[0].napi, tg3_poll);
 	for (i = 1; i < tp->irq_cnt; i++)
-		netif_napi_add(tp->dev, &tp->napi[i].napi, tg3_poll_msix, 64);
+		netif_napi_add(tp->dev, &tp->napi[i].napi, tg3_poll_msix);
 }
 
 static void tg3_napi_fini(struct tg3 *tp)
@@ -12302,9 +12302,9 @@ static void tg3_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info
 {
 	struct tg3 *tp = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
-	strlcpy(info->fw_version, tp->fw_ver, sizeof(info->fw_version));
-	strlcpy(info->bus_info, pci_name(tp->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->fw_version, tp->fw_ver, sizeof(info->fw_version));
+	strscpy(info->bus_info, pci_name(tp->pdev), sizeof(info->bus_info));
 }
 
 static void tg3_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
diff --git a/drivers/net/ethernet/brocade/bna/bnad.c b/drivers/net/ethernet/brocade/bna/bnad.c
index 29dd0f93d6c0..d6d90f9722a7 100644
--- a/drivers/net/ethernet/brocade/bna/bnad.c
+++ b/drivers/net/ethernet/brocade/bna/bnad.c
@@ -1891,7 +1891,7 @@ bnad_napi_add(struct bnad *bnad, u32 rx_id)
 	for (i = 0; i <	bnad->num_rxp_per_rx; i++) {
 		rx_ctrl = &bnad->rx_info[rx_id].rx_ctrl[i];
 		netif_napi_add(bnad->netdev, &rx_ctrl->napi,
-			       bnad_napi_poll_rx, NAPI_POLL_WEIGHT);
+			       bnad_napi_poll_rx);
 	}
 }
 
diff --git a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c
index 8aca768571b2..df10edff5603 100644
--- a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c
+++ b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c
@@ -114,7 +114,7 @@ static const char *bnad_net_stats_strings[] = {
 	"mac_tx_deferral",
 	"mac_tx_excessive_deferral",
 	"mac_tx_single_collision",
-	"mac_tx_muliple_collision",
+	"mac_tx_multiple_collision",
 	"mac_tx_late_collision",
 	"mac_tx_excessive_collision",
 	"mac_tx_total_collision",
@@ -283,7 +283,7 @@ bnad_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
 	struct bfa_ioc_attr *ioc_attr;
 	unsigned long flags;
 
-	strlcpy(drvinfo->driver, BNAD_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->driver, BNAD_NAME, sizeof(drvinfo->driver));
 
 	ioc_attr = kzalloc(sizeof(*ioc_attr), GFP_KERNEL);
 	if (ioc_attr) {
@@ -291,12 +291,12 @@ bnad_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
 		bfa_nw_ioc_get_attr(&bnad->bna.ioceth.ioc, ioc_attr);
 		spin_unlock_irqrestore(&bnad->bna_lock, flags);
 
-		strlcpy(drvinfo->fw_version, ioc_attr->adapter_attr.fw_ver,
+		strscpy(drvinfo->fw_version, ioc_attr->adapter_attr.fw_ver,
 			sizeof(drvinfo->fw_version));
 		kfree(ioc_attr);
 	}
 
-	strlcpy(drvinfo->bus_info, pci_name(bnad->pcidev),
+	strscpy(drvinfo->bus_info, pci_name(bnad->pcidev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c
index 66c7d08d376a..51c9fd6f68a4 100644
--- a/drivers/net/ethernet/cadence/macb_main.c
+++ b/drivers/net/ethernet/cadence/macb_main.c
@@ -38,6 +38,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/ptp_classify.h>
 #include <linux/reset.h>
+#include <linux/firmware/xlnx-zynqmp.h>
 #include "macb.h"
 
 /* This structure is only used for MACB on SiFive FU540 devices */
@@ -3977,8 +3978,8 @@ static int macb_init(struct platform_device *pdev)
 		queue = &bp->queues[q];
 		queue->bp = bp;
 		spin_lock_init(&queue->tx_ptr_lock);
-		netif_napi_add(dev, &queue->napi_rx, macb_rx_poll, NAPI_POLL_WEIGHT);
-		netif_napi_add(dev, &queue->napi_tx, macb_tx_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(dev, &queue->napi_rx, macb_rx_poll);
+		netif_napi_add(dev, &queue->napi_tx, macb_tx_poll);
 		if (hw_q) {
 			queue->ISR  = GEM_ISR(hw_q - 1);
 			queue->IER  = GEM_IER(hw_q - 1);
@@ -4621,6 +4622,25 @@ static int init_reset_optional(struct platform_device *pdev)
 					     "failed to init SGMII PHY\n");
 	}
 
+	ret = zynqmp_pm_is_function_supported(PM_IOCTL, IOCTL_SET_GEM_CONFIG);
+	if (!ret) {
+		u32 pm_info[2];
+
+		ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains",
+						 pm_info, ARRAY_SIZE(pm_info));
+		if (ret) {
+			dev_err(&pdev->dev, "Failed to read power management information\n");
+			goto err_out_phy_exit;
+		}
+		ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_FIXED, 0);
+		if (ret)
+			goto err_out_phy_exit;
+
+		ret = zynqmp_pm_set_gem_config(pm_info[1], GEM_CONFIG_SGMII_MODE, 1);
+		if (ret)
+			goto err_out_phy_exit;
+	}
+
 	/* Fully reset controller at hardware level if mapped in device tree */
 	ret = device_reset_optional(&pdev->dev);
 	if (ret) {
@@ -4629,6 +4649,8 @@ static int init_reset_optional(struct platform_device *pdev)
 	}
 
 	ret = macb_init(pdev);
+
+err_out_phy_exit:
 	if (ret)
 		phy_exit(bp->sgmii_phy);
 
@@ -5109,6 +5131,7 @@ static int __maybe_unused macb_suspend(struct device *dev)
 	if (!(bp->wol & MACB_WOL_ENABLED)) {
 		rtnl_lock();
 		phylink_stop(bp->phylink);
+		phy_exit(bp->sgmii_phy);
 		rtnl_unlock();
 		spin_lock_irqsave(&bp->lock, flags);
 		macb_reset_hw(bp);
@@ -5198,6 +5221,9 @@ static int __maybe_unused macb_resume(struct device *dev)
 	macb_set_rx_mode(netdev);
 	macb_restore_features(bp);
 	rtnl_lock();
+	if (!device_may_wakeup(&bp->dev->dev))
+		phy_init(bp->sgmii_phy);
+
 	phylink_start(bp->phylink);
 	rtnl_unlock();
 
diff --git a/drivers/net/ethernet/calxeda/xgmac.c b/drivers/net/ethernet/calxeda/xgmac.c
index 1281d1565ef8..f4f87dfa9687 100644
--- a/drivers/net/ethernet/calxeda/xgmac.c
+++ b/drivers/net/ethernet/calxeda/xgmac.c
@@ -1792,7 +1792,7 @@ static int xgmac_probe(struct platform_device *pdev)
 		netdev_warn(ndev, "MAC address %pM not valid",
 			 ndev->dev_addr);
 
-	netif_napi_add(ndev, &priv->napi, xgmac_poll, 64);
+	netif_napi_add(ndev, &priv->napi, xgmac_poll);
 	ret = register_netdev(ndev);
 	if (ret)
 		goto err_reg;
diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h
index 3f1c189646f4..a0fd32476225 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h
+++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_pf_regs.h
@@ -87,8 +87,8 @@
  */
 #define    CN23XX_SLI_PKT_IN_JABBER                0x29170
 /* The input jabber is used to determine the TSO max size.
- * Due to H/W limitation, this need to be reduced to 60000
- * in order to to H/W TSO and avoid the WQE malfarmation
+ * Due to H/W limitation, this needs to be reduced to 60000
+ * in order to use H/W TSO and avoid the WQE malformation
  * PKO_BUG_24989_WQE_LEN
  */
 #define    CN23XX_DEFAULT_INPUT_JABBER             0xEA60 /*60000*/
diff --git a/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_regs.h b/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_regs.h
index d33dd8f4226f..e956109415cd 100644
--- a/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_regs.h
+++ b/drivers/net/ethernet/cavium/liquidio/cn23xx_vf_regs.h
@@ -36,8 +36,8 @@
 #define     CN23XX_CONFIG_PCIE_FLTMSK              0x720
 
 /* The input jabber is used to determine the TSO max size.
- * Due to H/W limitation, this need to be reduced to 60000
- * in order to to H/W TSO and avoid the WQE malfarmation
+ * Due to H/W limitation, this needs to be reduced to 60000
+ * in order to use H/W TSO and avoid the WQE malformation
  * PKO_BUG_24989_WQE_LEN
  */
 #define    CN23XX_DEFAULT_INPUT_JABBER             0xEA60 /*60000*/
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_core.c b/drivers/net/ethernet/cavium/liquidio/lio_core.c
index 73cb03266549..882b2be06ea0 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_core.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_core.c
@@ -851,7 +851,7 @@ int liquidio_setup_io_queues(struct octeon_device *octeon_dev, int ifidx,
 		napi = &droq->napi;
 		dev_dbg(&octeon_dev->pci_dev->dev, "netif_napi_add netdev:%llx oct:%llx\n",
 			(u64)netdev, (u64)octeon_dev);
-		netif_napi_add(netdev, napi, liquidio_napi_poll, 64);
+		netif_napi_add(netdev, napi, liquidio_napi_poll);
 
 		/* designate a CPU for this droq */
 		droq->cpu_id = cpu_id;
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c
index bee35ce60171..d312bd594935 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c
@@ -92,11 +92,6 @@ static int octeon_console_debug_enabled(u32 console)
 /* time to wait for possible in-flight requests in milliseconds */
 #define WAIT_INFLIGHT_REQUEST	msecs_to_jiffies(1000)
 
-struct lio_trusted_vf_ctx {
-	struct completion complete;
-	int status;
-};
-
 struct oct_link_status_resp {
 	u64 rh;
 	struct oct_link_info link_info;
diff --git a/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c b/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c
index 103591dcea1c..edde0b8fa49c 100644
--- a/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c
+++ b/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c
@@ -1342,7 +1342,7 @@ static void octeon_mgmt_poll_controller(struct net_device *netdev)
 static void octeon_mgmt_get_drvinfo(struct net_device *netdev,
 				    struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 }
 
 static int octeon_mgmt_nway_reset(struct net_device *dev)
@@ -1396,8 +1396,8 @@ static int octeon_mgmt_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, netdev);
 	p = netdev_priv(netdev);
-	netif_napi_add(netdev, &p->napi, octeon_mgmt_napi_poll,
-		       OCTEON_MGMT_NAPI_WEIGHT);
+	netif_napi_add_weight(netdev, &p->napi, octeon_mgmt_napi_poll,
+			      OCTEON_MGMT_NAPI_WEIGHT);
 
 	p->netdev = netdev;
 	p->dev = &pdev->dev;
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
index 5a9fad61e9ea..e5c71f907852 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c
@@ -191,8 +191,8 @@ static void nicvf_get_drvinfo(struct net_device *netdev,
 {
 	struct nicvf *nic = netdev_priv(netdev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(nic->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(nic->pdev), sizeof(info->bus_info));
 }
 
 static u32 nicvf_get_msglevel(struct net_device *netdev)
diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c
index 768ea426d49f..98f3dc460ca7 100644
--- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c
+++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c
@@ -1472,8 +1472,7 @@ int nicvf_open(struct net_device *netdev)
 		}
 		cq_poll->cq_idx = qidx;
 		cq_poll->nicvf = nic;
-		netif_napi_add(netdev, &cq_poll->napi, nicvf_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(netdev, &cq_poll->napi, nicvf_poll);
 		napi_enable(&cq_poll->napi);
 		nic->napi[qidx] = cq_poll;
 	}
diff --git a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c
index f4054d2553ea..d2286adf09fe 100644
--- a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c
+++ b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c
@@ -429,8 +429,8 @@ static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
 	struct adapter *adapter = dev->ml_priv;
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(adapter->pdev),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(adapter->pdev),
 		sizeof(info->bus_info));
 }
 
@@ -1053,7 +1053,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		netdev->hard_header_len += (netdev->hw_features & NETIF_F_TSO) ?
 			sizeof(struct cpl_tx_pkt_lso) : sizeof(struct cpl_tx_pkt);
 
-		netif_napi_add(netdev, &adapter->napi, t1_poll, 64);
+		netif_napi_add(netdev, &adapter->napi, t1_poll);
 
 		netdev->ethtool_ops = &t1_ethtool_ops;
 
diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
index 174b1e156669..a52e6b6e2876 100644
--- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
@@ -609,8 +609,7 @@ static void init_napi(struct adapter *adap)
 		struct sge_qset *qs = &adap->sge.qs[i];
 
 		if (qs->adap)
-			netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
-				       64);
+			netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll);
 	}
 
 	/*
@@ -1627,8 +1626,8 @@ static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 	t3_get_tp_version(adapter, &tp_vers);
 	spin_unlock(&adapter->stats_lock);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(adapter->pdev),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(adapter->pdev),
 		sizeof(info->bus_info));
 	if (fw_vers)
 		snprintf(info->fw_version, sizeof(info->fw_version),
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c
index a7f291c89702..557c591a6ce3 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cudbg_lib.c
@@ -14,6 +14,7 @@
 #include "cudbg_entity.h"
 #include "cudbg_lib.h"
 #include "cudbg_zlib.h"
+#include "cxgb4_tc_mqprio.h"
 
 static const u32 t6_tp_pio_array[][IREG_NUM_ELEM] = {
 	{0x7e40, 0x7e44, 0x020, 28}, /* t6_tp_pio_regs_20_to_3b */
@@ -3458,7 +3459,7 @@ int cudbg_collect_qdesc(struct cudbg_init *pdbg_init,
 			for (i = 0; i < utxq->ntxq; i++)
 				QDESC_GET_TXQ(&utxq->uldtxq[i].q,
 					      cudbg_uld_txq_to_qtype(j),
-					      out_unlock);
+					      out_unlock_uld);
 		}
 	}
 
@@ -3475,7 +3476,7 @@ int cudbg_collect_qdesc(struct cudbg_init *pdbg_init,
 			for (i = 0; i < urxq->nrxq; i++)
 				QDESC_GET_RXQ(&urxq->uldrxq[i].rspq,
 					      cudbg_uld_rxq_to_qtype(j),
-					      out_unlock);
+					      out_unlock_uld);
 		}
 
 		/* ULD FLQ */
@@ -3487,7 +3488,7 @@ int cudbg_collect_qdesc(struct cudbg_init *pdbg_init,
 			for (i = 0; i < urxq->nrxq; i++)
 				QDESC_GET_FLQ(&urxq->uldrxq[i].fl,
 					      cudbg_uld_flq_to_qtype(j),
-					      out_unlock);
+					      out_unlock_uld);
 		}
 
 		/* ULD CIQ */
@@ -3500,29 +3501,34 @@ int cudbg_collect_qdesc(struct cudbg_init *pdbg_init,
 			for (i = 0; i < urxq->nciq; i++)
 				QDESC_GET_RXQ(&urxq->uldrxq[base + i].rspq,
 					      cudbg_uld_ciq_to_qtype(j),
-					      out_unlock);
+					      out_unlock_uld);
 		}
 	}
+	mutex_unlock(&uld_mutex);
+
+	if (!padap->tc_mqprio)
+		goto out;
 
+	mutex_lock(&padap->tc_mqprio->mqprio_mutex);
 	/* ETHOFLD TXQ */
 	if (s->eohw_txq)
 		for (i = 0; i < s->eoqsets; i++)
 			QDESC_GET_TXQ(&s->eohw_txq[i].q,
-				      CUDBG_QTYPE_ETHOFLD_TXQ, out);
+				      CUDBG_QTYPE_ETHOFLD_TXQ, out_unlock_mqprio);
 
 	/* ETHOFLD RXQ and FLQ */
 	if (s->eohw_rxq) {
 		for (i = 0; i < s->eoqsets; i++)
 			QDESC_GET_RXQ(&s->eohw_rxq[i].rspq,
-				      CUDBG_QTYPE_ETHOFLD_RXQ, out);
+				      CUDBG_QTYPE_ETHOFLD_RXQ, out_unlock_mqprio);
 
 		for (i = 0; i < s->eoqsets; i++)
 			QDESC_GET_FLQ(&s->eohw_rxq[i].fl,
-				      CUDBG_QTYPE_ETHOFLD_FLQ, out);
+				      CUDBG_QTYPE_ETHOFLD_FLQ, out_unlock_mqprio);
 	}
 
-out_unlock:
-	mutex_unlock(&uld_mutex);
+out_unlock_mqprio:
+	mutex_unlock(&padap->tc_mqprio->mqprio_mutex);
 
 out:
 	qdesc_info->qdesc_entry_size = sizeof(*qdesc_entry);
@@ -3559,6 +3565,10 @@ out_free:
 #undef QDESC_GET
 
 	return rc;
+
+out_unlock_uld:
+	mutex_unlock(&uld_mutex);
+	goto out;
 }
 
 int cudbg_collect_flash(struct cudbg_init *pdbg_init,
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
index 77897edd2bc0..8477a93cee6b 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c
@@ -199,8 +199,8 @@ static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 	struct adapter *adapter = netdev2adap(dev);
 	u32 exprom_vers;
 
-	strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(adapter->pdev),
+	strscpy(info->driver, cxgb4_driver_name, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(adapter->pdev),
 		sizeof(info->bus_info));
 	info->regdump_len = get_regs_len(dev);
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index d0061921529f..9cbce1faab26 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -3903,8 +3903,8 @@ static void cxgb4_mgmt_get_drvinfo(struct net_device *dev,
 {
 	struct adapter *adapter = netdev2adap(dev);
 
-	strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(adapter->pdev),
+	strscpy(info->driver, cxgb4_driver_name, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(adapter->pdev),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c
index ee52e3b1d74f..46809e2d94ee 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c
@@ -4467,7 +4467,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq,
 	if (ret)
 		goto err;
 
-	netif_napi_add(dev, &iq->napi, napi_rx_handler, 64);
+	netif_napi_add(dev, &iq->napi, napi_rx_handler);
 	iq->cur_desc = iq->desc;
 	iq->cidx = 0;
 	iq->gen = 1;
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
index c2822e635f89..54db79f4dcfe 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
@@ -1553,8 +1553,8 @@ static void cxgb4vf_get_drvinfo(struct net_device *dev,
 {
 	struct adapter *adapter = netdev2adap(dev);
 
-	strlcpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, pci_name(to_pci_dev(dev->dev.parent)),
+	strscpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->bus_info, pci_name(to_pci_dev(dev->dev.parent)),
 		sizeof(drvinfo->bus_info));
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
 		 "%u.%u.%u.%u, TP %u.%u.%u.%u",
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
index 43b2ceb6aa32..2d0cf76fb3c5 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c
@@ -2336,7 +2336,7 @@ int t4vf_sge_alloc_rxq(struct adapter *adapter, struct sge_rspq *rspq,
 	if (ret)
 		goto err;
 
-	netif_napi_add(dev, &rspq->napi, napi_rx_handler, 64);
+	netif_napi_add(dev, &rspq->napi, napi_rx_handler);
 	rspq->cur_desc = rspq->desc;
 	rspq->cidx = 0;
 	rspq->gen = 1;
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
index ddfe9208529a..f90bfba4b303 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
+++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
@@ -1069,8 +1069,7 @@ static void chtls_pass_accept_rpl(struct sk_buff *skb,
 	cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
 }
 
-static void inet_inherit_port(struct inet_hashinfo *hash_info,
-			      struct sock *lsk, struct sock *newsk)
+static void inet_inherit_port(struct sock *lsk, struct sock *newsk)
 {
 	local_bh_disable();
 	__inet_inherit_port(lsk, newsk);
@@ -1240,7 +1239,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
 						     ipv4.sysctl_tcp_window_scaling),
 					   tp->window_clamp);
 	neigh_release(n);
-	inet_inherit_port(&tcp_hashinfo, lsk, newsk);
+	inet_inherit_port(lsk, newsk);
 	csk_set_flag(csk, CSK_CONN_INLINE);
 	bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */
 
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c
index 9098b3eed4da..1e55b12fee51 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c
+++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_main.c
@@ -193,7 +193,7 @@ static void chtls_register_dev(struct chtls_dev *cdev)
 {
 	struct tls_toe_device *tlsdev = &cdev->tlsdev;
 
-	strlcpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX);
+	strscpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX);
 	strlcat(tlsdev->name, cdev->lldi->ports[0]->name,
 		TLS_TOE_DEVICE_NAME_MAX);
 	tlsdev->feature = chtls_inline_feature;
diff --git a/drivers/net/ethernet/cirrus/ep93xx_eth.c b/drivers/net/ethernet/cirrus/ep93xx_eth.c
index 21ba6e893072..8627ab19d470 100644
--- a/drivers/net/ethernet/cirrus/ep93xx_eth.c
+++ b/drivers/net/ethernet/cirrus/ep93xx_eth.c
@@ -689,7 +689,7 @@ static int ep93xx_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 
 static void ep93xx_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
 }
 
 static int ep93xx_get_link_ksettings(struct net_device *dev,
@@ -812,7 +812,7 @@ static int ep93xx_eth_probe(struct platform_device *pdev)
 	ep = netdev_priv(dev);
 	ep->dev = dev;
 	SET_NETDEV_DEV(dev, &pdev->dev);
-	netif_napi_add(dev, &ep->napi, ep93xx_poll, 64);
+	netif_napi_add(dev, &ep->napi, ep93xx_poll);
 
 	platform_set_drvdata(pdev, dev);
 
diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c
index 60d8c0fbc037..08b7cc0a1809 100644
--- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c
+++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c
@@ -131,10 +131,10 @@ static void enic_get_drvinfo(struct net_device *netdev,
 	if (err == -ENOMEM)
 		return;
 
-	strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->fw_version, fw_info->fw_version,
+	strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->fw_version, fw_info->fw_version,
 		sizeof(drvinfo->fw_version));
-	strlcpy(drvinfo->bus_info, pci_name(enic->pdev),
+	strscpy(drvinfo->bus_info, pci_name(enic->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c
index 372fb7b3a282..29500d32e362 100644
--- a/drivers/net/ethernet/cisco/enic/enic_main.c
+++ b/drivers/net/ethernet/cisco/enic/enic_main.c
@@ -2633,16 +2633,17 @@ static int enic_dev_init(struct enic *enic)
 
 	switch (vnic_dev_get_intr_mode(enic->vdev)) {
 	default:
-		netif_napi_add(netdev, &enic->napi[0], enic_poll, 64);
+		netif_napi_add(netdev, &enic->napi[0], enic_poll);
 		break;
 	case VNIC_DEV_INTR_MODE_MSIX:
 		for (i = 0; i < enic->rq_count; i++) {
 			netif_napi_add(netdev, &enic->napi[i],
-				enic_poll_msix_rq, NAPI_POLL_WEIGHT);
+				       enic_poll_msix_rq);
 		}
 		for (i = 0; i < enic->wq_count; i++)
-			netif_napi_add(netdev, &enic->napi[enic_cq_wq(enic, i)],
-				       enic_poll_msix_wq, NAPI_POLL_WEIGHT);
+			netif_napi_add(netdev,
+				       &enic->napi[enic_cq_wq(enic, i)],
+				       enic_poll_msix_wq);
 		break;
 	}
 
diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c
index 6dae768671e3..fdf10318758b 100644
--- a/drivers/net/ethernet/cortina/gemini.c
+++ b/drivers/net/ethernet/cortina/gemini.c
@@ -2471,7 +2471,7 @@ static int gemini_ethernet_port_probe(struct platform_device *pdev)
 	netdev->max_mtu = 10236 - VLAN_ETH_HLEN;
 
 	port->freeq_refill = 0;
-	netif_napi_add(netdev, &port->napi, gmac_napi_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, &port->napi, gmac_napi_poll);
 
 	ret = of_get_mac_address(np, mac);
 	if (!ret) {
diff --git a/drivers/net/ethernet/davicom/dm9000.c b/drivers/net/ethernet/davicom/dm9000.c
index 0985ab216566..b21e56de6167 100644
--- a/drivers/net/ethernet/davicom/dm9000.c
+++ b/drivers/net/ethernet/davicom/dm9000.c
@@ -28,8 +28,7 @@
 #include <linux/irq.h>
 #include <linux/slab.h>
 #include <linux/regulator/consumer.h>
-#include <linux/gpio.h>
-#include <linux/of_gpio.h>
+#include <linux/gpio/consumer.h>
 
 #include <asm/delay.h>
 #include <asm/irq.h>
@@ -540,8 +539,8 @@ static void dm9000_get_drvinfo(struct net_device *dev,
 {
 	struct board_info *dm = to_dm9000_board(dev);
 
-	strlcpy(info->driver, CARDNAME, sizeof(info->driver));
-	strlcpy(info->bus_info, to_platform_device(dm->dev)->name,
+	strscpy(info->driver, CARDNAME, sizeof(info->driver));
+	strscpy(info->bus_info, to_platform_device(dm->dev)->name,
 		sizeof(info->bus_info));
 }
 
@@ -1012,7 +1011,7 @@ static void dm9000_send_packet(struct net_device *dev,
  *  Hardware start transmission.
  *  Send a packet to media from the upper layer.
  */
-static int
+static netdev_tx_t
 dm9000_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	unsigned long flags;
@@ -1421,8 +1420,7 @@ dm9000_probe(struct platform_device *pdev)
 	int iosize;
 	int i;
 	u32 id_val;
-	int reset_gpios;
-	enum of_gpio_flags flags;
+	struct gpio_desc *reset_gpio;
 	struct regulator *power;
 	bool inv_mac_addr = false;
 	u8 addr[ETH_ALEN];
@@ -1442,20 +1440,24 @@ dm9000_probe(struct platform_device *pdev)
 		dev_dbg(dev, "regulator enabled\n");
 	}
 
-	reset_gpios = of_get_named_gpio_flags(dev->of_node, "reset-gpios", 0,
-					      &flags);
-	if (gpio_is_valid(reset_gpios)) {
-		ret = devm_gpio_request_one(dev, reset_gpios, flags,
-					    "dm9000_reset");
+	reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
+	ret = PTR_ERR_OR_ZERO(reset_gpio);
+	if (ret) {
+		dev_err(dev, "failed to request reset gpio: %d\n", ret);
+		goto out_regulator_disable;
+	}
+
+	if (reset_gpio) {
+		ret = gpiod_set_consumer_name(reset_gpio, "dm9000_reset");
 		if (ret) {
-			dev_err(dev, "failed to request reset gpio %d: %d\n",
-				reset_gpios, ret);
+			dev_err(dev, "failed to set reset gpio name: %d\n",
+				ret);
 			goto out_regulator_disable;
 		}
 
 		/* According to manual PWRST# Low Period Min 1ms */
 		msleep(2);
-		gpio_set_value(reset_gpios, 1);
+		gpiod_set_value_cansleep(reset_gpio, 0);
 		/* Needs 3ms to read eeprom when PWRST is deasserted */
 		msleep(4);
 	}
diff --git a/drivers/net/ethernet/dec/tulip/de2104x.c b/drivers/net/ethernet/dec/tulip/de2104x.c
index d51b3d24a0c8..cd3dc4b89518 100644
--- a/drivers/net/ethernet/dec/tulip/de2104x.c
+++ b/drivers/net/ethernet/dec/tulip/de2104x.c
@@ -1606,8 +1606,8 @@ static void de_get_drvinfo (struct net_device *dev,struct ethtool_drvinfo *info)
 {
 	struct de_private *de = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(de->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(de->pdev), sizeof(info->bus_info));
 }
 
 static int de_get_regs_len(struct net_device *dev)
diff --git a/drivers/net/ethernet/dec/tulip/dmfe.c b/drivers/net/ethernet/dec/tulip/dmfe.c
index 83f1727d1423..3188ba7b450f 100644
--- a/drivers/net/ethernet/dec/tulip/dmfe.c
+++ b/drivers/net/ethernet/dec/tulip/dmfe.c
@@ -1074,8 +1074,8 @@ static void dmfe_ethtool_get_drvinfo(struct net_device *dev,
 {
 	struct dmfe_board_info *np = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info));
 }
 
 static int dmfe_ethtool_set_wol(struct net_device *dev,
diff --git a/drivers/net/ethernet/dec/tulip/tulip_core.c b/drivers/net/ethernet/dec/tulip/tulip_core.c
index b8e46c4849ef..ecfad43df45a 100644
--- a/drivers/net/ethernet/dec/tulip/tulip_core.c
+++ b/drivers/net/ethernet/dec/tulip/tulip_core.c
@@ -858,8 +858,8 @@ static struct net_device_stats *tulip_get_stats(struct net_device *dev)
 static void tulip_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
 	struct tulip_private *np = netdev_priv(dev);
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info));
 }
 
 
diff --git a/drivers/net/ethernet/dec/tulip/uli526x.c b/drivers/net/ethernet/dec/tulip/uli526x.c
index 77d9058431e3..ff080ab0f116 100644
--- a/drivers/net/ethernet/dec/tulip/uli526x.c
+++ b/drivers/net/ethernet/dec/tulip/uli526x.c
@@ -971,8 +971,8 @@ static void netdev_get_drvinfo(struct net_device *dev,
 {
 	struct uli526x_board_info *np = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info));
 }
 
 static int netdev_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/dec/tulip/winbond-840.c b/drivers/net/ethernet/dec/tulip/winbond-840.c
index 1db19463fd46..37fba39c0056 100644
--- a/drivers/net/ethernet/dec/tulip/winbond-840.c
+++ b/drivers/net/ethernet/dec/tulip/winbond-840.c
@@ -1374,8 +1374,8 @@ static void netdev_get_drvinfo (struct net_device *dev, struct ethtool_drvinfo *
 {
 	struct netdev_private *np = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
 }
 
 static int netdev_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/dlink/dl2k.c b/drivers/net/ethernet/dlink/dl2k.c
index a301f7e6a440..2c67a857a42f 100644
--- a/drivers/net/ethernet/dlink/dl2k.c
+++ b/drivers/net/ethernet/dlink/dl2k.c
@@ -1235,8 +1235,8 @@ static void rio_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info
 {
 	struct netdev_private *np = netdev_priv(dev);
 
-	strlcpy(info->driver, "dl2k", sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, "dl2k", sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(np->pdev), sizeof(info->bus_info));
 }
 
 static int rio_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/dlink/sundance.c b/drivers/net/ethernet/dlink/sundance.c
index 8dd7bf9014ec..43def191f26f 100644
--- a/drivers/net/ethernet/dlink/sundance.c
+++ b/drivers/net/ethernet/dlink/sundance.c
@@ -1644,8 +1644,8 @@ static int check_if_running(struct net_device *dev)
 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
 	struct netdev_private *np = netdev_priv(dev);
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
 }
 
 static int get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/dnet.c b/drivers/net/ethernet/dnet.c
index 92462ed87bc4..08184f20f510 100644
--- a/drivers/net/ethernet/dnet.c
+++ b/drivers/net/ethernet/dnet.c
@@ -725,8 +725,8 @@ static struct net_device_stats *dnet_get_stats(struct net_device *dev)
 static void dnet_get_drvinfo(struct net_device *dev,
 			     struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, "0", sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, "0", sizeof(info->bus_info));
 }
 
 static const struct ethtool_ops dnet_ethtool_ops = {
@@ -788,7 +788,7 @@ static int dnet_probe(struct platform_device *pdev)
 	}
 
 	dev->netdev_ops = &dnet_netdev_ops;
-	netif_napi_add(dev, &bp->napi, dnet_poll, 64);
+	netif_napi_add(dev, &bp->napi, dnet_poll);
 	dev->ethtool_ops = &dnet_ethtool_ops;
 
 	dev->base_addr = (unsigned long)bp->regs;
diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c
index b4f5e57d0285..08ec84cd21c0 100644
--- a/drivers/net/ethernet/emulex/benet/be_cmds.c
+++ b/drivers/net/ethernet/emulex/benet/be_cmds.c
@@ -1878,9 +1878,9 @@ int be_cmd_get_fw_ver(struct be_adapter *adapter)
 	if (!status) {
 		struct be_cmd_resp_get_fw_version *resp = embedded_payload(wrb);
 
-		strlcpy(adapter->fw_ver, resp->firmware_version_string,
+		strscpy(adapter->fw_ver, resp->firmware_version_string,
 			sizeof(adapter->fw_ver));
-		strlcpy(adapter->fw_on_flash, resp->fw_on_flash_version_string,
+		strscpy(adapter->fw_on_flash, resp->fw_on_flash_version_string,
 			sizeof(adapter->fw_on_flash));
 	}
 err:
@@ -2373,7 +2373,7 @@ static int lancer_cmd_write_object(struct be_adapter *adapter,
 
 	be_dws_cpu_to_le(ctxt, sizeof(req->context));
 	req->write_offset = cpu_to_le32(data_offset);
-	strlcpy(req->object_name, obj_name, sizeof(req->object_name));
+	strscpy(req->object_name, obj_name, sizeof(req->object_name));
 	req->descriptor_count = cpu_to_le32(1);
 	req->buf_len = cpu_to_le32(data_size);
 	req->addr_low = cpu_to_le32((cmd->dma +
@@ -2442,9 +2442,9 @@ int be_cmd_query_sfp_info(struct be_adapter *adapter)
 	status = be_cmd_read_port_transceiver_data(adapter, TR_PAGE_A0,
 						   0, PAGE_DATA_LEN, page_data);
 	if (!status) {
-		strlcpy(adapter->phy.vendor_name, page_data +
+		strscpy(adapter->phy.vendor_name, page_data +
 			SFP_VENDOR_NAME_OFFSET, SFP_VENDOR_NAME_LEN - 1);
-		strlcpy(adapter->phy.vendor_pn,
+		strscpy(adapter->phy.vendor_pn,
 			page_data + SFP_VENDOR_PN_OFFSET,
 			SFP_VENDOR_NAME_LEN - 1);
 	}
@@ -2473,7 +2473,7 @@ static int lancer_cmd_delete_object(struct be_adapter *adapter,
 			       OPCODE_COMMON_DELETE_OBJECT,
 			       sizeof(*req), wrb, NULL);
 
-	strlcpy(req->object_name, obj_name, sizeof(req->object_name));
+	strscpy(req->object_name, obj_name, sizeof(req->object_name));
 
 	status = be_mcc_notify_wait(adapter);
 err:
diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c
index bd0df189d871..77edc3d9b505 100644
--- a/drivers/net/ethernet/emulex/benet/be_ethtool.c
+++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c
@@ -220,15 +220,15 @@ static void be_get_drvinfo(struct net_device *netdev,
 {
 	struct be_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
 	if (!memcmp(adapter->fw_ver, adapter->fw_on_flash, FW_VER_LEN))
-		strlcpy(drvinfo->fw_version, adapter->fw_ver,
+		strscpy(drvinfo->fw_version, adapter->fw_ver,
 			sizeof(drvinfo->fw_version));
 	else
 		snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
 			 "%s [%s]", adapter->fw_ver, adapter->fw_on_flash);
 
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/emulex/benet/be_main.c b/drivers/net/ethernet/emulex/benet/be_main.c
index 414362febbb9..a92a74761546 100644
--- a/drivers/net/ethernet/emulex/benet/be_main.c
+++ b/drivers/net/ethernet/emulex/benet/be_main.c
@@ -2982,8 +2982,7 @@ static int be_evt_queues_create(struct be_adapter *adapter)
 			return -ENOMEM;
 		cpumask_set_cpu(cpumask_local_spread(i, numa_node),
 				eqo->affinity_mask);
-		netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(adapter->netdev, &eqo->napi, be_poll);
 	}
 	return 0;
 }
diff --git a/drivers/net/ethernet/engleder/Kconfig b/drivers/net/ethernet/engleder/Kconfig
index f4e2b1102d8f..3df6bf476ae7 100644
--- a/drivers/net/ethernet/engleder/Kconfig
+++ b/drivers/net/ethernet/engleder/Kconfig
@@ -21,6 +21,7 @@ config TSNEP
 	depends on HAS_IOMEM && HAS_DMA
 	depends on PTP_1588_CLOCK_OPTIONAL
 	select PHYLIB
+	select PAGE_POOL
 	help
 	  Support for the Engleder TSN endpoint Ethernet MAC IP Core.
 
diff --git a/drivers/net/ethernet/engleder/Makefile b/drivers/net/ethernet/engleder/Makefile
index cce2191cb889..b6e3b16623de 100644
--- a/drivers/net/ethernet/engleder/Makefile
+++ b/drivers/net/ethernet/engleder/Makefile
@@ -6,5 +6,5 @@
 obj-$(CONFIG_TSNEP) += tsnep.o
 
 tsnep-objs := tsnep_main.o tsnep_ethtool.o tsnep_ptp.o tsnep_tc.o \
-	      $(tsnep-y)
+	      tsnep_rxnfc.o $(tsnep-y)
 tsnep-$(CONFIG_TSNEP_SELFTESTS) += tsnep_selftests.o
diff --git a/drivers/net/ethernet/engleder/tsnep.h b/drivers/net/ethernet/engleder/tsnep.h
index 23bbece6b7de..09a723b827c7 100644
--- a/drivers/net/ethernet/engleder/tsnep.h
+++ b/drivers/net/ethernet/engleder/tsnep.h
@@ -21,8 +21,6 @@
 #define TSNEP_RING_ENTRIES_PER_PAGE (PAGE_SIZE / TSNEP_DESC_SIZE)
 #define TSNEP_RING_PAGE_COUNT (TSNEP_RING_SIZE / TSNEP_RING_ENTRIES_PER_PAGE)
 
-#define TSNEP_QUEUES 1
-
 struct tsnep_gcl {
 	void __iomem *addr;
 
@@ -39,6 +37,24 @@ struct tsnep_gcl {
 	bool change;
 };
 
+enum tsnep_rxnfc_filter_type {
+	TSNEP_RXNFC_ETHER_TYPE,
+};
+
+struct tsnep_rxnfc_filter {
+	enum tsnep_rxnfc_filter_type type;
+	union {
+		u16 ether_type;
+	};
+};
+
+struct tsnep_rxnfc_rule {
+	struct list_head list;
+	struct tsnep_rxnfc_filter filter;
+	int queue_index;
+	int location;
+};
+
 struct tsnep_tx_entry {
 	struct tsnep_tx_desc *desc;
 	struct tsnep_tx_desc_wb *desc_wb;
@@ -55,6 +71,7 @@ struct tsnep_tx_entry {
 struct tsnep_tx {
 	struct tsnep_adapter *adapter;
 	void __iomem *addr;
+	int queue_index;
 
 	void *page[TSNEP_RING_PAGE_COUNT];
 	dma_addr_t page_dma[TSNEP_RING_PAGE_COUNT];
@@ -79,14 +96,15 @@ struct tsnep_rx_entry {
 
 	u32 properties;
 
-	struct sk_buff *skb;
+	struct page *page;
 	size_t len;
-	DEFINE_DMA_UNMAP_ADDR(dma);
+	dma_addr_t dma;
 };
 
 struct tsnep_rx {
 	struct tsnep_adapter *adapter;
 	void __iomem *addr;
+	int queue_index;
 
 	void *page[TSNEP_RING_PAGE_COUNT];
 	dma_addr_t page_dma[TSNEP_RING_PAGE_COUNT];
@@ -95,6 +113,7 @@ struct tsnep_rx {
 	int read;
 	u32 owner_counter;
 	int increment_owner_counter;
+	struct page_pool *page_pool;
 
 	u32 packets;
 	u32 bytes;
@@ -104,12 +123,14 @@ struct tsnep_rx {
 
 struct tsnep_queue {
 	struct tsnep_adapter *adapter;
+	char name[IFNAMSIZ + 9];
 
 	struct tsnep_tx *tx;
 	struct tsnep_rx *rx;
 
 	struct napi_struct napi;
 
+	int irq;
 	u32 irq_mask;
 };
 
@@ -125,7 +146,6 @@ struct tsnep_adapter {
 	struct platform_device *pdev;
 	struct device *dmadev;
 	void __iomem *addr;
-	int irq;
 
 	bool gate_control;
 	/* gate control lock */
@@ -140,6 +160,12 @@ struct tsnep_adapter {
 	/* ptp clock lock */
 	spinlock_t ptp_lock;
 
+	/* RX flow classification rules lock */
+	struct mutex rxnfc_lock;
+	struct list_head rxnfc_rules;
+	int rxnfc_count;
+	int rxnfc_max;
+
 	int num_tx_queues;
 	struct tsnep_tx tx[TSNEP_MAX_QUEUES];
 	int num_rx_queues;
@@ -160,6 +186,18 @@ void tsnep_tc_cleanup(struct tsnep_adapter *adapter);
 int tsnep_tc_setup(struct net_device *netdev, enum tc_setup_type type,
 		   void *type_data);
 
+int tsnep_rxnfc_init(struct tsnep_adapter *adapter);
+void tsnep_rxnfc_cleanup(struct tsnep_adapter *adapter);
+int tsnep_rxnfc_get_rule(struct tsnep_adapter *adapter,
+			 struct ethtool_rxnfc *cmd);
+int tsnep_rxnfc_get_all(struct tsnep_adapter *adapter,
+			struct ethtool_rxnfc *cmd,
+			u32 *rule_locs);
+int tsnep_rxnfc_add_rule(struct tsnep_adapter *adapter,
+			 struct ethtool_rxnfc *cmd);
+int tsnep_rxnfc_del_rule(struct tsnep_adapter *adapter,
+			 struct ethtool_rxnfc *cmd);
+
 #if IS_ENABLED(CONFIG_TSNEP_SELFTESTS)
 int tsnep_ethtool_get_test_count(void);
 void tsnep_ethtool_get_test_strings(u8 *data);
diff --git a/drivers/net/ethernet/engleder/tsnep_ethtool.c b/drivers/net/ethernet/engleder/tsnep_ethtool.c
index e6760dc68ddd..a713a126b227 100644
--- a/drivers/net/ethernet/engleder/tsnep_ethtool.c
+++ b/drivers/net/ethernet/engleder/tsnep_ethtool.c
@@ -250,6 +250,44 @@ static int tsnep_ethtool_get_sset_count(struct net_device *netdev, int sset)
 	}
 }
 
+static int tsnep_ethtool_get_rxnfc(struct net_device *dev,
+				   struct ethtool_rxnfc *cmd, u32 *rule_locs)
+{
+	struct tsnep_adapter *adapter = netdev_priv(dev);
+
+	switch (cmd->cmd) {
+	case ETHTOOL_GRXRINGS:
+		cmd->data = adapter->num_rx_queues;
+		return 0;
+	case ETHTOOL_GRXCLSRLCNT:
+		cmd->rule_cnt = adapter->rxnfc_count;
+		cmd->data = adapter->rxnfc_max;
+		cmd->data |= RX_CLS_LOC_SPECIAL;
+		return 0;
+	case ETHTOOL_GRXCLSRULE:
+		return tsnep_rxnfc_get_rule(adapter, cmd);
+	case ETHTOOL_GRXCLSRLALL:
+		return tsnep_rxnfc_get_all(adapter, cmd, rule_locs);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int tsnep_ethtool_set_rxnfc(struct net_device *dev,
+				   struct ethtool_rxnfc *cmd)
+{
+	struct tsnep_adapter *adapter = netdev_priv(dev);
+
+	switch (cmd->cmd) {
+	case ETHTOOL_SRXCLSRLINS:
+		return tsnep_rxnfc_add_rule(adapter, cmd);
+	case ETHTOOL_SRXCLSRLDEL:
+		return tsnep_rxnfc_del_rule(adapter, cmd);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 static int tsnep_ethtool_get_ts_info(struct net_device *dev,
 				     struct ethtool_ts_info *info)
 {
@@ -287,6 +325,8 @@ const struct ethtool_ops tsnep_ethtool_ops = {
 	.get_strings = tsnep_ethtool_get_strings,
 	.get_ethtool_stats = tsnep_ethtool_get_ethtool_stats,
 	.get_sset_count = tsnep_ethtool_get_sset_count,
+	.get_rxnfc = tsnep_ethtool_get_rxnfc,
+	.set_rxnfc = tsnep_ethtool_set_rxnfc,
 	.get_ts_info = tsnep_ethtool_get_ts_info,
 	.get_link_ksettings = phy_ethtool_get_link_ksettings,
 	.set_link_ksettings = phy_ethtool_set_link_ksettings,
diff --git a/drivers/net/ethernet/engleder/tsnep_hw.h b/drivers/net/ethernet/engleder/tsnep_hw.h
index 916ceac3ada2..315dada75323 100644
--- a/drivers/net/ethernet/engleder/tsnep_hw.h
+++ b/drivers/net/ethernet/engleder/tsnep_hw.h
@@ -34,6 +34,7 @@
 #define ECM_INT_LINK 0x00000020
 #define ECM_INT_TX_0 0x00000100
 #define ECM_INT_RX_0 0x00000200
+#define ECM_INT_TXRX_SHIFT 2
 #define ECM_INT_ALL 0x7FFFFFFF
 #define ECM_INT_DISABLE 0x80000000
 
@@ -92,8 +93,7 @@
 
 /* tsnep register */
 #define TSNEP_INFO 0x0100
-#define TSNEP_INFO_RX_ASSIGN 0x00010000
-#define TSNEP_INFO_TX_TIME 0x00020000
+#define TSNEP_INFO_TX_TIME 0x00010000
 #define TSNEP_CONTROL 0x0108
 #define TSNEP_CONTROL_TX_RESET 0x00000001
 #define TSNEP_CONTROL_TX_ENABLE 0x00000002
@@ -122,10 +122,6 @@
 #define TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL 0x0191
 #define TSNEP_RX_STATISTIC_FIFO_OVERFLOW 0x0192
 #define TSNEP_RX_STATISTIC_INVALID_FRAME 0x0193
-#define TSNEP_RX_ASSIGN 0x01A0
-#define TSNEP_RX_ASSIGN_ETHER_TYPE_ACTIVE 0x00000001
-#define TSNEP_RX_ASSIGN_ETHER_TYPE_MASK 0xFFFF0000
-#define TSNEP_RX_ASSIGN_ETHER_TYPE_SHIFT 16
 #define TSNEP_MAC_ADDRESS_LOW 0x0800
 #define TSNEP_MAC_ADDRESS_HIGH 0x0804
 #define TSNEP_RX_FILTER 0x0806
@@ -152,6 +148,14 @@
 #define TSNEP_GCL_A 0x2000
 #define TSNEP_GCL_B 0x2800
 #define TSNEP_GCL_SIZE SZ_2K
+#define TSNEP_RX_ASSIGN 0x0840
+#define TSNEP_RX_ASSIGN_ACTIVE 0x00000001
+#define TSNEP_RX_ASSIGN_QUEUE_MASK 0x00000006
+#define TSNEP_RX_ASSIGN_QUEUE_SHIFT 1
+#define TSNEP_RX_ASSIGN_OFFSET 1
+#define TSNEP_RX_ASSIGN_ETHER_TYPE 0x0880
+#define TSNEP_RX_ASSIGN_ETHER_TYPE_OFFSET 2
+#define TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT 2
 
 /* tsnep gate control list operation */
 struct tsnep_gcl_operation {
diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c
index a5f7152a1716..48fb391951dd 100644
--- a/drivers/net/ethernet/engleder/tsnep_main.c
+++ b/drivers/net/ethernet/engleder/tsnep_main.c
@@ -27,10 +27,10 @@
 #include <linux/phy.h>
 #include <linux/iopoll.h>
 
-#define RX_SKB_LENGTH (round_up(TSNEP_RX_INLINE_METADATA_SIZE + ETH_HLEN + \
-				TSNEP_MAX_FRAME_SIZE + ETH_FCS_LEN, 4))
-#define RX_SKB_RESERVE ((16 - TSNEP_RX_INLINE_METADATA_SIZE) + NET_IP_ALIGN)
-#define RX_SKB_ALLOC_LENGTH (RX_SKB_RESERVE + RX_SKB_LENGTH)
+#define TSNEP_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN)
+#define TSNEP_HEADROOM ALIGN(TSNEP_SKB_PAD, 4)
+#define TSNEP_MAX_RX_BUF_SIZE (PAGE_SIZE - TSNEP_HEADROOM - \
+			       SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
 
 #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
 #define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF))
@@ -60,22 +60,29 @@ static irqreturn_t tsnep_irq(int irq, void *arg)
 		iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE);
 
 	/* handle link interrupt */
-	if ((active & ECM_INT_LINK) != 0) {
-		if (adapter->netdev->phydev)
-			phy_mac_interrupt(adapter->netdev->phydev);
-	}
+	if ((active & ECM_INT_LINK) != 0)
+		phy_mac_interrupt(adapter->netdev->phydev);
 
 	/* handle TX/RX queue 0 interrupt */
 	if ((active & adapter->queue[0].irq_mask) != 0) {
-		if (adapter->netdev) {
-			tsnep_disable_irq(adapter, adapter->queue[0].irq_mask);
-			napi_schedule(&adapter->queue[0].napi);
-		}
+		tsnep_disable_irq(adapter, adapter->queue[0].irq_mask);
+		napi_schedule(&adapter->queue[0].napi);
 	}
 
 	return IRQ_HANDLED;
 }
 
+static irqreturn_t tsnep_irq_txrx(int irq, void *arg)
+{
+	struct tsnep_queue *queue = arg;
+
+	/* handle TX/RX queue interrupt */
+	tsnep_disable_irq(queue->adapter, queue->irq_mask);
+	napi_schedule(&queue->napi);
+
+	return IRQ_HANDLED;
+}
+
 static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum)
 {
 	struct tsnep_adapter *adapter = bus->priv;
@@ -124,30 +131,51 @@ static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum,
 	return 0;
 }
 
+static void tsnep_set_link_mode(struct tsnep_adapter *adapter)
+{
+	u32 mode;
+
+	switch (adapter->phydev->speed) {
+	case SPEED_100:
+		mode = ECM_LINK_MODE_100;
+		break;
+	case SPEED_1000:
+		mode = ECM_LINK_MODE_1000;
+		break;
+	default:
+		mode = ECM_LINK_MODE_OFF;
+		break;
+	}
+	iowrite32(mode, adapter->addr + ECM_STATUS);
+}
+
 static void tsnep_phy_link_status_change(struct net_device *netdev)
 {
 	struct tsnep_adapter *adapter = netdev_priv(netdev);
 	struct phy_device *phydev = netdev->phydev;
-	u32 mode;
 
-	if (phydev->link) {
-		switch (phydev->speed) {
-		case SPEED_100:
-			mode = ECM_LINK_MODE_100;
-			break;
-		case SPEED_1000:
-			mode = ECM_LINK_MODE_1000;
-			break;
-		default:
-			mode = ECM_LINK_MODE_OFF;
-			break;
-		}
-		iowrite32(mode, adapter->addr + ECM_STATUS);
-	}
+	if (phydev->link)
+		tsnep_set_link_mode(adapter);
 
 	phy_print_status(netdev->phydev);
 }
 
+static int tsnep_phy_loopback(struct tsnep_adapter *adapter, bool enable)
+{
+	int retval;
+
+	retval = phy_loopback(adapter->phydev, enable);
+
+	/* PHY link state change is not signaled if loopback is enabled, it
+	 * would delay a working loopback anyway, let's ensure that loopback
+	 * is working immediately by setting link mode directly
+	 */
+	if (!retval && enable)
+		tsnep_set_link_mode(adapter);
+
+	return retval;
+}
+
 static int tsnep_phy_open(struct tsnep_adapter *adapter)
 {
 	struct phy_device *phydev;
@@ -241,14 +269,14 @@ alloc_failed:
 	return retval;
 }
 
-static void tsnep_tx_activate(struct tsnep_tx *tx, int index, bool last)
+static void tsnep_tx_activate(struct tsnep_tx *tx, int index, int length,
+			      bool last)
 {
 	struct tsnep_tx_entry *entry = &tx->entry[index];
 
 	entry->properties = 0;
 	if (entry->skb) {
-		entry->properties =
-			skb_pagelen(entry->skb) & TSNEP_DESC_LENGTH_MASK;
+		entry->properties = length & TSNEP_DESC_LENGTH_MASK;
 		entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
 		if (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS)
 			entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG;
@@ -313,6 +341,7 @@ static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count)
 	struct tsnep_tx_entry *entry;
 	unsigned int len;
 	dma_addr_t dma;
+	int map_len = 0;
 	int i;
 
 	for (i = 0; i < count; i++) {
@@ -335,15 +364,18 @@ static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count)
 		dma_unmap_addr_set(entry, dma, dma);
 
 		entry->desc->tx = __cpu_to_le64(dma);
+
+		map_len += len;
 	}
 
-	return 0;
+	return map_len;
 }
 
-static void tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)
+static int tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)
 {
 	struct device *dmadev = tx->adapter->dmadev;
 	struct tsnep_tx_entry *entry;
+	int map_len = 0;
 	int i;
 
 	for (i = 0; i < count; i++) {
@@ -360,9 +392,12 @@ static void tsnep_tx_unmap(struct tsnep_tx *tx, int index, int count)
 					       dma_unmap_addr(entry, dma),
 					       dma_unmap_len(entry, len),
 					       DMA_TO_DEVICE);
+			map_len += entry->len;
 			entry->len = 0;
 		}
 	}
+
+	return map_len;
 }
 
 static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
@@ -371,6 +406,7 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
 	unsigned long flags;
 	int count = 1;
 	struct tsnep_tx_entry *entry;
+	int length;
 	int i;
 	int retval;
 
@@ -394,7 +430,7 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
 	entry->skb = skb;
 
 	retval = tsnep_tx_map(skb, tx, count);
-	if (retval != 0) {
+	if (retval < 0) {
 		tsnep_tx_unmap(tx, tx->write, count);
 		dev_kfree_skb_any(entry->skb);
 		entry->skb = NULL;
@@ -407,12 +443,13 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
 
 		return NETDEV_TX_OK;
 	}
+	length = retval;
 
 	if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
 
 	for (i = 0; i < count; i++)
-		tsnep_tx_activate(tx, (tx->write + i) % TSNEP_RING_SIZE,
+		tsnep_tx_activate(tx, (tx->write + i) % TSNEP_RING_SIZE, length,
 				  i == (count - 1));
 	tx->write = (tx->write + count) % TSNEP_RING_SIZE;
 
@@ -428,9 +465,6 @@ static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb,
 		netif_stop_queue(tx->adapter->netdev);
 	}
 
-	tx->packets++;
-	tx->bytes += skb_pagelen(entry->skb) + ETH_FCS_LEN;
-
 	spin_unlock_irqrestore(&tx->lock, flags);
 
 	return NETDEV_TX_OK;
@@ -442,6 +476,7 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
 	int budget = 128;
 	struct tsnep_tx_entry *entry;
 	int count;
+	int length;
 
 	spin_lock_irqsave(&tx->lock, flags);
 
@@ -464,7 +499,7 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
 		if (skb_shinfo(entry->skb)->nr_frags > 0)
 			count += skb_shinfo(entry->skb)->nr_frags;
 
-		tsnep_tx_unmap(tx, tx->read, count);
+		length = tsnep_tx_unmap(tx, tx->read, count);
 
 		if ((skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) &&
 		    (__le32_to_cpu(entry->desc_wb->properties) &
@@ -491,6 +526,9 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
 
 		tx->read = (tx->read + count) % TSNEP_RING_SIZE;
 
+		tx->packets++;
+		tx->bytes += length + ETH_FCS_LEN;
+
 		budget--;
 	} while (likely(budget));
 
@@ -505,7 +543,7 @@ static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget)
 }
 
 static int tsnep_tx_open(struct tsnep_adapter *adapter, void __iomem *addr,
-			 struct tsnep_tx *tx)
+			 int queue_index, struct tsnep_tx *tx)
 {
 	dma_addr_t dma;
 	int retval;
@@ -513,6 +551,7 @@ static int tsnep_tx_open(struct tsnep_adapter *adapter, void __iomem *addr,
 	memset(tx, 0, sizeof(*tx));
 	tx->adapter = adapter;
 	tx->addr = addr;
+	tx->queue_index = queue_index;
 
 	retval = tsnep_tx_ring_init(tx);
 	if (retval)
@@ -548,14 +587,15 @@ static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
 
 	for (i = 0; i < TSNEP_RING_SIZE; i++) {
 		entry = &rx->entry[i];
-		if (dma_unmap_addr(entry, dma))
-			dma_unmap_single(dmadev, dma_unmap_addr(entry, dma),
-					 dma_unmap_len(entry, len),
-					 DMA_FROM_DEVICE);
-		if (entry->skb)
-			dev_kfree_skb(entry->skb);
+		if (entry->page)
+			page_pool_put_full_page(rx->page_pool, entry->page,
+						false);
+		entry->page = NULL;
 	}
 
+	if (rx->page_pool)
+		page_pool_destroy(rx->page_pool);
+
 	memset(rx->entry, 0, sizeof(rx->entry));
 
 	for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) {
@@ -568,31 +608,19 @@ static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx)
 	}
 }
 
-static int tsnep_rx_alloc_and_map_skb(struct tsnep_rx *rx,
-				      struct tsnep_rx_entry *entry)
+static int tsnep_rx_alloc_buffer(struct tsnep_rx *rx,
+				 struct tsnep_rx_entry *entry)
 {
-	struct device *dmadev = rx->adapter->dmadev;
-	struct sk_buff *skb;
-	dma_addr_t dma;
+	struct page *page;
 
-	skb = __netdev_alloc_skb(rx->adapter->netdev, RX_SKB_ALLOC_LENGTH,
-				 GFP_ATOMIC | GFP_DMA);
-	if (!skb)
+	page = page_pool_dev_alloc_pages(rx->page_pool);
+	if (unlikely(!page))
 		return -ENOMEM;
 
-	skb_reserve(skb, RX_SKB_RESERVE);
-
-	dma = dma_map_single(dmadev, skb->data, RX_SKB_LENGTH,
-			     DMA_FROM_DEVICE);
-	if (dma_mapping_error(dmadev, dma)) {
-		dev_kfree_skb(skb);
-		return -ENOMEM;
-	}
-
-	entry->skb = skb;
-	entry->len = RX_SKB_LENGTH;
-	dma_unmap_addr_set(entry, dma, dma);
-	entry->desc->rx = __cpu_to_le64(dma);
+	entry->page = page;
+	entry->len = TSNEP_MAX_RX_BUF_SIZE;
+	entry->dma = page_pool_get_dma_addr(entry->page);
+	entry->desc->rx = __cpu_to_le64(entry->dma + TSNEP_SKB_PAD);
 
 	return 0;
 }
@@ -601,6 +629,7 @@ static int tsnep_rx_ring_init(struct tsnep_rx *rx)
 {
 	struct device *dmadev = rx->adapter->dmadev;
 	struct tsnep_rx_entry *entry;
+	struct page_pool_params pp_params = { 0 };
 	struct tsnep_rx_entry *next_entry;
 	int i, j;
 	int retval;
@@ -622,12 +651,28 @@ static int tsnep_rx_ring_init(struct tsnep_rx *rx)
 			entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j;
 		}
 	}
+
+	pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
+	pp_params.order = 0;
+	pp_params.pool_size = TSNEP_RING_SIZE;
+	pp_params.nid = dev_to_node(dmadev);
+	pp_params.dev = dmadev;
+	pp_params.dma_dir = DMA_FROM_DEVICE;
+	pp_params.max_len = TSNEP_MAX_RX_BUF_SIZE;
+	pp_params.offset = TSNEP_SKB_PAD;
+	rx->page_pool = page_pool_create(&pp_params);
+	if (IS_ERR(rx->page_pool)) {
+		retval = PTR_ERR(rx->page_pool);
+		rx->page_pool = NULL;
+		goto failed;
+	}
+
 	for (i = 0; i < TSNEP_RING_SIZE; i++) {
 		entry = &rx->entry[i];
 		next_entry = &rx->entry[(i + 1) % TSNEP_RING_SIZE];
 		entry->desc->next = __cpu_to_le64(next_entry->desc_dma);
 
-		retval = tsnep_rx_alloc_and_map_skb(rx, entry);
+		retval = tsnep_rx_alloc_buffer(rx, entry);
 		if (retval)
 			goto failed;
 	}
@@ -643,7 +688,7 @@ static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
 {
 	struct tsnep_rx_entry *entry = &rx->entry[index];
 
-	/* RX_SKB_LENGTH is a multiple of 4 */
+	/* TSNEP_MAX_RX_BUF_SIZE is a multiple of 4 */
 	entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK;
 	entry->properties |= TSNEP_DESC_INTERRUPT_FLAG;
 	if (index == rx->increment_owner_counter) {
@@ -666,19 +711,52 @@ static void tsnep_rx_activate(struct tsnep_rx *rx, int index)
 	entry->desc->properties = __cpu_to_le32(entry->properties);
 }
 
+static struct sk_buff *tsnep_build_skb(struct tsnep_rx *rx, struct page *page,
+				       int length)
+{
+	struct sk_buff *skb;
+
+	skb = napi_build_skb(page_address(page), PAGE_SIZE);
+	if (unlikely(!skb))
+		return NULL;
+
+	/* update pointers within the skb to store the data */
+	skb_reserve(skb, TSNEP_SKB_PAD + TSNEP_RX_INLINE_METADATA_SIZE);
+	__skb_put(skb, length - TSNEP_RX_INLINE_METADATA_SIZE - ETH_FCS_LEN);
+
+	if (rx->adapter->hwtstamp_config.rx_filter == HWTSTAMP_FILTER_ALL) {
+		struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
+		struct tsnep_rx_inline *rx_inline =
+			(struct tsnep_rx_inline *)(page_address(page) +
+						   TSNEP_SKB_PAD);
+
+		skb_shinfo(skb)->tx_flags |=
+			SKBTX_HW_TSTAMP_NETDEV;
+		memset(hwtstamps, 0, sizeof(*hwtstamps));
+		hwtstamps->netdev_data = rx_inline;
+	}
+
+	skb_record_rx_queue(skb, rx->queue_index);
+	skb->protocol = eth_type_trans(skb, rx->adapter->netdev);
+
+	return skb;
+}
+
 static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
 			 int budget)
 {
 	struct device *dmadev = rx->adapter->dmadev;
 	int done = 0;
+	enum dma_data_direction dma_dir;
 	struct tsnep_rx_entry *entry;
+	struct page *page;
 	struct sk_buff *skb;
-	size_t len;
-	dma_addr_t dma;
 	int length;
 	bool enable = false;
 	int retval;
 
+	dma_dir = page_pool_get_dma_dir(rx->page_pool);
+
 	while (likely(done < budget)) {
 		entry = &rx->entry[rx->read];
 		if ((__le32_to_cpu(entry->desc_wb->properties) &
@@ -691,42 +769,34 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
 		 */
 		dma_rmb();
 
-		skb = entry->skb;
-		len = dma_unmap_len(entry, len);
-		dma = dma_unmap_addr(entry, dma);
+		prefetch(page_address(entry->page) + TSNEP_SKB_PAD);
+		length = __le32_to_cpu(entry->desc_wb->properties) &
+			 TSNEP_DESC_LENGTH_MASK;
+		dma_sync_single_range_for_cpu(dmadev, entry->dma, TSNEP_SKB_PAD,
+					      length, dma_dir);
+		page = entry->page;
 
 		/* forward skb only if allocation is successful, otherwise
-		 * skb is reused and frame dropped
+		 * page is reused and frame dropped
 		 */
-		retval = tsnep_rx_alloc_and_map_skb(rx, entry);
+		retval = tsnep_rx_alloc_buffer(rx, entry);
 		if (!retval) {
-			dma_unmap_single(dmadev, dma, len, DMA_FROM_DEVICE);
-
-			length = __le32_to_cpu(entry->desc_wb->properties) &
-				 TSNEP_DESC_LENGTH_MASK;
-			skb_put(skb, length - ETH_FCS_LEN);
-			if (rx->adapter->hwtstamp_config.rx_filter ==
-			    HWTSTAMP_FILTER_ALL) {
-				struct skb_shared_hwtstamps *hwtstamps =
-					skb_hwtstamps(skb);
-				struct tsnep_rx_inline *rx_inline =
-					(struct tsnep_rx_inline *)skb->data;
-
-				skb_shinfo(skb)->tx_flags |=
-					SKBTX_HW_TSTAMP_NETDEV;
-				memset(hwtstamps, 0, sizeof(*hwtstamps));
-				hwtstamps->netdev_data = rx_inline;
-			}
-			skb_pull(skb, TSNEP_RX_INLINE_METADATA_SIZE);
-			skb->protocol = eth_type_trans(skb,
-						       rx->adapter->netdev);
+			skb = tsnep_build_skb(rx, page, length);
+			if (skb) {
+				page_pool_release_page(rx->page_pool, page);
 
-			rx->packets++;
-			rx->bytes += length - TSNEP_RX_INLINE_METADATA_SIZE;
-			if (skb->pkt_type == PACKET_MULTICAST)
-				rx->multicast++;
+				rx->packets++;
+				rx->bytes += length -
+					     TSNEP_RX_INLINE_METADATA_SIZE;
+				if (skb->pkt_type == PACKET_MULTICAST)
+					rx->multicast++;
 
-			napi_gro_receive(napi, skb);
+				napi_gro_receive(napi, skb);
+			} else {
+				page_pool_recycle_direct(rx->page_pool, page);
+
+				rx->dropped++;
+			}
 			done++;
 		} else {
 			rx->dropped++;
@@ -752,7 +822,7 @@ static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi,
 }
 
 static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr,
-			 struct tsnep_rx *rx)
+			 int queue_index, struct tsnep_rx *rx)
 {
 	dma_addr_t dma;
 	int i;
@@ -761,6 +831,7 @@ static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr,
 	memset(rx, 0, sizeof(*rx));
 	rx->adapter = adapter;
 	rx->addr = addr;
+	rx->queue_index = queue_index;
 
 	retval = tsnep_rx_ring_init(rx);
 	if (retval)
@@ -821,6 +892,56 @@ static int tsnep_poll(struct napi_struct *napi, int budget)
 	return min(done, budget - 1);
 }
 
+static int tsnep_request_irq(struct tsnep_queue *queue, bool first)
+{
+	const char *name = netdev_name(queue->adapter->netdev);
+	irq_handler_t handler;
+	void *dev;
+	int retval;
+
+	if (first) {
+		sprintf(queue->name, "%s-mac", name);
+		handler = tsnep_irq;
+		dev = queue->adapter;
+	} else {
+		if (queue->tx && queue->rx)
+			sprintf(queue->name, "%s-txrx-%d", name,
+				queue->rx->queue_index);
+		else if (queue->tx)
+			sprintf(queue->name, "%s-tx-%d", name,
+				queue->tx->queue_index);
+		else
+			sprintf(queue->name, "%s-rx-%d", name,
+				queue->rx->queue_index);
+		handler = tsnep_irq_txrx;
+		dev = queue;
+	}
+
+	retval = request_irq(queue->irq, handler, 0, queue->name, dev);
+	if (retval) {
+		/* if name is empty, then interrupt won't be freed */
+		memset(queue->name, 0, sizeof(queue->name));
+	}
+
+	return retval;
+}
+
+static void tsnep_free_irq(struct tsnep_queue *queue, bool first)
+{
+	void *dev;
+
+	if (!strlen(queue->name))
+		return;
+
+	if (first)
+		dev = queue->adapter;
+	else
+		dev = queue;
+
+	free_irq(queue->irq, dev);
+	memset(queue->name, 0, sizeof(queue->name));
+}
+
 static int tsnep_netdev_open(struct net_device *netdev)
 {
 	struct tsnep_adapter *adapter = netdev_priv(netdev);
@@ -830,15 +951,11 @@ static int tsnep_netdev_open(struct net_device *netdev)
 	int rx_queue_index = 0;
 	int retval;
 
-	retval = tsnep_phy_open(adapter);
-	if (retval)
-		return retval;
-
 	for (i = 0; i < adapter->num_queues; i++) {
 		adapter->queue[i].adapter = adapter;
 		if (adapter->queue[i].tx) {
 			addr = adapter->addr + TSNEP_QUEUE(tx_queue_index);
-			retval = tsnep_tx_open(adapter, addr,
+			retval = tsnep_tx_open(adapter, addr, tx_queue_index,
 					       adapter->queue[i].tx);
 			if (retval)
 				goto failed;
@@ -847,11 +964,20 @@ static int tsnep_netdev_open(struct net_device *netdev)
 		if (adapter->queue[i].rx) {
 			addr = adapter->addr + TSNEP_QUEUE(rx_queue_index);
 			retval = tsnep_rx_open(adapter, addr,
+					       rx_queue_index,
 					       adapter->queue[i].rx);
 			if (retval)
 				goto failed;
 			rx_queue_index++;
 		}
+
+		retval = tsnep_request_irq(&adapter->queue[i], i == 0);
+		if (retval) {
+			netif_err(adapter, drv, adapter->netdev,
+				  "can't get assigned irq %d.\n",
+				  adapter->queue[i].irq);
+			goto failed;
+		}
 	}
 
 	retval = netif_set_real_num_tx_queues(adapter->netdev,
@@ -863,9 +989,14 @@ static int tsnep_netdev_open(struct net_device *netdev)
 	if (retval)
 		goto failed;
 
+	tsnep_enable_irq(adapter, ECM_INT_LINK);
+	retval = tsnep_phy_open(adapter);
+	if (retval)
+		goto phy_failed;
+
 	for (i = 0; i < adapter->num_queues; i++) {
 		netif_napi_add(adapter->netdev, &adapter->queue[i].napi,
-			       tsnep_poll, 64);
+			       tsnep_poll);
 		napi_enable(&adapter->queue[i].napi);
 
 		tsnep_enable_irq(adapter, adapter->queue[i].irq_mask);
@@ -873,14 +1004,18 @@ static int tsnep_netdev_open(struct net_device *netdev)
 
 	return 0;
 
+phy_failed:
+	tsnep_disable_irq(adapter, ECM_INT_LINK);
+	tsnep_phy_close(adapter);
 failed:
 	for (i = 0; i < adapter->num_queues; i++) {
+		tsnep_free_irq(&adapter->queue[i], i == 0);
+
 		if (adapter->queue[i].rx)
 			tsnep_rx_close(adapter->queue[i].rx);
 		if (adapter->queue[i].tx)
 			tsnep_tx_close(adapter->queue[i].tx);
 	}
-	tsnep_phy_close(adapter);
 	return retval;
 }
 
@@ -889,20 +1024,23 @@ static int tsnep_netdev_close(struct net_device *netdev)
 	struct tsnep_adapter *adapter = netdev_priv(netdev);
 	int i;
 
+	tsnep_disable_irq(adapter, ECM_INT_LINK);
+	tsnep_phy_close(adapter);
+
 	for (i = 0; i < adapter->num_queues; i++) {
 		tsnep_disable_irq(adapter, adapter->queue[i].irq_mask);
 
 		napi_disable(&adapter->queue[i].napi);
 		netif_napi_del(&adapter->queue[i].napi);
 
+		tsnep_free_irq(&adapter->queue[i], i == 0);
+
 		if (adapter->queue[i].rx)
 			tsnep_rx_close(adapter->queue[i].rx);
 		if (adapter->queue[i].tx)
 			tsnep_tx_close(adapter->queue[i].tx);
 	}
 
-	tsnep_phy_close(adapter);
-
 	return 0;
 }
 
@@ -1017,6 +1155,22 @@ static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr)
 	return 0;
 }
 
+static int tsnep_netdev_set_features(struct net_device *netdev,
+				     netdev_features_t features)
+{
+	struct tsnep_adapter *adapter = netdev_priv(netdev);
+	netdev_features_t changed = netdev->features ^ features;
+	bool enable;
+	int retval = 0;
+
+	if (changed & NETIF_F_LOOPBACK) {
+		enable = !!(features & NETIF_F_LOOPBACK);
+		retval = tsnep_phy_loopback(adapter, enable);
+	}
+
+	return retval;
+}
+
 static ktime_t tsnep_netdev_get_tstamp(struct net_device *netdev,
 				       const struct skb_shared_hwtstamps *hwtstamps,
 				       bool cycles)
@@ -1038,9 +1192,9 @@ static const struct net_device_ops tsnep_netdev_ops = {
 	.ndo_start_xmit = tsnep_netdev_xmit_frame,
 	.ndo_eth_ioctl = tsnep_netdev_ioctl,
 	.ndo_set_rx_mode = tsnep_netdev_set_multicast,
-
 	.ndo_get_stats64 = tsnep_netdev_get_stats64,
 	.ndo_set_mac_address = tsnep_netdev_set_mac_address,
+	.ndo_set_features = tsnep_netdev_set_features,
 	.ndo_get_tstamp = tsnep_netdev_get_tstamp,
 	.ndo_setup_tc = tsnep_tc_setup,
 };
@@ -1141,6 +1295,52 @@ static int tsnep_phy_init(struct tsnep_adapter *adapter)
 	return 0;
 }
 
+static int tsnep_queue_init(struct tsnep_adapter *adapter, int queue_count)
+{
+	u32 irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0;
+	char name[8];
+	int i;
+	int retval;
+
+	/* one TX/RX queue pair for netdev is mandatory */
+	if (platform_irq_count(adapter->pdev) == 1)
+		retval = platform_get_irq(adapter->pdev, 0);
+	else
+		retval = platform_get_irq_byname(adapter->pdev, "mac");
+	if (retval < 0)
+		return retval;
+	adapter->num_tx_queues = 1;
+	adapter->num_rx_queues = 1;
+	adapter->num_queues = 1;
+	adapter->queue[0].irq = retval;
+	adapter->queue[0].tx = &adapter->tx[0];
+	adapter->queue[0].rx = &adapter->rx[0];
+	adapter->queue[0].irq_mask = irq_mask;
+
+	adapter->netdev->irq = adapter->queue[0].irq;
+
+	/* add additional TX/RX queue pairs only if dedicated interrupt is
+	 * available
+	 */
+	for (i = 1; i < queue_count; i++) {
+		sprintf(name, "txrx-%d", i);
+		retval = platform_get_irq_byname_optional(adapter->pdev, name);
+		if (retval < 0)
+			break;
+
+		adapter->num_tx_queues++;
+		adapter->num_rx_queues++;
+		adapter->num_queues++;
+		adapter->queue[i].irq = retval;
+		adapter->queue[i].tx = &adapter->tx[i];
+		adapter->queue[i].rx = &adapter->rx[i];
+		adapter->queue[i].irq_mask =
+			irq_mask << (ECM_INT_TXRX_SHIFT * i);
+	}
+
+	return 0;
+}
+
 static int tsnep_probe(struct platform_device *pdev)
 {
 	struct tsnep_adapter *adapter;
@@ -1149,6 +1349,7 @@ static int tsnep_probe(struct platform_device *pdev)
 	u32 type;
 	int revision;
 	int version;
+	int queue_count;
 	int retval;
 
 	netdev = devm_alloc_etherdev_mqs(&pdev->dev,
@@ -1170,41 +1371,39 @@ static int tsnep_probe(struct platform_device *pdev)
 	netdev->max_mtu = TSNEP_MAX_FRAME_SIZE;
 
 	mutex_init(&adapter->gate_control_lock);
+	mutex_init(&adapter->rxnfc_lock);
+	INIT_LIST_HEAD(&adapter->rxnfc_rules);
 
 	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	adapter->addr = devm_ioremap_resource(&pdev->dev, io);
 	if (IS_ERR(adapter->addr))
 		return PTR_ERR(adapter->addr);
-	adapter->irq = platform_get_irq(pdev, 0);
 	netdev->mem_start = io->start;
 	netdev->mem_end = io->end;
-	netdev->irq = adapter->irq;
 
 	type = ioread32(adapter->addr + ECM_TYPE);
 	revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT;
 	version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT;
+	queue_count = (type & ECM_QUEUE_COUNT_MASK) >> ECM_QUEUE_COUNT_SHIFT;
 	adapter->gate_control = type & ECM_GATE_CONTROL;
-
-	adapter->num_tx_queues = TSNEP_QUEUES;
-	adapter->num_rx_queues = TSNEP_QUEUES;
-	adapter->num_queues = TSNEP_QUEUES;
-	adapter->queue[0].tx = &adapter->tx[0];
-	adapter->queue[0].rx = &adapter->rx[0];
-	adapter->queue[0].irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0;
+	adapter->rxnfc_max = TSNEP_RX_ASSIGN_ETHER_TYPE_COUNT;
 
 	tsnep_disable_irq(adapter, ECM_INT_ALL);
-	retval = devm_request_irq(&adapter->pdev->dev, adapter->irq, tsnep_irq,
-				  0, TSNEP, adapter);
-	if (retval != 0) {
-		dev_err(&adapter->pdev->dev, "can't get assigned irq %d.\n",
-			adapter->irq);
+
+	retval = tsnep_queue_init(adapter, queue_count);
+	if (retval)
+		return retval;
+
+	retval = dma_set_mask_and_coherent(&adapter->pdev->dev,
+					   DMA_BIT_MASK(64));
+	if (retval) {
+		dev_err(&adapter->pdev->dev, "no usable DMA configuration.\n");
 		return retval;
 	}
-	tsnep_enable_irq(adapter, ECM_INT_LINK);
 
 	retval = tsnep_mac_init(adapter);
 	if (retval)
-		goto mac_init_failed;
+		return retval;
 
 	retval = tsnep_mdio_init(adapter);
 	if (retval)
@@ -1222,10 +1421,14 @@ static int tsnep_probe(struct platform_device *pdev)
 	if (retval)
 		goto tc_init_failed;
 
+	retval = tsnep_rxnfc_init(adapter);
+	if (retval)
+		goto rxnfc_init_failed;
+
 	netdev->netdev_ops = &tsnep_netdev_ops;
 	netdev->ethtool_ops = &tsnep_ethtool_ops;
 	netdev->features = NETIF_F_SG;
-	netdev->hw_features = netdev->features;
+	netdev->hw_features = netdev->features | NETIF_F_LOOPBACK;
 
 	/* carrier off reporting is important to ethtool even BEFORE open */
 	netif_carrier_off(netdev);
@@ -1242,6 +1445,8 @@ static int tsnep_probe(struct platform_device *pdev)
 	return 0;
 
 register_failed:
+	tsnep_rxnfc_cleanup(adapter);
+rxnfc_init_failed:
 	tsnep_tc_cleanup(adapter);
 tc_init_failed:
 	tsnep_ptp_cleanup(adapter);
@@ -1250,8 +1455,6 @@ phy_init_failed:
 	if (adapter->mdiobus)
 		mdiobus_unregister(adapter->mdiobus);
 mdio_init_failed:
-mac_init_failed:
-	tsnep_disable_irq(adapter, ECM_INT_ALL);
 	return retval;
 }
 
@@ -1261,6 +1464,8 @@ static int tsnep_remove(struct platform_device *pdev)
 
 	unregister_netdev(adapter->netdev);
 
+	tsnep_rxnfc_cleanup(adapter);
+
 	tsnep_tc_cleanup(adapter);
 
 	tsnep_ptp_cleanup(adapter);
diff --git a/drivers/net/ethernet/engleder/tsnep_rxnfc.c b/drivers/net/ethernet/engleder/tsnep_rxnfc.c
new file mode 100644
index 000000000000..9ac2a0cf3833
--- /dev/null
+++ b/drivers/net/ethernet/engleder/tsnep_rxnfc.c
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2022 Gerhard Engleder <gerhard@engleder-embedded.com> */
+
+#include "tsnep.h"
+
+#define ETHER_TYPE_FULL_MASK ((__force __be16)~0)
+
+static void tsnep_enable_rule(struct tsnep_adapter *adapter,
+			      struct tsnep_rxnfc_rule *rule)
+{
+	u8 rx_assign;
+	void __iomem *addr;
+
+	rx_assign = TSNEP_RX_ASSIGN_ACTIVE;
+	rx_assign |= (rule->queue_index << TSNEP_RX_ASSIGN_QUEUE_SHIFT) &
+		     TSNEP_RX_ASSIGN_QUEUE_MASK;
+
+	addr = adapter->addr + TSNEP_RX_ASSIGN_ETHER_TYPE +
+	       TSNEP_RX_ASSIGN_ETHER_TYPE_OFFSET * rule->location;
+	iowrite16(rule->filter.ether_type, addr);
+
+	/* enable rule after all settings are done */
+	addr = adapter->addr + TSNEP_RX_ASSIGN +
+	       TSNEP_RX_ASSIGN_OFFSET * rule->location;
+	iowrite8(rx_assign, addr);
+}
+
+static void tsnep_disable_rule(struct tsnep_adapter *adapter,
+			       struct tsnep_rxnfc_rule *rule)
+{
+	void __iomem *addr;
+
+	addr = adapter->addr + TSNEP_RX_ASSIGN +
+	       TSNEP_RX_ASSIGN_OFFSET * rule->location;
+	iowrite8(0, addr);
+}
+
+static struct tsnep_rxnfc_rule *tsnep_get_rule(struct tsnep_adapter *adapter,
+					       int location)
+{
+	struct tsnep_rxnfc_rule *rule;
+
+	list_for_each_entry(rule, &adapter->rxnfc_rules, list) {
+		if (rule->location == location)
+			return rule;
+		if (rule->location > location)
+			break;
+	}
+
+	return NULL;
+}
+
+static void tsnep_add_rule(struct tsnep_adapter *adapter,
+			   struct tsnep_rxnfc_rule *rule)
+{
+	struct tsnep_rxnfc_rule *pred, *cur;
+
+	tsnep_enable_rule(adapter, rule);
+
+	pred = NULL;
+	list_for_each_entry(cur, &adapter->rxnfc_rules, list) {
+		if (cur->location >= rule->location)
+			break;
+		pred = cur;
+	}
+
+	list_add(&rule->list, pred ? &pred->list : &adapter->rxnfc_rules);
+	adapter->rxnfc_count++;
+}
+
+static void tsnep_delete_rule(struct tsnep_adapter *adapter,
+			      struct tsnep_rxnfc_rule *rule)
+{
+	tsnep_disable_rule(adapter, rule);
+
+	list_del(&rule->list);
+	adapter->rxnfc_count--;
+
+	kfree(rule);
+}
+
+static void tsnep_flush_rules(struct tsnep_adapter *adapter)
+{
+	struct tsnep_rxnfc_rule *rule, *tmp;
+
+	mutex_lock(&adapter->rxnfc_lock);
+
+	list_for_each_entry_safe(rule, tmp, &adapter->rxnfc_rules, list)
+		tsnep_delete_rule(adapter, rule);
+
+	mutex_unlock(&adapter->rxnfc_lock);
+}
+
+int tsnep_rxnfc_get_rule(struct tsnep_adapter *adapter,
+			 struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fsp = &cmd->fs;
+	struct tsnep_rxnfc_rule *rule = NULL;
+
+	cmd->data = adapter->rxnfc_max;
+
+	mutex_lock(&adapter->rxnfc_lock);
+
+	rule = tsnep_get_rule(adapter, fsp->location);
+	if (!rule) {
+		mutex_unlock(&adapter->rxnfc_lock);
+
+		return -ENOENT;
+	}
+
+	fsp->flow_type = ETHER_FLOW;
+	fsp->ring_cookie = rule->queue_index;
+
+	if (rule->filter.type == TSNEP_RXNFC_ETHER_TYPE) {
+		fsp->h_u.ether_spec.h_proto = htons(rule->filter.ether_type);
+		fsp->m_u.ether_spec.h_proto = ETHER_TYPE_FULL_MASK;
+	}
+
+	mutex_unlock(&adapter->rxnfc_lock);
+
+	return 0;
+}
+
+int tsnep_rxnfc_get_all(struct tsnep_adapter *adapter,
+			struct ethtool_rxnfc *cmd,
+			u32 *rule_locs)
+{
+	struct tsnep_rxnfc_rule *rule;
+	int count = 0;
+
+	cmd->data = adapter->rxnfc_max;
+
+	mutex_lock(&adapter->rxnfc_lock);
+
+	list_for_each_entry(rule, &adapter->rxnfc_rules, list) {
+		if (count == cmd->rule_cnt) {
+			mutex_unlock(&adapter->rxnfc_lock);
+
+			return -EMSGSIZE;
+		}
+
+		rule_locs[count] = rule->location;
+		count++;
+	}
+
+	mutex_unlock(&adapter->rxnfc_lock);
+
+	cmd->rule_cnt = count;
+
+	return 0;
+}
+
+static int tsnep_rxnfc_find_location(struct tsnep_adapter *adapter)
+{
+	struct tsnep_rxnfc_rule *tmp;
+	int location = 0;
+
+	list_for_each_entry(tmp, &adapter->rxnfc_rules, list) {
+		if (tmp->location == location)
+			location++;
+		else
+			return location;
+	}
+
+	if (location >= adapter->rxnfc_max)
+		return -ENOSPC;
+
+	return location;
+}
+
+static void tsnep_rxnfc_init_rule(struct tsnep_rxnfc_rule *rule,
+				  const struct ethtool_rx_flow_spec *fsp)
+{
+	INIT_LIST_HEAD(&rule->list);
+
+	rule->queue_index = fsp->ring_cookie;
+	rule->location = fsp->location;
+
+	rule->filter.type = TSNEP_RXNFC_ETHER_TYPE;
+	rule->filter.ether_type = ntohs(fsp->h_u.ether_spec.h_proto);
+}
+
+static int tsnep_rxnfc_check_rule(struct tsnep_adapter *adapter,
+				  struct tsnep_rxnfc_rule *rule)
+{
+	struct net_device *dev = adapter->netdev;
+	struct tsnep_rxnfc_rule *tmp;
+
+	list_for_each_entry(tmp, &adapter->rxnfc_rules, list) {
+		if (!memcmp(&rule->filter, &tmp->filter, sizeof(rule->filter)) &&
+		    tmp->location != rule->location) {
+			netdev_dbg(dev, "rule already exists\n");
+
+			return -EEXIST;
+		}
+	}
+
+	return 0;
+}
+
+int tsnep_rxnfc_add_rule(struct tsnep_adapter *adapter,
+			 struct ethtool_rxnfc *cmd)
+{
+	struct net_device *netdev = adapter->netdev;
+	struct ethtool_rx_flow_spec *fsp =
+		(struct ethtool_rx_flow_spec *)&cmd->fs;
+	struct tsnep_rxnfc_rule *rule, *old_rule;
+	int retval;
+
+	/* only EtherType is supported */
+	if (fsp->flow_type != ETHER_FLOW ||
+	    !is_zero_ether_addr(fsp->m_u.ether_spec.h_dest) ||
+	    !is_zero_ether_addr(fsp->m_u.ether_spec.h_source) ||
+	    fsp->m_u.ether_spec.h_proto != ETHER_TYPE_FULL_MASK) {
+		netdev_dbg(netdev, "only ethernet protocol is supported\n");
+
+		return -EOPNOTSUPP;
+	}
+
+	if (fsp->ring_cookie >
+	    (TSNEP_RX_ASSIGN_QUEUE_MASK >> TSNEP_RX_ASSIGN_QUEUE_SHIFT)) {
+		netdev_dbg(netdev, "invalid action\n");
+
+		return -EINVAL;
+	}
+
+	if (fsp->location != RX_CLS_LOC_ANY &&
+	    fsp->location >= adapter->rxnfc_max) {
+		netdev_dbg(netdev, "invalid location\n");
+
+		return -EINVAL;
+	}
+
+	rule = kzalloc(sizeof(*rule), GFP_KERNEL);
+	if (!rule)
+		return -ENOMEM;
+
+	mutex_lock(&adapter->rxnfc_lock);
+
+	if (fsp->location == RX_CLS_LOC_ANY) {
+		retval = tsnep_rxnfc_find_location(adapter);
+		if (retval < 0)
+			goto failed;
+		fsp->location = retval;
+	}
+
+	tsnep_rxnfc_init_rule(rule, fsp);
+
+	retval = tsnep_rxnfc_check_rule(adapter, rule);
+	if (retval)
+		goto failed;
+
+	old_rule = tsnep_get_rule(adapter, fsp->location);
+	if (old_rule)
+		tsnep_delete_rule(adapter, old_rule);
+
+	tsnep_add_rule(adapter, rule);
+
+	mutex_unlock(&adapter->rxnfc_lock);
+
+	return 0;
+
+failed:
+	mutex_unlock(&adapter->rxnfc_lock);
+	kfree(rule);
+	return retval;
+}
+
+int tsnep_rxnfc_del_rule(struct tsnep_adapter *adapter,
+			 struct ethtool_rxnfc *cmd)
+{
+	struct ethtool_rx_flow_spec *fsp =
+		(struct ethtool_rx_flow_spec *)&cmd->fs;
+	struct tsnep_rxnfc_rule *rule;
+
+	mutex_lock(&adapter->rxnfc_lock);
+
+	rule = tsnep_get_rule(adapter, fsp->location);
+	if (!rule) {
+		mutex_unlock(&adapter->rxnfc_lock);
+
+		return -ENOENT;
+	}
+
+	tsnep_delete_rule(adapter, rule);
+
+	mutex_unlock(&adapter->rxnfc_lock);
+
+	return 0;
+}
+
+int tsnep_rxnfc_init(struct tsnep_adapter *adapter)
+{
+	int i;
+
+	/* disable all rules */
+	for (i = 0; i < adapter->rxnfc_max;
+	     i += sizeof(u32) / TSNEP_RX_ASSIGN_OFFSET)
+		iowrite32(0, adapter->addr + TSNEP_RX_ASSIGN + i);
+
+	return 0;
+}
+
+void tsnep_rxnfc_cleanup(struct tsnep_adapter *adapter)
+{
+	tsnep_flush_rules(adapter);
+}
diff --git a/drivers/net/ethernet/ethoc.c b/drivers/net/ethernet/ethoc.c
index 437c5acfe222..95cbad198b4b 100644
--- a/drivers/net/ethernet/ethoc.c
+++ b/drivers/net/ethernet/ethoc.c
@@ -1224,7 +1224,7 @@ static int ethoc_probe(struct platform_device *pdev)
 	netdev->ethtool_ops = &ethoc_ethtool_ops;
 
 	/* setup NAPI */
-	netif_napi_add(netdev, &priv->napi, ethoc_poll, 64);
+	netif_napi_add(netdev, &priv->napi, ethoc_poll);
 
 	spin_lock_init(&priv->lock);
 
diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c
index c03663785a8d..a03879a27b04 100644
--- a/drivers/net/ethernet/faraday/ftgmac100.c
+++ b/drivers/net/ethernet/faraday/ftgmac100.c
@@ -1063,8 +1063,8 @@ static int ftgmac100_mdiobus_write(struct mii_bus *bus, int phy_addr,
 static void ftgmac100_get_drvinfo(struct net_device *netdev,
 				  struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, dev_name(&netdev->dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, dev_name(&netdev->dev), sizeof(info->bus_info));
 }
 
 static void
@@ -1506,7 +1506,7 @@ static int ftgmac100_open(struct net_device *netdev)
 		goto err_hw;
 
 	/* Initialize NAPI */
-	netif_napi_add(netdev, &priv->napi, ftgmac100_poll, 64);
+	netif_napi_add(netdev, &priv->napi, ftgmac100_poll);
 
 	/* Grab our interrupt */
 	err = request_irq(netdev->irq, ftgmac100_interrupt, 0, netdev->name, netdev);
@@ -1701,10 +1701,14 @@ err_register_mdiobus:
 
 static void ftgmac100_phy_disconnect(struct net_device *netdev)
 {
+	struct ftgmac100 *priv = netdev_priv(netdev);
+
 	if (!netdev->phydev)
 		return;
 
 	phy_disconnect(netdev->phydev);
+	if (of_phy_is_fixed_link(priv->dev->of_node))
+		of_phy_deregister_fixed_link(priv->dev->of_node);
 }
 
 static void ftgmac100_destroy_mdio(struct net_device *netdev)
@@ -1867,6 +1871,26 @@ static int ftgmac100_probe(struct platform_device *pdev)
 			err = -EINVAL;
 			goto err_phy_connect;
 		}
+	} else if (np && of_phy_is_fixed_link(np)) {
+		struct phy_device *phy;
+
+		err = of_phy_register_fixed_link(np);
+		if (err) {
+			dev_err(&pdev->dev, "Failed to register fixed PHY\n");
+			goto err_phy_connect;
+		}
+
+		phy = of_phy_get_and_connect(priv->netdev, np,
+					     &ftgmac100_adjust_link);
+		if (!phy) {
+			dev_err(&pdev->dev, "Failed to connect to fixed PHY\n");
+			of_phy_deregister_fixed_link(np);
+			err = -EINVAL;
+			goto err_phy_connect;
+		}
+
+		/* Display what we found */
+		phy_attached_info(phy);
 	} else if (np && of_get_property(np, "phy-handle", NULL)) {
 		struct phy_device *phy;
 
diff --git a/drivers/net/ethernet/faraday/ftmac100.c b/drivers/net/ethernet/faraday/ftmac100.c
index 8a341e2d5833..d95d78230828 100644
--- a/drivers/net/ethernet/faraday/ftmac100.c
+++ b/drivers/net/ethernet/faraday/ftmac100.c
@@ -807,8 +807,8 @@ static void ftmac100_mdio_write(struct net_device *netdev, int phy_id, int reg,
 static void ftmac100_get_drvinfo(struct net_device *netdev,
 				 struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, dev_name(&netdev->dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, dev_name(&netdev->dev), sizeof(info->bus_info));
 }
 
 static int ftmac100_get_link_ksettings(struct net_device *netdev,
@@ -1075,6 +1075,11 @@ static int ftmac100_probe(struct platform_device *pdev)
 	SET_NETDEV_DEV(netdev, &pdev->dev);
 	netdev->ethtool_ops = &ftmac100_ethtool_ops;
 	netdev->netdev_ops = &ftmac100_netdev_ops;
+	netdev->max_mtu = MAX_PKT_SIZE;
+
+	err = platform_get_ethdev_address(&pdev->dev, netdev);
+	if (err == -EPROBE_DEFER)
+		goto defer_get_mac;
 
 	platform_set_drvdata(pdev, netdev);
 
@@ -1086,7 +1091,7 @@ static int ftmac100_probe(struct platform_device *pdev)
 	spin_lock_init(&priv->tx_lock);
 
 	/* initialize NAPI */
-	netif_napi_add(netdev, &priv->napi, ftmac100_poll, 64);
+	netif_napi_add(netdev, &priv->napi, ftmac100_poll);
 
 	/* map io memory */
 	priv->res = request_mem_region(res->start, resource_size(res),
@@ -1137,6 +1142,7 @@ err_ioremap:
 	release_resource(priv->res);
 err_req_mem:
 	netif_napi_del(&priv->napi);
+defer_get_mac:
 	free_netdev(netdev);
 err_alloc_etherdev:
 	return err;
diff --git a/drivers/net/ethernet/faraday/ftmac100.h b/drivers/net/ethernet/faraday/ftmac100.h
index fe986f1673fc..8af32f9070f4 100644
--- a/drivers/net/ethernet/faraday/ftmac100.h
+++ b/drivers/net/ethernet/faraday/ftmac100.h
@@ -122,9 +122,9 @@
  * Transmit descriptor, aligned to 16 bytes
  */
 struct ftmac100_txdes {
-	unsigned int	txdes0;
-	unsigned int	txdes1;
-	unsigned int	txdes2;	/* TXBUF_BADR */
+	__le32		txdes0;
+	__le32		txdes1;
+	__le32		txdes2;	/* TXBUF_BADR */
 	unsigned int	txdes3;	/* not used by HW */
 } __attribute__ ((aligned(16)));
 
@@ -143,9 +143,9 @@ struct ftmac100_txdes {
  * Receive descriptor, aligned to 16 bytes
  */
 struct ftmac100_rxdes {
-	unsigned int	rxdes0;
-	unsigned int	rxdes1;
-	unsigned int	rxdes2;	/* RXBUF_BADR */
+	__le32		rxdes0;
+	__le32		rxdes1;
+	__le32		rxdes2;	/* RXBUF_BADR */
 	unsigned int	rxdes3;	/* not used by HW */
 } __attribute__ ((aligned(16)));
 
diff --git a/drivers/net/ethernet/fealnx.c b/drivers/net/ethernet/fealnx.c
index b3939a5f7b03..ed18450fd2cc 100644
--- a/drivers/net/ethernet/fealnx.c
+++ b/drivers/net/ethernet/fealnx.c
@@ -1809,8 +1809,8 @@ static void netdev_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *i
 {
 	struct netdev_private *np = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
 }
 
 static int netdev_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/freescale/Kconfig b/drivers/net/ethernet/freescale/Kconfig
index e04e1c5cb013..ce866ae3df03 100644
--- a/drivers/net/ethernet/freescale/Kconfig
+++ b/drivers/net/ethernet/freescale/Kconfig
@@ -9,7 +9,7 @@ config NET_VENDOR_FREESCALE
 	depends on FSL_SOC || QUICC_ENGINE || CPM1 || CPM2 || PPC_MPC512x || \
 		   M523x || M527x || M5272 || M528x || M520x || M532x || \
 		   ARCH_MXC || ARCH_MXS || (PPC_MPC52xx && PPC_BESTCOMM) || \
-		   ARCH_LAYERSCAPE || COMPILE_TEST
+		   ARCH_LAYERSCAPE || ARCH_S32 || COMPILE_TEST
 	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
@@ -23,15 +23,16 @@ if NET_VENDOR_FREESCALE
 config FEC
 	tristate "FEC ethernet controller (of ColdFire and some i.MX CPUs)"
 	depends on (M523x || M527x || M5272 || M528x || M520x || M532x || \
-		   ARCH_MXC || SOC_IMX28 || COMPILE_TEST)
+		   ARCH_MXC || ARCH_S32 || SOC_IMX28 || COMPILE_TEST)
 	default ARCH_MXC || SOC_IMX28 if ARM
 	depends on PTP_1588_CLOCK_OPTIONAL
 	select CRC32
 	select PHYLIB
+	select PAGE_POOL
 	imply NET_SELFTESTS
 	help
 	  Say Y here if you want to use the built-in 10/100 Fast ethernet
-	  controller on some Motorola ColdFire and Freescale i.MX processors.
+	  controller on some Motorola ColdFire and Freescale i.MX/S32 processors.
 
 config FEC_MPC52xx
 	tristate "FEC MPC52xx driver"
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
index a770bab4d1ed..31cfa121333d 100644
--- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c
@@ -197,12 +197,15 @@ static int dpaa_rx_extra_headroom;
 #define dpaa_get_max_mtu()	\
 	(dpaa_max_frm - (VLAN_ETH_HLEN + ETH_FCS_LEN))
 
+static void dpaa_eth_cgr_set_speed(struct mac_device *mac_dev, int speed);
+
 static int dpaa_netdev_init(struct net_device *net_dev,
 			    const struct net_device_ops *dpaa_ops,
 			    u16 tx_timeout)
 {
 	struct dpaa_priv *priv = netdev_priv(net_dev);
 	struct device *dev = net_dev->dev.parent;
+	struct mac_device *mac_dev = priv->mac_dev;
 	struct dpaa_percpu_priv *percpu_priv;
 	const u8 *mac_addr;
 	int i, err;
@@ -216,10 +219,10 @@ static int dpaa_netdev_init(struct net_device *net_dev,
 	}
 
 	net_dev->netdev_ops = dpaa_ops;
-	mac_addr = priv->mac_dev->addr;
+	mac_addr = mac_dev->addr;
 
-	net_dev->mem_start = priv->mac_dev->res->start;
-	net_dev->mem_end = priv->mac_dev->res->end;
+	net_dev->mem_start = (unsigned long)mac_dev->vaddr;
+	net_dev->mem_end = (unsigned long)mac_dev->vaddr_end;
 
 	net_dev->min_mtu = ETH_MIN_MTU;
 	net_dev->max_mtu = dpaa_get_max_mtu();
@@ -246,7 +249,7 @@ static int dpaa_netdev_init(struct net_device *net_dev,
 		eth_hw_addr_set(net_dev, mac_addr);
 	} else {
 		eth_hw_addr_random(net_dev);
-		err = priv->mac_dev->change_addr(priv->mac_dev->fman_mac,
+		err = mac_dev->change_addr(mac_dev->fman_mac,
 			(const enet_addr_t *)net_dev->dev_addr);
 		if (err) {
 			dev_err(dev, "Failed to set random MAC address\n");
@@ -261,6 +264,9 @@ static int dpaa_netdev_init(struct net_device *net_dev,
 	net_dev->needed_headroom = priv->tx_headroom;
 	net_dev->watchdog_timeo = msecs_to_jiffies(tx_timeout);
 
+	mac_dev->net_dev = net_dev;
+	mac_dev->update_speed = dpaa_eth_cgr_set_speed;
+
 	/* start without the RUNNING flag, phylib controls it later */
 	netif_carrier_off(net_dev);
 
@@ -288,10 +294,9 @@ static int dpaa_stop(struct net_device *net_dev)
 	 */
 	msleep(200);
 
-	err = mac_dev->stop(mac_dev);
-	if (err < 0)
-		netif_err(priv, ifdown, net_dev, "mac_dev->stop() = %d\n",
-			  err);
+	if (mac_dev->phy_dev)
+		phy_stop(mac_dev->phy_dev);
+	mac_dev->disable(mac_dev->fman_mac);
 
 	for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) {
 		error = fman_port_disable(mac_dev->port[i]);
@@ -826,10 +831,10 @@ static int dpaa_eth_cgr_init(struct dpaa_priv *priv)
 	initcgr.we_mask = cpu_to_be16(QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES);
 	initcgr.cgr.cscn_en = QM_CGR_EN;
 
-	/* Set different thresholds based on the MAC speed.
-	 * This may turn suboptimal if the MAC is reconfigured at a speed
-	 * lower than its max, e.g. if a dTSEC later negotiates a 100Mbps link.
-	 * In such cases, we ought to reconfigure the threshold, too.
+	/* Set different thresholds based on the configured MAC speed.
+	 * This may turn suboptimal if the MAC is reconfigured at another
+	 * speed, so MACs must call dpaa_eth_cgr_set_speed in their adjust_link
+	 * callback.
 	 */
 	if (priv->mac_dev->if_support & SUPPORTED_10000baseT_Full)
 		cs_th = DPAA_CS_THRESHOLD_10G;
@@ -858,6 +863,31 @@ out_error:
 	return err;
 }
 
+static void dpaa_eth_cgr_set_speed(struct mac_device *mac_dev, int speed)
+{
+	struct net_device *net_dev = mac_dev->net_dev;
+	struct dpaa_priv *priv = netdev_priv(net_dev);
+	struct qm_mcc_initcgr opts = { };
+	u32 cs_th;
+	int err;
+
+	opts.we_mask = cpu_to_be16(QM_CGR_WE_CS_THRES);
+	switch (speed) {
+	case SPEED_10000:
+		cs_th = DPAA_CS_THRESHOLD_10G;
+		break;
+	case SPEED_1000:
+	default:
+		cs_th = DPAA_CS_THRESHOLD_1G;
+		break;
+	}
+	qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, cs_th, 1);
+
+	err = qman_update_cgr_safe(&priv->cgr_data.cgr, &opts);
+	if (err)
+		netdev_err(net_dev, "could not update speed: %d\n", err);
+}
+
 static inline void dpaa_setup_ingress(const struct dpaa_priv *priv,
 				      struct dpaa_fq *fq,
 				      const struct qman_fq *template)
@@ -2946,11 +2976,12 @@ static int dpaa_open(struct net_device *net_dev)
 			goto mac_start_failed;
 	}
 
-	err = priv->mac_dev->start(mac_dev);
+	err = priv->mac_dev->enable(mac_dev->fman_mac);
 	if (err < 0) {
-		netif_err(priv, ifup, net_dev, "mac_dev->start() = %d\n", err);
+		netif_err(priv, ifup, net_dev, "mac_dev->enable() = %d\n", err);
 		goto mac_start_failed;
 	}
+	phy_start(priv->mac_dev->phy_dev);
 
 	netif_tx_start_all_queues(net_dev);
 
@@ -3152,8 +3183,7 @@ static int dpaa_napi_add(struct net_device *net_dev)
 	for_each_possible_cpu(cpu) {
 		percpu_priv = per_cpu_ptr(priv->percpu_priv, cpu);
 
-		netif_napi_add(net_dev, &percpu_priv->np.napi,
-			       dpaa_eth_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(net_dev, &percpu_priv->np.napi, dpaa_eth_poll);
 	}
 
 	return 0;
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c
index 4fee74c024bd..258eb6c8f4c0 100644
--- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth_sysfs.c
@@ -18,7 +18,7 @@ static ssize_t dpaa_eth_show_addr(struct device *dev,
 
 	if (mac_dev)
 		return sprintf(buf, "%llx",
-				(unsigned long long)mac_dev->res->start);
+				(unsigned long long)mac_dev->vaddr);
 	else
 		return sprintf(buf, "none");
 }
diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c
index 73f07881ce2d..769e936a263c 100644
--- a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c
+++ b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c
@@ -80,9 +80,9 @@ static int dpaa_set_link_ksettings(struct net_device *net_dev,
 static void dpaa_get_drvinfo(struct net_device *net_dev,
 			     struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, KBUILD_MODNAME,
+	strscpy(drvinfo->driver, KBUILD_MODNAME,
 		sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, dev_name(net_dev->dev.parent->parent),
+	strscpy(drvinfo->bus_info, dev_name(net_dev->dev.parent->parent),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
index 75d51572693d..8d029addddad 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
+++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
@@ -4565,8 +4565,7 @@ static void dpaa2_eth_add_ch_napi(struct dpaa2_eth_priv *priv)
 	for (i = 0; i < priv->num_channels; i++) {
 		ch = priv->channel[i];
 		/* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */
-		netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll);
 	}
 }
 
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c
index c9bee9a0c9b2..49ff85633783 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c
+++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c
@@ -549,7 +549,7 @@ void dpaa2_mac_get_strings(u8 *data)
 	int i;
 
 	for (i = 0; i < DPAA2_MAC_NUM_STATS; i++) {
-		strlcpy(p, dpaa2_mac_ethtool_stats[i], ETH_GSTRING_LEN);
+		strscpy(p, dpaa2_mac_ethtool_stats[i], ETH_GSTRING_LEN);
 		p += ETH_GSTRING_LEN;
 	}
 }
diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c
index e507e9065214..2b5909fa93cf 100644
--- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c
+++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch.c
@@ -3373,9 +3373,8 @@ static int dpaa2_switch_probe(struct fsl_mc_device *sw_dev)
 	 * different queues for each switch ports.
 	 */
 	for (i = 0; i < DPAA2_SWITCH_RX_NUM_FQS; i++)
-		netif_napi_add(ethsw->ports[0]->netdev,
-			       &ethsw->fq[i].napi, dpaa2_switch_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(ethsw->ports[0]->netdev, &ethsw->fq[i].napi,
+			       dpaa2_switch_poll);
 
 	/* Setup IRQs */
 	err = dpaa2_switch_setup_irqs(sw_dev);
diff --git a/drivers/net/ethernet/freescale/enetc/enetc.c b/drivers/net/ethernet/freescale/enetc/enetc.c
index 9f5b921039bd..54bc92fc6bf0 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc.c
@@ -2116,13 +2116,14 @@ static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
 
 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
 {
+	struct enetc_hw *hw = &priv->si->hw;
 	int i;
 
 	for (i = 0; i < priv->num_tx_rings; i++)
-		enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]);
+		enetc_setup_txbdr(hw, priv->tx_ring[i]);
 
 	for (i = 0; i < priv->num_rx_rings; i++)
-		enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]);
+		enetc_setup_rxbdr(hw, priv->rx_ring[i]);
 }
 
 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
@@ -2155,13 +2156,14 @@ static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
 
 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
 {
+	struct enetc_hw *hw = &priv->si->hw;
 	int i;
 
 	for (i = 0; i < priv->num_tx_rings; i++)
-		enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]);
+		enetc_clear_txbdr(hw, priv->tx_ring[i]);
 
 	for (i = 0; i < priv->num_rx_rings; i++)
-		enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]);
+		enetc_clear_rxbdr(hw, priv->rx_ring[i]);
 
 	udelay(1);
 }
@@ -2169,13 +2171,13 @@ static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
 {
 	struct pci_dev *pdev = priv->si->pdev;
+	struct enetc_hw *hw = &priv->si->hw;
 	int i, j, err;
 
 	for (i = 0; i < priv->bdr_int_num; i++) {
 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
 		struct enetc_int_vector *v = priv->int_vector[i];
 		int entry = ENETC_BDR_INT_BASE_IDX + i;
-		struct enetc_hw *hw = &priv->si->hw;
 
 		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
 			 priv->ndev->name, i);
@@ -2263,13 +2265,14 @@ static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
 
 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
 {
+	struct enetc_hw *hw = &priv->si->hw;
 	int i;
 
 	for (i = 0; i < priv->num_tx_rings; i++)
-		enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0);
+		enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
 
 	for (i = 0; i < priv->num_rx_rings; i++)
-		enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0);
+		enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
 }
 
 static int enetc_phylink_connect(struct net_device *ndev)
@@ -2436,6 +2439,7 @@ int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
 {
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
 	struct tc_mqprio_qopt *mqprio = type_data;
+	struct enetc_hw *hw = &priv->si->hw;
 	struct enetc_bdr *tx_ring;
 	int num_stack_tx_queues;
 	u8 num_tc;
@@ -2452,7 +2456,7 @@ int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
 		/* Reset all ring priorities to 0 */
 		for (i = 0; i < priv->num_tx_rings; i++) {
 			tx_ring = priv->tx_ring[i];
-			enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0);
+			enetc_set_bdr_prio(hw, tx_ring->index, 0);
 		}
 
 		return 0;
@@ -2471,7 +2475,7 @@ int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
 	 */
 	for (i = 0; i < num_tc; i++) {
 		tx_ring = priv->tx_ring[i];
-		enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i);
+		enetc_set_bdr_prio(hw, tx_ring->index, i);
 	}
 
 	/* Reset the number of netdev queues based on the TC count */
@@ -2584,19 +2588,21 @@ static int enetc_set_rss(struct net_device *ndev, int en)
 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
 {
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_hw *hw = &priv->si->hw;
 	int i;
 
 	for (i = 0; i < priv->num_rx_rings; i++)
-		enetc_bdr_enable_rxvlan(&priv->si->hw, i, en);
+		enetc_bdr_enable_rxvlan(hw, i, en);
 }
 
 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
 {
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_hw *hw = &priv->si->hw;
 	int i;
 
 	for (i = 0; i < priv->num_tx_rings; i++)
-		enetc_bdr_enable_txvlan(&priv->si->hw, i, en);
+		enetc_bdr_enable_txvlan(hw, i, en);
 }
 
 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
@@ -2759,8 +2765,7 @@ int enetc_alloc_msix(struct enetc_ndev_priv *priv)
 			v->rx_dim_en = true;
 		}
 		INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
-		netif_napi_add(priv->ndev, &v->napi, enetc_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(priv->ndev, &v->napi, enetc_poll);
 		v->count_tx_rings = v_tx_rings;
 
 		for (j = 0; j < v_tx_rings; j++) {
diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h
index 2cfe6944ebd3..161930a65f61 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc.h
@@ -453,7 +453,11 @@ static inline void enetc_cbd_free_data_mem(struct enetc_si *si, int size,
 			  data, *dma);
 }
 
+void enetc_reset_ptcmsdur(struct enetc_hw *hw);
+void enetc_set_ptcmsdur(struct enetc_hw *hw, u32 *queue_max_sdu);
+
 #ifdef CONFIG_FSL_ENETC_QOS
+int enetc_qos_query_caps(struct net_device *ndev, void *type_data);
 int enetc_setup_tc_taprio(struct net_device *ndev, void *type_data);
 void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed);
 int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data);
@@ -467,19 +471,20 @@ int enetc_set_psfp(struct net_device *ndev, bool en);
 
 static inline void enetc_get_max_cap(struct enetc_ndev_priv *priv)
 {
+	struct enetc_hw *hw = &priv->si->hw;
 	u32 reg;
 
-	reg = enetc_port_rd(&priv->si->hw, ENETC_PSIDCAPR);
+	reg = enetc_port_rd(hw, ENETC_PSIDCAPR);
 	priv->psfp_cap.max_streamid = reg & ENETC_PSIDCAPR_MSK;
 	/* Port stream filter capability */
-	reg = enetc_port_rd(&priv->si->hw, ENETC_PSFCAPR);
+	reg = enetc_port_rd(hw, ENETC_PSFCAPR);
 	priv->psfp_cap.max_psfp_filter = reg & ENETC_PSFCAPR_MSK;
 	/* Port stream gate capability */
-	reg = enetc_port_rd(&priv->si->hw, ENETC_PSGCAPR);
+	reg = enetc_port_rd(hw, ENETC_PSGCAPR);
 	priv->psfp_cap.max_psfp_gate = (reg & ENETC_PSGCAPR_SGIT_MSK);
 	priv->psfp_cap.max_psfp_gatelist = (reg & ENETC_PSGCAPR_GCL_MSK) >> 16;
 	/* Port flow meter capability */
-	reg = enetc_port_rd(&priv->si->hw, ENETC_PFMCAPR);
+	reg = enetc_port_rd(hw, ENETC_PFMCAPR);
 	priv->psfp_cap.max_psfp_meter = reg & ENETC_PFMCAPR_MSK;
 }
 
@@ -520,6 +525,7 @@ static inline int enetc_psfp_disable(struct enetc_ndev_priv *priv)
 }
 
 #else
+#define enetc_qos_query_caps(ndev, type_data) -EOPNOTSUPP
 #define enetc_setup_tc_taprio(ndev, type_data) -EOPNOTSUPP
 #define enetc_sched_speed_set(priv, speed) (void)0
 #define enetc_setup_tc_cbs(ndev, type_data) -EOPNOTSUPP
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
index ff872e40ce85..c8369e3752b0 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c
@@ -125,68 +125,68 @@ static const struct {
 	int reg;
 	char name[ETH_GSTRING_LEN];
 } enetc_port_counters[] = {
-	{ ENETC_PM0_REOCT,  "MAC rx ethernet octets" },
-	{ ENETC_PM0_RALN,   "MAC rx alignment errors" },
-	{ ENETC_PM0_RXPF,   "MAC rx valid pause frames" },
-	{ ENETC_PM0_RFRM,   "MAC rx valid frames" },
-	{ ENETC_PM0_RFCS,   "MAC rx fcs errors" },
-	{ ENETC_PM0_RVLAN,  "MAC rx VLAN frames" },
-	{ ENETC_PM0_RERR,   "MAC rx frame errors" },
-	{ ENETC_PM0_RUCA,   "MAC rx unicast frames" },
-	{ ENETC_PM0_RMCA,   "MAC rx multicast frames" },
-	{ ENETC_PM0_RBCA,   "MAC rx broadcast frames" },
-	{ ENETC_PM0_RDRP,   "MAC rx dropped packets" },
-	{ ENETC_PM0_RPKT,   "MAC rx packets" },
-	{ ENETC_PM0_RUND,   "MAC rx undersized packets" },
-	{ ENETC_PM0_R64,    "MAC rx 64 byte packets" },
-	{ ENETC_PM0_R127,   "MAC rx 65-127 byte packets" },
-	{ ENETC_PM0_R255,   "MAC rx 128-255 byte packets" },
-	{ ENETC_PM0_R511,   "MAC rx 256-511 byte packets" },
-	{ ENETC_PM0_R1023,  "MAC rx 512-1023 byte packets" },
-	{ ENETC_PM0_R1522,  "MAC rx 1024-1522 byte packets" },
-	{ ENETC_PM0_R1523X, "MAC rx 1523 to max-octet packets" },
-	{ ENETC_PM0_ROVR,   "MAC rx oversized packets" },
-	{ ENETC_PM0_RJBR,   "MAC rx jabber packets" },
-	{ ENETC_PM0_RFRG,   "MAC rx fragment packets" },
-	{ ENETC_PM0_RCNP,   "MAC rx control packets" },
-	{ ENETC_PM0_RDRNTP, "MAC rx fifo drop" },
-	{ ENETC_PM0_TEOCT,  "MAC tx ethernet octets" },
-	{ ENETC_PM0_TOCT,   "MAC tx octets" },
-	{ ENETC_PM0_TCRSE,  "MAC tx carrier sense errors" },
-	{ ENETC_PM0_TXPF,   "MAC tx valid pause frames" },
-	{ ENETC_PM0_TFRM,   "MAC tx frames" },
-	{ ENETC_PM0_TFCS,   "MAC tx fcs errors" },
-	{ ENETC_PM0_TVLAN,  "MAC tx VLAN frames" },
-	{ ENETC_PM0_TERR,   "MAC tx frame errors" },
-	{ ENETC_PM0_TUCA,   "MAC tx unicast frames" },
-	{ ENETC_PM0_TMCA,   "MAC tx multicast frames" },
-	{ ENETC_PM0_TBCA,   "MAC tx broadcast frames" },
-	{ ENETC_PM0_TPKT,   "MAC tx packets" },
-	{ ENETC_PM0_TUND,   "MAC tx undersized packets" },
-	{ ENETC_PM0_T64,    "MAC tx 64 byte packets" },
-	{ ENETC_PM0_T127,   "MAC tx 65-127 byte packets" },
-	{ ENETC_PM0_T255,   "MAC tx 128-255 byte packets" },
-	{ ENETC_PM0_T511,   "MAC tx 256-511 byte packets" },
-	{ ENETC_PM0_T1023,  "MAC tx 512-1023 byte packets" },
-	{ ENETC_PM0_T1522,  "MAC tx 1024-1522 byte packets" },
-	{ ENETC_PM0_T1523X, "MAC tx 1523 to max-octet packets" },
-	{ ENETC_PM0_TCNP,   "MAC tx control packets" },
-	{ ENETC_PM0_TDFR,   "MAC tx deferred packets" },
-	{ ENETC_PM0_TMCOL,  "MAC tx multiple collisions" },
-	{ ENETC_PM0_TSCOL,  "MAC tx single collisions" },
-	{ ENETC_PM0_TLCOL,  "MAC tx late collisions" },
-	{ ENETC_PM0_TECOL,  "MAC tx excessive collisions" },
-	{ ENETC_UFDMF,      "SI MAC nomatch u-cast discards" },
-	{ ENETC_MFDMF,      "SI MAC nomatch m-cast discards" },
-	{ ENETC_PBFDSIR,    "SI MAC nomatch b-cast discards" },
-	{ ENETC_PUFDVFR,    "SI VLAN nomatch u-cast discards" },
-	{ ENETC_PMFDVFR,    "SI VLAN nomatch m-cast discards" },
-	{ ENETC_PBFDVFR,    "SI VLAN nomatch b-cast discards" },
-	{ ENETC_PFDMSAPR,   "SI pruning discarded frames" },
-	{ ENETC_PICDR(0),   "ICM DR0 discarded frames" },
-	{ ENETC_PICDR(1),   "ICM DR1 discarded frames" },
-	{ ENETC_PICDR(2),   "ICM DR2 discarded frames" },
-	{ ENETC_PICDR(3),   "ICM DR3 discarded frames" },
+	{ ENETC_PM_REOCT(0),	"MAC rx ethernet octets" },
+	{ ENETC_PM_RALN(0),	"MAC rx alignment errors" },
+	{ ENETC_PM_RXPF(0),	"MAC rx valid pause frames" },
+	{ ENETC_PM_RFRM(0),	"MAC rx valid frames" },
+	{ ENETC_PM_RFCS(0),	"MAC rx fcs errors" },
+	{ ENETC_PM_RVLAN(0),	"MAC rx VLAN frames" },
+	{ ENETC_PM_RERR(0),	"MAC rx frame errors" },
+	{ ENETC_PM_RUCA(0),	"MAC rx unicast frames" },
+	{ ENETC_PM_RMCA(0),	"MAC rx multicast frames" },
+	{ ENETC_PM_RBCA(0),	"MAC rx broadcast frames" },
+	{ ENETC_PM_RDRP(0),	"MAC rx dropped packets" },
+	{ ENETC_PM_RPKT(0),	"MAC rx packets" },
+	{ ENETC_PM_RUND(0),	"MAC rx undersized packets" },
+	{ ENETC_PM_R64(0),	"MAC rx 64 byte packets" },
+	{ ENETC_PM_R127(0),	"MAC rx 65-127 byte packets" },
+	{ ENETC_PM_R255(0),	"MAC rx 128-255 byte packets" },
+	{ ENETC_PM_R511(0),	"MAC rx 256-511 byte packets" },
+	{ ENETC_PM_R1023(0),	"MAC rx 512-1023 byte packets" },
+	{ ENETC_PM_R1522(0),	"MAC rx 1024-1522 byte packets" },
+	{ ENETC_PM_R1523X(0),	"MAC rx 1523 to max-octet packets" },
+	{ ENETC_PM_ROVR(0),	"MAC rx oversized packets" },
+	{ ENETC_PM_RJBR(0),	"MAC rx jabber packets" },
+	{ ENETC_PM_RFRG(0),	"MAC rx fragment packets" },
+	{ ENETC_PM_RCNP(0),	"MAC rx control packets" },
+	{ ENETC_PM_RDRNTP(0),	"MAC rx fifo drop" },
+	{ ENETC_PM_TEOCT(0),	"MAC tx ethernet octets" },
+	{ ENETC_PM_TOCT(0),	"MAC tx octets" },
+	{ ENETC_PM_TCRSE(0),	"MAC tx carrier sense errors" },
+	{ ENETC_PM_TXPF(0),	"MAC tx valid pause frames" },
+	{ ENETC_PM_TFRM(0),	"MAC tx frames" },
+	{ ENETC_PM_TFCS(0),	"MAC tx fcs errors" },
+	{ ENETC_PM_TVLAN(0),	"MAC tx VLAN frames" },
+	{ ENETC_PM_TERR(0),	"MAC tx frame errors" },
+	{ ENETC_PM_TUCA(0),	"MAC tx unicast frames" },
+	{ ENETC_PM_TMCA(0),	"MAC tx multicast frames" },
+	{ ENETC_PM_TBCA(0),	"MAC tx broadcast frames" },
+	{ ENETC_PM_TPKT(0),	"MAC tx packets" },
+	{ ENETC_PM_TUND(0),	"MAC tx undersized packets" },
+	{ ENETC_PM_T64(0),	"MAC tx 64 byte packets" },
+	{ ENETC_PM_T127(0),	"MAC tx 65-127 byte packets" },
+	{ ENETC_PM_T255(0),	"MAC tx 128-255 byte packets" },
+	{ ENETC_PM_T511(0),	"MAC tx 256-511 byte packets" },
+	{ ENETC_PM_T1023(0),	"MAC tx 512-1023 byte packets" },
+	{ ENETC_PM_T1522(0),	"MAC tx 1024-1522 byte packets" },
+	{ ENETC_PM_T1523X(0),	"MAC tx 1523 to max-octet packets" },
+	{ ENETC_PM_TCNP(0),	"MAC tx control packets" },
+	{ ENETC_PM_TDFR(0),	"MAC tx deferred packets" },
+	{ ENETC_PM_TMCOL(0),	"MAC tx multiple collisions" },
+	{ ENETC_PM_TSCOL(0),	"MAC tx single collisions" },
+	{ ENETC_PM_TLCOL(0),	"MAC tx late collisions" },
+	{ ENETC_PM_TECOL(0),	"MAC tx excessive collisions" },
+	{ ENETC_UFDMF,		"SI MAC nomatch u-cast discards" },
+	{ ENETC_MFDMF,		"SI MAC nomatch m-cast discards" },
+	{ ENETC_PBFDSIR,	"SI MAC nomatch b-cast discards" },
+	{ ENETC_PUFDVFR,	"SI VLAN nomatch u-cast discards" },
+	{ ENETC_PMFDVFR,	"SI VLAN nomatch m-cast discards" },
+	{ ENETC_PBFDVFR,	"SI VLAN nomatch b-cast discards" },
+	{ ENETC_PFDMSAPR,	"SI pruning discarded frames" },
+	{ ENETC_PICDR(0),	"ICM DR0 discarded frames" },
+	{ ENETC_PICDR(1),	"ICM DR1 discarded frames" },
+	{ ENETC_PICDR(2),	"ICM DR2 discarded frames" },
+	{ ENETC_PICDR(3),	"ICM DR3 discarded frames" },
 };
 
 static const char rx_ring_stats[][ETH_GSTRING_LEN] = {
@@ -236,7 +236,7 @@ static void enetc_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
 	switch (stringset) {
 	case ETH_SS_STATS:
 		for (i = 0; i < ARRAY_SIZE(enetc_si_counters); i++) {
-			strlcpy(p, enetc_si_counters[i].name, ETH_GSTRING_LEN);
+			strscpy(p, enetc_si_counters[i].name, ETH_GSTRING_LEN);
 			p += ETH_GSTRING_LEN;
 		}
 		for (i = 0; i < priv->num_tx_rings; i++) {
@@ -258,7 +258,7 @@ static void enetc_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
 			break;
 
 		for (i = 0; i < ARRAY_SIZE(enetc_port_counters); i++) {
-			strlcpy(p, enetc_port_counters[i].name,
+			strscpy(p, enetc_port_counters[i].name,
 				ETH_GSTRING_LEN);
 			p += ETH_GSTRING_LEN;
 		}
@@ -301,6 +301,113 @@ static void enetc_get_ethtool_stats(struct net_device *ndev,
 		data[o++] = enetc_port_rd(hw, enetc_port_counters[i].reg);
 }
 
+static void enetc_get_pause_stats(struct net_device *ndev,
+				  struct ethtool_pause_stats *pause_stats)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_hw *hw = &priv->si->hw;
+
+	pause_stats->tx_pause_frames = enetc_port_rd(hw, ENETC_PM_TXPF(0));
+	pause_stats->rx_pause_frames = enetc_port_rd(hw, ENETC_PM_RXPF(0));
+}
+
+static void enetc_mac_stats(struct enetc_hw *hw, int mac,
+			    struct ethtool_eth_mac_stats *s)
+{
+	s->FramesTransmittedOK = enetc_port_rd(hw, ENETC_PM_TFRM(mac));
+	s->SingleCollisionFrames = enetc_port_rd(hw, ENETC_PM_TSCOL(mac));
+	s->MultipleCollisionFrames = enetc_port_rd(hw, ENETC_PM_TMCOL(mac));
+	s->FramesReceivedOK = enetc_port_rd(hw, ENETC_PM_RFRM(mac));
+	s->FrameCheckSequenceErrors = enetc_port_rd(hw, ENETC_PM_RFCS(mac));
+	s->AlignmentErrors = enetc_port_rd(hw, ENETC_PM_RALN(mac));
+	s->OctetsTransmittedOK = enetc_port_rd(hw, ENETC_PM_TEOCT(mac));
+	s->FramesWithDeferredXmissions = enetc_port_rd(hw, ENETC_PM_TDFR(mac));
+	s->LateCollisions = enetc_port_rd(hw, ENETC_PM_TLCOL(mac));
+	s->FramesAbortedDueToXSColls = enetc_port_rd(hw, ENETC_PM_TECOL(mac));
+	s->FramesLostDueToIntMACXmitError = enetc_port_rd(hw, ENETC_PM_TERR(mac));
+	s->CarrierSenseErrors = enetc_port_rd(hw, ENETC_PM_TCRSE(mac));
+	s->OctetsReceivedOK = enetc_port_rd(hw, ENETC_PM_REOCT(mac));
+	s->FramesLostDueToIntMACRcvError = enetc_port_rd(hw, ENETC_PM_RDRNTP(mac));
+	s->MulticastFramesXmittedOK = enetc_port_rd(hw, ENETC_PM_TMCA(mac));
+	s->BroadcastFramesXmittedOK = enetc_port_rd(hw, ENETC_PM_TBCA(mac));
+	s->MulticastFramesReceivedOK = enetc_port_rd(hw, ENETC_PM_RMCA(mac));
+	s->BroadcastFramesReceivedOK = enetc_port_rd(hw, ENETC_PM_RBCA(mac));
+}
+
+static void enetc_ctrl_stats(struct enetc_hw *hw, int mac,
+			     struct ethtool_eth_ctrl_stats *s)
+{
+	s->MACControlFramesTransmitted = enetc_port_rd(hw, ENETC_PM_TCNP(mac));
+	s->MACControlFramesReceived = enetc_port_rd(hw, ENETC_PM_RCNP(mac));
+}
+
+static const struct ethtool_rmon_hist_range enetc_rmon_ranges[] = {
+	{   64,   64 },
+	{   65,  127 },
+	{  128,  255 },
+	{  256,  511 },
+	{  512, 1023 },
+	{ 1024, 1522 },
+	{ 1523, ENETC_MAC_MAXFRM_SIZE },
+	{},
+};
+
+static void enetc_rmon_stats(struct enetc_hw *hw, int mac,
+			     struct ethtool_rmon_stats *s,
+			     const struct ethtool_rmon_hist_range **ranges)
+{
+	s->undersize_pkts = enetc_port_rd(hw, ENETC_PM_RUND(mac));
+	s->oversize_pkts = enetc_port_rd(hw, ENETC_PM_ROVR(mac));
+	s->fragments = enetc_port_rd(hw, ENETC_PM_RFRG(mac));
+	s->jabbers = enetc_port_rd(hw, ENETC_PM_RJBR(mac));
+
+	s->hist[0] = enetc_port_rd(hw, ENETC_PM_R64(mac));
+	s->hist[1] = enetc_port_rd(hw, ENETC_PM_R127(mac));
+	s->hist[2] = enetc_port_rd(hw, ENETC_PM_R255(mac));
+	s->hist[3] = enetc_port_rd(hw, ENETC_PM_R511(mac));
+	s->hist[4] = enetc_port_rd(hw, ENETC_PM_R1023(mac));
+	s->hist[5] = enetc_port_rd(hw, ENETC_PM_R1522(mac));
+	s->hist[6] = enetc_port_rd(hw, ENETC_PM_R1523X(mac));
+
+	s->hist_tx[0] = enetc_port_rd(hw, ENETC_PM_T64(mac));
+	s->hist_tx[1] = enetc_port_rd(hw, ENETC_PM_T127(mac));
+	s->hist_tx[2] = enetc_port_rd(hw, ENETC_PM_T255(mac));
+	s->hist_tx[3] = enetc_port_rd(hw, ENETC_PM_T511(mac));
+	s->hist_tx[4] = enetc_port_rd(hw, ENETC_PM_T1023(mac));
+	s->hist_tx[5] = enetc_port_rd(hw, ENETC_PM_T1522(mac));
+	s->hist_tx[6] = enetc_port_rd(hw, ENETC_PM_T1523X(mac));
+
+	*ranges = enetc_rmon_ranges;
+}
+
+static void enetc_get_eth_mac_stats(struct net_device *ndev,
+				    struct ethtool_eth_mac_stats *mac_stats)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_hw *hw = &priv->si->hw;
+
+	enetc_mac_stats(hw, 0, mac_stats);
+}
+
+static void enetc_get_eth_ctrl_stats(struct net_device *ndev,
+				     struct ethtool_eth_ctrl_stats *ctrl_stats)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_hw *hw = &priv->si->hw;
+
+	enetc_ctrl_stats(hw, 0, ctrl_stats);
+}
+
+static void enetc_get_rmon_stats(struct net_device *ndev,
+				 struct ethtool_rmon_stats *rmon_stats,
+				 const struct ethtool_rmon_hist_range **ranges)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_hw *hw = &priv->si->hw;
+
+	enetc_rmon_stats(hw, 0, rmon_stats, ranges);
+}
+
 #define ENETC_RSSHASH_L3 (RXH_L2DA | RXH_VLAN | RXH_L3_PROTO | RXH_IP_SRC | \
 			  RXH_IP_DST)
 #define ENETC_RSSHASH_L4 (ENETC_RSSHASH_L3 | RXH_L4_B_0_1 | RXH_L4_B_2_3)
@@ -766,6 +873,10 @@ static const struct ethtool_ops enetc_pf_ethtool_ops = {
 	.get_sset_count = enetc_get_sset_count,
 	.get_strings = enetc_get_strings,
 	.get_ethtool_stats = enetc_get_ethtool_stats,
+	.get_pause_stats = enetc_get_pause_stats,
+	.get_rmon_stats = enetc_get_rmon_stats,
+	.get_eth_ctrl_stats = enetc_get_eth_ctrl_stats,
+	.get_eth_mac_stats = enetc_get_eth_mac_stats,
 	.get_rxnfc = enetc_get_rxnfc,
 	.set_rxnfc = enetc_set_rxnfc,
 	.get_rxfh_key_size = enetc_get_rxfh_key_size,
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_hw.h b/drivers/net/ethernet/freescale/enetc/enetc_hw.h
index 647c87f73bf7..18ca1f42b1f7 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_hw.h
+++ b/drivers/net/ethernet/freescale/enetc/enetc_hw.h
@@ -276,58 +276,60 @@ enum enetc_bdr_type {TX, RX};
 #define ENETC_PFMCAPR		0x1b38
 #define ENETC_PFMCAPR_MSK	GENMASK(15, 0)
 
-/* MAC counters */
-#define ENETC_PM0_REOCT		0x8100
-#define ENETC_PM0_RALN		0x8110
-#define ENETC_PM0_RXPF		0x8118
-#define ENETC_PM0_RFRM		0x8120
-#define ENETC_PM0_RFCS		0x8128
-#define ENETC_PM0_RVLAN		0x8130
-#define ENETC_PM0_RERR		0x8138
-#define ENETC_PM0_RUCA		0x8140
-#define ENETC_PM0_RMCA		0x8148
-#define ENETC_PM0_RBCA		0x8150
-#define ENETC_PM0_RDRP		0x8158
-#define ENETC_PM0_RPKT		0x8160
-#define ENETC_PM0_RUND		0x8168
-#define ENETC_PM0_R64		0x8170
-#define ENETC_PM0_R127		0x8178
-#define ENETC_PM0_R255		0x8180
-#define ENETC_PM0_R511		0x8188
-#define ENETC_PM0_R1023		0x8190
-#define ENETC_PM0_R1522		0x8198
-#define ENETC_PM0_R1523X	0x81A0
-#define ENETC_PM0_ROVR		0x81A8
-#define ENETC_PM0_RJBR		0x81B0
-#define ENETC_PM0_RFRG		0x81B8
-#define ENETC_PM0_RCNP		0x81C0
-#define ENETC_PM0_RDRNTP	0x81C8
-#define ENETC_PM0_TEOCT		0x8200
-#define ENETC_PM0_TOCT		0x8208
-#define ENETC_PM0_TCRSE		0x8210
-#define ENETC_PM0_TXPF		0x8218
-#define ENETC_PM0_TFRM		0x8220
-#define ENETC_PM0_TFCS		0x8228
-#define ENETC_PM0_TVLAN		0x8230
-#define ENETC_PM0_TERR		0x8238
-#define ENETC_PM0_TUCA		0x8240
-#define ENETC_PM0_TMCA		0x8248
-#define ENETC_PM0_TBCA		0x8250
-#define ENETC_PM0_TPKT		0x8260
-#define ENETC_PM0_TUND		0x8268
-#define ENETC_PM0_T64		0x8270
-#define ENETC_PM0_T127		0x8278
-#define ENETC_PM0_T255		0x8280
-#define ENETC_PM0_T511		0x8288
-#define ENETC_PM0_T1023		0x8290
-#define ENETC_PM0_T1522		0x8298
-#define ENETC_PM0_T1523X	0x82A0
-#define ENETC_PM0_TCNP		0x82C0
-#define ENETC_PM0_TDFR		0x82D0
-#define ENETC_PM0_TMCOL		0x82D8
-#define ENETC_PM0_TSCOL		0x82E0
-#define ENETC_PM0_TLCOL		0x82E8
-#define ENETC_PM0_TECOL		0x82F0
+/* Port MAC counters: Port MAC 0 corresponds to the eMAC and
+ * Port MAC 1 to the pMAC.
+ */
+#define ENETC_PM_REOCT(mac)	(0x8100 + 0x1000 * (mac))
+#define ENETC_PM_RALN(mac)	(0x8110 + 0x1000 * (mac))
+#define ENETC_PM_RXPF(mac)	(0x8118 + 0x1000 * (mac))
+#define ENETC_PM_RFRM(mac)	(0x8120 + 0x1000 * (mac))
+#define ENETC_PM_RFCS(mac)	(0x8128 + 0x1000 * (mac))
+#define ENETC_PM_RVLAN(mac)	(0x8130 + 0x1000 * (mac))
+#define ENETC_PM_RERR(mac)	(0x8138 + 0x1000 * (mac))
+#define ENETC_PM_RUCA(mac)	(0x8140 + 0x1000 * (mac))
+#define ENETC_PM_RMCA(mac)	(0x8148 + 0x1000 * (mac))
+#define ENETC_PM_RBCA(mac)	(0x8150 + 0x1000 * (mac))
+#define ENETC_PM_RDRP(mac)	(0x8158 + 0x1000 * (mac))
+#define ENETC_PM_RPKT(mac)	(0x8160 + 0x1000 * (mac))
+#define ENETC_PM_RUND(mac)	(0x8168 + 0x1000 * (mac))
+#define ENETC_PM_R64(mac)	(0x8170 + 0x1000 * (mac))
+#define ENETC_PM_R127(mac)	(0x8178 + 0x1000 * (mac))
+#define ENETC_PM_R255(mac)	(0x8180 + 0x1000 * (mac))
+#define ENETC_PM_R511(mac)	(0x8188 + 0x1000 * (mac))
+#define ENETC_PM_R1023(mac)	(0x8190 + 0x1000 * (mac))
+#define ENETC_PM_R1522(mac)	(0x8198 + 0x1000 * (mac))
+#define ENETC_PM_R1523X(mac)	(0x81A0 + 0x1000 * (mac))
+#define ENETC_PM_ROVR(mac)	(0x81A8 + 0x1000 * (mac))
+#define ENETC_PM_RJBR(mac)	(0x81B0 + 0x1000 * (mac))
+#define ENETC_PM_RFRG(mac)	(0x81B8 + 0x1000 * (mac))
+#define ENETC_PM_RCNP(mac)	(0x81C0 + 0x1000 * (mac))
+#define ENETC_PM_RDRNTP(mac)	(0x81C8 + 0x1000 * (mac))
+#define ENETC_PM_TEOCT(mac)	(0x8200 + 0x1000 * (mac))
+#define ENETC_PM_TOCT(mac)	(0x8208 + 0x1000 * (mac))
+#define ENETC_PM_TCRSE(mac)	(0x8210 + 0x1000 * (mac))
+#define ENETC_PM_TXPF(mac)	(0x8218 + 0x1000 * (mac))
+#define ENETC_PM_TFRM(mac)	(0x8220 + 0x1000 * (mac))
+#define ENETC_PM_TFCS(mac)	(0x8228 + 0x1000 * (mac))
+#define ENETC_PM_TVLAN(mac)	(0x8230 + 0x1000 * (mac))
+#define ENETC_PM_TERR(mac)	(0x8238 + 0x1000 * (mac))
+#define ENETC_PM_TUCA(mac)	(0x8240 + 0x1000 * (mac))
+#define ENETC_PM_TMCA(mac)	(0x8248 + 0x1000 * (mac))
+#define ENETC_PM_TBCA(mac)	(0x8250 + 0x1000 * (mac))
+#define ENETC_PM_TPKT(mac)	(0x8260 + 0x1000 * (mac))
+#define ENETC_PM_TUND(mac)	(0x8268 + 0x1000 * (mac))
+#define ENETC_PM_T64(mac)	(0x8270 + 0x1000 * (mac))
+#define ENETC_PM_T127(mac)	(0x8278 + 0x1000 * (mac))
+#define ENETC_PM_T255(mac)	(0x8280 + 0x1000 * (mac))
+#define ENETC_PM_T511(mac)	(0x8288 + 0x1000 * (mac))
+#define ENETC_PM_T1023(mac)	(0x8290 + 0x1000 * (mac))
+#define ENETC_PM_T1522(mac)	(0x8298 + 0x1000 * (mac))
+#define ENETC_PM_T1523X(mac)	(0x82A0 + 0x1000 * (mac))
+#define ENETC_PM_TCNP(mac)	(0x82C0 + 0x1000 * (mac))
+#define ENETC_PM_TDFR(mac)	(0x82D0 + 0x1000 * (mac))
+#define ENETC_PM_TMCOL(mac)	(0x82D8 + 0x1000 * (mac))
+#define ENETC_PM_TSCOL(mac)	(0x82E0 + 0x1000 * (mac))
+#define ENETC_PM_TLCOL(mac)	(0x82E8 + 0x1000 * (mac))
+#define ENETC_PM_TECOL(mac)	(0x82F0 + 0x1000 * (mac))
 
 /* Port counters */
 #define ENETC_PICDR(n)		(0x0700 + (n) * 8) /* n = [0..3] */
@@ -943,13 +945,13 @@ static inline u32 enetc_usecs_to_cycles(u32 usecs)
 }
 
 /* port time gating control register */
-#define ENETC_QBV_PTGCR_OFFSET		0x11a00
-#define ENETC_QBV_TGE			BIT(31)
-#define ENETC_QBV_TGPE			BIT(30)
+#define ENETC_PTGCR			0x11a00
+#define ENETC_PTGCR_TGE			BIT(31)
+#define ENETC_PTGCR_TGPE		BIT(30)
 
 /* Port time gating capability register */
-#define ENETC_QBV_PTGCAPR_OFFSET	0x11a08
-#define ENETC_QBV_MAX_GCL_LEN_MASK	GENMASK(15, 0)
+#define ENETC_PTGCAPR			0x11a08
+#define ENETC_PTGCAPR_MAX_GCL_LEN_MASK	GENMASK(15, 0)
 
 /* Port time specific departure */
 #define ENETC_PTCTSDR(n)	(0x1210 + 4 * (n))
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c
index bb7750222691..bdf94335ee99 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c
@@ -516,15 +516,34 @@ static void enetc_port_si_configure(struct enetc_si *si)
 	enetc_port_wr(hw, ENETC_PSIVLANFMR, ENETC_PSIVLANFMR_VS);
 }
 
-static void enetc_configure_port_mac(struct enetc_hw *hw)
+void enetc_set_ptcmsdur(struct enetc_hw *hw, u32 *max_sdu)
 {
 	int tc;
 
-	enetc_port_wr(hw, ENETC_PM0_MAXFRM,
-		      ENETC_SET_MAXFRM(ENETC_RX_MAXFRM_SIZE));
+	for (tc = 0; tc < 8; tc++) {
+		u32 val = ENETC_MAC_MAXFRM_SIZE;
+
+		if (max_sdu[tc])
+			val = max_sdu[tc] + VLAN_ETH_HLEN;
+
+		enetc_port_wr(hw, ENETC_PTCMSDUR(tc), val);
+	}
+}
+
+void enetc_reset_ptcmsdur(struct enetc_hw *hw)
+{
+	int tc;
 
 	for (tc = 0; tc < 8; tc++)
 		enetc_port_wr(hw, ENETC_PTCMSDUR(tc), ENETC_MAC_MAXFRM_SIZE);
+}
+
+static void enetc_configure_port_mac(struct enetc_hw *hw)
+{
+	enetc_port_wr(hw, ENETC_PM0_MAXFRM,
+		      ENETC_SET_MAXFRM(ENETC_RX_MAXFRM_SIZE));
+
+	enetc_reset_ptcmsdur(hw);
 
 	enetc_port_wr(hw, ENETC_PM0_CMD_CFG, ENETC_PM0_CMD_PHY_TX_EN |
 		      ENETC_PM0_CMD_TXP	| ENETC_PM0_PROMISC);
@@ -738,6 +757,8 @@ static int enetc_pf_setup_tc(struct net_device *ndev, enum tc_setup_type type,
 			     void *type_data)
 {
 	switch (type) {
+	case TC_QUERY_CAPS:
+		return enetc_qos_query_caps(ndev, type_data);
 	case TC_SETUP_QDISC_MQPRIO:
 		return enetc_setup_tc_mqprio(ndev, type_data);
 	case TC_SETUP_QDISC_TAPRIO:
diff --git a/drivers/net/ethernet/freescale/enetc/enetc_qos.c b/drivers/net/ethernet/freescale/enetc/enetc_qos.c
index f8a2f02ce22d..e6416332ec79 100644
--- a/drivers/net/ethernet/freescale/enetc/enetc_qos.c
+++ b/drivers/net/ethernet/freescale/enetc/enetc_qos.c
@@ -7,18 +7,19 @@
 #include <linux/math64.h>
 #include <linux/refcount.h>
 #include <net/pkt_cls.h>
+#include <net/pkt_sched.h>
 #include <net/tc_act/tc_gate.h>
 
 static u16 enetc_get_max_gcl_len(struct enetc_hw *hw)
 {
-	return enetc_rd(hw, ENETC_QBV_PTGCAPR_OFFSET)
-		& ENETC_QBV_MAX_GCL_LEN_MASK;
+	return enetc_rd(hw, ENETC_PTGCAPR) & ENETC_PTGCAPR_MAX_GCL_LEN_MASK;
 }
 
 void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed)
 {
+	struct enetc_hw *hw = &priv->si->hw;
 	u32 old_speed = priv->speed;
-	u32 pspeed;
+	u32 pspeed, tmp;
 
 	if (speed == old_speed)
 		return;
@@ -39,16 +40,15 @@ void enetc_sched_speed_set(struct enetc_ndev_priv *priv, int speed)
 	}
 
 	priv->speed = speed;
-	enetc_port_wr(&priv->si->hw, ENETC_PMR,
-		      (enetc_port_rd(&priv->si->hw, ENETC_PMR)
-		      & (~ENETC_PMR_PSPEED_MASK))
-		      | pspeed);
+	tmp = enetc_port_rd(hw, ENETC_PMR);
+	enetc_port_wr(hw, ENETC_PMR, (tmp & ~ENETC_PMR_PSPEED_MASK) | pspeed);
 }
 
 static int enetc_setup_taprio(struct net_device *ndev,
 			      struct tc_taprio_qopt_offload *admin_conf)
 {
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_hw *hw = &priv->si->hw;
 	struct enetc_cbd cbd = {.cmd = 0};
 	struct tgs_gcl_conf *gcl_config;
 	struct tgs_gcl_data *gcl_data;
@@ -61,15 +61,14 @@ static int enetc_setup_taprio(struct net_device *ndev,
 	int err;
 	int i;
 
-	if (admin_conf->num_entries > enetc_get_max_gcl_len(&priv->si->hw))
+	if (admin_conf->num_entries > enetc_get_max_gcl_len(hw))
 		return -EINVAL;
 	gcl_len = admin_conf->num_entries;
 
-	tge = enetc_rd(&priv->si->hw, ENETC_QBV_PTGCR_OFFSET);
+	tge = enetc_rd(hw, ENETC_PTGCR);
 	if (!admin_conf->enable) {
-		enetc_wr(&priv->si->hw,
-			 ENETC_QBV_PTGCR_OFFSET,
-			 tge & (~ENETC_QBV_TGE));
+		enetc_wr(hw, ENETC_PTGCR, tge & ~ENETC_PTGCR_TGE);
+		enetc_reset_ptcmsdur(hw);
 
 		priv->active_offloads &= ~ENETC_F_QBV;
 
@@ -117,27 +116,28 @@ static int enetc_setup_taprio(struct net_device *ndev,
 	cbd.cls = BDCR_CMD_PORT_GCL;
 	cbd.status_flags = 0;
 
-	enetc_wr(&priv->si->hw, ENETC_QBV_PTGCR_OFFSET,
-		 tge | ENETC_QBV_TGE);
+	enetc_wr(hw, ENETC_PTGCR, tge | ENETC_PTGCR_TGE);
 
 	err = enetc_send_cmd(priv->si, &cbd);
 	if (err)
-		enetc_wr(&priv->si->hw,
-			 ENETC_QBV_PTGCR_OFFSET,
-			 tge & (~ENETC_QBV_TGE));
+		enetc_wr(hw, ENETC_PTGCR, tge & ~ENETC_PTGCR_TGE);
 
 	enetc_cbd_free_data_mem(priv->si, data_size, tmp, &dma);
 
-	if (!err)
-		priv->active_offloads |= ENETC_F_QBV;
+	if (err)
+		return err;
 
-	return err;
+	enetc_set_ptcmsdur(hw, admin_conf->max_sdu);
+	priv->active_offloads |= ENETC_F_QBV;
+
+	return 0;
 }
 
 int enetc_setup_tc_taprio(struct net_device *ndev, void *type_data)
 {
 	struct tc_taprio_qopt_offload *taprio = type_data;
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct enetc_hw *hw = &priv->si->hw;
 	int err;
 	int i;
 
@@ -147,16 +147,14 @@ int enetc_setup_tc_taprio(struct net_device *ndev, void *type_data)
 			return -EBUSY;
 
 	for (i = 0; i < priv->num_tx_rings; i++)
-		enetc_set_bdr_prio(&priv->si->hw,
-				   priv->tx_ring[i]->index,
+		enetc_set_bdr_prio(hw, priv->tx_ring[i]->index,
 				   taprio->enable ? i : 0);
 
 	err = enetc_setup_taprio(ndev, taprio);
 
 	if (err)
 		for (i = 0; i < priv->num_tx_rings; i++)
-			enetc_set_bdr_prio(&priv->si->hw,
-					   priv->tx_ring[i]->index,
+			enetc_set_bdr_prio(hw, priv->tx_ring[i]->index,
 					   taprio->enable ? 0 : i);
 
 	return err;
@@ -178,7 +176,7 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data)
 	struct tc_cbs_qopt_offload *cbs = type_data;
 	u32 port_transmit_rate = priv->speed;
 	u8 tc_nums = netdev_get_num_tc(ndev);
-	struct enetc_si *si = priv->si;
+	struct enetc_hw *hw = &priv->si->hw;
 	u32 hi_credit_bit, hi_credit_reg;
 	u32 max_interference_size;
 	u32 port_frame_max_size;
@@ -199,15 +197,15 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data)
 		 * lower than this TC have been disabled.
 		 */
 		if (tc == prio_top &&
-		    enetc_get_cbs_enable(&si->hw, prio_next)) {
+		    enetc_get_cbs_enable(hw, prio_next)) {
 			dev_err(&ndev->dev,
 				"Disable TC%d before disable TC%d\n",
 				prio_next, tc);
 			return -EINVAL;
 		}
 
-		enetc_port_wr(&si->hw, ENETC_PTCCBSR1(tc), 0);
-		enetc_port_wr(&si->hw, ENETC_PTCCBSR0(tc), 0);
+		enetc_port_wr(hw, ENETC_PTCCBSR1(tc), 0);
+		enetc_port_wr(hw, ENETC_PTCCBSR0(tc), 0);
 
 		return 0;
 	}
@@ -224,13 +222,13 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data)
 	 * higher than this TC have been enabled.
 	 */
 	if (tc == prio_next) {
-		if (!enetc_get_cbs_enable(&si->hw, prio_top)) {
+		if (!enetc_get_cbs_enable(hw, prio_top)) {
 			dev_err(&ndev->dev,
 				"Enable TC%d first before enable TC%d\n",
 				prio_top, prio_next);
 			return -EINVAL;
 		}
-		bw_sum += enetc_get_cbs_bw(&si->hw, prio_top);
+		bw_sum += enetc_get_cbs_bw(hw, prio_top);
 	}
 
 	if (bw_sum + bw >= 100) {
@@ -239,7 +237,7 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data)
 		return -EINVAL;
 	}
 
-	enetc_port_rd(&si->hw, ENETC_PTCMSDUR(tc));
+	enetc_port_rd(hw, ENETC_PTCMSDUR(tc));
 
 	/* For top prio TC, the max_interfrence_size is maxSizedFrame.
 	 *
@@ -259,8 +257,8 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data)
 		u32 m0, ma, r0, ra;
 
 		m0 = port_frame_max_size * 8;
-		ma = enetc_port_rd(&si->hw, ENETC_PTCMSDUR(prio_top)) * 8;
-		ra = enetc_get_cbs_bw(&si->hw, prio_top) *
+		ma = enetc_port_rd(hw, ENETC_PTCMSDUR(prio_top)) * 8;
+		ra = enetc_get_cbs_bw(hw, prio_top) *
 			port_transmit_rate * 10000ULL;
 		r0 = port_transmit_rate * 1000000ULL;
 		max_interference_size = m0 + ma +
@@ -280,10 +278,10 @@ int enetc_setup_tc_cbs(struct net_device *ndev, void *type_data)
 	hi_credit_reg = (u32)div_u64((ENETC_CLK * 100ULL) * hi_credit_bit,
 				     port_transmit_rate * 1000000ULL);
 
-	enetc_port_wr(&si->hw, ENETC_PTCCBSR1(tc), hi_credit_reg);
+	enetc_port_wr(hw, ENETC_PTCCBSR1(tc), hi_credit_reg);
 
 	/* Set bw register and enable this traffic class */
-	enetc_port_wr(&si->hw, ENETC_PTCCBSR0(tc), bw | ENETC_CBSE);
+	enetc_port_wr(hw, ENETC_PTCCBSR0(tc), bw | ENETC_CBSE);
 
 	return 0;
 }
@@ -293,6 +291,7 @@ int enetc_setup_tc_txtime(struct net_device *ndev, void *type_data)
 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
 	struct tc_etf_qopt_offload *qopt = type_data;
 	u8 tc_nums = netdev_get_num_tc(ndev);
+	struct enetc_hw *hw = &priv->si->hw;
 	int tc;
 
 	if (!tc_nums)
@@ -304,12 +303,11 @@ int enetc_setup_tc_txtime(struct net_device *ndev, void *type_data)
 		return -EINVAL;
 
 	/* TSD and Qbv are mutually exclusive in hardware */
-	if (enetc_rd(&priv->si->hw, ENETC_QBV_PTGCR_OFFSET) & ENETC_QBV_TGE)
+	if (enetc_rd(hw, ENETC_PTGCR) & ENETC_PTGCR_TGE)
 		return -EBUSY;
 
 	priv->tx_ring[tc]->tsd_enable = qopt->enable;
-	enetc_port_wr(&priv->si->hw, ENETC_PTCTSDR(tc),
-		      qopt->enable ? ENETC_TSDE : 0);
+	enetc_port_wr(hw, ENETC_PTCTSDR(tc), qopt->enable ? ENETC_TSDE : 0);
 
 	return 0;
 }
@@ -1601,3 +1599,23 @@ int enetc_setup_tc_psfp(struct net_device *ndev, void *type_data)
 
 	return 0;
 }
+
+int enetc_qos_query_caps(struct net_device *ndev, void *type_data)
+{
+	struct enetc_ndev_priv *priv = netdev_priv(ndev);
+	struct tc_query_caps_base *base = type_data;
+	struct enetc_si *si = priv->si;
+
+	switch (base->type) {
+	case TC_SETUP_QDISC_TAPRIO: {
+		struct tc_taprio_caps *caps = base->caps;
+
+		if (si->hw_features & ENETC_SI_F_QBV)
+			caps->supports_queue_max_sdu = true;
+
+		return 0;
+	}
+	default:
+		return -EOPNOTSUPP;
+	}
+}
diff --git a/drivers/net/ethernet/freescale/fec.h b/drivers/net/ethernet/freescale/fec.h
index a5fed00cb971..33f84a30e167 100644
--- a/drivers/net/ethernet/freescale/fec.h
+++ b/drivers/net/ethernet/freescale/fec.h
@@ -17,8 +17,11 @@
 #include <linux/clocksource.h>
 #include <linux/net_tstamp.h>
 #include <linux/pm_qos.h>
+#include <linux/bpf.h>
 #include <linux/ptp_clock_kernel.h>
 #include <linux/timecounter.h>
+#include <dt-bindings/firmware/imx/rsrc.h>
+#include <linux/firmware/imx/sci.h>
 
 #if defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
     defined(CONFIG_M520x) || defined(CONFIG_M532x) || defined(CONFIG_ARM) || \
@@ -344,8 +347,11 @@ struct bufdesc_ex {
  * the skbuffer directly.
  */
 
+#define FEC_ENET_XDP_HEADROOM	(XDP_PACKET_HEADROOM)
+
 #define FEC_ENET_RX_PAGES	256
-#define FEC_ENET_RX_FRSIZE	2048
+#define FEC_ENET_RX_FRSIZE	(PAGE_SIZE - FEC_ENET_XDP_HEADROOM \
+		- SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
 #define FEC_ENET_RX_FRPPG	(PAGE_SIZE / FEC_ENET_RX_FRSIZE)
 #define RX_RING_SIZE		(FEC_ENET_RX_FRPPG * FEC_ENET_RX_PAGES)
 #define FEC_ENET_TX_FRSIZE	2048
@@ -515,6 +521,12 @@ struct bufdesc_prop {
 	unsigned char dsize_log2;
 };
 
+struct fec_enet_priv_txrx_info {
+	int	offset;
+	struct	page *page;
+	struct  sk_buff *skb;
+};
+
 struct fec_enet_priv_tx_q {
 	struct bufdesc_prop bd;
 	unsigned char *tx_bounce[TX_RING_SIZE];
@@ -530,7 +542,14 @@ struct fec_enet_priv_tx_q {
 
 struct fec_enet_priv_rx_q {
 	struct bufdesc_prop bd;
-	struct  sk_buff *rx_skbuff[RX_RING_SIZE];
+	struct  fec_enet_priv_txrx_info rx_skb_info[RX_RING_SIZE];
+
+	/* page_pool */
+	struct page_pool *page_pool;
+	struct xdp_rxq_info xdp_rxq;
+
+	/* rx queue number, in the range 0-7 */
+	u8 id;
 };
 
 struct fec_stop_mode_gpr {
@@ -583,6 +602,7 @@ struct fec_enet_private {
 	struct device_node *phy_node;
 	bool	rgmii_txc_dly;
 	bool	rgmii_rxc_dly;
+	bool	rpm_active;
 	int	link;
 	int	full_duplex;
 	int	speed;
@@ -639,6 +659,8 @@ struct fec_enet_private {
 	int pps_enable;
 	unsigned int next_counter;
 
+	struct imx_sc_ipc *ipc_handle;
+
 	u64 ethtool_stats[];
 };
 
diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c
index 92c55e1a5507..98d5cd313fdd 100644
--- a/drivers/net/ethernet/freescale/fec_main.c
+++ b/drivers/net/ethernet/freescale/fec_main.c
@@ -66,6 +66,8 @@
 #include <linux/mfd/syscon.h>
 #include <linux/regmap.h>
 #include <soc/imx/cpuidle.h>
+#include <linux/filter.h>
+#include <linux/bpf.h>
 
 #include <asm/cacheflush.h>
 
@@ -156,6 +158,13 @@ static const struct fec_devinfo fec_imx8qm_info = {
 		  FEC_QUIRK_DELAYED_CLKS_SUPPORT,
 };
 
+static const struct fec_devinfo fec_s32v234_info = {
+	.quirks = FEC_QUIRK_ENET_MAC | FEC_QUIRK_HAS_GBIT |
+		  FEC_QUIRK_HAS_BUFDESC_EX | FEC_QUIRK_HAS_CSUM |
+		  FEC_QUIRK_HAS_VLAN | FEC_QUIRK_HAS_AVB |
+		  FEC_QUIRK_ERR007885 | FEC_QUIRK_BUG_CAPTURE,
+};
+
 static struct platform_device_id fec_devtype[] = {
 	{
 		/* keep it for coldfire */
@@ -189,6 +198,9 @@ static struct platform_device_id fec_devtype[] = {
 		.name = "imx8qm-fec",
 		.driver_data = (kernel_ulong_t)&fec_imx8qm_info,
 	}, {
+		.name = "s32v234-fec",
+		.driver_data = (kernel_ulong_t)&fec_s32v234_info,
+	}, {
 		/* sentinel */
 	}
 };
@@ -204,6 +216,7 @@ enum imx_fec_type {
 	IMX6UL_FEC,
 	IMX8MQ_FEC,
 	IMX8QM_FEC,
+	S32V234_FEC,
 };
 
 static const struct of_device_id fec_dt_ids[] = {
@@ -216,6 +229,7 @@ static const struct of_device_id fec_dt_ids[] = {
 	{ .compatible = "fsl,imx6ul-fec", .data = &fec_devtype[IMX6UL_FEC], },
 	{ .compatible = "fsl,imx8mq-fec", .data = &fec_devtype[IMX8MQ_FEC], },
 	{ .compatible = "fsl,imx8qm-fec", .data = &fec_devtype[IMX8QM_FEC], },
+	{ .compatible = "fsl,s32v234-fec", .data = &fec_devtype[S32V234_FEC], },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, fec_dt_ids);
@@ -410,6 +424,48 @@ fec_enet_clear_csum(struct sk_buff *skb, struct net_device *ndev)
 	return 0;
 }
 
+static int
+fec_enet_create_page_pool(struct fec_enet_private *fep,
+			  struct fec_enet_priv_rx_q *rxq, int size)
+{
+	struct page_pool_params pp_params = {
+		.order = 0,
+		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
+		.pool_size = size,
+		.nid = dev_to_node(&fep->pdev->dev),
+		.dev = &fep->pdev->dev,
+		.dma_dir = DMA_FROM_DEVICE,
+		.offset = FEC_ENET_XDP_HEADROOM,
+		.max_len = FEC_ENET_RX_FRSIZE,
+	};
+	int err;
+
+	rxq->page_pool = page_pool_create(&pp_params);
+	if (IS_ERR(rxq->page_pool)) {
+		err = PTR_ERR(rxq->page_pool);
+		rxq->page_pool = NULL;
+		return err;
+	}
+
+	err = xdp_rxq_info_reg(&rxq->xdp_rxq, fep->netdev, rxq->id, 0);
+	if (err < 0)
+		goto err_free_pp;
+
+	err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL,
+					 rxq->page_pool);
+	if (err)
+		goto err_unregister_rxq;
+
+	return 0;
+
+err_unregister_rxq:
+	xdp_rxq_info_unreg(&rxq->xdp_rxq);
+err_free_pp:
+	page_pool_destroy(rxq->page_pool);
+	rxq->page_pool = NULL;
+	return err;
+}
+
 static struct bufdesc *
 fec_enet_txq_submit_frag_skb(struct fec_enet_priv_tx_q *txq,
 			     struct sk_buff *skb,
@@ -1168,6 +1224,34 @@ fec_restart(struct net_device *ndev)
 
 }
 
+static int fec_enet_ipc_handle_init(struct fec_enet_private *fep)
+{
+	if (!(of_machine_is_compatible("fsl,imx8qm") ||
+	      of_machine_is_compatible("fsl,imx8qxp") ||
+	      of_machine_is_compatible("fsl,imx8dxl")))
+		return 0;
+
+	return imx_scu_get_handle(&fep->ipc_handle);
+}
+
+static void fec_enet_ipg_stop_set(struct fec_enet_private *fep, bool enabled)
+{
+	struct device_node *np = fep->pdev->dev.of_node;
+	u32 rsrc_id, val;
+	int idx;
+
+	if (!np || !fep->ipc_handle)
+		return;
+
+	idx = of_alias_get_id(np, "ethernet");
+	if (idx < 0)
+		idx = 0;
+	rsrc_id = idx ? IMX_SC_R_ENET_1 : IMX_SC_R_ENET_0;
+
+	val = enabled ? 1 : 0;
+	imx_sc_misc_set_control(fep->ipc_handle, rsrc_id, IMX_SC_C_IPG_STOP, val);
+}
+
 static void fec_enet_stop_mode(struct fec_enet_private *fep, bool enabled)
 {
 	struct fec_platform_data *pdata = fep->pdev->dev.platform_data;
@@ -1183,6 +1267,8 @@ static void fec_enet_stop_mode(struct fec_enet_private *fep, bool enabled)
 					   BIT(stop_gpr->bit), 0);
 	} else if (pdata && pdata->sleep_mode_enable) {
 		pdata->sleep_mode_enable(enabled);
+	} else {
+		fec_enet_ipg_stop_set(fep, enabled);
 	}
 }
 
@@ -1408,7 +1494,7 @@ static void fec_enet_tx(struct net_device *ndev)
 		fec_enet_tx_queue(ndev, i);
 }
 
-static int
+static int __maybe_unused
 fec_enet_new_rxbdp(struct net_device *ndev, struct bufdesc *bdp, struct sk_buff *skb)
 {
 	struct  fec_enet_private *fep = netdev_priv(ndev);
@@ -1428,8 +1514,9 @@ fec_enet_new_rxbdp(struct net_device *ndev, struct bufdesc *bdp, struct sk_buff
 	return 0;
 }
 
-static bool fec_enet_copybreak(struct net_device *ndev, struct sk_buff **skb,
-			       struct bufdesc *bdp, u32 length, bool swap)
+static bool __maybe_unused
+fec_enet_copybreak(struct net_device *ndev, struct sk_buff **skb,
+		   struct bufdesc *bdp, u32 length, bool swap)
 {
 	struct  fec_enet_private *fep = netdev_priv(ndev);
 	struct sk_buff *new_skb;
@@ -1454,6 +1541,21 @@ static bool fec_enet_copybreak(struct net_device *ndev, struct sk_buff **skb,
 	return true;
 }
 
+static void fec_enet_update_cbd(struct fec_enet_priv_rx_q *rxq,
+				struct bufdesc *bdp, int index)
+{
+	struct page *new_page;
+	dma_addr_t phys_addr;
+
+	new_page = page_pool_dev_alloc_pages(rxq->page_pool);
+	WARN_ON(!new_page);
+	rxq->rx_skb_info[index].page = new_page;
+
+	rxq->rx_skb_info[index].offset = FEC_ENET_XDP_HEADROOM;
+	phys_addr = page_pool_get_dma_addr(new_page) + FEC_ENET_XDP_HEADROOM;
+	bdp->cbd_bufaddr = cpu_to_fec32(phys_addr);
+}
+
 /* During a receive, the bd_rx.cur points to the current incoming buffer.
  * When we update through the ring, if the next incoming buffer has
  * not been given to the system, we just set the empty indicator,
@@ -1466,7 +1568,6 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
 	struct fec_enet_priv_rx_q *rxq;
 	struct bufdesc *bdp;
 	unsigned short status;
-	struct  sk_buff *skb_new = NULL;
 	struct  sk_buff *skb;
 	ushort	pkt_len;
 	__u8 *data;
@@ -1475,8 +1576,8 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
 	bool	vlan_packet_rcvd = false;
 	u16	vlan_tag;
 	int	index = 0;
-	bool	is_copybreak;
 	bool	need_swap = fep->quirks & FEC_QUIRK_SWAP_FRAME;
+	struct page *page;
 
 #ifdef CONFIG_M532x
 	flush_cache_all();
@@ -1528,31 +1629,25 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
 		ndev->stats.rx_bytes += pkt_len;
 
 		index = fec_enet_get_bd_index(bdp, &rxq->bd);
-		skb = rxq->rx_skbuff[index];
+		page = rxq->rx_skb_info[index].page;
+		dma_sync_single_for_cpu(&fep->pdev->dev,
+					fec32_to_cpu(bdp->cbd_bufaddr),
+					pkt_len,
+					DMA_FROM_DEVICE);
+		prefetch(page_address(page));
+		fec_enet_update_cbd(rxq, bdp, index);
 
 		/* The packet length includes FCS, but we don't want to
 		 * include that when passing upstream as it messes up
 		 * bridging applications.
 		 */
-		is_copybreak = fec_enet_copybreak(ndev, &skb, bdp, pkt_len - 4,
-						  need_swap);
-		if (!is_copybreak) {
-			skb_new = netdev_alloc_skb(ndev, FEC_ENET_RX_FRSIZE);
-			if (unlikely(!skb_new)) {
-				ndev->stats.rx_dropped++;
-				goto rx_processing_done;
-			}
-			dma_unmap_single(&fep->pdev->dev,
-					 fec32_to_cpu(bdp->cbd_bufaddr),
-					 FEC_ENET_RX_FRSIZE - fep->rx_align,
-					 DMA_FROM_DEVICE);
-		}
-
-		prefetch(skb->data - NET_IP_ALIGN);
+		skb = build_skb(page_address(page), PAGE_SIZE);
+		skb_reserve(skb, FEC_ENET_XDP_HEADROOM);
 		skb_put(skb, pkt_len - 4);
+		skb_mark_for_recycle(skb);
 		data = skb->data;
 
-		if (!is_copybreak && need_swap)
+		if (need_swap)
 			swap_buffer(data, pkt_len);
 
 #if !defined(CONFIG_M5272)
@@ -1607,16 +1702,6 @@ fec_enet_rx_queue(struct net_device *ndev, int budget, u16 queue_id)
 		skb_record_rx_queue(skb, queue_id);
 		napi_gro_receive(&fep->napi, skb);
 
-		if (is_copybreak) {
-			dma_sync_single_for_device(&fep->pdev->dev,
-						   fec32_to_cpu(bdp->cbd_bufaddr),
-						   FEC_ENET_RX_FRSIZE - fep->rx_align,
-						   DMA_FROM_DEVICE);
-		} else {
-			rxq->rx_skbuff[index] = skb_new;
-			fec_enet_new_rxbdp(ndev, bdp, skb_new);
-		}
-
 rx_processing_done:
 		/* Clear the status flags for this buffer */
 		status &= ~BD_ENET_RX_STATS;
@@ -2105,13 +2190,13 @@ static int fec_enet_mii_probe(struct net_device *ndev)
 				continue;
 			if (dev_id--)
 				continue;
-			strlcpy(mdio_bus_id, fep->mii_bus->id, MII_BUS_ID_SIZE);
+			strscpy(mdio_bus_id, fep->mii_bus->id, MII_BUS_ID_SIZE);
 			break;
 		}
 
 		if (phy_id >= PHY_MAX_ADDR) {
 			netdev_info(ndev, "no PHY, assuming direct connection to switch\n");
-			strlcpy(mdio_bus_id, "fixed-0", MII_BUS_ID_SIZE);
+			strscpy(mdio_bus_id, "fixed-0", MII_BUS_ID_SIZE);
 			phy_id = 0;
 		}
 
@@ -2295,9 +2380,9 @@ static void fec_enet_get_drvinfo(struct net_device *ndev,
 {
 	struct fec_enet_private *fep = netdev_priv(ndev);
 
-	strlcpy(info->driver, fep->pdev->dev.driver->name,
+	strscpy(info->driver, fep->pdev->dev.driver->name,
 		sizeof(info->driver));
-	strlcpy(info->bus_info, dev_name(&ndev->dev), sizeof(info->bus_info));
+	strscpy(info->bus_info, dev_name(&ndev->dev), sizeof(info->bus_info));
 }
 
 static int fec_enet_get_regs_len(struct net_device *ndev)
@@ -2960,26 +3045,19 @@ static void fec_enet_free_buffers(struct net_device *ndev)
 	struct fec_enet_private *fep = netdev_priv(ndev);
 	unsigned int i;
 	struct sk_buff *skb;
-	struct bufdesc	*bdp;
 	struct fec_enet_priv_tx_q *txq;
 	struct fec_enet_priv_rx_q *rxq;
 	unsigned int q;
 
 	for (q = 0; q < fep->num_rx_queues; q++) {
 		rxq = fep->rx_queue[q];
-		bdp = rxq->bd.base;
-		for (i = 0; i < rxq->bd.ring_size; i++) {
-			skb = rxq->rx_skbuff[i];
-			rxq->rx_skbuff[i] = NULL;
-			if (skb) {
-				dma_unmap_single(&fep->pdev->dev,
-						 fec32_to_cpu(bdp->cbd_bufaddr),
-						 FEC_ENET_RX_FRSIZE - fep->rx_align,
-						 DMA_FROM_DEVICE);
-				dev_kfree_skb(skb);
-			}
-			bdp = fec_enet_get_nextdesc(bdp, &rxq->bd);
-		}
+		for (i = 0; i < rxq->bd.ring_size; i++)
+			page_pool_release_page(rxq->page_pool, rxq->rx_skb_info[i].page);
+
+		if (xdp_rxq_info_is_reg(&rxq->xdp_rxq))
+			xdp_rxq_info_unreg(&rxq->xdp_rxq);
+		page_pool_destroy(rxq->page_pool);
+		rxq->page_pool = NULL;
 	}
 
 	for (q = 0; q < fep->num_tx_queues; q++) {
@@ -3069,24 +3147,31 @@ static int
 fec_enet_alloc_rxq_buffers(struct net_device *ndev, unsigned int queue)
 {
 	struct fec_enet_private *fep = netdev_priv(ndev);
-	unsigned int i;
-	struct sk_buff *skb;
-	struct bufdesc	*bdp;
 	struct fec_enet_priv_rx_q *rxq;
+	dma_addr_t phys_addr;
+	struct bufdesc	*bdp;
+	struct page *page;
+	int i, err;
 
 	rxq = fep->rx_queue[queue];
 	bdp = rxq->bd.base;
+
+	err = fec_enet_create_page_pool(fep, rxq, rxq->bd.ring_size);
+	if (err < 0) {
+		netdev_err(ndev, "%s failed queue %d (%d)\n", __func__, queue, err);
+		return err;
+	}
+
 	for (i = 0; i < rxq->bd.ring_size; i++) {
-		skb = __netdev_alloc_skb(ndev, FEC_ENET_RX_FRSIZE, GFP_KERNEL);
-		if (!skb)
+		page = page_pool_dev_alloc_pages(rxq->page_pool);
+		if (!page)
 			goto err_alloc;
 
-		if (fec_enet_new_rxbdp(ndev, bdp, skb)) {
-			dev_kfree_skb(skb);
-			goto err_alloc;
-		}
+		phys_addr = page_pool_get_dma_addr(page) + FEC_ENET_XDP_HEADROOM;
+		bdp->cbd_bufaddr = cpu_to_fec32(phys_addr);
 
-		rxq->rx_skbuff[i] = skb;
+		rxq->rx_skb_info[i].page = page;
+		rxq->rx_skb_info[i].offset = FEC_ENET_XDP_HEADROOM;
 		bdp->cbd_sc = cpu_to_fec16(BD_ENET_RX_EMPTY);
 
 		if (fep->bufdesc_ex) {
@@ -3566,7 +3651,7 @@ static int fec_enet_init(struct net_device *ndev)
 	ndev->ethtool_ops = &fec_enet_ethtool_ops;
 
 	writel(FEC_RX_DISABLED_IMASK, fep->hwp + FEC_IMASK);
-	netif_napi_add(ndev, &fep->napi, fec_enet_rx_napi, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &fep->napi, fec_enet_rx_napi);
 
 	if (fep->quirks & FEC_QUIRK_HAS_VLAN)
 		/* enable hw VLAN support */
@@ -3824,6 +3909,10 @@ fec_probe(struct platform_device *pdev)
 	    !of_property_read_bool(np, "fsl,err006687-workaround-present"))
 		fep->quirks |= FEC_QUIRK_ERR006687;
 
+	ret = fec_enet_ipc_handle_init(fep);
+	if (ret)
+		goto failed_ipc_init;
+
 	if (of_get_property(np, "fsl,magic-packet", NULL))
 		fep->wol_flag |= FEC_WOL_HAS_MAGIC_PACKET;
 
@@ -4021,6 +4110,7 @@ failed_rgmii_delay:
 		of_phy_deregister_fixed_link(np);
 	of_node_put(phy_node);
 failed_stop_mode:
+failed_ipc_init:
 failed_phy:
 	dev_id--;
 failed_ioremap:
@@ -4065,6 +4155,7 @@ static int __maybe_unused fec_suspend(struct device *dev)
 {
 	struct net_device *ndev = dev_get_drvdata(dev);
 	struct fec_enet_private *fep = netdev_priv(ndev);
+	int ret;
 
 	rtnl_lock();
 	if (netif_running(ndev)) {
@@ -4089,6 +4180,15 @@ static int __maybe_unused fec_suspend(struct device *dev)
 		}
 		/* It's safe to disable clocks since interrupts are masked */
 		fec_enet_clk_enable(ndev, false);
+
+		fep->rpm_active = !pm_runtime_status_suspended(dev);
+		if (fep->rpm_active) {
+			ret = pm_runtime_force_suspend(dev);
+			if (ret < 0) {
+				rtnl_unlock();
+				return ret;
+			}
+		}
 	}
 	rtnl_unlock();
 
@@ -4119,6 +4219,9 @@ static int __maybe_unused fec_resume(struct device *dev)
 
 	rtnl_lock();
 	if (netif_running(ndev)) {
+		if (fep->rpm_active)
+			pm_runtime_force_resume(dev);
+
 		ret = fec_enet_clk_enable(ndev, true);
 		if (ret) {
 			rtnl_unlock();
diff --git a/drivers/net/ethernet/freescale/fec_ptp.c b/drivers/net/ethernet/freescale/fec_ptp.c
index 3dc3c0b626c2..cffd9ad499dd 100644
--- a/drivers/net/ethernet/freescale/fec_ptp.c
+++ b/drivers/net/ethernet/freescale/fec_ptp.c
@@ -578,7 +578,7 @@ void fec_ptp_init(struct platform_device *pdev, int irq_idx)
 	int ret;
 
 	fep->ptp_caps.owner = THIS_MODULE;
-	strlcpy(fep->ptp_caps.name, "fec ptp", sizeof(fep->ptp_caps.name));
+	strscpy(fep->ptp_caps.name, "fec ptp", sizeof(fep->ptp_caps.name));
 
 	fep->ptp_caps.max_adj = 250000000;
 	fep->ptp_caps.n_alarm = 0;
diff --git a/drivers/net/ethernet/freescale/fman/fman.c b/drivers/net/ethernet/freescale/fman/fman.c
index 8f0db61cb1f6..9d85fb136e34 100644
--- a/drivers/net/ethernet/freescale/fman/fman.c
+++ b/drivers/net/ethernet/freescale/fman/fman.c
@@ -1,34 +1,7 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
 /*
- * Copyright 2008-2015 Freescale Semiconductor Inc.
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  * Copyright 2020 NXP
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
diff --git a/drivers/net/ethernet/freescale/fman/fman.h b/drivers/net/ethernet/freescale/fman/fman.h
index f2ede1360f03..2ea575a46675 100644
--- a/drivers/net/ethernet/freescale/fman/fman.h
+++ b/drivers/net/ethernet/freescale/fman/fman.h
@@ -1,34 +1,7 @@
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */
 /*
- * Copyright 2008-2015 Freescale Semiconductor Inc.
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  * Copyright 2020 NXP
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #ifndef __FM_H
diff --git a/drivers/net/ethernet/freescale/fman/fman_dtsec.c b/drivers/net/ethernet/freescale/fman/fman_dtsec.c
index 1950a8936bc0..6617932fd3fd 100644
--- a/drivers/net/ethernet/freescale/fman/fman_dtsec.c
+++ b/drivers/net/ethernet/freescale/fman/fman_dtsec.c
@@ -1,39 +1,13 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
 /*
- * Copyright 2008-2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include "fman_dtsec.h"
 #include "fman.h"
+#include "mac.h"
 
 #include <linux/slab.h>
 #include <linux/bitrev.h>
@@ -327,7 +301,7 @@ struct fman_mac {
 	/* Ethernet physical interface */
 	phy_interface_t phy_if;
 	u16 max_speed;
-	void *dev_id; /* device cookie used by the exception cbs */
+	struct mac_device *dev_id; /* device cookie used by the exception cbs */
 	fman_mac_exception_cb *exception_cb;
 	fman_mac_exception_cb *event_cb;
 	/* Number of individual addresses in registers for this station */
@@ -840,73 +814,45 @@ static void free_init_resources(struct fman_mac *dtsec)
 	dtsec->unicast_addr_hash = NULL;
 }
 
-int dtsec_cfg_max_frame_len(struct fman_mac *dtsec, u16 new_val)
-{
-	if (is_init_done(dtsec->dtsec_drv_param))
-		return -EINVAL;
-
-	dtsec->dtsec_drv_param->maximum_frame = new_val;
-
-	return 0;
-}
-
-int dtsec_cfg_pad_and_crc(struct fman_mac *dtsec, bool new_val)
-{
-	if (is_init_done(dtsec->dtsec_drv_param))
-		return -EINVAL;
-
-	dtsec->dtsec_drv_param->tx_pad_crc = new_val;
-
-	return 0;
-}
-
-static void graceful_start(struct fman_mac *dtsec, enum comm_mode mode)
+static void graceful_start(struct fman_mac *dtsec)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
 
-	if (mode & COMM_MODE_TX)
-		iowrite32be(ioread32be(&regs->tctrl) &
-				~TCTRL_GTS, &regs->tctrl);
-	if (mode & COMM_MODE_RX)
-		iowrite32be(ioread32be(&regs->rctrl) &
-				~RCTRL_GRS, &regs->rctrl);
+	iowrite32be(ioread32be(&regs->tctrl) & ~TCTRL_GTS, &regs->tctrl);
+	iowrite32be(ioread32be(&regs->rctrl) & ~RCTRL_GRS, &regs->rctrl);
 }
 
-static void graceful_stop(struct fman_mac *dtsec, enum comm_mode mode)
+static void graceful_stop(struct fman_mac *dtsec)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
 	u32 tmp;
 
 	/* Graceful stop - Assert the graceful Rx stop bit */
-	if (mode & COMM_MODE_RX) {
-		tmp = ioread32be(&regs->rctrl) | RCTRL_GRS;
-		iowrite32be(tmp, &regs->rctrl);
+	tmp = ioread32be(&regs->rctrl) | RCTRL_GRS;
+	iowrite32be(tmp, &regs->rctrl);
 
-		if (dtsec->fm_rev_info.major == 2) {
-			/* Workaround for dTSEC Errata A002 */
-			usleep_range(100, 200);
-		} else {
-			/* Workaround for dTSEC Errata A004839 */
-			usleep_range(10, 50);
-		}
+	if (dtsec->fm_rev_info.major == 2) {
+		/* Workaround for dTSEC Errata A002 */
+		usleep_range(100, 200);
+	} else {
+		/* Workaround for dTSEC Errata A004839 */
+		usleep_range(10, 50);
 	}
 
 	/* Graceful stop - Assert the graceful Tx stop bit */
-	if (mode & COMM_MODE_TX) {
-		if (dtsec->fm_rev_info.major == 2) {
-			/* dTSEC Errata A004: Do not use TCTRL[GTS]=1 */
-			pr_debug("GTS not supported due to DTSEC_A004 Errata.\n");
-		} else {
-			tmp = ioread32be(&regs->tctrl) | TCTRL_GTS;
-			iowrite32be(tmp, &regs->tctrl);
+	if (dtsec->fm_rev_info.major == 2) {
+		/* dTSEC Errata A004: Do not use TCTRL[GTS]=1 */
+		pr_debug("GTS not supported due to DTSEC_A004 Errata.\n");
+	} else {
+		tmp = ioread32be(&regs->tctrl) | TCTRL_GTS;
+		iowrite32be(tmp, &regs->tctrl);
 
-			/* Workaround for dTSEC Errata A0012, A0014 */
-			usleep_range(10, 50);
-		}
+		/* Workaround for dTSEC Errata A0012, A0014 */
+		usleep_range(10, 50);
 	}
 }
 
-int dtsec_enable(struct fman_mac *dtsec, enum comm_mode mode)
+static int dtsec_enable(struct fman_mac *dtsec)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
 	u32 tmp;
@@ -916,58 +862,42 @@ int dtsec_enable(struct fman_mac *dtsec, enum comm_mode mode)
 
 	/* Enable */
 	tmp = ioread32be(&regs->maccfg1);
-	if (mode & COMM_MODE_RX)
-		tmp |= MACCFG1_RX_EN;
-	if (mode & COMM_MODE_TX)
-		tmp |= MACCFG1_TX_EN;
-
+	tmp |= MACCFG1_RX_EN | MACCFG1_TX_EN;
 	iowrite32be(tmp, &regs->maccfg1);
 
 	/* Graceful start - clear the graceful Rx/Tx stop bit */
-	graceful_start(dtsec, mode);
+	graceful_start(dtsec);
 
 	return 0;
 }
 
-int dtsec_disable(struct fman_mac *dtsec, enum comm_mode mode)
+static void dtsec_disable(struct fman_mac *dtsec)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
 	u32 tmp;
 
-	if (!is_init_done(dtsec->dtsec_drv_param))
-		return -EINVAL;
+	WARN_ON_ONCE(!is_init_done(dtsec->dtsec_drv_param));
 
 	/* Graceful stop - Assert the graceful Rx/Tx stop bit */
-	graceful_stop(dtsec, mode);
+	graceful_stop(dtsec);
 
 	tmp = ioread32be(&regs->maccfg1);
-	if (mode & COMM_MODE_RX)
-		tmp &= ~MACCFG1_RX_EN;
-	if (mode & COMM_MODE_TX)
-		tmp &= ~MACCFG1_TX_EN;
-
+	tmp &= ~(MACCFG1_RX_EN | MACCFG1_TX_EN);
 	iowrite32be(tmp, &regs->maccfg1);
-
-	return 0;
 }
 
-int dtsec_set_tx_pause_frames(struct fman_mac *dtsec,
-			      u8 __maybe_unused priority,
-			      u16 pause_time, u16 __maybe_unused thresh_time)
+static int dtsec_set_tx_pause_frames(struct fman_mac *dtsec,
+				     u8 __maybe_unused priority,
+				     u16 pause_time,
+				     u16 __maybe_unused thresh_time)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
-	enum comm_mode mode = COMM_MODE_NONE;
 	u32 ptv = 0;
 
 	if (!is_init_done(dtsec->dtsec_drv_param))
 		return -EINVAL;
 
-	if ((ioread32be(&regs->rctrl) & RCTRL_GRS) == 0)
-		mode |= COMM_MODE_RX;
-	if ((ioread32be(&regs->tctrl) & TCTRL_GTS) == 0)
-		mode |= COMM_MODE_TX;
-
-	graceful_stop(dtsec, mode);
+	graceful_stop(dtsec);
 
 	if (pause_time) {
 		/* FM_BAD_TX_TS_IN_B_2_B_ERRATA_DTSEC_A003 Errata workaround */
@@ -989,26 +919,20 @@ int dtsec_set_tx_pause_frames(struct fman_mac *dtsec,
 		iowrite32be(ioread32be(&regs->maccfg1) & ~MACCFG1_TX_FLOW,
 			    &regs->maccfg1);
 
-	graceful_start(dtsec, mode);
+	graceful_start(dtsec);
 
 	return 0;
 }
 
-int dtsec_accept_rx_pause_frames(struct fman_mac *dtsec, bool en)
+static int dtsec_accept_rx_pause_frames(struct fman_mac *dtsec, bool en)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
-	enum comm_mode mode = COMM_MODE_NONE;
 	u32 tmp;
 
 	if (!is_init_done(dtsec->dtsec_drv_param))
 		return -EINVAL;
 
-	if ((ioread32be(&regs->rctrl) & RCTRL_GRS) == 0)
-		mode |= COMM_MODE_RX;
-	if ((ioread32be(&regs->tctrl) & TCTRL_GTS) == 0)
-		mode |= COMM_MODE_TX;
-
-	graceful_stop(dtsec, mode);
+	graceful_stop(dtsec);
 
 	tmp = ioread32be(&regs->maccfg1);
 	if (en)
@@ -1017,25 +941,18 @@ int dtsec_accept_rx_pause_frames(struct fman_mac *dtsec, bool en)
 		tmp &= ~MACCFG1_RX_FLOW;
 	iowrite32be(tmp, &regs->maccfg1);
 
-	graceful_start(dtsec, mode);
+	graceful_start(dtsec);
 
 	return 0;
 }
 
-int dtsec_modify_mac_address(struct fman_mac *dtsec, const enet_addr_t *enet_addr)
+static int dtsec_modify_mac_address(struct fman_mac *dtsec,
+				    const enet_addr_t *enet_addr)
 {
-	struct dtsec_regs __iomem *regs = dtsec->regs;
-	enum comm_mode mode = COMM_MODE_NONE;
-
 	if (!is_init_done(dtsec->dtsec_drv_param))
 		return -EINVAL;
 
-	if ((ioread32be(&regs->rctrl) & RCTRL_GRS) == 0)
-		mode |= COMM_MODE_RX;
-	if ((ioread32be(&regs->tctrl) & TCTRL_GTS) == 0)
-		mode |= COMM_MODE_TX;
-
-	graceful_stop(dtsec, mode);
+	graceful_stop(dtsec);
 
 	/* Initialize MAC Station Address registers (1 & 2)
 	 * Station address have to be swapped (big endian to little endian
@@ -1043,12 +960,13 @@ int dtsec_modify_mac_address(struct fman_mac *dtsec, const enet_addr_t *enet_add
 	dtsec->addr = ENET_ADDR_TO_UINT64(*enet_addr);
 	set_mac_address(dtsec->regs, (const u8 *)(*enet_addr));
 
-	graceful_start(dtsec, mode);
+	graceful_start(dtsec);
 
 	return 0;
 }
 
-int dtsec_add_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr)
+static int dtsec_add_hash_mac_address(struct fman_mac *dtsec,
+				      enet_addr_t *eth_addr)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
 	struct eth_hash_entry *hash_entry;
@@ -1114,7 +1032,7 @@ int dtsec_add_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr)
 	return 0;
 }
 
-int dtsec_set_allmulti(struct fman_mac *dtsec, bool enable)
+static int dtsec_set_allmulti(struct fman_mac *dtsec, bool enable)
 {
 	u32 tmp;
 	struct dtsec_regs __iomem *regs = dtsec->regs;
@@ -1133,7 +1051,7 @@ int dtsec_set_allmulti(struct fman_mac *dtsec, bool enable)
 	return 0;
 }
 
-int dtsec_set_tstamp(struct fman_mac *dtsec, bool enable)
+static int dtsec_set_tstamp(struct fman_mac *dtsec, bool enable)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
 	u32 rctrl, tctrl;
@@ -1158,7 +1076,8 @@ int dtsec_set_tstamp(struct fman_mac *dtsec, bool enable)
 	return 0;
 }
 
-int dtsec_del_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr)
+static int dtsec_del_hash_mac_address(struct fman_mac *dtsec,
+				      enet_addr_t *eth_addr)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
 	struct list_head *pos;
@@ -1229,7 +1148,7 @@ int dtsec_del_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr)
 	return 0;
 }
 
-int dtsec_set_promiscuous(struct fman_mac *dtsec, bool new_val)
+static int dtsec_set_promiscuous(struct fman_mac *dtsec, bool new_val)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
 	u32 tmp;
@@ -1258,21 +1177,15 @@ int dtsec_set_promiscuous(struct fman_mac *dtsec, bool new_val)
 	return 0;
 }
 
-int dtsec_adjust_link(struct fman_mac *dtsec, u16 speed)
+static int dtsec_adjust_link(struct fman_mac *dtsec, u16 speed)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
-	enum comm_mode mode = COMM_MODE_NONE;
 	u32 tmp;
 
 	if (!is_init_done(dtsec->dtsec_drv_param))
 		return -EINVAL;
 
-	if ((ioread32be(&regs->rctrl) & RCTRL_GRS) == 0)
-		mode |= COMM_MODE_RX;
-	if ((ioread32be(&regs->tctrl) & TCTRL_GTS) == 0)
-		mode |= COMM_MODE_TX;
-
-	graceful_stop(dtsec, mode);
+	graceful_stop(dtsec);
 
 	tmp = ioread32be(&regs->maccfg2);
 
@@ -1293,12 +1206,12 @@ int dtsec_adjust_link(struct fman_mac *dtsec, u16 speed)
 		tmp &= ~DTSEC_ECNTRL_R100M;
 	iowrite32be(tmp, &regs->ecntrl);
 
-	graceful_start(dtsec, mode);
+	graceful_start(dtsec);
 
 	return 0;
 }
 
-int dtsec_restart_autoneg(struct fman_mac *dtsec)
+static int dtsec_restart_autoneg(struct fman_mac *dtsec)
 {
 	u16 tmp_reg16;
 
@@ -1316,20 +1229,31 @@ int dtsec_restart_autoneg(struct fman_mac *dtsec)
 	return 0;
 }
 
-int dtsec_get_version(struct fman_mac *dtsec, u32 *mac_version)
+static void adjust_link_dtsec(struct mac_device *mac_dev)
 {
-	struct dtsec_regs __iomem *regs = dtsec->regs;
+	struct phy_device *phy_dev = mac_dev->phy_dev;
+	struct fman_mac *fman_mac;
+	bool rx_pause, tx_pause;
+	int err;
 
-	if (!is_init_done(dtsec->dtsec_drv_param))
-		return -EINVAL;
+	fman_mac = mac_dev->fman_mac;
+	if (!phy_dev->link) {
+		dtsec_restart_autoneg(fman_mac);
 
-	*mac_version = ioread32be(&regs->tsec_id);
+		return;
+	}
 
-	return 0;
+	dtsec_adjust_link(fman_mac, phy_dev->speed);
+	mac_dev->update_speed(mac_dev, phy_dev->speed);
+	fman_get_pause_cfg(mac_dev, &rx_pause, &tx_pause);
+	err = fman_set_mac_active_pause(mac_dev, rx_pause, tx_pause);
+	if (err < 0)
+		dev_err(mac_dev->dev, "fman_set_mac_active_pause() = %d\n",
+			err);
 }
 
-int dtsec_set_exception(struct fman_mac *dtsec,
-			enum fman_mac_exceptions exception, bool enable)
+static int dtsec_set_exception(struct fman_mac *dtsec,
+			       enum fman_mac_exceptions exception, bool enable)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
 	u32 bit_mask = 0;
@@ -1382,7 +1306,7 @@ int dtsec_set_exception(struct fman_mac *dtsec,
 	return 0;
 }
 
-int dtsec_init(struct fman_mac *dtsec)
+static int dtsec_init(struct fman_mac *dtsec)
 {
 	struct dtsec_regs __iomem *regs = dtsec->regs;
 	struct dtsec_cfg *dtsec_drv_param;
@@ -1476,7 +1400,7 @@ int dtsec_init(struct fman_mac *dtsec)
 	return 0;
 }
 
-int dtsec_free(struct fman_mac *dtsec)
+static int dtsec_free(struct fman_mac *dtsec)
 {
 	free_init_resources(dtsec);
 
@@ -1487,13 +1411,11 @@ int dtsec_free(struct fman_mac *dtsec)
 	return 0;
 }
 
-struct fman_mac *dtsec_config(struct fman_mac_params *params)
+static struct fman_mac *dtsec_config(struct mac_device *mac_dev,
+				     struct fman_mac_params *params)
 {
 	struct fman_mac *dtsec;
 	struct dtsec_cfg *dtsec_drv_param;
-	void __iomem *base_addr;
-
-	base_addr = params->base_addr;
 
 	/* allocate memory for the UCC GETH data structure. */
 	dtsec = kzalloc(sizeof(*dtsec), GFP_KERNEL);
@@ -1510,10 +1432,10 @@ struct fman_mac *dtsec_config(struct fman_mac_params *params)
 
 	set_dflts(dtsec_drv_param);
 
-	dtsec->regs = base_addr;
-	dtsec->addr = ENET_ADDR_TO_UINT64(params->addr);
+	dtsec->regs = mac_dev->vaddr;
+	dtsec->addr = ENET_ADDR_TO_UINT64(mac_dev->addr);
 	dtsec->max_speed = params->max_speed;
-	dtsec->phy_if = params->phy_if;
+	dtsec->phy_if = mac_dev->phy_if;
 	dtsec->mac_id = params->mac_id;
 	dtsec->exceptions = (DTSEC_IMASK_BREN	|
 			     DTSEC_IMASK_RXCEN	|
@@ -1530,34 +1452,87 @@ struct fman_mac *dtsec_config(struct fman_mac_params *params)
 			     DTSEC_IMASK_RDPEEN);
 	dtsec->exception_cb = params->exception_cb;
 	dtsec->event_cb = params->event_cb;
-	dtsec->dev_id = params->dev_id;
+	dtsec->dev_id = mac_dev;
 	dtsec->ptp_tsu_enabled = dtsec->dtsec_drv_param->ptp_tsu_en;
 	dtsec->en_tsu_err_exception = dtsec->dtsec_drv_param->ptp_exception_en;
 
 	dtsec->fm = params->fm;
 	dtsec->basex_if = params->basex_if;
 
-	if (!params->internal_phy_node) {
+	/* Save FMan revision */
+	fman_get_revision(dtsec->fm, &dtsec->fm_rev_info);
+
+	return dtsec;
+
+err_dtsec:
+	kfree(dtsec);
+	return NULL;
+}
+
+int dtsec_initialization(struct mac_device *mac_dev,
+			 struct device_node *mac_node,
+			 struct fman_mac_params *params)
+{
+	int			err;
+	struct fman_mac		*dtsec;
+	struct device_node	*phy_node;
+
+	mac_dev->set_promisc		= dtsec_set_promiscuous;
+	mac_dev->change_addr		= dtsec_modify_mac_address;
+	mac_dev->add_hash_mac_addr	= dtsec_add_hash_mac_address;
+	mac_dev->remove_hash_mac_addr	= dtsec_del_hash_mac_address;
+	mac_dev->set_tx_pause		= dtsec_set_tx_pause_frames;
+	mac_dev->set_rx_pause		= dtsec_accept_rx_pause_frames;
+	mac_dev->set_exception		= dtsec_set_exception;
+	mac_dev->set_allmulti		= dtsec_set_allmulti;
+	mac_dev->set_tstamp		= dtsec_set_tstamp;
+	mac_dev->set_multi		= fman_set_multi;
+	mac_dev->adjust_link            = adjust_link_dtsec;
+	mac_dev->enable			= dtsec_enable;
+	mac_dev->disable		= dtsec_disable;
+
+	mac_dev->fman_mac = dtsec_config(mac_dev, params);
+	if (!mac_dev->fman_mac) {
+		err = -EINVAL;
+		goto _return;
+	}
+
+	dtsec = mac_dev->fman_mac;
+	dtsec->dtsec_drv_param->maximum_frame = fman_get_max_frm();
+	dtsec->dtsec_drv_param->tx_pad_crc = true;
+
+	phy_node = of_parse_phandle(mac_node, "tbi-handle", 0);
+	if (!phy_node) {
 		pr_err("TBI PHY node is not available\n");
-		goto err_dtsec_drv_param;
+		err = -EINVAL;
+		goto _return_fm_mac_free;
 	}
 
-	dtsec->tbiphy = of_phy_find_device(params->internal_phy_node);
+	dtsec->tbiphy = of_phy_find_device(phy_node);
 	if (!dtsec->tbiphy) {
 		pr_err("of_phy_find_device (TBI PHY) failed\n");
-		goto err_dtsec_drv_param;
+		err = -EINVAL;
+		goto _return_fm_mac_free;
 	}
-
 	put_device(&dtsec->tbiphy->mdio.dev);
 
-	/* Save FMan revision */
-	fman_get_revision(dtsec->fm, &dtsec->fm_rev_info);
+	err = dtsec_init(dtsec);
+	if (err < 0)
+		goto _return_fm_mac_free;
 
-	return dtsec;
+	/* For 1G MAC, disable by default the MIB counters overflow interrupt */
+	err = dtsec_set_exception(dtsec, FM_MAC_EX_1G_RX_MIB_CNT_OVFL, false);
+	if (err < 0)
+		goto _return_fm_mac_free;
 
-err_dtsec_drv_param:
-	kfree(dtsec_drv_param);
-err_dtsec:
-	kfree(dtsec);
-	return NULL;
+	dev_info(mac_dev->dev, "FMan dTSEC version: 0x%08x\n",
+		 ioread32be(&dtsec->regs->tsec_id));
+
+	goto _return;
+
+_return_fm_mac_free:
+	dtsec_free(dtsec);
+
+_return:
+	return err;
 }
diff --git a/drivers/net/ethernet/freescale/fman/fman_dtsec.h b/drivers/net/ethernet/freescale/fman/fman_dtsec.h
index 68512c3bd6e5..8c72d280c51a 100644
--- a/drivers/net/ethernet/freescale/fman/fman_dtsec.h
+++ b/drivers/net/ethernet/freescale/fman/fman_dtsec.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */
 /*
- * Copyright 2008-2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  */
 
 #ifndef __DTSEC_H
@@ -35,27 +8,10 @@
 
 #include "fman_mac.h"
 
-struct fman_mac *dtsec_config(struct fman_mac_params *params);
-int dtsec_set_promiscuous(struct fman_mac *dtsec, bool new_val);
-int dtsec_modify_mac_address(struct fman_mac *dtsec, const enet_addr_t *enet_addr);
-int dtsec_adjust_link(struct fman_mac *dtsec,
-		      u16 speed);
-int dtsec_restart_autoneg(struct fman_mac *dtsec);
-int dtsec_cfg_max_frame_len(struct fman_mac *dtsec, u16 new_val);
-int dtsec_cfg_pad_and_crc(struct fman_mac *dtsec, bool new_val);
-int dtsec_enable(struct fman_mac *dtsec, enum comm_mode mode);
-int dtsec_disable(struct fman_mac *dtsec, enum comm_mode mode);
-int dtsec_init(struct fman_mac *dtsec);
-int dtsec_free(struct fman_mac *dtsec);
-int dtsec_accept_rx_pause_frames(struct fman_mac *dtsec, bool en);
-int dtsec_set_tx_pause_frames(struct fman_mac *dtsec, u8 priority,
-			      u16 pause_time, u16 thresh_time);
-int dtsec_set_exception(struct fman_mac *dtsec,
-			enum fman_mac_exceptions exception, bool enable);
-int dtsec_add_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr);
-int dtsec_del_hash_mac_address(struct fman_mac *dtsec, enet_addr_t *eth_addr);
-int dtsec_get_version(struct fman_mac *dtsec, u32 *mac_version);
-int dtsec_set_allmulti(struct fman_mac *dtsec, bool enable);
-int dtsec_set_tstamp(struct fman_mac *dtsec, bool enable);
+struct mac_device;
+
+int dtsec_initialization(struct mac_device *mac_dev,
+			 struct device_node *mac_node,
+			 struct fman_mac_params *params);
 
 #endif /* __DTSEC_H */
diff --git a/drivers/net/ethernet/freescale/fman/fman_keygen.c b/drivers/net/ethernet/freescale/fman/fman_keygen.c
index e1bdfed16134..e73f6ef3c6ee 100644
--- a/drivers/net/ethernet/freescale/fman/fman_keygen.c
+++ b/drivers/net/ethernet/freescale/fman/fman_keygen.c
@@ -1,33 +1,6 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
 /*
  * Copyright 2017 NXP
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of NXP nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY NXP ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NXP BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
diff --git a/drivers/net/ethernet/freescale/fman/fman_keygen.h b/drivers/net/ethernet/freescale/fman/fman_keygen.h
index c4640de3f4cb..2cb0df453074 100644
--- a/drivers/net/ethernet/freescale/fman/fman_keygen.h
+++ b/drivers/net/ethernet/freescale/fman/fman_keygen.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */
 /*
  * Copyright 2017 NXP
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of NXP nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY NXP ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL NXP BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #ifndef __KEYGEN_H
diff --git a/drivers/net/ethernet/freescale/fman/fman_mac.h b/drivers/net/ethernet/freescale/fman/fman_mac.h
index 19f327efdaff..65887a3160d7 100644
--- a/drivers/net/ethernet/freescale/fman/fman_mac.h
+++ b/drivers/net/ethernet/freescale/fman/fman_mac.h
@@ -41,6 +41,7 @@
 #include <linux/if_ether.h>
 
 struct fman_mac;
+struct mac_device;
 
 /* Ethernet Address */
 typedef u8 enet_addr_t[ETH_ALEN];
@@ -75,16 +76,6 @@ typedef u8 enet_addr_t[ETH_ALEN];
 #define ETH_HASH_ENTRY_OBJ(ptr)	\
 	hlist_entry_safe(ptr, struct eth_hash_entry, node)
 
-/* Enumeration (bit flags) of communication modes (Transmit,
- * receive or both).
- */
-enum comm_mode {
-	COMM_MODE_NONE = 0,	/* No transmit/receive communication */
-	COMM_MODE_RX = 1,	/* Only receive communication */
-	COMM_MODE_TX = 2,	/* Only transmit communication */
-	COMM_MODE_RX_AND_TX = 3	/* Both transmit and receive communication */
-};
-
 /* FM MAC Exceptions */
 enum fman_mac_exceptions {
 	FM_MAC_EX_10G_MDIO_SCAN_EVENT = 0
@@ -168,30 +159,23 @@ struct eth_hash_entry {
 	struct list_head node;
 };
 
-typedef void (fman_mac_exception_cb)(void *dev_id,
-				    enum fman_mac_exceptions exceptions);
+typedef void (fman_mac_exception_cb)(struct mac_device *dev_id,
+				     enum fman_mac_exceptions exceptions);
 
 /* FMan MAC config input */
 struct fman_mac_params {
-	/* Base of memory mapped FM MAC registers */
-	void __iomem *base_addr;
-	/* MAC address of device; First octet is sent first */
-	enet_addr_t addr;
 	/* MAC ID; numbering of dTSEC and 1G-mEMAC:
 	 * 0 - FM_MAX_NUM_OF_1G_MACS;
 	 * numbering of 10G-MAC (TGEC) and 10G-mEMAC:
 	 * 0 - FM_MAX_NUM_OF_10G_MACS
 	 */
 	u8 mac_id;
-	/* PHY interface */
-	phy_interface_t	 phy_if;
 	/* Note that the speed should indicate the maximum rate that
 	 * this MAC should support rather than the actual speed;
 	 */
 	u16 max_speed;
 	/* A handle to the FM object this port related to */
 	void *fm;
-	void *dev_id; /* device cookie used by the exception cbs */
 	fman_mac_exception_cb *event_cb;    /* MDIO Events Callback Routine */
 	fman_mac_exception_cb *exception_cb;/* Exception Callback Routine */
 	/* SGMII/QSGII interface with 1000BaseX auto-negotiation between MAC
@@ -200,8 +184,6 @@ struct fman_mac_params {
 	 * synchronize with far-end phy at 10Mbps, 100Mbps or 1000Mbps
 	*/
 	bool basex_if;
-	/* Pointer to TBI/PCS PHY node, used for TBI/PCS PHY access */
-	struct device_node *internal_phy_node;
 };
 
 struct eth_hash_t {
diff --git a/drivers/net/ethernet/freescale/fman/fman_memac.c b/drivers/net/ethernet/freescale/fman/fman_memac.c
index 2216b7f51d26..32d26cf17843 100644
--- a/drivers/net/ethernet/freescale/fman/fman_memac.c
+++ b/drivers/net/ethernet/freescale/fman/fman_memac.c
@@ -1,39 +1,13 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
 /*
- * Copyright 2008-2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include "fman_memac.h"
 #include "fman.h"
+#include "mac.h"
 
 #include <linux/slab.h>
 #include <linux/io.h>
@@ -337,7 +311,7 @@ struct fman_mac {
 	/* Ethernet physical interface */
 	phy_interface_t phy_if;
 	u16 max_speed;
-	void *dev_id; /* device cookie used by the exception cbs */
+	struct mac_device *dev_id; /* device cookie used by the exception cbs */
 	fman_mac_exception_cb *exception_cb;
 	fman_mac_exception_cb *event_cb;
 	/* Pointer to driver's global address hash table  */
@@ -712,7 +686,7 @@ static bool is_init_done(struct memac_cfg *memac_drv_params)
 	return false;
 }
 
-int memac_enable(struct fman_mac *memac, enum comm_mode mode)
+static int memac_enable(struct fman_mac *memac)
 {
 	struct memac_regs __iomem *regs = memac->regs;
 	u32 tmp;
@@ -721,36 +695,26 @@ int memac_enable(struct fman_mac *memac, enum comm_mode mode)
 		return -EINVAL;
 
 	tmp = ioread32be(&regs->command_config);
-	if (mode & COMM_MODE_RX)
-		tmp |= CMD_CFG_RX_EN;
-	if (mode & COMM_MODE_TX)
-		tmp |= CMD_CFG_TX_EN;
-
+	tmp |= CMD_CFG_RX_EN | CMD_CFG_TX_EN;
 	iowrite32be(tmp, &regs->command_config);
 
 	return 0;
 }
 
-int memac_disable(struct fman_mac *memac, enum comm_mode mode)
+static void memac_disable(struct fman_mac *memac)
+
 {
 	struct memac_regs __iomem *regs = memac->regs;
 	u32 tmp;
 
-	if (!is_init_done(memac->memac_drv_param))
-		return -EINVAL;
+	WARN_ON_ONCE(!is_init_done(memac->memac_drv_param));
 
 	tmp = ioread32be(&regs->command_config);
-	if (mode & COMM_MODE_RX)
-		tmp &= ~CMD_CFG_RX_EN;
-	if (mode & COMM_MODE_TX)
-		tmp &= ~CMD_CFG_TX_EN;
-
+	tmp &= ~(CMD_CFG_RX_EN | CMD_CFG_TX_EN);
 	iowrite32be(tmp, &regs->command_config);
-
-	return 0;
 }
 
-int memac_set_promiscuous(struct fman_mac *memac, bool new_val)
+static int memac_set_promiscuous(struct fman_mac *memac, bool new_val)
 {
 	struct memac_regs __iomem *regs = memac->regs;
 	u32 tmp;
@@ -769,7 +733,7 @@ int memac_set_promiscuous(struct fman_mac *memac, bool new_val)
 	return 0;
 }
 
-int memac_adjust_link(struct fman_mac *memac, u16 speed)
+static int memac_adjust_link(struct fman_mac *memac, u16 speed)
 {
 	struct memac_regs __iomem *regs = memac->regs;
 	u32 tmp;
@@ -809,39 +773,26 @@ int memac_adjust_link(struct fman_mac *memac, u16 speed)
 	return 0;
 }
 
-int memac_cfg_max_frame_len(struct fman_mac *memac, u16 new_val)
-{
-	if (is_init_done(memac->memac_drv_param))
-		return -EINVAL;
-
-	memac->memac_drv_param->max_frame_length = new_val;
-
-	return 0;
-}
-
-int memac_cfg_reset_on_init(struct fman_mac *memac, bool enable)
-{
-	if (is_init_done(memac->memac_drv_param))
-		return -EINVAL;
-
-	memac->memac_drv_param->reset_on_init = enable;
-
-	return 0;
-}
-
-int memac_cfg_fixed_link(struct fman_mac *memac,
-			 struct fixed_phy_status *fixed_link)
+static void adjust_link_memac(struct mac_device *mac_dev)
 {
-	if (is_init_done(memac->memac_drv_param))
-		return -EINVAL;
+	struct phy_device *phy_dev = mac_dev->phy_dev;
+	struct fman_mac *fman_mac;
+	bool rx_pause, tx_pause;
+	int err;
 
-	memac->memac_drv_param->fixed_link = fixed_link;
+	fman_mac = mac_dev->fman_mac;
+	memac_adjust_link(fman_mac, phy_dev->speed);
+	mac_dev->update_speed(mac_dev, phy_dev->speed);
 
-	return 0;
+	fman_get_pause_cfg(mac_dev, &rx_pause, &tx_pause);
+	err = fman_set_mac_active_pause(mac_dev, rx_pause, tx_pause);
+	if (err < 0)
+		dev_err(mac_dev->dev, "fman_set_mac_active_pause() = %d\n",
+			err);
 }
 
-int memac_set_tx_pause_frames(struct fman_mac *memac, u8 priority,
-			      u16 pause_time, u16 thresh_time)
+static int memac_set_tx_pause_frames(struct fman_mac *memac, u8 priority,
+				     u16 pause_time, u16 thresh_time)
 {
 	struct memac_regs __iomem *regs = memac->regs;
 	u32 tmp;
@@ -878,7 +829,7 @@ int memac_set_tx_pause_frames(struct fman_mac *memac, u8 priority,
 	return 0;
 }
 
-int memac_accept_rx_pause_frames(struct fman_mac *memac, bool en)
+static int memac_accept_rx_pause_frames(struct fman_mac *memac, bool en)
 {
 	struct memac_regs __iomem *regs = memac->regs;
 	u32 tmp;
@@ -897,7 +848,8 @@ int memac_accept_rx_pause_frames(struct fman_mac *memac, bool en)
 	return 0;
 }
 
-int memac_modify_mac_address(struct fman_mac *memac, const enet_addr_t *enet_addr)
+static int memac_modify_mac_address(struct fman_mac *memac,
+				    const enet_addr_t *enet_addr)
 {
 	if (!is_init_done(memac->memac_drv_param))
 		return -EINVAL;
@@ -907,7 +859,8 @@ int memac_modify_mac_address(struct fman_mac *memac, const enet_addr_t *enet_add
 	return 0;
 }
 
-int memac_add_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr)
+static int memac_add_hash_mac_address(struct fman_mac *memac,
+				      enet_addr_t *eth_addr)
 {
 	struct memac_regs __iomem *regs = memac->regs;
 	struct eth_hash_entry *hash_entry;
@@ -940,7 +893,7 @@ int memac_add_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr)
 	return 0;
 }
 
-int memac_set_allmulti(struct fman_mac *memac, bool enable)
+static int memac_set_allmulti(struct fman_mac *memac, bool enable)
 {
 	u32 entry;
 	struct memac_regs __iomem *regs = memac->regs;
@@ -963,12 +916,13 @@ int memac_set_allmulti(struct fman_mac *memac, bool enable)
 	return 0;
 }
 
-int memac_set_tstamp(struct fman_mac *memac, bool enable)
+static int memac_set_tstamp(struct fman_mac *memac, bool enable)
 {
 	return 0; /* Always enabled. */
 }
 
-int memac_del_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr)
+static int memac_del_hash_mac_address(struct fman_mac *memac,
+				      enet_addr_t *eth_addr)
 {
 	struct memac_regs __iomem *regs = memac->regs;
 	struct eth_hash_entry *hash_entry = NULL;
@@ -1001,8 +955,8 @@ int memac_del_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr)
 	return 0;
 }
 
-int memac_set_exception(struct fman_mac *memac,
-			enum fman_mac_exceptions exception, bool enable)
+static int memac_set_exception(struct fman_mac *memac,
+			       enum fman_mac_exceptions exception, bool enable)
 {
 	u32 bit_mask = 0;
 
@@ -1024,13 +978,13 @@ int memac_set_exception(struct fman_mac *memac,
 	return 0;
 }
 
-int memac_init(struct fman_mac *memac)
+static int memac_init(struct fman_mac *memac)
 {
 	struct memac_cfg *memac_drv_param;
 	u8 i;
 	enet_addr_t eth_addr;
 	bool slow_10g_if = false;
-	struct fixed_phy_status *fixed_link;
+	struct fixed_phy_status *fixed_link = NULL;
 	int err;
 	u32 reg32 = 0;
 
@@ -1141,7 +1095,7 @@ int memac_init(struct fman_mac *memac)
 	return 0;
 }
 
-int memac_free(struct fman_mac *memac)
+static int memac_free(struct fman_mac *memac)
 {
 	free_init_resources(memac);
 
@@ -1154,13 +1108,12 @@ int memac_free(struct fman_mac *memac)
 	return 0;
 }
 
-struct fman_mac *memac_config(struct fman_mac_params *params)
+static struct fman_mac *memac_config(struct mac_device *mac_dev,
+				     struct fman_mac_params *params)
 {
 	struct fman_mac *memac;
 	struct memac_cfg *memac_drv_param;
-	void __iomem *base_addr;
 
-	base_addr = params->base_addr;
 	/* allocate memory for the m_emac data structure */
 	memac = kzalloc(sizeof(*memac), GFP_KERNEL);
 	if (!memac)
@@ -1178,38 +1131,121 @@ struct fman_mac *memac_config(struct fman_mac_params *params)
 
 	set_dflts(memac_drv_param);
 
-	memac->addr = ENET_ADDR_TO_UINT64(params->addr);
+	memac->addr = ENET_ADDR_TO_UINT64(mac_dev->addr);
 
-	memac->regs = base_addr;
+	memac->regs = mac_dev->vaddr;
 	memac->max_speed = params->max_speed;
-	memac->phy_if = params->phy_if;
+	memac->phy_if = mac_dev->phy_if;
 	memac->mac_id = params->mac_id;
 	memac->exceptions = (MEMAC_IMASK_TSECC_ER | MEMAC_IMASK_TECC_ER |
 			     MEMAC_IMASK_RECC_ER | MEMAC_IMASK_MGI);
 	memac->exception_cb = params->exception_cb;
 	memac->event_cb = params->event_cb;
-	memac->dev_id = params->dev_id;
+	memac->dev_id = mac_dev;
 	memac->fm = params->fm;
 	memac->basex_if = params->basex_if;
 
 	/* Save FMan revision */
 	fman_get_revision(memac->fm, &memac->fm_rev_info);
 
+	return memac;
+}
+
+int memac_initialization(struct mac_device *mac_dev,
+			 struct device_node *mac_node,
+			 struct fman_mac_params *params)
+{
+	int			 err;
+	struct device_node	*phy_node;
+	struct fixed_phy_status *fixed_link;
+	struct fman_mac		*memac;
+
+	mac_dev->set_promisc		= memac_set_promiscuous;
+	mac_dev->change_addr		= memac_modify_mac_address;
+	mac_dev->add_hash_mac_addr	= memac_add_hash_mac_address;
+	mac_dev->remove_hash_mac_addr	= memac_del_hash_mac_address;
+	mac_dev->set_tx_pause		= memac_set_tx_pause_frames;
+	mac_dev->set_rx_pause		= memac_accept_rx_pause_frames;
+	mac_dev->set_exception		= memac_set_exception;
+	mac_dev->set_allmulti		= memac_set_allmulti;
+	mac_dev->set_tstamp		= memac_set_tstamp;
+	mac_dev->set_multi		= fman_set_multi;
+	mac_dev->adjust_link            = adjust_link_memac;
+	mac_dev->enable			= memac_enable;
+	mac_dev->disable		= memac_disable;
+
+	if (params->max_speed == SPEED_10000)
+		mac_dev->phy_if = PHY_INTERFACE_MODE_XGMII;
+
+	mac_dev->fman_mac = memac_config(mac_dev, params);
+	if (!mac_dev->fman_mac) {
+		err = -EINVAL;
+		goto _return;
+	}
+
+	memac = mac_dev->fman_mac;
+	memac->memac_drv_param->max_frame_length = fman_get_max_frm();
+	memac->memac_drv_param->reset_on_init = true;
 	if (memac->phy_if == PHY_INTERFACE_MODE_SGMII ||
 	    memac->phy_if == PHY_INTERFACE_MODE_QSGMII) {
-		if (!params->internal_phy_node) {
+		phy_node = of_parse_phandle(mac_node, "pcsphy-handle", 0);
+		if (!phy_node) {
 			pr_err("PCS PHY node is not available\n");
-			memac_free(memac);
-			return NULL;
+			err = -EINVAL;
+			goto _return_fm_mac_free;
 		}
 
-		memac->pcsphy = of_phy_find_device(params->internal_phy_node);
+		memac->pcsphy = of_phy_find_device(phy_node);
 		if (!memac->pcsphy) {
 			pr_err("of_phy_find_device (PCS PHY) failed\n");
-			memac_free(memac);
-			return NULL;
+			err = -EINVAL;
+			goto _return_fm_mac_free;
 		}
 	}
 
-	return memac;
+	if (!mac_dev->phy_node && of_phy_is_fixed_link(mac_node)) {
+		struct phy_device *phy;
+
+		err = of_phy_register_fixed_link(mac_node);
+		if (err)
+			goto _return_fm_mac_free;
+
+		fixed_link = kzalloc(sizeof(*fixed_link), GFP_KERNEL);
+		if (!fixed_link) {
+			err = -ENOMEM;
+			goto _return_fm_mac_free;
+		}
+
+		mac_dev->phy_node = of_node_get(mac_node);
+		phy = of_phy_find_device(mac_dev->phy_node);
+		if (!phy) {
+			err = -EINVAL;
+			of_node_put(mac_dev->phy_node);
+			goto _return_fixed_link_free;
+		}
+
+		fixed_link->link = phy->link;
+		fixed_link->speed = phy->speed;
+		fixed_link->duplex = phy->duplex;
+		fixed_link->pause = phy->pause;
+		fixed_link->asym_pause = phy->asym_pause;
+
+		put_device(&phy->mdio.dev);
+		memac->memac_drv_param->fixed_link = fixed_link;
+	}
+
+	err = memac_init(mac_dev->fman_mac);
+	if (err < 0)
+		goto _return_fixed_link_free;
+
+	dev_info(mac_dev->dev, "FMan MEMAC\n");
+
+	goto _return;
+
+_return_fixed_link_free:
+	kfree(fixed_link);
+_return_fm_mac_free:
+	memac_free(mac_dev->fman_mac);
+_return:
+	return err;
 }
diff --git a/drivers/net/ethernet/freescale/fman/fman_memac.h b/drivers/net/ethernet/freescale/fman/fman_memac.h
index 3820f7a22983..5a3a14f9684f 100644
--- a/drivers/net/ethernet/freescale/fman/fman_memac.h
+++ b/drivers/net/ethernet/freescale/fman/fman_memac.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */
 /*
- * Copyright 2008-2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  */
 
 #ifndef __MEMAC_H
@@ -38,26 +11,10 @@
 #include <linux/netdevice.h>
 #include <linux/phy_fixed.h>
 
-struct fman_mac *memac_config(struct fman_mac_params *params);
-int memac_set_promiscuous(struct fman_mac *memac, bool new_val);
-int memac_modify_mac_address(struct fman_mac *memac, const enet_addr_t *enet_addr);
-int memac_adjust_link(struct fman_mac *memac, u16 speed);
-int memac_cfg_max_frame_len(struct fman_mac *memac, u16 new_val);
-int memac_cfg_reset_on_init(struct fman_mac *memac, bool enable);
-int memac_cfg_fixed_link(struct fman_mac *memac,
-			 struct fixed_phy_status *fixed_link);
-int memac_enable(struct fman_mac *memac, enum comm_mode mode);
-int memac_disable(struct fman_mac *memac, enum comm_mode mode);
-int memac_init(struct fman_mac *memac);
-int memac_free(struct fman_mac *memac);
-int memac_accept_rx_pause_frames(struct fman_mac *memac, bool en);
-int memac_set_tx_pause_frames(struct fman_mac *memac, u8 priority,
-			      u16 pause_time, u16 thresh_time);
-int memac_set_exception(struct fman_mac *memac,
-			enum fman_mac_exceptions exception, bool enable);
-int memac_add_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr);
-int memac_del_hash_mac_address(struct fman_mac *memac, enet_addr_t *eth_addr);
-int memac_set_allmulti(struct fman_mac *memac, bool enable);
-int memac_set_tstamp(struct fman_mac *memac, bool enable);
+struct mac_device;
+
+int memac_initialization(struct mac_device *mac_dev,
+			 struct device_node *mac_node,
+			 struct fman_mac_params *params);
 
 #endif /* __MEMAC_H */
diff --git a/drivers/net/ethernet/freescale/fman/fman_muram.c b/drivers/net/ethernet/freescale/fman/fman_muram.c
index 7ad317e622bc..f557d68e5b76 100644
--- a/drivers/net/ethernet/freescale/fman/fman_muram.c
+++ b/drivers/net/ethernet/freescale/fman/fman_muram.c
@@ -1,33 +1,6 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
 /*
- * Copyright 2008-2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  */
 
 #include "fman_muram.h"
diff --git a/drivers/net/ethernet/freescale/fman/fman_muram.h b/drivers/net/ethernet/freescale/fman/fman_muram.h
index 453bf849eee1..3643af61bae2 100644
--- a/drivers/net/ethernet/freescale/fman/fman_muram.h
+++ b/drivers/net/ethernet/freescale/fman/fman_muram.h
@@ -1,34 +1,8 @@
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */
 /*
- * Copyright 2008-2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  */
+
 #ifndef __FM_MURAM_EXT
 #define __FM_MURAM_EXT
 
diff --git a/drivers/net/ethernet/freescale/fman/fman_port.c b/drivers/net/ethernet/freescale/fman/fman_port.c
index 4c9d05c45c03..ab90fe2bee5e 100644
--- a/drivers/net/ethernet/freescale/fman/fman_port.c
+++ b/drivers/net/ethernet/freescale/fman/fman_port.c
@@ -1,33 +1,6 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
 /*
  * Copyright 2008 - 2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
diff --git a/drivers/net/ethernet/freescale/fman/fman_port.h b/drivers/net/ethernet/freescale/fman/fman_port.h
index 82f12661a46d..4917fe8f0617 100644
--- a/drivers/net/ethernet/freescale/fman/fman_port.h
+++ b/drivers/net/ethernet/freescale/fman/fman_port.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */
 /*
  * Copyright 2008 - 2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #ifndef __FMAN_PORT_H
diff --git a/drivers/net/ethernet/freescale/fman/fman_sp.c b/drivers/net/ethernet/freescale/fman/fman_sp.c
index 248f5bcca468..0fac60aa5283 100644
--- a/drivers/net/ethernet/freescale/fman/fman_sp.c
+++ b/drivers/net/ethernet/freescale/fman/fman_sp.c
@@ -1,33 +1,6 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
 /*
  * Copyright 2008 - 2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include "fman_sp.h"
diff --git a/drivers/net/ethernet/freescale/fman/fman_sp.h b/drivers/net/ethernet/freescale/fman/fman_sp.h
index 820b7f63088f..a62dd21c81f1 100644
--- a/drivers/net/ethernet/freescale/fman/fman_sp.h
+++ b/drivers/net/ethernet/freescale/fman/fman_sp.h
@@ -1,32 +1,6 @@
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */
 /*
  * Copyright 2008 - 2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *	 notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *	 notice, this list of conditions and the following disclaimer in the
- *	 documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *	 names of its contributors may be used to endorse or promote products
- *	 derived from this software without specific prior written permission.
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #ifndef __FM_SP_H
diff --git a/drivers/net/ethernet/freescale/fman/fman_tgec.c b/drivers/net/ethernet/freescale/fman/fman_tgec.c
index 311c1906e044..5a4be54ad459 100644
--- a/drivers/net/ethernet/freescale/fman/fman_tgec.c
+++ b/drivers/net/ethernet/freescale/fman/fman_tgec.c
@@ -1,39 +1,13 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
 /*
- * Copyright 2008-2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include "fman_tgec.h"
 #include "fman.h"
+#include "mac.h"
 
 #include <linux/slab.h>
 #include <linux/bitrev.h>
@@ -206,7 +180,7 @@ struct fman_mac {
 	/* MAC address of device; */
 	u64 addr;
 	u16 max_speed;
-	void *dev_id; /* device cookie used by the exception cbs */
+	struct mac_device *dev_id; /* device cookie used by the exception cbs */
 	fman_mac_exception_cb *exception_cb;
 	fman_mac_exception_cb *event_cb;
 	/* pointer to driver's global address hash table  */
@@ -419,7 +393,7 @@ static bool is_init_done(struct tgec_cfg *cfg)
 	return false;
 }
 
-int tgec_enable(struct fman_mac *tgec, enum comm_mode mode)
+static int tgec_enable(struct fman_mac *tgec)
 {
 	struct tgec_regs __iomem *regs = tgec->regs;
 	u32 tmp;
@@ -428,34 +402,25 @@ int tgec_enable(struct fman_mac *tgec, enum comm_mode mode)
 		return -EINVAL;
 
 	tmp = ioread32be(&regs->command_config);
-	if (mode & COMM_MODE_RX)
-		tmp |= CMD_CFG_RX_EN;
-	if (mode & COMM_MODE_TX)
-		tmp |= CMD_CFG_TX_EN;
+	tmp |= CMD_CFG_RX_EN | CMD_CFG_TX_EN;
 	iowrite32be(tmp, &regs->command_config);
 
 	return 0;
 }
 
-int tgec_disable(struct fman_mac *tgec, enum comm_mode mode)
+static void tgec_disable(struct fman_mac *tgec)
 {
 	struct tgec_regs __iomem *regs = tgec->regs;
 	u32 tmp;
 
-	if (!is_init_done(tgec->cfg))
-		return -EINVAL;
+	WARN_ON_ONCE(!is_init_done(tgec->cfg));
 
 	tmp = ioread32be(&regs->command_config);
-	if (mode & COMM_MODE_RX)
-		tmp &= ~CMD_CFG_RX_EN;
-	if (mode & COMM_MODE_TX)
-		tmp &= ~CMD_CFG_TX_EN;
+	tmp &= ~(CMD_CFG_RX_EN | CMD_CFG_TX_EN);
 	iowrite32be(tmp, &regs->command_config);
-
-	return 0;
 }
 
-int tgec_set_promiscuous(struct fman_mac *tgec, bool new_val)
+static int tgec_set_promiscuous(struct fman_mac *tgec, bool new_val)
 {
 	struct tgec_regs __iomem *regs = tgec->regs;
 	u32 tmp;
@@ -473,18 +438,9 @@ int tgec_set_promiscuous(struct fman_mac *tgec, bool new_val)
 	return 0;
 }
 
-int tgec_cfg_max_frame_len(struct fman_mac *tgec, u16 new_val)
-{
-	if (is_init_done(tgec->cfg))
-		return -EINVAL;
-
-	tgec->cfg->max_frame_length = new_val;
-
-	return 0;
-}
-
-int tgec_set_tx_pause_frames(struct fman_mac *tgec, u8 __maybe_unused priority,
-			     u16 pause_time, u16 __maybe_unused thresh_time)
+static int tgec_set_tx_pause_frames(struct fman_mac *tgec,
+				    u8 __maybe_unused priority, u16 pause_time,
+				    u16 __maybe_unused thresh_time)
 {
 	struct tgec_regs __iomem *regs = tgec->regs;
 
@@ -496,7 +452,7 @@ int tgec_set_tx_pause_frames(struct fman_mac *tgec, u8 __maybe_unused priority,
 	return 0;
 }
 
-int tgec_accept_rx_pause_frames(struct fman_mac *tgec, bool en)
+static int tgec_accept_rx_pause_frames(struct fman_mac *tgec, bool en)
 {
 	struct tgec_regs __iomem *regs = tgec->regs;
 	u32 tmp;
@@ -514,7 +470,8 @@ int tgec_accept_rx_pause_frames(struct fman_mac *tgec, bool en)
 	return 0;
 }
 
-int tgec_modify_mac_address(struct fman_mac *tgec, const enet_addr_t *p_enet_addr)
+static int tgec_modify_mac_address(struct fman_mac *tgec,
+				   const enet_addr_t *p_enet_addr)
 {
 	if (!is_init_done(tgec->cfg))
 		return -EINVAL;
@@ -525,7 +482,8 @@ int tgec_modify_mac_address(struct fman_mac *tgec, const enet_addr_t *p_enet_add
 	return 0;
 }
 
-int tgec_add_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr)
+static int tgec_add_hash_mac_address(struct fman_mac *tgec,
+				     enet_addr_t *eth_addr)
 {
 	struct tgec_regs __iomem *regs = tgec->regs;
 	struct eth_hash_entry *hash_entry;
@@ -562,7 +520,7 @@ int tgec_add_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr)
 	return 0;
 }
 
-int tgec_set_allmulti(struct fman_mac *tgec, bool enable)
+static int tgec_set_allmulti(struct fman_mac *tgec, bool enable)
 {
 	u32 entry;
 	struct tgec_regs __iomem *regs = tgec->regs;
@@ -585,7 +543,7 @@ int tgec_set_allmulti(struct fman_mac *tgec, bool enable)
 	return 0;
 }
 
-int tgec_set_tstamp(struct fman_mac *tgec, bool enable)
+static int tgec_set_tstamp(struct fman_mac *tgec, bool enable)
 {
 	struct tgec_regs __iomem *regs = tgec->regs;
 	u32 tmp;
@@ -605,7 +563,8 @@ int tgec_set_tstamp(struct fman_mac *tgec, bool enable)
 	return 0;
 }
 
-int tgec_del_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr)
+static int tgec_del_hash_mac_address(struct fman_mac *tgec,
+				     enet_addr_t *eth_addr)
 {
 	struct tgec_regs __iomem *regs = tgec->regs;
 	struct eth_hash_entry *hash_entry = NULL;
@@ -642,20 +601,15 @@ int tgec_del_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr)
 	return 0;
 }
 
-int tgec_get_version(struct fman_mac *tgec, u32 *mac_version)
+static void tgec_adjust_link(struct mac_device *mac_dev)
 {
-	struct tgec_regs __iomem *regs = tgec->regs;
-
-	if (!is_init_done(tgec->cfg))
-		return -EINVAL;
+	struct phy_device *phy_dev = mac_dev->phy_dev;
 
-	*mac_version = ioread32be(&regs->tgec_id);
-
-	return 0;
+	mac_dev->update_speed(mac_dev, phy_dev->speed);
 }
 
-int tgec_set_exception(struct fman_mac *tgec,
-		       enum fman_mac_exceptions exception, bool enable)
+static int tgec_set_exception(struct fman_mac *tgec,
+			      enum fman_mac_exceptions exception, bool enable)
 {
 	struct tgec_regs __iomem *regs = tgec->regs;
 	u32 bit_mask = 0;
@@ -681,7 +635,7 @@ int tgec_set_exception(struct fman_mac *tgec,
 	return 0;
 }
 
-int tgec_init(struct fman_mac *tgec)
+static int tgec_init(struct fman_mac *tgec)
 {
 	struct tgec_cfg *cfg;
 	enet_addr_t eth_addr;
@@ -764,7 +718,7 @@ int tgec_init(struct fman_mac *tgec)
 	return 0;
 }
 
-int tgec_free(struct fman_mac *tgec)
+static int tgec_free(struct fman_mac *tgec)
 {
 	free_init_resources(tgec);
 
@@ -774,13 +728,12 @@ int tgec_free(struct fman_mac *tgec)
 	return 0;
 }
 
-struct fman_mac *tgec_config(struct fman_mac_params *params)
+static struct fman_mac *tgec_config(struct mac_device *mac_dev,
+				    struct fman_mac_params *params)
 {
 	struct fman_mac *tgec;
 	struct tgec_cfg *cfg;
-	void __iomem *base_addr;
 
-	base_addr = params->base_addr;
 	/* allocate memory for the UCC GETH data structure. */
 	tgec = kzalloc(sizeof(*tgec), GFP_KERNEL);
 	if (!tgec)
@@ -798,8 +751,8 @@ struct fman_mac *tgec_config(struct fman_mac_params *params)
 
 	set_dflts(cfg);
 
-	tgec->regs = base_addr;
-	tgec->addr = ENET_ADDR_TO_UINT64(params->addr);
+	tgec->regs = mac_dev->vaddr;
+	tgec->addr = ENET_ADDR_TO_UINT64(mac_dev->addr);
 	tgec->max_speed = params->max_speed;
 	tgec->mac_id = params->mac_id;
 	tgec->exceptions = (TGEC_IMASK_MDIO_SCAN_EVENT	|
@@ -819,7 +772,7 @@ struct fman_mac *tgec_config(struct fman_mac_params *params)
 			    TGEC_IMASK_RX_ALIGN_ER);
 	tgec->exception_cb = params->exception_cb;
 	tgec->event_cb = params->event_cb;
-	tgec->dev_id = params->dev_id;
+	tgec->dev_id = mac_dev;
 	tgec->fm = params->fm;
 
 	/* Save FMan revision */
@@ -827,3 +780,52 @@ struct fman_mac *tgec_config(struct fman_mac_params *params)
 
 	return tgec;
 }
+
+int tgec_initialization(struct mac_device *mac_dev,
+			struct device_node *mac_node,
+			struct fman_mac_params *params)
+{
+	int err;
+	struct fman_mac		*tgec;
+
+	mac_dev->set_promisc		= tgec_set_promiscuous;
+	mac_dev->change_addr		= tgec_modify_mac_address;
+	mac_dev->add_hash_mac_addr	= tgec_add_hash_mac_address;
+	mac_dev->remove_hash_mac_addr	= tgec_del_hash_mac_address;
+	mac_dev->set_tx_pause		= tgec_set_tx_pause_frames;
+	mac_dev->set_rx_pause		= tgec_accept_rx_pause_frames;
+	mac_dev->set_exception		= tgec_set_exception;
+	mac_dev->set_allmulti		= tgec_set_allmulti;
+	mac_dev->set_tstamp		= tgec_set_tstamp;
+	mac_dev->set_multi		= fman_set_multi;
+	mac_dev->adjust_link            = tgec_adjust_link;
+	mac_dev->enable			= tgec_enable;
+	mac_dev->disable		= tgec_disable;
+
+	mac_dev->fman_mac = tgec_config(mac_dev, params);
+	if (!mac_dev->fman_mac) {
+		err = -EINVAL;
+		goto _return;
+	}
+
+	tgec = mac_dev->fman_mac;
+	tgec->cfg->max_frame_length = fman_get_max_frm();
+	err = tgec_init(tgec);
+	if (err < 0)
+		goto _return_fm_mac_free;
+
+	/* For 10G MAC, disable Tx ECC exception */
+	err = tgec_set_exception(tgec, FM_MAC_EX_10G_TX_ECC_ER, false);
+	if (err < 0)
+		goto _return_fm_mac_free;
+
+	pr_info("FMan XGEC version: 0x%08x\n",
+		ioread32be(&tgec->regs->tgec_id));
+	goto _return;
+
+_return_fm_mac_free:
+	tgec_free(mac_dev->fman_mac);
+
+_return:
+	return err;
+}
diff --git a/drivers/net/ethernet/freescale/fman/fman_tgec.h b/drivers/net/ethernet/freescale/fman/fman_tgec.h
index b28b20b26148..768b8d165e05 100644
--- a/drivers/net/ethernet/freescale/fman/fman_tgec.h
+++ b/drivers/net/ethernet/freescale/fman/fman_tgec.h
@@ -1,33 +1,6 @@
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */
 /*
- * Copyright 2008-2015 Freescale Semiconductor Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *       notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *       notice, this list of conditions and the following disclaimer in the
- *       documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *       names of its contributors may be used to endorse or promote products
- *       derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  */
 
 #ifndef __TGEC_H
@@ -35,23 +8,10 @@
 
 #include "fman_mac.h"
 
-struct fman_mac *tgec_config(struct fman_mac_params *params);
-int tgec_set_promiscuous(struct fman_mac *tgec, bool new_val);
-int tgec_modify_mac_address(struct fman_mac *tgec, const enet_addr_t *enet_addr);
-int tgec_cfg_max_frame_len(struct fman_mac *tgec, u16 new_val);
-int tgec_enable(struct fman_mac *tgec, enum comm_mode mode);
-int tgec_disable(struct fman_mac *tgec, enum comm_mode mode);
-int tgec_init(struct fman_mac *tgec);
-int tgec_free(struct fman_mac *tgec);
-int tgec_accept_rx_pause_frames(struct fman_mac *tgec, bool en);
-int tgec_set_tx_pause_frames(struct fman_mac *tgec, u8 priority,
-			     u16 pause_time, u16 thresh_time);
-int tgec_set_exception(struct fman_mac *tgec,
-		       enum fman_mac_exceptions exception, bool enable);
-int tgec_add_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr);
-int tgec_del_hash_mac_address(struct fman_mac *tgec, enet_addr_t *eth_addr);
-int tgec_get_version(struct fman_mac *tgec, u32 *mac_version);
-int tgec_set_allmulti(struct fman_mac *tgec, bool enable);
-int tgec_set_tstamp(struct fman_mac *tgec, bool enable);
+struct mac_device;
+
+int tgec_initialization(struct mac_device *mac_dev,
+			struct device_node *mac_node,
+			struct fman_mac_params *params);
 
 #endif /* __TGEC_H */
diff --git a/drivers/net/ethernet/freescale/fman/mac.c b/drivers/net/ethernet/freescale/fman/mac.c
index 39ae965cd4f6..7b7526fd7da3 100644
--- a/drivers/net/ethernet/freescale/fman/mac.c
+++ b/drivers/net/ethernet/freescale/fman/mac.c
@@ -1,32 +1,6 @@
-/* Copyright 2008-2015 Freescale Semiconductor, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *	 notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *	 notice, this list of conditions and the following disclaimer in the
- *	 documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *	 names of its contributors may be used to endorse or promote products
- *	 derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
+/*
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -54,20 +28,12 @@ MODULE_LICENSE("Dual BSD/GPL");
 MODULE_DESCRIPTION("FSL FMan MAC API based driver");
 
 struct mac_priv_s {
-	struct device			*dev;
-	void __iomem			*vaddr;
 	u8				cell_index;
 	struct fman			*fman;
-	struct device_node		*internal_phy_node;
 	/* List of multicast addresses */
 	struct list_head		mc_addr_list;
 	struct platform_device		*eth_dev;
-	struct fixed_phy_status		*fixed_link;
 	u16				speed;
-	u16				max_speed;
-
-	int (*enable)(struct fman_mac *mac_dev, enum comm_mode mode);
-	int (*disable)(struct fman_mac *mac_dev, enum comm_mode mode);
 };
 
 struct mac_address {
@@ -75,222 +41,21 @@ struct mac_address {
 	struct list_head list;
 };
 
-static void mac_exception(void *handle, enum fman_mac_exceptions ex)
+static void mac_exception(struct mac_device *mac_dev,
+			  enum fman_mac_exceptions ex)
 {
-	struct mac_device	*mac_dev;
-	struct mac_priv_s	*priv;
-
-	mac_dev = handle;
-	priv = mac_dev->priv;
-
 	if (ex == FM_MAC_EX_10G_RX_FIFO_OVFL) {
 		/* don't flag RX FIFO after the first */
 		mac_dev->set_exception(mac_dev->fman_mac,
 				       FM_MAC_EX_10G_RX_FIFO_OVFL, false);
-		dev_err(priv->dev, "10G MAC got RX FIFO Error = %x\n", ex);
+		dev_err(mac_dev->dev, "10G MAC got RX FIFO Error = %x\n", ex);
 	}
 
-	dev_dbg(priv->dev, "%s:%s() -> %d\n", KBUILD_BASENAME ".c",
+	dev_dbg(mac_dev->dev, "%s:%s() -> %d\n", KBUILD_BASENAME ".c",
 		__func__, ex);
 }
 
-static int set_fman_mac_params(struct mac_device *mac_dev,
-			       struct fman_mac_params *params)
-{
-	struct mac_priv_s *priv = mac_dev->priv;
-
-	params->base_addr = (typeof(params->base_addr))
-		devm_ioremap(priv->dev, mac_dev->res->start,
-			     resource_size(mac_dev->res));
-	if (!params->base_addr)
-		return -ENOMEM;
-
-	memcpy(&params->addr, mac_dev->addr, sizeof(mac_dev->addr));
-	params->max_speed	= priv->max_speed;
-	params->phy_if		= mac_dev->phy_if;
-	params->basex_if	= false;
-	params->mac_id		= priv->cell_index;
-	params->fm		= (void *)priv->fman;
-	params->exception_cb	= mac_exception;
-	params->event_cb	= mac_exception;
-	params->dev_id		= mac_dev;
-	params->internal_phy_node = priv->internal_phy_node;
-
-	return 0;
-}
-
-static int tgec_initialization(struct mac_device *mac_dev)
-{
-	int err;
-	struct mac_priv_s	*priv;
-	struct fman_mac_params	params;
-	u32			version;
-
-	priv = mac_dev->priv;
-
-	err = set_fman_mac_params(mac_dev, &params);
-	if (err)
-		goto _return;
-
-	mac_dev->fman_mac = tgec_config(&params);
-	if (!mac_dev->fman_mac) {
-		err = -EINVAL;
-		goto _return;
-	}
-
-	err = tgec_cfg_max_frame_len(mac_dev->fman_mac, fman_get_max_frm());
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	err = tgec_init(mac_dev->fman_mac);
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	/* For 10G MAC, disable Tx ECC exception */
-	err = mac_dev->set_exception(mac_dev->fman_mac,
-				     FM_MAC_EX_10G_TX_ECC_ER, false);
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	err = tgec_get_version(mac_dev->fman_mac, &version);
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	dev_info(priv->dev, "FMan XGEC version: 0x%08x\n", version);
-
-	goto _return;
-
-_return_fm_mac_free:
-	tgec_free(mac_dev->fman_mac);
-
-_return:
-	return err;
-}
-
-static int dtsec_initialization(struct mac_device *mac_dev)
-{
-	int			err;
-	struct mac_priv_s	*priv;
-	struct fman_mac_params	params;
-	u32			version;
-
-	priv = mac_dev->priv;
-
-	err = set_fman_mac_params(mac_dev, &params);
-	if (err)
-		goto _return;
-
-	mac_dev->fman_mac = dtsec_config(&params);
-	if (!mac_dev->fman_mac) {
-		err = -EINVAL;
-		goto _return;
-	}
-
-	err = dtsec_cfg_max_frame_len(mac_dev->fman_mac, fman_get_max_frm());
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	err = dtsec_cfg_pad_and_crc(mac_dev->fman_mac, true);
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	err = dtsec_init(mac_dev->fman_mac);
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	/* For 1G MAC, disable by default the MIB counters overflow interrupt */
-	err = mac_dev->set_exception(mac_dev->fman_mac,
-				     FM_MAC_EX_1G_RX_MIB_CNT_OVFL, false);
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	err = dtsec_get_version(mac_dev->fman_mac, &version);
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	dev_info(priv->dev, "FMan dTSEC version: 0x%08x\n", version);
-
-	goto _return;
-
-_return_fm_mac_free:
-	dtsec_free(mac_dev->fman_mac);
-
-_return:
-	return err;
-}
-
-static int memac_initialization(struct mac_device *mac_dev)
-{
-	int			 err;
-	struct mac_priv_s	*priv;
-	struct fman_mac_params	 params;
-
-	priv = mac_dev->priv;
-
-	err = set_fman_mac_params(mac_dev, &params);
-	if (err)
-		goto _return;
-
-	if (priv->max_speed == SPEED_10000)
-		params.phy_if = PHY_INTERFACE_MODE_XGMII;
-
-	mac_dev->fman_mac = memac_config(&params);
-	if (!mac_dev->fman_mac) {
-		err = -EINVAL;
-		goto _return;
-	}
-
-	err = memac_cfg_max_frame_len(mac_dev->fman_mac, fman_get_max_frm());
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	err = memac_cfg_reset_on_init(mac_dev->fman_mac, true);
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	err = memac_cfg_fixed_link(mac_dev->fman_mac, priv->fixed_link);
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	err = memac_init(mac_dev->fman_mac);
-	if (err < 0)
-		goto _return_fm_mac_free;
-
-	dev_info(priv->dev, "FMan MEMAC\n");
-
-	goto _return;
-
-_return_fm_mac_free:
-	memac_free(mac_dev->fman_mac);
-
-_return:
-	return err;
-}
-
-static int start(struct mac_device *mac_dev)
-{
-	int	 err;
-	struct phy_device *phy_dev = mac_dev->phy_dev;
-	struct mac_priv_s *priv = mac_dev->priv;
-
-	err = priv->enable(mac_dev->fman_mac, COMM_MODE_RX_AND_TX);
-	if (!err && phy_dev)
-		phy_start(phy_dev);
-
-	return err;
-}
-
-static int stop(struct mac_device *mac_dev)
-{
-	struct mac_priv_s *priv = mac_dev->priv;
-
-	if (mac_dev->phy_dev)
-		phy_stop(mac_dev->phy_dev);
-
-	return priv->disable(mac_dev->fman_mac, COMM_MODE_RX_AND_TX);
-}
-
-static int set_multi(struct net_device *net_dev, struct mac_device *mac_dev)
+int fman_set_multi(struct net_device *net_dev, struct mac_device *mac_dev)
 {
 	struct mac_priv_s	*priv;
 	struct mac_address	*old_addr, *tmp;
@@ -424,109 +189,6 @@ void fman_get_pause_cfg(struct mac_device *mac_dev, bool *rx_pause,
 }
 EXPORT_SYMBOL(fman_get_pause_cfg);
 
-static void adjust_link_void(struct mac_device *mac_dev)
-{
-}
-
-static void adjust_link_dtsec(struct mac_device *mac_dev)
-{
-	struct phy_device *phy_dev = mac_dev->phy_dev;
-	struct fman_mac *fman_mac;
-	bool rx_pause, tx_pause;
-	int err;
-
-	fman_mac = mac_dev->fman_mac;
-	if (!phy_dev->link) {
-		dtsec_restart_autoneg(fman_mac);
-
-		return;
-	}
-
-	dtsec_adjust_link(fman_mac, phy_dev->speed);
-	fman_get_pause_cfg(mac_dev, &rx_pause, &tx_pause);
-	err = fman_set_mac_active_pause(mac_dev, rx_pause, tx_pause);
-	if (err < 0)
-		dev_err(mac_dev->priv->dev, "fman_set_mac_active_pause() = %d\n",
-			err);
-}
-
-static void adjust_link_memac(struct mac_device *mac_dev)
-{
-	struct phy_device *phy_dev = mac_dev->phy_dev;
-	struct fman_mac *fman_mac;
-	bool rx_pause, tx_pause;
-	int err;
-
-	fman_mac = mac_dev->fman_mac;
-	memac_adjust_link(fman_mac, phy_dev->speed);
-
-	fman_get_pause_cfg(mac_dev, &rx_pause, &tx_pause);
-	err = fman_set_mac_active_pause(mac_dev, rx_pause, tx_pause);
-	if (err < 0)
-		dev_err(mac_dev->priv->dev, "fman_set_mac_active_pause() = %d\n",
-			err);
-}
-
-static void setup_dtsec(struct mac_device *mac_dev)
-{
-	mac_dev->init			= dtsec_initialization;
-	mac_dev->set_promisc		= dtsec_set_promiscuous;
-	mac_dev->change_addr		= dtsec_modify_mac_address;
-	mac_dev->add_hash_mac_addr	= dtsec_add_hash_mac_address;
-	mac_dev->remove_hash_mac_addr	= dtsec_del_hash_mac_address;
-	mac_dev->set_tx_pause		= dtsec_set_tx_pause_frames;
-	mac_dev->set_rx_pause		= dtsec_accept_rx_pause_frames;
-	mac_dev->set_exception		= dtsec_set_exception;
-	mac_dev->set_allmulti		= dtsec_set_allmulti;
-	mac_dev->set_tstamp		= dtsec_set_tstamp;
-	mac_dev->set_multi		= set_multi;
-	mac_dev->start			= start;
-	mac_dev->stop			= stop;
-	mac_dev->adjust_link            = adjust_link_dtsec;
-	mac_dev->priv->enable		= dtsec_enable;
-	mac_dev->priv->disable		= dtsec_disable;
-}
-
-static void setup_tgec(struct mac_device *mac_dev)
-{
-	mac_dev->init			= tgec_initialization;
-	mac_dev->set_promisc		= tgec_set_promiscuous;
-	mac_dev->change_addr		= tgec_modify_mac_address;
-	mac_dev->add_hash_mac_addr	= tgec_add_hash_mac_address;
-	mac_dev->remove_hash_mac_addr	= tgec_del_hash_mac_address;
-	mac_dev->set_tx_pause		= tgec_set_tx_pause_frames;
-	mac_dev->set_rx_pause		= tgec_accept_rx_pause_frames;
-	mac_dev->set_exception		= tgec_set_exception;
-	mac_dev->set_allmulti		= tgec_set_allmulti;
-	mac_dev->set_tstamp		= tgec_set_tstamp;
-	mac_dev->set_multi		= set_multi;
-	mac_dev->start			= start;
-	mac_dev->stop			= stop;
-	mac_dev->adjust_link            = adjust_link_void;
-	mac_dev->priv->enable		= tgec_enable;
-	mac_dev->priv->disable		= tgec_disable;
-}
-
-static void setup_memac(struct mac_device *mac_dev)
-{
-	mac_dev->init			= memac_initialization;
-	mac_dev->set_promisc		= memac_set_promiscuous;
-	mac_dev->change_addr		= memac_modify_mac_address;
-	mac_dev->add_hash_mac_addr	= memac_add_hash_mac_address;
-	mac_dev->remove_hash_mac_addr	= memac_del_hash_mac_address;
-	mac_dev->set_tx_pause		= memac_set_tx_pause_frames;
-	mac_dev->set_rx_pause		= memac_accept_rx_pause_frames;
-	mac_dev->set_exception		= memac_set_exception;
-	mac_dev->set_allmulti		= memac_set_allmulti;
-	mac_dev->set_tstamp		= memac_set_tstamp;
-	mac_dev->set_multi		= set_multi;
-	mac_dev->start			= start;
-	mac_dev->stop			= stop;
-	mac_dev->adjust_link            = adjust_link_memac;
-	mac_dev->priv->enable		= memac_enable;
-	mac_dev->priv->disable		= memac_disable;
-}
-
 #define DTSEC_SUPPORTED \
 	(SUPPORTED_10baseT_Half \
 	| SUPPORTED_10baseT_Full \
@@ -577,7 +239,7 @@ static struct platform_device *dpaa_eth_add_device(int fman_id,
 		goto no_mem;
 	}
 
-	pdev->dev.parent = priv->dev;
+	pdev->dev.parent = mac_dev->dev;
 
 	ret = platform_device_add_data(pdev, &data, sizeof(data));
 	if (ret)
@@ -601,9 +263,9 @@ no_mem:
 }
 
 static const struct of_device_id mac_match[] = {
-	{ .compatible	= "fsl,fman-dtsec" },
-	{ .compatible	= "fsl,fman-xgec" },
-	{ .compatible	= "fsl,fman-memac" },
+	{ .compatible	= "fsl,fman-dtsec", .data = dtsec_initialization },
+	{ .compatible	= "fsl,fman-xgec", .data = tgec_initialization },
+	{ .compatible	= "fsl,fman-memac", .data = memac_initialization },
 	{}
 };
 MODULE_DEVICE_TABLE(of, mac_match);
@@ -611,50 +273,33 @@ MODULE_DEVICE_TABLE(of, mac_match);
 static int mac_probe(struct platform_device *_of_dev)
 {
 	int			 err, i, nph;
+	int (*init)(struct mac_device *mac_dev, struct device_node *mac_node,
+		    struct fman_mac_params *params);
 	struct device		*dev;
 	struct device_node	*mac_node, *dev_node;
 	struct mac_device	*mac_dev;
 	struct platform_device	*of_dev;
-	struct resource		 res;
+	struct resource		*res;
 	struct mac_priv_s	*priv;
+	struct fman_mac_params	 params;
 	u32			 val;
 	u8			fman_id;
 	phy_interface_t          phy_if;
 
 	dev = &_of_dev->dev;
 	mac_node = dev->of_node;
+	init = of_device_get_match_data(dev);
 
 	mac_dev = devm_kzalloc(dev, sizeof(*mac_dev), GFP_KERNEL);
-	if (!mac_dev) {
-		err = -ENOMEM;
-		goto _return;
-	}
+	if (!mac_dev)
+		return -ENOMEM;
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv) {
-		err = -ENOMEM;
-		goto _return;
-	}
+	if (!priv)
+		return -ENOMEM;
 
 	/* Save private information */
 	mac_dev->priv = priv;
-	priv->dev = dev;
-
-	if (of_device_is_compatible(mac_node, "fsl,fman-dtsec")) {
-		setup_dtsec(mac_dev);
-		priv->internal_phy_node = of_parse_phandle(mac_node,
-							  "tbi-handle", 0);
-	} else if (of_device_is_compatible(mac_node, "fsl,fman-xgec")) {
-		setup_tgec(mac_dev);
-	} else if (of_device_is_compatible(mac_node, "fsl,fman-memac")) {
-		setup_memac(mac_dev);
-		priv->internal_phy_node = of_parse_phandle(mac_node,
-							  "pcsphy-handle", 0);
-	} else {
-		dev_err(dev, "MAC node (%pOF) contains unsupported MAC\n",
-			mac_node);
-		err = -EINVAL;
-		goto _return;
-	}
+	mac_dev->dev = dev;
 
 	INIT_LIST_HEAD(&priv->mc_addr_list);
 
@@ -663,8 +308,7 @@ static int mac_probe(struct platform_device *_of_dev)
 	if (!dev_node) {
 		dev_err(dev, "of_get_parent(%pOF) failed\n",
 			mac_node);
-		err = -EINVAL;
-		goto _return_of_get_parent;
+		return -EINVAL;
 	}
 
 	of_dev = of_find_device_by_node(dev_node);
@@ -694,42 +338,33 @@ static int mac_probe(struct platform_device *_of_dev)
 	of_node_put(dev_node);
 
 	/* Get the address of the memory mapped registers */
-	err = of_address_to_resource(mac_node, 0, &res);
-	if (err < 0) {
-		dev_err(dev, "of_address_to_resource(%pOF) = %d\n",
-			mac_node, err);
-		goto _return_of_get_parent;
+	res = platform_get_mem_or_io(_of_dev, 0);
+	if (!res) {
+		dev_err(dev, "could not get registers\n");
+		return -EINVAL;
 	}
 
-	mac_dev->res = __devm_request_region(dev,
-					     fman_get_mem_region(priv->fman),
-					     res.start, resource_size(&res),
-					     "mac");
-	if (!mac_dev->res) {
-		dev_err(dev, "__devm_request_mem_region(mac) failed\n");
-		err = -EBUSY;
-		goto _return_of_get_parent;
+	err = devm_request_resource(dev, fman_get_mem_region(priv->fman), res);
+	if (err) {
+		dev_err_probe(dev, err, "could not request resource\n");
+		return err;
 	}
 
-	priv->vaddr = devm_ioremap(dev, mac_dev->res->start,
-				   resource_size(mac_dev->res));
-	if (!priv->vaddr) {
+	mac_dev->vaddr = devm_ioremap(dev, res->start, resource_size(res));
+	if (!mac_dev->vaddr) {
 		dev_err(dev, "devm_ioremap() failed\n");
-		err = -EIO;
-		goto _return_of_get_parent;
+		return -EIO;
 	}
+	mac_dev->vaddr_end = mac_dev->vaddr + resource_size(res);
 
-	if (!of_device_is_available(mac_node)) {
-		err = -ENODEV;
-		goto _return_of_get_parent;
-	}
+	if (!of_device_is_available(mac_node))
+		return -ENODEV;
 
 	/* Get the cell-index */
 	err = of_property_read_u32(mac_node, "cell-index", &val);
 	if (err) {
 		dev_err(dev, "failed to read cell-index for %pOF\n", mac_node);
-		err = -EINVAL;
-		goto _return_of_get_parent;
+		return -EINVAL;
 	}
 	priv->cell_index = (u8)val;
 
@@ -743,15 +378,13 @@ static int mac_probe(struct platform_device *_of_dev)
 	if (unlikely(nph < 0)) {
 		dev_err(dev, "of_count_phandle_with_args(%pOF, fsl,fman-ports) failed\n",
 			mac_node);
-		err = nph;
-		goto _return_of_get_parent;
+		return nph;
 	}
 
 	if (nph != ARRAY_SIZE(mac_dev->port)) {
 		dev_err(dev, "Not supported number of fman-ports handles of mac node %pOF from device tree\n",
 			mac_node);
-		err = -EINVAL;
-		goto _return_of_get_parent;
+		return -EINVAL;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(mac_dev->port); i++) {
@@ -760,8 +393,7 @@ static int mac_probe(struct platform_device *_of_dev)
 		if (!dev_node) {
 			dev_err(dev, "of_parse_phandle(%pOF, fsl,fman-ports) failed\n",
 				mac_node);
-			err = -EINVAL;
-			goto _return_of_node_put;
+			return -EINVAL;
 		}
 
 		of_dev = of_find_device_by_node(dev_node);
@@ -793,7 +425,7 @@ static int mac_probe(struct platform_device *_of_dev)
 	mac_dev->phy_if = phy_if;
 
 	priv->speed		= phy2speed[mac_dev->phy_if];
-	priv->max_speed		= priv->speed;
+	params.max_speed	= priv->speed;
 	mac_dev->if_support	= DTSEC_SUPPORTED;
 	/* We don't support half-duplex in SGMII mode */
 	if (mac_dev->phy_if == PHY_INTERFACE_MODE_SGMII)
@@ -801,7 +433,7 @@ static int mac_probe(struct platform_device *_of_dev)
 					SUPPORTED_100baseT_Half);
 
 	/* Gigabit support (no half-duplex) */
-	if (priv->max_speed == 1000)
+	if (params.max_speed == 1000)
 		mac_dev->if_support |= SUPPORTED_1000baseT_Full;
 
 	/* The 10G interface only supports one mode */
@@ -810,42 +442,18 @@ static int mac_probe(struct platform_device *_of_dev)
 
 	/* Get the rest of the PHY information */
 	mac_dev->phy_node = of_parse_phandle(mac_node, "phy-handle", 0);
-	if (!mac_dev->phy_node && of_phy_is_fixed_link(mac_node)) {
-		struct phy_device *phy;
-
-		err = of_phy_register_fixed_link(mac_node);
-		if (err)
-			goto _return_of_get_parent;
-
-		priv->fixed_link = kzalloc(sizeof(*priv->fixed_link),
-					   GFP_KERNEL);
-		if (!priv->fixed_link) {
-			err = -ENOMEM;
-			goto _return_of_get_parent;
-		}
-
-		mac_dev->phy_node = of_node_get(mac_node);
-		phy = of_phy_find_device(mac_dev->phy_node);
-		if (!phy) {
-			err = -EINVAL;
-			of_node_put(mac_dev->phy_node);
-			goto _return_of_get_parent;
-		}
 
-		priv->fixed_link->link = phy->link;
-		priv->fixed_link->speed = phy->speed;
-		priv->fixed_link->duplex = phy->duplex;
-		priv->fixed_link->pause = phy->pause;
-		priv->fixed_link->asym_pause = phy->asym_pause;
+	params.basex_if		= false;
+	params.mac_id		= priv->cell_index;
+	params.fm		= (void *)priv->fman;
+	params.exception_cb	= mac_exception;
+	params.event_cb		= mac_exception;
 
-		put_device(&phy->mdio.dev);
-	}
-
-	err = mac_dev->init(mac_dev);
+	err = init(mac_dev, mac_node, &params);
 	if (err < 0) {
 		dev_err(dev, "mac_dev->init() = %d\n", err);
 		of_node_put(mac_dev->phy_node);
-		goto _return_of_get_parent;
+		return err;
 	}
 
 	/* pause frame autonegotiation enabled */
@@ -872,13 +480,10 @@ static int mac_probe(struct platform_device *_of_dev)
 		priv->eth_dev = NULL;
 	}
 
-	goto _return;
+	return err;
 
 _return_of_node_put:
 	of_node_put(dev_node);
-_return_of_get_parent:
-	kfree(priv->fixed_link);
-_return:
 	return err;
 }
 
diff --git a/drivers/net/ethernet/freescale/fman/mac.h b/drivers/net/ethernet/freescale/fman/mac.h
index daa285a9b8b2..b95d384271bd 100644
--- a/drivers/net/ethernet/freescale/fman/mac.h
+++ b/drivers/net/ethernet/freescale/fman/mac.h
@@ -1,32 +1,6 @@
-/* Copyright 2008-2015 Freescale Semiconductor, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *     * Redistributions of source code must retain the above copyright
- *	 notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above copyright
- *	 notice, this list of conditions and the following disclaimer in the
- *	 documentation and/or other materials provided with the distribution.
- *     * Neither the name of Freescale Semiconductor nor the
- *	 names of its contributors may be used to endorse or promote products
- *	 derived from this software without specific prior written permission.
- *
- *
- * ALTERNATIVELY, this software may be distributed under the terms of the
- * GNU General Public License ("GPL") as published by the Free Software
- * Foundation, either version 2 of that License or (at your option) any
- * later version.
- *
- * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later */
+/*
+ * Copyright 2008 - 2015 Freescale Semiconductor Inc.
  */
 
 #ifndef __MAC_H
@@ -45,13 +19,16 @@ struct fman_mac;
 struct mac_priv_s;
 
 struct mac_device {
-	struct resource		*res;
+	void __iomem		*vaddr;
+	void __iomem		*vaddr_end;
+	struct device		*dev;
 	u8			 addr[ETH_ALEN];
 	struct fman_port	*port[2];
 	u32			 if_support;
 	struct phy_device	*phy_dev;
 	phy_interface_t		phy_if;
 	struct device_node	*phy_node;
+	struct net_device	*net_dev;
 
 	bool autoneg_pause;
 	bool rx_pause_req;
@@ -61,9 +38,8 @@ struct mac_device {
 	bool promisc;
 	bool allmulti;
 
-	int (*init)(struct mac_device *mac_dev);
-	int (*start)(struct mac_device *mac_dev);
-	int (*stop)(struct mac_device *mac_dev);
+	int (*enable)(struct fman_mac *mac_dev);
+	void (*disable)(struct fman_mac *mac_dev);
 	void (*adjust_link)(struct mac_device *mac_dev);
 	int (*set_promisc)(struct fman_mac *mac_dev, bool enable);
 	int (*change_addr)(struct fman_mac *mac_dev, const enet_addr_t *enet_addr);
@@ -81,6 +57,8 @@ struct mac_device {
 	int (*remove_hash_mac_addr)(struct fman_mac *mac_dev,
 				    enet_addr_t *eth_addr);
 
+	void (*update_speed)(struct mac_device *mac_dev, int speed);
+
 	struct fman_mac		*fman_mac;
 	struct mac_priv_s	*priv;
 };
@@ -97,5 +75,6 @@ int fman_set_mac_active_pause(struct mac_device *mac_dev, bool rx, bool tx);
 
 void fman_get_pause_cfg(struct mac_device *mac_dev, bool *rx_pause,
 			bool *tx_pause);
+int fman_set_multi(struct net_device *net_dev, struct mac_device *mac_dev);
 
 #endif	/* __MAC_H */
diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
index b3dae17e067e..8844a9a04fcf 100644
--- a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
+++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c
@@ -791,7 +791,7 @@ static int fs_enet_close(struct net_device *dev)
 static void fs_get_drvinfo(struct net_device *dev,
 			    struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
 }
 
 static int fs_get_regs_len(struct net_device *dev)
@@ -883,9 +883,6 @@ static const struct ethtool_ops fs_ethtool_ops = {
 	.set_tunable = fs_set_tunable,
 };
 
-extern int fs_mii_connect(struct net_device *dev);
-extern void fs_mii_disconnect(struct net_device *dev);
-
 /**************************************************************************************/
 
 #ifdef CONFIG_FS_ENET_HAS_FEC
diff --git a/drivers/net/ethernet/freescale/fs_enet/mac-fec.c b/drivers/net/ethernet/freescale/fs_enet/mac-fec.c
index 99fe2c210d0f..61f4b6e50d29 100644
--- a/drivers/net/ethernet/freescale/fs_enet/mac-fec.c
+++ b/drivers/net/ethernet/freescale/fs_enet/mac-fec.c
@@ -98,7 +98,7 @@ static int do_pd_setup(struct fs_enet_private *fep)
 		return -EINVAL;
 
 	fep->fec.fecp = of_iomap(ofdev->dev.of_node, 0);
-	if (!fep->fcc.fccp)
+	if (!fep->fec.fecp)
 		return -EINVAL;
 
 	return 0;
diff --git a/drivers/net/ethernet/freescale/gianfar.c b/drivers/net/ethernet/freescale/gianfar.c
index e7bf1524b68e..b2def295523a 100644
--- a/drivers/net/ethernet/freescale/gianfar.c
+++ b/drivers/net/ethernet/freescale/gianfar.c
@@ -3233,7 +3233,7 @@ static int gfar_probe(struct platform_device *ofdev)
 	/* Register for napi ...We are registering NAPI for each grp */
 	for (i = 0; i < priv->num_grps; i++) {
 		netif_napi_add(dev, &priv->gfargrp[i].napi_rx,
-			       gfar_poll_rx_sq, NAPI_POLL_WEIGHT);
+			       gfar_poll_rx_sq);
 		netif_napi_add_tx_weight(dev, &priv->gfargrp[i].napi_tx,
 					 gfar_poll_tx_sq, 2);
 	}
diff --git a/drivers/net/ethernet/freescale/gianfar_ethtool.c b/drivers/net/ethernet/freescale/gianfar_ethtool.c
index 81fb68730138..b2b0d3c26fcc 100644
--- a/drivers/net/ethernet/freescale/gianfar_ethtool.c
+++ b/drivers/net/ethernet/freescale/gianfar_ethtool.c
@@ -163,7 +163,7 @@ static int gfar_sset_count(struct net_device *dev, int sset)
 static void gfar_gdrvinfo(struct net_device *dev,
 			  struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
 }
 
 /* Return the length of the register structure */
diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c
index 823221c912ab..7a4cb4f07c32 100644
--- a/drivers/net/ethernet/freescale/ucc_geth.c
+++ b/drivers/net/ethernet/freescale/ucc_geth.c
@@ -3712,7 +3712,7 @@ static int ucc_geth_probe(struct platform_device* ofdev)
 	dev->netdev_ops = &ucc_geth_netdev_ops;
 	dev->watchdog_timeo = TX_TIMEOUT;
 	INIT_WORK(&ugeth->timeout_work, ucc_geth_timeout_work);
-	netif_napi_add(dev, &ugeth->napi, ucc_geth_poll, 64);
+	netif_napi_add(dev, &ugeth->napi, ucc_geth_poll);
 	dev->mtu = 1500;
 	dev->max_mtu = 1518;
 
diff --git a/drivers/net/ethernet/freescale/ucc_geth_ethtool.c b/drivers/net/ethernet/freescale/ucc_geth_ethtool.c
index 69b2b98b1525..601beb93d3b3 100644
--- a/drivers/net/ethernet/freescale/ucc_geth_ethtool.c
+++ b/drivers/net/ethernet/freescale/ucc_geth_ethtool.c
@@ -337,8 +337,8 @@ static void
 uec_get_drvinfo(struct net_device *netdev,
                        struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, "QUICC ENGINE", sizeof(drvinfo->bus_info));
+	strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->bus_info, "QUICC ENGINE", sizeof(drvinfo->bus_info));
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/net/ethernet/freescale/xgmac_mdio.c b/drivers/net/ethernet/freescale/xgmac_mdio.c
index ec90da1de030..d7d39a58cd80 100644
--- a/drivers/net/ethernet/freescale/xgmac_mdio.c
+++ b/drivers/net/ethernet/freescale/xgmac_mdio.c
@@ -355,7 +355,7 @@ static int xgmac_mdio_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	fwnode = pdev->dev.fwnode;
+	fwnode = dev_fwnode(&pdev->dev);
 	if (is_of_node(fwnode))
 		ret = of_mdiobus_register(bus, to_of_node(fwnode));
 	else if (is_acpi_node(fwnode))
diff --git a/drivers/net/ethernet/fujitsu/fmvj18x_cs.c b/drivers/net/ethernet/fujitsu/fmvj18x_cs.c
index b0d733e9a7c6..4859493471db 100644
--- a/drivers/net/ethernet/fujitsu/fmvj18x_cs.c
+++ b/drivers/net/ethernet/fujitsu/fmvj18x_cs.c
@@ -1046,8 +1046,8 @@ static void fjn_rx(struct net_device *dev)
 static void netdev_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 	snprintf(info->bus_info, sizeof(info->bus_info),
 		"PCMCIA 0x%lx", dev->base_addr);
 }
diff --git a/drivers/net/ethernet/fungible/funeth/funeth_main.c b/drivers/net/ethernet/fungible/funeth/funeth_main.c
index f247b7ad3a88..095f51c4d9d9 100644
--- a/drivers/net/ethernet/fungible/funeth/funeth_main.c
+++ b/drivers/net/ethernet/fungible/funeth/funeth_main.c
@@ -339,8 +339,7 @@ static int fun_alloc_queue_irqs(struct net_device *dev, unsigned int ntx,
 			return PTR_ERR(irq);
 
 		fp->num_rx_irqs++;
-		netif_napi_add(dev, &irq->napi, fun_rxq_napi_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(dev, &irq->napi, fun_rxq_napi_poll);
 	}
 
 	netif_info(fp, intr, dev, "Reserved %u/%u IRQs for Tx/Rx queues\n",
@@ -1802,16 +1801,14 @@ static int fun_create_netdev(struct fun_ethdev *ed, unsigned int portid)
 	if (rc)
 		goto unreg_devlink;
 
-	if (fp->dl_port.devlink)
-		devlink_port_type_eth_set(&fp->dl_port, netdev);
+	devlink_port_type_eth_set(&fp->dl_port, netdev);
 
 	return 0;
 
 unreg_devlink:
 	ed->netdevs[portid] = NULL;
 	fun_ktls_cleanup(fp);
-	if (fp->dl_port.devlink)
-		devlink_port_unregister(&fp->dl_port);
+	devlink_port_unregister(&fp->dl_port);
 free_stats:
 	fun_free_stats_area(fp);
 free_rss:
@@ -1830,11 +1827,9 @@ static void fun_destroy_netdev(struct net_device *netdev)
 	struct funeth_priv *fp;
 
 	fp = netdev_priv(netdev);
-	if (fp->dl_port.devlink) {
-		devlink_port_type_clear(&fp->dl_port);
-		devlink_port_unregister(&fp->dl_port);
-	}
+	devlink_port_type_clear(&fp->dl_port);
 	unregister_netdev(netdev);
+	devlink_port_unregister(&fp->dl_port);
 	fun_ktls_cleanup(fp);
 	fun_free_stats_area(fp);
 	fun_free_rss(fp);
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 044db3ebb071..d3e3ac242bfc 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -526,8 +526,7 @@ static void gve_add_napi(struct gve_priv *priv, int ntfy_idx,
 {
 	struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
 
-	netif_napi_add(priv->dev, &block->napi, gve_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(priv->dev, &block->napi, gve_poll);
 }
 
 static void gve_remove_napi(struct gve_priv *priv, int ntfy_idx)
diff --git a/drivers/net/ethernet/hisilicon/hip04_eth.c b/drivers/net/ethernet/hisilicon/hip04_eth.c
index c84ef494bd60..50c3f5d6611f 100644
--- a/drivers/net/ethernet/hisilicon/hip04_eth.c
+++ b/drivers/net/ethernet/hisilicon/hip04_eth.c
@@ -830,8 +830,8 @@ static int hip04_set_coalesce(struct net_device *netdev,
 static void hip04_get_drvinfo(struct net_device *netdev,
 			      struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version));
+	strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version));
 }
 
 static const struct ethtool_ops hip04_ethtool_ops = {
@@ -990,7 +990,7 @@ static int hip04_mac_probe(struct platform_device *pdev)
 	ndev->watchdog_timeo = TX_TIMEOUT;
 	ndev->priv_flags |= IFF_UNICAST_FLT;
 	ndev->irq = irq;
-	netif_napi_add(ndev, &priv->napi, hip04_rx_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &priv->napi, hip04_rx_poll);
 
 	hip04_reset_dreq(priv);
 	hip04_reset_ppe(priv);
diff --git a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c
index d7e62eca050f..ffcf797dfa90 100644
--- a/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c
+++ b/drivers/net/ethernet/hisilicon/hix5hd2_gmac.c
@@ -1243,7 +1243,7 @@ static int hix5hd2_dev_probe(struct platform_device *pdev)
 	if (ret)
 		goto out_phy_node;
 
-	netif_napi_add(ndev, &priv->napi, hix5hd2_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &priv->napi, hix5hd2_poll);
 
 	if (HAS_CAP_TSO(priv->hw_cap)) {
 		ret = hix5hd2_init_sg_desc_queue(priv);
diff --git a/drivers/net/ethernet/hisilicon/hns/hns_enet.c b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
index d94cc8c6681f..7cf10d1e2b31 100644
--- a/drivers/net/ethernet/hisilicon/hns/hns_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns/hns_enet.c
@@ -2109,8 +2109,7 @@ static int hns_nic_init_ring_data(struct hns_nic_priv *priv)
 		rd->fini_process = is_ver1 ? hns_nic_tx_fini_pro :
 			hns_nic_tx_fini_pro_v2;
 
-		netif_napi_add(priv->netdev, &rd->napi,
-			       hns_nic_common_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(priv->netdev, &rd->napi, hns_nic_common_poll);
 		rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED;
 	}
 	for (i = h->q_num; i < h->q_num * 2; i++) {
@@ -2122,8 +2121,7 @@ static int hns_nic_init_ring_data(struct hns_nic_priv *priv)
 		rd->fini_process = is_ver1 ? hns_nic_rx_fini_pro :
 			hns_nic_rx_fini_pro_v2;
 
-		netif_napi_add(priv->netdev, &rd->napi,
-			       hns_nic_common_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(priv->netdev, &rd->napi, hns_nic_common_poll);
 		rd->ring->irq_init_flag = RCB_IRQ_NOT_INITED;
 	}
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h
index 7d4ae467f3ad..abcd7877f7d2 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h
@@ -233,6 +233,17 @@ struct hclgevf_mbx_arq_ring {
 	__le16 msg_q[HCLGE_MBX_MAX_ARQ_MSG_NUM][HCLGE_MBX_MAX_ARQ_MSG_SIZE];
 };
 
+struct hclge_dev;
+
+#define HCLGE_MBX_OPCODE_MAX 256
+struct hclge_mbx_ops_param {
+	struct hclge_vport *vport;
+	struct hclge_mbx_vf_to_pf_cmd *req;
+	struct hclge_respond_to_vf_msg *resp_msg;
+};
+
+typedef int (*hclge_mbx_ops_fn)(struct hclge_mbx_ops_param *param);
+
 #define hclge_mbx_ring_ptr_move_crq(crq) \
 	(crq->next_to_use = (crq->next_to_use + 1) % crq->desc_num)
 #define hclge_mbx_tail_ptr_move_arq(arq) \
diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
index 94f80e1c4020..0179fc288f5f 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h
@@ -97,13 +97,15 @@ enum HNAE3_DEV_CAP_BITS {
 	HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B,
 	HNAE3_DEV_SUPPORT_MC_MAC_MNG_B,
 	HNAE3_DEV_SUPPORT_CQ_B,
+	HNAE3_DEV_SUPPORT_FEC_STATS_B,
+	HNAE3_DEV_SUPPORT_LANE_NUM_B,
 };
 
-#define hnae3_dev_fd_supported(hdev) \
-	test_bit(HNAE3_DEV_SUPPORT_FD_B, (hdev)->ae_dev->caps)
+#define hnae3_ae_dev_fd_supported(ae_dev) \
+		test_bit(HNAE3_DEV_SUPPORT_FD_B, (ae_dev)->caps)
 
-#define hnae3_dev_gro_supported(hdev) \
-	test_bit(HNAE3_DEV_SUPPORT_GRO_B, (hdev)->ae_dev->caps)
+#define hnae3_ae_dev_gro_supported(ae_dev) \
+		test_bit(HNAE3_DEV_SUPPORT_GRO_B, (ae_dev)->caps)
 
 #define hnae3_dev_fec_supported(hdev) \
 	test_bit(HNAE3_DEV_SUPPORT_FEC_B, (hdev)->ae_dev->caps)
@@ -159,6 +161,12 @@ enum HNAE3_DEV_CAP_BITS {
 #define hnae3_ae_dev_cq_supported(ae_dev) \
 	test_bit(HNAE3_DEV_SUPPORT_CQ_B, (ae_dev)->caps)
 
+#define hnae3_ae_dev_fec_stats_supported(ae_dev) \
+	test_bit(HNAE3_DEV_SUPPORT_FEC_STATS_B, (ae_dev)->caps)
+
+#define hnae3_ae_dev_lane_num_supported(ae_dev) \
+	test_bit(HNAE3_DEV_SUPPORT_LANE_NUM_B, (ae_dev)->caps)
+
 enum HNAE3_PF_CAP_BITS {
 	HNAE3_PF_SUPPORT_VLAN_FLTR_MDF_B = 0,
 };
@@ -187,6 +195,7 @@ struct hns3_mac_stats {
 
 /* hnae3 loop mode */
 enum hnae3_loop {
+	HNAE3_LOOP_EXTERNAL,
 	HNAE3_LOOP_APP,
 	HNAE3_LOOP_SERIAL_SERDES,
 	HNAE3_LOOP_PARALLEL_SERDES,
@@ -223,6 +232,8 @@ enum hnae3_fec_mode {
 	HNAE3_FEC_AUTO = 0,
 	HNAE3_FEC_BASER,
 	HNAE3_FEC_RS,
+	HNAE3_FEC_LLRS,
+	HNAE3_FEC_NONE,
 	HNAE3_FEC_USER_DEF,
 };
 
@@ -270,6 +281,7 @@ enum hnae3_dbg_cmd {
 	HNAE3_DBG_CMD_TC_SCH_INFO,
 	HNAE3_DBG_CMD_QOS_PAUSE_CFG,
 	HNAE3_DBG_CMD_QOS_PRI_MAP,
+	HNAE3_DBG_CMD_QOS_DSCP_MAP,
 	HNAE3_DBG_CMD_QOS_BUF_CFG,
 	HNAE3_DBG_CMD_DEV_INFO,
 	HNAE3_DBG_CMD_TX_BD,
@@ -308,6 +320,11 @@ enum hnae3_dbg_cmd {
 	HNAE3_DBG_CMD_UNKNOWN,
 };
 
+enum hnae3_tc_map_mode {
+	HNAE3_TC_MAP_MODE_PRIO,
+	HNAE3_TC_MAP_MODE_DSCP,
+};
+
 struct hnae3_vector_info {
 	u8 __iomem *io_addr;
 	int vector;
@@ -560,14 +577,17 @@ struct hnae3_ae_ops {
 	void (*client_stop)(struct hnae3_handle *handle);
 	int (*get_status)(struct hnae3_handle *handle);
 	void (*get_ksettings_an_result)(struct hnae3_handle *handle,
-					u8 *auto_neg, u32 *speed, u8 *duplex);
+					u8 *auto_neg, u32 *speed, u8 *duplex,
+					u32 *lane_num);
 
 	int (*cfg_mac_speed_dup_h)(struct hnae3_handle *handle, int speed,
-				   u8 duplex);
+				   u8 duplex, u8 lane_num);
 
 	void (*get_media_type)(struct hnae3_handle *handle, u8 *media_type,
 			       u8 *module_type);
 	int (*check_port_speed)(struct hnae3_handle *handle, u32 speed);
+	void (*get_fec_stats)(struct hnae3_handle *handle,
+			      struct ethtool_fec_stats *fec_stats);
 	void (*get_fec)(struct hnae3_handle *handle, u8 *fec_ability,
 			u8 *fec_mode);
 	int (*set_fec)(struct hnae3_handle *handle, u32 fec_mode);
@@ -737,6 +757,8 @@ struct hnae3_ae_ops {
 	int (*get_link_diagnosis_info)(struct hnae3_handle *handle,
 				       u32 *status_code);
 	void (*clean_vf_config)(struct hnae3_ae_dev *ae_dev, int num_vfs);
+	int (*get_dscp_prio)(struct hnae3_handle *handle, u8 dscp,
+			     u8 *tc_map_mode, u8 *priority);
 };
 
 struct hnae3_dcb_ops {
@@ -745,6 +767,8 @@ struct hnae3_dcb_ops {
 	int (*ieee_setets)(struct hnae3_handle *, struct ieee_ets *);
 	int (*ieee_getpfc)(struct hnae3_handle *, struct ieee_pfc *);
 	int (*ieee_setpfc)(struct hnae3_handle *, struct ieee_pfc *);
+	int (*ieee_setapp)(struct hnae3_handle *h, struct dcb_app *app);
+	int (*ieee_delapp)(struct hnae3_handle *h, struct dcb_app *app);
 
 	/* DCBX configuration */
 	u8   (*getdcbx)(struct hnae3_handle *);
@@ -774,6 +798,8 @@ struct hnae3_tc_info {
 	bool mqprio_active;
 };
 
+#define HNAE3_MAX_DSCP			64
+#define HNAE3_PRIO_ID_INVALID		0xff
 struct hnae3_knic_private_info {
 	struct net_device *netdev; /* Set by KNIC client when init instance */
 	u16 rss_size;		   /* Allocated RSS queues */
@@ -784,6 +810,9 @@ struct hnae3_knic_private_info {
 	u32 tx_spare_buf_size;
 
 	struct hnae3_tc_info tc_info;
+	u8 tc_map_mode;
+	u8 dscp_app_cnt;
+	u8 dscp_prio[HNAE3_MAX_DSCP];
 
 	u16 num_tqps;		  /* total number of TQPs in this handle */
 	struct hnae3_queue **tqp;  /* array base of all TQPs in this instance */
@@ -815,6 +844,7 @@ struct hnae3_roce_private_info {
 #define HNAE3_SUPPORT_SERDES_SERIAL_LOOPBACK	BIT(2)
 #define HNAE3_SUPPORT_VF	      BIT(3)
 #define HNAE3_SUPPORT_SERDES_PARALLEL_LOOPBACK	BIT(4)
+#define HNAE3_SUPPORT_EXTERNAL_LOOPBACK	BIT(5)
 
 #define HNAE3_USER_UPE		BIT(0)	/* unicast promisc enabled by user */
 #define HNAE3_USER_MPE		BIT(1)	/* mulitcast promisc enabled by user */
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c
index c8b151d29f53..f671a63cecde 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.c
@@ -52,9 +52,9 @@ void hclge_comm_cmd_reuse_desc(struct hclge_desc *desc, bool is_read)
 static void hclge_comm_set_default_capability(struct hnae3_ae_dev *ae_dev,
 					      bool is_pf)
 {
-	set_bit(HNAE3_DEV_SUPPORT_FD_B, ae_dev->caps);
 	set_bit(HNAE3_DEV_SUPPORT_GRO_B, ae_dev->caps);
-	if (is_pf && ae_dev->dev_version == HNAE3_DEVICE_VERSION_V2) {
+	if (is_pf) {
+		set_bit(HNAE3_DEV_SUPPORT_FD_B, ae_dev->caps);
 		set_bit(HNAE3_DEV_SUPPORT_FEC_B, ae_dev->caps);
 		set_bit(HNAE3_DEV_SUPPORT_PAUSE_B, ae_dev->caps);
 	}
@@ -91,6 +91,7 @@ int hclge_comm_firmware_compat_config(struct hnae3_ae_dev *ae_dev,
 			hnae3_set_bit(compat, HCLGE_COMM_PHY_IMP_EN_B, 1);
 		hnae3_set_bit(compat, HCLGE_COMM_MAC_STATS_EXT_EN_B, 1);
 		hnae3_set_bit(compat, HCLGE_COMM_SYNC_RX_RING_HEAD_EN_B, 1);
+		hnae3_set_bit(compat, HCLGE_COMM_LLRS_FEC_EN_B, 1);
 
 		req->compat = cpu_to_le32(compat);
 	}
@@ -150,6 +151,10 @@ static const struct hclge_comm_caps_bit_map hclge_pf_cmd_caps[] = {
 	 HNAE3_DEV_SUPPORT_PORT_VLAN_BYPASS_B},
 	{HCLGE_COMM_CAP_PORT_VLAN_BYPASS_B, HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B},
 	{HCLGE_COMM_CAP_CQ_B, HNAE3_DEV_SUPPORT_CQ_B},
+	{HCLGE_COMM_CAP_GRO_B, HNAE3_DEV_SUPPORT_GRO_B},
+	{HCLGE_COMM_CAP_FD_B, HNAE3_DEV_SUPPORT_FD_B},
+	{HCLGE_COMM_CAP_FEC_STATS_B, HNAE3_DEV_SUPPORT_FEC_STATS_B},
+	{HCLGE_COMM_CAP_LANE_NUM_B, HNAE3_DEV_SUPPORT_LANE_NUM_B},
 };
 
 static const struct hclge_comm_caps_bit_map hclge_vf_cmd_caps[] = {
@@ -162,6 +167,7 @@ static const struct hclge_comm_caps_bit_map hclge_vf_cmd_caps[] = {
 	{HCLGE_COMM_CAP_TX_PUSH_B, HNAE3_DEV_SUPPORT_TX_PUSH_B},
 	{HCLGE_COMM_CAP_RXD_ADV_LAYOUT_B, HNAE3_DEV_SUPPORT_RXD_ADV_LAYOUT_B},
 	{HCLGE_COMM_CAP_CQ_B, HNAE3_DEV_SUPPORT_CQ_B},
+	{HCLGE_COMM_CAP_GRO_B, HNAE3_DEV_SUPPORT_GRO_B},
 };
 
 static void
@@ -220,8 +226,10 @@ int hclge_comm_cmd_query_version_and_capability(struct hnae3_ae_dev *ae_dev,
 					 HNAE3_PCI_REVISION_BIT_SIZE;
 	ae_dev->dev_version |= ae_dev->pdev->revision;
 
-	if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2)
+	if (ae_dev->dev_version == HNAE3_DEVICE_VERSION_V2) {
 		hclge_comm_set_default_capability(ae_dev, is_pf);
+		return 0;
+	}
 
 	hclge_comm_parse_capability(ae_dev, is_pf, resp);
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h
index 7a7d4cf9bf35..b1f9383b418f 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_common/hclge_comm_cmd.h
@@ -20,6 +20,7 @@
 #define HCLGE_COMM_PHY_IMP_EN_B			2
 #define HCLGE_COMM_MAC_STATS_EXT_EN_B		3
 #define HCLGE_COMM_SYNC_RX_RING_HEAD_EN_B	4
+#define HCLGE_COMM_LLRS_FEC_EN_B		5
 
 #define hclge_comm_dev_phy_imp_supported(ae_dev) \
 	test_bit(HNAE3_DEV_SUPPORT_PHY_IMP_B, (ae_dev)->caps)
@@ -102,6 +103,7 @@ enum hclge_opcode_type {
 	HCLGE_OPC_MAC_TNL_INT_EN	= 0x0311,
 	HCLGE_OPC_CLEAR_MAC_TNL_INT	= 0x0312,
 	HCLGE_OPC_COMMON_LOOPBACK       = 0x0315,
+	HCLGE_OPC_QUERY_FEC_STATS	= 0x0316,
 	HCLGE_OPC_CONFIG_FEC_MODE	= 0x031A,
 	HCLGE_OPC_QUERY_ROH_TYPE_INFO	= 0x0389,
 
@@ -339,6 +341,10 @@ enum HCLGE_COMM_CAP_BITS {
 	HCLGE_COMM_CAP_RXD_ADV_LAYOUT_B = 15,
 	HCLGE_COMM_CAP_PORT_VLAN_BYPASS_B = 17,
 	HCLGE_COMM_CAP_CQ_B = 18,
+	HCLGE_COMM_CAP_GRO_B = 20,
+	HCLGE_COMM_CAP_FD_B = 21,
+	HCLGE_COMM_CAP_FEC_STATS_B = 25,
+	HCLGE_COMM_CAP_LANE_NUM_B = 27,
 };
 
 enum HCLGE_COMM_API_CAP_BITS {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c b/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c
index d2ec4c573bf8..3b6dbf158b98 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_dcbnl.c
@@ -56,6 +56,32 @@ static int hns3_dcbnl_ieee_setpfc(struct net_device *ndev, struct ieee_pfc *pfc)
 	return -EOPNOTSUPP;
 }
 
+static int hns3_dcbnl_ieee_setapp(struct net_device *ndev, struct dcb_app *app)
+{
+	struct hnae3_handle *h = hns3_get_handle(ndev);
+
+	if (hns3_nic_resetting(ndev))
+		return -EBUSY;
+
+	if (h->kinfo.dcb_ops->ieee_setapp)
+		return h->kinfo.dcb_ops->ieee_setapp(h, app);
+
+	return -EOPNOTSUPP;
+}
+
+static int hns3_dcbnl_ieee_delapp(struct net_device *ndev, struct dcb_app *app)
+{
+	struct hnae3_handle *h = hns3_get_handle(ndev);
+
+	if (hns3_nic_resetting(ndev))
+		return -EBUSY;
+
+	if (h->kinfo.dcb_ops->ieee_setapp)
+		return h->kinfo.dcb_ops->ieee_delapp(h, app);
+
+	return -EOPNOTSUPP;
+}
+
 /* DCBX configuration */
 static u8 hns3_dcbnl_getdcbx(struct net_device *ndev)
 {
@@ -83,6 +109,8 @@ static const struct dcbnl_rtnl_ops hns3_dcbnl_ops = {
 	.ieee_setets	= hns3_dcbnl_ieee_setets,
 	.ieee_getpfc	= hns3_dcbnl_ieee_getpfc,
 	.ieee_setpfc	= hns3_dcbnl_ieee_setpfc,
+	.ieee_setapp    = hns3_dcbnl_ieee_setapp,
+	.ieee_delapp    = hns3_dcbnl_ieee_delapp,
 	.getdcbx	= hns3_dcbnl_getdcbx,
 	.setdcbx	= hns3_dcbnl_setdcbx,
 };
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c
index 93aeb615191d..66feb23f7b7b 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c
@@ -106,6 +106,13 @@ static struct hns3_dbg_cmd_info hns3_dbg_cmd[] = {
 		.init = hns3_dbg_common_file_init,
 	},
 	{
+		.name = "qos_dscp_map",
+		.cmd = HNAE3_DBG_CMD_QOS_DSCP_MAP,
+		.dentry = HNS3_DBG_DENTRY_TM,
+		.buf_len = HNS3_DBG_READ_LEN,
+		.init = hns3_dbg_common_file_init,
+	},
+	{
 		.name = "qos_buf_cfg",
 		.cmd = HNAE3_DBG_CMD_QOS_BUF_CFG,
 		.dentry = HNS3_DBG_DENTRY_TM,
@@ -395,6 +402,12 @@ static struct hns3_dbg_cap_info hns3_dbg_cap[] = {
 	}, {
 		.name = "support modify vlan filter state",
 		.cap_bit = HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B,
+	}, {
+		.name = "support FEC statistics",
+		.cap_bit = HNAE3_DEV_SUPPORT_FEC_STATS_B,
+	}, {
+		.name = "support lane num",
+		.cap_bit = HNAE3_DEV_SUPPORT_LANE_NUM_B,
 	}
 };
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
index 35d70041b9e8..4cb2421e71a7 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
@@ -2963,6 +2963,48 @@ static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)
 	return h->ae_algo->ops->set_vf_mac(h, vf_id, mac);
 }
 
+#define HNS3_INVALID_DSCP		0xff
+#define HNS3_DSCP_SHIFT			2
+
+static u8 hns3_get_skb_dscp(struct sk_buff *skb)
+{
+	__be16 protocol = skb->protocol;
+	u8 dscp = HNS3_INVALID_DSCP;
+
+	if (protocol == htons(ETH_P_8021Q))
+		protocol = vlan_get_protocol(skb);
+
+	if (protocol == htons(ETH_P_IP))
+		dscp = ipv4_get_dsfield(ip_hdr(skb)) >> HNS3_DSCP_SHIFT;
+	else if (protocol == htons(ETH_P_IPV6))
+		dscp = ipv6_get_dsfield(ipv6_hdr(skb)) >> HNS3_DSCP_SHIFT;
+
+	return dscp;
+}
+
+static u16 hns3_nic_select_queue(struct net_device *netdev,
+				 struct sk_buff *skb,
+				 struct net_device *sb_dev)
+{
+	struct hnae3_handle *h = hns3_get_handle(netdev);
+	u8 dscp;
+
+	if (h->kinfo.tc_map_mode != HNAE3_TC_MAP_MODE_DSCP ||
+	    !h->ae_algo->ops->get_dscp_prio)
+		goto out;
+
+	dscp = hns3_get_skb_dscp(skb);
+	if (unlikely(dscp >= HNAE3_MAX_DSCP))
+		goto out;
+
+	skb->priority = h->kinfo.dscp_prio[dscp];
+	if (skb->priority == HNAE3_PRIO_ID_INVALID)
+		skb->priority = 0;
+
+out:
+	return netdev_pick_tx(netdev, skb, sb_dev);
+}
+
 static const struct net_device_ops hns3_nic_netdev_ops = {
 	.ndo_open		= hns3_nic_net_open,
 	.ndo_stop		= hns3_nic_net_stop,
@@ -2988,6 +3030,7 @@ static const struct net_device_ops hns3_nic_netdev_ops = {
 	.ndo_set_vf_link_state	= hns3_nic_set_vf_link_state,
 	.ndo_set_vf_rate	= hns3_nic_set_vf_rate,
 	.ndo_set_vf_mac		= hns3_nic_set_vf_mac,
+	.ndo_select_queue	= hns3_nic_select_queue,
 };
 
 bool hns3_is_phys_func(struct pci_dev *pdev)
@@ -3271,12 +3314,11 @@ static void hns3_set_default_feature(struct net_device *netdev)
 		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |
 		NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;
 
-	if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2) {
+	if (hnae3_ae_dev_gro_supported(ae_dev))
 		netdev->features |= NETIF_F_GRO_HW;
 
-		if (!(h->flags & HNAE3_SUPPORT_VF))
-			netdev->features |= NETIF_F_NTUPLE;
-	}
+	if (hnae3_ae_dev_fd_supported(ae_dev))
+		netdev->features |= NETIF_F_NTUPLE;
 
 	if (test_bit(HNAE3_DEV_SUPPORT_UDP_GSO_B, ae_dev->caps))
 		netdev->features |= NETIF_F_GSO_UDP_L4;
@@ -4650,7 +4692,7 @@ static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)
 			goto map_ring_fail;
 
 		netif_napi_add(priv->netdev, &tqp_vector->napi,
-			       hns3_nic_common_poll, NAPI_POLL_WEIGHT);
+			       hns3_nic_common_poll);
 	}
 
 	return 0;
@@ -5782,6 +5824,57 @@ int hns3_set_channels(struct net_device *netdev,
 	return 0;
 }
 
+void hns3_external_lb_prepare(struct net_device *ndev, bool if_running)
+{
+	struct hns3_nic_priv *priv = netdev_priv(ndev);
+	struct hnae3_handle *h = priv->ae_handle;
+	int i;
+
+	if (!if_running)
+		return;
+
+	netif_carrier_off(ndev);
+	netif_tx_disable(ndev);
+
+	for (i = 0; i < priv->vector_num; i++)
+		hns3_vector_disable(&priv->tqp_vector[i]);
+
+	for (i = 0; i < h->kinfo.num_tqps; i++)
+		hns3_tqp_disable(h->kinfo.tqp[i]);
+
+	/* delay ring buffer clearing to hns3_reset_notify_uninit_enet
+	 * during reset process, because driver may not be able
+	 * to disable the ring through firmware when downing the netdev.
+	 */
+	if (!hns3_nic_resetting(ndev))
+		hns3_nic_reset_all_ring(priv->ae_handle);
+
+	hns3_reset_tx_queue(priv->ae_handle);
+}
+
+void hns3_external_lb_restore(struct net_device *ndev, bool if_running)
+{
+	struct hns3_nic_priv *priv = netdev_priv(ndev);
+	struct hnae3_handle *h = priv->ae_handle;
+	int i;
+
+	if (!if_running)
+		return;
+
+	hns3_nic_reset_all_ring(priv->ae_handle);
+
+	for (i = 0; i < priv->vector_num; i++)
+		hns3_vector_enable(&priv->tqp_vector[i]);
+
+	for (i = 0; i < h->kinfo.num_tqps; i++)
+		hns3_tqp_enable(h->kinfo.tqp[i]);
+
+	netif_tx_wake_all_queues(ndev);
+
+	if (h->ae_algo->ops->get_status(h))
+		netif_carrier_on(ndev);
+}
+
 static const struct hns3_hw_error_info hns3_hw_err[] = {
 	{ .type = HNAE3_PPU_POISON_ERROR,
 	  .msg = "PPU poison" },
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
index 4a3253692dcc..133a054af6b7 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h
@@ -744,4 +744,7 @@ u16 hns3_get_max_available_channels(struct hnae3_handle *h);
 void hns3_cq_period_mode_init(struct hns3_nic_priv *priv,
 			      enum dim_cq_period_mode tx_mode,
 			      enum dim_cq_period_mode rx_mode);
+
+void hns3_external_lb_prepare(struct net_device *ndev, bool if_running);
+void hns3_external_lb_restore(struct net_device *ndev, bool if_running);
 #endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
index 4c7988e308a2..cdf76fb58d45 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c
@@ -69,7 +69,6 @@ static const struct hns3_stats hns3_rxq_stats[] = {
 
 #define HNS3_TQP_STATS_COUNT (HNS3_TXQ_STATS_COUNT + HNS3_RXQ_STATS_COUNT)
 
-#define HNS3_SELF_TEST_TYPE_NUM         4
 #define HNS3_NIC_LB_TEST_PKT_NUM	1
 #define HNS3_NIC_LB_TEST_RING_ID	0
 #define HNS3_NIC_LB_TEST_PACKET_SIZE	128
@@ -95,6 +94,7 @@ static int hns3_lp_setup(struct net_device *ndev, enum hnae3_loop loop, bool en)
 	case HNAE3_LOOP_PARALLEL_SERDES:
 	case HNAE3_LOOP_APP:
 	case HNAE3_LOOP_PHY:
+	case HNAE3_LOOP_EXTERNAL:
 		ret = h->ae_algo->ops->set_loopback(h, loop, en);
 		break;
 	default:
@@ -304,6 +304,10 @@ out:
 
 static void hns3_set_selftest_param(struct hnae3_handle *h, int (*st_param)[2])
 {
+	st_param[HNAE3_LOOP_EXTERNAL][0] = HNAE3_LOOP_EXTERNAL;
+	st_param[HNAE3_LOOP_EXTERNAL][1] =
+			h->flags & HNAE3_SUPPORT_EXTERNAL_LOOPBACK;
+
 	st_param[HNAE3_LOOP_APP][0] = HNAE3_LOOP_APP;
 	st_param[HNAE3_LOOP_APP][1] =
 			h->flags & HNAE3_SUPPORT_APP_LOOPBACK;
@@ -322,17 +326,11 @@ static void hns3_set_selftest_param(struct hnae3_handle *h, int (*st_param)[2])
 			h->flags & HNAE3_SUPPORT_PHY_LOOPBACK;
 }
 
-static void hns3_selftest_prepare(struct net_device *ndev,
-				  bool if_running, int (*st_param)[2])
+static void hns3_selftest_prepare(struct net_device *ndev, bool if_running)
 {
 	struct hns3_nic_priv *priv = netdev_priv(ndev);
 	struct hnae3_handle *h = priv->ae_handle;
 
-	if (netif_msg_ifdown(h))
-		netdev_info(ndev, "self test start\n");
-
-	hns3_set_selftest_param(h, st_param);
-
 	if (if_running)
 		ndev->netdev_ops->ndo_stop(ndev);
 
@@ -371,18 +369,15 @@ static void hns3_selftest_restore(struct net_device *ndev, bool if_running)
 
 	if (if_running)
 		ndev->netdev_ops->ndo_open(ndev);
-
-	if (netif_msg_ifdown(h))
-		netdev_info(ndev, "self test end\n");
 }
 
 static void hns3_do_selftest(struct net_device *ndev, int (*st_param)[2],
 			     struct ethtool_test *eth_test, u64 *data)
 {
-	int test_index = 0;
+	int test_index = HNAE3_LOOP_APP;
 	u32 i;
 
-	for (i = 0; i < HNS3_SELF_TEST_TYPE_NUM; i++) {
+	for (i = HNAE3_LOOP_APP; i < HNAE3_LOOP_NONE; i++) {
 		enum hnae3_loop loop_type = (enum hnae3_loop)st_param[i][0];
 
 		if (!st_param[i][1])
@@ -401,6 +396,20 @@ static void hns3_do_selftest(struct net_device *ndev, int (*st_param)[2],
 	}
 }
 
+static void hns3_do_external_lb(struct net_device *ndev,
+				struct ethtool_test *eth_test, u64 *data)
+{
+	data[HNAE3_LOOP_EXTERNAL] = hns3_lp_up(ndev, HNAE3_LOOP_EXTERNAL);
+	if (!data[HNAE3_LOOP_EXTERNAL])
+		data[HNAE3_LOOP_EXTERNAL] = hns3_lp_run_test(ndev, HNAE3_LOOP_EXTERNAL);
+	hns3_lp_down(ndev, HNAE3_LOOP_EXTERNAL);
+
+	if (data[HNAE3_LOOP_EXTERNAL])
+		eth_test->flags |= ETH_TEST_FL_FAILED;
+
+	eth_test->flags |= ETH_TEST_FL_EXTERNAL_LB_DONE;
+}
+
 /**
  * hns3_self_test - self test
  * @ndev: net device
@@ -410,7 +419,9 @@ static void hns3_do_selftest(struct net_device *ndev, int (*st_param)[2],
 static void hns3_self_test(struct net_device *ndev,
 			   struct ethtool_test *eth_test, u64 *data)
 {
-	int st_param[HNS3_SELF_TEST_TYPE_NUM][2];
+	struct hns3_nic_priv *priv = netdev_priv(ndev);
+	struct hnae3_handle *h = priv->ae_handle;
+	int st_param[HNAE3_LOOP_NONE][2];
 	bool if_running = netif_running(ndev);
 
 	if (hns3_nic_resetting(ndev)) {
@@ -418,13 +429,29 @@ static void hns3_self_test(struct net_device *ndev,
 		return;
 	}
 
-	/* Only do offline selftest, or pass by default */
-	if (eth_test->flags != ETH_TEST_FL_OFFLINE)
+	if (!(eth_test->flags & ETH_TEST_FL_OFFLINE))
 		return;
 
-	hns3_selftest_prepare(ndev, if_running, st_param);
+	if (netif_msg_ifdown(h))
+		netdev_info(ndev, "self test start\n");
+
+	hns3_set_selftest_param(h, st_param);
+
+	/* external loopback test requires that the link is up and the duplex is
+	 * full, do external test first to reduce the whole test time
+	 */
+	if (eth_test->flags & ETH_TEST_FL_EXTERNAL_LB) {
+		hns3_external_lb_prepare(ndev, if_running);
+		hns3_do_external_lb(ndev, eth_test, data);
+		hns3_external_lb_restore(ndev, if_running);
+	}
+
+	hns3_selftest_prepare(ndev, if_running);
 	hns3_do_selftest(ndev, st_param, eth_test, data);
 	hns3_selftest_restore(ndev, if_running);
+
+	if (netif_msg_ifdown(h))
+		netdev_info(ndev, "self test end\n");
 }
 
 static void hns3_update_limit_promisc_mode(struct net_device *netdev,
@@ -712,7 +739,8 @@ static void hns3_get_ksettings(struct hnae3_handle *h,
 		ops->get_ksettings_an_result(h,
 					     &cmd->base.autoneg,
 					     &cmd->base.speed,
-					     &cmd->base.duplex);
+					     &cmd->base.duplex,
+					     &cmd->lanes);
 
 	/* 2.get link mode */
 	if (ops->get_link_mode)
@@ -794,6 +822,7 @@ static int hns3_check_ksettings_param(const struct net_device *netdev,
 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 	u8 module_type = HNAE3_MODULE_TYPE_UNKNOWN;
 	u8 media_type = HNAE3_MEDIA_TYPE_UNKNOWN;
+	u32 lane_num;
 	u8 autoneg;
 	u32 speed;
 	u8 duplex;
@@ -806,9 +835,9 @@ static int hns3_check_ksettings_param(const struct net_device *netdev,
 		return 0;
 
 	if (ops->get_ksettings_an_result) {
-		ops->get_ksettings_an_result(handle, &autoneg, &speed, &duplex);
+		ops->get_ksettings_an_result(handle, &autoneg, &speed, &duplex, &lane_num);
 		if (cmd->base.autoneg == autoneg && cmd->base.speed == speed &&
-		    cmd->base.duplex == duplex)
+		    cmd->base.duplex == duplex && cmd->lanes == lane_num)
 			return 0;
 	}
 
@@ -845,10 +874,14 @@ static int hns3_set_link_ksettings(struct net_device *netdev,
 	if (cmd->base.speed == SPEED_1000 && cmd->base.duplex == DUPLEX_HALF)
 		return -EINVAL;
 
+	if (cmd->lanes && !hnae3_ae_dev_lane_num_supported(ae_dev))
+		return -EOPNOTSUPP;
+
 	netif_dbg(handle, drv, netdev,
-		  "set link(%s): autoneg=%u, speed=%u, duplex=%u\n",
+		  "set link(%s): autoneg=%u, speed=%u, duplex=%u, lanes=%u\n",
 		  netdev->phydev ? "phy" : "mac",
-		  cmd->base.autoneg, cmd->base.speed, cmd->base.duplex);
+		  cmd->base.autoneg, cmd->base.speed, cmd->base.duplex,
+		  cmd->lanes);
 
 	/* Only support ksettings_set for netdev with phy attached for now */
 	if (netdev->phydev) {
@@ -886,7 +919,7 @@ static int hns3_set_link_ksettings(struct net_device *netdev,
 
 	if (ops->cfg_mac_speed_dup_h)
 		ret = ops->cfg_mac_speed_dup_h(handle, cmd->base.speed,
-					       cmd->base.duplex);
+					       cmd->base.duplex, (u8)(cmd->lanes));
 
 	return ret;
 }
@@ -1612,6 +1645,19 @@ static void hns3_set_msglevel(struct net_device *netdev, u32 msg_level)
 	h->msg_enable = msg_level;
 }
 
+static void hns3_get_fec_stats(struct net_device *netdev,
+			       struct ethtool_fec_stats *fec_stats)
+{
+	struct hnae3_handle *handle = hns3_get_handle(netdev);
+	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev);
+	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
+
+	if (!hnae3_ae_dev_fec_stats_supported(ae_dev) || !ops->get_fec_stats)
+		return;
+
+	ops->get_fec_stats(handle, fec_stats);
+}
+
 /* Translate local fec value into ethtool value. */
 static unsigned int loc_to_eth_fec(u8 loc_fec)
 {
@@ -1621,12 +1667,12 @@ static unsigned int loc_to_eth_fec(u8 loc_fec)
 		eth_fec |= ETHTOOL_FEC_AUTO;
 	if (loc_fec & BIT(HNAE3_FEC_RS))
 		eth_fec |= ETHTOOL_FEC_RS;
+	if (loc_fec & BIT(HNAE3_FEC_LLRS))
+		eth_fec |= ETHTOOL_FEC_LLRS;
 	if (loc_fec & BIT(HNAE3_FEC_BASER))
 		eth_fec |= ETHTOOL_FEC_BASER;
-
-	/* if nothing is set, then FEC is off */
-	if (!eth_fec)
-		eth_fec = ETHTOOL_FEC_OFF;
+	if (loc_fec & BIT(HNAE3_FEC_NONE))
+		eth_fec |= ETHTOOL_FEC_OFF;
 
 	return eth_fec;
 }
@@ -1637,12 +1683,13 @@ static unsigned int eth_to_loc_fec(unsigned int eth_fec)
 	u32 loc_fec = 0;
 
 	if (eth_fec & ETHTOOL_FEC_OFF)
-		return loc_fec;
-
+		loc_fec |= BIT(HNAE3_FEC_NONE);
 	if (eth_fec & ETHTOOL_FEC_AUTO)
 		loc_fec |= BIT(HNAE3_FEC_AUTO);
 	if (eth_fec & ETHTOOL_FEC_RS)
 		loc_fec |= BIT(HNAE3_FEC_RS);
+	if (eth_fec & ETHTOOL_FEC_LLRS)
+		loc_fec |= BIT(HNAE3_FEC_LLRS);
 	if (eth_fec & ETHTOOL_FEC_BASER)
 		loc_fec |= BIT(HNAE3_FEC_BASER);
 
@@ -1668,6 +1715,8 @@ static int hns3_get_fecparam(struct net_device *netdev,
 
 	fec->fec = loc_to_eth_fec(fec_ability);
 	fec->active_fec = loc_to_eth_fec(fec_mode);
+	if (!fec->active_fec)
+		fec->active_fec = ETHTOOL_FEC_OFF;
 
 	return 0;
 }
@@ -2051,6 +2100,7 @@ static const struct ethtool_ops hns3vf_ethtool_ops = {
 static const struct ethtool_ops hns3_ethtool_ops = {
 	.supported_coalesce_params = HNS3_ETHTOOL_COALESCE,
 	.supported_ring_params = HNS3_ETHTOOL_RING,
+	.cap_link_lanes_supported = true,
 	.self_test = hns3_self_test,
 	.get_drvinfo = hns3_get_drvinfo,
 	.get_link = hns3_get_link,
@@ -2081,6 +2131,7 @@ static const struct ethtool_ops hns3_ethtool_ops = {
 	.set_msglevel = hns3_set_msglevel,
 	.get_fecparam = hns3_get_fecparam,
 	.set_fecparam = hns3_set_fecparam,
+	.get_fec_stats = hns3_get_fec_stats,
 	.get_module_info = hns3_get_module_info,
 	.get_module_eeprom = hns3_get_module_eeprom,
 	.get_priv_flags = hns3_get_priv_flags,
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
index f9d89511eb32..43cada51d8cb 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_cmd.h
@@ -321,7 +321,9 @@ struct hclge_config_mac_speed_dup_cmd {
 
 #define HCLGE_CFG_MAC_SPEED_CHANGE_EN_B	0
 	u8 mac_change_fec_en;
-	u8 rsv[22];
+	u8 rsv[4];
+	u8 lane_num;
+	u8 rsv1[17];
 };
 
 #define HCLGE_TQP_ENABLE_B		0
@@ -347,7 +349,9 @@ struct hclge_sfp_info_cmd {
 	u8 autoneg_ability; /* whether support autoneg */
 	__le32 speed_ability; /* speed ability for current media */
 	__le32 module_type;
-	u8 rsv[8];
+	u8 fec_ability;
+	u8 lane_num;
+	u8 rsv[6];
 };
 
 #define HCLGE_MAC_CFG_FEC_AUTO_EN_B	0
@@ -359,12 +363,27 @@ struct hclge_sfp_info_cmd {
 #define HCLGE_MAC_FEC_OFF		0
 #define HCLGE_MAC_FEC_BASER		1
 #define HCLGE_MAC_FEC_RS		2
+#define HCLGE_MAC_FEC_LLRS		3
 struct hclge_config_fec_cmd {
 	u8 fec_mode;
 	u8 default_config;
 	u8 rsv[22];
 };
 
+#define HCLGE_FEC_STATS_CMD_NUM 4
+
+struct hclge_query_fec_stats_cmd {
+	/* fec rs mode total stats */
+	__le32 rs_fec_corr_blocks;
+	__le32 rs_fec_uncorr_blocks;
+	__le32 rs_fec_error_blocks;
+	/* fec base-r mode per lanes stats */
+	u8 base_r_lane_num;
+	u8 rsv[3];
+	__le32 base_r_fec_corr_blocks;
+	__le32 base_r_fec_uncorr_blocks;
+};
+
 #define HCLGE_MAC_UPLINK_PORT		0x100
 
 struct hclge_config_max_frm_size_cmd {
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c
index 69b8673436ca..c4aded65e848 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_dcb.c
@@ -359,6 +359,93 @@ static int hclge_ieee_setpfc(struct hnae3_handle *h, struct ieee_pfc *pfc)
 	return hclge_notify_client(hdev, HNAE3_UP_CLIENT);
 }
 
+static int hclge_ieee_setapp(struct hnae3_handle *h, struct dcb_app *app)
+{
+	struct hclge_vport *vport = hclge_get_vport(h);
+	struct net_device *netdev = h->kinfo.netdev;
+	struct hclge_dev *hdev = vport->back;
+	struct dcb_app old_app;
+	int ret;
+
+	if (app->selector != IEEE_8021QAZ_APP_SEL_DSCP ||
+	    app->protocol >= HNAE3_MAX_DSCP ||
+	    app->priority >= HNAE3_MAX_USER_PRIO)
+		return -EINVAL;
+
+	dev_info(&hdev->pdev->dev, "setapp dscp=%u priority=%u\n",
+		 app->protocol, app->priority);
+
+	if (app->priority == h->kinfo.dscp_prio[app->protocol])
+		return 0;
+
+	ret = dcb_ieee_setapp(netdev, app);
+	if (ret)
+		return ret;
+
+	old_app.selector = IEEE_8021QAZ_APP_SEL_DSCP;
+	old_app.protocol = app->protocol;
+	old_app.priority = h->kinfo.dscp_prio[app->protocol];
+
+	h->kinfo.dscp_prio[app->protocol] = app->priority;
+	ret = hclge_dscp_to_tc_map(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed to set dscp to tc map, ret = %d\n", ret);
+		h->kinfo.dscp_prio[app->protocol] = old_app.priority;
+		(void)dcb_ieee_delapp(netdev, app);
+		return ret;
+	}
+
+	vport->nic.kinfo.tc_map_mode = HNAE3_TC_MAP_MODE_DSCP;
+	if (old_app.priority == HNAE3_PRIO_ID_INVALID)
+		h->kinfo.dscp_app_cnt++;
+	else
+		ret = dcb_ieee_delapp(netdev, &old_app);
+
+	return ret;
+}
+
+static int hclge_ieee_delapp(struct hnae3_handle *h, struct dcb_app *app)
+{
+	struct hclge_vport *vport = hclge_get_vport(h);
+	struct net_device *netdev = h->kinfo.netdev;
+	struct hclge_dev *hdev = vport->back;
+	int ret;
+
+	if (app->selector != IEEE_8021QAZ_APP_SEL_DSCP ||
+	    app->protocol >= HNAE3_MAX_DSCP ||
+	    app->priority >= HNAE3_MAX_USER_PRIO ||
+	    app->priority != h->kinfo.dscp_prio[app->protocol])
+		return -EINVAL;
+
+	dev_info(&hdev->pdev->dev, "delapp dscp=%u priority=%u\n",
+		 app->protocol, app->priority);
+
+	ret = dcb_ieee_delapp(netdev, app);
+	if (ret)
+		return ret;
+
+	h->kinfo.dscp_prio[app->protocol] = HNAE3_PRIO_ID_INVALID;
+	ret = hclge_dscp_to_tc_map(hdev);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed to del dscp to tc map, ret = %d\n", ret);
+		h->kinfo.dscp_prio[app->protocol] = app->priority;
+		(void)dcb_ieee_setapp(netdev, app);
+		return ret;
+	}
+
+	if (h->kinfo.dscp_app_cnt)
+		h->kinfo.dscp_app_cnt--;
+
+	if (!h->kinfo.dscp_app_cnt) {
+		vport->nic.kinfo.tc_map_mode = HNAE3_TC_MAP_MODE_PRIO;
+		ret = hclge_up_to_tc_map(hdev);
+	}
+
+	return ret;
+}
+
 /* DCBX configuration */
 static u8 hclge_getdcbx(struct hnae3_handle *h)
 {
@@ -543,6 +630,8 @@ static const struct hnae3_dcb_ops hns3_dcb_ops = {
 	.ieee_setets	= hclge_ieee_setets,
 	.ieee_getpfc	= hclge_ieee_getpfc,
 	.ieee_setpfc	= hclge_ieee_setpfc,
+	.ieee_setapp    = hclge_ieee_setapp,
+	.ieee_delapp    = hclge_ieee_delapp,
 	.getdcbx	= hclge_getdcbx,
 	.setdcbx	= hclge_setdcbx,
 	.setup_tc	= hclge_setup_tc,
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c
index 9b870e79c290..142415c84c6b 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c
@@ -14,6 +14,8 @@ static const char * const hclge_mac_state_str[] = {
 	"TO_ADD", "TO_DEL", "ACTIVE"
 };
 
+static const char * const tc_map_mode_str[] = { "PRIO", "DSCP" };
+
 static const struct hclge_dbg_reg_type_info hclge_dbg_reg_info[] = {
 	{ .cmd = HNAE3_DBG_CMD_REG_BIOS_COMMON,
 	  .dfx_msg = &hclge_dbg_bios_common_reg[0],
@@ -1115,10 +1117,11 @@ static int hclge_dbg_dump_qos_pause_cfg(struct hclge_dev *hdev, char *buf,
 	return 0;
 }
 
+#define HCLGE_DBG_TC_MASK		0x0F
+
 static int hclge_dbg_dump_qos_pri_map(struct hclge_dev *hdev, char *buf,
 				      int len)
 {
-#define HCLGE_DBG_TC_MASK		0x0F
 #define HCLGE_DBG_TC_BIT_WIDTH		4
 
 	struct hclge_qos_pri_map_cmd *pri_map;
@@ -1152,6 +1155,58 @@ static int hclge_dbg_dump_qos_pri_map(struct hclge_dev *hdev, char *buf,
 	return 0;
 }
 
+static int hclge_dbg_dump_qos_dscp_map(struct hclge_dev *hdev, char *buf,
+				       int len)
+{
+	struct hnae3_knic_private_info *kinfo = &hdev->vport[0].nic.kinfo;
+	struct hclge_desc desc[HCLGE_DSCP_MAP_TC_BD_NUM];
+	u8 *req0 = (u8 *)desc[0].data;
+	u8 *req1 = (u8 *)desc[1].data;
+	u8 dscp_tc[HNAE3_MAX_DSCP];
+	int pos, ret;
+	u8 i, j;
+
+	pos = scnprintf(buf, len, "tc map mode: %s\n",
+			tc_map_mode_str[kinfo->tc_map_mode]);
+
+	if (kinfo->tc_map_mode != HNAE3_TC_MAP_MODE_DSCP)
+		return 0;
+
+	hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_QOS_MAP, true);
+	desc[0].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT);
+	hclge_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_QOS_MAP, true);
+	ret = hclge_cmd_send(&hdev->hw, desc, HCLGE_DSCP_MAP_TC_BD_NUM);
+	if (ret) {
+		dev_err(&hdev->pdev->dev,
+			"failed to dump qos dscp map, ret = %d\n", ret);
+		return ret;
+	}
+
+	pos += scnprintf(buf + pos, len - pos, "\nDSCP  PRIO  TC\n");
+
+	/* The low 32 dscp setting use bd0, high 32 dscp setting use bd1 */
+	for (i = 0; i < HNAE3_MAX_DSCP / HCLGE_DSCP_MAP_TC_BD_NUM; i++) {
+		j = i + HNAE3_MAX_DSCP / HCLGE_DSCP_MAP_TC_BD_NUM;
+		/* Each dscp setting has 4 bits, so each byte saves two dscp
+		 * setting
+		 */
+		dscp_tc[i] = req0[i >> 1] >> HCLGE_DSCP_TC_SHIFT(i);
+		dscp_tc[j] = req1[i >> 1] >> HCLGE_DSCP_TC_SHIFT(i);
+		dscp_tc[i] &= HCLGE_DBG_TC_MASK;
+		dscp_tc[j] &= HCLGE_DBG_TC_MASK;
+	}
+
+	for (i = 0; i < HNAE3_MAX_DSCP; i++) {
+		if (kinfo->dscp_prio[i] == HNAE3_PRIO_ID_INVALID)
+			continue;
+
+		pos += scnprintf(buf + pos, len - pos, " %2u    %u    %u\n",
+				 i, kinfo->dscp_prio[i], dscp_tc[i]);
+	}
+
+	return 0;
+}
+
 static int hclge_dbg_dump_tx_buf_cfg(struct hclge_dev *hdev, char *buf, int len)
 {
 	struct hclge_tx_buff_alloc_cmd *tx_buf_cmd;
@@ -1517,7 +1572,7 @@ static int hclge_dbg_dump_fd_tcam(struct hclge_dev *hdev, char *buf, int len)
 	char *tcam_buf;
 	int pos = 0;
 
-	if (!hnae3_dev_fd_supported(hdev)) {
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) {
 		dev_err(&hdev->pdev->dev,
 			"Only FD-supported dev supports dump fd tcam\n");
 		return -EOPNOTSUPP;
@@ -1585,6 +1640,9 @@ static int hclge_dbg_dump_fd_counter(struct hclge_dev *hdev, char *buf, int len)
 	u64 cnt;
 	u8 i;
 
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
+		return -EOPNOTSUPP;
+
 	pos += scnprintf(buf + pos, len - pos,
 			 "func_id\thit_times\n");
 
@@ -2374,6 +2432,10 @@ static const struct hclge_dbg_func hclge_dbg_cmd_func[] = {
 		.dbg_dump = hclge_dbg_dump_qos_pri_map,
 	},
 	{
+		.cmd = HNAE3_DBG_CMD_QOS_DSCP_MAP,
+		.dbg_dump = hclge_dbg_dump_qos_dscp_map,
+	},
+	{
 		.cmd = HNAE3_DBG_CMD_QOS_BUF_CFG,
 		.dbg_dump = hclge_dbg_dump_qos_buf_cfg,
 	},
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
index fae79764dc44..6962a9d69cf8 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
@@ -71,6 +71,7 @@ static void hclge_sync_mac_table(struct hclge_dev *hdev);
 static void hclge_restore_hw_table(struct hclge_dev *hdev);
 static void hclge_sync_promisc_mode(struct hclge_dev *hdev);
 static void hclge_sync_fd_table(struct hclge_dev *hdev);
+static void hclge_update_fec_stats(struct hclge_dev *hdev);
 
 static struct hnae3_ae_algo ae_algo;
 
@@ -148,10 +149,11 @@ static const u32 tqp_intr_reg_addr_list[] = {HCLGE_TQP_INTR_CTRL_REG,
 					     HCLGE_TQP_INTR_RL_REG};
 
 static const char hns3_nic_test_strs[][ETH_GSTRING_LEN] = {
-	"App    Loopback test",
-	"Serdes serial Loopback test",
-	"Serdes parallel Loopback test",
-	"Phy    Loopback test"
+	"External Loopback test",
+	"App      Loopback test",
+	"Serdes   serial Loopback test",
+	"Serdes   parallel Loopback test",
+	"Phy      Loopback test"
 };
 
 static const struct hclge_comm_stats_str g_mac_stats_string[] = {
@@ -679,6 +681,8 @@ static void hclge_update_stats_for_all(struct hclge_dev *hdev)
 		}
 	}
 
+	hclge_update_fec_stats(hdev);
+
 	status = hclge_mac_update_stats(hdev);
 	if (status)
 		dev_err(&hdev->pdev->dev,
@@ -715,7 +719,8 @@ static int hclge_get_sset_count(struct hnae3_handle *handle, int stringset)
 #define HCLGE_LOOPBACK_TEST_FLAGS (HNAE3_SUPPORT_APP_LOOPBACK | \
 		HNAE3_SUPPORT_PHY_LOOPBACK | \
 		HNAE3_SUPPORT_SERDES_SERIAL_LOOPBACK | \
-		HNAE3_SUPPORT_SERDES_PARALLEL_LOOPBACK)
+		HNAE3_SUPPORT_SERDES_PARALLEL_LOOPBACK | \
+		HNAE3_SUPPORT_EXTERNAL_LOOPBACK)
 
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_dev *hdev = vport->back;
@@ -737,9 +742,12 @@ static int hclge_get_sset_count(struct hnae3_handle *handle, int stringset)
 			handle->flags |= HNAE3_SUPPORT_APP_LOOPBACK;
 		}
 
-		count += 2;
+		count += 1;
 		handle->flags |= HNAE3_SUPPORT_SERDES_SERIAL_LOOPBACK;
+		count += 1;
 		handle->flags |= HNAE3_SUPPORT_SERDES_PARALLEL_LOOPBACK;
+		count += 1;
+		handle->flags |= HNAE3_SUPPORT_EXTERNAL_LOOPBACK;
 
 		if ((hdev->hw.mac.phydev && hdev->hw.mac.phydev->drv &&
 		     hdev->hw.mac.phydev->drv->set_loopback) ||
@@ -770,6 +778,11 @@ static void hclge_get_strings(struct hnae3_handle *handle, u32 stringset,
 					   size, p);
 		p = hclge_comm_tqps_get_strings(handle, p);
 	} else if (stringset == ETH_SS_TEST) {
+		if (handle->flags & HNAE3_SUPPORT_EXTERNAL_LOOPBACK) {
+			memcpy(p, hns3_nic_test_strs[HNAE3_LOOP_EXTERNAL],
+			       ETH_GSTRING_LEN);
+			p += ETH_GSTRING_LEN;
+		}
 		if (handle->flags & HNAE3_SUPPORT_APP_LOOPBACK) {
 			memcpy(p, hns3_nic_test_strs[HNAE3_LOOP_APP],
 			       ETH_GSTRING_LEN);
@@ -1003,6 +1016,27 @@ static int hclge_check_port_speed(struct hnae3_handle *handle, u32 speed)
 	return -EINVAL;
 }
 
+static void hclge_update_fec_support(struct hclge_mac *mac)
+{
+	linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, mac->supported);
+	linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, mac->supported);
+	linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_LLRS_BIT, mac->supported);
+	linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_NONE_BIT, mac->supported);
+
+	if (mac->fec_ability & BIT(HNAE3_FEC_BASER))
+		linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT,
+				 mac->supported);
+	if (mac->fec_ability & BIT(HNAE3_FEC_RS))
+		linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT,
+				 mac->supported);
+	if (mac->fec_ability & BIT(HNAE3_FEC_LLRS))
+		linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_LLRS_BIT,
+				 mac->supported);
+	if (mac->fec_ability & BIT(HNAE3_FEC_NONE))
+		linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_NONE_BIT,
+				 mac->supported);
+}
+
 static void hclge_convert_setting_sr(u16 speed_ability,
 				     unsigned long *link_mode)
 {
@@ -1101,34 +1135,36 @@ static void hclge_convert_setting_kr(u16 speed_ability,
 
 static void hclge_convert_setting_fec(struct hclge_mac *mac)
 {
-	linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, mac->supported);
-	linkmode_clear_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, mac->supported);
+	/* If firmware has reported fec_ability, don't need to convert by speed */
+	if (mac->fec_ability)
+		goto out;
 
 	switch (mac->speed) {
 	case HCLGE_MAC_SPEED_10G:
 	case HCLGE_MAC_SPEED_40G:
-		linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT,
-				 mac->supported);
-		mac->fec_ability =
-			BIT(HNAE3_FEC_BASER) | BIT(HNAE3_FEC_AUTO);
+		mac->fec_ability = BIT(HNAE3_FEC_BASER) | BIT(HNAE3_FEC_AUTO) |
+				   BIT(HNAE3_FEC_NONE);
 		break;
 	case HCLGE_MAC_SPEED_25G:
 	case HCLGE_MAC_SPEED_50G:
-		linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT,
-				 mac->supported);
-		mac->fec_ability =
-			BIT(HNAE3_FEC_BASER) | BIT(HNAE3_FEC_RS) |
-			BIT(HNAE3_FEC_AUTO);
+		mac->fec_ability = BIT(HNAE3_FEC_BASER) | BIT(HNAE3_FEC_RS) |
+				   BIT(HNAE3_FEC_AUTO) | BIT(HNAE3_FEC_NONE);
 		break;
 	case HCLGE_MAC_SPEED_100G:
+		mac->fec_ability = BIT(HNAE3_FEC_RS) | BIT(HNAE3_FEC_AUTO) |
+				   BIT(HNAE3_FEC_NONE);
+		break;
 	case HCLGE_MAC_SPEED_200G:
-		linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, mac->supported);
-		mac->fec_ability = BIT(HNAE3_FEC_RS) | BIT(HNAE3_FEC_AUTO);
+		mac->fec_ability = BIT(HNAE3_FEC_RS) | BIT(HNAE3_FEC_AUTO) |
+				   BIT(HNAE3_FEC_LLRS);
 		break;
 	default:
 		mac->fec_ability = 0;
 		break;
 	}
+
+out:
+	hclge_update_fec_support(mac);
 }
 
 static void hclge_parse_fiber_link_mode(struct hclge_dev *hdev,
@@ -1574,7 +1610,7 @@ static int hclge_configure(struct hclge_dev *hdev)
 	if (cfg.vlan_fliter_cap == HCLGE_VLAN_FLTR_CAN_MDF)
 		set_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps);
 
-	if (hnae3_dev_fd_supported(hdev)) {
+	if (hnae3_ae_dev_fd_supported(hdev->ae_dev)) {
 		hdev->fd_en = true;
 		hdev->fd_active_type = HCLGE_FD_RULE_NONE;
 	}
@@ -1617,7 +1653,7 @@ static int hclge_config_gro(struct hclge_dev *hdev)
 	struct hclge_desc desc;
 	int ret;
 
-	if (!hnae3_dev_gro_supported(hdev))
+	if (!hnae3_ae_dev_gro_supported(hdev->ae_dev))
 		return 0;
 
 	hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_GRO_GENERIC_CONFIG, false);
@@ -2589,7 +2625,7 @@ static int hclge_convert_to_fw_speed(u32 speed_drv, u32 *speed_fw)
 }
 
 static int hclge_cfg_mac_speed_dup_hw(struct hclge_dev *hdev, int speed,
-				      u8 duplex)
+				      u8 duplex, u8 lane_num)
 {
 	struct hclge_config_mac_speed_dup_cmd *req;
 	struct hclge_desc desc;
@@ -2613,6 +2649,7 @@ static int hclge_cfg_mac_speed_dup_hw(struct hclge_dev *hdev, int speed,
 			speed_fw);
 	hnae3_set_bit(req->mac_change_fec_en, HCLGE_CFG_MAC_SPEED_CHANGE_EN_B,
 		      1);
+	req->lane_num = lane_num;
 
 	ret = hclge_cmd_send(&hdev->hw, &desc, 1);
 	if (ret) {
@@ -2624,33 +2661,35 @@ static int hclge_cfg_mac_speed_dup_hw(struct hclge_dev *hdev, int speed,
 	return 0;
 }
 
-int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex)
+int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex, u8 lane_num)
 {
 	struct hclge_mac *mac = &hdev->hw.mac;
 	int ret;
 
 	duplex = hclge_check_speed_dup(duplex, speed);
 	if (!mac->support_autoneg && mac->speed == speed &&
-	    mac->duplex == duplex)
+	    mac->duplex == duplex && (mac->lane_num == lane_num || lane_num == 0))
 		return 0;
 
-	ret = hclge_cfg_mac_speed_dup_hw(hdev, speed, duplex);
+	ret = hclge_cfg_mac_speed_dup_hw(hdev, speed, duplex, lane_num);
 	if (ret)
 		return ret;
 
 	hdev->hw.mac.speed = speed;
 	hdev->hw.mac.duplex = duplex;
+	if (!lane_num)
+		hdev->hw.mac.lane_num = lane_num;
 
 	return 0;
 }
 
 static int hclge_cfg_mac_speed_dup_h(struct hnae3_handle *handle, int speed,
-				     u8 duplex)
+				     u8 duplex, u8 lane_num)
 {
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_dev *hdev = vport->back;
 
-	return hclge_cfg_mac_speed_dup(hdev, speed, duplex);
+	return hclge_cfg_mac_speed_dup(hdev, speed, duplex, lane_num);
 }
 
 static int hclge_set_autoneg_en(struct hclge_dev *hdev, bool enable)
@@ -2730,6 +2769,157 @@ static int hclge_halt_autoneg(struct hnae3_handle *handle, bool halt)
 	return 0;
 }
 
+static void hclge_parse_fec_stats_lanes(struct hclge_dev *hdev,
+					struct hclge_desc *desc, u32 desc_len)
+{
+	u32 lane_size = HCLGE_FEC_STATS_MAX_LANES * 2;
+	u32 desc_index = 0;
+	u32 data_index = 0;
+	u32 i;
+
+	for (i = 0; i < lane_size; i++) {
+		if (data_index >= HCLGE_DESC_DATA_LEN) {
+			desc_index++;
+			data_index = 0;
+		}
+
+		if (desc_index >= desc_len)
+			return;
+
+		hdev->fec_stats.per_lanes[i] +=
+			le32_to_cpu(desc[desc_index].data[data_index]);
+		data_index++;
+	}
+}
+
+static void hclge_parse_fec_stats(struct hclge_dev *hdev,
+				  struct hclge_desc *desc, u32 desc_len)
+{
+	struct hclge_query_fec_stats_cmd *req;
+
+	req = (struct hclge_query_fec_stats_cmd *)desc[0].data;
+
+	hdev->fec_stats.base_r_lane_num = req->base_r_lane_num;
+	hdev->fec_stats.rs_corr_blocks +=
+		le32_to_cpu(req->rs_fec_corr_blocks);
+	hdev->fec_stats.rs_uncorr_blocks +=
+		le32_to_cpu(req->rs_fec_uncorr_blocks);
+	hdev->fec_stats.rs_error_blocks +=
+		le32_to_cpu(req->rs_fec_error_blocks);
+	hdev->fec_stats.base_r_corr_blocks +=
+		le32_to_cpu(req->base_r_fec_corr_blocks);
+	hdev->fec_stats.base_r_uncorr_blocks +=
+		le32_to_cpu(req->base_r_fec_uncorr_blocks);
+
+	hclge_parse_fec_stats_lanes(hdev, &desc[1], desc_len - 1);
+}
+
+static int hclge_update_fec_stats_hw(struct hclge_dev *hdev)
+{
+	struct hclge_desc desc[HCLGE_FEC_STATS_CMD_NUM];
+	int ret;
+	u32 i;
+
+	for (i = 0; i < HCLGE_FEC_STATS_CMD_NUM; i++) {
+		hclge_cmd_setup_basic_desc(&desc[i], HCLGE_OPC_QUERY_FEC_STATS,
+					   true);
+		if (i != (HCLGE_FEC_STATS_CMD_NUM - 1))
+			desc[i].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT);
+	}
+
+	ret = hclge_cmd_send(&hdev->hw, desc, HCLGE_FEC_STATS_CMD_NUM);
+	if (ret)
+		return ret;
+
+	hclge_parse_fec_stats(hdev, desc, HCLGE_FEC_STATS_CMD_NUM);
+
+	return 0;
+}
+
+static void hclge_update_fec_stats(struct hclge_dev *hdev)
+{
+	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev);
+	int ret;
+
+	if (!hnae3_ae_dev_fec_stats_supported(ae_dev) ||
+	    test_and_set_bit(HCLGE_STATE_FEC_STATS_UPDATING, &hdev->state))
+		return;
+
+	ret = hclge_update_fec_stats_hw(hdev);
+	if (ret)
+		dev_err(&hdev->pdev->dev,
+			"failed to update fec stats, ret = %d\n", ret);
+
+	clear_bit(HCLGE_STATE_FEC_STATS_UPDATING, &hdev->state);
+}
+
+static void hclge_get_fec_stats_total(struct hclge_dev *hdev,
+				      struct ethtool_fec_stats *fec_stats)
+{
+	fec_stats->corrected_blocks.total = hdev->fec_stats.rs_corr_blocks;
+	fec_stats->uncorrectable_blocks.total =
+		hdev->fec_stats.rs_uncorr_blocks;
+}
+
+static void hclge_get_fec_stats_lanes(struct hclge_dev *hdev,
+				      struct ethtool_fec_stats *fec_stats)
+{
+	u32 i;
+
+	if (hdev->fec_stats.base_r_lane_num == 0 ||
+	    hdev->fec_stats.base_r_lane_num > HCLGE_FEC_STATS_MAX_LANES) {
+		dev_err(&hdev->pdev->dev,
+			"fec stats lane number(%llu) is invalid\n",
+			hdev->fec_stats.base_r_lane_num);
+		return;
+	}
+
+	for (i = 0; i < hdev->fec_stats.base_r_lane_num; i++) {
+		fec_stats->corrected_blocks.lanes[i] =
+			hdev->fec_stats.base_r_corr_per_lanes[i];
+		fec_stats->uncorrectable_blocks.lanes[i] =
+			hdev->fec_stats.base_r_uncorr_per_lanes[i];
+	}
+}
+
+static void hclge_comm_get_fec_stats(struct hclge_dev *hdev,
+				     struct ethtool_fec_stats *fec_stats)
+{
+	u32 fec_mode = hdev->hw.mac.fec_mode;
+
+	switch (fec_mode) {
+	case BIT(HNAE3_FEC_RS):
+	case BIT(HNAE3_FEC_LLRS):
+		hclge_get_fec_stats_total(hdev, fec_stats);
+		break;
+	case BIT(HNAE3_FEC_BASER):
+		hclge_get_fec_stats_lanes(hdev, fec_stats);
+		break;
+	default:
+		dev_err(&hdev->pdev->dev,
+			"fec stats is not supported by current fec mode(0x%x)\n",
+			fec_mode);
+		break;
+	}
+}
+
+static void hclge_get_fec_stats(struct hnae3_handle *handle,
+				struct ethtool_fec_stats *fec_stats)
+{
+	struct hclge_vport *vport = hclge_get_vport(handle);
+	struct hclge_dev *hdev = vport->back;
+	u32 fec_mode = hdev->hw.mac.fec_mode;
+
+	if (fec_mode == BIT(HNAE3_FEC_NONE) ||
+	    fec_mode == BIT(HNAE3_FEC_AUTO) ||
+	    fec_mode == BIT(HNAE3_FEC_USER_DEF))
+		return;
+
+	hclge_update_fec_stats(hdev);
+
+	hclge_comm_get_fec_stats(hdev, fec_stats);
+}
+
 static int hclge_set_fec_hw(struct hclge_dev *hdev, u32 fec_mode)
 {
 	struct hclge_config_fec_cmd *req;
@@ -2744,6 +2934,9 @@ static int hclge_set_fec_hw(struct hclge_dev *hdev, u32 fec_mode)
 	if (fec_mode & BIT(HNAE3_FEC_RS))
 		hnae3_set_field(req->fec_mode, HCLGE_MAC_CFG_FEC_MODE_M,
 				HCLGE_MAC_CFG_FEC_MODE_S, HCLGE_MAC_FEC_RS);
+	if (fec_mode & BIT(HNAE3_FEC_LLRS))
+		hnae3_set_field(req->fec_mode, HCLGE_MAC_CFG_FEC_MODE_M,
+				HCLGE_MAC_CFG_FEC_MODE_S, HCLGE_MAC_FEC_LLRS);
 	if (fec_mode & BIT(HNAE3_FEC_BASER))
 		hnae3_set_field(req->fec_mode, HCLGE_MAC_CFG_FEC_MODE_M,
 				HCLGE_MAC_CFG_FEC_MODE_S, HCLGE_MAC_FEC_BASER);
@@ -2796,7 +2989,7 @@ static int hclge_mac_init(struct hclge_dev *hdev)
 	hdev->support_sfp_query = true;
 	hdev->hw.mac.duplex = HCLGE_MAC_FULL;
 	ret = hclge_cfg_mac_speed_dup_hw(hdev, hdev->hw.mac.speed,
-					 hdev->hw.mac.duplex);
+					 hdev->hw.mac.duplex, hdev->hw.mac.lane_num);
 	if (ret)
 		return ret;
 
@@ -2988,6 +3181,9 @@ static void hclge_update_fec_advertising(struct hclge_mac *mac)
 	if (mac->fec_mode & BIT(HNAE3_FEC_RS))
 		linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT,
 				 mac->advertising);
+	else if (mac->fec_mode & BIT(HNAE3_FEC_LLRS))
+		linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_LLRS_BIT,
+				 mac->advertising);
 	else if (mac->fec_mode & BIT(HNAE3_FEC_BASER))
 		linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT,
 				 mac->advertising);
@@ -3037,7 +3233,6 @@ static void hclge_update_port_capability(struct hclge_dev *hdev,
 					 struct hclge_mac *mac)
 {
 	if (hnae3_dev_fec_supported(hdev))
-		/* update fec ability by speed */
 		hclge_convert_setting_fec(mac);
 
 	/* firmware can not identify back plane type, the media type
@@ -3119,10 +3314,12 @@ static int hclge_get_sfp_info(struct hclge_dev *hdev, struct hclge_mac *mac)
 		mac->autoneg = resp->autoneg;
 		mac->support_autoneg = resp->autoneg_ability;
 		mac->speed_type = QUERY_ACTIVE_SPEED;
+		mac->lane_num = resp->lane_num;
 		if (!resp->active_fec)
 			mac->fec_mode = 0;
 		else
 			mac->fec_mode = BIT(resp->active_fec);
+		mac->fec_ability = resp->fec_ability;
 	} else {
 		mac->speed_type = QUERY_SFP_SPEED;
 	}
@@ -3302,13 +3499,13 @@ static int hclge_update_port_info(struct hclge_dev *hdev)
 			return 0;
 		}
 		return hclge_cfg_mac_speed_dup(hdev, mac->speed,
-					       HCLGE_MAC_FULL);
+					       HCLGE_MAC_FULL, mac->lane_num);
 	} else {
 		if (speed == HCLGE_MAC_SPEED_UNKNOWN)
 			return 0; /* do nothing if no SFP */
 
 		/* must config full duplex for SFP */
-		return hclge_cfg_mac_speed_dup(hdev, speed, HCLGE_MAC_FULL);
+		return hclge_cfg_mac_speed_dup(hdev, speed, HCLGE_MAC_FULL, 0);
 	}
 }
 
@@ -5334,7 +5531,7 @@ static int hclge_init_fd_config(struct hclge_dev *hdev)
 	struct hclge_fd_key_cfg *key_cfg;
 	int ret;
 
-	if (!hnae3_dev_fd_supported(hdev))
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
 		return 0;
 
 	ret = hclge_get_fd_mode(hdev, &hdev->fd_cfg.fd_mode);
@@ -6339,7 +6536,7 @@ static int hclge_add_fd_entry(struct hnae3_handle *handle,
 	u8 action;
 	int ret;
 
-	if (!hnae3_dev_fd_supported(hdev)) {
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) {
 		dev_err(&hdev->pdev->dev,
 			"flow table director is not supported\n");
 		return -EOPNOTSUPP;
@@ -6395,7 +6592,7 @@ static int hclge_del_fd_entry(struct hnae3_handle *handle,
 	struct ethtool_rx_flow_spec *fs;
 	int ret;
 
-	if (!hnae3_dev_fd_supported(hdev))
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
 		return -EOPNOTSUPP;
 
 	fs = (struct ethtool_rx_flow_spec *)&cmd->fs;
@@ -6431,9 +6628,6 @@ static void hclge_clear_fd_rules_in_list(struct hclge_dev *hdev,
 	struct hlist_node *node;
 	u16 location;
 
-	if (!hnae3_dev_fd_supported(hdev))
-		return;
-
 	spin_lock_bh(&hdev->fd_rule_lock);
 
 	for_each_set_bit(location, hdev->fd_bmap,
@@ -6458,6 +6652,9 @@ static void hclge_clear_fd_rules_in_list(struct hclge_dev *hdev,
 
 static void hclge_del_all_fd_entries(struct hclge_dev *hdev)
 {
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
+		return;
+
 	hclge_clear_fd_rules_in_list(hdev, true);
 	hclge_fd_disable_user_def(hdev);
 }
@@ -6473,7 +6670,7 @@ static int hclge_restore_fd_entries(struct hnae3_handle *handle)
 	 * return value. If error is returned here, the reset process will
 	 * fail.
 	 */
-	if (!hnae3_dev_fd_supported(hdev))
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
 		return 0;
 
 	/* if fd is disabled, should not restore it when reset */
@@ -6497,7 +6694,7 @@ static int hclge_get_fd_rule_cnt(struct hnae3_handle *handle,
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_dev *hdev = vport->back;
 
-	if (!hnae3_dev_fd_supported(hdev) || hclge_is_cls_flower_active(handle))
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev) || hclge_is_cls_flower_active(handle))
 		return -EOPNOTSUPP;
 
 	cmd->rule_cnt = hdev->hclge_fd_rule_num;
@@ -6715,7 +6912,7 @@ static int hclge_get_fd_rule_info(struct hnae3_handle *handle,
 	struct hclge_dev *hdev = vport->back;
 	struct ethtool_rx_flow_spec *fs;
 
-	if (!hnae3_dev_fd_supported(hdev))
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
 		return -EOPNOTSUPP;
 
 	fs = (struct ethtool_rx_flow_spec *)&cmd->fs;
@@ -6778,7 +6975,7 @@ static int hclge_get_all_rules(struct hnae3_handle *handle,
 	struct hlist_node *node2;
 	int cnt = 0;
 
-	if (!hnae3_dev_fd_supported(hdev))
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
 		return -EOPNOTSUPP;
 
 	cmd->data = hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1];
@@ -6878,7 +7075,7 @@ static int hclge_add_fd_entry_by_arfs(struct hnae3_handle *handle, u16 queue_id,
 	struct hclge_fd_rule *rule;
 	u16 bit_id;
 
-	if (!hnae3_dev_fd_supported(hdev))
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
 		return -EOPNOTSUPP;
 
 	/* when there is already fd rule existed add by user,
@@ -7167,6 +7364,12 @@ static int hclge_add_cls_flower(struct hnae3_handle *handle,
 	struct hclge_fd_rule *rule;
 	int ret;
 
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) {
+		dev_err(&hdev->pdev->dev,
+			"cls flower is not supported\n");
+		return -EOPNOTSUPP;
+	}
+
 	ret = hclge_check_cls_flower(hdev, cls_flower, tc);
 	if (ret) {
 		dev_err(&hdev->pdev->dev,
@@ -7220,6 +7423,9 @@ static int hclge_del_cls_flower(struct hnae3_handle *handle,
 	struct hclge_fd_rule *rule;
 	int ret;
 
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
+		return -EOPNOTSUPP;
+
 	spin_lock_bh(&hdev->fd_rule_lock);
 
 	rule = hclge_find_cls_flower(hdev, cls_flower->cookie);
@@ -7282,6 +7488,9 @@ out:
 
 static void hclge_sync_fd_table(struct hclge_dev *hdev)
 {
+	if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
+		return;
+
 	if (test_and_clear_bit(HCLGE_STATE_FD_CLEAR_ALL, &hdev->state)) {
 		bool clear_list = hdev->fd_active_type == HCLGE_FD_ARFS_ACTIVE;
 
@@ -7705,7 +7914,7 @@ static int hclge_set_loopback(struct hnae3_handle *handle,
 {
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_dev *hdev = vport->back;
-	int ret;
+	int ret = 0;
 
 	/* Loopback can be enabled in three places: SSU, MAC, and serdes. By
 	 * default, SSU loopback is enabled, so if the SMAC and the DMAC are
@@ -7732,6 +7941,8 @@ static int hclge_set_loopback(struct hnae3_handle *handle,
 	case HNAE3_LOOP_PHY:
 		ret = hclge_set_phy_loopback(hdev, en);
 		break;
+	case HNAE3_LOOP_EXTERNAL:
+		break;
 	default:
 		ret = -ENOTSUPP;
 		dev_err(&hdev->pdev->dev,
@@ -10793,7 +11004,7 @@ static int hclge_set_pauseparam(struct hnae3_handle *handle, u32 auto_neg,
 }
 
 static void hclge_get_ksettings_an_result(struct hnae3_handle *handle,
-					  u8 *auto_neg, u32 *speed, u8 *duplex)
+					  u8 *auto_neg, u32 *speed, u8 *duplex, u32 *lane_num)
 {
 	struct hclge_vport *vport = hclge_get_vport(handle);
 	struct hclge_dev *hdev = vport->back;
@@ -10804,6 +11015,8 @@ static void hclge_get_ksettings_an_result(struct hnae3_handle *handle,
 		*duplex = hdev->hw.mac.duplex;
 	if (auto_neg)
 		*auto_neg = hdev->hw.mac.autoneg;
+	if (lane_num)
+		*lane_num = hdev->hw.mac.lane_num;
 }
 
 static void hclge_get_media_type(struct hnae3_handle *handle, u8 *media_type,
@@ -11443,6 +11656,10 @@ static int hclge_init_ae_dev(struct hnae3_ae_dev *ae_dev)
 	if (ret)
 		goto err_mdiobus_unreg;
 
+	ret = hclge_update_port_info(hdev);
+	if (ret)
+		goto err_mdiobus_unreg;
+
 	INIT_KFIFO(hdev->mac_tnl_log);
 
 	hclge_dcb_ops_set(hdev);
@@ -11510,6 +11727,7 @@ out:
 static void hclge_stats_clear(struct hclge_dev *hdev)
 {
 	memset(&hdev->mac_stats, 0, sizeof(hdev->mac_stats));
+	memset(&hdev->fec_stats, 0, sizeof(hdev->fec_stats));
 }
 
 static int hclge_set_mac_spoofchk(struct hclge_dev *hdev, int vf, bool enable)
@@ -12763,6 +12981,21 @@ static void hclge_clean_vport_config(struct hnae3_ae_dev *ae_dev, int num_vfs)
 	}
 }
 
+static int hclge_get_dscp_prio(struct hnae3_handle *h, u8 dscp, u8 *tc_mode,
+			       u8 *priority)
+{
+	if (dscp >= HNAE3_MAX_DSCP)
+		return -EINVAL;
+
+	if (tc_mode)
+		*tc_mode = h->kinfo.tc_map_mode;
+	if (priority)
+		*priority = h->kinfo.dscp_prio[dscp] == HNAE3_PRIO_ID_INVALID ? 0 :
+			    h->kinfo.dscp_prio[dscp];
+
+	return 0;
+}
+
 static const struct hnae3_ae_ops hclge_ops = {
 	.init_ae_dev = hclge_init_ae_dev,
 	.uninit_ae_dev = hclge_uninit_ae_dev,
@@ -12786,6 +13019,7 @@ static const struct hnae3_ae_ops hclge_ops = {
 	.cfg_mac_speed_dup_h = hclge_cfg_mac_speed_dup_h,
 	.get_media_type = hclge_get_media_type,
 	.check_port_speed = hclge_check_port_speed,
+	.get_fec_stats = hclge_get_fec_stats,
 	.get_fec = hclge_get_fec,
 	.set_fec = hclge_set_fec,
 	.get_rss_key_size = hclge_comm_get_rss_key_size,
@@ -12865,6 +13099,7 @@ static const struct hnae3_ae_ops hclge_ops = {
 	.get_ts_info = hclge_ptp_get_ts_info,
 	.get_link_diagnosis_info = hclge_get_link_diagnosis_info,
 	.clean_vf_config = hclge_clean_vport_config,
+	.get_dscp_prio = hclge_get_dscp_prio,
 };
 
 static struct hnae3_ae_algo ae_algo = {
@@ -12872,7 +13107,7 @@ static struct hnae3_ae_algo ae_algo = {
 	.pdev_id_table = ae_algo_pci_tbl,
 };
 
-static int hclge_init(void)
+static int __init hclge_init(void)
 {
 	pr_info("%s is initializing\n", HCLGE_NAME);
 
@@ -12887,7 +13122,7 @@ static int hclge_init(void)
 	return 0;
 }
 
-static void hclge_exit(void)
+static void __exit hclge_exit(void)
 {
 	hnae3_unregister_ae_algo_prepare(&ae_algo);
 	hnae3_unregister_ae_algo(&ae_algo);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
index 18caddd541f8..495b639b0dc2 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h
@@ -216,6 +216,7 @@ enum HCLGE_DEV_STATE {
 	HCLGE_STATE_FD_USER_DEF_CHANGED,
 	HCLGE_STATE_PTP_EN,
 	HCLGE_STATE_PTP_TX_HANDLING,
+	HCLGE_STATE_FEC_STATS_UPDATING,
 	HCLGE_STATE_MAX
 };
 
@@ -258,6 +259,7 @@ struct hclge_mac {
 	u8 duplex;
 	u8 support_autoneg;
 	u8 speed_type;	/* 0: sfp speed, 1: active speed */
+	u8 lane_num;
 	u32 speed;
 	u32 max_speed;
 	u32 speed_ability; /* speed ability supported by current media */
@@ -488,6 +490,26 @@ struct hclge_mac_stats {
 
 #define HCLGE_STATS_TIMER_INTERVAL	300UL
 
+/* fec stats ,opcode id: 0x0316 */
+#define HCLGE_FEC_STATS_MAX_LANES	8
+struct hclge_fec_stats {
+	/* fec rs mode total stats */
+	u64 rs_corr_blocks;
+	u64 rs_uncorr_blocks;
+	u64 rs_error_blocks;
+	/* fec base-r mode per lanes stats */
+	u64 base_r_lane_num;
+	u64 base_r_corr_blocks;
+	u64 base_r_uncorr_blocks;
+	union {
+		struct {
+			u64 base_r_corr_per_lanes[HCLGE_FEC_STATS_MAX_LANES];
+			u64 base_r_uncorr_per_lanes[HCLGE_FEC_STATS_MAX_LANES];
+		};
+		u64 per_lanes[HCLGE_FEC_STATS_MAX_LANES * 2];
+	};
+};
+
 struct hclge_vlan_type_cfg {
 	u16 rx_ot_fst_vlan_type;
 	u16 rx_ot_sec_vlan_type;
@@ -826,6 +848,7 @@ struct hclge_dev {
 	struct hclge_hw hw;
 	struct hclge_misc_vector misc_vector;
 	struct hclge_mac_stats mac_stats;
+	struct hclge_fec_stats fec_stats;
 	unsigned long state;
 	unsigned long flr_state;
 	unsigned long last_reset_time;
@@ -1070,7 +1093,7 @@ static inline int hclge_get_queue_id(struct hnae3_queue *queue)
 }
 
 int hclge_inform_reset_assert_to_vf(struct hclge_vport *vport);
-int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex);
+int hclge_cfg_mac_speed_dup(struct hclge_dev *hdev, int speed, u8 duplex, u8 lane_num);
 int hclge_set_vlan_filter(struct hnae3_handle *handle, __be16 proto,
 			  u16 vlan_id, bool is_kill);
 int hclge_en_hw_strip_rxvtag(struct hnae3_handle *handle, bool enable);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
index e1012f7f9b73..a7b06c63143c 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c
@@ -779,17 +779,284 @@ static void hclge_handle_vf_tbl(struct hclge_vport *vport,
 	}
 }
 
+static int
+hclge_mbx_map_ring_to_vector_handler(struct hclge_mbx_ops_param *param)
+{
+	return hclge_map_unmap_ring_to_vf_vector(param->vport, true,
+						 param->req);
+}
+
+static int
+hclge_mbx_unmap_ring_to_vector_handler(struct hclge_mbx_ops_param *param)
+{
+	return hclge_map_unmap_ring_to_vf_vector(param->vport, false,
+						 param->req);
+}
+
+static int
+hclge_mbx_get_ring_vector_map_handler(struct hclge_mbx_ops_param *param)
+{
+	int ret;
+
+	ret = hclge_get_vf_ring_vector_map(param->vport, param->req,
+					   param->resp_msg);
+	if (ret)
+		dev_err(&param->vport->back->pdev->dev,
+			"PF fail(%d) to get VF ring vector map\n",
+			ret);
+	return ret;
+}
+
+static int hclge_mbx_set_promisc_mode_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_set_vf_promisc_mode(param->vport, param->req);
+	return 0;
+}
+
+static int hclge_mbx_set_unicast_handler(struct hclge_mbx_ops_param *param)
+{
+	int ret;
+
+	ret = hclge_set_vf_uc_mac_addr(param->vport, param->req);
+	if (ret)
+		dev_err(&param->vport->back->pdev->dev,
+			"PF fail(%d) to set VF UC MAC Addr\n",
+			ret);
+	return ret;
+}
+
+static int hclge_mbx_set_multicast_handler(struct hclge_mbx_ops_param *param)
+{
+	int ret;
+
+	ret = hclge_set_vf_mc_mac_addr(param->vport, param->req);
+	if (ret)
+		dev_err(&param->vport->back->pdev->dev,
+			"PF fail(%d) to set VF MC MAC Addr\n",
+			ret);
+	return ret;
+}
+
+static int hclge_mbx_set_vlan_handler(struct hclge_mbx_ops_param *param)
+{
+	int ret;
+
+	ret = hclge_set_vf_vlan_cfg(param->vport, param->req, param->resp_msg);
+	if (ret)
+		dev_err(&param->vport->back->pdev->dev,
+			"PF failed(%d) to config VF's VLAN\n",
+			ret);
+	return ret;
+}
+
+static int hclge_mbx_set_alive_handler(struct hclge_mbx_ops_param *param)
+{
+	int ret;
+
+	ret = hclge_set_vf_alive(param->vport, param->req);
+	if (ret)
+		dev_err(&param->vport->back->pdev->dev,
+			"PF failed(%d) to set VF's ALIVE\n",
+			ret);
+	return ret;
+}
+
+static int hclge_mbx_get_qinfo_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_get_vf_queue_info(param->vport, param->resp_msg);
+	return 0;
+}
+
+static int hclge_mbx_get_qdepth_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_get_vf_queue_depth(param->vport, param->resp_msg);
+	return 0;
+}
+
+static int hclge_mbx_get_basic_info_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_get_basic_info(param->vport, param->resp_msg);
+	return 0;
+}
+
+static int hclge_mbx_get_link_status_handler(struct hclge_mbx_ops_param *param)
+{
+	int ret;
+
+	ret = hclge_push_vf_link_status(param->vport);
+	if (ret)
+		dev_err(&param->vport->back->pdev->dev,
+			"failed to inform link stat to VF, ret = %d\n",
+			ret);
+	return ret;
+}
+
+static int hclge_mbx_queue_reset_handler(struct hclge_mbx_ops_param *param)
+{
+	return hclge_mbx_reset_vf_queue(param->vport, param->req,
+					param->resp_msg);
+}
+
+static int hclge_mbx_reset_handler(struct hclge_mbx_ops_param *param)
+{
+	return hclge_reset_vf(param->vport);
+}
+
+static int hclge_mbx_keep_alive_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_vf_keep_alive(param->vport);
+	return 0;
+}
+
+static int hclge_mbx_set_mtu_handler(struct hclge_mbx_ops_param *param)
+{
+	int ret;
+
+	ret = hclge_set_vf_mtu(param->vport, param->req);
+	if (ret)
+		dev_err(&param->vport->back->pdev->dev,
+			"VF fail(%d) to set mtu\n", ret);
+	return ret;
+}
+
+static int hclge_mbx_get_qid_in_pf_handler(struct hclge_mbx_ops_param *param)
+{
+	return hclge_get_queue_id_in_pf(param->vport, param->req,
+					param->resp_msg);
+}
+
+static int hclge_mbx_get_rss_key_handler(struct hclge_mbx_ops_param *param)
+{
+	return hclge_get_rss_key(param->vport, param->req, param->resp_msg);
+}
+
+static int hclge_mbx_get_link_mode_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_get_link_mode(param->vport, param->req);
+	return 0;
+}
+
+static int
+hclge_mbx_get_vf_flr_status_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_rm_vport_all_mac_table(param->vport, false,
+				     HCLGE_MAC_ADDR_UC);
+	hclge_rm_vport_all_mac_table(param->vport, false,
+				     HCLGE_MAC_ADDR_MC);
+	hclge_rm_vport_all_vlan_table(param->vport, false);
+	return 0;
+}
+
+static int hclge_mbx_vf_uninit_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_rm_vport_all_mac_table(param->vport, true,
+				     HCLGE_MAC_ADDR_UC);
+	hclge_rm_vport_all_mac_table(param->vport, true,
+				     HCLGE_MAC_ADDR_MC);
+	hclge_rm_vport_all_vlan_table(param->vport, true);
+	return 0;
+}
+
+static int hclge_mbx_get_media_type_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_get_vf_media_type(param->vport, param->resp_msg);
+	return 0;
+}
+
+static int hclge_mbx_push_link_status_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_handle_link_change_event(param->vport->back, param->req);
+	return 0;
+}
+
+static int hclge_mbx_get_mac_addr_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_get_vf_mac_addr(param->vport, param->resp_msg);
+	return 0;
+}
+
+static int hclge_mbx_ncsi_error_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_handle_ncsi_error(param->vport->back);
+	return 0;
+}
+
+static int hclge_mbx_handle_vf_tbl_handler(struct hclge_mbx_ops_param *param)
+{
+	hclge_handle_vf_tbl(param->vport, param->req);
+	return 0;
+}
+
+static const hclge_mbx_ops_fn hclge_mbx_ops_list[HCLGE_MBX_OPCODE_MAX] = {
+	[HCLGE_MBX_RESET]   = hclge_mbx_reset_handler,
+	[HCLGE_MBX_SET_UNICAST] = hclge_mbx_set_unicast_handler,
+	[HCLGE_MBX_SET_MULTICAST] = hclge_mbx_set_multicast_handler,
+	[HCLGE_MBX_SET_VLAN] = hclge_mbx_set_vlan_handler,
+	[HCLGE_MBX_MAP_RING_TO_VECTOR] = hclge_mbx_map_ring_to_vector_handler,
+	[HCLGE_MBX_UNMAP_RING_TO_VECTOR] = hclge_mbx_unmap_ring_to_vector_handler,
+	[HCLGE_MBX_SET_PROMISC_MODE] = hclge_mbx_set_promisc_mode_handler,
+	[HCLGE_MBX_GET_QINFO] = hclge_mbx_get_qinfo_handler,
+	[HCLGE_MBX_GET_QDEPTH] = hclge_mbx_get_qdepth_handler,
+	[HCLGE_MBX_GET_BASIC_INFO] = hclge_mbx_get_basic_info_handler,
+	[HCLGE_MBX_GET_RSS_KEY] = hclge_mbx_get_rss_key_handler,
+	[HCLGE_MBX_GET_MAC_ADDR] = hclge_mbx_get_mac_addr_handler,
+	[HCLGE_MBX_GET_LINK_STATUS] = hclge_mbx_get_link_status_handler,
+	[HCLGE_MBX_QUEUE_RESET] = hclge_mbx_queue_reset_handler,
+	[HCLGE_MBX_KEEP_ALIVE] = hclge_mbx_keep_alive_handler,
+	[HCLGE_MBX_SET_ALIVE] = hclge_mbx_set_alive_handler,
+	[HCLGE_MBX_SET_MTU] = hclge_mbx_set_mtu_handler,
+	[HCLGE_MBX_GET_QID_IN_PF] = hclge_mbx_get_qid_in_pf_handler,
+	[HCLGE_MBX_GET_LINK_MODE] = hclge_mbx_get_link_mode_handler,
+	[HCLGE_MBX_GET_MEDIA_TYPE] = hclge_mbx_get_media_type_handler,
+	[HCLGE_MBX_VF_UNINIT] = hclge_mbx_vf_uninit_handler,
+	[HCLGE_MBX_HANDLE_VF_TBL] = hclge_mbx_handle_vf_tbl_handler,
+	[HCLGE_MBX_GET_RING_VECTOR_MAP] = hclge_mbx_get_ring_vector_map_handler,
+	[HCLGE_MBX_GET_VF_FLR_STATUS] = hclge_mbx_get_vf_flr_status_handler,
+	[HCLGE_MBX_PUSH_LINK_STATUS] = hclge_mbx_push_link_status_handler,
+	[HCLGE_MBX_NCSI_ERROR] = hclge_mbx_ncsi_error_handler,
+};
+
+static void hclge_mbx_request_handling(struct hclge_mbx_ops_param *param)
+{
+	hclge_mbx_ops_fn cmd_func = NULL;
+	struct hclge_dev *hdev;
+	int ret = 0;
+
+	hdev = param->vport->back;
+	cmd_func = hclge_mbx_ops_list[param->req->msg.code];
+	if (cmd_func)
+		ret = cmd_func(param);
+	else
+		dev_err(&hdev->pdev->dev,
+			"un-supported mailbox message, code = %u\n",
+			param->req->msg.code);
+
+	/* PF driver should not reply IMP */
+	if (hnae3_get_bit(param->req->mbx_need_resp, HCLGE_MBX_NEED_RESP_B) &&
+	    param->req->msg.code < HCLGE_MBX_GET_VF_FLR_STATUS) {
+		param->resp_msg->status = ret;
+		if (time_is_before_jiffies(hdev->last_mbx_scheduled +
+					   HCLGE_MBX_SCHED_TIMEOUT))
+			dev_warn(&hdev->pdev->dev,
+				 "resp vport%u mbx(%u,%u) late\n",
+				 param->req->mbx_src_vfid,
+				 param->req->msg.code,
+				 param->req->msg.subcode);
+
+		hclge_gen_resp_to_vf(param->vport, param->req, param->resp_msg);
+	}
+}
+
 void hclge_mbx_handler(struct hclge_dev *hdev)
 {
 	struct hclge_comm_cmq_ring *crq = &hdev->hw.hw.cmq.crq;
 	struct hclge_respond_to_vf_msg resp_msg;
 	struct hclge_mbx_vf_to_pf_cmd *req;
-	struct hclge_vport *vport;
+	struct hclge_mbx_ops_param param;
 	struct hclge_desc *desc;
-	bool is_del = false;
 	unsigned int flag;
-	int ret = 0;
 
+	param.resp_msg = &resp_msg;
 	/* handle all the mailbox requests in the queue */
 	while (!hclge_cmd_crq_empty(&hdev->hw)) {
 		if (test_bit(HCLGE_COMM_STATE_CMD_DISABLE,
@@ -814,152 +1081,16 @@ void hclge_mbx_handler(struct hclge_dev *hdev)
 			continue;
 		}
 
-		vport = &hdev->vport[req->mbx_src_vfid];
-
 		trace_hclge_pf_mbx_get(hdev, req);
 
 		/* clear the resp_msg before processing every mailbox message */
 		memset(&resp_msg, 0, sizeof(resp_msg));
-
-		switch (req->msg.code) {
-		case HCLGE_MBX_MAP_RING_TO_VECTOR:
-			ret = hclge_map_unmap_ring_to_vf_vector(vport, true,
-								req);
-			break;
-		case HCLGE_MBX_UNMAP_RING_TO_VECTOR:
-			ret = hclge_map_unmap_ring_to_vf_vector(vport, false,
-								req);
-			break;
-		case HCLGE_MBX_GET_RING_VECTOR_MAP:
-			ret = hclge_get_vf_ring_vector_map(vport, req,
-							   &resp_msg);
-			if (ret)
-				dev_err(&hdev->pdev->dev,
-					"PF fail(%d) to get VF ring vector map\n",
-					ret);
-			break;
-		case HCLGE_MBX_SET_PROMISC_MODE:
-			hclge_set_vf_promisc_mode(vport, req);
-			break;
-		case HCLGE_MBX_SET_UNICAST:
-			ret = hclge_set_vf_uc_mac_addr(vport, req);
-			if (ret)
-				dev_err(&hdev->pdev->dev,
-					"PF fail(%d) to set VF UC MAC Addr\n",
-					ret);
-			break;
-		case HCLGE_MBX_SET_MULTICAST:
-			ret = hclge_set_vf_mc_mac_addr(vport, req);
-			if (ret)
-				dev_err(&hdev->pdev->dev,
-					"PF fail(%d) to set VF MC MAC Addr\n",
-					ret);
-			break;
-		case HCLGE_MBX_SET_VLAN:
-			ret = hclge_set_vf_vlan_cfg(vport, req, &resp_msg);
-			if (ret)
-				dev_err(&hdev->pdev->dev,
-					"PF failed(%d) to config VF's VLAN\n",
-					ret);
-			break;
-		case HCLGE_MBX_SET_ALIVE:
-			ret = hclge_set_vf_alive(vport, req);
-			if (ret)
-				dev_err(&hdev->pdev->dev,
-					"PF failed(%d) to set VF's ALIVE\n",
-					ret);
-			break;
-		case HCLGE_MBX_GET_QINFO:
-			hclge_get_vf_queue_info(vport, &resp_msg);
-			break;
-		case HCLGE_MBX_GET_QDEPTH:
-			hclge_get_vf_queue_depth(vport, &resp_msg);
-			break;
-		case HCLGE_MBX_GET_BASIC_INFO:
-			hclge_get_basic_info(vport, &resp_msg);
-			break;
-		case HCLGE_MBX_GET_LINK_STATUS:
-			ret = hclge_push_vf_link_status(vport);
-			if (ret)
-				dev_err(&hdev->pdev->dev,
-					"failed to inform link stat to VF, ret = %d\n",
-					ret);
-			break;
-		case HCLGE_MBX_QUEUE_RESET:
-			ret = hclge_mbx_reset_vf_queue(vport, req, &resp_msg);
-			break;
-		case HCLGE_MBX_RESET:
-			ret = hclge_reset_vf(vport);
-			break;
-		case HCLGE_MBX_KEEP_ALIVE:
-			hclge_vf_keep_alive(vport);
-			break;
-		case HCLGE_MBX_SET_MTU:
-			ret = hclge_set_vf_mtu(vport, req);
-			if (ret)
-				dev_err(&hdev->pdev->dev,
-					"VF fail(%d) to set mtu\n", ret);
-			break;
-		case HCLGE_MBX_GET_QID_IN_PF:
-			ret = hclge_get_queue_id_in_pf(vport, req, &resp_msg);
-			break;
-		case HCLGE_MBX_GET_RSS_KEY:
-			ret = hclge_get_rss_key(vport, req, &resp_msg);
-			break;
-		case HCLGE_MBX_GET_LINK_MODE:
-			hclge_get_link_mode(vport, req);
-			break;
-		case HCLGE_MBX_GET_VF_FLR_STATUS:
-		case HCLGE_MBX_VF_UNINIT:
-			is_del = req->msg.code == HCLGE_MBX_VF_UNINIT;
-			hclge_rm_vport_all_mac_table(vport, is_del,
-						     HCLGE_MAC_ADDR_UC);
-			hclge_rm_vport_all_mac_table(vport, is_del,
-						     HCLGE_MAC_ADDR_MC);
-			hclge_rm_vport_all_vlan_table(vport, is_del);
-			break;
-		case HCLGE_MBX_GET_MEDIA_TYPE:
-			hclge_get_vf_media_type(vport, &resp_msg);
-			break;
-		case HCLGE_MBX_PUSH_LINK_STATUS:
-			hclge_handle_link_change_event(hdev, req);
-			break;
-		case HCLGE_MBX_GET_MAC_ADDR:
-			hclge_get_vf_mac_addr(vport, &resp_msg);
-			break;
-		case HCLGE_MBX_NCSI_ERROR:
-			hclge_handle_ncsi_error(hdev);
-			break;
-		case HCLGE_MBX_HANDLE_VF_TBL:
-			hclge_handle_vf_tbl(vport, req);
-			break;
-		default:
-			dev_err(&hdev->pdev->dev,
-				"un-supported mailbox message, code = %u\n",
-				req->msg.code);
-			break;
-		}
-
-		/* PF driver should not reply IMP */
-		if (hnae3_get_bit(req->mbx_need_resp, HCLGE_MBX_NEED_RESP_B) &&
-		    req->msg.code < HCLGE_MBX_GET_VF_FLR_STATUS) {
-			resp_msg.status = ret;
-			if (time_is_before_jiffies(hdev->last_mbx_scheduled +
-						   HCLGE_MBX_SCHED_TIMEOUT))
-				dev_warn(&hdev->pdev->dev,
-					 "resp vport%u mbx(%u,%u) late\n",
-					 req->mbx_src_vfid,
-					 req->msg.code,
-					 req->msg.subcode);
-
-			hclge_gen_resp_to_vf(vport, req, &resp_msg);
-		}
+		param.vport = &hdev->vport[req->mbx_src_vfid];
+		param.req = req;
+		hclge_mbx_request_handling(&param);
 
 		crq->desc[crq->next_to_use].flag = 0;
 		hclge_mbx_ring_ptr_move_crq(crq);
-
-		/* reinitialize ret after complete the mbx message processing */
-		ret = 0;
 	}
 
 	/* Write back CMDQ_RQ header pointer, M7 need this pointer */
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
index 03d63b6a9b2b..85fb11de43a1 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.c
@@ -187,7 +187,7 @@ static void hclge_mac_adjust_link(struct net_device *netdev)
 	speed = netdev->phydev->speed;
 	duplex = netdev->phydev->duplex;
 
-	ret = hclge_cfg_mac_speed_dup(hdev, speed, duplex);
+	ret = hclge_cfg_mac_speed_dup(hdev, speed, duplex, 0);
 	if (ret)
 		netdev_err(netdev, "failed to adjust link.\n");
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
index 2f33b036a47a..4a33f65190e2 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c
@@ -248,7 +248,7 @@ static int hclge_fill_pri_array(struct hclge_dev *hdev, u8 *pri, u8 pri_id)
 	return 0;
 }
 
-static int hclge_up_to_tc_map(struct hclge_dev *hdev)
+int hclge_up_to_tc_map(struct hclge_dev *hdev)
 {
 	struct hclge_desc desc;
 	u8 *pri = (u8 *)desc.data;
@@ -266,6 +266,47 @@ static int hclge_up_to_tc_map(struct hclge_dev *hdev)
 	return hclge_cmd_send(&hdev->hw, &desc, 1);
 }
 
+static void hclge_dscp_to_prio_map_init(struct hclge_dev *hdev)
+{
+	u8 i;
+
+	hdev->vport[0].nic.kinfo.tc_map_mode = HNAE3_TC_MAP_MODE_PRIO;
+	hdev->vport[0].nic.kinfo.dscp_app_cnt = 0;
+	for (i = 0; i < HNAE3_MAX_DSCP; i++)
+		hdev->vport[0].nic.kinfo.dscp_prio[i] = HNAE3_PRIO_ID_INVALID;
+}
+
+int hclge_dscp_to_tc_map(struct hclge_dev *hdev)
+{
+	struct hclge_desc desc[HCLGE_DSCP_MAP_TC_BD_NUM];
+	u8 *req0 = (u8 *)desc[0].data;
+	u8 *req1 = (u8 *)desc[1].data;
+	u8 pri_id, tc_id, i, j;
+
+	hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_QOS_MAP, false);
+	desc[0].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT);
+	hclge_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_QOS_MAP, false);
+
+	/* The low 32 dscp setting use bd0, high 32 dscp setting use bd1 */
+	for (i = 0; i < HNAE3_MAX_DSCP / HCLGE_DSCP_MAP_TC_BD_NUM; i++) {
+		pri_id = hdev->vport[0].nic.kinfo.dscp_prio[i];
+		pri_id = pri_id == HNAE3_PRIO_ID_INVALID ? 0 : pri_id;
+		tc_id = hdev->tm_info.prio_tc[pri_id];
+		/* Each dscp setting has 4 bits, so each byte saves two dscp
+		 * setting
+		 */
+		req0[i >> 1] |= tc_id << HCLGE_DSCP_TC_SHIFT(i);
+
+		j = i + HNAE3_MAX_DSCP / HCLGE_DSCP_MAP_TC_BD_NUM;
+		pri_id = hdev->vport[0].nic.kinfo.dscp_prio[j];
+		pri_id = pri_id == HNAE3_PRIO_ID_INVALID ? 0 : pri_id;
+		tc_id = hdev->tm_info.prio_tc[pri_id];
+		req1[i >> 1] |= tc_id << HCLGE_DSCP_TC_SHIFT(i);
+	}
+
+	return hclge_cmd_send(&hdev->hw, desc, HCLGE_DSCP_MAP_TC_BD_NUM);
+}
+
 static int hclge_tm_pg_to_pri_map_cfg(struct hclge_dev *hdev,
 				      u8 pg_id, u8 pri_bit_map)
 {
@@ -1275,6 +1316,12 @@ static int hclge_tm_map_cfg(struct hclge_dev *hdev)
 	if (ret)
 		return ret;
 
+	if (hdev->vport[0].nic.kinfo.tc_map_mode == HNAE3_TC_MAP_MODE_DSCP) {
+		ret = hclge_dscp_to_tc_map(hdev);
+		if (ret)
+			return ret;
+	}
+
 	ret = hclge_tm_pg_to_pri_map(hdev);
 	if (ret)
 		return ret;
@@ -1646,6 +1693,7 @@ int hclge_tm_schd_init(struct hclge_dev *hdev)
 		return -EINVAL;
 
 	hclge_tm_schd_info_init(hdev);
+	hclge_dscp_to_prio_map_init(hdev);
 
 	return hclge_tm_init_hw(hdev, true);
 }
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
index d943943912f7..68f28a98e380 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h
@@ -30,6 +30,9 @@ enum hclge_opcode_type;
 #define HCLGE_TM_PF_MAX_PRI_NUM		8
 #define HCLGE_TM_PF_MAX_QSET_NUM	8
 
+#define HCLGE_DSCP_MAP_TC_BD_NUM	2
+#define HCLGE_DSCP_TC_SHIFT(n)		(((n) & 1) * 4)
+
 struct hclge_pg_to_pri_link_cmd {
 	u8 pg_id;
 	u8 rsvd1[3];
@@ -262,4 +265,6 @@ int hclge_tm_get_pg_shaper(struct hclge_dev *hdev, u8 pg_id,
 			   struct hclge_tm_shaper_para *para);
 int hclge_tm_get_port_shaper(struct hclge_dev *hdev,
 			     struct hclge_tm_shaper_para *para);
+int hclge_up_to_tc_map(struct hclge_dev *hdev);
+int hclge_dscp_to_tc_map(struct hclge_dev *hdev);
 #endif
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
index 26f87330173e..db6f7cdba958 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
@@ -2125,7 +2125,7 @@ static int hclgevf_config_gro(struct hclgevf_dev *hdev)
 	struct hclge_desc desc;
 	int ret;
 
-	if (!hnae3_dev_gro_supported(hdev))
+	if (!hnae3_ae_dev_gro_supported(hdev->ae_dev))
 		return 0;
 
 	hclgevf_cmd_setup_basic_desc(&desc, HCLGE_OPC_GRO_GENERIC_CONFIG,
@@ -3177,7 +3177,7 @@ static int hclgevf_get_status(struct hnae3_handle *handle)
 
 static void hclgevf_get_ksettings_an_result(struct hnae3_handle *handle,
 					    u8 *auto_neg, u32 *speed,
-					    u8 *duplex)
+					    u8 *duplex, u32 *lane_num)
 {
 	struct hclgevf_dev *hdev = hclgevf_ae_get_hdev(handle);
 
@@ -3429,7 +3429,7 @@ static struct hnae3_ae_algo ae_algovf = {
 	.pdev_id_table = ae_algovf_pci_tbl,
 };
 
-static int hclgevf_init(void)
+static int __init hclgevf_init(void)
 {
 	pr_info("%s is initializing\n", HCLGEVF_NAME);
 
@@ -3444,7 +3444,7 @@ static int hclgevf_init(void)
 	return 0;
 }
 
-static void hclgevf_exit(void)
+static void __exit hclgevf_exit(void)
 {
 	hnae3_unregister_ae_algo(&ae_algovf);
 	destroy_workqueue(hclgevf_wq);
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_debugfs.h b/drivers/net/ethernet/huawei/hinic/hinic_debugfs.h
index e9e00cfa1329..e10f739d8339 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_debugfs.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_debugfs.h
@@ -12,7 +12,6 @@
 #define    TBL_ID_FUNC_CFG_SM_INST                      1
 
 #define HINIC_FUNCTION_CONFIGURE_TABLE_SIZE             64
-#define HINIC_FUNCTION_CONFIGURE_TABLE			1
 
 struct hinic_cmd_lt_rd {
 	u8	status;
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c
index 93192f58ac88..f4b680286911 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c
@@ -55,7 +55,6 @@
 #define COALESCE_ALL_QUEUE		0xFFFF
 #define COALESCE_MAX_PENDING_LIMIT	(255 * COALESCE_PENDING_LIMIT_UNIT)
 #define COALESCE_MAX_TIMER_CFG		(255 * COALESCE_TIMER_CFG_UNIT)
-#define OBJ_STR_MAX_LEN			32
 
 struct hw2ethtool_link_mode {
 	enum ethtool_link_mode_bit_indices link_mode_bit;
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c
index a627237f694b..78190e88cd75 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c
@@ -82,11 +82,6 @@
 						     struct hinic_func_to_io, \
 						     cmdqs)
 
-enum cmdq_wqe_type {
-	WQE_LCMD_TYPE = 0,
-	WQE_SCMD_TYPE = 1,
-};
-
 enum completion_format {
 	COMPLETE_DIRECT = 0,
 	COMPLETE_SGE    = 1,
@@ -509,8 +504,8 @@ int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs,
  *
  * Return 0 - Success, negative - Failure
  **/
-int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs,
-		      enum hinic_set_arm_qtype q_type, u32 q_id)
+static int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs,
+			     enum hinic_set_arm_qtype q_type, u32 q_id)
 {
 	struct hinic_cmdq *cmdq = &cmdqs->cmdq[HINIC_CMDQ_SYNC];
 	struct hinic_hwif *hwif = cmdqs->hwif;
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h
index 9c413e963a04..ff09cf0ed52b 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.h
@@ -177,9 +177,6 @@ int hinic_cmdq_direct_resp(struct hinic_cmdqs *cmdqs,
 			   enum hinic_mod_type mod, u8 cmd,
 			   struct hinic_cmdq_buf *buf_in, u64 *out_param);
 
-int hinic_set_arm_bit(struct hinic_cmdqs *cmdqs,
-		      enum hinic_set_arm_qtype q_type, u32 q_id);
-
 int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif,
 		     void __iomem **db_area);
 
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h
index 7e84e4e33fff..d56e7413ace0 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_csr.h
@@ -22,7 +22,6 @@
 	(HINIC_DMA_ATTR_BASE + (idx) * HINIC_DMA_ATTR_STRIDE)
 
 #define HINIC_PPF_ELECTION_STRIDE                       0x4
-#define HINIC_CSR_MAX_PORTS                             4
 
 #define HINIC_CSR_PPF_ELECTION_ADDR(idx)                \
 	(HINIC_ELECTION_BASE +  (idx) * HINIC_PPF_ELECTION_STRIDE)
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c
index 2127a48749a8..94f470556295 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c
@@ -29,7 +29,6 @@
 #include "hinic_hw_io.h"
 #include "hinic_hw_dev.h"
 
-#define IO_STATUS_TIMEOUT               100
 #define OUTBOUND_STATE_TIMEOUT          100
 #define DB_STATE_TIMEOUT                100
 
@@ -42,11 +41,6 @@ enum intr_type {
 	INTR_MSIX_TYPE,
 };
 
-enum io_status {
-	IO_STOPPED = 0,
-	IO_RUNNING = 1,
-};
-
 /**
  * parse_capability - convert device capabilities to NIC capabilities
  * @hwdev: the HW device to set and convert device capabilities for
@@ -837,8 +831,8 @@ static int hinic_l2nic_reset(struct hinic_hwdev *hwdev)
 	return 0;
 }
 
-int hinic_get_interrupt_cfg(struct hinic_hwdev *hwdev,
-			    struct hinic_msix_config *interrupt_info)
+static int hinic_get_interrupt_cfg(struct hinic_hwdev *hwdev,
+				   struct hinic_msix_config *interrupt_info)
 {
 	u16 out_size = sizeof(*interrupt_info);
 	struct hinic_pfhwdev *pfhwdev;
@@ -1041,13 +1035,6 @@ void hinic_free_hwdev(struct hinic_hwdev *hwdev)
 	hinic_free_hwif(hwdev->hwif);
 }
 
-int hinic_hwdev_max_num_qps(struct hinic_hwdev *hwdev)
-{
-	struct hinic_cap *nic_cap = &hwdev->nic_cap;
-
-	return nic_cap->max_qps;
-}
-
 /**
  * hinic_hwdev_num_qps - return the number QPs available for use
  * @hwdev: the NIC HW device
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h
index 416492e48274..d2d89b0a5ef0 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.h
@@ -566,8 +566,6 @@ struct hinic_hwdev *hinic_init_hwdev(struct pci_dev *pdev, struct devlink *devli
 
 void hinic_free_hwdev(struct hinic_hwdev *hwdev);
 
-int hinic_hwdev_max_num_qps(struct hinic_hwdev *hwdev);
-
 int hinic_hwdev_num_qps(struct hinic_hwdev *hwdev);
 
 struct hinic_sq *hinic_hwdev_get_sq(struct hinic_hwdev *hwdev, int i);
@@ -587,9 +585,6 @@ int hinic_hwdev_hw_ci_addr_set(struct hinic_hwdev *hwdev, struct hinic_sq *sq,
 void hinic_hwdev_set_msix_state(struct hinic_hwdev *hwdev, u16 msix_index,
 				enum hinic_msix_state flag);
 
-int hinic_get_interrupt_cfg(struct hinic_hwdev *hwdev,
-			    struct hinic_msix_config *interrupt_info);
-
 int hinic_set_interrupt_cfg(struct hinic_hwdev *hwdev,
 			    struct hinic_msix_config *interrupt_info);
 
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c
index 0428faa68e80..88567305d06e 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.c
@@ -58,39 +58,6 @@ int hinic_msix_attr_set(struct hinic_hwif *hwif, u16 msix_index,
 }
 
 /**
- * hinic_msix_attr_get - get message attribute of msix entry
- * @hwif: the HW interface of a pci function device
- * @msix_index: msix_index
- * @pending_limit: the maximum pending interrupt events (unit 8)
- * @coalesc_timer: coalesc period for interrupt (unit 8 us)
- * @lli_timer: replenishing period for low latency credit (unit 8 us)
- * @lli_credit_limit: maximum credits for low latency msix messages (unit 8)
- * @resend_timer: maximum wait for resending msix (unit coalesc period)
- *
- * Return 0 - Success, negative - Failure
- **/
-int hinic_msix_attr_get(struct hinic_hwif *hwif, u16 msix_index,
-			u8 *pending_limit, u8 *coalesc_timer,
-			u8 *lli_timer, u8 *lli_credit_limit,
-			u8 *resend_timer)
-{
-	u32 addr, val;
-
-	if (!VALID_MSIX_IDX(&hwif->attr, msix_index))
-		return -EINVAL;
-
-	addr = HINIC_CSR_MSIX_CTRL_ADDR(msix_index);
-	val  = hinic_hwif_read_reg(hwif, addr);
-
-	*pending_limit    = HINIC_MSIX_ATTR_GET(val, PENDING_LIMIT);
-	*coalesc_timer    = HINIC_MSIX_ATTR_GET(val, COALESC_TIMER);
-	*lli_timer        = HINIC_MSIX_ATTR_GET(val, LLI_TIMER);
-	*lli_credit_limit = HINIC_MSIX_ATTR_GET(val, LLI_CREDIT);
-	*resend_timer     = HINIC_MSIX_ATTR_GET(val, RESEND_TIMER);
-	return 0;
-}
-
-/**
  * hinic_msix_attr_cnt_clear - clear message attribute counters for msix entry
  * @hwif: the HW interface of a pci function device
  * @msix_index: msix_index
@@ -115,8 +82,6 @@ int hinic_msix_attr_cnt_clear(struct hinic_hwif *hwif, u16 msix_index)
  * hinic_set_pf_action - set action on pf channel
  * @hwif: the HW interface of a pci function device
  * @action: action on pf channel
- *
- * Return 0 - Success, negative - Failure
  **/
 void hinic_set_pf_action(struct hinic_hwif *hwif, enum hinic_pf_action action)
 {
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h
index c06f2253151e..3d588896a367 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_if.h
@@ -131,10 +131,6 @@
 	(((u32)(val) & HINIC_MSIX_##member##_MASK) <<           \
 	 HINIC_MSIX_##member##_SHIFT)
 
-#define HINIC_MSIX_ATTR_GET(val, member)                        \
-	(((val) >> HINIC_MSIX_##member##_SHIFT) &               \
-	 HINIC_MSIX_##member##_MASK)
-
 #define HINIC_MSIX_CNT_RESEND_TIMER_SHIFT                       29
 
 #define HINIC_MSIX_CNT_RESEND_TIMER_MASK                        0x1
@@ -269,11 +265,6 @@ int hinic_msix_attr_set(struct hinic_hwif *hwif, u16 msix_index,
 			u8 lli_timer_cfg, u8 lli_credit_limit,
 			u8 resend_timer);
 
-int hinic_msix_attr_get(struct hinic_hwif *hwif, u16 msix_index,
-			u8 *pending_limit, u8 *coalesc_timer_cfg,
-			u8 *lli_timer, u8 *lli_credit_limit,
-			u8 *resend_timer);
-
 void hinic_set_msix_state(struct hinic_hwif *hwif, u16 msix_idx,
 			  enum hinic_msix_state flag);
 
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c
index 5078c0c73863..3f9c31d29215 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.c
@@ -117,7 +117,6 @@ enum hinic_mbox_tx_status {
 #define MBOX_WB_STATUS_MASK			0xFF
 #define MBOX_WB_ERROR_CODE_MASK			0xFF00
 #define MBOX_WB_STATUS_FINISHED_SUCCESS		0xFF
-#define MBOX_WB_STATUS_FINISHED_WITH_ERR	0xFE
 #define MBOX_WB_STATUS_NOT_FINISHED		0x00
 
 #define MBOX_STATUS_FINISHED(wb)	\
@@ -130,11 +129,8 @@ enum hinic_mbox_tx_status {
 #define SEQ_ID_START_VAL			0
 #define SEQ_ID_MAX_VAL				42
 
-#define DST_AEQ_IDX_DEFAULT_VAL			0
-#define SRC_AEQ_IDX_DEFAULT_VAL			0
 #define NO_DMA_ATTRIBUTE_VAL			0
 
-#define HINIC_MGMT_RSP_AEQN			0
 #define HINIC_MBOX_RSP_AEQN			2
 #define HINIC_MBOX_RECV_AEQN			0
 
@@ -146,7 +142,6 @@ enum hinic_mbox_tx_status {
 
 #define IS_PF_OR_PPF_SRC(src_func_idx)	((src_func_idx) < HINIC_MAX_PF_FUNCS)
 
-#define MBOX_RESPONSE_ERROR		0x1
 #define MBOX_MSG_ID_MASK		0xFF
 #define MBOX_MSG_ID(func_to_func)	((func_to_func)->send_msg_id)
 #define MBOX_MSG_ID_INC(func_to_func_mbox) (MBOX_MSG_ID(func_to_func_mbox) = \
@@ -621,7 +616,7 @@ static bool check_vf_mbox_random_id(struct hinic_mbox_func_to_func *func_to_func
 	return false;
 }
 
-void hinic_mbox_func_aeqe_handler(void *handle, void *header, u8 size)
+static void hinic_mbox_func_aeqe_handler(void *handle, void *header, u8 size)
 {
 	struct hinic_mbox_func_to_func *func_to_func;
 	u64 mbox_header = *((u64 *)header);
@@ -649,7 +644,7 @@ void hinic_mbox_func_aeqe_handler(void *handle, void *header, u8 size)
 	recv_mbox_handler(func_to_func, (u64 *)header, recv_mbox);
 }
 
-void hinic_mbox_self_aeqe_handler(void *handle, void *header, u8 size)
+static void hinic_mbox_self_aeqe_handler(void *handle, void *header, u8 size)
 {
 	struct hinic_mbox_func_to_func *func_to_func;
 	struct hinic_send_mbox *send_mbox;
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.h
index 46953190d29e..33ac7814d3b3 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_mbox.h
@@ -150,10 +150,6 @@ void hinic_unregister_pf_mbox_cb(struct hinic_hwdev *hwdev,
 void hinic_unregister_vf_mbox_cb(struct hinic_hwdev *hwdev,
 				 enum hinic_mod_type mod);
 
-void hinic_mbox_func_aeqe_handler(void *handle, void *header, u8 size);
-
-void hinic_mbox_self_aeqe_handler(void *handle, void *header, u8 size);
-
 int hinic_func_to_func_init(struct hinic_hwdev *hwdev);
 
 void hinic_func_to_func_free(struct hinic_hwdev *hwdev);
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c
index 336248aa2e48..537a8098bc4e 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c
@@ -472,8 +472,7 @@ int hinic_get_rq_free_wqebbs(struct hinic_rq *rq)
 	return atomic_read(&wq->delta) - 1;
 }
 
-static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, u16 prod_idx,
-			    int nr_descs)
+static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, int nr_descs)
 {
 	u32 ctrl_size, task_size, bufdesc_size;
 
@@ -588,18 +587,16 @@ void hinic_set_tso_inner_l4(struct hinic_sq_task *task, u32 *queue_info,
 /**
  * hinic_sq_prepare_wqe - prepare wqe before insert to the queue
  * @sq: send queue
- * @prod_idx: pi value
  * @sq_wqe: wqe to prepare
  * @sges: sges for use by the wqe for send for buf addresses
  * @nr_sges: number of sges
  **/
-void hinic_sq_prepare_wqe(struct hinic_sq *sq, u16 prod_idx,
-			  struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,
-			  int nr_sges)
+void hinic_sq_prepare_wqe(struct hinic_sq *sq, struct hinic_sq_wqe *sq_wqe,
+			  struct hinic_sge *sges, int nr_sges)
 {
 	int i;
 
-	sq_prepare_ctrl(&sq_wqe->ctrl, prod_idx, nr_sges);
+	sq_prepare_ctrl(&sq_wqe->ctrl, nr_sges);
 
 	sq_prepare_task(&sq_wqe->task);
 
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h
index 0dfa51ad5855..178dcc874370 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_qp.h
@@ -175,9 +175,8 @@ void hinic_set_tso_inner_l4(struct hinic_sq_task *task,
 			    u32 l4_len,
 			    u32 offset, u32 ip_ident, u32 mss);
 
-void hinic_sq_prepare_wqe(struct hinic_sq *sq, u16 prod_idx,
-			  struct hinic_sq_wqe *wqe, struct hinic_sge *sges,
-			  int nr_sges);
+void hinic_sq_prepare_wqe(struct hinic_sq *sq, struct hinic_sq_wqe *wqe,
+			  struct hinic_sge *sges, int nr_sges);
 
 void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size,
 		       unsigned int cos);
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c
index 4daf6bf291ec..e1a1735c00c1 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c
@@ -175,8 +175,6 @@ static int cmdq_allocate_page(struct hinic_cmdq_pages *cmdq_pages)
 /**
  * cmdq_free_page - free page from cmdq
  * @cmdq_pages: the pages of the cmdq queue struct that hold the page
- *
- * Return 0 - Success, negative - Failure
  **/
 static void cmdq_free_page(struct hinic_cmdq_pages *cmdq_pages)
 {
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h b/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h
index f4b6d2c1061f..c6bdeed5606e 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_wqe.h
@@ -261,23 +261,6 @@
 #define HINIC_RSS_TYPE_GET(val, member)                        \
 		(((u32)(val) >> HINIC_RSS_TYPE_##member##_SHIFT) & 0x1)
 
-enum hinic_l4offload_type {
-	HINIC_L4_OFF_DISABLE            = 0,
-	HINIC_TCP_OFFLOAD_ENABLE        = 1,
-	HINIC_SCTP_OFFLOAD_ENABLE       = 2,
-	HINIC_UDP_OFFLOAD_ENABLE        = 3,
-};
-
-enum hinic_vlan_offload {
-	HINIC_VLAN_OFF_DISABLE = 0,
-	HINIC_VLAN_OFF_ENABLE  = 1,
-};
-
-enum hinic_pkt_parsed {
-	HINIC_PKT_NOT_PARSED = 0,
-	HINIC_PKT_PARSED     = 1,
-};
-
 enum hinic_l3_offload_type {
 	L3TYPE_UNKNOWN = 0,
 	IPV6_PKT = 1,
@@ -305,18 +288,10 @@ enum hinic_outer_l3type {
 	HINIC_OUTER_L3TYPE_IPV4_CHKSUM          = 3,
 };
 
-enum hinic_media_type {
-	HINIC_MEDIA_UNKNOWN = 0,
-};
-
 enum hinic_l2type {
 	HINIC_L2TYPE_ETH = 0,
 };
 
-enum hinc_tunnel_l4type {
-	HINIC_TUNNEL_L4TYPE_UNKNOWN = 0,
-};
-
 struct hinic_cmdq_header {
 	u32     header_info;
 	u32     saved_data;
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_main.c b/drivers/net/ethernet/huawei/hinic/hinic_main.c
index c23ee2ddbce3..e1f54a2f28b2 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_main.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_main.c
@@ -960,8 +960,6 @@ static void hinic_refresh_nic_cfg(struct hinic_dev *nic_dev)
  * @in_size: input size
  * @buf_out: output buffer
  * @out_size: returned output size
- *
- * Return 0 - Success, negative - Failure
  **/
 static void link_status_event_handler(void *handle, void *buf_in, u16 in_size,
 				      void *buf_out, u16 *out_size)
@@ -1382,8 +1380,6 @@ err_pci_regions:
 	return err;
 }
 
-#define HINIC_WAIT_SRIOV_CFG_TIMEOUT	15000
-
 static void wait_sriov_cfg_complete(struct hinic_dev *nic_dev)
 {
 	struct hinic_sriov_info *sriov_info = &nic_dev->sriov_info;
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_rx.c b/drivers/net/ethernet/huawei/hinic/hinic_rx.c
index e5828a658caf..d649c6e323c8 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_rx.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_rx.c
@@ -50,7 +50,7 @@
  * hinic_rxq_clean_stats - Clean the statistics of specific queue
  * @rxq: Logical Rx Queue
  **/
-void hinic_rxq_clean_stats(struct hinic_rxq *rxq)
+static void hinic_rxq_clean_stats(struct hinic_rxq *rxq)
 {
 	struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
 
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_rx.h b/drivers/net/ethernet/huawei/hinic/hinic_rx.h
index 507dcbae9085..8f7bd6a049bd 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_rx.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_rx.h
@@ -41,8 +41,6 @@ struct hinic_rxq {
 	struct napi_struct      napi;
 };
 
-void hinic_rxq_clean_stats(struct hinic_rxq *rxq);
-
 void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats);
 
 int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq,
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_sriov.c b/drivers/net/ethernet/huawei/hinic/hinic_sriov.c
index df555847afb5..a5f08b969e3f 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_sriov.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_sriov.c
@@ -24,6 +24,7 @@ MODULE_PARM_DESC(set_vf_link_state, "Set vf link state, 0 represents link auto,
 #define HINIC_VLAN_PRIORITY_SHIFT 13
 #define HINIC_ADD_VLAN_IN_MAC 0x8000
 #define HINIC_TX_RATE_TABLE_FULL 12
+#define HINIC_MAX_QOS 7
 
 static int hinic_set_mac(struct hinic_hwdev *hwdev, const u8 *mac_addr,
 			 u16 vlan_id, u16 func_id)
@@ -774,7 +775,7 @@ int hinic_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
 	u16 vlanprio, cur_vlanprio;
 
 	sriov_info = &nic_dev->sriov_info;
-	if (vf >= sriov_info->num_vfs || vlan > 4095 || qos > 7)
+	if (vf >= sriov_info->num_vfs || vlan >= VLAN_N_VID || qos > HINIC_MAX_QOS)
 		return -EINVAL;
 	if (vlan_proto != htons(ETH_P_8021Q))
 		return -EPROTONOSUPPORT;
@@ -820,7 +821,7 @@ int hinic_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting)
 
 	cur_trust = nic_io->vf_infos[vf].trust;
 	/* same request, so just return success */
-	if ((setting && cur_trust) || (!setting && !cur_trust))
+	if (setting == cur_trust)
 		return 0;
 
 	err = hinic_set_vf_trust(adapter->hwdev, vf, setting);
@@ -940,7 +941,7 @@ int hinic_ndo_set_vf_spoofchk(struct net_device *netdev, int vf, bool setting)
 	cur_spoofchk = nic_dev->hwdev->func_to_io.vf_infos[vf].spoofchk;
 
 	/* same request, so just return success */
-	if ((setting && cur_spoofchk) || (!setting && !cur_spoofchk))
+	if (setting == cur_spoofchk)
 		return 0;
 
 	err = hinic_set_vf_spoofchk(sriov_info->hwdev,
@@ -1131,8 +1132,8 @@ static void hinic_clear_vf_infos(struct hinic_dev *nic_dev, u16 vf_id)
 	hinic_init_vf_infos(&nic_dev->hwdev->func_to_io, HW_VF_ID_TO_OS(vf_id));
 }
 
-static int hinic_deinit_vf_hw(struct hinic_sriov_info *sriov_info,
-			      u16 start_vf_id, u16 end_vf_id)
+static void hinic_deinit_vf_hw(struct hinic_sriov_info *sriov_info,
+			       u16 start_vf_id, u16 end_vf_id)
 {
 	struct hinic_dev *nic_dev;
 	u16 func_idx, idx;
@@ -1145,8 +1146,6 @@ static int hinic_deinit_vf_hw(struct hinic_sriov_info *sriov_info,
 				       HINIC_HW_WQ_PAGE_SIZE);
 		hinic_clear_vf_infos(nic_dev, idx);
 	}
-
-	return 0;
 }
 
 int hinic_vf_func_init(struct hinic_hwdev *hwdev)
@@ -1293,7 +1292,7 @@ int hinic_pci_sriov_disable(struct pci_dev *pdev)
 	return 0;
 }
 
-int hinic_pci_sriov_enable(struct pci_dev *pdev, int num_vfs)
+static int hinic_pci_sriov_enable(struct pci_dev *pdev, int num_vfs)
 {
 	struct hinic_sriov_info *sriov_info;
 	int err;
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_sriov.h b/drivers/net/ethernet/huawei/hinic/hinic_sriov.h
index ba627a362f9a..d4d4e63d31ea 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_sriov.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_sriov.h
@@ -98,8 +98,6 @@ void hinic_notify_all_vfs_link_changed(struct hinic_hwdev *hwdev,
 
 int hinic_pci_sriov_disable(struct pci_dev *dev);
 
-int hinic_pci_sriov_enable(struct pci_dev *dev, int num_vfs);
-
 int hinic_vf_func_init(struct hinic_hwdev *hwdev);
 
 void hinic_vf_func_free(struct hinic_hwdev *hwdev);
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.c b/drivers/net/ethernet/huawei/hinic/hinic_tx.c
index 3b6c7b585737..e91476c8ff8b 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_tx.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.c
@@ -74,7 +74,7 @@ enum hinic_offload_type {
  * hinic_txq_clean_stats - Clean the statistics of specific queue
  * @txq: Logical Tx Queue
  **/
-void hinic_txq_clean_stats(struct hinic_txq *txq)
+static void hinic_txq_clean_stats(struct hinic_txq *txq)
 {
 	struct hinic_txq_stats *txq_stats = &txq->txq_stats;
 
@@ -530,7 +530,7 @@ netdev_tx_t hinic_lb_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
 	}
 
 process_sq_wqe:
-	hinic_sq_prepare_wqe(txq->sq, prod_idx, sq_wqe, txq->sges, nr_sges);
+	hinic_sq_prepare_wqe(txq->sq, sq_wqe, txq->sges, nr_sges);
 	hinic_sq_write_wqe(txq->sq, prod_idx, sq_wqe, skb, wqe_size);
 
 flush_skbs:
@@ -614,7 +614,7 @@ netdev_tx_t hinic_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
 	}
 
 process_sq_wqe:
-	hinic_sq_prepare_wqe(txq->sq, prod_idx, sq_wqe, txq->sges, nr_sges);
+	hinic_sq_prepare_wqe(txq->sq, sq_wqe, txq->sges, nr_sges);
 
 	err = hinic_tx_offload(skb, &sq_wqe->task, &sq_wqe->ctrl.queue_info);
 	if (err)
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.h b/drivers/net/ethernet/huawei/hinic/hinic_tx.h
index b3c8657774a7..91dc778362f3 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_tx.h
+++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.h
@@ -40,8 +40,6 @@ struct hinic_txq {
 	struct napi_struct      napi;
 };
 
-void hinic_txq_clean_stats(struct hinic_txq *txq);
-
 void hinic_txq_get_stats(struct hinic_txq *txq, struct hinic_txq_stats *stats);
 
 netdev_tx_t hinic_lb_xmit_frame(struct sk_buff *skb, struct net_device *netdev);
diff --git a/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c b/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c
index 6cb86032ce46..1db5b6790a41 100644
--- a/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c
+++ b/drivers/net/ethernet/ibm/ehea/ehea_ethtool.c
@@ -159,8 +159,8 @@ static int ehea_nway_reset(struct net_device *dev)
 static void ehea_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 }
 
 static u32 ehea_get_msglevel(struct net_device *dev)
diff --git a/drivers/net/ethernet/ibm/ehea/ehea_main.c b/drivers/net/ethernet/ibm/ehea/ehea_main.c
index 5dc302880f5f..294bdbbeacc3 100644
--- a/drivers/net/ethernet/ibm/ehea/ehea_main.c
+++ b/drivers/net/ethernet/ibm/ehea/ehea_main.c
@@ -1546,7 +1546,7 @@ static int ehea_init_port_res(struct ehea_port *port, struct ehea_port_res *pr,
 
 	kfree(init_attr);
 
-	netif_napi_add(pr->port->netdev, &pr->napi, ehea_poll, 64);
+	netif_napi_add(pr->port->netdev, &pr->napi, ehea_poll);
 
 	ret = 0;
 	goto out;
diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c
index fbea9f7efe8c..9b08e41ccc29 100644
--- a/drivers/net/ethernet/ibm/emac/core.c
+++ b/drivers/net/ethernet/ibm/emac/core.c
@@ -2284,8 +2284,8 @@ static void emac_ethtool_get_drvinfo(struct net_device *ndev,
 {
 	struct emac_instance *dev = netdev_priv(ndev);
 
-	strlcpy(info->driver, "ibm_emac", sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, "ibm_emac", sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 	snprintf(info->bus_info, sizeof(info->bus_info), "PPC 4xx EMAC-%d %pOF",
 		 dev->cell_index, dev->ofdev->dev.of_node);
 }
@@ -2979,11 +2979,9 @@ static int emac_init_config(struct emac_instance *dev)
 
 	/* Read MAC-address */
 	err = of_get_ethdev_address(np, dev->ndev);
-	if (err) {
-		if (err != -EPROBE_DEFER)
-			dev_err(&dev->ofdev->dev, "Can't get valid [local-]mac-address from OF !\n");
-		return err;
-	}
+	if (err)
+		return dev_err_probe(&dev->ofdev->dev, err,
+				     "Can't get valid [local-]mac-address from OF !\n");
 
 	/* IAHT and GAHT filter parameterization */
 	if (emac_has_feature(dev, EMAC_FTR_EMAC4SYNC)) {
diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c
index 5c6a04d29f5b..3b14dc93f59d 100644
--- a/drivers/net/ethernet/ibm/ibmveth.c
+++ b/drivers/net/ethernet/ibm/ibmveth.c
@@ -141,6 +141,13 @@ static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter)
 	return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_CSUM_GOOD;
 }
 
+static unsigned int ibmveth_real_max_tx_queues(void)
+{
+	unsigned int n_cpu = num_online_cpus();
+
+	return min(n_cpu, IBMVETH_MAX_QUEUES);
+}
+
 /* setup the initial settings for a buffer pool */
 static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool *pool,
 				     u32 pool_index, u32 pool_size,
@@ -456,6 +463,38 @@ static void ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter)
 	}
 }
 
+static void ibmveth_free_tx_ltb(struct ibmveth_adapter *adapter, int idx)
+{
+	dma_unmap_single(&adapter->vdev->dev, adapter->tx_ltb_dma[idx],
+			 adapter->tx_ltb_size, DMA_TO_DEVICE);
+	kfree(adapter->tx_ltb_ptr[idx]);
+	adapter->tx_ltb_ptr[idx] = NULL;
+}
+
+static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter *adapter, int idx)
+{
+	adapter->tx_ltb_ptr[idx] = kzalloc(adapter->tx_ltb_size,
+					   GFP_KERNEL);
+	if (!adapter->tx_ltb_ptr[idx]) {
+		netdev_err(adapter->netdev,
+			   "unable to allocate tx long term buffer\n");
+		return -ENOMEM;
+	}
+	adapter->tx_ltb_dma[idx] = dma_map_single(&adapter->vdev->dev,
+						  adapter->tx_ltb_ptr[idx],
+						  adapter->tx_ltb_size,
+						  DMA_TO_DEVICE);
+	if (dma_mapping_error(&adapter->vdev->dev, adapter->tx_ltb_dma[idx])) {
+		netdev_err(adapter->netdev,
+			   "unable to DMA map tx long term buffer\n");
+		kfree(adapter->tx_ltb_ptr[idx]);
+		adapter->tx_ltb_ptr[idx] = NULL;
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
 static int ibmveth_register_logical_lan(struct ibmveth_adapter *adapter,
         union ibmveth_buf_desc rxq_desc, u64 mac_address)
 {
@@ -538,6 +577,11 @@ static int ibmveth_open(struct net_device *netdev)
 		goto out_unmap_buffer_list;
 	}
 
+	for (i = 0; i < netdev->real_num_tx_queues; i++) {
+		if (ibmveth_allocate_tx_ltb(adapter, i))
+			goto out_free_tx_ltb;
+	}
+
 	adapter->rx_queue.index = 0;
 	adapter->rx_queue.num_slots = rxq_entries;
 	adapter->rx_queue.toggle = 1;
@@ -595,25 +639,15 @@ static int ibmveth_open(struct net_device *netdev)
 
 	rc = -ENOMEM;
 
-	adapter->bounce_buffer = dma_alloc_coherent(&adapter->vdev->dev,
-						    netdev->mtu + IBMVETH_BUFF_OH,
-						    &adapter->bounce_buffer_dma, GFP_KERNEL);
-	if (!adapter->bounce_buffer) {
-		netdev_err(netdev, "unable to alloc bounce buffer\n");
-		goto out_free_irq;
-	}
-
 	netdev_dbg(netdev, "initial replenish cycle\n");
 	ibmveth_interrupt(netdev->irq, netdev);
 
-	netif_start_queue(netdev);
+	netif_tx_start_all_queues(netdev);
 
 	netdev_dbg(netdev, "open complete\n");
 
 	return 0;
 
-out_free_irq:
-	free_irq(netdev->irq, netdev);
 out_free_buffer_pools:
 	while (--i >= 0) {
 		if (adapter->rx_buff_pool[i].active)
@@ -623,6 +657,12 @@ out_free_buffer_pools:
 out_unmap_filter_list:
 	dma_unmap_single(dev, adapter->filter_list_dma, 4096,
 			 DMA_BIDIRECTIONAL);
+
+out_free_tx_ltb:
+	while (--i >= 0) {
+		ibmveth_free_tx_ltb(adapter, i);
+	}
+
 out_unmap_buffer_list:
 	dma_unmap_single(dev, adapter->buffer_list_dma, 4096,
 			 DMA_BIDIRECTIONAL);
@@ -651,7 +691,7 @@ static int ibmveth_close(struct net_device *netdev)
 	napi_disable(&adapter->napi);
 
 	if (!adapter->pool_config)
-		netif_stop_queue(netdev);
+		netif_tx_stop_all_queues(netdev);
 
 	h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE);
 
@@ -685,9 +725,8 @@ static int ibmveth_close(struct net_device *netdev)
 			ibmveth_free_buffer_pool(adapter,
 						 &adapter->rx_buff_pool[i]);
 
-	dma_free_coherent(&adapter->vdev->dev,
-			  adapter->netdev->mtu + IBMVETH_BUFF_OH,
-			  adapter->bounce_buffer, adapter->bounce_buffer_dma);
+	for (i = 0; i < netdev->real_num_tx_queues; i++)
+		ibmveth_free_tx_ltb(adapter, i);
 
 	netdev_dbg(netdev, "close complete\n");
 
@@ -727,8 +766,8 @@ static void ibmveth_init_link_settings(struct net_device *dev)
 static void netdev_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, ibmveth_driver_name, sizeof(info->driver));
-	strlcpy(info->version, ibmveth_driver_version, sizeof(info->version));
+	strscpy(info->driver, ibmveth_driver_name, sizeof(info->driver));
+	strscpy(info->version, ibmveth_driver_version, sizeof(info->version));
 }
 
 static netdev_features_t ibmveth_fix_features(struct net_device *dev,
@@ -953,6 +992,69 @@ static void ibmveth_get_ethtool_stats(struct net_device *dev,
 		data[i] = IBMVETH_GET_STAT(adapter, ibmveth_stats[i].offset);
 }
 
+static void ibmveth_get_channels(struct net_device *netdev,
+				 struct ethtool_channels *channels)
+{
+	channels->max_tx = ibmveth_real_max_tx_queues();
+	channels->tx_count = netdev->real_num_tx_queues;
+
+	channels->max_rx = netdev->real_num_rx_queues;
+	channels->rx_count = netdev->real_num_rx_queues;
+}
+
+static int ibmveth_set_channels(struct net_device *netdev,
+				struct ethtool_channels *channels)
+{
+	struct ibmveth_adapter *adapter = netdev_priv(netdev);
+	unsigned int old = netdev->real_num_tx_queues,
+		     goal = channels->tx_count;
+	int rc, i;
+
+	/* If ndo_open has not been called yet then don't allocate, just set
+	 * desired netdev_queue's and return
+	 */
+	if (!(netdev->flags & IFF_UP))
+		return netif_set_real_num_tx_queues(netdev, goal);
+
+	/* We have IBMVETH_MAX_QUEUES netdev_queue's allocated
+	 * but we may need to alloc/free the ltb's.
+	 */
+	netif_tx_stop_all_queues(netdev);
+
+	/* Allocate any queue that we need */
+	for (i = old; i < goal; i++) {
+		if (adapter->tx_ltb_ptr[i])
+			continue;
+
+		rc = ibmveth_allocate_tx_ltb(adapter, i);
+		if (!rc)
+			continue;
+
+		/* if something goes wrong, free everything we just allocated */
+		netdev_err(netdev, "Failed to allocate more tx queues, returning to %d queues\n",
+			   old);
+		goal = old;
+		old = i;
+		break;
+	}
+	rc = netif_set_real_num_tx_queues(netdev, goal);
+	if (rc) {
+		netdev_err(netdev, "Failed to set real tx queues, returning to %d queues\n",
+			   old);
+		goal = old;
+		old = i;
+	}
+	/* Free any that are no longer needed */
+	for (i = old; i > goal; i--) {
+		if (adapter->tx_ltb_ptr[i - 1])
+			ibmveth_free_tx_ltb(adapter, i - 1);
+	}
+
+	netif_tx_wake_all_queues(netdev);
+
+	return rc;
+}
+
 static const struct ethtool_ops netdev_ethtool_ops = {
 	.get_drvinfo		         = netdev_get_drvinfo,
 	.get_link		         = ethtool_op_get_link,
@@ -961,6 +1063,8 @@ static const struct ethtool_ops netdev_ethtool_ops = {
 	.get_ethtool_stats	         = ibmveth_get_ethtool_stats,
 	.get_link_ksettings	         = ibmveth_get_link_ksettings,
 	.set_link_ksettings              = ibmveth_set_link_ksettings,
+	.get_channels			 = ibmveth_get_channels,
+	.set_channels			 = ibmveth_set_channels
 };
 
 static int ibmveth_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
@@ -969,7 +1073,7 @@ static int ibmveth_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
 }
 
 static int ibmveth_send(struct ibmveth_adapter *adapter,
-			union ibmveth_buf_desc *descs, unsigned long mss)
+			unsigned long desc, unsigned long mss)
 {
 	unsigned long correlator;
 	unsigned int retry_count;
@@ -982,12 +1086,9 @@ static int ibmveth_send(struct ibmveth_adapter *adapter,
 	retry_count = 1024;
 	correlator = 0;
 	do {
-		ret = h_send_logical_lan(adapter->vdev->unit_address,
-					     descs[0].desc, descs[1].desc,
-					     descs[2].desc, descs[3].desc,
-					     descs[4].desc, descs[5].desc,
-					     correlator, &correlator, mss,
-					     adapter->fw_large_send_support);
+		ret = h_send_logical_lan(adapter->vdev->unit_address, desc,
+					 correlator, &correlator, mss,
+					 adapter->fw_large_send_support);
 	} while ((ret == H_BUSY) && (retry_count--));
 
 	if (ret != H_SUCCESS && ret != H_DROPPED) {
@@ -1020,34 +1121,13 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 				      struct net_device *netdev)
 {
 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
-	unsigned int desc_flags;
-	union ibmveth_buf_desc descs[6];
-	int last, i;
-	int force_bounce = 0;
-	dma_addr_t dma_addr;
+	unsigned int desc_flags, total_bytes;
+	union ibmveth_buf_desc desc;
+	int i, queue_num = skb_get_queue_mapping(skb);
 	unsigned long mss = 0;
 
 	if (ibmveth_is_packet_unsupported(skb, netdev))
 		goto out;
-
-	/* veth doesn't handle frag_list, so linearize the skb.
-	 * When GRO is enabled SKB's can have frag_list.
-	 */
-	if (adapter->is_active_trunk &&
-	    skb_has_frag_list(skb) && __skb_linearize(skb)) {
-		netdev->stats.tx_dropped++;
-		goto out;
-	}
-
-	/*
-	 * veth handles a maximum of 6 segments including the header, so
-	 * we have to linearize the skb if there are more than this.
-	 */
-	if (skb_shinfo(skb)->nr_frags > 5 && __skb_linearize(skb)) {
-		netdev->stats.tx_dropped++;
-		goto out;
-	}
-
 	/* veth can't checksum offload UDP */
 	if (skb->ip_summed == CHECKSUM_PARTIAL &&
 	    ((skb->protocol == htons(ETH_P_IP) &&
@@ -1077,56 +1157,6 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 			desc_flags |= IBMVETH_BUF_LRG_SND;
 	}
 
-retry_bounce:
-	memset(descs, 0, sizeof(descs));
-
-	/*
-	 * If a linear packet is below the rx threshold then
-	 * copy it into the static bounce buffer. This avoids the
-	 * cost of a TCE insert and remove.
-	 */
-	if (force_bounce || (!skb_is_nonlinear(skb) &&
-				(skb->len < tx_copybreak))) {
-		skb_copy_from_linear_data(skb, adapter->bounce_buffer,
-					  skb->len);
-
-		descs[0].fields.flags_len = desc_flags | skb->len;
-		descs[0].fields.address = adapter->bounce_buffer_dma;
-
-		if (ibmveth_send(adapter, descs, 0)) {
-			adapter->tx_send_failed++;
-			netdev->stats.tx_dropped++;
-		} else {
-			netdev->stats.tx_packets++;
-			netdev->stats.tx_bytes += skb->len;
-		}
-
-		goto out;
-	}
-
-	/* Map the header */
-	dma_addr = dma_map_single(&adapter->vdev->dev, skb->data,
-				  skb_headlen(skb), DMA_TO_DEVICE);
-	if (dma_mapping_error(&adapter->vdev->dev, dma_addr))
-		goto map_failed;
-
-	descs[0].fields.flags_len = desc_flags | skb_headlen(skb);
-	descs[0].fields.address = dma_addr;
-
-	/* Map the frags */
-	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
-		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
-
-		dma_addr = skb_frag_dma_map(&adapter->vdev->dev, frag, 0,
-					    skb_frag_size(frag), DMA_TO_DEVICE);
-
-		if (dma_mapping_error(&adapter->vdev->dev, dma_addr))
-			goto map_failed_frags;
-
-		descs[i+1].fields.flags_len = desc_flags | skb_frag_size(frag);
-		descs[i+1].fields.address = dma_addr;
-	}
-
 	if (skb->ip_summed == CHECKSUM_PARTIAL && skb_is_gso(skb)) {
 		if (adapter->fw_large_send_support) {
 			mss = (unsigned long)skb_shinfo(skb)->gso_size;
@@ -1143,7 +1173,36 @@ retry_bounce:
 		}
 	}
 
-	if (ibmveth_send(adapter, descs, mss)) {
+	/* Copy header into mapped buffer */
+	if (unlikely(skb->len > adapter->tx_ltb_size)) {
+		netdev_err(adapter->netdev, "tx: packet size (%u) exceeds ltb (%u)\n",
+			   skb->len, adapter->tx_ltb_size);
+		netdev->stats.tx_dropped++;
+		goto out;
+	}
+	memcpy(adapter->tx_ltb_ptr[queue_num], skb->data, skb_headlen(skb));
+	total_bytes = skb_headlen(skb);
+	/* Copy frags into mapped buffers */
+	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+
+		memcpy(adapter->tx_ltb_ptr[queue_num] + total_bytes,
+		       skb_frag_address_safe(frag), skb_frag_size(frag));
+		total_bytes += skb_frag_size(frag);
+	}
+
+	if (unlikely(total_bytes != skb->len)) {
+		netdev_err(adapter->netdev, "tx: incorrect packet len copied into ltb (%u != %u)\n",
+			   skb->len, total_bytes);
+		netdev->stats.tx_dropped++;
+		goto out;
+	}
+	desc.fields.flags_len = desc_flags | skb->len;
+	desc.fields.address = adapter->tx_ltb_dma[queue_num];
+	/* finish writing to long_term_buff before VIOS accessing it */
+	dma_wmb();
+
+	if (ibmveth_send(adapter, desc.desc, mss)) {
 		adapter->tx_send_failed++;
 		netdev->stats.tx_dropped++;
 	} else {
@@ -1151,41 +1210,11 @@ retry_bounce:
 		netdev->stats.tx_bytes += skb->len;
 	}
 
-	dma_unmap_single(&adapter->vdev->dev,
-			 descs[0].fields.address,
-			 descs[0].fields.flags_len & IBMVETH_BUF_LEN_MASK,
-			 DMA_TO_DEVICE);
-
-	for (i = 1; i < skb_shinfo(skb)->nr_frags + 1; i++)
-		dma_unmap_page(&adapter->vdev->dev, descs[i].fields.address,
-			       descs[i].fields.flags_len & IBMVETH_BUF_LEN_MASK,
-			       DMA_TO_DEVICE);
-
 out:
 	dev_consume_skb_any(skb);
 	return NETDEV_TX_OK;
 
-map_failed_frags:
-	last = i+1;
-	for (i = 1; i < last; i++)
-		dma_unmap_page(&adapter->vdev->dev, descs[i].fields.address,
-			       descs[i].fields.flags_len & IBMVETH_BUF_LEN_MASK,
-			       DMA_TO_DEVICE);
 
-	dma_unmap_single(&adapter->vdev->dev,
-			 descs[0].fields.address,
-			 descs[0].fields.flags_len & IBMVETH_BUF_LEN_MASK,
-			 DMA_TO_DEVICE);
-map_failed:
-	if (!firmware_has_feature(FW_FEATURE_CMO))
-		netdev_err(netdev, "tx: unable to map xmit buffer\n");
-	adapter->tx_map_failed++;
-	if (skb_linearize(skb)) {
-		netdev->stats.tx_dropped++;
-		goto out;
-	}
-	force_bounce = 1;
-	goto retry_bounce;
 }
 
 static void ibmveth_rx_mss_helper(struct sk_buff *skb, u16 mss, int lrg_pkt)
@@ -1568,6 +1597,8 @@ static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev)
 
 	ret = IBMVETH_BUFF_LIST_SIZE + IBMVETH_FILT_LIST_SIZE;
 	ret += IOMMU_PAGE_ALIGN(netdev->mtu, tbl);
+	/* add size of mapped tx buffers */
+	ret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl);
 
 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
 		/* add the size of the active receive buffers */
@@ -1660,8 +1691,7 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
 		return -EINVAL;
 	}
 
-	netdev = alloc_etherdev(sizeof(struct ibmveth_adapter));
-
+	netdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter), IBMVETH_MAX_QUEUES, 1);
 	if (!netdev)
 		return -ENOMEM;
 
@@ -1727,6 +1757,17 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
 			kobject_uevent(kobj, KOBJ_ADD);
 	}
 
+	rc = netif_set_real_num_tx_queues(netdev, ibmveth_real_max_tx_queues());
+	if (rc) {
+		netdev_dbg(netdev, "failed to set number of tx queues rc=%d\n",
+			   rc);
+		free_netdev(netdev);
+		return rc;
+	}
+	adapter->tx_ltb_size = PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE);
+	for (i = 0; i < IBMVETH_MAX_QUEUES; i++)
+		adapter->tx_ltb_ptr[i] = NULL;
+
 	netdev_dbg(netdev, "adapter @ 0x%p\n", adapter);
 	netdev_dbg(netdev, "registering netdev...\n");
 
diff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h
index 27dfff200166..daf6f615c03f 100644
--- a/drivers/net/ethernet/ibm/ibmveth.h
+++ b/drivers/net/ethernet/ibm/ibmveth.h
@@ -46,23 +46,23 @@
 #define h_add_logical_lan_buffer(ua, buf) \
   plpar_hcall_norets(H_ADD_LOGICAL_LAN_BUFFER, ua, buf)
 
+/* FW allows us to send 6 descriptors but we only use one so mark
+ * the other 5 as unused (0)
+ */
 static inline long h_send_logical_lan(unsigned long unit_address,
-		unsigned long desc1, unsigned long desc2, unsigned long desc3,
-		unsigned long desc4, unsigned long desc5, unsigned long desc6,
-		unsigned long corellator_in, unsigned long *corellator_out,
-		unsigned long mss, unsigned long large_send_support)
+		unsigned long desc, unsigned long corellator_in,
+		unsigned long *corellator_out, unsigned long mss,
+		unsigned long large_send_support)
 {
 	long rc;
 	unsigned long retbuf[PLPAR_HCALL9_BUFSIZE];
 
 	if (large_send_support)
 		rc = plpar_hcall9(H_SEND_LOGICAL_LAN, retbuf, unit_address,
-				  desc1, desc2, desc3, desc4, desc5, desc6,
-				  corellator_in, mss);
+				  desc, 0, 0, 0, 0, 0, corellator_in, mss);
 	else
 		rc = plpar_hcall9(H_SEND_LOGICAL_LAN, retbuf, unit_address,
-				  desc1, desc2, desc3, desc4, desc5, desc6,
-				  corellator_in);
+				  desc, 0, 0, 0, 0, 0, corellator_in);
 
 	*corellator_out = retbuf[0];
 
@@ -98,6 +98,8 @@ static inline long h_illan_attributes(unsigned long unit_address,
 #define IBMVETH_BUFF_LIST_SIZE 4096
 #define IBMVETH_FILT_LIST_SIZE 4096
 #define IBMVETH_MAX_BUF_SIZE (1024 * 128)
+#define IBMVETH_MAX_TX_BUF_SIZE (1024 * 64)
+#define IBMVETH_MAX_QUEUES 16U
 
 static int pool_size[] = { 512, 1024 * 2, 1024 * 16, 1024 * 32, 1024 * 64 };
 static int pool_count[] = { 256, 512, 256, 256, 256 };
@@ -137,6 +139,9 @@ struct ibmveth_adapter {
     unsigned int mcastFilterSize;
     void * buffer_list_addr;
     void * filter_list_addr;
+    void *tx_ltb_ptr[IBMVETH_MAX_QUEUES];
+    unsigned int tx_ltb_size;
+    dma_addr_t tx_ltb_dma[IBMVETH_MAX_QUEUES];
     dma_addr_t buffer_list_dma;
     dma_addr_t filter_list_dma;
     struct ibmveth_buff_pool rx_buff_pool[IBMVETH_NUM_BUFF_POOLS];
@@ -145,8 +150,6 @@ struct ibmveth_adapter {
     int rx_csum;
     int large_send;
     bool is_active_trunk;
-    void *bounce_buffer;
-    dma_addr_t bounce_buffer_dma;
 
     u64 fw_ipv6_csum_support;
     u64 fw_ipv4_csum_support;
diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index 5ab7c0f81e9a..65dbfbec487a 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -1262,7 +1262,7 @@ static int init_napi(struct ibmvnic_adapter *adapter)
 	for (i = 0; i < adapter->req_rx_queues; i++) {
 		netdev_dbg(adapter->netdev, "Adding napi[%d]\n", i);
 		netif_napi_add(adapter->netdev, &adapter->napi[i],
-			       ibmvnic_poll, NAPI_POLL_WEIGHT);
+			       ibmvnic_poll);
 	}
 
 	adapter->num_active_rx_napi = adapter->req_rx_queues;
diff --git a/drivers/net/ethernet/intel/e100.c b/drivers/net/ethernet/intel/e100.c
index 11a884aa5082..560d1d442232 100644
--- a/drivers/net/ethernet/intel/e100.c
+++ b/drivers/net/ethernet/intel/e100.c
@@ -2431,8 +2431,8 @@ static void e100_get_drvinfo(struct net_device *netdev,
 	struct ethtool_drvinfo *info)
 {
 	struct nic *nic = netdev_priv(netdev);
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(nic->pdev),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(nic->pdev),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c
index 32803b0cf1e8..d06d29c6c037 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c
+++ b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c
@@ -531,10 +531,10 @@ static void e1000_get_drvinfo(struct net_device *netdev,
 {
 	struct e1000_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver,  e1000_driver_name,
+	strscpy(drvinfo->driver,  e1000_driver_name,
 		sizeof(drvinfo->driver));
 
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c
index 23299fc56199..61e60e4de600 100644
--- a/drivers/net/ethernet/intel/e1000/e1000_main.c
+++ b/drivers/net/ethernet/intel/e1000/e1000_main.c
@@ -1012,7 +1012,7 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	netdev->netdev_ops = &e1000_netdev_ops;
 	e1000_set_ethtool_ops(netdev);
 	netdev->watchdog_timeo = 5 * HZ;
-	netif_napi_add(netdev, &adapter->napi, e1000_clean, 64);
+	netif_napi_add(netdev, &adapter->napi, e1000_clean);
 
 	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
 
diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c
index b80ae9a82224..51a5afe9df2f 100644
--- a/drivers/net/ethernet/intel/e1000e/ethtool.c
+++ b/drivers/net/ethernet/intel/e1000e/ethtool.c
@@ -639,7 +639,7 @@ static void e1000_get_drvinfo(struct net_device *netdev,
 {
 	struct e1000_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, e1000e_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->driver, e1000e_driver_name, sizeof(drvinfo->driver));
 
 	/* EEPROM image version # is reported as firmware version # for
 	 * PCI-E controllers
@@ -650,7 +650,7 @@ static void e1000_get_drvinfo(struct net_device *netdev,
 		 (adapter->eeprom_vers & 0x0FF0) >> 4,
 		 (adapter->eeprom_vers & 0x000F));
 
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c
index 321f2a95ae3a..49e926959ad3 100644
--- a/drivers/net/ethernet/intel/e1000e/netdev.c
+++ b/drivers/net/ethernet/intel/e1000e/netdev.c
@@ -7267,7 +7267,7 @@ static void e1000_print_device_info(struct e1000_adapter *adapter)
 	ret_val = e1000_read_pba_string_generic(hw, pba_str,
 						E1000_PBANUM_LENGTH);
 	if (ret_val)
-		strlcpy((char *)pba_str, "Unknown", sizeof(pba_str));
+		strscpy((char *)pba_str, "Unknown", sizeof(pba_str));
 	e_info("MAC: %d, PHY: %d, PBA No: %s\n",
 	       hw->mac.type, hw->phy.type, pba_str);
 }
@@ -7479,8 +7479,8 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	netdev->netdev_ops = &e1000e_netdev_ops;
 	e1000e_set_ethtool_ops(netdev);
 	netdev->watchdog_timeo = 5 * HZ;
-	netif_napi_add(netdev, &adapter->napi, e1000e_poll, 64);
-	strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
+	netif_napi_add(netdev, &adapter->napi, e1000e_poll);
+	strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
 
 	netdev->mem_start = mmio_start;
 	netdev->mem_end = mmio_start + mmio_len;
@@ -7676,7 +7676,7 @@ static int e1000_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (hw->mac.type >= e1000_pch_cnp)
 		adapter->flags2 |= FLAG2_ENABLE_S0IX_FLOWS;
 
-	strlcpy(netdev->name, "eth%d", sizeof(netdev->name));
+	strscpy(netdev->name, "eth%d", sizeof(netdev->name));
 	err = register_netdev(netdev);
 	if (err)
 		goto err_register;
diff --git a/drivers/net/ethernet/intel/e1000e/phy.c b/drivers/net/ethernet/intel/e1000e/phy.c
index fd07c3679bb1..060b263348ce 100644
--- a/drivers/net/ethernet/intel/e1000e/phy.c
+++ b/drivers/net/ethernet/intel/e1000e/phy.c
@@ -2697,9 +2697,14 @@ static s32 e1000_access_phy_wakeup_reg_bm(struct e1000_hw *hw, u32 offset,
 void e1000_power_up_phy_copper(struct e1000_hw *hw)
 {
 	u16 mii_reg = 0;
+	int ret;
 
 	/* The PHY will retain its settings across a power down/up cycle */
-	e1e_rphy(hw, MII_BMCR, &mii_reg);
+	ret = e1e_rphy(hw, MII_BMCR, &mii_reg);
+	if (ret) {
+		e_dbg("Error reading PHY register\n");
+		return;
+	}
 	mii_reg &= ~BMCR_PDOWN;
 	e1e_wphy(hw, MII_BMCR, mii_reg);
 }
@@ -2715,9 +2720,14 @@ void e1000_power_up_phy_copper(struct e1000_hw *hw)
 void e1000_power_down_phy_copper(struct e1000_hw *hw)
 {
 	u16 mii_reg = 0;
+	int ret;
 
 	/* The PHY will retain its settings across a power down/up cycle */
-	e1e_rphy(hw, MII_BMCR, &mii_reg);
+	ret = e1e_rphy(hw, MII_BMCR, &mii_reg);
+	if (ret) {
+		e_dbg("Error reading PHY register\n");
+		return;
+	}
 	mii_reg |= BMCR_PDOWN;
 	e1e_wphy(hw, MII_BMCR, mii_reg);
 	usleep_range(1000, 2000);
@@ -3037,7 +3047,11 @@ s32 e1000_link_stall_workaround_hv(struct e1000_hw *hw)
 		return 0;
 
 	/* Do not apply workaround if in PHY loopback bit 14 set */
-	e1e_rphy(hw, MII_BMCR, &data);
+	ret_val = e1e_rphy(hw, MII_BMCR, &data);
+	if (ret_val) {
+		e_dbg("Error reading PHY register\n");
+		return ret_val;
+	}
 	if (data & BMCR_LOOPBACK)
 		return 0;
 
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_main.c b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
index 3362f26d7f99..4a6630586ec9 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_main.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_main.c
@@ -1595,8 +1595,7 @@ static int fm10k_alloc_q_vector(struct fm10k_intfc *interface,
 		return -ENOMEM;
 
 	/* initialize NAPI */
-	netif_napi_add(interface->netdev, &q_vector->napi,
-		       fm10k_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(interface->netdev, &q_vector->napi, fm10k_poll);
 
 	/* tie q_vector and interface together */
 	interface->q_vector[v_idx] = q_vector;
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h
index d86b6d349ea9..9a60d6b207f7 100644
--- a/drivers/net/ethernet/intel/i40e/i40e.h
+++ b/drivers/net/ethernet/intel/i40e/i40e.h
@@ -399,6 +399,20 @@ struct i40e_ddp_old_profile_list {
 				 I40E_FLEX_54_MASK | I40E_FLEX_55_MASK | \
 				 I40E_FLEX_56_MASK | I40E_FLEX_57_MASK)
 
+#define I40E_QINT_TQCTL_VAL(qp, vector, nextq_type) \
+	(I40E_QINT_TQCTL_CAUSE_ENA_MASK | \
+	(I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | \
+	((vector) << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | \
+	((qp) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | \
+	(I40E_QUEUE_TYPE_##nextq_type << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT))
+
+#define I40E_QINT_RQCTL_VAL(qp, vector, nextq_type) \
+	(I40E_QINT_RQCTL_CAUSE_ENA_MASK | \
+	(I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | \
+	((vector) << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | \
+	((qp) << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | \
+	(I40E_QUEUE_TYPE_##nextq_type << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT))
+
 struct i40e_flex_pit {
 	struct list_head list;
 	u16 src_offset;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_common.c b/drivers/net/ethernet/intel/i40e/i40e_common.c
index 2819e261a126..4f01e2a6b6bb 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_common.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_common.c
@@ -27,6 +27,7 @@ i40e_status i40e_set_mac_type(struct i40e_hw *hw)
 		case I40E_DEV_ID_QSFP_A:
 		case I40E_DEV_ID_QSFP_B:
 		case I40E_DEV_ID_QSFP_C:
+		case I40E_DEV_ID_1G_BASE_T_BC:
 		case I40E_DEV_ID_5G_BASE_T_BC:
 		case I40E_DEV_ID_10G_BASE_T:
 		case I40E_DEV_ID_10G_BASE_T4:
@@ -4974,6 +4975,7 @@ i40e_status i40e_write_phy_register(struct i40e_hw *hw,
 		status = i40e_write_phy_register_clause22(hw, reg, phy_addr,
 							  value);
 		break;
+	case I40E_DEV_ID_1G_BASE_T_BC:
 	case I40E_DEV_ID_5G_BASE_T_BC:
 	case I40E_DEV_ID_10G_BASE_T:
 	case I40E_DEV_ID_10G_BASE_T4:
@@ -5012,6 +5014,7 @@ i40e_status i40e_read_phy_register(struct i40e_hw *hw,
 		status = i40e_read_phy_register_clause22(hw, reg, phy_addr,
 							 value);
 		break;
+	case I40E_DEV_ID_1G_BASE_T_BC:
 	case I40E_DEV_ID_5G_BASE_T_BC:
 	case I40E_DEV_ID_10G_BASE_T:
 	case I40E_DEV_ID_10G_BASE_T4:
diff --git a/drivers/net/ethernet/intel/i40e/i40e_devids.h b/drivers/net/ethernet/intel/i40e/i40e_devids.h
index 2610338002fe..d9c51a238dcc 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_devids.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_devids.h
@@ -24,8 +24,10 @@
 #define I40E_DEV_ID_10G_B		0x104F
 #define I40E_DEV_ID_10G_SFP		0x104E
 #define I40E_DEV_ID_5G_BASE_T_BC	0x101F
+#define I40E_DEV_ID_1G_BASE_T_BC	0x0DD2
 #define I40E_IS_X710TL_DEVICE(d) \
-	(((d) == I40E_DEV_ID_5G_BASE_T_BC) || \
+	(((d) == I40E_DEV_ID_1G_BASE_T_BC) || \
+	 ((d) == I40E_DEV_ID_5G_BASE_T_BC) || \
 	 ((d) == I40E_DEV_ID_10G_BASE_T_BC))
 #define I40E_DEV_ID_KX_X722		0x37CE
 #define I40E_DEV_ID_QSFP_X722		0x37CF
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
index e9cd0fa6a0d2..7e75706f76db 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
@@ -2001,10 +2001,10 @@ static void i40e_get_drvinfo(struct net_device *netdev,
 	struct i40e_vsi *vsi = np->vsi;
 	struct i40e_pf *pf = vsi->back;
 
-	strlcpy(drvinfo->driver, i40e_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->fw_version, i40e_nvm_version_str(&pf->hw),
+	strscpy(drvinfo->driver, i40e_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->fw_version, i40e_nvm_version_str(&pf->hw),
 		sizeof(drvinfo->fw_version));
-	strlcpy(drvinfo->bus_info, pci_name(pf->pdev),
+	strscpy(drvinfo->bus_info, pci_name(pf->pdev),
 		sizeof(drvinfo->bus_info));
 	drvinfo->n_priv_flags = I40E_PRIV_FLAGS_STR_LEN;
 	if (pf->hw.pf_id == 0)
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index e3d9804aeb25..2c07fa8ecfc8 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -66,6 +66,7 @@ static const struct pci_device_id i40e_pci_tbl[] = {
 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
+	{PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0},
 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
@@ -3878,7 +3879,7 @@ static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
 		wr32(hw, I40E_PFINT_RATEN(vector - 1),
 		     i40e_intrl_usec_to_reg(vsi->int_rate_limit));
 
-		/* Linked list for the queuepairs assigned to this vector */
+		/* begin of linked list for RX queue assigned to this vector */
 		wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
 		for (q = 0; q < q_vector->num_ringpairs; q++) {
 			u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
@@ -3894,6 +3895,7 @@ static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
 			wr32(hw, I40E_QINT_RQCTL(qp), val);
 
 			if (has_xdp) {
+				/* TX queue with next queue set to TX */
 				val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
 				      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
 				      (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
@@ -3903,7 +3905,7 @@ static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
 
 				wr32(hw, I40E_QINT_TQCTL(nextqp), val);
 			}
-
+			/* TX queue with next RX or end of linked list */
 			val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
 			      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
 			      (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
@@ -3972,7 +3974,6 @@ static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
 	struct i40e_q_vector *q_vector = vsi->q_vectors[0];
 	struct i40e_pf *pf = vsi->back;
 	struct i40e_hw *hw = &pf->hw;
-	u32 val;
 
 	/* set the ITR configuration */
 	q_vector->rx.next_update = jiffies + 1;
@@ -3989,28 +3990,20 @@ static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
 	/* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
 	wr32(hw, I40E_PFINT_LNKLST0, 0);
 
-	/* Associate the queue pair to the vector and enable the queue int */
-	val = I40E_QINT_RQCTL_CAUSE_ENA_MASK		       |
-	      (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
-	      (nextqp	   << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
-	      (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
-
-	wr32(hw, I40E_QINT_RQCTL(0), val);
+	/* Associate the queue pair to the vector and enable the queue
+	 * interrupt RX queue in linked list with next queue set to TX
+	 */
+	wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX));
 
 	if (i40e_enabled_xdp_vsi(vsi)) {
-		val = I40E_QINT_TQCTL_CAUSE_ENA_MASK		     |
-		      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
-		      (I40E_QUEUE_TYPE_TX
-		       << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
-
-		wr32(hw, I40E_QINT_TQCTL(nextqp), val);
+		/* TX queue in linked list with next queue set to TX */
+		wr32(hw, I40E_QINT_TQCTL(nextqp),
+		     I40E_QINT_TQCTL_VAL(nextqp, 0, TX));
 	}
 
-	val = I40E_QINT_TQCTL_CAUSE_ENA_MASK		      |
-	      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
-	      (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
-
-	wr32(hw, I40E_QINT_TQCTL(0), val);
+	/* last TX queue so the next RX queue doesn't matter */
+	wr32(hw, I40E_QINT_TQCTL(0),
+	     I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX));
 	i40e_flush(hw);
 }
 
@@ -10724,7 +10717,7 @@ static void i40e_send_version(struct i40e_pf *pf)
 	dv.minor_version = 0xff;
 	dv.build_version = 0xff;
 	dv.subbuild_version = 0;
-	strlcpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
+	strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
 	i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
 }
 
@@ -11948,8 +11941,7 @@ static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
 	cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
 
 	if (vsi->netdev)
-		netif_napi_add(vsi->netdev, &q_vector->napi,
-			       i40e_napi_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll);
 
 	/* tie q_vector and vsi together */
 	vsi->q_vectors[v_idx] = q_vector;
@@ -16072,23 +16064,23 @@ static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 		switch (hw->bus.speed) {
 		case i40e_bus_speed_8000:
-			strlcpy(speed, "8.0", PCI_SPEED_SIZE); break;
+			strscpy(speed, "8.0", PCI_SPEED_SIZE); break;
 		case i40e_bus_speed_5000:
-			strlcpy(speed, "5.0", PCI_SPEED_SIZE); break;
+			strscpy(speed, "5.0", PCI_SPEED_SIZE); break;
 		case i40e_bus_speed_2500:
-			strlcpy(speed, "2.5", PCI_SPEED_SIZE); break;
+			strscpy(speed, "2.5", PCI_SPEED_SIZE); break;
 		default:
 			break;
 		}
 		switch (hw->bus.width) {
 		case i40e_bus_width_pcie_x8:
-			strlcpy(width, "8", PCI_WIDTH_SIZE); break;
+			strscpy(width, "8", PCI_WIDTH_SIZE); break;
 		case i40e_bus_width_pcie_x4:
-			strlcpy(width, "4", PCI_WIDTH_SIZE); break;
+			strscpy(width, "4", PCI_WIDTH_SIZE); break;
 		case i40e_bus_width_pcie_x2:
-			strlcpy(width, "2", PCI_WIDTH_SIZE); break;
+			strscpy(width, "2", PCI_WIDTH_SIZE); break;
 		case i40e_bus_width_pcie_x1:
-			strlcpy(width, "1", PCI_WIDTH_SIZE); break;
+			strscpy(width, "1", PCI_WIDTH_SIZE); break;
 		default:
 			break;
 		}
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
index 2d3533f38d7b..ffea0c9c82f1 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
@@ -1390,7 +1390,7 @@ static long i40e_ptp_create_clock(struct i40e_pf *pf)
 	if (!IS_ERR_OR_NULL(pf->ptp_clock))
 		return 0;
 
-	strlcpy(pf->ptp_caps.name, i40e_driver_name,
+	strscpy(pf->ptp_caps.name, i40e_driver_name,
 		sizeof(pf->ptp_caps.name) - 1);
 	pf->ptp_caps.owner = THIS_MODULE;
 	pf->ptp_caps.max_adj = 999999999;
diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
index e535d4c3da49..a056e1545615 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c
@@ -581,9 +581,9 @@ static void iavf_get_drvinfo(struct net_device *netdev,
 {
 	struct iavf_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, iavf_driver_name, 32);
-	strlcpy(drvinfo->fw_version, "N/A", 4);
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), 32);
+	strscpy(drvinfo->driver, iavf_driver_name, 32);
+	strscpy(drvinfo->fw_version, "N/A", 4);
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev), 32);
 	drvinfo->n_priv_flags = IAVF_PRIV_FLAGS_STR_LEN;
 }
 
diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c
index 0c89f16bf1e2..3fc572341781 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_main.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
@@ -1267,66 +1267,138 @@ static void iavf_up_complete(struct iavf_adapter *adapter)
 }
 
 /**
- * iavf_down - Shutdown the connection processing
+ * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF
+ * yet and mark other to be removed.
  * @adapter: board private structure
- *
- * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
  **/
-void iavf_down(struct iavf_adapter *adapter)
+static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter)
 {
-	struct net_device *netdev = adapter->netdev;
-	struct iavf_vlan_filter *vlf;
-	struct iavf_cloud_filter *cf;
-	struct iavf_fdir_fltr *fdir;
-	struct iavf_mac_filter *f;
-	struct iavf_adv_rss *rss;
-
-	if (adapter->state <= __IAVF_DOWN_PENDING)
-		return;
-
-	netif_carrier_off(netdev);
-	netif_tx_disable(netdev);
-	adapter->link_up = false;
-	iavf_napi_disable_all(adapter);
-	iavf_irq_disable(adapter);
+	struct iavf_vlan_filter *vlf, *vlftmp;
+	struct iavf_mac_filter *f, *ftmp;
 
 	spin_lock_bh(&adapter->mac_vlan_list_lock);
-
 	/* clear the sync flag on all filters */
 	__dev_uc_unsync(adapter->netdev, NULL);
 	__dev_mc_unsync(adapter->netdev, NULL);
 
 	/* remove all MAC filters */
-	list_for_each_entry(f, &adapter->mac_filter_list, list) {
-		f->remove = true;
+	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list,
+				 list) {
+		if (f->add) {
+			list_del(&f->list);
+			kfree(f);
+		} else {
+			f->remove = true;
+		}
 	}
 
 	/* remove all VLAN filters */
-	list_for_each_entry(vlf, &adapter->vlan_filter_list, list) {
-		vlf->remove = true;
+	list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
+				 list) {
+		if (vlf->add) {
+			list_del(&vlf->list);
+			kfree(vlf);
+		} else {
+			vlf->remove = true;
+		}
 	}
-
 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
+}
+
+/**
+ * iavf_clear_cloud_filters - Remove cloud filters not sent to PF yet and
+ * mark other to be removed.
+ * @adapter: board private structure
+ **/
+static void iavf_clear_cloud_filters(struct iavf_adapter *adapter)
+{
+	struct iavf_cloud_filter *cf, *cftmp;
 
 	/* remove all cloud filters */
 	spin_lock_bh(&adapter->cloud_filter_list_lock);
-	list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
-		cf->del = true;
+	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
+				 list) {
+		if (cf->add) {
+			list_del(&cf->list);
+			kfree(cf);
+			adapter->num_cloud_filters--;
+		} else {
+			cf->del = true;
+		}
 	}
 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
+}
+
+/**
+ * iavf_clear_fdir_filters - Remove fdir filters not sent to PF yet and mark
+ * other to be removed.
+ * @adapter: board private structure
+ **/
+static void iavf_clear_fdir_filters(struct iavf_adapter *adapter)
+{
+	struct iavf_fdir_fltr *fdir, *fdirtmp;
 
 	/* remove all Flow Director filters */
 	spin_lock_bh(&adapter->fdir_fltr_lock);
-	list_for_each_entry(fdir, &adapter->fdir_list_head, list) {
-		fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
+	list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head,
+				 list) {
+		if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) {
+			list_del(&fdir->list);
+			kfree(fdir);
+			adapter->fdir_active_fltr--;
+		} else {
+			fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
+		}
 	}
 	spin_unlock_bh(&adapter->fdir_fltr_lock);
+}
+
+/**
+ * iavf_clear_adv_rss_conf - Remove adv rss conf not sent to PF yet and mark
+ * other to be removed.
+ * @adapter: board private structure
+ **/
+static void iavf_clear_adv_rss_conf(struct iavf_adapter *adapter)
+{
+	struct iavf_adv_rss *rss, *rsstmp;
 
 	/* remove all advance RSS configuration */
 	spin_lock_bh(&adapter->adv_rss_lock);
-	list_for_each_entry(rss, &adapter->adv_rss_list_head, list)
-		rss->state = IAVF_ADV_RSS_DEL_REQUEST;
+	list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
+				 list) {
+		if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) {
+			list_del(&rss->list);
+			kfree(rss);
+		} else {
+			rss->state = IAVF_ADV_RSS_DEL_REQUEST;
+		}
+	}
 	spin_unlock_bh(&adapter->adv_rss_lock);
+}
+
+/**
+ * iavf_down - Shutdown the connection processing
+ * @adapter: board private structure
+ *
+ * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
+ **/
+void iavf_down(struct iavf_adapter *adapter)
+{
+	struct net_device *netdev = adapter->netdev;
+
+	if (adapter->state <= __IAVF_DOWN_PENDING)
+		return;
+
+	netif_carrier_off(netdev);
+	netif_tx_disable(netdev);
+	adapter->link_up = false;
+	iavf_napi_disable_all(adapter);
+	iavf_irq_disable(adapter);
+
+	iavf_clear_mac_vlan_filters(adapter);
+	iavf_clear_cloud_filters(adapter);
+	iavf_clear_fdir_filters(adapter);
+	iavf_clear_adv_rss_conf(adapter);
 
 	if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)) {
 		/* cancel any current operation */
@@ -1335,11 +1407,16 @@ void iavf_down(struct iavf_adapter *adapter)
 		 * here for this to complete. The watchdog is still running
 		 * and it will take care of this.
 		 */
-		adapter->aq_required = IAVF_FLAG_AQ_DEL_MAC_FILTER;
-		adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
-		adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
-		adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
-		adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
+		if (!list_empty(&adapter->mac_filter_list))
+			adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
+		if (!list_empty(&adapter->vlan_filter_list))
+			adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
+		if (!list_empty(&adapter->cloud_filter_list))
+			adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
+		if (!list_empty(&adapter->fdir_list_head))
+			adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
+		if (!list_empty(&adapter->adv_rss_list_head))
+			adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
 		adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES;
 	}
 
@@ -1754,7 +1831,7 @@ static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
 		q_vector->reg_idx = q_idx;
 		cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
 		netif_napi_add(adapter->netdev, &q_vector->napi,
-			       iavf_napi_poll, NAPI_POLL_WEIGHT);
+			       iavf_napi_poll);
 	}
 
 	return 0;
@@ -4178,6 +4255,7 @@ err_unlock:
 static int iavf_close(struct net_device *netdev)
 {
 	struct iavf_adapter *adapter = netdev_priv(netdev);
+	u64 aq_to_restore;
 	int status;
 
 	mutex_lock(&adapter->crit_lock);
@@ -4190,6 +4268,29 @@ static int iavf_close(struct net_device *netdev)
 	set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
 	if (CLIENT_ENABLED(adapter))
 		adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE;
+	/* We cannot send IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS before
+	 * IAVF_FLAG_AQ_DISABLE_QUEUES because in such case there is rtnl
+	 * deadlock with adminq_task() until iavf_close timeouts. We must send
+	 * IAVF_FLAG_AQ_GET_CONFIG before IAVF_FLAG_AQ_DISABLE_QUEUES to make
+	 * disable queues possible for vf. Give only necessary flags to
+	 * iavf_down and save other to set them right before iavf_close()
+	 * returns, when IAVF_FLAG_AQ_DISABLE_QUEUES will be already sent and
+	 * iavf will be in DOWN state.
+	 */
+	aq_to_restore = adapter->aq_required;
+	adapter->aq_required &= IAVF_FLAG_AQ_GET_CONFIG;
+
+	/* Remove flags which we do not want to send after close or we want to
+	 * send before disable queues.
+	 */
+	aq_to_restore &= ~(IAVF_FLAG_AQ_GET_CONFIG		|
+			   IAVF_FLAG_AQ_ENABLE_QUEUES		|
+			   IAVF_FLAG_AQ_CONFIGURE_QUEUES	|
+			   IAVF_FLAG_AQ_ADD_VLAN_FILTER		|
+			   IAVF_FLAG_AQ_ADD_MAC_FILTER		|
+			   IAVF_FLAG_AQ_ADD_CLOUD_FILTER	|
+			   IAVF_FLAG_AQ_ADD_FDIR_FILTER		|
+			   IAVF_FLAG_AQ_ADD_ADV_RSS_CFG);
 
 	iavf_down(adapter);
 	iavf_change_state(adapter, __IAVF_DOWN_PENDING);
@@ -4213,6 +4314,10 @@ static int iavf_close(struct net_device *netdev)
 				    msecs_to_jiffies(500));
 	if (!status)
 		netdev_warn(netdev, "Device resources not yet released\n");
+
+	mutex_lock(&adapter->crit_lock);
+	adapter->aq_required |= aq_to_restore;
+	mutex_unlock(&adapter->crit_lock);
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h
index 841fa149c407..001500afc4a6 100644
--- a/drivers/net/ethernet/intel/ice/ice.h
+++ b/drivers/net/ethernet/intel/ice/ice.h
@@ -864,6 +864,7 @@ ice_fetch_u64_stats_per_ring(struct u64_stats_sync *syncp,
 			     struct ice_q_stats stats, u64 *pkts, u64 *bytes);
 int ice_up(struct ice_vsi *vsi);
 int ice_down(struct ice_vsi *vsi);
+int ice_down_up(struct ice_vsi *vsi);
 int ice_vsi_cfg(struct ice_vsi *vsi);
 struct ice_vsi *ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi);
 int ice_vsi_determine_xdp_res(struct ice_vsi *vsi);
diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h
index 9939238573a4..1bdc70aa979d 100644
--- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h
+++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h
@@ -1423,6 +1423,56 @@ struct ice_aqc_set_port_id_led {
 	u8 rsvd[13];
 };
 
+/* Get Port Options (indirect, 0x06EA) */
+struct ice_aqc_get_port_options {
+	u8 lport_num;
+	u8 lport_num_valid;
+	u8 port_options_count;
+#define ICE_AQC_PORT_OPT_COUNT_M	GENMASK(3, 0)
+#define ICE_AQC_PORT_OPT_MAX		16
+
+	u8 innermost_phy_index;
+	u8 port_options;
+#define ICE_AQC_PORT_OPT_ACTIVE_M	GENMASK(3, 0)
+#define ICE_AQC_PORT_OPT_VALID		BIT(7)
+
+	u8 pending_port_option_status;
+#define ICE_AQC_PENDING_PORT_OPT_IDX_M	GENMASK(3, 0)
+#define ICE_AQC_PENDING_PORT_OPT_VALID	BIT(7)
+
+	u8 rsvd[2];
+	__le32 addr_high;
+	__le32 addr_low;
+};
+
+struct ice_aqc_get_port_options_elem {
+	u8 pmd;
+#define ICE_AQC_PORT_OPT_PMD_COUNT_M	GENMASK(3, 0)
+
+	u8 max_lane_speed;
+#define ICE_AQC_PORT_OPT_MAX_LANE_M	GENMASK(3, 0)
+#define ICE_AQC_PORT_OPT_MAX_LANE_100M	0
+#define ICE_AQC_PORT_OPT_MAX_LANE_1G	1
+#define ICE_AQC_PORT_OPT_MAX_LANE_2500M	2
+#define ICE_AQC_PORT_OPT_MAX_LANE_5G	3
+#define ICE_AQC_PORT_OPT_MAX_LANE_10G	4
+#define ICE_AQC_PORT_OPT_MAX_LANE_25G	5
+#define ICE_AQC_PORT_OPT_MAX_LANE_50G	6
+#define ICE_AQC_PORT_OPT_MAX_LANE_100G	7
+
+	u8 global_scid[2];
+	u8 phy_scid[2];
+	u8 pf2port_cid[2];
+};
+
+/* Set Port Option (direct, 0x06EB) */
+struct ice_aqc_set_port_option {
+	u8 lport_num;
+	u8 lport_num_valid;
+	u8 selected_port_option;
+	u8 rsvd[13];
+};
+
 /* Set/Get GPIO (direct, 0x06EC/0x06ED) */
 struct ice_aqc_gpio {
 	__le16 gpio_ctrl_handle;
@@ -1489,6 +1539,12 @@ struct ice_aqc_nvm {
 #define ICE_AQC_NVM_PERST_FLAG		1
 #define ICE_AQC_NVM_EMPR_FLAG		2
 #define ICE_AQC_NVM_EMPR_ENA		BIT(0) /* Write Activate reply only */
+	/* For Write Activate, several flags are sent as part of a separate
+	 * flags2 field using a separate byte. For simplicity of the software
+	 * interface, we pass the flags as a 16 bit value so these flags are
+	 * all offset by 8 bits
+	 */
+#define ICE_AQC_NVM_ACTIV_REQ_EMPR	BIT(8) /* NVM Write Activate only */
 	__le16 module_typeid;
 	__le16 length;
 #define ICE_AQC_NVM_ERASE_LEN	0xFFFF
@@ -2082,6 +2138,8 @@ struct ice_aq_desc {
 		struct ice_aqc_gpio read_write_gpio;
 		struct ice_aqc_sff_eeprom read_write_sff_param;
 		struct ice_aqc_set_port_id_led set_port_id_led;
+		struct ice_aqc_get_port_options get_port_options;
+		struct ice_aqc_set_port_option set_port_option;
 		struct ice_aqc_get_sw_cfg get_sw_conf;
 		struct ice_aqc_set_port_params set_port_params;
 		struct ice_aqc_sw_rules sw_rules;
@@ -2243,6 +2301,8 @@ enum ice_adminq_opc {
 	ice_aqc_opc_read_i2c				= 0x06E2,
 	ice_aqc_opc_write_i2c				= 0x06E3,
 	ice_aqc_opc_set_port_id_led			= 0x06E9,
+	ice_aqc_opc_get_port_options			= 0x06EA,
+	ice_aqc_opc_set_port_option			= 0x06EB,
 	ice_aqc_opc_set_gpio				= 0x06EC,
 	ice_aqc_opc_get_gpio				= 0x06ED,
 	ice_aqc_opc_sff_eeprom				= 0x06EE,
diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c
index 1e3243808178..9e36f01dfa4f 100644
--- a/drivers/net/ethernet/intel/ice/ice_base.c
+++ b/drivers/net/ethernet/intel/ice/ice_base.c
@@ -130,8 +130,7 @@ static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, u16 v_idx)
 	 * handler here (i.e. resume, reset/rebuild, etc.)
 	 */
 	if (vsi->netdev)
-		netif_napi_add(vsi->netdev, &q_vector->napi, ice_napi_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(vsi->netdev, &q_vector->napi, ice_napi_poll);
 
 out:
 	/* tie q_vector and VSI together */
@@ -405,7 +404,7 @@ static int ice_setup_rx_ctx(struct ice_rx_ring *ring)
 	/* Strip the Ethernet CRC bytes before the packet is posted to host
 	 * memory.
 	 */
-	rlan_ctx.crcstrip = 1;
+	rlan_ctx.crcstrip = !(ring->flags & ICE_RX_FLAGS_CRC_STRIP_DIS);
 
 	/* L2TSEL flag defines the reported L2 Tags in the receive descriptor
 	 * and it needs to remain 1 for non-DVM capable configurations to not
diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c
index 27d0cbbd29da..039342a0ed15 100644
--- a/drivers/net/ethernet/intel/ice/ice_common.c
+++ b/drivers/net/ethernet/intel/ice/ice_common.c
@@ -8,6 +8,108 @@
 
 #define ICE_PF_RESET_WAIT_COUNT	300
 
+static const char * const ice_link_mode_str_low[] = {
+	[0] = "100BASE_TX",
+	[1] = "100M_SGMII",
+	[2] = "1000BASE_T",
+	[3] = "1000BASE_SX",
+	[4] = "1000BASE_LX",
+	[5] = "1000BASE_KX",
+	[6] = "1G_SGMII",
+	[7] = "2500BASE_T",
+	[8] = "2500BASE_X",
+	[9] = "2500BASE_KX",
+	[10] = "5GBASE_T",
+	[11] = "5GBASE_KR",
+	[12] = "10GBASE_T",
+	[13] = "10G_SFI_DA",
+	[14] = "10GBASE_SR",
+	[15] = "10GBASE_LR",
+	[16] = "10GBASE_KR_CR1",
+	[17] = "10G_SFI_AOC_ACC",
+	[18] = "10G_SFI_C2C",
+	[19] = "25GBASE_T",
+	[20] = "25GBASE_CR",
+	[21] = "25GBASE_CR_S",
+	[22] = "25GBASE_CR1",
+	[23] = "25GBASE_SR",
+	[24] = "25GBASE_LR",
+	[25] = "25GBASE_KR",
+	[26] = "25GBASE_KR_S",
+	[27] = "25GBASE_KR1",
+	[28] = "25G_AUI_AOC_ACC",
+	[29] = "25G_AUI_C2C",
+	[30] = "40GBASE_CR4",
+	[31] = "40GBASE_SR4",
+	[32] = "40GBASE_LR4",
+	[33] = "40GBASE_KR4",
+	[34] = "40G_XLAUI_AOC_ACC",
+	[35] = "40G_XLAUI",
+	[36] = "50GBASE_CR2",
+	[37] = "50GBASE_SR2",
+	[38] = "50GBASE_LR2",
+	[39] = "50GBASE_KR2",
+	[40] = "50G_LAUI2_AOC_ACC",
+	[41] = "50G_LAUI2",
+	[42] = "50G_AUI2_AOC_ACC",
+	[43] = "50G_AUI2",
+	[44] = "50GBASE_CP",
+	[45] = "50GBASE_SR",
+	[46] = "50GBASE_FR",
+	[47] = "50GBASE_LR",
+	[48] = "50GBASE_KR_PAM4",
+	[49] = "50G_AUI1_AOC_ACC",
+	[50] = "50G_AUI1",
+	[51] = "100GBASE_CR4",
+	[52] = "100GBASE_SR4",
+	[53] = "100GBASE_LR4",
+	[54] = "100GBASE_KR4",
+	[55] = "100G_CAUI4_AOC_ACC",
+	[56] = "100G_CAUI4",
+	[57] = "100G_AUI4_AOC_ACC",
+	[58] = "100G_AUI4",
+	[59] = "100GBASE_CR_PAM4",
+	[60] = "100GBASE_KR_PAM4",
+	[61] = "100GBASE_CP2",
+	[62] = "100GBASE_SR2",
+	[63] = "100GBASE_DR",
+};
+
+static const char * const ice_link_mode_str_high[] = {
+	[0] = "100GBASE_KR2_PAM4",
+	[1] = "100G_CAUI2_AOC_ACC",
+	[2] = "100G_CAUI2",
+	[3] = "100G_AUI2_AOC_ACC",
+	[4] = "100G_AUI2",
+};
+
+/**
+ * ice_dump_phy_type - helper function to dump phy_type
+ * @hw: pointer to the HW structure
+ * @low: 64 bit value for phy_type_low
+ * @high: 64 bit value for phy_type_high
+ * @prefix: prefix string to differentiate multiple dumps
+ */
+static void
+ice_dump_phy_type(struct ice_hw *hw, u64 low, u64 high, const char *prefix)
+{
+	ice_debug(hw, ICE_DBG_PHY, "%s: phy_type_low: 0x%016llx\n", prefix, low);
+
+	for (u32 i = 0; i < BITS_PER_TYPE(typeof(low)); i++) {
+		if (low & BIT_ULL(i))
+			ice_debug(hw, ICE_DBG_PHY, "%s:   bit(%d): %s\n",
+				  prefix, i, ice_link_mode_str_low[i]);
+	}
+
+	ice_debug(hw, ICE_DBG_PHY, "%s: phy_type_high: 0x%016llx\n", prefix, high);
+
+	for (u32 i = 0; i < BITS_PER_TYPE(typeof(high)); i++) {
+		if (high & BIT_ULL(i))
+			ice_debug(hw, ICE_DBG_PHY, "%s:   bit(%d): %s\n",
+				  prefix, i, ice_link_mode_str_high[i]);
+	}
+}
+
 /**
  * ice_set_mac_type - Sets MAC type
  * @hw: pointer to the HW structure
@@ -80,9 +182,23 @@ bool ice_is_e810t(struct ice_hw *hw)
 {
 	switch (hw->device_id) {
 	case ICE_DEV_ID_E810C_SFP:
-		if (hw->subsystem_device_id == ICE_SUBDEV_ID_E810T ||
-		    hw->subsystem_device_id == ICE_SUBDEV_ID_E810T2)
+		switch (hw->subsystem_device_id) {
+		case ICE_SUBDEV_ID_E810T:
+		case ICE_SUBDEV_ID_E810T2:
+		case ICE_SUBDEV_ID_E810T3:
+		case ICE_SUBDEV_ID_E810T4:
+		case ICE_SUBDEV_ID_E810T6:
+		case ICE_SUBDEV_ID_E810T7:
+			return true;
+		}
+		break;
+	case ICE_DEV_ID_E810C_QSFP:
+		switch (hw->subsystem_device_id) {
+		case ICE_SUBDEV_ID_E810T2:
+		case ICE_SUBDEV_ID_E810T3:
+		case ICE_SUBDEV_ID_E810T5:
 			return true;
+		}
 		break;
 	default:
 		break;
@@ -183,6 +299,7 @@ ice_aq_get_phy_caps(struct ice_port_info *pi, bool qual_mods, u8 report_mode,
 	struct ice_aqc_get_phy_caps *cmd;
 	u16 pcaps_size = sizeof(*pcaps);
 	struct ice_aq_desc desc;
+	const char *prefix;
 	struct ice_hw *hw;
 	int status;
 
@@ -204,29 +321,48 @@ ice_aq_get_phy_caps(struct ice_port_info *pi, bool qual_mods, u8 report_mode,
 	cmd->param0 |= cpu_to_le16(report_mode);
 	status = ice_aq_send_cmd(hw, &desc, pcaps, pcaps_size, cd);
 
-	ice_debug(hw, ICE_DBG_LINK, "get phy caps - report_mode = 0x%x\n",
-		  report_mode);
-	ice_debug(hw, ICE_DBG_LINK, "	phy_type_low = 0x%llx\n",
-		  (unsigned long long)le64_to_cpu(pcaps->phy_type_low));
-	ice_debug(hw, ICE_DBG_LINK, "	phy_type_high = 0x%llx\n",
-		  (unsigned long long)le64_to_cpu(pcaps->phy_type_high));
-	ice_debug(hw, ICE_DBG_LINK, "	caps = 0x%x\n", pcaps->caps);
-	ice_debug(hw, ICE_DBG_LINK, "	low_power_ctrl_an = 0x%x\n",
+	ice_debug(hw, ICE_DBG_LINK, "get phy caps dump\n");
+
+	switch (report_mode) {
+	case ICE_AQC_REPORT_TOPO_CAP_MEDIA:
+		prefix = "phy_caps_media";
+		break;
+	case ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA:
+		prefix = "phy_caps_no_media";
+		break;
+	case ICE_AQC_REPORT_ACTIVE_CFG:
+		prefix = "phy_caps_active";
+		break;
+	case ICE_AQC_REPORT_DFLT_CFG:
+		prefix = "phy_caps_default";
+		break;
+	default:
+		prefix = "phy_caps_invalid";
+	}
+
+	ice_dump_phy_type(hw, le64_to_cpu(pcaps->phy_type_low),
+			  le64_to_cpu(pcaps->phy_type_high), prefix);
+
+	ice_debug(hw, ICE_DBG_LINK, "%s: report_mode = 0x%x\n",
+		  prefix, report_mode);
+	ice_debug(hw, ICE_DBG_LINK, "%s: caps = 0x%x\n", prefix, pcaps->caps);
+	ice_debug(hw, ICE_DBG_LINK, "%s: low_power_ctrl_an = 0x%x\n", prefix,
 		  pcaps->low_power_ctrl_an);
-	ice_debug(hw, ICE_DBG_LINK, "	eee_cap = 0x%x\n", pcaps->eee_cap);
-	ice_debug(hw, ICE_DBG_LINK, "	eeer_value = 0x%x\n",
+	ice_debug(hw, ICE_DBG_LINK, "%s: eee_cap = 0x%x\n", prefix,
+		  pcaps->eee_cap);
+	ice_debug(hw, ICE_DBG_LINK, "%s: eeer_value = 0x%x\n", prefix,
 		  pcaps->eeer_value);
-	ice_debug(hw, ICE_DBG_LINK, "	link_fec_options = 0x%x\n",
+	ice_debug(hw, ICE_DBG_LINK, "%s: link_fec_options = 0x%x\n", prefix,
 		  pcaps->link_fec_options);
-	ice_debug(hw, ICE_DBG_LINK, "	module_compliance_enforcement = 0x%x\n",
-		  pcaps->module_compliance_enforcement);
-	ice_debug(hw, ICE_DBG_LINK, "   extended_compliance_code = 0x%x\n",
-		  pcaps->extended_compliance_code);
-	ice_debug(hw, ICE_DBG_LINK, "   module_type[0] = 0x%x\n",
+	ice_debug(hw, ICE_DBG_LINK, "%s: module_compliance_enforcement = 0x%x\n",
+		  prefix, pcaps->module_compliance_enforcement);
+	ice_debug(hw, ICE_DBG_LINK, "%s: extended_compliance_code = 0x%x\n",
+		  prefix, pcaps->extended_compliance_code);
+	ice_debug(hw, ICE_DBG_LINK, "%s: module_type[0] = 0x%x\n", prefix,
 		  pcaps->module_type[0]);
-	ice_debug(hw, ICE_DBG_LINK, "   module_type[1] = 0x%x\n",
+	ice_debug(hw, ICE_DBG_LINK, "%s: module_type[1] = 0x%x\n", prefix,
 		  pcaps->module_type[1]);
-	ice_debug(hw, ICE_DBG_LINK, "   module_type[2] = 0x%x\n",
+	ice_debug(hw, ICE_DBG_LINK, "%s: module_type[2] = 0x%x\n", prefix,
 		  pcaps->module_type[2]);
 
 	if (!status && report_mode == ICE_AQC_REPORT_TOPO_CAP_MEDIA) {
@@ -2397,6 +2533,8 @@ ice_parse_1588_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p,
 	info->tmr1_owned = ((number & ICE_TS_TMR1_OWND_M) != 0);
 	info->tmr1_ena = ((number & ICE_TS_TMR1_ENA_M) != 0);
 
+	info->ts_ll_read = ((number & ICE_TS_LL_TX_TS_READ_M) != 0);
+
 	info->ena_ports = logical_id;
 	info->tmr_own_map = phys_id;
 
@@ -2414,6 +2552,8 @@ ice_parse_1588_dev_caps(struct ice_hw *hw, struct ice_hw_dev_caps *dev_p,
 		  info->tmr1_owned);
 	ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr1_ena = %u\n",
 		  info->tmr1_ena);
+	ice_debug(hw, ICE_DBG_INIT, "dev caps: ts_ll_read = %u\n",
+		  info->ts_ll_read);
 	ice_debug(hw, ICE_DBG_INIT, "dev caps: ieee_1588 ena_ports = %u\n",
 		  info->ena_ports);
 	ice_debug(hw, ICE_DBG_INIT, "dev caps: tmr_own_map = %u\n",
@@ -2776,6 +2916,26 @@ ice_aq_set_port_params(struct ice_port_info *pi, bool double_vlan,
 }
 
 /**
+ * ice_is_100m_speed_supported
+ * @hw: pointer to the HW struct
+ *
+ * returns true if 100M speeds are supported by the device,
+ * false otherwise.
+ */
+bool ice_is_100m_speed_supported(struct ice_hw *hw)
+{
+	switch (hw->device_id) {
+	case ICE_DEV_ID_E822C_SGMII:
+	case ICE_DEV_ID_E822L_SGMII:
+	case ICE_DEV_ID_E823L_1GBE:
+	case ICE_DEV_ID_E823C_SGMII:
+		return true;
+	default:
+		return false;
+	}
+}
+
+/**
  * ice_get_link_speed_based_on_phy_type - returns link speed
  * @phy_type_low: lower part of phy_type
  * @phy_type_high: higher part of phy_type
@@ -3535,6 +3695,121 @@ ice_aq_set_port_id_led(struct ice_port_info *pi, bool is_orig_mode,
 }
 
 /**
+ * ice_aq_get_port_options
+ * @hw: pointer to the HW struct
+ * @options: buffer for the resultant port options
+ * @option_count: input - size of the buffer in port options structures,
+ *                output - number of returned port options
+ * @lport: logical port to call the command with (optional)
+ * @lport_valid: when false, FW uses port owned by the PF instead of lport,
+ *               when PF owns more than 1 port it must be true
+ * @active_option_idx: index of active port option in returned buffer
+ * @active_option_valid: active option in returned buffer is valid
+ * @pending_option_idx: index of pending port option in returned buffer
+ * @pending_option_valid: pending option in returned buffer is valid
+ *
+ * Calls Get Port Options AQC (0x06ea) and verifies result.
+ */
+int
+ice_aq_get_port_options(struct ice_hw *hw,
+			struct ice_aqc_get_port_options_elem *options,
+			u8 *option_count, u8 lport, bool lport_valid,
+			u8 *active_option_idx, bool *active_option_valid,
+			u8 *pending_option_idx, bool *pending_option_valid)
+{
+	struct ice_aqc_get_port_options *cmd;
+	struct ice_aq_desc desc;
+	int status;
+	u8 i;
+
+	/* options buffer shall be able to hold max returned options */
+	if (*option_count < ICE_AQC_PORT_OPT_COUNT_M)
+		return -EINVAL;
+
+	cmd = &desc.params.get_port_options;
+	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_get_port_options);
+
+	if (lport_valid)
+		cmd->lport_num = lport;
+	cmd->lport_num_valid = lport_valid;
+
+	status = ice_aq_send_cmd(hw, &desc, options,
+				 *option_count * sizeof(*options), NULL);
+	if (status)
+		return status;
+
+	/* verify direct FW response & set output parameters */
+	*option_count = FIELD_GET(ICE_AQC_PORT_OPT_COUNT_M,
+				  cmd->port_options_count);
+	ice_debug(hw, ICE_DBG_PHY, "options: %x\n", *option_count);
+	*active_option_valid = FIELD_GET(ICE_AQC_PORT_OPT_VALID,
+					 cmd->port_options);
+	if (*active_option_valid) {
+		*active_option_idx = FIELD_GET(ICE_AQC_PORT_OPT_ACTIVE_M,
+					       cmd->port_options);
+		if (*active_option_idx > (*option_count - 1))
+			return -EIO;
+		ice_debug(hw, ICE_DBG_PHY, "active idx: %x\n",
+			  *active_option_idx);
+	}
+
+	*pending_option_valid = FIELD_GET(ICE_AQC_PENDING_PORT_OPT_VALID,
+					  cmd->pending_port_option_status);
+	if (*pending_option_valid) {
+		*pending_option_idx = FIELD_GET(ICE_AQC_PENDING_PORT_OPT_IDX_M,
+						cmd->pending_port_option_status);
+		if (*pending_option_idx > (*option_count - 1))
+			return -EIO;
+		ice_debug(hw, ICE_DBG_PHY, "pending idx: %x\n",
+			  *pending_option_idx);
+	}
+
+	/* mask output options fields */
+	for (i = 0; i < *option_count; i++) {
+		options[i].pmd = FIELD_GET(ICE_AQC_PORT_OPT_PMD_COUNT_M,
+					   options[i].pmd);
+		options[i].max_lane_speed = FIELD_GET(ICE_AQC_PORT_OPT_MAX_LANE_M,
+						      options[i].max_lane_speed);
+		ice_debug(hw, ICE_DBG_PHY, "pmds: %x max speed: %x\n",
+			  options[i].pmd, options[i].max_lane_speed);
+	}
+
+	return 0;
+}
+
+/**
+ * ice_aq_set_port_option
+ * @hw: pointer to the HW struct
+ * @lport: logical port to call the command with
+ * @lport_valid: when false, FW uses port owned by the PF instead of lport,
+ *               when PF owns more than 1 port it must be true
+ * @new_option: new port option to be written
+ *
+ * Calls Set Port Options AQC (0x06eb).
+ */
+int
+ice_aq_set_port_option(struct ice_hw *hw, u8 lport, u8 lport_valid,
+		       u8 new_option)
+{
+	struct ice_aqc_set_port_option *cmd;
+	struct ice_aq_desc desc;
+
+	if (new_option > ICE_AQC_PORT_OPT_COUNT_M)
+		return -EINVAL;
+
+	cmd = &desc.params.set_port_option;
+	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_set_port_option);
+
+	if (lport_valid)
+		cmd->lport_num = lport;
+
+	cmd->lport_num_valid = lport_valid;
+	cmd->selected_port_option = new_option;
+
+	return ice_aq_send_cmd(hw, &desc, NULL, 0, NULL);
+}
+
+/**
  * ice_aq_sff_eeprom
  * @hw: pointer to the HW struct
  * @lport: bits [7:0] = logical port, bit [8] = logical port valid
@@ -5029,20 +5304,22 @@ ice_aq_get_gpio(struct ice_hw *hw, u16 gpio_ctrl_handle, u8 pin_idx,
 }
 
 /**
- * ice_fw_supports_link_override
+ * ice_is_fw_api_min_ver
  * @hw: pointer to the hardware structure
+ * @maj: major version
+ * @min: minor version
+ * @patch: patch version
  *
- * Checks if the firmware supports link override
+ * Checks if the firmware API is minimum version
  */
-bool ice_fw_supports_link_override(struct ice_hw *hw)
+static bool ice_is_fw_api_min_ver(struct ice_hw *hw, u8 maj, u8 min, u8 patch)
 {
-	if (hw->api_maj_ver == ICE_FW_API_LINK_OVERRIDE_MAJ) {
-		if (hw->api_min_ver > ICE_FW_API_LINK_OVERRIDE_MIN)
+	if (hw->api_maj_ver == maj) {
+		if (hw->api_min_ver > min)
 			return true;
-		if (hw->api_min_ver == ICE_FW_API_LINK_OVERRIDE_MIN &&
-		    hw->api_patch >= ICE_FW_API_LINK_OVERRIDE_PATCH)
+		if (hw->api_min_ver == min && hw->api_patch >= patch)
 			return true;
-	} else if (hw->api_maj_ver > ICE_FW_API_LINK_OVERRIDE_MAJ) {
+	} else if (hw->api_maj_ver > maj) {
 		return true;
 	}
 
@@ -5050,6 +5327,19 @@ bool ice_fw_supports_link_override(struct ice_hw *hw)
 }
 
 /**
+ * ice_fw_supports_link_override
+ * @hw: pointer to the hardware structure
+ *
+ * Checks if the firmware supports link override
+ */
+bool ice_fw_supports_link_override(struct ice_hw *hw)
+{
+	return ice_is_fw_api_min_ver(hw, ICE_FW_API_LINK_OVERRIDE_MAJ,
+				     ICE_FW_API_LINK_OVERRIDE_MIN,
+				     ICE_FW_API_LINK_OVERRIDE_PATCH);
+}
+
+/**
  * ice_get_link_default_override
  * @ldo: pointer to the link default override struct
  * @pi: pointer to the port info struct
@@ -5179,16 +5469,9 @@ bool ice_fw_supports_lldp_fltr_ctrl(struct ice_hw *hw)
 	if (hw->mac_type != ICE_MAC_E810)
 		return false;
 
-	if (hw->api_maj_ver == ICE_FW_API_LLDP_FLTR_MAJ) {
-		if (hw->api_min_ver > ICE_FW_API_LLDP_FLTR_MIN)
-			return true;
-		if (hw->api_min_ver == ICE_FW_API_LLDP_FLTR_MIN &&
-		    hw->api_patch >= ICE_FW_API_LLDP_FLTR_PATCH)
-			return true;
-	} else if (hw->api_maj_ver > ICE_FW_API_LLDP_FLTR_MAJ) {
-		return true;
-	}
-	return false;
+	return ice_is_fw_api_min_ver(hw, ICE_FW_API_LLDP_FLTR_MAJ,
+				     ICE_FW_API_LLDP_FLTR_MIN,
+				     ICE_FW_API_LLDP_FLTR_PATCH);
 }
 
 /**
@@ -5225,14 +5508,7 @@ ice_lldp_fltr_add_remove(struct ice_hw *hw, u16 vsi_num, bool add)
  */
 bool ice_fw_supports_report_dflt_cfg(struct ice_hw *hw)
 {
-	if (hw->api_maj_ver == ICE_FW_API_REPORT_DFLT_CFG_MAJ) {
-		if (hw->api_min_ver > ICE_FW_API_REPORT_DFLT_CFG_MIN)
-			return true;
-		if (hw->api_min_ver == ICE_FW_API_REPORT_DFLT_CFG_MIN &&
-		    hw->api_patch >= ICE_FW_API_REPORT_DFLT_CFG_PATCH)
-			return true;
-	} else if (hw->api_maj_ver > ICE_FW_API_REPORT_DFLT_CFG_MAJ) {
-		return true;
-	}
-	return false;
+	return ice_is_fw_api_min_ver(hw, ICE_FW_API_REPORT_DFLT_CFG_MAJ,
+				     ICE_FW_API_REPORT_DFLT_CFG_MIN,
+				     ICE_FW_API_REPORT_DFLT_CFG_PATCH);
 }
diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h
index 61b7c60db689..8b6712b92e84 100644
--- a/drivers/net/ethernet/intel/ice/ice_common.h
+++ b/drivers/net/ethernet/intel/ice/ice_common.h
@@ -151,6 +151,15 @@ int
 ice_aq_set_port_id_led(struct ice_port_info *pi, bool is_orig_mode,
 		       struct ice_sq_cd *cd);
 int
+ice_aq_get_port_options(struct ice_hw *hw,
+			struct ice_aqc_get_port_options_elem *options,
+			u8 *option_count, u8 lport, bool lport_valid,
+			u8 *active_option_idx, bool *active_option_valid,
+			u8 *pending_option_idx, bool *pending_option_valid);
+int
+ice_aq_set_port_option(struct ice_hw *hw, u8 lport, u8 lport_valid,
+		       u8 new_option);
+int
 ice_aq_sff_eeprom(struct ice_hw *hw, u16 lport, u8 bus_addr,
 		  u16 mem_addr, u8 page, u8 set_page, u8 *data, u8 length,
 		  bool write, struct ice_sq_cd *cd);
@@ -204,6 +213,7 @@ ice_aq_set_gpio(struct ice_hw *hw, u16 gpio_ctrl_handle, u8 pin_idx, bool value,
 int
 ice_aq_get_gpio(struct ice_hw *hw, u16 gpio_ctrl_handle, u8 pin_idx,
 		bool *value, struct ice_sq_cd *cd);
+bool ice_is_100m_speed_supported(struct ice_hw *hw);
 int
 ice_aq_set_lldp_mib(struct ice_hw *hw, u8 mib_type, void *buf, u16 buf_size,
 		    struct ice_sq_cd *cd);
diff --git a/drivers/net/ethernet/intel/ice/ice_devids.h b/drivers/net/ethernet/intel/ice/ice_devids.h
index b41bc3dc1745..6d560d1c74a4 100644
--- a/drivers/net/ethernet/intel/ice/ice_devids.h
+++ b/drivers/net/ethernet/intel/ice/ice_devids.h
@@ -24,6 +24,11 @@
 #define ICE_DEV_ID_E810C_SFP		0x1593
 #define ICE_SUBDEV_ID_E810T		0x000E
 #define ICE_SUBDEV_ID_E810T2		0x000F
+#define ICE_SUBDEV_ID_E810T3		0x0010
+#define ICE_SUBDEV_ID_E810T4		0x0011
+#define ICE_SUBDEV_ID_E810T5		0x0012
+#define ICE_SUBDEV_ID_E810T6		0x02E9
+#define ICE_SUBDEV_ID_E810T7		0x02EA
 /* Intel(R) Ethernet Controller E810-XXV for backplane */
 #define ICE_DEV_ID_E810_XXV_BACKPLANE	0x1599
 /* Intel(R) Ethernet Controller E810-XXV for QSFP */
diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c
index 3337314a7b35..e6ec20079ced 100644
--- a/drivers/net/ethernet/intel/ice/ice_devlink.c
+++ b/drivers/net/ethernet/intel/ice/ice_devlink.c
@@ -9,6 +9,8 @@
 #include "ice_eswitch.h"
 #include "ice_fw_update.h"
 
+static int ice_active_port_option = -1;
+
 /* context for devlink info version reporting */
 struct ice_info_ctx {
 	char buf[128];
@@ -466,12 +468,259 @@ ice_devlink_reload_empr_finish(struct devlink *devlink,
 	return 0;
 }
 
+/**
+ * ice_devlink_port_opt_speed_str - convert speed to a string
+ * @speed: speed value
+ */
+static const char *ice_devlink_port_opt_speed_str(u8 speed)
+{
+	switch (speed & ICE_AQC_PORT_OPT_MAX_LANE_M) {
+	case ICE_AQC_PORT_OPT_MAX_LANE_100M:
+		return "0.1";
+	case ICE_AQC_PORT_OPT_MAX_LANE_1G:
+		return "1";
+	case ICE_AQC_PORT_OPT_MAX_LANE_2500M:
+		return "2.5";
+	case ICE_AQC_PORT_OPT_MAX_LANE_5G:
+		return "5";
+	case ICE_AQC_PORT_OPT_MAX_LANE_10G:
+		return "10";
+	case ICE_AQC_PORT_OPT_MAX_LANE_25G:
+		return "25";
+	case ICE_AQC_PORT_OPT_MAX_LANE_50G:
+		return "50";
+	case ICE_AQC_PORT_OPT_MAX_LANE_100G:
+		return "100";
+	}
+
+	return "-";
+}
+
+#define ICE_PORT_OPT_DESC_LEN	50
+/**
+ * ice_devlink_port_options_print - Print available port split options
+ * @pf: the PF to print split port options
+ *
+ * Prints a table with available port split options and max port speeds
+ */
+static void ice_devlink_port_options_print(struct ice_pf *pf)
+{
+	u8 i, j, options_count, cnt, speed, pending_idx, active_idx;
+	struct ice_aqc_get_port_options_elem *options, *opt;
+	struct device *dev = ice_pf_to_dev(pf);
+	bool active_valid, pending_valid;
+	char desc[ICE_PORT_OPT_DESC_LEN];
+	const char *str;
+	int status;
+
+	options = kcalloc(ICE_AQC_PORT_OPT_MAX * ICE_MAX_PORT_PER_PCI_DEV,
+			  sizeof(*options), GFP_KERNEL);
+	if (!options)
+		return;
+
+	for (i = 0; i < ICE_MAX_PORT_PER_PCI_DEV; i++) {
+		opt = options + i * ICE_AQC_PORT_OPT_MAX;
+		options_count = ICE_AQC_PORT_OPT_MAX;
+		active_valid = 0;
+
+		status = ice_aq_get_port_options(&pf->hw, opt, &options_count,
+						 i, true, &active_idx,
+						 &active_valid, &pending_idx,
+						 &pending_valid);
+		if (status) {
+			dev_dbg(dev, "Couldn't read port option for port %d, err %d\n",
+				i, status);
+			goto err;
+		}
+	}
+
+	dev_dbg(dev, "Available port split options and max port speeds (Gbps):\n");
+	dev_dbg(dev, "Status  Split      Quad 0          Quad 1\n");
+	dev_dbg(dev, "        count  L0  L1  L2  L3  L4  L5  L6  L7\n");
+
+	for (i = 0; i < options_count; i++) {
+		cnt = 0;
+
+		if (i == ice_active_port_option)
+			str = "Active";
+		else if ((i == pending_idx) && pending_valid)
+			str = "Pending";
+		else
+			str = "";
+
+		cnt += snprintf(&desc[cnt], ICE_PORT_OPT_DESC_LEN - cnt,
+				"%-8s", str);
+
+		cnt += snprintf(&desc[cnt], ICE_PORT_OPT_DESC_LEN - cnt,
+				"%-6u", options[i].pmd);
+
+		for (j = 0; j < ICE_MAX_PORT_PER_PCI_DEV; ++j) {
+			speed = options[i + j * ICE_AQC_PORT_OPT_MAX].max_lane_speed;
+			str = ice_devlink_port_opt_speed_str(speed);
+			cnt += snprintf(&desc[cnt], ICE_PORT_OPT_DESC_LEN - cnt,
+					"%3s ", str);
+		}
+
+		dev_dbg(dev, "%s\n", desc);
+	}
+
+err:
+	kfree(options);
+}
+
+/**
+ * ice_devlink_aq_set_port_option - Send set port option admin queue command
+ * @pf: the PF to print split port options
+ * @option_idx: selected port option
+ * @extack: extended netdev ack structure
+ *
+ * Sends set port option admin queue command with selected port option and
+ * calls NVM write activate.
+ */
+static int
+ice_devlink_aq_set_port_option(struct ice_pf *pf, u8 option_idx,
+			       struct netlink_ext_ack *extack)
+{
+	struct device *dev = ice_pf_to_dev(pf);
+	int status;
+
+	status = ice_aq_set_port_option(&pf->hw, 0, true, option_idx);
+	if (status) {
+		dev_dbg(dev, "ice_aq_set_port_option, err %d aq_err %d\n",
+			status, pf->hw.adminq.sq_last_status);
+		NL_SET_ERR_MSG_MOD(extack, "Port split request failed");
+		return -EIO;
+	}
+
+	status = ice_acquire_nvm(&pf->hw, ICE_RES_WRITE);
+	if (status) {
+		dev_dbg(dev, "ice_acquire_nvm failed, err %d aq_err %d\n",
+			status, pf->hw.adminq.sq_last_status);
+		NL_SET_ERR_MSG_MOD(extack, "Failed to acquire NVM semaphore");
+		return -EIO;
+	}
+
+	status = ice_nvm_write_activate(&pf->hw, ICE_AQC_NVM_ACTIV_REQ_EMPR, NULL);
+	if (status) {
+		dev_dbg(dev, "ice_nvm_write_activate failed, err %d aq_err %d\n",
+			status, pf->hw.adminq.sq_last_status);
+		NL_SET_ERR_MSG_MOD(extack, "Port split request failed to save data");
+		ice_release_nvm(&pf->hw);
+		return -EIO;
+	}
+
+	ice_release_nvm(&pf->hw);
+
+	NL_SET_ERR_MSG_MOD(extack, "Reboot required to finish port split");
+	return 0;
+}
+
+/**
+ * ice_devlink_port_split - .port_split devlink handler
+ * @devlink: devlink instance structure
+ * @port: devlink port structure
+ * @count: number of ports to split to
+ * @extack: extended netdev ack structure
+ *
+ * Callback for the devlink .port_split operation.
+ *
+ * Unfortunately, the devlink expression of available options is limited
+ * to just a number, so search for an FW port option which supports
+ * the specified number. As there could be multiple FW port options with
+ * the same port split count, allow switching between them. When the same
+ * port split count request is issued again, switch to the next FW port
+ * option with the same port split count.
+ *
+ * Return: zero on success or an error code on failure.
+ */
+static int
+ice_devlink_port_split(struct devlink *devlink, struct devlink_port *port,
+		       unsigned int count, struct netlink_ext_ack *extack)
+{
+	struct ice_aqc_get_port_options_elem options[ICE_AQC_PORT_OPT_MAX];
+	u8 i, j, active_idx, pending_idx, new_option;
+	struct ice_pf *pf = devlink_priv(devlink);
+	u8 option_count = ICE_AQC_PORT_OPT_MAX;
+	struct device *dev = ice_pf_to_dev(pf);
+	bool active_valid, pending_valid;
+	int status;
+
+	status = ice_aq_get_port_options(&pf->hw, options, &option_count,
+					 0, true, &active_idx, &active_valid,
+					 &pending_idx, &pending_valid);
+	if (status) {
+		dev_dbg(dev, "Couldn't read port split options, err = %d\n",
+			status);
+		NL_SET_ERR_MSG_MOD(extack, "Failed to get available port split options");
+		return -EIO;
+	}
+
+	new_option = ICE_AQC_PORT_OPT_MAX;
+	active_idx = pending_valid ? pending_idx : active_idx;
+	for (i = 1; i <= option_count; i++) {
+		/* In order to allow switching between FW port options with
+		 * the same port split count, search for a new option starting
+		 * from the active/pending option (with array wrap around).
+		 */
+		j = (active_idx + i) % option_count;
+
+		if (count == options[j].pmd) {
+			new_option = j;
+			break;
+		}
+	}
+
+	if (new_option == active_idx) {
+		dev_dbg(dev, "request to split: count: %u is already set and there are no other options\n",
+			count);
+		NL_SET_ERR_MSG_MOD(extack, "Requested split count is already set");
+		ice_devlink_port_options_print(pf);
+		return -EINVAL;
+	}
+
+	if (new_option == ICE_AQC_PORT_OPT_MAX) {
+		dev_dbg(dev, "request to split: count: %u not found\n", count);
+		NL_SET_ERR_MSG_MOD(extack, "Port split requested unsupported port config");
+		ice_devlink_port_options_print(pf);
+		return -EINVAL;
+	}
+
+	status = ice_devlink_aq_set_port_option(pf, new_option, extack);
+	if (status)
+		return status;
+
+	ice_devlink_port_options_print(pf);
+
+	return 0;
+}
+
+/**
+ * ice_devlink_port_unsplit - .port_unsplit devlink handler
+ * @devlink: devlink instance structure
+ * @port: devlink port structure
+ * @extack: extended netdev ack structure
+ *
+ * Callback for the devlink .port_unsplit operation.
+ * Calls ice_devlink_port_split with split count set to 1.
+ * There could be no FW option available with split count 1.
+ *
+ * Return: zero on success or an error code on failure.
+ */
+static int
+ice_devlink_port_unsplit(struct devlink *devlink, struct devlink_port *port,
+			 struct netlink_ext_ack *extack)
+{
+	return ice_devlink_port_split(devlink, port, 1, extack);
+}
+
 static const struct devlink_ops ice_devlink_ops = {
 	.supported_flash_update_params = DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK,
 	.reload_actions = BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE),
 	/* The ice driver currently does not support driver reinit */
 	.reload_down = ice_devlink_reload_empr_start,
 	.reload_up = ice_devlink_reload_empr_finish,
+	.port_split = ice_devlink_port_split,
+	.port_unsplit = ice_devlink_port_unsplit,
 	.eswitch_mode_get = ice_eswitch_mode_get,
 	.eswitch_mode_set = ice_eswitch_mode_set,
 	.info_get = ice_devlink_info_get,
@@ -695,6 +944,39 @@ void ice_devlink_unregister_params(struct ice_pf *pf)
 }
 
 /**
+ * ice_devlink_set_port_split_options - Set port split options
+ * @pf: the PF to set port split options
+ * @attrs: devlink attributes
+ *
+ * Sets devlink port split options based on available FW port options
+ */
+static void
+ice_devlink_set_port_split_options(struct ice_pf *pf,
+				   struct devlink_port_attrs *attrs)
+{
+	struct ice_aqc_get_port_options_elem options[ICE_AQC_PORT_OPT_MAX];
+	u8 i, active_idx, pending_idx, option_count = ICE_AQC_PORT_OPT_MAX;
+	bool active_valid, pending_valid;
+	int status;
+
+	status = ice_aq_get_port_options(&pf->hw, options, &option_count,
+					 0, true, &active_idx, &active_valid,
+					 &pending_idx, &pending_valid);
+	if (status) {
+		dev_dbg(ice_pf_to_dev(pf), "Couldn't read port split options, err = %d\n",
+			status);
+		return;
+	}
+
+	/* find the biggest available port split count */
+	for (i = 0; i < option_count; i++)
+		attrs->lanes = max_t(int, attrs->lanes, options[i].pmd);
+
+	attrs->splittable = attrs->lanes ? 1 : 0;
+	ice_active_port_option = active_idx;
+}
+
+/**
  * ice_devlink_create_pf_port - Create a devlink port for this PF
  * @pf: the PF to create a devlink port for
  *
@@ -722,6 +1004,12 @@ int ice_devlink_create_pf_port(struct ice_pf *pf)
 	attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
 	attrs.phys.port_number = pf->hw.bus.func;
 
+	/* As FW supports only port split options for whole device,
+	 * set port split options only for first PF.
+	 */
+	if (pf->hw.pf_id == 0)
+		ice_devlink_set_port_split_options(pf, &attrs);
+
 	ice_devlink_set_switch_id(pf, &attrs.switch_id);
 
 	devlink_port_attrs_set(devlink_port, &attrs);
diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.c b/drivers/net/ethernet/intel/ice/ice_eswitch.c
index e35371e61e07..f9f15acae90a 100644
--- a/drivers/net/ethernet/intel/ice/ice_eswitch.c
+++ b/drivers/net/ethernet/intel/ice/ice_eswitch.c
@@ -292,8 +292,8 @@ static int ice_eswitch_setup_reprs(struct ice_pf *pf)
 		if (max_vsi_num < vsi->vsi_num)
 			max_vsi_num = vsi->vsi_num;
 
-		netif_napi_add(vf->repr->netdev, &vf->repr->q_vector->napi, ice_napi_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(vf->repr->netdev, &vf->repr->q_vector->napi,
+			       ice_napi_poll);
 
 		netif_keep_dst(vf->repr->netdev);
 	}
diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c
index a6fff8ebaf9d..b7be84bbe72d 100644
--- a/drivers/net/ethernet/intel/ice/ice_ethtool.c
+++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c
@@ -136,6 +136,11 @@ static const struct ice_stats ice_gstrings_pf_stats[] = {
 	ICE_PF_STAT("mac_remote_faults.nic", stats.mac_remote_faults),
 	ICE_PF_STAT("fdir_sb_match.nic", stats.fd_sb_match),
 	ICE_PF_STAT("fdir_sb_status.nic", stats.fd_sb_status),
+	ICE_PF_STAT("tx_hwtstamp_skipped", ptp.tx_hwtstamp_skipped),
+	ICE_PF_STAT("tx_hwtstamp_timeouts", ptp.tx_hwtstamp_timeouts),
+	ICE_PF_STAT("tx_hwtstamp_flushed", ptp.tx_hwtstamp_flushed),
+	ICE_PF_STAT("tx_hwtstamp_discarded", ptp.tx_hwtstamp_discarded),
+	ICE_PF_STAT("late_cached_phc_updates", ptp.late_cached_phc_updates),
 };
 
 static const u32 ice_regs_dump_list[] = {
@@ -1284,10 +1289,7 @@ static int ice_set_priv_flags(struct net_device *netdev, u32 flags)
 	}
 	if (test_bit(ICE_FLAG_LEGACY_RX, change_flags)) {
 		/* down and up VSI so that changes of Rx cfg are reflected. */
-		if (!test_and_set_bit(ICE_VSI_DOWN, vsi->state)) {
-			ice_down(vsi);
-			ice_up(vsi);
-		}
+		ice_down_up(vsi);
 	}
 	/* don't allow modification of this flag when a single VF is in
 	 * promiscuous mode because it's not supported
@@ -1468,20 +1470,22 @@ ice_get_ethtool_stats(struct net_device *netdev,
 
 /**
  * ice_mask_min_supported_speeds
+ * @hw: pointer to the HW structure
  * @phy_types_high: PHY type high
  * @phy_types_low: PHY type low to apply minimum supported speeds mask
  *
  * Apply minimum supported speeds mask to PHY type low. These are the speeds
  * for ethtool supported link mode.
  */
-static
-void ice_mask_min_supported_speeds(u64 phy_types_high, u64 *phy_types_low)
+static void
+ice_mask_min_supported_speeds(struct ice_hw *hw,
+			      u64 phy_types_high, u64 *phy_types_low)
 {
 	/* if QSFP connection with 100G speed, minimum supported speed is 25G */
 	if (*phy_types_low & ICE_PHY_TYPE_LOW_MASK_100G ||
 	    phy_types_high & ICE_PHY_TYPE_HIGH_MASK_100G)
 		*phy_types_low &= ~ICE_PHY_TYPE_LOW_MASK_MIN_25G;
-	else
+	else if (!ice_is_100m_speed_supported(hw))
 		*phy_types_low &= ~ICE_PHY_TYPE_LOW_MASK_MIN_1G;
 }
 
@@ -1531,7 +1535,8 @@ ice_phy_type_to_ethtool(struct net_device *netdev,
 		phy_types_low = le64_to_cpu(pf->nvm_phy_type_lo);
 		phy_types_high = le64_to_cpu(pf->nvm_phy_type_hi);
 
-		ice_mask_min_supported_speeds(phy_types_high, &phy_types_low);
+		ice_mask_min_supported_speeds(&pf->hw, phy_types_high,
+					      &phy_types_low);
 		/* determine advertised modes based on link override only
 		 * if it's supported and if the FW doesn't abstract the
 		 * driver from having to account for link overrides
@@ -2826,6 +2831,7 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring,
 		tx_rings[i].count = new_tx_cnt;
 		tx_rings[i].desc = NULL;
 		tx_rings[i].tx_buf = NULL;
+		tx_rings[i].tx_tstamps = &pf->ptp.port.tx;
 		err = ice_setup_tx_ring(&tx_rings[i]);
 		if (err) {
 			while (i--)
@@ -2884,6 +2890,7 @@ process_rx:
 		/* clone ring and setup updated count */
 		rx_rings[i] = *vsi->rx_rings[i];
 		rx_rings[i].count = new_rx_cnt;
+		rx_rings[i].cached_phctime = pf->ptp.cached_phc_time;
 		rx_rings[i].desc = NULL;
 		rx_rings[i].rx_buf = NULL;
 		/* this is to allow wr32 to have something to write to
diff --git a/drivers/net/ethernet/intel/ice/ice_lag.c b/drivers/net/ethernet/intel/ice/ice_lag.c
index c9f7393b783d..ee5b36941ba3 100644
--- a/drivers/net/ethernet/intel/ice/ice_lag.c
+++ b/drivers/net/ethernet/intel/ice/ice_lag.c
@@ -61,13 +61,13 @@ static void ice_lag_set_backup(struct ice_lag *lag)
  */
 static void ice_display_lag_info(struct ice_lag *lag)
 {
-	const char *name, *peer, *upper, *role, *bonded, *master;
+	const char *name, *peer, *upper, *role, *bonded, *primary;
 	struct device *dev = &lag->pf->pdev->dev;
 
 	name = lag->netdev ? netdev_name(lag->netdev) : "unset";
 	peer = lag->peer_netdev ? netdev_name(lag->peer_netdev) : "unset";
 	upper = lag->upper_netdev ? netdev_name(lag->upper_netdev) : "unset";
-	master = lag->master ? "TRUE" : "FALSE";
+	primary = lag->primary ? "TRUE" : "FALSE";
 	bonded = lag->bonded ? "BONDED" : "UNBONDED";
 
 	switch (lag->role) {
@@ -87,8 +87,8 @@ static void ice_display_lag_info(struct ice_lag *lag)
 		role = "ERROR";
 	}
 
-	dev_dbg(dev, "%s %s, peer:%s, upper:%s, role:%s, master:%s\n", name,
-		bonded, peer, upper, role, master);
+	dev_dbg(dev, "%s %s, peer:%s, upper:%s, role:%s, primary:%s\n", name,
+		bonded, peer, upper, role, primary);
 }
 
 /**
@@ -119,7 +119,7 @@ static void ice_lag_info_event(struct ice_lag *lag, void *ptr)
 	}
 
 	if (strcmp(bonding_info->slave.slave_name, lag_netdev_name)) {
-		netdev_dbg(lag->netdev, "Bonding event recv, but slave info not for us\n");
+		netdev_dbg(lag->netdev, "Bonding event recv, but secondary info not for us\n");
 		goto lag_out;
 	}
 
@@ -164,8 +164,8 @@ ice_lag_link(struct ice_lag *lag, struct netdev_notifier_changeupper_info *info)
 	lag->bonded = true;
 	lag->role = ICE_LAG_UNSET;
 
-	/* if this is the first element in an LAG mark as master */
-	lag->master = !!(peers == 1);
+	/* if this is the first element in an LAG mark as primary */
+	lag->primary = !!(peers == 1);
 }
 
 /**
@@ -264,7 +264,7 @@ static void ice_lag_changeupper_event(struct ice_lag *lag, void *ptr)
 	netdev_dbg(netdev, "bonding %s\n", info->linking ? "LINK" : "UNLINK");
 
 	if (!netif_is_lag_master(info->upper_dev)) {
-		netdev_dbg(netdev, "changeupper rcvd, but not master. bail\n");
+		netdev_dbg(netdev, "changeupper rcvd, but not primary. bail\n");
 		return;
 	}
 
diff --git a/drivers/net/ethernet/intel/ice/ice_lag.h b/drivers/net/ethernet/intel/ice/ice_lag.h
index c2e3688dd8fd..51b5cf467ce2 100644
--- a/drivers/net/ethernet/intel/ice/ice_lag.h
+++ b/drivers/net/ethernet/intel/ice/ice_lag.h
@@ -24,7 +24,7 @@ struct ice_lag {
 	struct net_device *upper_netdev; /* upper bonding netdev */
 	struct notifier_block notif_block;
 	u8 bonded:1; /* currently bonded */
-	u8 master:1; /* this is a master */
+	u8 primary:1; /* this is primary */
 	u8 handler:1; /* did we register a rx_netdev_handler */
 	/* each thing blocking bonding will increment this value by one.
 	 * If this value is zero, then bonding is allowed.
diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c
index 58d483e2f539..938ba8c215cb 100644
--- a/drivers/net/ethernet/intel/ice/ice_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_lib.c
@@ -1524,6 +1524,7 @@ static int ice_vsi_alloc_rings(struct ice_vsi *vsi)
 		ring->netdev = vsi->netdev;
 		ring->dev = dev;
 		ring->count = vsi->num_rx_desc;
+		ring->cached_phctime = pf->ptp.cached_phc_time;
 		WRITE_ONCE(vsi->rx_rings[i], ring);
 	}
 
@@ -1564,6 +1565,22 @@ void ice_vsi_manage_rss_lut(struct ice_vsi *vsi, bool ena)
 }
 
 /**
+ * ice_vsi_cfg_crc_strip - Configure CRC stripping for a VSI
+ * @vsi: VSI to be configured
+ * @disable: set to true to have FCS / CRC in the frame data
+ */
+void ice_vsi_cfg_crc_strip(struct ice_vsi *vsi, bool disable)
+{
+	int i;
+
+	ice_for_each_rxq(vsi, i)
+		if (disable)
+			vsi->rx_rings[i]->flags |= ICE_RX_FLAGS_CRC_STRIP_DIS;
+		else
+			vsi->rx_rings[i]->flags &= ~ICE_RX_FLAGS_CRC_STRIP_DIS;
+}
+
+/**
  * ice_vsi_cfg_rss_lut_key - Configure RSS params for a VSI
  * @vsi: VSI to be configured
  */
@@ -2971,9 +2988,6 @@ int ice_vsi_release(struct ice_vsi *vsi)
 		clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
 	}
 
-	if (vsi->type == ICE_VSI_PF)
-		ice_devlink_destroy_pf_port(pf);
-
 	if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
 		ice_rss_clean(vsi);
 
@@ -3031,6 +3045,9 @@ int ice_vsi_release(struct ice_vsi *vsi)
 		}
 	}
 
+	if (vsi->type == ICE_VSI_PF)
+		ice_devlink_destroy_pf_port(pf);
+
 	if (vsi->type == ICE_VSI_VF &&
 	    vsi->agg_node && vsi->agg_node->valid)
 		vsi->agg_node->num_vsis--;
@@ -3278,6 +3295,12 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, bool init_vsi)
 			 */
 			if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
 				ice_vsi_cfg_rss_lut_key(vsi);
+
+		/* disable or enable CRC stripping */
+		if (vsi->netdev)
+			ice_vsi_cfg_crc_strip(vsi, !!(vsi->netdev->features &
+					      NETIF_F_RXFCS));
+
 		break;
 	case ICE_VSI_VF:
 		ret = ice_vsi_alloc_q_vectors(vsi);
diff --git a/drivers/net/ethernet/intel/ice/ice_lib.h b/drivers/net/ethernet/intel/ice/ice_lib.h
index 8712b1d2ceec..ec4bf0c89857 100644
--- a/drivers/net/ethernet/intel/ice/ice_lib.h
+++ b/drivers/net/ethernet/intel/ice/ice_lib.h
@@ -89,6 +89,8 @@ void ice_vsi_free_tx_rings(struct ice_vsi *vsi);
 
 void ice_vsi_manage_rss_lut(struct ice_vsi *vsi, bool ena);
 
+void ice_vsi_cfg_crc_strip(struct ice_vsi *vsi, bool disable);
+
 void ice_update_tx_ring_stats(struct ice_tx_ring *ring, u64 pkts, u64 bytes);
 
 void ice_update_rx_ring_stats(struct ice_rx_ring *ring, u64 pkts, u64 bytes);
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index e109cb93886b..0f6718719453 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -3093,7 +3093,8 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
 
 	if (oicr & PFINT_OICR_TSYN_TX_M) {
 		ena_mask &= ~PFINT_OICR_TSYN_TX_M;
-		ice_ptp_process_ts(pf);
+		if (!hw->reset_ongoing)
+			ret = IRQ_WAKE_THREAD;
 	}
 
 	if (oicr & PFINT_OICR_TSYN_EVNT_M) {
@@ -3128,7 +3129,8 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
 			ice_service_task_schedule(pf);
 		}
 	}
-	ret = IRQ_HANDLED;
+	if (!ret)
+		ret = IRQ_HANDLED;
 
 	ice_service_task_schedule(pf);
 	ice_irq_dynamic_ena(hw, NULL, NULL);
@@ -3137,6 +3139,24 @@ static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
 }
 
 /**
+ * ice_misc_intr_thread_fn - misc interrupt thread function
+ * @irq: interrupt number
+ * @data: pointer to a q_vector
+ */
+static irqreturn_t ice_misc_intr_thread_fn(int __always_unused irq, void *data)
+{
+	irqreturn_t ret = IRQ_HANDLED;
+	struct ice_pf *pf = data;
+	bool irq_handled;
+
+	irq_handled = ice_ptp_process_ts(pf);
+	if (!irq_handled)
+		ret = IRQ_WAKE_THREAD;
+
+	return ret;
+}
+
+/**
  * ice_dis_ctrlq_interrupts - disable control queue interrupts
  * @hw: pointer to HW structure
  */
@@ -3248,10 +3268,12 @@ static int ice_req_irq_msix_misc(struct ice_pf *pf)
 	pf->num_avail_sw_msix -= 1;
 	pf->oicr_idx = (u16)oicr_idx;
 
-	err = devm_request_irq(dev, pf->msix_entries[pf->oicr_idx].vector,
-			       ice_misc_intr, 0, pf->int_name, pf);
+	err = devm_request_threaded_irq(dev,
+					pf->msix_entries[pf->oicr_idx].vector,
+					ice_misc_intr, ice_misc_intr_thread_fn,
+					0, pf->int_name, pf);
 	if (err) {
-		dev_err(dev, "devm_request_irq for %s failed: %d\n",
+		dev_err(dev, "devm_request_threaded_irq for %s failed: %d\n",
 			pf->int_name, err);
 		ice_free_res(pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID);
 		pf->num_avail_sw_msix += 1;
@@ -3288,7 +3310,7 @@ static void ice_napi_add(struct ice_vsi *vsi)
 
 	ice_for_each_q_vector(vsi, v_idx)
 		netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi,
-			       ice_napi_poll, NAPI_POLL_WEIGHT);
+			       ice_napi_poll);
 }
 
 /**
@@ -3391,6 +3413,11 @@ static void ice_set_netdev_features(struct net_device *netdev)
 	if (is_dvm_ena)
 		netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX |
 			NETIF_F_HW_VLAN_STAG_TX;
+
+	/* Leave CRC / FCS stripping enabled by default, but allow the value to
+	 * be changed at runtime
+	 */
+	netdev->hw_features |= NETIF_F_RXFCS;
 }
 
 /**
@@ -3923,87 +3950,134 @@ static int ice_init_pf(struct ice_pf *pf)
 }
 
 /**
+ * ice_reduce_msix_usage - Reduce usage of MSI-X vectors
+ * @pf: board private structure
+ * @v_remain: number of remaining MSI-X vectors to be distributed
+ *
+ * Reduce the usage of MSI-X vectors when entire request cannot be fulfilled.
+ * pf->num_lan_msix and pf->num_rdma_msix values are set based on number of
+ * remaining vectors.
+ */
+static void ice_reduce_msix_usage(struct ice_pf *pf, int v_remain)
+{
+	int v_rdma;
+
+	if (!ice_is_rdma_ena(pf)) {
+		pf->num_lan_msix = v_remain;
+		return;
+	}
+
+	/* RDMA needs at least 1 interrupt in addition to AEQ MSIX */
+	v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1;
+
+	if (v_remain < ICE_MIN_LAN_TXRX_MSIX + ICE_MIN_RDMA_MSIX) {
+		dev_warn(ice_pf_to_dev(pf), "Not enough MSI-X vectors to support RDMA.\n");
+		clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
+
+		pf->num_rdma_msix = 0;
+		pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX;
+	} else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) ||
+		   (v_remain - v_rdma < v_rdma)) {
+		/* Support minimum RDMA and give remaining vectors to LAN MSIX */
+		pf->num_rdma_msix = ICE_MIN_RDMA_MSIX;
+		pf->num_lan_msix = v_remain - ICE_MIN_RDMA_MSIX;
+	} else {
+		/* Split remaining MSIX with RDMA after accounting for AEQ MSIX
+		 */
+		pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 +
+				    ICE_RDMA_NUM_AEQ_MSIX;
+		pf->num_lan_msix = v_remain - pf->num_rdma_msix;
+	}
+}
+
+/**
  * ice_ena_msix_range - Request a range of MSIX vectors from the OS
  * @pf: board private structure
  *
- * compute the number of MSIX vectors required (v_budget) and request from
- * the OS. Return the number of vectors reserved or negative on failure
+ * Compute the number of MSIX vectors wanted and request from the OS. Adjust
+ * device usage if there are not enough vectors. Return the number of vectors
+ * reserved or negative on failure.
  */
 static int ice_ena_msix_range(struct ice_pf *pf)
 {
-	int num_cpus, v_left, v_actual, v_other, v_budget = 0;
+	int num_cpus, hw_num_msix, v_other, v_wanted, v_actual;
 	struct device *dev = ice_pf_to_dev(pf);
-	int needed, err, i;
+	int err, i;
 
-	v_left = pf->hw.func_caps.common_cap.num_msix_vectors;
+	hw_num_msix = pf->hw.func_caps.common_cap.num_msix_vectors;
 	num_cpus = num_online_cpus();
 
-	/* reserve for LAN miscellaneous handler */
-	needed = ICE_MIN_LAN_OICR_MSIX;
-	if (v_left < needed)
-		goto no_hw_vecs_left_err;
-	v_budget += needed;
-	v_left -= needed;
+	/* LAN miscellaneous handler */
+	v_other = ICE_MIN_LAN_OICR_MSIX;
 
-	/* reserve for flow director */
-	if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) {
-		needed = ICE_FDIR_MSIX;
-		if (v_left < needed)
-			goto no_hw_vecs_left_err;
-		v_budget += needed;
-		v_left -= needed;
-	}
-
-	/* reserve for switchdev */
-	needed = ICE_ESWITCH_MSIX;
-	if (v_left < needed)
-		goto no_hw_vecs_left_err;
-	v_budget += needed;
-	v_left -= needed;
-
-	/* total used for non-traffic vectors */
-	v_other = v_budget;
-
-	/* reserve vectors for LAN traffic */
-	needed = num_cpus;
-	if (v_left < needed)
-		goto no_hw_vecs_left_err;
-	pf->num_lan_msix = needed;
-	v_budget += needed;
-	v_left -= needed;
-
-	/* reserve vectors for RDMA auxiliary driver */
+	/* Flow Director */
+	if (test_bit(ICE_FLAG_FD_ENA, pf->flags))
+		v_other += ICE_FDIR_MSIX;
+
+	/* switchdev */
+	v_other += ICE_ESWITCH_MSIX;
+
+	v_wanted = v_other;
+
+	/* LAN traffic */
+	pf->num_lan_msix = num_cpus;
+	v_wanted += pf->num_lan_msix;
+
+	/* RDMA auxiliary driver */
 	if (ice_is_rdma_ena(pf)) {
-		needed = num_cpus + ICE_RDMA_NUM_AEQ_MSIX;
-		if (v_left < needed)
-			goto no_hw_vecs_left_err;
-		pf->num_rdma_msix = needed;
-		v_budget += needed;
-		v_left -= needed;
+		pf->num_rdma_msix = num_cpus + ICE_RDMA_NUM_AEQ_MSIX;
+		v_wanted += pf->num_rdma_msix;
 	}
 
-	pf->msix_entries = devm_kcalloc(dev, v_budget,
+	if (v_wanted > hw_num_msix) {
+		int v_remain;
+
+		dev_warn(dev, "not enough device MSI-X vectors. wanted = %d, available = %d\n",
+			 v_wanted, hw_num_msix);
+
+		if (hw_num_msix < ICE_MIN_MSIX) {
+			err = -ERANGE;
+			goto exit_err;
+		}
+
+		v_remain = hw_num_msix - v_other;
+		if (v_remain < ICE_MIN_LAN_TXRX_MSIX) {
+			v_other = ICE_MIN_MSIX - ICE_MIN_LAN_TXRX_MSIX;
+			v_remain = ICE_MIN_LAN_TXRX_MSIX;
+		}
+
+		ice_reduce_msix_usage(pf, v_remain);
+		v_wanted = pf->num_lan_msix + pf->num_rdma_msix + v_other;
+
+		dev_notice(dev, "Reducing request to %d MSI-X vectors for LAN traffic.\n",
+			   pf->num_lan_msix);
+		if (ice_is_rdma_ena(pf))
+			dev_notice(dev, "Reducing request to %d MSI-X vectors for RDMA.\n",
+				   pf->num_rdma_msix);
+	}
+
+	pf->msix_entries = devm_kcalloc(dev, v_wanted,
 					sizeof(*pf->msix_entries), GFP_KERNEL);
 	if (!pf->msix_entries) {
 		err = -ENOMEM;
 		goto exit_err;
 	}
 
-	for (i = 0; i < v_budget; i++)
+	for (i = 0; i < v_wanted; i++)
 		pf->msix_entries[i].entry = i;
 
 	/* actually reserve the vectors */
 	v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries,
-					 ICE_MIN_MSIX, v_budget);
+					 ICE_MIN_MSIX, v_wanted);
 	if (v_actual < 0) {
 		dev_err(dev, "unable to reserve MSI-X vectors\n");
 		err = v_actual;
 		goto msix_err;
 	}
 
-	if (v_actual < v_budget) {
+	if (v_actual < v_wanted) {
 		dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n",
-			 v_budget, v_actual);
+			 v_wanted, v_actual);
 
 		if (v_actual < ICE_MIN_MSIX) {
 			/* error if we can't get minimum vectors */
@@ -4012,38 +4086,11 @@ static int ice_ena_msix_range(struct ice_pf *pf)
 			goto msix_err;
 		} else {
 			int v_remain = v_actual - v_other;
-			int v_rdma = 0, v_min_rdma = 0;
 
-			if (ice_is_rdma_ena(pf)) {
-				/* Need at least 1 interrupt in addition to
-				 * AEQ MSIX
-				 */
-				v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1;
-				v_min_rdma = ICE_MIN_RDMA_MSIX;
-			}
+			if (v_remain < ICE_MIN_LAN_TXRX_MSIX)
+				v_remain = ICE_MIN_LAN_TXRX_MSIX;
 
-			if (v_actual == ICE_MIN_MSIX ||
-			    v_remain < ICE_MIN_LAN_TXRX_MSIX + v_min_rdma) {
-				dev_warn(dev, "Not enough MSI-X vectors to support RDMA.\n");
-				clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
-
-				pf->num_rdma_msix = 0;
-				pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX;
-			} else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) ||
-				   (v_remain - v_rdma < v_rdma)) {
-				/* Support minimum RDMA and give remaining
-				 * vectors to LAN MSIX
-				 */
-				pf->num_rdma_msix = v_min_rdma;
-				pf->num_lan_msix = v_remain - v_min_rdma;
-			} else {
-				/* Split remaining MSIX with RDMA after
-				 * accounting for AEQ MSIX
-				 */
-				pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 +
-						    ICE_RDMA_NUM_AEQ_MSIX;
-				pf->num_lan_msix = v_remain - pf->num_rdma_msix;
-			}
+			ice_reduce_msix_usage(pf, v_remain);
 
 			dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n",
 				   pf->num_lan_msix);
@@ -4058,12 +4105,7 @@ static int ice_ena_msix_range(struct ice_pf *pf)
 
 msix_err:
 	devm_kfree(dev, pf->msix_entries);
-	goto exit_err;
 
-no_hw_vecs_left_err:
-	dev_err(dev, "not enough device MSI-X vectors. requested = %d, available = %d\n",
-		needed, v_left);
-	err = -ERANGE;
 exit_err:
 	pf->num_rdma_msix = 0;
 	pf->num_lan_msix = 0;
@@ -4557,6 +4599,10 @@ static int ice_register_netdev(struct ice_pf *pf)
 	if (!vsi || !vsi->netdev)
 		return -EIO;
 
+	err = ice_devlink_create_pf_port(pf);
+	if (err)
+		goto err_devlink_create;
+
 	err = register_netdev(vsi->netdev);
 	if (err)
 		goto err_register_netdev;
@@ -4564,17 +4610,13 @@ static int ice_register_netdev(struct ice_pf *pf)
 	set_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
 	netif_carrier_off(vsi->netdev);
 	netif_tx_stop_all_queues(vsi->netdev);
-	err = ice_devlink_create_pf_port(pf);
-	if (err)
-		goto err_devlink_create;
 
 	devlink_port_type_eth_set(&pf->devlink_port, vsi->netdev);
 
 	return 0;
-err_devlink_create:
-	unregister_netdev(vsi->netdev);
-	clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
 err_register_netdev:
+	ice_devlink_destroy_pf_port(pf);
+err_devlink_create:
 	free_netdev(vsi->netdev);
 	vsi->netdev = NULL;
 	clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state);
@@ -4682,8 +4724,6 @@ ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
 		ice_set_safe_mode_caps(hw);
 	}
 
-	hw->ucast_shared = true;
-
 	err = ice_init_pf(pf);
 	if (err) {
 		dev_err(dev, "ice_init_pf failed: %d\n", err);
@@ -5742,6 +5782,9 @@ ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[],
 					 NETIF_F_HW_VLAN_STAG_RX | \
 					 NETIF_F_HW_VLAN_STAG_TX)
 
+#define NETIF_VLAN_STRIPPING_FEATURES	(NETIF_F_HW_VLAN_CTAG_RX | \
+					 NETIF_F_HW_VLAN_STAG_RX)
+
 #define NETIF_VLAN_FILTERING_FEATURES	(NETIF_F_HW_VLAN_CTAG_FILTER | \
 					 NETIF_F_HW_VLAN_STAG_FILTER)
 
@@ -5828,6 +5871,14 @@ ice_fix_features(struct net_device *netdev, netdev_features_t features)
 			      NETIF_F_HW_VLAN_STAG_TX);
 	}
 
+	if (!(netdev->features & NETIF_F_RXFCS) &&
+	    (features & NETIF_F_RXFCS) &&
+	    (features & NETIF_VLAN_STRIPPING_FEATURES) &&
+	    !ice_vsi_has_non_zero_vlans(np->vsi)) {
+		netdev_warn(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n");
+		features &= ~NETIF_VLAN_STRIPPING_FEATURES;
+	}
+
 	return features;
 }
 
@@ -5921,6 +5972,13 @@ ice_set_vlan_features(struct net_device *netdev, netdev_features_t features)
 	current_vlan_features = netdev->features & NETIF_VLAN_OFFLOAD_FEATURES;
 	requested_vlan_features = features & NETIF_VLAN_OFFLOAD_FEATURES;
 	if (current_vlan_features ^ requested_vlan_features) {
+		if ((features & NETIF_F_RXFCS) &&
+		    (features & NETIF_VLAN_STRIPPING_FEATURES)) {
+			dev_err(ice_pf_to_dev(vsi->back),
+				"To enable VLAN stripping, you must first enable FCS/CRC stripping\n");
+			return -EIO;
+		}
+
 		err = ice_set_vlan_offload_features(vsi, features);
 		if (err)
 			return err;
@@ -6002,6 +6060,23 @@ ice_set_features(struct net_device *netdev, netdev_features_t features)
 	if (ret)
 		return ret;
 
+	/* Turn on receive of FCS aka CRC, and after setting this
+	 * flag the packet data will have the 4 byte CRC appended
+	 */
+	if (changed & NETIF_F_RXFCS) {
+		if ((features & NETIF_F_RXFCS) &&
+		    (features & NETIF_VLAN_STRIPPING_FEATURES)) {
+			dev_err(ice_pf_to_dev(vsi->back),
+				"To disable FCS/CRC stripping, you must first disable VLAN stripping\n");
+			return -EIO;
+		}
+
+		ice_vsi_cfg_crc_strip(vsi, !!(features & NETIF_F_RXFCS));
+		ret = ice_down_up(vsi);
+		if (ret)
+			return ret;
+	}
+
 	if (changed & NETIF_F_NTUPLE) {
 		bool ena = !!(features & NETIF_F_NTUPLE);
 
@@ -6699,6 +6774,31 @@ int ice_down(struct ice_vsi *vsi)
 }
 
 /**
+ * ice_down_up - shutdown the VSI connection and bring it up
+ * @vsi: the VSI to be reconnected
+ */
+int ice_down_up(struct ice_vsi *vsi)
+{
+	int ret;
+
+	/* if DOWN already set, nothing to do */
+	if (test_and_set_bit(ICE_VSI_DOWN, vsi->state))
+		return 0;
+
+	ret = ice_down(vsi);
+	if (ret)
+		return ret;
+
+	ret = ice_up(vsi);
+	if (ret) {
+		netdev_err(vsi->netdev, "reallocating resources failed during netdev features change, may need to reload driver\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+/**
  * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources
  * @vsi: VSI having resources allocated
  *
diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.c b/drivers/net/ethernet/intel/ice/ice_nvm.c
index 13cdb5ea594d..c262dc886e6a 100644
--- a/drivers/net/ethernet/intel/ice/ice_nvm.c
+++ b/drivers/net/ethernet/intel/ice/ice_nvm.c
@@ -1114,14 +1114,18 @@ int ice_nvm_validate_checksum(struct ice_hw *hw)
  * Update the control word with the required banks' validity bits
  * and dumps the Shadow RAM to flash (0x0707)
  *
- * cmd_flags controls which banks to activate, and the preservation level to
- * use when activating the NVM bank.
+ * cmd_flags controls which banks to activate, the preservation level to use
+ * when activating the NVM bank, and whether an EMP reset is required for
+ * activation.
+ *
+ * Note that the 16bit cmd_flags value is split between two separate 1 byte
+ * flag values in the descriptor.
  *
  * On successful return of the firmware command, the response_flags variable
  * is updated with the flags reported by firmware indicating certain status,
  * such as whether EMP reset is enabled.
  */
-int ice_nvm_write_activate(struct ice_hw *hw, u8 cmd_flags, u8 *response_flags)
+int ice_nvm_write_activate(struct ice_hw *hw, u16 cmd_flags, u8 *response_flags)
 {
 	struct ice_aqc_nvm *cmd;
 	struct ice_aq_desc desc;
@@ -1130,7 +1134,8 @@ int ice_nvm_write_activate(struct ice_hw *hw, u8 cmd_flags, u8 *response_flags)
 	cmd = &desc.params.nvm;
 	ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_nvm_write_activate);
 
-	cmd->cmd_flags = cmd_flags;
+	cmd->cmd_flags = (u8)(cmd_flags & 0xFF);
+	cmd->offset_high = (u8)((cmd_flags >> 8) & 0xFF);
 
 	err = ice_aq_send_cmd(hw, &desc, NULL, 0, NULL);
 	if (!err && response_flags)
diff --git a/drivers/net/ethernet/intel/ice/ice_nvm.h b/drivers/net/ethernet/intel/ice/ice_nvm.h
index 856d1ad4398b..774c2317967d 100644
--- a/drivers/net/ethernet/intel/ice/ice_nvm.h
+++ b/drivers/net/ethernet/intel/ice/ice_nvm.h
@@ -34,7 +34,7 @@ ice_aq_update_nvm(struct ice_hw *hw, u16 module_typeid, u32 offset,
 int
 ice_aq_erase_nvm(struct ice_hw *hw, u16 module_typeid, struct ice_sq_cd *cd);
 int ice_nvm_validate_checksum(struct ice_hw *hw);
-int ice_nvm_write_activate(struct ice_hw *hw, u8 cmd_flags, u8 *response_flags);
+int ice_nvm_write_activate(struct ice_hw *hw, u16 cmd_flags, u8 *response_flags);
 int ice_aq_nvm_update_empr(struct ice_hw *hw);
 int
 ice_nvm_set_pkg_data(struct ice_hw *hw, bool del_pkg_data_flag, u8 *data,
diff --git a/drivers/net/ethernet/intel/ice/ice_protocol_type.h b/drivers/net/ethernet/intel/ice/ice_protocol_type.h
index 560efc7654c7..02a4e1cf624e 100644
--- a/drivers/net/ethernet/intel/ice/ice_protocol_type.h
+++ b/drivers/net/ethernet/intel/ice/ice_protocol_type.h
@@ -44,6 +44,7 @@ enum ice_protocol_type {
 	ICE_GTP,
 	ICE_GTP_NO_PAY,
 	ICE_PPPOE,
+	ICE_L2TPV3,
 	ICE_VLAN_EX,
 	ICE_VLAN_IN,
 	ICE_VXLAN_GPE,
@@ -111,6 +112,7 @@ enum ice_prot_id {
 #define ICE_UDP_ILOS_HW		53
 #define ICE_GRE_OF_HW		64
 #define ICE_PPPOE_HW		103
+#define ICE_L2TPV3_HW		104
 
 #define ICE_UDP_OF_HW	52 /* UDP Tunnels */
 #define ICE_META_DATA_ID_HW 255 /* this is used for tunnel and VLAN type */
@@ -217,6 +219,11 @@ struct ice_pppoe_hdr {
 	__be16 ppp_prot_id; /* control and data only */
 };
 
+struct ice_l2tpv3_sess_hdr {
+	__be32 session_id;
+	__be64 cookie;
+};
+
 struct ice_nvgre_hdr {
 	__be16 flags;
 	__be16 protocol;
@@ -235,6 +242,7 @@ union ice_prot_hdr {
 	struct ice_nvgre_hdr nvgre_hdr;
 	struct ice_udp_gtp_hdr gtp_hdr;
 	struct ice_pppoe_hdr pppoe_hdr;
+	struct ice_l2tpv3_sess_hdr l2tpv3_sess_hdr;
 };
 
 /* This is mapping table entry that maps every word within a given protocol
diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.c b/drivers/net/ethernet/intel/ice/ice_ptp.c
index 72b663108a4a..011b727ab190 100644
--- a/drivers/net/ethernet/intel/ice/ice_ptp.c
+++ b/drivers/net/ethernet/intel/ice/ice_ptp.c
@@ -491,56 +491,6 @@ ice_ptp_read_src_clk_reg(struct ice_pf *pf, struct ptp_system_timestamp *sts)
 }
 
 /**
- * ice_ptp_update_cached_phctime - Update the cached PHC time values
- * @pf: Board specific private structure
- *
- * This function updates the system time values which are cached in the PF
- * structure and the Rx rings.
- *
- * This function must be called periodically to ensure that the cached value
- * is never more than 2 seconds old. It must also be called whenever the PHC
- * time has been changed.
- *
- * Return:
- * * 0 - OK, successfully updated
- * * -EAGAIN - PF was busy, need to reschedule the update
- */
-static int ice_ptp_update_cached_phctime(struct ice_pf *pf)
-{
-	u64 systime;
-	int i;
-
-	if (test_and_set_bit(ICE_CFG_BUSY, pf->state))
-		return -EAGAIN;
-
-	/* Read the current PHC time */
-	systime = ice_ptp_read_src_clk_reg(pf, NULL);
-
-	/* Update the cached PHC time stored in the PF structure */
-	WRITE_ONCE(pf->ptp.cached_phc_time, systime);
-
-	ice_for_each_vsi(pf, i) {
-		struct ice_vsi *vsi = pf->vsi[i];
-		int j;
-
-		if (!vsi)
-			continue;
-
-		if (vsi->type != ICE_VSI_PF)
-			continue;
-
-		ice_for_each_rxq(vsi, j) {
-			if (!vsi->rx_rings[j])
-				continue;
-			WRITE_ONCE(vsi->rx_rings[j]->cached_phctime, systime);
-		}
-	}
-	clear_bit(ICE_CFG_BUSY, pf->state);
-
-	return 0;
-}
-
-/**
  * ice_ptp_extend_32b_ts - Convert a 32b nanoseconds timestamp to 64b
  * @cached_phc_time: recently cached copy of PHC time
  * @in_tstamp: Ingress/egress 32b nanoseconds timestamp value
@@ -636,12 +586,400 @@ static u64 ice_ptp_extend_32b_ts(u64 cached_phc_time, u32 in_tstamp)
 static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp)
 {
 	const u64 mask = GENMASK_ULL(31, 0);
+	unsigned long discard_time;
+
+	/* Discard the hardware timestamp if the cached PHC time is too old */
+	discard_time = pf->ptp.cached_phc_jiffies + msecs_to_jiffies(2000);
+	if (time_is_before_jiffies(discard_time)) {
+		pf->ptp.tx_hwtstamp_discarded++;
+		return 0;
+	}
 
 	return ice_ptp_extend_32b_ts(pf->ptp.cached_phc_time,
 				     (in_tstamp >> 8) & mask);
 }
 
 /**
+ * ice_ptp_tx_tstamp - Process Tx timestamps for a port
+ * @tx: the PTP Tx timestamp tracker
+ *
+ * Process timestamps captured by the PHY associated with this port. To do
+ * this, loop over each index with a waiting skb.
+ *
+ * If a given index has a valid timestamp, perform the following steps:
+ *
+ * 1) copy the timestamp out of the PHY register
+ * 4) clear the timestamp valid bit in the PHY register
+ * 5) unlock the index by clearing the associated in_use bit.
+ * 2) extend the 40b timestamp value to get a 64bit timestamp
+ * 3) send that timestamp to the stack
+ *
+ * After looping, if we still have waiting SKBs, return true. This may cause us
+ * effectively poll even when not strictly necessary. We do this because it's
+ * possible a new timestamp was requested around the same time as the interrupt.
+ * In some cases hardware might not interrupt us again when the timestamp is
+ * captured.
+ *
+ * Note that we only take the tracking lock when clearing the bit and when
+ * checking if we need to re-queue this task. The only place where bits can be
+ * set is the hard xmit routine where an SKB has a request flag set. The only
+ * places where we clear bits are this work function, or the periodic cleanup
+ * thread. If the cleanup thread clears a bit we're processing we catch it
+ * when we lock to clear the bit and then grab the SKB pointer. If a Tx thread
+ * starts a new timestamp, we might not begin processing it right away but we
+ * will notice it at the end when we re-queue the task. If a Tx thread starts
+ * a new timestamp just after this function exits without re-queuing,
+ * the interrupt when the timestamp finishes should trigger. Avoiding holding
+ * the lock for the entire function is important in order to ensure that Tx
+ * threads do not get blocked while waiting for the lock.
+ */
+static bool ice_ptp_tx_tstamp(struct ice_ptp_tx *tx)
+{
+	struct ice_ptp_port *ptp_port;
+	bool ts_handled = true;
+	struct ice_pf *pf;
+	u8 idx;
+
+	if (!tx->init)
+		return false;
+
+	ptp_port = container_of(tx, struct ice_ptp_port, tx);
+	pf = ptp_port_to_pf(ptp_port);
+
+	for_each_set_bit(idx, tx->in_use, tx->len) {
+		struct skb_shared_hwtstamps shhwtstamps = {};
+		u8 phy_idx = idx + tx->quad_offset;
+		u64 raw_tstamp, tstamp;
+		struct sk_buff *skb;
+		int err;
+
+		ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx);
+
+		err = ice_read_phy_tstamp(&pf->hw, tx->quad, phy_idx,
+					  &raw_tstamp);
+		if (err)
+			continue;
+
+		ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx);
+
+		/* Check if the timestamp is invalid or stale */
+		if (!(raw_tstamp & ICE_PTP_TS_VALID) ||
+		    raw_tstamp == tx->tstamps[idx].cached_tstamp)
+			continue;
+
+		/* The timestamp is valid, so we'll go ahead and clear this
+		 * index and then send the timestamp up to the stack.
+		 */
+		spin_lock(&tx->lock);
+		tx->tstamps[idx].cached_tstamp = raw_tstamp;
+		clear_bit(idx, tx->in_use);
+		skb = tx->tstamps[idx].skb;
+		tx->tstamps[idx].skb = NULL;
+		spin_unlock(&tx->lock);
+
+		/* it's (unlikely but) possible we raced with the cleanup
+		 * thread for discarding old timestamp requests.
+		 */
+		if (!skb)
+			continue;
+
+		/* Extend the timestamp using cached PHC time */
+		tstamp = ice_ptp_extend_40b_ts(pf, raw_tstamp);
+		if (tstamp) {
+			shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
+			ice_trace(tx_tstamp_complete, skb, idx);
+		}
+
+		skb_tstamp_tx(skb, &shhwtstamps);
+		dev_kfree_skb_any(skb);
+	}
+
+	/* Check if we still have work to do. If so, re-queue this task to
+	 * poll for remaining timestamps.
+	 */
+	spin_lock(&tx->lock);
+	if (!bitmap_empty(tx->in_use, tx->len))
+		ts_handled = false;
+	spin_unlock(&tx->lock);
+
+	return ts_handled;
+}
+
+/**
+ * ice_ptp_alloc_tx_tracker - Initialize tracking for Tx timestamps
+ * @tx: Tx tracking structure to initialize
+ *
+ * Assumes that the length has already been initialized. Do not call directly,
+ * use the ice_ptp_init_tx_e822 or ice_ptp_init_tx_e810 instead.
+ */
+static int
+ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx)
+{
+	tx->tstamps = kcalloc(tx->len, sizeof(*tx->tstamps), GFP_KERNEL);
+	if (!tx->tstamps)
+		return -ENOMEM;
+
+	tx->in_use = bitmap_zalloc(tx->len, GFP_KERNEL);
+	if (!tx->in_use) {
+		kfree(tx->tstamps);
+		tx->tstamps = NULL;
+		return -ENOMEM;
+	}
+
+	spin_lock_init(&tx->lock);
+
+	tx->init = 1;
+
+	return 0;
+}
+
+/**
+ * ice_ptp_flush_tx_tracker - Flush any remaining timestamps from the tracker
+ * @pf: Board private structure
+ * @tx: the tracker to flush
+ */
+static void
+ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)
+{
+	u8 idx;
+
+	for (idx = 0; idx < tx->len; idx++) {
+		u8 phy_idx = idx + tx->quad_offset;
+
+		spin_lock(&tx->lock);
+		if (tx->tstamps[idx].skb) {
+			dev_kfree_skb_any(tx->tstamps[idx].skb);
+			tx->tstamps[idx].skb = NULL;
+			pf->ptp.tx_hwtstamp_flushed++;
+		}
+		clear_bit(idx, tx->in_use);
+		spin_unlock(&tx->lock);
+
+		/* Clear any potential residual timestamp in the PHY block */
+		if (!pf->hw.reset_ongoing)
+			ice_clear_phy_tstamp(&pf->hw, tx->quad, phy_idx);
+	}
+}
+
+/**
+ * ice_ptp_release_tx_tracker - Release allocated memory for Tx tracker
+ * @pf: Board private structure
+ * @tx: Tx tracking structure to release
+ *
+ * Free memory associated with the Tx timestamp tracker.
+ */
+static void
+ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)
+{
+	tx->init = 0;
+
+	ice_ptp_flush_tx_tracker(pf, tx);
+
+	kfree(tx->tstamps);
+	tx->tstamps = NULL;
+
+	bitmap_free(tx->in_use);
+	tx->in_use = NULL;
+
+	tx->len = 0;
+}
+
+/**
+ * ice_ptp_init_tx_e822 - Initialize tracking for Tx timestamps
+ * @pf: Board private structure
+ * @tx: the Tx tracking structure to initialize
+ * @port: the port this structure tracks
+ *
+ * Initialize the Tx timestamp tracker for this port. For generic MAC devices,
+ * the timestamp block is shared for all ports in the same quad. To avoid
+ * ports using the same timestamp index, logically break the block of
+ * registers into chunks based on the port number.
+ */
+static int
+ice_ptp_init_tx_e822(struct ice_pf *pf, struct ice_ptp_tx *tx, u8 port)
+{
+	tx->quad = port / ICE_PORTS_PER_QUAD;
+	tx->quad_offset = (port % ICE_PORTS_PER_QUAD) * INDEX_PER_PORT;
+	tx->len = INDEX_PER_PORT;
+
+	return ice_ptp_alloc_tx_tracker(tx);
+}
+
+/**
+ * ice_ptp_init_tx_e810 - Initialize tracking for Tx timestamps
+ * @pf: Board private structure
+ * @tx: the Tx tracking structure to initialize
+ *
+ * Initialize the Tx timestamp tracker for this PF. For E810 devices, each
+ * port has its own block of timestamps, independent of the other ports.
+ */
+static int
+ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx)
+{
+	tx->quad = pf->hw.port_info->lport;
+	tx->quad_offset = 0;
+	tx->len = INDEX_PER_QUAD;
+
+	return ice_ptp_alloc_tx_tracker(tx);
+}
+
+/**
+ * ice_ptp_tx_tstamp_cleanup - Cleanup old timestamp requests that got dropped
+ * @pf: pointer to the PF struct
+ * @tx: PTP Tx tracker to clean up
+ *
+ * Loop through the Tx timestamp requests and see if any of them have been
+ * waiting for a long time. Discard any SKBs that have been waiting for more
+ * than 2 seconds. This is long enough to be reasonably sure that the
+ * timestamp will never be captured. This might happen if the packet gets
+ * discarded before it reaches the PHY timestamping block.
+ */
+static void ice_ptp_tx_tstamp_cleanup(struct ice_pf *pf, struct ice_ptp_tx *tx)
+{
+	struct ice_hw *hw = &pf->hw;
+	u8 idx;
+
+	if (!tx->init)
+		return;
+
+	for_each_set_bit(idx, tx->in_use, tx->len) {
+		struct sk_buff *skb;
+		u64 raw_tstamp;
+
+		/* Check if this SKB has been waiting for too long */
+		if (time_is_after_jiffies(tx->tstamps[idx].start + 2 * HZ))
+			continue;
+
+		/* Read tstamp to be able to use this register again */
+		ice_read_phy_tstamp(hw, tx->quad, idx + tx->quad_offset,
+				    &raw_tstamp);
+
+		spin_lock(&tx->lock);
+		skb = tx->tstamps[idx].skb;
+		tx->tstamps[idx].skb = NULL;
+		clear_bit(idx, tx->in_use);
+		spin_unlock(&tx->lock);
+
+		/* Count the number of Tx timestamps which have timed out */
+		pf->ptp.tx_hwtstamp_timeouts++;
+
+		/* Free the SKB after we've cleared the bit */
+		dev_kfree_skb_any(skb);
+	}
+}
+
+/**
+ * ice_ptp_update_cached_phctime - Update the cached PHC time values
+ * @pf: Board specific private structure
+ *
+ * This function updates the system time values which are cached in the PF
+ * structure and the Rx rings.
+ *
+ * This function must be called periodically to ensure that the cached value
+ * is never more than 2 seconds old.
+ *
+ * Note that the cached copy in the PF PTP structure is always updated, even
+ * if we can't update the copy in the Rx rings.
+ *
+ * Return:
+ * * 0 - OK, successfully updated
+ * * -EAGAIN - PF was busy, need to reschedule the update
+ */
+static int ice_ptp_update_cached_phctime(struct ice_pf *pf)
+{
+	struct device *dev = ice_pf_to_dev(pf);
+	unsigned long update_before;
+	u64 systime;
+	int i;
+
+	update_before = pf->ptp.cached_phc_jiffies + msecs_to_jiffies(2000);
+	if (pf->ptp.cached_phc_time &&
+	    time_is_before_jiffies(update_before)) {
+		unsigned long time_taken = jiffies - pf->ptp.cached_phc_jiffies;
+
+		dev_warn(dev, "%u msecs passed between update to cached PHC time\n",
+			 jiffies_to_msecs(time_taken));
+		pf->ptp.late_cached_phc_updates++;
+	}
+
+	/* Read the current PHC time */
+	systime = ice_ptp_read_src_clk_reg(pf, NULL);
+
+	/* Update the cached PHC time stored in the PF structure */
+	WRITE_ONCE(pf->ptp.cached_phc_time, systime);
+	WRITE_ONCE(pf->ptp.cached_phc_jiffies, jiffies);
+
+	if (test_and_set_bit(ICE_CFG_BUSY, pf->state))
+		return -EAGAIN;
+
+	ice_for_each_vsi(pf, i) {
+		struct ice_vsi *vsi = pf->vsi[i];
+		int j;
+
+		if (!vsi)
+			continue;
+
+		if (vsi->type != ICE_VSI_PF)
+			continue;
+
+		ice_for_each_rxq(vsi, j) {
+			if (!vsi->rx_rings[j])
+				continue;
+			WRITE_ONCE(vsi->rx_rings[j]->cached_phctime, systime);
+		}
+	}
+	clear_bit(ICE_CFG_BUSY, pf->state);
+
+	return 0;
+}
+
+/**
+ * ice_ptp_reset_cached_phctime - Reset cached PHC time after an update
+ * @pf: Board specific private structure
+ *
+ * This function must be called when the cached PHC time is no longer valid,
+ * such as after a time adjustment. It discards any outstanding Tx timestamps,
+ * and updates the cached PHC time for both the PF and Rx rings. If updating
+ * the PHC time cannot be done immediately, a warning message is logged and
+ * the work item is scheduled.
+ *
+ * These steps are required in order to ensure that we do not accidentally
+ * report a timestamp extended by the wrong PHC cached copy. Note that we
+ * do not directly update the cached timestamp here because it is possible
+ * this might produce an error when ICE_CFG_BUSY is set. If this occurred, we
+ * would have to try again. During that time window, timestamps might be
+ * requested and returned with an invalid extension. Thus, on failure to
+ * immediately update the cached PHC time we would need to zero the value
+ * anyways. For this reason, we just zero the value immediately and queue the
+ * update work item.
+ */
+static void ice_ptp_reset_cached_phctime(struct ice_pf *pf)
+{
+	struct device *dev = ice_pf_to_dev(pf);
+	int err;
+
+	/* Update the cached PHC time immediately if possible, otherwise
+	 * schedule the work item to execute soon.
+	 */
+	err = ice_ptp_update_cached_phctime(pf);
+	if (err) {
+		/* If another thread is updating the Rx rings, we won't
+		 * properly reset them here. This could lead to reporting of
+		 * invalid timestamps, but there isn't much we can do.
+		 */
+		dev_warn(dev, "%s: ICE_CFG_BUSY, unable to immediately update cached PHC time\n",
+			 __func__);
+
+		/* Queue the work item to update the Rx rings when possible */
+		kthread_queue_delayed_work(pf->ptp.kworker, &pf->ptp.work,
+					   msecs_to_jiffies(10));
+	}
+
+	/* Flush any outstanding Tx timestamps */
+	ice_ptp_flush_tx_tracker(pf, &pf->ptp.port.tx);
+}
+
+/**
  * ice_ptp_read_time - Read the time from the device
  * @pf: Board private structure
  * @ts: timespec structure to hold the current time value
@@ -900,6 +1238,9 @@ static void ice_ptp_wait_for_offset_valid(struct kthread_work *work)
 	hw = &pf->hw;
 	dev = ice_pf_to_dev(pf);
 
+	if (ice_is_reset_in_progress(pf->state))
+		return;
+
 	if (ice_ptp_check_offset_valid(port)) {
 		/* Offsets not ready yet, try again later */
 		kthread_queue_delayed_work(pf->ptp.kworker,
@@ -1509,7 +1850,7 @@ ice_ptp_settime64(struct ptp_clock_info *info, const struct timespec64 *ts)
 	ice_ptp_unlock(hw);
 
 	if (!err)
-		ice_ptp_update_cached_phctime(pf);
+		ice_ptp_reset_cached_phctime(pf);
 
 	/* Reenable periodic outputs */
 	ice_ptp_enable_all_clkout(pf);
@@ -1588,7 +1929,7 @@ static int ice_ptp_adjtime(struct ptp_clock_info *info, s64 delta)
 		return err;
 	}
 
-	ice_ptp_update_cached_phctime(pf);
+	ice_ptp_reset_cached_phctime(pf);
 
 	return 0;
 }
@@ -1796,26 +2137,31 @@ void
 ice_ptp_rx_hwtstamp(struct ice_rx_ring *rx_ring,
 		    union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb)
 {
+	struct skb_shared_hwtstamps *hwtstamps;
+	u64 ts_ns, cached_time;
 	u32 ts_high;
-	u64 ts_ns;
 
-	/* Populate timesync data into skb */
-	if (rx_desc->wb.time_stamp_low & ICE_PTP_TS_VALID) {
-		struct skb_shared_hwtstamps *hwtstamps;
+	if (!(rx_desc->wb.time_stamp_low & ICE_PTP_TS_VALID))
+		return;
 
-		/* Use ice_ptp_extend_32b_ts directly, using the ring-specific
-		 * cached PHC value, rather than accessing the PF. This also
-		 * allows us to simply pass the upper 32bits of nanoseconds
-		 * directly. Calling ice_ptp_extend_40b_ts is unnecessary as
-		 * it would just discard these bits itself.
-		 */
-		ts_high = le32_to_cpu(rx_desc->wb.flex_ts.ts_high);
-		ts_ns = ice_ptp_extend_32b_ts(rx_ring->cached_phctime, ts_high);
+	cached_time = READ_ONCE(rx_ring->cached_phctime);
 
-		hwtstamps = skb_hwtstamps(skb);
-		memset(hwtstamps, 0, sizeof(*hwtstamps));
-		hwtstamps->hwtstamp = ns_to_ktime(ts_ns);
-	}
+	/* Do not report a timestamp if we don't have a cached PHC time */
+	if (!cached_time)
+		return;
+
+	/* Use ice_ptp_extend_32b_ts directly, using the ring-specific cached
+	 * PHC value, rather than accessing the PF. This also allows us to
+	 * simply pass the upper 32bits of nanoseconds directly. Calling
+	 * ice_ptp_extend_40b_ts is unnecessary as it would just discard these
+	 * bits itself.
+	 */
+	ts_high = le32_to_cpu(rx_desc->wb.flex_ts.ts_high);
+	ts_ns = ice_ptp_extend_32b_ts(cached_time, ts_high);
+
+	hwtstamps = skb_hwtstamps(skb);
+	memset(hwtstamps, 0, sizeof(*hwtstamps));
+	hwtstamps->hwtstamp = ns_to_ktime(ts_ns);
 }
 
 /**
@@ -1871,49 +2217,26 @@ ice_ptp_setup_sma_pins_e810t(struct ice_pf *pf, struct ptp_clock_info *info)
 }
 
 /**
- * ice_ptp_setup_pins_e810t - Setup PTP pins in sysfs
+ * ice_ptp_setup_pins_e810 - Setup PTP pins in sysfs
  * @pf: pointer to the PF instance
  * @info: PTP clock capabilities
  */
 static void
-ice_ptp_setup_pins_e810t(struct ice_pf *pf, struct ptp_clock_info *info)
+ice_ptp_setup_pins_e810(struct ice_pf *pf, struct ptp_clock_info *info)
 {
-	/* Check if SMA controller is in the netlist */
-	if (ice_is_feature_supported(pf, ICE_F_SMA_CTRL) &&
-	    !ice_is_pca9575_present(&pf->hw))
-		ice_clear_feature_support(pf, ICE_F_SMA_CTRL);
-
-	if (!ice_is_feature_supported(pf, ICE_F_SMA_CTRL)) {
-		info->n_ext_ts = N_EXT_TS_E810_NO_SMA;
-		info->n_per_out = N_PER_OUT_E810T_NO_SMA;
-		return;
-	}
+	info->n_per_out = N_PER_OUT_E810;
 
-	info->n_per_out = N_PER_OUT_E810T;
+	if (ice_is_feature_supported(pf, ICE_F_PTP_EXTTS))
+		info->n_ext_ts = N_EXT_TS_E810;
 
-	if (ice_is_feature_supported(pf, ICE_F_PTP_EXTTS)) {
+	if (ice_is_feature_supported(pf, ICE_F_SMA_CTRL)) {
 		info->n_ext_ts = N_EXT_TS_E810;
 		info->n_pins = NUM_PTP_PINS_E810T;
 		info->verify = ice_verify_pin_e810t;
-	}
 
-	/* Complete setup of the SMA pins */
-	ice_ptp_setup_sma_pins_e810t(pf, info);
-}
-
-/**
- * ice_ptp_setup_pins_e810 - Setup PTP pins in sysfs
- * @pf: pointer to the PF instance
- * @info: PTP clock capabilities
- */
-static void ice_ptp_setup_pins_e810(struct ice_pf *pf, struct ptp_clock_info *info)
-{
-	info->n_per_out = N_PER_OUT_E810;
-
-	if (!ice_is_feature_supported(pf, ICE_F_PTP_EXTTS))
-		return;
-
-	info->n_ext_ts = N_EXT_TS_E810;
+		/* Complete setup of the SMA pins */
+		ice_ptp_setup_sma_pins_e810t(pf, info);
+	}
 }
 
 /**
@@ -1950,11 +2273,7 @@ static void
 ice_ptp_set_funcs_e810(struct ice_pf *pf, struct ptp_clock_info *info)
 {
 	info->enable = ice_ptp_gpio_enable_e810;
-
-	if (ice_is_e810t(&pf->hw))
-		ice_ptp_setup_pins_e810t(pf, info);
-	else
-		ice_ptp_setup_pins_e810(pf, info);
+	ice_ptp_setup_pins_e810(pf, info);
 }
 
 /**
@@ -2016,112 +2335,6 @@ static long ice_ptp_create_clock(struct ice_pf *pf)
 }
 
 /**
- * ice_ptp_tx_tstamp_work - Process Tx timestamps for a port
- * @work: pointer to the kthread_work struct
- *
- * Process timestamps captured by the PHY associated with this port. To do
- * this, loop over each index with a waiting skb.
- *
- * If a given index has a valid timestamp, perform the following steps:
- *
- * 1) copy the timestamp out of the PHY register
- * 4) clear the timestamp valid bit in the PHY register
- * 5) unlock the index by clearing the associated in_use bit.
- * 2) extend the 40b timestamp value to get a 64bit timestamp
- * 3) send that timestamp to the stack
- *
- * After looping, if we still have waiting SKBs, then re-queue the work. This
- * may cause us effectively poll even when not strictly necessary. We do this
- * because it's possible a new timestamp was requested around the same time as
- * the interrupt. In some cases hardware might not interrupt us again when the
- * timestamp is captured.
- *
- * Note that we only take the tracking lock when clearing the bit and when
- * checking if we need to re-queue this task. The only place where bits can be
- * set is the hard xmit routine where an SKB has a request flag set. The only
- * places where we clear bits are this work function, or the periodic cleanup
- * thread. If the cleanup thread clears a bit we're processing we catch it
- * when we lock to clear the bit and then grab the SKB pointer. If a Tx thread
- * starts a new timestamp, we might not begin processing it right away but we
- * will notice it at the end when we re-queue the work item. If a Tx thread
- * starts a new timestamp just after this function exits without re-queuing,
- * the interrupt when the timestamp finishes should trigger. Avoiding holding
- * the lock for the entire function is important in order to ensure that Tx
- * threads do not get blocked while waiting for the lock.
- */
-static void ice_ptp_tx_tstamp_work(struct kthread_work *work)
-{
-	struct ice_ptp_port *ptp_port;
-	struct ice_ptp_tx *tx;
-	struct ice_pf *pf;
-	struct ice_hw *hw;
-	u8 idx;
-
-	tx = container_of(work, struct ice_ptp_tx, work);
-	if (!tx->init)
-		return;
-
-	ptp_port = container_of(tx, struct ice_ptp_port, tx);
-	pf = ptp_port_to_pf(ptp_port);
-	hw = &pf->hw;
-
-	for_each_set_bit(idx, tx->in_use, tx->len) {
-		struct skb_shared_hwtstamps shhwtstamps = {};
-		u8 phy_idx = idx + tx->quad_offset;
-		u64 raw_tstamp, tstamp;
-		struct sk_buff *skb;
-		int err;
-
-		ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx);
-
-		err = ice_read_phy_tstamp(hw, tx->quad, phy_idx,
-					  &raw_tstamp);
-		if (err)
-			continue;
-
-		ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx);
-
-		/* Check if the timestamp is invalid or stale */
-		if (!(raw_tstamp & ICE_PTP_TS_VALID) ||
-		    raw_tstamp == tx->tstamps[idx].cached_tstamp)
-			continue;
-
-		/* The timestamp is valid, so we'll go ahead and clear this
-		 * index and then send the timestamp up to the stack.
-		 */
-		spin_lock(&tx->lock);
-		tx->tstamps[idx].cached_tstamp = raw_tstamp;
-		clear_bit(idx, tx->in_use);
-		skb = tx->tstamps[idx].skb;
-		tx->tstamps[idx].skb = NULL;
-		spin_unlock(&tx->lock);
-
-		/* it's (unlikely but) possible we raced with the cleanup
-		 * thread for discarding old timestamp requests.
-		 */
-		if (!skb)
-			continue;
-
-		/* Extend the timestamp using cached PHC time */
-		tstamp = ice_ptp_extend_40b_ts(pf, raw_tstamp);
-		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
-
-		ice_trace(tx_tstamp_complete, skb, idx);
-
-		skb_tstamp_tx(skb, &shhwtstamps);
-		dev_kfree_skb_any(skb);
-	}
-
-	/* Check if we still have work to do. If so, re-queue this task to
-	 * poll for remaining timestamps.
-	 */
-	spin_lock(&tx->lock);
-	if (!bitmap_empty(tx->in_use, tx->len))
-		kthread_queue_work(pf->ptp.kworker, &tx->work);
-	spin_unlock(&tx->lock);
-}
-
-/**
  * ice_ptp_request_ts - Request an available Tx timestamp index
  * @tx: the PTP Tx timestamp tracker to request from
  * @skb: the SKB to associate with this timestamp request
@@ -2161,177 +2374,17 @@ s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb)
 }
 
 /**
- * ice_ptp_process_ts - Spawn kthread work to handle timestamps
+ * ice_ptp_process_ts - Process the PTP Tx timestamps
  * @pf: Board private structure
  *
- * Queue work required to process the PTP Tx timestamps outside of interrupt
- * context.
+ * Returns true if timestamps are processed.
  */
-void ice_ptp_process_ts(struct ice_pf *pf)
+bool ice_ptp_process_ts(struct ice_pf *pf)
 {
 	if (pf->ptp.port.tx.init)
-		kthread_queue_work(pf->ptp.kworker, &pf->ptp.port.tx.work);
-}
-
-/**
- * ice_ptp_alloc_tx_tracker - Initialize tracking for Tx timestamps
- * @tx: Tx tracking structure to initialize
- *
- * Assumes that the length has already been initialized. Do not call directly,
- * use the ice_ptp_init_tx_e822 or ice_ptp_init_tx_e810 instead.
- */
-static int
-ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx)
-{
-	tx->tstamps = kcalloc(tx->len, sizeof(*tx->tstamps), GFP_KERNEL);
-	if (!tx->tstamps)
-		return -ENOMEM;
-
-	tx->in_use = bitmap_zalloc(tx->len, GFP_KERNEL);
-	if (!tx->in_use) {
-		kfree(tx->tstamps);
-		tx->tstamps = NULL;
-		return -ENOMEM;
-	}
-
-	spin_lock_init(&tx->lock);
-	kthread_init_work(&tx->work, ice_ptp_tx_tstamp_work);
-
-	tx->init = 1;
-
-	return 0;
-}
-
-/**
- * ice_ptp_flush_tx_tracker - Flush any remaining timestamps from the tracker
- * @pf: Board private structure
- * @tx: the tracker to flush
- */
-static void
-ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)
-{
-	u8 idx;
-
-	for (idx = 0; idx < tx->len; idx++) {
-		u8 phy_idx = idx + tx->quad_offset;
-
-		spin_lock(&tx->lock);
-		if (tx->tstamps[idx].skb) {
-			dev_kfree_skb_any(tx->tstamps[idx].skb);
-			tx->tstamps[idx].skb = NULL;
-		}
-		clear_bit(idx, tx->in_use);
-		spin_unlock(&tx->lock);
-
-		/* Clear any potential residual timestamp in the PHY block */
-		if (!pf->hw.reset_ongoing)
-			ice_clear_phy_tstamp(&pf->hw, tx->quad, phy_idx);
-	}
-}
-
-/**
- * ice_ptp_release_tx_tracker - Release allocated memory for Tx tracker
- * @pf: Board private structure
- * @tx: Tx tracking structure to release
- *
- * Free memory associated with the Tx timestamp tracker.
- */
-static void
-ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)
-{
-	tx->init = 0;
-
-	kthread_cancel_work_sync(&tx->work);
-
-	ice_ptp_flush_tx_tracker(pf, tx);
-
-	kfree(tx->tstamps);
-	tx->tstamps = NULL;
-
-	bitmap_free(tx->in_use);
-	tx->in_use = NULL;
+		return ice_ptp_tx_tstamp(&pf->ptp.port.tx);
 
-	tx->len = 0;
-}
-
-/**
- * ice_ptp_init_tx_e822 - Initialize tracking for Tx timestamps
- * @pf: Board private structure
- * @tx: the Tx tracking structure to initialize
- * @port: the port this structure tracks
- *
- * Initialize the Tx timestamp tracker for this port. For generic MAC devices,
- * the timestamp block is shared for all ports in the same quad. To avoid
- * ports using the same timestamp index, logically break the block of
- * registers into chunks based on the port number.
- */
-static int
-ice_ptp_init_tx_e822(struct ice_pf *pf, struct ice_ptp_tx *tx, u8 port)
-{
-	tx->quad = port / ICE_PORTS_PER_QUAD;
-	tx->quad_offset = (port % ICE_PORTS_PER_QUAD) * INDEX_PER_PORT;
-	tx->len = INDEX_PER_PORT;
-
-	return ice_ptp_alloc_tx_tracker(tx);
-}
-
-/**
- * ice_ptp_init_tx_e810 - Initialize tracking for Tx timestamps
- * @pf: Board private structure
- * @tx: the Tx tracking structure to initialize
- *
- * Initialize the Tx timestamp tracker for this PF. For E810 devices, each
- * port has its own block of timestamps, independent of the other ports.
- */
-static int
-ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx)
-{
-	tx->quad = pf->hw.port_info->lport;
-	tx->quad_offset = 0;
-	tx->len = INDEX_PER_QUAD;
-
-	return ice_ptp_alloc_tx_tracker(tx);
-}
-
-/**
- * ice_ptp_tx_tstamp_cleanup - Cleanup old timestamp requests that got dropped
- * @hw: pointer to the hw struct
- * @tx: PTP Tx tracker to clean up
- *
- * Loop through the Tx timestamp requests and see if any of them have been
- * waiting for a long time. Discard any SKBs that have been waiting for more
- * than 2 seconds. This is long enough to be reasonably sure that the
- * timestamp will never be captured. This might happen if the packet gets
- * discarded before it reaches the PHY timestamping block.
- */
-static void ice_ptp_tx_tstamp_cleanup(struct ice_hw *hw, struct ice_ptp_tx *tx)
-{
-	u8 idx;
-
-	if (!tx->init)
-		return;
-
-	for_each_set_bit(idx, tx->in_use, tx->len) {
-		struct sk_buff *skb;
-		u64 raw_tstamp;
-
-		/* Check if this SKB has been waiting for too long */
-		if (time_is_after_jiffies(tx->tstamps[idx].start + 2 * HZ))
-			continue;
-
-		/* Read tstamp to be able to use this register again */
-		ice_read_phy_tstamp(hw, tx->quad, idx + tx->quad_offset,
-				    &raw_tstamp);
-
-		spin_lock(&tx->lock);
-		skb = tx->tstamps[idx].skb;
-		tx->tstamps[idx].skb = NULL;
-		clear_bit(idx, tx->in_use);
-		spin_unlock(&tx->lock);
-
-		/* Free the SKB after we've cleared the bit */
-		dev_kfree_skb_any(skb);
-	}
+	return false;
 }
 
 static void ice_ptp_periodic_work(struct kthread_work *work)
@@ -2345,7 +2398,7 @@ static void ice_ptp_periodic_work(struct kthread_work *work)
 
 	err = ice_ptp_update_cached_phctime(pf);
 
-	ice_ptp_tx_tstamp_cleanup(&pf->hw, &pf->ptp.port.tx);
+	ice_ptp_tx_tstamp_cleanup(pf, &pf->ptp.port.tx);
 
 	/* Run twice a second or reschedule if phc update failed */
 	kthread_queue_delayed_work(ptp->kworker, &ptp->work,
diff --git a/drivers/net/ethernet/intel/ice/ice_ptp.h b/drivers/net/ethernet/intel/ice/ice_ptp.h
index 10e396abf130..028349295b71 100644
--- a/drivers/net/ethernet/intel/ice/ice_ptp.h
+++ b/drivers/net/ethernet/intel/ice/ice_ptp.h
@@ -105,7 +105,6 @@ struct ice_tx_tstamp {
 
 /**
  * struct ice_ptp_tx - Tracking structure for all Tx timestamp requests on a port
- * @work: work function to handle processing of Tx timestamps
  * @lock: lock to prevent concurrent write to in_use bitmap
  * @tstamps: array of len to store outstanding requests
  * @in_use: bitmap of len to indicate which slots are in use
@@ -117,7 +116,6 @@ struct ice_tx_tstamp {
  *               window, timestamps are temporarily disabled.
  */
 struct ice_ptp_tx {
-	struct kthread_work work;
 	spinlock_t lock; /* lock protecting in_use bitmap */
 	struct ice_tx_tstamp *tstamps;
 	unsigned long *in_use;
@@ -163,6 +161,7 @@ struct ice_ptp_port {
  * @work: delayed work function for periodic tasks
  * @extts_work: work function for handling external Tx timestamps
  * @cached_phc_time: a cached copy of the PHC time for timestamp extension
+ * @cached_phc_jiffies: jiffies when cached_phc_time was last updated
  * @ext_ts_chan: the external timestamp channel in use
  * @ext_ts_irq: the external timestamp IRQ in use
  * @kworker: kwork thread for handling periodic work
@@ -171,12 +170,19 @@ struct ice_ptp_port {
  * @clock: pointer to registered PTP clock device
  * @tstamp_config: hardware timestamping configuration
  * @reset_time: kernel time after clock stop on reset
+ * @tx_hwtstamp_skipped: number of Tx time stamp requests skipped
+ * @tx_hwtstamp_timeouts: number of Tx skbs discarded with no time stamp
+ * @tx_hwtstamp_flushed: number of Tx skbs flushed due to interface closed
+ * @tx_hwtstamp_discarded: number of Tx skbs discarded due to cached PHC time
+ *                         being too old to correctly extend timestamp
+ * @late_cached_phc_updates: number of times cached PHC update is late
  */
 struct ice_ptp {
 	struct ice_ptp_port port;
 	struct kthread_delayed_work work;
 	struct kthread_work extts_work;
 	u64 cached_phc_time;
+	unsigned long cached_phc_jiffies;
 	u8 ext_ts_chan;
 	u8 ext_ts_irq;
 	struct kthread_worker *kworker;
@@ -185,6 +191,11 @@ struct ice_ptp {
 	struct ptp_clock *clock;
 	struct hwtstamp_config tstamp_config;
 	u64 reset_time;
+	u32 tx_hwtstamp_skipped;
+	u32 tx_hwtstamp_timeouts;
+	u32 tx_hwtstamp_flushed;
+	u32 tx_hwtstamp_discarded;
+	u32 late_cached_phc_updates;
 };
 
 #define __ptp_port_to_ptp(p) \
@@ -224,8 +235,8 @@ struct ice_ptp {
 #define N_EXT_TS_E810			3
 #define N_PER_OUT_E810			4
 #define N_PER_OUT_E810T			3
-#define N_PER_OUT_E810T_NO_SMA		2
-#define N_EXT_TS_E810_NO_SMA		2
+#define N_PER_OUT_NO_SMA_E810T		2
+#define N_EXT_TS_NO_SMA_E810T		2
 #define ETH_GLTSYN_ENA(_i)		(0x03000348 + ((_i) * 4))
 
 #if IS_ENABLED(CONFIG_PTP_1588_CLOCK)
@@ -236,7 +247,7 @@ void ice_ptp_cfg_timestamp(struct ice_pf *pf, bool ena);
 int ice_get_ptp_clock_index(struct ice_pf *pf);
 
 s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb);
-void ice_ptp_process_ts(struct ice_pf *pf);
+bool ice_ptp_process_ts(struct ice_pf *pf);
 
 void
 ice_ptp_rx_hwtstamp(struct ice_rx_ring *rx_ring,
@@ -269,7 +280,10 @@ ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb)
 	return -1;
 }
 
-static inline void ice_ptp_process_ts(struct ice_pf *pf) { }
+static inline bool ice_ptp_process_ts(struct ice_pf *pf)
+{
+	return true;
+}
 static inline void
 ice_ptp_rx_hwtstamp(struct ice_rx_ring *rx_ring,
 		    union ice_32b_rx_flex_desc *rx_desc, struct sk_buff *skb) { }
diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c
index 6dff97d53d81..772b1f566d6e 100644
--- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c
+++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright (C) 2021, Intel Corporation. */
 
+#include <linux/delay.h>
 #include "ice_common.h"
 #include "ice_ptp_hw.h"
 #include "ice_ptp_consts.h"
@@ -2587,38 +2588,113 @@ static int ice_write_phy_reg_e810(struct ice_hw *hw, u32 addr, u32 val)
 }
 
 /**
- * ice_read_phy_tstamp_e810 - Read a PHY timestamp out of the external PHY
+ * ice_read_phy_tstamp_ll_e810 - Read a PHY timestamp registers through the FW
+ * @hw: pointer to the HW struct
+ * @idx: the timestamp index to read
+ * @hi: 8 bit timestamp high value
+ * @lo: 32 bit timestamp low value
+ *
+ * Read a 8bit timestamp high value and 32 bit timestamp low value out of the
+ * timestamp block of the external PHY on the E810 device using the low latency
+ * timestamp read.
+ */
+static int
+ice_read_phy_tstamp_ll_e810(struct ice_hw *hw, u8 idx, u8 *hi, u32 *lo)
+{
+	u32 val;
+	u8 i;
+
+	/* Write TS index to read to the PF register so the FW can read it */
+	val = FIELD_PREP(TS_LL_READ_TS_IDX, idx) | TS_LL_READ_TS;
+	wr32(hw, PF_SB_ATQBAL, val);
+
+	/* Read the register repeatedly until the FW provides us the TS */
+	for (i = TS_LL_READ_RETRIES; i > 0; i--) {
+		val = rd32(hw, PF_SB_ATQBAL);
+
+		/* When the bit is cleared, the TS is ready in the register */
+		if (!(FIELD_GET(TS_LL_READ_TS, val))) {
+			/* High 8 bit value of the TS is on the bits 16:23 */
+			*hi = FIELD_GET(TS_LL_READ_TS_HIGH, val);
+
+			/* Read the low 32 bit value and set the TS valid bit */
+			*lo = rd32(hw, PF_SB_ATQBAH) | TS_VALID;
+			return 0;
+		}
+
+		udelay(10);
+	}
+
+	/* FW failed to provide the TS in time */
+	ice_debug(hw, ICE_DBG_PTP, "Failed to read PTP timestamp using low latency read\n");
+	return -EINVAL;
+}
+
+/**
+ * ice_read_phy_tstamp_sbq_e810 - Read a PHY timestamp registers through the sbq
  * @hw: pointer to the HW struct
  * @lport: the lport to read from
  * @idx: the timestamp index to read
- * @tstamp: on return, the 40bit timestamp value
+ * @hi: 8 bit timestamp high value
+ * @lo: 32 bit timestamp low value
  *
- * Read a 40bit timestamp value out of the timestamp block of the external PHY
- * on the E810 device.
+ * Read a 8bit timestamp high value and 32 bit timestamp low value out of the
+ * timestamp block of the external PHY on the E810 device using sideband queue.
  */
 static int
-ice_read_phy_tstamp_e810(struct ice_hw *hw, u8 lport, u8 idx, u64 *tstamp)
+ice_read_phy_tstamp_sbq_e810(struct ice_hw *hw, u8 lport, u8 idx, u8 *hi,
+			     u32 *lo)
 {
-	u32 lo_addr, hi_addr, lo, hi;
+	u32 hi_addr = TS_EXT(HIGH_TX_MEMORY_BANK_START, lport, idx);
+	u32 lo_addr = TS_EXT(LOW_TX_MEMORY_BANK_START, lport, idx);
+	u32 lo_val, hi_val;
 	int err;
 
-	lo_addr = TS_EXT(LOW_TX_MEMORY_BANK_START, lport, idx);
-	hi_addr = TS_EXT(HIGH_TX_MEMORY_BANK_START, lport, idx);
-
-	err = ice_read_phy_reg_e810(hw, lo_addr, &lo);
+	err = ice_read_phy_reg_e810(hw, lo_addr, &lo_val);
 	if (err) {
 		ice_debug(hw, ICE_DBG_PTP, "Failed to read low PTP timestamp register, err %d\n",
 			  err);
 		return err;
 	}
 
-	err = ice_read_phy_reg_e810(hw, hi_addr, &hi);
+	err = ice_read_phy_reg_e810(hw, hi_addr, &hi_val);
 	if (err) {
 		ice_debug(hw, ICE_DBG_PTP, "Failed to read high PTP timestamp register, err %d\n",
 			  err);
 		return err;
 	}
 
+	*lo = lo_val;
+	*hi = (u8)hi_val;
+
+	return 0;
+}
+
+/**
+ * ice_read_phy_tstamp_e810 - Read a PHY timestamp out of the external PHY
+ * @hw: pointer to the HW struct
+ * @lport: the lport to read from
+ * @idx: the timestamp index to read
+ * @tstamp: on return, the 40bit timestamp value
+ *
+ * Read a 40bit timestamp value out of the timestamp block of the external PHY
+ * on the E810 device.
+ */
+static int
+ice_read_phy_tstamp_e810(struct ice_hw *hw, u8 lport, u8 idx, u64 *tstamp)
+{
+	u32 lo = 0;
+	u8 hi = 0;
+	int err;
+
+	if (hw->dev_caps.ts_dev_info.ts_ll_read)
+		err = ice_read_phy_tstamp_ll_e810(hw, idx, &hi, &lo);
+	else
+		err = ice_read_phy_tstamp_sbq_e810(hw, lport, idx, &hi, &lo);
+
+	if (err)
+		return err;
+
 	/* For E810 devices, the timestamp is reported with the lower 32 bits
 	 * in the low register, and the upper 8 bits in the high register.
 	 */
diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h
index 1246e4ee4b5d..2bda64c76abc 100644
--- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h
+++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h
@@ -402,6 +402,7 @@ bool ice_is_pca9575_present(struct ice_hw *hw);
 #define INCVAL_HIGH_M			0xFF
 
 /* Timestamp block macros */
+#define TS_VALID			BIT(0)
 #define TS_LOW_M			0xFFFFFFFF
 #define TS_HIGH_M			0xFF
 #define TS_HIGH_S			32
@@ -413,6 +414,12 @@ bool ice_is_pca9575_present(struct ice_hw *hw);
 #define BYTES_PER_IDX_ADDR_L_U		8
 #define BYTES_PER_IDX_ADDR_L		4
 
+/* Tx timestamp low latency read definitions */
+#define TS_LL_READ_RETRIES		200
+#define TS_LL_READ_TS_HIGH		GENMASK(23, 16)
+#define TS_LL_READ_TS_IDX		GENMASK(29, 24)
+#define TS_LL_READ_TS			BIT(31)
+
 /* Internal PHY timestamp address */
 #define TS_L(a, idx) ((a) + ((idx) * BYTES_PER_IDX_ADDR_L_U))
 #define TS_H(a, idx) ((a) + ((idx) * BYTES_PER_IDX_ADDR_L_U +		\
diff --git a/drivers/net/ethernet/intel/ice/ice_repr.c b/drivers/net/ethernet/intel/ice/ice_repr.c
index 0dac67cd9c77..bd31748aae1b 100644
--- a/drivers/net/ethernet/intel/ice/ice_repr.c
+++ b/drivers/net/ethernet/intel/ice/ice_repr.c
@@ -377,10 +377,10 @@ static void ice_repr_rem(struct ice_vf *vf)
 	if (!vf->repr)
 		return;
 
-	ice_devlink_destroy_vf_port(vf);
 	kfree(vf->repr->q_vector);
 	vf->repr->q_vector = NULL;
 	unregister_netdev(vf->repr->netdev);
+	ice_devlink_destroy_vf_port(vf);
 	free_netdev(vf->repr->netdev);
 	vf->repr->netdev = NULL;
 #ifdef CONFIG_ICE_SWITCHDEV
diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c
index 7947223536e3..118595763bba 100644
--- a/drivers/net/ethernet/intel/ice/ice_sched.c
+++ b/drivers/net/ethernet/intel/ice/ice_sched.c
@@ -1212,7 +1212,7 @@ int ice_sched_init_port(struct ice_port_info *pi)
 	hw = pi->hw;
 
 	/* Query the Default Topology from FW */
-	buf = devm_kzalloc(ice_hw_to_dev(hw), ICE_AQ_MAX_BUF_LEN, GFP_KERNEL);
+	buf = kzalloc(ICE_AQ_MAX_BUF_LEN, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
 
@@ -1290,7 +1290,7 @@ err_init_port:
 		pi->root = NULL;
 	}
 
-	devm_kfree(ice_hw_to_dev(hw), buf);
+	kfree(buf);
 	return status;
 }
 
diff --git a/drivers/net/ethernet/intel/ice/ice_switch.c b/drivers/net/ethernet/intel/ice/ice_switch.c
index 3808034f7e7e..9b762f7972ce 100644
--- a/drivers/net/ethernet/intel/ice/ice_switch.c
+++ b/drivers/net/ethernet/intel/ice/ice_switch.c
@@ -42,6 +42,7 @@ enum {
 	ICE_PKT_GTP_NOPAY	= BIT(8),
 	ICE_PKT_KMALLOC		= BIT(9),
 	ICE_PKT_PPPOE		= BIT(10),
+	ICE_PKT_L2TPV3		= BIT(11),
 };
 
 struct ice_dummy_pkt_offsets {
@@ -1258,6 +1259,65 @@ ICE_DECLARE_PKT_TEMPLATE(pppoe_ipv6_udp) = {
 	0x00, 0x00,		/* 2 bytes for 4 bytes alignment */
 };
 
+ICE_DECLARE_PKT_OFFSETS(ipv4_l2tpv3) = {
+	{ ICE_MAC_OFOS,		0 },
+	{ ICE_ETYPE_OL,		12 },
+	{ ICE_IPV4_OFOS,	14 },
+	{ ICE_L2TPV3,		34 },
+	{ ICE_PROTOCOL_LAST,	0 },
+};
+
+ICE_DECLARE_PKT_TEMPLATE(ipv4_l2tpv3) = {
+	0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+
+	0x08, 0x00,		/* ICE_ETYPE_OL 12 */
+
+	0x45, 0x00, 0x00, 0x20, /* ICE_IPV4_IL 14 */
+	0x00, 0x00, 0x40, 0x00,
+	0x40, 0x73, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+
+	0x00, 0x00, 0x00, 0x00, /* ICE_L2TPV3 34 */
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00,		/* 2 bytes for 4 bytes alignment */
+};
+
+ICE_DECLARE_PKT_OFFSETS(ipv6_l2tpv3) = {
+	{ ICE_MAC_OFOS,		0 },
+	{ ICE_ETYPE_OL,		12 },
+	{ ICE_IPV6_OFOS,	14 },
+	{ ICE_L2TPV3,		54 },
+	{ ICE_PROTOCOL_LAST,	0 },
+};
+
+ICE_DECLARE_PKT_TEMPLATE(ipv6_l2tpv3) = {
+	0x00, 0x00, 0x00, 0x00, /* ICE_MAC_OFOS 0 */
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+
+	0x86, 0xDD,		/* ICE_ETYPE_OL 12 */
+
+	0x60, 0x00, 0x00, 0x00, /* ICE_IPV6_IL 14 */
+	0x00, 0x0c, 0x73, 0x40,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+
+	0x00, 0x00, 0x00, 0x00, /* ICE_L2TPV3 54 */
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00, 0x00, 0x00,
+	0x00, 0x00,		/* 2 bytes for 4 bytes alignment */
+};
+
 static const struct ice_dummy_pkt_profile ice_dummy_pkt_profiles[] = {
 	ICE_PKT_PROFILE(ipv6_gtp, ICE_PKT_TUN_GTPU | ICE_PKT_OUTER_IPV6 |
 				  ICE_PKT_GTP_NOPAY),
@@ -1297,6 +1357,8 @@ static const struct ice_dummy_pkt_profile ice_dummy_pkt_profiles[] = {
 	ICE_PKT_PROFILE(udp_tun_ipv6_tcp, ICE_PKT_TUN_UDP |
 					  ICE_PKT_INNER_IPV6 |
 					  ICE_PKT_INNER_TCP),
+	ICE_PKT_PROFILE(ipv6_l2tpv3, ICE_PKT_L2TPV3 | ICE_PKT_OUTER_IPV6),
+	ICE_PKT_PROFILE(ipv4_l2tpv3, ICE_PKT_L2TPV3),
 	ICE_PKT_PROFILE(udp_tun_tcp, ICE_PKT_TUN_UDP | ICE_PKT_INNER_TCP),
 	ICE_PKT_PROFILE(udp_tun_ipv6_udp, ICE_PKT_TUN_UDP |
 					  ICE_PKT_INNER_IPV6),
@@ -2274,9 +2336,7 @@ int ice_get_initial_sw_cfg(struct ice_hw *hw)
 	int status;
 	u16 i;
 
-	rbuf = devm_kzalloc(ice_hw_to_dev(hw), ICE_SW_CFG_MAX_BUF_LEN,
-			    GFP_KERNEL);
-
+	rbuf = kzalloc(ICE_SW_CFG_MAX_BUF_LEN, GFP_KERNEL);
 	if (!rbuf)
 		return -ENOMEM;
 
@@ -2324,7 +2384,7 @@ int ice_get_initial_sw_cfg(struct ice_hw *hw)
 		}
 	} while (req_desc && !status);
 
-	devm_kfree(ice_hw_to_dev(hw), rbuf);
+	kfree(rbuf);
 	return status;
 }
 
@@ -3449,31 +3509,15 @@ bool ice_vlan_fltr_exist(struct ice_hw *hw, u16 vlan_id, u16 vsi_handle)
  * ice_add_mac - Add a MAC address based filter rule
  * @hw: pointer to the hardware structure
  * @m_list: list of MAC addresses and forwarding information
- *
- * IMPORTANT: When the ucast_shared flag is set to false and m_list has
- * multiple unicast addresses, the function assumes that all the
- * addresses are unique in a given add_mac call. It doesn't
- * check for duplicates in this case, removing duplicates from a given
- * list should be taken care of in the caller of this function.
  */
 int ice_add_mac(struct ice_hw *hw, struct list_head *m_list)
 {
-	struct ice_sw_rule_lkup_rx_tx *s_rule, *r_iter;
 	struct ice_fltr_list_entry *m_list_itr;
-	struct list_head *rule_head;
-	u16 total_elem_left, s_rule_size;
-	struct ice_switch_info *sw;
-	struct mutex *rule_lock; /* Lock to protect filter rule list */
-	u16 num_unicast = 0;
 	int status = 0;
-	u8 elem_sent;
 
 	if (!m_list || !hw)
 		return -EINVAL;
 
-	s_rule = NULL;
-	sw = hw->switch_info;
-	rule_lock = &sw->recp_list[ICE_SW_LKUP_MAC].filt_rule_lock;
 	list_for_each_entry(m_list_itr, m_list, list_entry) {
 		u8 *add = &m_list_itr->fltr_info.l_data.mac.mac_addr[0];
 		u16 vsi_handle;
@@ -3492,106 +3536,13 @@ int ice_add_mac(struct ice_hw *hw, struct list_head *m_list)
 		if (m_list_itr->fltr_info.lkup_type != ICE_SW_LKUP_MAC ||
 		    is_zero_ether_addr(add))
 			return -EINVAL;
-		if (is_unicast_ether_addr(add) && !hw->ucast_shared) {
-			/* Don't overwrite the unicast address */
-			mutex_lock(rule_lock);
-			if (ice_find_rule_entry(hw, ICE_SW_LKUP_MAC,
-						&m_list_itr->fltr_info)) {
-				mutex_unlock(rule_lock);
-				return -EEXIST;
-			}
-			mutex_unlock(rule_lock);
-			num_unicast++;
-		} else if (is_multicast_ether_addr(add) ||
-			   (is_unicast_ether_addr(add) && hw->ucast_shared)) {
-			m_list_itr->status =
-				ice_add_rule_internal(hw, ICE_SW_LKUP_MAC,
-						      m_list_itr);
-			if (m_list_itr->status)
-				return m_list_itr->status;
-		}
-	}
 
-	mutex_lock(rule_lock);
-	/* Exit if no suitable entries were found for adding bulk switch rule */
-	if (!num_unicast) {
-		status = 0;
-		goto ice_add_mac_exit;
-	}
-
-	rule_head = &sw->recp_list[ICE_SW_LKUP_MAC].filt_rules;
-
-	/* Allocate switch rule buffer for the bulk update for unicast */
-	s_rule_size = ICE_SW_RULE_RX_TX_ETH_HDR_SIZE(s_rule);
-	s_rule = devm_kcalloc(ice_hw_to_dev(hw), num_unicast, s_rule_size,
-			      GFP_KERNEL);
-	if (!s_rule) {
-		status = -ENOMEM;
-		goto ice_add_mac_exit;
-	}
-
-	r_iter = s_rule;
-	list_for_each_entry(m_list_itr, m_list, list_entry) {
-		struct ice_fltr_info *f_info = &m_list_itr->fltr_info;
-		u8 *mac_addr = &f_info->l_data.mac.mac_addr[0];
-
-		if (is_unicast_ether_addr(mac_addr)) {
-			ice_fill_sw_rule(hw, &m_list_itr->fltr_info, r_iter,
-					 ice_aqc_opc_add_sw_rules);
-			r_iter = (typeof(s_rule))((u8 *)r_iter + s_rule_size);
-		}
-	}
-
-	/* Call AQ bulk switch rule update for all unicast addresses */
-	r_iter = s_rule;
-	/* Call AQ switch rule in AQ_MAX chunk */
-	for (total_elem_left = num_unicast; total_elem_left > 0;
-	     total_elem_left -= elem_sent) {
-		struct ice_sw_rule_lkup_rx_tx *entry = r_iter;
-
-		elem_sent = min_t(u8, total_elem_left,
-				  (ICE_AQ_MAX_BUF_LEN / s_rule_size));
-		status = ice_aq_sw_rules(hw, entry, elem_sent * s_rule_size,
-					 elem_sent, ice_aqc_opc_add_sw_rules,
-					 NULL);
-		if (status)
-			goto ice_add_mac_exit;
-		r_iter = (typeof(s_rule))
-			((u8 *)r_iter + (elem_sent * s_rule_size));
+		m_list_itr->status = ice_add_rule_internal(hw, ICE_SW_LKUP_MAC,
+							   m_list_itr);
+		if (m_list_itr->status)
+			return m_list_itr->status;
 	}
 
-	/* Fill up rule ID based on the value returned from FW */
-	r_iter = s_rule;
-	list_for_each_entry(m_list_itr, m_list, list_entry) {
-		struct ice_fltr_info *f_info = &m_list_itr->fltr_info;
-		u8 *mac_addr = &f_info->l_data.mac.mac_addr[0];
-		struct ice_fltr_mgmt_list_entry *fm_entry;
-
-		if (is_unicast_ether_addr(mac_addr)) {
-			f_info->fltr_rule_id = le16_to_cpu(r_iter->index);
-			f_info->fltr_act = ICE_FWD_TO_VSI;
-			/* Create an entry to track this MAC address */
-			fm_entry = devm_kzalloc(ice_hw_to_dev(hw),
-						sizeof(*fm_entry), GFP_KERNEL);
-			if (!fm_entry) {
-				status = -ENOMEM;
-				goto ice_add_mac_exit;
-			}
-			fm_entry->fltr_info = *f_info;
-			fm_entry->vsi_count = 1;
-			/* The book keeping entries will get removed when
-			 * base driver calls remove filter AQ command
-			 */
-
-			list_add(&fm_entry->list_entry, rule_head);
-			r_iter = (typeof(s_rule))((u8 *)r_iter + s_rule_size);
-		}
-	}
-
-ice_add_mac_exit:
-	mutex_unlock(rule_lock);
-	if (s_rule)
-		devm_kfree(ice_hw_to_dev(hw), s_rule);
 	return status;
 }
 
@@ -3979,38 +3930,6 @@ ice_check_if_dflt_vsi(struct ice_port_info *pi, u16 vsi_handle,
 }
 
 /**
- * ice_find_ucast_rule_entry - Search for a unicast MAC filter rule entry
- * @hw: pointer to the hardware structure
- * @recp_id: lookup type for which the specified rule needs to be searched
- * @f_info: rule information
- *
- * Helper function to search for a unicast rule entry - this is to be used
- * to remove unicast MAC filter that is not shared with other VSIs on the
- * PF switch.
- *
- * Returns pointer to entry storing the rule if found
- */
-static struct ice_fltr_mgmt_list_entry *
-ice_find_ucast_rule_entry(struct ice_hw *hw, u8 recp_id,
-			  struct ice_fltr_info *f_info)
-{
-	struct ice_switch_info *sw = hw->switch_info;
-	struct ice_fltr_mgmt_list_entry *list_itr;
-	struct list_head *list_head;
-
-	list_head = &sw->recp_list[recp_id].filt_rules;
-	list_for_each_entry(list_itr, list_head, list_entry) {
-		if (!memcmp(&f_info->l_data, &list_itr->fltr_info.l_data,
-			    sizeof(f_info->l_data)) &&
-		    f_info->fwd_id.hw_vsi_id ==
-		    list_itr->fltr_info.fwd_id.hw_vsi_id &&
-		    f_info->flag == list_itr->fltr_info.flag)
-			return list_itr;
-	}
-	return NULL;
-}
-
-/**
  * ice_remove_mac - remove a MAC address based filter rule
  * @hw: pointer to the hardware structure
  * @m_list: list of MAC addresses and forwarding information
@@ -4026,15 +3945,12 @@ ice_find_ucast_rule_entry(struct ice_hw *hw, u8 recp_id,
 int ice_remove_mac(struct ice_hw *hw, struct list_head *m_list)
 {
 	struct ice_fltr_list_entry *list_itr, *tmp;
-	struct mutex *rule_lock; /* Lock to protect filter rule list */
 
 	if (!m_list)
 		return -EINVAL;
 
-	rule_lock = &hw->switch_info->recp_list[ICE_SW_LKUP_MAC].filt_rule_lock;
 	list_for_each_entry_safe(list_itr, tmp, m_list, list_entry) {
 		enum ice_sw_lkup_type l_type = list_itr->fltr_info.lkup_type;
-		u8 *add = &list_itr->fltr_info.l_data.mac.mac_addr[0];
 		u16 vsi_handle;
 
 		if (l_type != ICE_SW_LKUP_MAC)
@@ -4046,19 +3962,7 @@ int ice_remove_mac(struct ice_hw *hw, struct list_head *m_list)
 
 		list_itr->fltr_info.fwd_id.hw_vsi_id =
 					ice_get_hw_vsi_num(hw, vsi_handle);
-		if (is_unicast_ether_addr(add) && !hw->ucast_shared) {
-			/* Don't remove the unicast address that belongs to
-			 * another VSI on the switch, since it is not being
-			 * shared...
-			 */
-			mutex_lock(rule_lock);
-			if (!ice_find_ucast_rule_entry(hw, ICE_SW_LKUP_MAC,
-						       &list_itr->fltr_info)) {
-				mutex_unlock(rule_lock);
-				return -ENOENT;
-			}
-			mutex_unlock(rule_lock);
-		}
+
 		list_itr->status = ice_remove_rule_internal(hw,
 							    ICE_SW_LKUP_MAC,
 							    list_itr);
@@ -4648,6 +4552,7 @@ static const struct ice_prot_ext_tbl_entry ice_prot_ext[ICE_PROTOCOL_LAST] = {
 	{ ICE_GTP,		{ 8, 10, 12, 14, 16, 18, 20, 22 } },
 	{ ICE_GTP_NO_PAY,	{ 8, 10, 12, 14 } },
 	{ ICE_PPPOE,		{ 0, 2, 4, 6 } },
+	{ ICE_L2TPV3,		{ 0, 2, 4, 6, 8, 10 } },
 	{ ICE_VLAN_EX,          { 2, 0 } },
 	{ ICE_VLAN_IN,          { 2, 0 } },
 };
@@ -4671,6 +4576,7 @@ static struct ice_protocol_entry ice_prot_id_tbl[ICE_PROTOCOL_LAST] = {
 	{ ICE_GTP,		ICE_UDP_OF_HW },
 	{ ICE_GTP_NO_PAY,	ICE_UDP_ILOS_HW },
 	{ ICE_PPPOE,		ICE_PPPOE_HW },
+	{ ICE_L2TPV3,		ICE_L2TPV3_HW },
 	{ ICE_VLAN_EX,          ICE_VLAN_OF_HW },
 	{ ICE_VLAN_IN,          ICE_VLAN_OL_HW },
 };
@@ -5754,7 +5660,8 @@ ice_find_dummy_packet(struct ice_adv_lkup_elem *lkups, u16 lkups_cnt,
 			if (lkups[i].h_u.pppoe_hdr.ppp_prot_id ==
 			    htons(PPP_IPV6))
 				match |= ICE_PKT_OUTER_IPV6;
-		}
+		} else if (lkups[i].type == ICE_L2TPV3)
+			match |= ICE_PKT_L2TPV3;
 	}
 
 	while (ret->match && (match & ret->match) != ret->match)
@@ -5855,6 +5762,9 @@ ice_fill_adv_dummy_packet(struct ice_adv_lkup_elem *lkups, u16 lkups_cnt,
 		case ICE_PPPOE:
 			len = sizeof(struct ice_pppoe_hdr);
 			break;
+		case ICE_L2TPV3:
+			len = sizeof(struct ice_l2tpv3_sess_hdr);
+			break;
 		default:
 			return -EINVAL;
 		}
diff --git a/drivers/net/ethernet/intel/ice/ice_tc_lib.c b/drivers/net/ethernet/intel/ice/ice_tc_lib.c
index a298862857a8..f68c555be4e9 100644
--- a/drivers/net/ethernet/intel/ice/ice_tc_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_tc_lib.c
@@ -36,6 +36,10 @@ ice_tc_count_lkups(u32 flags, struct ice_tc_flower_lyr_2_4_hdrs *headers,
 		     ICE_TC_FLWR_FIELD_ENC_DEST_IPV6))
 		lkups_cnt++;
 
+	if (flags & (ICE_TC_FLWR_FIELD_ENC_IP_TOS |
+		     ICE_TC_FLWR_FIELD_ENC_IP_TTL))
+		lkups_cnt++;
+
 	if (flags & ICE_TC_FLWR_FIELD_ENC_DEST_L4_PORT)
 		lkups_cnt++;
 
@@ -47,11 +51,11 @@ ice_tc_count_lkups(u32 flags, struct ice_tc_flower_lyr_2_4_hdrs *headers,
 		lkups_cnt++;
 
 	/* is VLAN specified? */
-	if (flags & ICE_TC_FLWR_FIELD_VLAN)
+	if (flags & (ICE_TC_FLWR_FIELD_VLAN | ICE_TC_FLWR_FIELD_VLAN_PRIO))
 		lkups_cnt++;
 
 	/* is CVLAN specified? */
-	if (flags & ICE_TC_FLWR_FIELD_CVLAN)
+	if (flags & (ICE_TC_FLWR_FIELD_CVLAN | ICE_TC_FLWR_FIELD_CVLAN_PRIO))
 		lkups_cnt++;
 
 	/* are PPPoE options specified? */
@@ -64,6 +68,13 @@ ice_tc_count_lkups(u32 flags, struct ice_tc_flower_lyr_2_4_hdrs *headers,
 		     ICE_TC_FLWR_FIELD_DEST_IPV6 | ICE_TC_FLWR_FIELD_SRC_IPV6))
 		lkups_cnt++;
 
+	if (flags & (ICE_TC_FLWR_FIELD_IP_TOS | ICE_TC_FLWR_FIELD_IP_TTL))
+		lkups_cnt++;
+
+	/* are L2TPv3 options specified? */
+	if (flags & ICE_TC_FLWR_FIELD_L2TPV3_SESSID)
+		lkups_cnt++;
+
 	/* is L4 (TCP/UDP/any other L4 protocol fields) specified? */
 	if (flags & (ICE_TC_FLWR_FIELD_DEST_L4_PORT |
 		     ICE_TC_FLWR_FIELD_SRC_L4_PORT))
@@ -257,6 +268,50 @@ ice_tc_fill_tunnel_outer(u32 flags, struct ice_tc_flower_fltr *fltr,
 		i++;
 	}
 
+	if (fltr->inner_headers.l2_key.n_proto == htons(ETH_P_IP) &&
+	    (flags & (ICE_TC_FLWR_FIELD_ENC_IP_TOS |
+		      ICE_TC_FLWR_FIELD_ENC_IP_TTL))) {
+		list[i].type = ice_proto_type_from_ipv4(false);
+
+		if (flags & ICE_TC_FLWR_FIELD_ENC_IP_TOS) {
+			list[i].h_u.ipv4_hdr.tos = hdr->l3_key.tos;
+			list[i].m_u.ipv4_hdr.tos = hdr->l3_mask.tos;
+		}
+
+		if (flags & ICE_TC_FLWR_FIELD_ENC_IP_TTL) {
+			list[i].h_u.ipv4_hdr.time_to_live = hdr->l3_key.ttl;
+			list[i].m_u.ipv4_hdr.time_to_live = hdr->l3_mask.ttl;
+		}
+
+		i++;
+	}
+
+	if (fltr->inner_headers.l2_key.n_proto == htons(ETH_P_IPV6) &&
+	    (flags & (ICE_TC_FLWR_FIELD_ENC_IP_TOS |
+		      ICE_TC_FLWR_FIELD_ENC_IP_TTL))) {
+		struct ice_ipv6_hdr *hdr_h, *hdr_m;
+
+		hdr_h = &list[i].h_u.ipv6_hdr;
+		hdr_m = &list[i].m_u.ipv6_hdr;
+		list[i].type = ice_proto_type_from_ipv6(false);
+
+		if (flags & ICE_TC_FLWR_FIELD_ENC_IP_TOS) {
+			be32p_replace_bits(&hdr_h->be_ver_tc_flow,
+					   hdr->l3_key.tos,
+					   ICE_IPV6_HDR_TC_MASK);
+			be32p_replace_bits(&hdr_m->be_ver_tc_flow,
+					   hdr->l3_mask.tos,
+					   ICE_IPV6_HDR_TC_MASK);
+		}
+
+		if (flags & ICE_TC_FLWR_FIELD_ENC_IP_TTL) {
+			hdr_h->hop_limit = hdr->l3_key.ttl;
+			hdr_m->hop_limit = hdr->l3_mask.ttl;
+		}
+
+		i++;
+	}
+
 	if ((flags & ICE_TC_FLWR_FIELD_ENC_DEST_L4_PORT) &&
 	    hdr->l3_key.ip_proto == IPPROTO_UDP) {
 		list[i].type = ICE_UDP_OF;
@@ -334,7 +389,7 @@ ice_tc_fill_rules(struct ice_hw *hw, u32 flags,
 	}
 
 	/* copy VLAN info */
-	if (flags & ICE_TC_FLWR_FIELD_VLAN) {
+	if (flags & (ICE_TC_FLWR_FIELD_VLAN | ICE_TC_FLWR_FIELD_VLAN_PRIO)) {
 		vlan_tpid = be16_to_cpu(headers->vlan_hdr.vlan_tpid);
 		rule_info->vlan_type =
 				ice_check_supported_vlan_tpid(vlan_tpid);
@@ -343,15 +398,45 @@ ice_tc_fill_rules(struct ice_hw *hw, u32 flags,
 			list[i].type = ICE_VLAN_EX;
 		else
 			list[i].type = ICE_VLAN_OFOS;
-		list[i].h_u.vlan_hdr.vlan = headers->vlan_hdr.vlan_id;
-		list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xFFFF);
+
+		if (flags & ICE_TC_FLWR_FIELD_VLAN) {
+			list[i].h_u.vlan_hdr.vlan = headers->vlan_hdr.vlan_id;
+			list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0x0FFF);
+		}
+
+		if (flags & ICE_TC_FLWR_FIELD_VLAN_PRIO) {
+			if (flags & ICE_TC_FLWR_FIELD_VLAN) {
+				list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xEFFF);
+			} else {
+				list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xE000);
+				list[i].h_u.vlan_hdr.vlan = 0;
+			}
+			list[i].h_u.vlan_hdr.vlan |=
+				headers->vlan_hdr.vlan_prio;
+		}
+
 		i++;
 	}
 
-	if (flags & ICE_TC_FLWR_FIELD_CVLAN) {
+	if (flags & (ICE_TC_FLWR_FIELD_CVLAN | ICE_TC_FLWR_FIELD_CVLAN_PRIO)) {
 		list[i].type = ICE_VLAN_IN;
-		list[i].h_u.vlan_hdr.vlan = headers->cvlan_hdr.vlan_id;
-		list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xFFFF);
+
+		if (flags & ICE_TC_FLWR_FIELD_CVLAN) {
+			list[i].h_u.vlan_hdr.vlan = headers->cvlan_hdr.vlan_id;
+			list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0x0FFF);
+		}
+
+		if (flags & ICE_TC_FLWR_FIELD_CVLAN_PRIO) {
+			if (flags & ICE_TC_FLWR_FIELD_CVLAN) {
+				list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xEFFF);
+			} else {
+				list[i].m_u.vlan_hdr.vlan = cpu_to_be16(0xE000);
+				list[i].h_u.vlan_hdr.vlan = 0;
+			}
+			list[i].h_u.vlan_hdr.vlan |=
+				headers->cvlan_hdr.vlan_prio;
+		}
+
 		i++;
 	}
 
@@ -420,6 +505,61 @@ ice_tc_fill_rules(struct ice_hw *hw, u32 flags,
 		i++;
 	}
 
+	if (headers->l2_key.n_proto == htons(ETH_P_IP) &&
+	    (flags & (ICE_TC_FLWR_FIELD_IP_TOS | ICE_TC_FLWR_FIELD_IP_TTL))) {
+		list[i].type = ice_proto_type_from_ipv4(inner);
+
+		if (flags & ICE_TC_FLWR_FIELD_IP_TOS) {
+			list[i].h_u.ipv4_hdr.tos = headers->l3_key.tos;
+			list[i].m_u.ipv4_hdr.tos = headers->l3_mask.tos;
+		}
+
+		if (flags & ICE_TC_FLWR_FIELD_IP_TTL) {
+			list[i].h_u.ipv4_hdr.time_to_live =
+				headers->l3_key.ttl;
+			list[i].m_u.ipv4_hdr.time_to_live =
+				headers->l3_mask.ttl;
+		}
+
+		i++;
+	}
+
+	if (headers->l2_key.n_proto == htons(ETH_P_IPV6) &&
+	    (flags & (ICE_TC_FLWR_FIELD_IP_TOS | ICE_TC_FLWR_FIELD_IP_TTL))) {
+		struct ice_ipv6_hdr *hdr_h, *hdr_m;
+
+		hdr_h = &list[i].h_u.ipv6_hdr;
+		hdr_m = &list[i].m_u.ipv6_hdr;
+		list[i].type = ice_proto_type_from_ipv6(inner);
+
+		if (flags & ICE_TC_FLWR_FIELD_IP_TOS) {
+			be32p_replace_bits(&hdr_h->be_ver_tc_flow,
+					   headers->l3_key.tos,
+					   ICE_IPV6_HDR_TC_MASK);
+			be32p_replace_bits(&hdr_m->be_ver_tc_flow,
+					   headers->l3_mask.tos,
+					   ICE_IPV6_HDR_TC_MASK);
+		}
+
+		if (flags & ICE_TC_FLWR_FIELD_IP_TTL) {
+			hdr_h->hop_limit = headers->l3_key.ttl;
+			hdr_m->hop_limit = headers->l3_mask.ttl;
+		}
+
+		i++;
+	}
+
+	if (flags & ICE_TC_FLWR_FIELD_L2TPV3_SESSID) {
+		list[i].type = ICE_L2TPV3;
+
+		list[i].h_u.l2tpv3_sess_hdr.session_id =
+			headers->l2tpv3_hdr.session_id;
+		list[i].m_u.l2tpv3_sess_hdr.session_id =
+			cpu_to_be32(0xFFFFFFFF);
+
+		i++;
+	}
+
 	/* copy L4 (src, dest) port */
 	if (flags & (ICE_TC_FLWR_FIELD_DEST_L4_PORT |
 		     ICE_TC_FLWR_FIELD_SRC_L4_PORT)) {
@@ -839,6 +979,40 @@ ice_tc_set_ipv6(struct flow_match_ipv6_addrs *match,
 }
 
 /**
+ * ice_tc_set_tos_ttl - Parse IP ToS/TTL from TC flower filter
+ * @match: Pointer to flow match structure
+ * @fltr: Pointer to filter structure
+ * @headers: inner or outer header fields
+ * @is_encap: set true for tunnel
+ */
+static void
+ice_tc_set_tos_ttl(struct flow_match_ip *match,
+		   struct ice_tc_flower_fltr *fltr,
+		   struct ice_tc_flower_lyr_2_4_hdrs *headers,
+		   bool is_encap)
+{
+	if (match->mask->tos) {
+		if (is_encap)
+			fltr->flags |= ICE_TC_FLWR_FIELD_ENC_IP_TOS;
+		else
+			fltr->flags |= ICE_TC_FLWR_FIELD_IP_TOS;
+
+		headers->l3_key.tos = match->key->tos;
+		headers->l3_mask.tos = match->mask->tos;
+	}
+
+	if (match->mask->ttl) {
+		if (is_encap)
+			fltr->flags |= ICE_TC_FLWR_FIELD_ENC_IP_TTL;
+		else
+			fltr->flags |= ICE_TC_FLWR_FIELD_IP_TTL;
+
+		headers->l3_key.ttl = match->key->ttl;
+		headers->l3_mask.ttl = match->mask->ttl;
+	}
+}
+
+/**
  * ice_tc_set_port - Parse ports from TC flower filter
  * @match: Flow match structure
  * @fltr: Pointer to filter structure
@@ -967,10 +1141,7 @@ ice_parse_tunnel_attr(struct net_device *dev, struct flow_rule *rule,
 		struct flow_match_ip match;
 
 		flow_rule_match_enc_ip(rule, &match);
-		headers->l3_key.tos = match.key->tos;
-		headers->l3_key.ttl = match.key->ttl;
-		headers->l3_mask.tos = match.mask->tos;
-		headers->l3_mask.ttl = match.mask->ttl;
+		ice_tc_set_tos_ttl(&match, fltr, headers, true);
 	}
 
 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_PORTS) &&
@@ -1039,9 +1210,11 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi,
 	      BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
 	      BIT(FLOW_DISSECTOR_KEY_ENC_PORTS) |
 	      BIT(FLOW_DISSECTOR_KEY_ENC_OPTS) |
+	      BIT(FLOW_DISSECTOR_KEY_IP) |
 	      BIT(FLOW_DISSECTOR_KEY_ENC_IP) |
 	      BIT(FLOW_DISSECTOR_KEY_PORTS) |
-	      BIT(FLOW_DISSECTOR_KEY_PPPOE))) {
+	      BIT(FLOW_DISSECTOR_KEY_PPPOE) |
+	      BIT(FLOW_DISSECTOR_KEY_L2TPV3))) {
 		NL_SET_ERR_MSG_MOD(fltr->extack, "Unsupported key used");
 		return -EOPNOTSUPP;
 	}
@@ -1137,16 +1310,22 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi,
 		if (match.mask->vlan_id) {
 			if (match.mask->vlan_id == VLAN_VID_MASK) {
 				fltr->flags |= ICE_TC_FLWR_FIELD_VLAN;
+				headers->vlan_hdr.vlan_id =
+					cpu_to_be16(match.key->vlan_id &
+						    VLAN_VID_MASK);
 			} else {
 				NL_SET_ERR_MSG_MOD(fltr->extack, "Bad VLAN mask");
 				return -EINVAL;
 			}
 		}
 
-		headers->vlan_hdr.vlan_id =
-				cpu_to_be16(match.key->vlan_id & VLAN_VID_MASK);
-		if (match.mask->vlan_priority)
-			headers->vlan_hdr.vlan_prio = match.key->vlan_priority;
+		if (match.mask->vlan_priority) {
+			fltr->flags |= ICE_TC_FLWR_FIELD_VLAN_PRIO;
+			headers->vlan_hdr.vlan_prio =
+				cpu_to_be16((match.key->vlan_priority <<
+					     VLAN_PRIO_SHIFT) & VLAN_PRIO_MASK);
+		}
+
 		if (match.mask->vlan_tpid)
 			headers->vlan_hdr.vlan_tpid = match.key->vlan_tpid;
 	}
@@ -1164,6 +1343,9 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi,
 		if (match.mask->vlan_id) {
 			if (match.mask->vlan_id == VLAN_VID_MASK) {
 				fltr->flags |= ICE_TC_FLWR_FIELD_CVLAN;
+				headers->cvlan_hdr.vlan_id =
+					cpu_to_be16(match.key->vlan_id &
+						    VLAN_VID_MASK);
 			} else {
 				NL_SET_ERR_MSG_MOD(fltr->extack,
 						   "Bad CVLAN mask");
@@ -1171,10 +1353,12 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi,
 			}
 		}
 
-		headers->cvlan_hdr.vlan_id =
-				cpu_to_be16(match.key->vlan_id & VLAN_VID_MASK);
-		if (match.mask->vlan_priority)
-			headers->cvlan_hdr.vlan_prio = match.key->vlan_priority;
+		if (match.mask->vlan_priority) {
+			fltr->flags |= ICE_TC_FLWR_FIELD_CVLAN_PRIO;
+			headers->cvlan_hdr.vlan_prio =
+				cpu_to_be16((match.key->vlan_priority <<
+					     VLAN_PRIO_SHIFT) & VLAN_PRIO_MASK);
+		}
 	}
 
 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PPPOE)) {
@@ -1217,6 +1401,22 @@ ice_parse_cls_flower(struct net_device *filter_dev, struct ice_vsi *vsi,
 			return -EINVAL;
 	}
 
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_IP)) {
+		struct flow_match_ip match;
+
+		flow_rule_match_ip(rule, &match);
+		ice_tc_set_tos_ttl(&match, fltr, headers, false);
+	}
+
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_L2TPV3)) {
+		struct flow_match_l2tpv3 match;
+
+		flow_rule_match_l2tpv3(rule, &match);
+
+		fltr->flags |= ICE_TC_FLWR_FIELD_L2TPV3_SESSID;
+		headers->l2tpv3_hdr.session_id = match.key->session_id;
+	}
+
 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
 		struct flow_match_ports match;
 
diff --git a/drivers/net/ethernet/intel/ice/ice_tc_lib.h b/drivers/net/ethernet/intel/ice/ice_tc_lib.h
index 91cd3d3778c7..92642faad595 100644
--- a/drivers/net/ethernet/intel/ice/ice_tc_lib.h
+++ b/drivers/net/ethernet/intel/ice/ice_tc_lib.h
@@ -26,9 +26,18 @@
 #define ICE_TC_FLWR_FIELD_CVLAN			BIT(19)
 #define ICE_TC_FLWR_FIELD_PPPOE_SESSID		BIT(20)
 #define ICE_TC_FLWR_FIELD_PPP_PROTO		BIT(21)
+#define ICE_TC_FLWR_FIELD_IP_TOS		BIT(22)
+#define ICE_TC_FLWR_FIELD_IP_TTL		BIT(23)
+#define ICE_TC_FLWR_FIELD_ENC_IP_TOS		BIT(24)
+#define ICE_TC_FLWR_FIELD_ENC_IP_TTL		BIT(25)
+#define ICE_TC_FLWR_FIELD_L2TPV3_SESSID		BIT(26)
+#define ICE_TC_FLWR_FIELD_VLAN_PRIO		BIT(27)
+#define ICE_TC_FLWR_FIELD_CVLAN_PRIO		BIT(28)
 
 #define ICE_TC_FLOWER_MASK_32   0xFFFFFFFF
 
+#define ICE_IPV6_HDR_TC_MASK 0xFF00000
+
 struct ice_indr_block_priv {
 	struct net_device *netdev;
 	struct ice_netdev_priv *np;
@@ -42,7 +51,7 @@ struct ice_tc_flower_action {
 
 struct ice_tc_vlan_hdr {
 	__be16 vlan_id; /* Only last 12 bits valid */
-	u16 vlan_prio; /* Only last 3 bits valid (valid values: 0..7) */
+	__be16 vlan_prio; /* Only last 3 bits valid (valid values: 0..7) */
 	__be16 vlan_tpid;
 };
 
@@ -80,6 +89,10 @@ struct ice_tc_l3_hdr {
 	u8 ttl;
 };
 
+struct ice_tc_l2tpv3_hdr {
+	__be32 session_id;
+};
+
 struct ice_tc_l4_hdr {
 	__be16 dst_port;
 	__be16 src_port;
@@ -92,6 +105,7 @@ struct ice_tc_flower_lyr_2_4_hdrs {
 	struct ice_tc_vlan_hdr vlan_hdr;
 	struct ice_tc_vlan_hdr cvlan_hdr;
 	struct ice_tc_pppoe_hdr pppoe_hdr;
+	struct ice_tc_l2tpv3_hdr l2tpv3_hdr;
 	/* L3 (IPv4[6]) layer fields with their mask */
 	struct ice_tc_l3_hdr l3_key;
 	struct ice_tc_l3_hdr l3_mask;
diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c
index 97453d1dfafe..dbe80e5053a8 100644
--- a/drivers/net/ethernet/intel/ice/ice_txrx.c
+++ b/drivers/net/ethernet/intel/ice/ice_txrx.c
@@ -1467,7 +1467,7 @@ int ice_napi_poll(struct napi_struct *napi, int budget)
 		bool wd;
 
 		if (tx_ring->xsk_pool)
-			wd = ice_xmit_zc(tx_ring, ICE_DESC_UNUSED(tx_ring), budget);
+			wd = ice_xmit_zc(tx_ring);
 		else if (ice_ring_is_xdp(tx_ring))
 			wd = true;
 		else
@@ -2258,8 +2258,10 @@ ice_tstamp(struct ice_tx_ring *tx_ring, struct sk_buff *skb,
 
 	/* Grab an open timestamp slot */
 	idx = ice_ptp_request_ts(tx_ring->tx_tstamps, skb);
-	if (idx < 0)
+	if (idx < 0) {
+		tx_ring->vsi->back->ptp.tx_hwtstamp_skipped++;
 		return;
+	}
 
 	off->cd_qw1 |= (u64)(ICE_TX_DESC_DTYPE_CTX |
 			     (ICE_TX_CTX_DESC_TSYN << ICE_TXD_CTX_QW1_CMD_S) |
diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h
index ca902af54bb4..932b5661ec4d 100644
--- a/drivers/net/ethernet/intel/ice/ice_txrx.h
+++ b/drivers/net/ethernet/intel/ice/ice_txrx.h
@@ -295,10 +295,11 @@ struct ice_rx_ring {
 	struct xsk_buff_pool *xsk_pool;
 	struct sk_buff *skb;
 	dma_addr_t dma;			/* physical address of ring */
-#define ICE_RX_FLAGS_RING_BUILD_SKB	BIT(1)
 	u64 cached_phctime;
 	u8 dcb_tc;			/* Traffic class of ring */
 	u8 ptp_rx;
+#define ICE_RX_FLAGS_RING_BUILD_SKB	BIT(1)
+#define ICE_RX_FLAGS_CRC_STRIP_DIS	BIT(2)
 	u8 flags;
 } ____cacheline_internodealigned_in_smp;
 
diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h
index 861b64322959..e1abfcee96dc 100644
--- a/drivers/net/ethernet/intel/ice/ice_type.h
+++ b/drivers/net/ethernet/intel/ice/ice_type.h
@@ -347,6 +347,7 @@ struct ice_ts_func_info {
 #define ICE_TS_DEV_ENA_M		BIT(24)
 #define ICE_TS_TMR0_ENA_M		BIT(25)
 #define ICE_TS_TMR1_ENA_M		BIT(26)
+#define ICE_TS_LL_TX_TS_READ_M		BIT(28)
 
 struct ice_ts_dev_info {
 	/* Device specific info */
@@ -359,6 +360,7 @@ struct ice_ts_dev_info {
 	u8 ena;
 	u8 tmr0_ena;
 	u8 tmr1_ena;
+	u8 ts_ll_read;
 };
 
 /* Function specific capabilities */
@@ -564,6 +566,8 @@ enum ice_rl_type {
 #define ICE_SCHED_INVAL_PROF_ID		0xFFFF
 #define ICE_SCHED_DFLT_BURST_SIZE	(15 * 1024)	/* in bytes (15k) */
 
+#define ICE_MAX_PORT_PER_PCI_DEV 8
+
  /* Data structure for saving BW information */
 enum ice_bw_type {
 	ICE_BW_TYPE_PRIO,
@@ -885,8 +889,6 @@ struct ice_hw {
 	/* INTRL granularity in 1 us */
 	u8 intrl_gran;
 
-	u8 ucast_shared;	/* true if VSIs can share unicast addr */
-
 #define ICE_PHY_PER_NAC		1
 #define ICE_MAX_QUAD		2
 #define ICE_NUM_QUAD_TYPE	2
diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c
index 03ce85f6e6df..056c904b83cc 100644
--- a/drivers/net/ethernet/intel/ice/ice_xsk.c
+++ b/drivers/net/ethernet/intel/ice/ice_xsk.c
@@ -392,13 +392,6 @@ int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid)
 		goto failure;
 	}
 
-	if (!is_power_of_2(vsi->rx_rings[qid]->count) ||
-	    !is_power_of_2(vsi->tx_rings[qid]->count)) {
-		netdev_err(vsi->netdev, "Please align ring sizes to power of 2\n");
-		pool_failure = -EINVAL;
-		goto failure;
-	}
-
 	if_running = netif_running(vsi->netdev) && ice_is_xdp_ena_vsi(vsi);
 
 	if (if_running) {
@@ -534,11 +527,10 @@ exit:
 bool ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count)
 {
 	u16 rx_thresh = ICE_RING_QUARTER(rx_ring);
-	u16 batched, leftover, i, tail_bumps;
+	u16 leftover, i, tail_bumps;
 
-	batched = ALIGN_DOWN(count, rx_thresh);
-	tail_bumps = batched / rx_thresh;
-	leftover = count & (rx_thresh - 1);
+	tail_bumps = count / rx_thresh;
+	leftover = count - (tail_bumps * rx_thresh);
 
 	for (i = 0; i < tail_bumps; i++)
 		if (!__ice_alloc_rx_bufs_zc(rx_ring, rx_thresh))
@@ -788,69 +780,57 @@ ice_clean_xdp_tx_buf(struct ice_tx_ring *xdp_ring, struct ice_tx_buf *tx_buf)
 }
 
 /**
- * ice_clean_xdp_irq_zc - Reclaim resources after transmit completes on XDP ring
- * @xdp_ring: XDP ring to clean
- * @napi_budget: amount of descriptors that NAPI allows us to clean
- *
- * Returns count of cleaned descriptors
+ * ice_clean_xdp_irq_zc - produce AF_XDP descriptors to CQ
+ * @xdp_ring: XDP Tx ring
  */
-static u16 ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring, int napi_budget)
+static void ice_clean_xdp_irq_zc(struct ice_tx_ring *xdp_ring)
 {
-	u16 tx_thresh = ICE_RING_QUARTER(xdp_ring);
-	int budget = napi_budget / tx_thresh;
-	u16 next_dd = xdp_ring->next_dd;
-	u16 ntc, cleared_dds = 0;
-
-	do {
-		struct ice_tx_desc *next_dd_desc;
-		u16 desc_cnt = xdp_ring->count;
-		struct ice_tx_buf *tx_buf;
-		u32 xsk_frames;
-		u16 i;
-
-		next_dd_desc = ICE_TX_DESC(xdp_ring, next_dd);
-		if (!(next_dd_desc->cmd_type_offset_bsz &
-		    cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE)))
-			break;
+	u16 ntc = xdp_ring->next_to_clean;
+	struct ice_tx_desc *tx_desc;
+	u16 cnt = xdp_ring->count;
+	struct ice_tx_buf *tx_buf;
+	u16 xsk_frames = 0;
+	u16 last_rs;
+	int i;
 
-		cleared_dds++;
-		xsk_frames = 0;
-		if (likely(!xdp_ring->xdp_tx_active)) {
-			xsk_frames = tx_thresh;
-			goto skip;
-		}
+	last_rs = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : cnt - 1;
+	tx_desc = ICE_TX_DESC(xdp_ring, last_rs);
+	if ((tx_desc->cmd_type_offset_bsz &
+	    cpu_to_le64(ICE_TX_DESC_DTYPE_DESC_DONE))) {
+		if (last_rs >= ntc)
+			xsk_frames = last_rs - ntc + 1;
+		else
+			xsk_frames = last_rs + cnt - ntc + 1;
+	}
 
-		ntc = xdp_ring->next_to_clean;
+	if (!xsk_frames)
+		return;
 
-		for (i = 0; i < tx_thresh; i++) {
-			tx_buf = &xdp_ring->tx_buf[ntc];
+	if (likely(!xdp_ring->xdp_tx_active))
+		goto skip;
 
-			if (tx_buf->raw_buf) {
-				ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
-				tx_buf->raw_buf = NULL;
-			} else {
-				xsk_frames++;
-			}
+	ntc = xdp_ring->next_to_clean;
+	for (i = 0; i < xsk_frames; i++) {
+		tx_buf = &xdp_ring->tx_buf[ntc];
 
-			ntc++;
-			if (ntc >= xdp_ring->count)
-				ntc = 0;
+		if (tx_buf->raw_buf) {
+			ice_clean_xdp_tx_buf(xdp_ring, tx_buf);
+			tx_buf->raw_buf = NULL;
+		} else {
+			xsk_frames++;
 		}
+
+		ntc++;
+		if (ntc >= xdp_ring->count)
+			ntc = 0;
+	}
 skip:
-		xdp_ring->next_to_clean += tx_thresh;
-		if (xdp_ring->next_to_clean >= desc_cnt)
-			xdp_ring->next_to_clean -= desc_cnt;
-		if (xsk_frames)
-			xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
-		next_dd_desc->cmd_type_offset_bsz = 0;
-		next_dd = next_dd + tx_thresh;
-		if (next_dd >= desc_cnt)
-			next_dd = tx_thresh - 1;
-	} while (--budget);
-
-	xdp_ring->next_dd = next_dd;
-
-	return cleared_dds * tx_thresh;
+	tx_desc->cmd_type_offset_bsz = 0;
+	xdp_ring->next_to_clean += xsk_frames;
+	if (xdp_ring->next_to_clean >= cnt)
+		xdp_ring->next_to_clean -= cnt;
+	if (xsk_frames)
+		xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames);
 }
 
 /**
@@ -885,7 +865,6 @@ static void ice_xmit_pkt(struct ice_tx_ring *xdp_ring, struct xdp_desc *desc,
 static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs,
 			       unsigned int *total_bytes)
 {
-	u16 tx_thresh = ICE_RING_QUARTER(xdp_ring);
 	u16 ntu = xdp_ring->next_to_use;
 	struct ice_tx_desc *tx_desc;
 	u32 i;
@@ -905,13 +884,6 @@ static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *de
 	}
 
 	xdp_ring->next_to_use = ntu;
-
-	if (xdp_ring->next_to_use > xdp_ring->next_rs) {
-		tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_rs);
-		tx_desc->cmd_type_offset_bsz |=
-			cpu_to_le64(ICE_TX_DESC_CMD_RS << ICE_TXD_QW1_CMD_S);
-		xdp_ring->next_rs += tx_thresh;
-	}
 }
 
 /**
@@ -924,7 +896,6 @@ static void ice_xmit_pkt_batch(struct ice_tx_ring *xdp_ring, struct xdp_desc *de
 static void ice_fill_tx_hw_ring(struct ice_tx_ring *xdp_ring, struct xdp_desc *descs,
 				u32 nb_pkts, unsigned int *total_bytes)
 {
-	u16 tx_thresh = ICE_RING_QUARTER(xdp_ring);
 	u32 batched, leftover, i;
 
 	batched = ALIGN_DOWN(nb_pkts, PKTS_PER_BATCH);
@@ -933,54 +904,54 @@ static void ice_fill_tx_hw_ring(struct ice_tx_ring *xdp_ring, struct xdp_desc *d
 		ice_xmit_pkt_batch(xdp_ring, &descs[i], total_bytes);
 	for (; i < batched + leftover; i++)
 		ice_xmit_pkt(xdp_ring, &descs[i], total_bytes);
+}
 
-	if (xdp_ring->next_to_use > xdp_ring->next_rs) {
-		struct ice_tx_desc *tx_desc;
+/**
+ * ice_set_rs_bit - set RS bit on last produced descriptor (one behind current NTU)
+ * @xdp_ring: XDP ring to produce the HW Tx descriptors on
+ */
+static void ice_set_rs_bit(struct ice_tx_ring *xdp_ring)
+{
+	u16 ntu = xdp_ring->next_to_use ? xdp_ring->next_to_use - 1 : xdp_ring->count - 1;
+	struct ice_tx_desc *tx_desc;
 
-		tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_rs);
-		tx_desc->cmd_type_offset_bsz |=
-			cpu_to_le64(ICE_TX_DESC_CMD_RS << ICE_TXD_QW1_CMD_S);
-		xdp_ring->next_rs += tx_thresh;
-	}
+	tx_desc = ICE_TX_DESC(xdp_ring, ntu);
+	tx_desc->cmd_type_offset_bsz |=
+		cpu_to_le64(ICE_TX_DESC_CMD_RS << ICE_TXD_QW1_CMD_S);
 }
 
 /**
  * ice_xmit_zc - take entries from XSK Tx ring and place them onto HW Tx ring
  * @xdp_ring: XDP ring to produce the HW Tx descriptors on
- * @budget: number of free descriptors on HW Tx ring that can be used
- * @napi_budget: amount of descriptors that NAPI allows us to clean
  *
  * Returns true if there is no more work that needs to be done, false otherwise
  */
-bool ice_xmit_zc(struct ice_tx_ring *xdp_ring, u32 budget, int napi_budget)
+bool ice_xmit_zc(struct ice_tx_ring *xdp_ring)
 {
 	struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs;
-	u16 tx_thresh = ICE_RING_QUARTER(xdp_ring);
 	u32 nb_pkts, nb_processed = 0;
 	unsigned int total_bytes = 0;
+	int budget;
+
+	ice_clean_xdp_irq_zc(xdp_ring);
 
-	if (budget < tx_thresh)
-		budget += ice_clean_xdp_irq_zc(xdp_ring, napi_budget);
+	budget = ICE_DESC_UNUSED(xdp_ring);
+	budget = min_t(u16, budget, ICE_RING_QUARTER(xdp_ring));
 
 	nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget);
 	if (!nb_pkts)
 		return true;
 
 	if (xdp_ring->next_to_use + nb_pkts >= xdp_ring->count) {
-		struct ice_tx_desc *tx_desc;
-
 		nb_processed = xdp_ring->count - xdp_ring->next_to_use;
 		ice_fill_tx_hw_ring(xdp_ring, descs, nb_processed, &total_bytes);
-		tx_desc = ICE_TX_DESC(xdp_ring, xdp_ring->next_rs);
-		tx_desc->cmd_type_offset_bsz |=
-			cpu_to_le64(ICE_TX_DESC_CMD_RS << ICE_TXD_QW1_CMD_S);
-		xdp_ring->next_rs = tx_thresh - 1;
 		xdp_ring->next_to_use = 0;
 	}
 
 	ice_fill_tx_hw_ring(xdp_ring, &descs[nb_processed], nb_pkts - nb_processed,
 			    &total_bytes);
 
+	ice_set_rs_bit(xdp_ring);
 	ice_xdp_ring_update_tail(xdp_ring);
 	ice_update_tx_ring_stats(xdp_ring, nb_pkts, total_bytes);
 
@@ -1058,14 +1029,16 @@ bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi)
  */
 void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring)
 {
-	u16 count_mask = rx_ring->count - 1;
 	u16 ntc = rx_ring->next_to_clean;
 	u16 ntu = rx_ring->next_to_use;
 
-	for ( ; ntc != ntu; ntc = (ntc + 1) & count_mask) {
+	while (ntc != ntu) {
 		struct xdp_buff *xdp = *ice_xdp_buf(rx_ring, ntc);
 
 		xsk_buff_free(xdp);
+		ntc++;
+		if (ntc >= rx_ring->count)
+			ntc = 0;
 	}
 }
 
diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.h b/drivers/net/ethernet/intel/ice/ice_xsk.h
index 4edbe81eb646..6fa181f080ef 100644
--- a/drivers/net/ethernet/intel/ice/ice_xsk.h
+++ b/drivers/net/ethernet/intel/ice/ice_xsk.h
@@ -26,13 +26,10 @@ bool ice_alloc_rx_bufs_zc(struct ice_rx_ring *rx_ring, u16 count);
 bool ice_xsk_any_rx_ring_ena(struct ice_vsi *vsi);
 void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring);
 void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring);
-bool ice_xmit_zc(struct ice_tx_ring *xdp_ring, u32 budget, int napi_budget);
+bool ice_xmit_zc(struct ice_tx_ring *xdp_ring);
 int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc);
 #else
-static inline bool
-ice_xmit_zc(struct ice_tx_ring __always_unused *xdp_ring,
-	    u32 __always_unused budget,
-	    int __always_unused napi_budget)
+static inline bool ice_xmit_zc(struct ice_tx_ring __always_unused *xdp_ring)
 {
 	return false;
 }
diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c
index c14fc871dd41..e5f3e7680dc6 100644
--- a/drivers/net/ethernet/intel/igb/igb_ethtool.c
+++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c
@@ -850,14 +850,14 @@ static void igb_get_drvinfo(struct net_device *netdev,
 {
 	struct igb_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver,  igb_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->driver,  igb_driver_name, sizeof(drvinfo->driver));
 
 	/* EEPROM image version # is reported as firmware version # for
 	 * 82575 controllers
 	 */
-	strlcpy(drvinfo->fw_version, adapter->fw_version,
+	strscpy(drvinfo->fw_version, adapter->fw_version,
 		sizeof(drvinfo->fw_version));
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 
 	drvinfo->n_priv_flags = IGB_PRIV_FLAGS_STR_LEN;
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index 2796e81d2726..f8e32833226c 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -1211,8 +1211,7 @@ static int igb_alloc_q_vector(struct igb_adapter *adapter,
 		return -ENOMEM;
 
 	/* initialize NAPI */
-	netif_napi_add(adapter->netdev, &q_vector->napi,
-		       igb_poll, 64);
+	netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll);
 
 	/* tie q_vector and adapter together */
 	adapter->q_vector[v_idx] = q_vector;
@@ -3138,7 +3137,7 @@ static s32 igb_init_i2c(struct igb_adapter *adapter)
 	adapter->i2c_algo.data = adapter;
 	adapter->i2c_adap.algo_data = &adapter->i2c_algo;
 	adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
-	strlcpy(adapter->i2c_adap.name, "igb BB",
+	strscpy(adapter->i2c_adap.name, "igb BB",
 		sizeof(adapter->i2c_adap.name));
 	status = i2c_bit_add_bus(&adapter->i2c_adap);
 	return status;
diff --git a/drivers/net/ethernet/intel/igbvf/ethtool.c b/drivers/net/ethernet/intel/igbvf/ethtool.c
index 9d4322b74163..83b97989a6bd 100644
--- a/drivers/net/ethernet/intel/igbvf/ethtool.c
+++ b/drivers/net/ethernet/intel/igbvf/ethtool.c
@@ -169,8 +169,8 @@ static void igbvf_get_drvinfo(struct net_device *netdev,
 {
 	struct igbvf_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver,  igbvf_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->driver,  igbvf_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c
index f4e91db89fe5..3a32809510fc 100644
--- a/drivers/net/ethernet/intel/igbvf/netdev.c
+++ b/drivers/net/ethernet/intel/igbvf/netdev.c
@@ -1109,7 +1109,7 @@ static int igbvf_alloc_queues(struct igbvf_adapter *adapter)
 		return -ENOMEM;
 	}
 
-	netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll, 64);
+	netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h
index 5c66b97c0cfa..4f9d7f013a95 100644
--- a/drivers/net/ethernet/intel/igc/igc_defines.h
+++ b/drivers/net/ethernet/intel/igc/igc_defines.h
@@ -610,7 +610,6 @@
 #define IGC_MDIC_OP_WRITE	0x04000000
 #define IGC_MDIC_OP_READ	0x08000000
 #define IGC_MDIC_READY		0x10000000
-#define IGC_MDIC_INT_EN		0x20000000
 #define IGC_MDIC_ERROR		0x40000000
 
 #define IGC_N0_QUEUE		-1
diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c
index ebff0e04045d..34889be63e78 100644
--- a/drivers/net/ethernet/intel/igc/igc_main.c
+++ b/drivers/net/ethernet/intel/igc/igc_main.c
@@ -2129,65 +2129,102 @@ static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count)
 	return ok;
 }
 
-static int igc_xdp_init_tx_buffer(struct igc_tx_buffer *buffer,
-				  struct xdp_frame *xdpf,
-				  struct igc_ring *ring)
-{
-	dma_addr_t dma;
-
-	dma = dma_map_single(ring->dev, xdpf->data, xdpf->len, DMA_TO_DEVICE);
-	if (dma_mapping_error(ring->dev, dma)) {
-		netdev_err_once(ring->netdev, "Failed to map DMA for TX\n");
-		return -ENOMEM;
-	}
-
-	buffer->type = IGC_TX_BUFFER_TYPE_XDP;
-	buffer->xdpf = xdpf;
-	buffer->protocol = 0;
-	buffer->bytecount = xdpf->len;
-	buffer->gso_segs = 1;
-	buffer->time_stamp = jiffies;
-	dma_unmap_len_set(buffer, len, xdpf->len);
-	dma_unmap_addr_set(buffer, dma, dma);
-	return 0;
-}
-
 /* This function requires __netif_tx_lock is held by the caller. */
 static int igc_xdp_init_tx_descriptor(struct igc_ring *ring,
 				      struct xdp_frame *xdpf)
 {
-	struct igc_tx_buffer *buffer;
-	union igc_adv_tx_desc *desc;
-	u32 cmd_type, olinfo_status;
-	int err;
+	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
+	u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
+	u16 count, index = ring->next_to_use;
+	struct igc_tx_buffer *head = &ring->tx_buffer_info[index];
+	struct igc_tx_buffer *buffer = head;
+	union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index);
+	u32 olinfo_status, len = xdpf->len, cmd_type;
+	void *data = xdpf->data;
+	u16 i;
 
-	if (!igc_desc_unused(ring))
-		return -EBUSY;
+	count = TXD_USE_COUNT(len);
+	for (i = 0; i < nr_frags; i++)
+		count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
 
-	buffer = &ring->tx_buffer_info[ring->next_to_use];
-	err = igc_xdp_init_tx_buffer(buffer, xdpf, ring);
-	if (err)
-		return err;
+	if (igc_maybe_stop_tx(ring, count + 3)) {
+		/* this is a hard error */
+		return -EBUSY;
+	}
 
-	cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
-		   IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
-		   buffer->bytecount;
-	olinfo_status = buffer->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
+	i = 0;
+	head->bytecount = xdp_get_frame_len(xdpf);
+	head->type = IGC_TX_BUFFER_TYPE_XDP;
+	head->gso_segs = 1;
+	head->xdpf = xdpf;
 
-	desc = IGC_TX_DESC(ring, ring->next_to_use);
-	desc->read.cmd_type_len = cpu_to_le32(cmd_type);
+	olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
 	desc->read.olinfo_status = cpu_to_le32(olinfo_status);
-	desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(buffer, dma));
 
-	netdev_tx_sent_queue(txring_txq(ring), buffer->bytecount);
+	for (;;) {
+		dma_addr_t dma;
 
-	buffer->next_to_watch = desc;
+		dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE);
+		if (dma_mapping_error(ring->dev, dma)) {
+			netdev_err_once(ring->netdev,
+					"Failed to map DMA for TX\n");
+			goto unmap;
+		}
 
-	ring->next_to_use++;
-	if (ring->next_to_use == ring->count)
-		ring->next_to_use = 0;
+		dma_unmap_len_set(buffer, len, len);
+		dma_unmap_addr_set(buffer, dma, dma);
+
+		cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
+			   IGC_ADVTXD_DCMD_IFCS | len;
+
+		desc->read.cmd_type_len = cpu_to_le32(cmd_type);
+		desc->read.buffer_addr = cpu_to_le64(dma);
+
+		buffer->protocol = 0;
+
+		if (++index == ring->count)
+			index = 0;
+
+		if (i == nr_frags)
+			break;
+
+		buffer = &ring->tx_buffer_info[index];
+		desc = IGC_TX_DESC(ring, index);
+		desc->read.olinfo_status = 0;
+
+		data = skb_frag_address(&sinfo->frags[i]);
+		len = skb_frag_size(&sinfo->frags[i]);
+		i++;
+	}
+	desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD);
+
+	netdev_tx_sent_queue(txring_txq(ring), head->bytecount);
+	/* set the timestamp */
+	head->time_stamp = jiffies;
+	/* set next_to_watch value indicating a packet is present */
+	head->next_to_watch = desc;
+	ring->next_to_use = index;
 
 	return 0;
+
+unmap:
+	for (;;) {
+		buffer = &ring->tx_buffer_info[index];
+		if (dma_unmap_len(buffer, len))
+			dma_unmap_page(ring->dev,
+				       dma_unmap_addr(buffer, dma),
+				       dma_unmap_len(buffer, len),
+				       DMA_TO_DEVICE);
+		dma_unmap_len_set(buffer, len, 0);
+		if (buffer == head)
+			break;
+
+		if (!index)
+			index += ring->count;
+		index--;
+	}
+
+	return -ENOMEM;
 }
 
 static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter,
@@ -2369,6 +2406,7 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
 			xdp_prepare_buff(&xdp, pktbuf - igc_rx_offset(rx_ring),
 					 igc_rx_offset(rx_ring) + pkt_offset,
 					 size, true);
+			xdp_buff_clear_frags_flag(&xdp);
 
 			skb = igc_xdp_run_prog(adapter, &xdp);
 		}
@@ -4356,8 +4394,7 @@ static int igc_alloc_q_vector(struct igc_adapter *adapter,
 		return -ENOMEM;
 
 	/* initialize NAPI */
-	netif_napi_add(adapter->netdev, &q_vector->napi,
-		       igc_poll, 64);
+	netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll);
 
 	/* tie q_vector and adapter together */
 	adapter->q_vector[v_idx] = q_vector;
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c
index 46efcfab7234..efa980514944 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c
@@ -456,9 +456,9 @@ ixgb_get_drvinfo(struct net_device *netdev,
 {
 	struct ixgb_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver,  ixgb_driver_name,
+	strscpy(drvinfo->driver,  ixgb_driver_name,
 		sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_main.c b/drivers/net/ethernet/intel/ixgb/ixgb_main.c
index 45be9a1ab6af..b4d47e7a76c8 100644
--- a/drivers/net/ethernet/intel/ixgb/ixgb_main.c
+++ b/drivers/net/ethernet/intel/ixgb/ixgb_main.c
@@ -414,7 +414,7 @@ ixgb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	netdev->netdev_ops = &ixgb_netdev_ops;
 	ixgb_set_ethtool_ops(netdev);
 	netdev->watchdog_timeo = 5 * HZ;
-	netif_napi_add(netdev, &adapter->napi, ixgb_clean, 64);
+	netif_napi_add(netdev, &adapter->napi, ixgb_clean);
 
 	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
index 04f453eabef6..e88e3dfac8c2 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
@@ -1106,12 +1106,12 @@ static void ixgbe_get_drvinfo(struct net_device *netdev,
 {
 	struct ixgbe_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, ixgbe_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->driver, ixgbe_driver_name, sizeof(drvinfo->driver));
 
-	strlcpy(drvinfo->fw_version, adapter->eeprom_id,
+	strscpy(drvinfo->fw_version, adapter->eeprom_id,
 		sizeof(drvinfo->fw_version));
 
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 
 	drvinfo->n_priv_flags = IXGBE_PRIV_FLAGS_STR_LEN;
@@ -1964,15 +1964,13 @@ static bool ixgbe_check_lbtest_frame(struct ixgbe_rx_buffer *rx_buffer,
 
 	frame_size >>= 1;
 
-	data = kmap(rx_buffer->page) + rx_buffer->page_offset;
+	data = page_address(rx_buffer->page) + rx_buffer->page_offset;
 
 	if (data[3] != 0xFF ||
 	    data[frame_size + 10] != 0xBE ||
 	    data[frame_size + 12] != 0xAF)
 		match = false;
 
-	kunmap(rx_buffer->page);
-
 	return match;
 }
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c
index 0fcd82036d4e..7311bd545acf 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c
@@ -1004,7 +1004,7 @@ int ixgbe_fcoe_get_hbainfo(struct net_device *netdev,
 		 ixgbe_driver_name,
 		 UTS_RELEASE);
 	/* Firmware Version */
-	strlcpy(info->firmware_version, adapter->eeprom_id,
+	strscpy(info->firmware_version, adapter->eeprom_id,
 		sizeof(info->firmware_version));
 
 	/* Model */
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
index 86b11164655e..f8156fe4b1dc 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c
@@ -874,8 +874,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter,
 
 #endif
 	/* initialize NAPI */
-	netif_napi_add(adapter->netdev, &q_vector->napi,
-		       ixgbe_poll, 64);
+	netif_napi_add(adapter->netdev, &q_vector->napi, ixgbe_poll);
 
 	/* tie q_vector and adapter together */
 	adapter->q_vector[v_idx] = q_vector;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index d1e430b8c8aa..298cfbfcb7b6 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -10849,7 +10849,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	netdev->netdev_ops = &ixgbe_netdev_ops;
 	ixgbe_set_ethtool_ops(netdev);
 	netdev->watchdog_timeo = 5 * HZ;
-	strlcpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
+	strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
 
 	/* Setup hw api */
 	hw->mac.ops   = *ii->mac_ops;
@@ -11140,7 +11140,7 @@ skip_sriov:
 
 	err = ixgbe_read_pba_string_generic(hw, part_str, sizeof(part_str));
 	if (err)
-		strlcpy(part_str, "Unknown", sizeof(part_str));
+		strscpy(part_str, "Unknown", sizeof(part_str));
 	if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
 		e_dev_info("MAC: %d, PHY: %d, SFP+: %d, PBA No: %s\n",
 			   hw->mac.type, hw->phy.type, hw->phy.sfp_type,
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
index 7f7ea468ffa9..2b00db92b08f 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
@@ -3712,7 +3712,9 @@ struct ixgbe_info {
 #define IXGBE_KRM_LINK_S1(P)		((P) ? 0x8200 : 0x4200)
 #define IXGBE_KRM_LINK_CTRL_1(P)	((P) ? 0x820C : 0x420C)
 #define IXGBE_KRM_AN_CNTL_1(P)		((P) ? 0x822C : 0x422C)
+#define IXGBE_KRM_AN_CNTL_4(P)		((P) ? 0x8238 : 0x4238)
 #define IXGBE_KRM_AN_CNTL_8(P)		((P) ? 0x8248 : 0x4248)
+#define IXGBE_KRM_PCS_KX_AN(P)		((P) ? 0x9918 : 0x5918)
 #define IXGBE_KRM_SGMII_CTRL(P)		((P) ? 0x82A0 : 0x42A0)
 #define IXGBE_KRM_LP_BASE_PAGE_HIGH(P)	((P) ? 0x836C : 0x436C)
 #define IXGBE_KRM_DSP_TXFFE_STATE_4(P)	((P) ? 0x8634 : 0x4634)
@@ -3722,6 +3724,7 @@ struct ixgbe_info {
 #define IXGBE_KRM_PMD_FLX_MASK_ST20(P)	((P) ? 0x9054 : 0x5054)
 #define IXGBE_KRM_TX_COEFF_CTRL_1(P)	((P) ? 0x9520 : 0x5520)
 #define IXGBE_KRM_RX_ANA_CTL(P)		((P) ? 0x9A00 : 0x5A00)
+#define IXGBE_KRM_FLX_TMRS_CTRL_ST31(P)	((P) ? 0x9180 : 0x5180)
 
 #define IXGBE_KRM_PMD_FLX_MASK_ST20_SFI_10G_DA		~(0x3 << 20)
 #define IXGBE_KRM_PMD_FLX_MASK_ST20_SFI_10G_SR		BIT(20)
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
index 35c2b9b8bd19..aa4bf6c9a2f7 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
@@ -1721,9 +1721,59 @@ static s32 ixgbe_setup_sfi_x550a(struct ixgbe_hw *hw, ixgbe_link_speed *speed)
 		return IXGBE_ERR_LINK_SETUP;
 	}
 
-	status = mac->ops.write_iosf_sb_reg(hw,
-				IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id),
-				IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val);
+	(void)mac->ops.write_iosf_sb_reg(hw,
+			IXGBE_KRM_PMD_FLX_MASK_ST20(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val);
+
+	/* change mode enforcement rules to hybrid */
+	(void)mac->ops.read_iosf_sb_reg(hw,
+			IXGBE_KRM_FLX_TMRS_CTRL_ST31(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, &reg_val);
+	reg_val |= 0x0400;
+
+	(void)mac->ops.write_iosf_sb_reg(hw,
+			IXGBE_KRM_FLX_TMRS_CTRL_ST31(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val);
+
+	/* manually control the config */
+	(void)mac->ops.read_iosf_sb_reg(hw,
+			IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, &reg_val);
+	reg_val |= 0x20002240;
+
+	(void)mac->ops.write_iosf_sb_reg(hw,
+			IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val);
+
+	/* move the AN base page values */
+	(void)mac->ops.read_iosf_sb_reg(hw,
+			IXGBE_KRM_PCS_KX_AN(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, &reg_val);
+	reg_val |= 0x1;
+
+	(void)mac->ops.write_iosf_sb_reg(hw,
+			IXGBE_KRM_PCS_KX_AN(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val);
+
+	/* set the AN37 over CB mode */
+	(void)mac->ops.read_iosf_sb_reg(hw,
+			IXGBE_KRM_AN_CNTL_4(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, &reg_val);
+	reg_val |= 0x20000000;
+
+	(void)mac->ops.write_iosf_sb_reg(hw,
+			IXGBE_KRM_AN_CNTL_4(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val);
+
+	/* restart AN manually */
+	(void)mac->ops.read_iosf_sb_reg(hw,
+			IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, &reg_val);
+	reg_val |= IXGBE_KRM_LINK_CTRL_1_TETH_AN_RESTART;
+
+	(void)mac->ops.write_iosf_sb_reg(hw,
+			IXGBE_KRM_LINK_CTRL_1(hw->bus.lan_id),
+			IXGBE_SB_IOSF_TARGET_KR_PHY, reg_val);
 
 	/* Toggle port SW reset by AN reset. */
 	status = ixgbe_restart_an_internal_phy_x550em(hw);
diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c
index fed46872af2b..ccfa6b91aac6 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c
@@ -213,8 +213,8 @@ static void ixgbevf_get_drvinfo(struct net_device *netdev,
 {
 	struct ixgbevf_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, ixgbevf_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->driver, ixgbevf_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 
 	drvinfo->n_priv_flags = IXGBEVF_PRIV_FLAGS_STR_LEN;
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index 2f12fbe229c1..99933e89717a 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -2733,7 +2733,7 @@ static int ixgbevf_alloc_q_vector(struct ixgbevf_adapter *adapter, int v_idx,
 		return -ENOMEM;
 
 	/* initialize NAPI */
-	netif_napi_add(adapter->netdev, &q_vector->napi, ixgbevf_poll, 64);
+	netif_napi_add(adapter->netdev, &q_vector->napi, ixgbevf_poll);
 
 	/* tie q_vector and adapter together */
 	adapter->q_vector[v_idx] = q_vector;
diff --git a/drivers/net/ethernet/jme.c b/drivers/net/ethernet/jme.c
index f43d6616bc0d..1732ec3c3dbd 100644
--- a/drivers/net/ethernet/jme.c
+++ b/drivers/net/ethernet/jme.c
@@ -2332,9 +2332,9 @@ jme_get_drvinfo(struct net_device *netdev,
 {
 	struct jme_adapter *jme = netdev_priv(netdev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(jme->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(jme->pdev), sizeof(info->bus_info));
 }
 
 static int
@@ -3009,7 +3009,7 @@ jme_init_one(struct pci_dev *pdev,
 		jwrite32(jme, JME_APMC, apmc);
 	}
 
-	netif_napi_add(netdev, &jme->napi, jme_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, &jme->napi, jme_poll);
 
 	spin_lock_init(&jme->phy_lock);
 	spin_lock_init(&jme->macaddr_lock);
diff --git a/drivers/net/ethernet/korina.c b/drivers/net/ethernet/korina.c
index df9a8eefa007..2b9335cb4bb3 100644
--- a/drivers/net/ethernet/korina.c
+++ b/drivers/net/ethernet/korina.c
@@ -416,7 +416,8 @@ static void korina_abort_rx(struct net_device *dev)
 }
 
 /* transmit packet */
-static int korina_send_packet(struct sk_buff *skb, struct net_device *dev)
+static netdev_tx_t korina_send_packet(struct sk_buff *skb,
+				      struct net_device *dev)
 {
 	struct korina_private *lp = netdev_priv(dev);
 	u32 chain_prev, chain_next;
@@ -938,9 +939,9 @@ static void netdev_get_drvinfo(struct net_device *dev,
 {
 	struct korina_private *lp = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, lp->dev->name, sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, lp->dev->name, sizeof(info->bus_info));
 }
 
 static int netdev_get_link_ksettings(struct net_device *dev,
@@ -1354,7 +1355,7 @@ static int korina_probe(struct platform_device *pdev)
 	dev->netdev_ops = &korina_netdev_ops;
 	dev->ethtool_ops = &netdev_ethtool_ops;
 	dev->watchdog_timeo = TX_TIMEOUT;
-	netif_napi_add(dev, &lp->napi, korina_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &lp->napi, korina_poll);
 
 	lp->mii_if.dev = dev;
 	lp->mii_if.mdio_read = korina_mdio_read;
diff --git a/drivers/net/ethernet/lantiq_etop.c b/drivers/net/ethernet/lantiq_etop.c
index 7cedbe1fdfd7..59aab4086dcc 100644
--- a/drivers/net/ethernet/lantiq_etop.c
+++ b/drivers/net/ethernet/lantiq_etop.c
@@ -470,7 +470,7 @@ ltq_etop_stop(struct net_device *dev)
 	return 0;
 }
 
-static int
+static netdev_tx_t
 ltq_etop_tx(struct sk_buff *skb, struct net_device *dev)
 {
 	int queue = skb_get_queue_mapping(skb);
diff --git a/drivers/net/ethernet/lantiq_xrx200.c b/drivers/net/ethernet/lantiq_xrx200.c
index 57f27cc7724e..8d646c7f8c82 100644
--- a/drivers/net/ethernet/lantiq_xrx200.c
+++ b/drivers/net/ethernet/lantiq_xrx200.c
@@ -620,8 +620,7 @@ static int xrx200_probe(struct platform_device *pdev)
 			 PMAC_HD_CTL);
 
 	/* setup NAPI */
-	netif_napi_add(net_dev, &priv->chan_rx.napi, xrx200_poll_rx,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(net_dev, &priv->chan_rx.napi, xrx200_poll_rx);
 	netif_napi_add_tx(net_dev, &priv->chan_tx.napi,
 			  xrx200_tx_housekeeping);
 
diff --git a/drivers/net/ethernet/litex/litex_liteeth.c b/drivers/net/ethernet/litex/litex_liteeth.c
index fdd99f0de424..35f24e0f0934 100644
--- a/drivers/net/ethernet/litex/litex_liteeth.c
+++ b/drivers/net/ethernet/litex/litex_liteeth.c
@@ -152,7 +152,8 @@ static int liteeth_stop(struct net_device *netdev)
 	return 0;
 }
 
-static int liteeth_start_xmit(struct sk_buff *skb, struct net_device *netdev)
+static netdev_tx_t liteeth_start_xmit(struct sk_buff *skb,
+				      struct net_device *netdev)
 {
 	struct liteeth *priv = netdev_priv(netdev);
 	void __iomem *txbuffer;
diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c
index b6be0552a6c1..707993b445d1 100644
--- a/drivers/net/ethernet/marvell/mv643xx_eth.c
+++ b/drivers/net/ethernet/marvell/mv643xx_eth.c
@@ -1603,12 +1603,12 @@ mv643xx_eth_set_link_ksettings(struct net_device *dev,
 static void mv643xx_eth_get_drvinfo(struct net_device *dev,
 				    struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, mv643xx_eth_driver_name,
+	strscpy(drvinfo->driver, mv643xx_eth_driver_name,
 		sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, mv643xx_eth_driver_version,
+	strscpy(drvinfo->version, mv643xx_eth_driver_version,
 		sizeof(drvinfo->version));
-	strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version));
-	strlcpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info));
+	strscpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version));
+	strscpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info));
 }
 
 static int mv643xx_eth_get_coalesce(struct net_device *dev,
@@ -3183,7 +3183,7 @@ static int mv643xx_eth_probe(struct platform_device *pdev)
 
 	INIT_WORK(&mp->tx_timeout_task, tx_timeout_task);
 
-	netif_napi_add(dev, &mp->napi, mv643xx_eth_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &mp->napi, mv643xx_eth_poll);
 
 	timer_setup(&mp->rx_oom, oom_timer_wrapper, 0);
 
diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c
index 0caa2df87c04..ff3e361e06e7 100644
--- a/drivers/net/ethernet/marvell/mvneta.c
+++ b/drivers/net/ethernet/marvell/mvneta.c
@@ -4656,11 +4656,11 @@ mvneta_ethtool_get_coalesce(struct net_device *dev,
 static void mvneta_ethtool_get_drvinfo(struct net_device *dev,
 				    struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, MVNETA_DRIVER_NAME,
+	strscpy(drvinfo->driver, MVNETA_DRIVER_NAME,
 		sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, MVNETA_DRIVER_VERSION,
+	strscpy(drvinfo->version, MVNETA_DRIVER_VERSION,
 		sizeof(drvinfo->version));
-	strlcpy(drvinfo->bus_info, dev_name(&dev->dev),
+	strscpy(drvinfo->bus_info, dev_name(&dev->dev),
 		sizeof(drvinfo->bus_info));
 }
 
@@ -5600,14 +5600,13 @@ static int mvneta_probe(struct platform_device *pdev)
 	 * operation, so only single NAPI should be initialized.
 	 */
 	if (pp->neta_armada3700) {
-		netif_napi_add(dev, &pp->napi, mvneta_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(dev, &pp->napi, mvneta_poll);
 	} else {
 		for_each_present_cpu(cpu) {
 			struct mvneta_pcpu_port *port =
 				per_cpu_ptr(pp->ports, cpu);
 
-			netif_napi_add(dev, &port->napi, mvneta_poll,
-				       NAPI_POLL_WEIGHT);
+			netif_napi_add(dev, &port->napi, mvneta_poll);
 			port->pp = pp;
 		}
 	}
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h
index ad73a488fc5f..11e603686a27 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2.h
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2.h
@@ -1530,6 +1530,7 @@ u32 mvpp2_read(struct mvpp2 *priv, u32 offset);
 void mvpp2_dbgfs_init(struct mvpp2 *priv, const char *name);
 
 void mvpp2_dbgfs_cleanup(struct mvpp2 *priv);
+void mvpp2_dbgfs_exit(void);
 
 void mvpp23_rx_fifo_fc_en(struct mvpp2 *priv, int port, bool en);
 
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c
index 0eec05d905eb..75e83ea2a926 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_debugfs.c
@@ -691,6 +691,13 @@ static int mvpp2_dbgfs_port_init(struct dentry *parent,
 	return 0;
 }
 
+static struct dentry *mvpp2_root;
+
+void mvpp2_dbgfs_exit(void)
+{
+	debugfs_remove(mvpp2_root);
+}
+
 void mvpp2_dbgfs_cleanup(struct mvpp2 *priv)
 {
 	debugfs_remove_recursive(priv->dbgfs_dir);
@@ -700,7 +707,6 @@ void mvpp2_dbgfs_cleanup(struct mvpp2 *priv)
 
 void mvpp2_dbgfs_init(struct mvpp2 *priv, const char *name)
 {
-	static struct dentry *mvpp2_root;
 	struct dentry *mvpp2_dir;
 	int ret, i;
 
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
index b84128b549b4..eb0fb8128096 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
@@ -5425,11 +5425,11 @@ mvpp2_ethtool_get_coalesce(struct net_device *dev,
 static void mvpp2_ethtool_get_drvinfo(struct net_device *dev,
 				      struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, MVPP2_DRIVER_NAME,
+	strscpy(drvinfo->driver, MVPP2_DRIVER_NAME,
 		sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, MVPP2_DRIVER_VERSION,
+	strscpy(drvinfo->version, MVPP2_DRIVER_VERSION,
 		sizeof(drvinfo->version));
-	strlcpy(drvinfo->bus_info, dev_name(&dev->dev),
+	strscpy(drvinfo->bus_info, dev_name(&dev->dev),
 		sizeof(drvinfo->bus_info));
 }
 
@@ -5770,8 +5770,7 @@ static int mvpp2_simple_queue_vectors_init(struct mvpp2_port *port,
 	v->irq = irq_of_parse_and_map(port_node, 0);
 	if (v->irq <= 0)
 		return -EINVAL;
-	netif_napi_add(port->dev, &v->napi, mvpp2_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(port->dev, &v->napi, mvpp2_poll);
 
 	port->nqvecs = 1;
 
@@ -5831,8 +5830,7 @@ static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port,
 			goto err;
 		}
 
-		netif_napi_add(port->dev, &v->napi, mvpp2_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(port->dev, &v->napi, mvpp2_poll);
 	}
 
 	return 0;
@@ -7706,7 +7704,18 @@ static struct platform_driver mvpp2_driver = {
 	},
 };
 
-module_platform_driver(mvpp2_driver);
+static int __init mvpp2_driver_init(void)
+{
+	return platform_driver_register(&mvpp2_driver);
+}
+module_init(mvpp2_driver_init);
+
+static void __exit mvpp2_driver_exit(void)
+{
+	platform_driver_unregister(&mvpp2_driver);
+	mvpp2_dbgfs_exit();
+}
+module_exit(mvpp2_driver_exit);
 
 MODULE_DESCRIPTION("Marvell PPv2 Ethernet Driver - www.marvell.com");
 MODULE_AUTHOR("Marcin Wojtas <mw@semihalf.com>");
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c
index 97f080c66dd4..9089adcb75f9 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_main.c
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_main.c
@@ -410,7 +410,7 @@ static void octep_napi_add(struct octep_device *oct)
 	for (i = 0; i < oct->num_oqs; i++) {
 		netdev_dbg(oct->netdev, "Adding NAPI on Q-%d\n", i);
 		netif_napi_add(oct->netdev, &oct->ioq_vector[i]->napi,
-			       octep_napi_poll, 64);
+			       octep_napi_poll);
 		oct->oq[i]->napi = &oct->ioq_vector[i]->napi;
 	}
 }
diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c
index d9ae0937d17a..392d9b0da0d7 100644
--- a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c
+++ b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c
@@ -158,8 +158,7 @@ static int octep_setup_oq(struct octep_device *oct, int q_no)
 		goto desc_dma_alloc_err;
 	}
 
-	oq->buff_info = (struct octep_rx_buffer *)
-			vzalloc(oq->max_count * OCTEP_OQ_RECVBUF_SIZE);
+	oq->buff_info = vzalloc(oq->max_count * OCTEP_OQ_RECVBUF_SIZE);
 	if (unlikely(!oq->buff_info)) {
 		dev_err(&oct->pdev->dev,
 			"Failed to allocate buffer info for OQ-%d\n", q_no);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/Makefile b/drivers/net/ethernet/marvell/octeontx2/af/Makefile
index 40203560b291..3cf4c8285c90 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/Makefile
+++ b/drivers/net/ethernet/marvell/octeontx2/af/Makefile
@@ -11,4 +11,4 @@ rvu_mbox-y := mbox.o rvu_trace.o
 rvu_af-y := cgx.o rvu.o rvu_cgx.o rvu_npa.o rvu_nix.o \
 		  rvu_reg.o rvu_npc.o rvu_debugfs.o ptp.o rvu_npc_fs.o \
 		  rvu_cpt.o rvu_devlink.o rpm.o rvu_cn10k.o rvu_switch.o \
-		  rvu_sdp.o rvu_npc_hash.o
+		  rvu_sdp.o rvu_npc_hash.o mcs.o mcs_rvu_if.o mcs_cnf10kb.o
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
index d7762577e285..8d5d5a0f68c4 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h
@@ -293,20 +293,74 @@ M(NIX_BANDPROF_ALLOC,	0x801d, nix_bandprof_alloc, nix_bandprof_alloc_req, \
 M(NIX_BANDPROF_FREE,	0x801e, nix_bandprof_free, nix_bandprof_free_req,   \
 				msg_rsp)				    \
 M(NIX_BANDPROF_GET_HWINFO, 0x801f, nix_bandprof_get_hwinfo, msg_req,		\
-				nix_bandprof_get_hwinfo_rsp)
-
-/* Messages initiated by AF (range 0xC00 - 0xDFF) */
+				nix_bandprof_get_hwinfo_rsp)		    \
+/* MCS mbox IDs (range 0xA000 - 0xBFFF) */					\
+M(MCS_ALLOC_RESOURCES,	0xa000, mcs_alloc_resources, mcs_alloc_rsrc_req,	\
+				mcs_alloc_rsrc_rsp)				\
+M(MCS_FREE_RESOURCES,	0xa001, mcs_free_resources, mcs_free_rsrc_req, msg_rsp) \
+M(MCS_FLOWID_ENTRY_WRITE, 0xa002, mcs_flowid_entry_write, mcs_flowid_entry_write_req,	\
+				msg_rsp)					\
+M(MCS_SECY_PLCY_WRITE,	0xa003, mcs_secy_plcy_write, mcs_secy_plcy_write_req,	\
+				msg_rsp)					\
+M(MCS_RX_SC_CAM_WRITE,	0xa004, mcs_rx_sc_cam_write, mcs_rx_sc_cam_write_req,	\
+				msg_rsp)					\
+M(MCS_SA_PLCY_WRITE,	0xa005, mcs_sa_plcy_write, mcs_sa_plcy_write_req,	\
+				msg_rsp)					\
+M(MCS_TX_SC_SA_MAP_WRITE, 0xa006, mcs_tx_sc_sa_map_write, mcs_tx_sc_sa_map,	\
+				  msg_rsp)					\
+M(MCS_RX_SC_SA_MAP_WRITE, 0xa007, mcs_rx_sc_sa_map_write, mcs_rx_sc_sa_map,	\
+				  msg_rsp)					\
+M(MCS_FLOWID_ENA_ENTRY,	0xa008, mcs_flowid_ena_entry, mcs_flowid_ena_dis_entry,	\
+				msg_rsp)					\
+M(MCS_PN_TABLE_WRITE,	0xa009, mcs_pn_table_write, mcs_pn_table_write_req,	\
+				msg_rsp)					\
+M(MCS_SET_ACTIVE_LMAC,	0xa00a,	mcs_set_active_lmac, mcs_set_active_lmac,	\
+				msg_rsp)					\
+M(MCS_GET_HW_INFO,	0xa00b,	mcs_get_hw_info, msg_req, mcs_hw_info)		\
+M(MCS_GET_FLOWID_STATS, 0xa00c, mcs_get_flowid_stats, mcs_stats_req,		\
+				mcs_flowid_stats)				\
+M(MCS_GET_SECY_STATS,	0xa00d, mcs_get_secy_stats, mcs_stats_req,		\
+				mcs_secy_stats)					\
+M(MCS_GET_SC_STATS,	0xa00e, mcs_get_sc_stats, mcs_stats_req, mcs_sc_stats)	\
+M(MCS_GET_SA_STATS,	0xa00f, mcs_get_sa_stats, mcs_stats_req, mcs_sa_stats)	\
+M(MCS_GET_PORT_STATS,	0xa010, mcs_get_port_stats, mcs_stats_req,		\
+				mcs_port_stats)					\
+M(MCS_CLEAR_STATS,	0xa011,	mcs_clear_stats, mcs_clear_stats, msg_rsp)	\
+M(MCS_INTR_CFG,		0xa012, mcs_intr_cfg, mcs_intr_cfg, msg_rsp)		\
+M(MCS_SET_LMAC_MODE,	0xa013, mcs_set_lmac_mode, mcs_set_lmac_mode, msg_rsp)	\
+M(MCS_SET_PN_THRESHOLD, 0xa014, mcs_set_pn_threshold, mcs_set_pn_threshold,	\
+				msg_rsp)					\
+M(MCS_ALLOC_CTRL_PKT_RULE, 0xa015, mcs_alloc_ctrl_pkt_rule,			\
+				   mcs_alloc_ctrl_pkt_rule_req,			\
+				   mcs_alloc_ctrl_pkt_rule_rsp)			\
+M(MCS_FREE_CTRL_PKT_RULE, 0xa016, mcs_free_ctrl_pkt_rule,			\
+				  mcs_free_ctrl_pkt_rule_req, msg_rsp)		\
+M(MCS_CTRL_PKT_RULE_WRITE, 0xa017, mcs_ctrl_pkt_rule_write,			\
+				   mcs_ctrl_pkt_rule_write_req, msg_rsp)	\
+M(MCS_PORT_RESET,	0xa018, mcs_port_reset, mcs_port_reset_req, msg_rsp)	\
+M(MCS_PORT_CFG_SET,	0xa019, mcs_port_cfg_set, mcs_port_cfg_set_req, msg_rsp)\
+M(MCS_PORT_CFG_GET,	0xa020, mcs_port_cfg_get, mcs_port_cfg_get_req,		\
+				mcs_port_cfg_get_rsp)				\
+M(MCS_CUSTOM_TAG_CFG_GET, 0xa021, mcs_custom_tag_cfg_get,			\
+				  mcs_custom_tag_cfg_get_req,			\
+				  mcs_custom_tag_cfg_get_rsp)
+
+/* Messages initiated by AF (range 0xC00 - 0xEFF) */
 #define MBOX_UP_CGX_MESSAGES						\
 M(CGX_LINK_EVENT,	0xC00, cgx_link_event, cgx_link_info_msg, msg_rsp)
 
 #define MBOX_UP_CPT_MESSAGES						\
 M(CPT_INST_LMTST,	0xD00, cpt_inst_lmtst, cpt_inst_lmtst_req, msg_rsp)
 
+#define MBOX_UP_MCS_MESSAGES						\
+M(MCS_INTR_NOTIFY,	0xE00, mcs_intr_notify, mcs_intr_info, msg_rsp)
+
 enum {
 #define M(_name, _id, _1, _2, _3) MBOX_MSG_ ## _name = _id,
 MBOX_MESSAGES
 MBOX_UP_CGX_MESSAGES
 MBOX_UP_CPT_MESSAGES
+MBOX_UP_MCS_MESSAGES
 #undef M
 };
 
@@ -1471,6 +1525,7 @@ enum ptp_op {
 	PTP_OP_GET_CLOCK = 1,
 	PTP_OP_GET_TSTMP = 2,
 	PTP_OP_SET_THRESH = 3,
+	PTP_OP_EXTTS_ON = 4,
 };
 
 struct ptp_req {
@@ -1478,6 +1533,7 @@ struct ptp_req {
 	u8 op;
 	s64 scaled_ppm;
 	u64 thresh;
+	int extts_on;
 };
 
 struct ptp_rsp {
@@ -1655,4 +1711,415 @@ enum cgx_af_status {
 	LMAC_AF_ERR_EXACT_MATCH_TBL_LOOK_UP_FAILED = -1110,
 };
 
+enum mcs_direction {
+	MCS_RX,
+	MCS_TX,
+};
+
+enum mcs_rsrc_type {
+	MCS_RSRC_TYPE_FLOWID,
+	MCS_RSRC_TYPE_SECY,
+	MCS_RSRC_TYPE_SC,
+	MCS_RSRC_TYPE_SA,
+};
+
+struct mcs_alloc_rsrc_req {
+	struct mbox_msghdr hdr;
+	u8 rsrc_type;
+	u8 rsrc_cnt;	/* Resources count */
+	u8 mcs_id;	/* MCS block ID	*/
+	u8 dir;		/* Macsec ingress or egress side */
+	u8 all;		/* Allocate all resource type one each */
+	u64 rsvd;
+};
+
+struct mcs_alloc_rsrc_rsp {
+	struct mbox_msghdr hdr;
+	u8 flow_ids[128];	/* Index of reserved entries */
+	u8 secy_ids[128];
+	u8 sc_ids[128];
+	u8 sa_ids[256];
+	u8 rsrc_type;
+	u8 rsrc_cnt;		/* No of entries reserved */
+	u8 mcs_id;
+	u8 dir;
+	u8 all;
+	u8 rsvd[256];		/* reserved fields for future expansion */
+};
+
+struct mcs_free_rsrc_req {
+	struct mbox_msghdr hdr;
+	u8 rsrc_id;		/* Index of the entry to be freed */
+	u8 rsrc_type;
+	u8 mcs_id;
+	u8 dir;
+	u8 all;			/* Free all the cam resources */
+	u64 rsvd;
+};
+
+struct mcs_flowid_entry_write_req {
+	struct mbox_msghdr hdr;
+	u64 data[4];
+	u64 mask[4];
+	u64 sci;	/* CNF10K-B for tx_secy_mem_map */
+	u8 flow_id;
+	u8 secy_id;	/* secyid for which flowid is mapped */
+	u8 sc_id;	/* Valid if dir = MCS_TX, SC_CAM id mapped to flowid */
+	u8 ena;		/* Enable tcam entry */
+	u8 ctrl_pkt;
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+struct mcs_secy_plcy_write_req {
+	struct mbox_msghdr hdr;
+	u64 plcy;
+	u8 secy_id;
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+/* RX SC_CAM mapping */
+struct mcs_rx_sc_cam_write_req {
+	struct mbox_msghdr hdr;
+	u64 sci;	/* SCI */
+	u64 secy_id;	/* secy index mapped to SC */
+	u8 sc_id;	/* SC CAM entry index */
+	u8 mcs_id;
+	u64 rsvd;
+};
+
+struct mcs_sa_plcy_write_req {
+	struct mbox_msghdr hdr;
+	u64 plcy[2][9];		/* Support 2 SA policy */
+	u8 sa_index[2];
+	u8 sa_cnt;
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+struct mcs_tx_sc_sa_map {
+	struct mbox_msghdr hdr;
+	u8 sa_index0;
+	u8 sa_index1;
+	u8 rekey_ena;
+	u8 sa_index0_vld;
+	u8 sa_index1_vld;
+	u8 tx_sa_active;
+	u64 sectag_sci;
+	u8 sc_id;	/* used as index for SA_MEM_MAP */
+	u8 mcs_id;
+	u64 rsvd;
+};
+
+struct mcs_rx_sc_sa_map {
+	struct mbox_msghdr hdr;
+	u8 sa_index;
+	u8 sa_in_use;
+	u8 sc_id;
+	u8 an;		/* value range 0-3, sc_id + an used as index SA_MEM_MAP */
+	u8 mcs_id;
+	u64 rsvd;
+};
+
+struct mcs_flowid_ena_dis_entry {
+	struct mbox_msghdr hdr;
+	u8 flow_id;
+	u8 ena;
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+struct mcs_pn_table_write_req {
+	struct mbox_msghdr hdr;
+	u64 next_pn;
+	u8 pn_id;
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+struct mcs_hw_info {
+	struct mbox_msghdr hdr;
+	u8 num_mcs_blks;	/* Number of MCS blocks */
+	u8 tcam_entries;	/* RX/TX Tcam entries per mcs block */
+	u8 secy_entries;	/* RX/TX SECY entries per mcs block */
+	u8 sc_entries;		/* RX/TX SC CAM entries per mcs block */
+	u8 sa_entries;		/* PN table entries = SA entries */
+	u64 rsvd[16];
+};
+
+struct mcs_set_active_lmac {
+	struct mbox_msghdr hdr;
+	u32 lmac_bmap;	/* bitmap of active lmac per mcs block */
+	u8 mcs_id;
+	u16 chan_base; /* MCS channel base */
+	u64 rsvd;
+};
+
+struct mcs_set_lmac_mode {
+	struct mbox_msghdr hdr;
+	u8 mode;	/* 1:Bypass 0:Operational */
+	u8 lmac_id;
+	u8 mcs_id;
+	u64 rsvd;
+};
+
+struct mcs_port_reset_req {
+	struct mbox_msghdr hdr;
+	u8 reset;
+	u8 mcs_id;
+	u8 port_id;
+	u64 rsvd;
+};
+
+struct mcs_port_cfg_set_req {
+	struct mbox_msghdr hdr;
+	u8 cstm_tag_rel_mode_sel;
+	u8 custom_hdr_enb;
+	u8 fifo_skid;
+	u8 port_mode;
+	u8 port_id;
+	u8 mcs_id;
+	u64 rsvd;
+};
+
+struct mcs_port_cfg_get_req {
+	struct mbox_msghdr hdr;
+	u8 port_id;
+	u8 mcs_id;
+	u64 rsvd;
+};
+
+struct mcs_port_cfg_get_rsp {
+	struct mbox_msghdr hdr;
+	u8 cstm_tag_rel_mode_sel;
+	u8 custom_hdr_enb;
+	u8 fifo_skid;
+	u8 port_mode;
+	u8 port_id;
+	u8 mcs_id;
+	u64 rsvd;
+};
+
+struct mcs_custom_tag_cfg_get_req {
+	struct mbox_msghdr hdr;
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+struct mcs_custom_tag_cfg_get_rsp {
+	struct mbox_msghdr hdr;
+	u16 cstm_etype[8];
+	u8 cstm_indx[8];
+	u8 cstm_etype_en;
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+/* MCS mailbox error codes
+ * Range 1201 - 1300.
+ */
+enum mcs_af_status {
+	MCS_AF_ERR_INVALID_MCSID        = -1201,
+	MCS_AF_ERR_NOT_MAPPED           = -1202,
+};
+
+struct mcs_set_pn_threshold {
+	struct mbox_msghdr hdr;
+	u64 threshold;
+	u8 xpn; /* '1' for setting xpn threshold */
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+enum mcs_ctrl_pkt_rulew_type {
+	MCS_CTRL_PKT_RULE_TYPE_ETH,
+	MCS_CTRL_PKT_RULE_TYPE_DA,
+	MCS_CTRL_PKT_RULE_TYPE_RANGE,
+	MCS_CTRL_PKT_RULE_TYPE_COMBO,
+	MCS_CTRL_PKT_RULE_TYPE_MAC,
+};
+
+struct mcs_alloc_ctrl_pkt_rule_req {
+	struct mbox_msghdr hdr;
+	u8 rule_type;
+	u8 mcs_id;	/* MCS block ID	*/
+	u8 dir;		/* Macsec ingress or egress side */
+	u64 rsvd;
+};
+
+struct mcs_alloc_ctrl_pkt_rule_rsp {
+	struct mbox_msghdr hdr;
+	u8 rule_idx;
+	u8 rule_type;
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+struct mcs_free_ctrl_pkt_rule_req {
+	struct mbox_msghdr hdr;
+	u8 rule_idx;
+	u8 rule_type;
+	u8 mcs_id;
+	u8 dir;
+	u8 all;
+	u64 rsvd;
+};
+
+struct mcs_ctrl_pkt_rule_write_req {
+	struct mbox_msghdr hdr;
+	u64 data0;
+	u64 data1;
+	u64 data2;
+	u8 rule_idx;
+	u8 rule_type;
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+struct mcs_stats_req {
+	struct mbox_msghdr hdr;
+	u8 id;
+	u8 mcs_id;
+	u8 dir;
+	u64 rsvd;
+};
+
+struct mcs_flowid_stats {
+	struct mbox_msghdr hdr;
+	u64 tcam_hit_cnt;
+	u64 rsvd;
+};
+
+struct mcs_secy_stats {
+	struct mbox_msghdr hdr;
+	u64 ctl_pkt_bcast_cnt;
+	u64 ctl_pkt_mcast_cnt;
+	u64 ctl_pkt_ucast_cnt;
+	u64 ctl_octet_cnt;
+	u64 unctl_pkt_bcast_cnt;
+	u64 unctl_pkt_mcast_cnt;
+	u64 unctl_pkt_ucast_cnt;
+	u64 unctl_octet_cnt;
+	/* Valid only for RX */
+	u64 octet_decrypted_cnt;
+	u64 octet_validated_cnt;
+	u64 pkt_port_disabled_cnt;
+	u64 pkt_badtag_cnt;
+	u64 pkt_nosa_cnt;
+	u64 pkt_nosaerror_cnt;
+	u64 pkt_tagged_ctl_cnt;
+	u64 pkt_untaged_cnt;
+	u64 pkt_ctl_cnt;	/* CN10K-B */
+	u64 pkt_notag_cnt;	/* CNF10K-B */
+	/* Valid only for TX */
+	u64 octet_encrypted_cnt;
+	u64 octet_protected_cnt;
+	u64 pkt_noactivesa_cnt;
+	u64 pkt_toolong_cnt;
+	u64 pkt_untagged_cnt;
+	u64 rsvd[4];
+};
+
+struct mcs_port_stats {
+	struct mbox_msghdr hdr;
+	u64 tcam_miss_cnt;
+	u64 parser_err_cnt;
+	u64 preempt_err_cnt;  /* CNF10K-B */
+	u64 sectag_insert_err_cnt;
+	u64 rsvd[4];
+};
+
+/* Only for CN10K-B */
+struct mcs_sa_stats {
+	struct mbox_msghdr hdr;
+	/* RX */
+	u64 pkt_invalid_cnt;
+	u64 pkt_nosaerror_cnt;
+	u64 pkt_notvalid_cnt;
+	u64 pkt_ok_cnt;
+	u64 pkt_nosa_cnt;
+	/* TX */
+	u64 pkt_encrypt_cnt;
+	u64 pkt_protected_cnt;
+	u64 rsvd[4];
+};
+
+struct mcs_sc_stats {
+	struct mbox_msghdr hdr;
+	/* RX */
+	u64 hit_cnt;
+	u64 pkt_invalid_cnt;
+	u64 pkt_late_cnt;
+	u64 pkt_notvalid_cnt;
+	u64 pkt_unchecked_cnt;
+	u64 pkt_delay_cnt;	/* CNF10K-B */
+	u64 pkt_ok_cnt;		/* CNF10K-B */
+	u64 octet_decrypt_cnt;	/* CN10K-B */
+	u64 octet_validate_cnt;	/* CN10K-B */
+	/* TX */
+	u64 pkt_encrypt_cnt;
+	u64 pkt_protected_cnt;
+	u64 octet_encrypt_cnt;		/* CN10K-B */
+	u64 octet_protected_cnt;	/* CN10K-B */
+	u64 rsvd[4];
+};
+
+struct mcs_clear_stats {
+	struct mbox_msghdr hdr;
+#define MCS_FLOWID_STATS	0
+#define MCS_SECY_STATS		1
+#define MCS_SC_STATS		2
+#define MCS_SA_STATS		3
+#define MCS_PORT_STATS		4
+	u8 type;	/* FLOWID, SECY, SC, SA, PORT */
+	u8 id;		/* type = PORT, If id = FF(invalid) port no is derived from pcifunc */
+	u8 mcs_id;
+	u8 dir;
+	u8 all;		/* All resources stats mapped to PF are cleared */
+};
+
+struct mcs_intr_cfg {
+	struct mbox_msghdr hdr;
+#define MCS_CPM_RX_SECTAG_V_EQ1_INT		BIT_ULL(0)
+#define MCS_CPM_RX_SECTAG_E_EQ0_C_EQ1_INT	BIT_ULL(1)
+#define MCS_CPM_RX_SECTAG_SL_GTE48_INT		BIT_ULL(2)
+#define MCS_CPM_RX_SECTAG_ES_EQ1_SC_EQ1_INT	BIT_ULL(3)
+#define MCS_CPM_RX_SECTAG_SC_EQ1_SCB_EQ1_INT	BIT_ULL(4)
+#define MCS_CPM_RX_PACKET_XPN_EQ0_INT		BIT_ULL(5)
+#define MCS_CPM_RX_PN_THRESH_REACHED_INT	BIT_ULL(6)
+#define MCS_CPM_TX_PACKET_XPN_EQ0_INT		BIT_ULL(7)
+#define MCS_CPM_TX_PN_THRESH_REACHED_INT	BIT_ULL(8)
+#define MCS_CPM_TX_SA_NOT_VALID_INT		BIT_ULL(9)
+#define MCS_BBE_RX_DFIFO_OVERFLOW_INT		BIT_ULL(10)
+#define MCS_BBE_RX_PLFIFO_OVERFLOW_INT		BIT_ULL(11)
+#define MCS_BBE_TX_DFIFO_OVERFLOW_INT		BIT_ULL(12)
+#define MCS_BBE_TX_PLFIFO_OVERFLOW_INT		BIT_ULL(13)
+#define MCS_PAB_RX_CHAN_OVERFLOW_INT		BIT_ULL(14)
+#define MCS_PAB_TX_CHAN_OVERFLOW_INT		BIT_ULL(15)
+	u64 intr_mask;		/* Interrupt enable mask */
+	u8 mcs_id;
+	u8 lmac_id;
+	u64 rsvd;
+};
+
+struct mcs_intr_info {
+	struct mbox_msghdr hdr;
+	u64 intr_mask;
+	int sa_id;
+	u8 mcs_id;
+	u8 lmac_id;
+	u64 rsvd;
+};
+
 #endif /* MBOX_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c
new file mode 100644
index 000000000000..5ba618aed6ad
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.c
@@ -0,0 +1,1601 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell MCS driver
+ *
+ * Copyright (C) 2022 Marvell.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include "mcs.h"
+#include "mcs_reg.h"
+
+#define DRV_NAME	"Marvell MCS Driver"
+
+#define PCI_CFG_REG_BAR_NUM	0
+
+static const struct pci_device_id mcs_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CN10K_MCS) },
+	{ 0, }  /* end of table */
+};
+
+static LIST_HEAD(mcs_list);
+
+void mcs_get_tx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id)
+{
+	u64 reg;
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLBCPKTSX(id);
+	stats->ctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLMCPKTSX(id);
+	stats->ctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLOCTETSX(id);
+	stats->ctl_octet_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLUCPKTSX(id);
+	stats->ctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLBCPKTSX(id);
+	stats->unctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLMCPKTSX(id);
+	stats->unctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLOCTETSX(id);
+	stats->unctl_octet_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLUCPKTSX(id);
+	stats->unctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYENCRYPTEDX(id);
+	stats->octet_encrypted_cnt =  mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYPROTECTEDX(id);
+	stats->octet_protected_cnt =  mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYNOACTIVESAX(id);
+	stats->pkt_noactivesa_cnt =  mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYTOOLONGX(id);
+	stats->pkt_toolong_cnt =  mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYUNTAGGEDX(id);
+	stats->pkt_untagged_cnt =  mcs_reg_read(mcs, reg);
+}
+
+void mcs_get_rx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id)
+{
+	u64 reg;
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLBCPKTSX(id);
+	stats->ctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLMCPKTSX(id);
+	stats->ctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLOCTETSX(id);
+	stats->ctl_octet_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_IFINCTLUCPKTSX(id);
+	stats->ctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLBCPKTSX(id);
+	stats->unctl_pkt_bcast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLMCPKTSX(id);
+	stats->unctl_pkt_mcast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLOCTETSX(id);
+	stats->unctl_octet_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLUCPKTSX(id);
+	stats->unctl_pkt_ucast_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYDECRYPTEDX(id);
+	stats->octet_decrypted_cnt =  mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYVALIDATEX(id);
+	stats->octet_validated_cnt =  mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSCTRLPORTDISABLEDX(id);
+	stats->pkt_port_disabled_cnt =  mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYBADTAGX(id);
+	stats->pkt_badtag_cnt =  mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAX(id);
+	stats->pkt_nosa_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAERRORX(id);
+	stats->pkt_nosaerror_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYTAGGEDCTLX(id);
+	stats->pkt_tagged_ctl_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDORNOTAGX(id);
+	stats->pkt_untaged_cnt = mcs_reg_read(mcs, reg);
+
+	reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYCTLX(id);
+	stats->pkt_ctl_cnt = mcs_reg_read(mcs, reg);
+
+	if (mcs->hw->mcs_blks > 1) {
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOTAGX(id);
+		stats->pkt_notag_cnt = mcs_reg_read(mcs, reg);
+	}
+}
+
+void mcs_get_flowid_stats(struct mcs *mcs, struct mcs_flowid_stats *stats,
+			  int id, int dir)
+{
+	u64 reg;
+
+	if (dir == MCS_RX)
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMHITX(id);
+	else
+		reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMHITX(id);
+
+	stats->tcam_hit_cnt = mcs_reg_read(mcs, reg);
+}
+
+void mcs_get_port_stats(struct mcs *mcs, struct mcs_port_stats *stats,
+			int id, int dir)
+{
+	u64 reg;
+
+	if (dir == MCS_RX) {
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMMISSX(id);
+		stats->tcam_miss_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSPARSEERRX(id);
+		stats->parser_err_cnt = mcs_reg_read(mcs, reg);
+		if (mcs->hw->mcs_blks > 1) {
+			reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSEARLYPREEMPTERRX(id);
+			stats->preempt_err_cnt = mcs_reg_read(mcs, reg);
+		}
+	} else {
+		reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMMISSX(id);
+		stats->tcam_miss_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSPARSEERRX(id);
+		stats->parser_err_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECTAGINSERTIONERRX(id);
+		stats->sectag_insert_err_cnt = mcs_reg_read(mcs, reg);
+	}
+}
+
+void mcs_get_sa_stats(struct mcs *mcs, struct mcs_sa_stats *stats, int id, int dir)
+{
+	u64 reg;
+
+	if (dir == MCS_RX) {
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAINVALIDX(id);
+		stats->pkt_invalid_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTUSINGSAERRORX(id);
+		stats->pkt_nosaerror_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTVALIDX(id);
+		stats->pkt_notvalid_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAOKX(id);
+		stats->pkt_ok_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSAUNUSEDSAX(id);
+		stats->pkt_nosa_cnt = mcs_reg_read(mcs, reg);
+	} else {
+		reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAENCRYPTEDX(id);
+		stats->pkt_encrypt_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAPROTECTEDX(id);
+		stats->pkt_protected_cnt = mcs_reg_read(mcs, reg);
+	}
+}
+
+void mcs_get_sc_stats(struct mcs *mcs, struct mcs_sc_stats *stats,
+		      int id, int dir)
+{
+	u64 reg;
+
+	if (dir == MCS_RX) {
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCCAMHITX(id);
+		stats->hit_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCINVALIDX(id);
+		stats->pkt_invalid_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCLATEORDELAYEDX(id);
+		stats->pkt_late_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCNOTVALIDX(id);
+		stats->pkt_notvalid_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCUNCHECKEDOROKX(id);
+		stats->pkt_unchecked_cnt = mcs_reg_read(mcs, reg);
+
+		if (mcs->hw->mcs_blks > 1) {
+			reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCDELAYEDX(id);
+			stats->pkt_delay_cnt = mcs_reg_read(mcs, reg);
+
+			reg = MCSX_CSE_RX_MEM_SLAVE_INPKTSSCOKX(id);
+			stats->pkt_ok_cnt = mcs_reg_read(mcs, reg);
+		}
+		if (mcs->hw->mcs_blks == 1) {
+			reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCDECRYPTEDX(id);
+			stats->octet_decrypt_cnt = mcs_reg_read(mcs, reg);
+
+			reg = MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCVALIDATEX(id);
+			stats->octet_validate_cnt = mcs_reg_read(mcs, reg);
+		}
+	} else {
+		reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCENCRYPTEDX(id);
+		stats->pkt_encrypt_cnt = mcs_reg_read(mcs, reg);
+
+		reg = MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCPROTECTEDX(id);
+		stats->pkt_protected_cnt = mcs_reg_read(mcs, reg);
+
+		if (mcs->hw->mcs_blks == 1) {
+			reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCENCRYPTEDX(id);
+			stats->octet_encrypt_cnt = mcs_reg_read(mcs, reg);
+
+			reg = MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCPROTECTEDX(id);
+			stats->octet_protected_cnt = mcs_reg_read(mcs, reg);
+		}
+	}
+}
+
+void mcs_clear_stats(struct mcs *mcs, u8 type, u8 id, int dir)
+{
+	struct mcs_flowid_stats flowid_st;
+	struct mcs_port_stats port_st;
+	struct mcs_secy_stats secy_st;
+	struct mcs_sc_stats sc_st;
+	struct mcs_sa_stats sa_st;
+	u64 reg;
+
+	if (dir == MCS_RX)
+		reg = MCSX_CSE_RX_SLAVE_CTRL;
+	else
+		reg = MCSX_CSE_TX_SLAVE_CTRL;
+
+	mcs_reg_write(mcs, reg, BIT_ULL(0));
+
+	switch (type) {
+	case MCS_FLOWID_STATS:
+		mcs_get_flowid_stats(mcs, &flowid_st, id, dir);
+		break;
+	case MCS_SECY_STATS:
+		if (dir == MCS_RX)
+			mcs_get_rx_secy_stats(mcs, &secy_st, id);
+		else
+			mcs_get_tx_secy_stats(mcs, &secy_st, id);
+		break;
+	case MCS_SC_STATS:
+		mcs_get_sc_stats(mcs, &sc_st, id, dir);
+		break;
+	case MCS_SA_STATS:
+		mcs_get_sa_stats(mcs, &sa_st, id, dir);
+		break;
+	case MCS_PORT_STATS:
+		mcs_get_port_stats(mcs, &port_st, id, dir);
+		break;
+	}
+
+	mcs_reg_write(mcs, reg, 0x0);
+}
+
+int mcs_clear_all_stats(struct mcs *mcs, u16 pcifunc, int dir)
+{
+	struct mcs_rsrc_map *map;
+	int id;
+
+	if (dir == MCS_RX)
+		map = &mcs->rx;
+	else
+		map = &mcs->tx;
+
+	/* Clear FLOWID stats */
+	for (id = 0; id < map->flow_ids.max; id++) {
+		if (map->flowid2pf_map[id] != pcifunc)
+			continue;
+		mcs_clear_stats(mcs, MCS_FLOWID_STATS, id, dir);
+	}
+
+	/* Clear SECY stats */
+	for (id = 0; id < map->secy.max; id++) {
+		if (map->secy2pf_map[id] != pcifunc)
+			continue;
+		mcs_clear_stats(mcs, MCS_SECY_STATS, id, dir);
+	}
+
+	/* Clear SC stats */
+	for (id = 0; id < map->secy.max; id++) {
+		if (map->sc2pf_map[id] != pcifunc)
+			continue;
+		mcs_clear_stats(mcs, MCS_SC_STATS, id, dir);
+	}
+
+	/* Clear SA stats */
+	for (id = 0; id < map->sa.max; id++) {
+		if (map->sa2pf_map[id] != pcifunc)
+			continue;
+		mcs_clear_stats(mcs, MCS_SA_STATS, id, dir);
+	}
+	return 0;
+}
+
+void mcs_pn_table_write(struct mcs *mcs, u8 pn_id, u64 next_pn, u8 dir)
+{
+	u64 reg;
+
+	if (dir == MCS_RX)
+		reg = MCSX_CPM_RX_SLAVE_SA_PN_TABLE_MEMX(pn_id);
+	else
+		reg = MCSX_CPM_TX_SLAVE_SA_PN_TABLE_MEMX(pn_id);
+	mcs_reg_write(mcs, reg, next_pn);
+}
+
+void cn10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map)
+{
+	u64 reg, val;
+
+	val = (map->sa_index0 & 0xFF) |
+	      (map->sa_index1 & 0xFF) << 9 |
+	      (map->rekey_ena & 0x1) << 18 |
+	      (map->sa_index0_vld & 0x1) << 19 |
+	      (map->sa_index1_vld & 0x1) << 20 |
+	      (map->tx_sa_active & 0x1) << 21 |
+	      map->sectag_sci << 22;
+	reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(map->sc_id);
+	mcs_reg_write(mcs, reg, val);
+
+	val = map->sectag_sci >> 42;
+	reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_1X(map->sc_id);
+	mcs_reg_write(mcs, reg, val);
+}
+
+void cn10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map)
+{
+	u64 val, reg;
+
+	val = (map->sa_index & 0xFF) | map->sa_in_use << 9;
+
+	reg = MCSX_CPM_RX_SLAVE_SA_MAP_MEMX((4 * map->sc_id) + map->an);
+	mcs_reg_write(mcs, reg, val);
+}
+
+void mcs_sa_plcy_write(struct mcs *mcs, u64 *plcy, int sa_id, int dir)
+{
+	int reg_id;
+	u64 reg;
+
+	if (dir == MCS_RX) {
+		for (reg_id = 0; reg_id < 8; reg_id++) {
+			reg =  MCSX_CPM_RX_SLAVE_SA_PLCY_MEMX(reg_id, sa_id);
+			mcs_reg_write(mcs, reg, plcy[reg_id]);
+		}
+	} else {
+		for (reg_id = 0; reg_id < 9; reg_id++) {
+			reg =  MCSX_CPM_TX_SLAVE_SA_PLCY_MEMX(reg_id, sa_id);
+			mcs_reg_write(mcs, reg, plcy[reg_id]);
+		}
+	}
+}
+
+void mcs_ena_dis_sc_cam_entry(struct mcs *mcs, int sc_id, int ena)
+{
+	u64 reg, val;
+
+	reg = MCSX_CPM_RX_SLAVE_SC_CAM_ENA(0);
+	if (sc_id > 63)
+		reg = MCSX_CPM_RX_SLAVE_SC_CAM_ENA(1);
+
+	if (ena)
+		val = mcs_reg_read(mcs, reg) | BIT_ULL(sc_id);
+	else
+		val = mcs_reg_read(mcs, reg) & ~BIT_ULL(sc_id);
+
+	mcs_reg_write(mcs, reg, val);
+}
+
+void mcs_rx_sc_cam_write(struct mcs *mcs, u64 sci, u64 secy, int sc_id)
+{
+	mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SC_CAMX(0, sc_id), sci);
+	mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SC_CAMX(1, sc_id), secy);
+	/* Enable SC CAM */
+	mcs_ena_dis_sc_cam_entry(mcs, sc_id, true);
+}
+
+void mcs_secy_plcy_write(struct mcs *mcs, u64 plcy, int secy_id, int dir)
+{
+	u64 reg;
+
+	if (dir == MCS_RX)
+		reg = MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_0X(secy_id);
+	else
+		reg = MCSX_CPM_TX_SLAVE_SECY_PLCY_MEMX(secy_id);
+
+	mcs_reg_write(mcs, reg, plcy);
+
+	if (mcs->hw->mcs_blks == 1 && dir == MCS_RX)
+		mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_1X(secy_id), 0x0ull);
+}
+
+void cn10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir)
+{
+	u64 reg, val;
+
+	val = (map->secy & 0x7F) | (map->ctrl_pkt & 0x1) << 8;
+	if (dir == MCS_RX) {
+		reg = MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(map->flow_id);
+	} else {
+		val |= (map->sc & 0x7F) << 9;
+		reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(map->flow_id);
+	}
+
+	mcs_reg_write(mcs, reg, val);
+}
+
+void mcs_ena_dis_flowid_entry(struct mcs *mcs, int flow_id, int dir, int ena)
+{
+	u64 reg, val;
+
+	if (dir == MCS_RX) {
+		reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_0;
+		if (flow_id > 63)
+			reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_1;
+	} else {
+		reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_0;
+		if (flow_id > 63)
+			reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_1;
+	}
+
+	/* Enable/Disable the tcam entry */
+	if (ena)
+		val = mcs_reg_read(mcs, reg) | BIT_ULL(flow_id);
+	else
+		val = mcs_reg_read(mcs, reg) & ~BIT_ULL(flow_id);
+
+	mcs_reg_write(mcs, reg, val);
+}
+
+void mcs_flowid_entry_write(struct mcs *mcs, u64 *data, u64 *mask, int flow_id, int dir)
+{
+	int reg_id;
+	u64 reg;
+
+	if (dir == MCS_RX) {
+		for (reg_id = 0; reg_id < 4; reg_id++) {
+			reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_DATAX(reg_id, flow_id);
+			mcs_reg_write(mcs, reg, data[reg_id]);
+			reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id);
+			mcs_reg_write(mcs, reg, mask[reg_id]);
+		}
+	} else {
+		for (reg_id = 0; reg_id < 4; reg_id++) {
+			reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_DATAX(reg_id, flow_id);
+			mcs_reg_write(mcs, reg, data[reg_id]);
+			reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id);
+			mcs_reg_write(mcs, reg, mask[reg_id]);
+		}
+	}
+}
+
+int mcs_install_flowid_bypass_entry(struct mcs *mcs)
+{
+	int flow_id, secy_id, reg_id;
+	struct secy_mem_map map;
+	u64 reg, plcy = 0;
+
+	/* Flow entry */
+	flow_id = mcs->hw->tcam_entries - MCS_RSRC_RSVD_CNT;
+	for (reg_id = 0; reg_id < 4; reg_id++) {
+		reg = MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id);
+		mcs_reg_write(mcs, reg, GENMASK_ULL(63, 0));
+	}
+	for (reg_id = 0; reg_id < 4; reg_id++) {
+		reg = MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(reg_id, flow_id);
+		mcs_reg_write(mcs, reg, GENMASK_ULL(63, 0));
+	}
+	/* secy */
+	secy_id = mcs->hw->secy_entries - MCS_RSRC_RSVD_CNT;
+
+	/* Set validate frames to NULL and enable control port */
+	plcy = 0x7ull;
+	if (mcs->hw->mcs_blks > 1)
+		plcy = BIT_ULL(0) | 0x3ull << 4;
+	mcs_secy_plcy_write(mcs, plcy, secy_id, MCS_RX);
+
+	/* Enable control port and set mtu to max */
+	plcy = BIT_ULL(0) | GENMASK_ULL(43, 28);
+	if (mcs->hw->mcs_blks > 1)
+		plcy = BIT_ULL(0) | GENMASK_ULL(63, 48);
+	mcs_secy_plcy_write(mcs, plcy, secy_id, MCS_TX);
+
+	/* Map flowid to secy */
+	map.secy = secy_id;
+	map.ctrl_pkt = 0;
+	map.flow_id = flow_id;
+	mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, MCS_RX);
+	map.sc = secy_id;
+	mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, MCS_TX);
+
+	/* Enable Flowid entry */
+	mcs_ena_dis_flowid_entry(mcs, flow_id, MCS_RX, true);
+	mcs_ena_dis_flowid_entry(mcs, flow_id, MCS_TX, true);
+	return 0;
+}
+
+void mcs_clear_secy_plcy(struct mcs *mcs, int secy_id, int dir)
+{
+	struct mcs_rsrc_map *map;
+	int flow_id;
+
+	if (dir == MCS_RX)
+		map = &mcs->rx;
+	else
+		map = &mcs->tx;
+
+	/* Clear secy memory to zero */
+	mcs_secy_plcy_write(mcs, 0, secy_id, dir);
+
+	/* Disable the tcam entry using this secy */
+	for (flow_id = 0; flow_id < map->flow_ids.max; flow_id++) {
+		if (map->flowid2secy_map[flow_id] != secy_id)
+			continue;
+		mcs_ena_dis_flowid_entry(mcs, flow_id, dir, false);
+	}
+}
+
+int mcs_alloc_ctrlpktrule(struct rsrc_bmap *rsrc, u16 *pf_map, u16 offset, u16 pcifunc)
+{
+	int rsrc_id;
+
+	if (!rsrc->bmap)
+		return -EINVAL;
+
+	rsrc_id = bitmap_find_next_zero_area(rsrc->bmap, rsrc->max, offset, 1, 0);
+	if (rsrc_id >= rsrc->max)
+		return -ENOSPC;
+
+	bitmap_set(rsrc->bmap, rsrc_id, 1);
+	pf_map[rsrc_id] = pcifunc;
+
+	return rsrc_id;
+}
+
+int mcs_free_ctrlpktrule(struct mcs *mcs, struct mcs_free_ctrl_pkt_rule_req *req)
+{
+	u16 pcifunc = req->hdr.pcifunc;
+	struct mcs_rsrc_map *map;
+	u64 dis, reg;
+	int id, rc;
+
+	reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ENABLE : MCSX_PEX_TX_SLAVE_RULE_ENABLE;
+	map = (req->dir == MCS_RX) ? &mcs->rx : &mcs->tx;
+
+	if (req->all) {
+		for (id = 0; id < map->ctrlpktrule.max; id++) {
+			if (map->ctrlpktrule2pf_map[id] != pcifunc)
+				continue;
+			mcs_free_rsrc(&map->ctrlpktrule, map->ctrlpktrule2pf_map, id, pcifunc);
+			dis = mcs_reg_read(mcs, reg);
+			dis &= ~BIT_ULL(id);
+			mcs_reg_write(mcs, reg, dis);
+		}
+		return 0;
+	}
+
+	rc = mcs_free_rsrc(&map->ctrlpktrule, map->ctrlpktrule2pf_map, req->rule_idx, pcifunc);
+	dis = mcs_reg_read(mcs, reg);
+	dis &= ~BIT_ULL(req->rule_idx);
+	mcs_reg_write(mcs, reg, dis);
+
+	return rc;
+}
+
+int mcs_ctrlpktrule_write(struct mcs *mcs, struct mcs_ctrl_pkt_rule_write_req *req)
+{
+	u64 reg, enb;
+	u64 idx;
+
+	switch (req->rule_type) {
+	case MCS_CTRL_PKT_RULE_TYPE_ETH:
+		req->data0 &= GENMASK(15, 0);
+		if (req->data0 != ETH_P_PAE)
+			return -EINVAL;
+
+		idx = req->rule_idx - MCS_CTRLPKT_ETYPE_RULE_OFFSET;
+		reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ETYPE_CFGX(idx) :
+		      MCSX_PEX_TX_SLAVE_RULE_ETYPE_CFGX(idx);
+
+		mcs_reg_write(mcs, reg, req->data0);
+		break;
+	case MCS_CTRL_PKT_RULE_TYPE_DA:
+		if (!(req->data0 & BIT_ULL(40)))
+			return -EINVAL;
+
+		idx = req->rule_idx - MCS_CTRLPKT_DA_RULE_OFFSET;
+		reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_DAX(idx) :
+		      MCSX_PEX_TX_SLAVE_RULE_DAX(idx);
+
+		mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0));
+		break;
+	case MCS_CTRL_PKT_RULE_TYPE_RANGE:
+		if (!(req->data0 & BIT_ULL(40)) || !(req->data1 & BIT_ULL(40)))
+			return -EINVAL;
+
+		idx = req->rule_idx - MCS_CTRLPKT_DA_RANGE_RULE_OFFSET;
+		if (req->dir == MCS_RX) {
+			reg = MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MINX(idx);
+			mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0));
+			reg = MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MAXX(idx);
+			mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0));
+		} else {
+			reg = MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MINX(idx);
+			mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0));
+			reg = MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MAXX(idx);
+			mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0));
+		}
+		break;
+	case MCS_CTRL_PKT_RULE_TYPE_COMBO:
+		req->data2 &= GENMASK(15, 0);
+		if (req->data2 != ETH_P_PAE || !(req->data0 & BIT_ULL(40)) ||
+		    !(req->data1 & BIT_ULL(40)))
+			return -EINVAL;
+
+		idx = req->rule_idx - MCS_CTRLPKT_COMBO_RULE_OFFSET;
+		if (req->dir == MCS_RX) {
+			reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_MINX(idx);
+			mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0));
+			reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_MAXX(idx);
+			mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0));
+			reg = MCSX_PEX_RX_SLAVE_RULE_COMBO_ETX(idx);
+			mcs_reg_write(mcs, reg, req->data2);
+		} else {
+			reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_MINX(idx);
+			mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0));
+			reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_MAXX(idx);
+			mcs_reg_write(mcs, reg, req->data1 & GENMASK_ULL(47, 0));
+			reg = MCSX_PEX_TX_SLAVE_RULE_COMBO_ETX(idx);
+			mcs_reg_write(mcs, reg, req->data2);
+		}
+		break;
+	case MCS_CTRL_PKT_RULE_TYPE_MAC:
+		if (!(req->data0 & BIT_ULL(40)))
+			return -EINVAL;
+
+		idx = req->rule_idx - MCS_CTRLPKT_MAC_EN_RULE_OFFSET;
+		reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_MAC :
+		      MCSX_PEX_TX_SLAVE_RULE_MAC;
+
+		mcs_reg_write(mcs, reg, req->data0 & GENMASK_ULL(47, 0));
+		break;
+	}
+
+	reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_RULE_ENABLE : MCSX_PEX_TX_SLAVE_RULE_ENABLE;
+
+	enb = mcs_reg_read(mcs, reg);
+	enb |= BIT_ULL(req->rule_idx);
+	mcs_reg_write(mcs, reg, enb);
+
+	return 0;
+}
+
+int mcs_free_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, int rsrc_id, u16 pcifunc)
+{
+	/* Check if the rsrc_id is mapped to PF/VF */
+	if (pf_map[rsrc_id] != pcifunc)
+		return -EINVAL;
+
+	rvu_free_rsrc(rsrc, rsrc_id);
+	pf_map[rsrc_id] = 0;
+	return 0;
+}
+
+/* Free all the cam resources mapped to pf */
+int mcs_free_all_rsrc(struct mcs *mcs, int dir, u16 pcifunc)
+{
+	struct mcs_rsrc_map *map;
+	int id;
+
+	if (dir == MCS_RX)
+		map = &mcs->rx;
+	else
+		map = &mcs->tx;
+
+	/* free tcam entries */
+	for (id = 0; id < map->flow_ids.max; id++) {
+		if (map->flowid2pf_map[id] != pcifunc)
+			continue;
+		mcs_free_rsrc(&map->flow_ids, map->flowid2pf_map,
+			      id, pcifunc);
+		mcs_ena_dis_flowid_entry(mcs, id, dir, false);
+	}
+
+	/* free secy entries */
+	for (id = 0; id < map->secy.max; id++) {
+		if (map->secy2pf_map[id] != pcifunc)
+			continue;
+		mcs_free_rsrc(&map->secy, map->secy2pf_map,
+			      id, pcifunc);
+		mcs_clear_secy_plcy(mcs, id, dir);
+	}
+
+	/* free sc entries */
+	for (id = 0; id < map->secy.max; id++) {
+		if (map->sc2pf_map[id] != pcifunc)
+			continue;
+		mcs_free_rsrc(&map->sc, map->sc2pf_map, id, pcifunc);
+
+		/* Disable SC CAM only on RX side */
+		if (dir == MCS_RX)
+			mcs_ena_dis_sc_cam_entry(mcs, id, false);
+	}
+
+	/* free sa entries */
+	for (id = 0; id < map->sa.max; id++) {
+		if (map->sa2pf_map[id] != pcifunc)
+			continue;
+		mcs_free_rsrc(&map->sa, map->sa2pf_map, id, pcifunc);
+	}
+	return 0;
+}
+
+int mcs_alloc_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, u16 pcifunc)
+{
+	int rsrc_id;
+
+	rsrc_id = rvu_alloc_rsrc(rsrc);
+	if (rsrc_id < 0)
+		return -ENOMEM;
+	pf_map[rsrc_id] = pcifunc;
+	return rsrc_id;
+}
+
+int mcs_alloc_all_rsrc(struct mcs *mcs, u8 *flow_id, u8 *secy_id,
+		       u8 *sc_id, u8 *sa1_id, u8 *sa2_id, u16 pcifunc, int dir)
+{
+	struct mcs_rsrc_map *map;
+	int id;
+
+	if (dir == MCS_RX)
+		map = &mcs->rx;
+	else
+		map = &mcs->tx;
+
+	id = mcs_alloc_rsrc(&map->flow_ids, map->flowid2pf_map, pcifunc);
+	if (id < 0)
+		return -ENOMEM;
+	*flow_id = id;
+
+	id = mcs_alloc_rsrc(&map->secy, map->secy2pf_map, pcifunc);
+	if (id < 0)
+		return -ENOMEM;
+	*secy_id = id;
+
+	id = mcs_alloc_rsrc(&map->sc, map->sc2pf_map, pcifunc);
+	if (id < 0)
+		return -ENOMEM;
+	*sc_id = id;
+
+	id =  mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc);
+	if (id < 0)
+		return -ENOMEM;
+	*sa1_id = id;
+
+	id =  mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc);
+	if (id < 0)
+		return -ENOMEM;
+	*sa2_id = id;
+
+	return 0;
+}
+
+static void cn10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs)
+{
+	struct mcs_intr_event event = { 0 };
+	struct rsrc_bmap *sc_bmap;
+	u64 val;
+	int sc;
+
+	sc_bmap = &mcs->tx.sc;
+
+	event.mcs_id = mcs->mcs_id;
+	event.intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT;
+
+	for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) {
+		val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc));
+
+		if (mcs->tx_sa_active[sc])
+			/* SA_index1 was used and got expired */
+			event.sa_id = (val >> 9) & 0xFF;
+		else
+			/* SA_index0 was used and got expired */
+			event.sa_id = val & 0xFF;
+
+		event.pcifunc = mcs->tx.sa2pf_map[event.sa_id];
+		mcs_add_intr_wq_entry(mcs, &event);
+	}
+}
+
+static void cn10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs)
+{
+	struct mcs_intr_event event = { 0 };
+	struct rsrc_bmap *sc_bmap;
+	u64 val, status;
+	int sc;
+
+	sc_bmap = &mcs->tx.sc;
+
+	event.mcs_id = mcs->mcs_id;
+	event.intr_mask = MCS_CPM_TX_PN_THRESH_REACHED_INT;
+
+	/* TX SA interrupt is raised only if autorekey is enabled.
+	 * MCS_CPM_TX_SLAVE_SA_MAP_MEM_0X[sc].tx_sa_active bit gets toggled if
+	 * one of two SAs mapped to SC gets expired. If tx_sa_active=0 implies
+	 * SA in SA_index1 got expired else SA in SA_index0 got expired.
+	 */
+	for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) {
+		val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc));
+		/* Auto rekey is enable */
+		if (!((val >> 18) & 0x1))
+			continue;
+
+		status = (val >> 21) & 0x1;
+
+		/* Check if tx_sa_active status had changed */
+		if (status == mcs->tx_sa_active[sc])
+			continue;
+		/* SA_index0 is expired */
+		if (status)
+			event.sa_id = val & 0xFF;
+		else
+			event.sa_id = (val >> 9) & 0xFF;
+
+		event.pcifunc = mcs->tx.sa2pf_map[event.sa_id];
+		mcs_add_intr_wq_entry(mcs, &event);
+	}
+}
+
+static void mcs_rx_pn_thresh_reached_handler(struct mcs *mcs)
+{
+	struct mcs_intr_event event = { 0 };
+	int sa, reg;
+	u64 intr;
+
+	/* Check expired SAs */
+	for (reg = 0; reg < (mcs->hw->sa_entries / 64); reg++) {
+		/* Bit high in *PN_THRESH_REACHEDX implies
+		 * corresponding SAs are expired.
+		 */
+		intr = mcs_reg_read(mcs, MCSX_CPM_RX_SLAVE_PN_THRESH_REACHEDX(reg));
+		for (sa = 0; sa < 64; sa++) {
+			if (!(intr & BIT_ULL(sa)))
+				continue;
+
+			event.mcs_id = mcs->mcs_id;
+			event.intr_mask = MCS_CPM_RX_PN_THRESH_REACHED_INT;
+			event.sa_id = sa + (reg * 64);
+			event.pcifunc = mcs->rx.sa2pf_map[event.sa_id];
+			mcs_add_intr_wq_entry(mcs, &event);
+		}
+	}
+}
+
+static void mcs_rx_misc_intr_handler(struct mcs *mcs, u64 intr)
+{
+	struct mcs_intr_event event = { 0 };
+
+	event.mcs_id = mcs->mcs_id;
+	event.pcifunc = mcs->pf_map[0];
+
+	if (intr & MCS_CPM_RX_INT_SECTAG_V_EQ1)
+		event.intr_mask = MCS_CPM_RX_SECTAG_V_EQ1_INT;
+	if (intr & MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1)
+		event.intr_mask |= MCS_CPM_RX_SECTAG_E_EQ0_C_EQ1_INT;
+	if (intr & MCS_CPM_RX_INT_SL_GTE48)
+		event.intr_mask |= MCS_CPM_RX_SECTAG_SL_GTE48_INT;
+	if (intr & MCS_CPM_RX_INT_ES_EQ1_SC_EQ1)
+		event.intr_mask |= MCS_CPM_RX_SECTAG_ES_EQ1_SC_EQ1_INT;
+	if (intr & MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1)
+		event.intr_mask |= MCS_CPM_RX_SECTAG_SC_EQ1_SCB_EQ1_INT;
+	if (intr & MCS_CPM_RX_INT_PACKET_XPN_EQ0)
+		event.intr_mask |= MCS_CPM_RX_PACKET_XPN_EQ0_INT;
+
+	mcs_add_intr_wq_entry(mcs, &event);
+}
+
+static void mcs_tx_misc_intr_handler(struct mcs *mcs, u64 intr)
+{
+	struct mcs_intr_event event = { 0 };
+
+	if (!(intr & MCS_CPM_TX_INT_SA_NOT_VALID))
+		return;
+
+	event.mcs_id = mcs->mcs_id;
+	event.pcifunc = mcs->pf_map[0];
+
+	event.intr_mask = MCS_CPM_TX_SA_NOT_VALID_INT;
+
+	mcs_add_intr_wq_entry(mcs, &event);
+}
+
+static void mcs_bbe_intr_handler(struct mcs *mcs, u64 intr, enum mcs_direction dir)
+{
+	struct mcs_intr_event event = { 0 };
+	int i;
+
+	if (!(intr & MCS_BBE_INT_MASK))
+		return;
+
+	event.mcs_id = mcs->mcs_id;
+	event.pcifunc = mcs->pf_map[0];
+
+	for (i = 0; i < MCS_MAX_BBE_INT; i++) {
+		if (!(intr & BIT_ULL(i)))
+			continue;
+
+		/* Lower nibble denotes data fifo overflow interrupts and
+		 * upper nibble indicates policy fifo overflow interrupts.
+		 */
+		if (intr & 0xFULL)
+			event.intr_mask = (dir == MCS_RX) ?
+					  MCS_BBE_RX_DFIFO_OVERFLOW_INT :
+					  MCS_BBE_TX_DFIFO_OVERFLOW_INT;
+		else
+			event.intr_mask = (dir == MCS_RX) ?
+					  MCS_BBE_RX_PLFIFO_OVERFLOW_INT :
+					  MCS_BBE_RX_PLFIFO_OVERFLOW_INT;
+
+		/* Notify the lmac_id info which ran into BBE fatal error */
+		event.lmac_id = i & 0x3ULL;
+		mcs_add_intr_wq_entry(mcs, &event);
+	}
+}
+
+static void mcs_pab_intr_handler(struct mcs *mcs, u64 intr, enum mcs_direction dir)
+{
+	struct mcs_intr_event event = { 0 };
+	int i;
+
+	if (!(intr & MCS_PAB_INT_MASK))
+		return;
+
+	event.mcs_id = mcs->mcs_id;
+	event.pcifunc = mcs->pf_map[0];
+
+	for (i = 0; i < MCS_MAX_PAB_INT; i++) {
+		if (!(intr & BIT_ULL(i)))
+			continue;
+
+		event.intr_mask = (dir == MCS_RX) ? MCS_PAB_RX_CHAN_OVERFLOW_INT :
+				  MCS_PAB_TX_CHAN_OVERFLOW_INT;
+
+		/* Notify the lmac_id info which ran into PAB fatal error */
+		event.lmac_id = i;
+		mcs_add_intr_wq_entry(mcs, &event);
+	}
+}
+
+static irqreturn_t mcs_ip_intr_handler(int irq, void *mcs_irq)
+{
+	struct mcs *mcs = (struct mcs *)mcs_irq;
+	u64 intr, cpm_intr, bbe_intr, pab_intr;
+
+	/* Disable and clear the interrupt */
+	mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1C, BIT_ULL(0));
+	mcs_reg_write(mcs, MCSX_IP_INT, BIT_ULL(0));
+
+	/* Check which block has interrupt*/
+	intr = mcs_reg_read(mcs, MCSX_TOP_SLAVE_INT_SUM);
+
+	/* CPM RX */
+	if (intr & MCS_CPM_RX_INT_ENA) {
+		/* Check for PN thresh interrupt bit */
+		cpm_intr = mcs_reg_read(mcs, MCSX_CPM_RX_SLAVE_RX_INT);
+
+		if (cpm_intr & MCS_CPM_RX_INT_PN_THRESH_REACHED)
+			mcs_rx_pn_thresh_reached_handler(mcs);
+
+		if (cpm_intr & MCS_CPM_RX_INT_ALL)
+			mcs_rx_misc_intr_handler(mcs, cpm_intr);
+
+		/* Clear the interrupt */
+		mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_RX_INT, cpm_intr);
+	}
+
+	/* CPM TX */
+	if (intr & MCS_CPM_TX_INT_ENA) {
+		cpm_intr = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_TX_INT);
+
+		if (cpm_intr & MCS_CPM_TX_INT_PN_THRESH_REACHED) {
+			if (mcs->hw->mcs_blks > 1)
+				cnf10kb_mcs_tx_pn_thresh_reached_handler(mcs);
+			else
+				cn10kb_mcs_tx_pn_thresh_reached_handler(mcs);
+		}
+
+		if (cpm_intr & MCS_CPM_TX_INT_SA_NOT_VALID)
+			mcs_tx_misc_intr_handler(mcs, cpm_intr);
+
+		if (cpm_intr & MCS_CPM_TX_INT_PACKET_XPN_EQ0) {
+			if (mcs->hw->mcs_blks > 1)
+				cnf10kb_mcs_tx_pn_wrapped_handler(mcs);
+			else
+				cn10kb_mcs_tx_pn_wrapped_handler(mcs);
+		}
+		/* Clear the interrupt */
+		mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_INT, cpm_intr);
+	}
+
+	/* BBE RX */
+	if (intr & MCS_BBE_RX_INT_ENA) {
+		bbe_intr = mcs_reg_read(mcs, MCSX_BBE_RX_SLAVE_BBE_INT);
+		mcs_bbe_intr_handler(mcs, bbe_intr, MCS_RX);
+
+		/* Clear the interrupt */
+		mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT_INTR_RW, 0);
+		mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT, bbe_intr);
+	}
+
+	/* BBE TX */
+	if (intr & MCS_BBE_TX_INT_ENA) {
+		bbe_intr = mcs_reg_read(mcs, MCSX_BBE_TX_SLAVE_BBE_INT);
+		mcs_bbe_intr_handler(mcs, bbe_intr, MCS_TX);
+
+		/* Clear the interrupt */
+		mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT_INTR_RW, 0);
+		mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT, bbe_intr);
+	}
+
+	/* PAB RX */
+	if (intr & MCS_PAB_RX_INT_ENA) {
+		pab_intr = mcs_reg_read(mcs, MCSX_PAB_RX_SLAVE_PAB_INT);
+		mcs_pab_intr_handler(mcs, pab_intr, MCS_RX);
+
+		/* Clear the interrupt */
+		mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT_INTR_RW, 0);
+		mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT, pab_intr);
+	}
+
+	/* PAB TX */
+	if (intr & MCS_PAB_TX_INT_ENA) {
+		pab_intr = mcs_reg_read(mcs, MCSX_PAB_TX_SLAVE_PAB_INT);
+		mcs_pab_intr_handler(mcs, pab_intr, MCS_TX);
+
+		/* Clear the interrupt */
+		mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT_INTR_RW, 0);
+		mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT, pab_intr);
+	}
+
+	/* Enable the interrupt */
+	mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1S, BIT_ULL(0));
+
+	return IRQ_HANDLED;
+}
+
+static void *alloc_mem(struct mcs *mcs, int n)
+{
+	return devm_kcalloc(mcs->dev, n, sizeof(u16), GFP_KERNEL);
+}
+
+static int mcs_alloc_struct_mem(struct mcs *mcs, struct mcs_rsrc_map *res)
+{
+	struct hwinfo *hw = mcs->hw;
+	int err;
+
+	res->flowid2pf_map = alloc_mem(mcs, hw->tcam_entries);
+	if (!res->flowid2pf_map)
+		return -ENOMEM;
+
+	res->secy2pf_map = alloc_mem(mcs, hw->secy_entries);
+	if (!res->secy2pf_map)
+		return -ENOMEM;
+
+	res->sc2pf_map = alloc_mem(mcs, hw->sc_entries);
+	if (!res->sc2pf_map)
+		return -ENOMEM;
+
+	res->sa2pf_map = alloc_mem(mcs, hw->sa_entries);
+	if (!res->sa2pf_map)
+		return -ENOMEM;
+
+	res->flowid2secy_map = alloc_mem(mcs, hw->tcam_entries);
+	if (!res->flowid2secy_map)
+		return -ENOMEM;
+
+	res->ctrlpktrule2pf_map = alloc_mem(mcs, MCS_MAX_CTRLPKT_RULES);
+	if (!res->ctrlpktrule2pf_map)
+		return -ENOMEM;
+
+	res->flow_ids.max = hw->tcam_entries - MCS_RSRC_RSVD_CNT;
+	err = rvu_alloc_bitmap(&res->flow_ids);
+	if (err)
+		return err;
+
+	res->secy.max = hw->secy_entries - MCS_RSRC_RSVD_CNT;
+	err = rvu_alloc_bitmap(&res->secy);
+	if (err)
+		return err;
+
+	res->sc.max = hw->sc_entries;
+	err = rvu_alloc_bitmap(&res->sc);
+	if (err)
+		return err;
+
+	res->sa.max = hw->sa_entries;
+	err = rvu_alloc_bitmap(&res->sa);
+	if (err)
+		return err;
+
+	res->ctrlpktrule.max = MCS_MAX_CTRLPKT_RULES;
+	err = rvu_alloc_bitmap(&res->ctrlpktrule);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int mcs_register_interrupts(struct mcs *mcs)
+{
+	int ret = 0;
+
+	mcs->num_vec = pci_msix_vec_count(mcs->pdev);
+
+	ret = pci_alloc_irq_vectors(mcs->pdev, mcs->num_vec,
+				    mcs->num_vec, PCI_IRQ_MSIX);
+	if (ret < 0) {
+		dev_err(mcs->dev, "MCS Request for %d msix vector failed err:%d\n",
+			mcs->num_vec, ret);
+		return ret;
+	}
+
+	ret = request_irq(pci_irq_vector(mcs->pdev, MCS_INT_VEC_IP),
+			  mcs_ip_intr_handler, 0, "MCS_IP", mcs);
+	if (ret) {
+		dev_err(mcs->dev, "MCS IP irq registration failed\n");
+		goto exit;
+	}
+
+	/* MCS enable IP interrupts */
+	mcs_reg_write(mcs, MCSX_IP_INT_ENA_W1S, BIT_ULL(0));
+
+	/* Enable CPM Rx/Tx interrupts */
+	mcs_reg_write(mcs, MCSX_TOP_SLAVE_INT_SUM_ENB,
+		      MCS_CPM_RX_INT_ENA | MCS_CPM_TX_INT_ENA |
+		      MCS_BBE_RX_INT_ENA | MCS_BBE_TX_INT_ENA |
+		      MCS_PAB_RX_INT_ENA | MCS_PAB_TX_INT_ENA);
+
+	mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_INT_ENB, 0x7ULL);
+	mcs_reg_write(mcs, MCSX_CPM_RX_SLAVE_RX_INT_ENB, 0x7FULL);
+
+	mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_BBE_INT_ENB, 0xff);
+	mcs_reg_write(mcs, MCSX_BBE_TX_SLAVE_BBE_INT_ENB, 0xff);
+
+	mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PAB_INT_ENB, 0xff);
+	mcs_reg_write(mcs, MCSX_PAB_TX_SLAVE_PAB_INT_ENB, 0xff);
+
+	mcs->tx_sa_active = alloc_mem(mcs, mcs->hw->sc_entries);
+	if (!mcs->tx_sa_active)
+		goto exit;
+
+	return ret;
+exit:
+	pci_free_irq_vectors(mcs->pdev);
+	mcs->num_vec = 0;
+	return ret;
+}
+
+int mcs_get_blkcnt(void)
+{
+	struct mcs *mcs;
+	int idmax = -ENODEV;
+
+	/* Check MCS block is present in hardware */
+	if (!pci_dev_present(mcs_id_table))
+		return 0;
+
+	list_for_each_entry(mcs, &mcs_list, mcs_list)
+		if (mcs->mcs_id > idmax)
+			idmax = mcs->mcs_id;
+
+	if (idmax < 0)
+		return 0;
+
+	return idmax + 1;
+}
+
+struct mcs *mcs_get_pdata(int mcs_id)
+{
+	struct mcs *mcs_dev;
+
+	list_for_each_entry(mcs_dev, &mcs_list, mcs_list) {
+		if (mcs_dev->mcs_id == mcs_id)
+			return mcs_dev;
+	}
+	return NULL;
+}
+
+void mcs_set_port_cfg(struct mcs *mcs, struct mcs_port_cfg_set_req *req)
+{
+	u64 val = 0;
+
+	mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_PORT_CFGX(req->port_id),
+		      req->port_mode & MCS_PORT_MODE_MASK);
+
+	req->cstm_tag_rel_mode_sel &= 0x3;
+
+	if (mcs->hw->mcs_blks > 1) {
+		req->fifo_skid &= MCS_PORT_FIFO_SKID_MASK;
+		val = (u32)req->fifo_skid << 0x10;
+		val |= req->fifo_skid;
+		mcs_reg_write(mcs, MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(req->port_id), val);
+		mcs_reg_write(mcs, MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(req->port_id),
+			      req->cstm_tag_rel_mode_sel);
+		val = mcs_reg_read(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION);
+
+		if (req->custom_hdr_enb)
+			val |= BIT_ULL(req->port_id);
+		else
+			val &= ~BIT_ULL(req->port_id);
+
+		mcs_reg_write(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION, val);
+	} else {
+		val = mcs_reg_read(mcs, MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id));
+		val |= (req->cstm_tag_rel_mode_sel << 2);
+		mcs_reg_write(mcs, MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id), val);
+	}
+}
+
+void mcs_get_port_cfg(struct mcs *mcs, struct mcs_port_cfg_get_req *req,
+		      struct mcs_port_cfg_get_rsp *rsp)
+{
+	u64 reg = 0;
+
+	rsp->port_mode = mcs_reg_read(mcs, MCSX_PAB_RX_SLAVE_PORT_CFGX(req->port_id)) &
+			 MCS_PORT_MODE_MASK;
+
+	if (mcs->hw->mcs_blks > 1) {
+		reg = MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(req->port_id);
+		rsp->fifo_skid = mcs_reg_read(mcs, reg) & MCS_PORT_FIFO_SKID_MASK;
+		reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(req->port_id);
+		rsp->cstm_tag_rel_mode_sel = mcs_reg_read(mcs, reg) & 0x3;
+		if (mcs_reg_read(mcs, MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION) & BIT_ULL(req->port_id))
+			rsp->custom_hdr_enb = 1;
+	} else {
+		reg = MCSX_PEX_TX_SLAVE_PORT_CONFIG(req->port_id);
+		rsp->cstm_tag_rel_mode_sel = mcs_reg_read(mcs, reg) >> 2;
+	}
+
+	rsp->port_id = req->port_id;
+	rsp->mcs_id = req->mcs_id;
+}
+
+void mcs_get_custom_tag_cfg(struct mcs *mcs, struct mcs_custom_tag_cfg_get_req *req,
+			    struct mcs_custom_tag_cfg_get_rsp *rsp)
+{
+	u64 reg = 0, val = 0;
+	u8 idx;
+
+	for (idx = 0; idx < MCS_MAX_CUSTOM_TAGS; idx++) {
+		if (mcs->hw->mcs_blks > 1)
+			reg  = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(idx) :
+				MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(idx);
+		else
+			reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_VLAN_CFGX(idx) :
+				MCSX_PEX_TX_SLAVE_VLAN_CFGX(idx);
+
+		val = mcs_reg_read(mcs, reg);
+		if (mcs->hw->mcs_blks > 1) {
+			rsp->cstm_etype[idx] = val & GENMASK(15, 0);
+			rsp->cstm_indx[idx] = (val >> 0x16) & 0x3;
+			reg = (req->dir == MCS_RX) ? MCSX_PEX_RX_SLAVE_ETYPE_ENABLE :
+				MCSX_PEX_TX_SLAVE_ETYPE_ENABLE;
+			rsp->cstm_etype_en = mcs_reg_read(mcs, reg) & 0xFF;
+		} else {
+			rsp->cstm_etype[idx] = (val >> 0x1) & GENMASK(15, 0);
+			rsp->cstm_indx[idx] = (val >> 0x11) & 0x3;
+			rsp->cstm_etype_en |= (val & 0x1) << idx;
+		}
+	}
+
+	rsp->mcs_id = req->mcs_id;
+	rsp->dir = req->dir;
+}
+
+void mcs_reset_port(struct mcs *mcs, u8 port_id, u8 reset)
+{
+	u64 reg = MCSX_MCS_TOP_SLAVE_PORT_RESET(port_id);
+
+	mcs_reg_write(mcs, reg, reset & 0x1);
+}
+
+/* Set lmac to bypass/operational mode */
+void mcs_set_lmac_mode(struct mcs *mcs, int lmac_id, u8 mode)
+{
+	u64 reg;
+
+	reg = MCSX_MCS_TOP_SLAVE_CHANNEL_CFG(lmac_id * 2);
+	mcs_reg_write(mcs, reg, (u64)mode);
+}
+
+void mcs_pn_threshold_set(struct mcs *mcs, struct mcs_set_pn_threshold *pn)
+{
+	u64 reg;
+
+	if (pn->dir == MCS_RX)
+		reg = pn->xpn ? MCSX_CPM_RX_SLAVE_XPN_THRESHOLD : MCSX_CPM_RX_SLAVE_PN_THRESHOLD;
+	else
+		reg = pn->xpn ? MCSX_CPM_TX_SLAVE_XPN_THRESHOLD : MCSX_CPM_TX_SLAVE_PN_THRESHOLD;
+
+	mcs_reg_write(mcs, reg, pn->threshold);
+}
+
+void cn10kb_mcs_parser_cfg(struct mcs *mcs)
+{
+	u64 reg, val;
+
+	/* VLAN CTag */
+	val = BIT_ULL(0) | (0x8100ull & 0xFFFF) << 1 | BIT_ULL(17);
+	/* RX */
+	reg = MCSX_PEX_RX_SLAVE_VLAN_CFGX(0);
+	mcs_reg_write(mcs, reg, val);
+
+	/* TX */
+	reg = MCSX_PEX_TX_SLAVE_VLAN_CFGX(0);
+	mcs_reg_write(mcs, reg, val);
+
+	/* VLAN STag */
+	val = BIT_ULL(0) | (0x88a8ull & 0xFFFF) << 1 | BIT_ULL(18);
+	/* RX */
+	reg = MCSX_PEX_RX_SLAVE_VLAN_CFGX(1);
+	mcs_reg_write(mcs, reg, val);
+
+	/* TX */
+	reg = MCSX_PEX_TX_SLAVE_VLAN_CFGX(1);
+	mcs_reg_write(mcs, reg, val);
+}
+
+static void mcs_lmac_init(struct mcs *mcs, int lmac_id)
+{
+	u64 reg;
+
+	/* Port mode 25GB */
+	reg = MCSX_PAB_RX_SLAVE_PORT_CFGX(lmac_id);
+	mcs_reg_write(mcs, reg, 0);
+
+	if (mcs->hw->mcs_blks > 1) {
+		reg = MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(lmac_id);
+		mcs_reg_write(mcs, reg, 0xe000e);
+		return;
+	}
+
+	reg = MCSX_PAB_TX_SLAVE_PORT_CFGX(lmac_id);
+	mcs_reg_write(mcs, reg, 0);
+}
+
+int mcs_set_lmac_channels(int mcs_id, u16 base)
+{
+	struct mcs *mcs;
+	int lmac;
+	u64 cfg;
+
+	mcs = mcs_get_pdata(mcs_id);
+	if (!mcs)
+		return -ENODEV;
+	for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++) {
+		cfg = mcs_reg_read(mcs, MCSX_LINK_LMACX_CFG(lmac));
+		cfg &= ~(MCSX_LINK_LMAC_BASE_MASK | MCSX_LINK_LMAC_RANGE_MASK);
+		cfg |=	FIELD_PREP(MCSX_LINK_LMAC_RANGE_MASK, ilog2(16));
+		cfg |=	FIELD_PREP(MCSX_LINK_LMAC_BASE_MASK, base);
+		mcs_reg_write(mcs, MCSX_LINK_LMACX_CFG(lmac), cfg);
+		base += 16;
+	}
+	return 0;
+}
+
+static int mcs_x2p_calibration(struct mcs *mcs)
+{
+	unsigned long timeout = jiffies + usecs_to_jiffies(20000);
+	int i, err = 0;
+	u64 val;
+
+	/* set X2P calibration */
+	val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL);
+	val |= BIT_ULL(5);
+	mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val);
+
+	/* Wait for calibration to complete */
+	while (!(mcs_reg_read(mcs, MCSX_MIL_RX_GBL_STATUS) & BIT_ULL(0))) {
+		if (time_before(jiffies, timeout)) {
+			usleep_range(80, 100);
+			continue;
+		} else {
+			err = -EBUSY;
+			dev_err(mcs->dev, "MCS X2P calibration failed..ignoring\n");
+			return err;
+		}
+	}
+
+	val = mcs_reg_read(mcs, MCSX_MIL_RX_GBL_STATUS);
+	for (i = 0; i < mcs->hw->mcs_x2p_intf; i++) {
+		if (val & BIT_ULL(1 + i))
+			continue;
+		err = -EBUSY;
+		dev_err(mcs->dev, "MCS:%d didn't respond to X2P calibration\n", i);
+	}
+	/* Clear X2P calibrate */
+	mcs_reg_write(mcs, MCSX_MIL_GLOBAL, mcs_reg_read(mcs, MCSX_MIL_GLOBAL) & ~BIT_ULL(5));
+
+	return err;
+}
+
+static void mcs_set_external_bypass(struct mcs *mcs, u8 bypass)
+{
+	u64 val;
+
+	/* Set MCS to external bypass */
+	val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL);
+	if (bypass)
+		val |= BIT_ULL(6);
+	else
+		val &= ~BIT_ULL(6);
+	mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val);
+}
+
+static void mcs_global_cfg(struct mcs *mcs)
+{
+	/* Disable external bypass */
+	mcs_set_external_bypass(mcs, false);
+
+	/* Reset TX/RX stats memory */
+	mcs_reg_write(mcs, MCSX_CSE_RX_SLAVE_STATS_CLEAR, 0x1F);
+	mcs_reg_write(mcs, MCSX_CSE_TX_SLAVE_STATS_CLEAR, 0x1F);
+
+	/* Set MCS to perform standard IEEE802.1AE macsec processing */
+	if (mcs->hw->mcs_blks == 1) {
+		mcs_reg_write(mcs, MCSX_IP_MODE, BIT_ULL(3));
+		return;
+	}
+
+	mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_CAL_ENTRY, 0xe4);
+	mcs_reg_write(mcs, MCSX_BBE_RX_SLAVE_CAL_LEN, 4);
+}
+
+void cn10kb_mcs_set_hw_capabilities(struct mcs *mcs)
+{
+	struct hwinfo *hw = mcs->hw;
+
+	hw->tcam_entries = 128;		/* TCAM entries */
+	hw->secy_entries  = 128;	/* SecY entries */
+	hw->sc_entries = 128;		/* SC CAM entries */
+	hw->sa_entries = 256;		/* SA entries */
+	hw->lmac_cnt = 20;		/* lmacs/ports per mcs block */
+	hw->mcs_x2p_intf = 5;		/* x2p clabration intf */
+	hw->mcs_blks = 1;		/* MCS blocks */
+}
+
+static struct mcs_ops cn10kb_mcs_ops = {
+	.mcs_set_hw_capabilities	= cn10kb_mcs_set_hw_capabilities,
+	.mcs_parser_cfg			= cn10kb_mcs_parser_cfg,
+	.mcs_tx_sa_mem_map_write	= cn10kb_mcs_tx_sa_mem_map_write,
+	.mcs_rx_sa_mem_map_write	= cn10kb_mcs_rx_sa_mem_map_write,
+	.mcs_flowid_secy_map		= cn10kb_mcs_flowid_secy_map,
+};
+
+static int mcs_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct device *dev = &pdev->dev;
+	int lmac, err = 0;
+	struct mcs *mcs;
+
+	mcs = devm_kzalloc(dev, sizeof(*mcs), GFP_KERNEL);
+	if (!mcs)
+		return -ENOMEM;
+
+	mcs->hw = devm_kzalloc(dev, sizeof(struct hwinfo), GFP_KERNEL);
+	if (!mcs->hw)
+		return -ENOMEM;
+
+	err = pci_enable_device(pdev);
+	if (err) {
+		dev_err(dev, "Failed to enable PCI device\n");
+		pci_set_drvdata(pdev, NULL);
+		return err;
+	}
+
+	err = pci_request_regions(pdev, DRV_NAME);
+	if (err) {
+		dev_err(dev, "PCI request regions failed 0x%x\n", err);
+		goto exit;
+	}
+
+	mcs->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
+	if (!mcs->reg_base) {
+		dev_err(dev, "mcs: Cannot map CSR memory space, aborting\n");
+		err = -ENOMEM;
+		goto exit;
+	}
+
+	pci_set_drvdata(pdev, mcs);
+	mcs->pdev = pdev;
+	mcs->dev = &pdev->dev;
+
+	if (pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_B)
+		mcs->mcs_ops = &cn10kb_mcs_ops;
+	else
+		mcs->mcs_ops = cnf10kb_get_mac_ops();
+
+	/* Set hardware capabilities */
+	mcs->mcs_ops->mcs_set_hw_capabilities(mcs);
+
+	mcs_global_cfg(mcs);
+
+	/* Perform X2P clibration */
+	err = mcs_x2p_calibration(mcs);
+	if (err)
+		goto err_x2p;
+
+	mcs->mcs_id = (pci_resource_start(pdev, PCI_CFG_REG_BAR_NUM) >> 24)
+			& MCS_ID_MASK;
+
+	/* Set mcs tx side resources */
+	err = mcs_alloc_struct_mem(mcs, &mcs->tx);
+	if (err)
+		goto err_x2p;
+
+	/* Set mcs rx side resources */
+	err = mcs_alloc_struct_mem(mcs, &mcs->rx);
+	if (err)
+		goto err_x2p;
+
+	/* per port config */
+	for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++)
+		mcs_lmac_init(mcs, lmac);
+
+	/* Parser configuration */
+	mcs->mcs_ops->mcs_parser_cfg(mcs);
+
+	err = mcs_register_interrupts(mcs);
+	if (err)
+		goto exit;
+
+	list_add(&mcs->mcs_list, &mcs_list);
+	mutex_init(&mcs->stats_lock);
+
+	return 0;
+
+err_x2p:
+	/* Enable external bypass */
+	mcs_set_external_bypass(mcs, true);
+exit:
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+	return err;
+}
+
+static void mcs_remove(struct pci_dev *pdev)
+{
+	struct mcs *mcs = pci_get_drvdata(pdev);
+
+	/* Set MCS to external bypass */
+	mcs_set_external_bypass(mcs, true);
+	pci_free_irq_vectors(pdev);
+	pci_release_regions(pdev);
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+}
+
+struct pci_driver mcs_driver = {
+	.name = DRV_NAME,
+	.id_table = mcs_id_table,
+	.probe = mcs_probe,
+	.remove = mcs_remove,
+};
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs.h b/drivers/net/ethernet/marvell/octeontx2/af/mcs.h
new file mode 100644
index 000000000000..64dc2b80e15d
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs.h
@@ -0,0 +1,246 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Marvell CN10K MCS driver
+ *
+ * Copyright (C) 2022 Marvell.
+ */
+
+#ifndef MCS_H
+#define MCS_H
+
+#include <linux/bits.h>
+#include "rvu.h"
+
+#define PCI_DEVID_CN10K_MCS		0xA096
+
+#define MCSX_LINK_LMAC_RANGE_MASK	GENMASK_ULL(19, 16)
+#define MCSX_LINK_LMAC_BASE_MASK	GENMASK_ULL(11, 0)
+
+#define MCS_ID_MASK			0x7
+#define MCS_MAX_PFS                     128
+
+#define MCS_PORT_MODE_MASK		0x3
+#define MCS_PORT_FIFO_SKID_MASK		0x3F
+#define MCS_MAX_CUSTOM_TAGS		0x8
+
+#define MCS_CTRLPKT_ETYPE_RULE_MAX	8
+#define MCS_CTRLPKT_DA_RULE_MAX		8
+#define MCS_CTRLPKT_DA_RANGE_RULE_MAX	4
+#define MCS_CTRLPKT_COMBO_RULE_MAX	4
+#define MCS_CTRLPKT_MAC_RULE_MAX	1
+
+#define MCS_MAX_CTRLPKT_RULES	(MCS_CTRLPKT_ETYPE_RULE_MAX + \
+				MCS_CTRLPKT_DA_RULE_MAX + \
+				MCS_CTRLPKT_DA_RANGE_RULE_MAX + \
+				MCS_CTRLPKT_COMBO_RULE_MAX + \
+				MCS_CTRLPKT_MAC_RULE_MAX)
+
+#define MCS_CTRLPKT_ETYPE_RULE_OFFSET		0
+#define MCS_CTRLPKT_DA_RULE_OFFSET		8
+#define MCS_CTRLPKT_DA_RANGE_RULE_OFFSET	16
+#define MCS_CTRLPKT_COMBO_RULE_OFFSET		20
+#define MCS_CTRLPKT_MAC_EN_RULE_OFFSET		24
+
+/* Reserved resources for default bypass entry */
+#define MCS_RSRC_RSVD_CNT		1
+
+/* MCS Interrupt Vector Enumeration */
+enum mcs_int_vec_e {
+	MCS_INT_VEC_MIL_RX_GBL		= 0x0,
+	MCS_INT_VEC_MIL_RX_LMACX	= 0x1,
+	MCS_INT_VEC_MIL_TX_LMACX	= 0x5,
+	MCS_INT_VEC_HIL_RX_GBL		= 0x9,
+	MCS_INT_VEC_HIL_RX_LMACX	= 0xa,
+	MCS_INT_VEC_HIL_TX_GBL		= 0xe,
+	MCS_INT_VEC_HIL_TX_LMACX	= 0xf,
+	MCS_INT_VEC_IP			= 0x13,
+	MCS_INT_VEC_CNT			= 0x14,
+};
+
+#define MCS_MAX_BBE_INT			8ULL
+#define MCS_BBE_INT_MASK		0xFFULL
+
+#define MCS_MAX_PAB_INT			4ULL
+#define MCS_PAB_INT_MASK		0xFULL
+
+#define MCS_BBE_RX_INT_ENA		BIT_ULL(0)
+#define MCS_BBE_TX_INT_ENA		BIT_ULL(1)
+#define MCS_CPM_RX_INT_ENA		BIT_ULL(2)
+#define MCS_CPM_TX_INT_ENA		BIT_ULL(3)
+#define MCS_PAB_RX_INT_ENA		BIT_ULL(4)
+#define MCS_PAB_TX_INT_ENA		BIT_ULL(5)
+
+#define MCS_CPM_TX_INT_PACKET_XPN_EQ0		BIT_ULL(0)
+#define MCS_CPM_TX_INT_PN_THRESH_REACHED	BIT_ULL(1)
+#define MCS_CPM_TX_INT_SA_NOT_VALID		BIT_ULL(2)
+
+#define MCS_CPM_RX_INT_SECTAG_V_EQ1		BIT_ULL(0)
+#define MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1	BIT_ULL(1)
+#define MCS_CPM_RX_INT_SL_GTE48			BIT_ULL(2)
+#define MCS_CPM_RX_INT_ES_EQ1_SC_EQ1		BIT_ULL(3)
+#define MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1		BIT_ULL(4)
+#define MCS_CPM_RX_INT_PACKET_XPN_EQ0		BIT_ULL(5)
+#define MCS_CPM_RX_INT_PN_THRESH_REACHED	BIT_ULL(6)
+
+#define MCS_CPM_RX_INT_ALL	(MCS_CPM_RX_INT_SECTAG_V_EQ1 |		\
+				 MCS_CPM_RX_INT_SECTAG_E_EQ0_C_EQ1 |    \
+				 MCS_CPM_RX_INT_SL_GTE48 |		\
+				 MCS_CPM_RX_INT_ES_EQ1_SC_EQ1 |		\
+				 MCS_CPM_RX_INT_SC_EQ1_SCB_EQ1 |	\
+				 MCS_CPM_RX_INT_PACKET_XPN_EQ0 |	\
+				 MCS_CPM_RX_INT_PN_THRESH_REACHED)
+
+struct mcs_pfvf {
+	u64 intr_mask;	/* Enabled Interrupt mask */
+};
+
+struct mcs_intr_event {
+	u16 pcifunc;
+	u64 intr_mask;
+	u64 sa_id;
+	u8 mcs_id;
+	u8 lmac_id;
+};
+
+struct mcs_intrq_entry {
+	struct list_head node;
+	struct mcs_intr_event intr_event;
+};
+
+struct secy_mem_map {
+	u8 flow_id;
+	u8 secy;
+	u8 ctrl_pkt;
+	u8 sc;
+	u64 sci;
+};
+
+struct mcs_rsrc_map {
+	u16 *flowid2pf_map;
+	u16 *secy2pf_map;
+	u16 *sc2pf_map;
+	u16 *sa2pf_map;
+	u16 *flowid2secy_map;	/* bitmap flowid mapped to secy*/
+	u16 *ctrlpktrule2pf_map;
+	struct rsrc_bmap	flow_ids;
+	struct rsrc_bmap	secy;
+	struct rsrc_bmap	sc;
+	struct rsrc_bmap	sa;
+	struct rsrc_bmap	ctrlpktrule;
+};
+
+struct hwinfo {
+	u8 tcam_entries;
+	u8 secy_entries;
+	u8 sc_entries;
+	u16 sa_entries;
+	u8 mcs_x2p_intf;
+	u8 lmac_cnt;
+	u8 mcs_blks;
+	unsigned long	lmac_bmap; /* bitmap of enabled mcs lmac */
+};
+
+struct mcs {
+	void __iomem		*reg_base;
+	struct pci_dev		*pdev;
+	struct device		*dev;
+	struct hwinfo		*hw;
+	struct mcs_rsrc_map	tx;
+	struct mcs_rsrc_map	rx;
+	u16                     pf_map[MCS_MAX_PFS]; /* List of PCIFUNC mapped to MCS */
+	u8			mcs_id;
+	struct mcs_ops		*mcs_ops;
+	struct list_head	mcs_list;
+	/* Lock for mcs stats */
+	struct mutex		stats_lock;
+	struct mcs_pfvf		*pf;
+	struct mcs_pfvf		*vf;
+	u16			num_vec;
+	void			*rvu;
+	u16			*tx_sa_active;
+};
+
+struct mcs_ops {
+	void	(*mcs_set_hw_capabilities)(struct mcs *mcs);
+	void	(*mcs_parser_cfg)(struct mcs *mcs);
+	void	(*mcs_tx_sa_mem_map_write)(struct mcs *mcs, struct mcs_tx_sc_sa_map *map);
+	void	(*mcs_rx_sa_mem_map_write)(struct mcs *mcs, struct mcs_rx_sc_sa_map *map);
+	void	(*mcs_flowid_secy_map)(struct mcs *mcs, struct secy_mem_map *map, int dir);
+};
+
+extern struct pci_driver mcs_driver;
+
+static inline void mcs_reg_write(struct mcs *mcs, u64 offset, u64 val)
+{
+	writeq(val, mcs->reg_base + offset);
+}
+
+static inline u64 mcs_reg_read(struct mcs *mcs, u64 offset)
+{
+	return readq(mcs->reg_base + offset);
+}
+
+/* MCS APIs */
+struct mcs *mcs_get_pdata(int mcs_id);
+int mcs_get_blkcnt(void);
+int mcs_set_lmac_channels(int mcs_id, u16 base);
+int mcs_alloc_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, u16 pcifunc);
+int mcs_free_rsrc(struct rsrc_bmap *rsrc, u16 *pf_map, int rsrc_id, u16 pcifunc);
+int mcs_alloc_all_rsrc(struct mcs *mcs, u8 *flowid, u8 *secy_id,
+		       u8 *sc_id, u8 *sa1_id, u8 *sa2_id, u16 pcifunc, int dir);
+int mcs_free_all_rsrc(struct mcs *mcs, int dir, u16 pcifunc);
+void mcs_clear_secy_plcy(struct mcs *mcs, int secy_id, int dir);
+void mcs_ena_dis_flowid_entry(struct mcs *mcs, int id, int dir, int ena);
+void mcs_ena_dis_sc_cam_entry(struct mcs *mcs, int id, int ena);
+void mcs_flowid_entry_write(struct mcs *mcs, u64 *data, u64 *mask, int id, int dir);
+void mcs_secy_plcy_write(struct mcs *mcs, u64 plcy, int id, int dir);
+void mcs_rx_sc_cam_write(struct mcs *mcs, u64 sci, u64 secy, int sc_id);
+void mcs_sa_plcy_write(struct mcs *mcs, u64 *plcy, int sa, int dir);
+void mcs_map_sc_to_sa(struct mcs *mcs, u64 *sa_map, int sc, int dir);
+void mcs_pn_table_write(struct mcs *mcs, u8 pn_id, u64 next_pn, u8 dir);
+void mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map);
+void mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir);
+void mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map);
+void mcs_pn_threshold_set(struct mcs *mcs, struct mcs_set_pn_threshold *pn);
+int mcs_install_flowid_bypass_entry(struct mcs *mcs);
+void mcs_set_lmac_mode(struct mcs *mcs, int lmac_id, u8 mode);
+void mcs_reset_port(struct mcs *mcs, u8 port_id, u8 reset);
+void mcs_set_port_cfg(struct mcs *mcs, struct mcs_port_cfg_set_req *req);
+void mcs_get_port_cfg(struct mcs *mcs, struct mcs_port_cfg_get_req *req,
+		      struct mcs_port_cfg_get_rsp *rsp);
+void mcs_get_custom_tag_cfg(struct mcs *mcs, struct mcs_custom_tag_cfg_get_req *req,
+			    struct mcs_custom_tag_cfg_get_rsp *rsp);
+int mcs_alloc_ctrlpktrule(struct rsrc_bmap *rsrc, u16 *pf_map, u16 offset, u16 pcifunc);
+int mcs_free_ctrlpktrule(struct mcs *mcs, struct mcs_free_ctrl_pkt_rule_req *req);
+int mcs_ctrlpktrule_write(struct mcs *mcs, struct mcs_ctrl_pkt_rule_write_req *req);
+
+/* CN10K-B APIs */
+void cn10kb_mcs_set_hw_capabilities(struct mcs *mcs);
+void cn10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map);
+void cn10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir);
+void cn10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map);
+void cn10kb_mcs_parser_cfg(struct mcs *mcs);
+
+/* CNF10K-B APIs */
+struct mcs_ops *cnf10kb_get_mac_ops(void);
+void cnf10kb_mcs_set_hw_capabilities(struct mcs *mcs);
+void cnf10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map);
+void cnf10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir);
+void cnf10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map);
+void cnf10kb_mcs_parser_cfg(struct mcs *mcs);
+void cnf10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs);
+void cnf10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs);
+
+/* Stats APIs */
+void mcs_get_sc_stats(struct mcs *mcs, struct mcs_sc_stats *stats, int id, int dir);
+void mcs_get_sa_stats(struct mcs *mcs, struct mcs_sa_stats *stats, int id, int dir);
+void mcs_get_port_stats(struct mcs *mcs, struct mcs_port_stats *stats, int id, int dir);
+void mcs_get_flowid_stats(struct mcs *mcs, struct mcs_flowid_stats *stats, int id, int dir);
+void mcs_get_rx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id);
+void mcs_get_tx_secy_stats(struct mcs *mcs, struct mcs_secy_stats *stats, int id);
+void mcs_clear_stats(struct mcs *mcs, u8 type, u8 id, int dir);
+int mcs_clear_all_stats(struct mcs *mcs, u16 pcifunc, int dir);
+int mcs_set_force_clk_en(struct mcs *mcs, bool set);
+
+int mcs_add_intr_wq_entry(struct mcs *mcs, struct mcs_intr_event *event);
+
+#endif /* MCS_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c
new file mode 100644
index 000000000000..7b6205414428
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_cnf10kb.c
@@ -0,0 +1,214 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell MCS driver
+ *
+ * Copyright (C) 2022 Marvell.
+ */
+
+#include "mcs.h"
+#include "mcs_reg.h"
+
+static struct mcs_ops cnf10kb_mcs_ops   = {
+	.mcs_set_hw_capabilities	= cnf10kb_mcs_set_hw_capabilities,
+	.mcs_parser_cfg			= cnf10kb_mcs_parser_cfg,
+	.mcs_tx_sa_mem_map_write	= cnf10kb_mcs_tx_sa_mem_map_write,
+	.mcs_rx_sa_mem_map_write	= cnf10kb_mcs_rx_sa_mem_map_write,
+	.mcs_flowid_secy_map		= cnf10kb_mcs_flowid_secy_map,
+};
+
+struct mcs_ops *cnf10kb_get_mac_ops(void)
+{
+	return &cnf10kb_mcs_ops;
+}
+
+void cnf10kb_mcs_set_hw_capabilities(struct mcs *mcs)
+{
+	struct hwinfo *hw = mcs->hw;
+
+	hw->tcam_entries = 64;		/* TCAM entries */
+	hw->secy_entries  = 64;		/* SecY entries */
+	hw->sc_entries = 64;		/* SC CAM entries */
+	hw->sa_entries = 128;		/* SA entries */
+	hw->lmac_cnt = 4;		/* lmacs/ports per mcs block */
+	hw->mcs_x2p_intf = 1;		/* x2p clabration intf */
+	hw->mcs_blks = 7;		/* MCS blocks */
+}
+
+void cnf10kb_mcs_parser_cfg(struct mcs *mcs)
+{
+	u64 reg, val;
+
+	/* VLAN Ctag */
+	val = (0x8100ull & 0xFFFF) | BIT_ULL(20) | BIT_ULL(22);
+
+	reg = MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(0);
+	mcs_reg_write(mcs, reg, val);
+
+	reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(0);
+	mcs_reg_write(mcs, reg, val);
+
+	/* VLAN STag */
+	val = (0x88a8ull & 0xFFFF) | BIT_ULL(20) | BIT_ULL(23);
+
+	/* RX */
+	reg = MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(1);
+	mcs_reg_write(mcs, reg, val);
+
+	/* TX */
+	reg = MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(1);
+	mcs_reg_write(mcs, reg, val);
+
+	/* Enable custom tage 0 and 1 and sectag */
+	val = BIT_ULL(0) | BIT_ULL(1) | BIT_ULL(12);
+
+	reg = MCSX_PEX_RX_SLAVE_ETYPE_ENABLE;
+	mcs_reg_write(mcs, reg, val);
+
+	reg = MCSX_PEX_TX_SLAVE_ETYPE_ENABLE;
+	mcs_reg_write(mcs, reg, val);
+}
+
+void cnf10kb_mcs_flowid_secy_map(struct mcs *mcs, struct secy_mem_map *map, int dir)
+{
+	u64 reg, val;
+
+	val = (map->secy & 0x3F) | (map->ctrl_pkt & 0x1) << 6;
+	if (dir == MCS_RX) {
+		reg = MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(map->flow_id);
+	} else {
+		reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(map->flow_id);
+		mcs_reg_write(mcs, reg, map->sci);
+		val |= (map->sc & 0x3F) << 7;
+		reg = MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_1X(map->flow_id);
+	}
+
+	mcs_reg_write(mcs, reg, val);
+}
+
+void cnf10kb_mcs_tx_sa_mem_map_write(struct mcs *mcs, struct mcs_tx_sc_sa_map *map)
+{
+	u64 reg, val;
+
+	val = (map->sa_index0 & 0x7F) | (map->sa_index1 & 0x7F) << 7;
+
+	reg = MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(map->sc_id);
+	mcs_reg_write(mcs, reg, val);
+
+	reg = MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0;
+	val = mcs_reg_read(mcs, reg);
+
+	if (map->rekey_ena)
+		val |= BIT_ULL(map->sc_id);
+	else
+		val &= ~BIT_ULL(map->sc_id);
+
+	mcs_reg_write(mcs, reg, val);
+
+	mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_SA_INDEX0_VLDX(map->sc_id), map->sa_index0_vld);
+	mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_SA_INDEX1_VLDX(map->sc_id), map->sa_index1_vld);
+
+	mcs_reg_write(mcs, MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(map->sc_id), map->tx_sa_active);
+}
+
+void cnf10kb_mcs_rx_sa_mem_map_write(struct mcs *mcs, struct mcs_rx_sc_sa_map *map)
+{
+	u64 val, reg;
+
+	val = (map->sa_index & 0x7F) | (map->sa_in_use << 7);
+
+	reg = MCSX_CPM_RX_SLAVE_SA_MAP_MEMX((4 * map->sc_id) + map->an);
+	mcs_reg_write(mcs, reg, val);
+}
+
+int mcs_set_force_clk_en(struct mcs *mcs, bool set)
+{
+	unsigned long timeout = jiffies + usecs_to_jiffies(2000);
+	u64 val;
+
+	val = mcs_reg_read(mcs, MCSX_MIL_GLOBAL);
+
+	if (set) {
+		val |= BIT_ULL(4);
+		mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val);
+
+		/* Poll till mcsx_mil_ip_gbl_status.mcs_ip_stats_ready value is 1 */
+		while (!(mcs_reg_read(mcs, MCSX_MIL_IP_GBL_STATUS) & BIT_ULL(0))) {
+			if (time_after(jiffies, timeout)) {
+				dev_err(mcs->dev, "MCS set force clk enable failed\n");
+				break;
+			}
+		}
+	} else {
+		val &= ~BIT_ULL(4);
+		mcs_reg_write(mcs, MCSX_MIL_GLOBAL, val);
+	}
+
+	return 0;
+}
+
+/* TX SA interrupt is raised only if autorekey is enabled.
+ * MCS_CPM_TX_SLAVE_SA_MAP_MEM_0X[sc].tx_sa_active bit gets toggled if
+ * one of two SAs mapped to SC gets expired. If tx_sa_active=0 implies
+ * SA in SA_index1 got expired else SA in SA_index0 got expired.
+ */
+void cnf10kb_mcs_tx_pn_thresh_reached_handler(struct mcs *mcs)
+{
+	struct mcs_intr_event event;
+	struct rsrc_bmap *sc_bmap;
+	unsigned long rekey_ena;
+	u64 val, sa_status;
+	int sc;
+
+	sc_bmap = &mcs->tx.sc;
+
+	event.mcs_id = mcs->mcs_id;
+	event.intr_mask = MCS_CPM_TX_PN_THRESH_REACHED_INT;
+
+	rekey_ena = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0);
+
+	for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) {
+		/* Auto rekey is enable */
+		if (!test_bit(sc, &rekey_ena))
+			continue;
+		sa_status = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(sc));
+		/* Check if tx_sa_active status had changed */
+		if (sa_status == mcs->tx_sa_active[sc])
+			continue;
+
+		/* SA_index0 is expired */
+		val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc));
+		if (sa_status)
+			event.sa_id = val & 0x7F;
+		else
+			event.sa_id = (val >> 7) & 0x7F;
+
+		event.pcifunc = mcs->tx.sa2pf_map[event.sa_id];
+		mcs_add_intr_wq_entry(mcs, &event);
+	}
+}
+
+void cnf10kb_mcs_tx_pn_wrapped_handler(struct mcs *mcs)
+{
+	struct mcs_intr_event event = { 0 };
+	struct rsrc_bmap *sc_bmap;
+	u64 val;
+	int sc;
+
+	sc_bmap = &mcs->tx.sc;
+
+	event.mcs_id = mcs->mcs_id;
+	event.intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT;
+
+	for_each_set_bit(sc, sc_bmap->bmap, mcs->hw->sc_entries) {
+		val = mcs_reg_read(mcs, MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(sc));
+
+		if (mcs->tx_sa_active[sc])
+			/* SA_index1 was used and got expired */
+			event.sa_id = (val >> 7) & 0x7F;
+		else
+			/* SA_index0 was used and got expired */
+			event.sa_id = val & 0x7F;
+
+		event.pcifunc = mcs->tx.sa2pf_map[event.sa_id];
+		mcs_add_intr_wq_entry(mcs, &event);
+	}
+}
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h b/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h
new file mode 100644
index 000000000000..c95a8b8f5eaf
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_reg.h
@@ -0,0 +1,1102 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Marvell MCS driver
+ *
+ * Copyright (C) 2022 Marvell.
+ */
+
+#ifndef MCS_REG_H
+#define MCS_REG_H
+
+#include <linux/bits.h>
+
+/* Registers */
+#define MCSX_IP_MODE					0x900c8ull
+#define MCSX_MCS_TOP_SLAVE_PORT_RESET(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x408ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xa28ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+
+#define MCSX_MCS_TOP_SLAVE_CHANNEL_CFG(a) ({		\
+	u64 offset;					\
+							\
+	offset = 0x808ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xa68ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_MIL_GLOBAL	({				\
+	u64 offset;					\
+							\
+	offset = 0x80000ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x60000ull;			\
+	offset; })
+
+#define MCSX_MIL_RX_LMACX_CFG(a) ({			\
+	u64 offset;					\
+							\
+	offset = 0x900a8ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x700a8ull;			\
+	offset += (a) * 0x800ull;			\
+	offset; })
+
+#define MCSX_HIL_GLOBAL ({				\
+	u64 offset;					\
+							\
+	offset = 0xc0000ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xa0000ull;			\
+	offset; })
+
+#define MCSX_LINK_LMACX_CFG(a) ({			\
+	u64 offset;					\
+							\
+	offset = 0x90000ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x70000ull;			\
+	offset += (a) * 0x800ull;			\
+	offset; })
+
+#define MCSX_MIL_RX_GBL_STATUS ({			\
+	u64 offset;					\
+							\
+	offset = 0x800c8ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x600c8ull;			\
+	offset; })
+
+#define MCSX_MIL_IP_GBL_STATUS ({			\
+	u64 offset;					\
+							\
+	offset = 0x800d0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x600d0ull;			\
+	offset; })
+
+/* PAB */
+#define MCSX_PAB_RX_SLAVE_PORT_CFGX(a) ({	\
+	u64 offset;				\
+						\
+	offset = 0x1718ull;			\
+	if (mcs->hw->mcs_blks > 1)		\
+		offset = 0x280ull;		\
+	offset += (a) * 0x40ull;		\
+	offset; })
+
+#define MCSX_PAB_TX_SLAVE_PORT_CFGX(a)			(0x2930ull + (a) * 0x40ull)
+
+/* PEX registers */
+#define MCSX_PEX_RX_SLAVE_VLAN_CFGX(a)          (0x3b58ull + (a) * 0x8ull)
+#define MCSX_PEX_TX_SLAVE_VLAN_CFGX(a)          (0x46f8ull + (a) * 0x8ull)
+#define MCSX_PEX_TX_SLAVE_CUSTOM_TAG_REL_MODE_SEL(a)	(0x788ull + (a) * 0x8ull)
+#define MCSX_PEX_TX_SLAVE_PORT_CONFIG(a)		(0x4738ull + (a) * 0x8ull)
+#define MCSX_PEX_RX_SLAVE_RULE_ETYPE_CFGX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x3fc0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x558ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_RX_SLAVE_RULE_DAX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4000ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x598ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MINX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4040ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x5d8ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_RX_SLAVE_RULE_DA_RANGE_MAXX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4048ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x5e0ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_RX_SLAVE_RULE_COMBO_MINX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4080ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x648ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_RX_SLAVE_RULE_COMBO_MAXX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4088ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x650ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_RX_SLAVE_RULE_COMBO_ETX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4090ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x658ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_RX_SLAVE_RULE_MAC ({	\
+	u64 offset;					\
+							\
+	offset = 0x40e0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x6d8ull;			\
+	offset; })
+
+#define MCSX_PEX_RX_SLAVE_RULE_ENABLE ({	\
+	u64 offset;					\
+							\
+	offset = 0x40e8ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x6e0ull;			\
+	offset; })
+
+#define MCSX_PEX_TX_SLAVE_RULE_ETYPE_CFGX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4b60ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x7d8ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_TX_SLAVE_RULE_DAX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4ba0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x818ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MINX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4be0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x858ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_TX_SLAVE_RULE_DA_RANGE_MAXX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4be8ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x860ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_TX_SLAVE_RULE_COMBO_MINX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4c20ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x8c8ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_TX_SLAVE_RULE_COMBO_MAXX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4c28ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x8d0ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_TX_SLAVE_RULE_COMBO_ETX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x4c30ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x8d8ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_PEX_TX_SLAVE_RULE_MAC ({	\
+	u64 offset;					\
+							\
+	offset = 0x4c80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x958ull;			\
+	offset; })
+
+#define MCSX_PEX_TX_SLAVE_RULE_ENABLE ({	\
+	u64 offset;					\
+							\
+	offset = 0x4c88ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x960ull;			\
+	offset; })
+
+#define MCSX_PEX_RX_SLAVE_PEX_CONFIGURATION ({		\
+	u64 offset;					\
+							\
+	offset = 0x3b50ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x4c0ull;			\
+	offset; })
+
+/* CNF10K-B */
+#define MCSX_PEX_RX_SLAVE_CUSTOM_TAGX(a)        (0x4c8ull + (a) * 0x8ull)
+#define MCSX_PEX_TX_SLAVE_CUSTOM_TAGX(a)        (0x748ull + (a) * 0x8ull)
+#define MCSX_PEX_RX_SLAVE_ETYPE_ENABLE          0x6e8ull
+#define MCSX_PEX_TX_SLAVE_ETYPE_ENABLE          0x968ull
+
+/* BEE */
+#define MCSX_BBE_RX_SLAVE_PADDING_CTL			0xe08ull
+#define MCSX_BBE_TX_SLAVE_PADDING_CTL			0x12f8ull
+#define MCSX_BBE_RX_SLAVE_CAL_ENTRY			0x180ull
+#define MCSX_BBE_RX_SLAVE_CAL_LEN			0x188ull
+#define MCSX_PAB_RX_SLAVE_FIFO_SKID_CFGX(a)		(0x290ull + (a) * 0x40ull)
+
+#define MCSX_BBE_RX_SLAVE_BBE_INT ({	\
+	u64 offset;			\
+					\
+	offset = 0xe00ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x160ull;	\
+	offset; })
+
+#define MCSX_BBE_RX_SLAVE_BBE_INT_ENB ({	\
+	u64 offset;			\
+					\
+	offset = 0xe08ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x168ull;	\
+	offset; })
+
+#define MCSX_BBE_RX_SLAVE_BBE_INT_INTR_RW ({	\
+	u64 offset;			\
+					\
+	offset = 0xe08ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x178ull;	\
+	offset; })
+
+#define MCSX_BBE_TX_SLAVE_BBE_INT ({	\
+	u64 offset;			\
+					\
+	offset = 0x1278ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x1e0ull;	\
+	offset; })
+
+#define MCSX_BBE_TX_SLAVE_BBE_INT_INTR_RW ({	\
+	u64 offset;			\
+					\
+	offset = 0x1278ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x1f8ull;	\
+	offset; })
+
+#define MCSX_BBE_TX_SLAVE_BBE_INT_ENB ({	\
+	u64 offset;			\
+					\
+	offset = 0x1280ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x1e8ull;	\
+	offset; })
+
+#define MCSX_PAB_RX_SLAVE_PAB_INT ({	\
+	u64 offset;			\
+					\
+	offset = 0x16f0ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x260ull;	\
+	offset; })
+
+#define MCSX_PAB_RX_SLAVE_PAB_INT_ENB ({	\
+	u64 offset;			\
+					\
+	offset = 0x16f8ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x268ull;	\
+	offset; })
+
+#define MCSX_PAB_RX_SLAVE_PAB_INT_INTR_RW ({	\
+	u64 offset;			\
+					\
+	offset = 0x16f8ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x278ull;	\
+	offset; })
+
+#define MCSX_PAB_TX_SLAVE_PAB_INT ({	\
+	u64 offset;			\
+					\
+	offset = 0x2908ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x380ull;	\
+	offset; })
+
+#define MCSX_PAB_TX_SLAVE_PAB_INT_ENB ({	\
+	u64 offset;			\
+					\
+	offset = 0x2910ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x388ull;	\
+	offset; })
+
+#define MCSX_PAB_TX_SLAVE_PAB_INT_INTR_RW ({	\
+	u64 offset;			\
+					\
+	offset = 0x16f8ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x398ull;	\
+	offset; })
+
+/* CPM registers */
+#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_DATAX(a, b) ({	\
+	u64 offset;					\
+							\
+	offset = 0x30740ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x3bf8ull;			\
+	offset += (a) * 0x8ull + (b) * 0x20ull;		\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_MASKX(a, b) ({	\
+	u64 offset;					\
+							\
+	offset = 0x34740ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x43f8ull;			\
+	offset += (a) * 0x8ull + (b) * 0x20ull;		\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_0 ({		\
+	u64 offset;					\
+							\
+	offset = 0x30700ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x3bd8ull;			\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_SC_CAMX(a, b)	({		\
+	u64 offset;					\
+							\
+	offset = 0x38780ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x4c08ull;			\
+	offset +=  (a) * 0x8ull + (b) * 0x10ull;	\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_SC_CAM_ENA(a)	({		\
+	u64 offset;					\
+							\
+	offset = 0x38740ull + (a) * 0x8ull;		\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x4bf8ull;			\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_SECY_MAP_MEMX(a) ({		\
+	u64 offset;					\
+							\
+	offset = 0x23ee0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xbd0ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_0X(a) ({	\
+	u64 offset;					\
+							\
+	offset = (0x246e0ull + (a) * 0x10ull);		\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = (0xdd0ull + (a) * 0x8ull);	\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_SA_KEY_LOCKOUTX(a) ({		\
+	u64 offset;					\
+							\
+	offset = 0x23E90ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xbb0ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_SA_MAP_MEMX(a) ({		\
+	u64 offset;					\
+							\
+	offset = 0x256e0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xfd0ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_SA_PLCY_MEMX(a, b) ({		\
+	u64 offset;					\
+							\
+	offset = 0x27700ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x17d8ull;			\
+	offset +=  (a) * 0x8ull + (b) * 0x40ull;	\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_SA_PN_TABLE_MEMX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x2f700ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x37d8;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_XPN_THRESHOLD	({		\
+	u64 offset;					\
+							\
+	offset = 0x23e40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xb90ull;			\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_PN_THRESHOLD	({		\
+	u64 offset;					\
+							\
+	offset = 0x23e48ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xb98ull;			\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_PN_THRESH_REACHEDX(a)	({	\
+	u64 offset;					\
+							\
+	offset = 0x23e50ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xba0ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_FLOWID_TCAM_ENA_1		0x30708ull
+#define MCSX_CPM_RX_SLAVE_SECY_PLCY_MEM_1X(a)		(0x246e8ull + (a) * 0x10ull)
+
+/* TX registers */
+#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_DATAX(a, b) ({	\
+	u64 offset;					\
+							\
+	offset = 0x51d50ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xa7c0ull;			\
+	offset += (a) * 0x8ull + (b) * 0x20ull;		\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_MASKX(a, b) ({	\
+	u64 offset;					\
+							\
+	offset = 0x55d50ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xafc0ull;			\
+	offset += (a) * 0x8ull + (b) * 0x20ull;		\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_0 ({		\
+	u64 offset;					\
+							\
+	offset = 0x51d10ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xa7a0ull;			\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_0X(a) ({		\
+	u64 offset;					\
+							\
+	offset = 0x3e508ull + (a) * 0x8ull;		\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x5550ull + (a) * 0x10ull;	\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_SECY_PLCY_MEMX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x3ed08ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x5950ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_SA_KEY_LOCKOUTX(a) ({		\
+	u64 offset;					\
+							\
+	offset = 0x3e4c0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x5538ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_SA_MAP_MEM_0X(a) ({		\
+	u64 offset;					\
+							\
+	offset = 0x3fd10ull + (a) * 0x10ull;		\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x6150ull + (a) * 0x8ull;	\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_SA_PLCY_MEMX(a, b) ({		\
+	u64 offset;					\
+							\
+	offset = 0x40d10ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x63a0ull;			\
+	offset += (a) * 0x8ull + (b) * 0x80ull;		\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_SA_PN_TABLE_MEMX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x50d10ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xa3a0ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_XPN_THRESHOLD ({		\
+	u64 offset;					\
+							\
+	offset = 0x3e4b0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x5528ull;			\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_PN_THRESHOLD ({		\
+	u64 offset;					\
+							\
+	offset = 0x3e4b8ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x5530ull;			\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_SA_MAP_MEM_1X(a)		(0x3fd18ull + (a) * 0x10ull)
+#define MCSX_CPM_TX_SLAVE_SECY_MAP_MEM_1X(a)		(0x5558ull + (a) * 0x10ull)
+#define MCSX_CPM_TX_SLAVE_FLOWID_TCAM_ENA_1		0x51d18ull
+#define MCSX_CPM_TX_SLAVE_TX_SA_ACTIVEX(a)		(0x5b50 + (a) * 0x8ull)
+#define MCSX_CPM_TX_SLAVE_SA_INDEX0_VLDX(a)		(0x5d50 + (a) * 0x8ull)
+#define MCSX_CPM_TX_SLAVE_SA_INDEX1_VLDX(a)		(0x5f50 + (a) * 0x8ull)
+#define MCSX_CPM_TX_SLAVE_AUTO_REKEY_ENABLE_0		0x5500ull
+
+/* CSE */
+#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLBCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x9e80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xc218ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLMCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x9680ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xc018ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLOCTETSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x6e80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xbc18ull;			\
+	offset +=  (a) * 0x8ull;			\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_IFINCTLUCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x8e80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xbe18ull;			\
+	offset +=  (a) * 0x8ull;			\
+	offset; })
+
+#define	MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLBCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x8680ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xca18ull;			\
+	offset +=  (a) * 0x8ull;			\
+	offset; })
+
+#define	MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLMCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x7e80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xc818ull;			\
+	offset +=  (a) * 0x8ull;			\
+	offset; })
+
+#define	MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLOCTETSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x6680ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xc418ull;			\
+	offset +=  (a) * 0x8ull;			\
+	offset; })
+
+#define	MCSX_CSE_RX_MEM_SLAVE_IFINUNCTLUCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x7680ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xc618ull;			\
+	offset +=  (a) * 0x8ull;			\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYDECRYPTEDX(a) ({ \
+	u64 offset;					\
+							\
+	offset = 0x5e80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xdc18ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSECYVALIDATEX(a)({ \
+	u64 offset;					\
+							\
+	offset = 0x5680ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xda18ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSCTRLPORTDISABLEDX(a) ({ \
+	u64 offset;					\
+							\
+	offset = 0xd680ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xce18ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMHITX(a) ({ \
+	u64 offset;					\
+							\
+	offset = 0x16a80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xec78ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSFLOWIDTCAMMISSX(a) ({ \
+	u64 offset;					\
+							\
+	offset = 0x16680ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xec38ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSPARSEERRX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x16880ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xec18ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCCAMHITX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0xfe80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xde18ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCINVALIDX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x10680ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xe418ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCNOTVALIDX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x10e80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xe218ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYBADTAGX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0xae80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xd418ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0xc680ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xd618ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOSAERRORX(a) ({ \
+	u64 offset;					\
+							\
+	offset = 0xce80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xd818ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYTAGGEDCTLX(a) ({ \
+	u64 offset;					\
+							\
+	offset = 0xbe80ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xcc18ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_RX_SLAVE_CTRL	({			\
+	u64 offset;					\
+							\
+	offset = 0x52a0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x9c0ull;			\
+	offset; })
+
+#define MCSX_CSE_RX_SLAVE_STATS_CLEAR	({		\
+	u64 offset;					\
+							\
+	offset = 0x52b8ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x9d8ull;			\
+	offset; })
+
+#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCDECRYPTEDX(a)	(0xe680ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INOCTETSSCVALIDATEX(a)	(0xde80ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDORNOTAGX(a)	(0xa680ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYNOTAGX(a)	(0xd218 + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYUNTAGGEDX(a)	(0xd018ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCUNCHECKEDOROKX(a)	(0xee80ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSECYCTLX(a)		(0xb680ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCLATEORDELAYEDX(a) (0xf680ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAINVALIDX(a)	(0x12680ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTUSINGSAERRORX(a) (0x15680ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSANOTVALIDX(a)	(0x13680ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAOKX(a)		(0x11680ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSAUNUSEDSAX(a)	(0x14680ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSEARLYPREEMPTERRX(a) (0xec58ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCOKX(a)		(0xea18ull + (a) * 0x8ull)
+#define MCSX_CSE_RX_MEM_SLAVE_INPKTSSCDELAYEDX(a)	(0xe618ull + (a) * 0x8ull)
+
+/* CSE TX */
+#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCOMMONOCTETSX(a)	(0x18440ull + (a) * 0x8ull)
+#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLBCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x1c440ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xf478ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLMCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x1bc40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xf278ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLOCTETSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x19440ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xee78ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_IFOUTCTLUCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x1b440ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xf078ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLBCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x1ac40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xfc78ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLMCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x1a440ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xfa78ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLOCTETSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x18c40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xf678ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_IFOUTUNCTLUCPKTSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x19c40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xf878ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYENCRYPTEDX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x17c40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x10878ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSECYPROTECTEDX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x17440ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x10678ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSCTRLPORTDISABLEDX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x1e440ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xfe78ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMHITX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x23240ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x10ed8ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSFLOWIDTCAMMISSX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x22c40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x10e98ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSPARSEERRX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x22e40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x10e78ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCENCRYPTEDX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x20440ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x10c78ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSCPROTECTEDX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x1fc40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x10a78ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECTAGINSERTIONERRX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x23040ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x110d8ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYNOACTIVESAX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x1dc40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x10278ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYTOOLONGX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x1d440ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x10478ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSECYUNTAGGEDX(a) ({	\
+	u64 offset;					\
+							\
+	offset = 0x1cc40ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0x10078ull;			\
+	offset += (a) * 0x8ull;				\
+	offset; })
+
+#define MCSX_CSE_TX_SLAVE_CTRL	({	\
+	u64 offset;					\
+							\
+	offset = 0x54a0ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xa00ull;			\
+	offset; })
+
+#define MCSX_CSE_TX_SLAVE_STATS_CLEAR ({		\
+	u64 offset;					\
+							\
+	offset = 0x54b8ull;				\
+	if (mcs->hw->mcs_blks > 1)			\
+		offset = 0xa18ull;			\
+	offset; })
+
+#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCENCRYPTEDX(a)	(0x1f440ull + (a) * 0x8ull)
+#define MCSX_CSE_TX_MEM_SLAVE_OUTOCTETSSCPROTECTEDX(a)	(0x1ec40ull + (a) * 0x8ull)
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSEARLYPREEMPTERRX(a) (0x10eb8ull + (a) * 0x8ull)
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAENCRYPTEDX(a)	(0x21c40ull + (a) * 0x8ull)
+#define MCSX_CSE_TX_MEM_SLAVE_OUTPKTSSAPROTECTEDX(a)	(0x20c40ull + (a) * 0x8ull)
+
+#define MCSX_IP_INT ({			\
+	u64 offset;			\
+					\
+	offset = 0x80028ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x60028ull;	\
+	offset; })
+
+#define MCSX_IP_INT_ENA_W1S ({		\
+	u64 offset;			\
+					\
+	offset = 0x80040ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x60040ull;	\
+	offset; })
+
+#define MCSX_IP_INT_ENA_W1C ({		\
+	u64 offset;			\
+					\
+	offset = 0x80038ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x60038ull;	\
+	offset; })
+
+#define MCSX_TOP_SLAVE_INT_SUM ({	\
+	u64 offset;			\
+					\
+	offset = 0xc20ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0xab8ull;	\
+	offset; })
+
+#define MCSX_TOP_SLAVE_INT_SUM_ENB ({	\
+	u64 offset;			\
+					\
+	offset = 0xc28ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0xac0ull;	\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_RX_INT ({	\
+	u64 offset;			\
+					\
+	offset = 0x23c00ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x0ad8ull;	\
+	offset; })
+
+#define MCSX_CPM_RX_SLAVE_RX_INT_ENB ({	\
+	u64 offset;			\
+					\
+	offset = 0x23c08ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0xae0ull;	\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_TX_INT ({	\
+	u64 offset;			\
+					\
+	offset = 0x3d490ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x54a0ull;	\
+	offset; })
+
+#define MCSX_CPM_TX_SLAVE_TX_INT_ENB ({	\
+	u64 offset;			\
+					\
+	offset = 0x3d498ull;		\
+	if (mcs->hw->mcs_blks > 1)	\
+		offset = 0x54a8ull;	\
+	offset; })
+
+#endif
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c
new file mode 100644
index 000000000000..fa8029a94068
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/af/mcs_rvu_if.c
@@ -0,0 +1,889 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell CN10K MCS driver
+ *
+ * Copyright (C) 2022 Marvell.
+ */
+
+#include <linux/types.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include "mcs.h"
+#include "rvu.h"
+#include "lmac_common.h"
+
+#define M(_name, _id, _fn_name, _req_type, _rsp_type)			\
+static struct _req_type __maybe_unused					\
+*otx2_mbox_alloc_msg_ ## _fn_name(struct rvu *rvu, int devid)		\
+{									\
+	struct _req_type *req;						\
+									\
+	req = (struct _req_type *)otx2_mbox_alloc_msg_rsp(		\
+		&rvu->afpf_wq_info.mbox_up, devid, sizeof(struct _req_type), \
+		sizeof(struct _rsp_type));				\
+	if (!req)							\
+		return NULL;						\
+	req->hdr.sig = OTX2_MBOX_REQ_SIG;				\
+	req->hdr.id = _id;						\
+	return req;							\
+}
+
+MBOX_UP_MCS_MESSAGES
+#undef M
+
+int rvu_mbox_handler_mcs_set_lmac_mode(struct rvu *rvu,
+				       struct mcs_set_lmac_mode *req,
+				       struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	if (BIT_ULL(req->lmac_id) & mcs->hw->lmac_bmap)
+		mcs_set_lmac_mode(mcs, req->lmac_id, req->mode);
+
+	return 0;
+}
+
+int mcs_add_intr_wq_entry(struct mcs *mcs, struct mcs_intr_event *event)
+{
+	struct mcs_intrq_entry *qentry;
+	u16 pcifunc = event->pcifunc;
+	struct rvu *rvu = mcs->rvu;
+	struct mcs_pfvf *pfvf;
+
+	/* Check if it is PF or VF */
+	if (pcifunc & RVU_PFVF_FUNC_MASK)
+		pfvf = &mcs->vf[rvu_get_hwvf(rvu, pcifunc)];
+	else
+		pfvf = &mcs->pf[rvu_get_pf(pcifunc)];
+
+	event->intr_mask &= pfvf->intr_mask;
+
+	/* Check PF/VF interrupt notification is enabled */
+	if (!(pfvf->intr_mask && event->intr_mask))
+		return 0;
+
+	qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC);
+	if (!qentry)
+		return -ENOMEM;
+
+	qentry->intr_event = *event;
+	spin_lock(&rvu->mcs_intrq_lock);
+	list_add_tail(&qentry->node, &rvu->mcs_intrq_head);
+	spin_unlock(&rvu->mcs_intrq_lock);
+	queue_work(rvu->mcs_intr_wq, &rvu->mcs_intr_work);
+
+	return 0;
+}
+
+static int mcs_notify_pfvf(struct mcs_intr_event *event, struct rvu *rvu)
+{
+	struct mcs_intr_info *req;
+	int err, pf;
+
+	pf = rvu_get_pf(event->pcifunc);
+
+	req = otx2_mbox_alloc_msg_mcs_intr_notify(rvu, pf);
+	if (!req)
+		return -ENOMEM;
+
+	req->mcs_id = event->mcs_id;
+	req->intr_mask = event->intr_mask;
+	req->sa_id = event->sa_id;
+	req->hdr.pcifunc = event->pcifunc;
+	req->lmac_id = event->lmac_id;
+
+	otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, pf);
+	err = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, pf);
+	if (err)
+		dev_warn(rvu->dev, "MCS notification to pf %d failed\n", pf);
+
+	return 0;
+}
+
+static void mcs_intr_handler_task(struct work_struct *work)
+{
+	struct rvu *rvu = container_of(work, struct rvu, mcs_intr_work);
+	struct mcs_intrq_entry *qentry;
+	struct mcs_intr_event *event;
+	unsigned long flags;
+
+	do {
+		spin_lock_irqsave(&rvu->mcs_intrq_lock, flags);
+		qentry = list_first_entry_or_null(&rvu->mcs_intrq_head,
+						  struct mcs_intrq_entry,
+						  node);
+		if (qentry)
+			list_del(&qentry->node);
+
+		spin_unlock_irqrestore(&rvu->mcs_intrq_lock, flags);
+		if (!qentry)
+			break; /* nothing more to process */
+
+		event = &qentry->intr_event;
+
+		mcs_notify_pfvf(event, rvu);
+		kfree(qentry);
+	} while (1);
+}
+
+int rvu_mbox_handler_mcs_intr_cfg(struct rvu *rvu,
+				  struct mcs_intr_cfg *req,
+				  struct msg_rsp *rsp)
+{
+	u16 pcifunc = req->hdr.pcifunc;
+	struct mcs_pfvf *pfvf;
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	/* Check if it is PF or VF */
+	if (pcifunc & RVU_PFVF_FUNC_MASK)
+		pfvf = &mcs->vf[rvu_get_hwvf(rvu, pcifunc)];
+	else
+		pfvf = &mcs->pf[rvu_get_pf(pcifunc)];
+
+	mcs->pf_map[0] = pcifunc;
+	pfvf->intr_mask = req->intr_mask;
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_get_hw_info(struct rvu *rvu,
+				     struct msg_req *req,
+				     struct mcs_hw_info *rsp)
+{
+	struct mcs *mcs;
+
+	if (!rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_NOT_MAPPED;
+
+	/* MCS resources are same across all blocks */
+	mcs = mcs_get_pdata(0);
+	rsp->num_mcs_blks = rvu->mcs_blk_cnt;
+	rsp->tcam_entries = mcs->hw->tcam_entries;
+	rsp->secy_entries = mcs->hw->secy_entries;
+	rsp->sc_entries = mcs->hw->sc_entries;
+	rsp->sa_entries = mcs->hw->sa_entries;
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_port_reset(struct rvu *rvu, struct mcs_port_reset_req *req,
+				    struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	mcs_reset_port(mcs, req->port_id, req->reset);
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_clear_stats(struct rvu *rvu,
+				     struct mcs_clear_stats *req,
+				     struct msg_rsp *rsp)
+{
+	u16 pcifunc = req->hdr.pcifunc;
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	mutex_lock(&mcs->stats_lock);
+	if (req->all)
+		mcs_clear_all_stats(mcs, pcifunc, req->dir);
+	else
+		mcs_clear_stats(mcs, req->type, req->id, req->dir);
+
+	mutex_unlock(&mcs->stats_lock);
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_get_flowid_stats(struct rvu *rvu,
+					  struct mcs_stats_req *req,
+					  struct mcs_flowid_stats *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	/* In CNF10K-B, before reading the statistics,
+	 * MCSX_MIL_GLOBAL.FORCE_CLK_EN_IP needs to be set
+	 * to get accurate statistics
+	 */
+	if (mcs->hw->mcs_blks > 1)
+		mcs_set_force_clk_en(mcs, true);
+
+	mutex_lock(&mcs->stats_lock);
+	mcs_get_flowid_stats(mcs, rsp, req->id, req->dir);
+	mutex_unlock(&mcs->stats_lock);
+
+	/* Clear MCSX_MIL_GLOBAL.FORCE_CLK_EN_IP after reading
+	 * the statistics
+	 */
+	if (mcs->hw->mcs_blks > 1)
+		mcs_set_force_clk_en(mcs, false);
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_get_secy_stats(struct rvu *rvu,
+					struct mcs_stats_req *req,
+					struct mcs_secy_stats *rsp)
+{	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	if (mcs->hw->mcs_blks > 1)
+		mcs_set_force_clk_en(mcs, true);
+
+	mutex_lock(&mcs->stats_lock);
+
+	if (req->dir == MCS_RX)
+		mcs_get_rx_secy_stats(mcs, rsp, req->id);
+	else
+		mcs_get_tx_secy_stats(mcs, rsp, req->id);
+
+	mutex_unlock(&mcs->stats_lock);
+
+	if (mcs->hw->mcs_blks > 1)
+		mcs_set_force_clk_en(mcs, false);
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_get_sc_stats(struct rvu *rvu,
+				      struct mcs_stats_req *req,
+				      struct mcs_sc_stats *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	if (mcs->hw->mcs_blks > 1)
+		mcs_set_force_clk_en(mcs, true);
+
+	mutex_lock(&mcs->stats_lock);
+	mcs_get_sc_stats(mcs, rsp, req->id, req->dir);
+	mutex_unlock(&mcs->stats_lock);
+
+	if (mcs->hw->mcs_blks > 1)
+		mcs_set_force_clk_en(mcs, false);
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_get_sa_stats(struct rvu *rvu,
+				      struct mcs_stats_req *req,
+				      struct mcs_sa_stats *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	if (mcs->hw->mcs_blks > 1)
+		mcs_set_force_clk_en(mcs, true);
+
+	mutex_lock(&mcs->stats_lock);
+	mcs_get_sa_stats(mcs, rsp, req->id, req->dir);
+	mutex_unlock(&mcs->stats_lock);
+
+	if (mcs->hw->mcs_blks > 1)
+		mcs_set_force_clk_en(mcs, false);
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_get_port_stats(struct rvu *rvu,
+					struct mcs_stats_req *req,
+					struct mcs_port_stats *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	if (mcs->hw->mcs_blks > 1)
+		mcs_set_force_clk_en(mcs, true);
+
+	mutex_lock(&mcs->stats_lock);
+	mcs_get_port_stats(mcs, rsp, req->id, req->dir);
+	mutex_unlock(&mcs->stats_lock);
+
+	if (mcs->hw->mcs_blks > 1)
+		mcs_set_force_clk_en(mcs, false);
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_set_active_lmac(struct rvu *rvu,
+					 struct mcs_set_active_lmac *req,
+					 struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+	if (!mcs)
+		return MCS_AF_ERR_NOT_MAPPED;
+
+	mcs->hw->lmac_bmap = req->lmac_bmap;
+	mcs_set_lmac_channels(req->mcs_id, req->chan_base);
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_port_cfg_set(struct rvu *rvu, struct mcs_port_cfg_set_req *req,
+				      struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	if (mcs->hw->lmac_cnt <= req->port_id || !(mcs->hw->lmac_bmap & BIT_ULL(req->port_id)))
+		return -EINVAL;
+
+	mcs_set_port_cfg(mcs, req);
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_port_cfg_get(struct rvu *rvu, struct mcs_port_cfg_get_req *req,
+				      struct mcs_port_cfg_get_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	if (mcs->hw->lmac_cnt <= req->port_id || !(mcs->hw->lmac_bmap & BIT_ULL(req->port_id)))
+		return -EINVAL;
+
+	mcs_get_port_cfg(mcs, req, rsp);
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_custom_tag_cfg_get(struct rvu *rvu, struct mcs_custom_tag_cfg_get_req *req,
+					    struct mcs_custom_tag_cfg_get_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	mcs_get_custom_tag_cfg(mcs, req, rsp);
+
+	return 0;
+}
+
+int rvu_mcs_flr_handler(struct rvu *rvu, u16 pcifunc)
+{
+	struct mcs *mcs;
+	int mcs_id;
+
+	/* CNF10K-B mcs0-6 are mapped to RPM2-8*/
+	if (rvu->mcs_blk_cnt > 1) {
+		for (mcs_id = 0; mcs_id < rvu->mcs_blk_cnt; mcs_id++) {
+			mcs = mcs_get_pdata(mcs_id);
+			mcs_free_all_rsrc(mcs, MCS_RX, pcifunc);
+			mcs_free_all_rsrc(mcs, MCS_TX, pcifunc);
+		}
+	} else {
+		/* CN10K-B has only one mcs block */
+		mcs = mcs_get_pdata(0);
+		mcs_free_all_rsrc(mcs, MCS_RX, pcifunc);
+		mcs_free_all_rsrc(mcs, MCS_TX, pcifunc);
+	}
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_flowid_ena_entry(struct rvu *rvu,
+					  struct mcs_flowid_ena_dis_entry *req,
+					  struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+	mcs_ena_dis_flowid_entry(mcs, req->flow_id, req->dir, req->ena);
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_pn_table_write(struct rvu *rvu,
+					struct mcs_pn_table_write_req *req,
+					struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+	mcs_pn_table_write(mcs, req->pn_id, req->next_pn, req->dir);
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_set_pn_threshold(struct rvu *rvu,
+					  struct mcs_set_pn_threshold *req,
+					  struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	mcs_pn_threshold_set(mcs, req);
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_rx_sc_sa_map_write(struct rvu *rvu,
+					    struct mcs_rx_sc_sa_map *req,
+					    struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+	mcs->mcs_ops->mcs_rx_sa_mem_map_write(mcs, req);
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_tx_sc_sa_map_write(struct rvu *rvu,
+					    struct mcs_tx_sc_sa_map *req,
+					    struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+	mcs->mcs_ops->mcs_tx_sa_mem_map_write(mcs, req);
+	mcs->tx_sa_active[req->sc_id] = req->tx_sa_active;
+
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_sa_plcy_write(struct rvu *rvu,
+				       struct mcs_sa_plcy_write_req *req,
+				       struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+	int i;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	for (i = 0; i < req->sa_cnt; i++)
+		mcs_sa_plcy_write(mcs, &req->plcy[i][0],
+				  req->sa_index[i], req->dir);
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_rx_sc_cam_write(struct rvu *rvu,
+					 struct mcs_rx_sc_cam_write_req *req,
+					 struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+	mcs_rx_sc_cam_write(mcs, req->sci, req->secy_id, req->sc_id);
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_secy_plcy_write(struct rvu *rvu,
+					 struct mcs_secy_plcy_write_req *req,
+					 struct msg_rsp *rsp)
+{	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	mcs_secy_plcy_write(mcs, req->plcy,
+			    req->secy_id, req->dir);
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_flowid_entry_write(struct rvu *rvu,
+					    struct mcs_flowid_entry_write_req *req,
+					    struct msg_rsp *rsp)
+{
+	struct secy_mem_map map;
+	struct mcs *mcs;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	/* TODO validate the flowid */
+	mcs_flowid_entry_write(mcs, req->data, req->mask,
+			       req->flow_id, req->dir);
+	map.secy = req->secy_id;
+	map.sc = req->sc_id;
+	map.ctrl_pkt = req->ctrl_pkt;
+	map.flow_id = req->flow_id;
+	map.sci = req->sci;
+	mcs->mcs_ops->mcs_flowid_secy_map(mcs, &map, req->dir);
+	if (req->ena)
+		mcs_ena_dis_flowid_entry(mcs, req->flow_id,
+					 req->dir, true);
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_free_resources(struct rvu *rvu,
+					struct mcs_free_rsrc_req *req,
+					struct msg_rsp *rsp)
+{
+	u16 pcifunc = req->hdr.pcifunc;
+	struct mcs_rsrc_map *map;
+	struct mcs *mcs;
+	int rc;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	if (req->dir == MCS_RX)
+		map = &mcs->rx;
+	else
+		map = &mcs->tx;
+
+	mutex_lock(&rvu->rsrc_lock);
+	/* Free all the cam resources mapped to PF/VF */
+	if (req->all) {
+		rc = mcs_free_all_rsrc(mcs, req->dir, pcifunc);
+		goto exit;
+	}
+
+	switch (req->rsrc_type) {
+	case MCS_RSRC_TYPE_FLOWID:
+		rc = mcs_free_rsrc(&map->flow_ids, map->flowid2pf_map, req->rsrc_id, pcifunc);
+		mcs_ena_dis_flowid_entry(mcs, req->rsrc_id, req->dir, false);
+		break;
+	case MCS_RSRC_TYPE_SECY:
+		rc =  mcs_free_rsrc(&map->secy, map->secy2pf_map, req->rsrc_id, pcifunc);
+		mcs_clear_secy_plcy(mcs, req->rsrc_id, req->dir);
+		break;
+	case MCS_RSRC_TYPE_SC:
+		rc = mcs_free_rsrc(&map->sc, map->sc2pf_map, req->rsrc_id, pcifunc);
+		/* Disable SC CAM only on RX side */
+		if (req->dir == MCS_RX)
+			mcs_ena_dis_sc_cam_entry(mcs, req->rsrc_id, false);
+		break;
+	case MCS_RSRC_TYPE_SA:
+		rc = mcs_free_rsrc(&map->sa, map->sa2pf_map, req->rsrc_id, pcifunc);
+		break;
+	}
+exit:
+	mutex_unlock(&rvu->rsrc_lock);
+	return rc;
+}
+
+int rvu_mbox_handler_mcs_alloc_resources(struct rvu *rvu,
+					 struct mcs_alloc_rsrc_req *req,
+					 struct mcs_alloc_rsrc_rsp *rsp)
+{
+	u16 pcifunc = req->hdr.pcifunc;
+	struct mcs_rsrc_map *map;
+	struct mcs *mcs;
+	int rsrc_id, i;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	if (req->dir == MCS_RX)
+		map = &mcs->rx;
+	else
+		map = &mcs->tx;
+
+	mutex_lock(&rvu->rsrc_lock);
+
+	if (req->all) {
+		rsrc_id = mcs_alloc_all_rsrc(mcs, &rsp->flow_ids[0],
+					     &rsp->secy_ids[0],
+					     &rsp->sc_ids[0],
+					     &rsp->sa_ids[0],
+					     &rsp->sa_ids[1],
+					     pcifunc, req->dir);
+		goto exit;
+	}
+
+	switch (req->rsrc_type) {
+	case MCS_RSRC_TYPE_FLOWID:
+		for (i = 0; i < req->rsrc_cnt; i++) {
+			rsrc_id = mcs_alloc_rsrc(&map->flow_ids, map->flowid2pf_map, pcifunc);
+			if (rsrc_id < 0)
+				goto exit;
+			rsp->flow_ids[i] = rsrc_id;
+			rsp->rsrc_cnt++;
+		}
+		break;
+	case MCS_RSRC_TYPE_SECY:
+		for (i = 0; i < req->rsrc_cnt; i++) {
+			rsrc_id = mcs_alloc_rsrc(&map->secy, map->secy2pf_map, pcifunc);
+			if (rsrc_id < 0)
+				goto exit;
+			rsp->secy_ids[i] = rsrc_id;
+			rsp->rsrc_cnt++;
+		}
+		break;
+	case MCS_RSRC_TYPE_SC:
+		for (i = 0; i < req->rsrc_cnt; i++) {
+			rsrc_id = mcs_alloc_rsrc(&map->sc, map->sc2pf_map, pcifunc);
+			if (rsrc_id < 0)
+				goto exit;
+			rsp->sc_ids[i] = rsrc_id;
+			rsp->rsrc_cnt++;
+		}
+		break;
+	case MCS_RSRC_TYPE_SA:
+		for (i = 0; i < req->rsrc_cnt; i++) {
+			rsrc_id = mcs_alloc_rsrc(&map->sa, map->sa2pf_map, pcifunc);
+			if (rsrc_id < 0)
+				goto exit;
+			rsp->sa_ids[i] = rsrc_id;
+			rsp->rsrc_cnt++;
+		}
+		break;
+	}
+
+	rsp->rsrc_type = req->rsrc_type;
+	rsp->dir = req->dir;
+	rsp->mcs_id = req->mcs_id;
+	rsp->all = req->all;
+
+exit:
+	if (rsrc_id < 0)
+		dev_err(rvu->dev, "Failed to allocate the mcs resources for PCIFUNC:%d\n", pcifunc);
+	mutex_unlock(&rvu->rsrc_lock);
+	return 0;
+}
+
+int rvu_mbox_handler_mcs_alloc_ctrl_pkt_rule(struct rvu *rvu,
+					     struct mcs_alloc_ctrl_pkt_rule_req *req,
+					     struct mcs_alloc_ctrl_pkt_rule_rsp *rsp)
+{
+	u16 pcifunc = req->hdr.pcifunc;
+	struct mcs_rsrc_map *map;
+	struct mcs *mcs;
+	int rsrc_id;
+	u16 offset;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	map = (req->dir == MCS_RX) ? &mcs->rx : &mcs->tx;
+
+	mutex_lock(&rvu->rsrc_lock);
+
+	switch (req->rule_type) {
+	case MCS_CTRL_PKT_RULE_TYPE_ETH:
+		offset = MCS_CTRLPKT_ETYPE_RULE_OFFSET;
+		break;
+	case MCS_CTRL_PKT_RULE_TYPE_DA:
+		offset = MCS_CTRLPKT_DA_RULE_OFFSET;
+		break;
+	case MCS_CTRL_PKT_RULE_TYPE_RANGE:
+		offset = MCS_CTRLPKT_DA_RANGE_RULE_OFFSET;
+		break;
+	case MCS_CTRL_PKT_RULE_TYPE_COMBO:
+		offset = MCS_CTRLPKT_COMBO_RULE_OFFSET;
+		break;
+	case MCS_CTRL_PKT_RULE_TYPE_MAC:
+		offset = MCS_CTRLPKT_MAC_EN_RULE_OFFSET;
+		break;
+	}
+
+	rsrc_id = mcs_alloc_ctrlpktrule(&map->ctrlpktrule, map->ctrlpktrule2pf_map, offset,
+					pcifunc);
+	if (rsrc_id < 0)
+		goto exit;
+
+	rsp->rule_idx = rsrc_id;
+	rsp->rule_type = req->rule_type;
+	rsp->dir = req->dir;
+	rsp->mcs_id = req->mcs_id;
+
+	mutex_unlock(&rvu->rsrc_lock);
+	return 0;
+exit:
+	if (rsrc_id < 0)
+		dev_err(rvu->dev, "Failed to allocate the mcs ctrl pkt rule for PCIFUNC:%d\n",
+			pcifunc);
+	mutex_unlock(&rvu->rsrc_lock);
+	return rsrc_id;
+}
+
+int rvu_mbox_handler_mcs_free_ctrl_pkt_rule(struct rvu *rvu,
+					    struct mcs_free_ctrl_pkt_rule_req *req,
+					    struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+	int rc;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	mutex_lock(&rvu->rsrc_lock);
+
+	rc = mcs_free_ctrlpktrule(mcs, req);
+
+	mutex_unlock(&rvu->rsrc_lock);
+
+	return rc;
+}
+
+int rvu_mbox_handler_mcs_ctrl_pkt_rule_write(struct rvu *rvu,
+					     struct mcs_ctrl_pkt_rule_write_req *req,
+					     struct msg_rsp *rsp)
+{
+	struct mcs *mcs;
+	int rc;
+
+	if (req->mcs_id >= rvu->mcs_blk_cnt)
+		return MCS_AF_ERR_INVALID_MCSID;
+
+	mcs = mcs_get_pdata(req->mcs_id);
+
+	rc = mcs_ctrlpktrule_write(mcs, req);
+
+	return rc;
+}
+
+static void rvu_mcs_set_lmac_bmap(struct rvu *rvu)
+{
+	struct mcs *mcs = mcs_get_pdata(0);
+	unsigned long lmac_bmap;
+	int cgx, lmac, port;
+
+	for (port = 0; port < mcs->hw->lmac_cnt; port++) {
+		cgx = port / rvu->hw->lmac_per_cgx;
+		lmac = port % rvu->hw->lmac_per_cgx;
+		if (!is_lmac_valid(rvu_cgx_pdata(cgx, rvu), lmac))
+			continue;
+		set_bit(port, &lmac_bmap);
+	}
+	mcs->hw->lmac_bmap = lmac_bmap;
+}
+
+int rvu_mcs_init(struct rvu *rvu)
+{
+	struct rvu_hwinfo *hw = rvu->hw;
+	int lmac, err = 0, mcs_id;
+	struct mcs *mcs;
+
+	rvu->mcs_blk_cnt = mcs_get_blkcnt();
+
+	if (!rvu->mcs_blk_cnt)
+		return 0;
+
+	/* Needed only for CN10K-B */
+	if (rvu->mcs_blk_cnt == 1) {
+		err = mcs_set_lmac_channels(0, hw->cgx_chan_base);
+		if (err)
+			return err;
+		/* Set active lmacs */
+		rvu_mcs_set_lmac_bmap(rvu);
+	}
+
+	/* Install default tcam bypass entry and set port to operational mode */
+	for (mcs_id = 0; mcs_id < rvu->mcs_blk_cnt; mcs_id++) {
+		mcs = mcs_get_pdata(mcs_id);
+		mcs_install_flowid_bypass_entry(mcs);
+		for (lmac = 0; lmac < mcs->hw->lmac_cnt; lmac++)
+			mcs_set_lmac_mode(mcs, lmac, 0);
+
+		mcs->rvu = rvu;
+
+		/* Allocated memory for PFVF data */
+		mcs->pf = devm_kcalloc(mcs->dev, hw->total_pfs,
+				       sizeof(struct mcs_pfvf), GFP_KERNEL);
+		if (!mcs->pf)
+			return -ENOMEM;
+
+		mcs->vf = devm_kcalloc(mcs->dev, hw->total_vfs,
+				       sizeof(struct mcs_pfvf), GFP_KERNEL);
+		if (!mcs->vf)
+			return -ENOMEM;
+	}
+
+	/* Initialize the wq for handling mcs interrupts */
+	INIT_LIST_HEAD(&rvu->mcs_intrq_head);
+	INIT_WORK(&rvu->mcs_intr_work, mcs_intr_handler_task);
+	rvu->mcs_intr_wq = alloc_workqueue("mcs_intr_wq", 0, 0);
+	if (!rvu->mcs_intr_wq) {
+		dev_err(rvu->dev, "mcs alloc workqueue failed\n");
+		return -ENOMEM;
+	}
+
+	return err;
+}
+
+void rvu_mcs_exit(struct rvu *rvu)
+{
+	if (!rvu->mcs_intr_wq)
+		return;
+
+	flush_workqueue(rvu->mcs_intr_wq);
+	destroy_workqueue(rvu->mcs_intr_wq);
+	rvu->mcs_intr_wq = NULL;
+}
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c
index 67a6821d2dff..3411e2e47d46 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/ptp.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/ptp.c
@@ -9,6 +9,8 @@
 #include <linux/device.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/hrtimer.h>
+#include <linux/ktime.h>
 
 #include "ptp.h"
 #include "mbox.h"
@@ -50,12 +52,23 @@
 #define PTP_CLOCK_COMP				0xF18ULL
 #define PTP_TIMESTAMP				0xF20ULL
 #define PTP_CLOCK_SEC				0xFD0ULL
+#define PTP_SEC_ROLLOVER			0xFD8ULL
 
 #define CYCLE_MULT				1000
 
 static struct ptp *first_ptp_block;
 static const struct pci_device_id ptp_id_table[];
 
+static bool is_ptp_dev_cnf10kb(struct ptp *ptp)
+{
+	return (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CNF10K_B_PTP) ? true : false;
+}
+
+static bool is_ptp_dev_cn10k(struct ptp *ptp)
+{
+	return (ptp->pdev->device == PCI_DEVID_CN10K_PTP) ? true : false;
+}
+
 static bool cn10k_ptp_errata(struct ptp *ptp)
 {
 	if (ptp->pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_A_PTP ||
@@ -72,6 +85,43 @@ static bool is_ptp_tsfmt_sec_nsec(struct ptp *ptp)
 	return false;
 }
 
+static enum hrtimer_restart ptp_reset_thresh(struct hrtimer *hrtimer)
+{
+	struct ptp *ptp = container_of(hrtimer, struct ptp, hrtimer);
+	ktime_t curr_ts = ktime_get();
+	ktime_t delta_ns, period_ns;
+	u64 ptp_clock_hi;
+
+	/* calculate the elapsed time since last restart */
+	delta_ns = ktime_to_ns(ktime_sub(curr_ts, ptp->last_ts));
+
+	/* if the ptp clock value has crossed 0.5 seconds,
+	 * its too late to update pps threshold value, so
+	 * update threshold after 1 second.
+	 */
+	ptp_clock_hi = readq(ptp->reg_base + PTP_CLOCK_HI);
+	if (ptp_clock_hi > 500000000) {
+		period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - ptp_clock_hi));
+	} else {
+		writeq(500000000, ptp->reg_base + PTP_PPS_THRESH_HI);
+		period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - delta_ns));
+	}
+
+	hrtimer_forward_now(hrtimer, period_ns);
+	ptp->last_ts = curr_ts;
+
+	return HRTIMER_RESTART;
+}
+
+static void ptp_hrtimer_start(struct ptp *ptp, ktime_t start_ns)
+{
+	ktime_t period_ns;
+
+	period_ns = ktime_set(0, (NSEC_PER_SEC + 100 - start_ns));
+	hrtimer_start(&ptp->hrtimer, period_ns, HRTIMER_MODE_REL);
+	ptp->last_ts = ktime_get();
+}
+
 static u64 read_ptp_tstmp_sec_nsec(struct ptp *ptp)
 {
 	u64 sec, sec1, nsec;
@@ -246,6 +296,10 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts)
 	/* sclk is in MHz */
 	ptp->clock_rate = sclk * 1000000;
 
+	/* Program the seconds rollover value to 1 second */
+	if (is_ptp_dev_cnf10kb(ptp))
+		writeq(0x3b9aca00, ptp->reg_base + PTP_SEC_ROLLOVER);
+
 	/* Enable PTP clock */
 	clock_cfg = readq(ptp->reg_base + PTP_CLOCK_CFG);
 
@@ -270,6 +324,18 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts)
 	/* Set 50% duty cycle for 1Hz output */
 	writeq(0x1dcd650000000000, ptp->reg_base + PTP_PPS_HI_INCR);
 	writeq(0x1dcd650000000000, ptp->reg_base + PTP_PPS_LO_INCR);
+	if (cn10k_ptp_errata(ptp)) {
+		/* The ptp_clock_hi rollsover to zero once clock cycle before it
+		 * reaches one second boundary. so, program the pps_lo_incr in
+		 * such a way that the pps threshold value comparison at one
+		 * second boundary will succeed and pps edge changes. After each
+		 * one second boundary, the hrtimer handler will be invoked and
+		 * reprograms the pps threshold value.
+		 */
+		ptp->clock_period = NSEC_PER_SEC / ptp->clock_rate;
+		writeq((0x1dcd6500ULL - ptp->clock_period) << 32,
+		       ptp->reg_base + PTP_PPS_LO_INCR);
+	}
 
 	if (cn10k_ptp_errata(ptp))
 		clock_comp = ptp_calc_adjusted_comp(ptp->clock_rate);
@@ -282,14 +348,39 @@ void ptp_start(struct ptp *ptp, u64 sclk, u32 ext_clk_freq, u32 extts)
 
 static int ptp_get_tstmp(struct ptp *ptp, u64 *clk)
 {
-	*clk = readq(ptp->reg_base + PTP_TIMESTAMP);
+	u64 timestamp;
+
+	if (is_ptp_dev_cn10k(ptp)) {
+		timestamp = readq(ptp->reg_base + PTP_TIMESTAMP);
+		*clk = (timestamp >> 32) * NSEC_PER_SEC + (timestamp & 0xFFFFFFFF);
+	} else {
+		*clk = readq(ptp->reg_base + PTP_TIMESTAMP);
+	}
 
 	return 0;
 }
 
 static int ptp_set_thresh(struct ptp *ptp, u64 thresh)
 {
-	writeq(thresh, ptp->reg_base + PTP_PPS_THRESH_HI);
+	if (!cn10k_ptp_errata(ptp))
+		writeq(thresh, ptp->reg_base + PTP_PPS_THRESH_HI);
+
+	return 0;
+}
+
+static int ptp_extts_on(struct ptp *ptp, int on)
+{
+	u64 ptp_clock_hi;
+
+	if (cn10k_ptp_errata(ptp)) {
+		if (on) {
+			ptp_clock_hi = readq(ptp->reg_base + PTP_CLOCK_HI);
+			ptp_hrtimer_start(ptp, (ktime_t)ptp_clock_hi);
+		} else {
+			if (hrtimer_active(&ptp->hrtimer))
+				hrtimer_cancel(&ptp->hrtimer);
+		}
+	}
 
 	return 0;
 }
@@ -329,6 +420,11 @@ static int ptp_probe(struct pci_dev *pdev,
 	else
 		ptp->read_ptp_tstmp = &read_ptp_tstmp_nsec;
 
+	if (cn10k_ptp_errata(ptp)) {
+		hrtimer_init(&ptp->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+		ptp->hrtimer.function = ptp_reset_thresh;
+	}
+
 	return 0;
 
 error_free:
@@ -353,6 +449,9 @@ static void ptp_remove(struct pci_dev *pdev)
 	struct ptp *ptp = pci_get_drvdata(pdev);
 	u64 clock_cfg;
 
+	if (cn10k_ptp_errata(ptp) && hrtimer_active(&ptp->hrtimer))
+		hrtimer_cancel(&ptp->hrtimer);
+
 	if (IS_ERR_OR_NULL(ptp))
 		return;
 
@@ -420,6 +519,9 @@ int rvu_mbox_handler_ptp_op(struct rvu *rvu, struct ptp_req *req,
 	case PTP_OP_SET_THRESH:
 		err = ptp_set_thresh(rvu->ptp, req->thresh);
 		break;
+	case PTP_OP_EXTTS_ON:
+		err = ptp_extts_on(rvu->ptp, req->extts_on);
+		break;
 	default:
 		err = -EINVAL;
 		break;
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/ptp.h b/drivers/net/ethernet/marvell/octeontx2/af/ptp.h
index 95a955159f40..b9d92abc3844 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/ptp.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/ptp.h
@@ -17,7 +17,10 @@ struct ptp {
 	void __iomem *reg_base;
 	u64 (*read_ptp_tstmp)(struct ptp *ptp);
 	spinlock_t ptp_lock; /* lock */
+	struct hrtimer hrtimer;
+	ktime_t last_ts;
 	u32 clock_rate;
+	u32 clock_period;
 };
 
 struct ptp *ptp_get(void);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c
index ef59de43b11e..a70e1153fa04 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.c
@@ -415,11 +415,26 @@ void rpm_lmac_ptp_config(void *rpmd, int lmac_id, bool enable)
 		return;
 
 	cfg = rpm_read(rpm, lmac_id, RPMX_CMRX_CFG);
-	if (enable)
+	if (enable) {
 		cfg |= RPMX_RX_TS_PREPEND;
-	else
+		cfg |= RPMX_TX_PTP_1S_SUPPORT;
+	} else {
 		cfg &= ~RPMX_RX_TS_PREPEND;
+		cfg &= ~RPMX_TX_PTP_1S_SUPPORT;
+	}
+
 	rpm_write(rpm, lmac_id, RPMX_CMRX_CFG, cfg);
+
+	cfg = rpm_read(rpm, lmac_id, RPMX_MTI_MAC100X_XIF_MODE);
+
+	if (enable) {
+		cfg |= RPMX_ONESTEP_ENABLE;
+		cfg &= ~RPMX_TS_BINARY_MODE;
+	} else {
+		cfg &= ~RPMX_ONESTEP_ENABLE;
+	}
+
+	rpm_write(rpm, lmac_id, RPMX_MTI_MAC100X_XIF_MODE, cfg);
 }
 
 int rpm_lmac_pfc_config(void *rpmd, int lmac_id, u8 tx_pause, u8 rx_pause, u16 pfc_en)
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h
index c2bd6e54ea51..77f2ef9e1425 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rpm.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rpm.h
@@ -16,6 +16,7 @@
 /* Registers */
 #define RPMX_CMRX_CFG			0x00
 #define RPMX_RX_TS_PREPEND              BIT_ULL(22)
+#define RPMX_TX_PTP_1S_SUPPORT          BIT_ULL(17)
 #define RPMX_CMRX_SW_INT                0x180
 #define RPMX_CMRX_SW_INT_W1S            0x188
 #define RPMX_CMRX_SW_INT_ENA_W1S        0x198
@@ -72,6 +73,10 @@
 #define RPMX_MTI_MAC100X_CL89_PAUSE_QUANTA		0x8108
 #define RPM_DEFAULT_PAUSE_TIME                          0x7FF
 
+#define RPMX_MTI_MAC100X_XIF_MODE		        0x8100
+#define RPMX_ONESTEP_ENABLE				BIT_ULL(5)
+#define RPMX_TS_BINARY_MODE				BIT_ULL(11)
+
 /* Function Declarations */
 int rpm_get_nr_lmacs(void *rpmd);
 u8 rpm_get_lmac_type(void *rpmd, int lmac_id);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
index 7282a826d81e..3f5e09b77d4b 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c
@@ -16,6 +16,7 @@
 #include "rvu.h"
 #include "rvu_reg.h"
 #include "ptp.h"
+#include "mcs.h"
 
 #include "rvu_trace.h"
 #include "rvu_npc_hash.h"
@@ -23,8 +24,6 @@
 #define DRV_NAME	"rvu_af"
 #define DRV_STRING      "Marvell OcteonTX2 RVU Admin Function Driver"
 
-static int rvu_get_hwvf(struct rvu *rvu, int pcifunc);
-
 static void rvu_set_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
 				struct rvu_block *block, int lf);
 static void rvu_clear_msix_offset(struct rvu *rvu, struct rvu_pfvf *pfvf,
@@ -418,7 +417,7 @@ void rvu_get_pf_numvfs(struct rvu *rvu, int pf, int *numvfs, int *hwvf)
 		*hwvf = cfg & 0xFFF;
 }
 
-static int rvu_get_hwvf(struct rvu *rvu, int pcifunc)
+int rvu_get_hwvf(struct rvu *rvu, int pcifunc)
 {
 	int pf, func;
 	u64 cfg;
@@ -1159,6 +1158,12 @@ cpt:
 
 	rvu_program_channels(rvu);
 
+	err = rvu_mcs_init(rvu);
+	if (err) {
+		dev_err(rvu->dev, "%s: Failed to initialize mcs\n", __func__);
+		goto nix_err;
+	}
+
 	return 0;
 
 nix_err:
@@ -3293,6 +3298,7 @@ err_mbox:
 err_hwsetup:
 	rvu_cgx_exit(rvu);
 	rvu_fwdata_exit(rvu);
+	rvu_mcs_exit(rvu);
 	rvu_reset_all_blocks(rvu);
 	rvu_free_hw_resources(rvu);
 	rvu_clear_rvum_blk_revid(rvu);
@@ -3319,6 +3325,7 @@ static void rvu_remove(struct pci_dev *pdev)
 	rvu_flr_wq_destroy(rvu);
 	rvu_cgx_exit(rvu);
 	rvu_fwdata_exit(rvu);
+	rvu_mcs_exit(rvu);
 	rvu_mbox_destroy(&rvu->afpf_wq_info);
 	rvu_disable_sriov(rvu);
 	rvu_reset_all_blocks(rvu);
@@ -3354,12 +3361,18 @@ static int __init rvu_init_module(void)
 	if (err < 0)
 		goto ptp_err;
 
+	err = pci_register_driver(&mcs_driver);
+	if (err < 0)
+		goto mcs_err;
+
 	err =  pci_register_driver(&rvu_driver);
 	if (err < 0)
 		goto rvu_err;
 
 	return 0;
 rvu_err:
+	pci_unregister_driver(&mcs_driver);
+mcs_err:
 	pci_unregister_driver(&ptp_driver);
 ptp_err:
 	pci_unregister_driver(&cgx_driver);
@@ -3370,6 +3383,7 @@ ptp_err:
 static void __exit rvu_cleanup_module(void)
 {
 	pci_unregister_driver(&rvu_driver);
+	pci_unregister_driver(&mcs_driver);
 	pci_unregister_driver(&ptp_driver);
 	pci_unregister_driver(&cgx_driver);
 }
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
index d15bc443335d..76474385a602 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h
@@ -25,6 +25,8 @@
 /* Subsystem Device ID */
 #define PCI_SUBSYS_DEVID_96XX                  0xB200
 #define PCI_SUBSYS_DEVID_CN10K_A	       0xB900
+#define PCI_SUBSYS_DEVID_CNF10K_B              0xBC00
+#define PCI_SUBSYS_DEVID_CN10K_B               0xBD00
 
 /* PCI BAR nos */
 #define	PCI_AF_REG_BAR_NUM			0
@@ -62,6 +64,10 @@ struct rvu_debugfs {
 	struct dentry *nix;
 	struct dentry *npc;
 	struct dentry *cpt;
+	struct dentry *mcs_root;
+	struct dentry *mcs;
+	struct dentry *mcs_rx;
+	struct dentry *mcs_tx;
 	struct dump_ctx npa_aura_ctx;
 	struct dump_ctx npa_pool_ctx;
 	struct dump_ctx nix_cq_ctx;
@@ -497,6 +503,8 @@ struct rvu {
 
 	struct ptp		*ptp;
 
+	int			mcs_blk_cnt;
+
 #ifdef CONFIG_DEBUG_FS
 	struct rvu_debugfs	rvu_dbg;
 #endif
@@ -504,6 +512,12 @@ struct rvu {
 
 	/* RVU switch implementation over NPC with DMAC rules */
 	struct rvu_switch	rswitch;
+
+	struct			work_struct mcs_intr_work;
+	struct			workqueue_struct *mcs_intr_wq;
+	struct list_head	mcs_intrq_head;
+	/* mcs interrupt queue lock */
+	spinlock_t		mcs_intrq_lock;
 };
 
 static inline void rvu_write64(struct rvu *rvu, u64 block, u64 offset, u64 val)
@@ -868,4 +882,11 @@ void rvu_switch_update_rules(struct rvu *rvu, u16 pcifunc);
 int rvu_npc_set_parse_mode(struct rvu *rvu, u16 pcifunc, u64 mode, u8 dir,
 			   u64 pkind, u8 var_len_off, u8 var_len_off_mask,
 			   u8 shift_dir);
+int rvu_get_hwvf(struct rvu *rvu, int pcifunc);
+
+/* CN10K MCS */
+int rvu_mcs_init(struct rvu *rvu);
+int rvu_mcs_flr_handler(struct rvu *rvu, u16 pcifunc);
+void rvu_mcs_exit(struct rvu *rvu);
+
 #endif /* RVU_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
index f42a09f04b25..a1970ebedf95 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c
@@ -19,6 +19,7 @@
 #include "lmac_common.h"
 #include "npc.h"
 #include "rvu_npc_hash.h"
+#include "mcs.h"
 
 #define DEBUGFS_DIR_NAME "octeontx2"
 
@@ -227,6 +228,350 @@ static const struct file_operations rvu_dbg_##name##_fops = { \
 
 static void print_nix_qsize(struct seq_file *filp, struct rvu_pfvf *pfvf);
 
+static int rvu_dbg_mcs_port_stats_display(struct seq_file *filp, void *unused, int dir)
+{
+	struct mcs *mcs = filp->private;
+	struct mcs_port_stats stats;
+	int lmac;
+
+	seq_puts(filp, "\n port stats\n");
+	mutex_lock(&mcs->stats_lock);
+	for_each_set_bit(lmac, &mcs->hw->lmac_bmap, mcs->hw->lmac_cnt) {
+		mcs_get_port_stats(mcs, &stats, lmac, dir);
+		seq_printf(filp, "port%d: Tcam Miss: %lld\n", lmac, stats.tcam_miss_cnt);
+		seq_printf(filp, "port%d: Parser errors: %lld\n", lmac, stats.parser_err_cnt);
+
+		if (dir == MCS_RX && mcs->hw->mcs_blks > 1)
+			seq_printf(filp, "port%d: Preempt error: %lld\n", lmac,
+				   stats.preempt_err_cnt);
+		if (dir == MCS_TX)
+			seq_printf(filp, "port%d: Sectag insert error: %lld\n", lmac,
+				   stats.sectag_insert_err_cnt);
+	}
+	mutex_unlock(&mcs->stats_lock);
+	return 0;
+}
+
+static int rvu_dbg_mcs_rx_port_stats_display(struct seq_file *filp, void *unused)
+{
+	return rvu_dbg_mcs_port_stats_display(filp, unused, MCS_RX);
+}
+
+RVU_DEBUG_SEQ_FOPS(mcs_rx_port_stats, mcs_rx_port_stats_display, NULL);
+
+static int rvu_dbg_mcs_tx_port_stats_display(struct seq_file *filp, void *unused)
+{
+	return rvu_dbg_mcs_port_stats_display(filp, unused, MCS_TX);
+}
+
+RVU_DEBUG_SEQ_FOPS(mcs_tx_port_stats, mcs_tx_port_stats_display, NULL);
+
+static int rvu_dbg_mcs_sa_stats_display(struct seq_file *filp, void *unused, int dir)
+{
+	struct mcs *mcs = filp->private;
+	struct mcs_sa_stats stats;
+	struct rsrc_bmap *map;
+	int sa_id;
+
+	if (dir == MCS_TX) {
+		map = &mcs->tx.sa;
+		mutex_lock(&mcs->stats_lock);
+		for_each_set_bit(sa_id, map->bmap, mcs->hw->sa_entries) {
+			seq_puts(filp, "\n TX SA stats\n");
+			mcs_get_sa_stats(mcs, &stats, sa_id, MCS_TX);
+			seq_printf(filp, "sa%d: Pkts encrypted: %lld\n", sa_id,
+				   stats.pkt_encrypt_cnt);
+
+			seq_printf(filp, "sa%d: Pkts protected: %lld\n", sa_id,
+				   stats.pkt_protected_cnt);
+		}
+		mutex_unlock(&mcs->stats_lock);
+		return 0;
+	}
+
+	/* RX stats */
+	map = &mcs->rx.sa;
+	mutex_lock(&mcs->stats_lock);
+	for_each_set_bit(sa_id, map->bmap, mcs->hw->sa_entries) {
+		seq_puts(filp, "\n RX SA stats\n");
+		mcs_get_sa_stats(mcs, &stats, sa_id, MCS_RX);
+		seq_printf(filp, "sa%d: Invalid pkts: %lld\n", sa_id, stats.pkt_invalid_cnt);
+		seq_printf(filp, "sa%d: Pkts no sa error: %lld\n", sa_id, stats.pkt_nosaerror_cnt);
+		seq_printf(filp, "sa%d: Pkts not valid: %lld\n", sa_id, stats.pkt_notvalid_cnt);
+		seq_printf(filp, "sa%d: Pkts ok: %lld\n", sa_id, stats.pkt_ok_cnt);
+		seq_printf(filp, "sa%d: Pkts no sa: %lld\n", sa_id, stats.pkt_nosa_cnt);
+	}
+	mutex_unlock(&mcs->stats_lock);
+	return 0;
+}
+
+static int rvu_dbg_mcs_rx_sa_stats_display(struct seq_file *filp, void *unused)
+{
+	return rvu_dbg_mcs_sa_stats_display(filp, unused, MCS_RX);
+}
+
+RVU_DEBUG_SEQ_FOPS(mcs_rx_sa_stats, mcs_rx_sa_stats_display, NULL);
+
+static int rvu_dbg_mcs_tx_sa_stats_display(struct seq_file *filp, void *unused)
+{
+	return rvu_dbg_mcs_sa_stats_display(filp, unused, MCS_TX);
+}
+
+RVU_DEBUG_SEQ_FOPS(mcs_tx_sa_stats, mcs_tx_sa_stats_display, NULL);
+
+static int rvu_dbg_mcs_tx_sc_stats_display(struct seq_file *filp, void *unused)
+{
+	struct mcs *mcs = filp->private;
+	struct mcs_sc_stats stats;
+	struct rsrc_bmap *map;
+	int sc_id;
+
+	map = &mcs->tx.sc;
+	seq_puts(filp, "\n SC stats\n");
+
+	mutex_lock(&mcs->stats_lock);
+	for_each_set_bit(sc_id, map->bmap, mcs->hw->sc_entries) {
+		mcs_get_sc_stats(mcs, &stats, sc_id, MCS_TX);
+		seq_printf(filp, "\n=======sc%d======\n\n", sc_id);
+		seq_printf(filp, "sc%d: Pkts encrypted: %lld\n", sc_id, stats.pkt_encrypt_cnt);
+		seq_printf(filp, "sc%d: Pkts protected: %lld\n", sc_id, stats.pkt_protected_cnt);
+
+		if (mcs->hw->mcs_blks == 1) {
+			seq_printf(filp, "sc%d: Octets encrypted: %lld\n", sc_id,
+				   stats.octet_encrypt_cnt);
+			seq_printf(filp, "sc%d: Octets protected: %lld\n", sc_id,
+				   stats.octet_protected_cnt);
+		}
+	}
+	mutex_unlock(&mcs->stats_lock);
+	return 0;
+}
+
+RVU_DEBUG_SEQ_FOPS(mcs_tx_sc_stats, mcs_tx_sc_stats_display, NULL);
+
+static int rvu_dbg_mcs_rx_sc_stats_display(struct seq_file *filp, void *unused)
+{
+	struct mcs *mcs = filp->private;
+	struct mcs_sc_stats stats;
+	struct rsrc_bmap *map;
+	int sc_id;
+
+	map = &mcs->rx.sc;
+	seq_puts(filp, "\n SC stats\n");
+
+	mutex_lock(&mcs->stats_lock);
+	for_each_set_bit(sc_id, map->bmap, mcs->hw->sc_entries) {
+		mcs_get_sc_stats(mcs, &stats, sc_id, MCS_RX);
+		seq_printf(filp, "\n=======sc%d======\n\n", sc_id);
+		seq_printf(filp, "sc%d: Cam hits: %lld\n", sc_id, stats.hit_cnt);
+		seq_printf(filp, "sc%d: Invalid pkts: %lld\n", sc_id, stats.pkt_invalid_cnt);
+		seq_printf(filp, "sc%d: Late pkts: %lld\n", sc_id, stats.pkt_late_cnt);
+		seq_printf(filp, "sc%d: Notvalid pkts: %lld\n", sc_id, stats.pkt_notvalid_cnt);
+		seq_printf(filp, "sc%d: Unchecked pkts: %lld\n", sc_id, stats.pkt_unchecked_cnt);
+
+		if (mcs->hw->mcs_blks > 1) {
+			seq_printf(filp, "sc%d: Delay pkts: %lld\n", sc_id, stats.pkt_delay_cnt);
+			seq_printf(filp, "sc%d: Pkts ok: %lld\n", sc_id, stats.pkt_ok_cnt);
+		}
+		if (mcs->hw->mcs_blks == 1) {
+			seq_printf(filp, "sc%d: Octets decrypted: %lld\n", sc_id,
+				   stats.octet_decrypt_cnt);
+			seq_printf(filp, "sc%d: Octets validated: %lld\n", sc_id,
+				   stats.octet_validate_cnt);
+		}
+	}
+	mutex_unlock(&mcs->stats_lock);
+	return 0;
+}
+
+RVU_DEBUG_SEQ_FOPS(mcs_rx_sc_stats, mcs_rx_sc_stats_display, NULL);
+
+static int rvu_dbg_mcs_flowid_stats_display(struct seq_file *filp, void *unused, int dir)
+{
+	struct mcs *mcs = filp->private;
+	struct mcs_flowid_stats stats;
+	struct rsrc_bmap *map;
+	int flow_id;
+
+	seq_puts(filp, "\n Flowid stats\n");
+
+	if (dir == MCS_RX)
+		map = &mcs->rx.flow_ids;
+	else
+		map = &mcs->tx.flow_ids;
+
+	mutex_lock(&mcs->stats_lock);
+	for_each_set_bit(flow_id, map->bmap, mcs->hw->tcam_entries) {
+		mcs_get_flowid_stats(mcs, &stats, flow_id, dir);
+		seq_printf(filp, "Flowid%d: Hit:%lld\n", flow_id, stats.tcam_hit_cnt);
+	}
+	mutex_unlock(&mcs->stats_lock);
+	return 0;
+}
+
+static int rvu_dbg_mcs_tx_flowid_stats_display(struct seq_file *filp, void *unused)
+{
+	return rvu_dbg_mcs_flowid_stats_display(filp, unused, MCS_TX);
+}
+
+RVU_DEBUG_SEQ_FOPS(mcs_tx_flowid_stats, mcs_tx_flowid_stats_display, NULL);
+
+static int rvu_dbg_mcs_rx_flowid_stats_display(struct seq_file *filp, void *unused)
+{
+	return rvu_dbg_mcs_flowid_stats_display(filp, unused, MCS_RX);
+}
+
+RVU_DEBUG_SEQ_FOPS(mcs_rx_flowid_stats, mcs_rx_flowid_stats_display, NULL);
+
+static int rvu_dbg_mcs_tx_secy_stats_display(struct seq_file *filp, void *unused)
+{
+	struct mcs *mcs = filp->private;
+	struct mcs_secy_stats stats;
+	struct rsrc_bmap *map;
+	int secy_id;
+
+	map = &mcs->tx.secy;
+	seq_puts(filp, "\n MCS TX secy stats\n");
+
+	mutex_lock(&mcs->stats_lock);
+	for_each_set_bit(secy_id, map->bmap, mcs->hw->secy_entries) {
+		mcs_get_tx_secy_stats(mcs, &stats, secy_id);
+		seq_printf(filp, "\n=======Secy%d======\n\n", secy_id);
+		seq_printf(filp, "secy%d: Ctrl bcast pkts: %lld\n", secy_id,
+			   stats.ctl_pkt_bcast_cnt);
+		seq_printf(filp, "secy%d: Ctrl Mcast pkts: %lld\n", secy_id,
+			   stats.ctl_pkt_mcast_cnt);
+		seq_printf(filp, "secy%d: Ctrl ucast pkts: %lld\n", secy_id,
+			   stats.ctl_pkt_ucast_cnt);
+		seq_printf(filp, "secy%d: Ctrl octets: %lld\n", secy_id, stats.ctl_octet_cnt);
+		seq_printf(filp, "secy%d: Unctrl bcast cnt: %lld\n", secy_id,
+			   stats.unctl_pkt_bcast_cnt);
+		seq_printf(filp, "secy%d: Unctrl mcast pkts: %lld\n", secy_id,
+			   stats.unctl_pkt_mcast_cnt);
+		seq_printf(filp, "secy%d: Unctrl ucast pkts: %lld\n", secy_id,
+			   stats.unctl_pkt_ucast_cnt);
+		seq_printf(filp, "secy%d: Unctrl octets: %lld\n", secy_id, stats.unctl_octet_cnt);
+		seq_printf(filp, "secy%d: Octet encrypted: %lld\n", secy_id,
+			   stats.octet_encrypted_cnt);
+		seq_printf(filp, "secy%d: octet protected: %lld\n", secy_id,
+			   stats.octet_protected_cnt);
+		seq_printf(filp, "secy%d: Pkts on active sa: %lld\n", secy_id,
+			   stats.pkt_noactivesa_cnt);
+		seq_printf(filp, "secy%d: Pkts too long: %lld\n", secy_id, stats.pkt_toolong_cnt);
+		seq_printf(filp, "secy%d: Pkts untagged: %lld\n", secy_id, stats.pkt_untagged_cnt);
+	}
+	mutex_unlock(&mcs->stats_lock);
+	return 0;
+}
+
+RVU_DEBUG_SEQ_FOPS(mcs_tx_secy_stats, mcs_tx_secy_stats_display, NULL);
+
+static int rvu_dbg_mcs_rx_secy_stats_display(struct seq_file *filp, void *unused)
+{
+	struct mcs *mcs = filp->private;
+	struct mcs_secy_stats stats;
+	struct rsrc_bmap *map;
+	int secy_id;
+
+	map = &mcs->rx.secy;
+	seq_puts(filp, "\n MCS secy stats\n");
+
+	mutex_lock(&mcs->stats_lock);
+	for_each_set_bit(secy_id, map->bmap, mcs->hw->secy_entries) {
+		mcs_get_rx_secy_stats(mcs, &stats, secy_id);
+		seq_printf(filp, "\n=======Secy%d======\n\n", secy_id);
+		seq_printf(filp, "secy%d: Ctrl bcast pkts: %lld\n", secy_id,
+			   stats.ctl_pkt_bcast_cnt);
+		seq_printf(filp, "secy%d: Ctrl Mcast pkts: %lld\n", secy_id,
+			   stats.ctl_pkt_mcast_cnt);
+		seq_printf(filp, "secy%d: Ctrl ucast pkts: %lld\n", secy_id,
+			   stats.ctl_pkt_ucast_cnt);
+		seq_printf(filp, "secy%d: Ctrl octets: %lld\n", secy_id, stats.ctl_octet_cnt);
+		seq_printf(filp, "secy%d: Unctrl bcast cnt: %lld\n", secy_id,
+			   stats.unctl_pkt_bcast_cnt);
+		seq_printf(filp, "secy%d: Unctrl mcast pkts: %lld\n", secy_id,
+			   stats.unctl_pkt_mcast_cnt);
+		seq_printf(filp, "secy%d: Unctrl ucast pkts: %lld\n", secy_id,
+			   stats.unctl_pkt_ucast_cnt);
+		seq_printf(filp, "secy%d: Unctrl octets: %lld\n", secy_id, stats.unctl_octet_cnt);
+		seq_printf(filp, "secy%d: Octet decrypted: %lld\n", secy_id,
+			   stats.octet_decrypted_cnt);
+		seq_printf(filp, "secy%d: octet validated: %lld\n", secy_id,
+			   stats.octet_validated_cnt);
+		seq_printf(filp, "secy%d: Pkts on disable port: %lld\n", secy_id,
+			   stats.pkt_port_disabled_cnt);
+		seq_printf(filp, "secy%d: Octets validated: %lld\n", secy_id, stats.pkt_badtag_cnt);
+		seq_printf(filp, "secy%d: Octets validated: %lld\n", secy_id, stats.pkt_nosa_cnt);
+		seq_printf(filp, "secy%d: Pkts with nosaerror: %lld\n", secy_id,
+			   stats.pkt_nosaerror_cnt);
+		seq_printf(filp, "secy%d: Tagged ctrl pkts: %lld\n", secy_id,
+			   stats.pkt_tagged_ctl_cnt);
+		seq_printf(filp, "secy%d: Untaged pkts: %lld\n", secy_id, stats.pkt_untaged_cnt);
+		seq_printf(filp, "secy%d: Ctrl pkts: %lld\n", secy_id, stats.pkt_ctl_cnt);
+		if (mcs->hw->mcs_blks > 1)
+			seq_printf(filp, "secy%d: pkts notag: %lld\n", secy_id,
+				   stats.pkt_notag_cnt);
+	}
+	mutex_unlock(&mcs->stats_lock);
+	return 0;
+}
+
+RVU_DEBUG_SEQ_FOPS(mcs_rx_secy_stats, mcs_rx_secy_stats_display, NULL);
+
+static void rvu_dbg_mcs_init(struct rvu *rvu)
+{
+	struct mcs *mcs;
+	char dname[10];
+	int i;
+
+	if (!rvu->mcs_blk_cnt)
+		return;
+
+	rvu->rvu_dbg.mcs_root = debugfs_create_dir("mcs", rvu->rvu_dbg.root);
+
+	for (i = 0; i < rvu->mcs_blk_cnt; i++) {
+		mcs = mcs_get_pdata(i);
+
+		sprintf(dname, "mcs%d", i);
+		rvu->rvu_dbg.mcs = debugfs_create_dir(dname,
+						      rvu->rvu_dbg.mcs_root);
+
+		rvu->rvu_dbg.mcs_rx = debugfs_create_dir("rx_stats", rvu->rvu_dbg.mcs);
+
+		debugfs_create_file("flowid", 0600, rvu->rvu_dbg.mcs_rx, mcs,
+				    &rvu_dbg_mcs_rx_flowid_stats_fops);
+
+		debugfs_create_file("secy", 0600, rvu->rvu_dbg.mcs_rx, mcs,
+				    &rvu_dbg_mcs_rx_secy_stats_fops);
+
+		debugfs_create_file("sc", 0600, rvu->rvu_dbg.mcs_rx, mcs,
+				    &rvu_dbg_mcs_rx_sc_stats_fops);
+
+		debugfs_create_file("sa", 0600, rvu->rvu_dbg.mcs_rx, mcs,
+				    &rvu_dbg_mcs_rx_sa_stats_fops);
+
+		debugfs_create_file("port", 0600, rvu->rvu_dbg.mcs_rx, mcs,
+				    &rvu_dbg_mcs_rx_port_stats_fops);
+
+		rvu->rvu_dbg.mcs_tx = debugfs_create_dir("tx_stats", rvu->rvu_dbg.mcs);
+
+		debugfs_create_file("flowid", 0600, rvu->rvu_dbg.mcs_tx, mcs,
+				    &rvu_dbg_mcs_tx_flowid_stats_fops);
+
+		debugfs_create_file("secy", 0600, rvu->rvu_dbg.mcs_tx, mcs,
+				    &rvu_dbg_mcs_tx_secy_stats_fops);
+
+		debugfs_create_file("sc", 0600, rvu->rvu_dbg.mcs_tx, mcs,
+				    &rvu_dbg_mcs_tx_sc_stats_fops);
+
+		debugfs_create_file("sa", 0600, rvu->rvu_dbg.mcs_tx, mcs,
+				    &rvu_dbg_mcs_tx_sa_stats_fops);
+
+		debugfs_create_file("port", 0600, rvu->rvu_dbg.mcs_tx, mcs,
+				    &rvu_dbg_mcs_tx_port_stats_fops);
+	}
+}
+
 #define LMT_MAPTBL_ENTRY_SIZE 16
 /* Dump LMTST map table */
 static ssize_t rvu_dbg_lmtst_map_table_display(struct file *filp,
@@ -3053,6 +3398,7 @@ create:
 	rvu_dbg_npc_init(rvu);
 	rvu_dbg_cpt_init(rvu, BLKADDR_CPT0);
 	rvu_dbg_cpt_init(rvu, BLKADDR_CPT1);
+	rvu_dbg_mcs_init(rvu);
 }
 
 void rvu_dbg_exit(struct rvu *rvu)
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
index 0879a48411f3..7646bb2ec89b 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c
@@ -4296,8 +4296,14 @@ static int rvu_nix_block_init(struct rvu *rvu, struct nix_hw *nix_hw)
 	/* Restore CINT timer delay to HW reset values */
 	rvu_write64(rvu, blkaddr, NIX_AF_CINT_DELAY, 0x0ULL);
 
+	cfg = rvu_read64(rvu, blkaddr, NIX_AF_SEB_CFG);
+
 	/* For better performance use NDC TX instead of NDC RX for SQ's SQEs" */
-	rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, 0x1ULL);
+	cfg |= 1ULL;
+	if (!is_rvu_otx2(rvu))
+		cfg |= NIX_PTP_1STEP_EN;
+
+	rvu_write64(rvu, blkaddr, NIX_AF_SEB_CFG, cfg);
 
 	if (is_block_implemented(hw, blkaddr)) {
 		err = nix_setup_txschq(rvu, nix_hw, blkaddr);
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h
index 77a9ade91f3e..0e0d536645ac 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h
+++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_reg.h
@@ -266,6 +266,7 @@
 #define NIX_AF_TX_NPC_CAPTURE_CONFIG	(0x0660)
 #define NIX_AF_TX_NPC_CAPTURE_INFO	(0x0670)
 #define NIX_AF_SEB_CFG			(0x05F0)
+#define NIX_PTP_1STEP_EN		BIT_ULL(2)
 
 #define NIX_AF_DEBUG_NPC_RESP_DATAX(a)          (0x680 | (a) << 3)
 #define NIX_AF_SMQX_CFG(a)                      (0x700 | (a) << 16)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile
index d463dc72d80a..73fdb8798614 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/Makefile
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/Makefile
@@ -13,5 +13,6 @@ rvu_nicvf-y := otx2_vf.o otx2_devlink.o
 
 rvu_nicpf-$(CONFIG_DCB) += otx2_dcbnl.o
 rvu_nicvf-$(CONFIG_DCB) += otx2_dcbnl.o
+rvu_nicpf-$(CONFIG_MACSEC) += cn10k_macsec.o
 
 ccflags-y += -I$(srctree)/drivers/net/ethernet/marvell/octeontx2/af
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
index fd4f083c699e..826f691de259 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k.c
@@ -86,8 +86,7 @@ int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura)
 	aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */
 	aq->sq.cq_ena = 1;
 	aq->sq.ena = 1;
-	/* Only one SMQ is allocated, map all SQ's to that SMQ  */
-	aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0];
+	aq->sq.smq = otx2_get_smq_idx(pfvf, qidx);
 	aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen);
 	aq->sq.default_chan = pfvf->hw.tx_chan_base;
 	aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c
new file mode 100644
index 000000000000..64f3acd7f67b
--- /dev/null
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c
@@ -0,0 +1,1668 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Marvell MACSEC hardware offload driver
+ *
+ * Copyright (C) 2022 Marvell.
+ */
+
+#include <linux/rtnetlink.h>
+#include <linux/bitfield.h>
+#include <net/macsec.h>
+#include "otx2_common.h"
+
+#define MCS_TCAM0_MAC_SA_MASK		GENMASK_ULL(63, 48)
+#define MCS_TCAM1_MAC_SA_MASK		GENMASK_ULL(31, 0)
+#define MCS_TCAM1_ETYPE_MASK		GENMASK_ULL(47, 32)
+
+#define MCS_SA_MAP_MEM_SA_USE		BIT_ULL(9)
+
+#define MCS_RX_SECY_PLCY_RW_MASK	GENMASK_ULL(49, 18)
+#define MCS_RX_SECY_PLCY_RP		BIT_ULL(17)
+#define MCS_RX_SECY_PLCY_AUTH_ENA	BIT_ULL(16)
+#define MCS_RX_SECY_PLCY_CIP		GENMASK_ULL(8, 5)
+#define MCS_RX_SECY_PLCY_VAL		GENMASK_ULL(2, 1)
+#define MCS_RX_SECY_PLCY_ENA		BIT_ULL(0)
+
+#define MCS_TX_SECY_PLCY_MTU		GENMASK_ULL(43, 28)
+#define MCS_TX_SECY_PLCY_ST_TCI		GENMASK_ULL(27, 22)
+#define MCS_TX_SECY_PLCY_ST_OFFSET	GENMASK_ULL(21, 15)
+#define MCS_TX_SECY_PLCY_INS_MODE	BIT_ULL(14)
+#define MCS_TX_SECY_PLCY_AUTH_ENA	BIT_ULL(13)
+#define MCS_TX_SECY_PLCY_CIP		GENMASK_ULL(5, 2)
+#define MCS_TX_SECY_PLCY_PROTECT	BIT_ULL(1)
+#define MCS_TX_SECY_PLCY_ENA		BIT_ULL(0)
+
+#define MCS_GCM_AES_128			0
+#define MCS_GCM_AES_256			1
+#define MCS_GCM_AES_XPN_128		2
+#define MCS_GCM_AES_XPN_256		3
+
+#define MCS_TCI_ES			0x40 /* end station */
+#define MCS_TCI_SC			0x20 /* SCI present */
+#define MCS_TCI_SCB			0x10 /* epon */
+#define MCS_TCI_E			0x08 /* encryption */
+#define MCS_TCI_C			0x04 /* changed text */
+
+static struct cn10k_mcs_txsc *cn10k_mcs_get_txsc(struct cn10k_mcs_cfg *cfg,
+						 struct macsec_secy *secy)
+{
+	struct cn10k_mcs_txsc *txsc;
+
+	list_for_each_entry(txsc, &cfg->txsc_list, entry) {
+		if (txsc->sw_secy == secy)
+			return txsc;
+	}
+
+	return NULL;
+}
+
+static struct cn10k_mcs_rxsc *cn10k_mcs_get_rxsc(struct cn10k_mcs_cfg *cfg,
+						 struct macsec_secy *secy,
+						 struct macsec_rx_sc *rx_sc)
+{
+	struct cn10k_mcs_rxsc *rxsc;
+
+	list_for_each_entry(rxsc, &cfg->rxsc_list, entry) {
+		if (rxsc->sw_rxsc == rx_sc && rxsc->sw_secy == secy)
+			return rxsc;
+	}
+
+	return NULL;
+}
+
+static const char *rsrc_name(enum mcs_rsrc_type rsrc_type)
+{
+	switch (rsrc_type) {
+	case MCS_RSRC_TYPE_FLOWID:
+		return "FLOW";
+	case MCS_RSRC_TYPE_SC:
+		return "SC";
+	case MCS_RSRC_TYPE_SECY:
+		return "SECY";
+	case MCS_RSRC_TYPE_SA:
+		return "SA";
+	default:
+		return "Unknown";
+	};
+
+	return "Unknown";
+}
+
+static int cn10k_mcs_alloc_rsrc(struct otx2_nic *pfvf, enum mcs_direction dir,
+				enum mcs_rsrc_type type, u16 *rsrc_id)
+{
+	struct mbox *mbox = &pfvf->mbox;
+	struct mcs_alloc_rsrc_req *req;
+	struct mcs_alloc_rsrc_rsp *rsp;
+	int ret = -ENOMEM;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_alloc_resources(mbox);
+	if (!req)
+		goto fail;
+
+	req->rsrc_type = type;
+	req->rsrc_cnt  = 1;
+	req->dir = dir;
+
+	ret = otx2_sync_mbox_msg(mbox);
+	if (ret)
+		goto fail;
+
+	rsp = (struct mcs_alloc_rsrc_rsp *)otx2_mbox_get_rsp(&pfvf->mbox.mbox,
+							     0, &req->hdr);
+	if (IS_ERR(rsp) || req->rsrc_cnt != rsp->rsrc_cnt ||
+	    req->rsrc_type != rsp->rsrc_type || req->dir != rsp->dir) {
+		ret = -EINVAL;
+		goto fail;
+	}
+
+	switch (rsp->rsrc_type) {
+	case MCS_RSRC_TYPE_FLOWID:
+		*rsrc_id = rsp->flow_ids[0];
+		break;
+	case MCS_RSRC_TYPE_SC:
+		*rsrc_id = rsp->sc_ids[0];
+		break;
+	case MCS_RSRC_TYPE_SECY:
+		*rsrc_id = rsp->secy_ids[0];
+		break;
+	case MCS_RSRC_TYPE_SA:
+		*rsrc_id = rsp->sa_ids[0];
+		break;
+	default:
+		ret = -EINVAL;
+		goto fail;
+	};
+
+	mutex_unlock(&mbox->lock);
+
+	return 0;
+fail:
+	dev_err(pfvf->dev, "Failed to allocate %s %s resource\n",
+		dir == MCS_TX ? "TX" : "RX", rsrc_name(type));
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static void cn10k_mcs_free_rsrc(struct otx2_nic *pfvf, enum mcs_direction dir,
+				enum mcs_rsrc_type type, u16 hw_rsrc_id,
+				bool all)
+{
+	struct mbox *mbox = &pfvf->mbox;
+	struct mcs_free_rsrc_req *req;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_free_resources(mbox);
+	if (!req)
+		goto fail;
+
+	req->rsrc_id = hw_rsrc_id;
+	req->rsrc_type = type;
+	req->dir = dir;
+	if (all)
+		req->all = 1;
+
+	if (otx2_sync_mbox_msg(&pfvf->mbox))
+		goto fail;
+
+	mutex_unlock(&mbox->lock);
+
+	return;
+fail:
+	dev_err(pfvf->dev, "Failed to free %s %s resource\n",
+		dir == MCS_TX ? "TX" : "RX", rsrc_name(type));
+	mutex_unlock(&mbox->lock);
+}
+
+static int cn10k_mcs_alloc_txsa(struct otx2_nic *pfvf, u16 *hw_sa_id)
+{
+	return cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SA, hw_sa_id);
+}
+
+static int cn10k_mcs_alloc_rxsa(struct otx2_nic *pfvf, u16 *hw_sa_id)
+{
+	return cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SA, hw_sa_id);
+}
+
+static void cn10k_mcs_free_txsa(struct otx2_nic *pfvf, u16 hw_sa_id)
+{
+	cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SA, hw_sa_id, false);
+}
+
+static void cn10k_mcs_free_rxsa(struct otx2_nic *pfvf, u16 hw_sa_id)
+{
+	cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SA, hw_sa_id, false);
+}
+
+static int cn10k_mcs_write_rx_secy(struct otx2_nic *pfvf,
+				   struct macsec_secy *secy, u8 hw_secy_id)
+{
+	struct mcs_secy_plcy_write_req *req;
+	struct mbox *mbox = &pfvf->mbox;
+	u64 policy;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_secy_plcy_write(mbox);
+	if (!req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	policy = FIELD_PREP(MCS_RX_SECY_PLCY_RW_MASK, secy->replay_window);
+	if (secy->replay_protect)
+		policy |= MCS_RX_SECY_PLCY_RP;
+
+	policy |= MCS_RX_SECY_PLCY_AUTH_ENA;
+	policy |= FIELD_PREP(MCS_RX_SECY_PLCY_CIP, MCS_GCM_AES_128);
+	policy |= FIELD_PREP(MCS_RX_SECY_PLCY_VAL, secy->validate_frames);
+
+	policy |= MCS_RX_SECY_PLCY_ENA;
+
+	req->plcy = policy;
+	req->secy_id = hw_secy_id;
+	req->dir = MCS_RX;
+
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_write_rx_flowid(struct otx2_nic *pfvf,
+				     struct cn10k_mcs_rxsc *rxsc, u8 hw_secy_id)
+{
+	struct macsec_rx_sc *sw_rx_sc = rxsc->sw_rxsc;
+	struct mcs_flowid_entry_write_req *req;
+	struct mbox *mbox = &pfvf->mbox;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_flowid_entry_write(mbox);
+	if (!req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	req->data[1] = FIELD_PREP(MCS_TCAM1_ETYPE_MASK, ETH_P_MACSEC);
+	req->mask[1] = ~0ULL;
+	req->mask[1] &= ~MCS_TCAM1_ETYPE_MASK;
+
+	req->mask[0] = ~0ULL;
+	req->mask[2] = ~0ULL;
+	req->mask[3] = ~0ULL;
+
+	req->flow_id = rxsc->hw_flow_id;
+	req->secy_id = hw_secy_id;
+	req->sc_id = rxsc->hw_sc_id;
+	req->dir = MCS_RX;
+
+	if (sw_rx_sc->active)
+		req->ena = 1;
+
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_write_sc_cam(struct otx2_nic *pfvf,
+				  struct cn10k_mcs_rxsc *rxsc, u8 hw_secy_id)
+{
+	struct macsec_rx_sc *sw_rx_sc = rxsc->sw_rxsc;
+	struct mcs_rx_sc_cam_write_req *sc_req;
+	struct mbox *mbox = &pfvf->mbox;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	sc_req = otx2_mbox_alloc_msg_mcs_rx_sc_cam_write(mbox);
+	if (!sc_req) {
+		return -ENOMEM;
+		goto fail;
+	}
+
+	sc_req->sci = (__force u64)cpu_to_be64((__force u64)sw_rx_sc->sci);
+	sc_req->sc_id = rxsc->hw_sc_id;
+	sc_req->secy_id = hw_secy_id;
+
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_write_rx_sa_plcy(struct otx2_nic *pfvf,
+				      struct macsec_secy *secy,
+				      struct cn10k_mcs_rxsc *rxsc,
+				      u8 assoc_num, bool sa_in_use)
+{
+	unsigned char *src = rxsc->sa_key[assoc_num];
+	struct mcs_sa_plcy_write_req *plcy_req;
+	struct mcs_rx_sc_sa_map *map_req;
+	struct mbox *mbox = &pfvf->mbox;
+	u8 reg, key_len;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	plcy_req = otx2_mbox_alloc_msg_mcs_sa_plcy_write(mbox);
+	if (!plcy_req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	map_req = otx2_mbox_alloc_msg_mcs_rx_sc_sa_map_write(mbox);
+	if (!map_req) {
+		otx2_mbox_reset(&mbox->mbox, 0);
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	for (reg = 0, key_len = 0; key_len < secy->key_len; key_len += 8) {
+		memcpy((u8 *)&plcy_req->plcy[0][reg],
+		       (src + reg * 8), 8);
+		reg++;
+	}
+
+	plcy_req->sa_index[0] = rxsc->hw_sa_id[assoc_num];
+	plcy_req->sa_cnt = 1;
+	plcy_req->dir = MCS_RX;
+
+	map_req->sa_index = rxsc->hw_sa_id[assoc_num];
+	map_req->sa_in_use = sa_in_use;
+	map_req->sc_id = rxsc->hw_sc_id;
+	map_req->an = assoc_num;
+
+	/* Send two messages together */
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_write_rx_sa_pn(struct otx2_nic *pfvf,
+				    struct cn10k_mcs_rxsc *rxsc,
+				    u8 assoc_num, u64 next_pn)
+{
+	struct mcs_pn_table_write_req *req;
+	struct mbox *mbox = &pfvf->mbox;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_pn_table_write(mbox);
+	if (!req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	req->pn_id = rxsc->hw_sa_id[assoc_num];
+	req->next_pn = next_pn;
+	req->dir = MCS_RX;
+
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_write_tx_secy(struct otx2_nic *pfvf,
+				   struct macsec_secy *secy,
+				   struct cn10k_mcs_txsc *txsc)
+{
+	struct mcs_secy_plcy_write_req *req;
+	struct mbox *mbox = &pfvf->mbox;
+	struct macsec_tx_sc *sw_tx_sc;
+	/* Insert SecTag after 12 bytes (DA+SA)*/
+	u8 tag_offset = 12;
+	u8 sectag_tci = 0;
+	u64 policy;
+	int ret;
+
+	sw_tx_sc = &secy->tx_sc;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_secy_plcy_write(mbox);
+	if (!req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	if (sw_tx_sc->send_sci) {
+		sectag_tci |= MCS_TCI_SC;
+	} else {
+		if (sw_tx_sc->end_station)
+			sectag_tci |= MCS_TCI_ES;
+		if (sw_tx_sc->scb)
+			sectag_tci |= MCS_TCI_SCB;
+	}
+
+	if (sw_tx_sc->encrypt)
+		sectag_tci |= (MCS_TCI_E | MCS_TCI_C);
+
+	policy = FIELD_PREP(MCS_TX_SECY_PLCY_MTU, secy->netdev->mtu);
+	/* Write SecTag excluding AN bits(1..0) */
+	policy |= FIELD_PREP(MCS_TX_SECY_PLCY_ST_TCI, sectag_tci >> 2);
+	policy |= FIELD_PREP(MCS_TX_SECY_PLCY_ST_OFFSET, tag_offset);
+	policy |= MCS_TX_SECY_PLCY_INS_MODE;
+	policy |= MCS_TX_SECY_PLCY_AUTH_ENA;
+	policy |= FIELD_PREP(MCS_TX_SECY_PLCY_CIP, MCS_GCM_AES_128);
+
+	if (secy->protect_frames)
+		policy |= MCS_TX_SECY_PLCY_PROTECT;
+
+	/* If the encodingsa does not exist/active and protect is
+	 * not set then frames can be sent out as it is. Hence enable
+	 * the policy irrespective of secy operational when !protect.
+	 */
+	if (!secy->protect_frames || secy->operational)
+		policy |= MCS_TX_SECY_PLCY_ENA;
+
+	req->plcy = policy;
+	req->secy_id = txsc->hw_secy_id_tx;
+	req->dir = MCS_TX;
+
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_write_tx_flowid(struct otx2_nic *pfvf,
+				     struct macsec_secy *secy,
+				     struct cn10k_mcs_txsc *txsc)
+{
+	struct mcs_flowid_entry_write_req *req;
+	struct mbox *mbox = &pfvf->mbox;
+	u64 mac_sa;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_flowid_entry_write(mbox);
+	if (!req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	mac_sa = ether_addr_to_u64(secy->netdev->dev_addr);
+
+	req->data[0] = FIELD_PREP(MCS_TCAM0_MAC_SA_MASK, mac_sa);
+	req->data[1] = FIELD_PREP(MCS_TCAM1_MAC_SA_MASK, mac_sa >> 16);
+
+	req->mask[0] = ~0ULL;
+	req->mask[0] &= ~MCS_TCAM0_MAC_SA_MASK;
+
+	req->mask[1] = ~0ULL;
+	req->mask[1] &= ~MCS_TCAM1_MAC_SA_MASK;
+
+	req->mask[2] = ~0ULL;
+	req->mask[3] = ~0ULL;
+
+	req->flow_id = txsc->hw_flow_id;
+	req->secy_id = txsc->hw_secy_id_tx;
+	req->sc_id = txsc->hw_sc_id;
+	req->sci = (__force u64)cpu_to_be64((__force u64)secy->sci);
+	req->dir = MCS_TX;
+	/* This can be enabled since stack xmits packets only when interface is up */
+	req->ena = 1;
+
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_link_tx_sa2sc(struct otx2_nic *pfvf,
+				   struct macsec_secy *secy,
+				   struct cn10k_mcs_txsc *txsc,
+				   u8 sa_num, bool sa_active)
+{
+	struct mcs_tx_sc_sa_map *map_req;
+	struct mbox *mbox = &pfvf->mbox;
+	int ret;
+
+	/* Link the encoding_sa only to SC out of all SAs */
+	if (txsc->encoding_sa != sa_num)
+		return 0;
+
+	mutex_lock(&mbox->lock);
+
+	map_req = otx2_mbox_alloc_msg_mcs_tx_sc_sa_map_write(mbox);
+	if (!map_req) {
+		otx2_mbox_reset(&mbox->mbox, 0);
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	map_req->sa_index0 = txsc->hw_sa_id[sa_num];
+	map_req->sa_index0_vld = sa_active;
+	map_req->sectag_sci = (__force u64)cpu_to_be64((__force u64)secy->sci);
+	map_req->sc_id = txsc->hw_sc_id;
+
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_write_tx_sa_plcy(struct otx2_nic *pfvf,
+				      struct macsec_secy *secy,
+				      struct cn10k_mcs_txsc *txsc,
+				      u8 assoc_num)
+{
+	unsigned char *src = txsc->sa_key[assoc_num];
+	struct mcs_sa_plcy_write_req *plcy_req;
+	struct mbox *mbox = &pfvf->mbox;
+	u8 reg, key_len;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	plcy_req = otx2_mbox_alloc_msg_mcs_sa_plcy_write(mbox);
+	if (!plcy_req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	for (reg = 0, key_len = 0; key_len < secy->key_len; key_len += 8) {
+		memcpy((u8 *)&plcy_req->plcy[0][reg], (src + reg * 8), 8);
+		reg++;
+	}
+
+	plcy_req->plcy[0][8] = assoc_num;
+	plcy_req->sa_index[0] = txsc->hw_sa_id[assoc_num];
+	plcy_req->sa_cnt = 1;
+	plcy_req->dir = MCS_TX;
+
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_write_tx_sa_pn(struct otx2_nic *pfvf,
+				struct cn10k_mcs_txsc *txsc,
+				u8 assoc_num, u64 next_pn)
+{
+	struct mcs_pn_table_write_req *req;
+	struct mbox *mbox = &pfvf->mbox;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_pn_table_write(mbox);
+	if (!req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	req->pn_id = txsc->hw_sa_id[assoc_num];
+	req->next_pn = next_pn;
+	req->dir = MCS_TX;
+
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_ena_dis_flowid(struct otx2_nic *pfvf, u16 hw_flow_id,
+				    bool enable, enum mcs_direction dir)
+{
+	struct mcs_flowid_ena_dis_entry *req;
+	struct mbox *mbox = &pfvf->mbox;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_flowid_ena_entry(mbox);
+	if (!req) {
+		return -ENOMEM;
+		goto fail;
+	}
+
+	req->flow_id = hw_flow_id;
+	req->ena = enable;
+	req->dir = dir;
+
+	ret = otx2_sync_mbox_msg(mbox);
+
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_sa_stats(struct otx2_nic *pfvf, u8 hw_sa_id,
+			      struct mcs_sa_stats *rsp_p,
+			      enum mcs_direction dir, bool clear)
+{
+	struct mcs_clear_stats *clear_req;
+	struct mbox *mbox = &pfvf->mbox;
+	struct mcs_stats_req *req;
+	struct mcs_sa_stats *rsp;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_get_sa_stats(mbox);
+	if (!req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	req->id = hw_sa_id;
+	req->dir = dir;
+
+	if (!clear)
+		goto send_msg;
+
+	clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox);
+	if (!clear_req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+	clear_req->id = hw_sa_id;
+	clear_req->dir = dir;
+	clear_req->type = MCS_RSRC_TYPE_SA;
+
+send_msg:
+	ret = otx2_sync_mbox_msg(mbox);
+	if (ret)
+		goto fail;
+
+	rsp = (struct mcs_sa_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox,
+						       0, &req->hdr);
+	if (IS_ERR(rsp)) {
+		ret = PTR_ERR(rsp);
+		goto fail;
+	}
+
+	memcpy(rsp_p, rsp, sizeof(*rsp_p));
+
+	mutex_unlock(&mbox->lock);
+
+	return 0;
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_sc_stats(struct otx2_nic *pfvf, u8 hw_sc_id,
+			      struct mcs_sc_stats *rsp_p,
+			      enum mcs_direction dir, bool clear)
+{
+	struct mcs_clear_stats *clear_req;
+	struct mbox *mbox = &pfvf->mbox;
+	struct mcs_stats_req *req;
+	struct mcs_sc_stats *rsp;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_get_sc_stats(mbox);
+	if (!req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	req->id = hw_sc_id;
+	req->dir = dir;
+
+	if (!clear)
+		goto send_msg;
+
+	clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox);
+	if (!clear_req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+	clear_req->id = hw_sc_id;
+	clear_req->dir = dir;
+	clear_req->type = MCS_RSRC_TYPE_SC;
+
+send_msg:
+	ret = otx2_sync_mbox_msg(mbox);
+	if (ret)
+		goto fail;
+
+	rsp = (struct mcs_sc_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox,
+						       0, &req->hdr);
+	if (IS_ERR(rsp)) {
+		ret = PTR_ERR(rsp);
+		goto fail;
+	}
+
+	memcpy(rsp_p, rsp, sizeof(*rsp_p));
+
+	mutex_unlock(&mbox->lock);
+
+	return 0;
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static int cn10k_mcs_secy_stats(struct otx2_nic *pfvf, u8 hw_secy_id,
+				struct mcs_secy_stats *rsp_p,
+				enum mcs_direction dir, bool clear)
+{
+	struct mcs_clear_stats *clear_req;
+	struct mbox *mbox = &pfvf->mbox;
+	struct mcs_secy_stats *rsp;
+	struct mcs_stats_req *req;
+	int ret;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_get_secy_stats(mbox);
+	if (!req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+
+	req->id = hw_secy_id;
+	req->dir = dir;
+
+	if (!clear)
+		goto send_msg;
+
+	clear_req = otx2_mbox_alloc_msg_mcs_clear_stats(mbox);
+	if (!clear_req) {
+		ret = -ENOMEM;
+		goto fail;
+	}
+	clear_req->id = hw_secy_id;
+	clear_req->dir = dir;
+	clear_req->type = MCS_RSRC_TYPE_SECY;
+
+send_msg:
+	ret = otx2_sync_mbox_msg(mbox);
+	if (ret)
+		goto fail;
+
+	rsp = (struct mcs_secy_stats *)otx2_mbox_get_rsp(&pfvf->mbox.mbox,
+							 0, &req->hdr);
+	if (IS_ERR(rsp)) {
+		ret = PTR_ERR(rsp);
+		goto fail;
+	}
+
+	memcpy(rsp_p, rsp, sizeof(*rsp_p));
+
+	mutex_unlock(&mbox->lock);
+
+	return 0;
+fail:
+	mutex_unlock(&mbox->lock);
+	return ret;
+}
+
+static struct cn10k_mcs_txsc *cn10k_mcs_create_txsc(struct otx2_nic *pfvf)
+{
+	struct cn10k_mcs_txsc *txsc;
+	int ret;
+
+	txsc = kzalloc(sizeof(*txsc), GFP_KERNEL);
+	if (!txsc)
+		return ERR_PTR(-ENOMEM);
+
+	ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID,
+				   &txsc->hw_flow_id);
+	if (ret)
+		goto fail;
+
+	/* For a SecY, one TX secy and one RX secy HW resources are needed */
+	ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY,
+				   &txsc->hw_secy_id_tx);
+	if (ret)
+		goto free_flowid;
+
+	ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY,
+				   &txsc->hw_secy_id_rx);
+	if (ret)
+		goto free_tx_secy;
+
+	ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SC,
+				   &txsc->hw_sc_id);
+	if (ret)
+		goto free_rx_secy;
+
+	return txsc;
+free_rx_secy:
+	cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY,
+			    txsc->hw_secy_id_rx, false);
+free_tx_secy:
+	cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY,
+			    txsc->hw_secy_id_tx, false);
+free_flowid:
+	cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID,
+			    txsc->hw_flow_id, false);
+fail:
+	return ERR_PTR(ret);
+}
+
+/* Free Tx SC and its SAs(if any) resources to AF
+ */
+static void cn10k_mcs_delete_txsc(struct otx2_nic *pfvf,
+				  struct cn10k_mcs_txsc *txsc)
+{
+	u8 sa_bmap = txsc->sa_bmap;
+	u8 sa_num = 0;
+
+	while (sa_bmap) {
+		if (sa_bmap & 1) {
+			cn10k_mcs_write_tx_sa_plcy(pfvf, txsc->sw_secy,
+						   txsc, sa_num);
+			cn10k_mcs_free_txsa(pfvf, txsc->hw_sa_id[sa_num]);
+		}
+		sa_num++;
+		sa_bmap >>= 1;
+	}
+
+	cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SC,
+			    txsc->hw_sc_id, false);
+	cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY,
+			    txsc->hw_secy_id_rx, false);
+	cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY,
+			    txsc->hw_secy_id_tx, false);
+	cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_FLOWID,
+			    txsc->hw_flow_id, false);
+}
+
+static struct cn10k_mcs_rxsc *cn10k_mcs_create_rxsc(struct otx2_nic *pfvf)
+{
+	struct cn10k_mcs_rxsc *rxsc;
+	int ret;
+
+	rxsc = kzalloc(sizeof(*rxsc), GFP_KERNEL);
+	if (!rxsc)
+		return ERR_PTR(-ENOMEM);
+
+	ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID,
+				   &rxsc->hw_flow_id);
+	if (ret)
+		goto fail;
+
+	ret = cn10k_mcs_alloc_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SC,
+				   &rxsc->hw_sc_id);
+	if (ret)
+		goto free_flowid;
+
+	return rxsc;
+free_flowid:
+	cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID,
+			    rxsc->hw_flow_id, false);
+fail:
+	return ERR_PTR(ret);
+}
+
+/* Free Rx SC and its SAs(if any) resources to AF
+ */
+static void cn10k_mcs_delete_rxsc(struct otx2_nic *pfvf,
+				  struct cn10k_mcs_rxsc *rxsc)
+{
+	u8 sa_bmap = rxsc->sa_bmap;
+	u8 sa_num = 0;
+
+	while (sa_bmap) {
+		if (sa_bmap & 1) {
+			cn10k_mcs_write_rx_sa_plcy(pfvf, rxsc->sw_secy, rxsc,
+						   sa_num, false);
+			cn10k_mcs_free_rxsa(pfvf, rxsc->hw_sa_id[sa_num]);
+		}
+		sa_num++;
+		sa_bmap >>= 1;
+	}
+
+	cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SC,
+			    rxsc->hw_sc_id, false);
+	cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_FLOWID,
+			    rxsc->hw_flow_id, false);
+}
+
+static int cn10k_mcs_secy_tx_cfg(struct otx2_nic *pfvf, struct macsec_secy *secy,
+				 struct cn10k_mcs_txsc *txsc,
+				 struct macsec_tx_sa *sw_tx_sa, u8 sa_num)
+{
+	if (sw_tx_sa) {
+		cn10k_mcs_write_tx_sa_plcy(pfvf, secy, txsc, sa_num);
+		cn10k_write_tx_sa_pn(pfvf, txsc, sa_num,
+				     sw_tx_sa->next_pn_halves.lower);
+		cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc, sa_num,
+					sw_tx_sa->active);
+	}
+
+	cn10k_mcs_write_tx_secy(pfvf, secy, txsc);
+	cn10k_mcs_write_tx_flowid(pfvf, secy, txsc);
+	/* When updating secy, change RX secy also */
+	cn10k_mcs_write_rx_secy(pfvf, secy, txsc->hw_secy_id_rx);
+
+	return 0;
+}
+
+static int cn10k_mcs_secy_rx_cfg(struct otx2_nic *pfvf,
+				 struct macsec_secy *secy, u8 hw_secy_id)
+{
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct cn10k_mcs_rxsc *mcs_rx_sc;
+	struct macsec_rx_sc *sw_rx_sc;
+	struct macsec_rx_sa *sw_rx_sa;
+	u8 sa_num;
+
+	for (sw_rx_sc = rcu_dereference_bh(secy->rx_sc); sw_rx_sc && sw_rx_sc->active;
+	     sw_rx_sc = rcu_dereference_bh(sw_rx_sc->next)) {
+		mcs_rx_sc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc);
+		if (unlikely(!mcs_rx_sc))
+			continue;
+
+		for (sa_num = 0; sa_num < CN10K_MCS_SA_PER_SC; sa_num++) {
+			sw_rx_sa = rcu_dereference_bh(sw_rx_sc->sa[sa_num]);
+			if (!sw_rx_sa)
+				continue;
+
+			cn10k_mcs_write_rx_sa_plcy(pfvf, secy, mcs_rx_sc,
+						   sa_num, sw_rx_sa->active);
+			cn10k_mcs_write_rx_sa_pn(pfvf, mcs_rx_sc, sa_num,
+						 sw_rx_sa->next_pn_halves.lower);
+		}
+
+		cn10k_mcs_write_rx_flowid(pfvf, mcs_rx_sc, hw_secy_id);
+		cn10k_mcs_write_sc_cam(pfvf, mcs_rx_sc, hw_secy_id);
+	}
+
+	return 0;
+}
+
+static int cn10k_mcs_disable_rxscs(struct otx2_nic *pfvf,
+				   struct macsec_secy *secy,
+				   bool delete)
+{
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct cn10k_mcs_rxsc *mcs_rx_sc;
+	struct macsec_rx_sc *sw_rx_sc;
+	int ret;
+
+	for (sw_rx_sc = rcu_dereference_bh(secy->rx_sc); sw_rx_sc && sw_rx_sc->active;
+	     sw_rx_sc = rcu_dereference_bh(sw_rx_sc->next)) {
+		mcs_rx_sc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc);
+		if (unlikely(!mcs_rx_sc))
+			continue;
+
+		ret = cn10k_mcs_ena_dis_flowid(pfvf, mcs_rx_sc->hw_flow_id,
+					       false, MCS_RX);
+		if (ret)
+			dev_err(pfvf->dev, "Failed to disable TCAM for SC %d\n",
+				mcs_rx_sc->hw_sc_id);
+		if (delete) {
+			cn10k_mcs_delete_rxsc(pfvf, mcs_rx_sc);
+			list_del(&mcs_rx_sc->entry);
+			kfree(mcs_rx_sc);
+		}
+	}
+
+	return 0;
+}
+
+static void cn10k_mcs_sync_stats(struct otx2_nic *pfvf, struct macsec_secy *secy,
+				 struct cn10k_mcs_txsc *txsc)
+{
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct mcs_secy_stats rx_rsp = { 0 };
+	struct mcs_sc_stats sc_rsp = { 0 };
+	struct cn10k_mcs_rxsc *rxsc;
+
+	/* Because of shared counters for some stats in the hardware, when
+	 * updating secy policy take a snapshot of current stats and reset them.
+	 * Below are the effected stats because of shared counters.
+	 */
+
+	/* Check if sync is really needed */
+	if (secy->validate_frames == txsc->last_validate_frames &&
+	    secy->protect_frames == txsc->last_protect_frames)
+		return;
+
+	cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_rx, &rx_rsp, MCS_RX, true);
+
+	txsc->stats.InPktsBadTag += rx_rsp.pkt_badtag_cnt;
+	txsc->stats.InPktsUnknownSCI += rx_rsp.pkt_nosa_cnt;
+	txsc->stats.InPktsNoSCI += rx_rsp.pkt_nosaerror_cnt;
+	if (txsc->last_validate_frames == MACSEC_VALIDATE_STRICT)
+		txsc->stats.InPktsNoTag += rx_rsp.pkt_untaged_cnt;
+	else
+		txsc->stats.InPktsUntagged += rx_rsp.pkt_untaged_cnt;
+
+	list_for_each_entry(rxsc, &cfg->rxsc_list, entry) {
+		cn10k_mcs_sc_stats(pfvf, rxsc->hw_sc_id, &sc_rsp, MCS_RX, true);
+
+		rxsc->stats.InOctetsValidated += sc_rsp.octet_validate_cnt;
+		rxsc->stats.InOctetsDecrypted += sc_rsp.octet_decrypt_cnt;
+
+		rxsc->stats.InPktsInvalid += sc_rsp.pkt_invalid_cnt;
+		rxsc->stats.InPktsNotValid += sc_rsp.pkt_notvalid_cnt;
+
+		if (txsc->last_protect_frames)
+			rxsc->stats.InPktsLate += sc_rsp.pkt_late_cnt;
+		else
+			rxsc->stats.InPktsDelayed += sc_rsp.pkt_late_cnt;
+
+		if (txsc->last_validate_frames == MACSEC_VALIDATE_CHECK)
+			rxsc->stats.InPktsUnchecked += sc_rsp.pkt_unchecked_cnt;
+		else
+			rxsc->stats.InPktsOK += sc_rsp.pkt_unchecked_cnt;
+	}
+
+	txsc->last_validate_frames = secy->validate_frames;
+	txsc->last_protect_frames = secy->protect_frames;
+}
+
+static int cn10k_mdo_open(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_secy *secy = ctx->secy;
+	struct macsec_tx_sa *sw_tx_sa;
+	struct cn10k_mcs_txsc *txsc;
+	u8 sa_num;
+	int err;
+
+	txsc = cn10k_mcs_get_txsc(cfg, ctx->secy);
+	if (!txsc)
+		return -ENOENT;
+
+	sa_num = txsc->encoding_sa;
+	sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[sa_num]);
+
+	err = cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, sw_tx_sa, sa_num);
+	if (err)
+		return err;
+
+	return cn10k_mcs_secy_rx_cfg(pfvf, secy, txsc->hw_secy_id_rx);
+}
+
+static int cn10k_mdo_stop(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct cn10k_mcs_txsc *txsc;
+	int err;
+
+	txsc = cn10k_mcs_get_txsc(cfg, ctx->secy);
+	if (!txsc)
+		return -ENOENT;
+
+	err = cn10k_mcs_ena_dis_flowid(pfvf, txsc->hw_flow_id, false, MCS_TX);
+	if (err)
+		return err;
+
+	return cn10k_mcs_disable_rxscs(pfvf, ctx->secy, false);
+}
+
+static int cn10k_mdo_add_secy(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_secy *secy = ctx->secy;
+	struct cn10k_mcs_txsc *txsc;
+
+	if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN)
+		return -EOPNOTSUPP;
+
+	/* Stick to 16 bytes key len until XPN support is added */
+	if (secy->key_len != 16)
+		return -EOPNOTSUPP;
+
+	if (secy->xpn)
+		return -EOPNOTSUPP;
+
+	txsc = cn10k_mcs_create_txsc(pfvf);
+	if (IS_ERR(txsc))
+		return -ENOSPC;
+
+	txsc->sw_secy = secy;
+	txsc->encoding_sa = secy->tx_sc.encoding_sa;
+	txsc->last_validate_frames = secy->validate_frames;
+	txsc->last_protect_frames = secy->protect_frames;
+
+	list_add(&txsc->entry, &cfg->txsc_list);
+
+	if (netif_running(secy->netdev))
+		return cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, NULL, 0);
+
+	return 0;
+}
+
+static int cn10k_mdo_upd_secy(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_secy *secy = ctx->secy;
+	struct macsec_tx_sa *sw_tx_sa;
+	struct cn10k_mcs_txsc *txsc;
+	u8 sa_num;
+	int err;
+
+	txsc = cn10k_mcs_get_txsc(cfg, secy);
+	if (!txsc)
+		return -ENOENT;
+
+	txsc->encoding_sa = secy->tx_sc.encoding_sa;
+
+	sa_num = txsc->encoding_sa;
+	sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[sa_num]);
+
+	if (netif_running(secy->netdev)) {
+		cn10k_mcs_sync_stats(pfvf, secy, txsc);
+
+		err = cn10k_mcs_secy_tx_cfg(pfvf, secy, txsc, sw_tx_sa, sa_num);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int cn10k_mdo_del_secy(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct cn10k_mcs_txsc *txsc;
+
+	txsc = cn10k_mcs_get_txsc(cfg, ctx->secy);
+	if (!txsc)
+		return -ENOENT;
+
+	cn10k_mcs_ena_dis_flowid(pfvf, txsc->hw_flow_id, false, MCS_TX);
+	cn10k_mcs_disable_rxscs(pfvf, ctx->secy, true);
+	cn10k_mcs_delete_txsc(pfvf, txsc);
+	list_del(&txsc->entry);
+	kfree(txsc);
+
+	return 0;
+}
+
+static int cn10k_mdo_add_txsa(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct macsec_tx_sa *sw_tx_sa = ctx->sa.tx_sa;
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_secy *secy = ctx->secy;
+	u8 sa_num = ctx->sa.assoc_num;
+	struct cn10k_mcs_txsc *txsc;
+	int err;
+
+	txsc = cn10k_mcs_get_txsc(cfg, secy);
+	if (!txsc)
+		return -ENOENT;
+
+	if (sa_num >= CN10K_MCS_SA_PER_SC)
+		return -EOPNOTSUPP;
+
+	if (cn10k_mcs_alloc_txsa(pfvf, &txsc->hw_sa_id[sa_num]))
+		return -ENOSPC;
+
+	memcpy(&txsc->sa_key[sa_num], ctx->sa.key, secy->key_len);
+	txsc->sa_bmap |= 1 << sa_num;
+
+	if (netif_running(secy->netdev)) {
+		err = cn10k_mcs_write_tx_sa_plcy(pfvf, secy, txsc, sa_num);
+		if (err)
+			return err;
+
+		err = cn10k_write_tx_sa_pn(pfvf, txsc, sa_num,
+					   sw_tx_sa->next_pn_halves.lower);
+		if (err)
+			return err;
+
+		err = cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc,
+					      sa_num, sw_tx_sa->active);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int cn10k_mdo_upd_txsa(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct macsec_tx_sa *sw_tx_sa = ctx->sa.tx_sa;
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_secy *secy = ctx->secy;
+	u8 sa_num = ctx->sa.assoc_num;
+	struct cn10k_mcs_txsc *txsc;
+	int err;
+
+	txsc = cn10k_mcs_get_txsc(cfg, secy);
+	if (!txsc)
+		return -ENOENT;
+
+	if (sa_num >= CN10K_MCS_SA_PER_SC)
+		return -EOPNOTSUPP;
+
+	if (netif_running(secy->netdev)) {
+		/* Keys cannot be changed after creation */
+		err = cn10k_write_tx_sa_pn(pfvf, txsc, sa_num,
+					   sw_tx_sa->next_pn_halves.lower);
+		if (err)
+			return err;
+
+		err = cn10k_mcs_link_tx_sa2sc(pfvf, secy, txsc,
+					      sa_num, sw_tx_sa->active);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int cn10k_mdo_del_txsa(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	u8 sa_num = ctx->sa.assoc_num;
+	struct cn10k_mcs_txsc *txsc;
+
+	txsc = cn10k_mcs_get_txsc(cfg, ctx->secy);
+	if (!txsc)
+		return -ENOENT;
+
+	if (sa_num >= CN10K_MCS_SA_PER_SC)
+		return -EOPNOTSUPP;
+
+	cn10k_mcs_free_txsa(pfvf, txsc->hw_sa_id[sa_num]);
+	txsc->sa_bmap &= ~(1 << sa_num);
+
+	return 0;
+}
+
+static int cn10k_mdo_add_rxsc(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_secy *secy = ctx->secy;
+	struct cn10k_mcs_rxsc *rxsc;
+	struct cn10k_mcs_txsc *txsc;
+	int err;
+
+	txsc = cn10k_mcs_get_txsc(cfg, secy);
+	if (!txsc)
+		return -ENOENT;
+
+	rxsc = cn10k_mcs_create_rxsc(pfvf);
+	if (IS_ERR(rxsc))
+		return -ENOSPC;
+
+	rxsc->sw_secy = ctx->secy;
+	rxsc->sw_rxsc = ctx->rx_sc;
+	list_add(&rxsc->entry, &cfg->rxsc_list);
+
+	if (netif_running(secy->netdev)) {
+		err = cn10k_mcs_write_rx_flowid(pfvf, rxsc, txsc->hw_secy_id_rx);
+		if (err)
+			return err;
+
+		err = cn10k_mcs_write_sc_cam(pfvf, rxsc, txsc->hw_secy_id_rx);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int cn10k_mdo_upd_rxsc(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_secy *secy = ctx->secy;
+	bool enable = ctx->rx_sc->active;
+	struct cn10k_mcs_rxsc *rxsc;
+
+	rxsc = cn10k_mcs_get_rxsc(cfg, secy, ctx->rx_sc);
+	if (!rxsc)
+		return -ENOENT;
+
+	if (netif_running(secy->netdev))
+		return cn10k_mcs_ena_dis_flowid(pfvf, rxsc->hw_flow_id,
+						enable, MCS_RX);
+
+	return 0;
+}
+
+static int cn10k_mdo_del_rxsc(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct cn10k_mcs_rxsc *rxsc;
+
+	rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, ctx->rx_sc);
+	if (!rxsc)
+		return -ENOENT;
+
+	cn10k_mcs_ena_dis_flowid(pfvf, rxsc->hw_flow_id, false, MCS_RX);
+	cn10k_mcs_delete_rxsc(pfvf, rxsc);
+	list_del(&rxsc->entry);
+	kfree(rxsc);
+
+	return 0;
+}
+
+static int cn10k_mdo_add_rxsa(struct macsec_context *ctx)
+{
+	struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc;
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_rx_sa *rx_sa = ctx->sa.rx_sa;
+	u64 next_pn = rx_sa->next_pn_halves.lower;
+	struct macsec_secy *secy = ctx->secy;
+	bool sa_in_use = rx_sa->active;
+	u8 sa_num = ctx->sa.assoc_num;
+	struct cn10k_mcs_rxsc *rxsc;
+	int err;
+
+	rxsc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc);
+	if (!rxsc)
+		return -ENOENT;
+
+	if (sa_num >= CN10K_MCS_SA_PER_SC)
+		return -EOPNOTSUPP;
+
+	if (cn10k_mcs_alloc_rxsa(pfvf, &rxsc->hw_sa_id[sa_num]))
+		return -ENOSPC;
+
+	memcpy(&rxsc->sa_key[sa_num], ctx->sa.key, ctx->secy->key_len);
+	rxsc->sa_bmap |= 1 << sa_num;
+
+	if (netif_running(secy->netdev)) {
+		err = cn10k_mcs_write_rx_sa_plcy(pfvf, secy, rxsc,
+						 sa_num, sa_in_use);
+		if (err)
+			return err;
+
+		err = cn10k_mcs_write_rx_sa_pn(pfvf, rxsc, sa_num, next_pn);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int cn10k_mdo_upd_rxsa(struct macsec_context *ctx)
+{
+	struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc;
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_rx_sa *rx_sa = ctx->sa.rx_sa;
+	u64 next_pn = rx_sa->next_pn_halves.lower;
+	struct macsec_secy *secy = ctx->secy;
+	bool sa_in_use = rx_sa->active;
+	u8 sa_num = ctx->sa.assoc_num;
+	struct cn10k_mcs_rxsc *rxsc;
+	int err;
+
+	rxsc = cn10k_mcs_get_rxsc(cfg, secy, sw_rx_sc);
+	if (!rxsc)
+		return -ENOENT;
+
+	if (sa_num >= CN10K_MCS_SA_PER_SC)
+		return -EOPNOTSUPP;
+
+	if (netif_running(secy->netdev)) {
+		err = cn10k_mcs_write_rx_sa_plcy(pfvf, secy, rxsc, sa_num, sa_in_use);
+		if (err)
+			return err;
+
+		err = cn10k_mcs_write_rx_sa_pn(pfvf, rxsc, sa_num, next_pn);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int cn10k_mdo_del_rxsa(struct macsec_context *ctx)
+{
+	struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc;
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	u8 sa_num = ctx->sa.assoc_num;
+	struct cn10k_mcs_rxsc *rxsc;
+
+	rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, sw_rx_sc);
+	if (!rxsc)
+		return -ENOENT;
+
+	if (sa_num >= CN10K_MCS_SA_PER_SC)
+		return -EOPNOTSUPP;
+
+	cn10k_mcs_write_rx_sa_plcy(pfvf, ctx->secy, rxsc, sa_num, false);
+	cn10k_mcs_free_rxsa(pfvf, rxsc->hw_sa_id[sa_num]);
+
+	rxsc->sa_bmap &= ~(1 << sa_num);
+
+	return 0;
+}
+
+static int cn10k_mdo_get_dev_stats(struct macsec_context *ctx)
+{
+	struct mcs_secy_stats tx_rsp = { 0 }, rx_rsp = { 0 };
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_secy *secy = ctx->secy;
+	struct cn10k_mcs_txsc *txsc;
+
+	txsc = cn10k_mcs_get_txsc(cfg, ctx->secy);
+	if (!txsc)
+		return -ENOENT;
+
+	cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_tx, &tx_rsp, MCS_TX, false);
+	ctx->stats.dev_stats->OutPktsUntagged = tx_rsp.pkt_untagged_cnt;
+	ctx->stats.dev_stats->OutPktsTooLong = tx_rsp.pkt_toolong_cnt;
+
+	cn10k_mcs_secy_stats(pfvf, txsc->hw_secy_id_rx, &rx_rsp, MCS_RX, true);
+	txsc->stats.InPktsBadTag += rx_rsp.pkt_badtag_cnt;
+	txsc->stats.InPktsUnknownSCI += rx_rsp.pkt_nosa_cnt;
+	txsc->stats.InPktsNoSCI += rx_rsp.pkt_nosaerror_cnt;
+	if (secy->validate_frames == MACSEC_VALIDATE_STRICT)
+		txsc->stats.InPktsNoTag += rx_rsp.pkt_untaged_cnt;
+	else
+		txsc->stats.InPktsUntagged += rx_rsp.pkt_untaged_cnt;
+	txsc->stats.InPktsOverrun = 0;
+
+	ctx->stats.dev_stats->InPktsNoTag = txsc->stats.InPktsNoTag;
+	ctx->stats.dev_stats->InPktsUntagged = txsc->stats.InPktsUntagged;
+	ctx->stats.dev_stats->InPktsBadTag = txsc->stats.InPktsBadTag;
+	ctx->stats.dev_stats->InPktsUnknownSCI = txsc->stats.InPktsUnknownSCI;
+	ctx->stats.dev_stats->InPktsNoSCI = txsc->stats.InPktsNoSCI;
+	ctx->stats.dev_stats->InPktsOverrun = txsc->stats.InPktsOverrun;
+
+	return 0;
+}
+
+static int cn10k_mdo_get_tx_sc_stats(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct mcs_sc_stats rsp = { 0 };
+	struct cn10k_mcs_txsc *txsc;
+
+	txsc = cn10k_mcs_get_txsc(cfg, ctx->secy);
+	if (!txsc)
+		return -ENOENT;
+
+	cn10k_mcs_sc_stats(pfvf, txsc->hw_sc_id, &rsp, MCS_TX, false);
+
+	ctx->stats.tx_sc_stats->OutPktsProtected = rsp.pkt_protected_cnt;
+	ctx->stats.tx_sc_stats->OutPktsEncrypted = rsp.pkt_encrypt_cnt;
+	ctx->stats.tx_sc_stats->OutOctetsProtected = rsp.octet_protected_cnt;
+	ctx->stats.tx_sc_stats->OutOctetsEncrypted = rsp.octet_encrypt_cnt;
+
+	return 0;
+}
+
+static int cn10k_mdo_get_tx_sa_stats(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct mcs_sa_stats rsp = { 0 };
+	u8 sa_num = ctx->sa.assoc_num;
+	struct cn10k_mcs_txsc *txsc;
+
+	txsc = cn10k_mcs_get_txsc(cfg, ctx->secy);
+	if (!txsc)
+		return -ENOENT;
+
+	if (sa_num >= CN10K_MCS_SA_PER_SC)
+		return -EOPNOTSUPP;
+
+	cn10k_mcs_sa_stats(pfvf, txsc->hw_sa_id[sa_num], &rsp, MCS_TX, false);
+
+	ctx->stats.tx_sa_stats->OutPktsProtected = rsp.pkt_protected_cnt;
+	ctx->stats.tx_sa_stats->OutPktsEncrypted = rsp.pkt_encrypt_cnt;
+
+	return 0;
+}
+
+static int cn10k_mdo_get_rx_sc_stats(struct macsec_context *ctx)
+{
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_secy *secy = ctx->secy;
+	struct mcs_sc_stats rsp = { 0 };
+	struct cn10k_mcs_rxsc *rxsc;
+
+	rxsc = cn10k_mcs_get_rxsc(cfg, secy, ctx->rx_sc);
+	if (!rxsc)
+		return -ENOENT;
+
+	cn10k_mcs_sc_stats(pfvf, rxsc->hw_sc_id, &rsp, MCS_RX, true);
+
+	rxsc->stats.InOctetsValidated += rsp.octet_validate_cnt;
+	rxsc->stats.InOctetsDecrypted += rsp.octet_decrypt_cnt;
+
+	rxsc->stats.InPktsInvalid += rsp.pkt_invalid_cnt;
+	rxsc->stats.InPktsNotValid += rsp.pkt_notvalid_cnt;
+
+	if (secy->protect_frames)
+		rxsc->stats.InPktsLate += rsp.pkt_late_cnt;
+	else
+		rxsc->stats.InPktsDelayed += rsp.pkt_late_cnt;
+
+	if (secy->validate_frames == MACSEC_VALIDATE_CHECK)
+		rxsc->stats.InPktsUnchecked += rsp.pkt_unchecked_cnt;
+	else
+		rxsc->stats.InPktsOK += rsp.pkt_unchecked_cnt;
+
+	ctx->stats.rx_sc_stats->InOctetsValidated = rxsc->stats.InOctetsValidated;
+	ctx->stats.rx_sc_stats->InOctetsDecrypted = rxsc->stats.InOctetsDecrypted;
+	ctx->stats.rx_sc_stats->InPktsInvalid = rxsc->stats.InPktsInvalid;
+	ctx->stats.rx_sc_stats->InPktsNotValid = rxsc->stats.InPktsNotValid;
+	ctx->stats.rx_sc_stats->InPktsLate = rxsc->stats.InPktsLate;
+	ctx->stats.rx_sc_stats->InPktsDelayed = rxsc->stats.InPktsDelayed;
+	ctx->stats.rx_sc_stats->InPktsUnchecked = rxsc->stats.InPktsUnchecked;
+	ctx->stats.rx_sc_stats->InPktsOK = rxsc->stats.InPktsOK;
+
+	return 0;
+}
+
+static int cn10k_mdo_get_rx_sa_stats(struct macsec_context *ctx)
+{
+	struct macsec_rx_sc *sw_rx_sc = ctx->sa.rx_sa->sc;
+	struct otx2_nic *pfvf = netdev_priv(ctx->netdev);
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct mcs_sa_stats rsp = { 0 };
+	u8 sa_num = ctx->sa.assoc_num;
+	struct cn10k_mcs_rxsc *rxsc;
+
+	rxsc = cn10k_mcs_get_rxsc(cfg, ctx->secy, sw_rx_sc);
+	if (!rxsc)
+		return -ENOENT;
+
+	if (sa_num >= CN10K_MCS_SA_PER_SC)
+		return -EOPNOTSUPP;
+
+	cn10k_mcs_sa_stats(pfvf, rxsc->hw_sa_id[sa_num], &rsp, MCS_RX, false);
+
+	ctx->stats.rx_sa_stats->InPktsOK = rsp.pkt_ok_cnt;
+	ctx->stats.rx_sa_stats->InPktsInvalid = rsp.pkt_invalid_cnt;
+	ctx->stats.rx_sa_stats->InPktsNotValid = rsp.pkt_notvalid_cnt;
+	ctx->stats.rx_sa_stats->InPktsNotUsingSA = rsp.pkt_nosaerror_cnt;
+	ctx->stats.rx_sa_stats->InPktsUnusedSA = rsp.pkt_nosa_cnt;
+
+	return 0;
+}
+
+static const struct macsec_ops cn10k_mcs_ops = {
+	.mdo_dev_open = cn10k_mdo_open,
+	.mdo_dev_stop = cn10k_mdo_stop,
+	.mdo_add_secy = cn10k_mdo_add_secy,
+	.mdo_upd_secy = cn10k_mdo_upd_secy,
+	.mdo_del_secy = cn10k_mdo_del_secy,
+	.mdo_add_rxsc = cn10k_mdo_add_rxsc,
+	.mdo_upd_rxsc = cn10k_mdo_upd_rxsc,
+	.mdo_del_rxsc = cn10k_mdo_del_rxsc,
+	.mdo_add_rxsa = cn10k_mdo_add_rxsa,
+	.mdo_upd_rxsa = cn10k_mdo_upd_rxsa,
+	.mdo_del_rxsa = cn10k_mdo_del_rxsa,
+	.mdo_add_txsa = cn10k_mdo_add_txsa,
+	.mdo_upd_txsa = cn10k_mdo_upd_txsa,
+	.mdo_del_txsa = cn10k_mdo_del_txsa,
+	.mdo_get_dev_stats = cn10k_mdo_get_dev_stats,
+	.mdo_get_tx_sc_stats = cn10k_mdo_get_tx_sc_stats,
+	.mdo_get_tx_sa_stats = cn10k_mdo_get_tx_sa_stats,
+	.mdo_get_rx_sc_stats = cn10k_mdo_get_rx_sc_stats,
+	.mdo_get_rx_sa_stats = cn10k_mdo_get_rx_sa_stats,
+};
+
+void cn10k_handle_mcs_event(struct otx2_nic *pfvf, struct mcs_intr_info *event)
+{
+	struct cn10k_mcs_cfg *cfg = pfvf->macsec_cfg;
+	struct macsec_tx_sa *sw_tx_sa = NULL;
+	struct macsec_secy *secy = NULL;
+	struct cn10k_mcs_txsc *txsc;
+	u8 an;
+
+	if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag))
+		return;
+
+	if (!(event->intr_mask & MCS_CPM_TX_PACKET_XPN_EQ0_INT))
+		return;
+
+	/* Find the SecY to which the expired hardware SA is mapped */
+	list_for_each_entry(txsc, &cfg->txsc_list, entry) {
+		for (an = 0; an < CN10K_MCS_SA_PER_SC; an++)
+			if (txsc->hw_sa_id[an] == event->sa_id) {
+				secy = txsc->sw_secy;
+				sw_tx_sa = rcu_dereference_bh(secy->tx_sc.sa[an]);
+			}
+	}
+
+	if (secy && sw_tx_sa)
+		macsec_pn_wrapped(secy, sw_tx_sa);
+}
+
+int cn10k_mcs_init(struct otx2_nic *pfvf)
+{
+	struct mbox *mbox = &pfvf->mbox;
+	struct cn10k_mcs_cfg *cfg;
+	struct mcs_intr_cfg *req;
+
+	if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag))
+		return 0;
+
+	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
+	if (!cfg)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&cfg->txsc_list);
+	INIT_LIST_HEAD(&cfg->rxsc_list);
+	pfvf->macsec_cfg = cfg;
+
+	pfvf->netdev->features |= NETIF_F_HW_MACSEC;
+	pfvf->netdev->macsec_ops = &cn10k_mcs_ops;
+
+	mutex_lock(&mbox->lock);
+
+	req = otx2_mbox_alloc_msg_mcs_intr_cfg(mbox);
+	if (!req)
+		goto fail;
+
+	req->intr_mask = MCS_CPM_TX_PACKET_XPN_EQ0_INT;
+
+	if (otx2_sync_mbox_msg(mbox))
+		goto fail;
+
+	mutex_unlock(&mbox->lock);
+
+	return 0;
+fail:
+	dev_err(pfvf->dev, "Cannot notify PN wrapped event\n");
+	return 0;
+}
+
+void cn10k_mcs_free(struct otx2_nic *pfvf)
+{
+	if (!test_bit(CN10K_HW_MACSEC, &pfvf->hw.cap_flag))
+		return;
+
+	cn10k_mcs_free_rsrc(pfvf, MCS_TX, MCS_RSRC_TYPE_SECY, 0, true);
+	cn10k_mcs_free_rsrc(pfvf, MCS_RX, MCS_RSRC_TYPE_SECY, 0, true);
+	kfree(pfvf->macsec_cfg);
+	pfvf->macsec_cfg = NULL;
+}
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index d686c7b6252f..9ac9e6615ae7 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -586,8 +586,9 @@ void otx2_get_mac_from_af(struct net_device *netdev)
 }
 EXPORT_SYMBOL(otx2_get_mac_from_af);
 
-int otx2_txschq_config(struct otx2_nic *pfvf, int lvl)
+int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool txschq_for_pfc)
 {
+	u16 (*schq_list)[MAX_TXSCHQ_PER_FUNC];
 	struct otx2_hw *hw = &pfvf->hw;
 	struct nix_txschq_config *req;
 	u64 schq, parent;
@@ -602,7 +603,13 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl)
 	req->lvl = lvl;
 	req->num_regs = 1;
 
-	schq = hw->txschq_list[lvl][0];
+	schq_list = hw->txschq_list;
+#ifdef CONFIG_DCB
+	if (txschq_for_pfc)
+		schq_list = pfvf->pfc_schq_list;
+#endif
+
+	schq = schq_list[lvl][prio];
 	/* Set topology e.t.c configuration */
 	if (lvl == NIX_TXSCH_LVL_SMQ) {
 		req->reg[0] = NIX_AF_SMQX_CFG(schq);
@@ -611,7 +618,7 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl)
 				  (0x2ULL << 36);
 		req->num_regs++;
 		/* MDQ config */
-		parent =  hw->txschq_list[NIX_TXSCH_LVL_TL4][0];
+		parent = schq_list[NIX_TXSCH_LVL_TL4][prio];
 		req->reg[1] = NIX_AF_MDQX_PARENT(schq);
 		req->regval[1] = parent << 16;
 		req->num_regs++;
@@ -619,14 +626,14 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl)
 		req->reg[2] = NIX_AF_MDQX_SCHEDULE(schq);
 		req->regval[2] =  dwrr_val;
 	} else if (lvl == NIX_TXSCH_LVL_TL4) {
-		parent =  hw->txschq_list[NIX_TXSCH_LVL_TL3][0];
+		parent = schq_list[NIX_TXSCH_LVL_TL3][prio];
 		req->reg[0] = NIX_AF_TL4X_PARENT(schq);
 		req->regval[0] = parent << 16;
 		req->num_regs++;
 		req->reg[1] = NIX_AF_TL4X_SCHEDULE(schq);
 		req->regval[1] = dwrr_val;
 	} else if (lvl == NIX_TXSCH_LVL_TL3) {
-		parent = hw->txschq_list[NIX_TXSCH_LVL_TL2][0];
+		parent = schq_list[NIX_TXSCH_LVL_TL2][prio];
 		req->reg[0] = NIX_AF_TL3X_PARENT(schq);
 		req->regval[0] = parent << 16;
 		req->num_regs++;
@@ -635,11 +642,13 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl)
 		if (lvl == hw->txschq_link_cfg_lvl) {
 			req->num_regs++;
 			req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link);
-			/* Enable this queue and backpressure */
-			req->regval[2] = BIT_ULL(13) | BIT_ULL(12);
+			/* Enable this queue and backpressure
+			 * and set relative channel
+			 */
+			req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio;
 		}
 	} else if (lvl == NIX_TXSCH_LVL_TL2) {
-		parent =  hw->txschq_list[NIX_TXSCH_LVL_TL1][0];
+		parent = schq_list[NIX_TXSCH_LVL_TL1][prio];
 		req->reg[0] = NIX_AF_TL2X_PARENT(schq);
 		req->regval[0] = parent << 16;
 
@@ -650,8 +659,10 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl)
 		if (lvl == hw->txschq_link_cfg_lvl) {
 			req->num_regs++;
 			req->reg[2] = NIX_AF_TL3_TL2X_LINKX_CFG(schq, hw->tx_link);
-			/* Enable this queue and backpressure */
-			req->regval[2] = BIT_ULL(13) | BIT_ULL(12);
+			/* Enable this queue and backpressure
+			 * and set relative channel
+			 */
+			req->regval[2] = BIT_ULL(13) | BIT_ULL(12) | prio;
 		}
 	} else if (lvl == NIX_TXSCH_LVL_TL1) {
 		/* Default config for TL1.
@@ -676,6 +687,31 @@ int otx2_txschq_config(struct otx2_nic *pfvf, int lvl)
 
 	return otx2_sync_mbox_msg(&pfvf->mbox);
 }
+EXPORT_SYMBOL(otx2_txschq_config);
+
+int otx2_smq_flush(struct otx2_nic *pfvf, int smq)
+{
+	struct nix_txschq_config *req;
+	int rc;
+
+	mutex_lock(&pfvf->mbox.lock);
+
+	req = otx2_mbox_alloc_msg_nix_txschq_cfg(&pfvf->mbox);
+	if (!req) {
+		mutex_unlock(&pfvf->mbox.lock);
+		return -ENOMEM;
+	}
+
+	req->lvl = NIX_TXSCH_LVL_SMQ;
+	req->reg[0] = NIX_AF_SMQX_CFG(smq);
+	req->regval[0] |= BIT_ULL(49);
+	req->num_regs++;
+
+	rc = otx2_sync_mbox_msg(&pfvf->mbox);
+	mutex_unlock(&pfvf->mbox.lock);
+	return rc;
+}
+EXPORT_SYMBOL(otx2_smq_flush);
 
 int otx2_txsch_alloc(struct otx2_nic *pfvf)
 {
@@ -806,8 +842,7 @@ int otx2_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura)
 	aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */
 	aq->sq.cq_ena = 1;
 	aq->sq.ena = 1;
-	/* Only one SMQ is allocated, map all SQ's to that SMQ  */
-	aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0];
+	aq->sq.smq = otx2_get_smq_idx(pfvf, qidx);
 	aq->sq.smq_rr_quantum = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen);
 	aq->sq.default_chan = pfvf->hw.tx_chan_base;
 	aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */
@@ -1792,4 +1827,5 @@ otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf,		\
 }									\
 EXPORT_SYMBOL(otx2_mbox_up_handler_ ## _fn_name);
 MBOX_UP_CGX_MESSAGES
+MBOX_UP_MCS_MESSAGES
 #undef M
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index b28029cc4316..282db6fe3b08 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -19,6 +19,7 @@
 #include <net/devlink.h>
 #include <linux/time64.h>
 #include <linux/dim.h>
+#include <uapi/linux/if_macsec.h>
 
 #include <mbox.h>
 #include <npc.h>
@@ -33,6 +34,7 @@
 #define PCI_DEVID_OCTEONTX2_RVU_AFVF		0xA0F8
 
 #define PCI_SUBSYS_DEVID_96XX_RVU_PFVF		0xB200
+#define PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF	0xBD00
 
 /* PCI BAR nos */
 #define PCI_CFG_REG_BAR_NUM                     2
@@ -40,6 +42,11 @@
 
 #define NAME_SIZE                               32
 
+#ifdef CONFIG_DCB
+/* Max priority supported for PFC */
+#define NIX_PF_PFC_PRIO_MAX			8
+#endif
+
 enum arua_mapped_qtypes {
 	AURA_NIX_RQ,
 	AURA_NIX_SQ,
@@ -196,7 +203,7 @@ struct otx2_hw {
 
 	/* NIX */
 	u8			txschq_link_cfg_lvl;
-	u16		txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC];
+	u16			txschq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC];
 	u16			matchall_ipolicer;
 	u32			dwrr_mtu;
 
@@ -238,6 +245,8 @@ struct otx2_hw {
 #define CN10K_MBOX		1
 #define CN10K_LMTST		2
 #define CN10K_RPM		3
+#define CN10K_PTP_ONESTEP	4
+#define CN10K_HW_MACSEC		5
 	unsigned long		cap_flag;
 
 #define LMT_LINE_SIZE		128
@@ -271,6 +280,13 @@ struct refill_work {
 	struct otx2_nic *pf;
 };
 
+/* PTPv2 originTimestamp structure */
+struct ptpv2_tstamp {
+	__be16 seconds_msb; /* 16 bits + */
+	__be32 seconds_lsb; /* 32 bits = 48 bits*/
+	__be32 nanoseconds;
+} __packed;
+
 struct otx2_ptp {
 	struct ptp_clock_info ptp_info;
 	struct ptp_clock *ptp_clock;
@@ -286,6 +302,9 @@ struct otx2_ptp {
 	struct ptp_pin_desc extts_config;
 	u64 (*convert_rx_ptp_tstmp)(u64 timestamp);
 	u64 (*convert_tx_ptp_tstmp)(u64 timestamp);
+	struct delayed_work synctstamp_work;
+	u64 tstamp;
+	u32 base_ns;
 };
 
 #define OTX2_HW_TIMESTAMP_LEN	8
@@ -335,6 +354,66 @@ struct dev_hw_ops {
 	void	(*aura_freeptr)(void *dev, int aura, u64 buf);
 };
 
+#define CN10K_MCS_SA_PER_SC	4
+
+/* Stats which need to be accumulated in software because
+ * of shared counters in hardware.
+ */
+struct cn10k_txsc_stats {
+	u64 InPktsUntagged;
+	u64 InPktsNoTag;
+	u64 InPktsBadTag;
+	u64 InPktsUnknownSCI;
+	u64 InPktsNoSCI;
+	u64 InPktsOverrun;
+};
+
+struct cn10k_rxsc_stats {
+	u64 InOctetsValidated;
+	u64 InOctetsDecrypted;
+	u64 InPktsUnchecked;
+	u64 InPktsDelayed;
+	u64 InPktsOK;
+	u64 InPktsInvalid;
+	u64 InPktsLate;
+	u64 InPktsNotValid;
+	u64 InPktsNotUsingSA;
+	u64 InPktsUnusedSA;
+};
+
+struct cn10k_mcs_txsc {
+	struct macsec_secy *sw_secy;
+	struct cn10k_txsc_stats stats;
+	struct list_head entry;
+	enum macsec_validation_type last_validate_frames;
+	bool last_protect_frames;
+	u16 hw_secy_id_tx;
+	u16 hw_secy_id_rx;
+	u16 hw_flow_id;
+	u16 hw_sc_id;
+	u16 hw_sa_id[CN10K_MCS_SA_PER_SC];
+	u8 sa_bmap;
+	u8 sa_key[CN10K_MCS_SA_PER_SC][MACSEC_MAX_KEY_LEN];
+	u8 encoding_sa;
+};
+
+struct cn10k_mcs_rxsc {
+	struct macsec_secy *sw_secy;
+	struct macsec_rx_sc *sw_rxsc;
+	struct cn10k_rxsc_stats stats;
+	struct list_head entry;
+	u16 hw_flow_id;
+	u16 hw_sc_id;
+	u16 hw_sa_id[CN10K_MCS_SA_PER_SC];
+	u8 sa_bmap;
+	u8 sa_key[CN10K_MCS_SA_PER_SC][MACSEC_MAX_KEY_LEN];
+};
+
+struct cn10k_mcs_cfg {
+	struct list_head txsc_list;
+	struct list_head rxsc_list;
+};
+
 struct otx2_nic {
 	void __iomem		*reg_base;
 	struct net_device	*netdev;
@@ -358,6 +437,7 @@ struct otx2_nic {
 #define OTX2_FLAG_TC_MATCHALL_EGRESS_ENABLED	BIT_ULL(12)
 #define OTX2_FLAG_TC_MATCHALL_INGRESS_ENABLED	BIT_ULL(13)
 #define OTX2_FLAG_DMACFLTR_SUPPORT		BIT_ULL(14)
+#define OTX2_FLAG_PTP_ONESTEP_SYNC		BIT_ULL(15)
 #define OTX2_FLAG_ADPTV_INT_COAL_ENABLED BIT_ULL(16)
 	u64			flags;
 	u64			*cq_op_addr;
@@ -415,10 +495,16 @@ struct otx2_nic {
 	/* PFC */
 	u8			pfc_en;
 	u8			*queue_to_pfc_map;
+	u16			pfc_schq_list[NIX_TXSCH_LVL_CNT][MAX_TXSCHQ_PER_FUNC];
+	bool			pfc_alloc_status[NIX_PF_PFC_PRIO_MAX];
 #endif
 
 	/* napi event count. It is needed for adaptive irq coalescing. */
 	u32 napi_events;
+
+#if IS_ENABLED(CONFIG_MACSEC)
+	struct cn10k_mcs_cfg	*macsec_cfg;
+#endif
 };
 
 static inline bool is_otx2_lbkvf(struct pci_dev *pdev)
@@ -458,6 +544,11 @@ static inline bool is_dev_otx2(struct pci_dev *pdev)
 		midr == PCI_REVISION_ID_95XXMM || midr == PCI_REVISION_ID_95XXO);
 }
 
+static inline bool is_dev_cn10kb(struct pci_dev *pdev)
+{
+	return pdev->subsystem_device == PCI_SUBSYS_DEVID_CN10K_B_RVU_PFVF;
+}
+
 static inline void otx2_setup_dev_hw_settings(struct otx2_nic *pfvf)
 {
 	struct otx2_hw *hw = &pfvf->hw;
@@ -487,7 +578,11 @@ static inline void otx2_setup_dev_hw_settings(struct otx2_nic *pfvf)
 		__set_bit(CN10K_MBOX, &hw->cap_flag);
 		__set_bit(CN10K_LMTST, &hw->cap_flag);
 		__set_bit(CN10K_RPM, &hw->cap_flag);
+		__set_bit(CN10K_PTP_ONESTEP, &hw->cap_flag);
 	}
+
+	if (is_dev_cn10kb(pfvf->pdev))
+		__set_bit(CN10K_HW_MACSEC, &hw->cap_flag);
 }
 
 /* Register read/write APIs */
@@ -743,6 +838,7 @@ otx2_mbox_up_handler_ ## _fn_name(struct otx2_nic *pfvf,		\
 				struct _rsp_type *rsp);			\
 
 MBOX_UP_CGX_MESSAGES
+MBOX_UP_MCS_MESSAGES
 #undef M
 
 /* Time to wait before watchdog kicks off */
@@ -785,6 +881,16 @@ static inline void otx2_dma_unmap_page(struct otx2_nic *pfvf,
 			     dir, DMA_ATTR_SKIP_CPU_SYNC);
 }
 
+static inline u16 otx2_get_smq_idx(struct otx2_nic *pfvf, u16 qidx)
+{
+#ifdef CONFIG_DCB
+	if (pfvf->pfc_alloc_status[qidx])
+		return pfvf->pfc_schq_list[NIX_TXSCH_LVL_SMQ][qidx];
+#endif
+
+	return pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0];
+}
+
 /* MSI-X APIs */
 void otx2_free_cints(struct otx2_nic *pfvf, int n);
 void otx2_set_cints_affinity(struct otx2_nic *pfvf);
@@ -807,7 +913,7 @@ void otx2_free_aura_ptr(struct otx2_nic *pfvf, int type);
 void otx2_sq_free_sqbs(struct otx2_nic *pfvf);
 int otx2_config_nix(struct otx2_nic *pfvf);
 int otx2_config_nix_queues(struct otx2_nic *pfvf);
-int otx2_txschq_config(struct otx2_nic *pfvf, int lvl);
+int otx2_txschq_config(struct otx2_nic *pfvf, int lvl, int prio, bool pfc_en);
 int otx2_txsch_alloc(struct otx2_nic *pfvf);
 int otx2_txschq_stop(struct otx2_nic *pfvf);
 void otx2_sqb_flush(struct otx2_nic *pfvf);
@@ -888,6 +994,8 @@ bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx);
 u16 otx2_get_max_mtu(struct otx2_nic *pfvf);
 int otx2_handle_ntuple_tc_features(struct net_device *netdev,
 				   netdev_features_t features);
+int otx2_smq_flush(struct otx2_nic *pfvf, int smq);
+
 /* tc support */
 int otx2_init_tc(struct otx2_nic *nic);
 void otx2_shutdown_tc(struct otx2_nic *nic);
@@ -907,5 +1015,24 @@ void otx2_dmacflt_update_pfmac_flow(struct otx2_nic *pfvf);
 void otx2_update_bpid_in_rqctx(struct otx2_nic *pfvf, int vlan_prio, int qidx, bool pfc_enable);
 int otx2_config_priority_flow_ctrl(struct otx2_nic *pfvf);
 int otx2_dcbnl_set_ops(struct net_device *dev);
+/* PFC support */
+int otx2_pfc_txschq_config(struct otx2_nic *pfvf);
+int otx2_pfc_txschq_alloc(struct otx2_nic *pfvf);
+int otx2_pfc_txschq_update(struct otx2_nic *pfvf);
+int otx2_pfc_txschq_stop(struct otx2_nic *pfvf);
 #endif
+
+#if IS_ENABLED(CONFIG_MACSEC)
+/* MACSEC offload support */
+int cn10k_mcs_init(struct otx2_nic *pfvf);
+void cn10k_mcs_free(struct otx2_nic *pfvf);
+void cn10k_handle_mcs_event(struct otx2_nic *pfvf, struct mcs_intr_info *event);
+#else
+static inline int cn10k_mcs_init(struct otx2_nic *pfvf) { return 0; }
+static inline void cn10k_mcs_free(struct otx2_nic *pfvf) {}
+static inline void cn10k_handle_mcs_event(struct otx2_nic *pfvf,
+					  struct mcs_intr_info *event)
+{}
+#endif /* CONFIG_MACSEC */
+
 #endif /* OTX2_COMMON_H */
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c
index 723d2506d309..ccaf97bb1ce0 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_dcbnl.c
@@ -7,6 +7,289 @@
 
 #include "otx2_common.h"
 
+static int otx2_check_pfc_config(struct otx2_nic *pfvf)
+{
+	u8 tx_queues = pfvf->hw.tx_queues, prio;
+	u8 pfc_en = pfvf->pfc_en;
+
+	for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) {
+		if ((pfc_en & (1 << prio)) &&
+		    prio > tx_queues - 1) {
+			dev_warn(pfvf->dev,
+				 "Increase number of tx queues from %d to %d to support PFC.\n",
+				 tx_queues, prio + 1);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+int otx2_pfc_txschq_config(struct otx2_nic *pfvf)
+{
+	u8 pfc_en, pfc_bit_set;
+	int prio, lvl, err;
+
+	pfc_en = pfvf->pfc_en;
+	for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) {
+		pfc_bit_set = pfc_en & (1 << prio);
+
+		/* Either PFC bit is not set
+		 * or tx scheduler is not allocated for the priority
+		 */
+		if (!pfc_bit_set || !pfvf->pfc_alloc_status[prio])
+			continue;
+
+		/* configure the scheduler for the tls*/
+		for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
+			err = otx2_txschq_config(pfvf, lvl, prio, true);
+			if (err) {
+				dev_err(pfvf->dev,
+					"%s configure PFC tx schq for lvl:%d, prio:%d failed!\n",
+					__func__, lvl, prio);
+				return err;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int otx2_pfc_txschq_alloc_one(struct otx2_nic *pfvf, u8 prio)
+{
+	struct nix_txsch_alloc_req *req;
+	struct nix_txsch_alloc_rsp *rsp;
+	int lvl, rc;
+
+	/* Get memory to put this msg */
+	req = otx2_mbox_alloc_msg_nix_txsch_alloc(&pfvf->mbox);
+	if (!req)
+		return -ENOMEM;
+
+	/* Request one schq per level upto max level as configured
+	 * link config level. These rest of the scheduler can be
+	 * same as hw.txschq_list.
+	 */
+	for (lvl = 0; lvl < pfvf->hw.txschq_link_cfg_lvl; lvl++)
+		req->schq[lvl] = 1;
+
+	rc = otx2_sync_mbox_msg(&pfvf->mbox);
+	if (rc)
+		return rc;
+
+	rsp = (struct nix_txsch_alloc_rsp *)
+	      otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr);
+	if (IS_ERR(rsp))
+		return PTR_ERR(rsp);
+
+	/* Setup transmit scheduler list */
+	for (lvl = 0; lvl < pfvf->hw.txschq_link_cfg_lvl; lvl++) {
+		if (!rsp->schq[lvl])
+			return -ENOSPC;
+
+		pfvf->pfc_schq_list[lvl][prio] = rsp->schq_list[lvl][0];
+	}
+
+	/* Set the Tx schedulers for rest of the levels same as
+	 * hw.txschq_list as those will be common for all.
+	 */
+	for (; lvl < NIX_TXSCH_LVL_CNT; lvl++)
+		pfvf->pfc_schq_list[lvl][prio] = pfvf->hw.txschq_list[lvl][0];
+
+	pfvf->pfc_alloc_status[prio] = true;
+	return 0;
+}
+
+int otx2_pfc_txschq_alloc(struct otx2_nic *pfvf)
+{
+	u8 pfc_en = pfvf->pfc_en;
+	u8 pfc_bit_set;
+	int err, prio;
+
+	for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) {
+		pfc_bit_set = pfc_en & (1 << prio);
+
+		if (!pfc_bit_set || pfvf->pfc_alloc_status[prio])
+			continue;
+
+		/* Add new scheduler to the priority */
+		err = otx2_pfc_txschq_alloc_one(pfvf, prio);
+		if (err) {
+			dev_err(pfvf->dev, "%s failed to allocate PFC TX schedulers\n", __func__);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static int otx2_pfc_txschq_stop_one(struct otx2_nic *pfvf, u8 prio)
+{
+	struct nix_txsch_free_req *free_req;
+
+	mutex_lock(&pfvf->mbox.lock);
+	/* free PFC TLx nodes */
+	free_req = otx2_mbox_alloc_msg_nix_txsch_free(&pfvf->mbox);
+	if (!free_req) {
+		mutex_unlock(&pfvf->mbox.lock);
+		return -ENOMEM;
+	}
+
+	free_req->flags = TXSCHQ_FREE_ALL;
+	otx2_sync_mbox_msg(&pfvf->mbox);
+	mutex_unlock(&pfvf->mbox.lock);
+
+	pfvf->pfc_alloc_status[prio] = false;
+	return 0;
+}
+
+static int otx2_pfc_update_sq_smq_mapping(struct otx2_nic *pfvf, int prio)
+{
+	struct nix_cn10k_aq_enq_req *cn10k_sq_aq;
+	struct net_device *dev = pfvf->netdev;
+	bool if_up = netif_running(dev);
+	struct nix_aq_enq_req *sq_aq;
+
+	if (if_up) {
+		if (pfvf->pfc_alloc_status[prio])
+			netif_tx_stop_all_queues(pfvf->netdev);
+		else
+			netif_tx_stop_queue(netdev_get_tx_queue(dev, prio));
+	}
+
+	if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) {
+		cn10k_sq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
+		if (!cn10k_sq_aq)
+			return -ENOMEM;
+
+		/* Fill AQ info */
+		cn10k_sq_aq->qidx = prio;
+		cn10k_sq_aq->ctype = NIX_AQ_CTYPE_SQ;
+		cn10k_sq_aq->op = NIX_AQ_INSTOP_WRITE;
+
+		/* Fill fields to update */
+		cn10k_sq_aq->sq.ena = 1;
+		cn10k_sq_aq->sq_mask.ena = 1;
+		cn10k_sq_aq->sq_mask.smq = GENMASK(9, 0);
+		cn10k_sq_aq->sq.smq = otx2_get_smq_idx(pfvf, prio);
+	} else {
+		sq_aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox);
+		if (!sq_aq)
+			return -ENOMEM;
+
+		/* Fill AQ info */
+		sq_aq->qidx = prio;
+		sq_aq->ctype = NIX_AQ_CTYPE_SQ;
+		sq_aq->op = NIX_AQ_INSTOP_WRITE;
+
+		/* Fill fields to update */
+		sq_aq->sq.ena = 1;
+		sq_aq->sq_mask.ena = 1;
+		sq_aq->sq_mask.smq = GENMASK(8, 0);
+		sq_aq->sq.smq = otx2_get_smq_idx(pfvf, prio);
+	}
+
+	otx2_sync_mbox_msg(&pfvf->mbox);
+
+	if (if_up) {
+		if (pfvf->pfc_alloc_status[prio])
+			netif_tx_start_all_queues(pfvf->netdev);
+		else
+			netif_tx_start_queue(netdev_get_tx_queue(dev, prio));
+	}
+
+	return 0;
+}
+
+int otx2_pfc_txschq_update(struct otx2_nic *pfvf)
+{
+	bool if_up = netif_running(pfvf->netdev);
+	u8 pfc_en = pfvf->pfc_en, pfc_bit_set;
+	struct mbox *mbox = &pfvf->mbox;
+	int err, prio;
+
+	mutex_lock(&mbox->lock);
+	for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) {
+		pfc_bit_set = pfc_en & (1 << prio);
+
+		/* tx scheduler was created but user wants to disable now */
+		if (!pfc_bit_set && pfvf->pfc_alloc_status[prio]) {
+			mutex_unlock(&mbox->lock);
+			if (if_up)
+				netif_tx_stop_all_queues(pfvf->netdev);
+
+			otx2_smq_flush(pfvf, pfvf->pfc_schq_list[NIX_TXSCH_LVL_SMQ][prio]);
+			if (if_up)
+				netif_tx_start_all_queues(pfvf->netdev);
+
+			/* delete the schq */
+			err = otx2_pfc_txschq_stop_one(pfvf, prio);
+			if (err) {
+				dev_err(pfvf->dev,
+					"%s failed to stop PFC tx schedulers for priority: %d\n",
+					__func__, prio);
+				return err;
+			}
+
+			mutex_lock(&mbox->lock);
+			goto update_sq_smq_map;
+		}
+
+		/* Either PFC bit is not set
+		 * or Tx scheduler is already mapped for the priority
+		 */
+		if (!pfc_bit_set || pfvf->pfc_alloc_status[prio])
+			continue;
+
+		/* Add new scheduler to the priority */
+		err = otx2_pfc_txschq_alloc_one(pfvf, prio);
+		if (err) {
+			mutex_unlock(&mbox->lock);
+			dev_err(pfvf->dev,
+				"%s failed to allocate PFC tx schedulers for priority: %d\n",
+				__func__, prio);
+			return err;
+		}
+
+update_sq_smq_map:
+		err = otx2_pfc_update_sq_smq_mapping(pfvf, prio);
+		if (err) {
+			mutex_unlock(&mbox->lock);
+			dev_err(pfvf->dev, "%s failed PFC Tx schq sq:%d mapping", __func__, prio);
+			return err;
+		}
+	}
+
+	err = otx2_pfc_txschq_config(pfvf);
+	mutex_unlock(&mbox->lock);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+int otx2_pfc_txschq_stop(struct otx2_nic *pfvf)
+{
+	u8 pfc_en, pfc_bit_set;
+	int prio, err;
+
+	pfc_en = pfvf->pfc_en;
+	for (prio = 0; prio < NIX_PF_PFC_PRIO_MAX; prio++) {
+		pfc_bit_set = pfc_en & (1 << prio);
+		if (!pfc_bit_set || !pfvf->pfc_alloc_status[prio])
+			continue;
+
+		/* Delete the existing scheduler */
+		err = otx2_pfc_txschq_stop_one(pfvf, prio);
+		if (err) {
+			dev_err(pfvf->dev, "%s failed to stop PFC TX schedulers\n", __func__);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
 int otx2_config_priority_flow_ctrl(struct otx2_nic *pfvf)
 {
 	struct cgx_pfc_cfg *req;
@@ -128,6 +411,17 @@ static int otx2_dcbnl_ieee_setpfc(struct net_device *dev, struct ieee_pfc *pfc)
 	/* Save PFC configuration to interface */
 	pfvf->pfc_en = pfc->pfc_en;
 
+	if (pfvf->hw.tx_queues >= NIX_PF_PFC_PRIO_MAX)
+		goto process_pfc;
+
+	/* Check if the PFC configuration can be
+	 * supported by the tx queue configuration
+	 */
+	err = otx2_check_pfc_config(pfvf);
+	if (err)
+		return err;
+
+process_pfc:
 	err = otx2_config_priority_flow_ctrl(pfvf);
 	if (err)
 		return err;
@@ -136,6 +430,12 @@ static int otx2_dcbnl_ieee_setpfc(struct net_device *dev, struct ieee_pfc *pfc)
 	if (pfc->pfc_en)
 		otx2_nix_config_bp(pfvf, true);
 
+	err = otx2_pfc_txschq_update(pfvf);
+	if (err) {
+		dev_err(pfvf->dev, "%s failed to update TX schedulers\n", __func__);
+		return err;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
index 3f60a80e34c8..0eb74e8c553d 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c
@@ -76,8 +76,8 @@ static void otx2_get_drvinfo(struct net_device *netdev,
 {
 	struct otx2_nic *pfvf = netdev_priv(netdev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(pfvf->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(pfvf->pdev), sizeof(info->bus_info));
 }
 
 static void otx2_get_qset_strings(struct otx2_nic *pfvf, u8 **data, int qset)
@@ -963,10 +963,12 @@ static int otx2_get_ts_info(struct net_device *netdev,
 
 	info->phc_index = otx2_ptp_clock_index(pfvf);
 
-	info->tx_types = (1 << HWTSTAMP_TX_OFF) | (1 << HWTSTAMP_TX_ON);
+	info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON);
+	if (test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag))
+		info->tx_types |= BIT(HWTSTAMP_TX_ONESTEP_SYNC);
 
-	info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) |
-			   (1 << HWTSTAMP_FILTER_ALL);
+	info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) |
+			   BIT(HWTSTAMP_FILTER_ALL);
 
 	return 0;
 }
@@ -1313,8 +1315,8 @@ static void otx2vf_get_drvinfo(struct net_device *netdev,
 {
 	struct otx2_nic *vf = netdev_priv(netdev);
 
-	strlcpy(info->driver, DRV_VF_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(vf->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_VF_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(vf->pdev), sizeof(info->bus_info));
 }
 
 static void otx2vf_get_strings(struct net_device *netdev, u32 sset, u8 *data)
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index 9376d0e62914..5803d7f9137c 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -858,6 +858,15 @@ static void otx2_handle_link_event(struct otx2_nic *pf)
 	}
 }
 
+int otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic *pf,
+					 struct mcs_intr_info *event,
+					 struct msg_rsp *rsp)
+{
+	cn10k_handle_mcs_event(pf, event);
+
+	return 0;
+}
+
 int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf,
 					struct cgx_link_info_msg *msg,
 					struct msg_rsp *rsp)
@@ -917,6 +926,7 @@ static int otx2_process_mbox_msg_up(struct otx2_nic *pf,
 		return err;						\
 	}
 MBOX_UP_CGX_MESSAGES
+MBOX_UP_MCS_MESSAGES
 #undef M
 		break;
 	default:
@@ -1389,18 +1399,40 @@ static int otx2_init_hw_resources(struct otx2_nic *pf)
 		goto err_free_sq_ptrs;
 	}
 
+#ifdef CONFIG_DCB
+	if (pf->pfc_en) {
+		err = otx2_pfc_txschq_alloc(pf);
+		if (err) {
+			mutex_unlock(&mbox->lock);
+			goto err_free_sq_ptrs;
+		}
+	}
+#endif
+
 	err = otx2_config_nix_queues(pf);
 	if (err) {
 		mutex_unlock(&mbox->lock);
 		goto err_free_txsch;
 	}
+
 	for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
-		err = otx2_txschq_config(pf, lvl);
+		err = otx2_txschq_config(pf, lvl, 0, false);
+		if (err) {
+			mutex_unlock(&mbox->lock);
+			goto err_free_nix_queues;
+		}
+	}
+
+#ifdef CONFIG_DCB
+	if (pf->pfc_en) {
+		err = otx2_pfc_txschq_config(pf);
 		if (err) {
 			mutex_unlock(&mbox->lock);
 			goto err_free_nix_queues;
 		}
 	}
+#endif
+
 	mutex_unlock(&mbox->lock);
 	return err;
 
@@ -1455,6 +1487,11 @@ static void otx2_free_hw_resources(struct otx2_nic *pf)
 	if (err)
 		dev_err(pf->dev, "RVUPF: Failed to stop/free TX schedulers\n");
 
+#ifdef CONFIG_DCB
+	if (pf->pfc_en)
+		otx2_pfc_txschq_stop(pf);
+#endif
+
 	mutex_lock(&mbox->lock);
 	/* Disable backpressure */
 	if (!(pf->pcifunc & RVU_PFVF_FUNC_MASK))
@@ -1634,8 +1671,7 @@ int otx2_open(struct net_device *netdev)
 		cq_poll->dev = (void *)pf;
 		cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
 		INIT_WORK(&cq_poll->dim.work, otx2_dim_work);
-		netif_napi_add(netdev, &cq_poll->napi,
-			       otx2_napi_handler, NAPI_POLL_WEIGHT);
+		netif_napi_add(netdev, &cq_poll->napi, otx2_napi_handler);
 		napi_enable(&cq_poll->napi);
 	}
 
@@ -1853,6 +1889,30 @@ static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev)
 	return NETDEV_TX_OK;
 }
 
+static u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb,
+			     struct net_device *sb_dev)
+{
+#ifdef CONFIG_DCB
+	struct otx2_nic *pf = netdev_priv(netdev);
+	u8 vlan_prio;
+#endif
+
+#ifdef CONFIG_DCB
+	if (!skb->vlan_present)
+		goto pick_tx;
+
+	vlan_prio = skb->vlan_tci >> 13;
+	if ((vlan_prio > pf->hw.tx_queues - 1) ||
+	    !pf->pfc_alloc_status[vlan_prio])
+		goto pick_tx;
+
+	return vlan_prio;
+
+pick_tx:
+#endif
+	return netdev_pick_tx(netdev, skb, NULL);
+}
+
 static netdev_features_t otx2_fix_features(struct net_device *dev,
 					   netdev_features_t features)
 {
@@ -1987,8 +2047,19 @@ int otx2_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr)
 
 	switch (config.tx_type) {
 	case HWTSTAMP_TX_OFF:
+		if (pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC)
+			pfvf->flags &= ~OTX2_FLAG_PTP_ONESTEP_SYNC;
+
+		cancel_delayed_work(&pfvf->ptp->synctstamp_work);
 		otx2_config_hw_tx_tstamp(pfvf, false);
 		break;
+	case HWTSTAMP_TX_ONESTEP_SYNC:
+		if (!test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag))
+			return -ERANGE;
+		pfvf->flags |= OTX2_FLAG_PTP_ONESTEP_SYNC;
+		schedule_delayed_work(&pfvf->ptp->synctstamp_work,
+				      msecs_to_jiffies(500));
+		fallthrough;
 	case HWTSTAMP_TX_ON:
 		otx2_config_hw_tx_tstamp(pfvf, true);
 		break;
@@ -2447,6 +2518,7 @@ static const struct net_device_ops otx2_netdev_ops = {
 	.ndo_open		= otx2_open,
 	.ndo_stop		= otx2_stop,
 	.ndo_start_xmit		= otx2_xmit,
+	.ndo_select_queue	= otx2_select_queue,
 	.ndo_fix_features	= otx2_fix_features,
 	.ndo_set_mac_address    = otx2_set_mac_address,
 	.ndo_change_mtu		= otx2_change_mtu,
@@ -2702,6 +2774,10 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (err)
 		goto err_ptp_destroy;
 
+	err = cn10k_mcs_init(pf);
+	if (err)
+		goto err_del_mcam_entries;
+
 	if (pf->flags & OTX2_FLAG_NTUPLE_SUPPORT)
 		netdev->hw_features |= NETIF_F_NTUPLE;
 
@@ -2916,6 +2992,8 @@ static void otx2_remove(struct pci_dev *pdev)
 		otx2_config_pause_frm(pf);
 	}
 
+	cn10k_mcs_free(pf);
+
 #ifdef CONFIG_DCB
 	/* Disable PFC config */
 	if (pf->pfc_en) {
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c
index fdc2c9315b91..896b2f9bac34 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ptp.c
@@ -10,6 +10,33 @@
 #include "otx2_common.h"
 #include "otx2_ptp.h"
 
+static u64 otx2_ptp_get_clock(struct otx2_ptp *ptp)
+{
+	struct ptp_req *req;
+	struct ptp_rsp *rsp;
+	int err;
+
+	if (!ptp->nic)
+		return 0;
+
+	req = otx2_mbox_alloc_msg_ptp_op(&ptp->nic->mbox);
+	if (!req)
+		return 0;
+
+	req->op = PTP_OP_GET_CLOCK;
+
+	err = otx2_sync_mbox_msg(&ptp->nic->mbox);
+	if (err)
+		return 0;
+
+	rsp = (struct ptp_rsp *)otx2_mbox_get_rsp(&ptp->nic->mbox.mbox, 0,
+						  &req->hdr);
+	if (IS_ERR(rsp))
+		return 0;
+
+	return rsp->clk;
+}
+
 static int otx2_ptp_adjfine(struct ptp_clock_info *ptp_info, long scaled_ppm)
 {
 	struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp,
@@ -46,32 +73,28 @@ static int ptp_set_thresh(struct otx2_ptp *ptp, u64 thresh)
 	return otx2_sync_mbox_msg(&ptp->nic->mbox);
 }
 
-static u64 ptp_cc_read(const struct cyclecounter *cc)
+static int ptp_extts_on(struct otx2_ptp *ptp, int on)
 {
-	struct otx2_ptp *ptp = container_of(cc, struct otx2_ptp, cycle_counter);
 	struct ptp_req *req;
-	struct ptp_rsp *rsp;
-	int err;
 
 	if (!ptp->nic)
-		return 0;
+		return -ENODEV;
 
 	req = otx2_mbox_alloc_msg_ptp_op(&ptp->nic->mbox);
 	if (!req)
-		return 0;
+		return -ENOMEM;
 
-	req->op = PTP_OP_GET_CLOCK;
+	req->op = PTP_OP_EXTTS_ON;
+	req->extts_on = on;
 
-	err = otx2_sync_mbox_msg(&ptp->nic->mbox);
-	if (err)
-		return 0;
+	return otx2_sync_mbox_msg(&ptp->nic->mbox);
+}
 
-	rsp = (struct ptp_rsp *)otx2_mbox_get_rsp(&ptp->nic->mbox.mbox, 0,
-						  &req->hdr);
-	if (IS_ERR(rsp))
-		return 0;
+static u64 ptp_cc_read(const struct cyclecounter *cc)
+{
+	struct otx2_ptp *ptp = container_of(cc, struct otx2_ptp, cycle_counter);
 
-	return rsp->clk;
+	return otx2_ptp_get_clock(ptp);
 }
 
 static u64 ptp_tstmp_read(struct otx2_ptp *ptp)
@@ -101,6 +124,15 @@ static u64 ptp_tstmp_read(struct otx2_ptp *ptp)
 	return rsp->clk;
 }
 
+static void otx2_get_ptpclock(struct otx2_ptp *ptp, u64 *tstamp)
+{
+	struct otx2_nic *pfvf = ptp->nic;
+
+	mutex_lock(&pfvf->mbox.lock);
+	*tstamp = timecounter_read(&ptp->time_counter);
+	mutex_unlock(&pfvf->mbox.lock);
+}
+
 static int otx2_ptp_adjtime(struct ptp_clock_info *ptp_info, s64 delta)
 {
 	struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp,
@@ -119,14 +151,10 @@ static int otx2_ptp_gettime(struct ptp_clock_info *ptp_info,
 {
 	struct otx2_ptp *ptp = container_of(ptp_info, struct otx2_ptp,
 					    ptp_info);
-	struct otx2_nic *pfvf = ptp->nic;
-	u64 nsec;
+	u64 tstamp;
 
-	mutex_lock(&pfvf->mbox.lock);
-	nsec = timecounter_read(&ptp->time_counter);
-	mutex_unlock(&pfvf->mbox.lock);
-
-	*ts = ns_to_timespec64(nsec);
+	otx2_get_ptpclock(ptp, &tstamp);
+	*ts = ns_to_timespec64(tstamp);
 
 	return 0;
 }
@@ -178,8 +206,6 @@ static void otx2_ptp_extts_check(struct work_struct *work)
 		event.index = 0;
 		event.timestamp = timecounter_cyc2time(&ptp->time_counter, tstmp);
 		ptp_clock_event(ptp->ptp_clock, &event);
-		ptp->last_extts = tstmp;
-
 		new_thresh = tstmp % 500000000;
 		if (ptp->thresh != new_thresh) {
 			mutex_lock(&ptp->nic->mbox.lock);
@@ -187,10 +213,28 @@ static void otx2_ptp_extts_check(struct work_struct *work)
 			mutex_unlock(&ptp->nic->mbox.lock);
 			ptp->thresh = new_thresh;
 		}
+		ptp->last_extts = tstmp;
 	}
 	schedule_delayed_work(&ptp->extts_work, msecs_to_jiffies(200));
 }
 
+static void otx2_sync_tstamp(struct work_struct *work)
+{
+	struct otx2_ptp *ptp = container_of(work, struct otx2_ptp,
+					    synctstamp_work.work);
+	struct otx2_nic *pfvf = ptp->nic;
+	u64 tstamp;
+
+	mutex_lock(&pfvf->mbox.lock);
+	tstamp = otx2_ptp_get_clock(ptp);
+	mutex_unlock(&pfvf->mbox.lock);
+
+	ptp->tstamp = timecounter_cyc2time(&pfvf->ptp->time_counter, tstamp);
+	ptp->base_ns = tstamp % NSEC_PER_SEC;
+
+	schedule_delayed_work(&ptp->synctstamp_work, msecs_to_jiffies(250));
+}
+
 static int otx2_ptp_enable(struct ptp_clock_info *ptp_info,
 			   struct ptp_clock_request *rq, int on)
 {
@@ -207,10 +251,13 @@ static int otx2_ptp_enable(struct ptp_clock_info *ptp_info,
 				   rq->extts.index);
 		if (pin < 0)
 			return -EBUSY;
-		if (on)
+		if (on) {
+			ptp_extts_on(ptp, on);
 			schedule_delayed_work(&ptp->extts_work, msecs_to_jiffies(200));
-		else
+		} else {
+			ptp_extts_on(ptp, on);
 			cancel_delayed_work_sync(&ptp->extts_work);
+		}
 		return 0;
 	default:
 		break;
@@ -302,6 +349,8 @@ int otx2_ptp_init(struct otx2_nic *pfvf)
 		ptp_ptr->convert_tx_ptp_tstmp = &cn10k_ptp_convert_timestamp;
 	}
 
+	INIT_DELAYED_WORK(&ptp_ptr->synctstamp_work, otx2_sync_tstamp);
+
 	pfvf->ptp = ptp_ptr;
 
 error:
@@ -316,6 +365,8 @@ void otx2_ptp_destroy(struct otx2_nic *pfvf)
 	if (!ptp)
 		return;
 
+	cancel_delayed_work(&pfvf->ptp->synctstamp_work);
+
 	ptp_clock_unregister(ptp->ptp_clock);
 	kfree(ptp);
 	pfvf->ptp = NULL;
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h
index 4bbd12ff26e6..aa205a0d158f 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_struct.h
@@ -236,8 +236,15 @@ struct nix_sqe_sg_s {
 
 /* NIX send memory subdescriptor structure */
 struct nix_sqe_mem_s {
-	u64 offset        : 16; /* W0 */
-	u64 rsvd_51_16    : 36;
+	u64 start_offset  : 8;
+	u64 rsvd_11_8	  : 4;
+	u64 rsvd_12	  : 1;
+	u64 udp_csum_crt  : 1;
+	u64 update64      : 1;
+	u64 rsvd_15_16    : 1;
+	u64 base_ns       : 32;
+	u64 step_type     : 1;
+	u64 rsvd_51_49    : 3;
 	u64 per_lso_seg   : 1;
 	u64 wmem          : 1;
 	u64 dsz           : 2;
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
index a18e8efd0f1e..5ec11d71bf60 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
@@ -19,6 +19,12 @@
 #include "cn10k.h"
 
 #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))
+#define PTP_PORT	        0x13F
+/* PTPv2 header Original Timestamp starts at byte offset 34 and
+ * contains 6 byte seconds field and 4 byte nano seconds field.
+ */
+#define PTP_SYNC_SEC_OFFSET	34
+
 static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
 				     struct bpf_prog *prog,
 				     struct nix_cqe_rx_s *cqe,
@@ -686,7 +692,8 @@ static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
 }
 
 static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset,
-			     int alg, u64 iova)
+			     int alg, u64 iova, int ptp_offset,
+			     u64 base_ns, int udp_csum)
 {
 	struct nix_sqe_mem_s *mem;
 
@@ -696,6 +703,13 @@ static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset,
 	mem->wmem = 1; /* wait for the memory operation */
 	mem->addr = iova;
 
+	if (ptp_offset) {
+		mem->start_offset = ptp_offset;
+		mem->udp_csum_crt = udp_csum;
+		mem->base_ns = base_ns;
+		mem->step_type = 1;
+	}
+
 	*offset += sizeof(*mem);
 }
 
@@ -952,16 +966,102 @@ static int otx2_get_sqe_count(struct otx2_nic *pfvf, struct sk_buff *skb)
 	return skb_shinfo(skb)->gso_segs;
 }
 
+static bool otx2_validate_network_transport(struct sk_buff *skb)
+{
+	if ((ip_hdr(skb)->protocol == IPPROTO_UDP) ||
+	    (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) {
+		struct udphdr *udph = udp_hdr(skb);
+
+		if (udph->source == htons(PTP_PORT) &&
+		    udph->dest == htons(PTP_PORT))
+			return true;
+	}
+
+	return false;
+}
+
+static bool otx2_ptp_is_sync(struct sk_buff *skb, int *offset, int *udp_csum)
+{
+	struct ethhdr *eth = (struct ethhdr *)(skb->data);
+	u16 nix_offload_hlen = 0, inner_vhlen = 0;
+	u8 *data = skb->data, *msgtype;
+	__be16 proto = eth->h_proto;
+	int network_depth = 0;
+
+	/* NIX is programmed to offload outer  VLAN header
+	 * in case of single vlan protocol field holds Network header ETH_IP/V6
+	 * in case of stacked vlan protocol field holds Inner vlan (8100)
+	 */
+	if (skb->dev->features & NETIF_F_HW_VLAN_CTAG_TX &&
+	    skb->dev->features & NETIF_F_HW_VLAN_STAG_TX) {
+		if (skb->vlan_proto == htons(ETH_P_8021AD)) {
+			/* Get vlan protocol */
+			proto = __vlan_get_protocol(skb, eth->h_proto, NULL);
+			/* SKB APIs like skb_transport_offset does not include
+			 * offloaded vlan header length. Need to explicitly add
+			 * the length
+			 */
+			nix_offload_hlen = VLAN_HLEN;
+			inner_vhlen = VLAN_HLEN;
+		} else if (skb->vlan_proto == htons(ETH_P_8021Q)) {
+			nix_offload_hlen = VLAN_HLEN;
+		}
+	} else if (eth_type_vlan(eth->h_proto)) {
+		proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
+	}
+
+	switch (ntohs(proto)) {
+	case ETH_P_1588:
+		if (network_depth)
+			*offset = network_depth;
+		else
+			*offset = ETH_HLEN + nix_offload_hlen +
+				  inner_vhlen;
+		break;
+	case ETH_P_IP:
+	case ETH_P_IPV6:
+		if (!otx2_validate_network_transport(skb))
+			return false;
+
+		*udp_csum = 1;
+		*offset = nix_offload_hlen + skb_transport_offset(skb) +
+			  sizeof(struct udphdr);
+	}
+
+	msgtype = data + *offset;
+
+	/* Check PTP messageId is SYNC or not */
+	return (*msgtype & 0xf) == 0;
+}
+
 static void otx2_set_txtstamp(struct otx2_nic *pfvf, struct sk_buff *skb,
 			      struct otx2_snd_queue *sq, int *offset)
 {
+	struct ptpv2_tstamp *origin_tstamp;
+	int ptp_offset = 0, udp_csum = 0;
+	struct timespec64 ts;
 	u64 iova;
 
-	if (!skb_shinfo(skb)->gso_size &&
-	    skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
-		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+	if (unlikely(!skb_shinfo(skb)->gso_size &&
+		     (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) {
+		if (unlikely(pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC)) {
+			if (otx2_ptp_is_sync(skb, &ptp_offset, &udp_csum)) {
+				origin_tstamp = (struct ptpv2_tstamp *)
+						((u8 *)skb->data + ptp_offset +
+						 PTP_SYNC_SEC_OFFSET);
+				ts = ns_to_timespec64(pfvf->ptp->tstamp);
+				origin_tstamp->seconds_msb = htons((ts.tv_sec >> 32) & 0xffff);
+				origin_tstamp->seconds_lsb = htonl(ts.tv_sec & 0xffffffff);
+				origin_tstamp->nanoseconds = htonl(ts.tv_nsec);
+				/* Point to correction field in PTP packet */
+				ptp_offset += 8;
+			}
+		} else {
+			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+		}
 		iova = sq->timestamps->iova + (sq->head * sizeof(u64));
-		otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova);
+		otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova,
+				 ptp_offset, pfvf->ptp->base_ns, udp_csum);
 	} else {
 		skb_tx_timestamp(skb);
 	}
diff --git a/drivers/net/ethernet/marvell/prestera/Makefile b/drivers/net/ethernet/marvell/prestera/Makefile
index d395f4131648..df14cee80153 100644
--- a/drivers/net/ethernet/marvell/prestera/Makefile
+++ b/drivers/net/ethernet/marvell/prestera/Makefile
@@ -4,6 +4,6 @@ prestera-objs		:= prestera_main.o prestera_hw.o prestera_dsa.o \
 			   prestera_rxtx.o prestera_devlink.o prestera_ethtool.o \
 			   prestera_switchdev.o prestera_acl.o prestera_flow.o \
 			   prestera_flower.o prestera_span.o prestera_counter.o \
-			   prestera_router.o prestera_router_hw.o
+			   prestera_router.o prestera_router_hw.o prestera_matchall.o
 
 obj-$(CONFIG_PRESTERA_PCI)	+= prestera_pci.o
diff --git a/drivers/net/ethernet/marvell/prestera/prestera.h b/drivers/net/ethernet/marvell/prestera/prestera.h
index 2f84d0fb4094..35554ee805cd 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera.h
+++ b/drivers/net/ethernet/marvell/prestera/prestera.h
@@ -306,17 +306,27 @@ struct prestera_switch {
 	struct prestera_counter *counter;
 	u8 lag_member_max;
 	u8 lag_max;
+	u32 size_tbl_router_nexthop;
 };
 
 struct prestera_router {
 	struct prestera_switch *sw;
 	struct list_head vr_list;
 	struct list_head rif_entry_list;
+	struct rhashtable nh_neigh_ht;
+	struct rhashtable nexthop_group_ht;
 	struct rhashtable fib_ht;
+	struct rhashtable kern_neigh_cache_ht;
 	struct rhashtable kern_fib_cache_ht;
 	struct notifier_block inetaddr_nb;
 	struct notifier_block inetaddr_valid_nb;
 	struct notifier_block fib_nb;
+	struct notifier_block netevent_nb;
+	u8 *nhgrp_hw_state_cache; /* Bitmap cached hw state of nhs */
+	unsigned long nhgrp_hw_cache_kick; /* jiffies */
+	struct {
+		struct delayed_work dw;
+	} neighs_update;
 };
 
 struct prestera_rxtx_params {
@@ -362,11 +372,15 @@ int prestera_port_cfg_mac_write(struct prestera_port *port,
 struct prestera_port *prestera_port_dev_lower_find(struct net_device *dev);
 
 void prestera_queue_work(struct work_struct *work);
+void prestera_queue_delayed_work(struct delayed_work *work, unsigned long delay);
+void prestera_queue_drain(void);
 
 int prestera_port_learning_set(struct prestera_port *port, bool learn_enable);
 int prestera_port_uc_flood_set(struct prestera_port *port, bool flood);
 int prestera_port_mc_flood_set(struct prestera_port *port, bool flood);
 
+int prestera_port_br_locked_set(struct prestera_port *port, bool br_locked);
+
 int prestera_port_pvid_set(struct prestera_port *port, u16 vid);
 
 bool prestera_netdev_check(const struct net_device *dev);
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.c b/drivers/net/ethernet/marvell/prestera/prestera_acl.c
index 3d4b85f2d541..cba89fda504b 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_acl.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.c
@@ -54,6 +54,10 @@ struct prestera_acl_ruleset {
 	struct prestera_acl_ruleset_ht_key ht_key;
 	struct rhashtable rule_ht;
 	struct prestera_acl *acl;
+	struct {
+		u32 min;
+		u32 max;
+	} prio;
 	unsigned long rule_count;
 	refcount_t refcount;
 	void *keymask;
@@ -162,6 +166,9 @@ prestera_acl_ruleset_create(struct prestera_acl *acl,
 	ruleset->pcl_id = PRESTERA_ACL_PCL_ID_MAKE((u8)uid, chain_index);
 	ruleset->index = uid;
 
+	ruleset->prio.min = UINT_MAX;
+	ruleset->prio.max = 0;
+
 	err = rhashtable_insert_fast(&acl->ruleset_ht, &ruleset->ht_node,
 				     prestera_acl_ruleset_ht_params);
 	if (err)
@@ -178,10 +185,14 @@ err_rhashtable_init:
 	return ERR_PTR(err);
 }
 
-void prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset,
-				      void *keymask)
+int prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset,
+				     void *keymask)
 {
 	ruleset->keymask = kmemdup(keymask, ACL_KEYMASK_SIZE, GFP_KERNEL);
+	if (!ruleset->keymask)
+		return -ENOMEM;
+
+	return 0;
 }
 
 int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset)
@@ -365,6 +376,26 @@ prestera_acl_ruleset_block_unbind(struct prestera_acl_ruleset *ruleset,
 	block->ruleset_zero = NULL;
 }
 
+static void
+prestera_acl_ruleset_prio_refresh(struct prestera_acl *acl,
+				  struct prestera_acl_ruleset *ruleset)
+{
+	struct prestera_acl_rule *rule;
+
+	ruleset->prio.min = UINT_MAX;
+	ruleset->prio.max = 0;
+
+	list_for_each_entry(rule, &acl->rules, list) {
+		if (ruleset->ingress != rule->ruleset->ingress)
+			continue;
+		if (ruleset->ht_key.chain_index != rule->chain_index)
+			continue;
+
+		ruleset->prio.min = min(ruleset->prio.min, rule->priority);
+		ruleset->prio.max = max(ruleset->prio.max, rule->priority);
+	}
+}
+
 void
 prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule, u16 pcl_id)
 {
@@ -389,6 +420,13 @@ u32 prestera_acl_ruleset_index_get(const struct prestera_acl_ruleset *ruleset)
 	return ruleset->index;
 }
 
+void prestera_acl_ruleset_prio_get(struct prestera_acl_ruleset *ruleset,
+				   u32 *prio_min, u32 *prio_max)
+{
+	*prio_min = ruleset->prio.min;
+	*prio_max = ruleset->prio.max;
+}
+
 bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset)
 {
 	return ruleset->offload;
@@ -429,6 +467,13 @@ void prestera_acl_rule_destroy(struct prestera_acl_rule *rule)
 	kfree(rule);
 }
 
+static void prestera_acl_ruleset_prio_update(struct prestera_acl_ruleset *ruleset,
+					     u32 prio)
+{
+	ruleset->prio.min = min(ruleset->prio.min, prio);
+	ruleset->prio.max = max(ruleset->prio.max, prio);
+}
+
 int prestera_acl_rule_add(struct prestera_switch *sw,
 			  struct prestera_acl_rule *rule)
 {
@@ -468,6 +513,7 @@ int prestera_acl_rule_add(struct prestera_switch *sw,
 
 	list_add_tail(&rule->list, &sw->acl->rules);
 	ruleset->rule_count++;
+	prestera_acl_ruleset_prio_update(ruleset, rule->priority);
 	return 0;
 
 err_acl_block_bind:
@@ -492,6 +538,7 @@ void prestera_acl_rule_del(struct prestera_switch *sw,
 	list_del(&rule->list);
 
 	prestera_acl_rule_entry_destroy(sw->acl, rule->re);
+	prestera_acl_ruleset_prio_refresh(sw->acl, ruleset);
 
 	/* unbind block (all ports) */
 	if (!ruleset->ht_key.chain_index && !ruleset->rule_count)
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.h b/drivers/net/ethernet/marvell/prestera/prestera_acl.h
index 03fc5b9dc925..a35cc0609a1d 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_acl.h
+++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.h
@@ -185,8 +185,8 @@ struct prestera_acl_ruleset *
 prestera_acl_ruleset_lookup(struct prestera_acl *acl,
 			    struct prestera_flow_block *block,
 			    u32 chain_index);
-void prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset,
-				      void *keymask);
+int prestera_acl_ruleset_keymask_set(struct prestera_acl_ruleset *ruleset,
+				     void *keymask);
 bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset);
 int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset);
 void prestera_acl_ruleset_put(struct prestera_acl_ruleset *ruleset);
@@ -195,6 +195,8 @@ int prestera_acl_ruleset_bind(struct prestera_acl_ruleset *ruleset,
 int prestera_acl_ruleset_unbind(struct prestera_acl_ruleset *ruleset,
 				struct prestera_port *port);
 u32 prestera_acl_ruleset_index_get(const struct prestera_acl_ruleset *ruleset);
+void prestera_acl_ruleset_prio_get(struct prestera_acl_ruleset *ruleset,
+				   u32 *prio_min, u32 *prio_max);
 void
 prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule,
 				     u16 pcl_id);
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c
index 1da7ff889417..2f52daba58e6 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_ethtool.c
@@ -300,8 +300,8 @@ static void prestera_ethtool_get_drvinfo(struct net_device *dev,
 	struct prestera_port *port = netdev_priv(dev);
 	struct prestera_switch *sw = port->sw;
 
-	strlcpy(drvinfo->driver, driver_kind, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, dev_name(prestera_dev(sw)),
+	strscpy(drvinfo->driver, driver_kind, sizeof(drvinfo->driver));
+	strscpy(drvinfo->bus_info, dev_name(prestera_dev(sw)),
 		sizeof(drvinfo->bus_info));
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
 		 "%d.%d.%d",
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.c b/drivers/net/ethernet/marvell/prestera/prestera_flow.c
index 2262693bd5cf..9f4267f326b0 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_flow.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.c
@@ -7,8 +7,9 @@
 #include "prestera.h"
 #include "prestera_acl.h"
 #include "prestera_flow.h"
-#include "prestera_span.h"
 #include "prestera_flower.h"
+#include "prestera_matchall.h"
+#include "prestera_span.h"
 
 static LIST_HEAD(prestera_block_cb_list);
 
@@ -17,9 +18,9 @@ static int prestera_flow_block_mall_cb(struct prestera_flow_block *block,
 {
 	switch (f->command) {
 	case TC_CLSMATCHALL_REPLACE:
-		return prestera_span_replace(block, f);
+		return prestera_mall_replace(block, f);
 	case TC_CLSMATCHALL_DESTROY:
-		prestera_span_destroy(block);
+		prestera_mall_destroy(block);
 		return 0;
 	default:
 		return -EOPNOTSUPP;
@@ -89,6 +90,9 @@ prestera_flow_block_create(struct prestera_switch *sw,
 	INIT_LIST_HEAD(&block->template_list);
 	block->net = net;
 	block->sw = sw;
+	block->mall.prio_min = UINT_MAX;
+	block->mall.prio_max = 0;
+	block->mall.bound = false;
 	block->ingress = ingress;
 
 	return block;
@@ -263,7 +267,7 @@ static void prestera_setup_flow_block_unbind(struct prestera_port *port,
 
 	block = flow_block_cb_priv(block_cb);
 
-	prestera_span_destroy(block);
+	prestera_mall_destroy(block);
 
 	err = prestera_flow_block_unbind(block, port);
 	if (err)
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.h b/drivers/net/ethernet/marvell/prestera/prestera_flow.h
index 0c9e13263261..a85a3eb40279 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_flow.h
+++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.h
@@ -22,6 +22,11 @@ struct prestera_flow_block {
 	struct prestera_acl_ruleset *ruleset_zero;
 	struct flow_block_cb *block_cb;
 	struct list_head template_list;
+	struct {
+		u32 prio_min;
+		u32 prio_max;
+		bool bound;
+	} mall;
 	unsigned int rule_count;
 	bool ingress;
 };
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.c b/drivers/net/ethernet/marvell/prestera/prestera_flower.c
index 19d3b55c578e..91a478b75cbf 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_flower.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.c
@@ -5,6 +5,7 @@
 #include "prestera_acl.h"
 #include "prestera_flow.h"
 #include "prestera_flower.h"
+#include "prestera_matchall.h"
 
 struct prestera_flower_template {
 	struct prestera_acl_ruleset *ruleset;
@@ -360,6 +361,49 @@ static int prestera_flower_parse(struct prestera_flow_block *block,
 					     f->common.extack);
 }
 
+static int prestera_flower_prio_check(struct prestera_flow_block *block,
+				      struct flow_cls_offload *f)
+{
+	u32 mall_prio_min;
+	u32 mall_prio_max;
+	int err;
+
+	err = prestera_mall_prio_get(block, &mall_prio_min, &mall_prio_max);
+	if (err == -ENOENT)
+		/* No matchall filters installed on this chain. */
+		return 0;
+
+	if (err) {
+		NL_SET_ERR_MSG(f->common.extack, "Failed to get matchall priorities");
+		return err;
+	}
+
+	if (f->common.prio <= mall_prio_max && block->ingress) {
+		NL_SET_ERR_MSG(f->common.extack,
+			       "Failed to add in front of existing matchall rules");
+		return -EOPNOTSUPP;
+	}
+	if (f->common.prio >= mall_prio_min && !block->ingress) {
+		NL_SET_ERR_MSG(f->common.extack, "Failed to add behind of existing matchall rules");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+int prestera_flower_prio_get(struct prestera_flow_block *block, u32 chain_index,
+			     u32 *prio_min, u32 *prio_max)
+{
+	struct prestera_acl_ruleset *ruleset;
+
+	ruleset = prestera_acl_ruleset_lookup(block->sw->acl, block, chain_index);
+	if (IS_ERR(ruleset))
+		return PTR_ERR(ruleset);
+
+	prestera_acl_ruleset_prio_get(ruleset, prio_min, prio_max);
+	return 0;
+}
+
 int prestera_flower_replace(struct prestera_flow_block *block,
 			    struct flow_cls_offload *f)
 {
@@ -368,6 +412,10 @@ int prestera_flower_replace(struct prestera_flow_block *block,
 	struct prestera_acl_rule *rule;
 	int err;
 
+	err = prestera_flower_prio_check(block, f);
+	if (err)
+		return err;
+
 	ruleset = prestera_acl_ruleset_get(acl, block, f->common.chain_index);
 	if (IS_ERR(ruleset))
 		return PTR_ERR(ruleset);
@@ -452,7 +500,9 @@ int prestera_flower_tmplt_create(struct prestera_flow_block *block,
 	}
 
 	/* preserve keymask/template to this ruleset */
-	prestera_acl_ruleset_keymask_set(ruleset, rule.re_key.match.mask);
+	err = prestera_acl_ruleset_keymask_set(ruleset, rule.re_key.match.mask);
+	if (err)
+		goto err_ruleset_keymask_set;
 
 	/* skip error, as it is not possible to reject template operation,
 	 * so, keep the reference to the ruleset for rules to be added
@@ -468,6 +518,8 @@ int prestera_flower_tmplt_create(struct prestera_flow_block *block,
 	list_add_rcu(&template->list, &block->template_list);
 	return 0;
 
+err_ruleset_keymask_set:
+	prestera_acl_ruleset_put(ruleset);
 err_ruleset_get:
 	kfree(template);
 err_malloc:
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.h b/drivers/net/ethernet/marvell/prestera/prestera_flower.h
index 495f151e6fa9..1181115fe6fa 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_flower.h
+++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.h
@@ -19,5 +19,7 @@ int prestera_flower_tmplt_create(struct prestera_flow_block *block,
 void prestera_flower_tmplt_destroy(struct prestera_flow_block *block,
 				   struct flow_cls_offload *f);
 void prestera_flower_template_cleanup(struct prestera_flow_block *block);
+int prestera_flower_prio_get(struct prestera_flow_block *block, u32 chain_index,
+			     u32 *prio_min, u32 *prio_max);
 
 #endif /* _PRESTERA_FLOWER_H_ */
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.c b/drivers/net/ethernet/marvell/prestera/prestera_hw.c
index 962d7e0c0cb5..fc6f7d2746e8 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_hw.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.c
@@ -10,11 +10,14 @@
 #include "prestera_hw.h"
 #include "prestera_acl.h"
 #include "prestera_counter.h"
+#include "prestera_router_hw.h"
 
 #define PRESTERA_SWITCH_INIT_TIMEOUT_MS (30 * 1000)
 
 #define PRESTERA_MIN_MTU 64
 
+#define PRESTERA_MSG_CHUNK_SIZE 1024
+
 enum prestera_cmd_type_t {
 	PRESTERA_CMD_TYPE_SWITCH_INIT = 0x1,
 	PRESTERA_CMD_TYPE_SWITCH_ATTR_SET = 0x2,
@@ -57,6 +60,10 @@ enum prestera_cmd_type_t {
 	PRESTERA_CMD_TYPE_ROUTER_RIF_DELETE = 0x601,
 	PRESTERA_CMD_TYPE_ROUTER_LPM_ADD = 0x610,
 	PRESTERA_CMD_TYPE_ROUTER_LPM_DELETE = 0x611,
+	PRESTERA_CMD_TYPE_ROUTER_NH_GRP_SET = 0x622,
+	PRESTERA_CMD_TYPE_ROUTER_NH_GRP_BLK_GET = 0x645,
+	PRESTERA_CMD_TYPE_ROUTER_NH_GRP_ADD = 0x623,
+	PRESTERA_CMD_TYPE_ROUTER_NH_GRP_DELETE = 0x624,
 	PRESTERA_CMD_TYPE_ROUTER_VR_CREATE = 0x630,
 	PRESTERA_CMD_TYPE_ROUTER_VR_DELETE = 0x631,
 
@@ -78,9 +85,11 @@ enum prestera_cmd_type_t {
 	PRESTERA_CMD_TYPE_STP_PORT_SET = 0x1000,
 
 	PRESTERA_CMD_TYPE_SPAN_GET = 0x1100,
-	PRESTERA_CMD_TYPE_SPAN_BIND = 0x1101,
-	PRESTERA_CMD_TYPE_SPAN_UNBIND = 0x1102,
+	PRESTERA_CMD_TYPE_SPAN_INGRESS_BIND = 0x1101,
+	PRESTERA_CMD_TYPE_SPAN_INGRESS_UNBIND = 0x1102,
 	PRESTERA_CMD_TYPE_SPAN_RELEASE = 0x1103,
+	PRESTERA_CMD_TYPE_SPAN_EGRESS_BIND = 0x1104,
+	PRESTERA_CMD_TYPE_SPAN_EGRESS_UNBIND = 0x1105,
 
 	PRESTERA_CMD_TYPE_POLICER_CREATE = 0x1500,
 	PRESTERA_CMD_TYPE_POLICER_RELEASE = 0x1501,
@@ -101,6 +110,7 @@ enum {
 	PRESTERA_CMD_PORT_ATTR_LEARNING = 7,
 	PRESTERA_CMD_PORT_ATTR_FLOOD = 8,
 	PRESTERA_CMD_PORT_ATTR_CAPABILITY = 9,
+	PRESTERA_CMD_PORT_ATTR_LOCKED = 10,
 	PRESTERA_CMD_PORT_ATTR_PHY_MODE = 12,
 	PRESTERA_CMD_PORT_ATTR_TYPE = 13,
 	PRESTERA_CMD_PORT_ATTR_STATS = 17,
@@ -285,6 +295,7 @@ union prestera_msg_port_param {
 	u8 duplex;
 	u8 fec;
 	u8 fc;
+	u8 br_locked;
 	union {
 		struct {
 			u8 admin;
@@ -538,6 +549,14 @@ struct prestera_msg_ip_addr {
 	u8 __pad[3];
 };
 
+struct prestera_msg_nh {
+	struct prestera_msg_iface oif;
+	__le32 hw_id;
+	u8 mac[ETH_ALEN];
+	u8 is_active;
+	u8 pad;
+};
+
 struct prestera_msg_rif_req {
 	struct prestera_msg_cmd cmd;
 	struct prestera_msg_iface iif;
@@ -563,6 +582,34 @@ struct prestera_msg_lpm_req {
 	u8 __pad[2];
 };
 
+struct prestera_msg_nh_req {
+	struct prestera_msg_cmd cmd;
+	struct prestera_msg_nh nh[PRESTERA_NHGR_SIZE_MAX];
+	__le32 size;
+	__le32 grp_id;
+};
+
+struct prestera_msg_nh_chunk_req {
+	struct prestera_msg_cmd cmd;
+	__le32 offset;
+};
+
+struct prestera_msg_nh_chunk_resp {
+	struct prestera_msg_ret ret;
+	u8 hw_state[PRESTERA_MSG_CHUNK_SIZE];
+};
+
+struct prestera_msg_nh_grp_req {
+	struct prestera_msg_cmd cmd;
+	__le32 grp_id;
+	__le32 size;
+};
+
+struct prestera_msg_nh_grp_resp {
+	struct prestera_msg_ret ret;
+	__le32 grp_id;
+};
+
 struct prestera_msg_vr_req {
 	struct prestera_msg_cmd cmd;
 	__le16 vr_id;
@@ -725,11 +772,15 @@ static void prestera_hw_build_tests(void)
 	BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_ports_reset_req) != 8);
 	BUILD_BUG_ON(sizeof(struct prestera_msg_mdb_create_req) != 16);
 	BUILD_BUG_ON(sizeof(struct prestera_msg_mdb_destroy_req) != 16);
+	BUILD_BUG_ON(sizeof(struct prestera_msg_nh_req) != 124);
+	BUILD_BUG_ON(sizeof(struct prestera_msg_nh_chunk_req) != 8);
+	BUILD_BUG_ON(sizeof(struct prestera_msg_nh_grp_req) != 12);
 
 	/*  structure that are part of req/resp fw messages */
 	BUILD_BUG_ON(sizeof(struct prestera_msg_iface) != 16);
 	BUILD_BUG_ON(sizeof(struct prestera_msg_ip_addr) != 20);
 	BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_port) != 12);
+	BUILD_BUG_ON(sizeof(struct prestera_msg_nh) != 28);
 
 	/* check responses */
 	BUILD_BUG_ON(sizeof(struct prestera_msg_common_resp) != 8);
@@ -745,6 +796,9 @@ static void prestera_hw_build_tests(void)
 	BUILD_BUG_ON(sizeof(struct prestera_msg_rif_resp) != 12);
 	BUILD_BUG_ON(sizeof(struct prestera_msg_vr_resp) != 12);
 	BUILD_BUG_ON(sizeof(struct prestera_msg_policer_resp) != 12);
+	BUILD_BUG_ON(sizeof(struct prestera_msg_flood_domain_create_resp) != 12);
+	BUILD_BUG_ON(sizeof(struct prestera_msg_nh_chunk_resp) != 1032);
+	BUILD_BUG_ON(sizeof(struct prestera_msg_nh_grp_resp) != 12);
 
 	/* check events */
 	BUILD_BUG_ON(sizeof(struct prestera_msg_event_port) != 20);
@@ -1022,6 +1076,8 @@ int prestera_hw_switch_init(struct prestera_switch *sw)
 	sw->id = resp.switch_id;
 	sw->lag_member_max = resp.lag_member_max;
 	sw->lag_max = resp.lag_max;
+	sw->size_tbl_router_nexthop =
+		__le32_to_cpu(resp.size_tbl_router_nexthop);
 
 	return 0;
 }
@@ -1431,27 +1487,39 @@ int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id)
 	return 0;
 }
 
-int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id)
+int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id,
+			  bool ingress)
 {
 	struct prestera_msg_span_req req = {
 		.port = __cpu_to_le32(port->hw_id),
 		.dev = __cpu_to_le32(port->dev_id),
 		.id = span_id,
 	};
+	enum prestera_cmd_type_t cmd_type;
+
+	if (ingress)
+		cmd_type = PRESTERA_CMD_TYPE_SPAN_INGRESS_BIND;
+	else
+		cmd_type = PRESTERA_CMD_TYPE_SPAN_EGRESS_BIND;
+
+	return prestera_cmd(port->sw, cmd_type, &req.cmd, sizeof(req));
 
-	return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_SPAN_BIND,
-			    &req.cmd, sizeof(req));
 }
 
-int prestera_hw_span_unbind(const struct prestera_port *port)
+int prestera_hw_span_unbind(const struct prestera_port *port, bool ingress)
 {
 	struct prestera_msg_span_req req = {
 		.port = __cpu_to_le32(port->hw_id),
 		.dev = __cpu_to_le32(port->dev_id),
 	};
+	enum prestera_cmd_type_t cmd_type;
 
-	return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_SPAN_UNBIND,
-			    &req.cmd, sizeof(req));
+	if (ingress)
+		cmd_type = PRESTERA_CMD_TYPE_SPAN_INGRESS_UNBIND;
+	else
+		cmd_type = PRESTERA_CMD_TYPE_SPAN_EGRESS_UNBIND;
+
+	return prestera_cmd(port->sw, cmd_type, &req.cmd, sizeof(req));
 }
 
 int prestera_hw_span_release(struct prestera_switch *sw, u8 span_id)
@@ -1639,6 +1707,22 @@ int prestera_hw_port_mc_flood_set(const struct prestera_port *port, bool flood)
 			    &req.cmd, sizeof(req));
 }
 
+int prestera_hw_port_br_locked_set(const struct prestera_port *port,
+				   bool br_locked)
+{
+	struct prestera_msg_port_attr_req req = {
+		.attr = __cpu_to_le32(PRESTERA_CMD_PORT_ATTR_LOCKED),
+		.port = __cpu_to_le32(port->hw_id),
+		.dev = __cpu_to_le32(port->dev_id),
+		.param = {
+			.br_locked = br_locked,
+		}
+	};
+
+	return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_PORT_ATTR_SET,
+			    &req.cmd, sizeof(req));
+}
+
 int prestera_hw_vlan_create(struct prestera_switch *sw, u16 vid)
 {
 	struct prestera_msg_vlan_req req = {
@@ -2004,6 +2088,85 @@ int prestera_hw_lpm_del(struct prestera_switch *sw, u16 vr_id,
 			    sizeof(req));
 }
 
+int prestera_hw_nh_entries_set(struct prestera_switch *sw, int count,
+			       struct prestera_neigh_info *nhs, u32 grp_id)
+{
+	struct prestera_msg_nh_req req = { .size = __cpu_to_le32((u32)count),
+			.grp_id = __cpu_to_le32(grp_id) };
+	int i, err;
+
+	for (i = 0; i < count; i++) {
+		req.nh[i].is_active = nhs[i].connected;
+		memcpy(&req.nh[i].mac, nhs[i].ha, ETH_ALEN);
+		err = prestera_iface_to_msg(&nhs[i].iface, &req.nh[i].oif);
+		if (err)
+			return err;
+	}
+
+	return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_SET, &req.cmd,
+			    sizeof(req));
+}
+
+int prestera_hw_nhgrp_blk_get(struct prestera_switch *sw,
+			      u8 *hw_state, u32 buf_size /* Buffer in bytes */)
+{
+	static struct prestera_msg_nh_chunk_resp resp;
+	struct prestera_msg_nh_chunk_req req;
+	u32 buf_offset;
+	int err;
+
+	memset(&hw_state[0], 0, buf_size);
+	buf_offset = 0;
+	while (1) {
+		if (buf_offset >= buf_size)
+			break;
+
+		memset(&req, 0, sizeof(req));
+		req.offset = __cpu_to_le32(buf_offset * 8); /* 8 bits in u8 */
+		err = prestera_cmd_ret(sw,
+				       PRESTERA_CMD_TYPE_ROUTER_NH_GRP_BLK_GET,
+				       &req.cmd, sizeof(req), &resp.ret,
+				       sizeof(resp));
+		if (err)
+			return err;
+
+		memcpy(&hw_state[buf_offset], &resp.hw_state[0],
+		       buf_offset + PRESTERA_MSG_CHUNK_SIZE > buf_size ?
+			buf_size - buf_offset : PRESTERA_MSG_CHUNK_SIZE);
+		buf_offset += PRESTERA_MSG_CHUNK_SIZE;
+	}
+
+	return 0;
+}
+
+int prestera_hw_nh_group_create(struct prestera_switch *sw, u16 nh_count,
+				u32 *grp_id)
+{
+	struct prestera_msg_nh_grp_req req = { .size = __cpu_to_le32((u32)nh_count) };
+	struct prestera_msg_nh_grp_resp resp;
+	int err;
+
+	err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_ADD,
+			       &req.cmd, sizeof(req), &resp.ret, sizeof(resp));
+	if (err)
+		return err;
+
+	*grp_id = __le32_to_cpu(resp.grp_id);
+	return err;
+}
+
+int prestera_hw_nh_group_delete(struct prestera_switch *sw, u16 nh_count,
+				u32 grp_id)
+{
+	struct prestera_msg_nh_grp_req req = {
+	    .grp_id = __cpu_to_le32(grp_id),
+	    .size = __cpu_to_le32(nh_count)
+	};
+
+	return prestera_cmd(sw, PRESTERA_CMD_TYPE_ROUTER_NH_GRP_DELETE,
+			    &req.cmd, sizeof(req));
+}
+
 int prestera_hw_rxtx_init(struct prestera_switch *sw,
 			  struct prestera_rxtx_params *params)
 {
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.h b/drivers/net/ethernet/marvell/prestera/prestera_hw.h
index 56e043146dd2..0a929279e1ce 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_hw.h
+++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.h
@@ -146,6 +146,7 @@ struct prestera_counter_stats;
 struct prestera_iface;
 struct prestera_flood_domain;
 struct prestera_mdb_entry;
+struct prestera_neigh_info;
 
 /* Switch API */
 int prestera_hw_switch_init(struct prestera_switch *sw);
@@ -183,6 +184,8 @@ int prestera_hw_port_speed_get(const struct prestera_port *port, u32 *speed);
 int prestera_hw_port_learning_set(struct prestera_port *port, bool enable);
 int prestera_hw_port_uc_flood_set(const struct prestera_port *port, bool flood);
 int prestera_hw_port_mc_flood_set(const struct prestera_port *port, bool flood);
+int prestera_hw_port_br_locked_set(const struct prestera_port *port,
+				   bool br_locked);
 int prestera_hw_port_accept_frm_type(struct prestera_port *port,
 				     enum prestera_accept_frm_type type);
 /* Vlan API */
@@ -243,8 +246,9 @@ int prestera_hw_counter_clear(struct prestera_switch *sw, u32 block_id,
 
 /* SPAN API */
 int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id);
-int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id);
-int prestera_hw_span_unbind(const struct prestera_port *port);
+int prestera_hw_span_bind(const struct prestera_port *port, u8 span_id,
+			  bool ingress);
+int prestera_hw_span_unbind(const struct prestera_port *port, bool ingress);
 int prestera_hw_span_release(struct prestera_switch *sw, u8 span_id);
 
 /* Router API */
@@ -263,6 +267,16 @@ int prestera_hw_lpm_add(struct prestera_switch *sw, u16 vr_id,
 int prestera_hw_lpm_del(struct prestera_switch *sw, u16 vr_id,
 			__be32 dst, u32 dst_len);
 
+/* NH API */
+int prestera_hw_nh_entries_set(struct prestera_switch *sw, int count,
+			       struct prestera_neigh_info *nhs, u32 grp_id);
+int prestera_hw_nhgrp_blk_get(struct prestera_switch *sw,
+			      u8 *hw_state, u32 buf_size /* Buffer in bytes */);
+int prestera_hw_nh_group_create(struct prestera_switch *sw, u16 nh_count,
+				u32 *grp_id);
+int prestera_hw_nh_group_delete(struct prestera_switch *sw, u16 nh_count,
+				u32 grp_id);
+
 /* Event handlers */
 int prestera_hw_event_handler_register(struct prestera_switch *sw,
 				       enum prestera_event_type type,
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c
index a895862b4821..24f9d6024745 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_main.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c
@@ -36,6 +36,17 @@ void prestera_queue_work(struct work_struct *work)
 	queue_work(prestera_owq, work);
 }
 
+void prestera_queue_delayed_work(struct delayed_work *work, unsigned long delay)
+{
+	queue_delayed_work(prestera_wq, work, delay);
+}
+
+void prestera_queue_drain(void)
+{
+	drain_workqueue(prestera_wq);
+	drain_workqueue(prestera_owq);
+}
+
 int prestera_port_learning_set(struct prestera_port *port, bool learn)
 {
 	return prestera_hw_port_learning_set(port, learn);
@@ -51,6 +62,11 @@ int prestera_port_mc_flood_set(struct prestera_port *port, bool flood)
 	return prestera_hw_port_mc_flood_set(port, flood);
 }
 
+int prestera_port_br_locked_set(struct prestera_port *port, bool br_locked)
+{
+	return prestera_hw_port_br_locked_set(port, br_locked);
+}
+
 int prestera_port_pvid_set(struct prestera_port *port, u16 vid)
 {
 	enum prestera_accept_frm_type frm_type;
@@ -799,32 +815,30 @@ static void prestera_port_handle_event(struct prestera_switch *sw,
 
 		caching_dw = &port->cached_hw_stats.caching_dw;
 
-		if (port->phy_link) {
-			memset(&smac, 0, sizeof(smac));
-			smac.valid = true;
-			smac.oper = pevt->data.mac.oper;
-			if (smac.oper) {
-				smac.mode = pevt->data.mac.mode;
-				smac.speed = pevt->data.mac.speed;
-				smac.duplex = pevt->data.mac.duplex;
-				smac.fc = pevt->data.mac.fc;
-				smac.fec = pevt->data.mac.fec;
-				phylink_mac_change(port->phy_link, true);
-			} else {
-				phylink_mac_change(port->phy_link, false);
-			}
-			prestera_port_mac_state_cache_write(port, &smac);
+		memset(&smac, 0, sizeof(smac));
+		smac.valid = true;
+		smac.oper = pevt->data.mac.oper;
+		if (smac.oper) {
+			smac.mode = pevt->data.mac.mode;
+			smac.speed = pevt->data.mac.speed;
+			smac.duplex = pevt->data.mac.duplex;
+			smac.fc = pevt->data.mac.fc;
+			smac.fec = pevt->data.mac.fec;
 		}
+		prestera_port_mac_state_cache_write(port, &smac);
 
 		if (port->state_mac.oper) {
-			if (!port->phy_link)
+			if (port->phy_link)
+				phylink_mac_change(port->phy_link, true);
+			else
 				netif_carrier_on(port->dev);
 
 			if (!delayed_work_pending(caching_dw))
 				queue_delayed_work(prestera_wq, caching_dw, 0);
-		} else if (netif_running(port->dev) &&
-			   netif_carrier_ok(port->dev)) {
-			if (!port->phy_link)
+		} else {
+			if (port->phy_link)
+				phylink_mac_change(port->phy_link, false);
+			else if (netif_running(port->dev) && netif_carrier_ok(port->dev))
 				netif_carrier_off(port->dev);
 
 			if (delayed_work_pending(caching_dw))
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_matchall.c b/drivers/net/ethernet/marvell/prestera/prestera_matchall.c
new file mode 100644
index 000000000000..6f2b95a5263e
--- /dev/null
+++ b/drivers/net/ethernet/marvell/prestera/prestera_matchall.c
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
+/* Copyright (c) 2019-2022 Marvell International Ltd. All rights reserved */
+
+#include <linux/kernel.h>
+#include <linux/list.h>
+
+#include "prestera.h"
+#include "prestera_hw.h"
+#include "prestera_flow.h"
+#include "prestera_flower.h"
+#include "prestera_matchall.h"
+#include "prestera_span.h"
+
+static int prestera_mall_prio_check(struct prestera_flow_block *block,
+				    struct tc_cls_matchall_offload *f)
+{
+	u32 flower_prio_min;
+	u32 flower_prio_max;
+	int err;
+
+	err = prestera_flower_prio_get(block, f->common.chain_index,
+				       &flower_prio_min, &flower_prio_max);
+	if (err == -ENOENT)
+		/* No flower filters installed on this chain. */
+		return 0;
+
+	if (err) {
+		NL_SET_ERR_MSG(f->common.extack, "Failed to get flower priorities");
+		return err;
+	}
+
+	if (f->common.prio <= flower_prio_max && !block->ingress) {
+		NL_SET_ERR_MSG(f->common.extack, "Failed to add in front of existing flower rules");
+		return -EOPNOTSUPP;
+	}
+	if (f->common.prio >= flower_prio_min && block->ingress) {
+		NL_SET_ERR_MSG(f->common.extack, "Failed to add behind of existing flower rules");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+int prestera_mall_prio_get(struct prestera_flow_block *block,
+			   u32 *prio_min, u32 *prio_max)
+{
+	if (!block->mall.bound)
+		return -ENOENT;
+
+	*prio_min = block->mall.prio_min;
+	*prio_max = block->mall.prio_max;
+	return 0;
+}
+
+static void prestera_mall_prio_update(struct prestera_flow_block *block,
+				      struct tc_cls_matchall_offload *f)
+{
+	block->mall.prio_min = min(block->mall.prio_min, f->common.prio);
+	block->mall.prio_max = max(block->mall.prio_max, f->common.prio);
+}
+
+int prestera_mall_replace(struct prestera_flow_block *block,
+			  struct tc_cls_matchall_offload *f)
+{
+	struct prestera_flow_block_binding *binding;
+	__be16 protocol = f->common.protocol;
+	struct flow_action_entry *act;
+	struct prestera_port *port;
+	int err;
+
+	if (!flow_offload_has_one_action(&f->rule->action)) {
+		NL_SET_ERR_MSG(f->common.extack,
+			       "Only singular actions are supported");
+		return -EOPNOTSUPP;
+	}
+
+	act = &f->rule->action.entries[0];
+
+	if (!prestera_netdev_check(act->dev)) {
+		NL_SET_ERR_MSG(f->common.extack,
+			       "Only Marvell Prestera port is supported");
+		return -EINVAL;
+	}
+	if (!tc_cls_can_offload_and_chain0(act->dev, &f->common))
+		return -EOPNOTSUPP;
+	if (act->id != FLOW_ACTION_MIRRED)
+		return -EOPNOTSUPP;
+	if (protocol != htons(ETH_P_ALL))
+		return -EOPNOTSUPP;
+
+	err = prestera_mall_prio_check(block, f);
+	if (err)
+		return err;
+
+	port = netdev_priv(act->dev);
+
+	list_for_each_entry(binding, &block->binding_list, list) {
+		err = prestera_span_rule_add(binding, port, block->ingress);
+		if (err)
+			goto rollback;
+	}
+
+	prestera_mall_prio_update(block, f);
+
+	block->mall.bound = true;
+	return 0;
+
+rollback:
+	list_for_each_entry_continue_reverse(binding,
+					     &block->binding_list, list)
+		prestera_span_rule_del(binding, block->ingress);
+	return err;
+}
+
+void prestera_mall_destroy(struct prestera_flow_block *block)
+{
+	struct prestera_flow_block_binding *binding;
+
+	list_for_each_entry(binding, &block->binding_list, list)
+		prestera_span_rule_del(binding, block->ingress);
+
+	block->mall.prio_min = UINT_MAX;
+	block->mall.prio_max = 0;
+	block->mall.bound = false;
+}
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_matchall.h b/drivers/net/ethernet/marvell/prestera/prestera_matchall.h
new file mode 100644
index 000000000000..fed08be80257
--- /dev/null
+++ b/drivers/net/ethernet/marvell/prestera/prestera_matchall.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */
+/* Copyright (c) 2022 Marvell International Ltd. All rights reserved. */
+
+#ifndef _PRESTERA_MATCHALL_H_
+#define _PRESTERA_MATCHALL_H_
+
+#include <net/pkt_cls.h>
+
+struct prestera_flow_block;
+
+int prestera_mall_replace(struct prestera_flow_block *block,
+			  struct tc_cls_matchall_offload *f);
+void prestera_mall_destroy(struct prestera_flow_block *block);
+int prestera_mall_prio_get(struct prestera_flow_block *block,
+			   u32 *prio_min, u32 *prio_max);
+
+#endif /* _PRESTERA_MATCHALL_H_ */
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router.c b/drivers/net/ethernet/marvell/prestera/prestera_router.c
index 58f4e44d5ad7..4046be0e86ff 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_router.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_router.c
@@ -7,10 +7,35 @@
 #include <net/inet_dscp.h>
 #include <net/switchdev.h>
 #include <linux/rhashtable.h>
+#include <net/nexthop.h>
+#include <net/arp.h>
+#include <linux/if_vlan.h>
+#include <linux/if_macvlan.h>
+#include <net/netevent.h>
 
 #include "prestera.h"
 #include "prestera_router_hw.h"
 
+#define PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH
+#define PRESTERA_NH_PROBE_INTERVAL 5000 /* ms */
+
+struct prestera_kern_neigh_cache_key {
+	struct prestera_ip_addr addr;
+	struct net_device *dev;
+};
+
+struct prestera_kern_neigh_cache {
+	struct prestera_kern_neigh_cache_key key;
+	struct rhash_head ht_node;
+	struct list_head kern_fib_cache_list;
+	/* Hold prepared nh_neigh info if is in_kernel */
+	struct prestera_neigh_info nh_neigh_info;
+	/* Indicate if neighbour is reachable by direct route */
+	bool reachable;
+	/* Lock cache if neigh is present in kernel */
+	bool in_kernel;
+};
+
 struct prestera_kern_fib_cache_key {
 	struct prestera_ip_addr addr;
 	u32 prefix_len;
@@ -23,15 +48,29 @@ struct prestera_kern_fib_cache {
 	struct {
 		struct prestera_fib_key fib_key;
 		enum prestera_fib_type fib_type;
+		struct prestera_nexthop_group_key nh_grp_key;
 	} lpm_info; /* hold prepared lpm info */
 	/* Indicate if route is not overlapped by another table */
 	struct rhash_head ht_node; /* node of prestera_router */
-	struct fib_info *fi;
-	dscp_t kern_dscp;
-	u8 kern_type;
+	struct prestera_kern_neigh_cache_head {
+		struct prestera_kern_fib_cache *this;
+		struct list_head head;
+		struct prestera_kern_neigh_cache *n_cache;
+	} kern_neigh_cache_head[PRESTERA_NHGR_SIZE_MAX];
+	union {
+		struct fib_notifier_info info; /* point to any of 4/6 */
+		struct fib_entry_notifier_info fen4_info;
+	};
 	bool reachable;
 };
 
+static const struct rhashtable_params __prestera_kern_neigh_cache_ht_params = {
+	.key_offset  = offsetof(struct prestera_kern_neigh_cache, key),
+	.head_offset = offsetof(struct prestera_kern_neigh_cache, ht_node),
+	.key_len     = sizeof(struct prestera_kern_neigh_cache_key),
+	.automatic_shrinking = true,
+};
+
 static const struct rhashtable_params __prestera_kern_fib_cache_ht_params = {
 	.key_offset  = offsetof(struct prestera_kern_fib_cache, key),
 	.head_offset = offsetof(struct prestera_kern_fib_cache, ht_node),
@@ -51,15 +90,450 @@ static u32 prestera_fix_tb_id(u32 tb_id)
 }
 
 static void
-prestera_util_fen_info2fib_cache_key(struct fib_entry_notifier_info *fen_info,
+prestera_util_fen_info2fib_cache_key(struct fib_notifier_info *info,
 				     struct prestera_kern_fib_cache_key *key)
 {
+	struct fib_entry_notifier_info *fen_info =
+		container_of(info, struct fib_entry_notifier_info, info);
+
 	memset(key, 0, sizeof(*key));
+	key->addr.v = PRESTERA_IPV4;
 	key->addr.u.ipv4 = cpu_to_be32(fen_info->dst);
 	key->prefix_len = fen_info->dst_len;
 	key->kern_tb_id = fen_info->tb_id;
 }
 
+static int prestera_util_nhc2nc_key(struct prestera_switch *sw,
+				    struct fib_nh_common *nhc,
+				    struct prestera_kern_neigh_cache_key *nk)
+{
+	memset(nk, 0, sizeof(*nk));
+	if (nhc->nhc_gw_family == AF_INET) {
+		nk->addr.v = PRESTERA_IPV4;
+		nk->addr.u.ipv4 = nhc->nhc_gw.ipv4;
+	} else {
+		nk->addr.v = PRESTERA_IPV6;
+		nk->addr.u.ipv6 = nhc->nhc_gw.ipv6;
+	}
+
+	nk->dev = nhc->nhc_dev;
+	return 0;
+}
+
+static void
+prestera_util_nc_key2nh_key(struct prestera_kern_neigh_cache_key *ck,
+			    struct prestera_nh_neigh_key *nk)
+{
+	memset(nk, 0, sizeof(*nk));
+	nk->addr = ck->addr;
+	nk->rif = (void *)ck->dev;
+}
+
+static bool
+prestera_util_nhc_eq_n_cache_key(struct prestera_switch *sw,
+				 struct fib_nh_common *nhc,
+				 struct prestera_kern_neigh_cache_key *nk)
+{
+	struct prestera_kern_neigh_cache_key tk;
+	int err;
+
+	err = prestera_util_nhc2nc_key(sw, nhc, &tk);
+	if (err)
+		return false;
+
+	if (memcmp(&tk, nk, sizeof(tk)))
+		return false;
+
+	return true;
+}
+
+static int
+prestera_util_neigh2nc_key(struct prestera_switch *sw, struct neighbour *n,
+			   struct prestera_kern_neigh_cache_key *key)
+{
+	memset(key, 0, sizeof(*key));
+	if (n->tbl->family == AF_INET) {
+		key->addr.v = PRESTERA_IPV4;
+		key->addr.u.ipv4 = *(__be32 *)n->primary_key;
+	} else {
+		return -ENOENT;
+	}
+
+	key->dev = n->dev;
+
+	return 0;
+}
+
+static bool __prestera_fi_is_direct(struct fib_info *fi)
+{
+	struct fib_nh *fib_nh;
+
+	if (fib_info_num_path(fi) == 1) {
+		fib_nh = fib_info_nh(fi, 0);
+		if (fib_nh->fib_nh_gw_family == AF_UNSPEC)
+			return true;
+	}
+
+	return false;
+}
+
+static bool prestera_fi_is_direct(struct fib_info *fi)
+{
+	if (fi->fib_type != RTN_UNICAST)
+		return false;
+
+	return __prestera_fi_is_direct(fi);
+}
+
+static bool prestera_fi_is_nh(struct fib_info *fi)
+{
+	if (fi->fib_type != RTN_UNICAST)
+		return false;
+
+	return !__prestera_fi_is_direct(fi);
+}
+
+static bool __prestera_fi6_is_direct(struct fib6_info *fi)
+{
+	if (!fi->fib6_nh->nh_common.nhc_gw_family)
+		return true;
+
+	return false;
+}
+
+static bool prestera_fi6_is_direct(struct fib6_info *fi)
+{
+	if (fi->fib6_type != RTN_UNICAST)
+		return false;
+
+	return __prestera_fi6_is_direct(fi);
+}
+
+static bool prestera_fi6_is_nh(struct fib6_info *fi)
+{
+	if (fi->fib6_type != RTN_UNICAST)
+		return false;
+
+	return !__prestera_fi6_is_direct(fi);
+}
+
+static bool prestera_fib_info_is_direct(struct fib_notifier_info *info)
+{
+	struct fib6_entry_notifier_info *fen6_info =
+		container_of(info, struct fib6_entry_notifier_info, info);
+	struct fib_entry_notifier_info *fen_info =
+		container_of(info, struct fib_entry_notifier_info, info);
+
+	if (info->family == AF_INET)
+		return prestera_fi_is_direct(fen_info->fi);
+	else
+		return prestera_fi6_is_direct(fen6_info->rt);
+}
+
+static bool prestera_fib_info_is_nh(struct fib_notifier_info *info)
+{
+	struct fib6_entry_notifier_info *fen6_info =
+		container_of(info, struct fib6_entry_notifier_info, info);
+	struct fib_entry_notifier_info *fen_info =
+		container_of(info, struct fib_entry_notifier_info, info);
+
+	if (info->family == AF_INET)
+		return prestera_fi_is_nh(fen_info->fi);
+	else
+		return prestera_fi6_is_nh(fen6_info->rt);
+}
+
+/* must be called with rcu_read_lock() */
+static int prestera_util_kern_get_route(struct fib_result *res, u32 tb_id,
+					__be32 *addr)
+{
+	struct flowi4 fl4;
+
+	/* TODO: walkthrough appropriate tables in kernel
+	 * to know if the same prefix exists in several tables
+	 */
+	memset(&fl4, 0, sizeof(fl4));
+	fl4.daddr = *addr;
+	return fib_lookup(&init_net, &fl4, res, 0 /* FIB_LOOKUP_NOREF */);
+}
+
+static bool
+__prestera_util_kern_n_is_reachable_v4(u32 tb_id, __be32 *addr,
+				       struct net_device *dev)
+{
+	struct fib_nh *fib_nh;
+	struct fib_result res;
+	bool reachable;
+
+	reachable = false;
+
+	if (!prestera_util_kern_get_route(&res, tb_id, addr))
+		if (prestera_fi_is_direct(res.fi)) {
+			fib_nh = fib_info_nh(res.fi, 0);
+			if (dev == fib_nh->fib_nh_dev)
+				reachable = true;
+		}
+
+	return reachable;
+}
+
+/* Check if neigh route is reachable */
+static bool
+prestera_util_kern_n_is_reachable(u32 tb_id,
+				  struct prestera_ip_addr *addr,
+				  struct net_device *dev)
+{
+	if (addr->v == PRESTERA_IPV4)
+		return __prestera_util_kern_n_is_reachable_v4(tb_id,
+							      &addr->u.ipv4,
+							      dev);
+	else
+		return false;
+}
+
+static void prestera_util_kern_set_neigh_offload(struct neighbour *n,
+						 bool offloaded)
+{
+	if (offloaded)
+		n->flags |= NTF_OFFLOADED;
+	else
+		n->flags &= ~NTF_OFFLOADED;
+}
+
+static void
+prestera_util_kern_set_nh_offload(struct fib_nh_common *nhc, bool offloaded, bool trap)
+{
+		if (offloaded)
+			nhc->nhc_flags |= RTNH_F_OFFLOAD;
+		else
+			nhc->nhc_flags &= ~RTNH_F_OFFLOAD;
+
+		if (trap)
+			nhc->nhc_flags |= RTNH_F_TRAP;
+		else
+			nhc->nhc_flags &= ~RTNH_F_TRAP;
+}
+
+static struct fib_nh_common *
+prestera_kern_fib_info_nhc(struct fib_notifier_info *info, int n)
+{
+	struct fib6_entry_notifier_info *fen6_info;
+	struct fib_entry_notifier_info *fen4_info;
+	struct fib6_info *iter;
+
+	if (info->family == AF_INET) {
+		fen4_info = container_of(info, struct fib_entry_notifier_info,
+					 info);
+		return &fib_info_nh(fen4_info->fi, n)->nh_common;
+	} else if (info->family == AF_INET6) {
+		fen6_info = container_of(info, struct fib6_entry_notifier_info,
+					 info);
+		if (!n)
+			return &fen6_info->rt->fib6_nh->nh_common;
+
+		list_for_each_entry(iter, &fen6_info->rt->fib6_siblings,
+				    fib6_siblings) {
+			if (!--n)
+				return &iter->fib6_nh->nh_common;
+		}
+	}
+
+	/* if family is incorrect - than upper functions has BUG */
+	/* if doesn't find requested index - there is alsi bug, because
+	 * valid index must be produced by nhs, which checks list length
+	 */
+	WARN(1, "Invalid parameters passed to %s n=%d i=%p",
+	     __func__, n, info);
+	return NULL;
+}
+
+static int prestera_kern_fib_info_nhs(struct fib_notifier_info *info)
+{
+	struct fib6_entry_notifier_info *fen6_info;
+	struct fib_entry_notifier_info *fen4_info;
+
+	if (info->family == AF_INET) {
+		fen4_info = container_of(info, struct fib_entry_notifier_info,
+					 info);
+		return fib_info_num_path(fen4_info->fi);
+	} else if (info->family == AF_INET6) {
+		fen6_info = container_of(info, struct fib6_entry_notifier_info,
+					 info);
+		return fen6_info->rt->fib6_nsiblings + 1;
+	}
+
+	return 0;
+}
+
+static unsigned char
+prestera_kern_fib_info_type(struct fib_notifier_info *info)
+{
+	struct fib6_entry_notifier_info *fen6_info;
+	struct fib_entry_notifier_info *fen4_info;
+
+	if (info->family == AF_INET) {
+		fen4_info = container_of(info, struct fib_entry_notifier_info,
+					 info);
+		return fen4_info->fi->fib_type;
+	} else if (info->family == AF_INET6) {
+		fen6_info = container_of(info, struct fib6_entry_notifier_info,
+					 info);
+		/* TODO: ECMP in ipv6 is several routes.
+		 * Every route has single nh.
+		 */
+		return fen6_info->rt->fib6_type;
+	}
+
+	return RTN_UNSPEC;
+}
+
+/* Decided, that uc_nh route with key==nh is obviously neighbour route */
+static bool
+prestera_fib_node_util_is_neighbour(struct prestera_fib_node *fib_node)
+{
+	if (fib_node->info.type != PRESTERA_FIB_TYPE_UC_NH)
+		return false;
+
+	if (fib_node->info.nh_grp->nh_neigh_head[1].neigh)
+		return false;
+
+	if (!fib_node->info.nh_grp->nh_neigh_head[0].neigh)
+		return false;
+
+	if (memcmp(&fib_node->info.nh_grp->nh_neigh_head[0].neigh->key.addr,
+		   &fib_node->key.addr, sizeof(struct prestera_ip_addr)))
+		return false;
+
+	return true;
+}
+
+static int prestera_dev_if_type(const struct net_device *dev)
+{
+	struct macvlan_dev *vlan;
+
+	if (is_vlan_dev(dev) &&
+	    netif_is_bridge_master(vlan_dev_real_dev(dev))) {
+		return PRESTERA_IF_VID_E;
+	} else if (netif_is_bridge_master(dev)) {
+		return PRESTERA_IF_VID_E;
+	} else if (netif_is_lag_master(dev)) {
+		return PRESTERA_IF_LAG_E;
+	} else if (netif_is_macvlan(dev)) {
+		vlan = netdev_priv(dev);
+		return prestera_dev_if_type(vlan->lowerdev);
+	} else {
+		return PRESTERA_IF_PORT_E;
+	}
+}
+
+static int
+prestera_neigh_iface_init(struct prestera_switch *sw,
+			  struct prestera_iface *iface,
+			  struct neighbour *n)
+{
+	struct prestera_port *port;
+
+	iface->vlan_id = 0; /* TODO: vlan egress */
+	iface->type = prestera_dev_if_type(n->dev);
+	if (iface->type != PRESTERA_IF_PORT_E)
+		return -EINVAL;
+
+	if (!prestera_netdev_check(n->dev))
+		return -EINVAL;
+
+	port = netdev_priv(n->dev);
+	iface->dev_port.hw_dev_num = port->dev_id;
+	iface->dev_port.port_num = port->hw_id;
+
+	return 0;
+}
+
+static struct prestera_kern_neigh_cache *
+prestera_kern_neigh_cache_find(struct prestera_switch *sw,
+			       struct prestera_kern_neigh_cache_key *key)
+{
+	struct prestera_kern_neigh_cache *n_cache;
+
+	n_cache =
+	 rhashtable_lookup_fast(&sw->router->kern_neigh_cache_ht, key,
+				__prestera_kern_neigh_cache_ht_params);
+	return IS_ERR(n_cache) ? NULL : n_cache;
+}
+
+static void
+__prestera_kern_neigh_cache_destruct(struct prestera_switch *sw,
+				     struct prestera_kern_neigh_cache *n_cache)
+{
+	dev_put(n_cache->key.dev);
+}
+
+static void
+__prestera_kern_neigh_cache_destroy(struct prestera_switch *sw,
+				    struct prestera_kern_neigh_cache *n_cache)
+{
+	rhashtable_remove_fast(&sw->router->kern_neigh_cache_ht,
+			       &n_cache->ht_node,
+			       __prestera_kern_neigh_cache_ht_params);
+	__prestera_kern_neigh_cache_destruct(sw, n_cache);
+	kfree(n_cache);
+}
+
+static struct prestera_kern_neigh_cache *
+__prestera_kern_neigh_cache_create(struct prestera_switch *sw,
+				   struct prestera_kern_neigh_cache_key *key)
+{
+	struct prestera_kern_neigh_cache *n_cache;
+	int err;
+
+	n_cache = kzalloc(sizeof(*n_cache), GFP_KERNEL);
+	if (!n_cache)
+		goto err_kzalloc;
+
+	memcpy(&n_cache->key, key, sizeof(*key));
+	dev_hold(n_cache->key.dev);
+
+	INIT_LIST_HEAD(&n_cache->kern_fib_cache_list);
+	err = rhashtable_insert_fast(&sw->router->kern_neigh_cache_ht,
+				     &n_cache->ht_node,
+				     __prestera_kern_neigh_cache_ht_params);
+	if (err)
+		goto err_ht_insert;
+
+	return n_cache;
+
+err_ht_insert:
+	dev_put(n_cache->key.dev);
+	kfree(n_cache);
+err_kzalloc:
+	return NULL;
+}
+
+static struct prestera_kern_neigh_cache *
+prestera_kern_neigh_cache_get(struct prestera_switch *sw,
+			      struct prestera_kern_neigh_cache_key *key)
+{
+	struct prestera_kern_neigh_cache *n_cache;
+
+	n_cache = prestera_kern_neigh_cache_find(sw, key);
+	if (!n_cache)
+		n_cache = __prestera_kern_neigh_cache_create(sw, key);
+
+	return n_cache;
+}
+
+static struct prestera_kern_neigh_cache *
+prestera_kern_neigh_cache_put(struct prestera_switch *sw,
+			      struct prestera_kern_neigh_cache *n_cache)
+{
+	if (!n_cache->in_kernel &&
+	    list_empty(&n_cache->kern_fib_cache_list)) {
+		__prestera_kern_neigh_cache_destroy(sw, n_cache);
+		return NULL;
+	}
+
+	return n_cache;
+}
+
 static struct prestera_kern_fib_cache *
 prestera_kern_fib_cache_find(struct prestera_switch *sw,
 			     struct prestera_kern_fib_cache_key *key)
@@ -73,24 +547,79 @@ prestera_kern_fib_cache_find(struct prestera_switch *sw,
 }
 
 static void
+__prestera_kern_fib_cache_destruct(struct prestera_switch *sw,
+				   struct prestera_kern_fib_cache *fib_cache)
+{
+	struct prestera_kern_neigh_cache *n_cache;
+	int i;
+
+	for (i = 0; i < PRESTERA_NHGR_SIZE_MAX; i++) {
+		n_cache = fib_cache->kern_neigh_cache_head[i].n_cache;
+		if (n_cache) {
+			list_del(&fib_cache->kern_neigh_cache_head[i].head);
+			prestera_kern_neigh_cache_put(sw, n_cache);
+		}
+	}
+
+	fib_info_put(fib_cache->fen4_info.fi);
+}
+
+static void
 prestera_kern_fib_cache_destroy(struct prestera_switch *sw,
 				struct prestera_kern_fib_cache *fib_cache)
 {
-	fib_info_put(fib_cache->fi);
 	rhashtable_remove_fast(&sw->router->kern_fib_cache_ht,
 			       &fib_cache->ht_node,
 			       __prestera_kern_fib_cache_ht_params);
+	__prestera_kern_fib_cache_destruct(sw, fib_cache);
 	kfree(fib_cache);
 }
 
+static int
+__prestera_kern_fib_cache_create_nhs(struct prestera_switch *sw,
+				     struct prestera_kern_fib_cache *fc)
+{
+	struct prestera_kern_neigh_cache_key nc_key;
+	struct prestera_kern_neigh_cache *n_cache;
+	struct fib_nh_common *nhc;
+	int i, nhs, err;
+
+	if (!prestera_fib_info_is_nh(&fc->info))
+		return 0;
+
+	nhs = prestera_kern_fib_info_nhs(&fc->info);
+	if (nhs > PRESTERA_NHGR_SIZE_MAX)
+		return 0;
+
+	for (i = 0; i < nhs; i++) {
+		nhc = prestera_kern_fib_info_nhc(&fc->fen4_info.info, i);
+		err = prestera_util_nhc2nc_key(sw, nhc, &nc_key);
+		if (err)
+			return 0;
+
+		n_cache = prestera_kern_neigh_cache_get(sw, &nc_key);
+		if (!n_cache)
+			return 0;
+
+		fc->kern_neigh_cache_head[i].this = fc;
+		fc->kern_neigh_cache_head[i].n_cache = n_cache;
+		list_add(&fc->kern_neigh_cache_head[i].head,
+			 &n_cache->kern_fib_cache_list);
+	}
+
+	return 0;
+}
+
 /* Operations on fi (offload, etc) must be wrapped in utils.
  * This function just create storage.
  */
 static struct prestera_kern_fib_cache *
 prestera_kern_fib_cache_create(struct prestera_switch *sw,
 			       struct prestera_kern_fib_cache_key *key,
-			       struct fib_info *fi, dscp_t dscp, u8 type)
+			       struct fib_notifier_info *info)
 {
+	struct fib_entry_notifier_info *fen_info =
+		container_of(info, struct fib_entry_notifier_info, info);
 	struct prestera_kern_fib_cache *fib_cache;
 	int err;
 
@@ -99,10 +628,8 @@ prestera_kern_fib_cache_create(struct prestera_switch *sw,
 		goto err_kzalloc;
 
 	memcpy(&fib_cache->key, key, sizeof(*key));
-	fib_info_hold(fi);
-	fib_cache->fi = fi;
-	fib_cache->kern_dscp = dscp;
-	fib_cache->kern_type = type;
+	fib_info_hold(fen_info->fi);
+	memcpy(&fib_cache->fen4_info, fen_info, sizeof(*fen_info));
 
 	err = rhashtable_insert_fast(&sw->router->kern_fib_cache_ht,
 				     &fib_cache->ht_node,
@@ -110,48 +637,270 @@ prestera_kern_fib_cache_create(struct prestera_switch *sw,
 	if (err)
 		goto err_ht_insert;
 
+	/* Handle nexthops */
+	err = __prestera_kern_fib_cache_create_nhs(sw, fib_cache);
+	if (err)
+		goto out; /* Not critical */
+
+out:
 	return fib_cache;
 
 err_ht_insert:
-	fib_info_put(fi);
+	fib_info_put(fen_info->fi);
 	kfree(fib_cache);
 err_kzalloc:
 	return NULL;
 }
 
 static void
+__prestera_k_arb_fib_nh_offload_set(struct prestera_switch *sw,
+				    struct prestera_kern_fib_cache *fibc,
+				    struct prestera_kern_neigh_cache *nc,
+				    bool offloaded, bool trap)
+{
+	struct fib_nh_common *nhc;
+	int i, nhs;
+
+	nhs = prestera_kern_fib_info_nhs(&fibc->info);
+	for (i = 0; i < nhs; i++) {
+		nhc = prestera_kern_fib_info_nhc(&fibc->info, i);
+		if (!nc) {
+			prestera_util_kern_set_nh_offload(nhc, offloaded, trap);
+			continue;
+		}
+
+		if (prestera_util_nhc_eq_n_cache_key(sw, nhc, &nc->key)) {
+			prestera_util_kern_set_nh_offload(nhc, offloaded, trap);
+			break;
+		}
+	}
+}
+
+static void
+__prestera_k_arb_n_offload_set(struct prestera_switch *sw,
+			       struct prestera_kern_neigh_cache *nc,
+			       bool offloaded)
+{
+	struct neighbour *n;
+
+	n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4,
+			 nc->key.dev);
+	if (!n)
+		return;
+
+	prestera_util_kern_set_neigh_offload(n, offloaded);
+	neigh_release(n);
+}
+
+static void
 __prestera_k_arb_fib_lpm_offload_set(struct prestera_switch *sw,
 				     struct prestera_kern_fib_cache *fc,
 				     bool fail, bool offload, bool trap)
 {
 	struct fib_rt_info fri;
 
-	if (fc->key.addr.v != PRESTERA_IPV4)
+	switch (fc->key.addr.v) {
+	case PRESTERA_IPV4:
+		fri.fi = fc->fen4_info.fi;
+		fri.tb_id = fc->key.kern_tb_id;
+		fri.dst = fc->key.addr.u.ipv4;
+		fri.dst_len = fc->key.prefix_len;
+		fri.dscp = fc->fen4_info.dscp;
+		fri.type = fc->fen4_info.type;
+		/* flags begin */
+		fri.offload = offload;
+		fri.trap = trap;
+		fri.offload_failed = fail;
+		/* flags end */
+		fib_alias_hw_flags_set(&init_net, &fri);
+		return;
+	case PRESTERA_IPV6:
+		/* TODO */
 		return;
+	}
+}
+
+static void
+__prestera_k_arb_n_lpm_set(struct prestera_switch *sw,
+			   struct prestera_kern_neigh_cache *n_cache,
+			   bool enabled)
+{
+	struct prestera_nexthop_group_key nh_grp_key;
+	struct prestera_kern_fib_cache_key fc_key;
+	struct prestera_kern_fib_cache *fib_cache;
+	struct prestera_fib_node *fib_node;
+	struct prestera_fib_key fib_key;
+
+	/* Exception for fc with prefix 32: LPM entry is already used by fib */
+	memset(&fc_key, 0, sizeof(fc_key));
+	fc_key.addr = n_cache->key.addr;
+	fc_key.prefix_len = PRESTERA_IP_ADDR_PLEN(n_cache->key.addr.v);
+	/* But better to use tb_id of route, which pointed to this neighbour. */
+	/* We take it from rif, because rif inconsistent.
+	 * Must be separated in_rif and out_rif.
+	 * Also note: for each fib pointed to this neigh should be separated
+	 *            neigh lpm entry (for each ingress vr)
+	 */
+	fc_key.kern_tb_id = l3mdev_fib_table(n_cache->key.dev);
+	fib_cache = prestera_kern_fib_cache_find(sw, &fc_key);
+	memset(&fib_key, 0, sizeof(fib_key));
+	fib_key.addr = n_cache->key.addr;
+	fib_key.prefix_len = PRESTERA_IP_ADDR_PLEN(n_cache->key.addr.v);
+	fib_key.tb_id = prestera_fix_tb_id(fc_key.kern_tb_id);
+	fib_node = prestera_fib_node_find(sw, &fib_key);
+	if (!fib_cache || !fib_cache->reachable) {
+		if (!enabled && fib_node) {
+			if (prestera_fib_node_util_is_neighbour(fib_node))
+				prestera_fib_node_destroy(sw, fib_node);
+			return;
+		}
+	}
+
+	if (enabled && !fib_node) {
+		memset(&nh_grp_key, 0, sizeof(nh_grp_key));
+		prestera_util_nc_key2nh_key(&n_cache->key,
+					    &nh_grp_key.neigh[0]);
+		fib_node = prestera_fib_node_create(sw, &fib_key,
+						    PRESTERA_FIB_TYPE_UC_NH,
+						    &nh_grp_key);
+		if (!fib_node)
+			pr_err("%s failed ip=%pI4n", "prestera_fib_node_create",
+			       &fib_key.addr.u.ipv4);
+		return;
+	}
+}
+
+static void
+__prestera_k_arb_nc_kern_fib_fetch(struct prestera_switch *sw,
+				   struct prestera_kern_neigh_cache *nc)
+{
+	if (prestera_util_kern_n_is_reachable(l3mdev_fib_table(nc->key.dev),
+					      &nc->key.addr, nc->key.dev))
+		nc->reachable = true;
+	else
+		nc->reachable = false;
+}
+
+/* Kernel neighbour -> neigh_cache info */
+static void
+__prestera_k_arb_nc_kern_n_fetch(struct prestera_switch *sw,
+				 struct prestera_kern_neigh_cache *nc)
+{
+	struct neighbour *n;
+	int err;
+
+	memset(&nc->nh_neigh_info, 0, sizeof(nc->nh_neigh_info));
+	n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4, nc->key.dev);
+	if (!n)
+		goto out;
+
+	read_lock_bh(&n->lock);
+	if (n->nud_state & NUD_VALID && !n->dead) {
+		err = prestera_neigh_iface_init(sw, &nc->nh_neigh_info.iface,
+						n);
+		if (err)
+			goto n_read_out;
 
-	fri.fi = fc->fi;
-	fri.tb_id = fc->key.kern_tb_id;
-	fri.dst = fc->key.addr.u.ipv4;
-	fri.dst_len = fc->key.prefix_len;
-	fri.dscp = fc->kern_dscp;
-	fri.type = fc->kern_type;
-	/* flags begin */
-	fri.offload = offload;
-	fri.trap = trap;
-	fri.offload_failed = fail;
-	/* flags end */
-	fib_alias_hw_flags_set(&init_net, &fri);
+		memcpy(&nc->nh_neigh_info.ha[0], &n->ha[0], ETH_ALEN);
+		nc->nh_neigh_info.connected = true;
+	}
+n_read_out:
+	read_unlock_bh(&n->lock);
+out:
+	nc->in_kernel = nc->nh_neigh_info.connected;
+	if (n)
+		neigh_release(n);
+}
+
+/* neigh_cache info -> lpm update */
+static void
+__prestera_k_arb_nc_apply(struct prestera_switch *sw,
+			  struct prestera_kern_neigh_cache *nc)
+{
+	struct prestera_kern_neigh_cache_head *nhead;
+	struct prestera_nh_neigh_key nh_key;
+	struct prestera_nh_neigh *nh_neigh;
+	int err;
+
+	__prestera_k_arb_n_lpm_set(sw, nc, nc->reachable && nc->in_kernel);
+	__prestera_k_arb_n_offload_set(sw, nc, nc->reachable && nc->in_kernel);
+
+	prestera_util_nc_key2nh_key(&nc->key, &nh_key);
+	nh_neigh = prestera_nh_neigh_find(sw, &nh_key);
+	if (!nh_neigh)
+		goto out;
+
+	/* Do hw update only if something changed to prevent nh flap */
+	if (memcmp(&nc->nh_neigh_info, &nh_neigh->info,
+		   sizeof(nh_neigh->info))) {
+		memcpy(&nh_neigh->info, &nc->nh_neigh_info,
+		       sizeof(nh_neigh->info));
+		err = prestera_nh_neigh_set(sw, nh_neigh);
+		if (err) {
+			pr_err("%s failed with err=%d ip=%pI4n mac=%pM",
+			       "prestera_nh_neigh_set", err,
+			       &nh_neigh->key.addr.u.ipv4,
+			       &nh_neigh->info.ha[0]);
+			goto out;
+		}
+	}
+
+out:
+	list_for_each_entry(nhead, &nc->kern_fib_cache_list, head) {
+		__prestera_k_arb_fib_nh_offload_set(sw, nhead->this, nc,
+						    nc->in_kernel,
+						    !nc->in_kernel);
+	}
 }
 
 static int
 __prestera_pr_k_arb_fc_lpm_info_calc(struct prestera_switch *sw,
 				     struct prestera_kern_fib_cache *fc)
 {
+	struct fib_nh_common *nhc;
+	int nh_cnt;
+
 	memset(&fc->lpm_info, 0, sizeof(fc->lpm_info));
 
-	switch (fc->fi->fib_type) {
+	switch (prestera_kern_fib_info_type(&fc->info)) {
 	case RTN_UNICAST:
-		fc->lpm_info.fib_type = PRESTERA_FIB_TYPE_TRAP;
+		if (prestera_fib_info_is_direct(&fc->info) &&
+		    fc->key.prefix_len ==
+			PRESTERA_IP_ADDR_PLEN(fc->key.addr.v)) {
+			/* This is special case.
+			 * When prefix is 32. Than we will have conflict in lpm
+			 * for direct route - once TRAP added, there is no
+			 * place for neighbour entry. So represent direct route
+			 * with prefix 32, as NH. So neighbour will be resolved
+			 * as nexthop of this route.
+			 */
+			nhc = prestera_kern_fib_info_nhc(&fc->info, 0);
+			fc->lpm_info.fib_type = PRESTERA_FIB_TYPE_UC_NH;
+			fc->lpm_info.nh_grp_key.neigh[0].addr =
+				fc->key.addr;
+			fc->lpm_info.nh_grp_key.neigh[0].rif =
+				nhc->nhc_dev;
+
+			break;
+		}
+
+		/* We can also get nh_grp_key from fi. This will be correct to
+		 * because cache not always represent, what actually written to
+		 * lpm. But we use nh cache, as well for now (for this case).
+		 */
+		for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) {
+			if (!fc->kern_neigh_cache_head[nh_cnt].n_cache)
+				break;
+
+			fc->lpm_info.nh_grp_key.neigh[nh_cnt].addr =
+				fc->kern_neigh_cache_head[nh_cnt].n_cache->key.addr;
+			fc->lpm_info.nh_grp_key.neigh[nh_cnt].rif =
+				fc->kern_neigh_cache_head[nh_cnt].n_cache->key.dev;
+		}
+
+		fc->lpm_info.fib_type = nh_cnt ?
+					PRESTERA_FIB_TYPE_UC_NH :
+					PRESTERA_FIB_TYPE_TRAP;
 		break;
 	/* Unsupported. Leave it for kernel: */
 	case RTN_BROADCAST:
@@ -191,7 +940,8 @@ static int __prestera_k_arb_f_lpm_set(struct prestera_switch *sw,
 		return 0;
 
 	fib_node = prestera_fib_node_create(sw, &fc->lpm_info.fib_key,
-					    fc->lpm_info.fib_type);
+					    fc->lpm_info.fib_type,
+					    &fc->lpm_info.nh_grp_key);
 
 	if (!fib_node) {
 		dev_err(sw->dev->dev, "fib_node=NULL %pI4n/%d kern_tb_id = %d",
@@ -220,6 +970,10 @@ static int __prestera_k_arb_fc_apply(struct prestera_switch *sw,
 	}
 
 	switch (fc->lpm_info.fib_type) {
+	case PRESTERA_FIB_TYPE_UC_NH:
+		__prestera_k_arb_fib_lpm_offload_set(sw, fc, false,
+						     fc->reachable, false);
+		break;
 	case PRESTERA_FIB_TYPE_TRAP:
 		__prestera_k_arb_fib_lpm_offload_set(sw, fc, false,
 						     false, fc->reachable);
@@ -271,17 +1025,140 @@ __prestera_k_arb_util_fib_overlapped(struct prestera_switch *sw,
 	return rfc;
 }
 
+static void __prestera_k_arb_hw_state_upd(struct prestera_switch *sw,
+					  struct prestera_kern_neigh_cache *nc)
+{
+	struct prestera_nh_neigh_key nh_key;
+	struct prestera_nh_neigh *nh_neigh;
+	struct neighbour *n;
+	bool hw_active;
+
+	prestera_util_nc_key2nh_key(&nc->key, &nh_key);
+	nh_neigh = prestera_nh_neigh_find(sw, &nh_key);
+	if (!nh_neigh) {
+		pr_err("Cannot find nh_neigh for cached %pI4n",
+		       &nc->key.addr.u.ipv4);
+		return;
+	}
+
+	hw_active = prestera_nh_neigh_util_hw_state(sw, nh_neigh);
+
+#ifdef PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH
+	if (!hw_active && nc->in_kernel)
+		goto out;
+#else /* PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH */
+	if (!hw_active)
+		goto out;
+#endif /* PRESTERA_IMPLICITY_RESOLVE_DEAD_NEIGH */
+
+	if (nc->key.addr.v == PRESTERA_IPV4) {
+		n = neigh_lookup(&arp_tbl, &nc->key.addr.u.ipv4,
+				 nc->key.dev);
+		if (!n)
+			n = neigh_create(&arp_tbl, &nc->key.addr.u.ipv4,
+					 nc->key.dev);
+	} else {
+		n = NULL;
+	}
+
+	if (!IS_ERR(n) && n) {
+		neigh_event_send(n, NULL);
+		neigh_release(n);
+	} else {
+		pr_err("Cannot create neighbour %pI4n", &nc->key.addr.u.ipv4);
+	}
+
+out:
+	return;
+}
+
+/* Propagate hw state to kernel */
+static void prestera_k_arb_hw_evt(struct prestera_switch *sw)
+{
+	struct prestera_kern_neigh_cache *n_cache;
+	struct rhashtable_iter iter;
+
+	rhashtable_walk_enter(&sw->router->kern_neigh_cache_ht, &iter);
+	rhashtable_walk_start(&iter);
+	while (1) {
+		n_cache = rhashtable_walk_next(&iter);
+
+		if (!n_cache)
+			break;
+
+		if (IS_ERR(n_cache))
+			continue;
+
+		rhashtable_walk_stop(&iter);
+		__prestera_k_arb_hw_state_upd(sw, n_cache);
+		rhashtable_walk_start(&iter);
+	}
+	rhashtable_walk_stop(&iter);
+	rhashtable_walk_exit(&iter);
+}
+
+/* Propagate kernel event to hw */
+static void prestera_k_arb_n_evt(struct prestera_switch *sw,
+				 struct neighbour *n)
+{
+	struct prestera_kern_neigh_cache_key n_key;
+	struct prestera_kern_neigh_cache *n_cache;
+	int err;
+
+	err = prestera_util_neigh2nc_key(sw, n, &n_key);
+	if (err)
+		return;
+
+	n_cache = prestera_kern_neigh_cache_find(sw, &n_key);
+	if (!n_cache) {
+		n_cache = prestera_kern_neigh_cache_get(sw, &n_key);
+		if (!n_cache)
+			return;
+		__prestera_k_arb_nc_kern_fib_fetch(sw, n_cache);
+	}
+
+	__prestera_k_arb_nc_kern_n_fetch(sw, n_cache);
+	__prestera_k_arb_nc_apply(sw, n_cache);
+
+	prestera_kern_neigh_cache_put(sw, n_cache);
+}
+
+static void __prestera_k_arb_fib_evt2nc(struct prestera_switch *sw)
+{
+	struct prestera_kern_neigh_cache *n_cache;
+	struct rhashtable_iter iter;
+
+	rhashtable_walk_enter(&sw->router->kern_neigh_cache_ht, &iter);
+	rhashtable_walk_start(&iter);
+	while (1) {
+		n_cache = rhashtable_walk_next(&iter);
+
+		if (!n_cache)
+			break;
+
+		if (IS_ERR(n_cache))
+			continue;
+
+		rhashtable_walk_stop(&iter);
+		__prestera_k_arb_nc_kern_fib_fetch(sw, n_cache);
+		__prestera_k_arb_nc_apply(sw, n_cache);
+		rhashtable_walk_start(&iter);
+	}
+	rhashtable_walk_stop(&iter);
+	rhashtable_walk_exit(&iter);
+}
+
 static int
 prestera_k_arb_fib_evt(struct prestera_switch *sw,
 		       bool replace, /* replace or del */
-		       struct fib_entry_notifier_info *fen_info)
+		       struct fib_notifier_info *info)
 {
 	struct prestera_kern_fib_cache *tfib_cache, *bfib_cache; /* top/btm */
 	struct prestera_kern_fib_cache_key fc_key;
 	struct prestera_kern_fib_cache *fib_cache;
 	int err;
 
-	prestera_util_fen_info2fib_cache_key(fen_info, &fc_key);
+	prestera_util_fen_info2fib_cache_key(info, &fc_key);
 	fib_cache = prestera_kern_fib_cache_find(sw, &fc_key);
 	if (fib_cache) {
 		fib_cache->reachable = false;
@@ -304,10 +1181,7 @@ prestera_k_arb_fib_evt(struct prestera_switch *sw,
 	}
 
 	if (replace) {
-		fib_cache = prestera_kern_fib_cache_create(sw, &fc_key,
-							   fen_info->fi,
-							   fen_info->dscp,
-							   fen_info->type);
+		fib_cache = prestera_kern_fib_cache_create(sw, &fc_key, info);
 		if (!fib_cache) {
 			dev_err(sw->dev->dev, "fib_cache == NULL");
 			return -ENOENT;
@@ -331,9 +1205,65 @@ prestera_k_arb_fib_evt(struct prestera_switch *sw,
 			dev_err(sw->dev->dev, "Applying fib_cache failed");
 	}
 
+	/* Update all neighs to resolve overlapped and apply related */
+	__prestera_k_arb_fib_evt2nc(sw);
+
 	return 0;
 }
 
+static void __prestera_k_arb_abort_neigh_ht_cb(void *ptr, void *arg)
+{
+	struct prestera_kern_neigh_cache *n_cache = ptr;
+	struct prestera_switch *sw = arg;
+
+	if (!list_empty(&n_cache->kern_fib_cache_list)) {
+		WARN_ON(1); /* BUG */
+		return;
+	}
+	__prestera_k_arb_n_offload_set(sw, n_cache, false);
+	n_cache->in_kernel = false;
+	/* No need to destroy lpm.
+	 * It will be aborted by destroy_ht
+	 */
+	__prestera_kern_neigh_cache_destruct(sw, n_cache);
+	kfree(n_cache);
+}
+
+static void __prestera_k_arb_abort_fib_ht_cb(void *ptr, void *arg)
+{
+	struct prestera_kern_fib_cache *fib_cache = ptr;
+	struct prestera_switch *sw = arg;
+
+	__prestera_k_arb_fib_lpm_offload_set(sw, fib_cache,
+					     false, false,
+					     false);
+	__prestera_k_arb_fib_nh_offload_set(sw, fib_cache, NULL,
+					    false, false);
+	/* No need to destroy lpm.
+	 * It will be aborted by destroy_ht
+	 */
+	__prestera_kern_fib_cache_destruct(sw, fib_cache);
+	kfree(fib_cache);
+}
+
+static void prestera_k_arb_abort(struct prestera_switch *sw)
+{
+	/* Function to remove all arbiter entries and related hw objects. */
+	/* Sequence:
+	 *   1) Clear arbiter tables, but don't touch hw
+	 *   2) Clear hw
+	 * We use such approach, because arbiter object is not directly mapped
+	 * to hw. So deletion of one arbiter object may even lead to creation of
+	 * hw object (e.g. in case of overlapped routes).
+	 */
+	rhashtable_free_and_destroy(&sw->router->kern_fib_cache_ht,
+				    __prestera_k_arb_abort_fib_ht_cb,
+				    sw);
+	rhashtable_free_and_destroy(&sw->router->kern_neigh_cache_ht,
+				    __prestera_k_arb_abort_neigh_ht_cb,
+				    sw);
+}
+
 static int __prestera_inetaddr_port_event(struct net_device *port_dev,
 					  unsigned long event,
 					  struct netlink_ext_ack *extack)
@@ -469,13 +1399,15 @@ static void __prestera_router_fib_event_work(struct work_struct *work)
 
 	switch (fib_work->event) {
 	case FIB_EVENT_ENTRY_REPLACE:
-		err = prestera_k_arb_fib_evt(sw, true, &fib_work->fen_info);
+		err = prestera_k_arb_fib_evt(sw, true,
+					     &fib_work->fen_info.info);
 		if (err)
 			goto err_out;
 
 		break;
 	case FIB_EVENT_ENTRY_DEL:
-		err = prestera_k_arb_fib_evt(sw, false, &fib_work->fen_info);
+		err = prestera_k_arb_fib_evt(sw, false,
+					     &fib_work->fen_info.info);
 		if (err)
 			goto err_out;
 
@@ -534,10 +1466,89 @@ static int __prestera_router_fib_event(struct notifier_block *nb,
 	return NOTIFY_DONE;
 }
 
+struct prestera_netevent_work {
+	struct work_struct work;
+	struct prestera_switch *sw;
+	struct neighbour *n;
+};
+
+static void prestera_router_neigh_event_work(struct work_struct *work)
+{
+	struct prestera_netevent_work *net_work =
+		container_of(work, struct prestera_netevent_work, work);
+	struct prestera_switch *sw = net_work->sw;
+	struct neighbour *n = net_work->n;
+
+	/* neigh - its not hw related object. It stored only in kernel. So... */
+	rtnl_lock();
+
+	prestera_k_arb_n_evt(sw, n);
+
+	neigh_release(n);
+	rtnl_unlock();
+	kfree(net_work);
+}
+
+static int prestera_router_netevent_event(struct notifier_block *nb,
+					  unsigned long event, void *ptr)
+{
+	struct prestera_netevent_work *net_work;
+	struct prestera_router *router;
+	struct neighbour *n = ptr;
+
+	router = container_of(nb, struct prestera_router, netevent_nb);
+
+	switch (event) {
+	case NETEVENT_NEIGH_UPDATE:
+		if (n->tbl->family != AF_INET)
+			return NOTIFY_DONE;
+
+		net_work = kzalloc(sizeof(*net_work), GFP_ATOMIC);
+		if (WARN_ON(!net_work))
+			return NOTIFY_BAD;
+
+		neigh_clone(n);
+		net_work->n = n;
+		net_work->sw = router->sw;
+		INIT_WORK(&net_work->work, prestera_router_neigh_event_work);
+		prestera_queue_work(&net_work->work);
+	}
+
+	return NOTIFY_DONE;
+}
+
+static void prestera_router_update_neighs_work(struct work_struct *work)
+{
+	struct prestera_router *router;
+
+	router = container_of(work, struct prestera_router,
+			      neighs_update.dw.work);
+	rtnl_lock();
+
+	prestera_k_arb_hw_evt(router->sw);
+
+	rtnl_unlock();
+	prestera_queue_delayed_work(&router->neighs_update.dw,
+				    msecs_to_jiffies(PRESTERA_NH_PROBE_INTERVAL));
+}
+
+static int prestera_neigh_work_init(struct prestera_switch *sw)
+{
+	INIT_DELAYED_WORK(&sw->router->neighs_update.dw,
+			  prestera_router_update_neighs_work);
+	prestera_queue_delayed_work(&sw->router->neighs_update.dw, 0);
+	return 0;
+}
+
+static void prestera_neigh_work_fini(struct prestera_switch *sw)
+{
+	cancel_delayed_work_sync(&sw->router->neighs_update.dw);
+}
+
 int prestera_router_init(struct prestera_switch *sw)
 {
 	struct prestera_router *router;
-	int err;
+	int err, nhgrp_cache_bytes;
 
 	router = kzalloc(sizeof(*sw->router), GFP_KERNEL);
 	if (!router)
@@ -555,6 +1566,22 @@ int prestera_router_init(struct prestera_switch *sw)
 	if (err)
 		goto err_kern_fib_cache_ht_init;
 
+	err = rhashtable_init(&router->kern_neigh_cache_ht,
+			      &__prestera_kern_neigh_cache_ht_params);
+	if (err)
+		goto err_kern_neigh_cache_ht_init;
+
+	nhgrp_cache_bytes = sw->size_tbl_router_nexthop / 8 + 1;
+	router->nhgrp_hw_state_cache = kzalloc(nhgrp_cache_bytes, GFP_KERNEL);
+	if (!router->nhgrp_hw_state_cache) {
+		err = -ENOMEM;
+		goto err_nh_state_cache_alloc;
+	}
+
+	err = prestera_neigh_work_init(sw);
+	if (err)
+		goto err_neigh_work_init;
+
 	router->inetaddr_valid_nb.notifier_call = __prestera_inetaddr_valid_cb;
 	err = register_inetaddr_validator_notifier(&router->inetaddr_valid_nb);
 	if (err)
@@ -565,6 +1592,11 @@ int prestera_router_init(struct prestera_switch *sw)
 	if (err)
 		goto err_register_inetaddr_notifier;
 
+	router->netevent_nb.notifier_call = prestera_router_netevent_event;
+	err = register_netevent_notifier(&router->netevent_nb);
+	if (err)
+		goto err_register_netevent_notifier;
+
 	router->fib_nb.notifier_call = __prestera_router_fib_event;
 	err = register_fib_notifier(&init_net, &router->fib_nb,
 				    /* TODO: flush fib entries */ NULL, NULL);
@@ -574,10 +1606,18 @@ int prestera_router_init(struct prestera_switch *sw)
 	return 0;
 
 err_register_fib_notifier:
+	unregister_netevent_notifier(&router->netevent_nb);
+err_register_netevent_notifier:
 	unregister_inetaddr_notifier(&router->inetaddr_nb);
 err_register_inetaddr_notifier:
 	unregister_inetaddr_validator_notifier(&router->inetaddr_valid_nb);
 err_register_inetaddr_validator_notifier:
+	prestera_neigh_work_fini(sw);
+err_neigh_work_init:
+	kfree(router->nhgrp_hw_state_cache);
+err_nh_state_cache_alloc:
+	rhashtable_destroy(&router->kern_neigh_cache_ht);
+err_kern_neigh_cache_ht_init:
 	rhashtable_destroy(&router->kern_fib_cache_ht);
 err_kern_fib_cache_ht_init:
 	prestera_router_hw_fini(sw);
@@ -589,8 +1629,15 @@ err_router_lib_init:
 void prestera_router_fini(struct prestera_switch *sw)
 {
 	unregister_fib_notifier(&init_net, &sw->router->fib_nb);
+	unregister_netevent_notifier(&sw->router->netevent_nb);
 	unregister_inetaddr_notifier(&sw->router->inetaddr_nb);
 	unregister_inetaddr_validator_notifier(&sw->router->inetaddr_valid_nb);
+	prestera_neigh_work_fini(sw);
+	prestera_queue_drain();
+
+	prestera_k_arb_abort(sw);
+
+	kfree(sw->router->nhgrp_hw_state_cache);
 	rhashtable_destroy(&sw->router->kern_fib_cache_ht);
 	prestera_router_hw_fini(sw);
 	kfree(sw->router);
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c
index 5b0cf3be9a9e..4f65df0ae5e8 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.c
@@ -8,10 +8,16 @@
 #include "prestera_router_hw.h"
 #include "prestera_acl.h"
 
-/*            +--+
- *   +------->|vr|<-+
- *   |        +--+  |
- *   |              |
+/*                                Nexthop is pointed
+ *                                to port (not rif)
+ *                                +-------+
+ *                              +>|nexthop|
+ *                              | +-------+
+ *                              |
+ *            +--+        +-----++
+ *   +------->|vr|<-+   +>|nh_grp|
+ *   |        +--+  |   | +------+
+ *   |              |   |
  * +-+-------+   +--+---+-+
  * |rif_entry|   |fib_node|
  * +---------+   +--------+
@@ -23,6 +29,8 @@
 
 #define PRESTERA_NHGR_UNUSED (0)
 #define PRESTERA_NHGR_DROP (0xFFFFFFFF)
+/* Need to merge it with router_manager */
+#define PRESTERA_NH_ACTIVE_JIFFER_FILTER 3000 /* ms */
 
 static const struct rhashtable_params __prestera_fib_ht_params = {
 	.key_offset  = offsetof(struct prestera_fib_node, key),
@@ -31,10 +39,45 @@ static const struct rhashtable_params __prestera_fib_ht_params = {
 	.automatic_shrinking = true,
 };
 
+static const struct rhashtable_params __prestera_nh_neigh_ht_params = {
+	.key_offset  = offsetof(struct prestera_nh_neigh, key),
+	.key_len     = sizeof(struct prestera_nh_neigh_key),
+	.head_offset = offsetof(struct prestera_nh_neigh, ht_node),
+};
+
+static const struct rhashtable_params __prestera_nexthop_group_ht_params = {
+	.key_offset  = offsetof(struct prestera_nexthop_group, key),
+	.key_len     = sizeof(struct prestera_nexthop_group_key),
+	.head_offset = offsetof(struct prestera_nexthop_group, ht_node),
+};
+
+static int prestera_nexthop_group_set(struct prestera_switch *sw,
+				      struct prestera_nexthop_group *nh_grp);
+static bool
+prestera_nexthop_group_util_hw_state(struct prestera_switch *sw,
+				     struct prestera_nexthop_group *nh_grp);
+static void prestera_fib_node_destroy_ht_cb(void *ptr, void *arg);
+
+/* TODO: move to router.h as macros */
+static bool prestera_nh_neigh_key_is_valid(struct prestera_nh_neigh_key *key)
+{
+	return memchr_inv(key, 0, sizeof(*key)) ? true : false;
+}
+
 int prestera_router_hw_init(struct prestera_switch *sw)
 {
 	int err;
 
+	err = rhashtable_init(&sw->router->nh_neigh_ht,
+			      &__prestera_nh_neigh_ht_params);
+	if (err)
+		goto err_nh_neigh_ht_init;
+
+	err = rhashtable_init(&sw->router->nexthop_group_ht,
+			      &__prestera_nexthop_group_ht_params);
+	if (err)
+		goto err_nexthop_grp_ht_init;
+
 	err = rhashtable_init(&sw->router->fib_ht,
 			      &__prestera_fib_ht_params);
 	if (err)
@@ -43,15 +86,25 @@ int prestera_router_hw_init(struct prestera_switch *sw)
 	INIT_LIST_HEAD(&sw->router->vr_list);
 	INIT_LIST_HEAD(&sw->router->rif_entry_list);
 
+	return 0;
+
 err_fib_ht_init:
+	rhashtable_destroy(&sw->router->nexthop_group_ht);
+err_nexthop_grp_ht_init:
+	rhashtable_destroy(&sw->router->nh_neigh_ht);
+err_nh_neigh_ht_init:
 	return 0;
 }
 
 void prestera_router_hw_fini(struct prestera_switch *sw)
 {
+	rhashtable_free_and_destroy(&sw->router->fib_ht,
+				    prestera_fib_node_destroy_ht_cb, sw);
 	WARN_ON(!list_empty(&sw->router->vr_list));
 	WARN_ON(!list_empty(&sw->router->rif_entry_list));
 	rhashtable_destroy(&sw->router->fib_ht);
+	rhashtable_destroy(&sw->router->nexthop_group_ht);
+	rhashtable_destroy(&sw->router->nh_neigh_ht);
 }
 
 static struct prestera_vr *__prestera_vr_find(struct prestera_switch *sw,
@@ -232,6 +285,286 @@ err_kzalloc:
 	return NULL;
 }
 
+static void __prestera_nh_neigh_destroy(struct prestera_switch *sw,
+					struct prestera_nh_neigh *neigh)
+{
+	rhashtable_remove_fast(&sw->router->nh_neigh_ht,
+			       &neigh->ht_node,
+			       __prestera_nh_neigh_ht_params);
+	kfree(neigh);
+}
+
+static struct prestera_nh_neigh *
+__prestera_nh_neigh_create(struct prestera_switch *sw,
+			   struct prestera_nh_neigh_key *key)
+{
+	struct prestera_nh_neigh *neigh;
+	int err;
+
+	neigh = kzalloc(sizeof(*neigh), GFP_KERNEL);
+	if (!neigh)
+		goto err_kzalloc;
+
+	memcpy(&neigh->key, key, sizeof(*key));
+	neigh->info.connected = false;
+	INIT_LIST_HEAD(&neigh->nexthop_group_list);
+	err = rhashtable_insert_fast(&sw->router->nh_neigh_ht,
+				     &neigh->ht_node,
+				     __prestera_nh_neigh_ht_params);
+	if (err)
+		goto err_rhashtable_insert;
+
+	return neigh;
+
+err_rhashtable_insert:
+	kfree(neigh);
+err_kzalloc:
+	return NULL;
+}
+
+struct prestera_nh_neigh *
+prestera_nh_neigh_find(struct prestera_switch *sw,
+		       struct prestera_nh_neigh_key *key)
+{
+	struct prestera_nh_neigh *nh_neigh;
+
+	nh_neigh = rhashtable_lookup_fast(&sw->router->nh_neigh_ht,
+					  key, __prestera_nh_neigh_ht_params);
+	return IS_ERR(nh_neigh) ? NULL : nh_neigh;
+}
+
+struct prestera_nh_neigh *
+prestera_nh_neigh_get(struct prestera_switch *sw,
+		      struct prestera_nh_neigh_key *key)
+{
+	struct prestera_nh_neigh *neigh;
+
+	neigh = prestera_nh_neigh_find(sw, key);
+	if (!neigh)
+		return __prestera_nh_neigh_create(sw, key);
+
+	return neigh;
+}
+
+void prestera_nh_neigh_put(struct prestera_switch *sw,
+			   struct prestera_nh_neigh *neigh)
+{
+	if (list_empty(&neigh->nexthop_group_list))
+		__prestera_nh_neigh_destroy(sw, neigh);
+}
+
+/* Updates new prestera_neigh_info */
+int prestera_nh_neigh_set(struct prestera_switch *sw,
+			  struct prestera_nh_neigh *neigh)
+{
+	struct prestera_nh_neigh_head *nh_head;
+	struct prestera_nexthop_group *nh_grp;
+	int err;
+
+	list_for_each_entry(nh_head, &neigh->nexthop_group_list, head) {
+		nh_grp = nh_head->this;
+		err = prestera_nexthop_group_set(sw, nh_grp);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+bool prestera_nh_neigh_util_hw_state(struct prestera_switch *sw,
+				     struct prestera_nh_neigh *nh_neigh)
+{
+	bool state;
+	struct prestera_nh_neigh_head *nh_head, *tmp;
+
+	state = false;
+	list_for_each_entry_safe(nh_head, tmp,
+				 &nh_neigh->nexthop_group_list, head) {
+		state = prestera_nexthop_group_util_hw_state(sw, nh_head->this);
+		if (state)
+			goto out;
+	}
+
+out:
+	return state;
+}
+
+static struct prestera_nexthop_group *
+__prestera_nexthop_group_create(struct prestera_switch *sw,
+				struct prestera_nexthop_group_key *key)
+{
+	struct prestera_nexthop_group *nh_grp;
+	struct prestera_nh_neigh *nh_neigh;
+	int nh_cnt, err, gid;
+
+	nh_grp = kzalloc(sizeof(*nh_grp), GFP_KERNEL);
+	if (!nh_grp)
+		goto err_kzalloc;
+
+	memcpy(&nh_grp->key, key, sizeof(*key));
+	for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) {
+		if (!prestera_nh_neigh_key_is_valid(&nh_grp->key.neigh[nh_cnt]))
+			break;
+
+		nh_neigh = prestera_nh_neigh_get(sw,
+						 &nh_grp->key.neigh[nh_cnt]);
+		if (!nh_neigh)
+			goto err_nh_neigh_get;
+
+		nh_grp->nh_neigh_head[nh_cnt].neigh = nh_neigh;
+		nh_grp->nh_neigh_head[nh_cnt].this = nh_grp;
+		list_add(&nh_grp->nh_neigh_head[nh_cnt].head,
+			 &nh_neigh->nexthop_group_list);
+	}
+
+	err = prestera_hw_nh_group_create(sw, nh_cnt, &nh_grp->grp_id);
+	if (err)
+		goto err_nh_group_create;
+
+	err = prestera_nexthop_group_set(sw, nh_grp);
+	if (err)
+		goto err_nexthop_group_set;
+
+	err = rhashtable_insert_fast(&sw->router->nexthop_group_ht,
+				     &nh_grp->ht_node,
+				     __prestera_nexthop_group_ht_params);
+	if (err)
+		goto err_ht_insert;
+
+	/* reset cache for created group */
+	gid = nh_grp->grp_id;
+	sw->router->nhgrp_hw_state_cache[gid / 8] &= ~BIT(gid % 8);
+
+	return nh_grp;
+
+err_ht_insert:
+err_nexthop_group_set:
+	prestera_hw_nh_group_delete(sw, nh_cnt, nh_grp->grp_id);
+err_nh_group_create:
+err_nh_neigh_get:
+	for (nh_cnt--; nh_cnt >= 0; nh_cnt--) {
+		list_del(&nh_grp->nh_neigh_head[nh_cnt].head);
+		prestera_nh_neigh_put(sw, nh_grp->nh_neigh_head[nh_cnt].neigh);
+	}
+
+	kfree(nh_grp);
+err_kzalloc:
+	return NULL;
+}
+
+static void
+__prestera_nexthop_group_destroy(struct prestera_switch *sw,
+				 struct prestera_nexthop_group *nh_grp)
+{
+	struct prestera_nh_neigh *nh_neigh;
+	int nh_cnt;
+
+	rhashtable_remove_fast(&sw->router->nexthop_group_ht,
+			       &nh_grp->ht_node,
+			       __prestera_nexthop_group_ht_params);
+
+	for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) {
+		nh_neigh = nh_grp->nh_neigh_head[nh_cnt].neigh;
+		if (!nh_neigh)
+			break;
+
+		list_del(&nh_grp->nh_neigh_head[nh_cnt].head);
+		prestera_nh_neigh_put(sw, nh_neigh);
+	}
+
+	prestera_hw_nh_group_delete(sw, nh_cnt, nh_grp->grp_id);
+	kfree(nh_grp);
+}
+
+static struct prestera_nexthop_group *
+__prestera_nexthop_group_find(struct prestera_switch *sw,
+			      struct prestera_nexthop_group_key *key)
+{
+	struct prestera_nexthop_group *nh_grp;
+
+	nh_grp = rhashtable_lookup_fast(&sw->router->nexthop_group_ht,
+					key, __prestera_nexthop_group_ht_params);
+	return IS_ERR(nh_grp) ? NULL : nh_grp;
+}
+
+static struct prestera_nexthop_group *
+prestera_nexthop_group_get(struct prestera_switch *sw,
+			   struct prestera_nexthop_group_key *key)
+{
+	struct prestera_nexthop_group *nh_grp;
+
+	nh_grp = __prestera_nexthop_group_find(sw, key);
+	if (nh_grp) {
+		refcount_inc(&nh_grp->refcount);
+	} else {
+		nh_grp = __prestera_nexthop_group_create(sw, key);
+		if (IS_ERR(nh_grp))
+			return ERR_CAST(nh_grp);
+
+		refcount_set(&nh_grp->refcount, 1);
+	}
+
+	return nh_grp;
+}
+
+static void prestera_nexthop_group_put(struct prestera_switch *sw,
+				       struct prestera_nexthop_group *nh_grp)
+{
+	if (refcount_dec_and_test(&nh_grp->refcount))
+		__prestera_nexthop_group_destroy(sw, nh_grp);
+}
+
+/* Updates with new nh_neigh's info */
+static int prestera_nexthop_group_set(struct prestera_switch *sw,
+				      struct prestera_nexthop_group *nh_grp)
+{
+	struct prestera_neigh_info info[PRESTERA_NHGR_SIZE_MAX];
+	struct prestera_nh_neigh *neigh;
+	int nh_cnt;
+
+	memset(&info[0], 0, sizeof(info));
+	for (nh_cnt = 0; nh_cnt < PRESTERA_NHGR_SIZE_MAX; nh_cnt++) {
+		neigh = nh_grp->nh_neigh_head[nh_cnt].neigh;
+		if (!neigh)
+			break;
+
+		memcpy(&info[nh_cnt], &neigh->info, sizeof(neigh->info));
+	}
+
+	return prestera_hw_nh_entries_set(sw, nh_cnt, &info[0], nh_grp->grp_id);
+}
+
+static bool
+prestera_nexthop_group_util_hw_state(struct prestera_switch *sw,
+				     struct prestera_nexthop_group *nh_grp)
+{
+	int err;
+	u32 buf_size = sw->size_tbl_router_nexthop / 8 + 1;
+	u32 gid = nh_grp->grp_id;
+	u8 *cache = sw->router->nhgrp_hw_state_cache;
+
+	/* Antijitter
+	 * Prevent situation, when we read state of nh_grp twice in short time,
+	 * and state bit is still cleared on second call. So just stuck active
+	 * state for PRESTERA_NH_ACTIVE_JIFFER_FILTER, after last occurred.
+	 */
+	if (!time_before(jiffies, sw->router->nhgrp_hw_cache_kick +
+			msecs_to_jiffies(PRESTERA_NH_ACTIVE_JIFFER_FILTER))) {
+		err = prestera_hw_nhgrp_blk_get(sw, cache, buf_size);
+		if (err) {
+			pr_err("Failed to get hw state nh_grp's");
+			return false;
+		}
+
+		sw->router->nhgrp_hw_cache_kick = jiffies;
+	}
+
+	if (cache[gid / 8] & BIT(gid % 8))
+		return true;
+
+	return false;
+}
+
 struct prestera_fib_node *
 prestera_fib_node_find(struct prestera_switch *sw, struct prestera_fib_key *key)
 {
@@ -251,6 +584,9 @@ static void __prestera_fib_node_destruct(struct prestera_switch *sw,
 	prestera_hw_lpm_del(sw, vr->hw_vr_id, fib_node->key.addr.u.ipv4,
 			    fib_node->key.prefix_len);
 	switch (fib_node->info.type) {
+	case PRESTERA_FIB_TYPE_UC_NH:
+		prestera_nexthop_group_put(sw, fib_node->info.nh_grp);
+		break;
 	case PRESTERA_FIB_TYPE_TRAP:
 		break;
 	case PRESTERA_FIB_TYPE_DROP:
@@ -272,10 +608,20 @@ void prestera_fib_node_destroy(struct prestera_switch *sw,
 	kfree(fib_node);
 }
 
+static void prestera_fib_node_destroy_ht_cb(void *ptr, void *arg)
+{
+	struct prestera_fib_node *node = ptr;
+	struct prestera_switch *sw = arg;
+
+	__prestera_fib_node_destruct(sw, node);
+	kfree(node);
+}
+
 struct prestera_fib_node *
 prestera_fib_node_create(struct prestera_switch *sw,
 			 struct prestera_fib_key *key,
-			 enum prestera_fib_type fib_type)
+			 enum prestera_fib_type fib_type,
+			 struct prestera_nexthop_group_key *nh_grp_key)
 {
 	struct prestera_fib_node *fib_node;
 	u32 grp_id;
@@ -302,6 +648,14 @@ prestera_fib_node_create(struct prestera_switch *sw,
 	case PRESTERA_FIB_TYPE_DROP:
 		grp_id = PRESTERA_NHGR_DROP;
 		break;
+	case PRESTERA_FIB_TYPE_UC_NH:
+		fib_node->info.nh_grp = prestera_nexthop_group_get(sw,
+								   nh_grp_key);
+		if (!fib_node->info.nh_grp)
+			goto err_nh_grp_get;
+
+		grp_id = fib_node->info.nh_grp->grp_id;
+		break;
 	default:
 		pr_err("Unsupported fib_type %d", fib_type);
 		goto err_nh_grp_get;
@@ -323,6 +677,8 @@ err_ht_insert:
 	prestera_hw_lpm_del(sw, vr->hw_vr_id, key->addr.u.ipv4,
 			    key->prefix_len);
 err_lpm_add:
+	if (fib_type == PRESTERA_FIB_TYPE_UC_NH)
+		prestera_nexthop_group_put(sw, fib_node->info.nh_grp);
 err_nh_grp_get:
 	prestera_vr_put(sw, vr);
 err_vr_get:
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h
index 67dbb49c8bd4..9ca97919c863 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h
+++ b/drivers/net/ethernet/marvell/prestera/prestera_router_hw.h
@@ -31,6 +31,63 @@ struct prestera_ip_addr {
 		PRESTERA_IPV4 = 0,
 		PRESTERA_IPV6
 	} v;
+#define PRESTERA_IP_ADDR_PLEN(V) ((V) == PRESTERA_IPV4 ? 32 : \
+				  /* (V) == PRESTERA_IPV6 ? */ 128 /* : 0 */)
+};
+
+struct prestera_nh_neigh_key {
+	struct prestera_ip_addr addr;
+	/* Seems like rif is obsolete, because there is iface in info ?
+	 * Key can contain functional fields, or fields, which is used to
+	 * filter duplicate objects on logical level (before you pass it to
+	 * HW)... also key can be used to cover hardware restrictions.
+	 * In our case rif - is logical interface (even can be VLAN), which
+	 * is used in combination with IP address (which is also not related to
+	 * hardware nexthop) to provide logical compression of created nexthops.
+	 * You even can imagine, that rif+IPaddr is just cookie.
+	 */
+	/* struct prestera_rif *rif; */
+	/* Use just as cookie, to divide ARP domains (in order with addr) */
+	void *rif;
+};
+
+/* Used for hw call */
+struct prestera_neigh_info {
+	struct prestera_iface iface;
+	unsigned char ha[ETH_ALEN];
+	u8 connected; /* bool. indicate, if mac/oif valid */
+	u8 __pad[1];
+};
+
+/* Used to notify nh about neigh change */
+struct prestera_nh_neigh {
+	struct prestera_nh_neigh_key key;
+	struct prestera_neigh_info info;
+	struct rhash_head ht_node; /* node of prestera_vr */
+	struct list_head nexthop_group_list;
+};
+
+#define PRESTERA_NHGR_SIZE_MAX 4
+
+struct prestera_nexthop_group {
+	struct prestera_nexthop_group_key {
+		struct prestera_nh_neigh_key neigh[PRESTERA_NHGR_SIZE_MAX];
+	} key;
+	/* Store intermediate object here.
+	 * This prevent overhead kzalloc call.
+	 */
+	/* nh_neigh is used only to notify nexthop_group */
+	struct prestera_nh_neigh_head {
+		struct prestera_nexthop_group *this;
+		struct list_head head;
+		/* ptr to neigh is not necessary.
+		 * It used to prevent lookup of nh_neigh by key (n) on destroy
+		 */
+		struct prestera_nh_neigh *neigh;
+	} nh_neigh_head[PRESTERA_NHGR_SIZE_MAX];
+	struct rhash_head ht_node; /* node of prestera_vr */
+	refcount_t refcount;
+	u32 grp_id; /* hw */
 };
 
 struct prestera_fib_key {
@@ -44,12 +101,16 @@ struct prestera_fib_info {
 	struct list_head vr_node;
 	enum prestera_fib_type {
 		PRESTERA_FIB_TYPE_INVALID = 0,
+		/* must be pointer to nh_grp id */
+		PRESTERA_FIB_TYPE_UC_NH,
 		/* It can be connected route
 		 * and will be overlapped with neighbours
 		 */
 		PRESTERA_FIB_TYPE_TRAP,
 		PRESTERA_FIB_TYPE_DROP
 	} type;
+	/* Valid only if type = UC_NH*/
+	struct prestera_nexthop_group *nh_grp;
 };
 
 struct prestera_fib_node {
@@ -67,6 +128,18 @@ struct prestera_rif_entry *
 prestera_rif_entry_create(struct prestera_switch *sw,
 			  struct prestera_rif_entry_key *k,
 			  u32 tb_id, const unsigned char *addr);
+struct prestera_nh_neigh *
+prestera_nh_neigh_find(struct prestera_switch *sw,
+		       struct prestera_nh_neigh_key *key);
+struct prestera_nh_neigh *
+prestera_nh_neigh_get(struct prestera_switch *sw,
+		      struct prestera_nh_neigh_key *key);
+void prestera_nh_neigh_put(struct prestera_switch *sw,
+			   struct prestera_nh_neigh *neigh);
+int prestera_nh_neigh_set(struct prestera_switch *sw,
+			  struct prestera_nh_neigh *neigh);
+bool prestera_nh_neigh_util_hw_state(struct prestera_switch *sw,
+				     struct prestera_nh_neigh *nh_neigh);
 struct prestera_fib_node *prestera_fib_node_find(struct prestera_switch *sw,
 						 struct prestera_fib_key *key);
 void prestera_fib_node_destroy(struct prestera_switch *sw,
@@ -74,7 +147,8 @@ void prestera_fib_node_destroy(struct prestera_switch *sw,
 struct prestera_fib_node *
 prestera_fib_node_create(struct prestera_switch *sw,
 			 struct prestera_fib_key *key,
-			 enum prestera_fib_type fib_type);
+			 enum prestera_fib_type fib_type,
+			 struct prestera_nexthop_group_key *nh_grp_key);
 int prestera_router_hw_init(struct prestera_switch *sw);
 void prestera_router_hw_fini(struct prestera_switch *sw);
 
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c
index dc3e3ddc60bf..42ee963e9f75 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_rxtx.c
@@ -659,7 +659,7 @@ static int prestera_sdma_switch_init(struct prestera_switch *sw)
 
 	init_dummy_netdev(&sdma->napi_dev);
 
-	netif_napi_add(&sdma->napi_dev, &sdma->rx_napi, prestera_sdma_rx_poll, 64);
+	netif_napi_add(&sdma->napi_dev, &sdma->rx_napi, prestera_sdma_rx_poll);
 	napi_enable(&sdma->rx_napi);
 
 	return 0;
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.c b/drivers/net/ethernet/marvell/prestera/prestera_span.c
index 845e9d8c8cc7..f0e9d6ea88c5 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_span.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_span.c
@@ -120,8 +120,9 @@ static int prestera_span_put(struct prestera_switch *sw, u8 span_id)
 	return 0;
 }
 
-static int prestera_span_rule_add(struct prestera_flow_block_binding *binding,
-				  struct prestera_port *to_port)
+int prestera_span_rule_add(struct prestera_flow_block_binding *binding,
+			   struct prestera_port *to_port,
+			   bool ingress)
 {
 	struct prestera_switch *sw = binding->port->sw;
 	u8 span_id;
@@ -135,7 +136,7 @@ static int prestera_span_rule_add(struct prestera_flow_block_binding *binding,
 	if (err)
 		return err;
 
-	err = prestera_hw_span_bind(binding->port, span_id);
+	err = prestera_hw_span_bind(binding->port, span_id, ingress);
 	if (err) {
 		prestera_span_put(sw, span_id);
 		return err;
@@ -145,11 +146,12 @@ static int prestera_span_rule_add(struct prestera_flow_block_binding *binding,
 	return 0;
 }
 
-static int prestera_span_rule_del(struct prestera_flow_block_binding *binding)
+int prestera_span_rule_del(struct prestera_flow_block_binding *binding,
+			   bool ingress)
 {
 	int err;
 
-	err = prestera_hw_span_unbind(binding->port);
+	err = prestera_hw_span_unbind(binding->port, ingress);
 	if (err)
 		return err;
 
@@ -161,60 +163,6 @@ static int prestera_span_rule_del(struct prestera_flow_block_binding *binding)
 	return 0;
 }
 
-int prestera_span_replace(struct prestera_flow_block *block,
-			  struct tc_cls_matchall_offload *f)
-{
-	struct prestera_flow_block_binding *binding;
-	__be16 protocol = f->common.protocol;
-	struct flow_action_entry *act;
-	struct prestera_port *port;
-	int err;
-
-	if (!flow_offload_has_one_action(&f->rule->action)) {
-		NL_SET_ERR_MSG(f->common.extack,
-			       "Only singular actions are supported");
-		return -EOPNOTSUPP;
-	}
-
-	act = &f->rule->action.entries[0];
-
-	if (!prestera_netdev_check(act->dev)) {
-		NL_SET_ERR_MSG(f->common.extack,
-			       "Only Marvell Prestera port is supported");
-		return -EINVAL;
-	}
-	if (!tc_cls_can_offload_and_chain0(act->dev, &f->common))
-		return -EOPNOTSUPP;
-	if (act->id != FLOW_ACTION_MIRRED)
-		return -EOPNOTSUPP;
-	if (protocol != htons(ETH_P_ALL))
-		return -EOPNOTSUPP;
-
-	port = netdev_priv(act->dev);
-
-	list_for_each_entry(binding, &block->binding_list, list) {
-		err = prestera_span_rule_add(binding, port);
-		if (err)
-			goto rollback;
-	}
-
-	return 0;
-
-rollback:
-	list_for_each_entry_continue_reverse(binding,
-					     &block->binding_list, list)
-		prestera_span_rule_del(binding);
-	return err;
-}
-
-void prestera_span_destroy(struct prestera_flow_block *block)
-{
-	struct prestera_flow_block_binding *binding;
-
-	list_for_each_entry(binding, &block->binding_list, list)
-		prestera_span_rule_del(binding);
-}
-
 int prestera_span_init(struct prestera_switch *sw)
 {
 	struct prestera_span *span;
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.h b/drivers/net/ethernet/marvell/prestera/prestera_span.h
index f0644521f78a..493b68524bcb 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_span.h
+++ b/drivers/net/ethernet/marvell/prestera/prestera_span.h
@@ -8,13 +8,17 @@
 
 #define PRESTERA_SPAN_INVALID_ID -1
 
+struct prestera_port;
 struct prestera_switch;
-struct prestera_flow_block;
+struct prestera_flow_block_binding;
 
 int prestera_span_init(struct prestera_switch *sw);
 void prestera_span_fini(struct prestera_switch *sw);
-int prestera_span_replace(struct prestera_flow_block *block,
-			  struct tc_cls_matchall_offload *f);
-void prestera_span_destroy(struct prestera_flow_block *block);
+
+int prestera_span_rule_add(struct prestera_flow_block_binding *binding,
+			   struct prestera_port *to_port,
+			   bool ingress);
+int prestera_span_rule_del(struct prestera_flow_block_binding *binding,
+			   bool ingress);
 
 #endif /* _PRESTERA_SPAN_H_ */
diff --git a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c
index 71cde97d85c8..e548cd32582e 100644
--- a/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c
+++ b/drivers/net/ethernet/marvell/prestera/prestera_switchdev.c
@@ -143,6 +143,7 @@ prestera_br_port_flags_reset(struct prestera_bridge_port *br_port,
 	prestera_port_uc_flood_set(port, false);
 	prestera_port_mc_flood_set(port, false);
 	prestera_port_learning_set(port, false);
+	prestera_port_br_locked_set(port, false);
 }
 
 static int prestera_br_port_flags_set(struct prestera_bridge_port *br_port,
@@ -162,6 +163,11 @@ static int prestera_br_port_flags_set(struct prestera_bridge_port *br_port,
 	if (err)
 		goto err_out;
 
+	err = prestera_port_br_locked_set(port,
+					  br_port->flags & BR_PORT_LOCKED);
+	if (err)
+		goto err_out;
+
 	return 0;
 
 err_out:
@@ -1163,7 +1169,7 @@ static int prestera_port_obj_attr_set(struct net_device *dev, const void *ctx,
 		break;
 	case SWITCHDEV_ATTR_ID_PORT_PRE_BRIDGE_FLAGS:
 		if (attr->u.brport_flags.mask &
-		    ~(BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD))
+		    ~(BR_LEARNING | BR_FLOOD | BR_MCAST_FLOOD | BR_PORT_LOCKED))
 			err = -EINVAL;
 		break;
 	case SWITCHDEV_ATTR_ID_PORT_BRIDGE_FLAGS:
diff --git a/drivers/net/ethernet/marvell/pxa168_eth.c b/drivers/net/ethernet/marvell/pxa168_eth.c
index 349b8a94e939..cf456d62677f 100644
--- a/drivers/net/ethernet/marvell/pxa168_eth.c
+++ b/drivers/net/ethernet/marvell/pxa168_eth.c
@@ -1354,10 +1354,10 @@ static void pxa168_eth_netpoll(struct net_device *dev)
 static void pxa168_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRIVER_VERSION, sizeof(info->version));
-	strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
-	strlcpy(info->bus_info, "N/A", sizeof(info->bus_info));
+	strscpy(info->driver, DRIVER_NAME, sizeof(info->driver));
+	strscpy(info->version, DRIVER_VERSION, sizeof(info->version));
+	strscpy(info->fw_version, "N/A", sizeof(info->fw_version));
+	strscpy(info->bus_info, "N/A", sizeof(info->bus_info));
 }
 
 static const struct ethtool_ops pxa168_ethtool_ops = {
diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c
index c1e985416c0e..1b43704baceb 100644
--- a/drivers/net/ethernet/marvell/skge.c
+++ b/drivers/net/ethernet/marvell/skge.c
@@ -394,9 +394,9 @@ static void skge_get_drvinfo(struct net_device *dev,
 {
 	struct skge_port *skge = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(skge->hw->pdev),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(skge->hw->pdev),
 		sizeof(info->bus_info));
 }
 
@@ -3832,7 +3832,7 @@ static struct net_device *skge_devinit(struct skge_hw *hw, int port,
 		dev->features |= NETIF_F_HIGHDMA;
 
 	skge = netdev_priv(dev);
-	netif_napi_add(dev, &skge->napi, skge_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &skge->napi, skge_poll);
 	skge->netdev = dev;
 	skge->hw = hw;
 	skge->msg_enable = netif_msg_init(debug, default_msg);
diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c
index bbea5458000b..ab33ba1c3023 100644
--- a/drivers/net/ethernet/marvell/sky2.c
+++ b/drivers/net/ethernet/marvell/sky2.c
@@ -3687,9 +3687,9 @@ static void sky2_get_drvinfo(struct net_device *dev,
 {
 	struct sky2_port *sky2 = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(sky2->hw->pdev),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(sky2->hw->pdev),
 		sizeof(info->bus_info));
 }
 
@@ -4937,7 +4937,7 @@ static int sky2_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		}
 	}
 
-	netif_napi_add(dev, &hw->napi, sky2_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &hw->napi, sky2_poll);
 
 	err = register_netdev(dev);
 	if (err) {
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
index b344632beadd..4fba7cb0144b 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
@@ -73,6 +73,12 @@ static const struct mtk_reg_map mtk_reg_map = {
 		.fq_blen	= 0x1b2c,
 	},
 	.gdm1_cnt		= 0x2400,
+	.gdma_to_ppe		= 0x4444,
+	.ppe_base		= 0x0c00,
+	.wdma_base = {
+		[0]		= 0x2800,
+		[1]		= 0x2c00,
+	},
 };
 
 static const struct mtk_reg_map mt7628_reg_map = {
@@ -126,6 +132,12 @@ static const struct mtk_reg_map mt7986_reg_map = {
 		.fq_blen	= 0x472c,
 	},
 	.gdm1_cnt		= 0x1c00,
+	.gdma_to_ppe		= 0x3333,
+	.ppe_base		= 0x2000,
+	.wdma_base = {
+		[0]		= 0x4800,
+		[1]		= 0x4c00,
+	},
 };
 
 /* strings used by ethtool */
@@ -1573,8 +1585,8 @@ static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf,
 		.last	= !xdp_frame_has_frags(xdpf),
 	};
 	int err, index = 0, n_desc = 1, nr_frags;
-	struct mtk_tx_dma *htxd, *txd, *txd_pdma;
 	struct mtk_tx_buf *htx_buf, *tx_buf;
+	struct mtk_tx_dma *htxd, *txd;
 	void *data = xdpf->data;
 
 	if (unlikely(test_bit(MTK_RESETTING, &eth->state)))
@@ -1608,7 +1620,6 @@ static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf,
 
 		if (MTK_HAS_CAPS(soc->caps, MTK_QDMA) || (index & 0x1)) {
 			txd = mtk_qdma_phys_to_virt(ring, txd->txd2);
-			txd_pdma = qdma_to_pdma(ring, txd);
 			if (txd == ring->last_free)
 				goto unmap;
 
@@ -1629,7 +1640,8 @@ static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf,
 	htx_buf->data = xdpf;
 
 	if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
-		txd_pdma = qdma_to_pdma(ring, txd);
+		struct mtk_tx_dma *txd_pdma = qdma_to_pdma(ring, txd);
+
 		if (index & 1)
 			txd_pdma->txd2 |= TX_DMA_LS0;
 		else
@@ -1660,13 +1672,15 @@ static int mtk_xdp_submit_frame(struct mtk_eth *eth, struct xdp_frame *xdpf,
 
 unmap:
 	while (htxd != txd) {
-		txd_pdma = qdma_to_pdma(ring, htxd);
 		tx_buf = mtk_desc_to_tx_buf(ring, htxd, soc->txrx.txd_size);
 		mtk_tx_unmap(eth, tx_buf, NULL, false);
 
 		htxd->txd3 = TX_DMA_LS0 | TX_DMA_OWNER_CPU;
-		if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA))
+		if (!MTK_HAS_CAPS(soc->caps, MTK_QDMA)) {
+			struct mtk_tx_dma *txd_pdma = qdma_to_pdma(ring, htxd);
+
 			txd_pdma->txd2 = TX_DMA_DESP2_DEF;
+		}
 
 		htxd = mtk_qdma_phys_to_virt(ring, htxd->txd2);
 	}
@@ -1892,12 +1906,14 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
 		bytes += skb->len;
 
 		if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
+			reason = FIELD_GET(MTK_RXD5_PPE_CPU_REASON, trxd.rxd5);
 			hash = trxd.rxd5 & MTK_RXD5_FOE_ENTRY;
 			if (hash != MTK_RXD5_FOE_ENTRY)
 				skb_set_hash(skb, jhash_1word(hash, 0),
 					     PKT_HASH_TYPE_L4);
 			rxdcsum = &trxd.rxd3;
 		} else {
+			reason = FIELD_GET(MTK_RXD4_PPE_CPU_REASON, trxd.rxd4);
 			hash = trxd.rxd4 & MTK_RXD4_FOE_ENTRY;
 			if (hash != MTK_RXD4_FOE_ENTRY)
 				skb_set_hash(skb, jhash_1word(hash, 0),
@@ -1911,9 +1927,8 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
 			skb_checksum_none_assert(skb);
 		skb->protocol = eth_type_trans(skb, netdev);
 
-		reason = FIELD_GET(MTK_RXD4_PPE_CPU_REASON, trxd.rxd4);
 		if (reason == MTK_PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED)
-			mtk_ppe_check_skb(eth->ppe, skb, hash);
+			mtk_ppe_check_skb(eth->ppe[0], skb, hash);
 
 		if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
 			if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
@@ -2976,21 +2991,25 @@ static int mtk_open(struct net_device *dev)
 
 	/* we run 2 netdevs on the same dma ring so we only bring it up once */
 	if (!refcount_read(&eth->dma_refcnt)) {
-		u32 gdm_config = MTK_GDMA_TO_PDMA;
+		const struct mtk_soc_data *soc = eth->soc;
+		u32 gdm_config;
+		int i;
 
 		err = mtk_start_dma(eth);
 		if (err)
 			return err;
 
-		if (eth->soc->offload_version && mtk_ppe_start(eth->ppe) == 0)
-			gdm_config = MTK_GDMA_TO_PPE;
+		for (i = 0; i < ARRAY_SIZE(eth->ppe); i++)
+			mtk_ppe_start(eth->ppe[i]);
 
+		gdm_config = soc->offload_version ? soc->reg_map->gdma_to_ppe
+						  : MTK_GDMA_TO_PDMA;
 		mtk_gdm_config(eth, gdm_config);
 
 		napi_enable(&eth->tx_napi);
 		napi_enable(&eth->rx_napi);
 		mtk_tx_irq_enable(eth, MTK_TX_DONE_INT);
-		mtk_rx_irq_enable(eth, eth->soc->txrx.rx_irq_done_mask);
+		mtk_rx_irq_enable(eth, soc->txrx.rx_irq_done_mask);
 		refcount_set(&eth->dma_refcnt, 1);
 	}
 	else
@@ -3028,6 +3047,7 @@ static int mtk_stop(struct net_device *dev)
 {
 	struct mtk_mac *mac = netdev_priv(dev);
 	struct mtk_eth *eth = mac->hw;
+	int i;
 
 	phylink_stop(mac->phylink);
 
@@ -3055,8 +3075,8 @@ static int mtk_stop(struct net_device *dev)
 
 	mtk_dma_free(eth);
 
-	if (eth->soc->offload_version)
-		mtk_ppe_stop(eth->ppe);
+	for (i = 0; i < ARRAY_SIZE(eth->ppe); i++)
+		mtk_ppe_stop(eth->ppe[i]);
 
 	return 0;
 }
@@ -3556,8 +3576,8 @@ static void mtk_get_drvinfo(struct net_device *dev,
 {
 	struct mtk_mac *mac = netdev_priv(dev);
 
-	strlcpy(info->driver, mac->hw->dev->driver->name, sizeof(info->driver));
-	strlcpy(info->bus_info, dev_name(mac->hw->dev), sizeof(info->bus_info));
+	strscpy(info->driver, mac->hw->dev->driver->name, sizeof(info->driver));
+	strscpy(info->bus_info, dev_name(mac->hw->dev), sizeof(info->bus_info));
 	info->n_stats = ARRAY_SIZE(mtk_ethtool_stats);
 }
 
@@ -3925,6 +3945,7 @@ void mtk_eth_set_dma_device(struct mtk_eth *eth, struct device *dma_dev)
 
 static int mtk_probe(struct platform_device *pdev)
 {
+	struct resource *res = NULL;
 	struct device_node *mac_np;
 	struct mtk_eth *eth;
 	int err, i;
@@ -4005,20 +4026,31 @@ static int mtk_probe(struct platform_device *pdev)
 		}
 	}
 
-	for (i = 0;; i++) {
-		struct device_node *np = of_parse_phandle(pdev->dev.of_node,
-							  "mediatek,wed", i);
-		static const u32 wdma_regs[] = {
-			MTK_WDMA0_BASE,
-			MTK_WDMA1_BASE
-		};
-		void __iomem *wdma;
-
-		if (!np || i >= ARRAY_SIZE(wdma_regs))
-			break;
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
+		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+		if (!res)
+			return -EINVAL;
+	}
 
-		wdma = eth->base + wdma_regs[i];
-		mtk_wed_add_hw(np, eth, wdma, i);
+	if (eth->soc->offload_version) {
+		for (i = 0;; i++) {
+			struct device_node *np;
+			phys_addr_t wdma_phy;
+			u32 wdma_base;
+
+			if (i >= ARRAY_SIZE(eth->soc->reg_map->wdma_base))
+				break;
+
+			np = of_parse_phandle(pdev->dev.of_node,
+					      "mediatek,wed", i);
+			if (!np)
+				break;
+
+			wdma_base = eth->soc->reg_map->wdma_base[i];
+			wdma_phy = res ? res->start + wdma_base : 0;
+			mtk_wed_add_hw(np, eth, eth->base + wdma_base,
+				       wdma_phy, i);
+		}
 	}
 
 	for (i = 0; i < 3; i++) {
@@ -4096,10 +4128,19 @@ static int mtk_probe(struct platform_device *pdev)
 	}
 
 	if (eth->soc->offload_version) {
-		eth->ppe = mtk_ppe_init(eth, eth->base + MTK_ETH_PPE_BASE, 2);
-		if (!eth->ppe) {
-			err = -ENOMEM;
-			goto err_free_dev;
+		u32 num_ppe;
+
+		num_ppe = MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2) ? 2 : 1;
+		num_ppe = min_t(u32, ARRAY_SIZE(eth->ppe), num_ppe);
+		for (i = 0; i < num_ppe; i++) {
+			u32 ppe_addr = eth->soc->reg_map->ppe_base + i * 0x400;
+
+			eth->ppe[i] = mtk_ppe_init(eth, eth->base + ppe_addr,
+						   eth->soc->offload_version, i);
+			if (!eth->ppe[i]) {
+				err = -ENOMEM;
+				goto err_free_dev;
+			}
 		}
 
 		err = mtk_eth_offload_init(eth);
@@ -4125,10 +4166,8 @@ static int mtk_probe(struct platform_device *pdev)
 	 * for NAPI to work
 	 */
 	init_dummy_netdev(&eth->dummy_dev);
-	netif_napi_add(&eth->dummy_dev, &eth->tx_napi, mtk_napi_tx,
-		       NAPI_POLL_WEIGHT);
-	netif_napi_add(&eth->dummy_dev, &eth->rx_napi, mtk_napi_rx,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(&eth->dummy_dev, &eth->tx_napi, mtk_napi_tx);
+	netif_napi_add(&eth->dummy_dev, &eth->rx_napi, mtk_napi_rx);
 
 	platform_set_drvdata(pdev, eth);
 
@@ -4192,6 +4231,8 @@ static const struct mtk_soc_data mt7621_data = {
 	.required_clks = MT7621_CLKS_BITMAP,
 	.required_pctl = false,
 	.offload_version = 2,
+	.hash_offset = 2,
+	.foe_entry_size = sizeof(struct mtk_foe_entry) - 16,
 	.txrx = {
 		.txd_size = sizeof(struct mtk_tx_dma),
 		.rxd_size = sizeof(struct mtk_rx_dma),
@@ -4210,6 +4251,8 @@ static const struct mtk_soc_data mt7622_data = {
 	.required_clks = MT7622_CLKS_BITMAP,
 	.required_pctl = false,
 	.offload_version = 2,
+	.hash_offset = 2,
+	.foe_entry_size = sizeof(struct mtk_foe_entry) - 16,
 	.txrx = {
 		.txd_size = sizeof(struct mtk_tx_dma),
 		.rxd_size = sizeof(struct mtk_rx_dma),
@@ -4227,6 +4270,8 @@ static const struct mtk_soc_data mt7623_data = {
 	.required_clks = MT7623_CLKS_BITMAP,
 	.required_pctl = true,
 	.offload_version = 2,
+	.hash_offset = 2,
+	.foe_entry_size = sizeof(struct mtk_foe_entry) - 16,
 	.txrx = {
 		.txd_size = sizeof(struct mtk_tx_dma),
 		.rxd_size = sizeof(struct mtk_rx_dma),
@@ -4258,8 +4303,11 @@ static const struct mtk_soc_data mt7986_data = {
 	.reg_map = &mt7986_reg_map,
 	.ana_rgc3 = 0x128,
 	.caps = MT7986_CAPS,
+	.hw_features = MTK_HW_FEATURES,
 	.required_clks = MT7986_CLKS_BITMAP,
 	.required_pctl = false,
+	.hash_offset = 4,
+	.foe_entry_size = sizeof(struct mtk_foe_entry),
 	.txrx = {
 		.txd_size = sizeof(struct mtk_tx_dma_v2),
 		.rxd_size = sizeof(struct mtk_rx_dma_v2),
diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.h b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
index ecf85e9ed824..b52f3b0177ef 100644
--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.h
+++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.h
@@ -105,7 +105,6 @@
 #define MTK_GDMA_TCS_EN		BIT(21)
 #define MTK_GDMA_UCS_EN		BIT(20)
 #define MTK_GDMA_TO_PDMA	0x0
-#define MTK_GDMA_TO_PPE		0x4444
 #define MTK_GDMA_DROP_ALL       0x7777
 
 /* Unicast Filter MAC Address Register - Low */
@@ -269,9 +268,6 @@
 #define TX_DMA_FPORT_MASK_V2	0xf
 #define TX_DMA_SWC_V2		BIT(30)
 
-#define MTK_WDMA0_BASE		0x2800
-#define MTK_WDMA1_BASE		0x2c00
-
 /* QDMA descriptor txd4 */
 #define TX_DMA_CHKSUM		(0x7 << 29)
 #define TX_DMA_TSO		BIT(28)
@@ -319,8 +315,8 @@
 #define MTK_RXD5_PPE_CPU_REASON	GENMASK(22, 18)
 #define MTK_RXD5_SRC_PORT	GENMASK(29, 26)
 
-#define RX_DMA_GET_SPORT(x)	(((x) >> 19) & 0xf)
-#define RX_DMA_GET_SPORT_V2(x)	(((x) >> 26) & 0x7)
+#define RX_DMA_GET_SPORT(x)	(((x) >> 19) & 0x7)
+#define RX_DMA_GET_SPORT_V2(x)	(((x) >> 26) & 0xf)
 
 /* PDMA V2 descriptor rxd3 */
 #define RX_DMA_VTAG_V2		BIT(0)
@@ -955,6 +951,9 @@ struct mtk_reg_map {
 		u32	fq_blen;	/* fq free page buffer length */
 	} qdma;
 	u32	gdm1_cnt;
+	u32	gdma_to_ppe;
+	u32	ppe_base;
+	u32	wdma_base[2];
 };
 
 /* struct mtk_eth_data -	This is the structure holding all differences
@@ -968,6 +967,8 @@ struct mtk_reg_map {
  *				the target SoC
  * @required_pctl		A bool value to show whether the SoC requires
  *				the extra setup for those pins used by GMAC.
+ * @hash_offset			Flow table hash offset.
+ * @foe_entry_size		Foe table entry size.
  * @txd_size			Tx DMA descriptor size.
  * @rxd_size			Rx DMA descriptor size.
  * @rx_irq_done_mask		Rx irq done register mask.
@@ -982,6 +983,8 @@ struct mtk_soc_data {
 	u32		required_clks;
 	bool		required_pctl;
 	u8		offload_version;
+	u8		hash_offset;
+	u16		foe_entry_size;
 	netdev_features_t hw_features;
 	struct {
 		u32	txd_size;
@@ -1111,7 +1114,7 @@ struct mtk_eth {
 
 	int				ip_align;
 
-	struct mtk_ppe			*ppe;
+	struct mtk_ppe			*ppe[2];
 	struct rhashtable		flow_table;
 
 	struct bpf_prog			__rcu *prog;
@@ -1142,6 +1145,86 @@ struct mtk_mac {
 /* the struct describing the SoC. these are declared in the soc_xyz.c files */
 extern const struct of_device_id of_mtk_match[];
 
+static inline struct mtk_foe_entry *
+mtk_foe_get_entry(struct mtk_ppe *ppe, u16 hash)
+{
+	const struct mtk_soc_data *soc = ppe->eth->soc;
+
+	return ppe->foe_table + hash * soc->foe_entry_size;
+}
+
+static inline u32 mtk_get_ib1_ts_mask(struct mtk_eth *eth)
+{
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
+		return MTK_FOE_IB1_BIND_TIMESTAMP_V2;
+
+	return MTK_FOE_IB1_BIND_TIMESTAMP;
+}
+
+static inline u32 mtk_get_ib1_ppoe_mask(struct mtk_eth *eth)
+{
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
+		return MTK_FOE_IB1_BIND_PPPOE_V2;
+
+	return MTK_FOE_IB1_BIND_PPPOE;
+}
+
+static inline u32 mtk_get_ib1_vlan_tag_mask(struct mtk_eth *eth)
+{
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
+		return MTK_FOE_IB1_BIND_VLAN_TAG_V2;
+
+	return MTK_FOE_IB1_BIND_VLAN_TAG;
+}
+
+static inline u32 mtk_get_ib1_vlan_layer_mask(struct mtk_eth *eth)
+{
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
+		return MTK_FOE_IB1_BIND_VLAN_LAYER_V2;
+
+	return MTK_FOE_IB1_BIND_VLAN_LAYER;
+}
+
+static inline u32 mtk_prep_ib1_vlan_layer(struct mtk_eth *eth, u32 val)
+{
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
+		return FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER_V2, val);
+
+	return FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, val);
+}
+
+static inline u32 mtk_get_ib1_vlan_layer(struct mtk_eth *eth, u32 val)
+{
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
+		return FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER_V2, val);
+
+	return FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER, val);
+}
+
+static inline u32 mtk_get_ib1_pkt_type_mask(struct mtk_eth *eth)
+{
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
+		return MTK_FOE_IB1_PACKET_TYPE_V2;
+
+	return MTK_FOE_IB1_PACKET_TYPE;
+}
+
+static inline u32 mtk_get_ib1_pkt_type(struct mtk_eth *eth, u32 val)
+{
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
+		return FIELD_GET(MTK_FOE_IB1_PACKET_TYPE_V2, val);
+
+	return FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, val);
+}
+
+static inline u32 mtk_get_ib2_multicast_mask(struct mtk_eth *eth)
+{
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2))
+		return MTK_FOE_IB2_MULTICAST_V2;
+
+	return MTK_FOE_IB2_MULTICAST;
+}
+
 /* read the hardware status register */
 void mtk_stats_update_mac(struct mtk_mac *mac);
 
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c
index cfe804bc8d20..ae00e572390d 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe.c
+++ b/drivers/net/ethernet/mediatek/mtk_ppe.c
@@ -56,7 +56,7 @@ static u32 ppe_clear(struct mtk_ppe *ppe, u32 reg, u32 val)
 
 static u32 mtk_eth_timestamp(struct mtk_eth *eth)
 {
-	return mtk_r32(eth, 0x0010) & MTK_FOE_IB1_BIND_TIMESTAMP;
+	return mtk_r32(eth, 0x0010) & mtk_get_ib1_ts_mask(eth);
 }
 
 static int mtk_ppe_wait_busy(struct mtk_ppe *ppe)
@@ -88,12 +88,12 @@ static void mtk_ppe_cache_enable(struct mtk_ppe *ppe, bool enable)
 		enable * MTK_PPE_CACHE_CTL_EN);
 }
 
-static u32 mtk_ppe_hash_entry(struct mtk_foe_entry *e)
+static u32 mtk_ppe_hash_entry(struct mtk_eth *eth, struct mtk_foe_entry *e)
 {
 	u32 hv1, hv2, hv3;
 	u32 hash;
 
-	switch (FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, e->ib1)) {
+	switch (mtk_get_ib1_pkt_type(eth, e->ib1)) {
 		case MTK_PPE_PKT_TYPE_IPV4_ROUTE:
 		case MTK_PPE_PKT_TYPE_IPV4_HNAPT:
 			hv1 = e->ipv4.orig.ports;
@@ -122,16 +122,16 @@ static u32 mtk_ppe_hash_entry(struct mtk_foe_entry *e)
 	hash = (hash >> 24) | ((hash & 0xffffff) << 8);
 	hash ^= hv1 ^ hv2 ^ hv3;
 	hash ^= hash >> 16;
-	hash <<= 1;
+	hash <<= (ffs(eth->soc->hash_offset) - 1);
 	hash &= MTK_PPE_ENTRIES - 1;
 
 	return hash;
 }
 
 static inline struct mtk_foe_mac_info *
-mtk_foe_entry_l2(struct mtk_foe_entry *entry)
+mtk_foe_entry_l2(struct mtk_eth *eth, struct mtk_foe_entry *entry)
 {
-	int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
+	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
 
 	if (type == MTK_PPE_PKT_TYPE_BRIDGE)
 		return &entry->bridge.l2;
@@ -143,9 +143,9 @@ mtk_foe_entry_l2(struct mtk_foe_entry *entry)
 }
 
 static inline u32 *
-mtk_foe_entry_ib2(struct mtk_foe_entry *entry)
+mtk_foe_entry_ib2(struct mtk_eth *eth, struct mtk_foe_entry *entry)
 {
-	int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
+	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
 
 	if (type == MTK_PPE_PKT_TYPE_BRIDGE)
 		return &entry->bridge.ib2;
@@ -156,27 +156,38 @@ mtk_foe_entry_ib2(struct mtk_foe_entry *entry)
 	return &entry->ipv4.ib2;
 }
 
-int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto,
-			  u8 pse_port, u8 *src_mac, u8 *dest_mac)
+int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry,
+			  int type, int l4proto, u8 pse_port, u8 *src_mac,
+			  u8 *dest_mac)
 {
 	struct mtk_foe_mac_info *l2;
 	u32 ports_pad, val;
 
 	memset(entry, 0, sizeof(*entry));
 
-	val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
-	      FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) |
-	      FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
-	      MTK_FOE_IB1_BIND_TTL |
-	      MTK_FOE_IB1_BIND_CACHE;
-	entry->ib1 = val;
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
+		val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
+		      FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE_V2, type) |
+		      FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
+		      MTK_FOE_IB1_BIND_CACHE_V2 | MTK_FOE_IB1_BIND_TTL_V2;
+		entry->ib1 = val;
 
-	val = FIELD_PREP(MTK_FOE_IB2_PORT_MG, 0x3f) |
-	      FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f) |
-	      FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port);
+		val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, pse_port) |
+		      FIELD_PREP(MTK_FOE_IB2_PORT_AG_V2, 0xf);
+	} else {
+		val = FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_BIND) |
+		      FIELD_PREP(MTK_FOE_IB1_PACKET_TYPE, type) |
+		      FIELD_PREP(MTK_FOE_IB1_UDP, l4proto == IPPROTO_UDP) |
+		      MTK_FOE_IB1_BIND_CACHE | MTK_FOE_IB1_BIND_TTL;
+		entry->ib1 = val;
+
+		val = FIELD_PREP(MTK_FOE_IB2_DEST_PORT, pse_port) |
+		      FIELD_PREP(MTK_FOE_IB2_PORT_MG, 0x3f) |
+		      FIELD_PREP(MTK_FOE_IB2_PORT_AG, 0x1f);
+	}
 
 	if (is_multicast_ether_addr(dest_mac))
-		val |= MTK_FOE_IB2_MULTICAST;
+		val |= mtk_get_ib2_multicast_mask(eth);
 
 	ports_pad = 0xa5a5a500 | (l4proto & 0xff);
 	if (type == MTK_PPE_PKT_TYPE_IPV4_ROUTE)
@@ -210,24 +221,30 @@ int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto,
 	return 0;
 }
 
-int mtk_foe_entry_set_pse_port(struct mtk_foe_entry *entry, u8 port)
+int mtk_foe_entry_set_pse_port(struct mtk_eth *eth,
+			       struct mtk_foe_entry *entry, u8 port)
 {
-	u32 *ib2 = mtk_foe_entry_ib2(entry);
-	u32 val;
+	u32 *ib2 = mtk_foe_entry_ib2(eth, entry);
+	u32 val = *ib2;
 
-	val = *ib2;
-	val &= ~MTK_FOE_IB2_DEST_PORT;
-	val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port);
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
+		val &= ~MTK_FOE_IB2_DEST_PORT_V2;
+		val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT_V2, port);
+	} else {
+		val &= ~MTK_FOE_IB2_DEST_PORT;
+		val |= FIELD_PREP(MTK_FOE_IB2_DEST_PORT, port);
+	}
 	*ib2 = val;
 
 	return 0;
 }
 
-int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool egress,
+int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth,
+				 struct mtk_foe_entry *entry, bool egress,
 				 __be32 src_addr, __be16 src_port,
 				 __be32 dest_addr, __be16 dest_port)
 {
-	int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
+	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
 	struct mtk_ipv4_tuple *t;
 
 	switch (type) {
@@ -262,11 +279,12 @@ int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool egress,
 	return 0;
 }
 
-int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry,
+int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth,
+				 struct mtk_foe_entry *entry,
 				 __be32 *src_addr, __be16 src_port,
 				 __be32 *dest_addr, __be16 dest_port)
 {
-	int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->ib1);
+	int type = mtk_get_ib1_pkt_type(eth, entry->ib1);
 	u32 *src, *dest;
 	int i;
 
@@ -297,39 +315,41 @@ int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry,
 	return 0;
 }
 
-int mtk_foe_entry_set_dsa(struct mtk_foe_entry *entry, int port)
+int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry,
+			  int port)
 {
-	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
+	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
 
 	l2->etype = BIT(port);
 
-	if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_LAYER))
-		entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1);
+	if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)))
+		entry->ib1 |= mtk_prep_ib1_vlan_layer(eth, 1);
 	else
 		l2->etype |= BIT(8);
 
-	entry->ib1 &= ~MTK_FOE_IB1_BIND_VLAN_TAG;
+	entry->ib1 &= ~mtk_get_ib1_vlan_tag_mask(eth);
 
 	return 0;
 }
 
-int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid)
+int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry,
+			   int vid)
 {
-	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
+	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
 
-	switch (FIELD_GET(MTK_FOE_IB1_BIND_VLAN_LAYER, entry->ib1)) {
+	switch (mtk_get_ib1_vlan_layer(eth, entry->ib1)) {
 	case 0:
-		entry->ib1 |= MTK_FOE_IB1_BIND_VLAN_TAG |
-			      FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1);
+		entry->ib1 |= mtk_get_ib1_vlan_tag_mask(eth) |
+			      mtk_prep_ib1_vlan_layer(eth, 1);
 		l2->vlan1 = vid;
 		return 0;
 	case 1:
-		if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_TAG)) {
+		if (!(entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth))) {
 			l2->vlan1 = vid;
 			l2->etype |= BIT(8);
 		} else {
 			l2->vlan2 = vid;
-			entry->ib1 += FIELD_PREP(MTK_FOE_IB1_BIND_VLAN_LAYER, 1);
+			entry->ib1 += mtk_prep_ib1_vlan_layer(eth, 1);
 		}
 		return 0;
 	default:
@@ -337,34 +357,42 @@ int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid)
 	}
 }
 
-int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid)
+int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry,
+			    int sid)
 {
-	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
+	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
 
-	if (!(entry->ib1 & MTK_FOE_IB1_BIND_VLAN_LAYER) ||
-	    (entry->ib1 & MTK_FOE_IB1_BIND_VLAN_TAG))
+	if (!(entry->ib1 & mtk_get_ib1_vlan_layer_mask(eth)) ||
+	    (entry->ib1 & mtk_get_ib1_vlan_tag_mask(eth)))
 		l2->etype = ETH_P_PPP_SES;
 
-	entry->ib1 |= MTK_FOE_IB1_BIND_PPPOE;
+	entry->ib1 |= mtk_get_ib1_ppoe_mask(eth);
 	l2->pppoe_id = sid;
 
 	return 0;
 }
 
-int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq,
-			   int bss, int wcid)
+int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry,
+			   int wdma_idx, int txq, int bss, int wcid)
 {
-	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(entry);
-	u32 *ib2 = mtk_foe_entry_ib2(entry);
-
-	*ib2 &= ~MTK_FOE_IB2_PORT_MG;
-	*ib2 |= MTK_FOE_IB2_WDMA_WINFO;
-	if (wdma_idx)
-		*ib2 |= MTK_FOE_IB2_WDMA_DEVIDX;
+	struct mtk_foe_mac_info *l2 = mtk_foe_entry_l2(eth, entry);
+	u32 *ib2 = mtk_foe_entry_ib2(eth, entry);
 
-	l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) |
-		    FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) |
-		    FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq);
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
+		*ib2 &= ~MTK_FOE_IB2_PORT_MG_V2;
+		*ib2 |=  FIELD_PREP(MTK_FOE_IB2_RX_IDX, txq) |
+			 MTK_FOE_IB2_WDMA_WINFO_V2;
+		l2->winfo = FIELD_PREP(MTK_FOE_WINFO_WCID, wcid) |
+			    FIELD_PREP(MTK_FOE_WINFO_BSS, bss);
+	} else {
+		*ib2 &= ~MTK_FOE_IB2_PORT_MG;
+		*ib2 |= MTK_FOE_IB2_WDMA_WINFO;
+		if (wdma_idx)
+			*ib2 |= MTK_FOE_IB2_WDMA_DEVIDX;
+		l2->vlan2 = FIELD_PREP(MTK_FOE_VLAN2_WINFO_BSS, bss) |
+			    FIELD_PREP(MTK_FOE_VLAN2_WINFO_WCID, wcid) |
+			    FIELD_PREP(MTK_FOE_VLAN2_WINFO_RING, txq);
+	}
 
 	return 0;
 }
@@ -376,14 +404,15 @@ static inline bool mtk_foe_entry_usable(struct mtk_foe_entry *entry)
 }
 
 static bool
-mtk_flow_entry_match(struct mtk_flow_entry *entry, struct mtk_foe_entry *data)
+mtk_flow_entry_match(struct mtk_eth *eth, struct mtk_flow_entry *entry,
+		     struct mtk_foe_entry *data)
 {
 	int type, len;
 
 	if ((data->ib1 ^ entry->data.ib1) & MTK_FOE_IB1_UDP)
 		return false;
 
-	type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->data.ib1);
+	type = mtk_get_ib1_pkt_type(eth, entry->data.ib1);
 	if (type > MTK_PPE_PKT_TYPE_IPV4_DSLITE)
 		len = offsetof(struct mtk_foe_entry, ipv6._rsv);
 	else
@@ -410,9 +439,10 @@ __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 
 	hlist_del_init(&entry->list);
 	if (entry->hash != 0xffff) {
-		ppe->foe_table[entry->hash].ib1 &= ~MTK_FOE_IB1_STATE;
-		ppe->foe_table[entry->hash].ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE,
-							      MTK_FOE_STATE_UNBIND);
+		struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, entry->hash);
+
+		hwe->ib1 &= ~MTK_FOE_IB1_STATE;
+		hwe->ib1 |= FIELD_PREP(MTK_FOE_IB1_STATE, MTK_FOE_STATE_INVALID);
 		dma_wmb();
 	}
 	entry->hash = 0xffff;
@@ -426,14 +456,12 @@ __mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 
 static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1)
 {
-	u16 timestamp;
-	u16 now;
-
-	now = mtk_eth_timestamp(ppe->eth) & MTK_FOE_IB1_BIND_TIMESTAMP;
-	timestamp = ib1 & MTK_FOE_IB1_BIND_TIMESTAMP;
+	u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth);
+	u16 now = mtk_eth_timestamp(ppe->eth);
+	u16 timestamp = ib1 & ib1_ts_mask;
 
 	if (timestamp > now)
-		return MTK_FOE_IB1_BIND_TIMESTAMP + 1 - timestamp + now;
+		return ib1_ts_mask + 1 - timestamp + now;
 	else
 		return now - timestamp;
 }
@@ -441,6 +469,7 @@ static int __mtk_foe_entry_idle_time(struct mtk_ppe *ppe, u32 ib1)
 static void
 mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 {
+	u32 ib1_ts_mask = mtk_get_ib1_ts_mask(ppe->eth);
 	struct mtk_flow_entry *cur;
 	struct mtk_foe_entry *hwe;
 	struct hlist_node *tmp;
@@ -451,7 +480,7 @@ mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 		int cur_idle;
 		u32 ib1;
 
-		hwe = &ppe->foe_table[cur->hash];
+		hwe = mtk_foe_get_entry(ppe, cur->hash);
 		ib1 = READ_ONCE(hwe->ib1);
 
 		if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND) {
@@ -465,16 +494,16 @@ mtk_flow_entry_update_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 			continue;
 
 		idle = cur_idle;
-		entry->data.ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
-		entry->data.ib1 |= hwe->ib1 & MTK_FOE_IB1_BIND_TIMESTAMP;
+		entry->data.ib1 &= ~ib1_ts_mask;
+		entry->data.ib1 |= hwe->ib1 & ib1_ts_mask;
 	}
 }
 
 static void
 mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 {
+	struct mtk_foe_entry foe = {};
 	struct mtk_foe_entry *hwe;
-	struct mtk_foe_entry foe;
 
 	spin_lock_bh(&ppe_lock);
 
@@ -486,9 +515,9 @@ mtk_flow_entry_update(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 	if (entry->hash == 0xffff)
 		goto out;
 
-	hwe = &ppe->foe_table[entry->hash];
-	memcpy(&foe, hwe, sizeof(foe));
-	if (!mtk_flow_entry_match(entry, &foe)) {
+	hwe = mtk_foe_get_entry(ppe, entry->hash);
+	memcpy(&foe, hwe, ppe->eth->soc->foe_entry_size);
+	if (!mtk_flow_entry_match(ppe->eth, entry, &foe)) {
 		entry->hash = 0xffff;
 		goto out;
 	}
@@ -503,16 +532,22 @@ static void
 __mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_foe_entry *entry,
 		       u16 hash)
 {
+	struct mtk_eth *eth = ppe->eth;
+	u16 timestamp = mtk_eth_timestamp(eth);
 	struct mtk_foe_entry *hwe;
-	u16 timestamp;
 
-	timestamp = mtk_eth_timestamp(ppe->eth);
-	timestamp &= MTK_FOE_IB1_BIND_TIMESTAMP;
-	entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
-	entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP, timestamp);
+	if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
+		entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP_V2;
+		entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP_V2,
+					 timestamp);
+	} else {
+		entry->ib1 &= ~MTK_FOE_IB1_BIND_TIMESTAMP;
+		entry->ib1 |= FIELD_PREP(MTK_FOE_IB1_BIND_TIMESTAMP,
+					 timestamp);
+	}
 
-	hwe = &ppe->foe_table[hash];
-	memcpy(&hwe->data, &entry->data, sizeof(hwe->data));
+	hwe = mtk_foe_get_entry(ppe, hash);
+	memcpy(&hwe->data, &entry->data, eth->soc->foe_entry_size - sizeof(hwe->ib1));
 	wmb();
 	hwe->ib1 = entry->ib1;
 
@@ -539,16 +574,17 @@ mtk_foe_entry_commit_l2(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 
 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 {
-	int type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, entry->data.ib1);
+	const struct mtk_soc_data *soc = ppe->eth->soc;
+	int type = mtk_get_ib1_pkt_type(ppe->eth, entry->data.ib1);
 	u32 hash;
 
 	if (type == MTK_PPE_PKT_TYPE_BRIDGE)
 		return mtk_foe_entry_commit_l2(ppe, entry);
 
-	hash = mtk_ppe_hash_entry(&entry->data);
+	hash = mtk_ppe_hash_entry(ppe->eth, &entry->data);
 	entry->hash = 0xffff;
 	spin_lock_bh(&ppe_lock);
-	hlist_add_head(&entry->list, &ppe->foe_flow[hash / 2]);
+	hlist_add_head(&entry->list, &ppe->foe_flow[hash / soc->hash_offset]);
 	spin_unlock_bh(&ppe_lock);
 
 	return 0;
@@ -558,10 +594,11 @@ static void
 mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry,
 			     u16 hash)
 {
+	const struct mtk_soc_data *soc = ppe->eth->soc;
 	struct mtk_flow_entry *flow_info;
-	struct mtk_foe_entry foe, *hwe;
+	struct mtk_foe_entry foe = {}, *hwe;
 	struct mtk_foe_mac_info *l2;
-	u32 ib1_mask = MTK_FOE_IB1_PACKET_TYPE | MTK_FOE_IB1_UDP;
+	u32 ib1_mask = mtk_get_ib1_pkt_type_mask(ppe->eth) | MTK_FOE_IB1_UDP;
 	int type;
 
 	flow_info = kzalloc(offsetof(struct mtk_flow_entry, l2_data.end),
@@ -572,32 +609,34 @@ mtk_foe_entry_commit_subflow(struct mtk_ppe *ppe, struct mtk_flow_entry *entry,
 	flow_info->l2_data.base_flow = entry;
 	flow_info->type = MTK_FLOW_TYPE_L2_SUBFLOW;
 	flow_info->hash = hash;
-	hlist_add_head(&flow_info->list, &ppe->foe_flow[hash / 2]);
+	hlist_add_head(&flow_info->list,
+		       &ppe->foe_flow[hash / soc->hash_offset]);
 	hlist_add_head(&flow_info->l2_data.list, &entry->l2_flows);
 
-	hwe = &ppe->foe_table[hash];
-	memcpy(&foe, hwe, sizeof(foe));
+	hwe = mtk_foe_get_entry(ppe, hash);
+	memcpy(&foe, hwe, soc->foe_entry_size);
 	foe.ib1 &= ib1_mask;
 	foe.ib1 |= entry->data.ib1 & ~ib1_mask;
 
-	l2 = mtk_foe_entry_l2(&foe);
+	l2 = mtk_foe_entry_l2(ppe->eth, &foe);
 	memcpy(l2, &entry->data.bridge.l2, sizeof(*l2));
 
-	type = FIELD_GET(MTK_FOE_IB1_PACKET_TYPE, foe.ib1);
+	type = mtk_get_ib1_pkt_type(ppe->eth, foe.ib1);
 	if (type == MTK_PPE_PKT_TYPE_IPV4_HNAPT)
 		memcpy(&foe.ipv4.new, &foe.ipv4.orig, sizeof(foe.ipv4.new));
 	else if (type >= MTK_PPE_PKT_TYPE_IPV6_ROUTE_3T && l2->etype == ETH_P_IP)
 		l2->etype = ETH_P_IPV6;
 
-	*mtk_foe_entry_ib2(&foe) = entry->data.bridge.ib2;
+	*mtk_foe_entry_ib2(ppe->eth, &foe) = entry->data.bridge.ib2;
 
 	__mtk_foe_entry_commit(ppe, &foe, hash);
 }
 
 void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
 {
-	struct hlist_head *head = &ppe->foe_flow[hash / 2];
-	struct mtk_foe_entry *hwe = &ppe->foe_table[hash];
+	const struct mtk_soc_data *soc = ppe->eth->soc;
+	struct hlist_head *head = &ppe->foe_flow[hash / soc->hash_offset];
+	struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, hash);
 	struct mtk_flow_entry *entry;
 	struct mtk_foe_bridge key = {};
 	struct hlist_node *n;
@@ -621,7 +660,7 @@ void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
 			continue;
 		}
 
-		if (found || !mtk_flow_entry_match(entry, hwe)) {
+		if (found || !mtk_flow_entry_match(ppe->eth, entry, hwe)) {
 			if (entry->hash != 0xffff)
 				entry->hash = 0xffff;
 			continue;
@@ -678,11 +717,13 @@ int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry)
 }
 
 struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
-		 int version)
+			     int version, int index)
 {
+	const struct mtk_soc_data *soc = eth->soc;
 	struct device *dev = eth->dev;
-	struct mtk_foe_entry *foe;
 	struct mtk_ppe *ppe;
+	u32 foe_flow_size;
+	void *foe;
 
 	ppe = devm_kzalloc(dev, sizeof(*ppe), GFP_KERNEL);
 	if (!ppe)
@@ -698,14 +739,21 @@ struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
 	ppe->dev = dev;
 	ppe->version = version;
 
-	foe = dmam_alloc_coherent(ppe->dev, MTK_PPE_ENTRIES * sizeof(*foe),
+	foe = dmam_alloc_coherent(ppe->dev,
+				  MTK_PPE_ENTRIES * soc->foe_entry_size,
 				  &ppe->foe_phys, GFP_KERNEL);
 	if (!foe)
 		return NULL;
 
 	ppe->foe_table = foe;
 
-	mtk_ppe_debugfs_init(ppe);
+	foe_flow_size = (MTK_PPE_ENTRIES / soc->hash_offset) *
+			sizeof(*ppe->foe_flow);
+	ppe->foe_flow = devm_kzalloc(dev, foe_flow_size, GFP_KERNEL);
+	if (!ppe->foe_flow)
+		return NULL;
+
+	mtk_ppe_debugfs_init(ppe, index);
 
 	return ppe;
 }
@@ -715,21 +763,30 @@ static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe)
 	static const u8 skip[] = { 12, 25, 38, 51, 76, 89, 102 };
 	int i, k;
 
-	memset(ppe->foe_table, 0, MTK_PPE_ENTRIES * sizeof(*ppe->foe_table));
+	memset(ppe->foe_table, 0,
+	       MTK_PPE_ENTRIES * ppe->eth->soc->foe_entry_size);
 
 	if (!IS_ENABLED(CONFIG_SOC_MT7621))
 		return;
 
 	/* skip all entries that cross the 1024 byte boundary */
-	for (i = 0; i < MTK_PPE_ENTRIES; i += 128)
-		for (k = 0; k < ARRAY_SIZE(skip); k++)
-			ppe->foe_table[i + skip[k]].ib1 |= MTK_FOE_IB1_STATIC;
+	for (i = 0; i < MTK_PPE_ENTRIES; i += 128) {
+		for (k = 0; k < ARRAY_SIZE(skip); k++) {
+			struct mtk_foe_entry *hwe;
+
+			hwe = mtk_foe_get_entry(ppe, i + skip[k]);
+			hwe->ib1 |= MTK_FOE_IB1_STATIC;
+		}
+	}
 }
 
-int mtk_ppe_start(struct mtk_ppe *ppe)
+void mtk_ppe_start(struct mtk_ppe *ppe)
 {
 	u32 val;
 
+	if (!ppe)
+		return;
+
 	mtk_ppe_init_foe_table(ppe);
 	ppe_w32(ppe, MTK_PPE_TB_BASE, ppe->foe_phys);
 
@@ -748,6 +805,8 @@ int mtk_ppe_start(struct mtk_ppe *ppe)
 			 MTK_PPE_SCAN_MODE_KEEPALIVE_AGE) |
 	      FIELD_PREP(MTK_PPE_TB_CFG_ENTRY_NUM,
 			 MTK_PPE_ENTRIES_SHIFT);
+	if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2))
+		val |= MTK_PPE_TB_CFG_INFO_SEL;
 	ppe_w32(ppe, MTK_PPE_TB_CFG, val);
 
 	ppe_w32(ppe, MTK_PPE_IP_PROTO_CHK,
@@ -755,15 +814,21 @@ int mtk_ppe_start(struct mtk_ppe *ppe)
 
 	mtk_ppe_cache_enable(ppe, true);
 
-	val = MTK_PPE_FLOW_CFG_IP4_TCP_FRAG |
-	      MTK_PPE_FLOW_CFG_IP4_UDP_FRAG |
-	      MTK_PPE_FLOW_CFG_IP6_3T_ROUTE |
+	val = MTK_PPE_FLOW_CFG_IP6_3T_ROUTE |
 	      MTK_PPE_FLOW_CFG_IP6_5T_ROUTE |
 	      MTK_PPE_FLOW_CFG_IP6_6RD |
 	      MTK_PPE_FLOW_CFG_IP4_NAT |
 	      MTK_PPE_FLOW_CFG_IP4_NAPT |
 	      MTK_PPE_FLOW_CFG_IP4_DSLITE |
 	      MTK_PPE_FLOW_CFG_IP4_NAT_FRAG;
+	if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2))
+		val |= MTK_PPE_MD_TOAP_BYP_CRSN0 |
+		       MTK_PPE_MD_TOAP_BYP_CRSN1 |
+		       MTK_PPE_MD_TOAP_BYP_CRSN2 |
+		       MTK_PPE_FLOW_CFG_IP4_HASH_GRE_KEY;
+	else
+		val |= MTK_PPE_FLOW_CFG_IP4_TCP_FRAG |
+		       MTK_PPE_FLOW_CFG_IP4_UDP_FRAG;
 	ppe_w32(ppe, MTK_PPE_FLOW_CFG, val);
 
 	val = FIELD_PREP(MTK_PPE_UNBIND_AGE_MIN_PACKETS, 1000) |
@@ -798,7 +863,10 @@ int mtk_ppe_start(struct mtk_ppe *ppe)
 
 	ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT, 0);
 
-	return 0;
+	if (MTK_HAS_CAPS(ppe->eth->soc->caps, MTK_NETSYS_V2)) {
+		ppe_w32(ppe, MTK_PPE_DEFAULT_CPU_PORT1, 0xcb777);
+		ppe_w32(ppe, MTK_PPE_SBW_CTRL, 0x7f);
+	}
 }
 
 int mtk_ppe_stop(struct mtk_ppe *ppe)
@@ -806,9 +874,15 @@ int mtk_ppe_stop(struct mtk_ppe *ppe)
 	u32 val;
 	int i;
 
-	for (i = 0; i < MTK_PPE_ENTRIES; i++)
-		ppe->foe_table[i].ib1 = FIELD_PREP(MTK_FOE_IB1_STATE,
-						   MTK_FOE_STATE_INVALID);
+	if (!ppe)
+		return 0;
+
+	for (i = 0; i < MTK_PPE_ENTRIES; i++) {
+		struct mtk_foe_entry *hwe = mtk_foe_get_entry(ppe, i);
+
+		hwe->ib1 = FIELD_PREP(MTK_FOE_IB1_STATE,
+				      MTK_FOE_STATE_INVALID);
+	}
 
 	mtk_ppe_cache_enable(ppe, false);
 
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.h b/drivers/net/ethernet/mediatek/mtk_ppe.h
index 69ffce04d630..0b7a67a958e4 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe.h
+++ b/drivers/net/ethernet/mediatek/mtk_ppe.h
@@ -8,8 +8,6 @@
 #include <linux/bitfield.h>
 #include <linux/rhashtable.h>
 
-#define MTK_ETH_PPE_BASE		0xc00
-
 #define MTK_PPE_ENTRIES_SHIFT		3
 #define MTK_PPE_ENTRIES			(1024 << MTK_PPE_ENTRIES_SHIFT)
 #define MTK_PPE_HASH_MASK		(MTK_PPE_ENTRIES - 1)
@@ -34,6 +32,15 @@
 #define MTK_FOE_IB1_UDP			BIT(30)
 #define MTK_FOE_IB1_STATIC		BIT(31)
 
+/* CONFIG_MEDIATEK_NETSYS_V2 */
+#define MTK_FOE_IB1_BIND_TIMESTAMP_V2	GENMASK(7, 0)
+#define MTK_FOE_IB1_BIND_VLAN_LAYER_V2	GENMASK(16, 14)
+#define MTK_FOE_IB1_BIND_PPPOE_V2	BIT(17)
+#define MTK_FOE_IB1_BIND_VLAN_TAG_V2	BIT(18)
+#define MTK_FOE_IB1_BIND_CACHE_V2	BIT(20)
+#define MTK_FOE_IB1_BIND_TTL_V2		BIT(22)
+#define MTK_FOE_IB1_PACKET_TYPE_V2	GENMASK(27, 23)
+
 enum {
 	MTK_PPE_PKT_TYPE_IPV4_HNAPT = 0,
 	MTK_PPE_PKT_TYPE_IPV4_ROUTE = 1,
@@ -55,14 +62,25 @@ enum {
 
 #define MTK_FOE_IB2_PORT_MG		GENMASK(17, 12)
 
+#define MTK_FOE_IB2_RX_IDX		GENMASK(18, 17)
 #define MTK_FOE_IB2_PORT_AG		GENMASK(23, 18)
 
 #define MTK_FOE_IB2_DSCP		GENMASK(31, 24)
 
+/* CONFIG_MEDIATEK_NETSYS_V2 */
+#define MTK_FOE_IB2_PORT_MG_V2		BIT(7)
+#define MTK_FOE_IB2_DEST_PORT_V2	GENMASK(12, 9)
+#define MTK_FOE_IB2_MULTICAST_V2	BIT(13)
+#define MTK_FOE_IB2_WDMA_WINFO_V2	BIT(19)
+#define MTK_FOE_IB2_PORT_AG_V2		GENMASK(23, 20)
+
 #define MTK_FOE_VLAN2_WINFO_BSS		GENMASK(5, 0)
 #define MTK_FOE_VLAN2_WINFO_WCID	GENMASK(13, 6)
 #define MTK_FOE_VLAN2_WINFO_RING	GENMASK(15, 14)
 
+#define MTK_FOE_WINFO_BSS		GENMASK(5, 0)
+#define MTK_FOE_WINFO_WCID		GENMASK(15, 6)
+
 enum {
 	MTK_FOE_STATE_INVALID,
 	MTK_FOE_STATE_UNBIND,
@@ -83,6 +101,9 @@ struct mtk_foe_mac_info {
 
 	u16 pppoe_id;
 	u16 src_mac_lo;
+
+	u16 minfo;
+	u16 winfo;
 };
 
 /* software-only entry type */
@@ -200,7 +221,7 @@ struct mtk_foe_entry {
 		struct mtk_foe_ipv4_dslite dslite;
 		struct mtk_foe_ipv6 ipv6;
 		struct mtk_foe_ipv6_6rd ipv6_6rd;
-		u32 data[19];
+		u32 data[23];
 	};
 };
 
@@ -249,6 +270,7 @@ struct mtk_flow_entry {
 	};
 	u8 type;
 	s8 wed_index;
+	u8 ppe_index;
 	u16 hash;
 	union {
 		struct mtk_foe_entry data;
@@ -267,20 +289,22 @@ struct mtk_ppe {
 	struct device *dev;
 	void __iomem *base;
 	int version;
+	char dirname[5];
 
-	struct mtk_foe_entry *foe_table;
+	void *foe_table;
 	dma_addr_t foe_phys;
 
 	u16 foe_check_time[MTK_PPE_ENTRIES];
-	struct hlist_head foe_flow[MTK_PPE_ENTRIES / 2];
+	struct hlist_head *foe_flow;
 
 	struct rhashtable l2_flows;
 
 	void *acct_table;
 };
 
-struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int version);
-int mtk_ppe_start(struct mtk_ppe *ppe);
+struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base,
+			     int version, int index);
+void mtk_ppe_start(struct mtk_ppe *ppe);
 int mtk_ppe_stop(struct mtk_ppe *ppe);
 
 void __mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash);
@@ -305,34 +329,30 @@ mtk_ppe_check_skb(struct mtk_ppe *ppe, struct sk_buff *skb, u16 hash)
 	__mtk_ppe_check_skb(ppe, skb, hash);
 }
 
-static inline int
-mtk_foe_entry_timestamp(struct mtk_ppe *ppe, u16 hash)
-{
-	u32 ib1 = READ_ONCE(ppe->foe_table[hash].ib1);
-
-	if (FIELD_GET(MTK_FOE_IB1_STATE, ib1) != MTK_FOE_STATE_BIND)
-		return -1;
-
-	return FIELD_GET(MTK_FOE_IB1_BIND_TIMESTAMP, ib1);
-}
-
-int mtk_foe_entry_prepare(struct mtk_foe_entry *entry, int type, int l4proto,
-			  u8 pse_port, u8 *src_mac, u8 *dest_mac);
-int mtk_foe_entry_set_pse_port(struct mtk_foe_entry *entry, u8 port);
-int mtk_foe_entry_set_ipv4_tuple(struct mtk_foe_entry *entry, bool orig,
+int mtk_foe_entry_prepare(struct mtk_eth *eth, struct mtk_foe_entry *entry,
+			  int type, int l4proto, u8 pse_port, u8 *src_mac,
+			  u8 *dest_mac);
+int mtk_foe_entry_set_pse_port(struct mtk_eth *eth,
+			       struct mtk_foe_entry *entry, u8 port);
+int mtk_foe_entry_set_ipv4_tuple(struct mtk_eth *eth,
+				 struct mtk_foe_entry *entry, bool orig,
 				 __be32 src_addr, __be16 src_port,
 				 __be32 dest_addr, __be16 dest_port);
-int mtk_foe_entry_set_ipv6_tuple(struct mtk_foe_entry *entry,
+int mtk_foe_entry_set_ipv6_tuple(struct mtk_eth *eth,
+				 struct mtk_foe_entry *entry,
 				 __be32 *src_addr, __be16 src_port,
 				 __be32 *dest_addr, __be16 dest_port);
-int mtk_foe_entry_set_dsa(struct mtk_foe_entry *entry, int port);
-int mtk_foe_entry_set_vlan(struct mtk_foe_entry *entry, int vid);
-int mtk_foe_entry_set_pppoe(struct mtk_foe_entry *entry, int sid);
-int mtk_foe_entry_set_wdma(struct mtk_foe_entry *entry, int wdma_idx, int txq,
-			   int bss, int wcid);
+int mtk_foe_entry_set_dsa(struct mtk_eth *eth, struct mtk_foe_entry *entry,
+			  int port);
+int mtk_foe_entry_set_vlan(struct mtk_eth *eth, struct mtk_foe_entry *entry,
+			   int vid);
+int mtk_foe_entry_set_pppoe(struct mtk_eth *eth, struct mtk_foe_entry *entry,
+			    int sid);
+int mtk_foe_entry_set_wdma(struct mtk_eth *eth, struct mtk_foe_entry *entry,
+			   int wdma_idx, int txq, int bss, int wcid);
 int mtk_foe_entry_commit(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
 void mtk_foe_entry_clear(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
 int mtk_foe_entry_idle_time(struct mtk_ppe *ppe, struct mtk_flow_entry *entry);
-int mtk_ppe_debugfs_init(struct mtk_ppe *ppe);
+int mtk_ppe_debugfs_init(struct mtk_ppe *ppe, int index);
 
 #endif
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
index eb0b598f14e4..391b071bcff3 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
+++ b/drivers/net/ethernet/mediatek/mtk_ppe_debugfs.c
@@ -79,7 +79,7 @@ mtk_ppe_debugfs_foe_show(struct seq_file *m, void *private, bool bind)
 	int i;
 
 	for (i = 0; i < MTK_PPE_ENTRIES; i++) {
-		struct mtk_foe_entry *entry = &ppe->foe_table[i];
+		struct mtk_foe_entry *entry = mtk_foe_get_entry(ppe, i);
 		struct mtk_foe_mac_info *l2;
 		struct mtk_flow_addr_info ai = {};
 		unsigned char h_source[ETH_ALEN];
@@ -162,52 +162,28 @@ mtk_ppe_debugfs_foe_show(struct seq_file *m, void *private, bool bind)
 }
 
 static int
-mtk_ppe_debugfs_foe_show_all(struct seq_file *m, void *private)
+mtk_ppe_debugfs_foe_all_show(struct seq_file *m, void *private)
 {
 	return mtk_ppe_debugfs_foe_show(m, private, false);
 }
+DEFINE_SHOW_ATTRIBUTE(mtk_ppe_debugfs_foe_all);
 
 static int
-mtk_ppe_debugfs_foe_show_bind(struct seq_file *m, void *private)
+mtk_ppe_debugfs_foe_bind_show(struct seq_file *m, void *private)
 {
 	return mtk_ppe_debugfs_foe_show(m, private, true);
 }
+DEFINE_SHOW_ATTRIBUTE(mtk_ppe_debugfs_foe_bind);
 
-static int
-mtk_ppe_debugfs_foe_open_all(struct inode *inode, struct file *file)
+int mtk_ppe_debugfs_init(struct mtk_ppe *ppe, int index)
 {
-	return single_open(file, mtk_ppe_debugfs_foe_show_all,
-			   inode->i_private);
-}
-
-static int
-mtk_ppe_debugfs_foe_open_bind(struct inode *inode, struct file *file)
-{
-	return single_open(file, mtk_ppe_debugfs_foe_show_bind,
-			   inode->i_private);
-}
-
-int mtk_ppe_debugfs_init(struct mtk_ppe *ppe)
-{
-	static const struct file_operations fops_all = {
-		.open = mtk_ppe_debugfs_foe_open_all,
-		.read = seq_read,
-		.llseek = seq_lseek,
-		.release = single_release,
-	};
-
-	static const struct file_operations fops_bind = {
-		.open = mtk_ppe_debugfs_foe_open_bind,
-		.read = seq_read,
-		.llseek = seq_lseek,
-		.release = single_release,
-	};
-
 	struct dentry *root;
 
-	root = debugfs_create_dir("mtk_ppe", NULL);
-	debugfs_create_file("entries", S_IRUGO, root, ppe, &fops_all);
-	debugfs_create_file("bind", S_IRUGO, root, ppe, &fops_bind);
+	snprintf(ppe->dirname, sizeof(ppe->dirname), "ppe%d", index);
+
+	root = debugfs_create_dir(ppe->dirname, NULL);
+	debugfs_create_file("entries", S_IRUGO, root, ppe, &mtk_ppe_debugfs_foe_all_fops);
+	debugfs_create_file("bind", S_IRUGO, root, ppe, &mtk_ppe_debugfs_foe_bind_fops);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
index 25dc3c3aa31d..28bbd1df3e30 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
+++ b/drivers/net/ethernet/mediatek/mtk_ppe_offload.c
@@ -52,18 +52,19 @@ static const struct rhashtable_params mtk_flow_ht_params = {
 };
 
 static int
-mtk_flow_set_ipv4_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data,
-		       bool egress)
+mtk_flow_set_ipv4_addr(struct mtk_eth *eth, struct mtk_foe_entry *foe,
+		       struct mtk_flow_data *data, bool egress)
 {
-	return mtk_foe_entry_set_ipv4_tuple(foe, egress,
+	return mtk_foe_entry_set_ipv4_tuple(eth, foe, egress,
 					    data->v4.src_addr, data->src_port,
 					    data->v4.dst_addr, data->dst_port);
 }
 
 static int
-mtk_flow_set_ipv6_addr(struct mtk_foe_entry *foe, struct mtk_flow_data *data)
+mtk_flow_set_ipv6_addr(struct mtk_eth *eth, struct mtk_foe_entry *foe,
+		       struct mtk_flow_data *data)
 {
-	return mtk_foe_entry_set_ipv6_tuple(foe,
+	return mtk_foe_entry_set_ipv6_tuple(eth, foe,
 					    data->v6.src_addr.s6_addr32, data->src_port,
 					    data->v6.dst_addr.s6_addr32, data->dst_port);
 }
@@ -173,7 +174,7 @@ mtk_flow_get_dsa_port(struct net_device **dev)
 	if (dp->cpu_dp->tag_ops->proto != DSA_TAG_PROTO_MTK)
 		return -ENODEV;
 
-	*dev = dp->cpu_dp->master;
+	*dev = dsa_port_to_master(dp);
 
 	return dp->index;
 #else
@@ -190,16 +191,29 @@ mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe,
 	int pse_port, dsa_port;
 
 	if (mtk_flow_get_wdma_info(dev, dest_mac, &info) == 0) {
-		mtk_foe_entry_set_wdma(foe, info.wdma_idx, info.queue, info.bss,
-				       info.wcid);
-		pse_port = 3;
+		mtk_foe_entry_set_wdma(eth, foe, info.wdma_idx, info.queue,
+				       info.bss, info.wcid);
+		if (MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2)) {
+			switch (info.wdma_idx) {
+			case 0:
+				pse_port = 8;
+				break;
+			case 1:
+				pse_port = 9;
+				break;
+			default:
+				return -EINVAL;
+			}
+		} else {
+			pse_port = 3;
+		}
 		*wed_index = info.wdma_idx;
 		goto out;
 	}
 
 	dsa_port = mtk_flow_get_dsa_port(&dev);
 	if (dsa_port >= 0)
-		mtk_foe_entry_set_dsa(foe, dsa_port);
+		mtk_foe_entry_set_dsa(eth, foe, dsa_port);
 
 	if (dev == eth->netdev[0])
 		pse_port = 1;
@@ -209,7 +223,7 @@ mtk_flow_set_output_device(struct mtk_eth *eth, struct mtk_foe_entry *foe,
 		return -EOPNOTSUPP;
 
 out:
-	mtk_foe_entry_set_pse_port(foe, pse_port);
+	mtk_foe_entry_set_pse_port(eth, foe, pse_port);
 
 	return 0;
 }
@@ -333,9 +347,8 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
 	    !is_valid_ether_addr(data.eth.h_dest))
 		return -EINVAL;
 
-	err = mtk_foe_entry_prepare(&foe, offload_type, l4proto, 0,
-				    data.eth.h_source,
-				    data.eth.h_dest);
+	err = mtk_foe_entry_prepare(eth, &foe, offload_type, l4proto, 0,
+				    data.eth.h_source, data.eth.h_dest);
 	if (err)
 		return err;
 
@@ -360,7 +373,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
 		data.v4.src_addr = addrs.key->src;
 		data.v4.dst_addr = addrs.key->dst;
 
-		mtk_flow_set_ipv4_addr(&foe, &data, false);
+		mtk_flow_set_ipv4_addr(eth, &foe, &data, false);
 	}
 
 	if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
@@ -371,7 +384,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
 		data.v6.src_addr = addrs.key->src;
 		data.v6.dst_addr = addrs.key->dst;
 
-		mtk_flow_set_ipv6_addr(&foe, &data);
+		mtk_flow_set_ipv6_addr(eth, &foe, &data);
 	}
 
 	flow_action_for_each(i, act, &rule->action) {
@@ -401,7 +414,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
 	}
 
 	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
-		err = mtk_flow_set_ipv4_addr(&foe, &data, true);
+		err = mtk_flow_set_ipv4_addr(eth, &foe, &data, true);
 		if (err)
 			return err;
 	}
@@ -413,10 +426,10 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
 		if (data.vlan.proto != htons(ETH_P_8021Q))
 			return -EOPNOTSUPP;
 
-		mtk_foe_entry_set_vlan(&foe, data.vlan.id);
+		mtk_foe_entry_set_vlan(eth, &foe, data.vlan.id);
 	}
 	if (data.pppoe.num == 1)
-		mtk_foe_entry_set_pppoe(&foe, data.pppoe.sid);
+		mtk_foe_entry_set_pppoe(eth, &foe, data.pppoe.sid);
 
 	err = mtk_flow_set_output_device(eth, &foe, odev, data.eth.h_dest,
 					 &wed_index);
@@ -434,7 +447,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
 	memcpy(&entry->data, &foe, sizeof(entry->data));
 	entry->wed_index = wed_index;
 
-	err = mtk_foe_entry_commit(eth->ppe, entry);
+	err = mtk_foe_entry_commit(eth->ppe[entry->ppe_index], entry);
 	if (err < 0)
 		goto free;
 
@@ -446,7 +459,7 @@ mtk_flow_offload_replace(struct mtk_eth *eth, struct flow_cls_offload *f)
 	return 0;
 
 clear:
-	mtk_foe_entry_clear(eth->ppe, entry);
+	mtk_foe_entry_clear(eth->ppe[entry->ppe_index], entry);
 free:
 	kfree(entry);
 	if (wed_index >= 0)
@@ -464,7 +477,7 @@ mtk_flow_offload_destroy(struct mtk_eth *eth, struct flow_cls_offload *f)
 	if (!entry)
 		return -ENOENT;
 
-	mtk_foe_entry_clear(eth->ppe, entry);
+	mtk_foe_entry_clear(eth->ppe[entry->ppe_index], entry);
 	rhashtable_remove_fast(&eth->flow_table, &entry->node,
 			       mtk_flow_ht_params);
 	if (entry->wed_index >= 0)
@@ -485,7 +498,7 @@ mtk_flow_offload_stats(struct mtk_eth *eth, struct flow_cls_offload *f)
 	if (!entry)
 		return -ENOENT;
 
-	idle = mtk_foe_entry_idle_time(eth->ppe, entry);
+	idle = mtk_foe_entry_idle_time(eth->ppe[entry->ppe_index], entry);
 	f->stats.lastused = jiffies - idle * HZ;
 
 	return 0;
@@ -537,7 +550,7 @@ mtk_eth_setup_tc_block(struct net_device *dev, struct flow_block_offload *f)
 	struct flow_block_cb *block_cb;
 	flow_setup_cb_t *cb;
 
-	if (!eth->ppe || !eth->ppe->foe_table)
+	if (!eth->soc->offload_version)
 		return -EOPNOTSUPP;
 
 	if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
@@ -589,8 +602,5 @@ int mtk_eth_setup_tc(struct net_device *dev, enum tc_setup_type type,
 
 int mtk_eth_offload_init(struct mtk_eth *eth)
 {
-	if (!eth->ppe || !eth->ppe->foe_table)
-		return 0;
-
 	return rhashtable_init(&eth->flow_table, &mtk_flow_ht_params);
 }
diff --git a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
index 0c45ea0900f1..59596d823d8b 100644
--- a/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
+++ b/drivers/net/ethernet/mediatek/mtk_ppe_regs.h
@@ -21,6 +21,9 @@
 #define MTK_PPE_GLO_CFG_BUSY			BIT(31)
 
 #define MTK_PPE_FLOW_CFG			0x204
+#define MTK_PPE_MD_TOAP_BYP_CRSN0		BIT(1)
+#define MTK_PPE_MD_TOAP_BYP_CRSN1		BIT(2)
+#define MTK_PPE_MD_TOAP_BYP_CRSN2		BIT(3)
 #define MTK_PPE_FLOW_CFG_IP4_TCP_FRAG		BIT(6)
 #define MTK_PPE_FLOW_CFG_IP4_UDP_FRAG		BIT(7)
 #define MTK_PPE_FLOW_CFG_IP6_3T_ROUTE		BIT(8)
@@ -54,6 +57,7 @@
 #define MTK_PPE_TB_CFG_HASH_MODE		GENMASK(15, 14)
 #define MTK_PPE_TB_CFG_SCAN_MODE		GENMASK(17, 16)
 #define MTK_PPE_TB_CFG_HASH_DEBUG		GENMASK(19, 18)
+#define MTK_PPE_TB_CFG_INFO_SEL			BIT(20)
 
 enum {
 	MTK_PPE_SCAN_MODE_DISABLED,
@@ -112,6 +116,8 @@ enum {
 #define MTK_PPE_DEFAULT_CPU_PORT		0x248
 #define MTK_PPE_DEFAULT_CPU_PORT_MASK(_n)	(GENMASK(2, 0) << ((_n) * 4))
 
+#define MTK_PPE_DEFAULT_CPU_PORT1		0x24c
+
 #define MTK_PPE_MTU_DROP			0x308
 
 #define MTK_PPE_VLAN_MTU0			0x30c
@@ -141,4 +147,6 @@ enum {
 #define MTK_PPE_MIB_CACHE_CTL_EN		BIT(0)
 #define MTK_PPE_MIB_CACHE_CTL_FLUSH		BIT(2)
 
+#define MTK_PPE_SBW_CTRL			0x374
+
 #endif
diff --git a/drivers/net/ethernet/mediatek/mtk_star_emac.c b/drivers/net/ethernet/mediatek/mtk_star_emac.c
index 3f0e5e64de50..7e890f81148e 100644
--- a/drivers/net/ethernet/mediatek/mtk_star_emac.c
+++ b/drivers/net/ethernet/mediatek/mtk_star_emac.c
@@ -1255,7 +1255,7 @@ static const struct net_device_ops mtk_star_netdev_ops = {
 static void mtk_star_get_drvinfo(struct net_device *dev,
 				 struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, MTK_STAR_DRVNAME, sizeof(info->driver));
+	strscpy(info->driver, MTK_STAR_DRVNAME, sizeof(info->driver));
 }
 
 /* TODO Add ethtool stats. */
@@ -1651,8 +1651,7 @@ static int mtk_star_probe(struct platform_device *pdev)
 	ndev->netdev_ops = &mtk_star_netdev_ops;
 	ndev->ethtool_ops = &mtk_star_ethtool_ops;
 
-	netif_napi_add(ndev, &priv->rx_napi, mtk_star_rx_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &priv->rx_napi, mtk_star_rx_poll);
 	netif_napi_add_tx(ndev, &priv->tx_napi, mtk_star_tx_poll);
 
 	return devm_register_netdev(dev, ndev);
diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c
index 29be2fcafea3..099b6e0df619 100644
--- a/drivers/net/ethernet/mediatek/mtk_wed.c
+++ b/drivers/net/ethernet/mediatek/mtk_wed.c
@@ -25,6 +25,11 @@
 
 #define MTK_WED_TX_RING_SIZE		2048
 #define MTK_WED_WDMA_RING_SIZE		1024
+#define MTK_WED_MAX_GROUP_SIZE		0x100
+#define MTK_WED_VLD_GROUP_SIZE		0x40
+#define MTK_WED_PER_GROUP_PKT		128
+
+#define MTK_WED_FBUF_SIZE		128
 
 static struct mtk_wed_hw *hw_list[2];
 static DEFINE_MUTEX(hw_lock);
@@ -80,11 +85,31 @@ static struct mtk_wed_hw *
 mtk_wed_assign(struct mtk_wed_device *dev)
 {
 	struct mtk_wed_hw *hw;
+	int i;
+
+	if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) {
+		hw = hw_list[pci_domain_nr(dev->wlan.pci_dev->bus)];
+		if (!hw)
+			return NULL;
+
+		if (!hw->wed_dev)
+			goto out;
+
+		if (hw->version == 1)
+			return NULL;
+
+		/* MT7986 WED devices do not have any pcie slot restrictions */
+	}
+	/* MT7986 PCIE or AXI */
+	for (i = 0; i < ARRAY_SIZE(hw_list); i++) {
+		hw = hw_list[i];
+		if (hw && !hw->wed_dev)
+			goto out;
+	}
 
-	hw = hw_list[pci_domain_nr(dev->wlan.pci_dev->bus)];
-	if (!hw || hw->wed_dev)
-		return NULL;
+	return NULL;
 
+out:
 	hw->wed_dev = dev;
 	return hw;
 }
@@ -150,10 +175,17 @@ mtk_wed_buffer_alloc(struct mtk_wed_device *dev)
 
 			desc->buf0 = cpu_to_le32(buf_phys);
 			desc->buf1 = cpu_to_le32(buf_phys + txd_size);
-			ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0, txd_size) |
-			       FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1,
-					  MTK_WED_BUF_SIZE - txd_size) |
-			       MTK_WDMA_DESC_CTRL_LAST_SEG1;
+
+			if (dev->hw->version == 1)
+				ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0, txd_size) |
+				       FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1,
+						  MTK_WED_BUF_SIZE - txd_size) |
+				       MTK_WDMA_DESC_CTRL_LAST_SEG1;
+			else
+				ctrl = FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN0, txd_size) |
+				       FIELD_PREP(MTK_WDMA_DESC_CTRL_LEN1_V2,
+						  MTK_WED_BUF_SIZE - txd_size) |
+				       MTK_WDMA_DESC_CTRL_LAST_SEG0;
 			desc->ctrl = cpu_to_le32(ctrl);
 			desc->info = 0;
 			desc++;
@@ -209,7 +241,7 @@ mtk_wed_free_ring(struct mtk_wed_device *dev, struct mtk_wed_ring *ring)
 	if (!ring->desc)
 		return;
 
-	dma_free_coherent(dev->hw->dev, ring->size * sizeof(*ring->desc),
+	dma_free_coherent(dev->hw->dev, ring->size * ring->desc_size,
 			  ring->desc, ring->desc_phys);
 }
 
@@ -229,6 +261,14 @@ mtk_wed_set_ext_int(struct mtk_wed_device *dev, bool en)
 {
 	u32 mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK;
 
+	if (dev->hw->version == 1)
+		mask |= MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR;
+	else
+		mask |= MTK_WED_EXT_INT_STATUS_RX_FBUF_LO_TH |
+			MTK_WED_EXT_INT_STATUS_RX_FBUF_HI_TH |
+			MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT |
+			MTK_WED_EXT_INT_STATUS_TX_DMA_W_RESP_ERR;
+
 	if (!dev->hw->num_flows)
 		mask &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;
 
@@ -237,9 +277,54 @@ mtk_wed_set_ext_int(struct mtk_wed_device *dev, bool en)
 }
 
 static void
+mtk_wed_set_512_support(struct mtk_wed_device *dev, bool enable)
+{
+	if (enable) {
+		wed_w32(dev, MTK_WED_TXDP_CTRL, MTK_WED_TXDP_DW9_OVERWR);
+		wed_w32(dev, MTK_WED_TXP_DW1,
+			FIELD_PREP(MTK_WED_WPDMA_WRITE_TXP, 0x0103));
+	} else {
+		wed_w32(dev, MTK_WED_TXP_DW1,
+			FIELD_PREP(MTK_WED_WPDMA_WRITE_TXP, 0x0100));
+		wed_clr(dev, MTK_WED_TXDP_CTRL, MTK_WED_TXDP_DW9_OVERWR);
+	}
+}
+
+static void
+mtk_wed_dma_disable(struct mtk_wed_device *dev)
+{
+	wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
+		MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
+		MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);
+
+	wed_clr(dev, MTK_WED_WDMA_GLO_CFG, MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
+
+	wed_clr(dev, MTK_WED_GLO_CFG,
+		MTK_WED_GLO_CFG_TX_DMA_EN |
+		MTK_WED_GLO_CFG_RX_DMA_EN);
+
+	wdma_m32(dev, MTK_WDMA_GLO_CFG,
+		 MTK_WDMA_GLO_CFG_TX_DMA_EN |
+		 MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |
+		 MTK_WDMA_GLO_CFG_RX_INFO2_PRERES, 0);
+
+	if (dev->hw->version == 1) {
+		regmap_write(dev->hw->mirror, dev->hw->index * 4, 0);
+		wdma_m32(dev, MTK_WDMA_GLO_CFG,
+			 MTK_WDMA_GLO_CFG_RX_INFO3_PRERES, 0);
+	} else {
+		wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
+			MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC |
+			MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC);
+
+		mtk_wed_set_512_support(dev, false);
+	}
+}
+
+static void
 mtk_wed_stop(struct mtk_wed_device *dev)
 {
-	regmap_write(dev->hw->mirror, dev->hw->index * 4, 0);
+	mtk_wed_dma_disable(dev);
 	mtk_wed_set_ext_int(dev, false);
 
 	wed_clr(dev, MTK_WED_CTRL,
@@ -252,21 +337,11 @@ mtk_wed_stop(struct mtk_wed_device *dev)
 	wdma_w32(dev, MTK_WDMA_INT_MASK, 0);
 	wdma_w32(dev, MTK_WDMA_INT_GRP2, 0);
 	wed_w32(dev, MTK_WED_WPDMA_INT_MASK, 0);
-
-	wed_clr(dev, MTK_WED_GLO_CFG,
-		MTK_WED_GLO_CFG_TX_DMA_EN |
-		MTK_WED_GLO_CFG_RX_DMA_EN);
-	wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
-		MTK_WED_WPDMA_GLO_CFG_TX_DRV_EN |
-		MTK_WED_WPDMA_GLO_CFG_RX_DRV_EN);
-	wed_clr(dev, MTK_WED_WDMA_GLO_CFG,
-		MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
 }
 
 static void
 mtk_wed_detach(struct mtk_wed_device *dev)
 {
-	struct device_node *wlan_node = dev->wlan.pci_dev->dev.of_node;
 	struct mtk_wed_hw *hw = dev->hw;
 
 	mutex_lock(&hw_lock);
@@ -281,9 +356,14 @@ mtk_wed_detach(struct mtk_wed_device *dev)
 	mtk_wed_free_buffer(dev);
 	mtk_wed_free_tx_rings(dev);
 
-	if (of_dma_is_coherent(wlan_node))
-		regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP,
-				   BIT(hw->index), BIT(hw->index));
+	if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) {
+		struct device_node *wlan_node;
+
+		wlan_node = dev->wlan.pci_dev->dev.of_node;
+		if (of_dma_is_coherent(wlan_node) && hw->hifsys)
+			regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP,
+					   BIT(hw->index), BIT(hw->index));
+	}
 
 	if (!hw_list[!hw->index]->wed_dev &&
 	    hw->eth->dma_dev != hw->eth->dev)
@@ -296,14 +376,76 @@ mtk_wed_detach(struct mtk_wed_device *dev)
 	mutex_unlock(&hw_lock);
 }
 
+#define PCIE_BASE_ADDR0		0x11280000
+static void
+mtk_wed_bus_init(struct mtk_wed_device *dev)
+{
+	switch (dev->wlan.bus_type) {
+	case MTK_WED_BUS_PCIE: {
+		struct device_node *np = dev->hw->eth->dev->of_node;
+		struct regmap *regs;
+
+		regs = syscon_regmap_lookup_by_phandle(np,
+						       "mediatek,wed-pcie");
+		if (IS_ERR(regs))
+			break;
+
+		regmap_update_bits(regs, 0, BIT(0), BIT(0));
+
+		wed_w32(dev, MTK_WED_PCIE_INT_CTRL,
+			FIELD_PREP(MTK_WED_PCIE_INT_CTRL_POLL_EN, 2));
+
+		/* pcie interrupt control: pola/source selection */
+		wed_set(dev, MTK_WED_PCIE_INT_CTRL,
+			MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA |
+			FIELD_PREP(MTK_WED_PCIE_INT_CTRL_SRC_SEL, 1));
+		wed_r32(dev, MTK_WED_PCIE_INT_CTRL);
+
+		wed_w32(dev, MTK_WED_PCIE_CFG_INTM, PCIE_BASE_ADDR0 | 0x180);
+		wed_w32(dev, MTK_WED_PCIE_CFG_BASE, PCIE_BASE_ADDR0 | 0x184);
+
+		/* pcie interrupt status trigger register */
+		wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, BIT(24));
+		wed_r32(dev, MTK_WED_PCIE_INT_TRIGGER);
+
+		/* pola setting */
+		wed_set(dev, MTK_WED_PCIE_INT_CTRL,
+			MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA);
+		break;
+	}
+	case MTK_WED_BUS_AXI:
+		wed_set(dev, MTK_WED_WPDMA_INT_CTRL,
+			MTK_WED_WPDMA_INT_CTRL_SIG_SRC |
+			FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_SRC_SEL, 0));
+		break;
+	default:
+		break;
+	}
+}
+
+static void
+mtk_wed_set_wpdma(struct mtk_wed_device *dev)
+{
+	if (dev->hw->version == 1) {
+		wed_w32(dev, MTK_WED_WPDMA_CFG_BASE,  dev->wlan.wpdma_phys);
+	} else {
+		mtk_wed_bus_init(dev);
+
+		wed_w32(dev, MTK_WED_WPDMA_CFG_BASE,  dev->wlan.wpdma_int);
+		wed_w32(dev, MTK_WED_WPDMA_CFG_INT_MASK,  dev->wlan.wpdma_mask);
+		wed_w32(dev, MTK_WED_WPDMA_CFG_TX,  dev->wlan.wpdma_tx);
+		wed_w32(dev, MTK_WED_WPDMA_CFG_TX_FREE,  dev->wlan.wpdma_txfree);
+	}
+}
+
 static void
 mtk_wed_hw_init_early(struct mtk_wed_device *dev)
 {
 	u32 mask, set;
-	u32 offset;
 
 	mtk_wed_stop(dev);
 	mtk_wed_reset(dev, MTK_WED_RESET_WED);
+	mtk_wed_set_wpdma(dev);
 
 	mask = MTK_WED_WDMA_GLO_CFG_BT_SIZE |
 	       MTK_WED_WDMA_GLO_CFG_DYNAMIC_DMAD_RECYCLE |
@@ -313,14 +455,33 @@ mtk_wed_hw_init_early(struct mtk_wed_device *dev)
 	      MTK_WED_WDMA_GLO_CFG_IDLE_DMAD_SUPPLY;
 	wed_m32(dev, MTK_WED_WDMA_GLO_CFG, mask, set);
 
-	wdma_set(dev, MTK_WDMA_GLO_CFG, MTK_WDMA_GLO_CFG_RX_INFO_PRERES);
+	if (dev->hw->version == 1) {
+		u32 offset = dev->hw->index ? 0x04000400 : 0;
 
-	offset = dev->hw->index ? 0x04000400 : 0;
-	wed_w32(dev, MTK_WED_WDMA_OFFSET0, 0x2a042a20 + offset);
-	wed_w32(dev, MTK_WED_WDMA_OFFSET1, 0x29002800 + offset);
+		wdma_set(dev, MTK_WDMA_GLO_CFG,
+			 MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |
+			 MTK_WDMA_GLO_CFG_RX_INFO2_PRERES |
+			 MTK_WDMA_GLO_CFG_RX_INFO3_PRERES);
 
-	wed_w32(dev, MTK_WED_PCIE_CFG_BASE, MTK_PCIE_BASE(dev->hw->index));
-	wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_phys);
+		wed_w32(dev, MTK_WED_WDMA_OFFSET0, 0x2a042a20 + offset);
+		wed_w32(dev, MTK_WED_WDMA_OFFSET1, 0x29002800 + offset);
+		wed_w32(dev, MTK_WED_PCIE_CFG_BASE,
+			MTK_PCIE_BASE(dev->hw->index));
+	} else {
+		wed_w32(dev, MTK_WED_WDMA_CFG_BASE, dev->hw->wdma_phy);
+		wed_set(dev, MTK_WED_CTRL, MTK_WED_CTRL_ETH_DMAD_FMT);
+		wed_w32(dev, MTK_WED_WDMA_OFFSET0,
+			FIELD_PREP(MTK_WED_WDMA_OFST0_GLO_INTS,
+				   MTK_WDMA_INT_STATUS) |
+			FIELD_PREP(MTK_WED_WDMA_OFST0_GLO_CFG,
+				   MTK_WDMA_GLO_CFG));
+
+		wed_w32(dev, MTK_WED_WDMA_OFFSET1,
+			FIELD_PREP(MTK_WED_WDMA_OFST1_TX_CTRL,
+				   MTK_WDMA_RING_TX(0)) |
+			FIELD_PREP(MTK_WED_WDMA_OFST1_RX_CTRL,
+				   MTK_WDMA_RING_RX(0)));
+	}
 }
 
 static void
@@ -340,37 +501,65 @@ mtk_wed_hw_init(struct mtk_wed_device *dev)
 
 	wed_w32(dev, MTK_WED_TX_BM_BASE, dev->buf_ring.desc_phys);
 
-	wed_w32(dev, MTK_WED_TX_BM_TKID,
-		FIELD_PREP(MTK_WED_TX_BM_TKID_START,
-			   dev->wlan.token_start) |
-		FIELD_PREP(MTK_WED_TX_BM_TKID_END,
-			   dev->wlan.token_start + dev->wlan.nbuf - 1));
-
 	wed_w32(dev, MTK_WED_TX_BM_BUF_LEN, MTK_WED_PKT_SIZE);
 
-	wed_w32(dev, MTK_WED_TX_BM_DYN_THR,
-		FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO, 1) |
-		MTK_WED_TX_BM_DYN_THR_HI);
+	if (dev->hw->version == 1) {
+		wed_w32(dev, MTK_WED_TX_BM_TKID,
+			FIELD_PREP(MTK_WED_TX_BM_TKID_START,
+				   dev->wlan.token_start) |
+			FIELD_PREP(MTK_WED_TX_BM_TKID_END,
+				   dev->wlan.token_start +
+				   dev->wlan.nbuf - 1));
+		wed_w32(dev, MTK_WED_TX_BM_DYN_THR,
+			FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO, 1) |
+			MTK_WED_TX_BM_DYN_THR_HI);
+	} else {
+		wed_w32(dev, MTK_WED_TX_BM_TKID_V2,
+			FIELD_PREP(MTK_WED_TX_BM_TKID_START,
+				   dev->wlan.token_start) |
+			FIELD_PREP(MTK_WED_TX_BM_TKID_END,
+				   dev->wlan.token_start +
+				   dev->wlan.nbuf - 1));
+		wed_w32(dev, MTK_WED_TX_BM_DYN_THR,
+			FIELD_PREP(MTK_WED_TX_BM_DYN_THR_LO_V2, 0) |
+			MTK_WED_TX_BM_DYN_THR_HI_V2);
+		wed_w32(dev, MTK_WED_TX_TKID_CTRL,
+			MTK_WED_TX_TKID_CTRL_PAUSE |
+			FIELD_PREP(MTK_WED_TX_TKID_CTRL_VLD_GRP_NUM,
+				   dev->buf_ring.size / 128) |
+			FIELD_PREP(MTK_WED_TX_TKID_CTRL_RSV_GRP_NUM,
+				   dev->buf_ring.size / 128));
+		wed_w32(dev, MTK_WED_TX_TKID_DYN_THR,
+			FIELD_PREP(MTK_WED_TX_TKID_DYN_THR_LO, 0) |
+			MTK_WED_TX_TKID_DYN_THR_HI);
+	}
 
 	mtk_wed_reset(dev, MTK_WED_RESET_TX_BM);
 
-	wed_set(dev, MTK_WED_CTRL,
-		MTK_WED_CTRL_WED_TX_BM_EN |
-		MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
+	if (dev->hw->version == 1)
+		wed_set(dev, MTK_WED_CTRL,
+			MTK_WED_CTRL_WED_TX_BM_EN |
+			MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
+	else
+		wed_clr(dev, MTK_WED_TX_TKID_CTRL, MTK_WED_TX_TKID_CTRL_PAUSE);
 
 	wed_clr(dev, MTK_WED_TX_BM_CTRL, MTK_WED_TX_BM_CTRL_PAUSE);
 }
 
 static void
-mtk_wed_ring_reset(struct mtk_wdma_desc *desc, int size)
+mtk_wed_ring_reset(struct mtk_wed_ring *ring, int size)
 {
+	void *head = (void *)ring->desc;
 	int i;
 
 	for (i = 0; i < size; i++) {
-		desc[i].buf0 = 0;
-		desc[i].ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE);
-		desc[i].buf1 = 0;
-		desc[i].info = 0;
+		struct mtk_wdma_desc *desc;
+
+		desc = (struct mtk_wdma_desc *)(head + i * ring->desc_size);
+		desc->buf0 = 0;
+		desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE);
+		desc->buf1 = 0;
+		desc->info = 0;
 	}
 }
 
@@ -421,12 +610,10 @@ mtk_wed_reset_dma(struct mtk_wed_device *dev)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) {
-		struct mtk_wdma_desc *desc = dev->tx_ring[i].desc;
-
-		if (!desc)
+		if (!dev->tx_ring[i].desc)
 			continue;
 
-		mtk_wed_ring_reset(desc, MTK_WED_TX_RING_SIZE);
+		mtk_wed_ring_reset(&dev->tx_ring[i], MTK_WED_TX_RING_SIZE);
 	}
 
 	if (mtk_wed_poll_busy(dev))
@@ -483,16 +670,16 @@ mtk_wed_reset_dma(struct mtk_wed_device *dev)
 
 static int
 mtk_wed_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring,
-		   int size)
+		   int size, u32 desc_size)
 {
-	ring->desc = dma_alloc_coherent(dev->hw->dev,
-					size * sizeof(*ring->desc),
+	ring->desc = dma_alloc_coherent(dev->hw->dev, size * desc_size,
 					&ring->desc_phys, GFP_KERNEL);
 	if (!ring->desc)
 		return -ENOMEM;
 
+	ring->desc_size = desc_size;
 	ring->size = size;
-	mtk_wed_ring_reset(ring->desc, size);
+	mtk_wed_ring_reset(ring, size);
 
 	return 0;
 }
@@ -500,9 +687,10 @@ mtk_wed_ring_alloc(struct mtk_wed_device *dev, struct mtk_wed_ring *ring,
 static int
 mtk_wed_wdma_ring_setup(struct mtk_wed_device *dev, int idx, int size)
 {
+	u32 desc_size = sizeof(struct mtk_wdma_desc) * dev->hw->version;
 	struct mtk_wed_ring *wdma = &dev->tx_wdma[idx];
 
-	if (mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE))
+	if (mtk_wed_ring_alloc(dev, wdma, MTK_WED_WDMA_RING_SIZE, desc_size))
 		return -ENOMEM;
 
 	wdma_w32(dev, MTK_WDMA_RING_RX(idx) + MTK_WED_RING_OFS_BASE,
@@ -520,43 +708,63 @@ mtk_wed_wdma_ring_setup(struct mtk_wed_device *dev, int idx, int size)
 }
 
 static void
-mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask)
+mtk_wed_configure_irq(struct mtk_wed_device *dev, u32 irq_mask)
 {
-	u32 wdma_mask;
-	u32 val;
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++)
-		if (!dev->tx_wdma[i].desc)
-			mtk_wed_wdma_ring_setup(dev, i, 16);
-
-	wdma_mask = FIELD_PREP(MTK_WDMA_INT_MASK_RX_DONE, GENMASK(1, 0));
-
-	mtk_wed_hw_init(dev);
+	u32 wdma_mask = FIELD_PREP(MTK_WDMA_INT_MASK_RX_DONE, GENMASK(1, 0));
 
+	/* wed control cr set */
 	wed_set(dev, MTK_WED_CTRL,
 		MTK_WED_CTRL_WDMA_INT_AGENT_EN |
 		MTK_WED_CTRL_WPDMA_INT_AGENT_EN |
 		MTK_WED_CTRL_WED_TX_BM_EN |
 		MTK_WED_CTRL_WED_TX_FREE_AGENT_EN);
 
-	wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER, MTK_WED_PCIE_INT_TRIGGER_STATUS);
+	if (dev->hw->version == 1) {
+		wed_w32(dev, MTK_WED_PCIE_INT_TRIGGER,
+			MTK_WED_PCIE_INT_TRIGGER_STATUS);
 
-	wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER,
-		MTK_WED_WPDMA_INT_TRIGGER_RX_DONE |
-		MTK_WED_WPDMA_INT_TRIGGER_TX_DONE);
+		wed_w32(dev, MTK_WED_WPDMA_INT_TRIGGER,
+			MTK_WED_WPDMA_INT_TRIGGER_RX_DONE |
+			MTK_WED_WPDMA_INT_TRIGGER_TX_DONE);
 
-	wed_set(dev, MTK_WED_WPDMA_INT_CTRL,
-		MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV);
+		wed_clr(dev, MTK_WED_WDMA_INT_CTRL, wdma_mask);
+	} else {
+		/* initail tx interrupt trigger */
+		wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_TX,
+			MTK_WED_WPDMA_INT_CTRL_TX0_DONE_EN |
+			MTK_WED_WPDMA_INT_CTRL_TX0_DONE_CLR |
+			MTK_WED_WPDMA_INT_CTRL_TX1_DONE_EN |
+			MTK_WED_WPDMA_INT_CTRL_TX1_DONE_CLR |
+			FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX0_DONE_TRIG,
+				   dev->wlan.tx_tbit[0]) |
+			FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX1_DONE_TRIG,
+				   dev->wlan.tx_tbit[1]));
+
+		/* initail txfree interrupt trigger */
+		wed_w32(dev, MTK_WED_WPDMA_INT_CTRL_TX_FREE,
+			MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_EN |
+			MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_CLR |
+			FIELD_PREP(MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_TRIG,
+				   dev->wlan.txfree_tbit));
+
+		wed_w32(dev, MTK_WED_WDMA_INT_CLR, wdma_mask);
+		wed_set(dev, MTK_WED_WDMA_INT_CTRL,
+			FIELD_PREP(MTK_WED_WDMA_INT_CTRL_POLL_SRC_SEL,
+				   dev->wdma_idx));
+	}
 
 	wed_w32(dev, MTK_WED_WDMA_INT_TRIGGER, wdma_mask);
-	wed_clr(dev, MTK_WED_WDMA_INT_CTRL, wdma_mask);
 
 	wdma_w32(dev, MTK_WDMA_INT_MASK, wdma_mask);
 	wdma_w32(dev, MTK_WDMA_INT_GRP2, wdma_mask);
-
 	wed_w32(dev, MTK_WED_WPDMA_INT_MASK, irq_mask);
 	wed_w32(dev, MTK_WED_INT_MASK, irq_mask);
+}
+
+static void
+mtk_wed_dma_enable(struct mtk_wed_device *dev)
+{
+	wed_set(dev, MTK_WED_WPDMA_INT_CTRL, MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV);
 
 	wed_set(dev, MTK_WED_GLO_CFG,
 		MTK_WED_GLO_CFG_TX_DMA_EN |
@@ -567,16 +775,54 @@ mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask)
 	wed_set(dev, MTK_WED_WDMA_GLO_CFG,
 		MTK_WED_WDMA_GLO_CFG_RX_DRV_EN);
 
+	wdma_set(dev, MTK_WDMA_GLO_CFG,
+		 MTK_WDMA_GLO_CFG_TX_DMA_EN |
+		 MTK_WDMA_GLO_CFG_RX_INFO1_PRERES |
+		 MTK_WDMA_GLO_CFG_RX_INFO2_PRERES);
+
+	if (dev->hw->version == 1) {
+		wdma_set(dev, MTK_WDMA_GLO_CFG,
+			 MTK_WDMA_GLO_CFG_RX_INFO3_PRERES);
+	} else {
+		wed_set(dev, MTK_WED_WPDMA_CTRL,
+			MTK_WED_WPDMA_CTRL_SDL1_FIXED);
+
+		wed_set(dev, MTK_WED_WPDMA_GLO_CFG,
+			MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC |
+			MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC);
+
+		wed_clr(dev, MTK_WED_WPDMA_GLO_CFG,
+			MTK_WED_WPDMA_GLO_CFG_TX_TKID_KEEP |
+			MTK_WED_WPDMA_GLO_CFG_TX_DMAD_DW3_PREV);
+	}
+}
+
+static void
+mtk_wed_start(struct mtk_wed_device *dev, u32 irq_mask)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++)
+		if (!dev->tx_wdma[i].desc)
+			mtk_wed_wdma_ring_setup(dev, i, 16);
+
+	mtk_wed_hw_init(dev);
+	mtk_wed_configure_irq(dev, irq_mask);
+
 	mtk_wed_set_ext_int(dev, true);
-	val = dev->wlan.wpdma_phys |
-	      MTK_PCIE_MIRROR_MAP_EN |
-	      FIELD_PREP(MTK_PCIE_MIRROR_MAP_WED_ID, dev->hw->index);
 
-	if (dev->hw->index)
-		val |= BIT(1);
-	val |= BIT(0);
-	regmap_write(dev->hw->mirror, dev->hw->index * 4, val);
+	if (dev->hw->version == 1) {
+		u32 val = dev->wlan.wpdma_phys | MTK_PCIE_MIRROR_MAP_EN |
+			  FIELD_PREP(MTK_PCIE_MIRROR_MAP_WED_ID,
+				     dev->hw->index);
+
+		val |= BIT(0) | (BIT(1) * !!dev->hw->index);
+		regmap_write(dev->hw->mirror, dev->hw->index * 4, val);
+	} else {
+		mtk_wed_set_512_support(dev, true);
+	}
 
+	mtk_wed_dma_enable(dev);
 	dev->running = true;
 }
 
@@ -585,12 +831,14 @@ mtk_wed_attach(struct mtk_wed_device *dev)
 	__releases(RCU)
 {
 	struct mtk_wed_hw *hw;
+	struct device *device;
 	int ret = 0;
 
 	RCU_LOCKDEP_WARN(!rcu_read_lock_held(),
 			 "mtk_wed_attach without holding the RCU read lock");
 
-	if (pci_domain_nr(dev->wlan.pci_dev->bus) > 1 ||
+	if ((dev->wlan.bus_type == MTK_WED_BUS_PCIE &&
+	     pci_domain_nr(dev->wlan.pci_dev->bus) > 1) ||
 	    !try_module_get(THIS_MODULE))
 		ret = -ENODEV;
 
@@ -608,7 +856,11 @@ mtk_wed_attach(struct mtk_wed_device *dev)
 		goto out;
 	}
 
-	dev_info(&dev->wlan.pci_dev->dev, "attaching wed device %d\n", hw->index);
+	device = dev->wlan.bus_type == MTK_WED_BUS_PCIE
+		? &dev->wlan.pci_dev->dev
+		: &dev->wlan.platform_dev->dev;
+	dev_info(device, "attaching wed device %d version %d\n",
+		 hw->index, hw->version);
 
 	dev->hw = hw;
 	dev->dev = hw->dev;
@@ -626,7 +878,9 @@ mtk_wed_attach(struct mtk_wed_device *dev)
 	}
 
 	mtk_wed_hw_init_early(dev);
-	regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, BIT(hw->index), 0);
+	if (hw->hifsys)
+		regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP,
+				   BIT(hw->index), 0);
 
 out:
 	mutex_unlock(&hw_lock);
@@ -653,7 +907,8 @@ mtk_wed_tx_ring_setup(struct mtk_wed_device *dev, int idx, void __iomem *regs)
 
 	BUG_ON(idx >= ARRAY_SIZE(dev->tx_ring));
 
-	if (mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE))
+	if (mtk_wed_ring_alloc(dev, ring, MTK_WED_TX_RING_SIZE,
+			       sizeof(*ring->desc)))
 		return -ENOMEM;
 
 	if (mtk_wed_wdma_ring_setup(dev, idx, MTK_WED_WDMA_RING_SIZE))
@@ -680,21 +935,21 @@ static int
 mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs)
 {
 	struct mtk_wed_ring *ring = &dev->txfree_ring;
-	int i;
+	int i, index = dev->hw->version == 1;
 
 	/*
 	 * For txfree event handling, the same DMA ring is shared between WED
 	 * and WLAN. The WLAN driver accesses the ring index registers through
 	 * WED
 	 */
-	ring->reg_base = MTK_WED_RING_RX(1);
+	ring->reg_base = MTK_WED_RING_RX(index);
 	ring->wpdma = regs;
 
 	for (i = 0; i < 12; i += 4) {
 		u32 val = readl(regs + i);
 
-		wed_w32(dev, MTK_WED_RING_RX(1) + i, val);
-		wed_w32(dev, MTK_WED_WPDMA_RING_RX(1) + i, val);
+		wed_w32(dev, MTK_WED_RING_RX(index) + i, val);
+		wed_w32(dev, MTK_WED_WPDMA_RING_RX(index) + i, val);
 	}
 
 	return 0;
@@ -703,11 +958,19 @@ mtk_wed_txfree_ring_setup(struct mtk_wed_device *dev, void __iomem *regs)
 static u32
 mtk_wed_irq_get(struct mtk_wed_device *dev, u32 mask)
 {
-	u32 val;
+	u32 val, ext_mask = MTK_WED_EXT_INT_STATUS_ERROR_MASK;
+
+	if (dev->hw->version == 1)
+		ext_mask |= MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR;
+	else
+		ext_mask |= MTK_WED_EXT_INT_STATUS_RX_FBUF_LO_TH |
+			    MTK_WED_EXT_INT_STATUS_RX_FBUF_HI_TH |
+			    MTK_WED_EXT_INT_STATUS_RX_DRV_COHERENT |
+			    MTK_WED_EXT_INT_STATUS_TX_DMA_W_RESP_ERR;
 
 	val = wed_r32(dev, MTK_WED_EXT_INT_STATUS);
 	wed_w32(dev, MTK_WED_EXT_INT_STATUS, val);
-	val &= MTK_WED_EXT_INT_STATUS_ERROR_MASK;
+	val &= ext_mask;
 	if (!dev->hw->num_flows)
 		val &= ~MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD;
 	if (val && net_ratelimit())
@@ -782,7 +1045,8 @@ out:
 }
 
 void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,
-		    void __iomem *wdma, int index)
+		    void __iomem *wdma, phys_addr_t wdma_phy,
+		    int index)
 {
 	static const struct mtk_wed_ops wed_ops = {
 		.attach = mtk_wed_attach,
@@ -829,26 +1093,33 @@ void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,
 	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
 	if (!hw)
 		goto unlock;
+
 	hw->node = np;
 	hw->regs = regs;
 	hw->eth = eth;
 	hw->dev = &pdev->dev;
+	hw->wdma_phy = wdma_phy;
 	hw->wdma = wdma;
 	hw->index = index;
 	hw->irq = irq;
-	hw->mirror = syscon_regmap_lookup_by_phandle(eth_np,
-						     "mediatek,pcie-mirror");
-	hw->hifsys = syscon_regmap_lookup_by_phandle(eth_np,
-						     "mediatek,hifsys");
-	if (IS_ERR(hw->mirror) || IS_ERR(hw->hifsys)) {
-		kfree(hw);
-		goto unlock;
-	}
+	hw->version = MTK_HAS_CAPS(eth->soc->caps, MTK_NETSYS_V2) ? 2 : 1;
+
+	if (hw->version == 1) {
+		hw->mirror = syscon_regmap_lookup_by_phandle(eth_np,
+				"mediatek,pcie-mirror");
+		hw->hifsys = syscon_regmap_lookup_by_phandle(eth_np,
+				"mediatek,hifsys");
+		if (IS_ERR(hw->mirror) || IS_ERR(hw->hifsys)) {
+			kfree(hw);
+			goto unlock;
+		}
 
-	if (!index) {
-		regmap_write(hw->mirror, 0, 0);
-		regmap_write(hw->mirror, 4, 0);
+		if (!index) {
+			regmap_write(hw->mirror, 0, 0);
+			regmap_write(hw->mirror, 4, 0);
+		}
 	}
+
 	mtk_wed_hw_add_debugfs(hw);
 
 	hw_list[index] = hw;
diff --git a/drivers/net/ethernet/mediatek/mtk_wed.h b/drivers/net/ethernet/mediatek/mtk_wed.h
index 981ec613f4b0..ae420ca01a48 100644
--- a/drivers/net/ethernet/mediatek/mtk_wed.h
+++ b/drivers/net/ethernet/mediatek/mtk_wed.h
@@ -18,11 +18,13 @@ struct mtk_wed_hw {
 	struct regmap *hifsys;
 	struct device *dev;
 	void __iomem *wdma;
+	phys_addr_t wdma_phy;
 	struct regmap *mirror;
 	struct dentry *debugfs_dir;
 	struct mtk_wed_device *wed_dev;
 	u32 debugfs_reg;
 	u32 num_flows;
+	u8 version;
 	char dirname[5];
 	int irq;
 	int index;
@@ -101,14 +103,16 @@ wpdma_txfree_w32(struct mtk_wed_device *dev, u32 reg, u32 val)
 }
 
 void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,
-		    void __iomem *wdma, int index);
+		    void __iomem *wdma, phys_addr_t wdma_phy,
+		    int index);
 void mtk_wed_exit(void);
 int mtk_wed_flow_add(int index);
 void mtk_wed_flow_remove(int index);
 #else
 static inline void
 mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth,
-	       void __iomem *wdma, int index)
+	       void __iomem *wdma, phys_addr_t wdma_phy,
+	       int index)
 {
 }
 static inline void
diff --git a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c
index a81d3fd1a439..f420f187e837 100644
--- a/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c
+++ b/drivers/net/ethernet/mediatek/mtk_wed_debugfs.c
@@ -116,6 +116,9 @@ wed_txinfo_show(struct seq_file *s, void *data)
 		DUMP_WDMA(WDMA_GLO_CFG),
 		DUMP_WDMA_RING(WDMA_RING_RX(0)),
 		DUMP_WDMA_RING(WDMA_RING_RX(1)),
+
+		DUMP_STR("TX FREE"),
+		DUMP_WED(WED_RX_MIB(0)),
 	};
 	struct mtk_wed_hw *hw = s->private;
 	struct mtk_wed_device *dev = hw->wed_dev;
diff --git a/drivers/net/ethernet/mediatek/mtk_wed_regs.h b/drivers/net/ethernet/mediatek/mtk_wed_regs.h
index 0a0465ea58b4..e270fb336143 100644
--- a/drivers/net/ethernet/mediatek/mtk_wed_regs.h
+++ b/drivers/net/ethernet/mediatek/mtk_wed_regs.h
@@ -5,6 +5,7 @@
 #define __MTK_WED_REGS_H
 
 #define MTK_WDMA_DESC_CTRL_LEN1			GENMASK(14, 0)
+#define MTK_WDMA_DESC_CTRL_LEN1_V2		GENMASK(13, 0)
 #define MTK_WDMA_DESC_CTRL_LAST_SEG1		BIT(15)
 #define MTK_WDMA_DESC_CTRL_BURST		BIT(16)
 #define MTK_WDMA_DESC_CTRL_LEN0			GENMASK(29, 16)
@@ -41,6 +42,7 @@ struct mtk_wdma_desc {
 #define MTK_WED_CTRL_RESERVE_EN				BIT(12)
 #define MTK_WED_CTRL_RESERVE_BUSY			BIT(13)
 #define MTK_WED_CTRL_FINAL_DIDX_READ			BIT(24)
+#define MTK_WED_CTRL_ETH_DMAD_FMT			BIT(25)
 #define MTK_WED_CTRL_MIB_READ_CLEAR			BIT(28)
 
 #define MTK_WED_EXT_INT_STATUS				0x020
@@ -57,7 +59,8 @@ struct mtk_wdma_desc {
 #define MTK_WED_EXT_INT_STATUS_RX_DRV_INIT_WDMA_EN	BIT(19)
 #define MTK_WED_EXT_INT_STATUS_RX_DRV_BM_DMAD_COHERENT	BIT(20)
 #define MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR	BIT(21)
-#define MTK_WED_EXT_INT_STATUS_TX_DRV_W_RESP_ERR	BIT(22)
+#define MTK_WED_EXT_INT_STATUS_TX_DMA_R_RESP_ERR	BIT(22)
+#define MTK_WED_EXT_INT_STATUS_TX_DMA_W_RESP_ERR	BIT(23)
 #define MTK_WED_EXT_INT_STATUS_RX_DRV_DMA_RECYCLE	BIT(24)
 #define MTK_WED_EXT_INT_STATUS_ERROR_MASK		(MTK_WED_EXT_INT_STATUS_TF_LEN_ERR | \
 							 MTK_WED_EXT_INT_STATUS_TKID_WO_PYLD | \
@@ -65,8 +68,7 @@ struct mtk_wdma_desc {
 							 MTK_WED_EXT_INT_STATUS_RX_DRV_R_RESP_ERR | \
 							 MTK_WED_EXT_INT_STATUS_RX_DRV_W_RESP_ERR | \
 							 MTK_WED_EXT_INT_STATUS_RX_DRV_INIT_WDMA_EN | \
-							 MTK_WED_EXT_INT_STATUS_TX_DRV_R_RESP_ERR | \
-							 MTK_WED_EXT_INT_STATUS_TX_DRV_W_RESP_ERR)
+							 MTK_WED_EXT_INT_STATUS_TX_DMA_R_RESP_ERR)
 
 #define MTK_WED_EXT_INT_MASK				0x028
 
@@ -81,6 +83,7 @@ struct mtk_wdma_desc {
 #define MTK_WED_TX_BM_BASE				0x084
 
 #define MTK_WED_TX_BM_TKID				0x088
+#define MTK_WED_TX_BM_TKID_V2				0x0c8
 #define MTK_WED_TX_BM_TKID_START			GENMASK(15, 0)
 #define MTK_WED_TX_BM_TKID_END				GENMASK(31, 16)
 
@@ -94,7 +97,25 @@ struct mtk_wdma_desc {
 
 #define MTK_WED_TX_BM_DYN_THR				0x0a0
 #define MTK_WED_TX_BM_DYN_THR_LO			GENMASK(6, 0)
+#define MTK_WED_TX_BM_DYN_THR_LO_V2			GENMASK(8, 0)
 #define MTK_WED_TX_BM_DYN_THR_HI			GENMASK(22, 16)
+#define MTK_WED_TX_BM_DYN_THR_HI_V2			GENMASK(24, 16)
+
+#define MTK_WED_TX_TKID_CTRL				0x0c0
+#define MTK_WED_TX_TKID_CTRL_VLD_GRP_NUM		GENMASK(6, 0)
+#define MTK_WED_TX_TKID_CTRL_RSV_GRP_NUM		GENMASK(22, 16)
+#define MTK_WED_TX_TKID_CTRL_PAUSE			BIT(28)
+
+#define MTK_WED_TX_TKID_DYN_THR				0x0e0
+#define MTK_WED_TX_TKID_DYN_THR_LO			GENMASK(6, 0)
+#define MTK_WED_TX_TKID_DYN_THR_HI			GENMASK(22, 16)
+
+#define MTK_WED_TXP_DW0					0x120
+#define MTK_WED_TXP_DW1					0x124
+#define MTK_WED_WPDMA_WRITE_TXP				GENMASK(31, 16)
+#define MTK_WED_TXDP_CTRL				0x130
+#define MTK_WED_TXDP_DW9_OVERWR				BIT(9)
+#define MTK_WED_RX_BM_TKID_MIB				0x1cc
 
 #define MTK_WED_INT_STATUS				0x200
 #define MTK_WED_INT_MASK				0x204
@@ -125,6 +146,7 @@ struct mtk_wdma_desc {
 #define MTK_WED_RESET_IDX_RX				GENMASK(17, 16)
 
 #define MTK_WED_TX_MIB(_n)				(0x2a0 + (_n) * 4)
+#define MTK_WED_RX_MIB(_n)				(0x2e0 + (_n) * 4)
 
 #define MTK_WED_RING_TX(_n)				(0x300 + (_n) * 0x10)
 
@@ -155,21 +177,64 @@ struct mtk_wdma_desc {
 #define MTK_WED_WPDMA_GLO_CFG_BYTE_SWAP			BIT(29)
 #define MTK_WED_WPDMA_GLO_CFG_RX_2B_OFFSET		BIT(31)
 
+/* CONFIG_MEDIATEK_NETSYS_V2 */
+#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_PKT_PROC	BIT(4)
+#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_R1_PKT_PROC	BIT(5)
+#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_R0_CRX_SYNC	BIT(6)
+#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_R1_CRX_SYNC	BIT(7)
+#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_EVENT_PKT_FMT_VER	GENMASK(18, 16)
+#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_UNSUPPORT_FMT	BIT(19)
+#define MTK_WED_WPDMA_GLO_CFG_RX_DRV_UEVENT_PKT_FMT_CHK BIT(20)
+#define MTK_WED_WPDMA_GLO_CFG_RX_DDONE2_WR		BIT(21)
+#define MTK_WED_WPDMA_GLO_CFG_TX_TKID_KEEP		BIT(24)
+#define MTK_WED_WPDMA_GLO_CFG_TX_DMAD_DW3_PREV		BIT(28)
+
 #define MTK_WED_WPDMA_RESET_IDX				0x50c
 #define MTK_WED_WPDMA_RESET_IDX_TX			GENMASK(3, 0)
 #define MTK_WED_WPDMA_RESET_IDX_RX			GENMASK(17, 16)
 
+#define MTK_WED_WPDMA_CTRL				0x518
+#define MTK_WED_WPDMA_CTRL_SDL1_FIXED			BIT(31)
+
 #define MTK_WED_WPDMA_INT_CTRL				0x520
 #define MTK_WED_WPDMA_INT_CTRL_SUBRT_ADV		BIT(21)
+#define MTK_WED_WPDMA_INT_CTRL_SIG_SRC			BIT(22)
+#define MTK_WED_WPDMA_INT_CTRL_SRC_SEL			GENMASK(17, 16)
 
 #define MTK_WED_WPDMA_INT_MASK				0x524
 
+#define MTK_WED_WPDMA_INT_CTRL_TX			0x530
+#define MTK_WED_WPDMA_INT_CTRL_TX0_DONE_EN		BIT(0)
+#define MTK_WED_WPDMA_INT_CTRL_TX0_DONE_CLR		BIT(1)
+#define MTK_WED_WPDMA_INT_CTRL_TX0_DONE_TRIG		GENMASK(6, 2)
+#define MTK_WED_WPDMA_INT_CTRL_TX1_DONE_EN		BIT(8)
+#define MTK_WED_WPDMA_INT_CTRL_TX1_DONE_CLR		BIT(9)
+#define MTK_WED_WPDMA_INT_CTRL_TX1_DONE_TRIG		GENMASK(14, 10)
+
+#define MTK_WED_WPDMA_INT_CTRL_RX			0x534
+
+#define MTK_WED_WPDMA_INT_CTRL_TX_FREE			0x538
+#define MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_EN		BIT(0)
+#define MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_CLR		BIT(1)
+#define MTK_WED_WPDMA_INT_CTRL_TX_FREE_DONE_TRIG	GENMASK(6, 2)
+
 #define MTK_WED_PCIE_CFG_BASE				0x560
 
+#define MTK_WED_PCIE_CFG_BASE				0x560
+#define MTK_WED_PCIE_CFG_INTM				0x564
+#define MTK_WED_PCIE_CFG_MSIS				0x568
 #define MTK_WED_PCIE_INT_TRIGGER			0x570
 #define MTK_WED_PCIE_INT_TRIGGER_STATUS			BIT(16)
 
+#define MTK_WED_PCIE_INT_CTRL				0x57c
+#define MTK_WED_PCIE_INT_CTRL_MSK_EN_POLA		BIT(20)
+#define MTK_WED_PCIE_INT_CTRL_SRC_SEL			GENMASK(17, 16)
+#define MTK_WED_PCIE_INT_CTRL_POLL_EN			GENMASK(13, 12)
+
 #define MTK_WED_WPDMA_CFG_BASE				0x580
+#define MTK_WED_WPDMA_CFG_INT_MASK			0x584
+#define MTK_WED_WPDMA_CFG_TX				0x588
+#define MTK_WED_WPDMA_CFG_TX_FREE			0x58c
 
 #define MTK_WED_WPDMA_TX_MIB(_n)			(0x5a0 + (_n) * 4)
 #define MTK_WED_WPDMA_TX_COHERENT_MIB(_n)		(0x5d0 + (_n) * 4)
@@ -203,15 +268,24 @@ struct mtk_wdma_desc {
 #define MTK_WED_WDMA_RESET_IDX_RX			GENMASK(17, 16)
 #define MTK_WED_WDMA_RESET_IDX_DRV			GENMASK(25, 24)
 
+#define MTK_WED_WDMA_INT_CLR				0xa24
+#define MTK_WED_WDMA_INT_CLR_RX_DONE			GENMASK(17, 16)
+
 #define MTK_WED_WDMA_INT_TRIGGER			0xa28
 #define MTK_WED_WDMA_INT_TRIGGER_RX_DONE		GENMASK(17, 16)
 
 #define MTK_WED_WDMA_INT_CTRL				0xa2c
 #define MTK_WED_WDMA_INT_CTRL_POLL_SRC_SEL		GENMASK(17, 16)
 
+#define MTK_WED_WDMA_CFG_BASE				0xaa0
 #define MTK_WED_WDMA_OFFSET0				0xaa4
 #define MTK_WED_WDMA_OFFSET1				0xaa8
 
+#define MTK_WED_WDMA_OFST0_GLO_INTS			GENMASK(15, 0)
+#define MTK_WED_WDMA_OFST0_GLO_CFG			GENMASK(31, 16)
+#define MTK_WED_WDMA_OFST1_TX_CTRL			GENMASK(15, 0)
+#define MTK_WED_WDMA_OFST1_RX_CTRL			GENMASK(31, 16)
+
 #define MTK_WED_WDMA_RX_MIB(_n)				(0xae0 + (_n) * 4)
 #define MTK_WED_WDMA_RX_RECYCLE_MIB(_n)			(0xae8 + (_n) * 4)
 #define MTK_WED_WDMA_RX_PROCESSED_MIB(_n)		(0xaf0 + (_n) * 4)
@@ -221,15 +295,22 @@ struct mtk_wdma_desc {
 #define MTK_WED_RING_OFS_CPU_IDX			0x08
 #define MTK_WED_RING_OFS_DMA_IDX			0x0c
 
+#define MTK_WDMA_RING_TX(_n)				(0x000 + (_n) * 0x10)
 #define MTK_WDMA_RING_RX(_n)				(0x100 + (_n) * 0x10)
 
 #define MTK_WDMA_GLO_CFG				0x204
-#define MTK_WDMA_GLO_CFG_RX_INFO_PRERES			GENMASK(28, 26)
+#define MTK_WDMA_GLO_CFG_TX_DMA_EN			BIT(0)
+#define MTK_WDMA_GLO_CFG_RX_DMA_EN			BIT(2)
+#define MTK_WDMA_GLO_CFG_RX_INFO3_PRERES		BIT(26)
+#define MTK_WDMA_GLO_CFG_RX_INFO2_PRERES		BIT(27)
+#define MTK_WDMA_GLO_CFG_RX_INFO1_PRERES		BIT(28)
 
 #define MTK_WDMA_RESET_IDX				0x208
 #define MTK_WDMA_RESET_IDX_TX				GENMASK(3, 0)
 #define MTK_WDMA_RESET_IDX_RX				GENMASK(17, 16)
 
+#define MTK_WDMA_INT_STATUS				0x220
+
 #define MTK_WDMA_INT_MASK				0x228
 #define MTK_WDMA_INT_MASK_TX_DONE			GENMASK(3, 0)
 #define MTK_WDMA_INT_MASK_RX_DONE			GENMASK(17, 16)
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_cq.c b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
index 6affbd241264..1184ac5751e1 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_cq.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_cq.c
@@ -152,7 +152,7 @@ int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq,
 		break;
 	case RX:
 		cq->mcq.comp = mlx4_en_rx_irq;
-		netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_rx_cq, 64);
+		netif_napi_add(cq->dev, &cq->napi, mlx4_en_poll_rx_cq);
 		napi_enable(&cq->napi);
 		break;
 	case TX_XDP:
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
index 6400a827173c..7d45f1d55f79 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c
@@ -89,15 +89,15 @@ mlx4_en_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo)
 	struct mlx4_en_priv *priv = netdev_priv(dev);
 	struct mlx4_en_dev *mdev = priv->mdev;
 
-	strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, DRV_VERSION,
+	strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, DRV_VERSION,
 		sizeof(drvinfo->version));
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
 		"%d.%d.%d",
 		(u16) (mdev->dev->caps.fw_ver >> 32),
 		(u16) ((mdev->dev->caps.fw_ver >> 16) & 0xffff),
 		(u16) (mdev->dev->caps.fw_ver & 0xffff));
-	strlcpy(drvinfo->bus_info, pci_name(mdev->dev->persist->pdev),
+	strscpy(drvinfo->bus_info, pci_name(mdev->dev->persist->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/fw.c b/drivers/net/ethernet/mellanox/mlx4/fw.c
index dcb9eb1899ce..fe48d20d6118 100644
--- a/drivers/net/ethernet/mellanox/mlx4/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx4/fw.c
@@ -1779,7 +1779,7 @@ static void get_board_id(void *vsd, char *board_id)
 
 	if (be16_to_cpup(vsd + VSD_OFFSET_SIG1) == VSD_SIGNATURE_TOPSPIN &&
 	    be16_to_cpup(vsd + VSD_OFFSET_SIG2) == VSD_SIGNATURE_TOPSPIN) {
-		strlcpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MLX4_BOARD_ID_LEN);
+		strscpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MLX4_BOARD_ID_LEN);
 	} else {
 		/*
 		 * The board ID is a string but the firmware byte
diff --git a/drivers/net/ethernet/mellanox/mlx4/icm.c b/drivers/net/ethernet/mellanox/mlx4/icm.c
index d89a3da89e5a..59b8b3c73582 100644
--- a/drivers/net/ethernet/mellanox/mlx4/icm.c
+++ b/drivers/net/ethernet/mellanox/mlx4/icm.c
@@ -208,7 +208,7 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
 						chunk->sg, chunk->npages,
 						DMA_BIDIRECTIONAL);
 
-			if (chunk->nsg <= 0)
+			if (!chunk->nsg)
 				goto fail;
 		}
 
@@ -222,7 +222,7 @@ struct mlx4_icm *mlx4_alloc_icm(struct mlx4_dev *dev, int npages,
 		chunk->nsg = dma_map_sg(&dev->persist->pdev->dev, chunk->sg,
 					chunk->npages, DMA_BIDIRECTIONAL);
 
-		if (chunk->nsg <= 0)
+		if (!chunk->nsg)
 			goto fail;
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 78c5f40382c9..d3fc86cd3c1d 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -3071,6 +3071,7 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
 	err = device_create_file(&dev->persist->pdev->dev, &info->port_attr);
 	if (err) {
 		mlx4_err(dev, "Failed to create file for port %d\n", port);
+		devlink_port_type_clear(&info->devlink_port);
 		devl_port_unregister(&info->devlink_port);
 		info->port = -1;
 		return err;
@@ -3093,6 +3094,7 @@ static int mlx4_init_port_info(struct mlx4_dev *dev, int port)
 		mlx4_err(dev, "Failed to create mtu file for port %d\n", port);
 		device_remove_file(&info->dev->persist->pdev->dev,
 				   &info->port_attr);
+		devlink_port_type_clear(&info->devlink_port);
 		devl_port_unregister(&info->devlink_port);
 		info->port = -1;
 		return err;
@@ -3109,6 +3111,7 @@ static void mlx4_cleanup_port_info(struct mlx4_port_info *info)
 	device_remove_file(&info->dev->persist->pdev->dev, &info->port_attr);
 	device_remove_file(&info->dev->persist->pdev->dev,
 			   &info->port_mtu_attr);
+	devlink_port_type_clear(&info->devlink_port);
 	devl_port_unregister(&info->devlink_port);
 
 #ifdef CONFIG_RFS_ACCEL
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
index bfc0cd5ec423..26685fd0fdaa 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig
@@ -139,6 +139,14 @@ config MLX5_CORE_IPOIB
 	help
 	  MLX5 IPoIB offloads & acceleration support.
 
+config MLX5_EN_MACSEC
+	bool "Connect-X support for MACSec offload"
+	depends on MLX5_CORE_EN
+	depends on MACSEC
+	default n
+	help
+	  Build support for MACsec cryptography-offload acceleration in the NIC.
+
 config MLX5_EN_IPSEC
 	bool "Mellanox Technologies IPsec Connect-X support"
 	depends on MLX5_CORE_EN
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
index a3773a8177ed..a22c32aabf11 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile
+++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile
@@ -92,6 +92,9 @@ mlx5_core-$(CONFIG_MLX5_CORE_IPOIB) += ipoib/ipoib.o ipoib/ethtool.o ipoib/ipoib
 #
 mlx5_core-$(CONFIG_MLX5_FPGA) += fpga/cmd.o fpga/core.o fpga/conn.o fpga/sdk.o
 
+mlx5_core-$(CONFIG_MLX5_EN_MACSEC) += en_accel/macsec.o en_accel/macsec_fs.o \
+				      en_accel/macsec_stats.o
+
 mlx5_core-$(CONFIG_MLX5_EN_IPSEC) += en_accel/ipsec.o en_accel/ipsec_rxtx.o \
 				     en_accel/ipsec_stats.o en_accel/ipsec_fs.o \
 				     en_accel/ipsec_offload.o
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index a560df446bac..26a23047f1f3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -93,29 +93,26 @@ struct page_pool;
 #define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \
 	MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD))
 
-#define MLX5_MPWRQ_LOG_WQE_SZ			18
-#define MLX5_MPWRQ_WQE_PAGE_ORDER  (MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT > 0 ? \
-				    MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT : 0)
-#define MLX5_MPWRQ_PAGES_PER_WQE		BIT(MLX5_MPWRQ_WQE_PAGE_ORDER)
-
-#define MLX5_ALIGN_MTTS(mtts)		(ALIGN(mtts, 8))
-#define MLX5_ALIGNED_MTTS_OCTW(mtts)	((mtts) / 2)
-#define MLX5_MTT_OCTW(mtts)		(MLX5_ALIGNED_MTTS_OCTW(MLX5_ALIGN_MTTS(mtts)))
-/* Add another page to MLX5E_REQUIRED_WQE_MTTS as a buffer between
- * WQEs, This page will absorb write overflow by the hardware, when
- * receiving packets larger than MTU. These oversize packets are
- * dropped by the driver at a later stage.
+#define MLX5_MPWRQ_MAX_LOG_WQE_SZ 18
+
+/* Keep in sync with mlx5e_mpwrq_log_wqe_sz.
+ * These are theoretical maximums, which can be further restricted by
+ * capabilities. These values are used for static resource allocations and
+ * sanity checks.
+ * MLX5_SEND_WQE_MAX_SIZE is a bit bigger than the maximum cacheline-aligned WQE
+ * size actually used at runtime, but it's not a problem when calculating static
+ * array sizes.
  */
-#define MLX5E_REQUIRED_WQE_MTTS		(MLX5_ALIGN_MTTS(MLX5_MPWRQ_PAGES_PER_WQE + 1))
-#define MLX5E_REQUIRED_MTTS(wqes)	(wqes * MLX5E_REQUIRED_WQE_MTTS)
+#define MLX5_UMR_MAX_MTT_SPACE \
+	(ALIGN_DOWN(MLX5_SEND_WQE_MAX_SIZE - sizeof(struct mlx5e_umr_wqe), \
+		    MLX5_UMR_MTT_ALIGNMENT))
+#define MLX5_MPWRQ_MAX_PAGES_PER_WQE \
+	rounddown_pow_of_two(MLX5_UMR_MAX_MTT_SPACE / sizeof(struct mlx5_mtt))
+
 #define MLX5E_MAX_RQ_NUM_MTTS	\
-	(ALIGN_DOWN(U16_MAX, 4) * 2) /* So that MLX5_MTT_OCTW(num_mtts) fits into u16 */
+	(ALIGN_DOWN(U16_MAX, 4) * 2) /* Fits into u16 and aligned by WQEBB. */
+#define MLX5E_MAX_RQ_NUM_KSMS (U16_MAX - 1) /* So that num_ksms fits into u16. */
 #define MLX5E_ORDER2_MAX_PACKET_MTU (order_base_2(10 * 1024))
-#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW	\
-		(ilog2(MLX5E_MAX_RQ_NUM_MTTS / MLX5E_REQUIRED_WQE_MTTS))
-#define MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW \
-	(MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW + \
-	 (MLX5_MPWRQ_LOG_WQE_SZ - MLX5E_ORDER2_MAX_PACKET_MTU))
 
 #define MLX5E_MIN_SKB_FRAG_SZ		(MLX5_SKB_FRAG_SZ(MLX5_RX_HEADROOM))
 #define MLX5E_LOG_MAX_RX_WQE_BULK	\
@@ -127,8 +124,7 @@ struct page_pool;
 
 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE (1 + MLX5E_LOG_MAX_RX_WQE_BULK)
 #define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE                0xa
-#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE min_t(u8, 0xd,	\
-					       MLX5E_LOG_MAX_RQ_NUM_PACKETS_MPW)
+#define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE		0xd
 
 #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW            0x2
 
@@ -150,13 +146,6 @@ struct page_pool;
 #define MLX5E_TX_XSK_POLL_BUDGET       64
 #define MLX5E_SQ_RECOVER_MIN_INTERVAL  500 /* msecs */
 
-#define MLX5E_UMR_WQE_INLINE_SZ \
-	(sizeof(struct mlx5e_umr_wqe) + \
-	 ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(struct mlx5_mtt), \
-	       MLX5_UMR_MTT_ALIGNMENT))
-#define MLX5E_UMR_WQEBBS \
-	(DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_BB))
-
 #define MLX5E_KLM_UMR_WQE_SZ(sgl_len)\
 	(sizeof(struct mlx5e_umr_wqe) +\
 	(sizeof(struct mlx5_klm) * (sgl_len)))
@@ -174,8 +163,7 @@ struct page_pool;
 	ALIGN_DOWN(MLX5E_KLM_MAX_ENTRIES_PER_WQE(wqe_size), MLX5_UMR_KLM_ALIGNMENT)
 
 #define MLX5E_MAX_KLM_PER_WQE(mdev) \
-	MLX5E_KLM_ENTRIES_PER_WQE(MLX5_SEND_WQE_BB * \
-		mlx5e_get_sw_max_sq_mpw_wqebbs(mlx5e_get_max_sq_wqebbs(mdev)))
+	MLX5E_KLM_ENTRIES_PER_WQE(MLX5_SEND_WQE_BB * mlx5e_get_max_sq_aligned_wqebbs(mdev))
 
 #define MLX5E_MSG_LEVEL			NETIF_MSG_LINK
 
@@ -189,12 +177,6 @@ do {                                                            \
 #define mlx5e_state_dereference(priv, p) \
 	rcu_dereference_protected((p), lockdep_is_held(&(priv)->state_lock))
 
-enum mlx5e_rq_group {
-	MLX5E_RQ_GROUP_REGULAR,
-	MLX5E_RQ_GROUP_XSK,
-#define MLX5E_NUM_RQ_GROUPS(g) (1 + MLX5E_RQ_GROUP_##g)
-};
-
 static inline u8 mlx5e_get_num_lag_ports(struct mlx5_core_dev *mdev)
 {
 	if (mlx5_lag_is_lacp_owner(mdev))
@@ -227,13 +209,15 @@ static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
  * bytes units. Driver hardens the limitation to 1KB (16
  * WQEBBs), unless firmware capability is stricter.
  */
-static inline u16 mlx5e_get_max_sq_wqebbs(struct mlx5_core_dev *mdev)
+static inline u8 mlx5e_get_max_sq_wqebbs(struct mlx5_core_dev *mdev)
 {
-	return min_t(u16, MLX5_SEND_WQE_MAX_WQEBBS,
-		     MLX5_CAP_GEN(mdev, max_wqe_sz_sq) / MLX5_SEND_WQE_BB);
+	BUILD_BUG_ON(MLX5_SEND_WQE_MAX_WQEBBS > U8_MAX);
+
+	return (u8)min_t(u16, MLX5_SEND_WQE_MAX_WQEBBS,
+			 MLX5_CAP_GEN(mdev, max_wqe_sz_sq) / MLX5_SEND_WQE_BB);
 }
 
-static inline u8 mlx5e_get_sw_max_sq_mpw_wqebbs(u8 max_sq_wqebbs)
+static inline u8 mlx5e_get_max_sq_aligned_wqebbs(struct mlx5_core_dev *mdev)
 {
 /* The return value will be multiplied by MLX5_SEND_WQEBB_NUM_DS.
  * Since max_sq_wqebbs may be up to MLX5_SEND_WQE_MAX_WQEBBS == 16,
@@ -242,8 +226,9 @@ static inline u8 mlx5e_get_sw_max_sq_mpw_wqebbs(u8 max_sq_wqebbs)
  * than MLX5_SEND_WQE_MAX_WQEBBS to let a full-session WQE be
  * cache-aligned.
  */
-	u8 wqebbs = min_t(u8, max_sq_wqebbs, MLX5_SEND_WQE_MAX_WQEBBS - 1);
+	u8 wqebbs = mlx5e_get_max_sq_wqebbs(mdev);
 
+	wqebbs = min_t(u8, wqebbs, MLX5_SEND_WQE_MAX_WQEBBS - 1);
 #if L1_CACHE_BYTES >= 128
 	wqebbs = ALIGN_DOWN(wqebbs, 2);
 #endif
@@ -272,6 +257,7 @@ struct mlx5e_umr_wqe {
 	union {
 		DECLARE_FLEX_ARRAY(struct mlx5_mtt, inline_mtts);
 		DECLARE_FLEX_ARRAY(struct mlx5_klm, inline_klms);
+		DECLARE_FLEX_ARRAY(struct mlx5_ksm, inline_ksms);
 	};
 };
 
@@ -476,15 +462,11 @@ struct mlx5e_txqsq {
 	struct work_struct         recover_work;
 	struct mlx5e_ptpsq        *ptpsq;
 	cqe_ts_to_ns               ptp_cyc2time;
-	u16                        max_sq_wqebbs;
 } ____cacheline_aligned_in_smp;
 
-struct mlx5e_dma_info {
-	dma_addr_t addr;
-	union {
-		struct page *page;
-		struct xdp_buff *xsk;
-	};
+union mlx5e_alloc_unit {
+	struct page *page;
+	struct xdp_buff *xsk;
 };
 
 /* XDP packets can be transmitted in different ways. On completion, we need to
@@ -580,7 +562,6 @@ struct mlx5e_xdpsq {
 	/* control path */
 	struct mlx5_wq_ctrl        wq_ctrl;
 	struct mlx5e_channel      *channel;
-	u16                        max_sq_wqebbs;
 } ____cacheline_aligned_in_smp;
 
 struct mlx5e_ktls_resync_resp;
@@ -609,25 +590,20 @@ struct mlx5e_icosq {
 	/* control path */
 	struct mlx5_wq_ctrl        wq_ctrl;
 	struct mlx5e_channel      *channel;
-	u16                        max_sq_wqebbs;
 
 	struct work_struct         recover_work;
 } ____cacheline_aligned_in_smp;
 
 struct mlx5e_wqe_frag_info {
-	struct mlx5e_dma_info *di;
+	union mlx5e_alloc_unit *au;
 	u32 offset;
 	bool last_in_page;
 };
 
-struct mlx5e_umr_dma_info {
-	struct mlx5e_dma_info  dma_info[MLX5_MPWRQ_PAGES_PER_WQE];
-};
-
 struct mlx5e_mpw_info {
-	struct mlx5e_umr_dma_info umr;
 	u16 consumed_strides;
-	DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
+	DECLARE_BITMAP(xdp_xmit_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE);
+	union mlx5e_alloc_unit alloc_units[];
 };
 
 #define MLX5E_MAX_RX_FRAGS 4
@@ -635,13 +611,13 @@ struct mlx5e_mpw_info {
 /* a single cache unit is capable to serve one napi call (for non-striding rq)
  * or a MPWQE (for striding rq).
  */
-#define MLX5E_CACHE_UNIT	(MLX5_MPWRQ_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \
-				 MLX5_MPWRQ_PAGES_PER_WQE : NAPI_POLL_WEIGHT)
+#define MLX5E_CACHE_UNIT (MLX5_MPWRQ_MAX_PAGES_PER_WQE > NAPI_POLL_WEIGHT ? \
+			  MLX5_MPWRQ_MAX_PAGES_PER_WQE : NAPI_POLL_WEIGHT)
 #define MLX5E_CACHE_SIZE	(4 * roundup_pow_of_two(MLX5E_CACHE_UNIT))
 struct mlx5e_page_cache {
 	u32 head;
 	u32 tail;
-	struct mlx5e_dma_info page_cache[MLX5E_CACHE_SIZE];
+	struct page *page_cache[MLX5E_CACHE_SIZE];
 };
 
 struct mlx5e_rq;
@@ -674,6 +650,12 @@ struct mlx5e_rq_frags_info {
 	u8 num_frags;
 	u8 log_num_frags;
 	u8 wqe_bulk;
+	u8 wqe_index_mask;
+};
+
+struct mlx5e_dma_info {
+	dma_addr_t addr;
+	struct page *page;
 };
 
 struct mlx5e_shampo_hd {
@@ -695,13 +677,20 @@ struct mlx5e_hw_gro_data {
 	int second_ip_id;
 };
 
+enum mlx5e_mpwrq_umr_mode {
+	MLX5E_MPWRQ_UMR_MODE_ALIGNED,
+	MLX5E_MPWRQ_UMR_MODE_UNALIGNED,
+	MLX5E_MPWRQ_UMR_MODE_OVERSIZED,
+	MLX5E_MPWRQ_UMR_MODE_TRIPLE,
+};
+
 struct mlx5e_rq {
 	/* data path */
 	union {
 		struct {
 			struct mlx5_wq_cyc          wq;
 			struct mlx5e_wqe_frag_info *frags;
-			struct mlx5e_dma_info      *di;
+			union mlx5e_alloc_unit     *alloc_units;
 			struct mlx5e_rq_frags_info  info;
 			mlx5e_fp_skb_from_cqe       skb_from_cqe;
 		} wqe;
@@ -710,6 +699,7 @@ struct mlx5e_rq {
 			struct mlx5e_umr_wqe   umr_wqe;
 			struct mlx5e_mpw_info *info;
 			mlx5e_fp_skb_from_cqe_mpwrq skb_from_cqe_mpwrq;
+			__be32                 umr_mkey_be;
 			u16                    num_strides;
 			u16                    actual_wq_head;
 			u8                     log_stride_sz;
@@ -717,6 +707,11 @@ struct mlx5e_rq {
 			u8                     umr_last_bulk;
 			u8                     umr_completed;
 			u8                     min_wqe_bulk;
+			u8                     page_shift;
+			u8                     pages_per_wqe;
+			u8                     umr_wqebbs;
+			u8                     mtts_per_wqe;
+			u8                     umr_mode;
 			struct mlx5e_shampo_hd *shampo;
 		} mpwqe;
 	};
@@ -767,7 +762,6 @@ struct mlx5e_rq {
 	u32                    rqn;
 	struct mlx5_core_dev  *mdev;
 	struct mlx5e_channel  *channel;
-	u32  umr_mkey;
 	struct mlx5e_dma_info  wqe_overflow;
 
 	/* XDP read-mostly */
@@ -856,11 +850,6 @@ enum {
 	MLX5E_STATE_XDP_ACTIVE,
 };
 
-enum {
-	MLX5E_TC_PRIO = 0,
-	MLX5E_NIC_PRIO
-};
-
 struct mlx5e_modify_sq_param {
 	int curr_state;
 	int next_state;
@@ -959,6 +948,9 @@ struct mlx5e_priv {
 
 	const struct mlx5e_profile *profile;
 	void                      *ppriv;
+#ifdef CONFIG_MLX5_EN_MACSEC
+	struct mlx5e_macsec       *macsec;
+#endif
 #ifdef CONFIG_MLX5_EN_IPSEC
 	struct mlx5e_ipsec        *ipsec;
 #endif
@@ -1010,7 +1002,6 @@ struct mlx5e_profile {
 	mlx5e_stats_grp_t *stats_grps;
 	const struct mlx5e_rx_handlers *rx_handlers;
 	int	max_tc;
-	u8	rq_groups;
 	u32     features;
 };
 
@@ -1019,7 +1010,8 @@ struct mlx5e_profile {
 
 void mlx5e_build_ptys2ethtool_map(void);
 
-bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev);
+bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,
+					    enum mlx5e_mpwrq_umr_mode umr_mode);
 
 void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close);
 void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
@@ -1047,6 +1039,7 @@ struct mlx5e_rq_param;
 int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
 		  struct mlx5e_xsk_param *xsk, int node,
 		  struct mlx5e_rq *rq);
+#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time);
 void mlx5e_close_rq(struct mlx5e_rq *rq);
 int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param);
@@ -1101,7 +1094,7 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv);
 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv);
 int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx);
 
-int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state);
+int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state);
 void mlx5e_activate_rq(struct mlx5e_rq *rq);
 void mlx5e_deactivate_rq(struct mlx5e_rq *rq);
 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq);
@@ -1136,6 +1129,7 @@ static inline bool mlx5_tx_swp_supported(struct mlx5_core_dev *mdev)
 
 extern const struct ethtool_ops mlx5e_ethtool_ops;
 
+int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey);
 int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev);
 void mlx5e_destroy_mdev_resources(struct mlx5_core_dev *mdev);
 int mlx5e_refresh_tirs(struct mlx5e_priv *priv, bool enable_uc_lb,
@@ -1148,8 +1142,6 @@ void mlx5e_destroy_q_counters(struct mlx5e_priv *priv);
 int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
 		       struct mlx5e_rq *drop_rq);
 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq);
-int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node);
-void mlx5e_free_di_list(struct mlx5e_rq *rq);
 
 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn);
 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c
index e7c14c0de0a7..48581ea3adcb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.c
@@ -10,28 +10,33 @@ unsigned int mlx5e_channels_get_num(struct mlx5e_channels *chs)
 	return chs->num;
 }
 
-void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn)
+static struct mlx5e_channel *mlx5e_channels_get(struct mlx5e_channels *chs, unsigned int ix)
 {
-	struct mlx5e_channel *c;
+	WARN_ON_ONCE(ix >= mlx5e_channels_get_num(chs));
+	return chs->c[ix];
+}
 
-	WARN_ON(ix >= mlx5e_channels_get_num(chs));
-	c = chs->c[ix];
+bool mlx5e_channels_is_xsk(struct mlx5e_channels *chs, unsigned int ix)
+{
+	struct mlx5e_channel *c = mlx5e_channels_get(chs, ix);
 
-	*rqn = c->rq.rqn;
+	return test_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
 }
 
-bool mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn)
+void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn)
 {
-	struct mlx5e_channel *c;
+	struct mlx5e_channel *c = mlx5e_channels_get(chs, ix);
 
-	WARN_ON(ix >= mlx5e_channels_get_num(chs));
-	c = chs->c[ix];
+	*rqn = c->rq.rqn;
+}
 
-	if (!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
-		return false;
+void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn)
+{
+	struct mlx5e_channel *c = mlx5e_channels_get(chs, ix);
+
+	WARN_ON_ONCE(!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state));
 
 	*rqn = c->xskrq.rqn;
-	return true;
 }
 
 bool mlx5e_channels_get_ptp_rqn(struct mlx5e_channels *chs, u32 *rqn)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h
index ca00cbc827cb..637ca90daaa8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/channels.h
@@ -9,8 +9,9 @@
 struct mlx5e_channels;
 
 unsigned int mlx5e_channels_get_num(struct mlx5e_channels *chs);
+bool mlx5e_channels_is_xsk(struct mlx5e_channels *chs, unsigned int ix);
 void mlx5e_channels_get_regular_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn);
-bool mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn);
+void mlx5e_channels_get_xsk_rqn(struct mlx5e_channels *chs, unsigned int ix, u32 *rqn);
 bool mlx5e_channels_get_ptp_rqn(struct mlx5e_channels *chs, u32 *rqn);
 
 #endif /* __MLX5_EN_CHANNELS_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
index 9b8cdf2e68ad..bf2741eb7f9b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h
@@ -8,6 +8,7 @@
 #include "lib/fs_ttc.h"
 
 struct mlx5e_post_act;
+struct mlx5e_tc_table;
 
 enum {
 	MLX5E_TC_FT_LEVEL = 0,
@@ -15,6 +16,11 @@ enum {
 	MLX5E_TC_MISS_LEVEL,
 };
 
+enum {
+	MLX5E_TC_PRIO = 0,
+	MLX5E_NIC_PRIO
+};
+
 struct mlx5e_flow_table {
 	int num_groups;
 	struct mlx5_flow_table *t;
@@ -83,54 +89,28 @@ enum {
 #endif
 };
 
-struct mlx5e_priv;
-
-#ifdef CONFIG_MLX5_EN_RXNFC
-
-struct mlx5e_ethtool_table {
-	struct mlx5_flow_table *ft;
-	int                    num_rules;
-};
-
-#define ETHTOOL_NUM_L3_L4_FTS 7
-#define ETHTOOL_NUM_L2_FTS 4
-
-struct mlx5e_ethtool_steering {
-	struct mlx5e_ethtool_table      l3_l4_ft[ETHTOOL_NUM_L3_L4_FTS];
-	struct mlx5e_ethtool_table      l2_ft[ETHTOOL_NUM_L2_FTS];
-	struct list_head                rules;
-	int                             tot_num_rules;
-};
-
-void mlx5e_ethtool_init_steering(struct mlx5e_priv *priv);
-void mlx5e_ethtool_cleanup_steering(struct mlx5e_priv *priv);
-int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd);
-int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv,
-			    struct ethtool_rxnfc *info, u32 *rule_locs);
-#else
-static inline void mlx5e_ethtool_init_steering(struct mlx5e_priv *priv)    { }
-static inline void mlx5e_ethtool_cleanup_steering(struct mlx5e_priv *priv) { }
-static inline int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd)
-{ return -EOPNOTSUPP; }
-static inline int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv,
-					  struct ethtool_rxnfc *info, u32 *rule_locs)
-{ return -EOPNOTSUPP; }
-#endif /* CONFIG_MLX5_EN_RXNFC */
+struct mlx5e_flow_steering;
+struct mlx5e_rx_res;
 
 #ifdef CONFIG_MLX5_EN_ARFS
 struct mlx5e_arfs_tables;
 
-int mlx5e_arfs_create_tables(struct mlx5e_priv *priv);
-void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv);
-int mlx5e_arfs_enable(struct mlx5e_priv *priv);
-int mlx5e_arfs_disable(struct mlx5e_priv *priv);
+int mlx5e_arfs_create_tables(struct mlx5e_flow_steering *fs,
+			     struct mlx5e_rx_res *rx_res, bool ntuple);
+void mlx5e_arfs_destroy_tables(struct mlx5e_flow_steering *fs, bool ntuple);
+int mlx5e_arfs_enable(struct mlx5e_flow_steering *fs);
+int mlx5e_arfs_disable(struct mlx5e_flow_steering *fs);
 int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
 			u16 rxq_index, u32 flow_id);
 #else
-static inline int mlx5e_arfs_create_tables(struct mlx5e_priv *priv) { return 0; }
-static inline void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv) {}
-static inline int mlx5e_arfs_enable(struct mlx5e_priv *priv) { return -EOPNOTSUPP; }
-static inline int mlx5e_arfs_disable(struct mlx5e_priv *priv) {	return -EOPNOTSUPP; }
+static inline int mlx5e_arfs_create_tables(struct mlx5e_flow_steering *fs,
+					   struct mlx5e_rx_res *rx_res, bool ntuple)
+{ return 0; }
+static inline void mlx5e_arfs_destroy_tables(struct mlx5e_flow_steering *fs, bool ntuple) {}
+static inline int mlx5e_arfs_enable(struct mlx5e_flow_steering *fs)
+{ return -EOPNOTSUPP; }
+static inline int mlx5e_arfs_disable(struct mlx5e_flow_steering *fs)
+{ return -EOPNOTSUPP; }
 #endif
 
 #ifdef CONFIG_MLX5_EN_TLS
@@ -142,54 +122,63 @@ struct mlx5e_fs_udp;
 struct mlx5e_fs_any;
 struct mlx5e_ptp_fs;
 
-struct mlx5e_flow_steering {
-	bool				state_destroy;
-	bool				vlan_strip_disable;
-	struct mlx5_core_dev		*mdev;
-	struct mlx5_flow_namespace      *ns;
-#ifdef CONFIG_MLX5_EN_RXNFC
-	struct mlx5e_ethtool_steering   ethtool;
-#endif
-	struct mlx5e_tc_table           *tc;
-	struct mlx5e_promisc_table      promisc;
-	struct mlx5e_vlan_table         *vlan;
-	struct mlx5e_l2_table           l2;
-	struct mlx5_ttc_table           *ttc;
-	struct mlx5_ttc_table           *inner_ttc;
-#ifdef CONFIG_MLX5_EN_ARFS
-	struct mlx5e_arfs_tables       *arfs;
-#endif
-#ifdef CONFIG_MLX5_EN_TLS
-	struct mlx5e_accel_fs_tcp      *accel_tcp;
-#endif
-	struct mlx5e_fs_udp            *udp;
-	struct mlx5e_fs_any            *any;
-	struct mlx5e_ptp_fs            *ptp_fs;
-};
-
-void mlx5e_set_ttc_params(struct mlx5e_priv *priv,
+void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs,
+			  struct mlx5e_rx_res *rx_res,
 			  struct ttc_params *ttc_params, bool tunnel);
 
-void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv);
-int mlx5e_create_ttc_table(struct mlx5e_priv *priv);
+void mlx5e_destroy_ttc_table(struct mlx5e_flow_steering *fs);
+int mlx5e_create_ttc_table(struct mlx5e_flow_steering  *fs,
+			   struct mlx5e_rx_res *rx_res);
 
 void mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft);
 
-void mlx5e_enable_cvlan_filter(struct mlx5e_priv *priv);
-void mlx5e_disable_cvlan_filter(struct mlx5e_priv *priv);
+void mlx5e_enable_cvlan_filter(struct mlx5e_flow_steering *fs, bool promisc);
+void mlx5e_disable_cvlan_filter(struct mlx5e_flow_steering *fs, bool promisc);
 
-int mlx5e_create_flow_steering(struct mlx5e_priv *priv);
-void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv);
+int mlx5e_create_flow_steering(struct mlx5e_flow_steering *fs,
+			       struct mlx5e_rx_res *rx_res,
+			       const struct mlx5e_profile *profile,
+			       struct net_device *netdev);
+void mlx5e_destroy_flow_steering(struct mlx5e_flow_steering *fs, bool ntuple,
+				 const struct mlx5e_profile *profile);
 
 struct mlx5e_flow_steering *mlx5e_fs_init(const struct mlx5e_profile *profile,
 					  struct mlx5_core_dev *mdev,
 					  bool state_destroy);
 void mlx5e_fs_cleanup(struct mlx5e_flow_steering *fs);
-
-int mlx5e_add_vlan_trap(struct mlx5e_priv *priv, int  trap_id, int tir_num);
-void mlx5e_remove_vlan_trap(struct mlx5e_priv *priv);
-int mlx5e_add_mac_trap(struct mlx5e_priv *priv, int  trap_id, int tir_num);
-void mlx5e_remove_mac_trap(struct mlx5e_priv *priv);
+struct mlx5e_vlan_table *mlx5e_fs_get_vlan(struct mlx5e_flow_steering *fs);
+void mlx5e_fs_set_tc(struct mlx5e_flow_steering *fs, struct mlx5e_tc_table *tc);
+struct mlx5e_tc_table *mlx5e_fs_get_tc(struct mlx5e_flow_steering *fs);
+struct mlx5e_l2_table *mlx5e_fs_get_l2(struct mlx5e_flow_steering *fs);
+struct mlx5_flow_namespace *mlx5e_fs_get_ns(struct mlx5e_flow_steering *fs, bool egress);
+void mlx5e_fs_set_ns(struct mlx5e_flow_steering *fs, struct mlx5_flow_namespace *ns, bool egress);
+#ifdef CONFIG_MLX5_EN_RXNFC
+struct mlx5e_ethtool_steering *mlx5e_fs_get_ethtool(struct mlx5e_flow_steering *fs);
+#endif
+struct mlx5_ttc_table *mlx5e_fs_get_ttc(struct mlx5e_flow_steering *fs, bool inner);
+void mlx5e_fs_set_ttc(struct mlx5e_flow_steering *fs, struct mlx5_ttc_table *ttc, bool inner);
+#ifdef CONFIG_MLX5_EN_ARFS
+struct mlx5e_arfs_tables *mlx5e_fs_get_arfs(struct mlx5e_flow_steering *fs);
+void mlx5e_fs_set_arfs(struct mlx5e_flow_steering *fs, struct mlx5e_arfs_tables *arfs);
+#endif
+struct mlx5e_ptp_fs *mlx5e_fs_get_ptp(struct mlx5e_flow_steering *fs);
+void mlx5e_fs_set_ptp(struct mlx5e_flow_steering *fs, struct mlx5e_ptp_fs *ptp_fs);
+struct mlx5e_fs_any *mlx5e_fs_get_any(struct mlx5e_flow_steering *fs);
+void mlx5e_fs_set_any(struct mlx5e_flow_steering *fs, struct mlx5e_fs_any *any);
+struct mlx5e_fs_udp *mlx5e_fs_get_udp(struct mlx5e_flow_steering *fs);
+void mlx5e_fs_set_udp(struct mlx5e_flow_steering *fs, struct mlx5e_fs_udp *udp);
+#ifdef CONFIG_MLX5_EN_TLS
+struct mlx5e_accel_fs_tcp *mlx5e_fs_get_accel_tcp(struct mlx5e_flow_steering *fs);
+void mlx5e_fs_set_accel_tcp(struct mlx5e_flow_steering *fs, struct mlx5e_accel_fs_tcp *accel_tcp);
+#endif
+void mlx5e_fs_set_state_destroy(struct mlx5e_flow_steering *fs, bool state_destroy);
+void mlx5e_fs_set_vlan_strip_disable(struct mlx5e_flow_steering *fs, bool vlan_strip_disable);
+
+struct mlx5_core_dev *mlx5e_fs_get_mdev(struct mlx5e_flow_steering *fs);
+int mlx5e_add_vlan_trap(struct mlx5e_flow_steering *fs, int  trap_id, int tir_num);
+void mlx5e_remove_vlan_trap(struct mlx5e_flow_steering *fs);
+int mlx5e_add_mac_trap(struct mlx5e_flow_steering *fs, int  trap_id, int tir_num);
+void mlx5e_remove_mac_trap(struct mlx5e_flow_steering *fs);
 void mlx5e_fs_set_rx_mode_work(struct mlx5e_flow_steering *fs, struct net_device *netdev);
 int mlx5e_fs_vlan_rx_add_vid(struct mlx5e_flow_steering *fs,
 			     struct net_device *netdev,
@@ -198,5 +187,18 @@ int mlx5e_fs_vlan_rx_kill_vid(struct mlx5e_flow_steering *fs,
 			      struct net_device *netdev,
 			      __be16 proto, u16 vid);
 void mlx5e_fs_init_l2_addr(struct mlx5e_flow_steering *fs, struct net_device *netdev);
+
+#define fs_err(fs, fmt, ...) \
+	mlx5_core_err(mlx5e_fs_get_mdev(fs), fmt, ##__VA_ARGS__)
+
+#define fs_dbg(fs, fmt, ...) \
+	mlx5_core_dbg(mlx5e_fs_get_mdev(fs), fmt, ##__VA_ARGS__)
+
+#define fs_warn(fs, fmt, ...) \
+	mlx5_core_warn(mlx5e_fs_get_mdev(fs), fmt, ##__VA_ARGS__)
+
+#define fs_warn_once(fs, fmt, ...) \
+	mlx5_core_warn_once(mlx5e_fs_get_mdev(fs), fmt, ##__VA_ARGS__)
+
 #endif /* __MLX5E_FLOW_STEER_H__ */
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_ethtool.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_ethtool.h
new file mode 100644
index 000000000000..9e276fd3c0cf
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_ethtool.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */
+
+#ifndef __MLX5E_FS_ETHTOOL_H__
+#define __MLX5E_FS_ETHTOOL_H__
+
+struct mlx5e_priv;
+struct mlx5e_ethtool_steering;
+#ifdef CONFIG_MLX5_EN_RXNFC
+int mlx5e_ethtool_alloc(struct mlx5e_ethtool_steering **ethtool);
+void mlx5e_ethtool_free(struct mlx5e_ethtool_steering *ethtool);
+void mlx5e_ethtool_init_steering(struct mlx5e_flow_steering *fs);
+void mlx5e_ethtool_cleanup_steering(struct mlx5e_flow_steering *fs);
+int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd);
+int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv,
+			    struct ethtool_rxnfc *info, u32 *rule_locs);
+#else
+static inline int mlx5e_ethtool_alloc(struct mlx5e_ethtool_steering **ethtool)
+{ return 0; }
+static inline void mlx5e_ethtool_free(struct mlx5e_ethtool_steering *ethtool) { }
+static inline void mlx5e_ethtool_init_steering(struct mlx5e_flow_steering *fs) { }
+static inline void mlx5e_ethtool_cleanup_steering(struct mlx5e_flow_steering *fs) { }
+static inline int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd)
+{ return -EOPNOTSUPP; }
+static inline int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv,
+					  struct ethtool_rxnfc *info, u32 *rule_locs)
+{ return -EOPNOTSUPP; }
+#endif
+#endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c
index e153d6119e02..03cb79adf912 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c
@@ -1,9 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */
 
-#include <linux/netdevice.h>
 #include "en/fs_tt_redirect.h"
 #include "fs_core.h"
+#include "mlx5_core.h"
 
 enum fs_udp_type {
 	FS_IPV4_UDP,
@@ -74,17 +74,17 @@ static void fs_udp_set_dport_flow(struct mlx5_flow_spec *spec, enum fs_udp_type
 }
 
 struct mlx5_flow_handle *
-mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv,
+mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_flow_steering *fs,
 				  enum mlx5_traffic_types ttc_type,
 				  u32 tir_num, u16 d_port)
 {
+	struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs);
 	enum fs_udp_type type = tt2fs_udp(ttc_type);
 	struct mlx5_flow_destination dest = {};
 	struct mlx5_flow_table *ft = NULL;
 	MLX5_DECLARE_FLOW_ACT(flow_act);
 	struct mlx5_flow_handle *rule;
 	struct mlx5_flow_spec *spec;
-	struct mlx5e_fs_udp *fs_udp;
 	int err;
 
 	if (type == FS_UDP_NUM_TYPES)
@@ -94,7 +94,6 @@ mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv,
 	if (!spec)
 		return ERR_PTR(-ENOMEM);
 
-	fs_udp = priv->fs->udp;
 	ft = fs_udp->tables[type].t;
 
 	fs_udp_set_dport_flow(spec, type, d_port);
@@ -106,31 +105,30 @@ mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv,
 
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
-		netdev_err(priv->netdev, "%s: add %s rule failed, err %d\n",
-			   __func__, fs_udp_type2str(type), err);
+		fs_err(fs, "%s: add %s rule failed, err %d\n",
+		       __func__, fs_udp_type2str(type), err);
 	}
 	return rule;
 }
 
-static int fs_udp_add_default_rule(struct mlx5e_priv *priv, enum fs_udp_type type)
+static int fs_udp_add_default_rule(struct mlx5e_flow_steering *fs, enum fs_udp_type type)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
+	struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs);
 	struct mlx5e_flow_table *fs_udp_t;
 	struct mlx5_flow_destination dest;
 	MLX5_DECLARE_FLOW_ACT(flow_act);
 	struct mlx5_flow_handle *rule;
-	struct mlx5e_fs_udp *fs_udp;
 	int err;
 
-	fs_udp = priv->fs->udp;
 	fs_udp_t = &fs_udp->tables[type];
 
-	dest = mlx5_ttc_get_default_dest(priv->fs->ttc, fs_udp2tt(type));
+	dest = mlx5_ttc_get_default_dest(ttc, fs_udp2tt(type));
 	rule = mlx5_add_flow_rules(fs_udp_t->t, NULL, &flow_act, &dest, 1);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
-		netdev_err(priv->netdev,
-			   "%s: add default rule failed, fs type=%d, err %d\n",
-			   __func__, type, err);
+		fs_err(fs, "%s: add default rule failed, fs type=%d, err %d\n",
+		       __func__, type, err);
 		return err;
 	}
 
@@ -206,33 +204,36 @@ out:
 	return err;
 }
 
-static int fs_udp_create_table(struct mlx5e_priv *priv, enum fs_udp_type type)
+static int fs_udp_create_table(struct mlx5e_flow_steering *fs, enum fs_udp_type type)
 {
-	struct mlx5e_flow_table *ft = &priv->fs->udp->tables[type];
+	struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false);
+	struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs);
 	struct mlx5_flow_table_attr ft_attr = {};
+	struct mlx5e_flow_table *ft;
 	int err;
 
+	ft = &fs_udp->tables[type];
 	ft->num_groups = 0;
 
 	ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE;
 	ft_attr.level = MLX5E_FS_TT_UDP_FT_LEVEL;
 	ft_attr.prio = MLX5E_NIC_PRIO;
 
-	ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr);
+	ft->t = mlx5_create_flow_table(ns, &ft_attr);
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
 		ft->t = NULL;
 		return err;
 	}
 
-	netdev_dbg(priv->netdev, "Created fs %s table id %u level %u\n",
-		   fs_udp_type2str(type), ft->t->id, ft->t->level);
+	mlx5_core_dbg(mlx5e_fs_get_mdev(fs), "Created fs %s table id %u level %u\n",
+		      fs_udp_type2str(type), ft->t->id, ft->t->level);
 
 	err = fs_udp_create_groups(ft, type);
 	if (err)
 		goto err;
 
-	err = fs_udp_add_default_rule(priv, type);
+	err = fs_udp_add_default_rule(fs, type);
 	if (err)
 		goto err;
 
@@ -253,17 +254,17 @@ static void fs_udp_destroy_table(struct mlx5e_fs_udp *fs_udp, int i)
 	fs_udp->tables[i].t = NULL;
 }
 
-static int fs_udp_disable(struct mlx5e_priv *priv)
+static int fs_udp_disable(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
 	int err, i;
 
 	for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
 		/* Modify ttc rules destination to point back to the indir TIRs */
-		err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, fs_udp2tt(i));
+		err = mlx5_ttc_fwd_default_dest(ttc, fs_udp2tt(i));
 		if (err) {
-			netdev_err(priv->netdev,
-				   "%s: modify ttc[%d] default destination failed, err(%d)\n",
-				   __func__, fs_udp2tt(i), err);
+			fs_err(fs, "%s: modify ttc[%d] default destination failed, err(%d)\n",
+			       __func__, fs_udp2tt(i), err);
 			return err;
 		}
 	}
@@ -271,30 +272,31 @@ static int fs_udp_disable(struct mlx5e_priv *priv)
 	return 0;
 }
 
-static int fs_udp_enable(struct mlx5e_priv *priv)
+static int fs_udp_enable(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
+	struct mlx5e_fs_udp *udp = mlx5e_fs_get_udp(fs);
 	struct mlx5_flow_destination dest = {};
 	int err, i;
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
-		dest.ft = priv->fs->udp->tables[i].t;
+		dest.ft = udp->tables[i].t;
 
 		/* Modify ttc rules destination to point on the accel_fs FTs */
-		err = mlx5_ttc_fwd_dest(priv->fs->ttc, fs_udp2tt(i), &dest);
+		err = mlx5_ttc_fwd_dest(ttc, fs_udp2tt(i), &dest);
 		if (err) {
-			netdev_err(priv->netdev,
-				   "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
-				   __func__, fs_udp2tt(i), err);
+			fs_err(fs, "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
+			       __func__, fs_udp2tt(i), err);
 			return err;
 		}
 	}
 	return 0;
 }
 
-void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv)
+void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_flow_steering *fs)
 {
-	struct mlx5e_fs_udp *fs_udp = priv->fs->udp;
+	struct mlx5e_fs_udp *fs_udp = mlx5e_fs_get_udp(fs);
 	int i;
 
 	if (!fs_udp)
@@ -303,48 +305,50 @@ void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv)
 	if (--fs_udp->ref_cnt)
 		return;
 
-	fs_udp_disable(priv);
+	fs_udp_disable(fs);
 
 	for (i = 0; i < FS_UDP_NUM_TYPES; i++)
 		fs_udp_destroy_table(fs_udp, i);
 
 	kfree(fs_udp);
-	priv->fs->udp = NULL;
+	mlx5e_fs_set_udp(fs, NULL);
 }
 
-int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv *priv)
+int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5e_fs_udp *udp = mlx5e_fs_get_udp(fs);
 	int i, err;
 
-	if (priv->fs->udp) {
-		priv->fs->udp->ref_cnt++;
+	if (udp) {
+		udp->ref_cnt++;
 		return 0;
 	}
 
-	priv->fs->udp = kzalloc(sizeof(*priv->fs->udp), GFP_KERNEL);
-	if (!priv->fs->udp)
+	udp = kzalloc(sizeof(*udp), GFP_KERNEL);
+	if (!udp)
 		return -ENOMEM;
+	mlx5e_fs_set_udp(fs, udp);
 
 	for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
-		err = fs_udp_create_table(priv, i);
+		err = fs_udp_create_table(fs, i);
 		if (err)
 			goto err_destroy_tables;
 	}
 
-	err = fs_udp_enable(priv);
+	err = fs_udp_enable(fs);
 	if (err)
 		goto err_destroy_tables;
 
-	priv->fs->udp->ref_cnt = 1;
+	udp->ref_cnt = 1;
 
 	return 0;
 
 err_destroy_tables:
 	while (--i >= 0)
-		fs_udp_destroy_table(priv->fs->udp, i);
+		fs_udp_destroy_table(udp, i);
 
-	kfree(priv->fs->udp);
-	priv->fs->udp = NULL;
+	kfree(udp);
+	mlx5e_fs_set_udp(fs, NULL);
 	return err;
 }
 
@@ -356,22 +360,21 @@ static void fs_any_set_ethertype_flow(struct mlx5_flow_spec *spec, u16 ether_typ
 }
 
 struct mlx5_flow_handle *
-mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv,
+mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_flow_steering *fs,
 				  u32 tir_num, u16 ether_type)
 {
+	struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs);
 	struct mlx5_flow_destination dest = {};
 	struct mlx5_flow_table *ft = NULL;
 	MLX5_DECLARE_FLOW_ACT(flow_act);
 	struct mlx5_flow_handle *rule;
 	struct mlx5_flow_spec *spec;
-	struct mlx5e_fs_any *fs_any;
 	int err;
 
 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
 	if (!spec)
 		return ERR_PTR(-ENOMEM);
 
-	fs_any = priv->fs->any;
 	ft = fs_any->table.t;
 
 	fs_any_set_ethertype_flow(spec, ether_type);
@@ -383,31 +386,29 @@ mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv,
 
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
-		netdev_err(priv->netdev, "%s: add ANY rule failed, err %d\n",
-			   __func__, err);
+		fs_err(fs, "%s: add ANY rule failed, err %d\n",
+		       __func__, err);
 	}
 	return rule;
 }
 
-static int fs_any_add_default_rule(struct mlx5e_priv *priv)
+static int fs_any_add_default_rule(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
+	struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs);
 	struct mlx5e_flow_table *fs_any_t;
 	struct mlx5_flow_destination dest;
 	MLX5_DECLARE_FLOW_ACT(flow_act);
 	struct mlx5_flow_handle *rule;
-	struct mlx5e_fs_any *fs_any;
 	int err;
 
-	fs_any = priv->fs->any;
 	fs_any_t = &fs_any->table;
-
-	dest = mlx5_ttc_get_default_dest(priv->fs->ttc, MLX5_TT_ANY);
+	dest = mlx5_ttc_get_default_dest(ttc, MLX5_TT_ANY);
 	rule = mlx5_add_flow_rules(fs_any_t->t, NULL, &flow_act, &dest, 1);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
-		netdev_err(priv->netdev,
-			   "%s: add default rule failed, fs type=ANY, err %d\n",
-			   __func__, err);
+		fs_err(fs, "%s: add default rule failed, fs type=ANY, err %d\n",
+		       __func__, err);
 		return err;
 	}
 
@@ -472,9 +473,11 @@ err:
 	return err;
 }
 
-static int fs_any_create_table(struct mlx5e_priv *priv)
+static int fs_any_create_table(struct mlx5e_flow_steering *fs)
 {
-	struct mlx5e_flow_table *ft = &priv->fs->any->table;
+	struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false);
+	struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs);
+	struct mlx5e_flow_table *ft = &fs_any->table;
 	struct mlx5_flow_table_attr ft_attr = {};
 	int err;
 
@@ -484,21 +487,21 @@ static int fs_any_create_table(struct mlx5e_priv *priv)
 	ft_attr.level = MLX5E_FS_TT_ANY_FT_LEVEL;
 	ft_attr.prio = MLX5E_NIC_PRIO;
 
-	ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr);
+	ft->t = mlx5_create_flow_table(ns, &ft_attr);
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
 		ft->t = NULL;
 		return err;
 	}
 
-	netdev_dbg(priv->netdev, "Created fs ANY table id %u level %u\n",
-		   ft->t->id, ft->t->level);
+	mlx5_core_dbg(mlx5e_fs_get_mdev(fs), "Created fs ANY table id %u level %u\n",
+		      ft->t->id, ft->t->level);
 
 	err = fs_any_create_groups(ft);
 	if (err)
 		goto err;
 
-	err = fs_any_add_default_rule(priv);
+	err = fs_any_add_default_rule(fs);
 	if (err)
 		goto err;
 
@@ -509,35 +512,38 @@ err:
 	return err;
 }
 
-static int fs_any_disable(struct mlx5e_priv *priv)
+static int fs_any_disable(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
 	int err;
 
 	/* Modify ttc rules destination to point back to the indir TIRs */
-	err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, MLX5_TT_ANY);
+	err = mlx5_ttc_fwd_default_dest(ttc, MLX5_TT_ANY);
 	if (err) {
-		netdev_err(priv->netdev,
-			   "%s: modify ttc[%d] default destination failed, err(%d)\n",
-			   __func__, MLX5_TT_ANY, err);
+		fs_err(fs,
+		       "%s: modify ttc[%d] default destination failed, err(%d)\n",
+		       __func__, MLX5_TT_ANY, err);
 		return err;
 	}
 	return 0;
 }
 
-static int fs_any_enable(struct mlx5e_priv *priv)
+static int fs_any_enable(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
+	struct mlx5e_fs_any *any = mlx5e_fs_get_any(fs);
 	struct mlx5_flow_destination dest = {};
 	int err;
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
-	dest.ft = priv->fs->any->table.t;
+	dest.ft = any->table.t;
 
 	/* Modify ttc rules destination to point on the accel_fs FTs */
-	err = mlx5_ttc_fwd_dest(priv->fs->ttc, MLX5_TT_ANY, &dest);
+	err = mlx5_ttc_fwd_dest(ttc, MLX5_TT_ANY, &dest);
 	if (err) {
-		netdev_err(priv->netdev,
-			   "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
-			   __func__, MLX5_TT_ANY, err);
+		fs_err(fs,
+		       "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
+		       __func__, MLX5_TT_ANY, err);
 		return err;
 	}
 	return 0;
@@ -553,9 +559,9 @@ static void fs_any_destroy_table(struct mlx5e_fs_any *fs_any)
 	fs_any->table.t = NULL;
 }
 
-void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv)
+void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_flow_steering *fs)
 {
-	struct mlx5e_fs_any *fs_any = priv->fs->any;
+	struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs);
 
 	if (!fs_any)
 		return;
@@ -563,43 +569,45 @@ void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv)
 	if (--fs_any->ref_cnt)
 		return;
 
-	fs_any_disable(priv);
+	fs_any_disable(fs);
 
 	fs_any_destroy_table(fs_any);
 
 	kfree(fs_any);
-	priv->fs->any = NULL;
+	mlx5e_fs_set_any(fs, NULL);
 }
 
-int mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv *priv)
+int mlx5e_fs_tt_redirect_any_create(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5e_fs_any *fs_any = mlx5e_fs_get_any(fs);
 	int err;
 
-	if (priv->fs->any) {
-		priv->fs->any->ref_cnt++;
+	if (fs_any) {
+		fs_any->ref_cnt++;
 		return 0;
 	}
 
-	priv->fs->any = kzalloc(sizeof(*priv->fs->any), GFP_KERNEL);
-	if (!priv->fs->any)
+	fs_any = kzalloc(sizeof(*fs_any), GFP_KERNEL);
+	if (!fs_any)
 		return -ENOMEM;
+	mlx5e_fs_set_any(fs, fs_any);
 
-	err = fs_any_create_table(priv);
+	err = fs_any_create_table(fs);
 	if (err)
 		return err;
 
-	err = fs_any_enable(priv);
+	err = fs_any_enable(fs);
 	if (err)
 		goto err_destroy_table;
 
-	priv->fs->any->ref_cnt = 1;
+	fs_any->ref_cnt = 1;
 
 	return 0;
 
 err_destroy_table:
-	fs_any_destroy_table(priv->fs->any);
+	fs_any_destroy_table(fs_any);
 
-	kfree(priv->fs->any);
-	priv->fs->any = NULL;
+	kfree(fs_any);
+	mlx5e_fs_set_any(fs, NULL);
 	return err;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.h
index 7a70c4f38fda..5780fd7ad507 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.h
@@ -4,23 +4,22 @@
 #ifndef __MLX5E_FS_TT_REDIRECT_H__
 #define __MLX5E_FS_TT_REDIRECT_H__
 
-#include "en.h"
 #include "en/fs.h"
 
 void mlx5e_fs_tt_redirect_del_rule(struct mlx5_flow_handle *rule);
 
 /* UDP traffic type redirect */
 struct mlx5_flow_handle *
-mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv,
+mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_flow_steering *fs,
 				  enum mlx5_traffic_types ttc_type,
 				  u32 tir_num, u16 d_port);
-void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv);
-int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv *priv);
+void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_flow_steering *fs);
+int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_flow_steering *fs);
 
 /* ANY traffic type redirect*/
 struct mlx5_flow_handle *
-mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv,
+mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_flow_steering *fs,
 				  u32 tir_num, u16 ether_type);
-void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv);
-int mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv *priv);
+void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_flow_steering *fs);
+int mlx5e_fs_tt_redirect_any_create(struct mlx5e_flow_steering *fs);
 #endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
index e025040350ba..29dd3a04c154 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c
@@ -6,11 +6,212 @@
 #include "en/port.h"
 #include "en_accel/en_accel.h"
 #include "en_accel/ipsec.h"
+#include <net/xdp_sock_drv.h>
 
-static bool mlx5e_rx_is_xdp(struct mlx5e_params *params,
-			    struct mlx5e_xsk_param *xsk)
+static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev)
+{
+	u8 min_page_shift = MLX5_CAP_GEN_2(mdev, log_min_mkey_entity_size);
+
+	return min_page_shift ? : 12;
+}
+
+u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
+{
+	u8 req_page_shift = xsk ? order_base_2(xsk->chunk_size) : PAGE_SHIFT;
+	u8 min_page_shift = mlx5e_mpwrq_min_page_shift(mdev);
+
+	/* Regular RQ uses order-0 pages, the NIC must be able to map them. */
+	if (WARN_ON_ONCE(!xsk && req_page_shift < min_page_shift))
+		min_page_shift = req_page_shift;
+
+	return max(req_page_shift, min_page_shift);
+}
+
+enum mlx5e_mpwrq_umr_mode
+mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk)
+{
+	/* Different memory management schemes use different mechanisms to map
+	 * user-mode memory. The stricter guarantees we have, the faster
+	 * mechanisms we use:
+	 * 1. MTT - direct mapping in page granularity.
+	 * 2. KSM - indirect mapping to another MKey to arbitrary addresses, but
+	 *    all mappings have the same size.
+	 * 3. KLM - indirect mapping to another MKey to arbitrary addresses, and
+	 *    mappings can have different sizes.
+	 */
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	bool unaligned = xsk ? xsk->unaligned : false;
+	bool oversized = false;
+
+	if (xsk) {
+		oversized = xsk->chunk_size < (1 << page_shift);
+		WARN_ON_ONCE(xsk->chunk_size > (1 << page_shift));
+	}
+
+	/* XSK frame size doesn't match the UMR page size, either because the
+	 * frame size is not a power of two, or it's smaller than the minimal
+	 * page size supported by the firmware.
+	 * It's possible to receive packets bigger than MTU in certain setups.
+	 * To avoid writing over the XSK frame boundary, the top region of each
+	 * stride is mapped to a garbage page, resulting in two mappings of
+	 * different sizes per frame.
+	 */
+	if (oversized) {
+		/* An optimization for frame sizes equal to 3 * power_of_two.
+		 * 3 KSMs point to the frame, and one KSM points to the garbage
+		 * page, which works faster than KLM.
+		 */
+		if (xsk->chunk_size % 3 == 0 && is_power_of_2(xsk->chunk_size / 3))
+			return MLX5E_MPWRQ_UMR_MODE_TRIPLE;
+
+		return MLX5E_MPWRQ_UMR_MODE_OVERSIZED;
+	}
+
+	/* XSK frames can start at arbitrary unaligned locations, but they all
+	 * have the same size which is a power of two. It allows to optimize to
+	 * one KSM per frame.
+	 */
+	if (unaligned)
+		return MLX5E_MPWRQ_UMR_MODE_UNALIGNED;
+
+	/* XSK: frames are naturally aligned, MTT can be used.
+	 * Non-XSK: Allocations happen in units of CPU pages, therefore, the
+	 * mappings are naturally aligned.
+	 */
+	return MLX5E_MPWRQ_UMR_MODE_ALIGNED;
+}
+
+u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode)
+{
+	switch (mode) {
+	case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
+		return sizeof(struct mlx5_mtt);
+	case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
+		return sizeof(struct mlx5_ksm);
+	case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
+		return sizeof(struct mlx5_klm) * 2;
+	case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
+		return sizeof(struct mlx5_ksm) * 4;
+	}
+	WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", mode);
+	return 0;
+}
+
+u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
+			  enum mlx5e_mpwrq_umr_mode umr_mode)
+{
+	u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode);
+	u8 max_pages_per_wqe, max_log_mpwqe_size;
+	u16 max_wqe_size;
+
+	/* Keep in sync with MLX5_MPWRQ_MAX_PAGES_PER_WQE. */
+	max_wqe_size = mlx5e_get_max_sq_aligned_wqebbs(mdev) * MLX5_SEND_WQE_BB;
+	max_pages_per_wqe = ALIGN_DOWN(max_wqe_size - sizeof(struct mlx5e_umr_wqe),
+				       MLX5_UMR_MTT_ALIGNMENT) / umr_entry_size;
+	max_log_mpwqe_size = ilog2(max_pages_per_wqe) + page_shift;
+
+	WARN_ON_ONCE(max_log_mpwqe_size < MLX5E_ORDER2_MAX_PACKET_MTU);
+
+	return min_t(u8, max_log_mpwqe_size, MLX5_MPWRQ_MAX_LOG_WQE_SZ);
+}
+
+u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,
+			     enum mlx5e_mpwrq_umr_mode umr_mode)
 {
-	return params->xdp_prog || xsk;
+	u8 log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
+	u8 pages_per_wqe;
+
+	pages_per_wqe = log_wqe_sz > page_shift ? (1 << (log_wqe_sz - page_shift)) : 1;
+
+	/* Two MTTs are needed to form an octword. The number of MTTs is encoded
+	 * in octwords in a UMR WQE, so we need at least two to avoid mapping
+	 * garbage addresses.
+	 */
+	if (WARN_ON_ONCE(pages_per_wqe < 2 && umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED))
+		pages_per_wqe = 2;
+
+	/* Sanity check for further calculations to succeed. */
+	BUILD_BUG_ON(MLX5_MPWRQ_MAX_PAGES_PER_WQE > 64);
+	if (WARN_ON_ONCE(pages_per_wqe > MLX5_MPWRQ_MAX_PAGES_PER_WQE))
+		return MLX5_MPWRQ_MAX_PAGES_PER_WQE;
+
+	return pages_per_wqe;
+}
+
+u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
+			   enum mlx5e_mpwrq_umr_mode umr_mode)
+{
+	u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, umr_mode);
+	u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode);
+	u16 umr_wqe_sz;
+
+	umr_wqe_sz = sizeof(struct mlx5e_umr_wqe) +
+		ALIGN(pages_per_wqe * umr_entry_size, MLX5_UMR_MTT_ALIGNMENT);
+
+	WARN_ON_ONCE(DIV_ROUND_UP(umr_wqe_sz, MLX5_SEND_WQE_DS) > MLX5_WQE_CTRL_DS_MASK);
+
+	return umr_wqe_sz;
+}
+
+u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift,
+			  enum mlx5e_mpwrq_umr_mode umr_mode)
+{
+	return DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(mdev, page_shift, umr_mode),
+			    MLX5_SEND_WQE_BB);
+}
+
+u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,
+			    enum mlx5e_mpwrq_umr_mode umr_mode)
+{
+	u8 pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, page_shift, umr_mode);
+
+	/* Add another page as a buffer between WQEs. This page will absorb
+	 * write overflow by the hardware, when receiving packets larger than
+	 * MTU. These oversize packets are dropped by the driver at a later
+	 * stage.
+	 */
+	return ALIGN(pages_per_wqe + 1,
+		     MLX5_SEND_WQE_BB / mlx5e_mpwrq_umr_entry_size(umr_mode));
+}
+
+u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev,
+				enum mlx5e_mpwrq_umr_mode umr_mode)
+{
+	/* Same limits apply to KSMs and KLMs. */
+	u32 klm_limit = min(MLX5E_MAX_RQ_NUM_KSMS,
+			    1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size));
+
+	switch (umr_mode) {
+	case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
+		return MLX5E_MAX_RQ_NUM_MTTS;
+	case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
+		return klm_limit;
+	case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
+		/* Each entry is two KLMs. */
+		return klm_limit / 2;
+	case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
+		/* Each entry is four KSMs. */
+		return klm_limit / 4;
+	}
+	WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode);
+	return 0;
+}
+
+static u8 mlx5e_mpwrq_max_log_rq_size(struct mlx5_core_dev *mdev, u8 page_shift,
+				      enum mlx5e_mpwrq_umr_mode umr_mode)
+{
+	u8 mtts_per_wqe = mlx5e_mpwrq_mtts_per_wqe(mdev, page_shift, umr_mode);
+	u32 max_entries = mlx5e_mpwrq_max_num_entries(mdev, umr_mode);
+
+	return ilog2(max_entries / mtts_per_wqe);
+}
+
+u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,
+			       enum mlx5e_mpwrq_umr_mode umr_mode)
+{
+	return mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode) +
+		mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -
+		MLX5E_ORDER2_MAX_PACKET_MTU;
 }
 
 u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
@@ -22,7 +223,7 @@ u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
 		return xsk->headroom;
 
 	headroom = NET_IP_ALIGN;
-	if (mlx5e_rx_is_xdp(params, xsk))
+	if (params->xdp_prog)
 		headroom += XDP_PACKET_HEADROOM;
 	else
 		headroom += MLX5_RX_HEADROOM;
@@ -30,70 +231,80 @@ u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
 	return headroom;
 }
 
-u32 mlx5e_rx_get_min_frag_sz(struct mlx5e_params *params,
-			     struct mlx5e_xsk_param *xsk)
+static u32 mlx5e_rx_get_linear_sz_xsk(struct mlx5e_params *params,
+				      struct mlx5e_xsk_param *xsk)
 {
 	u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
-	u16 linear_rq_headroom = mlx5e_get_linear_rq_headroom(params, xsk);
 
-	return linear_rq_headroom + hw_mtu;
+	return xsk->headroom + hw_mtu;
 }
 
-static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params,
-				       struct mlx5e_xsk_param *xsk)
+static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool xsk)
 {
-	u32 frag_sz = mlx5e_rx_get_min_frag_sz(params, xsk);
-
-	/* AF_XDP doesn't build SKBs in place. */
-	if (!xsk)
-		frag_sz = MLX5_SKB_FRAG_SZ(frag_sz);
+	/* SKBs built on XDP_PASS on XSK RQs don't have headroom. */
+	u16 headroom = xsk ? 0 : mlx5e_get_linear_rq_headroom(params, NULL);
+	u32 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
 
-	/* XDP in mlx5e doesn't support multiple packets per page. AF_XDP is a
-	 * special case. It can run with frames smaller than a page, as it
-	 * doesn't allocate pages dynamically. However, here we pretend that
-	 * fragments are page-sized: it allows to treat XSK frames like pages
-	 * by redirecting alloc and free operations to XSK rings and by using
-	 * the fact there are no multiple packets per "page" (which is a frame).
-	 * The latter is important, because frames may come in a random order,
-	 * and we will have trouble assemblying a real page of multiple frames.
-	 */
-	if (mlx5e_rx_is_xdp(params, xsk))
-		frag_sz = max_t(u32, frag_sz, PAGE_SIZE);
+	return MLX5_SKB_FRAG_SZ(headroom + hw_mtu);
+}
 
-	/* Even if we can go with a smaller fragment size, we must not put
-	 * multiple packets into a single frame.
+static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev,
+					 struct mlx5e_params *params,
+					 struct mlx5e_xsk_param *xsk,
+					 bool mpwqe)
+{
+	/* XSK frames are mapped as individual pages, because frames may come in
+	 * an arbitrary order from random locations in the UMEM.
 	 */
 	if (xsk)
-		frag_sz = max_t(u32, frag_sz, xsk->chunk_size);
+		return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE;
+
+	/* XDP in mlx5e doesn't support multiple packets per page. */
+	if (params->xdp_prog)
+		return PAGE_SIZE;
 
-	return frag_sz;
+	return roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, false));
 }
 
-u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params,
-				struct mlx5e_xsk_param *xsk)
+static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev,
+				       struct mlx5e_params *params,
+				       struct mlx5e_xsk_param *xsk)
 {
-	u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params, xsk);
+	u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true);
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
 
-	return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
+	return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -
+		order_base_2(linear_stride_sz);
 }
 
-bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params,
+bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
+			    struct mlx5e_params *params,
 			    struct mlx5e_xsk_param *xsk)
 {
-	/* AF_XDP allocates SKBs on XDP_PASS - ensure they don't occupy more
-	 * than one page. For this, check both with and without xsk.
+	if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE)
+		return false;
+
+	/* Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data
+	 * must fit into a CPU page.
 	 */
-	u32 linear_frag_sz = max(mlx5e_rx_get_linear_frag_sz(params, xsk),
-				 mlx5e_rx_get_linear_frag_sz(params, NULL));
+	if (mlx5e_rx_get_linear_sz_skb(params, xsk) > PAGE_SIZE)
+		return false;
+
+	/* XSK frames must be big enough to hold the packet data. */
+	if (xsk && mlx5e_rx_get_linear_sz_xsk(params, xsk) > xsk->chunk_size)
+		return false;
 
-	return params->packet_merge.type == MLX5E_PACKET_MERGE_NONE &&
-		linear_frag_sz <= PAGE_SIZE;
+	return true;
 }
 
-bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
-				   u8 log_stride_sz, u8 log_num_strides)
+static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
+					  u8 log_stride_sz, u8 log_num_strides,
+					  u8 page_shift,
+					  enum mlx5e_mpwrq_umr_mode umr_mode)
 {
-	if (log_stride_sz + log_num_strides != MLX5_MPWRQ_LOG_WQE_SZ)
+	if (log_stride_sz + log_num_strides !=
+	    mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode))
 		return false;
 
 	if (log_stride_sz < MLX5_MPWQE_LOG_STRIDE_SZ_BASE ||
@@ -113,28 +324,53 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
 				  struct mlx5e_params *params,
 				  struct mlx5e_xsk_param *xsk)
 {
-	s8 log_num_strides;
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	u8 log_num_strides;
 	u8 log_stride_sz;
+	u8 log_wqe_sz;
 
-	if (!mlx5e_rx_is_linear_skb(params, xsk))
+	if (!mlx5e_rx_is_linear_skb(mdev, params, xsk))
 		return false;
 
-	log_stride_sz = order_base_2(mlx5e_rx_get_linear_frag_sz(params, xsk));
-	log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - log_stride_sz;
+	log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
+	log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode);
 
-	return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz, log_num_strides);
+	if (log_wqe_sz < log_stride_sz)
+		return false;
+
+	log_num_strides = log_wqe_sz - log_stride_sz;
+
+	return mlx5e_verify_rx_mpwqe_strides(mdev, log_stride_sz,
+					     log_num_strides, page_shift,
+					     umr_mode);
 }
 
-u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params,
+u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
+			       struct mlx5e_params *params,
 			       struct mlx5e_xsk_param *xsk)
 {
-	u8 log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(params, xsk);
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+	u8 log_pkts_per_wqe, page_shift, max_log_rq_size;
+
+	log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk);
+	page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode);
 
 	/* Numbers are unsigned, don't subtract to avoid underflow. */
 	if (params->log_rq_mtu_frames <
 	    log_pkts_per_wqe + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
 		return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
 
+	/* Ethtool's rx_max_pending is calculated for regular RQ, that uses
+	 * pages of PAGE_SIZE. Max length of an XSK RQ might differ if it uses a
+	 * frame size not equal to PAGE_SIZE.
+	 * A stricter condition is checked in mlx5e_mpwrq_validate_xsk, WARN on
+	 * unexpected failure.
+	 */
+	if (WARN_ON_ONCE(params->log_rq_mtu_frames > log_pkts_per_wqe + max_log_rq_size))
+		return max_log_rq_size;
+
 	return params->log_rq_mtu_frames - log_pkts_per_wqe;
 }
 
@@ -164,7 +400,7 @@ u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
 				   struct mlx5e_xsk_param *xsk)
 {
 	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
-		return order_base_2(mlx5e_rx_get_linear_frag_sz(params, xsk));
+		return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true));
 
 	return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
 }
@@ -173,7 +409,10 @@ u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
 				   struct mlx5e_params *params,
 				   struct mlx5e_xsk_param *xsk)
 {
-	return MLX5_MPWRQ_LOG_WQE_SZ -
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+
+	return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) -
 		mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
 }
 
@@ -209,11 +448,11 @@ u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *par
 	stop_room  = mlx5e_ktls_get_stop_room(mdev, params);
 	stop_room += mlx5e_stop_room_for_max_wqe(mdev);
 	if (is_mpwqe)
-		/* A MPWQE can take up to the maximum-sized WQE + all the normal
-		 * stop room can be taken if a new packet breaks the active
-		 * MPWQE session and allocates its WQEs right away.
+		/* A MPWQE can take up to the maximum cacheline-aligned WQE +
+		 * all the normal stop room can be taken if a new packet breaks
+		 * the active MPWQE session and allocates its WQEs right away.
 		 */
-		stop_room += mlx5e_stop_room_for_max_wqe(mdev);
+		stop_room += mlx5e_stop_room_for_mpwqe(mdev);
 
 	return stop_room;
 }
@@ -320,22 +559,46 @@ bool slow_pci_heuristic(struct mlx5_core_dev *mdev)
 		link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw;
 }
 
-bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
-				struct mlx5e_params *params)
+int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
 {
-	if (!mlx5e_check_fragmented_striding_rq_cap(mdev))
-		return false;
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, NULL);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, NULL);
 
-	if (params->xdp_prog) {
-		/* XSK params are not considered here. If striding RQ is in use,
-		 * and an XSK is being opened, mlx5e_rx_mpwqe_is_linear_skb will
-		 * be called with the known XSK params.
-		 */
-		if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
-			return false;
+	if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode))
+		return -EOPNOTSUPP;
+
+	if (params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL))
+		return -EINVAL;
+
+	return 0;
+}
+
+int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params,
+			     struct mlx5e_xsk_param *xsk)
+{
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	bool unaligned = xsk ? xsk->unaligned : false;
+	u16 max_mtu_pkts;
+
+	if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode))
+		return -EOPNOTSUPP;
+
+	if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk))
+		return -EINVAL;
+
+	/* Current RQ length is too big for the given frame size, the
+	 * needed number of WQEs exceeds the maximum.
+	 */
+	max_mtu_pkts = min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE,
+			     mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift, unaligned));
+	if (params->log_rq_mtu_frames > max_mtu_pkts) {
+		mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n",
+			      1 << params->log_rq_mtu_frames, xsk->chunk_size);
+		return -EINVAL;
 	}
 
-	return true;
+	return 0;
 }
 
 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
@@ -348,7 +611,7 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
 	mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
 		       params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
 		       params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ?
-		       BIT(mlx5e_mpwqe_get_log_rq_size(params, NULL)) :
+		       BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, NULL)) :
 		       BIT(params->log_rq_mtu_frames),
 		       BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)),
 		       MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
@@ -356,8 +619,7 @@ void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
 
 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
 {
-	params->rq_wq_type = mlx5e_striding_rq_possible(mdev, params) &&
-		MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ?
+	params->rq_wq_type = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ?
 		MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
 		MLX5_WQ_TYPE_CYCLIC;
 }
@@ -374,9 +636,9 @@ void mlx5e_build_rq_params(struct mlx5_core_dev *mdev,
 	 */
 	if ((!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) ||
 	     MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index)) &&
-	    mlx5e_striding_rq_possible(mdev, params) &&
+	    !mlx5e_mpwrq_validate_regular(mdev, params) &&
 	    (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ||
-	     !mlx5e_rx_is_linear_skb(params, NULL)))
+	     !mlx5e_rx_is_linear_skb(mdev, params, NULL)))
 		MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);
 	mlx5e_set_rq_type(mdev, params);
 	mlx5e_init_rq_type_params(mdev, params);
@@ -419,16 +681,22 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
 	int max_mtu;
 	int i;
 
-	if (mlx5e_rx_is_linear_skb(params, xsk)) {
+	if (mlx5e_rx_is_linear_skb(mdev, params, xsk)) {
 		int frag_stride;
 
-		frag_stride = mlx5e_rx_get_linear_frag_sz(params, xsk);
-		frag_stride = roundup_pow_of_two(frag_stride);
+		frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, false);
 
 		info->arr[0].frag_size = byte_count;
 		info->arr[0].frag_stride = frag_stride;
 		info->num_frags = 1;
-		info->wqe_bulk = PAGE_SIZE / frag_stride;
+
+		/* N WQEs share the same page, N = PAGE_SIZE / frag_stride. The
+		 * first WQE in the page is responsible for allocation of this
+		 * page, this WQE's index is k*N. If WQEs [k*N+1; k*N+N-1] are
+		 * still not completed, the allocation must stop before k*N.
+		 */
+		info->wqe_index_mask = (PAGE_SIZE / frag_stride) - 1;
+
 		goto out;
 	}
 
@@ -477,11 +745,40 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
 		i++;
 	}
 	info->num_frags = i;
-	/* number of different wqes sharing a page */
-	info->wqe_bulk = 1 + (info->num_frags % 2);
+
+	/* The last fragment of WQE with index 2*N may share the page with the
+	 * first fragment of WQE with index 2*N+1 in certain cases. If WQE 2*N+1
+	 * is not completed yet, WQE 2*N must not be allocated, as it's
+	 * responsible for allocating a new page.
+	 */
+	if (frag_size_max == PAGE_SIZE) {
+		/* No WQE can start in the middle of a page. */
+		info->wqe_index_mask = 0;
+	} else {
+		/* PAGE_SIZEs starting from 8192 don't use 2K-sized fragments,
+		 * because there would be more than MLX5E_MAX_RX_FRAGS of them.
+		 */
+		WARN_ON(PAGE_SIZE != 2 * DEFAULT_FRAG_SIZE);
+
+		/* Odd number of fragments allows to pack the last fragment of
+		 * the previous WQE and the first fragment of the next WQE into
+		 * the same page.
+		 * As long as DEFAULT_FRAG_SIZE is 2048, and MLX5E_MAX_RX_FRAGS
+		 * is 4, the last fragment can be bigger than the rest only if
+		 * it's the fourth one, so WQEs consisting of 3 fragments will
+		 * always share a page.
+		 * When a page is shared, WQE bulk size is 2, otherwise just 1.
+		 */
+		info->wqe_index_mask = info->num_frags % 2;
+	}
 
 out:
-	info->wqe_bulk = max_t(u8, info->wqe_bulk, 8);
+	/* Bulking optimization to skip allocation until at least 8 WQEs can be
+	 * allocated in a row. At the same time, never start allocation when
+	 * the page is still used by older WQEs.
+	 */
+	info->wqe_bulk = max_t(u8, info->wqe_index_mask + 1, 8);
+
 	info->log_num_frags = order_base_2(info->num_frags);
 
 	return 0;
@@ -520,7 +817,7 @@ static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev,
 	u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
 	int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(mdev, params));
 	u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
-	int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(params, xsk));
+	int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
 	int wqe_size = BIT(log_stride_sz) * num_strides;
 
 	/* +1 is for the case that the pkt_per_rsrv dont consume the reservation
@@ -544,7 +841,7 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev,
 		if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
 			log_cq_size = mlx5e_shampo_get_log_cq_size(mdev, params, xsk);
 		else
-			log_cq_size = mlx5e_mpwqe_get_log_rq_size(params, xsk) +
+			log_cq_size = mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk) +
 				mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
 		break;
 	default: /* MLX5_WQ_TYPE_CYCLIC */
@@ -587,12 +884,16 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: {
 		u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk);
 		u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
+		enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+		u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
 
 		if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size,
-						   log_wqe_num_of_strides)) {
+						   log_wqe_num_of_strides,
+						   page_shift, umr_mode)) {
 			mlx5_core_err(mdev,
-				      "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u\n",
-				      log_wqe_stride_size, log_wqe_num_of_strides);
+				      "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n",
+				      log_wqe_stride_size, log_wqe_num_of_strides,
+				      umr_mode);
 			return -EINVAL;
 		}
 
@@ -600,7 +901,7 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev,
 			 log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
 		MLX5_SET(wq, wq, log_wqe_stride_size,
 			 log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
-		MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params, xsk));
+		MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
 		if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) {
 			MLX5_SET(wq, wq, shampo_enable, true);
 			MLX5_SET(wq, wq, log_reservation_size,
@@ -712,13 +1013,6 @@ static void mlx5e_build_ico_cq_param(struct mlx5_core_dev *mdev,
 	param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 }
 
-static u8 mlx5e_get_rq_log_wq_sz(void *rqc)
-{
-	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
-
-	return MLX5_GET(wq, wq, log_wq_sz);
-}
-
 /* This function calculates the maximum number of headers entries that are needed
  * per WQE, the formula is based on the size of the reservations and the
  * restriction we have about max packets for reservation that is equal to max
@@ -779,31 +1073,92 @@ static u32 mlx5e_shampo_icosq_sz(struct mlx5_core_dev *mdev,
 	return wqebbs;
 }
 
+static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev,
+					struct mlx5e_params *params,
+					struct mlx5e_xsk_param *xsk)
+{
+	enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+	u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+	u8 umr_wqebbs;
+
+	umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode);
+
+	return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk));
+}
+
 static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev,
 				      struct mlx5e_params *params,
 				      struct mlx5e_rq_param *rqp)
 {
-	u32 wqebbs;
+	u32 wqebbs, total_pages, useful_space;
 
 	/* MLX5_WQ_TYPE_CYCLIC */
 	if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
 		return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
 
-	wqebbs = MLX5E_UMR_WQEBBS * BIT(mlx5e_get_rq_log_wq_sz(rqp->rqc));
+	/* UMR WQEs for the regular RQ. */
+	wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL);
 
 	/* If XDP program is attached, XSK may be turned on at any time without
 	 * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of
 	 * both regular RQ and XSK RQ.
-	 * Although mlx5e_mpwqe_get_log_rq_size accepts mlx5e_xsk_param, it
-	 * doesn't affect its return value, as long as params->xdp_prog != NULL,
-	 * so we can just multiply by 2.
+	 *
+	 * XSK uses different values of page_shift, and the total number of UMR
+	 * WQEBBs depends on it. This dependency is complex and not monotonic,
+	 * especially taking into consideration that some of the parameters come
+	 * from capabilities. Hence, we have to try all valid values of XSK
+	 * frame size (and page_shift) to find the maximum.
 	 */
-	if (params->xdp_prog)
-		wqebbs *= 2;
+	if (params->xdp_prog) {
+		u32 max_xsk_wqebbs = 0;
+		u8 frame_shift;
+
+		for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT;
+		     frame_shift <= PAGE_SHIFT; frame_shift++) {
+			/* The headroom doesn't affect the calculation. */
+			struct mlx5e_xsk_param xsk = {
+				.chunk_size = 1 << frame_shift,
+				.unaligned = false,
+			};
+
+			/* XSK aligned mode. */
+			max_xsk_wqebbs = max(max_xsk_wqebbs,
+				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
+
+			/* XSK unaligned mode, frame size is a power of two. */
+			xsk.unaligned = true;
+			max_xsk_wqebbs = max(max_xsk_wqebbs,
+				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
+
+			/* XSK unaligned mode, frame size is not equal to stride size. */
+			xsk.chunk_size -= 1;
+			max_xsk_wqebbs = max(max_xsk_wqebbs,
+				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
+
+			/* XSK unaligned mode, frame size is a triple power of two. */
+			xsk.chunk_size = (1 << frame_shift) / 4 * 3;
+			max_xsk_wqebbs = max(max_xsk_wqebbs,
+				mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk));
+		}
+
+		wqebbs += max_xsk_wqebbs;
+	}
 
 	if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
 		wqebbs += mlx5e_shampo_icosq_sz(mdev, params, rqp);
 
+	/* UMR WQEs don't cross the page boundary, they are padded with NOPs.
+	 * This padding is always smaller than the max WQE size. That gives us
+	 * at least (PAGE_SIZE - (max WQE size - MLX5_SEND_WQE_BB)) useful bytes
+	 * per page. The number of pages is estimated as the total size of WQEs
+	 * divided by the useful space in page, rounding up. If some WQEs don't
+	 * fully fit into the useful space, they can occupy part of the padding,
+	 * which proves this estimation to be correct (reserve enough space).
+	 */
+	useful_space = PAGE_SIZE - mlx5e_get_max_sq_wqebbs(mdev) + MLX5_SEND_WQE_BB;
+	total_pages = DIV_ROUND_UP(wqebbs * MLX5_SEND_WQE_BB, useful_space);
+	wqebbs = total_pages * (PAGE_SIZE / MLX5_SEND_WQE_BB);
+
 	return max_t(u8, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE, order_base_2(wqebbs));
 }
 
@@ -857,7 +1212,7 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev,
 	mlx5e_build_sq_param_common(mdev, param);
 	MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
 	param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE);
-	param->is_xdp_mb = !mlx5e_rx_is_linear_skb(params, xsk);
+	param->is_xdp_mb = !mlx5e_rx_is_linear_skb(mdev, params, xsk);
 	mlx5e_build_tx_cq_param(mdev, params, &param->cqp);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
index f5c46e78eebc..034debd140bc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h
@@ -9,6 +9,7 @@
 struct mlx5e_xsk_param {
 	u16 headroom;
 	u16 chunk_size;
+	bool unaligned;
 };
 
 struct mlx5e_cq_param {
@@ -52,37 +53,26 @@ struct mlx5e_create_sq_param {
 	u8                          min_inline_mode;
 };
 
-static inline bool mlx5e_qid_get_ch_if_in_group(struct mlx5e_params *params,
-						u16 qid,
-						enum mlx5e_rq_group group,
-						u16 *ix)
-{
-	int nch = params->num_channels;
-	int ch = qid - nch * group;
-
-	if (ch < 0 || ch >= nch)
-		return false;
-
-	*ix = ch;
-	return true;
-}
-
-static inline void mlx5e_qid_get_ch_and_group(struct mlx5e_params *params,
-					      u16 qid,
-					      u16 *ix,
-					      enum mlx5e_rq_group *group)
-{
-	u16 nch = params->num_channels;
-
-	*ix = qid % nch;
-	*group = qid / nch;
-}
-
-static inline bool mlx5e_qid_validate(const struct mlx5e_profile *profile,
-				      struct mlx5e_params *params, u64 qid)
-{
-	return qid < params->num_channels * profile->rq_groups;
-}
+/* Striding RQ dynamic parameters */
+
+u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);
+enum mlx5e_mpwrq_umr_mode
+mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk);
+u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode);
+u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
+			  enum mlx5e_mpwrq_umr_mode umr_mode);
+u8 mlx5e_mpwrq_pages_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,
+			     enum mlx5e_mpwrq_umr_mode umr_mode);
+u16 mlx5e_mpwrq_umr_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift,
+			   enum mlx5e_mpwrq_umr_mode umr_mode);
+u8 mlx5e_mpwrq_umr_wqebbs(struct mlx5_core_dev *mdev, u8 page_shift,
+			  enum mlx5e_mpwrq_umr_mode umr_mode);
+u8 mlx5e_mpwrq_mtts_per_wqe(struct mlx5_core_dev *mdev, u8 page_shift,
+			    enum mlx5e_mpwrq_umr_mode umr_mode);
+u32 mlx5e_mpwrq_max_num_entries(struct mlx5_core_dev *mdev,
+				enum mlx5e_mpwrq_umr_mode umr_mode);
+u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift,
+			       enum mlx5e_mpwrq_umr_mode umr_mode);
 
 /* Parameter calculations */
 
@@ -92,25 +82,23 @@ void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode);
 
 bool slow_pci_heuristic(struct mlx5_core_dev *mdev);
-bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
+int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
+int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params,
+			     struct mlx5e_xsk_param *xsk);
 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params);
 
-bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev,
-				   u8 log_stride_sz, u8 log_num_strides);
 u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params,
 				 struct mlx5e_xsk_param *xsk);
-u32 mlx5e_rx_get_min_frag_sz(struct mlx5e_params *params,
-			     struct mlx5e_xsk_param *xsk);
-u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params,
-				struct mlx5e_xsk_param *xsk);
-bool mlx5e_rx_is_linear_skb(struct mlx5e_params *params,
+bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
+			    struct mlx5e_params *params,
 			    struct mlx5e_xsk_param *xsk);
 bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
 				  struct mlx5e_params *params,
 				  struct mlx5e_xsk_param *xsk);
-u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params,
+u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev,
+			       struct mlx5e_params *params,
 			       struct mlx5e_xsk_param *xsk);
 u8 mlx5e_shampo_get_log_hd_entry_size(struct mlx5_core_dev *mdev,
 				      struct mlx5e_params *params);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
index 903de88bab53..8469e9c38670 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c
@@ -622,37 +622,39 @@ static int mlx5e_ptp_set_state(struct mlx5e_ptp *c, struct mlx5e_params *params)
 	return bitmap_empty(c->state, MLX5E_PTP_STATE_NUM_STATES) ? -EINVAL : 0;
 }
 
-static void mlx5e_ptp_rx_unset_fs(struct mlx5e_priv *priv)
+static void mlx5e_ptp_rx_unset_fs(struct mlx5e_flow_steering *fs)
 {
-	struct mlx5e_ptp_fs *ptp_fs = priv->fs->ptp_fs;
+	struct mlx5e_ptp_fs *ptp_fs = mlx5e_fs_get_ptp(fs);
 
 	if (!ptp_fs->valid)
 		return;
 
 	mlx5e_fs_tt_redirect_del_rule(ptp_fs->l2_rule);
-	mlx5e_fs_tt_redirect_any_destroy(priv);
+	mlx5e_fs_tt_redirect_any_destroy(fs);
 
 	mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v6_rule);
 	mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v4_rule);
-	mlx5e_fs_tt_redirect_udp_destroy(priv);
+	mlx5e_fs_tt_redirect_udp_destroy(fs);
 	ptp_fs->valid = false;
 }
 
 static int mlx5e_ptp_rx_set_fs(struct mlx5e_priv *priv)
 {
 	u32 tirn = mlx5e_rx_res_get_tirn_ptp(priv->rx_res);
-	struct mlx5e_ptp_fs *ptp_fs = priv->fs->ptp_fs;
+	struct mlx5e_flow_steering *fs = priv->fs;
 	struct mlx5_flow_handle *rule;
+	struct mlx5e_ptp_fs *ptp_fs;
 	int err;
 
+	ptp_fs = mlx5e_fs_get_ptp(fs);
 	if (ptp_fs->valid)
 		return 0;
 
-	err = mlx5e_fs_tt_redirect_udp_create(priv);
+	err = mlx5e_fs_tt_redirect_udp_create(fs);
 	if (err)
 		goto out_free;
 
-	rule = mlx5e_fs_tt_redirect_udp_add_rule(priv, MLX5_TT_IPV4_UDP,
+	rule = mlx5e_fs_tt_redirect_udp_add_rule(fs, MLX5_TT_IPV4_UDP,
 						 tirn, PTP_EV_PORT);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
@@ -660,7 +662,7 @@ static int mlx5e_ptp_rx_set_fs(struct mlx5e_priv *priv)
 	}
 	ptp_fs->udp_v4_rule = rule;
 
-	rule = mlx5e_fs_tt_redirect_udp_add_rule(priv, MLX5_TT_IPV6_UDP,
+	rule = mlx5e_fs_tt_redirect_udp_add_rule(fs, MLX5_TT_IPV6_UDP,
 						 tirn, PTP_EV_PORT);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
@@ -668,11 +670,11 @@ static int mlx5e_ptp_rx_set_fs(struct mlx5e_priv *priv)
 	}
 	ptp_fs->udp_v6_rule = rule;
 
-	err = mlx5e_fs_tt_redirect_any_create(priv);
+	err = mlx5e_fs_tt_redirect_any_create(fs);
 	if (err)
 		goto out_destroy_udp_v6_rule;
 
-	rule = mlx5e_fs_tt_redirect_any_add_rule(priv, tirn, ETH_P_1588);
+	rule = mlx5e_fs_tt_redirect_any_add_rule(fs, tirn, ETH_P_1588);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
 		goto out_destroy_fs_any;
@@ -683,13 +685,13 @@ static int mlx5e_ptp_rx_set_fs(struct mlx5e_priv *priv)
 	return 0;
 
 out_destroy_fs_any:
-	mlx5e_fs_tt_redirect_any_destroy(priv);
+	mlx5e_fs_tt_redirect_any_destroy(fs);
 out_destroy_udp_v6_rule:
 	mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v6_rule);
 out_destroy_udp_v4_rule:
 	mlx5e_fs_tt_redirect_del_rule(ptp_fs->udp_v4_rule);
 out_destroy_fs_udp:
-	mlx5e_fs_tt_redirect_udp_destroy(priv);
+	mlx5e_fs_tt_redirect_udp_destroy(fs);
 out_free:
 	return err;
 }
@@ -723,7 +725,7 @@ int mlx5e_ptp_open(struct mlx5e_priv *priv, struct mlx5e_params *params,
 	if (err)
 		goto err_free;
 
-	netif_napi_add(netdev, &c->napi, mlx5e_ptp_napi_poll, 64);
+	netif_napi_add(netdev, &c->napi, mlx5e_ptp_napi_poll);
 
 	mlx5e_ptp_build_params(c, cparams, params);
 
@@ -797,29 +799,31 @@ int mlx5e_ptp_get_rqn(struct mlx5e_ptp *c, u32 *rqn)
 	return 0;
 }
 
-int mlx5e_ptp_alloc_rx_fs(struct mlx5e_priv *priv)
+int mlx5e_ptp_alloc_rx_fs(struct mlx5e_flow_steering *fs,
+			  const struct mlx5e_profile *profile)
 {
 	struct mlx5e_ptp_fs *ptp_fs;
 
-	if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX))
+	if (!mlx5e_profile_feature_cap(profile, PTP_RX))
 		return 0;
 
 	ptp_fs = kzalloc(sizeof(*ptp_fs), GFP_KERNEL);
 	if (!ptp_fs)
 		return -ENOMEM;
+	mlx5e_fs_set_ptp(fs, ptp_fs);
 
-	priv->fs->ptp_fs = ptp_fs;
 	return 0;
 }
 
-void mlx5e_ptp_free_rx_fs(struct mlx5e_priv *priv)
+void mlx5e_ptp_free_rx_fs(struct mlx5e_flow_steering *fs,
+			  const struct mlx5e_profile *profile)
 {
-	struct mlx5e_ptp_fs *ptp_fs = priv->fs->ptp_fs;
+	struct mlx5e_ptp_fs *ptp_fs = mlx5e_fs_get_ptp(fs);
 
-	if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX))
+	if (!mlx5e_profile_feature_cap(profile, PTP_RX))
 		return;
 
-	mlx5e_ptp_rx_unset_fs(priv);
+	mlx5e_ptp_rx_unset_fs(fs);
 	kfree(ptp_fs);
 }
 
@@ -845,6 +849,6 @@ int mlx5e_ptp_rx_manage_fs(struct mlx5e_priv *priv, bool set)
 		netdev_WARN_ONCE(priv->netdev, "Don't try to remove PTP RX-FS rules");
 		return -EINVAL;
 	}
-	mlx5e_ptp_rx_unset_fs(priv);
+	mlx5e_ptp_rx_unset_fs(priv->fs);
 	return 0;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h
index 92dbbec472ec..5bce554e131a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h
@@ -74,8 +74,10 @@ void mlx5e_ptp_close(struct mlx5e_ptp *c);
 void mlx5e_ptp_activate_channel(struct mlx5e_ptp *c);
 void mlx5e_ptp_deactivate_channel(struct mlx5e_ptp *c);
 int mlx5e_ptp_get_rqn(struct mlx5e_ptp *c, u32 *rqn);
-int mlx5e_ptp_alloc_rx_fs(struct mlx5e_priv *priv);
-void mlx5e_ptp_free_rx_fs(struct mlx5e_priv *priv);
+int mlx5e_ptp_alloc_rx_fs(struct mlx5e_flow_steering *fs,
+			  const struct mlx5e_profile *profile);
+void mlx5e_ptp_free_rx_fs(struct mlx5e_flow_steering *fs,
+			  const struct mlx5e_profile *profile);
 int mlx5e_ptp_rx_manage_fs(struct mlx5e_priv *priv, bool set);
 
 enum {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
index fc366e66d0b0..5f6f95ad6888 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_rx.c
@@ -134,38 +134,17 @@ out:
 	return err;
 }
 
-static int mlx5e_rq_to_ready(struct mlx5e_rq *rq, int curr_state)
-{
-	struct net_device *dev = rq->netdev;
-	int err;
-
-	err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST);
-	if (err) {
-		netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn);
-		return err;
-	}
-	err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
-	if (err) {
-		netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn);
-		return err;
-	}
-
-	return 0;
-}
-
 static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx)
 {
 	struct mlx5e_rq *rq = ctx;
 	int err;
 
 	mlx5e_deactivate_rq(rq);
-	mlx5e_free_rx_descs(rq);
-
-	err = mlx5e_rq_to_ready(rq, MLX5_RQC_STATE_ERR);
+	err = mlx5e_flush_rq(rq, MLX5_RQC_STATE_ERR);
+	clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state);
 	if (err)
-		goto out;
+		return err;
 
-	clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state);
 	mlx5e_activate_rq(rq);
 	rq->stats->recover++;
 	if (rq->channel)
@@ -173,9 +152,6 @@ static int mlx5e_rx_reporter_err_rq_cqe_recover(void *ctx)
 	else
 		mlx5e_trigger_napi_sched(rq->cq.napi);
 	return 0;
-out:
-	clear_bit(MLX5E_RQ_STATE_RECOVERING, &rq->state);
-	return err;
 }
 
 static int mlx5e_rx_reporter_timeout_recover(void *ctx)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
index 24c32f73040a..e1095bc36543 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
@@ -24,8 +24,6 @@ struct mlx5e_rx_res {
 	struct {
 		struct mlx5e_rqt direct_rqt;
 		struct mlx5e_tir direct_tir;
-		struct mlx5e_rqt xsk_rqt;
-		struct mlx5e_tir xsk_tir;
 	} *channels;
 
 	struct {
@@ -320,48 +318,8 @@ static int mlx5e_rx_res_channels_init(struct mlx5e_rx_res *res)
 		mlx5e_tir_builder_clear(builder);
 	}
 
-	if (!(res->features & MLX5E_RX_RES_FEATURE_XSK))
-		goto out;
-
-	for (ix = 0; ix < res->max_nch; ix++) {
-		err = mlx5e_rqt_init_direct(&res->channels[ix].xsk_rqt,
-					    res->mdev, false, res->drop_rqn);
-		if (err) {
-			mlx5_core_warn(res->mdev, "Failed to create an XSK RQT: err = %d, ix = %u\n",
-				       err, ix);
-			goto err_destroy_xsk_rqts;
-		}
-	}
-
-	for (ix = 0; ix < res->max_nch; ix++) {
-		mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn,
-					    mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt),
-					    inner_ft_support);
-		mlx5e_tir_builder_build_packet_merge(builder, &res->pkt_merge_param);
-		mlx5e_tir_builder_build_direct(builder);
-
-		err = mlx5e_tir_init(&res->channels[ix].xsk_tir, builder, res->mdev, true);
-		if (err) {
-			mlx5_core_warn(res->mdev, "Failed to create an XSK TIR: err = %d, ix = %u\n",
-				       err, ix);
-			goto err_destroy_xsk_tirs;
-		}
-
-		mlx5e_tir_builder_clear(builder);
-	}
-
 	goto out;
 
-err_destroy_xsk_tirs:
-	while (--ix >= 0)
-		mlx5e_tir_destroy(&res->channels[ix].xsk_tir);
-
-	ix = res->max_nch;
-err_destroy_xsk_rqts:
-	while (--ix >= 0)
-		mlx5e_rqt_destroy(&res->channels[ix].xsk_rqt);
-
-	ix = res->max_nch;
 err_destroy_direct_tirs:
 	while (--ix >= 0)
 		mlx5e_tir_destroy(&res->channels[ix].direct_tir);
@@ -420,12 +378,6 @@ static void mlx5e_rx_res_channels_destroy(struct mlx5e_rx_res *res)
 	for (ix = 0; ix < res->max_nch; ix++) {
 		mlx5e_tir_destroy(&res->channels[ix].direct_tir);
 		mlx5e_rqt_destroy(&res->channels[ix].direct_rqt);
-
-		if (!(res->features & MLX5E_RX_RES_FEATURE_XSK))
-			continue;
-
-		mlx5e_tir_destroy(&res->channels[ix].xsk_tir);
-		mlx5e_rqt_destroy(&res->channels[ix].xsk_rqt);
 	}
 
 	kvfree(res->channels);
@@ -491,13 +443,6 @@ u32 mlx5e_rx_res_get_tirn_direct(struct mlx5e_rx_res *res, unsigned int ix)
 	return mlx5e_tir_get_tirn(&res->channels[ix].direct_tir);
 }
 
-u32 mlx5e_rx_res_get_tirn_xsk(struct mlx5e_rx_res *res, unsigned int ix)
-{
-	WARN_ON(!(res->features & MLX5E_RX_RES_FEATURE_XSK));
-
-	return mlx5e_tir_get_tirn(&res->channels[ix].xsk_tir);
-}
-
 u32 mlx5e_rx_res_get_tirn_rss(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt)
 {
 	struct mlx5e_rss *rss = res->rss[0];
@@ -523,56 +468,53 @@ static u32 mlx5e_rx_res_get_rqtn_direct(struct mlx5e_rx_res *res, unsigned int i
 	return mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt);
 }
 
-void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs)
+static void mlx5e_rx_res_channel_activate_direct(struct mlx5e_rx_res *res,
+						 struct mlx5e_channels *chs,
+						 unsigned int ix)
 {
-	unsigned int nch, ix;
+	u32 rqn = res->rss_rqns[ix];
 	int err;
 
-	nch = mlx5e_channels_get_num(chs);
-
-	for (ix = 0; ix < chs->num; ix++)
-		mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]);
-	res->rss_nch = chs->num;
+	err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn);
+	if (err)
+		mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (channel %u): err = %d\n",
+			       mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt),
+			       rqn, ix, err);
+}
 
-	mlx5e_rx_res_rss_enable(res);
+static void mlx5e_rx_res_channel_deactivate_direct(struct mlx5e_rx_res *res,
+						   unsigned int ix)
+{
+	int err;
 
-	for (ix = 0; ix < nch; ix++) {
-		u32 rqn;
+	err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn);
+	if (err)
+		mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n",
+			       mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt),
+			       res->drop_rqn, ix, err);
+}
 
-		mlx5e_channels_get_regular_rqn(chs, ix, &rqn);
-		err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn);
-		if (err)
-			mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (channel %u): err = %d\n",
-				       mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt),
-				       rqn, ix, err);
+void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs)
+{
+	unsigned int nch, ix;
+	int err;
 
-		if (!(res->features & MLX5E_RX_RES_FEATURE_XSK))
-			continue;
+	nch = mlx5e_channels_get_num(chs);
 
-		if (!mlx5e_channels_get_xsk_rqn(chs, ix, &rqn))
-			rqn = res->drop_rqn;
-		err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, rqn);
-		if (err)
-			mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to RQ %#x (channel %u): err = %d\n",
-				       mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt),
-				       rqn, ix, err);
+	for (ix = 0; ix < chs->num; ix++) {
+		if (mlx5e_channels_is_xsk(chs, ix))
+			mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]);
+		else
+			mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]);
 	}
-	for (ix = nch; ix < res->max_nch; ix++) {
-		err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn);
-		if (err)
-			mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n",
-				       mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt),
-				       res->drop_rqn, ix, err);
+	res->rss_nch = chs->num;
 
-		if (!(res->features & MLX5E_RX_RES_FEATURE_XSK))
-			continue;
+	mlx5e_rx_res_rss_enable(res);
 
-		err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, res->drop_rqn);
-		if (err)
-			mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to drop RQ %#x (channel %u): err = %d\n",
-				       mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt),
-				       res->drop_rqn, ix, err);
-	}
+	for (ix = 0; ix < nch; ix++)
+		mlx5e_rx_res_channel_activate_direct(res, chs, ix);
+	for (ix = nch; ix < res->max_nch; ix++)
+		mlx5e_rx_res_channel_deactivate_direct(res, ix);
 
 	if (res->features & MLX5E_RX_RES_FEATURE_PTP) {
 		u32 rqn;
@@ -595,22 +537,8 @@ void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res)
 
 	mlx5e_rx_res_rss_disable(res);
 
-	for (ix = 0; ix < res->max_nch; ix++) {
-		err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn);
-		if (err)
-			mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n",
-				       mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt),
-				       res->drop_rqn, ix, err);
-
-		if (!(res->features & MLX5E_RX_RES_FEATURE_XSK))
-			continue;
-
-		err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, res->drop_rqn);
-		if (err)
-			mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to drop RQ %#x (channel %u): err = %d\n",
-				       mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt),
-				       res->drop_rqn, ix, err);
-	}
+	for (ix = 0; ix < res->max_nch; ix++)
+		mlx5e_rx_res_channel_deactivate_direct(res, ix);
 
 	if (res->features & MLX5E_RX_RES_FEATURE_PTP) {
 		err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, res->drop_rqn);
@@ -621,33 +549,17 @@ void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res)
 	}
 }
 
-int mlx5e_rx_res_xsk_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs,
-			      unsigned int ix)
+void mlx5e_rx_res_xsk_update(struct mlx5e_rx_res *res, struct mlx5e_channels *chs,
+			     unsigned int ix, bool xsk)
 {
-	u32 rqn;
-	int err;
-
-	if (!mlx5e_channels_get_xsk_rqn(chs, ix, &rqn))
-		return -EINVAL;
-
-	err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, rqn);
-	if (err)
-		mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to XSK RQ %#x (channel %u): err = %d\n",
-			       mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt),
-			       rqn, ix, err);
-	return err;
-}
+	if (xsk)
+		mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]);
+	else
+		mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]);
 
-int mlx5e_rx_res_xsk_deactivate(struct mlx5e_rx_res *res, unsigned int ix)
-{
-	int err;
+	mlx5e_rx_res_rss_enable(res);
 
-	err = mlx5e_rqt_redirect_direct(&res->channels[ix].xsk_rqt, res->drop_rqn);
-	if (err)
-		mlx5_core_warn(res->mdev, "Failed to redirect XSK RQT %#x to drop RQ %#x (channel %u): err = %d\n",
-			       mlx5e_rqt_get_rqtn(&res->channels[ix].xsk_rqt),
-			       res->drop_rqn, ix, err);
-	return err;
+	mlx5e_rx_res_channel_activate_direct(res, chs, ix);
 }
 
 int mlx5e_rx_res_packet_merge_set_param(struct mlx5e_rx_res *res,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
index b39b20a720e0..5d5f64fab60f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
@@ -17,8 +17,7 @@ struct mlx5e_rss_params_hash;
 
 enum mlx5e_rx_res_features {
 	MLX5E_RX_RES_FEATURE_INNER_FT = BIT(0),
-	MLX5E_RX_RES_FEATURE_XSK = BIT(1),
-	MLX5E_RX_RES_FEATURE_PTP = BIT(2),
+	MLX5E_RX_RES_FEATURE_PTP = BIT(1),
 };
 
 /* Setup */
@@ -32,7 +31,6 @@ void mlx5e_rx_res_free(struct mlx5e_rx_res *res);
 
 /* TIRN getters for flow steering */
 u32 mlx5e_rx_res_get_tirn_direct(struct mlx5e_rx_res *res, unsigned int ix);
-u32 mlx5e_rx_res_get_tirn_xsk(struct mlx5e_rx_res *res, unsigned int ix);
 u32 mlx5e_rx_res_get_tirn_rss(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt);
 u32 mlx5e_rx_res_get_tirn_rss_inner(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt);
 u32 mlx5e_rx_res_get_tirn_ptp(struct mlx5e_rx_res *res);
@@ -40,9 +38,8 @@ u32 mlx5e_rx_res_get_tirn_ptp(struct mlx5e_rx_res *res);
 /* Activate/deactivate API */
 void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs);
 void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res);
-int mlx5e_rx_res_xsk_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs,
-			      unsigned int ix);
-int mlx5e_rx_res_xsk_deactivate(struct mlx5e_rx_res *res, unsigned int ix);
+void mlx5e_rx_res_xsk_update(struct mlx5e_rx_res *res, struct mlx5e_channels *chs,
+			     unsigned int ix, bool xsk);
 
 /* Configuration API */
 void mlx5e_rx_res_rss_set_indir_uniform(struct mlx5e_rx_res *res, unsigned int nch);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c
index 69949ab830b6..25174f68613e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c
@@ -12,6 +12,7 @@ validate_goto_chain(struct mlx5e_priv *priv,
 		    const struct flow_action_entry *act,
 		    struct netlink_ext_ack *extack)
 {
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	bool is_esw = mlx5e_is_eswitch_flow(flow);
 	bool ft_flow = mlx5e_is_ft_flow(flow);
 	u32 dest_chain = act->chain_index;
@@ -21,7 +22,7 @@ validate_goto_chain(struct mlx5e_priv *priv,
 	u32 max_chain;
 
 	esw = priv->mdev->priv.eswitch;
-	chains = is_esw ? esw_chains(esw) : mlx5e_nic_chains(priv->fs->tc);
+	chains = is_esw ? esw_chains(esw) : mlx5e_nic_chains(tc);
 	max_chain = mlx5_chains_get_chain_range(chains);
 	reformat_and_fwd = is_esw ?
 			   MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev, reformat_and_fwd_to_table) :
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c
index 11f2a7fb72a9..201ac7dd338f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/trap.c
@@ -147,7 +147,7 @@ static struct mlx5e_trap *mlx5e_open_trap(struct mlx5e_priv *priv)
 	t->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.hw_objs.mkey);
 	t->stats    = &priv->trap_stats.ch;
 
-	netif_napi_add(netdev, &t->napi, mlx5e_trap_napi_poll, 64);
+	netif_napi_add(netdev, &t->napi, mlx5e_trap_napi_poll);
 
 	err = mlx5e_open_trap_rq(priv, t);
 	if (unlikely(err))
@@ -230,12 +230,12 @@ static int mlx5e_handle_action_trap(struct mlx5e_priv *priv, int trap_id)
 
 	switch (trap_id) {
 	case DEVLINK_TRAP_GENERIC_ID_INGRESS_VLAN_FILTER:
-		err = mlx5e_add_vlan_trap(priv, trap_id, mlx5e_trap_get_tirn(priv->en_trap));
+		err = mlx5e_add_vlan_trap(priv->fs, trap_id, mlx5e_trap_get_tirn(priv->en_trap));
 		if (err)
 			goto err_out;
 		break;
 	case DEVLINK_TRAP_GENERIC_ID_DMAC_FILTER:
-		err = mlx5e_add_mac_trap(priv, trap_id, mlx5e_trap_get_tirn(priv->en_trap));
+		err = mlx5e_add_mac_trap(priv->fs, trap_id, mlx5e_trap_get_tirn(priv->en_trap));
 		if (err)
 			goto err_out;
 		break;
@@ -256,10 +256,10 @@ static int mlx5e_handle_action_drop(struct mlx5e_priv *priv, int trap_id)
 {
 	switch (trap_id) {
 	case DEVLINK_TRAP_GENERIC_ID_INGRESS_VLAN_FILTER:
-		mlx5e_remove_vlan_trap(priv);
+		mlx5e_remove_vlan_trap(priv->fs);
 		break;
 	case DEVLINK_TRAP_GENERIC_ID_DMAC_FILTER:
-		mlx5e_remove_mac_trap(priv);
+		mlx5e_remove_mac_trap(priv->fs);
 		break;
 	default:
 		netdev_warn(priv->netdev, "%s: Unknown trap id %d\n", __func__, trap_id);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
index c208ea307bff..4456ad5cedf1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/txrx.h
@@ -439,16 +439,24 @@ static inline u16 mlx5e_stop_room_for_max_wqe(struct mlx5_core_dev *mdev)
 	return MLX5E_STOP_ROOM(mlx5e_get_max_sq_wqebbs(mdev));
 }
 
-static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size)
+static inline u16 mlx5e_stop_room_for_mpwqe(struct mlx5_core_dev *mdev)
 {
-	u16 room = sq->reserved_room;
+	u8 mpwqe_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
 
-	WARN_ONCE(wqe_size > sq->max_sq_wqebbs,
-		  "wqe_size %u is greater than max SQ WQEBBs %u",
-		  wqe_size, sq->max_sq_wqebbs);
+	return mlx5e_stop_room_for_wqe(mdev, mpwqe_wqebbs);
+}
 
-	room += MLX5E_STOP_ROOM(wqe_size);
+static inline bool mlx5e_icosq_can_post_wqe(struct mlx5e_icosq *sq, u16 wqe_size)
+{
+	u16 room = sq->reserved_room + MLX5E_STOP_ROOM(wqe_size);
 
 	return mlx5e_wqc_has_room_for(&sq->wq, sq->cc, sq->pc, room);
 }
+
+static inline struct mlx5e_mpw_info *mlx5e_get_mpw_info(struct mlx5e_rq *rq, int i)
+{
+	size_t isz = struct_size(rq->mpwqe.info, alloc_units, rq->mpwqe.pages_per_wqe);
+
+	return (struct mlx5e_mpw_info *)((char *)rq->mpwqe.info + array_size(i, isz));
+}
 #endif
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index 8f321a6c0809..4685c652c97e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -333,7 +333,7 @@ mlx5e_xmit_xdp_frame_mpwqe(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptx
 
 	mlx5e_xdp_mpwqe_add_dseg(sq, xdptxd, stats);
 
-	if (unlikely(mlx5e_xdp_mpqwe_is_full(session, sq->max_sq_mpw_wqebbs)))
+	if (unlikely(mlx5e_xdp_mpwqe_is_full(session, sq->max_sq_mpw_wqebbs)))
 		mlx5e_xdp_mpwqe_complete(sq);
 
 	stats->xmit++;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
index 287e17911251..bc2d9034af5b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
@@ -122,7 +122,7 @@ static inline bool mlx5e_xdp_get_inline_state(struct mlx5e_xdpsq *sq, bool cur)
 	return cur;
 }
 
-static inline bool mlx5e_xdp_mpqwe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs)
+static inline bool mlx5e_xdp_mpwqe_is_full(struct mlx5e_tx_mpwqe *session, u8 max_sq_mpw_wqebbs)
 {
 	if (session->inline_on)
 		return session->ds_count + MLX5E_XDP_INLINE_WQE_MAX_DS_CNT >
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
index 2c520394aa1d..ebada0c5af3c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c
@@ -72,6 +72,7 @@ void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *x
 {
 	xsk->headroom = xsk_pool_get_headroom(pool);
 	xsk->chunk_size = xsk_pool_get_chunk_size(pool);
+	xsk->unaligned = pool->unaligned;
 }
 
 static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
@@ -98,6 +99,15 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
 
 	mlx5e_build_xsk_param(pool, &xsk);
 
+	if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
+	    mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) {
+		const char *recommendation = is_power_of_2(xsk.chunk_size) ?
+			"Upgrade firmware" : "Disable striding RQ";
+
+		mlx5_core_warn(priv->mdev, "Expected slowdown with XSK frame size %u. %s for better performance.\n",
+			       xsk.chunk_size, recommendation);
+	}
+
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
 		/* XSK objects will be created on open. */
 		goto validate_closed;
@@ -123,15 +133,12 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
 	 * any Fill Ring entries at the setup stage.
 	 */
 
-	err = mlx5e_rx_res_xsk_activate(priv->rx_res, &priv->channels, ix);
-	if (unlikely(err))
-		goto err_deactivate;
+	mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, true);
 
-	return 0;
+	mlx5e_deactivate_rq(&c->rq);
+	mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY);
 
-err_deactivate:
-	mlx5e_deactivate_xsk(c);
-	mlx5e_close_xsk(c);
+	return 0;
 
 err_remove_pool:
 	mlx5e_xsk_remove_pool(&priv->xsk, ix);
@@ -170,7 +177,13 @@ static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix)
 		goto remove_pool;
 
 	c = priv->channels.c[ix];
-	mlx5e_rx_res_xsk_deactivate(priv->rx_res, ix);
+
+	mlx5e_activate_rq(&c->rq);
+	mlx5e_trigger_napi_icosq(c);
+	mlx5e_wait_for_min_rx_wqes(&c->rq, MLX5E_RQ_WQES_TIMEOUT);
+
+	mlx5e_rx_res_xsk_update(priv->rx_res, &priv->channels, ix, false);
+
 	mlx5e_deactivate_xsk(c);
 	mlx5e_close_xsk(c);
 
@@ -208,11 +221,10 @@ int mlx5e_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	struct mlx5e_params *params = &priv->channels.params;
-	u16 ix;
 
-	if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix)))
+	if (unlikely(qid >= params->num_channels))
 		return -EINVAL;
 
-	return pool ? mlx5e_xsk_enable_pool(priv, pool, ix) :
-		      mlx5e_xsk_disable_pool(priv, ix);
+	return pool ? mlx5e_xsk_enable_pool(priv, pool, qid) :
+		      mlx5e_xsk_disable_pool(priv, qid);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
index 9a1553598a7c..c91b54d9ff27 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.c
@@ -8,18 +8,221 @@
 
 /* RX data path */
 
-static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, void *data,
-					       u32 cqe_bcnt)
+int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 {
+	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix);
+	struct mlx5e_icosq *icosq = rq->icosq;
+	struct mlx5_wq_cyc *wq = &icosq->wq;
+	struct mlx5e_umr_wqe *umr_wqe;
+	int batch, i;
+	u32 offset; /* 17-bit value with MTT. */
+	u16 pi;
+
+	if (unlikely(!xsk_buff_can_alloc(rq->xsk_pool, rq->mpwqe.pages_per_wqe)))
+		goto err;
+
+	BUILD_BUG_ON(sizeof(wi->alloc_units[0]) != sizeof(wi->alloc_units[0].xsk));
+	batch = xsk_buff_alloc_batch(rq->xsk_pool, (struct xdp_buff **)wi->alloc_units,
+				     rq->mpwqe.pages_per_wqe);
+
+	/* If batch < pages_per_wqe, either:
+	 * 1. Some (or all) descriptors were invalid.
+	 * 2. dma_need_sync is true, and it fell back to allocating one frame.
+	 * In either case, try to continue allocating frames one by one, until
+	 * the first error, which will mean there are no more valid descriptors.
+	 */
+	for (; batch < rq->mpwqe.pages_per_wqe; batch++) {
+		wi->alloc_units[batch].xsk = xsk_buff_alloc(rq->xsk_pool);
+		if (unlikely(!wi->alloc_units[batch].xsk))
+			goto err_reuse_batch;
+	}
+
+	pi = mlx5e_icosq_get_next_pi(icosq, rq->mpwqe.umr_wqebbs);
+	umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
+	memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe));
+
+	if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED)) {
+		for (i = 0; i < batch; i++) {
+			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);
+
+			umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {
+				.ptag = cpu_to_be64(addr | MLX5_EN_WR),
+			};
+		}
+	} else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED)) {
+		for (i = 0; i < batch; i++) {
+			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);
+
+			umr_wqe->inline_ksms[i] = (struct mlx5_ksm) {
+				.key = rq->mkey_be,
+				.va = cpu_to_be64(addr),
+			};
+		}
+	} else if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)) {
+		u32 mapping_size = 1 << (rq->mpwqe.page_shift - 2);
+
+		for (i = 0; i < batch; i++) {
+			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);
+
+			umr_wqe->inline_ksms[i << 2] = (struct mlx5_ksm) {
+				.key = rq->mkey_be,
+				.va = cpu_to_be64(addr),
+			};
+			umr_wqe->inline_ksms[(i << 2) + 1] = (struct mlx5_ksm) {
+				.key = rq->mkey_be,
+				.va = cpu_to_be64(addr + mapping_size),
+			};
+			umr_wqe->inline_ksms[(i << 2) + 2] = (struct mlx5_ksm) {
+				.key = rq->mkey_be,
+				.va = cpu_to_be64(addr + mapping_size * 2),
+			};
+			umr_wqe->inline_ksms[(i << 2) + 3] = (struct mlx5_ksm) {
+				.key = rq->mkey_be,
+				.va = cpu_to_be64(rq->wqe_overflow.addr),
+			};
+		}
+	} else {
+		__be32 pad_size = cpu_to_be32((1 << rq->mpwqe.page_shift) -
+					      rq->xsk_pool->chunk_size);
+		__be32 frame_size = cpu_to_be32(rq->xsk_pool->chunk_size);
+
+		for (i = 0; i < batch; i++) {
+			dma_addr_t addr = xsk_buff_xdp_get_frame_dma(wi->alloc_units[i].xsk);
+
+			umr_wqe->inline_klms[i << 1] = (struct mlx5_klm) {
+				.key = rq->mkey_be,
+				.va = cpu_to_be64(addr),
+				.bcount = frame_size,
+			};
+			umr_wqe->inline_klms[(i << 1) + 1] = (struct mlx5_klm) {
+				.key = rq->mkey_be,
+				.va = cpu_to_be64(rq->wqe_overflow.addr),
+				.bcount = pad_size,
+			};
+		}
+	}
+
+	bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe);
+	wi->consumed_strides = 0;
+
+	umr_wqe->ctrl.opmod_idx_opcode =
+		cpu_to_be32((icosq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | MLX5_OPCODE_UMR);
+
+	/* Optimized for speed: keep in sync with mlx5e_mpwrq_umr_entry_size. */
+	offset = ix * rq->mpwqe.mtts_per_wqe;
+	if (likely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_ALIGNED))
+		offset = offset * sizeof(struct mlx5_mtt) / MLX5_OCTWORD;
+	else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_OVERSIZED))
+		offset = offset * sizeof(struct mlx5_klm) * 2 / MLX5_OCTWORD;
+	else if (unlikely(rq->mpwqe.umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE))
+		offset = offset * sizeof(struct mlx5_ksm) * 4 / MLX5_OCTWORD;
+	umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset);
+
+	icosq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {
+		.wqe_type = MLX5E_ICOSQ_WQE_UMR_RX,
+		.num_wqebbs = rq->mpwqe.umr_wqebbs,
+		.umr.rq = rq,
+	};
+
+	icosq->pc += rq->mpwqe.umr_wqebbs;
+
+	icosq->doorbell_cseg = &umr_wqe->ctrl;
+
+	return 0;
+
+err_reuse_batch:
+	while (--batch >= 0)
+		xsk_buff_free(wi->alloc_units[batch].xsk);
+
+err:
+	rq->stats->buff_alloc_err++;
+	return -ENOMEM;
+}
+
+int mlx5e_xsk_alloc_rx_wqes_batched(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
+{
+	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
+	struct xdp_buff **buffs;
+	u32 contig, alloc;
+	int i;
+
+	/* mlx5e_init_frags_partition creates a 1:1 mapping between
+	 * rq->wqe.frags and rq->wqe.alloc_units, which allows us to
+	 * allocate XDP buffers straight into alloc_units.
+	 */
+	BUILD_BUG_ON(sizeof(rq->wqe.alloc_units[0]) !=
+		     sizeof(rq->wqe.alloc_units[0].xsk));
+	buffs = (struct xdp_buff **)rq->wqe.alloc_units;
+	contig = mlx5_wq_cyc_get_size(wq) - ix;
+	if (wqe_bulk <= contig) {
+		alloc = xsk_buff_alloc_batch(rq->xsk_pool, buffs + ix, wqe_bulk);
+	} else {
+		alloc = xsk_buff_alloc_batch(rq->xsk_pool, buffs + ix, contig);
+		if (likely(alloc == contig))
+			alloc += xsk_buff_alloc_batch(rq->xsk_pool, buffs, wqe_bulk - contig);
+	}
+
+	for (i = 0; i < alloc; i++) {
+		int j = mlx5_wq_cyc_ctr2ix(wq, ix + i);
+		struct mlx5e_wqe_frag_info *frag;
+		struct mlx5e_rx_wqe_cyc *wqe;
+		dma_addr_t addr;
+
+		wqe = mlx5_wq_cyc_get_wqe(wq, j);
+		/* Assumes log_num_frags == 0. */
+		frag = &rq->wqe.frags[j];
+
+		addr = xsk_buff_xdp_get_frame_dma(frag->au->xsk);
+		wqe->data[0].addr = cpu_to_be64(addr + rq->buff.headroom);
+	}
+
+	return alloc;
+}
+
+int mlx5e_xsk_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
+{
+	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
+	int i;
+
+	for (i = 0; i < wqe_bulk; i++) {
+		int j = mlx5_wq_cyc_ctr2ix(wq, ix + i);
+		struct mlx5e_wqe_frag_info *frag;
+		struct mlx5e_rx_wqe_cyc *wqe;
+		dma_addr_t addr;
+
+		wqe = mlx5_wq_cyc_get_wqe(wq, j);
+		/* Assumes log_num_frags == 0. */
+		frag = &rq->wqe.frags[j];
+
+		frag->au->xsk = xsk_buff_alloc(rq->xsk_pool);
+		if (unlikely(!frag->au->xsk))
+			return i;
+
+		addr = xsk_buff_xdp_get_frame_dma(frag->au->xsk);
+		wqe->data[0].addr = cpu_to_be64(addr + rq->buff.headroom);
+	}
+
+	return wqe_bulk;
+}
+
+static struct sk_buff *mlx5e_xsk_construct_skb(struct mlx5e_rq *rq, struct xdp_buff *xdp)
+{
+	u32 totallen = xdp->data_end - xdp->data_meta;
+	u32 metalen = xdp->data - xdp->data_meta;
 	struct sk_buff *skb;
 
-	skb = napi_alloc_skb(rq->cq.napi, cqe_bcnt);
+	skb = napi_alloc_skb(rq->cq.napi, totallen);
 	if (unlikely(!skb)) {
 		rq->stats->buff_alloc_err++;
 		return NULL;
 	}
 
-	skb_put_data(skb, data, cqe_bcnt);
+	skb_put_data(skb, xdp->data_meta, totallen);
+
+	if (metalen) {
+		skb_metadata_set(skb, metalen);
+		__skb_pull(skb, metalen);
+	}
 
 	return skb;
 }
@@ -30,7 +233,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
 						    u32 head_offset,
 						    u32 page_idx)
 {
-	struct xdp_buff *xdp = wi->umr.dma_info[page_idx].xsk;
+	struct xdp_buff *xdp = wi->alloc_units[page_idx].xsk;
 	struct bpf_prog *prog;
 
 	/* Check packet size. Note LRO doesn't use linear SKB */
@@ -46,8 +249,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
 	 */
 	WARN_ON_ONCE(head_offset);
 
-	xdp->data_end = xdp->data + cqe_bcnt;
-	xdp_set_data_meta_invalid(xdp);
+	xsk_buff_set_size(xdp, cqe_bcnt);
 	xsk_buff_dma_sync_for_cpu(xdp, rq->xsk_pool);
 	net_prefetch(xdp->data);
 
@@ -76,14 +278,14 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
 	/* XDP_PASS: copy the data from the UMEM to a new SKB and reuse the
 	 * frame. On SKB allocation failure, NULL is returned.
 	 */
-	return mlx5e_xsk_construct_skb(rq, xdp->data, xdp->data_end - xdp->data);
+	return mlx5e_xsk_construct_skb(rq, xdp);
 }
 
 struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
 					      struct mlx5e_wqe_frag_info *wi,
 					      u32 cqe_bcnt)
 {
-	struct xdp_buff *xdp = wi->di->xsk;
+	struct xdp_buff *xdp = wi->au->xsk;
 	struct bpf_prog *prog;
 
 	/* wi->offset is not used in this function, because xdp->data and the
@@ -93,8 +295,7 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
 	 */
 	WARN_ON_ONCE(wi->offset);
 
-	xdp->data_end = xdp->data + cqe_bcnt;
-	xdp_set_data_meta_invalid(xdp);
+	xsk_buff_set_size(xdp, cqe_bcnt);
 	xsk_buff_dma_sync_for_cpu(xdp, rq->xsk_pool);
 	net_prefetch(xdp->data);
 
@@ -103,8 +304,8 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
 		return NULL; /* page/packet was consumed by XDP */
 
 	/* XDP_PASS: copy the data from the UMEM to a new SKB. The frame reuse
-	 * will be handled by mlx5e_put_rx_frag.
+	 * will be handled by mlx5e_free_rx_wqe.
 	 * On SKB allocation failure, NULL is returned.
 	 */
-	return mlx5e_xsk_construct_skb(rq, xdp->data, xdp->data_end - xdp->data);
+	return mlx5e_xsk_construct_skb(rq, xdp);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
index cc18d97d8ee0..087c943bd8e9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/rx.h
@@ -5,12 +5,12 @@
 #define __MLX5_EN_XSK_RX_H__
 
 #include "en.h"
-#include <net/xdp_sock_drv.h>
-
-#define MLX5E_MTT_PTAG_MASK 0xfffffffffffffff8ULL
 
 /* RX data path */
 
+int mlx5e_xsk_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
+int mlx5e_xsk_alloc_rx_wqes_batched(struct mlx5e_rq *rq, u16 ix, int wqe_bulk);
+int mlx5e_xsk_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk);
 struct sk_buff *mlx5e_xsk_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
 						    struct mlx5e_mpw_info *wi,
 						    u16 cqe_bcnt,
@@ -20,46 +20,4 @@ struct sk_buff *mlx5e_xsk_skb_from_cqe_linear(struct mlx5e_rq *rq,
 					      struct mlx5e_wqe_frag_info *wi,
 					      u32 cqe_bcnt);
 
-static inline int mlx5e_xsk_page_alloc_pool(struct mlx5e_rq *rq,
-					    struct mlx5e_dma_info *dma_info)
-{
-retry:
-	dma_info->xsk = xsk_buff_alloc(rq->xsk_pool);
-	if (!dma_info->xsk)
-		return -ENOMEM;
-
-	/* Store the DMA address without headroom. In striding RQ case, we just
-	 * provide pages for UMR, and headroom is counted at the setup stage
-	 * when creating a WQE. In non-striding RQ case, headroom is accounted
-	 * in mlx5e_alloc_rx_wqe.
-	 */
-	dma_info->addr = xsk_buff_xdp_get_frame_dma(dma_info->xsk);
-
-	/* MTT page mapping has alignment requirements. If they are not
-	 * satisfied, leak the descriptor so that it won't come again, and try
-	 * to allocate a new one.
-	 */
-	if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
-		if (unlikely(dma_info->addr & ~MLX5E_MTT_PTAG_MASK)) {
-			xsk_buff_discard(dma_info->xsk);
-			goto retry;
-		}
-	}
-
-	return 0;
-}
-
-static inline bool mlx5e_xsk_update_rx_wakeup(struct mlx5e_rq *rq, bool alloc_err)
-{
-	if (!xsk_uses_need_wakeup(rq->xsk_pool))
-		return alloc_err;
-
-	if (unlikely(alloc_err))
-		xsk_set_rx_need_wakeup(rq->xsk_pool);
-	else
-		xsk_clear_rx_need_wakeup(rq->xsk_pool);
-
-	return false;
-}
-
 #endif /* __MLX5_EN_XSK_RX_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
index 98ed9ef3a6bd..ff03c43833bb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
@@ -5,24 +5,19 @@
 #include "en/params.h"
 #include "en/txrx.h"
 #include "en/health.h"
+#include <net/xdp_sock_drv.h>
 
-/* It matches XDP_UMEM_MIN_CHUNK_SIZE, but as this constant is private and may
- * change unexpectedly, and mlx5e has a minimum valid stride size for striding
- * RQ, keep this check in the driver.
+/* The limitation of 2048 can be altered, but shouldn't go beyond the minimal
+ * stride size of striding RQ.
  */
-#define MLX5E_MIN_XSK_CHUNK_SIZE 2048
+#define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE)
 
 bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
 			      struct mlx5e_xsk_param *xsk,
 			      struct mlx5_core_dev *mdev)
 {
 	/* AF_XDP doesn't support frames larger than PAGE_SIZE. */
-	if (xsk->chunk_size > PAGE_SIZE ||
-			xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE)
-		return false;
-
-	/* Current MTU and XSK headroom don't allow packets to fit the frames. */
-	if (mlx5e_rx_get_min_frag_sz(params, xsk) > xsk->chunk_size)
+	if (xsk->chunk_size > PAGE_SIZE || xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE)
 		return false;
 
 	/* frag_sz is different for regular and XSK RQs, so ensure that linear
@@ -30,9 +25,9 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
 	 */
 	switch (params->rq_wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
-		return mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk);
+		return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk);
 	default: /* MLX5_WQ_TYPE_CYCLIC */
-		return mlx5e_rx_is_linear_skb(params, xsk);
+		return mlx5e_rx_is_linear_skb(mdev, params, xsk);
 	}
 }
 
@@ -71,7 +66,7 @@ static int mlx5e_init_xsk_rq(struct mlx5e_channel *c,
 	rq->xsk_pool     = pool;
 	rq->stats        = &c->priv->channel_stats[c->ix]->xskrq;
 	rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev);
-	rq_xdp_ix        = c->ix + params->num_channels * MLX5E_RQ_GROUP_XSK;
+	rq_xdp_ix        = c->ix;
 	err = mlx5e_rq_set_handlers(rq, params, xsk);
 	if (err)
 		return err;
@@ -159,7 +154,7 @@ err_free_cparam:
 void mlx5e_close_xsk(struct mlx5e_channel *c)
 {
 	clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
-	synchronize_net(); /* Sync with the XSK wakeup and with NAPI. */
+	synchronize_net(); /* Sync with NAPI. */
 
 	mlx5e_close_rq(&c->xskrq);
 	mlx5e_close_cq(&c->xskrq.cq);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
index 4902ef74fedf..367a9505ca4f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.c
@@ -12,18 +12,14 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	struct mlx5e_params *params = &priv->channels.params;
 	struct mlx5e_channel *c;
-	u16 ix;
 
 	if (unlikely(!mlx5e_xdp_is_active(priv)))
 		return -ENETDOWN;
 
-	if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix)))
+	if (unlikely(qid >= params->num_channels))
 		return -EINVAL;
 
-	c = priv->channels.c[ix];
-
-	if (unlikely(!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)))
-		return -EINVAL;
+	c = priv->channels.c[qid];
 
 	if (!napi_if_scheduled_mark_missed(&c->napi)) {
 		/* To avoid WQE overrun, don't post a NOP if async_icosq is not
@@ -36,9 +32,7 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
 		if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state))
 			return 0;
 
-		spin_lock_bh(&c->async_icosq_lock);
-		mlx5e_trigger_irq(&c->async_icosq);
-		spin_unlock_bh(&c->async_icosq_lock);
+		mlx5e_trigger_napi_icosq(c);
 	}
 
 	return 0;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
index a05085035f23..9c505158b975 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/tx.h
@@ -5,7 +5,6 @@
 #define __MLX5_EN_XSK_TX_H__
 
 #include "en.h"
-#include <net/xdp_sock_drv.h>
 
 /* TX data path */
 
@@ -13,15 +12,4 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags);
 
 bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget);
 
-static inline void mlx5e_xsk_update_tx_wakeup(struct mlx5e_xdpsq *sq)
-{
-	if (!xsk_uses_need_wakeup(sq->xsk_pool))
-		return;
-
-	if (sq->pc != sq->cc)
-		xsk_clear_tx_need_wakeup(sq->xsk_pool);
-	else
-		xsk_set_tx_need_wakeup(sq->xsk_pool);
-}
-
 #endif /* __MLX5_EN_XSK_TX_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h
index 1839f1ab1ddd..07187028f0d3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h
@@ -39,6 +39,7 @@
 #include "en_accel/ipsec_rxtx.h"
 #include "en_accel/ktls.h"
 #include "en_accel/ktls_txrx.h"
+#include <en_accel/macsec.h>
 #include "en.h"
 #include "en/txrx.h"
 
@@ -137,6 +138,15 @@ static inline bool mlx5e_accel_tx_begin(struct net_device *dev,
 	}
 #endif
 
+#ifdef CONFIG_MLX5_EN_MACSEC
+	if (unlikely(mlx5e_macsec_skb_is_offload(skb))) {
+		struct mlx5e_priv *priv = netdev_priv(dev);
+
+		if (unlikely(!mlx5e_macsec_handle_tx_skb(priv->macsec, skb)))
+			return false;
+	}
+#endif
+
 	return true;
 }
 
@@ -163,6 +173,11 @@ static inline void mlx5e_accel_tx_eseg(struct mlx5e_priv *priv,
 		mlx5e_ipsec_tx_build_eseg(priv, skb, eseg);
 #endif
 
+#ifdef CONFIG_MLX5_EN_MACSEC
+	if (unlikely(mlx5e_macsec_skb_is_offload(skb)))
+		mlx5e_macsec_tx_build_eseg(priv->macsec, skb, eseg);
+#endif
+
 #if IS_ENABLED(CONFIG_GENEVE)
 	if (skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL)
 		mlx5e_tx_tunnel_accel(skb, eseg, ihs);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c
index 20a4f1e585af..285d32d2fd08 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */
 
-#include <linux/netdevice.h>
+#include <mlx5_core.h>
 #include "en_accel/fs_tcp.h"
 #include "fs_core.h"
 
@@ -71,13 +71,13 @@ void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule)
 	mlx5_del_flow_rules(rule);
 }
 
-struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv,
+struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_flow_steering *fs,
 					       struct sock *sk, u32 tirn,
 					       uint32_t flow_tag)
 {
+	struct mlx5e_accel_fs_tcp *fs_tcp = mlx5e_fs_get_accel_tcp(fs);
 	struct mlx5_flow_destination dest = {};
 	struct mlx5e_flow_table *ft = NULL;
-	struct mlx5e_accel_fs_tcp *fs_tcp;
 	MLX5_DECLARE_FLOW_ACT(flow_act);
 	struct mlx5_flow_handle *flow;
 	struct mlx5_flow_spec *spec;
@@ -86,19 +86,17 @@ struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv,
 	if (!spec)
 		return ERR_PTR(-ENOMEM);
 
-	fs_tcp = priv->fs->accel_tcp;
-
 	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
 
 	switch (sk->sk_family) {
 	case AF_INET:
 		accel_fs_tcp_set_ipv4_flow(spec, sk);
 		ft = &fs_tcp->tables[ACCEL_FS_IPV4_TCP];
-		mlx5e_dbg(HW, priv, "%s flow is %pI4:%d -> %pI4:%d\n", __func__,
-			  &inet_sk(sk)->inet_rcv_saddr,
-			  inet_sk(sk)->inet_sport,
-			  &inet_sk(sk)->inet_daddr,
-			  inet_sk(sk)->inet_dport);
+		fs_dbg(fs, "%s flow is %pI4:%d -> %pI4:%d\n", __func__,
+		       &inet_sk(sk)->inet_rcv_saddr,
+		       inet_sk(sk)->inet_sport,
+		       &inet_sk(sk)->inet_daddr,
+		       inet_sk(sk)->inet_dport);
 		break;
 #if IS_ENABLED(CONFIG_IPV6)
 	case AF_INET6:
@@ -140,34 +138,32 @@ struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv,
 	flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1);
 
 	if (IS_ERR(flow))
-		netdev_err(priv->netdev, "mlx5_add_flow_rules() failed, flow is %ld\n",
-			   PTR_ERR(flow));
+		fs_err(fs, "mlx5_add_flow_rules() failed, flow is %ld\n", PTR_ERR(flow));
 
 out:
 	kvfree(spec);
 	return flow;
 }
 
-static int accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv,
+static int accel_fs_tcp_add_default_rule(struct mlx5e_flow_steering *fs,
 					 enum accel_fs_tcp_type type)
 {
+	struct mlx5e_accel_fs_tcp *fs_tcp = mlx5e_fs_get_accel_tcp(fs);
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
 	struct mlx5e_flow_table *accel_fs_t;
 	struct mlx5_flow_destination dest;
-	struct mlx5e_accel_fs_tcp *fs_tcp;
 	MLX5_DECLARE_FLOW_ACT(flow_act);
 	struct mlx5_flow_handle *rule;
 	int err = 0;
 
-	fs_tcp = priv->fs->accel_tcp;
 	accel_fs_t = &fs_tcp->tables[type];
 
-	dest = mlx5_ttc_get_default_dest(priv->fs->ttc, fs_accel2tt(type));
+	dest = mlx5_ttc_get_default_dest(ttc, fs_accel2tt(type));
 	rule = mlx5_add_flow_rules(accel_fs_t->t, NULL, &flow_act, &dest, 1);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
-		netdev_err(priv->netdev,
-			   "%s: add default rule failed, accel_fs type=%d, err %d\n",
-			   __func__, type, err);
+		fs_err(fs, "%s: add default rule failed, accel_fs type=%d, err %d\n",
+		       __func__, type, err);
 		return err;
 	}
 
@@ -265,9 +261,11 @@ out:
 	return err;
 }
 
-static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_type type)
+static int accel_fs_tcp_create_table(struct mlx5e_flow_steering *fs, enum accel_fs_tcp_type type)
 {
-	struct mlx5e_flow_table *ft = &priv->fs->accel_tcp->tables[type];
+	struct mlx5e_accel_fs_tcp *accel_tcp = mlx5e_fs_get_accel_tcp(fs);
+	struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false);
+	struct mlx5e_flow_table *ft = &accel_tcp->tables[type];
 	struct mlx5_flow_table_attr ft_attr = {};
 	int err;
 
@@ -277,21 +275,21 @@ static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_
 	ft_attr.level = MLX5E_ACCEL_FS_TCP_FT_LEVEL;
 	ft_attr.prio = MLX5E_NIC_PRIO;
 
-	ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr);
+	ft->t = mlx5_create_flow_table(ns, &ft_attr);
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
 		ft->t = NULL;
 		return err;
 	}
 
-	netdev_dbg(priv->netdev, "Created fs accel table id %u level %u\n",
-		   ft->t->id, ft->t->level);
+	fs_dbg(fs, "Created fs accel table id %u level %u\n",
+	       ft->t->id, ft->t->level);
 
 	err = accel_fs_tcp_create_groups(ft, type);
 	if (err)
 		goto err;
 
-	err = accel_fs_tcp_add_default_rule(priv, type);
+	err = accel_fs_tcp_add_default_rule(fs, type);
 	if (err)
 		goto err;
 
@@ -301,17 +299,18 @@ err:
 	return err;
 }
 
-static int accel_fs_tcp_disable(struct mlx5e_priv *priv)
+static int accel_fs_tcp_disable(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
 	int err, i;
 
 	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
 		/* Modify ttc rules destination to point back to the indir TIRs */
-		err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, fs_accel2tt(i));
+		err = mlx5_ttc_fwd_default_dest(ttc, fs_accel2tt(i));
 		if (err) {
-			netdev_err(priv->netdev,
-				   "%s: modify ttc[%d] default destination failed, err(%d)\n",
-				   __func__, fs_accel2tt(i), err);
+			fs_err(fs,
+			       "%s: modify ttc[%d] default destination failed, err(%d)\n",
+			       __func__, fs_accel2tt(i), err);
 			return err;
 		}
 	}
@@ -319,32 +318,32 @@ static int accel_fs_tcp_disable(struct mlx5e_priv *priv)
 	return 0;
 }
 
-static int accel_fs_tcp_enable(struct mlx5e_priv *priv)
+static int accel_fs_tcp_enable(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5e_accel_fs_tcp *accel_tcp = mlx5e_fs_get_accel_tcp(fs);
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
 	struct mlx5_flow_destination dest = {};
 	int err, i;
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
-		dest.ft = priv->fs->accel_tcp->tables[i].t;
+		dest.ft = accel_tcp->tables[i].t;
 
 		/* Modify ttc rules destination to point on the accel_fs FTs */
-		err = mlx5_ttc_fwd_dest(priv->fs->ttc, fs_accel2tt(i), &dest);
+		err = mlx5_ttc_fwd_dest(ttc, fs_accel2tt(i), &dest);
 		if (err) {
-			netdev_err(priv->netdev,
-				   "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
-				   __func__, fs_accel2tt(i), err);
+			fs_err(fs, "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
+			       __func__, fs_accel2tt(i), err);
 			return err;
 		}
 	}
 	return 0;
 }
 
-static void accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i)
+static void accel_fs_tcp_destroy_table(struct mlx5e_flow_steering *fs, int i)
 {
-	struct mlx5e_accel_fs_tcp *fs_tcp;
+	struct mlx5e_accel_fs_tcp *fs_tcp = mlx5e_fs_get_accel_tcp(fs);
 
-	fs_tcp = priv->fs->accel_tcp;
 	if (IS_ERR_OR_NULL(fs_tcp->tables[i].t))
 		return;
 
@@ -353,40 +352,43 @@ static void accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i)
 	fs_tcp->tables[i].t = NULL;
 }
 
-void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv)
+void mlx5e_accel_fs_tcp_destroy(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5e_accel_fs_tcp *accel_tcp = mlx5e_fs_get_accel_tcp(fs);
 	int i;
 
-	if (!priv->fs->accel_tcp)
+	if (!accel_tcp)
 		return;
 
-	accel_fs_tcp_disable(priv);
+	accel_fs_tcp_disable(fs);
 
 	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++)
-		accel_fs_tcp_destroy_table(priv, i);
+		accel_fs_tcp_destroy_table(fs, i);
 
-	kfree(priv->fs->accel_tcp);
-	priv->fs->accel_tcp = NULL;
+	kfree(accel_tcp);
+	mlx5e_fs_set_accel_tcp(fs, NULL);
 }
 
-int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv)
+int mlx5e_accel_fs_tcp_create(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5e_accel_fs_tcp *accel_tcp;
 	int i, err;
 
-	if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version))
+	if (!MLX5_CAP_FLOWTABLE_NIC_RX(mlx5e_fs_get_mdev(fs), ft_field_support.outer_ip_version))
 		return -EOPNOTSUPP;
 
-	priv->fs->accel_tcp = kzalloc(sizeof(*priv->fs->accel_tcp), GFP_KERNEL);
-	if (!priv->fs->accel_tcp)
+	accel_tcp = kvzalloc(sizeof(*accel_tcp), GFP_KERNEL);
+	if (!accel_tcp)
 		return -ENOMEM;
+	mlx5e_fs_set_accel_tcp(fs, accel_tcp);
 
 	for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) {
-		err = accel_fs_tcp_create_table(priv, i);
+		err = accel_fs_tcp_create_table(fs, i);
 		if (err)
 			goto err_destroy_tables;
 	}
 
-	err = accel_fs_tcp_enable(priv);
+	err = accel_fs_tcp_enable(fs);
 	if (err)
 		goto err_destroy_tables;
 
@@ -394,9 +396,8 @@ int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv)
 
 err_destroy_tables:
 	while (--i >= 0)
-		accel_fs_tcp_destroy_table(priv, i);
-
-	kfree(priv->fs->accel_tcp);
-	priv->fs->accel_tcp = NULL;
+		accel_fs_tcp_destroy_table(fs, i);
+	kfree(accel_tcp);
+	mlx5e_fs_set_accel_tcp(fs, NULL);
 	return err;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h
index 589235824543..a032bff482a6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.h
@@ -4,19 +4,19 @@
 #ifndef __MLX5E_ACCEL_FS_TCP_H__
 #define __MLX5E_ACCEL_FS_TCP_H__
 
-#include "en.h"
+#include "en/fs.h"
 
 #ifdef CONFIG_MLX5_EN_TLS
-int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv);
-void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv);
-struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv,
+int mlx5e_accel_fs_tcp_create(struct mlx5e_flow_steering *fs);
+void mlx5e_accel_fs_tcp_destroy(struct mlx5e_flow_steering *fs);
+struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_flow_steering *fs,
 					       struct sock *sk, u32 tirn,
 					       uint32_t flow_tag);
 void mlx5e_accel_fs_del_sk(struct mlx5_flow_handle *rule);
 #else
-static inline int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) { return 0; }
-static inline void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) {}
-static inline struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv,
+static inline int mlx5e_accel_fs_tcp_create(struct mlx5e_flow_steering *fs) { return 0; }
+static inline void mlx5e_accel_fs_tcp_destroy(struct mlx5e_flow_steering *fs) {}
+static inline struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_flow_steering *fs,
 							     struct sock *sk, u32 tirn,
 							     uint32_t flow_tag)
 { return ERR_PTR(-EOPNOTSUPP); }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
index f8113fd23265..b859e4a4c744 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
@@ -174,6 +174,8 @@ static void rx_destroy(struct mlx5e_priv *priv, enum accel_fs_esp_type type)
 
 static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type)
 {
+	struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(priv->fs, false);
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false);
 	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5e_accel_fs_esp_prot *fs_prot;
 	struct mlx5e_accel_fs_esp *accel_esp;
@@ -182,15 +184,14 @@ static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type)
 
 	accel_esp = priv->ipsec->rx_fs;
 	fs_prot = &accel_esp->fs_prot[type];
-
 	fs_prot->default_dest =
-		mlx5_ttc_get_default_dest(priv->fs->ttc, fs_esp2tt(type));
+		mlx5_ttc_get_default_dest(ttc, fs_esp2tt(type));
 
 	ft_attr.max_fte = 1;
 	ft_attr.autogroup.max_num_groups = 1;
 	ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL;
 	ft_attr.prio = MLX5E_NIC_PRIO;
-	ft = mlx5_create_auto_grouped_flow_table(priv->fs->ns, &ft_attr);
+	ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
 	if (IS_ERR(ft))
 		return PTR_ERR(ft);
 
@@ -205,7 +206,7 @@ static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type)
 	ft_attr.prio = MLX5E_NIC_PRIO;
 	ft_attr.autogroup.num_reserved_entries = 1;
 	ft_attr.autogroup.max_num_groups = 1;
-	ft = mlx5_create_auto_grouped_flow_table(priv->fs->ns, &ft_attr);
+	ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
 	if (IS_ERR(ft)) {
 		err = PTR_ERR(ft);
 		goto err_fs_ft;
@@ -230,6 +231,7 @@ err_add:
 
 static int rx_ft_get(struct mlx5e_priv *priv, enum accel_fs_esp_type type)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false);
 	struct mlx5e_accel_fs_esp_prot *fs_prot;
 	struct mlx5_flow_destination dest = {};
 	struct mlx5e_accel_fs_esp *accel_esp;
@@ -249,7 +251,7 @@ static int rx_ft_get(struct mlx5e_priv *priv, enum accel_fs_esp_type type)
 	/* connect */
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	dest.ft = fs_prot->ft;
-	mlx5_ttc_fwd_dest(priv->fs->ttc, fs_esp2tt(type), &dest);
+	mlx5_ttc_fwd_dest(ttc, fs_esp2tt(type), &dest);
 
 skip:
 	fs_prot->refcnt++;
@@ -260,6 +262,7 @@ out:
 
 static void rx_ft_put(struct mlx5e_priv *priv, enum accel_fs_esp_type type)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false);
 	struct mlx5e_accel_fs_esp_prot *fs_prot;
 	struct mlx5e_accel_fs_esp *accel_esp;
 
@@ -271,7 +274,7 @@ static void rx_ft_put(struct mlx5e_priv *priv, enum accel_fs_esp_type type)
 		goto out;
 
 	/* disconnect */
-	mlx5_ttc_fwd_default_dest(priv->fs->ttc, fs_esp2tt(type));
+	mlx5_ttc_fwd_default_dest(ttc, fs_esp2tt(type));
 
 	/* remove FT */
 	rx_destroy(priv, type);
@@ -385,7 +388,8 @@ static void setup_fte_common(struct mlx5_accel_esp_xfrm_attrs *attrs,
 		       0xff, 16);
 	}
 
-	flow_act->ipsec_obj_id = ipsec_obj_id;
+	flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC;
+	flow_act->crypto.obj_id = ipsec_obj_id;
 	flow_act->flags |= FLOW_ACT_NO_APPEND;
 }
 
@@ -441,7 +445,7 @@ static int rx_add_rule(struct mlx5e_priv *priv,
 	}
 
 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
-			  MLX5_FLOW_CONTEXT_ACTION_IPSEC_DECRYPT |
+			  MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
 			  MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	flow_act.modify_hdr = modify_hdr;
@@ -497,7 +501,7 @@ static int tx_add_rule(struct mlx5e_priv *priv,
 		 MLX5_ETH_WQE_FT_META_IPSEC);
 
 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW |
-			  MLX5_FLOW_CONTEXT_ACTION_IPSEC_ENCRYPT;
+			  MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT;
 	rule = mlx5_add_flow_rules(priv->ipsec->tx_fs->ft, spec, &flow_act, NULL, 0);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
@@ -573,7 +577,7 @@ int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec)
 	int err = -ENOMEM;
 
 	ns = mlx5_get_flow_namespace(ipsec->mdev,
-				     MLX5_FLOW_NAMESPACE_EGRESS_KERNEL);
+				     MLX5_FLOW_NAMESPACE_EGRESS_IPSEC);
 	if (!ns)
 		return -EOPNOTSUPP;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
index 0ae4e12ce528..1878a70b9031 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
@@ -39,9 +39,9 @@
 #include "en.h"
 #include "en/txrx.h"
 
-/* Bit31: IPsec marker, Bit30-24: IPsec syndrome, Bit23-0: IPsec obj id */
+/* Bit31: IPsec marker, Bit30: reserved, Bit29-24: IPsec syndrome, Bit23-0: IPsec obj id */
 #define MLX5_IPSEC_METADATA_MARKER(metadata)  (((metadata) >> 31) & 0x1)
-#define MLX5_IPSEC_METADATA_SYNDROM(metadata) (((metadata) >> 24) & GENMASK(6, 0))
+#define MLX5_IPSEC_METADATA_SYNDROM(metadata) (((metadata) >> 24) & GENMASK(5, 0))
 #define MLX5_IPSEC_METADATA_HANDLE(metadata)  ((metadata) & GENMASK(23, 0))
 
 struct mlx5e_accel_tx_ipsec_state {
@@ -77,11 +77,6 @@ static inline bool mlx5_ipsec_is_rx_flow(struct mlx5_cqe64 *cqe)
 	return MLX5_IPSEC_METADATA_MARKER(be32_to_cpu(cqe->ft_metadata));
 }
 
-static inline bool mlx5e_ipsec_is_tx_flow(struct mlx5e_accel_tx_ipsec_state *ipsec_st)
-{
-	return ipsec_st->x;
-}
-
 static inline bool mlx5e_ipsec_eseg_meta(struct mlx5_wqe_eth_seg *eseg)
 {
 	return eseg->flow_table_metadata & cpu_to_be32(MLX5_ETH_WQE_FT_META_IPSEC);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
index 30a70d139046..da2184c94203 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.c
@@ -92,6 +92,24 @@ static const struct tlsdev_ops mlx5e_ktls_ops = {
 	.tls_dev_resync = mlx5e_ktls_resync,
 };
 
+bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev)
+{
+	u8 max_sq_wqebbs = mlx5e_get_max_sq_wqebbs(mdev);
+
+	if (is_kdump_kernel() || !MLX5_CAP_GEN(mdev, tls_rx))
+		return false;
+
+	/* Check the possibility to post the required ICOSQ WQEs. */
+	if (WARN_ON_ONCE(max_sq_wqebbs < MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS))
+		return false;
+	if (WARN_ON_ONCE(max_sq_wqebbs < MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS))
+		return false;
+	if (WARN_ON_ONCE(max_sq_wqebbs < MLX5E_KTLS_GET_PROGRESS_WQEBBS))
+		return false;
+
+	return true;
+}
+
 void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv)
 {
 	struct net_device *netdev = priv->netdev;
@@ -118,9 +136,9 @@ int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable)
 
 	mutex_lock(&priv->state_lock);
 	if (enable)
-		err = mlx5e_accel_fs_tcp_create(priv);
+		err = mlx5e_accel_fs_tcp_create(priv->fs);
 	else
-		mlx5e_accel_fs_tcp_destroy(priv);
+		mlx5e_accel_fs_tcp_destroy(priv->fs);
 	mutex_unlock(&priv->state_lock);
 
 	return err;
@@ -138,7 +156,7 @@ int mlx5e_ktls_init_rx(struct mlx5e_priv *priv)
 		return -ENOMEM;
 
 	if (priv->netdev->features & NETIF_F_HW_TLS_RX) {
-		err = mlx5e_accel_fs_tcp_create(priv);
+		err = mlx5e_accel_fs_tcp_create(priv->fs);
 		if (err) {
 			destroy_workqueue(priv->tls->rx_wq);
 			return err;
@@ -154,7 +172,7 @@ void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv)
 		return;
 
 	if (priv->netdev->features & NETIF_F_HW_TLS_RX)
-		mlx5e_accel_fs_tcp_destroy(priv);
+		mlx5e_accel_fs_tcp_destroy(priv->fs);
 
 	destroy_workqueue(priv->tls->rx_wq);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
index 948400dee525..1c35045e41fb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h
@@ -25,7 +25,8 @@ static inline bool mlx5e_is_ktls_device(struct mlx5_core_dev *mdev)
 	if (!MLX5_CAP_GEN(mdev, log_max_dek))
 		return false;
 
-	return MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128);
+	return (MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_128) ||
+		MLX5_CAP_TLS(mdev, tls_1_2_aes_gcm_256));
 }
 
 static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev,
@@ -36,6 +37,10 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev,
 		if (crypto_info->version == TLS_1_2_VERSION)
 			return MLX5_CAP_TLS(mdev,  tls_1_2_aes_gcm_128);
 		break;
+	case TLS_CIPHER_AES_GCM_256:
+		if (crypto_info->version == TLS_1_2_VERSION)
+			return MLX5_CAP_TLS(mdev,  tls_1_2_aes_gcm_256);
+		break;
 	}
 
 	return false;
@@ -56,10 +61,7 @@ static inline bool mlx5e_is_ktls_tx(struct mlx5_core_dev *mdev)
 	return !is_kdump_kernel() && MLX5_CAP_GEN(mdev, tls_tx);
 }
 
-static inline bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev)
-{
-	return !is_kdump_kernel() && MLX5_CAP_GEN(mdev, tls_rx);
-}
+bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev);
 
 struct mlx5e_tls_sw_stats {
 	atomic64_t tx_tls_ctx;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
index 27483aa7be8a..3e54834747ce 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
@@ -43,7 +43,7 @@ struct mlx5e_ktls_rx_resync_ctx {
 };
 
 struct mlx5e_ktls_offload_context_rx {
-	struct tls12_crypto_info_aes_gcm_128 crypto_info;
+	union mlx5e_crypto_info crypto_info;
 	struct accel_rule rule;
 	struct sock *sk;
 	struct mlx5e_rq_stats *rq_stats;
@@ -111,7 +111,7 @@ static void accel_rule_handle_work(struct work_struct *work)
 	if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
 		goto out;
 
-	rule = mlx5e_accel_fs_add_sk(accel_rule->priv, priv_rx->sk,
+	rule = mlx5e_accel_fs_add_sk(accel_rule->priv->fs, priv_rx->sk,
 				     mlx5e_tir_get_tirn(&priv_rx->tir),
 				     MLX5_FS_DEFAULT_FLOW_TAG);
 	if (!IS_ERR_OR_NULL(rule))
@@ -362,7 +362,6 @@ static void resync_init(struct mlx5e_ktls_rx_resync_ctx *resync,
 static void resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx,
 				    struct mlx5e_channel *c)
 {
-	struct tls12_crypto_info_aes_gcm_128 *info = &priv_rx->crypto_info;
 	struct mlx5e_ktls_resync_resp *ktls_resync;
 	struct mlx5e_icosq *sq;
 	bool trigger_poll;
@@ -373,7 +372,31 @@ static void resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_r
 
 	spin_lock_bh(&ktls_resync->lock);
 	spin_lock_bh(&priv_rx->lock);
-	memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, sizeof(info->rec_seq));
+	switch (priv_rx->crypto_info.crypto_info.cipher_type) {
+	case TLS_CIPHER_AES_GCM_128: {
+		struct tls12_crypto_info_aes_gcm_128 *info =
+			&priv_rx->crypto_info.crypto_info_128;
+
+		memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be,
+		       sizeof(info->rec_seq));
+		break;
+	}
+	case TLS_CIPHER_AES_GCM_256: {
+		struct tls12_crypto_info_aes_gcm_256 *info =
+			&priv_rx->crypto_info.crypto_info_256;
+
+		memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be,
+		       sizeof(info->rec_seq));
+		break;
+	}
+	default:
+		WARN_ONCE(1, "Unsupported cipher type %u\n",
+			  priv_rx->crypto_info.crypto_info.cipher_type);
+		spin_unlock_bh(&priv_rx->lock);
+		spin_unlock_bh(&ktls_resync->lock);
+		return;
+	}
+
 	if (list_empty(&priv_rx->list)) {
 		list_add_tail(&priv_rx->list, &ktls_resync->list);
 		trigger_poll = !test_and_set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state);
@@ -461,6 +484,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb)
 {
 	struct ethhdr *eth = (struct ethhdr *)(skb->data);
 	struct net_device *netdev = rq->netdev;
+	struct net *net = dev_net(netdev);
 	struct sock *sk = NULL;
 	unsigned int datalen;
 	struct iphdr *iph;
@@ -475,7 +499,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb)
 		depth += sizeof(struct iphdr);
 		th = (void *)iph + sizeof(struct iphdr);
 
-		sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo,
+		sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 					     iph->saddr, th->source, iph->daddr,
 					     th->dest, netdev->ifindex);
 #if IS_ENABLED(CONFIG_IPV6)
@@ -485,7 +509,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb)
 		depth += sizeof(struct ipv6hdr);
 		th = (void *)ipv6h + sizeof(struct ipv6hdr);
 
-		sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo,
+		sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 						&ipv6h->saddr, th->source,
 						&ipv6h->daddr, ntohs(th->dest),
 						netdev->ifindex, 0);
@@ -603,8 +627,20 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
 
 	INIT_LIST_HEAD(&priv_rx->list);
 	spin_lock_init(&priv_rx->lock);
-	priv_rx->crypto_info  =
-		*(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
+	switch (crypto_info->cipher_type) {
+	case TLS_CIPHER_AES_GCM_128:
+		priv_rx->crypto_info.crypto_info_128 =
+			*(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
+		break;
+	case TLS_CIPHER_AES_GCM_256:
+		priv_rx->crypto_info.crypto_info_256 =
+			*(struct tls12_crypto_info_aes_gcm_256 *)crypto_info;
+		break;
+	default:
+		WARN_ONCE(1, "Unsupported cipher type %u\n",
+			  crypto_info->cipher_type);
+		return -EOPNOTSUPP;
+	}
 
 	rxq = mlx5e_ktls_sk_get_rxq(sk);
 	priv_rx->rxq = rxq;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c
index 3a1f76eac542..2e0335246967 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c
@@ -93,7 +93,7 @@ struct mlx5e_ktls_offload_context_tx {
 	bool ctx_post_pending;
 	/* control / resync */
 	struct list_head list_node; /* member of the pool */
-	struct tls12_crypto_info_aes_gcm_128 crypto_info;
+	union mlx5e_crypto_info crypto_info;
 	struct tls_offload_context_tx *tx_ctx;
 	struct mlx5_core_dev *mdev;
 	struct mlx5e_tls_sw_stats *sw_stats;
@@ -485,8 +485,20 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk,
 		goto err_create_key;
 
 	priv_tx->expected_seq = start_offload_tcp_sn;
-	priv_tx->crypto_info  =
-		*(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
+	switch (crypto_info->cipher_type) {
+	case TLS_CIPHER_AES_GCM_128:
+		priv_tx->crypto_info.crypto_info_128 =
+			*(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
+		break;
+	case TLS_CIPHER_AES_GCM_256:
+		priv_tx->crypto_info.crypto_info_256 =
+			*(struct tls12_crypto_info_aes_gcm_256 *)crypto_info;
+		break;
+	default:
+		WARN_ONCE(1, "Unsupported cipher type %u\n",
+			  crypto_info->cipher_type);
+		return -EOPNOTSUPP;
+	}
 	priv_tx->tx_ctx = tls_offload_ctx_tx(tls_ctx);
 
 	mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx);
@@ -671,14 +683,31 @@ tx_post_resync_params(struct mlx5e_txqsq *sq,
 		      struct mlx5e_ktls_offload_context_tx *priv_tx,
 		      u64 rcd_sn)
 {
-	struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info;
 	__be64 rn_be = cpu_to_be64(rcd_sn);
 	bool skip_static_post;
 	u16 rec_seq_sz;
 	char *rec_seq;
 
-	rec_seq = info->rec_seq;
-	rec_seq_sz = sizeof(info->rec_seq);
+	switch (priv_tx->crypto_info.crypto_info.cipher_type) {
+	case TLS_CIPHER_AES_GCM_128: {
+		struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info.crypto_info_128;
+
+		rec_seq = info->rec_seq;
+		rec_seq_sz = sizeof(info->rec_seq);
+		break;
+	}
+	case TLS_CIPHER_AES_GCM_256: {
+		struct tls12_crypto_info_aes_gcm_256 *info = &priv_tx->crypto_info.crypto_info_256;
+
+		rec_seq = info->rec_seq;
+		rec_seq_sz = sizeof(info->rec_seq);
+		break;
+	}
+	default:
+		WARN_ONCE(1, "Unsupported cipher type %u\n",
+			  priv_tx->crypto_info.crypto_info.cipher_type);
+		return;
+	}
 
 	skip_static_post = !memcmp(rec_seq, &rn_be, rec_seq_sz);
 	if (!skip_static_post)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
index ac29aeb8af49..570a912dd6fa 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.c
@@ -21,7 +21,7 @@ enum {
 
 static void
 fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
-		   struct tls12_crypto_info_aes_gcm_128 *info,
+		   union mlx5e_crypto_info *crypto_info,
 		   u32 key_id, u32 resync_tcp_sn)
 {
 	char *initial_rn, *gcm_iv;
@@ -32,7 +32,26 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
 
 	ctx = params->ctx;
 
-	EXTRACT_INFO_FIELDS;
+	switch (crypto_info->crypto_info.cipher_type) {
+	case TLS_CIPHER_AES_GCM_128: {
+		struct tls12_crypto_info_aes_gcm_128 *info =
+			&crypto_info->crypto_info_128;
+
+		EXTRACT_INFO_FIELDS;
+		break;
+	}
+	case TLS_CIPHER_AES_GCM_256: {
+		struct tls12_crypto_info_aes_gcm_256 *info =
+			&crypto_info->crypto_info_256;
+
+		EXTRACT_INFO_FIELDS;
+		break;
+	}
+	default:
+		WARN_ONCE(1, "Unsupported cipher type %u\n",
+			  crypto_info->crypto_info.cipher_type);
+		return;
+	}
 
 	gcm_iv      = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv);
 	initial_rn  = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number);
@@ -54,7 +73,7 @@ fill_static_params(struct mlx5_wqe_tls_static_params_seg *params,
 void
 mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe,
 			       u16 pc, u32 sqn,
-			       struct tls12_crypto_info_aes_gcm_128 *info,
+			       union mlx5e_crypto_info *crypto_info,
 			       u32 tis_tir_num, u32 key_id, u32 resync_tcp_sn,
 			       bool fence, enum tls_offload_ctx_dir direction)
 {
@@ -75,7 +94,7 @@ mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe,
 	ucseg->flags = MLX5_UMR_INLINE;
 	ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16);
 
-	fill_static_params(&wqe->params, info, key_id, resync_tcp_sn);
+	fill_static_params(&wqe->params, crypto_info, key_id, resync_tcp_sn);
 }
 
 static void
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h
index 0dc715c4c10d..3d79cd379890 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_utils.h
@@ -27,6 +27,12 @@ int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
 void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx);
 void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk, u32 seq, u8 *rcd_sn);
 
+union mlx5e_crypto_info {
+	struct tls_crypto_info crypto_info;
+	struct tls12_crypto_info_aes_gcm_128 crypto_info_128;
+	struct tls12_crypto_info_aes_gcm_256 crypto_info_256;
+};
+
 struct mlx5e_set_tls_static_params_wqe {
 	struct mlx5_wqe_ctrl_seg ctrl;
 	struct mlx5_wqe_umr_ctrl_seg uctrl;
@@ -72,7 +78,7 @@ struct mlx5e_get_tls_progress_params_wqe {
 void
 mlx5e_ktls_build_static_params(struct mlx5e_set_tls_static_params_wqe *wqe,
 			       u16 pc, u32 sqn,
-			       struct tls12_crypto_info_aes_gcm_128 *info,
+			       union mlx5e_crypto_info *crypto_info,
 			       u32 tis_tir_num, u32 key_id, u32 resync_tcp_sn,
 			       bool fence, enum tls_offload_ctx_dir direction);
 void
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c
new file mode 100644
index 000000000000..5da746da898d
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.c
@@ -0,0 +1,1870 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#include <linux/mlx5/device.h>
+#include <linux/mlx5/mlx5_ifc.h>
+#include <linux/xarray.h>
+
+#include "en.h"
+#include "lib/aso.h"
+#include "lib/mlx5.h"
+#include "en_accel/macsec.h"
+#include "en_accel/macsec_fs.h"
+
+#define MLX5_MACSEC_EPN_SCOPE_MID 0x80000000L
+#define MLX5E_MACSEC_ASO_CTX_SZ MLX5_ST_SZ_BYTES(macsec_aso)
+
+enum mlx5_macsec_aso_event_arm {
+	MLX5E_ASO_EPN_ARM = BIT(0),
+};
+
+enum {
+	MLX5_MACSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET,
+};
+
+struct mlx5e_macsec_handle {
+	struct mlx5e_macsec *macsec;
+	u32 obj_id;
+	u8 idx;
+};
+
+enum {
+	MLX5_MACSEC_EPN,
+};
+
+struct mlx5e_macsec_aso_out {
+	u8 event_arm;
+	u32 mode_param;
+};
+
+struct mlx5e_macsec_aso_in {
+	u8 mode;
+	u32 obj_id;
+};
+
+struct mlx5e_macsec_epn_state {
+	u32 epn_msb;
+	u8 epn_enabled;
+	u8 overlap;
+};
+
+struct mlx5e_macsec_async_work {
+	struct mlx5e_macsec *macsec;
+	struct mlx5_core_dev *mdev;
+	struct work_struct work;
+	u32 obj_id;
+};
+
+struct mlx5e_macsec_sa {
+	bool active;
+	u8  assoc_num;
+	u32 macsec_obj_id;
+	u32 enc_key_id;
+	u32 next_pn;
+	sci_t sci;
+	salt_t salt;
+
+	struct rhash_head hash;
+	u32 fs_id;
+	union mlx5e_macsec_rule *macsec_rule;
+	struct rcu_head rcu_head;
+	struct mlx5e_macsec_epn_state epn_state;
+};
+
+struct mlx5e_macsec_rx_sc;
+struct mlx5e_macsec_rx_sc_xarray_element {
+	u32 fs_id;
+	struct mlx5e_macsec_rx_sc *rx_sc;
+};
+
+struct mlx5e_macsec_rx_sc {
+	bool active;
+	sci_t sci;
+	struct mlx5e_macsec_sa *rx_sa[MACSEC_NUM_AN];
+	struct list_head rx_sc_list_element;
+	struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element;
+	struct metadata_dst *md_dst;
+	struct rcu_head rcu_head;
+};
+
+struct mlx5e_macsec_umr {
+	dma_addr_t dma_addr;
+	u8 ctx[MLX5_ST_SZ_BYTES(macsec_aso)];
+	u32 mkey;
+};
+
+struct mlx5e_macsec_aso {
+	/* ASO */
+	struct mlx5_aso *maso;
+	/* Protects macsec ASO */
+	struct mutex aso_lock;
+	/* UMR */
+	struct mlx5e_macsec_umr *umr;
+
+	u32 pdn;
+};
+
+static const struct rhashtable_params rhash_sci = {
+	.key_len = sizeof_field(struct mlx5e_macsec_sa, sci),
+	.key_offset = offsetof(struct mlx5e_macsec_sa, sci),
+	.head_offset = offsetof(struct mlx5e_macsec_sa, hash),
+	.automatic_shrinking = true,
+	.min_size = 1,
+};
+
+struct mlx5e_macsec_device {
+	const struct net_device *netdev;
+	struct mlx5e_macsec_sa *tx_sa[MACSEC_NUM_AN];
+	struct list_head macsec_rx_sc_list_head;
+	unsigned char *dev_addr;
+	struct list_head macsec_device_list_element;
+};
+
+struct mlx5e_macsec {
+	struct list_head macsec_device_list_head;
+	int num_of_devices;
+	struct mlx5e_macsec_fs *macsec_fs;
+	struct mutex lock; /* Protects mlx5e_macsec internal contexts */
+
+	/* Tx sci -> fs id mapping handling */
+	struct rhashtable sci_hash;      /* sci -> mlx5e_macsec_sa */
+
+	/* Rx fs_id -> rx_sc mapping */
+	struct xarray sc_xarray;
+
+	struct mlx5_core_dev *mdev;
+
+	/* Stats manage */
+	struct mlx5e_macsec_stats stats;
+
+	/* ASO */
+	struct mlx5e_macsec_aso aso;
+
+	struct notifier_block nb;
+	struct workqueue_struct *wq;
+};
+
+struct mlx5_macsec_obj_attrs {
+	u32 aso_pdn;
+	u32 next_pn;
+	__be64 sci;
+	u32 enc_key_id;
+	bool encrypt;
+	struct mlx5e_macsec_epn_state epn_state;
+	salt_t salt;
+	__be32 ssci;
+	bool replay_protect;
+	u32 replay_window;
+};
+
+struct mlx5_aso_ctrl_param {
+	u8   data_mask_mode;
+	u8   condition_0_operand;
+	u8   condition_1_operand;
+	u8   condition_0_offset;
+	u8   condition_1_offset;
+	u8   data_offset;
+	u8   condition_operand;
+	u32  condition_0_data;
+	u32  condition_0_mask;
+	u32  condition_1_data;
+	u32  condition_1_mask;
+	u64  bitwise_data;
+	u64  data_mask;
+};
+
+static int mlx5e_macsec_aso_reg_mr(struct mlx5_core_dev *mdev, struct mlx5e_macsec_aso *aso)
+{
+	struct mlx5e_macsec_umr *umr;
+	struct device *dma_device;
+	dma_addr_t dma_addr;
+	int err;
+
+	umr = kzalloc(sizeof(*umr), GFP_KERNEL);
+	if (!umr) {
+		err = -ENOMEM;
+		return err;
+	}
+
+	dma_device = &mdev->pdev->dev;
+	dma_addr = dma_map_single(dma_device, umr->ctx, sizeof(umr->ctx), DMA_BIDIRECTIONAL);
+	err = dma_mapping_error(dma_device, dma_addr);
+	if (err) {
+		mlx5_core_err(mdev, "Can't map dma device, err=%d\n", err);
+		goto out_dma;
+	}
+
+	err = mlx5e_create_mkey(mdev, aso->pdn, &umr->mkey);
+	if (err) {
+		mlx5_core_err(mdev, "Can't create mkey, err=%d\n", err);
+		goto out_mkey;
+	}
+
+	umr->dma_addr = dma_addr;
+
+	aso->umr = umr;
+
+	return 0;
+
+out_mkey:
+	dma_unmap_single(dma_device, dma_addr, sizeof(umr->ctx), DMA_BIDIRECTIONAL);
+out_dma:
+	kfree(umr);
+	return err;
+}
+
+static void mlx5e_macsec_aso_dereg_mr(struct mlx5_core_dev *mdev, struct mlx5e_macsec_aso *aso)
+{
+	struct mlx5e_macsec_umr *umr = aso->umr;
+
+	mlx5_core_destroy_mkey(mdev, umr->mkey);
+	dma_unmap_single(&mdev->pdev->dev, umr->dma_addr, sizeof(umr->ctx), DMA_BIDIRECTIONAL);
+	kfree(umr);
+}
+
+static int macsec_set_replay_protection(struct mlx5_macsec_obj_attrs *attrs, void *aso_ctx)
+{
+	u8 window_sz;
+
+	if (!attrs->replay_protect)
+		return 0;
+
+	switch (attrs->replay_window) {
+	case 256:
+		window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_256BIT;
+		break;
+	case 128:
+		window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_128BIT;
+		break;
+	case 64:
+		window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_64BIT;
+		break;
+	case 32:
+		window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_32BIT;
+		break;
+	default:
+		return -EINVAL;
+	}
+	MLX5_SET(macsec_aso, aso_ctx, window_size, window_sz);
+	MLX5_SET(macsec_aso, aso_ctx, mode, MLX5_MACSEC_ASO_REPLAY_PROTECTION);
+
+	return 0;
+}
+
+static int mlx5e_macsec_create_object(struct mlx5_core_dev *mdev,
+				      struct mlx5_macsec_obj_attrs *attrs,
+				      bool is_tx,
+				      u32 *macsec_obj_id)
+{
+	u32 in[MLX5_ST_SZ_DW(create_macsec_obj_in)] = {};
+	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+	void *aso_ctx;
+	void *obj;
+	int err;
+
+	obj = MLX5_ADDR_OF(create_macsec_obj_in, in, macsec_object);
+	aso_ctx = MLX5_ADDR_OF(macsec_offload_obj, obj, macsec_aso);
+
+	MLX5_SET(macsec_offload_obj, obj, confidentiality_en, attrs->encrypt);
+	MLX5_SET(macsec_offload_obj, obj, dekn, attrs->enc_key_id);
+	MLX5_SET(macsec_offload_obj, obj, aso_return_reg, MLX5_MACSEC_ASO_REG_C_4_5);
+	MLX5_SET(macsec_offload_obj, obj, macsec_aso_access_pd, attrs->aso_pdn);
+	MLX5_SET(macsec_aso, aso_ctx, mode_parameter, attrs->next_pn);
+
+	/* Epn */
+	if (attrs->epn_state.epn_enabled) {
+		void *salt_p;
+		int i;
+
+		MLX5_SET(macsec_aso, aso_ctx, epn_event_arm, 1);
+		MLX5_SET(macsec_offload_obj, obj, epn_en, 1);
+		MLX5_SET(macsec_offload_obj, obj, epn_msb, attrs->epn_state.epn_msb);
+		MLX5_SET(macsec_offload_obj, obj, epn_overlap, attrs->epn_state.overlap);
+		MLX5_SET64(macsec_offload_obj, obj, sci, (__force u64)attrs->ssci);
+		salt_p = MLX5_ADDR_OF(macsec_offload_obj, obj, salt);
+		for (i = 0; i < 3 ; i++)
+			memcpy((u32 *)salt_p + i, &attrs->salt.bytes[4 * (2 - i)], 4);
+	} else {
+		MLX5_SET64(macsec_offload_obj, obj, sci, (__force u64)(attrs->sci));
+	}
+
+	MLX5_SET(macsec_aso, aso_ctx, valid, 0x1);
+	if (is_tx) {
+		MLX5_SET(macsec_aso, aso_ctx, mode, MLX5_MACSEC_ASO_INC_SN);
+	} else {
+		err = macsec_set_replay_protection(attrs, aso_ctx);
+		if (err)
+			return err;
+	}
+
+	/* general object fields set */
+	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_MACSEC);
+
+	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+	if (err) {
+		mlx5_core_err(mdev,
+			      "MACsec offload: Failed to create MACsec object (err = %d)\n",
+			      err);
+		return err;
+	}
+
+	*macsec_obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
+
+	return err;
+}
+
+static void mlx5e_macsec_destroy_object(struct mlx5_core_dev *mdev, u32 macsec_obj_id)
+{
+	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
+	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
+
+	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_MACSEC);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, macsec_obj_id);
+
+	mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+static void mlx5e_macsec_cleanup_sa(struct mlx5e_macsec *macsec,
+				    struct mlx5e_macsec_sa *sa,
+				    bool is_tx)
+{
+	int action =  (is_tx) ?  MLX5_ACCEL_MACSEC_ACTION_ENCRYPT :
+				 MLX5_ACCEL_MACSEC_ACTION_DECRYPT;
+
+	if ((is_tx) && sa->fs_id) {
+		/* Make sure ongoing datapath readers sees a valid SA */
+		rhashtable_remove_fast(&macsec->sci_hash, &sa->hash, rhash_sci);
+		sa->fs_id = 0;
+	}
+
+	if (!sa->macsec_rule)
+		return;
+
+	mlx5e_macsec_fs_del_rule(macsec->macsec_fs, sa->macsec_rule, action);
+	mlx5e_macsec_destroy_object(macsec->mdev, sa->macsec_obj_id);
+	sa->macsec_rule = NULL;
+}
+
+static int mlx5e_macsec_init_sa(struct macsec_context *ctx,
+				struct mlx5e_macsec_sa *sa,
+				bool encrypt,
+				bool is_tx)
+{
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_macsec *macsec = priv->macsec;
+	struct mlx5_macsec_rule_attrs rule_attrs;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5_macsec_obj_attrs obj_attrs;
+	union mlx5e_macsec_rule *macsec_rule;
+	struct macsec_key *key;
+	int err;
+
+	obj_attrs.next_pn = sa->next_pn;
+	obj_attrs.sci = cpu_to_be64((__force u64)sa->sci);
+	obj_attrs.enc_key_id = sa->enc_key_id;
+	obj_attrs.encrypt = encrypt;
+	obj_attrs.aso_pdn = macsec->aso.pdn;
+	obj_attrs.epn_state = sa->epn_state;
+
+	if (is_tx) {
+		obj_attrs.ssci = cpu_to_be32((__force u32)ctx->sa.tx_sa->ssci);
+		key = &ctx->sa.tx_sa->key;
+	} else {
+		obj_attrs.ssci = cpu_to_be32((__force u32)ctx->sa.rx_sa->ssci);
+		key = &ctx->sa.rx_sa->key;
+	}
+
+	memcpy(&obj_attrs.salt, &key->salt, sizeof(key->salt));
+	obj_attrs.replay_window = ctx->secy->replay_window;
+	obj_attrs.replay_protect = ctx->secy->replay_protect;
+
+	err = mlx5e_macsec_create_object(mdev, &obj_attrs, is_tx, &sa->macsec_obj_id);
+	if (err)
+		return err;
+
+	rule_attrs.macsec_obj_id = sa->macsec_obj_id;
+	rule_attrs.sci = sa->sci;
+	rule_attrs.assoc_num = sa->assoc_num;
+	rule_attrs.action = (is_tx) ? MLX5_ACCEL_MACSEC_ACTION_ENCRYPT :
+				      MLX5_ACCEL_MACSEC_ACTION_DECRYPT;
+
+	macsec_rule = mlx5e_macsec_fs_add_rule(macsec->macsec_fs, ctx, &rule_attrs, &sa->fs_id);
+	if (!macsec_rule) {
+		err = -ENOMEM;
+		goto destroy_macsec_object;
+	}
+
+	sa->macsec_rule = macsec_rule;
+
+	if (is_tx) {
+		err = rhashtable_insert_fast(&macsec->sci_hash, &sa->hash, rhash_sci);
+		if (err)
+			goto destroy_macsec_object_and_rule;
+	}
+
+	return 0;
+
+destroy_macsec_object_and_rule:
+	mlx5e_macsec_cleanup_sa(macsec, sa, is_tx);
+destroy_macsec_object:
+	mlx5e_macsec_destroy_object(mdev, sa->macsec_obj_id);
+
+	return err;
+}
+
+static struct mlx5e_macsec_rx_sc *
+mlx5e_macsec_get_rx_sc_from_sc_list(const struct list_head *list, sci_t sci)
+{
+	struct mlx5e_macsec_rx_sc *iter;
+
+	list_for_each_entry_rcu(iter, list, rx_sc_list_element) {
+		if (iter->sci == sci)
+			return iter;
+	}
+
+	return NULL;
+}
+
+static int mlx5e_macsec_update_rx_sa(struct mlx5e_macsec *macsec,
+				     struct mlx5e_macsec_sa *rx_sa,
+				     bool active)
+{
+	struct mlx5_core_dev *mdev = macsec->mdev;
+	struct mlx5_macsec_obj_attrs attrs;
+	int err = 0;
+
+	if (rx_sa->active != active)
+		return 0;
+
+	rx_sa->active = active;
+	if (!active) {
+		mlx5e_macsec_cleanup_sa(macsec, rx_sa, false);
+		return 0;
+	}
+
+	attrs.sci = rx_sa->sci;
+	attrs.enc_key_id = rx_sa->enc_key_id;
+	err = mlx5e_macsec_create_object(mdev, &attrs, false, &rx_sa->macsec_obj_id);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static bool mlx5e_macsec_secy_features_validate(struct macsec_context *ctx)
+{
+	const struct net_device *netdev = ctx->netdev;
+	const struct macsec_secy *secy = ctx->secy;
+
+	if (secy->validate_frames != MACSEC_VALIDATE_STRICT) {
+		netdev_err(netdev,
+			   "MACsec offload is supported only when validate_frame is in strict mode\n");
+		return false;
+	}
+
+	if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN) {
+		netdev_err(netdev, "MACsec offload is supported only when icv_len is %d\n",
+			   MACSEC_DEFAULT_ICV_LEN);
+		return false;
+	}
+
+	if (!secy->protect_frames) {
+		netdev_err(netdev,
+			   "MACsec offload is supported only when protect_frames is set\n");
+		return false;
+	}
+
+	return true;
+}
+
+static struct mlx5e_macsec_device *
+mlx5e_macsec_get_macsec_device_context(const struct mlx5e_macsec *macsec,
+				       const struct macsec_context *ctx)
+{
+	struct mlx5e_macsec_device *iter;
+	const struct list_head *list;
+
+	list = &macsec->macsec_device_list_head;
+	list_for_each_entry_rcu(iter, list, macsec_device_list_element) {
+		if (iter->netdev == ctx->secy->netdev)
+			return iter;
+	}
+
+	return NULL;
+}
+
+static void update_macsec_epn(struct mlx5e_macsec_sa *sa, const struct macsec_key *key,
+			      const pn_t *next_pn_halves)
+{
+	struct mlx5e_macsec_epn_state *epn_state = &sa->epn_state;
+
+	sa->salt = key->salt;
+	epn_state->epn_enabled = 1;
+	epn_state->epn_msb = next_pn_halves->upper;
+	epn_state->overlap = next_pn_halves->lower < MLX5_MACSEC_EPN_SCOPE_MID ? 0 : 1;
+}
+
+static int mlx5e_macsec_add_txsa(struct macsec_context *ctx)
+{
+	const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc;
+	const struct macsec_tx_sa *ctx_tx_sa = ctx->sa.tx_sa;
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	const struct macsec_secy *secy = ctx->secy;
+	struct mlx5e_macsec_device *macsec_device;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u8 assoc_num = ctx->sa.assoc_num;
+	struct mlx5e_macsec_sa *tx_sa;
+	struct mlx5e_macsec *macsec;
+	int err = 0;
+
+	mutex_lock(&priv->macsec->lock);
+
+	macsec = priv->macsec;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
+		err = -EEXIST;
+		goto out;
+	}
+
+	if (macsec_device->tx_sa[assoc_num]) {
+		netdev_err(ctx->netdev, "MACsec offload tx_sa: %d already exist\n", assoc_num);
+		err = -EEXIST;
+		goto out;
+	}
+
+	tx_sa = kzalloc(sizeof(*tx_sa), GFP_KERNEL);
+	if (!tx_sa) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	tx_sa->active = ctx_tx_sa->active;
+	tx_sa->next_pn = ctx_tx_sa->next_pn_halves.lower;
+	tx_sa->sci = secy->sci;
+	tx_sa->assoc_num = assoc_num;
+
+	if (secy->xpn)
+		update_macsec_epn(tx_sa, &ctx_tx_sa->key, &ctx_tx_sa->next_pn_halves);
+
+	err = mlx5_create_encryption_key(mdev, ctx->sa.key, secy->key_len,
+					 MLX5_ACCEL_OBJ_MACSEC_KEY,
+					 &tx_sa->enc_key_id);
+	if (err)
+		goto destroy_sa;
+
+	macsec_device->tx_sa[assoc_num] = tx_sa;
+	if (!secy->operational ||
+	    assoc_num != tx_sc->encoding_sa ||
+	    !tx_sa->active)
+		goto out;
+
+	err = mlx5e_macsec_init_sa(ctx, tx_sa, tx_sc->encrypt, true);
+	if (err)
+		goto destroy_encryption_key;
+
+	mutex_unlock(&macsec->lock);
+
+	return 0;
+
+destroy_encryption_key:
+	macsec_device->tx_sa[assoc_num] = NULL;
+	mlx5_destroy_encryption_key(mdev, tx_sa->enc_key_id);
+destroy_sa:
+	kfree(tx_sa);
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static int mlx5e_macsec_upd_txsa(struct macsec_context *ctx)
+{
+	const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc;
+	const struct macsec_tx_sa *ctx_tx_sa = ctx->sa.tx_sa;
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_macsec_device *macsec_device;
+	u8 assoc_num = ctx->sa.assoc_num;
+	struct mlx5e_macsec_sa *tx_sa;
+	struct mlx5e_macsec *macsec;
+	struct net_device *netdev;
+	int err = 0;
+
+	mutex_lock(&priv->macsec->lock);
+
+	macsec = priv->macsec;
+	netdev = ctx->netdev;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(netdev, "MACsec offload: Failed to find device context\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	tx_sa = macsec_device->tx_sa[assoc_num];
+	if (!tx_sa) {
+		netdev_err(netdev, "MACsec offload: TX sa 0x%x doesn't exist\n", assoc_num);
+		err = -EEXIST;
+		goto out;
+	}
+
+	if (tx_sa->next_pn != ctx_tx_sa->next_pn_halves.lower) {
+		netdev_err(netdev, "MACsec offload: update TX sa %d PN isn't supported\n",
+			   assoc_num);
+		err = -EINVAL;
+		goto out;
+	}
+
+	if (tx_sa->active == ctx_tx_sa->active)
+		goto out;
+
+	if (tx_sa->assoc_num != tx_sc->encoding_sa)
+		goto out;
+
+	if (ctx_tx_sa->active) {
+		err = mlx5e_macsec_init_sa(ctx, tx_sa, tx_sc->encrypt, true);
+		if (err)
+			goto out;
+	} else {
+		if (!tx_sa->macsec_rule) {
+			err = -EINVAL;
+			goto out;
+		}
+
+		mlx5e_macsec_cleanup_sa(macsec, tx_sa, true);
+	}
+
+	tx_sa->active = ctx_tx_sa->active;
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static int mlx5e_macsec_del_txsa(struct macsec_context *ctx)
+{
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_macsec_device *macsec_device;
+	u8 assoc_num = ctx->sa.assoc_num;
+	struct mlx5e_macsec_sa *tx_sa;
+	struct mlx5e_macsec *macsec;
+	int err = 0;
+
+	mutex_lock(&priv->macsec->lock);
+	macsec = priv->macsec;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	tx_sa = macsec_device->tx_sa[assoc_num];
+	if (!tx_sa) {
+		netdev_err(ctx->netdev, "MACsec offload: TX sa 0x%x doesn't exist\n", assoc_num);
+		err = -EEXIST;
+		goto out;
+	}
+
+	mlx5e_macsec_cleanup_sa(macsec, tx_sa, true);
+	mlx5_destroy_encryption_key(macsec->mdev, tx_sa->enc_key_id);
+	kfree_rcu(tx_sa);
+	macsec_device->tx_sa[assoc_num] = NULL;
+
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static u32 mlx5e_macsec_get_sa_from_hashtable(struct rhashtable *sci_hash, sci_t *sci)
+{
+	struct mlx5e_macsec_sa *macsec_sa;
+	u32 fs_id = 0;
+
+	rcu_read_lock();
+	macsec_sa = rhashtable_lookup(sci_hash, sci, rhash_sci);
+	if (macsec_sa)
+		fs_id = macsec_sa->fs_id;
+	rcu_read_unlock();
+
+	return fs_id;
+}
+
+static int mlx5e_macsec_add_rxsc(struct macsec_context *ctx)
+{
+	struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element;
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	const struct macsec_rx_sc *ctx_rx_sc = ctx->rx_sc;
+	struct mlx5e_macsec_device *macsec_device;
+	struct mlx5e_macsec_rx_sc *rx_sc;
+	struct list_head *rx_sc_list;
+	struct mlx5e_macsec *macsec;
+	int err = 0;
+
+	mutex_lock(&priv->macsec->lock);
+	macsec = priv->macsec;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	rx_sc_list = &macsec_device->macsec_rx_sc_list_head;
+	rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(rx_sc_list, ctx_rx_sc->sci);
+	if (rx_sc) {
+		netdev_err(ctx->netdev, "MACsec offload: rx_sc (sci %lld) already exists\n",
+			   ctx_rx_sc->sci);
+		err = -EEXIST;
+		goto out;
+	}
+
+	rx_sc = kzalloc(sizeof(*rx_sc), GFP_KERNEL);
+	if (!rx_sc) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	sc_xarray_element = kzalloc(sizeof(*sc_xarray_element), GFP_KERNEL);
+	if (!sc_xarray_element) {
+		err = -ENOMEM;
+		goto destroy_rx_sc;
+	}
+
+	sc_xarray_element->rx_sc = rx_sc;
+	err = xa_alloc(&macsec->sc_xarray, &sc_xarray_element->fs_id, sc_xarray_element,
+		       XA_LIMIT(1, USHRT_MAX), GFP_KERNEL);
+	if (err)
+		goto destroy_sc_xarray_elemenet;
+
+	rx_sc->md_dst = metadata_dst_alloc(0, METADATA_MACSEC, GFP_KERNEL);
+	if (!rx_sc->md_dst) {
+		err = -ENOMEM;
+		goto erase_xa_alloc;
+	}
+
+	rx_sc->sci = ctx_rx_sc->sci;
+	rx_sc->active = ctx_rx_sc->active;
+	list_add_rcu(&rx_sc->rx_sc_list_element, rx_sc_list);
+
+	rx_sc->sc_xarray_element = sc_xarray_element;
+	rx_sc->md_dst->u.macsec_info.sci = rx_sc->sci;
+	mutex_unlock(&macsec->lock);
+
+	return 0;
+
+erase_xa_alloc:
+	xa_erase(&macsec->sc_xarray, sc_xarray_element->fs_id);
+destroy_sc_xarray_elemenet:
+	kfree(sc_xarray_element);
+destroy_rx_sc:
+	kfree(rx_sc);
+
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static int mlx5e_macsec_upd_rxsc(struct macsec_context *ctx)
+{
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	const struct macsec_rx_sc *ctx_rx_sc = ctx->rx_sc;
+	struct mlx5e_macsec_device *macsec_device;
+	struct mlx5e_macsec_rx_sc *rx_sc;
+	struct mlx5e_macsec_sa *rx_sa;
+	struct mlx5e_macsec *macsec;
+	struct list_head *list;
+	int i;
+	int err = 0;
+
+	mutex_lock(&priv->macsec->lock);
+
+	macsec = priv->macsec;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	list = &macsec_device->macsec_rx_sc_list_head;
+	rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, ctx_rx_sc->sci);
+	if (!rx_sc) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	rx_sc->active = ctx_rx_sc->active;
+	if (rx_sc->active == ctx_rx_sc->active)
+		goto out;
+
+	for (i = 0; i < MACSEC_NUM_AN; ++i) {
+		rx_sa = rx_sc->rx_sa[i];
+		if (!rx_sa)
+			continue;
+
+		err = mlx5e_macsec_update_rx_sa(macsec, rx_sa, rx_sa->active && ctx_rx_sc->active);
+		if (err)
+			goto out;
+	}
+
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static int mlx5e_macsec_del_rxsc(struct macsec_context *ctx)
+{
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_macsec_device *macsec_device;
+	struct mlx5e_macsec_rx_sc *rx_sc;
+	struct mlx5e_macsec_sa *rx_sa;
+	struct mlx5e_macsec *macsec;
+	struct list_head *list;
+	int err = 0;
+	int i;
+
+	mutex_lock(&priv->macsec->lock);
+
+	macsec = priv->macsec;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	list = &macsec_device->macsec_rx_sc_list_head;
+	rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, ctx->rx_sc->sci);
+	if (!rx_sc) {
+		netdev_err(ctx->netdev,
+			   "MACsec offload rx_sc sci %lld doesn't exist\n",
+			   ctx->sa.rx_sa->sc->sci);
+		err = -EINVAL;
+		goto out;
+	}
+
+	for (i = 0; i < MACSEC_NUM_AN; ++i) {
+		rx_sa = rx_sc->rx_sa[i];
+		if (!rx_sa)
+			continue;
+
+		mlx5e_macsec_cleanup_sa(macsec, rx_sa, false);
+		mlx5_destroy_encryption_key(macsec->mdev, rx_sa->enc_key_id);
+
+		kfree(rx_sa);
+		rx_sc->rx_sa[i] = NULL;
+	}
+
+/*
+ * At this point the relevant MACsec offload Rx rule already removed at
+ * mlx5e_macsec_cleanup_sa need to wait for datapath to finish current
+ * Rx related data propagating using xa_erase which uses rcu to sync,
+ * once fs_id is erased then this rx_sc is hidden from datapath.
+ */
+	list_del_rcu(&rx_sc->rx_sc_list_element);
+	xa_erase(&macsec->sc_xarray, rx_sc->sc_xarray_element->fs_id);
+	metadata_dst_free(rx_sc->md_dst);
+	kfree(rx_sc->sc_xarray_element);
+
+	kfree_rcu(rx_sc);
+
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static int mlx5e_macsec_add_rxsa(struct macsec_context *ctx)
+{
+	const struct macsec_rx_sa *ctx_rx_sa = ctx->sa.rx_sa;
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_macsec_device *macsec_device;
+	struct mlx5_core_dev *mdev = priv->mdev;
+	u8 assoc_num = ctx->sa.assoc_num;
+	struct mlx5e_macsec_rx_sc *rx_sc;
+	sci_t sci = ctx_rx_sa->sc->sci;
+	struct mlx5e_macsec_sa *rx_sa;
+	struct mlx5e_macsec *macsec;
+	struct list_head *list;
+	int err = 0;
+
+	mutex_lock(&priv->macsec->lock);
+
+	macsec = priv->macsec;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	list = &macsec_device->macsec_rx_sc_list_head;
+	rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, sci);
+	if (!rx_sc) {
+		netdev_err(ctx->netdev,
+			   "MACsec offload rx_sc sci %lld doesn't exist\n",
+			   ctx->sa.rx_sa->sc->sci);
+		err = -EINVAL;
+		goto out;
+	}
+
+	if (rx_sc->rx_sa[assoc_num]) {
+		netdev_err(ctx->netdev,
+			   "MACsec offload rx_sc sci %lld rx_sa %d already exist\n",
+			   sci, assoc_num);
+		err = -EEXIST;
+		goto out;
+	}
+
+	rx_sa = kzalloc(sizeof(*rx_sa), GFP_KERNEL);
+	if (!rx_sa) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	rx_sa->active = ctx_rx_sa->active;
+	rx_sa->next_pn = ctx_rx_sa->next_pn;
+	rx_sa->sci = sci;
+	rx_sa->assoc_num = assoc_num;
+	rx_sa->fs_id = rx_sc->sc_xarray_element->fs_id;
+
+	if (ctx->secy->xpn)
+		update_macsec_epn(rx_sa, &ctx_rx_sa->key, &ctx_rx_sa->next_pn_halves);
+
+	err = mlx5_create_encryption_key(mdev, ctx->sa.key, ctx->secy->key_len,
+					 MLX5_ACCEL_OBJ_MACSEC_KEY,
+					 &rx_sa->enc_key_id);
+	if (err)
+		goto destroy_sa;
+
+	rx_sc->rx_sa[assoc_num] = rx_sa;
+	if (!rx_sa->active)
+		goto out;
+
+	//TODO - add support for both authentication and encryption flows
+	err = mlx5e_macsec_init_sa(ctx, rx_sa, true, false);
+	if (err)
+		goto destroy_encryption_key;
+
+	goto out;
+
+destroy_encryption_key:
+	rx_sc->rx_sa[assoc_num] = NULL;
+	mlx5_destroy_encryption_key(mdev, rx_sa->enc_key_id);
+destroy_sa:
+	kfree(rx_sa);
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static int mlx5e_macsec_upd_rxsa(struct macsec_context *ctx)
+{
+	const struct macsec_rx_sa *ctx_rx_sa = ctx->sa.rx_sa;
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_macsec_device *macsec_device;
+	u8 assoc_num = ctx->sa.assoc_num;
+	struct mlx5e_macsec_rx_sc *rx_sc;
+	sci_t sci = ctx_rx_sa->sc->sci;
+	struct mlx5e_macsec_sa *rx_sa;
+	struct mlx5e_macsec *macsec;
+	struct list_head *list;
+	int err = 0;
+
+	mutex_lock(&priv->macsec->lock);
+
+	macsec = priv->macsec;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	list = &macsec_device->macsec_rx_sc_list_head;
+	rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, sci);
+	if (!rx_sc) {
+		netdev_err(ctx->netdev,
+			   "MACsec offload rx_sc sci %lld doesn't exist\n",
+			   ctx->sa.rx_sa->sc->sci);
+		err = -EINVAL;
+		goto out;
+	}
+
+	rx_sa = rx_sc->rx_sa[assoc_num];
+	if (rx_sa) {
+		netdev_err(ctx->netdev,
+			   "MACsec offload rx_sc sci %lld rx_sa %d already exist\n",
+			   sci, assoc_num);
+		err = -EEXIST;
+		goto out;
+	}
+
+	if (rx_sa->next_pn != ctx_rx_sa->next_pn_halves.lower) {
+		netdev_err(ctx->netdev,
+			   "MACsec offload update RX sa %d PN isn't supported\n",
+			   assoc_num);
+		err = -EINVAL;
+		goto out;
+	}
+
+	err = mlx5e_macsec_update_rx_sa(macsec, rx_sa, ctx_rx_sa->active);
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static int mlx5e_macsec_del_rxsa(struct macsec_context *ctx)
+{
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_macsec_device *macsec_device;
+	sci_t sci = ctx->sa.rx_sa->sc->sci;
+	struct mlx5e_macsec_rx_sc *rx_sc;
+	u8 assoc_num = ctx->sa.assoc_num;
+	struct mlx5e_macsec_sa *rx_sa;
+	struct mlx5e_macsec *macsec;
+	struct list_head *list;
+	int err = 0;
+
+	mutex_lock(&priv->macsec->lock);
+
+	macsec = priv->macsec;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	list = &macsec_device->macsec_rx_sc_list_head;
+	rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, sci);
+	if (!rx_sc) {
+		netdev_err(ctx->netdev,
+			   "MACsec offload rx_sc sci %lld doesn't exist\n",
+			   ctx->sa.rx_sa->sc->sci);
+		err = -EINVAL;
+		goto out;
+	}
+
+	rx_sa = rx_sc->rx_sa[assoc_num];
+	if (rx_sa) {
+		netdev_err(ctx->netdev,
+			   "MACsec offload rx_sc sci %lld rx_sa %d already exist\n",
+			   sci, assoc_num);
+		err = -EEXIST;
+		goto out;
+	}
+
+	mlx5e_macsec_cleanup_sa(macsec, rx_sa, false);
+	mlx5_destroy_encryption_key(macsec->mdev, rx_sa->enc_key_id);
+	kfree(rx_sa);
+	rx_sc->rx_sa[assoc_num] = NULL;
+
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static int mlx5e_macsec_add_secy(struct macsec_context *ctx)
+{
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	const struct net_device *dev = ctx->secy->netdev;
+	const struct net_device *netdev = ctx->netdev;
+	struct mlx5e_macsec_device *macsec_device;
+	struct mlx5e_macsec *macsec;
+	int err = 0;
+
+	if (!mlx5e_macsec_secy_features_validate(ctx))
+		return -EINVAL;
+
+	mutex_lock(&priv->macsec->lock);
+	macsec = priv->macsec;
+	if (mlx5e_macsec_get_macsec_device_context(macsec, ctx)) {
+		netdev_err(netdev, "MACsec offload: MACsec net_device already exist\n");
+		goto out;
+	}
+
+	if (macsec->num_of_devices >= MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES) {
+		netdev_err(netdev, "Currently, only %d MACsec offload devices can be set\n",
+			   MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES);
+		err = -EBUSY;
+		goto out;
+	}
+
+	macsec_device = kzalloc(sizeof(*macsec_device), GFP_KERNEL);
+	if (!macsec_device) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	macsec_device->dev_addr = kmemdup(dev->dev_addr, dev->addr_len, GFP_KERNEL);
+	if (!macsec_device->dev_addr) {
+		kfree(macsec_device);
+		err = -ENOMEM;
+		goto out;
+	}
+
+	macsec_device->netdev = dev;
+
+	INIT_LIST_HEAD_RCU(&macsec_device->macsec_rx_sc_list_head);
+	list_add_rcu(&macsec_device->macsec_device_list_element, &macsec->macsec_device_list_head);
+
+	++macsec->num_of_devices;
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static int macsec_upd_secy_hw_address(struct macsec_context *ctx,
+				      struct mlx5e_macsec_device *macsec_device)
+{
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	const struct net_device *dev = ctx->secy->netdev;
+	struct mlx5e_macsec *macsec = priv->macsec;
+	struct mlx5e_macsec_rx_sc *rx_sc, *tmp;
+	struct mlx5e_macsec_sa *rx_sa;
+	struct list_head *list;
+	int i, err = 0;
+
+
+	list = &macsec_device->macsec_rx_sc_list_head;
+	list_for_each_entry_safe(rx_sc, tmp, list, rx_sc_list_element) {
+		for (i = 0; i < MACSEC_NUM_AN; ++i) {
+			rx_sa = rx_sc->rx_sa[i];
+			if (!rx_sa || !rx_sa->macsec_rule)
+				continue;
+
+			mlx5e_macsec_cleanup_sa(macsec, rx_sa, false);
+		}
+	}
+
+	list_for_each_entry_safe(rx_sc, tmp, list, rx_sc_list_element) {
+		for (i = 0; i < MACSEC_NUM_AN; ++i) {
+			rx_sa = rx_sc->rx_sa[i];
+			if (!rx_sa)
+				continue;
+
+			if (rx_sa->active) {
+				err = mlx5e_macsec_init_sa(ctx, rx_sa, false, false);
+				if (err)
+					goto out;
+			}
+		}
+	}
+
+	memcpy(macsec_device->dev_addr, dev->dev_addr, dev->addr_len);
+out:
+	return err;
+}
+
+/* this function is called from 2 macsec ops functions:
+ *  macsec_set_mac_address – MAC address was changed, therefore we need to destroy
+ *  and create new Tx contexts(macsec object + steering).
+ *  macsec_changelink – in this case the tx SC or SecY may be changed, therefore need to
+ *  destroy Tx and Rx contexts(macsec object + steering)
+ */
+static int mlx5e_macsec_upd_secy(struct macsec_context *ctx)
+{
+	const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc;
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	const struct net_device *dev = ctx->secy->netdev;
+	struct mlx5e_macsec_device *macsec_device;
+	struct mlx5e_macsec_sa *tx_sa;
+	struct mlx5e_macsec *macsec;
+	int i, err = 0;
+
+	if (!mlx5e_macsec_secy_features_validate(ctx))
+		return -EINVAL;
+
+	mutex_lock(&priv->macsec->lock);
+
+	macsec = priv->macsec;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	/* if the dev_addr hasn't change, it mean the callback is from macsec_changelink */
+	if (!memcmp(macsec_device->dev_addr, dev->dev_addr, dev->addr_len)) {
+		err = macsec_upd_secy_hw_address(ctx, macsec_device);
+		if (err)
+			goto out;
+	}
+
+	for (i = 0; i < MACSEC_NUM_AN; ++i) {
+		tx_sa = macsec_device->tx_sa[i];
+		if (!tx_sa)
+			continue;
+
+		mlx5e_macsec_cleanup_sa(macsec, tx_sa, true);
+	}
+
+	for (i = 0; i < MACSEC_NUM_AN; ++i) {
+		tx_sa = macsec_device->tx_sa[i];
+		if (!tx_sa)
+			continue;
+
+		if (tx_sa->assoc_num == tx_sc->encoding_sa && tx_sa->active) {
+			err = mlx5e_macsec_init_sa(ctx, tx_sa, tx_sc->encrypt, true);
+			if (err)
+				goto out;
+		}
+	}
+
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static int mlx5e_macsec_del_secy(struct macsec_context *ctx)
+{
+	struct mlx5e_priv *priv = netdev_priv(ctx->netdev);
+	struct mlx5e_macsec_device *macsec_device;
+	struct mlx5e_macsec_rx_sc *rx_sc, *tmp;
+	struct mlx5e_macsec_sa *rx_sa;
+	struct mlx5e_macsec_sa *tx_sa;
+	struct mlx5e_macsec *macsec;
+	struct list_head *list;
+	int err = 0;
+	int i;
+
+	mutex_lock(&priv->macsec->lock);
+	macsec = priv->macsec;
+	macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
+	if (!macsec_device) {
+		netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
+		err = -EINVAL;
+
+		goto out;
+	}
+
+	for (i = 0; i < MACSEC_NUM_AN; ++i) {
+		tx_sa = macsec_device->tx_sa[i];
+		if (!tx_sa)
+			continue;
+
+		mlx5e_macsec_cleanup_sa(macsec, tx_sa, true);
+		mlx5_destroy_encryption_key(macsec->mdev, tx_sa->enc_key_id);
+		kfree(tx_sa);
+		macsec_device->tx_sa[i] = NULL;
+	}
+
+	list = &macsec_device->macsec_rx_sc_list_head;
+	list_for_each_entry_safe(rx_sc, tmp, list, rx_sc_list_element) {
+		for (i = 0; i < MACSEC_NUM_AN; ++i) {
+			rx_sa = rx_sc->rx_sa[i];
+			if (!rx_sa)
+				continue;
+
+			mlx5e_macsec_cleanup_sa(macsec, rx_sa, false);
+			mlx5_destroy_encryption_key(macsec->mdev, rx_sa->enc_key_id);
+			kfree(rx_sa);
+			rx_sc->rx_sa[i] = NULL;
+		}
+
+		list_del_rcu(&rx_sc->rx_sc_list_element);
+
+		kfree_rcu(rx_sc);
+	}
+
+	kfree(macsec_device->dev_addr);
+	macsec_device->dev_addr = NULL;
+
+	list_del_rcu(&macsec_device->macsec_device_list_element);
+	--macsec->num_of_devices;
+
+out:
+	mutex_unlock(&macsec->lock);
+
+	return err;
+}
+
+static void macsec_build_accel_attrs(struct mlx5e_macsec_sa *sa,
+				     struct mlx5_macsec_obj_attrs *attrs)
+{
+	attrs->epn_state.epn_msb = sa->epn_state.epn_msb;
+	attrs->epn_state.overlap = sa->epn_state.overlap;
+}
+
+static void macsec_aso_build_wqe_ctrl_seg(struct mlx5e_macsec_aso *macsec_aso,
+					  struct mlx5_wqe_aso_ctrl_seg *aso_ctrl,
+					  struct mlx5_aso_ctrl_param *param)
+{
+	memset(aso_ctrl, 0, sizeof(*aso_ctrl));
+	if (macsec_aso->umr->dma_addr) {
+		aso_ctrl->va_l  = cpu_to_be32(macsec_aso->umr->dma_addr | ASO_CTRL_READ_EN);
+		aso_ctrl->va_h  = cpu_to_be32((u64)macsec_aso->umr->dma_addr >> 32);
+		aso_ctrl->l_key = cpu_to_be32(macsec_aso->umr->mkey);
+	}
+
+	if (!param)
+		return;
+
+	aso_ctrl->data_mask_mode = param->data_mask_mode << 6;
+	aso_ctrl->condition_1_0_operand = param->condition_1_operand |
+						param->condition_0_operand << 4;
+	aso_ctrl->condition_1_0_offset = param->condition_1_offset |
+						param->condition_0_offset << 4;
+	aso_ctrl->data_offset_condition_operand = param->data_offset |
+						param->condition_operand << 6;
+	aso_ctrl->condition_0_data = cpu_to_be32(param->condition_0_data);
+	aso_ctrl->condition_0_mask = cpu_to_be32(param->condition_0_mask);
+	aso_ctrl->condition_1_data = cpu_to_be32(param->condition_1_data);
+	aso_ctrl->condition_1_mask = cpu_to_be32(param->condition_1_mask);
+	aso_ctrl->bitwise_data = cpu_to_be64(param->bitwise_data);
+	aso_ctrl->data_mask = cpu_to_be64(param->data_mask);
+}
+
+static int mlx5e_macsec_modify_obj(struct mlx5_core_dev *mdev, struct mlx5_macsec_obj_attrs *attrs,
+				   u32 macsec_id)
+{
+	u32 in[MLX5_ST_SZ_DW(modify_macsec_obj_in)] = {};
+	u32 out[MLX5_ST_SZ_DW(query_macsec_obj_out)];
+	u64 modify_field_select = 0;
+	void *obj;
+	int err;
+
+	/* General object fields set */
+	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_MACSEC);
+	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, macsec_id);
+	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+	if (err) {
+		mlx5_core_err(mdev, "Query MACsec object failed (Object id %d), err = %d\n",
+			      macsec_id, err);
+		return err;
+	}
+
+	obj = MLX5_ADDR_OF(query_macsec_obj_out, out, macsec_object);
+	modify_field_select = MLX5_GET64(macsec_offload_obj, obj, modify_field_select);
+
+	/* EPN */
+	if (!(modify_field_select & MLX5_MODIFY_MACSEC_BITMASK_EPN_OVERLAP) ||
+	    !(modify_field_select & MLX5_MODIFY_MACSEC_BITMASK_EPN_MSB)) {
+		mlx5_core_dbg(mdev, "MACsec object field is not modifiable (Object id %d)\n",
+			      macsec_id);
+		return -EOPNOTSUPP;
+	}
+
+	obj = MLX5_ADDR_OF(modify_macsec_obj_in, in, macsec_object);
+	MLX5_SET64(macsec_offload_obj, obj, modify_field_select,
+		   MLX5_MODIFY_MACSEC_BITMASK_EPN_OVERLAP | MLX5_MODIFY_MACSEC_BITMASK_EPN_MSB);
+	MLX5_SET(macsec_offload_obj, obj, epn_msb, attrs->epn_state.epn_msb);
+	MLX5_SET(macsec_offload_obj, obj, epn_overlap, attrs->epn_state.overlap);
+
+	/* General object fields set */
+	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
+
+	return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
+}
+
+static void macsec_aso_build_ctrl(struct mlx5e_macsec_aso *aso,
+				  struct mlx5_wqe_aso_ctrl_seg *aso_ctrl,
+				  struct mlx5e_macsec_aso_in *in)
+{
+	struct mlx5_aso_ctrl_param param = {};
+
+	param.data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT;
+	param.condition_0_operand = MLX5_ASO_ALWAYS_TRUE;
+	param.condition_1_operand = MLX5_ASO_ALWAYS_TRUE;
+	if (in->mode == MLX5_MACSEC_EPN) {
+		param.data_offset = MLX5_MACSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
+		param.bitwise_data = BIT_ULL(54);
+		param.data_mask = param.bitwise_data;
+	}
+	macsec_aso_build_wqe_ctrl_seg(aso, aso_ctrl, &param);
+}
+
+static int macsec_aso_set_arm_event(struct mlx5_core_dev *mdev, struct mlx5e_macsec *macsec,
+				    struct mlx5e_macsec_aso_in *in)
+{
+	struct mlx5e_macsec_aso *aso;
+	struct mlx5_aso_wqe *aso_wqe;
+	struct mlx5_aso *maso;
+	int err;
+
+	aso = &macsec->aso;
+	maso = aso->maso;
+
+	mutex_lock(&aso->aso_lock);
+	aso_wqe = mlx5_aso_get_wqe(maso);
+	mlx5_aso_build_wqe(maso, MLX5_MACSEC_ASO_DS_CNT, aso_wqe, in->obj_id,
+			   MLX5_ACCESS_ASO_OPC_MOD_MACSEC);
+	macsec_aso_build_ctrl(aso, &aso_wqe->aso_ctrl, in);
+	mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl);
+	err = mlx5_aso_poll_cq(maso, false, 10);
+	mutex_unlock(&aso->aso_lock);
+
+	return err;
+}
+
+static int macsec_aso_query(struct mlx5_core_dev *mdev, struct mlx5e_macsec *macsec,
+			    struct mlx5e_macsec_aso_in *in, struct mlx5e_macsec_aso_out *out)
+{
+	struct mlx5e_macsec_aso *aso;
+	struct mlx5_aso_wqe *aso_wqe;
+	struct mlx5_aso *maso;
+	int err;
+
+	aso = &macsec->aso;
+	maso = aso->maso;
+
+	mutex_lock(&aso->aso_lock);
+
+	aso_wqe = mlx5_aso_get_wqe(maso);
+	mlx5_aso_build_wqe(maso, MLX5_MACSEC_ASO_DS_CNT, aso_wqe, in->obj_id,
+			   MLX5_ACCESS_ASO_OPC_MOD_MACSEC);
+	macsec_aso_build_wqe_ctrl_seg(aso, &aso_wqe->aso_ctrl, NULL);
+
+	mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl);
+	err = mlx5_aso_poll_cq(maso, false, 10);
+	if (err)
+		goto err_out;
+
+	if (MLX5_GET(macsec_aso, aso->umr->ctx, epn_event_arm))
+		out->event_arm |= MLX5E_ASO_EPN_ARM;
+
+	out->mode_param = MLX5_GET(macsec_aso, aso->umr->ctx, mode_parameter);
+
+err_out:
+	mutex_unlock(&aso->aso_lock);
+	return err;
+}
+
+static struct mlx5e_macsec_sa *get_macsec_tx_sa_from_obj_id(const struct mlx5e_macsec *macsec,
+							    const u32 obj_id)
+{
+	const struct list_head *device_list;
+	struct mlx5e_macsec_sa *macsec_sa;
+	struct mlx5e_macsec_device *iter;
+	int i;
+
+	device_list = &macsec->macsec_device_list_head;
+
+	list_for_each_entry(iter, device_list, macsec_device_list_element) {
+		for (i = 0; i < MACSEC_NUM_AN; ++i) {
+			macsec_sa = iter->tx_sa[i];
+			if (!macsec_sa || !macsec_sa->active)
+				continue;
+			if (macsec_sa->macsec_obj_id == obj_id)
+				return macsec_sa;
+		}
+	}
+
+	return NULL;
+}
+
+static struct mlx5e_macsec_sa *get_macsec_rx_sa_from_obj_id(const struct mlx5e_macsec *macsec,
+							    const u32 obj_id)
+{
+	const struct list_head *device_list, *sc_list;
+	struct mlx5e_macsec_rx_sc *mlx5e_rx_sc;
+	struct mlx5e_macsec_sa *macsec_sa;
+	struct mlx5e_macsec_device *iter;
+	int i;
+
+	device_list = &macsec->macsec_device_list_head;
+
+	list_for_each_entry(iter, device_list, macsec_device_list_element) {
+		sc_list = &iter->macsec_rx_sc_list_head;
+		list_for_each_entry(mlx5e_rx_sc, sc_list, rx_sc_list_element) {
+			for (i = 0; i < MACSEC_NUM_AN; ++i) {
+				macsec_sa = mlx5e_rx_sc->rx_sa[i];
+				if (!macsec_sa || !macsec_sa->active)
+					continue;
+				if (macsec_sa->macsec_obj_id == obj_id)
+					return macsec_sa;
+			}
+		}
+	}
+
+	return NULL;
+}
+
+static void macsec_epn_update(struct mlx5e_macsec *macsec, struct mlx5_core_dev *mdev,
+			      struct mlx5e_macsec_sa *sa, u32 obj_id, u32 mode_param)
+{
+	struct mlx5_macsec_obj_attrs attrs = {};
+	struct mlx5e_macsec_aso_in in = {};
+
+	/* When the bottom of the replay protection window (mode_param) crosses 2^31 (half sequence
+	 * number wraparound) hence mode_param > MLX5_MACSEC_EPN_SCOPE_MID the SW should update the
+	 * esn_overlap to OLD (1).
+	 * When the bottom of the replay protection window (mode_param) crosses 2^32 (full sequence
+	 * number wraparound) hence mode_param < MLX5_MACSEC_EPN_SCOPE_MID since it did a
+	 * wraparound, the SW should update the esn_overlap to NEW (0), and increment the esn_msb.
+	 */
+
+	if (mode_param < MLX5_MACSEC_EPN_SCOPE_MID) {
+		sa->epn_state.epn_msb++;
+		sa->epn_state.overlap = 0;
+	} else {
+		sa->epn_state.overlap = 1;
+	}
+
+	macsec_build_accel_attrs(sa, &attrs);
+	mlx5e_macsec_modify_obj(mdev, &attrs, obj_id);
+
+	/* Re-set EPN arm event */
+	in.obj_id = obj_id;
+	in.mode = MLX5_MACSEC_EPN;
+	macsec_aso_set_arm_event(mdev, macsec, &in);
+}
+
+static void macsec_async_event(struct work_struct *work)
+{
+	struct mlx5e_macsec_async_work *async_work;
+	struct mlx5e_macsec_aso_out out = {};
+	struct mlx5e_macsec_aso_in in = {};
+	struct mlx5e_macsec_sa *macsec_sa;
+	struct mlx5e_macsec *macsec;
+	struct mlx5_core_dev *mdev;
+	u32 obj_id;
+
+	async_work = container_of(work, struct mlx5e_macsec_async_work, work);
+	macsec = async_work->macsec;
+	mdev = async_work->mdev;
+	obj_id = async_work->obj_id;
+	macsec_sa = get_macsec_tx_sa_from_obj_id(macsec, obj_id);
+	if (!macsec_sa) {
+		macsec_sa = get_macsec_rx_sa_from_obj_id(macsec, obj_id);
+		if (!macsec_sa) {
+			mlx5_core_dbg(mdev, "MACsec SA is not found (SA object id %d)\n", obj_id);
+			goto out_async_work;
+		}
+	}
+
+	/* Query MACsec ASO context */
+	in.obj_id = obj_id;
+	macsec_aso_query(mdev, macsec, &in, &out);
+
+	/* EPN case */
+	if (macsec_sa->epn_state.epn_enabled && !(out.event_arm & MLX5E_ASO_EPN_ARM))
+		macsec_epn_update(macsec, mdev, macsec_sa, obj_id, out.mode_param);
+
+out_async_work:
+	kfree(async_work);
+}
+
+static int macsec_obj_change_event(struct notifier_block *nb, unsigned long event, void *data)
+{
+	struct mlx5e_macsec *macsec = container_of(nb, struct mlx5e_macsec, nb);
+	struct mlx5e_macsec_async_work *async_work;
+	struct mlx5_eqe_obj_change *obj_change;
+	struct mlx5_eqe *eqe = data;
+	u16 obj_type;
+	u32 obj_id;
+
+	if (event != MLX5_EVENT_TYPE_OBJECT_CHANGE)
+		return NOTIFY_DONE;
+
+	obj_change = &eqe->data.obj_change;
+	obj_type = be16_to_cpu(obj_change->obj_type);
+	obj_id = be32_to_cpu(obj_change->obj_id);
+
+	if (obj_type != MLX5_GENERAL_OBJECT_TYPES_MACSEC)
+		return NOTIFY_DONE;
+
+	async_work = kzalloc(sizeof(*async_work), GFP_ATOMIC);
+	if (!async_work)
+		return NOTIFY_DONE;
+
+	async_work->macsec = macsec;
+	async_work->mdev = macsec->mdev;
+	async_work->obj_id = obj_id;
+
+	INIT_WORK(&async_work->work, macsec_async_event);
+
+	WARN_ON(!queue_work(macsec->wq, &async_work->work));
+
+	return NOTIFY_OK;
+}
+
+static int mlx5e_macsec_aso_init(struct mlx5e_macsec_aso *aso, struct mlx5_core_dev *mdev)
+{
+	struct mlx5_aso *maso;
+	int err;
+
+	err = mlx5_core_alloc_pd(mdev, &aso->pdn);
+	if (err) {
+		mlx5_core_err(mdev,
+			      "MACsec offload: Failed to alloc pd for MACsec ASO, err=%d\n",
+			      err);
+		return err;
+	}
+
+	maso = mlx5_aso_create(mdev, aso->pdn);
+	if (IS_ERR(maso)) {
+		err = PTR_ERR(maso);
+		goto err_aso;
+	}
+
+	err = mlx5e_macsec_aso_reg_mr(mdev, aso);
+	if (err)
+		goto err_aso_reg;
+
+	mutex_init(&aso->aso_lock);
+
+	aso->maso = maso;
+
+	return 0;
+
+err_aso_reg:
+	mlx5_aso_destroy(maso);
+err_aso:
+	mlx5_core_dealloc_pd(mdev, aso->pdn);
+	return err;
+}
+
+static void mlx5e_macsec_aso_cleanup(struct mlx5e_macsec_aso *aso, struct mlx5_core_dev *mdev)
+{
+	if (!aso)
+		return;
+
+	mlx5e_macsec_aso_dereg_mr(mdev, aso);
+
+	mlx5_aso_destroy(aso->maso);
+
+	mlx5_core_dealloc_pd(mdev, aso->pdn);
+}
+
+bool mlx5e_is_macsec_device(const struct mlx5_core_dev *mdev)
+{
+	if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) &
+	    MLX5_GENERAL_OBJ_TYPES_CAP_MACSEC_OFFLOAD))
+		return false;
+
+	if (!MLX5_CAP_GEN(mdev, log_max_dek))
+		return false;
+
+	if (!MLX5_CAP_MACSEC(mdev, log_max_macsec_offload))
+		return false;
+
+	if (!MLX5_CAP_FLOWTABLE_NIC_RX(mdev, macsec_decrypt) ||
+	    !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_remove_macsec))
+		return false;
+
+	if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, macsec_encrypt) ||
+	    !MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_macsec))
+		return false;
+
+	if (!MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_128_encrypt) &&
+	    !MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_256_encrypt))
+		return false;
+
+	if (!MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_128_decrypt) &&
+	    !MLX5_CAP_MACSEC(mdev, macsec_crypto_esp_aes_gcm_256_decrypt))
+		return false;
+
+	return true;
+}
+
+void mlx5e_macsec_get_stats_fill(struct mlx5e_macsec *macsec, void *macsec_stats)
+{
+	mlx5e_macsec_fs_get_stats_fill(macsec->macsec_fs, macsec_stats);
+}
+
+struct mlx5e_macsec_stats *mlx5e_macsec_get_stats(struct mlx5e_macsec *macsec)
+{
+	if (!macsec)
+		return NULL;
+
+	return &macsec->stats;
+}
+
+static const struct macsec_ops macsec_offload_ops = {
+	.mdo_add_txsa = mlx5e_macsec_add_txsa,
+	.mdo_upd_txsa = mlx5e_macsec_upd_txsa,
+	.mdo_del_txsa = mlx5e_macsec_del_txsa,
+	.mdo_add_rxsc = mlx5e_macsec_add_rxsc,
+	.mdo_upd_rxsc = mlx5e_macsec_upd_rxsc,
+	.mdo_del_rxsc = mlx5e_macsec_del_rxsc,
+	.mdo_add_rxsa = mlx5e_macsec_add_rxsa,
+	.mdo_upd_rxsa = mlx5e_macsec_upd_rxsa,
+	.mdo_del_rxsa = mlx5e_macsec_del_rxsa,
+	.mdo_add_secy = mlx5e_macsec_add_secy,
+	.mdo_upd_secy = mlx5e_macsec_upd_secy,
+	.mdo_del_secy = mlx5e_macsec_del_secy,
+};
+
+bool mlx5e_macsec_handle_tx_skb(struct mlx5e_macsec *macsec, struct sk_buff *skb)
+{
+	struct metadata_dst *md_dst = skb_metadata_dst(skb);
+	u32 fs_id;
+
+	fs_id = mlx5e_macsec_get_sa_from_hashtable(&macsec->sci_hash, &md_dst->u.macsec_info.sci);
+	if (!fs_id)
+		goto err_out;
+
+	return true;
+
+err_out:
+	dev_kfree_skb_any(skb);
+	return false;
+}
+
+void mlx5e_macsec_tx_build_eseg(struct mlx5e_macsec *macsec,
+				struct sk_buff *skb,
+				struct mlx5_wqe_eth_seg *eseg)
+{
+	struct metadata_dst *md_dst = skb_metadata_dst(skb);
+	u32 fs_id;
+
+	fs_id = mlx5e_macsec_get_sa_from_hashtable(&macsec->sci_hash, &md_dst->u.macsec_info.sci);
+	if (!fs_id)
+		return;
+
+	eseg->flow_table_metadata = cpu_to_be32(MLX5_ETH_WQE_FT_META_MACSEC | fs_id << 2);
+}
+
+void mlx5e_macsec_offload_handle_rx_skb(struct net_device *netdev,
+					struct sk_buff *skb,
+					struct mlx5_cqe64 *cqe)
+{
+	struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element;
+	u32 macsec_meta_data = be32_to_cpu(cqe->ft_metadata);
+	struct mlx5e_priv *priv = netdev_priv(netdev);
+	struct mlx5e_macsec_rx_sc *rx_sc;
+	struct mlx5e_macsec *macsec;
+	u32  fs_id;
+
+	macsec = priv->macsec;
+	if (!macsec)
+		return;
+
+	fs_id = MLX5_MACSEC_METADATA_HANDLE(macsec_meta_data);
+
+	rcu_read_lock();
+	sc_xarray_element = xa_load(&macsec->sc_xarray, fs_id);
+	rx_sc = sc_xarray_element->rx_sc;
+	if (rx_sc) {
+		dst_hold(&rx_sc->md_dst->dst);
+		skb_dst_set(skb, &rx_sc->md_dst->dst);
+	}
+
+	rcu_read_unlock();
+}
+
+void mlx5e_macsec_build_netdev(struct mlx5e_priv *priv)
+{
+	struct net_device *netdev = priv->netdev;
+
+	if (!mlx5e_is_macsec_device(priv->mdev))
+		return;
+
+	/* Enable MACsec */
+	mlx5_core_dbg(priv->mdev, "mlx5e: MACsec acceleration enabled\n");
+	netdev->macsec_ops = &macsec_offload_ops;
+	netdev->features |= NETIF_F_HW_MACSEC;
+	netif_keep_dst(netdev);
+}
+
+int mlx5e_macsec_init(struct mlx5e_priv *priv)
+{
+	struct mlx5_core_dev *mdev = priv->mdev;
+	struct mlx5e_macsec *macsec = NULL;
+	struct mlx5e_macsec_fs *macsec_fs;
+	int err;
+
+	if (!mlx5e_is_macsec_device(priv->mdev)) {
+		mlx5_core_dbg(mdev, "Not a MACsec offload device\n");
+		return 0;
+	}
+
+	macsec = kzalloc(sizeof(*macsec), GFP_KERNEL);
+	if (!macsec)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&macsec->macsec_device_list_head);
+	mutex_init(&macsec->lock);
+
+	err = rhashtable_init(&macsec->sci_hash, &rhash_sci);
+	if (err) {
+		mlx5_core_err(mdev, "MACsec offload: Failed to init SCI hash table, err=%d\n",
+			      err);
+		goto err_hash;
+	}
+
+	err = mlx5e_macsec_aso_init(&macsec->aso, priv->mdev);
+	if (err) {
+		mlx5_core_err(mdev, "MACsec offload: Failed to init aso, err=%d\n", err);
+		goto err_aso;
+	}
+
+	macsec->wq = alloc_ordered_workqueue("mlx5e_macsec_%s", 0, priv->netdev->name);
+	if (!macsec->wq) {
+		err = -ENOMEM;
+		goto err_wq;
+	}
+
+	xa_init_flags(&macsec->sc_xarray, XA_FLAGS_ALLOC1);
+
+	priv->macsec = macsec;
+
+	macsec->mdev = mdev;
+
+	macsec_fs = mlx5e_macsec_fs_init(mdev, priv->netdev);
+	if (!macsec_fs) {
+		err = -ENOMEM;
+		goto err_out;
+	}
+
+	macsec->macsec_fs = macsec_fs;
+
+	macsec->nb.notifier_call = macsec_obj_change_event;
+	mlx5_notifier_register(mdev, &macsec->nb);
+
+	mlx5_core_dbg(mdev, "MACsec attached to netdevice\n");
+
+	return 0;
+
+err_out:
+	destroy_workqueue(macsec->wq);
+err_wq:
+	mlx5e_macsec_aso_cleanup(&macsec->aso, priv->mdev);
+err_aso:
+	rhashtable_destroy(&macsec->sci_hash);
+err_hash:
+	kfree(macsec);
+	priv->macsec = NULL;
+	return err;
+}
+
+void mlx5e_macsec_cleanup(struct mlx5e_priv *priv)
+{
+	struct mlx5e_macsec *macsec = priv->macsec;
+	struct mlx5_core_dev *mdev = macsec->mdev;
+
+	if (!macsec)
+		return;
+
+	mlx5_notifier_unregister(mdev, &macsec->nb);
+
+	mlx5e_macsec_fs_cleanup(macsec->macsec_fs);
+
+	/* Cleanup workqueue */
+	destroy_workqueue(macsec->wq);
+
+	mlx5e_macsec_aso_cleanup(&macsec->aso, mdev);
+
+	priv->macsec = NULL;
+
+	rhashtable_destroy(&macsec->sci_hash);
+
+	mutex_destroy(&macsec->lock);
+
+	kfree(macsec);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.h
new file mode 100644
index 000000000000..d580b4a91253
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec.h
@@ -0,0 +1,71 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#ifndef __MLX5_EN_ACCEL_MACSEC_H__
+#define __MLX5_EN_ACCEL_MACSEC_H__
+
+#ifdef CONFIG_MLX5_EN_MACSEC
+
+#include <linux/mlx5/driver.h>
+#include <net/macsec.h>
+#include <net/dst_metadata.h>
+
+/* Bit31 - 30: MACsec marker, Bit3-0: MACsec id */
+#define MLX5_MACSEC_METADATA_MARKER(metadata)  ((((metadata) >> 30) & 0x3)  == 0x1)
+#define MLX5_MACSEC_METADATA_HANDLE(metadata)  ((metadata) & GENMASK(3, 0))
+
+struct mlx5e_priv;
+struct mlx5e_macsec;
+
+struct mlx5e_macsec_stats {
+	u64 macsec_rx_pkts;
+	u64 macsec_rx_bytes;
+	u64 macsec_rx_pkts_drop;
+	u64 macsec_rx_bytes_drop;
+	u64 macsec_tx_pkts;
+	u64 macsec_tx_bytes;
+	u64 macsec_tx_pkts_drop;
+	u64 macsec_tx_bytes_drop;
+};
+
+void mlx5e_macsec_build_netdev(struct mlx5e_priv *priv);
+int mlx5e_macsec_init(struct mlx5e_priv *priv);
+void mlx5e_macsec_cleanup(struct mlx5e_priv *priv);
+bool mlx5e_macsec_handle_tx_skb(struct mlx5e_macsec *macsec, struct sk_buff *skb);
+void mlx5e_macsec_tx_build_eseg(struct mlx5e_macsec *macsec,
+				struct sk_buff *skb,
+				struct mlx5_wqe_eth_seg *eseg);
+
+static inline bool mlx5e_macsec_skb_is_offload(struct sk_buff *skb)
+{
+	struct metadata_dst *md_dst = skb_metadata_dst(skb);
+
+	return md_dst && (md_dst->type == METADATA_MACSEC);
+}
+
+static inline bool mlx5e_macsec_is_rx_flow(struct mlx5_cqe64 *cqe)
+{
+	return MLX5_MACSEC_METADATA_MARKER(be32_to_cpu(cqe->ft_metadata));
+}
+
+void mlx5e_macsec_offload_handle_rx_skb(struct net_device *netdev, struct sk_buff *skb,
+					struct mlx5_cqe64 *cqe);
+bool mlx5e_is_macsec_device(const struct mlx5_core_dev *mdev);
+void mlx5e_macsec_get_stats_fill(struct mlx5e_macsec *macsec, void *macsec_stats);
+struct mlx5e_macsec_stats *mlx5e_macsec_get_stats(struct mlx5e_macsec *macsec);
+
+#else
+
+static inline void mlx5e_macsec_build_netdev(struct mlx5e_priv *priv) {}
+static inline int mlx5e_macsec_init(struct mlx5e_priv *priv) { return 0; }
+static inline void mlx5e_macsec_cleanup(struct mlx5e_priv *priv) {}
+static inline bool mlx5e_macsec_skb_is_offload(struct sk_buff *skb) { return false; }
+static inline bool mlx5e_macsec_is_rx_flow(struct mlx5_cqe64 *cqe) { return false; }
+static inline void mlx5e_macsec_offload_handle_rx_skb(struct net_device *netdev,
+						      struct sk_buff *skb,
+						      struct mlx5_cqe64 *cqe)
+{}
+static inline bool mlx5e_is_macsec_device(const struct mlx5_core_dev *mdev) { return false; }
+#endif  /* CONFIG_MLX5_EN_MACSEC */
+
+#endif	/* __MLX5_ACCEL_EN_MACSEC_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c
new file mode 100644
index 000000000000..13dc628b988a
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.c
@@ -0,0 +1,1384 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#include <net/macsec.h>
+#include <linux/netdevice.h>
+#include <linux/mlx5/qp.h>
+#include "fs_core.h"
+#include "en/fs.h"
+#include "en_accel/macsec_fs.h"
+#include "mlx5_core.h"
+
+/* MACsec TX flow steering */
+#define CRYPTO_NUM_MAXSEC_FTE BIT(15)
+#define CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE 1
+
+#define TX_CRYPTO_TABLE_LEVEL 0
+#define TX_CRYPTO_TABLE_NUM_GROUPS 3
+#define TX_CRYPTO_TABLE_MKE_GROUP_SIZE 1
+#define TX_CRYPTO_TABLE_SA_GROUP_SIZE \
+	(CRYPTO_NUM_MAXSEC_FTE - (TX_CRYPTO_TABLE_MKE_GROUP_SIZE + \
+				  CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE))
+#define TX_CHECK_TABLE_LEVEL 1
+#define TX_CHECK_TABLE_NUM_FTE 2
+#define RX_CRYPTO_TABLE_LEVEL 0
+#define RX_CHECK_TABLE_LEVEL 1
+#define RX_CHECK_TABLE_NUM_FTE 3
+#define RX_CRYPTO_TABLE_NUM_GROUPS 3
+#define RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE \
+	((CRYPTO_NUM_MAXSEC_FTE - CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE) / 2)
+#define RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE \
+	(CRYPTO_NUM_MAXSEC_FTE - RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE)
+#define RX_NUM_OF_RULES_PER_SA 2
+
+#define MLX5_MACSEC_TAG_LEN 8 /* SecTAG length with ethertype and without the optional SCI */
+#define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK 0x23
+#define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET 0x8
+#define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET 0x5
+#define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT (0x1 << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET)
+#define MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI 0x8
+#define MLX5_SECTAG_HEADER_SIZE_WITH_SCI (MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI + MACSEC_SCI_LEN)
+
+/* MACsec RX flow steering */
+#define MLX5_ETH_WQE_FT_META_MACSEC_MASK 0x3E
+
+struct mlx5_sectag_header {
+	__be16 ethertype;
+	u8 tci_an;
+	u8 sl;
+	u32 pn;
+	u8 sci[MACSEC_SCI_LEN]; /* optional */
+}  __packed;
+
+struct mlx5e_macsec_tx_rule {
+	struct mlx5_flow_handle *rule;
+	struct mlx5_pkt_reformat *pkt_reformat;
+	u32 fs_id;
+};
+
+struct mlx5e_macsec_tables {
+	struct mlx5e_flow_table ft_crypto;
+	struct mlx5_flow_handle *crypto_miss_rule;
+
+	struct mlx5_flow_table *ft_check;
+	struct mlx5_flow_group  *ft_check_group;
+	struct mlx5_fc *check_miss_rule_counter;
+	struct mlx5_flow_handle *check_miss_rule;
+	struct mlx5_fc *check_rule_counter;
+
+	u32 refcnt;
+};
+
+struct mlx5e_macsec_tx {
+	struct mlx5_flow_handle *crypto_mke_rule;
+	struct mlx5_flow_handle *check_rule;
+
+	struct ida tx_halloc;
+
+	struct mlx5e_macsec_tables tables;
+};
+
+struct mlx5e_macsec_rx_rule {
+	struct mlx5_flow_handle *rule[RX_NUM_OF_RULES_PER_SA];
+	struct mlx5_modify_hdr *meta_modhdr;
+};
+
+struct mlx5e_macsec_rx {
+	struct mlx5_flow_handle *check_rule[2];
+	struct mlx5_pkt_reformat *check_rule_pkt_reformat[2];
+
+	struct mlx5e_macsec_tables tables;
+};
+
+union mlx5e_macsec_rule {
+	struct mlx5e_macsec_tx_rule tx_rule;
+	struct mlx5e_macsec_rx_rule rx_rule;
+};
+
+struct mlx5e_macsec_fs {
+	struct mlx5_core_dev *mdev;
+	struct net_device *netdev;
+	struct mlx5e_macsec_tx *tx_fs;
+	struct mlx5e_macsec_rx *rx_fs;
+};
+
+static void macsec_fs_tx_destroy(struct mlx5e_macsec_fs *macsec_fs)
+{
+	struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
+	struct mlx5e_macsec_tables *tx_tables;
+
+	tx_tables = &tx_fs->tables;
+
+	/* Tx check table */
+	if (tx_fs->check_rule) {
+		mlx5_del_flow_rules(tx_fs->check_rule);
+		tx_fs->check_rule = NULL;
+	}
+
+	if (tx_tables->check_miss_rule) {
+		mlx5_del_flow_rules(tx_tables->check_miss_rule);
+		tx_tables->check_miss_rule = NULL;
+	}
+
+	if (tx_tables->ft_check_group) {
+		mlx5_destroy_flow_group(tx_tables->ft_check_group);
+		tx_tables->ft_check_group = NULL;
+	}
+
+	if (tx_tables->ft_check) {
+		mlx5_destroy_flow_table(tx_tables->ft_check);
+		tx_tables->ft_check = NULL;
+	}
+
+	/* Tx crypto table */
+	if (tx_fs->crypto_mke_rule) {
+		mlx5_del_flow_rules(tx_fs->crypto_mke_rule);
+		tx_fs->crypto_mke_rule = NULL;
+	}
+
+	if (tx_tables->crypto_miss_rule) {
+		mlx5_del_flow_rules(tx_tables->crypto_miss_rule);
+		tx_tables->crypto_miss_rule = NULL;
+	}
+
+	mlx5e_destroy_flow_table(&tx_tables->ft_crypto);
+}
+
+static int macsec_fs_tx_create_crypto_table_groups(struct mlx5e_flow_table *ft)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	int mclen = MLX5_ST_SZ_BYTES(fte_match_param);
+	int ix = 0;
+	u32 *in;
+	int err;
+	u8 *mc;
+
+	ft->g = kcalloc(TX_CRYPTO_TABLE_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
+	if (!ft->g)
+		return -ENOMEM;
+	in = kvzalloc(inlen, GFP_KERNEL);
+
+	if (!in) {
+		kfree(ft->g);
+		return -ENOMEM;
+	}
+
+	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
+
+	/* Flow Group for MKE match */
+	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
+	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
+
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += TX_CRYPTO_TABLE_MKE_GROUP_SIZE;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
+	if (IS_ERR(ft->g[ft->num_groups]))
+		goto err;
+	ft->num_groups++;
+
+	/* Flow Group for SA rules */
+	memset(in, 0, inlen);
+	memset(mc, 0, mclen);
+	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS_2);
+	MLX5_SET(fte_match_param, mc, misc_parameters_2.metadata_reg_a,
+		 MLX5_ETH_WQE_FT_META_MACSEC_MASK);
+
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += TX_CRYPTO_TABLE_SA_GROUP_SIZE;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
+	if (IS_ERR(ft->g[ft->num_groups]))
+		goto err;
+	ft->num_groups++;
+
+	/* Flow Group for l2 traps */
+	memset(in, 0, inlen);
+	memset(mc, 0, mclen);
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
+	if (IS_ERR(ft->g[ft->num_groups]))
+		goto err;
+	ft->num_groups++;
+
+	kvfree(in);
+	return 0;
+
+err:
+	err = PTR_ERR(ft->g[ft->num_groups]);
+	ft->g[ft->num_groups] = NULL;
+	kvfree(in);
+
+	return err;
+}
+
+static struct mlx5_flow_table
+	*macsec_fs_auto_group_table_create(struct mlx5_flow_namespace *ns, int flags,
+					   int level, int max_fte)
+{
+	struct mlx5_flow_table_attr ft_attr = {};
+	struct mlx5_flow_table *fdb = NULL;
+
+	/* reserve entry for the match all miss group and rule */
+	ft_attr.autogroup.num_reserved_entries = 1;
+	ft_attr.autogroup.max_num_groups = 1;
+	ft_attr.prio = 0;
+	ft_attr.flags = flags;
+	ft_attr.level = level;
+	ft_attr.max_fte = max_fte;
+
+	fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
+
+	return fdb;
+}
+
+static int macsec_fs_tx_create(struct mlx5e_macsec_fs *macsec_fs)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
+	struct net_device *netdev = macsec_fs->netdev;
+	struct mlx5_flow_table_attr ft_attr = {};
+	struct mlx5_flow_destination dest = {};
+	struct mlx5e_macsec_tables *tx_tables;
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5e_flow_table *ft_crypto;
+	struct mlx5_flow_table *flow_table;
+	struct mlx5_flow_group *flow_group;
+	struct mlx5_flow_namespace *ns;
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	u32 *flow_group_in;
+	int err = 0;
+
+	ns = mlx5_get_flow_namespace(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_EGRESS_MACSEC);
+	if (!ns)
+		return -ENOMEM;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return -ENOMEM;
+
+	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
+	if (!flow_group_in)
+		goto out_spec;
+
+	tx_tables = &tx_fs->tables;
+	ft_crypto = &tx_tables->ft_crypto;
+
+	/* Tx crypto table  */
+	ft_attr.flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT;
+	ft_attr.level = TX_CRYPTO_TABLE_LEVEL;
+	ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE;
+
+	flow_table = mlx5_create_flow_table(ns, &ft_attr);
+	if (IS_ERR(flow_table)) {
+		err = PTR_ERR(flow_table);
+		netdev_err(netdev, "Failed to create MACsec Tx crypto table err(%d)\n", err);
+		goto out_flow_group;
+	}
+	ft_crypto->t = flow_table;
+
+	/* Tx crypto table groups */
+	err = macsec_fs_tx_create_crypto_table_groups(ft_crypto);
+	if (err) {
+		netdev_err(netdev,
+			   "Failed to create default flow group for MACsec Tx crypto table err(%d)\n",
+			   err);
+		goto err;
+	}
+
+	/* Tx crypto table MKE rule - MKE packets shouldn't be offloaded */
+	memset(&flow_act, 0, sizeof(flow_act));
+	memset(spec, 0, sizeof(*spec));
+	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
+	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_PAE);
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW;
+
+	rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, NULL, 0);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		netdev_err(netdev, "Failed to add MACsec TX MKE rule, err=%d\n", err);
+		goto err;
+	}
+	tx_fs->crypto_mke_rule = rule;
+
+	/* Tx crypto table Default miss rule */
+	memset(&flow_act, 0, sizeof(flow_act));
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW;
+	rule = mlx5_add_flow_rules(ft_crypto->t, NULL, &flow_act, NULL, 0);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		netdev_err(netdev, "Failed to add MACsec Tx table default miss rule %d\n", err);
+		goto err;
+	}
+	tx_tables->crypto_miss_rule = rule;
+
+	/* Tx check table */
+	flow_table = macsec_fs_auto_group_table_create(ns, 0, TX_CHECK_TABLE_LEVEL,
+						       TX_CHECK_TABLE_NUM_FTE);
+	if (IS_ERR(flow_table)) {
+		err = PTR_ERR(flow_table);
+		netdev_err(netdev, "fail to create MACsec TX check table, err(%d)\n", err);
+		goto err;
+	}
+	tx_tables->ft_check = flow_table;
+
+	/* Tx check table Default miss group/rule */
+	memset(flow_group_in, 0, inlen);
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1);
+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1);
+	flow_group = mlx5_create_flow_group(tx_tables->ft_check, flow_group_in);
+	if (IS_ERR(flow_group)) {
+		err = PTR_ERR(flow_group);
+		netdev_err(netdev,
+			   "Failed to create default flow group for MACsec Tx crypto table err(%d)\n",
+			   err);
+		goto err;
+	}
+	tx_tables->ft_check_group = flow_group;
+
+	/* Tx check table default drop rule */
+	memset(&dest, 0, sizeof(struct mlx5_flow_destination));
+	memset(&flow_act, 0, sizeof(flow_act));
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+	dest.counter_id = mlx5_fc_id(tx_tables->check_miss_rule_counter);
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;
+	rule = mlx5_add_flow_rules(tx_tables->ft_check,  NULL, &flow_act, &dest, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		netdev_err(netdev, "Failed to added MACsec tx check drop rule, err(%d)\n", err);
+		goto err;
+	}
+	tx_tables->check_miss_rule = rule;
+
+	/* Tx check table rule */
+	memset(spec, 0, sizeof(struct mlx5_flow_spec));
+	memset(&dest, 0, sizeof(struct mlx5_flow_destination));
+	memset(&flow_act, 0, sizeof(flow_act));
+
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4);
+	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0);
+	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
+
+	flow_act.flags = FLOW_ACT_NO_APPEND;
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | MLX5_FLOW_CONTEXT_ACTION_COUNT;
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+	dest.counter_id = mlx5_fc_id(tx_tables->check_rule_counter);
+	rule = mlx5_add_flow_rules(tx_tables->ft_check, spec, &flow_act, &dest, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		netdev_err(netdev, "Failed to add MACsec check rule, err=%d\n", err);
+		goto err;
+	}
+	tx_fs->check_rule = rule;
+
+	goto out_flow_group;
+
+err:
+	macsec_fs_tx_destroy(macsec_fs);
+out_flow_group:
+	kvfree(flow_group_in);
+out_spec:
+	kvfree(spec);
+	return err;
+}
+
+static int macsec_fs_tx_ft_get(struct mlx5e_macsec_fs *macsec_fs)
+{
+	struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
+	struct mlx5e_macsec_tables *tx_tables;
+	int err = 0;
+
+	tx_tables = &tx_fs->tables;
+	if (tx_tables->refcnt)
+		goto out;
+
+	err = macsec_fs_tx_create(macsec_fs);
+	if (err)
+		return err;
+
+out:
+	tx_tables->refcnt++;
+	return err;
+}
+
+static void macsec_fs_tx_ft_put(struct mlx5e_macsec_fs *macsec_fs)
+{
+	struct mlx5e_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables;
+
+	if (--tx_tables->refcnt)
+		return;
+
+	macsec_fs_tx_destroy(macsec_fs);
+}
+
+static int macsec_fs_tx_setup_fte(struct mlx5e_macsec_fs *macsec_fs,
+				  struct mlx5_flow_spec *spec,
+				  struct mlx5_flow_act *flow_act,
+				  u32 macsec_obj_id,
+				  u32 *fs_id)
+{
+	struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
+	int err = 0;
+	u32 id;
+
+	err = ida_alloc_range(&tx_fs->tx_halloc, 1,
+			      MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES,
+			      GFP_KERNEL);
+	if (err < 0)
+		return err;
+
+	id = err;
+	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
+
+	/* Metadata match */
+	MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_a,
+		 MLX5_ETH_WQE_FT_META_MACSEC_MASK);
+	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_a,
+		 MLX5_ETH_WQE_FT_META_MACSEC | id << 2);
+
+	*fs_id = id;
+	flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC;
+	flow_act->crypto.obj_id = macsec_obj_id;
+
+	mlx5_core_dbg(macsec_fs->mdev, "Tx fte: macsec obj_id %u, fs_id %u\n", macsec_obj_id, id);
+	return 0;
+}
+
+static void macsec_fs_tx_create_sectag_header(const struct macsec_context *ctx,
+					      char *reformatbf,
+					      size_t *reformat_size)
+{
+	const struct macsec_secy *secy = ctx->secy;
+	bool sci_present = macsec_send_sci(secy);
+	struct mlx5_sectag_header sectag = {};
+	const struct macsec_tx_sc *tx_sc;
+
+	tx_sc = &secy->tx_sc;
+	sectag.ethertype = htons(ETH_P_MACSEC);
+
+	if (sci_present) {
+		sectag.tci_an |= MACSEC_TCI_SC;
+		memcpy(&sectag.sci, &secy->sci,
+		       sizeof(sectag.sci));
+	} else {
+		if (tx_sc->end_station)
+			sectag.tci_an |= MACSEC_TCI_ES;
+		if (tx_sc->scb)
+			sectag.tci_an |= MACSEC_TCI_SCB;
+	}
+
+	/* With GCM, C/E clear for !encrypt, both set for encrypt */
+	if (tx_sc->encrypt)
+		sectag.tci_an |= MACSEC_TCI_CONFID;
+	else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN)
+		sectag.tci_an |= MACSEC_TCI_C;
+
+	sectag.tci_an |= tx_sc->encoding_sa;
+
+	*reformat_size = MLX5_MACSEC_TAG_LEN + (sci_present ? MACSEC_SCI_LEN : 0);
+
+	memcpy(reformatbf, &sectag, *reformat_size);
+}
+
+static void macsec_fs_tx_del_rule(struct mlx5e_macsec_fs *macsec_fs,
+				  struct mlx5e_macsec_tx_rule *tx_rule)
+{
+	if (tx_rule->rule) {
+		mlx5_del_flow_rules(tx_rule->rule);
+		tx_rule->rule = NULL;
+	}
+
+	if (tx_rule->pkt_reformat) {
+		mlx5_packet_reformat_dealloc(macsec_fs->mdev, tx_rule->pkt_reformat);
+		tx_rule->pkt_reformat = NULL;
+	}
+
+	if (tx_rule->fs_id) {
+		ida_free(&macsec_fs->tx_fs->tx_halloc, tx_rule->fs_id);
+		tx_rule->fs_id = 0;
+	}
+
+	kfree(tx_rule);
+
+	macsec_fs_tx_ft_put(macsec_fs);
+}
+
+static union mlx5e_macsec_rule *
+macsec_fs_tx_add_rule(struct mlx5e_macsec_fs *macsec_fs,
+		      const struct macsec_context *macsec_ctx,
+		      struct mlx5_macsec_rule_attrs *attrs,
+		      u32 *sa_fs_id)
+{
+	char reformatbf[MLX5_MACSEC_TAG_LEN + MACSEC_SCI_LEN];
+	struct mlx5_pkt_reformat_params reformat_params = {};
+	struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
+	struct net_device *netdev = macsec_fs->netdev;
+	union mlx5e_macsec_rule *macsec_rule = NULL;
+	struct mlx5_flow_destination dest = {};
+	struct mlx5e_macsec_tables *tx_tables;
+	struct mlx5e_macsec_tx_rule *tx_rule;
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	size_t reformat_size;
+	int err = 0;
+	u32 fs_id;
+
+	tx_tables = &tx_fs->tables;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return NULL;
+
+	err = macsec_fs_tx_ft_get(macsec_fs);
+	if (err)
+		goto out_spec;
+
+	macsec_rule = kzalloc(sizeof(*macsec_rule), GFP_KERNEL);
+	if (!macsec_rule) {
+		macsec_fs_tx_ft_put(macsec_fs);
+		goto out_spec;
+	}
+
+	tx_rule = &macsec_rule->tx_rule;
+
+	/* Tx crypto table crypto rule */
+	macsec_fs_tx_create_sectag_header(macsec_ctx, reformatbf, &reformat_size);
+
+	reformat_params.type = MLX5_REFORMAT_TYPE_ADD_MACSEC;
+	reformat_params.size = reformat_size;
+	reformat_params.data = reformatbf;
+	flow_act.pkt_reformat = mlx5_packet_reformat_alloc(macsec_fs->mdev,
+							   &reformat_params,
+							   MLX5_FLOW_NAMESPACE_EGRESS_MACSEC);
+	if (IS_ERR(flow_act.pkt_reformat)) {
+		err = PTR_ERR(flow_act.pkt_reformat);
+		netdev_err(netdev, "Failed to allocate MACsec Tx reformat context err=%d\n",  err);
+		goto err;
+	}
+	tx_rule->pkt_reformat = flow_act.pkt_reformat;
+
+	err = macsec_fs_tx_setup_fte(macsec_fs, spec, &flow_act, attrs->macsec_obj_id, &fs_id);
+	if (err) {
+		netdev_err(netdev,
+			   "Failed to add packet reformat for MACsec TX crypto rule, err=%d\n",
+			   err);
+		goto err;
+	}
+
+	tx_rule->fs_id = fs_id;
+	*sa_fs_id = fs_id;
+
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
+			  MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT |
+			  MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest.ft = tx_tables->ft_check;
+	rule = mlx5_add_flow_rules(tx_tables->ft_crypto.t, spec, &flow_act, &dest, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		netdev_err(netdev, "Failed to add MACsec TX crypto rule, err=%d\n", err);
+		goto err;
+	}
+	tx_rule->rule = rule;
+
+	goto out_spec;
+
+err:
+	macsec_fs_tx_del_rule(macsec_fs, tx_rule);
+	macsec_rule = NULL;
+out_spec:
+	kvfree(spec);
+
+	return macsec_rule;
+}
+
+static void macsec_fs_tx_cleanup(struct mlx5e_macsec_fs *macsec_fs)
+{
+	struct mlx5e_macsec_tx *tx_fs = macsec_fs->tx_fs;
+	struct mlx5_core_dev *mdev = macsec_fs->mdev;
+	struct mlx5e_macsec_tables *tx_tables;
+
+	if (!tx_fs)
+		return;
+
+	tx_tables = &tx_fs->tables;
+	if (tx_tables->refcnt) {
+		netdev_err(macsec_fs->netdev,
+			   "Can't destroy MACsec offload tx_fs, refcnt(%u) isn't 0\n",
+			   tx_tables->refcnt);
+		return;
+	}
+
+	ida_destroy(&tx_fs->tx_halloc);
+
+	if (tx_tables->check_miss_rule_counter) {
+		mlx5_fc_destroy(mdev, tx_tables->check_miss_rule_counter);
+		tx_tables->check_miss_rule_counter = NULL;
+	}
+
+	if (tx_tables->check_rule_counter) {
+		mlx5_fc_destroy(mdev, tx_tables->check_rule_counter);
+		tx_tables->check_rule_counter = NULL;
+	}
+
+	kfree(tx_fs);
+	macsec_fs->tx_fs = NULL;
+}
+
+static int macsec_fs_tx_init(struct mlx5e_macsec_fs *macsec_fs)
+{
+	struct net_device *netdev = macsec_fs->netdev;
+	struct mlx5_core_dev *mdev = macsec_fs->mdev;
+	struct mlx5e_macsec_tables *tx_tables;
+	struct mlx5e_macsec_tx *tx_fs;
+	struct mlx5_fc *flow_counter;
+	int err;
+
+	tx_fs = kzalloc(sizeof(*tx_fs), GFP_KERNEL);
+	if (!tx_fs)
+		return -ENOMEM;
+
+	tx_tables = &tx_fs->tables;
+
+	flow_counter = mlx5_fc_create(mdev, false);
+	if (IS_ERR(flow_counter)) {
+		err = PTR_ERR(flow_counter);
+		netdev_err(netdev,
+			   "Failed to create MACsec Tx encrypt flow counter, err(%d)\n",
+			   err);
+		goto err_encrypt_counter;
+	}
+	tx_tables->check_rule_counter = flow_counter;
+
+	flow_counter = mlx5_fc_create(mdev, false);
+	if (IS_ERR(flow_counter)) {
+		err = PTR_ERR(flow_counter);
+		netdev_err(netdev,
+			   "Failed to create MACsec Tx drop flow counter, err(%d)\n",
+			   err);
+		goto err_drop_counter;
+	}
+	tx_tables->check_miss_rule_counter = flow_counter;
+
+	ida_init(&tx_fs->tx_halloc);
+
+	macsec_fs->tx_fs = tx_fs;
+
+	return 0;
+
+err_drop_counter:
+	mlx5_fc_destroy(mdev, tx_tables->check_rule_counter);
+	tx_tables->check_rule_counter = NULL;
+
+err_encrypt_counter:
+	kfree(tx_fs);
+	macsec_fs->tx_fs = NULL;
+
+	return err;
+}
+
+static void macsec_fs_rx_destroy(struct mlx5e_macsec_fs *macsec_fs)
+{
+	struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs;
+	struct mlx5e_macsec_tables *rx_tables;
+	int i;
+
+	/* Rx check table */
+	for (i = 1; i >= 0; --i) {
+		if (rx_fs->check_rule[i]) {
+			mlx5_del_flow_rules(rx_fs->check_rule[i]);
+			rx_fs->check_rule[i] = NULL;
+		}
+
+		if (rx_fs->check_rule_pkt_reformat[i]) {
+			mlx5_packet_reformat_dealloc(macsec_fs->mdev,
+						     rx_fs->check_rule_pkt_reformat[i]);
+			rx_fs->check_rule_pkt_reformat[i] = NULL;
+		}
+	}
+
+	rx_tables = &rx_fs->tables;
+
+	if (rx_tables->check_miss_rule) {
+		mlx5_del_flow_rules(rx_tables->check_miss_rule);
+		rx_tables->check_miss_rule = NULL;
+	}
+
+	if (rx_tables->ft_check_group) {
+		mlx5_destroy_flow_group(rx_tables->ft_check_group);
+		rx_tables->ft_check_group = NULL;
+	}
+
+	if (rx_tables->ft_check) {
+		mlx5_destroy_flow_table(rx_tables->ft_check);
+		rx_tables->ft_check = NULL;
+	}
+
+	/* Rx crypto table */
+	if (rx_tables->crypto_miss_rule) {
+		mlx5_del_flow_rules(rx_tables->crypto_miss_rule);
+		rx_tables->crypto_miss_rule = NULL;
+	}
+
+	mlx5e_destroy_flow_table(&rx_tables->ft_crypto);
+}
+
+static int macsec_fs_rx_create_crypto_table_groups(struct mlx5e_flow_table *ft)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	int mclen = MLX5_ST_SZ_BYTES(fte_match_param);
+	int ix = 0;
+	u32 *in;
+	int err;
+	u8 *mc;
+
+	ft->g = kcalloc(RX_CRYPTO_TABLE_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
+	if (!ft->g)
+		return -ENOMEM;
+
+	in = kvzalloc(inlen, GFP_KERNEL);
+	if (!in) {
+		kfree(ft->g);
+		return -ENOMEM;
+	}
+
+	mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
+
+	/* Flow group for SA rule with SCI */
+	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS |
+						MLX5_MATCH_MISC_PARAMETERS_5);
+	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
+
+	MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0,
+		 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK <<
+		 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
+	MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_2);
+	MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_3);
+
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
+	if (IS_ERR(ft->g[ft->num_groups]))
+		goto err;
+	ft->num_groups++;
+
+	/* Flow group for SA rule without SCI */
+	memset(in, 0, inlen);
+	memset(mc, 0, mclen);
+	MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS |
+						MLX5_MATCH_MISC_PARAMETERS_5);
+	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_47_16);
+	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_15_0);
+	MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype);
+
+	MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0,
+		 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
+
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
+	if (IS_ERR(ft->g[ft->num_groups]))
+		goto err;
+	ft->num_groups++;
+
+	/* Flow Group for l2 traps */
+	memset(in, 0, inlen);
+	memset(mc, 0, mclen);
+	MLX5_SET_CFG(in, start_flow_index, ix);
+	ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE;
+	MLX5_SET_CFG(in, end_flow_index, ix - 1);
+	ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
+	if (IS_ERR(ft->g[ft->num_groups]))
+		goto err;
+	ft->num_groups++;
+
+	kvfree(in);
+	return 0;
+
+err:
+	err = PTR_ERR(ft->g[ft->num_groups]);
+	ft->g[ft->num_groups] = NULL;
+	kvfree(in);
+
+	return err;
+}
+
+static int macsec_fs_rx_create_check_decap_rule(struct mlx5e_macsec_fs *macsec_fs,
+						struct mlx5_flow_destination *dest,
+						struct mlx5_flow_act *flow_act,
+						struct mlx5_flow_spec *spec,
+						int reformat_param_size)
+{
+	int rule_index = (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI) ? 0 : 1;
+	u8 mlx5_reformat_buf[MLX5_SECTAG_HEADER_SIZE_WITH_SCI];
+	struct mlx5_pkt_reformat_params reformat_params = {};
+	struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs;
+	struct net_device *netdev = macsec_fs->netdev;
+	struct mlx5e_macsec_tables *rx_tables;
+	struct mlx5_flow_handle *rule;
+	int err = 0;
+
+	rx_tables = &rx_fs->tables;
+
+	/* Rx check table decap 16B rule */
+	memset(dest, 0, sizeof(*dest));
+	memset(flow_act, 0, sizeof(*flow_act));
+	memset(spec, 0, sizeof(*spec));
+
+	reformat_params.type = MLX5_REFORMAT_TYPE_DEL_MACSEC;
+	reformat_params.size = reformat_param_size;
+	reformat_params.data = mlx5_reformat_buf;
+	flow_act->pkt_reformat = mlx5_packet_reformat_alloc(macsec_fs->mdev,
+							    &reformat_params,
+							    MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC);
+	if (IS_ERR(flow_act->pkt_reformat)) {
+		err = PTR_ERR(flow_act->pkt_reformat);
+		netdev_err(netdev, "Failed to allocate MACsec Rx reformat context err=%d\n", err);
+		return err;
+	}
+	rx_fs->check_rule_pkt_reformat[rule_index] = flow_act->pkt_reformat;
+
+	spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
+	/* MACsec syndrome match */
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.macsec_syndrome);
+	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.macsec_syndrome, 0);
+	/* ASO return reg syndrome match */
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4);
+	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0);
+
+	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5;
+	/* Sectag TCI SC present bit*/
+	MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0,
+		 MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
+
+	if (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI)
+		MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0,
+			 MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT <<
+			 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
+
+	flow_act->flags = FLOW_ACT_NO_APPEND;
+	flow_act->action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO |
+			   MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT |
+			   MLX5_FLOW_CONTEXT_ACTION_COUNT;
+	dest->type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+	dest->counter_id = mlx5_fc_id(rx_tables->check_rule_counter);
+	rule = mlx5_add_flow_rules(rx_tables->ft_check, spec, flow_act, dest, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		netdev_err(netdev, "Failed to add MACsec Rx check rule, err=%d\n", err);
+		return err;
+	}
+
+	rx_fs->check_rule[rule_index] = rule;
+
+	return 0;
+}
+
+static int macsec_fs_rx_create(struct mlx5e_macsec_fs *macsec_fs)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs;
+	struct net_device *netdev = macsec_fs->netdev;
+	struct mlx5_flow_table_attr ft_attr = {};
+	struct mlx5_flow_destination dest = {};
+	struct mlx5e_macsec_tables *rx_tables;
+	struct mlx5e_flow_table *ft_crypto;
+	struct mlx5_flow_table *flow_table;
+	struct mlx5_flow_group *flow_group;
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_namespace *ns;
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	u32 *flow_group_in;
+	int err = 0;
+
+	ns = mlx5_get_flow_namespace(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC);
+	if (!ns)
+		return -ENOMEM;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return -ENOMEM;
+
+	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
+	if (!flow_group_in)
+		goto free_spec;
+
+	rx_tables = &rx_fs->tables;
+	ft_crypto = &rx_tables->ft_crypto;
+
+	/* Rx crypto table */
+	ft_attr.level = RX_CRYPTO_TABLE_LEVEL;
+	ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE;
+
+	flow_table = mlx5_create_flow_table(ns, &ft_attr);
+	if (IS_ERR(flow_table)) {
+		err = PTR_ERR(flow_table);
+		netdev_err(netdev, "Failed to create MACsec Rx crypto table err(%d)\n", err);
+		goto out_flow_group;
+	}
+	ft_crypto->t = flow_table;
+
+	/* Rx crypto table groups */
+	err = macsec_fs_rx_create_crypto_table_groups(ft_crypto);
+	if (err) {
+		netdev_err(netdev,
+			   "Failed to create default flow group for MACsec Tx crypto table err(%d)\n",
+			   err);
+		goto err;
+	}
+
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO;
+	rule = mlx5_add_flow_rules(ft_crypto->t, NULL, &flow_act, NULL, 0);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		netdev_err(netdev,
+			   "Failed to add MACsec Rx crypto table default miss rule %d\n",
+			   err);
+		goto err;
+	}
+	rx_tables->crypto_miss_rule = rule;
+
+	/* Rx check table */
+	flow_table = macsec_fs_auto_group_table_create(ns,
+						       MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT,
+						       RX_CHECK_TABLE_LEVEL,
+						       RX_CHECK_TABLE_NUM_FTE);
+	if (IS_ERR(flow_table)) {
+		err = PTR_ERR(flow_table);
+		netdev_err(netdev, "fail to create MACsec RX check table, err(%d)\n", err);
+		goto err;
+	}
+	rx_tables->ft_check = flow_table;
+
+	/* Rx check table Default miss group/rule */
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1);
+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1);
+	flow_group = mlx5_create_flow_group(rx_tables->ft_check, flow_group_in);
+	if (IS_ERR(flow_group)) {
+		err = PTR_ERR(flow_group);
+		netdev_err(netdev,
+			   "Failed to create default flow group for MACsec Rx check table err(%d)\n",
+			   err);
+		goto err;
+	}
+	rx_tables->ft_check_group = flow_group;
+
+	/* Rx check table default drop rule */
+	memset(&flow_act, 0, sizeof(flow_act));
+
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
+	dest.counter_id = mlx5_fc_id(rx_tables->check_miss_rule_counter);
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT;
+	rule = mlx5_add_flow_rules(rx_tables->ft_check,  NULL, &flow_act, &dest, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		netdev_err(netdev, "Failed to added MACsec Rx check drop rule, err(%d)\n", err);
+		goto err;
+	}
+	rx_tables->check_miss_rule = rule;
+
+	/* Rx check table decap rules */
+	err = macsec_fs_rx_create_check_decap_rule(macsec_fs, &dest, &flow_act, spec,
+						   MLX5_SECTAG_HEADER_SIZE_WITH_SCI);
+	if (err)
+		goto err;
+
+	err = macsec_fs_rx_create_check_decap_rule(macsec_fs, &dest, &flow_act, spec,
+						   MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI);
+	if (err)
+		goto err;
+
+	goto out_flow_group;
+
+err:
+	macsec_fs_rx_destroy(macsec_fs);
+out_flow_group:
+	kvfree(flow_group_in);
+free_spec:
+	kvfree(spec);
+	return err;
+}
+
+static int macsec_fs_rx_ft_get(struct mlx5e_macsec_fs *macsec_fs)
+{
+	struct mlx5e_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables;
+	int err = 0;
+
+	if (rx_tables->refcnt)
+		goto out;
+
+	err = macsec_fs_rx_create(macsec_fs);
+	if (err)
+		return err;
+
+out:
+	rx_tables->refcnt++;
+	return err;
+}
+
+static void macsec_fs_rx_ft_put(struct mlx5e_macsec_fs *macsec_fs)
+{
+	struct mlx5e_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables;
+
+	if (--rx_tables->refcnt)
+		return;
+
+	macsec_fs_rx_destroy(macsec_fs);
+}
+
+static void macsec_fs_rx_del_rule(struct mlx5e_macsec_fs *macsec_fs,
+				  struct mlx5e_macsec_rx_rule *rx_rule)
+{
+	int i;
+
+	for (i = 0; i < RX_NUM_OF_RULES_PER_SA; ++i) {
+		if (rx_rule->rule[i]) {
+			mlx5_del_flow_rules(rx_rule->rule[i]);
+			rx_rule->rule[i] = NULL;
+		}
+	}
+
+	if (rx_rule->meta_modhdr) {
+		mlx5_modify_header_dealloc(macsec_fs->mdev, rx_rule->meta_modhdr);
+		rx_rule->meta_modhdr = NULL;
+	}
+
+	kfree(rx_rule);
+
+	macsec_fs_rx_ft_put(macsec_fs);
+}
+
+static void macsec_fs_rx_setup_fte(struct mlx5_flow_spec *spec,
+				   struct mlx5_flow_act *flow_act,
+				   struct mlx5_macsec_rule_attrs *attrs,
+				   bool sci_present)
+{
+	u8 tci_an = (sci_present << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET) | attrs->assoc_num;
+	struct mlx5_flow_act_crypto_params *crypto_params = &flow_act->crypto;
+	__be32 *sci_p = (__be32 *)(&attrs->sci);
+
+	spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+
+	/* MACsec ethertype */
+	MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
+	MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_MACSEC);
+
+	spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5;
+
+	/* Sectag AN + TCI SC present bit*/
+	MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0,
+		 MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
+	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0,
+		 tci_an << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET);
+
+	if (sci_present) {
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+				 misc_parameters_5.macsec_tag_2);
+		MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_2,
+			 be32_to_cpu(sci_p[0]));
+
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+				 misc_parameters_5.macsec_tag_3);
+		MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_3,
+			 be32_to_cpu(sci_p[1]));
+	} else {
+		/* When SCI isn't present in the Sectag, need to match the source */
+		/* MAC address only if the SCI contains the default MACsec PORT	  */
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_47_16);
+		MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_15_0);
+		memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.smac_47_16),
+		       sci_p, ETH_ALEN);
+	}
+
+	crypto_params->type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC;
+	crypto_params->obj_id = attrs->macsec_obj_id;
+}
+
+static union mlx5e_macsec_rule *
+macsec_fs_rx_add_rule(struct mlx5e_macsec_fs *macsec_fs,
+		      const struct macsec_context *macsec_ctx,
+		      struct mlx5_macsec_rule_attrs *attrs,
+		      u32 fs_id)
+{
+	u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
+	struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs;
+	struct net_device *netdev = macsec_fs->netdev;
+	union mlx5e_macsec_rule *macsec_rule = NULL;
+	struct mlx5_modify_hdr *modify_hdr = NULL;
+	struct mlx5_flow_destination dest = {};
+	struct mlx5e_macsec_tables *rx_tables;
+	struct mlx5e_macsec_rx_rule *rx_rule;
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5e_flow_table *ft_crypto;
+	struct mlx5_flow_handle *rule;
+	struct mlx5_flow_spec *spec;
+	int err = 0;
+
+	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+	if (!spec)
+		return NULL;
+
+	err = macsec_fs_rx_ft_get(macsec_fs);
+	if (err)
+		goto out_spec;
+
+	macsec_rule = kzalloc(sizeof(*macsec_rule), GFP_KERNEL);
+	if (!macsec_rule) {
+		macsec_fs_rx_ft_put(macsec_fs);
+		goto out_spec;
+	}
+
+	rx_rule = &macsec_rule->rx_rule;
+	rx_tables = &rx_fs->tables;
+	ft_crypto = &rx_tables->ft_crypto;
+
+	/* Set bit[31 - 30] macsec marker - 0x01 */
+	/* Set bit[3-0] fs id */
+	MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET);
+	MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B);
+	MLX5_SET(set_action_in, action, data, fs_id | BIT(30));
+	MLX5_SET(set_action_in, action, offset, 0);
+	MLX5_SET(set_action_in, action, length, 32);
+
+	modify_hdr = mlx5_modify_header_alloc(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC,
+					      1, action);
+	if (IS_ERR(modify_hdr)) {
+		err = PTR_ERR(modify_hdr);
+		netdev_err(netdev, "fail to alloc MACsec set modify_header_id err=%d\n", err);
+		modify_hdr = NULL;
+		goto err;
+	}
+	rx_rule->meta_modhdr = modify_hdr;
+
+	/* Rx crypto table with SCI rule */
+	macsec_fs_rx_setup_fte(spec, &flow_act, attrs, true);
+
+	flow_act.modify_hdr = modify_hdr;
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
+			  MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
+			  MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+
+	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+	dest.ft = rx_tables->ft_check;
+	rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, &dest, 1);
+	if (IS_ERR(rule)) {
+		err = PTR_ERR(rule);
+		netdev_err(netdev,
+			   "Failed to add SA with SCI rule to Rx crypto rule, err=%d\n",
+			   err);
+		goto err;
+	}
+	rx_rule->rule[0] = rule;
+
+	/* Rx crypto table without SCI rule */
+	if (cpu_to_be64((__force u64)attrs->sci) & ntohs(MACSEC_PORT_ES)) {
+		memset(spec, 0, sizeof(struct mlx5_flow_spec));
+		memset(&dest, 0, sizeof(struct mlx5_flow_destination));
+		memset(&flow_act, 0, sizeof(flow_act));
+
+		macsec_fs_rx_setup_fte(spec, &flow_act, attrs, false);
+
+		flow_act.modify_hdr = modify_hdr;
+		flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
+				  MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
+				  MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
+
+		dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
+		dest.ft = rx_tables->ft_check;
+		rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, &dest, 1);
+		if (IS_ERR(rule)) {
+			err = PTR_ERR(rule);
+			netdev_err(netdev,
+				   "Failed to add SA without SCI rule to Rx crypto rule, err=%d\n",
+				   err);
+			goto err;
+		}
+		rx_rule->rule[1] = rule;
+	}
+
+	return macsec_rule;
+
+err:
+	macsec_fs_rx_del_rule(macsec_fs, rx_rule);
+	macsec_rule = NULL;
+out_spec:
+	kvfree(spec);
+	return macsec_rule;
+}
+
+static int macsec_fs_rx_init(struct mlx5e_macsec_fs *macsec_fs)
+{
+	struct net_device *netdev = macsec_fs->netdev;
+	struct mlx5_core_dev *mdev = macsec_fs->mdev;
+	struct mlx5e_macsec_tables *rx_tables;
+	struct mlx5e_macsec_rx *rx_fs;
+	struct mlx5_fc *flow_counter;
+	int err;
+
+	rx_fs =	kzalloc(sizeof(*rx_fs), GFP_KERNEL);
+	if (!rx_fs)
+		return -ENOMEM;
+
+	flow_counter = mlx5_fc_create(mdev, false);
+	if (IS_ERR(flow_counter)) {
+		err = PTR_ERR(flow_counter);
+		netdev_err(netdev,
+			   "Failed to create MACsec Rx encrypt flow counter, err(%d)\n",
+			   err);
+		goto err_encrypt_counter;
+	}
+
+	rx_tables = &rx_fs->tables;
+	rx_tables->check_rule_counter = flow_counter;
+
+	flow_counter = mlx5_fc_create(mdev, false);
+	if (IS_ERR(flow_counter)) {
+		err = PTR_ERR(flow_counter);
+		netdev_err(netdev,
+			   "Failed to create MACsec Rx drop flow counter, err(%d)\n",
+			   err);
+		goto err_drop_counter;
+	}
+	rx_tables->check_miss_rule_counter = flow_counter;
+
+	macsec_fs->rx_fs = rx_fs;
+
+	return 0;
+
+err_drop_counter:
+	mlx5_fc_destroy(mdev, rx_tables->check_rule_counter);
+	rx_tables->check_rule_counter = NULL;
+
+err_encrypt_counter:
+	kfree(rx_fs);
+	macsec_fs->rx_fs = NULL;
+
+	return err;
+}
+
+static void macsec_fs_rx_cleanup(struct mlx5e_macsec_fs *macsec_fs)
+{
+	struct mlx5e_macsec_rx *rx_fs = macsec_fs->rx_fs;
+	struct mlx5_core_dev *mdev = macsec_fs->mdev;
+	struct mlx5e_macsec_tables *rx_tables;
+
+	if (!rx_fs)
+		return;
+
+	rx_tables = &rx_fs->tables;
+
+	if (rx_tables->refcnt) {
+		netdev_err(macsec_fs->netdev,
+			   "Can't destroy MACsec offload rx_fs, refcnt(%u) isn't 0\n",
+			   rx_tables->refcnt);
+		return;
+	}
+
+	if (rx_tables->check_miss_rule_counter) {
+		mlx5_fc_destroy(mdev, rx_tables->check_miss_rule_counter);
+		rx_tables->check_miss_rule_counter = NULL;
+	}
+
+	if (rx_tables->check_rule_counter) {
+		mlx5_fc_destroy(mdev, rx_tables->check_rule_counter);
+		rx_tables->check_rule_counter = NULL;
+	}
+
+	kfree(rx_fs);
+	macsec_fs->rx_fs = NULL;
+}
+
+void mlx5e_macsec_fs_get_stats_fill(struct mlx5e_macsec_fs *macsec_fs, void *macsec_stats)
+{
+	struct mlx5e_macsec_stats *stats = (struct mlx5e_macsec_stats *)macsec_stats;
+	struct mlx5e_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables;
+	struct mlx5e_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables;
+	struct mlx5_core_dev *mdev = macsec_fs->mdev;
+
+	if (tx_tables->check_rule_counter)
+		mlx5_fc_query(mdev, tx_tables->check_rule_counter,
+			      &stats->macsec_tx_pkts, &stats->macsec_tx_bytes);
+
+	if (tx_tables->check_miss_rule_counter)
+		mlx5_fc_query(mdev, tx_tables->check_miss_rule_counter,
+			      &stats->macsec_tx_pkts_drop, &stats->macsec_tx_bytes_drop);
+
+	if (rx_tables->check_rule_counter)
+		mlx5_fc_query(mdev, rx_tables->check_rule_counter,
+			      &stats->macsec_rx_pkts, &stats->macsec_rx_bytes);
+
+	if (rx_tables->check_miss_rule_counter)
+		mlx5_fc_query(mdev, rx_tables->check_miss_rule_counter,
+			      &stats->macsec_rx_pkts_drop, &stats->macsec_rx_bytes_drop);
+}
+
+union mlx5e_macsec_rule *
+mlx5e_macsec_fs_add_rule(struct mlx5e_macsec_fs *macsec_fs,
+			 const struct macsec_context *macsec_ctx,
+			 struct mlx5_macsec_rule_attrs *attrs,
+			 u32 *sa_fs_id)
+{
+	return (attrs->action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ?
+		macsec_fs_tx_add_rule(macsec_fs, macsec_ctx, attrs, sa_fs_id) :
+		macsec_fs_rx_add_rule(macsec_fs, macsec_ctx, attrs, *sa_fs_id);
+}
+
+void mlx5e_macsec_fs_del_rule(struct mlx5e_macsec_fs *macsec_fs,
+			      union mlx5e_macsec_rule *macsec_rule,
+			      int action)
+{
+	(action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ?
+		macsec_fs_tx_del_rule(macsec_fs, &macsec_rule->tx_rule) :
+		macsec_fs_rx_del_rule(macsec_fs, &macsec_rule->rx_rule);
+}
+
+void mlx5e_macsec_fs_cleanup(struct mlx5e_macsec_fs *macsec_fs)
+{
+	macsec_fs_rx_cleanup(macsec_fs);
+	macsec_fs_tx_cleanup(macsec_fs);
+	kfree(macsec_fs);
+}
+
+struct mlx5e_macsec_fs *
+mlx5e_macsec_fs_init(struct mlx5_core_dev *mdev,
+		     struct net_device *netdev)
+{
+	struct mlx5e_macsec_fs *macsec_fs;
+	int err;
+
+	macsec_fs = kzalloc(sizeof(*macsec_fs), GFP_KERNEL);
+	if (!macsec_fs)
+		return NULL;
+
+	macsec_fs->mdev = mdev;
+	macsec_fs->netdev = netdev;
+
+	err = macsec_fs_tx_init(macsec_fs);
+	if (err) {
+		netdev_err(netdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err);
+		goto err;
+	}
+
+	err = macsec_fs_rx_init(macsec_fs);
+	if (err) {
+		netdev_err(netdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err);
+		goto tx_cleanup;
+	}
+
+	return macsec_fs;
+
+tx_cleanup:
+	macsec_fs_tx_cleanup(macsec_fs);
+err:
+	kfree(macsec_fs);
+	return NULL;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.h
new file mode 100644
index 000000000000..b429648d4ee7
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_fs.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#ifndef __MLX5_MACSEC_STEERING_H__
+#define __MLX5_MACSEC_STEERING_H__
+
+#ifdef CONFIG_MLX5_EN_MACSEC
+
+#include "en_accel/macsec.h"
+
+#define MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES 16
+
+struct mlx5e_macsec_fs;
+union mlx5e_macsec_rule;
+
+struct mlx5_macsec_rule_attrs {
+	sci_t sci;
+	u32 macsec_obj_id;
+	u8 assoc_num;
+	int action;
+};
+
+enum mlx5_macsec_action {
+	MLX5_ACCEL_MACSEC_ACTION_ENCRYPT,
+	MLX5_ACCEL_MACSEC_ACTION_DECRYPT,
+};
+
+void mlx5e_macsec_fs_cleanup(struct mlx5e_macsec_fs *macsec_fs);
+
+struct mlx5e_macsec_fs *
+mlx5e_macsec_fs_init(struct mlx5_core_dev *mdev, struct net_device *netdev);
+
+union mlx5e_macsec_rule *
+mlx5e_macsec_fs_add_rule(struct mlx5e_macsec_fs *macsec_fs,
+			 const struct macsec_context *ctx,
+			 struct mlx5_macsec_rule_attrs *attrs,
+			 u32 *sa_fs_id);
+
+void mlx5e_macsec_fs_del_rule(struct mlx5e_macsec_fs *macsec_fs,
+			      union mlx5e_macsec_rule *macsec_rule,
+			      int action);
+
+void mlx5e_macsec_fs_get_stats_fill(struct mlx5e_macsec_fs *macsec_fs, void *macsec_stats);
+
+#endif
+
+#endif /* __MLX5_MACSEC_STEERING_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_stats.c
new file mode 100644
index 000000000000..e50a2e3f3d18
--- /dev/null
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/macsec_stats.c
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
+
+#include <linux/ethtool.h>
+#include <net/sock.h>
+
+#include "en.h"
+#include "en_accel/macsec.h"
+
+static const struct counter_desc mlx5e_macsec_hw_stats_desc[] = {
+	{ MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_rx_pkts) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_rx_bytes) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_rx_pkts_drop) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_rx_bytes_drop) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_tx_pkts) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_tx_bytes) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_tx_pkts_drop) },
+	{ MLX5E_DECLARE_STAT(struct mlx5e_macsec_stats, macsec_tx_bytes_drop) },
+};
+
+#define NUM_MACSEC_HW_COUNTERS ARRAY_SIZE(mlx5e_macsec_hw_stats_desc)
+
+static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(macsec_hw)
+{
+	if (!priv->macsec)
+		return 0;
+
+	if (mlx5e_is_macsec_device(priv->mdev))
+		return NUM_MACSEC_HW_COUNTERS;
+
+	return 0;
+}
+
+static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(macsec_hw) {}
+
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(macsec_hw)
+{
+	unsigned int i;
+
+	if (!priv->macsec)
+		return idx;
+
+	if (!mlx5e_is_macsec_device(priv->mdev))
+		return idx;
+
+	for (i = 0; i < NUM_MACSEC_HW_COUNTERS; i++)
+		strcpy(data + (idx++) * ETH_GSTRING_LEN,
+		       mlx5e_macsec_hw_stats_desc[i].format);
+
+	return idx;
+}
+
+static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(macsec_hw)
+{
+	int i;
+
+	if (!priv->macsec)
+		return idx;
+
+	if (!mlx5e_is_macsec_device(priv->mdev))
+		return idx;
+
+	mlx5e_macsec_get_stats_fill(priv->macsec, mlx5e_macsec_get_stats(priv->macsec));
+	for (i = 0; i < NUM_MACSEC_HW_COUNTERS; i++)
+		data[idx++] = MLX5E_READ_CTR64_CPU(mlx5e_macsec_get_stats(priv->macsec),
+						   mlx5e_macsec_hw_stats_desc,
+						   i);
+
+	return idx;
+}
+
+MLX5E_DEFINE_STATS_GRP(macsec_hw, 0);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c
index cd7f245dcf14..0ae1865086ff 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c
@@ -114,47 +114,49 @@ static enum mlx5_traffic_types arfs_get_tt(enum arfs_type type)
 	}
 }
 
-static int arfs_disable(struct mlx5e_priv *priv)
+static int arfs_disable(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
 	int err, i;
 
 	for (i = 0; i < ARFS_NUM_TYPES; i++) {
 		/* Modify ttc rules destination back to their default */
-		err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, arfs_get_tt(i));
+		err = mlx5_ttc_fwd_default_dest(ttc, arfs_get_tt(i));
 		if (err) {
-			netdev_err(priv->netdev,
-				   "%s: modify ttc[%d] default destination failed, err(%d)\n",
-				   __func__, arfs_get_tt(i), err);
+			fs_err(fs,
+			       "%s: modify ttc[%d] default destination failed, err(%d)\n",
+			       __func__, arfs_get_tt(i), err);
 			return err;
 		}
 	}
 	return 0;
 }
 
-static void arfs_del_rules(struct mlx5e_priv *priv);
+static void arfs_del_rules(struct mlx5e_flow_steering *fs);
 
-int mlx5e_arfs_disable(struct mlx5e_priv *priv)
+int mlx5e_arfs_disable(struct mlx5e_flow_steering *fs)
 {
-	arfs_del_rules(priv);
+	arfs_del_rules(fs);
 
-	return arfs_disable(priv);
+	return arfs_disable(fs);
 }
 
-int mlx5e_arfs_enable(struct mlx5e_priv *priv)
+int mlx5e_arfs_enable(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs, false);
+	struct mlx5e_arfs_tables *arfs =  mlx5e_fs_get_arfs(fs);
 	struct mlx5_flow_destination dest = {};
 	int err, i;
 
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 	for (i = 0; i < ARFS_NUM_TYPES; i++) {
-		dest.ft = priv->fs->arfs->arfs_tables[i].ft.t;
+		dest.ft = arfs->arfs_tables[i].ft.t;
 		/* Modify ttc rules destination to point on the aRFS FTs */
-		err = mlx5_ttc_fwd_dest(priv->fs->ttc, arfs_get_tt(i), &dest);
+		err = mlx5_ttc_fwd_dest(ttc, arfs_get_tt(i), &dest);
 		if (err) {
-			netdev_err(priv->netdev,
-				   "%s: modify ttc[%d] dest to arfs, failed err(%d)\n",
-				   __func__, arfs_get_tt(i), err);
-			arfs_disable(priv);
+			fs_err(fs, "%s: modify ttc[%d] dest to arfs, failed err(%d)\n",
+			       __func__, arfs_get_tt(i), err);
+			arfs_disable(fs);
 			return err;
 		}
 	}
@@ -167,31 +169,37 @@ static void arfs_destroy_table(struct arfs_table *arfs_t)
 	mlx5e_destroy_flow_table(&arfs_t->ft);
 }
 
-static void _mlx5e_cleanup_tables(struct mlx5e_priv *priv)
+static void _mlx5e_cleanup_tables(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5e_arfs_tables *arfs =  mlx5e_fs_get_arfs(fs);
 	int i;
 
-	arfs_del_rules(priv);
-	destroy_workqueue(priv->fs->arfs->wq);
+	arfs_del_rules(fs);
+	destroy_workqueue(arfs->wq);
 	for (i = 0; i < ARFS_NUM_TYPES; i++) {
-		if (!IS_ERR_OR_NULL(priv->fs->arfs->arfs_tables[i].ft.t))
-			arfs_destroy_table(&priv->fs->arfs->arfs_tables[i]);
+		if (!IS_ERR_OR_NULL(arfs->arfs_tables[i].ft.t))
+			arfs_destroy_table(&arfs->arfs_tables[i]);
 	}
 }
 
-void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv)
+void mlx5e_arfs_destroy_tables(struct mlx5e_flow_steering *fs, bool ntuple)
 {
-	if (!(priv->netdev->hw_features & NETIF_F_NTUPLE))
+	struct mlx5e_arfs_tables *arfs =  mlx5e_fs_get_arfs(fs);
+
+	if (!ntuple)
 		return;
 
-	_mlx5e_cleanup_tables(priv);
-	kvfree(priv->fs->arfs);
+	_mlx5e_cleanup_tables(fs);
+	mlx5e_fs_set_arfs(fs, NULL);
+	kvfree(arfs);
 }
 
-static int arfs_add_default_rule(struct mlx5e_priv *priv,
+static int arfs_add_default_rule(struct mlx5e_flow_steering *fs,
+				 struct mlx5e_rx_res *rx_res,
 				 enum arfs_type type)
 {
-	struct arfs_table *arfs_t = &priv->fs->arfs->arfs_tables[type];
+	struct mlx5e_arfs_tables *arfs =  mlx5e_fs_get_arfs(fs);
+	struct arfs_table *arfs_t = &arfs->arfs_tables[type];
 	struct mlx5_flow_destination dest = {};
 	MLX5_DECLARE_FLOW_ACT(flow_act);
 	enum mlx5_traffic_types tt;
@@ -200,23 +208,21 @@ static int arfs_add_default_rule(struct mlx5e_priv *priv,
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 	tt = arfs_get_tt(type);
 	if (tt == -EINVAL) {
-		netdev_err(priv->netdev, "%s: bad arfs_type: %d\n",
-			   __func__, type);
+		fs_err(fs, "%s: bad arfs_type: %d\n", __func__, type);
 		return -EINVAL;
 	}
 
 	/* FIXME: Must use mlx5_ttc_get_default_dest(),
 	 * but can't since TTC default is not setup yet !
 	 */
-	dest.tir_num = mlx5e_rx_res_get_tirn_rss(priv->rx_res, tt);
+	dest.tir_num = mlx5e_rx_res_get_tirn_rss(rx_res, tt);
 	arfs_t->default_rule = mlx5_add_flow_rules(arfs_t->ft.t, NULL,
 						   &flow_act,
 						   &dest, 1);
 	if (IS_ERR(arfs_t->default_rule)) {
 		err = PTR_ERR(arfs_t->default_rule);
 		arfs_t->default_rule = NULL;
-		netdev_err(priv->netdev, "%s: add rule failed, arfs type=%d\n",
-			   __func__, type);
+		fs_err(fs, "%s: add rule failed, arfs type=%d\n", __func__, type);
 	}
 
 	return err;
@@ -318,10 +324,12 @@ out:
 	return err;
 }
 
-static int arfs_create_table(struct mlx5e_priv *priv,
+static int arfs_create_table(struct mlx5e_flow_steering *fs,
+			     struct mlx5e_rx_res *rx_res,
 			     enum arfs_type type)
 {
-	struct mlx5e_arfs_tables *arfs = priv->fs->arfs;
+	struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs, false);
+	struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs);
 	struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft;
 	struct mlx5_flow_table_attr ft_attr = {};
 	int err;
@@ -332,7 +340,7 @@ static int arfs_create_table(struct mlx5e_priv *priv,
 	ft_attr.level = MLX5E_ARFS_FT_LEVEL;
 	ft_attr.prio = MLX5E_NIC_PRIO;
 
-	ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr);
+	ft->t = mlx5_create_flow_table(ns, &ft_attr);
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
 		ft->t = NULL;
@@ -343,7 +351,7 @@ static int arfs_create_table(struct mlx5e_priv *priv,
 	if (err)
 		goto err;
 
-	err = arfs_add_default_rule(priv, type);
+	err = arfs_add_default_rule(fs, rx_res,  type);
 	if (err)
 		goto err;
 
@@ -353,35 +361,40 @@ err:
 	return err;
 }
 
-int mlx5e_arfs_create_tables(struct mlx5e_priv *priv)
+int mlx5e_arfs_create_tables(struct mlx5e_flow_steering *fs,
+			     struct mlx5e_rx_res *rx_res, bool ntuple)
 {
+	struct mlx5e_arfs_tables *arfs;
 	int err = -ENOMEM;
 	int i;
 
-	if (!(priv->netdev->hw_features & NETIF_F_NTUPLE))
+	if (!ntuple)
 		return 0;
 
-	priv->fs->arfs = kvzalloc(sizeof(*priv->fs->arfs), GFP_KERNEL);
-	if (!priv->fs->arfs)
+	arfs = kvzalloc(sizeof(*arfs), GFP_KERNEL);
+	if (!arfs)
 		return -ENOMEM;
 
-	spin_lock_init(&priv->fs->arfs->arfs_lock);
-	INIT_LIST_HEAD(&priv->fs->arfs->rules);
-	priv->fs->arfs->wq = create_singlethread_workqueue("mlx5e_arfs");
-	if (!priv->fs->arfs->wq)
+	spin_lock_init(&arfs->arfs_lock);
+	INIT_LIST_HEAD(&arfs->rules);
+	arfs->wq = create_singlethread_workqueue("mlx5e_arfs");
+	if (!arfs->wq)
 		goto err;
 
+	mlx5e_fs_set_arfs(fs, arfs);
+
 	for (i = 0; i < ARFS_NUM_TYPES; i++) {
-		err = arfs_create_table(priv, i);
+		err = arfs_create_table(fs, rx_res, i);
 		if (err)
 			goto err_des;
 	}
 	return 0;
 
 err_des:
-	_mlx5e_cleanup_tables(priv);
+	_mlx5e_cleanup_tables(fs);
 err:
-	kvfree(priv->fs->arfs);
+	mlx5e_fs_set_arfs(fs, NULL);
+	kvfree(arfs);
 	return err;
 }
 
@@ -389,6 +402,7 @@ err:
 
 static void arfs_may_expire_flow(struct mlx5e_priv *priv)
 {
+	struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs);
 	struct arfs_rule *arfs_rule;
 	struct hlist_node *htmp;
 	HLIST_HEAD(del_list);
@@ -396,8 +410,8 @@ static void arfs_may_expire_flow(struct mlx5e_priv *priv)
 	int i;
 	int j;
 
-	spin_lock_bh(&priv->fs->arfs->arfs_lock);
-	mlx5e_for_each_arfs_rule(arfs_rule, htmp, priv->fs->arfs->arfs_tables, i, j) {
+	spin_lock_bh(&arfs->arfs_lock);
+	mlx5e_for_each_arfs_rule(arfs_rule, htmp, arfs->arfs_tables, i, j) {
 		if (!work_pending(&arfs_rule->arfs_work) &&
 		    rps_may_expire_flow(priv->netdev,
 					arfs_rule->rxq, arfs_rule->flow_id,
@@ -408,7 +422,7 @@ static void arfs_may_expire_flow(struct mlx5e_priv *priv)
 				break;
 		}
 	}
-	spin_unlock_bh(&priv->fs->arfs->arfs_lock);
+	spin_unlock_bh(&arfs->arfs_lock);
 	hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) {
 		if (arfs_rule->rule)
 			mlx5_del_flow_rules(arfs_rule->rule);
@@ -417,20 +431,21 @@ static void arfs_may_expire_flow(struct mlx5e_priv *priv)
 	}
 }
 
-static void arfs_del_rules(struct mlx5e_priv *priv)
+static void arfs_del_rules(struct mlx5e_flow_steering *fs)
 {
+	struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(fs);
 	struct hlist_node *htmp;
 	struct arfs_rule *rule;
 	HLIST_HEAD(del_list);
 	int i;
 	int j;
 
-	spin_lock_bh(&priv->fs->arfs->arfs_lock);
-	mlx5e_for_each_arfs_rule(rule, htmp, priv->fs->arfs->arfs_tables, i, j) {
+	spin_lock_bh(&arfs->arfs_lock);
+	mlx5e_for_each_arfs_rule(rule, htmp, arfs->arfs_tables, i, j) {
 		hlist_del_init(&rule->hlist);
 		hlist_add_head(&rule->hlist, &del_list);
 	}
-	spin_unlock_bh(&priv->fs->arfs->arfs_lock);
+	spin_unlock_bh(&arfs->arfs_lock);
 
 	hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) {
 		cancel_work_sync(&rule->arfs_work);
@@ -474,7 +489,7 @@ static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs,
 static struct mlx5_flow_handle *arfs_add_rule(struct mlx5e_priv *priv,
 					      struct arfs_rule *arfs_rule)
 {
-	struct mlx5e_arfs_tables *arfs = priv->fs->arfs;
+	struct mlx5e_arfs_tables *arfs = mlx5e_fs_get_arfs(priv->fs);
 	struct arfs_tuple *tuple = &arfs_rule->tuple;
 	struct mlx5_flow_handle *rule = NULL;
 	struct mlx5_flow_destination dest = {};
@@ -588,13 +603,15 @@ static void arfs_handle_work(struct work_struct *work)
 						   struct arfs_rule,
 						   arfs_work);
 	struct mlx5e_priv *priv = arfs_rule->priv;
+	struct mlx5e_arfs_tables *arfs;
 	struct mlx5_flow_handle *rule;
 
+	arfs = mlx5e_fs_get_arfs(priv->fs);
 	mutex_lock(&priv->state_lock);
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
-		spin_lock_bh(&priv->fs->arfs->arfs_lock);
+		spin_lock_bh(&arfs->arfs_lock);
 		hlist_del(&arfs_rule->hlist);
-		spin_unlock_bh(&priv->fs->arfs->arfs_lock);
+		spin_unlock_bh(&arfs->arfs_lock);
 
 		mutex_unlock(&priv->state_lock);
 		kfree(arfs_rule);
@@ -620,6 +637,7 @@ static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv,
 					 const struct flow_keys *fk,
 					 u16 rxq, u32 flow_id)
 {
+	struct mlx5e_arfs_tables *arfs =  mlx5e_fs_get_arfs(priv->fs);
 	struct arfs_rule *rule;
 	struct arfs_tuple *tuple;
 
@@ -647,7 +665,7 @@ static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv,
 	tuple->dst_port = fk->ports.dst;
 
 	rule->flow_id = flow_id;
-	rule->filter_id = priv->fs->arfs->last_filter_id++ % RPS_NO_FILTER;
+	rule->filter_id = arfs->last_filter_id++ % RPS_NO_FILTER;
 
 	hlist_add_head(&rule->hlist,
 		       arfs_hash_bucket(arfs_t, tuple->src_port,
@@ -691,11 +709,12 @@ int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
 			u16 rxq_index, u32 flow_id)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
-	struct mlx5e_arfs_tables *arfs = priv->fs->arfs;
-	struct arfs_table *arfs_t;
+	struct mlx5e_arfs_tables *arfs;
 	struct arfs_rule *arfs_rule;
+	struct arfs_table *arfs_t;
 	struct flow_keys fk;
 
+	arfs =  mlx5e_fs_get_arfs(priv->fs);
 	if (!skb_flow_dissect_flow_keys(skb, &fk, 0))
 		return -EPROTONOSUPPORT;
 
@@ -725,7 +744,7 @@ int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
 			return -ENOMEM;
 		}
 	}
-	queue_work(priv->fs->arfs->wq, &arfs_rule->arfs_work);
+	queue_work(arfs->wq, &arfs_rule->arfs_work);
 	spin_unlock_bh(&arfs->arfs_lock);
 	return arfs_rule->filter_id;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
index c0f409c195bf..68f19324db93 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
@@ -46,8 +46,7 @@ void mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc)
 	MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_pci_enable && ro_write);
 }
 
-static int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
-			     u32 *mkey)
+int mlx5e_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, u32 *mkey)
 {
 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
 	void *mkc;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index b811207fe5ed..24aa25da482b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -38,18 +38,19 @@
 #include "en/xsk/pool.h"
 #include "en/ptp.h"
 #include "lib/clock.h"
+#include "en/fs_ethtool.h"
 
 void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
 			       struct ethtool_drvinfo *drvinfo)
 {
 	struct mlx5_core_dev *mdev = priv->mdev;
 
-	strlcpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver));
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
 		 "%d.%d.%04d (%.16s)",
 		 fw_rev_maj(mdev), fw_rev_min(mdev), fw_rev_sub(mdev),
 		 mdev->board_id);
-	strlcpy(drvinfo->bus_info, dev_name(mdev->device),
+	strscpy(drvinfo->bus_info, dev_name(mdev->device),
 		sizeof(drvinfo->bus_info));
 }
 
@@ -310,7 +311,15 @@ void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv,
 				 struct ethtool_ringparam *param,
 				 struct kernel_ethtool_ringparam *kernel_param)
 {
-	param->rx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE;
+	/* Limitation for regular RQ. XSK RQ may clamp the queue length in
+	 * mlx5e_mpwqe_get_log_rq_size.
+	 */
+	u8 max_log_mpwrq_pkts = mlx5e_mpwrq_max_log_rq_pkts(priv->mdev,
+							    PAGE_SHIFT,
+							    MLX5E_MPWRQ_UMR_MODE_ALIGNED);
+
+	param->rx_max_pending = 1 << min_t(u8, MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE,
+					   max_log_mpwrq_pkts);
 	param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE;
 	param->rx_pending     = 1 << priv->channels.params.log_rq_mtu_frames;
 	param->tx_pending     = 1 << priv->channels.params.log_sq_size;
@@ -494,14 +503,14 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
 
 	arfs_enabled = opened && (priv->netdev->features & NETIF_F_NTUPLE);
 	if (arfs_enabled)
-		mlx5e_arfs_disable(priv);
+		mlx5e_arfs_disable(priv->fs);
 
 	/* Switch to new channels, set new parameters and close old ones */
 	err = mlx5e_safe_switch_params(priv, &new_params,
 				       mlx5e_num_channels_changed_ctx, NULL, true);
 
 	if (arfs_enabled) {
-		int err2 = mlx5e_arfs_enable(priv);
+		int err2 = mlx5e_arfs_enable(priv->fs);
 
 		if (err2)
 			netdev_err(priv->netdev, "%s: mlx5e_arfs_enable failed: %d\n",
@@ -1996,10 +2005,14 @@ static int set_pflag_rx_striding_rq(struct net_device *netdev, bool enable)
 	struct mlx5e_params new_params;
 
 	if (enable) {
-		if (!mlx5e_check_fragmented_striding_rq_cap(mdev))
-			return -EOPNOTSUPP;
-		if (!mlx5e_striding_rq_possible(mdev, &priv->channels.params))
-			return -EINVAL;
+		/* Checking the regular RQ here; mlx5e_validate_xsk_param called
+		 * from mlx5e_open_xsk will check for each XSK queue, and
+		 * mlx5e_safe_switch_params will be reverted if any check fails.
+		 */
+		int err = mlx5e_mpwrq_validate_regular(mdev, &priv->channels.params);
+
+		if (err)
+			return err;
 	} else if (priv->channels.params.packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
 		netdev_warn(netdev, "Can't set legacy RQ with HW-GRO/LRO, disable them first\n");
 		return -EINVAL;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
index e0ce5a233d0b..1892ccb889b3 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c
@@ -36,10 +36,38 @@
 #include <linux/tcp.h>
 #include <linux/mlx5/fs.h>
 #include <linux/mlx5/mpfs.h>
-#include "en.h"
 #include "en_tc.h"
 #include "lib/mpfs.h"
 #include "en/ptp.h"
+#include "en/fs_ethtool.h"
+
+struct mlx5e_flow_steering {
+	struct work_struct		set_rx_mode_work;
+	bool				state_destroy;
+	bool				vlan_strip_disable;
+	struct mlx5_core_dev		*mdev;
+	struct net_device		*netdev;
+	struct mlx5_flow_namespace      *ns;
+	struct mlx5_flow_namespace      *egress_ns;
+#ifdef CONFIG_MLX5_EN_RXNFC
+	struct mlx5e_ethtool_steering   *ethtool;
+#endif
+	struct mlx5e_tc_table           *tc;
+	struct mlx5e_promisc_table      promisc;
+	struct mlx5e_vlan_table         *vlan;
+	struct mlx5e_l2_table           l2;
+	struct mlx5_ttc_table           *ttc;
+	struct mlx5_ttc_table           *inner_ttc;
+#ifdef CONFIG_MLX5_EN_ARFS
+	struct mlx5e_arfs_tables       *arfs;
+#endif
+#ifdef CONFIG_MLX5_EN_TLS
+	struct mlx5e_accel_fs_tcp      *accel_tcp;
+#endif
+	struct mlx5e_fs_udp            *udp;
+	struct mlx5e_fs_any            *any;
+	struct mlx5e_ptp_fs            *ptp_fs;
+};
 
 static int mlx5e_add_l2_flow_rule(struct mlx5e_flow_steering *fs,
 				  struct mlx5e_l2_rule *ai, int type);
@@ -148,9 +176,8 @@ static int mlx5e_vport_context_update_vlans(struct mlx5e_flow_steering *fs)
 	max_list_size = 1 << MLX5_CAP_GEN(fs->mdev, log_max_vlan_list);
 
 	if (list_size > max_list_size) {
-		mlx5_core_warn(fs->mdev,
-			       "netdev vlans list size (%d) > (%d) max vport list size, some vlans will be dropped\n",
-			       list_size, max_list_size);
+		fs_warn(fs, "netdev vlans list size (%d) > (%d) max vport list size, some vlans will be dropped\n",
+			list_size, max_list_size);
 		list_size = max_list_size;
 	}
 
@@ -167,8 +194,8 @@ static int mlx5e_vport_context_update_vlans(struct mlx5e_flow_steering *fs)
 
 	err = mlx5_modify_nic_vport_vlans(fs->mdev, vlans, list_size);
 	if (err)
-		mlx5_core_err(fs->mdev, "Failed to modify vport vlans list err(%d)\n",
-			      err);
+		fs_err(fs, "Failed to modify vport vlans list err(%d)\n",
+		       err);
 
 	kvfree(vlans);
 	return err;
@@ -249,7 +276,7 @@ static int __mlx5e_add_vlan_rule(struct mlx5e_flow_steering *fs,
 	if (IS_ERR(*rule_p)) {
 		err = PTR_ERR(*rule_p);
 		*rule_p = NULL;
-		mlx5_core_err(fs->mdev, "%s: add rule failed\n", __func__);
+		fs_err(fs, "%s: add rule failed\n", __func__);
 	}
 
 	return err;
@@ -351,78 +378,78 @@ mlx5e_add_trap_rule(struct mlx5_flow_table *ft, int trap_id, int tir_num)
 	return rule;
 }
 
-int mlx5e_add_vlan_trap(struct mlx5e_priv *priv, int trap_id, int tir_num)
+int mlx5e_add_vlan_trap(struct mlx5e_flow_steering *fs, int trap_id, int tir_num)
 {
-	struct mlx5_flow_table *ft = priv->fs->vlan->ft.t;
+	struct mlx5_flow_table *ft = fs->vlan->ft.t;
 	struct mlx5_flow_handle *rule;
 	int err;
 
 	rule = mlx5e_add_trap_rule(ft, trap_id, tir_num);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
-		priv->fs->vlan->trap_rule = NULL;
-		mlx5_core_err(priv->fs->mdev, "%s: add VLAN trap rule failed, err %d\n",
-			      __func__, err);
+		fs->vlan->trap_rule = NULL;
+		fs_err(fs, "%s: add VLAN trap rule failed, err %d\n",
+		       __func__, err);
 		return err;
 	}
-	priv->fs->vlan->trap_rule = rule;
+	fs->vlan->trap_rule = rule;
 	return 0;
 }
 
-void mlx5e_remove_vlan_trap(struct mlx5e_priv *priv)
+void mlx5e_remove_vlan_trap(struct mlx5e_flow_steering *fs)
 {
-	if (priv->fs->vlan->trap_rule) {
-		mlx5_del_flow_rules(priv->fs->vlan->trap_rule);
-		priv->fs->vlan->trap_rule = NULL;
+	if (fs->vlan->trap_rule) {
+		mlx5_del_flow_rules(fs->vlan->trap_rule);
+		fs->vlan->trap_rule = NULL;
 	}
 }
 
-int mlx5e_add_mac_trap(struct mlx5e_priv *priv, int trap_id, int tir_num)
+int mlx5e_add_mac_trap(struct mlx5e_flow_steering *fs, int trap_id, int tir_num)
 {
-	struct mlx5_flow_table *ft = priv->fs->l2.ft.t;
+	struct mlx5_flow_table *ft = fs->l2.ft.t;
 	struct mlx5_flow_handle *rule;
 	int err;
 
 	rule = mlx5e_add_trap_rule(ft, trap_id, tir_num);
 	if (IS_ERR(rule)) {
 		err = PTR_ERR(rule);
-		priv->fs->l2.trap_rule = NULL;
-		mlx5_core_err(priv->fs->mdev, "%s: add MAC trap rule failed, err %d\n",
-			      __func__, err);
+		fs->l2.trap_rule = NULL;
+		fs_err(fs, "%s: add MAC trap rule failed, err %d\n",
+		       __func__, err);
 		return err;
 	}
-	priv->fs->l2.trap_rule = rule;
+	fs->l2.trap_rule = rule;
 	return 0;
 }
 
-void mlx5e_remove_mac_trap(struct mlx5e_priv *priv)
+void mlx5e_remove_mac_trap(struct mlx5e_flow_steering *fs)
 {
-	if (priv->fs->l2.trap_rule) {
-		mlx5_del_flow_rules(priv->fs->l2.trap_rule);
-		priv->fs->l2.trap_rule = NULL;
+	if (fs->l2.trap_rule) {
+		mlx5_del_flow_rules(fs->l2.trap_rule);
+		fs->l2.trap_rule = NULL;
 	}
 }
 
-void mlx5e_enable_cvlan_filter(struct mlx5e_priv *priv)
+void mlx5e_enable_cvlan_filter(struct mlx5e_flow_steering *fs, bool promisc)
 {
-	if (!priv->fs->vlan->cvlan_filter_disabled)
+	if (!fs->vlan->cvlan_filter_disabled)
 		return;
 
-	priv->fs->vlan->cvlan_filter_disabled = false;
-	if (priv->netdev->flags & IFF_PROMISC)
+	fs->vlan->cvlan_filter_disabled = false;
+	if (promisc)
 		return;
-	mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0);
+	mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0);
 }
 
-void mlx5e_disable_cvlan_filter(struct mlx5e_priv *priv)
+void mlx5e_disable_cvlan_filter(struct mlx5e_flow_steering *fs, bool promisc)
 {
-	if (priv->fs->vlan->cvlan_filter_disabled)
+	if (fs->vlan->cvlan_filter_disabled)
 		return;
 
-	priv->fs->vlan->cvlan_filter_disabled = true;
-	if (priv->netdev->flags & IFF_PROMISC)
+	fs->vlan->cvlan_filter_disabled = true;
+	if (promisc)
 		return;
-	mlx5e_add_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0);
+	mlx5e_add_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0);
 }
 
 static int mlx5e_vlan_rx_add_cvid(struct mlx5e_flow_steering *fs, u16 vid)
@@ -462,7 +489,7 @@ int mlx5e_fs_vlan_rx_add_vid(struct mlx5e_flow_steering *fs,
 {
 
 	if (!fs->vlan) {
-		mlx5_core_err(fs->mdev, "Vlan doesn't exist\n");
+		fs_err(fs, "Vlan doesn't exist\n");
 		return -EINVAL;
 	}
 
@@ -479,7 +506,7 @@ int mlx5e_fs_vlan_rx_kill_vid(struct mlx5e_flow_steering *fs,
 			      __be16 proto, u16 vid)
 {
 	if (!fs->vlan) {
-		mlx5_core_err(fs->mdev, "Vlan doesn't exist\n");
+		fs_err(fs, "Vlan doesn't exist\n");
 		return -EINVAL;
 	}
 
@@ -512,28 +539,28 @@ static void mlx5e_fs_add_vlan_rules(struct mlx5e_flow_steering *fs)
 		mlx5e_fs_add_any_vid_rules(fs);
 }
 
-static void mlx5e_del_vlan_rules(struct mlx5e_priv *priv)
+static void mlx5e_del_vlan_rules(struct mlx5e_flow_steering *fs)
 {
 	int i;
 
-	mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
+	mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0);
 
-	for_each_set_bit(i, priv->fs->vlan->active_cvlans, VLAN_N_VID) {
-		mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, i);
+	for_each_set_bit(i, fs->vlan->active_cvlans, VLAN_N_VID) {
+		mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, i);
 	}
 
-	for_each_set_bit(i, priv->fs->vlan->active_svlans, VLAN_N_VID)
-		mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, i);
+	for_each_set_bit(i, fs->vlan->active_svlans, VLAN_N_VID)
+		mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, i);
 
-	WARN_ON_ONCE(priv->fs->state_destroy);
+	WARN_ON_ONCE(fs->state_destroy);
 
-	mlx5e_remove_vlan_trap(priv);
+	mlx5e_remove_vlan_trap(fs);
 
 	/* must be called after DESTROY bit is set and
 	 * set_rx_mode is called and flushed
 	 */
-	if (priv->fs->vlan->cvlan_filter_disabled)
-		mlx5e_fs_del_any_vid_rules(priv->fs);
+	if (fs->vlan->cvlan_filter_disabled)
+		mlx5e_fs_del_any_vid_rules(fs);
 }
 
 #define mlx5e_for_each_hash_node(hn, tmp, hash, i) \
@@ -568,8 +595,9 @@ static void mlx5e_execute_l2_action(struct mlx5e_flow_steering *fs,
 	}
 
 	if (l2_err)
-		mlx5_core_warn(fs->mdev, "MPFS, failed to %s mac %pM, err(%d)\n",
-			       action == MLX5E_ACTION_ADD ? "add" : "del", mac_addr, l2_err);
+		fs_warn(fs, "MPFS, failed to %s mac %pM, err(%d)\n",
+			action == MLX5E_ACTION_ADD ? "add" : "del",
+			mac_addr, l2_err);
 }
 
 static void mlx5e_sync_netdev_addr(struct mlx5e_flow_steering *fs,
@@ -640,9 +668,8 @@ static void mlx5e_vport_context_update_addr_list(struct mlx5e_flow_steering *fs,
 		size++;
 
 	if (size > max_size) {
-		mlx5_core_warn(fs->mdev,
-			       "mdev %s list size (%d) > (%d) max vport list size, some addresses will be dropped\n",
-			      is_uc ? "UC" : "MC", size, max_size);
+		fs_warn(fs, "mdev %s list size (%d) > (%d) max vport list size, some addresses will be dropped\n",
+			is_uc ? "UC" : "MC", size, max_size);
 		size = max_size;
 	}
 
@@ -658,9 +685,8 @@ static void mlx5e_vport_context_update_addr_list(struct mlx5e_flow_steering *fs,
 	err = mlx5_modify_nic_vport_mac_list(fs->mdev, list_type, addr_array, size);
 out:
 	if (err)
-		mlx5_core_err(fs->mdev,
-			      "Failed to modify vport %s list err(%d)\n",
-			      is_uc ? "UC" : "MC", err);
+		fs_err(fs, "Failed to modify vport %s list err(%d)\n",
+		       is_uc ? "UC" : "MC", err);
 	kfree(addr_array);
 }
 
@@ -730,7 +756,7 @@ static int mlx5e_add_promisc_rule(struct mlx5e_flow_steering *fs)
 	if (IS_ERR(*rule_p)) {
 		err = PTR_ERR(*rule_p);
 		*rule_p = NULL;
-		mlx5_core_err(fs->mdev, "%s: add promiscuous rule failed\n", __func__);
+		fs_err(fs, "%s: add promiscuous rule failed\n", __func__);
 	}
 	kvfree(spec);
 	return err;
@@ -750,7 +776,7 @@ static int mlx5e_create_promisc_table(struct mlx5e_flow_steering *fs)
 	ft->t = mlx5_create_auto_grouped_flow_table(fs->ns, &ft_attr);
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
-		mlx5_core_err(fs->mdev, "fail to create promisc table err=%d\n", err);
+		fs_err(fs, "fail to create promisc table err=%d\n", err);
 		return err;
 	}
 
@@ -807,8 +833,8 @@ void mlx5e_fs_set_rx_mode_work(struct mlx5e_flow_steering *fs,
 		if (err)
 			enable_promisc = false;
 		if (!fs->vlan_strip_disable && !err)
-			mlx5_core_warn_once(fs->mdev,
-					    "S-tagged traffic will be dropped while C-tag vlan stripping is enabled\n");
+			fs_warn_once(fs,
+				     "S-tagged traffic will be dropped while C-tag vlan stripping is enabled\n");
 	}
 	if (enable_allmulti)
 		mlx5e_add_l2_flow_rule(fs, &ea->allmulti, MLX5E_ALLMULTI);
@@ -856,14 +882,15 @@ void mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft)
 	ft->t = NULL;
 }
 
-static void mlx5e_set_inner_ttc_params(struct mlx5e_priv *priv,
+static void mlx5e_set_inner_ttc_params(struct mlx5e_flow_steering *fs,
+				       struct mlx5e_rx_res *rx_res,
 				       struct ttc_params *ttc_params)
 {
 	struct mlx5_flow_table_attr *ft_attr = &ttc_params->ft_attr;
 	int tt;
 
 	memset(ttc_params, 0, sizeof(*ttc_params));
-	ttc_params->ns = mlx5_get_flow_namespace(priv->fs->mdev,
+	ttc_params->ns = mlx5_get_flow_namespace(fs->mdev,
 						 MLX5_FLOW_NAMESPACE_KERNEL);
 	ft_attr->level = MLX5E_INNER_TTC_FT_LEVEL;
 	ft_attr->prio = MLX5E_NIC_PRIO;
@@ -872,13 +899,14 @@ static void mlx5e_set_inner_ttc_params(struct mlx5e_priv *priv,
 		ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 		ttc_params->dests[tt].tir_num =
 			tt == MLX5_TT_ANY ?
-				mlx5e_rx_res_get_tirn_direct(priv->rx_res, 0) :
-				mlx5e_rx_res_get_tirn_rss_inner(priv->rx_res,
+				mlx5e_rx_res_get_tirn_direct(rx_res, 0) :
+				mlx5e_rx_res_get_tirn_rss_inner(rx_res,
 								tt);
 	}
 }
 
-void mlx5e_set_ttc_params(struct mlx5e_priv *priv,
+void mlx5e_set_ttc_params(struct mlx5e_flow_steering *fs,
+			  struct mlx5e_rx_res *rx_res,
 			  struct ttc_params *ttc_params, bool tunnel)
 
 {
@@ -886,7 +914,7 @@ void mlx5e_set_ttc_params(struct mlx5e_priv *priv,
 	int tt;
 
 	memset(ttc_params, 0, sizeof(*ttc_params));
-	ttc_params->ns = mlx5_get_flow_namespace(priv->fs->mdev,
+	ttc_params->ns = mlx5_get_flow_namespace(fs->mdev,
 						 MLX5_FLOW_NAMESPACE_KERNEL);
 	ft_attr->level = MLX5E_TTC_FT_LEVEL;
 	ft_attr->prio = MLX5E_NIC_PRIO;
@@ -895,19 +923,19 @@ void mlx5e_set_ttc_params(struct mlx5e_priv *priv,
 		ttc_params->dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_TIR;
 		ttc_params->dests[tt].tir_num =
 			tt == MLX5_TT_ANY ?
-				mlx5e_rx_res_get_tirn_direct(priv->rx_res, 0) :
-				mlx5e_rx_res_get_tirn_rss(priv->rx_res, tt);
+				mlx5e_rx_res_get_tirn_direct(rx_res, 0) :
+				mlx5e_rx_res_get_tirn_rss(rx_res, tt);
 	}
 
 	ttc_params->inner_ttc = tunnel;
-	if (!tunnel || !mlx5_tunnel_inner_ft_supported(priv->fs->mdev))
+	if (!tunnel || !mlx5_tunnel_inner_ft_supported(fs->mdev))
 		return;
 
 	for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
 		ttc_params->tunnel_dests[tt].type =
 			MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 		ttc_params->tunnel_dests[tt].ft =
-			mlx5_get_ttc_flow_table(priv->fs->inner_ttc);
+			mlx5_get_ttc_flow_table(fs->inner_ttc);
 	}
 }
 
@@ -959,8 +987,7 @@ static int mlx5e_add_l2_flow_rule(struct mlx5e_flow_steering *fs,
 
 	ai->rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
 	if (IS_ERR(ai->rule)) {
-		mlx5_core_err(fs->mdev, "%s: add l2 rule(mac:%pM) failed\n",
-			      __func__, mv_dmac);
+		fs_err(fs, "%s: add l2 rule(mac:%pM) failed\n", __func__, mv_dmac);
 		err = PTR_ERR(ai->rule);
 		ai->rule = NULL;
 	}
@@ -1044,14 +1071,14 @@ err_destroy_groups:
 	return err;
 }
 
-static void mlx5e_destroy_l2_table(struct mlx5e_priv *priv)
+static void mlx5e_destroy_l2_table(struct mlx5e_flow_steering *fs)
 {
-	mlx5e_destroy_flow_table(&priv->fs->l2.ft);
+	mlx5e_destroy_flow_table(&fs->l2.ft);
 }
 
-static int mlx5e_create_l2_table(struct mlx5e_priv *priv)
+static int mlx5e_create_l2_table(struct mlx5e_flow_steering *fs)
 {
-	struct mlx5e_l2_table *l2_table = &priv->fs->l2;
+	struct mlx5e_l2_table *l2_table = &fs->l2;
 	struct mlx5e_flow_table *ft = &l2_table->ft;
 	struct mlx5_flow_table_attr ft_attr = {};
 	int err;
@@ -1062,7 +1089,7 @@ static int mlx5e_create_l2_table(struct mlx5e_priv *priv)
 	ft_attr.level = MLX5E_L2_FT_LEVEL;
 	ft_attr.prio = MLX5E_NIC_PRIO;
 
-	ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr);
+	ft->t = mlx5_create_flow_table(fs->ns, &ft_attr);
 	if (IS_ERR(ft->t)) {
 		err = PTR_ERR(ft->t);
 		ft->t = NULL;
@@ -1221,126 +1248,128 @@ err_destroy_vlan_table:
 	return err;
 }
 
-static void mlx5e_destroy_vlan_table(struct mlx5e_priv *priv)
+static void mlx5e_destroy_vlan_table(struct mlx5e_flow_steering *fs)
 {
-	mlx5e_del_vlan_rules(priv);
-	mlx5e_destroy_flow_table(&priv->fs->vlan->ft);
+	mlx5e_del_vlan_rules(fs);
+	mlx5e_destroy_flow_table(&fs->vlan->ft);
 }
 
-static void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv)
+static void mlx5e_destroy_inner_ttc_table(struct mlx5e_flow_steering *fs)
 {
-	if (!mlx5_tunnel_inner_ft_supported(priv->fs->mdev))
+	if (!mlx5_tunnel_inner_ft_supported(fs->mdev))
 		return;
-	mlx5_destroy_ttc_table(priv->fs->inner_ttc);
+	mlx5_destroy_ttc_table(fs->inner_ttc);
 }
 
-void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv)
+void mlx5e_destroy_ttc_table(struct mlx5e_flow_steering *fs)
 {
-	mlx5_destroy_ttc_table(priv->fs->ttc);
+	mlx5_destroy_ttc_table(fs->ttc);
 }
 
-static int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv)
+static int mlx5e_create_inner_ttc_table(struct mlx5e_flow_steering *fs,
+					struct mlx5e_rx_res *rx_res)
 {
 	struct ttc_params ttc_params = {};
 
-	if (!mlx5_tunnel_inner_ft_supported(priv->fs->mdev))
+	if (!mlx5_tunnel_inner_ft_supported(fs->mdev))
 		return 0;
 
-	mlx5e_set_inner_ttc_params(priv, &ttc_params);
-	priv->fs->inner_ttc = mlx5_create_inner_ttc_table(priv->fs->mdev,
-							  &ttc_params);
-	if (IS_ERR(priv->fs->inner_ttc))
-		return PTR_ERR(priv->fs->inner_ttc);
+	mlx5e_set_inner_ttc_params(fs, rx_res, &ttc_params);
+	fs->inner_ttc = mlx5_create_inner_ttc_table(fs->mdev,
+						    &ttc_params);
+	if (IS_ERR(fs->inner_ttc))
+		return PTR_ERR(fs->inner_ttc);
 	return 0;
 }
 
-int mlx5e_create_ttc_table(struct mlx5e_priv *priv)
+int mlx5e_create_ttc_table(struct mlx5e_flow_steering *fs,
+			   struct mlx5e_rx_res *rx_res)
 {
 	struct ttc_params ttc_params = {};
 
-	mlx5e_set_ttc_params(priv, &ttc_params, true);
-	priv->fs->ttc = mlx5_create_ttc_table(priv->fs->mdev, &ttc_params);
-	if (IS_ERR(priv->fs->ttc))
-		return PTR_ERR(priv->fs->ttc);
+	mlx5e_set_ttc_params(fs, rx_res, &ttc_params, true);
+	fs->ttc = mlx5_create_ttc_table(fs->mdev, &ttc_params);
+	if (IS_ERR(fs->ttc))
+		return PTR_ERR(fs->ttc);
 	return 0;
 }
 
-int mlx5e_create_flow_steering(struct mlx5e_priv *priv)
+int mlx5e_create_flow_steering(struct mlx5e_flow_steering *fs,
+			       struct mlx5e_rx_res *rx_res,
+			       const struct mlx5e_profile *profile,
+			       struct net_device *netdev)
 {
+	struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(fs->mdev,
+								 MLX5_FLOW_NAMESPACE_KERNEL);
 	int err;
 
-	priv->fs->ns = mlx5_get_flow_namespace(priv->fs->mdev,
-					       MLX5_FLOW_NAMESPACE_KERNEL);
-
-	if (!priv->fs->ns)
+	if (!ns)
 		return -EOPNOTSUPP;
 
-	err = mlx5e_arfs_create_tables(priv);
+	mlx5e_fs_set_ns(fs, ns, false);
+	err = mlx5e_arfs_create_tables(fs, rx_res,
+				       !!(netdev->hw_features & NETIF_F_NTUPLE));
 	if (err) {
-		mlx5_core_err(priv->fs->mdev, "Failed to create arfs tables, err=%d\n",
-			      err);
-		priv->netdev->hw_features &= ~NETIF_F_NTUPLE;
+		fs_err(fs, "Failed to create arfs tables, err=%d\n", err);
+		netdev->hw_features &= ~NETIF_F_NTUPLE;
 	}
 
-	err = mlx5e_create_inner_ttc_table(priv);
+	err = mlx5e_create_inner_ttc_table(fs, rx_res);
 	if (err) {
-		mlx5_core_err(priv->fs->mdev,
-			      "Failed to create inner ttc table, err=%d\n", err);
+		fs_err(fs, "Failed to create inner ttc table, err=%d\n", err);
 		goto err_destroy_arfs_tables;
 	}
 
-	err = mlx5e_create_ttc_table(priv);
+	err = mlx5e_create_ttc_table(fs, rx_res);
 	if (err) {
-		mlx5_core_err(priv->fs->mdev, "Failed to create ttc table, err=%d\n",
-			      err);
+		fs_err(fs, "Failed to create ttc table, err=%d\n", err);
 		goto err_destroy_inner_ttc_table;
 	}
 
-	err = mlx5e_create_l2_table(priv);
+	err = mlx5e_create_l2_table(fs);
 	if (err) {
-		mlx5_core_err(priv->fs->mdev, "Failed to create l2 table, err=%d\n",
-			      err);
+		fs_err(fs, "Failed to create l2 table, err=%d\n", err);
 		goto err_destroy_ttc_table;
 	}
 
-	err = mlx5e_fs_create_vlan_table(priv->fs);
+	err = mlx5e_fs_create_vlan_table(fs);
 	if (err) {
-		mlx5_core_err(priv->fs->mdev, "Failed to create vlan table, err=%d\n",
-			      err);
+		fs_err(fs, "Failed to create vlan table, err=%d\n", err);
 		goto err_destroy_l2_table;
 	}
 
-	err = mlx5e_ptp_alloc_rx_fs(priv);
+	err = mlx5e_ptp_alloc_rx_fs(fs, profile);
 	if (err)
 		goto err_destory_vlan_table;
 
-	mlx5e_ethtool_init_steering(priv);
+	mlx5e_ethtool_init_steering(fs);
 
 	return 0;
 
 err_destory_vlan_table:
-	mlx5e_destroy_vlan_table(priv);
+	mlx5e_destroy_vlan_table(fs);
 err_destroy_l2_table:
-	mlx5e_destroy_l2_table(priv);
+	mlx5e_destroy_l2_table(fs);
 err_destroy_ttc_table:
-	mlx5e_destroy_ttc_table(priv);
+	mlx5e_destroy_ttc_table(fs);
 err_destroy_inner_ttc_table:
-	mlx5e_destroy_inner_ttc_table(priv);
+	mlx5e_destroy_inner_ttc_table(fs);
 err_destroy_arfs_tables:
-	mlx5e_arfs_destroy_tables(priv);
+	mlx5e_arfs_destroy_tables(fs, !!(netdev->hw_features & NETIF_F_NTUPLE));
 
 	return err;
 }
 
-void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv)
+void mlx5e_destroy_flow_steering(struct mlx5e_flow_steering *fs, bool ntuple,
+				 const struct mlx5e_profile *profile)
 {
-	mlx5e_ptp_free_rx_fs(priv);
-	mlx5e_destroy_vlan_table(priv);
-	mlx5e_destroy_l2_table(priv);
-	mlx5e_destroy_ttc_table(priv);
-	mlx5e_destroy_inner_ttc_table(priv);
-	mlx5e_arfs_destroy_tables(priv);
-	mlx5e_ethtool_cleanup_steering(priv);
+	mlx5e_ptp_free_rx_fs(fs, profile);
+	mlx5e_destroy_vlan_table(fs);
+	mlx5e_destroy_l2_table(fs);
+	mlx5e_destroy_ttc_table(fs);
+	mlx5e_destroy_inner_ttc_table(fs);
+	mlx5e_arfs_destroy_tables(fs, ntuple);
+	mlx5e_ethtool_cleanup_steering(fs);
 }
 
 static int mlx5e_fs_vlan_alloc(struct mlx5e_flow_steering *fs)
@@ -1356,6 +1385,11 @@ static void mlx5e_fs_vlan_free(struct mlx5e_flow_steering *fs)
 	kvfree(fs->vlan);
 }
 
+struct mlx5e_vlan_table *mlx5e_fs_get_vlan(struct mlx5e_flow_steering *fs)
+{
+	return fs->vlan;
+}
+
 static int mlx5e_fs_tc_alloc(struct mlx5e_flow_steering *fs)
 {
 	fs->tc = mlx5e_tc_table_alloc();
@@ -1369,6 +1403,32 @@ static void mlx5e_fs_tc_free(struct mlx5e_flow_steering *fs)
 	mlx5e_tc_table_free(fs->tc);
 }
 
+struct mlx5e_tc_table *mlx5e_fs_get_tc(struct mlx5e_flow_steering *fs)
+{
+	return fs->tc;
+}
+
+#ifdef CONFIG_MLX5_EN_RXNFC
+static int mlx5e_fs_ethtool_alloc(struct mlx5e_flow_steering *fs)
+{
+	return mlx5e_ethtool_alloc(&fs->ethtool);
+}
+
+static void mlx5e_fs_ethtool_free(struct mlx5e_flow_steering *fs)
+{
+	mlx5e_ethtool_free(fs->ethtool);
+}
+
+struct mlx5e_ethtool_steering *mlx5e_fs_get_ethtool(struct mlx5e_flow_steering *fs)
+{
+	return fs->ethtool;
+}
+#else
+static int mlx5e_fs_ethtool_alloc(struct mlx5e_flow_steering *fs)
+{ return 0; }
+static void mlx5e_fs_ethtool_free(struct mlx5e_flow_steering *fs) { }
+#endif
+
 struct mlx5e_flow_steering *mlx5e_fs_init(const struct mlx5e_profile *profile,
 					  struct mlx5_core_dev *mdev,
 					  bool state_destroy)
@@ -1394,8 +1454,13 @@ struct mlx5e_flow_steering *mlx5e_fs_init(const struct mlx5e_profile *profile,
 			goto err_free_vlan;
 	}
 
-	return fs;
+	err = mlx5e_fs_ethtool_alloc(fs);
+	if (err)
+		goto err_free_tc;
 
+	return fs;
+err_free_tc:
+	mlx5e_fs_tc_free(fs);
 err_free_vlan:
 	mlx5e_fs_vlan_free(fs);
 err_free_fs:
@@ -1406,7 +1471,109 @@ err:
 
 void mlx5e_fs_cleanup(struct mlx5e_flow_steering *fs)
 {
+	mlx5e_fs_ethtool_free(fs);
 	mlx5e_fs_tc_free(fs);
 	mlx5e_fs_vlan_free(fs);
 	kvfree(fs);
 }
+
+struct mlx5e_l2_table *mlx5e_fs_get_l2(struct mlx5e_flow_steering *fs)
+{
+	return &fs->l2;
+}
+
+struct mlx5_flow_namespace *mlx5e_fs_get_ns(struct mlx5e_flow_steering *fs, bool egress)
+{
+	return  egress ? fs->egress_ns : fs->ns;
+}
+
+void mlx5e_fs_set_ns(struct mlx5e_flow_steering *fs, struct mlx5_flow_namespace *ns, bool egress)
+{
+	if (!egress)
+		fs->ns = ns;
+	else
+		fs->egress_ns = ns;
+}
+
+struct mlx5_ttc_table *mlx5e_fs_get_ttc(struct mlx5e_flow_steering *fs, bool inner)
+{
+	return inner ? fs->inner_ttc : fs->ttc;
+}
+
+void mlx5e_fs_set_ttc(struct mlx5e_flow_steering *fs, struct mlx5_ttc_table *ttc, bool inner)
+{
+	if (!inner)
+		fs->ttc = ttc;
+	else
+		fs->inner_ttc = ttc;
+}
+
+#ifdef CONFIG_MLX5_EN_ARFS
+struct mlx5e_arfs_tables *mlx5e_fs_get_arfs(struct mlx5e_flow_steering *fs)
+{
+	return fs->arfs;
+}
+
+void mlx5e_fs_set_arfs(struct mlx5e_flow_steering *fs, struct mlx5e_arfs_tables *arfs)
+{
+	fs->arfs = arfs;
+}
+#endif
+
+struct mlx5e_ptp_fs *mlx5e_fs_get_ptp(struct mlx5e_flow_steering *fs)
+{
+	return fs->ptp_fs;
+}
+
+void mlx5e_fs_set_ptp(struct mlx5e_flow_steering *fs, struct mlx5e_ptp_fs *ptp_fs)
+{
+	fs->ptp_fs = ptp_fs;
+}
+
+struct mlx5e_fs_any *mlx5e_fs_get_any(struct mlx5e_flow_steering *fs)
+{
+	return fs->any;
+}
+
+void mlx5e_fs_set_any(struct mlx5e_flow_steering *fs, struct mlx5e_fs_any *any)
+{
+	fs->any = any;
+}
+
+#ifdef CONFIG_MLX5_EN_TLS
+struct mlx5e_accel_fs_tcp *mlx5e_fs_get_accel_tcp(struct mlx5e_flow_steering *fs)
+{
+	return fs->accel_tcp;
+}
+
+void mlx5e_fs_set_accel_tcp(struct mlx5e_flow_steering *fs, struct mlx5e_accel_fs_tcp *accel_tcp)
+{
+	fs->accel_tcp = accel_tcp;
+}
+#endif
+
+void mlx5e_fs_set_state_destroy(struct mlx5e_flow_steering *fs, bool state_destroy)
+{
+	fs->state_destroy = state_destroy;
+}
+
+void mlx5e_fs_set_vlan_strip_disable(struct mlx5e_flow_steering *fs,
+				     bool vlan_strip_disable)
+{
+	fs->vlan_strip_disable = vlan_strip_disable;
+}
+
+struct mlx5e_fs_udp *mlx5e_fs_get_udp(struct mlx5e_flow_steering *fs)
+{
+	return fs->udp;
+}
+
+void mlx5e_fs_set_udp(struct mlx5e_flow_steering *fs, struct mlx5e_fs_udp *udp)
+{
+	fs->udp = udp;
+}
+
+struct mlx5_core_dev *mlx5e_fs_get_mdev(struct mlx5e_flow_steering *fs)
+{
+	return fs->mdev;
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c
index 3e4bc7836ef4..aac32e505c14 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c
@@ -34,6 +34,22 @@
 #include "en.h"
 #include "en/params.h"
 #include "en/xsk/pool.h"
+#include "en/fs_ethtool.h"
+
+struct mlx5e_ethtool_table {
+	struct mlx5_flow_table *ft;
+	int                    num_rules;
+};
+
+#define ETHTOOL_NUM_L3_L4_FTS 7
+#define ETHTOOL_NUM_L2_FTS 4
+
+struct mlx5e_ethtool_steering {
+	struct mlx5e_ethtool_table      l3_l4_ft[ETHTOOL_NUM_L3_L4_FTS];
+	struct mlx5e_ethtool_table      l2_ft[ETHTOOL_NUM_L2_FTS];
+	struct list_head                rules;
+	int                             tot_num_rules;
+};
 
 static int flow_type_to_traffic_type(u32 flow_type);
 
@@ -66,6 +82,7 @@ static struct mlx5e_ethtool_table *get_flow_table(struct mlx5e_priv *priv,
 						  struct ethtool_rx_flow_spec *fs,
 						  int num_tuples)
 {
+	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs);
 	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5e_ethtool_table *eth_ft;
 	struct mlx5_flow_namespace *ns;
@@ -81,18 +98,18 @@ static struct mlx5e_ethtool_table *get_flow_table(struct mlx5e_priv *priv,
 	case UDP_V6_FLOW:
 		max_tuples = ETHTOOL_NUM_L3_L4_FTS;
 		prio = MLX5E_ETHTOOL_L3_L4_PRIO + (max_tuples - num_tuples);
-		eth_ft = &priv->fs->ethtool.l3_l4_ft[prio];
+		eth_ft = &ethtool->l3_l4_ft[prio];
 		break;
 	case IP_USER_FLOW:
 	case IPV6_USER_FLOW:
 		max_tuples = ETHTOOL_NUM_L3_L4_FTS;
 		prio = MLX5E_ETHTOOL_L3_L4_PRIO + (max_tuples - num_tuples);
-		eth_ft = &priv->fs->ethtool.l3_l4_ft[prio];
+		eth_ft = &ethtool->l3_l4_ft[prio];
 		break;
 	case ETHER_FLOW:
 		max_tuples = ETHTOOL_NUM_L2_FTS;
 		prio = max_tuples - num_tuples;
-		eth_ft = &priv->fs->ethtool.l2_ft[prio];
+		eth_ft = &ethtool->l2_ft[prio];
 		prio += MLX5E_ETHTOOL_L2_PRIO;
 		break;
 	default:
@@ -382,15 +399,16 @@ static int set_flow_attrs(u32 *match_c, u32 *match_v,
 static void add_rule_to_list(struct mlx5e_priv *priv,
 			     struct mlx5e_ethtool_rule *rule)
 {
+	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs);
+	struct list_head *head = &ethtool->rules;
 	struct mlx5e_ethtool_rule *iter;
-	struct list_head *head = &priv->fs->ethtool.rules;
 
-	list_for_each_entry(iter, &priv->fs->ethtool.rules, list) {
+	list_for_each_entry(iter, &ethtool->rules, list) {
 		if (iter->flow_spec.location > rule->flow_spec.location)
 			break;
 		head = &iter->list;
 	}
-	priv->fs->ethtool.tot_num_rules++;
+	ethtool->tot_num_rules++;
 	list_add(&rule->list, head);
 }
 
@@ -433,15 +451,7 @@ static int flow_get_tirn(struct mlx5e_priv *priv,
 		eth_rule->rss = rss;
 		mlx5e_rss_refcnt_inc(eth_rule->rss);
 	} else {
-		struct mlx5e_params *params = &priv->channels.params;
-		enum mlx5e_rq_group group;
-		u16 ix;
-
-		mlx5e_qid_get_ch_and_group(params, fs->ring_cookie, &ix, &group);
-
-		*tirn = group == MLX5E_RQ_GROUP_XSK ?
-			mlx5e_rx_res_get_tirn_xsk(priv->rx_res, ix) :
-			mlx5e_rx_res_get_tirn_direct(priv->rx_res, ix);
+		*tirn = mlx5e_rx_res_get_tirn_direct(priv->rx_res, fs->ring_cookie);
 	}
 
 	return 0;
@@ -499,15 +509,16 @@ free:
 	return err ? ERR_PTR(err) : rule;
 }
 
-static void del_ethtool_rule(struct mlx5e_priv *priv,
+static void del_ethtool_rule(struct mlx5e_flow_steering *fs,
 			     struct mlx5e_ethtool_rule *eth_rule)
 {
+	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(fs);
 	if (eth_rule->rule)
 		mlx5_del_flow_rules(eth_rule->rule);
 	if (eth_rule->rss)
 		mlx5e_rss_refcnt_dec(eth_rule->rss);
 	list_del(&eth_rule->list);
-	priv->fs->ethtool.tot_num_rules--;
+	ethtool->tot_num_rules--;
 	put_flow_table(eth_rule->eth_ft);
 	kfree(eth_rule);
 }
@@ -515,9 +526,10 @@ static void del_ethtool_rule(struct mlx5e_priv *priv,
 static struct mlx5e_ethtool_rule *find_ethtool_rule(struct mlx5e_priv *priv,
 						    int location)
 {
+	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs);
 	struct mlx5e_ethtool_rule *iter;
 
-	list_for_each_entry(iter, &priv->fs->ethtool.rules, list) {
+	list_for_each_entry(iter, &ethtool->rules, list) {
 		if (iter->flow_spec.location == location)
 			return iter;
 	}
@@ -531,7 +543,7 @@ static struct mlx5e_ethtool_rule *get_ethtool_rule(struct mlx5e_priv *priv,
 
 	eth_rule = find_ethtool_rule(priv, location);
 	if (eth_rule)
-		del_ethtool_rule(priv, eth_rule);
+		del_ethtool_rule(priv->fs, eth_rule);
 
 	eth_rule = kzalloc(sizeof(*eth_rule), GFP_KERNEL);
 	if (!eth_rule)
@@ -662,8 +674,7 @@ static int validate_flow(struct mlx5e_priv *priv,
 		return -ENOSPC;
 
 	if (fs->ring_cookie != RX_CLS_FLOW_DISC)
-		if (!mlx5e_qid_validate(priv->profile, &priv->channels.params,
-					fs->ring_cookie))
+		if (fs->ring_cookie >= priv->channels.params.num_channels)
 			return -EINVAL;
 
 	switch (flow_type_mask(fs->flow_type)) {
@@ -754,7 +765,7 @@ mlx5e_ethtool_flow_replace(struct mlx5e_priv *priv,
 	return 0;
 
 del_ethtool_rule:
-	del_ethtool_rule(priv, eth_rule);
+	del_ethtool_rule(priv->fs, eth_rule);
 
 	return err;
 }
@@ -774,7 +785,7 @@ mlx5e_ethtool_flow_remove(struct mlx5e_priv *priv, int location)
 		goto out;
 	}
 
-	del_ethtool_rule(priv, eth_rule);
+	del_ethtool_rule(priv->fs, eth_rule);
 out:
 	return err;
 }
@@ -783,12 +794,13 @@ static int
 mlx5e_ethtool_get_flow(struct mlx5e_priv *priv,
 		       struct ethtool_rxnfc *info, int location)
 {
+	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs);
 	struct mlx5e_ethtool_rule *eth_rule;
 
 	if (location < 0 || location >= MAX_NUM_OF_ETHTOOL_RULES)
 		return -EINVAL;
 
-	list_for_each_entry(eth_rule, &priv->fs->ethtool.rules, list) {
+	list_for_each_entry(eth_rule, &ethtool->rules, list) {
 		int index;
 
 		if (eth_rule->flow_spec.location != location)
@@ -826,18 +838,34 @@ mlx5e_ethtool_get_all_flows(struct mlx5e_priv *priv,
 	return err;
 }
 
-void mlx5e_ethtool_cleanup_steering(struct mlx5e_priv *priv)
+int mlx5e_ethtool_alloc(struct mlx5e_ethtool_steering **ethtool)
 {
+	*ethtool =  kvzalloc(sizeof(**ethtool), GFP_KERNEL);
+	if (!*ethtool)
+		return -ENOMEM;
+	return 0;
+}
+
+void mlx5e_ethtool_free(struct mlx5e_ethtool_steering *ethtool)
+{
+	kvfree(ethtool);
+}
+
+void mlx5e_ethtool_cleanup_steering(struct mlx5e_flow_steering *fs)
+{
+	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(fs);
 	struct mlx5e_ethtool_rule *iter;
 	struct mlx5e_ethtool_rule *temp;
 
-	list_for_each_entry_safe(iter, temp, &priv->fs->ethtool.rules, list)
-		del_ethtool_rule(priv, iter);
+	list_for_each_entry_safe(iter, temp, &ethtool->rules, list)
+		del_ethtool_rule(fs, iter);
 }
 
-void mlx5e_ethtool_init_steering(struct mlx5e_priv *priv)
+void mlx5e_ethtool_init_steering(struct mlx5e_flow_steering *fs)
 {
-	INIT_LIST_HEAD(&priv->fs->ethtool.rules);
+	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(fs);
+
+	INIT_LIST_HEAD(&ethtool->rules);
 }
 
 static int flow_type_to_traffic_type(u32 flow_type)
@@ -959,11 +987,12 @@ int mlx5e_ethtool_set_rxnfc(struct mlx5e_priv *priv, struct ethtool_rxnfc *cmd)
 int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv,
 			    struct ethtool_rxnfc *info, u32 *rule_locs)
 {
+	struct mlx5e_ethtool_steering *ethtool = mlx5e_fs_get_ethtool(priv->fs);
 	int err = 0;
 
 	switch (info->cmd) {
 	case ETHTOOL_GRXCLSRLCNT:
-		info->rule_cnt = priv->fs->ethtool.tot_num_rules;
+		info->rule_cnt = ethtool->tot_num_rules;
 		break;
 	case ETHTOOL_GRXCLSRULE:
 		err = mlx5e_ethtool_get_flow(priv, info, info->fs.location);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 02eb2f0fa2ae..364f04309149 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -45,6 +45,7 @@
 #include "en_tc.h"
 #include "en_rep.h"
 #include "en_accel/ipsec.h"
+#include "en_accel/macsec.h"
 #include "en_accel/en_accel.h"
 #include "en_accel/ktls.h"
 #include "lib/vxlan.h"
@@ -67,22 +68,25 @@
 #include "qos.h"
 #include "en/trap.h"
 
-bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
+bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,
+					    enum mlx5e_mpwrq_umr_mode umr_mode)
 {
-	bool striding_rq_umr, inline_umr;
-	u16 max_wqe_sz_cap;
+	u16 umr_wqebbs, max_wqebbs;
+	bool striding_rq_umr;
 
 	striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
 			  MLX5_CAP_ETH(mdev, reg_umr_sq);
-	max_wqe_sz_cap = mlx5e_get_max_sq_wqebbs(mdev) * MLX5_SEND_WQE_BB;
-	inline_umr = max_wqe_sz_cap >= MLX5E_UMR_WQE_INLINE_SZ;
 	if (!striding_rq_umr)
 		return false;
-	if (!inline_umr) {
-		mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n",
-			       (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap);
+
+	umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode);
+	max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
+	/* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is
+	 * calculated from mlx5e_get_max_sq_aligned_wqebbs.
+	 */
+	if (WARN_ON(umr_wqebbs > max_wqebbs))
 		return false;
-	}
+
 	return true;
 }
 
@@ -199,21 +203,35 @@ static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv)
 	mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb);
 }
 
+static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode)
+{
+	u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode);
+
+	WARN_ON_ONCE(entries * umr_entry_size % MLX5_OCTWORD);
+
+	return entries * umr_entry_size / MLX5_OCTWORD;
+}
+
 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
 				       struct mlx5e_icosq *sq,
 				       struct mlx5e_umr_wqe *wqe)
 {
 	struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
 	struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
-	u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS);
+	u16 octowords;
+	u8 ds_cnt;
+
+	ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift,
+						     rq->mpwqe.umr_mode),
+			      MLX5_SEND_WQE_DS);
 
 	cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
 				      ds_cnt);
-	cseg->umr_mkey  = rq->mkey_be;
+	cseg->umr_mkey  = rq->mpwqe.umr_mkey_be;
 
 	ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
-	ucseg->xlt_octowords =
-		cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
+	octowords = mlx5e_mpwrq_umr_octowords(rq->mpwqe.pages_per_wqe, rq->mpwqe.umr_mode);
+	ucseg->xlt_octowords = cpu_to_be16(octowords);
 	ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
 }
 
@@ -259,10 +277,12 @@ static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq)
 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
 {
 	int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
+	size_t alloc_size;
+
+	alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info, alloc_units,
+						   rq->mpwqe.pages_per_wqe));
 
-	rq->mpwqe.info = kvzalloc_node(array_size(wq_sz,
-						  sizeof(*rq->mpwqe.info)),
-				       GFP_KERNEL, node);
+	rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node);
 	if (!rq->mpwqe.info)
 		return -ENOMEM;
 
@@ -271,18 +291,52 @@ static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
 	return 0;
 }
 
-static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev,
-				     u64 npages, u8 page_shift, u32 *umr_mkey,
-				     dma_addr_t filler_addr)
+
+static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode)
+{
+	switch (umr_mode) {
+	case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
+		return MLX5_MKC_ACCESS_MODE_MTT;
+	case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
+		return MLX5_MKC_ACCESS_MODE_KSM;
+	case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
+		return MLX5_MKC_ACCESS_MODE_KLMS;
+	case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
+		return MLX5_MKC_ACCESS_MODE_KSM;
+	}
+	WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode);
+	return 0;
+}
+
+static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
+				 u32 npages, u8 page_shift, u32 *umr_mkey,
+				 dma_addr_t filler_addr,
+				 enum mlx5e_mpwrq_umr_mode umr_mode,
+				 u32 xsk_chunk_size)
 {
 	struct mlx5_mtt *mtt;
+	struct mlx5_ksm *ksm;
+	struct mlx5_klm *klm;
+	u32 octwords;
 	int inlen;
 	void *mkc;
 	u32 *in;
 	int err;
 	int i;
 
-	inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + sizeof(*mtt) * npages;
+	if ((umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED ||
+	     umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) &&
+	    !MLX5_CAP_GEN(mdev, fixed_buffer_size)) {
+		mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n");
+		return -EINVAL;
+	}
+
+	octwords = mlx5e_mpwrq_umr_octowords(npages, umr_mode);
+
+	inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in),
+				    MLX5_OCTWORD, octwords);
+	if (inlen < 0)
+		return inlen;
 
 	in = kvzalloc(inlen, GFP_KERNEL);
 	if (!in)
@@ -294,16 +348,17 @@ static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev,
 	MLX5_SET(mkc, mkc, umr_en, 1);
 	MLX5_SET(mkc, mkc, lw, 1);
 	MLX5_SET(mkc, mkc, lr, 1);
-	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
+	MLX5_SET(mkc, mkc, access_mode_1_0, mlx5e_mpwrq_access_mode(umr_mode));
 	mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
 	MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
 	MLX5_SET64(mkc, mkc, len, npages << page_shift);
-	MLX5_SET(mkc, mkc, translations_octword_size,
-		 MLX5_MTT_OCTW(npages));
-	MLX5_SET(mkc, mkc, log_page_size, page_shift);
-	MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
-		 MLX5_MTT_OCTW(npages));
+	MLX5_SET(mkc, mkc, translations_octword_size, octwords);
+	if (umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)
+		MLX5_SET(mkc, mkc, log_page_size, page_shift - 2);
+	else if (umr_mode != MLX5E_MPWRQ_UMR_MODE_OVERSIZED)
+		MLX5_SET(mkc, mkc, log_page_size, page_shift);
+	MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords);
 
 	/* Initialize the mkey with all MTTs pointing to a default
 	 * page (filler_addr). When the channels are activated, UMR
@@ -311,9 +366,47 @@ static int mlx5e_create_umr_mtt_mkey(struct mlx5_core_dev *mdev,
 	 * the RQ's pool, while the gaps (wqe_overflow) remain mapped
 	 * to the default page.
 	 */
-	mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
-	for (i = 0 ; i < npages ; i++)
-		mtt[i].ptag = cpu_to_be64(filler_addr);
+	switch (umr_mode) {
+	case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
+		klm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
+		for (i = 0; i < npages; i++) {
+			klm[i << 1] = (struct mlx5_klm) {
+				.va = cpu_to_be64(filler_addr),
+				.bcount = cpu_to_be32(xsk_chunk_size),
+				.key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
+			};
+			klm[(i << 1) + 1] = (struct mlx5_klm) {
+				.va = cpu_to_be64(filler_addr),
+				.bcount = cpu_to_be32((1 << page_shift) - xsk_chunk_size),
+				.key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
+			};
+		}
+		break;
+	case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
+		ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
+		for (i = 0; i < npages; i++)
+			ksm[i] = (struct mlx5_ksm) {
+				.key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
+				.va = cpu_to_be64(filler_addr),
+			};
+		break;
+	case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
+		mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
+		for (i = 0; i < npages; i++)
+			mtt[i] = (struct mlx5_mtt) {
+				.ptag = cpu_to_be64(filler_addr),
+			};
+		break;
+	case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
+		ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
+		for (i = 0; i < npages * 4; i++) {
+			ksm[i] = (struct mlx5_ksm) {
+				.key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
+				.va = cpu_to_be64(filler_addr),
+			};
+		}
+		break;
+	}
 
 	err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
 
@@ -356,10 +449,27 @@ static int mlx5e_create_umr_klm_mkey(struct mlx5_core_dev *mdev,
 
 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
 {
-	u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq));
+	u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0;
+	u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
+	u32 num_entries, max_num_entries;
+	u32 umr_mkey;
+	int err;
+
+	max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.umr_mode);
+
+	/* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */
+	if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe,
+					    &num_entries) ||
+			 num_entries > max_num_entries))
+		mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n",
+			      __func__, wq_size, rq->mpwqe.mtts_per_wqe,
+			      max_num_entries);
 
-	return mlx5e_create_umr_mtt_mkey(mdev, num_mtts, PAGE_SHIFT,
-					 &rq->umr_mkey, rq->wqe_overflow.addr);
+	err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift,
+				    &umr_mkey, rq->wqe_overflow.addr,
+				    rq->mpwqe.umr_mode, xsk_chunk_size);
+	rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey);
+	return err;
 }
 
 static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev,
@@ -376,18 +486,20 @@ static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev,
 					 &rq->mpwqe.shampo->mkey);
 }
 
-static u64 mlx5e_get_mpwqe_offset(u16 wqe_ix)
-{
-	return MLX5E_REQUIRED_MTTS(wqe_ix) << PAGE_SHIFT;
-}
-
 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
 {
 	struct mlx5e_wqe_frag_info next_frag = {};
 	struct mlx5e_wqe_frag_info *prev = NULL;
 	int i;
 
-	next_frag.di = &rq->wqe.di[0];
+	if (rq->xsk_pool) {
+		/* Assumptions used by XSK batched allocator. */
+		WARN_ON(rq->wqe.info.num_frags != 1);
+		WARN_ON(rq->wqe.info.log_num_frags != 0);
+		WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE);
+	}
+
+	next_frag.au = &rq->wqe.alloc_units[0];
 
 	for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
 		struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
@@ -397,7 +509,7 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
 
 		for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
 			if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
-				next_frag.di++;
+				next_frag.au++;
 				next_frag.offset = 0;
 				if (prev)
 					prev->last_in_page = true;
@@ -414,12 +526,13 @@ static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
 		prev->last_in_page = true;
 }
 
-int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node)
+static int mlx5e_init_au_list(struct mlx5e_rq *rq, int wq_sz, int node)
 {
 	int len = wq_sz << rq->wqe.info.log_num_frags;
 
-	rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), GFP_KERNEL, node);
-	if (!rq->wqe.di)
+	rq->wqe.alloc_units = kvzalloc_node(array_size(len, sizeof(*rq->wqe.alloc_units)),
+					    GFP_KERNEL, node);
+	if (!rq->wqe.alloc_units)
 		return -ENOMEM;
 
 	mlx5e_init_frags_partition(rq);
@@ -427,9 +540,9 @@ int mlx5e_init_di_list(struct mlx5e_rq *rq, int wq_sz, int node)
 	return 0;
 }
 
-void mlx5e_free_di_list(struct mlx5e_rq *rq)
+static void mlx5e_free_au_list(struct mlx5e_rq *rq)
 {
-	kvfree(rq->wqe.di);
+	kvfree(rq->wqe.alloc_units);
 }
 
 static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work)
@@ -485,7 +598,7 @@ static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *param
 	if (err)
 		return err;
 
-	return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, 0);
+	return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, c->napi.napi_id);
 }
 
 static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
@@ -572,6 +685,8 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 	rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
 	pool_size = 1 << params->log_rq_mtu_frames;
 
+	rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
+
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 		err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
@@ -587,8 +702,20 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 
 		wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
 
-		pool_size = MLX5_MPWRQ_PAGES_PER_WQE <<
-			mlx5e_mpwqe_get_log_rq_size(params, xsk);
+		rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
+		rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
+		rq->mpwqe.pages_per_wqe =
+			mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift,
+						  rq->mpwqe.umr_mode);
+		rq->mpwqe.umr_wqebbs =
+			mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift,
+					       rq->mpwqe.umr_mode);
+		rq->mpwqe.mtts_per_wqe =
+			mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift,
+						 rq->mpwqe.umr_mode);
+
+		pool_size = rq->mpwqe.pages_per_wqe <<
+			mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk);
 
 		rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
 		rq->mpwqe.num_strides =
@@ -600,7 +727,6 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 		err = mlx5e_create_rq_umr_mkey(mdev, rq);
 		if (err)
 			goto err_rq_drop_page;
-		rq->mkey_be = cpu_to_be32(rq->umr_mkey);
 
 		err = mlx5e_rq_alloc_mpwqe_info(rq, node);
 		if (err)
@@ -608,7 +734,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 
 		err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node);
 		if (err)
-			goto err_free_by_rq_type;
+			goto err_free_mpwqe_info;
 
 		break;
 	default: /* MLX5_WQ_TYPE_CYCLIC */
@@ -633,11 +759,9 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 			goto err_rq_wq_destroy;
 		}
 
-		err = mlx5e_init_di_list(rq, wq_sz, node);
+		err = mlx5e_init_au_list(rq, wq_sz, node);
 		if (err)
 			goto err_rq_frags;
-
-		rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
 	}
 
 	if (xsk) {
@@ -662,14 +786,14 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 		if (IS_ERR(rq->page_pool)) {
 			err = PTR_ERR(rq->page_pool);
 			rq->page_pool = NULL;
-			goto err_free_shampo;
+			goto err_free_by_rq_type;
 		}
 		if (xdp_rxq_info_is_reg(&rq->xdp_rxq))
 			err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
 							 MEM_TYPE_PAGE_POOL, rq->page_pool);
 	}
 	if (err)
-		goto err_free_shampo;
+		goto err_destroy_page_pool;
 
 	for (i = 0; i < wq_sz; i++) {
 		if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
@@ -677,13 +801,14 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 				mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i);
 			u32 byte_count =
 				rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
-			u64 dma_offset = mlx5e_get_mpwqe_offset(i);
+			u64 dma_offset = mul_u32_u32(i, rq->mpwqe.mtts_per_wqe) <<
+				rq->mpwqe.page_shift;
 			u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ?
 				       0 : rq->buff.headroom;
 
 			wqe->data[0].addr = cpu_to_be64(dma_offset + headroom);
 			wqe->data[0].byte_count = cpu_to_be32(byte_count);
-			wqe->data[0].lkey = rq->mkey_be;
+			wqe->data[0].lkey = rq->mpwqe.umr_mkey_be;
 		} else {
 			struct mlx5e_rx_wqe_cyc *wqe =
 				mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i);
@@ -721,19 +846,21 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 
 	return 0;
 
-err_free_shampo:
-	mlx5e_rq_free_shampo(rq);
+err_destroy_page_pool:
+	page_pool_destroy(rq->page_pool);
 err_free_by_rq_type:
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
+		mlx5e_rq_free_shampo(rq);
+err_free_mpwqe_info:
 		kvfree(rq->mpwqe.info);
 err_rq_mkey:
-		mlx5_core_destroy_mkey(mdev, rq->umr_mkey);
+		mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be));
 err_rq_drop_page:
 		mlx5e_free_mpwqe_rq_drop_page(rq);
 		break;
 	default: /* MLX5_WQ_TYPE_CYCLIC */
-		mlx5e_free_di_list(rq);
+		mlx5e_free_au_list(rq);
 err_rq_frags:
 		kvfree(rq->wqe.frags);
 	}
@@ -761,24 +888,22 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq)
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
 		kvfree(rq->mpwqe.info);
-		mlx5_core_destroy_mkey(rq->mdev, rq->umr_mkey);
+		mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be));
 		mlx5e_free_mpwqe_rq_drop_page(rq);
 		mlx5e_rq_free_shampo(rq);
 		break;
 	default: /* MLX5_WQ_TYPE_CYCLIC */
 		kvfree(rq->wqe.frags);
-		mlx5e_free_di_list(rq);
+		mlx5e_free_au_list(rq);
 	}
 
 	for (i = rq->page_cache.head; i != rq->page_cache.tail;
 	     i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
-		struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
-
 		/* With AF_XDP, page_cache is not used, so this loop is not
 		 * entered, and it's safe to call mlx5e_page_release_dynamic
 		 * directly.
 		 */
-		mlx5e_page_release_dynamic(rq, dma_info->page, false);
+		mlx5e_page_release_dynamic(rq, rq->page_cache.page_cache[i], false);
 	}
 
 	xdp_rxq_info_unreg(&rq->xdp_rxq);
@@ -833,7 +958,7 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
 	return err;
 }
 
-int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
+static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
 {
 	struct mlx5_core_dev *mdev = rq->mdev;
 
@@ -862,6 +987,32 @@ int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
 	return err;
 }
 
+static int mlx5e_rq_to_ready(struct mlx5e_rq *rq, int curr_state)
+{
+	struct net_device *dev = rq->netdev;
+	int err;
+
+	err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST);
+	if (err) {
+		netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn);
+		return err;
+	}
+	err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
+	if (err) {
+		netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn);
+		return err;
+	}
+
+	return 0;
+}
+
+int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state)
+{
+	mlx5e_free_rx_descs(rq);
+
+	return mlx5e_rq_to_ready(rq, curr_state);
+}
+
 static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable)
 {
 	struct mlx5_core_dev *mdev = rq->mdev;
@@ -1154,9 +1305,9 @@ static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
 		is_redirect ?
 			&c->priv->channel_stats[c->ix]->xdpsq :
 			&c->priv->channel_stats[c->ix]->rq_xdpsq;
-	sq->max_sq_wqebbs = mlx5e_get_max_sq_wqebbs(mdev);
-	sq->stop_room = MLX5E_STOP_ROOM(sq->max_sq_wqebbs);
-	sq->max_sq_mpw_wqebbs = mlx5e_get_sw_max_sq_mpw_wqebbs(sq->max_sq_wqebbs);
+	sq->stop_room = param->is_mpw ? mlx5e_stop_room_for_mpwqe(mdev) :
+					mlx5e_stop_room_for_max_wqe(mdev);
+	sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
 
 	param->wq.db_numa_node = cpu_to_node(c->cpu);
 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
@@ -1231,7 +1382,6 @@ static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
 	sq->channel   = c;
 	sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
 	sq->reserved_room = param->stop_room;
-	sq->max_sq_wqebbs = mlx5e_get_max_sq_wqebbs(mdev);
 
 	param->wq.db_numa_node = cpu_to_node(c->cpu);
 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
@@ -1317,8 +1467,7 @@ static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
 	sq->uar_map   = mdev->mlx5e_res.hw_objs.bfreg.map;
 	sq->min_inline_mode = params->tx_min_inline_mode;
 	sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
-	sq->max_sq_wqebbs = mlx5e_get_max_sq_wqebbs(mdev);
-	sq->max_sq_mpw_wqebbs = mlx5e_get_sw_max_sq_mpw_wqebbs(sq->max_sq_wqebbs);
+	sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
 	INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work);
 	if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert))
 		set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state);
@@ -2280,7 +2429,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
 	c->aff_mask = irq_get_effective_affinity_mask(irq);
 	c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix);
 
-	netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
+	netif_napi_add(netdev, &c->napi, mlx5e_napi_poll);
 
 	err = mlx5e_open_queues(c, params, cparam);
 	if (unlikely(err))
@@ -2318,10 +2467,11 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c)
 		mlx5e_activate_txqsq(&c->sq[tc]);
 	mlx5e_activate_icosq(&c->icosq);
 	mlx5e_activate_icosq(&c->async_icosq);
-	mlx5e_activate_rq(&c->rq);
 
 	if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
 		mlx5e_activate_xsk(c);
+	else
+		mlx5e_activate_rq(&c->rq);
 
 	mlx5e_trigger_napi_icosq(c);
 }
@@ -2332,8 +2482,9 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
 
 	if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
 		mlx5e_deactivate_xsk(c);
+	else
+		mlx5e_deactivate_rq(&c->rq);
 
-	mlx5e_deactivate_rq(&c->rq);
 	mlx5e_deactivate_icosq(&c->async_icosq);
 	mlx5e_deactivate_icosq(&c->icosq);
 	for (tc = 0; tc < c->num_tc; tc++)
@@ -2425,8 +2576,6 @@ static void mlx5e_activate_channels(struct mlx5e_channels *chs)
 		mlx5e_ptp_activate_channel(chs->ptp);
 }
 
-#define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */
-
 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
 {
 	int err = 0;
@@ -2434,8 +2583,12 @@ static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
 
 	for (i = 0; i < chs->num; i++) {
 		int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;
+		struct mlx5e_channel *c = chs->c[i];
 
-		err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout);
+		if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
+			continue;
+
+		err |= mlx5e_wait_for_min_rx_wqes(&c->rq, timeout);
 
 		/* Don't wait on the XSK RQ, because the newer xdpsock sample
 		 * doesn't provide any Fill Ring entries at the setup stage.
@@ -2600,7 +2753,7 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv)
 	struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq;
 	struct net_device *netdev = priv->netdev;
 	int old_num_txqs, old_ntc;
-	int num_rxqs, nch, ntc;
+	int nch, ntc;
 	int err;
 	int i;
 
@@ -2611,7 +2764,6 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv)
 
 	nch = priv->channels.params.num_channels;
 	ntc = priv->channels.params.mqprio.num_tc;
-	num_rxqs = nch * priv->profile->rq_groups;
 	tc_to_txq = priv->channels.params.mqprio.tc_to_txq;
 
 	err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq);
@@ -2620,7 +2772,7 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv)
 	err = mlx5e_update_tx_netdev_queues(priv);
 	if (err)
 		goto err_tcs;
-	err = netif_set_real_num_rx_queues(netdev, num_rxqs);
+	err = netif_set_real_num_rx_queues(netdev, nch);
 	if (err) {
 		netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err);
 		goto err_txqs;
@@ -2738,7 +2890,7 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
 	netif_tx_start_all_queues(priv->netdev);
 
 	if (mlx5e_is_vport_rep(priv))
-		mlx5e_add_sqs_fwd_rules(priv);
+		mlx5e_rep_activate_channels(priv);
 
 	mlx5e_wait_channels_min_rx_wqes(&priv->channels);
 
@@ -2752,7 +2904,7 @@ void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
 		mlx5e_rx_res_channels_deactivate(priv->rx_res);
 
 	if (mlx5e_is_vport_rep(priv))
-		mlx5e_remove_sqs_fwd_rules(priv);
+		mlx5e_rep_deactivate_channels(priv);
 
 	/* The results of ndo_select_queue are unreliable, while netdev config
 	 * is being changed (real_num_tx_queues, num_tc). Stop all queues to
@@ -3547,7 +3699,8 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
 	stats->rx_length_errors =
 		PPORT_802_3_GET(pstats, a_in_range_length_errors) +
 		PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
-		PPORT_802_3_GET(pstats, a_frame_too_long_errors);
+		PPORT_802_3_GET(pstats, a_frame_too_long_errors) +
+		VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small);
 	stats->rx_crc_errors =
 		PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
 	stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
@@ -3669,9 +3822,11 @@ static int set_feature_cvlan_filter(struct net_device *netdev, bool enable)
 	struct mlx5e_priv *priv = netdev_priv(netdev);
 
 	if (enable)
-		mlx5e_enable_cvlan_filter(priv);
+		mlx5e_enable_cvlan_filter(priv->fs,
+					  !!(priv->netdev->flags & IFF_PROMISC));
 	else
-		mlx5e_disable_cvlan_filter(priv);
+		mlx5e_disable_cvlan_filter(priv->fs,
+					   !!(priv->netdev->flags & IFF_PROMISC));
 
 	return 0;
 }
@@ -3780,7 +3935,7 @@ static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
 
 	mutex_lock(&priv->state_lock);
 
-	priv->fs->vlan_strip_disable = !enable;
+	mlx5e_fs_set_vlan_strip_disable(priv->fs, !enable);
 	priv->channels.params.vlan_strip_disable = !enable;
 
 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
@@ -3788,7 +3943,7 @@ static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
 
 	err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
 	if (err) {
-		priv->fs->vlan_strip_disable = enable;
+		mlx5e_fs_set_vlan_strip_disable(priv->fs, enable);
 		priv->channels.params.vlan_strip_disable = enable;
 	}
 unlock:
@@ -3826,9 +3981,9 @@ static int set_feature_arfs(struct net_device *netdev, bool enable)
 	int err;
 
 	if (enable)
-		err = mlx5e_arfs_enable(priv);
+		err = mlx5e_arfs_enable(priv->fs);
 	else
-		err = mlx5e_arfs_disable(priv);
+		err = mlx5e_arfs_disable(priv->fs);
 
 	return err;
 }
@@ -3912,12 +4067,14 @@ static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
 					    netdev_features_t features)
 {
 	struct mlx5e_priv *priv = netdev_priv(netdev);
+	struct mlx5e_vlan_table *vlan;
 	struct mlx5e_params *params;
 
+	vlan = mlx5e_fs_get_vlan(priv->fs);
 	mutex_lock(&priv->state_lock);
 	params = &priv->channels.params;
-	if (!priv->fs->vlan ||
-	    !bitmap_empty(mlx5e_vlan_get_active_svlans(priv->fs->vlan), VLAN_N_VID)) {
+	if (!vlan ||
+	    !bitmap_empty(mlx5e_vlan_get_active_svlans(vlan), VLAN_N_VID)) {
 		/* HW strips the outer C-tag header, this is a problem
 		 * for S-tag traffic.
 		 */
@@ -4006,7 +4163,7 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
 			 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE.
 			 */
 			max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr);
-			max_mtu_page = mlx5e_xdp_max_mtu(new_params, &xsk);
+			max_mtu_page = MLX5E_HW2SW_MTU(new_params, SKB_MAX_HEAD(0));
 			max_mtu = min(max_mtu_frame, max_mtu_page);
 
 			netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u. Try MTU <= %d\n",
@@ -4018,14 +4175,16 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
 	return true;
 }
 
-static bool mlx5e_params_validate_xdp(struct net_device *netdev, struct mlx5e_params *params)
+static bool mlx5e_params_validate_xdp(struct net_device *netdev,
+				      struct mlx5_core_dev *mdev,
+				      struct mlx5e_params *params)
 {
 	bool is_linear;
 
 	/* No XSK params: AF_XDP can't be enabled yet at the point of setting
 	 * the XDP program.
 	 */
-	is_linear = mlx5e_rx_is_linear_skb(params, NULL);
+	is_linear = mlx5e_rx_is_linear_skb(mdev, params, NULL);
 
 	if (!is_linear && params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) {
 		netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n",
@@ -4062,7 +4221,8 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
 	if (err)
 		goto out;
 
-	if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, &new_params)) {
+	if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev,
+							      &new_params)) {
 		err = -EINVAL;
 		goto out;
 	}
@@ -4077,19 +4237,21 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
 	if (params->packet_merge.type == MLX5E_PACKET_MERGE_LRO)
 		reset = false;
 
-	if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
+	if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
+	    params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) {
 		bool is_linear_old = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, params, NULL);
 		bool is_linear_new = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev,
 								  &new_params, NULL);
-		u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params, NULL);
-		u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_params, NULL);
+		u8 sz_old = mlx5e_mpwqe_get_log_rq_size(priv->mdev, params, NULL);
+		u8 sz_new = mlx5e_mpwqe_get_log_rq_size(priv->mdev, &new_params, NULL);
 
 		/* Always reset in linear mode - hw_mtu is used in data path.
 		 * Check that the mode was non-linear and didn't change.
 		 * If XSK is active, XSK RQs are linear.
+		 * Reset if the RQ size changed, even if it's non-linear.
 		 */
 		if (!is_linear_old && !is_linear_new && !priv->xsk.refcnt &&
-		    ppw_old == ppw_new)
+		    sz_old == sz_new)
 			reset = false;
 	}
 
@@ -4539,7 +4701,7 @@ static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog)
 	new_params = priv->channels.params;
 	new_params.xdp_prog = prog;
 
-	if (!mlx5e_params_validate_xdp(netdev, &new_params))
+	if (!mlx5e_params_validate_xdp(netdev, priv->mdev, &new_params))
 		return -EINVAL;
 
 	return 0;
@@ -4577,8 +4739,20 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
 
 	new_params = priv->channels.params;
 	new_params.xdp_prog = prog;
-	if (reset)
-		mlx5e_set_rq_type(priv->mdev, &new_params);
+
+	/* XDP affects striding RQ parameters. Block XDP if striding RQ won't be
+	 * supported with the new parameters: if PAGE_SIZE is bigger than
+	 * MLX5_MPWQE_LOG_STRIDE_SZ_MAX, striding RQ can't be used, even though
+	 * the MTU is small enough for the linear mode, because XDP uses strides
+	 * of PAGE_SIZE on regular RQs.
+	 */
+	if (reset && MLX5E_GET_PFLAG(&new_params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
+		/* Checking for regular RQs here; XSK RQs were checked on XSK bind. */
+		err = mlx5e_mpwrq_validate_regular(priv->mdev, &new_params);
+		if (err)
+			goto unlock;
+	}
+
 	old_prog = priv->channels.params.xdp_prog;
 
 	err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset);
@@ -4898,7 +5072,8 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 	if (!!MLX5_CAP_ETH(mdev, lro_cap) &&
 	    !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) &&
 	    !MLX5_CAP_ETH(mdev, tunnel_lro_gre) &&
-	    mlx5e_check_fragmented_striding_rq_cap(mdev))
+	    mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT,
+						   MLX5E_MPWRQ_UMR_MODE_ALIGNED))
 		netdev->vlan_features    |= NETIF_F_LRO;
 
 	netdev->hw_features       = netdev->vlan_features;
@@ -4986,6 +5161,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 
 	netif_set_tso_max_size(netdev, GSO_MAX_SIZE);
 	mlx5e_set_netdev_dev_addr(netdev);
+	mlx5e_macsec_build_netdev(priv);
 	mlx5e_ipsec_build_netdev(priv);
 	mlx5e_ktls_build_netdev(priv);
 }
@@ -5087,7 +5263,7 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
 		goto err_destroy_q_counters;
 	}
 
-	features = MLX5E_RX_RES_FEATURE_XSK | MLX5E_RX_RES_FEATURE_PTP;
+	features = MLX5E_RX_RES_FEATURE_PTP;
 	if (priv->channels.params.tunneled_offload_en)
 		features |= MLX5E_RX_RES_FEATURE_INNER_FT;
 	err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, features,
@@ -5097,7 +5273,8 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
 	if (err)
 		goto err_close_drop_rq;
 
-	err = mlx5e_create_flow_steering(priv);
+	err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile,
+					 priv->netdev);
 	if (err) {
 		mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
 		goto err_destroy_rx_res;
@@ -5120,7 +5297,8 @@ static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
 err_tc_nic_cleanup:
 	mlx5e_tc_nic_cleanup(priv);
 err_destroy_flow_steering:
-	mlx5e_destroy_flow_steering(priv);
+	mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE),
+				    priv->profile);
 err_destroy_rx_res:
 	mlx5e_rx_res_destroy(priv->rx_res);
 err_close_drop_rq:
@@ -5136,7 +5314,8 @@ static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
 {
 	mlx5e_accel_cleanup_rx(priv);
 	mlx5e_tc_nic_cleanup(priv);
-	mlx5e_destroy_flow_steering(priv);
+	mlx5e_destroy_flow_steering(priv->fs, !!(priv->netdev->hw_features & NETIF_F_NTUPLE),
+				    priv->profile);
 	mlx5e_rx_res_destroy(priv->rx_res);
 	mlx5e_close_drop_rq(&priv->drop_rq);
 	mlx5e_destroy_q_counters(priv);
@@ -5188,9 +5367,14 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv)
 {
 	struct net_device *netdev = priv->netdev;
 	struct mlx5_core_dev *mdev = priv->mdev;
+	int err;
 
 	mlx5e_fs_init_l2_addr(priv->fs, netdev);
 
+	err = mlx5e_macsec_init(priv);
+	if (err)
+		mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err);
+
 	/* Marking the link as currently not needed by the Driver */
 	if (!netif_running(netdev))
 		mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN);
@@ -5248,6 +5432,7 @@ static void mlx5e_nic_disable(struct mlx5e_priv *priv)
 	mlx5e_disable_async_events(priv);
 	mlx5_lag_remove_netdev(mdev, priv->netdev);
 	mlx5_vxlan_reset_to_default(mdev->vxlan);
+	mlx5e_macsec_cleanup(priv);
 }
 
 int mlx5e_update_nic_rx(struct mlx5e_priv *priv)
@@ -5269,7 +5454,6 @@ static const struct mlx5e_profile mlx5e_nic_profile = {
 	.update_carrier	   = mlx5e_update_carrier,
 	.rx_handlers       = &mlx5e_rx_handlers_nic,
 	.max_tc		   = MLX5E_MAX_NUM_TC,
-	.rq_groups	   = MLX5E_NUM_RQ_GROUPS(XSK),
 	.stats_grps	   = mlx5e_nic_stats_grps,
 	.stats_grps_num	   = mlx5e_nic_stats_grps_num,
 	.features          = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) |
@@ -5302,8 +5486,7 @@ mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev,
 	max_nch = mlx5e_profile_max_num_channels(mdev, profile);
 
 	/* netdev rx queues */
-	tmp = netdev->num_rx_queues / max_t(u8, profile->rq_groups, 1);
-	max_nch = min_t(unsigned int, max_nch, tmp);
+	max_nch = min_t(unsigned int, max_nch, netdev->num_rx_queues);
 
 	/* netdev tx queues */
 	tmp = netdev->num_tx_queues;
@@ -5447,11 +5630,7 @@ static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev,
 static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev,
 					   const struct mlx5e_profile *profile)
 {
-	unsigned int nch;
-
-	nch = mlx5e_profile_max_num_channels(mdev, profile);
-
-	return nch * profile->rq_groups;
+	return mlx5e_profile_max_num_channels(mdev, profile);
 }
 
 struct net_device *
@@ -5512,7 +5691,8 @@ int mlx5e_attach_netdev(struct mlx5e_priv *priv)
 
 	clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
 	if (priv->fs)
-		priv->fs->state_destroy = !test_bit(MLX5E_STATE_DESTROYING, &priv->state);
+		mlx5e_fs_set_state_destroy(priv->fs,
+					   !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
 
 	/* max number of channels may have changed */
 	max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile);
@@ -5573,7 +5753,8 @@ out:
 	mlx5e_reset_channels(priv->netdev);
 	set_bit(MLX5E_STATE_DESTROYING, &priv->state);
 	if (priv->fs)
-		priv->fs->state_destroy = !test_bit(MLX5E_STATE_DESTROYING, &priv->state);
+		mlx5e_fs_set_state_destroy(priv->fs,
+					   !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
 	cancel_work_sync(&priv->update_stats_work);
 	return err;
 }
@@ -5584,7 +5765,8 @@ void mlx5e_detach_netdev(struct mlx5e_priv *priv)
 
 	set_bit(MLX5E_STATE_DESTROYING, &priv->state);
 	if (priv->fs)
-		priv->fs->state_destroy = !test_bit(MLX5E_STATE_DESTROYING, &priv->state);
+		mlx5e_fs_set_state_destroy(priv->fs,
+					   !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
 
 	if (profile->disable)
 		profile->disable(priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 759f7d3c2cfd..794cd8dfe9c9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -56,6 +56,7 @@
 #include "en_accel/ipsec.h"
 #include "en/tc/int_port.h"
 #include "en/ptp.h"
+#include "en/fs_ethtool.h"
 
 #define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \
 	max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE)
@@ -69,7 +70,7 @@ static void mlx5e_rep_get_drvinfo(struct net_device *dev,
 	struct mlx5e_priv *priv = netdev_priv(dev);
 	struct mlx5_core_dev *mdev = priv->mdev;
 
-	strlcpy(drvinfo->driver, mlx5e_rep_driver_name,
+	strscpy(drvinfo->driver, mlx5e_rep_driver_name,
 		sizeof(drvinfo->driver));
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
 		 "%d.%d.%04d (%.16s)",
@@ -397,7 +398,8 @@ out_err:
 	return err;
 }
 
-int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv)
+static int
+mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv)
 {
 	int sqs_per_channel = mlx5e_get_dcb_num_tc(&priv->channels.params);
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
@@ -451,7 +453,8 @@ out:
 	return err;
 }
 
-void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv)
+static void
+mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv)
 {
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	struct mlx5e_rep_priv *rpriv = priv->ppriv;
@@ -460,6 +463,49 @@ void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv)
 	mlx5e_sqs2vport_stop(esw, rep);
 }
 
+static int
+mlx5e_rep_add_meta_tunnel_rule(struct mlx5e_priv *priv)
+{
+	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
+	struct mlx5e_rep_priv *rpriv = priv->ppriv;
+	struct mlx5_eswitch_rep *rep = rpriv->rep;
+	struct mlx5_flow_handle *flow_rule;
+	struct mlx5_flow_group *g;
+
+	g = esw->fdb_table.offloads.send_to_vport_meta_grp;
+	if (!g)
+		return 0;
+
+	flow_rule = mlx5_eswitch_add_send_to_vport_meta_rule(esw, rep->vport);
+	if (IS_ERR(flow_rule))
+		return PTR_ERR(flow_rule);
+
+	rpriv->send_to_vport_meta_rule = flow_rule;
+
+	return 0;
+}
+
+static void
+mlx5e_rep_del_meta_tunnel_rule(struct mlx5e_priv *priv)
+{
+	struct mlx5e_rep_priv *rpriv = priv->ppriv;
+
+	if (rpriv->send_to_vport_meta_rule)
+		mlx5_eswitch_del_send_to_vport_meta_rule(rpriv->send_to_vport_meta_rule);
+}
+
+void mlx5e_rep_activate_channels(struct mlx5e_priv *priv)
+{
+	mlx5e_add_sqs_fwd_rules(priv);
+	mlx5e_rep_add_meta_tunnel_rule(priv);
+}
+
+void mlx5e_rep_deactivate_channels(struct mlx5e_priv *priv)
+{
+	mlx5e_rep_del_meta_tunnel_rule(priv);
+	mlx5e_remove_sqs_fwd_rules(priv);
+}
+
 static int mlx5e_rep_open(struct net_device *dev)
 {
 	struct mlx5e_priv *priv = netdev_priv(dev);
@@ -747,19 +793,20 @@ static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv)
 	struct ttc_params ttc_params = {};
 	int err;
 
-	priv->fs->ns = mlx5_get_flow_namespace(priv->mdev,
-					       MLX5_FLOW_NAMESPACE_KERNEL);
+	mlx5e_fs_set_ns(priv->fs,
+			mlx5_get_flow_namespace(priv->mdev,
+						MLX5_FLOW_NAMESPACE_KERNEL), false);
 
 	/* The inner_ttc in the ttc params is intentionally not set */
-	mlx5e_set_ttc_params(priv, &ttc_params, false);
+	mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false);
 
 	if (rep->vport != MLX5_VPORT_UPLINK)
 		/* To give uplik rep TTC a lower level for chaining from root ft */
 		ttc_params.ft_attr.level = MLX5E_TTC_FT_LEVEL + 1;
 
-	priv->fs->ttc = mlx5_create_ttc_table(priv->mdev, &ttc_params);
-	if (IS_ERR(priv->fs->ttc)) {
-		err = PTR_ERR(priv->fs->ttc);
+	mlx5e_fs_set_ttc(priv->fs, mlx5_create_ttc_table(priv->mdev, &ttc_params), false);
+	if (IS_ERR(mlx5e_fs_get_ttc(priv->fs, false))) {
+		err = PTR_ERR(mlx5e_fs_get_ttc(priv->fs, false));
 		netdev_err(priv->netdev, "Failed to create rep ttc table, err=%d\n",
 			   err);
 		return err;
@@ -779,7 +826,7 @@ static int mlx5e_create_rep_root_ft(struct mlx5e_priv *priv)
 		/* non uplik reps will skip any bypass tables and go directly to
 		 * their own ttc
 		 */
-		rpriv->root_ft = mlx5_get_ttc_flow_table(priv->fs->ttc);
+		rpriv->root_ft = mlx5_get_ttc_flow_table(mlx5e_fs_get_ttc(priv->fs, false));
 		return 0;
 	}
 
@@ -887,14 +934,14 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv)
 	if (err)
 		goto err_destroy_root_ft;
 
-	mlx5e_ethtool_init_steering(priv);
+	mlx5e_ethtool_init_steering(priv->fs);
 
 	return 0;
 
 err_destroy_root_ft:
 	mlx5e_destroy_rep_root_ft(priv);
 err_destroy_ttc_table:
-	mlx5_destroy_ttc_table(priv->fs->ttc);
+	mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv->fs, false));
 err_destroy_rx_res:
 	mlx5e_rx_res_destroy(priv->rx_res);
 err_close_drop_rq:
@@ -908,10 +955,10 @@ err_free_fs:
 
 static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv)
 {
-	mlx5e_ethtool_cleanup_steering(priv);
+	mlx5e_ethtool_cleanup_steering(priv->fs);
 	rep_vport_rx_rule_destroy(priv);
 	mlx5e_destroy_rep_root_ft(priv);
-	mlx5_destroy_ttc_table(priv->fs->ttc);
+	mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv->fs, false));
 	mlx5e_rx_res_destroy(priv->rx_res);
 	mlx5e_close_drop_rq(&priv->drop_rq);
 	mlx5e_rx_res_free(priv->rx_res);
@@ -1177,7 +1224,6 @@ static const struct mlx5e_profile mlx5e_rep_profile = {
 	.update_stats           = mlx5e_stats_update_ndo_stats,
 	.rx_handlers            = &mlx5e_rx_handlers_rep,
 	.max_tc			= 1,
-	.rq_groups		= MLX5E_NUM_RQ_GROUPS(REGULAR),
 	.stats_grps		= mlx5e_rep_stats_grps,
 	.stats_grps_num		= mlx5e_rep_stats_grps_num,
 	.max_nch_limit		= mlx5e_rep_max_nch_limit,
@@ -1197,8 +1243,6 @@ static const struct mlx5e_profile mlx5e_uplink_rep_profile = {
 	.update_carrier	        = mlx5e_update_carrier,
 	.rx_handlers            = &mlx5e_rx_handlers_rep,
 	.max_tc			= MLX5E_MAX_NUM_TC,
-	/* XSK is needed so we can replace profile with NIC netdev */
-	.rq_groups		= MLX5E_NUM_RQ_GROUPS(XSK),
 	.stats_grps		= mlx5e_ul_rep_stats_grps,
 	.stats_grps_num		= mlx5e_ul_rep_stats_grps_num,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
index dec183ccd4ac..b4e691760da9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h
@@ -111,6 +111,7 @@ struct mlx5e_rep_priv {
 	struct list_head       vport_sqs_list;
 	struct mlx5_rep_uplink_priv uplink_priv; /* valid for uplink rep */
 	struct rtnl_link_stats64 prev_vf_vport_stats;
+	struct mlx5_flow_handle *send_to_vport_meta_rule;
 	struct rhashtable tc_ht;
 };
 
@@ -241,8 +242,8 @@ int mlx5e_rep_get_offload_stats(int attr_id, const struct net_device *dev,
 				void *sp);
 
 bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv);
-int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv);
-void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv);
+void mlx5e_rep_activate_channels(struct mlx5e_priv *priv);
+void mlx5e_rep_deactivate_channels(struct mlx5e_priv *priv);
 
 void mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv *priv);
 
@@ -256,8 +257,8 @@ static inline bool mlx5e_eswitch_rep(const struct net_device *netdev)
 
 #else /* CONFIG_MLX5_ESWITCH */
 static inline bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv) { return false; }
-static inline int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv) { return 0; }
-static inline void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv) {}
+static inline void mlx5e_rep_activate_channels(struct mlx5e_priv *priv) {}
+static inline void mlx5e_rep_deactivate_channels(struct mlx5e_priv *priv) {}
 static inline int mlx5e_rep_init(void) { return 0; };
 static inline void mlx5e_rep_cleanup(void) {};
 static inline bool mlx5e_rep_has_offload_stats(const struct net_device *dev,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 24de37b79f5a..58084650151f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -41,6 +41,7 @@
 #include <net/gro.h>
 #include <net/udp.h>
 #include <net/tcp.h>
+#include <net/xdp_sock_drv.h>
 #include "en.h"
 #include "en/txrx.h"
 #include "en_tc.h"
@@ -49,6 +50,7 @@
 #include "en/rep/tc.h"
 #include "ipoib/ipoib.h"
 #include "en_accel/ipsec.h"
+#include "en_accel/macsec.h"
 #include "en_accel/ipsec_rxtx.h"
 #include "en_accel/ktls_txrx.h"
 #include "en/xdp.h"
@@ -237,69 +239,61 @@ static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, struct page *page)
 		return false;
 	}
 
-	cache->page_cache[cache->tail].page = page;
-	cache->page_cache[cache->tail].addr = page_pool_get_dma_addr(page);
+	cache->page_cache[cache->tail] = page;
 	cache->tail = tail_next;
 	return true;
 }
 
-static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq,
-				      struct mlx5e_dma_info *dma_info)
+static inline bool mlx5e_rx_cache_get(struct mlx5e_rq *rq, union mlx5e_alloc_unit *au)
 {
 	struct mlx5e_page_cache *cache = &rq->page_cache;
 	struct mlx5e_rq_stats *stats = rq->stats;
+	dma_addr_t addr;
 
 	if (unlikely(cache->head == cache->tail)) {
 		stats->cache_empty++;
 		return false;
 	}
 
-	if (page_ref_count(cache->page_cache[cache->head].page) != 1) {
+	if (page_ref_count(cache->page_cache[cache->head]) != 1) {
 		stats->cache_busy++;
 		return false;
 	}
 
-	*dma_info = cache->page_cache[cache->head];
+	au->page = cache->page_cache[cache->head];
 	cache->head = (cache->head + 1) & (MLX5E_CACHE_SIZE - 1);
 	stats->cache_reuse++;
 
-	dma_sync_single_for_device(rq->pdev, dma_info->addr,
-				   PAGE_SIZE,
-				   DMA_FROM_DEVICE);
+	addr = page_pool_get_dma_addr(au->page);
+	/* Non-XSK always uses PAGE_SIZE. */
+	dma_sync_single_for_device(rq->pdev, addr, PAGE_SIZE, DMA_FROM_DEVICE);
 	return true;
 }
 
-static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq,
-					struct mlx5e_dma_info *dma_info)
+static inline int mlx5e_page_alloc_pool(struct mlx5e_rq *rq, union mlx5e_alloc_unit *au)
 {
-	if (mlx5e_rx_cache_get(rq, dma_info))
+	dma_addr_t addr;
+
+	if (mlx5e_rx_cache_get(rq, au))
 		return 0;
 
-	dma_info->page = page_pool_dev_alloc_pages(rq->page_pool);
-	if (unlikely(!dma_info->page))
+	au->page = page_pool_dev_alloc_pages(rq->page_pool);
+	if (unlikely(!au->page))
 		return -ENOMEM;
 
-	dma_info->addr = dma_map_page_attrs(rq->pdev, dma_info->page, 0, PAGE_SIZE,
-					    rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC);
-	if (unlikely(dma_mapping_error(rq->pdev, dma_info->addr))) {
-		page_pool_recycle_direct(rq->page_pool, dma_info->page);
-		dma_info->page = NULL;
+	/* Non-XSK always uses PAGE_SIZE. */
+	addr = dma_map_page_attrs(rq->pdev, au->page, 0, PAGE_SIZE,
+				  rq->buff.map_dir, DMA_ATTR_SKIP_CPU_SYNC);
+	if (unlikely(dma_mapping_error(rq->pdev, addr))) {
+		page_pool_recycle_direct(rq->page_pool, au->page);
+		au->page = NULL;
 		return -ENOMEM;
 	}
-	page_pool_set_dma_addr(dma_info->page, dma_info->addr);
+	page_pool_set_dma_addr(au->page, addr);
 
 	return 0;
 }
 
-static inline int mlx5e_page_alloc(struct mlx5e_rq *rq,
-				   struct mlx5e_dma_info *dma_info)
-{
-	if (rq->xsk_pool)
-		return mlx5e_xsk_page_alloc_pool(rq, dma_info);
-	else
-		return mlx5e_page_alloc_pool(rq, dma_info);
-}
-
 void mlx5e_page_dma_unmap(struct mlx5e_rq *rq, struct page *page)
 {
 	dma_addr_t dma_addr = page_pool_get_dma_addr(page);
@@ -324,32 +318,18 @@ void mlx5e_page_release_dynamic(struct mlx5e_rq *rq, struct page *page, bool rec
 	}
 }
 
-static inline void mlx5e_page_release(struct mlx5e_rq *rq,
-				      struct mlx5e_dma_info *dma_info,
-				      bool recycle)
-{
-	if (rq->xsk_pool)
-		/* The `recycle` parameter is ignored, and the page is always
-		 * put into the Reuse Ring, because there is no way to return
-		 * the page to the userspace when the interface goes down.
-		 */
-		xsk_buff_free(dma_info->xsk);
-	else
-		mlx5e_page_release_dynamic(rq, dma_info->page, recycle);
-}
-
 static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq,
 				    struct mlx5e_wqe_frag_info *frag)
 {
 	int err = 0;
 
 	if (!frag->offset)
-		/* On first frag (offset == 0), replenish page (dma_info actually).
-		 * Other frags that point to the same dma_info (with a different
+		/* On first frag (offset == 0), replenish page (alloc_unit actually).
+		 * Other frags that point to the same alloc_unit (with a different
 		 * offset) should just use the new one without replenishing again
 		 * by themselves.
 		 */
-		err = mlx5e_page_alloc(rq, frag->di);
+		err = mlx5e_page_alloc_pool(rq, frag->au);
 
 	return err;
 }
@@ -359,7 +339,7 @@ static inline void mlx5e_put_rx_frag(struct mlx5e_rq *rq,
 				     bool recycle)
 {
 	if (frag->last_in_page)
-		mlx5e_page_release(rq, frag->di, recycle);
+		mlx5e_page_release_dynamic(rq, frag->au->page, recycle);
 }
 
 static inline struct mlx5e_wqe_frag_info *get_frag(struct mlx5e_rq *rq, u16 ix)
@@ -375,6 +355,7 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe,
 	int i;
 
 	for (i = 0; i < rq->wqe.info.num_frags; i++, frag++) {
+		dma_addr_t addr;
 		u16 headroom;
 
 		err = mlx5e_get_rx_frag(rq, frag);
@@ -382,8 +363,8 @@ static int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe_cyc *wqe,
 			goto free_frags;
 
 		headroom = i == 0 ? rq->buff.headroom : 0;
-		wqe->data[i].addr = cpu_to_be64(frag->di->addr +
-						frag->offset + headroom);
+		addr = page_pool_get_dma_addr(frag->au->page);
+		wqe->data[i].addr = cpu_to_be64(addr + frag->offset + headroom);
 	}
 
 	return 0;
@@ -401,6 +382,15 @@ static inline void mlx5e_free_rx_wqe(struct mlx5e_rq *rq,
 {
 	int i;
 
+	if (rq->xsk_pool) {
+		/* The `recycle` parameter is ignored, and the page is always
+		 * put into the Reuse Ring, because there is no way to return
+		 * the page to the userspace when the interface goes down.
+		 */
+		xsk_buff_free(wi->au->xsk);
+		return;
+	}
+
 	for (i = 0; i < rq->wqe.info.num_frags; i++, wi++)
 		mlx5e_put_rx_frag(rq, wi, recycle);
 }
@@ -412,84 +402,76 @@ static void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix)
 	mlx5e_free_rx_wqe(rq, wi, false);
 }
 
-static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, u8 wqe_bulk)
+static int mlx5e_alloc_rx_wqes(struct mlx5e_rq *rq, u16 ix, int wqe_bulk)
 {
 	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
-	int err;
 	int i;
 
-	if (rq->xsk_pool) {
-		int pages_desired = wqe_bulk << rq->wqe.info.log_num_frags;
-
-		/* Check in advance that we have enough frames, instead of
-		 * allocating one-by-one, failing and moving frames to the
-		 * Reuse Ring.
-		 */
-		if (unlikely(!xsk_buff_can_alloc(rq->xsk_pool, pages_desired)))
-			return -ENOMEM;
-	}
-
 	for (i = 0; i < wqe_bulk; i++) {
-		struct mlx5e_rx_wqe_cyc *wqe = mlx5_wq_cyc_get_wqe(wq, ix + i);
-
-		err = mlx5e_alloc_rx_wqe(rq, wqe, ix + i);
-		if (unlikely(err))
-			goto free_wqes;
-	}
+		int j = mlx5_wq_cyc_ctr2ix(wq, ix + i);
+		struct mlx5e_rx_wqe_cyc *wqe;
 
-	return 0;
+		wqe = mlx5_wq_cyc_get_wqe(wq, j);
 
-free_wqes:
-	while (--i >= 0)
-		mlx5e_dealloc_rx_wqe(rq, ix + i);
+		if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, j)))
+			break;
+	}
 
-	return err;
+	return i;
 }
 
 static inline void
 mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb,
-		   struct mlx5e_dma_info *di, u32 frag_offset, u32 len,
+		   union mlx5e_alloc_unit *au, u32 frag_offset, u32 len,
 		   unsigned int truesize)
 {
-	dma_sync_single_for_cpu(rq->pdev,
-				di->addr + frag_offset,
-				len, DMA_FROM_DEVICE);
-	page_ref_inc(di->page);
+	dma_addr_t addr = page_pool_get_dma_addr(au->page);
+
+	dma_sync_single_for_cpu(rq->pdev, addr + frag_offset, len, DMA_FROM_DEVICE);
+	page_ref_inc(au->page);
 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
-			di->page, frag_offset, len, truesize);
+			au->page, frag_offset, len, truesize);
 }
 
 static inline void
 mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb,
-		      struct mlx5e_dma_info *dma_info,
+		      struct page *page, dma_addr_t addr,
 		      int offset_from, int dma_offset, u32 headlen)
 {
-	const void *from = page_address(dma_info->page) + offset_from;
+	const void *from = page_address(page) + offset_from;
 	/* Aligning len to sizeof(long) optimizes memcpy performance */
 	unsigned int len = ALIGN(headlen, sizeof(long));
 
-	dma_sync_single_for_cpu(pdev, dma_info->addr + dma_offset, len,
-				DMA_FROM_DEVICE);
+	dma_sync_single_for_cpu(pdev, addr + dma_offset, len, DMA_FROM_DEVICE);
 	skb_copy_to_linear_data(skb, from, len);
 }
 
 static void
 mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi, bool recycle)
 {
+	union mlx5e_alloc_unit *alloc_units = wi->alloc_units;
 	bool no_xdp_xmit;
-	struct mlx5e_dma_info *dma_info = wi->umr.dma_info;
 	int i;
 
 	/* A common case for AF_XDP. */
-	if (bitmap_full(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE))
+	if (bitmap_full(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe))
 		return;
 
-	no_xdp_xmit = bitmap_empty(wi->xdp_xmit_bitmap,
-				   MLX5_MPWRQ_PAGES_PER_WQE);
+	no_xdp_xmit = bitmap_empty(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe);
 
-	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++)
-		if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
-			mlx5e_page_release(rq, &dma_info[i], recycle);
+	if (rq->xsk_pool) {
+		/* The `recycle` parameter is ignored, and the page is always
+		 * put into the Reuse Ring, because there is no way to return
+		 * the page to the userspace when the interface goes down.
+		 */
+		for (i = 0; i < rq->mpwqe.pages_per_wqe; i++)
+			if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
+				xsk_buff_free(alloc_units[i].xsk);
+	} else {
+		for (i = 0; i < rq->mpwqe.pages_per_wqe; i++)
+			if (no_xdp_xmit || !test_bit(i, wi->xdp_xmit_bitmap))
+				mlx5e_page_release_dynamic(rq, alloc_units[i].page, recycle);
+	}
 }
 
 static void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq, u8 n)
@@ -574,11 +556,13 @@ static int mlx5e_build_shampo_hd_umr(struct mlx5e_rq *rq,
 		header_offset = (index & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) <<
 			MLX5E_SHAMPO_LOG_MAX_HEADER_ENTRY_SIZE;
 		if (!(header_offset & (PAGE_SIZE - 1))) {
-			err = mlx5e_page_alloc(rq, dma_info);
+			union mlx5e_alloc_unit au;
+
+			err = mlx5e_page_alloc_pool(rq, &au);
 			if (unlikely(err))
 				goto err_unmap;
-			addr = dma_info->addr;
-			page = dma_info->page;
+			page = dma_info->page = au.page;
+			addr = dma_info->addr = page_pool_get_dma_addr(au.page);
 		} else {
 			dma_info->addr = addr + header_offset;
 			dma_info->page = page;
@@ -611,7 +595,7 @@ err_unmap:
 		dma_info = &shampo->info[--index];
 		if (!(i & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1))) {
 			dma_info->addr = ALIGN_DOWN(dma_info->addr, PAGE_SIZE);
-			mlx5e_page_release(rq, dma_info, true);
+			mlx5e_page_release_dynamic(rq, dma_info->page, true);
 		}
 	}
 	rq->stats->buff_alloc_err++;
@@ -659,57 +643,55 @@ static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq)
 
 static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 {
-	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
-	struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[0];
+	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix);
+	union mlx5e_alloc_unit *au = &wi->alloc_units[0];
 	struct mlx5e_icosq *sq = rq->icosq;
 	struct mlx5_wq_cyc *wq = &sq->wq;
 	struct mlx5e_umr_wqe *umr_wqe;
+	u32 offset; /* 17-bit value with MTT. */
 	u16 pi;
 	int err;
 	int i;
 
-	/* Check in advance that we have enough frames, instead of allocating
-	 * one-by-one, failing and moving frames to the Reuse Ring.
-	 */
-	if (rq->xsk_pool &&
-	    unlikely(!xsk_buff_can_alloc(rq->xsk_pool, MLX5_MPWRQ_PAGES_PER_WQE))) {
-		err = -ENOMEM;
-		goto err;
-	}
-
 	if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
 		err = mlx5e_alloc_rx_hd_mpwqe(rq);
 		if (unlikely(err))
 			goto err;
 	}
 
-	pi = mlx5e_icosq_get_next_pi(sq, MLX5E_UMR_WQEBBS);
+	pi = mlx5e_icosq_get_next_pi(sq, rq->mpwqe.umr_wqebbs);
 	umr_wqe = mlx5_wq_cyc_get_wqe(wq, pi);
-	memcpy(umr_wqe, &rq->mpwqe.umr_wqe, offsetof(struct mlx5e_umr_wqe, inline_mtts));
+	memcpy(umr_wqe, &rq->mpwqe.umr_wqe, sizeof(struct mlx5e_umr_wqe));
+
+	for (i = 0; i < rq->mpwqe.pages_per_wqe; i++, au++) {
+		dma_addr_t addr;
 
-	for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++, dma_info++) {
-		err = mlx5e_page_alloc(rq, dma_info);
+		err = mlx5e_page_alloc_pool(rq, au);
 		if (unlikely(err))
 			goto err_unmap;
-		umr_wqe->inline_mtts[i].ptag = cpu_to_be64(dma_info->addr | MLX5_EN_WR);
+		addr = page_pool_get_dma_addr(au->page);
+		umr_wqe->inline_mtts[i] = (struct mlx5_mtt) {
+			.ptag = cpu_to_be64(addr | MLX5_EN_WR),
+		};
 	}
 
-	bitmap_zero(wi->xdp_xmit_bitmap, MLX5_MPWRQ_PAGES_PER_WQE);
+	bitmap_zero(wi->xdp_xmit_bitmap, rq->mpwqe.pages_per_wqe);
 	wi->consumed_strides = 0;
 
 	umr_wqe->ctrl.opmod_idx_opcode =
 		cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
 			    MLX5_OPCODE_UMR);
-	umr_wqe->uctrl.xlt_offset =
-		cpu_to_be16(MLX5_ALIGNED_MTTS_OCTW(MLX5E_REQUIRED_MTTS(ix)));
+
+	offset = (ix * rq->mpwqe.mtts_per_wqe) * sizeof(struct mlx5_mtt) / MLX5_OCTWORD;
+	umr_wqe->uctrl.xlt_offset = cpu_to_be16(offset);
 
 	sq->db.wqe_info[pi] = (struct mlx5e_icosq_wqe_info) {
 		.wqe_type   = MLX5E_ICOSQ_WQE_UMR_RX,
-		.num_wqebbs = MLX5E_UMR_WQEBBS,
+		.num_wqebbs = rq->mpwqe.umr_wqebbs,
 		.umr.rq     = rq,
 	};
 
-	sq->pc += MLX5E_UMR_WQEBBS;
+	sq->pc += rq->mpwqe.umr_wqebbs;
 
 	sq->doorbell_cseg = &umr_wqe->ctrl;
 
@@ -717,8 +699,8 @@ static int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 
 err_unmap:
 	while (--i >= 0) {
-		dma_info--;
-		mlx5e_page_release(rq, dma_info, true);
+		au--;
+		mlx5e_page_release_dynamic(rq, au->page, true);
 	}
 
 err:
@@ -752,7 +734,7 @@ void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close
 		hd_info->addr = ALIGN_DOWN(hd_info->addr, PAGE_SIZE);
 		if (hd_info->page != deleted_page) {
 			deleted_page = hd_info->page;
-			mlx5e_page_release(rq, hd_info, false);
+			mlx5e_page_release_dynamic(rq, hd_info->page, false);
 		}
 	}
 
@@ -767,7 +749,7 @@ void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close
 
 static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 {
-	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[ix];
+	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, ix);
 	/* Don't recycle, this function is called on rq/netdev close */
 	mlx5e_free_rx_mpwqe(rq, wi, false);
 }
@@ -775,38 +757,51 @@ static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
 INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
 {
 	struct mlx5_wq_cyc *wq = &rq->wqe.wq;
-	u8 wqe_bulk;
-	int err;
+	int wqe_bulk, count;
+	bool busy = false;
+	u16 head;
 
 	if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
 		return false;
 
-	wqe_bulk = rq->wqe.info.wqe_bulk;
-
-	if (mlx5_wq_cyc_missing(wq) < wqe_bulk)
+	if (mlx5_wq_cyc_missing(wq) < rq->wqe.info.wqe_bulk)
 		return false;
 
 	if (rq->page_pool)
 		page_pool_nid_changed(rq->page_pool, numa_mem_id());
 
-	do {
-		u16 head = mlx5_wq_cyc_get_head(wq);
+	wqe_bulk = mlx5_wq_cyc_missing(wq);
+	head = mlx5_wq_cyc_get_head(wq);
 
-		err = mlx5e_alloc_rx_wqes(rq, head, wqe_bulk);
-		if (unlikely(err)) {
-			rq->stats->buff_alloc_err++;
-			break;
-		}
+	/* Don't allow any newly allocated WQEs to share the same page with old
+	 * WQEs that aren't completed yet. Stop earlier.
+	 */
+	wqe_bulk -= (head + wqe_bulk) & rq->wqe.info.wqe_index_mask;
 
-		mlx5_wq_cyc_push_n(wq, wqe_bulk);
-	} while (mlx5_wq_cyc_missing(wq) >= wqe_bulk);
+	if (!rq->xsk_pool)
+		count = mlx5e_alloc_rx_wqes(rq, head, wqe_bulk);
+	else if (likely(!rq->xsk_pool->dma_need_sync))
+		count = mlx5e_xsk_alloc_rx_wqes_batched(rq, head, wqe_bulk);
+	else
+		/* If dma_need_sync is true, it's more efficient to call
+		 * xsk_buff_alloc in a loop, rather than xsk_buff_alloc_batch,
+		 * because the latter does the same check and returns only one
+		 * frame.
+		 */
+		count = mlx5e_xsk_alloc_rx_wqes(rq, head, wqe_bulk);
+
+	mlx5_wq_cyc_push_n(wq, count);
+	if (unlikely(count != wqe_bulk)) {
+		rq->stats->buff_alloc_err++;
+		busy = true;
+	}
 
 	/* ensure wqes are visible to device before updating doorbell record */
 	dma_wmb();
 
 	mlx5_wq_cyc_update_db_record(wq);
 
-	return !!err;
+	return busy;
 }
 
 void mlx5e_free_icosq_descs(struct mlx5e_icosq *sq)
@@ -974,7 +969,8 @@ INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_mpwqes(struct mlx5e_rq *rq)
 	head = rq->mpwqe.actual_wq_head;
 	i = missing;
 	do {
-		alloc_err = mlx5e_alloc_rx_mpwqe(rq, head);
+		alloc_err = rq->xsk_pool ? mlx5e_xsk_alloc_rx_mpwqe(rq, head) :
+					   mlx5e_alloc_rx_mpwqe(rq, head);
 
 		if (unlikely(alloc_err))
 			break;
@@ -1421,6 +1417,9 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
 	if (unlikely(mlx5_ipsec_is_rx_flow(cqe)))
 		mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, cqe);
 
+	if (unlikely(mlx5e_macsec_is_rx_flow(cqe)))
+		mlx5e_macsec_offload_handle_rx_skb(netdev, skb, cqe);
+
 	if (lro_num_seg > 1) {
 		mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt);
 		skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg);
@@ -1524,19 +1523,21 @@ static struct sk_buff *
 mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
 			  u32 cqe_bcnt)
 {
-	struct mlx5e_dma_info *di = wi->di;
+	union mlx5e_alloc_unit *au = wi->au;
 	u16 rx_headroom = rq->buff.headroom;
 	struct bpf_prog *prog;
 	struct sk_buff *skb;
 	u32 metasize = 0;
 	void *va, *data;
+	dma_addr_t addr;
 	u32 frag_size;
 
-	va             = page_address(di->page) + wi->offset;
+	va             = page_address(au->page) + wi->offset;
 	data           = va + rx_headroom;
 	frag_size      = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
 
-	dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
+	addr = page_pool_get_dma_addr(au->page);
+	dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset,
 				      frag_size, DMA_FROM_DEVICE);
 	net_prefetch(data);
 
@@ -1546,7 +1547,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
 
 		net_prefetchw(va); /* xdp_frame data area */
 		mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
-		if (mlx5e_xdp_handle(rq, di->page, prog, &xdp))
+		if (mlx5e_xdp_handle(rq, au->page, prog, &xdp))
 			return NULL; /* page/packet was consumed by XDP */
 
 		rx_headroom = xdp.data - xdp.data_hard_start;
@@ -1559,7 +1560,7 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
 		return NULL;
 
 	/* queue up for recycling/reuse */
-	page_ref_inc(di->page);
+	page_ref_inc(au->page);
 
 	return skb;
 }
@@ -1570,20 +1571,22 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
 {
 	struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
 	struct mlx5e_wqe_frag_info *head_wi = wi;
+	union mlx5e_alloc_unit *au = wi->au;
 	u16 rx_headroom = rq->buff.headroom;
-	struct mlx5e_dma_info *di = wi->di;
 	struct skb_shared_info *sinfo;
 	u32 frag_consumed_bytes;
 	struct bpf_prog *prog;
 	struct xdp_buff xdp;
 	struct sk_buff *skb;
+	dma_addr_t addr;
 	u32 truesize;
 	void *va;
 
-	va = page_address(di->page) + wi->offset;
+	va = page_address(au->page) + wi->offset;
 	frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt);
 
-	dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
+	addr = page_pool_get_dma_addr(au->page);
+	dma_sync_single_range_for_cpu(rq->pdev, addr, wi->offset,
 				      rq->buff.frame0_sz, DMA_FROM_DEVICE);
 	net_prefetchw(va); /* xdp_frame data area */
 	net_prefetch(va + rx_headroom);
@@ -1599,11 +1602,12 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
 	while (cqe_bcnt) {
 		skb_frag_t *frag;
 
-		di = wi->di;
+		au = wi->au;
 
 		frag_consumed_bytes = min_t(u32, frag_info->frag_size, cqe_bcnt);
 
-		dma_sync_single_for_cpu(rq->pdev, di->addr + wi->offset,
+		addr = page_pool_get_dma_addr(au->page);
+		dma_sync_single_for_cpu(rq->pdev, addr + wi->offset,
 					frag_consumed_bytes, DMA_FROM_DEVICE);
 
 		if (!xdp_buff_has_frags(&xdp)) {
@@ -1616,11 +1620,11 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
 		}
 
 		frag = &sinfo->frags[sinfo->nr_frags++];
-		__skb_frag_set_page(frag, di->page);
+		__skb_frag_set_page(frag, au->page);
 		skb_frag_off_set(frag, wi->offset);
 		skb_frag_size_set(frag, frag_consumed_bytes);
 
-		if (page_is_pfmemalloc(di->page))
+		if (page_is_pfmemalloc(au->page))
 			xdp_buff_set_frag_pfmemalloc(&xdp);
 
 		sinfo->xdp_frags_size += frag_consumed_bytes;
@@ -1631,10 +1635,10 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
 		wi++;
 	}
 
-	di = head_wi->di;
+	au = head_wi->au;
 
 	prog = rcu_dereference(rq->xdp_prog);
-	if (prog && mlx5e_xdp_handle(rq, di->page, prog, &xdp)) {
+	if (prog && mlx5e_xdp_handle(rq, au->page, prog, &xdp)) {
 		if (test_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags)) {
 			int i;
 
@@ -1651,7 +1655,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi
 	if (unlikely(!skb))
 		return NULL;
 
-	page_ref_inc(di->page);
+	page_ref_inc(au->page);
 
 	if (unlikely(xdp_buff_has_frags(&xdp))) {
 		int i;
@@ -1706,9 +1710,10 @@ static void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe)
 		goto free_wqe;
 	}
 
-	skb = INDIRECT_CALL_2(rq->wqe.skb_from_cqe,
+	skb = INDIRECT_CALL_3(rq->wqe.skb_from_cqe,
 			      mlx5e_skb_from_cqe_linear,
 			      mlx5e_skb_from_cqe_nonlinear,
+			      mlx5e_xsk_skb_from_cqe_linear,
 			      rq, wi, cqe_bcnt);
 	if (!skb) {
 		/* probably for XDP */
@@ -1791,11 +1796,11 @@ static void mlx5e_handle_rx_cqe_mpwrq_rep(struct mlx5e_rq *rq, struct mlx5_cqe64
 {
 	u16 cstrides       = mpwrq_get_cqe_consumed_strides(cqe);
 	u16 wqe_id         = be16_to_cpu(cqe->wqe_id);
-	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
+	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, wqe_id);
 	u16 stride_ix      = mpwrq_get_cqe_stride_index(cqe);
 	u32 wqe_offset     = stride_ix << rq->mpwqe.log_stride_sz;
-	u32 head_offset    = wqe_offset & (PAGE_SIZE - 1);
-	u32 page_idx       = wqe_offset >> PAGE_SHIFT;
+	u32 head_offset    = wqe_offset & ((1 << rq->mpwqe.page_shift) - 1);
+	u32 page_idx       = wqe_offset >> rq->mpwqe.page_shift;
 	struct mlx5e_rx_wqe_ll *wqe;
 	struct mlx5_wq_ll *wq;
 	struct sk_buff *skb;
@@ -1846,12 +1851,13 @@ const struct mlx5e_rx_handlers mlx5e_rx_handlers_rep = {
 #endif
 
 static void
-mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_dma_info *di,
-		    u32 data_bcnt, u32 data_offset)
+mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq,
+		    union mlx5e_alloc_unit *au, u32 data_bcnt, u32 data_offset)
 {
 	net_prefetchw(skb->data);
 
 	while (data_bcnt) {
+		/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
 		u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - data_offset, data_bcnt);
 		unsigned int truesize;
 
@@ -1860,12 +1866,12 @@ mlx5e_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_dma_i
 		else
 			truesize = ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz));
 
-		mlx5e_add_skb_frag(rq, skb, di, data_offset,
+		mlx5e_add_skb_frag(rq, skb, au, data_offset,
 				   pg_consumed_bytes, truesize);
 
 		data_bcnt -= pg_consumed_bytes;
 		data_offset = 0;
-		di++;
+		au++;
 	}
 }
 
@@ -1873,12 +1879,13 @@ static struct sk_buff *
 mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 				   u16 cqe_bcnt, u32 head_offset, u32 page_idx)
 {
+	union mlx5e_alloc_unit *au = &wi->alloc_units[page_idx];
 	u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt);
-	struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
 	u32 frag_offset    = head_offset + headlen;
 	u32 byte_cnt       = cqe_bcnt - headlen;
-	struct mlx5e_dma_info *head_di = di;
+	union mlx5e_alloc_unit *head_au = au;
 	struct sk_buff *skb;
+	dma_addr_t addr;
 
 	skb = napi_alloc_skb(rq->cq.napi,
 			     ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long)));
@@ -1889,14 +1896,17 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
 
 	net_prefetchw(skb->data);
 
+	/* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */
 	if (unlikely(frag_offset >= PAGE_SIZE)) {
-		di++;
+		au++;
 		frag_offset -= PAGE_SIZE;
 	}
 
-	mlx5e_fill_skb_data(skb, rq, di, byte_cnt, frag_offset);
+	mlx5e_fill_skb_data(skb, rq, au, byte_cnt, frag_offset);
 	/* copy header */
-	mlx5e_copy_skb_header(rq->pdev, skb, head_di, head_offset, head_offset, headlen);
+	addr = page_pool_get_dma_addr(head_au->page);
+	mlx5e_copy_skb_header(rq->pdev, skb, head_au->page, addr,
+			      head_offset, head_offset, headlen);
 	/* skb linear part was allocated with headlen and aligned to long */
 	skb->tail += headlen;
 	skb->len  += headlen;
@@ -1908,12 +1918,13 @@ static struct sk_buff *
 mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 				u16 cqe_bcnt, u32 head_offset, u32 page_idx)
 {
-	struct mlx5e_dma_info *di = &wi->umr.dma_info[page_idx];
+	union mlx5e_alloc_unit *au = &wi->alloc_units[page_idx];
 	u16 rx_headroom = rq->buff.headroom;
 	struct bpf_prog *prog;
 	struct sk_buff *skb;
 	u32 metasize = 0;
 	void *va, *data;
+	dma_addr_t addr;
 	u32 frag_size;
 
 	/* Check packet size. Note LRO doesn't use linear SKB */
@@ -1922,11 +1933,12 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 		return NULL;
 	}
 
-	va             = page_address(di->page) + head_offset;
+	va             = page_address(au->page) + head_offset;
 	data           = va + rx_headroom;
 	frag_size      = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
 
-	dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset,
+	addr = page_pool_get_dma_addr(au->page);
+	dma_sync_single_range_for_cpu(rq->pdev, addr, head_offset,
 				      frag_size, DMA_FROM_DEVICE);
 	net_prefetch(data);
 
@@ -1936,7 +1948,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 
 		net_prefetchw(va); /* xdp_frame data area */
 		mlx5e_fill_xdp_buff(rq, va, rx_headroom, cqe_bcnt, &xdp);
-		if (mlx5e_xdp_handle(rq, di->page, prog, &xdp)) {
+		if (mlx5e_xdp_handle(rq, au->page, prog, &xdp)) {
 			if (__test_and_clear_bit(MLX5E_RQ_FLAG_XDP_XMIT, rq->flags))
 				__set_bit(page_idx, wi->xdp_xmit_bitmap); /* non-atomic */
 			return NULL; /* page/packet was consumed by XDP */
@@ -1952,7 +1964,7 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 		return NULL;
 
 	/* queue up for recycling/reuse */
-	page_ref_inc(di->page);
+	page_ref_inc(au->page);
 
 	return skb;
 }
@@ -1997,7 +2009,7 @@ mlx5e_skb_from_cqe_shampo(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 		}
 
 		prefetchw(skb->data);
-		mlx5e_copy_skb_header(rq->pdev, skb, head,
+		mlx5e_copy_skb_header(rq->pdev, skb, head->page, head->addr,
 				      head_offset + rx_headroom,
 				      rx_headroom, head_size);
 		/* skb linear part was allocated with headlen and aligned to long */
@@ -2049,7 +2061,7 @@ mlx5e_free_rx_shampo_hd_entry(struct mlx5e_rq *rq, u16 header_index)
 
 	if (((header_index + 1) & (MLX5E_SHAMPO_WQ_HEADER_PER_PAGE - 1)) == 0) {
 		shampo->info[header_index].addr = ALIGN_DOWN(addr, PAGE_SIZE);
-		mlx5e_page_release(rq, &shampo->info[header_index], true);
+		mlx5e_page_release_dynamic(rq, shampo->info[header_index].page, true);
 	}
 	bitmap_clear(shampo->bitmap, header_index, 1);
 }
@@ -2070,11 +2082,11 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq
 	bool match		= cqe->shampo.match;
 	struct mlx5e_rq_stats *stats = rq->stats;
 	struct mlx5e_rx_wqe_ll *wqe;
-	struct mlx5e_dma_info *di;
+	union mlx5e_alloc_unit *au;
 	struct mlx5e_mpw_info *wi;
 	struct mlx5_wq_ll *wq;
 
-	wi = &rq->mpwqe.info[wqe_id];
+	wi = mlx5e_get_mpw_info(rq, wqe_id);
 	wi->consumed_strides += cstrides;
 
 	if (unlikely(MLX5E_RX_ERR_CQE(cqe))) {
@@ -2120,8 +2132,8 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq
 	}
 
 	if (likely(head_size)) {
-		di = &wi->umr.dma_info[page_idx];
-		mlx5e_fill_skb_data(*skb, rq, di, data_bcnt, data_offset);
+		au = &wi->alloc_units[page_idx];
+		mlx5e_fill_skb_data(*skb, rq, au, data_bcnt, data_offset);
 	}
 
 	mlx5e_shampo_complete_rx_cqe(rq, cqe, cqe_bcnt, *skb);
@@ -2143,11 +2155,11 @@ static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cq
 {
 	u16 cstrides       = mpwrq_get_cqe_consumed_strides(cqe);
 	u16 wqe_id         = be16_to_cpu(cqe->wqe_id);
-	struct mlx5e_mpw_info *wi = &rq->mpwqe.info[wqe_id];
+	struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, wqe_id);
 	u16 stride_ix      = mpwrq_get_cqe_stride_index(cqe);
 	u32 wqe_offset     = stride_ix << rq->mpwqe.log_stride_sz;
-	u32 head_offset    = wqe_offset & (PAGE_SIZE - 1);
-	u32 page_idx       = wqe_offset >> PAGE_SHIFT;
+	u32 head_offset    = wqe_offset & ((1 << rq->mpwqe.page_shift) - 1);
+	u32 page_idx       = wqe_offset >> rq->mpwqe.page_shift;
 	struct mlx5e_rx_wqe_ll *wqe;
 	struct mlx5_wq_ll *wq;
 	struct sk_buff *skb;
@@ -2170,9 +2182,10 @@ static void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cq
 
 	cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe);
 
-	skb = INDIRECT_CALL_2(rq->mpwqe.skb_from_cqe_mpwrq,
+	skb = INDIRECT_CALL_3(rq->mpwqe.skb_from_cqe_mpwrq,
 			      mlx5e_skb_from_cqe_mpwrq_linear,
 			      mlx5e_skb_from_cqe_mpwrq_nonlinear,
+			      mlx5e_xsk_skb_from_cqe_mpwrq_linear,
 			      rq, wi, cqe_bcnt, head_offset, page_idx);
 	if (!skb)
 		goto mpwrq_cqe_out;
@@ -2417,7 +2430,7 @@ int mlx5e_rq_set_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params, bool
 	default: /* MLX5_WQ_TYPE_CYCLIC */
 		rq->wqe.skb_from_cqe = xsk ?
 			mlx5e_xsk_skb_from_cqe_linear :
-			mlx5e_rx_is_linear_skb(params, NULL) ?
+			mlx5e_rx_is_linear_skb(mdev, params, NULL) ?
 				mlx5e_skb_from_cqe_linear :
 				mlx5e_skb_from_cqe_nonlinear;
 		rq->post_wqes = mlx5e_post_rx_wqes;
@@ -2471,7 +2484,7 @@ free_wqe:
 
 void mlx5e_rq_set_trap_handlers(struct mlx5e_rq *rq, struct mlx5e_params *params)
 {
-	rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params, NULL) ?
+	rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(rq->mdev, params, NULL) ?
 			       mlx5e_skb_from_cqe_linear :
 			       mlx5e_skb_from_cqe_nonlinear;
 	rq->post_wqes = mlx5e_post_rx_wqes;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
index 7409829d1201..03c1841970f1 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
@@ -641,17 +641,26 @@ static const struct counter_desc vnic_env_stats_dev_oob_desc[] = {
 		VNIC_ENV_OFF(vport_env.internal_rq_out_of_buffer) },
 };
 
+static const struct counter_desc vnic_env_stats_drop_desc[] = {
+	{ "rx_oversize_pkts_buffer",
+		VNIC_ENV_OFF(vport_env.eth_wqe_too_small) },
+};
+
 #define NUM_VNIC_ENV_STEER_COUNTERS(dev) \
 	(MLX5_CAP_GEN(dev, nic_receive_steering_discard) ? \
 	 ARRAY_SIZE(vnic_env_stats_steer_desc) : 0)
 #define NUM_VNIC_ENV_DEV_OOB_COUNTERS(dev) \
 	(MLX5_CAP_GEN(dev, vnic_env_int_rq_oob) ? \
 	 ARRAY_SIZE(vnic_env_stats_dev_oob_desc) : 0)
+#define NUM_VNIC_ENV_DROP_COUNTERS(dev) \
+	(MLX5_CAP_GEN(dev, eth_wqe_too_small) ? \
+	 ARRAY_SIZE(vnic_env_stats_drop_desc) : 0)
 
 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vnic_env)
 {
 	return NUM_VNIC_ENV_STEER_COUNTERS(priv->mdev) +
-		NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev);
+	       NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev) +
+	       NUM_VNIC_ENV_DROP_COUNTERS(priv->mdev);
 }
 
 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vnic_env)
@@ -665,6 +674,11 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vnic_env)
 	for (i = 0; i < NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev); i++)
 		strcpy(data + (idx++) * ETH_GSTRING_LEN,
 		       vnic_env_stats_dev_oob_desc[i].format);
+
+	for (i = 0; i < NUM_VNIC_ENV_DROP_COUNTERS(priv->mdev); i++)
+		strcpy(data + (idx++) * ETH_GSTRING_LEN,
+		       vnic_env_stats_drop_desc[i].format);
+
 	return idx;
 }
 
@@ -679,6 +693,11 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vnic_env)
 	for (i = 0; i < NUM_VNIC_ENV_DEV_OOB_COUNTERS(priv->mdev); i++)
 		data[idx++] = MLX5E_READ_CTR32_BE(priv->stats.vnic.query_vnic_env_out,
 						  vnic_env_stats_dev_oob_desc, i);
+
+	for (i = 0; i < NUM_VNIC_ENV_DROP_COUNTERS(priv->mdev); i++)
+		data[idx++] = MLX5E_READ_CTR32_BE(priv->stats.vnic.query_vnic_env_out,
+						  vnic_env_stats_drop_desc, i);
+
 	return idx;
 }
 
@@ -2451,6 +2470,9 @@ mlx5e_stats_grp_t mlx5e_nic_stats_grps[] = {
 	&MLX5E_STATS_GRP(per_port_buff_congest),
 	&MLX5E_STATS_GRP(ptp),
 	&MLX5E_STATS_GRP(qos),
+#ifdef CONFIG_MLX5_EN_MACSEC
+	&MLX5E_STATS_GRP(macsec_hw),
+#endif
 };
 
 unsigned int mlx5e_nic_stats_grps_num(struct mlx5e_priv *priv)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
index ed4fc940e4ef..9f781085be47 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
@@ -273,6 +273,10 @@ struct mlx5e_qcounter_stats {
 	u32 rx_if_down_packets;
 };
 
+#define VNIC_ENV_GET(vnic_env_stats, c) \
+	MLX5_GET(query_vnic_env_out, (vnic_env_stats)->query_vnic_env_out, \
+		 vport_env.c)
+
 struct mlx5e_vnic_env_stats {
 	__be64 query_vnic_env_out[MLX5_ST_SZ_QW(query_vnic_env_out)];
 };
@@ -486,5 +490,6 @@ extern MLX5E_DECLARE_STATS_GRP(channels);
 extern MLX5E_DECLARE_STATS_GRP(per_port_buff_congest);
 extern MLX5E_DECLARE_STATS_GRP(ipsec_sw);
 extern MLX5E_DECLARE_STATS_GRP(ptp);
+extern MLX5E_DECLARE_STATS_GRP(macsec_hw);
 
 #endif /* __MLX5_EN_STATS_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index f154bda668ad..70a7a61f9708 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -311,6 +311,7 @@ mlx5e_get_flow_meters(struct mlx5_core_dev *dev)
 static struct mlx5_tc_ct_priv *
 get_ct_priv(struct mlx5e_priv *priv)
 {
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	struct mlx5_rep_uplink_priv *uplink_priv;
 	struct mlx5e_rep_priv *uplink_rpriv;
@@ -322,7 +323,7 @@ get_ct_priv(struct mlx5e_priv *priv)
 		return uplink_priv->ct_priv;
 	}
 
-	return priv->fs->tc->ct;
+	return tc->ct;
 }
 
 static struct mlx5e_tc_psample *
@@ -345,6 +346,7 @@ get_sample_priv(struct mlx5e_priv *priv)
 static struct mlx5e_post_act *
 get_post_action(struct mlx5e_priv *priv)
 {
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 	struct mlx5_rep_uplink_priv *uplink_priv;
 	struct mlx5e_rep_priv *uplink_rpriv;
@@ -356,7 +358,7 @@ get_post_action(struct mlx5e_priv *priv)
 		return uplink_priv->post_act;
 	}
 
-	return priv->fs->tc->post_act;
+	return tc->post_act;
 }
 
 struct mlx5_flow_handle *
@@ -607,11 +609,12 @@ int mlx5e_get_flow_namespace(struct mlx5e_tc_flow *flow)
 static struct mod_hdr_tbl *
 get_mod_hdr_table(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow)
 {
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 
 	return mlx5e_get_flow_namespace(flow) == MLX5_FLOW_NAMESPACE_FDB ?
 		&esw->offloads.mod_hdr :
-		&priv->fs->tc->mod_hdr;
+		&tc->mod_hdr;
 }
 
 static int mlx5e_attach_mod_hdr(struct mlx5e_priv *priv,
@@ -810,6 +813,7 @@ static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin *hp)
 {
 	struct mlx5e_priv *priv = hp->func_priv;
 	struct ttc_params ttc_params;
+	struct mlx5_ttc_table *ttc;
 	int err;
 
 	err = mlx5e_hairpin_create_indirect_rqt(hp);
@@ -827,9 +831,10 @@ static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin *hp)
 		goto err_create_ttc_table;
 	}
 
+	ttc = mlx5e_fs_get_ttc(priv->fs, false);
 	netdev_dbg(priv->netdev, "add hairpin: using %d channels rss ttc table id %x\n",
 		   hp->num_channels,
-		   mlx5_get_ttc_flow_table(priv->fs->ttc)->id);
+		   mlx5_get_ttc_flow_table(ttc)->id);
 
 	return 0;
 
@@ -916,10 +921,11 @@ static inline u32 hash_hairpin_info(u16 peer_vhca_id, u8 prio)
 static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv,
 						     u16 peer_vhca_id, u8 prio)
 {
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	struct mlx5e_hairpin_entry *hpe;
 	u32 hash_key = hash_hairpin_info(peer_vhca_id, prio);
 
-	hash_for_each_possible(priv->fs->tc->hairpin_tbl, hpe,
+	hash_for_each_possible(tc->hairpin_tbl, hpe,
 			       hairpin_hlist, hash_key) {
 		if (hpe->peer_vhca_id == peer_vhca_id && hpe->prio == prio) {
 			refcount_inc(&hpe->refcnt);
@@ -933,11 +939,12 @@ static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv,
 static void mlx5e_hairpin_put(struct mlx5e_priv *priv,
 			      struct mlx5e_hairpin_entry *hpe)
 {
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	/* no more hairpin flows for us, release the hairpin pair */
-	if (!refcount_dec_and_mutex_lock(&hpe->refcnt, &priv->fs->tc->hairpin_tbl_lock))
+	if (!refcount_dec_and_mutex_lock(&hpe->refcnt, &tc->hairpin_tbl_lock))
 		return;
 	hash_del(&hpe->hairpin_hlist);
-	mutex_unlock(&priv->fs->tc->hairpin_tbl_lock);
+	mutex_unlock(&tc->hairpin_tbl_lock);
 
 	if (!IS_ERR_OR_NULL(hpe->hp)) {
 		netdev_dbg(priv->netdev, "del hairpin: peer %s\n",
@@ -993,6 +1000,7 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
 				  struct mlx5e_tc_flow_parse_attr *parse_attr,
 				  struct netlink_ext_ack *extack)
 {
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	int peer_ifindex = parse_attr->mirred_ifindex[0];
 	struct mlx5_hairpin_params params;
 	struct mlx5_core_dev *peer_mdev;
@@ -1021,10 +1029,10 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
 	if (err)
 		return err;
 
-	mutex_lock(&priv->fs->tc->hairpin_tbl_lock);
+	mutex_lock(&tc->hairpin_tbl_lock);
 	hpe = mlx5e_hairpin_get(priv, peer_id, match_prio);
 	if (hpe) {
-		mutex_unlock(&priv->fs->tc->hairpin_tbl_lock);
+		mutex_unlock(&tc->hairpin_tbl_lock);
 		wait_for_completion(&hpe->res_ready);
 
 		if (IS_ERR(hpe->hp)) {
@@ -1036,7 +1044,7 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
 
 	hpe = kzalloc(sizeof(*hpe), GFP_KERNEL);
 	if (!hpe) {
-		mutex_unlock(&priv->fs->tc->hairpin_tbl_lock);
+		mutex_unlock(&tc->hairpin_tbl_lock);
 		return -ENOMEM;
 	}
 
@@ -1048,9 +1056,9 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv,
 	refcount_set(&hpe->refcnt, 1);
 	init_completion(&hpe->res_ready);
 
-	hash_add(priv->fs->tc->hairpin_tbl, &hpe->hairpin_hlist,
+	hash_add(tc->hairpin_tbl, &hpe->hairpin_hlist,
 		 hash_hairpin_info(peer_id, match_prio));
-	mutex_unlock(&priv->fs->tc->hairpin_tbl_lock);
+	mutex_unlock(&tc->hairpin_tbl_lock);
 
 	params.log_data_size = 16;
 	params.log_data_size = min_t(u8, params.log_data_size,
@@ -1126,8 +1134,9 @@ mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv,
 			     struct mlx5_flow_attr *attr)
 {
 	struct mlx5_flow_context *flow_context = &spec->flow_context;
+	struct mlx5e_vlan_table *vlan = mlx5e_fs_get_vlan(priv->fs);
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	struct mlx5_nic_flow_attr *nic_attr = attr->nic_attr;
-	struct mlx5e_tc_table *tc = priv->fs->tc;
 	struct mlx5_flow_destination dest[2] = {};
 	struct mlx5_fs_chains *nic_chains;
 	struct mlx5_flow_act flow_act = {
@@ -1163,7 +1172,7 @@ mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv,
 			if (IS_ERR(dest[dest_ix].ft))
 				return ERR_CAST(dest[dest_ix].ft);
 		} else {
-			dest[dest_ix].ft = mlx5e_vlan_get_flowtable(priv->fs->vlan);
+			dest[dest_ix].ft = mlx5e_vlan_get_flowtable(vlan);
 		}
 		dest_ix++;
 	}
@@ -1191,7 +1200,7 @@ mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv,
 			mutex_unlock(&tc->t_lock);
 			netdev_err(priv->netdev,
 				   "Failed to create tc offload table\n");
-			rule = ERR_CAST(priv->fs->tc->t);
+			rule = ERR_CAST(tc->t);
 			goto err_ft_get;
 		}
 	}
@@ -1293,8 +1302,10 @@ void mlx5e_del_offloaded_nic_rule(struct mlx5e_priv *priv,
 				  struct mlx5_flow_handle *rule,
 				  struct mlx5_flow_attr *attr)
 {
-	struct mlx5_fs_chains *nic_chains = mlx5e_nic_chains(priv->fs->tc);
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
+	struct mlx5_fs_chains *nic_chains;
 
+	nic_chains = mlx5e_nic_chains(tc);
 	mlx5_del_flow_rules(rule);
 
 	if (attr->chain || attr->prio)
@@ -1309,8 +1320,8 @@ void mlx5e_del_offloaded_nic_rule(struct mlx5e_priv *priv,
 static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv,
 				  struct mlx5e_tc_flow *flow)
 {
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	struct mlx5_flow_attr *attr = flow->attr;
-	struct mlx5e_tc_table *tc = priv->fs->tc;
 
 	flow_flag_clear(flow, OFFLOADED);
 
@@ -1322,13 +1333,13 @@ static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv,
 	/* Remove root table if no rules are left to avoid
 	 * extra steering hops.
 	 */
-	mutex_lock(&priv->fs->tc->t_lock);
+	mutex_lock(&tc->t_lock);
 	if (!mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD)) &&
 	    !IS_ERR_OR_NULL(tc->t)) {
 		mlx5_chains_put_table(mlx5e_nic_chains(tc), 0, 1, MLX5E_TC_FT_LEVEL);
-		priv->fs->tc->t = NULL;
+		tc->t = NULL;
 	}
-	mutex_unlock(&priv->fs->tc->t_lock);
+	mutex_unlock(&tc->t_lock);
 
 	if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
 		mlx5e_detach_mod_hdr(priv, flow);
@@ -1494,8 +1505,11 @@ bool mlx5e_tc_is_vf_tunnel(struct net_device *out_dev, struct net_device *route_
 	route_priv = netdev_priv(route_dev);
 	route_mdev = route_priv->mdev;
 
-	if (out_mdev->coredev_type != MLX5_COREDEV_PF ||
-	    route_mdev->coredev_type != MLX5_COREDEV_VF)
+	if (out_mdev->coredev_type != MLX5_COREDEV_PF)
+		return false;
+
+	if (route_mdev->coredev_type != MLX5_COREDEV_VF &&
+	    route_mdev->coredev_type != MLX5_COREDEV_SF)
 		return false;
 
 	return mlx5e_same_hw_devs(out_priv, route_priv);
@@ -4058,13 +4072,14 @@ static const struct rhashtable_params tc_ht_params = {
 static struct rhashtable *get_tc_ht(struct mlx5e_priv *priv,
 				    unsigned long flags)
 {
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	struct mlx5e_rep_priv *rpriv;
 
 	if (flags & MLX5_TC_FLAG(ESW_OFFLOAD)) {
 		rpriv = priv->ppriv;
 		return &rpriv->tc_ht;
 	} else /* NIC offload */
-		return &priv->fs->tc->ht;
+		return &tc->ht;
 }
 
 static bool is_peer_flow_needed(struct mlx5e_tc_flow *flow)
@@ -4448,7 +4463,7 @@ int mlx5e_configure_flower(struct net_device *dev, struct mlx5e_priv *priv,
 	int err = 0;
 
 	if (!mlx5_esw_hold(priv->mdev))
-		return -EAGAIN;
+		return -EBUSY;
 
 	mlx5_esw_get(priv->mdev);
 
@@ -4772,6 +4787,7 @@ void mlx5e_tc_stats_matchall(struct mlx5e_priv *priv,
 static void mlx5e_tc_hairpin_update_dead_peer(struct mlx5e_priv *priv,
 					      struct mlx5e_priv *peer_priv)
 {
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	struct mlx5_core_dev *peer_mdev = peer_priv->mdev;
 	struct mlx5e_hairpin_entry *hpe, *tmp;
 	LIST_HEAD(init_wait_list);
@@ -4783,11 +4799,11 @@ static void mlx5e_tc_hairpin_update_dead_peer(struct mlx5e_priv *priv,
 
 	peer_vhca_id = MLX5_CAP_GEN(peer_mdev, vhca_id);
 
-	mutex_lock(&priv->fs->tc->hairpin_tbl_lock);
-	hash_for_each(priv->fs->tc->hairpin_tbl, bkt, hpe, hairpin_hlist)
+	mutex_lock(&tc->hairpin_tbl_lock);
+	hash_for_each(tc->hairpin_tbl, bkt, hpe, hairpin_hlist)
 		if (refcount_inc_not_zero(&hpe->refcnt))
 			list_add(&hpe->dead_peer_wait_list, &init_wait_list);
-	mutex_unlock(&priv->fs->tc->hairpin_tbl_lock);
+	mutex_unlock(&tc->hairpin_tbl_lock);
 
 	list_for_each_entry_safe(hpe, tmp, &init_wait_list, dead_peer_wait_list) {
 		wait_for_completion(&hpe->res_ready);
@@ -4841,7 +4857,8 @@ static int mlx5e_tc_nic_get_ft_size(struct mlx5_core_dev *dev)
 
 static int mlx5e_tc_nic_create_miss_table(struct mlx5e_priv *priv)
 {
-	struct mlx5_flow_table **ft = &priv->fs->tc->miss_t;
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
+	struct mlx5_flow_table **ft = &tc->miss_t;
 	struct mlx5_flow_table_attr ft_attr = {};
 	struct mlx5_flow_namespace *ns;
 	int err = 0;
@@ -4863,12 +4880,14 @@ static int mlx5e_tc_nic_create_miss_table(struct mlx5e_priv *priv)
 
 static void mlx5e_tc_nic_destroy_miss_table(struct mlx5e_priv *priv)
 {
-	mlx5_destroy_flow_table(priv->fs->tc->miss_t);
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
+
+	mlx5_destroy_flow_table(tc->miss_t);
 }
 
 int mlx5e_tc_nic_init(struct mlx5e_priv *priv)
 {
-	struct mlx5e_tc_table *tc = priv->fs->tc;
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 	struct mlx5_core_dev *dev = priv->mdev;
 	struct mapping_ctx *chains_mapping;
 	struct mlx5_chains_attr attr = {};
@@ -4909,7 +4928,7 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv)
 	attr.ns = MLX5_FLOW_NAMESPACE_KERNEL;
 	attr.max_ft_sz = mlx5e_tc_nic_get_ft_size(dev);
 	attr.max_grp_num = MLX5E_TC_TABLE_NUM_GROUPS;
-	attr.default_ft = priv->fs->tc->miss_t;
+	attr.default_ft = tc->miss_t;
 	attr.mapping = chains_mapping;
 
 	tc->chains = mlx5_chains_create(dev, &attr);
@@ -4958,7 +4977,7 @@ static void _mlx5e_tc_del_flow(void *ptr, void *arg)
 
 void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv)
 {
-	struct mlx5e_tc_table *tc = priv->fs->tc;
+	struct mlx5e_tc_table *tc = mlx5e_fs_get_tc(priv->fs);
 
 	if (tc->netdevice_nb.notifier_call)
 		unregister_netdevice_notifier_dev_net(priv->netdev,
@@ -5163,13 +5182,13 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe,
 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
 	u32 chain = 0, chain_tag, reg_b, zone_restore_id;
 	struct mlx5e_priv *priv = netdev_priv(skb->dev);
-	struct mlx5e_tc_table *tc = priv->fs->tc;
 	struct mlx5_mapped_obj mapped_obj;
 	struct tc_skb_ext *tc_skb_ext;
+	struct mlx5e_tc_table *tc;
 	int err;
 
 	reg_b = be32_to_cpu(cqe->ft_metadata);
-
+	tc = mlx5e_fs_get_tc(priv->fs);
 	chain_tag = reg_b & MLX5E_TC_TABLE_CHAIN_TAG_MASK;
 
 	err = mapping_find(tc->mapping, chain_tag, &mapped_obj);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
index 6ce1ab6b86b7..48241317a535 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h
@@ -54,6 +54,7 @@
 			    ESW_FLOW_ATTR_SZ :\
 			    NIC_FLOW_ATTR_SZ)
 
+struct mlx5_fs_chains *mlx5e_nic_chains(struct mlx5e_tc_table *tc);
 int mlx5e_tc_num_filters(struct mlx5e_priv *priv, unsigned long flags);
 
 struct mlx5e_tc_update_priv {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
index 27f791feb517..bf2232a2a836 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -39,6 +39,7 @@
 #include "ipoib/ipoib.h"
 #include "en_accel/en_accel.h"
 #include "en_accel/ipsec_rxtx.h"
+#include "en_accel/macsec.h"
 #include "en/ptp.h"
 #include <net/ipv6.h>
 
@@ -485,7 +486,7 @@ err_drop:
 static bool mlx5e_tx_skb_supports_mpwqe(struct sk_buff *skb, struct mlx5e_tx_attr *attr)
 {
 	return !skb_is_nonlinear(skb) && !skb_vlan_tag_present(skb) && !attr->ihs &&
-	       !attr->insz;
+	       !attr->insz && !mlx5e_macsec_skb_is_offload(skb);
 }
 
 static bool mlx5e_tx_mpwqe_same_eseg(struct mlx5e_txqsq *sq, struct mlx5_wqe_eth_seg *eseg)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index 833be29170a1..9a458a5d9853 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -31,6 +31,7 @@
  */
 
 #include <linux/irq.h>
+#include <net/xdp_sock_drv.h>
 #include "en.h"
 #include "en/txrx.h"
 #include "en/xdp.h"
@@ -86,26 +87,36 @@ void mlx5e_trigger_irq(struct mlx5e_icosq *sq)
 
 static bool mlx5e_napi_xsk_post(struct mlx5e_xdpsq *xsksq, struct mlx5e_rq *xskrq)
 {
+	bool need_wakeup = xsk_uses_need_wakeup(xskrq->xsk_pool);
 	bool busy_xsk = false, xsk_rx_alloc_err;
 
-	/* Handle the race between the application querying need_wakeup and the
-	 * driver setting it:
-	 * 1. Update need_wakeup both before and after the TX. If it goes to
-	 * "yes", it can only happen with the first update.
-	 * 2. If the application queried need_wakeup before we set it, the
-	 * packets will be transmitted anyway, even w/o a wakeup.
-	 * 3. Give a chance to clear need_wakeup after new packets were queued
-	 * for TX.
+	/* If SQ is empty, there are no TX completions to trigger NAPI, so set
+	 * need_wakeup. Do it before queuing packets for TX to avoid race
+	 * condition with userspace.
 	 */
-	mlx5e_xsk_update_tx_wakeup(xsksq);
+	if (need_wakeup && xsksq->pc == xsksq->cc)
+		xsk_set_tx_need_wakeup(xsksq->xsk_pool);
 	busy_xsk |= mlx5e_xsk_tx(xsksq, MLX5E_TX_XSK_POLL_BUDGET);
-	mlx5e_xsk_update_tx_wakeup(xsksq);
+	/* If we queued some packets for TX, no need for wakeup anymore. */
+	if (need_wakeup && xsksq->pc != xsksq->cc)
+		xsk_clear_tx_need_wakeup(xsksq->xsk_pool);
 
+	/* If WQ is empty, RX won't trigger NAPI, so set need_wakeup. Do it
+	 * before refilling to avoid race condition with userspace.
+	 */
+	if (need_wakeup && !mlx5e_rqwq_get_cur_sz(xskrq))
+		xsk_set_rx_need_wakeup(xskrq->xsk_pool);
 	xsk_rx_alloc_err = INDIRECT_CALL_2(xskrq->post_wqes,
 					   mlx5e_post_rx_mpwqes,
 					   mlx5e_post_rx_wqes,
 					   xskrq);
-	busy_xsk |= mlx5e_xsk_update_rx_wakeup(xskrq, xsk_rx_alloc_err);
+	/* Ask for wakeup if WQ is not full after refill. */
+	if (!need_wakeup)
+		busy_xsk |= xsk_rx_alloc_err;
+	else if (xsk_rx_alloc_err)
+		xsk_set_rx_need_wakeup(xskrq->xsk_pool);
+	else
+		xsk_clear_rx_need_wakeup(xskrq->xsk_pool);
 
 	return busy_xsk;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eq.c b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
index 229728c80233..a0242dc15741 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eq.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eq.c
@@ -575,6 +575,9 @@ static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4])
 	if (MLX5_CAP_GEN_MAX(dev, vhca_state))
 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_VHCA_STATE_CHANGE);
 
+	if (MLX5_CAP_MACSEC(dev, log_max_macsec_offload))
+		async_event_mask |= (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE);
+
 	mask[0] = async_event_mask;
 
 	if (MLX5_CAP_GEN(dev, event_cap))
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c
index 0abef71cb839..c9a91158e99c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c
@@ -78,12 +78,16 @@ mlx5_esw_indir_table_needed(struct mlx5_eswitch *esw,
 			    struct mlx5_core_dev *dest_mdev)
 {
 	struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
+	bool vf_sf_vport;
+
+	vf_sf_vport = mlx5_eswitch_is_vf_vport(esw, vport_num) ||
+		      mlx5_esw_is_sf_vport(esw, vport_num);
 
 	/* Use indirect table for all IP traffic from UL to VF with vport
 	 * destination when source rewrite flag is set.
 	 */
 	return esw_attr->in_rep->vport == MLX5_VPORT_UPLINK &&
-		mlx5_eswitch_is_vf_vport(esw, vport_num) &&
+		vf_sf_vport &&
 		esw->dev == dest_mdev &&
 		attr->ip_version &&
 		attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
index 694c54066955..4f8a24d84a86 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c
@@ -924,12 +924,16 @@ int mlx5_esw_qos_vport_update_group(struct mlx5_eswitch *esw,
 				    struct mlx5_esw_rate_group *group,
 				    struct netlink_ext_ack *extack)
 {
-	int err;
+	int err = 0;
 
 	mutex_lock(&esw->state_lock);
+	if (!vport->qos.enabled && !group)
+		goto unlock;
+
 	err = esw_qos_vport_enable(esw, vport, 0, 0, extack);
 	if (!err)
 		err = esw_qos_vport_update_group(esw, vport, group, extack);
+unlock:
 	mutex_unlock(&esw->state_lock);
 	return err;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index 6aa58044b949..c59107fa9e6d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -1360,7 +1360,6 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf)
 	if (esw->mode == MLX5_ESWITCH_OFFLOADS) {
 		struct devlink *devlink = priv_to_devlink(esw->dev);
 
-		esw_offloads_del_send_to_vport_meta_rules(esw);
 		devl_rate_nodes_destroy(devlink);
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 87ce5a208cb5..f68dc2d0dbe6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -244,6 +244,8 @@ struct mlx5_esw_offload {
 
 	struct mlx5_flow_table *ft_offloads;
 	struct mlx5_flow_group *vport_rx_group;
+	struct mlx5_flow_group *vport_rx_drop_group;
+	struct mlx5_flow_handle *vport_rx_drop_rule;
 	struct xarray vport_reps;
 	struct list_head peer_flows;
 	struct mutex peer_mutex;
@@ -344,7 +346,10 @@ void esw_offloads_disable(struct mlx5_eswitch *esw);
 int esw_offloads_enable(struct mlx5_eswitch *esw);
 void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw);
 int esw_offloads_init_reps(struct mlx5_eswitch *esw);
-void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw);
+
+struct mlx5_flow_handle *
+mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num);
+void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule);
 
 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw);
 int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index a9f4c652f859..4e50df3139c6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -70,6 +70,8 @@
 #define MLX5_ESW_VPORT_TBL_SIZE 128
 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS  4
 
+#define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
+
 static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
 	.max_fte = MLX5_ESW_VPORT_TBL_SIZE,
 	.max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
@@ -481,25 +483,27 @@ esw_setup_dests(struct mlx5_flow_destination *dest,
 	    !(attr->flags & MLX5_ATTR_FLAG_SLOW_PATH)) {
 		esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i);
 		(*i)++;
-	} else if (attr->dest_ft) {
-		esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i);
-		(*i)++;
 	} else if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) {
 		esw_setup_slow_path_dest(dest, flow_act, esw, *i);
 		(*i)++;
 	} else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) {
 		esw_setup_accept_dest(dest, flow_act, chains, *i);
 		(*i)++;
-	} else if (attr->dest_chain) {
-		err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
-					   1, 0, *i);
-		(*i)++;
 	} else if (esw_is_indir_table(esw, attr)) {
 		err = esw_setup_indir_table(dest, flow_act, esw, attr, spec, true, i);
 	} else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
 		err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
 	} else {
 		*i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i);
+
+		if (attr->dest_ft) {
+			err = esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i);
+			(*i)++;
+		} else if (attr->dest_chain) {
+			err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
+						   1, 0, *i);
+			(*i)++;
+		}
 	}
 
 	return err;
@@ -1058,52 +1062,23 @@ void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
 	mlx5_del_flow_rules(rule);
 }
 
-static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw)
+void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule)
 {
-	struct mlx5_flow_handle **flows = esw->fdb_table.offloads.send_to_vport_meta_rules;
-	int i = 0, num_vfs = esw->esw_funcs.num_vfs;
-
-	if (!num_vfs || !flows)
-		return;
-
-	for (i = 0; i < num_vfs; i++)
-		mlx5_del_flow_rules(flows[i]);
-
-	kvfree(flows);
-	/* If changing eswitch mode from switchdev to legacy, but num_vfs is not 0,
-	 * meta rules could be freed again. So set it to NULL.
-	 */
-	esw->fdb_table.offloads.send_to_vport_meta_rules = NULL;
+	if (rule)
+		mlx5_del_flow_rules(rule);
 }
 
-void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw)
-{
-	mlx5_eswitch_del_send_to_vport_meta_rules(esw);
-}
-
-static int
-mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch *esw)
+struct mlx5_flow_handle *
+mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num)
 {
 	struct mlx5_flow_destination dest = {};
 	struct mlx5_flow_act flow_act = {0};
-	int num_vfs, rule_idx = 0, err = 0;
 	struct mlx5_flow_handle *flow_rule;
-	struct mlx5_flow_handle **flows;
 	struct mlx5_flow_spec *spec;
-	struct mlx5_vport *vport;
-	unsigned long i;
-	u16 vport_num;
-
-	num_vfs = esw->esw_funcs.num_vfs;
-	flows = kvcalloc(num_vfs, sizeof(*flows), GFP_KERNEL);
-	if (!flows)
-		return -ENOMEM;
 
 	spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
-	if (!spec) {
-		err = -ENOMEM;
-		goto alloc_err;
-	}
+	if (!spec)
+		return ERR_PTR(-ENOMEM);
 
 	MLX5_SET(fte_match_param, spec->match_criteria,
 		 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
@@ -1116,34 +1091,18 @@ mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch *esw)
 	dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
 	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
 
-	mlx5_esw_for_each_vf_vport(esw, i, vport, num_vfs) {
-		vport_num = vport->vport;
-		MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
-			 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
-		dest.vport.num = vport_num;
-
-		flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
-						spec, &flow_act, &dest, 1);
-		if (IS_ERR(flow_rule)) {
-			err = PTR_ERR(flow_rule);
-			esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule idx %d, err %ld\n",
-				 rule_idx, PTR_ERR(flow_rule));
-			goto rule_err;
-		}
-		flows[rule_idx++] = flow_rule;
-	}
+	MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
+		 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
+	dest.vport.num = vport_num;
 
-	esw->fdb_table.offloads.send_to_vport_meta_rules = flows;
-	kvfree(spec);
-	return 0;
+	flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
+					spec, &flow_act, &dest, 1);
+	if (IS_ERR(flow_rule))
+		esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n",
+			 vport_num, PTR_ERR(flow_rule));
 
-rule_err:
-	while (--rule_idx >= 0)
-		mlx5_del_flow_rules(flows[rule_idx]);
 	kvfree(spec);
-alloc_err:
-	kvfree(flows);
-	return err;
+	return flow_rule;
 }
 
 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
@@ -1668,18 +1627,200 @@ esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
 
 #endif
 
+static int
+esw_create_send_to_vport_group(struct mlx5_eswitch *esw,
+			       struct mlx5_flow_table *fdb,
+			       u32 *flow_group_in,
+			       int *ix)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *g;
+	void *match_criteria;
+	int count, err = 0;
+
+	memset(flow_group_in, 0, inlen);
+
+	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
+		 MLX5_MATCH_MISC_PARAMETERS);
+
+	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
+
+	MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
+	MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port);
+	if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
+		MLX5_SET_TO_ONES(fte_match_param, match_criteria,
+				 misc_parameters.source_eswitch_owner_vhca_id);
+		MLX5_SET(create_flow_group_in, flow_group_in,
+			 source_eswitch_owner_vhca_id_valid, 1);
+	}
+
+	/* See comment at table_size calculation */
+	count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1);
+	*ix += count;
+
+	g = mlx5_create_flow_group(fdb, flow_group_in);
+	if (IS_ERR(g)) {
+		err = PTR_ERR(g);
+		esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err);
+		goto out;
+	}
+	esw->fdb_table.offloads.send_to_vport_grp = g;
+
+out:
+	return err;
+}
+
+static int
+esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw,
+				    struct mlx5_flow_table *fdb,
+				    u32 *flow_group_in,
+				    int *ix)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *g;
+	void *match_criteria;
+	int err = 0;
+
+	if (!esw_src_port_rewrite_supported(esw))
+		return 0;
+
+	memset(flow_group_in, 0, inlen);
+
+	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
+		 MLX5_MATCH_MISC_PARAMETERS_2);
+
+	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
+
+	MLX5_SET(fte_match_param, match_criteria,
+		 misc_parameters_2.metadata_reg_c_0,
+		 mlx5_eswitch_get_vport_metadata_mask());
+	MLX5_SET(fte_match_param, match_criteria,
+		 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
+
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
+	MLX5_SET(create_flow_group_in, flow_group_in,
+		 end_flow_index, *ix + esw->total_vports - 1);
+	*ix += esw->total_vports;
+
+	g = mlx5_create_flow_group(fdb, flow_group_in);
+	if (IS_ERR(g)) {
+		err = PTR_ERR(g);
+		esw_warn(esw->dev,
+			 "Failed to create send-to-vport meta flow group err(%d)\n", err);
+		goto send_vport_meta_err;
+	}
+	esw->fdb_table.offloads.send_to_vport_meta_grp = g;
+
+	return 0;
+
+send_vport_meta_err:
+	return err;
+}
+
+static int
+esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw,
+			       struct mlx5_flow_table *fdb,
+			       u32 *flow_group_in,
+			       int *ix)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *g;
+	void *match_criteria;
+	int err = 0;
+
+	if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
+		return 0;
+
+	memset(flow_group_in, 0, inlen);
+
+	esw_set_flow_group_source_port(esw, flow_group_in);
+
+	if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
+		match_criteria = MLX5_ADDR_OF(create_flow_group_in,
+					      flow_group_in,
+					      match_criteria);
+
+		MLX5_SET_TO_ONES(fte_match_param, match_criteria,
+				 misc_parameters.source_eswitch_owner_vhca_id);
+
+		MLX5_SET(create_flow_group_in, flow_group_in,
+			 source_eswitch_owner_vhca_id_valid, 1);
+	}
+
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
+		 *ix + esw->total_vports - 1);
+	*ix += esw->total_vports;
+
+	g = mlx5_create_flow_group(fdb, flow_group_in);
+	if (IS_ERR(g)) {
+		err = PTR_ERR(g);
+		esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err);
+		goto out;
+	}
+	esw->fdb_table.offloads.peer_miss_grp = g;
+
+out:
+	return err;
+}
+
+static int
+esw_create_miss_group(struct mlx5_eswitch *esw,
+		      struct mlx5_flow_table *fdb,
+		      u32 *flow_group_in,
+		      int *ix)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *g;
+	void *match_criteria;
+	int err = 0;
+	u8 *dmac;
+
+	memset(flow_group_in, 0, inlen);
+
+	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
+		 MLX5_MATCH_OUTER_HEADERS);
+	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
+				      match_criteria);
+	dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
+			    outer_headers.dmac_47_16);
+	dmac[0] = 0x01;
+
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix);
+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
+		 *ix + MLX5_ESW_MISS_FLOWS);
+
+	g = mlx5_create_flow_group(fdb, flow_group_in);
+	if (IS_ERR(g)) {
+		err = PTR_ERR(g);
+		esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err);
+		goto miss_err;
+	}
+	esw->fdb_table.offloads.miss_grp = g;
+
+	err = esw_add_fdb_miss_rule(esw);
+	if (err)
+		goto miss_rule_err;
+
+	return 0;
+
+miss_rule_err:
+	mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
+miss_err:
+	return err;
+}
+
 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
 {
 	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
 	struct mlx5_flow_table_attr ft_attr = {};
-	int num_vfs, table_size, ix, err = 0;
 	struct mlx5_core_dev *dev = esw->dev;
 	struct mlx5_flow_namespace *root_ns;
 	struct mlx5_flow_table *fdb = NULL;
+	int table_size, ix = 0, err = 0;
 	u32 flags = 0, *flow_group_in;
-	struct mlx5_flow_group *g;
-	void *match_criteria;
-	u8 *dmac;
 
 	esw_debug(esw->dev, "Create offloads FDB Tables\n");
 
@@ -1713,7 +1854,7 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
 	 * total vports of the peer (currently is also uses esw->total_vports).
 	 */
 	table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
-		MLX5_ESW_MISS_FLOWS + esw->total_vports + esw->esw_funcs.num_vfs;
+		     esw->total_vports * 2 + MLX5_ESW_MISS_FLOWS;
 
 	/* create the slow path fdb with encap set, so further table instances
 	 * can be created at run time while VFs are probed if the FW allows that.
@@ -1754,139 +1895,29 @@ static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
 		goto fdb_chains_err;
 	}
 
-	/* create send-to-vport group */
-	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
-		 MLX5_MATCH_MISC_PARAMETERS);
-
-	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
-
-	MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
-	MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port);
-	if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
-		MLX5_SET_TO_ONES(fte_match_param, match_criteria,
-				 misc_parameters.source_eswitch_owner_vhca_id);
-		MLX5_SET(create_flow_group_in, flow_group_in,
-			 source_eswitch_owner_vhca_id_valid, 1);
-	}
-
-	/* See comment above table_size calculation */
-	ix = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
-	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
-	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1);
-
-	g = mlx5_create_flow_group(fdb, flow_group_in);
-	if (IS_ERR(g)) {
-		err = PTR_ERR(g);
-		esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err);
+	err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix);
+	if (err)
 		goto send_vport_err;
-	}
-	esw->fdb_table.offloads.send_to_vport_grp = g;
-
-	if (esw_src_port_rewrite_supported(esw)) {
-		/* meta send to vport */
-		memset(flow_group_in, 0, inlen);
-		MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
-			 MLX5_MATCH_MISC_PARAMETERS_2);
-
-		match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
-
-		MLX5_SET(fte_match_param, match_criteria,
-			 misc_parameters_2.metadata_reg_c_0,
-			 mlx5_eswitch_get_vport_metadata_mask());
-		MLX5_SET(fte_match_param, match_criteria,
-			 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
-
-		num_vfs = esw->esw_funcs.num_vfs;
-		if (num_vfs) {
-			MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
-			MLX5_SET(create_flow_group_in, flow_group_in,
-				 end_flow_index, ix + num_vfs - 1);
-			ix += num_vfs;
-
-			g = mlx5_create_flow_group(fdb, flow_group_in);
-			if (IS_ERR(g)) {
-				err = PTR_ERR(g);
-				esw_warn(dev, "Failed to create send-to-vport meta flow group err(%d)\n",
-					 err);
-				goto send_vport_meta_err;
-			}
-			esw->fdb_table.offloads.send_to_vport_meta_grp = g;
-
-			err = mlx5_eswitch_add_send_to_vport_meta_rules(esw);
-			if (err)
-				goto meta_rule_err;
-		}
-	}
-
-	if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
-		/* create peer esw miss group */
-		memset(flow_group_in, 0, inlen);
-
-		esw_set_flow_group_source_port(esw, flow_group_in);
-
-		if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
-			match_criteria = MLX5_ADDR_OF(create_flow_group_in,
-						      flow_group_in,
-						      match_criteria);
-
-			MLX5_SET_TO_ONES(fte_match_param, match_criteria,
-					 misc_parameters.source_eswitch_owner_vhca_id);
-
-			MLX5_SET(create_flow_group_in, flow_group_in,
-				 source_eswitch_owner_vhca_id_valid, 1);
-		}
-
-		MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
-		MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
-			 ix + esw->total_vports - 1);
-		ix += esw->total_vports;
-
-		g = mlx5_create_flow_group(fdb, flow_group_in);
-		if (IS_ERR(g)) {
-			err = PTR_ERR(g);
-			esw_warn(dev, "Failed to create peer miss flow group err(%d)\n", err);
-			goto peer_miss_err;
-		}
-		esw->fdb_table.offloads.peer_miss_grp = g;
-	}
 
-	/* create miss group */
-	memset(flow_group_in, 0, inlen);
-	MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
-		 MLX5_MATCH_OUTER_HEADERS);
-	match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
-				      match_criteria);
-	dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
-			    outer_headers.dmac_47_16);
-	dmac[0] = 0x01;
-
-	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
-	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
-		 ix + MLX5_ESW_MISS_FLOWS);
+	err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix);
+	if (err)
+		goto send_vport_meta_err;
 
-	g = mlx5_create_flow_group(fdb, flow_group_in);
-	if (IS_ERR(g)) {
-		err = PTR_ERR(g);
-		esw_warn(dev, "Failed to create miss flow group err(%d)\n", err);
-		goto miss_err;
-	}
-	esw->fdb_table.offloads.miss_grp = g;
+	err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix);
+	if (err)
+		goto peer_miss_err;
 
-	err = esw_add_fdb_miss_rule(esw);
+	err = esw_create_miss_group(esw, fdb, flow_group_in, &ix);
 	if (err)
-		goto miss_rule_err;
+		goto miss_err;
 
 	kvfree(flow_group_in);
 	return 0;
 
-miss_rule_err:
-	mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
 miss_err:
 	if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
 		mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
 peer_miss_err:
-	mlx5_eswitch_del_send_to_vport_meta_rules(esw);
-meta_rule_err:
 	if (esw->fdb_table.offloads.send_to_vport_meta_grp)
 		mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
 send_vport_meta_err:
@@ -1913,7 +1944,6 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
 	esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
 	mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi);
 	mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
-	mlx5_eswitch_del_send_to_vport_meta_rules(esw);
 	mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
 	if (esw->fdb_table.offloads.send_to_vport_meta_grp)
 		mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
@@ -1931,7 +1961,7 @@ static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
 	atomic64_set(&esw->user_count, 0);
 }
 
-static int esw_get_offloads_ft_size(struct mlx5_eswitch *esw)
+static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw)
 {
 	int nvports;
 
@@ -1956,7 +1986,8 @@ static int esw_create_offloads_table(struct mlx5_eswitch *esw)
 		return -EOPNOTSUPP;
 	}
 
-	ft_attr.max_fte = esw_get_offloads_ft_size(esw);
+	ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) +
+			  MLX5_ESW_FT_OFFLOADS_DROP_RULE;
 	ft_attr.prio = 1;
 
 	ft_offloads = mlx5_create_flow_table(ns, &ft_attr);
@@ -1985,7 +2016,7 @@ static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
 	int nvports;
 	int err = 0;
 
-	nvports = esw_get_offloads_ft_size(esw);
+	nvports = esw_get_nr_ft_offloads_steering_src_ports(esw);
 	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
 	if (!flow_group_in)
 		return -ENOMEM;
@@ -2015,6 +2046,52 @@ static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
 	mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
 }
 
+static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw)
+{
+	/* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1)
+	 * for the drop rule, which is placed at the end of the table.
+	 * So return the total of vport and int_port as rule index.
+	 */
+	return esw_get_nr_ft_offloads_steering_src_ports(esw);
+}
+
+static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw)
+{
+	int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
+	struct mlx5_flow_group *g;
+	u32 *flow_group_in;
+	int flow_index;
+	int err = 0;
+
+	flow_index = esw_create_vport_rx_drop_rule_index(esw);
+
+	flow_group_in = kvzalloc(inlen, GFP_KERNEL);
+	if (!flow_group_in)
+		return -ENOMEM;
+
+	MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index);
+	MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index);
+
+	g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
+
+	if (IS_ERR(g)) {
+		err = PTR_ERR(g);
+		mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err);
+		goto out;
+	}
+
+	esw->offloads.vport_rx_drop_group = g;
+out:
+	kvfree(flow_group_in);
+	return err;
+}
+
+static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw)
+{
+	if (esw->offloads.vport_rx_drop_group)
+		mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group);
+}
+
 struct mlx5_flow_handle *
 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
 				  struct mlx5_flow_destination *dest)
@@ -2063,6 +2140,32 @@ out:
 	return flow_rule;
 }
 
+static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw)
+{
+	struct mlx5_flow_act flow_act = {};
+	struct mlx5_flow_handle *flow_rule;
+
+	flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP;
+	flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL,
+					&flow_act, NULL, 0);
+	if (IS_ERR(flow_rule)) {
+		esw_warn(esw->dev,
+			 "fs offloads: Failed to add vport rx drop rule err %ld\n",
+			 PTR_ERR(flow_rule));
+		return PTR_ERR(flow_rule);
+	}
+
+	esw->offloads.vport_rx_drop_rule = flow_rule;
+
+	return 0;
+}
+
+static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw)
+{
+	if (esw->offloads.vport_rx_drop_rule)
+		mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule);
+}
+
 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
 {
 	u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
@@ -3063,8 +3166,20 @@ static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
 	if (err)
 		goto create_fg_err;
 
+	err = esw_create_vport_rx_drop_group(esw);
+	if (err)
+		goto create_rx_drop_fg_err;
+
+	err = esw_create_vport_rx_drop_rule(esw);
+	if (err)
+		goto create_rx_drop_rule_err;
+
 	return 0;
 
+create_rx_drop_rule_err:
+	esw_destroy_vport_rx_drop_group(esw);
+create_rx_drop_fg_err:
+	esw_destroy_vport_rx_group(esw);
 create_fg_err:
 	esw_destroy_offloads_fdb_tables(esw);
 create_fdb_err:
@@ -3082,6 +3197,8 @@ create_indir_err:
 
 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
 {
+	esw_destroy_vport_rx_drop_rule(esw);
+	esw_destroy_vport_rx_drop_group(esw);
 	esw_destroy_vport_rx_group(esw);
 	esw_destroy_offloads_fdb_tables(esw);
 	esw_destroy_restore_table(esw);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/events.c b/drivers/net/ethernet/mellanox/mlx5/core/events.c
index a1ac3a654962..9459e56ee90a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/events.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/events.c
@@ -36,6 +36,7 @@ static struct mlx5_nb events_nbs_ref[] = {
 	/* Events to be forwarded (as is) to mlx5 core interfaces (mlx5e/mlx5_ib) */
 	{.nb.notifier_call = forward_event,   .event_type = MLX5_EVENT_TYPE_PORT_CHANGE },
 	{.nb.notifier_call = forward_event,   .event_type = MLX5_EVENT_TYPE_GENERAL_EVENT },
+	{.nb.notifier_call = forward_event,   .event_type = MLX5_EVENT_TYPE_OBJECT_CHANGE },
 	/* QP/WQ resource events to forward */
 	{.nb.notifier_call = forward_event,   .event_type = MLX5_EVENT_TYPE_DCT_DRAINED },
 	{.nb.notifier_call = forward_event,   .event_type = MLX5_EVENT_TYPE_PATH_MIG },
@@ -132,6 +133,8 @@ static const char *eqe_type_str(u8 type)
 		return "MLX5_EVENT_TYPE_MONITOR_COUNTER";
 	case MLX5_EVENT_TYPE_DEVICE_TRACER:
 		return "MLX5_EVENT_TYPE_DEVICE_TRACER";
+	case MLX5_EVENT_TYPE_OBJECT_CHANGE:
+		return "MLX5_EVENT_TYPE_OBJECT_CHANGE";
 	default:
 		return "Unrecognized event";
 	}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
index e735e19461ba..32d4c967469c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c
@@ -577,7 +577,10 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev,
 		MLX5_SET(flow_context, in_flow_context, modify_header_id,
 			 fte->action.modify_hdr->id);
 
-	MLX5_SET(flow_context, in_flow_context, ipsec_obj_id, fte->action.ipsec_obj_id);
+	MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_type,
+		 fte->action.crypto.type);
+	MLX5_SET(flow_context, in_flow_context, encrypt_decrypt_obj_id,
+		 fte->action.crypto.obj_id);
 
 	vlan = MLX5_ADDR_OF(flow_context, in_flow_context, push_vlan);
 
@@ -919,13 +922,15 @@ static int mlx5_cmd_modify_header_alloc(struct mlx5_flow_root_namespace *ns,
 		max_actions = MLX5_CAP_ESW_FLOWTABLE_FDB(dev, max_modify_header_actions);
 		table_type = FS_FT_FDB;
 		break;
+	case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC:
 	case MLX5_FLOW_NAMESPACE_KERNEL:
 	case MLX5_FLOW_NAMESPACE_BYPASS:
 		max_actions = MLX5_CAP_FLOWTABLE_NIC_RX(dev, max_modify_header_actions);
 		table_type = FS_FT_NIC_RX;
 		break;
 	case MLX5_FLOW_NAMESPACE_EGRESS:
-	case MLX5_FLOW_NAMESPACE_EGRESS_KERNEL:
+	case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC:
+	case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC:
 		max_actions = MLX5_CAP_FLOWTABLE_NIC_TX(dev, max_modify_header_actions);
 		table_type = FS_FT_NIC_TX;
 		break;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
index e3960cdf5131..d53749248fa0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
@@ -104,6 +104,10 @@
 #define BY_PASS_MIN_LEVEL (ETHTOOL_MIN_LEVEL + MLX5_BY_PASS_NUM_PRIOS +\
 			   LEFTOVERS_NUM_PRIOS)
 
+#define KERNEL_RX_MACSEC_NUM_PRIOS  1
+#define KERNEL_RX_MACSEC_NUM_LEVELS 2
+#define KERNEL_RX_MACSEC_MIN_LEVEL (BY_PASS_MIN_LEVEL + KERNEL_RX_MACSEC_NUM_PRIOS)
+
 #define ETHTOOL_PRIO_NUM_LEVELS 1
 #define ETHTOOL_NUM_PRIOS 11
 #define ETHTOOL_MIN_LEVEL (KERNEL_MIN_LEVEL + ETHTOOL_NUM_PRIOS)
@@ -126,11 +130,15 @@
 
 #define LAG_PRIO_NUM_LEVELS 1
 #define LAG_NUM_PRIOS 1
-#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + 1)
+#define LAG_MIN_LEVEL (OFFLOADS_MIN_LEVEL + KERNEL_RX_MACSEC_MIN_LEVEL + 1)
 
 #define KERNEL_TX_IPSEC_NUM_PRIOS  1
 #define KERNEL_TX_IPSEC_NUM_LEVELS 1
-#define KERNEL_TX_MIN_LEVEL        (KERNEL_TX_IPSEC_NUM_LEVELS)
+#define KERNEL_TX_IPSEC_MIN_LEVEL        (KERNEL_TX_IPSEC_NUM_LEVELS)
+
+#define KERNEL_TX_MACSEC_NUM_PRIOS  1
+#define KERNEL_TX_MACSEC_NUM_LEVELS 2
+#define KERNEL_TX_MACSEC_MIN_LEVEL       (KERNEL_TX_IPSEC_MIN_LEVEL + KERNEL_TX_MACSEC_NUM_PRIOS)
 
 struct node_caps {
 	size_t	arr_sz;
@@ -149,12 +157,16 @@ static struct init_tree_node {
 	enum mlx5_flow_table_miss_action def_miss_action;
 } root_fs = {
 	.type = FS_TYPE_NAMESPACE,
-	.ar_size = 7,
+	.ar_size = 8,
 	  .children = (struct init_tree_node[]){
 		  ADD_PRIO(0, BY_PASS_MIN_LEVEL, 0, FS_CHAINING_CAPS,
 			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
 				  ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
 						    BY_PASS_PRIO_NUM_LEVELS))),
+		  ADD_PRIO(0, KERNEL_RX_MACSEC_MIN_LEVEL, 0, FS_CHAINING_CAPS,
+			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				  ADD_MULTIPLE_PRIO(KERNEL_RX_MACSEC_NUM_PRIOS,
+						    KERNEL_RX_MACSEC_NUM_LEVELS))),
 		  ADD_PRIO(0, LAG_MIN_LEVEL, 0, FS_CHAINING_CAPS,
 			   ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
 				  ADD_MULTIPLE_PRIO(LAG_NUM_PRIOS,
@@ -186,18 +198,23 @@ static struct init_tree_node {
 
 static struct init_tree_node egress_root_fs = {
 	.type = FS_TYPE_NAMESPACE,
-	.ar_size = 2,
+	.ar_size = 3,
 	.children = (struct init_tree_node[]) {
 		ADD_PRIO(0, MLX5_BY_PASS_NUM_PRIOS, 0,
 			 FS_CHAINING_CAPS_EGRESS,
 			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
 				ADD_MULTIPLE_PRIO(MLX5_BY_PASS_NUM_PRIOS,
 						  BY_PASS_PRIO_NUM_LEVELS))),
-		ADD_PRIO(0, KERNEL_TX_MIN_LEVEL, 0,
+		ADD_PRIO(0, KERNEL_TX_IPSEC_MIN_LEVEL, 0,
 			 FS_CHAINING_CAPS_EGRESS,
 			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
 				ADD_MULTIPLE_PRIO(KERNEL_TX_IPSEC_NUM_PRIOS,
 						  KERNEL_TX_IPSEC_NUM_LEVELS))),
+		ADD_PRIO(0, KERNEL_TX_MACSEC_MIN_LEVEL, 0,
+			 FS_CHAINING_CAPS_EGRESS,
+			 ADD_NS(MLX5_FLOW_TABLE_MISS_ACTION_DEF,
+				ADD_MULTIPLE_PRIO(KERNEL_TX_MACSEC_NUM_PRIOS,
+						  KERNEL_TX_MACSEC_NUM_LEVELS))),
 	}
 };
 
@@ -2269,6 +2286,7 @@ static bool is_nic_rx_ns(enum mlx5_flow_namespace_type type)
 {
 	switch (type) {
 	case MLX5_FLOW_NAMESPACE_BYPASS:
+	case MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC:
 	case MLX5_FLOW_NAMESPACE_LAG:
 	case MLX5_FLOW_NAMESPACE_OFFLOADS:
 	case MLX5_FLOW_NAMESPACE_ETHTOOL:
@@ -2315,7 +2333,8 @@ struct mlx5_flow_namespace *mlx5_get_flow_namespace(struct mlx5_core_dev *dev,
 		prio =  FDB_BYPASS_PATH;
 		break;
 	case MLX5_FLOW_NAMESPACE_EGRESS:
-	case MLX5_FLOW_NAMESPACE_EGRESS_KERNEL:
+	case MLX5_FLOW_NAMESPACE_EGRESS_IPSEC:
+	case MLX5_FLOW_NAMESPACE_EGRESS_MACSEC:
 		root_ns = steering->egress_root_ns;
 		prio = type - MLX5_FLOW_NAMESPACE_EGRESS;
 		break;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
index 079fa44ada71..f34e758a2f1f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c
@@ -273,6 +273,19 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev)
 			return err;
 	}
 
+	if (MLX5_CAP_GEN_64(dev, general_obj_types) &
+	    MLX5_GENERAL_OBJ_TYPES_CAP_MACSEC_OFFLOAD) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_MACSEC);
+		if (err)
+			return err;
+	}
+
+	if (MLX5_CAP_GEN(dev, adv_virtualization)) {
+		err = mlx5_core_get_caps(dev, MLX5_CAP_ADV_VIRTUALIZATION);
+		if (err)
+			return err;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c
index 2cf2c9948446..86ed87d704f7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/health.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c
@@ -601,7 +601,7 @@ static void mlx5_fw_reporter_err_work(struct work_struct *work)
 	fw_reporter_ctx.miss_counter = health->miss_counter;
 	if (fw_reporter_ctx.err_synd) {
 		devlink_health_report(health->fw_reporter,
-				      "FW syndrom reported", &fw_reporter_ctx);
+				      "FW syndrome reported", &fw_reporter_ctx);
 		return;
 	}
 	if (fw_reporter_ctx.miss_counter)
@@ -702,11 +702,25 @@ static const struct devlink_health_reporter_ops mlx5_fw_fatal_reporter_ops = {
 		.dump = mlx5_fw_fatal_reporter_dump,
 };
 
-#define MLX5_REPORTER_FW_GRACEFUL_PERIOD 1200000
+#define MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD 180000
+#define MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD 60000
+#define MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD 30000
+#define MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD MLX5_FW_REPORTER_VF_GRACEFUL_PERIOD
+
 static void mlx5_fw_reporters_create(struct mlx5_core_dev *dev)
 {
 	struct mlx5_core_health *health = &dev->priv.health;
 	struct devlink *devlink = priv_to_devlink(dev);
+	u64 grace_period;
+
+	if (mlx5_core_is_ecpf(dev)) {
+		grace_period = MLX5_FW_REPORTER_ECPF_GRACEFUL_PERIOD;
+	} else if (mlx5_core_is_pf(dev)) {
+		grace_period = MLX5_FW_REPORTER_PF_GRACEFUL_PERIOD;
+	} else {
+		/* VF or SF */
+		grace_period = MLX5_FW_REPORTER_DEFAULT_GRACEFUL_PERIOD;
+	}
 
 	health->fw_reporter =
 		devlink_health_reporter_create(devlink, &mlx5_fw_reporter_ops,
@@ -718,7 +732,7 @@ static void mlx5_fw_reporters_create(struct mlx5_core_dev *dev)
 	health->fw_fatal_reporter =
 		devlink_health_reporter_create(devlink,
 					       &mlx5_fw_fatal_reporter_ops,
-					       MLX5_REPORTER_FW_GRACEFUL_PERIOD,
+					       grace_period,
 					       dev);
 	if (IS_ERR(health->fw_fatal_reporter))
 		mlx5_core_warn(dev, "Failed to create fw fatal reporter, err = %ld\n",
@@ -843,9 +857,6 @@ void mlx5_start_health_poll(struct mlx5_core_dev *dev)
 
 	health->timer.expires = jiffies + msecs_to_jiffies(poll_interval_ms);
 	add_timer(&health->timer);
-
-	if (mlx5_core_is_pf(dev) && MLX5_CAP_MCAM_REG(dev, mrtc))
-		queue_delayed_work(health->wq, &health->update_fw_log_ts_work, 0);
 }
 
 void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health)
@@ -862,6 +873,14 @@ void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health)
 	del_timer_sync(&health->timer);
 }
 
+void mlx5_start_health_fw_log_up(struct mlx5_core_dev *dev)
+{
+	struct mlx5_core_health *health = &dev->priv.health;
+
+	if (mlx5_core_is_pf(dev) && MLX5_CAP_MCAM_REG(dev, mrtc))
+		queue_delayed_work(health->wq, &health->update_fw_log_ts_work, 0);
+}
+
 void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
 {
 	struct mlx5_core_health *health = &dev->priv.health;
@@ -875,13 +894,6 @@ void mlx5_drain_health_wq(struct mlx5_core_dev *dev)
 	cancel_work_sync(&health->fatal_report_work);
 }
 
-void mlx5_health_flush(struct mlx5_core_dev *dev)
-{
-	struct mlx5_core_health *health = &dev->priv.health;
-
-	flush_workqueue(health->wq);
-}
-
 void mlx5_health_cleanup(struct mlx5_core_dev *dev)
 {
 	struct mlx5_core_health *health = &dev->priv.health;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c
index ac3757beaea2..c247cca154e9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c
@@ -32,6 +32,7 @@
 
 #include "en.h"
 #include "ipoib.h"
+#include "en/fs_ethtool.h"
 
 static void mlx5i_get_drvinfo(struct net_device *dev,
 			      struct ethtool_drvinfo *drvinfo)
@@ -39,7 +40,7 @@ static void mlx5i_get_drvinfo(struct net_device *dev,
 	struct mlx5e_priv *priv = mlx5i_epriv(dev);
 
 	mlx5e_ethtool_get_drvinfo(priv, drvinfo);
-	strlcpy(drvinfo->driver, KBUILD_MODNAME "[ib_ipoib]",
+	strscpy(drvinfo->driver, KBUILD_MODNAME "[ib_ipoib]",
 		sizeof(drvinfo->driver));
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
index c02b7b08fb4c..4e3a75496dd9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c
@@ -35,6 +35,7 @@
 #include "en.h"
 #include "en/params.h"
 #include "ipoib.h"
+#include "en/fs_ethtool.h"
 
 #define IB_DEFAULT_Q_KEY   0xb1b
 #define MLX5I_PARAMS_DEFAULT_LOG_RQ_SIZE 9
@@ -320,43 +321,47 @@ static void mlx5i_cleanup_tx(struct mlx5e_priv *priv)
 
 static int mlx5i_create_flow_steering(struct mlx5e_priv *priv)
 {
+	struct mlx5_flow_namespace *ns =
+		mlx5_get_flow_namespace(priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL);
 	int err;
 
-	priv->fs->ns = mlx5_get_flow_namespace(priv->mdev,
-					       MLX5_FLOW_NAMESPACE_KERNEL);
 
-	if (!priv->fs->ns)
+	if (!ns)
 		return -EINVAL;
 
-	err = mlx5e_arfs_create_tables(priv);
+	mlx5e_fs_set_ns(priv->fs, ns, false);
+	err = mlx5e_arfs_create_tables(priv->fs, priv->rx_res,
+				       !!(priv->netdev->hw_features & NETIF_F_NTUPLE));
 	if (err) {
 		netdev_err(priv->netdev, "Failed to create arfs tables, err=%d\n",
 			   err);
 		priv->netdev->hw_features &= ~NETIF_F_NTUPLE;
 	}
 
-	err = mlx5e_create_ttc_table(priv);
+	err = mlx5e_create_ttc_table(priv->fs, priv->rx_res);
 	if (err) {
 		netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n",
 			   err);
 		goto err_destroy_arfs_tables;
 	}
 
-	mlx5e_ethtool_init_steering(priv);
+	mlx5e_ethtool_init_steering(priv->fs);
 
 	return 0;
 
 err_destroy_arfs_tables:
-	mlx5e_arfs_destroy_tables(priv);
+	mlx5e_arfs_destroy_tables(priv->fs,
+				  !!(priv->netdev->hw_features & NETIF_F_NTUPLE));
 
 	return err;
 }
 
 static void mlx5i_destroy_flow_steering(struct mlx5e_priv *priv)
 {
-	mlx5e_destroy_ttc_table(priv);
-	mlx5e_arfs_destroy_tables(priv);
-	mlx5e_ethtool_cleanup_steering(priv);
+	mlx5e_destroy_ttc_table(priv->fs);
+	mlx5e_arfs_destroy_tables(priv->fs,
+				  !!(priv->netdev->hw_features & NETIF_F_NTUPLE));
+	mlx5e_ethtool_cleanup_steering(priv->fs);
 }
 
 static int mlx5i_init_rx(struct mlx5e_priv *priv)
@@ -458,7 +463,6 @@ static const struct mlx5e_profile mlx5i_nic_profile = {
 	.update_carrier    = NULL, /* no HW update in IB link */
 	.rx_handlers       = &mlx5i_rx_handlers,
 	.max_tc		   = MLX5I_MAX_NUM_TC,
-	.rq_groups	   = MLX5E_NUM_RQ_GROUPS(REGULAR),
 	.stats_grps        = mlx5i_stats_grps,
 	.stats_grps_num    = mlx5i_stats_grps_num,
 };
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c
index 0b86e78dbc0e..0227a521d301 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib_vlan.c
@@ -349,7 +349,6 @@ static const struct mlx5e_profile mlx5i_pkey_nic_profile = {
 	.update_stats	   = NULL,
 	.rx_handlers       = &mlx5i_rx_handlers,
 	.max_tc		   = MLX5I_MAX_NUM_TC,
-	.rq_groups	   = MLX5E_NUM_RQ_GROUPS(REGULAR),
 };
 
 const struct mlx5e_profile *mlx5i_pkey_get_profile(void)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
index 065102278cb8..a9f4ede4a9bf 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
@@ -65,6 +65,21 @@ static int get_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags)
 	return MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY;
 }
 
+static u8 lag_active_port_bits(struct mlx5_lag *ldev)
+{
+	u8 enabled_ports[MLX5_MAX_PORTS] = {};
+	u8 active_port = 0;
+	int num_enabled;
+	int idx;
+
+	mlx5_infer_tx_enabled(&ldev->tracker, ldev->ports, enabled_ports,
+			      &num_enabled);
+	for (idx = 0; idx < num_enabled; idx++)
+		active_port |= BIT_MASK(enabled_ports[idx]);
+
+	return active_port;
+}
+
 static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, u8 *ports, int mode,
 			       unsigned long flags)
 {
@@ -77,9 +92,21 @@ static int mlx5_cmd_create_lag(struct mlx5_core_dev *dev, u8 *ports, int mode,
 	lag_ctx = MLX5_ADDR_OF(create_lag_in, in, ctx);
 	MLX5_SET(create_lag_in, in, opcode, MLX5_CMD_OP_CREATE_LAG);
 	MLX5_SET(lagc, lag_ctx, fdb_selection_mode, fdb_sel_mode);
-	if (port_sel_mode == MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY) {
+
+	switch (port_sel_mode) {
+	case MLX5_LAG_PORT_SELECT_MODE_QUEUE_AFFINITY:
 		MLX5_SET(lagc, lag_ctx, tx_remap_affinity_1, ports[0]);
 		MLX5_SET(lagc, lag_ctx, tx_remap_affinity_2, ports[1]);
+		break;
+	case MLX5_LAG_PORT_SELECT_MODE_PORT_SELECT_FT:
+		if (!MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass))
+			break;
+
+		MLX5_SET(lagc, lag_ctx, active_port,
+			 lag_active_port_bits(mlx5_lag_dev(dev)));
+		break;
+	default:
+		break;
 	}
 	MLX5_SET(lagc, lag_ctx, port_select_mode, port_sel_mode);
 
@@ -386,12 +413,37 @@ static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev,
 	}
 }
 
+static int mlx5_cmd_modify_active_port(struct mlx5_core_dev *dev, u8 ports)
+{
+	u32 in[MLX5_ST_SZ_DW(modify_lag_in)] = {};
+	void *lag_ctx;
+
+	lag_ctx = MLX5_ADDR_OF(modify_lag_in, in, ctx);
+
+	MLX5_SET(modify_lag_in, in, opcode, MLX5_CMD_OP_MODIFY_LAG);
+	MLX5_SET(modify_lag_in, in, field_select, 0x2);
+
+	MLX5_SET(lagc, lag_ctx, active_port, ports);
+
+	return mlx5_cmd_exec_in(dev, modify_lag, in);
+}
+
 static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports)
 {
 	struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev;
+	u8 active_ports;
+	int ret;
+
+	if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags)) {
+		ret = mlx5_lag_port_sel_modify(ldev, ports);
+		if (ret ||
+		    !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table_bypass))
+			return ret;
 
-	if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags))
-		return mlx5_lag_port_sel_modify(ldev, ports);
+		active_ports = lag_active_port_bits(ldev);
+
+		return mlx5_cmd_modify_active_port(dev0, active_ports);
+	}
 	return mlx5_cmd_modify_lag(dev0, ldev->ports, ports);
 }
 
@@ -432,21 +484,22 @@ void mlx5_modify_lag(struct mlx5_lag *ldev,
 		mlx5_lag_drop_rule_setup(ldev, tracker);
 }
 
-#define MLX5_LAG_ROCE_HASH_PORTS_SUPPORTED 4
 static int mlx5_lag_set_port_sel_mode_roce(struct mlx5_lag *ldev,
 					   unsigned long *flags)
 {
-	struct lag_func *dev0 = &ldev->pf[MLX5_LAG_P1];
+	struct mlx5_core_dev *dev0 = ldev->pf[MLX5_LAG_P1].dev;
 
-	if (ldev->ports == MLX5_LAG_ROCE_HASH_PORTS_SUPPORTED) {
-		/* Four ports are support only in hash mode */
-		if (!MLX5_CAP_PORT_SELECTION(dev0->dev, port_select_flow_table))
-			return -EINVAL;
-		set_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, flags);
+	if (!MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table)) {
 		if (ldev->ports > 2)
-			ldev->buckets = MLX5_LAG_MAX_HASH_BUCKETS;
+			return -EINVAL;
+		return 0;
 	}
 
+	if (ldev->ports > 2)
+		ldev->buckets = MLX5_LAG_MAX_HASH_BUCKETS;
+
+	set_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, flags);
+
 	return 0;
 }
 
@@ -1275,6 +1328,22 @@ bool mlx5_lag_is_active(struct mlx5_core_dev *dev)
 }
 EXPORT_SYMBOL(mlx5_lag_is_active);
 
+bool mlx5_lag_mode_is_hash(struct mlx5_core_dev *dev)
+{
+	struct mlx5_lag *ldev;
+	unsigned long flags;
+	bool res = 0;
+
+	spin_lock_irqsave(&lag_lock, flags);
+	ldev = mlx5_lag_dev(dev);
+	if (ldev)
+		res = test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags);
+	spin_unlock_irqrestore(&lag_lock, flags);
+
+	return res;
+}
+EXPORT_SYMBOL(mlx5_lag_mode_is_hash);
+
 bool mlx5_lag_is_master(struct mlx5_core_dev *dev)
 {
 	struct mlx5_lag *ldev;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h
index b3bbf284fe71..d854e01d7fc5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h
@@ -11,7 +11,9 @@
 	(DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_BB))
 #define MLX5_ASO_WQEBBS_DATA \
 	(DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_BB))
+#define ASO_CTRL_READ_EN BIT(0)
 #define MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT 24
+#define MLX5_MACSEC_ASO_DS_CNT (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_DS))
 
 struct mlx5_wqe_aso_ctrl_seg {
 	__be32  va_h;
@@ -70,6 +72,7 @@ enum {
 
 enum {
 	MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER = 0x2,
+	MLX5_ACCESS_ASO_OPC_MOD_MACSEC = 0x5,
 };
 
 struct mlx5_aso;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
index 91e806c1aa21..d3a9ae80fd30 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c
@@ -65,6 +65,8 @@ enum {
 	MLX5_MTPPS_FS_TIME_STAMP		= BIT(0x4),
 	MLX5_MTPPS_FS_OUT_PULSE_DURATION	= BIT(0x5),
 	MLX5_MTPPS_FS_ENH_OUT_PER_ADJ		= BIT(0x7),
+	MLX5_MTPPS_FS_NPPS_PERIOD               = BIT(0x9),
+	MLX5_MTPPS_FS_OUT_PULSE_DURATION_NS     = BIT(0xa),
 };
 
 static bool mlx5_real_time_mode(struct mlx5_core_dev *mdev)
@@ -72,6 +74,13 @@ static bool mlx5_real_time_mode(struct mlx5_core_dev *mdev)
 	return (mlx5_is_real_time_rq(mdev) || mlx5_is_real_time_sq(mdev));
 }
 
+static bool mlx5_npps_real_time_supported(struct mlx5_core_dev *mdev)
+{
+	return (mlx5_real_time_mode(mdev) &&
+		MLX5_CAP_MCAM_FEATURE(mdev, npps_period) &&
+		MLX5_CAP_MCAM_FEATURE(mdev, out_pulse_duration_ns));
+}
+
 static bool mlx5_modify_mtutc_allowed(struct mlx5_core_dev *mdev)
 {
 	return MLX5_CAP_MCAM_FEATURE(mdev, ptpcyc2realtime_modify);
@@ -459,9 +468,95 @@ static u64 perout_conf_internal_timer(struct mlx5_core_dev *mdev, s64 sec)
 	return find_target_cycles(mdev, target_ns);
 }
 
-static u64 perout_conf_real_time(s64 sec)
+static u64 perout_conf_real_time(s64 sec, u32 nsec)
+{
+	return (u64)nsec | (u64)sec << 32;
+}
+
+static int perout_conf_1pps(struct mlx5_core_dev *mdev, struct ptp_clock_request *rq,
+			    u64 *time_stamp, bool real_time)
+{
+	struct timespec64 ts;
+	s64 ns;
+
+	ts.tv_nsec = rq->perout.period.nsec;
+	ts.tv_sec = rq->perout.period.sec;
+	ns = timespec64_to_ns(&ts);
+
+	if ((ns >> 1) != 500000000LL)
+		return -EINVAL;
+
+	*time_stamp = real_time ? perout_conf_real_time(rq->perout.start.sec, 0) :
+		      perout_conf_internal_timer(mdev, rq->perout.start.sec);
+
+	return 0;
+}
+
+#define MLX5_MAX_PULSE_DURATION (BIT(__mlx5_bit_sz(mtpps_reg, out_pulse_duration_ns)) - 1)
+static int mlx5_perout_conf_out_pulse_duration(struct mlx5_core_dev *mdev,
+					       struct ptp_clock_request *rq,
+					       u32 *out_pulse_duration_ns)
 {
-	return (u64)sec << 32;
+	struct mlx5_pps *pps_info = &mdev->clock.pps_info;
+	u32 out_pulse_duration;
+	struct timespec64 ts;
+
+	if (rq->perout.flags & PTP_PEROUT_DUTY_CYCLE) {
+		ts.tv_sec = rq->perout.on.sec;
+		ts.tv_nsec = rq->perout.on.nsec;
+		out_pulse_duration = (u32)timespec64_to_ns(&ts);
+	} else {
+		/* out_pulse_duration_ns should be up to 50% of the
+		 * pulse period as default
+		 */
+		ts.tv_sec = rq->perout.period.sec;
+		ts.tv_nsec = rq->perout.period.nsec;
+		out_pulse_duration = (u32)timespec64_to_ns(&ts) >> 1;
+	}
+
+	if (out_pulse_duration < pps_info->min_out_pulse_duration_ns ||
+	    out_pulse_duration > MLX5_MAX_PULSE_DURATION) {
+		mlx5_core_err(mdev, "NPPS pulse duration %u is not in [%llu, %lu]\n",
+			      out_pulse_duration, pps_info->min_out_pulse_duration_ns,
+			      MLX5_MAX_PULSE_DURATION);
+		return -EINVAL;
+	}
+	*out_pulse_duration_ns = out_pulse_duration;
+
+	return 0;
+}
+
+static int perout_conf_npps_real_time(struct mlx5_core_dev *mdev, struct ptp_clock_request *rq,
+				      u32 *field_select, u32 *out_pulse_duration_ns,
+				      u64 *period, u64 *time_stamp)
+{
+	struct mlx5_pps *pps_info = &mdev->clock.pps_info;
+	struct ptp_clock_time *time = &rq->perout.start;
+	struct timespec64 ts;
+
+	ts.tv_sec = rq->perout.period.sec;
+	ts.tv_nsec = rq->perout.period.nsec;
+	if (timespec64_to_ns(&ts) < pps_info->min_npps_period) {
+		mlx5_core_err(mdev, "NPPS period is lower than minimal npps period %llu\n",
+			      pps_info->min_npps_period);
+		return -EINVAL;
+	}
+	*period = perout_conf_real_time(rq->perout.period.sec, rq->perout.period.nsec);
+
+	if (mlx5_perout_conf_out_pulse_duration(mdev, rq, out_pulse_duration_ns))
+		return -EINVAL;
+
+	*time_stamp = perout_conf_real_time(time->sec, time->nsec);
+	*field_select |= MLX5_MTPPS_FS_NPPS_PERIOD |
+			 MLX5_MTPPS_FS_OUT_PULSE_DURATION_NS;
+
+	return 0;
+}
+
+static bool mlx5_perout_verify_flags(struct mlx5_core_dev *mdev, unsigned int flags)
+{
+	return ((!mlx5_npps_real_time_supported(mdev) && flags) ||
+		(mlx5_npps_real_time_supported(mdev) && flags & ~PTP_PEROUT_DUTY_CYCLE));
 }
 
 static int mlx5_perout_configure(struct ptp_clock_info *ptp,
@@ -474,20 +569,20 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp,
 			container_of(clock, struct mlx5_core_dev, clock);
 	bool rt_mode = mlx5_real_time_mode(mdev);
 	u32 in[MLX5_ST_SZ_DW(mtpps_reg)] = {0};
-	struct timespec64 ts;
+	u32 out_pulse_duration_ns = 0;
 	u32 field_select = 0;
+	u64 npps_period = 0;
 	u64 time_stamp = 0;
 	u8 pin_mode = 0;
 	u8 pattern = 0;
 	int pin = -1;
 	int err = 0;
-	s64 ns;
 
 	if (!MLX5_PPS_CAP(mdev))
 		return -EOPNOTSUPP;
 
 	/* Reject requests with unsupported flags */
-	if (rq->perout.flags)
+	if (mlx5_perout_verify_flags(mdev, rq->perout.flags))
 		return -EOPNOTSUPP;
 
 	if (rq->perout.index >= clock->ptp_info.n_pins)
@@ -500,29 +595,25 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp,
 
 	if (on) {
 		bool rt_mode = mlx5_real_time_mode(mdev);
-		s64 sec = rq->perout.start.sec;
-
-		if (rq->perout.start.nsec)
-			return -EINVAL;
 
 		pin_mode = MLX5_PIN_MODE_OUT;
 		pattern = MLX5_OUT_PATTERN_PERIODIC;
-		ts.tv_sec = rq->perout.period.sec;
-		ts.tv_nsec = rq->perout.period.nsec;
-		ns = timespec64_to_ns(&ts);
 
-		if ((ns >> 1) != 500000000LL)
+		if (rt_mode &&  rq->perout.start.sec > U32_MAX)
 			return -EINVAL;
 
-		if (rt_mode && sec > U32_MAX)
-			return -EINVAL;
-
-		time_stamp = rt_mode ? perout_conf_real_time(sec) :
-				       perout_conf_internal_timer(mdev, sec);
-
 		field_select |= MLX5_MTPPS_FS_PIN_MODE |
 				MLX5_MTPPS_FS_PATTERN |
 				MLX5_MTPPS_FS_TIME_STAMP;
+
+		if (mlx5_npps_real_time_supported(mdev))
+			err = perout_conf_npps_real_time(mdev, rq, &field_select,
+							 &out_pulse_duration_ns, &npps_period,
+							 &time_stamp);
+		else
+			err = perout_conf_1pps(mdev, rq, &time_stamp, rt_mode);
+		if (err)
+			return err;
 	}
 
 	MLX5_SET(mtpps_reg, in, pin, pin);
@@ -531,7 +622,8 @@ static int mlx5_perout_configure(struct ptp_clock_info *ptp,
 	MLX5_SET(mtpps_reg, in, enable, on);
 	MLX5_SET64(mtpps_reg, in, time_stamp, time_stamp);
 	MLX5_SET(mtpps_reg, in, field_select, field_select);
-
+	MLX5_SET64(mtpps_reg, in, npps_period, npps_period);
+	MLX5_SET(mtpps_reg, in, out_pulse_duration_ns, out_pulse_duration_ns);
 	err = mlx5_set_mtpps(mdev, in, sizeof(in));
 	if (err)
 		return err;
@@ -687,6 +779,13 @@ static void mlx5_get_pps_caps(struct mlx5_core_dev *mdev)
 	clock->ptp_info.n_per_out = MLX5_GET(mtpps_reg, out,
 					     cap_max_num_of_pps_out_pins);
 
+	if (MLX5_CAP_MCAM_FEATURE(mdev, npps_period))
+		clock->pps_info.min_npps_period = 1 << MLX5_GET(mtpps_reg, out,
+								cap_log_min_npps_period);
+	if (MLX5_CAP_MCAM_FEATURE(mdev, out_pulse_duration_ns))
+		clock->pps_info.min_out_pulse_duration_ns = 1 << MLX5_GET(mtpps_reg, out,
+								cap_log_min_out_pulse_duration_ns);
+
 	clock->pps_info.pin_caps[0] = MLX5_GET(mtpps_reg, out, cap_pin_0_mode);
 	clock->pps_info.pin_caps[1] = MLX5_GET(mtpps_reg, out, cap_pin_1_mode);
 	clock->pps_info.pin_caps[2] = MLX5_GET(mtpps_reg, out, cap_pin_2_mode);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h
index 2f536c5d30b1..032adb21ad4b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/mlx5.h
@@ -83,6 +83,7 @@ int mlx5_notifier_call_chain(struct mlx5_events *events, unsigned int event, voi
 enum {
 	MLX5_ACCEL_OBJ_TLS_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS,
 	MLX5_ACCEL_OBJ_IPSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC,
+	MLX5_ACCEL_OBJ_MACSEC_KEY = MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_MACSEC,
 };
 
 int mlx5_create_encryption_key(struct mlx5_core_dev *mdev,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 89b2d9cea33f..0b459d841c3a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -671,6 +671,33 @@ static int handle_hca_cap_roce(struct mlx5_core_dev *dev, void *set_ctx)
 	return err;
 }
 
+static int handle_hca_cap_port_selection(struct mlx5_core_dev *dev,
+					 void *set_ctx)
+{
+	void *set_hca_cap;
+	int err;
+
+	if (!MLX5_CAP_GEN(dev, port_selection_cap))
+		return 0;
+
+	err = mlx5_core_get_caps(dev, MLX5_CAP_PORT_SELECTION);
+	if (err)
+		return err;
+
+	if (MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass) ||
+	    !MLX5_CAP_PORT_SELECTION_MAX(dev, port_select_flow_table_bypass))
+		return 0;
+
+	set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
+	memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_PORT_SELECTION]->cur,
+	       MLX5_ST_SZ_BYTES(port_selection_cap));
+	MLX5_SET(port_selection_cap, set_hca_cap, port_select_flow_table_bypass, 1);
+
+	err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MODE_PORT_SELECTION);
+
+	return err;
+}
+
 static int set_hca_cap(struct mlx5_core_dev *dev)
 {
 	int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
@@ -715,6 +742,13 @@ static int set_hca_cap(struct mlx5_core_dev *dev)
 		goto out;
 	}
 
+	memset(set_ctx, 0, set_sz);
+	err = handle_hca_cap_port_selection(dev, set_ctx);
+	if (err) {
+		mlx5_core_err(dev, "handle_hca_cap_port_selection failed\n");
+		goto out;
+	}
+
 out:
 	kfree(set_ctx);
 	return err;
@@ -1058,7 +1092,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
 	mlx5_devcom_unregister_device(dev->priv.devcom);
 }
 
-static int mlx5_function_setup(struct mlx5_core_dev *dev, u64 timeout)
+static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot, u64 timeout)
 {
 	int err;
 
@@ -1096,10 +1130,12 @@ static int mlx5_function_setup(struct mlx5_core_dev *dev, u64 timeout)
 
 	mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_UP);
 
+	mlx5_start_health_poll(dev);
+
 	err = mlx5_core_enable_hca(dev, 0);
 	if (err) {
 		mlx5_core_err(dev, "enable hca failed\n");
-		goto err_cmd_cleanup;
+		goto stop_health_poll;
 	}
 
 	err = mlx5_core_set_issi(dev);
@@ -1151,8 +1187,7 @@ static int mlx5_function_setup(struct mlx5_core_dev *dev, u64 timeout)
 		mlx5_core_err(dev, "query hca failed\n");
 		goto reclaim_boot_pages;
 	}
-
-	mlx5_start_health_poll(dev);
+	mlx5_start_health_fw_log_up(dev);
 
 	return 0;
 
@@ -1160,6 +1195,8 @@ reclaim_boot_pages:
 	mlx5_reclaim_startup_pages(dev);
 err_disable_hca:
 	mlx5_core_disable_hca(dev, 0);
+stop_health_poll:
+	mlx5_stop_health_poll(dev, boot);
 err_cmd_cleanup:
 	mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
 	mlx5_cmd_cleanup(dev);
@@ -1171,7 +1208,6 @@ static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot)
 {
 	int err;
 
-	mlx5_stop_health_poll(dev, boot);
 	err = mlx5_cmd_teardown_hca(dev);
 	if (err) {
 		mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n");
@@ -1179,6 +1215,7 @@ static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot)
 	}
 	mlx5_reclaim_startup_pages(dev);
 	mlx5_core_disable_hca(dev, 0);
+	mlx5_stop_health_poll(dev, boot);
 	mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
 	mlx5_cmd_cleanup(dev);
 
@@ -1328,7 +1365,7 @@ int mlx5_init_one(struct mlx5_core_dev *dev)
 	mutex_lock(&dev->intf_state_mutex);
 	dev->state = MLX5_DEVICE_STATE_UP;
 
-	err = mlx5_function_setup(dev, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT));
+	err = mlx5_function_setup(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT));
 	if (err)
 		goto err_function;
 
@@ -1416,7 +1453,7 @@ int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery)
 		timeout = mlx5_tout_ms(dev, FW_PRE_INIT_ON_RECOVERY_TIMEOUT);
 	else
 		timeout = mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT);
-	err = mlx5_function_setup(dev, timeout);
+	err = mlx5_function_setup(dev, false, timeout);
 	if (err)
 		goto err_function;
 
@@ -1507,6 +1544,8 @@ static const int types[] = {
 	MLX5_CAP_IPSEC,
 	MLX5_CAP_PORT_SELECTION,
 	MLX5_CAP_DEV_SHAMPO,
+	MLX5_CAP_MACSEC,
+	MLX5_CAP_ADV_VIRTUALIZATION,
 };
 
 static void mlx5_hca_caps_free(struct mlx5_core_dev *dev)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index ad61b86d5769..a806e3de7b7c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -143,6 +143,36 @@ enum mlx5_semaphore_space_address {
 
 #define MLX5_DEFAULT_PROF       2
 
+static inline int mlx5_flexible_inlen(struct mlx5_core_dev *dev, size_t fixed,
+				      size_t item_size, size_t num_items,
+				      const char *func, int line)
+{
+	int inlen;
+
+	if (fixed > INT_MAX || item_size > INT_MAX || num_items > INT_MAX) {
+		mlx5_core_err(dev, "%s: %s:%d: input values too big: %zu + %zu * %zu\n",
+			      __func__, func, line, fixed, item_size, num_items);
+		return -ENOMEM;
+	}
+
+	if (check_mul_overflow((int)item_size, (int)num_items, &inlen)) {
+		mlx5_core_err(dev, "%s: %s:%d: multiplication overflow: %zu + %zu * %zu\n",
+			      __func__, func, line, fixed, item_size, num_items);
+		return -ENOMEM;
+	}
+
+	if (check_add_overflow((int)fixed, inlen, &inlen)) {
+		mlx5_core_err(dev, "%s: %s:%d: addition overflow: %zu + %zu * %zu\n",
+			      __func__, func, line, fixed, item_size, num_items);
+		return -ENOMEM;
+	}
+
+	return inlen;
+}
+
+#define MLX5_FLEXIBLE_INLEN(dev, fixed, item_size, num_items) \
+	mlx5_flexible_inlen(dev, fixed, item_size, num_items, __func__, __LINE__)
+
 int mlx5_query_hca_caps(struct mlx5_core_dev *dev);
 int mlx5_query_board_id(struct mlx5_core_dev *dev);
 int mlx5_cmd_init(struct mlx5_core_dev *dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/port.c b/drivers/net/ethernet/mellanox/mlx5/core/port.c
index e1bd54574ea5..a1548e6bfb35 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/port.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/port.c
@@ -493,29 +493,6 @@ int mlx5_query_port_vl_hw_cap(struct mlx5_core_dev *dev,
 }
 EXPORT_SYMBOL_GPL(mlx5_query_port_vl_hw_cap);
 
-int mlx5_core_query_ib_ppcnt(struct mlx5_core_dev *dev,
-			     u8 port_num, void *out, size_t sz)
-{
-	u32 *in;
-	int err;
-
-	in  = kvzalloc(sz, GFP_KERNEL);
-	if (!in) {
-		err = -ENOMEM;
-		return err;
-	}
-
-	MLX5_SET(ppcnt_reg, in, local_port, port_num);
-
-	MLX5_SET(ppcnt_reg, in, grp, MLX5_INFINIBAND_PORT_COUNTERS_GROUP);
-	err = mlx5_core_access_reg(dev, in, sz, out,
-				   sz, MLX5_REG_PPCNT, 0, 0);
-
-	kvfree(in);
-	return err;
-}
-EXPORT_SYMBOL_GPL(mlx5_core_query_ib_ppcnt);
-
 static int mlx5_query_pfcc_reg(struct mlx5_core_dev *dev, u32 *out,
 			       u32 out_size)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h
index 062c7c74a1f3..1777a1e508e7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h
@@ -1294,20 +1294,6 @@ struct mlx5dr_cmd_gid_attr {
 	u32 roce_ver;
 };
 
-struct mlx5dr_cmd_qp_create_attr {
-	u32 page_id;
-	u32 pdn;
-	u32 cqn;
-	u32 pm_state;
-	u32 service_type;
-	u32 buff_umem_id;
-	u32 db_umem_id;
-	u32 sq_wqe_cnt;
-	u32 rq_wqe_cnt;
-	u32 rq_wqe_shift;
-	u8 isolate_vl_tc:1;
-};
-
 int mlx5dr_cmd_query_gid(struct mlx5_core_dev *mdev, u8 vhca_port_num,
 			 u16 index, struct mlx5dr_cmd_gid_attr *attr);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.h
index 1fb185d6ac7f..d168622063d5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.h
@@ -14,10 +14,6 @@ struct mlx5_fs_dr_action {
 	struct mlx5dr_action *dr_action;
 };
 
-struct mlx5_fs_dr_ns {
-	struct mlx5_dr_ns *dr_ns;
-};
-
 struct mlx5_fs_dr_rule {
 	struct mlx5dr_rule    *dr_rule;
 	/* Only actions created by fs_dr */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wq.h b/drivers/net/ethernet/mellanox/mlx5/core/wq.h
index e5c4dcd1425e..4d629e5ddbc7 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/wq.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/wq.h
@@ -123,7 +123,7 @@ static inline void mlx5_wq_cyc_push(struct mlx5_wq_cyc *wq)
 	wq->cur_sz++;
 }
 
-static inline void mlx5_wq_cyc_push_n(struct mlx5_wq_cyc *wq, u8 n)
+static inline void mlx5_wq_cyc_push_n(struct mlx5_wq_cyc *wq, u16 n)
 {
 	wq->wqe_ctr += n;
 	wq->cur_sz += n;
diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c
index b03e1c66bac0..2292d63a279c 100644
--- a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c
+++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_main.c
@@ -156,7 +156,7 @@ static int mlxbf_gige_open(struct net_device *netdev)
 
 	phy_start(phydev);
 
-	netif_napi_add(netdev, &priv->napi, mlxbf_gige_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, &priv->napi, mlxbf_gige_poll);
 	napi_enable(&priv->napi);
 	netif_start_queue(netdev);
 
diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c
index 4aeb927c3715..aa780b1614a3 100644
--- a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c
+++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_mdio.c
@@ -246,8 +246,8 @@ int mlxbf_gige_mdio_probe(struct platform_device *pdev, struct mlxbf_gige *priv)
 	}
 
 	priv->clk_io = devm_ioremap(dev, res->start, resource_size(res));
-	if (IS_ERR(priv->clk_io))
-		return PTR_ERR(priv->clk_io);
+	if (!priv->clk_io)
+		return -ENOMEM;
 
 	mlxbf_gige_mdio_cfg(priv);
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/cmd.h b/drivers/net/ethernet/mellanox/mlxsw/cmd.h
index 60232fb8ccd7..09bef04b11d1 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/cmd.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/cmd.h
@@ -703,6 +703,9 @@ MLXSW_ITEM32(cmd_mbox, config_profile, max_vepa_channels, 0x10, 0, 8);
 
 /* cmd_mbox_config_profile_max_lag
  * Maximum number of LAG IDs requested.
+ * Reserved when Spectrum-1/2/3, supported from Spectrum-4 and above.
+ * For Spectrum-4, firmware sets 128 for values between 1-128 and 256 for values
+ * between 129-256.
  */
 MLXSW_ITEM32(cmd_mbox, config_profile, max_lag, 0x14, 0, 16);
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c
index 75553eb2c7f2..e2a985ec2c76 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.c
@@ -70,6 +70,8 @@ struct mlxsw_core {
 	struct workqueue_struct *emad_wq;
 	struct list_head rx_listener_list;
 	struct list_head event_listener_list;
+	struct list_head irq_event_handler_list;
+	struct mutex irq_event_handler_lock; /* Locks access to handlers list */
 	struct {
 		atomic64_t tid;
 		struct list_head trans_list;
@@ -184,6 +186,23 @@ unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core)
 }
 EXPORT_SYMBOL(mlxsw_core_max_ports);
 
+int mlxsw_core_max_lag(struct mlxsw_core *mlxsw_core, u16 *p_max_lag)
+{
+	struct mlxsw_driver *driver = mlxsw_core->driver;
+
+	if (driver->profile->used_max_lag) {
+		*p_max_lag = driver->profile->max_lag;
+		return 0;
+	}
+
+	if (!MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG))
+		return -EIO;
+
+	*p_max_lag = MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG);
+	return 0;
+}
+EXPORT_SYMBOL(mlxsw_core_max_lag);
+
 void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core)
 {
 	return mlxsw_core->driver_priv;
@@ -633,7 +652,7 @@ static void mlxsw_emad_process_string_tlv(const struct sk_buff *skb,
 		return;
 
 	string = mlxsw_emad_string_tlv_string_data(string_tlv);
-	strlcpy(trans->emad_err_string, string,
+	strscpy(trans->emad_err_string, string,
 		MLXSW_EMAD_STRING_TLV_STRING_LEN);
 }
 
@@ -1305,21 +1324,6 @@ mlxsw_devlink_sb_pool_set(struct devlink *devlink,
 					 extack);
 }
 
-static int mlxsw_devlink_port_type_set(struct devlink_port *devlink_port,
-				       enum devlink_port_type port_type)
-{
-	struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
-	struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
-	struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
-
-	if (!mlxsw_driver->port_type_set)
-		return -EOPNOTSUPP;
-
-	return mlxsw_driver->port_type_set(mlxsw_core,
-					   mlxsw_core_port->local_port,
-					   port_type);
-}
-
 static int mlxsw_devlink_sb_port_pool_get(struct devlink_port *devlink_port,
 					  unsigned int sb_index, u16 pool_index,
 					  u32 *p_threshold)
@@ -1650,7 +1654,6 @@ static const struct devlink_ops mlxsw_devlink_ops = {
 				  BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE),
 	.reload_down		= mlxsw_devlink_core_bus_device_reload_down,
 	.reload_up		= mlxsw_devlink_core_bus_device_reload_up,
-	.port_type_set			= mlxsw_devlink_port_type_set,
 	.port_split			= mlxsw_devlink_port_split,
 	.port_unsplit			= mlxsw_devlink_port_unsplit,
 	.sb_pool_get			= mlxsw_devlink_sb_pool_get,
@@ -2090,6 +2093,18 @@ static void mlxsw_core_health_fini(struct mlxsw_core *mlxsw_core)
 	devlink_health_reporter_destroy(mlxsw_core->health.fw_fatal);
 }
 
+static void mlxsw_core_irq_event_handler_init(struct mlxsw_core *mlxsw_core)
+{
+	INIT_LIST_HEAD(&mlxsw_core->irq_event_handler_list);
+	mutex_init(&mlxsw_core->irq_event_handler_lock);
+}
+
+static void mlxsw_core_irq_event_handler_fini(struct mlxsw_core *mlxsw_core)
+{
+	mutex_destroy(&mlxsw_core->irq_event_handler_lock);
+	WARN_ON(!list_empty(&mlxsw_core->irq_event_handler_list));
+}
+
 static int
 __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 				 const struct mlxsw_bus *mlxsw_bus,
@@ -2101,6 +2116,7 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 	struct mlxsw_core *mlxsw_core;
 	struct mlxsw_driver *mlxsw_driver;
 	size_t alloc_size;
+	u16 max_lag;
 	int err;
 
 	mlxsw_driver = mlxsw_core_driver_get(device_kind);
@@ -2125,6 +2141,7 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 	mlxsw_core->bus = mlxsw_bus;
 	mlxsw_core->bus_priv = bus_priv;
 	mlxsw_core->bus_info = mlxsw_bus_info;
+	mlxsw_core_irq_event_handler_init(mlxsw_core);
 
 	err = mlxsw_bus->init(bus_priv, mlxsw_core, mlxsw_driver->profile,
 			      &mlxsw_core->res);
@@ -2141,10 +2158,9 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
 	if (err)
 		goto err_ports_init;
 
-	if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG) &&
-	    MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) {
-		alloc_size = sizeof(*mlxsw_core->lag.mapping) *
-			MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG) *
+	err = mlxsw_core_max_lag(mlxsw_core, &max_lag);
+	if (!err && MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) {
+		alloc_size = sizeof(*mlxsw_core->lag.mapping) * max_lag *
 			MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS);
 		mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL);
 		if (!mlxsw_core->lag.mapping) {
@@ -2233,6 +2249,7 @@ err_ports_init:
 err_register_resources:
 	mlxsw_bus->fini(bus_priv);
 err_bus_init:
+	mlxsw_core_irq_event_handler_fini(mlxsw_core);
 	if (!reload) {
 		devl_unlock(devlink);
 		devlink_free(devlink);
@@ -2302,6 +2319,7 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core,
 	if (!reload)
 		devl_resources_unregister(devlink);
 	mlxsw_core->bus->fini(mlxsw_core->bus_priv);
+	mlxsw_core_irq_event_handler_fini(mlxsw_core);
 	if (!reload) {
 		devl_unlock(devlink);
 		devlink_free(devlink);
@@ -2772,6 +2790,57 @@ int mlxsw_reg_trans_bulk_wait(struct list_head *bulk_list)
 }
 EXPORT_SYMBOL(mlxsw_reg_trans_bulk_wait);
 
+struct mlxsw_core_irq_event_handler_item {
+	struct list_head list;
+	void (*cb)(struct mlxsw_core *mlxsw_core);
+};
+
+int mlxsw_core_irq_event_handler_register(struct mlxsw_core *mlxsw_core,
+					  mlxsw_irq_event_cb_t cb)
+{
+	struct mlxsw_core_irq_event_handler_item *item;
+
+	item = kzalloc(sizeof(*item), GFP_KERNEL);
+	if (!item)
+		return -ENOMEM;
+	item->cb = cb;
+	mutex_lock(&mlxsw_core->irq_event_handler_lock);
+	list_add_tail(&item->list, &mlxsw_core->irq_event_handler_list);
+	mutex_unlock(&mlxsw_core->irq_event_handler_lock);
+	return 0;
+}
+EXPORT_SYMBOL(mlxsw_core_irq_event_handler_register);
+
+void mlxsw_core_irq_event_handler_unregister(struct mlxsw_core *mlxsw_core,
+					     mlxsw_irq_event_cb_t cb)
+{
+	struct mlxsw_core_irq_event_handler_item *item, *tmp;
+
+	mutex_lock(&mlxsw_core->irq_event_handler_lock);
+	list_for_each_entry_safe(item, tmp,
+				 &mlxsw_core->irq_event_handler_list, list) {
+		if (item->cb == cb) {
+			list_del(&item->list);
+			kfree(item);
+		}
+	}
+	mutex_unlock(&mlxsw_core->irq_event_handler_lock);
+}
+EXPORT_SYMBOL(mlxsw_core_irq_event_handler_unregister);
+
+void mlxsw_core_irq_event_handlers_call(struct mlxsw_core *mlxsw_core)
+{
+	struct mlxsw_core_irq_event_handler_item *item;
+
+	mutex_lock(&mlxsw_core->irq_event_handler_lock);
+	list_for_each_entry(item, &mlxsw_core->irq_event_handler_list, list) {
+		if (item->cb)
+			item->cb(mlxsw_core);
+	}
+	mutex_unlock(&mlxsw_core->irq_event_handler_lock);
+}
+EXPORT_SYMBOL(mlxsw_core_irq_event_handlers_call);
+
 static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core,
 				     const struct mlxsw_reg_info *reg,
 				     char *payload,
@@ -3115,18 +3184,6 @@ void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u16 local_port,
 }
 EXPORT_SYMBOL(mlxsw_core_port_eth_set);
 
-void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u16 local_port,
-			    void *port_driver_priv)
-{
-	struct mlxsw_core_port *mlxsw_core_port =
-					&mlxsw_core->ports[local_port];
-	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
-
-	mlxsw_core_port->port_driver_priv = port_driver_priv;
-	devlink_port_type_ib_set(devlink_port, NULL);
-}
-EXPORT_SYMBOL(mlxsw_core_port_ib_set);
-
 void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u16 local_port,
 			   void *port_driver_priv)
 {
@@ -3139,18 +3196,6 @@ void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u16 local_port,
 }
 EXPORT_SYMBOL(mlxsw_core_port_clear);
 
-enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core,
-						u16 local_port)
-{
-	struct mlxsw_core_port *mlxsw_core_port =
-					&mlxsw_core->ports[local_port];
-	struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
-
-	return devlink_port->type;
-}
-EXPORT_SYMBOL(mlxsw_core_port_type_get);
-
-
 struct devlink_port *
 mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core,
 				 u16 local_port)
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h
index 02d9cc2ef0c8..ca0c3d2bee6b 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/core.h
@@ -35,6 +35,8 @@ struct mlxsw_fw_rev;
 
 unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core);
 
+int mlxsw_core_max_lag(struct mlxsw_core *mlxsw_core, u16 *p_max_lag);
+
 void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core);
 
 struct mlxsw_linecards *mlxsw_core_linecards(struct mlxsw_core *mlxsw_core);
@@ -215,6 +217,14 @@ int mlxsw_reg_trans_write(struct mlxsw_core *mlxsw_core,
 			  mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv);
 int mlxsw_reg_trans_bulk_wait(struct list_head *bulk_list);
 
+typedef void mlxsw_irq_event_cb_t(struct mlxsw_core *mlxsw_core);
+
+int mlxsw_core_irq_event_handler_register(struct mlxsw_core *mlxsw_core,
+					  mlxsw_irq_event_cb_t cb);
+void mlxsw_core_irq_event_handler_unregister(struct mlxsw_core *mlxsw_core,
+					     mlxsw_irq_event_cb_t cb);
+void mlxsw_core_irq_event_handlers_call(struct mlxsw_core *mlxsw_core);
+
 int mlxsw_reg_query(struct mlxsw_core *mlxsw_core,
 		    const struct mlxsw_reg_info *reg, char *payload);
 int mlxsw_reg_write(struct mlxsw_core *mlxsw_core,
@@ -256,12 +266,8 @@ int mlxsw_core_cpu_port_init(struct mlxsw_core *mlxsw_core,
 void mlxsw_core_cpu_port_fini(struct mlxsw_core *mlxsw_core);
 void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u16 local_port,
 			     void *port_driver_priv, struct net_device *dev);
-void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u16 local_port,
-			    void *port_driver_priv);
 void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u16 local_port,
 			   void *port_driver_priv);
-enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core,
-						u16 local_port);
 struct devlink_port *
 mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core,
 				 u16 local_port);
@@ -291,6 +297,7 @@ struct mlxsw_swid_config {
 
 struct mlxsw_config_profile {
 	u16	used_max_vepa_channels:1,
+		used_max_lag:1,
 		used_max_mid:1,
 		used_max_pgt:1,
 		used_max_system_port:1,
@@ -306,6 +313,7 @@ struct mlxsw_config_profile {
 		used_kvd_sizes:1,
 		used_cqe_time_stamp_type:1;
 	u8	max_vepa_channels;
+	u16	max_lag;
 	u16	max_mid;
 	u16	max_pgt;
 	u16	max_system_port;
@@ -341,8 +349,6 @@ struct mlxsw_driver {
 		    const struct mlxsw_bus_info *mlxsw_bus_info,
 		    struct netlink_ext_ack *extack);
 	void (*fini)(struct mlxsw_core *mlxsw_core);
-	int (*port_type_set)(struct mlxsw_core *mlxsw_core, u16 local_port,
-			     enum devlink_port_type new_type);
 	int (*port_split)(struct mlxsw_core *mlxsw_core, u16 local_port,
 			  unsigned int count, struct netlink_ext_ack *extack);
 	int (*port_unsplit)(struct mlxsw_core *mlxsw_core, u16 local_port,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c
index 636db9a87457..9dfe7148199f 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c
@@ -737,8 +737,9 @@ mlxsw_afa_cookie_create(struct mlxsw_afa *mlxsw_afa,
 	if (!cookie)
 		return ERR_PTR(-ENOMEM);
 	refcount_set(&cookie->ref_count, 1);
-	memcpy(&cookie->fa_cookie, fa_cookie,
-	       sizeof(*fa_cookie) + fa_cookie->cookie_len);
+	cookie->fa_cookie = *fa_cookie;
+	memcpy(cookie->fa_cookie.cookie, fa_cookie->cookie,
+	       fa_cookie->cookie_len);
 
 	err = rhashtable_insert_fast(&mlxsw_afa->cookie_ht, &cookie->ht_node,
 				     mlxsw_afa_cookie_ht_params);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c b/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c
index ca59f0b946da..83d2dc91ba2c 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core_linecards.c
@@ -785,6 +785,21 @@ static int mlxsw_linecard_status_get_and_process(struct mlxsw_core *mlxsw_core,
 	return mlxsw_linecard_status_process(linecards, linecard, mddq_pl);
 }
 
+static void mlxsw_linecards_irq_event_handler(struct mlxsw_core *mlxsw_core)
+{
+	struct mlxsw_linecards *linecards = mlxsw_core_linecards(mlxsw_core);
+	int i;
+
+	/* Handle change of line card active state. */
+	for (i = 0; i < linecards->count; i++) {
+		struct mlxsw_linecard *linecard = mlxsw_linecard_get(linecards,
+								     i + 1);
+
+		mlxsw_linecard_status_get_and_process(mlxsw_core, linecards,
+						      linecard);
+	}
+}
+
 static const char * const mlxsw_linecard_status_event_type_name[] = {
 	[MLXSW_LINECARD_STATUS_EVENT_TYPE_PROVISION] = "provision",
 	[MLXSW_LINECARD_STATUS_EVENT_TYPE_UNPROVISION] = "unprovision",
@@ -1238,7 +1253,6 @@ static int mlxsw_linecard_init(struct mlxsw_core *mlxsw_core,
 {
 	struct devlink_linecard *devlink_linecard;
 	struct mlxsw_linecard *linecard;
-	int err;
 
 	linecard = mlxsw_linecard_get(linecards, slot_index);
 	linecard->slot_index = slot_index;
@@ -1248,17 +1262,45 @@ static int mlxsw_linecard_init(struct mlxsw_core *mlxsw_core,
 	devlink_linecard = devlink_linecard_create(priv_to_devlink(mlxsw_core),
 						   slot_index, &mlxsw_linecard_ops,
 						   linecard);
-	if (IS_ERR(devlink_linecard)) {
-		err = PTR_ERR(devlink_linecard);
-		goto err_devlink_linecard_create;
-	}
+	if (IS_ERR(devlink_linecard))
+		return PTR_ERR(devlink_linecard);
+
 	linecard->devlink_linecard = devlink_linecard;
 	INIT_DELAYED_WORK(&linecard->status_event_to_dw,
 			  &mlxsw_linecard_status_event_to_work);
 
+	return 0;
+}
+
+static void mlxsw_linecard_fini(struct mlxsw_core *mlxsw_core,
+				struct mlxsw_linecards *linecards,
+				u8 slot_index)
+{
+	struct mlxsw_linecard *linecard;
+
+	linecard = mlxsw_linecard_get(linecards, slot_index);
+	cancel_delayed_work_sync(&linecard->status_event_to_dw);
+	/* Make sure all scheduled events are processed */
+	mlxsw_core_flush_owq();
+	if (linecard->active)
+		mlxsw_linecard_active_clear(linecard);
+	mlxsw_linecard_bdev_del(linecard);
+	devlink_linecard_destroy(linecard->devlink_linecard);
+	mutex_destroy(&linecard->lock);
+}
+
+static int
+mlxsw_linecard_event_delivery_init(struct mlxsw_core *mlxsw_core,
+				   struct mlxsw_linecards *linecards,
+				   u8 slot_index)
+{
+	struct mlxsw_linecard *linecard;
+	int err;
+
+	linecard = mlxsw_linecard_get(linecards, slot_index);
 	err = mlxsw_linecard_event_delivery_set(mlxsw_core, linecard, true);
 	if (err)
-		goto err_event_delivery_set;
+		return err;
 
 	err = mlxsw_linecard_status_get_and_process(mlxsw_core, linecards,
 						    linecard);
@@ -1269,29 +1311,18 @@ static int mlxsw_linecard_init(struct mlxsw_core *mlxsw_core,
 
 err_status_get_and_process:
 	mlxsw_linecard_event_delivery_set(mlxsw_core, linecard, false);
-err_event_delivery_set:
-	devlink_linecard_destroy(linecard->devlink_linecard);
-err_devlink_linecard_create:
-	mutex_destroy(&linecard->lock);
 	return err;
 }
 
-static void mlxsw_linecard_fini(struct mlxsw_core *mlxsw_core,
-				struct mlxsw_linecards *linecards,
-				u8 slot_index)
+static void
+mlxsw_linecard_event_delivery_fini(struct mlxsw_core *mlxsw_core,
+				   struct mlxsw_linecards *linecards,
+				   u8 slot_index)
 {
 	struct mlxsw_linecard *linecard;
 
 	linecard = mlxsw_linecard_get(linecards, slot_index);
 	mlxsw_linecard_event_delivery_set(mlxsw_core, linecard, false);
-	cancel_delayed_work_sync(&linecard->status_event_to_dw);
-	/* Make sure all scheduled events are processed */
-	mlxsw_core_flush_owq();
-	if (linecard->active)
-		mlxsw_linecard_active_clear(linecard);
-	mlxsw_linecard_bdev_del(linecard);
-	devlink_linecard_destroy(linecard->devlink_linecard);
-	mutex_destroy(&linecard->lock);
 }
 
 /*       LINECARDS INI BUNDLE FILE
@@ -1505,6 +1536,11 @@ int mlxsw_linecards_init(struct mlxsw_core *mlxsw_core,
 	if (err)
 		goto err_traps_register;
 
+	err = mlxsw_core_irq_event_handler_register(mlxsw_core,
+						    mlxsw_linecards_irq_event_handler);
+	if (err)
+		goto err_irq_event_handler_register;
+
 	mlxsw_core_linecards_set(mlxsw_core, linecards);
 
 	for (i = 0; i < linecards->count; i++) {
@@ -1513,11 +1549,25 @@ int mlxsw_linecards_init(struct mlxsw_core *mlxsw_core,
 			goto err_linecard_init;
 	}
 
+	for (i = 0; i < linecards->count; i++) {
+		err = mlxsw_linecard_event_delivery_init(mlxsw_core, linecards,
+							 i + 1);
+		if (err)
+			goto err_linecard_event_delivery_init;
+	}
+
 	return 0;
 
+err_linecard_event_delivery_init:
+	for (i--; i >= 0; i--)
+		mlxsw_linecard_event_delivery_fini(mlxsw_core, linecards, i + 1);
+	i = linecards->count;
 err_linecard_init:
 	for (i--; i >= 0; i--)
 		mlxsw_linecard_fini(mlxsw_core, linecards, i + 1);
+	mlxsw_core_irq_event_handler_unregister(mlxsw_core,
+						mlxsw_linecards_irq_event_handler);
+err_irq_event_handler_register:
 	mlxsw_core_traps_unregister(mlxsw_core, mlxsw_linecard_listener,
 				    ARRAY_SIZE(mlxsw_linecard_listener),
 				    mlxsw_core);
@@ -1536,7 +1586,11 @@ void mlxsw_linecards_fini(struct mlxsw_core *mlxsw_core)
 	if (!linecards)
 		return;
 	for (i = 0; i < linecards->count; i++)
+		mlxsw_linecard_event_delivery_fini(mlxsw_core, linecards, i + 1);
+	for (i = 0; i < linecards->count; i++)
 		mlxsw_linecard_fini(mlxsw_core, linecards, i + 1);
+	mlxsw_core_irq_event_handler_unregister(mlxsw_core,
+						mlxsw_linecards_irq_event_handler);
 	mlxsw_core_traps_unregister(mlxsw_core, mlxsw_linecard_listener,
 				    ARRAY_SIZE(mlxsw_linecard_listener),
 				    mlxsw_core);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
index 3548fe1df7c8..987fe5c9d5a3 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/core_thermal.c
@@ -21,7 +21,6 @@
 #define MLXSW_THERMAL_ASIC_TEMP_HOT	105000	/* 105C */
 #define MLXSW_THERMAL_HYSTERESIS_TEMP	5000	/* 5C */
 #define MLXSW_THERMAL_MODULE_TEMP_SHIFT	(MLXSW_THERMAL_HYSTERESIS_TEMP * 2)
-#define MLXSW_THERMAL_TEMP_SCORE_MAX	GENMASK(31, 0)
 #define MLXSW_THERMAL_MAX_STATE	10
 #define MLXSW_THERMAL_MIN_STATE	2
 #define MLXSW_THERMAL_MAX_DUTY	255
@@ -101,8 +100,6 @@ struct mlxsw_thermal {
 	struct thermal_cooling_device *cdevs[MLXSW_MFCR_PWMS_MAX];
 	u8 cooling_levels[MLXSW_THERMAL_MAX_STATE + 1];
 	struct mlxsw_thermal_trip trips[MLXSW_THERMAL_NUM_TRIPS];
-	unsigned int tz_highest_score;
-	struct thermal_zone_device *tz_highest_dev;
 	struct mlxsw_thermal_area line_cards[];
 };
 
@@ -193,34 +190,6 @@ mlxsw_thermal_module_trips_update(struct device *dev, struct mlxsw_core *core,
 	return 0;
 }
 
-static void mlxsw_thermal_tz_score_update(struct mlxsw_thermal *thermal,
-					  struct thermal_zone_device *tzdev,
-					  struct mlxsw_thermal_trip *trips,
-					  int temp)
-{
-	struct mlxsw_thermal_trip *trip = trips;
-	unsigned int score, delta, i, shift = 1;
-
-	/* Calculate thermal zone score, if temperature is above the hot
-	 * threshold score is set to MLXSW_THERMAL_TEMP_SCORE_MAX.
-	 */
-	score = MLXSW_THERMAL_TEMP_SCORE_MAX;
-	for (i = MLXSW_THERMAL_TEMP_TRIP_NORM; i < MLXSW_THERMAL_NUM_TRIPS;
-	     i++, trip++) {
-		if (temp < trip->temp) {
-			delta = DIV_ROUND_CLOSEST(temp, trip->temp - temp);
-			score = delta * shift;
-			break;
-		}
-		shift *= 256;
-	}
-
-	if (score > thermal->tz_highest_score) {
-		thermal->tz_highest_score = score;
-		thermal->tz_highest_dev = tzdev;
-	}
-}
-
 static int mlxsw_thermal_bind(struct thermal_zone_device *tzdev,
 			      struct thermal_cooling_device *cdev)
 {
@@ -286,9 +255,6 @@ static int mlxsw_thermal_get_temp(struct thermal_zone_device *tzdev,
 		return err;
 	}
 	mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL, NULL, NULL);
-	if (temp > 0)
-		mlxsw_thermal_tz_score_update(thermal, tzdev, thermal->trips,
-					      temp);
 
 	*p_temp = temp;
 	return 0;
@@ -349,21 +315,6 @@ static int mlxsw_thermal_set_trip_hyst(struct thermal_zone_device *tzdev,
 	return 0;
 }
 
-static int mlxsw_thermal_trend_get(struct thermal_zone_device *tzdev,
-				   int trip, enum thermal_trend *trend)
-{
-	struct mlxsw_thermal *thermal = tzdev->devdata;
-
-	if (trip < 0 || trip >= MLXSW_THERMAL_NUM_TRIPS)
-		return -EINVAL;
-
-	if (tzdev == thermal->tz_highest_dev)
-		return 1;
-
-	*trend = THERMAL_TREND_STABLE;
-	return 0;
-}
-
 static struct thermal_zone_params mlxsw_thermal_params = {
 	.no_hwmon = true,
 };
@@ -377,7 +328,6 @@ static struct thermal_zone_device_ops mlxsw_thermal_ops = {
 	.set_trip_temp	= mlxsw_thermal_set_trip_temp,
 	.get_trip_hyst	= mlxsw_thermal_get_trip_hyst,
 	.set_trip_hyst	= mlxsw_thermal_set_trip_hyst,
-	.get_trend	= mlxsw_thermal_trend_get,
 };
 
 static int mlxsw_thermal_module_bind(struct thermal_zone_device *tzdev,
@@ -463,7 +413,6 @@ static int mlxsw_thermal_module_temp_get(struct thermal_zone_device *tzdev,
 	int temp, crit_temp, emerg_temp;
 	struct device *dev;
 	u16 sensor_index;
-	int err;
 
 	dev = thermal->bus_info->dev;
 	sensor_index = MLXSW_REG_MTMP_MODULE_INDEX_MIN + tz->module;
@@ -479,10 +428,8 @@ static int mlxsw_thermal_module_temp_get(struct thermal_zone_device *tzdev,
 		return 0;
 
 	/* Update trip points. */
-	err = mlxsw_thermal_module_trips_update(dev, thermal->core, tz,
-						crit_temp, emerg_temp);
-	if (!err && temp > 0)
-		mlxsw_thermal_tz_score_update(thermal, tzdev, tz->trips, temp);
+	mlxsw_thermal_module_trips_update(dev, thermal->core, tz,
+					  crit_temp, emerg_temp);
 
 	return 0;
 }
@@ -546,22 +493,6 @@ mlxsw_thermal_module_trip_hyst_set(struct thermal_zone_device *tzdev, int trip,
 	return 0;
 }
 
-static int mlxsw_thermal_module_trend_get(struct thermal_zone_device *tzdev,
-					  int trip, enum thermal_trend *trend)
-{
-	struct mlxsw_thermal_module *tz = tzdev->devdata;
-	struct mlxsw_thermal *thermal = tz->parent;
-
-	if (trip < 0 || trip >= MLXSW_THERMAL_NUM_TRIPS)
-		return -EINVAL;
-
-	if (tzdev == thermal->tz_highest_dev)
-		return 1;
-
-	*trend = THERMAL_TREND_STABLE;
-	return 0;
-}
-
 static struct thermal_zone_device_ops mlxsw_thermal_module_ops = {
 	.bind		= mlxsw_thermal_module_bind,
 	.unbind		= mlxsw_thermal_module_unbind,
@@ -571,7 +502,6 @@ static struct thermal_zone_device_ops mlxsw_thermal_module_ops = {
 	.set_trip_temp	= mlxsw_thermal_module_trip_temp_set,
 	.get_trip_hyst	= mlxsw_thermal_module_trip_hyst_get,
 	.set_trip_hyst	= mlxsw_thermal_module_trip_hyst_set,
-	.get_trend	= mlxsw_thermal_module_trend_get,
 };
 
 static int mlxsw_thermal_gearbox_temp_get(struct thermal_zone_device *tzdev,
@@ -592,8 +522,6 @@ static int mlxsw_thermal_gearbox_temp_get(struct thermal_zone_device *tzdev,
 		return err;
 
 	mlxsw_reg_mtmp_unpack(mtmp_pl, &temp, NULL, NULL, NULL, NULL);
-	if (temp > 0)
-		mlxsw_thermal_tz_score_update(thermal, tzdev, tz->trips, temp);
 
 	*p_temp = temp;
 	return 0;
@@ -608,7 +536,6 @@ static struct thermal_zone_device_ops mlxsw_thermal_gearbox_ops = {
 	.set_trip_temp	= mlxsw_thermal_module_trip_temp_set,
 	.get_trip_hyst	= mlxsw_thermal_module_trip_hyst_get,
 	.set_trip_hyst	= mlxsw_thermal_module_trip_hyst_set,
-	.get_trend	= mlxsw_thermal_module_trend_get,
 };
 
 static int mlxsw_thermal_get_max_state(struct thermal_cooling_device *cdev,
diff --git a/drivers/net/ethernet/mellanox/mlxsw/i2c.c b/drivers/net/ethernet/mellanox/mlxsw/i2c.c
index ce843ea91464..f5f5f8dc3d19 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/i2c.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/i2c.c
@@ -9,6 +9,7 @@
 #include <linux/mutex.h>
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
+#include <linux/platform_data/mlxreg.h>
 #include <linux/slab.h>
 
 #include "cmd.h"
@@ -51,6 +52,15 @@
 #define MLXSW_I2C_TIMEOUT_MSECS		5000
 #define MLXSW_I2C_MAX_DATA_SIZE		256
 
+/* Driver can be initialized by kernel platform driver or from the user
+ * space. In the first case IRQ line number is passed through the platform
+ * data, otherwise default IRQ line is to be used. Default IRQ is relevant
+ * only for specific I2C slave address, allowing 3.4 MHz I2C path to the chip
+ * (special hardware feature for I2C acceleration).
+ */
+#define MLXSW_I2C_DEFAULT_IRQ		17
+#define MLXSW_FAST_I2C_SLAVE		0x37
+
 /**
  * struct mlxsw_i2c - device private data:
  * @cmd: command attributes;
@@ -63,6 +73,9 @@
  * @core: switch core pointer;
  * @bus_info: bus info block;
  * @block_size: maximum block size allowed to pass to under layer;
+ * @pdata: device platform data;
+ * @irq_work: interrupts work item;
+ * @irq: IRQ line number;
  */
 struct mlxsw_i2c {
 	struct {
@@ -76,6 +89,9 @@ struct mlxsw_i2c {
 	struct mlxsw_core *core;
 	struct mlxsw_bus_info bus_info;
 	u16 block_size;
+	struct mlxreg_core_hotplug_platform_data *pdata;
+	struct work_struct irq_work;
+	int irq;
 };
 
 #define MLXSW_I2C_READ_MSG(_client, _addr_buf, _buf, _len) {	\
@@ -546,6 +562,67 @@ static void mlxsw_i2c_fini(void *bus_priv)
 	mlxsw_i2c->core = NULL;
 }
 
+static void mlxsw_i2c_work_handler(struct work_struct *work)
+{
+	struct mlxsw_i2c *mlxsw_i2c;
+
+	mlxsw_i2c = container_of(work, struct mlxsw_i2c, irq_work);
+	mlxsw_core_irq_event_handlers_call(mlxsw_i2c->core);
+}
+
+static irqreturn_t mlxsw_i2c_irq_handler(int irq, void *dev)
+{
+	struct mlxsw_i2c *mlxsw_i2c = dev;
+
+	mlxsw_core_schedule_work(&mlxsw_i2c->irq_work);
+
+	/* Interrupt handler shares IRQ line with 'main' interrupt handler.
+	 * Return here IRQ_NONE, while main handler will return IRQ_HANDLED.
+	 */
+	return IRQ_NONE;
+}
+
+static int mlxsw_i2c_irq_init(struct mlxsw_i2c *mlxsw_i2c, u8 addr)
+{
+	int err;
+
+	/* Initialize interrupt handler if system hotplug driver is reachable,
+	 * otherwise interrupt line is not enabled and interrupts will not be
+	 * raised to CPU. Also request_irq() call will be not valid.
+	 */
+	if (!IS_REACHABLE(CONFIG_MLXREG_HOTPLUG))
+		return 0;
+
+	/* Set default interrupt line. */
+	if (mlxsw_i2c->pdata && mlxsw_i2c->pdata->irq)
+		mlxsw_i2c->irq = mlxsw_i2c->pdata->irq;
+	else if (addr == MLXSW_FAST_I2C_SLAVE)
+		mlxsw_i2c->irq = MLXSW_I2C_DEFAULT_IRQ;
+
+	if (!mlxsw_i2c->irq)
+		return 0;
+
+	INIT_WORK(&mlxsw_i2c->irq_work, mlxsw_i2c_work_handler);
+	err = request_irq(mlxsw_i2c->irq, mlxsw_i2c_irq_handler,
+			  IRQF_TRIGGER_FALLING | IRQF_SHARED, "mlxsw-i2c",
+			  mlxsw_i2c);
+	if (err) {
+		dev_err(mlxsw_i2c->bus_info.dev, "Failed to request irq: %d\n",
+			err);
+		return err;
+	}
+
+	return 0;
+}
+
+static void mlxsw_i2c_irq_fini(struct mlxsw_i2c *mlxsw_i2c)
+{
+	if (!IS_REACHABLE(CONFIG_MLXREG_HOTPLUG) || !mlxsw_i2c->irq)
+		return;
+	cancel_work_sync(&mlxsw_i2c->irq_work);
+	free_irq(mlxsw_i2c->irq, mlxsw_i2c);
+}
+
 static const struct mlxsw_bus mlxsw_i2c_bus = {
 	.kind			= "i2c",
 	.init			= mlxsw_i2c_init,
@@ -638,17 +715,24 @@ static int mlxsw_i2c_probe(struct i2c_client *client,
 	mlxsw_i2c->bus_info.dev = &client->dev;
 	mlxsw_i2c->bus_info.low_frequency = true;
 	mlxsw_i2c->dev = &client->dev;
+	mlxsw_i2c->pdata = client->dev.platform_data;
+
+	err = mlxsw_i2c_irq_init(mlxsw_i2c, client->addr);
+	if (err)
+		goto errout;
 
 	err = mlxsw_core_bus_device_register(&mlxsw_i2c->bus_info,
 					     &mlxsw_i2c_bus, mlxsw_i2c, false,
 					     NULL, NULL);
 	if (err) {
 		dev_err(&client->dev, "Fail to register core bus\n");
-		return err;
+		goto err_bus_device_register;
 	}
 
 	return 0;
 
+err_bus_device_register:
+	mlxsw_i2c_irq_fini(mlxsw_i2c);
 errout:
 	mutex_destroy(&mlxsw_i2c->cmd.lock);
 	i2c_set_clientdata(client, NULL);
@@ -656,14 +740,13 @@ errout:
 	return err;
 }
 
-static int mlxsw_i2c_remove(struct i2c_client *client)
+static void mlxsw_i2c_remove(struct i2c_client *client)
 {
 	struct mlxsw_i2c *mlxsw_i2c = i2c_get_clientdata(client);
 
 	mlxsw_core_bus_device_unregister(mlxsw_i2c->core, false);
+	mlxsw_i2c_irq_fini(mlxsw_i2c);
 	mutex_destroy(&mlxsw_i2c->cmd.lock);
-
-	return 0;
 }
 
 int mlxsw_i2c_driver_register(struct i2c_driver *i2c_driver)
diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c
index bb1cd4bae82e..55b3c42bb007 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c
@@ -26,20 +26,29 @@ static const struct mlxsw_fw_rev mlxsw_m_fw_rev = {
 
 struct mlxsw_m_port;
 
+struct mlxsw_m_line_card {
+	bool active;
+	int module_to_port[];
+};
+
 struct mlxsw_m {
 	struct mlxsw_m_port **ports;
-	int *module_to_port;
 	struct mlxsw_core *core;
 	const struct mlxsw_bus_info *bus_info;
 	u8 base_mac[ETH_ALEN];
 	u8 max_ports;
+	u8 max_modules_per_slot; /* Maximum number of modules per-slot. */
+	u8 num_of_slots; /* Including the main board. */
+	struct mlxsw_m_line_card **line_cards;
 };
 
 struct mlxsw_m_port {
 	struct net_device *dev;
 	struct mlxsw_m *mlxsw_m;
 	u16 local_port;
+	u8 slot_index;
 	u8 module;
+	u8 module_offset;
 };
 
 static int mlxsw_m_base_mac_get(struct mlxsw_m *mlxsw_m)
@@ -94,14 +103,14 @@ static void mlxsw_m_module_get_drvinfo(struct net_device *dev,
 	struct mlxsw_m_port *mlxsw_m_port = netdev_priv(dev);
 	struct mlxsw_m *mlxsw_m = mlxsw_m_port->mlxsw_m;
 
-	strlcpy(drvinfo->driver, mlxsw_m->bus_info->device_kind,
+	strscpy(drvinfo->driver, mlxsw_m->bus_info->device_kind,
 		sizeof(drvinfo->driver));
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
 		 "%d.%d.%d",
 		 mlxsw_m->bus_info->fw_rev.major,
 		 mlxsw_m->bus_info->fw_rev.minor,
 		 mlxsw_m->bus_info->fw_rev.subminor);
-	strlcpy(drvinfo->bus_info, mlxsw_m->bus_info->device_name,
+	strscpy(drvinfo->bus_info, mlxsw_m->bus_info->device_name,
 		sizeof(drvinfo->bus_info));
 }
 
@@ -111,8 +120,9 @@ static int mlxsw_m_get_module_info(struct net_device *netdev,
 	struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev);
 	struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core;
 
-	return mlxsw_env_get_module_info(netdev, core, 0, mlxsw_m_port->module,
-					 modinfo);
+	return mlxsw_env_get_module_info(netdev, core,
+					 mlxsw_m_port->slot_index,
+					 mlxsw_m_port->module, modinfo);
 }
 
 static int
@@ -122,7 +132,8 @@ mlxsw_m_get_module_eeprom(struct net_device *netdev, struct ethtool_eeprom *ee,
 	struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev);
 	struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core;
 
-	return mlxsw_env_get_module_eeprom(netdev, core, 0,
+	return mlxsw_env_get_module_eeprom(netdev, core,
+					   mlxsw_m_port->slot_index,
 					   mlxsw_m_port->module, ee, data);
 }
 
@@ -134,7 +145,8 @@ mlxsw_m_get_module_eeprom_by_page(struct net_device *netdev,
 	struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev);
 	struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core;
 
-	return mlxsw_env_get_module_eeprom_by_page(core, 0,
+	return mlxsw_env_get_module_eeprom_by_page(core,
+						   mlxsw_m_port->slot_index,
 						   mlxsw_m_port->module,
 						   page, extack);
 }
@@ -144,7 +156,8 @@ static int mlxsw_m_reset(struct net_device *netdev, u32 *flags)
 	struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev);
 	struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core;
 
-	return mlxsw_env_reset_module(netdev, core, 0, mlxsw_m_port->module,
+	return mlxsw_env_reset_module(netdev, core, mlxsw_m_port->slot_index,
+				      mlxsw_m_port->module,
 				      flags);
 }
 
@@ -156,7 +169,8 @@ mlxsw_m_get_module_power_mode(struct net_device *netdev,
 	struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev);
 	struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core;
 
-	return mlxsw_env_get_module_power_mode(core, 0, mlxsw_m_port->module,
+	return mlxsw_env_get_module_power_mode(core, mlxsw_m_port->slot_index,
+					       mlxsw_m_port->module,
 					       params, extack);
 }
 
@@ -168,7 +182,8 @@ mlxsw_m_set_module_power_mode(struct net_device *netdev,
 	struct mlxsw_m_port *mlxsw_m_port = netdev_priv(netdev);
 	struct mlxsw_core *core = mlxsw_m_port->mlxsw_m->core;
 
-	return mlxsw_env_set_module_power_mode(core, 0, mlxsw_m_port->module,
+	return mlxsw_env_set_module_power_mode(core, mlxsw_m_port->slot_index,
+					       mlxsw_m_port->module,
 					       params->policy, extack);
 }
 
@@ -184,7 +199,7 @@ static const struct ethtool_ops mlxsw_m_port_ethtool_ops = {
 
 static int
 mlxsw_m_port_module_info_get(struct mlxsw_m *mlxsw_m, u16 local_port,
-			     u8 *p_module, u8 *p_width)
+			     u8 *p_module, u8 *p_width, u8 *p_slot_index)
 {
 	char pmlp_pl[MLXSW_REG_PMLP_LEN];
 	int err;
@@ -195,6 +210,7 @@ mlxsw_m_port_module_info_get(struct mlxsw_m *mlxsw_m, u16 local_port,
 		return err;
 	*p_module = mlxsw_reg_pmlp_module_get(pmlp_pl, 0);
 	*p_width = mlxsw_reg_pmlp_width_get(pmlp_pl);
+	*p_slot_index = mlxsw_reg_pmlp_slot_index_get(pmlp_pl, 0);
 
 	return 0;
 }
@@ -212,18 +228,25 @@ mlxsw_m_port_dev_addr_get(struct mlxsw_m_port *mlxsw_m_port)
 	if (err)
 		return err;
 	mlxsw_reg_ppad_mac_memcpy_from(ppad_pl, addr);
-	eth_hw_addr_gen(mlxsw_m_port->dev, addr, mlxsw_m_port->module + 1);
+	eth_hw_addr_gen(mlxsw_m_port->dev, addr, mlxsw_m_port->module + 1 +
+			mlxsw_m_port->module_offset);
 	return 0;
 }
 
+static bool mlxsw_m_port_created(struct mlxsw_m *mlxsw_m, u16 local_port)
+{
+	return mlxsw_m->ports[local_port];
+}
+
 static int
-mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u16 local_port, u8 module)
+mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u16 local_port, u8 slot_index,
+		    u8 module)
 {
 	struct mlxsw_m_port *mlxsw_m_port;
 	struct net_device *dev;
 	int err;
 
-	err = mlxsw_core_port_init(mlxsw_m->core, local_port, 0,
+	err = mlxsw_core_port_init(mlxsw_m->core, local_port, slot_index,
 				   module + 1, false, 0, false,
 				   0, mlxsw_m->base_mac,
 				   sizeof(mlxsw_m->base_mac));
@@ -246,6 +269,15 @@ mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u16 local_port, u8 module)
 	mlxsw_m_port->mlxsw_m = mlxsw_m;
 	mlxsw_m_port->local_port = local_port;
 	mlxsw_m_port->module = module;
+	mlxsw_m_port->slot_index = slot_index;
+	/* Add module offset for line card. Offset for main board iz zero.
+	 * For line card in slot #n offset is calculated as (#n - 1)
+	 * multiplied by maximum modules number, which could be found on a line
+	 * card.
+	 */
+	mlxsw_m_port->module_offset = mlxsw_m_port->slot_index ?
+				      (mlxsw_m_port->slot_index - 1) *
+				      mlxsw_m->max_modules_per_slot : 0;
 
 	dev->netdev_ops = &mlxsw_m_port_netdev_ops;
 	dev->ethtool_ops = &mlxsw_m_port_ethtool_ops;
@@ -291,19 +323,29 @@ static void mlxsw_m_port_remove(struct mlxsw_m *mlxsw_m, u16 local_port)
 	mlxsw_core_port_fini(mlxsw_m->core, local_port);
 }
 
+static int*
+mlxsw_m_port_mapping_get(struct mlxsw_m *mlxsw_m, u8 slot_index, u8 module)
+{
+	return &mlxsw_m->line_cards[slot_index]->module_to_port[module];
+}
+
 static int mlxsw_m_port_module_map(struct mlxsw_m *mlxsw_m, u16 local_port,
 				   u8 *last_module)
 {
 	unsigned int max_ports = mlxsw_core_max_ports(mlxsw_m->core);
-	u8 module, width;
+	u8 module, width, slot_index;
+	int *module_to_port;
 	int err;
 
 	/* Fill out to local port mapping array */
 	err = mlxsw_m_port_module_info_get(mlxsw_m, local_port, &module,
-					   &width);
+					   &width, &slot_index);
 	if (err)
 		return err;
 
+	/* Skip if line card has been already configured */
+	if (mlxsw_m->line_cards[slot_index]->active)
+		return 0;
 	if (!width)
 		return 0;
 	/* Skip, if port belongs to the cluster */
@@ -313,91 +355,220 @@ static int mlxsw_m_port_module_map(struct mlxsw_m *mlxsw_m, u16 local_port,
 
 	if (WARN_ON_ONCE(module >= max_ports))
 		return -EINVAL;
-	mlxsw_env_module_port_map(mlxsw_m->core, 0, module);
-	mlxsw_m->module_to_port[module] = ++mlxsw_m->max_ports;
+	mlxsw_env_module_port_map(mlxsw_m->core, slot_index, module);
+	module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, slot_index, module);
+	*module_to_port = local_port;
 
 	return 0;
 }
 
-static void mlxsw_m_port_module_unmap(struct mlxsw_m *mlxsw_m, u8 module)
+static void
+mlxsw_m_port_module_unmap(struct mlxsw_m *mlxsw_m, u8 slot_index, u8 module)
 {
-	mlxsw_m->module_to_port[module] = -1;
-	mlxsw_env_module_port_unmap(mlxsw_m->core, 0, module);
+	int *module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, slot_index,
+						       module);
+	*module_to_port = -1;
+	mlxsw_env_module_port_unmap(mlxsw_m->core, slot_index, module);
 }
 
-static int mlxsw_m_ports_create(struct mlxsw_m *mlxsw_m)
+static int mlxsw_m_linecards_init(struct mlxsw_m *mlxsw_m)
 {
 	unsigned int max_ports = mlxsw_core_max_ports(mlxsw_m->core);
-	u8 last_module = max_ports;
-	int i;
-	int err;
+	char mgpir_pl[MLXSW_REG_MGPIR_LEN];
+	u8 num_of_modules;
+	int i, j, err;
+
+	mlxsw_reg_mgpir_pack(mgpir_pl, 0);
+	err = mlxsw_reg_query(mlxsw_m->core, MLXSW_REG(mgpir), mgpir_pl);
+	if (err)
+		return err;
+
+	mlxsw_reg_mgpir_unpack(mgpir_pl, NULL, NULL, NULL, &num_of_modules,
+			       &mlxsw_m->num_of_slots);
+	/* If the system is modular, get the maximum number of modules per-slot.
+	 * Otherwise, get the maximum number of modules on the main board.
+	 */
+	if (mlxsw_m->num_of_slots)
+		mlxsw_m->max_modules_per_slot =
+			mlxsw_reg_mgpir_max_modules_per_slot_get(mgpir_pl);
+	else
+		mlxsw_m->max_modules_per_slot = num_of_modules;
+	/* Add slot for main board. */
+	mlxsw_m->num_of_slots += 1;
 
 	mlxsw_m->ports = kcalloc(max_ports, sizeof(*mlxsw_m->ports),
 				 GFP_KERNEL);
 	if (!mlxsw_m->ports)
 		return -ENOMEM;
 
-	mlxsw_m->module_to_port = kmalloc_array(max_ports, sizeof(int),
-						GFP_KERNEL);
-	if (!mlxsw_m->module_to_port) {
+	mlxsw_m->line_cards = kcalloc(mlxsw_m->num_of_slots,
+				      sizeof(*mlxsw_m->line_cards),
+				      GFP_KERNEL);
+	if (!mlxsw_m->line_cards) {
 		err = -ENOMEM;
-		goto err_module_to_port_alloc;
+		goto err_kcalloc;
 	}
 
-	/* Invalidate the entries of module to local port mapping array */
-	for (i = 0; i < max_ports; i++)
-		mlxsw_m->module_to_port[i] = -1;
+	for (i = 0; i < mlxsw_m->num_of_slots; i++) {
+		mlxsw_m->line_cards[i] =
+			kzalloc(struct_size(mlxsw_m->line_cards[i],
+					    module_to_port,
+					    mlxsw_m->max_modules_per_slot),
+				GFP_KERNEL);
+		if (!mlxsw_m->line_cards[i]) {
+			err = -ENOMEM;
+			goto err_kmalloc_array;
+		}
 
-	/* Fill out module to local port mapping array */
-	for (i = 1; i < max_ports; i++) {
-		err = mlxsw_m_port_module_map(mlxsw_m, i, &last_module);
-		if (err)
-			goto err_module_to_port_map;
+		/* Invalidate the entries of module to local port mapping array. */
+		for (j = 0; j < mlxsw_m->max_modules_per_slot; j++)
+			mlxsw_m->line_cards[i]->module_to_port[j] = -1;
 	}
 
-	/* Create port objects for each valid entry */
-	for (i = 0; i < mlxsw_m->max_ports; i++) {
-		if (mlxsw_m->module_to_port[i] > 0) {
-			err = mlxsw_m_port_create(mlxsw_m,
-						  mlxsw_m->module_to_port[i],
-						  i);
+	return 0;
+
+err_kmalloc_array:
+	for (i--; i >= 0; i--)
+		kfree(mlxsw_m->line_cards[i]);
+err_kcalloc:
+	kfree(mlxsw_m->ports);
+	return err;
+}
+
+static void mlxsw_m_linecards_fini(struct mlxsw_m *mlxsw_m)
+{
+	int i = mlxsw_m->num_of_slots;
+
+	for (i--; i >= 0; i--)
+		kfree(mlxsw_m->line_cards[i]);
+	kfree(mlxsw_m->line_cards);
+	kfree(mlxsw_m->ports);
+}
+
+static void
+mlxsw_m_linecard_port_module_unmap(struct mlxsw_m *mlxsw_m, u8 slot_index)
+{
+	int i;
+
+	for (i = mlxsw_m->max_modules_per_slot - 1; i >= 0; i--) {
+		int *module_to_port;
+
+		module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, slot_index, i);
+		if (*module_to_port > 0)
+			mlxsw_m_port_module_unmap(mlxsw_m, slot_index, i);
+	}
+}
+
+static int
+mlxsw_m_linecard_ports_create(struct mlxsw_m *mlxsw_m, u8 slot_index)
+{
+	int *module_to_port;
+	int i, err;
+
+	for (i = 0; i < mlxsw_m->max_modules_per_slot; i++) {
+		module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, slot_index, i);
+		if (*module_to_port > 0) {
+			err = mlxsw_m_port_create(mlxsw_m, *module_to_port,
+						  slot_index, i);
 			if (err)
-				goto err_module_to_port_create;
+				goto err_port_create;
+			/* Mark slot as active */
+			if (!mlxsw_m->line_cards[slot_index]->active)
+				mlxsw_m->line_cards[slot_index]->active = true;
 		}
 	}
-
 	return 0;
 
-err_module_to_port_create:
+err_port_create:
 	for (i--; i >= 0; i--) {
-		if (mlxsw_m->module_to_port[i] > 0)
-			mlxsw_m_port_remove(mlxsw_m,
-					    mlxsw_m->module_to_port[i]);
+		module_to_port = mlxsw_m_port_mapping_get(mlxsw_m, slot_index, i);
+		if (*module_to_port > 0 &&
+		    mlxsw_m_port_created(mlxsw_m, *module_to_port)) {
+			mlxsw_m_port_remove(mlxsw_m, *module_to_port);
+			/* Mark slot as inactive */
+			if (mlxsw_m->line_cards[slot_index]->active)
+				mlxsw_m->line_cards[slot_index]->active = false;
+		}
 	}
-	i = max_ports;
-err_module_to_port_map:
-	for (i--; i > 0; i--)
-		mlxsw_m_port_module_unmap(mlxsw_m, i);
-	kfree(mlxsw_m->module_to_port);
-err_module_to_port_alloc:
-	kfree(mlxsw_m->ports);
 	return err;
 }
 
-static void mlxsw_m_ports_remove(struct mlxsw_m *mlxsw_m)
+static void
+mlxsw_m_linecard_ports_remove(struct mlxsw_m *mlxsw_m, u8 slot_index)
 {
 	int i;
 
-	for (i = 0; i < mlxsw_m->max_ports; i++) {
-		if (mlxsw_m->module_to_port[i] > 0) {
-			mlxsw_m_port_remove(mlxsw_m,
-					    mlxsw_m->module_to_port[i]);
-			mlxsw_m_port_module_unmap(mlxsw_m, i);
+	for (i = 0; i < mlxsw_m->max_modules_per_slot; i++) {
+		int *module_to_port = mlxsw_m_port_mapping_get(mlxsw_m,
+							       slot_index, i);
+
+		if (*module_to_port > 0 &&
+		    mlxsw_m_port_created(mlxsw_m, *module_to_port)) {
+			mlxsw_m_port_remove(mlxsw_m, *module_to_port);
+			mlxsw_m_port_module_unmap(mlxsw_m, slot_index, i);
 		}
 	}
+}
 
-	kfree(mlxsw_m->module_to_port);
-	kfree(mlxsw_m->ports);
+static int mlxsw_m_ports_module_map(struct mlxsw_m *mlxsw_m)
+{
+	unsigned int max_ports = mlxsw_core_max_ports(mlxsw_m->core);
+	u8 last_module = max_ports;
+	int i, err;
+
+	for (i = 1; i < max_ports; i++) {
+		err = mlxsw_m_port_module_map(mlxsw_m, i, &last_module);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static int mlxsw_m_ports_create(struct mlxsw_m *mlxsw_m)
+{
+	int err;
+
+	/* Fill out module to local port mapping array */
+	err = mlxsw_m_ports_module_map(mlxsw_m);
+	if (err)
+		goto err_ports_module_map;
+
+	/* Create port objects for each valid entry */
+	err = mlxsw_m_linecard_ports_create(mlxsw_m, 0);
+	if (err)
+		goto err_linecard_ports_create;
+
+	return 0;
+
+err_linecard_ports_create:
+err_ports_module_map:
+	mlxsw_m_linecard_port_module_unmap(mlxsw_m, 0);
+
+	return err;
+}
+
+static void mlxsw_m_ports_remove(struct mlxsw_m *mlxsw_m)
+{
+	mlxsw_m_linecard_ports_remove(mlxsw_m, 0);
+}
+
+static void
+mlxsw_m_ports_remove_selected(struct mlxsw_core *mlxsw_core,
+			      bool (*selector)(void *priv, u16 local_port),
+			      void *priv)
+{
+	struct mlxsw_m *mlxsw_m = mlxsw_core_driver_priv(mlxsw_core);
+	struct mlxsw_linecard *linecard_priv = priv;
+	struct mlxsw_m_line_card *linecard;
+
+	linecard = mlxsw_m->line_cards[linecard_priv->slot_index];
+
+	if (WARN_ON(!linecard->active))
+		return;
+
+	mlxsw_m_linecard_ports_remove(mlxsw_m, linecard_priv->slot_index);
+	linecard->active = false;
 }
 
 static int mlxsw_m_fw_rev_validate(struct mlxsw_m *mlxsw_m)
@@ -418,6 +589,60 @@ static int mlxsw_m_fw_rev_validate(struct mlxsw_m *mlxsw_m)
 	return -EINVAL;
 }
 
+static void
+mlxsw_m_got_active(struct mlxsw_core *mlxsw_core, u8 slot_index, void *priv)
+{
+	struct mlxsw_m_line_card *linecard;
+	struct mlxsw_m *mlxsw_m = priv;
+	int err;
+
+	linecard = mlxsw_m->line_cards[slot_index];
+	/* Skip if line card has been already configured during init */
+	if (linecard->active)
+		return;
+
+	/* Fill out module to local port mapping array */
+	err = mlxsw_m_ports_module_map(mlxsw_m);
+	if (err)
+		goto err_ports_module_map;
+
+	/* Create port objects for each valid entry */
+	err = mlxsw_m_linecard_ports_create(mlxsw_m, slot_index);
+	if (err) {
+		dev_err(mlxsw_m->bus_info->dev, "Failed to create port for line card at slot %d\n",
+			slot_index);
+		goto err_linecard_ports_create;
+	}
+
+	linecard->active = true;
+
+	return;
+
+err_linecard_ports_create:
+err_ports_module_map:
+	mlxsw_m_linecard_port_module_unmap(mlxsw_m, slot_index);
+}
+
+static void
+mlxsw_m_got_inactive(struct mlxsw_core *mlxsw_core, u8 slot_index, void *priv)
+{
+	struct mlxsw_m_line_card *linecard;
+	struct mlxsw_m *mlxsw_m = priv;
+
+	linecard = mlxsw_m->line_cards[slot_index];
+
+	if (WARN_ON(!linecard->active))
+		return;
+
+	mlxsw_m_linecard_ports_remove(mlxsw_m, slot_index);
+	linecard->active = false;
+}
+
+static struct mlxsw_linecards_event_ops mlxsw_m_event_ops = {
+	.got_active = mlxsw_m_got_active,
+	.got_inactive = mlxsw_m_got_inactive,
+};
+
 static int mlxsw_m_init(struct mlxsw_core *mlxsw_core,
 			const struct mlxsw_bus_info *mlxsw_bus_info,
 			struct netlink_ext_ack *extack)
@@ -438,13 +663,33 @@ static int mlxsw_m_init(struct mlxsw_core *mlxsw_core,
 		return err;
 	}
 
+	err = mlxsw_m_linecards_init(mlxsw_m);
+	if (err) {
+		dev_err(mlxsw_m->bus_info->dev, "Failed to create line cards\n");
+		return err;
+	}
+
+	err = mlxsw_linecards_event_ops_register(mlxsw_core,
+						 &mlxsw_m_event_ops, mlxsw_m);
+	if (err) {
+		dev_err(mlxsw_m->bus_info->dev, "Failed to register line cards operations\n");
+		goto linecards_event_ops_register;
+	}
+
 	err = mlxsw_m_ports_create(mlxsw_m);
 	if (err) {
 		dev_err(mlxsw_m->bus_info->dev, "Failed to create ports\n");
-		return err;
+		goto err_ports_create;
 	}
 
 	return 0;
+
+err_ports_create:
+	mlxsw_linecards_event_ops_unregister(mlxsw_core,
+					     &mlxsw_m_event_ops, mlxsw_m);
+linecards_event_ops_register:
+	mlxsw_m_linecards_fini(mlxsw_m);
+	return err;
 }
 
 static void mlxsw_m_fini(struct mlxsw_core *mlxsw_core)
@@ -452,6 +697,9 @@ static void mlxsw_m_fini(struct mlxsw_core *mlxsw_core)
 	struct mlxsw_m *mlxsw_m = mlxsw_core_driver_priv(mlxsw_core);
 
 	mlxsw_m_ports_remove(mlxsw_m);
+	mlxsw_linecards_event_ops_unregister(mlxsw_core,
+					     &mlxsw_m_event_ops, mlxsw_m);
+	mlxsw_m_linecards_fini(mlxsw_m);
 }
 
 static const struct mlxsw_config_profile mlxsw_m_config_profile;
@@ -461,6 +709,7 @@ static struct mlxsw_driver mlxsw_m_driver = {
 	.priv_size		= sizeof(struct mlxsw_m),
 	.init			= mlxsw_m_init,
 	.fini			= mlxsw_m_fini,
+	.ports_remove_selected	= mlxsw_m_ports_remove_selected,
 	.profile		= &mlxsw_m_config_profile,
 };
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c
index 50527adc5b5a..c968309657dd 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/pci.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c
@@ -1187,6 +1187,11 @@ static int mlxsw_pci_config_profile(struct mlxsw_pci *mlxsw_pci, char *mbox,
 		mlxsw_cmd_mbox_config_profile_max_vepa_channels_set(
 			mbox, profile->max_vepa_channels);
 	}
+	if (profile->used_max_lag) {
+		mlxsw_cmd_mbox_config_profile_set_max_lag_set(mbox, 1);
+		mlxsw_cmd_mbox_config_profile_max_lag_set(mbox,
+							  profile->max_lag);
+	}
 	if (profile->used_max_mid) {
 		mlxsw_cmd_mbox_config_profile_set_max_mid_set(
 			mbox, 1);
diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h
index f27bdecdf952..0777bed5bb1a 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/reg.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h
@@ -2218,76 +2218,6 @@ static inline void mlxsw_reg_smpe_pack(char *payload, u16 local_port,
 	mlxsw_reg_smpe_evid_set(payload, evid);
 }
 
-/* SFTR-V2 - Switch Flooding Table Version 2 Register
- * --------------------------------------------------
- * The switch flooding table is used for flooding packet replication. The table
- * defines a bit mask of ports for packet replication.
- */
-#define MLXSW_REG_SFTR2_ID 0x202F
-#define MLXSW_REG_SFTR2_LEN 0x120
-
-MLXSW_REG_DEFINE(sftr2, MLXSW_REG_SFTR2_ID, MLXSW_REG_SFTR2_LEN);
-
-/* reg_sftr2_swid
- * Switch partition ID with which to associate the port.
- * Access: Index
- */
-MLXSW_ITEM32(reg, sftr2, swid, 0x00, 24, 8);
-
-/* reg_sftr2_flood_table
- * Flooding table index to associate with the specific type on the specific
- * switch partition.
- * Access: Index
- */
-MLXSW_ITEM32(reg, sftr2, flood_table, 0x00, 16, 6);
-
-/* reg_sftr2_index
- * Index. Used as an index into the Flooding Table in case the table is
- * configured to use VID / FID or FID Offset.
- * Access: Index
- */
-MLXSW_ITEM32(reg, sftr2, index, 0x00, 0, 16);
-
-/* reg_sftr2_table_type
- * See mlxsw_flood_table_type
- * Access: RW
- */
-MLXSW_ITEM32(reg, sftr2, table_type, 0x04, 16, 3);
-
-/* reg_sftr2_range
- * Range of entries to update
- * Access: Index
- */
-MLXSW_ITEM32(reg, sftr2, range, 0x04, 0, 16);
-
-/* reg_sftr2_port
- * Local port membership (1 bit per port).
- * Access: RW
- */
-MLXSW_ITEM_BIT_ARRAY(reg, sftr2, port, 0x20, 0x80, 1);
-
-/* reg_sftr2_port_mask
- * Local port mask (1 bit per port).
- * Access: WO
- */
-MLXSW_ITEM_BIT_ARRAY(reg, sftr2, port_mask, 0xA0, 0x80, 1);
-
-static inline void mlxsw_reg_sftr2_pack(char *payload,
-					unsigned int flood_table,
-					unsigned int index,
-					enum mlxsw_flood_table_type table_type,
-					unsigned int range, u16 port, bool set)
-{
-	MLXSW_REG_ZERO(sftr2, payload);
-	mlxsw_reg_sftr2_swid_set(payload, 0);
-	mlxsw_reg_sftr2_flood_table_set(payload, flood_table);
-	mlxsw_reg_sftr2_index_set(payload, index);
-	mlxsw_reg_sftr2_table_type_set(payload, table_type);
-	mlxsw_reg_sftr2_range_set(payload, range);
-	mlxsw_reg_sftr2_port_set(payload, port, set);
-	mlxsw_reg_sftr2_port_mask_set(payload, port, 1);
-}
-
 /* SMID-V2 - Switch Multicast ID Version 2 Register
  * ------------------------------------------------
  * The MID record maps from a MID (Multicast ID), which is a unique identifier
@@ -4729,25 +4659,6 @@ MLXSW_ITEM32(reg, ptys, ext_eth_proto_cap, 0x08, 0, 32);
  */
 MLXSW_ITEM32(reg, ptys, eth_proto_cap, 0x0C, 0, 32);
 
-/* reg_ptys_ib_link_width_cap
- * IB port supported widths.
- * Access: RO
- */
-MLXSW_ITEM32(reg, ptys, ib_link_width_cap, 0x10, 16, 16);
-
-#define MLXSW_REG_PTYS_IB_SPEED_SDR	BIT(0)
-#define MLXSW_REG_PTYS_IB_SPEED_DDR	BIT(1)
-#define MLXSW_REG_PTYS_IB_SPEED_QDR	BIT(2)
-#define MLXSW_REG_PTYS_IB_SPEED_FDR10	BIT(3)
-#define MLXSW_REG_PTYS_IB_SPEED_FDR	BIT(4)
-#define MLXSW_REG_PTYS_IB_SPEED_EDR	BIT(5)
-
-/* reg_ptys_ib_proto_cap
- * IB port supported speeds and protocols.
- * Access: RO
- */
-MLXSW_ITEM32(reg, ptys, ib_proto_cap, 0x10, 0, 16);
-
 /* reg_ptys_ext_eth_proto_admin
  * Extended speed and protocol to set port to.
  * Access: RW
@@ -4760,18 +4671,6 @@ MLXSW_ITEM32(reg, ptys, ext_eth_proto_admin, 0x14, 0, 32);
  */
 MLXSW_ITEM32(reg, ptys, eth_proto_admin, 0x18, 0, 32);
 
-/* reg_ptys_ib_link_width_admin
- * IB width to set port to.
- * Access: RW
- */
-MLXSW_ITEM32(reg, ptys, ib_link_width_admin, 0x1C, 16, 16);
-
-/* reg_ptys_ib_proto_admin
- * IB speeds and protocols to set port to.
- * Access: RW
- */
-MLXSW_ITEM32(reg, ptys, ib_proto_admin, 0x1C, 0, 16);
-
 /* reg_ptys_ext_eth_proto_oper
  * The extended current speed and protocol configured for the port.
  * Access: RO
@@ -4784,18 +4683,6 @@ MLXSW_ITEM32(reg, ptys, ext_eth_proto_oper, 0x20, 0, 32);
  */
 MLXSW_ITEM32(reg, ptys, eth_proto_oper, 0x24, 0, 32);
 
-/* reg_ptys_ib_link_width_oper
- * The current IB width to set port to.
- * Access: RO
- */
-MLXSW_ITEM32(reg, ptys, ib_link_width_oper, 0x28, 16, 16);
-
-/* reg_ptys_ib_proto_oper
- * The current IB speed and protocol.
- * Access: RO
- */
-MLXSW_ITEM32(reg, ptys, ib_proto_oper, 0x28, 0, 16);
-
 enum mlxsw_reg_ptys_connector_type {
 	MLXSW_REG_PTYS_CONNECTOR_TYPE_UNKNOWN_OR_NO_CONNECTOR,
 	MLXSW_REG_PTYS_CONNECTOR_TYPE_PORT_NONE,
@@ -4866,33 +4753,6 @@ static inline void mlxsw_reg_ptys_ext_eth_unpack(char *payload,
 			mlxsw_reg_ptys_ext_eth_proto_oper_get(payload);
 }
 
-static inline void mlxsw_reg_ptys_ib_pack(char *payload, u16 local_port,
-					  u16 proto_admin, u16 link_width)
-{
-	MLXSW_REG_ZERO(ptys, payload);
-	mlxsw_reg_ptys_local_port_set(payload, local_port);
-	mlxsw_reg_ptys_proto_mask_set(payload, MLXSW_REG_PTYS_PROTO_MASK_IB);
-	mlxsw_reg_ptys_ib_proto_admin_set(payload, proto_admin);
-	mlxsw_reg_ptys_ib_link_width_admin_set(payload, link_width);
-}
-
-static inline void mlxsw_reg_ptys_ib_unpack(char *payload, u16 *p_ib_proto_cap,
-					    u16 *p_ib_link_width_cap,
-					    u16 *p_ib_proto_oper,
-					    u16 *p_ib_link_width_oper)
-{
-	if (p_ib_proto_cap)
-		*p_ib_proto_cap = mlxsw_reg_ptys_ib_proto_cap_get(payload);
-	if (p_ib_link_width_cap)
-		*p_ib_link_width_cap =
-			mlxsw_reg_ptys_ib_link_width_cap_get(payload);
-	if (p_ib_proto_oper)
-		*p_ib_proto_oper = mlxsw_reg_ptys_ib_proto_oper_get(payload);
-	if (p_ib_link_width_oper)
-		*p_ib_link_width_oper =
-			mlxsw_reg_ptys_ib_link_width_oper_get(payload);
-}
-
 /* PPAD - Port Physical Address Register
  * -------------------------------------
  * The PPAD register configures the per port physical MAC address.
@@ -5666,27 +5526,6 @@ static inline void mlxsw_reg_ppcnt_pack(char *payload, u16 local_port,
 	mlxsw_reg_ppcnt_prio_tc_set(payload, prio_tc);
 }
 
-/* PLIB - Port Local to InfiniBand Port
- * ------------------------------------
- * The PLIB register performs mapping from Local Port into InfiniBand Port.
- */
-#define MLXSW_REG_PLIB_ID 0x500A
-#define MLXSW_REG_PLIB_LEN 0x10
-
-MLXSW_REG_DEFINE(plib, MLXSW_REG_PLIB_ID, MLXSW_REG_PLIB_LEN);
-
-/* reg_plib_local_port
- * Local port number.
- * Access: Index
- */
-MLXSW_ITEM32_LP(reg, plib, 0x00, 16, 0x00, 12);
-
-/* reg_plib_ib_port
- * InfiniBand port remapping for local_port.
- * Access: RW
- */
-MLXSW_ITEM32(reg, plib, ib_port, 0x00, 0, 8);
-
 /* PPTB - Port Prio To Buffer Register
  * -----------------------------------
  * Configures the switch priority to buffer table.
@@ -12924,7 +12763,6 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = {
 	MLXSW_REG(spvc),
 	MLXSW_REG(spevet),
 	MLXSW_REG(smpe),
-	MLXSW_REG(sftr2),
 	MLXSW_REG(smid2),
 	MLXSW_REG(cwtp),
 	MLXSW_REG(cwtpm),
@@ -12962,7 +12800,6 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = {
 	MLXSW_REG(paos),
 	MLXSW_REG(pfcc),
 	MLXSW_REG(ppcnt),
-	MLXSW_REG(plib),
 	MLXSW_REG(pptb),
 	MLXSW_REG(pbmc),
 	MLXSW_REG(pspa),
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index 30c7b0e15721..5bcf5bceff71 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -2691,6 +2691,7 @@ static void mlxsw_sp_traps_fini(struct mlxsw_sp *mlxsw_sp)
 static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp)
 {
 	char slcr_pl[MLXSW_REG_SLCR_LEN];
+	u16 max_lag;
 	u32 seed;
 	int err;
 
@@ -2709,12 +2710,14 @@ static int mlxsw_sp_lag_init(struct mlxsw_sp *mlxsw_sp)
 	if (err)
 		return err;
 
-	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LAG) ||
-	    !MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LAG_MEMBERS))
+	err = mlxsw_core_max_lag(mlxsw_sp->core, &max_lag);
+	if (err)
+		return err;
+
+	if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_LAG_MEMBERS))
 		return -EIO;
 
-	mlxsw_sp->lags = kcalloc(MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LAG),
-				 sizeof(struct mlxsw_sp_upper),
+	mlxsw_sp->lags = kcalloc(max_lag, sizeof(struct mlxsw_sp_upper),
 				 GFP_KERNEL);
 	if (!mlxsw_sp->lags)
 		return -ENOMEM;
@@ -3509,6 +3512,33 @@ static const struct mlxsw_config_profile mlxsw_sp2_config_profile = {
 	.cqe_time_stamp_type		= MLXSW_CMD_MBOX_CONFIG_PROFILE_CQE_TIME_STAMP_TYPE_UTC,
 };
 
+/* Reduce number of LAGs from full capacity (256) to the maximum supported LAGs
+ * in Spectrum-2/3, to avoid regression in number of free entries in the PGT
+ * table.
+ */
+#define MLXSW_SP4_CONFIG_PROFILE_MAX_LAG 128
+
+static const struct mlxsw_config_profile mlxsw_sp4_config_profile = {
+	.used_max_lag			= 1,
+	.max_lag			= MLXSW_SP4_CONFIG_PROFILE_MAX_LAG,
+	.used_flood_mode                = 1,
+	.flood_mode                     = MLXSW_CMD_MBOX_CONFIG_PROFILE_FLOOD_MODE_CONTROLLED,
+	.used_max_ib_mc			= 1,
+	.max_ib_mc			= 0,
+	.used_max_pkey			= 1,
+	.max_pkey			= 0,
+	.used_ubridge			= 1,
+	.ubridge			= 1,
+	.swid_config			= {
+		{
+			.used_type	= 1,
+			.type		= MLXSW_PORT_SWID_TYPE_ETH,
+		}
+	},
+	.used_cqe_time_stamp_type	= 1,
+	.cqe_time_stamp_type		= MLXSW_CMD_MBOX_CONFIG_PROFILE_CQE_TIME_STAMP_TYPE_UTC,
+};
+
 static void
 mlxsw_sp_resource_size_params_prepare(struct mlxsw_core *mlxsw_core,
 				      struct devlink_resource_size_params *kvd_size_params,
@@ -4039,7 +4069,7 @@ static struct mlxsw_driver mlxsw_sp4_driver = {
 	.params_unregister		= mlxsw_sp2_params_unregister,
 	.ptp_transmitted		= mlxsw_sp_ptp_transmitted,
 	.txhdr_len			= MLXSW_TXHDR_LEN,
-	.profile			= &mlxsw_sp2_config_profile,
+	.profile			= &mlxsw_sp4_config_profile,
 	.sdq_supports_cqe_v2		= true,
 };
 
@@ -4263,10 +4293,13 @@ static int mlxsw_sp_lag_index_get(struct mlxsw_sp *mlxsw_sp,
 {
 	struct mlxsw_sp_upper *lag;
 	int free_lag_id = -1;
-	u64 max_lag;
-	int i;
+	u16 max_lag;
+	int err, i;
+
+	err = mlxsw_core_max_lag(mlxsw_sp->core, &max_lag);
+	if (err)
+		return err;
 
-	max_lag = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_LAG);
 	for (i = 0; i < max_lag; i++) {
 		lag = mlxsw_sp_lag_get(mlxsw_sp, i);
 		if (lag->ref_count) {
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c
index 915dffb85a1c..dcd79d7e2af4 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c
@@ -14,16 +14,16 @@ static void mlxsw_sp_port_get_drvinfo(struct net_device *dev,
 	struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
 	struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 
-	strlcpy(drvinfo->driver, mlxsw_sp->bus_info->device_kind,
+	strscpy(drvinfo->driver, mlxsw_sp->bus_info->device_kind,
 		sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, mlxsw_sp_driver_version,
+	strscpy(drvinfo->version, mlxsw_sp_driver_version,
 		sizeof(drvinfo->version));
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
 		 "%d.%d.%d",
 		 mlxsw_sp->bus_info->fw_rev.major,
 		 mlxsw_sp->bus_info->fw_rev.minor,
 		 mlxsw_sp->bus_info->fw_rev.subminor);
-	strlcpy(drvinfo->bus_info, mlxsw_sp->bus_info->device_name,
+	strscpy(drvinfo->bus_info, mlxsw_sp->bus_info->device_name,
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
index 2c4443c6b964..48f1fa62a4fd 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
@@ -1819,7 +1819,7 @@ void mlxsw_sp_ipip_entry_demote_tunnel(struct mlxsw_sp *mlxsw_sp,
 /* The configuration where several tunnels have the same local address in the
  * same underlay table needs special treatment in the HW. That is currently not
  * implemented in the driver. This function finds and demotes the first tunnel
- * with a given source address, except the one passed in in the argument
+ * with a given source address, except the one passed in the argument
  * `except'.
  */
 bool
diff --git a/drivers/net/ethernet/micrel/ks8851.h b/drivers/net/ethernet/micrel/ks8851.h
index 6f34a61739b6..fecd43754cea 100644
--- a/drivers/net/ethernet/micrel/ks8851.h
+++ b/drivers/net/ethernet/micrel/ks8851.h
@@ -403,7 +403,7 @@ struct ks8851_net {
 	struct eeprom_93cx6	eeprom;
 	struct regulator	*vdd_reg;
 	struct regulator	*vdd_io;
-	int			gpio;
+	struct gpio_desc	*gpio;
 	struct mii_bus		*mii_bus;
 
 	void			(*lock)(struct ks8851_net *ks,
diff --git a/drivers/net/ethernet/micrel/ks8851_common.c b/drivers/net/ethernet/micrel/ks8851_common.c
index 691206f19ea7..cfbc900d4aeb 100644
--- a/drivers/net/ethernet/micrel/ks8851_common.c
+++ b/drivers/net/ethernet/micrel/ks8851_common.c
@@ -17,10 +17,9 @@
 #include <linux/cache.h>
 #include <linux/crc32.h>
 #include <linux/mii.h>
+#include <linux/gpio/consumer.h>
 #include <linux/regulator/consumer.h>
 
-#include <linux/gpio.h>
-#include <linux/of_gpio.h>
 #include <linux/of_mdio.h>
 #include <linux/of_net.h>
 
@@ -703,9 +702,9 @@ static const struct net_device_ops ks8851_netdev_ops = {
 static void ks8851_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *di)
 {
-	strlcpy(di->driver, "KS8851", sizeof(di->driver));
-	strlcpy(di->version, "1.00", sizeof(di->version));
-	strlcpy(di->bus_info, dev_name(dev->dev.parent), sizeof(di->bus_info));
+	strscpy(di->driver, "KS8851", sizeof(di->driver));
+	strscpy(di->version, "1.00", sizeof(di->version));
+	strscpy(di->bus_info, dev_name(dev->dev.parent), sizeof(di->bus_info));
 }
 
 static u32 ks8851_get_msglevel(struct net_device *dev)
@@ -1117,24 +1116,23 @@ int ks8851_probe_common(struct net_device *netdev, struct device *dev,
 {
 	struct ks8851_net *ks = netdev_priv(netdev);
 	unsigned cider;
-	int gpio;
 	int ret;
 
 	ks->netdev = netdev;
 	ks->tx_space = 6144;
 
-	gpio = of_get_named_gpio_flags(dev->of_node, "reset-gpios", 0, NULL);
-	if (gpio == -EPROBE_DEFER)
-		return gpio;
-
-	ks->gpio = gpio;
-	if (gpio_is_valid(gpio)) {
-		ret = devm_gpio_request_one(dev, gpio,
-					    GPIOF_OUT_INIT_LOW, "ks8851_rst_n");
-		if (ret) {
-			dev_err(dev, "reset gpio request failed\n");
-			return ret;
-		}
+	ks->gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
+	ret = PTR_ERR_OR_ZERO(ks->gpio);
+	if (ret) {
+		if (ret != -EPROBE_DEFER)
+			dev_err(dev, "reset gpio request failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = gpiod_set_consumer_name(ks->gpio, "ks8851_rst_n");
+	if (ret) {
+		dev_err(dev, "failed to set reset gpio name: %d\n", ret);
+		return ret;
 	}
 
 	ks->vdd_io = devm_regulator_get(dev, "vdd-io");
@@ -1161,9 +1159,9 @@ int ks8851_probe_common(struct net_device *netdev, struct device *dev,
 		goto err_reg;
 	}
 
-	if (gpio_is_valid(gpio)) {
+	if (ks->gpio) {
 		usleep_range(10000, 11000);
-		gpio_set_value(gpio, 1);
+		gpiod_set_value_cansleep(ks->gpio, 0);
 	}
 
 	spin_lock_init(&ks->statelock);
@@ -1239,8 +1237,8 @@ int ks8851_probe_common(struct net_device *netdev, struct device *dev,
 err_id:
 	ks8851_unregister_mdiobus(ks);
 err_mdio:
-	if (gpio_is_valid(gpio))
-		gpio_set_value(gpio, 0);
+	if (ks->gpio)
+		gpiod_set_value_cansleep(ks->gpio, 1);
 	regulator_disable(ks->vdd_reg);
 err_reg:
 	regulator_disable(ks->vdd_io);
@@ -1259,8 +1257,8 @@ void ks8851_remove_common(struct device *dev)
 		dev_info(dev, "remove\n");
 
 	unregister_netdev(priv->netdev);
-	if (gpio_is_valid(priv->gpio))
-		gpio_set_value(priv->gpio, 0);
+	if (priv->gpio)
+		gpiod_set_value_cansleep(priv->gpio, 1);
 	regulator_disable(priv->vdd_reg);
 	regulator_disable(priv->vdd_io);
 }
diff --git a/drivers/net/ethernet/micrel/ks8851_spi.c b/drivers/net/ethernet/micrel/ks8851_spi.c
index 82d55fc27edc..70bc7253454f 100644
--- a/drivers/net/ethernet/micrel/ks8851_spi.c
+++ b/drivers/net/ethernet/micrel/ks8851_spi.c
@@ -413,7 +413,8 @@ static int ks8851_probe_spi(struct spi_device *spi)
 
 	spi->bits_per_word = 8;
 
-	ks = netdev_priv(netdev);
+	kss = netdev_priv(netdev);
+	ks = &kss->ks8851;
 
 	ks->lock = ks8851_lock_spi;
 	ks->unlock = ks8851_unlock_spi;
@@ -433,8 +434,6 @@ static int ks8851_probe_spi(struct spi_device *spi)
 		 IRQ_RXPSI)	/* RX process stop */
 	ks->rc_ier = STD_IRQ;
 
-	kss = to_ks8851_spi(ks);
-
 	kss->spidev = spi;
 	mutex_init(&kss->lock);
 	INIT_WORK(&kss->tx_work, ks8851_tx_work);
diff --git a/drivers/net/ethernet/micrel/ksz884x.c b/drivers/net/ethernet/micrel/ksz884x.c
index 2b3eb5ed8233..468520079c65 100644
--- a/drivers/net/ethernet/micrel/ksz884x.c
+++ b/drivers/net/ethernet/micrel/ksz884x.c
@@ -5998,9 +5998,9 @@ static void netdev_get_drvinfo(struct net_device *dev,
 	struct dev_priv *priv = netdev_priv(dev);
 	struct dev_info *hw_priv = priv->adapter;
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(hw_priv->pdev),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(hw_priv->pdev),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/microchip/enc28j60.c b/drivers/net/ethernet/microchip/enc28j60.c
index 559ad94a44d0..176efbeae127 100644
--- a/drivers/net/ethernet/microchip/enc28j60.c
+++ b/drivers/net/ethernet/microchip/enc28j60.c
@@ -1467,9 +1467,9 @@ static void enc28j60_restart_work_handler(struct work_struct *work)
 static void
 enc28j60_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info,
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info,
 		dev_name(dev->dev.parent), sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/microchip/encx24j600.c b/drivers/net/ethernet/microchip/encx24j600.c
index dc1840cb5b10..d7c8aa77ec75 100644
--- a/drivers/net/ethernet/microchip/encx24j600.c
+++ b/drivers/net/ethernet/microchip/encx24j600.c
@@ -925,9 +925,9 @@ static void encx24j600_get_regs(struct net_device *dev,
 static void encx24j600_get_drvinfo(struct net_device *dev,
 				   struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, dev_name(dev->dev.parent),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, dev_name(dev->dev.parent),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/microchip/lan743x_ethtool.c b/drivers/net/ethernet/microchip/lan743x_ethtool.c
index b1c74e6cb012..c739d60ee17d 100644
--- a/drivers/net/ethernet/microchip/lan743x_ethtool.c
+++ b/drivers/net/ethernet/microchip/lan743x_ethtool.c
@@ -579,8 +579,8 @@ static void lan743x_ethtool_get_drvinfo(struct net_device *netdev,
 {
 	struct lan743x_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info,
+	strscpy(info->driver, DRIVER_NAME, sizeof(info->driver));
+	strscpy(info->bus_info,
 		pci_name(adapter->pdev), sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c
index a9a1dea6d731..50eeecba1f18 100644
--- a/drivers/net/ethernet/microchip/lan743x_main.c
+++ b/drivers/net/ethernet/microchip/lan743x_main.c
@@ -1585,6 +1585,9 @@ static void lan743x_rfe_set_multicast(struct lan743x_adapter *adapter)
 			rfctl |= RFE_CTL_AM_;
 	}
 
+	if (netdev->features & NETIF_F_RXCSUM)
+		rfctl |= RFE_CTL_IP_COE_ | RFE_CTL_TCP_UDP_COE_;
+
 	memset(hash_table, 0, DP_SEL_VHF_HASH_LEN * sizeof(u32));
 	if (netdev_mc_count(netdev)) {
 		struct netdev_hw_addr *ha;
@@ -2066,11 +2069,13 @@ static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx,
 {
 	int required_number_of_descriptors = 0;
 	unsigned int start_frame_length = 0;
+	netdev_tx_t retval = NETDEV_TX_OK;
 	unsigned int frame_length = 0;
 	unsigned int head_length = 0;
 	unsigned long irq_flags = 0;
 	bool do_timestamp = false;
 	bool ignore_sync = false;
+	struct netdev_queue *txq;
 	int nr_frags = 0;
 	bool gso = false;
 	int j;
@@ -2083,9 +2088,12 @@ static netdev_tx_t lan743x_tx_xmit_frame(struct lan743x_tx *tx,
 		if (required_number_of_descriptors > (tx->ring_size - 1)) {
 			dev_kfree_skb_irq(skb);
 		} else {
-			/* save to overflow buffer */
-			tx->overflow_skb = skb;
-			netif_stop_queue(tx->adapter->netdev);
+			/* save how many descriptors we needed to restart the queue */
+			tx->rqd_descriptors = required_number_of_descriptors;
+			retval = NETDEV_TX_BUSY;
+			txq = netdev_get_tx_queue(tx->adapter->netdev,
+						  tx->channel_number);
+			netif_tx_stop_queue(txq);
 		}
 		goto unlock;
 	}
@@ -2144,15 +2152,15 @@ finish:
 
 unlock:
 	spin_unlock_irqrestore(&tx->ring_lock, irq_flags);
-	return NETDEV_TX_OK;
+	return retval;
 }
 
 static int lan743x_tx_napi_poll(struct napi_struct *napi, int weight)
 {
 	struct lan743x_tx *tx = container_of(napi, struct lan743x_tx, napi);
 	struct lan743x_adapter *adapter = tx->adapter;
-	bool start_transmitter = false;
 	unsigned long irq_flags = 0;
+	struct netdev_queue *txq;
 	u32 ioc_bit = 0;
 
 	ioc_bit = DMAC_INT_BIT_TX_IOC_(tx->channel_number);
@@ -2163,24 +2171,20 @@ static int lan743x_tx_napi_poll(struct napi_struct *napi, int weight)
 
 	/* clean up tx ring */
 	lan743x_tx_release_completed_descriptors(tx);
-	if (netif_queue_stopped(adapter->netdev)) {
-		if (tx->overflow_skb) {
-			if (lan743x_tx_get_desc_cnt(tx, tx->overflow_skb) <=
-				lan743x_tx_get_avail_desc(tx))
-				start_transmitter = true;
+	txq = netdev_get_tx_queue(adapter->netdev, tx->channel_number);
+	if (netif_tx_queue_stopped(txq)) {
+		if (tx->rqd_descriptors) {
+			if (tx->rqd_descriptors <=
+			    lan743x_tx_get_avail_desc(tx)) {
+				tx->rqd_descriptors = 0;
+				netif_tx_wake_queue(txq);
+			}
 		} else {
-			netif_wake_queue(adapter->netdev);
+			netif_tx_wake_queue(txq);
 		}
 	}
 	spin_unlock_irqrestore(&tx->ring_lock, irq_flags);
 
-	if (start_transmitter) {
-		/* space is now available, transmit overflow skb */
-		lan743x_tx_xmit_frame(tx, tx->overflow_skb);
-		tx->overflow_skb = NULL;
-		netif_wake_queue(adapter->netdev);
-	}
-
 	if (!napi_complete(napi))
 		goto done;
 
@@ -2304,10 +2308,7 @@ static void lan743x_tx_close(struct lan743x_tx *tx)
 
 	lan743x_tx_release_all_descriptors(tx);
 
-	if (tx->overflow_skb) {
-		dev_kfree_skb(tx->overflow_skb);
-		tx->overflow_skb = NULL;
-	}
+	tx->rqd_descriptors = 0;
 
 	lan743x_tx_ring_cleanup(tx);
 }
@@ -2387,7 +2388,7 @@ static int lan743x_tx_open(struct lan743x_tx *tx)
 							 (tx->channel_number));
 	netif_napi_add_tx_weight(adapter->netdev,
 				 &tx->napi, lan743x_tx_napi_poll,
-				 tx->ring_size - 1);
+				 NAPI_POLL_WEIGHT);
 	napi_enable(&tx->napi);
 
 	data = 0;
@@ -2549,6 +2550,7 @@ static int lan743x_rx_process_buffer(struct lan743x_rx *rx)
 	int result = RX_PROCESS_RESULT_NOTHING_TO_DO;
 	struct lan743x_rx_buffer_info *buffer_info;
 	int frame_length, buffer_length;
+	bool is_ice, is_tce, is_icsm;
 	int extension_index = -1;
 	bool is_last, is_first;
 	struct sk_buff *skb;
@@ -2595,6 +2597,9 @@ static int lan743x_rx_process_buffer(struct lan743x_rx *rx)
 	frame_length =
 		RX_DESC_DATA0_FRAME_LENGTH_GET_(le32_to_cpu(descriptor->data0));
 	buffer_length = buffer_info->buffer_length;
+	is_ice = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_ICE_;
+	is_tce = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_TCE_;
+	is_icsm = le32_to_cpu(descriptor->data1) & RX_DESC_DATA1_STATUS_ICSM_;
 
 	netdev_dbg(netdev, "%s%schunk: %d/%d",
 		   is_first ? "first " : "      ",
@@ -2663,6 +2668,10 @@ process_extension:
 	if (is_last && rx->skb_head) {
 		rx->skb_head->protocol = eth_type_trans(rx->skb_head,
 							rx->adapter->netdev);
+		if (rx->adapter->netdev->features & NETIF_F_RXCSUM) {
+			if (!is_ice && !is_tce && !is_icsm)
+				skb->ip_summed = CHECKSUM_UNNECESSARY;
+		}
 		netdev_dbg(netdev, "sending %d byte frame to OS",
 			   rx->skb_head->len);
 		napi_gro_receive(&rx->napi, rx->skb_head);
@@ -2866,9 +2875,7 @@ static int lan743x_rx_open(struct lan743x_rx *rx)
 	if (ret)
 		goto return_error;
 
-	netif_napi_add(adapter->netdev,
-		       &rx->napi, lan743x_rx_napi_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(adapter->netdev, &rx->napi, lan743x_rx_napi_poll);
 
 	lan743x_csr_write(adapter, DMAC_CMD,
 			  DMAC_CMD_RX_SWR_(rx->channel_number));
@@ -3347,8 +3354,10 @@ static int lan743x_pcidev_probe(struct pci_dev *pdev,
 						 PCI11X1X_USED_TX_CHANNELS,
 						 LAN743X_USED_RX_CHANNELS);
 	} else {
-		netdev = devm_alloc_etherdev(&pdev->dev,
-					     sizeof(struct lan743x_adapter));
+		netdev = devm_alloc_etherdev_mqs(&pdev->dev,
+						 sizeof(struct lan743x_adapter),
+						 LAN743X_USED_TX_CHANNELS,
+						 LAN743X_USED_RX_CHANNELS);
 	}
 
 	if (!netdev)
@@ -3383,7 +3392,8 @@ static int lan743x_pcidev_probe(struct pci_dev *pdev,
 
 	adapter->netdev->netdev_ops = &lan743x_netdev_ops;
 	adapter->netdev->ethtool_ops = &lan743x_ethtool_ops;
-	adapter->netdev->features = NETIF_F_SG | NETIF_F_TSO | NETIF_F_HW_CSUM;
+	adapter->netdev->features = NETIF_F_SG | NETIF_F_TSO |
+				    NETIF_F_HW_CSUM | NETIF_F_RXCSUM;
 	adapter->netdev->hw_features = adapter->netdev->features;
 
 	/* carrier off reporting is important to ethtool even BEFORE open */
diff --git a/drivers/net/ethernet/microchip/lan743x_main.h b/drivers/net/ethernet/microchip/lan743x_main.h
index 72adae4f2aa0..67877d3b6dd9 100644
--- a/drivers/net/ethernet/microchip/lan743x_main.h
+++ b/drivers/net/ethernet/microchip/lan743x_main.h
@@ -266,6 +266,8 @@
 #define RFE_ADDR_FILT_LO(x)		(0x404 + (8 * (x)))
 
 #define RFE_CTL				(0x508)
+#define RFE_CTL_TCP_UDP_COE_		BIT(12)
+#define RFE_CTL_IP_COE_			BIT(11)
 #define RFE_CTL_AB_			BIT(10)
 #define RFE_CTL_AM_			BIT(9)
 #define RFE_CTL_AU_			BIT(8)
@@ -954,8 +956,7 @@ struct lan743x_tx {
 
 	struct napi_struct napi;
 	u32 frame_count;
-
-	struct sk_buff *overflow_skb;
+	u32 rqd_descriptors;
 };
 
 void lan743x_tx_set_timestamping_mode(struct lan743x_tx *tx,
@@ -1110,7 +1111,7 @@ struct lan743x_tx_buffer_info {
 	unsigned int    buffer_length;
 };
 
-#define LAN743X_TX_RING_SIZE    (50)
+#define LAN743X_TX_RING_SIZE    (128)
 
 /* OWN bit is set. ie, Descs are owned by RX DMAC */
 #define RX_DESC_DATA0_OWN_                (0x00008000)
@@ -1122,6 +1123,9 @@ struct lan743x_tx_buffer_info {
 	(((data0) & RX_DESC_DATA0_FRAME_LENGTH_MASK_) >> 16)
 #define RX_DESC_DATA0_EXT_                (0x00004000)
 #define RX_DESC_DATA0_BUF_LENGTH_MASK_    (0x00003FFF)
+#define RX_DESC_DATA1_STATUS_ICE_         (0x00020000)
+#define RX_DESC_DATA1_STATUS_TCE_         (0x00010000)
+#define RX_DESC_DATA1_STATUS_ICSM_        (0x00000001)
 #define RX_DESC_DATA2_TS_NS_MASK_         (0x3FFFFFFF)
 
 #if ((NET_IP_ALIGN != 0) && (NET_IP_ALIGN != 2))
diff --git a/drivers/net/ethernet/microchip/lan743x_ptp.c b/drivers/net/ethernet/microchip/lan743x_ptp.c
index 6a11e2ceb013..da3ea905adbb 100644
--- a/drivers/net/ethernet/microchip/lan743x_ptp.c
+++ b/drivers/net/ethernet/microchip/lan743x_ptp.c
@@ -1049,6 +1049,10 @@ static int lan743x_ptpci_verify_pin_config(struct ptp_clock_info *ptp,
 					   enum ptp_pin_function func,
 					   unsigned int chan)
 {
+	struct lan743x_ptp *lan_ptp =
+		container_of(ptp, struct lan743x_ptp, ptp_clock_info);
+	struct lan743x_adapter *adapter =
+		container_of(lan_ptp, struct lan743x_adapter, ptp);
 	int result = 0;
 
 	/* Confirm the requested function is supported. Parameter
@@ -1057,7 +1061,10 @@ static int lan743x_ptpci_verify_pin_config(struct ptp_clock_info *ptp,
 	switch (func) {
 	case PTP_PF_NONE:
 	case PTP_PF_PEROUT:
+		break;
 	case PTP_PF_EXTTS:
+		if (!adapter->is_pci11x1x)
+			result = -1;
 		break;
 	case PTP_PF_PHYSYNC:
 	default:
diff --git a/drivers/net/ethernet/microchip/lan966x/Kconfig b/drivers/net/ethernet/microchip/lan966x/Kconfig
index 4241ff0e5098..49e1464a4313 100644
--- a/drivers/net/ethernet/microchip/lan966x/Kconfig
+++ b/drivers/net/ethernet/microchip/lan966x/Kconfig
@@ -4,6 +4,7 @@ config LAN966X_SWITCH
 	depends on HAS_IOMEM
 	depends on OF
 	depends on NET_SWITCHDEV
+	depends on BRIDGE || BRIDGE=n
 	select PHYLINK
 	select PACKING
 	help
diff --git a/drivers/net/ethernet/microchip/lan966x/Makefile b/drivers/net/ethernet/microchip/lan966x/Makefile
index fd2e0ebb2427..962f7c5f9e7d 100644
--- a/drivers/net/ethernet/microchip/lan966x/Makefile
+++ b/drivers/net/ethernet/microchip/lan966x/Makefile
@@ -8,4 +8,7 @@ obj-$(CONFIG_LAN966X_SWITCH) += lan966x-switch.o
 lan966x-switch-objs  := lan966x_main.o lan966x_phylink.o lan966x_port.o \
 			lan966x_mac.o lan966x_ethtool.o lan966x_switchdev.o \
 			lan966x_vlan.o lan966x_fdb.o lan966x_mdb.o \
-			lan966x_ptp.o lan966x_fdma.o
+			lan966x_ptp.o lan966x_fdma.o lan966x_lag.o \
+			lan966x_tc.o lan966x_mqprio.o lan966x_taprio.o \
+			lan966x_tbf.o lan966x_cbs.o lan966x_ets.o \
+			lan966x_tc_matchall.o lan966x_police.o lan966x_mirror.o
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_cbs.c b/drivers/net/ethernet/microchip/lan966x/lan966x_cbs.c
new file mode 100644
index 000000000000..70cbbf8d2b67
--- /dev/null
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_cbs.c
@@ -0,0 +1,70 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include "lan966x_main.h"
+
+int lan966x_cbs_add(struct lan966x_port *port,
+		    struct tc_cbs_qopt_offload *qopt)
+{
+	struct lan966x *lan966x = port->lan966x;
+	u32 cir, cbs;
+	u8 se_idx;
+
+	/* Check for invalid values */
+	if (qopt->idleslope <= 0 ||
+	    qopt->sendslope >= 0 ||
+	    qopt->locredit >= qopt->hicredit)
+		return -EINVAL;
+
+	se_idx = SE_IDX_QUEUE + port->chip_port * NUM_PRIO_QUEUES + qopt->queue;
+	cir = qopt->idleslope;
+	cbs = (qopt->idleslope - qopt->sendslope) *
+		(qopt->hicredit - qopt->locredit) /
+		-qopt->sendslope;
+
+	/* Rate unit is 100 kbps */
+	cir = DIV_ROUND_UP(cir, 100);
+	/* Avoid using zero rate */
+	cir = cir ?: 1;
+	/* Burst unit is 4kB */
+	cbs = DIV_ROUND_UP(cbs, 4096);
+	/* Avoid using zero burst */
+	cbs = cbs ?: 1;
+
+	/* Check that actually the result can be written */
+	if (cir > GENMASK(15, 0) ||
+	    cbs > GENMASK(6, 0))
+		return -EINVAL;
+
+	lan_rmw(QSYS_SE_CFG_SE_AVB_ENA_SET(1) |
+		QSYS_SE_CFG_SE_FRM_MODE_SET(1),
+		QSYS_SE_CFG_SE_AVB_ENA |
+		QSYS_SE_CFG_SE_FRM_MODE,
+		lan966x, QSYS_SE_CFG(se_idx));
+
+	lan_wr(QSYS_CIR_CFG_CIR_RATE_SET(cir) |
+	       QSYS_CIR_CFG_CIR_BURST_SET(cbs),
+	       lan966x, QSYS_CIR_CFG(se_idx));
+
+	return 0;
+}
+
+int lan966x_cbs_del(struct lan966x_port *port,
+		    struct tc_cbs_qopt_offload *qopt)
+{
+	struct lan966x *lan966x = port->lan966x;
+	u8 se_idx;
+
+	se_idx = SE_IDX_QUEUE + port->chip_port * NUM_PRIO_QUEUES + qopt->queue;
+
+	lan_rmw(QSYS_SE_CFG_SE_AVB_ENA_SET(1) |
+		QSYS_SE_CFG_SE_FRM_MODE_SET(0),
+		QSYS_SE_CFG_SE_AVB_ENA |
+		QSYS_SE_CFG_SE_FRM_MODE,
+		lan966x, QSYS_SE_CFG(se_idx));
+
+	lan_wr(QSYS_CIR_CFG_CIR_RATE_SET(0) |
+	       QSYS_CIR_CFG_CIR_BURST_SET(0),
+	       lan966x, QSYS_CIR_CFG(se_idx));
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ets.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ets.c
new file mode 100644
index 000000000000..8310d3f35404
--- /dev/null
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ets.c
@@ -0,0 +1,96 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include "lan966x_main.h"
+
+#define DWRR_COST_BIT_WIDTH	BIT(5)
+
+static u32 lan966x_ets_hw_cost(u32 w_min, u32 weight)
+{
+	u32 res;
+
+	/* Round half up: Multiply with 16 before division,
+	 * add 8 and divide result with 16 again
+	 */
+	res = (((DWRR_COST_BIT_WIDTH << 4) * w_min / weight) + 8) >> 4;
+	return max_t(u32, 1, res) - 1;
+}
+
+int lan966x_ets_add(struct lan966x_port *port,
+		    struct tc_ets_qopt_offload *qopt)
+{
+	struct tc_ets_qopt_offload_replace_params *params;
+	struct lan966x *lan966x = port->lan966x;
+	u32 w_min = 100;
+	u8 count = 0;
+	u32 se_idx;
+	u8 i;
+
+	/* Check the input */
+	if (qopt->parent != TC_H_ROOT)
+		return -EINVAL;
+
+	params = &qopt->replace_params;
+	if (params->bands != NUM_PRIO_QUEUES)
+		return -EINVAL;
+
+	for (i = 0; i < params->bands; ++i) {
+		/* In the switch the DWRR is always on the lowest consecutive
+		 * priorities. Due to this, the first priority must map to the
+		 * first DWRR band.
+		 */
+		if (params->priomap[i] != (7 - i))
+			return -EINVAL;
+
+		if (params->quanta[i] && params->weights[i] == 0)
+			return -EINVAL;
+	}
+
+	se_idx = SE_IDX_PORT + port->chip_port;
+
+	/* Find minimum weight */
+	for (i = 0; i < params->bands; ++i) {
+		if (params->quanta[i] == 0)
+			continue;
+
+		w_min = min(w_min, params->weights[i]);
+	}
+
+	for (i = 0; i < params->bands; ++i) {
+		if (params->quanta[i] == 0)
+			continue;
+
+		++count;
+
+		lan_wr(lan966x_ets_hw_cost(w_min, params->weights[i]),
+		       lan966x, QSYS_SE_DWRR_CFG(se_idx, 7 - i));
+	}
+
+	lan_rmw(QSYS_SE_CFG_SE_DWRR_CNT_SET(count) |
+		QSYS_SE_CFG_SE_RR_ENA_SET(0),
+		QSYS_SE_CFG_SE_DWRR_CNT |
+		QSYS_SE_CFG_SE_RR_ENA,
+		lan966x, QSYS_SE_CFG(se_idx));
+
+	return 0;
+}
+
+int lan966x_ets_del(struct lan966x_port *port,
+		    struct tc_ets_qopt_offload *qopt)
+{
+	struct lan966x *lan966x = port->lan966x;
+	u32 se_idx;
+	int i;
+
+	se_idx = SE_IDX_PORT + port->chip_port;
+
+	for (i = 0; i < NUM_PRIO_QUEUES; ++i)
+		lan_wr(0, lan966x, QSYS_SE_DWRR_CFG(se_idx, i));
+
+	lan_rmw(QSYS_SE_CFG_SE_DWRR_CNT_SET(0) |
+		QSYS_SE_CFG_SE_RR_ENA_SET(0),
+		QSYS_SE_CFG_SE_DWRR_CNT |
+		QSYS_SE_CFG_SE_RR_ENA,
+		lan966x, QSYS_SE_CFG(se_idx));
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdb.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdb.c
index da5ca7188679..2ea263e893ee 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdb.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdb.c
@@ -8,6 +8,7 @@ struct lan966x_fdb_event_work {
 	struct work_struct work;
 	struct switchdev_notifier_fdb_info fdb_info;
 	struct net_device *dev;
+	struct net_device *orig_dev;
 	struct lan966x *lan966x;
 	unsigned long event;
 };
@@ -127,75 +128,119 @@ void lan966x_fdb_deinit(struct lan966x *lan966x)
 	lan966x_fdb_purge_entries(lan966x);
 }
 
-static void lan966x_fdb_event_work(struct work_struct *work)
+void lan966x_fdb_flush_workqueue(struct lan966x *lan966x)
+{
+	flush_workqueue(lan966x->fdb_work);
+}
+
+static void lan966x_fdb_port_event_work(struct lan966x_fdb_event_work *fdb_work)
 {
-	struct lan966x_fdb_event_work *fdb_work =
-		container_of(work, struct lan966x_fdb_event_work, work);
 	struct switchdev_notifier_fdb_info *fdb_info;
-	struct net_device *dev = fdb_work->dev;
 	struct lan966x_port *port;
 	struct lan966x *lan966x;
-	int ret;
 
-	fdb_info = &fdb_work->fdb_info;
 	lan966x = fdb_work->lan966x;
+	port = netdev_priv(fdb_work->orig_dev);
+	fdb_info = &fdb_work->fdb_info;
 
-	if (lan966x_netdevice_check(dev)) {
-		port = netdev_priv(dev);
+	switch (fdb_work->event) {
+	case SWITCHDEV_FDB_ADD_TO_DEVICE:
+		if (!fdb_info->added_by_user)
+			break;
+		lan966x_mac_add_entry(lan966x, port, fdb_info->addr,
+				      fdb_info->vid);
+		break;
+	case SWITCHDEV_FDB_DEL_TO_DEVICE:
+		if (!fdb_info->added_by_user)
+			break;
+		lan966x_mac_del_entry(lan966x, fdb_info->addr,
+				      fdb_info->vid);
+		break;
+	}
+}
+
+static void lan966x_fdb_bridge_event_work(struct lan966x_fdb_event_work *fdb_work)
+{
+	struct switchdev_notifier_fdb_info *fdb_info;
+	struct lan966x *lan966x;
+	int ret;
 
-		switch (fdb_work->event) {
-		case SWITCHDEV_FDB_ADD_TO_DEVICE:
-			if (!fdb_info->added_by_user)
-				break;
-			lan966x_mac_add_entry(lan966x, port, fdb_info->addr,
-					      fdb_info->vid);
+	lan966x = fdb_work->lan966x;
+	fdb_info = &fdb_work->fdb_info;
+
+	/* In case the bridge is called */
+	switch (fdb_work->event) {
+	case SWITCHDEV_FDB_ADD_TO_DEVICE:
+		/* If there is no front port in this vlan, there is no
+		 * point to copy the frame to CPU because it would be
+		 * just dropped at later point. So add it only if
+		 * there is a port but it is required to store the fdb
+		 * entry for later point when a port actually gets in
+		 * the vlan.
+		 */
+		lan966x_fdb_add_entry(lan966x, fdb_info);
+		if (!lan966x_vlan_cpu_member_cpu_vlan_mask(lan966x,
+							   fdb_info->vid))
 			break;
-		case SWITCHDEV_FDB_DEL_TO_DEVICE:
-			if (!fdb_info->added_by_user)
-				break;
-			lan966x_mac_del_entry(lan966x, fdb_info->addr,
-					      fdb_info->vid);
+
+		lan966x_mac_cpu_learn(lan966x, fdb_info->addr,
+				      fdb_info->vid);
+		break;
+	case SWITCHDEV_FDB_DEL_TO_DEVICE:
+		ret = lan966x_fdb_del_entry(lan966x, fdb_info);
+		if (!lan966x_vlan_cpu_member_cpu_vlan_mask(lan966x,
+							   fdb_info->vid))
 			break;
-		}
-	} else {
-		if (!netif_is_bridge_master(dev))
-			goto out;
-
-		/* In case the bridge is called */
-		switch (fdb_work->event) {
-		case SWITCHDEV_FDB_ADD_TO_DEVICE:
-			/* If there is no front port in this vlan, there is no
-			 * point to copy the frame to CPU because it would be
-			 * just dropped at later point. So add it only if
-			 * there is a port but it is required to store the fdb
-			 * entry for later point when a port actually gets in
-			 * the vlan.
-			 */
-			lan966x_fdb_add_entry(lan966x, fdb_info);
-			if (!lan966x_vlan_cpu_member_cpu_vlan_mask(lan966x,
-								   fdb_info->vid))
-				break;
-
-			lan966x_mac_cpu_learn(lan966x, fdb_info->addr,
-					      fdb_info->vid);
+
+		if (ret)
+			lan966x_mac_cpu_forget(lan966x, fdb_info->addr,
+					       fdb_info->vid);
+		break;
+	}
+}
+
+static void lan966x_fdb_lag_event_work(struct lan966x_fdb_event_work *fdb_work)
+{
+	struct switchdev_notifier_fdb_info *fdb_info;
+	struct lan966x_port *port;
+	struct lan966x *lan966x;
+
+	if (!lan966x_lag_first_port(fdb_work->orig_dev, fdb_work->dev))
+		return;
+
+	lan966x = fdb_work->lan966x;
+	port = netdev_priv(fdb_work->dev);
+	fdb_info = &fdb_work->fdb_info;
+
+	switch (fdb_work->event) {
+	case SWITCHDEV_FDB_ADD_TO_DEVICE:
+		if (!fdb_info->added_by_user)
 			break;
-		case SWITCHDEV_FDB_DEL_TO_DEVICE:
-			ret = lan966x_fdb_del_entry(lan966x, fdb_info);
-			if (!lan966x_vlan_cpu_member_cpu_vlan_mask(lan966x,
-								   fdb_info->vid))
-				break;
-
-			if (ret)
-				lan966x_mac_cpu_forget(lan966x, fdb_info->addr,
-						       fdb_info->vid);
+		lan966x_mac_add_entry(lan966x, port, fdb_info->addr,
+				      fdb_info->vid);
+		break;
+	case SWITCHDEV_FDB_DEL_TO_DEVICE:
+		if (!fdb_info->added_by_user)
 			break;
-		}
+		lan966x_mac_del_entry(lan966x, fdb_info->addr, fdb_info->vid);
+		break;
 	}
+}
+
+static void lan966x_fdb_event_work(struct work_struct *work)
+{
+	struct lan966x_fdb_event_work *fdb_work =
+		container_of(work, struct lan966x_fdb_event_work, work);
+
+	if (lan966x_netdevice_check(fdb_work->orig_dev))
+		lan966x_fdb_port_event_work(fdb_work);
+	else if (netif_is_bridge_master(fdb_work->orig_dev))
+		lan966x_fdb_bridge_event_work(fdb_work);
+	else if (netif_is_lag_master(fdb_work->orig_dev))
+		lan966x_fdb_lag_event_work(fdb_work);
 
-out:
 	kfree(fdb_work->fdb_info.addr);
 	kfree(fdb_work);
-	dev_put(dev);
 }
 
 int lan966x_handle_fdb(struct net_device *dev,
@@ -221,7 +266,8 @@ int lan966x_handle_fdb(struct net_device *dev,
 		if (!fdb_work)
 			return -ENOMEM;
 
-		fdb_work->dev = orig_dev;
+		fdb_work->dev = dev;
+		fdb_work->orig_dev = orig_dev;
 		fdb_work->lan966x = lan966x;
 		fdb_work->event = event;
 		INIT_WORK(&fdb_work->work, lan966x_fdb_event_work);
@@ -231,7 +277,6 @@ int lan966x_handle_fdb(struct net_device *dev,
 			goto err_addr_alloc;
 
 		ether_addr_copy((u8 *)fdb_work->fdb_info.addr, fdb_info->addr);
-		dev_hold(orig_dev);
 
 		queue_work(lan966x->fdb_work, &fdb_work->work);
 		break;
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
index 51f8a0816377..7e4061c854f0 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma.c
@@ -787,8 +787,7 @@ void lan966x_fdma_netdev_init(struct lan966x *lan966x, struct net_device *dev)
 		return;
 
 	lan966x->fdma_ndev = dev;
-	netif_napi_add(dev, &lan966x->napi, lan966x_fdma_napi_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &lan966x->napi, lan966x_fdma_napi_poll);
 	napi_enable(&lan966x->napi);
 }
 
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_lag.c b/drivers/net/ethernet/microchip/lan966x/lan966x_lag.c
new file mode 100644
index 000000000000..41fa2523d91d
--- /dev/null
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_lag.c
@@ -0,0 +1,363 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <linux/if_bridge.h>
+
+#include "lan966x_main.h"
+
+static void lan966x_lag_set_aggr_pgids(struct lan966x *lan966x)
+{
+	u32 visited = GENMASK(lan966x->num_phys_ports - 1, 0);
+	int p, lag, i;
+
+	/* Reset destination and aggregation PGIDS */
+	for (p = 0; p < lan966x->num_phys_ports; ++p)
+		lan_wr(ANA_PGID_PGID_SET(BIT(p)),
+		       lan966x, ANA_PGID(p));
+
+	for (p = PGID_AGGR; p < PGID_SRC; ++p)
+		lan_wr(ANA_PGID_PGID_SET(visited),
+		       lan966x, ANA_PGID(p));
+
+	/* The visited ports bitmask holds the list of ports offloading any
+	 * bonding interface. Initially we mark all these ports as unvisited,
+	 * then every time we visit a port in this bitmask, we know that it is
+	 * the lowest numbered port, i.e. the one whose logical ID == physical
+	 * port ID == LAG ID. So we mark as visited all further ports in the
+	 * bitmask that are offloading the same bonding interface. This way,
+	 * we set up the aggregation PGIDs only once per bonding interface.
+	 */
+	for (p = 0; p < lan966x->num_phys_ports; ++p) {
+		struct lan966x_port *port = lan966x->ports[p];
+
+		if (!port || !port->bond)
+			continue;
+
+		visited &= ~BIT(p);
+	}
+
+	/* Now, set PGIDs for each active LAG */
+	for (lag = 0; lag < lan966x->num_phys_ports; ++lag) {
+		struct net_device *bond = lan966x->ports[lag]->bond;
+		int num_active_ports = 0;
+		unsigned long bond_mask;
+		u8 aggr_idx[16];
+
+		if (!bond || (visited & BIT(lag)))
+			continue;
+
+		bond_mask = lan966x_lag_get_mask(lan966x, bond);
+
+		for_each_set_bit(p, &bond_mask, lan966x->num_phys_ports) {
+			struct lan966x_port *port = lan966x->ports[p];
+
+			lan_wr(ANA_PGID_PGID_SET(bond_mask),
+			       lan966x, ANA_PGID(p));
+			if (port->lag_tx_active)
+				aggr_idx[num_active_ports++] = p;
+		}
+
+		for (i = PGID_AGGR; i < PGID_SRC; ++i) {
+			u32 ac;
+
+			ac = lan_rd(lan966x, ANA_PGID(i));
+			ac &= ~bond_mask;
+			/* Don't do division by zero if there was no active
+			 * port. Just make all aggregation codes zero.
+			 */
+			if (num_active_ports)
+				ac |= BIT(aggr_idx[i % num_active_ports]);
+			lan_wr(ANA_PGID_PGID_SET(ac),
+			       lan966x, ANA_PGID(i));
+		}
+
+		/* Mark all ports in the same LAG as visited to avoid applying
+		 * the same config again.
+		 */
+		for (p = lag; p < lan966x->num_phys_ports; p++) {
+			struct lan966x_port *port = lan966x->ports[p];
+
+			if (!port)
+				continue;
+
+			if (port->bond == bond)
+				visited |= BIT(p);
+		}
+	}
+}
+
+static void lan966x_lag_set_port_ids(struct lan966x *lan966x)
+{
+	struct lan966x_port *port;
+	u32 bond_mask;
+	u32 lag_id;
+	int p;
+
+	for (p = 0; p < lan966x->num_phys_ports; ++p) {
+		port = lan966x->ports[p];
+		if (!port)
+			continue;
+
+		lag_id = port->chip_port;
+
+		bond_mask = lan966x_lag_get_mask(lan966x, port->bond);
+		if (bond_mask)
+			lag_id = __ffs(bond_mask);
+
+		lan_rmw(ANA_PORT_CFG_PORTID_VAL_SET(lag_id),
+			ANA_PORT_CFG_PORTID_VAL,
+			lan966x, ANA_PORT_CFG(port->chip_port));
+	}
+}
+
+static void lan966x_lag_update_ids(struct lan966x *lan966x)
+{
+	lan966x_lag_set_port_ids(lan966x);
+	lan966x_update_fwd_mask(lan966x);
+	lan966x_lag_set_aggr_pgids(lan966x);
+}
+
+int lan966x_lag_port_join(struct lan966x_port *port,
+			  struct net_device *brport_dev,
+			  struct net_device *bond,
+			  struct netlink_ext_ack *extack)
+{
+	struct lan966x *lan966x = port->lan966x;
+	struct net_device *dev = port->dev;
+	u32 lag_id = -1;
+	u32 bond_mask;
+	int err;
+
+	bond_mask = lan966x_lag_get_mask(lan966x, bond);
+	if (bond_mask)
+		lag_id = __ffs(bond_mask);
+
+	port->bond = bond;
+	lan966x_lag_update_ids(lan966x);
+
+	err = switchdev_bridge_port_offload(brport_dev, dev, port,
+					    &lan966x_switchdev_nb,
+					    &lan966x_switchdev_blocking_nb,
+					    false, extack);
+	if (err)
+		goto out;
+
+	lan966x_port_stp_state_set(port, br_port_get_stp_state(brport_dev));
+
+	if (lan966x_lag_first_port(port->bond, port->dev) &&
+	    lag_id != -1)
+		lan966x_mac_lag_replace_port_entry(lan966x,
+						   lan966x->ports[lag_id],
+						   port);
+
+	return 0;
+
+out:
+	port->bond = NULL;
+	lan966x_lag_update_ids(lan966x);
+
+	return err;
+}
+
+void lan966x_lag_port_leave(struct lan966x_port *port, struct net_device *bond)
+{
+	struct lan966x *lan966x = port->lan966x;
+	u32 bond_mask;
+	u32 lag_id;
+
+	if (lan966x_lag_first_port(port->bond, port->dev)) {
+		bond_mask = lan966x_lag_get_mask(lan966x, port->bond);
+		bond_mask &= ~BIT(port->chip_port);
+		if (bond_mask) {
+			lag_id = __ffs(bond_mask);
+			lan966x_mac_lag_replace_port_entry(lan966x, port,
+							   lan966x->ports[lag_id]);
+		} else {
+			lan966x_mac_lag_remove_port_entry(lan966x, port);
+		}
+	}
+
+	port->bond = NULL;
+	lan966x_lag_update_ids(lan966x);
+	lan966x_port_stp_state_set(port, BR_STATE_FORWARDING);
+}
+
+static bool lan966x_lag_port_check_hash_types(struct lan966x *lan966x,
+					      enum netdev_lag_hash hash_type)
+{
+	int p;
+
+	for (p = 0; p < lan966x->num_phys_ports; ++p) {
+		struct lan966x_port *port = lan966x->ports[p];
+
+		if (!port || !port->bond)
+			continue;
+
+		if (port->hash_type != hash_type)
+			return false;
+	}
+
+	return true;
+}
+
+int lan966x_lag_port_prechangeupper(struct net_device *dev,
+				    struct netdev_notifier_changeupper_info *info)
+{
+	struct lan966x_port *port = netdev_priv(dev);
+	struct lan966x *lan966x = port->lan966x;
+	struct netdev_lag_upper_info *lui;
+	struct netlink_ext_ack *extack;
+
+	extack = netdev_notifier_info_to_extack(&info->info);
+	lui = info->upper_info;
+	if (!lui) {
+		port->hash_type = NETDEV_LAG_HASH_NONE;
+		return NOTIFY_DONE;
+	}
+
+	if (lui->tx_type != NETDEV_LAG_TX_TYPE_HASH) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "LAG device using unsupported Tx type");
+		return -EINVAL;
+	}
+
+	if (!lan966x_lag_port_check_hash_types(lan966x, lui->hash_type)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "LAG devices can have only the same hash_type");
+		return -EINVAL;
+	}
+
+	switch (lui->hash_type) {
+	case NETDEV_LAG_HASH_L2:
+		lan_wr(ANA_AGGR_CFG_AC_DMAC_ENA_SET(1) |
+		       ANA_AGGR_CFG_AC_SMAC_ENA_SET(1),
+		       lan966x, ANA_AGGR_CFG);
+		break;
+	case NETDEV_LAG_HASH_L34:
+		lan_wr(ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA_SET(1) |
+		       ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA_SET(1) |
+		       ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA_SET(1),
+		       lan966x, ANA_AGGR_CFG);
+		break;
+	case NETDEV_LAG_HASH_L23:
+		lan_wr(ANA_AGGR_CFG_AC_DMAC_ENA_SET(1) |
+		       ANA_AGGR_CFG_AC_SMAC_ENA_SET(1) |
+		       ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA_SET(1) |
+		       ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA_SET(1),
+		       lan966x, ANA_AGGR_CFG);
+		break;
+	default:
+		NL_SET_ERR_MSG_MOD(extack,
+				   "LAG device using unsupported hash type");
+		return -EINVAL;
+	}
+
+	port->hash_type = lui->hash_type;
+
+	return NOTIFY_OK;
+}
+
+int lan966x_lag_port_changelowerstate(struct net_device *dev,
+				      struct netdev_notifier_changelowerstate_info *info)
+{
+	struct netdev_lag_lower_state_info *lag = info->lower_state_info;
+	struct lan966x_port *port = netdev_priv(dev);
+	struct lan966x *lan966x = port->lan966x;
+	bool is_active;
+
+	if (!port->bond)
+		return NOTIFY_DONE;
+
+	is_active = lag->link_up && lag->tx_enabled;
+	if (port->lag_tx_active == is_active)
+		return NOTIFY_DONE;
+
+	port->lag_tx_active = is_active;
+	lan966x_lag_set_aggr_pgids(lan966x);
+
+	return NOTIFY_OK;
+}
+
+int lan966x_lag_netdev_prechangeupper(struct net_device *dev,
+				      struct netdev_notifier_changeupper_info *info)
+{
+	struct lan966x_port *port;
+	struct net_device *lower;
+	struct list_head *iter;
+	int err;
+
+	netdev_for_each_lower_dev(dev, lower, iter) {
+		if (!lan966x_netdevice_check(lower))
+			continue;
+
+		port = netdev_priv(lower);
+		if (port->bond != dev)
+			continue;
+
+		err = lan966x_port_prechangeupper(lower, dev, info);
+		if (err)
+			return err;
+	}
+
+	return NOTIFY_DONE;
+}
+
+int lan966x_lag_netdev_changeupper(struct net_device *dev,
+				   struct netdev_notifier_changeupper_info *info)
+{
+	struct lan966x_port *port;
+	struct net_device *lower;
+	struct list_head *iter;
+	int err;
+
+	netdev_for_each_lower_dev(dev, lower, iter) {
+		if (!lan966x_netdevice_check(lower))
+			continue;
+
+		port = netdev_priv(lower);
+		if (port->bond != dev)
+			continue;
+
+		err = lan966x_port_changeupper(lower, dev, info);
+		if (err)
+			return err;
+	}
+
+	return NOTIFY_DONE;
+}
+
+bool lan966x_lag_first_port(struct net_device *lag, struct net_device *dev)
+{
+	struct lan966x_port *port = netdev_priv(dev);
+	struct lan966x *lan966x = port->lan966x;
+	unsigned long bond_mask;
+
+	if (port->bond != lag)
+		return false;
+
+	bond_mask = lan966x_lag_get_mask(lan966x, lag);
+	if (bond_mask && port->chip_port == __ffs(bond_mask))
+		return true;
+
+	return false;
+}
+
+u32 lan966x_lag_get_mask(struct lan966x *lan966x, struct net_device *bond)
+{
+	struct lan966x_port *port;
+	u32 mask = 0;
+	int p;
+
+	if (!bond)
+		return mask;
+
+	for (p = 0; p < lan966x->num_phys_ports; p++) {
+		port = lan966x->ports[p];
+		if (!port)
+			continue;
+
+		if (port->bond == bond)
+			mask |= BIT(p);
+	}
+
+	return mask;
+}
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c b/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c
index 5893770bfd94..baa3a30c039f 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c
@@ -22,6 +22,7 @@ struct lan966x_mac_entry {
 	u16 vid;
 	u16 port_index;
 	int row;
+	bool lag;
 };
 
 struct lan966x_mac_raw_entry {
@@ -69,15 +70,14 @@ static void lan966x_mac_select(struct lan966x *lan966x,
 	lan_wr(mach, lan966x, ANA_MACHDATA);
 }
 
-static int __lan966x_mac_learn(struct lan966x *lan966x, int pgid,
-			       bool cpu_copy,
-			       const unsigned char mac[ETH_ALEN],
-			       unsigned int vid,
-			       enum macaccess_entry_type type)
+static int __lan966x_mac_learn_locked(struct lan966x *lan966x, int pgid,
+				      bool cpu_copy,
+				      const unsigned char mac[ETH_ALEN],
+				      unsigned int vid,
+				      enum macaccess_entry_type type)
 {
-	int ret;
+	lockdep_assert_held(&lan966x->mac_lock);
 
-	spin_lock(&lan966x->mac_lock);
 	lan966x_mac_select(lan966x, mac, vid);
 
 	/* Issue a write command */
@@ -89,7 +89,19 @@ static int __lan966x_mac_learn(struct lan966x *lan966x, int pgid,
 	       ANA_MACACCESS_MAC_TABLE_CMD_SET(MACACCESS_CMD_LEARN),
 	       lan966x, ANA_MACACCESS);
 
-	ret = lan966x_mac_wait_for_completion(lan966x);
+	return lan966x_mac_wait_for_completion(lan966x);
+}
+
+static int __lan966x_mac_learn(struct lan966x *lan966x, int pgid,
+			       bool cpu_copy,
+			       const unsigned char mac[ETH_ALEN],
+			       unsigned int vid,
+			       enum macaccess_entry_type type)
+{
+	int ret;
+
+	spin_lock(&lan966x->mac_lock);
+	ret = __lan966x_mac_learn_locked(lan966x, pgid, cpu_copy, mac, vid, type);
 	spin_unlock(&lan966x->mac_lock);
 
 	return ret;
@@ -119,6 +131,16 @@ int lan966x_mac_learn(struct lan966x *lan966x, int port,
 	return __lan966x_mac_learn(lan966x, port, false, mac, vid, type);
 }
 
+static int lan966x_mac_learn_locked(struct lan966x *lan966x, int port,
+				    const unsigned char mac[ETH_ALEN],
+				    unsigned int vid,
+				    enum macaccess_entry_type type)
+{
+	WARN_ON(type != ENTRYTYPE_NORMAL && type != ENTRYTYPE_LOCKED);
+
+	return __lan966x_mac_learn_locked(lan966x, port, false, mac, vid, type);
+}
+
 static int lan966x_mac_forget_locked(struct lan966x *lan966x,
 				     const unsigned char mac[ETH_ALEN],
 				     unsigned int vid,
@@ -178,8 +200,9 @@ void lan966x_mac_init(struct lan966x *lan966x)
 	INIT_LIST_HEAD(&lan966x->mac_entries);
 }
 
-static struct lan966x_mac_entry *lan966x_mac_alloc_entry(const unsigned char *mac,
-							 u16 vid, u16 port_index)
+static struct lan966x_mac_entry *lan966x_mac_alloc_entry(struct lan966x_port *port,
+							 const unsigned char *mac,
+							 u16 vid)
 {
 	struct lan966x_mac_entry *mac_entry;
 
@@ -189,8 +212,9 @@ static struct lan966x_mac_entry *lan966x_mac_alloc_entry(const unsigned char *ma
 
 	memcpy(mac_entry->mac, mac, ETH_ALEN);
 	mac_entry->vid = vid;
-	mac_entry->port_index = port_index;
+	mac_entry->port_index = port->chip_port;
 	mac_entry->row = LAN966X_MAC_INVALID_ROW;
+	mac_entry->lag = port->bond ? true : false;
 	return mac_entry;
 }
 
@@ -269,7 +293,7 @@ int lan966x_mac_add_entry(struct lan966x *lan966x, struct lan966x_port *port,
 		goto mac_learn;
 	}
 
-	mac_entry = lan966x_mac_alloc_entry(addr, vid, port->chip_port);
+	mac_entry = lan966x_mac_alloc_entry(port, addr, vid);
 	if (!mac_entry) {
 		spin_unlock(&lan966x->mac_lock);
 		return -ENOMEM;
@@ -278,7 +302,8 @@ int lan966x_mac_add_entry(struct lan966x *lan966x, struct lan966x_port *port,
 	list_add_tail(&mac_entry->list, &lan966x->mac_entries);
 	spin_unlock(&lan966x->mac_lock);
 
-	lan966x_fdb_call_notifiers(SWITCHDEV_FDB_OFFLOADED, addr, vid, port->dev);
+	lan966x_fdb_call_notifiers(SWITCHDEV_FDB_OFFLOADED, addr, vid,
+				   port->bond ?: port->dev);
 
 mac_learn:
 	lan966x_mac_learn(lan966x, port->chip_port, addr, vid, ENTRYTYPE_LOCKED);
@@ -309,6 +334,50 @@ int lan966x_mac_del_entry(struct lan966x *lan966x, const unsigned char *addr,
 	return 0;
 }
 
+void lan966x_mac_lag_replace_port_entry(struct lan966x *lan966x,
+					struct lan966x_port *src,
+					struct lan966x_port *dst)
+{
+	struct lan966x_mac_entry *mac_entry;
+
+	spin_lock(&lan966x->mac_lock);
+	list_for_each_entry(mac_entry, &lan966x->mac_entries, list) {
+		if (mac_entry->port_index == src->chip_port &&
+		    mac_entry->lag) {
+			lan966x_mac_forget_locked(lan966x, mac_entry->mac,
+						  mac_entry->vid,
+						  ENTRYTYPE_LOCKED);
+
+			lan966x_mac_learn_locked(lan966x, dst->chip_port,
+						 mac_entry->mac, mac_entry->vid,
+						 ENTRYTYPE_LOCKED);
+			mac_entry->port_index = dst->chip_port;
+		}
+	}
+	spin_unlock(&lan966x->mac_lock);
+}
+
+void lan966x_mac_lag_remove_port_entry(struct lan966x *lan966x,
+				       struct lan966x_port *src)
+{
+	struct lan966x_mac_entry *mac_entry, *tmp;
+
+	spin_lock(&lan966x->mac_lock);
+	list_for_each_entry_safe(mac_entry, tmp, &lan966x->mac_entries,
+				 list) {
+		if (mac_entry->port_index == src->chip_port &&
+		    mac_entry->lag) {
+			lan966x_mac_forget_locked(lan966x, mac_entry->mac,
+						  mac_entry->vid,
+						  ENTRYTYPE_LOCKED);
+
+			list_del(&mac_entry->list);
+			kfree(mac_entry);
+		}
+	}
+	spin_unlock(&lan966x->mac_lock);
+}
+
 void lan966x_mac_purge_entries(struct lan966x *lan966x)
 {
 	struct lan966x_mac_entry *mac_entry, *tmp;
@@ -354,6 +423,7 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row,
 	struct lan966x_mac_entry *mac_entry, *tmp;
 	unsigned char mac[ETH_ALEN] __aligned(2);
 	struct list_head mac_deleted_entries;
+	struct lan966x_port *port;
 	u32 dest_idx;
 	u32 column;
 	u16 vid;
@@ -406,9 +476,10 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row,
 		/* Notify the bridge that the entry doesn't exist
 		 * anymore in the HW
 		 */
+		port = lan966x->ports[mac_entry->port_index];
 		lan966x_mac_notifiers(SWITCHDEV_FDB_DEL_TO_BRIDGE,
 				      mac_entry->mac, mac_entry->vid,
-				      lan966x->ports[mac_entry->port_index]->dev);
+				      port->bond ?: port->dev);
 		list_del(&mac_entry->list);
 		kfree(mac_entry);
 	}
@@ -440,7 +511,8 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row,
 			continue;
 		}
 
-		mac_entry = lan966x_mac_alloc_entry(mac, vid, dest_idx);
+		port = lan966x->ports[dest_idx];
+		mac_entry = lan966x_mac_alloc_entry(port, mac, vid);
 		if (!mac_entry) {
 			spin_unlock(&lan966x->mac_lock);
 			return;
@@ -451,7 +523,7 @@ static void lan966x_mac_irq_process(struct lan966x *lan966x, u32 row,
 		spin_unlock(&lan966x->mac_lock);
 
 		lan966x_mac_notifiers(SWITCHDEV_FDB_ADD_TO_BRIDGE,
-				      mac, vid, lan966x->ports[dest_idx]->dev);
+				      mac, vid, port->bond ?: port->dev);
 	}
 }
 
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c
index d928b75f3780..be2fd030cccb 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c
@@ -344,7 +344,8 @@ static void lan966x_ifh_set_timestamp(void *ifh, u64 timestamp)
 		IFH_POS_TIMESTAMP, IFH_LEN * 4, PACK, 0);
 }
 
-static int lan966x_port_xmit(struct sk_buff *skb, struct net_device *dev)
+static netdev_tx_t lan966x_port_xmit(struct sk_buff *skb,
+				     struct net_device *dev)
 {
 	struct lan966x_port *port = netdev_priv(dev);
 	struct lan966x *lan966x = port->lan966x;
@@ -466,6 +467,7 @@ static const struct net_device_ops lan966x_port_netdev_ops = {
 	.ndo_set_mac_address		= lan966x_port_set_mac_address,
 	.ndo_get_port_parent_id		= lan966x_port_get_parent_id,
 	.ndo_eth_ioctl			= lan966x_port_ioctl,
+	.ndo_setup_tc			= lan966x_tc_setup,
 };
 
 bool lan966x_netdevice_check(const struct net_device *dev)
@@ -738,7 +740,8 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p,
 		return -EINVAL;
 
 	dev = devm_alloc_etherdev_mqs(lan966x->dev,
-				      sizeof(struct lan966x_port), 8, 1);
+				      sizeof(struct lan966x_port),
+				      NUM_PRIO_QUEUES, 1);
 	if (!dev)
 		return -ENOMEM;
 
@@ -754,7 +757,9 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p,
 	dev->netdev_ops = &lan966x_port_netdev_ops;
 	dev->ethtool_ops = &lan966x_ethtool_ops;
 	dev->features |= NETIF_F_HW_VLAN_CTAG_TX |
-			 NETIF_F_HW_VLAN_STAG_TX;
+			 NETIF_F_HW_VLAN_STAG_TX |
+			 NETIF_F_HW_TC;
+	dev->hw_features |= NETIF_F_HW_TC;
 	dev->needed_headroom = IFH_LEN * sizeof(u32);
 
 	eth_hw_addr_gen(dev, lan966x->base_mac, p + 1);
@@ -770,6 +775,7 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p,
 	port->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
 		MAC_10 | MAC_100 | MAC_1000FD | MAC_2500FD;
 
+	phy_interface_set_rgmii(port->phylink_config.supported_interfaces);
 	__set_bit(PHY_INTERFACE_MODE_MII,
 		  port->phylink_config.supported_interfaces);
 	__set_bit(PHY_INTERFACE_MODE_GMII,
@@ -778,6 +784,8 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p,
 		  port->phylink_config.supported_interfaces);
 	__set_bit(PHY_INTERFACE_MODE_QSGMII,
 		  port->phylink_config.supported_interfaces);
+	__set_bit(PHY_INTERFACE_MODE_QUSGMII,
+		  port->phylink_config.supported_interfaces);
 	__set_bit(PHY_INTERFACE_MODE_1000BASEX,
 		  port->phylink_config.supported_interfaces);
 	__set_bit(PHY_INTERFACE_MODE_2500BASEX,
@@ -956,6 +964,8 @@ static void lan966x_init(struct lan966x *lan966x)
 		lan966x, ANA_ANAINTR);
 
 	spin_lock_init(&lan966x->tx_lock);
+
+	lan966x_taprio_init(lan966x);
 }
 
 static int lan966x_ram_init(struct lan966x *lan966x)
@@ -969,7 +979,8 @@ static int lan966x_reset_switch(struct lan966x *lan966x)
 	int val = 0;
 	int ret;
 
-	switch_reset = devm_reset_control_get_shared(lan966x->dev, "switch");
+	switch_reset = devm_reset_control_get_optional_shared(lan966x->dev,
+							      "switch");
 	if (IS_ERR(switch_reset))
 		return dev_err_probe(lan966x->dev, PTR_ERR(switch_reset),
 				     "Could not obtain switch reset");
@@ -1164,6 +1175,7 @@ static int lan966x_remove(struct platform_device *pdev)
 {
 	struct lan966x *lan966x = platform_get_drvdata(pdev);
 
+	lan966x_taprio_deinit(lan966x);
 	lan966x_fdma_deinit(lan966x);
 	lan966x_cleanup_ports(lan966x);
 
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
index 2787055c1847..9656071b8289 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h
@@ -9,6 +9,8 @@
 #include <linux/phy.h>
 #include <linux/phylink.h>
 #include <linux/ptp_clock_kernel.h>
+#include <net/pkt_cls.h>
+#include <net/pkt_sched.h>
 #include <net/switchdev.h>
 
 #include "lan966x_regs.h"
@@ -36,6 +38,7 @@
 
 #define NUM_PHYS_PORTS			8
 #define CPU_PORT			8
+#define NUM_PRIO_QUEUES			8
 
 /* Reserved PGIDs */
 #define PGID_CPU			(PGID_AGGR - 6)
@@ -79,6 +82,9 @@
 #define FDMA_INJ_CHANNEL		0
 #define FDMA_DCB_MAX			512
 
+#define SE_IDX_QUEUE			0  /* 0-79 : Queue scheduler elements */
+#define SE_IDX_PORT			80 /* 80-89 : Port schedular elements */
+
 /* MAC table entry types.
  * ENTRYTYPE_NORMAL is subject to aging.
  * ENTRYTYPE_LOCKED is not subject to aging.
@@ -258,6 +264,11 @@ struct lan966x {
 	struct lan966x_rx rx;
 	struct lan966x_tx tx;
 	struct napi_struct napi;
+
+	/* Mirror */
+	struct lan966x_port *mirror_monitor;
+	u32 mirror_mask[2];
+	u32 mirror_count;
 };
 
 struct lan966x_port_config {
@@ -270,6 +281,15 @@ struct lan966x_port_config {
 	bool autoneg;
 };
 
+struct lan966x_port_tc {
+	bool ingress_shared_block;
+	unsigned long police_id;
+	unsigned long ingress_mirror_id;
+	unsigned long egress_mirror_id;
+	struct flow_stats police_stat;
+	struct flow_stats mirror_stat;
+};
+
 struct lan966x_port {
 	struct net_device *dev;
 	struct lan966x *lan966x;
@@ -292,11 +312,19 @@ struct lan966x_port {
 	u8 ptp_cmd;
 	u16 ts_id;
 	struct sk_buff_head tx_skbs;
+
+	struct net_device *bond;
+	bool lag_tx_active;
+	enum netdev_lag_hash hash_type;
+
+	struct lan966x_port_tc tc;
 };
 
 extern const struct phylink_mac_ops lan966x_phylink_mac_ops;
 extern const struct phylink_pcs_ops lan966x_phylink_pcs_ops;
 extern const struct ethtool_ops lan966x_ethtool_ops;
+extern struct notifier_block lan966x_switchdev_nb __read_mostly;
+extern struct notifier_block lan966x_switchdev_blocking_nb __read_mostly;
 
 bool lan966x_netdevice_check(const struct net_device *dev);
 
@@ -345,6 +373,11 @@ int lan966x_mac_add_entry(struct lan966x *lan966x,
 			  struct lan966x_port *port,
 			  const unsigned char *addr,
 			  u16 vid);
+void lan966x_mac_lag_replace_port_entry(struct lan966x *lan966x,
+					struct lan966x_port *src,
+					struct lan966x_port *dst);
+void lan966x_mac_lag_remove_port_entry(struct lan966x *lan966x,
+				       struct lan966x_port *src);
 void lan966x_mac_purge_entries(struct lan966x *lan966x);
 irqreturn_t lan966x_mac_irq_handler(struct lan966x *lan966x);
 
@@ -369,6 +402,7 @@ void lan966x_fdb_write_entries(struct lan966x *lan966x, u16 vid);
 void lan966x_fdb_erase_entries(struct lan966x *lan966x, u16 vid);
 int lan966x_fdb_init(struct lan966x *lan966x);
 void lan966x_fdb_deinit(struct lan966x *lan966x);
+void lan966x_fdb_flush_workqueue(struct lan966x *lan966x);
 int lan966x_handle_fdb(struct net_device *dev,
 		       struct net_device *orig_dev,
 		       unsigned long event, const void *ctx,
@@ -397,6 +431,8 @@ void lan966x_ptp_txtstamp_release(struct lan966x_port *port,
 				  struct sk_buff *skb);
 irqreturn_t lan966x_ptp_irq_handler(int irq, void *args);
 irqreturn_t lan966x_ptp_ext_irq_handler(int irq, void *args);
+u32 lan966x_ptp_get_period_ps(void);
+int lan966x_ptp_gettime64(struct ptp_clock_info *ptp, struct timespec64 *ts);
 
 int lan966x_fdma_xmit(struct sk_buff *skb, __be32 *ifh, struct net_device *dev);
 int lan966x_fdma_change_mtu(struct lan966x *lan966x);
@@ -406,6 +442,89 @@ int lan966x_fdma_init(struct lan966x *lan966x);
 void lan966x_fdma_deinit(struct lan966x *lan966x);
 irqreturn_t lan966x_fdma_irq_handler(int irq, void *args);
 
+int lan966x_lag_port_join(struct lan966x_port *port,
+			  struct net_device *brport_dev,
+			  struct net_device *bond,
+			  struct netlink_ext_ack *extack);
+void lan966x_lag_port_leave(struct lan966x_port *port, struct net_device *bond);
+int lan966x_lag_port_prechangeupper(struct net_device *dev,
+				    struct netdev_notifier_changeupper_info *info);
+int lan966x_lag_port_changelowerstate(struct net_device *dev,
+				      struct netdev_notifier_changelowerstate_info *info);
+int lan966x_lag_netdev_prechangeupper(struct net_device *dev,
+				      struct netdev_notifier_changeupper_info *info);
+int lan966x_lag_netdev_changeupper(struct net_device *dev,
+				   struct netdev_notifier_changeupper_info *info);
+bool lan966x_lag_first_port(struct net_device *lag, struct net_device *dev);
+u32 lan966x_lag_get_mask(struct lan966x *lan966x, struct net_device *bond);
+
+int lan966x_port_changeupper(struct net_device *dev,
+			     struct net_device *brport_dev,
+			     struct netdev_notifier_changeupper_info *info);
+int lan966x_port_prechangeupper(struct net_device *dev,
+				struct net_device *brport_dev,
+				struct netdev_notifier_changeupper_info *info);
+void lan966x_port_stp_state_set(struct lan966x_port *port, u8 state);
+void lan966x_port_ageing_set(struct lan966x_port *port,
+			     unsigned long ageing_clock_t);
+void lan966x_update_fwd_mask(struct lan966x *lan966x);
+
+int lan966x_tc_setup(struct net_device *dev, enum tc_setup_type type,
+		     void *type_data);
+
+int lan966x_mqprio_add(struct lan966x_port *port, u8 num_tc);
+int lan966x_mqprio_del(struct lan966x_port *port);
+
+void lan966x_taprio_init(struct lan966x *lan966x);
+void lan966x_taprio_deinit(struct lan966x *lan966x);
+int lan966x_taprio_add(struct lan966x_port *port,
+		       struct tc_taprio_qopt_offload *qopt);
+int lan966x_taprio_del(struct lan966x_port *port);
+int lan966x_taprio_speed_set(struct lan966x_port *port, int speed);
+
+int lan966x_tbf_add(struct lan966x_port *port,
+		    struct tc_tbf_qopt_offload *qopt);
+int lan966x_tbf_del(struct lan966x_port *port,
+		    struct tc_tbf_qopt_offload *qopt);
+
+int lan966x_cbs_add(struct lan966x_port *port,
+		    struct tc_cbs_qopt_offload *qopt);
+int lan966x_cbs_del(struct lan966x_port *port,
+		    struct tc_cbs_qopt_offload *qopt);
+
+int lan966x_ets_add(struct lan966x_port *port,
+		    struct tc_ets_qopt_offload *qopt);
+int lan966x_ets_del(struct lan966x_port *port,
+		    struct tc_ets_qopt_offload *qopt);
+
+int lan966x_tc_matchall(struct lan966x_port *port,
+			struct tc_cls_matchall_offload *f,
+			bool ingress);
+
+int lan966x_police_port_add(struct lan966x_port *port,
+			    struct flow_action *action,
+			    struct flow_action_entry *act,
+			    unsigned long police_id,
+			    bool ingress,
+			    struct netlink_ext_ack *extack);
+int lan966x_police_port_del(struct lan966x_port *port,
+			    unsigned long police_id,
+			    struct netlink_ext_ack *extack);
+void lan966x_police_port_stats(struct lan966x_port *port,
+			       struct flow_stats *stats);
+
+int lan966x_mirror_port_add(struct lan966x_port *port,
+			    struct flow_action_entry *action,
+			    unsigned long mirror_id,
+			    bool ingress,
+			    struct netlink_ext_ack *extack);
+int lan966x_mirror_port_del(struct lan966x_port *port,
+			    bool ingress,
+			    struct netlink_ext_ack *extack);
+void lan966x_mirror_port_stats(struct lan966x_port *port,
+			       struct flow_stats *stats,
+			       bool ingress);
+
 static inline void __iomem *lan_addr(void __iomem *base[],
 				     int id, int tinst, int tcnt,
 				     int gbase, int ginst,
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_mirror.c b/drivers/net/ethernet/microchip/lan966x/lan966x_mirror.c
new file mode 100644
index 000000000000..7e1ba3f40c35
--- /dev/null
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_mirror.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include "lan966x_main.h"
+
+int lan966x_mirror_port_add(struct lan966x_port *port,
+			    struct flow_action_entry *action,
+			    unsigned long mirror_id,
+			    bool ingress,
+			    struct netlink_ext_ack *extack)
+{
+	struct lan966x *lan966x = port->lan966x;
+	struct lan966x_port *monitor_port;
+
+	if (!lan966x_netdevice_check(action->dev)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Destination not an lan966x port");
+		return -EOPNOTSUPP;
+	}
+
+	monitor_port = netdev_priv(action->dev);
+
+	if (lan966x->mirror_mask[ingress] & BIT(port->chip_port)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Mirror already exists");
+		return -EEXIST;
+	}
+
+	if (lan966x->mirror_monitor &&
+	    lan966x->mirror_monitor != monitor_port) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot change mirror port while in use");
+		return -EBUSY;
+	}
+
+	if (port == monitor_port) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Cannot mirror the monitor port");
+		return -EINVAL;
+	}
+
+	lan966x->mirror_mask[ingress] |= BIT(port->chip_port);
+
+	lan966x->mirror_monitor = monitor_port;
+	lan_wr(BIT(monitor_port->chip_port), lan966x, ANA_MIRRORPORTS);
+
+	if (ingress) {
+		lan_rmw(ANA_PORT_CFG_SRC_MIRROR_ENA_SET(1),
+			ANA_PORT_CFG_SRC_MIRROR_ENA,
+			lan966x, ANA_PORT_CFG(port->chip_port));
+	} else {
+		lan_wr(lan966x->mirror_mask[0], lan966x,
+		       ANA_EMIRRORPORTS);
+	}
+
+	lan966x->mirror_count++;
+
+	if (ingress)
+		port->tc.ingress_mirror_id = mirror_id;
+	else
+		port->tc.egress_mirror_id = mirror_id;
+
+	return 0;
+}
+
+int lan966x_mirror_port_del(struct lan966x_port *port,
+			    bool ingress,
+			    struct netlink_ext_ack *extack)
+{
+	struct lan966x *lan966x = port->lan966x;
+
+	if (!(lan966x->mirror_mask[ingress] & BIT(port->chip_port))) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "There is no mirroring for this port");
+		return -ENOENT;
+	}
+
+	lan966x->mirror_mask[ingress] &= ~BIT(port->chip_port);
+
+	if (ingress) {
+		lan_rmw(ANA_PORT_CFG_SRC_MIRROR_ENA_SET(0),
+			ANA_PORT_CFG_SRC_MIRROR_ENA,
+			lan966x, ANA_PORT_CFG(port->chip_port));
+	} else {
+		lan_wr(lan966x->mirror_mask[0], lan966x,
+		       ANA_EMIRRORPORTS);
+	}
+
+	lan966x->mirror_count--;
+
+	if (lan966x->mirror_count == 0) {
+		lan966x->mirror_monitor = NULL;
+		lan_wr(0, lan966x, ANA_MIRRORPORTS);
+	}
+
+	if (ingress)
+		port->tc.ingress_mirror_id = 0;
+	else
+		port->tc.egress_mirror_id = 0;
+
+	return 0;
+}
+
+void lan966x_mirror_port_stats(struct lan966x_port *port,
+			       struct flow_stats *stats,
+			       bool ingress)
+{
+	struct rtnl_link_stats64 new_stats;
+	struct flow_stats *old_stats;
+
+	old_stats = &port->tc.mirror_stat;
+	lan966x_stats_get(port->dev, &new_stats);
+
+	if (ingress) {
+		flow_stats_update(stats,
+				  new_stats.rx_bytes - old_stats->bytes,
+				  new_stats.rx_packets - old_stats->pkts,
+				  new_stats.rx_dropped - old_stats->drops,
+				  old_stats->lastused,
+				  FLOW_ACTION_HW_STATS_IMMEDIATE);
+
+		old_stats->bytes = new_stats.rx_bytes;
+		old_stats->pkts = new_stats.rx_packets;
+		old_stats->drops = new_stats.rx_dropped;
+		old_stats->lastused = jiffies;
+	} else {
+		flow_stats_update(stats,
+				  new_stats.tx_bytes - old_stats->bytes,
+				  new_stats.tx_packets - old_stats->pkts,
+				  new_stats.tx_dropped - old_stats->drops,
+				  old_stats->lastused,
+				  FLOW_ACTION_HW_STATS_IMMEDIATE);
+
+		old_stats->bytes = new_stats.tx_bytes;
+		old_stats->pkts = new_stats.tx_packets;
+		old_stats->drops = new_stats.tx_dropped;
+		old_stats->lastused = jiffies;
+	}
+}
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_mqprio.c b/drivers/net/ethernet/microchip/lan966x/lan966x_mqprio.c
new file mode 100644
index 000000000000..7fa76e74f9e2
--- /dev/null
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_mqprio.c
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include "lan966x_main.h"
+
+int lan966x_mqprio_add(struct lan966x_port *port, u8 num_tc)
+{
+	u8 i;
+
+	if (num_tc != NUM_PRIO_QUEUES) {
+		netdev_err(port->dev, "Only %d traffic classes supported\n",
+			   NUM_PRIO_QUEUES);
+		return -EINVAL;
+	}
+
+	netdev_set_num_tc(port->dev, num_tc);
+
+	for (i = 0; i < num_tc; ++i)
+		netdev_set_tc_queue(port->dev, i, 1, i);
+
+	return 0;
+}
+
+int lan966x_mqprio_del(struct lan966x_port *port)
+{
+	netdev_reset_tc(port->dev);
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c b/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c
index 38a7e95d69b4..e4ac59480514 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c
@@ -28,11 +28,12 @@ static int lan966x_phylink_mac_prepare(struct phylink_config *config,
 				       phy_interface_t iface)
 {
 	struct lan966x_port *port = netdev_priv(to_net_dev(config->dev));
+	phy_interface_t serdes_mode = iface;
 	int err;
 
 	if (port->serdes) {
 		err = phy_set_mode_ext(port->serdes, PHY_MODE_ETHERNET,
-				       iface);
+				       serdes_mode);
 		if (err) {
 			netdev_err(to_net_dev(config->dev),
 				   "Could not set mode of SerDes\n");
@@ -59,6 +60,9 @@ static void lan966x_phylink_mac_link_up(struct phylink_config *config,
 	port_config->pause |= tx_pause ? MLO_PAUSE_TX : 0;
 	port_config->pause |= rx_pause ? MLO_PAUSE_RX : 0;
 
+	if (phy_interface_mode_is_rgmii(interface))
+		phy_set_speed(port->serdes, speed);
+
 	lan966x_port_config_up(port);
 }
 
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_police.c b/drivers/net/ethernet/microchip/lan966x/lan966x_police.c
new file mode 100644
index 000000000000..a9aec900d608
--- /dev/null
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_police.c
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include "lan966x_main.h"
+
+/* 0-8 : 9 port policers */
+#define POL_IDX_PORT	0
+
+/* Policer order: Serial (QoS -> Port -> VCAP) */
+#define POL_ORDER	0x1d3
+
+struct lan966x_tc_policer {
+	/* kilobit per second */
+	u32 rate;
+	/* bytes */
+	u32 burst;
+};
+
+static int lan966x_police_add(struct lan966x_port *port,
+			      struct lan966x_tc_policer *pol,
+			      u16 pol_idx)
+{
+	struct lan966x *lan966x = port->lan966x;
+
+	/* Rate unit is 33 1/3 kpps */
+	pol->rate = DIV_ROUND_UP(pol->rate * 3, 100);
+	/* Avoid zero burst size */
+	pol->burst = pol->burst ?: 1;
+	/* Unit is 4kB */
+	pol->burst = DIV_ROUND_UP(pol->burst, 4096);
+
+	if (pol->rate > GENMASK(15, 0) ||
+	    pol->burst > GENMASK(6, 0))
+		return -EINVAL;
+
+	lan_wr(ANA_POL_MODE_DROP_ON_YELLOW_ENA_SET(0) |
+	       ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA_SET(0) |
+	       ANA_POL_MODE_IPG_SIZE_SET(20) |
+	       ANA_POL_MODE_FRM_MODE_SET(1) |
+	       ANA_POL_MODE_OVERSHOOT_ENA_SET(1),
+	       lan966x, ANA_POL_MODE(pol_idx));
+
+	lan_wr(ANA_POL_PIR_STATE_PIR_LVL_SET(0),
+	       lan966x, ANA_POL_PIR_STATE(pol_idx));
+
+	lan_wr(ANA_POL_PIR_CFG_PIR_RATE_SET(pol->rate) |
+	       ANA_POL_PIR_CFG_PIR_BURST_SET(pol->burst),
+	       lan966x, ANA_POL_PIR_CFG(pol_idx));
+
+	return 0;
+}
+
+static int lan966x_police_del(struct lan966x_port *port,
+			      u16 pol_idx)
+{
+	struct lan966x *lan966x = port->lan966x;
+
+	lan_wr(ANA_POL_MODE_DROP_ON_YELLOW_ENA_SET(0) |
+	       ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA_SET(0) |
+	       ANA_POL_MODE_IPG_SIZE_SET(20) |
+	       ANA_POL_MODE_FRM_MODE_SET(2) |
+	       ANA_POL_MODE_OVERSHOOT_ENA_SET(1),
+	       lan966x, ANA_POL_MODE(pol_idx));
+
+	lan_wr(ANA_POL_PIR_STATE_PIR_LVL_SET(0),
+	       lan966x, ANA_POL_PIR_STATE(pol_idx));
+
+	lan_wr(ANA_POL_PIR_CFG_PIR_RATE_SET(GENMASK(14, 0)) |
+	       ANA_POL_PIR_CFG_PIR_BURST_SET(0),
+	       lan966x, ANA_POL_PIR_CFG(pol_idx));
+
+	return 0;
+}
+
+static int lan966x_police_validate(struct lan966x_port *port,
+				   const struct flow_action *action,
+				   const struct flow_action_entry *act,
+				   unsigned long police_id,
+				   bool ingress,
+				   struct netlink_ext_ack *extack)
+{
+	if (act->police.exceed.act_id != FLOW_ACTION_DROP) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when exceed action is not drop");
+		return -EOPNOTSUPP;
+	}
+
+	if (act->police.notexceed.act_id != FLOW_ACTION_PIPE &&
+	    act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when conform action is not pipe or ok");
+		return -EOPNOTSUPP;
+	}
+
+	if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT &&
+	    !flow_action_is_last_entry(action, act)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when conform action is ok, but action is not last");
+		return -EOPNOTSUPP;
+	}
+
+	if (act->police.peakrate_bytes_ps ||
+	    act->police.avrate || act->police.overhead) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when peakrate/avrate/overhead is configured");
+		return -EOPNOTSUPP;
+	}
+
+	if (act->police.rate_pkt_ps) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "QoS offload not support packets per second");
+		return -EOPNOTSUPP;
+	}
+
+	if (!ingress) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Policer is not supported on egress");
+		return -EOPNOTSUPP;
+	}
+
+	if (port->tc.ingress_shared_block) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Policer is not supported on shared ingress blocks");
+		return -EOPNOTSUPP;
+	}
+
+	if (port->tc.police_id && port->tc.police_id != police_id) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Only one policer per port is supported");
+		return -EEXIST;
+	}
+
+	return 0;
+}
+
+int lan966x_police_port_add(struct lan966x_port *port,
+			    struct flow_action *action,
+			    struct flow_action_entry *act,
+			    unsigned long police_id,
+			    bool ingress,
+			    struct netlink_ext_ack *extack)
+{
+	struct lan966x *lan966x = port->lan966x;
+	struct rtnl_link_stats64 new_stats;
+	struct lan966x_tc_policer pol;
+	struct flow_stats *old_stats;
+	int err;
+
+	err = lan966x_police_validate(port, action, act, police_id, ingress,
+				      extack);
+	if (err)
+		return err;
+
+	memset(&pol, 0, sizeof(pol));
+
+	pol.rate = div_u64(act->police.rate_bytes_ps, 1000) * 8;
+	pol.burst = act->police.burst;
+
+	err = lan966x_police_add(port, &pol, POL_IDX_PORT + port->chip_port);
+	if (err) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Failed to add policer to port");
+		return err;
+	}
+
+	lan_rmw(ANA_POL_CFG_PORT_POL_ENA_SET(1) |
+		ANA_POL_CFG_POL_ORDER_SET(POL_ORDER),
+		ANA_POL_CFG_PORT_POL_ENA |
+		ANA_POL_CFG_POL_ORDER,
+		lan966x, ANA_POL_CFG(port->chip_port));
+
+	port->tc.police_id = police_id;
+
+	/* Setup initial stats */
+	old_stats = &port->tc.police_stat;
+	lan966x_stats_get(port->dev, &new_stats);
+	old_stats->bytes = new_stats.rx_bytes;
+	old_stats->pkts = new_stats.rx_packets;
+	old_stats->drops = new_stats.rx_dropped;
+	old_stats->lastused = jiffies;
+
+	return 0;
+}
+
+int lan966x_police_port_del(struct lan966x_port *port,
+			    unsigned long police_id,
+			    struct netlink_ext_ack *extack)
+{
+	struct lan966x *lan966x = port->lan966x;
+	int err;
+
+	if (port->tc.police_id != police_id) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Invalid policer id");
+		return -EINVAL;
+	}
+
+	err = lan966x_police_del(port, port->tc.police_id);
+	if (err) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Failed to add policer to port");
+		return err;
+	}
+
+	lan_rmw(ANA_POL_CFG_PORT_POL_ENA_SET(0) |
+		ANA_POL_CFG_POL_ORDER_SET(POL_ORDER),
+		ANA_POL_CFG_PORT_POL_ENA |
+		ANA_POL_CFG_POL_ORDER,
+		lan966x, ANA_POL_CFG(port->chip_port));
+
+	port->tc.police_id = 0;
+
+	return 0;
+}
+
+void lan966x_police_port_stats(struct lan966x_port *port,
+			       struct flow_stats *stats)
+{
+	struct rtnl_link_stats64 new_stats;
+	struct flow_stats *old_stats;
+
+	old_stats = &port->tc.police_stat;
+	lan966x_stats_get(port->dev, &new_stats);
+
+	flow_stats_update(stats,
+			  new_stats.rx_bytes - old_stats->bytes,
+			  new_stats.rx_packets - old_stats->pkts,
+			  new_stats.rx_dropped - old_stats->drops,
+			  old_stats->lastused,
+			  FLOW_ACTION_HW_STATS_IMMEDIATE);
+
+	old_stats->bytes = new_stats.rx_bytes;
+	old_stats->pkts = new_stats.rx_packets;
+	old_stats->drops = new_stats.rx_dropped;
+	old_stats->lastused = jiffies;
+}
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_port.c b/drivers/net/ethernet/microchip/lan966x/lan966x_port.c
index f141644e4372..1a61c6cdb077 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_port.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_port.c
@@ -165,10 +165,12 @@ static void lan966x_port_link_up(struct lan966x_port *port)
 		break;
 	}
 
+	lan966x_taprio_speed_set(port, config->speed);
+
 	/* Also the GIGA_MODE_ENA(1) needs to be set regardless of the
 	 * port speed for QSGMII ports.
 	 */
-	if (config->portmode == PHY_INTERFACE_MODE_QSGMII)
+	if (phy_interface_num_ports(config->portmode) == 4)
 		mode = DEV_MAC_MODE_CFG_GIGA_MODE_ENA_SET(1);
 
 	lan_wr(config->duplex | mode,
@@ -331,10 +333,14 @@ int lan966x_port_pcs_set(struct lan966x_port *port,
 	struct lan966x *lan966x = port->lan966x;
 	bool inband_aneg = false;
 	bool outband;
+	bool full_preamble = false;
+
+	if (config->portmode == PHY_INTERFACE_MODE_QUSGMII)
+		full_preamble = true;
 
 	if (config->inband) {
 		if (config->portmode == PHY_INTERFACE_MODE_SGMII ||
-		    config->portmode == PHY_INTERFACE_MODE_QSGMII)
+		    phy_interface_num_ports(config->portmode) == 4)
 			inband_aneg = true; /* Cisco-SGMII in-band-aneg */
 		else if (config->portmode == PHY_INTERFACE_MODE_1000BASEX &&
 			 config->autoneg)
@@ -345,9 +351,15 @@ int lan966x_port_pcs_set(struct lan966x_port *port,
 		outband = true;
 	}
 
-	/* Disable or enable inband */
-	lan_rmw(DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA_SET(outband),
-		DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA,
+	/* Disable or enable inband.
+	 * For QUSGMII, we rely on the preamble to transmit data such as
+	 * timestamps, therefore force full preamble transmission, and prevent
+	 * premable shortening
+	 */
+	lan_rmw(DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA_SET(outband) |
+		DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA_SET(full_preamble),
+		DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA |
+		DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA,
 		lan966x, DEV_PCS1G_MODE_CFG(port->chip_port));
 
 	/* Enable PCS */
@@ -396,7 +408,7 @@ void lan966x_port_init(struct lan966x_port *port)
 	if (lan966x->fdma)
 		lan966x_fdma_netdev_init(lan966x, port->dev);
 
-	if (config->portmode != PHY_INTERFACE_MODE_QSGMII)
+	if (phy_interface_num_ports(config->portmode) != 4)
 		return;
 
 	lan_rmw(DEV_CLOCK_CFG_PCS_RX_RST_SET(0) |
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c
index 3a621c5165bc..e5a2bbe064f8 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ptp.c
@@ -464,8 +464,7 @@ static int lan966x_ptp_settime64(struct ptp_clock_info *ptp,
 	return 0;
 }
 
-static int lan966x_ptp_gettime64(struct ptp_clock_info *ptp,
-				 struct timespec64 *ts)
+int lan966x_ptp_gettime64(struct ptp_clock_info *ptp, struct timespec64 *ts)
 {
 	struct lan966x_phc *phc = container_of(ptp, struct lan966x_phc, info);
 	struct lan966x *lan966x = phc->lan966x;
@@ -890,3 +889,9 @@ void lan966x_ptp_rxtstamp(struct lan966x *lan966x, struct sk_buff *skb,
 	shhwtstamps = skb_hwtstamps(skb);
 	shhwtstamps->hwtstamp = full_ts_in_ns;
 }
+
+u32 lan966x_ptp_get_period_ps(void)
+{
+	/* This represents the system clock period in picoseconds */
+	return 15125;
+}
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h b/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h
index 8265ad89f0bc..1d90b93dd417 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h
@@ -90,6 +90,24 @@ enum lan966x_target {
 #define ANA_AUTOAGE_AGE_PERIOD_GET(x)\
 	FIELD_GET(ANA_AUTOAGE_AGE_PERIOD, x)
 
+/*      ANA:ANA:MIRRORPORTS */
+#define ANA_MIRRORPORTS           __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 60, 0, 1, 4)
+
+#define ANA_MIRRORPORTS_MIRRORPORTS              GENMASK(8, 0)
+#define ANA_MIRRORPORTS_MIRRORPORTS_SET(x)\
+	FIELD_PREP(ANA_MIRRORPORTS_MIRRORPORTS, x)
+#define ANA_MIRRORPORTS_MIRRORPORTS_GET(x)\
+	FIELD_GET(ANA_MIRRORPORTS_MIRRORPORTS, x)
+
+/*      ANA:ANA:EMIRRORPORTS */
+#define ANA_EMIRRORPORTS          __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 64, 0, 1, 4)
+
+#define ANA_EMIRRORPORTS_EMIRRORPORTS            GENMASK(8, 0)
+#define ANA_EMIRRORPORTS_EMIRRORPORTS_SET(x)\
+	FIELD_PREP(ANA_EMIRRORPORTS_EMIRRORPORTS, x)
+#define ANA_EMIRRORPORTS_EMIRRORPORTS_GET(x)\
+	FIELD_GET(ANA_EMIRRORPORTS_EMIRRORPORTS, x)
+
 /*      ANA:ANA:FLOODING */
 #define ANA_FLOODING(r)           __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 68, r, 8, 4)
 
@@ -330,6 +348,12 @@ enum lan966x_target {
 /*      ANA:PORT:PORT_CFG */
 #define ANA_PORT_CFG(g)           __REG(TARGET_ANA, 0, 1, 28672, g, 9, 128, 112, 0, 1, 4)
 
+#define ANA_PORT_CFG_SRC_MIRROR_ENA              BIT(13)
+#define ANA_PORT_CFG_SRC_MIRROR_ENA_SET(x)\
+	FIELD_PREP(ANA_PORT_CFG_SRC_MIRROR_ENA, x)
+#define ANA_PORT_CFG_SRC_MIRROR_ENA_GET(x)\
+	FIELD_GET(ANA_PORT_CFG_SRC_MIRROR_ENA, x)
+
 #define ANA_PORT_CFG_LEARNAUTO                   BIT(6)
 #define ANA_PORT_CFG_LEARNAUTO_SET(x)\
 	FIELD_PREP(ANA_PORT_CFG_LEARNAUTO, x)
@@ -354,6 +378,21 @@ enum lan966x_target {
 #define ANA_PORT_CFG_PORTID_VAL_GET(x)\
 	FIELD_GET(ANA_PORT_CFG_PORTID_VAL, x)
 
+/*      ANA:PORT:POL_CFG */
+#define ANA_POL_CFG(g)            __REG(TARGET_ANA, 0, 1, 28672, g, 9, 128, 116, 0, 1, 4)
+
+#define ANA_POL_CFG_PORT_POL_ENA                 BIT(17)
+#define ANA_POL_CFG_PORT_POL_ENA_SET(x)\
+	FIELD_PREP(ANA_POL_CFG_PORT_POL_ENA, x)
+#define ANA_POL_CFG_PORT_POL_ENA_GET(x)\
+	FIELD_GET(ANA_POL_CFG_PORT_POL_ENA, x)
+
+#define ANA_POL_CFG_POL_ORDER                    GENMASK(8, 0)
+#define ANA_POL_CFG_POL_ORDER_SET(x)\
+	FIELD_PREP(ANA_POL_CFG_POL_ORDER, x)
+#define ANA_POL_CFG_POL_ORDER_GET(x)\
+	FIELD_GET(ANA_POL_CFG_POL_ORDER, x)
+
 /*      ANA:PFC:PFC_CFG */
 #define ANA_PFC_CFG(g)            __REG(TARGET_ANA, 0, 1, 30720, g, 8, 64, 0, 0, 1, 4)
 
@@ -363,6 +402,108 @@ enum lan966x_target {
 #define ANA_PFC_CFG_FC_LINK_SPEED_GET(x)\
 	FIELD_GET(ANA_PFC_CFG_FC_LINK_SPEED, x)
 
+/*      ANA:COMMON:AGGR_CFG */
+#define ANA_AGGR_CFG              __REG(TARGET_ANA, 0, 1, 31232, 0, 1, 552, 0, 0, 1, 4)
+
+#define ANA_AGGR_CFG_AC_RND_ENA                  BIT(6)
+#define ANA_AGGR_CFG_AC_RND_ENA_SET(x)\
+	FIELD_PREP(ANA_AGGR_CFG_AC_RND_ENA, x)
+#define ANA_AGGR_CFG_AC_RND_ENA_GET(x)\
+	FIELD_GET(ANA_AGGR_CFG_AC_RND_ENA, x)
+
+#define ANA_AGGR_CFG_AC_DMAC_ENA                 BIT(5)
+#define ANA_AGGR_CFG_AC_DMAC_ENA_SET(x)\
+	FIELD_PREP(ANA_AGGR_CFG_AC_DMAC_ENA, x)
+#define ANA_AGGR_CFG_AC_DMAC_ENA_GET(x)\
+	FIELD_GET(ANA_AGGR_CFG_AC_DMAC_ENA, x)
+
+#define ANA_AGGR_CFG_AC_SMAC_ENA                 BIT(4)
+#define ANA_AGGR_CFG_AC_SMAC_ENA_SET(x)\
+	FIELD_PREP(ANA_AGGR_CFG_AC_SMAC_ENA, x)
+#define ANA_AGGR_CFG_AC_SMAC_ENA_GET(x)\
+	FIELD_GET(ANA_AGGR_CFG_AC_SMAC_ENA, x)
+
+#define ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA         BIT(3)
+#define ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA_SET(x)\
+	FIELD_PREP(ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA, x)
+#define ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA_GET(x)\
+	FIELD_GET(ANA_AGGR_CFG_AC_IP6_FLOW_LBL_ENA, x)
+
+#define ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA           BIT(2)
+#define ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA_SET(x)\
+	FIELD_PREP(ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA, x)
+#define ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA_GET(x)\
+	FIELD_GET(ANA_AGGR_CFG_AC_IP6_TCPUDP_ENA, x)
+
+#define ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA           BIT(1)
+#define ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA_SET(x)\
+	FIELD_PREP(ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA, x)
+#define ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA_GET(x)\
+	FIELD_GET(ANA_AGGR_CFG_AC_IP4_SIPDIP_ENA, x)
+
+#define ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA           BIT(0)
+#define ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA_SET(x)\
+	FIELD_PREP(ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA, x)
+#define ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA_GET(x)\
+	FIELD_GET(ANA_AGGR_CFG_AC_IP4_TCPUDP_ENA, x)
+
+/*      ANA:POL:POL_PIR_CFG */
+#define ANA_POL_PIR_CFG(g)        __REG(TARGET_ANA, 0, 1, 16384, g, 345, 32, 0, 0, 1, 4)
+
+#define ANA_POL_PIR_CFG_PIR_RATE                 GENMASK(20, 6)
+#define ANA_POL_PIR_CFG_PIR_RATE_SET(x)\
+	FIELD_PREP(ANA_POL_PIR_CFG_PIR_RATE, x)
+#define ANA_POL_PIR_CFG_PIR_RATE_GET(x)\
+	FIELD_GET(ANA_POL_PIR_CFG_PIR_RATE, x)
+
+#define ANA_POL_PIR_CFG_PIR_BURST                GENMASK(5, 0)
+#define ANA_POL_PIR_CFG_PIR_BURST_SET(x)\
+	FIELD_PREP(ANA_POL_PIR_CFG_PIR_BURST, x)
+#define ANA_POL_PIR_CFG_PIR_BURST_GET(x)\
+	FIELD_GET(ANA_POL_PIR_CFG_PIR_BURST, x)
+
+/*      ANA:POL:POL_MODE_CFG */
+#define ANA_POL_MODE(g)           __REG(TARGET_ANA, 0, 1, 16384, g, 345, 32, 8, 0, 1, 4)
+
+#define ANA_POL_MODE_DROP_ON_YELLOW_ENA          BIT(11)
+#define ANA_POL_MODE_DROP_ON_YELLOW_ENA_SET(x)\
+	FIELD_PREP(ANA_POL_MODE_DROP_ON_YELLOW_ENA, x)
+#define ANA_POL_MODE_DROP_ON_YELLOW_ENA_GET(x)\
+	FIELD_GET(ANA_POL_MODE_DROP_ON_YELLOW_ENA, x)
+
+#define ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA       BIT(10)
+#define ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA_SET(x)\
+	FIELD_PREP(ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA, x)
+#define ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA_GET(x)\
+	FIELD_GET(ANA_POL_MODE_MARK_ALL_FRMS_RED_ENA, x)
+
+#define ANA_POL_MODE_IPG_SIZE                    GENMASK(9, 5)
+#define ANA_POL_MODE_IPG_SIZE_SET(x)\
+	FIELD_PREP(ANA_POL_MODE_IPG_SIZE, x)
+#define ANA_POL_MODE_IPG_SIZE_GET(x)\
+	FIELD_GET(ANA_POL_MODE_IPG_SIZE, x)
+
+#define ANA_POL_MODE_FRM_MODE                    GENMASK(4, 3)
+#define ANA_POL_MODE_FRM_MODE_SET(x)\
+	FIELD_PREP(ANA_POL_MODE_FRM_MODE, x)
+#define ANA_POL_MODE_FRM_MODE_GET(x)\
+	FIELD_GET(ANA_POL_MODE_FRM_MODE, x)
+
+#define ANA_POL_MODE_OVERSHOOT_ENA               BIT(0)
+#define ANA_POL_MODE_OVERSHOOT_ENA_SET(x)\
+	FIELD_PREP(ANA_POL_MODE_OVERSHOOT_ENA, x)
+#define ANA_POL_MODE_OVERSHOOT_ENA_GET(x)\
+	FIELD_GET(ANA_POL_MODE_OVERSHOOT_ENA, x)
+
+/*      ANA:POL:POL_PIR_STATE */
+#define ANA_POL_PIR_STATE(g)      __REG(TARGET_ANA, 0, 1, 16384, g, 345, 32, 12, 0, 1, 4)
+
+#define ANA_POL_PIR_STATE_PIR_LVL                GENMASK(21, 0)
+#define ANA_POL_PIR_STATE_PIR_LVL_SET(x)\
+	FIELD_PREP(ANA_POL_PIR_STATE_PIR_LVL, x)
+#define ANA_POL_PIR_STATE_PIR_LVL_GET(x)\
+	FIELD_GET(ANA_POL_PIR_STATE_PIR_LVL, x)
+
 /*      CHIP_TOP:CUPHY_CFG:CUPHY_PORT_CFG */
 #define CHIP_TOP_CUPHY_PORT_CFG(r) __REG(TARGET_CHIP_TOP, 0, 1, 16, 0, 1, 20, 8, r, 2, 4)
 
@@ -504,6 +645,12 @@ enum lan966x_target {
 #define DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA_GET(x)\
 	FIELD_GET(DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA, x)
 
+#define DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA        BIT(1)
+#define DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA_SET(x)\
+	FIELD_PREP(DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA, x)
+#define DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA_GET(x)\
+	FIELD_GET(DEV_PCS1G_MODE_CFG_SAVE_PREAMBLE_ENA, x)
+
 /*      DEV:PCS1G_CFG_STATUS:PCS1G_SD_CFG */
 #define DEV_PCS1G_SD_CFG(t)       __REG(TARGET_DEV, t, 8, 72, 0, 1, 68, 8, 0, 1, 4)
 
@@ -967,6 +1114,215 @@ enum lan966x_target {
 /*      QSYS:RES_CTRL:RES_CFG */
 #define QSYS_RES_CFG(g)           __REG(TARGET_QSYS, 0, 1, 32768, g, 1024, 8, 0, 0, 1, 4)
 
+/*      QSYS:HSCH:CIR_CFG */
+#define QSYS_CIR_CFG(g)           __REG(TARGET_QSYS, 0, 1, 16384, g, 90, 128, 0, 0, 1, 4)
+
+#define QSYS_CIR_CFG_CIR_RATE                    GENMASK(20, 6)
+#define QSYS_CIR_CFG_CIR_RATE_SET(x)\
+	FIELD_PREP(QSYS_CIR_CFG_CIR_RATE, x)
+#define QSYS_CIR_CFG_CIR_RATE_GET(x)\
+	FIELD_GET(QSYS_CIR_CFG_CIR_RATE, x)
+
+#define QSYS_CIR_CFG_CIR_BURST                   GENMASK(5, 0)
+#define QSYS_CIR_CFG_CIR_BURST_SET(x)\
+	FIELD_PREP(QSYS_CIR_CFG_CIR_BURST, x)
+#define QSYS_CIR_CFG_CIR_BURST_GET(x)\
+	FIELD_GET(QSYS_CIR_CFG_CIR_BURST, x)
+
+/*      QSYS:HSCH:SE_CFG */
+#define QSYS_SE_CFG(g)            __REG(TARGET_QSYS, 0, 1, 16384, g, 90, 128, 8, 0, 1, 4)
+
+#define QSYS_SE_CFG_SE_DWRR_CNT                  GENMASK(9, 6)
+#define QSYS_SE_CFG_SE_DWRR_CNT_SET(x)\
+	FIELD_PREP(QSYS_SE_CFG_SE_DWRR_CNT, x)
+#define QSYS_SE_CFG_SE_DWRR_CNT_GET(x)\
+	FIELD_GET(QSYS_SE_CFG_SE_DWRR_CNT, x)
+
+#define QSYS_SE_CFG_SE_RR_ENA                    BIT(5)
+#define QSYS_SE_CFG_SE_RR_ENA_SET(x)\
+	FIELD_PREP(QSYS_SE_CFG_SE_RR_ENA, x)
+#define QSYS_SE_CFG_SE_RR_ENA_GET(x)\
+	FIELD_GET(QSYS_SE_CFG_SE_RR_ENA, x)
+
+#define QSYS_SE_CFG_SE_AVB_ENA                   BIT(4)
+#define QSYS_SE_CFG_SE_AVB_ENA_SET(x)\
+	FIELD_PREP(QSYS_SE_CFG_SE_AVB_ENA, x)
+#define QSYS_SE_CFG_SE_AVB_ENA_GET(x)\
+	FIELD_GET(QSYS_SE_CFG_SE_AVB_ENA, x)
+
+#define QSYS_SE_CFG_SE_FRM_MODE                  GENMASK(3, 2)
+#define QSYS_SE_CFG_SE_FRM_MODE_SET(x)\
+	FIELD_PREP(QSYS_SE_CFG_SE_FRM_MODE, x)
+#define QSYS_SE_CFG_SE_FRM_MODE_GET(x)\
+	FIELD_GET(QSYS_SE_CFG_SE_FRM_MODE, x)
+
+#define QSYS_SE_DWRR_CFG(g, r)    __REG(TARGET_QSYS, 0, 1, 16384, g, 90, 128, 12, r, 12, 4)
+
+#define QSYS_SE_DWRR_CFG_DWRR_COST               GENMASK(4, 0)
+#define QSYS_SE_DWRR_CFG_DWRR_COST_SET(x)\
+	FIELD_PREP(QSYS_SE_DWRR_CFG_DWRR_COST, x)
+#define QSYS_SE_DWRR_CFG_DWRR_COST_GET(x)\
+	FIELD_GET(QSYS_SE_DWRR_CFG_DWRR_COST, x)
+
+/*      QSYS:TAS_CONFIG:TAS_CFG_CTRL */
+#define QSYS_TAS_CFG_CTRL         __REG(TARGET_QSYS, 0, 1, 57372, 0, 1, 12, 0, 0, 1, 4)
+
+#define QSYS_TAS_CFG_CTRL_LIST_NUM_MAX           GENMASK(27, 23)
+#define QSYS_TAS_CFG_CTRL_LIST_NUM_MAX_SET(x)\
+	FIELD_PREP(QSYS_TAS_CFG_CTRL_LIST_NUM_MAX, x)
+#define QSYS_TAS_CFG_CTRL_LIST_NUM_MAX_GET(x)\
+	FIELD_GET(QSYS_TAS_CFG_CTRL_LIST_NUM_MAX, x)
+
+#define QSYS_TAS_CFG_CTRL_LIST_NUM               GENMASK(22, 18)
+#define QSYS_TAS_CFG_CTRL_LIST_NUM_SET(x)\
+	FIELD_PREP(QSYS_TAS_CFG_CTRL_LIST_NUM, x)
+#define QSYS_TAS_CFG_CTRL_LIST_NUM_GET(x)\
+	FIELD_GET(QSYS_TAS_CFG_CTRL_LIST_NUM, x)
+
+#define QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q        BIT(17)
+#define QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q_SET(x)\
+	FIELD_PREP(QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q, x)
+#define QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q_GET(x)\
+	FIELD_GET(QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q, x)
+
+#define QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM          GENMASK(16, 5)
+#define QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM_SET(x)\
+	FIELD_PREP(QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM, x)
+#define QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM_GET(x)\
+	FIELD_GET(QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM, x)
+
+/*      QSYS:TAS_CONFIG:TAS_GATE_STATE_CTRL */
+#define QSYS_TAS_GS_CTRL          __REG(TARGET_QSYS, 0, 1, 57372, 0, 1, 12, 4, 0, 1, 4)
+
+#define QSYS_TAS_GS_CTRL_HSCH_POS                GENMASK(2, 0)
+#define QSYS_TAS_GS_CTRL_HSCH_POS_SET(x)\
+	FIELD_PREP(QSYS_TAS_GS_CTRL_HSCH_POS, x)
+#define QSYS_TAS_GS_CTRL_HSCH_POS_GET(x)\
+	FIELD_GET(QSYS_TAS_GS_CTRL_HSCH_POS, x)
+
+/*      QSYS:TAS_CONFIG:TAS_STATEMACHINE_CFG */
+#define QSYS_TAS_STM_CFG          __REG(TARGET_QSYS, 0, 1, 57372, 0, 1, 12, 8, 0, 1, 4)
+
+#define QSYS_TAS_STM_CFG_REVISIT_DLY             GENMASK(7, 0)
+#define QSYS_TAS_STM_CFG_REVISIT_DLY_SET(x)\
+	FIELD_PREP(QSYS_TAS_STM_CFG_REVISIT_DLY, x)
+#define QSYS_TAS_STM_CFG_REVISIT_DLY_GET(x)\
+	FIELD_GET(QSYS_TAS_STM_CFG_REVISIT_DLY, x)
+
+/*      QSYS:TAS_PROFILE_CFG:TAS_PROFILE_CONFIG */
+#define QSYS_TAS_PROFILE_CFG(g)   __REG(TARGET_QSYS, 0, 1, 30720, g, 16, 64, 32, 0, 1, 4)
+
+#define QSYS_TAS_PROFILE_CFG_PORT_NUM            GENMASK(21, 19)
+#define QSYS_TAS_PROFILE_CFG_PORT_NUM_SET(x)\
+	FIELD_PREP(QSYS_TAS_PROFILE_CFG_PORT_NUM, x)
+#define QSYS_TAS_PROFILE_CFG_PORT_NUM_GET(x)\
+	FIELD_GET(QSYS_TAS_PROFILE_CFG_PORT_NUM, x)
+
+#define QSYS_TAS_PROFILE_CFG_LINK_SPEED          GENMASK(18, 16)
+#define QSYS_TAS_PROFILE_CFG_LINK_SPEED_SET(x)\
+	FIELD_PREP(QSYS_TAS_PROFILE_CFG_LINK_SPEED, x)
+#define QSYS_TAS_PROFILE_CFG_LINK_SPEED_GET(x)\
+	FIELD_GET(QSYS_TAS_PROFILE_CFG_LINK_SPEED, x)
+
+/*      QSYS:TAS_LIST_CFG:TAS_BASE_TIME_NSEC */
+#define QSYS_TAS_BT_NSEC          __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 0, 0, 1, 4)
+
+#define QSYS_TAS_BT_NSEC_NSEC                    GENMASK(29, 0)
+#define QSYS_TAS_BT_NSEC_NSEC_SET(x)\
+	FIELD_PREP(QSYS_TAS_BT_NSEC_NSEC, x)
+#define QSYS_TAS_BT_NSEC_NSEC_GET(x)\
+	FIELD_GET(QSYS_TAS_BT_NSEC_NSEC, x)
+
+/*      QSYS:TAS_LIST_CFG:TAS_BASE_TIME_SEC_LSB */
+#define QSYS_TAS_BT_SEC_LSB       __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 4, 0, 1, 4)
+
+/*      QSYS:TAS_LIST_CFG:TAS_BASE_TIME_SEC_MSB */
+#define QSYS_TAS_BT_SEC_MSB       __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 8, 0, 1, 4)
+
+#define QSYS_TAS_BT_SEC_MSB_SEC_MSB              GENMASK(15, 0)
+#define QSYS_TAS_BT_SEC_MSB_SEC_MSB_SET(x)\
+	FIELD_PREP(QSYS_TAS_BT_SEC_MSB_SEC_MSB, x)
+#define QSYS_TAS_BT_SEC_MSB_SEC_MSB_GET(x)\
+	FIELD_GET(QSYS_TAS_BT_SEC_MSB_SEC_MSB, x)
+
+/*      QSYS:TAS_LIST_CFG:TAS_CYCLE_TIME_CFG */
+#define QSYS_TAS_CT_CFG           __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 24, 0, 1, 4)
+
+/*      QSYS:TAS_LIST_CFG:TAS_STARTUP_CFG */
+#define QSYS_TAS_STARTUP_CFG      __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 28, 0, 1, 4)
+
+#define QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX        GENMASK(27, 23)
+#define QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX_SET(x)\
+	FIELD_PREP(QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX, x)
+#define QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX_GET(x)\
+	FIELD_GET(QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX, x)
+
+/*      QSYS:TAS_LIST_CFG:TAS_LIST_CFG */
+#define QSYS_TAS_LIST_CFG         __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 32, 0, 1, 4)
+
+#define QSYS_TAS_LIST_CFG_LIST_BASE_ADDR         GENMASK(11, 0)
+#define QSYS_TAS_LIST_CFG_LIST_BASE_ADDR_SET(x)\
+	FIELD_PREP(QSYS_TAS_LIST_CFG_LIST_BASE_ADDR, x)
+#define QSYS_TAS_LIST_CFG_LIST_BASE_ADDR_GET(x)\
+	FIELD_GET(QSYS_TAS_LIST_CFG_LIST_BASE_ADDR, x)
+
+/*      QSYS:TAS_LIST_CFG:TAS_LIST_STATE */
+#define QSYS_TAS_LST              __REG(TARGET_QSYS, 0, 1, 27904, 0, 1, 64, 36, 0, 1, 4)
+
+#define QSYS_TAS_LST_LIST_STATE                  GENMASK(2, 0)
+#define QSYS_TAS_LST_LIST_STATE_SET(x)\
+	FIELD_PREP(QSYS_TAS_LST_LIST_STATE, x)
+#define QSYS_TAS_LST_LIST_STATE_GET(x)\
+	FIELD_GET(QSYS_TAS_LST_LIST_STATE, x)
+
+/*      QSYS:TAS_GCL_CFG:TAS_GCL_CTRL_CFG */
+#define QSYS_TAS_GCL_CT_CFG       __REG(TARGET_QSYS, 0, 1, 27968, 0, 1, 16, 0, 0, 1, 4)
+
+#define QSYS_TAS_GCL_CT_CFG_HSCH_POS             GENMASK(12, 10)
+#define QSYS_TAS_GCL_CT_CFG_HSCH_POS_SET(x)\
+	FIELD_PREP(QSYS_TAS_GCL_CT_CFG_HSCH_POS, x)
+#define QSYS_TAS_GCL_CT_CFG_HSCH_POS_GET(x)\
+	FIELD_GET(QSYS_TAS_GCL_CT_CFG_HSCH_POS, x)
+
+#define QSYS_TAS_GCL_CT_CFG_GATE_STATE           GENMASK(9, 2)
+#define QSYS_TAS_GCL_CT_CFG_GATE_STATE_SET(x)\
+	FIELD_PREP(QSYS_TAS_GCL_CT_CFG_GATE_STATE, x)
+#define QSYS_TAS_GCL_CT_CFG_GATE_STATE_GET(x)\
+	FIELD_GET(QSYS_TAS_GCL_CT_CFG_GATE_STATE, x)
+
+#define QSYS_TAS_GCL_CT_CFG_OP_TYPE              GENMASK(1, 0)
+#define QSYS_TAS_GCL_CT_CFG_OP_TYPE_SET(x)\
+	FIELD_PREP(QSYS_TAS_GCL_CT_CFG_OP_TYPE, x)
+#define QSYS_TAS_GCL_CT_CFG_OP_TYPE_GET(x)\
+	FIELD_GET(QSYS_TAS_GCL_CT_CFG_OP_TYPE, x)
+
+/*      QSYS:TAS_GCL_CFG:TAS_GCL_CTRL_CFG2 */
+#define QSYS_TAS_GCL_CT_CFG2      __REG(TARGET_QSYS, 0, 1, 27968, 0, 1, 16, 4, 0, 1, 4)
+
+#define QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE        GENMASK(15, 12)
+#define QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE_SET(x)\
+	FIELD_PREP(QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE, x)
+#define QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE_GET(x)\
+	FIELD_GET(QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE, x)
+
+#define QSYS_TAS_GCL_CT_CFG2_NEXT_GCL            GENMASK(11, 0)
+#define QSYS_TAS_GCL_CT_CFG2_NEXT_GCL_SET(x)\
+	FIELD_PREP(QSYS_TAS_GCL_CT_CFG2_NEXT_GCL, x)
+#define QSYS_TAS_GCL_CT_CFG2_NEXT_GCL_GET(x)\
+	FIELD_GET(QSYS_TAS_GCL_CT_CFG2_NEXT_GCL, x)
+
+/*      QSYS:TAS_GCL_CFG:TAS_GCL_TIME_CFG */
+#define QSYS_TAS_GCL_TM_CFG       __REG(TARGET_QSYS, 0, 1, 27968, 0, 1, 16, 8, 0, 1, 4)
+
+/*      QSYS:HSCH_TAS_STATE:TAS_GATE_STATE */
+#define QSYS_TAS_GATE_STATE       __REG(TARGET_QSYS, 0, 1, 28004, 0, 1, 4, 0, 0, 1, 4)
+
+#define QSYS_TAS_GATE_STATE_TAS_GATE_STATE       GENMASK(7, 0)
+#define QSYS_TAS_GATE_STATE_TAS_GATE_STATE_SET(x)\
+	FIELD_PREP(QSYS_TAS_GATE_STATE_TAS_GATE_STATE, x)
+#define QSYS_TAS_GATE_STATE_TAS_GATE_STATE_GET(x)\
+	FIELD_GET(QSYS_TAS_GATE_STATE_TAS_GATE_STATE, x)
+
 /*      REW:PORT:PORT_VLAN_CFG */
 #define REW_PORT_VLAN_CFG(g)      __REG(TARGET_REW, 0, 1, 0, g, 10, 128, 0, 0, 1, 4)
 
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c b/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c
index df2bee678559..1c88120eb291 100644
--- a/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_switchdev.c
@@ -6,8 +6,6 @@
 #include "lan966x_main.h"
 
 static struct notifier_block lan966x_netdevice_nb __read_mostly;
-static struct notifier_block lan966x_switchdev_nb __read_mostly;
-static struct notifier_block lan966x_switchdev_blocking_nb __read_mostly;
 
 static void lan966x_port_set_mcast_ip_flood(struct lan966x_port *port,
 					    u32 pgid_ip)
@@ -132,7 +130,7 @@ static int lan966x_port_pre_bridge_flags(struct lan966x_port *port,
 	return 0;
 }
 
-static void lan966x_update_fwd_mask(struct lan966x *lan966x)
+void lan966x_update_fwd_mask(struct lan966x *lan966x)
 {
 	int i;
 
@@ -140,9 +138,14 @@ static void lan966x_update_fwd_mask(struct lan966x *lan966x)
 		struct lan966x_port *port = lan966x->ports[i];
 		unsigned long mask = 0;
 
-		if (port && lan966x->bridge_fwd_mask & BIT(i))
+		if (port && lan966x->bridge_fwd_mask & BIT(i)) {
 			mask = lan966x->bridge_fwd_mask & ~BIT(i);
 
+			if (port->bond)
+				mask &= ~lan966x_lag_get_mask(lan966x,
+							      port->bond);
+		}
+
 		mask |= BIT(CPU_PORT);
 
 		lan_wr(ANA_PGID_PGID_SET(mask),
@@ -150,7 +153,7 @@ static void lan966x_update_fwd_mask(struct lan966x *lan966x)
 	}
 }
 
-static void lan966x_port_stp_state_set(struct lan966x_port *port, u8 state)
+void lan966x_port_stp_state_set(struct lan966x_port *port, u8 state)
 {
 	struct lan966x *lan966x = port->lan966x;
 	bool learn_ena = false;
@@ -171,8 +174,8 @@ static void lan966x_port_stp_state_set(struct lan966x_port *port, u8 state)
 	lan966x_update_fwd_mask(lan966x);
 }
 
-static void lan966x_port_ageing_set(struct lan966x_port *port,
-				    unsigned long ageing_clock_t)
+void lan966x_port_ageing_set(struct lan966x_port *port,
+			     unsigned long ageing_clock_t)
 {
 	unsigned long ageing_jiffies = clock_t_to_jiffies(ageing_clock_t);
 	u32 ageing_time = jiffies_to_msecs(ageing_jiffies) / 1000;
@@ -241,6 +244,7 @@ static int lan966x_port_attr_set(struct net_device *dev, const void *ctx,
 }
 
 static int lan966x_port_bridge_join(struct lan966x_port *port,
+				    struct net_device *brport_dev,
 				    struct net_device *bridge,
 				    struct netlink_ext_ack *extack)
 {
@@ -258,7 +262,7 @@ static int lan966x_port_bridge_join(struct lan966x_port *port,
 		}
 	}
 
-	err = switchdev_bridge_port_offload(dev, dev, port,
+	err = switchdev_bridge_port_offload(brport_dev, dev, port,
 					    &lan966x_switchdev_nb,
 					    &lan966x_switchdev_blocking_nb,
 					    false, extack);
@@ -295,8 +299,9 @@ static void lan966x_port_bridge_leave(struct lan966x_port *port,
 	lan966x_vlan_port_apply(port);
 }
 
-static int lan966x_port_changeupper(struct net_device *dev,
-				    struct netdev_notifier_changeupper_info *info)
+int lan966x_port_changeupper(struct net_device *dev,
+			     struct net_device *brport_dev,
+			     struct netdev_notifier_changeupper_info *info)
 {
 	struct lan966x_port *port = netdev_priv(dev);
 	struct netlink_ext_ack *extack;
@@ -306,44 +311,68 @@ static int lan966x_port_changeupper(struct net_device *dev,
 
 	if (netif_is_bridge_master(info->upper_dev)) {
 		if (info->linking)
-			err = lan966x_port_bridge_join(port, info->upper_dev,
+			err = lan966x_port_bridge_join(port, brport_dev,
+						       info->upper_dev,
 						       extack);
 		else
 			lan966x_port_bridge_leave(port, info->upper_dev);
 	}
 
+	if (netif_is_lag_master(info->upper_dev)) {
+		if (info->linking)
+			err = lan966x_lag_port_join(port, info->upper_dev,
+						    info->upper_dev,
+						    extack);
+		else
+			lan966x_lag_port_leave(port, info->upper_dev);
+	}
+
 	return err;
 }
 
-static int lan966x_port_prechangeupper(struct net_device *dev,
-				       struct netdev_notifier_changeupper_info *info)
+int lan966x_port_prechangeupper(struct net_device *dev,
+				struct net_device *brport_dev,
+				struct netdev_notifier_changeupper_info *info)
 {
 	struct lan966x_port *port = netdev_priv(dev);
+	int err = NOTIFY_DONE;
 
-	if (netif_is_bridge_master(info->upper_dev) && !info->linking)
-		switchdev_bridge_port_unoffload(port->dev, port,
-						NULL, NULL);
+	if (netif_is_bridge_master(info->upper_dev) && !info->linking) {
+		switchdev_bridge_port_unoffload(port->dev, port, NULL, NULL);
+		lan966x_fdb_flush_workqueue(port->lan966x);
+	}
 
-	return NOTIFY_DONE;
+	if (netif_is_lag_master(info->upper_dev)) {
+		err = lan966x_lag_port_prechangeupper(dev, info);
+		if (err || info->linking)
+			return err;
+
+		switchdev_bridge_port_unoffload(brport_dev, port, NULL, NULL);
+		lan966x_fdb_flush_workqueue(port->lan966x);
+	}
+
+	return err;
 }
 
-static int lan966x_foreign_bridging_check(struct net_device *bridge,
+static int lan966x_foreign_bridging_check(struct net_device *upper,
+					  bool *has_foreign,
+					  bool *seen_lan966x,
 					  struct netlink_ext_ack *extack)
 {
 	struct lan966x *lan966x = NULL;
-	bool has_foreign = false;
 	struct net_device *dev;
 	struct list_head *iter;
 
-	if (!netif_is_bridge_master(bridge))
+	if (!netif_is_bridge_master(upper) &&
+	    !netif_is_lag_master(upper))
 		return 0;
 
-	netdev_for_each_lower_dev(bridge, dev, iter) {
+	netdev_for_each_lower_dev(upper, dev, iter) {
 		if (lan966x_netdevice_check(dev)) {
 			struct lan966x_port *port = netdev_priv(dev);
 
 			if (lan966x) {
-				/* Bridge already has at least one port of a
+				/* Upper already has at least one port of a
 				 * lan966x switch inside it, check that it's
 				 * the same instance of the driver.
 				 */
@@ -354,15 +383,24 @@ static int lan966x_foreign_bridging_check(struct net_device *bridge,
 				}
 			} else {
 				/* This is the first lan966x port inside this
-				 * bridge
+				 * upper device
 				 */
 				lan966x = port->lan966x;
+				*seen_lan966x = true;
 			}
+		} else if (netif_is_lag_master(dev)) {
+			/* Allow to have bond interfaces that have only lan966x
+			 * devices
+			 */
+			if (lan966x_foreign_bridging_check(dev, has_foreign,
+							   seen_lan966x,
+							   extack))
+				return -EINVAL;
 		} else {
-			has_foreign = true;
+			*has_foreign = true;
 		}
 
-		if (lan966x && has_foreign) {
+		if (*seen_lan966x && *has_foreign) {
 			NL_SET_ERR_MSG_MOD(extack,
 					   "Bridging lan966x ports with foreign interfaces disallowed");
 			return -EINVAL;
@@ -375,7 +413,12 @@ static int lan966x_foreign_bridging_check(struct net_device *bridge,
 static int lan966x_bridge_check(struct net_device *dev,
 				struct netdev_notifier_changeupper_info *info)
 {
+	bool has_foreign = false;
+	bool seen_lan966x = false;
+
 	return lan966x_foreign_bridging_check(info->upper_dev,
+					      &has_foreign,
+					      &seen_lan966x,
 					      info->info.extack);
 }
 
@@ -386,21 +429,44 @@ static int lan966x_netdevice_port_event(struct net_device *dev,
 	int err = 0;
 
 	if (!lan966x_netdevice_check(dev)) {
-		if (event == NETDEV_CHANGEUPPER)
-			return lan966x_bridge_check(dev, ptr);
+		switch (event) {
+		case NETDEV_CHANGEUPPER:
+		case NETDEV_PRECHANGEUPPER:
+			err = lan966x_bridge_check(dev, ptr);
+			if (err)
+				return err;
+
+			if (netif_is_lag_master(dev)) {
+				if (event == NETDEV_CHANGEUPPER)
+					err = lan966x_lag_netdev_changeupper(dev,
+									     ptr);
+				else
+					err = lan966x_lag_netdev_prechangeupper(dev,
+										ptr);
+
+				return err;
+			}
+			break;
+		default:
+			return 0;
+		}
+
 		return 0;
 	}
 
 	switch (event) {
 	case NETDEV_PRECHANGEUPPER:
-		err = lan966x_port_prechangeupper(dev, ptr);
+		err = lan966x_port_prechangeupper(dev, dev, ptr);
 		break;
 	case NETDEV_CHANGEUPPER:
 		err = lan966x_bridge_check(dev, ptr);
 		if (err)
 			return err;
 
-		err = lan966x_port_changeupper(dev, ptr);
+		err = lan966x_port_changeupper(dev, dev, ptr);
+		break;
+	case NETDEV_CHANGELOWERSTATE:
+		err = lan966x_lag_port_changelowerstate(dev, ptr);
 		break;
 	}
 
@@ -418,19 +484,23 @@ static int lan966x_netdevice_event(struct notifier_block *nb,
 	return notifier_from_errno(ret);
 }
 
-/* We don't offload uppers such as LAG as bridge ports, so every device except
- * the bridge itself is foreign.
- */
 static bool lan966x_foreign_dev_check(const struct net_device *dev,
 				      const struct net_device *foreign_dev)
 {
 	struct lan966x_port *port = netdev_priv(dev);
 	struct lan966x *lan966x = port->lan966x;
+	int i;
 
 	if (netif_is_bridge_master(foreign_dev))
 		if (lan966x->bridge == foreign_dev)
 			return false;
 
+	if (netif_is_lag_master(foreign_dev))
+		for (i = 0; i < lan966x->num_phys_ports; ++i)
+			if (lan966x->ports[i] &&
+			    lan966x->ports[i]->bond == foreign_dev)
+				return false;
+
 	return true;
 }
 
@@ -571,11 +641,11 @@ static struct notifier_block lan966x_netdevice_nb __read_mostly = {
 	.notifier_call = lan966x_netdevice_event,
 };
 
-static struct notifier_block lan966x_switchdev_nb __read_mostly = {
+struct notifier_block lan966x_switchdev_nb __read_mostly = {
 	.notifier_call = lan966x_switchdev_event,
 };
 
-static struct notifier_block lan966x_switchdev_blocking_nb __read_mostly = {
+struct notifier_block lan966x_switchdev_blocking_nb __read_mostly = {
 	.notifier_call = lan966x_switchdev_blocking_event,
 };
 
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_taprio.c b/drivers/net/ethernet/microchip/lan966x/lan966x_taprio.c
new file mode 100644
index 000000000000..3f5b212066c5
--- /dev/null
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_taprio.c
@@ -0,0 +1,528 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include "lan966x_main.h"
+
+#define LAN966X_TAPRIO_TIMEOUT_MS		1000
+#define LAN966X_TAPRIO_ENTRIES_PER_PORT		2
+
+/* Minimum supported cycle time in nanoseconds */
+#define LAN966X_TAPRIO_MIN_CYCLE_TIME_NS	NSEC_PER_USEC
+
+/* Maximum supported cycle time in nanoseconds */
+#define LAN966X_TAPRIO_MAX_CYCLE_TIME_NS	(NSEC_PER_SEC - 1)
+
+/* Total number of TAS GCL entries */
+#define LAN966X_TAPRIO_NUM_GCL			256
+
+/* TAPRIO link speeds for calculation of guard band */
+enum lan966x_taprio_link_speed {
+	LAN966X_TAPRIO_SPEED_NO_GB,
+	LAN966X_TAPRIO_SPEED_10,
+	LAN966X_TAPRIO_SPEED_100,
+	LAN966X_TAPRIO_SPEED_1000,
+	LAN966X_TAPRIO_SPEED_2500,
+};
+
+/* TAPRIO list states */
+enum lan966x_taprio_state {
+	LAN966X_TAPRIO_STATE_ADMIN,
+	LAN966X_TAPRIO_STATE_ADVANCING,
+	LAN966X_TAPRIO_STATE_PENDING,
+	LAN966X_TAPRIO_STATE_OPERATING,
+	LAN966X_TAPRIO_STATE_TERMINATING,
+	LAN966X_TAPRIO_STATE_MAX,
+};
+
+/* TAPRIO GCL command */
+enum lan966x_taprio_gcl_cmd {
+	LAN966X_TAPRIO_GCL_CMD_SET_GATE_STATES = 0,
+};
+
+static u32 lan966x_taprio_list_index(struct lan966x_port *port, u8 entry)
+{
+	return port->chip_port * LAN966X_TAPRIO_ENTRIES_PER_PORT + entry;
+}
+
+static u32 lan966x_taprio_list_state_get(struct lan966x_port *port)
+{
+	struct lan966x *lan966x = port->lan966x;
+	u32 val;
+
+	val = lan_rd(lan966x, QSYS_TAS_LST);
+	return QSYS_TAS_LST_LIST_STATE_GET(val);
+}
+
+static u32 lan966x_taprio_list_index_state_get(struct lan966x_port *port,
+					       u32 list)
+{
+	struct lan966x *lan966x = port->lan966x;
+
+	lan_rmw(QSYS_TAS_CFG_CTRL_LIST_NUM_SET(list),
+		QSYS_TAS_CFG_CTRL_LIST_NUM,
+		lan966x, QSYS_TAS_CFG_CTRL);
+
+	return lan966x_taprio_list_state_get(port);
+}
+
+static void lan966x_taprio_list_state_set(struct lan966x_port *port,
+					  u32 state)
+{
+	struct lan966x *lan966x = port->lan966x;
+
+	lan_rmw(QSYS_TAS_LST_LIST_STATE_SET(state),
+		QSYS_TAS_LST_LIST_STATE,
+		lan966x, QSYS_TAS_LST);
+}
+
+static int lan966x_taprio_list_shutdown(struct lan966x_port *port,
+					u32 list)
+{
+	struct lan966x *lan966x = port->lan966x;
+	bool pending, operating;
+	unsigned long end;
+	u32 state;
+
+	end = jiffies +  msecs_to_jiffies(LAN966X_TAPRIO_TIMEOUT_MS);
+	/* It is required to try multiple times to set the state of list,
+	 * because the HW can overwrite this.
+	 */
+	do {
+		state = lan966x_taprio_list_state_get(port);
+
+		pending = false;
+		operating = false;
+
+		if (state == LAN966X_TAPRIO_STATE_ADVANCING ||
+		    state == LAN966X_TAPRIO_STATE_PENDING) {
+			lan966x_taprio_list_state_set(port,
+						      LAN966X_TAPRIO_STATE_ADMIN);
+			pending = true;
+		}
+
+		if (state == LAN966X_TAPRIO_STATE_OPERATING) {
+			lan966x_taprio_list_state_set(port,
+						      LAN966X_TAPRIO_STATE_TERMINATING);
+			operating = true;
+		}
+
+		/* If the entry was in pending and now gets in admin, then there
+		 * is nothing else to do, so just bail out
+		 */
+		state = lan966x_taprio_list_state_get(port);
+		if (pending &&
+		    state == LAN966X_TAPRIO_STATE_ADMIN)
+			return 0;
+
+		/* If the list was in operating and now is in terminating or
+		 * admin, then is OK to exit but it needs to wait until the list
+		 * will get in admin. It is not required to set the state
+		 * again.
+		 */
+		if (operating &&
+		    (state == LAN966X_TAPRIO_STATE_TERMINATING ||
+		     state == LAN966X_TAPRIO_STATE_ADMIN))
+			break;
+
+	} while (!time_after(jiffies, end));
+
+	end = jiffies + msecs_to_jiffies(LAN966X_TAPRIO_TIMEOUT_MS);
+	do {
+		state = lan966x_taprio_list_state_get(port);
+		if (state == LAN966X_TAPRIO_STATE_ADMIN)
+			break;
+
+	} while (!time_after(jiffies, end));
+
+	/* If the list was in operating mode, it could be stopped while some
+	 * queues where closed, so make sure to restore "all-queues-open"
+	 */
+	if (operating) {
+		lan_wr(QSYS_TAS_GS_CTRL_HSCH_POS_SET(port->chip_port),
+		       lan966x, QSYS_TAS_GS_CTRL);
+
+		lan_wr(QSYS_TAS_GATE_STATE_TAS_GATE_STATE_SET(0xff),
+		       lan966x, QSYS_TAS_GATE_STATE);
+	}
+
+	return 0;
+}
+
+static int lan966x_taprio_shutdown(struct lan966x_port *port)
+{
+	u32 i, list, state;
+	int err;
+
+	for (i = 0; i < LAN966X_TAPRIO_ENTRIES_PER_PORT; ++i) {
+		list = lan966x_taprio_list_index(port, i);
+		state = lan966x_taprio_list_index_state_get(port, list);
+		if (state == LAN966X_TAPRIO_STATE_ADMIN)
+			continue;
+
+		err = lan966x_taprio_list_shutdown(port, list);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+/* Find a suitable list for a new schedule. First priority is a list in state
+ * pending. Second priority is a list in state admin.
+ */
+static int lan966x_taprio_find_list(struct lan966x_port *port,
+				    struct tc_taprio_qopt_offload *qopt,
+				    int *new_list, int *obs_list)
+{
+	int state[LAN966X_TAPRIO_ENTRIES_PER_PORT];
+	int list[LAN966X_TAPRIO_ENTRIES_PER_PORT];
+	int err, oper = -1;
+	u32 i;
+
+	*new_list = -1;
+	*obs_list = -1;
+
+	/* If there is already an entry in operating mode, return this list in
+	 * obs_list, such that when the new list will get activated the
+	 * operating list will be stopped. In this way is possible to have
+	 * smooth transitions between the lists
+	 */
+	for (i = 0; i < LAN966X_TAPRIO_ENTRIES_PER_PORT; ++i) {
+		list[i] = lan966x_taprio_list_index(port, i);
+		state[i] = lan966x_taprio_list_index_state_get(port, list[i]);
+		if (state[i] == LAN966X_TAPRIO_STATE_OPERATING)
+			oper = list[i];
+	}
+
+	for (i = 0; i < LAN966X_TAPRIO_ENTRIES_PER_PORT; ++i) {
+		if (state[i] == LAN966X_TAPRIO_STATE_PENDING) {
+			err = lan966x_taprio_shutdown(port);
+			if (err)
+				return err;
+
+			*new_list = list[i];
+			*obs_list = (oper == -1) ? *new_list : oper;
+			return 0;
+		}
+	}
+
+	for (i = 0; i < LAN966X_TAPRIO_ENTRIES_PER_PORT; ++i) {
+		if (state[i] == LAN966X_TAPRIO_STATE_ADMIN) {
+			*new_list = list[i];
+			*obs_list = (oper == -1) ? *new_list : oper;
+			return 0;
+		}
+	}
+
+	return -ENOSPC;
+}
+
+static int lan966x_taprio_check(struct tc_taprio_qopt_offload *qopt)
+{
+	u64 total_time = 0;
+	u32 i;
+
+	/* This is not supported by th HW */
+	if (qopt->cycle_time_extension)
+		return -EOPNOTSUPP;
+
+	/* There is a limited number of gcl entries that can be used, they are
+	 * shared by all ports
+	 */
+	if (qopt->num_entries > LAN966X_TAPRIO_NUM_GCL)
+		return -EINVAL;
+
+	/* Don't allow cycle times bigger than 1 sec or smaller than 1 usec */
+	if (qopt->cycle_time < LAN966X_TAPRIO_MIN_CYCLE_TIME_NS ||
+	    qopt->cycle_time > LAN966X_TAPRIO_MAX_CYCLE_TIME_NS)
+		return -EINVAL;
+
+	for (i = 0; i < qopt->num_entries; ++i) {
+		struct tc_taprio_sched_entry *entry = &qopt->entries[i];
+
+		/* Don't allow intervals bigger than 1 sec or smaller than 1
+		 * usec
+		 */
+		if (entry->interval < LAN966X_TAPRIO_MIN_CYCLE_TIME_NS ||
+		    entry->interval > LAN966X_TAPRIO_MAX_CYCLE_TIME_NS)
+			return -EINVAL;
+
+		if (qopt->entries[i].command != TC_TAPRIO_CMD_SET_GATES)
+			return -EINVAL;
+
+		total_time += qopt->entries[i].interval;
+	}
+
+	/* Don't allow the total time of intervals be bigger than 1 sec */
+	if (total_time > LAN966X_TAPRIO_MAX_CYCLE_TIME_NS)
+		return -EINVAL;
+
+	/* The HW expects that the cycle time to be at least as big as sum of
+	 * each interval of gcl
+	 */
+	if (qopt->cycle_time < total_time)
+		return -EINVAL;
+
+	return 0;
+}
+
+static int lan966x_taprio_gcl_free_get(struct lan966x_port *port,
+				       unsigned long *free_list)
+{
+	struct lan966x *lan966x = port->lan966x;
+	u32 num_free, state, list;
+	u32 base, next, max_list;
+
+	/* By default everything is free */
+	bitmap_fill(free_list, LAN966X_TAPRIO_NUM_GCL);
+	num_free = LAN966X_TAPRIO_NUM_GCL;
+
+	/* Iterate over all gcl entries and find out which are free. And mark
+	 * those that are not free.
+	 */
+	max_list = lan966x->num_phys_ports * LAN966X_TAPRIO_ENTRIES_PER_PORT;
+	for (list = 0; list < max_list; ++list) {
+		state = lan966x_taprio_list_index_state_get(port, list);
+		if (state == LAN966X_TAPRIO_STATE_ADMIN)
+			continue;
+
+		base = lan_rd(lan966x, QSYS_TAS_LIST_CFG);
+		base = QSYS_TAS_LIST_CFG_LIST_BASE_ADDR_GET(base);
+		next = base;
+
+		do {
+			clear_bit(next, free_list);
+			num_free--;
+
+			lan_rmw(QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM_SET(next),
+				QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM,
+				lan966x, QSYS_TAS_CFG_CTRL);
+
+			next = lan_rd(lan966x, QSYS_TAS_GCL_CT_CFG2);
+			next = QSYS_TAS_GCL_CT_CFG2_NEXT_GCL_GET(next);
+		} while (base != next);
+	}
+
+	return num_free;
+}
+
+static void lan966x_taprio_gcl_setup_entry(struct lan966x_port *port,
+					   struct tc_taprio_sched_entry *entry,
+					   u32 next_entry)
+{
+	struct lan966x *lan966x = port->lan966x;
+
+	/* Setup a single gcl entry */
+	lan_wr(QSYS_TAS_GCL_CT_CFG_GATE_STATE_SET(entry->gate_mask) |
+	       QSYS_TAS_GCL_CT_CFG_HSCH_POS_SET(port->chip_port) |
+	       QSYS_TAS_GCL_CT_CFG_OP_TYPE_SET(LAN966X_TAPRIO_GCL_CMD_SET_GATE_STATES),
+	       lan966x, QSYS_TAS_GCL_CT_CFG);
+
+	lan_wr(QSYS_TAS_GCL_CT_CFG2_PORT_PROFILE_SET(port->chip_port) |
+	       QSYS_TAS_GCL_CT_CFG2_NEXT_GCL_SET(next_entry),
+	       lan966x, QSYS_TAS_GCL_CT_CFG2);
+
+	lan_wr(entry->interval, lan966x, QSYS_TAS_GCL_TM_CFG);
+}
+
+static int lan966x_taprio_gcl_setup(struct lan966x_port *port,
+				    struct tc_taprio_qopt_offload *qopt,
+				    int list)
+{
+	DECLARE_BITMAP(free_list, LAN966X_TAPRIO_NUM_GCL);
+	struct lan966x *lan966x = port->lan966x;
+	u32 i, base, next;
+
+	if (lan966x_taprio_gcl_free_get(port, free_list) < qopt->num_entries)
+		return -ENOSPC;
+
+	/* Select list */
+	lan_rmw(QSYS_TAS_CFG_CTRL_LIST_NUM_SET(list),
+		QSYS_TAS_CFG_CTRL_LIST_NUM,
+		lan966x, QSYS_TAS_CFG_CTRL);
+
+	/* Setup the address of the first gcl entry */
+	base = find_first_bit(free_list, LAN966X_TAPRIO_NUM_GCL);
+	lan_rmw(QSYS_TAS_LIST_CFG_LIST_BASE_ADDR_SET(base),
+		QSYS_TAS_LIST_CFG_LIST_BASE_ADDR,
+		lan966x, QSYS_TAS_LIST_CFG);
+
+	/* Iterate over entries and add them to the gcl list */
+	next = base;
+	for (i = 0; i < qopt->num_entries; ++i) {
+		lan_rmw(QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM_SET(next),
+			QSYS_TAS_CFG_CTRL_GCL_ENTRY_NUM,
+			lan966x, QSYS_TAS_CFG_CTRL);
+
+		/* If the entry is last, point back to the start of the list */
+		if (i == qopt->num_entries - 1)
+			next = base;
+		else
+			next = find_next_bit(free_list, LAN966X_TAPRIO_NUM_GCL,
+					     next + 1);
+
+		lan966x_taprio_gcl_setup_entry(port, &qopt->entries[i], next);
+	}
+
+	return 0;
+}
+
+/* Calculate new base_time based on cycle_time. The HW recommends to have the
+ * new base time at least 2 * cycle type + current time
+ */
+static void lan966x_taprio_new_base_time(struct lan966x *lan966x,
+					 const u32 cycle_time,
+					 const ktime_t org_base_time,
+					 ktime_t *new_base_time)
+{
+	ktime_t current_time, threshold_time;
+	struct timespec64 ts;
+
+	/* Get the current time and calculate the threshold_time */
+	lan966x_ptp_gettime64(&lan966x->phc[LAN966X_PHC_PORT].info, &ts);
+	current_time = timespec64_to_ktime(ts);
+	threshold_time = current_time + (2 * cycle_time);
+
+	/* If the org_base_time is in enough in future just use it */
+	if (org_base_time >= threshold_time) {
+		*new_base_time = org_base_time;
+		return;
+	}
+
+	/* If the org_base_time is smaller than current_time, calculate the new
+	 * base time as following.
+	 */
+	if (org_base_time <= current_time) {
+		u64 tmp = current_time - org_base_time;
+		u32 rem = 0;
+
+		if (tmp > cycle_time)
+			div_u64_rem(tmp, cycle_time, &rem);
+		rem = cycle_time - rem;
+		*new_base_time = threshold_time + rem;
+		return;
+	}
+
+	/* The only left place for org_base_time is between current_time and
+	 * threshold_time. In this case the new_base_time is calculated like
+	 * org_base_time + 2 * cycletime
+	 */
+	*new_base_time = org_base_time + 2 * cycle_time;
+}
+
+int lan966x_taprio_speed_set(struct lan966x_port *port, int speed)
+{
+	struct lan966x *lan966x = port->lan966x;
+	u8 taprio_speed;
+
+	switch (speed) {
+	case SPEED_10:
+		taprio_speed = LAN966X_TAPRIO_SPEED_10;
+		break;
+	case SPEED_100:
+		taprio_speed = LAN966X_TAPRIO_SPEED_100;
+		break;
+	case SPEED_1000:
+		taprio_speed = LAN966X_TAPRIO_SPEED_1000;
+		break;
+	case SPEED_2500:
+		taprio_speed = LAN966X_TAPRIO_SPEED_2500;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	lan_rmw(QSYS_TAS_PROFILE_CFG_LINK_SPEED_SET(taprio_speed),
+		QSYS_TAS_PROFILE_CFG_LINK_SPEED,
+		lan966x, QSYS_TAS_PROFILE_CFG(port->chip_port));
+
+	return 0;
+}
+
+int lan966x_taprio_add(struct lan966x_port *port,
+		       struct tc_taprio_qopt_offload *qopt)
+{
+	struct lan966x *lan966x = port->lan966x;
+	int err, new_list, obs_list;
+	struct timespec64 ts;
+	ktime_t base_time;
+
+	err = lan966x_taprio_check(qopt);
+	if (err)
+		return err;
+
+	err = lan966x_taprio_find_list(port, qopt, &new_list, &obs_list);
+	if (err)
+		return err;
+
+	err = lan966x_taprio_gcl_setup(port, qopt, new_list);
+	if (err)
+		return err;
+
+	lan966x_taprio_new_base_time(lan966x, qopt->cycle_time,
+				     qopt->base_time, &base_time);
+
+	ts = ktime_to_timespec64(base_time);
+	lan_wr(QSYS_TAS_BT_NSEC_NSEC_SET(ts.tv_nsec),
+	       lan966x, QSYS_TAS_BT_NSEC);
+
+	lan_wr(lower_32_bits(ts.tv_sec),
+	       lan966x, QSYS_TAS_BT_SEC_LSB);
+
+	lan_wr(QSYS_TAS_BT_SEC_MSB_SEC_MSB_SET(upper_32_bits(ts.tv_sec)),
+	       lan966x, QSYS_TAS_BT_SEC_MSB);
+
+	lan_wr(qopt->cycle_time, lan966x, QSYS_TAS_CT_CFG);
+
+	lan_rmw(QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX_SET(obs_list),
+		QSYS_TAS_STARTUP_CFG_OBSOLETE_IDX,
+		lan966x, QSYS_TAS_STARTUP_CFG);
+
+	/* Start list processing */
+	lan_rmw(QSYS_TAS_LST_LIST_STATE_SET(LAN966X_TAPRIO_STATE_ADVANCING),
+		QSYS_TAS_LST_LIST_STATE,
+		lan966x, QSYS_TAS_LST);
+
+	return err;
+}
+
+int lan966x_taprio_del(struct lan966x_port *port)
+{
+	return lan966x_taprio_shutdown(port);
+}
+
+void lan966x_taprio_init(struct lan966x *lan966x)
+{
+	int num_taprio_lists;
+	int p;
+
+	lan_wr(QSYS_TAS_STM_CFG_REVISIT_DLY_SET((256 * 1000) /
+						lan966x_ptp_get_period_ps()),
+	       lan966x, QSYS_TAS_STM_CFG);
+
+	num_taprio_lists = lan966x->num_phys_ports *
+			   LAN966X_TAPRIO_ENTRIES_PER_PORT;
+
+	/* For now we always use guard band on all queues */
+	lan_rmw(QSYS_TAS_CFG_CTRL_LIST_NUM_MAX_SET(num_taprio_lists) |
+		QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q_SET(1),
+		QSYS_TAS_CFG_CTRL_LIST_NUM_MAX |
+		QSYS_TAS_CFG_CTRL_ALWAYS_GB_SCH_Q,
+		lan966x, QSYS_TAS_CFG_CTRL);
+
+	for (p = 0; p < lan966x->num_phys_ports; p++)
+		lan_rmw(QSYS_TAS_PROFILE_CFG_PORT_NUM_SET(p),
+			QSYS_TAS_PROFILE_CFG_PORT_NUM,
+			lan966x, QSYS_TAS_PROFILE_CFG(p));
+}
+
+void lan966x_taprio_deinit(struct lan966x *lan966x)
+{
+	int p;
+
+	for (p = 0; p < lan966x->num_phys_ports; ++p) {
+		if (!lan966x->ports[p])
+			continue;
+
+		lan966x_taprio_del(lan966x->ports[p]);
+	}
+}
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_tbf.c b/drivers/net/ethernet/microchip/lan966x/lan966x_tbf.c
new file mode 100644
index 000000000000..4555a35d0d28
--- /dev/null
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_tbf.c
@@ -0,0 +1,85 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include "lan966x_main.h"
+
+int lan966x_tbf_add(struct lan966x_port *port,
+		    struct tc_tbf_qopt_offload *qopt)
+{
+	struct lan966x *lan966x = port->lan966x;
+	bool root = qopt->parent == TC_H_ROOT;
+	u32 queue = 0;
+	u32 cir, cbs;
+	u32 se_idx;
+
+	if (!root) {
+		queue = TC_H_MIN(qopt->parent) - 1;
+		if (queue >= NUM_PRIO_QUEUES)
+			return -EOPNOTSUPP;
+	}
+
+	if (root)
+		se_idx = SE_IDX_PORT + port->chip_port;
+	else
+		se_idx = SE_IDX_QUEUE + port->chip_port * NUM_PRIO_QUEUES + queue;
+
+	cir = div_u64(qopt->replace_params.rate.rate_bytes_ps, 1000) * 8;
+	cbs = qopt->replace_params.max_size;
+
+	/* Rate unit is 100 kbps */
+	cir = DIV_ROUND_UP(cir, 100);
+	/* Avoid using zero rate */
+	cir = cir ?: 1;
+	/* Burst unit is 4kB */
+	cbs = DIV_ROUND_UP(cbs, 4096);
+	/* Avoid using zero burst */
+	cbs = cbs ?: 1;
+
+	/* Check that actually the result can be written */
+	if (cir > GENMASK(15, 0) ||
+	    cbs > GENMASK(6, 0))
+		return -EINVAL;
+
+	lan_rmw(QSYS_SE_CFG_SE_AVB_ENA_SET(0) |
+		QSYS_SE_CFG_SE_FRM_MODE_SET(1),
+		QSYS_SE_CFG_SE_AVB_ENA |
+		QSYS_SE_CFG_SE_FRM_MODE,
+		lan966x, QSYS_SE_CFG(se_idx));
+
+	lan_wr(QSYS_CIR_CFG_CIR_RATE_SET(cir) |
+	       QSYS_CIR_CFG_CIR_BURST_SET(cbs),
+	       lan966x, QSYS_CIR_CFG(se_idx));
+
+	return 0;
+}
+
+int lan966x_tbf_del(struct lan966x_port *port,
+		    struct tc_tbf_qopt_offload *qopt)
+{
+	struct lan966x *lan966x = port->lan966x;
+	bool root = qopt->parent == TC_H_ROOT;
+	u32 queue = 0;
+	u32 se_idx;
+
+	if (!root) {
+		queue = TC_H_MIN(qopt->parent) - 1;
+		if (queue >= NUM_PRIO_QUEUES)
+			return -EOPNOTSUPP;
+	}
+
+	if (root)
+		se_idx = SE_IDX_PORT + port->chip_port;
+	else
+		se_idx = SE_IDX_QUEUE + port->chip_port * NUM_PRIO_QUEUES + queue;
+
+	lan_rmw(QSYS_SE_CFG_SE_AVB_ENA_SET(0) |
+		QSYS_SE_CFG_SE_FRM_MODE_SET(0),
+		QSYS_SE_CFG_SE_AVB_ENA |
+		QSYS_SE_CFG_SE_FRM_MODE,
+		lan966x, QSYS_SE_CFG(se_idx));
+
+	lan_wr(QSYS_CIR_CFG_CIR_RATE_SET(0) |
+	       QSYS_CIR_CFG_CIR_BURST_SET(0),
+	       lan966x, QSYS_CIR_CFG(se_idx));
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_tc.c b/drivers/net/ethernet/microchip/lan966x/lan966x_tc.c
new file mode 100644
index 000000000000..651d5493ae55
--- /dev/null
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_tc.c
@@ -0,0 +1,133 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include <net/pkt_cls.h>
+
+#include "lan966x_main.h"
+
+static LIST_HEAD(lan966x_tc_block_cb_list);
+
+static int lan966x_tc_setup_qdisc_mqprio(struct lan966x_port *port,
+					 struct tc_mqprio_qopt_offload *mqprio)
+{
+	u8 num_tc = mqprio->qopt.num_tc;
+
+	mqprio->qopt.hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+
+	return num_tc ? lan966x_mqprio_add(port, num_tc) :
+			lan966x_mqprio_del(port);
+}
+
+static int lan966x_tc_setup_qdisc_taprio(struct lan966x_port *port,
+					 struct tc_taprio_qopt_offload *taprio)
+{
+	return taprio->enable ? lan966x_taprio_add(port, taprio) :
+				lan966x_taprio_del(port);
+}
+
+static int lan966x_tc_setup_qdisc_tbf(struct lan966x_port *port,
+				      struct tc_tbf_qopt_offload *qopt)
+{
+	switch (qopt->command) {
+	case TC_TBF_REPLACE:
+		return lan966x_tbf_add(port, qopt);
+	case TC_TBF_DESTROY:
+		return lan966x_tbf_del(port, qopt);
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int lan966x_tc_setup_qdisc_cbs(struct lan966x_port *port,
+				      struct tc_cbs_qopt_offload *qopt)
+{
+	return qopt->enable ? lan966x_cbs_add(port, qopt) :
+			      lan966x_cbs_del(port, qopt);
+}
+
+static int lan966x_tc_setup_qdisc_ets(struct lan966x_port *port,
+				      struct tc_ets_qopt_offload *qopt)
+{
+	switch (qopt->command) {
+	case TC_ETS_REPLACE:
+		return lan966x_ets_add(port, qopt);
+	case TC_ETS_DESTROY:
+		return lan966x_ets_del(port, qopt);
+	default:
+		return -EOPNOTSUPP;
+	};
+
+	return -EOPNOTSUPP;
+}
+
+static int lan966x_tc_block_cb(enum tc_setup_type type, void *type_data,
+			       void *cb_priv, bool ingress)
+{
+	struct lan966x_port *port = cb_priv;
+
+	switch (type) {
+	case TC_SETUP_CLSMATCHALL:
+		return lan966x_tc_matchall(port, type_data, ingress);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+static int lan966x_tc_block_cb_ingress(enum tc_setup_type type,
+				       void *type_data, void *cb_priv)
+{
+	return lan966x_tc_block_cb(type, type_data, cb_priv, true);
+}
+
+static int lan966x_tc_block_cb_egress(enum tc_setup_type type,
+				      void *type_data, void *cb_priv)
+{
+	return lan966x_tc_block_cb(type, type_data, cb_priv, false);
+}
+
+static int lan966x_tc_setup_block(struct lan966x_port *port,
+				  struct flow_block_offload *f)
+{
+	flow_setup_cb_t *cb;
+	bool ingress;
+
+	if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) {
+		cb = lan966x_tc_block_cb_ingress;
+		port->tc.ingress_shared_block = f->block_shared;
+		ingress = true;
+	} else if (f->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) {
+		cb = lan966x_tc_block_cb_egress;
+		ingress = false;
+	} else {
+		return -EOPNOTSUPP;
+	}
+
+	return flow_block_cb_setup_simple(f, &lan966x_tc_block_cb_list,
+					  cb, port, port, ingress);
+}
+
+int lan966x_tc_setup(struct net_device *dev, enum tc_setup_type type,
+		     void *type_data)
+{
+	struct lan966x_port *port = netdev_priv(dev);
+
+	switch (type) {
+	case TC_SETUP_QDISC_MQPRIO:
+		return lan966x_tc_setup_qdisc_mqprio(port, type_data);
+	case TC_SETUP_QDISC_TAPRIO:
+		return lan966x_tc_setup_qdisc_taprio(port, type_data);
+	case TC_SETUP_QDISC_TBF:
+		return lan966x_tc_setup_qdisc_tbf(port, type_data);
+	case TC_SETUP_QDISC_CBS:
+		return lan966x_tc_setup_qdisc_cbs(port, type_data);
+	case TC_SETUP_QDISC_ETS:
+		return lan966x_tc_setup_qdisc_ets(port, type_data);
+	case TC_SETUP_BLOCK:
+		return lan966x_tc_setup_block(port, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_tc_matchall.c b/drivers/net/ethernet/microchip/lan966x/lan966x_tc_matchall.c
new file mode 100644
index 000000000000..7368433b9277
--- /dev/null
+++ b/drivers/net/ethernet/microchip/lan966x/lan966x_tc_matchall.c
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+#include "lan966x_main.h"
+
+static int lan966x_tc_matchall_add(struct lan966x_port *port,
+				   struct tc_cls_matchall_offload *f,
+				   bool ingress)
+{
+	struct flow_action_entry *act;
+
+	if (!flow_offload_has_one_action(&f->rule->action)) {
+		NL_SET_ERR_MSG_MOD(f->common.extack,
+				   "Only once action per filter is supported");
+		return -EOPNOTSUPP;
+	}
+
+	act = &f->rule->action.entries[0];
+	switch (act->id) {
+	case FLOW_ACTION_POLICE:
+		return lan966x_police_port_add(port, &f->rule->action, act,
+					       f->cookie, ingress,
+					       f->common.extack);
+	case FLOW_ACTION_MIRRED:
+		return lan966x_mirror_port_add(port, act, f->cookie,
+					       ingress, f->common.extack);
+	default:
+		NL_SET_ERR_MSG_MOD(f->common.extack,
+				   "Unsupported action");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int lan966x_tc_matchall_del(struct lan966x_port *port,
+				   struct tc_cls_matchall_offload *f,
+				   bool ingress)
+{
+	if (f->cookie == port->tc.police_id) {
+		return lan966x_police_port_del(port, f->cookie,
+					       f->common.extack);
+	} else if (f->cookie == port->tc.ingress_mirror_id ||
+		   f->cookie == port->tc.egress_mirror_id) {
+		return lan966x_mirror_port_del(port, ingress,
+					       f->common.extack);
+	} else {
+		NL_SET_ERR_MSG_MOD(f->common.extack,
+				   "Unsupported action");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int lan966x_tc_matchall_stats(struct lan966x_port *port,
+				     struct tc_cls_matchall_offload *f,
+				     bool ingress)
+{
+	if (f->cookie == port->tc.police_id) {
+		lan966x_police_port_stats(port, &f->stats);
+	} else if (f->cookie == port->tc.ingress_mirror_id ||
+		   f->cookie == port->tc.egress_mirror_id) {
+		lan966x_mirror_port_stats(port, &f->stats, ingress);
+	} else {
+		NL_SET_ERR_MSG_MOD(f->common.extack,
+				   "Unsupported action");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+int lan966x_tc_matchall(struct lan966x_port *port,
+			struct tc_cls_matchall_offload *f,
+			bool ingress)
+{
+	if (!tc_cls_can_offload_and_chain0(port->dev, &f->common)) {
+		NL_SET_ERR_MSG_MOD(f->common.extack,
+				   "Only chain zero is supported");
+		return -EOPNOTSUPP;
+	}
+
+	switch (f->command) {
+	case TC_CLSMATCHALL_REPLACE:
+		return lan966x_tc_matchall_add(port, f, ingress);
+	case TC_CLSMATCHALL_DESTROY:
+		return lan966x_tc_matchall_del(port, f, ingress);
+	case TC_CLSMATCHALL_STATS:
+		return lan966x_tc_matchall_stats(port, f, ingress);
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/microchip/sparx5/Makefile b/drivers/net/ethernet/microchip/sparx5/Makefile
index 4402c3ed1dc5..d1c6ad966747 100644
--- a/drivers/net/ethernet/microchip/sparx5/Makefile
+++ b/drivers/net/ethernet/microchip/sparx5/Makefile
@@ -8,4 +8,4 @@ obj-$(CONFIG_SPARX5_SWITCH) += sparx5-switch.o
 sparx5-switch-objs  := sparx5_main.o sparx5_packet.o \
  sparx5_netdev.o sparx5_phylink.o sparx5_port.o sparx5_mactable.o sparx5_vlan.o \
  sparx5_switchdev.o sparx5_calendar.o sparx5_ethtool.o sparx5_fdma.o \
- sparx5_ptp.o sparx5_pgid.o
+ sparx5_ptp.o sparx5_pgid.o sparx5_tc.o sparx5_qos.o
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c b/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c
index a5837dbe0c7e..4af285918ea2 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c
@@ -186,8 +186,8 @@ bool sparx5_mact_getnext(struct sparx5 *sparx5,
 	return ret == 0;
 }
 
-bool sparx5_mact_find(struct sparx5 *sparx5,
-		      const unsigned char mac[ETH_ALEN], u16 vid, u32 *pcfg2)
+int sparx5_mact_find(struct sparx5 *sparx5,
+		     const unsigned char mac[ETH_ALEN], u16 vid, u32 *pcfg2)
 {
 	int ret;
 	u32 cfg2;
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c
index 01be7bd84181..62a325e96345 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c
@@ -27,6 +27,7 @@
 #include "sparx5_main_regs.h"
 #include "sparx5_main.h"
 #include "sparx5_port.h"
+#include "sparx5_qos.h"
 
 #define QLIM_WM(fraction) \
 	((SPX5_BUFFER_MEMORY / SPX5_BUFFER_CELL_SZ - 100) * (fraction) / 100)
@@ -277,6 +278,7 @@ static int sparx5_create_port(struct sparx5 *sparx5,
 	spx5_port->custom_etype = 0x8880; /* Vitesse */
 	spx5_port->phylink_pcs.poll = true;
 	spx5_port->phylink_pcs.ops = &sparx5_phylink_pcs_ops;
+	spx5_port->is_mrouter = false;
 	sparx5->ports[config->portno] = spx5_port;
 
 	err = sparx5_port_init(sparx5, spx5_port, &config->conf);
@@ -661,6 +663,9 @@ static int sparx5_start(struct sparx5 *sparx5)
 	queue_delayed_work(sparx5->mact_queue, &sparx5->mact_work,
 			   SPX5_MACT_PULL_DELAY);
 
+	mutex_init(&sparx5->mdb_lock);
+	INIT_LIST_HEAD(&sparx5->mdb_entries);
+
 	err = sparx5_register_netdevs(sparx5);
 	if (err)
 		return err;
@@ -864,6 +869,12 @@ static int mchp_sparx5_probe(struct platform_device *pdev)
 		goto cleanup_ports;
 	}
 
+	err = sparx5_qos_init(sparx5);
+	if (err) {
+		dev_err(sparx5->dev, "Failed to initialize QoS\n");
+		goto cleanup_ports;
+	}
+
 	err = sparx5_ptp_init(sparx5);
 	if (err) {
 		dev_err(sparx5->dev, "PTP failed\n");
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
index b197129044b5..7a83222caa73 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h
@@ -190,6 +190,7 @@ struct sparx5_port {
 	u8 ptp_cmd;
 	u16 ts_id;
 	struct sk_buff_head tx_skbs;
+	bool is_mrouter;
 };
 
 enum sparx5_core_clockfreq {
@@ -215,6 +216,15 @@ struct sparx5_skb_cb {
 	unsigned long jiffies;
 };
 
+struct sparx5_mdb_entry {
+	struct list_head list;
+	DECLARE_BITMAP(port_mask, SPX5_PORTS);
+	unsigned char addr[ETH_ALEN];
+	bool cpu_copy;
+	u16 vid;
+	u16 pgid_idx;
+};
+
 #define SPARX5_PTP_TIMEOUT		msecs_to_jiffies(10)
 #define SPARX5_SKB_CB(skb) \
 	((struct sparx5_skb_cb *)((skb)->cb))
@@ -256,6 +266,10 @@ struct sparx5 {
 	struct list_head mact_entries;
 	/* mac table list (mact_entries) mutex */
 	struct mutex mact_lock;
+	/* SW MDB table */
+	struct list_head mdb_entries;
+	/* mdb list mutex */
+	struct mutex mdb_lock;
 	struct delayed_work mact_work;
 	struct workqueue_struct *mact_queue;
 	/* Board specifics */
@@ -291,7 +305,7 @@ struct frame_info {
 void sparx5_xtr_flush(struct sparx5 *sparx5, u8 grp);
 void sparx5_ifh_parse(u32 *ifh, struct frame_info *info);
 irqreturn_t sparx5_xtr_handler(int irq, void *_priv);
-int sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev);
+netdev_tx_t sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev);
 int sparx5_manual_injection_mode(struct sparx5 *sparx5);
 void sparx5_port_inj_timer_setup(struct sparx5_port *port);
 
@@ -307,8 +321,8 @@ int sparx5_mact_learn(struct sparx5 *sparx5, int port,
 		      const unsigned char mac[ETH_ALEN], u16 vid);
 bool sparx5_mact_getnext(struct sparx5 *sparx5,
 			 unsigned char mac[ETH_ALEN], u16 *vid, u32 *pcfg2);
-bool sparx5_mact_find(struct sparx5 *sparx5,
-		      const unsigned char mac[ETH_ALEN], u16 vid, u32 *pcfg2);
+int sparx5_mact_find(struct sparx5 *sparx5,
+		     const unsigned char mac[ETH_ALEN], u16 vid, u32 *pcfg2);
 int sparx5_mact_forget(struct sparx5 *sparx5,
 		       const unsigned char mac[ETH_ALEN], u16 vid);
 int sparx5_add_mact_entry(struct sparx5 *sparx5,
@@ -325,6 +339,7 @@ void sparx5_mact_init(struct sparx5 *sparx5);
 
 /* sparx5_vlan.c */
 void sparx5_pgid_update_mask(struct sparx5_port *port, int pgid, bool enable);
+void sparx5_pgid_clear(struct sparx5 *spx5, int pgid);
 void sparx5_pgid_read_mask(struct sparx5 *sparx5, int pgid, u32 portmask[3]);
 void sparx5_update_fwd(struct sparx5 *sparx5);
 void sparx5_vlan_init(struct sparx5 *sparx5);
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h
index c94de436b281..fa2eb70f487a 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main_regs.h
@@ -2993,6 +2993,147 @@ enum sparx5_target {
 #define GCB_SIO_CLOCK_SYS_CLK_PERIOD_GET(x)\
 	FIELD_GET(GCB_SIO_CLOCK_SYS_CLK_PERIOD, x)
 
+/*      HSCH:HSCH_CFG:CIR_CFG */
+#define HSCH_CIR_CFG(g)           __REG(TARGET_HSCH, 0, 1, 0, g, 5040, 32, 0, 0, 1, 4)
+
+#define HSCH_CIR_CFG_CIR_RATE                    GENMASK(22, 6)
+#define HSCH_CIR_CFG_CIR_RATE_SET(x)\
+	FIELD_PREP(HSCH_CIR_CFG_CIR_RATE, x)
+#define HSCH_CIR_CFG_CIR_RATE_GET(x)\
+	FIELD_GET(HSCH_CIR_CFG_CIR_RATE, x)
+
+#define HSCH_CIR_CFG_CIR_BURST                   GENMASK(5, 0)
+#define HSCH_CIR_CFG_CIR_BURST_SET(x)\
+	FIELD_PREP(HSCH_CIR_CFG_CIR_BURST, x)
+#define HSCH_CIR_CFG_CIR_BURST_GET(x)\
+	FIELD_GET(HSCH_CIR_CFG_CIR_BURST, x)
+
+/*      HSCH:HSCH_CFG:EIR_CFG */
+#define HSCH_EIR_CFG(g)           __REG(TARGET_HSCH, 0, 1, 0, g, 5040, 32, 4, 0, 1, 4)
+
+#define HSCH_EIR_CFG_EIR_RATE                    GENMASK(22, 6)
+#define HSCH_EIR_CFG_EIR_RATE_SET(x)\
+	FIELD_PREP(HSCH_EIR_CFG_EIR_RATE, x)
+#define HSCH_EIR_CFG_EIR_RATE_GET(x)\
+	FIELD_GET(HSCH_EIR_CFG_EIR_RATE, x)
+
+#define HSCH_EIR_CFG_EIR_BURST                   GENMASK(5, 0)
+#define HSCH_EIR_CFG_EIR_BURST_SET(x)\
+	FIELD_PREP(HSCH_EIR_CFG_EIR_BURST, x)
+#define HSCH_EIR_CFG_EIR_BURST_GET(x)\
+	FIELD_GET(HSCH_EIR_CFG_EIR_BURST, x)
+
+/*      HSCH:HSCH_CFG:SE_CFG */
+#define HSCH_SE_CFG(g)            __REG(TARGET_HSCH, 0, 1, 0, g, 5040, 32, 8, 0, 1, 4)
+
+#define HSCH_SE_CFG_SE_DWRR_CNT                  GENMASK(12, 6)
+#define HSCH_SE_CFG_SE_DWRR_CNT_SET(x)\
+	FIELD_PREP(HSCH_SE_CFG_SE_DWRR_CNT, x)
+#define HSCH_SE_CFG_SE_DWRR_CNT_GET(x)\
+	FIELD_GET(HSCH_SE_CFG_SE_DWRR_CNT, x)
+
+#define HSCH_SE_CFG_SE_AVB_ENA                   BIT(5)
+#define HSCH_SE_CFG_SE_AVB_ENA_SET(x)\
+	FIELD_PREP(HSCH_SE_CFG_SE_AVB_ENA, x)
+#define HSCH_SE_CFG_SE_AVB_ENA_GET(x)\
+	FIELD_GET(HSCH_SE_CFG_SE_AVB_ENA, x)
+
+#define HSCH_SE_CFG_SE_FRM_MODE                  GENMASK(4, 3)
+#define HSCH_SE_CFG_SE_FRM_MODE_SET(x)\
+	FIELD_PREP(HSCH_SE_CFG_SE_FRM_MODE, x)
+#define HSCH_SE_CFG_SE_FRM_MODE_GET(x)\
+	FIELD_GET(HSCH_SE_CFG_SE_FRM_MODE, x)
+
+#define HSCH_SE_CFG_SE_DWRR_FRM_MODE             GENMASK(2, 1)
+#define HSCH_SE_CFG_SE_DWRR_FRM_MODE_SET(x)\
+	FIELD_PREP(HSCH_SE_CFG_SE_DWRR_FRM_MODE, x)
+#define HSCH_SE_CFG_SE_DWRR_FRM_MODE_GET(x)\
+	FIELD_GET(HSCH_SE_CFG_SE_DWRR_FRM_MODE, x)
+
+#define HSCH_SE_CFG_SE_STOP                      BIT(0)
+#define HSCH_SE_CFG_SE_STOP_SET(x)\
+	FIELD_PREP(HSCH_SE_CFG_SE_STOP, x)
+#define HSCH_SE_CFG_SE_STOP_GET(x)\
+	FIELD_GET(HSCH_SE_CFG_SE_STOP, x)
+
+/*      HSCH:HSCH_CFG:SE_CONNECT */
+#define HSCH_SE_CONNECT(g)        __REG(TARGET_HSCH, 0, 1, 0, g, 5040, 32, 12, 0, 1, 4)
+
+#define HSCH_SE_CONNECT_SE_LEAK_LINK             GENMASK(15, 0)
+#define HSCH_SE_CONNECT_SE_LEAK_LINK_SET(x)\
+	FIELD_PREP(HSCH_SE_CONNECT_SE_LEAK_LINK, x)
+#define HSCH_SE_CONNECT_SE_LEAK_LINK_GET(x)\
+	FIELD_GET(HSCH_SE_CONNECT_SE_LEAK_LINK, x)
+
+/*      HSCH:HSCH_CFG:SE_DLB_SENSE */
+#define HSCH_SE_DLB_SENSE(g)      __REG(TARGET_HSCH, 0, 1, 0, g, 5040, 32, 16, 0, 1, 4)
+
+#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO            GENMASK(12, 10)
+#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO_SET(x)\
+	FIELD_PREP(HSCH_SE_DLB_SENSE_SE_DLB_PRIO, x)
+#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO_GET(x)\
+	FIELD_GET(HSCH_SE_DLB_SENSE_SE_DLB_PRIO, x)
+
+#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT           GENMASK(9, 3)
+#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT_SET(x)\
+	FIELD_PREP(HSCH_SE_DLB_SENSE_SE_DLB_DPORT, x)
+#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT_GET(x)\
+	FIELD_GET(HSCH_SE_DLB_SENSE_SE_DLB_DPORT, x)
+
+#define HSCH_SE_DLB_SENSE_SE_DLB_SE_ENA          BIT(2)
+#define HSCH_SE_DLB_SENSE_SE_DLB_SE_ENA_SET(x)\
+	FIELD_PREP(HSCH_SE_DLB_SENSE_SE_DLB_SE_ENA, x)
+#define HSCH_SE_DLB_SENSE_SE_DLB_SE_ENA_GET(x)\
+	FIELD_GET(HSCH_SE_DLB_SENSE_SE_DLB_SE_ENA, x)
+
+#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO_ENA        BIT(1)
+#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO_ENA_SET(x)\
+	FIELD_PREP(HSCH_SE_DLB_SENSE_SE_DLB_PRIO_ENA, x)
+#define HSCH_SE_DLB_SENSE_SE_DLB_PRIO_ENA_GET(x)\
+	FIELD_GET(HSCH_SE_DLB_SENSE_SE_DLB_PRIO_ENA, x)
+
+#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT_ENA       BIT(0)
+#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT_ENA_SET(x)\
+	FIELD_PREP(HSCH_SE_DLB_SENSE_SE_DLB_DPORT_ENA, x)
+#define HSCH_SE_DLB_SENSE_SE_DLB_DPORT_ENA_GET(x)\
+	FIELD_GET(HSCH_SE_DLB_SENSE_SE_DLB_DPORT_ENA, x)
+
+/*      HSCH:HSCH_DWRR:DWRR_ENTRY */
+#define HSCH_DWRR_ENTRY(g)        __REG(TARGET_HSCH, 0, 1, 162816, g, 72, 4, 0, 0, 1, 4)
+
+#define HSCH_DWRR_ENTRY_DWRR_COST                GENMASK(24, 20)
+#define HSCH_DWRR_ENTRY_DWRR_COST_SET(x)\
+	FIELD_PREP(HSCH_DWRR_ENTRY_DWRR_COST, x)
+#define HSCH_DWRR_ENTRY_DWRR_COST_GET(x)\
+	FIELD_GET(HSCH_DWRR_ENTRY_DWRR_COST, x)
+
+#define HSCH_DWRR_ENTRY_DWRR_BALANCE             GENMASK(19, 0)
+#define HSCH_DWRR_ENTRY_DWRR_BALANCE_SET(x)\
+	FIELD_PREP(HSCH_DWRR_ENTRY_DWRR_BALANCE, x)
+#define HSCH_DWRR_ENTRY_DWRR_BALANCE_GET(x)\
+	FIELD_GET(HSCH_DWRR_ENTRY_DWRR_BALANCE, x)
+
+/*      HSCH:HSCH_MISC:HSCH_CFG_CFG */
+#define HSCH_HSCH_CFG_CFG         __REG(TARGET_HSCH, 0, 1, 163104, 0, 1, 648, 284, 0, 1, 4)
+
+#define HSCH_HSCH_CFG_CFG_CFG_SE_IDX             GENMASK(26, 14)
+#define HSCH_HSCH_CFG_CFG_CFG_SE_IDX_SET(x)\
+	FIELD_PREP(HSCH_HSCH_CFG_CFG_CFG_SE_IDX, x)
+#define HSCH_HSCH_CFG_CFG_CFG_SE_IDX_GET(x)\
+	FIELD_GET(HSCH_HSCH_CFG_CFG_CFG_SE_IDX, x)
+
+#define HSCH_HSCH_CFG_CFG_HSCH_LAYER             GENMASK(13, 12)
+#define HSCH_HSCH_CFG_CFG_HSCH_LAYER_SET(x)\
+	FIELD_PREP(HSCH_HSCH_CFG_CFG_HSCH_LAYER, x)
+#define HSCH_HSCH_CFG_CFG_HSCH_LAYER_GET(x)\
+	FIELD_GET(HSCH_HSCH_CFG_CFG_HSCH_LAYER, x)
+
+#define HSCH_HSCH_CFG_CFG_CSR_GRANT              GENMASK(11, 0)
+#define HSCH_HSCH_CFG_CFG_CSR_GRANT_SET(x)\
+	FIELD_PREP(HSCH_HSCH_CFG_CFG_CSR_GRANT, x)
+#define HSCH_HSCH_CFG_CFG_CSR_GRANT_GET(x)\
+	FIELD_GET(HSCH_HSCH_CFG_CFG_CSR_GRANT, x)
+
 /*      HSCH:HSCH_MISC:SYS_CLK_PER */
 #define HSCH_SYS_CLK_PER          __REG(TARGET_HSCH, 0, 1, 163104, 0, 1, 648, 640, 0, 1, 4)
 
@@ -3002,6 +3143,30 @@ enum sparx5_target {
 #define HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS_GET(x)\
 	FIELD_GET(HSCH_SYS_CLK_PER_SYS_CLK_PER_100PS, x)
 
+/*      HSCH:HSCH_LEAK_LISTS:HSCH_TIMER_CFG */
+#define HSCH_HSCH_TIMER_CFG(g, r) __REG(TARGET_HSCH, 0, 1, 161664, g, 4, 32, 0, r, 4, 4)
+
+#define HSCH_HSCH_TIMER_CFG_LEAK_TIME            GENMASK(17, 0)
+#define HSCH_HSCH_TIMER_CFG_LEAK_TIME_SET(x)\
+	FIELD_PREP(HSCH_HSCH_TIMER_CFG_LEAK_TIME, x)
+#define HSCH_HSCH_TIMER_CFG_LEAK_TIME_GET(x)\
+	FIELD_GET(HSCH_HSCH_TIMER_CFG_LEAK_TIME, x)
+
+/*      HSCH:HSCH_LEAK_LISTS:HSCH_LEAK_CFG */
+#define HSCH_HSCH_LEAK_CFG(g, r)  __REG(TARGET_HSCH, 0, 1, 161664, g, 4, 32, 16, r, 4, 4)
+
+#define HSCH_HSCH_LEAK_CFG_LEAK_FIRST            GENMASK(16, 1)
+#define HSCH_HSCH_LEAK_CFG_LEAK_FIRST_SET(x)\
+	FIELD_PREP(HSCH_HSCH_LEAK_CFG_LEAK_FIRST, x)
+#define HSCH_HSCH_LEAK_CFG_LEAK_FIRST_GET(x)\
+	FIELD_GET(HSCH_HSCH_LEAK_CFG_LEAK_FIRST, x)
+
+#define HSCH_HSCH_LEAK_CFG_LEAK_ERR              BIT(0)
+#define HSCH_HSCH_LEAK_CFG_LEAK_ERR_SET(x)\
+	FIELD_PREP(HSCH_HSCH_LEAK_CFG_LEAK_ERR, x)
+#define HSCH_HSCH_LEAK_CFG_LEAK_ERR_GET(x)\
+	FIELD_GET(HSCH_HSCH_LEAK_CFG_LEAK_ERR, x)
+
 /*      HSCH:SYSTEM:FLUSH_CTRL */
 #define HSCH_FLUSH_CTRL           __REG(TARGET_HSCH, 0, 1, 184000, 0, 1, 312, 4, 0, 1, 4)
 
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c
index af4d3e1f1a6d..19516ccad533 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_netdev.c
@@ -7,6 +7,7 @@
 #include "sparx5_main_regs.h"
 #include "sparx5_main.h"
 #include "sparx5_port.h"
+#include "sparx5_tc.h"
 
 /* The IFH bit position of the first VSTAX bit. This is because the
  * VSTAX bit positions in Data sheet is starting from zero.
@@ -228,6 +229,7 @@ static const struct net_device_ops sparx5_port_netdev_ops = {
 	.ndo_get_stats64        = sparx5_get_stats64,
 	.ndo_get_port_parent_id = sparx5_get_port_parent_id,
 	.ndo_eth_ioctl          = sparx5_port_ioctl,
+	.ndo_setup_tc           = sparx5_port_setup_tc,
 };
 
 bool sparx5_netdevice_check(const struct net_device *dev)
@@ -240,10 +242,14 @@ struct net_device *sparx5_create_netdev(struct sparx5 *sparx5, u32 portno)
 	struct sparx5_port *spx5_port;
 	struct net_device *ndev;
 
-	ndev = devm_alloc_etherdev(sparx5->dev, sizeof(struct sparx5_port));
+	ndev = devm_alloc_etherdev_mqs(sparx5->dev, sizeof(struct sparx5_port),
+				       SPX5_PRIOS, 1);
 	if (!ndev)
 		return ERR_PTR(-ENOMEM);
 
+	ndev->hw_features |= NETIF_F_HW_TC;
+	ndev->features |= NETIF_F_HW_TC;
+
 	SET_NETDEV_DEV(ndev, sparx5->dev);
 	spx5_port = netdev_priv(ndev);
 	spx5_port->ndev = ndev;
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c b/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c
index 21844beba72d..83c16ca5b30f 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_packet.c
@@ -222,13 +222,13 @@ static int sparx5_inject(struct sparx5 *sparx5,
 	return NETDEV_TX_OK;
 }
 
-int sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev)
+netdev_tx_t sparx5_port_xmit_impl(struct sk_buff *skb, struct net_device *dev)
 {
 	struct net_device_stats *stats = &dev->stats;
 	struct sparx5_port *port = netdev_priv(dev);
 	struct sparx5 *sparx5 = port->sparx5;
 	u32 ifh[IFH_LEN];
-	int ret;
+	netdev_tx_t ret;
 
 	memset(ifh, 0, IFH_LEN * 4);
 	sparx5_set_port_ifh(ifh, port->portno);
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.c b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.c
new file mode 100644
index 000000000000..1e79d0ef0cb8
--- /dev/null
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.c
@@ -0,0 +1,513 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Microchip Sparx5 Switch driver
+ *
+ * Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries.
+ */
+
+#include <net/pkt_cls.h>
+
+#include "sparx5_main.h"
+#include "sparx5_qos.h"
+
+/* Max rates for leak groups */
+static const u32 spx5_hsch_max_group_rate[SPX5_HSCH_LEAK_GRP_CNT] = {
+	1048568, /*  1.049 Gbps */
+	2621420, /*  2.621 Gbps */
+	10485680, /* 10.486 Gbps */
+	26214200 /* 26.214 Gbps */
+};
+
+static struct sparx5_layer layers[SPX5_HSCH_LAYER_CNT];
+
+static u32 sparx5_lg_get_leak_time(struct sparx5 *sparx5, u32 layer, u32 group)
+{
+	u32 value;
+
+	value = spx5_rd(sparx5, HSCH_HSCH_TIMER_CFG(layer, group));
+	return HSCH_HSCH_TIMER_CFG_LEAK_TIME_GET(value);
+}
+
+static void sparx5_lg_set_leak_time(struct sparx5 *sparx5, u32 layer, u32 group,
+				    u32 leak_time)
+{
+	spx5_wr(HSCH_HSCH_TIMER_CFG_LEAK_TIME_SET(leak_time), sparx5,
+		HSCH_HSCH_TIMER_CFG(layer, group));
+}
+
+static u32 sparx5_lg_get_first(struct sparx5 *sparx5, u32 layer, u32 group)
+{
+	u32 value;
+
+	value = spx5_rd(sparx5, HSCH_HSCH_LEAK_CFG(layer, group));
+	return HSCH_HSCH_LEAK_CFG_LEAK_FIRST_GET(value);
+}
+
+static u32 sparx5_lg_get_next(struct sparx5 *sparx5, u32 layer, u32 group,
+			      u32 idx)
+
+{
+	u32 value;
+
+	value = spx5_rd(sparx5, HSCH_SE_CONNECT(idx));
+	return HSCH_SE_CONNECT_SE_LEAK_LINK_GET(value);
+}
+
+static u32 sparx5_lg_get_last(struct sparx5 *sparx5, u32 layer, u32 group)
+{
+	u32 itr, next;
+
+	itr = sparx5_lg_get_first(sparx5, layer, group);
+
+	for (;;) {
+		next = sparx5_lg_get_next(sparx5, layer, group, itr);
+		if (itr == next)
+			return itr;
+
+		itr = next;
+	}
+}
+
+static bool sparx5_lg_is_last(struct sparx5 *sparx5, u32 layer, u32 group,
+			      u32 idx)
+{
+	return idx == sparx5_lg_get_next(sparx5, layer, group, idx);
+}
+
+static bool sparx5_lg_is_first(struct sparx5 *sparx5, u32 layer, u32 group,
+			       u32 idx)
+{
+	return idx == sparx5_lg_get_first(sparx5, layer, group);
+}
+
+static bool sparx5_lg_is_empty(struct sparx5 *sparx5, u32 layer, u32 group)
+{
+	return sparx5_lg_get_leak_time(sparx5, layer, group) == 0;
+}
+
+static bool sparx5_lg_is_singular(struct sparx5 *sparx5, u32 layer, u32 group)
+{
+	if (sparx5_lg_is_empty(sparx5, layer, group))
+		return false;
+
+	return sparx5_lg_get_first(sparx5, layer, group) ==
+	       sparx5_lg_get_last(sparx5, layer, group);
+}
+
+static void sparx5_lg_enable(struct sparx5 *sparx5, u32 layer, u32 group,
+			     u32 leak_time)
+{
+	sparx5_lg_set_leak_time(sparx5, layer, group, leak_time);
+}
+
+static void sparx5_lg_disable(struct sparx5 *sparx5, u32 layer, u32 group)
+{
+	sparx5_lg_set_leak_time(sparx5, layer, group, 0);
+}
+
+static int sparx5_lg_get_group_by_index(struct sparx5 *sparx5, u32 layer,
+					u32 idx, u32 *group)
+{
+	u32 itr, next;
+	int i;
+
+	for (i = 0; i < SPX5_HSCH_LEAK_GRP_CNT; i++) {
+		if (sparx5_lg_is_empty(sparx5, layer, i))
+			continue;
+
+		itr = sparx5_lg_get_first(sparx5, layer, i);
+
+		for (;;) {
+			next = sparx5_lg_get_next(sparx5, layer, i, itr);
+
+			if (itr == idx) {
+				*group = i;
+				return 0; /* Found it */
+			}
+			if (itr == next)
+				break; /* Was not found */
+
+			itr = next;
+		}
+	}
+
+	return -1;
+}
+
+static int sparx5_lg_get_group_by_rate(u32 layer, u32 rate, u32 *group)
+{
+	struct sparx5_layer *l = &layers[layer];
+	struct sparx5_lg *lg;
+	u32 i;
+
+	for (i = 0; i < SPX5_HSCH_LEAK_GRP_CNT; i++) {
+		lg = &l->leak_groups[i];
+		if (rate <= lg->max_rate) {
+			*group = i;
+			return 0;
+		}
+	}
+
+	return -1;
+}
+
+static int sparx5_lg_get_adjacent(struct sparx5 *sparx5, u32 layer, u32 group,
+				  u32 idx, u32 *prev, u32 *next, u32 *first)
+{
+	u32 itr;
+
+	*first = sparx5_lg_get_first(sparx5, layer, group);
+	*prev = *first;
+	*next = *first;
+	itr = *first;
+
+	for (;;) {
+		*next = sparx5_lg_get_next(sparx5, layer, group, itr);
+
+		if (itr == idx)
+			return 0; /* Found it */
+
+		if (itr == *next)
+			return -1; /* Was not found */
+
+		*prev = itr;
+		itr = *next;
+	}
+
+	return -1;
+}
+
+static int sparx5_lg_conf_set(struct sparx5 *sparx5, u32 layer, u32 group,
+			      u32 se_first, u32 idx, u32 idx_next, bool empty)
+{
+	u32 leak_time = layers[layer].leak_groups[group].leak_time;
+
+	/* Stop leaking */
+	sparx5_lg_disable(sparx5, layer, group);
+
+	if (empty)
+		return 0;
+
+	/* Select layer */
+	spx5_rmw(HSCH_HSCH_CFG_CFG_HSCH_LAYER_SET(layer),
+		 HSCH_HSCH_CFG_CFG_HSCH_LAYER, sparx5, HSCH_HSCH_CFG_CFG);
+
+	/* Link elements */
+	spx5_wr(HSCH_SE_CONNECT_SE_LEAK_LINK_SET(idx_next), sparx5,
+		HSCH_SE_CONNECT(idx));
+
+	/* Set the first element. */
+	spx5_rmw(HSCH_HSCH_LEAK_CFG_LEAK_FIRST_SET(se_first),
+		 HSCH_HSCH_LEAK_CFG_LEAK_FIRST, sparx5,
+		 HSCH_HSCH_LEAK_CFG(layer, group));
+
+	/* Start leaking */
+	sparx5_lg_enable(sparx5, layer, group, leak_time);
+
+	return 0;
+}
+
+static int sparx5_lg_del(struct sparx5 *sparx5, u32 layer, u32 group, u32 idx)
+{
+	u32 first, next, prev;
+	bool empty = false;
+
+	/* idx *must* be present in the leak group */
+	WARN_ON(sparx5_lg_get_adjacent(sparx5, layer, group, idx, &prev, &next,
+				       &first) < 0);
+
+	if (sparx5_lg_is_singular(sparx5, layer, group)) {
+		empty = true;
+	} else if (sparx5_lg_is_last(sparx5, layer, group, idx)) {
+		/* idx is removed, prev is now last */
+		idx = prev;
+		next = prev;
+	} else if (sparx5_lg_is_first(sparx5, layer, group, idx)) {
+		/* idx is removed and points to itself, first is next */
+		first = next;
+		next = idx;
+	} else {
+		/* Next is not touched */
+		idx = prev;
+	}
+
+	return sparx5_lg_conf_set(sparx5, layer, group, first, idx, next,
+				  empty);
+}
+
+static int sparx5_lg_add(struct sparx5 *sparx5, u32 layer, u32 new_group,
+			 u32 idx)
+{
+	u32 first, next, old_group;
+
+	pr_debug("ADD: layer: %d, new_group: %d, idx: %d", layer, new_group,
+		 idx);
+
+	/* Is this SE already shaping ? */
+	if (sparx5_lg_get_group_by_index(sparx5, layer, idx, &old_group) >= 0) {
+		if (old_group != new_group) {
+			/* Delete from old group */
+			sparx5_lg_del(sparx5, layer, old_group, idx);
+		} else {
+			/* Nothing to do here */
+			return 0;
+		}
+	}
+
+	/* We always add to head of the list */
+	first = idx;
+
+	if (sparx5_lg_is_empty(sparx5, layer, new_group))
+		next = idx;
+	else
+		next = sparx5_lg_get_first(sparx5, layer, new_group);
+
+	return sparx5_lg_conf_set(sparx5, layer, new_group, first, idx, next,
+				  false);
+}
+
+static int sparx5_shaper_conf_set(struct sparx5_port *port,
+				  const struct sparx5_shaper *sh, u32 layer,
+				  u32 idx, u32 group)
+{
+	int (*sparx5_lg_action)(struct sparx5 *, u32, u32, u32);
+	struct sparx5 *sparx5 = port->sparx5;
+
+	if (!sh->rate && !sh->burst)
+		sparx5_lg_action = &sparx5_lg_del;
+	else
+		sparx5_lg_action = &sparx5_lg_add;
+
+	/* Select layer */
+	spx5_rmw(HSCH_HSCH_CFG_CFG_HSCH_LAYER_SET(layer),
+		 HSCH_HSCH_CFG_CFG_HSCH_LAYER, sparx5, HSCH_HSCH_CFG_CFG);
+
+	/* Set frame mode */
+	spx5_rmw(HSCH_SE_CFG_SE_FRM_MODE_SET(sh->mode), HSCH_SE_CFG_SE_FRM_MODE,
+		 sparx5, HSCH_SE_CFG(idx));
+
+	/* Set committed rate and burst */
+	spx5_wr(HSCH_CIR_CFG_CIR_RATE_SET(sh->rate) |
+			HSCH_CIR_CFG_CIR_BURST_SET(sh->burst),
+		sparx5, HSCH_CIR_CFG(idx));
+
+	/* This has to be done after the shaper configuration has been set */
+	sparx5_lg_action(sparx5, layer, group, idx);
+
+	return 0;
+}
+
+static u32 sparx5_weight_to_hw_cost(u32 weight_min, u32 weight)
+{
+	return ((((SPX5_DWRR_COST_MAX << 4) * weight_min / weight) + 8) >> 4) -
+	       1;
+}
+
+static int sparx5_dwrr_conf_set(struct sparx5_port *port,
+				struct sparx5_dwrr *dwrr)
+{
+	int i;
+
+	spx5_rmw(HSCH_HSCH_CFG_CFG_HSCH_LAYER_SET(2) |
+		 HSCH_HSCH_CFG_CFG_CFG_SE_IDX_SET(port->portno),
+		 HSCH_HSCH_CFG_CFG_HSCH_LAYER | HSCH_HSCH_CFG_CFG_CFG_SE_IDX,
+		 port->sparx5, HSCH_HSCH_CFG_CFG);
+
+	/* Number of *lower* indexes that are arbitrated dwrr */
+	spx5_rmw(HSCH_SE_CFG_SE_DWRR_CNT_SET(dwrr->count),
+		 HSCH_SE_CFG_SE_DWRR_CNT, port->sparx5,
+		 HSCH_SE_CFG(port->portno));
+
+	for (i = 0; i < dwrr->count; i++) {
+		spx5_rmw(HSCH_DWRR_ENTRY_DWRR_COST_SET(dwrr->cost[i]),
+			 HSCH_DWRR_ENTRY_DWRR_COST, port->sparx5,
+			 HSCH_DWRR_ENTRY(i));
+	}
+
+	return 0;
+}
+
+static int sparx5_leak_groups_init(struct sparx5 *sparx5)
+{
+	struct sparx5_layer *layer;
+	u32 sys_clk_per_100ps;
+	struct sparx5_lg *lg;
+	u32 leak_time_us;
+	int i, ii;
+
+	sys_clk_per_100ps = spx5_rd(sparx5, HSCH_SYS_CLK_PER);
+
+	for (i = 0; i < SPX5_HSCH_LAYER_CNT; i++) {
+		layer = &layers[i];
+		for (ii = 0; ii < SPX5_HSCH_LEAK_GRP_CNT; ii++) {
+			lg = &layer->leak_groups[ii];
+			lg->max_rate = spx5_hsch_max_group_rate[ii];
+
+			/* Calculate the leak time in us, to serve a maximum
+			 * rate of 'max_rate' for this group
+			 */
+			leak_time_us = (SPX5_SE_RATE_MAX * 1000) / lg->max_rate;
+
+			/* Hardware wants leak time in ns */
+			lg->leak_time = 1000 * leak_time_us;
+
+			/* Calculate resolution */
+			lg->resolution = 1000 / leak_time_us;
+
+			/* Maximum number of shapers that can be served by
+			 * this leak group
+			 */
+			lg->max_ses = (1000 * leak_time_us) / sys_clk_per_100ps;
+
+			/* Example:
+			 * Wanted bandwidth is 100Mbit:
+			 *
+			 * 100 mbps can be served by leak group zero.
+			 *
+			 * leak_time is 125000 ns.
+			 * resolution is: 8
+			 *
+			 * cir          = 100000 / 8 = 12500
+			 * leaks_pr_sec = 125000 / 10^9 = 8000
+			 * bw           = 12500 * 8000 = 10^8 (100 Mbit)
+			 */
+
+			/* Disable by default - this also indicates an empty
+			 * leak group
+			 */
+			sparx5_lg_disable(sparx5, i, ii);
+		}
+	}
+
+	return 0;
+}
+
+int sparx5_qos_init(struct sparx5 *sparx5)
+{
+	int ret;
+
+	ret = sparx5_leak_groups_init(sparx5);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+int sparx5_tc_mqprio_add(struct net_device *ndev, u8 num_tc)
+{
+	int i;
+
+	if (num_tc != SPX5_PRIOS) {
+		netdev_err(ndev, "Only %d traffic classes supported\n",
+			   SPX5_PRIOS);
+		return -EINVAL;
+	}
+
+	netdev_set_num_tc(ndev, num_tc);
+
+	for (i = 0; i < num_tc; i++)
+		netdev_set_tc_queue(ndev, i, 1, i);
+
+	netdev_dbg(ndev, "dev->num_tc %u dev->real_num_tx_queues %u\n",
+		   ndev->num_tc, ndev->real_num_tx_queues);
+
+	return 0;
+}
+
+int sparx5_tc_mqprio_del(struct net_device *ndev)
+{
+	netdev_reset_tc(ndev);
+
+	netdev_dbg(ndev, "dev->num_tc %u dev->real_num_tx_queues %u\n",
+		   ndev->num_tc, ndev->real_num_tx_queues);
+
+	return 0;
+}
+
+int sparx5_tc_tbf_add(struct sparx5_port *port,
+		      struct tc_tbf_qopt_offload_replace_params *params,
+		      u32 layer, u32 idx)
+{
+	struct sparx5_shaper sh = {
+		.mode = SPX5_SE_MODE_DATARATE,
+		.rate = div_u64(params->rate.rate_bytes_ps, 1000) * 8,
+		.burst = params->max_size,
+	};
+	struct sparx5_lg *lg;
+	u32 group;
+
+	/* Find suitable group for this se */
+	if (sparx5_lg_get_group_by_rate(layer, sh.rate, &group) < 0) {
+		pr_debug("Could not find leak group for se with rate: %d",
+			 sh.rate);
+		return -EINVAL;
+	}
+
+	lg = &layers[layer].leak_groups[group];
+
+	pr_debug("Found matching group (speed: %d)\n", lg->max_rate);
+
+	if (sh.rate < SPX5_SE_RATE_MIN || sh.burst < SPX5_SE_BURST_MIN)
+		return -EINVAL;
+
+	/* Calculate committed rate and burst */
+	sh.rate = DIV_ROUND_UP(sh.rate, lg->resolution);
+	sh.burst = DIV_ROUND_UP(sh.burst, SPX5_SE_BURST_UNIT);
+
+	if (sh.rate > SPX5_SE_RATE_MAX || sh.burst > SPX5_SE_BURST_MAX)
+		return -EINVAL;
+
+	return sparx5_shaper_conf_set(port, &sh, layer, idx, group);
+}
+
+int sparx5_tc_tbf_del(struct sparx5_port *port, u32 layer, u32 idx)
+{
+	struct sparx5_shaper sh = {0};
+	u32 group;
+
+	sparx5_lg_get_group_by_index(port->sparx5, layer, idx, &group);
+
+	return sparx5_shaper_conf_set(port, &sh, layer, idx, group);
+}
+
+int sparx5_tc_ets_add(struct sparx5_port *port,
+		      struct tc_ets_qopt_offload_replace_params *params)
+{
+	struct sparx5_dwrr dwrr = {0};
+	/* Minimum weight for each iteration */
+	unsigned int w_min = 100;
+	int i;
+
+	/* Find minimum weight for all dwrr bands */
+	for (i = 0; i < SPX5_PRIOS; i++) {
+		if (params->quanta[i] == 0)
+			continue;
+		w_min = min(w_min, params->weights[i]);
+	}
+
+	for (i = 0; i < SPX5_PRIOS; i++) {
+		/* Strict band; skip */
+		if (params->quanta[i] == 0)
+			continue;
+
+		dwrr.count++;
+
+		/* On the sparx5, bands with higher indexes are preferred and
+		 * arbitrated strict. Strict bands are put in the lower indexes,
+		 * by tc, so we reverse the bands here.
+		 *
+		 * Also convert the weight to something the hardware
+		 * understands.
+		 */
+		dwrr.cost[SPX5_PRIOS - i - 1] =
+			sparx5_weight_to_hw_cost(w_min, params->weights[i]);
+	}
+
+	return sparx5_dwrr_conf_set(port, &dwrr);
+}
+
+int sparx5_tc_ets_del(struct sparx5_port *port)
+{
+	struct sparx5_dwrr dwrr = {0};
+
+	return sparx5_dwrr_conf_set(port, &dwrr);
+}
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h
new file mode 100644
index 000000000000..ced35033a6c5
--- /dev/null
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_qos.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/* Microchip Sparx5 Switch driver
+ *
+ * Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries.
+ */
+
+#ifndef __SPARX5_QOS_H__
+#define __SPARX5_QOS_H__
+
+#include <linux/netdevice.h>
+
+/* Number of Layers */
+#define SPX5_HSCH_LAYER_CNT 3
+
+/* Scheduling elements per layer */
+#define SPX5_HSCH_L0_SE_CNT 5040
+#define SPX5_HSCH_L1_SE_CNT 64
+#define SPX5_HSCH_L2_SE_CNT 64
+
+/* Calculate Layer 0 Scheduler Element when using normal hierarchy */
+#define SPX5_HSCH_L0_GET_IDX(port, queue) ((64 * (port)) + (8 * (queue)))
+
+/* Number of leak groups */
+#define SPX5_HSCH_LEAK_GRP_CNT 4
+
+/* Scheduler modes */
+#define SPX5_SE_MODE_LINERATE 0
+#define SPX5_SE_MODE_DATARATE 1
+
+/* Rate and burst */
+#define SPX5_SE_RATE_MAX 262143
+#define SPX5_SE_BURST_MAX 127
+#define SPX5_SE_RATE_MIN 1
+#define SPX5_SE_BURST_MIN 1
+#define SPX5_SE_BURST_UNIT 4096
+
+/* Dwrr */
+#define SPX5_DWRR_COST_MAX 63
+
+struct sparx5_shaper {
+	u32 mode;
+	u32 rate;
+	u32 burst;
+};
+
+struct sparx5_lg {
+	u32 max_rate;
+	u32 resolution;
+	u32 leak_time;
+	u32 max_ses;
+};
+
+struct sparx5_layer {
+	struct sparx5_lg leak_groups[SPX5_HSCH_LEAK_GRP_CNT];
+};
+
+struct sparx5_dwrr {
+	u32 count; /* Number of inputs running dwrr */
+	u8 cost[SPX5_PRIOS];
+};
+
+int sparx5_qos_init(struct sparx5 *sparx5);
+
+/* Multi-Queue Priority */
+int sparx5_tc_mqprio_add(struct net_device *ndev, u8 num_tc);
+int sparx5_tc_mqprio_del(struct net_device *ndev);
+
+/* Token Bucket Filter */
+struct tc_tbf_qopt_offload_replace_params;
+int sparx5_tc_tbf_add(struct sparx5_port *port,
+		      struct tc_tbf_qopt_offload_replace_params *params,
+		      u32 layer, u32 idx);
+int sparx5_tc_tbf_del(struct sparx5_port *port, u32 layer, u32 idx);
+
+/* Enhanced Transmission Selection */
+struct tc_ets_qopt_offload_replace_params;
+int sparx5_tc_ets_add(struct sparx5_port *port,
+		      struct tc_ets_qopt_offload_replace_params *params);
+
+int sparx5_tc_ets_del(struct sparx5_port *port);
+
+#endif	/* __SPARX5_QOS_H__ */
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c b/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c
index ec07f7d0528c..4af85d108a06 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_switchdev.c
@@ -29,14 +29,23 @@ static int sparx5_port_attr_pre_bridge_flags(struct sparx5_port *port,
 	return 0;
 }
 
+static void sparx5_port_update_mcast_ip_flood(struct sparx5_port *port, bool flood_flag)
+{
+	bool should_flood = flood_flag || port->is_mrouter;
+	int pgid;
+
+	for (pgid = PGID_IPV4_MC_DATA; pgid <= PGID_IPV6_MC_CTRL; pgid++)
+		sparx5_pgid_update_mask(port, pgid, should_flood);
+}
+
 static void sparx5_port_attr_bridge_flags(struct sparx5_port *port,
 					  struct switchdev_brport_flags flags)
 {
-	int pgid;
+	if (flags.mask & BR_MCAST_FLOOD) {
+		sparx5_pgid_update_mask(port, PGID_MC_FLOOD, !!(flags.val & BR_MCAST_FLOOD));
+		sparx5_port_update_mcast_ip_flood(port, !!(flags.val & BR_MCAST_FLOOD));
+	}
 
-	if (flags.mask & BR_MCAST_FLOOD)
-		for (pgid = PGID_MC_FLOOD; pgid <= PGID_IPV6_MC_CTRL; pgid++)
-			sparx5_pgid_update_mask(port, pgid, !!(flags.val & BR_MCAST_FLOOD));
 	if (flags.mask & BR_FLOOD)
 		sparx5_pgid_update_mask(port, PGID_UC_FLOOD, !!(flags.val & BR_FLOOD));
 	if (flags.mask & BR_BCAST_FLOOD)
@@ -82,6 +91,37 @@ static void sparx5_port_attr_ageing_set(struct sparx5_port *port,
 	sparx5_set_ageing(port->sparx5, ageing_time);
 }
 
+static void sparx5_port_attr_mrouter_set(struct sparx5_port *port,
+					 struct net_device *orig_dev,
+					 bool enable)
+{
+	struct sparx5 *sparx5 = port->sparx5;
+	struct sparx5_mdb_entry *e;
+	bool flood_flag;
+
+	if ((enable && port->is_mrouter) || (!enable && !port->is_mrouter))
+		return;
+
+	/* Add/del mrouter port on all active mdb entries in HW.
+	 * Don't change entry port mask, since that represents
+	 * ports that actually joined that group.
+	 */
+	mutex_lock(&sparx5->mdb_lock);
+	list_for_each_entry(e, &sparx5->mdb_entries, list) {
+		if (!test_bit(port->portno, e->port_mask) &&
+		    ether_addr_is_ip_mcast(e->addr))
+			sparx5_pgid_update_mask(port, e->pgid_idx, enable);
+	}
+	mutex_unlock(&sparx5->mdb_lock);
+
+	/* Enable/disable flooding depending on if port is mrouter port
+	 * or if mcast flood is enabled.
+	 */
+	port->is_mrouter = enable;
+	flood_flag = br_port_flag_is_set(port->ndev, BR_MCAST_FLOOD);
+	sparx5_port_update_mcast_ip_flood(port, flood_flag);
+}
+
 static int sparx5_port_attr_set(struct net_device *dev, const void *ctx,
 				const struct switchdev_attr *attr,
 				struct netlink_ext_ack *extack)
@@ -110,6 +150,11 @@ static int sparx5_port_attr_set(struct net_device *dev, const void *ctx,
 		port->vlan_aware = attr->u.vlan_filtering;
 		sparx5_vlan_port_apply(port->sparx5, port);
 		break;
+	case SWITCHDEV_ATTR_ID_PORT_MROUTER:
+		sparx5_port_attr_mrouter_set(port,
+					     attr->orig_dev,
+					     attr->u.mrouter);
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -386,16 +431,95 @@ static int sparx5_handle_port_vlan_add(struct net_device *dev,
 				  v->flags & BRIDGE_VLAN_INFO_UNTAGGED);
 }
 
+static int sparx5_alloc_mdb_entry(struct sparx5 *sparx5,
+				  const unsigned char *addr,
+				  u16 vid,
+				  struct sparx5_mdb_entry **entry_out)
+{
+	struct sparx5_mdb_entry *entry;
+	u16 pgid_idx;
+	int err;
+
+	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
+	if (!entry)
+		return -ENOMEM;
+
+	err = sparx5_pgid_alloc_mcast(sparx5, &pgid_idx);
+	if (err) {
+		kfree(entry);
+		return err;
+	}
+
+	memcpy(entry->addr, addr, ETH_ALEN);
+	entry->vid = vid;
+	entry->pgid_idx = pgid_idx;
+
+	mutex_lock(&sparx5->mdb_lock);
+	list_add_tail(&entry->list, &sparx5->mdb_entries);
+	mutex_unlock(&sparx5->mdb_lock);
+
+	*entry_out = entry;
+	return 0;
+}
+
+static void sparx5_free_mdb_entry(struct sparx5 *sparx5,
+				  const unsigned char *addr,
+				  u16 vid)
+{
+	struct sparx5_mdb_entry *entry, *tmp;
+
+	mutex_lock(&sparx5->mdb_lock);
+	list_for_each_entry_safe(entry, tmp, &sparx5->mdb_entries, list) {
+		if ((vid == 0 || entry->vid == vid) &&
+		    ether_addr_equal(addr, entry->addr)) {
+			list_del(&entry->list);
+
+			sparx5_pgid_free(sparx5, entry->pgid_idx);
+			kfree(entry);
+			goto out;
+		}
+	}
+
+out:
+	mutex_unlock(&sparx5->mdb_lock);
+}
+
+static struct sparx5_mdb_entry *sparx5_mdb_get_entry(struct sparx5 *sparx5,
+						     const unsigned char *addr,
+						     u16 vid)
+{
+	struct sparx5_mdb_entry *e, *found = NULL;
+
+	mutex_lock(&sparx5->mdb_lock);
+	list_for_each_entry(e, &sparx5->mdb_entries, list) {
+		if (ether_addr_equal(e->addr, addr) && e->vid == vid) {
+			found = e;
+			goto out;
+		}
+	}
+
+out:
+	mutex_unlock(&sparx5->mdb_lock);
+	return found;
+}
+
+static void sparx5_cpu_copy_ena(struct sparx5 *spx5, u16 pgid, bool enable)
+{
+	spx5_rmw(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(enable),
+		 ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA, spx5,
+		 ANA_AC_PGID_MISC_CFG(pgid));
+}
+
 static int sparx5_handle_port_mdb_add(struct net_device *dev,
 				      struct notifier_block *nb,
 				      const struct switchdev_obj_port_mdb *v)
 {
 	struct sparx5_port *port = netdev_priv(dev);
 	struct sparx5 *spx5 = port->sparx5;
-	u16 pgid_idx, vid;
-	u32 mact_entry;
-	bool is_host;
-	int res, err;
+	struct sparx5_mdb_entry *entry;
+	bool is_host, is_new;
+	int err, i;
+	u16 vid;
 
 	if (!sparx5_netdevice_check(dev))
 		return -EOPNOTSUPP;
@@ -410,66 +534,36 @@ static int sparx5_handle_port_mdb_add(struct net_device *dev,
 	else
 		vid = v->vid;
 
-	res = sparx5_mact_find(spx5, v->addr, vid, &mact_entry);
-
-	if (res == 0) {
-		pgid_idx = LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_GET(mact_entry);
-
-		/* MC_IDX starts after the port masks in the PGID table */
-		pgid_idx += SPX5_PORTS;
-
-		if (is_host)
-			spx5_rmw(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(1),
-				 ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA, spx5,
-				 ANA_AC_PGID_MISC_CFG(pgid_idx));
-		else
-			sparx5_pgid_update_mask(port, pgid_idx, true);
-
-	} else {
-		err = sparx5_pgid_alloc_mcast(spx5, &pgid_idx);
-		if (err) {
-			netdev_warn(dev, "multicast pgid table full\n");
+	is_new = false;
+	entry = sparx5_mdb_get_entry(spx5, v->addr, vid);
+	if (!entry) {
+		err = sparx5_alloc_mdb_entry(spx5, v->addr, vid, &entry);
+		is_new = true;
+		if (err)
 			return err;
-		}
-
-		if (is_host)
-			spx5_rmw(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(1),
-				 ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA, spx5,
-				 ANA_AC_PGID_MISC_CFG(pgid_idx));
-		else
-			sparx5_pgid_update_mask(port, pgid_idx, true);
-
-		err = sparx5_mact_learn(spx5, pgid_idx, v->addr, vid);
-
-		if (err) {
-			netdev_warn(dev, "could not learn mac address %pM\n", v->addr);
-			sparx5_pgid_free(spx5, pgid_idx);
-			sparx5_pgid_update_mask(port, pgid_idx, false);
-			return err;
-		}
 	}
 
-	return 0;
-}
+	mutex_lock(&spx5->mdb_lock);
+
+	/* Add any mrouter ports to the new entry */
+	if (is_new && ether_addr_is_ip_mcast(v->addr))
+		for (i = 0; i < SPX5_PORTS; i++)
+			if (spx5->ports[i] && spx5->ports[i]->is_mrouter)
+				sparx5_pgid_update_mask(spx5->ports[i],
+							entry->pgid_idx,
+							true);
+
+	if (is_host && !entry->cpu_copy) {
+		sparx5_cpu_copy_ena(spx5, entry->pgid_idx, true);
+		entry->cpu_copy = true;
+	} else if (!is_host) {
+		sparx5_pgid_update_mask(port, entry->pgid_idx, true);
+		set_bit(port->portno, entry->port_mask);
+	}
+	mutex_unlock(&spx5->mdb_lock);
 
-static int sparx5_mdb_del_entry(struct net_device *dev,
-				struct sparx5 *spx5,
-				const unsigned char mac[ETH_ALEN],
-				const u16 vid,
-				u16 pgid_idx)
-{
-	int err;
+	sparx5_mact_learn(spx5, entry->pgid_idx, entry->addr, entry->vid);
 
-	err = sparx5_mact_forget(spx5, mac, vid);
-	if (err) {
-		netdev_warn(dev, "could not forget mac address %pM", mac);
-		return err;
-	}
-	err = sparx5_pgid_free(spx5, pgid_idx);
-	if (err) {
-		netdev_err(dev, "attempted to free already freed pgid\n");
-		return err;
-	}
 	return 0;
 }
 
@@ -479,42 +573,45 @@ static int sparx5_handle_port_mdb_del(struct net_device *dev,
 {
 	struct sparx5_port *port = netdev_priv(dev);
 	struct sparx5 *spx5 = port->sparx5;
-	u16 pgid_idx, vid;
-	u32 mact_entry, res, pgid_entry[3], misc_cfg;
-	bool host_ena;
+	struct sparx5_mdb_entry *entry;
+	bool is_host;
+	u16 vid;
 
 	if (!sparx5_netdevice_check(dev))
 		return -EOPNOTSUPP;
 
+	is_host = netif_is_bridge_master(v->obj.orig_dev);
+
 	if (!br_vlan_enabled(spx5->hw_bridge_dev))
 		vid = 1;
 	else
 		vid = v->vid;
 
-	res = sparx5_mact_find(spx5, v->addr, vid, &mact_entry);
-
-	if (res == 0) {
-		pgid_idx = LRN_MAC_ACCESS_CFG_2_MAC_ENTRY_ADDR_GET(mact_entry);
-
-		/* MC_IDX starts after the port masks in the PGID table */
-		pgid_idx += SPX5_PORTS;
-
-		if (netif_is_bridge_master(v->obj.orig_dev))
-			spx5_rmw(ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_SET(0),
-				 ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA, spx5,
-				 ANA_AC_PGID_MISC_CFG(pgid_idx));
-		else
-			sparx5_pgid_update_mask(port, pgid_idx, false);
+	entry = sparx5_mdb_get_entry(spx5, v->addr, vid);
+	if (!entry)
+		return 0;
 
-		misc_cfg = spx5_rd(spx5, ANA_AC_PGID_MISC_CFG(pgid_idx));
-		host_ena = ANA_AC_PGID_MISC_CFG_PGID_CPU_COPY_ENA_GET(misc_cfg);
+	mutex_lock(&spx5->mdb_lock);
+	if (is_host && entry->cpu_copy) {
+		sparx5_cpu_copy_ena(spx5, entry->pgid_idx, false);
+		entry->cpu_copy = false;
+	} else if (!is_host) {
+		clear_bit(port->portno, entry->port_mask);
 
-		sparx5_pgid_read_mask(spx5, pgid_idx, pgid_entry);
-		if (bitmap_empty((unsigned long *)pgid_entry, SPX5_PORTS) && !host_ena)
-			/* No ports or CPU are in MC group. Remove entry */
-			return sparx5_mdb_del_entry(dev, spx5, v->addr, vid, pgid_idx);
+		/* Port not mrouter port or addr is L2 mcast, remove port from mask. */
+		if (!port->is_mrouter || !ether_addr_is_ip_mcast(v->addr))
+			sparx5_pgid_update_mask(port, entry->pgid_idx, false);
+	}
+	mutex_unlock(&spx5->mdb_lock);
+
+	if (bitmap_empty(entry->port_mask, SPX5_PORTS) && !entry->cpu_copy) {
+		 /* Clear pgid in case mrouter ports exists
+		  * that are not part of the group.
+		  */
+		sparx5_pgid_clear(spx5, entry->pgid_idx);
+		sparx5_mact_forget(spx5, entry->addr, entry->vid);
+		sparx5_free_mdb_entry(spx5, entry->addr, entry->vid);
 	}
-
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_tc.c b/drivers/net/ethernet/microchip/sparx5/sparx5_tc.c
new file mode 100644
index 000000000000..e05429c751ee
--- /dev/null
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_tc.c
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Microchip Sparx5 Switch driver
+ *
+ * Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries.
+ */
+
+#include <net/pkt_cls.h>
+
+#include "sparx5_tc.h"
+#include "sparx5_main.h"
+#include "sparx5_qos.h"
+
+static void sparx5_tc_get_layer_and_idx(u32 parent, u32 portno, u32 *layer,
+					u32 *idx)
+{
+	if (parent == TC_H_ROOT) {
+		*layer = 2;
+		*idx = portno;
+	} else {
+		u32 queue = TC_H_MIN(parent) - 1;
+		*layer = 0;
+		*idx = SPX5_HSCH_L0_GET_IDX(portno, queue);
+	}
+}
+
+static int sparx5_tc_setup_qdisc_mqprio(struct net_device *ndev,
+					struct tc_mqprio_qopt_offload *m)
+{
+	m->qopt.hw = TC_MQPRIO_HW_OFFLOAD_TCS;
+
+	if (m->qopt.num_tc == 0)
+		return sparx5_tc_mqprio_del(ndev);
+	else
+		return sparx5_tc_mqprio_add(ndev, m->qopt.num_tc);
+}
+
+static int sparx5_tc_setup_qdisc_tbf(struct net_device *ndev,
+				     struct tc_tbf_qopt_offload *qopt)
+{
+	struct sparx5_port *port = netdev_priv(ndev);
+	u32 layer, se_idx;
+
+	sparx5_tc_get_layer_and_idx(qopt->parent, port->portno, &layer,
+				    &se_idx);
+
+	switch (qopt->command) {
+	case TC_TBF_REPLACE:
+		return sparx5_tc_tbf_add(port, &qopt->replace_params, layer,
+					 se_idx);
+	case TC_TBF_DESTROY:
+		return sparx5_tc_tbf_del(port, layer, se_idx);
+	case TC_TBF_STATS:
+		return -EOPNOTSUPP;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int sparx5_tc_setup_qdisc_ets(struct net_device *ndev,
+				     struct tc_ets_qopt_offload *qopt)
+{
+	struct tc_ets_qopt_offload_replace_params *params =
+		&qopt->replace_params;
+	struct sparx5_port *port = netdev_priv(ndev);
+	int i;
+
+	/* Only allow ets on ports  */
+	if (qopt->parent != TC_H_ROOT)
+		return -EOPNOTSUPP;
+
+	switch (qopt->command) {
+	case TC_ETS_REPLACE:
+
+		/* We support eight priorities */
+		if (params->bands != SPX5_PRIOS)
+			return -EOPNOTSUPP;
+
+		/* Sanity checks */
+		for (i = 0; i < SPX5_PRIOS; ++i) {
+			/* Priority map is *always* reverse e.g: 7 6 5 .. 0 */
+			if (params->priomap[i] != (7 - i))
+				return -EOPNOTSUPP;
+			/* Throw an error if we receive zero weights by tc */
+			if (params->quanta[i] && params->weights[i] == 0) {
+				pr_err("Invalid ets configuration; band %d has weight zero",
+				       i);
+				return -EINVAL;
+			}
+		}
+
+		sparx5_tc_ets_add(port, params);
+		break;
+	case TC_ETS_DESTROY:
+
+		sparx5_tc_ets_del(port);
+
+		break;
+	case TC_ETS_GRAFT:
+		return -EOPNOTSUPP;
+
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+int sparx5_port_setup_tc(struct net_device *ndev, enum tc_setup_type type,
+			 void *type_data)
+{
+	switch (type) {
+	case TC_SETUP_QDISC_MQPRIO:
+		return sparx5_tc_setup_qdisc_mqprio(ndev, type_data);
+	case TC_SETUP_QDISC_TBF:
+		return sparx5_tc_setup_qdisc_tbf(ndev, type_data);
+	case TC_SETUP_QDISC_ETS:
+		return sparx5_tc_setup_qdisc_ets(ndev, type_data);
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_tc.h b/drivers/net/ethernet/microchip/sparx5/sparx5_tc.h
new file mode 100644
index 000000000000..5b55e11b77e1
--- /dev/null
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_tc.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/* Microchip Sparx5 Switch driver
+ *
+ * Copyright (c) 2022 Microchip Technology Inc. and its subsidiaries.
+ */
+
+#ifndef __SPARX5_TC_H__
+#define __SPARX5_TC_H__
+
+#include <linux/netdevice.h>
+
+int sparx5_port_setup_tc(struct net_device *ndev, enum tc_setup_type type,
+			 void *type_data);
+
+#endif	/* __SPARX5_TC_H__ */
diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c
index 37e4ac965849..34f954bbf815 100644
--- a/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c
+++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vlan.c
@@ -138,6 +138,13 @@ void sparx5_pgid_update_mask(struct sparx5_port *port, int pgid, bool enable)
 	}
 }
 
+void sparx5_pgid_clear(struct sparx5 *spx5, int pgid)
+{
+	spx5_wr(0, spx5, ANA_AC_PGID_CFG(pgid));
+	spx5_wr(0, spx5, ANA_AC_PGID_CFG1(pgid));
+	spx5_wr(0, spx5, ANA_AC_PGID_CFG2(pgid));
+}
+
 void sparx5_pgid_read_mask(struct sparx5 *spx5, int pgid, u32 portmask[3])
 {
 	portmask[0] = spx5_rd(spx5, ANA_AC_PGID_CFG(pgid));
diff --git a/drivers/net/ethernet/moxa/moxart_ether.c b/drivers/net/ethernet/moxa/moxart_ether.c
index 9e57d23e57bf..3da99b62797d 100644
--- a/drivers/net/ethernet/moxa/moxart_ether.c
+++ b/drivers/net/ethernet/moxa/moxart_ether.c
@@ -29,12 +29,12 @@
 
 #include "moxart_ether.h"
 
-static inline void moxart_desc_write(u32 data, u32 *desc)
+static inline void moxart_desc_write(u32 data, __le32 *desc)
 {
 	*desc = cpu_to_le32(data);
 }
 
-static inline u32 moxart_desc_read(u32 *desc)
+static inline u32 moxart_desc_read(__le32 *desc)
 {
 	return le32_to_cpu(*desc);
 }
diff --git a/drivers/net/ethernet/mscc/Makefile b/drivers/net/ethernet/mscc/Makefile
index 41b34a509308..5d435a565d4c 100644
--- a/drivers/net/ethernet/mscc/Makefile
+++ b/drivers/net/ethernet/mscc/Makefile
@@ -2,16 +2,17 @@
 obj-$(CONFIG_MSCC_OCELOT_SWITCH_LIB) += mscc_ocelot_switch_lib.o
 mscc_ocelot_switch_lib-y := \
 	ocelot.o \
+	ocelot_devlink.o \
+	ocelot_flower.o \
 	ocelot_io.o \
 	ocelot_police.o \
-	ocelot_vcap.o \
-	ocelot_flower.o \
 	ocelot_ptp.o \
-	ocelot_devlink.o \
+	ocelot_stats.o \
+	ocelot_vcap.o \
 	vsc7514_regs.o
 mscc_ocelot_switch_lib-$(CONFIG_BRIDGE_MRP) += ocelot_mrp.o
 obj-$(CONFIG_MSCC_OCELOT_SWITCH) += mscc_ocelot.o
 mscc_ocelot-y := \
 	ocelot_fdma.o \
-	ocelot_vsc7514.o \
-	ocelot_net.o
+	ocelot_net.o \
+	ocelot_vsc7514.o
diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c
index 306026e6aa11..13b14110a060 100644
--- a/drivers/net/ethernet/mscc/ocelot.c
+++ b/drivers/net/ethernet/mscc/ocelot.c
@@ -6,7 +6,6 @@
  */
 #include <linux/dsa/ocelot.h>
 #include <linux/if_bridge.h>
-#include <linux/ptp_classify.h>
 #include <soc/mscc/ocelot_vcap.h>
 #include "ocelot.h"
 #include "ocelot_vcap.h"
@@ -290,6 +289,13 @@ static int ocelot_port_num_untagged_vlans(struct ocelot *ocelot, int port)
 		if (!(vlan->portmask & BIT(port)))
 			continue;
 
+		/* Ignore the VLAN added by ocelot_add_vlan_unaware_pvid(),
+		 * because this is never active in hardware at the same time as
+		 * the bridge VLANs, which only matter in VLAN-aware mode.
+		 */
+		if (vlan->vid >= OCELOT_RSV_VLAN_RANGE_START)
+			continue;
+
 		if (vlan->untagged & BIT(port))
 			num_untagged++;
 	}
@@ -910,211 +916,6 @@ void ocelot_phylink_mac_link_up(struct ocelot *ocelot, int port,
 }
 EXPORT_SYMBOL_GPL(ocelot_phylink_mac_link_up);
 
-static int ocelot_port_add_txtstamp_skb(struct ocelot *ocelot, int port,
-					struct sk_buff *clone)
-{
-	struct ocelot_port *ocelot_port = ocelot->ports[port];
-	unsigned long flags;
-
-	spin_lock_irqsave(&ocelot->ts_id_lock, flags);
-
-	if (ocelot_port->ptp_skbs_in_flight == OCELOT_MAX_PTP_ID ||
-	    ocelot->ptp_skbs_in_flight == OCELOT_PTP_FIFO_SIZE) {
-		spin_unlock_irqrestore(&ocelot->ts_id_lock, flags);
-		return -EBUSY;
-	}
-
-	skb_shinfo(clone)->tx_flags |= SKBTX_IN_PROGRESS;
-	/* Store timestamp ID in OCELOT_SKB_CB(clone)->ts_id */
-	OCELOT_SKB_CB(clone)->ts_id = ocelot_port->ts_id;
-
-	ocelot_port->ts_id++;
-	if (ocelot_port->ts_id == OCELOT_MAX_PTP_ID)
-		ocelot_port->ts_id = 0;
-
-	ocelot_port->ptp_skbs_in_flight++;
-	ocelot->ptp_skbs_in_flight++;
-
-	skb_queue_tail(&ocelot_port->tx_skbs, clone);
-
-	spin_unlock_irqrestore(&ocelot->ts_id_lock, flags);
-
-	return 0;
-}
-
-static bool ocelot_ptp_is_onestep_sync(struct sk_buff *skb,
-				       unsigned int ptp_class)
-{
-	struct ptp_header *hdr;
-	u8 msgtype, twostep;
-
-	hdr = ptp_parse_header(skb, ptp_class);
-	if (!hdr)
-		return false;
-
-	msgtype = ptp_get_msgtype(hdr, ptp_class);
-	twostep = hdr->flag_field[0] & 0x2;
-
-	if (msgtype == PTP_MSGTYPE_SYNC && twostep == 0)
-		return true;
-
-	return false;
-}
-
-int ocelot_port_txtstamp_request(struct ocelot *ocelot, int port,
-				 struct sk_buff *skb,
-				 struct sk_buff **clone)
-{
-	struct ocelot_port *ocelot_port = ocelot->ports[port];
-	u8 ptp_cmd = ocelot_port->ptp_cmd;
-	unsigned int ptp_class;
-	int err;
-
-	/* Don't do anything if PTP timestamping not enabled */
-	if (!ptp_cmd)
-		return 0;
-
-	ptp_class = ptp_classify_raw(skb);
-	if (ptp_class == PTP_CLASS_NONE)
-		return -EINVAL;
-
-	/* Store ptp_cmd in OCELOT_SKB_CB(skb)->ptp_cmd */
-	if (ptp_cmd == IFH_REW_OP_ORIGIN_PTP) {
-		if (ocelot_ptp_is_onestep_sync(skb, ptp_class)) {
-			OCELOT_SKB_CB(skb)->ptp_cmd = ptp_cmd;
-			return 0;
-		}
-
-		/* Fall back to two-step timestamping */
-		ptp_cmd = IFH_REW_OP_TWO_STEP_PTP;
-	}
-
-	if (ptp_cmd == IFH_REW_OP_TWO_STEP_PTP) {
-		*clone = skb_clone_sk(skb);
-		if (!(*clone))
-			return -ENOMEM;
-
-		err = ocelot_port_add_txtstamp_skb(ocelot, port, *clone);
-		if (err)
-			return err;
-
-		OCELOT_SKB_CB(skb)->ptp_cmd = ptp_cmd;
-		OCELOT_SKB_CB(*clone)->ptp_class = ptp_class;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL(ocelot_port_txtstamp_request);
-
-static void ocelot_get_hwtimestamp(struct ocelot *ocelot,
-				   struct timespec64 *ts)
-{
-	unsigned long flags;
-	u32 val;
-
-	spin_lock_irqsave(&ocelot->ptp_clock_lock, flags);
-
-	/* Read current PTP time to get seconds */
-	val = ocelot_read_rix(ocelot, PTP_PIN_CFG, TOD_ACC_PIN);
-
-	val &= ~(PTP_PIN_CFG_SYNC | PTP_PIN_CFG_ACTION_MASK | PTP_PIN_CFG_DOM);
-	val |= PTP_PIN_CFG_ACTION(PTP_PIN_ACTION_SAVE);
-	ocelot_write_rix(ocelot, val, PTP_PIN_CFG, TOD_ACC_PIN);
-	ts->tv_sec = ocelot_read_rix(ocelot, PTP_PIN_TOD_SEC_LSB, TOD_ACC_PIN);
-
-	/* Read packet HW timestamp from FIFO */
-	val = ocelot_read(ocelot, SYS_PTP_TXSTAMP);
-	ts->tv_nsec = SYS_PTP_TXSTAMP_PTP_TXSTAMP(val);
-
-	/* Sec has incremented since the ts was registered */
-	if ((ts->tv_sec & 0x1) != !!(val & SYS_PTP_TXSTAMP_PTP_TXSTAMP_SEC))
-		ts->tv_sec--;
-
-	spin_unlock_irqrestore(&ocelot->ptp_clock_lock, flags);
-}
-
-static bool ocelot_validate_ptp_skb(struct sk_buff *clone, u16 seqid)
-{
-	struct ptp_header *hdr;
-
-	hdr = ptp_parse_header(clone, OCELOT_SKB_CB(clone)->ptp_class);
-	if (WARN_ON(!hdr))
-		return false;
-
-	return seqid == ntohs(hdr->sequence_id);
-}
-
-void ocelot_get_txtstamp(struct ocelot *ocelot)
-{
-	int budget = OCELOT_PTP_QUEUE_SZ;
-
-	while (budget--) {
-		struct sk_buff *skb, *skb_tmp, *skb_match = NULL;
-		struct skb_shared_hwtstamps shhwtstamps;
-		u32 val, id, seqid, txport;
-		struct ocelot_port *port;
-		struct timespec64 ts;
-		unsigned long flags;
-
-		val = ocelot_read(ocelot, SYS_PTP_STATUS);
-
-		/* Check if a timestamp can be retrieved */
-		if (!(val & SYS_PTP_STATUS_PTP_MESS_VLD))
-			break;
-
-		WARN_ON(val & SYS_PTP_STATUS_PTP_OVFL);
-
-		/* Retrieve the ts ID and Tx port */
-		id = SYS_PTP_STATUS_PTP_MESS_ID_X(val);
-		txport = SYS_PTP_STATUS_PTP_MESS_TXPORT_X(val);
-		seqid = SYS_PTP_STATUS_PTP_MESS_SEQ_ID(val);
-
-		port = ocelot->ports[txport];
-
-		spin_lock(&ocelot->ts_id_lock);
-		port->ptp_skbs_in_flight--;
-		ocelot->ptp_skbs_in_flight--;
-		spin_unlock(&ocelot->ts_id_lock);
-
-		/* Retrieve its associated skb */
-try_again:
-		spin_lock_irqsave(&port->tx_skbs.lock, flags);
-
-		skb_queue_walk_safe(&port->tx_skbs, skb, skb_tmp) {
-			if (OCELOT_SKB_CB(skb)->ts_id != id)
-				continue;
-			__skb_unlink(skb, &port->tx_skbs);
-			skb_match = skb;
-			break;
-		}
-
-		spin_unlock_irqrestore(&port->tx_skbs.lock, flags);
-
-		if (WARN_ON(!skb_match))
-			continue;
-
-		if (!ocelot_validate_ptp_skb(skb_match, seqid)) {
-			dev_err_ratelimited(ocelot->dev,
-					    "port %d received stale TX timestamp for seqid %d, discarding\n",
-					    txport, seqid);
-			dev_kfree_skb_any(skb);
-			goto try_again;
-		}
-
-		/* Get the h/w timestamp */
-		ocelot_get_hwtimestamp(ocelot, &ts);
-
-		/* Set the timestamp into the skb */
-		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
-		shhwtstamps.hwtstamp = ktime_set(ts.tv_sec, ts.tv_nsec);
-		skb_complete_tx_timestamp(skb_match, &shhwtstamps);
-
-		/* Next ts */
-		ocelot_write(ocelot, SYS_PTP_NXT_PTP_NXT, SYS_PTP_NXT);
-	}
-}
-EXPORT_SYMBOL(ocelot_get_txtstamp);
-
 static int ocelot_rx_frame_word(struct ocelot *ocelot, u8 grp, bool ifh,
 				u32 *rval)
 {
@@ -1366,50 +1167,6 @@ int ocelot_fdb_del(struct ocelot *ocelot, int port, const unsigned char *addr,
 }
 EXPORT_SYMBOL(ocelot_fdb_del);
 
-int ocelot_port_fdb_do_dump(const unsigned char *addr, u16 vid,
-			    bool is_static, void *data)
-{
-	struct ocelot_dump_ctx *dump = data;
-	u32 portid = NETLINK_CB(dump->cb->skb).portid;
-	u32 seq = dump->cb->nlh->nlmsg_seq;
-	struct nlmsghdr *nlh;
-	struct ndmsg *ndm;
-
-	if (dump->idx < dump->cb->args[2])
-		goto skip;
-
-	nlh = nlmsg_put(dump->skb, portid, seq, RTM_NEWNEIGH,
-			sizeof(*ndm), NLM_F_MULTI);
-	if (!nlh)
-		return -EMSGSIZE;
-
-	ndm = nlmsg_data(nlh);
-	ndm->ndm_family  = AF_BRIDGE;
-	ndm->ndm_pad1    = 0;
-	ndm->ndm_pad2    = 0;
-	ndm->ndm_flags   = NTF_SELF;
-	ndm->ndm_type    = 0;
-	ndm->ndm_ifindex = dump->dev->ifindex;
-	ndm->ndm_state   = is_static ? NUD_NOARP : NUD_REACHABLE;
-
-	if (nla_put(dump->skb, NDA_LLADDR, ETH_ALEN, addr))
-		goto nla_put_failure;
-
-	if (vid && nla_put_u16(dump->skb, NDA_VLAN, vid))
-		goto nla_put_failure;
-
-	nlmsg_end(dump->skb, nlh);
-
-skip:
-	dump->idx++;
-	return 0;
-
-nla_put_failure:
-	nlmsg_cancel(dump->skb, nlh);
-	return -EMSGSIZE;
-}
-EXPORT_SYMBOL(ocelot_port_fdb_do_dump);
-
 /* Caller must hold &ocelot->mact_lock */
 static int ocelot_mact_read(struct ocelot *ocelot, int port, int row, int col,
 			    struct ocelot_mact_entry *entry)
@@ -1541,53 +1298,6 @@ int ocelot_fdb_dump(struct ocelot *ocelot, int port,
 }
 EXPORT_SYMBOL(ocelot_fdb_dump);
 
-static void ocelot_populate_l2_ptp_trap_key(struct ocelot_vcap_filter *trap)
-{
-	trap->key_type = OCELOT_VCAP_KEY_ETYPE;
-	*(__be16 *)trap->key.etype.etype.value = htons(ETH_P_1588);
-	*(__be16 *)trap->key.etype.etype.mask = htons(0xffff);
-}
-
-static void
-ocelot_populate_ipv4_ptp_event_trap_key(struct ocelot_vcap_filter *trap)
-{
-	trap->key_type = OCELOT_VCAP_KEY_IPV4;
-	trap->key.ipv4.proto.value[0] = IPPROTO_UDP;
-	trap->key.ipv4.proto.mask[0] = 0xff;
-	trap->key.ipv4.dport.value = PTP_EV_PORT;
-	trap->key.ipv4.dport.mask = 0xffff;
-}
-
-static void
-ocelot_populate_ipv6_ptp_event_trap_key(struct ocelot_vcap_filter *trap)
-{
-	trap->key_type = OCELOT_VCAP_KEY_IPV6;
-	trap->key.ipv4.proto.value[0] = IPPROTO_UDP;
-	trap->key.ipv4.proto.mask[0] = 0xff;
-	trap->key.ipv6.dport.value = PTP_EV_PORT;
-	trap->key.ipv6.dport.mask = 0xffff;
-}
-
-static void
-ocelot_populate_ipv4_ptp_general_trap_key(struct ocelot_vcap_filter *trap)
-{
-	trap->key_type = OCELOT_VCAP_KEY_IPV4;
-	trap->key.ipv4.proto.value[0] = IPPROTO_UDP;
-	trap->key.ipv4.proto.mask[0] = 0xff;
-	trap->key.ipv4.dport.value = PTP_GEN_PORT;
-	trap->key.ipv4.dport.mask = 0xffff;
-}
-
-static void
-ocelot_populate_ipv6_ptp_general_trap_key(struct ocelot_vcap_filter *trap)
-{
-	trap->key_type = OCELOT_VCAP_KEY_IPV6;
-	trap->key.ipv4.proto.value[0] = IPPROTO_UDP;
-	trap->key.ipv4.proto.mask[0] = 0xff;
-	trap->key.ipv6.dport.value = PTP_GEN_PORT;
-	trap->key.ipv6.dport.mask = 0xffff;
-}
-
 int ocelot_trap_add(struct ocelot *ocelot, int port,
 		    unsigned long cookie, bool take_ts,
 		    void (*populate)(struct ocelot_vcap_filter *f))
@@ -1656,381 +1366,6 @@ int ocelot_trap_del(struct ocelot *ocelot, int port, unsigned long cookie)
 	return ocelot_vcap_filter_replace(ocelot, trap);
 }
 
-static int ocelot_l2_ptp_trap_add(struct ocelot *ocelot, int port)
-{
-	unsigned long l2_cookie = OCELOT_VCAP_IS2_L2_PTP_TRAP(ocelot);
-
-	return ocelot_trap_add(ocelot, port, l2_cookie, true,
-			       ocelot_populate_l2_ptp_trap_key);
-}
-
-static int ocelot_l2_ptp_trap_del(struct ocelot *ocelot, int port)
-{
-	unsigned long l2_cookie = OCELOT_VCAP_IS2_L2_PTP_TRAP(ocelot);
-
-	return ocelot_trap_del(ocelot, port, l2_cookie);
-}
-
-static int ocelot_ipv4_ptp_trap_add(struct ocelot *ocelot, int port)
-{
-	unsigned long ipv4_gen_cookie = OCELOT_VCAP_IS2_IPV4_GEN_PTP_TRAP(ocelot);
-	unsigned long ipv4_ev_cookie = OCELOT_VCAP_IS2_IPV4_EV_PTP_TRAP(ocelot);
-	int err;
-
-	err = ocelot_trap_add(ocelot, port, ipv4_ev_cookie, true,
-			      ocelot_populate_ipv4_ptp_event_trap_key);
-	if (err)
-		return err;
-
-	err = ocelot_trap_add(ocelot, port, ipv4_gen_cookie, false,
-			      ocelot_populate_ipv4_ptp_general_trap_key);
-	if (err)
-		ocelot_trap_del(ocelot, port, ipv4_ev_cookie);
-
-	return err;
-}
-
-static int ocelot_ipv4_ptp_trap_del(struct ocelot *ocelot, int port)
-{
-	unsigned long ipv4_gen_cookie = OCELOT_VCAP_IS2_IPV4_GEN_PTP_TRAP(ocelot);
-	unsigned long ipv4_ev_cookie = OCELOT_VCAP_IS2_IPV4_EV_PTP_TRAP(ocelot);
-	int err;
-
-	err = ocelot_trap_del(ocelot, port, ipv4_ev_cookie);
-	err |= ocelot_trap_del(ocelot, port, ipv4_gen_cookie);
-	return err;
-}
-
-static int ocelot_ipv6_ptp_trap_add(struct ocelot *ocelot, int port)
-{
-	unsigned long ipv6_gen_cookie = OCELOT_VCAP_IS2_IPV6_GEN_PTP_TRAP(ocelot);
-	unsigned long ipv6_ev_cookie = OCELOT_VCAP_IS2_IPV6_EV_PTP_TRAP(ocelot);
-	int err;
-
-	err = ocelot_trap_add(ocelot, port, ipv6_ev_cookie, true,
-			      ocelot_populate_ipv6_ptp_event_trap_key);
-	if (err)
-		return err;
-
-	err = ocelot_trap_add(ocelot, port, ipv6_gen_cookie, false,
-			      ocelot_populate_ipv6_ptp_general_trap_key);
-	if (err)
-		ocelot_trap_del(ocelot, port, ipv6_ev_cookie);
-
-	return err;
-}
-
-static int ocelot_ipv6_ptp_trap_del(struct ocelot *ocelot, int port)
-{
-	unsigned long ipv6_gen_cookie = OCELOT_VCAP_IS2_IPV6_GEN_PTP_TRAP(ocelot);
-	unsigned long ipv6_ev_cookie = OCELOT_VCAP_IS2_IPV6_EV_PTP_TRAP(ocelot);
-	int err;
-
-	err = ocelot_trap_del(ocelot, port, ipv6_ev_cookie);
-	err |= ocelot_trap_del(ocelot, port, ipv6_gen_cookie);
-	return err;
-}
-
-static int ocelot_setup_ptp_traps(struct ocelot *ocelot, int port,
-				  bool l2, bool l4)
-{
-	int err;
-
-	if (l2)
-		err = ocelot_l2_ptp_trap_add(ocelot, port);
-	else
-		err = ocelot_l2_ptp_trap_del(ocelot, port);
-	if (err)
-		return err;
-
-	if (l4) {
-		err = ocelot_ipv4_ptp_trap_add(ocelot, port);
-		if (err)
-			goto err_ipv4;
-
-		err = ocelot_ipv6_ptp_trap_add(ocelot, port);
-		if (err)
-			goto err_ipv6;
-	} else {
-		err = ocelot_ipv4_ptp_trap_del(ocelot, port);
-
-		err |= ocelot_ipv6_ptp_trap_del(ocelot, port);
-	}
-	if (err)
-		return err;
-
-	return 0;
-
-err_ipv6:
-	ocelot_ipv4_ptp_trap_del(ocelot, port);
-err_ipv4:
-	if (l2)
-		ocelot_l2_ptp_trap_del(ocelot, port);
-	return err;
-}
-
-int ocelot_hwstamp_get(struct ocelot *ocelot, int port, struct ifreq *ifr)
-{
-	return copy_to_user(ifr->ifr_data, &ocelot->hwtstamp_config,
-			    sizeof(ocelot->hwtstamp_config)) ? -EFAULT : 0;
-}
-EXPORT_SYMBOL(ocelot_hwstamp_get);
-
-int ocelot_hwstamp_set(struct ocelot *ocelot, int port, struct ifreq *ifr)
-{
-	struct ocelot_port *ocelot_port = ocelot->ports[port];
-	bool l2 = false, l4 = false;
-	struct hwtstamp_config cfg;
-	int err;
-
-	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
-		return -EFAULT;
-
-	/* Tx type sanity check */
-	switch (cfg.tx_type) {
-	case HWTSTAMP_TX_ON:
-		ocelot_port->ptp_cmd = IFH_REW_OP_TWO_STEP_PTP;
-		break;
-	case HWTSTAMP_TX_ONESTEP_SYNC:
-		/* IFH_REW_OP_ONE_STEP_PTP updates the correctional field, we
-		 * need to update the origin time.
-		 */
-		ocelot_port->ptp_cmd = IFH_REW_OP_ORIGIN_PTP;
-		break;
-	case HWTSTAMP_TX_OFF:
-		ocelot_port->ptp_cmd = 0;
-		break;
-	default:
-		return -ERANGE;
-	}
-
-	mutex_lock(&ocelot->ptp_lock);
-
-	switch (cfg.rx_filter) {
-	case HWTSTAMP_FILTER_NONE:
-		break;
-	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
-	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
-	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
-		l4 = true;
-		break;
-	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
-	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
-	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
-		l2 = true;
-		break;
-	case HWTSTAMP_FILTER_PTP_V2_EVENT:
-	case HWTSTAMP_FILTER_PTP_V2_SYNC:
-	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
-		l2 = true;
-		l4 = true;
-		break;
-	default:
-		mutex_unlock(&ocelot->ptp_lock);
-		return -ERANGE;
-	}
-
-	err = ocelot_setup_ptp_traps(ocelot, port, l2, l4);
-	if (err) {
-		mutex_unlock(&ocelot->ptp_lock);
-		return err;
-	}
-
-	if (l2 && l4)
-		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
-	else if (l2)
-		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
-	else if (l4)
-		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
-	else
-		cfg.rx_filter = HWTSTAMP_FILTER_NONE;
-
-	/* Commit back the result & save it */
-	memcpy(&ocelot->hwtstamp_config, &cfg, sizeof(cfg));
-	mutex_unlock(&ocelot->ptp_lock);
-
-	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
-}
-EXPORT_SYMBOL(ocelot_hwstamp_set);
-
-void ocelot_get_strings(struct ocelot *ocelot, int port, u32 sset, u8 *data)
-{
-	int i;
-
-	if (sset != ETH_SS_STATS)
-		return;
-
-	for (i = 0; i < OCELOT_NUM_STATS; i++) {
-		if (ocelot->stats_layout[i].name[0] == '\0')
-			continue;
-
-		memcpy(data + i * ETH_GSTRING_LEN, ocelot->stats_layout[i].name,
-		       ETH_GSTRING_LEN);
-	}
-}
-EXPORT_SYMBOL(ocelot_get_strings);
-
-/* Caller must hold &ocelot->stats_lock */
-static int ocelot_port_update_stats(struct ocelot *ocelot, int port)
-{
-	unsigned int idx = port * OCELOT_NUM_STATS;
-	struct ocelot_stats_region *region;
-	int err, j;
-
-	/* Configure the port to read the stats from */
-	ocelot_write(ocelot, SYS_STAT_CFG_STAT_VIEW(port), SYS_STAT_CFG);
-
-	list_for_each_entry(region, &ocelot->stats_regions, node) {
-		err = ocelot_bulk_read(ocelot, region->base, region->buf,
-				       region->count);
-		if (err)
-			return err;
-
-		for (j = 0; j < region->count; j++) {
-			u64 *stat = &ocelot->stats[idx + j];
-			u64 val = region->buf[j];
-
-			if (val < (*stat & U32_MAX))
-				*stat += (u64)1 << 32;
-
-			*stat = (*stat & ~(u64)U32_MAX) + val;
-		}
-
-		idx += region->count;
-	}
-
-	return err;
-}
-
-static void ocelot_check_stats_work(struct work_struct *work)
-{
-	struct delayed_work *del_work = to_delayed_work(work);
-	struct ocelot *ocelot = container_of(del_work, struct ocelot,
-					     stats_work);
-	int i, err;
-
-	spin_lock(&ocelot->stats_lock);
-	for (i = 0; i < ocelot->num_phys_ports; i++) {
-		err = ocelot_port_update_stats(ocelot, i);
-		if (err)
-			break;
-	}
-	spin_unlock(&ocelot->stats_lock);
-
-	if (err)
-		dev_err(ocelot->dev, "Error %d updating ethtool stats\n",  err);
-
-	queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work,
-			   OCELOT_STATS_CHECK_DELAY);
-}
-
-void ocelot_get_ethtool_stats(struct ocelot *ocelot, int port, u64 *data)
-{
-	int i, err;
-
-	spin_lock(&ocelot->stats_lock);
-
-	/* check and update now */
-	err = ocelot_port_update_stats(ocelot, port);
-
-	/* Copy all supported counters */
-	for (i = 0; i < OCELOT_NUM_STATS; i++) {
-		int index = port * OCELOT_NUM_STATS + i;
-
-		if (ocelot->stats_layout[i].name[0] == '\0')
-			continue;
-
-		*data++ = ocelot->stats[index];
-	}
-
-	spin_unlock(&ocelot->stats_lock);
-
-	if (err)
-		dev_err(ocelot->dev, "Error %d updating ethtool stats\n", err);
-}
-EXPORT_SYMBOL(ocelot_get_ethtool_stats);
-
-int ocelot_get_sset_count(struct ocelot *ocelot, int port, int sset)
-{
-	int i, num_stats = 0;
-
-	if (sset != ETH_SS_STATS)
-		return -EOPNOTSUPP;
-
-	for (i = 0; i < OCELOT_NUM_STATS; i++)
-		if (ocelot->stats_layout[i].name[0] != '\0')
-			num_stats++;
-
-	return num_stats;
-}
-EXPORT_SYMBOL(ocelot_get_sset_count);
-
-static int ocelot_prepare_stats_regions(struct ocelot *ocelot)
-{
-	struct ocelot_stats_region *region = NULL;
-	unsigned int last;
-	int i;
-
-	INIT_LIST_HEAD(&ocelot->stats_regions);
-
-	for (i = 0; i < OCELOT_NUM_STATS; i++) {
-		if (ocelot->stats_layout[i].name[0] == '\0')
-			continue;
-
-		if (region && ocelot->stats_layout[i].reg == last + 4) {
-			region->count++;
-		} else {
-			region = devm_kzalloc(ocelot->dev, sizeof(*region),
-					      GFP_KERNEL);
-			if (!region)
-				return -ENOMEM;
-
-			region->base = ocelot->stats_layout[i].reg;
-			region->count = 1;
-			list_add_tail(&region->node, &ocelot->stats_regions);
-		}
-
-		last = ocelot->stats_layout[i].reg;
-	}
-
-	list_for_each_entry(region, &ocelot->stats_regions, node) {
-		region->buf = devm_kcalloc(ocelot->dev, region->count,
-					   sizeof(*region->buf), GFP_KERNEL);
-		if (!region->buf)
-			return -ENOMEM;
-	}
-
-	return 0;
-}
-
-int ocelot_get_ts_info(struct ocelot *ocelot, int port,
-		       struct ethtool_ts_info *info)
-{
-	info->phc_index = ocelot->ptp_clock ?
-			  ptp_clock_index(ocelot->ptp_clock) : -1;
-	if (info->phc_index == -1) {
-		info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE |
-					 SOF_TIMESTAMPING_RX_SOFTWARE |
-					 SOF_TIMESTAMPING_SOFTWARE;
-		return 0;
-	}
-	info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE |
-				 SOF_TIMESTAMPING_RX_SOFTWARE |
-				 SOF_TIMESTAMPING_SOFTWARE |
-				 SOF_TIMESTAMPING_TX_HARDWARE |
-				 SOF_TIMESTAMPING_RX_HARDWARE |
-				 SOF_TIMESTAMPING_RAW_HARDWARE;
-	info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON) |
-			 BIT(HWTSTAMP_TX_ONESTEP_SYNC);
-	info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) |
-			   BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) |
-			   BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
-			   BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT);
-
-	return 0;
-}
-EXPORT_SYMBOL(ocelot_get_ts_info);
-
 static u32 ocelot_get_bond_mask(struct ocelot *ocelot, struct net_device *bond)
 {
 	u32 mask = 0;
@@ -2054,7 +1389,7 @@ static u32 ocelot_get_bond_mask(struct ocelot *ocelot, struct net_device *bond)
 /* The logical port number of a LAG is equal to the lowest numbered physical
  * port ID present in that LAG. It may change if that port ever leaves the LAG.
  */
-static int ocelot_bond_get_id(struct ocelot *ocelot, struct net_device *bond)
+int ocelot_bond_get_id(struct ocelot *ocelot, struct net_device *bond)
 {
 	int bond_mask = ocelot_get_bond_mask(ocelot, bond);
 
@@ -2063,7 +1398,18 @@ static int ocelot_bond_get_id(struct ocelot *ocelot, struct net_device *bond)
 
 	return __ffs(bond_mask);
 }
+EXPORT_SYMBOL_GPL(ocelot_bond_get_id);
 
+/* Returns the mask of user ports assigned to this DSA tag_8021q CPU port.
+ * Note that when CPU ports are in a LAG, the user ports are assigned to the
+ * 'primary' CPU port, the one whose physical port number gives the logical
+ * port number of the LAG.
+ *
+ * We leave PGID_SRC poorly configured for the 'secondary' CPU port in the LAG
+ * (to which no user port is assigned), but it appears that forwarding from
+ * this secondary CPU port looks at the PGID_SRC associated with the logical
+ * port ID that it's assigned to, which *is* configured properly.
+ */
 static u32 ocelot_dsa_8021q_cpu_assigned_ports(struct ocelot *ocelot,
 					       struct ocelot_port *cpu)
 {
@@ -2080,9 +1426,15 @@ static u32 ocelot_dsa_8021q_cpu_assigned_ports(struct ocelot *ocelot,
 			mask |= BIT(port);
 	}
 
+	if (cpu->bond)
+		mask &= ~ocelot_get_bond_mask(ocelot, cpu->bond);
+
 	return mask;
 }
 
+/* Returns the DSA tag_8021q CPU port that the given port is assigned to,
+ * or the bit mask of CPU ports if said CPU port is in a LAG.
+ */
 u32 ocelot_port_assigned_dsa_8021q_cpu_mask(struct ocelot *ocelot, int port)
 {
 	struct ocelot_port *ocelot_port = ocelot->ports[port];
@@ -2091,6 +1443,9 @@ u32 ocelot_port_assigned_dsa_8021q_cpu_mask(struct ocelot *ocelot, int port)
 	if (!cpu_port)
 		return 0;
 
+	if (cpu_port->bond)
+		return ocelot_get_bond_mask(ocelot, cpu_port->bond);
+
 	return BIT(cpu_port->index);
 }
 EXPORT_SYMBOL_GPL(ocelot_port_assigned_dsa_8021q_cpu_mask);
@@ -2214,61 +1569,61 @@ static void ocelot_update_pgid_cpu(struct ocelot *ocelot)
 	ocelot_write_rix(ocelot, pgid_cpu, ANA_PGID_PGID, PGID_CPU);
 }
 
-void ocelot_port_assign_dsa_8021q_cpu(struct ocelot *ocelot, int port,
-				      int cpu)
+void ocelot_port_setup_dsa_8021q_cpu(struct ocelot *ocelot, int cpu)
 {
 	struct ocelot_port *cpu_port = ocelot->ports[cpu];
 	u16 vid;
 
 	mutex_lock(&ocelot->fwd_domain_lock);
 
-	ocelot->ports[port]->dsa_8021q_cpu = cpu_port;
-
-	if (!cpu_port->is_dsa_8021q_cpu) {
-		cpu_port->is_dsa_8021q_cpu = true;
+	cpu_port->is_dsa_8021q_cpu = true;
 
-		for (vid = OCELOT_RSV_VLAN_RANGE_START; vid < VLAN_N_VID; vid++)
-			ocelot_vlan_member_add(ocelot, cpu, vid, true);
+	for (vid = OCELOT_RSV_VLAN_RANGE_START; vid < VLAN_N_VID; vid++)
+		ocelot_vlan_member_add(ocelot, cpu, vid, true);
 
-		ocelot_update_pgid_cpu(ocelot);
-	}
-
-	ocelot_apply_bridge_fwd_mask(ocelot, true);
+	ocelot_update_pgid_cpu(ocelot);
 
 	mutex_unlock(&ocelot->fwd_domain_lock);
 }
-EXPORT_SYMBOL_GPL(ocelot_port_assign_dsa_8021q_cpu);
+EXPORT_SYMBOL_GPL(ocelot_port_setup_dsa_8021q_cpu);
 
-void ocelot_port_unassign_dsa_8021q_cpu(struct ocelot *ocelot, int port)
+void ocelot_port_teardown_dsa_8021q_cpu(struct ocelot *ocelot, int cpu)
 {
-	struct ocelot_port *cpu_port = ocelot->ports[port]->dsa_8021q_cpu;
-	bool keep = false;
+	struct ocelot_port *cpu_port = ocelot->ports[cpu];
 	u16 vid;
-	int p;
 
 	mutex_lock(&ocelot->fwd_domain_lock);
 
-	ocelot->ports[port]->dsa_8021q_cpu = NULL;
+	cpu_port->is_dsa_8021q_cpu = false;
 
-	for (p = 0; p < ocelot->num_phys_ports; p++) {
-		if (!ocelot->ports[p])
-			continue;
+	for (vid = OCELOT_RSV_VLAN_RANGE_START; vid < VLAN_N_VID; vid++)
+		ocelot_vlan_member_del(ocelot, cpu_port->index, vid);
 
-		if (ocelot->ports[p]->dsa_8021q_cpu == cpu_port) {
-			keep = true;
-			break;
-		}
-	}
+	ocelot_update_pgid_cpu(ocelot);
 
-	if (!keep) {
-		cpu_port->is_dsa_8021q_cpu = false;
+	mutex_unlock(&ocelot->fwd_domain_lock);
+}
+EXPORT_SYMBOL_GPL(ocelot_port_teardown_dsa_8021q_cpu);
 
-		for (vid = OCELOT_RSV_VLAN_RANGE_START; vid < VLAN_N_VID; vid++)
-			ocelot_vlan_member_del(ocelot, cpu_port->index, vid);
+void ocelot_port_assign_dsa_8021q_cpu(struct ocelot *ocelot, int port,
+				      int cpu)
+{
+	struct ocelot_port *cpu_port = ocelot->ports[cpu];
 
-		ocelot_update_pgid_cpu(ocelot);
-	}
+	mutex_lock(&ocelot->fwd_domain_lock);
+
+	ocelot->ports[port]->dsa_8021q_cpu = cpu_port;
+	ocelot_apply_bridge_fwd_mask(ocelot, true);
+
+	mutex_unlock(&ocelot->fwd_domain_lock);
+}
+EXPORT_SYMBOL_GPL(ocelot_port_assign_dsa_8021q_cpu);
+
+void ocelot_port_unassign_dsa_8021q_cpu(struct ocelot *ocelot, int port)
+{
+	mutex_lock(&ocelot->fwd_domain_lock);
 
+	ocelot->ports[port]->dsa_8021q_cpu = NULL;
 	ocelot_apply_bridge_fwd_mask(ocelot, true);
 
 	mutex_unlock(&ocelot->fwd_domain_lock);
@@ -2785,10 +2140,14 @@ static void ocelot_migrate_lag_fdbs(struct ocelot *ocelot,
 
 int ocelot_port_lag_join(struct ocelot *ocelot, int port,
 			 struct net_device *bond,
-			 struct netdev_lag_upper_info *info)
+			 struct netdev_lag_upper_info *info,
+			 struct netlink_ext_ack *extack)
 {
-	if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH)
+	if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Can only offload LAG using hash TX type");
 		return -EOPNOTSUPP;
+	}
 
 	mutex_lock(&ocelot->fwd_domain_lock);
 
@@ -3358,7 +2717,6 @@ static void ocelot_detect_features(struct ocelot *ocelot)
 
 int ocelot_init(struct ocelot *ocelot)
 {
-	char queue_name[32];
 	int i, ret;
 	u32 port;
 
@@ -3370,29 +2728,21 @@ int ocelot_init(struct ocelot *ocelot)
 		}
 	}
 
-	ocelot->stats = devm_kcalloc(ocelot->dev,
-				     ocelot->num_phys_ports * OCELOT_NUM_STATS,
-				     sizeof(u64), GFP_KERNEL);
-	if (!ocelot->stats)
-		return -ENOMEM;
-
-	spin_lock_init(&ocelot->stats_lock);
 	mutex_init(&ocelot->ptp_lock);
 	mutex_init(&ocelot->mact_lock);
 	mutex_init(&ocelot->fwd_domain_lock);
 	mutex_init(&ocelot->tas_lock);
 	spin_lock_init(&ocelot->ptp_clock_lock);
 	spin_lock_init(&ocelot->ts_id_lock);
-	snprintf(queue_name, sizeof(queue_name), "%s-stats",
-		 dev_name(ocelot->dev));
-	ocelot->stats_queue = create_singlethread_workqueue(queue_name);
-	if (!ocelot->stats_queue)
-		return -ENOMEM;
 
 	ocelot->owq = alloc_ordered_workqueue("ocelot-owq", 0);
-	if (!ocelot->owq) {
-		destroy_workqueue(ocelot->stats_queue);
+	if (!ocelot->owq)
 		return -ENOMEM;
+
+	ret = ocelot_stats_init(ocelot);
+	if (ret) {
+		destroy_workqueue(ocelot->owq);
+		return ret;
 	}
 
 	INIT_LIST_HEAD(&ocelot->multicast);
@@ -3504,25 +2854,13 @@ int ocelot_init(struct ocelot *ocelot)
 				 ANA_CPUQ_8021_CFG_CPUQ_BPDU_VAL(6),
 				 ANA_CPUQ_8021_CFG, i);
 
-	ret = ocelot_prepare_stats_regions(ocelot);
-	if (ret) {
-		destroy_workqueue(ocelot->stats_queue);
-		destroy_workqueue(ocelot->owq);
-		return ret;
-	}
-
-	INIT_DELAYED_WORK(&ocelot->stats_work, ocelot_check_stats_work);
-	queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work,
-			   OCELOT_STATS_CHECK_DELAY);
-
 	return 0;
 }
 EXPORT_SYMBOL(ocelot_init);
 
 void ocelot_deinit(struct ocelot *ocelot)
 {
-	cancel_delayed_work(&ocelot->stats_work);
-	destroy_workqueue(ocelot->stats_queue);
+	ocelot_stats_deinit(ocelot);
 	destroy_workqueue(ocelot->owq);
 }
 EXPORT_SYMBOL(ocelot_deinit);
diff --git a/drivers/net/ethernet/mscc/ocelot.h b/drivers/net/ethernet/mscc/ocelot.h
index 6d65cc87d757..70dbd9c4e512 100644
--- a/drivers/net/ethernet/mscc/ocelot.h
+++ b/drivers/net/ethernet/mscc/ocelot.h
@@ -51,13 +51,6 @@ struct ocelot_port_private {
 	struct ocelot_port_tc tc;
 };
 
-struct ocelot_dump_ctx {
-	struct net_device *dev;
-	struct sk_buff *skb;
-	struct netlink_callback *cb;
-	int idx;
-};
-
 /* A (PGID) port mask structure, encoding the 2^ocelot->num_phys_ports
  * possibilities of egress port masks for L2 multicast traffic.
  * For a switch with 9 user ports, there are 512 possible port masks, but the
@@ -84,8 +77,6 @@ struct ocelot_multicast {
 int ocelot_bridge_num_find(struct ocelot *ocelot,
 			   const struct net_device *bridge);
 
-int ocelot_port_fdb_do_dump(const unsigned char *addr, u16 vid,
-			    bool is_static, void *data);
 int ocelot_mact_learn(struct ocelot *ocelot, int port,
 		      const unsigned char mac[ETH_ALEN],
 		      unsigned int vid, enum macaccess_entry_type type);
@@ -115,6 +106,9 @@ struct ocelot_mirror *ocelot_mirror_get(struct ocelot *ocelot, int to,
 					struct netlink_ext_ack *extack);
 void ocelot_mirror_put(struct ocelot *ocelot);
 
+int ocelot_stats_init(struct ocelot *ocelot);
+void ocelot_stats_deinit(struct ocelot *ocelot);
+
 extern struct notifier_block ocelot_netdevice_nb;
 extern struct notifier_block ocelot_switchdev_nb;
 extern struct notifier_block ocelot_switchdev_blocking_nb;
diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c
index 330d30841cdc..50858cc10fef 100644
--- a/drivers/net/ethernet/mscc/ocelot_net.c
+++ b/drivers/net/ethernet/mscc/ocelot_net.c
@@ -20,6 +20,13 @@
 
 #define OCELOT_MAC_QUIRKS	OCELOT_QUIRK_QSGMII_PORTS_MUST_BE_UP
 
+struct ocelot_dump_ctx {
+	struct net_device *dev;
+	struct sk_buff *skb;
+	struct netlink_callback *cb;
+	int idx;
+};
+
 static bool ocelot_netdevice_dev_check(const struct net_device *dev);
 
 static struct ocelot *devlink_port_to_ocelot(struct devlink_port *dlp)
@@ -725,42 +732,8 @@ static void ocelot_get_stats64(struct net_device *dev,
 	struct ocelot_port_private *priv = netdev_priv(dev);
 	struct ocelot *ocelot = priv->port.ocelot;
 	int port = priv->port.index;
-	u64 *s;
-
-	spin_lock(&ocelot->stats_lock);
-
-	s = &ocelot->stats[port * OCELOT_NUM_STATS];
-
-	/* Get Rx stats */
-	stats->rx_bytes = s[OCELOT_STAT_RX_OCTETS];
-	stats->rx_packets = s[OCELOT_STAT_RX_SHORTS] +
-			    s[OCELOT_STAT_RX_FRAGMENTS] +
-			    s[OCELOT_STAT_RX_JABBERS] +
-			    s[OCELOT_STAT_RX_LONGS] +
-			    s[OCELOT_STAT_RX_64] +
-			    s[OCELOT_STAT_RX_65_127] +
-			    s[OCELOT_STAT_RX_128_255] +
-			    s[OCELOT_STAT_RX_256_511] +
-			    s[OCELOT_STAT_RX_512_1023] +
-			    s[OCELOT_STAT_RX_1024_1526] +
-			    s[OCELOT_STAT_RX_1527_MAX];
-	stats->multicast = s[OCELOT_STAT_RX_MULTICAST];
-	stats->rx_dropped = dev->stats.rx_dropped;
-
-	/* Get Tx stats */
-	stats->tx_bytes = s[OCELOT_STAT_TX_OCTETS];
-	stats->tx_packets = s[OCELOT_STAT_TX_64] +
-			    s[OCELOT_STAT_TX_65_127] +
-			    s[OCELOT_STAT_TX_128_255] +
-			    s[OCELOT_STAT_TX_256_511] +
-			    s[OCELOT_STAT_TX_512_1023] +
-			    s[OCELOT_STAT_TX_1024_1526] +
-			    s[OCELOT_STAT_TX_1527_MAX];
-	stats->tx_dropped = s[OCELOT_STAT_TX_DROPS] +
-			    s[OCELOT_STAT_TX_AGED];
-	stats->collisions = s[OCELOT_STAT_TX_COLLISION];
-
-	spin_unlock(&ocelot->stats_lock);
+
+	return ocelot_port_get_stats64(ocelot, port, stats);
 }
 
 static int ocelot_port_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
@@ -790,6 +763,49 @@ static int ocelot_port_fdb_del(struct ndmsg *ndm, struct nlattr *tb[],
 	return ocelot_fdb_del(ocelot, port, addr, vid, ocelot_port->bridge);
 }
 
+static int ocelot_port_fdb_do_dump(const unsigned char *addr, u16 vid,
+				   bool is_static, void *data)
+{
+	struct ocelot_dump_ctx *dump = data;
+	u32 portid = NETLINK_CB(dump->cb->skb).portid;
+	u32 seq = dump->cb->nlh->nlmsg_seq;
+	struct nlmsghdr *nlh;
+	struct ndmsg *ndm;
+
+	if (dump->idx < dump->cb->args[2])
+		goto skip;
+
+	nlh = nlmsg_put(dump->skb, portid, seq, RTM_NEWNEIGH,
+			sizeof(*ndm), NLM_F_MULTI);
+	if (!nlh)
+		return -EMSGSIZE;
+
+	ndm = nlmsg_data(nlh);
+	ndm->ndm_family  = AF_BRIDGE;
+	ndm->ndm_pad1    = 0;
+	ndm->ndm_pad2    = 0;
+	ndm->ndm_flags   = NTF_SELF;
+	ndm->ndm_type    = 0;
+	ndm->ndm_ifindex = dump->dev->ifindex;
+	ndm->ndm_state   = is_static ? NUD_NOARP : NUD_REACHABLE;
+
+	if (nla_put(dump->skb, NDA_LLADDR, ETH_ALEN, addr))
+		goto nla_put_failure;
+
+	if (vid && nla_put_u16(dump->skb, NDA_VLAN, vid))
+		goto nla_put_failure;
+
+	nlmsg_end(dump->skb, nlh);
+
+skip:
+	dump->idx++;
+	return 0;
+
+nla_put_failure:
+	nlmsg_cancel(dump->skb, nlh);
+	return -EMSGSIZE;
+}
+
 static int ocelot_port_fdb_dump(struct sk_buff *skb,
 				struct netlink_callback *cb,
 				struct net_device *dev,
@@ -1396,11 +1412,10 @@ static int ocelot_netdevice_lag_join(struct net_device *dev,
 	int port = priv->port.index;
 	int err;
 
-	err = ocelot_port_lag_join(ocelot, port, bond, info);
-	if (err == -EOPNOTSUPP) {
-		NL_SET_ERR_MSG_MOD(extack, "Offloading not supported");
+	err = ocelot_port_lag_join(ocelot, port, bond, info, extack);
+	if (err == -EOPNOTSUPP)
+		/* Offloading not supported, fall back to software LAG */
 		return 0;
-	}
 
 	bridge_dev = netdev_master_upper_dev_get(bond);
 	if (!bridge_dev || !netif_is_bridge_master(bridge_dev))
diff --git a/drivers/net/ethernet/mscc/ocelot_ptp.c b/drivers/net/ethernet/mscc/ocelot_ptp.c
index 09c703efe946..1a82f10c8853 100644
--- a/drivers/net/ethernet/mscc/ocelot_ptp.c
+++ b/drivers/net/ethernet/mscc/ocelot_ptp.c
@@ -6,9 +6,13 @@
  */
 #include <linux/time64.h>
 
+#include <linux/dsa/ocelot.h>
+#include <linux/ptp_classify.h>
 #include <soc/mscc/ocelot_ptp.h>
 #include <soc/mscc/ocelot_sys.h>
+#include <soc/mscc/ocelot_vcap.h>
 #include <soc/mscc/ocelot.h>
+#include "ocelot.h"
 
 int ocelot_ptp_gettime64(struct ptp_clock_info *ptp, struct timespec64 *ts)
 {
@@ -310,6 +314,483 @@ int ocelot_ptp_enable(struct ptp_clock_info *ptp,
 }
 EXPORT_SYMBOL(ocelot_ptp_enable);
 
+static void ocelot_populate_l2_ptp_trap_key(struct ocelot_vcap_filter *trap)
+{
+	trap->key_type = OCELOT_VCAP_KEY_ETYPE;
+	*(__be16 *)trap->key.etype.etype.value = htons(ETH_P_1588);
+	*(__be16 *)trap->key.etype.etype.mask = htons(0xffff);
+}
+
+static void
+ocelot_populate_ipv4_ptp_event_trap_key(struct ocelot_vcap_filter *trap)
+{
+	trap->key_type = OCELOT_VCAP_KEY_IPV4;
+	trap->key.ipv4.proto.value[0] = IPPROTO_UDP;
+	trap->key.ipv4.proto.mask[0] = 0xff;
+	trap->key.ipv4.dport.value = PTP_EV_PORT;
+	trap->key.ipv4.dport.mask = 0xffff;
+}
+
+static void
+ocelot_populate_ipv6_ptp_event_trap_key(struct ocelot_vcap_filter *trap)
+{
+	trap->key_type = OCELOT_VCAP_KEY_IPV6;
+	trap->key.ipv4.proto.value[0] = IPPROTO_UDP;
+	trap->key.ipv4.proto.mask[0] = 0xff;
+	trap->key.ipv6.dport.value = PTP_EV_PORT;
+	trap->key.ipv6.dport.mask = 0xffff;
+}
+
+static void
+ocelot_populate_ipv4_ptp_general_trap_key(struct ocelot_vcap_filter *trap)
+{
+	trap->key_type = OCELOT_VCAP_KEY_IPV4;
+	trap->key.ipv4.proto.value[0] = IPPROTO_UDP;
+	trap->key.ipv4.proto.mask[0] = 0xff;
+	trap->key.ipv4.dport.value = PTP_GEN_PORT;
+	trap->key.ipv4.dport.mask = 0xffff;
+}
+
+static void
+ocelot_populate_ipv6_ptp_general_trap_key(struct ocelot_vcap_filter *trap)
+{
+	trap->key_type = OCELOT_VCAP_KEY_IPV6;
+	trap->key.ipv4.proto.value[0] = IPPROTO_UDP;
+	trap->key.ipv4.proto.mask[0] = 0xff;
+	trap->key.ipv6.dport.value = PTP_GEN_PORT;
+	trap->key.ipv6.dport.mask = 0xffff;
+}
+
+static int ocelot_l2_ptp_trap_add(struct ocelot *ocelot, int port)
+{
+	unsigned long l2_cookie = OCELOT_VCAP_IS2_L2_PTP_TRAP(ocelot);
+
+	return ocelot_trap_add(ocelot, port, l2_cookie, true,
+			       ocelot_populate_l2_ptp_trap_key);
+}
+
+static int ocelot_l2_ptp_trap_del(struct ocelot *ocelot, int port)
+{
+	unsigned long l2_cookie = OCELOT_VCAP_IS2_L2_PTP_TRAP(ocelot);
+
+	return ocelot_trap_del(ocelot, port, l2_cookie);
+}
+
+static int ocelot_ipv4_ptp_trap_add(struct ocelot *ocelot, int port)
+{
+	unsigned long ipv4_gen_cookie = OCELOT_VCAP_IS2_IPV4_GEN_PTP_TRAP(ocelot);
+	unsigned long ipv4_ev_cookie = OCELOT_VCAP_IS2_IPV4_EV_PTP_TRAP(ocelot);
+	int err;
+
+	err = ocelot_trap_add(ocelot, port, ipv4_ev_cookie, true,
+			      ocelot_populate_ipv4_ptp_event_trap_key);
+	if (err)
+		return err;
+
+	err = ocelot_trap_add(ocelot, port, ipv4_gen_cookie, false,
+			      ocelot_populate_ipv4_ptp_general_trap_key);
+	if (err)
+		ocelot_trap_del(ocelot, port, ipv4_ev_cookie);
+
+	return err;
+}
+
+static int ocelot_ipv4_ptp_trap_del(struct ocelot *ocelot, int port)
+{
+	unsigned long ipv4_gen_cookie = OCELOT_VCAP_IS2_IPV4_GEN_PTP_TRAP(ocelot);
+	unsigned long ipv4_ev_cookie = OCELOT_VCAP_IS2_IPV4_EV_PTP_TRAP(ocelot);
+	int err;
+
+	err = ocelot_trap_del(ocelot, port, ipv4_ev_cookie);
+	err |= ocelot_trap_del(ocelot, port, ipv4_gen_cookie);
+	return err;
+}
+
+static int ocelot_ipv6_ptp_trap_add(struct ocelot *ocelot, int port)
+{
+	unsigned long ipv6_gen_cookie = OCELOT_VCAP_IS2_IPV6_GEN_PTP_TRAP(ocelot);
+	unsigned long ipv6_ev_cookie = OCELOT_VCAP_IS2_IPV6_EV_PTP_TRAP(ocelot);
+	int err;
+
+	err = ocelot_trap_add(ocelot, port, ipv6_ev_cookie, true,
+			      ocelot_populate_ipv6_ptp_event_trap_key);
+	if (err)
+		return err;
+
+	err = ocelot_trap_add(ocelot, port, ipv6_gen_cookie, false,
+			      ocelot_populate_ipv6_ptp_general_trap_key);
+	if (err)
+		ocelot_trap_del(ocelot, port, ipv6_ev_cookie);
+
+	return err;
+}
+
+static int ocelot_ipv6_ptp_trap_del(struct ocelot *ocelot, int port)
+{
+	unsigned long ipv6_gen_cookie = OCELOT_VCAP_IS2_IPV6_GEN_PTP_TRAP(ocelot);
+	unsigned long ipv6_ev_cookie = OCELOT_VCAP_IS2_IPV6_EV_PTP_TRAP(ocelot);
+	int err;
+
+	err = ocelot_trap_del(ocelot, port, ipv6_ev_cookie);
+	err |= ocelot_trap_del(ocelot, port, ipv6_gen_cookie);
+	return err;
+}
+
+static int ocelot_setup_ptp_traps(struct ocelot *ocelot, int port,
+				  bool l2, bool l4)
+{
+	int err;
+
+	if (l2)
+		err = ocelot_l2_ptp_trap_add(ocelot, port);
+	else
+		err = ocelot_l2_ptp_trap_del(ocelot, port);
+	if (err)
+		return err;
+
+	if (l4) {
+		err = ocelot_ipv4_ptp_trap_add(ocelot, port);
+		if (err)
+			goto err_ipv4;
+
+		err = ocelot_ipv6_ptp_trap_add(ocelot, port);
+		if (err)
+			goto err_ipv6;
+	} else {
+		err = ocelot_ipv4_ptp_trap_del(ocelot, port);
+
+		err |= ocelot_ipv6_ptp_trap_del(ocelot, port);
+	}
+	if (err)
+		return err;
+
+	return 0;
+
+err_ipv6:
+	ocelot_ipv4_ptp_trap_del(ocelot, port);
+err_ipv4:
+	if (l2)
+		ocelot_l2_ptp_trap_del(ocelot, port);
+	return err;
+}
+
+int ocelot_hwstamp_get(struct ocelot *ocelot, int port, struct ifreq *ifr)
+{
+	return copy_to_user(ifr->ifr_data, &ocelot->hwtstamp_config,
+			    sizeof(ocelot->hwtstamp_config)) ? -EFAULT : 0;
+}
+EXPORT_SYMBOL(ocelot_hwstamp_get);
+
+int ocelot_hwstamp_set(struct ocelot *ocelot, int port, struct ifreq *ifr)
+{
+	struct ocelot_port *ocelot_port = ocelot->ports[port];
+	bool l2 = false, l4 = false;
+	struct hwtstamp_config cfg;
+	int err;
+
+	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
+		return -EFAULT;
+
+	/* Tx type sanity check */
+	switch (cfg.tx_type) {
+	case HWTSTAMP_TX_ON:
+		ocelot_port->ptp_cmd = IFH_REW_OP_TWO_STEP_PTP;
+		break;
+	case HWTSTAMP_TX_ONESTEP_SYNC:
+		/* IFH_REW_OP_ONE_STEP_PTP updates the correctional field, we
+		 * need to update the origin time.
+		 */
+		ocelot_port->ptp_cmd = IFH_REW_OP_ORIGIN_PTP;
+		break;
+	case HWTSTAMP_TX_OFF:
+		ocelot_port->ptp_cmd = 0;
+		break;
+	default:
+		return -ERANGE;
+	}
+
+	mutex_lock(&ocelot->ptp_lock);
+
+	switch (cfg.rx_filter) {
+	case HWTSTAMP_FILTER_NONE:
+		break;
+	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
+	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
+	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
+		l4 = true;
+		break;
+	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
+	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
+	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
+		l2 = true;
+		break;
+	case HWTSTAMP_FILTER_PTP_V2_EVENT:
+	case HWTSTAMP_FILTER_PTP_V2_SYNC:
+	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
+		l2 = true;
+		l4 = true;
+		break;
+	default:
+		mutex_unlock(&ocelot->ptp_lock);
+		return -ERANGE;
+	}
+
+	err = ocelot_setup_ptp_traps(ocelot, port, l2, l4);
+	if (err) {
+		mutex_unlock(&ocelot->ptp_lock);
+		return err;
+	}
+
+	if (l2 && l4)
+		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
+	else if (l2)
+		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
+	else if (l4)
+		cfg.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
+	else
+		cfg.rx_filter = HWTSTAMP_FILTER_NONE;
+
+	/* Commit back the result & save it */
+	memcpy(&ocelot->hwtstamp_config, &cfg, sizeof(cfg));
+	mutex_unlock(&ocelot->ptp_lock);
+
+	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
+}
+EXPORT_SYMBOL(ocelot_hwstamp_set);
+
+int ocelot_get_ts_info(struct ocelot *ocelot, int port,
+		       struct ethtool_ts_info *info)
+{
+	info->phc_index = ocelot->ptp_clock ?
+			  ptp_clock_index(ocelot->ptp_clock) : -1;
+	if (info->phc_index == -1) {
+		info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE |
+					 SOF_TIMESTAMPING_RX_SOFTWARE |
+					 SOF_TIMESTAMPING_SOFTWARE;
+		return 0;
+	}
+	info->so_timestamping |= SOF_TIMESTAMPING_TX_SOFTWARE |
+				 SOF_TIMESTAMPING_RX_SOFTWARE |
+				 SOF_TIMESTAMPING_SOFTWARE |
+				 SOF_TIMESTAMPING_TX_HARDWARE |
+				 SOF_TIMESTAMPING_RX_HARDWARE |
+				 SOF_TIMESTAMPING_RAW_HARDWARE;
+	info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON) |
+			 BIT(HWTSTAMP_TX_ONESTEP_SYNC);
+	info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) |
+			   BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) |
+			   BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) |
+			   BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT);
+
+	return 0;
+}
+EXPORT_SYMBOL(ocelot_get_ts_info);
+
+static int ocelot_port_add_txtstamp_skb(struct ocelot *ocelot, int port,
+					struct sk_buff *clone)
+{
+	struct ocelot_port *ocelot_port = ocelot->ports[port];
+	unsigned long flags;
+
+	spin_lock_irqsave(&ocelot->ts_id_lock, flags);
+
+	if (ocelot_port->ptp_skbs_in_flight == OCELOT_MAX_PTP_ID ||
+	    ocelot->ptp_skbs_in_flight == OCELOT_PTP_FIFO_SIZE) {
+		spin_unlock_irqrestore(&ocelot->ts_id_lock, flags);
+		return -EBUSY;
+	}
+
+	skb_shinfo(clone)->tx_flags |= SKBTX_IN_PROGRESS;
+	/* Store timestamp ID in OCELOT_SKB_CB(clone)->ts_id */
+	OCELOT_SKB_CB(clone)->ts_id = ocelot_port->ts_id;
+
+	ocelot_port->ts_id++;
+	if (ocelot_port->ts_id == OCELOT_MAX_PTP_ID)
+		ocelot_port->ts_id = 0;
+
+	ocelot_port->ptp_skbs_in_flight++;
+	ocelot->ptp_skbs_in_flight++;
+
+	skb_queue_tail(&ocelot_port->tx_skbs, clone);
+
+	spin_unlock_irqrestore(&ocelot->ts_id_lock, flags);
+
+	return 0;
+}
+
+static bool ocelot_ptp_is_onestep_sync(struct sk_buff *skb,
+				       unsigned int ptp_class)
+{
+	struct ptp_header *hdr;
+	u8 msgtype, twostep;
+
+	hdr = ptp_parse_header(skb, ptp_class);
+	if (!hdr)
+		return false;
+
+	msgtype = ptp_get_msgtype(hdr, ptp_class);
+	twostep = hdr->flag_field[0] & 0x2;
+
+	if (msgtype == PTP_MSGTYPE_SYNC && twostep == 0)
+		return true;
+
+	return false;
+}
+
+int ocelot_port_txtstamp_request(struct ocelot *ocelot, int port,
+				 struct sk_buff *skb,
+				 struct sk_buff **clone)
+{
+	struct ocelot_port *ocelot_port = ocelot->ports[port];
+	u8 ptp_cmd = ocelot_port->ptp_cmd;
+	unsigned int ptp_class;
+	int err;
+
+	/* Don't do anything if PTP timestamping not enabled */
+	if (!ptp_cmd)
+		return 0;
+
+	ptp_class = ptp_classify_raw(skb);
+	if (ptp_class == PTP_CLASS_NONE)
+		return -EINVAL;
+
+	/* Store ptp_cmd in OCELOT_SKB_CB(skb)->ptp_cmd */
+	if (ptp_cmd == IFH_REW_OP_ORIGIN_PTP) {
+		if (ocelot_ptp_is_onestep_sync(skb, ptp_class)) {
+			OCELOT_SKB_CB(skb)->ptp_cmd = ptp_cmd;
+			return 0;
+		}
+
+		/* Fall back to two-step timestamping */
+		ptp_cmd = IFH_REW_OP_TWO_STEP_PTP;
+	}
+
+	if (ptp_cmd == IFH_REW_OP_TWO_STEP_PTP) {
+		*clone = skb_clone_sk(skb);
+		if (!(*clone))
+			return -ENOMEM;
+
+		err = ocelot_port_add_txtstamp_skb(ocelot, port, *clone);
+		if (err)
+			return err;
+
+		OCELOT_SKB_CB(skb)->ptp_cmd = ptp_cmd;
+		OCELOT_SKB_CB(*clone)->ptp_class = ptp_class;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(ocelot_port_txtstamp_request);
+
+static void ocelot_get_hwtimestamp(struct ocelot *ocelot,
+				   struct timespec64 *ts)
+{
+	unsigned long flags;
+	u32 val;
+
+	spin_lock_irqsave(&ocelot->ptp_clock_lock, flags);
+
+	/* Read current PTP time to get seconds */
+	val = ocelot_read_rix(ocelot, PTP_PIN_CFG, TOD_ACC_PIN);
+
+	val &= ~(PTP_PIN_CFG_SYNC | PTP_PIN_CFG_ACTION_MASK | PTP_PIN_CFG_DOM);
+	val |= PTP_PIN_CFG_ACTION(PTP_PIN_ACTION_SAVE);
+	ocelot_write_rix(ocelot, val, PTP_PIN_CFG, TOD_ACC_PIN);
+	ts->tv_sec = ocelot_read_rix(ocelot, PTP_PIN_TOD_SEC_LSB, TOD_ACC_PIN);
+
+	/* Read packet HW timestamp from FIFO */
+	val = ocelot_read(ocelot, SYS_PTP_TXSTAMP);
+	ts->tv_nsec = SYS_PTP_TXSTAMP_PTP_TXSTAMP(val);
+
+	/* Sec has incremented since the ts was registered */
+	if ((ts->tv_sec & 0x1) != !!(val & SYS_PTP_TXSTAMP_PTP_TXSTAMP_SEC))
+		ts->tv_sec--;
+
+	spin_unlock_irqrestore(&ocelot->ptp_clock_lock, flags);
+}
+
+static bool ocelot_validate_ptp_skb(struct sk_buff *clone, u16 seqid)
+{
+	struct ptp_header *hdr;
+
+	hdr = ptp_parse_header(clone, OCELOT_SKB_CB(clone)->ptp_class);
+	if (WARN_ON(!hdr))
+		return false;
+
+	return seqid == ntohs(hdr->sequence_id);
+}
+
+void ocelot_get_txtstamp(struct ocelot *ocelot)
+{
+	int budget = OCELOT_PTP_QUEUE_SZ;
+
+	while (budget--) {
+		struct sk_buff *skb, *skb_tmp, *skb_match = NULL;
+		struct skb_shared_hwtstamps shhwtstamps;
+		u32 val, id, seqid, txport;
+		struct ocelot_port *port;
+		struct timespec64 ts;
+		unsigned long flags;
+
+		val = ocelot_read(ocelot, SYS_PTP_STATUS);
+
+		/* Check if a timestamp can be retrieved */
+		if (!(val & SYS_PTP_STATUS_PTP_MESS_VLD))
+			break;
+
+		WARN_ON(val & SYS_PTP_STATUS_PTP_OVFL);
+
+		/* Retrieve the ts ID and Tx port */
+		id = SYS_PTP_STATUS_PTP_MESS_ID_X(val);
+		txport = SYS_PTP_STATUS_PTP_MESS_TXPORT_X(val);
+		seqid = SYS_PTP_STATUS_PTP_MESS_SEQ_ID(val);
+
+		port = ocelot->ports[txport];
+
+		spin_lock(&ocelot->ts_id_lock);
+		port->ptp_skbs_in_flight--;
+		ocelot->ptp_skbs_in_flight--;
+		spin_unlock(&ocelot->ts_id_lock);
+
+		/* Retrieve its associated skb */
+try_again:
+		spin_lock_irqsave(&port->tx_skbs.lock, flags);
+
+		skb_queue_walk_safe(&port->tx_skbs, skb, skb_tmp) {
+			if (OCELOT_SKB_CB(skb)->ts_id != id)
+				continue;
+			__skb_unlink(skb, &port->tx_skbs);
+			skb_match = skb;
+			break;
+		}
+
+		spin_unlock_irqrestore(&port->tx_skbs.lock, flags);
+
+		if (WARN_ON(!skb_match))
+			continue;
+
+		if (!ocelot_validate_ptp_skb(skb_match, seqid)) {
+			dev_err_ratelimited(ocelot->dev,
+					    "port %d received stale TX timestamp for seqid %d, discarding\n",
+					    txport, seqid);
+			dev_kfree_skb_any(skb);
+			goto try_again;
+		}
+
+		/* Get the h/w timestamp */
+		ocelot_get_hwtimestamp(ocelot, &ts);
+
+		/* Set the timestamp into the skb */
+		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
+		shhwtstamps.hwtstamp = ktime_set(ts.tv_sec, ts.tv_nsec);
+		skb_complete_tx_timestamp(skb_match, &shhwtstamps);
+
+		/* Next ts */
+		ocelot_write(ocelot, SYS_PTP_NXT_PTP_NXT, SYS_PTP_NXT);
+	}
+}
+EXPORT_SYMBOL(ocelot_get_txtstamp);
+
 int ocelot_init_timestamp(struct ocelot *ocelot,
 			  const struct ptp_clock_info *info)
 {
diff --git a/drivers/net/ethernet/mscc/ocelot_stats.c b/drivers/net/ethernet/mscc/ocelot_stats.c
new file mode 100644
index 000000000000..dbd20b125cea
--- /dev/null
+++ b/drivers/net/ethernet/mscc/ocelot_stats.c
@@ -0,0 +1,458 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/* Statistics for Ocelot switch family
+ *
+ * Copyright (c) 2017 Microsemi Corporation
+ * Copyright 2022 NXP
+ */
+#include <linux/spinlock.h>
+#include <linux/mutex.h>
+#include <linux/workqueue.h>
+#include "ocelot.h"
+
+/* Read the counters from hardware and keep them in region->buf.
+ * Caller must hold &ocelot->stat_view_lock.
+ */
+static int ocelot_port_update_stats(struct ocelot *ocelot, int port)
+{
+	struct ocelot_stats_region *region;
+	int err;
+
+	/* Configure the port to read the stats from */
+	ocelot_write(ocelot, SYS_STAT_CFG_STAT_VIEW(port), SYS_STAT_CFG);
+
+	list_for_each_entry(region, &ocelot->stats_regions, node) {
+		err = ocelot_bulk_read(ocelot, region->base, region->buf,
+				       region->count);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+/* Transfer the counters from region->buf to ocelot->stats.
+ * Caller must hold &ocelot->stat_view_lock and &ocelot->stats_lock.
+ */
+static void ocelot_port_transfer_stats(struct ocelot *ocelot, int port)
+{
+	unsigned int idx = port * OCELOT_NUM_STATS;
+	struct ocelot_stats_region *region;
+	int j;
+
+	list_for_each_entry(region, &ocelot->stats_regions, node) {
+		for (j = 0; j < region->count; j++) {
+			u64 *stat = &ocelot->stats[idx + j];
+			u64 val = region->buf[j];
+
+			if (val < (*stat & U32_MAX))
+				*stat += (u64)1 << 32;
+
+			*stat = (*stat & ~(u64)U32_MAX) + val;
+		}
+
+		idx += region->count;
+	}
+}
+
+static void ocelot_check_stats_work(struct work_struct *work)
+{
+	struct delayed_work *del_work = to_delayed_work(work);
+	struct ocelot *ocelot = container_of(del_work, struct ocelot,
+					     stats_work);
+	int port, err;
+
+	mutex_lock(&ocelot->stat_view_lock);
+
+	for (port = 0; port < ocelot->num_phys_ports; port++) {
+		err = ocelot_port_update_stats(ocelot, port);
+		if (err)
+			break;
+
+		spin_lock(&ocelot->stats_lock);
+		ocelot_port_transfer_stats(ocelot, port);
+		spin_unlock(&ocelot->stats_lock);
+	}
+
+	if (!err && ocelot->ops->update_stats)
+		ocelot->ops->update_stats(ocelot);
+
+	mutex_unlock(&ocelot->stat_view_lock);
+
+	if (err)
+		dev_err(ocelot->dev, "Error %d updating ethtool stats\n",  err);
+
+	queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work,
+			   OCELOT_STATS_CHECK_DELAY);
+}
+
+void ocelot_get_strings(struct ocelot *ocelot, int port, u32 sset, u8 *data)
+{
+	int i;
+
+	if (sset != ETH_SS_STATS)
+		return;
+
+	for (i = 0; i < OCELOT_NUM_STATS; i++) {
+		if (ocelot->stats_layout[i].name[0] == '\0')
+			continue;
+
+		memcpy(data + i * ETH_GSTRING_LEN, ocelot->stats_layout[i].name,
+		       ETH_GSTRING_LEN);
+	}
+}
+EXPORT_SYMBOL(ocelot_get_strings);
+
+/* Update ocelot->stats for the given port and run the given callback */
+static void ocelot_port_stats_run(struct ocelot *ocelot, int port, void *priv,
+				  void (*cb)(struct ocelot *ocelot, int port,
+					     void *priv))
+{
+	int err;
+
+	mutex_lock(&ocelot->stat_view_lock);
+
+	err = ocelot_port_update_stats(ocelot, port);
+	if (err) {
+		dev_err(ocelot->dev, "Failed to update port %d stats: %pe\n",
+			port, ERR_PTR(err));
+		goto out_unlock;
+	}
+
+	spin_lock(&ocelot->stats_lock);
+
+	ocelot_port_transfer_stats(ocelot, port);
+	cb(ocelot, port, priv);
+
+	spin_unlock(&ocelot->stats_lock);
+
+out_unlock:
+	mutex_unlock(&ocelot->stat_view_lock);
+}
+
+int ocelot_get_sset_count(struct ocelot *ocelot, int port, int sset)
+{
+	int i, num_stats = 0;
+
+	if (sset != ETH_SS_STATS)
+		return -EOPNOTSUPP;
+
+	for (i = 0; i < OCELOT_NUM_STATS; i++)
+		if (ocelot->stats_layout[i].name[0] != '\0')
+			num_stats++;
+
+	return num_stats;
+}
+EXPORT_SYMBOL(ocelot_get_sset_count);
+
+static void ocelot_port_ethtool_stats_cb(struct ocelot *ocelot, int port,
+					 void *priv)
+{
+	u64 *data = priv;
+	int i;
+
+	/* Copy all supported counters */
+	for (i = 0; i < OCELOT_NUM_STATS; i++) {
+		int index = port * OCELOT_NUM_STATS + i;
+
+		if (ocelot->stats_layout[i].name[0] == '\0')
+			continue;
+
+		*data++ = ocelot->stats[index];
+	}
+}
+
+void ocelot_get_ethtool_stats(struct ocelot *ocelot, int port, u64 *data)
+{
+	ocelot_port_stats_run(ocelot, port, data, ocelot_port_ethtool_stats_cb);
+}
+EXPORT_SYMBOL(ocelot_get_ethtool_stats);
+
+static void ocelot_port_pause_stats_cb(struct ocelot *ocelot, int port, void *priv)
+{
+	u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS];
+	struct ethtool_pause_stats *pause_stats = priv;
+
+	pause_stats->tx_pause_frames = s[OCELOT_STAT_TX_PAUSE];
+	pause_stats->rx_pause_frames = s[OCELOT_STAT_RX_PAUSE];
+}
+
+void ocelot_port_get_pause_stats(struct ocelot *ocelot, int port,
+				 struct ethtool_pause_stats *pause_stats)
+{
+	ocelot_port_stats_run(ocelot, port, pause_stats,
+			      ocelot_port_pause_stats_cb);
+}
+EXPORT_SYMBOL_GPL(ocelot_port_get_pause_stats);
+
+static const struct ethtool_rmon_hist_range ocelot_rmon_ranges[] = {
+	{   64,    64 },
+	{   65,   127 },
+	{  128,   255 },
+	{  256,   511 },
+	{  512,  1023 },
+	{ 1024,  1526 },
+	{ 1527, 65535 },
+	{},
+};
+
+static void ocelot_port_rmon_stats_cb(struct ocelot *ocelot, int port, void *priv)
+{
+	u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS];
+	struct ethtool_rmon_stats *rmon_stats = priv;
+
+	rmon_stats->undersize_pkts = s[OCELOT_STAT_RX_SHORTS];
+	rmon_stats->oversize_pkts = s[OCELOT_STAT_RX_LONGS];
+	rmon_stats->fragments = s[OCELOT_STAT_RX_FRAGMENTS];
+	rmon_stats->jabbers = s[OCELOT_STAT_RX_JABBERS];
+
+	rmon_stats->hist[0] = s[OCELOT_STAT_RX_64];
+	rmon_stats->hist[1] = s[OCELOT_STAT_RX_65_127];
+	rmon_stats->hist[2] = s[OCELOT_STAT_RX_128_255];
+	rmon_stats->hist[3] = s[OCELOT_STAT_RX_256_511];
+	rmon_stats->hist[4] = s[OCELOT_STAT_RX_512_1023];
+	rmon_stats->hist[5] = s[OCELOT_STAT_RX_1024_1526];
+	rmon_stats->hist[6] = s[OCELOT_STAT_RX_1527_MAX];
+
+	rmon_stats->hist_tx[0] = s[OCELOT_STAT_TX_64];
+	rmon_stats->hist_tx[1] = s[OCELOT_STAT_TX_65_127];
+	rmon_stats->hist_tx[2] = s[OCELOT_STAT_TX_128_255];
+	rmon_stats->hist_tx[3] = s[OCELOT_STAT_TX_128_255];
+	rmon_stats->hist_tx[4] = s[OCELOT_STAT_TX_256_511];
+	rmon_stats->hist_tx[5] = s[OCELOT_STAT_TX_512_1023];
+	rmon_stats->hist_tx[6] = s[OCELOT_STAT_TX_1024_1526];
+}
+
+void ocelot_port_get_rmon_stats(struct ocelot *ocelot, int port,
+				struct ethtool_rmon_stats *rmon_stats,
+				const struct ethtool_rmon_hist_range **ranges)
+{
+	*ranges = ocelot_rmon_ranges;
+
+	ocelot_port_stats_run(ocelot, port, rmon_stats,
+			      ocelot_port_rmon_stats_cb);
+}
+EXPORT_SYMBOL_GPL(ocelot_port_get_rmon_stats);
+
+static void ocelot_port_ctrl_stats_cb(struct ocelot *ocelot, int port, void *priv)
+{
+	u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS];
+	struct ethtool_eth_ctrl_stats *ctrl_stats = priv;
+
+	ctrl_stats->MACControlFramesReceived = s[OCELOT_STAT_RX_CONTROL];
+}
+
+void ocelot_port_get_eth_ctrl_stats(struct ocelot *ocelot, int port,
+				    struct ethtool_eth_ctrl_stats *ctrl_stats)
+{
+	ocelot_port_stats_run(ocelot, port, ctrl_stats,
+			      ocelot_port_ctrl_stats_cb);
+}
+EXPORT_SYMBOL_GPL(ocelot_port_get_eth_ctrl_stats);
+
+static void ocelot_port_mac_stats_cb(struct ocelot *ocelot, int port, void *priv)
+{
+	u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS];
+	struct ethtool_eth_mac_stats *mac_stats = priv;
+
+	mac_stats->OctetsTransmittedOK = s[OCELOT_STAT_TX_OCTETS];
+	mac_stats->FramesTransmittedOK = s[OCELOT_STAT_TX_64] +
+					 s[OCELOT_STAT_TX_65_127] +
+					 s[OCELOT_STAT_TX_128_255] +
+					 s[OCELOT_STAT_TX_256_511] +
+					 s[OCELOT_STAT_TX_512_1023] +
+					 s[OCELOT_STAT_TX_1024_1526] +
+					 s[OCELOT_STAT_TX_1527_MAX];
+	mac_stats->OctetsReceivedOK = s[OCELOT_STAT_RX_OCTETS];
+	mac_stats->FramesReceivedOK = s[OCELOT_STAT_RX_GREEN_PRIO_0] +
+				      s[OCELOT_STAT_RX_GREEN_PRIO_1] +
+				      s[OCELOT_STAT_RX_GREEN_PRIO_2] +
+				      s[OCELOT_STAT_RX_GREEN_PRIO_3] +
+				      s[OCELOT_STAT_RX_GREEN_PRIO_4] +
+				      s[OCELOT_STAT_RX_GREEN_PRIO_5] +
+				      s[OCELOT_STAT_RX_GREEN_PRIO_6] +
+				      s[OCELOT_STAT_RX_GREEN_PRIO_7] +
+				      s[OCELOT_STAT_RX_YELLOW_PRIO_0] +
+				      s[OCELOT_STAT_RX_YELLOW_PRIO_1] +
+				      s[OCELOT_STAT_RX_YELLOW_PRIO_2] +
+				      s[OCELOT_STAT_RX_YELLOW_PRIO_3] +
+				      s[OCELOT_STAT_RX_YELLOW_PRIO_4] +
+				      s[OCELOT_STAT_RX_YELLOW_PRIO_5] +
+				      s[OCELOT_STAT_RX_YELLOW_PRIO_6] +
+				      s[OCELOT_STAT_RX_YELLOW_PRIO_7];
+	mac_stats->MulticastFramesXmittedOK = s[OCELOT_STAT_TX_MULTICAST];
+	mac_stats->BroadcastFramesXmittedOK = s[OCELOT_STAT_TX_BROADCAST];
+	mac_stats->MulticastFramesReceivedOK = s[OCELOT_STAT_RX_MULTICAST];
+	mac_stats->BroadcastFramesReceivedOK = s[OCELOT_STAT_RX_BROADCAST];
+	mac_stats->FrameTooLongErrors = s[OCELOT_STAT_RX_LONGS];
+	/* Sadly, C_RX_CRC is the sum of FCS and alignment errors, they are not
+	 * counted individually.
+	 */
+	mac_stats->FrameCheckSequenceErrors = s[OCELOT_STAT_RX_CRC_ALIGN_ERRS];
+	mac_stats->AlignmentErrors = s[OCELOT_STAT_RX_CRC_ALIGN_ERRS];
+}
+
+void ocelot_port_get_eth_mac_stats(struct ocelot *ocelot, int port,
+				   struct ethtool_eth_mac_stats *mac_stats)
+{
+	ocelot_port_stats_run(ocelot, port, mac_stats,
+			      ocelot_port_mac_stats_cb);
+}
+EXPORT_SYMBOL_GPL(ocelot_port_get_eth_mac_stats);
+
+static void ocelot_port_phy_stats_cb(struct ocelot *ocelot, int port, void *priv)
+{
+	u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS];
+	struct ethtool_eth_phy_stats *phy_stats = priv;
+
+	phy_stats->SymbolErrorDuringCarrier = s[OCELOT_STAT_RX_SYM_ERRS];
+}
+
+void ocelot_port_get_eth_phy_stats(struct ocelot *ocelot, int port,
+				   struct ethtool_eth_phy_stats *phy_stats)
+{
+	ocelot_port_stats_run(ocelot, port, phy_stats,
+			      ocelot_port_phy_stats_cb);
+}
+EXPORT_SYMBOL_GPL(ocelot_port_get_eth_phy_stats);
+
+void ocelot_port_get_stats64(struct ocelot *ocelot, int port,
+			     struct rtnl_link_stats64 *stats)
+{
+	u64 *s = &ocelot->stats[port * OCELOT_NUM_STATS];
+
+	spin_lock(&ocelot->stats_lock);
+
+	/* Get Rx stats */
+	stats->rx_bytes = s[OCELOT_STAT_RX_OCTETS];
+	stats->rx_packets = s[OCELOT_STAT_RX_SHORTS] +
+			    s[OCELOT_STAT_RX_FRAGMENTS] +
+			    s[OCELOT_STAT_RX_JABBERS] +
+			    s[OCELOT_STAT_RX_LONGS] +
+			    s[OCELOT_STAT_RX_64] +
+			    s[OCELOT_STAT_RX_65_127] +
+			    s[OCELOT_STAT_RX_128_255] +
+			    s[OCELOT_STAT_RX_256_511] +
+			    s[OCELOT_STAT_RX_512_1023] +
+			    s[OCELOT_STAT_RX_1024_1526] +
+			    s[OCELOT_STAT_RX_1527_MAX];
+	stats->multicast = s[OCELOT_STAT_RX_MULTICAST];
+	stats->rx_missed_errors = s[OCELOT_STAT_DROP_TAIL];
+	stats->rx_dropped = s[OCELOT_STAT_RX_RED_PRIO_0] +
+			    s[OCELOT_STAT_RX_RED_PRIO_1] +
+			    s[OCELOT_STAT_RX_RED_PRIO_2] +
+			    s[OCELOT_STAT_RX_RED_PRIO_3] +
+			    s[OCELOT_STAT_RX_RED_PRIO_4] +
+			    s[OCELOT_STAT_RX_RED_PRIO_5] +
+			    s[OCELOT_STAT_RX_RED_PRIO_6] +
+			    s[OCELOT_STAT_RX_RED_PRIO_7] +
+			    s[OCELOT_STAT_DROP_LOCAL] +
+			    s[OCELOT_STAT_DROP_YELLOW_PRIO_0] +
+			    s[OCELOT_STAT_DROP_YELLOW_PRIO_1] +
+			    s[OCELOT_STAT_DROP_YELLOW_PRIO_2] +
+			    s[OCELOT_STAT_DROP_YELLOW_PRIO_3] +
+			    s[OCELOT_STAT_DROP_YELLOW_PRIO_4] +
+			    s[OCELOT_STAT_DROP_YELLOW_PRIO_5] +
+			    s[OCELOT_STAT_DROP_YELLOW_PRIO_6] +
+			    s[OCELOT_STAT_DROP_YELLOW_PRIO_7] +
+			    s[OCELOT_STAT_DROP_GREEN_PRIO_0] +
+			    s[OCELOT_STAT_DROP_GREEN_PRIO_1] +
+			    s[OCELOT_STAT_DROP_GREEN_PRIO_2] +
+			    s[OCELOT_STAT_DROP_GREEN_PRIO_3] +
+			    s[OCELOT_STAT_DROP_GREEN_PRIO_4] +
+			    s[OCELOT_STAT_DROP_GREEN_PRIO_5] +
+			    s[OCELOT_STAT_DROP_GREEN_PRIO_6] +
+			    s[OCELOT_STAT_DROP_GREEN_PRIO_7];
+
+	/* Get Tx stats */
+	stats->tx_bytes = s[OCELOT_STAT_TX_OCTETS];
+	stats->tx_packets = s[OCELOT_STAT_TX_64] +
+			    s[OCELOT_STAT_TX_65_127] +
+			    s[OCELOT_STAT_TX_128_255] +
+			    s[OCELOT_STAT_TX_256_511] +
+			    s[OCELOT_STAT_TX_512_1023] +
+			    s[OCELOT_STAT_TX_1024_1526] +
+			    s[OCELOT_STAT_TX_1527_MAX];
+	stats->tx_dropped = s[OCELOT_STAT_TX_DROPS] +
+			    s[OCELOT_STAT_TX_AGED];
+	stats->collisions = s[OCELOT_STAT_TX_COLLISION];
+
+	spin_unlock(&ocelot->stats_lock);
+}
+EXPORT_SYMBOL(ocelot_port_get_stats64);
+
+static int ocelot_prepare_stats_regions(struct ocelot *ocelot)
+{
+	struct ocelot_stats_region *region = NULL;
+	unsigned int last;
+	int i;
+
+	INIT_LIST_HEAD(&ocelot->stats_regions);
+
+	for (i = 0; i < OCELOT_NUM_STATS; i++) {
+		if (!ocelot->stats_layout[i].reg)
+			continue;
+
+		if (region && ocelot->stats_layout[i].reg == last + 4) {
+			region->count++;
+		} else {
+			region = devm_kzalloc(ocelot->dev, sizeof(*region),
+					      GFP_KERNEL);
+			if (!region)
+				return -ENOMEM;
+
+			region->base = ocelot->stats_layout[i].reg;
+			region->count = 1;
+			list_add_tail(&region->node, &ocelot->stats_regions);
+		}
+
+		last = ocelot->stats_layout[i].reg;
+	}
+
+	list_for_each_entry(region, &ocelot->stats_regions, node) {
+		region->buf = devm_kcalloc(ocelot->dev, region->count,
+					   sizeof(*region->buf), GFP_KERNEL);
+		if (!region->buf)
+			return -ENOMEM;
+	}
+
+	return 0;
+}
+
+int ocelot_stats_init(struct ocelot *ocelot)
+{
+	char queue_name[32];
+	int ret;
+
+	ocelot->stats = devm_kcalloc(ocelot->dev,
+				     ocelot->num_phys_ports * OCELOT_NUM_STATS,
+				     sizeof(u64), GFP_KERNEL);
+	if (!ocelot->stats)
+		return -ENOMEM;
+
+	snprintf(queue_name, sizeof(queue_name), "%s-stats",
+		 dev_name(ocelot->dev));
+	ocelot->stats_queue = create_singlethread_workqueue(queue_name);
+	if (!ocelot->stats_queue)
+		return -ENOMEM;
+
+	spin_lock_init(&ocelot->stats_lock);
+	mutex_init(&ocelot->stat_view_lock);
+
+	ret = ocelot_prepare_stats_regions(ocelot);
+	if (ret) {
+		destroy_workqueue(ocelot->stats_queue);
+		return ret;
+	}
+
+	INIT_DELAYED_WORK(&ocelot->stats_work, ocelot_check_stats_work);
+	queue_delayed_work(ocelot->stats_queue, &ocelot->stats_work,
+			   OCELOT_STATS_CHECK_DELAY);
+
+	return 0;
+}
+
+void ocelot_stats_deinit(struct ocelot *ocelot)
+{
+	cancel_delayed_work(&ocelot->stats_work);
+	destroy_workqueue(ocelot->stats_queue);
+}
diff --git a/drivers/net/ethernet/mscc/ocelot_vsc7514.c b/drivers/net/ethernet/mscc/ocelot_vsc7514.c
index 9c488953f541..6f22aea08a64 100644
--- a/drivers/net/ethernet/mscc/ocelot_vsc7514.c
+++ b/drivers/net/ethernet/mscc/ocelot_vsc7514.c
@@ -6,6 +6,7 @@
  */
 #include <linux/dsa/ocelot.h>
 #include <linux/interrupt.h>
+#include <linux/iopoll.h>
 #include <linux/module.h>
 #include <linux/of_net.h>
 #include <linux/netdevice.h>
@@ -25,6 +26,9 @@
 #define VSC7514_VCAP_POLICER_BASE			128
 #define VSC7514_VCAP_POLICER_MAX			191
 
+#define MEM_INIT_SLEEP_US				1000
+#define MEM_INIT_TIMEOUT_US				100000
+
 static const u32 *ocelot_regmap[TARGET_MAX] = {
 	[ANA] = vsc7514_ana_regmap,
 	[QS] = vsc7514_qs_regmap,
@@ -97,378 +101,7 @@ static const struct reg_field ocelot_regfields[REGFIELD_MAX] = {
 };
 
 static const struct ocelot_stat_layout ocelot_stats_layout[OCELOT_NUM_STATS] = {
-	[OCELOT_STAT_RX_OCTETS] = {
-		.name = "rx_octets",
-		.reg = SYS_COUNT_RX_OCTETS,
-	},
-	[OCELOT_STAT_RX_UNICAST] = {
-		.name = "rx_unicast",
-		.reg = SYS_COUNT_RX_UNICAST,
-	},
-	[OCELOT_STAT_RX_MULTICAST] = {
-		.name = "rx_multicast",
-		.reg = SYS_COUNT_RX_MULTICAST,
-	},
-	[OCELOT_STAT_RX_BROADCAST] = {
-		.name = "rx_broadcast",
-		.reg = SYS_COUNT_RX_BROADCAST,
-	},
-	[OCELOT_STAT_RX_SHORTS] = {
-		.name = "rx_shorts",
-		.reg = SYS_COUNT_RX_SHORTS,
-	},
-	[OCELOT_STAT_RX_FRAGMENTS] = {
-		.name = "rx_fragments",
-		.reg = SYS_COUNT_RX_FRAGMENTS,
-	},
-	[OCELOT_STAT_RX_JABBERS] = {
-		.name = "rx_jabbers",
-		.reg = SYS_COUNT_RX_JABBERS,
-	},
-	[OCELOT_STAT_RX_CRC_ALIGN_ERRS] = {
-		.name = "rx_crc_align_errs",
-		.reg = SYS_COUNT_RX_CRC_ALIGN_ERRS,
-	},
-	[OCELOT_STAT_RX_SYM_ERRS] = {
-		.name = "rx_sym_errs",
-		.reg = SYS_COUNT_RX_SYM_ERRS,
-	},
-	[OCELOT_STAT_RX_64] = {
-		.name = "rx_frames_below_65_octets",
-		.reg = SYS_COUNT_RX_64,
-	},
-	[OCELOT_STAT_RX_65_127] = {
-		.name = "rx_frames_65_to_127_octets",
-		.reg = SYS_COUNT_RX_65_127,
-	},
-	[OCELOT_STAT_RX_128_255] = {
-		.name = "rx_frames_128_to_255_octets",
-		.reg = SYS_COUNT_RX_128_255,
-	},
-	[OCELOT_STAT_RX_256_511] = {
-		.name = "rx_frames_256_to_511_octets",
-		.reg = SYS_COUNT_RX_256_511,
-	},
-	[OCELOT_STAT_RX_512_1023] = {
-		.name = "rx_frames_512_to_1023_octets",
-		.reg = SYS_COUNT_RX_512_1023,
-	},
-	[OCELOT_STAT_RX_1024_1526] = {
-		.name = "rx_frames_1024_to_1526_octets",
-		.reg = SYS_COUNT_RX_1024_1526,
-	},
-	[OCELOT_STAT_RX_1527_MAX] = {
-		.name = "rx_frames_over_1526_octets",
-		.reg = SYS_COUNT_RX_1527_MAX,
-	},
-	[OCELOT_STAT_RX_PAUSE] = {
-		.name = "rx_pause",
-		.reg = SYS_COUNT_RX_PAUSE,
-	},
-	[OCELOT_STAT_RX_CONTROL] = {
-		.name = "rx_control",
-		.reg = SYS_COUNT_RX_CONTROL,
-	},
-	[OCELOT_STAT_RX_LONGS] = {
-		.name = "rx_longs",
-		.reg = SYS_COUNT_RX_LONGS,
-	},
-	[OCELOT_STAT_RX_CLASSIFIED_DROPS] = {
-		.name = "rx_classified_drops",
-		.reg = SYS_COUNT_RX_CLASSIFIED_DROPS,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_0] = {
-		.name = "rx_red_prio_0",
-		.reg = SYS_COUNT_RX_RED_PRIO_0,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_1] = {
-		.name = "rx_red_prio_1",
-		.reg = SYS_COUNT_RX_RED_PRIO_1,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_2] = {
-		.name = "rx_red_prio_2",
-		.reg = SYS_COUNT_RX_RED_PRIO_2,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_3] = {
-		.name = "rx_red_prio_3",
-		.reg = SYS_COUNT_RX_RED_PRIO_3,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_4] = {
-		.name = "rx_red_prio_4",
-		.reg = SYS_COUNT_RX_RED_PRIO_4,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_5] = {
-		.name = "rx_red_prio_5",
-		.reg = SYS_COUNT_RX_RED_PRIO_5,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_6] = {
-		.name = "rx_red_prio_6",
-		.reg = SYS_COUNT_RX_RED_PRIO_6,
-	},
-	[OCELOT_STAT_RX_RED_PRIO_7] = {
-		.name = "rx_red_prio_7",
-		.reg = SYS_COUNT_RX_RED_PRIO_7,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_0] = {
-		.name = "rx_yellow_prio_0",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_0,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_1] = {
-		.name = "rx_yellow_prio_1",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_1,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_2] = {
-		.name = "rx_yellow_prio_2",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_2,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_3] = {
-		.name = "rx_yellow_prio_3",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_3,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_4] = {
-		.name = "rx_yellow_prio_4",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_4,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_5] = {
-		.name = "rx_yellow_prio_5",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_5,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_6] = {
-		.name = "rx_yellow_prio_6",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_6,
-	},
-	[OCELOT_STAT_RX_YELLOW_PRIO_7] = {
-		.name = "rx_yellow_prio_7",
-		.reg = SYS_COUNT_RX_YELLOW_PRIO_7,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_0] = {
-		.name = "rx_green_prio_0",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_0,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_1] = {
-		.name = "rx_green_prio_1",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_1,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_2] = {
-		.name = "rx_green_prio_2",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_2,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_3] = {
-		.name = "rx_green_prio_3",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_3,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_4] = {
-		.name = "rx_green_prio_4",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_4,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_5] = {
-		.name = "rx_green_prio_5",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_5,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_6] = {
-		.name = "rx_green_prio_6",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_6,
-	},
-	[OCELOT_STAT_RX_GREEN_PRIO_7] = {
-		.name = "rx_green_prio_7",
-		.reg = SYS_COUNT_RX_GREEN_PRIO_7,
-	},
-	[OCELOT_STAT_TX_OCTETS] = {
-		.name = "tx_octets",
-		.reg = SYS_COUNT_TX_OCTETS,
-	},
-	[OCELOT_STAT_TX_UNICAST] = {
-		.name = "tx_unicast",
-		.reg = SYS_COUNT_TX_UNICAST,
-	},
-	[OCELOT_STAT_TX_MULTICAST] = {
-		.name = "tx_multicast",
-		.reg = SYS_COUNT_TX_MULTICAST,
-	},
-	[OCELOT_STAT_TX_BROADCAST] = {
-		.name = "tx_broadcast",
-		.reg = SYS_COUNT_TX_BROADCAST,
-	},
-	[OCELOT_STAT_TX_COLLISION] = {
-		.name = "tx_collision",
-		.reg = SYS_COUNT_TX_COLLISION,
-	},
-	[OCELOT_STAT_TX_DROPS] = {
-		.name = "tx_drops",
-		.reg = SYS_COUNT_TX_DROPS,
-	},
-	[OCELOT_STAT_TX_PAUSE] = {
-		.name = "tx_pause",
-		.reg = SYS_COUNT_TX_PAUSE,
-	},
-	[OCELOT_STAT_TX_64] = {
-		.name = "tx_frames_below_65_octets",
-		.reg = SYS_COUNT_TX_64,
-	},
-	[OCELOT_STAT_TX_65_127] = {
-		.name = "tx_frames_65_to_127_octets",
-		.reg = SYS_COUNT_TX_65_127,
-	},
-	[OCELOT_STAT_TX_128_255] = {
-		.name = "tx_frames_128_255_octets",
-		.reg = SYS_COUNT_TX_128_255,
-	},
-	[OCELOT_STAT_TX_256_511] = {
-		.name = "tx_frames_256_511_octets",
-		.reg = SYS_COUNT_TX_256_511,
-	},
-	[OCELOT_STAT_TX_512_1023] = {
-		.name = "tx_frames_512_1023_octets",
-		.reg = SYS_COUNT_TX_512_1023,
-	},
-	[OCELOT_STAT_TX_1024_1526] = {
-		.name = "tx_frames_1024_1526_octets",
-		.reg = SYS_COUNT_TX_1024_1526,
-	},
-	[OCELOT_STAT_TX_1527_MAX] = {
-		.name = "tx_frames_over_1526_octets",
-		.reg = SYS_COUNT_TX_1527_MAX,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_0] = {
-		.name = "tx_yellow_prio_0",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_0,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_1] = {
-		.name = "tx_yellow_prio_1",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_1,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_2] = {
-		.name = "tx_yellow_prio_2",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_2,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_3] = {
-		.name = "tx_yellow_prio_3",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_3,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_4] = {
-		.name = "tx_yellow_prio_4",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_4,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_5] = {
-		.name = "tx_yellow_prio_5",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_5,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_6] = {
-		.name = "tx_yellow_prio_6",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_6,
-	},
-	[OCELOT_STAT_TX_YELLOW_PRIO_7] = {
-		.name = "tx_yellow_prio_7",
-		.reg = SYS_COUNT_TX_YELLOW_PRIO_7,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_0] = {
-		.name = "tx_green_prio_0",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_0,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_1] = {
-		.name = "tx_green_prio_1",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_1,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_2] = {
-		.name = "tx_green_prio_2",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_2,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_3] = {
-		.name = "tx_green_prio_3",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_3,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_4] = {
-		.name = "tx_green_prio_4",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_4,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_5] = {
-		.name = "tx_green_prio_5",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_5,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_6] = {
-		.name = "tx_green_prio_6",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_6,
-	},
-	[OCELOT_STAT_TX_GREEN_PRIO_7] = {
-		.name = "tx_green_prio_7",
-		.reg = SYS_COUNT_TX_GREEN_PRIO_7,
-	},
-	[OCELOT_STAT_TX_AGED] = {
-		.name = "tx_aged",
-		.reg = SYS_COUNT_TX_AGING,
-	},
-	[OCELOT_STAT_DROP_LOCAL] = {
-		.name = "drop_local",
-		.reg = SYS_COUNT_DROP_LOCAL,
-	},
-	[OCELOT_STAT_DROP_TAIL] = {
-		.name = "drop_tail",
-		.reg = SYS_COUNT_DROP_TAIL,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_0] = {
-		.name = "drop_yellow_prio_0",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_0,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_1] = {
-		.name = "drop_yellow_prio_1",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_1,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_2] = {
-		.name = "drop_yellow_prio_2",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_2,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_3] = {
-		.name = "drop_yellow_prio_3",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_3,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_4] = {
-		.name = "drop_yellow_prio_4",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_4,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_5] = {
-		.name = "drop_yellow_prio_5",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_5,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_6] = {
-		.name = "drop_yellow_prio_6",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_6,
-	},
-	[OCELOT_STAT_DROP_YELLOW_PRIO_7] = {
-		.name = "drop_yellow_prio_7",
-		.reg = SYS_COUNT_DROP_YELLOW_PRIO_7,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_0] = {
-		.name = "drop_green_prio_0",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_0,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_1] = {
-		.name = "drop_green_prio_1",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_1,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_2] = {
-		.name = "drop_green_prio_2",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_2,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_3] = {
-		.name = "drop_green_prio_3",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_3,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_4] = {
-		.name = "drop_green_prio_4",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_4,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_5] = {
-		.name = "drop_green_prio_5",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_5,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_6] = {
-		.name = "drop_green_prio_6",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_6,
-	},
-	[OCELOT_STAT_DROP_GREEN_PRIO_7] = {
-		.name = "drop_green_prio_7",
-		.reg = SYS_COUNT_DROP_GREEN_PRIO_7,
-	},
+	OCELOT_COMMON_STATS,
 };
 
 static void ocelot_pll5_init(struct ocelot *ocelot)
@@ -562,27 +195,43 @@ static const struct of_device_id mscc_ocelot_match[] = {
 };
 MODULE_DEVICE_TABLE(of, mscc_ocelot_match);
 
+static int ocelot_mem_init_status(struct ocelot *ocelot)
+{
+	unsigned int val;
+	int err;
+
+	err = regmap_field_read(ocelot->regfields[SYS_RESET_CFG_MEM_INIT],
+				&val);
+
+	return err ?: val;
+}
+
 static int ocelot_reset(struct ocelot *ocelot)
 {
-	int retries = 100;
+	int err;
 	u32 val;
 
-	regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_INIT], 1);
-	regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_ENA], 1);
+	err = regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_INIT], 1);
+	if (err)
+		return err;
 
-	do {
-		msleep(1);
-		regmap_field_read(ocelot->regfields[SYS_RESET_CFG_MEM_INIT],
-				  &val);
-	} while (val && --retries);
+	err = regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_ENA], 1);
+	if (err)
+		return err;
 
-	if (!retries)
-		return -ETIMEDOUT;
+	/* MEM_INIT is a self-clearing bit. Wait for it to be cleared (should be
+	 * 100us) before enabling the switch core.
+	 */
+	err = readx_poll_timeout(ocelot_mem_init_status, ocelot, val, !val,
+				 MEM_INIT_SLEEP_US, MEM_INIT_TIMEOUT_US);
+	if (err)
+		return err;
 
-	regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_ENA], 1);
-	regmap_field_write(ocelot->regfields[SYS_RESET_CFG_CORE_ENA], 1);
+	err = regmap_field_write(ocelot->regfields[SYS_RESET_CFG_MEM_ENA], 1);
+	if (err)
+		return err;
 
-	return 0;
+	return regmap_field_write(ocelot->regfields[SYS_RESET_CFG_CORE_ENA], 1);
 }
 
 /* Watermark encode
diff --git a/drivers/net/ethernet/mscc/vsc7514_regs.c b/drivers/net/ethernet/mscc/vsc7514_regs.c
index 9cf82ecf191c..9d2d3e13cacf 100644
--- a/drivers/net/ethernet/mscc/vsc7514_regs.c
+++ b/drivers/net/ethernet/mscc/vsc7514_regs.c
@@ -242,7 +242,7 @@ const u32 vsc7514_sys_regmap[] = {
 	REG(SYS_COUNT_TX_GREEN_PRIO_5,			0x00016c),
 	REG(SYS_COUNT_TX_GREEN_PRIO_6,			0x000170),
 	REG(SYS_COUNT_TX_GREEN_PRIO_7,			0x000174),
-	REG(SYS_COUNT_TX_AGING,				0x000178),
+	REG(SYS_COUNT_TX_AGED,				0x000178),
 	REG(SYS_COUNT_DROP_LOCAL,			0x000200),
 	REG(SYS_COUNT_DROP_TAIL,			0x000204),
 	REG(SYS_COUNT_DROP_YELLOW_PRIO_0,		0x000208),
@@ -283,7 +283,6 @@ const u32 vsc7514_sys_regmap[] = {
 	REG(SYS_MMGT_FAST,				0x0006a0),
 	REG(SYS_EVENTS_DIF,				0x0006a4),
 	REG(SYS_EVENTS_CORE,				0x0006b4),
-	REG(SYS_CNT,					0x000000),
 	REG(SYS_PTP_STATUS,				0x0006b8),
 	REG(SYS_PTP_TXSTAMP,				0x0006bc),
 	REG(SYS_PTP_NXT,				0x0006c0),
diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
index 971dde8c3286..9063e2e22cd5 100644
--- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
+++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c
@@ -1647,10 +1647,10 @@ myri10ge_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *info)
 {
 	struct myri10ge_priv *mgp = netdev_priv(netdev);
 
-	strlcpy(info->driver, "myri10ge", sizeof(info->driver));
-	strlcpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version));
-	strlcpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version));
-	strlcpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, "myri10ge", sizeof(info->driver));
+	strscpy(info->version, MYRI10GE_VERSION_STR, sizeof(info->version));
+	strscpy(info->fw_version, mgp->fw_version, sizeof(info->fw_version));
+	strscpy(info->bus_info, pci_name(mgp->pdev), sizeof(info->bus_info));
 }
 
 static int myri10ge_get_coalesce(struct net_device *netdev,
diff --git a/drivers/net/ethernet/natsemi/natsemi.c b/drivers/net/ethernet/natsemi/natsemi.c
index 9aae7f1eb5d2..650a5a166070 100644
--- a/drivers/net/ethernet/natsemi/natsemi.c
+++ b/drivers/net/ethernet/natsemi/natsemi.c
@@ -869,7 +869,7 @@ static int natsemi_probe1(struct pci_dev *pdev, const struct pci_device_id *ent)
 	np = netdev_priv(dev);
 	np->ioaddr = ioaddr;
 
-	netif_napi_add(dev, &np->napi, natsemi_poll, 64);
+	netif_napi_add(dev, &np->napi, natsemi_poll);
 	np->dev = dev;
 
 	np->pci_dev = pdev;
@@ -2564,9 +2564,9 @@ static void set_rx_mode(struct net_device *dev)
 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
 	struct netdev_private *np = netdev_priv(dev);
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
 }
 
 static int get_regs_len(struct net_device *dev)
diff --git a/drivers/net/ethernet/natsemi/ns83820.c b/drivers/net/ethernet/natsemi/ns83820.c
index 49ea130c9067..998586872599 100644
--- a/drivers/net/ethernet/natsemi/ns83820.c
+++ b/drivers/net/ethernet/natsemi/ns83820.c
@@ -1351,9 +1351,9 @@ static int ns83820_set_link_ksettings(struct net_device *ndev,
 static void ns83820_get_drvinfo(struct net_device *ndev, struct ethtool_drvinfo *info)
 {
 	struct ns83820 *dev = PRIV(ndev);
-	strlcpy(info->driver, "ns83820", sizeof(info->driver));
-	strlcpy(info->version, VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(dev->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, "ns83820", sizeof(info->driver));
+	strscpy(info->version, VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(dev->pci_dev), sizeof(info->bus_info));
 }
 
 static u32 ns83820_get_link(struct net_device *ndev)
diff --git a/drivers/net/ethernet/neterion/s2io.c b/drivers/net/ethernet/neterion/s2io.c
index 30f955efa830..dcf8212119f9 100644
--- a/drivers/net/ethernet/neterion/s2io.c
+++ b/drivers/net/ethernet/neterion/s2io.c
@@ -5348,9 +5348,9 @@ static void s2io_ethtool_gdrvinfo(struct net_device *dev,
 {
 	struct s2io_nic *sp = netdev_priv(dev);
 
-	strlcpy(info->driver, s2io_driver_name, sizeof(info->driver));
-	strlcpy(info->version, s2io_driver_version, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(sp->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, s2io_driver_name, sizeof(info->driver));
+	strscpy(info->version, s2io_driver_version, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(sp->pdev), sizeof(info->bus_info));
 }
 
 /**
@@ -7359,10 +7359,9 @@ static int rx_osm_handler(struct ring_info *ring_data, struct RxD_t * rxdp)
 		int get_off = ring_data->rx_curr_get_info.offset;
 		int buf0_len = RXD_GET_BUFFER0_SIZE_3(rxdp->Control_2);
 		int buf2_len = RXD_GET_BUFFER2_SIZE_3(rxdp->Control_2);
-		unsigned char *buff = skb_push(skb, buf0_len);
 
 		struct buffAdd *ba = &ring_data->ba[get_block][get_off];
-		memcpy(buff, ba->ba_0, buf0_len);
+		skb_put_data(skb, ba->ba_0, buf0_len);
 		skb_put(skb, buf2_len);
 	}
 
@@ -7905,10 +7904,10 @@ s2io_init_nic(struct pci_dev *pdev, const struct pci_device_id *pre)
 		for (i = 0; i < config->rx_ring_num ; i++) {
 			struct ring_info *ring = &mac_control->rings[i];
 
-			netif_napi_add(dev, &ring->napi, s2io_poll_msix, 64);
+			netif_napi_add(dev, &ring->napi, s2io_poll_msix);
 		}
 	} else {
-		netif_napi_add(dev, &sp->napi, s2io_poll_inta, 64);
+		netif_napi_add(dev, &sp->napi, s2io_poll_inta);
 	}
 
 	/* Not needed for Herc */
diff --git a/drivers/net/ethernet/netronome/nfp/crypto/tls.c b/drivers/net/ethernet/netronome/nfp/crypto/tls.c
index 78368e71ce83..f80f1a6953fa 100644
--- a/drivers/net/ethernet/netronome/nfp/crypto/tls.c
+++ b/drivers/net/ethernet/netronome/nfp/crypto/tls.c
@@ -474,6 +474,7 @@ int nfp_net_tls_rx_resync_req(struct net_device *netdev,
 {
 	struct nfp_net *nn = netdev_priv(netdev);
 	struct nfp_net_tls_offload_ctx *ntls;
+	struct net *net = dev_net(netdev);
 	struct ipv6hdr *ipv6h;
 	struct tcphdr *th;
 	struct iphdr *iph;
@@ -494,13 +495,13 @@ int nfp_net_tls_rx_resync_req(struct net_device *netdev,
 
 	switch (ipv6h->version) {
 	case 4:
-		sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo,
+		sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 					     iph->saddr, th->source, iph->daddr,
 					     th->dest, netdev->ifindex);
 		break;
 #if IS_ENABLED(CONFIG_IPV6)
 	case 6:
-		sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo,
+		sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 						&ipv6h->saddr, th->source,
 						&ipv6h->daddr, ntohs(th->dest),
 						netdev->ifindex, 0);
diff --git a/drivers/net/ethernet/netronome/nfp/flower/conntrack.c b/drivers/net/ethernet/netronome/nfp/flower/conntrack.c
index b3b2a23b8d89..f693119541d5 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/conntrack.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/conntrack.c
@@ -1,6 +1,9 @@
 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
 /* Copyright (C) 2021 Corigine, Inc. */
 
+#include <net/tc_act/tc_csum.h>
+#include <net/tc_act/tc_ct.h>
+
 #include "conntrack.h"
 #include "../nfp_port.h"
 
@@ -56,9 +59,17 @@ bool is_pre_ct_flow(struct flow_cls_offload *flow)
 	int i;
 
 	flow_action_for_each(i, act, &flow->rule->action) {
-		if (act->id == FLOW_ACTION_CT && !act->ct.action)
-			return true;
+		if (act->id == FLOW_ACTION_CT) {
+			/* The pre_ct rule only have the ct or ct nat action, cannot
+			 * contains other ct action e.g ct commit and so on.
+			 */
+			if ((!act->ct.action || act->ct.action == TCA_CT_ACT_NAT))
+				return true;
+			else
+				return false;
+		}
 	}
+
 	return false;
 }
 
@@ -66,13 +77,37 @@ bool is_post_ct_flow(struct flow_cls_offload *flow)
 {
 	struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
 	struct flow_dissector *dissector = rule->match.dissector;
+	struct flow_action_entry *act;
+	bool exist_ct_clear = false;
 	struct flow_match_ct ct;
+	int i;
+
+	/* post ct entry cannot contains any ct action except ct_clear. */
+	flow_action_for_each(i, act, &flow->rule->action) {
+		if (act->id == FLOW_ACTION_CT) {
+			/* ignore ct clear action. */
+			if (act->ct.action == TCA_CT_ACT_CLEAR) {
+				exist_ct_clear = true;
+				continue;
+			}
+
+			return false;
+		}
+	}
 
 	if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) {
 		flow_rule_match_ct(rule, &ct);
 		if (ct.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)
 			return true;
+	} else {
+		/* when do nat with ct, the post ct entry ignore the ct status,
+		 * will match the nat field(sip/dip) instead. In this situation,
+		 * the flow chain index is not zero and contains ct clear action.
+		 */
+		if (flow->common.chain_index && exist_ct_clear)
+			return true;
 	}
+
 	return false;
 }
 
@@ -168,6 +203,20 @@ static void *get_mangled_tos_ttl(struct flow_rule *rule, void *buf,
 	return buf;
 }
 
+/* Note entry1 and entry2 are not swappable. only skip ip and
+ * tport merge check for pre_ct and post_ct when pre_ct do nat.
+ */
+static bool nfp_ct_merge_check_cannot_skip(struct nfp_fl_ct_flow_entry *entry1,
+					   struct nfp_fl_ct_flow_entry *entry2)
+{
+	/* only pre_ct have NFP_FL_ACTION_DO_NAT flag. */
+	if ((entry1->flags & NFP_FL_ACTION_DO_NAT) &&
+	    entry2->type == CT_TYPE_POST_CT)
+		return false;
+
+	return true;
+}
+
 /* Note entry1 and entry2 are not swappable, entry1 should be
  * the former flow whose mangle action need be taken into account
  * if existed, and entry2 should be the latter flow whose action
@@ -225,7 +274,12 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
 			goto check_failed;
 	}
 
-	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
+	/* if pre ct entry do nat, the nat ip exists in nft entry,
+	 * will be do merge check when do nft and post ct merge,
+	 * so skip this ip merge check here.
+	 */
+	if ((ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS)) &&
+	    nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
 		struct flow_match_ipv4_addrs match1, match2;
 
 		flow_rule_match_ipv4_addrs(entry1->rule, &match1);
@@ -242,7 +296,12 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
 			goto check_failed;
 	}
 
-	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
+	/* if pre ct entry do nat, the nat ip exists in nft entry,
+	 * will be do merge check when do nft and post ct merge,
+	 * so skip this ip merge check here.
+	 */
+	if ((ovlp_keys & BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS)) &&
+	    nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
 		struct flow_match_ipv6_addrs match1, match2;
 
 		flow_rule_match_ipv6_addrs(entry1->rule, &match1);
@@ -259,7 +318,12 @@ static int nfp_ct_merge_check(struct nfp_fl_ct_flow_entry *entry1,
 			goto check_failed;
 	}
 
-	if (ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) {
+	/* if pre ct entry do nat, the nat tport exists in nft entry,
+	 * will be do merge check when do nft and post ct merge,
+	 * so skip this tport merge check here.
+	 */
+	if ((ovlp_keys & BIT(FLOW_DISSECTOR_KEY_PORTS)) &&
+	    nfp_ct_merge_check_cannot_skip(entry1, entry2)) {
 		enum flow_action_mangle_base htype = FLOW_ACT_MANGLE_UNSPEC;
 		struct flow_match_ports match1, match2;
 
@@ -404,12 +468,55 @@ check_failed:
 	return -EINVAL;
 }
 
+static int nfp_ct_check_vlan_merge(struct flow_action_entry *a_in,
+				   struct flow_rule *rule)
+{
+	struct flow_match_vlan match;
+
+	if (unlikely(flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)))
+		return -EOPNOTSUPP;
+
+	/* post_ct does not match VLAN KEY, can be merged. */
+	if (likely(!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)))
+		return 0;
+
+	switch (a_in->id) {
+	/* pre_ct has pop vlan, post_ct cannot match VLAN KEY, cannot be merged. */
+	case FLOW_ACTION_VLAN_POP:
+		return -EOPNOTSUPP;
+
+	case FLOW_ACTION_VLAN_PUSH:
+	case FLOW_ACTION_VLAN_MANGLE:
+		flow_rule_match_vlan(rule, &match);
+		/* different vlan id, cannot be merged. */
+		if ((match.key->vlan_id & match.mask->vlan_id) ^
+		    (a_in->vlan.vid & match.mask->vlan_id))
+			return -EOPNOTSUPP;
+
+		/* different tpid, cannot be merged. */
+		if ((match.key->vlan_tpid & match.mask->vlan_tpid) ^
+		    (a_in->vlan.proto & match.mask->vlan_tpid))
+			return -EOPNOTSUPP;
+
+		/* different priority, cannot be merged. */
+		if ((match.key->vlan_priority & match.mask->vlan_priority) ^
+		    (a_in->vlan.prio & match.mask->vlan_priority))
+			return -EOPNOTSUPP;
+
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
 static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
 				  struct nfp_fl_ct_flow_entry *post_ct_entry,
 				  struct nfp_fl_ct_flow_entry *nft_entry)
 {
 	struct flow_action_entry *act;
-	int i;
+	int i, err;
 
 	/* Check for pre_ct->action conflicts */
 	flow_action_for_each(i, act, &pre_ct_entry->rule->action) {
@@ -417,6 +524,10 @@ static int nfp_ct_merge_act_check(struct nfp_fl_ct_flow_entry *pre_ct_entry,
 		case FLOW_ACTION_VLAN_PUSH:
 		case FLOW_ACTION_VLAN_POP:
 		case FLOW_ACTION_VLAN_MANGLE:
+			err = nfp_ct_check_vlan_merge(act, post_ct_entry->rule);
+			if (err)
+				return err;
+			break;
 		case FLOW_ACTION_MPLS_PUSH:
 		case FLOW_ACTION_MPLS_POP:
 		case FLOW_ACTION_MPLS_MANGLE:
@@ -468,6 +579,12 @@ static int nfp_ct_check_meta(struct nfp_fl_ct_flow_entry *post_ct_entry,
 			return -EINVAL;
 
 		return 0;
+	} else {
+		/* post_ct with ct clear action will not match the
+		 * ct status when nft is nat entry.
+		 */
+		if (nft_entry->flags & NFP_FL_ACTION_DO_MANGLE)
+			return 0;
 	}
 
 	return -EINVAL;
@@ -537,11 +654,37 @@ nfp_fl_calc_key_layers_sz(struct nfp_fl_key_ls in_key_ls, uint16_t *map)
 	return key_size;
 }
 
+/* get the csum flag according the ip proto and mangle action. */
+static void nfp_fl_get_csum_flag(struct flow_action_entry *a_in, u8 ip_proto, u32 *csum)
+{
+	if (a_in->id != FLOW_ACTION_MANGLE)
+		return;
+
+	switch (a_in->mangle.htype) {
+	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
+		*csum |= TCA_CSUM_UPDATE_FLAG_IPV4HDR;
+		if (ip_proto == IPPROTO_TCP)
+			*csum |= TCA_CSUM_UPDATE_FLAG_TCP;
+		else if (ip_proto == IPPROTO_UDP)
+			*csum |= TCA_CSUM_UPDATE_FLAG_UDP;
+		break;
+	case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
+		*csum |= TCA_CSUM_UPDATE_FLAG_TCP;
+		break;
+	case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
+		*csum |= TCA_CSUM_UPDATE_FLAG_UDP;
+		break;
+	default:
+		break;
+	}
+}
+
 static int nfp_fl_merge_actions_offload(struct flow_rule **rules,
 					struct nfp_flower_priv *priv,
 					struct net_device *netdev,
 					struct nfp_fl_payload *flow_pay)
 {
+	enum flow_action_hw_stats tmp_stats = FLOW_ACTION_HW_STATS_DONT_CARE;
 	struct flow_action_entry *a_in;
 	int i, j, num_actions, id;
 	struct flow_rule *a_rule;
@@ -551,15 +694,25 @@ static int nfp_fl_merge_actions_offload(struct flow_rule **rules,
 		      rules[CT_TYPE_NFT]->action.num_entries +
 		      rules[CT_TYPE_POST_CT]->action.num_entries;
 
-	a_rule = flow_rule_alloc(num_actions);
+	/* Add one action to make sure there is enough room to add an checksum action
+	 * when do nat.
+	 */
+	a_rule = flow_rule_alloc(num_actions + 1);
 	if (!a_rule)
 		return -ENOMEM;
 
 	/* Actions need a BASIC dissector. */
 	a_rule->match = rules[CT_TYPE_PRE_CT]->match;
+	/* post_ct entry have one action at least. */
+	if (rules[CT_TYPE_POST_CT]->action.num_entries != 0) {
+		tmp_stats = rules[CT_TYPE_POST_CT]->action.entries[0].hw_stats;
+	}
 
 	/* Copy actions */
 	for (j = 0; j < _CT_TYPE_MAX; j++) {
+		u32 csum_updated = 0;
+		u8 ip_proto = 0;
+
 		if (flow_rule_match_key(rules[j], FLOW_DISSECTOR_KEY_BASIC)) {
 			struct flow_match_basic match;
 
@@ -571,8 +724,10 @@ static int nfp_fl_merge_actions_offload(struct flow_rule **rules,
 			 * through the subflows and assign the proper subflow to a_rule
 			 */
 			flow_rule_match_basic(rules[j], &match);
-			if (match.mask->ip_proto)
+			if (match.mask->ip_proto) {
 				a_rule->match = rules[j]->match;
+				ip_proto = match.key->ip_proto;
+			}
 		}
 
 		for (i = 0; i < rules[j]->action.num_entries; i++) {
@@ -589,11 +744,32 @@ static int nfp_fl_merge_actions_offload(struct flow_rule **rules,
 			case FLOW_ACTION_CT_METADATA:
 				continue;
 			default:
+				/* nft entry is generated by tc ct, which mangle action do not care
+				 * the stats, inherit the post entry stats to meet the
+				 * flow_action_hw_stats_check.
+				 */
+				if (j == CT_TYPE_NFT) {
+					if (a_in->hw_stats == FLOW_ACTION_HW_STATS_DONT_CARE)
+						a_in->hw_stats = tmp_stats;
+					nfp_fl_get_csum_flag(a_in, ip_proto, &csum_updated);
+				}
 				memcpy(&a_rule->action.entries[offset++],
 				       a_in, sizeof(struct flow_action_entry));
 				break;
 			}
 		}
+		/* nft entry have mangle action, but do not have checksum action when do NAT,
+		 * hardware will automatically fix IPv4 and TCP/UDP checksum. so add an csum action
+		 * to meet csum action check.
+		 */
+		if (csum_updated) {
+			struct flow_action_entry *csum_action;
+
+			csum_action = &a_rule->action.entries[offset++];
+			csum_action->id = FLOW_ACTION_CSUM;
+			csum_action->csum_flags = csum_updated;
+			csum_action->hw_stats = tmp_stats;
+		}
 	}
 
 	/* Some actions would have been ignored, so update the num_entries field */
@@ -1191,6 +1367,49 @@ static struct net_device *get_netdev_from_rule(struct flow_rule *rule)
 	return NULL;
 }
 
+static void nfp_nft_ct_translate_mangle_action(struct flow_action_entry *mangle_action)
+{
+	if (mangle_action->id != FLOW_ACTION_MANGLE)
+		return;
+
+	switch (mangle_action->mangle.htype) {
+	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
+	case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
+		mangle_action->mangle.val = (__force u32)cpu_to_be32(mangle_action->mangle.val);
+		mangle_action->mangle.mask = (__force u32)cpu_to_be32(mangle_action->mangle.mask);
+		return;
+
+	case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
+	case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
+		mangle_action->mangle.val = (__force u16)cpu_to_be16(mangle_action->mangle.val);
+		mangle_action->mangle.mask = (__force u16)cpu_to_be16(mangle_action->mangle.mask);
+		return;
+
+	default:
+		return;
+	}
+}
+
+static int nfp_nft_ct_set_flow_flag(struct flow_action_entry *act,
+				    struct nfp_fl_ct_flow_entry *entry)
+{
+	switch (act->id) {
+	case FLOW_ACTION_CT:
+		if (act->ct.action == TCA_CT_ACT_NAT)
+			entry->flags |= NFP_FL_ACTION_DO_NAT;
+		break;
+
+	case FLOW_ACTION_MANGLE:
+		entry->flags |= NFP_FL_ACTION_DO_MANGLE;
+		break;
+
+	default:
+		break;
+	}
+
+	return 0;
+}
+
 static struct
 nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
 					 struct net_device *netdev,
@@ -1257,6 +1476,13 @@ nfp_fl_ct_flow_entry *nfp_fl_ct_add_flow(struct nfp_fl_ct_zone_entry *zt,
 
 		new_act = &entry->rule->action.entries[i];
 		memcpy(new_act, act, sizeof(struct flow_action_entry));
+		/* nft entry mangle field is host byte order, need translate to
+		 * network byte order.
+		 */
+		if (is_nft)
+			nfp_nft_ct_translate_mangle_action(new_act);
+
+		nfp_nft_ct_set_flow_flag(new_act, entry);
 		/* Entunnel is a special case, need to allocate and copy
 		 * tunnel info.
 		 */
diff --git a/drivers/net/ethernet/netronome/nfp/flower/conntrack.h b/drivers/net/ethernet/netronome/nfp/flower/conntrack.h
index beb6cceff9d8..762c0b36e269 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/conntrack.h
+++ b/drivers/net/ethernet/netronome/nfp/flower/conntrack.h
@@ -103,6 +103,10 @@ enum nfp_nfp_layer_name {
 	_FLOW_PAY_LAYERS_MAX
 };
 
+/* NFP flow entry flags. */
+#define NFP_FL_ACTION_DO_NAT		BIT(0)
+#define NFP_FL_ACTION_DO_MANGLE		BIT(1)
+
 /**
  * struct nfp_fl_ct_flow_entry - Flow entry containing conntrack flow information
  * @cookie:	Flow cookie, same as original TC flow, used as key
@@ -115,6 +119,7 @@ enum nfp_nfp_layer_name {
  * @rule:	Reference to the original TC flow rule
  * @stats:	Used to cache stats for updating
  * @tun_offset: Used to indicate tunnel action offset in action list
+ * @flags:	Used to indicate flow flag like NAT which used by merge.
  */
 struct nfp_fl_ct_flow_entry {
 	unsigned long cookie;
@@ -127,6 +132,7 @@ struct nfp_fl_ct_flow_entry {
 	struct flow_rule *rule;
 	struct flow_stats stats;
 	u8 tun_offset;		// Set to NFP_FL_CT_NO_TUN if no tun
+	u8 flags;
 };
 
 /**
diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c
index 83c97154c0c7..3ab3e4536b99 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/offload.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c
@@ -1301,9 +1301,14 @@ static bool offload_pre_check(struct flow_cls_offload *flow)
 {
 	struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
 	struct flow_dissector *dissector = rule->match.dissector;
+	struct flow_match_ct ct;
 
-	if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT))
-		return false;
+	if (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CT)) {
+		flow_rule_match_ct(rule, &ct);
+		/* Allow special case where CT match is all 0 */
+		if (memchr_inv(ct.key, 0, sizeof(*ct.key)))
+			return false;
+	}
 
 	if (flow->common.chain_index)
 		return false;
diff --git a/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c b/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c
index 7b92026e1a6f..99052a925d9e 100644
--- a/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c
+++ b/drivers/net/ethernet/netronome/nfp/flower/qos_conf.c
@@ -119,7 +119,8 @@ int nfp_flower_offload_one_police(struct nfp_app *app, bool ingress,
 
 static int nfp_policer_validate(const struct flow_action *action,
 				const struct flow_action_entry *act,
-				struct netlink_ext_ack *extack)
+				struct netlink_ext_ack *extack,
+				bool ingress)
 {
 	if (act->police.exceed.act_id != FLOW_ACTION_DROP) {
 		NL_SET_ERR_MSG_MOD(extack,
@@ -127,12 +128,20 @@ static int nfp_policer_validate(const struct flow_action *action,
 		return -EOPNOTSUPP;
 	}
 
-	if (act->police.notexceed.act_id != FLOW_ACTION_CONTINUE &&
-	    act->police.notexceed.act_id != FLOW_ACTION_PIPE &&
-	    act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Offload not supported when conform action is not continue, pipe or ok");
-		return -EOPNOTSUPP;
+	if (ingress) {
+		if (act->police.notexceed.act_id != FLOW_ACTION_CONTINUE &&
+		    act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Offload not supported when conform action is not continue or ok");
+			return -EOPNOTSUPP;
+		}
+	} else {
+		if (act->police.notexceed.act_id != FLOW_ACTION_PIPE &&
+		    act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Offload not supported when conform action is not pipe or ok");
+			return -EOPNOTSUPP;
+		}
 	}
 
 	if (act->police.notexceed.act_id == FLOW_ACTION_ACCEPT &&
@@ -218,7 +227,7 @@ nfp_flower_install_rate_limiter(struct nfp_app *app, struct net_device *netdev,
 			return -EOPNOTSUPP;
 		}
 
-		err = nfp_policer_validate(&flow->rule->action, action, extack);
+		err = nfp_policer_validate(&flow->rule->action, action, extack, true);
 		if (err)
 			return err;
 
@@ -687,6 +696,7 @@ nfp_act_install_actions(struct nfp_app *app, struct flow_offload_action *fl_act,
 	bool pps_support, pps;
 	bool add = false;
 	u64 rate;
+	int err;
 
 	pps_support = !!(fl_priv->flower_ext_feats & NFP_FL_FEATS_QOS_PPS);
 
@@ -698,6 +708,11 @@ nfp_act_install_actions(struct nfp_app *app, struct flow_offload_action *fl_act,
 					   "unsupported offload: qos rate limit offload requires police action");
 			continue;
 		}
+
+		err = nfp_policer_validate(&fl_act->action, action, extack, false);
+		if (err)
+			return err;
+
 		if (action->police.rate_bytes_ps > 0) {
 			rate = action->police.rate_bytes_ps;
 			burst = action->police.burst;
diff --git a/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c b/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c
index 65e243168765..5d9db8c2a5b4 100644
--- a/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c
+++ b/drivers/net/ethernet/netronome/nfp/nfd3/xsk.c
@@ -84,7 +84,7 @@ static void nfp_nfd3_xsk_rx_skb(struct nfp_net_rx_ring *rx_ring,
 		nfp_net_xsk_rx_drop(r_vec, xrxbuf);
 		return;
 	}
-	memcpy(skb_put(skb, pkt_len), xrxbuf->xdp->data, pkt_len);
+	skb_put_data(skb, xrxbuf->xdp->data, pkt_len);
 
 	skb->mark = meta->mark;
 	skb_set_hash(skb, meta->hash, meta->hash_type);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.c b/drivers/net/ethernet/netronome/nfp/nfp_main.c
index 873429f7a6da..e66e548919d4 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_main.c
@@ -691,6 +691,71 @@ static int nfp_pf_find_rtsyms(struct nfp_pf *pf)
 	return 0;
 }
 
+int nfp_net_pf_get_app_id(struct nfp_pf *pf)
+{
+	return nfp_pf_rtsym_read_optional(pf, "_pf%u_net_app_id",
+					  NFP_APP_CORE_NIC);
+}
+
+static u64 nfp_net_pf_get_app_cap(struct nfp_pf *pf)
+{
+	char name[32];
+	int err = 0;
+	u64 val;
+
+	snprintf(name, sizeof(name), "_pf%u_net_app_cap", nfp_cppcore_pcie_unit(pf->cpp));
+
+	val = nfp_rtsym_read_le(pf->rtbl, name, &err);
+	if (err) {
+		if (err != -ENOENT)
+			nfp_err(pf->cpp, "Unable to read symbol %s\n", name);
+
+		return 0;
+	}
+
+	return val;
+}
+
+static int nfp_pf_cfg_hwinfo(struct nfp_pf *pf, bool sp_indiff)
+{
+	struct nfp_nsp *nsp;
+	char hwinfo[32];
+	int err;
+
+	nsp = nfp_nsp_open(pf->cpp);
+	if (IS_ERR(nsp))
+		return PTR_ERR(nsp);
+
+	snprintf(hwinfo, sizeof(hwinfo), "sp_indiff=%d", sp_indiff);
+	err = nfp_nsp_hwinfo_set(nsp, hwinfo, sizeof(hwinfo));
+	/* Not a fatal error, no need to return error to stop driver from loading */
+	if (err) {
+		nfp_warn(pf->cpp, "HWinfo(sp_indiff=%d) set failed: %d\n", sp_indiff, err);
+	} else {
+		/* Need reinit eth_tbl since the eth table state may change
+		 * after sp_indiff is configured.
+		 */
+		kfree(pf->eth_tbl);
+		pf->eth_tbl = __nfp_eth_read_ports(pf->cpp, nsp);
+	}
+
+	nfp_nsp_close(nsp);
+	return 0;
+}
+
+static int nfp_pf_nsp_cfg(struct nfp_pf *pf)
+{
+	bool sp_indiff = (nfp_net_pf_get_app_id(pf) == NFP_APP_FLOWER_NIC) ||
+			 (nfp_net_pf_get_app_cap(pf) & NFP_NET_APP_CAP_SP_INDIFF);
+
+	return nfp_pf_cfg_hwinfo(pf, sp_indiff);
+}
+
+static void nfp_pf_nsp_clean(struct nfp_pf *pf)
+{
+	nfp_pf_cfg_hwinfo(pf, false);
+}
+
 static int nfp_pci_probe(struct pci_dev *pdev,
 			 const struct pci_device_id *pci_id)
 {
@@ -791,10 +856,14 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 		goto err_fw_unload;
 	}
 
-	err = nfp_net_pci_probe(pf);
+	err = nfp_pf_nsp_cfg(pf);
 	if (err)
 		goto err_fw_unload;
 
+	err = nfp_net_pci_probe(pf);
+	if (err)
+		goto err_nsp_clean;
+
 	err = nfp_hwmon_register(pf);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to register hwmon info\n");
@@ -805,6 +874,8 @@ static int nfp_pci_probe(struct pci_dev *pdev,
 
 err_net_remove:
 	nfp_net_pci_remove(pf);
+err_nsp_clean:
+	nfp_pf_nsp_clean(pf);
 err_fw_unload:
 	kfree(pf->rtbl);
 	nfp_mip_close(pf->mip);
@@ -844,6 +915,7 @@ static void __nfp_pci_shutdown(struct pci_dev *pdev, bool unload_fw)
 
 	nfp_net_pci_remove(pf);
 
+	nfp_pf_nsp_clean(pf);
 	vfree(pf->dumpspec);
 	kfree(pf->rtbl);
 	nfp_mip_close(pf->mip);
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.h b/drivers/net/ethernet/netronome/nfp/nfp_main.h
index f56ca11de134..afd3edfa2428 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_main.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_main.h
@@ -161,6 +161,7 @@ bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb);
 
 int nfp_pf_rtsym_read_optional(struct nfp_pf *pf, const char *format,
 			       unsigned int default_val);
+int nfp_net_pf_get_app_id(struct nfp_pf *pf);
 u8 __iomem *
 nfp_pf_map_rtsym(struct nfp_pf *pf, const char *name, const char *sym_fmt,
 		 unsigned int min_size, struct nfp_cpp_area **area);
@@ -190,4 +191,7 @@ int nfp_shared_buf_pool_set(struct nfp_pf *pf, unsigned int sb,
 
 int nfp_devlink_params_register(struct nfp_pf *pf);
 void nfp_devlink_params_unregister(struct nfp_pf *pf);
+
+unsigned int nfp_net_lr2speed(unsigned int linkrate);
+unsigned int nfp_net_speed2lr(unsigned int speed);
 #endif /* NFP_MAIN_H */
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
index 349a2b1a19a2..27f4786ace4f 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c
@@ -474,19 +474,22 @@ static void nfp_net_read_link_status(struct nfp_net *nn)
 {
 	unsigned long flags;
 	bool link_up;
-	u32 sts;
+	u16 sts;
 
 	spin_lock_irqsave(&nn->link_status_lock, flags);
 
-	sts = nn_readl(nn, NFP_NET_CFG_STS);
+	sts = nn_readw(nn, NFP_NET_CFG_STS);
 	link_up = !!(sts & NFP_NET_CFG_STS_LINK);
 
 	if (nn->link_up == link_up)
 		goto out;
 
 	nn->link_up = link_up;
-	if (nn->port)
+	if (nn->port) {
 		set_bit(NFP_PORT_CHANGED, &nn->port->flags);
+		if (nn->port->link_cb)
+			nn->port->link_cb(nn->port);
+	}
 
 	if (nn->link_up) {
 		netif_carrier_on(nn->dp.netdev);
@@ -768,9 +771,7 @@ nfp_net_napi_add(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec, int idx)
 {
 	if (dp->netdev)
 		netif_napi_add(dp->netdev, &r_vec->napi,
-			       nfp_net_has_xsk_pool_slow(dp, idx) ?
-			       dp->ops->xsk_poll : dp->ops->poll,
-			       NAPI_POLL_WEIGHT);
+			       nfp_net_has_xsk_pool_slow(dp, idx) ? dp->ops->xsk_poll : dp->ops->poll);
 	else
 		tasklet_enable(&r_vec->tasklet);
 }
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
index ac05ec34d69e..6714d5e8fdab 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ctrl.h
@@ -14,6 +14,9 @@
 
 #include <linux/types.h>
 
+/* 64-bit per app capabilities */
+#define NFP_NET_APP_CAP_SP_INDIFF	BIT_ULL(0) /* indifferent to port speed */
+
 /* Configuration BAR size.
  *
  * The configuration BAR is 8K in size, but due to
@@ -193,6 +196,10 @@
 #define   NFP_NET_CFG_STS_LINK_RATE_40G		  5
 #define   NFP_NET_CFG_STS_LINK_RATE_50G		  6
 #define   NFP_NET_CFG_STS_LINK_RATE_100G	  7
+/* NSP Link rate is a 16-bit word. It's determined by NSP and
+ * written to CFG BAR by NFP driver.
+ */
+#define NFP_NET_CFG_STS_NSP_LINK_RATE	0x0036
 #define NFP_NET_CFG_CAP			0x0038
 #define NFP_NET_CFG_MAX_TXRINGS		0x003c
 #define NFP_NET_CFG_MAX_RXRINGS		0x0040
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
index b1b1b648e40c..22a5d2419084 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c
@@ -205,7 +205,7 @@ nfp_get_drvinfo(struct nfp_app *app, struct pci_dev *pdev,
 {
 	char nsp_version[ETHTOOL_FWVERS_LEN] = {};
 
-	strlcpy(drvinfo->driver, dev_driver_string(&pdev->dev),
+	strscpy(drvinfo->driver, dev_driver_string(&pdev->dev),
 		sizeof(drvinfo->driver));
 	nfp_net_get_nspinfo(app, nsp_version);
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
@@ -222,18 +222,49 @@ nfp_net_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
 	snprintf(vnic_version, sizeof(vnic_version), "%d.%d.%d.%d",
 		 nn->fw_ver.extend, nn->fw_ver.class,
 		 nn->fw_ver.major, nn->fw_ver.minor);
-	strlcpy(drvinfo->bus_info, pci_name(nn->pdev),
+	strscpy(drvinfo->bus_info, pci_name(nn->pdev),
 		sizeof(drvinfo->bus_info));
 
 	nfp_get_drvinfo(nn->app, nn->pdev, vnic_version, drvinfo);
 }
 
+static int
+nfp_net_nway_reset(struct net_device *netdev)
+{
+	struct nfp_eth_table_port *eth_port;
+	struct nfp_port *port;
+	int err;
+
+	port = nfp_port_from_netdev(netdev);
+	eth_port = nfp_port_get_eth_port(port);
+	if (!eth_port)
+		return -EOPNOTSUPP;
+
+	if (!netif_running(netdev))
+		return 0;
+
+	err = nfp_eth_set_configured(port->app->cpp, eth_port->index, false);
+	if (err) {
+		netdev_info(netdev, "Link down failed: %d\n", err);
+		return err;
+	}
+
+	err = nfp_eth_set_configured(port->app->cpp, eth_port->index, true);
+	if (err) {
+		netdev_info(netdev, "Link up failed: %d\n", err);
+		return err;
+	}
+
+	netdev_info(netdev, "Link reset succeeded\n");
+	return 0;
+}
+
 static void
 nfp_app_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
 {
 	struct nfp_app *app = nfp_app_from_netdev(netdev);
 
-	strlcpy(drvinfo->bus_info, pci_name(app->pdev),
+	strscpy(drvinfo->bus_info, pci_name(app->pdev),
 		sizeof(drvinfo->bus_info));
 	nfp_get_drvinfo(app, app->pdev, "*", drvinfo);
 }
@@ -273,20 +304,11 @@ static int
 nfp_net_get_link_ksettings(struct net_device *netdev,
 			   struct ethtool_link_ksettings *cmd)
 {
-	static const u32 ls_to_ethtool[] = {
-		[NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED]	= 0,
-		[NFP_NET_CFG_STS_LINK_RATE_UNKNOWN]	= SPEED_UNKNOWN,
-		[NFP_NET_CFG_STS_LINK_RATE_1G]		= SPEED_1000,
-		[NFP_NET_CFG_STS_LINK_RATE_10G]		= SPEED_10000,
-		[NFP_NET_CFG_STS_LINK_RATE_25G]		= SPEED_25000,
-		[NFP_NET_CFG_STS_LINK_RATE_40G]		= SPEED_40000,
-		[NFP_NET_CFG_STS_LINK_RATE_50G]		= SPEED_50000,
-		[NFP_NET_CFG_STS_LINK_RATE_100G]	= SPEED_100000,
-	};
 	struct nfp_eth_table_port *eth_port;
 	struct nfp_port *port;
 	struct nfp_net *nn;
-	u32 sts, ls;
+	unsigned int speed;
+	u16 sts;
 
 	/* Init to unknowns */
 	ethtool_link_ksettings_add_link_mode(cmd, supported, FIBRE);
@@ -299,8 +321,13 @@ nfp_net_get_link_ksettings(struct net_device *netdev,
 	if (eth_port) {
 		ethtool_link_ksettings_add_link_mode(cmd, supported, Pause);
 		ethtool_link_ksettings_add_link_mode(cmd, advertising, Pause);
-		cmd->base.autoneg = eth_port->aneg != NFP_ANEG_DISABLED ?
-			AUTONEG_ENABLE : AUTONEG_DISABLE;
+		if (eth_port->supp_aneg) {
+			ethtool_link_ksettings_add_link_mode(cmd, supported, Autoneg);
+			if (eth_port->aneg == NFP_ANEG_AUTO) {
+				ethtool_link_ksettings_add_link_mode(cmd, advertising, Autoneg);
+				cmd->base.autoneg = AUTONEG_ENABLE;
+			}
+		}
 		nfp_net_set_fec_link_mode(eth_port, cmd);
 	}
 
@@ -319,18 +346,15 @@ nfp_net_get_link_ksettings(struct net_device *netdev,
 		return -EOPNOTSUPP;
 	nn = netdev_priv(netdev);
 
-	sts = nn_readl(nn, NFP_NET_CFG_STS);
-
-	ls = FIELD_GET(NFP_NET_CFG_STS_LINK_RATE, sts);
-	if (ls == NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED)
+	sts = nn_readw(nn, NFP_NET_CFG_STS);
+	speed = nfp_net_lr2speed(FIELD_GET(NFP_NET_CFG_STS_LINK_RATE, sts));
+	if (!speed)
 		return -EOPNOTSUPP;
 
-	if (ls == NFP_NET_CFG_STS_LINK_RATE_UNKNOWN ||
-	    ls >= ARRAY_SIZE(ls_to_ethtool))
-		return 0;
-
-	cmd->base.speed = ls_to_ethtool[ls];
-	cmd->base.duplex = DUPLEX_FULL;
+	if (speed != SPEED_UNKNOWN) {
+		cmd->base.speed = speed;
+		cmd->base.duplex = DUPLEX_FULL;
+	}
 
 	return 0;
 }
@@ -339,6 +363,7 @@ static int
 nfp_net_set_link_ksettings(struct net_device *netdev,
 			   const struct ethtool_link_ksettings *cmd)
 {
+	bool req_aneg = (cmd->base.autoneg == AUTONEG_ENABLE);
 	struct nfp_eth_table_port *eth_port;
 	struct nfp_port *port;
 	struct nfp_nsp *nsp;
@@ -358,13 +383,25 @@ nfp_net_set_link_ksettings(struct net_device *netdev,
 	if (IS_ERR(nsp))
 		return PTR_ERR(nsp);
 
-	err = __nfp_eth_set_aneg(nsp, cmd->base.autoneg == AUTONEG_ENABLE ?
-				 NFP_ANEG_AUTO : NFP_ANEG_DISABLED);
+	if (req_aneg && !eth_port->supp_aneg) {
+		netdev_warn(netdev, "Autoneg is not supported.\n");
+		err = -EOPNOTSUPP;
+		goto err_bad_set;
+	}
+
+	err = __nfp_eth_set_aneg(nsp, req_aneg ? NFP_ANEG_AUTO : NFP_ANEG_DISABLED);
 	if (err)
 		goto err_bad_set;
+
 	if (cmd->base.speed != SPEED_UNKNOWN) {
 		u32 speed = cmd->base.speed / eth_port->lanes;
 
+		if (req_aneg) {
+			netdev_err(netdev, "Speed changing is not allowed when working on autoneg mode.\n");
+			err = -EINVAL;
+			goto err_bad_set;
+		}
+
 		err = __nfp_eth_set_speed(nsp, speed);
 		if (err)
 			goto err_bad_set;
@@ -1008,7 +1045,7 @@ nfp_port_get_fecparam(struct net_device *netdev,
 		return 0;
 
 	param->fec = nfp_port_fec_nsp_to_ethtool(eth_port->fec_modes_supported);
-	param->active_fec = nfp_port_fec_nsp_to_ethtool(eth_port->fec);
+	param->active_fec = nfp_port_fec_nsp_to_ethtool(BIT(eth_port->act_fec));
 
 	return 0;
 }
@@ -1676,11 +1713,166 @@ static int nfp_net_set_phys_id(struct net_device *netdev,
 	return err;
 }
 
+#define NFP_EEPROM_LEN ETH_ALEN
+
+static int
+nfp_net_get_eeprom_len(struct net_device *netdev)
+{
+	struct nfp_eth_table_port *eth_port;
+	struct nfp_port *port;
+
+	port = nfp_port_from_netdev(netdev);
+	eth_port = __nfp_port_get_eth_port(port);
+	if (!eth_port)
+		return 0;
+
+	return NFP_EEPROM_LEN;
+}
+
+static int
+nfp_net_get_nsp_hwindex(struct net_device *netdev,
+			struct nfp_nsp **nspptr,
+			u32 *index)
+{
+	struct nfp_eth_table_port *eth_port;
+	struct nfp_port *port;
+	struct nfp_nsp *nsp;
+	int err;
+
+	port = nfp_port_from_netdev(netdev);
+	eth_port = __nfp_port_get_eth_port(port);
+	if (!eth_port)
+		return -EOPNOTSUPP;
+
+	nsp = nfp_nsp_open(port->app->cpp);
+	if (IS_ERR(nsp)) {
+		err = PTR_ERR(nsp);
+		netdev_err(netdev, "Failed to access the NSP: %d\n", err);
+		return err;
+	}
+
+	if (!nfp_nsp_has_hwinfo_lookup(nsp)) {
+		netdev_err(netdev, "NSP doesn't support PF MAC generation\n");
+		nfp_nsp_close(nsp);
+		return -EOPNOTSUPP;
+	}
+
+	*nspptr = nsp;
+	*index = eth_port->eth_index;
+
+	return 0;
+}
+
+static int
+nfp_net_get_port_mac_by_hwinfo(struct net_device *netdev,
+			       u8 *mac_addr)
+{
+	char hwinfo[32] = {};
+	struct nfp_nsp *nsp;
+	u32 index;
+	int err;
+
+	err = nfp_net_get_nsp_hwindex(netdev, &nsp, &index);
+	if (err)
+		return err;
+
+	snprintf(hwinfo, sizeof(hwinfo), "eth%u.mac", index);
+	err = nfp_nsp_hwinfo_lookup(nsp, hwinfo, sizeof(hwinfo));
+	nfp_nsp_close(nsp);
+	if (err) {
+		netdev_err(netdev, "Reading persistent MAC address failed: %d\n",
+			   err);
+		return -EOPNOTSUPP;
+	}
+
+	if (sscanf(hwinfo, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",
+		   &mac_addr[0], &mac_addr[1], &mac_addr[2],
+		   &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) {
+		netdev_err(netdev, "Can't parse persistent MAC address (%s)\n",
+			   hwinfo);
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int
+nfp_net_set_port_mac_by_hwinfo(struct net_device *netdev,
+			       u8 *mac_addr)
+{
+	char hwinfo[32] = {};
+	struct nfp_nsp *nsp;
+	u32 index;
+	int err;
+
+	err = nfp_net_get_nsp_hwindex(netdev, &nsp, &index);
+	if (err)
+		return err;
+
+	snprintf(hwinfo, sizeof(hwinfo),
+		 "eth%u.mac=%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx",
+		 index, mac_addr[0], mac_addr[1], mac_addr[2], mac_addr[3],
+		 mac_addr[4], mac_addr[5]);
+
+	err = nfp_nsp_hwinfo_set(nsp, hwinfo, sizeof(hwinfo));
+	nfp_nsp_close(nsp);
+	if (err) {
+		netdev_err(netdev, "HWinfo set failed: %d, hwinfo: %s\n",
+			   err, hwinfo);
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int
+nfp_net_get_eeprom(struct net_device *netdev,
+		   struct ethtool_eeprom *eeprom, u8 *bytes)
+{
+	struct nfp_net *nn = netdev_priv(netdev);
+	u8 buf[NFP_EEPROM_LEN] = {};
+
+	if (eeprom->len == 0)
+		return -EINVAL;
+
+	if (nfp_net_get_port_mac_by_hwinfo(netdev, buf))
+		return -EOPNOTSUPP;
+
+	eeprom->magic = nn->pdev->vendor | (nn->pdev->device << 16);
+	memcpy(bytes, buf + eeprom->offset, eeprom->len);
+
+	return 0;
+}
+
+static int
+nfp_net_set_eeprom(struct net_device *netdev,
+		   struct ethtool_eeprom *eeprom, u8 *bytes)
+{
+	struct nfp_net *nn = netdev_priv(netdev);
+	u8 buf[NFP_EEPROM_LEN] = {};
+
+	if (eeprom->len == 0)
+		return -EINVAL;
+
+	if (eeprom->magic != (nn->pdev->vendor | nn->pdev->device << 16))
+		return -EINVAL;
+
+	if (nfp_net_get_port_mac_by_hwinfo(netdev, buf))
+		return -EOPNOTSUPP;
+
+	memcpy(buf + eeprom->offset, bytes, eeprom->len);
+	if (nfp_net_set_port_mac_by_hwinfo(netdev, buf))
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
 static const struct ethtool_ops nfp_net_ethtool_ops = {
 	.supported_coalesce_params = ETHTOOL_COALESCE_USECS |
 				     ETHTOOL_COALESCE_MAX_FRAMES |
 				     ETHTOOL_COALESCE_USE_ADAPTIVE,
 	.get_drvinfo		= nfp_net_get_drvinfo,
+	.nway_reset             = nfp_net_nway_reset,
 	.get_link		= ethtool_op_get_link,
 	.get_ringparam		= nfp_net_get_ringparam,
 	.set_ringparam		= nfp_net_set_ringparam,
@@ -1699,6 +1891,9 @@ static const struct ethtool_ops nfp_net_ethtool_ops = {
 	.set_dump		= nfp_app_set_dump,
 	.get_dump_flag		= nfp_app_get_dump_flag,
 	.get_dump_data		= nfp_app_get_dump_data,
+	.get_eeprom_len         = nfp_net_get_eeprom_len,
+	.get_eeprom             = nfp_net_get_eeprom,
+	.set_eeprom             = nfp_net_set_eeprom,
 	.get_module_info	= nfp_port_get_module_info,
 	.get_module_eeprom	= nfp_port_get_module_eeprom,
 	.get_coalesce           = nfp_net_get_coalesce,
@@ -1715,6 +1910,7 @@ static const struct ethtool_ops nfp_net_ethtool_ops = {
 
 const struct ethtool_ops nfp_port_ethtool_ops = {
 	.get_drvinfo		= nfp_app_get_drvinfo,
+	.nway_reset             = nfp_net_nway_reset,
 	.get_link		= ethtool_op_get_link,
 	.get_strings		= nfp_port_get_strings,
 	.get_ethtool_stats	= nfp_port_get_stats,
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
index ca4e05650fe6..3bae92dc899e 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c
@@ -77,12 +77,6 @@ static int nfp_net_pf_get_num_ports(struct nfp_pf *pf)
 	return nfp_pf_rtsym_read_optional(pf, "nfd_cfg_pf%u_num_ports", 1);
 }
 
-static int nfp_net_pf_get_app_id(struct nfp_pf *pf)
-{
-	return nfp_pf_rtsym_read_optional(pf, "_pf%u_net_app_id",
-					  NFP_APP_CORE_NIC);
-}
-
 static void nfp_net_pf_free_vnic(struct nfp_pf *pf, struct nfp_net *nn)
 {
 	if (nfp_net_is_data_vnic(nn))
@@ -202,6 +196,9 @@ nfp_net_pf_alloc_vnics(struct nfp_pf *pf, void __iomem *ctrl_bar,
 			goto err_free_prev;
 		}
 
+		if (nn->port)
+			nn->port->link_cb = nfp_net_refresh_port_table;
+
 		ctrl_bar += NFP_PF_CSR_SLICE_SIZE;
 
 		/* Kill the vNIC if app init marked it as invalid */
@@ -523,6 +520,57 @@ err_unmap_ctrl:
 	return err;
 }
 
+static const unsigned int lr_to_speed[] = {
+	[NFP_NET_CFG_STS_LINK_RATE_UNSUPPORTED]	= 0,
+	[NFP_NET_CFG_STS_LINK_RATE_UNKNOWN]	= SPEED_UNKNOWN,
+	[NFP_NET_CFG_STS_LINK_RATE_1G]		= SPEED_1000,
+	[NFP_NET_CFG_STS_LINK_RATE_10G]		= SPEED_10000,
+	[NFP_NET_CFG_STS_LINK_RATE_25G]		= SPEED_25000,
+	[NFP_NET_CFG_STS_LINK_RATE_40G]		= SPEED_40000,
+	[NFP_NET_CFG_STS_LINK_RATE_50G]		= SPEED_50000,
+	[NFP_NET_CFG_STS_LINK_RATE_100G]	= SPEED_100000,
+};
+
+unsigned int nfp_net_lr2speed(unsigned int linkrate)
+{
+	if (linkrate < ARRAY_SIZE(lr_to_speed))
+		return lr_to_speed[linkrate];
+
+	return SPEED_UNKNOWN;
+}
+
+unsigned int nfp_net_speed2lr(unsigned int speed)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(lr_to_speed); i++) {
+		if (speed == lr_to_speed[i])
+			return i;
+	}
+
+	return NFP_NET_CFG_STS_LINK_RATE_UNKNOWN;
+}
+
+static void nfp_net_notify_port_speed(struct nfp_port *port)
+{
+	struct net_device *netdev = port->netdev;
+	struct nfp_net *nn;
+	u16 sts;
+
+	if (!nfp_netdev_is_nfp_net(netdev))
+		return;
+
+	nn = netdev_priv(netdev);
+	sts = nn_readw(nn, NFP_NET_CFG_STS);
+
+	if (!(sts & NFP_NET_CFG_STS_LINK)) {
+		nn_writew(nn, NFP_NET_CFG_STS_NSP_LINK_RATE, NFP_NET_CFG_STS_LINK_RATE_UNKNOWN);
+		return;
+	}
+
+	nn_writew(nn, NFP_NET_CFG_STS_NSP_LINK_RATE, nfp_net_speed2lr(port->eth_port->speed));
+}
+
 static int
 nfp_net_eth_port_update(struct nfp_cpp *cpp, struct nfp_port *port,
 			struct nfp_eth_table *eth_table)
@@ -544,6 +592,7 @@ nfp_net_eth_port_update(struct nfp_cpp *cpp, struct nfp_port *port,
 	}
 
 	memcpy(port->eth_port, eth_port, sizeof(*eth_port));
+	nfp_net_notify_port_speed(port);
 
 	return 0;
 }
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.h b/drivers/net/ethernet/netronome/nfp/nfp_port.h
index d1ebe6c72f7f..6793cdf9ff11 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_port.h
+++ b/drivers/net/ethernet/netronome/nfp/nfp_port.h
@@ -46,6 +46,7 @@ enum nfp_port_flags {
  * @tc_offload_cnt:	number of active TC offloads, how offloads are counted
  *			is not defined, use as a boolean
  * @app:	backpointer to the app structure
+ * @link_cb:	callback when link status changed
  * @dl_port:	devlink port structure
  * @eth_id:	for %NFP_PORT_PHYS_PORT port ID in NFP enumeration scheme
  * @eth_forced:	for %NFP_PORT_PHYS_PORT port is forced UP or DOWN, don't change
@@ -66,6 +67,7 @@ struct nfp_port {
 	unsigned long tc_offload_cnt;
 
 	struct nfp_app *app;
+	void (*link_cb)(struct nfp_port *port);
 
 	struct devlink_port dl_port;
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h
index 77d66855be42..992d72ac98d3 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp.h
@@ -132,6 +132,7 @@ enum nfp_eth_fec {
  * @ports.interface:	interface (module) plugged in
  * @ports.media:	media type of the @interface
  * @ports.fec:		forward error correction mode
+ * @ports.act_fec:	active forward error correction mode
  * @ports.aneg:		auto negotiation mode
  * @ports.mac_addr:	interface MAC address
  * @ports.label_port:	port id
@@ -162,6 +163,7 @@ struct nfp_eth_table {
 		enum nfp_eth_media media;
 
 		enum nfp_eth_fec fec;
+		enum nfp_eth_fec act_fec;
 		enum nfp_eth_aneg aneg;
 
 		u8 mac_addr[ETH_ALEN];
@@ -172,6 +174,7 @@ struct nfp_eth_table {
 		bool enabled;
 		bool tx_enabled;
 		bool rx_enabled;
+		bool supp_aneg;
 
 		bool override_changed;
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c
index 4cc38799eabc..bb64efec4c46 100644
--- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c
+++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nsp_eth.c
@@ -27,6 +27,7 @@
 #define NSP_ETH_PORT_PHYLABEL		GENMASK_ULL(59, 54)
 #define NSP_ETH_PORT_FEC_SUPP_BASER	BIT_ULL(60)
 #define NSP_ETH_PORT_FEC_SUPP_RS	BIT_ULL(61)
+#define NSP_ETH_PORT_SUPP_ANEG		BIT_ULL(63)
 
 #define NSP_ETH_PORT_LANES_MASK		cpu_to_le64(NSP_ETH_PORT_LANES)
 
@@ -40,6 +41,7 @@
 #define NSP_ETH_STATE_OVRD_CHNG		BIT_ULL(22)
 #define NSP_ETH_STATE_ANEG		GENMASK_ULL(25, 23)
 #define NSP_ETH_STATE_FEC		GENMASK_ULL(27, 26)
+#define NSP_ETH_STATE_ACT_FEC		GENMASK_ULL(29, 28)
 
 #define NSP_ETH_CTRL_CONFIGURED		BIT_ULL(0)
 #define NSP_ETH_CTRL_ENABLED		BIT_ULL(1)
@@ -170,7 +172,14 @@ nfp_eth_port_translate(struct nfp_nsp *nsp, const union eth_table_entry *src,
 	if (dst->fec_modes_supported)
 		dst->fec_modes_supported |= NFP_FEC_AUTO | NFP_FEC_DISABLED;
 
-	dst->fec = 1 << FIELD_GET(NSP_ETH_STATE_FEC, state);
+	dst->fec = FIELD_GET(NSP_ETH_STATE_FEC, state);
+	dst->act_fec = dst->fec;
+
+	if (nfp_nsp_get_abi_ver_minor(nsp) < 33)
+		return;
+
+	dst->act_fec = FIELD_GET(NSP_ETH_STATE_ACT_FEC, state);
+	dst->supp_aneg = FIELD_GET(NSP_ETH_PORT_SUPP_ANEG, port);
 }
 
 static void
diff --git a/drivers/net/ethernet/ni/nixge.c b/drivers/net/ethernet/ni/nixge.c
index 4b3482ce90a1..3db4a2431741 100644
--- a/drivers/net/ethernet/ni/nixge.c
+++ b/drivers/net/ethernet/ni/nixge.c
@@ -990,8 +990,8 @@ static const struct net_device_ops nixge_netdev_ops = {
 static void nixge_ethtools_get_drvinfo(struct net_device *ndev,
 				       struct ethtool_drvinfo *ed)
 {
-	strlcpy(ed->driver, "nixge", sizeof(ed->driver));
-	strlcpy(ed->bus_info, "platform", sizeof(ed->bus_info));
+	strscpy(ed->driver, "nixge", sizeof(ed->driver));
+	strscpy(ed->bus_info, "platform", sizeof(ed->bus_info));
 }
 
 static int
@@ -1294,7 +1294,7 @@ static int nixge_probe(struct platform_device *pdev)
 	priv->ndev = ndev;
 	priv->dev = &pdev->dev;
 
-	netif_napi_add(ndev, &priv->napi, nixge_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &priv->napi, nixge_poll);
 	err = nixge_of_get_resources(pdev);
 	if (err)
 		goto free_netdev;
diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c
index 5116badaf091..daa028729d44 100644
--- a/drivers/net/ethernet/nvidia/forcedeth.c
+++ b/drivers/net/ethernet/nvidia/forcedeth.c
@@ -4291,9 +4291,9 @@ static void nv_do_stats_poll(struct timer_list *t)
 static void nv_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
 	struct fe_priv *np = netdev_priv(dev);
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, FORCEDETH_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, FORCEDETH_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
 }
 
 static void nv_get_wol(struct net_device *dev, struct ethtool_wolinfo *wolinfo)
@@ -5876,7 +5876,7 @@ static int nv_probe(struct pci_dev *pci_dev, const struct pci_device_id *id)
 	else
 		dev->netdev_ops = &nv_netdev_ops_optimized;
 
-	netif_napi_add(dev, &np->napi, nv_napi_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &np->napi, nv_napi_poll);
 	dev->ethtool_ops = &ops;
 	dev->watchdog_timeo = NV_WATCHDOG_TIMEO;
 
diff --git a/drivers/net/ethernet/nxp/lpc_eth.c b/drivers/net/ethernet/nxp/lpc_eth.c
index f606d75b33b4..1a4a272f4c5c 100644
--- a/drivers/net/ethernet/nxp/lpc_eth.c
+++ b/drivers/net/ethernet/nxp/lpc_eth.c
@@ -1184,9 +1184,9 @@ static int lpc_eth_open(struct net_device *ndev)
 static void lpc_eth_ethtool_getdrvinfo(struct net_device *ndev,
 	struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, MODNAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, dev_name(ndev->dev.parent),
+	strscpy(info->driver, MODNAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, dev_name(ndev->dev.parent),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c
index 84cc79e928c8..541b8bcd3223 100644
--- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c
+++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c
@@ -169,9 +169,9 @@ static void pch_gbe_get_drvinfo(struct net_device *netdev,
 {
 	struct pch_gbe_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, pch_driver_version, sizeof(drvinfo->version));
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, pch_driver_version, sizeof(drvinfo->version));
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
index 46da937ad27f..3f2c30184752 100644
--- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
+++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c
@@ -2516,8 +2516,7 @@ static int pch_gbe_probe(struct pci_dev *pdev,
 
 	netdev->netdev_ops = &pch_gbe_netdev_ops;
 	netdev->watchdog_timeo = PCH_GBE_WATCHDOG_PERIOD;
-	netif_napi_add(netdev, &adapter->napi,
-		       pch_gbe_napi_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, &adapter->napi, pch_gbe_napi_poll);
 	netdev->hw_features = NETIF_F_RXCSUM |
 		NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
 	netdev->features = netdev->hw_features;
diff --git a/drivers/net/ethernet/packetengines/hamachi.c b/drivers/net/ethernet/packetengines/hamachi.c
index 9c408328be0d..1cc001087193 100644
--- a/drivers/net/ethernet/packetengines/hamachi.c
+++ b/drivers/net/ethernet/packetengines/hamachi.c
@@ -1819,9 +1819,9 @@ static void hamachi_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *
 {
 	struct hamachi_private *np = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
 }
 
 static int hamachi_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/packetengines/yellowfin.c b/drivers/net/ethernet/packetengines/yellowfin.c
index 03650022d444..640ac01689fb 100644
--- a/drivers/net/ethernet/packetengines/yellowfin.c
+++ b/drivers/net/ethernet/packetengines/yellowfin.c
@@ -1340,9 +1340,9 @@ static void yellowfin_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo
 {
 	struct yellowfin_private *np = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
 }
 
 static const struct ethtool_ops ethtool_ops = {
diff --git a/drivers/net/ethernet/pasemi/pasemi_mac.c b/drivers/net/ethernet/pasemi/pasemi_mac.c
index f0ace3a0e85c..aaab590ef548 100644
--- a/drivers/net/ethernet/pasemi/pasemi_mac.c
+++ b/drivers/net/ethernet/pasemi/pasemi_mac.c
@@ -1697,7 +1697,7 @@ pasemi_mac_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	mac->pdev = pdev;
 	mac->netdev = dev;
 
-	netif_napi_add(dev, &mac->napi, pasemi_mac_poll, 64);
+	netif_napi_add(dev, &mac->napi, pasemi_mac_poll);
 
 	dev->features = NETIF_F_IP_CSUM | NETIF_F_LLTX | NETIF_F_SG |
 			NETIF_F_HIGHDMA | NETIF_F_GSO;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
index 0a7a757494bc..ce436e97324a 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
@@ -320,16 +320,16 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 			dev_err(dev, "Cannot enable existing VFs: %d\n", err);
 	}
 
-	err = ionic_lif_register(ionic->lif);
+	err = ionic_devlink_register(ionic);
 	if (err) {
-		dev_err(dev, "Cannot register LIF: %d, aborting\n", err);
+		dev_err(dev, "Cannot register devlink: %d\n", err);
 		goto err_out_deinit_lifs;
 	}
 
-	err = ionic_devlink_register(ionic);
+	err = ionic_lif_register(ionic->lif);
 	if (err) {
-		dev_err(dev, "Cannot register devlink: %d\n", err);
-		goto err_out_deregister_lifs;
+		dev_err(dev, "Cannot register LIF: %d, aborting\n", err);
+		goto err_out_deregister_devlink;
 	}
 
 	mod_timer(&ionic->watchdog_timer,
@@ -337,8 +337,8 @@ static int ionic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	return 0;
 
-err_out_deregister_lifs:
-	ionic_lif_unregister(ionic->lif);
+err_out_deregister_devlink:
+	ionic_devlink_unregister(ionic);
 err_out_deinit_lifs:
 	ionic_vf_dealloc(ionic);
 	ionic_lif_deinit(ionic->lif);
@@ -380,8 +380,8 @@ static void ionic_remove(struct pci_dev *pdev)
 	del_timer_sync(&ionic->watchdog_timer);
 
 	if (ionic->lif) {
-		ionic_devlink_unregister(ionic);
 		ionic_lif_unregister(ionic->lif);
+		ionic_devlink_unregister(ionic);
 		ionic_lif_deinit(ionic->lif);
 		ionic_lif_free(ionic->lif);
 		ionic->lif = NULL;
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
index 0be79c516781..5d58fd99be3c 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c
@@ -774,8 +774,7 @@ static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 	dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
 
 	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
-		netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);
 
 	qcq->flags |= IONIC_QCQ_F_INITED;
 
@@ -830,11 +829,9 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 	dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
 
 	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
-		netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);
 	else
-		netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);
 
 	qcq->flags |= IONIC_QCQ_F_INITED;
 
@@ -3165,8 +3162,7 @@ static int ionic_lif_adminq_init(struct ionic_lif *lif)
 	dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
 	dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
 
-	netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);
 
 	napi_enable(&qcq->napi);
 
diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c
index 3c4a84ea6321..8c4cb910e09b 100644
--- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c
+++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c
@@ -65,9 +65,9 @@ netxen_nic_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo)
 	u32 fw_minor = 0;
 	u32 fw_build = 0;
 
-	strlcpy(drvinfo->driver, netxen_nic_driver_name,
+	strscpy(drvinfo->driver, netxen_nic_driver_name,
 		sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, NETXEN_NIC_LINUX_VERSIONID,
+	strscpy(drvinfo->version, NETXEN_NIC_LINUX_VERSIONID,
 		sizeof(drvinfo->version));
 	fw_major = NXRD32(adapter, NETXEN_FW_VERSION_MAJOR);
 	fw_minor = NXRD32(adapter, NETXEN_FW_VERSION_MINOR);
@@ -75,7 +75,7 @@ netxen_nic_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo)
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
 		"%d.%d.%d", fw_major, fw_minor, fw_build);
 
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
index 4e6f00af17d9..de8d54b23f73 100644
--- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
+++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c
@@ -173,8 +173,7 @@ netxen_napi_add(struct netxen_adapter *adapter, struct net_device *netdev)
 
 	for (ring = 0; ring < adapter->max_sds_rings; ring++) {
 		sds_ring = &recv_ctx->sds_rings[ring];
-		netif_napi_add(netdev, &sds_ring->napi,
-				netxen_nic_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(netdev, &sds_ring->napi, netxen_nic_poll);
 	}
 
 	return 0;
diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.c b/drivers/net/ethernet/qlogic/qed/qed_int.c
index d701ecd3ba00..2661c483c67e 100644
--- a/drivers/net/ethernet/qlogic/qed/qed_int.c
+++ b/drivers/net/ethernet/qlogic/qed/qed_int.c
@@ -1119,7 +1119,7 @@ static int qed_int_deassertion(struct qed_hwfn  *p_hwfn,
 						snprintf(bit_name, 30,
 							 p_aeu->bit_name, num);
 					else
-						strlcpy(bit_name,
+						strscpy(bit_name,
 							p_aeu->bit_name, 30);
 
 					/* We now need to pass bitmask in its
diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
index 97a7ab0826ed..8034d812d5a0 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c
@@ -624,7 +624,7 @@ static void qede_get_drvinfo(struct net_device *ndev,
 	struct qede_dev *edev = netdev_priv(ndev);
 	char mbi[ETHTOOL_FWVERS_LEN];
 
-	strlcpy(info->driver, "qede", sizeof(info->driver));
+	strscpy(info->driver, "qede", sizeof(info->driver));
 
 	snprintf(storm, ETHTOOL_FWVERS_LEN, "%d.%d.%d.%d",
 		 edev->dev_info.common.fw_major,
@@ -661,7 +661,7 @@ static void qede_get_drvinfo(struct net_device *ndev,
 			 "mfw %s", mfw);
 	}
 
-	strlcpy(info->bus_info, pci_name(edev->pdev), sizeof(info->bus_info));
+	strscpy(info->bus_info, pci_name(edev->pdev), sizeof(info->bus_info));
 }
 
 static void qede_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol)
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index f56b679adb4b..953f304b8588 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -1214,7 +1214,7 @@ static int __qede_probe(struct pci_dev *pdev, u32 dp_module, u8 dp_level,
 	/* Start the Slowpath-process */
 	memset(&sp_params, 0, sizeof(sp_params));
 	sp_params.int_mode = QED_INT_MODE_MSIX;
-	strlcpy(sp_params.name, "qede LAN", QED_DRV_VER_STR_SIZE);
+	strscpy(sp_params.name, "qede LAN", QED_DRV_VER_STR_SIZE);
 	rc = qed_ops->common->slowpath_start(cdev, &sp_params);
 	if (rc) {
 		pr_notice("Cannot start slowpath\n");
@@ -1904,8 +1904,7 @@ static void qede_napi_add_enable(struct qede_dev *edev)
 
 	/* Add NAPI objects */
 	for_each_queue(i) {
-		netif_napi_add(edev->ndev, &edev->fp_array[i].napi,
-			       qede_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(edev->ndev, &edev->fp_array[i].napi, qede_poll);
 		napi_enable(&edev->fp_array[i].napi);
 	}
 }
diff --git a/drivers/net/ethernet/qlogic/qla3xxx.c b/drivers/net/ethernet/qlogic/qla3xxx.c
index 06f4d9a9e938..76072f8c3d2f 100644
--- a/drivers/net/ethernet/qlogic/qla3xxx.c
+++ b/drivers/net/ethernet/qlogic/qla3xxx.c
@@ -1736,10 +1736,10 @@ static void ql_get_drvinfo(struct net_device *ndev,
 			   struct ethtool_drvinfo *drvinfo)
 {
 	struct ql3_adapter *qdev = netdev_priv(ndev);
-	strlcpy(drvinfo->driver, ql3xxx_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, ql3xxx_driver_version,
+	strscpy(drvinfo->driver, ql3xxx_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, ql3xxx_driver_version,
 		sizeof(drvinfo->version));
-	strlcpy(drvinfo->bus_info, pci_name(qdev->pdev),
+	strscpy(drvinfo->bus_info, pci_name(qdev->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
@@ -3813,7 +3813,7 @@ static int ql3xxx_probe(struct pci_dev *pdev,
 	ndev->ethtool_ops = &ql3xxx_ethtool_ops;
 	ndev->watchdog_timeo = 5 * HZ;
 
-	netif_napi_add(ndev, &qdev->napi, ql_poll, 64);
+	netif_napi_add(ndev, &qdev->napi, ql_poll);
 
 	ndev->irq = pdev->irq;
 
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c
index 54a2d653be63..1ee491f78c6b 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c
@@ -277,10 +277,10 @@ qlcnic_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *drvinfo)
 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
 		"%d.%d.%d", fw_major, fw_minor, fw_build);
 
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
-	strlcpy(drvinfo->driver, qlcnic_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, QLCNIC_LINUX_VERSIONID,
+	strscpy(drvinfo->driver, qlcnic_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, QLCNIC_LINUX_VERSIONID,
 		sizeof(drvinfo->version));
 }
 
diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c
index 9da5e97f8a0a..92930a055cbc 100644
--- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c
+++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c
@@ -1586,17 +1586,15 @@ int qlcnic_82xx_napi_add(struct qlcnic_adapter *adapter,
 		sds_ring = &recv_ctx->sds_rings[ring];
 		if (qlcnic_check_multi_tx(adapter) &&
 		    !adapter->ahw->diag_test) {
-			netif_napi_add(netdev, &sds_ring->napi, qlcnic_rx_poll,
-				       NAPI_POLL_WEIGHT);
+			netif_napi_add(netdev, &sds_ring->napi,
+				       qlcnic_rx_poll);
 		} else {
 			if (ring == (adapter->drv_sds_rings - 1))
 				netif_napi_add(netdev, &sds_ring->napi,
-					       qlcnic_poll,
-					       NAPI_POLL_WEIGHT);
+					       qlcnic_poll);
 			else
 				netif_napi_add(netdev, &sds_ring->napi,
-					       qlcnic_rx_poll,
-					       NAPI_POLL_WEIGHT);
+					       qlcnic_rx_poll);
 		}
 	}
 
@@ -2115,17 +2113,14 @@ int qlcnic_83xx_napi_add(struct qlcnic_adapter *adapter,
 		if (adapter->flags & QLCNIC_MSIX_ENABLED) {
 			if (!(adapter->flags & QLCNIC_TX_INTR_SHARED))
 				netif_napi_add(netdev, &sds_ring->napi,
-					       qlcnic_83xx_rx_poll,
-					       NAPI_POLL_WEIGHT);
+					       qlcnic_83xx_rx_poll);
 			else
 				netif_napi_add(netdev, &sds_ring->napi,
-					       qlcnic_83xx_msix_sriov_vf_poll,
-					       NAPI_POLL_WEIGHT);
+					       qlcnic_83xx_msix_sriov_vf_poll);
 
 		} else {
 			netif_napi_add(netdev, &sds_ring->napi,
-				       qlcnic_83xx_poll,
-				       NAPI_POLL_WEIGHT);
+				       qlcnic_83xx_poll);
 		}
 	}
 
diff --git a/drivers/net/ethernet/qualcomm/emac/emac.c b/drivers/net/ethernet/qualcomm/emac/emac.c
index a55c52696d49..3115b2c12898 100644
--- a/drivers/net/ethernet/qualcomm/emac/emac.c
+++ b/drivers/net/ethernet/qualcomm/emac/emac.c
@@ -684,8 +684,7 @@ static int emac_probe(struct platform_device *pdev)
 	/* Initialize queues */
 	emac_mac_rx_tx_ring_init_all(pdev, adpt);
 
-	netif_napi_add(netdev, &adpt->rx_q.napi, emac_napi_rtx,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, &adpt->rx_q.napi, emac_napi_rtx);
 
 	ret = register_netdev(netdev);
 	if (ret) {
diff --git a/drivers/net/ethernet/qualcomm/qca_debug.c b/drivers/net/ethernet/qualcomm/qca_debug.c
index 792ce9a323cd..f62c39544e08 100644
--- a/drivers/net/ethernet/qualcomm/qca_debug.c
+++ b/drivers/net/ethernet/qualcomm/qca_debug.c
@@ -164,10 +164,10 @@ qcaspi_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *p)
 {
 	struct qcaspi *qca = netdev_priv(dev);
 
-	strlcpy(p->driver, QCASPI_DRV_NAME, sizeof(p->driver));
-	strlcpy(p->version, QCASPI_DRV_VERSION, sizeof(p->version));
-	strlcpy(p->fw_version, "QCA7000", sizeof(p->fw_version));
-	strlcpy(p->bus_info, dev_name(&qca->spi_dev->dev),
+	strscpy(p->driver, QCASPI_DRV_NAME, sizeof(p->driver));
+	strscpy(p->version, QCASPI_DRV_VERSION, sizeof(p->version));
+	strscpy(p->fw_version, "QCA7000", sizeof(p->fw_version));
+	strscpy(p->bus_info, dev_name(&qca->spi_dev->dev),
 		sizeof(p->bus_info));
 }
 
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
index e5a0b38f7dbe..2b033060fc20 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h
@@ -19,7 +19,7 @@ struct rmnet_map_control_command {
 			__be16 flow_control_seq_num;
 			__be32 qos_id;
 		} flow_control;
-		u8 data[0];
+		DECLARE_FLEX_ARRAY(u8, data);
 	};
 }  __aligned(1);
 
diff --git a/drivers/net/ethernet/rdc/r6040.c b/drivers/net/ethernet/rdc/r6040.c
index a6bf7d505178..eecd52ed1ed2 100644
--- a/drivers/net/ethernet/rdc/r6040.c
+++ b/drivers/net/ethernet/rdc/r6040.c
@@ -939,9 +939,9 @@ static void netdev_get_drvinfo(struct net_device *dev,
 {
 	struct r6040_private *rp = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(rp->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(rp->pdev), sizeof(info->bus_info));
 }
 
 static const struct ethtool_ops netdev_ethtool_ops = {
@@ -1127,7 +1127,7 @@ static int r6040_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	dev->ethtool_ops = &netdev_ethtool_ops;
 	dev->watchdog_timeo = TX_TIMEOUT;
 
-	netif_napi_add(dev, &lp->napi, r6040_poll, 64);
+	netif_napi_add(dev, &lp->napi, r6040_poll);
 
 	lp->mii_bus = mdiobus_alloc();
 	if (!lp->mii_bus) {
diff --git a/drivers/net/ethernet/realtek/8139cp.c b/drivers/net/ethernet/realtek/8139cp.c
index e0feeec13da6..f5786d78ed23 100644
--- a/drivers/net/ethernet/realtek/8139cp.c
+++ b/drivers/net/ethernet/realtek/8139cp.c
@@ -1382,9 +1382,9 @@ static void cp_get_drvinfo (struct net_device *dev, struct ethtool_drvinfo *info
 {
 	struct cp_private *cp = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(cp->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(cp->pdev), sizeof(info->bus_info));
 }
 
 static void cp_get_ringparam(struct net_device *dev,
diff --git a/drivers/net/ethernet/realtek/8139too.c b/drivers/net/ethernet/realtek/8139too.c
index 15b40fd93cd2..469e2e229c6e 100644
--- a/drivers/net/ethernet/realtek/8139too.c
+++ b/drivers/net/ethernet/realtek/8139too.c
@@ -1002,7 +1002,7 @@ static int rtl8139_init_one(struct pci_dev *pdev,
 	dev->netdev_ops = &rtl8139_netdev_ops;
 	dev->ethtool_ops = &rtl8139_ethtool_ops;
 	dev->watchdog_timeo = TX_TIMEOUT;
-	netif_napi_add(dev, &tp->napi, rtl8139_poll, 64);
+	netif_napi_add(dev, &tp->napi, rtl8139_poll);
 
 	/* note: the hardware is not capable of sg/csum/highdma, however
 	 * through the use of skb_copy_and_csum_dev we enable these
@@ -2380,9 +2380,9 @@ static int rtl8139_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
 static void rtl8139_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
 	struct rtl8139_private *tp = netdev_priv(dev);
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info));
 }
 
 static int rtl8139_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/realtek/r8169.h b/drivers/net/ethernet/realtek/r8169.h
index 8da4b66b71b5..55ef8251feb5 100644
--- a/drivers/net/ethernet/realtek/r8169.h
+++ b/drivers/net/ethernet/realtek/r8169.h
@@ -23,10 +23,10 @@ enum mac_version {
 	RTL_GIGA_MAC_VER_09,
 	RTL_GIGA_MAC_VER_10,
 	RTL_GIGA_MAC_VER_11,
-	RTL_GIGA_MAC_VER_12,
-	RTL_GIGA_MAC_VER_13,
+	/* RTL_GIGA_MAC_VER_12 was handled the same as VER_17 */
+	/* RTL_GIGA_MAC_VER_13 was merged with VER_10 */
 	RTL_GIGA_MAC_VER_14,
-	RTL_GIGA_MAC_VER_16,
+	/* RTL_GIGA_MAC_VER_16 was merged with VER_10 */
 	RTL_GIGA_MAC_VER_17,
 	RTL_GIGA_MAC_VER_18,
 	RTL_GIGA_MAC_VER_19,
@@ -51,20 +51,20 @@ enum mac_version {
 	RTL_GIGA_MAC_VER_38,
 	RTL_GIGA_MAC_VER_39,
 	RTL_GIGA_MAC_VER_40,
-	RTL_GIGA_MAC_VER_41,
+	/* support for RTL_GIGA_MAC_VER_41 has been removed */
 	RTL_GIGA_MAC_VER_42,
 	RTL_GIGA_MAC_VER_43,
 	RTL_GIGA_MAC_VER_44,
-	RTL_GIGA_MAC_VER_45,
+	/* support for RTL_GIGA_MAC_VER_45 has been removed */
 	RTL_GIGA_MAC_VER_46,
-	RTL_GIGA_MAC_VER_47,
+	/* support for RTL_GIGA_MAC_VER_47 has been removed */
 	RTL_GIGA_MAC_VER_48,
-	RTL_GIGA_MAC_VER_49,
-	RTL_GIGA_MAC_VER_50,
+	/* support for RTL_GIGA_MAC_VER_49 has been removed */
+	/* support for RTL_GIGA_MAC_VER_50 has been removed */
 	RTL_GIGA_MAC_VER_51,
 	RTL_GIGA_MAC_VER_52,
 	RTL_GIGA_MAC_VER_53,
-	RTL_GIGA_MAC_VER_60,
+	/* support for RTL_GIGA_MAC_VER_60 has been removed */
 	RTL_GIGA_MAC_VER_61,
 	RTL_GIGA_MAC_VER_63,
 	RTL_GIGA_MAC_NONE
diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c
index 1b7fdb4f056b..a73d061d9fcb 100644
--- a/drivers/net/ethernet/realtek/r8169_main.c
+++ b/drivers/net/ethernet/realtek/r8169_main.c
@@ -49,10 +49,8 @@
 #define FIRMWARE_8106E_2	"rtl_nic/rtl8106e-2.fw"
 #define FIRMWARE_8168G_2	"rtl_nic/rtl8168g-2.fw"
 #define FIRMWARE_8168G_3	"rtl_nic/rtl8168g-3.fw"
-#define FIRMWARE_8168H_1	"rtl_nic/rtl8168h-1.fw"
 #define FIRMWARE_8168H_2	"rtl_nic/rtl8168h-2.fw"
 #define FIRMWARE_8168FP_3	"rtl_nic/rtl8168fp-3.fw"
-#define FIRMWARE_8107E_1	"rtl_nic/rtl8107e-1.fw"
 #define FIRMWARE_8107E_2	"rtl_nic/rtl8107e-2.fw"
 #define FIRMWARE_8125A_3	"rtl_nic/rtl8125a-3.fw"
 #define FIRMWARE_8125B_2	"rtl_nic/rtl8125b-2.fw"
@@ -102,12 +100,9 @@ static const struct {
 	[RTL_GIGA_MAC_VER_07] = {"RTL8102e"				},
 	[RTL_GIGA_MAC_VER_08] = {"RTL8102e"				},
 	[RTL_GIGA_MAC_VER_09] = {"RTL8102e/RTL8103e"			},
-	[RTL_GIGA_MAC_VER_10] = {"RTL8101e"				},
+	[RTL_GIGA_MAC_VER_10] = {"RTL8101e/RTL8100e"			},
 	[RTL_GIGA_MAC_VER_11] = {"RTL8168b/8111b"			},
-	[RTL_GIGA_MAC_VER_12] = {"RTL8168b/8111b"			},
-	[RTL_GIGA_MAC_VER_13] = {"RTL8101e/RTL8100e"			},
 	[RTL_GIGA_MAC_VER_14] = {"RTL8401"				},
-	[RTL_GIGA_MAC_VER_16] = {"RTL8101e"				},
 	[RTL_GIGA_MAC_VER_17] = {"RTL8168b/8111b"			},
 	[RTL_GIGA_MAC_VER_18] = {"RTL8168cp/8111cp"			},
 	[RTL_GIGA_MAC_VER_19] = {"RTL8168c/8111c"			},
@@ -131,20 +126,14 @@ static const struct {
 	[RTL_GIGA_MAC_VER_38] = {"RTL8411",		FIRMWARE_8411_1 },
 	[RTL_GIGA_MAC_VER_39] = {"RTL8106e",		FIRMWARE_8106E_1},
 	[RTL_GIGA_MAC_VER_40] = {"RTL8168g/8111g",	FIRMWARE_8168G_2},
-	[RTL_GIGA_MAC_VER_41] = {"RTL8168g/8111g"			},
 	[RTL_GIGA_MAC_VER_42] = {"RTL8168gu/8111gu",	FIRMWARE_8168G_3},
 	[RTL_GIGA_MAC_VER_43] = {"RTL8106eus",		FIRMWARE_8106E_2},
 	[RTL_GIGA_MAC_VER_44] = {"RTL8411b",		FIRMWARE_8411_2 },
-	[RTL_GIGA_MAC_VER_45] = {"RTL8168h/8111h",	FIRMWARE_8168H_1},
 	[RTL_GIGA_MAC_VER_46] = {"RTL8168h/8111h",	FIRMWARE_8168H_2},
-	[RTL_GIGA_MAC_VER_47] = {"RTL8107e",		FIRMWARE_8107E_1},
 	[RTL_GIGA_MAC_VER_48] = {"RTL8107e",		FIRMWARE_8107E_2},
-	[RTL_GIGA_MAC_VER_49] = {"RTL8168ep/8111ep"			},
-	[RTL_GIGA_MAC_VER_50] = {"RTL8168ep/8111ep"			},
 	[RTL_GIGA_MAC_VER_51] = {"RTL8168ep/8111ep"			},
 	[RTL_GIGA_MAC_VER_52] = {"RTL8168fp/RTL8117",  FIRMWARE_8168FP_3},
 	[RTL_GIGA_MAC_VER_53] = {"RTL8168fp/RTL8117",			},
-	[RTL_GIGA_MAC_VER_60] = {"RTL8125A"				},
 	[RTL_GIGA_MAC_VER_61] = {"RTL8125A",		FIRMWARE_8125A_3},
 	/* reserve 62 for CFG_METHOD_4 in the vendor driver */
 	[RTL_GIGA_MAC_VER_63] = {"RTL8125B",		FIRMWARE_8125B_2},
@@ -658,10 +647,8 @@ MODULE_FIRMWARE(FIRMWARE_8106E_1);
 MODULE_FIRMWARE(FIRMWARE_8106E_2);
 MODULE_FIRMWARE(FIRMWARE_8168G_2);
 MODULE_FIRMWARE(FIRMWARE_8168G_3);
-MODULE_FIRMWARE(FIRMWARE_8168H_1);
 MODULE_FIRMWARE(FIRMWARE_8168H_2);
 MODULE_FIRMWARE(FIRMWARE_8168FP_3);
-MODULE_FIRMWARE(FIRMWARE_8107E_1);
 MODULE_FIRMWARE(FIRMWARE_8107E_2);
 MODULE_FIRMWARE(FIRMWARE_8125A_3);
 MODULE_FIRMWARE(FIRMWARE_8125B_2);
@@ -689,7 +676,7 @@ static void rtl_pci_commit(struct rtl8169_private *tp)
 
 static bool rtl_is_8125(struct rtl8169_private *tp)
 {
-	return tp->mac_version >= RTL_GIGA_MAC_VER_60;
+	return tp->mac_version >= RTL_GIGA_MAC_VER_61;
 }
 
 static bool rtl_is_8168evl_up(struct rtl8169_private *tp)
@@ -892,8 +879,6 @@ static void rtl8168g_phy_suspend_quirk(struct rtl8169_private *tp, int value)
 {
 	switch (tp->mac_version) {
 	case RTL_GIGA_MAC_VER_40:
-	case RTL_GIGA_MAC_VER_41:
-	case RTL_GIGA_MAC_VER_49:
 		if (value & BMCR_RESET || !(value & BMCR_PDOWN))
 			rtl_eri_set_bits(tp, 0x1a8, 0xfc000000);
 		else
@@ -1207,7 +1192,7 @@ static enum rtl_dash_type rtl_check_dash(struct rtl8169_private *tp)
 	case RTL_GIGA_MAC_VER_28:
 	case RTL_GIGA_MAC_VER_31:
 		return r8168dp_check_dash(tp) ? RTL_DASH_DP : RTL_DASH_NONE;
-	case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_53:
+	case RTL_GIGA_MAC_VER_51 ... RTL_GIGA_MAC_VER_53:
 		return r8168ep_check_dash(tp) ? RTL_DASH_EP : RTL_DASH_NONE;
 	default:
 		return RTL_DASH_NONE;
@@ -1423,11 +1408,11 @@ static void rtl8169_get_drvinfo(struct net_device *dev,
 	struct rtl8169_private *tp = netdev_priv(dev);
 	struct rtl_fw *rtl_fw = tp->rtl_fw;
 
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
-	strlcpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->bus_info, pci_name(tp->pci_dev), sizeof(info->bus_info));
 	BUILD_BUG_ON(sizeof(info->fw_version) < sizeof(rtl_fw->version));
 	if (rtl_fw)
-		strlcpy(info->fw_version, rtl_fw->version,
+		strscpy(info->fw_version, rtl_fw->version,
 			sizeof(info->fw_version));
 }
 
@@ -2011,7 +1996,10 @@ static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii)
 
 		/* 8168F family. */
 		{ 0x7c8, 0x488,	RTL_GIGA_MAC_VER_38 },
-		{ 0x7cf, 0x481,	RTL_GIGA_MAC_VER_36 },
+		/* It seems this chip version never made it to
+		 * the wild. Let's disable detection.
+		 * { 0x7cf, 0x481,	RTL_GIGA_MAC_VER_36 },
+		 */
 		{ 0x7cf, 0x480,	RTL_GIGA_MAC_VER_35 },
 
 		/* 8168E family. */
@@ -2041,7 +2029,6 @@ static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii)
 		{ 0x7c8, 0x3c0,	RTL_GIGA_MAC_VER_22 },
 
 		/* 8168B family. */
-		{ 0x7cf, 0x380,	RTL_GIGA_MAC_VER_12 },
 		{ 0x7c8, 0x380,	RTL_GIGA_MAC_VER_17 },
 		{ 0x7c8, 0x300,	RTL_GIGA_MAC_VER_11 },
 
@@ -2054,19 +2041,10 @@ static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii)
 		{ 0x7cf, 0x249,	RTL_GIGA_MAC_VER_08 },
 		{ 0x7cf, 0x348,	RTL_GIGA_MAC_VER_07 },
 		{ 0x7cf, 0x248,	RTL_GIGA_MAC_VER_07 },
-		{ 0x7cf, 0x340,	RTL_GIGA_MAC_VER_13 },
 		{ 0x7cf, 0x240,	RTL_GIGA_MAC_VER_14 },
-		{ 0x7cf, 0x343,	RTL_GIGA_MAC_VER_10 },
-		{ 0x7cf, 0x342,	RTL_GIGA_MAC_VER_16 },
 		{ 0x7c8, 0x348,	RTL_GIGA_MAC_VER_09 },
 		{ 0x7c8, 0x248,	RTL_GIGA_MAC_VER_09 },
-		{ 0x7c8, 0x340,	RTL_GIGA_MAC_VER_16 },
-		/* FIXME: where did these entries come from ? -- FR
-		 * Not even r8101 vendor driver knows these id's,
-		 * so let's disable detection for now. -- HK
-		 * { 0xfc8, 0x388,	RTL_GIGA_MAC_VER_13 },
-		 * { 0xfc8, 0x308,	RTL_GIGA_MAC_VER_13 },
-		 */
+		{ 0x7c8, 0x340,	RTL_GIGA_MAC_VER_10 },
 
 		/* 8110 family. */
 		{ 0xfc8, 0x980,	RTL_GIGA_MAC_VER_06 },
@@ -2088,8 +2066,6 @@ static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii)
 	if (ver != RTL_GIGA_MAC_NONE && !gmii) {
 		if (ver == RTL_GIGA_MAC_VER_42)
 			ver = RTL_GIGA_MAC_VER_43;
-		else if (ver == RTL_GIGA_MAC_VER_45)
-			ver = RTL_GIGA_MAC_VER_47;
 		else if (ver == RTL_GIGA_MAC_VER_46)
 			ver = RTL_GIGA_MAC_VER_48;
 	}
@@ -2271,7 +2247,7 @@ static void rtl_init_rxcfg(struct rtl8169_private *tp)
 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_53:
 		RTL_W32(tp, RxConfig, RX128_INT_EN | RX_MULTI_EN | RX_DMA_BURST | RX_EARLY_OFF);
 		break;
-	case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63:
+	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63:
 		RTL_W32(tp, RxConfig, RX_FETCH_DFLT_8125 | RX_DMA_BURST);
 		break;
 	default:
@@ -2338,7 +2314,6 @@ static void rtl_jumbo_config(struct rtl8169_private *tp)
 
 	rtl_unlock_config_regs(tp);
 	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_12:
 	case RTL_GIGA_MAC_VER_17:
 		if (jumbo) {
 			readrq = 512;
@@ -2455,7 +2430,7 @@ static void rtl_wait_txrx_fifo_empty(struct rtl8169_private *tp)
 		rtl_loop_wait_high(tp, &rtl_txcfg_empty_cond, 100, 42);
 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
 		break;
-	case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_61:
+	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_61:
 		rtl_loop_wait_high(tp, &rtl_rxtx_empty_cond, 100, 42);
 		break;
 	case RTL_GIGA_MAC_VER_63:
@@ -2468,6 +2443,11 @@ static void rtl_wait_txrx_fifo_empty(struct rtl8169_private *tp)
 	}
 }
 
+static void rtl_disable_rxdvgate(struct rtl8169_private *tp)
+{
+	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
+}
+
 static void rtl_enable_rxdvgate(struct rtl8169_private *tp)
 {
 	RTL_W32(tp, MISC, RTL_R32(tp, MISC) | RXDV_GATED_EN);
@@ -2700,8 +2680,8 @@ static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
 		RTL_W8(tp, Config2, RTL_R8(tp, Config2) | ClkReqEn);
 
 		switch (tp->mac_version) {
-		case RTL_GIGA_MAC_VER_45 ... RTL_GIGA_MAC_VER_48:
-		case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63:
+		case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
+		case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63:
 			/* reset ephy tx/rx disable timer */
 			r8168_mac_ocp_modify(tp, 0xe094, 0xff00, 0);
 			/* chip can trigger L1.2 */
@@ -2712,8 +2692,8 @@ static void rtl_hw_aspm_clkreq_enable(struct rtl8169_private *tp, bool enable)
 		}
 	} else {
 		switch (tp->mac_version) {
-		case RTL_GIGA_MAC_VER_45 ... RTL_GIGA_MAC_VER_48:
-		case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63:
+		case RTL_GIGA_MAC_VER_46 ... RTL_GIGA_MAC_VER_48:
+		case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63:
 			r8168_mac_ocp_modify(tp, 0xe092, 0x00ff, 0);
 			break;
 		default:
@@ -2985,7 +2965,7 @@ static void rtl_hw_start_8168g(struct rtl8169_private *tp)
 	rtl_reset_packet_filter(tp);
 	rtl_eri_write(tp, 0x2f8, ERIAR_MASK_0011, 0x1d8f);
 
-	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
+	rtl_disable_rxdvgate(tp);
 
 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
@@ -3223,7 +3203,7 @@ static void rtl_hw_start_8168h_1(struct rtl8169_private *tp)
 
 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
 
-	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
+	rtl_disable_rxdvgate(tp);
 
 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
@@ -3274,7 +3254,7 @@ static void rtl_hw_start_8168ep(struct rtl8169_private *tp)
 
 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
 
-	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
+	rtl_disable_rxdvgate(tp);
 
 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
@@ -3288,45 +3268,6 @@ static void rtl_hw_start_8168ep(struct rtl8169_private *tp)
 	rtl_pcie_state_l2l3_disable(tp);
 }
 
-static void rtl_hw_start_8168ep_1(struct rtl8169_private *tp)
-{
-	static const struct ephy_info e_info_8168ep_1[] = {
-		{ 0x00, 0xffff,	0x10ab },
-		{ 0x06, 0xffff,	0xf030 },
-		{ 0x08, 0xffff,	0x2006 },
-		{ 0x0d, 0xffff,	0x1666 },
-		{ 0x0c, 0x3ff0,	0x0000 }
-	};
-
-	/* disable aspm and clock request before access ephy */
-	rtl_hw_aspm_clkreq_enable(tp, false);
-	rtl_ephy_init(tp, e_info_8168ep_1);
-
-	rtl_hw_start_8168ep(tp);
-
-	rtl_hw_aspm_clkreq_enable(tp, true);
-}
-
-static void rtl_hw_start_8168ep_2(struct rtl8169_private *tp)
-{
-	static const struct ephy_info e_info_8168ep_2[] = {
-		{ 0x00, 0xffff,	0x10a3 },
-		{ 0x19, 0xffff,	0xfc00 },
-		{ 0x1e, 0xffff,	0x20ea }
-	};
-
-	/* disable aspm and clock request before access ephy */
-	rtl_hw_aspm_clkreq_enable(tp, false);
-	rtl_ephy_init(tp, e_info_8168ep_2);
-
-	rtl_hw_start_8168ep(tp);
-
-	RTL_W8(tp, DLLPR, RTL_R8(tp, DLLPR) & ~PFM_EN);
-	RTL_W8(tp, MISC_1, RTL_R8(tp, MISC_1) & ~PFM_D3COLD_EN);
-
-	rtl_hw_aspm_clkreq_enable(tp, true);
-}
-
 static void rtl_hw_start_8168ep_3(struct rtl8169_private *tp)
 {
 	static const struct ephy_info e_info_8168ep_3[] = {
@@ -3377,7 +3318,7 @@ static void rtl_hw_start_8117(struct rtl8169_private *tp)
 
 	rtl_eri_write(tp, 0x5f0, ERIAR_MASK_0011, 0x4f87);
 
-	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
+	rtl_disable_rxdvgate(tp);
 
 	rtl_eri_write(tp, 0xc0, ERIAR_MASK_0011, 0x0000);
 	rtl_eri_write(tp, 0xb8, ERIAR_MASK_0011, 0x0000);
@@ -3621,48 +3562,7 @@ static void rtl_hw_start_8125_common(struct rtl8169_private *tp)
 	else
 		rtl8125a_config_eee_mac(tp);
 
-	RTL_W32(tp, MISC, RTL_R32(tp, MISC) & ~RXDV_GATED_EN);
-	udelay(10);
-}
-
-static void rtl_hw_start_8125a_1(struct rtl8169_private *tp)
-{
-	static const struct ephy_info e_info_8125a_1[] = {
-		{ 0x01, 0xffff, 0xa812 },
-		{ 0x09, 0xffff, 0x520c },
-		{ 0x04, 0xffff, 0xd000 },
-		{ 0x0d, 0xffff, 0xf702 },
-		{ 0x0a, 0xffff, 0x8653 },
-		{ 0x06, 0xffff, 0x001e },
-		{ 0x08, 0xffff, 0x3595 },
-		{ 0x20, 0xffff, 0x9455 },
-		{ 0x21, 0xffff, 0x99ff },
-		{ 0x02, 0xffff, 0x6046 },
-		{ 0x29, 0xffff, 0xfe00 },
-		{ 0x23, 0xffff, 0xab62 },
-
-		{ 0x41, 0xffff, 0xa80c },
-		{ 0x49, 0xffff, 0x520c },
-		{ 0x44, 0xffff, 0xd000 },
-		{ 0x4d, 0xffff, 0xf702 },
-		{ 0x4a, 0xffff, 0x8653 },
-		{ 0x46, 0xffff, 0x001e },
-		{ 0x48, 0xffff, 0x3595 },
-		{ 0x60, 0xffff, 0x9455 },
-		{ 0x61, 0xffff, 0x99ff },
-		{ 0x42, 0xffff, 0x6046 },
-		{ 0x69, 0xffff, 0xfe00 },
-		{ 0x63, 0xffff, 0xab62 },
-	};
-
-	rtl_set_def_aspm_entry_latency(tp);
-
-	/* disable aspm and clock request before access ephy */
-	rtl_hw_aspm_clkreq_enable(tp, false);
-	rtl_ephy_init(tp, e_info_8125a_1);
-
-	rtl_hw_start_8125_common(tp);
-	rtl_hw_aspm_clkreq_enable(tp, true);
+	rtl_disable_rxdvgate(tp);
 }
 
 static void rtl_hw_start_8125a_2(struct rtl8169_private *tp)
@@ -3721,10 +3621,7 @@ static void rtl_hw_config(struct rtl8169_private *tp)
 		[RTL_GIGA_MAC_VER_09] = rtl_hw_start_8102e_2,
 		[RTL_GIGA_MAC_VER_10] = NULL,
 		[RTL_GIGA_MAC_VER_11] = rtl_hw_start_8168b,
-		[RTL_GIGA_MAC_VER_12] = rtl_hw_start_8168b,
-		[RTL_GIGA_MAC_VER_13] = NULL,
 		[RTL_GIGA_MAC_VER_14] = rtl_hw_start_8401,
-		[RTL_GIGA_MAC_VER_16] = NULL,
 		[RTL_GIGA_MAC_VER_17] = rtl_hw_start_8168b,
 		[RTL_GIGA_MAC_VER_18] = rtl_hw_start_8168cp_1,
 		[RTL_GIGA_MAC_VER_19] = rtl_hw_start_8168c_1,
@@ -3748,20 +3645,14 @@ static void rtl_hw_config(struct rtl8169_private *tp)
 		[RTL_GIGA_MAC_VER_38] = rtl_hw_start_8411,
 		[RTL_GIGA_MAC_VER_39] = rtl_hw_start_8106,
 		[RTL_GIGA_MAC_VER_40] = rtl_hw_start_8168g_1,
-		[RTL_GIGA_MAC_VER_41] = rtl_hw_start_8168g_1,
 		[RTL_GIGA_MAC_VER_42] = rtl_hw_start_8168g_2,
 		[RTL_GIGA_MAC_VER_43] = rtl_hw_start_8168g_2,
 		[RTL_GIGA_MAC_VER_44] = rtl_hw_start_8411_2,
-		[RTL_GIGA_MAC_VER_45] = rtl_hw_start_8168h_1,
 		[RTL_GIGA_MAC_VER_46] = rtl_hw_start_8168h_1,
-		[RTL_GIGA_MAC_VER_47] = rtl_hw_start_8168h_1,
 		[RTL_GIGA_MAC_VER_48] = rtl_hw_start_8168h_1,
-		[RTL_GIGA_MAC_VER_49] = rtl_hw_start_8168ep_1,
-		[RTL_GIGA_MAC_VER_50] = rtl_hw_start_8168ep_2,
 		[RTL_GIGA_MAC_VER_51] = rtl_hw_start_8168ep_3,
 		[RTL_GIGA_MAC_VER_52] = rtl_hw_start_8117,
 		[RTL_GIGA_MAC_VER_53] = rtl_hw_start_8117,
-		[RTL_GIGA_MAC_VER_60] = rtl_hw_start_8125a_1,
 		[RTL_GIGA_MAC_VER_61] = rtl_hw_start_8125a_2,
 		[RTL_GIGA_MAC_VER_63] = rtl_hw_start_8125b,
 	};
@@ -4156,7 +4047,6 @@ static unsigned int rtl_quirk_packet_padto(struct rtl8169_private *tp,
 
 	switch (tp->mac_version) {
 	case RTL_GIGA_MAC_VER_34:
-	case RTL_GIGA_MAC_VER_60:
 	case RTL_GIGA_MAC_VER_61:
 	case RTL_GIGA_MAC_VER_63:
 		padto = max_t(unsigned int, padto, ETH_ZLEN);
@@ -4677,8 +4567,7 @@ static void r8169_phylink_handler(struct net_device *ndev)
 		pm_runtime_idle(&tp->pci_dev->dev);
 	}
 
-	if (net_ratelimit())
-		phy_print_status(tp->phydev);
+	phy_print_status(tp->phydev);
 }
 
 static int r8169_phy_connect(struct rtl8169_private *tp)
@@ -4954,23 +4843,6 @@ static const struct dev_pm_ops rtl8169_pm_ops = {
 		       rtl8169_runtime_idle)
 };
 
-static void rtl_wol_shutdown_quirk(struct rtl8169_private *tp)
-{
-	/* WoL fails with 8168b when the receiver is disabled. */
-	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_11:
-	case RTL_GIGA_MAC_VER_12:
-	case RTL_GIGA_MAC_VER_17:
-		pci_clear_master(tp->pci_dev);
-
-		RTL_W8(tp, ChipCmd, CmdRxEnb);
-		rtl_pci_commit(tp);
-		break;
-	default:
-		break;
-	}
-}
-
 static void rtl_shutdown(struct pci_dev *pdev)
 {
 	struct rtl8169_private *tp = pci_get_drvdata(pdev);
@@ -4984,9 +4856,6 @@ static void rtl_shutdown(struct pci_dev *pdev)
 
 	if (system_state == SYSTEM_POWER_OFF &&
 	    tp->dash_type == RTL_DASH_NONE) {
-		if (tp->saved_wolopts)
-			rtl_wol_shutdown_quirk(tp);
-
 		pci_wake_from_d3(pdev, tp->saved_wolopts);
 		pci_set_power_state(pdev, PCI_D3hot);
 	}
@@ -5194,13 +5063,13 @@ static void rtl_hw_init_8125(struct rtl8169_private *tp)
 static void rtl_hw_initialize(struct rtl8169_private *tp)
 {
 	switch (tp->mac_version) {
-	case RTL_GIGA_MAC_VER_49 ... RTL_GIGA_MAC_VER_53:
+	case RTL_GIGA_MAC_VER_51 ... RTL_GIGA_MAC_VER_53:
 		rtl8168ep_stop_cmac(tp);
 		fallthrough;
 	case RTL_GIGA_MAC_VER_40 ... RTL_GIGA_MAC_VER_48:
 		rtl_hw_init_8168g(tp);
 		break;
-	case RTL_GIGA_MAC_VER_60 ... RTL_GIGA_MAC_VER_63:
+	case RTL_GIGA_MAC_VER_61 ... RTL_GIGA_MAC_VER_63:
 		rtl_hw_init_8125(tp);
 		break;
 	default:
@@ -5220,7 +5089,6 @@ static int rtl_jumbo_max(struct rtl8169_private *tp)
 		return JUMBO_7K;
 	/* RTL8168b */
 	case RTL_GIGA_MAC_VER_11:
-	case RTL_GIGA_MAC_VER_12:
 	case RTL_GIGA_MAC_VER_17:
 		return JUMBO_4K;
 	/* RTL8168c */
@@ -5231,37 +5099,6 @@ static int rtl_jumbo_max(struct rtl8169_private *tp)
 	}
 }
 
-static void rtl_disable_clk(void *data)
-{
-	clk_disable_unprepare(data);
-}
-
-static int rtl_get_ether_clk(struct rtl8169_private *tp)
-{
-	struct device *d = tp_to_dev(tp);
-	struct clk *clk;
-	int rc;
-
-	clk = devm_clk_get(d, "ether_clk");
-	if (IS_ERR(clk)) {
-		rc = PTR_ERR(clk);
-		if (rc == -ENOENT)
-			/* clk-core allows NULL (for suspend / resume) */
-			rc = 0;
-		else
-			dev_err_probe(d, rc, "failed to get clk\n");
-	} else {
-		tp->clk = clk;
-		rc = clk_prepare_enable(clk);
-		if (rc)
-			dev_err(d, "failed to enable clk: %d\n", rc);
-		else
-			rc = devm_add_action_or_reset(d, rtl_disable_clk, clk);
-	}
-
-	return rc;
-}
-
 static void rtl_init_mac_address(struct rtl8169_private *tp)
 {
 	u8 mac_addr[ETH_ALEN] __aligned(2) = {};
@@ -5291,7 +5128,7 @@ done:
 /* register is set if system vendor successfully tested ASPM 1.2 */
 static bool rtl_aspm_is_safe(struct rtl8169_private *tp)
 {
-	if (tp->mac_version >= RTL_GIGA_MAC_VER_60 &&
+	if (tp->mac_version >= RTL_GIGA_MAC_VER_61 &&
 	    r8168_mac_ocp_read(tp, 0xc0b2) & 0xf)
 		return true;
 
@@ -5325,9 +5162,9 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		return -ENOMEM;
 
 	/* Get the *optional* external "ether_clk" used on some boards */
-	rc = rtl_get_ether_clk(tp);
-	if (rc)
-		return rc;
+	tp->clk = devm_clk_get_optional_enabled(&pdev->dev, "ether_clk");
+	if (IS_ERR(tp->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(tp->clk), "failed to get ether_clk\n");
 
 	/* enable device (incl. PCI PM wakeup and hotplug setup) */
 	rc = pcim_enable_device(pdev);
@@ -5346,12 +5183,6 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		return -ENODEV;
 	}
 
-	/* check for weird/broken PCI region reporting */
-	if (pci_resource_len(pdev, region) < R8169_REGS_SIZE) {
-		dev_err(&pdev->dev, "Invalid PCI region size(s), aborting\n");
-		return -ENODEV;
-	}
-
 	rc = pcim_iomap_regions(pdev, BIT(region), KBUILD_MODNAME);
 	if (rc < 0) {
 		dev_err(&pdev->dev, "cannot remap MMIO, aborting\n");
@@ -5378,7 +5209,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	 */
 	if (rtl_aspm_is_safe(tp))
 		rc = 0;
-	else if (tp->mac_version >= RTL_GIGA_MAC_VER_45)
+	else if (tp->mac_version >= RTL_GIGA_MAC_VER_46)
 		rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2);
 	else
 		rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1);
@@ -5413,7 +5244,7 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	dev->ethtool_ops = &rtl8169_ethtool_ops;
 
-	netif_napi_add(dev, &tp->napi, rtl8169_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &tp->napi, rtl8169_poll);
 
 	dev->hw_features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM |
 			   NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
diff --git a/drivers/net/ethernet/realtek/r8169_phy_config.c b/drivers/net/ethernet/realtek/r8169_phy_config.c
index 15c295f90196..930496cd34ed 100644
--- a/drivers/net/ethernet/realtek/r8169_phy_config.c
+++ b/drivers/net/ethernet/realtek/r8169_phy_config.c
@@ -793,71 +793,6 @@ static void rtl8168g_2_hw_phy_config(struct rtl8169_private *tp,
 	rtl8168g_config_eee_phy(phydev);
 }
 
-static void rtl8168h_1_hw_phy_config(struct rtl8169_private *tp,
-				     struct phy_device *phydev)
-{
-	u16 dout_tapbin;
-	u32 data;
-
-	r8169_apply_firmware(tp);
-
-	/* CHN EST parameters adjust - giga master */
-	r8168g_phy_param(phydev, 0x809b, 0xf800, 0x8000);
-	r8168g_phy_param(phydev, 0x80a2, 0xff00, 0x8000);
-	r8168g_phy_param(phydev, 0x80a4, 0xff00, 0x8500);
-	r8168g_phy_param(phydev, 0x809c, 0xff00, 0xbd00);
-
-	/* CHN EST parameters adjust - giga slave */
-	r8168g_phy_param(phydev, 0x80ad, 0xf800, 0x7000);
-	r8168g_phy_param(phydev, 0x80b4, 0xff00, 0x5000);
-	r8168g_phy_param(phydev, 0x80ac, 0xff00, 0x4000);
-
-	/* CHN EST parameters adjust - fnet */
-	r8168g_phy_param(phydev, 0x808e, 0xff00, 0x1200);
-	r8168g_phy_param(phydev, 0x8090, 0xff00, 0xe500);
-	r8168g_phy_param(phydev, 0x8092, 0xff00, 0x9f00);
-
-	/* enable R-tune & PGA-retune function */
-	dout_tapbin = 0;
-	data = phy_read_paged(phydev, 0x0a46, 0x13);
-	data &= 3;
-	data <<= 2;
-	dout_tapbin |= data;
-	data = phy_read_paged(phydev, 0x0a46, 0x12);
-	data &= 0xc000;
-	data >>= 14;
-	dout_tapbin |= data;
-	dout_tapbin = ~(dout_tapbin ^ 0x08);
-	dout_tapbin <<= 12;
-	dout_tapbin &= 0xf000;
-
-	r8168g_phy_param(phydev, 0x827a, 0xf000, dout_tapbin);
-	r8168g_phy_param(phydev, 0x827b, 0xf000, dout_tapbin);
-	r8168g_phy_param(phydev, 0x827c, 0xf000, dout_tapbin);
-	r8168g_phy_param(phydev, 0x827d, 0xf000, dout_tapbin);
-	r8168g_phy_param(phydev, 0x0811, 0x0000, 0x0800);
-	phy_modify_paged(phydev, 0x0a42, 0x16, 0x0000, 0x0002);
-
-	rtl8168g_enable_gphy_10m(phydev);
-
-	/* SAR ADC performance */
-	phy_modify_paged(phydev, 0x0bca, 0x17, BIT(12) | BIT(13), BIT(14));
-
-	r8168g_phy_param(phydev, 0x803f, 0x3000, 0x0000);
-	r8168g_phy_param(phydev, 0x8047, 0x3000, 0x0000);
-	r8168g_phy_param(phydev, 0x804f, 0x3000, 0x0000);
-	r8168g_phy_param(phydev, 0x8057, 0x3000, 0x0000);
-	r8168g_phy_param(phydev, 0x805f, 0x3000, 0x0000);
-	r8168g_phy_param(phydev, 0x8067, 0x3000, 0x0000);
-	r8168g_phy_param(phydev, 0x806f, 0x3000, 0x0000);
-
-	/* disable phy pfm mode */
-	phy_modify_paged(phydev, 0x0a44, 0x11, BIT(7), 0);
-
-	rtl8168g_disable_aldps(phydev);
-	rtl8168h_config_eee_phy(phydev);
-}
-
 static void rtl8168h_2_hw_phy_config(struct rtl8169_private *tp,
 				     struct phy_device *phydev)
 {
@@ -895,27 +830,6 @@ static void rtl8168h_2_hw_phy_config(struct rtl8169_private *tp,
 	rtl8168g_config_eee_phy(phydev);
 }
 
-static void rtl8168ep_1_hw_phy_config(struct rtl8169_private *tp,
-				      struct phy_device *phydev)
-{
-	/* Enable PHY auto speed down */
-	phy_modify_paged(phydev, 0x0a44, 0x11, 0, BIT(3) | BIT(2));
-
-	rtl8168g_phy_adjust_10m_aldps(phydev);
-
-	/* Enable EEE auto-fallback function */
-	phy_modify_paged(phydev, 0x0a4b, 0x11, 0, BIT(2));
-
-	/* Enable UC LPF tune function */
-	r8168g_phy_param(phydev, 0x8012, 0x0000, 0x8000);
-
-	/* set rg_sel_sdm_rate */
-	phy_modify_paged(phydev, 0x0c42, 0x11, BIT(13), BIT(14));
-
-	rtl8168g_disable_aldps(phydev);
-	rtl8168g_config_eee_phy(phydev);
-}
-
 static void rtl8168ep_2_hw_phy_config(struct rtl8169_private *tp,
 				      struct phy_device *phydev)
 {
@@ -1081,44 +995,6 @@ static void rtl8125_legacy_force_mode(struct phy_device *phydev)
 	phy_modify_paged(phydev, 0xa5b, 0x12, BIT(15), 0);
 }
 
-static void rtl8125a_1_hw_phy_config(struct rtl8169_private *tp,
-				     struct phy_device *phydev)
-{
-	phy_modify_paged(phydev, 0xad4, 0x10, 0x03ff, 0x0084);
-	phy_modify_paged(phydev, 0xad4, 0x17, 0x0000, 0x0010);
-	phy_modify_paged(phydev, 0xad1, 0x13, 0x03ff, 0x0006);
-	phy_modify_paged(phydev, 0xad3, 0x11, 0x003f, 0x0006);
-	phy_modify_paged(phydev, 0xac0, 0x14, 0x0000, 0x1100);
-	phy_modify_paged(phydev, 0xac8, 0x15, 0xf000, 0x7000);
-	phy_modify_paged(phydev, 0xad1, 0x14, 0x0000, 0x0400);
-	phy_modify_paged(phydev, 0xad1, 0x15, 0x0000, 0x03ff);
-	phy_modify_paged(phydev, 0xad1, 0x16, 0x0000, 0x03ff);
-
-	r8168g_phy_param(phydev, 0x80ea, 0xff00, 0xc400);
-	r8168g_phy_param(phydev, 0x80eb, 0x0700, 0x0300);
-	r8168g_phy_param(phydev, 0x80f8, 0xff00, 0x1c00);
-	r8168g_phy_param(phydev, 0x80f1, 0xff00, 0x3000);
-	r8168g_phy_param(phydev, 0x80fe, 0xff00, 0xa500);
-	r8168g_phy_param(phydev, 0x8102, 0xff00, 0x5000);
-	r8168g_phy_param(phydev, 0x8105, 0xff00, 0x3300);
-	r8168g_phy_param(phydev, 0x8100, 0xff00, 0x7000);
-	r8168g_phy_param(phydev, 0x8104, 0xff00, 0xf000);
-	r8168g_phy_param(phydev, 0x8106, 0xff00, 0x6500);
-	r8168g_phy_param(phydev, 0x80dc, 0xff00, 0xed00);
-	r8168g_phy_param(phydev, 0x80df, 0x0000, 0x0100);
-	r8168g_phy_param(phydev, 0x80e1, 0x0100, 0x0000);
-
-	phy_modify_paged(phydev, 0xbf0, 0x13, 0x003f, 0x0038);
-	r8168g_phy_param(phydev, 0x819f, 0xffff, 0xd0b6);
-
-	phy_write_paged(phydev, 0xbc3, 0x12, 0x5555);
-	phy_modify_paged(phydev, 0xbf0, 0x15, 0x0e00, 0x0a00);
-	phy_modify_paged(phydev, 0xa5c, 0x10, 0x0400, 0x0000);
-	rtl8168g_enable_gphy_10m(phydev);
-
-	rtl8125a_config_eee_phy(phydev);
-}
-
 static void rtl8125a_2_hw_phy_config(struct rtl8169_private *tp,
 				     struct phy_device *phydev)
 {
@@ -1239,10 +1115,7 @@ void r8169_hw_phy_config(struct rtl8169_private *tp, struct phy_device *phydev,
 		[RTL_GIGA_MAC_VER_09] = rtl8102e_hw_phy_config,
 		[RTL_GIGA_MAC_VER_10] = NULL,
 		[RTL_GIGA_MAC_VER_11] = rtl8168bb_hw_phy_config,
-		[RTL_GIGA_MAC_VER_12] = rtl8168bef_hw_phy_config,
-		[RTL_GIGA_MAC_VER_13] = NULL,
 		[RTL_GIGA_MAC_VER_14] = rtl8401_hw_phy_config,
-		[RTL_GIGA_MAC_VER_16] = NULL,
 		[RTL_GIGA_MAC_VER_17] = rtl8168bef_hw_phy_config,
 		[RTL_GIGA_MAC_VER_18] = rtl8168cp_1_hw_phy_config,
 		[RTL_GIGA_MAC_VER_19] = rtl8168c_1_hw_phy_config,
@@ -1266,20 +1139,14 @@ void r8169_hw_phy_config(struct rtl8169_private *tp, struct phy_device *phydev,
 		[RTL_GIGA_MAC_VER_38] = rtl8411_hw_phy_config,
 		[RTL_GIGA_MAC_VER_39] = rtl8106e_hw_phy_config,
 		[RTL_GIGA_MAC_VER_40] = rtl8168g_1_hw_phy_config,
-		[RTL_GIGA_MAC_VER_41] = NULL,
 		[RTL_GIGA_MAC_VER_42] = rtl8168g_2_hw_phy_config,
 		[RTL_GIGA_MAC_VER_43] = rtl8168g_2_hw_phy_config,
 		[RTL_GIGA_MAC_VER_44] = rtl8168g_2_hw_phy_config,
-		[RTL_GIGA_MAC_VER_45] = rtl8168h_1_hw_phy_config,
 		[RTL_GIGA_MAC_VER_46] = rtl8168h_2_hw_phy_config,
-		[RTL_GIGA_MAC_VER_47] = rtl8168h_1_hw_phy_config,
 		[RTL_GIGA_MAC_VER_48] = rtl8168h_2_hw_phy_config,
-		[RTL_GIGA_MAC_VER_49] = rtl8168ep_1_hw_phy_config,
-		[RTL_GIGA_MAC_VER_50] = rtl8168ep_2_hw_phy_config,
 		[RTL_GIGA_MAC_VER_51] = rtl8168ep_2_hw_phy_config,
 		[RTL_GIGA_MAC_VER_52] = rtl8117_hw_phy_config,
 		[RTL_GIGA_MAC_VER_53] = rtl8117_hw_phy_config,
-		[RTL_GIGA_MAC_VER_60] = rtl8125a_1_hw_phy_config,
 		[RTL_GIGA_MAC_VER_61] = rtl8125a_2_hw_phy_config,
 		[RTL_GIGA_MAC_VER_63] = rtl8125b_hw_phy_config,
 	};
diff --git a/drivers/net/ethernet/renesas/ravb.h b/drivers/net/ethernet/renesas/ravb.h
index b980bce763d3..e0f8276cffed 100644
--- a/drivers/net/ethernet/renesas/ravb.h
+++ b/drivers/net/ethernet/renesas/ravb.h
@@ -189,6 +189,7 @@ enum ravb_reg {
 	PSR	= 0x0528,
 	PIPR	= 0x052c,
 	CXR31	= 0x0530,	/* RZ/G2L only */
+	CXR35	= 0x0540,	/* RZ/G2L only */
 	MPR	= 0x0558,
 	PFTCR	= 0x055c,
 	PFRCR	= 0x0560,
@@ -965,6 +966,13 @@ enum CXR31_BIT {
 	CXR31_SEL_LINK1	= 0x00000008,
 };
 
+enum CXR35_BIT {
+	CXR35_SEL_XMII		= 0x00000003,
+	CXR35_SEL_XMII_RGMII	= 0x00000000,
+	CXR35_SEL_XMII_MII	= 0x00000002,
+	CXR35_HALFCYC_CLKSW	= 0xffff0000,
+};
+
 enum CSR0_BIT {
 	CSR0_TPE	= 0x00000010,
 	CSR0_RPE	= 0x00000020,
diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c
index 7e32b04eb0c7..36324126db6d 100644
--- a/drivers/net/ethernet/renesas/ravb_main.c
+++ b/drivers/net/ethernet/renesas/ravb_main.c
@@ -540,7 +540,13 @@ static void ravb_emac_init_gbeth(struct net_device *ndev)
 	/* E-MAC interrupt enable register */
 	ravb_write(ndev, ECSIPR_ICDIP, ECSIPR);
 
-	ravb_modify(ndev, CXR31, CXR31_SEL_LINK0 | CXR31_SEL_LINK1, CXR31_SEL_LINK0);
+	if (priv->phy_interface == PHY_INTERFACE_MODE_MII) {
+		ravb_modify(ndev, CXR31, CXR31_SEL_LINK0 | CXR31_SEL_LINK1, 0);
+		ravb_write(ndev, (1000 << 16) | CXR35_SEL_XMII_MII, CXR35);
+	} else {
+		ravb_modify(ndev, CXR31, CXR31_SEL_LINK0 | CXR31_SEL_LINK1,
+			    CXR31_SEL_LINK0);
+	}
 }
 
 static void ravb_emac_init_rcar(struct net_device *ndev)
@@ -2514,6 +2520,7 @@ static const struct of_device_id ravb_match_table[] = {
 	{ .compatible = "renesas,etheravb-rcar-gen2", .data = &ravb_gen2_hw_info },
 	{ .compatible = "renesas,etheravb-r8a7795", .data = &ravb_gen3_hw_info },
 	{ .compatible = "renesas,etheravb-rcar-gen3", .data = &ravb_gen3_hw_info },
+	{ .compatible = "renesas,etheravb-rcar-gen4", .data = &ravb_gen3_hw_info },
 	{ .compatible = "renesas,etheravb-rzv2m", .data = &ravb_rzv2m_hw_info },
 	{ .compatible = "renesas,rzg2l-gbeth", .data = &gbeth_hw_info },
 	{ }
@@ -2834,9 +2841,9 @@ static int ravb_probe(struct platform_device *pdev)
 		goto out_dma_free;
 	}
 
-	netif_napi_add(ndev, &priv->napi[RAVB_BE], ravb_poll, 64);
+	netif_napi_add(ndev, &priv->napi[RAVB_BE], ravb_poll);
 	if (info->nc_queues)
-		netif_napi_add(ndev, &priv->napi[RAVB_NC], ravb_poll, 64);
+		netif_napi_add(ndev, &priv->napi[RAVB_NC], ravb_poll);
 
 	/* Network device register */
 	error = register_netdev(ndev);
diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c
index 7fd8828d3a84..71a499113308 100644
--- a/drivers/net/ethernet/renesas/sh_eth.c
+++ b/drivers/net/ethernet/renesas/sh_eth.c
@@ -3368,7 +3368,7 @@ static int sh_eth_drv_probe(struct platform_device *pdev)
 		goto out_release;
 	}
 
-	netif_napi_add(ndev, &mdp->napi, sh_eth_poll, 64);
+	netif_napi_add(ndev, &mdp->napi, sh_eth_poll);
 
 	/* network device register */
 	ret = register_netdev(ndev);
diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c
index fc83ec23bd1d..023682cd2768 100644
--- a/drivers/net/ethernet/rocker/rocker_main.c
+++ b/drivers/net/ethernet/rocker/rocker_main.c
@@ -2226,8 +2226,8 @@ rocker_port_set_link_ksettings(struct net_device *dev,
 static void rocker_port_get_drvinfo(struct net_device *dev,
 				    struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, rocker_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version));
+	strscpy(drvinfo->driver, rocker_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version));
 }
 
 static struct rocker_port_stats {
@@ -2574,8 +2574,7 @@ static int rocker_probe_port(struct rocker *rocker, unsigned int port_number)
 	dev->netdev_ops = &rocker_port_netdev_ops;
 	dev->ethtool_ops = &rocker_port_ethtool_ops;
 	netif_napi_add_tx(dev, &rocker_port->napi_tx, rocker_port_poll_tx);
-	netif_napi_add(dev, &rocker_port->napi_rx, rocker_port_poll_rx,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &rocker_port->napi_rx, rocker_port_poll_rx);
 	rocker_carrier_init(rocker_port);
 
 	dev->features |= NETIF_F_NETNS_LOCAL | NETIF_F_SG;
diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c
index 98edb01024f0..8ba017ec9849 100644
--- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c
+++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_ethtool.c
@@ -175,8 +175,8 @@ static int sxgbe_set_eee(struct net_device *dev,
 static void sxgbe_getdrvinfo(struct net_device *dev,
 			     struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 }
 
 static u32 sxgbe_getmsglevel(struct net_device *dev)
diff --git a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c
index a1c10b61269b..9664f029fa16 100644
--- a/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c
+++ b/drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c
@@ -2143,7 +2143,7 @@ struct sxgbe_priv_data *sxgbe_drv_probe(struct device *device,
 		pr_info("Enable RX Mitigation via HW Watchdog Timer\n");
 	}
 
-	netif_napi_add(ndev, &priv->napi, sxgbe_poll, 64);
+	netif_napi_add(ndev, &priv->napi, sxgbe_poll);
 
 	spin_lock_init(&priv->stats_lock);
 
diff --git a/drivers/net/ethernet/sfc/Makefile b/drivers/net/ethernet/sfc/Makefile
index bb06fa228367..b5e45fc6337e 100644
--- a/drivers/net/ethernet/sfc/Makefile
+++ b/drivers/net/ethernet/sfc/Makefile
@@ -9,7 +9,7 @@ sfc-y			+= efx.o efx_common.o efx_channels.o nic.o \
 			   ef100_ethtool.o ef100_rx.o ef100_tx.o
 sfc-$(CONFIG_SFC_MTD)	+= mtd.o
 sfc-$(CONFIG_SFC_SRIOV)	+= sriov.o ef10_sriov.o ef100_sriov.o ef100_rep.o \
-                           mae.o tc.o
+                           mae.o tc.o tc_bindings.o
 
 obj-$(CONFIG_SFC)	+= sfc.o
 
diff --git a/drivers/net/ethernet/sfc/ef10.c b/drivers/net/ethernet/sfc/ef10.c
index ee734b69150f..d1e1aa19a68e 100644
--- a/drivers/net/ethernet/sfc/ef10.c
+++ b/drivers/net/ethernet/sfc/ef10.c
@@ -4213,7 +4213,7 @@ const struct efx_nic_type efx_hunt_a0_nic_type = {
 	.ev_test_generate = efx_ef10_ev_test_generate,
 	.filter_table_probe = efx_ef10_filter_table_probe,
 	.filter_table_restore = efx_mcdi_filter_table_restore,
-	.filter_table_remove = efx_mcdi_filter_table_remove,
+	.filter_table_remove = efx_ef10_filter_table_remove,
 	.filter_update_rx_scatter = efx_mcdi_update_rx_scatter,
 	.filter_insert = efx_mcdi_filter_insert,
 	.filter_remove_safe = efx_mcdi_filter_remove_safe,
diff --git a/drivers/net/ethernet/sfc/ef100_ethtool.c b/drivers/net/ethernet/sfc/ef100_ethtool.c
index 702abbe59b76..135ece2f1375 100644
--- a/drivers/net/ethernet/sfc/ef100_ethtool.c
+++ b/drivers/net/ethernet/sfc/ef100_ethtool.c
@@ -43,6 +43,8 @@ const struct ethtool_ops ef100_ethtool_ops = {
 	.get_pauseparam         = efx_ethtool_get_pauseparam,
 	.set_pauseparam         = efx_ethtool_set_pauseparam,
 	.get_sset_count		= efx_ethtool_get_sset_count,
+	.get_priv_flags		= efx_ethtool_get_priv_flags,
+	.set_priv_flags		= efx_ethtool_set_priv_flags,
 	.self_test		= efx_ethtool_self_test,
 	.get_strings		= efx_ethtool_get_strings,
 	.get_link_ksettings	= efx_ethtool_get_link_ksettings,
diff --git a/drivers/net/ethernet/sfc/ef100_netdev.c b/drivers/net/ethernet/sfc/ef100_netdev.c
index 17b9d37218cb..88fa29572e23 100644
--- a/drivers/net/ethernet/sfc/ef100_netdev.c
+++ b/drivers/net/ethernet/sfc/ef100_netdev.c
@@ -23,6 +23,7 @@
 #include "mcdi_filters.h"
 #include "rx_common.h"
 #include "ef100_sriov.h"
+#include "tc_bindings.h"
 
 static void ef100_update_name(struct efx_nic *efx)
 {
@@ -246,6 +247,9 @@ static const struct net_device_ops ef100_netdev_ops = {
 #ifdef CONFIG_RFS_ACCEL
 	.ndo_rx_flow_steer      = efx_filter_rfs,
 #endif
+#ifdef CONFIG_SFC_SRIOV
+	.ndo_setup_tc		= efx_tc_setup,
+#endif
 };
 
 /*	Netdev registration
diff --git a/drivers/net/ethernet/sfc/ef100_nic.c b/drivers/net/ethernet/sfc/ef100_nic.c
index 8061efdaf82c..ad686c671ab8 100644
--- a/drivers/net/ethernet/sfc/ef100_nic.c
+++ b/drivers/net/ethernet/sfc/ef100_nic.c
@@ -1137,6 +1137,9 @@ int ef100_probe_netdev_pf(struct efx_nic *efx)
 		 */
 		netif_warn(efx, probe, net_dev, "Failed to probe MAE rc %d\n",
 			   rc);
+	} else {
+		net_dev->features |= NETIF_F_HW_TC;
+		efx->fixed_features |= NETIF_F_HW_TC;
 	}
 #endif
 	return 0;
diff --git a/drivers/net/ethernet/sfc/ef100_rep.c b/drivers/net/ethernet/sfc/ef100_rep.c
index 73ae4656a6e7..81ab22c74635 100644
--- a/drivers/net/ethernet/sfc/ef100_rep.c
+++ b/drivers/net/ethernet/sfc/ef100_rep.c
@@ -14,6 +14,7 @@
 #include "ef100_nic.h"
 #include "mae.h"
 #include "rx_common.h"
+#include "tc_bindings.h"
 
 #define EFX_EF100_REP_DRIVER	"efx_ef100_rep"
 
@@ -42,8 +43,7 @@ static int efx_ef100_rep_open(struct net_device *net_dev)
 {
 	struct efx_rep *efv = netdev_priv(net_dev);
 
-	netif_napi_add(net_dev, &efv->napi, efx_ef100_rep_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(net_dev, &efv->napi, efx_ef100_rep_poll);
 	napi_enable(&efv->napi);
 	return 0;
 }
@@ -107,6 +107,20 @@ static int efx_ef100_rep_get_phys_port_name(struct net_device *dev,
 	return 0;
 }
 
+static int efx_ef100_rep_setup_tc(struct net_device *net_dev,
+				  enum tc_setup_type type, void *type_data)
+{
+	struct efx_rep *efv = netdev_priv(net_dev);
+	struct efx_nic *efx = efv->parent;
+
+	if (type == TC_SETUP_CLSFLOWER)
+		return efx_tc_flower(efx, net_dev, type_data, efv);
+	if (type == TC_SETUP_BLOCK)
+		return efx_tc_setup_block(net_dev, efx, type_data, efv);
+
+	return -EOPNOTSUPP;
+}
+
 static void efx_ef100_rep_get_stats64(struct net_device *dev,
 				      struct rtnl_link_stats64 *stats)
 {
@@ -120,13 +134,14 @@ static void efx_ef100_rep_get_stats64(struct net_device *dev,
 	stats->tx_errors = atomic64_read(&efv->stats.tx_errors);
 }
 
-static const struct net_device_ops efx_ef100_rep_netdev_ops = {
+const struct net_device_ops efx_ef100_rep_netdev_ops = {
 	.ndo_open		= efx_ef100_rep_open,
 	.ndo_stop		= efx_ef100_rep_close,
 	.ndo_start_xmit		= efx_ef100_rep_xmit,
 	.ndo_get_port_parent_id	= efx_ef100_rep_get_port_parent_id,
 	.ndo_get_phys_port_name	= efx_ef100_rep_get_phys_port_name,
 	.ndo_get_stats64	= efx_ef100_rep_get_stats64,
+	.ndo_setup_tc		= efx_ef100_rep_setup_tc,
 };
 
 static void efx_ef100_rep_get_drvinfo(struct net_device *dev,
diff --git a/drivers/net/ethernet/sfc/ef100_rep.h b/drivers/net/ethernet/sfc/ef100_rep.h
index 070f700893c1..c21bc716f847 100644
--- a/drivers/net/ethernet/sfc/ef100_rep.h
+++ b/drivers/net/ethernet/sfc/ef100_rep.h
@@ -66,4 +66,5 @@ void efx_ef100_rep_rx_packet(struct efx_rep *efv, struct efx_rx_buffer *rx_buf);
  * Caller must hold rcu_read_lock().
  */
 struct efx_rep *efx_ef100_find_rep_by_mport(struct efx_nic *efx, u16 mport);
+extern const struct net_device_ops efx_ef100_rep_netdev_ops;
 #endif /* EF100_REP_H */
diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c
index 153d68e29b8b..054d5ce6029e 100644
--- a/drivers/net/ethernet/sfc/efx.c
+++ b/drivers/net/ethernet/sfc/efx.c
@@ -778,7 +778,7 @@ static void efx_unregister_netdev(struct efx_nic *efx)
 		return;
 
 	if (efx_dev_registered(efx)) {
-		strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
+		strscpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
 		efx_fini_mcdi_logging(efx);
 		device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
 		unregister_netdev(efx->net_dev);
@@ -1175,6 +1175,17 @@ static int efx_pm_freeze(struct device *dev)
 	return 0;
 }
 
+static void efx_pci_shutdown(struct pci_dev *pci_dev)
+{
+	struct efx_nic *efx = pci_get_drvdata(pci_dev);
+
+	if (!efx)
+		return;
+
+	efx_pm_freeze(&pci_dev->dev);
+	pci_disable_device(pci_dev);
+}
+
 static int efx_pm_thaw(struct device *dev)
 {
 	int rc;
@@ -1279,6 +1290,7 @@ static struct pci_driver efx_pci_driver = {
 	.probe		= efx_pci_probe,
 	.remove		= efx_pci_remove,
 	.driver.pm	= &efx_pm_ops,
+	.shutdown	= efx_pci_shutdown,
 	.err_handler	= &efx_err_handlers,
 #ifdef CONFIG_SFC_SRIOV
 	.sriov_configure = efx_pci_sriov_configure,
diff --git a/drivers/net/ethernet/sfc/efx_channels.c b/drivers/net/ethernet/sfc/efx_channels.c
index 5b4d661ab986..aaa381743bca 100644
--- a/drivers/net/ethernet/sfc/efx_channels.c
+++ b/drivers/net/ethernet/sfc/efx_channels.c
@@ -1313,7 +1313,7 @@ void efx_init_napi_channel(struct efx_channel *channel)
 	struct efx_nic *efx = channel->efx;
 
 	channel->napi_dev = efx->net_dev;
-	netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll, 64);
+	netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll);
 }
 
 void efx_init_napi(struct efx_nic *efx)
diff --git a/drivers/net/ethernet/sfc/efx_common.c b/drivers/net/ethernet/sfc/efx_common.c
index a929a1aaba92..c2224e41a694 100644
--- a/drivers/net/ethernet/sfc/efx_common.c
+++ b/drivers/net/ethernet/sfc/efx_common.c
@@ -996,7 +996,7 @@ int efx_init_struct(struct efx_nic *efx, struct pci_dev *pci_dev)
 	efx->pci_dev = pci_dev;
 	efx->msg_enable = debug;
 	efx->state = STATE_UNINIT;
-	strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
+	strscpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
 
 	efx->rx_prefix_size = efx->type->rx_prefix_size;
 	efx->rx_ip_align =
diff --git a/drivers/net/ethernet/sfc/ethtool_common.c b/drivers/net/ethernet/sfc/ethtool_common.c
index bc840ede3053..6649a2327d03 100644
--- a/drivers/net/ethernet/sfc/ethtool_common.c
+++ b/drivers/net/ethernet/sfc/ethtool_common.c
@@ -101,15 +101,23 @@ static const struct efx_sw_stat_desc efx_sw_stat_desc[] = {
 
 #define EFX_ETHTOOL_SW_STAT_COUNT ARRAY_SIZE(efx_sw_stat_desc)
 
+static const char efx_ethtool_priv_flags_strings[][ETH_GSTRING_LEN] = {
+	"log-tc-errors",
+};
+
+#define EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS		BIT(0)
+
+#define EFX_ETHTOOL_PRIV_FLAGS_COUNT ARRAY_SIZE(efx_ethtool_priv_flags_strings)
+
 void efx_ethtool_get_drvinfo(struct net_device *net_dev,
 			     struct ethtool_drvinfo *info)
 {
 	struct efx_nic *efx = efx_netdev_priv(net_dev);
 
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
 	efx_mcdi_print_fwver(efx, info->fw_version,
 			     sizeof(info->fw_version));
-	strlcpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info));
+	strscpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info));
 }
 
 u32 efx_ethtool_get_msglevel(struct net_device *net_dev)
@@ -452,6 +460,8 @@ int efx_ethtool_get_sset_count(struct net_device *net_dev, int string_set)
 		       efx_ptp_describe_stats(efx, NULL);
 	case ETH_SS_TEST:
 		return efx_ethtool_fill_self_tests(efx, NULL, NULL, NULL);
+	case ETH_SS_PRIV_FLAGS:
+		return EFX_ETHTOOL_PRIV_FLAGS_COUNT;
 	default:
 		return -EINVAL;
 	}
@@ -468,7 +478,7 @@ void efx_ethtool_get_strings(struct net_device *net_dev,
 		strings += (efx->type->describe_stats(efx, strings) *
 			    ETH_GSTRING_LEN);
 		for (i = 0; i < EFX_ETHTOOL_SW_STAT_COUNT; i++)
-			strlcpy(strings + i * ETH_GSTRING_LEN,
+			strscpy(strings + i * ETH_GSTRING_LEN,
 				efx_sw_stat_desc[i].name, ETH_GSTRING_LEN);
 		strings += EFX_ETHTOOL_SW_STAT_COUNT * ETH_GSTRING_LEN;
 		strings += (efx_describe_per_queue_stats(efx, strings) *
@@ -478,12 +488,39 @@ void efx_ethtool_get_strings(struct net_device *net_dev,
 	case ETH_SS_TEST:
 		efx_ethtool_fill_self_tests(efx, NULL, strings, NULL);
 		break;
+	case ETH_SS_PRIV_FLAGS:
+		for (i = 0; i < EFX_ETHTOOL_PRIV_FLAGS_COUNT; i++)
+			strscpy(strings + i * ETH_GSTRING_LEN,
+				efx_ethtool_priv_flags_strings[i],
+				ETH_GSTRING_LEN);
+		break;
 	default:
 		/* No other string sets */
 		break;
 	}
 }
 
+u32 efx_ethtool_get_priv_flags(struct net_device *net_dev)
+{
+	struct efx_nic *efx = efx_netdev_priv(net_dev);
+	u32 ret_flags = 0;
+
+	if (efx->log_tc_errs)
+		ret_flags |= EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS;
+
+	return ret_flags;
+}
+
+int efx_ethtool_set_priv_flags(struct net_device *net_dev, u32 flags)
+{
+	struct efx_nic *efx = efx_netdev_priv(net_dev);
+
+	efx->log_tc_errs =
+		!!(flags & EFX_ETHTOOL_PRIV_FLAGS_LOG_TC_ERRS);
+
+	return 0;
+}
+
 void efx_ethtool_get_stats(struct net_device *net_dev,
 			   struct ethtool_stats *stats,
 			   u64 *data)
diff --git a/drivers/net/ethernet/sfc/ethtool_common.h b/drivers/net/ethernet/sfc/ethtool_common.h
index 659491932101..0afc74021a5e 100644
--- a/drivers/net/ethernet/sfc/ethtool_common.h
+++ b/drivers/net/ethernet/sfc/ethtool_common.h
@@ -27,6 +27,8 @@ int efx_ethtool_fill_self_tests(struct efx_nic *efx,
 int efx_ethtool_get_sset_count(struct net_device *net_dev, int string_set);
 void efx_ethtool_get_strings(struct net_device *net_dev, u32 string_set,
 			     u8 *strings);
+u32 efx_ethtool_get_priv_flags(struct net_device *net_dev);
+int efx_ethtool_set_priv_flags(struct net_device *net_dev, u32 flags);
 void efx_ethtool_get_stats(struct net_device *net_dev,
 			   struct ethtool_stats *stats __attribute__ ((unused)),
 			   u64 *data);
diff --git a/drivers/net/ethernet/sfc/falcon/efx.c b/drivers/net/ethernet/sfc/falcon/efx.c
index a63f40b09856..e151b0957751 100644
--- a/drivers/net/ethernet/sfc/falcon/efx.c
+++ b/drivers/net/ethernet/sfc/falcon/efx.c
@@ -2012,7 +2012,7 @@ static void ef4_init_napi_channel(struct ef4_channel *channel)
 	struct ef4_nic *efx = channel->efx;
 
 	channel->napi_dev = efx->net_dev;
-	netif_napi_add(channel->napi_dev, &channel->napi_str, ef4_poll, 64);
+	netif_napi_add(channel->napi_dev, &channel->napi_str, ef4_poll);
 }
 
 static void ef4_init_napi(struct ef4_nic *efx)
@@ -2329,7 +2329,7 @@ static void ef4_unregister_netdev(struct ef4_nic *efx)
 	BUG_ON(netdev_priv(efx->net_dev) != efx);
 
 	if (ef4_dev_registered(efx)) {
-		strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
+		strscpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
 		device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
 		unregister_netdev(efx->net_dev);
 	}
@@ -2640,7 +2640,7 @@ static int ef4_init_struct(struct ef4_nic *efx,
 	efx->pci_dev = pci_dev;
 	efx->msg_enable = debug;
 	efx->state = STATE_UNINIT;
-	strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
+	strscpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
 
 	efx->net_dev = net_dev;
 	efx->rx_prefix_size = efx->type->rx_prefix_size;
diff --git a/drivers/net/ethernet/sfc/falcon/ethtool.c b/drivers/net/ethernet/sfc/falcon/ethtool.c
index 907254b36663..3976a333f7e3 100644
--- a/drivers/net/ethernet/sfc/falcon/ethtool.c
+++ b/drivers/net/ethernet/sfc/falcon/ethtool.c
@@ -162,9 +162,9 @@ static void ef4_ethtool_get_drvinfo(struct net_device *net_dev,
 {
 	struct ef4_nic *efx = netdev_priv(net_dev);
 
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
-	strlcpy(info->version, EF4_DRIVER_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->version, EF4_DRIVER_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info));
 }
 
 static int ef4_ethtool_get_regs_len(struct net_device *net_dev)
@@ -412,7 +412,7 @@ static void ef4_ethtool_get_strings(struct net_device *net_dev,
 		strings += (efx->type->describe_stats(efx, strings) *
 			    ETH_GSTRING_LEN);
 		for (i = 0; i < EF4_ETHTOOL_SW_STAT_COUNT; i++)
-			strlcpy(strings + i * ETH_GSTRING_LEN,
+			strscpy(strings + i * ETH_GSTRING_LEN,
 				ef4_sw_stat_desc[i].name, ETH_GSTRING_LEN);
 		strings += EF4_ETHTOOL_SW_STAT_COUNT * ETH_GSTRING_LEN;
 		strings += (ef4_describe_per_queue_stats(efx, strings) *
diff --git a/drivers/net/ethernet/sfc/falcon/falcon.c b/drivers/net/ethernet/sfc/falcon/falcon.c
index 3324a6219a09..7a1c9337081b 100644
--- a/drivers/net/ethernet/sfc/falcon/falcon.c
+++ b/drivers/net/ethernet/sfc/falcon/falcon.c
@@ -2387,7 +2387,7 @@ static int falcon_probe_nic(struct ef4_nic *efx)
 	board->i2c_data.data = efx;
 	board->i2c_adap.algo_data = &board->i2c_data;
 	board->i2c_adap.dev.parent = &efx->pci_dev->dev;
-	strlcpy(board->i2c_adap.name, "SFC4000 GPIO",
+	strscpy(board->i2c_adap.name, "SFC4000 GPIO",
 		sizeof(board->i2c_adap.name));
 	rc = i2c_bit_add_bus(&board->i2c_adap);
 	if (rc)
diff --git a/drivers/net/ethernet/sfc/falcon/nic.c b/drivers/net/ethernet/sfc/falcon/nic.c
index 156da315ec89..78c851b5a56f 100644
--- a/drivers/net/ethernet/sfc/falcon/nic.c
+++ b/drivers/net/ethernet/sfc/falcon/nic.c
@@ -452,7 +452,7 @@ size_t ef4_nic_describe_stats(const struct ef4_hw_stat_desc *desc, size_t count,
 	for_each_set_bit(index, mask, count) {
 		if (desc[index].name) {
 			if (names) {
-				strlcpy(names, desc[index].name,
+				strscpy(names, desc[index].name,
 					ETH_GSTRING_LEN);
 				names += ETH_GSTRING_LEN;
 			}
diff --git a/drivers/net/ethernet/sfc/filter.h b/drivers/net/ethernet/sfc/filter.h
index 4d928839d292..be72e71da027 100644
--- a/drivers/net/ethernet/sfc/filter.h
+++ b/drivers/net/ethernet/sfc/filter.h
@@ -9,6 +9,7 @@
 
 #include <linux/types.h>
 #include <linux/if_ether.h>
+#include <linux/in6.h>
 #include <asm/byteorder.h>
 
 /**
@@ -224,6 +225,27 @@ efx_filter_set_ipv4_local(struct efx_filter_spec *spec, u8 proto,
 }
 
 /**
+ * efx_filter_set_ipv6_local - specify IPv6 host, transport protocol and port
+ * @spec: Specification to initialise
+ * @proto: Transport layer protocol number
+ * @host: Local host address (network byte order)
+ * @port: Local port (network byte order)
+ */
+static inline int
+efx_filter_set_ipv6_local(struct efx_filter_spec *spec, u8 proto,
+			  const struct in6_addr *host, __be16 port)
+{
+	spec->match_flags |=
+		EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO |
+		EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT;
+	spec->ether_type = htons(ETH_P_IPV6);
+	spec->ip_proto = proto;
+	memcpy(spec->loc_host, host, sizeof(spec->loc_host));
+	spec->loc_port = port;
+	return 0;
+}
+
+/**
  * efx_filter_set_ipv4_full - specify IPv4 hosts, transport protocol and ports
  * @spec: Specification to initialise
  * @proto: Transport layer protocol number
diff --git a/drivers/net/ethernet/sfc/mae.c b/drivers/net/ethernet/sfc/mae.c
index 97627f5e3674..874c765b2465 100644
--- a/drivers/net/ethernet/sfc/mae.c
+++ b/drivers/net/ethernet/sfc/mae.c
@@ -112,6 +112,167 @@ int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id)
 	return 0;
 }
 
+static int efx_mae_get_basic_caps(struct efx_nic *efx, struct mae_caps *caps)
+{
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_GET_CAPS_OUT_LEN);
+	size_t outlen;
+	int rc;
+
+	BUILD_BUG_ON(MC_CMD_MAE_GET_CAPS_IN_LEN);
+
+	rc = efx_mcdi_rpc(efx, MC_CMD_MAE_GET_CAPS, NULL, 0, outbuf,
+			  sizeof(outbuf), &outlen);
+	if (rc)
+		return rc;
+	if (outlen < sizeof(outbuf))
+		return -EIO;
+	caps->match_field_count = MCDI_DWORD(outbuf, MAE_GET_CAPS_OUT_MATCH_FIELD_COUNT);
+	caps->action_prios = MCDI_DWORD(outbuf, MAE_GET_CAPS_OUT_ACTION_PRIOS);
+	return 0;
+}
+
+static int efx_mae_get_rule_fields(struct efx_nic *efx, u32 cmd,
+				   u8 *field_support)
+{
+	MCDI_DECLARE_BUF(outbuf, MC_CMD_MAE_GET_AR_CAPS_OUT_LEN(MAE_NUM_FIELDS));
+	MCDI_DECLARE_STRUCT_PTR(caps);
+	unsigned int count;
+	size_t outlen;
+	int rc, i;
+
+	BUILD_BUG_ON(MC_CMD_MAE_GET_AR_CAPS_IN_LEN);
+
+	rc = efx_mcdi_rpc(efx, cmd, NULL, 0, outbuf, sizeof(outbuf), &outlen);
+	if (rc)
+		return rc;
+	count = MCDI_DWORD(outbuf, MAE_GET_AR_CAPS_OUT_COUNT);
+	memset(field_support, MAE_FIELD_UNSUPPORTED, MAE_NUM_FIELDS);
+	caps = _MCDI_DWORD(outbuf, MAE_GET_AR_CAPS_OUT_FIELD_FLAGS);
+	/* We're only interested in the support status enum, not any other
+	 * flags, so just extract that from each entry.
+	 */
+	for (i = 0; i < count; i++)
+		if (i * sizeof(*outbuf) + MC_CMD_MAE_GET_AR_CAPS_OUT_FIELD_FLAGS_OFST < outlen)
+			field_support[i] = EFX_DWORD_FIELD(caps[i], MAE_FIELD_FLAGS_SUPPORT_STATUS);
+	return 0;
+}
+
+int efx_mae_get_caps(struct efx_nic *efx, struct mae_caps *caps)
+{
+	int rc;
+
+	rc = efx_mae_get_basic_caps(efx, caps);
+	if (rc)
+		return rc;
+	return efx_mae_get_rule_fields(efx, MC_CMD_MAE_GET_AR_CAPS,
+				       caps->action_rule_fields);
+}
+
+/* Bit twiddling:
+ * Prefix: 1...110...0
+ *      ~: 0...001...1
+ *    + 1: 0...010...0 is power of two
+ * so (~x) & ((~x) + 1) == 0.  Converse holds also.
+ */
+#define is_prefix_byte(_x)	!(((_x) ^ 0xff) & (((_x) ^ 0xff) + 1))
+
+enum mask_type { MASK_ONES, MASK_ZEROES, MASK_PREFIX, MASK_OTHER };
+
+static const char *mask_type_name(enum mask_type typ)
+{
+	switch (typ) {
+	case MASK_ONES:
+		return "all-1s";
+	case MASK_ZEROES:
+		return "all-0s";
+	case MASK_PREFIX:
+		return "prefix";
+	case MASK_OTHER:
+		return "arbitrary";
+	default: /* can't happen */
+		return "unknown";
+	}
+}
+
+/* Checks a (big-endian) bytestring is a bit prefix */
+static enum mask_type classify_mask(const u8 *mask, size_t len)
+{
+	bool zeroes = true; /* All bits seen so far are zeroes */
+	bool ones = true; /* All bits seen so far are ones */
+	bool prefix = true; /* Valid prefix so far */
+	size_t i;
+
+	for (i = 0; i < len; i++) {
+		if (ones) {
+			if (!is_prefix_byte(mask[i]))
+				prefix = false;
+		} else if (mask[i]) {
+			prefix = false;
+		}
+		if (mask[i] != 0xff)
+			ones = false;
+		if (mask[i])
+			zeroes = false;
+	}
+	if (ones)
+		return MASK_ONES;
+	if (zeroes)
+		return MASK_ZEROES;
+	if (prefix)
+		return MASK_PREFIX;
+	return MASK_OTHER;
+}
+
+static int efx_mae_match_check_cap_typ(u8 support, enum mask_type typ)
+{
+	switch (support) {
+	case MAE_FIELD_UNSUPPORTED:
+	case MAE_FIELD_SUPPORTED_MATCH_NEVER:
+		if (typ == MASK_ZEROES)
+			return 0;
+		return -EOPNOTSUPP;
+	case MAE_FIELD_SUPPORTED_MATCH_OPTIONAL:
+		if (typ == MASK_ZEROES)
+			return 0;
+		fallthrough;
+	case MAE_FIELD_SUPPORTED_MATCH_ALWAYS:
+		if (typ == MASK_ONES)
+			return 0;
+		return -EINVAL;
+	case MAE_FIELD_SUPPORTED_MATCH_PREFIX:
+		if (typ == MASK_OTHER)
+			return -EOPNOTSUPP;
+		return 0;
+	case MAE_FIELD_SUPPORTED_MATCH_MASK:
+		return 0;
+	default:
+		return -EIO;
+	}
+}
+
+int efx_mae_match_check_caps(struct efx_nic *efx,
+			     const struct efx_tc_match_fields *mask,
+			     struct netlink_ext_ack *extack)
+{
+	const u8 *supported_fields = efx->tc->caps->action_rule_fields;
+	__be32 ingress_port = cpu_to_be32(mask->ingress_port);
+	enum mask_type ingress_port_mask_type;
+	int rc;
+
+	/* Check for _PREFIX assumes big-endian, so we need to convert */
+	ingress_port_mask_type = classify_mask((const u8 *)&ingress_port,
+					       sizeof(ingress_port));
+	rc = efx_mae_match_check_cap_typ(supported_fields[MAE_FIELD_INGRESS_PORT],
+					 ingress_port_mask_type);
+	if (rc) {
+		efx_tc_err(efx, "No support for %s mask in field ingress_port\n",
+			   mask_type_name(ingress_port_mask_type));
+		NL_SET_ERR_MSG_MOD(extack, "Unsupported mask type for ingress_port");
+		return rc;
+	}
+	return 0;
+}
+
 static bool efx_mae_asl_id(u32 id)
 {
 	return !!(id & BIT(31));
@@ -279,6 +440,10 @@ static int efx_mae_populate_match_criteria(MCDI_DECLARE_STRUCT_PTR(match_crit),
 	}
 	MCDI_STRUCT_SET_DWORD(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_INGRESS_MPORT_SELECTOR_MASK,
 			      match->mask.ingress_port);
+	MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID,
+			     match->value.recirc_id);
+	MCDI_STRUCT_SET_BYTE(match_crit, MAE_FIELD_MASK_VALUE_PAIRS_V2_RECIRC_ID_MASK,
+			     match->mask.recirc_id);
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/sfc/mae.h b/drivers/net/ethernet/sfc/mae.h
index 0369be4d8983..3e0cd238d523 100644
--- a/drivers/net/ethernet/sfc/mae.h
+++ b/drivers/net/ethernet/sfc/mae.h
@@ -27,6 +27,20 @@ void efx_mae_mport_mport(struct efx_nic *efx, u32 mport_id, u32 *out);
 
 int efx_mae_lookup_mport(struct efx_nic *efx, u32 selector, u32 *id);
 
+#define MAE_NUM_FIELDS	(MAE_FIELD_ENC_VNET_ID + 1)
+
+struct mae_caps {
+	u32 match_field_count;
+	u32 action_prios;
+	u8 action_rule_fields[MAE_NUM_FIELDS];
+};
+
+int efx_mae_get_caps(struct efx_nic *efx, struct mae_caps *caps);
+
+int efx_mae_match_check_caps(struct efx_nic *efx,
+			     const struct efx_tc_match_fields *mask,
+			     struct netlink_ext_ack *extack);
+
 int efx_mae_alloc_action_set(struct efx_nic *efx, struct efx_tc_action_set *act);
 int efx_mae_free_action_set(struct efx_nic *efx, u32 fw_id);
 
diff --git a/drivers/net/ethernet/sfc/mcdi.h b/drivers/net/ethernet/sfc/mcdi.h
index 26bc69f76801..1f18e9dc62e8 100644
--- a/drivers/net/ethernet/sfc/mcdi.h
+++ b/drivers/net/ethernet/sfc/mcdi.h
@@ -201,6 +201,12 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev);
 	((u8 *)(_buf) + (_offset))
 #define MCDI_PTR(_buf, _field)						\
 	_MCDI_PTR(_buf, MC_CMD_ ## _field ## _OFST)
+/* Use MCDI_STRUCT_ functions to access members of MCDI structuredefs.
+ * _buf should point to the start of the structure, typically obtained with
+ * MCDI_DECLARE_STRUCT_PTR(structure) = _MCDI_DWORD(mcdi_buf, FIELD_WHICH_IS_STRUCT);
+ */
+#define MCDI_STRUCT_PTR(_buf, _field)					\
+	_MCDI_PTR(_buf, _field ## _OFST)
 #define _MCDI_CHECK_ALIGN(_ofst, _align)				\
 	((_ofst) + BUILD_BUG_ON_ZERO((_ofst) & (_align - 1)))
 #define _MCDI_DWORD(_buf, _field)					\
@@ -208,6 +214,10 @@ void efx_mcdi_sensor_event(struct efx_nic *efx, efx_qword_t *ev);
 #define _MCDI_STRUCT_DWORD(_buf, _field)				\
 	((_buf) + (_MCDI_CHECK_ALIGN(_field ## _OFST, 4) >> 2))
 
+#define MCDI_STRUCT_SET_BYTE(_buf, _field, _value) do {			\
+	BUILD_BUG_ON(_field ## _LEN != 1);				\
+	*(u8 *)MCDI_STRUCT_PTR(_buf, _field) = _value;			\
+	} while (0)
 #define MCDI_BYTE(_buf, _field)						\
 	((void)BUILD_BUG_ON_ZERO(MC_CMD_ ## _field ## _LEN != 1),	\
 	 *MCDI_PTR(_buf, _field))
diff --git a/drivers/net/ethernet/sfc/mcdi_mon.c b/drivers/net/ethernet/sfc/mcdi_mon.c
index 5954fcfee2b1..f5128db7c7e7 100644
--- a/drivers/net/ethernet/sfc/mcdi_mon.c
+++ b/drivers/net/ethernet/sfc/mcdi_mon.c
@@ -285,7 +285,7 @@ efx_mcdi_mon_add_attr(struct efx_nic *efx, const char *name,
 	struct efx_mcdi_mon *hwmon = efx_mcdi_mon(efx);
 	struct efx_mcdi_mon_attribute *attr = &hwmon->attrs[hwmon->n_attrs];
 
-	strlcpy(attr->name, name, sizeof(attr->name));
+	strscpy(attr->name, name, sizeof(attr->name));
 	attr->index = index;
 	attr->type = type;
 	if (type < ARRAY_SIZE(efx_mcdi_sensor_type))
diff --git a/drivers/net/ethernet/sfc/net_driver.h b/drivers/net/ethernet/sfc/net_driver.h
index 7ef823d7a89a..2e9ba0cfe848 100644
--- a/drivers/net/ethernet/sfc/net_driver.h
+++ b/drivers/net/ethernet/sfc/net_driver.h
@@ -855,6 +855,7 @@ enum efx_xdp_tx_queues_mode {
  * @timer_max_ns: Interrupt timer maximum value, in nanoseconds
  * @irq_rx_adaptive: Adaptive IRQ moderation enabled for RX event queues
  * @irqs_hooked: Channel interrupts are hooked
+ * @log_tc_errs: Error logging for TC filter insertion is enabled
  * @irq_rx_mod_step_us: Step size for IRQ moderation for RX event queues
  * @irq_rx_moderation_us: IRQ moderation time for RX event queues
  * @msg_enable: Log message enable flags
@@ -1017,6 +1018,7 @@ struct efx_nic {
 	unsigned int timer_max_ns;
 	bool irq_rx_adaptive;
 	bool irqs_hooked;
+	bool log_tc_errs;
 	unsigned int irq_mod_step_us;
 	unsigned int irq_rx_moderation_us;
 	u32 msg_enable;
diff --git a/drivers/net/ethernet/sfc/nic.c b/drivers/net/ethernet/sfc/nic.c
index 22fbb0ae77fb..63e2394382bb 100644
--- a/drivers/net/ethernet/sfc/nic.c
+++ b/drivers/net/ethernet/sfc/nic.c
@@ -465,7 +465,7 @@ size_t efx_nic_describe_stats(const struct efx_hw_stat_desc *desc, size_t count,
 	for_each_set_bit(index, mask, count) {
 		if (desc[index].name) {
 			if (names) {
-				strlcpy(names, desc[index].name,
+				strscpy(names, desc[index].name,
 					ETH_GSTRING_LEN);
 				names += ETH_GSTRING_LEN;
 			}
diff --git a/drivers/net/ethernet/sfc/ptp.c b/drivers/net/ethernet/sfc/ptp.c
index 10ad0b93d283..eaef4a15008a 100644
--- a/drivers/net/ethernet/sfc/ptp.c
+++ b/drivers/net/ethernet/sfc/ptp.c
@@ -118,9 +118,14 @@
 
 #define	PTP_MIN_LENGTH		63
 
-#define PTP_ADDRESS		0xe0000181	/* 224.0.1.129 */
+#define PTP_RXFILTERS_LEN	5
+
+#define PTP_ADDR_IPV4		0xe0000181	/* 224.0.1.129 */
+#define PTP_ADDR_IPV6		{0xff, 0x0e, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
+				0, 0x01, 0x81}	/* ff0e::181 */
 #define PTP_EVENT_PORT		319
 #define PTP_GENERAL_PORT	320
+#define PTP_ADDR_ETHER		{0x01, 0x1b, 0x19, 0, 0, 0} /* 01-1B-19-00-00-00 */
 
 /* Annoyingly the format of the version numbers are different between
  * versions 1 and 2 so it isn't possible to simply look for 1 or 2.
@@ -224,9 +229,8 @@ struct efx_ptp_timeset {
  * @work: Work task
  * @reset_required: A serious error has occurred and the PTP task needs to be
  *                  reset (disable, enable).
- * @rxfilter_event: Receive filter when operating
- * @rxfilter_general: Receive filter when operating
- * @rxfilter_installed: Receive filter installed
+ * @rxfilters: Receive filters when operating
+ * @rxfilters_count: Num of installed rxfilters, should be == PTP_RXFILTERS_LEN
  * @config: Current timestamp configuration
  * @enabled: PTP operation enabled
  * @mode: Mode in which PTP operating (PTP version)
@@ -295,9 +299,8 @@ struct efx_ptp_data {
 	struct workqueue_struct *workwq;
 	struct work_struct work;
 	bool reset_required;
-	u32 rxfilter_event;
-	u32 rxfilter_general;
-	bool rxfilter_installed;
+	u32 rxfilters[PTP_RXFILTERS_LEN];
+	size_t rxfilters_count;
 	struct hwtstamp_config config;
 	bool enabled;
 	unsigned int mode;
@@ -1290,61 +1293,108 @@ static void efx_ptp_remove_multicast_filters(struct efx_nic *efx)
 {
 	struct efx_ptp_data *ptp = efx->ptp_data;
 
-	if (ptp->rxfilter_installed) {
-		efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,
-					  ptp->rxfilter_general);
+	while (ptp->rxfilters_count) {
+		ptp->rxfilters_count--;
 		efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,
-					  ptp->rxfilter_event);
-		ptp->rxfilter_installed = false;
+					  ptp->rxfilters[ptp->rxfilters_count]);
 	}
 }
 
-static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)
+static void efx_ptp_init_filter(struct efx_nic *efx,
+				struct efx_filter_spec *rxfilter)
+{
+	struct efx_channel *channel = efx->ptp_data->channel;
+	struct efx_rx_queue *queue = efx_channel_get_rx_queue(channel);
+
+	efx_filter_init_rx(rxfilter, EFX_FILTER_PRI_REQUIRED, 0,
+			   efx_rx_queue_index(queue));
+}
+
+static int efx_ptp_insert_filter(struct efx_nic *efx,
+				 struct efx_filter_spec *rxfilter)
 {
 	struct efx_ptp_data *ptp = efx->ptp_data;
+
+	int rc = efx_filter_insert_filter(efx, rxfilter, true);
+	if (rc < 0)
+		return rc;
+	ptp->rxfilters[ptp->rxfilters_count] = rc;
+	ptp->rxfilters_count++;
+	return 0;
+}
+
+static int efx_ptp_insert_ipv4_filter(struct efx_nic *efx, u16 port)
+{
 	struct efx_filter_spec rxfilter;
+
+	efx_ptp_init_filter(efx, &rxfilter);
+	efx_filter_set_ipv4_local(&rxfilter, IPPROTO_UDP, htonl(PTP_ADDR_IPV4),
+				  htons(port));
+	return efx_ptp_insert_filter(efx, &rxfilter);
+}
+
+static int efx_ptp_insert_ipv6_filter(struct efx_nic *efx, u16 port)
+{
+	const struct in6_addr addr = {{PTP_ADDR_IPV6}};
+	struct efx_filter_spec rxfilter;
+
+	efx_ptp_init_filter(efx, &rxfilter);
+	efx_filter_set_ipv6_local(&rxfilter, IPPROTO_UDP, &addr, htons(port));
+	return efx_ptp_insert_filter(efx, &rxfilter);
+}
+
+static int efx_ptp_insert_eth_filter(struct efx_nic *efx)
+{
+	const u8 addr[ETH_ALEN] = PTP_ADDR_ETHER;
+	struct efx_filter_spec rxfilter;
+
+	efx_ptp_init_filter(efx, &rxfilter);
+	efx_filter_set_eth_local(&rxfilter, EFX_FILTER_VID_UNSPEC, addr);
+	rxfilter.match_flags |= EFX_FILTER_MATCH_ETHER_TYPE;
+	rxfilter.ether_type = htons(ETH_P_1588);
+	return efx_ptp_insert_filter(efx, &rxfilter);
+}
+
+static int efx_ptp_insert_multicast_filters(struct efx_nic *efx)
+{
+	struct efx_ptp_data *ptp = efx->ptp_data;
 	int rc;
 
-	if (!ptp->channel || ptp->rxfilter_installed)
+	if (!ptp->channel || ptp->rxfilters_count)
 		return 0;
 
 	/* Must filter on both event and general ports to ensure
 	 * that there is no packet re-ordering.
 	 */
-	efx_filter_init_rx(&rxfilter, EFX_FILTER_PRI_REQUIRED, 0,
-			   efx_rx_queue_index(
-				   efx_channel_get_rx_queue(ptp->channel)));
-	rc = efx_filter_set_ipv4_local(&rxfilter, IPPROTO_UDP,
-				       htonl(PTP_ADDRESS),
-				       htons(PTP_EVENT_PORT));
-	if (rc != 0)
-		return rc;
-
-	rc = efx_filter_insert_filter(efx, &rxfilter, true);
+	rc = efx_ptp_insert_ipv4_filter(efx, PTP_EVENT_PORT);
 	if (rc < 0)
-		return rc;
-	ptp->rxfilter_event = rc;
-
-	efx_filter_init_rx(&rxfilter, EFX_FILTER_PRI_REQUIRED, 0,
-			   efx_rx_queue_index(
-				   efx_channel_get_rx_queue(ptp->channel)));
-	rc = efx_filter_set_ipv4_local(&rxfilter, IPPROTO_UDP,
-				       htonl(PTP_ADDRESS),
-				       htons(PTP_GENERAL_PORT));
-	if (rc != 0)
 		goto fail;
 
-	rc = efx_filter_insert_filter(efx, &rxfilter, true);
+	rc = efx_ptp_insert_ipv4_filter(efx, PTP_GENERAL_PORT);
 	if (rc < 0)
 		goto fail;
-	ptp->rxfilter_general = rc;
 
-	ptp->rxfilter_installed = true;
+	/* if the NIC supports hw timestamps by the MAC, we can support
+	 * PTP over IPv6 and Ethernet
+	 */
+	if (efx_ptp_use_mac_tx_timestamps(efx)) {
+		rc = efx_ptp_insert_ipv6_filter(efx, PTP_EVENT_PORT);
+		if (rc < 0)
+			goto fail;
+
+		rc = efx_ptp_insert_ipv6_filter(efx, PTP_GENERAL_PORT);
+		if (rc < 0)
+			goto fail;
+
+		rc = efx_ptp_insert_eth_filter(efx);
+		if (rc < 0)
+			goto fail;
+	}
+
 	return 0;
 
 fail:
-	efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED,
-				  ptp->rxfilter_event);
+	efx_ptp_remove_multicast_filters(efx);
 	return rc;
 }
 
diff --git a/drivers/net/ethernet/sfc/siena/efx.c b/drivers/net/ethernet/sfc/siena/efx.c
index 63d999e63960..60e5b7c8ccf9 100644
--- a/drivers/net/ethernet/sfc/siena/efx.c
+++ b/drivers/net/ethernet/sfc/siena/efx.c
@@ -775,7 +775,7 @@ static void efx_unregister_netdev(struct efx_nic *efx)
 	BUG_ON(netdev_priv(efx->net_dev) != efx);
 
 	if (efx_dev_registered(efx)) {
-		strlcpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
+		strscpy(efx->name, pci_name(efx->pci_dev), sizeof(efx->name));
 		efx_siena_fini_mcdi_logging(efx);
 		device_remove_file(&efx->pci_dev->dev, &dev_attr_phy_type);
 		unregister_netdev(efx->net_dev);
@@ -1148,6 +1148,17 @@ static int efx_pm_freeze(struct device *dev)
 	return 0;
 }
 
+static void efx_pci_shutdown(struct pci_dev *pci_dev)
+{
+	struct efx_nic *efx = pci_get_drvdata(pci_dev);
+
+	if (!efx)
+		return;
+
+	efx_pm_freeze(&pci_dev->dev);
+	pci_disable_device(pci_dev);
+}
+
 static int efx_pm_thaw(struct device *dev)
 {
 	int rc;
@@ -1252,6 +1263,7 @@ static struct pci_driver efx_pci_driver = {
 	.probe		= efx_pci_probe,
 	.remove		= efx_pci_remove,
 	.driver.pm	= &efx_pm_ops,
+	.shutdown	= efx_pci_shutdown,
 	.err_handler	= &efx_siena_err_handlers,
 #ifdef CONFIG_SFC_SIENA_SRIOV
 	.sriov_configure = efx_pci_sriov_configure,
diff --git a/drivers/net/ethernet/sfc/siena/efx_channels.c b/drivers/net/ethernet/sfc/siena/efx_channels.c
index f54ebd007286..06ed74994e36 100644
--- a/drivers/net/ethernet/sfc/siena/efx_channels.c
+++ b/drivers/net/ethernet/sfc/siena/efx_channels.c
@@ -1317,7 +1317,7 @@ static void efx_init_napi_channel(struct efx_channel *channel)
 	struct efx_nic *efx = channel->efx;
 
 	channel->napi_dev = efx->net_dev;
-	netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll, 64);
+	netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll);
 }
 
 void efx_siena_init_napi(struct efx_nic *efx)
diff --git a/drivers/net/ethernet/sfc/siena/efx_common.c b/drivers/net/ethernet/sfc/siena/efx_common.c
index 954daf464abb..1fd396b00bfb 100644
--- a/drivers/net/ethernet/sfc/siena/efx_common.c
+++ b/drivers/net/ethernet/sfc/siena/efx_common.c
@@ -1006,7 +1006,7 @@ int efx_siena_init_struct(struct efx_nic *efx,
 	efx->pci_dev = pci_dev;
 	efx->msg_enable = debug;
 	efx->state = STATE_UNINIT;
-	strlcpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
+	strscpy(efx->name, pci_name(pci_dev), sizeof(efx->name));
 
 	efx->net_dev = net_dev;
 	efx->rx_prefix_size = efx->type->rx_prefix_size;
diff --git a/drivers/net/ethernet/sfc/siena/ethtool_common.c b/drivers/net/ethernet/sfc/siena/ethtool_common.c
index 0207d07f54e3..f590e87e5a23 100644
--- a/drivers/net/ethernet/sfc/siena/ethtool_common.c
+++ b/drivers/net/ethernet/sfc/siena/ethtool_common.c
@@ -105,10 +105,10 @@ void efx_siena_ethtool_get_drvinfo(struct net_device *net_dev,
 {
 	struct efx_nic *efx = netdev_priv(net_dev);
 
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
 	efx_siena_mcdi_print_fwver(efx, info->fw_version,
 				   sizeof(info->fw_version));
-	strlcpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info));
+	strscpy(info->bus_info, pci_name(efx->pci_dev), sizeof(info->bus_info));
 }
 
 u32 efx_siena_ethtool_get_msglevel(struct net_device *net_dev)
@@ -467,7 +467,7 @@ void efx_siena_ethtool_get_strings(struct net_device *net_dev,
 		strings += (efx->type->describe_stats(efx, strings) *
 			    ETH_GSTRING_LEN);
 		for (i = 0; i < EFX_ETHTOOL_SW_STAT_COUNT; i++)
-			strlcpy(strings + i * ETH_GSTRING_LEN,
+			strscpy(strings + i * ETH_GSTRING_LEN,
 				efx_sw_stat_desc[i].name, ETH_GSTRING_LEN);
 		strings += EFX_ETHTOOL_SW_STAT_COUNT * ETH_GSTRING_LEN;
 		strings += (efx_describe_per_queue_stats(efx, strings) *
diff --git a/drivers/net/ethernet/sfc/siena/mcdi_mon.c b/drivers/net/ethernet/sfc/siena/mcdi_mon.c
index c7ea703c5d7a..56a9c56ed9e3 100644
--- a/drivers/net/ethernet/sfc/siena/mcdi_mon.c
+++ b/drivers/net/ethernet/sfc/siena/mcdi_mon.c
@@ -285,7 +285,7 @@ efx_mcdi_mon_add_attr(struct efx_nic *efx, const char *name,
 	struct efx_mcdi_mon *hwmon = efx_mcdi_mon(efx);
 	struct efx_mcdi_mon_attribute *attr = &hwmon->attrs[hwmon->n_attrs];
 
-	strlcpy(attr->name, name, sizeof(attr->name));
+	strscpy(attr->name, name, sizeof(attr->name));
 	attr->index = index;
 	attr->type = type;
 	if (type < ARRAY_SIZE(efx_mcdi_sensor_type))
diff --git a/drivers/net/ethernet/sfc/siena/nic.c b/drivers/net/ethernet/sfc/siena/nic.c
index abf9a4adf139..0ea0433a6230 100644
--- a/drivers/net/ethernet/sfc/siena/nic.c
+++ b/drivers/net/ethernet/sfc/siena/nic.c
@@ -458,7 +458,7 @@ size_t efx_siena_describe_stats(const struct efx_hw_stat_desc *desc, size_t coun
 	for_each_set_bit(index, mask, count) {
 		if (desc[index].name) {
 			if (names) {
-				strlcpy(names, desc[index].name,
+				strscpy(names, desc[index].name,
 					ETH_GSTRING_LEN);
 				names += ETH_GSTRING_LEN;
 			}
diff --git a/drivers/net/ethernet/sfc/tc.c b/drivers/net/ethernet/sfc/tc.c
index 0c0aeb91f500..3478860d4023 100644
--- a/drivers/net/ethernet/sfc/tc.c
+++ b/drivers/net/ethernet/sfc/tc.c
@@ -9,11 +9,60 @@
  * by the Free Software Foundation, incorporated herein by reference.
  */
 
+#include <net/pkt_cls.h>
 #include "tc.h"
+#include "tc_bindings.h"
 #include "mae.h"
 #include "ef100_rep.h"
 #include "efx.h"
 
+#define EFX_EFV_PF	NULL
+/* Look up the representor information (efv) for a device.
+ * May return NULL for the PF (us), or an error pointer for a device that
+ * isn't supported as a TC offload endpoint
+ */
+static struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx,
+						struct net_device *dev)
+{
+	struct efx_rep *efv;
+
+	if (!dev)
+		return ERR_PTR(-EOPNOTSUPP);
+	/* Is it us (the PF)? */
+	if (dev == efx->net_dev)
+		return EFX_EFV_PF;
+	/* Is it an efx vfrep at all? */
+	if (dev->netdev_ops != &efx_ef100_rep_netdev_ops)
+		return ERR_PTR(-EOPNOTSUPP);
+	/* Is it ours?  We don't support TC rules that include another
+	 * EF100's netdevices (not even on another port of the same NIC).
+	 */
+	efv = netdev_priv(dev);
+	if (efv->parent != efx)
+		return ERR_PTR(-EOPNOTSUPP);
+	return efv;
+}
+
+/* Convert a driver-internal vport ID into an external device (wire or VF) */
+static s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv)
+{
+	u32 mport;
+
+	if (IS_ERR(efv))
+		return PTR_ERR(efv);
+	if (!efv) /* device is PF (us) */
+		efx_mae_mport_wire(efx, &mport);
+	else /* device is repr */
+		efx_mae_mport_mport(efx, efv->mport, &mport);
+	return mport;
+}
+
+static const struct rhashtable_params efx_tc_match_action_ht_params = {
+	.key_len	= sizeof(unsigned long),
+	.key_offset	= offsetof(struct efx_tc_flow_rule, cookie),
+	.head_offset	= offsetof(struct efx_tc_flow_rule, linkage),
+};
+
 static void efx_tc_free_action_set(struct efx_nic *efx,
 				   struct efx_tc_action_set *act, bool in_hw)
 {
@@ -58,6 +107,333 @@ static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rul
 	rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
 }
 
+static void efx_tc_flow_free(void *ptr, void *arg)
+{
+	struct efx_tc_flow_rule *rule = ptr;
+	struct efx_nic *efx = arg;
+
+	netif_err(efx, drv, efx->net_dev,
+		  "tc rule %lx still present at teardown, removing\n",
+		  rule->cookie);
+
+	efx_mae_delete_rule(efx, rule->fw_id);
+
+	/* Release entries in subsidiary tables */
+	efx_tc_free_action_set_list(efx, &rule->acts, true);
+
+	kfree(rule);
+}
+
+static int efx_tc_flower_parse_match(struct efx_nic *efx,
+				     struct flow_rule *rule,
+				     struct efx_tc_match *match,
+				     struct netlink_ext_ack *extack)
+{
+	struct flow_dissector *dissector = rule->match.dissector;
+
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
+		struct flow_match_control fm;
+
+		flow_rule_match_control(rule, &fm);
+
+		if (fm.mask->flags) {
+			efx_tc_err(efx, "Unsupported match on control.flags %#x\n",
+				   fm.mask->flags);
+			NL_SET_ERR_MSG_MOD(extack, "Unsupported match on control.flags");
+			return -EOPNOTSUPP;
+		}
+	}
+	if (dissector->used_keys &
+	    ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
+	      BIT(FLOW_DISSECTOR_KEY_BASIC))) {
+		efx_tc_err(efx, "Unsupported flower keys %#x\n", dissector->used_keys);
+		NL_SET_ERR_MSG_MOD(extack, "Unsupported flower keys encountered");
+		return -EOPNOTSUPP;
+	}
+
+	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
+		struct flow_match_basic fm;
+
+		flow_rule_match_basic(rule, &fm);
+		if (fm.mask->n_proto) {
+			EFX_TC_ERR_MSG(efx, extack, "Unsupported eth_proto match\n");
+			return -EOPNOTSUPP;
+		}
+		if (fm.mask->ip_proto) {
+			EFX_TC_ERR_MSG(efx, extack, "Unsupported ip_proto match\n");
+			return -EOPNOTSUPP;
+		}
+	}
+
+	return 0;
+}
+
+static int efx_tc_flower_replace(struct efx_nic *efx,
+				 struct net_device *net_dev,
+				 struct flow_cls_offload *tc,
+				 struct efx_rep *efv)
+{
+	struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
+	struct netlink_ext_ack *extack = tc->common.extack;
+	struct efx_tc_flow_rule *rule = NULL, *old;
+	struct efx_tc_action_set *act = NULL;
+	const struct flow_action_entry *fa;
+	struct efx_rep *from_efv, *to_efv;
+	struct efx_tc_match match;
+	s64 rc;
+	int i;
+
+	if (!tc_can_offload_extack(efx->net_dev, extack))
+		return -EOPNOTSUPP;
+	if (WARN_ON(!efx->tc))
+		return -ENETDOWN;
+	if (WARN_ON(!efx->tc->up))
+		return -ENETDOWN;
+
+	from_efv = efx_tc_flower_lookup_efv(efx, net_dev);
+	if (IS_ERR(from_efv)) {
+		/* Might be a tunnel decap rule from an indirect block.
+		 * Support for those not implemented yet.
+		 */
+		return -EOPNOTSUPP;
+	}
+
+	if (efv != from_efv) {
+		/* can't happen */
+		efx_tc_err(efx, "for %s efv is %snull but from_efv is %snull\n",
+			   netdev_name(net_dev), efv ? "non-" : "",
+			   from_efv ? "non-" : "");
+		if (efv)
+			NL_SET_ERR_MSG_MOD(extack, "vfrep filter has PF net_dev (can't happen)");
+		else
+			NL_SET_ERR_MSG_MOD(extack, "PF filter has vfrep net_dev (can't happen)");
+		return -EINVAL;
+	}
+
+	/* Parse match */
+	memset(&match, 0, sizeof(match));
+	rc = efx_tc_flower_external_mport(efx, from_efv);
+	if (rc < 0) {
+		EFX_TC_ERR_MSG(efx, extack, "Failed to identify ingress m-port");
+		return rc;
+	}
+	match.value.ingress_port = rc;
+	match.mask.ingress_port = ~0;
+	rc = efx_tc_flower_parse_match(efx, fr, &match, extack);
+	if (rc)
+		return rc;
+
+	if (tc->common.chain_index) {
+		EFX_TC_ERR_MSG(efx, extack, "No support for nonzero chain_index");
+		return -EOPNOTSUPP;
+	}
+	match.mask.recirc_id = 0xff;
+
+	rc = efx_mae_match_check_caps(efx, &match.mask, extack);
+	if (rc)
+		return rc;
+
+	rule = kzalloc(sizeof(*rule), GFP_USER);
+	if (!rule)
+		return -ENOMEM;
+	INIT_LIST_HEAD(&rule->acts.list);
+	rule->cookie = tc->cookie;
+	old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,
+						&rule->linkage,
+						efx_tc_match_action_ht_params);
+	if (old) {
+		netif_dbg(efx, drv, efx->net_dev,
+			  "Already offloaded rule (cookie %lx)\n", tc->cookie);
+		rc = -EEXIST;
+		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
+		goto release;
+	}
+
+	/* Parse actions */
+	act = kzalloc(sizeof(*act), GFP_USER);
+	if (!act) {
+		rc = -ENOMEM;
+		goto release;
+	}
+
+	flow_action_for_each(i, fa, &fr->action) {
+		struct efx_tc_action_set save;
+
+		if (!act) {
+			/* more actions after a non-pipe action */
+			EFX_TC_ERR_MSG(efx, extack, "Action follows non-pipe action");
+			rc = -EINVAL;
+			goto release;
+		}
+
+		switch (fa->id) {
+		case FLOW_ACTION_DROP:
+			rc = efx_mae_alloc_action_set(efx, act);
+			if (rc) {
+				EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (drop)");
+				goto release;
+			}
+			list_add_tail(&act->list, &rule->acts.list);
+			act = NULL; /* end of the line */
+			break;
+		case FLOW_ACTION_REDIRECT:
+		case FLOW_ACTION_MIRRED:
+			save = *act;
+			to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
+			if (IS_ERR(to_efv)) {
+				EFX_TC_ERR_MSG(efx, extack, "Mirred egress device not on switch");
+				rc = PTR_ERR(to_efv);
+				goto release;
+			}
+			rc = efx_tc_flower_external_mport(efx, to_efv);
+			if (rc < 0) {
+				EFX_TC_ERR_MSG(efx, extack, "Failed to identify egress m-port");
+				goto release;
+			}
+			act->dest_mport = rc;
+			act->deliver = 1;
+			rc = efx_mae_alloc_action_set(efx, act);
+			if (rc) {
+				EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (mirred)");
+				goto release;
+			}
+			list_add_tail(&act->list, &rule->acts.list);
+			act = NULL;
+			if (fa->id == FLOW_ACTION_REDIRECT)
+				break; /* end of the line */
+			/* Mirror, so continue on with saved act */
+			act = kzalloc(sizeof(*act), GFP_USER);
+			if (!act) {
+				rc = -ENOMEM;
+				goto release;
+			}
+			*act = save;
+			break;
+		default:
+			efx_tc_err(efx, "Unhandled action %u\n", fa->id);
+			rc = -EOPNOTSUPP;
+			NL_SET_ERR_MSG_MOD(extack, "Unsupported action");
+			goto release;
+		}
+	}
+
+	if (act) {
+		/* Not shot/redirected, so deliver to default dest */
+		if (from_efv == EFX_EFV_PF)
+			/* Rule applies to traffic from the wire,
+			 * and default dest is thus the PF
+			 */
+			efx_mae_mport_uplink(efx, &act->dest_mport);
+		else
+			/* Representor, so rule applies to traffic from
+			 * representee, and default dest is thus the rep.
+			 * All reps use the same mport for delivery
+			 */
+			efx_mae_mport_mport(efx, efx->tc->reps_mport_id,
+					    &act->dest_mport);
+		act->deliver = 1;
+		rc = efx_mae_alloc_action_set(efx, act);
+		if (rc) {
+			EFX_TC_ERR_MSG(efx, extack, "Failed to write action set to hw (deliver)");
+			goto release;
+		}
+		list_add_tail(&act->list, &rule->acts.list);
+		act = NULL; /* Prevent double-free in error path */
+	}
+
+	netif_dbg(efx, drv, efx->net_dev,
+		  "Successfully parsed filter (cookie %lx)\n",
+		  tc->cookie);
+
+	rule->match = match;
+
+	rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
+	if (rc) {
+		EFX_TC_ERR_MSG(efx, extack, "Failed to write action set list to hw");
+		goto release;
+	}
+	rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
+				 rule->acts.fw_id, &rule->fw_id);
+	if (rc) {
+		EFX_TC_ERR_MSG(efx, extack, "Failed to insert rule in hw");
+		goto release_acts;
+	}
+	return 0;
+
+release_acts:
+	efx_mae_free_action_set_list(efx, &rule->acts);
+release:
+	/* We failed to insert the rule, so free up any entries we created in
+	 * subsidiary tables.
+	 */
+	if (act)
+		efx_tc_free_action_set(efx, act, false);
+	if (rule) {
+		rhashtable_remove_fast(&efx->tc->match_action_ht,
+				       &rule->linkage,
+				       efx_tc_match_action_ht_params);
+		efx_tc_free_action_set_list(efx, &rule->acts, false);
+	}
+	kfree(rule);
+	return rc;
+}
+
+static int efx_tc_flower_destroy(struct efx_nic *efx,
+				 struct net_device *net_dev,
+				 struct flow_cls_offload *tc)
+{
+	struct netlink_ext_ack *extack = tc->common.extack;
+	struct efx_tc_flow_rule *rule;
+
+	rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie,
+				      efx_tc_match_action_ht_params);
+	if (!rule) {
+		/* Only log a message if we're the ingress device.  Otherwise
+		 * it's a foreign filter and we might just not have been
+		 * interested (e.g. we might not have been the egress device
+		 * either).
+		 */
+		if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
+			netif_warn(efx, drv, efx->net_dev,
+				   "Filter %lx not found to remove\n", tc->cookie);
+		NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
+		return -ENOENT;
+	}
+
+	/* Remove it from HW */
+	efx_tc_delete_rule(efx, rule);
+	/* Delete it from SW */
+	rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage,
+			       efx_tc_match_action_ht_params);
+	netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie);
+	kfree(rule);
+	return 0;
+}
+
+int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
+		  struct flow_cls_offload *tc, struct efx_rep *efv)
+{
+	int rc;
+
+	if (!efx->tc)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&efx->tc->mutex);
+	switch (tc->command) {
+	case FLOW_CLS_REPLACE:
+		rc = efx_tc_flower_replace(efx, net_dev, tc, efv);
+		break;
+	case FLOW_CLS_DESTROY:
+		rc = efx_tc_flower_destroy(efx, net_dev, tc);
+		break;
+	default:
+		rc = -EOPNOTSUPP;
+		break;
+	}
+	mutex_unlock(&efx->tc->mutex);
+	return rc;
+}
+
 static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port,
 					 u32 eg_port, struct efx_tc_flow_rule *rule)
 {
@@ -201,13 +577,37 @@ int efx_init_tc(struct efx_nic *efx)
 {
 	int rc;
 
+	rc = efx_mae_get_caps(efx, efx->tc->caps);
+	if (rc)
+		return rc;
+	if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS)
+		/* Firmware supports some match fields the driver doesn't know
+		 * about.  Not fatal, unless any of those fields are required
+		 * (MAE_FIELD_SUPPORTED_MATCH_ALWAYS) but if so we don't know.
+		 */
+		netif_warn(efx, probe, efx->net_dev,
+			   "FW reports additional match fields %u\n",
+			   efx->tc->caps->match_field_count);
+	if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) {
+		netif_err(efx, probe, efx->net_dev,
+			  "Too few action prios supported (have %u, need %u)\n",
+			  efx->tc->caps->action_prios, EFX_TC_PRIO__NUM);
+		return -EIO;
+	}
 	rc = efx_tc_configure_default_rule_pf(efx);
 	if (rc)
 		return rc;
 	rc = efx_tc_configure_default_rule_wire(efx);
 	if (rc)
 		return rc;
-	return efx_tc_configure_rep_mport(efx);
+	rc = efx_tc_configure_rep_mport(efx);
+	if (rc)
+		return rc;
+	efx->tc->up = true;
+	rc = flow_indr_dev_register(efx_tc_indr_setup_cb, efx);
+	if (rc)
+		return rc;
+	return 0;
 }
 
 void efx_fini_tc(struct efx_nic *efx)
@@ -215,20 +615,35 @@ void efx_fini_tc(struct efx_nic *efx)
 	/* We can get called even if efx_init_struct_tc() failed */
 	if (!efx->tc)
 		return;
+	if (efx->tc->up)
+		flow_indr_dev_unregister(efx_tc_indr_setup_cb, efx, efx_tc_block_unbind);
 	efx_tc_deconfigure_rep_mport(efx);
 	efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf);
 	efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire);
+	efx->tc->up = false;
 }
 
 int efx_init_struct_tc(struct efx_nic *efx)
 {
+	int rc;
+
 	if (efx->type->is_vf)
 		return 0;
 
 	efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL);
 	if (!efx->tc)
 		return -ENOMEM;
+	efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL);
+	if (!efx->tc->caps) {
+		rc = -ENOMEM;
+		goto fail_alloc_caps;
+	}
+	INIT_LIST_HEAD(&efx->tc->block_list);
 
+	mutex_init(&efx->tc->mutex);
+	rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params);
+	if (rc < 0)
+		goto fail_match_action_ht;
 	efx->tc->reps_filter_uc = -1;
 	efx->tc->reps_filter_mc = -1;
 	INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list);
@@ -236,6 +651,13 @@ int efx_init_struct_tc(struct efx_nic *efx)
 	INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list);
 	efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
 	return 0;
+fail_match_action_ht:
+	mutex_destroy(&efx->tc->mutex);
+	kfree(efx->tc->caps);
+fail_alloc_caps:
+	kfree(efx->tc);
+	efx->tc = NULL;
+	return rc;
 }
 
 void efx_fini_struct_tc(struct efx_nic *efx)
@@ -243,10 +665,16 @@ void efx_fini_struct_tc(struct efx_nic *efx)
 	if (!efx->tc)
 		return;
 
+	mutex_lock(&efx->tc->mutex);
 	EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id !=
 			     MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
 	EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id !=
 			     MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
+	rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free,
+				    efx);
+	mutex_unlock(&efx->tc->mutex);
+	mutex_destroy(&efx->tc->mutex);
+	kfree(efx->tc->caps);
 	kfree(efx->tc);
 	efx->tc = NULL;
 }
diff --git a/drivers/net/ethernet/sfc/tc.h b/drivers/net/ethernet/sfc/tc.h
index 309123c6b386..196fd74ed973 100644
--- a/drivers/net/ethernet/sfc/tc.h
+++ b/drivers/net/ethernet/sfc/tc.h
@@ -11,8 +11,28 @@
 
 #ifndef EFX_TC_H
 #define EFX_TC_H
+#include <net/flow_offload.h>
+#include <linux/rhashtable.h>
 #include "net_driver.h"
 
+/* Error reporting: convenience macros.  For indicating why a given filter
+ * insertion is not supported; errors in internal operation or in the
+ * hardware should be netif_err()s instead.
+ */
+/* Used when error message is constant. */
+#define EFX_TC_ERR_MSG(efx, extack, message)	do {			\
+	NL_SET_ERR_MSG_MOD(extack, message);				\
+	if (efx->log_tc_errs)						\
+		netif_info(efx, drv, efx->net_dev, "%s\n", message);	\
+} while (0)
+/* Used when error message is not constant; caller should also supply a
+ * constant extack message with NL_SET_ERR_MSG_MOD().
+ */
+#define efx_tc_err(efx, fmt, args...)	do {		\
+if (efx->log_tc_errs)					\
+	netif_info(efx, drv, efx->net_dev, fmt, ##args);\
+} while (0)
+
 struct efx_tc_action_set {
 	u16 deliver:1;
 	u32 dest_mport;
@@ -23,6 +43,7 @@ struct efx_tc_action_set {
 struct efx_tc_match_fields {
 	/* L1 */
 	u32 ingress_port;
+	u8 recirc_id;
 };
 
 struct efx_tc_match {
@@ -36,12 +57,15 @@ struct efx_tc_action_set_list {
 };
 
 struct efx_tc_flow_rule {
+	unsigned long cookie;
+	struct rhash_head linkage;
 	struct efx_tc_match match;
 	struct efx_tc_action_set_list acts;
 	u32 fw_id;
 };
 
 enum efx_tc_rule_prios {
+	EFX_TC_PRIO_TC, /* Rule inserted by TC */
 	EFX_TC_PRIO_DFLT, /* Default switch rule; one of efx_tc_default_rules */
 	EFX_TC_PRIO__NUM
 };
@@ -49,6 +73,10 @@ enum efx_tc_rule_prios {
 /**
  * struct efx_tc_state - control plane data for TC offload
  *
+ * @caps: MAE capabilities reported by MCDI
+ * @block_list: List of &struct efx_tc_block_binding
+ * @mutex: Used to serialise operations on TC hashtables
+ * @match_action_ht: Hashtable of TC match-action rules
  * @reps_mport_id: MAE port allocated for representor RX
  * @reps_filter_uc: VNIC filter for representor unicast RX (promisc)
  * @reps_filter_mc: VNIC filter for representor multicast RX (allmulti)
@@ -57,14 +85,20 @@ enum efx_tc_rule_prios {
  *	%EFX_TC_PRIO_DFLT.  Named by *ingress* port
  * @dflt.pf: rule for traffic ingressing from PF (egresses to wire)
  * @dflt.wire: rule for traffic ingressing from wire (egresses to PF)
+ * @up: have TC datastructures been set up?
  */
 struct efx_tc_state {
+	struct mae_caps *caps;
+	struct list_head block_list;
+	struct mutex mutex;
+	struct rhashtable match_action_ht;
 	u32 reps_mport_id, reps_mport_vport_id;
 	s32 reps_filter_uc, reps_filter_mc;
 	struct {
 		struct efx_tc_flow_rule pf;
 		struct efx_tc_flow_rule wire;
 	} dflt;
+	bool up;
 };
 
 struct efx_rep;
@@ -72,6 +106,8 @@ struct efx_rep;
 int efx_tc_configure_default_rule_rep(struct efx_rep *efv);
 void efx_tc_deconfigure_default_rule(struct efx_nic *efx,
 				     struct efx_tc_flow_rule *rule);
+int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
+		  struct flow_cls_offload *tc, struct efx_rep *efv);
 
 int efx_tc_insert_rep_filters(struct efx_nic *efx);
 void efx_tc_remove_rep_filters(struct efx_nic *efx);
diff --git a/drivers/net/ethernet/sfc/tc_bindings.c b/drivers/net/ethernet/sfc/tc_bindings.c
new file mode 100644
index 000000000000..c18d64519c2d
--- /dev/null
+++ b/drivers/net/ethernet/sfc/tc_bindings.c
@@ -0,0 +1,228 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/****************************************************************************
+ * Driver for Solarflare network controllers and boards
+ * Copyright 2022 Xilinx Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#include "tc_bindings.h"
+#include "tc.h"
+
+struct efx_tc_block_binding {
+	struct list_head list;
+	struct efx_nic *efx;
+	struct efx_rep *efv;
+	struct net_device *otherdev; /* may actually be us */
+	struct flow_block *block;
+};
+
+static struct efx_tc_block_binding *efx_tc_find_binding(struct efx_nic *efx,
+							struct net_device *otherdev)
+{
+	struct efx_tc_block_binding *binding;
+
+	ASSERT_RTNL();
+	list_for_each_entry(binding, &efx->tc->block_list, list)
+		if (binding->otherdev == otherdev)
+			return binding;
+	return NULL;
+}
+
+static int efx_tc_block_cb(enum tc_setup_type type, void *type_data,
+			   void *cb_priv)
+{
+	struct efx_tc_block_binding *binding = cb_priv;
+	struct flow_cls_offload *tcf = type_data;
+
+	switch (type) {
+	case TC_SETUP_CLSFLOWER:
+		return efx_tc_flower(binding->efx, binding->otherdev,
+				     tcf, binding->efv);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+void efx_tc_block_unbind(void *cb_priv)
+{
+	struct efx_tc_block_binding *binding = cb_priv;
+
+	list_del(&binding->list);
+	kfree(binding);
+}
+
+static struct efx_tc_block_binding *efx_tc_create_binding(
+			struct efx_nic *efx, struct efx_rep *efv,
+			struct net_device *otherdev, struct flow_block *block)
+{
+	struct efx_tc_block_binding *binding = kmalloc(sizeof(*binding), GFP_KERNEL);
+
+	if (!binding)
+		return ERR_PTR(-ENOMEM);
+	binding->efx = efx;
+	binding->efv = efv;
+	binding->otherdev = otherdev;
+	binding->block = block;
+	list_add(&binding->list, &efx->tc->block_list);
+	return binding;
+}
+
+int efx_tc_setup_block(struct net_device *net_dev, struct efx_nic *efx,
+		       struct flow_block_offload *tcb, struct efx_rep *efv)
+{
+	struct efx_tc_block_binding *binding;
+	struct flow_block_cb *block_cb;
+	int rc;
+
+	if (tcb->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
+		return -EOPNOTSUPP;
+
+	if (WARN_ON(!efx->tc))
+		return -ENETDOWN;
+
+	switch (tcb->command) {
+	case FLOW_BLOCK_BIND:
+		binding = efx_tc_create_binding(efx, efv, net_dev, tcb->block);
+		if (IS_ERR(binding))
+			return PTR_ERR(binding);
+		block_cb = flow_block_cb_alloc(efx_tc_block_cb, binding,
+					       binding, efx_tc_block_unbind);
+		rc = PTR_ERR_OR_ZERO(block_cb);
+		netif_dbg(efx, drv, efx->net_dev,
+			  "bind %sdirect block for device %s, rc %d\n",
+			  net_dev == efx->net_dev ? "" :
+			  efv ? "semi" : "in",
+			  net_dev ? net_dev->name : NULL, rc);
+		if (rc) {
+			list_del(&binding->list);
+			kfree(binding);
+		} else {
+			flow_block_cb_add(block_cb, tcb);
+		}
+		return rc;
+	case FLOW_BLOCK_UNBIND:
+		binding = efx_tc_find_binding(efx, net_dev);
+		if (binding) {
+			block_cb = flow_block_cb_lookup(tcb->block,
+							efx_tc_block_cb,
+							binding);
+			if (block_cb) {
+				flow_block_cb_remove(block_cb, tcb);
+				netif_dbg(efx, drv, efx->net_dev,
+					  "unbound %sdirect block for device %s\n",
+					  net_dev == efx->net_dev ? "" :
+					  binding->efv ? "semi" : "in",
+					  net_dev ? net_dev->name : NULL);
+				return 0;
+			}
+		}
+		/* If we're in driver teardown, then we expect to have
+		 * already unbound all our blocks (we did it early while
+		 * we still had MCDI to remove the filters), so getting
+		 * unbind callbacks now isn't a problem.
+		 */
+		netif_cond_dbg(efx, drv, efx->net_dev,
+			       !efx->tc->up, warn,
+			       "%sdirect block unbind for device %s, was never bound\n",
+			       net_dev == efx->net_dev ? "" : "in",
+			       net_dev ? net_dev->name : NULL);
+		return -ENOENT;
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+int efx_tc_indr_setup_cb(struct net_device *net_dev, struct Qdisc *sch,
+			 void *cb_priv, enum tc_setup_type type,
+			 void *type_data, void *data,
+			 void (*cleanup)(struct flow_block_cb *block_cb))
+{
+	struct flow_block_offload *tcb = type_data;
+	struct efx_tc_block_binding *binding;
+	struct flow_block_cb *block_cb;
+	struct efx_nic *efx = cb_priv;
+	bool is_ovs_int_port;
+	int rc;
+
+	if (!net_dev)
+		return -EOPNOTSUPP;
+
+	if (tcb->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS &&
+	    tcb->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS)
+		return -EOPNOTSUPP;
+
+	is_ovs_int_port = netif_is_ovs_master(net_dev);
+	if (tcb->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS &&
+	    !is_ovs_int_port)
+		return -EOPNOTSUPP;
+
+	if (is_ovs_int_port)
+		return -EOPNOTSUPP;
+
+	switch (type) {
+	case TC_SETUP_BLOCK:
+		switch (tcb->command) {
+		case FLOW_BLOCK_BIND:
+			binding = efx_tc_create_binding(efx, NULL, net_dev, tcb->block);
+			if (IS_ERR(binding))
+				return PTR_ERR(binding);
+			block_cb = flow_indr_block_cb_alloc(efx_tc_block_cb, binding,
+							    binding, efx_tc_block_unbind,
+							    tcb, net_dev, sch, data, binding,
+							    cleanup);
+			rc = PTR_ERR_OR_ZERO(block_cb);
+			netif_dbg(efx, drv, efx->net_dev,
+				  "bind indr block for device %s, rc %d\n",
+				  net_dev ? net_dev->name : NULL, rc);
+			if (rc) {
+				list_del(&binding->list);
+				kfree(binding);
+			} else {
+				flow_block_cb_add(block_cb, tcb);
+			}
+			return rc;
+		case FLOW_BLOCK_UNBIND:
+			binding = efx_tc_find_binding(efx, net_dev);
+			if (!binding)
+				return -ENOENT;
+			block_cb = flow_block_cb_lookup(tcb->block,
+							efx_tc_block_cb,
+							binding);
+			if (!block_cb)
+				return -ENOENT;
+			flow_indr_block_cb_remove(block_cb, tcb);
+			netif_dbg(efx, drv, efx->net_dev,
+				  "unbind indr block for device %s\n",
+				  net_dev ? net_dev->name : NULL);
+			return 0;
+		default:
+			return -EOPNOTSUPP;
+		}
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
+/* .ndo_setup_tc implementation
+ * Entry point for flower block and filter management.
+ */
+int efx_tc_setup(struct net_device *net_dev, enum tc_setup_type type,
+		 void *type_data)
+{
+	struct efx_nic *efx = efx_netdev_priv(net_dev);
+
+	if (efx->type->is_vf)
+		return -EOPNOTSUPP;
+	if (!efx->tc)
+		return -EOPNOTSUPP;
+
+	if (type == TC_SETUP_CLSFLOWER)
+		return efx_tc_flower(efx, net_dev, type_data, NULL);
+	if (type == TC_SETUP_BLOCK)
+		return efx_tc_setup_block(net_dev, efx, type_data, NULL);
+
+	return -EOPNOTSUPP;
+}
diff --git a/drivers/net/ethernet/sfc/tc_bindings.h b/drivers/net/ethernet/sfc/tc_bindings.h
new file mode 100644
index 000000000000..c210bb09150e
--- /dev/null
+++ b/drivers/net/ethernet/sfc/tc_bindings.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/****************************************************************************
+ * Driver for Solarflare network controllers and boards
+ * Copyright 2022 Xilinx Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, incorporated herein by reference.
+ */
+
+#ifndef EFX_TC_BINDINGS_H
+#define EFX_TC_BINDINGS_H
+#include "net_driver.h"
+
+#include <net/sch_generic.h>
+
+struct efx_rep;
+
+void efx_tc_block_unbind(void *cb_priv);
+int efx_tc_setup_block(struct net_device *net_dev, struct efx_nic *efx,
+		       struct flow_block_offload *tcb, struct efx_rep *efv);
+int efx_tc_setup(struct net_device *net_dev, enum tc_setup_type type,
+		 void *type_data);
+
+int efx_tc_indr_setup_cb(struct net_device *net_dev, struct Qdisc *sch,
+			 void *cb_priv, enum tc_setup_type type,
+			 void *type_data, void *data,
+			 void (*cleanup)(struct flow_block_cb *block_cb));
+#endif /* EFX_TC_BINDINGS_H */
diff --git a/drivers/net/ethernet/sgi/ioc3-eth.c b/drivers/net/ethernet/sgi/ioc3-eth.c
index e2d009866a7b..8fc3f5272fa7 100644
--- a/drivers/net/ethernet/sgi/ioc3-eth.c
+++ b/drivers/net/ethernet/sgi/ioc3-eth.c
@@ -1158,9 +1158,9 @@ static inline unsigned int ioc3_hash(const unsigned char *addr)
 static void ioc3_get_drvinfo(struct net_device *dev,
 			     struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, IOC3_NAME, sizeof(info->driver));
-	strlcpy(info->version, IOC3_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(to_pci_dev(dev->dev.parent)),
+	strscpy(info->driver, IOC3_NAME, sizeof(info->driver));
+	strscpy(info->version, IOC3_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(to_pci_dev(dev->dev.parent)),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/sis/sis190.c b/drivers/net/ethernet/sis/sis190.c
index 216bb2d34d7c..dda4e488c77a 100644
--- a/drivers/net/ethernet/sis/sis190.c
+++ b/drivers/net/ethernet/sis/sis190.c
@@ -1769,9 +1769,9 @@ static void sis190_get_drvinfo(struct net_device *dev,
 {
 	struct sis190_private *tp = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(tp->pci_dev),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(tp->pci_dev),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/sis/sis900.c b/drivers/net/ethernet/sis/sis900.c
index 23a336c5096e..cb7fec226cab 100644
--- a/drivers/net/ethernet/sis/sis900.c
+++ b/drivers/net/ethernet/sis/sis900.c
@@ -2027,9 +2027,9 @@ static void sis900_get_drvinfo(struct net_device *net_dev,
 {
 	struct sis900_private *sis_priv = netdev_priv(net_dev);
 
-	strlcpy(info->driver, SIS900_MODULE_NAME, sizeof(info->driver));
-	strlcpy(info->version, SIS900_DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(sis_priv->pci_dev),
+	strscpy(info->driver, SIS900_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->version, SIS900_DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(sis_priv->pci_dev),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/smsc/epic100.c b/drivers/net/ethernet/smsc/epic100.c
index 0329caf63279..013e90d69182 100644
--- a/drivers/net/ethernet/smsc/epic100.c
+++ b/drivers/net/ethernet/smsc/epic100.c
@@ -482,7 +482,7 @@ static int epic_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	dev->netdev_ops = &epic_netdev_ops;
 	dev->ethtool_ops = &netdev_ethtool_ops;
 	dev->watchdog_timeo = TX_TIMEOUT;
-	netif_napi_add(dev, &ep->napi, epic_poll, 64);
+	netif_napi_add(dev, &ep->napi, epic_poll);
 
 	ret = register_netdev(dev);
 	if (ret < 0)
@@ -1392,9 +1392,9 @@ static void netdev_get_drvinfo (struct net_device *dev, struct ethtool_drvinfo *
 {
 	struct epic_private *np = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(np->pci_dev), sizeof(info->bus_info));
 }
 
 static int netdev_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/smsc/smc911x.c b/drivers/net/ethernet/smsc/smc911x.c
index 24d66af797d4..52ecfb461c41 100644
--- a/drivers/net/ethernet/smsc/smc911x.c
+++ b/drivers/net/ethernet/smsc/smc911x.c
@@ -1509,9 +1509,9 @@ smc911x_ethtool_set_link_ksettings(struct net_device *dev,
 static void
 smc911x_ethtool_getdrvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, CARDNAME, sizeof(info->driver));
-	strlcpy(info->version, version, sizeof(info->version));
-	strlcpy(info->bus_info, dev_name(dev->dev.parent),
+	strscpy(info->driver, CARDNAME, sizeof(info->driver));
+	strscpy(info->version, version, sizeof(info->version));
+	strscpy(info->bus_info, dev_name(dev->dev.parent),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/smsc/smc91c92_cs.c b/drivers/net/ethernet/smsc/smc91c92_cs.c
index 37c822e27207..29bb19f42de9 100644
--- a/drivers/net/ethernet/smsc/smc91c92_cs.c
+++ b/drivers/net/ethernet/smsc/smc91c92_cs.c
@@ -1909,8 +1909,8 @@ static int check_if_running(struct net_device *dev)
 
 static void smc_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 }
 
 static int smc_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/smsc/smc91x.c b/drivers/net/ethernet/smsc/smc91x.c
index a31c159e96ea..35e99bf0c401 100644
--- a/drivers/net/ethernet/smsc/smc91x.c
+++ b/drivers/net/ethernet/smsc/smc91x.c
@@ -1588,9 +1588,9 @@ smc_ethtool_set_link_ksettings(struct net_device *dev,
 static void
 smc_ethtool_getdrvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, CARDNAME, sizeof(info->driver));
-	strlcpy(info->version, version, sizeof(info->version));
-	strlcpy(info->bus_info, dev_name(dev->dev.parent),
+	strscpy(info->driver, CARDNAME, sizeof(info->driver));
+	strscpy(info->version, version, sizeof(info->version));
+	strscpy(info->bus_info, dev_name(dev->dev.parent),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/smsc/smsc911x.c b/drivers/net/ethernet/smsc/smsc911x.c
index 3829c2805b16..a2e511912e6a 100644
--- a/drivers/net/ethernet/smsc/smsc911x.c
+++ b/drivers/net/ethernet/smsc/smsc911x.c
@@ -1955,9 +1955,9 @@ static int smsc911x_set_mac_address(struct net_device *dev, void *p)
 static void smsc911x_ethtool_getdrvinfo(struct net_device *dev,
 					struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, SMSC_CHIPNAME, sizeof(info->driver));
-	strlcpy(info->version, SMSC_DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, dev_name(dev->dev.parent),
+	strscpy(info->driver, SMSC_CHIPNAME, sizeof(info->driver));
+	strscpy(info->version, SMSC_DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, dev_name(dev->dev.parent),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/smsc/smsc9420.c b/drivers/net/ethernet/smsc/smsc9420.c
index 0c68c7f8056d..71fbb358bb7d 100644
--- a/drivers/net/ethernet/smsc/smsc9420.c
+++ b/drivers/net/ethernet/smsc/smsc9420.c
@@ -215,10 +215,10 @@ static void smsc9420_ethtool_get_drvinfo(struct net_device *netdev,
 {
 	struct smsc9420_pdata *pd = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->bus_info, pci_name(pd->pdev),
+	strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->bus_info, pci_name(pd->pdev),
 		sizeof(drvinfo->bus_info));
-	strlcpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version));
+	strscpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version));
 }
 
 static u32 smsc9420_ethtool_get_msglevel(struct net_device *netdev)
@@ -1585,7 +1585,7 @@ smsc9420_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	dev->netdev_ops = &smsc9420_netdev_ops;
 	dev->ethtool_ops = &smsc9420_ethtool_ops;
 
-	netif_napi_add(dev, &pd->napi, smsc9420_rx_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &pd->napi, smsc9420_rx_poll);
 
 	result = register_netdev(dev);
 	if (result) {
diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c
index b0c5a44785fa..2240f6d0b89b 100644
--- a/drivers/net/ethernet/socionext/netsec.c
+++ b/drivers/net/ethernet/socionext/netsec.c
@@ -526,8 +526,8 @@ static int netsec_phy_read(struct mii_bus *bus, int phy_addr, int reg_addr)
 static void netsec_et_get_drvinfo(struct net_device *net_device,
 				  struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, "netsec", sizeof(info->driver));
-	strlcpy(info->bus_info, dev_name(net_device->dev.parent),
+	strscpy(info->driver, "netsec", sizeof(info->driver));
+	strscpy(info->bus_info, dev_name(net_device->dev.parent),
 		sizeof(info->bus_info));
 }
 
@@ -2093,7 +2093,7 @@ static int netsec_probe(struct platform_device *pdev)
 	dev_info(&pdev->dev, "hardware revision %d.%d\n",
 		 hw_ver >> 16, hw_ver & 0xffff);
 
-	netif_napi_add(ndev, &priv->napi, netsec_napi_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &priv->napi, netsec_napi_poll);
 
 	ndev->netdev_ops = &netsec_netdev_ops;
 	ndev->ethtool_ops = &netsec_ethtool_ops;
diff --git a/drivers/net/ethernet/socionext/sni_ave.c b/drivers/net/ethernet/socionext/sni_ave.c
index f0c8de2c6075..1fa09b49ba7f 100644
--- a/drivers/net/ethernet/socionext/sni_ave.c
+++ b/drivers/net/ethernet/socionext/sni_ave.c
@@ -395,8 +395,8 @@ static void ave_ethtool_get_drvinfo(struct net_device *ndev,
 {
 	struct device *dev = ndev->dev.parent;
 
-	strlcpy(info->driver, dev->driver->name, sizeof(info->driver));
-	strlcpy(info->bus_info, dev_name(dev), sizeof(info->bus_info));
+	strscpy(info->driver, dev->driver->name, sizeof(info->driver));
+	strscpy(info->bus_info, dev_name(dev), sizeof(info->bus_info));
 	ave_hw_read_version(ndev, info->fw_version, sizeof(info->fw_version));
 }
 
@@ -1687,8 +1687,7 @@ static int ave_probe(struct platform_device *pdev)
 		 pdev->name, pdev->id);
 
 	/* Register as a NAPI supported driver */
-	netif_napi_add(ndev, &priv->napi_rx, ave_napi_poll_rx,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &priv->napi_rx, ave_napi_poll_rx);
 	netif_napi_add_tx(ndev, &priv->napi_tx, ave_napi_poll_tx);
 
 	platform_set_drvdata(pdev, ndev);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c
index 358fc26f8d1f..80efdeeb0b59 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c
@@ -445,9 +445,7 @@ static int dwc_eth_dwmac_probe(struct platform_device *pdev)
 
 	ret = data->probe(pdev, plat_dat, &stmmac_res);
 	if (ret < 0) {
-		if (ret != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "failed to probe subdriver: %d\n",
-				ret);
+		dev_err_probe(&pdev->dev, ret, "failed to probe subdriver\n");
 
 		goto remove_config;
 	}
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
index 9af25be42401..0a2afc1a3124 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
@@ -610,7 +610,6 @@ static int intel_mgbe_common_data(struct pci_dev *pdev,
 	plat->int_snapshot_num = AUX_SNAPSHOT1;
 	plat->ext_snapshot_num = AUX_SNAPSHOT0;
 
-	plat->has_crossts = true;
 	plat->crosststamp = intel_crosststamp;
 	plat->int_snapshot_en = 0;
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
index c469abc91fa1..f7269d79a385 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
@@ -32,6 +32,8 @@ struct rk_gmac_ops {
 	void (*set_to_rmii)(struct rk_priv_data *bsp_priv);
 	void (*set_rgmii_speed)(struct rk_priv_data *bsp_priv, int speed);
 	void (*set_rmii_speed)(struct rk_priv_data *bsp_priv, int speed);
+	void (*set_clock_selection)(struct rk_priv_data *bsp_priv, bool input,
+				    bool enable);
 	void (*integrated_phy_powerup)(struct rk_priv_data *bsp_priv);
 	bool regs_valid;
 	u32 regs[];
@@ -66,6 +68,7 @@ struct rk_priv_data {
 	int rx_delay;
 
 	struct regmap *grf;
+	struct regmap *php_grf;
 };
 
 #define HIWORD_UPDATE(val, mask, shift) \
@@ -1101,6 +1104,147 @@ static const struct rk_gmac_ops rk3568_ops = {
 	},
 };
 
+/* sys_grf */
+#define RK3588_GRF_GMAC_CON7			0X031c
+#define RK3588_GRF_GMAC_CON8			0X0320
+#define RK3588_GRF_GMAC_CON9			0X0324
+
+#define RK3588_GMAC_RXCLK_DLY_ENABLE(id)	GRF_BIT(2 * (id) + 3)
+#define RK3588_GMAC_RXCLK_DLY_DISABLE(id)	GRF_CLR_BIT(2 * (id) + 3)
+#define RK3588_GMAC_TXCLK_DLY_ENABLE(id)	GRF_BIT(2 * (id) + 2)
+#define RK3588_GMAC_TXCLK_DLY_DISABLE(id)	GRF_CLR_BIT(2 * (id) + 2)
+
+#define RK3588_GMAC_CLK_RX_DL_CFG(val)		HIWORD_UPDATE(val, 0xFF, 8)
+#define RK3588_GMAC_CLK_TX_DL_CFG(val)		HIWORD_UPDATE(val, 0xFF, 0)
+
+/* php_grf */
+#define RK3588_GRF_GMAC_CON0			0X0008
+#define RK3588_GRF_CLK_CON1			0X0070
+
+#define RK3588_GMAC_PHY_INTF_SEL_RGMII(id)	\
+	(GRF_BIT(3 + (id) * 6) | GRF_CLR_BIT(4 + (id) * 6) | GRF_CLR_BIT(5 + (id) * 6))
+#define RK3588_GMAC_PHY_INTF_SEL_RMII(id)	\
+	(GRF_CLR_BIT(3 + (id) * 6) | GRF_CLR_BIT(4 + (id) * 6) | GRF_BIT(5 + (id) * 6))
+
+#define RK3588_GMAC_CLK_RMII_MODE(id)		GRF_BIT(5 * (id))
+#define RK3588_GMAC_CLK_RGMII_MODE(id)		GRF_CLR_BIT(5 * (id))
+
+#define RK3588_GMAC_CLK_SELET_CRU(id)		GRF_BIT(5 * (id) + 4)
+#define RK3588_GMAC_CLK_SELET_IO(id)		GRF_CLR_BIT(5 * (id) + 4)
+
+#define RK3588_GMA_CLK_RMII_DIV2(id)		GRF_BIT(5 * (id) + 2)
+#define RK3588_GMA_CLK_RMII_DIV20(id)		GRF_CLR_BIT(5 * (id) + 2)
+
+#define RK3588_GMAC_CLK_RGMII_DIV1(id)		\
+			(GRF_CLR_BIT(5 * (id) + 2) | GRF_CLR_BIT(5 * (id) + 3))
+#define RK3588_GMAC_CLK_RGMII_DIV5(id)		\
+			(GRF_BIT(5 * (id) + 2) | GRF_BIT(5 * (id) + 3))
+#define RK3588_GMAC_CLK_RGMII_DIV50(id)		\
+			(GRF_CLR_BIT(5 * (id) + 2) | GRF_BIT(5 * (id) + 3))
+
+#define RK3588_GMAC_CLK_RMII_GATE(id)		GRF_BIT(5 * (id) + 1)
+#define RK3588_GMAC_CLK_RMII_NOGATE(id)		GRF_CLR_BIT(5 * (id) + 1)
+
+static void rk3588_set_to_rgmii(struct rk_priv_data *bsp_priv,
+				int tx_delay, int rx_delay)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+	u32 offset_con, id = bsp_priv->id;
+
+	if (IS_ERR(bsp_priv->grf) || IS_ERR(bsp_priv->php_grf)) {
+		dev_err(dev, "Missing rockchip,grf or rockchip,php_grf property\n");
+		return;
+	}
+
+	offset_con = bsp_priv->id == 1 ? RK3588_GRF_GMAC_CON9 :
+					 RK3588_GRF_GMAC_CON8;
+
+	regmap_write(bsp_priv->php_grf, RK3588_GRF_GMAC_CON0,
+		     RK3588_GMAC_PHY_INTF_SEL_RGMII(id));
+
+	regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1,
+		     RK3588_GMAC_CLK_RGMII_MODE(id));
+
+	regmap_write(bsp_priv->grf, RK3588_GRF_GMAC_CON7,
+		     RK3588_GMAC_RXCLK_DLY_ENABLE(id) |
+		     RK3588_GMAC_TXCLK_DLY_ENABLE(id));
+
+	regmap_write(bsp_priv->grf, offset_con,
+		     RK3588_GMAC_CLK_RX_DL_CFG(rx_delay) |
+		     RK3588_GMAC_CLK_TX_DL_CFG(tx_delay));
+}
+
+static void rk3588_set_to_rmii(struct rk_priv_data *bsp_priv)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->php_grf)) {
+		dev_err(dev, "%s: Missing rockchip,php_grf property\n", __func__);
+		return;
+	}
+
+	regmap_write(bsp_priv->php_grf, RK3588_GRF_GMAC_CON0,
+		     RK3588_GMAC_PHY_INTF_SEL_RMII(bsp_priv->id));
+
+	regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1,
+		     RK3588_GMAC_CLK_RMII_MODE(bsp_priv->id));
+}
+
+static void rk3588_set_gmac_speed(struct rk_priv_data *bsp_priv, int speed)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+	unsigned int val = 0, id = bsp_priv->id;
+
+	switch (speed) {
+	case 10:
+		if (bsp_priv->phy_iface == PHY_INTERFACE_MODE_RMII)
+			val = RK3588_GMA_CLK_RMII_DIV20(id);
+		else
+			val = RK3588_GMAC_CLK_RGMII_DIV50(id);
+		break;
+	case 100:
+		if (bsp_priv->phy_iface == PHY_INTERFACE_MODE_RMII)
+			val = RK3588_GMA_CLK_RMII_DIV2(id);
+		else
+			val = RK3588_GMAC_CLK_RGMII_DIV5(id);
+		break;
+	case 1000:
+		if (bsp_priv->phy_iface != PHY_INTERFACE_MODE_RMII)
+			val = RK3588_GMAC_CLK_RGMII_DIV1(id);
+		else
+			goto err;
+		break;
+	default:
+		goto err;
+	}
+
+	regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1, val);
+
+	return;
+err:
+	dev_err(dev, "unknown speed value for GMAC speed=%d", speed);
+}
+
+static void rk3588_set_clock_selection(struct rk_priv_data *bsp_priv, bool input,
+				       bool enable)
+{
+	unsigned int val = input ? RK3588_GMAC_CLK_SELET_IO(bsp_priv->id) :
+				   RK3588_GMAC_CLK_SELET_CRU(bsp_priv->id);
+
+	val |= enable ? RK3588_GMAC_CLK_RMII_NOGATE(bsp_priv->id) :
+			RK3588_GMAC_CLK_RMII_GATE(bsp_priv->id);
+
+	regmap_write(bsp_priv->php_grf, RK3588_GRF_CLK_CON1, val);
+}
+
+static const struct rk_gmac_ops rk3588_ops = {
+	.set_to_rgmii = rk3588_set_to_rgmii,
+	.set_to_rmii = rk3588_set_to_rmii,
+	.set_rgmii_speed = rk3588_set_gmac_speed,
+	.set_rmii_speed = rk3588_set_gmac_speed,
+	.set_clock_selection = rk3588_set_clock_selection,
+};
+
 #define RV1108_GRF_GMAC_CON0		0X0900
 
 /* RV1108_GRF_GMAC_CON0 */
@@ -1153,6 +1297,130 @@ static const struct rk_gmac_ops rv1108_ops = {
 	.set_rmii_speed = rv1108_set_rmii_speed,
 };
 
+#define RV1126_GRF_GMAC_CON0		0X0070
+#define RV1126_GRF_GMAC_CON1		0X0074
+#define RV1126_GRF_GMAC_CON2		0X0078
+
+/* RV1126_GRF_GMAC_CON0 */
+#define RV1126_GMAC_PHY_INTF_SEL_RGMII	\
+		(GRF_BIT(4) | GRF_CLR_BIT(5) | GRF_CLR_BIT(6))
+#define RV1126_GMAC_PHY_INTF_SEL_RMII	\
+		(GRF_CLR_BIT(4) | GRF_CLR_BIT(5) | GRF_BIT(6))
+#define RV1126_GMAC_FLOW_CTRL			GRF_BIT(7)
+#define RV1126_GMAC_FLOW_CTRL_CLR		GRF_CLR_BIT(7)
+#define RV1126_GMAC_M0_RXCLK_DLY_ENABLE		GRF_BIT(1)
+#define RV1126_GMAC_M0_RXCLK_DLY_DISABLE	GRF_CLR_BIT(1)
+#define RV1126_GMAC_M0_TXCLK_DLY_ENABLE		GRF_BIT(0)
+#define RV1126_GMAC_M0_TXCLK_DLY_DISABLE	GRF_CLR_BIT(0)
+#define RV1126_GMAC_M1_RXCLK_DLY_ENABLE		GRF_BIT(3)
+#define RV1126_GMAC_M1_RXCLK_DLY_DISABLE	GRF_CLR_BIT(3)
+#define RV1126_GMAC_M1_TXCLK_DLY_ENABLE		GRF_BIT(2)
+#define RV1126_GMAC_M1_TXCLK_DLY_DISABLE	GRF_CLR_BIT(2)
+
+/* RV1126_GRF_GMAC_CON1 */
+#define RV1126_GMAC_M0_CLK_RX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 8)
+#define RV1126_GMAC_M0_CLK_TX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 0)
+/* RV1126_GRF_GMAC_CON2 */
+#define RV1126_GMAC_M1_CLK_RX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 8)
+#define RV1126_GMAC_M1_CLK_TX_DL_CFG(val)	HIWORD_UPDATE(val, 0x7F, 0)
+
+static void rv1126_set_to_rgmii(struct rk_priv_data *bsp_priv,
+				int tx_delay, int rx_delay)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "Missing rockchip,grf property\n");
+		return;
+	}
+
+	regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON0,
+		     RV1126_GMAC_PHY_INTF_SEL_RGMII |
+		     RV1126_GMAC_M0_RXCLK_DLY_ENABLE |
+		     RV1126_GMAC_M0_TXCLK_DLY_ENABLE |
+		     RV1126_GMAC_M1_RXCLK_DLY_ENABLE |
+		     RV1126_GMAC_M1_TXCLK_DLY_ENABLE);
+
+	regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON1,
+		     RV1126_GMAC_M0_CLK_RX_DL_CFG(rx_delay) |
+		     RV1126_GMAC_M0_CLK_TX_DL_CFG(tx_delay));
+
+	regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON2,
+		     RV1126_GMAC_M1_CLK_RX_DL_CFG(rx_delay) |
+		     RV1126_GMAC_M1_CLK_TX_DL_CFG(tx_delay));
+}
+
+static void rv1126_set_to_rmii(struct rk_priv_data *bsp_priv)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+
+	if (IS_ERR(bsp_priv->grf)) {
+		dev_err(dev, "%s: Missing rockchip,grf property\n", __func__);
+		return;
+	}
+
+	regmap_write(bsp_priv->grf, RV1126_GRF_GMAC_CON0,
+		     RV1126_GMAC_PHY_INTF_SEL_RMII);
+}
+
+static void rv1126_set_rgmii_speed(struct rk_priv_data *bsp_priv, int speed)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+	unsigned long rate;
+	int ret;
+
+	switch (speed) {
+	case 10:
+		rate = 2500000;
+		break;
+	case 100:
+		rate = 25000000;
+		break;
+	case 1000:
+		rate = 125000000;
+		break;
+	default:
+		dev_err(dev, "unknown speed value for RGMII speed=%d", speed);
+		return;
+	}
+
+	ret = clk_set_rate(bsp_priv->clk_mac_speed, rate);
+	if (ret)
+		dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n",
+			__func__, rate, ret);
+}
+
+static void rv1126_set_rmii_speed(struct rk_priv_data *bsp_priv, int speed)
+{
+	struct device *dev = &bsp_priv->pdev->dev;
+	unsigned long rate;
+	int ret;
+
+	switch (speed) {
+	case 10:
+		rate = 2500000;
+		break;
+	case 100:
+		rate = 25000000;
+		break;
+	default:
+		dev_err(dev, "unknown speed value for RGMII speed=%d", speed);
+		return;
+	}
+
+	ret = clk_set_rate(bsp_priv->clk_mac_speed, rate);
+	if (ret)
+		dev_err(dev, "%s: set clk_mac_speed rate %ld failed %d\n",
+			__func__, rate, ret);
+}
+
+static const struct rk_gmac_ops rv1126_ops = {
+	.set_to_rgmii = rv1126_set_to_rgmii,
+	.set_to_rmii = rv1126_set_to_rmii,
+	.set_rgmii_speed = rv1126_set_rgmii_speed,
+	.set_rmii_speed = rv1126_set_rmii_speed,
+};
+
 #define RK_GRF_MACPHY_CON0		0xb00
 #define RK_GRF_MACPHY_CON1		0xb04
 #define RK_GRF_MACPHY_CON2		0xb08
@@ -1304,6 +1572,10 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable)
 			if (!IS_ERR(bsp_priv->clk_mac_speed))
 				clk_prepare_enable(bsp_priv->clk_mac_speed);
 
+			if (bsp_priv->ops && bsp_priv->ops->set_clock_selection)
+				bsp_priv->ops->set_clock_selection(bsp_priv,
+					       bsp_priv->clock_input, true);
+
 			/**
 			 * if (!IS_ERR(bsp_priv->clk_mac))
 			 *	clk_prepare_enable(bsp_priv->clk_mac);
@@ -1330,6 +1602,10 @@ static int gmac_clk_enable(struct rk_priv_data *bsp_priv, bool enable)
 			clk_disable_unprepare(bsp_priv->mac_clk_tx);
 
 			clk_disable_unprepare(bsp_priv->clk_mac_speed);
+
+			if (bsp_priv->ops && bsp_priv->ops->set_clock_selection)
+				bsp_priv->ops->set_clock_selection(bsp_priv,
+					      bsp_priv->clock_input, false);
 			/**
 			 * if (!IS_ERR(bsp_priv->clk_mac))
 			 *	clk_disable_unprepare(bsp_priv->clk_mac);
@@ -1444,6 +1720,8 @@ static struct rk_priv_data *rk_gmac_setup(struct platform_device *pdev,
 
 	bsp_priv->grf = syscon_regmap_lookup_by_phandle(dev->of_node,
 							"rockchip,grf");
+	bsp_priv->php_grf = syscon_regmap_lookup_by_phandle(dev->of_node,
+							    "rockchip,php-grf");
 
 	if (plat->phy_node) {
 		bsp_priv->integrated_phy = of_property_read_bool(plat->phy_node,
@@ -1680,7 +1958,9 @@ static const struct of_device_id rk_gmac_dwmac_match[] = {
 	{ .compatible = "rockchip,rk3368-gmac", .data = &rk3368_ops },
 	{ .compatible = "rockchip,rk3399-gmac", .data = &rk3399_ops },
 	{ .compatible = "rockchip,rk3568-gmac", .data = &rk3568_ops },
+	{ .compatible = "rockchip,rk3588-gmac", .data = &rk3588_ops },
 	{ .compatible = "rockchip,rv1108-gmac", .data = &rv1108_ops },
+	{ .compatible = "rockchip,rv1126-gmac", .data = &rv1126_ops },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, rk_gmac_dwmac_match);
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100.h b/drivers/net/ethernet/stmicro/stmmac/dwmac100.h
index 35ab8d0bdce7..7ab791c8d355 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac100.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100.h
@@ -56,7 +56,7 @@
 #define MAC_CONTROL_TE		0x00000008	/* Transmitter Enable */
 #define MAC_CONTROL_RE		0x00000004	/* Receiver Enable */
 
-#define MAC_CORE_INIT (MAC_CONTROL_HBD | MAC_CONTROL_ASTP)
+#define MAC_CORE_INIT (MAC_CONTROL_HBD)
 
 /* MAC FLOW CTRL defines */
 #define MAC_FLOW_CTRL_PT_MASK	0xffff0000	/* Pause Time Mask */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h
index 3c73453725f9..4296ddda8aaa 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h
@@ -126,7 +126,7 @@ enum inter_frame_gap {
 #define GMAC_CONTROL_TE		0x00000008	/* Transmitter Enable */
 #define GMAC_CONTROL_RE		0x00000004	/* Receiver Enable */
 
-#define GMAC_CORE_INIT (GMAC_CONTROL_JD | GMAC_CONTROL_PS | GMAC_CONTROL_ACS | \
+#define GMAC_CORE_INIT (GMAC_CONTROL_JD | GMAC_CONTROL_PS | \
 			GMAC_CONTROL_BE | GMAC_CONTROL_DCRS)
 
 /* GMAC Frame Filter defines */
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
index 76edb9b72675..0e00dd83d027 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c
@@ -15,7 +15,6 @@
 #include <linux/crc32.h>
 #include <linux/slab.h>
 #include <linux/ethtool.h>
-#include <net/dsa.h>
 #include <asm/io.h>
 #include "stmmac.h"
 #include "stmmac_pcs.h"
@@ -24,7 +23,6 @@
 static void dwmac1000_core_init(struct mac_device_info *hw,
 				struct net_device *dev)
 {
-	struct stmmac_priv *priv = netdev_priv(dev);
 	void __iomem *ioaddr = hw->pcsr;
 	u32 value = readl(ioaddr + GMAC_CONTROL);
 	int mtu = dev->mtu;
@@ -32,13 +30,6 @@ static void dwmac1000_core_init(struct mac_device_info *hw,
 	/* Configure GMAC core */
 	value |= GMAC_CORE_INIT;
 
-	/* Clear ACS bit because Ethernet switch tagging formats such as
-	 * Broadcom tags can look like invalid LLC/SNAP packets and cause the
-	 * hardware to truncate packets on reception.
-	 */
-	if (netdev_uses_dsa(dev) || !priv->plat->enh_desc)
-		value &= ~GMAC_CONTROL_ACS;
-
 	if (mtu > 1500)
 		value |= GMAC_CONTROL_2K;
 	if (mtu > 2000)
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
index 75071a7d551a..a6e8d7bd9588 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c
@@ -15,7 +15,6 @@
 *******************************************************************************/
 
 #include <linux/crc32.h>
-#include <net/dsa.h>
 #include <asm/io.h>
 #include "stmmac.h"
 #include "dwmac100.h"
@@ -28,13 +27,6 @@ static void dwmac100_core_init(struct mac_device_info *hw,
 
 	value |= MAC_CORE_INIT;
 
-	/* Clear ASTP bit because Ethernet switch tagging formats such as
-	 * Broadcom tags can look like invalid LLC/SNAP packets and cause the
-	 * hardware to truncate packets on reception.
-	 */
-	if (netdev_uses_dsa(dev))
-		value &= ~MAC_CONTROL_ASTP;
-
 	writel(value, ioaddr + MAC_CONTROL);
 
 #ifdef STMMAC_VLAN_TAG_USED
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
index d8f1fbc25bdd..c25bfecb4a2d 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c
@@ -14,7 +14,6 @@
 #include <linux/slab.h>
 #include <linux/ethtool.h>
 #include <linux/io.h>
-#include <net/dsa.h>
 #include "stmmac.h"
 #include "stmmac_pcs.h"
 #include "dwmac4.h"
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
index d6a44d53fe08..f453b0d09366 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c
@@ -287,15 +287,15 @@ static void stmmac_ethtool_getdrvinfo(struct net_device *dev,
 	struct stmmac_priv *priv = netdev_priv(dev);
 
 	if (priv->plat->has_gmac || priv->plat->has_gmac4)
-		strlcpy(info->driver, GMAC_ETHTOOL_NAME, sizeof(info->driver));
+		strscpy(info->driver, GMAC_ETHTOOL_NAME, sizeof(info->driver));
 	else if (priv->plat->has_xgmac)
-		strlcpy(info->driver, XGMAC_ETHTOOL_NAME, sizeof(info->driver));
+		strscpy(info->driver, XGMAC_ETHTOOL_NAME, sizeof(info->driver));
 	else
-		strlcpy(info->driver, MAC100_ETHTOOL_NAME,
+		strscpy(info->driver, MAC100_ETHTOOL_NAME,
 			sizeof(info->driver));
 
 	if (priv->plat->pdev) {
-		strlcpy(info->bus_info, pci_name(priv->plat->pdev),
+		strscpy(info->bus_info, pci_name(priv->plat->pdev),
 			sizeof(info->bus_info));
 	}
 }
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
index 592d29abcb1c..65c96773c6d2 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c
@@ -3801,6 +3801,15 @@ static int __stmmac_open(struct net_device *dev,
 
 	stmmac_reset_queues_param(priv);
 
+	if (priv->plat->serdes_powerup) {
+		ret = priv->plat->serdes_powerup(dev, priv->plat->bsp_priv);
+		if (ret < 0) {
+			netdev_err(priv->dev, "%s: Serdes powerup failed\n",
+				   __func__);
+			goto init_error;
+		}
+	}
+
 	ret = stmmac_hw_setup(dev, true);
 	if (ret < 0) {
 		netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
@@ -3904,6 +3913,10 @@ static int stmmac_release(struct net_device *dev)
 	/* Disable the MAC Rx/Tx */
 	stmmac_mac_set(priv, priv->ioaddr, false);
 
+	/* Powerdown Serdes if there is */
+	if (priv->plat->serdes_powerdown)
+		priv->plat->serdes_powerdown(dev, priv->plat->bsp_priv);
+
 	netif_carrier_off(dev);
 
 	stmmac_release_ptp(priv);
@@ -5076,16 +5089,8 @@ read_again:
 		buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
 		len += buf1_len;
 
-		/* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
-		 * Type frames (LLC/LLC-SNAP)
-		 *
-		 * llc_snap is never checked in GMAC >= 4, so this ACS
-		 * feature is always disabled and packets need to be
-		 * stripped manually.
-		 */
-		if (likely(!(status & rx_not_ls)) &&
-		    (likely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
-		     unlikely(status != llc_snap))) {
+		/* ACS is disabled; strip manually. */
+		if (likely(!(status & rx_not_ls))) {
 			buf1_len -= ETH_FCS_LEN;
 			len -= ETH_FCS_LEN;
 		}
@@ -5262,16 +5267,8 @@ read_again:
 		buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
 		len += buf2_len;
 
-		/* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
-		 * Type frames (LLC/LLC-SNAP)
-		 *
-		 * llc_snap is never checked in GMAC >= 4, so this ACS
-		 * feature is always disabled and packets need to be
-		 * stripped manually.
-		 */
-		if (likely(!(status & rx_not_ls)) &&
-		    (likely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
-		     unlikely(status != llc_snap))) {
+		/* ACS is disabled; strip manually. */
+		if (likely(!(status & rx_not_ls))) {
 			if (buf2_len) {
 				buf2_len -= ETH_FCS_LEN;
 				len -= ETH_FCS_LEN;
@@ -6890,8 +6887,7 @@ static void stmmac_napi_add(struct net_device *dev)
 		spin_lock_init(&ch->lock);
 
 		if (queue < priv->plat->rx_queues_to_use) {
-			netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx,
-				       NAPI_POLL_WEIGHT);
+			netif_napi_add(dev, &ch->rx_napi, stmmac_napi_poll_rx);
 		}
 		if (queue < priv->plat->tx_queues_to_use) {
 			netif_napi_add_tx(dev, &ch->tx_napi,
@@ -6900,8 +6896,7 @@ static void stmmac_napi_add(struct net_device *dev)
 		if (queue < priv->plat->rx_queues_to_use &&
 		    queue < priv->plat->tx_queues_to_use) {
 			netif_napi_add(dev, &ch->rxtx_napi,
-				       stmmac_napi_poll_rxtx,
-				       NAPI_POLL_WEIGHT);
+				       stmmac_napi_poll_rxtx);
 		}
 	}
 }
@@ -7293,14 +7288,6 @@ int stmmac_dvr_probe(struct device *device,
 		goto error_netdev_register;
 	}
 
-	if (priv->plat->serdes_powerup) {
-		ret = priv->plat->serdes_powerup(ndev,
-						 priv->plat->bsp_priv);
-
-		if (ret < 0)
-			goto error_serdes_powerup;
-	}
-
 #ifdef CONFIG_DEBUG_FS
 	stmmac_init_fs(ndev);
 #endif
@@ -7315,8 +7302,6 @@ int stmmac_dvr_probe(struct device *device,
 
 	return ret;
 
-error_serdes_powerup:
-	unregister_netdev(ndev);
 error_netdev_register:
 	phylink_destroy(priv->phylink);
 error_xpcs_setup:
diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
index 9f5cac4000da..50f6b4a14be4 100644
--- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
+++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c
@@ -440,11 +440,12 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac)
 	/* Default to phy auto-detection */
 	plat->phy_addr = -1;
 
-	/* Default to get clk_csr from stmmac_clk_crs_set(),
+	/* Default to get clk_csr from stmmac_clk_csr_set(),
 	 * or get clk_csr from device tree.
 	 */
 	plat->clk_csr = -1;
-	of_property_read_u32(np, "clk_csr", &plat->clk_csr);
+	if (of_property_read_u32(np, "snps,clk-csr", &plat->clk_csr))
+		of_property_read_u32(np, "clk_csr", &plat->clk_csr);
 
 	/* "snps,phy-addr" is not a standard property. Mark it as deprecated
 	 * and warn of its use. Remove this when phy node support is added.
diff --git a/drivers/net/ethernet/sun/cassini.c b/drivers/net/ethernet/sun/cassini.c
index 0b08b0e085e8..0aca193d9550 100644
--- a/drivers/net/ethernet/sun/cassini.c
+++ b/drivers/net/ethernet/sun/cassini.c
@@ -4484,9 +4484,9 @@ static void cas_set_multicast(struct net_device *dev)
 static void cas_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
 	struct cas *cp = netdev_priv(dev);
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(cp->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(cp->pdev), sizeof(info->bus_info));
 }
 
 static int cas_get_link_ksettings(struct net_device *dev,
@@ -5050,7 +5050,7 @@ static int cas_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	dev->watchdog_timeo = CAS_TX_TIMEOUT;
 
 #ifdef USE_NAPI
-	netif_napi_add(dev, &cp->napi, cas_poll, 64);
+	netif_napi_add(dev, &cp->napi, cas_poll);
 #endif
 	dev->irq = pdev->irq;
 	dev->dma = 0;
diff --git a/drivers/net/ethernet/sun/ldmvsw.c b/drivers/net/ethernet/sun/ldmvsw.c
index 0cd8493b810f..8addee6d04bd 100644
--- a/drivers/net/ethernet/sun/ldmvsw.c
+++ b/drivers/net/ethernet/sun/ldmvsw.c
@@ -63,8 +63,8 @@ static struct vio_version vsw_versions[] = {
 static void vsw_get_drvinfo(struct net_device *dev,
 			    struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
 }
 
 static u32 vsw_get_msglevel(struct net_device *dev)
@@ -354,8 +354,7 @@ static int vsw_port_probe(struct vio_dev *vdev, const struct vio_device_id *id)
 
 	dev_set_drvdata(&vdev->dev, port);
 
-	netif_napi_add(dev, &port->napi, sunvnet_poll_common,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &port->napi, sunvnet_poll_common);
 
 	spin_lock_irqsave(&vp->lock, flags);
 	list_add_rcu(&port->list, &vp->port_list);
diff --git a/drivers/net/ethernet/sun/niu.c b/drivers/net/ethernet/sun/niu.c
index df70df29deea..e6144d963eaa 100644
--- a/drivers/net/ethernet/sun/niu.c
+++ b/drivers/net/ethernet/sun/niu.c
@@ -6798,12 +6798,12 @@ static void niu_get_drvinfo(struct net_device *dev,
 	struct niu *np = netdev_priv(dev);
 	struct niu_vpd *vpd = &np->vpd;
 
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
 	snprintf(info->fw_version, sizeof(info->fw_version), "%d.%d",
 		vpd->fcode_major, vpd->fcode_minor);
 	if (np->parent->plat_type != PLAT_TYPE_NIU)
-		strlcpy(info->bus_info, pci_name(np->pdev),
+		strscpy(info->bus_info, pci_name(np->pdev),
 			sizeof(info->bus_info));
 }
 
@@ -9115,7 +9115,7 @@ static int niu_ldg_init(struct niu *np)
 	for (i = 0; i < np->num_ldg; i++) {
 		struct niu_ldg *lp = &np->ldg[i];
 
-		netif_napi_add(np->dev, &lp->napi, niu_poll, 64);
+		netif_napi_add(np->dev, &lp->napi, niu_poll);
 
 		lp->np = np;
 		lp->ldg_num = ldg_num_map[i];
diff --git a/drivers/net/ethernet/sun/sunbmac.c b/drivers/net/ethernet/sun/sunbmac.c
index 531a6f449afa..34b94153bf0c 100644
--- a/drivers/net/ethernet/sun/sunbmac.c
+++ b/drivers/net/ethernet/sun/sunbmac.c
@@ -1038,8 +1038,8 @@ static void bigmac_set_multicast(struct net_device *dev)
 /* Ethtool support... */
 static void bigmac_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, "sunbmac", sizeof(info->driver));
-	strlcpy(info->version, "2.0", sizeof(info->version));
+	strscpy(info->driver, "sunbmac", sizeof(info->driver));
+	strscpy(info->version, "2.0", sizeof(info->version));
 }
 
 static u32 bigmac_get_link(struct net_device *dev)
diff --git a/drivers/net/ethernet/sun/sungem.c b/drivers/net/ethernet/sun/sungem.c
index a14591b41acb..4154e68639ac 100644
--- a/drivers/net/ethernet/sun/sungem.c
+++ b/drivers/net/ethernet/sun/sungem.c
@@ -2521,9 +2521,9 @@ static void gem_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info
 {
 	struct gem *gp = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(gp->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(gp->pdev), sizeof(info->bus_info));
 }
 
 static int gem_get_link_ksettings(struct net_device *dev,
@@ -2980,7 +2980,7 @@ static int gem_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto err_out_free_consistent;
 
 	dev->netdev_ops = &gem_netdev_ops;
-	netif_napi_add(dev, &gp->napi, gem_poll, 64);
+	netif_napi_add(dev, &gp->napi, gem_poll);
 	dev->ethtool_ops = &gem_ethtool_ops;
 	dev->watchdog_timeo = 5 * HZ;
 	dev->dma = 0;
diff --git a/drivers/net/ethernet/sun/sunhme.c b/drivers/net/ethernet/sun/sunhme.c
index 88aa0d310aee..62deed210a95 100644
--- a/drivers/net/ethernet/sun/sunhme.c
+++ b/drivers/net/ethernet/sun/sunhme.c
@@ -61,15 +61,8 @@
 #include "sunhme.h"
 
 #define DRV_NAME	"sunhme"
-#define DRV_VERSION	"3.10"
-#define DRV_RELDATE	"August 26, 2008"
-#define DRV_AUTHOR	"David S. Miller (davem@davemloft.net)"
 
-static char version[] =
-	DRV_NAME ".c:v" DRV_VERSION " " DRV_RELDATE " " DRV_AUTHOR "\n";
-
-MODULE_VERSION(DRV_VERSION);
-MODULE_AUTHOR(DRV_AUTHOR);
+MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
 MODULE_DESCRIPTION("Sun HappyMealEthernet(HME) 10/100baseT ethernet driver");
 MODULE_LICENSE("GPL");
 
@@ -87,13 +80,17 @@ static struct quattro *qfe_sbus_list;
 static struct quattro *qfe_pci_list;
 #endif
 
-#undef HMEDEBUG
-#undef SXDEBUG
-#undef RXDEBUG
-#undef TXDEBUG
-#undef TXLOGGING
+#define hme_debug(fmt, ...) pr_debug("%s: " fmt, __func__, ##__VA_ARGS__)
+#define HMD hme_debug
+
+/* "Auto Switch Debug" aka phy debug */
+#if 1
+#define ASD hme_debug
+#else
+#define ASD(...)
+#endif
 
-#ifdef TXLOGGING
+#if 0
 struct hme_tx_logent {
 	unsigned int tstamp;
 	int tx_new, tx_old;
@@ -128,46 +125,16 @@ static __inline__ void tx_dump_log(void)
 
 	this = txlog_cur_entry;
 	for (i = 0; i < TX_LOG_LEN; i++) {
-		printk("TXLOG[%d]: j[%08x] tx[N(%d)O(%d)] action[%08x] stat[%08x]\n", i,
+		pr_err("TXLOG[%d]: j[%08x] tx[N(%d)O(%d)] action[%08x] stat[%08x]\n", i,
 		       tx_log[this].tstamp,
 		       tx_log[this].tx_new, tx_log[this].tx_old,
 		       tx_log[this].action, tx_log[this].status);
 		this = (this + 1) & (TX_LOG_LEN - 1);
 	}
 }
-static __inline__ void tx_dump_ring(struct happy_meal *hp)
-{
-	struct hmeal_init_block *hb = hp->happy_block;
-	struct happy_meal_txd *tp = &hb->happy_meal_txd[0];
-	int i;
-
-	for (i = 0; i < TX_RING_SIZE; i+=4) {
-		printk("TXD[%d..%d]: [%08x:%08x] [%08x:%08x] [%08x:%08x] [%08x:%08x]\n",
-		       i, i + 4,
-		       le32_to_cpu(tp[i].tx_flags), le32_to_cpu(tp[i].tx_addr),
-		       le32_to_cpu(tp[i + 1].tx_flags), le32_to_cpu(tp[i + 1].tx_addr),
-		       le32_to_cpu(tp[i + 2].tx_flags), le32_to_cpu(tp[i + 2].tx_addr),
-		       le32_to_cpu(tp[i + 3].tx_flags), le32_to_cpu(tp[i + 3].tx_addr));
-	}
-}
-#else
-#define tx_add_log(hp, a, s)		do { } while(0)
-#define tx_dump_log()			do { } while(0)
-#define tx_dump_ring(hp)		do { } while(0)
-#endif
-
-#ifdef HMEDEBUG
-#define HMD(x)  printk x
-#else
-#define HMD(x)
-#endif
-
-/* #define AUTO_SWITCH_DEBUG */
-
-#ifdef AUTO_SWITCH_DEBUG
-#define ASD(x)  printk x
 #else
-#define ASD(x)
+#define tx_add_log(hp, a, s)
+#define tx_dump_log()
 #endif
 
 #define DEFAULT_IPG0      16 /* For lance-mode only */
@@ -343,8 +310,6 @@ static int happy_meal_bb_read(struct happy_meal *hp,
 	int retval = 0;
 	int i;
 
-	ASD(("happy_meal_bb_read: reg=%d ", reg));
-
 	/* Enable the MIF BitBang outputs. */
 	hme_write32(hp, tregs + TCVR_BBOENAB, 1);
 
@@ -378,7 +343,7 @@ static int happy_meal_bb_read(struct happy_meal *hp,
 	(void) BB_GET_BIT2(hp, tregs, (hp->tcvr_type == internal));
 	(void) BB_GET_BIT2(hp, tregs, (hp->tcvr_type == internal));
 	(void) BB_GET_BIT2(hp, tregs, (hp->tcvr_type == internal));
-	ASD(("value=%x\n", retval));
+	ASD("reg=%d value=%x\n", reg, retval);
 	return retval;
 }
 
@@ -389,7 +354,7 @@ static void happy_meal_bb_write(struct happy_meal *hp,
 	u32 tmp;
 	int i;
 
-	ASD(("happy_meal_bb_write: reg=%d value=%x\n", reg, value));
+	ASD("reg=%d value=%x\n", reg, value);
 
 	/* Enable the MIF BitBang outputs. */
 	hme_write32(hp, tregs + TCVR_BBOENAB, 1);
@@ -433,14 +398,13 @@ static int happy_meal_tcvr_read(struct happy_meal *hp,
 	int tries = TCVR_READ_TRIES;
 	int retval;
 
-	ASD(("happy_meal_tcvr_read: reg=0x%02x ", reg));
 	if (hp->tcvr_type == none) {
-		ASD(("no transceiver, value=TCVR_FAILURE\n"));
+		ASD("no transceiver, value=TCVR_FAILURE\n");
 		return TCVR_FAILURE;
 	}
 
 	if (!(hp->happy_flags & HFLAG_FENABLE)) {
-		ASD(("doing bit bang\n"));
+		ASD("doing bit bang\n");
 		return happy_meal_bb_read(hp, tregs, reg);
 	}
 
@@ -449,11 +413,11 @@ static int happy_meal_tcvr_read(struct happy_meal *hp,
 	while (!(hme_read32(hp, tregs + TCVR_FRAME) & 0x10000) && --tries)
 		udelay(20);
 	if (!tries) {
-		printk(KERN_ERR "happy meal: Aieee, transceiver MIF read bolixed\n");
+		netdev_err(hp->dev, "Aieee, transceiver MIF read bolixed\n");
 		return TCVR_FAILURE;
 	}
 	retval = hme_read32(hp, tregs + TCVR_FRAME) & 0xffff;
-	ASD(("value=%04x\n", retval));
+	ASD("reg=0x%02x value=%04x\n", reg, retval);
 	return retval;
 }
 
@@ -465,7 +429,7 @@ static void happy_meal_tcvr_write(struct happy_meal *hp,
 {
 	int tries = TCVR_WRITE_TRIES;
 
-	ASD(("happy_meal_tcvr_write: reg=0x%02x value=%04x\n", reg, value));
+	ASD("reg=0x%02x value=%04x\n", reg, value);
 
 	/* Welcome to Sun Microsystems, can I take your order please? */
 	if (!(hp->happy_flags & HFLAG_FENABLE)) {
@@ -482,7 +446,7 @@ static void happy_meal_tcvr_write(struct happy_meal *hp,
 
 	/* Anything else? */
 	if (!tries)
-		printk(KERN_ERR "happy meal: Aieee, transceiver MIF write bolixed\n");
+		netdev_err(hp->dev, "Aieee, transceiver MIF write bolixed\n");
 
 	/* Fifty-two cents is your change, have a nice day. */
 }
@@ -660,8 +624,8 @@ static void happy_meal_timer(struct timer_list *t)
 			/* Enter force mode. */
 	do_force_mode:
 			hp->sw_bmcr = happy_meal_tcvr_read(hp, tregs, MII_BMCR);
-			printk(KERN_NOTICE "%s: Auto-Negotiation unsuccessful, trying force link mode\n",
-			       hp->dev->name);
+			netdev_notice(hp->dev,
+				      "Auto-Negotiation unsuccessful, trying force link mode\n");
 			hp->sw_bmcr = BMCR_SPEED100;
 			happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr);
 
@@ -720,8 +684,8 @@ static void happy_meal_timer(struct timer_list *t)
 			restart_timer = 0;
 		} else {
 			if (hp->timer_ticks >= 10) {
-				printk(KERN_NOTICE "%s: Auto negotiation successful, link still "
-				       "not completely up.\n", hp->dev->name);
+				netdev_notice(hp->dev,
+					      "Auto negotiation successful, link still not completely up.\n");
 				hp->timer_ticks = 0;
 				restart_timer = 1;
 			} else {
@@ -776,14 +740,14 @@ static void happy_meal_timer(struct timer_list *t)
 					 */
 
 					/* Let the user know... */
-					printk(KERN_NOTICE "%s: Link down, cable problem?\n",
-					       hp->dev->name);
+					netdev_notice(hp->dev,
+						      "Link down, cable problem?\n");
 
 					ret = happy_meal_init(hp);
 					if (ret) {
 						/* ho hum... */
-						printk(KERN_ERR "%s: Error, cannot re-init the "
-						       "Happy Meal.\n", hp->dev->name);
+						netdev_err(hp->dev,
+							   "Error, cannot re-init the Happy Meal.\n");
 					}
 					goto out;
 				}
@@ -805,8 +769,8 @@ static void happy_meal_timer(struct timer_list *t)
 	case asleep:
 	default:
 		/* Can't happens.... */
-		printk(KERN_ERR "%s: Aieee, link timer is asleep but we got one anyways!\n",
-		       hp->dev->name);
+		netdev_err(hp->dev,
+			   "Aieee, link timer is asleep but we got one anyways!\n");
 		restart_timer = 0;
 		hp->timer_ticks = 0;
 		hp->timer_state = asleep; /* foo on you */
@@ -830,7 +794,7 @@ static void happy_meal_tx_reset(struct happy_meal *hp, void __iomem *bregs)
 {
 	int tries = TX_RESET_TRIES;
 
-	HMD(("happy_meal_tx_reset: reset, "));
+	HMD("reset...\n");
 
 	/* Would you like to try our SMCC Delux? */
 	hme_write32(hp, bregs + BMAC_TXSWRESET, 0);
@@ -839,10 +803,10 @@ static void happy_meal_tx_reset(struct happy_meal *hp, void __iomem *bregs)
 
 	/* Lettuce, tomato, buggy hardware (no extra charge)? */
 	if (!tries)
-		printk(KERN_ERR "happy meal: Transceiver BigMac ATTACK!");
+		netdev_err(hp->dev, "Transceiver BigMac ATTACK!");
 
 	/* Take care. */
-	HMD(("done\n"));
+	HMD("done\n");
 }
 
 /* hp->happy_lock must be held */
@@ -850,7 +814,7 @@ static void happy_meal_rx_reset(struct happy_meal *hp, void __iomem *bregs)
 {
 	int tries = RX_RESET_TRIES;
 
-	HMD(("happy_meal_rx_reset: reset, "));
+	HMD("reset...\n");
 
 	/* We have a special on GNU/Viking hardware bugs today. */
 	hme_write32(hp, bregs + BMAC_RXSWRESET, 0);
@@ -859,10 +823,10 @@ static void happy_meal_rx_reset(struct happy_meal *hp, void __iomem *bregs)
 
 	/* Will that be all? */
 	if (!tries)
-		printk(KERN_ERR "happy meal: Receiver BigMac ATTACK!");
+		netdev_err(hp->dev, "Receiver BigMac ATTACK!\n");
 
 	/* Don't forget your vik_1137125_wa.  Have a nice day. */
-	HMD(("done\n"));
+	HMD("done\n");
 }
 
 #define STOP_TRIES         16
@@ -872,7 +836,7 @@ static void happy_meal_stop(struct happy_meal *hp, void __iomem *gregs)
 {
 	int tries = STOP_TRIES;
 
-	HMD(("happy_meal_stop: reset, "));
+	HMD("reset...\n");
 
 	/* We're consolidating our STB products, it's your lucky day. */
 	hme_write32(hp, gregs + GREG_SWRESET, GREG_RESET_ALL);
@@ -881,10 +845,10 @@ static void happy_meal_stop(struct happy_meal *hp, void __iomem *gregs)
 
 	/* Come back next week when we are "Sun Microelectronics". */
 	if (!tries)
-		printk(KERN_ERR "happy meal: Fry guys.");
+		netdev_err(hp->dev, "Fry guys.\n");
 
 	/* Remember: "Different name, same old buggy as shit hardware." */
-	HMD(("done\n"));
+	HMD("done\n");
 }
 
 /* hp->happy_lock must be held */
@@ -913,21 +877,18 @@ static void happy_meal_get_counters(struct happy_meal *hp, void __iomem *bregs)
 /* hp->happy_lock must be held */
 static void happy_meal_poll_stop(struct happy_meal *hp, void __iomem *tregs)
 {
-	ASD(("happy_meal_poll_stop: "));
-
 	/* If polling disabled or not polling already, nothing to do. */
 	if ((hp->happy_flags & (HFLAG_POLLENABLE | HFLAG_POLL)) !=
 	   (HFLAG_POLLENABLE | HFLAG_POLL)) {
-		HMD(("not polling, return\n"));
+		ASD("not polling, return\n");
 		return;
 	}
 
 	/* Shut up the MIF. */
-	ASD(("were polling, mif ints off, "));
+	ASD("were polling, mif ints off, polling off\n");
 	hme_write32(hp, tregs + TCVR_IMASK, 0xffff);
 
 	/* Turn off polling. */
-	ASD(("polling off, "));
 	hme_write32(hp, tregs + TCVR_CFG,
 		    hme_read32(hp, tregs + TCVR_CFG) & ~(TCV_CFG_PENABLE));
 
@@ -936,7 +897,7 @@ static void happy_meal_poll_stop(struct happy_meal *hp, void __iomem *tregs)
 
 	/* Let the bits set. */
 	udelay(200);
-	ASD(("done\n"));
+	ASD("done\n");
 }
 
 /* Only Sun can take such nice parts and fuck up the programming interface
@@ -952,44 +913,40 @@ static int happy_meal_tcvr_reset(struct happy_meal *hp, void __iomem *tregs)
 	int result, tries = TCVR_RESET_TRIES;
 
 	tconfig = hme_read32(hp, tregs + TCVR_CFG);
-	ASD(("happy_meal_tcvr_reset: tcfg<%08lx> ", tconfig));
+	ASD("tcfg=%08x\n", tconfig);
 	if (hp->tcvr_type == external) {
-		ASD(("external<"));
 		hme_write32(hp, tregs + TCVR_CFG, tconfig & ~(TCV_CFG_PSELECT));
 		hp->tcvr_type = internal;
 		hp->paddr = TCV_PADDR_ITX;
-		ASD(("ISOLATE,"));
 		happy_meal_tcvr_write(hp, tregs, MII_BMCR,
 				      (BMCR_LOOPBACK|BMCR_PDOWN|BMCR_ISOLATE));
 		result = happy_meal_tcvr_read(hp, tregs, MII_BMCR);
 		if (result == TCVR_FAILURE) {
-			ASD(("phyread_fail>\n"));
+			ASD("phyread_fail\n");
 			return -1;
 		}
-		ASD(("phyread_ok,PSELECT>"));
+		ASD("external: ISOLATE, phyread_ok, PSELECT\n");
 		hme_write32(hp, tregs + TCVR_CFG, tconfig | TCV_CFG_PSELECT);
 		hp->tcvr_type = external;
 		hp->paddr = TCV_PADDR_ETX;
 	} else {
 		if (tconfig & TCV_CFG_MDIO1) {
-			ASD(("internal<PSELECT,"));
 			hme_write32(hp, tregs + TCVR_CFG, (tconfig | TCV_CFG_PSELECT));
-			ASD(("ISOLATE,"));
 			happy_meal_tcvr_write(hp, tregs, MII_BMCR,
 					      (BMCR_LOOPBACK|BMCR_PDOWN|BMCR_ISOLATE));
 			result = happy_meal_tcvr_read(hp, tregs, MII_BMCR);
 			if (result == TCVR_FAILURE) {
-				ASD(("phyread_fail>\n"));
+				ASD("phyread_fail>\n");
 				return -1;
 			}
-			ASD(("phyread_ok,~PSELECT>"));
+			ASD("internal: PSELECT, ISOLATE, phyread_ok, ~PSELECT\n");
 			hme_write32(hp, tregs + TCVR_CFG, (tconfig & ~(TCV_CFG_PSELECT)));
 			hp->tcvr_type = internal;
 			hp->paddr = TCV_PADDR_ITX;
 		}
 	}
 
-	ASD(("BMCR_RESET "));
+	ASD("BMCR_RESET...\n");
 	happy_meal_tcvr_write(hp, tregs, MII_BMCR, BMCR_RESET);
 
 	while (--tries) {
@@ -1002,10 +959,10 @@ static int happy_meal_tcvr_reset(struct happy_meal *hp, void __iomem *tregs)
 		udelay(20);
 	}
 	if (!tries) {
-		ASD(("BMCR RESET FAILED!\n"));
+		ASD("BMCR RESET FAILED!\n");
 		return -1;
 	}
-	ASD(("RESET_OK\n"));
+	ASD("RESET_OK\n");
 
 	/* Get fresh copies of the PHY registers. */
 	hp->sw_bmsr      = happy_meal_tcvr_read(hp, tregs, MII_BMSR);
@@ -1013,7 +970,7 @@ static int happy_meal_tcvr_reset(struct happy_meal *hp, void __iomem *tregs)
 	hp->sw_physid2   = happy_meal_tcvr_read(hp, tregs, MII_PHYSID2);
 	hp->sw_advertise = happy_meal_tcvr_read(hp, tregs, MII_ADVERTISE);
 
-	ASD(("UNISOLATE"));
+	ASD("UNISOLATE...\n");
 	hp->sw_bmcr &= ~(BMCR_ISOLATE);
 	happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr);
 
@@ -1027,10 +984,10 @@ static int happy_meal_tcvr_reset(struct happy_meal *hp, void __iomem *tregs)
 		udelay(20);
 	}
 	if (!tries) {
-		ASD((" FAILED!\n"));
+		ASD("UNISOLATE FAILED!\n");
 		return -1;
 	}
-	ASD((" SUCCESS and CSCONFIG_DFBYPASS\n"));
+	ASD("SUCCESS and CSCONFIG_DFBYPASS\n");
 	if (!is_lucent_phy(hp)) {
 		result = happy_meal_tcvr_read(hp, tregs,
 					      DP83840_CSCONFIG);
@@ -1048,60 +1005,55 @@ static void happy_meal_transceiver_check(struct happy_meal *hp, void __iomem *tr
 {
 	unsigned long tconfig = hme_read32(hp, tregs + TCVR_CFG);
 
-	ASD(("happy_meal_transceiver_check: tcfg=%08lx ", tconfig));
+	ASD("tcfg=%08lx\n", tconfig);
 	if (hp->happy_flags & HFLAG_POLL) {
 		/* If we are polling, we must stop to get the transceiver type. */
-		ASD(("<polling> "));
 		if (hp->tcvr_type == internal) {
 			if (tconfig & TCV_CFG_MDIO1) {
-				ASD(("<internal> <poll stop> "));
 				happy_meal_poll_stop(hp, tregs);
 				hp->paddr = TCV_PADDR_ETX;
 				hp->tcvr_type = external;
-				ASD(("<external>\n"));
 				tconfig &= ~(TCV_CFG_PENABLE);
 				tconfig |= TCV_CFG_PSELECT;
 				hme_write32(hp, tregs + TCVR_CFG, tconfig);
+				ASD("poll stop, internal->external\n");
 			}
 		} else {
 			if (hp->tcvr_type == external) {
-				ASD(("<external> "));
 				if (!(hme_read32(hp, tregs + TCVR_STATUS) >> 16)) {
-					ASD(("<poll stop> "));
 					happy_meal_poll_stop(hp, tregs);
 					hp->paddr = TCV_PADDR_ITX;
 					hp->tcvr_type = internal;
-					ASD(("<internal>\n"));
 					hme_write32(hp, tregs + TCVR_CFG,
 						    hme_read32(hp, tregs + TCVR_CFG) &
 						    ~(TCV_CFG_PSELECT));
+					ASD("poll stop, external->internal\n");
 				}
-				ASD(("\n"));
 			} else {
-				ASD(("<none>\n"));
+				ASD("polling, none\n");
 			}
 		}
 	} else {
 		u32 reread = hme_read32(hp, tregs + TCVR_CFG);
 
 		/* Else we can just work off of the MDIO bits. */
-		ASD(("<not polling> "));
 		if (reread & TCV_CFG_MDIO1) {
 			hme_write32(hp, tregs + TCVR_CFG, tconfig | TCV_CFG_PSELECT);
 			hp->paddr = TCV_PADDR_ETX;
 			hp->tcvr_type = external;
-			ASD(("<external>\n"));
+			ASD("not polling, external\n");
 		} else {
 			if (reread & TCV_CFG_MDIO0) {
 				hme_write32(hp, tregs + TCVR_CFG,
 					    tconfig & ~(TCV_CFG_PSELECT));
 				hp->paddr = TCV_PADDR_ITX;
 				hp->tcvr_type = internal;
-				ASD(("<internal>\n"));
+				ASD("not polling, internal\n");
 			} else {
-				printk(KERN_ERR "happy meal: Transceiver and a coke please.");
+				netdev_err(hp->dev,
+					   "Transceiver and a coke please.");
 				hp->tcvr_type = none; /* Grrr... */
-				ASD(("<none>\n"));
+				ASD("not polling, none\n");
 			}
 		}
 	}
@@ -1208,15 +1160,14 @@ static void happy_meal_init_rings(struct happy_meal *hp)
 	struct hmeal_init_block *hb = hp->happy_block;
 	int i;
 
-	HMD(("happy_meal_init_rings: counters to zero, "));
+	HMD("counters to zero\n");
 	hp->rx_new = hp->rx_old = hp->tx_new = hp->tx_old = 0;
 
 	/* Free any skippy bufs left around in the rings. */
-	HMD(("clean, "));
 	happy_meal_clean_rings(hp);
 
 	/* Now get new skippy bufs for the receive ring. */
-	HMD(("init rxring, "));
+	HMD("init rxring\n");
 	for (i = 0; i < RX_RING_SIZE; i++) {
 		struct sk_buff *skb;
 		u32 mapping;
@@ -1243,11 +1194,11 @@ static void happy_meal_init_rings(struct happy_meal *hp)
 		skb_reserve(skb, RX_OFFSET);
 	}
 
-	HMD(("init txring, "));
+	HMD("init txring\n");
 	for (i = 0; i < TX_RING_SIZE; i++)
 		hme_write_txd(hp, &hb->happy_meal_txd[i], 0, 0);
 
-	HMD(("done\n"));
+	HMD("done\n");
 }
 
 /* hp->happy_lock must be held */
@@ -1294,17 +1245,11 @@ happy_meal_begin_auto_negotiation(struct happy_meal *hp,
 		 * XXX so I completely skip checking for it in the BMSR for now.
 		 */
 
-#ifdef AUTO_SWITCH_DEBUG
-		ASD(("%s: Advertising [ ", hp->dev->name));
-		if (hp->sw_advertise & ADVERTISE_10HALF)
-			ASD(("10H "));
-		if (hp->sw_advertise & ADVERTISE_10FULL)
-			ASD(("10F "));
-		if (hp->sw_advertise & ADVERTISE_100HALF)
-			ASD(("100H "));
-		if (hp->sw_advertise & ADVERTISE_100FULL)
-			ASD(("100F "));
-#endif
+		ASD("Advertising [ %s%s%s%s]\n",
+		    hp->sw_advertise & ADVERTISE_10HALF ? "10H " : "",
+		    hp->sw_advertise & ADVERTISE_10FULL ? "10F " : "",
+		    hp->sw_advertise & ADVERTISE_100HALF ? "100H " : "",
+		    hp->sw_advertise & ADVERTISE_100FULL ? "100F " : "");
 
 		/* Enable Auto-Negotiation, this is usually on already... */
 		hp->sw_bmcr |= BMCR_ANENABLE;
@@ -1324,10 +1269,11 @@ happy_meal_begin_auto_negotiation(struct happy_meal *hp,
 			udelay(10);
 		}
 		if (!timeout) {
-			printk(KERN_ERR "%s: Happy Meal would not start auto negotiation "
-			       "BMCR=0x%04x\n", hp->dev->name, hp->sw_bmcr);
-			printk(KERN_NOTICE "%s: Performing force link detection.\n",
-			       hp->dev->name);
+			netdev_err(hp->dev,
+				   "Happy Meal would not start auto negotiation BMCR=0x%04x\n",
+				   hp->sw_bmcr);
+			netdev_notice(hp->dev,
+				      "Performing force link detection.\n");
 			goto force_link;
 		} else {
 			hp->timer_state = arbwait;
@@ -1382,70 +1328,69 @@ static int happy_meal_init(struct happy_meal *hp)
 	void __iomem *erxregs      = hp->erxregs;
 	void __iomem *bregs        = hp->bigmacregs;
 	void __iomem *tregs        = hp->tcvregs;
+	const char *bursts;
 	u32 regtmp, rxcfg;
 
 	/* If auto-negotiation timer is running, kill it. */
 	del_timer(&hp->happy_timer);
 
-	HMD(("happy_meal_init: happy_flags[%08x] ",
-	     hp->happy_flags));
+	HMD("happy_flags[%08x]\n", hp->happy_flags);
 	if (!(hp->happy_flags & HFLAG_INIT)) {
-		HMD(("set HFLAG_INIT, "));
+		HMD("set HFLAG_INIT\n");
 		hp->happy_flags |= HFLAG_INIT;
 		happy_meal_get_counters(hp, bregs);
 	}
 
 	/* Stop polling. */
-	HMD(("to happy_meal_poll_stop\n"));
+	HMD("to happy_meal_poll_stop\n");
 	happy_meal_poll_stop(hp, tregs);
 
 	/* Stop transmitter and receiver. */
-	HMD(("happy_meal_init: to happy_meal_stop\n"));
+	HMD("to happy_meal_stop\n");
 	happy_meal_stop(hp, gregs);
 
 	/* Alloc and reset the tx/rx descriptor chains. */
-	HMD(("happy_meal_init: to happy_meal_init_rings\n"));
+	HMD("to happy_meal_init_rings\n");
 	happy_meal_init_rings(hp);
 
 	/* Shut up the MIF. */
-	HMD(("happy_meal_init: Disable all MIF irqs (old[%08x]), ",
-	     hme_read32(hp, tregs + TCVR_IMASK)));
+	HMD("Disable all MIF irqs (old[%08x])\n",
+	    hme_read32(hp, tregs + TCVR_IMASK));
 	hme_write32(hp, tregs + TCVR_IMASK, 0xffff);
 
 	/* See if we can enable the MIF frame on this card to speak to the DP83840. */
 	if (hp->happy_flags & HFLAG_FENABLE) {
-		HMD(("use frame old[%08x], ",
-		     hme_read32(hp, tregs + TCVR_CFG)));
+		HMD("use frame old[%08x]\n",
+		    hme_read32(hp, tregs + TCVR_CFG));
 		hme_write32(hp, tregs + TCVR_CFG,
 			    hme_read32(hp, tregs + TCVR_CFG) & ~(TCV_CFG_BENABLE));
 	} else {
-		HMD(("use bitbang old[%08x], ",
-		     hme_read32(hp, tregs + TCVR_CFG)));
+		HMD("use bitbang old[%08x]\n",
+		    hme_read32(hp, tregs + TCVR_CFG));
 		hme_write32(hp, tregs + TCVR_CFG,
 			    hme_read32(hp, tregs + TCVR_CFG) | TCV_CFG_BENABLE);
 	}
 
 	/* Check the state of the transceiver. */
-	HMD(("to happy_meal_transceiver_check\n"));
+	HMD("to happy_meal_transceiver_check\n");
 	happy_meal_transceiver_check(hp, tregs);
 
 	/* Put the Big Mac into a sane state. */
-	HMD(("happy_meal_init: "));
 	switch(hp->tcvr_type) {
 	case none:
 		/* Cannot operate if we don't know the transceiver type! */
-		HMD(("AAIEEE no transceiver type, EAGAIN"));
+		HMD("AAIEEE no transceiver type, EAGAIN\n");
 		return -EAGAIN;
 
 	case internal:
 		/* Using the MII buffers. */
-		HMD(("internal, using MII, "));
+		HMD("internal, using MII\n");
 		hme_write32(hp, bregs + BMAC_XIFCFG, 0);
 		break;
 
 	case external:
 		/* Not using the MII, disable it. */
-		HMD(("external, disable MII, "));
+		HMD("external, disable MII\n");
 		hme_write32(hp, bregs + BMAC_XIFCFG, BIGMAC_XCFG_MIIDISAB);
 		break;
 	}
@@ -1454,18 +1399,16 @@ static int happy_meal_init(struct happy_meal *hp)
 		return -EAGAIN;
 
 	/* Reset the Happy Meal Big Mac transceiver and the receiver. */
-	HMD(("tx/rx reset, "));
+	HMD("tx/rx reset\n");
 	happy_meal_tx_reset(hp, bregs);
 	happy_meal_rx_reset(hp, bregs);
 
 	/* Set jam size and inter-packet gaps to reasonable defaults. */
-	HMD(("jsize/ipg1/ipg2, "));
 	hme_write32(hp, bregs + BMAC_JSIZE, DEFAULT_JAMSIZE);
 	hme_write32(hp, bregs + BMAC_IGAP1, DEFAULT_IPG1);
 	hme_write32(hp, bregs + BMAC_IGAP2, DEFAULT_IPG2);
 
 	/* Load up the MAC address and random seed. */
-	HMD(("rseed/macaddr, "));
 
 	/* The docs recommend to use the 10LSB of our MAC here. */
 	hme_write32(hp, bregs + BMAC_RSEED, ((e[5] | e[4]<<8)&0x3ff));
@@ -1474,7 +1417,6 @@ static int happy_meal_init(struct happy_meal *hp)
 	hme_write32(hp, bregs + BMAC_MACADDR1, ((e[2] << 8) | e[3]));
 	hme_write32(hp, bregs + BMAC_MACADDR0, ((e[0] << 8) | e[1]));
 
-	HMD(("htable, "));
 	if ((hp->dev->flags & IFF_ALLMULTI) ||
 	    (netdev_mc_count(hp->dev) > 64)) {
 		hme_write32(hp, bregs + BMAC_HTABLE0, 0xffff);
@@ -1504,9 +1446,9 @@ static int happy_meal_init(struct happy_meal *hp)
 	}
 
 	/* Set the RX and TX ring ptrs. */
-	HMD(("ring ptrs rxr[%08x] txr[%08x]\n",
-	     ((__u32)hp->hblock_dvma + hblock_offset(happy_meal_rxd, 0)),
-	     ((__u32)hp->hblock_dvma + hblock_offset(happy_meal_txd, 0))));
+	HMD("ring ptrs rxr[%08x] txr[%08x]\n",
+	    ((__u32)hp->hblock_dvma + hblock_offset(happy_meal_rxd, 0)),
+	    ((__u32)hp->hblock_dvma + hblock_offset(happy_meal_txd, 0)));
 	hme_write32(hp, erxregs + ERX_RING,
 		    ((__u32)hp->hblock_dvma + hblock_offset(happy_meal_rxd, 0)));
 	hme_write32(hp, etxregs + ETX_RING,
@@ -1524,9 +1466,6 @@ static int happy_meal_init(struct happy_meal *hp)
 			    | 0x4);
 
 	/* Set the supported burst sizes. */
-	HMD(("happy_meal_init: old[%08x] bursts<",
-	     hme_read32(hp, gregs + GREG_CFG)));
-
 #ifndef CONFIG_SPARC
 	/* It is always PCI and can handle 64byte bursts. */
 	hme_write32(hp, gregs + GREG_CFG, GREG_CFG_BURST64);
@@ -1554,34 +1493,35 @@ static int happy_meal_init(struct happy_meal *hp)
 		}
 #endif
 
-		HMD(("64>"));
+		bursts = "64";
 		hme_write32(hp, gregs + GREG_CFG, gcfg);
 	} else if (hp->happy_bursts & DMA_BURST32) {
-		HMD(("32>"));
+		bursts = "32";
 		hme_write32(hp, gregs + GREG_CFG, GREG_CFG_BURST32);
 	} else if (hp->happy_bursts & DMA_BURST16) {
-		HMD(("16>"));
+		bursts = "16";
 		hme_write32(hp, gregs + GREG_CFG, GREG_CFG_BURST16);
 	} else {
-		HMD(("XXX>"));
+		bursts = "XXX";
 		hme_write32(hp, gregs + GREG_CFG, 0);
 	}
 #endif /* CONFIG_SPARC */
 
+	HMD("old[%08x] bursts<%s>\n",
+	    hme_read32(hp, gregs + GREG_CFG), bursts);
+
 	/* Turn off interrupts we do not want to hear. */
-	HMD((", enable global interrupts, "));
 	hme_write32(hp, gregs + GREG_IMASK,
 		    (GREG_IMASK_GOTFRAME | GREG_IMASK_RCNTEXP |
 		     GREG_IMASK_SENTFRAME | GREG_IMASK_TXPERR));
 
 	/* Set the transmit ring buffer size. */
-	HMD(("tx rsize=%d oreg[%08x], ", (int)TX_RING_SIZE,
-	     hme_read32(hp, etxregs + ETX_RSIZE)));
+	HMD("tx rsize=%d oreg[%08x]\n", (int)TX_RING_SIZE,
+	    hme_read32(hp, etxregs + ETX_RSIZE));
 	hme_write32(hp, etxregs + ETX_RSIZE, (TX_RING_SIZE >> ETX_RSIZE_SHIFT) - 1);
 
 	/* Enable transmitter DVMA. */
-	HMD(("tx dma enable old[%08x], ",
-	     hme_read32(hp, etxregs + ETX_CFG)));
+	HMD("tx dma enable old[%08x]\n", hme_read32(hp, etxregs + ETX_CFG));
 	hme_write32(hp, etxregs + ETX_CFG,
 		    hme_read32(hp, etxregs + ETX_CFG) | ETX_CFG_DMAENABLE);
 
@@ -1590,21 +1530,23 @@ static int happy_meal_init(struct happy_meal *hp)
 	 * properly.  I cannot think of a sane way to provide complete
 	 * coverage for this hardware bug yet.
 	 */
-	HMD(("erx regs bug old[%08x]\n",
-	     hme_read32(hp, erxregs + ERX_CFG)));
+	HMD("erx regs bug old[%08x]\n",
+	    hme_read32(hp, erxregs + ERX_CFG));
 	hme_write32(hp, erxregs + ERX_CFG, ERX_CFG_DEFAULT(RX_OFFSET));
 	regtmp = hme_read32(hp, erxregs + ERX_CFG);
 	hme_write32(hp, erxregs + ERX_CFG, ERX_CFG_DEFAULT(RX_OFFSET));
 	if (hme_read32(hp, erxregs + ERX_CFG) != ERX_CFG_DEFAULT(RX_OFFSET)) {
-		printk(KERN_ERR "happy meal: Eieee, rx config register gets greasy fries.\n");
-		printk(KERN_ERR "happy meal: Trying to set %08x, reread gives %08x\n",
-		       ERX_CFG_DEFAULT(RX_OFFSET), regtmp);
+		netdev_err(hp->dev,
+			   "Eieee, rx config register gets greasy fries.\n");
+		netdev_err(hp->dev,
+			   "Trying to set %08x, reread gives %08x\n",
+			   ERX_CFG_DEFAULT(RX_OFFSET), regtmp);
 		/* XXX Should return failure here... */
 	}
 
 	/* Enable Big Mac hash table filter. */
-	HMD(("happy_meal_init: enable hash rx_cfg_old[%08x], ",
-	     hme_read32(hp, bregs + BMAC_RXCFG)));
+	HMD("enable hash rx_cfg_old[%08x]\n",
+	    hme_read32(hp, bregs + BMAC_RXCFG));
 	rxcfg = BIGMAC_RXCFG_HENABLE | BIGMAC_RXCFG_REJME;
 	if (hp->dev->flags & IFF_PROMISC)
 		rxcfg |= BIGMAC_RXCFG_PMISC;
@@ -1614,7 +1556,7 @@ static int happy_meal_init(struct happy_meal *hp)
 	udelay(10);
 
 	/* Ok, configure the Big Mac transmitter. */
-	HMD(("BIGMAC init, "));
+	HMD("BIGMAC init\n");
 	regtmp = 0;
 	if (hp->happy_flags & HFLAG_FULL)
 		regtmp |= BIGMAC_TXCFG_FULLDPLX;
@@ -1638,14 +1580,13 @@ static int happy_meal_init(struct happy_meal *hp)
 	if (hp->tcvr_type == external)
 		regtmp |= BIGMAC_XCFG_MIIDISAB;
 
-	HMD(("XIF config old[%08x], ",
-	     hme_read32(hp, bregs + BMAC_XIFCFG)));
+	HMD("XIF config old[%08x]\n", hme_read32(hp, bregs + BMAC_XIFCFG));
 	hme_write32(hp, bregs + BMAC_XIFCFG, regtmp);
 
 	/* Start things up. */
-	HMD(("tx old[%08x] and rx [%08x] ON!\n",
-	     hme_read32(hp, bregs + BMAC_TXCFG),
-	     hme_read32(hp, bregs + BMAC_RXCFG)));
+	HMD("tx old[%08x] and rx [%08x] ON!\n",
+	    hme_read32(hp, bregs + BMAC_TXCFG),
+	    hme_read32(hp, bregs + BMAC_RXCFG));
 
 	/* Set larger TX/RX size to allow for 802.1q */
 	hme_write32(hp, bregs + BMAC_TXMAX, ETH_FRAME_LEN + 8);
@@ -1735,25 +1676,26 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status)
 		      GREG_STAT_MIFIRQ | GREG_STAT_TXEACK | GREG_STAT_TXLERR |
 		      GREG_STAT_TXPERR | GREG_STAT_TXTERR | GREG_STAT_SLVERR |
 		      GREG_STAT_SLVPERR))
-		printk(KERN_ERR "%s: Error interrupt for happy meal, status = %08x\n",
-		       hp->dev->name, status);
+		netdev_err(hp->dev,
+			   "Error interrupt for happy meal, status = %08x\n",
+			   status);
 
 	if (status & GREG_STAT_RFIFOVF) {
 		/* Receive FIFO overflow is harmless and the hardware will take
 		   care of it, just some packets are lost. Who cares. */
-		printk(KERN_DEBUG "%s: Happy Meal receive FIFO overflow.\n", hp->dev->name);
+		netdev_dbg(hp->dev, "Happy Meal receive FIFO overflow.\n");
 	}
 
 	if (status & GREG_STAT_STSTERR) {
 		/* BigMAC SQE link test failed. */
-		printk(KERN_ERR "%s: Happy Meal BigMAC SQE test failed.\n", hp->dev->name);
+		netdev_err(hp->dev, "Happy Meal BigMAC SQE test failed.\n");
 		reset = 1;
 	}
 
 	if (status & GREG_STAT_TFIFO_UND) {
 		/* Transmit FIFO underrun, again DMA error likely. */
-		printk(KERN_ERR "%s: Happy Meal transmitter FIFO underrun, DMA error.\n",
-		       hp->dev->name);
+		netdev_err(hp->dev,
+			   "Happy Meal transmitter FIFO underrun, DMA error.\n");
 		reset = 1;
 	}
 
@@ -1761,7 +1703,7 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status)
 		/* Driver error, tried to transmit something larger
 		 * than ethernet max mtu.
 		 */
-		printk(KERN_ERR "%s: Happy Meal MAX Packet size error.\n", hp->dev->name);
+		netdev_err(hp->dev, "Happy Meal MAX Packet size error.\n");
 		reset = 1;
 	}
 
@@ -1771,21 +1713,16 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status)
 		 * faster than the interrupt handler could keep up
 		 * with.
 		 */
-		printk(KERN_INFO "%s: Happy Meal out of receive "
-		       "descriptors, packet dropped.\n",
-		       hp->dev->name);
+		netdev_info(hp->dev,
+			    "Happy Meal out of receive descriptors, packet dropped.\n");
 	}
 
 	if (status & (GREG_STAT_RXERR|GREG_STAT_RXPERR|GREG_STAT_RXTERR)) {
 		/* All sorts of DMA receive errors. */
-		printk(KERN_ERR "%s: Happy Meal rx DMA errors [ ", hp->dev->name);
-		if (status & GREG_STAT_RXERR)
-			printk("GenericError ");
-		if (status & GREG_STAT_RXPERR)
-			printk("ParityError ");
-		if (status & GREG_STAT_RXTERR)
-			printk("RxTagBotch ");
-		printk("]\n");
+		netdev_err(hp->dev, "Happy Meal rx DMA errors [ %s%s%s]\n",
+			   status & GREG_STAT_RXERR ? "GenericError " : "",
+			   status & GREG_STAT_RXPERR ? "ParityError " : "",
+			   status & GREG_STAT_RXTERR ? "RxTagBotch " : "");
 		reset = 1;
 	}
 
@@ -1793,29 +1730,24 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status)
 		/* Driver bug, didn't set EOP bit in tx descriptor given
 		 * to the happy meal.
 		 */
-		printk(KERN_ERR "%s: EOP not set in happy meal transmit descriptor!\n",
-		       hp->dev->name);
+		netdev_err(hp->dev,
+			   "EOP not set in happy meal transmit descriptor!\n");
 		reset = 1;
 	}
 
 	if (status & GREG_STAT_MIFIRQ) {
 		/* MIF signalled an interrupt, were we polling it? */
-		printk(KERN_ERR "%s: Happy Meal MIF interrupt.\n", hp->dev->name);
+		netdev_err(hp->dev, "Happy Meal MIF interrupt.\n");
 	}
 
 	if (status &
 	    (GREG_STAT_TXEACK|GREG_STAT_TXLERR|GREG_STAT_TXPERR|GREG_STAT_TXTERR)) {
 		/* All sorts of transmit DMA errors. */
-		printk(KERN_ERR "%s: Happy Meal tx DMA errors [ ", hp->dev->name);
-		if (status & GREG_STAT_TXEACK)
-			printk("GenericError ");
-		if (status & GREG_STAT_TXLERR)
-			printk("LateError ");
-		if (status & GREG_STAT_TXPERR)
-			printk("ParityError ");
-		if (status & GREG_STAT_TXTERR)
-			printk("TagBotch ");
-		printk("]\n");
+		netdev_err(hp->dev, "Happy Meal tx DMA errors [ %s%s%s%s]\n",
+			   status & GREG_STAT_TXEACK ? "GenericError " : "",
+			   status & GREG_STAT_TXLERR ? "LateError " : "",
+			   status & GREG_STAT_TXPERR ? "ParityError " : "",
+			   status & GREG_STAT_TXTERR ? "TagBotch " : "");
 		reset = 1;
 	}
 
@@ -1823,14 +1755,14 @@ static int happy_meal_is_not_so_happy(struct happy_meal *hp, u32 status)
 		/* Bus or parity error when cpu accessed happy meal registers
 		 * or it's internal FIFO's.  Should never see this.
 		 */
-		printk(KERN_ERR "%s: Happy Meal register access SBUS slave (%s) error.\n",
-		       hp->dev->name,
-		       (status & GREG_STAT_SLVPERR) ? "parity" : "generic");
+		netdev_err(hp->dev,
+			   "Happy Meal register access SBUS slave (%s) error.\n",
+			   (status & GREG_STAT_SLVPERR) ? "parity" : "generic");
 		reset = 1;
 	}
 
 	if (reset) {
-		printk(KERN_NOTICE "%s: Resetting...\n", hp->dev->name);
+		netdev_notice(hp->dev, "Resetting...\n");
 		happy_meal_init(hp);
 		return 1;
 	}
@@ -1842,22 +1774,22 @@ static void happy_meal_mif_interrupt(struct happy_meal *hp)
 {
 	void __iomem *tregs = hp->tcvregs;
 
-	printk(KERN_INFO "%s: Link status change.\n", hp->dev->name);
+	netdev_info(hp->dev, "Link status change.\n");
 	hp->sw_bmcr = happy_meal_tcvr_read(hp, tregs, MII_BMCR);
 	hp->sw_lpa = happy_meal_tcvr_read(hp, tregs, MII_LPA);
 
 	/* Use the fastest transmission protocol possible. */
 	if (hp->sw_lpa & LPA_100FULL) {
-		printk(KERN_INFO "%s: Switching to 100Mbps at full duplex.", hp->dev->name);
+		netdev_info(hp->dev, "Switching to 100Mbps at full duplex.\n");
 		hp->sw_bmcr |= (BMCR_FULLDPLX | BMCR_SPEED100);
 	} else if (hp->sw_lpa & LPA_100HALF) {
-		printk(KERN_INFO "%s: Switching to 100MBps at half duplex.", hp->dev->name);
+		netdev_info(hp->dev, "Switching to 100MBps at half duplex.\n");
 		hp->sw_bmcr |= BMCR_SPEED100;
 	} else if (hp->sw_lpa & LPA_10FULL) {
-		printk(KERN_INFO "%s: Switching to 10MBps at full duplex.", hp->dev->name);
+		netdev_info(hp->dev, "Switching to 10MBps at full duplex.\n");
 		hp->sw_bmcr |= BMCR_FULLDPLX;
 	} else {
-		printk(KERN_INFO "%s: Using 10Mbps at half duplex.", hp->dev->name);
+		netdev_info(hp->dev, "Using 10Mbps at half duplex.\n");
 	}
 	happy_meal_tcvr_write(hp, tregs, MII_BMCR, hp->sw_bmcr);
 
@@ -1865,12 +1797,6 @@ static void happy_meal_mif_interrupt(struct happy_meal *hp)
 	happy_meal_poll_stop(hp, tregs);
 }
 
-#ifdef TXDEBUG
-#define TXD(x) printk x
-#else
-#define TXD(x)
-#endif
-
 /* hp->happy_lock must be held */
 static void happy_meal_tx(struct happy_meal *hp)
 {
@@ -1880,13 +1806,12 @@ static void happy_meal_tx(struct happy_meal *hp)
 	int elem;
 
 	elem = hp->tx_old;
-	TXD(("TX<"));
 	while (elem != hp->tx_new) {
 		struct sk_buff *skb;
 		u32 flags, dma_addr, dma_len;
 		int frag;
 
-		TXD(("[%d]", elem));
+		netdev_vdbg(hp->dev, "TX[%d]\n", elem);
 		this = &txbase[elem];
 		flags = hme_read_desc32(hp, &this->tx_flags);
 		if (flags & TXFLAG_OWN)
@@ -1922,19 +1847,12 @@ static void happy_meal_tx(struct happy_meal *hp)
 		dev->stats.tx_packets++;
 	}
 	hp->tx_old = elem;
-	TXD((">"));
 
 	if (netif_queue_stopped(dev) &&
 	    TX_BUFFS_AVAIL(hp) > (MAX_SKB_FRAGS + 1))
 		netif_wake_queue(dev);
 }
 
-#ifdef RXDEBUG
-#define RXD(x) printk x
-#else
-#define RXD(x)
-#endif
-
 /* Originally I used to handle the allocation failure by just giving back just
  * that one ring buffer to the happy meal.  Problem is that usually when that
  * condition is triggered, the happy meal expects you to do something reasonable
@@ -1951,7 +1869,6 @@ static void happy_meal_rx(struct happy_meal *hp, struct net_device *dev)
 	int elem = hp->rx_new, drops = 0;
 	u32 flags;
 
-	RXD(("RX<"));
 	this = &rxbase[elem];
 	while (!((flags = hme_read_desc32(hp, &this->rx_flags)) & RXFLAG_OWN)) {
 		struct sk_buff *skb;
@@ -1959,11 +1876,9 @@ static void happy_meal_rx(struct happy_meal *hp, struct net_device *dev)
 		u16 csum = flags & RXFLAG_CSUM;
 		u32 dma_addr = hme_read_desc32(hp, &this->rx_addr);
 
-		RXD(("[%d ", elem));
-
 		/* Check for errors. */
 		if ((len < ETH_ZLEN) || (flags & RXFLAG_OVERFLOW)) {
-			RXD(("ERR(%08x)]", flags));
+			netdev_vdbg(dev, "RX[%d ERR(%08x)]", elem, flags);
 			dev->stats.rx_errors++;
 			if (len < ETH_ZLEN)
 				dev->stats.rx_length_errors++;
@@ -2035,7 +1950,7 @@ static void happy_meal_rx(struct happy_meal *hp, struct net_device *dev)
 		skb->csum = csum_unfold(~(__force __sum16)htons(csum));
 		skb->ip_summed = CHECKSUM_COMPLETE;
 
-		RXD(("len=%d csum=%4x]", len, csum));
+		netdev_vdbg(dev, "RX[%d len=%d csum=%4x]", elem, len, csum);
 		skb->protocol = eth_type_trans(skb, dev);
 		netif_rx(skb);
 
@@ -2047,8 +1962,7 @@ static void happy_meal_rx(struct happy_meal *hp, struct net_device *dev)
 	}
 	hp->rx_new = elem;
 	if (drops)
-		printk(KERN_INFO "%s: Memory squeeze, deferring packet.\n", hp->dev->name);
-	RXD((">"));
+		netdev_info(hp->dev, "Memory squeeze, deferring packet.\n");
 }
 
 static irqreturn_t happy_meal_interrupt(int irq, void *dev_id)
@@ -2057,32 +1971,25 @@ static irqreturn_t happy_meal_interrupt(int irq, void *dev_id)
 	struct happy_meal *hp  = netdev_priv(dev);
 	u32 happy_status       = hme_read32(hp, hp->gregs + GREG_STAT);
 
-	HMD(("happy_meal_interrupt: status=%08x ", happy_status));
+	HMD("status=%08x\n", happy_status);
 
 	spin_lock(&hp->happy_lock);
 
 	if (happy_status & GREG_STAT_ERRORS) {
-		HMD(("ERRORS "));
 		if (happy_meal_is_not_so_happy(hp, /* un- */ happy_status))
 			goto out;
 	}
 
-	if (happy_status & GREG_STAT_MIFIRQ) {
-		HMD(("MIFIRQ "));
+	if (happy_status & GREG_STAT_MIFIRQ)
 		happy_meal_mif_interrupt(hp);
-	}
 
-	if (happy_status & GREG_STAT_TXALL) {
-		HMD(("TXALL "));
+	if (happy_status & GREG_STAT_TXALL)
 		happy_meal_tx(hp);
-	}
 
-	if (happy_status & GREG_STAT_RXTOHOST) {
-		HMD(("RXTOHOST "));
+	if (happy_status & GREG_STAT_RXTOHOST)
 		happy_meal_rx(hp, dev);
-	}
 
-	HMD(("done\n"));
+	HMD("done\n");
 out:
 	spin_unlock(&hp->happy_lock);
 
@@ -2100,7 +2007,7 @@ static irqreturn_t quattro_sbus_interrupt(int irq, void *cookie)
 		struct happy_meal *hp  = netdev_priv(dev);
 		u32 happy_status       = hme_read32(hp, hp->gregs + GREG_STAT);
 
-		HMD(("quattro_interrupt: status=%08x ", happy_status));
+		HMD("status=%08x\n", happy_status);
 
 		if (!(happy_status & (GREG_STAT_ERRORS |
 				      GREG_STAT_MIFIRQ |
@@ -2110,31 +2017,23 @@ static irqreturn_t quattro_sbus_interrupt(int irq, void *cookie)
 
 		spin_lock(&hp->happy_lock);
 
-		if (happy_status & GREG_STAT_ERRORS) {
-			HMD(("ERRORS "));
+		if (happy_status & GREG_STAT_ERRORS)
 			if (happy_meal_is_not_so_happy(hp, happy_status))
 				goto next;
-		}
 
-		if (happy_status & GREG_STAT_MIFIRQ) {
-			HMD(("MIFIRQ "));
+		if (happy_status & GREG_STAT_MIFIRQ)
 			happy_meal_mif_interrupt(hp);
-		}
 
-		if (happy_status & GREG_STAT_TXALL) {
-			HMD(("TXALL "));
+		if (happy_status & GREG_STAT_TXALL)
 			happy_meal_tx(hp);
-		}
 
-		if (happy_status & GREG_STAT_RXTOHOST) {
-			HMD(("RXTOHOST "));
+		if (happy_status & GREG_STAT_RXTOHOST)
 			happy_meal_rx(hp, dev);
-		}
 
 	next:
 		spin_unlock(&hp->happy_lock);
 	}
-	HMD(("done\n"));
+	HMD("done\n");
 
 	return IRQ_HANDLED;
 }
@@ -2145,8 +2044,6 @@ static int happy_meal_open(struct net_device *dev)
 	struct happy_meal *hp = netdev_priv(dev);
 	int res;
 
-	HMD(("happy_meal_open: "));
-
 	/* On SBUS Quattro QFE cards, all hme interrupts are concentrated
 	 * into a single source which we register handling at probe time.
 	 */
@@ -2154,15 +2051,14 @@ static int happy_meal_open(struct net_device *dev)
 		res = request_irq(hp->irq, happy_meal_interrupt, IRQF_SHARED,
 				  dev->name, dev);
 		if (res) {
-			HMD(("EAGAIN\n"));
-			printk(KERN_ERR "happy_meal(SBUS): Can't order irq %d to go.\n",
-			       hp->irq);
+			HMD("EAGAIN\n");
+			netdev_err(dev, "Can't order irq %d to go.\n", hp->irq);
 
 			return -EAGAIN;
 		}
 	}
 
-	HMD(("to happy_meal_init\n"));
+	HMD("to happy_meal_init\n");
 
 	spin_lock_irq(&hp->happy_lock);
 	res = happy_meal_init(hp);
@@ -2196,22 +2092,16 @@ static int happy_meal_close(struct net_device *dev)
 	return 0;
 }
 
-#ifdef SXDEBUG
-#define SXD(x) printk x
-#else
-#define SXD(x)
-#endif
-
 static void happy_meal_tx_timeout(struct net_device *dev, unsigned int txqueue)
 {
 	struct happy_meal *hp = netdev_priv(dev);
 
-	printk (KERN_ERR "%s: transmit timed out, resetting\n", dev->name);
+	netdev_err(dev, "transmit timed out, resetting\n");
 	tx_dump_log();
-	printk (KERN_ERR "%s: Happy Status %08x TX[%08x:%08x]\n", dev->name,
-		hme_read32(hp, hp->gregs + GREG_STAT),
-		hme_read32(hp, hp->etxregs + ETX_CFG),
-		hme_read32(hp, hp->bigmacregs + BMAC_TXCFG));
+	netdev_err(dev, "Happy Status %08x TX[%08x:%08x]\n",
+		   hme_read32(hp, hp->gregs + GREG_STAT),
+		   hme_read32(hp, hp->etxregs + ETX_CFG),
+		   hme_read32(hp, hp->bigmacregs + BMAC_TXCFG));
 
 	spin_lock_irq(&hp->happy_lock);
 	happy_meal_init(hp);
@@ -2261,13 +2151,12 @@ static netdev_tx_t happy_meal_start_xmit(struct sk_buff *skb,
 	if (TX_BUFFS_AVAIL(hp) <= (skb_shinfo(skb)->nr_frags + 1)) {
 		netif_stop_queue(dev);
 		spin_unlock_irq(&hp->happy_lock);
-		printk(KERN_ERR "%s: BUG! Tx Ring full when queue awake!\n",
-		       dev->name);
+		netdev_err(dev, "BUG! Tx Ring full when queue awake!\n");
 		return NETDEV_TX_BUSY;
 	}
 
 	entry = hp->tx_new;
-	SXD(("SX<l[%d]e[%d]>", len, entry));
+	netdev_vdbg(dev, "SX<l[%d]e[%d]>\n", skb->len, entry);
 	hp->tx_skbs[entry] = skb;
 
 	if (skb_shinfo(skb)->nr_frags == 0) {
@@ -2467,11 +2356,10 @@ static void hme_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info
 {
 	struct happy_meal *hp = netdev_priv(dev);
 
-	strlcpy(info->driver, "sunhme", sizeof(info->driver));
-	strlcpy(info->version, "2.02", sizeof(info->version));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 	if (hp->happy_flags & HFLAG_PCI) {
 		struct pci_dev *pdev = hp->happy_dev;
-		strlcpy(info->bus_info, pci_name(pdev), sizeof(info->bus_info));
+		strscpy(info->bus_info, pci_name(pdev), sizeof(info->bus_info));
 	}
 #ifdef CONFIG_SBUS
 	else {
@@ -2504,8 +2392,6 @@ static const struct ethtool_ops hme_ethtool_ops = {
 	.set_link_ksettings	= hme_set_link_ksettings,
 };
 
-static int hme_version_printed;
-
 #ifdef CONFIG_SBUS
 /* Given a happy meal sbus device, find it's quattro parent.
  * If none exist, allocate and return a new one.
@@ -2523,19 +2409,15 @@ static struct quattro *quattro_sbus_find(struct platform_device *child)
 	if (qp)
 		return qp;
 
-	qp = kmalloc(sizeof(struct quattro), GFP_KERNEL);
-	if (qp != NULL) {
-		int i;
-
-		for (i = 0; i < 4; i++)
-			qp->happy_meals[i] = NULL;
+	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+	if (!qp)
+		return NULL;
 
-		qp->quattro_dev = child;
-		qp->next = qfe_sbus_list;
-		qfe_sbus_list = qp;
+	qp->quattro_dev = child;
+	qp->next = qfe_sbus_list;
+	qfe_sbus_list = qp;
 
-		platform_set_drvdata(op, qp);
-	}
+	platform_set_drvdata(op, qp);
 	return qp;
 }
 
@@ -2563,8 +2445,9 @@ static int __init quattro_sbus_register_irqs(void)
 				  IRQF_SHARED, "Quattro",
 				  qp);
 		if (err != 0) {
-			printk(KERN_ERR "Quattro HME: IRQ registration "
-			       "error %d.\n", err);
+			dev_err(&op->dev,
+				"Quattro HME: IRQ registration error %d.\n",
+				err);
 			return err;
 		}
 	}
@@ -2595,30 +2478,33 @@ static void quattro_sbus_free_irqs(void)
 #ifdef CONFIG_PCI
 static struct quattro *quattro_pci_find(struct pci_dev *pdev)
 {
+	int i;
 	struct pci_dev *bdev = pdev->bus->self;
 	struct quattro *qp;
 
-	if (!bdev) return NULL;
+	if (!bdev)
+		return ERR_PTR(-ENODEV);
+
 	for (qp = qfe_pci_list; qp != NULL; qp = qp->next) {
 		struct pci_dev *qpdev = qp->quattro_dev;
 
 		if (qpdev == bdev)
 			return qp;
 	}
+
 	qp = kmalloc(sizeof(struct quattro), GFP_KERNEL);
-	if (qp != NULL) {
-		int i;
+	if (!qp)
+		return ERR_PTR(-ENOMEM);
 
-		for (i = 0; i < 4; i++)
-			qp->happy_meals[i] = NULL;
+	for (i = 0; i < 4; i++)
+		qp->happy_meals[i] = NULL;
 
-		qp->quattro_dev = bdev;
-		qp->next = qfe_pci_list;
-		qfe_pci_list = qp;
+	qp->quattro_dev = bdev;
+	qp->next = qfe_pci_list;
+	qfe_pci_list = qp;
 
-		/* No range tricks necessary on PCI. */
-		qp->nranges = 0;
-	}
+	/* No range tricks necessary on PCI. */
+	qp->nranges = 0;
 	return qp;
 }
 #endif /* CONFIG_PCI */
@@ -2668,9 +2554,6 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe)
 		goto err_out;
 	SET_NETDEV_DEV(dev, &op->dev);
 
-	if (hme_version_printed++ == 0)
-		printk(KERN_INFO "%s", version);
-
 	/* If user did not specify a MAC address specifically, use
 	 * the Quattro local-mac-address property...
 	 */
@@ -2712,35 +2595,35 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe)
 	hp->gregs = of_ioremap(&op->resource[0], 0,
 			       GREG_REG_SIZE, "HME Global Regs");
 	if (!hp->gregs) {
-		printk(KERN_ERR "happymeal: Cannot map global registers.\n");
+		dev_err(&op->dev, "Cannot map global registers.\n");
 		goto err_out_free_netdev;
 	}
 
 	hp->etxregs = of_ioremap(&op->resource[1], 0,
 				 ETX_REG_SIZE, "HME TX Regs");
 	if (!hp->etxregs) {
-		printk(KERN_ERR "happymeal: Cannot map MAC TX registers.\n");
+		dev_err(&op->dev, "Cannot map MAC TX registers.\n");
 		goto err_out_iounmap;
 	}
 
 	hp->erxregs = of_ioremap(&op->resource[2], 0,
 				 ERX_REG_SIZE, "HME RX Regs");
 	if (!hp->erxregs) {
-		printk(KERN_ERR "happymeal: Cannot map MAC RX registers.\n");
+		dev_err(&op->dev, "Cannot map MAC RX registers.\n");
 		goto err_out_iounmap;
 	}
 
 	hp->bigmacregs = of_ioremap(&op->resource[3], 0,
 				    BMAC_REG_SIZE, "HME BIGMAC Regs");
 	if (!hp->bigmacregs) {
-		printk(KERN_ERR "happymeal: Cannot map BIGMAC registers.\n");
+		dev_err(&op->dev, "Cannot map BIGMAC registers.\n");
 		goto err_out_iounmap;
 	}
 
 	hp->tcvregs = of_ioremap(&op->resource[4], 0,
 				 TCVR_REG_SIZE, "HME Tranceiver Regs");
 	if (!hp->tcvregs) {
-		printk(KERN_ERR "happymeal: Cannot map TCVR registers.\n");
+		dev_err(&op->dev, "Cannot map TCVR registers.\n");
 		goto err_out_iounmap;
 	}
 
@@ -2807,21 +2690,19 @@ static int happy_meal_sbus_probe_one(struct platform_device *op, int is_qfe)
 
 	err = register_netdev(hp->dev);
 	if (err) {
-		printk(KERN_ERR "happymeal: Cannot register net device, "
-		       "aborting.\n");
+		dev_err(&op->dev, "Cannot register net device, aborting.\n");
 		goto err_out_free_coherent;
 	}
 
 	platform_set_drvdata(op, hp);
 
 	if (qfe_slot != -1)
-		printk(KERN_INFO "%s: Quattro HME slot %d (SBUS) 10/100baseT Ethernet ",
-		       dev->name, qfe_slot);
+		netdev_info(dev,
+			    "Quattro HME slot %d (SBUS) 10/100baseT Ethernet %pM\n",
+			    qfe_slot, dev->dev_addr);
 	else
-		printk(KERN_INFO "%s: HAPPY MEAL (SBUS) 10/100baseT Ethernet ",
-		       dev->name);
-
-	printk("%pM\n", dev->dev_addr);
+		netdev_info(dev, "HAPPY MEAL (SBUS) 10/100baseT Ethernet %pM\n",
+			    dev->dev_addr);
 
 	return 0;
 
@@ -2949,7 +2830,7 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 	struct happy_meal *hp;
 	struct net_device *dev;
 	void __iomem *hpreg_base;
-	unsigned long hpreg_res;
+	struct resource *hpreg_res;
 	int i, qfe_slot = -1;
 	char prom_name[64];
 	u8 addr[ETH_ALEN];
@@ -2966,32 +2847,33 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 		strcpy(prom_name, "SUNW,hme");
 #endif
 
-	err = -ENODEV;
-
-	if (pci_enable_device(pdev))
+	err = pcim_enable_device(pdev);
+	if (err)
 		goto err_out;
 	pci_set_master(pdev);
 
 	if (!strcmp(prom_name, "SUNW,qfe") || !strcmp(prom_name, "qfe")) {
 		qp = quattro_pci_find(pdev);
-		if (qp == NULL)
+		if (IS_ERR(qp)) {
+			err = PTR_ERR(qp);
 			goto err_out;
+		}
+
 		for (qfe_slot = 0; qfe_slot < 4; qfe_slot++)
-			if (qp->happy_meals[qfe_slot] == NULL)
+			if (!qp->happy_meals[qfe_slot])
 				break;
+
 		if (qfe_slot == 4)
 			goto err_out;
 	}
 
-	dev = alloc_etherdev(sizeof(struct happy_meal));
-	err = -ENOMEM;
-	if (!dev)
+	dev = devm_alloc_etherdev(&pdev->dev, sizeof(struct happy_meal));
+	if (!dev) {
+		err = -ENOMEM;
 		goto err_out;
+	}
 	SET_NETDEV_DEV(dev, &pdev->dev);
 
-	if (hme_version_printed++ == 0)
-		printk(KERN_INFO "%s", version);
-
 	hp = netdev_priv(dev);
 
 	hp->happy_dev = pdev;
@@ -3005,21 +2887,26 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 		qp->happy_meals[qfe_slot] = dev;
 	}
 
-	hpreg_res = pci_resource_start(pdev, 0);
-	err = -ENODEV;
+	err = -EINVAL;
 	if ((pci_resource_flags(pdev, 0) & IORESOURCE_IO) != 0) {
-		printk(KERN_ERR "happymeal(PCI): Cannot find proper PCI device base address.\n");
+		dev_err(&pdev->dev,
+			"Cannot find proper PCI device base address.\n");
 		goto err_out_clear_quattro;
 	}
-	if (pci_request_regions(pdev, DRV_NAME)) {
-		printk(KERN_ERR "happymeal(PCI): Cannot obtain PCI resources, "
-		       "aborting.\n");
+
+	hpreg_res = devm_request_region(&pdev->dev, pci_resource_start(pdev, 0),
+					pci_resource_len(pdev, 0), DRV_NAME);
+	if (IS_ERR(hpreg_res)) {
+		err = PTR_ERR(hpreg_res);
+		dev_err(&pdev->dev, "Cannot obtain PCI resources, aborting.\n");
 		goto err_out_clear_quattro;
 	}
 
-	if ((hpreg_base = ioremap(hpreg_res, 0x8000)) == NULL) {
-		printk(KERN_ERR "happymeal(PCI): Unable to remap card memory.\n");
-		goto err_out_free_res;
+	hpreg_base = pcim_iomap(pdev, 0, 0x8000);
+	if (!hpreg_base) {
+		err = -ENOMEM;
+		dev_err(&pdev->dev, "Unable to remap card memory.\n");
+		goto err_out_clear_quattro;
 	}
 
 	for (i = 0; i < 6; i++) {
@@ -3085,11 +2972,12 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 	hp->happy_bursts = DMA_BURSTBITS;
 #endif
 
-	hp->happy_block = dma_alloc_coherent(&pdev->dev, PAGE_SIZE,
-					     &hp->hblock_dvma, GFP_KERNEL);
-	err = -ENODEV;
-	if (!hp->happy_block)
-		goto err_out_iounmap;
+	hp->happy_block = dmam_alloc_coherent(&pdev->dev, PAGE_SIZE,
+					      &hp->hblock_dvma, GFP_KERNEL);
+	if (!hp->happy_block) {
+		err = -ENOMEM;
+		goto err_out_clear_quattro;
+	}
 
 	hp->linkcheck = 0;
 	hp->timer_state = asleep;
@@ -3123,11 +3011,10 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 	happy_meal_set_initial_advertisement(hp);
 	spin_unlock_irq(&hp->happy_lock);
 
-	err = register_netdev(hp->dev);
+	err = devm_register_netdev(&pdev->dev, dev);
 	if (err) {
-		printk(KERN_ERR "happymeal(PCI): Cannot register net device, "
-		       "aborting.\n");
-		goto err_out_free_coherent;
+		dev_err(&pdev->dev, "Cannot register net device, aborting.\n");
+		goto err_out_clear_quattro;
 	}
 
 	pci_set_drvdata(pdev, hp);
@@ -3140,61 +3027,30 @@ static int happy_meal_pci_probe(struct pci_dev *pdev,
 			int i = simple_strtoul(dev->name + 3, NULL, 10);
 			sprintf(prom_name, "-%d", i + 3);
 		}
-		printk(KERN_INFO "%s%s: Quattro HME (PCI/CheerIO) 10/100baseT Ethernet ", dev->name, prom_name);
-		if (qpdev->vendor == PCI_VENDOR_ID_DEC &&
-		    qpdev->device == PCI_DEVICE_ID_DEC_21153)
-			printk("DEC 21153 PCI Bridge\n");
-		else
-			printk("unknown bridge %04x.%04x\n",
-				qpdev->vendor, qpdev->device);
+		netdev_info(dev,
+			    "%s: Quattro HME (PCI/CheerIO) 10/100baseT Ethernet bridge %04x.%04x\n",
+			    prom_name, qpdev->vendor, qpdev->device);
 	}
 
 	if (qfe_slot != -1)
-		printk(KERN_INFO "%s: Quattro HME slot %d (PCI/CheerIO) 10/100baseT Ethernet ",
-		       dev->name, qfe_slot);
+		netdev_info(dev,
+			    "Quattro HME slot %d (PCI/CheerIO) 10/100baseT Ethernet %pM\n",
+			    qfe_slot, dev->dev_addr);
 	else
-		printk(KERN_INFO "%s: HAPPY MEAL (PCI/CheerIO) 10/100BaseT Ethernet ",
-		       dev->name);
-
-	printk("%pM\n", dev->dev_addr);
+		netdev_info(dev,
+			    "HAPPY MEAL (PCI/CheerIO) 10/100BaseT Ethernet %pM\n",
+			    dev->dev_addr);
 
 	return 0;
 
-err_out_free_coherent:
-	dma_free_coherent(hp->dma_dev, PAGE_SIZE,
-			  hp->happy_block, hp->hblock_dvma);
-
-err_out_iounmap:
-	iounmap(hp->gregs);
-
-err_out_free_res:
-	pci_release_regions(pdev);
-
 err_out_clear_quattro:
 	if (qp != NULL)
 		qp->happy_meals[qfe_slot] = NULL;
 
-	free_netdev(dev);
-
 err_out:
 	return err;
 }
 
-static void happy_meal_pci_remove(struct pci_dev *pdev)
-{
-	struct happy_meal *hp = pci_get_drvdata(pdev);
-	struct net_device *net_dev = hp->dev;
-
-	unregister_netdev(net_dev);
-
-	dma_free_coherent(hp->dma_dev, PAGE_SIZE,
-			  hp->happy_block, hp->hblock_dvma);
-	iounmap(hp->gregs);
-	pci_release_regions(hp->happy_dev);
-
-	free_netdev(net_dev);
-}
-
 static const struct pci_device_id happymeal_pci_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_SUN, PCI_DEVICE_ID_SUN_HAPPYMEAL) },
 	{ }			/* Terminating entry */
@@ -3206,7 +3062,6 @@ static struct pci_driver hme_pci_driver = {
 	.name		= "hme",
 	.id_table	= happymeal_pci_ids,
 	.probe		= happy_meal_pci_probe,
-	.remove		= happy_meal_pci_remove,
 };
 
 static int __init happy_meal_pci_init(void)
diff --git a/drivers/net/ethernet/sun/sunqe.c b/drivers/net/ethernet/sun/sunqe.c
index efe0d33f6024..6418fcc3139f 100644
--- a/drivers/net/ethernet/sun/sunqe.c
+++ b/drivers/net/ethernet/sun/sunqe.c
@@ -684,8 +684,8 @@ static void qe_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 	struct sunqe *qep = netdev_priv(dev);
 	struct platform_device *op;
 
-	strlcpy(info->driver, "sunqe", sizeof(info->driver));
-	strlcpy(info->version, "3.0", sizeof(info->version));
+	strscpy(info->driver, "sunqe", sizeof(info->driver));
+	strscpy(info->version, "3.0", sizeof(info->version));
 
 	op = qep->op;
 	regs = of_get_property(op->dev.of_node, "reg", NULL);
diff --git a/drivers/net/ethernet/sun/sunvnet.c b/drivers/net/ethernet/sun/sunvnet.c
index da8119625cf3..acda6cbd0238 100644
--- a/drivers/net/ethernet/sun/sunvnet.c
+++ b/drivers/net/ethernet/sun/sunvnet.c
@@ -60,8 +60,8 @@ static struct vio_version vnet_versions[] = {
 static void vnet_get_drvinfo(struct net_device *dev,
 			     struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_MODULE_VERSION, sizeof(info->version));
 }
 
 static u32 vnet_get_msglevel(struct net_device *dev)
@@ -467,8 +467,7 @@ static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id)
 	if (err)
 		goto err_out_free_port;
 
-	netif_napi_add(port->vp->dev, &port->napi, sunvnet_poll_common,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(port->vp->dev, &port->napi, sunvnet_poll_common);
 
 	INIT_HLIST_NODE(&port->hash);
 	INIT_LIST_HEAD(&port->list);
diff --git a/drivers/net/ethernet/sunplus/spl2sw_driver.c b/drivers/net/ethernet/sunplus/spl2sw_driver.c
index 546206640492..9be585237277 100644
--- a/drivers/net/ethernet/sunplus/spl2sw_driver.c
+++ b/drivers/net/ethernet/sunplus/spl2sw_driver.c
@@ -62,7 +62,8 @@ static int spl2sw_ethernet_stop(struct net_device *ndev)
 	return 0;
 }
 
-static int spl2sw_ethernet_start_xmit(struct sk_buff *skb, struct net_device *ndev)
+static netdev_tx_t spl2sw_ethernet_start_xmit(struct sk_buff *skb,
+					      struct net_device *ndev)
 {
 	struct spl2sw_mac *mac = netdev_priv(ndev);
 	struct spl2sw_common *comm = mac->comm;
@@ -248,8 +249,8 @@ static int spl2sw_nvmem_get_mac_address(struct device *dev, struct device_node *
 
 	/* Check if mac address is valid */
 	if (!is_valid_ether_addr(mac)) {
-		kfree(mac);
 		dev_info(dev, "Invalid mac address in nvmem (%pM)!\n", mac);
+		kfree(mac);
 		return -EINVAL;
 	}
 
@@ -492,7 +493,7 @@ static int spl2sw_probe(struct platform_device *pdev)
 	}
 
 	/* Add and enable napi. */
-	netif_napi_add(ndev, &comm->rx_napi, spl2sw_rx_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &comm->rx_napi, spl2sw_rx_poll);
 	napi_enable(&comm->rx_napi);
 	netif_napi_add_tx(ndev, &comm->tx_napi, spl2sw_tx_poll);
 	napi_enable(&comm->tx_napi);
diff --git a/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c b/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c
index 5c9b6c90942b..f8e133604146 100644
--- a/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c
+++ b/drivers/net/ethernet/synopsys/dwc-xlgmac-common.c
@@ -54,8 +54,8 @@ static void xlgmac_default_config(struct xlgmac_pdata *pdata)
 	pdata->phy_speed = SPEED_25000;
 	pdata->sysclk_rate = XLGMAC_SYSCLOCK;
 
-	strlcpy(pdata->drv_name, XLGMAC_DRV_NAME, sizeof(pdata->drv_name));
-	strlcpy(pdata->drv_ver, XLGMAC_DRV_VERSION, sizeof(pdata->drv_ver));
+	strscpy(pdata->drv_name, XLGMAC_DRV_NAME, sizeof(pdata->drv_name));
+	strscpy(pdata->drv_ver, XLGMAC_DRV_VERSION, sizeof(pdata->drv_ver));
 }
 
 static void xlgmac_init_all_ops(struct xlgmac_pdata *pdata)
diff --git a/drivers/net/ethernet/synopsys/dwc-xlgmac-ethtool.c b/drivers/net/ethernet/synopsys/dwc-xlgmac-ethtool.c
index 49f8c6be9459..e794da727fe0 100644
--- a/drivers/net/ethernet/synopsys/dwc-xlgmac-ethtool.c
+++ b/drivers/net/ethernet/synopsys/dwc-xlgmac-ethtool.c
@@ -102,9 +102,9 @@ static void xlgmac_ethtool_get_drvinfo(struct net_device *netdev,
 	u32 ver = pdata->hw_feat.version;
 	u32 snpsver, devid, userver;
 
-	strlcpy(drvinfo->driver, pdata->drv_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, pdata->drv_ver, sizeof(drvinfo->version));
-	strlcpy(drvinfo->bus_info, dev_name(pdata->dev),
+	strscpy(drvinfo->driver, pdata->drv_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, pdata->drv_ver, sizeof(drvinfo->version));
+	strscpy(drvinfo->bus_info, dev_name(pdata->dev),
 		sizeof(drvinfo->bus_info));
 	/* S|SNPSVER: Synopsys-defined Version
 	 * D|DEVID: Indicates the Device family
diff --git a/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c b/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c
index e54ce73396ee..36b948820c1e 100644
--- a/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c
+++ b/drivers/net/ethernet/synopsys/dwc-xlgmac-net.c
@@ -419,15 +419,14 @@ static void xlgmac_napi_enable(struct xlgmac_pdata *pdata, unsigned int add)
 		for (i = 0; i < pdata->channel_count; i++, channel++) {
 			if (add)
 				netif_napi_add(pdata->netdev, &channel->napi,
-					       xlgmac_one_poll,
-					       NAPI_POLL_WEIGHT);
+					       xlgmac_one_poll);
 
 			napi_enable(&channel->napi);
 		}
 	} else {
 		if (add)
 			netif_napi_add(pdata->netdev, &pdata->napi,
-				       xlgmac_all_poll, NAPI_POLL_WEIGHT);
+				       xlgmac_all_poll);
 
 		napi_enable(&pdata->napi);
 	}
diff --git a/drivers/net/ethernet/tehuti/tehuti.c b/drivers/net/ethernet/tehuti/tehuti.c
index 985073eba3bd..ca409515ead5 100644
--- a/drivers/net/ethernet/tehuti/tehuti.c
+++ b/drivers/net/ethernet/tehuti/tehuti.c
@@ -1994,7 +1994,7 @@ bdx_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		priv->nic = nic;
 		priv->msg_enable = BDX_DEF_MSG_ENABLE;
 
-		netif_napi_add(ndev, &priv->napi, bdx_poll, 64);
+		netif_napi_add(ndev, &priv->napi, bdx_poll);
 
 		if ((readl(nic->regs + FPGA_VER) & 0xFFF) == 308) {
 			DBG("HW statistics not supported\n");
@@ -2133,10 +2133,10 @@ bdx_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
 {
 	struct bdx_priv *priv = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, BDX_DRV_NAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, BDX_DRV_VERSION, sizeof(drvinfo->version));
-	strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version));
-	strlcpy(drvinfo->bus_info, pci_name(priv->pdev),
+	strscpy(drvinfo->driver, BDX_DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, BDX_DRV_VERSION, sizeof(drvinfo->version));
+	strscpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version));
+	strscpy(drvinfo->bus_info, pci_name(priv->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/ti/Kconfig b/drivers/net/ethernet/ti/Kconfig
index fb30bc5d56cb..fce06663e1e1 100644
--- a/drivers/net/ethernet/ti/Kconfig
+++ b/drivers/net/ethernet/ti/Kconfig
@@ -33,6 +33,7 @@ config TI_DAVINCI_MDIO
 	tristate "TI DaVinci MDIO Support"
 	depends on ARCH_DAVINCI || ARCH_OMAP2PLUS || ARCH_KEYSTONE || ARCH_K3 || COMPILE_TEST
 	select PHYLIB
+	select MDIO_BITBANG
 	help
 	  This driver supports TI's DaVinci MDIO module.
 
diff --git a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c
index abc1e4276cf0..c51e2af91f69 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c
@@ -402,9 +402,9 @@ static void am65_cpsw_get_drvinfo(struct net_device *ndev,
 {
 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
 
-	strlcpy(info->driver, dev_driver_string(common->dev),
+	strscpy(info->driver, dev_driver_string(common->dev),
 		sizeof(info->driver));
-	strlcpy(info->bus_info, dev_name(common->dev), sizeof(info->bus_info));
+	strscpy(info->bus_info, dev_name(common->dev), sizeof(info->bus_info));
 }
 
 static u32 am65_cpsw_get_msglevel(struct net_device *ndev)
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
index f4a6b590a1e3..3cbe4ec46234 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
@@ -74,6 +74,9 @@
 #define AM65_CPSW_PORTN_REG_TS_VLAN_LTYPE_REG	0x318
 #define AM65_CPSW_PORTN_REG_TS_CTL_LTYPE2       0x31C
 
+#define AM65_CPSW_SGMII_CONTROL_REG		0x010
+#define AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE	BIT(0)
+
 #define AM65_CPSW_CTL_VLAN_AWARE		BIT(1)
 #define AM65_CPSW_CTL_P0_ENABLE			BIT(2)
 #define AM65_CPSW_CTL_P0_TX_CRC_REMOVE		BIT(13)
@@ -360,8 +363,7 @@ static void am65_cpsw_init_host_port_emac(struct am65_cpsw_common *common);
 static void am65_cpsw_init_port_switch_ale(struct am65_cpsw_port *port);
 static void am65_cpsw_init_port_emac_ale(struct am65_cpsw_port *port);
 
-static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common,
-				      netdev_features_t features)
+static int am65_cpsw_nuss_common_open(struct am65_cpsw_common *common)
 {
 	struct am65_cpsw_host *host_p = am65_common_get_host(common);
 	int port_idx, i, ret;
@@ -574,7 +576,7 @@ static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
 	for (i = 0; i < common->tx_ch_num; i++)
 		netdev_tx_reset_queue(netdev_get_tx_queue(ndev, i));
 
-	ret = am65_cpsw_nuss_common_open(common, ndev->features);
+	ret = am65_cpsw_nuss_common_open(common);
 	if (ret)
 		return ret;
 
@@ -590,11 +592,6 @@ static int am65_cpsw_nuss_ndo_slave_open(struct net_device *ndev)
 	/* mac_sl should be configured via phy-link interface */
 	am65_cpsw_sl_ctl_reset(port);
 
-	ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET,
-			       port->slave.phy_if);
-	if (ret)
-		goto error_cleanup;
-
 	ret = phylink_of_phy_connect(port->slave.phylink, port->slave.phy_node, 0);
 	if (ret)
 		goto error_cleanup;
@@ -1409,7 +1406,14 @@ static const struct net_device_ops am65_cpsw_nuss_netdev_ops = {
 static void am65_cpsw_nuss_mac_config(struct phylink_config *config, unsigned int mode,
 				      const struct phylink_link_state *state)
 {
-	/* Currently not used */
+	struct am65_cpsw_slave_data *slave = container_of(config, struct am65_cpsw_slave_data,
+							  phylink_config);
+	struct am65_cpsw_port *port = container_of(slave, struct am65_cpsw_port, slave);
+	struct am65_cpsw_common *common = port->common;
+
+	if (common->pdata.extra_modes & BIT(state->interface))
+		writel(AM65_CPSW_SGMII_CONTROL_MR_AN_ENABLE,
+		       port->sgmii_base + AM65_CPSW_SGMII_CONTROL_REG);
 }
 
 static void am65_cpsw_nuss_mac_link_down(struct phylink_config *config, unsigned int mode,
@@ -1847,6 +1851,8 @@ static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common)
 		port->common = common;
 		port->port_base = common->cpsw_base + AM65_CPSW_NU_PORTS_BASE +
 				  AM65_CPSW_NU_PORTS_OFFSET * (port_id);
+		if (common->pdata.extra_modes)
+			port->sgmii_base = common->ss_base + AM65_CPSW_SGMII_BASE * (port_id);
 		port->stat_base = common->cpsw_base + AM65_CPSW_NU_STATS_BASE +
 				  (AM65_CPSW_NU_STATS_PORT_OFFSET * port_id);
 		port->name = of_get_property(port_np, "label", NULL);
@@ -1886,6 +1892,10 @@ static int am65_cpsw_nuss_init_slave_ports(struct am65_cpsw_common *common)
 			goto of_node_put;
 		}
 
+		ret = phy_set_mode_ext(port->slave.ifphy, PHY_MODE_ETHERNET, port->slave.phy_if);
+		if (ret)
+			goto of_node_put;
+
 		ret = of_get_mac_address(port_np, port->slave.mac_addr);
 		if (ret) {
 			am65_cpsw_am654_get_efuse_macid(port_np,
@@ -1981,7 +1991,18 @@ am65_cpsw_nuss_init_port_ndev(struct am65_cpsw_common *common, u32 port_idx)
 	port->slave.phylink_config.type = PHYLINK_NETDEV;
 	port->slave.phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000FD;
 
-	phy_interface_set_rgmii(port->slave.phylink_config.supported_interfaces);
+	if (phy_interface_mode_is_rgmii(port->slave.phy_if)) {
+		phy_interface_set_rgmii(port->slave.phylink_config.supported_interfaces);
+	} else if (port->slave.phy_if == PHY_INTERFACE_MODE_RMII) {
+		__set_bit(PHY_INTERFACE_MODE_RMII,
+			  port->slave.phylink_config.supported_interfaces);
+	} else if (common->pdata.extra_modes & BIT(port->slave.phy_if)) {
+		__set_bit(PHY_INTERFACE_MODE_QSGMII,
+			  port->slave.phylink_config.supported_interfaces);
+	} else {
+		dev_err(dev, "selected phy-mode is not supported\n");
+		return -EOPNOTSUPP;
+	}
 
 	phylink = phylink_create(&port->slave.phylink_config,
 				 of_node_to_fwnode(port->slave.phy_node),
@@ -2023,7 +2044,7 @@ static int am65_cpsw_nuss_init_ndevs(struct am65_cpsw_common *common)
 	}
 
 	netif_napi_add(common->dma_ndev, &common->napi_rx,
-		       am65_cpsw_nuss_rx_poll, NAPI_POLL_WEIGHT);
+		       am65_cpsw_nuss_rx_poll);
 
 	return ret;
 }
@@ -2611,10 +2632,18 @@ static const struct am65_cpsw_pdata am64x_cpswxg_pdata = {
 	.fdqring_mode = K3_RINGACC_RING_MODE_RING,
 };
 
+static const struct am65_cpsw_pdata j7200_cpswxg_pdata = {
+	.quirks = 0,
+	.ale_dev_id = "am64-cpswxg",
+	.fdqring_mode = K3_RINGACC_RING_MODE_RING,
+	.extra_modes = BIT(PHY_INTERFACE_MODE_QSGMII),
+};
+
 static const struct of_device_id am65_cpsw_nuss_of_mtable[] = {
 	{ .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0},
 	{ .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata},
 	{ .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata},
+	{ .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata},
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, am65_cpsw_nuss_of_mtable);
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.h b/drivers/net/ethernet/ti/am65-cpsw-nuss.h
index ac945631bf2f..2c9850fdfcb6 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.h
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.h
@@ -46,6 +46,7 @@ struct am65_cpsw_port {
 	const char			*name;
 	u32				port_id;
 	void __iomem			*port_base;
+	void __iomem			*sgmii_base;
 	void __iomem			*stat_base;
 	void __iomem			*fetch_ram_base;
 	bool				disabled;
@@ -88,6 +89,7 @@ struct am65_cpsw_rx_chn {
 
 struct am65_cpsw_pdata {
 	u32	quirks;
+	u64	extra_modes;
 	enum k3_ring_mode fdqring_mode;
 	const char	*ale_dev_id;
 };
diff --git a/drivers/net/ethernet/ti/am65-cpts.c b/drivers/net/ethernet/ti/am65-cpts.c
index c30a6e510aa3..e2f0fb286143 100644
--- a/drivers/net/ethernet/ti/am65-cpts.c
+++ b/drivers/net/ethernet/ti/am65-cpts.c
@@ -943,9 +943,7 @@ struct am65_cpts *am65_cpts_create(struct device *dev, void __iomem *regs,
 	cpts->irq = of_irq_get_byname(node, "cpts");
 	if (cpts->irq <= 0) {
 		ret = cpts->irq ?: -ENXIO;
-		if (ret != -EPROBE_DEFER)
-			dev_err(dev, "Failed to get IRQ number (err = %d)\n",
-				ret);
+		dev_err_probe(dev, ret, "Failed to get IRQ number\n");
 		return ERR_PTR(ret);
 	}
 
@@ -965,8 +963,7 @@ struct am65_cpts *am65_cpts_create(struct device *dev, void __iomem *regs,
 	cpts->refclk = devm_get_clk_from_child(dev, node, "cpts");
 	if (IS_ERR(cpts->refclk)) {
 		ret = PTR_ERR(cpts->refclk);
-		if (ret != -EPROBE_DEFER)
-			dev_err(dev, "Failed to get refclk %d\n", ret);
+		dev_err_probe(dev, ret, "Failed to get refclk\n");
 		return ERR_PTR(ret);
 	}
 
diff --git a/drivers/net/ethernet/ti/cpmac.c b/drivers/net/ethernet/ti/cpmac.c
index bef5e68dac31..80eeeb463c4f 100644
--- a/drivers/net/ethernet/ti/cpmac.c
+++ b/drivers/net/ethernet/ti/cpmac.c
@@ -851,8 +851,8 @@ static int cpmac_set_ringparam(struct net_device *dev,
 static void cpmac_get_drvinfo(struct net_device *dev,
 			      struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, "cpmac", sizeof(info->driver));
-	strlcpy(info->version, CPMAC_VERSION, sizeof(info->version));
+	strscpy(info->driver, "cpmac", sizeof(info->driver));
+	strscpy(info->version, CPMAC_VERSION, sizeof(info->version));
 	snprintf(info->bus_info, sizeof(info->bus_info), "%s", "cpmac");
 }
 
@@ -1109,7 +1109,7 @@ static int cpmac_probe(struct platform_device *pdev)
 	dev->netdev_ops = &cpmac_netdev_ops;
 	dev->ethtool_ops = &cpmac_ethtool_ops;
 
-	netif_napi_add(dev, &priv->napi, cpmac_poll, 64);
+	netif_napi_add(dev, &priv->napi, cpmac_poll);
 
 	spin_lock_init(&priv->lock);
 	spin_lock_init(&priv->rx_lock);
@@ -1169,7 +1169,7 @@ static struct platform_driver cpmac_driver = {
 	.remove = cpmac_remove,
 };
 
-int cpmac_init(void)
+int __init cpmac_init(void)
 {
 	u32 mask;
 	int i, res;
@@ -1239,7 +1239,7 @@ fail_alloc:
 	return res;
 }
 
-void cpmac_exit(void)
+void __exit cpmac_exit(void)
 {
 	platform_driver_unregister(&cpmac_driver);
 	mdiobus_unregister(cpmac_mii);
diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c
index ed66c4d4d830..709ca6dd6ecb 100644
--- a/drivers/net/ethernet/ti/cpsw.c
+++ b/drivers/net/ethernet/ti/cpsw.c
@@ -1172,9 +1172,9 @@ static void cpsw_get_drvinfo(struct net_device *ndev,
 	struct cpsw_common *cpsw = ndev_to_cpsw(ndev);
 	struct platform_device	*pdev = to_platform_device(cpsw->dev);
 
-	strlcpy(info->driver, "cpsw", sizeof(info->driver));
-	strlcpy(info->version, "1.0", sizeof(info->version));
-	strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info));
+	strscpy(info->driver, "cpsw", sizeof(info->driver));
+	strscpy(info->version, "1.0", sizeof(info->version));
+	strscpy(info->bus_info, pdev->name, sizeof(info->bus_info));
 }
 
 static int cpsw_set_pauseparam(struct net_device *ndev,
@@ -1319,8 +1319,7 @@ static int cpsw_probe_dt(struct cpsw_platform_data *data,
 			 */
 			ret = of_phy_register_fixed_link(slave_node);
 			if (ret) {
-				if (ret != -EPROBE_DEFER)
-					dev_err(&pdev->dev, "failed to register fixed-link phy: %d\n", ret);
+				dev_err_probe(&pdev->dev, ret, "failed to register fixed-link phy\n");
 				goto err_node_put;
 			}
 			slave_data->phy_node = of_node_get(slave_node);
@@ -1638,8 +1637,7 @@ static int cpsw_probe(struct platform_device *pdev)
 	ndev->netdev_ops = &cpsw_netdev_ops;
 	ndev->ethtool_ops = &cpsw_ethtool_ops;
 	netif_napi_add(ndev, &cpsw->napi_rx,
-		       cpsw->quirk_irq ? cpsw_rx_poll : cpsw_rx_mq_poll,
-		       NAPI_POLL_WEIGHT);
+		       cpsw->quirk_irq ? cpsw_rx_poll : cpsw_rx_mq_poll);
 	netif_napi_add_tx(ndev, &cpsw->napi_tx,
 			  cpsw->quirk_irq ? cpsw_tx_poll : cpsw_tx_mq_poll);
 
diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c
index 353e58b22c51..83596ec0c7cb 100644
--- a/drivers/net/ethernet/ti/cpsw_new.c
+++ b/drivers/net/ethernet/ti/cpsw_new.c
@@ -1146,9 +1146,9 @@ static void cpsw_get_drvinfo(struct net_device *ndev,
 	struct platform_device *pdev;
 
 	pdev = to_platform_device(cpsw->dev);
-	strlcpy(info->driver, "cpsw-switch", sizeof(info->driver));
-	strlcpy(info->version, "2.0", sizeof(info->version));
-	strlcpy(info->bus_info, pdev->name, sizeof(info->bus_info));
+	strscpy(info->driver, "cpsw-switch", sizeof(info->driver));
+	strscpy(info->version, "2.0", sizeof(info->version));
+	strscpy(info->bus_info, pdev->name, sizeof(info->bus_info));
 }
 
 static int cpsw_set_pauseparam(struct net_device *ndev,
@@ -1288,9 +1288,8 @@ static int cpsw_probe_dt(struct cpsw_common *cpsw)
 		if (of_phy_is_fixed_link(port_np)) {
 			ret = of_phy_register_fixed_link(port_np);
 			if (ret) {
-				if (ret != -EPROBE_DEFER)
-					dev_err(dev, "%pOF failed to register fixed-link phy: %d\n",
-						port_np, ret);
+				dev_err_probe(dev, ret, "%pOF failed to register fixed-link phy\n",
+					      port_np);
 				goto err_node_put;
 			}
 			slave_data->phy_node = of_node_get(port_np);
@@ -1417,9 +1416,7 @@ static int cpsw_create_ports(struct cpsw_common *cpsw)
 			 * accordingly.
 			 */
 			netif_napi_add(ndev, &cpsw->napi_rx,
-				       cpsw->quirk_irq ?
-				       cpsw_rx_poll : cpsw_rx_mq_poll,
-				       NAPI_POLL_WEIGHT);
+				       cpsw->quirk_irq ? cpsw_rx_poll : cpsw_rx_mq_poll);
 			netif_napi_add_tx(ndev, &cpsw->napi_tx,
 					  cpsw->quirk_irq ?
 					  cpsw_tx_poll : cpsw_tx_mq_poll);
diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c
index 2a3e4e842fa5..2eb9d5a32588 100644
--- a/drivers/net/ethernet/ti/davinci_emac.c
+++ b/drivers/net/ethernet/ti/davinci_emac.c
@@ -374,8 +374,8 @@ static char *emac_rxhost_errcodes[16] = {
 static void emac_get_drvinfo(struct net_device *ndev,
 			     struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, emac_version_string, sizeof(info->driver));
-	strlcpy(info->version, EMAC_MODULE_VERSION, sizeof(info->version));
+	strscpy(info->driver, emac_version_string, sizeof(info->driver));
+	strscpy(info->version, EMAC_MODULE_VERSION, sizeof(info->version));
 }
 
 /**
@@ -949,7 +949,7 @@ static void emac_tx_handler(void *token, int len, int status)
  *
  * Returns success(NETDEV_TX_OK) or error code (typically out of desc's)
  */
-static int emac_dev_xmit(struct sk_buff *skb, struct net_device *ndev)
+static netdev_tx_t emac_dev_xmit(struct sk_buff *skb, struct net_device *ndev)
 {
 	struct device *emac_dev = &ndev->dev;
 	int ret_code;
@@ -1948,7 +1948,7 @@ static int davinci_emac_probe(struct platform_device *pdev)
 
 	ndev->netdev_ops = &emac_netdev_ops;
 	ndev->ethtool_ops = &ethtool_ops;
-	netif_napi_add(ndev, &priv->napi, emac_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &priv->napi, emac_poll);
 
 	pm_runtime_enable(&pdev->dev);
 	rc = pm_runtime_resume_and_get(&pdev->dev);
diff --git a/drivers/net/ethernet/ti/davinci_mdio.c b/drivers/net/ethernet/ti/davinci_mdio.c
index ea3772618043..946b9753ccfb 100644
--- a/drivers/net/ethernet/ti/davinci_mdio.c
+++ b/drivers/net/ethernet/ti/davinci_mdio.c
@@ -26,6 +26,8 @@
 #include <linux/of_device.h>
 #include <linux/of_mdio.h>
 #include <linux/pinctrl/consumer.h>
+#include <linux/mdio-bitbang.h>
+#include <linux/sys_soc.h>
 
 /*
  * This timeout definition is a worst-case ultra defensive measure against
@@ -41,6 +43,7 @@
 
 struct davinci_mdio_of_param {
 	int autosuspend_delay_ms;
+	bool manual_mode;
 };
 
 struct davinci_mdio_regs {
@@ -49,6 +52,15 @@ struct davinci_mdio_regs {
 #define CONTROL_IDLE		BIT(31)
 #define CONTROL_ENABLE		BIT(30)
 #define CONTROL_MAX_DIV		(0xffff)
+#define CONTROL_CLKDIV		GENMASK(15, 0)
+
+#define MDIO_MAN_MDCLK_O	BIT(2)
+#define MDIO_MAN_OE		BIT(1)
+#define MDIO_MAN_PIN		BIT(0)
+#define MDIO_MANUALMODE		BIT(31)
+
+#define MDIO_PIN               0
+
 
 	u32	alive;
 	u32	link;
@@ -59,7 +71,9 @@ struct davinci_mdio_regs {
 	u32	userintmasked;
 	u32	userintmaskset;
 	u32	userintmaskclr;
-	u32	__reserved_1[20];
+	u32	manualif;
+	u32	poll;
+	u32	__reserved_1[18];
 
 	struct {
 		u32	access;
@@ -79,6 +93,7 @@ static const struct mdio_platform_data default_pdata = {
 
 struct davinci_mdio_data {
 	struct mdio_platform_data pdata;
+	struct mdiobb_ctrl bb_ctrl;
 	struct davinci_mdio_regs __iomem *regs;
 	struct clk	*clk;
 	struct device	*dev;
@@ -90,6 +105,7 @@ struct davinci_mdio_data {
 	 */
 	bool		skip_scan;
 	u32		clk_div;
+	bool		manual_mode;
 };
 
 static void davinci_mdio_init_clk(struct davinci_mdio_data *data)
@@ -128,9 +144,122 @@ static void davinci_mdio_enable(struct davinci_mdio_data *data)
 	writel(data->clk_div | CONTROL_ENABLE, &data->regs->control);
 }
 
-static int davinci_mdio_reset(struct mii_bus *bus)
+static void davinci_mdio_disable(struct davinci_mdio_data *data)
+{
+	u32 reg;
+
+	/* Disable MDIO state machine */
+	reg = readl(&data->regs->control);
+
+	reg &= ~CONTROL_CLKDIV;
+	reg |= data->clk_div;
+
+	reg &= ~CONTROL_ENABLE;
+	writel(reg, &data->regs->control);
+}
+
+static void davinci_mdio_enable_manual_mode(struct davinci_mdio_data *data)
+{
+	u32 reg;
+	/* set manual mode */
+	reg = readl(&data->regs->poll);
+	reg |= MDIO_MANUALMODE;
+	writel(reg, &data->regs->poll);
+}
+
+static void davinci_set_mdc(struct mdiobb_ctrl *ctrl, int level)
+{
+	struct davinci_mdio_data *data;
+	u32 reg;
+
+	data = container_of(ctrl, struct davinci_mdio_data, bb_ctrl);
+	reg = readl(&data->regs->manualif);
+
+	if (level)
+		reg |= MDIO_MAN_MDCLK_O;
+	else
+		reg &= ~MDIO_MAN_MDCLK_O;
+
+	writel(reg, &data->regs->manualif);
+}
+
+static void davinci_set_mdio_dir(struct mdiobb_ctrl *ctrl, int output)
+{
+	struct davinci_mdio_data *data;
+	u32 reg;
+
+	data = container_of(ctrl, struct davinci_mdio_data, bb_ctrl);
+	reg = readl(&data->regs->manualif);
+
+	if (output)
+		reg |= MDIO_MAN_OE;
+	else
+		reg &= ~MDIO_MAN_OE;
+
+	writel(reg, &data->regs->manualif);
+}
+
+static void  davinci_set_mdio_data(struct mdiobb_ctrl *ctrl, int value)
+{
+	struct davinci_mdio_data *data;
+	u32 reg;
+
+	data = container_of(ctrl, struct davinci_mdio_data, bb_ctrl);
+	reg = readl(&data->regs->manualif);
+
+	if (value)
+		reg |= MDIO_MAN_PIN;
+	else
+		reg &= ~MDIO_MAN_PIN;
+
+	writel(reg, &data->regs->manualif);
+}
+
+static int davinci_get_mdio_data(struct mdiobb_ctrl *ctrl)
+{
+	struct davinci_mdio_data *data;
+	unsigned long reg;
+
+	data = container_of(ctrl, struct davinci_mdio_data, bb_ctrl);
+	reg = readl(&data->regs->manualif);
+	return test_bit(MDIO_PIN, &reg);
+}
+
+static int davinci_mdiobb_read(struct mii_bus *bus, int phy, int reg)
+{
+	int ret;
+
+	ret = pm_runtime_resume_and_get(bus->parent);
+	if (ret < 0)
+		return ret;
+
+	ret = mdiobb_read(bus, phy, reg);
+
+	pm_runtime_mark_last_busy(bus->parent);
+	pm_runtime_put_autosuspend(bus->parent);
+
+	return ret;
+}
+
+static int davinci_mdiobb_write(struct mii_bus *bus, int phy, int reg,
+				u16 val)
+{
+	int ret;
+
+	ret = pm_runtime_resume_and_get(bus->parent);
+	if (ret < 0)
+		return ret;
+
+	ret = mdiobb_write(bus, phy, reg, val);
+
+	pm_runtime_mark_last_busy(bus->parent);
+	pm_runtime_put_autosuspend(bus->parent);
+
+	return ret;
+}
+
+static int davinci_mdio_common_reset(struct davinci_mdio_data *data)
 {
-	struct davinci_mdio_data *data = bus->priv;
 	u32 phy_mask, ver;
 	int ret;
 
@@ -138,6 +267,11 @@ static int davinci_mdio_reset(struct mii_bus *bus)
 	if (ret < 0)
 		return ret;
 
+	if (data->manual_mode) {
+		davinci_mdio_disable(data);
+		davinci_mdio_enable_manual_mode(data);
+	}
+
 	/* wait for scan logic to settle */
 	msleep(PHY_MAX_ADDR * data->access_time);
 
@@ -171,6 +305,23 @@ done:
 	return 0;
 }
 
+static int davinci_mdio_reset(struct mii_bus *bus)
+{
+	struct davinci_mdio_data *data = bus->priv;
+
+	return davinci_mdio_common_reset(data);
+}
+
+static int davinci_mdiobb_reset(struct mii_bus *bus)
+{
+	struct mdiobb_ctrl *ctrl = bus->priv;
+	struct davinci_mdio_data *data;
+
+	data = container_of(ctrl, struct davinci_mdio_data, bb_ctrl);
+
+	return davinci_mdio_common_reset(data);
+}
+
 /* wait until hardware is ready for another user access */
 static inline int wait_for_user_access(struct davinci_mdio_data *data)
 {
@@ -318,6 +469,28 @@ static int davinci_mdio_probe_dt(struct mdio_platform_data *data,
 	return 0;
 }
 
+struct k3_mdio_soc_data {
+	bool manual_mode;
+};
+
+static const struct k3_mdio_soc_data am65_mdio_soc_data = {
+	.manual_mode = true,
+};
+
+static const struct soc_device_attribute k3_mdio_socinfo[] = {
+	{ .family = "AM62X", .revision = "SR1.0", .data = &am65_mdio_soc_data },
+	{ .family = "AM64X", .revision = "SR1.0", .data = &am65_mdio_soc_data },
+	{ .family = "AM64X", .revision = "SR2.0", .data = &am65_mdio_soc_data },
+	{ .family = "AM65X", .revision = "SR1.0", .data = &am65_mdio_soc_data },
+	{ .family = "AM65X", .revision = "SR2.0", .data = &am65_mdio_soc_data },
+	{ .family = "J7200", .revision = "SR1.0", .data = &am65_mdio_soc_data },
+	{ .family = "J7200", .revision = "SR2.0", .data = &am65_mdio_soc_data },
+	{ .family = "J721E", .revision = "SR1.0", .data = &am65_mdio_soc_data },
+	{ .family = "J721E", .revision = "SR2.0", .data = &am65_mdio_soc_data },
+	{ .family = "J721S2", .revision = "SR1.0", .data = &am65_mdio_soc_data},
+	{ /* sentinel */ },
+};
+
 #if IS_ENABLED(CONFIG_OF)
 static const struct davinci_mdio_of_param of_cpsw_mdio_data = {
 	.autosuspend_delay_ms = 100,
@@ -331,6 +504,14 @@ static const struct of_device_id davinci_mdio_of_mtable[] = {
 MODULE_DEVICE_TABLE(of, davinci_mdio_of_mtable);
 #endif
 
+static const struct mdiobb_ops davinci_mdiobb_ops = {
+	.owner = THIS_MODULE,
+	.set_mdc = davinci_set_mdc,
+	.set_mdio_dir = davinci_set_mdio_dir,
+	.set_mdio_data = davinci_set_mdio_data,
+	.get_mdio_data = davinci_get_mdio_data,
+};
+
 static int davinci_mdio_probe(struct platform_device *pdev)
 {
 	struct mdio_platform_data *pdata = dev_get_platdata(&pdev->dev);
@@ -345,7 +526,26 @@ static int davinci_mdio_probe(struct platform_device *pdev)
 	if (!data)
 		return -ENOMEM;
 
-	data->bus = devm_mdiobus_alloc(dev);
+	data->manual_mode = false;
+	data->bb_ctrl.ops = &davinci_mdiobb_ops;
+
+	if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
+		const struct soc_device_attribute *soc_match_data;
+
+		soc_match_data = soc_device_match(k3_mdio_socinfo);
+		if (soc_match_data && soc_match_data->data) {
+			const struct k3_mdio_soc_data *socdata =
+						soc_match_data->data;
+
+			data->manual_mode = socdata->manual_mode;
+		}
+	}
+
+	if (data->manual_mode)
+		data->bus = alloc_mdio_bitbang(&data->bb_ctrl);
+	else
+		data->bus = devm_mdiobus_alloc(dev);
+
 	if (!data->bus) {
 		dev_err(dev, "failed to alloc mii bus\n");
 		return -ENOMEM;
@@ -371,11 +571,20 @@ static int davinci_mdio_probe(struct platform_device *pdev)
 	}
 
 	data->bus->name		= dev_name(dev);
-	data->bus->read		= davinci_mdio_read;
-	data->bus->write	= davinci_mdio_write;
-	data->bus->reset	= davinci_mdio_reset;
+
+	if (data->manual_mode) {
+		data->bus->read		= davinci_mdiobb_read;
+		data->bus->write	= davinci_mdiobb_write;
+		data->bus->reset	= davinci_mdiobb_reset;
+
+		dev_info(dev, "Configuring MDIO in manual mode\n");
+	} else {
+		data->bus->read		= davinci_mdio_read;
+		data->bus->write	= davinci_mdio_write;
+		data->bus->reset	= davinci_mdio_reset;
+		data->bus->priv		= data;
+	}
 	data->bus->parent	= dev;
-	data->bus->priv		= data;
 
 	data->clk = devm_clk_get(dev, "fck");
 	if (IS_ERR(data->clk)) {
@@ -433,9 +642,13 @@ static int davinci_mdio_remove(struct platform_device *pdev)
 {
 	struct davinci_mdio_data *data = platform_get_drvdata(pdev);
 
-	if (data->bus)
+	if (data->bus) {
 		mdiobus_unregister(data->bus);
 
+		if (data->manual_mode)
+			free_mdio_bitbang(data->bus);
+	}
+
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
@@ -452,7 +665,9 @@ static int davinci_mdio_runtime_suspend(struct device *dev)
 	ctrl = readl(&data->regs->control);
 	ctrl &= ~CONTROL_ENABLE;
 	writel(ctrl, &data->regs->control);
-	wait_for_idle(data);
+
+	if (!data->manual_mode)
+		wait_for_idle(data);
 
 	return 0;
 }
@@ -461,7 +676,12 @@ static int davinci_mdio_runtime_resume(struct device *dev)
 {
 	struct davinci_mdio_data *data = dev_get_drvdata(dev);
 
-	davinci_mdio_enable(data);
+	if (data->manual_mode) {
+		davinci_mdio_disable(data);
+		davinci_mdio_enable_manual_mode(data);
+	} else {
+		davinci_mdio_enable(data);
+	}
 	return 0;
 }
 #endif
diff --git a/drivers/net/ethernet/ti/netcp_core.c b/drivers/net/ethernet/ti/netcp_core.c
index b15d44261e76..aba70bef4894 100644
--- a/drivers/net/ethernet/ti/netcp_core.c
+++ b/drivers/net/ethernet/ti/netcp_core.c
@@ -2095,7 +2095,7 @@ static int netcp_create_interface(struct netcp_device *netcp_device,
 	}
 
 	/* NAPI register */
-	netif_napi_add(ndev, &netcp->rx_napi, netcp_rx_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &netcp->rx_napi, netcp_rx_poll);
 	netif_napi_add_tx(ndev, &netcp->tx_napi, netcp_tx_poll);
 
 	/* Register the network device */
diff --git a/drivers/net/ethernet/ti/tlan.c b/drivers/net/ethernet/ti/tlan.c
index 741c42c6a417..b3da76efa8f5 100644
--- a/drivers/net/ethernet/ti/tlan.c
+++ b/drivers/net/ethernet/ti/tlan.c
@@ -762,12 +762,12 @@ static void tlan_get_drvinfo(struct net_device *dev,
 {
 	struct tlan_priv *priv = netdev_priv(dev);
 
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
 	if (priv->pci_dev)
-		strlcpy(info->bus_info, pci_name(priv->pci_dev),
+		strscpy(info->bus_info, pci_name(priv->pci_dev),
 			sizeof(info->bus_info));
 	else
-		strlcpy(info->bus_info, "EISA",	sizeof(info->bus_info));
+		strscpy(info->bus_info, "EISA",	sizeof(info->bus_info));
 }
 
 static int tlan_get_eeprom_len(struct net_device *dev)
diff --git a/drivers/net/ethernet/toshiba/ps3_gelic_net.c b/drivers/net/ethernet/toshiba/ps3_gelic_net.c
index 3dbfb1b20649..cf8de8a7a8a1 100644
--- a/drivers/net/ethernet/toshiba/ps3_gelic_net.c
+++ b/drivers/net/ethernet/toshiba/ps3_gelic_net.c
@@ -1187,8 +1187,8 @@ int gelic_net_open(struct net_device *netdev)
 void gelic_net_get_drvinfo(struct net_device *netdev,
 			   struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 }
 
 static int gelic_ether_get_link_ksettings(struct net_device *netdev,
@@ -1441,7 +1441,7 @@ static void gelic_ether_setup_netdev_ops(struct net_device *netdev,
 {
 	netdev->watchdog_timeo = GELIC_NET_WATCHDOG_TIMEOUT;
 	/* NAPI */
-	netif_napi_add(netdev, napi, gelic_net_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, napi, gelic_net_poll);
 	netdev->ethtool_ops = &gelic_ether_ethtool_ops;
 	netdev->netdev_ops = &gelic_netdevice_ops;
 }
diff --git a/drivers/net/ethernet/toshiba/spider_net.c b/drivers/net/ethernet/toshiba/spider_net.c
index bc4914c758ad..50d7eacfec58 100644
--- a/drivers/net/ethernet/toshiba/spider_net.c
+++ b/drivers/net/ethernet/toshiba/spider_net.c
@@ -2270,8 +2270,7 @@ spider_net_setup_netdev(struct spider_net_card *card)
 	card->aneg_count = 0;
 	timer_setup(&card->aneg_timer, spider_net_link_phy, 0);
 
-	netif_napi_add(netdev, &card->napi,
-		       spider_net_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, &card->napi, spider_net_poll);
 
 	spider_net_setup_netdev_ops(netdev);
 
diff --git a/drivers/net/ethernet/toshiba/spider_net_ethtool.c b/drivers/net/ethernet/toshiba/spider_net_ethtool.c
index 93110dba0bfa..fef9fd127b5e 100644
--- a/drivers/net/ethernet/toshiba/spider_net_ethtool.c
+++ b/drivers/net/ethernet/toshiba/spider_net_ethtool.c
@@ -63,12 +63,12 @@ spider_net_ethtool_get_drvinfo(struct net_device *netdev,
 	card = netdev_priv(netdev);
 
 	/* clear and fill out info */
-	strlcpy(drvinfo->driver, spider_net_driver_name,
+	strscpy(drvinfo->driver, spider_net_driver_name,
 		sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, VERSION, sizeof(drvinfo->version));
-	strlcpy(drvinfo->fw_version, "no information",
+	strscpy(drvinfo->version, VERSION, sizeof(drvinfo->version));
+	strscpy(drvinfo->fw_version, "no information",
 		sizeof(drvinfo->fw_version));
-	strlcpy(drvinfo->bus_info, pci_name(card->pdev),
+	strscpy(drvinfo->bus_info, pci_name(card->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/ethernet/toshiba/tc35815.c b/drivers/net/ethernet/toshiba/tc35815.c
index 47aab9c132c8..b50be67b398b 100644
--- a/drivers/net/ethernet/toshiba/tc35815.c
+++ b/drivers/net/ethernet/toshiba/tc35815.c
@@ -1956,9 +1956,9 @@ static void tc35815_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *
 {
 	struct tc35815_local *lp = netdev_priv(dev);
 
-	strlcpy(info->driver, MODNAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(lp->pci_dev), sizeof(info->bus_info));
+	strscpy(info->driver, MODNAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(lp->pci_dev), sizeof(info->bus_info));
 }
 
 static u32 tc35815_get_msglevel(struct net_device *dev)
diff --git a/drivers/net/ethernet/tundra/tsi108_eth.c b/drivers/net/ethernet/tundra/tsi108_eth.c
index 5251fc324221..2cd2afc3fff0 100644
--- a/drivers/net/ethernet/tundra/tsi108_eth.c
+++ b/drivers/net/ethernet/tundra/tsi108_eth.c
@@ -59,9 +59,6 @@
 /* Check the phy status every half a second. */
 #define CHECK_PHY_INTERVAL (HZ/2)
 
-static int tsi108_init_one(struct platform_device *pdev);
-static int tsi108_ether_remove(struct platform_device *pdev);
-
 struct tsi108_prv_data {
 	void  __iomem *regs;	/* Base of normal regs */
 	void  __iomem *phyregs;	/* Base of register bank used for PHY access */
@@ -144,16 +141,6 @@ struct tsi108_prv_data {
 	struct platform_device *pdev;
 };
 
-/* Structure for a device driver */
-
-static struct platform_driver tsi_eth_driver = {
-	.probe = tsi108_init_one,
-	.remove = tsi108_ether_remove,
-	.driver	= {
-		.name = "tsi-ethernet",
-	},
-};
-
 static void tsi108_timed_checker(struct timer_list *t);
 
 #ifdef DEBUG
@@ -1598,7 +1585,7 @@ tsi108_init_one(struct platform_device *pdev)
 	data->phy_type = einfo->phy_type;
 	data->irq_num = einfo->irq_num;
 	data->id = pdev->id;
-	netif_napi_add(dev, &data->napi, tsi108_poll, 64);
+	netif_napi_add(dev, &data->napi, tsi108_poll);
 	dev->netdev_ops = &tsi108_netdev_ops;
 	dev->ethtool_ops = &tsi108_ethtool_ops;
 
@@ -1683,6 +1670,16 @@ static int tsi108_ether_remove(struct platform_device *pdev)
 
 	return 0;
 }
+
+/* Structure for a device driver */
+
+static struct platform_driver tsi_eth_driver = {
+	.probe = tsi108_init_one,
+	.remove = tsi108_ether_remove,
+	.driver	= {
+		.name = "tsi-ethernet",
+	},
+};
 module_platform_driver(tsi_eth_driver);
 
 MODULE_AUTHOR("Tundra Semiconductor Corporation");
diff --git a/drivers/net/ethernet/vertexcom/mse102x.c b/drivers/net/ethernet/vertexcom/mse102x.c
index eb39a45de012..aeed2a093e34 100644
--- a/drivers/net/ethernet/vertexcom/mse102x.c
+++ b/drivers/net/ethernet/vertexcom/mse102x.c
@@ -750,6 +750,13 @@ static const struct of_device_id mse102x_match_table[] = {
 };
 MODULE_DEVICE_TABLE(of, mse102x_match_table);
 
+static const struct spi_device_id mse102x_ids[] = {
+	{ "mse1021" },
+	{ "mse1022" },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, mse102x_ids);
+
 static struct spi_driver mse102x_driver = {
 	.driver = {
 		.name = DRV_NAME,
@@ -758,10 +765,11 @@ static struct spi_driver mse102x_driver = {
 	},
 	.probe = mse102x_probe_spi,
 	.remove = mse102x_remove_spi,
+	.id_table = mse102x_ids,
 };
 module_spi_driver(mse102x_driver);
 
 MODULE_DESCRIPTION("MSE102x Network driver");
-MODULE_AUTHOR("Stefan Wahren <stefan.wahren@in-tech.com>");
+MODULE_AUTHOR("Stefan Wahren <stefan.wahren@chargebyte.com>");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("spi:" DRV_NAME);
diff --git a/drivers/net/ethernet/via/via-rhine.c b/drivers/net/ethernet/via/via-rhine.c
index 509c5e9b29df..0fb15a17b547 100644
--- a/drivers/net/ethernet/via/via-rhine.c
+++ b/drivers/net/ethernet/via/via-rhine.c
@@ -965,7 +965,7 @@ static int rhine_init_one_common(struct device *hwdev, u32 quirks,
 	dev->ethtool_ops = &netdev_ethtool_ops;
 	dev->watchdog_timeo = TX_TIMEOUT;
 
-	netif_napi_add(dev, &rp->napi, rhine_napipoll, 64);
+	netif_napi_add(dev, &rp->napi, rhine_napipoll);
 
 	if (rp->quirks & rqRhineI)
 		dev->features |= NETIF_F_SG|NETIF_F_HW_CSUM;
@@ -2281,8 +2281,8 @@ static void netdev_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *i
 {
 	struct device *hwdev = dev->dev.parent;
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->bus_info, dev_name(hwdev), sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->bus_info, dev_name(hwdev), sizeof(info->bus_info));
 }
 
 static int netdev_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/ethernet/via/via-velocity.c b/drivers/net/ethernet/via/via-velocity.c
index ff0c102cb578..a502812ac418 100644
--- a/drivers/net/ethernet/via/via-velocity.c
+++ b/drivers/net/ethernet/via/via-velocity.c
@@ -2846,7 +2846,7 @@ static int velocity_probe(struct device *dev, int irq,
 
 	netdev->netdev_ops = &velocity_netdev_ops;
 	netdev->ethtool_ops = &velocity_ethtool_ops;
-	netif_napi_add(netdev, &vptr->napi, velocity_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, &vptr->napi, velocity_poll);
 
 	netdev->hw_features = NETIF_F_IP_CSUM | NETIF_F_SG |
 			   NETIF_F_HW_VLAN_CTAG_TX;
@@ -3419,13 +3419,13 @@ static void velocity_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo
 {
 	struct velocity_info *vptr = netdev_priv(dev);
 
-	strlcpy(info->driver, VELOCITY_NAME, sizeof(info->driver));
-	strlcpy(info->version, VELOCITY_VERSION, sizeof(info->version));
+	strscpy(info->driver, VELOCITY_NAME, sizeof(info->driver));
+	strscpy(info->version, VELOCITY_VERSION, sizeof(info->version));
 	if (vptr->pdev)
-		strlcpy(info->bus_info, pci_name(vptr->pdev),
+		strscpy(info->bus_info, pci_name(vptr->pdev),
 						sizeof(info->bus_info));
 	else
-		strlcpy(info->bus_info, "platform", sizeof(info->bus_info));
+		strscpy(info->bus_info, "platform", sizeof(info->bus_info));
 }
 
 static void velocity_ethtool_get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
diff --git a/drivers/net/ethernet/wangxun/Kconfig b/drivers/net/ethernet/wangxun/Kconfig
index b4a4fa0a58f8..f5d43d8c9629 100644
--- a/drivers/net/ethernet/wangxun/Kconfig
+++ b/drivers/net/ethernet/wangxun/Kconfig
@@ -16,6 +16,19 @@ config NET_VENDOR_WANGXUN
 
 if NET_VENDOR_WANGXUN
 
+config NGBE
+	tristate "Wangxun(R) GbE PCI Express adapters support"
+	depends on PCI
+	help
+	  This driver supports Wangxun(R) GbE PCI Express family of
+	  adapters.
+
+	  More specific information on configuring the driver is in
+	  <file:Documentation/networking/device_drivers/ethernet/wangxun/ngbe.rst>.
+
+	  To compile this driver as a module, choose M here. The module
+	  will be called ngbe.
+
 config TXGBE
 	tristate "Wangxun(R) 10GbE PCI Express adapters support"
 	depends on PCI
diff --git a/drivers/net/ethernet/wangxun/Makefile b/drivers/net/ethernet/wangxun/Makefile
index c34db1bead25..ac3fb06b233c 100644
--- a/drivers/net/ethernet/wangxun/Makefile
+++ b/drivers/net/ethernet/wangxun/Makefile
@@ -4,3 +4,4 @@
 #
 
 obj-$(CONFIG_TXGBE) += txgbe/
+obj-$(CONFIG_NGBE) += ngbe/
diff --git a/drivers/net/ethernet/wangxun/ngbe/Makefile b/drivers/net/ethernet/wangxun/ngbe/Makefile
new file mode 100644
index 000000000000..0baf75907496
--- /dev/null
+++ b/drivers/net/ethernet/wangxun/ngbe/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd.
+#
+# Makefile for the Wangxun(R) GbE PCI Express ethernet driver
+#
+
+obj-$(CONFIG_NGBE) += ngbe.o
+
+ngbe-objs := ngbe_main.o
diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe.h b/drivers/net/ethernet/wangxun/ngbe/ngbe.h
new file mode 100644
index 000000000000..f5fa6e5238cc
--- /dev/null
+++ b/drivers/net/ethernet/wangxun/ngbe/ngbe.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */
+
+#ifndef _NGBE_H_
+#define _NGBE_H_
+
+#include "ngbe_type.h"
+
+#define NGBE_MAX_FDIR_INDICES		7
+
+#define NGBE_MAX_RX_QUEUES		(NGBE_MAX_FDIR_INDICES + 1)
+#define NGBE_MAX_TX_QUEUES		(NGBE_MAX_FDIR_INDICES + 1)
+
+/* board specific private data structure */
+struct ngbe_adapter {
+	u8 __iomem *io_addr;    /* Mainly for iounmap use */
+	/* OS defined structs */
+	struct net_device *netdev;
+	struct pci_dev *pdev;
+};
+
+extern char ngbe_driver_name[];
+
+#endif /* _NGBE_H_ */
diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
new file mode 100644
index 000000000000..7674cb6e5700
--- /dev/null
+++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
@@ -0,0 +1,170 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */
+
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/string.h>
+#include <linux/aer.h>
+#include <linux/etherdevice.h>
+
+#include "ngbe.h"
+char ngbe_driver_name[] = "ngbe";
+
+/* ngbe_pci_tbl - PCI Device ID Table
+ *
+ * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
+ *   Class, Class Mask, private data (not used) }
+ */
+static const struct pci_device_id ngbe_pci_tbl[] = {
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL_W), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A2), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A2S), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A4), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A4S), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL2), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL2S), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL4), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL4S), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860LC), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A1), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A1L), 0},
+	/* required last entry */
+	{ .device = 0 }
+};
+
+static void ngbe_dev_shutdown(struct pci_dev *pdev, bool *enable_wake)
+{
+	struct ngbe_adapter *adapter = pci_get_drvdata(pdev);
+	struct net_device *netdev = adapter->netdev;
+
+	netif_device_detach(netdev);
+
+	pci_disable_device(pdev);
+}
+
+static void ngbe_shutdown(struct pci_dev *pdev)
+{
+	bool wake;
+
+	ngbe_dev_shutdown(pdev, &wake);
+
+	if (system_state == SYSTEM_POWER_OFF) {
+		pci_wake_from_d3(pdev, wake);
+		pci_set_power_state(pdev, PCI_D3hot);
+	}
+}
+
+/**
+ * ngbe_probe - Device Initialization Routine
+ * @pdev: PCI device information struct
+ * @ent: entry in ngbe_pci_tbl
+ *
+ * Returns 0 on success, negative on failure
+ *
+ * ngbe_probe initializes an adapter identified by a pci_dev structure.
+ * The OS initialization, configuring of the adapter private structure,
+ * and a hardware reset occur.
+ **/
+static int ngbe_probe(struct pci_dev *pdev,
+		      const struct pci_device_id __always_unused *ent)
+{
+	struct ngbe_adapter *adapter = NULL;
+	struct net_device *netdev;
+	int err;
+
+	err = pci_enable_device_mem(pdev);
+	if (err)
+		return err;
+
+	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (err) {
+		dev_err(&pdev->dev,
+			"No usable DMA configuration, aborting\n");
+		goto err_pci_disable_dev;
+	}
+
+	err = pci_request_selected_regions(pdev,
+					   pci_select_bars(pdev, IORESOURCE_MEM),
+					   ngbe_driver_name);
+	if (err) {
+		dev_err(&pdev->dev,
+			"pci_request_selected_regions failed %d\n", err);
+		goto err_pci_disable_dev;
+	}
+
+	pci_enable_pcie_error_reporting(pdev);
+	pci_set_master(pdev);
+
+	netdev = devm_alloc_etherdev_mqs(&pdev->dev,
+					 sizeof(struct ngbe_adapter),
+					 NGBE_MAX_TX_QUEUES,
+					 NGBE_MAX_RX_QUEUES);
+	if (!netdev) {
+		err = -ENOMEM;
+		goto err_pci_release_regions;
+	}
+
+	SET_NETDEV_DEV(netdev, &pdev->dev);
+
+	adapter = netdev_priv(netdev);
+	adapter->netdev = netdev;
+	adapter->pdev = pdev;
+
+	adapter->io_addr = devm_ioremap(&pdev->dev,
+					pci_resource_start(pdev, 0),
+					pci_resource_len(pdev, 0));
+	if (!adapter->io_addr) {
+		err = -EIO;
+		goto err_pci_release_regions;
+	}
+
+	netdev->features |= NETIF_F_HIGHDMA;
+
+	pci_set_drvdata(pdev, adapter);
+
+	return 0;
+
+err_pci_release_regions:
+	pci_disable_pcie_error_reporting(pdev);
+	pci_release_selected_regions(pdev,
+				     pci_select_bars(pdev, IORESOURCE_MEM));
+err_pci_disable_dev:
+	pci_disable_device(pdev);
+	return err;
+}
+
+/**
+ * ngbe_remove - Device Removal Routine
+ * @pdev: PCI device information struct
+ *
+ * ngbe_remove is called by the PCI subsystem to alert the driver
+ * that it should release a PCI device.  The could be caused by a
+ * Hot-Plug event, or because the driver is going to be removed from
+ * memory.
+ **/
+static void ngbe_remove(struct pci_dev *pdev)
+{
+	pci_release_selected_regions(pdev,
+				     pci_select_bars(pdev, IORESOURCE_MEM));
+
+	pci_disable_pcie_error_reporting(pdev);
+
+	pci_disable_device(pdev);
+}
+
+static struct pci_driver ngbe_driver = {
+	.name     = ngbe_driver_name,
+	.id_table = ngbe_pci_tbl,
+	.probe    = ngbe_probe,
+	.remove   = ngbe_remove,
+	.shutdown = ngbe_shutdown,
+};
+
+module_pci_driver(ngbe_driver);
+
+MODULE_DEVICE_TABLE(pci, ngbe_pci_tbl);
+MODULE_AUTHOR("Beijing WangXun Technology Co., Ltd, <software@net-swift.com>");
+MODULE_DESCRIPTION("WangXun(R) Gigabit PCI Express Network Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h b/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h
new file mode 100644
index 000000000000..26e776c3539a
--- /dev/null
+++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2019 - 2022 Beijing WangXun Technology Co., Ltd. */
+
+#ifndef _NGBE_TYPE_H_
+#define _NGBE_TYPE_H_
+
+#include <linux/types.h>
+#include <linux/netdevice.h>
+
+/************ NGBE_register.h ************/
+/* Vendor ID */
+#ifndef PCI_VENDOR_ID_WANGXUN
+#define PCI_VENDOR_ID_WANGXUN			0x8088
+#endif
+
+/* Device IDs */
+#define NGBE_DEV_ID_EM_WX1860AL_W		0x0100
+#define NGBE_DEV_ID_EM_WX1860A2			0x0101
+#define NGBE_DEV_ID_EM_WX1860A2S		0x0102
+#define NGBE_DEV_ID_EM_WX1860A4			0x0103
+#define NGBE_DEV_ID_EM_WX1860A4S		0x0104
+#define NGBE_DEV_ID_EM_WX1860AL2		0x0105
+#define NGBE_DEV_ID_EM_WX1860AL2S		0x0106
+#define NGBE_DEV_ID_EM_WX1860AL4		0x0107
+#define NGBE_DEV_ID_EM_WX1860AL4S		0x0108
+#define NGBE_DEV_ID_EM_WX1860LC			0x0109
+#define NGBE_DEV_ID_EM_WX1860A1			0x010a
+#define NGBE_DEV_ID_EM_WX1860A1L		0x010b
+
+/* Subsystem ID */
+#define NGBE_SUBID_M88E1512_SFP			0x0003
+#define NGBE_SUBID_OCP_CARD			0x0040
+#define NGBE_SUBID_LY_M88E1512_SFP		0x0050
+#define NGBE_SUBID_M88E1512_RJ45		0x0051
+#define NGBE_SUBID_M88E1512_MIX			0x0052
+#define NGBE_SUBID_YT8521S_SFP			0x0060
+#define NGBE_SUBID_INTERNAL_YT8521S_SFP		0x0061
+#define NGBE_SUBID_YT8521S_SFP_GPIO		0x0062
+#define NGBE_SUBID_INTERNAL_YT8521S_SFP_GPIO	0x0064
+#define NGBE_SUBID_LY_YT8521S_SFP		0x0070
+#define NGBE_SUBID_RGMII_FPGA			0x0080
+
+#define NGBE_OEM_MASK				0x00FF
+
+#define NGBE_NCSI_SUP				0x8000
+#define NGBE_NCSI_MASK				0x8000
+#define NGBE_WOL_SUP				0x4000
+#define NGBE_WOL_MASK				0x4000
+
+#endif /* _NGBE_TYPE_H_ */
diff --git a/drivers/net/ethernet/wiznet/w5100.c b/drivers/net/ethernet/wiznet/w5100.c
index acd78120e53c..634946e87e5f 100644
--- a/drivers/net/ethernet/wiznet/w5100.c
+++ b/drivers/net/ethernet/wiznet/w5100.c
@@ -719,9 +719,9 @@ static void w5100_hw_close(struct w5100_priv *priv)
 static void w5100_get_drvinfo(struct net_device *ndev,
 			      struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, dev_name(ndev->dev.parent),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, dev_name(ndev->dev.parent),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/wiznet/w5300.c b/drivers/net/ethernet/wiznet/w5300.c
index 773f8c77909a..b0958fe8111e 100644
--- a/drivers/net/ethernet/wiznet/w5300.c
+++ b/drivers/net/ethernet/wiznet/w5300.c
@@ -282,9 +282,9 @@ static void w5300_hw_close(struct w5300_priv *priv)
 static void w5300_get_drvinfo(struct net_device *ndev,
 			      struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, dev_name(ndev->dev.parent),
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, dev_name(ndev->dev.parent),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/ethernet/xilinx/ll_temac.h b/drivers/net/ethernet/xilinx/ll_temac.h
index c6395c406418..6668d1b760d8 100644
--- a/drivers/net/ethernet/xilinx/ll_temac.h
+++ b/drivers/net/ethernet/xilinx/ll_temac.h
@@ -21,36 +21,45 @@
 /*  Configuration options */
 
 /*  Accept all incoming packets.
- *  This option defaults to disabled (cleared) */
+ *  This option defaults to disabled (cleared)
+ */
 #define XTE_OPTION_PROMISC                      (1 << 0)
 /*  Jumbo frame support for Tx & Rx.
- *  This option defaults to disabled (cleared) */
+ *  This option defaults to disabled (cleared)
+ */
 #define XTE_OPTION_JUMBO                        (1 << 1)
 /*  VLAN Rx & Tx frame support.
- *  This option defaults to disabled (cleared) */
+ *  This option defaults to disabled (cleared)
+ */
 #define XTE_OPTION_VLAN                         (1 << 2)
 /*  Enable recognition of flow control frames on Rx
- *  This option defaults to enabled (set) */
+ *  This option defaults to enabled (set)
+ */
 #define XTE_OPTION_FLOW_CONTROL                 (1 << 4)
 /*  Strip FCS and PAD from incoming frames.
  *  Note: PAD from VLAN frames is not stripped.
- *  This option defaults to disabled (set) */
+ *  This option defaults to disabled (set)
+ */
 #define XTE_OPTION_FCS_STRIP                    (1 << 5)
 /*  Generate FCS field and add PAD automatically for outgoing frames.
- *  This option defaults to enabled (set) */
+ *  This option defaults to enabled (set)
+ */
 #define XTE_OPTION_FCS_INSERT                   (1 << 6)
 /*  Enable Length/Type error checking for incoming frames. When this option is
-set, the MAC will filter frames that have a mismatched type/length field
-and if XTE_OPTION_REPORT_RXERR is set, the user is notified when these
-types of frames are encountered. When this option is cleared, the MAC will
-allow these types of frames to be received.
-This option defaults to enabled (set) */
+ *  set, the MAC will filter frames that have a mismatched type/length field
+ *  and if XTE_OPTION_REPORT_RXERR is set, the user is notified when these
+ *  types of frames are encountered. When this option is cleared, the MAC will
+ *  allow these types of frames to be received.
+ *  This option defaults to enabled (set)
+ */
 #define XTE_OPTION_LENTYPE_ERR                  (1 << 7)
 /*  Enable the transmitter.
- *  This option defaults to enabled (set) */
+ *  This option defaults to enabled (set)
+ */
 #define XTE_OPTION_TXEN                         (1 << 11)
 /*  Enable the receiver
-*   This option defaults to enabled (set) */
+ *  This option defaults to enabled (set)
+ */
 #define XTE_OPTION_RXEN                         (1 << 12)
 
 /*  Default options set when device is initialized or reset */
@@ -68,18 +77,18 @@ This option defaults to enabled (set) */
 #define TX_TAILDESC_PTR     0x04            /* rw */
 #define TX_CHNL_CTRL        0x05            /* rw */
 /*
- 0:7      24:31       IRQTimeout
- 8:15     16:23       IRQCount
- 16:20    11:15       Reserved
- 21       10          0
- 22       9           UseIntOnEnd
- 23       8           LdIRQCnt
- 24       7           IRQEn
- 25:28    3:6         Reserved
- 29       2           IrqErrEn
- 30       1           IrqDlyEn
- 31       0           IrqCoalEn
-*/
+ *  0:7      24:31       IRQTimeout
+ *  8:15     16:23       IRQCount
+ *  16:20    11:15       Reserved
+ *  21       10          0
+ *  22       9           UseIntOnEnd
+ *  23       8           LdIRQCnt
+ *  24       7           IRQEn
+ *  25:28    3:6         Reserved
+ *  29       2           IrqErrEn
+ *  30       1           IrqDlyEn
+ *  31       0           IrqCoalEn
+ */
 #define CHNL_CTRL_IRQ_IOE       (1 << 9)
 #define CHNL_CTRL_IRQ_EN        (1 << 7)
 #define CHNL_CTRL_IRQ_ERR_EN    (1 << 2)
@@ -87,35 +96,35 @@ This option defaults to enabled (set) */
 #define CHNL_CTRL_IRQ_COAL_EN   (1 << 0)
 #define TX_IRQ_REG          0x06            /* rw */
 /*
-  0:7      24:31       DltTmrValue
- 8:15     16:23       ClscCntrValue
- 16:17    14:15       Reserved
- 18:21    10:13       ClscCnt
- 22:23    8:9         DlyCnt
- 24:28    3::7        Reserved
- 29       2           ErrIrq
- 30       1           DlyIrq
- 31       0           CoalIrq
+ *  0:7      24:31       DltTmrValue
+ *  8:15     16:23       ClscCntrValue
+ *  16:17    14:15       Reserved
+ *  18:21    10:13       ClscCnt
+ *  22:23    8:9         DlyCnt
+ *  24:28    3::7        Reserved
+ *  29       2           ErrIrq
+ *  30       1           DlyIrq
+ *  31       0           CoalIrq
  */
 #define TX_CHNL_STS         0x07            /* r */
 /*
-   0:9      22:31   Reserved
- 10       21      TailPErr
- 11       20      CmpErr
- 12       19      AddrErr
- 13       18      NxtPErr
- 14       17      CurPErr
- 15       16      BsyWr
- 16:23    8:15    Reserved
- 24       7       Error
- 25       6       IOE
- 26       5       SOE
- 27       4       Cmplt
- 28       3       SOP
- 29       2       EOP
- 30       1       EngBusy
- 31       0       Reserved
-*/
+ *  0:9      22:31   Reserved
+ *  10       21      TailPErr
+ *  11       20      CmpErr
+ *  12       19      AddrErr
+ *  13       18      NxtPErr
+ *  14       17      CurPErr
+ *  15       16      BsyWr
+ *  16:23    8:15    Reserved
+ *  24       7       Error
+ *  25       6       IOE
+ *  26       5       SOE
+ *  27       4       Cmplt
+ *  28       3       SOP
+ *  29       2       EOP
+ *  30       1       EngBusy
+ *  31       0       Reserved
+ */
 
 #define RX_NXTDESC_PTR      0x08            /* r */
 #define RX_CURBUF_ADDR      0x09            /* r */
@@ -124,17 +133,17 @@ This option defaults to enabled (set) */
 #define RX_TAILDESC_PTR     0x0c            /* rw */
 #define RX_CHNL_CTRL        0x0d            /* rw */
 /*
- 0:7      24:31       IRQTimeout
- 8:15     16:23       IRQCount
- 16:20    11:15       Reserved
- 21       10          0
- 22       9           UseIntOnEnd
- 23       8           LdIRQCnt
- 24       7           IRQEn
- 25:28    3:6         Reserved
- 29       2           IrqErrEn
- 30       1           IrqDlyEn
- 31       0           IrqCoalEn
+ *  0:7      24:31       IRQTimeout
+ *  8:15     16:23       IRQCount
+ *  16:20    11:15       Reserved
+ *  21       10          0
+ *  22       9           UseIntOnEnd
+ *  23       8           LdIRQCnt
+ *  24       7           IRQEn
+ *  25:28    3:6         Reserved
+ *  29       2           IrqErrEn
+ *  30       1           IrqDlyEn
+ *  31       0           IrqCoalEn
  */
 #define RX_IRQ_REG          0x0e            /* rw */
 #define IRQ_COAL        (1 << 0)
@@ -142,13 +151,13 @@ This option defaults to enabled (set) */
 #define IRQ_ERR         (1 << 2)
 #define IRQ_DMAERR      (1 << 7)            /* this is not documented ??? */
 /*
- 0:7      24:31       DltTmrValue
- 8:15     16:23       ClscCntrValue
- 16:17    14:15       Reserved
- 18:21    10:13       ClscCnt
- 22:23    8:9         DlyCnt
- 24:28    3::7        Reserved
-*/
+ *  0:7      24:31       DltTmrValue
+ *  8:15     16:23       ClscCntrValue
+ *  16:17    14:15       Reserved
+ *  18:21    10:13       ClscCnt
+ *  22:23    8:9         DlyCnt
+ *  24:28    3::7        Reserved
+ */
 #define RX_CHNL_STS         0x0f        /* r */
 #define CHNL_STS_ENGBUSY    (1 << 1)
 #define CHNL_STS_EOP        (1 << 2)
@@ -165,23 +174,23 @@ This option defaults to enabled (set) */
 #define CHNL_STS_CMPERR     (1 << 20)
 #define CHNL_STS_TAILERR    (1 << 21)
 /*
- 0:9      22:31   Reserved
- 10       21      TailPErr
- 11       20      CmpErr
- 12       19      AddrErr
- 13       18      NxtPErr
- 14       17      CurPErr
- 15       16      BsyWr
- 16:23    8:15    Reserved
- 24       7       Error
- 25       6       IOE
- 26       5       SOE
- 27       4       Cmplt
- 28       3       SOP
- 29       2       EOP
- 30       1       EngBusy
- 31       0       Reserved
-*/
+ *  0:9      22:31   Reserved
+ *  10       21      TailPErr
+ *  11       20      CmpErr
+ *  12       19      AddrErr
+ *  13       18      NxtPErr
+ *  14       17      CurPErr
+ *  15       16      BsyWr
+ *  16:23    8:15    Reserved
+ *  24       7       Error
+ *  25       6       IOE
+ *  26       5       SOE
+ *  27       4       Cmplt
+ *  28       3       SOP
+ *  29       2       EOP
+ *  30       1       EngBusy
+ *  31       0       Reserved
+ */
 
 #define DMA_CONTROL_REG             0x10            /* rw */
 #define DMA_CONTROL_RST                 (1 << 0)
diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c
index 3f6b9dfca095..1066420d6a83 100644
--- a/drivers/net/ethernet/xilinx/ll_temac_main.c
+++ b/drivers/net/ethernet/xilinx/ll_temac_main.c
@@ -117,8 +117,8 @@ int temac_indirect_busywait(struct temac_local *lp)
 	spin_until_cond(hard_acs_rdy_or_timeout(lp, timeout));
 	if (WARN_ON(!hard_acs_rdy(lp)))
 		return -ETIMEDOUT;
-	else
-		return 0;
+
+	return 0;
 }
 
 /*
@@ -261,7 +261,7 @@ static void temac_dma_dcr_out(struct temac_local *lp, int reg, u32 value)
  * I/O  functions
  */
 static int temac_dcr_setup(struct temac_local *lp, struct platform_device *op,
-				struct device_node *np)
+			   struct device_node *np)
 {
 	unsigned int dcrs;
 
@@ -286,7 +286,7 @@ static int temac_dcr_setup(struct temac_local *lp, struct platform_device *op,
  * such as with MicroBlaze and x86
  */
 static int temac_dcr_setup(struct temac_local *lp, struct platform_device *op,
-				struct device_node *np)
+			   struct device_node *np)
 {
 	return -1;
 }
@@ -307,11 +307,9 @@ static void temac_dma_bd_release(struct net_device *ndev)
 	for (i = 0; i < lp->rx_bd_num; i++) {
 		if (!lp->rx_skb[i])
 			break;
-		else {
-			dma_unmap_single(ndev->dev.parent, lp->rx_bd_v[i].phys,
-					XTE_MAX_JUMBO_FRAME_SIZE, DMA_FROM_DEVICE);
-			dev_kfree_skb(lp->rx_skb[i]);
-		}
+		dma_unmap_single(ndev->dev.parent, lp->rx_bd_v[i].phys,
+				 XTE_MAX_JUMBO_FRAME_SIZE, DMA_FROM_DEVICE);
+		dev_kfree_skb(lp->rx_skb[i]);
 	}
 	if (lp->rx_bd_v)
 		dma_free_coherent(ndev->dev.parent,
@@ -430,7 +428,8 @@ static void temac_do_set_mac_address(struct net_device *ndev)
 				    (ndev->dev_addr[2] << 16) |
 				    (ndev->dev_addr[3] << 24));
 	/* There are reserved bits in EUAW1
-	 * so don't affect them Set MAC bits [47:32] in EUAW1 */
+	 * so don't affect them Set MAC bits [47:32] in EUAW1
+	 */
 	temac_indirect_out32_locked(lp, XTE_UAW1_OFFSET,
 				    (ndev->dev_addr[4] & 0x000000ff) |
 				    (ndev->dev_addr[5] << 8));
@@ -530,66 +529,66 @@ static struct temac_option {
 	{
 		.opt = XTE_OPTION_JUMBO,
 		.reg = XTE_RXC1_OFFSET,
-		.m_or =XTE_RXC1_RXJMBO_MASK,
+		.m_or = XTE_RXC1_RXJMBO_MASK,
 	},
 	/* Turn on VLAN packet support for both Rx and Tx */
 	{
 		.opt = XTE_OPTION_VLAN,
 		.reg = XTE_TXC_OFFSET,
-		.m_or =XTE_TXC_TXVLAN_MASK,
+		.m_or = XTE_TXC_TXVLAN_MASK,
 	},
 	{
 		.opt = XTE_OPTION_VLAN,
 		.reg = XTE_RXC1_OFFSET,
-		.m_or =XTE_RXC1_RXVLAN_MASK,
+		.m_or = XTE_RXC1_RXVLAN_MASK,
 	},
 	/* Turn on FCS stripping on receive packets */
 	{
 		.opt = XTE_OPTION_FCS_STRIP,
 		.reg = XTE_RXC1_OFFSET,
-		.m_or =XTE_RXC1_RXFCS_MASK,
+		.m_or = XTE_RXC1_RXFCS_MASK,
 	},
 	/* Turn on FCS insertion on transmit packets */
 	{
 		.opt = XTE_OPTION_FCS_INSERT,
 		.reg = XTE_TXC_OFFSET,
-		.m_or =XTE_TXC_TXFCS_MASK,
+		.m_or = XTE_TXC_TXFCS_MASK,
 	},
 	/* Turn on length/type field checking on receive packets */
 	{
 		.opt = XTE_OPTION_LENTYPE_ERR,
 		.reg = XTE_RXC1_OFFSET,
-		.m_or =XTE_RXC1_RXLT_MASK,
+		.m_or = XTE_RXC1_RXLT_MASK,
 	},
 	/* Turn on flow control */
 	{
 		.opt = XTE_OPTION_FLOW_CONTROL,
 		.reg = XTE_FCC_OFFSET,
-		.m_or =XTE_FCC_RXFLO_MASK,
+		.m_or = XTE_FCC_RXFLO_MASK,
 	},
 	/* Turn on flow control */
 	{
 		.opt = XTE_OPTION_FLOW_CONTROL,
 		.reg = XTE_FCC_OFFSET,
-		.m_or =XTE_FCC_TXFLO_MASK,
+		.m_or = XTE_FCC_TXFLO_MASK,
 	},
 	/* Turn on promiscuous frame filtering (all frames are received ) */
 	{
 		.opt = XTE_OPTION_PROMISC,
 		.reg = XTE_AFM_OFFSET,
-		.m_or =XTE_AFM_EPPRM_MASK,
+		.m_or = XTE_AFM_EPPRM_MASK,
 	},
 	/* Enable transmitter if not already enabled */
 	{
 		.opt = XTE_OPTION_TXEN,
 		.reg = XTE_TXC_OFFSET,
-		.m_or =XTE_TXC_TXEN_MASK,
+		.m_or = XTE_TXC_TXEN_MASK,
 	},
 	/* Enable receiver? */
 	{
 		.opt = XTE_OPTION_RXEN,
 		.reg = XTE_RXC1_OFFSET,
-		.m_or =XTE_RXC1_RXEN_MASK,
+		.m_or = XTE_RXC1_RXEN_MASK,
 	},
 	{}
 };
@@ -641,7 +640,7 @@ static void temac_device_reset(struct net_device *ndev)
 		udelay(1);
 		if (--timeout == 0) {
 			dev_err(&ndev->dev,
-				"temac_device_reset RX reset timeout!!\n");
+				"%s RX reset timeout!!\n", __func__);
 			break;
 		}
 	}
@@ -653,7 +652,7 @@ static void temac_device_reset(struct net_device *ndev)
 		udelay(1);
 		if (--timeout == 0) {
 			dev_err(&ndev->dev,
-				"temac_device_reset TX reset timeout!!\n");
+				"%s TX reset timeout!!\n", __func__);
 			break;
 		}
 	}
@@ -672,7 +671,7 @@ static void temac_device_reset(struct net_device *ndev)
 		udelay(1);
 		if (--timeout == 0) {
 			dev_err(&ndev->dev,
-				"temac_device_reset DMA reset timeout!!\n");
+				"%s DMA reset timeout!!\n", __func__);
 			break;
 		}
 	}
@@ -680,7 +679,7 @@ static void temac_device_reset(struct net_device *ndev)
 
 	if (temac_dma_bd_init(ndev)) {
 		dev_err(&ndev->dev,
-				"temac_device_reset descriptor allocation failed\n");
+			"%s descriptor allocation failed\n", __func__);
 	}
 
 	spin_lock_irqsave(lp->indirect_lock, flags);
@@ -691,7 +690,8 @@ static void temac_device_reset(struct net_device *ndev)
 	spin_unlock_irqrestore(lp->indirect_lock, flags);
 
 	/* Sync default options with HW
-	 * but leave receiver and transmitter disabled.  */
+	 * but leave receiver and transmitter disabled.
+	 */
 	temac_setoptions(ndev,
 			 lp->options & ~(XTE_OPTION_TXEN | XTE_OPTION_RXEN));
 
@@ -723,9 +723,15 @@ static void temac_adjust_link(struct net_device *ndev)
 		mii_speed &= ~XTE_EMCFG_LINKSPD_MASK;
 
 		switch (phy->speed) {
-		case SPEED_1000: mii_speed |= XTE_EMCFG_LINKSPD_1000; break;
-		case SPEED_100: mii_speed |= XTE_EMCFG_LINKSPD_100; break;
-		case SPEED_10: mii_speed |= XTE_EMCFG_LINKSPD_10; break;
+		case SPEED_1000:
+			mii_speed |= XTE_EMCFG_LINKSPD_1000;
+			break;
+		case SPEED_100:
+			mii_speed |= XTE_EMCFG_LINKSPD_100;
+			break;
+		case SPEED_10:
+			mii_speed |= XTE_EMCFG_LINKSPD_10;
+			break;
 		}
 
 		/* Write new speed setting out to TEMAC */
@@ -1007,7 +1013,6 @@ static void ll_temac_recv(struct net_device *ndev)
 		if (((lp->temac_features & TEMAC_FEATURE_RX_CSUM) != 0) &&
 		    (skb->protocol == htons(ETH_P_IP)) &&
 		    (skb->len > 64)) {
-
 			/* Convert from device endianness (be32) to cpu
 			 * endianness, and if necessary swap the bytes
 			 * (back) for proper IP checksum byte order
@@ -1563,16 +1568,12 @@ static int temac_probe(struct platform_device *pdev)
 	}
 
 	/* Error handle returned DMA RX and TX interrupts */
-	if (lp->rx_irq < 0) {
-		if (lp->rx_irq != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "could not get DMA RX irq\n");
-		return lp->rx_irq;
-	}
-	if (lp->tx_irq < 0) {
-		if (lp->tx_irq != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "could not get DMA TX irq\n");
-		return lp->tx_irq;
-	}
+	if (lp->rx_irq < 0)
+		return dev_err_probe(&pdev->dev, lp->rx_irq,
+				     "could not get DMA RX irq\n");
+	if (lp->tx_irq < 0)
+		return dev_err_probe(&pdev->dev, lp->tx_irq,
+				     "could not get DMA TX irq\n");
 
 	if (temac_np) {
 		/* Retrieve the MAC address */
diff --git a/drivers/net/ethernet/xilinx/ll_temac_mdio.c b/drivers/net/ethernet/xilinx/ll_temac_mdio.c
index 6fd2dea4e60f..2371c072b53f 100644
--- a/drivers/net/ethernet/xilinx/ll_temac_mdio.c
+++ b/drivers/net/ethernet/xilinx/ll_temac_mdio.c
@@ -29,7 +29,8 @@ static int temac_mdio_read(struct mii_bus *bus, int phy_id, int reg)
 
 	/* Write the PHY address to the MIIM Access Initiator register.
 	 * When the transfer completes, the PHY register value will appear
-	 * in the LSW0 register */
+	 * in the LSW0 register
+	 */
 	spin_lock_irqsave(lp->indirect_lock, flags);
 	temac_iow(lp, XTE_LSW0_OFFSET, (phy_id << 5) | reg);
 	rc = temac_indirect_in32_locked(lp, XTE_MIIMAI_OFFSET);
@@ -88,7 +89,8 @@ int temac_mdio_setup(struct temac_local *lp, struct platform_device *pdev)
 	}
 
 	/* Enable the MDIO bus by asserting the enable bit and writing
-	 * in the clock config */
+	 * in the clock config
+	 */
 	temac_indirect_out32(lp, XTE_MC_OFFSET, 1 << 6 | clk_div);
 
 	bus = devm_mdiobus_alloc(&pdev->dev);
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet.h b/drivers/net/ethernet/xilinx/xilinx_axienet.h
index f2e2261b4b7d..6370c447ac5c 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet.h
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet.h
@@ -402,6 +402,9 @@ struct axidma_bd {
  * @rx_bd_num:	Size of RX buffer descriptor ring
  * @rx_bd_ci:	Stores the index of the Rx buffer descriptor in the ring being
  *		accessed currently.
+ * @rx_packets: RX packet count for statistics
+ * @rx_bytes:	RX byte count for statistics
+ * @rx_stat_sync: Synchronization object for RX stats
  * @napi_tx:	NAPI TX control structure
  * @tx_dma_cr:  Nominal content of TX DMA control register
  * @tx_bd_v:	Virtual address of the TX buffer descriptor ring
@@ -411,6 +414,9 @@ struct axidma_bd {
  *		complete. Only updated at runtime by TX NAPI poll.
  * @tx_bd_tail:	Stores the index of the next Tx buffer descriptor in the ring
  *              to be populated.
+ * @tx_packets: TX packet count for statistics
+ * @tx_bytes:	TX byte count for statistics
+ * @tx_stat_sync: Synchronization object for TX stats
  * @dma_err_task: Work structure to process Axi DMA errors
  * @tx_irq:	Axidma TX IRQ number
  * @rx_irq:	Axidma RX IRQ number
@@ -458,6 +464,9 @@ struct axienet_local {
 	dma_addr_t rx_bd_p;
 	u32 rx_bd_num;
 	u32 rx_bd_ci;
+	u64_stats_t rx_packets;
+	u64_stats_t rx_bytes;
+	struct u64_stats_sync rx_stat_sync;
 
 	struct napi_struct napi_tx;
 	u32 tx_dma_cr;
@@ -466,6 +475,9 @@ struct axienet_local {
 	u32 tx_bd_num;
 	u32 tx_bd_ci;
 	u32 tx_bd_tail;
+	u64_stats_t tx_packets;
+	u64_stats_t tx_bytes;
+	struct u64_stats_sync tx_stat_sync;
 
 	struct work_struct dma_err_task;
 
@@ -591,7 +603,7 @@ static inline void axienet_dma_out_addr(struct axienet_local *lp, off_t reg,
 #else /* CONFIG_64BIT */
 
 static inline void axienet_dma_out_addr(struct axienet_local *lp, off_t reg,
-				 dma_addr_t addr)
+					dma_addr_t addr)
 {
 	axienet_dma_out32(lp, reg, lower_32_bits(addr));
 }
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
index 1760930ec0c4..d1d772580da9 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
@@ -597,7 +597,7 @@ static int axienet_device_reset(struct net_device *ndev)
 	lp->options &= (~XAE_OPTION_JUMBO);
 
 	if ((ndev->mtu > XAE_MTU) &&
-		(ndev->mtu <= XAE_JUMBO_MTU)) {
+	    (ndev->mtu <= XAE_JUMBO_MTU)) {
 		lp->max_frm_size = ndev->mtu + VLAN_ETH_HLEN +
 					XAE_TRL_SIZE;
 
@@ -645,7 +645,7 @@ static int axienet_device_reset(struct net_device *ndev)
  * @nr_bds:	Max number of descriptors to clean up
  * @force:	Whether to clean descriptors even if not complete
  * @sizep:	Pointer to a u32 filled with the total sum of all bytes
- * 		in all cleaned-up descriptors. Ignored if NULL.
+ *		in all cleaned-up descriptors. Ignored if NULL.
  * @budget:	NAPI budget (use 0 when not called from NAPI poll)
  *
  * Would either be called after a successful transmit operation, or after
@@ -752,8 +752,10 @@ static int axienet_tx_poll(struct napi_struct *napi, int budget)
 		if (lp->tx_bd_ci >= lp->tx_bd_num)
 			lp->tx_bd_ci %= lp->tx_bd_num;
 
-		ndev->stats.tx_packets += packets;
-		ndev->stats.tx_bytes += size;
+		u64_stats_update_begin(&lp->tx_stat_sync);
+		u64_stats_add(&lp->tx_packets, packets);
+		u64_stats_add(&lp->tx_bytes, size);
+		u64_stats_update_end(&lp->tx_stat_sync);
 
 		/* Matches barrier in axienet_start_xmit */
 		smp_mb();
@@ -984,8 +986,10 @@ static int axienet_rx_poll(struct napi_struct *napi, int budget)
 		cur_p = &lp->rx_bd_v[lp->rx_bd_ci];
 	}
 
-	lp->ndev->stats.rx_packets += packets;
-	lp->ndev->stats.rx_bytes += size;
+	u64_stats_update_begin(&lp->rx_stat_sync);
+	u64_stats_add(&lp->rx_packets, packets);
+	u64_stats_add(&lp->rx_bytes, size);
+	u64_stats_update_end(&lp->rx_stat_sync);
 
 	if (tail_p)
 		axienet_dma_out_addr(lp, XAXIDMA_RX_TDESC_OFFSET, tail_p);
@@ -1292,10 +1296,32 @@ static int axienet_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
 	return phylink_mii_ioctl(lp->phylink, rq, cmd);
 }
 
+static void
+axienet_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
+{
+	struct axienet_local *lp = netdev_priv(dev);
+	unsigned int start;
+
+	netdev_stats_to_stats64(stats, &dev->stats);
+
+	do {
+		start = u64_stats_fetch_begin_irq(&lp->rx_stat_sync);
+		stats->rx_packets = u64_stats_read(&lp->rx_packets);
+		stats->rx_bytes = u64_stats_read(&lp->rx_bytes);
+	} while (u64_stats_fetch_retry_irq(&lp->rx_stat_sync, start));
+
+	do {
+		start = u64_stats_fetch_begin_irq(&lp->tx_stat_sync);
+		stats->tx_packets = u64_stats_read(&lp->tx_packets);
+		stats->tx_bytes = u64_stats_read(&lp->tx_bytes);
+	} while (u64_stats_fetch_retry_irq(&lp->tx_stat_sync, start));
+}
+
 static const struct net_device_ops axienet_netdev_ops = {
 	.ndo_open = axienet_open,
 	.ndo_stop = axienet_stop,
 	.ndo_start_xmit = axienet_start_xmit,
+	.ndo_get_stats64 = axienet_get_stats64,
 	.ndo_change_mtu	= axienet_change_mtu,
 	.ndo_set_mac_address = netdev_set_mac_address,
 	.ndo_validate_addr = eth_validate_addr,
@@ -1317,8 +1343,8 @@ static const struct net_device_ops axienet_netdev_ops = {
 static void axienet_ethtools_get_drvinfo(struct net_device *ndev,
 					 struct ethtool_drvinfo *ed)
 {
-	strlcpy(ed->driver, DRIVER_NAME, sizeof(ed->driver));
-	strlcpy(ed->version, DRIVER_VERSION, sizeof(ed->version));
+	strscpy(ed->driver, DRIVER_NAME, sizeof(ed->driver));
+	strscpy(ed->version, DRIVER_VERSION, sizeof(ed->version));
 }
 
 /**
@@ -1349,7 +1375,7 @@ static int axienet_ethtools_get_regs_len(struct net_device *ndev)
 static void axienet_ethtools_get_regs(struct net_device *ndev,
 				      struct ethtool_regs *regs, void *ret)
 {
-	u32 *data = (u32 *) ret;
+	u32 *data = (u32 *)ret;
 	size_t len = sizeof(u32) * AXIENET_REGS_N;
 	struct axienet_local *lp = netdev_priv(ndev);
 
@@ -1850,8 +1876,11 @@ static int axienet_probe(struct platform_device *pdev)
 	lp->rx_bd_num = RX_BD_NUM_DEFAULT;
 	lp->tx_bd_num = TX_BD_NUM_DEFAULT;
 
-	netif_napi_add(ndev, &lp->napi_rx, axienet_rx_poll, NAPI_POLL_WEIGHT);
-	netif_napi_add(ndev, &lp->napi_tx, axienet_tx_poll, NAPI_POLL_WEIGHT);
+	u64_stats_init(&lp->rx_stat_sync);
+	u64_stats_init(&lp->tx_stat_sync);
+
+	netif_napi_add(ndev, &lp->napi_rx, axienet_rx_poll);
+	netif_napi_add(ndev, &lp->napi_tx, axienet_tx_poll);
 
 	lp->axi_clk = devm_clk_get_optional(&pdev->dev, "s_axi_lite_clk");
 	if (!lp->axi_clk) {
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_mdio.c b/drivers/net/ethernet/xilinx/xilinx_axienet_mdio.c
index 2772a79cd3ed..0b3b6935c558 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet_mdio.c
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet_mdio.c
@@ -126,7 +126,7 @@ static int axienet_mdio_write(struct mii_bus *bus, int phy_id, int reg,
 		return ret;
 	}
 
-	axienet_iow(lp, XAE_MDIO_MWD_OFFSET, (u32) val);
+	axienet_iow(lp, XAE_MDIO_MWD_OFFSET, (u32)val);
 	axienet_iow(lp, XAE_MDIO_MCR_OFFSET,
 		    (((phy_id << XAE_MDIO_MCR_PHYAD_SHIFT) &
 		      XAE_MDIO_MCR_PHYAD_MASK) |
diff --git a/drivers/net/ethernet/xilinx/xilinx_emaclite.c b/drivers/net/ethernet/xilinx/xilinx_emaclite.c
index 016a9c4f2c6c..05848ff15fb5 100644
--- a/drivers/net/ethernet/xilinx/xilinx_emaclite.c
+++ b/drivers/net/ethernet/xilinx/xilinx_emaclite.c
@@ -1060,7 +1060,7 @@ static bool get_bool(struct platform_device *ofdev, const char *s)
 static void xemaclite_ethtools_get_drvinfo(struct net_device *ndev,
 					   struct ethtool_drvinfo *ed)
 {
-	strlcpy(ed->driver, DRIVER_NAME, sizeof(ed->driver));
+	strscpy(ed->driver, DRIVER_NAME, sizeof(ed->driver));
 }
 
 static const struct ethtool_ops xemaclite_ethtool_ops = {
diff --git a/drivers/net/ethernet/xircom/xirc2ps_cs.c b/drivers/net/ethernet/xircom/xirc2ps_cs.c
index f9587e55b842..894e92ef415b 100644
--- a/drivers/net/ethernet/xircom/xirc2ps_cs.c
+++ b/drivers/net/ethernet/xircom/xirc2ps_cs.c
@@ -1402,7 +1402,7 @@ do_open(struct net_device *dev)
 static void netdev_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, "xirc2ps_cs", sizeof(info->driver));
+	strscpy(info->driver, "xirc2ps_cs", sizeof(info->driver));
 	snprintf(info->bus_info, sizeof(info->bus_info), "PCMCIA 0x%lx",
 		 dev->base_addr);
 }
diff --git a/drivers/net/ethernet/xscale/ixp4xx_eth.c b/drivers/net/ethernet/xscale/ixp4xx_eth.c
index 3591b9edc9a1..3b0c5f177447 100644
--- a/drivers/net/ethernet/xscale/ixp4xx_eth.c
+++ b/drivers/net/ethernet/xscale/ixp4xx_eth.c
@@ -841,7 +841,7 @@ static void eth_txdone_irq(void *unused)
 	}
 }
 
-static int eth_xmit(struct sk_buff *skb, struct net_device *dev)
+static netdev_tx_t eth_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct port *port = netdev_priv(dev);
 	unsigned int txreadyq = port->plat->txreadyq;
@@ -999,11 +999,11 @@ static void ixp4xx_get_drvinfo(struct net_device *dev,
 {
 	struct port *port = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
 	snprintf(info->fw_version, sizeof(info->fw_version), "%u:%u:%u:%u",
 		 port->firmware[0], port->firmware[1],
 		 port->firmware[2], port->firmware[3]);
-	strlcpy(info->bus_info, "internal", sizeof(info->bus_info));
+	strscpy(info->bus_info, "internal", sizeof(info->bus_info));
 }
 
 static int ixp4xx_get_ts_info(struct net_device *dev,
diff --git a/drivers/net/fjes/fjes_ethtool.c b/drivers/net/fjes/fjes_ethtool.c
index 746736c83873..19c99529566b 100644
--- a/drivers/net/fjes/fjes_ethtool.c
+++ b/drivers/net/fjes/fjes_ethtool.c
@@ -151,11 +151,11 @@ static void fjes_get_drvinfo(struct net_device *netdev,
 
 	plat_dev = adapter->plat_dev;
 
-	strlcpy(drvinfo->driver, fjes_driver_name, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, fjes_driver_version,
+	strscpy(drvinfo->driver, fjes_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, fjes_driver_version,
 		sizeof(drvinfo->version));
 
-	strlcpy(drvinfo->fw_version, "none", sizeof(drvinfo->fw_version));
+	strscpy(drvinfo->fw_version, "none", sizeof(drvinfo->fw_version));
 	snprintf(drvinfo->bus_info, sizeof(drvinfo->bus_info),
 		 "platform:%s", plat_dev->name);
 }
diff --git a/drivers/net/fjes/fjes_main.c b/drivers/net/fjes/fjes_main.c
index 5805e4a56385..1eff202f6a1f 100644
--- a/drivers/net/fjes/fjes_main.c
+++ b/drivers/net/fjes/fjes_main.c
@@ -32,68 +32,12 @@ MODULE_VERSION(DRV_VERSION);
 
 #define ACPI_MOTHERBOARD_RESOURCE_HID "PNP0C02"
 
-static int fjes_request_irq(struct fjes_adapter *);
-static void fjes_free_irq(struct fjes_adapter *);
-
-static int fjes_open(struct net_device *);
-static int fjes_close(struct net_device *);
-static int fjes_setup_resources(struct fjes_adapter *);
-static void fjes_free_resources(struct fjes_adapter *);
-static netdev_tx_t fjes_xmit_frame(struct sk_buff *, struct net_device *);
-static void fjes_raise_intr_rxdata_task(struct work_struct *);
-static void fjes_tx_stall_task(struct work_struct *);
-static void fjes_force_close_task(struct work_struct *);
-static irqreturn_t fjes_intr(int, void*);
-static void fjes_get_stats64(struct net_device *, struct rtnl_link_stats64 *);
-static int fjes_change_mtu(struct net_device *, int);
-static int fjes_vlan_rx_add_vid(struct net_device *, __be16 proto, u16);
-static int fjes_vlan_rx_kill_vid(struct net_device *, __be16 proto, u16);
-static void fjes_tx_retry(struct net_device *, unsigned int txqueue);
-
-static int fjes_acpi_add(struct acpi_device *);
-static int fjes_acpi_remove(struct acpi_device *);
-static acpi_status fjes_get_acpi_resource(struct acpi_resource *, void*);
-
-static int fjes_probe(struct platform_device *);
-static int fjes_remove(struct platform_device *);
-
-static int fjes_sw_init(struct fjes_adapter *);
-static void fjes_netdev_setup(struct net_device *);
-static void fjes_irq_watch_task(struct work_struct *);
-static void fjes_watch_unshare_task(struct work_struct *);
-static void fjes_rx_irq(struct fjes_adapter *, int);
-static int fjes_poll(struct napi_struct *, int);
-
 static const struct acpi_device_id fjes_acpi_ids[] = {
 	{ACPI_MOTHERBOARD_RESOURCE_HID, 0},
 	{"", 0},
 };
 MODULE_DEVICE_TABLE(acpi, fjes_acpi_ids);
 
-static struct acpi_driver fjes_acpi_driver = {
-	.name = DRV_NAME,
-	.class = DRV_NAME,
-	.owner = THIS_MODULE,
-	.ids = fjes_acpi_ids,
-	.ops = {
-		.add = fjes_acpi_add,
-		.remove = fjes_acpi_remove,
-	},
-};
-
-static struct platform_driver fjes_driver = {
-	.driver = {
-		.name = DRV_NAME,
-	},
-	.probe = fjes_probe,
-	.remove = fjes_remove,
-};
-
-static struct resource fjes_resource[] = {
-	DEFINE_RES_MEM(0, 1),
-	DEFINE_RES_IRQ(0)
-};
-
 static bool is_extended_socket_device(struct acpi_device *device)
 {
 	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL};
@@ -139,43 +83,6 @@ static int acpi_check_extended_socket_status(struct acpi_device *device)
 	return 0;
 }
 
-static int fjes_acpi_add(struct acpi_device *device)
-{
-	struct platform_device *plat_dev;
-	acpi_status status;
-
-	if (!is_extended_socket_device(device))
-		return -ENODEV;
-
-	if (acpi_check_extended_socket_status(device))
-		return -ENODEV;
-
-	status = acpi_walk_resources(device->handle, METHOD_NAME__CRS,
-				     fjes_get_acpi_resource, fjes_resource);
-	if (ACPI_FAILURE(status))
-		return -ENODEV;
-
-	/* create platform_device */
-	plat_dev = platform_device_register_simple(DRV_NAME, 0, fjes_resource,
-						   ARRAY_SIZE(fjes_resource));
-	if (IS_ERR(plat_dev))
-		return PTR_ERR(plat_dev);
-
-	device->driver_data = plat_dev;
-
-	return 0;
-}
-
-static int fjes_acpi_remove(struct acpi_device *device)
-{
-	struct platform_device *plat_dev;
-
-	plat_dev = (struct platform_device *)acpi_driver_data(device);
-	platform_device_unregister(plat_dev);
-
-	return 0;
-}
-
 static acpi_status
 fjes_get_acpi_resource(struct acpi_resource *acpi_res, void *data)
 {
@@ -206,143 +113,59 @@ fjes_get_acpi_resource(struct acpi_resource *acpi_res, void *data)
 	return AE_OK;
 }
 
-static int fjes_request_irq(struct fjes_adapter *adapter)
-{
-	struct net_device *netdev = adapter->netdev;
-	int result = -1;
-
-	adapter->interrupt_watch_enable = true;
-	if (!delayed_work_pending(&adapter->interrupt_watch_task)) {
-		queue_delayed_work(adapter->control_wq,
-				   &adapter->interrupt_watch_task,
-				   FJES_IRQ_WATCH_DELAY);
-	}
-
-	if (!adapter->irq_registered) {
-		result = request_irq(adapter->hw.hw_res.irq, fjes_intr,
-				     IRQF_SHARED, netdev->name, adapter);
-		if (result)
-			adapter->irq_registered = false;
-		else
-			adapter->irq_registered = true;
-	}
-
-	return result;
-}
-
-static void fjes_free_irq(struct fjes_adapter *adapter)
-{
-	struct fjes_hw *hw = &adapter->hw;
-
-	adapter->interrupt_watch_enable = false;
-	cancel_delayed_work_sync(&adapter->interrupt_watch_task);
-
-	fjes_hw_set_irqmask(hw, REG_ICTL_MASK_ALL, true);
-
-	if (adapter->irq_registered) {
-		free_irq(adapter->hw.hw_res.irq, adapter);
-		adapter->irq_registered = false;
-	}
-}
-
-static const struct net_device_ops fjes_netdev_ops = {
-	.ndo_open		= fjes_open,
-	.ndo_stop		= fjes_close,
-	.ndo_start_xmit		= fjes_xmit_frame,
-	.ndo_get_stats64	= fjes_get_stats64,
-	.ndo_change_mtu		= fjes_change_mtu,
-	.ndo_tx_timeout		= fjes_tx_retry,
-	.ndo_vlan_rx_add_vid	= fjes_vlan_rx_add_vid,
-	.ndo_vlan_rx_kill_vid = fjes_vlan_rx_kill_vid,
+static struct resource fjes_resource[] = {
+	DEFINE_RES_MEM(0, 1),
+	DEFINE_RES_IRQ(0)
 };
 
-/* fjes_open - Called when a network interface is made active */
-static int fjes_open(struct net_device *netdev)
+static int fjes_acpi_add(struct acpi_device *device)
 {
-	struct fjes_adapter *adapter = netdev_priv(netdev);
-	struct fjes_hw *hw = &adapter->hw;
-	int result;
-
-	if (adapter->open_guard)
-		return -ENXIO;
-
-	result = fjes_setup_resources(adapter);
-	if (result)
-		goto err_setup_res;
-
-	hw->txrx_stop_req_bit = 0;
-	hw->epstop_req_bit = 0;
+	struct platform_device *plat_dev;
+	acpi_status status;
 
-	napi_enable(&adapter->napi);
+	if (!is_extended_socket_device(device))
+		return -ENODEV;
 
-	fjes_hw_capture_interrupt_status(hw);
+	if (acpi_check_extended_socket_status(device))
+		return -ENODEV;
 
-	result = fjes_request_irq(adapter);
-	if (result)
-		goto err_req_irq;
+	status = acpi_walk_resources(device->handle, METHOD_NAME__CRS,
+				     fjes_get_acpi_resource, fjes_resource);
+	if (ACPI_FAILURE(status))
+		return -ENODEV;
 
-	fjes_hw_set_irqmask(hw, REG_ICTL_MASK_ALL, false);
+	/* create platform_device */
+	plat_dev = platform_device_register_simple(DRV_NAME, 0, fjes_resource,
+						   ARRAY_SIZE(fjes_resource));
+	if (IS_ERR(plat_dev))
+		return PTR_ERR(plat_dev);
 
-	netif_tx_start_all_queues(netdev);
-	netif_carrier_on(netdev);
+	device->driver_data = plat_dev;
 
 	return 0;
-
-err_req_irq:
-	fjes_free_irq(adapter);
-	napi_disable(&adapter->napi);
-
-err_setup_res:
-	fjes_free_resources(adapter);
-	return result;
 }
 
-/* fjes_close - Disables a network interface */
-static int fjes_close(struct net_device *netdev)
+static int fjes_acpi_remove(struct acpi_device *device)
 {
-	struct fjes_adapter *adapter = netdev_priv(netdev);
-	struct fjes_hw *hw = &adapter->hw;
-	unsigned long flags;
-	int epidx;
-
-	netif_tx_stop_all_queues(netdev);
-	netif_carrier_off(netdev);
-
-	fjes_hw_raise_epstop(hw);
-
-	napi_disable(&adapter->napi);
-
-	spin_lock_irqsave(&hw->rx_status_lock, flags);
-	for (epidx = 0; epidx < hw->max_epid; epidx++) {
-		if (epidx == hw->my_epid)
-			continue;
-
-		if (fjes_hw_get_partner_ep_status(hw, epidx) ==
-		    EP_PARTNER_SHARED)
-			adapter->hw.ep_shm_info[epidx]
-				   .tx.info->v1i.rx_status &=
-				~FJES_RX_POLL_WORK;
-	}
-	spin_unlock_irqrestore(&hw->rx_status_lock, flags);
-
-	fjes_free_irq(adapter);
-
-	cancel_delayed_work_sync(&adapter->interrupt_watch_task);
-	cancel_work_sync(&adapter->unshare_watch_task);
-	adapter->unshare_watch_bitmask = 0;
-	cancel_work_sync(&adapter->raise_intr_rxdata_task);
-	cancel_work_sync(&adapter->tx_stall_task);
-
-	cancel_work_sync(&hw->update_zone_task);
-	cancel_work_sync(&hw->epstop_task);
-
-	fjes_hw_wait_epstop(hw);
+	struct platform_device *plat_dev;
 
-	fjes_free_resources(adapter);
+	plat_dev = (struct platform_device *)acpi_driver_data(device);
+	platform_device_unregister(plat_dev);
 
 	return 0;
 }
 
+static struct acpi_driver fjes_acpi_driver = {
+	.name = DRV_NAME,
+	.class = DRV_NAME,
+	.owner = THIS_MODULE,
+	.ids = fjes_acpi_ids,
+	.ops = {
+		.add = fjes_acpi_add,
+		.remove = fjes_acpi_remove,
+	},
+};
+
 static int fjes_setup_resources(struct fjes_adapter *adapter)
 {
 	struct net_device *netdev = adapter->netdev;
@@ -421,6 +244,188 @@ static int fjes_setup_resources(struct fjes_adapter *adapter)
 	return 0;
 }
 
+static void fjes_rx_irq(struct fjes_adapter *adapter, int src_epid)
+{
+	struct fjes_hw *hw = &adapter->hw;
+
+	fjes_hw_set_irqmask(hw, REG_ICTL_MASK_RX_DATA, true);
+
+	adapter->unset_rx_last = true;
+	napi_schedule(&adapter->napi);
+}
+
+static void fjes_stop_req_irq(struct fjes_adapter *adapter, int src_epid)
+{
+	struct fjes_hw *hw = &adapter->hw;
+	enum ep_partner_status status;
+	unsigned long flags;
+
+	set_bit(src_epid, &hw->hw_info.buffer_unshare_reserve_bit);
+
+	status = fjes_hw_get_partner_ep_status(hw, src_epid);
+	trace_fjes_stop_req_irq_pre(hw, src_epid, status);
+	switch (status) {
+	case EP_PARTNER_WAITING:
+		spin_lock_irqsave(&hw->rx_status_lock, flags);
+		hw->ep_shm_info[src_epid].tx.info->v1i.rx_status |=
+				FJES_RX_STOP_REQ_DONE;
+		spin_unlock_irqrestore(&hw->rx_status_lock, flags);
+		clear_bit(src_epid, &hw->txrx_stop_req_bit);
+		fallthrough;
+	case EP_PARTNER_UNSHARE:
+	case EP_PARTNER_COMPLETE:
+	default:
+		set_bit(src_epid, &adapter->unshare_watch_bitmask);
+		if (!work_pending(&adapter->unshare_watch_task))
+			queue_work(adapter->control_wq,
+				   &adapter->unshare_watch_task);
+		break;
+	case EP_PARTNER_SHARED:
+		set_bit(src_epid, &hw->epstop_req_bit);
+
+		if (!work_pending(&hw->epstop_task))
+			queue_work(adapter->control_wq, &hw->epstop_task);
+		break;
+	}
+	trace_fjes_stop_req_irq_post(hw, src_epid);
+}
+
+static void fjes_txrx_stop_req_irq(struct fjes_adapter *adapter,
+				   int src_epid)
+{
+	struct fjes_hw *hw = &adapter->hw;
+	enum ep_partner_status status;
+	unsigned long flags;
+
+	status = fjes_hw_get_partner_ep_status(hw, src_epid);
+	trace_fjes_txrx_stop_req_irq_pre(hw, src_epid, status);
+	switch (status) {
+	case EP_PARTNER_UNSHARE:
+	case EP_PARTNER_COMPLETE:
+	default:
+		break;
+	case EP_PARTNER_WAITING:
+		if (src_epid < hw->my_epid) {
+			spin_lock_irqsave(&hw->rx_status_lock, flags);
+			hw->ep_shm_info[src_epid].tx.info->v1i.rx_status |=
+				FJES_RX_STOP_REQ_DONE;
+			spin_unlock_irqrestore(&hw->rx_status_lock, flags);
+
+			clear_bit(src_epid, &hw->txrx_stop_req_bit);
+			set_bit(src_epid, &adapter->unshare_watch_bitmask);
+
+			if (!work_pending(&adapter->unshare_watch_task))
+				queue_work(adapter->control_wq,
+					   &adapter->unshare_watch_task);
+		}
+		break;
+	case EP_PARTNER_SHARED:
+		if (hw->ep_shm_info[src_epid].rx.info->v1i.rx_status &
+		    FJES_RX_STOP_REQ_REQUEST) {
+			set_bit(src_epid, &hw->epstop_req_bit);
+			if (!work_pending(&hw->epstop_task))
+				queue_work(adapter->control_wq,
+					   &hw->epstop_task);
+		}
+		break;
+	}
+	trace_fjes_txrx_stop_req_irq_post(hw, src_epid);
+}
+
+static void fjes_update_zone_irq(struct fjes_adapter *adapter,
+				 int src_epid)
+{
+	struct fjes_hw *hw = &adapter->hw;
+
+	if (!work_pending(&hw->update_zone_task))
+		queue_work(adapter->control_wq, &hw->update_zone_task);
+}
+
+static irqreturn_t fjes_intr(int irq, void *data)
+{
+	struct fjes_adapter *adapter = data;
+	struct fjes_hw *hw = &adapter->hw;
+	irqreturn_t ret;
+	u32 icr;
+
+	icr = fjes_hw_capture_interrupt_status(hw);
+
+	if (icr & REG_IS_MASK_IS_ASSERT) {
+		if (icr & REG_ICTL_MASK_RX_DATA) {
+			fjes_rx_irq(adapter, icr & REG_IS_MASK_EPID);
+			hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats
+				.recv_intr_rx += 1;
+		}
+
+		if (icr & REG_ICTL_MASK_DEV_STOP_REQ) {
+			fjes_stop_req_irq(adapter, icr & REG_IS_MASK_EPID);
+			hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats
+				.recv_intr_stop += 1;
+		}
+
+		if (icr & REG_ICTL_MASK_TXRX_STOP_REQ) {
+			fjes_txrx_stop_req_irq(adapter, icr & REG_IS_MASK_EPID);
+			hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats
+				.recv_intr_unshare += 1;
+		}
+
+		if (icr & REG_ICTL_MASK_TXRX_STOP_DONE)
+			fjes_hw_set_irqmask(hw,
+					    REG_ICTL_MASK_TXRX_STOP_DONE, true);
+
+		if (icr & REG_ICTL_MASK_INFO_UPDATE) {
+			fjes_update_zone_irq(adapter, icr & REG_IS_MASK_EPID);
+			hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats
+				.recv_intr_zoneupdate += 1;
+		}
+
+		ret = IRQ_HANDLED;
+	} else {
+		ret = IRQ_NONE;
+	}
+
+	return ret;
+}
+
+static int fjes_request_irq(struct fjes_adapter *adapter)
+{
+	struct net_device *netdev = adapter->netdev;
+	int result = -1;
+
+	adapter->interrupt_watch_enable = true;
+	if (!delayed_work_pending(&adapter->interrupt_watch_task)) {
+		queue_delayed_work(adapter->control_wq,
+				   &adapter->interrupt_watch_task,
+				   FJES_IRQ_WATCH_DELAY);
+	}
+
+	if (!adapter->irq_registered) {
+		result = request_irq(adapter->hw.hw_res.irq, fjes_intr,
+				     IRQF_SHARED, netdev->name, adapter);
+		if (result)
+			adapter->irq_registered = false;
+		else
+			adapter->irq_registered = true;
+	}
+
+	return result;
+}
+
+static void fjes_free_irq(struct fjes_adapter *adapter)
+{
+	struct fjes_hw *hw = &adapter->hw;
+
+	adapter->interrupt_watch_enable = false;
+	cancel_delayed_work_sync(&adapter->interrupt_watch_task);
+
+	fjes_hw_set_irqmask(hw, REG_ICTL_MASK_ALL, true);
+
+	if (adapter->irq_registered) {
+		free_irq(adapter->hw.hw_res.irq, adapter);
+		adapter->irq_registered = false;
+	}
+}
+
 static void fjes_free_resources(struct fjes_adapter *adapter)
 {
 	struct net_device *netdev = adapter->netdev;
@@ -477,121 +482,91 @@ static void fjes_free_resources(struct fjes_adapter *adapter)
 	}
 }
 
-static void fjes_tx_stall_task(struct work_struct *work)
+/* fjes_open - Called when a network interface is made active */
+static int fjes_open(struct net_device *netdev)
 {
-	struct fjes_adapter *adapter = container_of(work,
-			struct fjes_adapter, tx_stall_task);
-	struct net_device *netdev = adapter->netdev;
+	struct fjes_adapter *adapter = netdev_priv(netdev);
 	struct fjes_hw *hw = &adapter->hw;
-	int all_queue_available, sendable;
-	enum ep_partner_status pstatus;
-	int max_epid, my_epid, epid;
-	union ep_buffer_info *info;
-	int i;
-
-	if (((long)jiffies -
-		dev_trans_start(netdev)) > FJES_TX_TX_STALL_TIMEOUT) {
-		netif_wake_queue(netdev);
-		return;
-	}
-
-	my_epid = hw->my_epid;
-	max_epid = hw->max_epid;
+	int result;
 
-	for (i = 0; i < 5; i++) {
-		all_queue_available = 1;
+	if (adapter->open_guard)
+		return -ENXIO;
 
-		for (epid = 0; epid < max_epid; epid++) {
-			if (my_epid == epid)
-				continue;
+	result = fjes_setup_resources(adapter);
+	if (result)
+		goto err_setup_res;
 
-			pstatus = fjes_hw_get_partner_ep_status(hw, epid);
-			sendable = (pstatus == EP_PARTNER_SHARED);
-			if (!sendable)
-				continue;
+	hw->txrx_stop_req_bit = 0;
+	hw->epstop_req_bit = 0;
 
-			info = adapter->hw.ep_shm_info[epid].tx.info;
+	napi_enable(&adapter->napi);
 
-			if (!(info->v1i.rx_status & FJES_RX_MTU_CHANGING_DONE))
-				return;
+	fjes_hw_capture_interrupt_status(hw);
 
-			if (EP_RING_FULL(info->v1i.head, info->v1i.tail,
-					 info->v1i.count_max)) {
-				all_queue_available = 0;
-				break;
-			}
-		}
+	result = fjes_request_irq(adapter);
+	if (result)
+		goto err_req_irq;
 
-		if (all_queue_available) {
-			netif_wake_queue(netdev);
-			return;
-		}
-	}
+	fjes_hw_set_irqmask(hw, REG_ICTL_MASK_ALL, false);
 
-	usleep_range(50, 100);
+	netif_tx_start_all_queues(netdev);
+	netif_carrier_on(netdev);
 
-	queue_work(adapter->txrx_wq, &adapter->tx_stall_task);
-}
+	return 0;
 
-static void fjes_force_close_task(struct work_struct *work)
-{
-	struct fjes_adapter *adapter = container_of(work,
-			struct fjes_adapter, force_close_task);
-	struct net_device *netdev = adapter->netdev;
+err_req_irq:
+	fjes_free_irq(adapter);
+	napi_disable(&adapter->napi);
 
-	rtnl_lock();
-	dev_close(netdev);
-	rtnl_unlock();
+err_setup_res:
+	fjes_free_resources(adapter);
+	return result;
 }
 
-static void fjes_raise_intr_rxdata_task(struct work_struct *work)
+/* fjes_close - Disables a network interface */
+static int fjes_close(struct net_device *netdev)
 {
-	struct fjes_adapter *adapter = container_of(work,
-			struct fjes_adapter, raise_intr_rxdata_task);
+	struct fjes_adapter *adapter = netdev_priv(netdev);
 	struct fjes_hw *hw = &adapter->hw;
-	enum ep_partner_status pstatus;
-	int max_epid, my_epid, epid;
+	unsigned long flags;
+	int epidx;
 
-	my_epid = hw->my_epid;
-	max_epid = hw->max_epid;
+	netif_tx_stop_all_queues(netdev);
+	netif_carrier_off(netdev);
 
-	for (epid = 0; epid < max_epid; epid++)
-		hw->ep_shm_info[epid].tx_status_work = 0;
+	fjes_hw_raise_epstop(hw);
 
-	for (epid = 0; epid < max_epid; epid++) {
-		if (epid == my_epid)
-			continue;
+	napi_disable(&adapter->napi);
 
-		pstatus = fjes_hw_get_partner_ep_status(hw, epid);
-		if (pstatus == EP_PARTNER_SHARED) {
-			hw->ep_shm_info[epid].tx_status_work =
-				hw->ep_shm_info[epid].tx.info->v1i.tx_status;
+	spin_lock_irqsave(&hw->rx_status_lock, flags);
+	for (epidx = 0; epidx < hw->max_epid; epidx++) {
+		if (epidx == hw->my_epid)
+			continue;
 
-			if (hw->ep_shm_info[epid].tx_status_work ==
-				FJES_TX_DELAY_SEND_PENDING) {
-				hw->ep_shm_info[epid].tx.info->v1i.tx_status =
-					FJES_TX_DELAY_SEND_NONE;
-			}
-		}
+		if (fjes_hw_get_partner_ep_status(hw, epidx) ==
+		    EP_PARTNER_SHARED)
+			adapter->hw.ep_shm_info[epidx]
+				   .tx.info->v1i.rx_status &=
+				~FJES_RX_POLL_WORK;
 	}
+	spin_unlock_irqrestore(&hw->rx_status_lock, flags);
 
-	for (epid = 0; epid < max_epid; epid++) {
-		if (epid == my_epid)
-			continue;
+	fjes_free_irq(adapter);
 
-		pstatus = fjes_hw_get_partner_ep_status(hw, epid);
-		if ((hw->ep_shm_info[epid].tx_status_work ==
-		     FJES_TX_DELAY_SEND_PENDING) &&
-		    (pstatus == EP_PARTNER_SHARED) &&
-		    !(hw->ep_shm_info[epid].rx.info->v1i.rx_status &
-		      FJES_RX_POLL_WORK)) {
-			fjes_hw_raise_interrupt(hw, epid,
-						REG_ICTL_MASK_RX_DATA);
-			hw->ep_shm_info[epid].ep_stats.send_intr_rx += 1;
-		}
-	}
+	cancel_delayed_work_sync(&adapter->interrupt_watch_task);
+	cancel_work_sync(&adapter->unshare_watch_task);
+	adapter->unshare_watch_bitmask = 0;
+	cancel_work_sync(&adapter->raise_intr_rxdata_task);
+	cancel_work_sync(&adapter->tx_stall_task);
 
-	usleep_range(500, 1000);
+	cancel_work_sync(&hw->update_zone_task);
+	cancel_work_sync(&hw->epstop_task);
+
+	fjes_hw_wait_epstop(hw);
+
+	fjes_free_resources(adapter);
+
+	return 0;
 }
 
 static int fjes_tx_send(struct fjes_adapter *adapter, int dest,
@@ -787,13 +762,6 @@ fjes_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
 	return ret;
 }
 
-static void fjes_tx_retry(struct net_device *netdev, unsigned int txqueue)
-{
-	struct netdev_queue *queue = netdev_get_tx_queue(netdev, 0);
-
-	netif_tx_wake_queue(queue);
-}
-
 static void
 fjes_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
 {
@@ -871,6 +839,13 @@ static int fjes_change_mtu(struct net_device *netdev, int new_mtu)
 	return ret;
 }
 
+static void fjes_tx_retry(struct net_device *netdev, unsigned int txqueue)
+{
+	struct netdev_queue *queue = netdev_get_tx_queue(netdev, 0);
+
+	netif_tx_wake_queue(queue);
+}
+
 static int fjes_vlan_rx_add_vid(struct net_device *netdev,
 				__be16 proto, u16 vid)
 {
@@ -907,137 +882,29 @@ static int fjes_vlan_rx_kill_vid(struct net_device *netdev,
 	return 0;
 }
 
-static void fjes_txrx_stop_req_irq(struct fjes_adapter *adapter,
-				   int src_epid)
-{
-	struct fjes_hw *hw = &adapter->hw;
-	enum ep_partner_status status;
-	unsigned long flags;
-
-	status = fjes_hw_get_partner_ep_status(hw, src_epid);
-	trace_fjes_txrx_stop_req_irq_pre(hw, src_epid, status);
-	switch (status) {
-	case EP_PARTNER_UNSHARE:
-	case EP_PARTNER_COMPLETE:
-	default:
-		break;
-	case EP_PARTNER_WAITING:
-		if (src_epid < hw->my_epid) {
-			spin_lock_irqsave(&hw->rx_status_lock, flags);
-			hw->ep_shm_info[src_epid].tx.info->v1i.rx_status |=
-				FJES_RX_STOP_REQ_DONE;
-			spin_unlock_irqrestore(&hw->rx_status_lock, flags);
-
-			clear_bit(src_epid, &hw->txrx_stop_req_bit);
-			set_bit(src_epid, &adapter->unshare_watch_bitmask);
-
-			if (!work_pending(&adapter->unshare_watch_task))
-				queue_work(adapter->control_wq,
-					   &adapter->unshare_watch_task);
-		}
-		break;
-	case EP_PARTNER_SHARED:
-		if (hw->ep_shm_info[src_epid].rx.info->v1i.rx_status &
-		    FJES_RX_STOP_REQ_REQUEST) {
-			set_bit(src_epid, &hw->epstop_req_bit);
-			if (!work_pending(&hw->epstop_task))
-				queue_work(adapter->control_wq,
-					   &hw->epstop_task);
-		}
-		break;
-	}
-	trace_fjes_txrx_stop_req_irq_post(hw, src_epid);
-}
-
-static void fjes_stop_req_irq(struct fjes_adapter *adapter, int src_epid)
-{
-	struct fjes_hw *hw = &adapter->hw;
-	enum ep_partner_status status;
-	unsigned long flags;
-
-	set_bit(src_epid, &hw->hw_info.buffer_unshare_reserve_bit);
-
-	status = fjes_hw_get_partner_ep_status(hw, src_epid);
-	trace_fjes_stop_req_irq_pre(hw, src_epid, status);
-	switch (status) {
-	case EP_PARTNER_WAITING:
-		spin_lock_irqsave(&hw->rx_status_lock, flags);
-		hw->ep_shm_info[src_epid].tx.info->v1i.rx_status |=
-				FJES_RX_STOP_REQ_DONE;
-		spin_unlock_irqrestore(&hw->rx_status_lock, flags);
-		clear_bit(src_epid, &hw->txrx_stop_req_bit);
-		fallthrough;
-	case EP_PARTNER_UNSHARE:
-	case EP_PARTNER_COMPLETE:
-	default:
-		set_bit(src_epid, &adapter->unshare_watch_bitmask);
-		if (!work_pending(&adapter->unshare_watch_task))
-			queue_work(adapter->control_wq,
-				   &adapter->unshare_watch_task);
-		break;
-	case EP_PARTNER_SHARED:
-		set_bit(src_epid, &hw->epstop_req_bit);
-
-		if (!work_pending(&hw->epstop_task))
-			queue_work(adapter->control_wq, &hw->epstop_task);
-		break;
-	}
-	trace_fjes_stop_req_irq_post(hw, src_epid);
-}
-
-static void fjes_update_zone_irq(struct fjes_adapter *adapter,
-				 int src_epid)
-{
-	struct fjes_hw *hw = &adapter->hw;
-
-	if (!work_pending(&hw->update_zone_task))
-		queue_work(adapter->control_wq, &hw->update_zone_task);
-}
+static const struct net_device_ops fjes_netdev_ops = {
+	.ndo_open		= fjes_open,
+	.ndo_stop		= fjes_close,
+	.ndo_start_xmit		= fjes_xmit_frame,
+	.ndo_get_stats64	= fjes_get_stats64,
+	.ndo_change_mtu		= fjes_change_mtu,
+	.ndo_tx_timeout		= fjes_tx_retry,
+	.ndo_vlan_rx_add_vid	= fjes_vlan_rx_add_vid,
+	.ndo_vlan_rx_kill_vid = fjes_vlan_rx_kill_vid,
+};
 
-static irqreturn_t fjes_intr(int irq, void *data)
+/* fjes_netdev_setup - netdevice initialization routine */
+static void fjes_netdev_setup(struct net_device *netdev)
 {
-	struct fjes_adapter *adapter = data;
-	struct fjes_hw *hw = &adapter->hw;
-	irqreturn_t ret;
-	u32 icr;
-
-	icr = fjes_hw_capture_interrupt_status(hw);
-
-	if (icr & REG_IS_MASK_IS_ASSERT) {
-		if (icr & REG_ICTL_MASK_RX_DATA) {
-			fjes_rx_irq(adapter, icr & REG_IS_MASK_EPID);
-			hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats
-				.recv_intr_rx += 1;
-		}
-
-		if (icr & REG_ICTL_MASK_DEV_STOP_REQ) {
-			fjes_stop_req_irq(adapter, icr & REG_IS_MASK_EPID);
-			hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats
-				.recv_intr_stop += 1;
-		}
-
-		if (icr & REG_ICTL_MASK_TXRX_STOP_REQ) {
-			fjes_txrx_stop_req_irq(adapter, icr & REG_IS_MASK_EPID);
-			hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats
-				.recv_intr_unshare += 1;
-		}
-
-		if (icr & REG_ICTL_MASK_TXRX_STOP_DONE)
-			fjes_hw_set_irqmask(hw,
-					    REG_ICTL_MASK_TXRX_STOP_DONE, true);
-
-		if (icr & REG_ICTL_MASK_INFO_UPDATE) {
-			fjes_update_zone_irq(adapter, icr & REG_IS_MASK_EPID);
-			hw->ep_shm_info[icr & REG_IS_MASK_EPID].ep_stats
-				.recv_intr_zoneupdate += 1;
-		}
-
-		ret = IRQ_HANDLED;
-	} else {
-		ret = IRQ_NONE;
-	}
+	ether_setup(netdev);
 
-	return ret;
+	netdev->watchdog_timeo = FJES_TX_RETRY_INTERVAL;
+	netdev->netdev_ops = &fjes_netdev_ops;
+	fjes_set_ethtool_ops(netdev);
+	netdev->mtu = fjes_support_mtu[3];
+	netdev->min_mtu = fjes_support_mtu[0];
+	netdev->max_mtu = fjes_support_mtu[3];
+	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
 }
 
 static int fjes_rxframe_search_exist(struct fjes_adapter *adapter,
@@ -1087,16 +954,6 @@ static void fjes_rxframe_release(struct fjes_adapter *adapter, int cur_epid)
 	fjes_hw_epbuf_rx_curpkt_drop(&adapter->hw.ep_shm_info[cur_epid].rx);
 }
 
-static void fjes_rx_irq(struct fjes_adapter *adapter, int src_epid)
-{
-	struct fjes_hw *hw = &adapter->hw;
-
-	fjes_hw_set_irqmask(hw, REG_ICTL_MASK_RX_DATA, true);
-
-	adapter->unset_rx_last = true;
-	napi_schedule(&adapter->napi);
-}
-
 static int fjes_poll(struct napi_struct *napi, int budget)
 {
 	struct fjes_adapter *adapter =
@@ -1196,182 +1053,130 @@ static int fjes_poll(struct napi_struct *napi, int budget)
 	return work_done;
 }
 
-/* fjes_probe - Device Initialization Routine */
-static int fjes_probe(struct platform_device *plat_dev)
+static int fjes_sw_init(struct fjes_adapter *adapter)
 {
-	struct fjes_adapter *adapter;
-	struct net_device *netdev;
-	struct resource *res;
-	struct fjes_hw *hw;
-	u8 addr[ETH_ALEN];
-	int err;
-
-	err = -ENOMEM;
-	netdev = alloc_netdev_mq(sizeof(struct fjes_adapter), "es%d",
-				 NET_NAME_UNKNOWN, fjes_netdev_setup,
-				 FJES_MAX_QUEUES);
-
-	if (!netdev)
-		goto err_out;
+	struct net_device *netdev = adapter->netdev;
 
-	SET_NETDEV_DEV(netdev, &plat_dev->dev);
+	netif_napi_add(netdev, &adapter->napi, fjes_poll);
 
-	dev_set_drvdata(&plat_dev->dev, netdev);
-	adapter = netdev_priv(netdev);
-	adapter->netdev = netdev;
-	adapter->plat_dev = plat_dev;
-	hw = &adapter->hw;
-	hw->back = adapter;
+	return 0;
+}
 
-	/* setup the private structure */
-	err = fjes_sw_init(adapter);
-	if (err)
-		goto err_free_netdev;
+static void fjes_force_close_task(struct work_struct *work)
+{
+	struct fjes_adapter *adapter = container_of(work,
+			struct fjes_adapter, force_close_task);
+	struct net_device *netdev = adapter->netdev;
 
-	INIT_WORK(&adapter->force_close_task, fjes_force_close_task);
-	adapter->force_reset = false;
-	adapter->open_guard = false;
+	rtnl_lock();
+	dev_close(netdev);
+	rtnl_unlock();
+}
 
-	adapter->txrx_wq = alloc_workqueue(DRV_NAME "/txrx", WQ_MEM_RECLAIM, 0);
-	if (unlikely(!adapter->txrx_wq)) {
-		err = -ENOMEM;
-		goto err_free_netdev;
-	}
+static void fjes_tx_stall_task(struct work_struct *work)
+{
+	struct fjes_adapter *adapter = container_of(work,
+			struct fjes_adapter, tx_stall_task);
+	struct net_device *netdev = adapter->netdev;
+	struct fjes_hw *hw = &adapter->hw;
+	int all_queue_available, sendable;
+	enum ep_partner_status pstatus;
+	int max_epid, my_epid, epid;
+	union ep_buffer_info *info;
+	int i;
 
-	adapter->control_wq = alloc_workqueue(DRV_NAME "/control",
-					      WQ_MEM_RECLAIM, 0);
-	if (unlikely(!adapter->control_wq)) {
-		err = -ENOMEM;
-		goto err_free_txrx_wq;
+	if (((long)jiffies -
+		dev_trans_start(netdev)) > FJES_TX_TX_STALL_TIMEOUT) {
+		netif_wake_queue(netdev);
+		return;
 	}
 
-	INIT_WORK(&adapter->tx_stall_task, fjes_tx_stall_task);
-	INIT_WORK(&adapter->raise_intr_rxdata_task,
-		  fjes_raise_intr_rxdata_task);
-	INIT_WORK(&adapter->unshare_watch_task, fjes_watch_unshare_task);
-	adapter->unshare_watch_bitmask = 0;
+	my_epid = hw->my_epid;
+	max_epid = hw->max_epid;
 
-	INIT_DELAYED_WORK(&adapter->interrupt_watch_task, fjes_irq_watch_task);
-	adapter->interrupt_watch_enable = false;
+	for (i = 0; i < 5; i++) {
+		all_queue_available = 1;
 
-	res = platform_get_resource(plat_dev, IORESOURCE_MEM, 0);
-	if (!res) {
-		err = -EINVAL;
-		goto err_free_control_wq;
-	}
-	hw->hw_res.start = res->start;
-	hw->hw_res.size = resource_size(res);
-	hw->hw_res.irq = platform_get_irq(plat_dev, 0);
-	if (hw->hw_res.irq < 0) {
-		err = hw->hw_res.irq;
-		goto err_free_control_wq;
-	}
+		for (epid = 0; epid < max_epid; epid++) {
+			if (my_epid == epid)
+				continue;
 
-	err = fjes_hw_init(&adapter->hw);
-	if (err)
-		goto err_free_control_wq;
+			pstatus = fjes_hw_get_partner_ep_status(hw, epid);
+			sendable = (pstatus == EP_PARTNER_SHARED);
+			if (!sendable)
+				continue;
 
-	/* setup MAC address (02:00:00:00:00:[epid])*/
-	addr[0] = 2;
-	addr[1] = 0;
-	addr[2] = 0;
-	addr[3] = 0;
-	addr[4] = 0;
-	addr[5] = hw->my_epid; /* EPID */
-	eth_hw_addr_set(netdev, addr);
+			info = adapter->hw.ep_shm_info[epid].tx.info;
 
-	err = register_netdev(netdev);
-	if (err)
-		goto err_hw_exit;
+			if (!(info->v1i.rx_status & FJES_RX_MTU_CHANGING_DONE))
+				return;
 
-	netif_carrier_off(netdev);
+			if (EP_RING_FULL(info->v1i.head, info->v1i.tail,
+					 info->v1i.count_max)) {
+				all_queue_available = 0;
+				break;
+			}
+		}
 
-	fjes_dbg_adapter_init(adapter);
+		if (all_queue_available) {
+			netif_wake_queue(netdev);
+			return;
+		}
+	}
 
-	return 0;
+	usleep_range(50, 100);
 
-err_hw_exit:
-	fjes_hw_exit(&adapter->hw);
-err_free_control_wq:
-	destroy_workqueue(adapter->control_wq);
-err_free_txrx_wq:
-	destroy_workqueue(adapter->txrx_wq);
-err_free_netdev:
-	free_netdev(netdev);
-err_out:
-	return err;
+	queue_work(adapter->txrx_wq, &adapter->tx_stall_task);
 }
 
-/* fjes_remove - Device Removal Routine */
-static int fjes_remove(struct platform_device *plat_dev)
+static void fjes_raise_intr_rxdata_task(struct work_struct *work)
 {
-	struct net_device *netdev = dev_get_drvdata(&plat_dev->dev);
-	struct fjes_adapter *adapter = netdev_priv(netdev);
+	struct fjes_adapter *adapter = container_of(work,
+			struct fjes_adapter, raise_intr_rxdata_task);
 	struct fjes_hw *hw = &adapter->hw;
+	enum ep_partner_status pstatus;
+	int max_epid, my_epid, epid;
 
-	fjes_dbg_adapter_exit(adapter);
-
-	cancel_delayed_work_sync(&adapter->interrupt_watch_task);
-	cancel_work_sync(&adapter->unshare_watch_task);
-	cancel_work_sync(&adapter->raise_intr_rxdata_task);
-	cancel_work_sync(&adapter->tx_stall_task);
-	if (adapter->control_wq)
-		destroy_workqueue(adapter->control_wq);
-	if (adapter->txrx_wq)
-		destroy_workqueue(adapter->txrx_wq);
-
-	unregister_netdev(netdev);
-
-	fjes_hw_exit(hw);
-
-	netif_napi_del(&adapter->napi);
-
-	free_netdev(netdev);
-
-	return 0;
-}
-
-static int fjes_sw_init(struct fjes_adapter *adapter)
-{
-	struct net_device *netdev = adapter->netdev;
-
-	netif_napi_add(netdev, &adapter->napi, fjes_poll, 64);
-
-	return 0;
-}
+	my_epid = hw->my_epid;
+	max_epid = hw->max_epid;
 
-/* fjes_netdev_setup - netdevice initialization routine */
-static void fjes_netdev_setup(struct net_device *netdev)
-{
-	ether_setup(netdev);
+	for (epid = 0; epid < max_epid; epid++)
+		hw->ep_shm_info[epid].tx_status_work = 0;
 
-	netdev->watchdog_timeo = FJES_TX_RETRY_INTERVAL;
-	netdev->netdev_ops = &fjes_netdev_ops;
-	fjes_set_ethtool_ops(netdev);
-	netdev->mtu = fjes_support_mtu[3];
-	netdev->min_mtu = fjes_support_mtu[0];
-	netdev->max_mtu = fjes_support_mtu[3];
-	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
-}
+	for (epid = 0; epid < max_epid; epid++) {
+		if (epid == my_epid)
+			continue;
 
-static void fjes_irq_watch_task(struct work_struct *work)
-{
-	struct fjes_adapter *adapter = container_of(to_delayed_work(work),
-			struct fjes_adapter, interrupt_watch_task);
+		pstatus = fjes_hw_get_partner_ep_status(hw, epid);
+		if (pstatus == EP_PARTNER_SHARED) {
+			hw->ep_shm_info[epid].tx_status_work =
+				hw->ep_shm_info[epid].tx.info->v1i.tx_status;
 
-	local_irq_disable();
-	fjes_intr(adapter->hw.hw_res.irq, adapter);
-	local_irq_enable();
+			if (hw->ep_shm_info[epid].tx_status_work ==
+				FJES_TX_DELAY_SEND_PENDING) {
+				hw->ep_shm_info[epid].tx.info->v1i.tx_status =
+					FJES_TX_DELAY_SEND_NONE;
+			}
+		}
+	}
 
-	if (fjes_rxframe_search_exist(adapter, 0) >= 0)
-		napi_schedule(&adapter->napi);
+	for (epid = 0; epid < max_epid; epid++) {
+		if (epid == my_epid)
+			continue;
 
-	if (adapter->interrupt_watch_enable) {
-		if (!delayed_work_pending(&adapter->interrupt_watch_task))
-			queue_delayed_work(adapter->control_wq,
-					   &adapter->interrupt_watch_task,
-					   FJES_IRQ_WATCH_DELAY);
+		pstatus = fjes_hw_get_partner_ep_status(hw, epid);
+		if ((hw->ep_shm_info[epid].tx_status_work ==
+		     FJES_TX_DELAY_SEND_PENDING) &&
+		    (pstatus == EP_PARTNER_SHARED) &&
+		    !(hw->ep_shm_info[epid].rx.info->v1i.rx_status &
+		      FJES_RX_POLL_WORK)) {
+			fjes_hw_raise_interrupt(hw, epid,
+						REG_ICTL_MASK_RX_DATA);
+			hw->ep_shm_info[epid].ep_stats.send_intr_rx += 1;
+		}
 	}
+
+	usleep_range(500, 1000);
 }
 
 static void fjes_watch_unshare_task(struct work_struct *work)
@@ -1508,6 +1313,169 @@ static void fjes_watch_unshare_task(struct work_struct *work)
 	}
 }
 
+static void fjes_irq_watch_task(struct work_struct *work)
+{
+	struct fjes_adapter *adapter = container_of(to_delayed_work(work),
+			struct fjes_adapter, interrupt_watch_task);
+
+	local_irq_disable();
+	fjes_intr(adapter->hw.hw_res.irq, adapter);
+	local_irq_enable();
+
+	if (fjes_rxframe_search_exist(adapter, 0) >= 0)
+		napi_schedule(&adapter->napi);
+
+	if (adapter->interrupt_watch_enable) {
+		if (!delayed_work_pending(&adapter->interrupt_watch_task))
+			queue_delayed_work(adapter->control_wq,
+					   &adapter->interrupt_watch_task,
+					   FJES_IRQ_WATCH_DELAY);
+	}
+}
+
+/* fjes_probe - Device Initialization Routine */
+static int fjes_probe(struct platform_device *plat_dev)
+{
+	struct fjes_adapter *adapter;
+	struct net_device *netdev;
+	struct resource *res;
+	struct fjes_hw *hw;
+	u8 addr[ETH_ALEN];
+	int err;
+
+	err = -ENOMEM;
+	netdev = alloc_netdev_mq(sizeof(struct fjes_adapter), "es%d",
+				 NET_NAME_UNKNOWN, fjes_netdev_setup,
+				 FJES_MAX_QUEUES);
+
+	if (!netdev)
+		goto err_out;
+
+	SET_NETDEV_DEV(netdev, &plat_dev->dev);
+
+	dev_set_drvdata(&plat_dev->dev, netdev);
+	adapter = netdev_priv(netdev);
+	adapter->netdev = netdev;
+	adapter->plat_dev = plat_dev;
+	hw = &adapter->hw;
+	hw->back = adapter;
+
+	/* setup the private structure */
+	err = fjes_sw_init(adapter);
+	if (err)
+		goto err_free_netdev;
+
+	INIT_WORK(&adapter->force_close_task, fjes_force_close_task);
+	adapter->force_reset = false;
+	adapter->open_guard = false;
+
+	adapter->txrx_wq = alloc_workqueue(DRV_NAME "/txrx", WQ_MEM_RECLAIM, 0);
+	if (unlikely(!adapter->txrx_wq)) {
+		err = -ENOMEM;
+		goto err_free_netdev;
+	}
+
+	adapter->control_wq = alloc_workqueue(DRV_NAME "/control",
+					      WQ_MEM_RECLAIM, 0);
+	if (unlikely(!adapter->control_wq)) {
+		err = -ENOMEM;
+		goto err_free_txrx_wq;
+	}
+
+	INIT_WORK(&adapter->tx_stall_task, fjes_tx_stall_task);
+	INIT_WORK(&adapter->raise_intr_rxdata_task,
+		  fjes_raise_intr_rxdata_task);
+	INIT_WORK(&adapter->unshare_watch_task, fjes_watch_unshare_task);
+	adapter->unshare_watch_bitmask = 0;
+
+	INIT_DELAYED_WORK(&adapter->interrupt_watch_task, fjes_irq_watch_task);
+	adapter->interrupt_watch_enable = false;
+
+	res = platform_get_resource(plat_dev, IORESOURCE_MEM, 0);
+	if (!res) {
+		err = -EINVAL;
+		goto err_free_control_wq;
+	}
+	hw->hw_res.start = res->start;
+	hw->hw_res.size = resource_size(res);
+	hw->hw_res.irq = platform_get_irq(plat_dev, 0);
+	if (hw->hw_res.irq < 0) {
+		err = hw->hw_res.irq;
+		goto err_free_control_wq;
+	}
+
+	err = fjes_hw_init(&adapter->hw);
+	if (err)
+		goto err_free_control_wq;
+
+	/* setup MAC address (02:00:00:00:00:[epid])*/
+	addr[0] = 2;
+	addr[1] = 0;
+	addr[2] = 0;
+	addr[3] = 0;
+	addr[4] = 0;
+	addr[5] = hw->my_epid; /* EPID */
+	eth_hw_addr_set(netdev, addr);
+
+	err = register_netdev(netdev);
+	if (err)
+		goto err_hw_exit;
+
+	netif_carrier_off(netdev);
+
+	fjes_dbg_adapter_init(adapter);
+
+	return 0;
+
+err_hw_exit:
+	fjes_hw_exit(&adapter->hw);
+err_free_control_wq:
+	destroy_workqueue(adapter->control_wq);
+err_free_txrx_wq:
+	destroy_workqueue(adapter->txrx_wq);
+err_free_netdev:
+	free_netdev(netdev);
+err_out:
+	return err;
+}
+
+/* fjes_remove - Device Removal Routine */
+static int fjes_remove(struct platform_device *plat_dev)
+{
+	struct net_device *netdev = dev_get_drvdata(&plat_dev->dev);
+	struct fjes_adapter *adapter = netdev_priv(netdev);
+	struct fjes_hw *hw = &adapter->hw;
+
+	fjes_dbg_adapter_exit(adapter);
+
+	cancel_delayed_work_sync(&adapter->interrupt_watch_task);
+	cancel_work_sync(&adapter->unshare_watch_task);
+	cancel_work_sync(&adapter->raise_intr_rxdata_task);
+	cancel_work_sync(&adapter->tx_stall_task);
+	if (adapter->control_wq)
+		destroy_workqueue(adapter->control_wq);
+	if (adapter->txrx_wq)
+		destroy_workqueue(adapter->txrx_wq);
+
+	unregister_netdev(netdev);
+
+	fjes_hw_exit(hw);
+
+	netif_napi_del(&adapter->napi);
+
+	free_netdev(netdev);
+
+	return 0;
+}
+
+static struct platform_driver fjes_driver = {
+	.driver = {
+		.name = DRV_NAME,
+	},
+	.probe = fjes_probe,
+	.remove = fjes_remove,
+};
+
 static acpi_status
 acpi_find_extended_socket_device(acpi_handle obj_handle, u32 level,
 				 void *context, void **return_value)
diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index 7962c37b3f14..f393e454f45c 100644
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -503,12 +503,9 @@ static struct sk_buff *geneve_gro_receive(struct sock *sk,
 
 	off_gnv = skb_gro_offset(skb);
 	hlen = off_gnv + sizeof(*gh);
-	gh = skb_gro_header_fast(skb, off_gnv);
-	if (skb_gro_header_hard(skb, hlen)) {
-		gh = skb_gro_header_slow(skb, hlen, off_gnv);
-		if (unlikely(!gh))
-			goto out;
-	}
+	gh = skb_gro_header(skb, hlen, off_gnv);
+	if (unlikely(!gh))
+		goto out;
 
 	if (gh->ver != GENEVE_VER || gh->oam)
 		goto out;
@@ -1200,8 +1197,8 @@ static const struct net_device_ops geneve_netdev_ops = {
 static void geneve_get_drvinfo(struct net_device *dev,
 			       struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->version, GENEVE_NETDEV_VER, sizeof(drvinfo->version));
-	strlcpy(drvinfo->driver, "geneve", sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, GENEVE_NETDEV_VER, sizeof(drvinfo->version));
+	strscpy(drvinfo->driver, "geneve", sizeof(drvinfo->driver));
 }
 
 static const struct ethtool_ops geneve_ethtool_ops = {
diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c
index a208e2b1a9af..15c7dc82107f 100644
--- a/drivers/net/gtp.c
+++ b/drivers/net/gtp.c
@@ -1859,6 +1859,7 @@ static struct genl_family gtp_genl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= gtp_genl_ops,
 	.n_small_ops	= ARRAY_SIZE(gtp_genl_ops),
+	.resv_start_op	= GTP_CMD_ECHOREQ + 1,
 	.mcgrps		= gtp_genl_mcgrps,
 	.n_mcgrps	= ARRAY_SIZE(gtp_genl_mcgrps),
 };
diff --git a/drivers/net/hamradio/hdlcdrv.c b/drivers/net/hamradio/hdlcdrv.c
index 8297411e87ea..a6184d6c7b15 100644
--- a/drivers/net/hamradio/hdlcdrv.c
+++ b/drivers/net/hamradio/hdlcdrv.c
@@ -600,7 +600,7 @@ static int hdlcdrv_siocdevprivate(struct net_device *dev, struct ifreq *ifr,
 
 	case HDLCDRVCTL_DRIVERNAME:
 		if (s->ops && s->ops->drvname) {
-			strlcpy(bi.data.drivername, s->ops->drvname,
+			strscpy(bi.data.drivername, s->ops->drvname,
 				sizeof(bi.data.drivername));
 			break;
 		}
diff --git a/drivers/net/hippi/rrunner.c b/drivers/net/hippi/rrunner.c
index 74e845fa2e07..aa8f828a0ae7 100644
--- a/drivers/net/hippi/rrunner.c
+++ b/drivers/net/hippi/rrunner.c
@@ -213,6 +213,7 @@ static int rr_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 		pci_iounmap(pdev, rrpriv->regs);
 	if (pdev)
 		pci_release_regions(pdev);
+	pci_disable_device(pdev);
  out2:
 	free_netdev(dev);
  out3:
diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c
index 6e42cb03e226..f066de0da492 100644
--- a/drivers/net/hyperv/netvsc.c
+++ b/drivers/net/hyperv/netvsc.c
@@ -1779,8 +1779,7 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device,
 	}
 
 	/* Enable NAPI handler before init callbacks */
-	netif_napi_add(ndev, &net_device->chan_table[0].napi,
-		       netvsc_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(ndev, &net_device->chan_table[0].napi, netvsc_poll);
 
 	/* Open the channel */
 	device->channel->next_request_id_callback = vmbus_next_request_id;
diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c
index 15ebd5426604..5f08482065ca 100644
--- a/drivers/net/hyperv/netvsc_drv.c
+++ b/drivers/net/hyperv/netvsc_drv.c
@@ -935,8 +935,8 @@ int netvsc_recv_callback(struct net_device *net,
 static void netvsc_get_drvinfo(struct net_device *net,
 			       struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
-	strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->fw_version, "N/A", sizeof(info->fw_version));
 }
 
 static void netvsc_get_channels(struct net_device *net,
diff --git a/drivers/net/hyperv/rndis_filter.c b/drivers/net/hyperv/rndis_filter.c
index 6da36cb8af80..11f767a20444 100644
--- a/drivers/net/hyperv/rndis_filter.c
+++ b/drivers/net/hyperv/rndis_filter.c
@@ -1575,7 +1575,7 @@ struct netvsc_device *rndis_filter_device_add(struct hv_device *dev,
 
 	for (i = 1; i < net_device->num_chn; i++)
 		netif_napi_add(net, &net_device->chan_table[i].napi,
-			       netvsc_poll, NAPI_POLL_WEIGHT);
+			       netvsc_poll);
 
 	return net_device;
 
diff --git a/drivers/net/ieee802154/mac802154_hwsim.c b/drivers/net/ieee802154/mac802154_hwsim.c
index 38c217bd7c82..2f0544dd7c2a 100644
--- a/drivers/net/ieee802154/mac802154_hwsim.c
+++ b/drivers/net/ieee802154/mac802154_hwsim.c
@@ -630,6 +630,7 @@ static struct genl_family hwsim_genl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.small_ops = hwsim_nl_ops,
 	.n_small_ops = ARRAY_SIZE(hwsim_nl_ops),
+	.resv_start_op = MAC802154_HWSIM_CMD_NEW_EDGE + 1,
 	.mcgrps = hwsim_mcgrps,
 	.n_mcgrps = ARRAY_SIZE(hwsim_mcgrps),
 };
diff --git a/drivers/net/ipa/Makefile b/drivers/net/ipa/Makefile
index 8b2220eb6b92..48255fc4b25c 100644
--- a/drivers/net/ipa/Makefile
+++ b/drivers/net/ipa/Makefile
@@ -13,4 +13,6 @@ ipa-y			:=	ipa_main.o ipa_power.o ipa_reg.o ipa_mem.o \
 				ipa_resource.o ipa_qmi.o ipa_qmi_msg.o \
 				ipa_sysfs.o
 
+ipa-y			+=	$(IPA_VERSIONS:%=reg/ipa_reg-v%.o)
+
 ipa-y			+=	$(IPA_VERSIONS:%=data/ipa_data-v%.o)
diff --git a/drivers/net/ipa/data/ipa_data-v3.1.c b/drivers/net/ipa/data/ipa_data-v3.1.c
index 1c1895aea811..e0d71f609272 100644
--- a/drivers/net/ipa/data/ipa_data-v3.1.c
+++ b/drivers/net/ipa/data/ipa_data-v3.1.c
@@ -526,7 +526,7 @@ static const struct ipa_power_data ipa_power_data = {
 /* Configuration data for an SoC having IPA v3.1 */
 const struct ipa_data ipa_data_v3_1 = {
 	.version	= IPA_VERSION_3_1,
-	.backward_compat = BCR_CMDQ_L_LACK_ONE_ENTRY_FMASK,
+	.backward_compat = BIT(BCR_CMDQ_L_LACK_ONE_ENTRY),
 	.qsb_count	= ARRAY_SIZE(ipa_qsb_data),
 	.qsb_data	= ipa_qsb_data,
 	.endpoint_count	= ARRAY_SIZE(ipa_gsi_endpoint_data),
diff --git a/drivers/net/ipa/data/ipa_data-v3.5.1.c b/drivers/net/ipa/data/ipa_data-v3.5.1.c
index 58b708d2fc75..383ef1890065 100644
--- a/drivers/net/ipa/data/ipa_data-v3.5.1.c
+++ b/drivers/net/ipa/data/ipa_data-v3.5.1.c
@@ -407,11 +407,11 @@ static const struct ipa_power_data ipa_power_data = {
 /* Configuration data for an SoC having IPA v3.5.1 */
 const struct ipa_data ipa_data_v3_5_1 = {
 	.version	= IPA_VERSION_3_5_1,
-	.backward_compat = BCR_CMDQ_L_LACK_ONE_ENTRY_FMASK |
-			   BCR_TX_NOT_USING_BRESP_FMASK |
-			   BCR_SUSPEND_L2_IRQ_FMASK |
-			   BCR_HOLB_DROP_L2_IRQ_FMASK |
-			   BCR_DUAL_TX_FMASK,
+	.backward_compat = BIT(BCR_CMDQ_L_LACK_ONE_ENTRY) |
+			   BIT(BCR_TX_NOT_USING_BRESP) |
+			   BIT(BCR_SUSPEND_L2_IRQ) |
+			   BIT(BCR_HOLB_DROP_L2_IRQ) |
+			   BIT(BCR_DUAL_TX),
 	.qsb_count	= ARRAY_SIZE(ipa_qsb_data),
 	.qsb_data	= ipa_qsb_data,
 	.endpoint_count	= ARRAY_SIZE(ipa_gsi_endpoint_data),
diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c
index 9e307eebd33f..bea2da1c4c51 100644
--- a/drivers/net/ipa/gsi.c
+++ b/drivers/net/ipa/gsi.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2021 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
@@ -56,9 +56,9 @@
  * element can also contain an immediate command, requesting the IPA perform
  * actions other than data transfer.
  *
- * Each TRE refers to a block of data--also located DRAM.  After writing one
- * or more TREs to a channel, the writer (either the IPA or an EE) writes a
- * doorbell register to inform the receiving side how many elements have
+ * Each TRE refers to a block of data--also located in DRAM.  After writing
+ * one or more TREs to a channel, the writer (either the IPA or an EE) writes
+ * a doorbell register to inform the receiving side how many elements have
  * been written.
  *
  * Each channel has a GSI "event ring" associated with it.  An event ring
@@ -710,43 +710,32 @@ static void gsi_evt_ring_program(struct gsi *gsi, u32 evt_ring_id)
 static struct gsi_trans *gsi_channel_trans_last(struct gsi_channel *channel)
 {
 	struct gsi_trans_info *trans_info = &channel->trans_info;
-	const struct list_head *list;
+	u32 pending_id = trans_info->pending_id;
 	struct gsi_trans *trans;
-
-	spin_lock_bh(&trans_info->spinlock);
-
-	/* There is a small chance a TX transaction got allocated just
-	 * before we disabled transmits, so check for that.
-	 */
-	if (channel->toward_ipa) {
-		list = &trans_info->alloc;
-		if (!list_empty(list))
-			goto done;
-		list = &trans_info->committed;
-		if (!list_empty(list))
-			goto done;
-		list = &trans_info->pending;
-		if (!list_empty(list))
-			goto done;
+	u16 trans_id;
+
+	if (channel->toward_ipa && pending_id != trans_info->free_id) {
+		/* There is a small chance a TX transaction got allocated
+		 * just before we disabled transmits, so check for that.
+		 * The last allocated, committed, or pending transaction
+		 * precedes the first free transaction.
+		 */
+		trans_id = trans_info->free_id - 1;
+	} else if (trans_info->polled_id != pending_id) {
+		/* Otherwise (TX or RX) we want to wait for anything that
+		 * has completed, or has been polled but not released yet.
+		 *
+		 * The last completed or polled transaction precedes the
+		 * first pending transaction.
+		 */
+		trans_id = pending_id - 1;
+	} else {
+		return NULL;
 	}
 
-	/* Otherwise (TX or RX) we want to wait for anything that
-	 * has completed, or has been polled but not released yet.
-	 */
-	list = &trans_info->complete;
-	if (!list_empty(list))
-		goto done;
-	list = &trans_info->polled;
-	if (list_empty(list))
-		list = NULL;
-done:
-	trans = list ? list_last_entry(list, struct gsi_trans, links) : NULL;
-
 	/* Caller will wait for this, so take a reference */
-	if (trans)
-		refcount_inc(&trans->refcount);
-
-	spin_unlock_bh(&trans_info->spinlock);
+	trans = &trans_info->trans[trans_id % channel->tre_count];
+	refcount_inc(&trans->refcount);
 
 	return trans;
 }
@@ -1358,8 +1347,8 @@ gsi_event_trans(struct gsi *gsi, struct gsi_event *event)
  * we update transactions to record their actual received lengths.
  *
  * When an event for a TX channel arrives we use information in the
- * transaction to report the number of requests and bytes have been
- * transferred.
+ * transaction to report the number of requests and bytes that have
+ * been transferred.
  *
  * This function is called whenever we learn that the GSI hardware has filled
  * new events since the last time we checked.  The ring's index field tells
@@ -1485,8 +1474,8 @@ void gsi_channel_doorbell(struct gsi_channel *channel)
 	iowrite32(val, gsi->virt + GSI_CH_C_DOORBELL_0_OFFSET(channel_id));
 }
 
-/* Consult hardware, move any newly completed transactions to completed list */
-static struct gsi_trans *gsi_channel_update(struct gsi_channel *channel)
+/* Consult hardware, move newly completed transactions to completed state */
+void gsi_channel_update(struct gsi_channel *channel)
 {
 	u32 evt_ring_id = channel->evt_ring_id;
 	struct gsi *gsi = channel->gsi;
@@ -1505,12 +1494,12 @@ static struct gsi_trans *gsi_channel_update(struct gsi_channel *channel)
 	offset = GSI_EV_CH_E_CNTXT_4_OFFSET(evt_ring_id);
 	index = gsi_ring_index(ring, ioread32(gsi->virt + offset));
 	if (index == ring->index % ring->count)
-		return NULL;
+		return;
 
 	/* Get the transaction for the latest completed event. */
 	trans = gsi_event_trans(gsi, gsi_ring_virt(ring, index - 1));
 	if (!trans)
-		return NULL;
+		return;
 
 	/* For RX channels, update each completed transaction with the number
 	 * of bytes that were actually received.  For TX channels, report
@@ -1518,8 +1507,6 @@ static struct gsi_trans *gsi_channel_update(struct gsi_channel *channel)
 	 * up the network stack.
 	 */
 	gsi_evt_ring_update(gsi, evt_ring_id, index);
-
-	return gsi_channel_trans_complete(channel);
 }
 
 /**
@@ -1528,21 +1515,18 @@ static struct gsi_trans *gsi_channel_update(struct gsi_channel *channel)
  *
  * Return:	Transaction pointer, or null if none are available
  *
- * This function returns the first entry on a channel's completed transaction
- * list.  If that list is empty, the hardware is consulted to determine
- * whether any new transactions have completed.  If so, they're moved to the
- * completed list and the new first entry is returned.  If there are no more
- * completed transactions, a null pointer is returned.
+ * This function returns the first of a channel's completed transactions.
+ * If no transactions are in completed state, the hardware is consulted to
+ * determine whether any new transactions have completed.  If so, they're
+ * moved to completed state and the first such transaction is returned.
+ * If there are no more completed transactions, a null pointer is returned.
  */
 static struct gsi_trans *gsi_channel_poll_one(struct gsi_channel *channel)
 {
 	struct gsi_trans *trans;
 
-	/* Get the first transaction from the completed list */
+	/* Get the first completed transaction */
 	trans = gsi_channel_trans_complete(channel);
-	if (!trans)	/* List is empty; see if there's more to do */
-		trans = gsi_channel_update(channel);
-
 	if (trans)
 		gsi_trans_move_polled(trans);
 
@@ -1623,7 +1607,7 @@ static int gsi_channel_setup_one(struct gsi *gsi, u32 channel_id)
 				  gsi_channel_poll);
 	else
 		netif_napi_add(&gsi->dummy_dev, &channel->napi,
-			       gsi_channel_poll, NAPI_POLL_WEIGHT);
+			       gsi_channel_poll);
 
 	return 0;
 
diff --git a/drivers/net/ipa/gsi.h b/drivers/net/ipa/gsi.h
index 23de5f67374c..49dcadba4e0b 100644
--- a/drivers/net/ipa/gsi.h
+++ b/drivers/net/ipa/gsi.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2021 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #ifndef _GSI_H_
 #define _GSI_H_
@@ -31,14 +31,6 @@ struct gsi_trans;
 struct gsi_channel_data;
 struct ipa_gsi_endpoint_data;
 
-/* Execution environment IDs */
-enum gsi_ee_id {
-	GSI_EE_AP				= 0x0,
-	GSI_EE_MODEM				= 0x1,
-	GSI_EE_UC				= 0x2,
-	GSI_EE_TZ				= 0x3,
-};
-
 struct gsi_ring {
 	void *virt;			/* ring array base address */
 	dma_addr_t addr;		/* primarily low 32 bits used */
@@ -82,18 +74,18 @@ struct gsi_trans_pool {
 
 struct gsi_trans_info {
 	atomic_t tre_avail;		/* TREs available for allocation */
-	struct gsi_trans_pool pool;	/* transaction pool */
+
+	u16 free_id;			/* first free trans in array */
+	u16 allocated_id;		/* first allocated transaction */
+	u16 committed_id;		/* first committed transaction */
+	u16 pending_id;			/* first pending transaction */
+	u16 completed_id;		/* first completed transaction */
+	u16 polled_id;			/* first polled transaction */
+	struct gsi_trans *trans;	/* transaction array */
 	struct gsi_trans **map;		/* TRE -> transaction map */
 
 	struct gsi_trans_pool sg_pool;	/* scatterlist pool */
 	struct gsi_trans_pool cmd_pool;	/* command payload DMA pool */
-
-	spinlock_t spinlock;		/* protects updates to the lists */
-	struct list_head alloc;		/* allocated, not committed */
-	struct list_head committed;	/* committed, awaiting doorbell */
-	struct list_head pending;	/* pending, awaiting completion */
-	struct list_head complete;	/* completed, awaiting poll */
-	struct list_head polled;	/* returned by gsi_channel_poll_one() */
 };
 
 /* Hardware values signifying the state of a channel */
diff --git a/drivers/net/ipa/gsi_private.h b/drivers/net/ipa/gsi_private.h
index 0b2516fa21b5..c65f7c5cdc8d 100644
--- a/drivers/net/ipa/gsi_private.h
+++ b/drivers/net/ipa/gsi_private.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #ifndef _GSI_PRIVATE_H_
 #define _GSI_PRIVATE_H_
@@ -18,13 +18,13 @@ struct gsi_channel;
 
 /**
  * gsi_trans_move_complete() - Mark a GSI transaction completed
- * @trans:	Transaction to commit
+ * @trans:	Transaction whose state is to be updated
  */
 void gsi_trans_move_complete(struct gsi_trans *trans);
 
 /**
  * gsi_trans_move_polled() - Mark a transaction polled
- * @trans:	Transaction to update
+ * @trans:	Transaction whose state is to be updated
  */
 void gsi_trans_move_polled(struct gsi_trans *trans);
 
@@ -94,6 +94,14 @@ void gsi_channel_trans_exit(struct gsi_channel *channel);
  */
 void gsi_channel_doorbell(struct gsi_channel *channel);
 
+/* gsi_channel_update() - Update knowledge of channel hardware state
+ * @channel:	Channel to be updated
+ *
+ * Consult hardware, change the state of any newly-completed transactions
+ * on a channel.
+ */
+void gsi_channel_update(struct gsi_channel *channel);
+
 /**
  * gsi_ring_virt() - Return virtual address for a ring entry
  * @ring:	Ring whose address is to be translated
diff --git a/drivers/net/ipa/gsi_reg.h b/drivers/net/ipa/gsi_reg.h
index 5bd8b31656d3..3763359f208f 100644
--- a/drivers/net/ipa/gsi_reg.h
+++ b/drivers/net/ipa/gsi_reg.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2015-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2021 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #ifndef _GSI_REG_H_
 #define _GSI_REG_H_
@@ -55,14 +55,10 @@
 /* The inter-EE IRQ registers are relative to gsi->virt_raw (IPA v3.5+) */
 
 #define GSI_INTER_EE_SRC_CH_IRQ_MSK_OFFSET \
-			GSI_INTER_EE_N_SRC_CH_IRQ_MSK_OFFSET(GSI_EE_AP)
-#define GSI_INTER_EE_N_SRC_CH_IRQ_MSK_OFFSET(ee) \
-			(0x0000c020 + 0x1000 * (ee))
+			(0x0000c020 + 0x1000 * GSI_EE_AP)
 
 #define GSI_INTER_EE_SRC_EV_CH_IRQ_MSK_OFFSET \
-			GSI_INTER_EE_N_SRC_EV_CH_IRQ_MSK_OFFSET(GSI_EE_AP)
-#define GSI_INTER_EE_N_SRC_EV_CH_IRQ_MSK_OFFSET(ee) \
-			(0x0000c024 + 0x1000 * (ee))
+			(0x0000c024 + 0x1000 * GSI_EE_AP)
 
 /* All other register offsets are relative to gsi->virt */
 
@@ -81,9 +77,7 @@ enum gsi_channel_type {
 };
 
 #define GSI_CH_C_CNTXT_0_OFFSET(ch) \
-		GSI_EE_N_CH_C_CNTXT_0_OFFSET((ch), GSI_EE_AP)
-#define GSI_EE_N_CH_C_CNTXT_0_OFFSET(ch, ee) \
-		(0x0001c000 + 0x4000 * (ee) + 0x80 * (ch))
+			(0x0001c000 + 0x4000 * GSI_EE_AP + 0x80 * (ch))
 #define CHTYPE_PROTOCOL_FMASK		GENMASK(2, 0)
 #define CHTYPE_DIR_FMASK		GENMASK(3, 3)
 #define EE_FMASK			GENMASK(7, 4)
@@ -112,9 +106,7 @@ chtype_protocol_encoded(enum ipa_version version, enum gsi_channel_type type)
 }
 
 #define GSI_CH_C_CNTXT_1_OFFSET(ch) \
-		GSI_EE_N_CH_C_CNTXT_1_OFFSET((ch), GSI_EE_AP)
-#define GSI_EE_N_CH_C_CNTXT_1_OFFSET(ch, ee) \
-		(0x0001c004 + 0x4000 * (ee) + 0x80 * (ch))
+			(0x0001c004 + 0x4000 * GSI_EE_AP + 0x80 * (ch))
 
 /* Encoded value for CH_C_CNTXT_1 register R_LENGTH field */
 static inline u32 r_length_encoded(enum ipa_version version, u32 length)
@@ -125,19 +117,13 @@ static inline u32 r_length_encoded(enum ipa_version version, u32 length)
 }
 
 #define GSI_CH_C_CNTXT_2_OFFSET(ch) \
-		GSI_EE_N_CH_C_CNTXT_2_OFFSET((ch), GSI_EE_AP)
-#define GSI_EE_N_CH_C_CNTXT_2_OFFSET(ch, ee) \
-		(0x0001c008 + 0x4000 * (ee) + 0x80 * (ch))
+			(0x0001c008 + 0x4000 * GSI_EE_AP + 0x80 * (ch))
 
 #define GSI_CH_C_CNTXT_3_OFFSET(ch) \
-		GSI_EE_N_CH_C_CNTXT_3_OFFSET((ch), GSI_EE_AP)
-#define GSI_EE_N_CH_C_CNTXT_3_OFFSET(ch, ee) \
-		(0x0001c00c + 0x4000 * (ee) + 0x80 * (ch))
+			(0x0001c00c + 0x4000 * GSI_EE_AP + 0x80 * (ch))
 
 #define GSI_CH_C_QOS_OFFSET(ch) \
-		GSI_EE_N_CH_C_QOS_OFFSET((ch), GSI_EE_AP)
-#define GSI_EE_N_CH_C_QOS_OFFSET(ch, ee) \
-		(0x0001c05c + 0x4000 * (ee) + 0x80 * (ch))
+			(0x0001c05c + 0x4000 * GSI_EE_AP + 0x80 * (ch))
 #define WRR_WEIGHT_FMASK		GENMASK(3, 0)
 #define MAX_PREFETCH_FMASK		GENMASK(8, 8)
 #define USE_DB_ENG_FMASK		GENMASK(9, 9)
@@ -158,29 +144,19 @@ enum gsi_prefetch_mode {
 };
 
 #define GSI_CH_C_SCRATCH_0_OFFSET(ch) \
-		GSI_EE_N_CH_C_SCRATCH_0_OFFSET((ch), GSI_EE_AP)
-#define GSI_EE_N_CH_C_SCRATCH_0_OFFSET(ch, ee) \
-		(0x0001c060 + 0x4000 * (ee) + 0x80 * (ch))
+			(0x0001c060 + 0x4000 * GSI_EE_AP + 0x80 * (ch))
 
 #define GSI_CH_C_SCRATCH_1_OFFSET(ch) \
-		GSI_EE_N_CH_C_SCRATCH_1_OFFSET((ch), GSI_EE_AP)
-#define GSI_EE_N_CH_C_SCRATCH_1_OFFSET(ch, ee) \
-		(0x0001c064 + 0x4000 * (ee) + 0x80 * (ch))
+			(0x0001c064 + 0x4000 * GSI_EE_AP + 0x80 * (ch))
 
 #define GSI_CH_C_SCRATCH_2_OFFSET(ch) \
-		GSI_EE_N_CH_C_SCRATCH_2_OFFSET((ch), GSI_EE_AP)
-#define GSI_EE_N_CH_C_SCRATCH_2_OFFSET(ch, ee) \
-		(0x0001c068 + 0x4000 * (ee) + 0x80 * (ch))
+			(0x0001c068 + 0x4000 * GSI_EE_AP + 0x80 * (ch))
 
 #define GSI_CH_C_SCRATCH_3_OFFSET(ch) \
-		GSI_EE_N_CH_C_SCRATCH_3_OFFSET((ch), GSI_EE_AP)
-#define GSI_EE_N_CH_C_SCRATCH_3_OFFSET(ch, ee) \
-		(0x0001c06c + 0x4000 * (ee) + 0x80 * (ch))
+			(0x0001c06c + 0x4000 * GSI_EE_AP + 0x80 * (ch))
 
 #define GSI_EV_CH_E_CNTXT_0_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_0_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_0_OFFSET(ev, ee) \
-		(0x0001d000 + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d000 + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 /* enum gsi_channel_type defines EV_CHTYPE field values in EV_CH_E_CNTXT_0 */
 #define EV_CHTYPE_FMASK			GENMASK(3, 0)
 #define EV_EE_FMASK			GENMASK(7, 4)
@@ -190,9 +166,7 @@ enum gsi_prefetch_mode {
 #define EV_ELEMENT_SIZE_FMASK		GENMASK(31, 24)
 
 #define GSI_EV_CH_E_CNTXT_1_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_1_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_1_OFFSET(ev, ee) \
-		(0x0001d004 + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d004 + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 /* Encoded value for EV_CH_C_CNTXT_1 register EV_R_LENGTH field */
 static inline u32 ev_r_length_encoded(enum ipa_version version, u32 length)
 {
@@ -202,83 +176,53 @@ static inline u32 ev_r_length_encoded(enum ipa_version version, u32 length)
 }
 
 #define GSI_EV_CH_E_CNTXT_2_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_2_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_2_OFFSET(ev, ee) \
-		(0x0001d008 + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d008 + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 
 #define GSI_EV_CH_E_CNTXT_3_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_3_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_3_OFFSET(ev, ee) \
-		(0x0001d00c + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d00c + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 
 #define GSI_EV_CH_E_CNTXT_4_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_4_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_4_OFFSET(ev, ee) \
-		(0x0001d010 + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d010 + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 
 #define GSI_EV_CH_E_CNTXT_8_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_8_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_8_OFFSET(ev, ee) \
-		(0x0001d020 + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d020 + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 #define MODT_FMASK			GENMASK(15, 0)
 #define MODC_FMASK			GENMASK(23, 16)
 #define MOD_CNT_FMASK			GENMASK(31, 24)
 
 #define GSI_EV_CH_E_CNTXT_9_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_9_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_9_OFFSET(ev, ee) \
-		(0x0001d024 + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d024 + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 
 #define GSI_EV_CH_E_CNTXT_10_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_10_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_10_OFFSET(ev, ee) \
-		(0x0001d028 + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d028 + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 
 #define GSI_EV_CH_E_CNTXT_11_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_11_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_11_OFFSET(ev, ee) \
-		(0x0001d02c + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d02c + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 
 #define GSI_EV_CH_E_CNTXT_12_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_12_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_12_OFFSET(ev, ee) \
-		(0x0001d030 + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d030 + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 
 #define GSI_EV_CH_E_CNTXT_13_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_CNTXT_13_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_CNTXT_13_OFFSET(ev, ee) \
-		(0x0001d034 + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d034 + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 
 #define GSI_EV_CH_E_SCRATCH_0_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_SCRATCH_0_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_SCRATCH_0_OFFSET(ev, ee) \
-		(0x0001d048 + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d048 + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 
 #define GSI_EV_CH_E_SCRATCH_1_OFFSET(ev) \
-		GSI_EE_N_EV_CH_E_SCRATCH_1_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_SCRATCH_1_OFFSET(ev, ee) \
-		(0x0001d04c + 0x4000 * (ee) + 0x80 * (ev))
+			(0x0001d04c + 0x4000 * GSI_EE_AP + 0x80 * (ev))
 
 #define GSI_CH_C_DOORBELL_0_OFFSET(ch) \
-		GSI_EE_N_CH_C_DOORBELL_0_OFFSET((ch), GSI_EE_AP)
-#define GSI_EE_N_CH_C_DOORBELL_0_OFFSET(ch, ee) \
-			(0x0001e000 + 0x4000 * (ee) + 0x08 * (ch))
+			(0x0001e000 + 0x4000 * GSI_EE_AP + 0x08 * (ch))
 
 #define GSI_EV_CH_E_DOORBELL_0_OFFSET(ev) \
-			GSI_EE_N_EV_CH_E_DOORBELL_0_OFFSET((ev), GSI_EE_AP)
-#define GSI_EE_N_EV_CH_E_DOORBELL_0_OFFSET(ev, ee) \
-			(0x0001e100 + 0x4000 * (ee) + 0x08 * (ev))
+			(0x0001e100 + 0x4000 * GSI_EE_AP + 0x08 * (ev))
 
 #define GSI_GSI_STATUS_OFFSET \
-			GSI_EE_N_GSI_STATUS_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_GSI_STATUS_OFFSET(ee) \
-			(0x0001f000 + 0x4000 * (ee))
+			(0x0001f000 + 0x4000 * GSI_EE_AP)
 #define ENABLED_FMASK			GENMASK(0, 0)
 
 #define GSI_CH_CMD_OFFSET \
-			GSI_EE_N_CH_CMD_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CH_CMD_OFFSET(ee) \
-			(0x0001f008 + 0x4000 * (ee))
+			(0x0001f008 + 0x4000 * GSI_EE_AP)
 #define CH_CHID_FMASK			GENMASK(7, 0)
 #define CH_OPCODE_FMASK			GENMASK(31, 24)
 
@@ -293,9 +237,7 @@ enum gsi_ch_cmd_opcode {
 };
 
 #define GSI_EV_CH_CMD_OFFSET \
-			GSI_EE_N_EV_CH_CMD_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_EV_CH_CMD_OFFSET(ee) \
-			(0x0001f010 + 0x4000 * (ee))
+			(0x0001f010 + 0x4000 * GSI_EE_AP)
 #define EV_CHID_FMASK			GENMASK(7, 0)
 #define EV_OPCODE_FMASK			GENMASK(31, 24)
 
@@ -307,9 +249,7 @@ enum gsi_evt_cmd_opcode {
 };
 
 #define GSI_GENERIC_CMD_OFFSET \
-			GSI_EE_N_GENERIC_CMD_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_GENERIC_CMD_OFFSET(ee) \
-			(0x0001f018 + 0x4000 * (ee))
+			(0x0001f018 + 0x4000 * GSI_EE_AP)
 #define GENERIC_OPCODE_FMASK		GENMASK(4, 0)
 #define GENERIC_CHID_FMASK		GENMASK(9, 5)
 #define GENERIC_EE_FMASK		GENMASK(13, 10)
@@ -326,9 +266,7 @@ enum gsi_generic_cmd_opcode {
 
 /* The next register is present for IPA v3.5.1 and above */
 #define GSI_GSI_HW_PARAM_2_OFFSET \
-			GSI_EE_N_GSI_HW_PARAM_2_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_GSI_HW_PARAM_2_OFFSET(ee) \
-			(0x0001f040 + 0x4000 * (ee))
+			(0x0001f040 + 0x4000 * GSI_EE_AP)
 #define IRAM_SIZE_FMASK			GENMASK(2, 0)
 #define NUM_CH_PER_EE_FMASK		GENMASK(7, 3)
 #define NUM_EV_PER_EE_FMASK		GENMASK(12, 8)
@@ -357,13 +295,9 @@ enum gsi_iram_size {
 
 /* IRQ condition for each type is cleared by writing type-specific register */
 #define GSI_CNTXT_TYPE_IRQ_OFFSET \
-			GSI_EE_N_CNTXT_TYPE_IRQ_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_TYPE_IRQ_OFFSET(ee) \
-			(0x0001f080 + 0x4000 * (ee))
+			(0x0001f080 + 0x4000 * GSI_EE_AP)
 #define GSI_CNTXT_TYPE_IRQ_MSK_OFFSET \
-			GSI_EE_N_CNTXT_TYPE_IRQ_MSK_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_TYPE_IRQ_MSK_OFFSET(ee) \
-			(0x0001f088 + 0x4000 * (ee))
+			(0x0001f088 + 0x4000 * GSI_EE_AP)
 
 /* Values here are bit positions in the TYPE_IRQ and TYPE_IRQ_MSK registers */
 enum gsi_irq_type_id {
@@ -377,62 +311,38 @@ enum gsi_irq_type_id {
 };
 
 #define GSI_CNTXT_SRC_CH_IRQ_OFFSET \
-			GSI_EE_N_CNTXT_SRC_CH_IRQ_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_SRC_CH_IRQ_OFFSET(ee) \
-			(0x0001f090 + 0x4000 * (ee))
+			(0x0001f090 + 0x4000 * GSI_EE_AP)
 
 #define GSI_CNTXT_SRC_EV_CH_IRQ_OFFSET \
-			GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_OFFSET(ee) \
-			(0x0001f094 + 0x4000 * (ee))
+			(0x0001f094 + 0x4000 * GSI_EE_AP)
 
 #define GSI_CNTXT_SRC_CH_IRQ_MSK_OFFSET \
-			GSI_EE_N_CNTXT_SRC_CH_IRQ_MSK_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_SRC_CH_IRQ_MSK_OFFSET(ee) \
-			(0x0001f098 + 0x4000 * (ee))
+			(0x0001f098 + 0x4000 * GSI_EE_AP)
 
 #define GSI_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET \
-			GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_MSK_OFFSET(ee) \
-			(0x0001f09c + 0x4000 * (ee))
+			(0x0001f09c + 0x4000 * GSI_EE_AP)
 
 #define GSI_CNTXT_SRC_CH_IRQ_CLR_OFFSET \
-			GSI_EE_N_CNTXT_SRC_CH_IRQ_CLR_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_SRC_CH_IRQ_CLR_OFFSET(ee) \
-			(0x0001f0a0 + 0x4000 * (ee))
+			(0x0001f0a0 + 0x4000 * GSI_EE_AP)
 
 #define GSI_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET \
-			GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_SRC_EV_CH_IRQ_CLR_OFFSET(ee) \
-			(0x0001f0a4 + 0x4000 * (ee))
+			(0x0001f0a4 + 0x4000 * GSI_EE_AP)
 
 #define GSI_CNTXT_SRC_IEOB_IRQ_OFFSET \
-			GSI_EE_N_CNTXT_SRC_IEOB_IRQ_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_SRC_IEOB_IRQ_OFFSET(ee) \
-			(0x0001f0b0 + 0x4000 * (ee))
+			(0x0001f0b0 + 0x4000 * GSI_EE_AP)
 
 #define GSI_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET \
-			GSI_EE_N_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_SRC_IEOB_IRQ_MSK_OFFSET(ee) \
-			(0x0001f0b8 + 0x4000 * (ee))
+			(0x0001f0b8 + 0x4000 * GSI_EE_AP)
 
 #define GSI_CNTXT_SRC_IEOB_IRQ_CLR_OFFSET \
-			GSI_EE_N_CNTXT_SRC_IEOB_IRQ_CLR_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_SRC_IEOB_IRQ_CLR_OFFSET(ee) \
-			(0x0001f0c0 + 0x4000 * (ee))
+			(0x0001f0c0 + 0x4000 * GSI_EE_AP)
 
 #define GSI_CNTXT_GLOB_IRQ_STTS_OFFSET \
-			GSI_EE_N_CNTXT_GLOB_IRQ_STTS_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_GLOB_IRQ_STTS_OFFSET(ee) \
-			(0x0001f100 + 0x4000 * (ee))
+			(0x0001f100 + 0x4000 * GSI_EE_AP)
 #define GSI_CNTXT_GLOB_IRQ_EN_OFFSET \
-			GSI_EE_N_CNTXT_GLOB_IRQ_EN_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_GLOB_IRQ_EN_OFFSET(ee) \
-			(0x0001f108 + 0x4000 * (ee))
+			(0x0001f108 + 0x4000 * GSI_EE_AP)
 #define GSI_CNTXT_GLOB_IRQ_CLR_OFFSET \
-			GSI_EE_N_CNTXT_GLOB_IRQ_CLR_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_GLOB_IRQ_CLR_OFFSET(ee) \
-			(0x0001f110 + 0x4000 * (ee))
+			(0x0001f110 + 0x4000 * GSI_EE_AP)
 /* Values here are bit positions in the GLOB_IRQ_* registers */
 enum gsi_global_irq_id {
 	ERROR_INT				= 0x0,
@@ -442,17 +352,11 @@ enum gsi_global_irq_id {
 };
 
 #define GSI_CNTXT_GSI_IRQ_STTS_OFFSET \
-			GSI_EE_N_CNTXT_GSI_IRQ_STTS_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_GSI_IRQ_STTS_OFFSET(ee) \
-			(0x0001f118 + 0x4000 * (ee))
+			(0x0001f118 + 0x4000 * GSI_EE_AP)
 #define GSI_CNTXT_GSI_IRQ_EN_OFFSET \
-			GSI_EE_N_CNTXT_GSI_IRQ_EN_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_GSI_IRQ_EN_OFFSET(ee) \
-			(0x0001f120 + 0x4000 * (ee))
+			(0x0001f120 + 0x4000 * GSI_EE_AP)
 #define GSI_CNTXT_GSI_IRQ_CLR_OFFSET \
-			GSI_EE_N_CNTXT_GSI_IRQ_CLR_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_GSI_IRQ_CLR_OFFSET(ee) \
-			(0x0001f128 + 0x4000 * (ee))
+			(0x0001f128 + 0x4000 * GSI_EE_AP)
 /* Values here are bit positions in the (general) GSI_IRQ_* registers */
 enum gsi_general_id {
 	BREAK_POINT				= 0x0,
@@ -462,15 +366,11 @@ enum gsi_general_id {
 };
 
 #define GSI_CNTXT_INTSET_OFFSET \
-			GSI_EE_N_CNTXT_INTSET_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_INTSET_OFFSET(ee) \
-			(0x0001f180 + 0x4000 * (ee))
+			(0x0001f180 + 0x4000 * GSI_EE_AP)
 #define INTYPE_FMASK			GENMASK(0, 0)
 
 #define GSI_ERROR_LOG_OFFSET \
-			GSI_EE_N_ERROR_LOG_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_ERROR_LOG_OFFSET(ee) \
-			(0x0001f200 + 0x4000 * (ee))
+			(0x0001f200 + 0x4000 * GSI_EE_AP)
 
 /* Fields below are present for IPA v3.5.1 and above */
 #define ERR_ARG3_FMASK			GENMASK(3, 0)
@@ -501,14 +401,10 @@ enum gsi_err_type {
 };
 
 #define GSI_ERROR_LOG_CLR_OFFSET \
-			GSI_EE_N_ERROR_LOG_CLR_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_ERROR_LOG_CLR_OFFSET(ee) \
-			(0x0001f210 + 0x4000 * (ee))
+			(0x0001f210 + 0x4000 * GSI_EE_AP)
 
 #define GSI_CNTXT_SCRATCH_0_OFFSET \
-			GSI_EE_N_CNTXT_SCRATCH_0_OFFSET(GSI_EE_AP)
-#define GSI_EE_N_CNTXT_SCRATCH_0_OFFSET(ee) \
-			(0x0001f400 + 0x4000 * (ee))
+			(0x0001f400 + 0x4000 * GSI_EE_AP)
 #define INTER_EE_RESULT_FMASK		GENMASK(2, 0)
 #define GENERIC_EE_RESULT_FMASK		GENMASK(7, 5)
 
diff --git a/drivers/net/ipa/gsi_trans.c b/drivers/net/ipa/gsi_trans.c
index 18e7e8c405be..26b7f683a3e1 100644
--- a/drivers/net/ipa/gsi_trans.c
+++ b/drivers/net/ipa/gsi_trans.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2020 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
@@ -22,37 +22,36 @@
  * DOC: GSI Transactions
  *
  * A GSI transaction abstracts the behavior of a GSI channel by representing
- * everything about a related group of IPA commands in a single structure.
- * (A "command" in this sense is either a data transfer or an IPA immediate
+ * everything about a related group of IPA operations in a single structure.
+ * (A "operation" in this sense is either a data transfer or an IPA immediate
  * command.)  Most details of interaction with the GSI hardware are managed
- * by the GSI transaction core, allowing users to simply describe commands
+ * by the GSI transaction core, allowing users to simply describe operations
  * to be performed.  When a transaction has completed a callback function
  * (dependent on the type of endpoint associated with the channel) allows
  * cleanup of resources associated with the transaction.
  *
- * To perform a command (or set of them), a user of the GSI transaction
+ * To perform an operation (or set of them), a user of the GSI transaction
  * interface allocates a transaction, indicating the number of TREs required
- * (one per command).  If sufficient TREs are available, they are reserved
+ * (one per operation).  If sufficient TREs are available, they are reserved
  * for use in the transaction and the allocation succeeds.  This way
- * exhaustion of the available TREs in a channel ring is detected
- * as early as possible.  All resources required to complete a transaction
- * are allocated at transaction allocation time.
+ * exhaustion of the available TREs in a channel ring is detected as early
+ * as possible.  Any other resources that might be needed to complete a
+ * transaction are also allocated when the transaction is allocated.
  *
- * Commands performed as part of a transaction are represented in an array
- * of Linux scatterlist structures.  This array is allocated with the
- * transaction, and its entries are initialized using standard scatterlist
- * functions (such as sg_set_buf() or skb_to_sgvec()).
+ * Operations performed as part of a transaction are represented in an array
+ * of Linux scatterlist structures, allocated with the transaction.  These
+ * scatterlist structures are initialized by "adding" operations to the
+ * transaction.  If a buffer in an operation must be mapped for DMA, this is
+ * done at the time it is added to the transaction.  It is possible for a
+ * mapping error to occur when an operation is added.  In this case the
+ * transaction should simply be freed; this correctly releases resources
+ * associated with the transaction.
  *
- * Once a transaction's scatterlist structures have been initialized, the
- * transaction is committed.  The caller is responsible for mapping buffers
- * for DMA if necessary, and this should be done *before* allocating
- * the transaction.  Between a successful allocation and commit of a
- * transaction no errors should occur.
- *
- * Committing transfers ownership of the entire transaction to the GSI
- * transaction core.  The GSI transaction code formats the content of
- * the scatterlist array into the channel ring buffer and informs the
- * hardware that new TREs are available to process.
+ * Once all operations have been successfully added to a transaction, the
+ * transaction is committed.  Committing transfers ownership of the entire
+ * transaction to the GSI transaction core.  The GSI transaction code
+ * formats the content of the scatterlist array into the channel ring
+ * buffer and informs the hardware that new TREs are available to process.
  *
  * The last TRE in each transaction is marked to interrupt the AP when the
  * GSI hardware has completed it.  Because transfers described by TREs are
@@ -125,11 +124,10 @@ void gsi_trans_pool_exit(struct gsi_trans_pool *pool)
 	memset(pool, 0, sizeof(*pool));
 }
 
-/* Allocate the requested number of (zeroed) entries from the pool */
-/* Home-grown DMA pool.  This way we can preallocate and use the tre_count
- * to guarantee allocations will succeed.  Even though we specify max_alloc
- * (and it can be more than one), we only allow allocation of a single
- * element from a DMA pool.
+/* Home-grown DMA pool.  This way we can preallocate the pool, and guarantee
+ * allocations will succeed.  The immediate commands in a transaction can
+ * require up to max_alloc elements from the pool.  But we only allow
+ * allocation of a single element from a DMA pool at a time.
  */
 int gsi_trans_pool_init_dma(struct device *dev, struct gsi_trans_pool *pool,
 			    size_t size, u32 count, u32 max_alloc)
@@ -237,68 +235,63 @@ gsi_channel_trans_mapped(struct gsi_channel *channel, u32 index)
 /* Return the oldest completed transaction for a channel (or null) */
 struct gsi_trans *gsi_channel_trans_complete(struct gsi_channel *channel)
 {
-	return list_first_entry_or_null(&channel->trans_info.complete,
-					struct gsi_trans, links);
+	struct gsi_trans_info *trans_info = &channel->trans_info;
+	u16 trans_id = trans_info->completed_id;
+
+	if (trans_id == trans_info->pending_id) {
+		gsi_channel_update(channel);
+		if (trans_id == trans_info->pending_id)
+			return NULL;
+	}
+
+	return &trans_info->trans[trans_id %= channel->tre_count];
 }
 
-/* Move a transaction from the allocated list to the committed list */
+/* Move a transaction from allocated to committed state */
 static void gsi_trans_move_committed(struct gsi_trans *trans)
 {
 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];
 	struct gsi_trans_info *trans_info = &channel->trans_info;
 
-	spin_lock_bh(&trans_info->spinlock);
-
-	list_move_tail(&trans->links, &trans_info->committed);
-
-	spin_unlock_bh(&trans_info->spinlock);
+	/* This allocated transaction is now committed */
+	trans_info->allocated_id++;
 }
 
-/* Move transactions from the committed list to the pending list */
+/* Move committed transactions to pending state */
 static void gsi_trans_move_pending(struct gsi_trans *trans)
 {
 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];
 	struct gsi_trans_info *trans_info = &channel->trans_info;
-	struct list_head list;
-
-	spin_lock_bh(&trans_info->spinlock);
+	u16 trans_index = trans - &trans_info->trans[0];
+	u16 delta;
 
-	/* Move this transaction and all predecessors to the pending list */
-	list_cut_position(&list, &trans_info->committed, &trans->links);
-	list_splice_tail(&list, &trans_info->pending);
-
-	spin_unlock_bh(&trans_info->spinlock);
+	/* These committed transactions are now pending */
+	delta = trans_index - trans_info->committed_id + 1;
+	trans_info->committed_id += delta % channel->tre_count;
 }
 
-/* Move a transaction and all of its predecessors from the pending list
- * to the completed list.
- */
+/* Move pending transactions to completed state */
 void gsi_trans_move_complete(struct gsi_trans *trans)
 {
 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];
 	struct gsi_trans_info *trans_info = &channel->trans_info;
-	struct list_head list;
+	u16 trans_index = trans - trans_info->trans;
+	u16 delta;
 
-	spin_lock_bh(&trans_info->spinlock);
-
-	/* Move this transaction and all predecessors to completed list */
-	list_cut_position(&list, &trans_info->pending, &trans->links);
-	list_splice_tail(&list, &trans_info->complete);
-
-	spin_unlock_bh(&trans_info->spinlock);
+	/* These pending transactions are now completed */
+	delta = trans_index - trans_info->pending_id + 1;
+	delta %= channel->tre_count;
+	trans_info->pending_id += delta;
 }
 
-/* Move a transaction from the completed list to the polled list */
+/* Move a transaction from completed to polled state */
 void gsi_trans_move_polled(struct gsi_trans *trans)
 {
 	struct gsi_channel *channel = &trans->gsi->channel[trans->channel_id];
 	struct gsi_trans_info *trans_info = &channel->trans_info;
 
-	spin_lock_bh(&trans_info->spinlock);
-
-	list_move_tail(&trans->links, &trans_info->polled);
-
-	spin_unlock_bh(&trans_info->spinlock);
+	/* This completed transaction is now polled */
+	trans_info->completed_id++;
 }
 
 /* Reserve some number of TREs on a channel.  Returns true if successful */
@@ -343,20 +336,22 @@ struct gsi_trans *gsi_channel_trans_alloc(struct gsi *gsi, u32 channel_id,
 	struct gsi_channel *channel = &gsi->channel[channel_id];
 	struct gsi_trans_info *trans_info;
 	struct gsi_trans *trans;
+	u16 trans_index;
 
 	if (WARN_ON(tre_count > channel->trans_tre_max))
 		return NULL;
 
 	trans_info = &channel->trans_info;
 
-	/* We reserve the TREs now, but consume them at commit time.
-	 * If there aren't enough available, we're done.
-	 */
+	/* If we can't reserve the TREs for the transaction, we're done */
 	if (!gsi_trans_tre_reserve(trans_info, tre_count))
 		return NULL;
 
-	/* Allocate and initialize non-zero fields in the transaction */
-	trans = gsi_trans_pool_alloc(&trans_info->pool, 1);
+	trans_index = trans_info->free_id % channel->tre_count;
+	trans = &trans_info->trans[trans_index];
+	memset(trans, 0, sizeof(*trans));
+
+	/* Initialize non-zero fields in the transaction */
 	trans->gsi = gsi;
 	trans->channel_id = channel_id;
 	trans->rsvd_count = tre_count;
@@ -367,45 +362,37 @@ struct gsi_trans *gsi_channel_trans_alloc(struct gsi *gsi, u32 channel_id,
 	sg_init_marker(trans->sgl, tre_count);
 
 	trans->direction = direction;
-
-	spin_lock_bh(&trans_info->spinlock);
-
-	list_add_tail(&trans->links, &trans_info->alloc);
-
-	spin_unlock_bh(&trans_info->spinlock);
-
 	refcount_set(&trans->refcount, 1);
 
+	/* This free transaction is now allocated */
+	trans_info->free_id++;
+
 	return trans;
 }
 
 /* Free a previously-allocated transaction */
 void gsi_trans_free(struct gsi_trans *trans)
 {
-	refcount_t *refcount = &trans->refcount;
 	struct gsi_trans_info *trans_info;
-	bool last;
 
-	/* We must hold the lock to release the last reference */
-	if (refcount_dec_not_one(refcount))
+	if (!refcount_dec_and_test(&trans->refcount))
 		return;
 
+	/* Unused transactions are allocated but never committed, pending,
+	 * completed, or polled.
+	 */
 	trans_info = &trans->gsi->channel[trans->channel_id].trans_info;
-
-	spin_lock_bh(&trans_info->spinlock);
-
-	/* Reference might have been added before we got the lock */
-	last = refcount_dec_and_test(refcount);
-	if (last)
-		list_del(&trans->links);
-
-	spin_unlock_bh(&trans_info->spinlock);
-
-	if (!last)
-		return;
-
-	if (trans->used_count)
+	if (!trans->used_count) {
+		trans_info->allocated_id++;
+		trans_info->committed_id++;
+		trans_info->pending_id++;
+		trans_info->completed_id++;
+	} else {
 		ipa_gsi_trans_release(trans);
+	}
+
+	/* This transaction is now free */
+	trans_info->polled_id++;
 
 	/* Releasing the reserved TREs implicitly frees the sgl[] and
 	 * (if present) info[] arrays, plus the transaction itself.
@@ -548,8 +535,8 @@ static void gsi_trans_tre_fill(struct gsi_tre *dest_tre, dma_addr_t addr,
  *
  * Formats channel ring TRE entries based on the content of the scatterlist.
  * Maps a transaction pointer to the last ring entry used for the transaction,
- * so it can be recovered when it completes.  Moves the transaction to the
- * pending list.  Finally, updates the channel ring pointer and optionally
+ * so it can be recovered when it completes.  Moves the transaction to
+ * pending state.  Finally, updates the channel ring pointer and optionally
  * rings the doorbell.
  */
 static void __gsi_trans_commit(struct gsi_trans *trans, bool ring_db)
@@ -654,23 +641,27 @@ void gsi_trans_complete(struct gsi_trans *trans)
 void gsi_channel_trans_cancel_pending(struct gsi_channel *channel)
 {
 	struct gsi_trans_info *trans_info = &channel->trans_info;
-	struct gsi_trans *trans;
-	bool cancelled;
+	u16 trans_id = trans_info->pending_id;
 
 	/* channel->gsi->mutex is held by caller */
-	spin_lock_bh(&trans_info->spinlock);
 
-	cancelled = !list_empty(&trans_info->pending);
-	list_for_each_entry(trans, &trans_info->pending, links)
-		trans->cancelled = true;
+	/* If there are no pending transactions, we're done */
+	if (trans_id == trans_info->committed_id)
+		return;
 
-	list_splice_tail_init(&trans_info->pending, &trans_info->complete);
+	/* Mark all pending transactions cancelled */
+	do {
+		struct gsi_trans *trans;
+
+		trans = &trans_info->trans[trans_id % channel->tre_count];
+		trans->cancelled = true;
+	} while (++trans_id != trans_info->committed_id);
 
-	spin_unlock_bh(&trans_info->spinlock);
+	/* All pending transactions are now completed */
+	trans_info->pending_id = trans_info->committed_id;
 
 	/* Schedule NAPI polling to complete the cancelled transactions */
-	if (cancelled)
-		napi_schedule(&channel->napi);
+	napi_schedule(&channel->napi);
 }
 
 /* Issue a command to read a single byte from a channel */
@@ -736,10 +727,16 @@ int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id)
 	 * modulo that number to determine the next one that's free.
 	 * Transactions are allocated one at a time.
 	 */
-	ret = gsi_trans_pool_init(&trans_info->pool, sizeof(struct gsi_trans),
-				  tre_max, 1);
-	if (ret)
+	trans_info->trans = kcalloc(tre_count, sizeof(*trans_info->trans),
+				    GFP_KERNEL);
+	if (!trans_info->trans)
 		return -ENOMEM;
+	trans_info->free_id = 0;	/* all modulo channel->tre_count */
+	trans_info->allocated_id = 0;
+	trans_info->committed_id = 0;
+	trans_info->pending_id = 0;
+	trans_info->completed_id = 0;
+	trans_info->polled_id = 0;
 
 	/* A completion event contains a pointer to the TRE that caused
 	 * the event (which will be the last one used by the transaction).
@@ -765,19 +762,13 @@ int gsi_channel_trans_init(struct gsi *gsi, u32 channel_id)
 	if (ret)
 		goto err_map_free;
 
-	spin_lock_init(&trans_info->spinlock);
-	INIT_LIST_HEAD(&trans_info->alloc);
-	INIT_LIST_HEAD(&trans_info->committed);
-	INIT_LIST_HEAD(&trans_info->pending);
-	INIT_LIST_HEAD(&trans_info->complete);
-	INIT_LIST_HEAD(&trans_info->polled);
 
 	return 0;
 
 err_map_free:
 	kfree(trans_info->map);
 err_trans_free:
-	gsi_trans_pool_exit(&trans_info->pool);
+	kfree(trans_info->trans);
 
 	dev_err(gsi->dev, "error %d initializing channel %u transactions\n",
 		ret, channel_id);
@@ -791,6 +782,6 @@ void gsi_channel_trans_exit(struct gsi_channel *channel)
 	struct gsi_trans_info *trans_info = &channel->trans_info;
 
 	gsi_trans_pool_exit(&trans_info->sg_pool);
-	gsi_trans_pool_exit(&trans_info->pool);
+	kfree(trans_info->trans);
 	kfree(trans_info->map);
 }
diff --git a/drivers/net/ipa/gsi_trans.h b/drivers/net/ipa/gsi_trans.h
index 7084507830c2..30c1c2dc77c6 100644
--- a/drivers/net/ipa/gsi_trans.h
+++ b/drivers/net/ipa/gsi_trans.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2020 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 #ifndef _GSI_TRANS_H_
 #define _GSI_TRANS_H_
@@ -29,7 +29,6 @@ struct gsi_trans_pool;
  * struct gsi_trans - a GSI transaction
  *
  * Most fields in this structure for internal use by the transaction core code:
- * @links:	Links for channel transaction lists by state
  * @gsi:	GSI pointer
  * @channel_id: Channel number transaction is associated with
  * @cancelled:	If set by the core code, transaction was cancelled
@@ -50,8 +49,6 @@ struct gsi_trans_pool;
  * received.
  */
 struct gsi_trans {
-	struct list_head links;		/* gsi_channel lists */
-
 	struct gsi *gsi;
 	u8 channel_id;
 
@@ -77,7 +74,7 @@ struct gsi_trans {
 
 /**
  * gsi_trans_pool_init() - Initialize a pool of structures for transactions
- * @pool:	GSI transaction poll pointer
+ * @pool:	GSI transaction pool pointer
  * @size:	Size of elements in the pool
  * @count:	Minimum number of elements in the pool
  * @max_alloc:	Maximum number of elements allocated at a time from pool
diff --git a/drivers/net/ipa/ipa.h b/drivers/net/ipa/ipa.h
index 4fc3c72359f5..09ead433ec38 100644
--- a/drivers/net/ipa/ipa.h
+++ b/drivers/net/ipa/ipa.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #ifndef _IPA_H_
 #define _IPA_H_
@@ -44,6 +44,7 @@ struct ipa_interrupt;
  * @uc_loaded:		true after microcontroller has reported it's ready
  * @reg_addr:		DMA address used for IPA register access
  * @reg_virt:		Virtual address used for IPA register access
+ * @regs:		IPA register definitions
  * @mem_addr:		DMA address of IPA-local memory space
  * @mem_virt:		Virtual address of IPA-local memory space
  * @mem_offset:		Offset from @mem_virt used for access to IPA memory
@@ -90,6 +91,7 @@ struct ipa {
 
 	dma_addr_t reg_addr;
 	void __iomem *reg_virt;
+	const struct ipa_regs *regs;
 
 	dma_addr_t mem_addr;
 	void *mem_virt;
diff --git a/drivers/net/ipa/ipa_cmd.c b/drivers/net/ipa/ipa_cmd.c
index 6dea40259b60..26c3db9f52b1 100644
--- a/drivers/net/ipa/ipa_cmd.c
+++ b/drivers/net/ipa/ipa_cmd.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2021 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
@@ -32,7 +32,7 @@
  * immediate command's opcode.  The payload for a command resides in AP
  * memory and is described by a single scatterlist entry in its transaction.
  * Commands do not require a transaction completion callback, and are
- * (currently) always issued using gsi_trans_commit_wait().
+ * always issued using gsi_trans_commit_wait().
  */
 
 /* Some commands can wait until indicated pipeline stages are clear */
@@ -305,6 +305,7 @@ static bool ipa_cmd_register_write_offset_valid(struct ipa *ipa,
 /* Check whether offsets passed to register_write are valid */
 static bool ipa_cmd_register_write_valid(struct ipa *ipa)
 {
+	const struct ipa_reg *reg;
 	const char *name;
 	u32 offset;
 
@@ -312,7 +313,8 @@ static bool ipa_cmd_register_write_valid(struct ipa *ipa)
 	 * offset will fit in a register write IPA immediate command.
 	 */
 	if (ipa_table_hash_support(ipa)) {
-		offset = ipa_reg_filt_rout_hash_flush_offset(ipa->version);
+		reg = ipa_reg(ipa, FILT_ROUT_HASH_FLUSH);
+		offset = ipa_reg_offset(reg);
 		name = "filter/route hash flush";
 		if (!ipa_cmd_register_write_offset_valid(ipa, name, offset))
 			return false;
@@ -325,7 +327,8 @@ static bool ipa_cmd_register_write_valid(struct ipa *ipa)
 	 * worst case (highest endpoint number) offset of that endpoint
 	 * fits in the register write command field(s) that must hold it.
 	 */
-	offset = IPA_REG_ENDP_STATUS_N_OFFSET(IPA_ENDPOINT_COUNT - 1);
+	reg = ipa_reg(ipa, ENDP_STATUS);
+	offset = ipa_reg_n_offset(reg, IPA_ENDPOINT_COUNT - 1);
 	name = "maximal endpoint status";
 	if (!ipa_cmd_register_write_offset_valid(ipa, name, offset))
 		return false;
diff --git a/drivers/net/ipa/ipa_cmd.h b/drivers/net/ipa/ipa_cmd.h
index 9215ddad1010..8e4243c1f0bb 100644
--- a/drivers/net/ipa/ipa_cmd.h
+++ b/drivers/net/ipa/ipa_cmd.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2020 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 #ifndef _IPA_CMD_H_
 #define _IPA_CMD_H_
diff --git a/drivers/net/ipa/ipa_data.h b/drivers/net/ipa/ipa_data.h
index e15eb3cd3e33..e5a6ce75c7dd 100644
--- a/drivers/net/ipa/ipa_data.h
+++ b/drivers/net/ipa/ipa_data.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2021 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 #ifndef _IPA_DATA_H_
 #define _IPA_DATA_H_
@@ -31,7 +31,7 @@
  * communication path between the IPA and a particular execution environment
  * (EE), such as the AP or Modem.  Each EE has a set of channels associated
  * with it, and each channel has an ID unique for that EE.  For the most part
- * the only GSI channels of concern to this driver belong to the AP
+ * the only GSI channels of concern to this driver belong to the AP.
  *
  * An endpoint is an IPA construct representing a single channel anywhere
  * in the system.  An IPA endpoint ID maps directly to an (EE, channel_id)
diff --git a/drivers/net/ipa/ipa_endpoint.c b/drivers/net/ipa/ipa_endpoint.c
index 66d2bfdf9e42..093e11ec7c2d 100644
--- a/drivers/net/ipa/ipa_endpoint.c
+++ b/drivers/net/ipa/ipa_endpoint.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2021 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
@@ -23,8 +23,6 @@
 #include "ipa_gsi.h"
 #include "ipa_power.h"
 
-#define atomic_dec_not_zero(v)	atomic_add_unless((v), -1, 0)
-
 /* Hardware is told about receive buffers once a "batch" has been queued */
 #define IPA_REPLENISH_BATCH	16		/* Must be non-zero */
 
@@ -72,14 +70,6 @@ struct ipa_status {
 #define IPA_STATUS_FLAGS1_RT_RULE_ID_FMASK	GENMASK(31, 22)
 #define IPA_STATUS_FLAGS2_TAG_FMASK		GENMASK_ULL(63, 16)
 
-static u32 aggr_byte_limit_max(enum ipa_version version)
-{
-	if (version < IPA_VERSION_4_5)
-		return field_max(aggr_byte_limit_fmask(true));
-
-	return field_max(aggr_byte_limit_fmask(false));
-}
-
 /* Compute the aggregation size value to use for a given buffer size */
 static u32 ipa_aggr_size_kb(u32 rx_buffer_size, bool aggr_hard_limit)
 {
@@ -111,6 +101,7 @@ static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count,
 
 	if (!data->toward_ipa) {
 		const struct ipa_endpoint_rx *rx_config;
+		const struct ipa_reg *reg;
 		u32 buffer_size;
 		u32 aggr_size;
 		u32 limit;
@@ -171,7 +162,9 @@ static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count,
 		 */
 		aggr_size = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD,
 					     rx_config->aggr_hard_limit);
-		limit = aggr_byte_limit_max(ipa->version);
+		reg = ipa_reg(ipa, ENDP_INIT_AGGR);
+
+		limit = ipa_reg_field_max(reg, BYTE_LIMIT);
 		if (aggr_size > limit) {
 			dev_err(dev, "aggregated size too large for RX endpoint %u (%u KB > %u KB)\n",
 				data->endpoint_id, aggr_size, limit);
@@ -182,6 +175,15 @@ static bool ipa_endpoint_data_valid_one(struct ipa *ipa, u32 count,
 		return true;	/* Nothing more to check for RX */
 	}
 
+	/* Starting with IPA v4.5 sequencer replication is obsolete */
+	if (ipa->version >= IPA_VERSION_4_5) {
+		if (data->endpoint.config.tx.seq_rep_type) {
+			dev_err(dev, "no-zero seq_rep_type TX endpoint %u\n",
+				data->endpoint_id);
+			return false;
+		}
+	}
+
 	if (data->endpoint.config.status_enable) {
 		other_name = data->endpoint.config.tx.status_endpoint;
 		if (other_name >= count) {
@@ -299,8 +301,10 @@ static struct gsi_trans *ipa_endpoint_trans_alloc(struct ipa_endpoint *endpoint,
 static bool
 ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay)
 {
-	u32 offset = IPA_REG_ENDP_INIT_CTRL_N_OFFSET(endpoint->endpoint_id);
 	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
+	u32 field_id;
+	u32 offset;
 	bool state;
 	u32 mask;
 	u32 val;
@@ -310,9 +314,13 @@ ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay)
 	else
 		WARN_ON(ipa->version >= IPA_VERSION_4_0);
 
-	mask = endpoint->toward_ipa ? ENDP_DELAY_FMASK : ENDP_SUSPEND_FMASK;
-
+	reg = ipa_reg(ipa, ENDP_INIT_CTRL);
+	offset = ipa_reg_n_offset(reg, endpoint->endpoint_id);
 	val = ioread32(ipa->reg_virt + offset);
+
+	field_id = endpoint->toward_ipa ? ENDP_DELAY : ENDP_SUSPEND;
+	mask = ipa_reg_bit(reg, field_id);
+
 	state = !!(val & mask);
 
 	/* Don't bother if it's already in the requested state */
@@ -339,13 +347,13 @@ static bool ipa_endpoint_aggr_active(struct ipa_endpoint *endpoint)
 {
 	u32 mask = BIT(endpoint->endpoint_id);
 	struct ipa *ipa = endpoint->ipa;
-	u32 offset;
+	const struct ipa_reg *reg;
 	u32 val;
 
 	WARN_ON(!(mask & ipa->available));
 
-	offset = ipa_reg_state_aggr_active_offset(ipa->version);
-	val = ioread32(ipa->reg_virt + offset);
+	reg = ipa_reg(ipa, STATE_AGGR_ACTIVE);
+	val = ioread32(ipa->reg_virt + ipa_reg_offset(reg));
 
 	return !!(val & mask);
 }
@@ -354,10 +362,12 @@ static void ipa_endpoint_force_close(struct ipa_endpoint *endpoint)
 {
 	u32 mask = BIT(endpoint->endpoint_id);
 	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 
 	WARN_ON(!(mask & ipa->available));
 
-	iowrite32(mask, ipa->reg_virt + IPA_REG_AGGR_FORCE_CLOSE_OFFSET);
+	reg = ipa_reg(ipa, AGGR_FORCE_CLOSE);
+	iowrite32(mask, ipa->reg_virt + ipa_reg_offset(reg));
 }
 
 /**
@@ -456,6 +466,7 @@ int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa)
 	while (initialized) {
 		u32 endpoint_id = __ffs(initialized);
 		struct ipa_endpoint *endpoint;
+		const struct ipa_reg *reg;
 		u32 offset;
 
 		initialized ^= BIT(endpoint_id);
@@ -465,7 +476,8 @@ int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa)
 		if (!(endpoint->ee_id == GSI_EE_MODEM && endpoint->toward_ipa))
 			continue;
 
-		offset = IPA_REG_ENDP_STATUS_N_OFFSET(endpoint_id);
+		reg = ipa_reg(ipa, ENDP_STATUS);
+		offset = ipa_reg_n_offset(reg, endpoint_id);
 
 		/* Value written is 0, and all bits are updated.  That
 		 * means status is disabled on the endpoint, and as a
@@ -485,22 +497,23 @@ int ipa_endpoint_modem_exception_reset_all(struct ipa *ipa)
 
 static void ipa_endpoint_init_cfg(struct ipa_endpoint *endpoint)
 {
-	u32 offset = IPA_REG_ENDP_INIT_CFG_N_OFFSET(endpoint->endpoint_id);
+	u32 endpoint_id = endpoint->endpoint_id;
+	struct ipa *ipa = endpoint->ipa;
 	enum ipa_cs_offload_en enabled;
+	const struct ipa_reg *reg;
 	u32 val = 0;
 
+	reg = ipa_reg(ipa, ENDP_INIT_CFG);
 	/* FRAG_OFFLOAD_EN is 0 */
 	if (endpoint->config.checksum) {
-		enum ipa_version version = endpoint->ipa->version;
+		enum ipa_version version = ipa->version;
 
 		if (endpoint->toward_ipa) {
-			u32 checksum_offset;
+			u32 off;
 
 			/* Checksum header offset is in 4-byte units */
-			checksum_offset = sizeof(struct rmnet_map_header);
-			checksum_offset /= sizeof(u32);
-			val |= u32_encode_bits(checksum_offset,
-					       CS_METADATA_HDR_OFFSET_FMASK);
+			off = sizeof(struct rmnet_map_header) / sizeof(u32);
+			val |= ipa_reg_encode(reg, CS_METADATA_HDR_OFFSET, off);
 
 			enabled = version < IPA_VERSION_4_5
 					? IPA_CS_OFFLOAD_UL
@@ -513,24 +526,26 @@ static void ipa_endpoint_init_cfg(struct ipa_endpoint *endpoint)
 	} else {
 		enabled = IPA_CS_OFFLOAD_NONE;
 	}
-	val |= u32_encode_bits(enabled, CS_OFFLOAD_EN_FMASK);
+	val |= ipa_reg_encode(reg, CS_OFFLOAD_EN, enabled);
 	/* CS_GEN_QMB_MASTER_SEL is 0 */
 
-	iowrite32(val, endpoint->ipa->reg_virt + offset);
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));
 }
 
 static void ipa_endpoint_init_nat(struct ipa_endpoint *endpoint)
 {
-	u32 offset;
+	u32 endpoint_id = endpoint->endpoint_id;
+	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 	u32 val;
 
 	if (!endpoint->toward_ipa)
 		return;
 
-	offset = IPA_REG_ENDP_INIT_NAT_N_OFFSET(endpoint->endpoint_id);
-	val = u32_encode_bits(IPA_NAT_BYPASS, NAT_EN_FMASK);
+	reg = ipa_reg(ipa, ENDP_INIT_NAT);
+	val = ipa_reg_encode(reg, NAT_EN, IPA_NAT_BYPASS);
 
-	iowrite32(val, endpoint->ipa->reg_virt + offset);
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));
 }
 
 static u32
@@ -554,6 +569,50 @@ ipa_qmap_header_size(enum ipa_version version, struct ipa_endpoint *endpoint)
 	return header_size;
 }
 
+/* Encoded value for ENDP_INIT_HDR register HDR_LEN* field(s) */
+static u32 ipa_header_size_encode(enum ipa_version version,
+				  const struct ipa_reg *reg, u32 header_size)
+{
+	u32 field_max = ipa_reg_field_max(reg, HDR_LEN);
+	u32 val;
+
+	/* We know field_max can be used as a mask (2^n - 1) */
+	val = ipa_reg_encode(reg, HDR_LEN, header_size & field_max);
+	if (version < IPA_VERSION_4_5) {
+		WARN_ON(header_size > field_max);
+		return val;
+	}
+
+	/* IPA v4.5 adds a few more most-significant bits */
+	header_size >>= hweight32(field_max);
+	WARN_ON(header_size > ipa_reg_field_max(reg, HDR_LEN_MSB));
+	val |= ipa_reg_encode(reg, HDR_LEN_MSB, header_size);
+
+	return val;
+}
+
+/* Encoded value for ENDP_INIT_HDR register OFST_METADATA* field(s) */
+static u32 ipa_metadata_offset_encode(enum ipa_version version,
+				      const struct ipa_reg *reg, u32 offset)
+{
+	u32 field_max = ipa_reg_field_max(reg, HDR_OFST_METADATA);
+	u32 val;
+
+	/* We know field_max can be used as a mask (2^n - 1) */
+	val = ipa_reg_encode(reg, HDR_OFST_METADATA, offset);
+	if (version < IPA_VERSION_4_5) {
+		WARN_ON(offset > field_max);
+		return val;
+	}
+
+	/* IPA v4.5 adds a few more most-significant bits */
+	offset >>= hweight32(field_max);
+	WARN_ON(offset > ipa_reg_field_max(reg, HDR_OFST_METADATA_MSB));
+	val |= ipa_reg_encode(reg, HDR_OFST_METADATA_MSB, offset);
+
+	return val;
+}
+
 /**
  * ipa_endpoint_init_hdr() - Initialize HDR endpoint configuration register
  * @endpoint:	Endpoint pointer
@@ -577,36 +636,38 @@ ipa_qmap_header_size(enum ipa_version version, struct ipa_endpoint *endpoint)
  */
 static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint)
 {
-	u32 offset = IPA_REG_ENDP_INIT_HDR_N_OFFSET(endpoint->endpoint_id);
+	u32 endpoint_id = endpoint->endpoint_id;
 	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 	u32 val = 0;
 
+	reg = ipa_reg(ipa, ENDP_INIT_HDR);
 	if (endpoint->config.qmap) {
 		enum ipa_version version = ipa->version;
 		size_t header_size;
 
 		header_size = ipa_qmap_header_size(version, endpoint);
-		val = ipa_header_size_encoded(version, header_size);
+		val = ipa_header_size_encode(version, reg, header_size);
 
 		/* Define how to fill fields in a received QMAP header */
 		if (!endpoint->toward_ipa) {
-			u32 offset;	/* Field offset within header */
+			u32 off;     /* Field offset within header */
 
 			/* Where IPA will write the metadata value */
-			offset = offsetof(struct rmnet_map_header, mux_id);
-			val |= ipa_metadata_offset_encoded(version, offset);
+			off = offsetof(struct rmnet_map_header, mux_id);
+			val |= ipa_metadata_offset_encode(version, reg, off);
 
 			/* Where IPA will write the length */
-			offset = offsetof(struct rmnet_map_header, pkt_len);
+			off = offsetof(struct rmnet_map_header, pkt_len);
 			/* Upper bits are stored in HDR_EXT with IPA v4.5 */
 			if (version >= IPA_VERSION_4_5)
-				offset &= field_mask(HDR_OFST_PKT_SIZE_FMASK);
+				off &= ipa_reg_field_max(reg, HDR_OFST_PKT_SIZE);
 
-			val |= HDR_OFST_PKT_SIZE_VALID_FMASK;
-			val |= u32_encode_bits(offset, HDR_OFST_PKT_SIZE_FMASK);
+			val |= ipa_reg_bit(reg, HDR_OFST_PKT_SIZE_VALID);
+			val |= ipa_reg_encode(reg, HDR_OFST_PKT_SIZE, off);
 		}
 		/* For QMAP TX, metadata offset is 0 (modem assumes this) */
-		val |= HDR_OFST_METADATA_VALID_FMASK;
+		val |= ipa_reg_bit(reg, HDR_OFST_METADATA_VALID);
 
 		/* HDR_ADDITIONAL_CONST_LEN is 0; (RX only) */
 		/* HDR_A5_MUX is 0 */
@@ -614,19 +675,21 @@ static void ipa_endpoint_init_hdr(struct ipa_endpoint *endpoint)
 		/* HDR_METADATA_REG_VALID is 0 (TX only, version < v4.5) */
 	}
 
-	iowrite32(val, ipa->reg_virt + offset);
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));
 }
 
 static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint)
 {
-	u32 offset = IPA_REG_ENDP_INIT_HDR_EXT_N_OFFSET(endpoint->endpoint_id);
 	u32 pad_align = endpoint->config.rx.pad_align;
+	u32 endpoint_id = endpoint->endpoint_id;
 	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 	u32 val = 0;
 
+	reg = ipa_reg(ipa, ENDP_INIT_HDR_EXT);
 	if (endpoint->config.qmap) {
 		/* We have a header, so we must specify its endianness */
-		val |= HDR_ENDIANNESS_FMASK;	/* big endian */
+		val |= ipa_reg_bit(reg, HDR_ENDIANNESS);	/* big endian */
 
 		/* A QMAP header contains a 6 bit pad field at offset 0.
 		 * The RMNet driver assumes this field is meaningful in
@@ -636,16 +699,16 @@ static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint)
 		 * (although 0) should be ignored.
 		 */
 		if (!endpoint->toward_ipa) {
-			val |= HDR_TOTAL_LEN_OR_PAD_VALID_FMASK;
+			val |= ipa_reg_bit(reg, HDR_TOTAL_LEN_OR_PAD_VALID);
 			/* HDR_TOTAL_LEN_OR_PAD is 0 (pad, not total_len) */
-			val |= HDR_PAYLOAD_LEN_INC_PADDING_FMASK;
+			val |= ipa_reg_bit(reg, HDR_PAYLOAD_LEN_INC_PADDING);
 			/* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0 */
 		}
 	}
 
 	/* HDR_PAYLOAD_LEN_INC_PADDING is 0 */
 	if (!endpoint->toward_ipa)
-		val |= u32_encode_bits(pad_align, HDR_PAD_TO_ALIGNMENT_FMASK);
+		val |= ipa_reg_encode(reg, HDR_PAD_TO_ALIGNMENT, pad_align);
 
 	/* IPA v4.5 adds some most-significant bits to a few fields,
 	 * two of which are defined in the HDR (not HDR_EXT) register.
@@ -653,191 +716,170 @@ static void ipa_endpoint_init_hdr_ext(struct ipa_endpoint *endpoint)
 	if (ipa->version >= IPA_VERSION_4_5) {
 		/* HDR_TOTAL_LEN_OR_PAD_OFFSET is 0, so MSB is 0 */
 		if (endpoint->config.qmap && !endpoint->toward_ipa) {
-			u32 offset;
+			u32 mask = ipa_reg_field_max(reg, HDR_OFST_PKT_SIZE);
+			u32 off;     /* Field offset within header */
 
-			offset = offsetof(struct rmnet_map_header, pkt_len);
-			offset >>= hweight32(HDR_OFST_PKT_SIZE_FMASK);
-			val |= u32_encode_bits(offset,
-					       HDR_OFST_PKT_SIZE_MSB_FMASK);
+			off = offsetof(struct rmnet_map_header, pkt_len);
+			/* Low bits are in the ENDP_INIT_HDR register */
+			off >>= hweight32(mask);
+			val |= ipa_reg_encode(reg, HDR_OFST_PKT_SIZE_MSB, off);
 			/* HDR_ADDITIONAL_CONST_LEN is 0 so MSB is 0 */
 		}
 	}
-	iowrite32(val, ipa->reg_virt + offset);
+
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));
 }
 
 static void ipa_endpoint_init_hdr_metadata_mask(struct ipa_endpoint *endpoint)
 {
 	u32 endpoint_id = endpoint->endpoint_id;
+	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 	u32 val = 0;
 	u32 offset;
 
 	if (endpoint->toward_ipa)
 		return;		/* Register not valid for TX endpoints */
 
-	offset = IPA_REG_ENDP_INIT_HDR_METADATA_MASK_N_OFFSET(endpoint_id);
+	reg = ipa_reg(ipa,  ENDP_INIT_HDR_METADATA_MASK);
+	offset = ipa_reg_n_offset(reg, endpoint_id);
 
 	/* Note that HDR_ENDIANNESS indicates big endian header fields */
 	if (endpoint->config.qmap)
 		val = (__force u32)cpu_to_be32(IPA_ENDPOINT_QMAP_METADATA_MASK);
 
-	iowrite32(val, endpoint->ipa->reg_virt + offset);
+	iowrite32(val, ipa->reg_virt + offset);
 }
 
 static void ipa_endpoint_init_mode(struct ipa_endpoint *endpoint)
 {
-	u32 offset = IPA_REG_ENDP_INIT_MODE_N_OFFSET(endpoint->endpoint_id);
+	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
+	u32 offset;
 	u32 val;
 
 	if (!endpoint->toward_ipa)
 		return;		/* Register not valid for RX endpoints */
 
+	reg = ipa_reg(ipa, ENDP_INIT_MODE);
 	if (endpoint->config.dma_mode) {
 		enum ipa_endpoint_name name = endpoint->config.dma_endpoint;
-		u32 dma_endpoint_id;
+		u32 dma_endpoint_id = ipa->name_map[name]->endpoint_id;
 
-		dma_endpoint_id = endpoint->ipa->name_map[name]->endpoint_id;
-
-		val = u32_encode_bits(IPA_DMA, MODE_FMASK);
-		val |= u32_encode_bits(dma_endpoint_id, DEST_PIPE_INDEX_FMASK);
+		val = ipa_reg_encode(reg, ENDP_MODE, IPA_DMA);
+		val |= ipa_reg_encode(reg, DEST_PIPE_INDEX, dma_endpoint_id);
 	} else {
-		val = u32_encode_bits(IPA_BASIC, MODE_FMASK);
+		val = ipa_reg_encode(reg, ENDP_MODE, IPA_BASIC);
 	}
 	/* All other bits unspecified (and 0) */
 
-	iowrite32(val, endpoint->ipa->reg_virt + offset);
+	offset = ipa_reg_n_offset(reg, endpoint->endpoint_id);
+	iowrite32(val, ipa->reg_virt + offset);
 }
 
-/* Encoded values for AGGR endpoint register fields */
-static u32 aggr_byte_limit_encoded(enum ipa_version version, u32 limit)
+/* For IPA v4.5+, times are expressed using Qtime.  The AP uses one of two
+ * pulse generators (0 and 1) to measure elapsed time.  In ipa_qtime_config()
+ * they're configured to have granularity 100 usec and 1 msec, respectively.
+ *
+ * The return value is the positive or negative Qtime value to use to
+ * express the (microsecond) time provided.  A positive return value
+ * means pulse generator 0 can be used; otherwise use pulse generator 1.
+ */
+static int ipa_qtime_val(u32 microseconds, u32 max)
 {
-	if (version < IPA_VERSION_4_5)
-		return u32_encode_bits(limit, aggr_byte_limit_fmask(true));
+	u32 val;
 
-	return u32_encode_bits(limit, aggr_byte_limit_fmask(false));
+	/* Use 100 microsecond granularity if possible */
+	val = DIV_ROUND_CLOSEST(microseconds, 100);
+	if (val <= max)
+		return (int)val;
+
+	/* Have to use pulse generator 1 (millisecond granularity) */
+	val = DIV_ROUND_CLOSEST(microseconds, 1000);
+	WARN_ON(val > max);
+
+	return (int)-val;
 }
 
 /* Encode the aggregation timer limit (microseconds) based on IPA version */
-static u32 aggr_time_limit_encoded(enum ipa_version version, u32 limit)
+static u32 aggr_time_limit_encode(struct ipa *ipa, const struct ipa_reg *reg,
+				  u32 microseconds)
 {
-	u32 gran_sel;
-	u32 fmask;
+	u32 max;
 	u32 val;
 
-	if (version < IPA_VERSION_4_5) {
-		/* We set aggregation granularity in ipa_hardware_config() */
-		fmask = aggr_time_limit_fmask(true);
-		val = DIV_ROUND_CLOSEST(limit, IPA_AGGR_GRANULARITY);
-		WARN(val > field_max(fmask),
-		     "aggr_time_limit too large (%u > %u usec)\n",
-		     val, field_max(fmask) * IPA_AGGR_GRANULARITY);
-
-		return u32_encode_bits(val, fmask);
-	}
-
-	/* IPA v4.5 expresses the time limit using Qtime.  The AP has
-	 * pulse generators 0 and 1 available, which were configured
-	 * in ipa_qtime_config() to have granularity 100 usec and
-	 * 1 msec, respectively.  Use pulse generator 0 if possible,
-	 * otherwise fall back to pulse generator 1.
-	 */
-	fmask = aggr_time_limit_fmask(false);
-	val = DIV_ROUND_CLOSEST(limit, 100);
-	if (val > field_max(fmask)) {
-		/* Have to use pulse generator 1 (millisecond granularity) */
-		gran_sel = AGGR_GRAN_SEL_FMASK;
-		val = DIV_ROUND_CLOSEST(limit, 1000);
-		WARN(val > field_max(fmask),
-		     "aggr_time_limit too large (%u > %u usec)\n",
-		     limit, field_max(fmask) * 1000);
-	} else {
-		/* We can use pulse generator 0 (100 usec granularity) */
-		gran_sel = 0;
-	}
+	if (!microseconds)
+		return 0;	/* Nothing to compute if time limit is 0 */
 
-	return gran_sel | u32_encode_bits(val, fmask);
-}
+	max = ipa_reg_field_max(reg, TIME_LIMIT);
+	if (ipa->version >= IPA_VERSION_4_5) {
+		u32 gran_sel;
+		int ret;
+
+		/* Compute the Qtime limit value to use */
+		ret = ipa_qtime_val(microseconds, max);
+		if (ret < 0) {
+			val = -ret;
+			gran_sel = ipa_reg_bit(reg, AGGR_GRAN_SEL);
+		} else {
+			val = ret;
+			gran_sel = 0;
+		}
 
-static u32 aggr_sw_eof_active_encoded(enum ipa_version version, bool enabled)
-{
-	u32 val = enabled ? 1 : 0;
+		return gran_sel | ipa_reg_encode(reg, TIME_LIMIT, val);
+	}
 
-	if (version < IPA_VERSION_4_5)
-		return u32_encode_bits(val, aggr_sw_eof_active_fmask(true));
+	/* We program aggregation granularity in ipa_hardware_config() */
+	val = DIV_ROUND_CLOSEST(microseconds, IPA_AGGR_GRANULARITY);
+	WARN(val > max, "aggr_time_limit too large (%u > %u usec)\n",
+	     microseconds, max * IPA_AGGR_GRANULARITY);
 
-	return u32_encode_bits(val, aggr_sw_eof_active_fmask(false));
+	return ipa_reg_encode(reg, TIME_LIMIT, val);
 }
 
 static void ipa_endpoint_init_aggr(struct ipa_endpoint *endpoint)
 {
-	u32 offset = IPA_REG_ENDP_INIT_AGGR_N_OFFSET(endpoint->endpoint_id);
-	enum ipa_version version = endpoint->ipa->version;
+	u32 endpoint_id = endpoint->endpoint_id;
+	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 	u32 val = 0;
 
+	reg = ipa_reg(ipa, ENDP_INIT_AGGR);
 	if (endpoint->config.aggregation) {
 		if (!endpoint->toward_ipa) {
 			const struct ipa_endpoint_rx *rx_config;
 			u32 buffer_size;
-			bool close_eof;
 			u32 limit;
 
 			rx_config = &endpoint->config.rx;
-			val |= u32_encode_bits(IPA_ENABLE_AGGR, AGGR_EN_FMASK);
-			val |= u32_encode_bits(IPA_GENERIC, AGGR_TYPE_FMASK);
+			val |= ipa_reg_encode(reg, AGGR_EN, IPA_ENABLE_AGGR);
+			val |= ipa_reg_encode(reg, AGGR_TYPE, IPA_GENERIC);
 
 			buffer_size = rx_config->buffer_size;
 			limit = ipa_aggr_size_kb(buffer_size - NET_SKB_PAD,
 						 rx_config->aggr_hard_limit);
-			val |= aggr_byte_limit_encoded(version, limit);
+			val |= ipa_reg_encode(reg, BYTE_LIMIT, limit);
 
 			limit = rx_config->aggr_time_limit;
-			val |= aggr_time_limit_encoded(version, limit);
+			val |= aggr_time_limit_encode(ipa, reg, limit);
 
 			/* AGGR_PKT_LIMIT is 0 (unlimited) */
 
-			close_eof = rx_config->aggr_close_eof;
-			val |= aggr_sw_eof_active_encoded(version, close_eof);
+			if (rx_config->aggr_close_eof)
+				val |= ipa_reg_bit(reg, SW_EOF_ACTIVE);
 		} else {
-			val |= u32_encode_bits(IPA_ENABLE_DEAGGR,
-					       AGGR_EN_FMASK);
-			val |= u32_encode_bits(IPA_QCMAP, AGGR_TYPE_FMASK);
+			val |= ipa_reg_encode(reg, AGGR_EN, IPA_ENABLE_DEAGGR);
+			val |= ipa_reg_encode(reg, AGGR_TYPE, IPA_QCMAP);
 			/* other fields ignored */
 		}
 		/* AGGR_FORCE_CLOSE is 0 */
 		/* AGGR_GRAN_SEL is 0 for IPA v4.5 */
 	} else {
-		val |= u32_encode_bits(IPA_BYPASS_AGGR, AGGR_EN_FMASK);
+		val |= ipa_reg_encode(reg, AGGR_EN, IPA_BYPASS_AGGR);
 		/* other fields ignored */
 	}
 
-	iowrite32(val, endpoint->ipa->reg_virt + offset);
-}
-
-/* Return the Qtime-based head-of-line blocking timer value that
- * represents the given number of microseconds.  The result
- * includes both the timer value and the selected timer granularity.
- */
-static u32 hol_block_timer_qtime_val(struct ipa *ipa, u32 microseconds)
-{
-	u32 gran_sel;
-	u32 val;
-
-	/* IPA v4.5 expresses time limits using Qtime.  The AP has
-	 * pulse generators 0 and 1 available, which were configured
-	 * in ipa_qtime_config() to have granularity 100 usec and
-	 * 1 msec, respectively.  Use pulse generator 0 if possible,
-	 * otherwise fall back to pulse generator 1.
-	 */
-	val = DIV_ROUND_CLOSEST(microseconds, 100);
-	if (val > field_max(TIME_LIMIT_FMASK)) {
-		/* Have to use pulse generator 1 (millisecond granularity) */
-		gran_sel = GRAN_SEL_FMASK;
-		val = DIV_ROUND_CLOSEST(microseconds, 1000);
-	} else {
-		/* We can use pulse generator 0 (100 usec granularity) */
-		gran_sel = 0;
-	}
-
-	return gran_sel | u32_encode_bits(val, TIME_LIMIT_FMASK);
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));
 }
 
 /* The head-of-line blocking timer is defined as a tick count.  For
@@ -845,12 +887,11 @@ static u32 hol_block_timer_qtime_val(struct ipa *ipa, u32 microseconds)
  * derived from the 19.2 MHz SoC XO clock.  For older IPA versions
  * each tick represents 128 cycles of the IPA core clock.
  *
- * Return the encoded value that should be written to that register
- * that represents the timeout period provided.  For IPA v4.2 this
- * encodes a base and scale value, while for earlier versions the
- * value is a simple tick count.
+ * Return the encoded value representing the timeout period provided
+ * that should be written to the ENDP_INIT_HOL_BLOCK_TIMER register.
  */
-static u32 hol_block_timer_val(struct ipa *ipa, u32 microseconds)
+static u32 hol_block_timer_encode(struct ipa *ipa, const struct ipa_reg *reg,
+				  u32 microseconds)
 {
 	u32 width;
 	u32 scale;
@@ -862,18 +903,34 @@ static u32 hol_block_timer_val(struct ipa *ipa, u32 microseconds)
 	if (!microseconds)
 		return 0;	/* Nothing to compute if timer period is 0 */
 
-	if (ipa->version >= IPA_VERSION_4_5)
-		return hol_block_timer_qtime_val(ipa, microseconds);
+	if (ipa->version >= IPA_VERSION_4_5) {
+		u32 max = ipa_reg_field_max(reg, TIMER_LIMIT);
+		u32 gran_sel;
+		int ret;
+
+		/* Compute the Qtime limit value to use */
+		ret = ipa_qtime_val(microseconds, max);
+		if (ret < 0) {
+			val = -ret;
+			gran_sel = ipa_reg_bit(reg, TIMER_GRAN_SEL);
+		} else {
+			val = ret;
+			gran_sel = 0;
+		}
 
-	/* Use 64 bit arithmetic to avoid overflow... */
+		return gran_sel | ipa_reg_encode(reg, TIMER_LIMIT, val);
+	}
+
+	/* Use 64 bit arithmetic to avoid overflow */
 	rate = ipa_core_clock_rate(ipa);
 	ticks = DIV_ROUND_CLOSEST(microseconds * rate, 128 * USEC_PER_SEC);
-	/* ...but we still need to fit into a 32-bit register */
-	WARN_ON(ticks > U32_MAX);
+
+	/* We still need the result to fit into the field */
+	WARN_ON(ticks > ipa_reg_field_max(reg, TIMER_BASE_VALUE));
 
 	/* IPA v3.5.1 through v4.1 just record the tick count */
 	if (ipa->version < IPA_VERSION_4_2)
-		return (u32)ticks;
+		return ipa_reg_encode(reg, TIMER_BASE_VALUE, (u32)ticks);
 
 	/* For IPA v4.2, the tick count is represented by base and
 	 * scale fields within the 32-bit timer register, where:
@@ -883,8 +940,8 @@ static u32 hol_block_timer_val(struct ipa *ipa, u32 microseconds)
 	 * count, and extract the number of bits in the base field
 	 * such that high bit is included.
 	 */
-	high = fls(ticks);		/* 1..32 */
-	width = HWEIGHT32(BASE_VALUE_FMASK);
+	high = fls(ticks);		/* 1..32 (or warning above) */
+	width = hweight32(ipa_reg_fmask(reg, TIMER_BASE_VALUE));
 	scale = high > width ? high - width : 0;
 	if (scale) {
 		/* If we're scaling, round up to get a closer result */
@@ -894,8 +951,8 @@ static u32 hol_block_timer_val(struct ipa *ipa, u32 microseconds)
 			scale++;
 	}
 
-	val = u32_encode_bits(scale, SCALE_FMASK);
-	val |= u32_encode_bits(ticks >> scale, BASE_VALUE_FMASK);
+	val = ipa_reg_encode(reg, TIMER_SCALE, scale);
+	val |= ipa_reg_encode(reg, TIMER_BASE_VALUE, (u32)ticks >> scale);
 
 	return val;
 }
@@ -906,28 +963,34 @@ static void ipa_endpoint_init_hol_block_timer(struct ipa_endpoint *endpoint,
 {
 	u32 endpoint_id = endpoint->endpoint_id;
 	struct ipa *ipa = endpoint->ipa;
-	u32 offset;
+	const struct ipa_reg *reg;
 	u32 val;
 
 	/* This should only be changed when HOL_BLOCK_EN is disabled */
-	offset = IPA_REG_ENDP_INIT_HOL_BLOCK_TIMER_N_OFFSET(endpoint_id);
-	val = hol_block_timer_val(ipa, microseconds);
-	iowrite32(val, ipa->reg_virt + offset);
+	reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_TIMER);
+	val = hol_block_timer_encode(ipa, reg, microseconds);
+
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));
 }
 
 static void
 ipa_endpoint_init_hol_block_en(struct ipa_endpoint *endpoint, bool enable)
 {
 	u32 endpoint_id = endpoint->endpoint_id;
+	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 	u32 offset;
 	u32 val;
 
-	val = enable ? HOL_BLOCK_EN_FMASK : 0;
-	offset = IPA_REG_ENDP_INIT_HOL_BLOCK_EN_N_OFFSET(endpoint_id);
-	iowrite32(val, endpoint->ipa->reg_virt + offset);
+	reg = ipa_reg(ipa, ENDP_INIT_HOL_BLOCK_EN);
+	offset = ipa_reg_n_offset(reg, endpoint_id);
+	val = enable ? ipa_reg_bit(reg, HOL_BLOCK_EN) : 0;
+
+	iowrite32(val, ipa->reg_virt + offset);
+
 	/* When enabling, the register must be written twice for IPA v4.5+ */
-	if (enable && endpoint->ipa->version >= IPA_VERSION_4_5)
-		iowrite32(val, endpoint->ipa->reg_virt + offset);
+	if (enable && ipa->version >= IPA_VERSION_4_5)
+		iowrite32(val, ipa->reg_virt + offset);
 }
 
 /* Assumes HOL_BLOCK is in disabled state */
@@ -960,46 +1023,58 @@ void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa)
 
 static void ipa_endpoint_init_deaggr(struct ipa_endpoint *endpoint)
 {
-	u32 offset = IPA_REG_ENDP_INIT_DEAGGR_N_OFFSET(endpoint->endpoint_id);
+	u32 endpoint_id = endpoint->endpoint_id;
+	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 	u32 val = 0;
 
 	if (!endpoint->toward_ipa)
 		return;		/* Register not valid for RX endpoints */
 
+	reg = ipa_reg(ipa, ENDP_INIT_DEAGGR);
 	/* DEAGGR_HDR_LEN is 0 */
 	/* PACKET_OFFSET_VALID is 0 */
 	/* PACKET_OFFSET_LOCATION is ignored (not valid) */
 	/* MAX_PACKET_LEN is 0 (not enforced) */
 
-	iowrite32(val, endpoint->ipa->reg_virt + offset);
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));
 }
 
 static void ipa_endpoint_init_rsrc_grp(struct ipa_endpoint *endpoint)
 {
-	u32 offset = IPA_REG_ENDP_INIT_RSRC_GRP_N_OFFSET(endpoint->endpoint_id);
+	u32 resource_group = endpoint->config.resource_group;
+	u32 endpoint_id = endpoint->endpoint_id;
 	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 	u32 val;
 
-	val = rsrc_grp_encoded(ipa->version, endpoint->config.resource_group);
-	iowrite32(val, ipa->reg_virt + offset);
+	reg = ipa_reg(ipa, ENDP_INIT_RSRC_GRP);
+	val = ipa_reg_encode(reg, ENDP_RSRC_GRP, resource_group);
+
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));
 }
 
 static void ipa_endpoint_init_seq(struct ipa_endpoint *endpoint)
 {
-	u32 offset = IPA_REG_ENDP_INIT_SEQ_N_OFFSET(endpoint->endpoint_id);
-	u32 val = 0;
+	u32 endpoint_id = endpoint->endpoint_id;
+	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
+	u32 val;
 
 	if (!endpoint->toward_ipa)
 		return;		/* Register not valid for RX endpoints */
 
+	reg = ipa_reg(ipa, ENDP_INIT_SEQ);
+
 	/* Low-order byte configures primary packet processing */
-	val |= u32_encode_bits(endpoint->config.tx.seq_type, SEQ_TYPE_FMASK);
+	val = ipa_reg_encode(reg, SEQ_TYPE, endpoint->config.tx.seq_type);
 
-	/* Second byte configures replicated packet processing */
-	val |= u32_encode_bits(endpoint->config.tx.seq_rep_type,
-			       SEQ_REP_TYPE_FMASK);
+	/* Second byte (if supported) configures replicated packet processing */
+	if (ipa->version < IPA_VERSION_4_5)
+		val |= ipa_reg_encode(reg, SEQ_REP_TYPE,
+				      endpoint->config.tx.seq_rep_type);
 
-	iowrite32(val, endpoint->ipa->reg_virt + offset);
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));
 }
 
 /**
@@ -1049,13 +1124,12 @@ static void ipa_endpoint_status(struct ipa_endpoint *endpoint)
 {
 	u32 endpoint_id = endpoint->endpoint_id;
 	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 	u32 val = 0;
-	u32 offset;
-
-	offset = IPA_REG_ENDP_STATUS_N_OFFSET(endpoint_id);
 
+	reg = ipa_reg(ipa, ENDP_STATUS);
 	if (endpoint->config.status_enable) {
-		val |= STATUS_EN_FMASK;
+		val |= ipa_reg_bit(reg, STATUS_EN);
 		if (endpoint->toward_ipa) {
 			enum ipa_endpoint_name name;
 			u32 status_endpoint_id;
@@ -1063,16 +1137,16 @@ static void ipa_endpoint_status(struct ipa_endpoint *endpoint)
 			name = endpoint->config.tx.status_endpoint;
 			status_endpoint_id = ipa->name_map[name]->endpoint_id;
 
-			val |= u32_encode_bits(status_endpoint_id,
-					       STATUS_ENDP_FMASK);
+			val |= ipa_reg_encode(reg, STATUS_ENDP,
+					      status_endpoint_id);
 		}
 		/* STATUS_LOCATION is 0, meaning status element precedes
-		 * packet (not present for IPA v4.5)
+		 * packet (not present for IPA v4.5+)
 		 */
-		/* STATUS_PKT_SUPPRESS_FMASK is 0 (not present for v3.5.1) */
+		/* STATUS_PKT_SUPPRESS_FMASK is 0 (not present for v4.0+) */
 	}
 
-	iowrite32(val, ipa->reg_virt + offset);
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, endpoint_id));
 }
 
 static int ipa_endpoint_replenish_one(struct ipa_endpoint *endpoint,
@@ -1412,16 +1486,18 @@ void ipa_endpoint_trans_release(struct ipa_endpoint *endpoint,
 
 void ipa_endpoint_default_route_set(struct ipa *ipa, u32 endpoint_id)
 {
+	const struct ipa_reg *reg;
 	u32 val;
 
+	reg = ipa_reg(ipa, ROUTE);
 	/* ROUTE_DIS is 0 */
-	val = u32_encode_bits(endpoint_id, ROUTE_DEF_PIPE_FMASK);
-	val |= ROUTE_DEF_HDR_TABLE_FMASK;
-	val |= u32_encode_bits(0, ROUTE_DEF_HDR_OFST_FMASK);
-	val |= u32_encode_bits(endpoint_id, ROUTE_FRAG_DEF_PIPE_FMASK);
-	val |= ROUTE_DEF_RETAIN_HDR_FMASK;
+	val = ipa_reg_encode(reg, ROUTE_DEF_PIPE, endpoint_id);
+	val |= ipa_reg_bit(reg, ROUTE_DEF_HDR_TABLE);
+	/* ROUTE_DEF_HDR_OFST is 0 */
+	val |= ipa_reg_encode(reg, ROUTE_FRAG_DEF_PIPE, endpoint_id);
+	val |= ipa_reg_bit(reg, ROUTE_DEF_RETAIN_HDR);
 
-	iowrite32(val, ipa->reg_virt + IPA_REG_ROUTE_OFFSET);
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
 }
 
 void ipa_endpoint_default_route_clear(struct ipa *ipa)
@@ -1765,6 +1841,7 @@ void ipa_endpoint_teardown(struct ipa *ipa)
 int ipa_endpoint_config(struct ipa *ipa)
 {
 	struct device *dev = &ipa->pdev->dev;
+	const struct ipa_reg *reg;
 	u32 initialized;
 	u32 rx_base;
 	u32 rx_mask;
@@ -1791,11 +1868,12 @@ int ipa_endpoint_config(struct ipa *ipa)
 	/* Find out about the endpoints supplied by the hardware, and ensure
 	 * the highest one doesn't exceed the number we support.
 	 */
-	val = ioread32(ipa->reg_virt + IPA_REG_FLAVOR_0_OFFSET);
+	reg = ipa_reg(ipa, FLAVOR_0);
+	val = ioread32(ipa->reg_virt + ipa_reg_offset(reg));
 
 	/* Our RX is an IPA producer */
-	rx_base = u32_get_bits(val, IPA_PROD_LOWEST_FMASK);
-	max = rx_base + u32_get_bits(val, IPA_MAX_PROD_PIPES_FMASK);
+	rx_base = ipa_reg_decode(reg, PROD_LOWEST, val);
+	max = rx_base + ipa_reg_decode(reg, MAX_PROD_PIPES, val);
 	if (max > IPA_ENDPOINT_MAX) {
 		dev_err(dev, "too many endpoints (%u > %u)\n",
 			max, IPA_ENDPOINT_MAX);
@@ -1804,7 +1882,7 @@ int ipa_endpoint_config(struct ipa *ipa)
 	rx_mask = GENMASK(max - 1, rx_base);
 
 	/* Our TX is an IPA consumer */
-	max = u32_get_bits(val, IPA_MAX_CONS_PIPES_FMASK);
+	max = ipa_reg_decode(reg, MAX_CONS_PIPES, val);
 	tx_mask = GENMASK(max - 1, 0);
 
 	ipa->available = rx_mask | tx_mask;
diff --git a/drivers/net/ipa/ipa_endpoint.h b/drivers/net/ipa/ipa_endpoint.h
index 28e0a7386fd7..d8dfa24f5214 100644
--- a/drivers/net/ipa/ipa_endpoint.h
+++ b/drivers/net/ipa/ipa_endpoint.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2020 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 #ifndef _IPA_ENDPOINT_H_
 #define _IPA_ENDPOINT_H_
diff --git a/drivers/net/ipa/ipa_interrupt.c b/drivers/net/ipa/ipa_interrupt.c
index 307bed2ee707..c269432f9c2e 100644
--- a/drivers/net/ipa/ipa_interrupt.c
+++ b/drivers/net/ipa/ipa_interrupt.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 
 /* DOC: IPA Interrupts
@@ -53,13 +53,15 @@ static void ipa_interrupt_process(struct ipa_interrupt *interrupt, u32 irq_id)
 {
 	bool uc_irq = ipa_interrupt_uc(interrupt, irq_id);
 	struct ipa *ipa = interrupt->ipa;
+	const struct ipa_reg *reg;
 	u32 mask = BIT(irq_id);
 	u32 offset;
 
 	/* For microcontroller interrupts, clear the interrupt right away,
 	 * "to avoid clearing unhandled interrupts."
 	 */
-	offset = ipa_reg_irq_clr_offset(ipa->version);
+	reg = ipa_reg(ipa, IPA_IRQ_CLR);
+	offset = ipa_reg_offset(reg);
 	if (uc_irq)
 		iowrite32(mask, ipa->reg_virt + offset);
 
@@ -80,6 +82,7 @@ static irqreturn_t ipa_isr_thread(int irq, void *dev_id)
 	struct ipa_interrupt *interrupt = dev_id;
 	struct ipa *ipa = interrupt->ipa;
 	u32 enabled = interrupt->enabled;
+	const struct ipa_reg *reg;
 	struct device *dev;
 	u32 pending;
 	u32 offset;
@@ -95,7 +98,8 @@ static irqreturn_t ipa_isr_thread(int irq, void *dev_id)
 	 * including conditions whose interrupt is not enabled.  Handle
 	 * only the enabled ones.
 	 */
-	offset = ipa_reg_irq_stts_offset(ipa->version);
+	reg = ipa_reg(ipa, IPA_IRQ_STTS);
+	offset = ipa_reg_offset(reg);
 	pending = ioread32(ipa->reg_virt + offset);
 	while ((mask = pending & enabled)) {
 		do {
@@ -112,7 +116,8 @@ static irqreturn_t ipa_isr_thread(int irq, void *dev_id)
 	if (pending) {
 		dev_dbg(dev, "clearing disabled IPA interrupts 0x%08x\n",
 			pending);
-		offset = ipa_reg_irq_clr_offset(ipa->version);
+		reg = ipa_reg(ipa, IPA_IRQ_CLR);
+		offset = ipa_reg_offset(reg);
 		iowrite32(pending, ipa->reg_virt + offset);
 	}
 out_power_put:
@@ -128,6 +133,7 @@ static void ipa_interrupt_suspend_control(struct ipa_interrupt *interrupt,
 {
 	struct ipa *ipa = interrupt->ipa;
 	u32 mask = BIT(endpoint_id);
+	const struct ipa_reg *reg;
 	u32 offset;
 	u32 val;
 
@@ -137,7 +143,8 @@ static void ipa_interrupt_suspend_control(struct ipa_interrupt *interrupt,
 	if (ipa->version == IPA_VERSION_3_0)
 		return;
 
-	offset = ipa_reg_irq_suspend_en_offset(ipa->version);
+	reg = ipa_reg(ipa, IRQ_SUSPEND_EN);
+	offset = ipa_reg_offset(reg);
 	val = ioread32(ipa->reg_virt + offset);
 	if (enable)
 		val |= mask;
@@ -164,18 +171,18 @@ ipa_interrupt_suspend_disable(struct ipa_interrupt *interrupt, u32 endpoint_id)
 void ipa_interrupt_suspend_clear_all(struct ipa_interrupt *interrupt)
 {
 	struct ipa *ipa = interrupt->ipa;
-	u32 offset;
+	const struct ipa_reg *reg;
 	u32 val;
 
-	offset = ipa_reg_irq_suspend_info_offset(ipa->version);
-	val = ioread32(ipa->reg_virt + offset);
+	reg = ipa_reg(ipa, IRQ_SUSPEND_INFO);
+	val = ioread32(ipa->reg_virt + ipa_reg_offset(reg));
 
 	/* SUSPEND interrupt status isn't cleared on IPA version 3.0 */
 	if (ipa->version == IPA_VERSION_3_0)
 		return;
 
-	offset = ipa_reg_irq_suspend_clr_offset(ipa->version);
-	iowrite32(val, ipa->reg_virt + offset);
+	reg = ipa_reg(ipa, IRQ_SUSPEND_CLR);
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
 }
 
 /* Simulate arrival of an IPA TX_SUSPEND interrupt */
@@ -189,7 +196,7 @@ void ipa_interrupt_add(struct ipa_interrupt *interrupt,
 		       enum ipa_irq_id ipa_irq, ipa_irq_handler_t handler)
 {
 	struct ipa *ipa = interrupt->ipa;
-	u32 offset;
+	const struct ipa_reg *reg;
 
 	if (WARN_ON(ipa_irq >= IPA_IRQ_COUNT))
 		return;
@@ -198,8 +205,9 @@ void ipa_interrupt_add(struct ipa_interrupt *interrupt,
 
 	/* Update the IPA interrupt mask to enable it */
 	interrupt->enabled |= BIT(ipa_irq);
-	offset = ipa_reg_irq_en_offset(ipa->version);
-	iowrite32(interrupt->enabled, ipa->reg_virt + offset);
+
+	reg = ipa_reg(ipa, IPA_IRQ_EN);
+	iowrite32(interrupt->enabled, ipa->reg_virt + ipa_reg_offset(reg));
 }
 
 /* Remove the handler for an IPA interrupt type */
@@ -207,15 +215,16 @@ void
 ipa_interrupt_remove(struct ipa_interrupt *interrupt, enum ipa_irq_id ipa_irq)
 {
 	struct ipa *ipa = interrupt->ipa;
-	u32 offset;
+	const struct ipa_reg *reg;
 
 	if (WARN_ON(ipa_irq >= IPA_IRQ_COUNT))
 		return;
 
 	/* Update the IPA interrupt mask to disable it */
 	interrupt->enabled &= ~BIT(ipa_irq);
-	offset = ipa_reg_irq_en_offset(ipa->version);
-	iowrite32(interrupt->enabled, ipa->reg_virt + offset);
+
+	reg = ipa_reg(ipa, IPA_IRQ_EN);
+	iowrite32(interrupt->enabled, ipa->reg_virt + ipa_reg_offset(reg));
 
 	interrupt->handler[ipa_irq] = NULL;
 }
@@ -225,8 +234,8 @@ struct ipa_interrupt *ipa_interrupt_config(struct ipa *ipa)
 {
 	struct device *dev = &ipa->pdev->dev;
 	struct ipa_interrupt *interrupt;
+	const struct ipa_reg *reg;
 	unsigned int irq;
-	u32 offset;
 	int ret;
 
 	ret = platform_get_irq_byname(ipa->pdev, "ipa");
@@ -244,8 +253,8 @@ struct ipa_interrupt *ipa_interrupt_config(struct ipa *ipa)
 	interrupt->irq = irq;
 
 	/* Start with all IPA interrupts disabled */
-	offset = ipa_reg_irq_en_offset(ipa->version);
-	iowrite32(0, ipa->reg_virt + offset);
+	reg = ipa_reg(ipa, IPA_IRQ_EN);
+	iowrite32(0, ipa->reg_virt + ipa_reg_offset(reg));
 
 	ret = request_threaded_irq(irq, NULL, ipa_isr_thread, IRQF_ONESHOT,
 				   "ipa", interrupt);
diff --git a/drivers/net/ipa/ipa_interrupt.h b/drivers/net/ipa/ipa_interrupt.h
index 231390cea52a..f31fd9965fdc 100644
--- a/drivers/net/ipa/ipa_interrupt.h
+++ b/drivers/net/ipa/ipa_interrupt.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #ifndef _IPA_INTERRUPT_H_
 #define _IPA_INTERRUPT_H_
diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c
index 32962d885acd..3461ad3029ab 100644
--- a/drivers/net/ipa/ipa_main.c
+++ b/drivers/net/ipa/ipa_main.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2021 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
@@ -183,31 +183,97 @@ static void ipa_teardown(struct ipa *ipa)
 	gsi_teardown(&ipa->gsi);
 }
 
+static void
+ipa_hardware_config_bcr(struct ipa *ipa, const struct ipa_data *data)
+{
+	const struct ipa_reg *reg;
+	u32 val;
+
+	/* IPA v4.5+ has no backward compatibility register */
+	if (ipa->version >= IPA_VERSION_4_5)
+		return;
+
+	reg = ipa_reg(ipa, IPA_BCR);
+	val = data->backward_compat;
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
+}
+
+static void ipa_hardware_config_tx(struct ipa *ipa)
+{
+	enum ipa_version version = ipa->version;
+	const struct ipa_reg *reg;
+	u32 offset;
+	u32 val;
+
+	if (version <= IPA_VERSION_4_0 || version >= IPA_VERSION_4_5)
+		return;
+
+	/* Disable PA mask to allow HOLB drop */
+	reg = ipa_reg(ipa, IPA_TX_CFG);
+	offset = ipa_reg_offset(reg);
+
+	val = ioread32(ipa->reg_virt + offset);
+
+	val &= ~ipa_reg_bit(reg, PA_MASK_EN);
+
+	iowrite32(val, ipa->reg_virt + offset);
+}
+
+static void ipa_hardware_config_clkon(struct ipa *ipa)
+{
+	enum ipa_version version = ipa->version;
+	const struct ipa_reg *reg;
+	u32 val;
+
+	if (version >= IPA_VERSION_4_5)
+		return;
+
+	if (version < IPA_VERSION_4_0 && version != IPA_VERSION_3_1)
+		return;
+
+	/* Implement some hardware workarounds */
+	reg = ipa_reg(ipa, CLKON_CFG);
+	if (version == IPA_VERSION_3_1) {
+		/* Disable MISC clock gating */
+		val = ipa_reg_bit(reg, CLKON_MISC);
+	} else {	/* IPA v4.0+ */
+		/* Enable open global clocks in the CLKON configuration */
+		val = ipa_reg_bit(reg, CLKON_GLOBAL);
+		val |= ipa_reg_bit(reg, GLOBAL_2X_CLK);
+	}
+
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
+}
+
 /* Configure bus access behavior for IPA components */
 static void ipa_hardware_config_comp(struct ipa *ipa)
 {
+	const struct ipa_reg *reg;
+	u32 offset;
 	u32 val;
 
 	/* Nothing to configure prior to IPA v4.0 */
 	if (ipa->version < IPA_VERSION_4_0)
 		return;
 
-	val = ioread32(ipa->reg_virt + IPA_REG_COMP_CFG_OFFSET);
+	reg = ipa_reg(ipa, COMP_CFG);
+	offset = ipa_reg_offset(reg);
+	val = ioread32(ipa->reg_virt + offset);
 
 	if (ipa->version == IPA_VERSION_4_0) {
-		val &= ~IPA_QMB_SELECT_CONS_EN_FMASK;
-		val &= ~IPA_QMB_SELECT_PROD_EN_FMASK;
-		val &= ~IPA_QMB_SELECT_GLOBAL_EN_FMASK;
+		val &= ~ipa_reg_bit(reg, IPA_QMB_SELECT_CONS_EN);
+		val &= ~ipa_reg_bit(reg, IPA_QMB_SELECT_PROD_EN);
+		val &= ~ipa_reg_bit(reg, IPA_QMB_SELECT_GLOBAL_EN);
 	} else if (ipa->version < IPA_VERSION_4_5) {
-		val |= GSI_MULTI_AXI_MASTERS_DIS_FMASK;
+		val |= ipa_reg_bit(reg, GSI_MULTI_AXI_MASTERS_DIS);
 	} else {
-		/* For IPA v4.5 IPA_FULL_FLUSH_WAIT_RSC_CLOSE_EN is 0 */
+		/* For IPA v4.5 FULL_FLUSH_WAIT_RS_CLOSURE_EN is 0 */
 	}
 
-	val |= GSI_MULTI_INORDER_RD_DIS_FMASK;
-	val |= GSI_MULTI_INORDER_WR_DIS_FMASK;
+	val |= ipa_reg_bit(reg, GSI_MULTI_INORDER_RD_DIS);
+	val |= ipa_reg_bit(reg, GSI_MULTI_INORDER_WR_DIS);
 
-	iowrite32(val, ipa->reg_virt + IPA_REG_COMP_CFG_OFFSET);
+	iowrite32(val, ipa->reg_virt + offset);
 }
 
 /* Configure DDR and (possibly) PCIe max read/write QSB values */
@@ -216,6 +282,7 @@ ipa_hardware_config_qsb(struct ipa *ipa, const struct ipa_data *data)
 {
 	const struct ipa_qsb_data *data0;
 	const struct ipa_qsb_data *data1;
+	const struct ipa_reg *reg;
 	u32 val;
 
 	/* QMB 0 represents DDR; QMB 1 (if present) represents PCIe */
@@ -224,25 +291,31 @@ ipa_hardware_config_qsb(struct ipa *ipa, const struct ipa_data *data)
 		data1 = &data->qsb_data[IPA_QSB_MASTER_PCIE];
 
 	/* Max outstanding write accesses for QSB masters */
-	val = u32_encode_bits(data0->max_writes, GEN_QMB_0_MAX_WRITES_FMASK);
+	reg = ipa_reg(ipa, QSB_MAX_WRITES);
+
+	val = ipa_reg_encode(reg, GEN_QMB_0_MAX_WRITES, data0->max_writes);
 	if (data->qsb_count > 1)
-		val |= u32_encode_bits(data1->max_writes,
-				       GEN_QMB_1_MAX_WRITES_FMASK);
-	iowrite32(val, ipa->reg_virt + IPA_REG_QSB_MAX_WRITES_OFFSET);
+		val |= ipa_reg_encode(reg, GEN_QMB_1_MAX_WRITES,
+				      data1->max_writes);
+
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
 
 	/* Max outstanding read accesses for QSB masters */
-	val = u32_encode_bits(data0->max_reads, GEN_QMB_0_MAX_READS_FMASK);
+	reg = ipa_reg(ipa, QSB_MAX_READS);
+
+	val = ipa_reg_encode(reg, GEN_QMB_0_MAX_READS, data0->max_reads);
 	if (ipa->version >= IPA_VERSION_4_0)
-		val |= u32_encode_bits(data0->max_reads_beats,
-				       GEN_QMB_0_MAX_READS_BEATS_FMASK);
+		val |= ipa_reg_encode(reg, GEN_QMB_0_MAX_READS_BEATS,
+				      data0->max_reads_beats);
 	if (data->qsb_count > 1) {
-		val |= u32_encode_bits(data1->max_reads,
-				       GEN_QMB_1_MAX_READS_FMASK);
+		val = ipa_reg_encode(reg, GEN_QMB_1_MAX_READS,
+				     data1->max_reads);
 		if (ipa->version >= IPA_VERSION_4_0)
-			val |= u32_encode_bits(data1->max_reads_beats,
-					       GEN_QMB_1_MAX_READS_BEATS_FMASK);
+			val |= ipa_reg_encode(reg, GEN_QMB_1_MAX_READS_BEATS,
+					      data1->max_reads_beats);
 	}
-	iowrite32(val, ipa->reg_virt + IPA_REG_QSB_MAX_READS_OFFSET);
+
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
 }
 
 /* The internal inactivity timer clock is used for the aggregation timer */
@@ -278,48 +351,96 @@ static __always_inline u32 ipa_aggr_granularity_val(u32 usec)
  */
 static void ipa_qtime_config(struct ipa *ipa)
 {
+	const struct ipa_reg *reg;
+	u32 offset;
 	u32 val;
 
 	/* Timer clock divider must be disabled when we change the rate */
-	iowrite32(0, ipa->reg_virt + IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET);
+	reg = ipa_reg(ipa, TIMERS_XO_CLK_DIV_CFG);
+	iowrite32(0, ipa->reg_virt + ipa_reg_offset(reg));
 
+	reg = ipa_reg(ipa, QTIME_TIMESTAMP_CFG);
 	/* Set DPL time stamp resolution to use Qtime (instead of 1 msec) */
-	val = u32_encode_bits(DPL_TIMESTAMP_SHIFT, DPL_TIMESTAMP_LSB_FMASK);
-	val |= u32_encode_bits(1, DPL_TIMESTAMP_SEL_FMASK);
+	val = ipa_reg_encode(reg, DPL_TIMESTAMP_LSB, DPL_TIMESTAMP_SHIFT);
+	val |= ipa_reg_bit(reg, DPL_TIMESTAMP_SEL);
 	/* Configure tag and NAT Qtime timestamp resolution as well */
-	val |= u32_encode_bits(TAG_TIMESTAMP_SHIFT, TAG_TIMESTAMP_LSB_FMASK);
-	val |= u32_encode_bits(NAT_TIMESTAMP_SHIFT, NAT_TIMESTAMP_LSB_FMASK);
-	iowrite32(val, ipa->reg_virt + IPA_REG_QTIME_TIMESTAMP_CFG_OFFSET);
+	val = ipa_reg_encode(reg, TAG_TIMESTAMP_LSB, TAG_TIMESTAMP_SHIFT);
+	val = ipa_reg_encode(reg, NAT_TIMESTAMP_LSB, NAT_TIMESTAMP_SHIFT);
+
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
 
 	/* Set granularity of pulse generators used for other timers */
-	val = u32_encode_bits(IPA_GRAN_100_US, GRAN_0_FMASK);
-	val |= u32_encode_bits(IPA_GRAN_1_MS, GRAN_1_FMASK);
-	val |= u32_encode_bits(IPA_GRAN_1_MS, GRAN_2_FMASK);
-	iowrite32(val, ipa->reg_virt + IPA_REG_TIMERS_PULSE_GRAN_CFG_OFFSET);
+	reg = ipa_reg(ipa, TIMERS_PULSE_GRAN_CFG);
+	val = ipa_reg_encode(reg, PULSE_GRAN_0, IPA_GRAN_100_US);
+	val |= ipa_reg_encode(reg, PULSE_GRAN_1, IPA_GRAN_1_MS);
+	val |= ipa_reg_encode(reg, PULSE_GRAN_2, IPA_GRAN_1_MS);
+
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
 
 	/* Actual divider is 1 more than value supplied here */
-	val = u32_encode_bits(IPA_XO_CLOCK_DIVIDER - 1, DIV_VALUE_FMASK);
-	iowrite32(val, ipa->reg_virt + IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET);
+	reg = ipa_reg(ipa, TIMERS_XO_CLK_DIV_CFG);
+	offset = ipa_reg_offset(reg);
+	val = ipa_reg_encode(reg, DIV_VALUE, IPA_XO_CLOCK_DIVIDER - 1);
+
+	iowrite32(val, ipa->reg_virt + offset);
 
 	/* Divider value is set; re-enable the common timer clock divider */
-	val |= u32_encode_bits(1, DIV_ENABLE_FMASK);
-	iowrite32(val, ipa->reg_virt + IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET);
+	val |= ipa_reg_bit(reg, DIV_ENABLE);
+
+	iowrite32(val, ipa->reg_virt + offset);
+}
+
+/* Before IPA v4.5 timing is controlled by a counter register */
+static void ipa_hardware_config_counter(struct ipa *ipa)
+{
+	u32 granularity = ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY);
+	const struct ipa_reg *reg;
+	u32 val;
+
+	reg = ipa_reg(ipa, COUNTER_CFG);
+	/* If defined, EOT_COAL_GRANULARITY is 0 */
+	val = ipa_reg_encode(reg, AGGR_GRANULARITY, granularity);
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
+}
+
+static void ipa_hardware_config_timing(struct ipa *ipa)
+{
+	if (ipa->version < IPA_VERSION_4_5)
+		ipa_hardware_config_counter(ipa);
+	else
+		ipa_qtime_config(ipa);
+}
+
+static void ipa_hardware_config_hashing(struct ipa *ipa)
+{
+	const struct ipa_reg *reg;
+
+	if (ipa->version != IPA_VERSION_4_2)
+		return;
+
+	/* IPA v4.2 does not support hashed tables, so disable them */
+	reg = ipa_reg(ipa, FILT_ROUT_HASH_EN);
+
+	/* IPV6_ROUTER_HASH, IPV6_FILTER_HASH, IPV4_ROUTER_HASH,
+	 * IPV4_FILTER_HASH are all zero.
+	 */
+	iowrite32(0, ipa->reg_virt + ipa_reg_offset(reg));
 }
 
 static void ipa_idle_indication_cfg(struct ipa *ipa,
 				    u32 enter_idle_debounce_thresh,
 				    bool const_non_idle_enable)
 {
-	u32 offset;
+	const struct ipa_reg *reg;
 	u32 val;
 
-	val = u32_encode_bits(enter_idle_debounce_thresh,
-			      ENTER_IDLE_DEBOUNCE_THRESH_FMASK);
+	reg = ipa_reg(ipa, IDLE_INDICATION_CFG);
+	val = ipa_reg_encode(reg, ENTER_IDLE_DEBOUNCE_THRESH,
+			     enter_idle_debounce_thresh);
 	if (const_non_idle_enable)
-		val |= CONST_NON_IDLE_ENABLE_FMASK;
+		val |= ipa_reg_bit(reg, CONST_NON_IDLE_ENABLE);
 
-	offset = ipa_reg_idle_indication_cfg_offset(ipa->version);
-	iowrite32(val, ipa->reg_virt + offset);
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
 }
 
 /**
@@ -349,55 +470,13 @@ static void ipa_hardware_dcd_deconfig(struct ipa *ipa)
  */
 static void ipa_hardware_config(struct ipa *ipa, const struct ipa_data *data)
 {
-	enum ipa_version version = ipa->version;
-	u32 granularity;
-	u32 val;
-
-	/* IPA v4.5+ has no backward compatibility register */
-	if (version < IPA_VERSION_4_5) {
-		val = data->backward_compat;
-		iowrite32(val, ipa->reg_virt + IPA_REG_BCR_OFFSET);
-	}
-
-	/* Implement some hardware workarounds */
-	if (version >= IPA_VERSION_4_0 && version < IPA_VERSION_4_5) {
-		/* Disable PA mask to allow HOLB drop */
-		val = ioread32(ipa->reg_virt + IPA_REG_TX_CFG_OFFSET);
-		val &= ~PA_MASK_EN_FMASK;
-		iowrite32(val, ipa->reg_virt + IPA_REG_TX_CFG_OFFSET);
-
-		/* Enable open global clocks in the CLKON configuration */
-		val = GLOBAL_FMASK | GLOBAL_2X_CLK_FMASK;
-	} else if (version == IPA_VERSION_3_1) {
-		val = MISC_FMASK;	/* Disable MISC clock gating */
-	} else {
-		val = 0;		/* No CLKON configuration needed */
-	}
-	if (val)
-		iowrite32(val, ipa->reg_virt + IPA_REG_CLKON_CFG_OFFSET);
-
+	ipa_hardware_config_bcr(ipa, data);
+	ipa_hardware_config_tx(ipa);
+	ipa_hardware_config_clkon(ipa);
 	ipa_hardware_config_comp(ipa);
-
-	/* Configure system bus limits */
 	ipa_hardware_config_qsb(ipa, data);
-
-	if (version < IPA_VERSION_4_5) {
-		/* Configure aggregation timer granularity */
-		granularity = ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY);
-		val = u32_encode_bits(granularity, AGGR_GRANULARITY_FMASK);
-		iowrite32(val, ipa->reg_virt + IPA_REG_COUNTER_CFG_OFFSET);
-	} else {
-		ipa_qtime_config(ipa);
-	}
-
-	/* IPA v4.2 does not support hashed tables, so disable them */
-	if (version == IPA_VERSION_4_2) {
-		u32 offset = ipa_reg_filt_rout_hash_en_offset(version);
-
-		iowrite32(0, ipa->reg_virt + offset);
-	}
-
-	/* Enable dynamic clock division */
+	ipa_hardware_config_timing(ipa);
+	ipa_hardware_config_hashing(ipa);
 	ipa_hardware_dcd_config(ipa);
 }
 
@@ -612,29 +691,6 @@ static void ipa_validate_build(void)
 
 	/* Aggregation granularity value can't be 0, and must fit */
 	BUILD_BUG_ON(!ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY));
-	BUILD_BUG_ON(ipa_aggr_granularity_val(IPA_AGGR_GRANULARITY) >
-			field_max(AGGR_GRANULARITY_FMASK));
-}
-
-static bool ipa_version_valid(enum ipa_version version)
-{
-	switch (version) {
-	case IPA_VERSION_3_0:
-	case IPA_VERSION_3_1:
-	case IPA_VERSION_3_5:
-	case IPA_VERSION_3_5_1:
-	case IPA_VERSION_4_0:
-	case IPA_VERSION_4_1:
-	case IPA_VERSION_4_2:
-	case IPA_VERSION_4_5:
-	case IPA_VERSION_4_7:
-	case IPA_VERSION_4_9:
-	case IPA_VERSION_4_11:
-		return true;
-
-	default:
-		return false;
-	}
 }
 
 /**
@@ -678,8 +734,8 @@ static int ipa_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	if (!ipa_version_valid(data->version)) {
-		dev_err(dev, "invalid IPA version\n");
+	if (!ipa_version_supported(data->version)) {
+		dev_err(dev, "unsupported IPA version %u\n", data->version);
 		return -EINVAL;
 	}
 
diff --git a/drivers/net/ipa/ipa_mem.c b/drivers/net/ipa/ipa_mem.c
index 53a1dbeaffa6..f84c6830495a 100644
--- a/drivers/net/ipa/ipa_mem.c
+++ b/drivers/net/ipa/ipa_mem.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2021 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
@@ -75,6 +75,7 @@ ipa_mem_zero_region_add(struct gsi_trans *trans, enum ipa_mem_id mem_id)
 int ipa_mem_setup(struct ipa *ipa)
 {
 	dma_addr_t addr = ipa->zero_addr;
+	const struct ipa_reg *reg;
 	const struct ipa_mem *mem;
 	struct gsi_trans *trans;
 	u32 offset;
@@ -112,8 +113,10 @@ int ipa_mem_setup(struct ipa *ipa)
 	/* Tell the hardware where the processing context area is located */
 	mem = ipa_mem_find(ipa, IPA_MEM_MODEM_PROC_CTX);
 	offset = ipa->mem_offset + mem->offset;
-	val = proc_cntxt_base_addr_encoded(ipa->version, offset);
-	iowrite32(val, ipa->reg_virt + IPA_REG_LOCAL_PKT_PROC_CNTXT_OFFSET);
+
+	reg = ipa_reg(ipa, LOCAL_PKT_PROC_CNTXT);
+	val = ipa_reg_encode(reg, IPA_BASE_ADDR, offset);
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
 
 	return 0;
 }
@@ -306,6 +309,7 @@ static bool ipa_mem_size_valid(struct ipa *ipa)
 int ipa_mem_config(struct ipa *ipa)
 {
 	struct device *dev = &ipa->pdev->dev;
+	const struct ipa_reg *reg;
 	const struct ipa_mem *mem;
 	dma_addr_t addr;
 	u32 mem_size;
@@ -314,12 +318,14 @@ int ipa_mem_config(struct ipa *ipa)
 	u32 i;
 
 	/* Check the advertised location and size of the shared memory area */
-	val = ioread32(ipa->reg_virt + IPA_REG_SHARED_MEM_SIZE_OFFSET);
+	reg = ipa_reg(ipa, SHARED_MEM_SIZE);
+	val = ioread32(ipa->reg_virt + ipa_reg_offset(reg));
 
 	/* The fields in the register are in 8 byte units */
-	ipa->mem_offset = 8 * u32_get_bits(val, SHARED_MEM_BADDR_FMASK);
+	ipa->mem_offset = 8 * ipa_reg_decode(reg, MEM_BADDR, val);
+
 	/* Make sure the end is within the region's mapped space */
-	mem_size = 8 * u32_get_bits(val, SHARED_MEM_SIZE_FMASK);
+	mem_size = 8 * ipa_reg_decode(reg, MEM_SIZE, val);
 
 	/* If the sizes don't match, issue a warning */
 	if (ipa->mem_offset + mem_size < ipa->mem_size) {
diff --git a/drivers/net/ipa/ipa_modem.c b/drivers/net/ipa/ipa_modem.c
index c8b1c4d9c507..423422a2a445 100644
--- a/drivers/net/ipa/ipa_modem.c
+++ b/drivers/net/ipa/ipa_modem.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2014-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2021 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 
 #include <linux/errno.h>
diff --git a/drivers/net/ipa/ipa_modem.h b/drivers/net/ipa/ipa_modem.h
index e64ccc2402e9..d85718db9a57 100644
--- a/drivers/net/ipa/ipa_modem.h
+++ b/drivers/net/ipa/ipa_modem.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #ifndef _IPA_MODEM_H_
 #define _IPA_MODEM_H_
diff --git a/drivers/net/ipa/ipa_power.c b/drivers/net/ipa/ipa_power.c
index db5ac7552286..8420f93128a2 100644
--- a/drivers/net/ipa/ipa_power.c
+++ b/drivers/net/ipa/ipa_power.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2021 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 
 #include <linux/clk.h>
diff --git a/drivers/net/ipa/ipa_power.h b/drivers/net/ipa/ipa_power.h
index 6f84f057a209..896f052e51a1 100644
--- a/drivers/net/ipa/ipa_power.h
+++ b/drivers/net/ipa/ipa_power.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #ifndef _IPA_POWER_H_
 #define _IPA_POWER_H_
diff --git a/drivers/net/ipa/ipa_qmi.c b/drivers/net/ipa/ipa_qmi.c
index 6f874f99b910..8295fd4b70d1 100644
--- a/drivers/net/ipa/ipa_qmi.c
+++ b/drivers/net/ipa/ipa_qmi.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2013-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
diff --git a/drivers/net/ipa/ipa_qmi.h b/drivers/net/ipa/ipa_qmi.h
index 856ef629ccc8..1c236826c17a 100644
--- a/drivers/net/ipa/ipa_qmi.h
+++ b/drivers/net/ipa/ipa_qmi.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #ifndef _IPA_QMI_H_
 #define _IPA_QMI_H_
diff --git a/drivers/net/ipa/ipa_qmi_msg.c b/drivers/net/ipa/ipa_qmi_msg.c
index 75d3fc0092e9..97c0befe8d86 100644
--- a/drivers/net/ipa/ipa_qmi_msg.c
+++ b/drivers/net/ipa/ipa_qmi_msg.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #include <linux/stddef.h>
 #include <linux/soc/qcom/qmi.h>
diff --git a/drivers/net/ipa/ipa_qmi_msg.h b/drivers/net/ipa/ipa_qmi_msg.h
index 9651aa59b596..e29663965f43 100644
--- a/drivers/net/ipa/ipa_qmi_msg.h
+++ b/drivers/net/ipa/ipa_qmi_msg.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #ifndef _IPA_QMI_MSG_H_
 #define _IPA_QMI_MSG_H_
diff --git a/drivers/net/ipa/ipa_reg.c b/drivers/net/ipa/ipa_reg.c
index e6147a1cd787..22f067741d9b 100644
--- a/drivers/net/ipa/ipa_reg.c
+++ b/drivers/net/ipa/ipa_reg.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2020 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 
 #include <linux/io.h>
@@ -9,11 +9,105 @@
 #include "ipa.h"
 #include "ipa_reg.h"
 
+/* Is this register valid and defined for the current IPA version? */
+static bool ipa_reg_valid(struct ipa *ipa, enum ipa_reg_id reg_id)
+{
+	enum ipa_version version = ipa->version;
+	bool valid;
+
+	/* Check for bogus (out of range) register IDs */
+	if ((u32)reg_id >= ipa->regs->reg_count)
+		return false;
+
+	switch (reg_id) {
+	case IPA_BCR:
+	case COUNTER_CFG:
+		valid = version < IPA_VERSION_4_5;
+		break;
+
+	case IPA_TX_CFG:
+	case FLAVOR_0:
+	case IDLE_INDICATION_CFG:
+		valid = version >= IPA_VERSION_3_5;
+		break;
+
+	case QTIME_TIMESTAMP_CFG:
+	case TIMERS_XO_CLK_DIV_CFG:
+	case TIMERS_PULSE_GRAN_CFG:
+		valid = version >= IPA_VERSION_4_5;
+		break;
+
+	case SRC_RSRC_GRP_45_RSRC_TYPE:
+	case DST_RSRC_GRP_45_RSRC_TYPE:
+		valid = version <= IPA_VERSION_3_1 ||
+			version == IPA_VERSION_4_5;
+		break;
+
+	case SRC_RSRC_GRP_67_RSRC_TYPE:
+	case DST_RSRC_GRP_67_RSRC_TYPE:
+		valid = version <= IPA_VERSION_3_1;
+		break;
+
+	case ENDP_FILTER_ROUTER_HSH_CFG:
+		valid = version != IPA_VERSION_4_2;
+		break;
+
+	case IRQ_SUSPEND_EN:
+	case IRQ_SUSPEND_CLR:
+		valid = version >= IPA_VERSION_3_1;
+		break;
+
+	default:
+		valid = true;	/* Others should be defined for all versions */
+		break;
+	}
+
+	/* To be valid, it must be defined */
+
+	return valid && ipa->regs->reg[reg_id];
+}
+
+const struct ipa_reg *ipa_reg(struct ipa *ipa, enum ipa_reg_id reg_id)
+{
+	if (WARN_ON(!ipa_reg_valid(ipa, reg_id)))
+		return NULL;
+
+	return ipa->regs->reg[reg_id];
+}
+
+static const struct ipa_regs *ipa_regs(enum ipa_version version)
+{
+	switch (version) {
+	case IPA_VERSION_3_1:
+		return &ipa_regs_v3_1;
+	case IPA_VERSION_3_5_1:
+		return &ipa_regs_v3_5_1;
+	case IPA_VERSION_4_2:
+		return &ipa_regs_v4_2;
+	case IPA_VERSION_4_5:
+		return &ipa_regs_v4_5;
+	case IPA_VERSION_4_9:
+		return &ipa_regs_v4_9;
+	case IPA_VERSION_4_11:
+		return &ipa_regs_v4_11;
+	default:
+		return NULL;
+	}
+}
+
 int ipa_reg_init(struct ipa *ipa)
 {
 	struct device *dev = &ipa->pdev->dev;
+	const struct ipa_regs *regs;
 	struct resource *res;
 
+	regs = ipa_regs(ipa->version);
+	if (!regs)
+		return -EINVAL;
+
+	if (WARN_ON(regs->reg_count > IPA_REG_ID_COUNT))
+		return -EINVAL;
+
 	/* Setup IPA register memory  */
 	res = platform_get_resource_byname(ipa->pdev, IORESOURCE_MEM,
 					   "ipa-reg");
@@ -28,6 +122,7 @@ int ipa_reg_init(struct ipa *ipa)
 		return -ENOMEM;
 	}
 	ipa->reg_addr = res->start;
+	ipa->regs = regs;
 
 	return 0;
 }
diff --git a/drivers/net/ipa/ipa_reg.h b/drivers/net/ipa/ipa_reg.h
index 6f35438cda89..7bf70f70f63f 100644
--- a/drivers/net/ipa/ipa_reg.h
+++ b/drivers/net/ipa/ipa_reg.h
@@ -1,12 +1,13 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2021 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 #ifndef _IPA_REG_H_
 #define _IPA_REG_H_
 
 #include <linux/bitfield.h>
+#include <linux/bug.h>
 
 #include "ipa_version.h"
 
@@ -16,304 +17,325 @@ struct ipa;
  * DOC: IPA Registers
  *
  * IPA registers are located within the "ipa-reg" address space defined by
- * Device Tree.  The offset of each register within that space is specified
- * by symbols defined below.  The address space is mapped to virtual memory
- * space in ipa_mem_init().  All IPA registers are 32 bits wide.
+ * Device Tree.  Each register has a specified offset within that space,
+ * which is mapped into virtual memory space in ipa_mem_init().  Each
+ * has a unique identifer, taken from the ipa_reg_id enumerated type.
+ * All IPA registers are 32 bits wide.
  *
- * Certain register types are duplicated for a number of instances of
- * something.  For example, each IPA endpoint has an set of registers
- * defining its configuration.  The offset to an endpoint's set of registers
- * is computed based on an "base" offset, plus an endpoint's ID multiplied
- * and a "stride" value for the register.  For such registers, the offset is
- * computed by a function-like macro that takes a parameter used in the
- * computation.
+ * Certain "parameterized" register types are duplicated for a number of
+ * instances of something.  For example, each IPA endpoint has an set of
+ * registers defining its configuration.  The offset to an endpoint's set
+ * of registers is computed based on an "base" offset, plus an endpoint's
+ * ID multiplied and a "stride" value for the register.  Similarly, some
+ * registers have an offset that depends on execution environment.  In
+ * this case, the stride is multiplied by a member of the gsi_ee_id
+ * enumerated type.
  *
- * Some register offsets depend on execution environment.  For these an "ee"
- * parameter is supplied to the offset macro.  The "ee" value is a member of
- * the gsi_ee enumerated type.
+ * Each version of IPA implements an array of ipa_reg structures indexed
+ * by register ID.  Each entry in the array specifies the base offset and
+ * (for parameterized registers) a non-zero stride value.  Not all versions
+ * of IPA define all registers.  The offset for a register is returned by
+ * ipa_reg_offset() when the register's ipa_reg structure is supplied;
+ * zero is returned for an undefined register (this should never happen).
  *
- * The offset of a register dependent on endpoint ID is computed by a macro
- * that is supplied a parameter "ep", "txep", or "rxep".  A register with an
- * "ep" parameter is valid for any endpoint; a register with a "txep" or
- * "rxep" parameter is valid only for TX or RX endpoints, respectively.  The
- * "*ep" value is assumed to be less than the maximum valid endpoint ID
- * for the current hardware, and that will not exceed IPA_ENDPOINT_MAX.
- *
- * The offset of registers related to filter and route tables is computed
- * by a macro that is supplied a parameter "er".  The "er" represents an
- * endpoint ID for filters, or a route ID for routes.  For filters, the
- * endpoint ID must be less than IPA_ENDPOINT_MAX, but is further restricted
- * because not all endpoints support filtering.  For routes, the route ID
- * must be less than IPA_ROUTE_MAX.
- *
- * The offset of registers related to resource types is computed by a macro
- * that is supplied a parameter "rt".  The "rt" represents a resource type,
- * which is a member of the ipa_resource_type_src enumerated type for
- * source endpoint resources or the ipa_resource_type_dst enumerated type
- * for destination endpoint resources.
- *
- * Some registers encode multiple fields within them.  For these, each field
- * has a symbol below defining a field mask that encodes both the position
- * and width of the field within its register.
- *
- * In some cases, different versions of IPA hardware use different offset or
- * field mask values.  In such cases an inline_function(ipa) is used rather
- * than a MACRO to define the offset or field mask to use.
- *
- * Finally, some registers hold bitmasks representing endpoints.  In such
- * cases the @available field in the @ipa structure defines the "full" set
- * of valid bits for the register.
+ * Some registers encode multiple fields within them.  Each field in
+ * such a register has a unique identifier (from an enumerated type).
+ * The position and width of the fields in a register are defined by
+ * an array of field masks, indexed by field ID.  Two functions are
+ * used to access register fields; both take an ipa_reg structure as
+ * argument.  To encode a value to be represented in a register field,
+ * the value and field ID are passed to ipa_reg_encode().  To extract
+ * a value encoded in a register field, the field ID is passed to
+ * ipa_reg_decode().  In addition, for single-bit fields, ipa_reg_bit()
+ * can be used to either encode the bit value, or to generate a mask
+ * used to extract the bit value.
  */
 
-#define IPA_REG_COMP_CFG_OFFSET				0x0000003c
-/* The next field is not supported for IPA v4.0+, not present for IPA v4.5+ */
-#define ENABLE_FMASK				GENMASK(0, 0)
-/* The next field is present for IPA v4.7+ */
-#define RAM_ARB_PRI_CLIENT_SAMP_FIX_DIS_FMASK	GENMASK(0, 0)
-#define GSI_SNOC_BYPASS_DIS_FMASK		GENMASK(1, 1)
-#define GEN_QMB_0_SNOC_BYPASS_DIS_FMASK		GENMASK(2, 2)
-#define GEN_QMB_1_SNOC_BYPASS_DIS_FMASK		GENMASK(3, 3)
-/* The next field is not present for IPA v4.5+ */
-#define IPA_DCMP_FAST_CLK_EN_FMASK		GENMASK(4, 4)
-/* The next twelve fields are present for IPA v4.0+ */
-#define IPA_QMB_SELECT_CONS_EN_FMASK		GENMASK(5, 5)
-#define IPA_QMB_SELECT_PROD_EN_FMASK		GENMASK(6, 6)
-#define GSI_MULTI_INORDER_RD_DIS_FMASK		GENMASK(7, 7)
-#define GSI_MULTI_INORDER_WR_DIS_FMASK		GENMASK(8, 8)
-#define GEN_QMB_0_MULTI_INORDER_RD_DIS_FMASK	GENMASK(9, 9)
-#define GEN_QMB_1_MULTI_INORDER_RD_DIS_FMASK	GENMASK(10, 10)
-#define GEN_QMB_0_MULTI_INORDER_WR_DIS_FMASK	GENMASK(11, 11)
-#define GEN_QMB_1_MULTI_INORDER_WR_DIS_FMASK	GENMASK(12, 12)
-#define GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS_FMASK	GENMASK(13, 13)
-#define GSI_SNOC_CNOC_LOOP_PROT_DISABLE_FMASK	GENMASK(14, 14)
-#define GSI_MULTI_AXI_MASTERS_DIS_FMASK		GENMASK(15, 15)
-#define IPA_QMB_SELECT_GLOBAL_EN_FMASK		GENMASK(16, 16)
-/* The next five fields are present for IPA v4.9+ */
-#define QMB_RAM_RD_CACHE_DISABLE_FMASK		GENMASK(19, 19)
-#define GENQMB_AOOOWR_FMASK			GENMASK(20, 20)
-#define IF_OUT_OF_BUF_STOP_RESET_MASK_EN_FMASK	GENMASK(21, 21)
-#define GEN_QMB_1_DYNAMIC_ASIZE_FMASK		GENMASK(30, 30)
-#define GEN_QMB_0_DYNAMIC_ASIZE_FMASK		GENMASK(31, 31)
-
-/* Encoded value for COMP_CFG register ATOMIC_FETCHER_ARB_LOCK_DIS field */
-static inline u32 arbitration_lock_disable_encoded(enum ipa_version version,
-						   u32 mask)
-{
-	WARN_ON(version < IPA_VERSION_4_0);
+/* enum ipa_reg_id - IPA register IDs */
+enum ipa_reg_id {
+	COMP_CFG,
+	CLKON_CFG,
+	ROUTE,
+	SHARED_MEM_SIZE,
+	QSB_MAX_WRITES,
+	QSB_MAX_READS,
+	FILT_ROUT_HASH_EN,
+	FILT_ROUT_HASH_FLUSH,
+	STATE_AGGR_ACTIVE,
+	IPA_BCR,					/* Not IPA v4.5+ */
+	LOCAL_PKT_PROC_CNTXT,
+	AGGR_FORCE_CLOSE,
+	COUNTER_CFG,					/* Not IPA v4.5+ */
+	IPA_TX_CFG,					/* IPA v3.5+ */
+	FLAVOR_0,					/* IPA v3.5+ */
+	IDLE_INDICATION_CFG,				/* IPA v3.5+ */
+	QTIME_TIMESTAMP_CFG,				/* IPA v4.5+ */
+	TIMERS_XO_CLK_DIV_CFG,				/* IPA v4.5+ */
+	TIMERS_PULSE_GRAN_CFG,				/* IPA v4.5+ */
+	SRC_RSRC_GRP_01_RSRC_TYPE,
+	SRC_RSRC_GRP_23_RSRC_TYPE,
+	SRC_RSRC_GRP_45_RSRC_TYPE,		/* Not IPA v3.5+, IPA v4.5 */
+	SRC_RSRC_GRP_67_RSRC_TYPE,			/* Not IPA v3.5+ */
+	DST_RSRC_GRP_01_RSRC_TYPE,
+	DST_RSRC_GRP_23_RSRC_TYPE,
+	DST_RSRC_GRP_45_RSRC_TYPE,		/* Not IPA v3.5+, IPA v4.5 */
+	DST_RSRC_GRP_67_RSRC_TYPE,			/* Not IPA v3.5+ */
+	ENDP_INIT_CTRL,		/* Not IPA v4.2+ for TX, not IPA v4.0+ for RX */
+	ENDP_INIT_CFG,
+	ENDP_INIT_NAT,			/* TX only */
+	ENDP_INIT_HDR,
+	ENDP_INIT_HDR_EXT,
+	ENDP_INIT_HDR_METADATA_MASK,	/* RX only */
+	ENDP_INIT_MODE,			/* TX only */
+	ENDP_INIT_AGGR,
+	ENDP_INIT_HOL_BLOCK_EN,		/* RX only */
+	ENDP_INIT_HOL_BLOCK_TIMER,	/* RX only */
+	ENDP_INIT_DEAGGR,		/* TX only */
+	ENDP_INIT_RSRC_GRP,
+	ENDP_INIT_SEQ,			/* TX only */
+	ENDP_STATUS,
+	ENDP_FILTER_ROUTER_HSH_CFG,			/* Not IPA v4.2 */
+	/* The IRQ registers are only used for GSI_EE_AP */
+	IPA_IRQ_STTS,
+	IPA_IRQ_EN,
+	IPA_IRQ_CLR,
+	IPA_IRQ_UC,
+	IRQ_SUSPEND_INFO,
+	IRQ_SUSPEND_EN,					/* IPA v3.1+ */
+	IRQ_SUSPEND_CLR,				/* IPA v3.1+ */
+	IPA_REG_ID_COUNT,				/* Last; not an ID */
+};
 
-	if (version < IPA_VERSION_4_9)
-		return u32_encode_bits(mask, GENMASK(20, 17));
+/**
+ * struct ipa_reg - An IPA register descriptor
+ * @offset:	Register offset relative to base of the "ipa-reg" memory
+ * @stride:	Distance between two instances, if parameterized
+ * @fcount:	Number of entries in the @fmask array
+ * @fmask:	Array of mask values defining position and width of fields
+ * @name:	Upper-case name of the IPA register
+ */
+struct ipa_reg {
+	u32 offset;
+	u32 stride;
+	u32 fcount;
+	const u32 *fmask;			/* BIT(nr) or GENMASK(h, l) */
+	const char *name;
+};
 
-	if (version == IPA_VERSION_4_9)
-		return u32_encode_bits(mask, GENMASK(24, 22));
+/* Helper macro for defining "simple" (non-parameterized) registers */
+#define IPA_REG(__NAME, __reg_id, __offset)				\
+	IPA_REG_STRIDE(__NAME, __reg_id, __offset, 0)
 
-	return u32_encode_bits(mask, GENMASK(23, 22));
-}
+/* Helper macro for defining parameterized registers, specifying stride */
+#define IPA_REG_STRIDE(__NAME, __reg_id, __offset, __stride)		\
+	static const struct ipa_reg ipa_reg_ ## __reg_id = {		\
+		.name	= #__NAME,					\
+		.offset	= __offset,					\
+		.stride	= __stride,					\
+	}
 
-/* Encoded value for COMP_CFG register FULL_FLUSH_WAIT_RS_CLOSURE_EN field */
-static inline u32 full_flush_rsc_closure_en_encoded(enum ipa_version version,
-						    bool enable)
-{
-	u32 val = enable ? 1 : 0;
+#define IPA_REG_FIELDS(__NAME, __name, __offset)			\
+	IPA_REG_STRIDE_FIELDS(__NAME, __name, __offset, 0)
 
-	WARN_ON(version < IPA_VERSION_4_5);
+#define IPA_REG_STRIDE_FIELDS(__NAME, __name, __offset, __stride)	\
+	static const struct ipa_reg ipa_reg_ ## __name = {		\
+		.name   = #__NAME,					\
+		.offset = __offset,					\
+		.stride = __stride,					\
+		.fcount = ARRAY_SIZE(ipa_reg_ ## __name ## _fmask),	\
+		.fmask  = ipa_reg_ ## __name ## _fmask,			\
+	}
 
-	if (version == IPA_VERSION_4_5 || version == IPA_VERSION_4_7)
-		return u32_encode_bits(val, GENMASK(21, 21));
+/**
+ * struct ipa_regs - Description of registers supported by hardware
+ * @reg_count:	Number of registers in the @reg[] array
+ * @reg:		Array of register descriptors
+ */
+struct ipa_regs {
+	u32 reg_count;
+	const struct ipa_reg **reg;
+};
 
-	return u32_encode_bits(val, GENMASK(17, 17));
-}
+/* COMP_CFG register */
+enum ipa_reg_comp_cfg_field_id {
+	COMP_CFG_ENABLE,				/* Not IPA v4.0+ */
+	RAM_ARB_PRI_CLIENT_SAMP_FIX_DIS,		/* IPA v4.7+ */
+	GSI_SNOC_BYPASS_DIS,
+	GEN_QMB_0_SNOC_BYPASS_DIS,
+	GEN_QMB_1_SNOC_BYPASS_DIS,
+	IPA_DCMP_FAST_CLK_EN,				/* Not IPA v4.5+ */
+	IPA_QMB_SELECT_CONS_EN,				/* IPA v4.0+ */
+	IPA_QMB_SELECT_PROD_EN,				/* IPA v4.0+ */
+	GSI_MULTI_INORDER_RD_DIS,			/* IPA v4.0+ */
+	GSI_MULTI_INORDER_WR_DIS,			/* IPA v4.0+ */
+	GEN_QMB_0_MULTI_INORDER_RD_DIS,			/* IPA v4.0+ */
+	GEN_QMB_1_MULTI_INORDER_RD_DIS,			/* IPA v4.0+ */
+	GEN_QMB_0_MULTI_INORDER_WR_DIS,			/* IPA v4.0+ */
+	GEN_QMB_1_MULTI_INORDER_WR_DIS,			/* IPA v4.0+ */
+	GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS,		/* IPA v4.0+ */
+	GSI_SNOC_CNOC_LOOP_PROT_DISABLE,		/* IPA v4.0+ */
+	GSI_MULTI_AXI_MASTERS_DIS,			/* IPA v4.0+ */
+	IPA_QMB_SELECT_GLOBAL_EN,			/* IPA v4.0+ */
+	QMB_RAM_RD_CACHE_DISABLE,			/* IPA v4.9+ */
+	GENQMB_AOOOWR,					/* IPA v4.9+ */
+	IF_OUT_OF_BUF_STOP_RESET_MASK_EN,		/* IPA v4.9+ */
+	GEN_QMB_1_DYNAMIC_ASIZE,			/* IPA v4.9+ */
+	GEN_QMB_0_DYNAMIC_ASIZE,			/* IPA v4.9+ */
+	ATOMIC_FETCHER_ARB_LOCK_DIS,			/* IPA v4.0+ */
+	FULL_FLUSH_WAIT_RS_CLOSURE_EN,			/* IPA v4.5+ */
+};
 
-#define IPA_REG_CLKON_CFG_OFFSET			0x00000044
-#define RX_FMASK				GENMASK(0, 0)
-#define PROC_FMASK				GENMASK(1, 1)
-#define TX_WRAPPER_FMASK			GENMASK(2, 2)
-#define MISC_FMASK				GENMASK(3, 3)
-#define RAM_ARB_FMASK				GENMASK(4, 4)
-#define FTCH_HPS_FMASK				GENMASK(5, 5)
-#define FTCH_DPS_FMASK				GENMASK(6, 6)
-#define HPS_FMASK				GENMASK(7, 7)
-#define DPS_FMASK				GENMASK(8, 8)
-#define RX_HPS_CMDQS_FMASK			GENMASK(9, 9)
-#define HPS_DPS_CMDQS_FMASK			GENMASK(10, 10)
-#define DPS_TX_CMDQS_FMASK			GENMASK(11, 11)
-#define RSRC_MNGR_FMASK				GENMASK(12, 12)
-#define CTX_HANDLER_FMASK			GENMASK(13, 13)
-#define ACK_MNGR_FMASK				GENMASK(14, 14)
-#define D_DCPH_FMASK				GENMASK(15, 15)
-#define H_DCPH_FMASK				GENMASK(16, 16)
-/* The next field is not present for IPA v4.5+ */
-#define DCMP_FMASK				GENMASK(17, 17)
-/* The next three fields are present for IPA v3.5+ */
-#define NTF_TX_CMDQS_FMASK			GENMASK(18, 18)
-#define TX_0_FMASK				GENMASK(19, 19)
-#define TX_1_FMASK				GENMASK(20, 20)
-/* The next field is present for IPA v3.5.1+ */
-#define FNR_FMASK				GENMASK(21, 21)
-/* The next eight fields are present for IPA v4.0+ */
-#define QSB2AXI_CMDQ_L_FMASK			GENMASK(22, 22)
-#define AGGR_WRAPPER_FMASK			GENMASK(23, 23)
-#define RAM_SLAVEWAY_FMASK			GENMASK(24, 24)
-#define QMB_FMASK				GENMASK(25, 25)
-#define WEIGHT_ARB_FMASK			GENMASK(26, 26)
-#define GSI_IF_FMASK				GENMASK(27, 27)
-#define GLOBAL_FMASK				GENMASK(28, 28)
-#define GLOBAL_2X_CLK_FMASK			GENMASK(29, 29)
-/* The next field is present for IPA v4.5+ */
-#define DPL_FIFO_FMASK				GENMASK(30, 30)
-/* The next field is present for IPA v4.7+ */
-#define DRBIP_FMASK				GENMASK(31, 31)
-
-#define IPA_REG_ROUTE_OFFSET				0x00000048
-#define ROUTE_DIS_FMASK				GENMASK(0, 0)
-#define ROUTE_DEF_PIPE_FMASK			GENMASK(5, 1)
-#define ROUTE_DEF_HDR_TABLE_FMASK		GENMASK(6, 6)
-#define ROUTE_DEF_HDR_OFST_FMASK		GENMASK(16, 7)
-#define ROUTE_FRAG_DEF_PIPE_FMASK		GENMASK(21, 17)
-#define ROUTE_DEF_RETAIN_HDR_FMASK		GENMASK(24, 24)
-
-#define IPA_REG_SHARED_MEM_SIZE_OFFSET			0x00000054
-#define SHARED_MEM_SIZE_FMASK			GENMASK(15, 0)
-#define SHARED_MEM_BADDR_FMASK			GENMASK(31, 16)
-
-#define IPA_REG_QSB_MAX_WRITES_OFFSET			0x00000074
-#define GEN_QMB_0_MAX_WRITES_FMASK		GENMASK(3, 0)
-#define GEN_QMB_1_MAX_WRITES_FMASK		GENMASK(7, 4)
-
-#define IPA_REG_QSB_MAX_READS_OFFSET			0x00000078
-#define GEN_QMB_0_MAX_READS_FMASK		GENMASK(3, 0)
-#define GEN_QMB_1_MAX_READS_FMASK		GENMASK(7, 4)
-/* The next two fields are present for IPA v4.0+ */
-#define GEN_QMB_0_MAX_READS_BEATS_FMASK		GENMASK(23, 16)
-#define GEN_QMB_1_MAX_READS_BEATS_FMASK		GENMASK(31, 24)
-
-static inline u32 ipa_reg_filt_rout_hash_en_offset(enum ipa_version version)
-{
-	if (version < IPA_VERSION_4_0)
-		return 0x000008c;
+/* CLKON_CFG register */
+enum ipa_reg_clkon_cfg_field_id {
+	CLKON_RX,
+	CLKON_PROC,
+	TX_WRAPPER,
+	CLKON_MISC,
+	RAM_ARB,
+	FTCH_HPS,
+	FTCH_DPS,
+	CLKON_HPS,
+	CLKON_DPS,
+	RX_HPS_CMDQS,
+	HPS_DPS_CMDQS,
+	DPS_TX_CMDQS,
+	RSRC_MNGR,
+	CTX_HANDLER,
+	ACK_MNGR,
+	D_DCPH,
+	H_DCPH,
+	CLKON_DCMP,					/* IPA v4.5+ */
+	NTF_TX_CMDQS,					/* IPA v3.5+ */
+	CLKON_TX_0,					/* IPA v3.5+ */
+	CLKON_TX_1,					/* IPA v3.5+ */
+	CLKON_FNR,					/* IPA v3.5.1+ */
+	QSB2AXI_CMDQ_L,					/* IPA v4.0+ */
+	AGGR_WRAPPER,					/* IPA v4.0+ */
+	RAM_SLAVEWAY,					/* IPA v4.0+ */
+	CLKON_QMB,					/* IPA v4.0+ */
+	WEIGHT_ARB,					/* IPA v4.0+ */
+	GSI_IF,						/* IPA v4.0+ */
+	CLKON_GLOBAL,					/* IPA v4.0+ */
+	GLOBAL_2X_CLK,					/* IPA v4.0+ */
+	DPL_FIFO,					/* IPA v4.5+ */
+	DRBIP,						/* IPA v4.7+ */
+};
 
-	return 0x0000148;
-}
+/* ROUTE register */
+enum ipa_reg_route_field_id {
+	ROUTE_DIS,
+	ROUTE_DEF_PIPE,
+	ROUTE_DEF_HDR_TABLE,
+	ROUTE_DEF_HDR_OFST,
+	ROUTE_FRAG_DEF_PIPE,
+	ROUTE_DEF_RETAIN_HDR,
+};
 
-static inline u32 ipa_reg_filt_rout_hash_flush_offset(enum ipa_version version)
-{
-	if (version < IPA_VERSION_4_0)
-		return 0x0000090;
+/* SHARED_MEM_SIZE register */
+enum ipa_reg_shared_mem_size_field_id {
+	MEM_SIZE,
+	MEM_BADDR,
+};
 
-	return 0x000014c;
-}
+/* QSB_MAX_WRITES register */
+enum ipa_reg_qsb_max_writes_field_id {
+	GEN_QMB_0_MAX_WRITES,
+	GEN_QMB_1_MAX_WRITES,
+};
 
-/* The next four fields are used for the hash enable and flush registers */
-#define IPV6_ROUTER_HASH_FMASK			GENMASK(0, 0)
-#define IPV6_FILTER_HASH_FMASK			GENMASK(4, 4)
-#define IPV4_ROUTER_HASH_FMASK			GENMASK(8, 8)
-#define IPV4_FILTER_HASH_FMASK			GENMASK(12, 12)
+/* QSB_MAX_READS register */
+enum ipa_reg_qsb_max_reads_field_id {
+	GEN_QMB_0_MAX_READS,
+	GEN_QMB_1_MAX_READS,
+	GEN_QMB_0_MAX_READS_BEATS,			/* IPA v4.0+ */
+	GEN_QMB_1_MAX_READS_BEATS,			/* IPA v4.0+ */
+};
 
-/* ipa->available defines the valid bits in the STATE_AGGR_ACTIVE register */
-static inline u32 ipa_reg_state_aggr_active_offset(enum ipa_version version)
-{
-	if (version < IPA_VERSION_4_0)
-		return 0x0000010c;
+/* FILT_ROUT_HASH_EN and FILT_ROUT_HASH_FLUSH registers */
+enum ipa_reg_rout_hash_field_id {
+	IPV6_ROUTER_HASH,
+	IPV6_FILTER_HASH,
+	IPV4_ROUTER_HASH,
+	IPV4_FILTER_HASH,
+};
 
-	return 0x000000b4;
-}
+/* BCR register */
+enum ipa_bcr_compat {
+	BCR_CMDQ_L_LACK_ONE_ENTRY		= 0x0,	/* Not IPA v4.2+ */
+	BCR_TX_NOT_USING_BRESP			= 0x1,	/* Not IPA v4.2+ */
+	BCR_TX_SUSPEND_IRQ_ASSERT_ONCE		= 0x2,	/* Not IPA v4.0+ */
+	BCR_SUSPEND_L2_IRQ			= 0x3,	/* Not IPA v4.2+ */
+	BCR_HOLB_DROP_L2_IRQ			= 0x4,	/* Not IPA v4.2+ */
+	BCR_DUAL_TX				= 0x5,	/* IPA v3.5+ */
+	BCR_ENABLE_FILTER_DATA_CACHE		= 0x6,	/* IPA v3.5+ */
+	BCR_NOTIF_PRIORITY_OVER_ZLT		= 0x7,	/* IPA v3.5+ */
+	BCR_FILTER_PREFETCH_EN			= 0x8,	/* IPA v3.5+ */
+	BCR_ROUTER_PREFETCH_EN			= 0x9,	/* IPA v3.5+ */
+};
 
-/* The next register is not present for IPA v4.5+ */
-#define IPA_REG_BCR_OFFSET				0x000001d0
-/* The next two fields are not present for IPA v4.2+ */
-#define BCR_CMDQ_L_LACK_ONE_ENTRY_FMASK		GENMASK(0, 0)
-#define BCR_TX_NOT_USING_BRESP_FMASK		GENMASK(1, 1)
-/* The next field is invalid for IPA v4.0+ */
-#define BCR_TX_SUSPEND_IRQ_ASSERT_ONCE_FMASK	GENMASK(2, 2)
-/* The next two fields are not present for IPA v4.2+ */
-#define BCR_SUSPEND_L2_IRQ_FMASK		GENMASK(3, 3)
-#define BCR_HOLB_DROP_L2_IRQ_FMASK		GENMASK(4, 4)
-/* The next five fields are present for IPA v3.5+ */
-#define BCR_DUAL_TX_FMASK			GENMASK(5, 5)
-#define BCR_ENABLE_FILTER_DATA_CACHE_FMASK	GENMASK(6, 6)
-#define BCR_NOTIF_PRIORITY_OVER_ZLT_FMASK	GENMASK(7, 7)
-#define BCR_FILTER_PREFETCH_EN_FMASK		GENMASK(8, 8)
-#define BCR_ROUTER_PREFETCH_EN_FMASK		GENMASK(9, 9)
-
-/* The value of the next register must be a multiple of 8 (bottom 3 bits 0) */
-#define IPA_REG_LOCAL_PKT_PROC_CNTXT_OFFSET		0x000001e8
-
-/* Encoded value for LOCAL_PKT_PROC_CNTXT register BASE_ADDR field */
-static inline u32 proc_cntxt_base_addr_encoded(enum ipa_version version,
-					       u32 addr)
-{
-	if (version < IPA_VERSION_4_5)
-		return u32_encode_bits(addr, GENMASK(16, 0));
+/* LOCAL_PKT_PROC_CNTXT register */
+enum ipa_reg_local_pkt_proc_cntxt_field_id {
+	IPA_BASE_ADDR,
+};
 
-	return u32_encode_bits(addr, GENMASK(17, 0));
-}
+/* COUNTER_CFG register */
+enum ipa_reg_counter_cfg_field_id {
+	EOT_COAL_GRANULARITY,				/* Not v3.5+ */
+	AGGR_GRANULARITY,
+};
 
-/* ipa->available defines the valid bits in the AGGR_FORCE_CLOSE register */
-#define IPA_REG_AGGR_FORCE_CLOSE_OFFSET			0x000001ec
-
-/* The next register is not present for IPA v4.5+ */
-#define IPA_REG_COUNTER_CFG_OFFSET			0x000001f0
-/* The next field is not present for IPA v3.5+ */
-#define EOT_COAL_GRANULARITY			GENMASK(3, 0)
-#define AGGR_GRANULARITY_FMASK			GENMASK(8, 4)
-
-/* The next register is present for IPA v3.5+ */
-#define IPA_REG_TX_CFG_OFFSET				0x000001fc
-/* The next three fields are not present for IPA v4.0+ */
-#define TX0_PREFETCH_DISABLE_FMASK		GENMASK(0, 0)
-#define TX1_PREFETCH_DISABLE_FMASK		GENMASK(1, 1)
-#define PREFETCH_ALMOST_EMPTY_SIZE_FMASK	GENMASK(4, 2)
-/* The next six fields are present for IPA v4.0+ */
-#define PREFETCH_ALMOST_EMPTY_SIZE_TX0_FMASK	GENMASK(5, 2)
-#define DMAW_SCND_OUTSD_PRED_THRESHOLD_FMASK	GENMASK(9, 6)
-#define DMAW_SCND_OUTSD_PRED_EN_FMASK		GENMASK(10, 10)
-#define DMAW_MAX_BEATS_256_DIS_FMASK		GENMASK(11, 11)
-#define PA_MASK_EN_FMASK			GENMASK(12, 12)
-#define PREFETCH_ALMOST_EMPTY_SIZE_TX1_FMASK	GENMASK(16, 13)
-/* The next field is present for IPA v4.5+ */
-#define DUAL_TX_ENABLE_FMASK			GENMASK(17, 17)
-/* The next field is present for IPA v4.2+, but not IPA v4.5 */
-#define SSPND_PA_NO_START_STATE_FMASK		GENMASK(18, 18)
-/* The next field is present for IPA v4.2 only */
-#define SSPND_PA_NO_BQ_STATE_FMASK		GENMASK(19, 19)
-
-/* The next register is present for IPA v3.5+ */
-#define IPA_REG_FLAVOR_0_OFFSET				0x00000210
-#define IPA_MAX_PIPES_FMASK			GENMASK(3, 0)
-#define IPA_MAX_CONS_PIPES_FMASK		GENMASK(12, 8)
-#define IPA_MAX_PROD_PIPES_FMASK		GENMASK(20, 16)
-#define IPA_PROD_LOWEST_FMASK			GENMASK(27, 24)
-
-/* The next register is present for IPA v3.5+ */
-static inline u32 ipa_reg_idle_indication_cfg_offset(enum ipa_version version)
-{
-	if (version >= IPA_VERSION_4_2)
-		return 0x00000240;
+/* IPA_TX_CFG register */
+enum ipa_reg_ipa_tx_cfg_field_id {
+	TX0_PREFETCH_DISABLE,				/* Not v4.0+ */
+	TX1_PREFETCH_DISABLE,				/* Not v4.0+ */
+	PREFETCH_ALMOST_EMPTY_SIZE,			/* Not v4.0+ */
+	PREFETCH_ALMOST_EMPTY_SIZE_TX0,			/* v4.0+ */
+	DMAW_SCND_OUTSD_PRED_THRESHOLD,			/* v4.0+ */
+	DMAW_SCND_OUTSD_PRED_EN,			/* v4.0+ */
+	DMAW_MAX_BEATS_256_DIS,				/* v4.0+ */
+	PA_MASK_EN,					/* v4.0+ */
+	PREFETCH_ALMOST_EMPTY_SIZE_TX1,			/* v4.0+ */
+	DUAL_TX_ENABLE,					/* v4.5+ */
+	SSPND_PA_NO_START_STATE,			/* v4,2+, not v4.5 */
+	SSPND_PA_NO_BQ_STATE,				/* v4.2 only */
+};
 
-	return 0x00000220;
-}
+/* FLAVOR_0 register */
+enum ipa_reg_flavor_0_field_id {
+	MAX_PIPES,
+	MAX_CONS_PIPES,
+	MAX_PROD_PIPES,
+	PROD_LOWEST,
+};
+
+/* IDLE_INDICATION_CFG register */
+enum ipa_reg_idle_indication_cfg_field_id {
+	ENTER_IDLE_DEBOUNCE_THRESH,
+	CONST_NON_IDLE_ENABLE,
+};
+
+/* QTIME_TIMESTAMP_CFG register */
+enum ipa_reg_qtime_timestamp_cfg_field_id {
+	DPL_TIMESTAMP_LSB,
+	DPL_TIMESTAMP_SEL,
+	TAG_TIMESTAMP_LSB,
+	NAT_TIMESTAMP_LSB,
+};
 
-#define ENTER_IDLE_DEBOUNCE_THRESH_FMASK	GENMASK(15, 0)
-#define CONST_NON_IDLE_ENABLE_FMASK		GENMASK(16, 16)
-
-/* The next register is present for IPA v4.5+ */
-#define IPA_REG_QTIME_TIMESTAMP_CFG_OFFSET		0x0000024c
-#define DPL_TIMESTAMP_LSB_FMASK			GENMASK(4, 0)
-#define DPL_TIMESTAMP_SEL_FMASK			GENMASK(7, 7)
-#define TAG_TIMESTAMP_LSB_FMASK			GENMASK(12, 8)
-#define NAT_TIMESTAMP_LSB_FMASK			GENMASK(20, 16)
-
-/* The next register is present for IPA v4.5+ */
-#define IPA_REG_TIMERS_XO_CLK_DIV_CFG_OFFSET		0x00000250
-#define DIV_VALUE_FMASK				GENMASK(8, 0)
-#define DIV_ENABLE_FMASK			GENMASK(31, 31)
-
-/* The next register is present for IPA v4.5+ */
-#define IPA_REG_TIMERS_PULSE_GRAN_CFG_OFFSET		0x00000254
-#define GRAN_0_FMASK				GENMASK(2, 0)
-#define GRAN_1_FMASK				GENMASK(5, 3)
-#define GRAN_2_FMASK				GENMASK(8, 6)
-/* Values for GRAN_x fields of TIMERS_PULSE_GRAN_CFG */
+/* TIMERS_XO_CLK_DIV_CFG register */
+enum ipa_reg_timers_xo_clk_div_cfg_field_id {
+	DIV_VALUE,
+	DIV_ENABLE,
+};
+
+/* TIMERS_PULSE_GRAN_CFG register */
+enum ipa_reg_timers_pulse_gran_cfg_field_id {
+	PULSE_GRAN_0,
+	PULSE_GRAN_1,
+	PULSE_GRAN_2,
+};
+
+/* Values for IPA_GRAN_x fields of TIMERS_PULSE_GRAN_CFG */
 enum ipa_pulse_gran {
 	IPA_GRAN_10_US				= 0x0,
 	IPA_GRAN_20_US				= 0x1,
@@ -325,267 +347,160 @@ enum ipa_pulse_gran {
 	IPA_GRAN_655350_US			= 0x7,
 };
 
-/* Not all of the following are present (depends on IPA version) */
-#define IPA_REG_SRC_RSRC_GRP_01_RSRC_TYPE_N_OFFSET(rt) \
-					(0x00000400 + 0x0020 * (rt))
-#define IPA_REG_SRC_RSRC_GRP_23_RSRC_TYPE_N_OFFSET(rt) \
-					(0x00000404 + 0x0020 * (rt))
-#define IPA_REG_SRC_RSRC_GRP_45_RSRC_TYPE_N_OFFSET(rt) \
-					(0x00000408 + 0x0020 * (rt))
-#define IPA_REG_SRC_RSRC_GRP_67_RSRC_TYPE_N_OFFSET(rt) \
-					(0x0000040c + 0x0020 * (rt))
-#define IPA_REG_DST_RSRC_GRP_01_RSRC_TYPE_N_OFFSET(rt) \
-					(0x00000500 + 0x0020 * (rt))
-#define IPA_REG_DST_RSRC_GRP_23_RSRC_TYPE_N_OFFSET(rt) \
-					(0x00000504 + 0x0020 * (rt))
-#define IPA_REG_DST_RSRC_GRP_45_RSRC_TYPE_N_OFFSET(rt) \
-					(0x00000508 + 0x0020 * (rt))
-#define IPA_REG_DST_RSRC_GRP_67_RSRC_TYPE_N_OFFSET(rt) \
-					(0x0000050c + 0x0020 * (rt))
-/* The next four fields are used for all resource group registers */
-#define X_MIN_LIM_FMASK				GENMASK(5, 0)
-#define X_MAX_LIM_FMASK				GENMASK(13, 8)
-/* The next two fields are not always present (if resource count is odd) */
-#define Y_MIN_LIM_FMASK				GENMASK(21, 16)
-#define Y_MAX_LIM_FMASK				GENMASK(29, 24)
-
-#define IPA_REG_ENDP_INIT_CTRL_N_OFFSET(ep) \
-					(0x00000800 + 0x0070 * (ep))
-/* Valid only for RX (IPA producer) endpoints (do not use for IPA v4.0+) */
-#define ENDP_SUSPEND_FMASK			GENMASK(0, 0)
-/* Valid only for TX (IPA consumer) endpoints */
-#define ENDP_DELAY_FMASK			GENMASK(1, 1)
-
-#define IPA_REG_ENDP_INIT_CFG_N_OFFSET(ep) \
-					(0x00000808 + 0x0070 * (ep))
-#define FRAG_OFFLOAD_EN_FMASK			GENMASK(0, 0)
-#define CS_OFFLOAD_EN_FMASK			GENMASK(2, 1)
-#define CS_METADATA_HDR_OFFSET_FMASK		GENMASK(6, 3)
-#define CS_GEN_QMB_MASTER_SEL_FMASK		GENMASK(8, 8)
+/* {SRC,DST}_RSRC_GRP_{01,23,45,67}_RSRC_TYPE registers */
+enum ipa_reg_rsrc_grp_rsrc_type_field_id {
+	X_MIN_LIM,
+	X_MAX_LIM,
+	Y_MIN_LIM,
+	Y_MAX_LIM,
+};
+
+/* ENDP_INIT_CTRL register */
+enum ipa_reg_endp_init_ctrl_field_id {
+	ENDP_SUSPEND,					/* Not v4.0+ */
+	ENDP_DELAY,					/* Not v4.2+ */
+};
+
+/* ENDP_INIT_CFG register */
+enum ipa_reg_endp_init_cfg_field_id {
+	FRAG_OFFLOAD_EN,
+	CS_OFFLOAD_EN,
+	CS_METADATA_HDR_OFFSET,
+	CS_GEN_QMB_MASTER_SEL,
+};
 
 /** enum ipa_cs_offload_en - ENDP_INIT_CFG register CS_OFFLOAD_EN field value */
 enum ipa_cs_offload_en {
-	IPA_CS_OFFLOAD_NONE		= 0x0,
-	IPA_CS_OFFLOAD_UL		= 0x1,	/* Before IPA v4.5 (TX) */
-	IPA_CS_OFFLOAD_DL		= 0x2,	/* Before IPA v4.5 (RX) */
-	IPA_CS_OFFLOAD_INLINE		= 0x1,	/* IPA v4.5 (TX and RX) */
+	IPA_CS_OFFLOAD_NONE			= 0x0,
+	IPA_CS_OFFLOAD_UL	/* TX */	= 0x1,	/* Not IPA v4.5+ */
+	IPA_CS_OFFLOAD_DL	/* RX */	= 0x2,	/* Not IPA v4.5+ */
+	IPA_CS_OFFLOAD_INLINE	/* TX and RX */	= 0x1,	/* IPA v4.5+ */
 };
 
-/* Valid only for TX (IPA consumer) endpoints */
-#define IPA_REG_ENDP_INIT_NAT_N_OFFSET(ep) \
-					(0x0000080c + 0x0070 * (ep))
-#define NAT_EN_FMASK				GENMASK(1, 0)
+/* ENDP_INIT_NAT register */
+enum ipa_reg_endp_init_nat_field_id {
+	NAT_EN,
+};
 
 /** enum ipa_nat_en - ENDP_INIT_NAT register NAT_EN field value */
 enum ipa_nat_en {
-	IPA_NAT_BYPASS			= 0x0,
-	IPA_NAT_SRC			= 0x1,
-	IPA_NAT_DST			= 0x2,
-};
-
-#define IPA_REG_ENDP_INIT_HDR_N_OFFSET(ep) \
-					(0x00000810 + 0x0070 * (ep))
-#define HDR_LEN_FMASK				GENMASK(5, 0)
-#define HDR_OFST_METADATA_VALID_FMASK		GENMASK(6, 6)
-#define HDR_OFST_METADATA_FMASK			GENMASK(12, 7)
-#define HDR_ADDITIONAL_CONST_LEN_FMASK		GENMASK(18, 13)
-#define HDR_OFST_PKT_SIZE_VALID_FMASK		GENMASK(19, 19)
-#define HDR_OFST_PKT_SIZE_FMASK			GENMASK(25, 20)
-/* The next field is not present for IPA v4.9+ */
-#define HDR_A5_MUX_FMASK			GENMASK(26, 26)
-#define HDR_LEN_INC_DEAGG_HDR_FMASK		GENMASK(27, 27)
-/* The next field is not present for IPA v4.5+ */
-#define HDR_METADATA_REG_VALID_FMASK		GENMASK(28, 28)
-/* The next two fields are present for IPA v4.5+ */
-#define HDR_LEN_MSB_FMASK			GENMASK(29, 28)
-#define HDR_OFST_METADATA_MSB_FMASK		GENMASK(31, 30)
-
-/* Encoded value for ENDP_INIT_HDR register HDR_LEN* field(s) */
-static inline u32 ipa_header_size_encoded(enum ipa_version version,
-					  u32 header_size)
-{
-	u32 size = header_size & field_mask(HDR_LEN_FMASK);
-	u32 val;
-
-	val = u32_encode_bits(size, HDR_LEN_FMASK);
-	if (version < IPA_VERSION_4_5) {
-		WARN_ON(header_size != size);
-		return val;
-	}
-
-	/* IPA v4.5 adds a few more most-significant bits */
-	size = header_size >> hweight32(HDR_LEN_FMASK);
-	val |= u32_encode_bits(size, HDR_LEN_MSB_FMASK);
-
-	return val;
-}
-
-/* Encoded value for ENDP_INIT_HDR register OFST_METADATA* field(s) */
-static inline u32 ipa_metadata_offset_encoded(enum ipa_version version,
-					      u32 offset)
-{
-	u32 off = offset & field_mask(HDR_OFST_METADATA_FMASK);
-	u32 val;
-
-	val = u32_encode_bits(off, HDR_OFST_METADATA_FMASK);
-	if (version < IPA_VERSION_4_5) {
-		WARN_ON(offset != off);
-		return val;
-	}
+	IPA_NAT_BYPASS				= 0x0,
+	IPA_NAT_SRC				= 0x1,
+	IPA_NAT_DST				= 0x2,
+};
 
-	/* IPA v4.5 adds a few more most-significant bits */
-	off = offset >> hweight32(HDR_OFST_METADATA_FMASK);
-	val |= u32_encode_bits(off, HDR_OFST_METADATA_MSB_FMASK);
+/* ENDP_INIT_HDR register */
+enum ipa_reg_endp_init_hdr_field_id {
+	HDR_LEN,
+	HDR_OFST_METADATA_VALID,
+	HDR_OFST_METADATA,
+	HDR_ADDITIONAL_CONST_LEN,
+	HDR_OFST_PKT_SIZE_VALID,
+	HDR_OFST_PKT_SIZE,
+	HDR_A5_MUX,					/* Not v4.9+ */
+	HDR_LEN_INC_DEAGG_HDR,
+	HDR_METADATA_REG_VALID,				/* Not v4.5+ */
+	HDR_LEN_MSB,					/* v4.5+ */
+	HDR_OFST_METADATA_MSB,				/* v4.5+ */
+};
 
-	return val;
-}
+/* ENDP_INIT_HDR_EXT register */
+enum ipa_reg_endp_init_hdr_ext_field_id {
+	HDR_ENDIANNESS,
+	HDR_TOTAL_LEN_OR_PAD_VALID,
+	HDR_TOTAL_LEN_OR_PAD,
+	HDR_PAYLOAD_LEN_INC_PADDING,
+	HDR_TOTAL_LEN_OR_PAD_OFFSET,
+	HDR_PAD_TO_ALIGNMENT,
+	HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB,		/* v4.5+ */
+	HDR_OFST_PKT_SIZE_MSB,				/* v4.5+ */
+	HDR_ADDITIONAL_CONST_LEN_MSB,			/* v4.5+ */
+};
 
-#define IPA_REG_ENDP_INIT_HDR_EXT_N_OFFSET(ep) \
-					(0x00000814 + 0x0070 * (ep))
-#define HDR_ENDIANNESS_FMASK			GENMASK(0, 0)
-#define HDR_TOTAL_LEN_OR_PAD_VALID_FMASK	GENMASK(1, 1)
-#define HDR_TOTAL_LEN_OR_PAD_FMASK		GENMASK(2, 2)
-#define HDR_PAYLOAD_LEN_INC_PADDING_FMASK	GENMASK(3, 3)
-#define HDR_TOTAL_LEN_OR_PAD_OFFSET_FMASK	GENMASK(9, 4)
-#define HDR_PAD_TO_ALIGNMENT_FMASK		GENMASK(13, 10)
-/* The next three fields are present for IPA v4.5+ */
-#define HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB_FMASK	GENMASK(17, 16)
-#define HDR_OFST_PKT_SIZE_MSB_FMASK		GENMASK(19, 18)
-#define HDR_ADDITIONAL_CONST_LEN_MSB_FMASK	GENMASK(21, 20)
-
-/* Valid only for RX (IPA producer) endpoints */
-#define IPA_REG_ENDP_INIT_HDR_METADATA_MASK_N_OFFSET(rxep) \
-					(0x00000818 + 0x0070 * (rxep))
-
-/* Valid only for TX (IPA consumer) endpoints */
-#define IPA_REG_ENDP_INIT_MODE_N_OFFSET(txep) \
-					(0x00000820 + 0x0070 * (txep))
-#define MODE_FMASK				GENMASK(2, 0)
-/* The next field is present for IPA v4.5+ */
-#define DCPH_ENABLE_FMASK			GENMASK(3, 3)
-#define DEST_PIPE_INDEX_FMASK			GENMASK(8, 4)
-#define BYTE_THRESHOLD_FMASK			GENMASK(27, 12)
-#define PIPE_REPLICATION_EN_FMASK		GENMASK(28, 28)
-#define PAD_EN_FMASK				GENMASK(29, 29)
-/* The next field is not present for IPA v4.5+ */
-#define HDR_FTCH_DISABLE_FMASK			GENMASK(30, 30)
-/* The next field is present for IPA v4.9+ */
-#define DRBIP_ACL_ENABLE			GENMASK(30, 30)
+/* ENDP_INIT_MODE register */
+enum ipa_reg_endp_init_mode_field_id {
+	ENDP_MODE,
+	DCPH_ENABLE,					/* v4.5+ */
+	DEST_PIPE_INDEX,
+	BYTE_THRESHOLD,
+	PIPE_REPLICATION_EN,
+	PAD_EN,
+	HDR_FTCH_DISABLE,				/* v4.5+ */
+	DRBIP_ACL_ENABLE,				/* v4.9+ */
+};
 
 /** enum ipa_mode - ENDP_INIT_MODE register MODE field value */
 enum ipa_mode {
-	IPA_BASIC			= 0x0,
-	IPA_ENABLE_FRAMING_HDLC		= 0x1,
-	IPA_ENABLE_DEFRAMING_HDLC	= 0x2,
-	IPA_DMA				= 0x3,
+	IPA_BASIC				= 0x0,
+	IPA_ENABLE_FRAMING_HDLC			= 0x1,
+	IPA_ENABLE_DEFRAMING_HDLC		= 0x2,
+	IPA_DMA					= 0x3,
 };
 
-#define IPA_REG_ENDP_INIT_AGGR_N_OFFSET(ep) \
-					(0x00000824 +  0x0070 * (ep))
-#define AGGR_EN_FMASK				GENMASK(1, 0)
-#define AGGR_TYPE_FMASK				GENMASK(4, 2)
-
-/* The legacy value is used for IPA hardware before IPA v4.5 */
-static inline u32 aggr_byte_limit_fmask(bool legacy)
-{
-	return legacy ? GENMASK(9, 5) : GENMASK(10, 5);
-}
-
-/* The legacy value is used for IPA hardware before IPA v4.5 */
-static inline u32 aggr_time_limit_fmask(bool legacy)
-{
-	return legacy ? GENMASK(14, 10) : GENMASK(16, 12);
-}
-
-/* The legacy value is used for IPA hardware before IPA v4.5 */
-static inline u32 aggr_pkt_limit_fmask(bool legacy)
-{
-	return legacy ? GENMASK(20, 15) : GENMASK(22, 17);
-}
-
-/* The legacy value is used for IPA hardware before IPA v4.5 */
-static inline u32 aggr_sw_eof_active_fmask(bool legacy)
-{
-	return legacy ? GENMASK(21, 21) : GENMASK(23, 23);
-}
-
-/* The legacy value is used for IPA hardware before IPA v4.5 */
-static inline u32 aggr_force_close_fmask(bool legacy)
-{
-	return legacy ? GENMASK(22, 22) : GENMASK(24, 24);
-}
-
-/* The legacy value is used for IPA hardware before IPA v4.5 */
-static inline u32 aggr_hard_byte_limit_enable_fmask(bool legacy)
-{
-	return legacy ? GENMASK(24, 24) : GENMASK(26, 26);
-}
-
-/* The next field is present for IPA v4.5+ */
-#define AGGR_GRAN_SEL_FMASK			GENMASK(27, 27)
+/* ENDP_INIT_AGGR register */
+enum ipa_reg_endp_init_aggr_field_id {
+	AGGR_EN,
+	AGGR_TYPE,
+	BYTE_LIMIT,
+	TIME_LIMIT,
+	PKT_LIMIT,
+	SW_EOF_ACTIVE,
+	FORCE_CLOSE,
+	HARD_BYTE_LIMIT_EN,
+	AGGR_GRAN_SEL,
+};
 
 /** enum ipa_aggr_en - ENDP_INIT_AGGR register AGGR_EN field value */
 enum ipa_aggr_en {
-	IPA_BYPASS_AGGR			= 0x0,	/* (TX, RX) */
-	IPA_ENABLE_AGGR			= 0x1,	/* (RX) */
-	IPA_ENABLE_DEAGGR		= 0x2,	/* (TX) */
+	IPA_BYPASS_AGGR		/* TX and RX */	= 0x0,
+	IPA_ENABLE_AGGR		/* RX */	= 0x1,
+	IPA_ENABLE_DEAGGR	/* TX */	= 0x2,
 };
 
 /** enum ipa_aggr_type - ENDP_INIT_AGGR register AGGR_TYPE field value */
 enum ipa_aggr_type {
-	IPA_MBIM_16			= 0x0,
-	IPA_HDLC			= 0x1,
-	IPA_TLP				= 0x2,
-	IPA_RNDIS			= 0x3,
-	IPA_GENERIC			= 0x4,
-	IPA_COALESCE			= 0x5,
-	IPA_QCMAP			= 0x6,
-};
-
-/* Valid only for RX (IPA producer) endpoints */
-#define IPA_REG_ENDP_INIT_HOL_BLOCK_EN_N_OFFSET(rxep) \
-					(0x0000082c +  0x0070 * (rxep))
-#define HOL_BLOCK_EN_FMASK			GENMASK(0, 0)
-
-/* Valid only for RX (IPA producer) endpoints */
-#define IPA_REG_ENDP_INIT_HOL_BLOCK_TIMER_N_OFFSET(rxep) \
-					(0x00000830 +  0x0070 * (rxep))
-/* The next two fields are present for IPA v4.2 only */
-#define BASE_VALUE_FMASK			GENMASK(4, 0)
-#define SCALE_FMASK				GENMASK(12, 8)
-/* The next two fields are present for IPA v4.5 */
-#define TIME_LIMIT_FMASK			GENMASK(4, 0)
-#define GRAN_SEL_FMASK				GENMASK(8, 8)
-
-/* Valid only for TX (IPA consumer) endpoints */
-#define IPA_REG_ENDP_INIT_DEAGGR_N_OFFSET(txep) \
-					(0x00000834 + 0x0070 * (txep))
-#define DEAGGR_HDR_LEN_FMASK			GENMASK(5, 0)
-#define SYSPIPE_ERR_DETECTION_FMASK		GENMASK(6, 6)
-#define PACKET_OFFSET_VALID_FMASK		GENMASK(7, 7)
-#define PACKET_OFFSET_LOCATION_FMASK		GENMASK(13, 8)
-#define IGNORE_MIN_PKT_ERR_FMASK		GENMASK(14, 14)
-#define MAX_PACKET_LEN_FMASK			GENMASK(31, 16)
-
-#define IPA_REG_ENDP_INIT_RSRC_GRP_N_OFFSET(ep) \
-					(0x00000838 + 0x0070 * (ep))
-/* Encoded value for ENDP_INIT_RSRC_GRP register RSRC_GRP field */
-static inline u32 rsrc_grp_encoded(enum ipa_version version, u32 rsrc_grp)
-{
-	if (version < IPA_VERSION_3_5 || version == IPA_VERSION_4_5)
-		return u32_encode_bits(rsrc_grp, GENMASK(2, 0));
+	IPA_MBIM_16				= 0x0,
+	IPA_HDLC				= 0x1,
+	IPA_TLP					= 0x2,
+	IPA_RNDIS				= 0x3,
+	IPA_GENERIC				= 0x4,
+	IPA_COALESCE				= 0x5,
+	IPA_QCMAP				= 0x6,
+};
 
-	if (version == IPA_VERSION_4_2 || version == IPA_VERSION_4_7)
-		return u32_encode_bits(rsrc_grp, GENMASK(0, 0));
+/* ENDP_INIT_HOL_BLOCK_EN register */
+enum ipa_reg_endp_init_hol_block_en_field_id {
+	HOL_BLOCK_EN,
+};
 
-	return u32_encode_bits(rsrc_grp, GENMASK(1, 0));
-}
+/* ENDP_INIT_HOL_BLOCK_TIMER register */
+enum ipa_reg_endp_init_hol_block_timer_field_id {
+	TIMER_BASE_VALUE,				/* Not v4.5+ */
+	TIMER_SCALE,					/* v4.2 only */
+	TIMER_LIMIT,					/* v4.5+ */
+	TIMER_GRAN_SEL,					/* v4.5+ */
+};
 
-/* Valid only for TX (IPA consumer) endpoints */
-#define IPA_REG_ENDP_INIT_SEQ_N_OFFSET(txep) \
-					(0x0000083c + 0x0070 * (txep))
-#define SEQ_TYPE_FMASK				GENMASK(7, 0)
-#define SEQ_REP_TYPE_FMASK			GENMASK(15, 8)
+/* ENDP_INIT_DEAGGR register */
+enum ipa_reg_endp_deaggr_field_id {
+	DEAGGR_HDR_LEN,
+	SYSPIPE_ERR_DETECTION,
+	PACKET_OFFSET_VALID,
+	PACKET_OFFSET_LOCATION,
+	IGNORE_MIN_PKT_ERR,
+	MAX_PACKET_LEN,
+};
+
+/* ENDP_INIT_RSRC_GRP register */
+enum ipa_reg_endp_init_rsrc_grp_field_id {
+	ENDP_RSRC_GRP,
+};
+
+/* ENDP_INIT_SEQ register */
+enum ipa_reg_endp_init_seq_field_id {
+	SEQ_TYPE,
+	SEQ_REP_TYPE,					/* Not v4.5+ */
+};
 
 /**
  * enum ipa_seq_type - HPS and DPS sequencer type
@@ -629,76 +544,36 @@ enum ipa_seq_rep_type {
 	IPA_SEQ_REP_DMA_PARSER			= 0x08,
 };
 
-#define IPA_REG_ENDP_STATUS_N_OFFSET(ep) \
-					(0x00000840 + 0x0070 * (ep))
-#define STATUS_EN_FMASK				GENMASK(0, 0)
-#define STATUS_ENDP_FMASK			GENMASK(5, 1)
-/* The next field is not present for IPA v4.5+ */
-#define STATUS_LOCATION_FMASK			GENMASK(8, 8)
-/* The next field is present for IPA v4.0+ */
-#define STATUS_PKT_SUPPRESS_FMASK		GENMASK(9, 9)
-
-/* The next register is not present for IPA v4.2 (which no hashing support) */
-#define IPA_REG_ENDP_FILTER_ROUTER_HSH_CFG_N_OFFSET(er) \
-					(0x0000085c + 0x0070 * (er))
-#define FILTER_HASH_MSK_SRC_ID_FMASK		GENMASK(0, 0)
-#define FILTER_HASH_MSK_SRC_IP_FMASK		GENMASK(1, 1)
-#define FILTER_HASH_MSK_DST_IP_FMASK		GENMASK(2, 2)
-#define FILTER_HASH_MSK_SRC_PORT_FMASK		GENMASK(3, 3)
-#define FILTER_HASH_MSK_DST_PORT_FMASK		GENMASK(4, 4)
-#define FILTER_HASH_MSK_PROTOCOL_FMASK		GENMASK(5, 5)
-#define FILTER_HASH_MSK_METADATA_FMASK		GENMASK(6, 6)
-#define IPA_REG_ENDP_FILTER_HASH_MSK_ALL	GENMASK(6, 0)
-
-#define ROUTER_HASH_MSK_SRC_ID_FMASK		GENMASK(16, 16)
-#define ROUTER_HASH_MSK_SRC_IP_FMASK		GENMASK(17, 17)
-#define ROUTER_HASH_MSK_DST_IP_FMASK		GENMASK(18, 18)
-#define ROUTER_HASH_MSK_SRC_PORT_FMASK		GENMASK(19, 19)
-#define ROUTER_HASH_MSK_DST_PORT_FMASK		GENMASK(20, 20)
-#define ROUTER_HASH_MSK_PROTOCOL_FMASK		GENMASK(21, 21)
-#define ROUTER_HASH_MSK_METADATA_FMASK		GENMASK(22, 22)
-#define IPA_REG_ENDP_ROUTER_HASH_MSK_ALL	GENMASK(22, 16)
-
-static inline u32 ipa_reg_irq_stts_ee_n_offset(enum ipa_version version,
-					       u32 ee)
-{
-	if (version < IPA_VERSION_4_9)
-		return 0x00003008 + 0x1000 * ee;
-
-	return 0x00004008 + 0x1000 * ee;
-}
-
-static inline u32 ipa_reg_irq_stts_offset(enum ipa_version version)
-{
-	return ipa_reg_irq_stts_ee_n_offset(version, GSI_EE_AP);
-}
-
-static inline u32 ipa_reg_irq_en_ee_n_offset(enum ipa_version version, u32 ee)
-{
-	if (version < IPA_VERSION_4_9)
-		return 0x0000300c + 0x1000 * ee;
-
-	return 0x0000400c + 0x1000 * ee;
-}
-
-static inline u32 ipa_reg_irq_en_offset(enum ipa_version version)
-{
-	return ipa_reg_irq_en_ee_n_offset(version, GSI_EE_AP);
-}
-
-static inline u32 ipa_reg_irq_clr_ee_n_offset(enum ipa_version version, u32 ee)
-{
-	if (version < IPA_VERSION_4_9)
-		return 0x00003010 + 0x1000 * ee;
-
-	return 0x00004010 + 0x1000 * ee;
-}
+/* ENDP_STATUS register */
+enum ipa_reg_endp_status_field_id {
+	STATUS_EN,
+	STATUS_ENDP,
+	STATUS_LOCATION,				/* Not v4.5+ */
+	STATUS_PKT_SUPPRESS,				/* v4.0+ */
+};
 
-static inline u32 ipa_reg_irq_clr_offset(enum ipa_version version)
-{
-	return ipa_reg_irq_clr_ee_n_offset(version, GSI_EE_AP);
-}
+/* ENDP_FILTER_ROUTER_HSH_CFG register */
+enum ipa_reg_endp_filter_router_hsh_cfg_field_id {
+	FILTER_HASH_MSK_SRC_ID,
+	FILTER_HASH_MSK_SRC_IP,
+	FILTER_HASH_MSK_DST_IP,
+	FILTER_HASH_MSK_SRC_PORT,
+	FILTER_HASH_MSK_DST_PORT,
+	FILTER_HASH_MSK_PROTOCOL,
+	FILTER_HASH_MSK_METADATA,
+	FILTER_HASH_MSK_ALL,		/* Bitwise OR of the above 6 fields */
+
+	ROUTER_HASH_MSK_SRC_ID,
+	ROUTER_HASH_MSK_SRC_IP,
+	ROUTER_HASH_MSK_DST_IP,
+	ROUTER_HASH_MSK_SRC_PORT,
+	ROUTER_HASH_MSK_DST_PORT,
+	ROUTER_HASH_MSK_PROTOCOL,
+	ROUTER_HASH_MSK_METADATA,
+	ROUTER_HASH_MSK_ALL,		/* Bitwise OR of the above 6 fields */
+};
 
+/* IPA_IRQ_STTS, IPA_IRQ_EN, and IPA_IRQ_CLR registers */
 /**
  * enum ipa_irq_id - Bit positions representing type of IPA IRQ
  * @IPA_IRQ_UC_0:	Microcontroller event interrupt
@@ -774,74 +649,82 @@ enum ipa_irq_id {
 	IPA_IRQ_COUNT,				/* Last; not an id */
 };
 
-static inline u32 ipa_reg_irq_uc_ee_n_offset(enum ipa_version version, u32 ee)
-{
-	if (version < IPA_VERSION_4_9)
-		return 0x0000301c + 0x1000 * ee;
+/* IPA_IRQ_UC register */
+enum ipa_reg_ipa_irq_uc_field_id {
+	UC_INTR,
+};
 
-	return 0x0000401c + 0x1000 * ee;
-}
+extern const struct ipa_regs ipa_regs_v3_1;
+extern const struct ipa_regs ipa_regs_v3_5_1;
+extern const struct ipa_regs ipa_regs_v4_2;
+extern const struct ipa_regs ipa_regs_v4_5;
+extern const struct ipa_regs ipa_regs_v4_9;
+extern const struct ipa_regs ipa_regs_v4_11;
 
-static inline u32 ipa_reg_irq_uc_offset(enum ipa_version version)
+/* Return the field mask for a field in a register */
+static inline u32 ipa_reg_fmask(const struct ipa_reg *reg, u32 field_id)
 {
-	return ipa_reg_irq_uc_ee_n_offset(version, GSI_EE_AP);
-}
+	if (!reg || WARN_ON(field_id >= reg->fcount))
+		return 0;
 
-#define UC_INTR_FMASK				GENMASK(0, 0)
+	return reg->fmask[field_id];
+}
 
-/* ipa->available defines the valid bits in the SUSPEND_INFO register */
-static inline u32
-ipa_reg_irq_suspend_info_ee_n_offset(enum ipa_version version, u32 ee)
+/* Return the mask for a single-bit field in a register */
+static inline u32 ipa_reg_bit(const struct ipa_reg *reg, u32 field_id)
 {
-	if (version == IPA_VERSION_3_0)
-		return 0x00003098 + 0x1000 * ee;
+	u32 fmask = ipa_reg_fmask(reg, field_id);
 
-	if (version < IPA_VERSION_4_9)
-		return 0x00003030 + 0x1000 * ee;
+	WARN_ON(!is_power_of_2(fmask));
 
-	return 0x00004030 + 0x1000 * ee;
+	return fmask;
 }
 
+/* Encode a value into the given field of a register */
 static inline u32
-ipa_reg_irq_suspend_info_offset(enum ipa_version version)
+ipa_reg_encode(const struct ipa_reg *reg, u32 field_id, u32 val)
 {
-	return ipa_reg_irq_suspend_info_ee_n_offset(version, GSI_EE_AP);
-}
+	u32 fmask = ipa_reg_fmask(reg, field_id);
 
-/* ipa->available defines the valid bits in the SUSPEND_EN register */
-static inline u32
-ipa_reg_irq_suspend_en_ee_n_offset(enum ipa_version version, u32 ee)
-{
-	WARN_ON(version == IPA_VERSION_3_0);
+	if (!fmask)
+		return 0;
 
-	if (version < IPA_VERSION_4_9)
-		return 0x00003034 + 0x1000 * ee;
+	val <<= __ffs(fmask);
+	if (WARN_ON(val & ~fmask))
+		return 0;
 
-	return 0x00004034 + 0x1000 * ee;
+	return val;
 }
 
+/* Given a register value, decode (extract) the value in the given field */
 static inline u32
-ipa_reg_irq_suspend_en_offset(enum ipa_version version)
+ipa_reg_decode(const struct ipa_reg *reg, u32 field_id, u32 val)
 {
-	return ipa_reg_irq_suspend_en_ee_n_offset(version, GSI_EE_AP);
+	u32 fmask = ipa_reg_fmask(reg, field_id);
+
+	return fmask ? (val & fmask) >> __ffs(fmask) : 0;
 }
 
-/* ipa->available defines the valid bits in the SUSPEND_CLR register */
-static inline u32
-ipa_reg_irq_suspend_clr_ee_n_offset(enum ipa_version version, u32 ee)
+/* Return the maximum value representable by the given field; always 2^n - 1 */
+static inline u32 ipa_reg_field_max(const struct ipa_reg *reg, u32 field_id)
 {
-	WARN_ON(version == IPA_VERSION_3_0);
+	u32 fmask = ipa_reg_fmask(reg, field_id);
 
-	if (version < IPA_VERSION_4_9)
-		return 0x00003038 + 0x1000 * ee;
+	return fmask ? fmask >> __ffs(fmask) : 0;
+}
+
+const struct ipa_reg *ipa_reg(struct ipa *ipa, enum ipa_reg_id reg_id);
 
-	return 0x00004038 + 0x1000 * ee;
+/* Returns 0 for NULL reg; warning will have already been issued */
+static inline u32 ipa_reg_offset(const struct ipa_reg *reg)
+{
+	return reg ? reg->offset : 0;
 }
 
-static inline u32
-ipa_reg_irq_suspend_clr_offset(enum ipa_version version)
+/* Returns 0 for NULL reg; warning will have already been issued */
+static inline u32 ipa_reg_n_offset(const struct ipa_reg *reg, u32 n)
 {
-	return ipa_reg_irq_suspend_clr_ee_n_offset(version, GSI_EE_AP);
+	return reg ? reg->offset + n * reg->stride : 0;
 }
 
 int ipa_reg_init(struct ipa *ipa);
diff --git a/drivers/net/ipa/ipa_resource.c b/drivers/net/ipa/ipa_resource.c
index 06cec7199382..a257f0e5e361 100644
--- a/drivers/net/ipa/ipa_resource.c
+++ b/drivers/net/ipa/ipa_resource.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2021 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
@@ -69,20 +69,21 @@ static bool ipa_resource_limits_valid(struct ipa *ipa,
 }
 
 static void
-ipa_resource_config_common(struct ipa *ipa, u32 offset,
+ipa_resource_config_common(struct ipa *ipa, u32 resource_type,
+			   const struct ipa_reg *reg,
 			   const struct ipa_resource_limits *xlimits,
 			   const struct ipa_resource_limits *ylimits)
 {
 	u32 val;
 
-	val = u32_encode_bits(xlimits->min, X_MIN_LIM_FMASK);
-	val |= u32_encode_bits(xlimits->max, X_MAX_LIM_FMASK);
+	val = ipa_reg_encode(reg, X_MIN_LIM, xlimits->min);
+	val |= ipa_reg_encode(reg, X_MAX_LIM, xlimits->max);
 	if (ylimits) {
-		val |= u32_encode_bits(ylimits->min, Y_MIN_LIM_FMASK);
-		val |= u32_encode_bits(ylimits->max, Y_MAX_LIM_FMASK);
+		val |= ipa_reg_encode(reg, Y_MIN_LIM, ylimits->min);
+		val |= ipa_reg_encode(reg, Y_MAX_LIM, ylimits->max);
 	}
 
-	iowrite32(val, ipa->reg_virt + offset);
+	iowrite32(val, ipa->reg_virt + ipa_reg_n_offset(reg, resource_type));
 }
 
 static void ipa_resource_config_src(struct ipa *ipa, u32 resource_type,
@@ -91,34 +92,35 @@ static void ipa_resource_config_src(struct ipa *ipa, u32 resource_type,
 	u32 group_count = data->rsrc_group_src_count;
 	const struct ipa_resource_limits *ylimits;
 	const struct ipa_resource *resource;
-	u32 offset;
+	const struct ipa_reg *reg;
 
 	resource = &data->resource_src[resource_type];
 
-	offset = IPA_REG_SRC_RSRC_GRP_01_RSRC_TYPE_N_OFFSET(resource_type);
+	reg = ipa_reg(ipa, SRC_RSRC_GRP_01_RSRC_TYPE);
 	ylimits = group_count == 1 ? NULL : &resource->limits[1];
-	ipa_resource_config_common(ipa, offset, &resource->limits[0], ylimits);
-
+	ipa_resource_config_common(ipa, resource_type, reg,
+				   &resource->limits[0], ylimits);
 	if (group_count < 3)
 		return;
 
-	offset = IPA_REG_SRC_RSRC_GRP_23_RSRC_TYPE_N_OFFSET(resource_type);
+	reg = ipa_reg(ipa, SRC_RSRC_GRP_23_RSRC_TYPE);
 	ylimits = group_count == 3 ? NULL : &resource->limits[3];
-	ipa_resource_config_common(ipa, offset, &resource->limits[2], ylimits);
-
+	ipa_resource_config_common(ipa, resource_type, reg,
+				   &resource->limits[2], ylimits);
 	if (group_count < 5)
 		return;
 
-	offset = IPA_REG_SRC_RSRC_GRP_45_RSRC_TYPE_N_OFFSET(resource_type);
+	reg = ipa_reg(ipa, SRC_RSRC_GRP_45_RSRC_TYPE);
 	ylimits = group_count == 5 ? NULL : &resource->limits[5];
-	ipa_resource_config_common(ipa, offset, &resource->limits[4], ylimits);
-
+	ipa_resource_config_common(ipa, resource_type, reg,
+				   &resource->limits[4], ylimits);
 	if (group_count < 7)
 		return;
 
-	offset = IPA_REG_SRC_RSRC_GRP_67_RSRC_TYPE_N_OFFSET(resource_type);
+	reg = ipa_reg(ipa, SRC_RSRC_GRP_67_RSRC_TYPE);
 	ylimits = group_count == 7 ? NULL : &resource->limits[7];
-	ipa_resource_config_common(ipa, offset, &resource->limits[6], ylimits);
+	ipa_resource_config_common(ipa, resource_type, reg,
+				   &resource->limits[6], ylimits);
 }
 
 static void ipa_resource_config_dst(struct ipa *ipa, u32 resource_type,
@@ -127,34 +129,35 @@ static void ipa_resource_config_dst(struct ipa *ipa, u32 resource_type,
 	u32 group_count = data->rsrc_group_dst_count;
 	const struct ipa_resource_limits *ylimits;
 	const struct ipa_resource *resource;
-	u32 offset;
+	const struct ipa_reg *reg;
 
 	resource = &data->resource_dst[resource_type];
 
-	offset = IPA_REG_DST_RSRC_GRP_01_RSRC_TYPE_N_OFFSET(resource_type);
+	reg = ipa_reg(ipa, DST_RSRC_GRP_01_RSRC_TYPE);
 	ylimits = group_count == 1 ? NULL : &resource->limits[1];
-	ipa_resource_config_common(ipa, offset, &resource->limits[0], ylimits);
-
+	ipa_resource_config_common(ipa, resource_type, reg,
+				   &resource->limits[0], ylimits);
 	if (group_count < 3)
 		return;
 
-	offset = IPA_REG_DST_RSRC_GRP_23_RSRC_TYPE_N_OFFSET(resource_type);
+	reg = ipa_reg(ipa, DST_RSRC_GRP_23_RSRC_TYPE);
 	ylimits = group_count == 3 ? NULL : &resource->limits[3];
-	ipa_resource_config_common(ipa, offset, &resource->limits[2], ylimits);
-
+	ipa_resource_config_common(ipa, resource_type, reg,
+				   &resource->limits[2], ylimits);
 	if (group_count < 5)
 		return;
 
-	offset = IPA_REG_DST_RSRC_GRP_45_RSRC_TYPE_N_OFFSET(resource_type);
+	reg = ipa_reg(ipa, DST_RSRC_GRP_45_RSRC_TYPE);
 	ylimits = group_count == 5 ? NULL : &resource->limits[5];
-	ipa_resource_config_common(ipa, offset, &resource->limits[4], ylimits);
-
+	ipa_resource_config_common(ipa, resource_type, reg,
+				   &resource->limits[4], ylimits);
 	if (group_count < 7)
 		return;
 
-	offset = IPA_REG_DST_RSRC_GRP_67_RSRC_TYPE_N_OFFSET(resource_type);
+	reg = ipa_reg(ipa, DST_RSRC_GRP_67_RSRC_TYPE);
 	ylimits = group_count == 7 ? NULL : &resource->limits[7];
-	ipa_resource_config_common(ipa, offset, &resource->limits[6], ylimits);
+	ipa_resource_config_common(ipa, resource_type, reg,
+				   &resource->limits[6], ylimits);
 }
 
 /* Configure resources; there is no ipa_resource_deconfig() */
diff --git a/drivers/net/ipa/ipa_smp2p.c b/drivers/net/ipa/ipa_smp2p.c
index 211233612039..5620dc271fac 100644
--- a/drivers/net/ipa/ipa_smp2p.c
+++ b/drivers/net/ipa/ipa_smp2p.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2020 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
diff --git a/drivers/net/ipa/ipa_smp2p.h b/drivers/net/ipa/ipa_smp2p.h
index 59cee31a7383..9b969b03d1a4 100644
--- a/drivers/net/ipa/ipa_smp2p.h
+++ b/drivers/net/ipa/ipa_smp2p.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2020 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 #ifndef _IPA_SMP2P_H_
 #define _IPA_SMP2P_H_
diff --git a/drivers/net/ipa/ipa_sysfs.c b/drivers/net/ipa/ipa_sysfs.c
index c0c8641cdd14..5cbc15a971f9 100644
--- a/drivers/net/ipa/ipa_sysfs.c
+++ b/drivers/net/ipa/ipa_sysfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 
-/* Copyright (C) 2021 Linaro Ltd. */
+/* Copyright (C) 2021-2022 Linaro Ltd. */
 
 #include <linux/kernel.h>
 #include <linux/types.h>
diff --git a/drivers/net/ipa/ipa_sysfs.h b/drivers/net/ipa/ipa_sysfs.h
index 4a3ffd1e4e3f..58ba22810bab 100644
--- a/drivers/net/ipa/ipa_sysfs.h
+++ b/drivers/net/ipa/ipa_sysfs.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2021 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 #ifndef _IPA_SYSFS_H_
 #define _IPA_SYSFS_H_
diff --git a/drivers/net/ipa/ipa_table.c b/drivers/net/ipa/ipa_table.c
index 69efe672ca52..510ff2dc8999 100644
--- a/drivers/net/ipa/ipa_table.c
+++ b/drivers/net/ipa/ipa_table.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2021 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
@@ -384,8 +384,9 @@ void ipa_table_reset(struct ipa *ipa, bool modem)
 
 int ipa_table_hash_flush(struct ipa *ipa)
 {
-	u32 offset = ipa_reg_filt_rout_hash_flush_offset(ipa->version);
+	const struct ipa_reg *reg;
 	struct gsi_trans *trans;
+	u32 offset;
 	u32 val;
 
 	if (!ipa_table_hash_support(ipa))
@@ -397,8 +398,13 @@ int ipa_table_hash_flush(struct ipa *ipa)
 		return -EBUSY;
 	}
 
-	val = IPV4_FILTER_HASH_FMASK | IPV6_FILTER_HASH_FMASK;
-	val |= IPV6_ROUTER_HASH_FMASK | IPV4_ROUTER_HASH_FMASK;
+	reg = ipa_reg(ipa, FILT_ROUT_HASH_FLUSH);
+	offset = ipa_reg_offset(reg);
+
+	val = ipa_reg_bit(reg, IPV6_ROUTER_HASH);
+	val |= ipa_reg_bit(reg, IPV6_FILTER_HASH);
+	val |= ipa_reg_bit(reg, IPV4_ROUTER_HASH);
+	val |= ipa_reg_bit(reg, IPV4_FILTER_HASH);
 
 	ipa_cmd_register_write_add(trans, offset, val, val, false);
 
@@ -516,15 +522,18 @@ int ipa_table_setup(struct ipa *ipa)
 static void ipa_filter_tuple_zero(struct ipa_endpoint *endpoint)
 {
 	u32 endpoint_id = endpoint->endpoint_id;
+	struct ipa *ipa = endpoint->ipa;
+	const struct ipa_reg *reg;
 	u32 offset;
 	u32 val;
 
-	offset = IPA_REG_ENDP_FILTER_ROUTER_HSH_CFG_N_OFFSET(endpoint_id);
+	reg = ipa_reg(ipa, ENDP_FILTER_ROUTER_HSH_CFG);
 
+	offset = ipa_reg_n_offset(reg, endpoint_id);
 	val = ioread32(endpoint->ipa->reg_virt + offset);
 
 	/* Zero all filter-related fields, preserving the rest */
-	u32p_replace_bits(&val, 0, IPA_REG_ENDP_FILTER_HASH_MSK_ALL);
+	val &= ~ipa_reg_fmask(reg, FILTER_HASH_MSK_ALL);
 
 	iowrite32(val, endpoint->ipa->reg_virt + offset);
 }
@@ -565,13 +574,17 @@ static bool ipa_route_id_modem(u32 route_id)
  */
 static void ipa_route_tuple_zero(struct ipa *ipa, u32 route_id)
 {
-	u32 offset = IPA_REG_ENDP_FILTER_ROUTER_HSH_CFG_N_OFFSET(route_id);
+	const struct ipa_reg *reg;
+	u32 offset;
 	u32 val;
 
+	reg = ipa_reg(ipa, ENDP_FILTER_ROUTER_HSH_CFG);
+	offset = ipa_reg_n_offset(reg, route_id);
+
 	val = ioread32(ipa->reg_virt + offset);
 
 	/* Zero all route-related fields, preserving the rest */
-	u32p_replace_bits(&val, 0, IPA_REG_ENDP_ROUTER_HASH_MSK_ALL);
+	val &= ~ipa_reg_fmask(reg, ROUTER_HASH_MSK_ALL);
 
 	iowrite32(val, ipa->reg_virt + offset);
 }
diff --git a/drivers/net/ipa/ipa_table.h b/drivers/net/ipa/ipa_table.h
index 1538e2e1732f..395189f75d78 100644
--- a/drivers/net/ipa/ipa_table.h
+++ b/drivers/net/ipa/ipa_table.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2021 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 #ifndef _IPA_TABLE_H_
 #define _IPA_TABLE_H_
diff --git a/drivers/net/ipa/ipa_uc.c b/drivers/net/ipa/ipa_uc.c
index fe11910518d9..f0ee47281015 100644
--- a/drivers/net/ipa/ipa_uc.c
+++ b/drivers/net/ipa/ipa_uc.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2018-2020 Linaro Ltd.
+ * Copyright (C) 2018-2022 Linaro Ltd.
  */
 
 #include <linux/types.h>
@@ -222,7 +222,7 @@ void ipa_uc_power(struct ipa *ipa)
 static void send_uc_command(struct ipa *ipa, u32 command, u32 command_param)
 {
 	struct ipa_uc_mem_area *shared = ipa_uc_shared(ipa);
-	u32 offset;
+	const struct ipa_reg *reg;
 	u32 val;
 
 	/* Fill in the command data */
@@ -233,9 +233,10 @@ static void send_uc_command(struct ipa *ipa, u32 command, u32 command_param)
 	shared->response_param = 0;
 
 	/* Use an interrupt to tell the microcontroller the command is ready */
-	val = u32_encode_bits(1, UC_INTR_FMASK);
-	offset = ipa_reg_irq_uc_offset(ipa->version);
-	iowrite32(val, ipa->reg_virt + offset);
+	reg = ipa_reg(ipa, IPA_IRQ_UC);
+	val = ipa_reg_bit(reg, UC_INTR);
+
+	iowrite32(val, ipa->reg_virt + ipa_reg_offset(reg));
 }
 
 /* Tell the microcontroller the AP is shutting down */
diff --git a/drivers/net/ipa/ipa_uc.h b/drivers/net/ipa/ipa_uc.h
index 23847f934d64..8514096e6f36 100644
--- a/drivers/net/ipa/ipa_uc.h
+++ b/drivers/net/ipa/ipa_uc.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2020 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 #ifndef _IPA_UC_H_
 #define _IPA_UC_H_
diff --git a/drivers/net/ipa/ipa_version.h b/drivers/net/ipa/ipa_version.h
index 6c16c895d842..7870e0cc3d7c 100644
--- a/drivers/net/ipa/ipa_version.h
+++ b/drivers/net/ipa/ipa_version.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 
 /* Copyright (c) 2012-2018, The Linux Foundation. All rights reserved.
- * Copyright (C) 2019-2020 Linaro Ltd.
+ * Copyright (C) 2019-2022 Linaro Ltd.
  */
 #ifndef _IPA_VERSION_H_
 #define _IPA_VERSION_H_
@@ -19,10 +19,10 @@
  * @IPA_VERSION_4_7:	IPA version 4.7/GSI version 2.7
  * @IPA_VERSION_4_9:	IPA version 4.9/GSI version 2.9
  * @IPA_VERSION_4_11:	IPA version 4.11/GSI version 2.11 (2.1.1)
+ * @IPA_VERSION_COUNT:	Number of defined IPA versions
  *
  * Defines the version of IPA (and GSI) hardware present on the platform.
- * Please update ipa_version_valid() and ipa_version_string() whenever a
- * new version is added.
+ * Please update ipa_version_string() whenever a new version is added.
  */
 enum ipa_version {
 	IPA_VERSION_3_0,
@@ -36,6 +36,30 @@ enum ipa_version {
 	IPA_VERSION_4_7,
 	IPA_VERSION_4_9,
 	IPA_VERSION_4_11,
+	IPA_VERSION_COUNT,			/* Last; not a version */
+};
+
+static inline bool ipa_version_supported(enum ipa_version version)
+{
+	switch (version) {
+	case IPA_VERSION_3_1:
+	case IPA_VERSION_3_5_1:
+	case IPA_VERSION_4_2:
+	case IPA_VERSION_4_5:
+	case IPA_VERSION_4_9:
+	case IPA_VERSION_4_11:
+		return true;
+	default:
+		return false;
+	}
+}
+
+/* Execution environment IDs */
+enum gsi_ee_id {
+	GSI_EE_AP		= 0x0,
+	GSI_EE_MODEM		= 0x1,
+	GSI_EE_UC		= 0x2,
+	GSI_EE_TZ		= 0x3,
 };
 
 #endif /* _IPA_VERSION_H_ */
diff --git a/drivers/net/ipa/reg/ipa_reg-v3.1.c b/drivers/net/ipa/reg/ipa_reg-v3.1.c
new file mode 100644
index 000000000000..116b27717e3d
--- /dev/null
+++ b/drivers/net/ipa/reg/ipa_reg-v3.1.c
@@ -0,0 +1,478 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* Copyright (C) 2022 Linaro Ltd. */
+
+#include <linux/types.h>
+
+#include "../ipa.h"
+#include "../ipa_reg.h"
+
+static const u32 ipa_reg_comp_cfg_fmask[] = {
+	[COMP_CFG_ENABLE]				= BIT(0),
+	[GSI_SNOC_BYPASS_DIS]				= BIT(1),
+	[GEN_QMB_0_SNOC_BYPASS_DIS]			= BIT(2),
+	[GEN_QMB_1_SNOC_BYPASS_DIS]			= BIT(3),
+	[IPA_DCMP_FAST_CLK_EN]				= BIT(4),
+						/* Bits 5-31 reserved */
+};
+
+IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c);
+
+static const u32 ipa_reg_clkon_cfg_fmask[] = {
+	[CLKON_RX]					= BIT(0),
+	[CLKON_PROC]					= BIT(1),
+	[TX_WRAPPER]					= BIT(2),
+	[CLKON_MISC]					= BIT(3),
+	[RAM_ARB]					= BIT(4),
+	[FTCH_HPS]					= BIT(5),
+	[FTCH_DPS]					= BIT(6),
+	[CLKON_HPS]					= BIT(7),
+	[CLKON_DPS]					= BIT(8),
+	[RX_HPS_CMDQS]					= BIT(9),
+	[HPS_DPS_CMDQS]					= BIT(10),
+	[DPS_TX_CMDQS]					= BIT(11),
+	[RSRC_MNGR]					= BIT(12),
+	[CTX_HANDLER]					= BIT(13),
+	[ACK_MNGR]					= BIT(14),
+	[D_DCPH]					= BIT(15),
+	[H_DCPH]					= BIT(16),
+						/* Bits 17-31 reserved */
+};
+
+IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044);
+
+static const u32 ipa_reg_route_fmask[] = {
+	[ROUTE_DIS]					= BIT(0),
+	[ROUTE_DEF_PIPE]				= GENMASK(5, 1),
+	[ROUTE_DEF_HDR_TABLE]				= BIT(6),
+	[ROUTE_DEF_HDR_OFST]				= GENMASK(16, 7),
+	[ROUTE_FRAG_DEF_PIPE]				= GENMASK(21, 17),
+						/* Bits 22-23 reserved */
+	[ROUTE_DEF_RETAIN_HDR]				= BIT(24),
+						/* Bits 25-31 reserved */
+};
+
+IPA_REG_FIELDS(ROUTE, route, 0x00000048);
+
+static const u32 ipa_reg_shared_mem_size_fmask[] = {
+	[MEM_SIZE]					= GENMASK(15, 0),
+	[MEM_BADDR]					= GENMASK(31, 16),
+};
+
+IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054);
+
+static const u32 ipa_reg_qsb_max_writes_fmask[] = {
+	[GEN_QMB_0_MAX_WRITES]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_WRITES]				= GENMASK(7, 4),
+						/* Bits 8-31 reserved */
+};
+
+IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074);
+
+static const u32 ipa_reg_qsb_max_reads_fmask[] = {
+	[GEN_QMB_0_MAX_READS]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_READS]				= GENMASK(7, 4),
+};
+
+IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078);
+
+static const u32 ipa_reg_filt_rout_hash_en_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x000008c);
+
+static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x0000090);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x0000010c);
+
+IPA_REG(IPA_BCR, ipa_bcr, 0x000001d0);
+
+static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = {
+	[IPA_BASE_ADDR]					= GENMASK(16, 0),
+						/* Bits 17-31 reserved */
+};
+
+/* Offset must be a multiple of 8 */
+IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec);
+
+static const u32 ipa_reg_counter_cfg_fmask[] = {
+	[EOT_COAL_GRANULARITY]				= GENMASK(3, 0),
+	[AGGR_GRANULARITY]				= GENMASK(8, 4),
+						/* Bits 5-31 reserved */
+};
+
+IPA_REG_FIELDS(COUNTER_CFG, counter_cfg, 0x000001f0);
+
+static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type,
+		      0x00000400, 0x0020);
+
+static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type,
+		      0x00000404, 0x0020);
+
+static const u32 ipa_reg_src_rsrc_grp_45_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_45_RSRC_TYPE, src_rsrc_grp_45_rsrc_type,
+		      0x00000408, 0x0020);
+
+static const u32 ipa_reg_src_rsrc_grp_67_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_67_RSRC_TYPE, src_rsrc_grp_67_rsrc_type,
+		      0x0000040c, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type,
+		      0x00000500, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type,
+		      0x00000504, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_45_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_45_RSRC_TYPE, dst_rsrc_grp_45_rsrc_type,
+		      0x00000508, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_67_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_67_RSRC_TYPE, dst_rsrc_grp_67_rsrc_type,
+		      0x0000050c, 0x0020);
+
+static const u32 ipa_reg_endp_init_ctrl_fmask[] = {
+	[ENDP_SUSPEND]					= BIT(0),
+	[ENDP_DELAY]					= BIT(1),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_CTRL, endp_init_ctrl, 0x00000800, 0x0070);
+
+static const u32 ipa_reg_endp_init_cfg_fmask[] = {
+	[FRAG_OFFLOAD_EN]				= BIT(0),
+	[CS_OFFLOAD_EN]					= GENMASK(2, 1),
+	[CS_METADATA_HDR_OFFSET]			= GENMASK(6, 3),
+						/* Bit 7 reserved */
+	[CS_GEN_QMB_MASTER_SEL]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070);
+
+static const u32 ipa_reg_endp_init_nat_fmask[] = {
+	[NAT_EN]					= GENMASK(1, 0),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_fmask[] = {
+	[HDR_LEN]					= GENMASK(5, 0),
+	[HDR_OFST_METADATA_VALID]			= BIT(6),
+	[HDR_OFST_METADATA]				= GENMASK(12, 7),
+	[HDR_ADDITIONAL_CONST_LEN]			= GENMASK(18, 13),
+	[HDR_OFST_PKT_SIZE_VALID]			= BIT(19),
+	[HDR_OFST_PKT_SIZE]				= GENMASK(25, 20),
+	[HDR_A5_MUX]					= BIT(26),
+	[HDR_LEN_INC_DEAGG_HDR]				= BIT(27),
+	[HDR_METADATA_REG_VALID]			= BIT(28),
+						/* Bits 29-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = {
+	[HDR_ENDIANNESS]				= BIT(0),
+	[HDR_TOTAL_LEN_OR_PAD_VALID]			= BIT(1),
+	[HDR_TOTAL_LEN_OR_PAD]				= BIT(2),
+	[HDR_PAYLOAD_LEN_INC_PADDING]			= BIT(3),
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET]			= GENMASK(9, 4),
+	[HDR_PAD_TO_ALIGNMENT]				= GENMASK(13, 10),
+						/* Bits 14-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070);
+
+IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask,
+	       0x00000818, 0x0070);
+
+static const u32 ipa_reg_endp_init_mode_fmask[] = {
+	[ENDP_MODE]					= GENMASK(2, 0),
+						/* Bit 3 reserved */
+	[DEST_PIPE_INDEX]				= GENMASK(8, 4),
+						/* Bits 9-11 reserved */
+	[BYTE_THRESHOLD]				= GENMASK(27, 12),
+	[PIPE_REPLICATION_EN]				= BIT(28),
+	[PAD_EN]					= BIT(29),
+	[HDR_FTCH_DISABLE]				= BIT(30),
+						/* Bit 31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070);
+
+static const u32 ipa_reg_endp_init_aggr_fmask[] = {
+	[AGGR_EN]					= GENMASK(1, 0),
+	[AGGR_TYPE]					= GENMASK(4, 2),
+	[BYTE_LIMIT]					= GENMASK(9, 5),
+	[TIME_LIMIT]					= GENMASK(14, 10),
+	[PKT_LIMIT]					= GENMASK(20, 15),
+	[SW_EOF_ACTIVE]					= BIT(21),
+	[FORCE_CLOSE]					= BIT(22),
+						/* Bit 23 reserved */
+	[HARD_BYTE_LIMIT_EN]				= BIT(24),
+						/* Bits 25-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070);
+
+static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = {
+	[HOL_BLOCK_EN]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en,
+		      0x0000082c, 0x0070);
+
+/* Entire register is a tick count */
+static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = {
+	[TIMER_BASE_VALUE]				= GENMASK(31, 0),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer,
+		      0x00000830, 0x0070);
+
+static const u32 ipa_reg_endp_init_deaggr_fmask[] = {
+	[DEAGGR_HDR_LEN]				= GENMASK(5, 0),
+	[SYSPIPE_ERR_DETECTION]				= BIT(6),
+	[PACKET_OFFSET_VALID]				= BIT(7),
+	[PACKET_OFFSET_LOCATION]			= GENMASK(13, 8),
+	[IGNORE_MIN_PKT_ERR]				= BIT(14),
+						/* Bit 15 reserved */
+	[MAX_PACKET_LEN]				= GENMASK(31, 16),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070);
+
+static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = {
+	[ENDP_RSRC_GRP]					= GENMASK(2, 0),
+						/* Bits 3-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp,
+		      0x00000838, 0x0070);
+
+static const u32 ipa_reg_endp_init_seq_fmask[] = {
+	[SEQ_TYPE]					= GENMASK(7, 0),
+	[SEQ_REP_TYPE]					= GENMASK(15, 8),
+						/* Bits 16-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070);
+
+static const u32 ipa_reg_endp_status_fmask[] = {
+	[STATUS_EN]					= BIT(0),
+	[STATUS_ENDP]					= GENMASK(5, 1),
+						/* Bits 6-7 reserved */
+	[STATUS_LOCATION]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070);
+
+static const u32 ipa_reg_endp_filter_router_hsh_cfg_fmask[] = {
+	[FILTER_HASH_MSK_SRC_ID]			= BIT(0),
+	[FILTER_HASH_MSK_SRC_IP]			= BIT(1),
+	[FILTER_HASH_MSK_DST_IP]			= BIT(2),
+	[FILTER_HASH_MSK_SRC_PORT]			= BIT(3),
+	[FILTER_HASH_MSK_DST_PORT]			= BIT(4),
+	[FILTER_HASH_MSK_PROTOCOL]			= BIT(5),
+	[FILTER_HASH_MSK_METADATA]			= BIT(6),
+	[FILTER_HASH_MSK_ALL]				= GENMASK(6, 0),
+						/* Bits 7-15 reserved */
+	[ROUTER_HASH_MSK_SRC_ID]			= BIT(16),
+	[ROUTER_HASH_MSK_SRC_IP]			= BIT(17),
+	[ROUTER_HASH_MSK_DST_IP]			= BIT(18),
+	[ROUTER_HASH_MSK_SRC_PORT]			= BIT(19),
+	[ROUTER_HASH_MSK_DST_PORT]			= BIT(20),
+	[ROUTER_HASH_MSK_PROTOCOL]			= BIT(21),
+	[ROUTER_HASH_MSK_METADATA]			= BIT(22),
+	[ROUTER_HASH_MSK_ALL]				= GENMASK(22, 16),
+						/* Bits 23-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_FILTER_ROUTER_HSH_CFG, endp_filter_router_hsh_cfg,
+		      0x0000085c, 0x0070);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00003008 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000300c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00003010 + 0x1000 * GSI_EE_AP);
+
+static const u32 ipa_reg_ipa_irq_uc_fmask[] = {
+	[UC_INTR]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000301c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00003030 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00003034 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00003038 + 0x1000 * GSI_EE_AP);
+
+static const struct ipa_reg *ipa_reg_array[] = {
+	[COMP_CFG]			= &ipa_reg_comp_cfg,
+	[CLKON_CFG]			= &ipa_reg_clkon_cfg,
+	[ROUTE]				= &ipa_reg_route,
+	[SHARED_MEM_SIZE]		= &ipa_reg_shared_mem_size,
+	[QSB_MAX_WRITES]		= &ipa_reg_qsb_max_writes,
+	[QSB_MAX_READS]			= &ipa_reg_qsb_max_reads,
+	[FILT_ROUT_HASH_EN]		= &ipa_reg_filt_rout_hash_en,
+	[FILT_ROUT_HASH_FLUSH]		= &ipa_reg_filt_rout_hash_flush,
+	[STATE_AGGR_ACTIVE]		= &ipa_reg_state_aggr_active,
+	[IPA_BCR]			= &ipa_reg_ipa_bcr,
+	[LOCAL_PKT_PROC_CNTXT]		= &ipa_reg_local_pkt_proc_cntxt,
+	[AGGR_FORCE_CLOSE]		= &ipa_reg_aggr_force_close,
+	[COUNTER_CFG]			= &ipa_reg_counter_cfg,
+	[SRC_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_01_rsrc_type,
+	[SRC_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_23_rsrc_type,
+	[SRC_RSRC_GRP_45_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_45_rsrc_type,
+	[SRC_RSRC_GRP_67_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_67_rsrc_type,
+	[DST_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_01_rsrc_type,
+	[DST_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_23_rsrc_type,
+	[DST_RSRC_GRP_45_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_45_rsrc_type,
+	[DST_RSRC_GRP_67_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_67_rsrc_type,
+	[ENDP_INIT_CTRL]		= &ipa_reg_endp_init_ctrl,
+	[ENDP_INIT_CFG]			= &ipa_reg_endp_init_cfg,
+	[ENDP_INIT_NAT]			= &ipa_reg_endp_init_nat,
+	[ENDP_INIT_HDR]			= &ipa_reg_endp_init_hdr,
+	[ENDP_INIT_HDR_EXT]		= &ipa_reg_endp_init_hdr_ext,
+	[ENDP_INIT_HDR_METADATA_MASK]	= &ipa_reg_endp_init_hdr_metadata_mask,
+	[ENDP_INIT_MODE]		= &ipa_reg_endp_init_mode,
+	[ENDP_INIT_AGGR]		= &ipa_reg_endp_init_aggr,
+	[ENDP_INIT_HOL_BLOCK_EN]	= &ipa_reg_endp_init_hol_block_en,
+	[ENDP_INIT_HOL_BLOCK_TIMER]	= &ipa_reg_endp_init_hol_block_timer,
+	[ENDP_INIT_DEAGGR]		= &ipa_reg_endp_init_deaggr,
+	[ENDP_INIT_RSRC_GRP]		= &ipa_reg_endp_init_rsrc_grp,
+	[ENDP_INIT_SEQ]			= &ipa_reg_endp_init_seq,
+	[ENDP_STATUS]			= &ipa_reg_endp_status,
+	[ENDP_FILTER_ROUTER_HSH_CFG]	= &ipa_reg_endp_filter_router_hsh_cfg,
+	[IPA_IRQ_STTS]			= &ipa_reg_ipa_irq_stts,
+	[IPA_IRQ_EN]			= &ipa_reg_ipa_irq_en,
+	[IPA_IRQ_CLR]			= &ipa_reg_ipa_irq_clr,
+	[IPA_IRQ_UC]			= &ipa_reg_ipa_irq_uc,
+	[IRQ_SUSPEND_INFO]		= &ipa_reg_irq_suspend_info,
+	[IRQ_SUSPEND_EN]		= &ipa_reg_irq_suspend_en,
+	[IRQ_SUSPEND_CLR]		= &ipa_reg_irq_suspend_clr,
+};
+
+const struct ipa_regs ipa_regs_v3_1 = {
+	.reg_count	= ARRAY_SIZE(ipa_reg_array),
+	.reg		= ipa_reg_array,
+};
diff --git a/drivers/net/ipa/reg/ipa_reg-v3.5.1.c b/drivers/net/ipa/reg/ipa_reg-v3.5.1.c
new file mode 100644
index 000000000000..6e2f939b18f1
--- /dev/null
+++ b/drivers/net/ipa/reg/ipa_reg-v3.5.1.c
@@ -0,0 +1,456 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* Copyright (C) 2022 Linaro Ltd. */
+
+#include <linux/types.h>
+
+#include "../ipa.h"
+#include "../ipa_reg.h"
+
+static const u32 ipa_reg_comp_cfg_fmask[] = {
+	[COMP_CFG_ENABLE]				= BIT(0),
+	[GSI_SNOC_BYPASS_DIS]				= BIT(1),
+	[GEN_QMB_0_SNOC_BYPASS_DIS]			= BIT(2),
+	[GEN_QMB_1_SNOC_BYPASS_DIS]			= BIT(3),
+	[IPA_DCMP_FAST_CLK_EN]				= BIT(4),
+						/* Bits 5-31 reserved */
+};
+
+IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c);
+
+static const u32 ipa_reg_clkon_cfg_fmask[] = {
+	[CLKON_RX]					= BIT(0),
+	[CLKON_PROC]					= BIT(1),
+	[TX_WRAPPER]					= BIT(2),
+	[CLKON_MISC]					= BIT(3),
+	[RAM_ARB]					= BIT(4),
+	[FTCH_HPS]					= BIT(5),
+	[FTCH_DPS]					= BIT(6),
+	[CLKON_HPS]					= BIT(7),
+	[CLKON_DPS]					= BIT(8),
+	[RX_HPS_CMDQS]					= BIT(9),
+	[HPS_DPS_CMDQS]					= BIT(10),
+	[DPS_TX_CMDQS]					= BIT(11),
+	[RSRC_MNGR]					= BIT(12),
+	[CTX_HANDLER]					= BIT(13),
+	[ACK_MNGR]					= BIT(14),
+	[D_DCPH]					= BIT(15),
+	[H_DCPH]					= BIT(16),
+						/* Bit 17 reserved */
+	[NTF_TX_CMDQS]					= BIT(18),
+	[CLKON_TX_0]					= BIT(19),
+	[CLKON_TX_1]					= BIT(20),
+	[CLKON_FNR]					= BIT(21),
+						/* Bits 22-31 reserved */
+};
+
+IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044);
+
+static const u32 ipa_reg_route_fmask[] = {
+	[ROUTE_DIS]					= BIT(0),
+	[ROUTE_DEF_PIPE]				= GENMASK(5, 1),
+	[ROUTE_DEF_HDR_TABLE]				= BIT(6),
+	[ROUTE_DEF_HDR_OFST]				= GENMASK(16, 7),
+	[ROUTE_FRAG_DEF_PIPE]				= GENMASK(21, 17),
+						/* Bits 22-23 reserved */
+	[ROUTE_DEF_RETAIN_HDR]				= BIT(24),
+						/* Bits 25-31 reserved */
+};
+
+IPA_REG_FIELDS(ROUTE, route, 0x00000048);
+
+static const u32 ipa_reg_shared_mem_size_fmask[] = {
+	[MEM_SIZE]					= GENMASK(15, 0),
+	[MEM_BADDR]					= GENMASK(31, 16),
+};
+
+IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054);
+
+static const u32 ipa_reg_qsb_max_writes_fmask[] = {
+	[GEN_QMB_0_MAX_WRITES]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_WRITES]				= GENMASK(7, 4),
+						/* Bits 8-31 reserved */
+};
+
+IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074);
+
+static const u32 ipa_reg_qsb_max_reads_fmask[] = {
+	[GEN_QMB_0_MAX_READS]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_READS]				= GENMASK(7, 4),
+};
+
+IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078);
+
+static const u32 ipa_reg_filt_rout_hash_en_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x000008c);
+
+static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x0000090);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x0000010c);
+
+IPA_REG(IPA_BCR, ipa_bcr, 0x000001d0);
+
+static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = {
+	[IPA_BASE_ADDR]					= GENMASK(16, 0),
+						/* Bits 17-31 reserved */
+};
+
+/* Offset must be a multiple of 8 */
+IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec);
+
+static const u32 ipa_reg_counter_cfg_fmask[] = {
+						/* Bits 0-3 reserved */
+	[AGGR_GRANULARITY]				= GENMASK(8, 4),
+						/* Bits 5-31 reserved */
+};
+
+IPA_REG_FIELDS(COUNTER_CFG, counter_cfg, 0x000001f0);
+
+static const u32 ipa_reg_ipa_tx_cfg_fmask[] = {
+	[TX0_PREFETCH_DISABLE]				= BIT(0),
+	[TX1_PREFETCH_DISABLE]				= BIT(1),
+	[PREFETCH_ALMOST_EMPTY_SIZE]			= GENMASK(4, 2),
+						/* Bits 5-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x000001fc);
+
+static const u32 ipa_reg_flavor_0_fmask[] = {
+	[MAX_PIPES]					= GENMASK(3, 0),
+						/* Bits 4-7 reserved */
+	[MAX_CONS_PIPES]				= GENMASK(12, 8),
+						/* Bits 13-15 reserved */
+	[MAX_PROD_PIPES]				= GENMASK(20, 16),
+						/* Bits 21-23 reserved */
+	[PROD_LOWEST]					= GENMASK(27, 24),
+						/* Bits 28-31 reserved */
+};
+
+IPA_REG_FIELDS(FLAVOR_0, flavor_0, 0x00000210);
+
+static const u32 ipa_reg_idle_indication_cfg_fmask[] = {
+	[ENTER_IDLE_DEBOUNCE_THRESH]			= GENMASK(15, 0),
+	[CONST_NON_IDLE_ENABLE]				= BIT(16),
+						/* Bits 17-31 reserved */
+};
+
+IPA_REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x00000220);
+
+static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type,
+		      0x00000400, 0x0020);
+
+static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type,
+		      0x00000404, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type,
+		      0x00000500, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type,
+		      0x00000504, 0x0020);
+
+static const u32 ipa_reg_endp_init_ctrl_fmask[] = {
+	[ENDP_SUSPEND]					= BIT(0),
+	[ENDP_DELAY]					= BIT(1),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_CTRL, endp_init_ctrl, 0x00000800, 0x0070);
+
+static const u32 ipa_reg_endp_init_cfg_fmask[] = {
+	[FRAG_OFFLOAD_EN]				= BIT(0),
+	[CS_OFFLOAD_EN]					= GENMASK(2, 1),
+	[CS_METADATA_HDR_OFFSET]			= GENMASK(6, 3),
+						/* Bit 7 reserved */
+	[CS_GEN_QMB_MASTER_SEL]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070);
+
+static const u32 ipa_reg_endp_init_nat_fmask[] = {
+	[NAT_EN]					= GENMASK(1, 0),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_fmask[] = {
+	[HDR_LEN]					= GENMASK(5, 0),
+	[HDR_OFST_METADATA_VALID]			= BIT(6),
+	[HDR_OFST_METADATA]				= GENMASK(12, 7),
+	[HDR_ADDITIONAL_CONST_LEN]			= GENMASK(18, 13),
+	[HDR_OFST_PKT_SIZE_VALID]			= BIT(19),
+	[HDR_OFST_PKT_SIZE]				= GENMASK(25, 20),
+	[HDR_A5_MUX]					= BIT(26),
+	[HDR_LEN_INC_DEAGG_HDR]				= BIT(27),
+	[HDR_METADATA_REG_VALID]			= BIT(28),
+						/* Bits 29-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = {
+	[HDR_ENDIANNESS]				= BIT(0),
+	[HDR_TOTAL_LEN_OR_PAD_VALID]			= BIT(1),
+	[HDR_TOTAL_LEN_OR_PAD]				= BIT(2),
+	[HDR_PAYLOAD_LEN_INC_PADDING]			= BIT(3),
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET]			= GENMASK(9, 4),
+	[HDR_PAD_TO_ALIGNMENT]				= GENMASK(13, 10),
+						/* Bits 14-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070);
+
+IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask,
+	       0x00000818, 0x0070);
+
+static const u32 ipa_reg_endp_init_mode_fmask[] = {
+	[ENDP_MODE]					= GENMASK(2, 0),
+						/* Bit 3 reserved */
+	[DEST_PIPE_INDEX]				= GENMASK(8, 4),
+						/* Bits 9-11 reserved */
+	[BYTE_THRESHOLD]				= GENMASK(27, 12),
+	[PIPE_REPLICATION_EN]				= BIT(28),
+	[PAD_EN]					= BIT(29),
+	[HDR_FTCH_DISABLE]				= BIT(30),
+						/* Bit 31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070);
+
+static const u32 ipa_reg_endp_init_aggr_fmask[] = {
+	[AGGR_EN]					= GENMASK(1, 0),
+	[AGGR_TYPE]					= GENMASK(4, 2),
+	[BYTE_LIMIT]					= GENMASK(9, 5),
+	[TIME_LIMIT]					= GENMASK(14, 10),
+	[PKT_LIMIT]					= GENMASK(20, 15),
+	[SW_EOF_ACTIVE]					= BIT(21),
+	[FORCE_CLOSE]					= BIT(22),
+						/* Bit 23 reserved */
+	[HARD_BYTE_LIMIT_EN]				= BIT(24),
+						/* Bits 25-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070);
+
+static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = {
+	[HOL_BLOCK_EN]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en,
+		      0x0000082c, 0x0070);
+
+/* Entire register is a tick count */
+static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = {
+	[TIMER_BASE_VALUE]				= GENMASK(31, 0),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer,
+		      0x00000830, 0x0070);
+
+static const u32 ipa_reg_endp_init_deaggr_fmask[] = {
+	[DEAGGR_HDR_LEN]				= GENMASK(5, 0),
+	[SYSPIPE_ERR_DETECTION]				= BIT(6),
+	[PACKET_OFFSET_VALID]				= BIT(7),
+	[PACKET_OFFSET_LOCATION]			= GENMASK(13, 8),
+	[IGNORE_MIN_PKT_ERR]				= BIT(14),
+						/* Bit 15 reserved */
+	[MAX_PACKET_LEN]				= GENMASK(31, 16),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070);
+
+static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = {
+	[ENDP_RSRC_GRP]					= GENMASK(1, 0),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp,
+		      0x00000838, 0x0070);
+
+static const u32 ipa_reg_endp_init_seq_fmask[] = {
+	[SEQ_TYPE]					= GENMASK(7, 0),
+	[SEQ_REP_TYPE]					= GENMASK(15, 8),
+						/* Bits 16-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070);
+
+static const u32 ipa_reg_endp_status_fmask[] = {
+	[STATUS_EN]					= BIT(0),
+	[STATUS_ENDP]					= GENMASK(5, 1),
+						/* Bits 6-7 reserved */
+	[STATUS_LOCATION]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070);
+
+static const u32 ipa_reg_endp_filter_router_hsh_cfg_fmask[] = {
+	[FILTER_HASH_MSK_SRC_ID]			= BIT(0),
+	[FILTER_HASH_MSK_SRC_IP]			= BIT(1),
+	[FILTER_HASH_MSK_DST_IP]			= BIT(2),
+	[FILTER_HASH_MSK_SRC_PORT]			= BIT(3),
+	[FILTER_HASH_MSK_DST_PORT]			= BIT(4),
+	[FILTER_HASH_MSK_PROTOCOL]			= BIT(5),
+	[FILTER_HASH_MSK_METADATA]			= BIT(6),
+	[FILTER_HASH_MSK_ALL]				= GENMASK(6, 0),
+						/* Bits 7-15 reserved */
+	[ROUTER_HASH_MSK_SRC_ID]			= BIT(16),
+	[ROUTER_HASH_MSK_SRC_IP]			= BIT(17),
+	[ROUTER_HASH_MSK_DST_IP]			= BIT(18),
+	[ROUTER_HASH_MSK_SRC_PORT]			= BIT(19),
+	[ROUTER_HASH_MSK_DST_PORT]			= BIT(20),
+	[ROUTER_HASH_MSK_PROTOCOL]			= BIT(21),
+	[ROUTER_HASH_MSK_METADATA]			= BIT(22),
+	[ROUTER_HASH_MSK_ALL]				= GENMASK(22, 16),
+						/* Bits 23-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_FILTER_ROUTER_HSH_CFG, endp_filter_router_hsh_cfg,
+		      0x0000085c, 0x0070);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00003008 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000300c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00003010 + 0x1000 * GSI_EE_AP);
+
+static const u32 ipa_reg_ipa_irq_uc_fmask[] = {
+	[UC_INTR]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000301c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00003030 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00003034 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00003038 + 0x1000 * GSI_EE_AP);
+
+static const struct ipa_reg *ipa_reg_array[] = {
+	[COMP_CFG]			= &ipa_reg_comp_cfg,
+	[CLKON_CFG]			= &ipa_reg_clkon_cfg,
+	[ROUTE]				= &ipa_reg_route,
+	[SHARED_MEM_SIZE]		= &ipa_reg_shared_mem_size,
+	[QSB_MAX_WRITES]		= &ipa_reg_qsb_max_writes,
+	[QSB_MAX_READS]			= &ipa_reg_qsb_max_reads,
+	[FILT_ROUT_HASH_EN]		= &ipa_reg_filt_rout_hash_en,
+	[FILT_ROUT_HASH_FLUSH]		= &ipa_reg_filt_rout_hash_flush,
+	[STATE_AGGR_ACTIVE]		= &ipa_reg_state_aggr_active,
+	[IPA_BCR]			= &ipa_reg_ipa_bcr,
+	[LOCAL_PKT_PROC_CNTXT]		= &ipa_reg_local_pkt_proc_cntxt,
+	[AGGR_FORCE_CLOSE]		= &ipa_reg_aggr_force_close,
+	[COUNTER_CFG]			= &ipa_reg_counter_cfg,
+	[IPA_TX_CFG]			= &ipa_reg_ipa_tx_cfg,
+	[FLAVOR_0]			= &ipa_reg_flavor_0,
+	[IDLE_INDICATION_CFG]		= &ipa_reg_idle_indication_cfg,
+	[SRC_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_01_rsrc_type,
+	[SRC_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_23_rsrc_type,
+	[DST_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_01_rsrc_type,
+	[DST_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_23_rsrc_type,
+	[ENDP_INIT_CTRL]		= &ipa_reg_endp_init_ctrl,
+	[ENDP_INIT_CFG]			= &ipa_reg_endp_init_cfg,
+	[ENDP_INIT_NAT]			= &ipa_reg_endp_init_nat,
+	[ENDP_INIT_HDR]			= &ipa_reg_endp_init_hdr,
+	[ENDP_INIT_HDR_EXT]		= &ipa_reg_endp_init_hdr_ext,
+	[ENDP_INIT_HDR_METADATA_MASK]	= &ipa_reg_endp_init_hdr_metadata_mask,
+	[ENDP_INIT_MODE]		= &ipa_reg_endp_init_mode,
+	[ENDP_INIT_AGGR]		= &ipa_reg_endp_init_aggr,
+	[ENDP_INIT_HOL_BLOCK_EN]	= &ipa_reg_endp_init_hol_block_en,
+	[ENDP_INIT_HOL_BLOCK_TIMER]	= &ipa_reg_endp_init_hol_block_timer,
+	[ENDP_INIT_DEAGGR]		= &ipa_reg_endp_init_deaggr,
+	[ENDP_INIT_RSRC_GRP]		= &ipa_reg_endp_init_rsrc_grp,
+	[ENDP_INIT_SEQ]			= &ipa_reg_endp_init_seq,
+	[ENDP_STATUS]			= &ipa_reg_endp_status,
+	[ENDP_FILTER_ROUTER_HSH_CFG]	= &ipa_reg_endp_filter_router_hsh_cfg,
+	[IPA_IRQ_STTS]			= &ipa_reg_ipa_irq_stts,
+	[IPA_IRQ_EN]			= &ipa_reg_ipa_irq_en,
+	[IPA_IRQ_CLR]			= &ipa_reg_ipa_irq_clr,
+	[IPA_IRQ_UC]			= &ipa_reg_ipa_irq_uc,
+	[IRQ_SUSPEND_INFO]		= &ipa_reg_irq_suspend_info,
+	[IRQ_SUSPEND_EN]		= &ipa_reg_irq_suspend_en,
+	[IRQ_SUSPEND_CLR]		= &ipa_reg_irq_suspend_clr,
+};
+
+const struct ipa_regs ipa_regs_v3_5_1 = {
+	.reg_count	= ARRAY_SIZE(ipa_reg_array),
+	.reg		= ipa_reg_array,
+};
diff --git a/drivers/net/ipa/reg/ipa_reg-v4.11.c b/drivers/net/ipa/reg/ipa_reg-v4.11.c
new file mode 100644
index 000000000000..8fd36569bb9f
--- /dev/null
+++ b/drivers/net/ipa/reg/ipa_reg-v4.11.c
@@ -0,0 +1,512 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* Copyright (C) 2022 Linaro Ltd. */
+
+#include <linux/types.h>
+
+#include "../ipa.h"
+#include "../ipa_reg.h"
+
+static const u32 ipa_reg_comp_cfg_fmask[] = {
+	[RAM_ARB_PRI_CLIENT_SAMP_FIX_DIS]		= BIT(0),
+	[GSI_SNOC_BYPASS_DIS]				= BIT(1),
+	[GEN_QMB_0_SNOC_BYPASS_DIS]			= BIT(2),
+	[GEN_QMB_1_SNOC_BYPASS_DIS]			= BIT(3),
+						/* Bit 4 reserved */
+	[IPA_QMB_SELECT_CONS_EN]			= BIT(5),
+	[IPA_QMB_SELECT_PROD_EN]			= BIT(6),
+	[GSI_MULTI_INORDER_RD_DIS]			= BIT(7),
+	[GSI_MULTI_INORDER_WR_DIS]			= BIT(8),
+	[GEN_QMB_0_MULTI_INORDER_RD_DIS]		= BIT(9),
+	[GEN_QMB_1_MULTI_INORDER_RD_DIS]		= BIT(10),
+	[GEN_QMB_0_MULTI_INORDER_WR_DIS]		= BIT(11),
+	[GEN_QMB_1_MULTI_INORDER_WR_DIS]		= BIT(12),
+	[GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS]		= BIT(13),
+	[GSI_SNOC_CNOC_LOOP_PROT_DISABLE]		= BIT(14),
+	[GSI_MULTI_AXI_MASTERS_DIS]			= BIT(15),
+	[IPA_QMB_SELECT_GLOBAL_EN]			= BIT(16),
+	[FULL_FLUSH_WAIT_RS_CLOSURE_EN]			= BIT(17),
+						/* Bit 18 reserved */
+	[QMB_RAM_RD_CACHE_DISABLE]			= BIT(19),
+	[GENQMB_AOOOWR]					= BIT(20),
+	[IF_OUT_OF_BUF_STOP_RESET_MASK_EN]		= BIT(21),
+	[ATOMIC_FETCHER_ARB_LOCK_DIS]			= GENMASK(23, 22),
+						/* Bits 24-29 reserved */
+	[GEN_QMB_1_DYNAMIC_ASIZE]			= BIT(30),
+	[GEN_QMB_0_DYNAMIC_ASIZE]			= BIT(31),
+};
+
+IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c);
+
+static const u32 ipa_reg_clkon_cfg_fmask[] = {
+	[CLKON_RX]					= BIT(0),
+	[CLKON_PROC]					= BIT(1),
+	[TX_WRAPPER]					= BIT(2),
+	[CLKON_MISC]					= BIT(3),
+	[RAM_ARB]					= BIT(4),
+	[FTCH_HPS]					= BIT(5),
+	[FTCH_DPS]					= BIT(6),
+	[CLKON_HPS]					= BIT(7),
+	[CLKON_DPS]					= BIT(8),
+	[RX_HPS_CMDQS]					= BIT(9),
+	[HPS_DPS_CMDQS]					= BIT(10),
+	[DPS_TX_CMDQS]					= BIT(11),
+	[RSRC_MNGR]					= BIT(12),
+	[CTX_HANDLER]					= BIT(13),
+	[ACK_MNGR]					= BIT(14),
+	[D_DCPH]					= BIT(15),
+	[H_DCPH]					= BIT(16),
+						/* Bit 17 reserved */
+	[NTF_TX_CMDQS]					= BIT(18),
+	[CLKON_TX_0]					= BIT(19),
+	[CLKON_TX_1]					= BIT(20),
+	[CLKON_FNR]					= BIT(21),
+	[QSB2AXI_CMDQ_L]				= BIT(22),
+	[AGGR_WRAPPER]					= BIT(23),
+	[RAM_SLAVEWAY]					= BIT(24),
+	[CLKON_QMB]					= BIT(25),
+	[WEIGHT_ARB]					= BIT(26),
+	[GSI_IF]					= BIT(27),
+	[CLKON_GLOBAL]					= BIT(28),
+	[GLOBAL_2X_CLK]					= BIT(29),
+	[DPL_FIFO]					= BIT(30),
+	[DRBIP]						= BIT(31),
+};
+
+IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044);
+
+static const u32 ipa_reg_route_fmask[] = {
+	[ROUTE_DIS]					= BIT(0),
+	[ROUTE_DEF_PIPE]				= GENMASK(5, 1),
+	[ROUTE_DEF_HDR_TABLE]				= BIT(6),
+	[ROUTE_DEF_HDR_OFST]				= GENMASK(16, 7),
+	[ROUTE_FRAG_DEF_PIPE]				= GENMASK(21, 17),
+						/* Bits 22-23 reserved */
+	[ROUTE_DEF_RETAIN_HDR]				= BIT(24),
+						/* Bits 25-31 reserved */
+};
+
+IPA_REG_FIELDS(ROUTE, route, 0x00000048);
+
+static const u32 ipa_reg_shared_mem_size_fmask[] = {
+	[MEM_SIZE]					= GENMASK(15, 0),
+	[MEM_BADDR]					= GENMASK(31, 16),
+};
+
+IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054);
+
+static const u32 ipa_reg_qsb_max_writes_fmask[] = {
+	[GEN_QMB_0_MAX_WRITES]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_WRITES]				= GENMASK(7, 4),
+						/* Bits 8-31 reserved */
+};
+
+IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074);
+
+static const u32 ipa_reg_qsb_max_reads_fmask[] = {
+	[GEN_QMB_0_MAX_READS]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_READS]				= GENMASK(7, 4),
+						/* Bits 8-15 reserved */
+	[GEN_QMB_0_MAX_READS_BEATS]			= GENMASK(23, 16),
+	[GEN_QMB_1_MAX_READS_BEATS]			= GENMASK(31, 24),
+};
+
+IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078);
+
+static const u32 ipa_reg_filt_rout_hash_en_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x0000148);
+
+static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x000014c);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x000000b4);
+
+static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = {
+	[IPA_BASE_ADDR]					= GENMASK(17, 0),
+						/* Bits 18-31 reserved */
+};
+
+/* Offset must be a multiple of 8 */
+IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec);
+
+static const u32 ipa_reg_ipa_tx_cfg_fmask[] = {
+						/* Bits 0-1 reserved */
+	[PREFETCH_ALMOST_EMPTY_SIZE_TX0]		= GENMASK(5, 2),
+	[DMAW_SCND_OUTSD_PRED_THRESHOLD]		= GENMASK(9, 6),
+	[DMAW_SCND_OUTSD_PRED_EN]			= BIT(10),
+	[DMAW_MAX_BEATS_256_DIS]			= BIT(11),
+	[PA_MASK_EN]					= BIT(12),
+	[PREFETCH_ALMOST_EMPTY_SIZE_TX1]		= GENMASK(16, 13),
+	[DUAL_TX_ENABLE]				= BIT(17),
+	[SSPND_PA_NO_START_STATE]			= BIT(18),
+						/* Bits 19-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x000001fc);
+
+static const u32 ipa_reg_flavor_0_fmask[] = {
+	[MAX_PIPES]					= GENMASK(4, 0),
+						/* Bits 5-7 reserved */
+	[MAX_CONS_PIPES]				= GENMASK(12, 8),
+						/* Bits 13-15 reserved */
+	[MAX_PROD_PIPES]				= GENMASK(20, 16),
+						/* Bits 21-23 reserved */
+	[PROD_LOWEST]					= GENMASK(27, 24),
+						/* Bits 28-31 reserved */
+};
+
+IPA_REG_FIELDS(FLAVOR_0, flavor_0, 0x00000210);
+
+static const u32 ipa_reg_idle_indication_cfg_fmask[] = {
+	[ENTER_IDLE_DEBOUNCE_THRESH]			= GENMASK(15, 0),
+	[CONST_NON_IDLE_ENABLE]				= BIT(16),
+						/* Bits 17-31 reserved */
+};
+
+IPA_REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x00000240);
+
+static const u32 ipa_reg_qtime_timestamp_cfg_fmask[] = {
+	[DPL_TIMESTAMP_LSB]				= GENMASK(4, 0),
+						/* Bits 5-6 reserved */
+	[DPL_TIMESTAMP_SEL]				= BIT(7),
+	[TAG_TIMESTAMP_LSB]				= GENMASK(12, 8),
+						/* Bits 13-15 reserved */
+	[NAT_TIMESTAMP_LSB]				= GENMASK(20, 16),
+						/* Bits 21-31 reserved */
+};
+
+IPA_REG_FIELDS(QTIME_TIMESTAMP_CFG, qtime_timestamp_cfg, 0x0000024c);
+
+static const u32 ipa_reg_timers_xo_clk_div_cfg_fmask[] = {
+	[DIV_VALUE]					= GENMASK(8, 0),
+						/* Bits 9-30 reserved */
+	[DIV_ENABLE]					= BIT(31),
+};
+
+IPA_REG_FIELDS(TIMERS_XO_CLK_DIV_CFG, timers_xo_clk_div_cfg, 0x00000250);
+
+static const u32 ipa_reg_timers_pulse_gran_cfg_fmask[] = {
+	[PULSE_GRAN_0]					= GENMASK(2, 0),
+	[PULSE_GRAN_1]					= GENMASK(5, 3),
+	[PULSE_GRAN_2]					= GENMASK(8, 6),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_FIELDS(TIMERS_PULSE_GRAN_CFG, timers_pulse_gran_cfg, 0x00000254);
+
+static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type,
+		      0x00000400, 0x0020);
+
+static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type,
+		      0x00000404, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type,
+		      0x00000500, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type,
+		      0x00000504, 0x0020);
+
+static const u32 ipa_reg_endp_init_cfg_fmask[] = {
+	[FRAG_OFFLOAD_EN]				= BIT(0),
+	[CS_OFFLOAD_EN]					= GENMASK(2, 1),
+	[CS_METADATA_HDR_OFFSET]			= GENMASK(6, 3),
+						/* Bit 7 reserved */
+	[CS_GEN_QMB_MASTER_SEL]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070);
+
+static const u32 ipa_reg_endp_init_nat_fmask[] = {
+	[NAT_EN]					= GENMASK(1, 0),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_fmask[] = {
+	[HDR_LEN]					= GENMASK(5, 0),
+	[HDR_OFST_METADATA_VALID]			= BIT(6),
+	[HDR_OFST_METADATA]				= GENMASK(12, 7),
+	[HDR_ADDITIONAL_CONST_LEN]			= GENMASK(18, 13),
+	[HDR_OFST_PKT_SIZE_VALID]			= BIT(19),
+	[HDR_OFST_PKT_SIZE]				= GENMASK(25, 20),
+						/* Bit 26 reserved */
+	[HDR_LEN_INC_DEAGG_HDR]				= BIT(27),
+	[HDR_LEN_MSB]					= GENMASK(29, 28),
+	[HDR_OFST_METADATA_MSB]				= GENMASK(31, 30),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = {
+	[HDR_ENDIANNESS]				= BIT(0),
+	[HDR_TOTAL_LEN_OR_PAD_VALID]			= BIT(1),
+	[HDR_TOTAL_LEN_OR_PAD]				= BIT(2),
+	[HDR_PAYLOAD_LEN_INC_PADDING]			= BIT(3),
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET]			= GENMASK(9, 4),
+	[HDR_PAD_TO_ALIGNMENT]				= GENMASK(13, 10),
+						/* Bits 14-15 reserved */
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB]		= GENMASK(17, 16),
+	[HDR_OFST_PKT_SIZE_MSB]				= GENMASK(19, 18),
+	[HDR_ADDITIONAL_CONST_LEN_MSB]			= GENMASK(21, 20),
+						/* Bits 22-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070);
+
+IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask,
+	       0x00000818, 0x0070);
+
+static const u32 ipa_reg_endp_init_mode_fmask[] = {
+	[ENDP_MODE]					= GENMASK(2, 0),
+	[DCPH_ENABLE]					= BIT(3),
+	[DEST_PIPE_INDEX]				= GENMASK(8, 4),
+						/* Bits 9-11 reserved */
+	[BYTE_THRESHOLD]				= GENMASK(27, 12),
+	[PIPE_REPLICATION_EN]				= BIT(28),
+	[PAD_EN]					= BIT(29),
+	[DRBIP_ACL_ENABLE]				= BIT(30),
+						/* Bit 31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070);
+
+static const u32 ipa_reg_endp_init_aggr_fmask[] = {
+	[AGGR_EN]					= GENMASK(1, 0),
+	[AGGR_TYPE]					= GENMASK(4, 2),
+	[BYTE_LIMIT]					= GENMASK(10, 5),
+						/* Bit 11 reserved */
+	[TIME_LIMIT]					= GENMASK(16, 12),
+	[PKT_LIMIT]					= GENMASK(22, 17),
+	[SW_EOF_ACTIVE]					= BIT(23),
+	[FORCE_CLOSE]					= BIT(24),
+						/* Bit 25 reserved */
+	[HARD_BYTE_LIMIT_EN]				= BIT(26),
+	[AGGR_GRAN_SEL]					= BIT(27),
+						/* Bits 28-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070);
+
+static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = {
+	[HOL_BLOCK_EN]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en,
+		      0x0000082c, 0x0070);
+
+static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = {
+	[TIMER_LIMIT]					= GENMASK(4, 0),
+						/* Bits 5-7 reserved */
+	[TIMER_GRAN_SEL]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer,
+		      0x00000830, 0x0070);
+
+static const u32 ipa_reg_endp_init_deaggr_fmask[] = {
+	[DEAGGR_HDR_LEN]				= GENMASK(5, 0),
+	[SYSPIPE_ERR_DETECTION]				= BIT(6),
+	[PACKET_OFFSET_VALID]				= BIT(7),
+	[PACKET_OFFSET_LOCATION]			= GENMASK(13, 8),
+	[IGNORE_MIN_PKT_ERR]				= BIT(14),
+						/* Bit 15 reserved */
+	[MAX_PACKET_LEN]				= GENMASK(31, 16),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070);
+
+static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = {
+	[ENDP_RSRC_GRP]					= GENMASK(1, 0),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp,
+		      0x00000838, 0x0070);
+
+static const u32 ipa_reg_endp_init_seq_fmask[] = {
+	[SEQ_TYPE]					= GENMASK(7, 0),
+						/* Bits 8-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070);
+
+static const u32 ipa_reg_endp_status_fmask[] = {
+	[STATUS_EN]					= BIT(0),
+	[STATUS_ENDP]					= GENMASK(5, 1),
+						/* Bits 6-8 reserved */
+	[STATUS_PKT_SUPPRESS]				= BIT(9),
+						/* Bits 10-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070);
+
+static const u32 ipa_reg_endp_filter_router_hsh_cfg_fmask[] = {
+	[FILTER_HASH_MSK_SRC_ID]			= BIT(0),
+	[FILTER_HASH_MSK_SRC_IP]			= BIT(1),
+	[FILTER_HASH_MSK_DST_IP]			= BIT(2),
+	[FILTER_HASH_MSK_SRC_PORT]			= BIT(3),
+	[FILTER_HASH_MSK_DST_PORT]			= BIT(4),
+	[FILTER_HASH_MSK_PROTOCOL]			= BIT(5),
+	[FILTER_HASH_MSK_METADATA]			= BIT(6),
+	[FILTER_HASH_MSK_ALL]				= GENMASK(6, 0),
+						/* Bits 7-15 reserved */
+	[ROUTER_HASH_MSK_SRC_ID]			= BIT(16),
+	[ROUTER_HASH_MSK_SRC_IP]			= BIT(17),
+	[ROUTER_HASH_MSK_DST_IP]			= BIT(18),
+	[ROUTER_HASH_MSK_SRC_PORT]			= BIT(19),
+	[ROUTER_HASH_MSK_DST_PORT]			= BIT(20),
+	[ROUTER_HASH_MSK_PROTOCOL]			= BIT(21),
+	[ROUTER_HASH_MSK_METADATA]			= BIT(22),
+	[ROUTER_HASH_MSK_ALL]				= GENMASK(22, 16),
+						/* Bits 23-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_FILTER_ROUTER_HSH_CFG, endp_filter_router_hsh_cfg,
+		      0x0000085c, 0x0070);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00004008 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000400c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00004010 + 0x1000 * GSI_EE_AP);
+
+static const u32 ipa_reg_ipa_irq_uc_fmask[] = {
+	[UC_INTR]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000401c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00004030 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00004034 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00004038 + 0x1000 * GSI_EE_AP);
+
+static const struct ipa_reg *ipa_reg_array[] = {
+	[COMP_CFG]			= &ipa_reg_comp_cfg,
+	[CLKON_CFG]			= &ipa_reg_clkon_cfg,
+	[ROUTE]				= &ipa_reg_route,
+	[SHARED_MEM_SIZE]		= &ipa_reg_shared_mem_size,
+	[QSB_MAX_WRITES]		= &ipa_reg_qsb_max_writes,
+	[QSB_MAX_READS]			= &ipa_reg_qsb_max_reads,
+	[FILT_ROUT_HASH_EN]		= &ipa_reg_filt_rout_hash_en,
+	[FILT_ROUT_HASH_FLUSH]		= &ipa_reg_filt_rout_hash_flush,
+	[STATE_AGGR_ACTIVE]		= &ipa_reg_state_aggr_active,
+	[LOCAL_PKT_PROC_CNTXT]		= &ipa_reg_local_pkt_proc_cntxt,
+	[AGGR_FORCE_CLOSE]		= &ipa_reg_aggr_force_close,
+	[IPA_TX_CFG]			= &ipa_reg_ipa_tx_cfg,
+	[FLAVOR_0]			= &ipa_reg_flavor_0,
+	[IDLE_INDICATION_CFG]		= &ipa_reg_idle_indication_cfg,
+	[QTIME_TIMESTAMP_CFG]		= &ipa_reg_qtime_timestamp_cfg,
+	[TIMERS_XO_CLK_DIV_CFG]		= &ipa_reg_timers_xo_clk_div_cfg,
+	[TIMERS_PULSE_GRAN_CFG]		= &ipa_reg_timers_pulse_gran_cfg,
+	[SRC_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_01_rsrc_type,
+	[SRC_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_23_rsrc_type,
+	[DST_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_01_rsrc_type,
+	[DST_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_23_rsrc_type,
+	[ENDP_INIT_CFG]			= &ipa_reg_endp_init_cfg,
+	[ENDP_INIT_NAT]			= &ipa_reg_endp_init_nat,
+	[ENDP_INIT_HDR]			= &ipa_reg_endp_init_hdr,
+	[ENDP_INIT_HDR_EXT]		= &ipa_reg_endp_init_hdr_ext,
+	[ENDP_INIT_HDR_METADATA_MASK]	= &ipa_reg_endp_init_hdr_metadata_mask,
+	[ENDP_INIT_MODE]		= &ipa_reg_endp_init_mode,
+	[ENDP_INIT_AGGR]		= &ipa_reg_endp_init_aggr,
+	[ENDP_INIT_HOL_BLOCK_EN]	= &ipa_reg_endp_init_hol_block_en,
+	[ENDP_INIT_HOL_BLOCK_TIMER]	= &ipa_reg_endp_init_hol_block_timer,
+	[ENDP_INIT_DEAGGR]		= &ipa_reg_endp_init_deaggr,
+	[ENDP_INIT_RSRC_GRP]		= &ipa_reg_endp_init_rsrc_grp,
+	[ENDP_INIT_SEQ]			= &ipa_reg_endp_init_seq,
+	[ENDP_STATUS]			= &ipa_reg_endp_status,
+	[ENDP_FILTER_ROUTER_HSH_CFG]	= &ipa_reg_endp_filter_router_hsh_cfg,
+	[IPA_IRQ_STTS]			= &ipa_reg_ipa_irq_stts,
+	[IPA_IRQ_EN]			= &ipa_reg_ipa_irq_en,
+	[IPA_IRQ_CLR]			= &ipa_reg_ipa_irq_clr,
+	[IPA_IRQ_UC]			= &ipa_reg_ipa_irq_uc,
+	[IRQ_SUSPEND_INFO]		= &ipa_reg_irq_suspend_info,
+	[IRQ_SUSPEND_EN]		= &ipa_reg_irq_suspend_en,
+	[IRQ_SUSPEND_CLR]		= &ipa_reg_irq_suspend_clr,
+};
+
+const struct ipa_regs ipa_regs_v4_11 = {
+	.reg_count	= ARRAY_SIZE(ipa_reg_array),
+	.reg		= ipa_reg_array,
+};
diff --git a/drivers/net/ipa/reg/ipa_reg-v4.2.c b/drivers/net/ipa/reg/ipa_reg-v4.2.c
new file mode 100644
index 000000000000..f8e78e1907c8
--- /dev/null
+++ b/drivers/net/ipa/reg/ipa_reg-v4.2.c
@@ -0,0 +1,456 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* Copyright (C) 2022 Linaro Ltd. */
+
+#include <linux/types.h>
+
+#include "../ipa.h"
+#include "../ipa_reg.h"
+
+static const u32 ipa_reg_comp_cfg_fmask[] = {
+						/* Bit 0 reserved */
+	[GSI_SNOC_BYPASS_DIS]				= BIT(1),
+	[GEN_QMB_0_SNOC_BYPASS_DIS]			= BIT(2),
+	[GEN_QMB_1_SNOC_BYPASS_DIS]			= BIT(3),
+	[IPA_DCMP_FAST_CLK_EN]				= BIT(4),
+	[IPA_QMB_SELECT_CONS_EN]			= BIT(5),
+	[IPA_QMB_SELECT_PROD_EN]			= BIT(6),
+	[GSI_MULTI_INORDER_RD_DIS]			= BIT(7),
+	[GSI_MULTI_INORDER_WR_DIS]			= BIT(8),
+	[GEN_QMB_0_MULTI_INORDER_RD_DIS]		= BIT(9),
+	[GEN_QMB_1_MULTI_INORDER_RD_DIS]		= BIT(10),
+	[GEN_QMB_0_MULTI_INORDER_WR_DIS]		= BIT(11),
+	[GEN_QMB_1_MULTI_INORDER_WR_DIS]		= BIT(12),
+	[GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS]		= BIT(13),
+	[GSI_SNOC_CNOC_LOOP_PROT_DISABLE]		= BIT(14),
+	[GSI_MULTI_AXI_MASTERS_DIS]			= BIT(15),
+	[IPA_QMB_SELECT_GLOBAL_EN]			= BIT(16),
+	[ATOMIC_FETCHER_ARB_LOCK_DIS]			= GENMASK(20, 17),
+						/* Bits 21-31 reserved */
+};
+
+IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c);
+
+static const u32 ipa_reg_clkon_cfg_fmask[] = {
+	[CLKON_RX]					= BIT(0),
+	[CLKON_PROC]					= BIT(1),
+	[TX_WRAPPER]					= BIT(2),
+	[CLKON_MISC]					= BIT(3),
+	[RAM_ARB]					= BIT(4),
+	[FTCH_HPS]					= BIT(5),
+	[FTCH_DPS]					= BIT(6),
+	[CLKON_HPS]					= BIT(7),
+	[CLKON_DPS]					= BIT(8),
+	[RX_HPS_CMDQS]					= BIT(9),
+	[HPS_DPS_CMDQS]					= BIT(10),
+	[DPS_TX_CMDQS]					= BIT(11),
+	[RSRC_MNGR]					= BIT(12),
+	[CTX_HANDLER]					= BIT(13),
+	[ACK_MNGR]					= BIT(14),
+	[D_DCPH]					= BIT(15),
+	[H_DCPH]					= BIT(16),
+						/* Bit 17 reserved */
+	[NTF_TX_CMDQS]					= BIT(18),
+	[CLKON_TX_0]					= BIT(19),
+	[CLKON_TX_1]					= BIT(20),
+	[CLKON_FNR]					= BIT(21),
+	[QSB2AXI_CMDQ_L]				= BIT(22),
+	[AGGR_WRAPPER]					= BIT(23),
+	[RAM_SLAVEWAY]					= BIT(24),
+	[CLKON_QMB]					= BIT(25),
+	[WEIGHT_ARB]					= BIT(26),
+	[GSI_IF]					= BIT(27),
+	[CLKON_GLOBAL]					= BIT(28),
+	[GLOBAL_2X_CLK]					= BIT(29),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044);
+
+static const u32 ipa_reg_route_fmask[] = {
+	[ROUTE_DIS]					= BIT(0),
+	[ROUTE_DEF_PIPE]				= GENMASK(5, 1),
+	[ROUTE_DEF_HDR_TABLE]				= BIT(6),
+	[ROUTE_DEF_HDR_OFST]				= GENMASK(16, 7),
+	[ROUTE_FRAG_DEF_PIPE]				= GENMASK(21, 17),
+						/* Bits 22-23 reserved */
+	[ROUTE_DEF_RETAIN_HDR]				= BIT(24),
+						/* Bits 25-31 reserved */
+};
+
+IPA_REG_FIELDS(ROUTE, route, 0x00000048);
+
+static const u32 ipa_reg_shared_mem_size_fmask[] = {
+	[MEM_SIZE]					= GENMASK(15, 0),
+	[MEM_BADDR]					= GENMASK(31, 16),
+};
+
+IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054);
+
+static const u32 ipa_reg_qsb_max_writes_fmask[] = {
+	[GEN_QMB_0_MAX_WRITES]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_WRITES]				= GENMASK(7, 4),
+						/* Bits 8-31 reserved */
+};
+
+IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074);
+
+static const u32 ipa_reg_qsb_max_reads_fmask[] = {
+	[GEN_QMB_0_MAX_READS]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_READS]				= GENMASK(7, 4),
+						/* Bits 8-15 reserved */
+	[GEN_QMB_0_MAX_READS_BEATS]			= GENMASK(23, 16),
+	[GEN_QMB_1_MAX_READS_BEATS]			= GENMASK(31, 24),
+};
+
+IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078);
+
+static const u32 ipa_reg_filt_rout_hash_en_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x0000148);
+
+static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x000014c);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x000000b4);
+
+IPA_REG(IPA_BCR, ipa_bcr, 0x000001d0);
+
+static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = {
+	[IPA_BASE_ADDR]					= GENMASK(16, 0),
+						/* Bits 17-31 reserved */
+};
+
+/* Offset must be a multiple of 8 */
+IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec);
+
+static const u32 ipa_reg_counter_cfg_fmask[] = {
+						/* Bits 0-3 reserved */
+	[AGGR_GRANULARITY]				= GENMASK(8, 4),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_FIELDS(COUNTER_CFG, counter_cfg, 0x000001f0);
+
+static const u32 ipa_reg_ipa_tx_cfg_fmask[] = {
+						/* Bits 0-1 reserved */
+	[PREFETCH_ALMOST_EMPTY_SIZE_TX0]		= GENMASK(5, 2),
+	[DMAW_SCND_OUTSD_PRED_THRESHOLD]		= GENMASK(9, 6),
+	[DMAW_SCND_OUTSD_PRED_EN]			= BIT(10),
+	[DMAW_MAX_BEATS_256_DIS]			= BIT(11),
+	[PA_MASK_EN]					= BIT(12),
+	[PREFETCH_ALMOST_EMPTY_SIZE_TX1]		= GENMASK(16, 13),
+						/* Bit 17 reserved */
+	[SSPND_PA_NO_START_STATE]			= BIT(18),
+	[SSPND_PA_NO_BQ_STATE]				= BIT(19),
+						/* Bits 20-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x000001fc);
+
+static const u32 ipa_reg_flavor_0_fmask[] = {
+	[MAX_PIPES]					= GENMASK(3, 0),
+						/* Bits 4-7 reserved */
+	[MAX_CONS_PIPES]				= GENMASK(12, 8),
+						/* Bits 13-15 reserved */
+	[MAX_PROD_PIPES]				= GENMASK(20, 16),
+						/* Bits 21-23 reserved */
+	[PROD_LOWEST]					= GENMASK(27, 24),
+						/* Bits 28-31 reserved */
+};
+
+IPA_REG_FIELDS(FLAVOR_0, flavor_0, 0x00000210);
+
+static const u32 ipa_reg_idle_indication_cfg_fmask[] = {
+	[ENTER_IDLE_DEBOUNCE_THRESH]			= GENMASK(15, 0),
+	[CONST_NON_IDLE_ENABLE]				= BIT(16),
+						/* Bits 17-31 reserved */
+};
+
+IPA_REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x00000240);
+
+static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type,
+		      0x00000400, 0x0020);
+
+static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type,
+		      0x00000404, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type,
+		      0x00000500, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type,
+		      0x00000504, 0x0020);
+
+static const u32 ipa_reg_endp_init_cfg_fmask[] = {
+	[FRAG_OFFLOAD_EN]				= BIT(0),
+	[CS_OFFLOAD_EN]					= GENMASK(2, 1),
+	[CS_METADATA_HDR_OFFSET]			= GENMASK(6, 3),
+						/* Bit 7 reserved */
+	[CS_GEN_QMB_MASTER_SEL]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070);
+
+static const u32 ipa_reg_endp_init_nat_fmask[] = {
+	[NAT_EN]					= GENMASK(1, 0),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_fmask[] = {
+	[HDR_LEN]					= GENMASK(5, 0),
+	[HDR_OFST_METADATA_VALID]			= BIT(6),
+	[HDR_OFST_METADATA]				= GENMASK(12, 7),
+	[HDR_ADDITIONAL_CONST_LEN]			= GENMASK(18, 13),
+	[HDR_OFST_PKT_SIZE_VALID]			= BIT(19),
+	[HDR_OFST_PKT_SIZE]				= GENMASK(25, 20),
+	[HDR_A5_MUX]					= BIT(26),
+	[HDR_LEN_INC_DEAGG_HDR]				= BIT(27),
+	[HDR_METADATA_REG_VALID]			= BIT(28),
+						/* Bits 29-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = {
+	[HDR_ENDIANNESS]				= BIT(0),
+	[HDR_TOTAL_LEN_OR_PAD_VALID]			= BIT(1),
+	[HDR_TOTAL_LEN_OR_PAD]				= BIT(2),
+	[HDR_PAYLOAD_LEN_INC_PADDING]			= BIT(3),
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET]			= GENMASK(9, 4),
+	[HDR_PAD_TO_ALIGNMENT]				= GENMASK(13, 10),
+						/* Bits 14-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070);
+
+IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask,
+	       0x00000818, 0x0070);
+
+static const u32 ipa_reg_endp_init_mode_fmask[] = {
+	[ENDP_MODE]					= GENMASK(2, 0),
+						/* Bit 3 reserved */
+	[DEST_PIPE_INDEX]				= GENMASK(8, 4),
+						/* Bits 9-11 reserved */
+	[BYTE_THRESHOLD]				= GENMASK(27, 12),
+	[PIPE_REPLICATION_EN]				= BIT(28),
+	[PAD_EN]					= BIT(29),
+	[HDR_FTCH_DISABLE]				= BIT(30),
+						/* Bit 31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070);
+
+static const u32 ipa_reg_endp_init_aggr_fmask[] = {
+	[AGGR_EN]					= GENMASK(1, 0),
+	[AGGR_TYPE]					= GENMASK(4, 2),
+	[BYTE_LIMIT]					= GENMASK(9, 5),
+	[TIME_LIMIT]					= GENMASK(14, 10),
+	[PKT_LIMIT]					= GENMASK(20, 15),
+	[SW_EOF_ACTIVE]					= BIT(21),
+	[FORCE_CLOSE]					= BIT(22),
+						/* Bit 23 reserved */
+	[HARD_BYTE_LIMIT_EN]				= BIT(24),
+						/* Bits 25-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070);
+
+static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = {
+	[HOL_BLOCK_EN]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en,
+		      0x0000082c, 0x0070);
+
+static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = {
+	[TIMER_BASE_VALUE]				= GENMASK(4, 0),
+						/* Bits 5-7 reserved */
+	[TIMER_SCALE]					= GENMASK(12, 8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer,
+		      0x00000830, 0x0070);
+
+static const u32 ipa_reg_endp_init_deaggr_fmask[] = {
+	[DEAGGR_HDR_LEN]				= GENMASK(5, 0),
+	[SYSPIPE_ERR_DETECTION]				= BIT(6),
+	[PACKET_OFFSET_VALID]				= BIT(7),
+	[PACKET_OFFSET_LOCATION]			= GENMASK(13, 8),
+	[IGNORE_MIN_PKT_ERR]				= BIT(14),
+						/* Bit 15 reserved */
+	[MAX_PACKET_LEN]				= GENMASK(31, 16),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070);
+
+static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = {
+	[ENDP_RSRC_GRP]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp,
+		      0x00000838, 0x0070);
+
+static const u32 ipa_reg_endp_init_seq_fmask[] = {
+	[SEQ_TYPE]					= GENMASK(7, 0),
+	[SEQ_REP_TYPE]					= GENMASK(15, 8),
+						/* Bits 16-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070);
+
+static const u32 ipa_reg_endp_status_fmask[] = {
+	[STATUS_EN]					= BIT(0),
+	[STATUS_ENDP]					= GENMASK(5, 1),
+						/* Bits 6-7 reserved */
+	[STATUS_LOCATION]				= BIT(8),
+	[STATUS_PKT_SUPPRESS]				= BIT(9),
+						/* Bits 10-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00003008 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000300c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00003010 + 0x1000 * GSI_EE_AP);
+
+static const u32 ipa_reg_ipa_irq_uc_fmask[] = {
+	[UC_INTR]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000301c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00003030 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00003034 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00003038 + 0x1000 * GSI_EE_AP);
+
+static const struct ipa_reg *ipa_reg_array[] = {
+	[COMP_CFG]			= &ipa_reg_comp_cfg,
+	[CLKON_CFG]			= &ipa_reg_clkon_cfg,
+	[ROUTE]				= &ipa_reg_route,
+	[SHARED_MEM_SIZE]		= &ipa_reg_shared_mem_size,
+	[QSB_MAX_WRITES]		= &ipa_reg_qsb_max_writes,
+	[QSB_MAX_READS]			= &ipa_reg_qsb_max_reads,
+	[FILT_ROUT_HASH_EN]		= &ipa_reg_filt_rout_hash_en,
+	[FILT_ROUT_HASH_FLUSH]		= &ipa_reg_filt_rout_hash_flush,
+	[STATE_AGGR_ACTIVE]		= &ipa_reg_state_aggr_active,
+	[IPA_BCR]			= &ipa_reg_ipa_bcr,
+	[LOCAL_PKT_PROC_CNTXT]		= &ipa_reg_local_pkt_proc_cntxt,
+	[AGGR_FORCE_CLOSE]		= &ipa_reg_aggr_force_close,
+	[COUNTER_CFG]			= &ipa_reg_counter_cfg,
+	[IPA_TX_CFG]			= &ipa_reg_ipa_tx_cfg,
+	[FLAVOR_0]			= &ipa_reg_flavor_0,
+	[IDLE_INDICATION_CFG]		= &ipa_reg_idle_indication_cfg,
+	[SRC_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_01_rsrc_type,
+	[SRC_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_23_rsrc_type,
+	[DST_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_01_rsrc_type,
+	[DST_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_23_rsrc_type,
+	[ENDP_INIT_CFG]			= &ipa_reg_endp_init_cfg,
+	[ENDP_INIT_NAT]			= &ipa_reg_endp_init_nat,
+	[ENDP_INIT_HDR]			= &ipa_reg_endp_init_hdr,
+	[ENDP_INIT_HDR_EXT]		= &ipa_reg_endp_init_hdr_ext,
+	[ENDP_INIT_HDR_METADATA_MASK]	= &ipa_reg_endp_init_hdr_metadata_mask,
+	[ENDP_INIT_MODE]		= &ipa_reg_endp_init_mode,
+	[ENDP_INIT_AGGR]		= &ipa_reg_endp_init_aggr,
+	[ENDP_INIT_HOL_BLOCK_EN]	= &ipa_reg_endp_init_hol_block_en,
+	[ENDP_INIT_HOL_BLOCK_TIMER]	= &ipa_reg_endp_init_hol_block_timer,
+	[ENDP_INIT_DEAGGR]		= &ipa_reg_endp_init_deaggr,
+	[ENDP_INIT_RSRC_GRP]		= &ipa_reg_endp_init_rsrc_grp,
+	[ENDP_INIT_SEQ]			= &ipa_reg_endp_init_seq,
+	[ENDP_STATUS]			= &ipa_reg_endp_status,
+	[IPA_IRQ_STTS]			= &ipa_reg_ipa_irq_stts,
+	[IPA_IRQ_EN]			= &ipa_reg_ipa_irq_en,
+	[IPA_IRQ_CLR]			= &ipa_reg_ipa_irq_clr,
+	[IPA_IRQ_UC]			= &ipa_reg_ipa_irq_uc,
+	[IRQ_SUSPEND_INFO]		= &ipa_reg_irq_suspend_info,
+	[IRQ_SUSPEND_EN]		= &ipa_reg_irq_suspend_en,
+	[IRQ_SUSPEND_CLR]		= &ipa_reg_irq_suspend_clr,
+};
+
+const struct ipa_regs ipa_regs_v4_2 = {
+	.reg_count	= ARRAY_SIZE(ipa_reg_array),
+	.reg		= ipa_reg_array,
+};
diff --git a/drivers/net/ipa/reg/ipa_reg-v4.5.c b/drivers/net/ipa/reg/ipa_reg-v4.5.c
new file mode 100644
index 000000000000..d32b805abb11
--- /dev/null
+++ b/drivers/net/ipa/reg/ipa_reg-v4.5.c
@@ -0,0 +1,533 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* Copyright (C) 2022 Linaro Ltd. */
+
+#include <linux/types.h>
+
+#include "../ipa.h"
+#include "../ipa_reg.h"
+
+static const u32 ipa_reg_comp_cfg_fmask[] = {
+						/* Bit 0 reserved */
+	[GSI_SNOC_BYPASS_DIS]				= BIT(1),
+	[GEN_QMB_0_SNOC_BYPASS_DIS]			= BIT(2),
+	[GEN_QMB_1_SNOC_BYPASS_DIS]			= BIT(3),
+						/* Bit 4 reserved */
+	[IPA_QMB_SELECT_CONS_EN]			= BIT(5),
+	[IPA_QMB_SELECT_PROD_EN]			= BIT(6),
+	[GSI_MULTI_INORDER_RD_DIS]			= BIT(7),
+	[GSI_MULTI_INORDER_WR_DIS]			= BIT(8),
+	[GEN_QMB_0_MULTI_INORDER_RD_DIS]		= BIT(9),
+	[GEN_QMB_1_MULTI_INORDER_RD_DIS]		= BIT(10),
+	[GEN_QMB_0_MULTI_INORDER_WR_DIS]		= BIT(11),
+	[GEN_QMB_1_MULTI_INORDER_WR_DIS]		= BIT(12),
+	[GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS]		= BIT(13),
+	[GSI_SNOC_CNOC_LOOP_PROT_DISABLE]		= BIT(14),
+	[GSI_MULTI_AXI_MASTERS_DIS]			= BIT(15),
+	[IPA_QMB_SELECT_GLOBAL_EN]			= BIT(16),
+	[ATOMIC_FETCHER_ARB_LOCK_DIS]			= GENMASK(20, 17),
+	[FULL_FLUSH_WAIT_RS_CLOSURE_EN]			= BIT(21),
+						/* Bits 22-31 reserved */
+};
+
+IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c);
+
+static const u32 ipa_reg_clkon_cfg_fmask[] = {
+	[CLKON_RX]					= BIT(0),
+	[CLKON_PROC]					= BIT(1),
+	[TX_WRAPPER]					= BIT(2),
+	[CLKON_MISC]					= BIT(3),
+	[RAM_ARB]					= BIT(4),
+	[FTCH_HPS]					= BIT(5),
+	[FTCH_DPS]					= BIT(6),
+	[CLKON_HPS]					= BIT(7),
+	[CLKON_DPS]					= BIT(8),
+	[RX_HPS_CMDQS]					= BIT(9),
+	[HPS_DPS_CMDQS]					= BIT(10),
+	[DPS_TX_CMDQS]					= BIT(11),
+	[RSRC_MNGR]					= BIT(12),
+	[CTX_HANDLER]					= BIT(13),
+	[ACK_MNGR]					= BIT(14),
+	[D_DCPH]					= BIT(15),
+	[H_DCPH]					= BIT(16),
+	[CLKON_DCMP]					= BIT(17),
+	[NTF_TX_CMDQS]					= BIT(18),
+	[CLKON_TX_0]					= BIT(19),
+	[CLKON_TX_1]					= BIT(20),
+	[CLKON_FNR]					= BIT(21),
+	[QSB2AXI_CMDQ_L]				= BIT(22),
+	[AGGR_WRAPPER]					= BIT(23),
+	[RAM_SLAVEWAY]					= BIT(24),
+	[CLKON_QMB]					= BIT(25),
+	[WEIGHT_ARB]					= BIT(26),
+	[GSI_IF]					= BIT(27),
+	[CLKON_GLOBAL]					= BIT(28),
+	[GLOBAL_2X_CLK]					= BIT(29),
+	[DPL_FIFO]					= BIT(30),
+						/* Bit 31 reserved */
+};
+
+IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044);
+
+static const u32 ipa_reg_route_fmask[] = {
+	[ROUTE_DIS]					= BIT(0),
+	[ROUTE_DEF_PIPE]				= GENMASK(5, 1),
+	[ROUTE_DEF_HDR_TABLE]				= BIT(6),
+	[ROUTE_DEF_HDR_OFST]				= GENMASK(16, 7),
+	[ROUTE_FRAG_DEF_PIPE]				= GENMASK(21, 17),
+						/* Bits 22-23 reserved */
+	[ROUTE_DEF_RETAIN_HDR]				= BIT(24),
+						/* Bits 25-31 reserved */
+};
+
+IPA_REG_FIELDS(ROUTE, route, 0x00000048);
+
+static const u32 ipa_reg_shared_mem_size_fmask[] = {
+	[MEM_SIZE]					= GENMASK(15, 0),
+	[MEM_BADDR]					= GENMASK(31, 16),
+};
+
+IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054);
+
+static const u32 ipa_reg_qsb_max_writes_fmask[] = {
+	[GEN_QMB_0_MAX_WRITES]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_WRITES]				= GENMASK(7, 4),
+						/* Bits 8-31 reserved */
+};
+
+IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074);
+
+static const u32 ipa_reg_qsb_max_reads_fmask[] = {
+	[GEN_QMB_0_MAX_READS]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_READS]				= GENMASK(7, 4),
+						/* Bits 8-15 reserved */
+	[GEN_QMB_0_MAX_READS_BEATS]			= GENMASK(23, 16),
+	[GEN_QMB_1_MAX_READS_BEATS]			= GENMASK(31, 24),
+};
+
+IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078);
+
+static const u32 ipa_reg_filt_rout_hash_en_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x0000148);
+
+static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x000014c);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x000000b4);
+
+static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = {
+	[IPA_BASE_ADDR]					= GENMASK(17, 0),
+						/* Bits 18-31 reserved */
+};
+
+/* Offset must be a multiple of 8 */
+IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec);
+
+static const u32 ipa_reg_ipa_tx_cfg_fmask[] = {
+						/* Bits 0-1 reserved */
+	[PREFETCH_ALMOST_EMPTY_SIZE_TX0]		= GENMASK(5, 2),
+	[DMAW_SCND_OUTSD_PRED_THRESHOLD]		= GENMASK(9, 6),
+	[DMAW_SCND_OUTSD_PRED_EN]			= BIT(10),
+	[DMAW_MAX_BEATS_256_DIS]			= BIT(11),
+	[PA_MASK_EN]					= BIT(12),
+	[PREFETCH_ALMOST_EMPTY_SIZE_TX1]		= GENMASK(16, 13),
+	[DUAL_TX_ENABLE]				= BIT(17),
+						/* Bits 18-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x000001fc);
+
+static const u32 ipa_reg_flavor_0_fmask[] = {
+	[MAX_PIPES]					= GENMASK(3, 0),
+						/* Bits 4-7 reserved */
+	[MAX_CONS_PIPES]				= GENMASK(12, 8),
+						/* Bits 13-15 reserved */
+	[MAX_PROD_PIPES]				= GENMASK(20, 16),
+						/* Bits 21-23 reserved */
+	[PROD_LOWEST]					= GENMASK(27, 24),
+						/* Bits 28-31 reserved */
+};
+
+IPA_REG_FIELDS(FLAVOR_0, flavor_0, 0x00000210);
+
+static const u32 ipa_reg_idle_indication_cfg_fmask[] = {
+	[ENTER_IDLE_DEBOUNCE_THRESH]			= GENMASK(15, 0),
+	[CONST_NON_IDLE_ENABLE]				= BIT(16),
+						/* Bits 17-31 reserved */
+};
+
+IPA_REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x00000240);
+
+static const u32 ipa_reg_qtime_timestamp_cfg_fmask[] = {
+	[DPL_TIMESTAMP_LSB]				= GENMASK(4, 0),
+						/* Bits 5-6 reserved */
+	[DPL_TIMESTAMP_SEL]				= BIT(7),
+	[TAG_TIMESTAMP_LSB]				= GENMASK(12, 8),
+						/* Bits 13-15 reserved */
+	[NAT_TIMESTAMP_LSB]				= GENMASK(20, 16),
+						/* Bits 21-31 reserved */
+};
+
+IPA_REG_FIELDS(QTIME_TIMESTAMP_CFG, qtime_timestamp_cfg, 0x0000024c);
+
+static const u32 ipa_reg_timers_xo_clk_div_cfg_fmask[] = {
+	[DIV_VALUE]					= GENMASK(8, 0),
+						/* Bits 9-30 reserved */
+	[DIV_ENABLE]					= BIT(31),
+};
+
+IPA_REG_FIELDS(TIMERS_XO_CLK_DIV_CFG, timers_xo_clk_div_cfg, 0x00000250);
+
+static const u32 ipa_reg_timers_pulse_gran_cfg_fmask[] = {
+	[PULSE_GRAN_0]					= GENMASK(2, 0),
+	[PULSE_GRAN_1]					= GENMASK(5, 3),
+	[PULSE_GRAN_2]					= GENMASK(8, 6),
+};
+
+IPA_REG_FIELDS(TIMERS_PULSE_GRAN_CFG, timers_pulse_gran_cfg, 0x00000254);
+
+static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type,
+		      0x00000400, 0x0020);
+
+static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type,
+		      0x00000404, 0x0020);
+
+static const u32 ipa_reg_src_rsrc_grp_45_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_45_RSRC_TYPE, src_rsrc_grp_45_rsrc_type,
+		      0x00000408, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type,
+		      0x00000500, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type,
+		      0x00000504, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_45_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_45_RSRC_TYPE, dst_rsrc_grp_45_rsrc_type,
+		      0x00000508, 0x0020);
+
+static const u32 ipa_reg_endp_init_cfg_fmask[] = {
+	[FRAG_OFFLOAD_EN]				= BIT(0),
+	[CS_OFFLOAD_EN]					= GENMASK(2, 1),
+	[CS_METADATA_HDR_OFFSET]			= GENMASK(6, 3),
+						/* Bit 7 reserved */
+	[CS_GEN_QMB_MASTER_SEL]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070);
+
+static const u32 ipa_reg_endp_init_nat_fmask[] = {
+	[NAT_EN]					= GENMASK(1, 0),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_fmask[] = {
+	[HDR_LEN]					= GENMASK(5, 0),
+	[HDR_OFST_METADATA_VALID]			= BIT(6),
+	[HDR_OFST_METADATA]				= GENMASK(12, 7),
+	[HDR_ADDITIONAL_CONST_LEN]			= GENMASK(18, 13),
+	[HDR_OFST_PKT_SIZE_VALID]			= BIT(19),
+	[HDR_OFST_PKT_SIZE]				= GENMASK(25, 20),
+	[HDR_A5_MUX]					= BIT(26),
+	[HDR_LEN_INC_DEAGG_HDR]				= BIT(27),
+	[HDR_LEN_MSB]					= GENMASK(29, 28),
+	[HDR_OFST_METADATA_MSB]				= GENMASK(31, 30),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = {
+	[HDR_ENDIANNESS]				= BIT(0),
+	[HDR_TOTAL_LEN_OR_PAD_VALID]			= BIT(1),
+	[HDR_TOTAL_LEN_OR_PAD]				= BIT(2),
+	[HDR_PAYLOAD_LEN_INC_PADDING]			= BIT(3),
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET]			= GENMASK(9, 4),
+	[HDR_PAD_TO_ALIGNMENT]				= GENMASK(13, 10),
+						/* Bits 14-15 reserved */
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB]		= GENMASK(17, 16),
+	[HDR_OFST_PKT_SIZE_MSB]				= GENMASK(19, 18),
+	[HDR_ADDITIONAL_CONST_LEN_MSB]			= GENMASK(21, 20),
+						/* Bits 22-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070);
+
+IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask,
+	       0x00000818, 0x0070);
+
+static const u32 ipa_reg_endp_init_mode_fmask[] = {
+	[ENDP_MODE]					= GENMASK(2, 0),
+	[DCPH_ENABLE]					= BIT(3),
+	[DEST_PIPE_INDEX]				= GENMASK(8, 4),
+						/* Bits 9-11 reserved */
+	[BYTE_THRESHOLD]				= GENMASK(27, 12),
+	[PIPE_REPLICATION_EN]				= BIT(28),
+	[PAD_EN]					= BIT(29),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070);
+
+static const u32 ipa_reg_endp_init_aggr_fmask[] = {
+	[AGGR_EN]					= GENMASK(1, 0),
+	[AGGR_TYPE]					= GENMASK(4, 2),
+	[BYTE_LIMIT]					= GENMASK(10, 5),
+						/* Bit 11 reserved */
+	[TIME_LIMIT]					= GENMASK(16, 12),
+	[PKT_LIMIT]					= GENMASK(22, 17),
+	[SW_EOF_ACTIVE]					= BIT(23),
+	[FORCE_CLOSE]					= BIT(24),
+						/* Bit 25 reserved */
+	[HARD_BYTE_LIMIT_EN]				= BIT(26),
+	[AGGR_GRAN_SEL]					= BIT(27),
+						/* Bits 28-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070);
+
+static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = {
+	[HOL_BLOCK_EN]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en,
+		      0x0000082c, 0x0070);
+
+static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = {
+	[TIMER_LIMIT]					= GENMASK(4, 0),
+						/* Bits 5-7 reserved */
+	[TIMER_GRAN_SEL]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer,
+		      0x00000830, 0x0070);
+
+static const u32 ipa_reg_endp_init_deaggr_fmask[] = {
+	[DEAGGR_HDR_LEN]				= GENMASK(5, 0),
+	[SYSPIPE_ERR_DETECTION]				= BIT(6),
+	[PACKET_OFFSET_VALID]				= BIT(7),
+	[PACKET_OFFSET_LOCATION]			= GENMASK(13, 8),
+	[IGNORE_MIN_PKT_ERR]				= BIT(14),
+						/* Bit 15 reserved */
+	[MAX_PACKET_LEN]				= GENMASK(31, 16),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070);
+
+static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = {
+	[ENDP_RSRC_GRP]					= GENMASK(2, 0),
+						/* Bits 3-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp,
+		      0x00000838, 0x0070);
+
+static const u32 ipa_reg_endp_init_seq_fmask[] = {
+	[SEQ_TYPE]					= GENMASK(7, 0),
+						/* Bits 8-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070);
+
+static const u32 ipa_reg_endp_status_fmask[] = {
+	[STATUS_EN]					= BIT(0),
+	[STATUS_ENDP]					= GENMASK(5, 1),
+						/* Bits 6-8 reserved */
+	[STATUS_PKT_SUPPRESS]				= BIT(9),
+						/* Bits 10-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070);
+
+static const u32 ipa_reg_endp_filter_router_hsh_cfg_fmask[] = {
+	[FILTER_HASH_MSK_SRC_ID]			= BIT(0),
+	[FILTER_HASH_MSK_SRC_IP]			= BIT(1),
+	[FILTER_HASH_MSK_DST_IP]			= BIT(2),
+	[FILTER_HASH_MSK_SRC_PORT]			= BIT(3),
+	[FILTER_HASH_MSK_DST_PORT]			= BIT(4),
+	[FILTER_HASH_MSK_PROTOCOL]			= BIT(5),
+	[FILTER_HASH_MSK_METADATA]			= BIT(6),
+	[FILTER_HASH_MSK_ALL]				= GENMASK(6, 0),
+						/* Bits 7-15 reserved */
+	[ROUTER_HASH_MSK_SRC_ID]			= BIT(16),
+	[ROUTER_HASH_MSK_SRC_IP]			= BIT(17),
+	[ROUTER_HASH_MSK_DST_IP]			= BIT(18),
+	[ROUTER_HASH_MSK_SRC_PORT]			= BIT(19),
+	[ROUTER_HASH_MSK_DST_PORT]			= BIT(20),
+	[ROUTER_HASH_MSK_PROTOCOL]			= BIT(21),
+	[ROUTER_HASH_MSK_METADATA]			= BIT(22),
+	[ROUTER_HASH_MSK_ALL]				= GENMASK(22, 16),
+						/* Bits 23-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_FILTER_ROUTER_HSH_CFG, endp_filter_router_hsh_cfg,
+		      0x0000085c, 0x0070);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00003008 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000300c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00003010 + 0x1000 * GSI_EE_AP);
+
+static const u32 ipa_reg_ipa_irq_uc_fmask[] = {
+	[UC_INTR]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000301c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00003030 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00003034 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00003038 + 0x1000 * GSI_EE_AP);
+
+static const struct ipa_reg *ipa_reg_array[] = {
+	[COMP_CFG]			= &ipa_reg_comp_cfg,
+	[CLKON_CFG]			= &ipa_reg_clkon_cfg,
+	[ROUTE]				= &ipa_reg_route,
+	[SHARED_MEM_SIZE]		= &ipa_reg_shared_mem_size,
+	[QSB_MAX_WRITES]		= &ipa_reg_qsb_max_writes,
+	[QSB_MAX_READS]			= &ipa_reg_qsb_max_reads,
+	[FILT_ROUT_HASH_EN]		= &ipa_reg_filt_rout_hash_en,
+	[FILT_ROUT_HASH_FLUSH]		= &ipa_reg_filt_rout_hash_flush,
+	[STATE_AGGR_ACTIVE]		= &ipa_reg_state_aggr_active,
+	[LOCAL_PKT_PROC_CNTXT]		= &ipa_reg_local_pkt_proc_cntxt,
+	[AGGR_FORCE_CLOSE]		= &ipa_reg_aggr_force_close,
+	[IPA_TX_CFG]			= &ipa_reg_ipa_tx_cfg,
+	[FLAVOR_0]			= &ipa_reg_flavor_0,
+	[IDLE_INDICATION_CFG]		= &ipa_reg_idle_indication_cfg,
+	[QTIME_TIMESTAMP_CFG]		= &ipa_reg_qtime_timestamp_cfg,
+	[TIMERS_XO_CLK_DIV_CFG]		= &ipa_reg_timers_xo_clk_div_cfg,
+	[TIMERS_PULSE_GRAN_CFG]		= &ipa_reg_timers_pulse_gran_cfg,
+	[SRC_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_01_rsrc_type,
+	[SRC_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_23_rsrc_type,
+	[SRC_RSRC_GRP_45_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_45_rsrc_type,
+	[DST_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_01_rsrc_type,
+	[DST_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_23_rsrc_type,
+	[DST_RSRC_GRP_45_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_45_rsrc_type,
+	[ENDP_INIT_CFG]			= &ipa_reg_endp_init_cfg,
+	[ENDP_INIT_NAT]			= &ipa_reg_endp_init_nat,
+	[ENDP_INIT_HDR]			= &ipa_reg_endp_init_hdr,
+	[ENDP_INIT_HDR_EXT]		= &ipa_reg_endp_init_hdr_ext,
+	[ENDP_INIT_HDR_METADATA_MASK]	= &ipa_reg_endp_init_hdr_metadata_mask,
+	[ENDP_INIT_MODE]		= &ipa_reg_endp_init_mode,
+	[ENDP_INIT_AGGR]		= &ipa_reg_endp_init_aggr,
+	[ENDP_INIT_HOL_BLOCK_EN]	= &ipa_reg_endp_init_hol_block_en,
+	[ENDP_INIT_HOL_BLOCK_TIMER]	= &ipa_reg_endp_init_hol_block_timer,
+	[ENDP_INIT_DEAGGR]		= &ipa_reg_endp_init_deaggr,
+	[ENDP_INIT_RSRC_GRP]		= &ipa_reg_endp_init_rsrc_grp,
+	[ENDP_INIT_SEQ]			= &ipa_reg_endp_init_seq,
+	[ENDP_STATUS]			= &ipa_reg_endp_status,
+	[ENDP_FILTER_ROUTER_HSH_CFG]	= &ipa_reg_endp_filter_router_hsh_cfg,
+	[IPA_IRQ_STTS]			= &ipa_reg_ipa_irq_stts,
+	[IPA_IRQ_EN]			= &ipa_reg_ipa_irq_en,
+	[IPA_IRQ_CLR]			= &ipa_reg_ipa_irq_clr,
+	[IPA_IRQ_UC]			= &ipa_reg_ipa_irq_uc,
+	[IRQ_SUSPEND_INFO]		= &ipa_reg_irq_suspend_info,
+	[IRQ_SUSPEND_EN]		= &ipa_reg_irq_suspend_en,
+	[IRQ_SUSPEND_CLR]		= &ipa_reg_irq_suspend_clr,
+};
+
+const struct ipa_regs ipa_regs_v4_5 = {
+	.reg_count	= ARRAY_SIZE(ipa_reg_array),
+	.reg		= ipa_reg_array,
+};
diff --git a/drivers/net/ipa/reg/ipa_reg-v4.9.c b/drivers/net/ipa/reg/ipa_reg-v4.9.c
new file mode 100644
index 000000000000..eabbc5451937
--- /dev/null
+++ b/drivers/net/ipa/reg/ipa_reg-v4.9.c
@@ -0,0 +1,509 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* Copyright (C) 2022 Linaro Ltd. */
+
+#include <linux/types.h>
+
+#include "../ipa.h"
+#include "../ipa_reg.h"
+
+static const u32 ipa_reg_comp_cfg_fmask[] = {
+	[RAM_ARB_PRI_CLIENT_SAMP_FIX_DIS]		= BIT(0),
+	[GSI_SNOC_BYPASS_DIS]				= BIT(1),
+	[GEN_QMB_0_SNOC_BYPASS_DIS]			= BIT(2),
+	[GEN_QMB_1_SNOC_BYPASS_DIS]			= BIT(3),
+						/* Bit 4 reserved */
+	[IPA_QMB_SELECT_CONS_EN]			= BIT(5),
+	[IPA_QMB_SELECT_PROD_EN]			= BIT(6),
+	[GSI_MULTI_INORDER_RD_DIS]			= BIT(7),
+	[GSI_MULTI_INORDER_WR_DIS]			= BIT(8),
+	[GEN_QMB_0_MULTI_INORDER_RD_DIS]		= BIT(9),
+	[GEN_QMB_1_MULTI_INORDER_RD_DIS]		= BIT(10),
+	[GEN_QMB_0_MULTI_INORDER_WR_DIS]		= BIT(11),
+	[GEN_QMB_1_MULTI_INORDER_WR_DIS]		= BIT(12),
+	[GEN_QMB_0_SNOC_CNOC_LOOP_PROT_DIS]		= BIT(13),
+	[GSI_SNOC_CNOC_LOOP_PROT_DISABLE]		= BIT(14),
+	[GSI_MULTI_AXI_MASTERS_DIS]			= BIT(15),
+	[IPA_QMB_SELECT_GLOBAL_EN]			= BIT(16),
+	[FULL_FLUSH_WAIT_RS_CLOSURE_EN]			= BIT(17),
+	[QMB_RAM_RD_CACHE_DISABLE]			= BIT(19),
+	[GENQMB_AOOOWR]					= BIT(20),
+	[IF_OUT_OF_BUF_STOP_RESET_MASK_EN]		= BIT(21),
+	[ATOMIC_FETCHER_ARB_LOCK_DIS]			= GENMASK(24, 22),
+						/* Bits 25-29 reserved */
+	[GEN_QMB_1_DYNAMIC_ASIZE]			= BIT(30),
+	[GEN_QMB_0_DYNAMIC_ASIZE]			= BIT(31),
+};
+
+IPA_REG_FIELDS(COMP_CFG, comp_cfg, 0x0000003c);
+
+static const u32 ipa_reg_clkon_cfg_fmask[] = {
+	[CLKON_RX]					= BIT(0),
+	[CLKON_PROC]					= BIT(1),
+	[TX_WRAPPER]					= BIT(2),
+	[CLKON_MISC]					= BIT(3),
+	[RAM_ARB]					= BIT(4),
+	[FTCH_HPS]					= BIT(5),
+	[FTCH_DPS]					= BIT(6),
+	[CLKON_HPS]					= BIT(7),
+	[CLKON_DPS]					= BIT(8),
+	[RX_HPS_CMDQS]					= BIT(9),
+	[HPS_DPS_CMDQS]					= BIT(10),
+	[DPS_TX_CMDQS]					= BIT(11),
+	[RSRC_MNGR]					= BIT(12),
+	[CTX_HANDLER]					= BIT(13),
+	[ACK_MNGR]					= BIT(14),
+	[D_DCPH]					= BIT(15),
+	[H_DCPH]					= BIT(16),
+	[CLKON_DCMP]					= BIT(17),
+	[NTF_TX_CMDQS]					= BIT(18),
+	[CLKON_TX_0]					= BIT(19),
+	[CLKON_TX_1]					= BIT(20),
+	[CLKON_FNR]					= BIT(21),
+	[QSB2AXI_CMDQ_L]				= BIT(22),
+	[AGGR_WRAPPER]					= BIT(23),
+	[RAM_SLAVEWAY]					= BIT(24),
+	[CLKON_QMB]					= BIT(25),
+	[WEIGHT_ARB]					= BIT(26),
+	[GSI_IF]					= BIT(27),
+	[CLKON_GLOBAL]					= BIT(28),
+	[GLOBAL_2X_CLK]					= BIT(29),
+	[DPL_FIFO]					= BIT(30),
+	[DRBIP]						= BIT(31),
+};
+
+IPA_REG_FIELDS(CLKON_CFG, clkon_cfg, 0x00000044);
+
+static const u32 ipa_reg_route_fmask[] = {
+	[ROUTE_DIS]					= BIT(0),
+	[ROUTE_DEF_PIPE]				= GENMASK(5, 1),
+	[ROUTE_DEF_HDR_TABLE]				= BIT(6),
+	[ROUTE_DEF_HDR_OFST]				= GENMASK(16, 7),
+	[ROUTE_FRAG_DEF_PIPE]				= GENMASK(21, 17),
+						/* Bits 22-23 reserved */
+	[ROUTE_DEF_RETAIN_HDR]				= BIT(24),
+						/* Bits 25-31 reserved */
+};
+
+IPA_REG_FIELDS(ROUTE, route, 0x00000048);
+
+static const u32 ipa_reg_shared_mem_size_fmask[] = {
+	[MEM_SIZE]					= GENMASK(15, 0),
+	[MEM_BADDR]					= GENMASK(31, 16),
+};
+
+IPA_REG_FIELDS(SHARED_MEM_SIZE, shared_mem_size, 0x00000054);
+
+static const u32 ipa_reg_qsb_max_writes_fmask[] = {
+	[GEN_QMB_0_MAX_WRITES]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_WRITES]				= GENMASK(7, 4),
+						/* Bits 8-31 reserved */
+};
+
+IPA_REG_FIELDS(QSB_MAX_WRITES, qsb_max_writes, 0x00000074);
+
+static const u32 ipa_reg_qsb_max_reads_fmask[] = {
+	[GEN_QMB_0_MAX_READS]				= GENMASK(3, 0),
+	[GEN_QMB_1_MAX_READS]				= GENMASK(7, 4),
+						/* Bits 8-15 reserved */
+	[GEN_QMB_0_MAX_READS_BEATS]			= GENMASK(23, 16),
+	[GEN_QMB_1_MAX_READS_BEATS]			= GENMASK(31, 24),
+};
+
+IPA_REG_FIELDS(QSB_MAX_READS, qsb_max_reads, 0x00000078);
+
+static const u32 ipa_reg_filt_rout_hash_en_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_EN, filt_rout_hash_en, 0x0000148);
+
+static const u32 ipa_reg_filt_rout_hash_flush_fmask[] = {
+	[IPV6_ROUTER_HASH]				= BIT(0),
+						/* Bits 1-3 reserved */
+	[IPV6_FILTER_HASH]				= BIT(4),
+						/* Bits 5-7 reserved */
+	[IPV4_ROUTER_HASH]				= BIT(8),
+						/* Bits 9-11 reserved */
+	[IPV4_FILTER_HASH]				= BIT(12),
+						/* Bits 13-31 reserved */
+};
+
+IPA_REG_FIELDS(FILT_ROUT_HASH_FLUSH, filt_rout_hash_flush, 0x000014c);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(STATE_AGGR_ACTIVE, state_aggr_active, 0x000000b4);
+
+static const u32 ipa_reg_local_pkt_proc_cntxt_fmask[] = {
+	[IPA_BASE_ADDR]					= GENMASK(17, 0),
+						/* Bits 18-31 reserved */
+};
+
+/* Offset must be a multiple of 8 */
+IPA_REG_FIELDS(LOCAL_PKT_PROC_CNTXT, local_pkt_proc_cntxt, 0x000001e8);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(AGGR_FORCE_CLOSE, aggr_force_close, 0x000001ec);
+
+static const u32 ipa_reg_ipa_tx_cfg_fmask[] = {
+						/* Bits 0-1 reserved */
+	[PREFETCH_ALMOST_EMPTY_SIZE_TX0]		= GENMASK(5, 2),
+	[DMAW_SCND_OUTSD_PRED_THRESHOLD]		= GENMASK(9, 6),
+	[DMAW_SCND_OUTSD_PRED_EN]			= BIT(10),
+	[DMAW_MAX_BEATS_256_DIS]			= BIT(11),
+	[PA_MASK_EN]					= BIT(12),
+	[PREFETCH_ALMOST_EMPTY_SIZE_TX1]		= GENMASK(16, 13),
+	[DUAL_TX_ENABLE]				= BIT(17),
+	[SSPND_PA_NO_START_STATE]			= BIT(18),
+						/* Bits 19-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_TX_CFG, ipa_tx_cfg, 0x000001fc);
+
+static const u32 ipa_reg_flavor_0_fmask[] = {
+	[MAX_PIPES]					= GENMASK(3, 0),
+						/* Bits 4-7 reserved */
+	[MAX_CONS_PIPES]				= GENMASK(12, 8),
+						/* Bits 13-15 reserved */
+	[MAX_PROD_PIPES]				= GENMASK(20, 16),
+						/* Bits 21-23 reserved */
+	[PROD_LOWEST]					= GENMASK(27, 24),
+						/* Bits 28-31 reserved */
+};
+
+IPA_REG_FIELDS(FLAVOR_0, flavor_0, 0x00000210);
+
+static const u32 ipa_reg_idle_indication_cfg_fmask[] = {
+	[ENTER_IDLE_DEBOUNCE_THRESH]			= GENMASK(15, 0),
+	[CONST_NON_IDLE_ENABLE]				= BIT(16),
+						/* Bits 17-31 reserved */
+};
+
+IPA_REG_FIELDS(IDLE_INDICATION_CFG, idle_indication_cfg, 0x00000240);
+
+static const u32 ipa_reg_qtime_timestamp_cfg_fmask[] = {
+	[DPL_TIMESTAMP_LSB]				= GENMASK(4, 0),
+						/* Bits 5-6 reserved */
+	[DPL_TIMESTAMP_SEL]				= BIT(7),
+	[TAG_TIMESTAMP_LSB]				= GENMASK(12, 8),
+						/* Bits 13-15 reserved */
+	[NAT_TIMESTAMP_LSB]				= GENMASK(20, 16),
+						/* Bits 21-31 reserved */
+};
+
+IPA_REG_FIELDS(QTIME_TIMESTAMP_CFG, qtime_timestamp_cfg, 0x0000024c);
+
+static const u32 ipa_reg_timers_xo_clk_div_cfg_fmask[] = {
+	[DIV_VALUE]					= GENMASK(8, 0),
+						/* Bits 9-30 reserved */
+	[DIV_ENABLE]					= BIT(31),
+};
+
+IPA_REG_FIELDS(TIMERS_XO_CLK_DIV_CFG, timers_xo_clk_div_cfg, 0x00000250);
+
+static const u32 ipa_reg_timers_pulse_gran_cfg_fmask[] = {
+	[PULSE_GRAN_0]					= GENMASK(2, 0),
+	[PULSE_GRAN_1]					= GENMASK(5, 3),
+	[PULSE_GRAN_2]					= GENMASK(8, 6),
+};
+
+IPA_REG_FIELDS(TIMERS_PULSE_GRAN_CFG, timers_pulse_gran_cfg, 0x00000254);
+
+static const u32 ipa_reg_src_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_01_RSRC_TYPE, src_rsrc_grp_01_rsrc_type,
+		      0x00000400, 0x0020);
+
+static const u32 ipa_reg_src_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(SRC_RSRC_GRP_23_RSRC_TYPE, src_rsrc_grp_23_rsrc_type,
+		      0x00000404, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_01_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_01_RSRC_TYPE, dst_rsrc_grp_01_rsrc_type,
+		      0x00000500, 0x0020);
+
+static const u32 ipa_reg_dst_rsrc_grp_23_rsrc_type_fmask[] = {
+	[X_MIN_LIM]					= GENMASK(5, 0),
+						/* Bits 6-7 reserved */
+	[X_MAX_LIM]					= GENMASK(13, 8),
+						/* Bits 14-15 reserved */
+	[Y_MIN_LIM]					= GENMASK(21, 16),
+						/* Bits 22-23 reserved */
+	[Y_MAX_LIM]					= GENMASK(29, 24),
+						/* Bits 30-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(DST_RSRC_GRP_23_RSRC_TYPE, dst_rsrc_grp_23_rsrc_type,
+		      0x00000504, 0x0020);
+
+static const u32 ipa_reg_endp_init_cfg_fmask[] = {
+	[FRAG_OFFLOAD_EN]				= BIT(0),
+	[CS_OFFLOAD_EN]					= GENMASK(2, 1),
+	[CS_METADATA_HDR_OFFSET]			= GENMASK(6, 3),
+						/* Bit 7 reserved */
+	[CS_GEN_QMB_MASTER_SEL]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_CFG, endp_init_cfg, 0x00000808, 0x0070);
+
+static const u32 ipa_reg_endp_init_nat_fmask[] = {
+	[NAT_EN]					= GENMASK(1, 0),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_NAT, endp_init_nat, 0x0000080c, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_fmask[] = {
+	[HDR_LEN]					= GENMASK(5, 0),
+	[HDR_OFST_METADATA_VALID]			= BIT(6),
+	[HDR_OFST_METADATA]				= GENMASK(12, 7),
+	[HDR_ADDITIONAL_CONST_LEN]			= GENMASK(18, 13),
+	[HDR_OFST_PKT_SIZE_VALID]			= BIT(19),
+	[HDR_OFST_PKT_SIZE]				= GENMASK(25, 20),
+	[HDR_LEN_INC_DEAGG_HDR]				= BIT(27),
+	[HDR_LEN_MSB]					= GENMASK(29, 28),
+	[HDR_OFST_METADATA_MSB]				= GENMASK(31, 30),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR, endp_init_hdr, 0x00000810, 0x0070);
+
+static const u32 ipa_reg_endp_init_hdr_ext_fmask[] = {
+	[HDR_ENDIANNESS]				= BIT(0),
+	[HDR_TOTAL_LEN_OR_PAD_VALID]			= BIT(1),
+	[HDR_TOTAL_LEN_OR_PAD]				= BIT(2),
+	[HDR_PAYLOAD_LEN_INC_PADDING]			= BIT(3),
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET]			= GENMASK(9, 4),
+	[HDR_PAD_TO_ALIGNMENT]				= GENMASK(13, 10),
+						/* Bits 14-15 reserved */
+	[HDR_TOTAL_LEN_OR_PAD_OFFSET_MSB]		= GENMASK(17, 16),
+	[HDR_OFST_PKT_SIZE_MSB]				= GENMASK(19, 18),
+	[HDR_ADDITIONAL_CONST_LEN_MSB]			= GENMASK(21, 20),
+						/* Bits 22-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HDR_EXT, endp_init_hdr_ext, 0x00000814, 0x0070);
+
+IPA_REG_STRIDE(ENDP_INIT_HDR_METADATA_MASK, endp_init_hdr_metadata_mask,
+	       0x00000818, 0x0070);
+
+static const u32 ipa_reg_endp_init_mode_fmask[] = {
+	[ENDP_MODE]					= GENMASK(2, 0),
+	[DCPH_ENABLE]					= BIT(3),
+	[DEST_PIPE_INDEX]				= GENMASK(8, 4),
+						/* Bits 9-11 reserved */
+	[BYTE_THRESHOLD]				= GENMASK(27, 12),
+	[PIPE_REPLICATION_EN]				= BIT(28),
+	[PAD_EN]					= BIT(29),
+	[DRBIP_ACL_ENABLE]				= BIT(30),
+						/* Bit 31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_MODE, endp_init_mode, 0x00000820, 0x0070);
+
+static const u32 ipa_reg_endp_init_aggr_fmask[] = {
+	[AGGR_EN]					= GENMASK(1, 0),
+	[AGGR_TYPE]					= GENMASK(4, 2),
+	[BYTE_LIMIT]					= GENMASK(10, 5),
+						/* Bit 11 reserved */
+	[TIME_LIMIT]					= GENMASK(16, 12),
+	[PKT_LIMIT]					= GENMASK(22, 17),
+	[SW_EOF_ACTIVE]					= BIT(23),
+	[FORCE_CLOSE]					= BIT(24),
+						/* Bit 25 reserved */
+	[HARD_BYTE_LIMIT_EN]				= BIT(26),
+	[AGGR_GRAN_SEL]					= BIT(27),
+						/* Bits 28-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_AGGR, endp_init_aggr, 0x00000824, 0x0070);
+
+static const u32 ipa_reg_endp_init_hol_block_en_fmask[] = {
+	[HOL_BLOCK_EN]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_EN, endp_init_hol_block_en,
+		      0x0000082c, 0x0070);
+
+static const u32 ipa_reg_endp_init_hol_block_timer_fmask[] = {
+	[TIMER_LIMIT]					= GENMASK(4, 0),
+						/* Bits 5-7 reserved */
+	[TIMER_GRAN_SEL]				= BIT(8),
+						/* Bits 9-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_HOL_BLOCK_TIMER, endp_init_hol_block_timer,
+		      0x00000830, 0x0070);
+
+static const u32 ipa_reg_endp_init_deaggr_fmask[] = {
+	[DEAGGR_HDR_LEN]				= GENMASK(5, 0),
+	[SYSPIPE_ERR_DETECTION]				= BIT(6),
+	[PACKET_OFFSET_VALID]				= BIT(7),
+	[PACKET_OFFSET_LOCATION]			= GENMASK(13, 8),
+	[IGNORE_MIN_PKT_ERR]				= BIT(14),
+						/* Bit 15 reserved */
+	[MAX_PACKET_LEN]				= GENMASK(31, 16),
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_DEAGGR, endp_init_deaggr, 0x00000834, 0x0070);
+
+static const u32 ipa_reg_endp_init_rsrc_grp_fmask[] = {
+	[ENDP_RSRC_GRP]					= GENMASK(1, 0),
+						/* Bits 2-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_RSRC_GRP, endp_init_rsrc_grp,
+		      0x00000838, 0x0070);
+
+static const u32 ipa_reg_endp_init_seq_fmask[] = {
+	[SEQ_TYPE]					= GENMASK(7, 0),
+						/* Bits 8-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_INIT_SEQ, endp_init_seq, 0x0000083c, 0x0070);
+
+static const u32 ipa_reg_endp_status_fmask[] = {
+	[STATUS_EN]					= BIT(0),
+	[STATUS_ENDP]					= GENMASK(5, 1),
+						/* Bits 6-8 reserved */
+	[STATUS_PKT_SUPPRESS]				= BIT(9),
+						/* Bits 10-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_STATUS, endp_status, 0x00000840, 0x0070);
+
+static const u32 ipa_reg_endp_filter_router_hsh_cfg_fmask[] = {
+	[FILTER_HASH_MSK_SRC_ID]			= BIT(0),
+	[FILTER_HASH_MSK_SRC_IP]			= BIT(1),
+	[FILTER_HASH_MSK_DST_IP]			= BIT(2),
+	[FILTER_HASH_MSK_SRC_PORT]			= BIT(3),
+	[FILTER_HASH_MSK_DST_PORT]			= BIT(4),
+	[FILTER_HASH_MSK_PROTOCOL]			= BIT(5),
+	[FILTER_HASH_MSK_METADATA]			= BIT(6),
+	[FILTER_HASH_MSK_ALL]				= GENMASK(6, 0),
+						/* Bits 7-15 reserved */
+	[ROUTER_HASH_MSK_SRC_ID]			= BIT(16),
+	[ROUTER_HASH_MSK_SRC_IP]			= BIT(17),
+	[ROUTER_HASH_MSK_DST_IP]			= BIT(18),
+	[ROUTER_HASH_MSK_SRC_PORT]			= BIT(19),
+	[ROUTER_HASH_MSK_DST_PORT]			= BIT(20),
+	[ROUTER_HASH_MSK_PROTOCOL]			= BIT(21),
+	[ROUTER_HASH_MSK_METADATA]			= BIT(22),
+	[ROUTER_HASH_MSK_ALL]				= GENMASK(22, 16),
+						/* Bits 23-31 reserved */
+};
+
+IPA_REG_STRIDE_FIELDS(ENDP_FILTER_ROUTER_HSH_CFG, endp_filter_router_hsh_cfg,
+		      0x0000085c, 0x0070);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_STTS, ipa_irq_stts, 0x00004008 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_EN, ipa_irq_en, 0x0000400c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by enum ipa_irq_id; only used for GSI_EE_AP */
+IPA_REG(IPA_IRQ_CLR, ipa_irq_clr, 0x00004010 + 0x1000 * GSI_EE_AP);
+
+static const u32 ipa_reg_ipa_irq_uc_fmask[] = {
+	[UC_INTR]					= BIT(0),
+						/* Bits 1-31 reserved */
+};
+
+IPA_REG_FIELDS(IPA_IRQ_UC, ipa_irq_uc, 0x0000401c + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_INFO, irq_suspend_info, 0x00004030 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_EN, irq_suspend_en, 0x00004034 + 0x1000 * GSI_EE_AP);
+
+/* Valid bits defined by ipa->available */
+IPA_REG(IRQ_SUSPEND_CLR, irq_suspend_clr, 0x00004038 + 0x1000 * GSI_EE_AP);
+
+static const struct ipa_reg *ipa_reg_array[] = {
+	[COMP_CFG]			= &ipa_reg_comp_cfg,
+	[CLKON_CFG]			= &ipa_reg_clkon_cfg,
+	[ROUTE]				= &ipa_reg_route,
+	[SHARED_MEM_SIZE]		= &ipa_reg_shared_mem_size,
+	[QSB_MAX_WRITES]		= &ipa_reg_qsb_max_writes,
+	[QSB_MAX_READS]			= &ipa_reg_qsb_max_reads,
+	[FILT_ROUT_HASH_EN]		= &ipa_reg_filt_rout_hash_en,
+	[FILT_ROUT_HASH_FLUSH]		= &ipa_reg_filt_rout_hash_flush,
+	[STATE_AGGR_ACTIVE]		= &ipa_reg_state_aggr_active,
+	[LOCAL_PKT_PROC_CNTXT]		= &ipa_reg_local_pkt_proc_cntxt,
+	[AGGR_FORCE_CLOSE]		= &ipa_reg_aggr_force_close,
+	[IPA_TX_CFG]			= &ipa_reg_ipa_tx_cfg,
+	[FLAVOR_0]			= &ipa_reg_flavor_0,
+	[IDLE_INDICATION_CFG]		= &ipa_reg_idle_indication_cfg,
+	[QTIME_TIMESTAMP_CFG]		= &ipa_reg_qtime_timestamp_cfg,
+	[TIMERS_XO_CLK_DIV_CFG]		= &ipa_reg_timers_xo_clk_div_cfg,
+	[TIMERS_PULSE_GRAN_CFG]		= &ipa_reg_timers_pulse_gran_cfg,
+	[SRC_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_01_rsrc_type,
+	[SRC_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_src_rsrc_grp_23_rsrc_type,
+	[DST_RSRC_GRP_01_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_01_rsrc_type,
+	[DST_RSRC_GRP_23_RSRC_TYPE]	= &ipa_reg_dst_rsrc_grp_23_rsrc_type,
+	[ENDP_INIT_CFG]			= &ipa_reg_endp_init_cfg,
+	[ENDP_INIT_NAT]			= &ipa_reg_endp_init_nat,
+	[ENDP_INIT_HDR]			= &ipa_reg_endp_init_hdr,
+	[ENDP_INIT_HDR_EXT]		= &ipa_reg_endp_init_hdr_ext,
+	[ENDP_INIT_HDR_METADATA_MASK]	= &ipa_reg_endp_init_hdr_metadata_mask,
+	[ENDP_INIT_MODE]		= &ipa_reg_endp_init_mode,
+	[ENDP_INIT_AGGR]		= &ipa_reg_endp_init_aggr,
+	[ENDP_INIT_HOL_BLOCK_EN]	= &ipa_reg_endp_init_hol_block_en,
+	[ENDP_INIT_HOL_BLOCK_TIMER]	= &ipa_reg_endp_init_hol_block_timer,
+	[ENDP_INIT_DEAGGR]		= &ipa_reg_endp_init_deaggr,
+	[ENDP_INIT_RSRC_GRP]		= &ipa_reg_endp_init_rsrc_grp,
+	[ENDP_INIT_SEQ]			= &ipa_reg_endp_init_seq,
+	[ENDP_STATUS]			= &ipa_reg_endp_status,
+	[ENDP_FILTER_ROUTER_HSH_CFG]	= &ipa_reg_endp_filter_router_hsh_cfg,
+	[IPA_IRQ_STTS]			= &ipa_reg_ipa_irq_stts,
+	[IPA_IRQ_EN]			= &ipa_reg_ipa_irq_en,
+	[IPA_IRQ_CLR]			= &ipa_reg_ipa_irq_clr,
+	[IPA_IRQ_UC]			= &ipa_reg_ipa_irq_uc,
+	[IRQ_SUSPEND_INFO]		= &ipa_reg_irq_suspend_info,
+	[IRQ_SUSPEND_EN]		= &ipa_reg_irq_suspend_en,
+	[IRQ_SUSPEND_CLR]		= &ipa_reg_irq_suspend_clr,
+};
+
+const struct ipa_regs ipa_regs_v4_9 = {
+	.reg_count	= ARRAY_SIZE(ipa_reg_array),
+	.reg		= ipa_reg_array,
+};
diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c
index 49ba8a50dfb1..54c94a69c2bb 100644
--- a/drivers/net/ipvlan/ipvlan_main.c
+++ b/drivers/net/ipvlan/ipvlan_main.c
@@ -408,8 +408,8 @@ static int ipvlan_ethtool_get_link_ksettings(struct net_device *dev,
 static void ipvlan_ethtool_get_drvinfo(struct net_device *dev,
 				       struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, IPVLAN_DRV, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, IPV_DRV_VER, sizeof(drvinfo->version));
+	strscpy(drvinfo->driver, IPVLAN_DRV, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, IPV_DRV_VER, sizeof(drvinfo->version));
 }
 
 static u32 ipvlan_ethtool_get_msglevel(struct net_device *dev)
diff --git a/drivers/net/macsec.c b/drivers/net/macsec.c
index c6d271e5687e..c891b60937a7 100644
--- a/drivers/net/macsec.c
+++ b/drivers/net/macsec.c
@@ -18,14 +18,13 @@
 #include <net/sock.h>
 #include <net/gro_cells.h>
 #include <net/macsec.h>
+#include <net/dst_metadata.h>
 #include <linux/phy.h>
 #include <linux/byteorder/generic.h>
 #include <linux/if_arp.h>
 
 #include <uapi/linux/if_macsec.h>
 
-#define MACSEC_SCI_LEN 8
-
 /* SecTAG length = macsec_eth_header without the optional SCI */
 #define MACSEC_TAG_LEN 6
 
@@ -46,20 +45,10 @@ struct macsec_eth_header {
 	u8 secure_channel_id[8]; /* optional */
 } __packed;
 
-#define MACSEC_TCI_VERSION 0x80
-#define MACSEC_TCI_ES      0x40 /* end station */
-#define MACSEC_TCI_SC      0x20 /* SCI present */
-#define MACSEC_TCI_SCB     0x10 /* epon */
-#define MACSEC_TCI_E       0x08 /* encryption */
-#define MACSEC_TCI_C       0x04 /* changed text */
-#define MACSEC_AN_MASK     0x03 /* association number */
-#define MACSEC_TCI_CONFID  (MACSEC_TCI_E | MACSEC_TCI_C)
-
 /* minimum secure data length deemed "not short", see IEEE 802.1AE-2006 9.7 */
 #define MIN_NON_SHORT_LEN 48
 
 #define GCM_AES_IV_LEN 12
-#define DEFAULT_ICV_LEN 16
 
 #define for_each_rxsc(secy, sc)				\
 	for (sc = rcu_dereference_bh(secy->rx_sc);	\
@@ -243,7 +232,6 @@ static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb)
 	return (struct macsec_cb *)skb->cb;
 }
 
-#define MACSEC_PORT_ES (htons(0x0001))
 #define MACSEC_PORT_SCB (0x0000)
 #define MACSEC_UNDEF_SCI ((__force sci_t)0xffffffffffffffffULL)
 #define MACSEC_UNDEF_SSCI ((__force ssci_t)0xffffffff)
@@ -258,14 +246,6 @@ static struct macsec_cb *macsec_skb_cb(struct sk_buff *skb)
 #define DEFAULT_ENCODING_SA 0
 #define MACSEC_XPN_MAX_REPLAY_WINDOW (((1 << 30) - 1))
 
-static bool send_sci(const struct macsec_secy *secy)
-{
-	const struct macsec_tx_sc *tx_sc = &secy->tx_sc;
-
-	return tx_sc->send_sci ||
-		(secy->n_rx_sc > 1 && !tx_sc->end_station && !tx_sc->scb);
-}
-
 static sci_t make_sci(const u8 *addr, __be16 port)
 {
 	sci_t sci;
@@ -330,7 +310,7 @@ static void macsec_fill_sectag(struct macsec_eth_header *h,
 	/* with GCM, C/E clear for !encrypt, both set for encrypt */
 	if (tx_sc->encrypt)
 		h->tci_an |= MACSEC_TCI_CONFID;
-	else if (secy->icv_len != DEFAULT_ICV_LEN)
+	else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN)
 		h->tci_an |= MACSEC_TCI_C;
 
 	h->tci_an |= tx_sc->encoding_sa;
@@ -654,7 +634,7 @@ static struct sk_buff *macsec_encrypt(struct sk_buff *skb,
 
 	unprotected_len = skb->len;
 	eth = eth_hdr(skb);
-	sci_present = send_sci(secy);
+	sci_present = macsec_send_sci(secy);
 	hh = skb_push(skb, macsec_extra_len(sci_present));
 	memmove(hh, eth, 2 * ETH_ALEN);
 
@@ -1024,11 +1004,13 @@ static enum rx_handler_result handle_not_macsec(struct sk_buff *skb)
 	/* Deliver to the uncontrolled port by default */
 	enum rx_handler_result ret = RX_HANDLER_PASS;
 	struct ethhdr *hdr = eth_hdr(skb);
+	struct metadata_dst *md_dst;
 	struct macsec_rxh_data *rxd;
 	struct macsec_dev *macsec;
 
 	rcu_read_lock();
 	rxd = macsec_data_rcu(skb->dev);
+	md_dst = skb_metadata_dst(skb);
 
 	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
 		struct sk_buff *nskb;
@@ -1039,6 +1021,10 @@ static enum rx_handler_result handle_not_macsec(struct sk_buff *skb)
 		 * the SecTAG, so we have to deduce which port to deliver to.
 		 */
 		if (macsec_is_offloaded(macsec) && netif_running(ndev)) {
+			if (md_dst && md_dst->type == METADATA_MACSEC &&
+			    (!find_rx_sc(&macsec->secy, md_dst->u.macsec_info.sci)))
+				continue;
+
 			if (ether_addr_equal_64bits(hdr->h_dest,
 						    ndev->dev_addr)) {
 				/* exact match, divert skb to this port */
@@ -1296,7 +1282,7 @@ nosci:
 	/* 10.6.1 if the SC is not found */
 	cbit = !!(hdr->tci_an & MACSEC_TCI_C);
 	if (!cbit)
-		macsec_finalize_skb(skb, DEFAULT_ICV_LEN,
+		macsec_finalize_skb(skb, MACSEC_DEFAULT_ICV_LEN,
 				    macsec_extra_len(macsec_skb_cb(skb)->has_sci));
 
 	list_for_each_entry_rcu(macsec, &rxd->secys, secys) {
@@ -1677,22 +1663,8 @@ static int macsec_offload(int (* const func)(struct macsec_context *),
 	if (ctx->offload == MACSEC_OFFLOAD_PHY)
 		mutex_lock(&ctx->phydev->lock);
 
-	/* Phase I: prepare. The drive should fail here if there are going to be
-	 * issues in the commit phase.
-	 */
-	ctx->prepare = true;
 	ret = (*func)(ctx);
-	if (ret)
-		goto phy_unlock;
 
-	/* Phase II: commit. This step cannot fail. */
-	ctx->prepare = false;
-	ret = (*func)(ctx);
-	/* This should never happen: commit is not allowed to fail */
-	if (unlikely(ret))
-		WARN(1, "MACsec offloading commit failed (%d)\n", ret);
-
-phy_unlock:
 	if (ctx->offload == MACSEC_OFFLOAD_PHY)
 		mutex_unlock(&ctx->phydev->lock);
 
@@ -1842,6 +1814,12 @@ static int macsec_add_rxsa(struct sk_buff *skb, struct genl_info *info)
 
 	rx_sa->sc = rx_sc;
 
+	if (secy->xpn) {
+		rx_sa->ssci = nla_get_ssci(tb_sa[MACSEC_SA_ATTR_SSCI]);
+		nla_memcpy(rx_sa->key.salt.bytes, tb_sa[MACSEC_SA_ATTR_SALT],
+			   MACSEC_SALT_LEN);
+	}
+
 	/* If h/w offloading is available, propagate to the device */
 	if (macsec_is_offloaded(netdev_priv(dev))) {
 		const struct macsec_ops *ops;
@@ -1864,12 +1842,6 @@ static int macsec_add_rxsa(struct sk_buff *skb, struct genl_info *info)
 			goto cleanup;
 	}
 
-	if (secy->xpn) {
-		rx_sa->ssci = nla_get_ssci(tb_sa[MACSEC_SA_ATTR_SSCI]);
-		nla_memcpy(rx_sa->key.salt.bytes, tb_sa[MACSEC_SA_ATTR_SALT],
-			   MACSEC_SALT_LEN);
-	}
-
 	nla_memcpy(rx_sa->key.id, tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN);
 	rcu_assign_pointer(rx_sc->sa[assoc_num], rx_sa);
 
@@ -2084,6 +2056,12 @@ static int macsec_add_txsa(struct sk_buff *skb, struct genl_info *info)
 	if (assoc_num == tx_sc->encoding_sa && tx_sa->active)
 		secy->operational = true;
 
+	if (secy->xpn) {
+		tx_sa->ssci = nla_get_ssci(tb_sa[MACSEC_SA_ATTR_SSCI]);
+		nla_memcpy(tx_sa->key.salt.bytes, tb_sa[MACSEC_SA_ATTR_SALT],
+			   MACSEC_SALT_LEN);
+	}
+
 	/* If h/w offloading is available, propagate to the device */
 	if (macsec_is_offloaded(netdev_priv(dev))) {
 		const struct macsec_ops *ops;
@@ -2106,12 +2084,6 @@ static int macsec_add_txsa(struct sk_buff *skb, struct genl_info *info)
 			goto cleanup;
 	}
 
-	if (secy->xpn) {
-		tx_sa->ssci = nla_get_ssci(tb_sa[MACSEC_SA_ATTR_SSCI]);
-		nla_memcpy(tx_sa->key.salt.bytes, tb_sa[MACSEC_SA_ATTR_SALT],
-			   MACSEC_SALT_LEN);
-	}
-
 	nla_memcpy(tx_sa->key.id, tb_sa[MACSEC_SA_ATTR_KEYID], MACSEC_KEYID_LEN);
 	rcu_assign_pointer(tx_sc->sa[assoc_num], tx_sa);
 
@@ -3404,6 +3376,7 @@ static struct genl_family macsec_fam __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= macsec_genl_ops,
 	.n_small_ops	= ARRAY_SIZE(macsec_genl_ops),
+	.resv_start_op	= MACSEC_CMD_UPD_OFFLOAD + 1,
 };
 
 static netdev_tx_t macsec_start_xmit(struct sk_buff *skb,
@@ -3415,6 +3388,11 @@ static netdev_tx_t macsec_start_xmit(struct sk_buff *skb,
 	int ret, len;
 
 	if (macsec_is_offloaded(netdev_priv(dev))) {
+		struct metadata_dst *md_dst = secy->tx_sc.md_dst;
+
+		skb_dst_drop(skb);
+		dst_hold(&md_dst->dst);
+		skb_dst_set(skb, &md_dst->dst);
 		skb->dev = macsec->real_dev;
 		return dev_queue_xmit(skb);
 	}
@@ -3742,6 +3720,8 @@ static void macsec_free_netdev(struct net_device *dev)
 {
 	struct macsec_dev *macsec = macsec_priv(dev);
 
+	if (macsec->secy.tx_sc.md_dst)
+		metadata_dst_free(macsec->secy.tx_sc.md_dst);
 	free_percpu(macsec->stats);
 	free_percpu(macsec->secy.tx_sc.stats);
 
@@ -4014,6 +3994,13 @@ static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)
 		return -ENOMEM;
 	}
 
+	secy->tx_sc.md_dst = metadata_dst_alloc(0, METADATA_MACSEC, GFP_KERNEL);
+	if (!secy->tx_sc.md_dst) {
+		free_percpu(secy->tx_sc.stats);
+		free_percpu(macsec->stats);
+		return -ENOMEM;
+	}
+
 	if (sci == MACSEC_UNDEF_SCI)
 		sci = dev_to_sci(dev, MACSEC_PORT_ES);
 
@@ -4027,6 +4014,7 @@ static int macsec_add_dev(struct net_device *dev, sci_t sci, u8 icv_len)
 	secy->xpn = DEFAULT_XPN;
 
 	secy->sci = sci;
+	secy->tx_sc.md_dst->u.macsec_info.sci = sci;
 	secy->tx_sc.active = true;
 	secy->tx_sc.encoding_sa = DEFAULT_ENCODING_SA;
 	secy->tx_sc.encrypt = DEFAULT_ENCRYPT;
@@ -4045,7 +4033,7 @@ static int macsec_newlink(struct net *net, struct net_device *dev,
 {
 	struct macsec_dev *macsec = macsec_priv(dev);
 	rx_handler_func_t *rx_handler;
-	u8 icv_len = DEFAULT_ICV_LEN;
+	u8 icv_len = MACSEC_DEFAULT_ICV_LEN;
 	struct net_device *real_dev;
 	int err, mtu;
 	sci_t sci;
@@ -4169,7 +4157,7 @@ static int macsec_validate_attr(struct nlattr *tb[], struct nlattr *data[],
 				struct netlink_ext_ack *extack)
 {
 	u64 csid = MACSEC_DEFAULT_CIPHER_ID;
-	u8 icv_len = DEFAULT_ICV_LEN;
+	u8 icv_len = MACSEC_DEFAULT_ICV_LEN;
 	int flag;
 	bool es, scb, sci;
 
@@ -4181,7 +4169,7 @@ static int macsec_validate_attr(struct nlattr *tb[], struct nlattr *data[],
 
 	if (data[IFLA_MACSEC_ICV_LEN]) {
 		icv_len = nla_get_u8(data[IFLA_MACSEC_ICV_LEN]);
-		if (icv_len != DEFAULT_ICV_LEN) {
+		if (icv_len != MACSEC_DEFAULT_ICV_LEN) {
 			char dummy_key[DEFAULT_SAK_LEN] = { 0 };
 			struct crypto_aead *dummy_tfm;
 
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index 1080d6ebff63..713e3354cb2e 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -1043,8 +1043,8 @@ static int macvlan_fdb_del(struct ndmsg *ndm, struct nlattr *tb[],
 static void macvlan_ethtool_get_drvinfo(struct net_device *dev,
 					struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, "macvlan", sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, "0.1", sizeof(drvinfo->version));
+	strscpy(drvinfo->driver, "macvlan", sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, "0.1", sizeof(drvinfo->version));
 }
 
 static int macvlan_ethtool_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c
index cecf8c63096c..d1f435788e90 100644
--- a/drivers/net/macvtap.c
+++ b/drivers/net/macvtap.c
@@ -207,7 +207,7 @@ static struct notifier_block macvtap_notifier_block __read_mostly = {
 	.notifier_call	= macvtap_device_event,
 };
 
-static int macvtap_init(void)
+static int __init macvtap_init(void)
 {
 	int err;
 
@@ -241,7 +241,7 @@ out1:
 }
 module_init(macvtap_init);
 
-static void macvtap_exit(void)
+static void __exit macvtap_exit(void)
 {
 	rtnl_link_unregister(&macvtap_link_ops);
 	unregister_netdevice_notifier(&macvtap_notifier_block);
diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c
index 53846c6b56ca..0762c735dd8a 100644
--- a/drivers/net/mctp/mctp-i2c.c
+++ b/drivers/net/mctp/mctp-i2c.c
@@ -986,7 +986,7 @@ out:
 	return rc;
 }
 
-static int mctp_i2c_remove(struct i2c_client *client)
+static void mctp_i2c_remove(struct i2c_client *client)
 {
 	struct mctp_i2c_client *mcli = i2c_get_clientdata(client);
 	struct mctp_i2c_dev *midev = NULL, *tmp = NULL;
@@ -999,8 +999,6 @@ static int mctp_i2c_remove(struct i2c_client *client)
 
 	mctp_i2c_free_client(mcli);
 	mutex_unlock(&driver_clients_lock);
-	/* Callers ignore return code */
-	return 0;
 }
 
 /* We look for a 'mctp-controller' property on I2C busses as they are
diff --git a/drivers/net/mdio/fwnode_mdio.c b/drivers/net/mdio/fwnode_mdio.c
index 1c1584fca632..689e728345ce 100644
--- a/drivers/net/mdio/fwnode_mdio.c
+++ b/drivers/net/mdio/fwnode_mdio.c
@@ -10,10 +10,31 @@
 #include <linux/fwnode_mdio.h>
 #include <linux/of.h>
 #include <linux/phy.h>
+#include <linux/pse-pd/pse.h>
 
 MODULE_AUTHOR("Calvin Johnson <calvin.johnson@oss.nxp.com>");
 MODULE_LICENSE("GPL");
 
+static struct pse_control *
+fwnode_find_pse_control(struct fwnode_handle *fwnode)
+{
+	struct pse_control *psec;
+	struct device_node *np;
+
+	if (!IS_ENABLED(CONFIG_PSE_CONTROLLER))
+		return NULL;
+
+	np = to_of_node(fwnode);
+	if (!np)
+		return NULL;
+
+	psec = of_pse_control_get(np);
+	if (PTR_ERR(psec) == -ENOENT)
+		return NULL;
+
+	return psec;
+}
+
 static struct mii_timestamper *
 fwnode_find_mii_timestamper(struct fwnode_handle *fwnode)
 {
@@ -91,14 +112,21 @@ int fwnode_mdiobus_register_phy(struct mii_bus *bus,
 				struct fwnode_handle *child, u32 addr)
 {
 	struct mii_timestamper *mii_ts = NULL;
+	struct pse_control *psec = NULL;
 	struct phy_device *phy;
 	bool is_c45 = false;
 	u32 phy_id;
 	int rc;
 
+	psec = fwnode_find_pse_control(child);
+	if (IS_ERR(psec))
+		return PTR_ERR(psec);
+
 	mii_ts = fwnode_find_mii_timestamper(child);
-	if (IS_ERR(mii_ts))
-		return PTR_ERR(mii_ts);
+	if (IS_ERR(mii_ts)) {
+		rc = PTR_ERR(mii_ts);
+		goto clean_pse;
+	}
 
 	rc = fwnode_property_match_string(child, "compatible",
 					  "ethernet-phy-ieee802.3-c45");
@@ -110,8 +138,8 @@ int fwnode_mdiobus_register_phy(struct mii_bus *bus,
 	else
 		phy = phy_device_create(bus, addr, phy_id, 0, NULL);
 	if (IS_ERR(phy)) {
-		unregister_mii_timestamper(mii_ts);
-		return PTR_ERR(phy);
+		rc = PTR_ERR(phy);
+		goto clean_mii_ts;
 	}
 
 	if (is_acpi_node(child)) {
@@ -125,25 +153,33 @@ int fwnode_mdiobus_register_phy(struct mii_bus *bus,
 		/* All data is now stored in the phy struct, so register it */
 		rc = phy_device_register(phy);
 		if (rc) {
-			phy_device_free(phy);
 			fwnode_handle_put(phy->mdio.dev.fwnode);
-			return rc;
+			goto clean_phy;
 		}
 	} else if (is_of_node(child)) {
 		rc = fwnode_mdiobus_phy_device_register(bus, phy, child, addr);
-		if (rc) {
-			unregister_mii_timestamper(mii_ts);
-			phy_device_free(phy);
-			return rc;
-		}
+		if (rc)
+			goto clean_phy;
 	}
 
+	phy->psec = psec;
+
 	/* phy->mii_ts may already be defined by the PHY driver. A
 	 * mii_timestamper probed via the device tree will still have
 	 * precedence.
 	 */
 	if (mii_ts)
 		phy->mii_ts = mii_ts;
+
 	return 0;
+
+clean_phy:
+	phy_device_free(phy);
+clean_mii_ts:
+	unregister_mii_timestamper(mii_ts);
+clean_pse:
+	pse_control_put(psec);
+
+	return rc;
 }
 EXPORT_SYMBOL(fwnode_mdiobus_register_phy);
diff --git a/drivers/net/mdio/mdio-i2c.c b/drivers/net/mdio/mdio-i2c.c
index 09200a70b315..bf8bf5e20faf 100644
--- a/drivers/net/mdio/mdio-i2c.c
+++ b/drivers/net/mdio/mdio-i2c.c
@@ -3,6 +3,7 @@
  * MDIO I2C bridge
  *
  * Copyright (C) 2015-2016 Russell King
+ * Copyright (C) 2021 Marek Behun
  *
  * Network PHYs can appear on I2C buses when they are part of SFP module.
  * This driver exposes these PHYs to the networking PHY code, allowing
@@ -12,6 +13,7 @@
 #include <linux/i2c.h>
 #include <linux/mdio/mdio-i2c.h>
 #include <linux/phy.h>
+#include <linux/sfp.h>
 
 /*
  * I2C bus addresses 0x50 and 0x51 are normally an EEPROM, which is
@@ -28,7 +30,7 @@ static unsigned int i2c_mii_phy_addr(int phy_id)
 	return phy_id + 0x40;
 }
 
-static int i2c_mii_read(struct mii_bus *bus, int phy_id, int reg)
+static int i2c_mii_read_default(struct mii_bus *bus, int phy_id, int reg)
 {
 	struct i2c_adapter *i2c = bus->priv;
 	struct i2c_msg msgs[2];
@@ -62,7 +64,8 @@ static int i2c_mii_read(struct mii_bus *bus, int phy_id, int reg)
 	return data[0] << 8 | data[1];
 }
 
-static int i2c_mii_write(struct mii_bus *bus, int phy_id, int reg, u16 val)
+static int i2c_mii_write_default(struct mii_bus *bus, int phy_id, int reg,
+				 u16 val)
 {
 	struct i2c_adapter *i2c = bus->priv;
 	struct i2c_msg msg;
@@ -91,9 +94,288 @@ static int i2c_mii_write(struct mii_bus *bus, int phy_id, int reg, u16 val)
 	return ret < 0 ? ret : 0;
 }
 
-struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c)
+/* RollBall SFPs do not access internal PHY via I2C address 0x56, but
+ * instead via address 0x51, when SFP page is set to 0x03 and password to
+ * 0xffffffff.
+ *
+ * address  size  contents  description
+ * -------  ----  --------  -----------
+ * 0x80     1     CMD       0x01/0x02/0x04 for write/read/done
+ * 0x81     1     DEV       Clause 45 device
+ * 0x82     2     REG       Clause 45 register
+ * 0x84     2     VAL       Register value
+ */
+#define ROLLBALL_PHY_I2C_ADDR		0x51
+
+#define ROLLBALL_PASSWORD		(SFP_VSL + 3)
+
+#define ROLLBALL_CMD_ADDR		0x80
+#define ROLLBALL_DATA_ADDR		0x81
+
+#define ROLLBALL_CMD_WRITE		0x01
+#define ROLLBALL_CMD_READ		0x02
+#define ROLLBALL_CMD_DONE		0x04
+
+#define SFP_PAGE_ROLLBALL_MDIO		3
+
+static int __i2c_transfer_err(struct i2c_adapter *i2c, struct i2c_msg *msgs,
+			      int num)
+{
+	int ret;
+
+	ret = __i2c_transfer(i2c, msgs, num);
+	if (ret < 0)
+		return ret;
+	else if (ret != num)
+		return -EIO;
+	else
+		return 0;
+}
+
+static int __i2c_rollball_get_page(struct i2c_adapter *i2c, int bus_addr,
+				   u8 *page)
+{
+	struct i2c_msg msgs[2];
+	u8 addr = SFP_PAGE;
+
+	msgs[0].addr = bus_addr;
+	msgs[0].flags = 0;
+	msgs[0].len = 1;
+	msgs[0].buf = &addr;
+
+	msgs[1].addr = bus_addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = 1;
+	msgs[1].buf = page;
+
+	return __i2c_transfer_err(i2c, msgs, 2);
+}
+
+static int __i2c_rollball_set_page(struct i2c_adapter *i2c, int bus_addr,
+				   u8 page)
+{
+	struct i2c_msg msg;
+	u8 buf[2];
+
+	buf[0] = SFP_PAGE;
+	buf[1] = page;
+
+	msg.addr = bus_addr;
+	msg.flags = 0;
+	msg.len = 2;
+	msg.buf = buf;
+
+	return __i2c_transfer_err(i2c, &msg, 1);
+}
+
+/* In order to not interfere with other SFP code (which possibly may manipulate
+ * SFP_PAGE), for every transfer we do this:
+ *   1. lock the bus
+ *   2. save content of SFP_PAGE
+ *   3. set SFP_PAGE to 3
+ *   4. do the transfer
+ *   5. restore original SFP_PAGE
+ *   6. unlock the bus
+ * Note that one might think that steps 2 to 5 could be theoretically done all
+ * in one call to i2c_transfer (by constructing msgs array in such a way), but
+ * unfortunately tests show that this does not work :-( Changed SFP_PAGE does
+ * not take into account until i2c_transfer() is done.
+ */
+static int i2c_transfer_rollball(struct i2c_adapter *i2c,
+				 struct i2c_msg *msgs, int num)
+{
+	int ret, main_err = 0;
+	u8 saved_page;
+
+	i2c_lock_bus(i2c, I2C_LOCK_SEGMENT);
+
+	/* save original page */
+	ret = __i2c_rollball_get_page(i2c, msgs->addr, &saved_page);
+	if (ret)
+		goto unlock;
+
+	/* change to RollBall MDIO page */
+	ret = __i2c_rollball_set_page(i2c, msgs->addr, SFP_PAGE_ROLLBALL_MDIO);
+	if (ret)
+		goto unlock;
+
+	/* do the transfer; we try to restore original page if this fails */
+	ret = __i2c_transfer_err(i2c, msgs, num);
+	if (ret)
+		main_err = ret;
+
+	/* restore original page */
+	ret = __i2c_rollball_set_page(i2c, msgs->addr, saved_page);
+
+unlock:
+	i2c_unlock_bus(i2c, I2C_LOCK_SEGMENT);
+
+	return main_err ? : ret;
+}
+
+static int i2c_rollball_mii_poll(struct mii_bus *bus, int bus_addr, u8 *buf,
+				 size_t len)
+{
+	struct i2c_adapter *i2c = bus->priv;
+	struct i2c_msg msgs[2];
+	u8 cmd_addr, tmp, *res;
+	int i, ret;
+
+	cmd_addr = ROLLBALL_CMD_ADDR;
+
+	res = buf ? buf : &tmp;
+	len = buf ? len : 1;
+
+	msgs[0].addr = bus_addr;
+	msgs[0].flags = 0;
+	msgs[0].len = 1;
+	msgs[0].buf = &cmd_addr;
+
+	msgs[1].addr = bus_addr;
+	msgs[1].flags = I2C_M_RD;
+	msgs[1].len = len;
+	msgs[1].buf = res;
+
+	/* By experiment it takes up to 70 ms to access a register for these
+	 * SFPs. Sleep 20ms between iterations and try 10 times.
+	 */
+	i = 10;
+	do {
+		msleep(20);
+
+		ret = i2c_transfer_rollball(i2c, msgs, ARRAY_SIZE(msgs));
+		if (ret)
+			return ret;
+
+		if (*res == ROLLBALL_CMD_DONE)
+			return 0;
+	} while (i-- > 0);
+
+	dev_dbg(&bus->dev, "poll timed out\n");
+
+	return -ETIMEDOUT;
+}
+
+static int i2c_rollball_mii_cmd(struct mii_bus *bus, int bus_addr, u8 cmd,
+				u8 *data, size_t len)
+{
+	struct i2c_adapter *i2c = bus->priv;
+	struct i2c_msg msgs[2];
+	u8 cmdbuf[2];
+
+	cmdbuf[0] = ROLLBALL_CMD_ADDR;
+	cmdbuf[1] = cmd;
+
+	msgs[0].addr = bus_addr;
+	msgs[0].flags = 0;
+	msgs[0].len = len;
+	msgs[0].buf = data;
+
+	msgs[1].addr = bus_addr;
+	msgs[1].flags = 0;
+	msgs[1].len = sizeof(cmdbuf);
+	msgs[1].buf = cmdbuf;
+
+	return i2c_transfer_rollball(i2c, msgs, ARRAY_SIZE(msgs));
+}
+
+static int i2c_mii_read_rollball(struct mii_bus *bus, int phy_id, int reg)
+{
+	u8 buf[4], res[6];
+	int bus_addr, ret;
+	u16 val;
+
+	if (!(reg & MII_ADDR_C45))
+		return -EOPNOTSUPP;
+
+	bus_addr = i2c_mii_phy_addr(phy_id);
+	if (bus_addr != ROLLBALL_PHY_I2C_ADDR)
+		return 0xffff;
+
+	buf[0] = ROLLBALL_DATA_ADDR;
+	buf[1] = (reg >> 16) & 0x1f;
+	buf[2] = (reg >> 8) & 0xff;
+	buf[3] = reg & 0xff;
+
+	ret = i2c_rollball_mii_cmd(bus, bus_addr, ROLLBALL_CMD_READ, buf,
+				   sizeof(buf));
+	if (ret < 0)
+		return ret;
+
+	ret = i2c_rollball_mii_poll(bus, bus_addr, res, sizeof(res));
+	if (ret == -ETIMEDOUT)
+		return 0xffff;
+	else if (ret < 0)
+		return ret;
+
+	val = res[4] << 8 | res[5];
+
+	return val;
+}
+
+static int i2c_mii_write_rollball(struct mii_bus *bus, int phy_id, int reg,
+				  u16 val)
+{
+	int bus_addr, ret;
+	u8 buf[6];
+
+	if (!(reg & MII_ADDR_C45))
+		return -EOPNOTSUPP;
+
+	bus_addr = i2c_mii_phy_addr(phy_id);
+	if (bus_addr != ROLLBALL_PHY_I2C_ADDR)
+		return 0;
+
+	buf[0] = ROLLBALL_DATA_ADDR;
+	buf[1] = (reg >> 16) & 0x1f;
+	buf[2] = (reg >> 8) & 0xff;
+	buf[3] = reg & 0xff;
+	buf[4] = val >> 8;
+	buf[5] = val & 0xff;
+
+	ret = i2c_rollball_mii_cmd(bus, bus_addr, ROLLBALL_CMD_WRITE, buf,
+				   sizeof(buf));
+	if (ret < 0)
+		return ret;
+
+	ret = i2c_rollball_mii_poll(bus, bus_addr, NULL, 0);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
+static int i2c_mii_init_rollball(struct i2c_adapter *i2c)
+{
+	struct i2c_msg msg;
+	u8 pw[5];
+	int ret;
+
+	pw[0] = ROLLBALL_PASSWORD;
+	pw[1] = 0xff;
+	pw[2] = 0xff;
+	pw[3] = 0xff;
+	pw[4] = 0xff;
+
+	msg.addr = ROLLBALL_PHY_I2C_ADDR;
+	msg.flags = 0;
+	msg.len = sizeof(pw);
+	msg.buf = pw;
+
+	ret = i2c_transfer(i2c, &msg, 1);
+	if (ret < 0)
+		return ret;
+	else if (ret != 1)
+		return -EIO;
+	else
+		return 0;
+}
+
+struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c,
+			       enum mdio_i2c_proto protocol)
 {
 	struct mii_bus *mii;
+	int ret;
 
 	if (!i2c_check_functionality(i2c, I2C_FUNC_I2C))
 		return ERR_PTR(-EINVAL);
@@ -104,10 +386,28 @@ struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c)
 
 	snprintf(mii->id, MII_BUS_ID_SIZE, "i2c:%s", dev_name(parent));
 	mii->parent = parent;
-	mii->read = i2c_mii_read;
-	mii->write = i2c_mii_write;
 	mii->priv = i2c;
 
+	switch (protocol) {
+	case MDIO_I2C_ROLLBALL:
+		ret = i2c_mii_init_rollball(i2c);
+		if (ret < 0) {
+			dev_err(parent,
+				"Cannot initialize RollBall MDIO I2C protocol: %d\n",
+				ret);
+			mdiobus_free(mii);
+			return ERR_PTR(ret);
+		}
+
+		mii->read = i2c_mii_read_rollball;
+		mii->write = i2c_mii_write_rollball;
+		break;
+	default:
+		mii->read = i2c_mii_read_default;
+		mii->write = i2c_mii_write_default;
+		break;
+	}
+
 	return mii;
 }
 EXPORT_SYMBOL_GPL(mdio_i2c_alloc);
diff --git a/drivers/net/mdio/mdio-mscc-miim.c b/drivers/net/mdio/mdio-mscc-miim.c
index 08541007b18a..51f68daac152 100644
--- a/drivers/net/mdio/mdio-mscc-miim.c
+++ b/drivers/net/mdio/mdio-mscc-miim.c
@@ -12,6 +12,7 @@
 #include <linux/iopoll.h>
 #include <linux/kernel.h>
 #include <linux/mdio/mdio-mscc-miim.h>
+#include <linux/mfd/ocelot.h>
 #include <linux/module.h>
 #include <linux/of_mdio.h>
 #include <linux/phy.h>
@@ -270,44 +271,25 @@ static int mscc_miim_clk_set(struct mii_bus *bus)
 
 static int mscc_miim_probe(struct platform_device *pdev)
 {
-	struct regmap *mii_regmap, *phy_regmap = NULL;
 	struct device_node *np = pdev->dev.of_node;
+	struct regmap *mii_regmap, *phy_regmap;
 	struct device *dev = &pdev->dev;
-	void __iomem *regs, *phy_regs;
 	struct mscc_miim_dev *miim;
-	struct resource *res;
 	struct mii_bus *bus;
 	int ret;
 
-	regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
-	if (IS_ERR(regs)) {
-		dev_err(dev, "Unable to map MIIM registers\n");
-		return PTR_ERR(regs);
-	}
-
-	mii_regmap = devm_regmap_init_mmio(dev, regs, &mscc_miim_regmap_config);
-
-	if (IS_ERR(mii_regmap)) {
-		dev_err(dev, "Unable to create MIIM regmap\n");
-		return PTR_ERR(mii_regmap);
-	}
+	mii_regmap = ocelot_regmap_from_resource(pdev, 0,
+						 &mscc_miim_regmap_config);
+	if (IS_ERR(mii_regmap))
+		return dev_err_probe(dev, PTR_ERR(mii_regmap),
+				     "Unable to create MIIM regmap\n");
 
 	/* This resource is optional */
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-	if (res) {
-		phy_regs = devm_ioremap_resource(dev, res);
-		if (IS_ERR(phy_regs)) {
-			dev_err(dev, "Unable to map internal phy registers\n");
-			return PTR_ERR(phy_regs);
-		}
-
-		phy_regmap = devm_regmap_init_mmio(dev, phy_regs,
-						   &mscc_miim_phy_regmap_config);
-		if (IS_ERR(phy_regmap)) {
-			dev_err(dev, "Unable to create phy register regmap\n");
-			return PTR_ERR(phy_regmap);
-		}
-	}
+	phy_regmap = ocelot_regmap_from_resource_optional(pdev, 1,
+						 &mscc_miim_phy_regmap_config);
+	if (IS_ERR(phy_regmap))
+		return dev_err_probe(dev, PTR_ERR(phy_regmap),
+				     "Unable to create phy register regmap\n");
 
 	ret = mscc_miim_setup(dev, &bus, "mscc_miim", mii_regmap, 0);
 	if (ret < 0) {
diff --git a/drivers/net/mdio/mdio-mux-meson-g12a.c b/drivers/net/mdio/mdio-mux-meson-g12a.c
index b8866bc3f2e8..4a2e94faf57e 100644
--- a/drivers/net/mdio/mdio-mux-meson-g12a.c
+++ b/drivers/net/mdio/mdio-mux-meson-g12a.c
@@ -233,11 +233,9 @@ static int g12a_ephy_glue_clk_register(struct device *dev)
 
 		snprintf(in_name, sizeof(in_name), "clkin%d", i);
 		clk = devm_clk_get(dev, in_name);
-		if (IS_ERR(clk)) {
-			if (PTR_ERR(clk) != -EPROBE_DEFER)
-				dev_err(dev, "Missing clock %s\n", in_name);
-			return PTR_ERR(clk);
-		}
+		if (IS_ERR(clk))
+			return dev_err_probe(dev, PTR_ERR(clk),
+					     "Missing clock %s\n", in_name);
 
 		parent_names[i] = __clk_get_name(clk);
 	}
@@ -317,12 +315,9 @@ static int g12a_mdio_mux_probe(struct platform_device *pdev)
 		return PTR_ERR(priv->regs);
 
 	priv->pclk = devm_clk_get(dev, "pclk");
-	if (IS_ERR(priv->pclk)) {
-		ret = PTR_ERR(priv->pclk);
-		if (ret != -EPROBE_DEFER)
-			dev_err(dev, "failed to get peripheral clock\n");
-		return ret;
-	}
+	if (IS_ERR(priv->pclk))
+		return dev_err_probe(dev, PTR_ERR(priv->pclk),
+				     "failed to get peripheral clock\n");
 
 	/* Make sure the device registers are clocked */
 	ret = clk_prepare_enable(priv->pclk);
@@ -339,8 +334,7 @@ static int g12a_mdio_mux_probe(struct platform_device *pdev)
 	ret = mdio_mux_init(dev, dev->of_node, g12a_mdio_switch_fn,
 			    &priv->mux_handle, dev, NULL);
 	if (ret) {
-		if (ret != -EPROBE_DEFER)
-			dev_err(dev, "mdio multiplexer init failed: %d", ret);
+		dev_err_probe(dev, ret, "mdio multiplexer init failed\n");
 		goto err;
 	}
 
diff --git a/drivers/net/mdio/mdio-mux-mmioreg.c b/drivers/net/mdio/mdio-mux-mmioreg.c
index c02fb2a067ee..c02c9c660016 100644
--- a/drivers/net/mdio/mdio-mux-mmioreg.c
+++ b/drivers/net/mdio/mdio-mux-mmioreg.c
@@ -159,12 +159,9 @@ static int mdio_mux_mmioreg_probe(struct platform_device *pdev)
 	ret = mdio_mux_init(&pdev->dev, pdev->dev.of_node,
 			    mdio_mux_mmioreg_switch_fn,
 			    &s->mux_handle, s, NULL);
-	if (ret) {
-		if (ret != -EPROBE_DEFER)
-			dev_err(&pdev->dev,
-				"failed to register mdio-mux bus %pOF\n", np);
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "failed to register mdio-mux bus %pOF\n", np);
 
 	pdev->dev.platform_data = s;
 
diff --git a/drivers/net/mdio/mdio-mux-multiplexer.c b/drivers/net/mdio/mdio-mux-multiplexer.c
index 527acfc3c045..bfa5af577b0a 100644
--- a/drivers/net/mdio/mdio-mux-multiplexer.c
+++ b/drivers/net/mdio/mdio-mux-multiplexer.c
@@ -72,12 +72,9 @@ static int mdio_mux_multiplexer_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	s->muxc = devm_mux_control_get(dev, NULL);
-	if (IS_ERR(s->muxc)) {
-		ret = PTR_ERR(s->muxc);
-		if (ret != -EPROBE_DEFER)
-			dev_err(&pdev->dev, "Failed to get mux: %d\n", ret);
-		return ret;
-	}
+	if (IS_ERR(s->muxc))
+		return dev_err_probe(&pdev->dev, PTR_ERR(s->muxc),
+				     "Failed to get mux\n");
 
 	platform_set_drvdata(pdev, s);
 
diff --git a/drivers/net/net_failover.c b/drivers/net/net_failover.c
index 21a0435c02de..7a28e082436e 100644
--- a/drivers/net/net_failover.c
+++ b/drivers/net/net_failover.c
@@ -324,8 +324,8 @@ static const struct net_device_ops failover_dev_ops = {
 static void nfo_ethtool_get_drvinfo(struct net_device *dev,
 				    struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, FAILOVER_NAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, FAILOVER_VERSION, sizeof(drvinfo->version));
+	strscpy(drvinfo->driver, FAILOVER_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, FAILOVER_VERSION, sizeof(drvinfo->version));
 }
 
 static int nfo_ethtool_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c
index ddac61d79145..bdff9ac5056d 100644
--- a/drivers/net/netconsole.c
+++ b/drivers/net/netconsole.c
@@ -55,7 +55,7 @@ MODULE_PARM_DESC(oops_only, "Only log oops messages");
 #ifndef	MODULE
 static int __init option_setup(char *opt)
 {
-	strlcpy(config, opt, MAX_PARAM_LENGTH);
+	strscpy(config, opt, MAX_PARAM_LENGTH);
 	return 1;
 }
 __setup("netconsole=", option_setup);
@@ -178,7 +178,7 @@ static struct netconsole_target *alloc_param_target(char *target_config)
 		goto fail;
 
 	nt->np.name = "netconsole";
-	strlcpy(nt->np.dev_name, "eth0", IFNAMSIZ);
+	strscpy(nt->np.dev_name, "eth0", IFNAMSIZ);
 	nt->np.local_port = 6665;
 	nt->np.remote_port = 6666;
 	eth_broadcast_addr(nt->np.remote_mac);
@@ -414,7 +414,7 @@ static ssize_t dev_name_store(struct config_item *item, const char *buf,
 		return -EINVAL;
 	}
 
-	strlcpy(nt->np.dev_name, buf, IFNAMSIZ);
+	strscpy(nt->np.dev_name, buf, IFNAMSIZ);
 
 	/* Get rid of possible trailing newline from echo(1) */
 	len = strnlen(nt->np.dev_name, IFNAMSIZ);
@@ -630,7 +630,7 @@ static struct config_item *make_netconsole_target(struct config_group *group,
 		return ERR_PTR(-ENOMEM);
 
 	nt->np.name = "netconsole";
-	strlcpy(nt->np.dev_name, "eth0", IFNAMSIZ);
+	strscpy(nt->np.dev_name, "eth0", IFNAMSIZ);
 	nt->np.local_port = 6665;
 	nt->np.remote_port = 6666;
 	eth_broadcast_addr(nt->np.remote_mac);
@@ -708,7 +708,7 @@ restart:
 		if (nt->np.dev == dev) {
 			switch (event) {
 			case NETDEV_CHANGENAME:
-				strlcpy(nt->np.dev_name, dev->name, IFNAMSIZ);
+				strscpy(nt->np.dev_name, dev->name, IFNAMSIZ);
 				break;
 			case NETDEV_RELEASE:
 			case NETDEV_JOIN:
diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c
index e88f783c297e..794fc0cc73b8 100644
--- a/drivers/net/netdevsim/dev.c
+++ b/drivers/net/netdevsim/dev.c
@@ -965,7 +965,6 @@ static int nsim_dev_reload_up(struct devlink *devlink, enum devlink_reload_actio
 			      struct netlink_ext_ack *extack)
 {
 	struct nsim_dev *nsim_dev = devlink_priv(devlink);
-	int ret;
 
 	if (nsim_dev->fail_reload) {
 		/* For testing purposes, user set debugfs fail_reload
@@ -976,15 +975,25 @@ static int nsim_dev_reload_up(struct devlink *devlink, enum devlink_reload_actio
 	}
 
 	*actions_performed = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT);
-	ret = nsim_dev_reload_create(nsim_dev, extack);
-	return ret;
+
+	return nsim_dev_reload_create(nsim_dev, extack);
 }
 
 static int nsim_dev_info_get(struct devlink *devlink,
 			     struct devlink_info_req *req,
 			     struct netlink_ext_ack *extack)
 {
-	return devlink_info_driver_name_put(req, DRV_NAME);
+	int err;
+
+	err = devlink_info_driver_name_put(req, DRV_NAME);
+	if (err)
+		return err;
+	err = devlink_info_version_stored_put_ext(req, "fw.mgmt", "10.20.30",
+						  DEVLINK_INFO_VERSION_TYPE_COMPONENT);
+	if (err)
+		return err;
+	return devlink_info_version_running_put_ext(req, "fw.mgmt", "10.20.30",
+						    DEVLINK_INFO_VERSION_TYPE_COMPONENT);
 }
 
 #define NSIM_DEV_FLASH_SIZE 500000
@@ -1312,8 +1321,7 @@ nsim_dev_devlink_trap_drop_counter_get(struct devlink *devlink,
 static const struct devlink_ops nsim_dev_devlink_ops = {
 	.eswitch_mode_set = nsim_devlink_eswitch_mode_set,
 	.eswitch_mode_get = nsim_devlink_eswitch_mode_get,
-	.supported_flash_update_params = DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT |
-					 DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK,
+	.supported_flash_update_params = DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK,
 	.reload_actions = BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT),
 	.reload_down = nsim_dev_reload_down,
 	.reload_up = nsim_dev_reload_up,
diff --git a/drivers/net/ntb_netdev.c b/drivers/net/ntb_netdev.c
index 80bdc07f2cd3..464d88ca8ab0 100644
--- a/drivers/net/ntb_netdev.c
+++ b/drivers/net/ntb_netdev.c
@@ -364,9 +364,9 @@ static void ntb_get_drvinfo(struct net_device *ndev,
 {
 	struct ntb_netdev *dev = netdev_priv(ndev);
 
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
-	strlcpy(info->version, NTB_NETDEV_VER, sizeof(info->version));
-	strlcpy(info->bus_info, pci_name(dev->pdev), sizeof(info->bus_info));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->version, NTB_NETDEV_VER, sizeof(info->version));
+	strscpy(info->bus_info, pci_name(dev->pdev), sizeof(info->bus_info));
 }
 
 static int ntb_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/pcs/Kconfig b/drivers/net/pcs/Kconfig
index 6289b7c765f1..6e7e6c346a3e 100644
--- a/drivers/net/pcs/Kconfig
+++ b/drivers/net/pcs/Kconfig
@@ -26,4 +26,10 @@ config PCS_RZN1_MIIC
 	  on RZ/N1 SoCs. This PCS converts MII to RMII/RGMII or can be set in
 	  pass-through mode for MII.
 
+config PCS_ALTERA_TSE
+	tristate
+	help
+	  This module provides helper functions for the Altera Triple Speed
+	  Ethernet SGMII PCS, that can be found on the Intel Socfpga family.
+
 endmenu
diff --git a/drivers/net/pcs/Makefile b/drivers/net/pcs/Makefile
index 0ff5388fcdea..4c780d8f2e98 100644
--- a/drivers/net/pcs/Makefile
+++ b/drivers/net/pcs/Makefile
@@ -6,3 +6,4 @@ pcs_xpcs-$(CONFIG_PCS_XPCS)	:= pcs-xpcs.o pcs-xpcs-nxp.o
 obj-$(CONFIG_PCS_XPCS)		+= pcs_xpcs.o
 obj-$(CONFIG_PCS_LYNX)		+= pcs-lynx.o
 obj-$(CONFIG_PCS_RZN1_MIIC)	+= pcs-rzn1-miic.o
+obj-$(CONFIG_PCS_ALTERA_TSE)	+= pcs-altera-tse.o
diff --git a/drivers/net/pcs/pcs-altera-tse.c b/drivers/net/pcs/pcs-altera-tse.c
new file mode 100644
index 000000000000..97a7cabff962
--- /dev/null
+++ b/drivers/net/pcs/pcs-altera-tse.c
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2022 Bootlin
+ *
+ * Maxime Chevallier <maxime.chevallier@bootlin.com>
+ */
+
+#include <linux/netdevice.h>
+#include <linux/phy.h>
+#include <linux/phylink.h>
+#include <linux/pcs-altera-tse.h>
+
+/* SGMII PCS register addresses
+ */
+#define SGMII_PCS_SCRATCH	0x10
+#define SGMII_PCS_REV		0x11
+#define SGMII_PCS_LINK_TIMER_0	0x12
+#define   SGMII_PCS_LINK_TIMER_REG(x)		(0x12 + (x))
+#define SGMII_PCS_LINK_TIMER_1	0x13
+#define SGMII_PCS_IF_MODE	0x14
+#define   PCS_IF_MODE_SGMII_ENA		BIT(0)
+#define   PCS_IF_MODE_USE_SGMII_AN	BIT(1)
+#define   PCS_IF_MODE_SGMI_SPEED_MASK	GENMASK(3, 2)
+#define   PCS_IF_MODE_SGMI_SPEED_10	(0 << 2)
+#define   PCS_IF_MODE_SGMI_SPEED_100	(1 << 2)
+#define   PCS_IF_MODE_SGMI_SPEED_1000	(2 << 2)
+#define   PCS_IF_MODE_SGMI_HALF_DUPLEX	BIT(4)
+#define   PCS_IF_MODE_SGMI_PHY_AN	BIT(5)
+#define SGMII_PCS_DIS_READ_TO	0x15
+#define SGMII_PCS_READ_TO	0x16
+#define SGMII_PCS_SW_RESET_TIMEOUT 100 /* usecs */
+
+struct altera_tse_pcs {
+	struct phylink_pcs pcs;
+	void __iomem *base;
+	int reg_width;
+};
+
+static struct altera_tse_pcs *phylink_pcs_to_tse_pcs(struct phylink_pcs *pcs)
+{
+	return container_of(pcs, struct altera_tse_pcs, pcs);
+}
+
+static u16 tse_pcs_read(struct altera_tse_pcs *tse_pcs, int regnum)
+{
+	if (tse_pcs->reg_width == 4)
+		return readl(tse_pcs->base + regnum * 4);
+	else
+		return readw(tse_pcs->base + regnum * 2);
+}
+
+static void tse_pcs_write(struct altera_tse_pcs *tse_pcs, int regnum,
+			  u16 value)
+{
+	if (tse_pcs->reg_width == 4)
+		writel(value, tse_pcs->base + regnum * 4);
+	else
+		writew(value, tse_pcs->base + regnum * 2);
+}
+
+static int tse_pcs_reset(struct altera_tse_pcs *tse_pcs)
+{
+	int i = 0;
+	u16 bmcr;
+
+	/* Reset PCS block */
+	bmcr = tse_pcs_read(tse_pcs, MII_BMCR);
+	bmcr |= BMCR_RESET;
+	tse_pcs_write(tse_pcs, MII_BMCR, bmcr);
+
+	for (i = 0; i < SGMII_PCS_SW_RESET_TIMEOUT; i++) {
+		if (!(tse_pcs_read(tse_pcs, MII_BMCR) & BMCR_RESET))
+			return 0;
+		udelay(1);
+	}
+
+	return -ETIMEDOUT;
+}
+
+static int alt_tse_pcs_validate(struct phylink_pcs *pcs,
+				unsigned long *supported,
+				const struct phylink_link_state *state)
+{
+	if (state->interface == PHY_INTERFACE_MODE_SGMII ||
+	    state->interface == PHY_INTERFACE_MODE_1000BASEX)
+		return 1;
+
+	return -EINVAL;
+}
+
+static int alt_tse_pcs_config(struct phylink_pcs *pcs, unsigned int mode,
+			      phy_interface_t interface,
+			      const unsigned long *advertising,
+			      bool permit_pause_to_mac)
+{
+	struct altera_tse_pcs *tse_pcs = phylink_pcs_to_tse_pcs(pcs);
+	u32 ctrl, if_mode;
+
+	ctrl = tse_pcs_read(tse_pcs, MII_BMCR);
+	if_mode = tse_pcs_read(tse_pcs, SGMII_PCS_IF_MODE);
+
+	/* Set link timer to 1.6ms, as per the MegaCore Function User Guide */
+	tse_pcs_write(tse_pcs, SGMII_PCS_LINK_TIMER_0, 0x0D40);
+	tse_pcs_write(tse_pcs, SGMII_PCS_LINK_TIMER_1, 0x03);
+
+	if (interface == PHY_INTERFACE_MODE_SGMII) {
+		if_mode |= PCS_IF_MODE_USE_SGMII_AN | PCS_IF_MODE_SGMII_ENA;
+	} else if (interface == PHY_INTERFACE_MODE_1000BASEX) {
+		if_mode &= ~(PCS_IF_MODE_USE_SGMII_AN | PCS_IF_MODE_SGMII_ENA);
+		if_mode |= PCS_IF_MODE_SGMI_SPEED_1000;
+	}
+
+	ctrl |= (BMCR_SPEED1000 | BMCR_FULLDPLX | BMCR_ANENABLE);
+
+	tse_pcs_write(tse_pcs, MII_BMCR, ctrl);
+	tse_pcs_write(tse_pcs, SGMII_PCS_IF_MODE, if_mode);
+
+	return tse_pcs_reset(tse_pcs);
+}
+
+static void alt_tse_pcs_get_state(struct phylink_pcs *pcs,
+				  struct phylink_link_state *state)
+{
+	struct altera_tse_pcs *tse_pcs = phylink_pcs_to_tse_pcs(pcs);
+	u16 bmsr, lpa;
+
+	bmsr = tse_pcs_read(tse_pcs, MII_BMSR);
+	lpa = tse_pcs_read(tse_pcs, MII_LPA);
+
+	phylink_mii_c22_pcs_decode_state(state, bmsr, lpa);
+}
+
+static void alt_tse_pcs_an_restart(struct phylink_pcs *pcs)
+{
+	struct altera_tse_pcs *tse_pcs = phylink_pcs_to_tse_pcs(pcs);
+	u16 bmcr;
+
+	bmcr = tse_pcs_read(tse_pcs, MII_BMCR);
+	bmcr |= BMCR_ANRESTART;
+	tse_pcs_write(tse_pcs, MII_BMCR, bmcr);
+
+	/* This PCS seems to require a soft reset to re-sync the AN logic */
+	tse_pcs_reset(tse_pcs);
+}
+
+static const struct phylink_pcs_ops alt_tse_pcs_ops = {
+	.pcs_validate = alt_tse_pcs_validate,
+	.pcs_get_state = alt_tse_pcs_get_state,
+	.pcs_config = alt_tse_pcs_config,
+	.pcs_an_restart = alt_tse_pcs_an_restart,
+};
+
+struct phylink_pcs *alt_tse_pcs_create(struct net_device *ndev,
+				       void __iomem *pcs_base, int reg_width)
+{
+	struct altera_tse_pcs *tse_pcs;
+
+	if (reg_width != 4 && reg_width != 2)
+		return ERR_PTR(-EINVAL);
+
+	tse_pcs = devm_kzalloc(&ndev->dev, sizeof(*tse_pcs), GFP_KERNEL);
+	if (!tse_pcs)
+		return ERR_PTR(-ENOMEM);
+
+	tse_pcs->pcs.ops = &alt_tse_pcs_ops;
+	tse_pcs->base = pcs_base;
+	tse_pcs->reg_width = reg_width;
+
+	return &tse_pcs->pcs;
+}
+EXPORT_SYMBOL_GPL(alt_tse_pcs_create);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Altera TSE PCS driver");
+MODULE_AUTHOR("Maxime Chevallier <maxime.chevallier@bootlin.com>");
diff --git a/drivers/net/phy/adin.c b/drivers/net/phy/adin.c
index ee374a85544a..134637584a83 100644
--- a/drivers/net/phy/adin.c
+++ b/drivers/net/phy/adin.c
@@ -749,7 +749,7 @@ static void adin_get_strings(struct phy_device *phydev, u8 *data)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(adin_hw_stats); i++) {
-		strlcpy(&data[i * ETH_GSTRING_LEN],
+		strscpy(&data[i * ETH_GSTRING_LEN],
 			adin_hw_stats[i].string, ETH_GSTRING_LEN);
 	}
 }
diff --git a/drivers/net/phy/adin1100.c b/drivers/net/phy/adin1100.c
index b6d139501199..7619d6185801 100644
--- a/drivers/net/phy/adin1100.c
+++ b/drivers/net/phy/adin1100.c
@@ -15,6 +15,8 @@
 #include <linux/property.h>
 
 #define PHY_ID_ADIN1100				0x0283bc81
+#define PHY_ID_ADIN1110				0x0283bc91
+#define PHY_ID_ADIN2111				0x0283bca1
 
 #define ADIN_FORCED_MODE			0x8000
 #define   ADIN_FORCED_MODE_EN			BIT(0)
@@ -265,7 +267,8 @@ static int adin_probe(struct phy_device *phydev)
 
 static struct phy_driver adin_driver[] = {
 	{
-		PHY_ID_MATCH_MODEL(PHY_ID_ADIN1100),
+		.phy_id			= PHY_ID_ADIN1100,
+		.phy_id_mask		= 0xffffffcf,
 		.name			= "ADIN1100",
 		.get_features		= adin_get_features,
 		.soft_reset		= adin_soft_reset,
@@ -284,6 +287,8 @@ module_phy_driver(adin_driver);
 
 static struct mdio_device_id __maybe_unused adin_tbl[] = {
 	{ PHY_ID_MATCH_MODEL(PHY_ID_ADIN1100) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_ADIN1110) },
+	{ PHY_ID_MATCH_MODEL(PHY_ID_ADIN2111) },
 	{ }
 };
 
diff --git a/drivers/net/phy/aquantia_main.c b/drivers/net/phy/aquantia_main.c
index 7111e2e958e9..47a76df36b74 100644
--- a/drivers/net/phy/aquantia_main.c
+++ b/drivers/net/phy/aquantia_main.c
@@ -27,9 +27,12 @@
 #define MDIO_PHYXS_VEND_IF_STATUS		0xe812
 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_MASK	GENMASK(7, 3)
 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_KR	0
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_KX	1
 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_XFI	2
 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_USXGMII	3
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_XAUI	4
 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_SGMII	6
+#define MDIO_PHYXS_VEND_IF_STATUS_TYPE_RXAUI	7
 #define MDIO_PHYXS_VEND_IF_STATUS_TYPE_OCSGMII	10
 
 #define MDIO_AN_VEND_PROV			0xc400
@@ -94,6 +97,19 @@
 #define VEND1_GLOBAL_GEN_STAT2			0xc831
 #define VEND1_GLOBAL_GEN_STAT2_OP_IN_PROG	BIT(15)
 
+/* The following registers all have similar layouts; first the registers... */
+#define VEND1_GLOBAL_CFG_10M			0x0310
+#define VEND1_GLOBAL_CFG_100M			0x031b
+#define VEND1_GLOBAL_CFG_1G			0x031c
+#define VEND1_GLOBAL_CFG_2_5G			0x031d
+#define VEND1_GLOBAL_CFG_5G			0x031e
+#define VEND1_GLOBAL_CFG_10G			0x031f
+/* ...and now the fields */
+#define VEND1_GLOBAL_CFG_RATE_ADAPT		GENMASK(8, 7)
+#define VEND1_GLOBAL_CFG_RATE_ADAPT_NONE	0
+#define VEND1_GLOBAL_CFG_RATE_ADAPT_USX		1
+#define VEND1_GLOBAL_CFG_RATE_ADAPT_PAUSE	2
+
 #define VEND1_GLOBAL_RSVD_STAT1			0xc885
 #define VEND1_GLOBAL_RSVD_STAT1_FW_BUILD_ID	GENMASK(7, 4)
 #define VEND1_GLOBAL_RSVD_STAT1_PROV_ID		GENMASK(3, 0)
@@ -344,40 +360,57 @@ static int aqr_read_status(struct phy_device *phydev)
 
 static int aqr107_read_rate(struct phy_device *phydev)
 {
+	u32 config_reg;
 	int val;
 
 	val = phy_read_mmd(phydev, MDIO_MMD_AN, MDIO_AN_TX_VEND_STATUS1);
 	if (val < 0)
 		return val;
 
+	if (val & MDIO_AN_TX_VEND_STATUS1_FULL_DUPLEX)
+		phydev->duplex = DUPLEX_FULL;
+	else
+		phydev->duplex = DUPLEX_HALF;
+
 	switch (FIELD_GET(MDIO_AN_TX_VEND_STATUS1_RATE_MASK, val)) {
 	case MDIO_AN_TX_VEND_STATUS1_10BASET:
 		phydev->speed = SPEED_10;
+		config_reg = VEND1_GLOBAL_CFG_10M;
 		break;
 	case MDIO_AN_TX_VEND_STATUS1_100BASETX:
 		phydev->speed = SPEED_100;
+		config_reg = VEND1_GLOBAL_CFG_100M;
 		break;
 	case MDIO_AN_TX_VEND_STATUS1_1000BASET:
 		phydev->speed = SPEED_1000;
+		config_reg = VEND1_GLOBAL_CFG_1G;
 		break;
 	case MDIO_AN_TX_VEND_STATUS1_2500BASET:
 		phydev->speed = SPEED_2500;
+		config_reg = VEND1_GLOBAL_CFG_2_5G;
 		break;
 	case MDIO_AN_TX_VEND_STATUS1_5000BASET:
 		phydev->speed = SPEED_5000;
+		config_reg = VEND1_GLOBAL_CFG_5G;
 		break;
 	case MDIO_AN_TX_VEND_STATUS1_10GBASET:
 		phydev->speed = SPEED_10000;
+		config_reg = VEND1_GLOBAL_CFG_10G;
 		break;
 	default:
 		phydev->speed = SPEED_UNKNOWN;
-		break;
+		return 0;
 	}
 
-	if (val & MDIO_AN_TX_VEND_STATUS1_FULL_DUPLEX)
-		phydev->duplex = DUPLEX_FULL;
+	val = phy_read_mmd(phydev, MDIO_MMD_VEND1, config_reg);
+	if (val < 0)
+		return val;
+
+	if (FIELD_GET(VEND1_GLOBAL_CFG_RATE_ADAPT, val) ==
+	    VEND1_GLOBAL_CFG_RATE_ADAPT_PAUSE)
+		phydev->rate_matching = RATE_MATCH_PAUSE;
 	else
-		phydev->duplex = DUPLEX_HALF;
+		phydev->rate_matching = RATE_MATCH_NONE;
 
 	return 0;
 }
@@ -401,15 +434,24 @@ static int aqr107_read_status(struct phy_device *phydev)
 	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_KR:
 		phydev->interface = PHY_INTERFACE_MODE_10GKR;
 		break;
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_KX:
+		phydev->interface = PHY_INTERFACE_MODE_1000BASEKX;
+		break;
 	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_XFI:
 		phydev->interface = PHY_INTERFACE_MODE_10GBASER;
 		break;
 	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_USXGMII:
 		phydev->interface = PHY_INTERFACE_MODE_USXGMII;
 		break;
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_XAUI:
+		phydev->interface = PHY_INTERFACE_MODE_XAUI;
+		break;
 	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_SGMII:
 		phydev->interface = PHY_INTERFACE_MODE_SGMII;
 		break;
+	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_RXAUI:
+		phydev->interface = PHY_INTERFACE_MODE_RXAUI;
+		break;
 	case MDIO_PHYXS_VEND_IF_STATUS_TYPE_OCSGMII:
 		phydev->interface = PHY_INTERFACE_MODE_2500BASEX;
 		break;
@@ -522,11 +564,14 @@ static int aqr107_config_init(struct phy_device *phydev)
 
 	/* Check that the PHY interface type is compatible */
 	if (phydev->interface != PHY_INTERFACE_MODE_SGMII &&
+	    phydev->interface != PHY_INTERFACE_MODE_1000BASEKX &&
 	    phydev->interface != PHY_INTERFACE_MODE_2500BASEX &&
 	    phydev->interface != PHY_INTERFACE_MODE_XGMII &&
 	    phydev->interface != PHY_INTERFACE_MODE_USXGMII &&
 	    phydev->interface != PHY_INTERFACE_MODE_10GKR &&
-	    phydev->interface != PHY_INTERFACE_MODE_10GBASER)
+	    phydev->interface != PHY_INTERFACE_MODE_10GBASER &&
+	    phydev->interface != PHY_INTERFACE_MODE_XAUI &&
+	    phydev->interface != PHY_INTERFACE_MODE_RXAUI)
 		return -ENODEV;
 
 	WARN(phydev->interface == PHY_INTERFACE_MODE_XGMII,
@@ -630,6 +675,16 @@ static int aqr107_wait_processor_intensive_op(struct phy_device *phydev)
 	return 0;
 }
 
+static int aqr107_get_rate_matching(struct phy_device *phydev,
+				    phy_interface_t iface)
+{
+	if (iface == PHY_INTERFACE_MODE_10GBASER ||
+	    iface == PHY_INTERFACE_MODE_2500BASEX ||
+	    iface == PHY_INTERFACE_MODE_NA)
+		return RATE_MATCH_PAUSE;
+	return RATE_MATCH_NONE;
+}
+
 static int aqr107_suspend(struct phy_device *phydev)
 {
 	int err;
@@ -703,6 +758,7 @@ static struct phy_driver aqr_driver[] = {
 	PHY_ID_MATCH_MODEL(PHY_ID_AQR107),
 	.name		= "Aquantia AQR107",
 	.probe		= aqr107_probe,
+	.get_rate_matching = aqr107_get_rate_matching,
 	.config_init	= aqr107_config_init,
 	.config_aneg    = aqr_config_aneg,
 	.config_intr	= aqr_config_intr,
@@ -721,6 +777,7 @@ static struct phy_driver aqr_driver[] = {
 	PHY_ID_MATCH_MODEL(PHY_ID_AQCS109),
 	.name		= "Aquantia AQCS109",
 	.probe		= aqr107_probe,
+	.get_rate_matching = aqr107_get_rate_matching,
 	.config_init	= aqcs109_config_init,
 	.config_aneg    = aqr_config_aneg,
 	.config_intr	= aqr_config_intr,
@@ -747,6 +804,7 @@ static struct phy_driver aqr_driver[] = {
 	PHY_ID_MATCH_MODEL(PHY_ID_AQR113C),
 	.name           = "Aquantia AQR113C",
 	.probe          = aqr107_probe,
+	.get_rate_matching = aqr107_get_rate_matching,
 	.config_init    = aqr107_config_init,
 	.config_aneg    = aqr_config_aneg,
 	.config_intr    = aqr_config_intr,
diff --git a/drivers/net/phy/at803x.c b/drivers/net/phy/at803x.c
index 59fe356942b5..9e9adde335c8 100644
--- a/drivers/net/phy/at803x.c
+++ b/drivers/net/phy/at803x.c
@@ -115,6 +115,7 @@
 #define AT803X_DEBUG_REG_HIB_CTRL		0x0b
 #define   AT803X_DEBUG_HIB_CTRL_SEL_RST_80U	BIT(10)
 #define   AT803X_DEBUG_HIB_CTRL_EN_ANY_CHANGE	BIT(13)
+#define   AT803X_DEBUG_HIB_CTRL_PS_HIB_EN	BIT(15)
 
 #define AT803X_DEBUG_REG_3C			0x3C
 
@@ -192,6 +193,9 @@
 #define AT803X_KEEP_PLL_ENABLED			BIT(0)
 #define AT803X_DISABLE_SMARTEEE			BIT(1)
 
+/* disable hibernation mode */
+#define AT803X_DISABLE_HIBERNATION_MODE		BIT(2)
+
 /* ADC threshold */
 #define QCA808X_PHY_DEBUG_ADC_THRESHOLD		0x2c80
 #define QCA808X_ADC_THRESHOLD_MASK		GENMASK(7, 0)
@@ -672,6 +676,7 @@ static int at803x_sfp_insert(void *upstream, const struct sfp_eeprom_id *id)
 	struct phy_device *phydev = upstream;
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(phy_support);
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(sfp_support);
+	DECLARE_PHY_INTERFACE_MASK(interfaces);
 	phy_interface_t iface;
 
 	linkmode_zero(phy_support);
@@ -682,7 +687,7 @@ static int at803x_sfp_insert(void *upstream, const struct sfp_eeprom_id *id)
 	phylink_set(phy_support, Asym_Pause);
 
 	linkmode_zero(sfp_support);
-	sfp_parse_support(phydev->sfp_bus, id, sfp_support);
+	sfp_parse_support(phydev->sfp_bus, id, sfp_support, interfaces);
 	/* Some modules support 10G modes as well as others we support.
 	 * Mask out non-supported modes so the correct interface is picked.
 	 */
@@ -730,6 +735,9 @@ static int at803x_parse_dt(struct phy_device *phydev)
 	if (of_property_read_bool(node, "qca,disable-smarteee"))
 		priv->flags |= AT803X_DISABLE_SMARTEEE;
 
+	if (of_property_read_bool(node, "qca,disable-hibernation-mode"))
+		priv->flags |= AT803X_DISABLE_HIBERNATION_MODE;
+
 	if (!of_property_read_u32(node, "qca,smarteee-tw-us-1g", &tw)) {
 		if (!tw || tw > 255) {
 			phydev_err(phydev, "invalid qca,smarteee-tw-us-1g\n");
@@ -999,6 +1007,20 @@ static int at8031_pll_config(struct phy_device *phydev)
 					     AT803X_DEBUG_PLL_ON, 0);
 }
 
+static int at803x_hibernation_mode_config(struct phy_device *phydev)
+{
+	struct at803x_priv *priv = phydev->priv;
+
+	/* The default after hardware reset is hibernation mode enabled. After
+	 * software reset, the value is retained.
+	 */
+	if (!(priv->flags & AT803X_DISABLE_HIBERNATION_MODE))
+		return 0;
+
+	return at803x_debug_reg_mask(phydev, AT803X_DEBUG_REG_HIB_CTRL,
+					 AT803X_DEBUG_HIB_CTRL_PS_HIB_EN, 0);
+}
+
 static int at803x_config_init(struct phy_device *phydev)
 {
 	struct at803x_priv *priv = phydev->priv;
@@ -1051,6 +1073,10 @@ static int at803x_config_init(struct phy_device *phydev)
 	if (ret < 0)
 		return ret;
 
+	ret = at803x_hibernation_mode_config(phydev);
+	if (ret < 0)
+		return ret;
+
 	/* Ar803x extended next page bit is enabled by default. Cisco
 	 * multigig switches read this bit and attempt to negotiate 10Gbps
 	 * rates even if the next page bit is disabled. This is incorrect
diff --git a/drivers/net/phy/bcm-phy-lib.c b/drivers/net/phy/bcm-phy-lib.c
index 287cccf8f7f4..b2c0baa51f39 100644
--- a/drivers/net/phy/bcm-phy-lib.c
+++ b/drivers/net/phy/bcm-phy-lib.c
@@ -519,7 +519,7 @@ void bcm_phy_get_strings(struct phy_device *phydev, u8 *data)
 	unsigned int i;
 
 	for (i = 0; i < ARRAY_SIZE(bcm_phy_hw_stats); i++)
-		strlcpy(data + i * ETH_GSTRING_LEN,
+		strscpy(data + i * ETH_GSTRING_LEN,
 			bcm_phy_hw_stats[i].string, ETH_GSTRING_LEN);
 }
 EXPORT_SYMBOL_GPL(bcm_phy_get_strings);
diff --git a/drivers/net/phy/broadcom.c b/drivers/net/phy/broadcom.c
index 31fbcdddc9ad..ad71c88c87e7 100644
--- a/drivers/net/phy/broadcom.c
+++ b/drivers/net/phy/broadcom.c
@@ -766,6 +766,41 @@ static irqreturn_t brcm_fet_handle_interrupt(struct phy_device *phydev)
 	return IRQ_HANDLED;
 }
 
+static int brcm_fet_suspend(struct phy_device *phydev)
+{
+	int reg, err, err2, brcmtest;
+
+	/* We cannot use a read/modify/write here otherwise the PHY continues
+	 * to drive LEDs which defeats the purpose of low power mode.
+	 */
+	err = phy_write(phydev, MII_BMCR, BMCR_PDOWN);
+	if (err < 0)
+		return err;
+
+	/* Enable shadow register access */
+	brcmtest = phy_read(phydev, MII_BRCM_FET_BRCMTEST);
+	if (brcmtest < 0)
+		return brcmtest;
+
+	reg = brcmtest | MII_BRCM_FET_BT_SRE;
+
+	err = phy_write(phydev, MII_BRCM_FET_BRCMTEST, reg);
+	if (err < 0)
+		return err;
+
+	/* Set standby mode */
+	err = phy_modify(phydev, MII_BRCM_FET_SHDW_AUXMODE4,
+			 MII_BRCM_FET_SHDW_AM4_STANDBY,
+			 MII_BRCM_FET_SHDW_AM4_STANDBY);
+
+	/* Disable shadow register access */
+	err2 = phy_write(phydev, MII_BRCM_FET_BRCMTEST, brcmtest);
+	if (!err)
+		err = err2;
+
+	return err;
+}
+
 static int bcm54xx_phy_probe(struct phy_device *phydev)
 {
 	struct bcm54xx_phy_priv *priv;
@@ -1033,6 +1068,8 @@ static struct phy_driver broadcom_drivers[] = {
 	.config_init	= brcm_fet_config_init,
 	.config_intr	= brcm_fet_config_intr,
 	.handle_interrupt = brcm_fet_handle_interrupt,
+	.suspend	= brcm_fet_suspend,
+	.resume		= brcm_fet_config_init,
 }, {
 	.phy_id		= PHY_ID_BCM5241,
 	.phy_id_mask	= 0xfffffff0,
@@ -1041,6 +1078,8 @@ static struct phy_driver broadcom_drivers[] = {
 	.config_init	= brcm_fet_config_init,
 	.config_intr	= brcm_fet_config_intr,
 	.handle_interrupt = brcm_fet_handle_interrupt,
+	.suspend	= brcm_fet_suspend,
+	.resume		= brcm_fet_config_init,
 }, {
 	.phy_id		= PHY_ID_BCM5395,
 	.phy_id_mask	= 0xfffffff0,
diff --git a/drivers/net/phy/marvell-88x2222.c b/drivers/net/phy/marvell-88x2222.c
index f070776ca904..fd9ad4820192 100644
--- a/drivers/net/phy/marvell-88x2222.c
+++ b/drivers/net/phy/marvell-88x2222.c
@@ -478,6 +478,7 @@ static int mv2222_config_init(struct phy_device *phydev)
 
 static int mv2222_sfp_insert(void *upstream, const struct sfp_eeprom_id *id)
 {
+	DECLARE_PHY_INTERFACE_MASK(interfaces);
 	struct phy_device *phydev = upstream;
 	phy_interface_t sfp_interface;
 	struct mv2222_data *priv;
@@ -489,7 +490,7 @@ static int mv2222_sfp_insert(void *upstream, const struct sfp_eeprom_id *id)
 	priv = (struct mv2222_data *)phydev->priv;
 	dev = &phydev->mdio.dev;
 
-	sfp_parse_support(phydev->sfp_bus, id, sfp_supported);
+	sfp_parse_support(phydev->sfp_bus, id, sfp_supported, interfaces);
 	phydev->port = sfp_parse_port(phydev->sfp_bus, id, sfp_supported);
 	sfp_interface = sfp_select_interface(phydev->sfp_bus, sfp_supported);
 
diff --git a/drivers/net/phy/marvell.c b/drivers/net/phy/marvell.c
index a714150f5e8c..2810f4f9da0c 100644
--- a/drivers/net/phy/marvell.c
+++ b/drivers/net/phy/marvell.c
@@ -1952,7 +1952,7 @@ static void marvell_get_strings(struct phy_device *phydev, u8 *data)
 	int i;
 
 	for (i = 0; i < count; i++) {
-		strlcpy(data + i * ETH_GSTRING_LEN,
+		strscpy(data + i * ETH_GSTRING_LEN,
 			marvell_hw_stats[i].string, ETH_GSTRING_LEN);
 	}
 }
@@ -2845,6 +2845,7 @@ static int marvell_probe(struct phy_device *phydev)
 
 static int m88e1510_sfp_insert(void *upstream, const struct sfp_eeprom_id *id)
 {
+	DECLARE_PHY_INTERFACE_MASK(interfaces);
 	struct phy_device *phydev = upstream;
 	phy_interface_t interface;
 	struct device *dev;
@@ -2856,7 +2857,7 @@ static int m88e1510_sfp_insert(void *upstream, const struct sfp_eeprom_id *id)
 
 	dev = &phydev->mdio.dev;
 
-	sfp_parse_support(phydev->sfp_bus, id, supported);
+	sfp_parse_support(phydev->sfp_bus, id, supported, interfaces);
 	interface = sfp_select_interface(phydev->sfp_bus, supported);
 
 	dev_info(dev, "%s SFP module inserted\n", phy_modes(interface));
diff --git a/drivers/net/phy/marvell10g.c b/drivers/net/phy/marvell10g.c
index 2b7d0720720b..383a9c9f36e5 100644
--- a/drivers/net/phy/marvell10g.c
+++ b/drivers/net/phy/marvell10g.c
@@ -96,6 +96,11 @@ enum {
 	MV_PCS_PORT_INFO_NPORTS_MASK	= 0x0380,
 	MV_PCS_PORT_INFO_NPORTS_SHIFT	= 7,
 
+	/* SerDes reinitialization 88E21X0 */
+	MV_AN_21X0_SERDES_CTRL2	= 0x800f,
+	MV_AN_21X0_SERDES_CTRL2_AUTO_INIT_DIS	= BIT(13),
+	MV_AN_21X0_SERDES_CTRL2_RUN_INIT	= BIT(15),
+
 	/* These registers appear at 0x800X and 0xa00X - the 0xa00X control
 	 * registers appear to set themselves to the 0x800X when AN is
 	 * restarted, but status registers appear readable from either.
@@ -117,16 +122,16 @@ enum {
 	MV_V2_33X0_PORT_CTRL_MACTYPE_10GBASER_NO_SGMII_AN	= 0x5,
 	MV_V2_33X0_PORT_CTRL_MACTYPE_10GBASER_RATE_MATCH	= 0x6,
 	MV_V2_33X0_PORT_CTRL_MACTYPE_USXGMII			= 0x7,
-	MV_V2_PORT_INTR_STS     = 0xf040,
-	MV_V2_PORT_INTR_MASK    = 0xf043,
-	MV_V2_PORT_INTR_STS_WOL_EN      = BIT(8),
-	MV_V2_MAGIC_PKT_WORD0   = 0xf06b,
-	MV_V2_MAGIC_PKT_WORD1   = 0xf06c,
-	MV_V2_MAGIC_PKT_WORD2   = 0xf06d,
+	MV_V2_PORT_INTR_STS		= 0xf040,
+	MV_V2_PORT_INTR_MASK		= 0xf043,
+	MV_V2_PORT_INTR_STS_WOL_EN	= BIT(8),
+	MV_V2_MAGIC_PKT_WORD0		= 0xf06b,
+	MV_V2_MAGIC_PKT_WORD1		= 0xf06c,
+	MV_V2_MAGIC_PKT_WORD2		= 0xf06d,
 	/* Wake on LAN registers */
-	MV_V2_WOL_CTRL          = 0xf06e,
-	MV_V2_WOL_CTRL_CLEAR_STS        = BIT(15),
-	MV_V2_WOL_CTRL_MAGIC_PKT_EN     = BIT(0),
+	MV_V2_WOL_CTRL			= 0xf06e,
+	MV_V2_WOL_CTRL_CLEAR_STS	= BIT(15),
+	MV_V2_WOL_CTRL_MAGIC_PKT_EN	= BIT(0),
 	/* Temperature control/read registers (88X3310 only) */
 	MV_V2_TEMP_CTRL		= 0xf08a,
 	MV_V2_TEMP_CTRL_MASK	= 0xc000,
@@ -140,6 +145,8 @@ struct mv3310_chip {
 	bool (*has_downshift)(struct phy_device *phydev);
 	void (*init_supported_interfaces)(unsigned long *mask);
 	int (*get_mactype)(struct phy_device *phydev);
+	int (*set_mactype)(struct phy_device *phydev, int mactype);
+	int (*select_mactype)(unsigned long *interfaces);
 	int (*init_interface)(struct phy_device *phydev, int mactype);
 
 #ifdef CONFIG_HWMON
@@ -466,9 +473,10 @@ static int mv3310_sfp_insert(void *upstream, const struct sfp_eeprom_id *id)
 {
 	struct phy_device *phydev = upstream;
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(support) = { 0, };
+	DECLARE_PHY_INTERFACE_MASK(interfaces);
 	phy_interface_t iface;
 
-	sfp_parse_support(phydev->sfp_bus, id, support);
+	sfp_parse_support(phydev->sfp_bus, id, support, interfaces);
 	iface = sfp_select_interface(phydev->sfp_bus, support);
 
 	if (iface != PHY_INTERFACE_MODE_10GBASER) {
@@ -593,6 +601,49 @@ static int mv2110_get_mactype(struct phy_device *phydev)
 	return mactype & MV_PMA_21X0_PORT_CTRL_MACTYPE_MASK;
 }
 
+static int mv2110_set_mactype(struct phy_device *phydev, int mactype)
+{
+	int err, val;
+
+	mactype &= MV_PMA_21X0_PORT_CTRL_MACTYPE_MASK;
+	err = phy_modify_mmd(phydev, MDIO_MMD_PMAPMD, MV_PMA_21X0_PORT_CTRL,
+			     MV_PMA_21X0_PORT_CTRL_SWRST |
+			     MV_PMA_21X0_PORT_CTRL_MACTYPE_MASK,
+			     MV_PMA_21X0_PORT_CTRL_SWRST | mactype);
+	if (err)
+		return err;
+
+	err = phy_set_bits_mmd(phydev, MDIO_MMD_AN, MV_AN_21X0_SERDES_CTRL2,
+			       MV_AN_21X0_SERDES_CTRL2_AUTO_INIT_DIS |
+			       MV_AN_21X0_SERDES_CTRL2_RUN_INIT);
+	if (err)
+		return err;
+
+	err = phy_read_mmd_poll_timeout(phydev, MDIO_MMD_AN,
+					MV_AN_21X0_SERDES_CTRL2, val,
+					!(val &
+					  MV_AN_21X0_SERDES_CTRL2_RUN_INIT),
+					5000, 100000, true);
+	if (err)
+		return err;
+
+	return phy_clear_bits_mmd(phydev, MDIO_MMD_AN, MV_AN_21X0_SERDES_CTRL2,
+				  MV_AN_21X0_SERDES_CTRL2_AUTO_INIT_DIS);
+}
+
+static int mv2110_select_mactype(unsigned long *interfaces)
+{
+	if (test_bit(PHY_INTERFACE_MODE_USXGMII, interfaces))
+		return MV_PMA_21X0_PORT_CTRL_MACTYPE_USXGMII;
+	else if (test_bit(PHY_INTERFACE_MODE_SGMII, interfaces) &&
+		 !test_bit(PHY_INTERFACE_MODE_10GBASER, interfaces))
+		return MV_PMA_21X0_PORT_CTRL_MACTYPE_5GBASER;
+	else if (test_bit(PHY_INTERFACE_MODE_10GBASER, interfaces))
+		return MV_PMA_21X0_PORT_CTRL_MACTYPE_10GBASER_RATE_MATCH;
+	else
+		return -1;
+}
+
 static int mv3310_get_mactype(struct phy_device *phydev)
 {
 	int mactype;
@@ -604,6 +655,46 @@ static int mv3310_get_mactype(struct phy_device *phydev)
 	return mactype & MV_V2_33X0_PORT_CTRL_MACTYPE_MASK;
 }
 
+static int mv3310_set_mactype(struct phy_device *phydev, int mactype)
+{
+	int ret;
+
+	mactype &= MV_V2_33X0_PORT_CTRL_MACTYPE_MASK;
+	ret = phy_modify_mmd_changed(phydev, MDIO_MMD_VEND2, MV_V2_PORT_CTRL,
+				     MV_V2_33X0_PORT_CTRL_MACTYPE_MASK,
+				     mactype);
+	if (ret <= 0)
+		return ret;
+
+	return phy_set_bits_mmd(phydev, MDIO_MMD_VEND2, MV_V2_PORT_CTRL,
+				MV_V2_33X0_PORT_CTRL_SWRST);
+}
+
+static int mv3310_select_mactype(unsigned long *interfaces)
+{
+	if (test_bit(PHY_INTERFACE_MODE_USXGMII, interfaces))
+		return MV_V2_33X0_PORT_CTRL_MACTYPE_USXGMII;
+	else if (test_bit(PHY_INTERFACE_MODE_SGMII, interfaces) &&
+		 test_bit(PHY_INTERFACE_MODE_10GBASER, interfaces))
+		return MV_V2_33X0_PORT_CTRL_MACTYPE_10GBASER;
+	else if (test_bit(PHY_INTERFACE_MODE_SGMII, interfaces) &&
+		 test_bit(PHY_INTERFACE_MODE_RXAUI, interfaces))
+		return MV_V2_33X0_PORT_CTRL_MACTYPE_RXAUI;
+	else if (test_bit(PHY_INTERFACE_MODE_SGMII, interfaces) &&
+		 test_bit(PHY_INTERFACE_MODE_XAUI, interfaces))
+		return MV_V2_3310_PORT_CTRL_MACTYPE_XAUI;
+	else if (test_bit(PHY_INTERFACE_MODE_10GBASER, interfaces))
+		return MV_V2_33X0_PORT_CTRL_MACTYPE_10GBASER_RATE_MATCH;
+	else if (test_bit(PHY_INTERFACE_MODE_RXAUI, interfaces))
+		return MV_V2_33X0_PORT_CTRL_MACTYPE_RXAUI_RATE_MATCH;
+	else if (test_bit(PHY_INTERFACE_MODE_XAUI, interfaces))
+		return MV_V2_3310_PORT_CTRL_MACTYPE_XAUI_RATE_MATCH;
+	else if (test_bit(PHY_INTERFACE_MODE_SGMII, interfaces))
+		return MV_V2_33X0_PORT_CTRL_MACTYPE_10GBASER;
+	else
+		return -1;
+}
+
 static int mv2110_init_interface(struct phy_device *phydev, int mactype)
 {
 	struct mv3310_priv *priv = dev_get_drvdata(&phydev->mdio.dev);
@@ -687,6 +778,20 @@ static int mv3310_config_init(struct phy_device *phydev)
 	if (err)
 		return err;
 
+	/* If host provided host supported interface modes, try to select the
+	 * best one
+	 */
+	if (!phy_interface_empty(phydev->host_interfaces)) {
+		mactype = chip->select_mactype(phydev->host_interfaces);
+		if (mactype >= 0) {
+			phydev_info(phydev, "Changing MACTYPE to %i\n",
+				    mactype);
+			err = chip->set_mactype(phydev, mactype);
+			if (err)
+				return err;
+		}
+	}
+
 	mactype = chip->get_mactype(phydev);
 	if (mactype < 0)
 		return mactype;
@@ -1049,6 +1154,8 @@ static const struct mv3310_chip mv3310_type = {
 	.has_downshift = mv3310_has_downshift,
 	.init_supported_interfaces = mv3310_init_supported_interfaces,
 	.get_mactype = mv3310_get_mactype,
+	.set_mactype = mv3310_set_mactype,
+	.select_mactype = mv3310_select_mactype,
 	.init_interface = mv3310_init_interface,
 
 #ifdef CONFIG_HWMON
@@ -1060,6 +1167,8 @@ static const struct mv3310_chip mv3340_type = {
 	.has_downshift = mv3310_has_downshift,
 	.init_supported_interfaces = mv3340_init_supported_interfaces,
 	.get_mactype = mv3310_get_mactype,
+	.set_mactype = mv3310_set_mactype,
+	.select_mactype = mv3310_select_mactype,
 	.init_interface = mv3340_init_interface,
 
 #ifdef CONFIG_HWMON
@@ -1070,6 +1179,8 @@ static const struct mv3310_chip mv3340_type = {
 static const struct mv3310_chip mv2110_type = {
 	.init_supported_interfaces = mv2110_init_supported_interfaces,
 	.get_mactype = mv2110_get_mactype,
+	.set_mactype = mv2110_set_mactype,
+	.select_mactype = mv2110_select_mactype,
 	.init_interface = mv2110_init_interface,
 
 #ifdef CONFIG_HWMON
@@ -1080,6 +1191,8 @@ static const struct mv3310_chip mv2110_type = {
 static const struct mv3310_chip mv2111_type = {
 	.init_supported_interfaces = mv2111_init_supported_interfaces,
 	.get_mactype = mv2110_get_mactype,
+	.set_mactype = mv2110_set_mactype,
+	.select_mactype = mv2110_select_mactype,
 	.init_interface = mv2110_init_interface,
 
 #ifdef CONFIG_HWMON
diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c
index 8a2dbe849866..f82090bdf7ab 100644
--- a/drivers/net/phy/mdio_bus.c
+++ b/drivers/net/phy/mdio_bus.c
@@ -232,7 +232,7 @@ static ssize_t mdio_bus_stat_field_show(struct device *dev,
 		val = mdio_bus_get_stat(&bus->stats[sattr->addr],
 					sattr->field_offset);
 
-	return sprintf(buf, "%llu\n", val);
+	return sysfs_emit(buf, "%llu\n", val);
 }
 
 static ssize_t mdio_bus_device_stat_field_show(struct device *dev,
@@ -251,7 +251,7 @@ static ssize_t mdio_bus_device_stat_field_show(struct device *dev,
 
 	val = mdio_bus_get_stat(&bus->stats[addr], sattr->field_offset);
 
-	return sprintf(buf, "%llu\n", val);
+	return sysfs_emit(buf, "%llu\n", val);
 }
 
 #define MDIO_BUS_STATS_ATTR_DECL(field, file)				\
diff --git a/drivers/net/phy/micrel.c b/drivers/net/phy/micrel.c
index 38234d7e14c5..3757e069c486 100644
--- a/drivers/net/phy/micrel.c
+++ b/drivers/net/phy/micrel.c
@@ -92,6 +92,15 @@
 #define KSZ9x31_LMD_VCT_DATA_HI_PULSE_MASK	GENMASK(1, 0)
 #define KSZ9x31_LMD_VCT_DATA_MASK		GENMASK(7, 0)
 
+#define KSZPHY_WIRE_PAIR_MASK			0x3
+
+#define LAN8814_CABLE_DIAG			0x12
+#define LAN8814_CABLE_DIAG_STAT_MASK		GENMASK(9, 8)
+#define LAN8814_CABLE_DIAG_VCT_DATA_MASK	GENMASK(7, 0)
+#define LAN8814_PAIR_BIT_SHIFT			12
+
+#define LAN8814_WIRE_PAIR_MASK			0xF
+
 /* Lan8814 general Interrupt control/status reg in GPHY specific block. */
 #define LAN8814_INTC				0x18
 #define LAN8814_INTS				0x1B
@@ -257,6 +266,8 @@ static struct kszphy_hw_stat kszphy_hw_stats[] = {
 struct kszphy_type {
 	u32 led_mode_reg;
 	u16 interrupt_level_mask;
+	u16 cable_diag_reg;
+	unsigned long pair_mask;
 	bool has_broadcast_disable;
 	bool has_nand_tree_disable;
 	bool has_rmii_ref_clk_sel;
@@ -313,6 +324,13 @@ struct kszphy_priv {
 
 static const struct kszphy_type lan8814_type = {
 	.led_mode_reg		= ~LAN8814_LED_CTRL_1,
+	.cable_diag_reg		= LAN8814_CABLE_DIAG,
+	.pair_mask		= LAN8814_WIRE_PAIR_MASK,
+};
+
+static const struct kszphy_type ksz886x_type = {
+	.cable_diag_reg		= KSZ8081_LMD,
+	.pair_mask		= KSZPHY_WIRE_PAIR_MASK,
 };
 
 static const struct kszphy_type ksz8021_type = {
@@ -1650,7 +1668,7 @@ static void kszphy_get_strings(struct phy_device *phydev, u8 *data)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(kszphy_hw_stats); i++) {
-		strlcpy(data + i * ETH_GSTRING_LEN,
+		strscpy(data + i * ETH_GSTRING_LEN,
 			kszphy_hw_stats[i].string, ETH_GSTRING_LEN);
 	}
 }
@@ -1796,6 +1814,17 @@ static int kszphy_probe(struct phy_device *phydev)
 	return 0;
 }
 
+static int lan8814_cable_test_start(struct phy_device *phydev)
+{
+	/* If autoneg is enabled, we won't be able to test cross pair
+	 * short. In this case, the PHY will "detect" a link and
+	 * confuse the internal state machine - disable auto neg here.
+	 * Set the speed to 1000mbit and full duplex.
+	 */
+	return phy_modify(phydev, MII_BMCR, BMCR_ANENABLE | BMCR_SPEED100,
+			  BMCR_SPEED1000 | BMCR_FULLDPLX);
+}
+
 static int ksz886x_cable_test_start(struct phy_device *phydev)
 {
 	if (phydev->dev_flags & MICREL_KSZ8_P1_ERRATA)
@@ -1809,9 +1838,9 @@ static int ksz886x_cable_test_start(struct phy_device *phydev)
 	return phy_clear_bits(phydev, MII_BMCR, BMCR_ANENABLE | BMCR_SPEED100);
 }
 
-static int ksz886x_cable_test_result_trans(u16 status)
+static int ksz886x_cable_test_result_trans(u16 status, u16 mask)
 {
-	switch (FIELD_GET(KSZ8081_LMD_STAT_MASK, status)) {
+	switch (FIELD_GET(mask, status)) {
 	case KSZ8081_LMD_STAT_NORMAL:
 		return ETHTOOL_A_CABLE_RESULT_CODE_OK;
 	case KSZ8081_LMD_STAT_SHORT:
@@ -1825,15 +1854,15 @@ static int ksz886x_cable_test_result_trans(u16 status)
 	}
 }
 
-static bool ksz886x_cable_test_failed(u16 status)
+static bool ksz886x_cable_test_failed(u16 status, u16 mask)
 {
-	return FIELD_GET(KSZ8081_LMD_STAT_MASK, status) ==
+	return FIELD_GET(mask, status) ==
 		KSZ8081_LMD_STAT_FAIL;
 }
 
-static bool ksz886x_cable_test_fault_length_valid(u16 status)
+static bool ksz886x_cable_test_fault_length_valid(u16 status, u16 mask)
 {
-	switch (FIELD_GET(KSZ8081_LMD_STAT_MASK, status)) {
+	switch (FIELD_GET(mask, status)) {
 	case KSZ8081_LMD_STAT_OPEN:
 		fallthrough;
 	case KSZ8081_LMD_STAT_SHORT:
@@ -1842,29 +1871,79 @@ static bool ksz886x_cable_test_fault_length_valid(u16 status)
 	return false;
 }
 
-static int ksz886x_cable_test_fault_length(u16 status)
+static int ksz886x_cable_test_fault_length(struct phy_device *phydev, u16 status, u16 data_mask)
 {
 	int dt;
 
 	/* According to the data sheet the distance to the fault is
-	 * DELTA_TIME * 0.4 meters.
+	 * DELTA_TIME * 0.4 meters for ksz phys.
+	 * (DELTA_TIME - 22) * 0.8 for lan8814 phy.
 	 */
-	dt = FIELD_GET(KSZ8081_LMD_DELTA_TIME_MASK, status);
+	dt = FIELD_GET(data_mask, status);
 
-	return (dt * 400) / 10;
+	if ((phydev->phy_id & MICREL_PHY_ID_MASK) == PHY_ID_LAN8814)
+		return ((dt - 22) * 800) / 10;
+	else
+		return (dt * 400) / 10;
 }
 
 static int ksz886x_cable_test_wait_for_completion(struct phy_device *phydev)
 {
+	const struct kszphy_type *type = phydev->drv->driver_data;
 	int val, ret;
 
-	ret = phy_read_poll_timeout(phydev, KSZ8081_LMD, val,
+	ret = phy_read_poll_timeout(phydev, type->cable_diag_reg, val,
 				    !(val & KSZ8081_LMD_ENABLE_TEST),
 				    30000, 100000, true);
 
 	return ret < 0 ? ret : 0;
 }
 
+static int lan8814_cable_test_one_pair(struct phy_device *phydev, int pair)
+{
+	static const int ethtool_pair[] = { ETHTOOL_A_CABLE_PAIR_A,
+					    ETHTOOL_A_CABLE_PAIR_B,
+					    ETHTOOL_A_CABLE_PAIR_C,
+					    ETHTOOL_A_CABLE_PAIR_D,
+					  };
+	u32 fault_length;
+	int ret;
+	int val;
+
+	val = KSZ8081_LMD_ENABLE_TEST;
+	val = val | (pair << LAN8814_PAIR_BIT_SHIFT);
+
+	ret = phy_write(phydev, LAN8814_CABLE_DIAG, val);
+	if (ret < 0)
+		return ret;
+
+	ret = ksz886x_cable_test_wait_for_completion(phydev);
+	if (ret)
+		return ret;
+
+	val = phy_read(phydev, LAN8814_CABLE_DIAG);
+	if (val < 0)
+		return val;
+
+	if (ksz886x_cable_test_failed(val, LAN8814_CABLE_DIAG_STAT_MASK))
+		return -EAGAIN;
+
+	ret = ethnl_cable_test_result(phydev, ethtool_pair[pair],
+				      ksz886x_cable_test_result_trans(val,
+								      LAN8814_CABLE_DIAG_STAT_MASK
+								      ));
+	if (ret)
+		return ret;
+
+	if (!ksz886x_cable_test_fault_length_valid(val, LAN8814_CABLE_DIAG_STAT_MASK))
+		return 0;
+
+	fault_length = ksz886x_cable_test_fault_length(phydev, val,
+						       LAN8814_CABLE_DIAG_VCT_DATA_MASK);
+
+	return ethnl_cable_test_fault_length(phydev, ethtool_pair[pair], fault_length);
+}
+
 static int ksz886x_cable_test_one_pair(struct phy_device *phydev, int pair)
 {
 	static const int ethtool_pair[] = {
@@ -1872,6 +1951,7 @@ static int ksz886x_cable_test_one_pair(struct phy_device *phydev, int pair)
 		ETHTOOL_A_CABLE_PAIR_B,
 	};
 	int ret, val, mdix;
+	u32 fault_length;
 
 	/* There is no way to choice the pair, like we do one ksz9031.
 	 * We can workaround this limitation by using the MDI-X functionality.
@@ -1910,25 +1990,27 @@ static int ksz886x_cable_test_one_pair(struct phy_device *phydev, int pair)
 	if (val < 0)
 		return val;
 
-	if (ksz886x_cable_test_failed(val))
+	if (ksz886x_cable_test_failed(val, KSZ8081_LMD_STAT_MASK))
 		return -EAGAIN;
 
 	ret = ethnl_cable_test_result(phydev, ethtool_pair[pair],
-				      ksz886x_cable_test_result_trans(val));
+				      ksz886x_cable_test_result_trans(val, KSZ8081_LMD_STAT_MASK));
 	if (ret)
 		return ret;
 
-	if (!ksz886x_cable_test_fault_length_valid(val))
+	if (!ksz886x_cable_test_fault_length_valid(val, KSZ8081_LMD_STAT_MASK))
 		return 0;
 
-	return ethnl_cable_test_fault_length(phydev, ethtool_pair[pair],
-					     ksz886x_cable_test_fault_length(val));
+	fault_length = ksz886x_cable_test_fault_length(phydev, val, KSZ8081_LMD_DELTA_TIME_MASK);
+
+	return ethnl_cable_test_fault_length(phydev, ethtool_pair[pair], fault_length);
 }
 
 static int ksz886x_cable_test_get_status(struct phy_device *phydev,
 					 bool *finished)
 {
-	unsigned long pair_mask = 0x3;
+	const struct kszphy_type *type = phydev->drv->driver_data;
+	unsigned long pair_mask = type->pair_mask;
 	int retries = 20;
 	int pair, ret;
 
@@ -1937,7 +2019,10 @@ static int ksz886x_cable_test_get_status(struct phy_device *phydev,
 	/* Try harder if link partner is active */
 	while (pair_mask && retries--) {
 		for_each_set_bit(pair, &pair_mask, 4) {
-			ret = ksz886x_cable_test_one_pair(phydev, pair);
+			if (type->cable_diag_reg == LAN8814_CABLE_DIAG)
+				ret = lan8814_cable_test_one_pair(phydev, pair);
+			else
+				ret = ksz886x_cable_test_one_pair(phydev, pair);
 			if (ret == -EAGAIN)
 				continue;
 			if (ret < 0)
@@ -2676,6 +2761,66 @@ static int lan8804_config_init(struct phy_device *phydev)
 	return 0;
 }
 
+static irqreturn_t lan8804_handle_interrupt(struct phy_device *phydev)
+{
+	int status;
+
+	status = phy_read(phydev, LAN8814_INTS);
+	if (status < 0) {
+		phy_error(phydev);
+		return IRQ_NONE;
+	}
+
+	if (status > 0)
+		phy_trigger_machine(phydev);
+
+	return IRQ_HANDLED;
+}
+
+#define LAN8804_OUTPUT_CONTROL			25
+#define LAN8804_OUTPUT_CONTROL_INTR_BUFFER	BIT(14)
+#define LAN8804_CONTROL				31
+#define LAN8804_CONTROL_INTR_POLARITY		BIT(14)
+
+static int lan8804_config_intr(struct phy_device *phydev)
+{
+	int err;
+
+	/* This is an internal PHY of lan966x and is not possible to change the
+	 * polarity on the GIC found in lan966x, therefore change the polarity
+	 * of the interrupt in the PHY from being active low instead of active
+	 * high.
+	 */
+	phy_write(phydev, LAN8804_CONTROL, LAN8804_CONTROL_INTR_POLARITY);
+
+	/* By default interrupt buffer is open-drain in which case the interrupt
+	 * can be active only low. Therefore change the interrupt buffer to be
+	 * push-pull to be able to change interrupt polarity
+	 */
+	phy_write(phydev, LAN8804_OUTPUT_CONTROL,
+		  LAN8804_OUTPUT_CONTROL_INTR_BUFFER);
+
+	if (phydev->interrupts == PHY_INTERRUPT_ENABLED) {
+		err = phy_read(phydev, LAN8814_INTS);
+		if (err < 0)
+			return err;
+
+		err = phy_write(phydev, LAN8814_INTC, LAN8814_INT_LINK);
+		if (err)
+			return err;
+	} else {
+		err = phy_write(phydev, LAN8814_INTC, 0);
+		if (err)
+			return err;
+
+		err = phy_read(phydev, LAN8814_INTS);
+		if (err < 0)
+			return err;
+	}
+
+	return 0;
+}
+
 static irqreturn_t lan8814_handle_interrupt(struct phy_device *phydev)
 {
 	int irq_status, tsu_irq_status;
@@ -2735,9 +2880,9 @@ static int lan8814_config_intr(struct phy_device *phydev)
 		if (err)
 			return err;
 
-		err =  phy_write(phydev, LAN8814_INTC, LAN8814_INT_LINK);
+		err = phy_write(phydev, LAN8814_INTC, LAN8814_INT_LINK);
 	} else {
-		err =  phy_write(phydev, LAN8814_INTC, 0);
+		err = phy_write(phydev, LAN8814_INTC, 0);
 		if (err)
 			return err;
 
@@ -3117,6 +3262,7 @@ static struct phy_driver ksphy_driver[] = {
 	.phy_id		= PHY_ID_LAN8814,
 	.phy_id_mask	= MICREL_PHY_ID_MASK,
 	.name		= "Microchip INDY Gigabit Quad PHY",
+	.flags          = PHY_POLL_CABLE_TEST,
 	.config_init	= lan8814_config_init,
 	.driver_data	= &lan8814_type,
 	.probe		= lan8814_probe,
@@ -3129,6 +3275,8 @@ static struct phy_driver ksphy_driver[] = {
 	.resume		= kszphy_resume,
 	.config_intr	= lan8814_config_intr,
 	.handle_interrupt = lan8814_handle_interrupt,
+	.cable_test_start	= lan8814_cable_test_start,
+	.cable_test_get_status	= ksz886x_cable_test_get_status,
 }, {
 	.phy_id		= PHY_ID_LAN8804,
 	.phy_id_mask	= MICREL_PHY_ID_MASK,
@@ -3143,6 +3291,8 @@ static struct phy_driver ksphy_driver[] = {
 	.get_stats	= kszphy_get_stats,
 	.suspend	= genphy_suspend,
 	.resume		= kszphy_resume,
+	.config_intr	= lan8804_config_intr,
+	.handle_interrupt = lan8804_handle_interrupt,
 }, {
 	.phy_id		= PHY_ID_KSZ9131,
 	.phy_id_mask	= MICREL_PHY_ID_MASK,
@@ -3175,6 +3325,7 @@ static struct phy_driver ksphy_driver[] = {
 	.phy_id		= PHY_ID_KSZ886X,
 	.phy_id_mask	= MICREL_PHY_ID_MASK,
 	.name		= "Micrel KSZ8851 Ethernet MAC or KSZ886X Switch",
+	.driver_data	= &ksz886x_type,
 	/* PHY_BASIC_FEATURES */
 	.flags		= PHY_POLL_CABLE_TEST,
 	.config_init	= kszphy_config_init,
@@ -3197,6 +3348,8 @@ static struct phy_driver ksphy_driver[] = {
 	.name		= "Microchip KSZ9477",
 	/* PHY_GBIT_FEATURES */
 	.config_init	= kszphy_config_init,
+	.config_intr	= kszphy_config_intr,
+	.handle_interrupt = kszphy_handle_interrupt,
 	.suspend	= genphy_suspend,
 	.resume		= genphy_resume,
 } };
diff --git a/drivers/net/phy/mscc/mscc_macsec.c b/drivers/net/phy/mscc/mscc_macsec.c
index b7b2521c73fb..ee5b17edca39 100644
--- a/drivers/net/phy/mscc/mscc_macsec.c
+++ b/drivers/net/phy/mscc/mscc_macsec.c
@@ -706,14 +706,6 @@ static int __vsc8584_macsec_add_rxsa(struct macsec_context *ctx,
 	struct phy_device *phydev = ctx->phydev;
 	struct vsc8531_private *priv = phydev->priv;
 
-	if (!flow) {
-		flow = vsc8584_macsec_alloc_flow(priv, MACSEC_INGR);
-		if (IS_ERR(flow))
-			return PTR_ERR(flow);
-
-		memcpy(flow->key, ctx->sa.key, priv->secy->key_len);
-	}
-
 	flow->assoc_num = ctx->sa.assoc_num;
 	flow->rx_sa = ctx->sa.rx_sa;
 
@@ -730,24 +722,13 @@ static int __vsc8584_macsec_add_rxsa(struct macsec_context *ctx,
 static int __vsc8584_macsec_add_txsa(struct macsec_context *ctx,
 				     struct macsec_flow *flow, bool update)
 {
-	struct phy_device *phydev = ctx->phydev;
-	struct vsc8531_private *priv = phydev->priv;
-
-	if (!flow) {
-		flow = vsc8584_macsec_alloc_flow(priv, MACSEC_EGR);
-		if (IS_ERR(flow))
-			return PTR_ERR(flow);
-
-		memcpy(flow->key, ctx->sa.key, priv->secy->key_len);
-	}
-
 	flow->assoc_num = ctx->sa.assoc_num;
 	flow->tx_sa = ctx->sa.tx_sa;
 
 	/* Always match untagged packets on egress */
 	flow->match.untagged = 1;
 
-	return vsc8584_macsec_add_flow(phydev, flow, update);
+	return vsc8584_macsec_add_flow(ctx->phydev, flow, update);
 }
 
 static int vsc8584_macsec_dev_open(struct macsec_context *ctx)
@@ -755,10 +736,6 @@ static int vsc8584_macsec_dev_open(struct macsec_context *ctx)
 	struct vsc8531_private *priv = ctx->phydev->priv;
 	struct macsec_flow *flow, *tmp;
 
-	/* No operation to perform before the commit step */
-	if (ctx->prepare)
-		return 0;
-
 	list_for_each_entry_safe(flow, tmp, &priv->macsec_flows, list)
 		vsc8584_macsec_flow_enable(ctx->phydev, flow);
 
@@ -770,10 +747,6 @@ static int vsc8584_macsec_dev_stop(struct macsec_context *ctx)
 	struct vsc8531_private *priv = ctx->phydev->priv;
 	struct macsec_flow *flow, *tmp;
 
-	/* No operation to perform before the commit step */
-	if (ctx->prepare)
-		return 0;
-
 	list_for_each_entry_safe(flow, tmp, &priv->macsec_flows, list)
 		vsc8584_macsec_flow_disable(ctx->phydev, flow);
 
@@ -785,12 +758,8 @@ static int vsc8584_macsec_add_secy(struct macsec_context *ctx)
 	struct vsc8531_private *priv = ctx->phydev->priv;
 	struct macsec_secy *secy = ctx->secy;
 
-	if (ctx->prepare) {
-		if (priv->secy)
-			return -EEXIST;
-
-		return 0;
-	}
+	if (priv->secy)
+		return -EEXIST;
 
 	priv->secy = secy;
 
@@ -807,10 +776,6 @@ static int vsc8584_macsec_del_secy(struct macsec_context *ctx)
 	struct vsc8531_private *priv = ctx->phydev->priv;
 	struct macsec_flow *flow, *tmp;
 
-	/* No operation to perform before the commit step */
-	if (ctx->prepare)
-		return 0;
-
 	list_for_each_entry_safe(flow, tmp, &priv->macsec_flows, list)
 		vsc8584_macsec_del_flow(ctx->phydev, flow);
 
@@ -823,10 +788,6 @@ static int vsc8584_macsec_del_secy(struct macsec_context *ctx)
 
 static int vsc8584_macsec_upd_secy(struct macsec_context *ctx)
 {
-	/* No operation to perform before the commit step */
-	if (ctx->prepare)
-		return 0;
-
 	vsc8584_macsec_del_secy(ctx);
 	return vsc8584_macsec_add_secy(ctx);
 }
@@ -847,10 +808,6 @@ static int vsc8584_macsec_del_rxsc(struct macsec_context *ctx)
 	struct vsc8531_private *priv = ctx->phydev->priv;
 	struct macsec_flow *flow, *tmp;
 
-	/* No operation to perform before the commit step */
-	if (ctx->prepare)
-		return 0;
-
 	list_for_each_entry_safe(flow, tmp, &priv->macsec_flows, list) {
 		if (flow->bank == MACSEC_INGR && flow->rx_sa &&
 		    flow->rx_sa->sc->sci == ctx->rx_sc->sci)
@@ -862,33 +819,40 @@ static int vsc8584_macsec_del_rxsc(struct macsec_context *ctx)
 
 static int vsc8584_macsec_add_rxsa(struct macsec_context *ctx)
 {
-	struct macsec_flow *flow = NULL;
-
-	if (ctx->prepare)
-		return __vsc8584_macsec_add_rxsa(ctx, flow, false);
+	struct phy_device *phydev = ctx->phydev;
+	struct vsc8531_private *priv = phydev->priv;
+	struct macsec_flow *flow;
+	int ret;
 
-	flow = vsc8584_macsec_find_flow(ctx, MACSEC_INGR);
+	flow = vsc8584_macsec_alloc_flow(priv, MACSEC_INGR);
 	if (IS_ERR(flow))
 		return PTR_ERR(flow);
 
-	vsc8584_macsec_flow_enable(ctx->phydev, flow);
+	memcpy(flow->key, ctx->sa.key, priv->secy->key_len);
+
+	ret = __vsc8584_macsec_add_rxsa(ctx, flow, false);
+	if (ret)
+		return ret;
+
+	vsc8584_macsec_flow_enable(phydev, flow);
 	return 0;
 }
 
 static int vsc8584_macsec_upd_rxsa(struct macsec_context *ctx)
 {
 	struct macsec_flow *flow;
+	int ret;
 
 	flow = vsc8584_macsec_find_flow(ctx, MACSEC_INGR);
 	if (IS_ERR(flow))
 		return PTR_ERR(flow);
 
-	if (ctx->prepare) {
-		/* Make sure the flow is disabled before updating it */
-		vsc8584_macsec_flow_disable(ctx->phydev, flow);
+	/* Make sure the flow is disabled before updating it */
+	vsc8584_macsec_flow_disable(ctx->phydev, flow);
 
-		return __vsc8584_macsec_add_rxsa(ctx, flow, true);
-	}
+	ret = __vsc8584_macsec_add_rxsa(ctx, flow, true);
+	if (ret)
+		return ret;
 
 	vsc8584_macsec_flow_enable(ctx->phydev, flow);
 	return 0;
@@ -899,11 +863,8 @@ static int vsc8584_macsec_del_rxsa(struct macsec_context *ctx)
 	struct macsec_flow *flow;
 
 	flow = vsc8584_macsec_find_flow(ctx, MACSEC_INGR);
-
 	if (IS_ERR(flow))
 		return PTR_ERR(flow);
-	if (ctx->prepare)
-		return 0;
 
 	vsc8584_macsec_del_flow(ctx->phydev, flow);
 	return 0;
@@ -911,33 +872,40 @@ static int vsc8584_macsec_del_rxsa(struct macsec_context *ctx)
 
 static int vsc8584_macsec_add_txsa(struct macsec_context *ctx)
 {
-	struct macsec_flow *flow = NULL;
-
-	if (ctx->prepare)
-		return __vsc8584_macsec_add_txsa(ctx, flow, false);
+	struct phy_device *phydev = ctx->phydev;
+	struct vsc8531_private *priv = phydev->priv;
+	struct macsec_flow *flow;
+	int ret;
 
-	flow = vsc8584_macsec_find_flow(ctx, MACSEC_EGR);
+	flow = vsc8584_macsec_alloc_flow(priv, MACSEC_EGR);
 	if (IS_ERR(flow))
 		return PTR_ERR(flow);
 
-	vsc8584_macsec_flow_enable(ctx->phydev, flow);
+	memcpy(flow->key, ctx->sa.key, priv->secy->key_len);
+
+	ret = __vsc8584_macsec_add_txsa(ctx, flow, false);
+	if (ret)
+		return ret;
+
+	vsc8584_macsec_flow_enable(phydev, flow);
 	return 0;
 }
 
 static int vsc8584_macsec_upd_txsa(struct macsec_context *ctx)
 {
 	struct macsec_flow *flow;
+	int ret;
 
 	flow = vsc8584_macsec_find_flow(ctx, MACSEC_EGR);
 	if (IS_ERR(flow))
 		return PTR_ERR(flow);
 
-	if (ctx->prepare) {
-		/* Make sure the flow is disabled before updating it */
-		vsc8584_macsec_flow_disable(ctx->phydev, flow);
+	/* Make sure the flow is disabled before updating it */
+	vsc8584_macsec_flow_disable(ctx->phydev, flow);
 
-		return __vsc8584_macsec_add_txsa(ctx, flow, true);
-	}
+	ret = __vsc8584_macsec_add_txsa(ctx, flow, true);
+	if (ret)
+		return ret;
 
 	vsc8584_macsec_flow_enable(ctx->phydev, flow);
 	return 0;
@@ -948,11 +916,8 @@ static int vsc8584_macsec_del_txsa(struct macsec_context *ctx)
 	struct macsec_flow *flow;
 
 	flow = vsc8584_macsec_find_flow(ctx, MACSEC_EGR);
-
 	if (IS_ERR(flow))
 		return PTR_ERR(flow);
-	if (ctx->prepare)
-		return 0;
 
 	vsc8584_macsec_del_flow(ctx->phydev, flow);
 	return 0;
diff --git a/drivers/net/phy/mscc/mscc_main.c b/drivers/net/phy/mscc/mscc_main.c
index 7e3017e7a1c0..8a13b1ad9a33 100644
--- a/drivers/net/phy/mscc/mscc_main.c
+++ b/drivers/net/phy/mscc/mscc_main.c
@@ -136,7 +136,7 @@ static void vsc85xx_get_strings(struct phy_device *phydev, u8 *data)
 		return;
 
 	for (i = 0; i < priv->nstats; i++)
-		strlcpy(data + i * ETH_GSTRING_LEN, priv->hw_stats[i].string,
+		strscpy(data + i * ETH_GSTRING_LEN, priv->hw_stats[i].string,
 			ETH_GSTRING_LEN);
 }
 
diff --git a/drivers/net/phy/nxp-tja11xx.c b/drivers/net/phy/nxp-tja11xx.c
index 2a8195c50d14..ec91e671f8aa 100644
--- a/drivers/net/phy/nxp-tja11xx.c
+++ b/drivers/net/phy/nxp-tja11xx.c
@@ -10,6 +10,7 @@
 #include <linux/mdio.h>
 #include <linux/mii.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/phy.h>
 #include <linux/hwmon.h>
 #include <linux/bitfield.h>
@@ -34,6 +35,11 @@
 #define MII_CFG1			18
 #define MII_CFG1_MASTER_SLAVE		BIT(15)
 #define MII_CFG1_AUTO_OP		BIT(14)
+#define MII_CFG1_INTERFACE_MODE_MASK	GENMASK(9, 8)
+#define MII_CFG1_MII_MODE				(0x0 << 8)
+#define MII_CFG1_RMII_MODE_REFCLK_IN	BIT(8)
+#define MII_CFG1_RMII_MODE_REFCLK_OUT	BIT(9)
+#define MII_CFG1_REVMII_MODE			GENMASK(9, 8)
 #define MII_CFG1_SLEEP_CONFIRM		BIT(6)
 #define MII_CFG1_LED_MODE_MASK		GENMASK(5, 4)
 #define MII_CFG1_LED_MODE_LINKUP	0
@@ -72,11 +78,15 @@
 #define MII_COMMCFG			27
 #define MII_COMMCFG_AUTO_OP		BIT(15)
 
+/* Configure REF_CLK as input in RMII mode */
+#define TJA110X_RMII_MODE_REFCLK_IN       BIT(0)
+
 struct tja11xx_priv {
 	char		*hwmon_name;
 	struct device	*hwmon_dev;
 	struct phy_device *phydev;
 	struct work_struct phy_register_work;
+	u32 flags;
 };
 
 struct tja11xx_phy_stats {
@@ -251,8 +261,34 @@ do_test:
 	return __genphy_config_aneg(phydev, changed);
 }
 
+static int tja11xx_get_interface_mode(struct phy_device *phydev)
+{
+	struct tja11xx_priv *priv = phydev->priv;
+	int mii_mode;
+
+	switch (phydev->interface) {
+	case PHY_INTERFACE_MODE_MII:
+		mii_mode = MII_CFG1_MII_MODE;
+		break;
+	case PHY_INTERFACE_MODE_REVMII:
+		mii_mode = MII_CFG1_REVMII_MODE;
+		break;
+	case PHY_INTERFACE_MODE_RMII:
+		if (priv->flags & TJA110X_RMII_MODE_REFCLK_IN)
+			mii_mode = MII_CFG1_RMII_MODE_REFCLK_IN;
+		else
+			mii_mode = MII_CFG1_RMII_MODE_REFCLK_OUT;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return mii_mode;
+}
+
 static int tja11xx_config_init(struct phy_device *phydev)
 {
+	u16 reg_mask, reg_val;
 	int ret;
 
 	ret = tja11xx_enable_reg_write(phydev);
@@ -265,15 +301,32 @@ static int tja11xx_config_init(struct phy_device *phydev)
 
 	switch (phydev->phy_id & PHY_ID_MASK) {
 	case PHY_ID_TJA1100:
-		ret = phy_modify(phydev, MII_CFG1,
-				 MII_CFG1_AUTO_OP | MII_CFG1_LED_MODE_MASK |
-				 MII_CFG1_LED_ENABLE,
-				 MII_CFG1_AUTO_OP | MII_CFG1_LED_MODE_LINKUP |
-				 MII_CFG1_LED_ENABLE);
+		reg_mask = MII_CFG1_AUTO_OP | MII_CFG1_LED_MODE_MASK |
+			   MII_CFG1_LED_ENABLE;
+		reg_val = MII_CFG1_AUTO_OP | MII_CFG1_LED_MODE_LINKUP |
+			  MII_CFG1_LED_ENABLE;
+
+		reg_mask |= MII_CFG1_INTERFACE_MODE_MASK;
+		ret = tja11xx_get_interface_mode(phydev);
+		if (ret < 0)
+			return ret;
+
+		reg_val |= (ret & 0xffff);
+		ret = phy_modify(phydev, MII_CFG1, reg_mask, reg_val);
 		if (ret)
 			return ret;
 		break;
 	case PHY_ID_TJA1101:
+		reg_mask = MII_CFG1_INTERFACE_MODE_MASK;
+		ret = tja11xx_get_interface_mode(phydev);
+		if (ret < 0)
+			return ret;
+
+		reg_val = ret & 0xffff;
+		ret = phy_modify(phydev, MII_CFG1, reg_mask, reg_val);
+		if (ret)
+			return ret;
+		fallthrough;
 	case PHY_ID_TJA1102:
 		ret = phy_set_bits(phydev, MII_COMMCFG, MII_COMMCFG_AUTO_OP);
 		if (ret)
@@ -458,16 +511,36 @@ static int tja11xx_hwmon_register(struct phy_device *phydev,
 	return PTR_ERR_OR_ZERO(priv->hwmon_dev);
 }
 
+static int tja11xx_parse_dt(struct phy_device *phydev)
+{
+	struct device_node *node = phydev->mdio.dev.of_node;
+	struct tja11xx_priv *priv = phydev->priv;
+
+	if (!IS_ENABLED(CONFIG_OF_MDIO))
+		return 0;
+
+	if (of_property_read_bool(node, "nxp,rmii-refclk-in"))
+		priv->flags |= TJA110X_RMII_MODE_REFCLK_IN;
+
+	return 0;
+}
+
 static int tja11xx_probe(struct phy_device *phydev)
 {
 	struct device *dev = &phydev->mdio.dev;
 	struct tja11xx_priv *priv;
+	int ret;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
 	priv->phydev = phydev;
+	phydev->priv = priv;
+
+	ret = tja11xx_parse_dt(phydev);
+	if (ret)
+		return ret;
 
 	return tja11xx_hwmon_register(phydev, priv);
 }
diff --git a/drivers/net/phy/phy-core.c b/drivers/net/phy/phy-core.c
index 1f2531a1a876..2c8bf438ea61 100644
--- a/drivers/net/phy/phy-core.c
+++ b/drivers/net/phy/phy-core.c
@@ -74,6 +74,80 @@ const char *phy_duplex_to_str(unsigned int duplex)
 }
 EXPORT_SYMBOL_GPL(phy_duplex_to_str);
 
+/**
+ * phy_rate_matching_to_str - Return a string describing the rate matching
+ *
+ * @rate_matching: Type of rate matching to describe
+ */
+const char *phy_rate_matching_to_str(int rate_matching)
+{
+	switch (rate_matching) {
+	case RATE_MATCH_NONE:
+		return "none";
+	case RATE_MATCH_PAUSE:
+		return "pause";
+	case RATE_MATCH_CRS:
+		return "crs";
+	case RATE_MATCH_OPEN_LOOP:
+		return "open-loop";
+	}
+	return "Unsupported (update phy-core.c)";
+}
+EXPORT_SYMBOL_GPL(phy_rate_matching_to_str);
+
+/**
+ * phy_interface_num_ports - Return the number of links that can be carried by
+ *			     a given MAC-PHY physical link. Returns 0 if this is
+ *			     unknown, the number of links else.
+ *
+ * @interface: The interface mode we want to get the number of ports
+ */
+int phy_interface_num_ports(phy_interface_t interface)
+{
+	switch (interface) {
+	case PHY_INTERFACE_MODE_NA:
+		return 0;
+	case PHY_INTERFACE_MODE_INTERNAL:
+	case PHY_INTERFACE_MODE_MII:
+	case PHY_INTERFACE_MODE_GMII:
+	case PHY_INTERFACE_MODE_TBI:
+	case PHY_INTERFACE_MODE_REVMII:
+	case PHY_INTERFACE_MODE_RMII:
+	case PHY_INTERFACE_MODE_REVRMII:
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+	case PHY_INTERFACE_MODE_RTBI:
+	case PHY_INTERFACE_MODE_XGMII:
+	case PHY_INTERFACE_MODE_XLGMII:
+	case PHY_INTERFACE_MODE_MOCA:
+	case PHY_INTERFACE_MODE_TRGMII:
+	case PHY_INTERFACE_MODE_USXGMII:
+	case PHY_INTERFACE_MODE_SGMII:
+	case PHY_INTERFACE_MODE_SMII:
+	case PHY_INTERFACE_MODE_1000BASEX:
+	case PHY_INTERFACE_MODE_2500BASEX:
+	case PHY_INTERFACE_MODE_5GBASER:
+	case PHY_INTERFACE_MODE_10GBASER:
+	case PHY_INTERFACE_MODE_25GBASER:
+	case PHY_INTERFACE_MODE_10GKR:
+	case PHY_INTERFACE_MODE_100BASEX:
+	case PHY_INTERFACE_MODE_RXAUI:
+	case PHY_INTERFACE_MODE_XAUI:
+	case PHY_INTERFACE_MODE_1000BASEKX:
+		return 1;
+	case PHY_INTERFACE_MODE_QSGMII:
+	case PHY_INTERFACE_MODE_QUSGMII:
+		return 4;
+	case PHY_INTERFACE_MODE_MAX:
+		WARN_ONCE(1, "PHY_INTERFACE_MODE_MAX isn't a valid interface mode");
+		return 0;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(phy_interface_num_ports);
+
 /* A mapping of all SUPPORTED settings to speed/duplex.  This table
  * must be grouped by speed and sorted in descending match priority
  * - iow, descending speed.
diff --git a/drivers/net/phy/phy.c b/drivers/net/phy/phy.c
index 8d3ee3a6495b..e741d8aebffe 100644
--- a/drivers/net/phy/phy.c
+++ b/drivers/net/phy/phy.c
@@ -115,6 +115,33 @@ void phy_print_status(struct phy_device *phydev)
 EXPORT_SYMBOL(phy_print_status);
 
 /**
+ * phy_get_rate_matching - determine if rate matching is supported
+ * @phydev: The phy device to return rate matching for
+ * @iface: The interface mode to use
+ *
+ * This determines the type of rate matching (if any) that @phy supports
+ * using @iface. @iface may be %PHY_INTERFACE_MODE_NA to determine if any
+ * interface supports rate matching.
+ *
+ * Return: The type of rate matching @phy supports for @iface, or
+ *         %RATE_MATCH_NONE.
+ */
+int phy_get_rate_matching(struct phy_device *phydev,
+			  phy_interface_t iface)
+{
+	int ret = RATE_MATCH_NONE;
+
+	if (phydev->drv->get_rate_matching) {
+		mutex_lock(&phydev->lock);
+		ret = phydev->drv->get_rate_matching(phydev, iface);
+		mutex_unlock(&phydev->lock);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(phy_get_rate_matching);
+
+/**
  * phy_config_interrupt - configure the PHY device for the requested interrupts
  * @phydev: the phy_device struct
  * @interrupts: interrupt flags to configure for this @phydev
@@ -256,6 +283,7 @@ void phy_ethtool_ksettings_get(struct phy_device *phydev,
 	cmd->base.duplex = phydev->duplex;
 	cmd->base.master_slave_cfg = phydev->master_slave_get;
 	cmd->base.master_slave_state = phydev->master_slave_state;
+	cmd->base.rate_matching = phydev->rate_matching;
 	if (phydev->interface == PHY_INTERFACE_MODE_MOCA)
 		cmd->base.port = PORT_BNC;
 	else
diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
index 12ff276b80ae..57849ac0384e 100644
--- a/drivers/net/phy/phy_device.c
+++ b/drivers/net/phy/phy_device.c
@@ -26,6 +26,7 @@
 #include <linux/netdevice.h>
 #include <linux/phy.h>
 #include <linux/phy_led_triggers.h>
+#include <linux/pse-pd/pse.h>
 #include <linux/property.h>
 #include <linux/sfp.h>
 #include <linux/skbuff.h>
@@ -316,11 +317,13 @@ static __maybe_unused int mdio_bus_phy_resume(struct device *dev)
 
 	phydev->suspended_by_mdio_bus = 0;
 
-	/* If we manged to get here with the PHY state machine in a state neither
-	 * PHY_HALTED nor PHY_READY this is an indication that something went wrong
-	 * and we should most likely be using MAC managed PM and we are not.
+	/* If we managed to get here with the PHY state machine in a state
+	 * neither PHY_HALTED, PHY_READY nor PHY_UP, this is an indication
+	 * that something went wrong and we should most likely be using
+	 * MAC managed PM, but we are not.
 	 */
-	WARN_ON(phydev->state != PHY_HALTED && phydev->state != PHY_READY);
+	WARN_ON(phydev->state != PHY_HALTED && phydev->state != PHY_READY &&
+		phydev->state != PHY_UP);
 
 	ret = phy_init_hw(phydev);
 	if (ret < 0)
@@ -370,7 +373,7 @@ int phy_register_fixup(const char *bus_id, u32 phy_uid, u32 phy_uid_mask,
 	if (!fixup)
 		return -ENOMEM;
 
-	strlcpy(fixup->bus_id, bus_id, sizeof(fixup->bus_id));
+	strscpy(fixup->bus_id, bus_id, sizeof(fixup->bus_id));
 	fixup->phy_uid = phy_uid;
 	fixup->phy_uid_mask = phy_uid_mask;
 	fixup->run = run;
@@ -520,7 +523,7 @@ phy_id_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct phy_device *phydev = to_phy_device(dev);
 
-	return sprintf(buf, "0x%.8lx\n", (unsigned long)phydev->phy_id);
+	return sysfs_emit(buf, "0x%.8lx\n", (unsigned long)phydev->phy_id);
 }
 static DEVICE_ATTR_RO(phy_id);
 
@@ -535,7 +538,7 @@ phy_interface_show(struct device *dev, struct device_attribute *attr, char *buf)
 	else
 		mode = phy_modes(phydev->interface);
 
-	return sprintf(buf, "%s\n", mode);
+	return sysfs_emit(buf, "%s\n", mode);
 }
 static DEVICE_ATTR_RO(phy_interface);
 
@@ -545,7 +548,7 @@ phy_has_fixups_show(struct device *dev, struct device_attribute *attr,
 {
 	struct phy_device *phydev = to_phy_device(dev);
 
-	return sprintf(buf, "%d\n", phydev->has_fixups);
+	return sysfs_emit(buf, "%d\n", phydev->has_fixups);
 }
 static DEVICE_ATTR_RO(phy_has_fixups);
 
@@ -555,7 +558,7 @@ static ssize_t phy_dev_flags_show(struct device *dev,
 {
 	struct phy_device *phydev = to_phy_device(dev);
 
-	return sprintf(buf, "0x%08x\n", phydev->dev_flags);
+	return sysfs_emit(buf, "0x%08x\n", phydev->dev_flags);
 }
 static DEVICE_ATTR_RO(phy_dev_flags);
 
@@ -989,6 +992,7 @@ EXPORT_SYMBOL(phy_device_register);
 void phy_device_remove(struct phy_device *phydev)
 {
 	unregister_mii_timestamper(phydev->mii_ts);
+	pse_control_put(phydev->psec);
 
 	device_del(&phydev->mdio.dev);
 
@@ -1310,7 +1314,7 @@ phy_standalone_show(struct device *dev, struct device_attribute *attr,
 {
 	struct phy_device *phydev = to_phy_device(dev);
 
-	return sprintf(buf, "%d\n", !phydev->attached_dev);
+	return sysfs_emit(buf, "%d\n", !phydev->attached_dev);
 }
 static DEVICE_ATTR_RO(phy_standalone);
 
diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c
index 9bd69328dc4d..75464df191ef 100644
--- a/drivers/net/phy/phylink.c
+++ b/drivers/net/phy/phylink.c
@@ -77,6 +77,7 @@ struct phylink {
 
 	struct sfp_bus *sfp_bus;
 	bool sfp_may_have_phy;
+	DECLARE_PHY_INTERFACE_MASK(sfp_interfaces);
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(sfp_support);
 	u8 sfp_port;
 };
@@ -155,8 +156,84 @@ static const char *phylink_an_mode_str(unsigned int mode)
 	return mode < ARRAY_SIZE(modestr) ? modestr[mode] : "unknown";
 }
 
-static void phylink_caps_to_linkmodes(unsigned long *linkmodes,
-				      unsigned long caps)
+/**
+ * phylink_interface_max_speed() - get the maximum speed of a phy interface
+ * @interface: phy interface mode defined by &typedef phy_interface_t
+ *
+ * Determine the maximum speed of a phy interface. This is intended to help
+ * determine the correct speed to pass to the MAC when the phy is performing
+ * rate matching.
+ *
+ * Return: The maximum speed of @interface
+ */
+static int phylink_interface_max_speed(phy_interface_t interface)
+{
+	switch (interface) {
+	case PHY_INTERFACE_MODE_100BASEX:
+	case PHY_INTERFACE_MODE_REVRMII:
+	case PHY_INTERFACE_MODE_RMII:
+	case PHY_INTERFACE_MODE_SMII:
+	case PHY_INTERFACE_MODE_REVMII:
+	case PHY_INTERFACE_MODE_MII:
+		return SPEED_100;
+
+	case PHY_INTERFACE_MODE_TBI:
+	case PHY_INTERFACE_MODE_MOCA:
+	case PHY_INTERFACE_MODE_RTBI:
+	case PHY_INTERFACE_MODE_1000BASEX:
+	case PHY_INTERFACE_MODE_1000BASEKX:
+	case PHY_INTERFACE_MODE_TRGMII:
+	case PHY_INTERFACE_MODE_RGMII_TXID:
+	case PHY_INTERFACE_MODE_RGMII_RXID:
+	case PHY_INTERFACE_MODE_RGMII_ID:
+	case PHY_INTERFACE_MODE_RGMII:
+	case PHY_INTERFACE_MODE_QSGMII:
+	case PHY_INTERFACE_MODE_SGMII:
+	case PHY_INTERFACE_MODE_GMII:
+		return SPEED_1000;
+
+	case PHY_INTERFACE_MODE_2500BASEX:
+		return SPEED_2500;
+
+	case PHY_INTERFACE_MODE_5GBASER:
+		return SPEED_5000;
+
+	case PHY_INTERFACE_MODE_XGMII:
+	case PHY_INTERFACE_MODE_RXAUI:
+	case PHY_INTERFACE_MODE_XAUI:
+	case PHY_INTERFACE_MODE_10GBASER:
+	case PHY_INTERFACE_MODE_10GKR:
+	case PHY_INTERFACE_MODE_USXGMII:
+	case PHY_INTERFACE_MODE_QUSGMII:
+		return SPEED_10000;
+
+	case PHY_INTERFACE_MODE_25GBASER:
+		return SPEED_25000;
+
+	case PHY_INTERFACE_MODE_XLGMII:
+		return SPEED_40000;
+
+	case PHY_INTERFACE_MODE_INTERNAL:
+	case PHY_INTERFACE_MODE_NA:
+	case PHY_INTERFACE_MODE_MAX:
+		/* No idea! Garbage in, unknown out */
+		return SPEED_UNKNOWN;
+	}
+
+	/* If we get here, someone forgot to add an interface mode above */
+	WARN_ON_ONCE(1);
+	return SPEED_UNKNOWN;
+}
+
+/**
+ * phylink_caps_to_linkmodes() - Convert capabilities to ethtool link modes
+ * @linkmodes: ethtool linkmode mask (must be already initialised)
+ * @caps: bitmask of MAC capabilities
+ *
+ * Set all possible pause, speed and duplex linkmodes in @linkmodes that are
+ * supported by the @caps. @linkmodes must have been initialised previously.
+ */
+void phylink_caps_to_linkmodes(unsigned long *linkmodes, unsigned long caps)
 {
 	if (caps & MAC_SYM_PAUSE)
 		__set_bit(ETHTOOL_LINK_MODE_Pause_BIT, linkmodes);
@@ -295,21 +372,72 @@ static void phylink_caps_to_linkmodes(unsigned long *linkmodes,
 		__set_bit(ETHTOOL_LINK_MODE_400000baseCR4_Full_BIT, linkmodes);
 	}
 }
+EXPORT_SYMBOL_GPL(phylink_caps_to_linkmodes);
+
+static struct {
+	unsigned long mask;
+	int speed;
+	unsigned int duplex;
+} phylink_caps_params[] = {
+	{ MAC_400000FD, SPEED_400000, DUPLEX_FULL },
+	{ MAC_200000FD, SPEED_200000, DUPLEX_FULL },
+	{ MAC_100000FD, SPEED_100000, DUPLEX_FULL },
+	{ MAC_56000FD,  SPEED_56000,  DUPLEX_FULL },
+	{ MAC_50000FD,  SPEED_50000,  DUPLEX_FULL },
+	{ MAC_40000FD,  SPEED_40000,  DUPLEX_FULL },
+	{ MAC_25000FD,  SPEED_25000,  DUPLEX_FULL },
+	{ MAC_20000FD,  SPEED_20000,  DUPLEX_FULL },
+	{ MAC_10000FD,  SPEED_10000,  DUPLEX_FULL },
+	{ MAC_5000FD,   SPEED_5000,   DUPLEX_FULL },
+	{ MAC_2500FD,   SPEED_2500,   DUPLEX_FULL },
+	{ MAC_1000FD,   SPEED_1000,   DUPLEX_FULL },
+	{ MAC_1000HD,   SPEED_1000,   DUPLEX_HALF },
+	{ MAC_100FD,    SPEED_100,    DUPLEX_FULL },
+	{ MAC_100HD,    SPEED_100,    DUPLEX_HALF },
+	{ MAC_10FD,     SPEED_10,     DUPLEX_FULL },
+	{ MAC_10HD,     SPEED_10,     DUPLEX_HALF },
+};
 
 /**
- * phylink_get_linkmodes() - get acceptable link modes
- * @linkmodes: ethtool linkmode mask (must be already initialised)
+ * phylink_cap_from_speed_duplex - Get mac capability from speed/duplex
+ * @speed: the speed to search for
+ * @duplex: the duplex to search for
+ *
+ * Find the mac capability for a given speed and duplex.
+ *
+ * Return: A mask with the mac capability patching @speed and @duplex, or 0 if
+ *         there were no matches.
+ */
+static unsigned long phylink_cap_from_speed_duplex(int speed,
+						   unsigned int duplex)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(phylink_caps_params); i++) {
+		if (speed == phylink_caps_params[i].speed &&
+		    duplex == phylink_caps_params[i].duplex)
+			return phylink_caps_params[i].mask;
+	}
+
+	return 0;
+}
+
+/**
+ * phylink_get_capabilities() - get capabilities for a given MAC
  * @interface: phy interface mode defined by &typedef phy_interface_t
  * @mac_capabilities: bitmask of MAC capabilities
+ * @rate_matching: type of rate matching being performed
  *
- * Set all possible pause, speed and duplex linkmodes in @linkmodes that
- * are supported by the @interface mode and @mac_capabilities. @linkmodes
- * must have been initialised previously.
+ * Get the MAC capabilities that are supported by the @interface mode and
+ * @mac_capabilities.
  */
-void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface,
-			   unsigned long mac_capabilities)
+unsigned long phylink_get_capabilities(phy_interface_t interface,
+				       unsigned long mac_capabilities,
+				       int rate_matching)
 {
+	int max_speed = phylink_interface_max_speed(interface);
 	unsigned long caps = MAC_SYM_PAUSE | MAC_ASYM_PAUSE;
+	unsigned long matched_caps = 0;
 
 	switch (interface) {
 	case PHY_INTERFACE_MODE_USXGMII:
@@ -321,6 +449,7 @@ void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface,
 	case PHY_INTERFACE_MODE_RGMII_ID:
 	case PHY_INTERFACE_MODE_RGMII:
 	case PHY_INTERFACE_MODE_QSGMII:
+	case PHY_INTERFACE_MODE_QUSGMII:
 	case PHY_INTERFACE_MODE_SGMII:
 	case PHY_INTERFACE_MODE_GMII:
 		caps |= MAC_1000HD | MAC_1000FD;
@@ -344,6 +473,7 @@ void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface,
 	case PHY_INTERFACE_MODE_1000BASEX:
 		caps |= MAC_1000HD;
 		fallthrough;
+	case PHY_INTERFACE_MODE_1000BASEKX:
 	case PHY_INTERFACE_MODE_TRGMII:
 		caps |= MAC_1000FD;
 		break;
@@ -381,9 +511,55 @@ void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface,
 		break;
 	}
 
-	phylink_caps_to_linkmodes(linkmodes, caps & mac_capabilities);
+	switch (rate_matching) {
+	case RATE_MATCH_OPEN_LOOP:
+		/* TODO */
+		fallthrough;
+	case RATE_MATCH_NONE:
+		matched_caps = 0;
+		break;
+	case RATE_MATCH_PAUSE: {
+		/* The MAC must support asymmetric pause towards the local
+		 * device for this. We could allow just symmetric pause, but
+		 * then we might have to renegotiate if the link partner
+		 * doesn't support pause. This is because there's no way to
+		 * accept pause frames without transmitting them if we only
+		 * support symmetric pause.
+		 */
+		if (!(mac_capabilities & MAC_SYM_PAUSE) ||
+		    !(mac_capabilities & MAC_ASYM_PAUSE))
+			break;
+
+		/* We can't adapt if the MAC doesn't support the interface's
+		 * max speed at full duplex.
+		 */
+		if (mac_capabilities &
+		    phylink_cap_from_speed_duplex(max_speed, DUPLEX_FULL)) {
+			/* Although a duplex-matching phy might exist, we
+			 * conservatively remove these modes because the MAC
+			 * will not be aware of the half-duplex nature of the
+			 * link.
+			 */
+			matched_caps = GENMASK(__fls(caps), __fls(MAC_10HD));
+			matched_caps &= ~(MAC_1000HD | MAC_100HD | MAC_10HD);
+		}
+		break;
+	}
+	case RATE_MATCH_CRS:
+		/* The MAC must support half duplex at the interface's max
+		 * speed.
+		 */
+		if (mac_capabilities &
+		    phylink_cap_from_speed_duplex(max_speed, DUPLEX_HALF)) {
+			matched_caps = GENMASK(__fls(caps), __fls(MAC_10HD));
+			matched_caps &= mac_capabilities;
+		}
+		break;
+	}
+
+	return (caps & mac_capabilities) | matched_caps;
 }
-EXPORT_SYMBOL_GPL(phylink_get_linkmodes);
+EXPORT_SYMBOL_GPL(phylink_get_capabilities);
 
 /**
  * phylink_generic_validate() - generic validate() callback implementation
@@ -400,10 +576,14 @@ void phylink_generic_validate(struct phylink_config *config,
 			      struct phylink_link_state *state)
 {
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
+	unsigned long caps;
 
 	phylink_set_port_modes(mask);
 	phylink_set(mask, Autoneg);
-	phylink_get_linkmodes(mask, state->interface, config->mac_capabilities);
+	caps = phylink_get_capabilities(state->interface,
+					config->mac_capabilities,
+					state->rate_matching);
+	phylink_caps_to_linkmodes(mask, caps);
 
 	linkmode_and(supported, supported, mask);
 	linkmode_and(state->advertising, state->advertising, mask);
@@ -458,8 +638,9 @@ static int phylink_validate_mac_and_pcs(struct phylink *pl,
 	return phylink_is_empty_linkmode(supported) ? -EINVAL : 0;
 }
 
-static int phylink_validate_any(struct phylink *pl, unsigned long *supported,
-				struct phylink_link_state *state)
+static int phylink_validate_mask(struct phylink *pl, unsigned long *supported,
+				 struct phylink_link_state *state,
+				 const unsigned long *interfaces)
 {
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(all_adv) = { 0, };
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(all_s) = { 0, };
@@ -468,7 +649,7 @@ static int phylink_validate_any(struct phylink *pl, unsigned long *supported,
 	int intf;
 
 	for (intf = 0; intf < PHY_INTERFACE_MODE_MAX; intf++) {
-		if (test_bit(intf, pl->config->supported_interfaces)) {
+		if (test_bit(intf, interfaces)) {
 			linkmode_copy(s, supported);
 
 			t = *state;
@@ -489,12 +670,14 @@ static int phylink_validate_any(struct phylink *pl, unsigned long *supported,
 static int phylink_validate(struct phylink *pl, unsigned long *supported,
 			    struct phylink_link_state *state)
 {
-	if (!phy_interface_empty(pl->config->supported_interfaces)) {
+	const unsigned long *interfaces = pl->config->supported_interfaces;
+
+	if (!phy_interface_empty(interfaces)) {
 		if (state->interface == PHY_INTERFACE_MODE_NA)
-			return phylink_validate_any(pl, supported, state);
+			return phylink_validate_mask(pl, supported, state,
+						     interfaces);
 
-		if (!test_bit(state->interface,
-			      pl->config->supported_interfaces))
+		if (!test_bit(state->interface, interfaces))
 			return -EINVAL;
 	}
 
@@ -632,6 +815,12 @@ static int phylink_parse_mode(struct phylink *pl, struct fwnode_handle *fwnode)
 		switch (pl->link_config.interface) {
 		case PHY_INTERFACE_MODE_SGMII:
 		case PHY_INTERFACE_MODE_QSGMII:
+		case PHY_INTERFACE_MODE_QUSGMII:
+		case PHY_INTERFACE_MODE_RGMII:
+		case PHY_INTERFACE_MODE_RGMII_ID:
+		case PHY_INTERFACE_MODE_RGMII_RXID:
+		case PHY_INTERFACE_MODE_RGMII_TXID:
+		case PHY_INTERFACE_MODE_RTBI:
 			phylink_set(pl->supported, 10baseT_Half);
 			phylink_set(pl->supported, 10baseT_Full);
 			phylink_set(pl->supported, 100baseT_Half);
@@ -774,11 +963,12 @@ static void phylink_mac_config(struct phylink *pl,
 			       const struct phylink_link_state *state)
 {
 	phylink_dbg(pl,
-		    "%s: mode=%s/%s/%s/%s adv=%*pb pause=%02x link=%u an=%u\n",
+		    "%s: mode=%s/%s/%s/%s/%s adv=%*pb pause=%02x link=%u an=%u\n",
 		    __func__, phylink_an_mode_str(pl->cur_link_an_mode),
 		    phy_modes(state->interface),
 		    phy_speed_to_str(state->speed),
 		    phy_duplex_to_str(state->duplex),
+		    phy_rate_matching_to_str(state->rate_matching),
 		    __ETHTOOL_LINK_MODE_MASK_NBITS, state->advertising,
 		    state->pause, state->link, state->an_enabled);
 
@@ -915,7 +1105,8 @@ static void phylink_mac_pcs_get_state(struct phylink *pl,
 	linkmode_zero(state->lp_advertising);
 	state->interface = pl->link_config.interface;
 	state->an_enabled = pl->link_config.an_enabled;
-	if  (state->an_enabled) {
+	state->rate_matching = pl->link_config.rate_matching;
+	if (state->an_enabled) {
 		state->speed = SPEED_UNKNOWN;
 		state->duplex = DUPLEX_UNKNOWN;
 		state->pause = MLO_PAUSE_NONE;
@@ -998,19 +1189,43 @@ static void phylink_link_up(struct phylink *pl,
 			    struct phylink_link_state link_state)
 {
 	struct net_device *ndev = pl->netdev;
+	int speed, duplex;
+	bool rx_pause;
+
+	speed = link_state.speed;
+	duplex = link_state.duplex;
+	rx_pause = !!(link_state.pause & MLO_PAUSE_RX);
+
+	switch (link_state.rate_matching) {
+	case RATE_MATCH_PAUSE:
+		/* The PHY is doing rate matchion from the media rate (in
+		 * the link_state) to the interface speed, and will send
+		 * pause frames to the MAC to limit its transmission speed.
+		 */
+		speed = phylink_interface_max_speed(link_state.interface);
+		duplex = DUPLEX_FULL;
+		rx_pause = true;
+		break;
+
+	case RATE_MATCH_CRS:
+		/* The PHY is doing rate matchion from the media rate (in
+		 * the link_state) to the interface speed, and will cause
+		 * collisions to the MAC to limit its transmission speed.
+		 */
+		speed = phylink_interface_max_speed(link_state.interface);
+		duplex = DUPLEX_HALF;
+		break;
+	}
 
 	pl->cur_interface = link_state.interface;
 
 	if (pl->pcs && pl->pcs->ops->pcs_link_up)
 		pl->pcs->ops->pcs_link_up(pl->pcs, pl->cur_link_an_mode,
-					 pl->cur_interface,
-					 link_state.speed, link_state.duplex);
+					  pl->cur_interface, speed, duplex);
 
-	pl->mac_ops->mac_link_up(pl->config, pl->phydev,
-				 pl->cur_link_an_mode, pl->cur_interface,
-				 link_state.speed, link_state.duplex,
-				 !!(link_state.pause & MLO_PAUSE_TX),
-				 !!(link_state.pause & MLO_PAUSE_RX));
+	pl->mac_ops->mac_link_up(pl->config, pl->phydev, pl->cur_link_an_mode,
+				 pl->cur_interface, speed, duplex,
+				 !!(link_state.pause & MLO_PAUSE_TX), rx_pause);
 
 	if (ndev)
 		netif_carrier_on(ndev);
@@ -1102,6 +1317,17 @@ static void phylink_resolve(struct work_struct *w)
 				}
 				link_state.interface = pl->phy_state.interface;
 
+				/* If we are doing rate matching, then the
+				 * link speed/duplex comes from the PHY
+				 */
+				if (pl->phy_state.rate_matching) {
+					link_state.rate_matching =
+						pl->phy_state.rate_matching;
+					link_state.speed = pl->phy_state.speed;
+					link_state.duplex =
+						pl->phy_state.duplex;
+				}
+
 				/* If we have a PHY, we need to update with
 				 * the PHY flow control bits.
 				 */
@@ -1336,6 +1562,7 @@ static void phylink_phy_change(struct phy_device *phydev, bool up)
 	mutex_lock(&pl->state_mutex);
 	pl->phy_state.speed = phydev->speed;
 	pl->phy_state.duplex = phydev->duplex;
+	pl->phy_state.rate_matching = phydev->rate_matching;
 	pl->phy_state.pause = MLO_PAUSE_NONE;
 	if (tx_pause)
 		pl->phy_state.pause |= MLO_PAUSE_TX;
@@ -1347,10 +1574,11 @@ static void phylink_phy_change(struct phy_device *phydev, bool up)
 
 	phylink_run_resolve(pl);
 
-	phylink_dbg(pl, "phy link %s %s/%s/%s/%s\n", up ? "up" : "down",
+	phylink_dbg(pl, "phy link %s %s/%s/%s/%s/%s\n", up ? "up" : "down",
 		    phy_modes(phydev->interface),
 		    phy_speed_to_str(phydev->speed),
 		    phy_duplex_to_str(phydev->duplex),
+		    phy_rate_matching_to_str(phydev->rate_matching),
 		    phylink_pause_to_str(pl->phy_state.pause));
 }
 
@@ -1387,6 +1615,7 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy,
 		config.interface = PHY_INTERFACE_MODE_NA;
 	else
 		config.interface = interface;
+	config.rate_matching = phy_get_rate_matching(phy, config.interface);
 
 	ret = phylink_validate(pl, supported, &config);
 	if (ret) {
@@ -1414,6 +1643,7 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy,
 	pl->phy_state.pause = MLO_PAUSE_NONE;
 	pl->phy_state.speed = SPEED_UNKNOWN;
 	pl->phy_state.duplex = DUPLEX_UNKNOWN;
+	pl->phy_state.rate_matching = RATE_MATCH_NONE;
 	linkmode_copy(pl->supported, supported);
 	linkmode_copy(pl->link_config.advertising, config.advertising);
 
@@ -1439,7 +1669,7 @@ static int phylink_attach_phy(struct phylink *pl, struct phy_device *phy,
 {
 	if (WARN_ON(pl->cfg_link_an_mode == MLO_AN_FIXED ||
 		    (pl->cfg_link_an_mode == MLO_AN_INBAND &&
-		     phy_interface_mode_is_8023z(interface))))
+		     phy_interface_mode_is_8023z(interface) && !pl->sfp_bus)))
 		return -EINVAL;
 
 	if (pl->phydev)
@@ -1856,8 +2086,10 @@ static void phylink_get_ksettings(const struct phylink_link_state *state,
 {
 	phylink_merge_link_mode(kset->link_modes.advertising, state->advertising);
 	linkmode_copy(kset->link_modes.lp_advertising, state->lp_advertising);
-	kset->base.speed = state->speed;
-	kset->base.duplex = state->duplex;
+	if (kset->base.rate_matching == RATE_MATCH_NONE) {
+		kset->base.speed = state->speed;
+		kset->base.duplex = state->duplex;
+	}
 	kset->base.autoneg = state->an_enabled ? AUTONEG_ENABLE :
 				AUTONEG_DISABLE;
 }
@@ -2571,21 +2803,85 @@ static void phylink_sfp_detach(void *upstream, struct sfp_bus *bus)
 	pl->netdev->sfp_bus = NULL;
 }
 
-static int phylink_sfp_config(struct phylink *pl, u8 mode,
-			      const unsigned long *supported,
-			      const unsigned long *advertising)
+static const phy_interface_t phylink_sfp_interface_preference[] = {
+	PHY_INTERFACE_MODE_25GBASER,
+	PHY_INTERFACE_MODE_USXGMII,
+	PHY_INTERFACE_MODE_10GBASER,
+	PHY_INTERFACE_MODE_5GBASER,
+	PHY_INTERFACE_MODE_2500BASEX,
+	PHY_INTERFACE_MODE_SGMII,
+	PHY_INTERFACE_MODE_1000BASEX,
+	PHY_INTERFACE_MODE_100BASEX,
+};
+
+static DECLARE_PHY_INTERFACE_MASK(phylink_sfp_interfaces);
+
+static phy_interface_t phylink_choose_sfp_interface(struct phylink *pl,
+						    const unsigned long *intf)
+{
+	phy_interface_t interface;
+	size_t i;
+
+	interface = PHY_INTERFACE_MODE_NA;
+	for (i = 0; i < ARRAY_SIZE(phylink_sfp_interface_preference); i++)
+		if (test_bit(phylink_sfp_interface_preference[i], intf)) {
+			interface = phylink_sfp_interface_preference[i];
+			break;
+		}
+
+	return interface;
+}
+
+static void phylink_sfp_set_config(struct phylink *pl, u8 mode,
+				   unsigned long *supported,
+				   struct phylink_link_state *state)
+{
+	bool changed = false;
+
+	phylink_dbg(pl, "requesting link mode %s/%s with support %*pb\n",
+		    phylink_an_mode_str(mode), phy_modes(state->interface),
+		    __ETHTOOL_LINK_MODE_MASK_NBITS, supported);
+
+	if (!linkmode_equal(pl->supported, supported)) {
+		linkmode_copy(pl->supported, supported);
+		changed = true;
+	}
+
+	if (!linkmode_equal(pl->link_config.advertising, state->advertising)) {
+		linkmode_copy(pl->link_config.advertising, state->advertising);
+		changed = true;
+	}
+
+	if (pl->cur_link_an_mode != mode ||
+	    pl->link_config.interface != state->interface) {
+		pl->cur_link_an_mode = mode;
+		pl->link_config.interface = state->interface;
+
+		changed = true;
+
+		phylink_info(pl, "switched to %s/%s link mode\n",
+			     phylink_an_mode_str(mode),
+			     phy_modes(state->interface));
+	}
+
+	if (changed && !test_bit(PHYLINK_DISABLE_STOPPED,
+				 &pl->phylink_disable_state))
+		phylink_mac_initial_config(pl, false);
+}
+
+static int phylink_sfp_config_phy(struct phylink *pl, u8 mode,
+				  struct phy_device *phy)
 {
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(support1);
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(support);
 	struct phylink_link_state config;
 	phy_interface_t iface;
-	bool changed;
 	int ret;
 
-	linkmode_copy(support, supported);
+	linkmode_copy(support, phy->supported);
 
 	memset(&config, 0, sizeof(config));
-	linkmode_copy(config.advertising, advertising);
+	linkmode_copy(config.advertising, phy->advertising);
 	config.interface = PHY_INTERFACE_MODE_NA;
 	config.speed = SPEED_UNKNOWN;
 	config.duplex = DUPLEX_UNKNOWN;
@@ -2622,60 +2918,100 @@ static int phylink_sfp_config(struct phylink *pl, u8 mode,
 		return ret;
 	}
 
-	phylink_dbg(pl, "requesting link mode %s/%s with support %*pb\n",
-		    phylink_an_mode_str(mode), phy_modes(config.interface),
-		    __ETHTOOL_LINK_MODE_MASK_NBITS, support);
+	pl->link_port = pl->sfp_port;
+
+	phylink_sfp_set_config(pl, mode, support, &config);
+
+	return 0;
+}
+
+static int phylink_sfp_config_optical(struct phylink *pl)
+{
+	__ETHTOOL_DECLARE_LINK_MODE_MASK(support);
+	DECLARE_PHY_INTERFACE_MASK(interfaces);
+	struct phylink_link_state config;
+	phy_interface_t interface;
+	int ret;
+
+	phylink_dbg(pl, "optical SFP: interfaces=[mac=%*pbl, sfp=%*pbl]\n",
+		    (int)PHY_INTERFACE_MODE_MAX,
+		    pl->config->supported_interfaces,
+		    (int)PHY_INTERFACE_MODE_MAX,
+		    pl->sfp_interfaces);
 
-	if (phy_interface_mode_is_8023z(iface) && pl->phydev)
+	/* Find the union of the supported interfaces by the PCS/MAC and
+	 * the SFP module.
+	 */
+	phy_interface_and(interfaces, pl->config->supported_interfaces,
+			  pl->sfp_interfaces);
+	if (phy_interface_empty(interfaces)) {
+		phylink_err(pl, "unsupported SFP module: no common interface modes\n");
 		return -EINVAL;
+	}
 
-	changed = !linkmode_equal(pl->supported, support) ||
-		  !linkmode_equal(pl->link_config.advertising,
-				  config.advertising);
-	if (changed) {
-		linkmode_copy(pl->supported, support);
-		linkmode_copy(pl->link_config.advertising, config.advertising);
+	memset(&config, 0, sizeof(config));
+	linkmode_copy(support, pl->sfp_support);
+	linkmode_copy(config.advertising, pl->sfp_support);
+	config.speed = SPEED_UNKNOWN;
+	config.duplex = DUPLEX_UNKNOWN;
+	config.pause = MLO_PAUSE_AN;
+	config.an_enabled = true;
+
+	/* For all the interfaces that are supported, reduce the sfp_support
+	 * mask to only those link modes that can be supported.
+	 */
+	ret = phylink_validate_mask(pl, pl->sfp_support, &config, interfaces);
+	if (ret) {
+		phylink_err(pl, "unsupported SFP module: validation with support %*pb failed\n",
+			    __ETHTOOL_LINK_MODE_MASK_NBITS, support);
+		return ret;
 	}
 
-	if (pl->cur_link_an_mode != mode ||
-	    pl->link_config.interface != config.interface) {
-		pl->link_config.interface = config.interface;
-		pl->cur_link_an_mode = mode;
+	interface = phylink_choose_sfp_interface(pl, interfaces);
+	if (interface == PHY_INTERFACE_MODE_NA) {
+		phylink_err(pl, "failed to select SFP interface\n");
+		return -EINVAL;
+	}
 
-		changed = true;
+	phylink_dbg(pl, "optical SFP: chosen %s interface\n",
+		    phy_modes(interface));
 
-		phylink_info(pl, "switched to %s/%s link mode\n",
-			     phylink_an_mode_str(mode),
-			     phy_modes(config.interface));
+	config.interface = interface;
+
+	/* Ignore errors if we're expecting a PHY to attach later */
+	ret = phylink_validate(pl, support, &config);
+	if (ret) {
+		phylink_err(pl, "validation with support %*pb failed: %pe\n",
+			    __ETHTOOL_LINK_MODE_MASK_NBITS, support,
+			    ERR_PTR(ret));
+		return ret;
 	}
 
 	pl->link_port = pl->sfp_port;
 
-	if (changed && !test_bit(PHYLINK_DISABLE_STOPPED,
-				 &pl->phylink_disable_state))
-		phylink_mac_initial_config(pl, false);
+	phylink_sfp_set_config(pl, MLO_AN_INBAND, pl->sfp_support, &config);
 
-	return ret;
+	return 0;
 }
 
 static int phylink_sfp_module_insert(void *upstream,
 				     const struct sfp_eeprom_id *id)
 {
 	struct phylink *pl = upstream;
-	unsigned long *support = pl->sfp_support;
 
 	ASSERT_RTNL();
 
-	linkmode_zero(support);
-	sfp_parse_support(pl->sfp_bus, id, support);
-	pl->sfp_port = sfp_parse_port(pl->sfp_bus, id, support);
+	linkmode_zero(pl->sfp_support);
+	phy_interface_zero(pl->sfp_interfaces);
+	sfp_parse_support(pl->sfp_bus, id, pl->sfp_support, pl->sfp_interfaces);
+	pl->sfp_port = sfp_parse_port(pl->sfp_bus, id, pl->sfp_support);
 
 	/* If this module may have a PHY connecting later, defer until later */
 	pl->sfp_may_have_phy = sfp_may_have_phy(pl->sfp_bus, id);
 	if (pl->sfp_may_have_phy)
 		return 0;
 
-	return phylink_sfp_config(pl, MLO_AN_INBAND, support, support);
+	return phylink_sfp_config_optical(pl);
 }
 
 static int phylink_sfp_module_start(void *upstream)
@@ -2694,8 +3030,7 @@ static int phylink_sfp_module_start(void *upstream)
 	if (!pl->sfp_may_have_phy)
 		return 0;
 
-	return phylink_sfp_config(pl, MLO_AN_INBAND,
-				  pl->sfp_support, pl->sfp_support);
+	return phylink_sfp_config_optical(pl);
 }
 
 static void phylink_sfp_module_stop(void *upstream)
@@ -2755,8 +3090,12 @@ static int phylink_sfp_connect_phy(void *upstream, struct phy_device *phy)
 	else
 		mode = MLO_AN_INBAND;
 
+	/* Set the PHY's host supported interfaces */
+	phy_interface_and(phy->host_interfaces, phylink_sfp_interfaces,
+			  pl->config->supported_interfaces);
+
 	/* Do the initial configuration */
-	ret = phylink_sfp_config(pl, mode, phy->supported, phy->advertising);
+	ret = phylink_sfp_config_phy(pl, mode, phy);
 	if (ret < 0)
 		return ret;
 
@@ -2929,6 +3268,7 @@ void phylink_mii_c22_pcs_decode_state(struct phylink_link_state *state,
 
 	case PHY_INTERFACE_MODE_SGMII:
 	case PHY_INTERFACE_MODE_QSGMII:
+	case PHY_INTERFACE_MODE_QUSGMII:
 		phylink_decode_sgmii_word(state, lpa);
 		break;
 
@@ -3107,4 +3447,15 @@ void phylink_mii_c45_pcs_get_state(struct mdio_device *pcs,
 }
 EXPORT_SYMBOL_GPL(phylink_mii_c45_pcs_get_state);
 
+static int __init phylink_init(void)
+{
+	for (int i = 0; i < ARRAY_SIZE(phylink_sfp_interface_preference); ++i)
+		__set_bit(phylink_sfp_interface_preference[i],
+			  phylink_sfp_interfaces);
+
+	return 0;
+}
+
+module_init(phylink_init);
+
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/phy/realtek.c b/drivers/net/phy/realtek.c
index a5671ab896b3..3d99fd6664d7 100644
--- a/drivers/net/phy/realtek.c
+++ b/drivers/net/phy/realtek.c
@@ -70,6 +70,7 @@
 #define RTLGEN_SPEED_MASK			0x0630
 
 #define RTL_GENERIC_PHYID			0x001cc800
+#define RTL_8211FVD_PHYID			0x001cc878
 
 MODULE_DESCRIPTION("Realtek PHY driver");
 MODULE_AUTHOR("Johnson Leung");
@@ -78,6 +79,7 @@ MODULE_LICENSE("GPL");
 struct rtl821x_priv {
 	u16 phycr1;
 	u16 phycr2;
+	bool has_phycr2;
 };
 
 static int rtl821x_read_page(struct phy_device *phydev)
@@ -94,6 +96,7 @@ static int rtl821x_probe(struct phy_device *phydev)
 {
 	struct device *dev = &phydev->mdio.dev;
 	struct rtl821x_priv *priv;
+	u32 phy_id = phydev->drv->phy_id;
 	int ret;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
@@ -108,13 +111,16 @@ static int rtl821x_probe(struct phy_device *phydev)
 	if (of_property_read_bool(dev->of_node, "realtek,aldps-enable"))
 		priv->phycr1 |= RTL8211F_ALDPS_PLL_OFF | RTL8211F_ALDPS_ENABLE | RTL8211F_ALDPS_XTAL_OFF;
 
-	ret = phy_read_paged(phydev, 0xa43, RTL8211F_PHYCR2);
-	if (ret < 0)
-		return ret;
+	priv->has_phycr2 = !(phy_id == RTL_8211FVD_PHYID);
+	if (priv->has_phycr2) {
+		ret = phy_read_paged(phydev, 0xa43, RTL8211F_PHYCR2);
+		if (ret < 0)
+			return ret;
 
-	priv->phycr2 = ret & RTL8211F_CLKOUT_EN;
-	if (of_property_read_bool(dev->of_node, "realtek,clkout-disable"))
-		priv->phycr2 &= ~RTL8211F_CLKOUT_EN;
+		priv->phycr2 = ret & RTL8211F_CLKOUT_EN;
+		if (of_property_read_bool(dev->of_node, "realtek,clkout-disable"))
+			priv->phycr2 &= ~RTL8211F_CLKOUT_EN;
+	}
 
 	phydev->priv = priv;
 
@@ -400,12 +406,14 @@ static int rtl8211f_config_init(struct phy_device *phydev)
 			val_rxdly ? "enabled" : "disabled");
 	}
 
-	ret = phy_modify_paged(phydev, 0xa43, RTL8211F_PHYCR2,
-			       RTL8211F_CLKOUT_EN, priv->phycr2);
-	if (ret < 0) {
-		dev_err(dev, "clkout configuration failed: %pe\n",
-			ERR_PTR(ret));
-		return ret;
+	if (priv->has_phycr2) {
+		ret = phy_modify_paged(phydev, 0xa43, RTL8211F_PHYCR2,
+				       RTL8211F_CLKOUT_EN, priv->phycr2);
+		if (ret < 0) {
+			dev_err(dev, "clkout configuration failed: %pe\n",
+				ERR_PTR(ret));
+			return ret;
+		}
 	}
 
 	return genphy_soft_reset(phydev);
@@ -924,6 +932,18 @@ static struct phy_driver realtek_drvs[] = {
 		.read_page	= rtl821x_read_page,
 		.write_page	= rtl821x_write_page,
 	}, {
+		PHY_ID_MATCH_EXACT(RTL_8211FVD_PHYID),
+		.name		= "RTL8211F-VD Gigabit Ethernet",
+		.probe		= rtl821x_probe,
+		.config_init	= &rtl8211f_config_init,
+		.read_status	= rtlgen_read_status,
+		.config_intr	= &rtl8211f_config_intr,
+		.handle_interrupt = rtl8211f_handle_interrupt,
+		.suspend	= genphy_suspend,
+		.resume		= rtl821x_resume,
+		.read_page	= rtl821x_read_page,
+		.write_page	= rtl821x_write_page,
+	}, {
 		.name		= "Generic FE-GE Realtek PHY",
 		.match_phy_device = rtlgen_match_phy_device,
 		.read_status	= rtlgen_read_status,
diff --git a/drivers/net/phy/sfp-bus.c b/drivers/net/phy/sfp-bus.c
index 15aa5ac1ff49..29e3fa86bac3 100644
--- a/drivers/net/phy/sfp-bus.c
+++ b/drivers/net/phy/sfp-bus.c
@@ -10,12 +10,6 @@
 
 #include "sfp.h"
 
-struct sfp_quirk {
-	const char *vendor;
-	const char *part;
-	void (*modes)(const struct sfp_eeprom_id *id, unsigned long *modes);
-};
-
 /**
  * struct sfp_bus - internal representation of a sfp bus
  */
@@ -38,93 +32,6 @@ struct sfp_bus {
 	bool started;
 };
 
-static void sfp_quirk_2500basex(const struct sfp_eeprom_id *id,
-				unsigned long *modes)
-{
-	phylink_set(modes, 2500baseX_Full);
-}
-
-static void sfp_quirk_ubnt_uf_instant(const struct sfp_eeprom_id *id,
-				      unsigned long *modes)
-{
-	/* Ubiquiti U-Fiber Instant module claims that support all transceiver
-	 * types including 10G Ethernet which is not truth. So clear all claimed
-	 * modes and set only one mode which module supports: 1000baseX_Full.
-	 */
-	phylink_zero(modes);
-	phylink_set(modes, 1000baseX_Full);
-}
-
-static const struct sfp_quirk sfp_quirks[] = {
-	{
-		// Alcatel Lucent G-010S-P can operate at 2500base-X, but
-		// incorrectly report 2500MBd NRZ in their EEPROM
-		.vendor = "ALCATELLUCENT",
-		.part = "G010SP",
-		.modes = sfp_quirk_2500basex,
-	}, {
-		// Alcatel Lucent G-010S-A can operate at 2500base-X, but
-		// report 3.2GBd NRZ in their EEPROM
-		.vendor = "ALCATELLUCENT",
-		.part = "3FE46541AA",
-		.modes = sfp_quirk_2500basex,
-	}, {
-		// Huawei MA5671A can operate at 2500base-X, but report 1.2GBd
-		// NRZ in their EEPROM
-		.vendor = "HUAWEI",
-		.part = "MA5671A",
-		.modes = sfp_quirk_2500basex,
-	}, {
-		// Lantech 8330-262D-E can operate at 2500base-X, but
-		// incorrectly report 2500MBd NRZ in their EEPROM
-		.vendor = "Lantech",
-		.part = "8330-262D-E",
-		.modes = sfp_quirk_2500basex,
-	}, {
-		.vendor = "UBNT",
-		.part = "UF-INSTANT",
-		.modes = sfp_quirk_ubnt_uf_instant,
-	},
-};
-
-static size_t sfp_strlen(const char *str, size_t maxlen)
-{
-	size_t size, i;
-
-	/* Trailing characters should be filled with space chars */
-	for (i = 0, size = 0; i < maxlen; i++)
-		if (str[i] != ' ')
-			size = i + 1;
-
-	return size;
-}
-
-static bool sfp_match(const char *qs, const char *str, size_t len)
-{
-	if (!qs)
-		return true;
-	if (strlen(qs) != len)
-		return false;
-	return !strncmp(qs, str, len);
-}
-
-static const struct sfp_quirk *sfp_lookup_quirk(const struct sfp_eeprom_id *id)
-{
-	const struct sfp_quirk *q;
-	unsigned int i;
-	size_t vs, ps;
-
-	vs = sfp_strlen(id->base.vendor_name, ARRAY_SIZE(id->base.vendor_name));
-	ps = sfp_strlen(id->base.vendor_pn, ARRAY_SIZE(id->base.vendor_pn));
-
-	for (i = 0, q = sfp_quirks; i < ARRAY_SIZE(sfp_quirks); i++, q++)
-		if (sfp_match(q->vendor, id->base.vendor_name, vs) &&
-		    sfp_match(q->part, id->base.vendor_pn, ps))
-			return q;
-
-	return NULL;
-}
-
 /**
  * sfp_parse_port() - Parse the EEPROM base ID, setting the port type
  * @bus: a pointer to the &struct sfp_bus structure for the sfp module
@@ -232,12 +139,14 @@ EXPORT_SYMBOL_GPL(sfp_may_have_phy);
  * @bus: a pointer to the &struct sfp_bus structure for the sfp module
  * @id: a pointer to the module's &struct sfp_eeprom_id
  * @support: pointer to an array of unsigned long for the ethtool support mask
+ * @interfaces: pointer to an array of unsigned long for phy interface modes
+ *		mask
  *
  * Parse the EEPROM identification information and derive the supported
  * ethtool link modes for the module.
  */
 void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
-		       unsigned long *support)
+		       unsigned long *support, unsigned long *interfaces)
 {
 	unsigned int br_min, br_nom, br_max;
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(modes) = { 0, };
@@ -264,54 +173,81 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 	}
 
 	/* Set ethtool support from the compliance fields. */
-	if (id->base.e10g_base_sr)
+	if (id->base.e10g_base_sr) {
 		phylink_set(modes, 10000baseSR_Full);
-	if (id->base.e10g_base_lr)
+		__set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces);
+	}
+	if (id->base.e10g_base_lr) {
 		phylink_set(modes, 10000baseLR_Full);
-	if (id->base.e10g_base_lrm)
+		__set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces);
+	}
+	if (id->base.e10g_base_lrm) {
 		phylink_set(modes, 10000baseLRM_Full);
-	if (id->base.e10g_base_er)
+		__set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces);
+	}
+	if (id->base.e10g_base_er) {
 		phylink_set(modes, 10000baseER_Full);
+		__set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces);
+	}
 	if (id->base.e1000_base_sx ||
 	    id->base.e1000_base_lx ||
-	    id->base.e1000_base_cx)
+	    id->base.e1000_base_cx) {
 		phylink_set(modes, 1000baseX_Full);
+		__set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces);
+	}
 	if (id->base.e1000_base_t) {
 		phylink_set(modes, 1000baseT_Half);
 		phylink_set(modes, 1000baseT_Full);
+		__set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces);
+		__set_bit(PHY_INTERFACE_MODE_SGMII, interfaces);
 	}
 
 	/* 1000Base-PX or 1000Base-BX10 */
 	if ((id->base.e_base_px || id->base.e_base_bx10) &&
-	    br_min <= 1300 && br_max >= 1200)
+	    br_min <= 1300 && br_max >= 1200) {
 		phylink_set(modes, 1000baseX_Full);
+		__set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces);
+	}
 
 	/* 100Base-FX, 100Base-LX, 100Base-PX, 100Base-BX10 */
-	if (id->base.e100_base_fx || id->base.e100_base_lx)
+	if (id->base.e100_base_fx || id->base.e100_base_lx) {
 		phylink_set(modes, 100baseFX_Full);
-	if ((id->base.e_base_px || id->base.e_base_bx10) && br_nom == 100)
+		__set_bit(PHY_INTERFACE_MODE_100BASEX, interfaces);
+	}
+	if ((id->base.e_base_px || id->base.e_base_bx10) && br_nom == 100) {
 		phylink_set(modes, 100baseFX_Full);
+		__set_bit(PHY_INTERFACE_MODE_100BASEX, interfaces);
+	}
 
 	/* For active or passive cables, select the link modes
 	 * based on the bit rates and the cable compliance bytes.
 	 */
 	if ((id->base.sfp_ct_passive || id->base.sfp_ct_active) && br_nom) {
 		/* This may look odd, but some manufacturers use 12000MBd */
-		if (br_min <= 12000 && br_max >= 10300)
+		if (br_min <= 12000 && br_max >= 10300) {
 			phylink_set(modes, 10000baseCR_Full);
-		if (br_min <= 3200 && br_max >= 3100)
+			__set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces);
+		}
+		if (br_min <= 3200 && br_max >= 3100) {
 			phylink_set(modes, 2500baseX_Full);
-		if (br_min <= 1300 && br_max >= 1200)
+			__set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces);
+		}
+		if (br_min <= 1300 && br_max >= 1200) {
 			phylink_set(modes, 1000baseX_Full);
+			__set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces);
+		}
 	}
 	if (id->base.sfp_ct_passive) {
-		if (id->base.passive.sff8431_app_e)
+		if (id->base.passive.sff8431_app_e) {
 			phylink_set(modes, 10000baseCR_Full);
+			__set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces);
+		}
 	}
 	if (id->base.sfp_ct_active) {
 		if (id->base.active.sff8431_app_e ||
 		    id->base.active.sff8431_lim) {
 			phylink_set(modes, 10000baseCR_Full);
+			__set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces);
 		}
 	}
 
@@ -336,12 +272,14 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 	case SFF8024_ECC_10GBASE_T_SFI:
 	case SFF8024_ECC_10GBASE_T_SR:
 		phylink_set(modes, 10000baseT_Full);
+		__set_bit(PHY_INTERFACE_MODE_10GBASER, interfaces);
 		break;
 	case SFF8024_ECC_5GBASE_T:
 		phylink_set(modes, 5000baseT_Full);
 		break;
 	case SFF8024_ECC_2_5GBASE_T:
 		phylink_set(modes, 2500baseT_Full);
+		__set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces);
 		break;
 	default:
 		dev_warn(bus->sfp_dev,
@@ -354,10 +292,14 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 	if (id->base.fc_speed_100 ||
 	    id->base.fc_speed_200 ||
 	    id->base.fc_speed_400) {
-		if (id->base.br_nominal >= 31)
+		if (id->base.br_nominal >= 31) {
 			phylink_set(modes, 2500baseX_Full);
-		if (id->base.br_nominal >= 12)
+			__set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces);
+		}
+		if (id->base.br_nominal >= 12) {
 			phylink_set(modes, 1000baseX_Full);
+			__set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces);
+		}
 	}
 
 	/* If we haven't discovered any modes that this module supports, try
@@ -370,14 +312,18 @@ void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 	 * 2500BASE-X, so we allow some slack here.
 	 */
 	if (bitmap_empty(modes, __ETHTOOL_LINK_MODE_MASK_NBITS) && br_nom) {
-		if (br_min <= 1300 && br_max >= 1200)
+		if (br_min <= 1300 && br_max >= 1200) {
 			phylink_set(modes, 1000baseX_Full);
-		if (br_min <= 3200 && br_max >= 2500)
+			__set_bit(PHY_INTERFACE_MODE_1000BASEX, interfaces);
+		}
+		if (br_min <= 3200 && br_max >= 2500) {
 			phylink_set(modes, 2500baseX_Full);
+			__set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces);
+		}
 	}
 
-	if (bus->sfp_quirk)
-		bus->sfp_quirk->modes(id, modes);
+	if (bus->sfp_quirk && bus->sfp_quirk->modes)
+		bus->sfp_quirk->modes(id, modes, interfaces);
 
 	linkmode_or(support, support, modes);
 
@@ -786,12 +732,13 @@ void sfp_link_down(struct sfp_bus *bus)
 }
 EXPORT_SYMBOL_GPL(sfp_link_down);
 
-int sfp_module_insert(struct sfp_bus *bus, const struct sfp_eeprom_id *id)
+int sfp_module_insert(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
+		      const struct sfp_quirk *quirk)
 {
 	const struct sfp_upstream_ops *ops = sfp_get_upstream_ops(bus);
 	int ret = 0;
 
-	bus->sfp_quirk = sfp_lookup_quirk(id);
+	bus->sfp_quirk = quirk;
 
 	if (ops && ops->module_insert)
 		ret = ops->module_insert(bus->upstream, id);
diff --git a/drivers/net/phy/sfp.c b/drivers/net/phy/sfp.c
index 63f90fe9a4d2..40c9a64c5e30 100644
--- a/drivers/net/phy/sfp.c
+++ b/drivers/net/phy/sfp.c
@@ -166,6 +166,7 @@ static const enum gpiod_flags gpio_flags[] = {
  * on board (for a copper SFP) time to initialise.
  */
 #define T_WAIT			msecs_to_jiffies(50)
+#define T_WAIT_ROLLBALL		msecs_to_jiffies(25000)
 #define T_START_UP		msecs_to_jiffies(300)
 #define T_START_UP_BAD_GPON	msecs_to_jiffies(60000)
 
@@ -205,8 +206,11 @@ static const enum gpiod_flags gpio_flags[] = {
 
 /* SFP modules appear to always have their PHY configured for bus address
  * 0x56 (which with mdio-i2c, translates to a PHY address of 22).
+ * RollBall SFPs access phy via SFP Enhanced Digital Diagnostic Interface
+ * via address 0x51 (mdio-i2c will use RollBall protocol on this address).
  */
-#define SFP_PHY_ADDR	22
+#define SFP_PHY_ADDR		22
+#define SFP_PHY_ADDR_ROLLBALL	17
 
 struct sff_data {
 	unsigned int gpios;
@@ -218,6 +222,7 @@ struct sfp {
 	struct i2c_adapter *i2c;
 	struct mii_bus *i2c_mii;
 	struct sfp_bus *sfp_bus;
+	enum mdio_i2c_proto mdio_protocol;
 	struct phy_device *mod_phy;
 	const struct sff_data *type;
 	size_t i2c_block_size;
@@ -234,6 +239,7 @@ struct sfp {
 	bool need_poll;
 
 	struct mutex st_mutex;			/* Protects state */
+	unsigned int state_hw_mask;
 	unsigned int state_soft_mask;
 	unsigned int state;
 	struct delayed_work poll;
@@ -250,8 +256,11 @@ struct sfp {
 	struct sfp_eeprom_id id;
 	unsigned int module_power_mW;
 	unsigned int module_t_start_up;
+	unsigned int module_t_wait;
 	bool tx_fault_ignore;
 
+	const struct sfp_quirk *quirk;
+
 #if IS_ENABLED(CONFIG_HWMON)
 	struct sfp_diag diag;
 	struct delayed_work hwmon_probe;
@@ -308,6 +317,136 @@ static const struct of_device_id sfp_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, sfp_of_match);
 
+static void sfp_fixup_long_startup(struct sfp *sfp)
+{
+	sfp->module_t_start_up = T_START_UP_BAD_GPON;
+}
+
+static void sfp_fixup_ignore_tx_fault(struct sfp *sfp)
+{
+	sfp->tx_fault_ignore = true;
+}
+
+static void sfp_fixup_halny_gsfp(struct sfp *sfp)
+{
+	/* Ignore the TX_FAULT and LOS signals on this module.
+	 * these are possibly used for other purposes on this
+	 * module, e.g. a serial port.
+	 */
+	sfp->state_hw_mask &= ~(SFP_F_TX_FAULT | SFP_F_LOS);
+}
+
+static void sfp_fixup_rollball(struct sfp *sfp)
+{
+	sfp->mdio_protocol = MDIO_I2C_ROLLBALL;
+	sfp->module_t_wait = T_WAIT_ROLLBALL;
+}
+
+static void sfp_fixup_rollball_cc(struct sfp *sfp)
+{
+	sfp_fixup_rollball(sfp);
+
+	/* Some RollBall SFPs may have wrong (zero) extended compliance code
+	 * burned in EEPROM. For PHY probing we need the correct one.
+	 */
+	sfp->id.base.extended_cc = SFF8024_ECC_10GBASE_T_SFI;
+}
+
+static void sfp_quirk_2500basex(const struct sfp_eeprom_id *id,
+				unsigned long *modes,
+				unsigned long *interfaces)
+{
+	linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, modes);
+	__set_bit(PHY_INTERFACE_MODE_2500BASEX, interfaces);
+}
+
+static void sfp_quirk_ubnt_uf_instant(const struct sfp_eeprom_id *id,
+				      unsigned long *modes,
+				      unsigned long *interfaces)
+{
+	/* Ubiquiti U-Fiber Instant module claims that support all transceiver
+	 * types including 10G Ethernet which is not truth. So clear all claimed
+	 * modes and set only one mode which module supports: 1000baseX_Full.
+	 */
+	linkmode_zero(modes);
+	linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseX_Full_BIT, modes);
+}
+
+#define SFP_QUIRK(_v, _p, _m, _f) \
+	{ .vendor = _v, .part = _p, .modes = _m, .fixup = _f, }
+#define SFP_QUIRK_M(_v, _p, _m) SFP_QUIRK(_v, _p, _m, NULL)
+#define SFP_QUIRK_F(_v, _p, _f) SFP_QUIRK(_v, _p, NULL, _f)
+
+static const struct sfp_quirk sfp_quirks[] = {
+	// Alcatel Lucent G-010S-P can operate at 2500base-X, but incorrectly
+	// report 2500MBd NRZ in their EEPROM
+	SFP_QUIRK_M("ALCATELLUCENT", "G010SP", sfp_quirk_2500basex),
+
+	// Alcatel Lucent G-010S-A can operate at 2500base-X, but report 3.2GBd
+	// NRZ in their EEPROM
+	SFP_QUIRK("ALCATELLUCENT", "3FE46541AA", sfp_quirk_2500basex,
+		  sfp_fixup_long_startup),
+
+	SFP_QUIRK_F("HALNy", "HL-GSFP", sfp_fixup_halny_gsfp),
+
+	// Huawei MA5671A can operate at 2500base-X, but report 1.2GBd NRZ in
+	// their EEPROM
+	SFP_QUIRK("HUAWEI", "MA5671A", sfp_quirk_2500basex,
+		  sfp_fixup_ignore_tx_fault),
+
+	// Lantech 8330-262D-E can operate at 2500base-X, but incorrectly report
+	// 2500MBd NRZ in their EEPROM
+	SFP_QUIRK_M("Lantech", "8330-262D-E", sfp_quirk_2500basex),
+
+	SFP_QUIRK_M("UBNT", "UF-INSTANT", sfp_quirk_ubnt_uf_instant),
+
+	SFP_QUIRK_F("OEM", "SFP-10G-T", sfp_fixup_rollball_cc),
+	SFP_QUIRK_F("OEM", "RTSFP-10", sfp_fixup_rollball_cc),
+	SFP_QUIRK_F("OEM", "RTSFP-10G", sfp_fixup_rollball_cc),
+	SFP_QUIRK_F("Turris", "RTSFP-10", sfp_fixup_rollball),
+	SFP_QUIRK_F("Turris", "RTSFP-10G", sfp_fixup_rollball),
+};
+
+static size_t sfp_strlen(const char *str, size_t maxlen)
+{
+	size_t size, i;
+
+	/* Trailing characters should be filled with space chars, but
+	 * some manufacturers can't read SFF-8472 and use NUL.
+	 */
+	for (i = 0, size = 0; i < maxlen; i++)
+		if (str[i] != ' ' && str[i] != '\0')
+			size = i + 1;
+
+	return size;
+}
+
+static bool sfp_match(const char *qs, const char *str, size_t len)
+{
+	if (!qs)
+		return true;
+	if (strlen(qs) != len)
+		return false;
+	return !strncmp(qs, str, len);
+}
+
+static const struct sfp_quirk *sfp_lookup_quirk(const struct sfp_eeprom_id *id)
+{
+	const struct sfp_quirk *q;
+	unsigned int i;
+	size_t vs, ps;
+
+	vs = sfp_strlen(id->base.vendor_name, ARRAY_SIZE(id->base.vendor_name));
+	ps = sfp_strlen(id->base.vendor_pn, ARRAY_SIZE(id->base.vendor_pn));
+
+	for (i = 0, q = sfp_quirks; i < ARRAY_SIZE(sfp_quirks); i++, q++)
+		if (sfp_match(q->vendor, id->base.vendor_name, vs) &&
+		    sfp_match(q->part, id->base.vendor_pn, ps))
+			return q;
+
+	return NULL;
+}
+
 static unsigned long poll_jiffies;
 
 static unsigned int sfp_gpio_get_state(struct sfp *sfp)
@@ -419,9 +558,6 @@ static int sfp_i2c_write(struct sfp *sfp, bool a2, u8 dev_addr, void *buf,
 
 static int sfp_i2c_configure(struct sfp *sfp, struct i2c_adapter *i2c)
 {
-	struct mii_bus *i2c_mii;
-	int ret;
-
 	if (!i2c_check_functionality(i2c, I2C_FUNC_I2C))
 		return -EINVAL;
 
@@ -429,7 +565,15 @@ static int sfp_i2c_configure(struct sfp *sfp, struct i2c_adapter *i2c)
 	sfp->read = sfp_i2c_read;
 	sfp->write = sfp_i2c_write;
 
-	i2c_mii = mdio_i2c_alloc(sfp->dev, i2c);
+	return 0;
+}
+
+static int sfp_i2c_mdiobus_create(struct sfp *sfp)
+{
+	struct mii_bus *i2c_mii;
+	int ret;
+
+	i2c_mii = mdio_i2c_alloc(sfp->dev, sfp->i2c, sfp->mdio_protocol);
 	if (IS_ERR(i2c_mii))
 		return PTR_ERR(i2c_mii);
 
@@ -447,6 +591,12 @@ static int sfp_i2c_configure(struct sfp *sfp, struct i2c_adapter *i2c)
 	return 0;
 }
 
+static void sfp_i2c_mdiobus_destroy(struct sfp *sfp)
+{
+	mdiobus_unregister(sfp->i2c_mii);
+	sfp->i2c_mii = NULL;
+}
+
 /* Interface */
 static int sfp_read(struct sfp *sfp, bool a2, u8 addr, void *buf, size_t len)
 {
@@ -499,17 +649,18 @@ static void sfp_soft_set_state(struct sfp *sfp, unsigned int state)
 static void sfp_soft_start_poll(struct sfp *sfp)
 {
 	const struct sfp_eeprom_id *id = &sfp->id;
+	unsigned int mask = 0;
 
 	sfp->state_soft_mask = 0;
-	if (id->ext.enhopts & SFP_ENHOPTS_SOFT_TX_DISABLE &&
-	    !sfp->gpio[GPIO_TX_DISABLE])
-		sfp->state_soft_mask |= SFP_F_TX_DISABLE;
-	if (id->ext.enhopts & SFP_ENHOPTS_SOFT_TX_FAULT &&
-	    !sfp->gpio[GPIO_TX_FAULT])
-		sfp->state_soft_mask |= SFP_F_TX_FAULT;
-	if (id->ext.enhopts & SFP_ENHOPTS_SOFT_RX_LOS &&
-	    !sfp->gpio[GPIO_LOS])
-		sfp->state_soft_mask |= SFP_F_LOS;
+	if (id->ext.enhopts & SFP_ENHOPTS_SOFT_TX_DISABLE)
+		mask |= SFP_F_TX_DISABLE;
+	if (id->ext.enhopts & SFP_ENHOPTS_SOFT_TX_FAULT)
+		mask |= SFP_F_TX_FAULT;
+	if (id->ext.enhopts & SFP_ENHOPTS_SOFT_RX_LOS)
+		mask |= SFP_F_LOS;
+
+	// Poll the soft state for hardware pins we want to ignore
+	sfp->state_soft_mask = ~sfp->state_hw_mask & mask;
 
 	if (sfp->state_soft_mask & (SFP_F_LOS | SFP_F_TX_FAULT) &&
 	    !sfp->need_poll)
@@ -523,10 +674,11 @@ static void sfp_soft_stop_poll(struct sfp *sfp)
 
 static unsigned int sfp_get_state(struct sfp *sfp)
 {
-	unsigned int state = sfp->get_state(sfp);
+	unsigned int soft = sfp->state_soft_mask & (SFP_F_LOS | SFP_F_TX_FAULT);
+	unsigned int state;
 
-	if (state & SFP_F_PRESENT &&
-	    sfp->state_soft_mask & (SFP_F_LOS | SFP_F_TX_FAULT))
+	state = sfp->get_state(sfp) & sfp->state_hw_mask;
+	if (state & SFP_F_PRESENT && soft)
 		state |= sfp_soft_get_state(sfp);
 
 	return state;
@@ -1195,90 +1347,45 @@ static const struct hwmon_ops sfp_hwmon_ops = {
 	.read_string = sfp_hwmon_read_string,
 };
 
-static u32 sfp_hwmon_chip_config[] = {
-	HWMON_C_REGISTER_TZ,
-	0,
-};
-
-static const struct hwmon_channel_info sfp_hwmon_chip = {
-	.type = hwmon_chip,
-	.config = sfp_hwmon_chip_config,
-};
-
-static u32 sfp_hwmon_temp_config[] = {
-	HWMON_T_INPUT |
-	HWMON_T_MAX | HWMON_T_MIN |
-	HWMON_T_MAX_ALARM | HWMON_T_MIN_ALARM |
-	HWMON_T_CRIT | HWMON_T_LCRIT |
-	HWMON_T_CRIT_ALARM | HWMON_T_LCRIT_ALARM |
-	HWMON_T_LABEL,
-	0,
-};
-
-static const struct hwmon_channel_info sfp_hwmon_temp_channel_info = {
-	.type = hwmon_temp,
-	.config = sfp_hwmon_temp_config,
-};
-
-static u32 sfp_hwmon_vcc_config[] = {
-	HWMON_I_INPUT |
-	HWMON_I_MAX | HWMON_I_MIN |
-	HWMON_I_MAX_ALARM | HWMON_I_MIN_ALARM |
-	HWMON_I_CRIT | HWMON_I_LCRIT |
-	HWMON_I_CRIT_ALARM | HWMON_I_LCRIT_ALARM |
-	HWMON_I_LABEL,
-	0,
-};
-
-static const struct hwmon_channel_info sfp_hwmon_vcc_channel_info = {
-	.type = hwmon_in,
-	.config = sfp_hwmon_vcc_config,
-};
-
-static u32 sfp_hwmon_bias_config[] = {
-	HWMON_C_INPUT |
-	HWMON_C_MAX | HWMON_C_MIN |
-	HWMON_C_MAX_ALARM | HWMON_C_MIN_ALARM |
-	HWMON_C_CRIT | HWMON_C_LCRIT |
-	HWMON_C_CRIT_ALARM | HWMON_C_LCRIT_ALARM |
-	HWMON_C_LABEL,
-	0,
-};
-
-static const struct hwmon_channel_info sfp_hwmon_bias_channel_info = {
-	.type = hwmon_curr,
-	.config = sfp_hwmon_bias_config,
-};
-
-static u32 sfp_hwmon_power_config[] = {
-	/* Transmit power */
-	HWMON_P_INPUT |
-	HWMON_P_MAX | HWMON_P_MIN |
-	HWMON_P_MAX_ALARM | HWMON_P_MIN_ALARM |
-	HWMON_P_CRIT | HWMON_P_LCRIT |
-	HWMON_P_CRIT_ALARM | HWMON_P_LCRIT_ALARM |
-	HWMON_P_LABEL,
-	/* Receive power */
-	HWMON_P_INPUT |
-	HWMON_P_MAX | HWMON_P_MIN |
-	HWMON_P_MAX_ALARM | HWMON_P_MIN_ALARM |
-	HWMON_P_CRIT | HWMON_P_LCRIT |
-	HWMON_P_CRIT_ALARM | HWMON_P_LCRIT_ALARM |
-	HWMON_P_LABEL,
-	0,
-};
-
-static const struct hwmon_channel_info sfp_hwmon_power_channel_info = {
-	.type = hwmon_power,
-	.config = sfp_hwmon_power_config,
-};
-
 static const struct hwmon_channel_info *sfp_hwmon_info[] = {
-	&sfp_hwmon_chip,
-	&sfp_hwmon_vcc_channel_info,
-	&sfp_hwmon_temp_channel_info,
-	&sfp_hwmon_bias_channel_info,
-	&sfp_hwmon_power_channel_info,
+	HWMON_CHANNEL_INFO(chip,
+			   HWMON_C_REGISTER_TZ),
+	HWMON_CHANNEL_INFO(in,
+			   HWMON_I_INPUT |
+			   HWMON_I_MAX | HWMON_I_MIN |
+			   HWMON_I_MAX_ALARM | HWMON_I_MIN_ALARM |
+			   HWMON_I_CRIT | HWMON_I_LCRIT |
+			   HWMON_I_CRIT_ALARM | HWMON_I_LCRIT_ALARM |
+			   HWMON_I_LABEL),
+	HWMON_CHANNEL_INFO(temp,
+			   HWMON_T_INPUT |
+			   HWMON_T_MAX | HWMON_T_MIN |
+			   HWMON_T_MAX_ALARM | HWMON_T_MIN_ALARM |
+			   HWMON_T_CRIT | HWMON_T_LCRIT |
+			   HWMON_T_CRIT_ALARM | HWMON_T_LCRIT_ALARM |
+			   HWMON_T_LABEL),
+	HWMON_CHANNEL_INFO(curr,
+			   HWMON_C_INPUT |
+			   HWMON_C_MAX | HWMON_C_MIN |
+			   HWMON_C_MAX_ALARM | HWMON_C_MIN_ALARM |
+			   HWMON_C_CRIT | HWMON_C_LCRIT |
+			   HWMON_C_CRIT_ALARM | HWMON_C_LCRIT_ALARM |
+			   HWMON_C_LABEL),
+	HWMON_CHANNEL_INFO(power,
+			   /* Transmit power */
+			   HWMON_P_INPUT |
+			   HWMON_P_MAX | HWMON_P_MIN |
+			   HWMON_P_MAX_ALARM | HWMON_P_MIN_ALARM |
+			   HWMON_P_CRIT | HWMON_P_LCRIT |
+			   HWMON_P_CRIT_ALARM | HWMON_P_LCRIT_ALARM |
+			   HWMON_P_LABEL,
+			   /* Receive power */
+			   HWMON_P_INPUT |
+			   HWMON_P_MAX | HWMON_P_MIN |
+			   HWMON_P_MAX_ALARM | HWMON_P_MIN_ALARM |
+			   HWMON_P_CRIT | HWMON_P_LCRIT |
+			   HWMON_P_CRIT_ALARM | HWMON_P_LCRIT_ALARM |
+			   HWMON_P_LABEL),
 	NULL,
 };
 
@@ -1505,12 +1612,12 @@ static void sfp_sm_phy_detach(struct sfp *sfp)
 	sfp->mod_phy = NULL;
 }
 
-static int sfp_sm_probe_phy(struct sfp *sfp, bool is_c45)
+static int sfp_sm_probe_phy(struct sfp *sfp, int addr, bool is_c45)
 {
 	struct phy_device *phy;
 	int err;
 
-	phy = get_phy_device(sfp->i2c_mii, SFP_PHY_ADDR, is_c45);
+	phy = get_phy_device(sfp->i2c_mii, addr, is_c45);
 	if (phy == ERR_PTR(-ENODEV))
 		return PTR_ERR(phy);
 	if (IS_ERR(phy)) {
@@ -1606,6 +1713,14 @@ static void sfp_sm_fault(struct sfp *sfp, unsigned int next_state, bool warn)
 	}
 }
 
+static int sfp_sm_add_mdio_bus(struct sfp *sfp)
+{
+	if (sfp->mdio_protocol != MDIO_I2C_NONE)
+		return sfp_i2c_mdiobus_create(sfp);
+
+	return 0;
+}
+
 /* Probe a SFP for a PHY device if the module supports copper - the PHY
  * normally sits at I2C bus address 0x56, and may either be a clause 22
  * or clause 45 PHY.
@@ -1621,19 +1736,23 @@ static int sfp_sm_probe_for_phy(struct sfp *sfp)
 {
 	int err = 0;
 
-	switch (sfp->id.base.extended_cc) {
-	case SFF8024_ECC_10GBASE_T_SFI:
-	case SFF8024_ECC_10GBASE_T_SR:
-	case SFF8024_ECC_5GBASE_T:
-	case SFF8024_ECC_2_5GBASE_T:
-		err = sfp_sm_probe_phy(sfp, true);
+	switch (sfp->mdio_protocol) {
+	case MDIO_I2C_NONE:
 		break;
 
-	default:
-		if (sfp->id.base.e1000_base_t)
-			err = sfp_sm_probe_phy(sfp, false);
+	case MDIO_I2C_MARVELL_C22:
+		err = sfp_sm_probe_phy(sfp, SFP_PHY_ADDR, false);
+		break;
+
+	case MDIO_I2C_C45:
+		err = sfp_sm_probe_phy(sfp, SFP_PHY_ADDR, true);
+		break;
+
+	case MDIO_I2C_ROLLBALL:
+		err = sfp_sm_probe_phy(sfp, SFP_PHY_ADDR_ROLLBALL, true);
 		break;
 	}
+
 	return err;
 }
 
@@ -1947,17 +2066,33 @@ static int sfp_sm_mod_probe(struct sfp *sfp, bool report)
 	if (ret < 0)
 		return ret;
 
-	if (!memcmp(id.base.vendor_name, "ALCATELLUCENT   ", 16) &&
-	    !memcmp(id.base.vendor_pn, "3FE46541AA      ", 16))
-		sfp->module_t_start_up = T_START_UP_BAD_GPON;
+	/* Initialise state bits to use from hardware */
+	sfp->state_hw_mask = SFP_F_PRESENT;
+	if (sfp->gpio[GPIO_TX_DISABLE])
+		sfp->state_hw_mask |= SFP_F_TX_DISABLE;
+	if (sfp->gpio[GPIO_TX_FAULT])
+		sfp->state_hw_mask |= SFP_F_TX_FAULT;
+	if (sfp->gpio[GPIO_LOS])
+		sfp->state_hw_mask |= SFP_F_LOS;
+
+	sfp->module_t_start_up = T_START_UP;
+	sfp->module_t_wait = T_WAIT;
+
+	sfp->tx_fault_ignore = false;
+
+	if (sfp->id.base.extended_cc == SFF8024_ECC_10GBASE_T_SFI ||
+	    sfp->id.base.extended_cc == SFF8024_ECC_10GBASE_T_SR ||
+	    sfp->id.base.extended_cc == SFF8024_ECC_5GBASE_T ||
+	    sfp->id.base.extended_cc == SFF8024_ECC_2_5GBASE_T)
+		sfp->mdio_protocol = MDIO_I2C_C45;
+	else if (sfp->id.base.e1000_base_t)
+		sfp->mdio_protocol = MDIO_I2C_MARVELL_C22;
 	else
-		sfp->module_t_start_up = T_START_UP;
+		sfp->mdio_protocol = MDIO_I2C_NONE;
 
-	if (!memcmp(id.base.vendor_name, "HUAWEI          ", 16) &&
-	    !memcmp(id.base.vendor_pn, "MA5671A         ", 16))
-		sfp->tx_fault_ignore = true;
-	else
-		sfp->tx_fault_ignore = false;
+	sfp->quirk = sfp_lookup_quirk(&id);
+	if (sfp->quirk && sfp->quirk->fixup)
+		sfp->quirk->fixup(sfp);
 
 	return 0;
 }
@@ -2071,7 +2206,8 @@ static void sfp_sm_module(struct sfp *sfp, unsigned int event)
 			break;
 
 		/* Report the module insertion to the upstream device */
-		err = sfp_module_insert(sfp->sfp_bus, &sfp->id);
+		err = sfp_module_insert(sfp->sfp_bus, &sfp->id,
+					sfp->quirk);
 		if (err < 0) {
 			sfp_sm_mod_next(sfp, SFP_MOD_ERROR, 0);
 			break;
@@ -2130,6 +2266,8 @@ static void sfp_sm_main(struct sfp *sfp, unsigned int event)
 			sfp_module_stop(sfp->sfp_bus);
 		if (sfp->mod_phy)
 			sfp_sm_phy_detach(sfp);
+		if (sfp->i2c_mii)
+			sfp_i2c_mdiobus_destroy(sfp);
 		sfp_module_tx_disable(sfp);
 		sfp_soft_stop_poll(sfp);
 		sfp_sm_next(sfp, SFP_S_DOWN, 0);
@@ -2153,9 +2291,10 @@ static void sfp_sm_main(struct sfp *sfp, unsigned int event)
 
 		/* We need to check the TX_FAULT state, which is not defined
 		 * while TX_DISABLE is asserted. The earliest we want to do
-		 * anything (such as probe for a PHY) is 50ms.
+		 * anything (such as probe for a PHY) is 50ms (or more on
+		 * specific modules).
 		 */
-		sfp_sm_next(sfp, SFP_S_WAIT, T_WAIT);
+		sfp_sm_next(sfp, SFP_S_WAIT, sfp->module_t_wait);
 		break;
 
 	case SFP_S_WAIT:
@@ -2169,8 +2308,8 @@ static void sfp_sm_main(struct sfp *sfp, unsigned int event)
 			 * deasserting.
 			 */
 			timeout = sfp->module_t_start_up;
-			if (timeout > T_WAIT)
-				timeout -= T_WAIT;
+			if (timeout > sfp->module_t_wait)
+				timeout -= sfp->module_t_wait;
 			else
 				timeout = 1;
 
@@ -2192,6 +2331,12 @@ static void sfp_sm_main(struct sfp *sfp, unsigned int event)
 				     sfp->sm_fault_retries == N_FAULT_INIT);
 		} else if (event == SFP_E_TIMEOUT || event == SFP_E_TX_CLEAR) {
 	init_done:
+			/* Create mdiobus and start trying for PHY */
+			ret = sfp_sm_add_mdio_bus(sfp);
+			if (ret < 0) {
+				sfp_sm_next(sfp, SFP_S_FAIL, 0);
+				break;
+			}
 			sfp->sm_phy_retries = R_PHY_RETRY;
 			goto phy_probe;
 		}
@@ -2573,6 +2718,8 @@ static int sfp_probe(struct platform_device *pdev)
 				return PTR_ERR(sfp->gpio[i]);
 		}
 
+	sfp->state_hw_mask = SFP_F_PRESENT;
+
 	sfp->get_state = sfp_gpio_get_state;
 	sfp->set_state = sfp_gpio_set_state;
 
diff --git a/drivers/net/phy/sfp.h b/drivers/net/phy/sfp.h
index 27226535c72b..6cf1643214d3 100644
--- a/drivers/net/phy/sfp.h
+++ b/drivers/net/phy/sfp.h
@@ -6,6 +6,14 @@
 
 struct sfp;
 
+struct sfp_quirk {
+	const char *vendor;
+	const char *part;
+	void (*modes)(const struct sfp_eeprom_id *id, unsigned long *modes,
+		      unsigned long *interfaces);
+	void (*fixup)(struct sfp *sfp);
+};
+
 struct sfp_socket_ops {
 	void (*attach)(struct sfp *sfp);
 	void (*detach)(struct sfp *sfp);
@@ -23,7 +31,8 @@ int sfp_add_phy(struct sfp_bus *bus, struct phy_device *phydev);
 void sfp_remove_phy(struct sfp_bus *bus);
 void sfp_link_up(struct sfp_bus *bus);
 void sfp_link_down(struct sfp_bus *bus);
-int sfp_module_insert(struct sfp_bus *bus, const struct sfp_eeprom_id *id);
+int sfp_module_insert(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
+		      const struct sfp_quirk *quirk);
 void sfp_module_remove(struct sfp_bus *bus);
 int sfp_module_start(struct sfp_bus *bus);
 void sfp_module_stop(struct sfp_bus *bus);
diff --git a/drivers/net/phy/smsc.c b/drivers/net/phy/smsc.c
index 69423b8965b3..ac7481ce2fc1 100644
--- a/drivers/net/phy/smsc.c
+++ b/drivers/net/phy/smsc.c
@@ -46,7 +46,6 @@ static struct smsc_hw_stat smsc_hw_stats[] = {
 struct smsc_phy_priv {
 	u16 intmask;
 	bool energy_enable;
-	struct clk *refclk;
 };
 
 static int smsc_phy_ack_interrupt(struct phy_device *phydev)
@@ -285,20 +284,12 @@ static void smsc_get_stats(struct phy_device *phydev,
 		data[i] = smsc_get_stat(phydev, i);
 }
 
-static void smsc_phy_remove(struct phy_device *phydev)
-{
-	struct smsc_phy_priv *priv = phydev->priv;
-
-	clk_disable_unprepare(priv->refclk);
-	clk_put(priv->refclk);
-}
-
 static int smsc_phy_probe(struct phy_device *phydev)
 {
 	struct device *dev = &phydev->mdio.dev;
 	struct device_node *of_node = dev->of_node;
 	struct smsc_phy_priv *priv;
-	int ret;
+	struct clk *refclk;
 
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
@@ -312,22 +303,12 @@ static int smsc_phy_probe(struct phy_device *phydev)
 	phydev->priv = priv;
 
 	/* Make clk optional to keep DTB backward compatibility. */
-	priv->refclk = clk_get_optional(dev, NULL);
-	if (IS_ERR(priv->refclk))
-		return dev_err_probe(dev, PTR_ERR(priv->refclk),
+	refclk = devm_clk_get_optional_enabled(dev, NULL);
+	if (IS_ERR(refclk))
+		return dev_err_probe(dev, PTR_ERR(refclk),
 				     "Failed to request clock\n");
 
-	ret = clk_prepare_enable(priv->refclk);
-	if (ret)
-		return ret;
-
-	ret = clk_set_rate(priv->refclk, 50 * 1000 * 1000);
-	if (ret) {
-		clk_disable_unprepare(priv->refclk);
-		return ret;
-	}
-
-	return 0;
+	return clk_set_rate(refclk, 50 * 1000 * 1000);
 }
 
 static struct phy_driver smsc_phy_driver[] = {
@@ -429,7 +410,6 @@ static struct phy_driver smsc_phy_driver[] = {
 	/* PHY_BASIC_FEATURES */
 
 	.probe		= smsc_phy_probe,
-	.remove		= smsc_phy_remove,
 
 	/* basic functions */
 	.read_status	= lan87xx_read_status,
diff --git a/drivers/net/phy/spi_ks8995.c b/drivers/net/phy/spi_ks8995.c
index ff37f8ba6758..d4202d40d47a 100644
--- a/drivers/net/phy/spi_ks8995.c
+++ b/drivers/net/phy/spi_ks8995.c
@@ -17,7 +17,6 @@
 #include <linux/device.h>
 #include <linux/gpio/consumer.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 
 #include <linux/spi/spi.h>
 
@@ -137,15 +136,10 @@ static const struct ks8995_chip_params ks8995_chip[] = {
 	},
 };
 
-struct ks8995_pdata {
-	int reset_gpio;
-	enum of_gpio_flags reset_gpio_flags;
-};
-
 struct ks8995_switch {
 	struct spi_device	*spi;
 	struct mutex		lock;
-	struct ks8995_pdata	*pdata;
+	struct gpio_desc	*reset_gpio;
 	struct bin_attribute	regs_attr;
 	const struct ks8995_chip_params	*chip;
 	int			revision_id;
@@ -401,24 +395,6 @@ err_out:
 	return err;
 }
 
-/* ks8995_parse_dt - setup platform data from devicetree
- * @ks: pointer to switch instance
- *
- * Parses supported DT properties and sets up platform data
- * accordingly.
- */
-static void ks8995_parse_dt(struct ks8995_switch *ks)
-{
-	struct device_node *np = ks->spi->dev.of_node;
-	struct ks8995_pdata *pdata = ks->pdata;
-
-	if (!np)
-		return;
-
-	pdata->reset_gpio = of_get_named_gpio_flags(np, "reset-gpios", 0,
-		&pdata->reset_gpio_flags);
-}
-
 static const struct bin_attribute ks8995_registers_attr = {
 	.attr = {
 		.name   = "registers",
@@ -449,38 +425,22 @@ static int ks8995_probe(struct spi_device *spi)
 	ks->spi = spi;
 	ks->chip = &ks8995_chip[variant];
 
-	if (ks->spi->dev.of_node) {
-		ks->pdata = devm_kzalloc(&spi->dev, sizeof(*ks->pdata),
-					 GFP_KERNEL);
-		if (!ks->pdata)
-			return -ENOMEM;
-
-		ks->pdata->reset_gpio = -1;
-
-		ks8995_parse_dt(ks);
+	ks->reset_gpio = devm_gpiod_get_optional(&spi->dev, "reset",
+						 GPIOD_OUT_HIGH);
+	err = PTR_ERR_OR_ZERO(ks->reset_gpio);
+	if (err) {
+		dev_err(&spi->dev,
+			"failed to get reset gpio: %d\n", err);
+		return err;
 	}
 
-	if (!ks->pdata)
-		ks->pdata = spi->dev.platform_data;
+	err = gpiod_set_consumer_name(ks->reset_gpio, "switch-reset");
+	if (err)
+		return err;
 
 	/* de-assert switch reset */
-	if (ks->pdata && gpio_is_valid(ks->pdata->reset_gpio)) {
-		unsigned long flags;
-
-		flags = (ks->pdata->reset_gpio_flags == OF_GPIO_ACTIVE_LOW ?
-			 GPIOF_ACTIVE_LOW : 0);
-
-		err = devm_gpio_request_one(&spi->dev,
-					    ks->pdata->reset_gpio,
-					    flags, "switch-reset");
-		if (err) {
-			dev_err(&spi->dev,
-				"failed to get reset-gpios: %d\n", err);
-			return -EIO;
-		}
-
-		gpiod_set_value(gpio_to_desc(ks->pdata->reset_gpio), 0);
-	}
+	/* FIXME: this likely requires a delay */
+	gpiod_set_value_cansleep(ks->reset_gpio, 0);
 
 	spi_set_drvdata(spi, ks);
 
@@ -524,8 +484,7 @@ static void ks8995_remove(struct spi_device *spi)
 	sysfs_remove_bin_file(&spi->dev.kobj, &ks->regs_attr);
 
 	/* assert reset */
-	if (ks->pdata && gpio_is_valid(ks->pdata->reset_gpio))
-		gpiod_set_value(gpio_to_desc(ks->pdata->reset_gpio), 1);
+	gpiod_set_value_cansleep(ks->reset_gpio, 1);
 }
 
 /* ------------------------------------------------------------------------ */
diff --git a/drivers/net/pse-pd/Kconfig b/drivers/net/pse-pd/Kconfig
new file mode 100644
index 000000000000..73d163704068
--- /dev/null
+++ b/drivers/net/pse-pd/Kconfig
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Ethernet Power Sourcing Equipment drivers
+#
+
+menuconfig PSE_CONTROLLER
+	bool "Ethernet Power Sourcing Equipment Support"
+	help
+	  Generic Power Sourcing Equipment Controller support.
+
+	  If unsure, say no.
+
+if PSE_CONTROLLER
+
+config PSE_REGULATOR
+	tristate "Regulator based PSE controller"
+	help
+	  This module provides support for simple regulator based Ethernet Power
+	  Sourcing Equipment without automatic classification support. For
+	  example for basic implementation of PoDL (802.3bu) specification.
+
+endif
diff --git a/drivers/net/pse-pd/Makefile b/drivers/net/pse-pd/Makefile
new file mode 100644
index 000000000000..1b8aa4c70f0b
--- /dev/null
+++ b/drivers/net/pse-pd/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Makefile for Linux PSE drivers
+
+obj-$(CONFIG_PSE_CONTROLLER) += pse_core.o
+
+obj-$(CONFIG_PSE_REGULATOR) += pse_regulator.o
diff --git a/drivers/net/pse-pd/pse_core.c b/drivers/net/pse-pd/pse_core.c
new file mode 100644
index 000000000000..146b81f08a89
--- /dev/null
+++ b/drivers/net/pse-pd/pse_core.c
@@ -0,0 +1,314 @@
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// Framework for Ethernet Power Sourcing Equipment
+//
+// Copyright (c) 2022 Pengutronix, Oleksij Rempel <kernel@pengutronix.de>
+//
+
+#include <linux/device.h>
+#include <linux/of.h>
+#include <linux/pse-pd/pse.h>
+
+static DEFINE_MUTEX(pse_list_mutex);
+static LIST_HEAD(pse_controller_list);
+
+/**
+ * struct pse_control - a PSE control
+ * @pcdev: a pointer to the PSE controller device
+ *         this PSE control belongs to
+ * @list: list entry for the pcdev's PSE controller list
+ * @id: ID of the PSE line in the PSE controller device
+ * @refcnt: Number of gets of this pse_control
+ */
+struct pse_control {
+	struct pse_controller_dev *pcdev;
+	struct list_head list;
+	unsigned int id;
+	struct kref refcnt;
+};
+
+/**
+ * of_pse_zero_xlate - dummy function for controllers with one only control
+ * @pcdev: a pointer to the PSE controller device
+ * @pse_spec: PSE line specifier as found in the device tree
+ *
+ * This static translation function is used by default if of_xlate in
+ * :c:type:`pse_controller_dev` is not set. It is useful for all PSE
+ * controllers with #pse-cells = <0>.
+ */
+static int of_pse_zero_xlate(struct pse_controller_dev *pcdev,
+			     const struct of_phandle_args *pse_spec)
+{
+	return 0;
+}
+
+/**
+ * of_pse_simple_xlate - translate pse_spec to the PSE line number
+ * @pcdev: a pointer to the PSE controller device
+ * @pse_spec: PSE line specifier as found in the device tree
+ *
+ * This static translation function is used by default if of_xlate in
+ * :c:type:`pse_controller_dev` is not set. It is useful for all PSE
+ * controllers with 1:1 mapping, where PSE lines can be indexed by number
+ * without gaps.
+ */
+static int of_pse_simple_xlate(struct pse_controller_dev *pcdev,
+			       const struct of_phandle_args *pse_spec)
+{
+	if (pse_spec->args[0] >= pcdev->nr_lines)
+		return -EINVAL;
+
+	return pse_spec->args[0];
+}
+
+/**
+ * pse_controller_register - register a PSE controller device
+ * @pcdev: a pointer to the initialized PSE controller device
+ */
+int pse_controller_register(struct pse_controller_dev *pcdev)
+{
+	if (!pcdev->of_xlate) {
+		if (pcdev->of_pse_n_cells == 0)
+			pcdev->of_xlate = of_pse_zero_xlate;
+		else if (pcdev->of_pse_n_cells == 1)
+			pcdev->of_xlate = of_pse_simple_xlate;
+	}
+
+	mutex_init(&pcdev->lock);
+	INIT_LIST_HEAD(&pcdev->pse_control_head);
+
+	mutex_lock(&pse_list_mutex);
+	list_add(&pcdev->list, &pse_controller_list);
+	mutex_unlock(&pse_list_mutex);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pse_controller_register);
+
+/**
+ * pse_controller_unregister - unregister a PSE controller device
+ * @pcdev: a pointer to the PSE controller device
+ */
+void pse_controller_unregister(struct pse_controller_dev *pcdev)
+{
+	mutex_lock(&pse_list_mutex);
+	list_del(&pcdev->list);
+	mutex_unlock(&pse_list_mutex);
+}
+EXPORT_SYMBOL_GPL(pse_controller_unregister);
+
+static void devm_pse_controller_release(struct device *dev, void *res)
+{
+	pse_controller_unregister(*(struct pse_controller_dev **)res);
+}
+
+/**
+ * devm_pse_controller_register - resource managed pse_controller_register()
+ * @dev: device that is registering this PSE controller
+ * @pcdev: a pointer to the initialized PSE controller device
+ *
+ * Managed pse_controller_register(). For PSE controllers registered by
+ * this function, pse_controller_unregister() is automatically called on
+ * driver detach. See pse_controller_register() for more information.
+ */
+int devm_pse_controller_register(struct device *dev,
+				 struct pse_controller_dev *pcdev)
+{
+	struct pse_controller_dev **pcdevp;
+	int ret;
+
+	pcdevp = devres_alloc(devm_pse_controller_release, sizeof(*pcdevp),
+			      GFP_KERNEL);
+	if (!pcdevp)
+		return -ENOMEM;
+
+	ret = pse_controller_register(pcdev);
+	if (ret) {
+		devres_free(pcdevp);
+		return ret;
+	}
+
+	*pcdevp = pcdev;
+	devres_add(dev, pcdevp);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(devm_pse_controller_register);
+
+/* PSE control section */
+
+static void __pse_control_release(struct kref *kref)
+{
+	struct pse_control *psec = container_of(kref, struct pse_control,
+						  refcnt);
+
+	lockdep_assert_held(&pse_list_mutex);
+
+	module_put(psec->pcdev->owner);
+
+	list_del(&psec->list);
+	kfree(psec);
+}
+
+static void __pse_control_put_internal(struct pse_control *psec)
+{
+	lockdep_assert_held(&pse_list_mutex);
+
+	kref_put(&psec->refcnt, __pse_control_release);
+}
+
+/**
+ * pse_control_put - free the PSE control
+ * @psec: PSE control pointer
+ */
+void pse_control_put(struct pse_control *psec)
+{
+	if (IS_ERR_OR_NULL(psec))
+		return;
+
+	mutex_lock(&pse_list_mutex);
+	__pse_control_put_internal(psec);
+	mutex_unlock(&pse_list_mutex);
+}
+EXPORT_SYMBOL_GPL(pse_control_put);
+
+static struct pse_control *
+pse_control_get_internal(struct pse_controller_dev *pcdev, unsigned int index)
+{
+	struct pse_control *psec;
+
+	lockdep_assert_held(&pse_list_mutex);
+
+	list_for_each_entry(psec, &pcdev->pse_control_head, list) {
+		if (psec->id == index) {
+			kref_get(&psec->refcnt);
+			return psec;
+		}
+	}
+
+	psec = kzalloc(sizeof(*psec), GFP_KERNEL);
+	if (!psec)
+		return ERR_PTR(-ENOMEM);
+
+	if (!try_module_get(pcdev->owner)) {
+		kfree(psec);
+		return ERR_PTR(-ENODEV);
+	}
+
+	psec->pcdev = pcdev;
+	list_add(&psec->list, &pcdev->pse_control_head);
+	psec->id = index;
+	kref_init(&psec->refcnt);
+
+	return psec;
+}
+
+struct pse_control *
+of_pse_control_get(struct device_node *node)
+{
+	struct pse_controller_dev *r, *pcdev;
+	struct of_phandle_args args;
+	struct pse_control *psec;
+	int psec_id;
+	int ret;
+
+	if (!node)
+		return ERR_PTR(-EINVAL);
+
+	ret = of_parse_phandle_with_args(node, "pses", "#pse-cells", 0, &args);
+	if (ret)
+		return ERR_PTR(ret);
+
+	mutex_lock(&pse_list_mutex);
+	pcdev = NULL;
+	list_for_each_entry(r, &pse_controller_list, list) {
+		if (args.np == r->dev->of_node) {
+			pcdev = r;
+			break;
+		}
+	}
+
+	if (!pcdev) {
+		psec = ERR_PTR(-EPROBE_DEFER);
+		goto out;
+	}
+
+	if (WARN_ON(args.args_count != pcdev->of_pse_n_cells)) {
+		psec = ERR_PTR(-EINVAL);
+		goto out;
+	}
+
+	psec_id = pcdev->of_xlate(pcdev, &args);
+	if (psec_id < 0) {
+		psec = ERR_PTR(psec_id);
+		goto out;
+	}
+
+	/* pse_list_mutex also protects the pcdev's pse_control list */
+	psec = pse_control_get_internal(pcdev, psec_id);
+
+out:
+	mutex_unlock(&pse_list_mutex);
+	of_node_put(args.np);
+
+	return psec;
+}
+EXPORT_SYMBOL_GPL(of_pse_control_get);
+
+/**
+ * pse_ethtool_get_status - get status of PSE control
+ * @psec: PSE control pointer
+ * @extack: extack for reporting useful error messages
+ * @status: struct to store PSE status
+ */
+int pse_ethtool_get_status(struct pse_control *psec,
+			   struct netlink_ext_ack *extack,
+			   struct pse_control_status *status)
+{
+	const struct pse_controller_ops *ops;
+	int err;
+
+	ops = psec->pcdev->ops;
+
+	if (!ops->ethtool_get_status) {
+		NL_SET_ERR_MSG(extack,
+			       "PSE driver does not support status report");
+		return -EOPNOTSUPP;
+	}
+
+	mutex_lock(&psec->pcdev->lock);
+	err = ops->ethtool_get_status(psec->pcdev, psec->id, extack, status);
+	mutex_unlock(&psec->pcdev->lock);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(pse_ethtool_get_status);
+
+/**
+ * pse_ethtool_set_config - set PSE control configuration
+ * @psec: PSE control pointer
+ * @extack: extack for reporting useful error messages
+ * @config: Configuration of the test to run
+ */
+int pse_ethtool_set_config(struct pse_control *psec,
+			   struct netlink_ext_ack *extack,
+			   const struct pse_control_config *config)
+{
+	const struct pse_controller_ops *ops;
+	int err;
+
+	ops = psec->pcdev->ops;
+
+	if (!ops->ethtool_set_config) {
+		NL_SET_ERR_MSG(extack,
+			       "PSE driver does not configuration");
+		return -EOPNOTSUPP;
+	}
+
+	mutex_lock(&psec->pcdev->lock);
+	err = ops->ethtool_set_config(psec->pcdev, psec->id, extack, config);
+	mutex_unlock(&psec->pcdev->lock);
+
+	return err;
+}
+EXPORT_SYMBOL_GPL(pse_ethtool_set_config);
diff --git a/drivers/net/pse-pd/pse_regulator.c b/drivers/net/pse-pd/pse_regulator.c
new file mode 100644
index 000000000000..e2bf8306ca90
--- /dev/null
+++ b/drivers/net/pse-pd/pse_regulator.c
@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// Driver for the regulator based Ethernet Power Sourcing Equipment, without
+// auto classification support.
+//
+// Copyright (c) 2022 Pengutronix, Oleksij Rempel <kernel@pengutronix.de>
+//
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pse-pd/pse.h>
+#include <linux/regulator/consumer.h>
+
+struct pse_reg_priv {
+	struct pse_controller_dev pcdev;
+	struct regulator *ps; /*power source */
+	enum ethtool_podl_pse_admin_state admin_state;
+};
+
+static struct pse_reg_priv *to_pse_reg(struct pse_controller_dev *pcdev)
+{
+	return container_of(pcdev, struct pse_reg_priv, pcdev);
+}
+
+static int
+pse_reg_ethtool_set_config(struct pse_controller_dev *pcdev, unsigned long id,
+			   struct netlink_ext_ack *extack,
+			   const struct pse_control_config *config)
+{
+	struct pse_reg_priv *priv = to_pse_reg(pcdev);
+	int ret;
+
+	if (priv->admin_state == config->admin_cotrol)
+		return 0;
+
+	switch (config->admin_cotrol) {
+	case ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED:
+		ret = regulator_enable(priv->ps);
+		break;
+	case ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED:
+		ret = regulator_disable(priv->ps);
+		break;
+	default:
+		dev_err(pcdev->dev, "Unknown admin state %i\n",
+			config->admin_cotrol);
+		ret = -ENOTSUPP;
+	}
+
+	if (ret)
+		return ret;
+
+	priv->admin_state = config->admin_cotrol;
+
+	return 0;
+}
+
+static int
+pse_reg_ethtool_get_status(struct pse_controller_dev *pcdev, unsigned long id,
+			   struct netlink_ext_ack *extack,
+			   struct pse_control_status *status)
+{
+	struct pse_reg_priv *priv = to_pse_reg(pcdev);
+	int ret;
+
+	ret = regulator_is_enabled(priv->ps);
+	if (ret < 0)
+		return ret;
+
+	if (!ret)
+		status->podl_pw_status = ETHTOOL_PODL_PSE_PW_D_STATUS_DISABLED;
+	else
+		status->podl_pw_status =
+			ETHTOOL_PODL_PSE_PW_D_STATUS_DELIVERING;
+
+	status->podl_admin_state = priv->admin_state;
+
+	return 0;
+}
+
+static const struct pse_controller_ops pse_reg_ops = {
+	.ethtool_get_status = pse_reg_ethtool_get_status,
+	.ethtool_set_config = pse_reg_ethtool_set_config,
+};
+
+static int
+pse_reg_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct pse_reg_priv *priv;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	if (!pdev->dev.of_node)
+		return -ENOENT;
+
+	priv->ps = devm_regulator_get_exclusive(dev, "pse");
+	if (IS_ERR(priv->ps))
+		return dev_err_probe(dev, PTR_ERR(priv->ps),
+				     "failed to get PSE regulator.\n");
+
+	platform_set_drvdata(pdev, priv);
+
+	ret = regulator_is_enabled(priv->ps);
+	if (ret < 0)
+		return ret;
+
+	if (ret)
+		priv->admin_state = ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED;
+	else
+		priv->admin_state = ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED;
+
+	priv->pcdev.owner = THIS_MODULE;
+	priv->pcdev.ops = &pse_reg_ops;
+	priv->pcdev.dev = dev;
+	ret = devm_pse_controller_register(dev, &priv->pcdev);
+	if (ret) {
+		dev_err(dev, "failed to register PSE controller (%pe)\n",
+			ERR_PTR(ret));
+		return ret;
+	}
+
+	return 0;
+}
+
+static const __maybe_unused struct of_device_id pse_reg_of_match[] = {
+	{ .compatible = "podl-pse-regulator", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, pse_reg_of_match);
+
+static struct platform_driver pse_reg_driver = {
+	.probe		= pse_reg_probe,
+	.driver		= {
+		.name		= "PSE regulator",
+		.of_match_table = of_match_ptr(pse_reg_of_match),
+	},
+};
+module_platform_driver(pse_reg_driver);
+
+MODULE_AUTHOR("Oleksij Rempel <kernel@pengutronix.de>");
+MODULE_DESCRIPTION("regulator based Ethernet Power Sourcing Equipment");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:pse-regulator");
diff --git a/drivers/net/rionet.c b/drivers/net/rionet.c
index 39e61e07e489..fbcb9d05da64 100644
--- a/drivers/net/rionet.c
+++ b/drivers/net/rionet.c
@@ -443,10 +443,10 @@ static void rionet_get_drvinfo(struct net_device *ndev,
 {
 	struct rionet_private *rnet = netdev_priv(ndev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
-	strlcpy(info->fw_version, "n/a", sizeof(info->fw_version));
-	strlcpy(info->bus_info, rnet->mport->name, sizeof(info->bus_info));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->fw_version, "n/a", sizeof(info->fw_version));
+	strscpy(info->bus_info, rnet->mport->name, sizeof(info->bus_info));
 }
 
 static u32 rionet_get_msglevel(struct net_device *ndev)
diff --git a/drivers/net/team/team.c b/drivers/net/team/team.c
index 154a3c0a6dfd..62ade69295a9 100644
--- a/drivers/net/team/team.c
+++ b/drivers/net/team/team.c
@@ -2082,8 +2082,8 @@ static const struct net_device_ops team_netdev_ops = {
 static void team_ethtool_get_drvinfo(struct net_device *dev,
 				     struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
-	strlcpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version));
+	strscpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version));
 }
 
 static int team_ethtool_get_link_ksettings(struct net_device *dev,
@@ -2852,6 +2852,7 @@ static struct genl_family team_nl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= team_nl_ops,
 	.n_small_ops	= ARRAY_SIZE(team_nl_ops),
+	.resv_start_op	= TEAM_CMD_PORT_LIST_GET + 1,
 	.mcgrps		= team_nl_mcgrps,
 	.n_mcgrps	= ARRAY_SIZE(team_nl_mcgrps),
 };
diff --git a/drivers/net/thunderbolt.c b/drivers/net/thunderbolt.c
index ff5d0e98a088..83fcaeb2ac5e 100644
--- a/drivers/net/thunderbolt.c
+++ b/drivers/net/thunderbolt.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Networking over Thunderbolt cable using Apple ThunderboltIP protocol
+ * Networking over Thunderbolt/USB4 cables using USB4NET protocol
+ * (formerly Apple ThunderboltIP).
  *
  * Copyright (C) 2017, Intel Corporation
  * Authors: Amir Levy <amir.jer.levy@intel.com>
@@ -30,6 +31,7 @@
 #define TBNET_RING_SIZE		256
 #define TBNET_LOGIN_RETRIES	60
 #define TBNET_LOGOUT_RETRIES	10
+#define TBNET_E2E		BIT(0)
 #define TBNET_MATCH_FRAGS_ID	BIT(1)
 #define TBNET_64K_FRAMES	BIT(2)
 #define TBNET_MAX_MTU		SZ_64K
@@ -209,6 +211,10 @@ static const uuid_t tbnet_svc_uuid =
 
 static struct tb_property_dir *tbnet_dir;
 
+static bool tbnet_e2e = true;
+module_param_named(e2e, tbnet_e2e, bool, 0444);
+MODULE_PARM_DESC(e2e, "USB4NET full end-to-end flow control (default: true)");
+
 static void tbnet_fill_header(struct thunderbolt_ip_header *hdr, u64 route,
 	u8 sequence, const uuid_t *initiator_uuid, const uuid_t *target_uuid,
 	enum thunderbolt_ip_type type, size_t size, u32 command_id)
@@ -612,18 +618,13 @@ static void tbnet_connected_work(struct work_struct *work)
 		return;
 	}
 
-	/* Both logins successful so enable the high-speed DMA paths and
-	 * start the network device queue.
+	/* Both logins successful so enable the rings, high-speed DMA
+	 * paths and start the network device queue.
+	 *
+	 * Note we enable the DMA paths last to make sure we have primed
+	 * the Rx ring before any incoming packets are allowed to
+	 * arrive.
 	 */
-	ret = tb_xdomain_enable_paths(net->xd, net->local_transmit_path,
-				      net->rx_ring.ring->hop,
-				      net->remote_transmit_path,
-				      net->tx_ring.ring->hop);
-	if (ret) {
-		netdev_err(net->dev, "failed to enable DMA paths\n");
-		return;
-	}
-
 	tb_ring_start(net->tx_ring.ring);
 	tb_ring_start(net->rx_ring.ring);
 
@@ -635,10 +636,21 @@ static void tbnet_connected_work(struct work_struct *work)
 	if (ret)
 		goto err_free_rx_buffers;
 
+	ret = tb_xdomain_enable_paths(net->xd, net->local_transmit_path,
+				      net->rx_ring.ring->hop,
+				      net->remote_transmit_path,
+				      net->tx_ring.ring->hop);
+	if (ret) {
+		netdev_err(net->dev, "failed to enable DMA paths\n");
+		goto err_free_tx_buffers;
+	}
+
 	netif_carrier_on(net->dev);
 	netif_start_queue(net->dev);
 	return;
 
+err_free_tx_buffers:
+	tbnet_free_buffers(&net->tx_ring);
 err_free_rx_buffers:
 	tbnet_free_buffers(&net->rx_ring);
 err_stop_rings:
@@ -867,6 +879,7 @@ static int tbnet_open(struct net_device *dev)
 	struct tb_xdomain *xd = net->xd;
 	u16 sof_mask, eof_mask;
 	struct tb_ring *ring;
+	unsigned int flags;
 	int hopid;
 
 	netif_carrier_off(dev);
@@ -891,9 +904,14 @@ static int tbnet_open(struct net_device *dev)
 	sof_mask = BIT(TBIP_PDF_FRAME_START);
 	eof_mask = BIT(TBIP_PDF_FRAME_END);
 
-	ring = tb_ring_alloc_rx(xd->tb->nhi, -1, TBNET_RING_SIZE,
-				RING_FLAG_FRAME, 0, sof_mask, eof_mask,
-				tbnet_start_poll, net);
+	flags = RING_FLAG_FRAME;
+	/* Only enable full E2E if the other end supports it too */
+	if (tbnet_e2e && net->svc->prtcstns & TBNET_E2E)
+		flags |= RING_FLAG_E2E;
+
+	ring = tb_ring_alloc_rx(xd->tb->nhi, -1, TBNET_RING_SIZE, flags,
+				net->tx_ring.ring->hop, sof_mask,
+				eof_mask, tbnet_start_poll, net);
 	if (!ring) {
 		netdev_err(dev, "failed to allocate Rx ring\n");
 		tb_ring_free(net->tx_ring.ring);
@@ -1264,7 +1282,7 @@ static int tbnet_probe(struct tb_service *svc, const struct tb_service_id *id)
 	dev->features = dev->hw_features | NETIF_F_HIGHDMA;
 	dev->hard_header_len += sizeof(struct thunderbolt_ip_frame_header);
 
-	netif_napi_add(dev, &net->napi, tbnet_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(dev, &net->napi, tbnet_poll);
 
 	/* MTU range: 68 - 65522 */
 	dev->min_mtu = ETH_MIN_MTU;
@@ -1356,6 +1374,7 @@ static struct tb_service_driver tbnet_driver = {
 
 static int __init tbnet_init(void)
 {
+	unsigned int flags;
 	int ret;
 
 	tbnet_dir = tb_property_create_dir(&tbnet_dir_uuid);
@@ -1365,12 +1384,11 @@ static int __init tbnet_init(void)
 	tb_property_add_immediate(tbnet_dir, "prtcid", 1);
 	tb_property_add_immediate(tbnet_dir, "prtcvers", 1);
 	tb_property_add_immediate(tbnet_dir, "prtcrevs", 1);
-	/* Currently only announce support for match frags ID (bit 1). Bit 0
-	 * is reserved for full E2E flow control which we do not support at
-	 * the moment.
-	 */
-	tb_property_add_immediate(tbnet_dir, "prtcstns",
-				  TBNET_MATCH_FRAGS_ID | TBNET_64K_FRAMES);
+
+	flags = TBNET_MATCH_FRAGS_ID | TBNET_64K_FRAMES;
+	if (tbnet_e2e)
+		flags |= TBNET_E2E;
+	tb_property_add_immediate(tbnet_dir, "prtcstns", flags);
 
 	ret = tb_register_property_dir("network", tbnet_dir);
 	if (ret) {
@@ -1393,5 +1411,5 @@ module_exit(tbnet_exit);
 MODULE_AUTHOR("Amir Levy <amir.jer.levy@intel.com>");
 MODULE_AUTHOR("Michael Jamet <michael.jamet@intel.com>");
 MODULE_AUTHOR("Mika Westerberg <mika.westerberg@linux.intel.com>");
-MODULE_DESCRIPTION("Thunderbolt network driver");
+MODULE_DESCRIPTION("Thunderbolt/USB4 network driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index 259b2b84b2b3..27c6d235cbda 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -2664,7 +2664,7 @@ static ssize_t tun_flags_show(struct device *dev, struct device_attribute *attr,
 			      char *buf)
 {
 	struct tun_struct *tun = netdev_priv(to_net_dev(dev));
-	return sprintf(buf, "0x%x\n", tun_flags(tun));
+	return sysfs_emit(buf, "0x%x\n", tun_flags(tun));
 }
 
 static ssize_t owner_show(struct device *dev, struct device_attribute *attr,
@@ -2672,9 +2672,9 @@ static ssize_t owner_show(struct device *dev, struct device_attribute *attr,
 {
 	struct tun_struct *tun = netdev_priv(to_net_dev(dev));
 	return uid_valid(tun->owner)?
-		sprintf(buf, "%u\n",
-			from_kuid_munged(current_user_ns(), tun->owner)):
-		sprintf(buf, "-1\n");
+		sysfs_emit(buf, "%u\n",
+			   from_kuid_munged(current_user_ns(), tun->owner)) :
+		sysfs_emit(buf, "-1\n");
 }
 
 static ssize_t group_show(struct device *dev, struct device_attribute *attr,
@@ -2682,9 +2682,9 @@ static ssize_t group_show(struct device *dev, struct device_attribute *attr,
 {
 	struct tun_struct *tun = netdev_priv(to_net_dev(dev));
 	return gid_valid(tun->group) ?
-		sprintf(buf, "%u\n",
-			from_kgid_munged(current_user_ns(), tun->group)):
-		sprintf(buf, "-1\n");
+		sysfs_emit(buf, "%u\n",
+			   from_kgid_munged(current_user_ns(), tun->group)) :
+		sysfs_emit(buf, "-1\n");
 }
 
 static DEVICE_ATTR_RO(tun_flags);
@@ -2828,7 +2828,10 @@ static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
 		rcu_assign_pointer(tfile->tun, tun);
 	}
 
-	netif_carrier_on(tun->dev);
+	if (ifr->ifr_flags & IFF_NO_CARRIER)
+		netif_carrier_off(tun->dev);
+	else
+		netif_carrier_on(tun->dev);
 
 	/* Make sure persistent devices do not get stuck in
 	 * xoff state.
@@ -3056,8 +3059,8 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
 		 * This is needed because we never checked for invalid flags on
 		 * TUNSETIFF.
 		 */
-		return put_user(IFF_TUN | IFF_TAP | TUN_FEATURES,
-				(unsigned int __user*)argp);
+		return put_user(IFF_TUN | IFF_TAP | IFF_NO_CARRIER |
+				TUN_FEATURES, (unsigned int __user*)argp);
 	} else if (cmd == TUNSETQUEUE) {
 		return tun_set_queue(file, &ifr);
 	} else if (cmd == SIOCGSKNS) {
@@ -3540,15 +3543,15 @@ static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info
 {
 	struct tun_struct *tun = netdev_priv(dev);
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 
 	switch (tun->flags & TUN_TYPE_MASK) {
 	case IFF_TUN:
-		strlcpy(info->bus_info, "tun", sizeof(info->bus_info));
+		strscpy(info->bus_info, "tun", sizeof(info->bus_info));
 		break;
 	case IFF_TAP:
-		strlcpy(info->bus_info, "tap", sizeof(info->bus_info));
+		strscpy(info->bus_info, "tap", sizeof(info->bus_info));
 		break;
 	}
 }
diff --git a/drivers/net/usb/Kconfig b/drivers/net/usb/Kconfig
index 76659c1c525a..4402eedb3d1a 100644
--- a/drivers/net/usb/Kconfig
+++ b/drivers/net/usb/Kconfig
@@ -168,7 +168,7 @@ config USB_NET_AX8817X
 	tristate "ASIX AX88xxx Based USB 2.0 Ethernet Adapters"
 	depends on USB_USBNET
 	select CRC32
-	select PHYLIB
+	select PHYLINK
 	select AX88796B_PHY
 	imply NET_SELFTESTS
 	default y
diff --git a/drivers/net/usb/aqc111.c b/drivers/net/usb/aqc111.c
index 3020e81159d0..a017e9de2119 100644
--- a/drivers/net/usb/aqc111.c
+++ b/drivers/net/usb/aqc111.c
@@ -201,7 +201,7 @@ static void aqc111_get_drvinfo(struct net_device *net,
 
 	/* Inherit standard device info */
 	usbnet_get_drvinfo(net, info);
-	strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver));
+	strscpy(info->driver, DRIVER_NAME, sizeof(info->driver));
 	snprintf(info->fw_version, sizeof(info->fw_version), "%u.%u.%u",
 		 aqc111_data->fw_ver.major,
 		 aqc111_data->fw_ver.minor,
diff --git a/drivers/net/usb/asix.h b/drivers/net/usb/asix.h
index 21c1ca275cc4..74162190bccc 100644
--- a/drivers/net/usb/asix.h
+++ b/drivers/net/usb/asix.h
@@ -27,6 +27,7 @@
 #include <linux/if_vlan.h>
 #include <linux/phy.h>
 #include <net/selftests.h>
+#include <linux/phylink.h>
 
 #define DRIVER_VERSION "22-Dec-2011"
 #define DRIVER_NAME "asix"
@@ -185,6 +186,8 @@ struct asix_common_private {
 	struct mii_bus *mdio;
 	struct phy_device *phydev;
 	struct phy_device *phydev_int;
+	struct phylink *phylink;
+	struct phylink_config phylink_config;
 	u16 phy_addr;
 	bool embd_phy;
 	u8 chipcode;
diff --git a/drivers/net/usb/asix_common.c b/drivers/net/usb/asix_common.c
index 9ea91c3ff045..72ffc89b477a 100644
--- a/drivers/net/usb/asix_common.c
+++ b/drivers/net/usb/asix_common.c
@@ -752,8 +752,8 @@ void asix_get_drvinfo(struct net_device *net, struct ethtool_drvinfo *info)
 {
 	/* Inherit standard device info */
 	usbnet_get_drvinfo(net, info);
-	strlcpy(info->driver, DRIVER_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRIVER_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRIVER_NAME, sizeof(info->driver));
+	strscpy(info->version, DRIVER_VERSION, sizeof(info->version));
 }
 
 int asix_set_mac_address(struct net_device *net, void *p)
diff --git a/drivers/net/usb/asix_devices.c b/drivers/net/usb/asix_devices.c
index 5b5eb630c4b7..11f60d32be82 100644
--- a/drivers/net/usb/asix_devices.c
+++ b/drivers/net/usb/asix_devices.c
@@ -303,6 +303,24 @@ static int ax88772_ethtool_get_sset_count(struct net_device *ndev, int sset)
 	}
 }
 
+static void ax88772_ethtool_get_pauseparam(struct net_device *ndev,
+					  struct ethtool_pauseparam *pause)
+{
+	struct usbnet *dev = netdev_priv(ndev);
+	struct asix_common_private *priv = dev->driver_priv;
+
+	phylink_ethtool_get_pauseparam(priv->phylink, pause);
+}
+
+static int ax88772_ethtool_set_pauseparam(struct net_device *ndev,
+					 struct ethtool_pauseparam *pause)
+{
+	struct usbnet *dev = netdev_priv(ndev);
+	struct asix_common_private *priv = dev->driver_priv;
+
+	return phylink_ethtool_set_pauseparam(priv->phylink, pause);
+}
+
 static const struct ethtool_ops ax88772_ethtool_ops = {
 	.get_drvinfo		= asix_get_drvinfo,
 	.get_link		= usbnet_get_link,
@@ -319,6 +337,8 @@ static const struct ethtool_ops ax88772_ethtool_ops = {
 	.self_test		= net_selftest,
 	.get_strings		= ax88772_ethtool_get_strings,
 	.get_sset_count		= ax88772_ethtool_get_sset_count,
+	.get_pauseparam		= ax88772_ethtool_get_pauseparam,
+	.set_pauseparam		= ax88772_ethtool_set_pauseparam,
 };
 
 static int ax88772_reset(struct usbnet *dev)
@@ -343,7 +363,7 @@ static int ax88772_reset(struct usbnet *dev)
 	if (ret < 0)
 		goto out;
 
-	phy_start(priv->phydev);
+	phylink_start(priv->phylink);
 
 	return 0;
 
@@ -590,8 +610,11 @@ static void ax88772_suspend(struct usbnet *dev)
 	struct asix_common_private *priv = dev->driver_priv;
 	u16 medium;
 
-	if (netif_running(dev->net))
-		phy_stop(priv->phydev);
+	if (netif_running(dev->net)) {
+		rtnl_lock();
+		phylink_suspend(priv->phylink, false);
+		rtnl_unlock();
+	}
 
 	/* Stop MAC operation */
 	medium = asix_read_medium_status(dev, 1);
@@ -622,8 +645,11 @@ static void ax88772_resume(struct usbnet *dev)
 		if (!priv->reset(dev, 1))
 			break;
 
-	if (netif_running(dev->net))
-		phy_start(priv->phydev);
+	if (netif_running(dev->net)) {
+		rtnl_lock();
+		phylink_resume(priv->phylink);
+		rtnl_unlock();
+	}
 }
 
 static int asix_resume(struct usb_interface *intf)
@@ -667,8 +693,7 @@ static int ax88772_init_phy(struct usbnet *dev)
 		return -ENODEV;
 	}
 
-	ret = phy_connect_direct(dev->net, priv->phydev, &asix_adjust_link,
-				 PHY_INTERFACE_MODE_INTERNAL);
+	ret = phylink_connect_phy(priv->phylink, priv->phydev);
 	if (ret) {
 		netdev_err(dev->net, "Could not connect PHY\n");
 		return ret;
@@ -688,6 +713,9 @@ static int ax88772_init_phy(struct usbnet *dev)
 	 */
 	priv->phydev_int = mdiobus_get_phy(priv->mdio, AX_EMBD_PHY_ADDR);
 	if (!priv->phydev_int) {
+		rtnl_lock();
+		phylink_disconnect_phy(priv->phylink);
+		rtnl_unlock();
 		netdev_err(dev->net, "Could not find internal PHY\n");
 		return -ENODEV;
 	}
@@ -698,6 +726,89 @@ static int ax88772_init_phy(struct usbnet *dev)
 	return 0;
 }
 
+static void ax88772_mac_config(struct phylink_config *config, unsigned int mode,
+			      const struct phylink_link_state *state)
+{
+	/* Nothing to do */
+}
+
+static void ax88772_mac_link_down(struct phylink_config *config,
+				 unsigned int mode, phy_interface_t interface)
+{
+	struct usbnet *dev = netdev_priv(to_net_dev(config->dev));
+
+	asix_write_medium_mode(dev, 0, 0);
+	usbnet_link_change(dev, false, false);
+}
+
+static void ax88772_mac_link_up(struct phylink_config *config,
+			       struct phy_device *phy,
+			       unsigned int mode, phy_interface_t interface,
+			       int speed, int duplex,
+			       bool tx_pause, bool rx_pause)
+{
+	struct usbnet *dev = netdev_priv(to_net_dev(config->dev));
+	u16 m = AX_MEDIUM_AC | AX_MEDIUM_RE;
+
+	m |= duplex ? AX_MEDIUM_FD : 0;
+
+	switch (speed) {
+	case SPEED_100:
+		m |= AX_MEDIUM_PS;
+		break;
+	case SPEED_10:
+		break;
+	default:
+		return;
+	}
+
+	if (tx_pause)
+		m |= AX_MEDIUM_TFC;
+
+	if (rx_pause)
+		m |= AX_MEDIUM_RFC;
+
+	asix_write_medium_mode(dev, m, 0);
+	usbnet_link_change(dev, true, false);
+}
+
+static const struct phylink_mac_ops ax88772_phylink_mac_ops = {
+	.validate = phylink_generic_validate,
+	.mac_config = ax88772_mac_config,
+	.mac_link_down = ax88772_mac_link_down,
+	.mac_link_up = ax88772_mac_link_up,
+};
+
+static int ax88772_phylink_setup(struct usbnet *dev)
+{
+	struct asix_common_private *priv = dev->driver_priv;
+	phy_interface_t phy_if_mode;
+	struct phylink *phylink;
+
+	priv->phylink_config.dev = &dev->net->dev;
+	priv->phylink_config.type = PHYLINK_NETDEV;
+	priv->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE |
+		MAC_10 | MAC_100;
+
+	__set_bit(PHY_INTERFACE_MODE_INTERNAL,
+		  priv->phylink_config.supported_interfaces);
+	__set_bit(PHY_INTERFACE_MODE_RMII,
+		  priv->phylink_config.supported_interfaces);
+
+	if (priv->embd_phy)
+		phy_if_mode = PHY_INTERFACE_MODE_INTERNAL;
+	else
+		phy_if_mode = PHY_INTERFACE_MODE_RMII;
+
+	phylink = phylink_create(&priv->phylink_config, dev->net->dev.fwnode,
+				 phy_if_mode, &ax88772_phylink_mac_ops);
+	if (IS_ERR(phylink))
+		return PTR_ERR(phylink);
+
+	priv->phylink = phylink;
+	return 0;
+}
+
 static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf)
 {
 	struct asix_common_private *priv;
@@ -788,14 +899,22 @@ static int ax88772_bind(struct usbnet *dev, struct usb_interface *intf)
 	if (ret)
 		return ret;
 
-	return ax88772_init_phy(dev);
+	ret = ax88772_phylink_setup(dev);
+	if (ret)
+		return ret;
+
+	ret = ax88772_init_phy(dev);
+	if (ret)
+		phylink_destroy(priv->phylink);
+
+	return ret;
 }
 
 static int ax88772_stop(struct usbnet *dev)
 {
 	struct asix_common_private *priv = dev->driver_priv;
 
-	phy_stop(priv->phydev);
+	phylink_stop(priv->phylink);
 
 	return 0;
 }
@@ -804,7 +923,10 @@ static void ax88772_unbind(struct usbnet *dev, struct usb_interface *intf)
 {
 	struct asix_common_private *priv = dev->driver_priv;
 
-	phy_disconnect(priv->phydev);
+	rtnl_lock();
+	phylink_disconnect_phy(priv->phylink);
+	rtnl_unlock();
+	phylink_destroy(priv->phylink);
 	asix_rx_fixup_common_free(dev->driver_priv);
 }
 
diff --git a/drivers/net/usb/catc.c b/drivers/net/usb/catc.c
index 843893482abd..ff439ef535ac 100644
--- a/drivers/net/usb/catc.c
+++ b/drivers/net/usb/catc.c
@@ -672,8 +672,8 @@ static void catc_get_drvinfo(struct net_device *dev,
 			     struct ethtool_drvinfo *info)
 {
 	struct catc *catc = netdev_priv(dev);
-	strlcpy(info->driver, driver_name, sizeof(info->driver));
-	strlcpy(info->version, DRIVER_VERSION, sizeof(info->version));
+	strscpy(info->driver, driver_name, sizeof(info->driver));
+	strscpy(info->version, DRIVER_VERSION, sizeof(info->version));
 	usb_make_path(catc->usbdev, info->bus_info, sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c
index 3226ab33afae..f18ab8e220db 100644
--- a/drivers/net/usb/lan78xx.c
+++ b/drivers/net/usb/lan78xx.c
@@ -4374,7 +4374,7 @@ static int lan78xx_probe(struct usb_interface *intf,
 
 	netif_set_tso_max_size(netdev, LAN78XX_TSO_SIZE(dev));
 
-	netif_napi_add(netdev, &dev->napi, lan78xx_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(netdev, &dev->napi, lan78xx_poll);
 
 	INIT_DELAYED_WORK(&dev->wq, lan78xx_delayedwork);
 	init_usb_anchor(&dev->deferred);
diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c
index feb247e355f7..81ca64debc5b 100644
--- a/drivers/net/usb/pegasus.c
+++ b/drivers/net/usb/pegasus.c
@@ -894,7 +894,7 @@ static void pegasus_get_drvinfo(struct net_device *dev,
 {
 	pegasus_t *pegasus = netdev_priv(dev);
 
-	strlcpy(info->driver, driver_name, sizeof(info->driver));
+	strscpy(info->driver, driver_name, sizeof(info->driver));
 	usb_make_path(pegasus->usb, info->bus_info, sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c
index 0cb187def5bc..26c34a7c21bd 100644
--- a/drivers/net/usb/qmi_wwan.c
+++ b/drivers/net/usb/qmi_wwan.c
@@ -1402,6 +1402,7 @@ static const struct usb_device_id products[] = {
 	{QMI_FIXED_INTF(0x413c, 0x81b3, 8)},	/* Dell Wireless 5809e Gobi(TM) 4G LTE Mobile Broadband Card (rev3) */
 	{QMI_FIXED_INTF(0x413c, 0x81b6, 8)},	/* Dell Wireless 5811e */
 	{QMI_FIXED_INTF(0x413c, 0x81b6, 10)},	/* Dell Wireless 5811e */
+	{QMI_FIXED_INTF(0x413c, 0x81c2, 8)},	/* Dell Wireless 5811e */
 	{QMI_FIXED_INTF(0x413c, 0x81cc, 8)},	/* Dell Wireless 5816e */
 	{QMI_FIXED_INTF(0x413c, 0x81d7, 0)},	/* Dell Wireless 5821e */
 	{QMI_FIXED_INTF(0x413c, 0x81d7, 1)},	/* Dell Wireless 5821e preproduction config */
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index 688905ea0a6d..a481a1d831e2 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -770,6 +770,7 @@ enum rtl8152_flags {
 	RX_EPROTO,
 };
 
+#define DEVICE_ID_LENOVO_USB_C_TRAVEL_HUB		0x721e
 #define DEVICE_ID_THINKPAD_ONELINK_PLUS_DOCK		0x3054
 #define DEVICE_ID_THINKPAD_THUNDERBOLT3_DOCK_GEN2	0x3082
 #define DEVICE_ID_THINKPAD_USB_C_DONGLE			0x720c
@@ -1874,7 +1875,9 @@ static void intr_callback(struct urb *urb)
 			   "Stop submitting intr, status %d\n", status);
 		return;
 	case -EOVERFLOW:
-		netif_info(tp, intr, tp->netdev, "intr status -EOVERFLOW\n");
+		if (net_ratelimit())
+			netif_info(tp, intr, tp->netdev,
+				   "intr status -EOVERFLOW\n");
 		goto resubmit;
 	/* -EPIPE:  should clear the halt */
 	default:
@@ -2727,22 +2730,26 @@ static void _rtl8152_set_rx_mode(struct net_device *netdev)
 		ocp_data |= RCR_AM | RCR_AAP;
 		mc_filter[1] = 0xffffffff;
 		mc_filter[0] = 0xffffffff;
-	} else if ((netdev_mc_count(netdev) > multicast_filter_limit) ||
-		   (netdev->flags & IFF_ALLMULTI)) {
+	} else if ((netdev->flags & IFF_MULTICAST &&
+				netdev_mc_count(netdev) > multicast_filter_limit) ||
+			   (netdev->flags & IFF_ALLMULTI)) {
 		/* Too many to filter perfectly -- accept all multicasts. */
 		ocp_data |= RCR_AM;
 		mc_filter[1] = 0xffffffff;
 		mc_filter[0] = 0xffffffff;
 	} else {
-		struct netdev_hw_addr *ha;
-
 		mc_filter[1] = 0;
 		mc_filter[0] = 0;
-		netdev_for_each_mc_addr(ha, netdev) {
-			int bit_nr = ether_crc(ETH_ALEN, ha->addr) >> 26;
 
-			mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31);
-			ocp_data |= RCR_AM;
+		if (netdev->flags & IFF_MULTICAST) {
+			struct netdev_hw_addr *ha;
+
+			netdev_for_each_mc_addr(ha, netdev) {
+				int bit_nr = ether_crc(ETH_ALEN, ha->addr) >> 26;
+
+				mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 31);
+				ocp_data |= RCR_AM;
+			}
 		}
 	}
 
@@ -8602,11 +8609,11 @@ static void rtl8152_get_drvinfo(struct net_device *netdev,
 {
 	struct r8152 *tp = netdev_priv(netdev);
 
-	strlcpy(info->driver, MODULENAME, sizeof(info->driver));
-	strlcpy(info->version, DRIVER_VERSION, sizeof(info->version));
+	strscpy(info->driver, MODULENAME, sizeof(info->driver));
+	strscpy(info->version, DRIVER_VERSION, sizeof(info->version));
 	usb_make_path(tp->udev, info->bus_info, sizeof(info->bus_info));
 	if (!IS_ERR_OR_NULL(tp->rtl_fw.fw))
-		strlcpy(info->fw_version, tp->rtl_fw.version,
+		strscpy(info->fw_version, tp->rtl_fw.version,
 			sizeof(info->fw_version));
 }
 
@@ -9582,6 +9589,7 @@ static bool rtl8152_supports_lenovo_macpassthru(struct usb_device *udev)
 
 	if (vendor_id == VENDOR_ID_LENOVO) {
 		switch (product_id) {
+		case DEVICE_ID_LENOVO_USB_C_TRAVEL_HUB:
 		case DEVICE_ID_THINKPAD_ONELINK_PLUS_DOCK:
 		case DEVICE_ID_THINKPAD_THUNDERBOLT3_DOCK_GEN2:
 		case DEVICE_ID_THINKPAD_USB_C_DOCK_GEN2:
diff --git a/drivers/net/usb/rtl8150.c b/drivers/net/usb/rtl8150.c
index 3d2bf2acca94..97afd7335d86 100644
--- a/drivers/net/usb/rtl8150.c
+++ b/drivers/net/usb/rtl8150.c
@@ -769,8 +769,8 @@ static void rtl8150_get_drvinfo(struct net_device *netdev, struct ethtool_drvinf
 {
 	rtl8150_t *dev = netdev_priv(netdev);
 
-	strlcpy(info->driver, driver_name, sizeof(info->driver));
-	strlcpy(info->version, DRIVER_VERSION, sizeof(info->version));
+	strscpy(info->driver, driver_name, sizeof(info->driver));
+	strscpy(info->version, DRIVER_VERSION, sizeof(info->version));
 	usb_make_path(dev->udev, info->bus_info, sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/usb/sierra_net.c b/drivers/net/usb/sierra_net.c
index bb4cbe8fc846..b3ae949e6f1c 100644
--- a/drivers/net/usb/sierra_net.c
+++ b/drivers/net/usb/sierra_net.c
@@ -612,8 +612,8 @@ static void sierra_net_get_drvinfo(struct net_device *net,
 {
 	/* Inherit standard device info */
 	usbnet_get_drvinfo(net, info);
-	strlcpy(info->driver, driver_name, sizeof(info->driver));
-	strlcpy(info->version, DRIVER_VERSION, sizeof(info->version));
+	strscpy(info->driver, driver_name, sizeof(info->driver));
+	strscpy(info->version, DRIVER_VERSION, sizeof(info->version));
 }
 
 static u32 sierra_net_get_link(struct net_device *net)
diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c
index aaa89b4cfd50..64a9a80b2309 100644
--- a/drivers/net/usb/usbnet.c
+++ b/drivers/net/usb/usbnet.c
@@ -1050,9 +1050,9 @@ void usbnet_get_drvinfo (struct net_device *net, struct ethtool_drvinfo *info)
 {
 	struct usbnet *dev = netdev_priv(net);
 
-	strlcpy (info->driver, dev->driver_name, sizeof info->driver);
-	strlcpy (info->fw_version, dev->driver_info->description,
-		sizeof info->fw_version);
+	strscpy(info->driver, dev->driver_name, sizeof(info->driver));
+	strscpy(info->fw_version, dev->driver_info->description,
+		sizeof(info->fw_version));
 	usb_make_path (dev->udev, info->bus_info, sizeof info->bus_info);
 }
 EXPORT_SYMBOL_GPL(usbnet_get_drvinfo);
@@ -1598,6 +1598,7 @@ void usbnet_disconnect (struct usb_interface *intf)
 	struct usbnet		*dev;
 	struct usb_device	*xdev;
 	struct net_device	*net;
+	struct urb		*urb;
 
 	dev = usb_get_intfdata(intf);
 	usb_set_intfdata(intf, NULL);
@@ -1614,7 +1615,11 @@ void usbnet_disconnect (struct usb_interface *intf)
 	net = dev->net;
 	unregister_netdev (net);
 
-	usb_scuttle_anchored_urbs(&dev->deferred);
+	while ((urb = usb_get_from_anchor(&dev->deferred))) {
+		dev_kfree_skb(urb->context);
+		kfree(urb->sg);
+		usb_free_urb(urb);
+	}
 
 	if (dev->driver_info->unbind)
 		dev->driver_info->unbind(dev, intf);
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 466da01ba2e3..09682ea3354e 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -128,8 +128,8 @@ static int veth_get_link_ksettings(struct net_device *dev,
 
 static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 }
 
 static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
@@ -1070,7 +1070,7 @@ static int veth_enable_xdp_range(struct net_device *dev, int start, int end,
 		struct veth_rq *rq = &priv->rq[i];
 
 		if (!napi_already_on)
-			netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
+			netif_napi_add(dev, &rq->xdp_napi, veth_poll);
 		err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id);
 		if (err < 0)
 			goto err_rxq_reg;
@@ -1184,7 +1184,7 @@ static int veth_napi_enable_range(struct net_device *dev, int start, int end)
 	for (i = start; i < end; i++) {
 		struct veth_rq *rq = &priv->rq[i];
 
-		netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT);
+		netif_napi_add(dev, &rq->xdp_napi, veth_poll);
 	}
 
 	err = __veth_napi_enable_range(dev, start, end);
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 9cce7dec7366..e0e57083d442 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -2594,9 +2594,9 @@ static void virtnet_get_drvinfo(struct net_device *dev,
 	struct virtnet_info *vi = netdev_priv(dev);
 	struct virtio_device *vdev = vi->vdev;
 
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
-	strlcpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
-	strlcpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version));
+	strscpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info));
 
 }
 
diff --git a/drivers/net/vmxnet3/vmxnet3_drv.c b/drivers/net/vmxnet3/vmxnet3_drv.c
index 53b3b241e027..d3e7b27eb933 100644
--- a/drivers/net/vmxnet3/vmxnet3_drv.c
+++ b/drivers/net/vmxnet3/vmxnet3_drv.c
@@ -3882,11 +3882,11 @@ vmxnet3_probe_device(struct pci_dev *pdev,
 		for (i = 0; i < adapter->num_rx_queues; i++) {
 			netif_napi_add(adapter->netdev,
 				       &adapter->rx_queue[i].napi,
-				       vmxnet3_poll_rx_only, 64);
+				       vmxnet3_poll_rx_only);
 		}
 	} else {
 		netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
-			       vmxnet3_poll, 64);
+			       vmxnet3_poll);
 	}
 
 	netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
diff --git a/drivers/net/vmxnet3/vmxnet3_ethtool.c b/drivers/net/vmxnet3/vmxnet3_ethtool.c
index e2034adc3a1a..18cf7c723201 100644
--- a/drivers/net/vmxnet3/vmxnet3_ethtool.c
+++ b/drivers/net/vmxnet3/vmxnet3_ethtool.c
@@ -209,12 +209,12 @@ vmxnet3_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo)
 {
 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
 
-	strlcpy(drvinfo->driver, vmxnet3_driver_name, sizeof(drvinfo->driver));
+	strscpy(drvinfo->driver, vmxnet3_driver_name, sizeof(drvinfo->driver));
 
-	strlcpy(drvinfo->version, VMXNET3_DRIVER_VERSION_REPORT,
+	strscpy(drvinfo->version, VMXNET3_DRIVER_VERSION_REPORT,
 		sizeof(drvinfo->version));
 
-	strlcpy(drvinfo->bus_info, pci_name(adapter->pdev),
+	strscpy(drvinfo->bus_info, pci_name(adapter->pdev),
 		sizeof(drvinfo->bus_info));
 }
 
diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c
index 5df7a0abc39d..badf6f09ae51 100644
--- a/drivers/net/vrf.c
+++ b/drivers/net/vrf.c
@@ -1541,8 +1541,8 @@ static const struct l3mdev_ops vrf_l3mdev_ops = {
 static void vrf_get_drvinfo(struct net_device *dev,
 			    struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 }
 
 static const struct ethtool_ops vrf_ethtool_ops = {
diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c
index c3285242f74f..6ab669dcd1c6 100644
--- a/drivers/net/vxlan/vxlan_core.c
+++ b/drivers/net/vxlan/vxlan_core.c
@@ -713,12 +713,9 @@ static struct sk_buff *vxlan_gro_receive(struct sock *sk,
 
 	off_vx = skb_gro_offset(skb);
 	hlen = off_vx + sizeof(*vh);
-	vh   = skb_gro_header_fast(skb, off_vx);
-	if (skb_gro_header_hard(skb, hlen)) {
-		vh = skb_gro_header_slow(skb, hlen, off_vx);
-		if (unlikely(!vh))
-			goto out;
-	}
+	vh = skb_gro_header(skb, hlen, off_vx);
+	if (unlikely(!vh))
+		goto out;
 
 	skb_gro_postpull_rcsum(skb, vh, sizeof(struct vxlanhdr));
 
@@ -3313,8 +3310,8 @@ static int vxlan_validate(struct nlattr *tb[], struct nlattr *data[],
 static void vxlan_get_drvinfo(struct net_device *netdev,
 			      struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->version, VXLAN_VERSION, sizeof(drvinfo->version));
-	strlcpy(drvinfo->driver, "vxlan", sizeof(drvinfo->driver));
+	strscpy(drvinfo->version, VXLAN_VERSION, sizeof(drvinfo->version));
+	strscpy(drvinfo->driver, "vxlan", sizeof(drvinfo->driver));
 }
 
 static int vxlan_get_link_ksettings(struct net_device *dev,
diff --git a/drivers/net/wireguard/netlink.c b/drivers/net/wireguard/netlink.c
index 5c804bcabfe6..43c8c84e7ea8 100644
--- a/drivers/net/wireguard/netlink.c
+++ b/drivers/net/wireguard/netlink.c
@@ -620,6 +620,7 @@ static const struct genl_ops genl_ops[] = {
 static struct genl_family genl_family __ro_after_init = {
 	.ops = genl_ops,
 	.n_ops = ARRAY_SIZE(genl_ops),
+	.resv_start_op = WG_CMD_SET_DEVICE + 1,
 	.name = WG_GENL_NAME,
 	.version = WG_GENL_VERSION,
 	.maxattr = WGDEVICE_A_MAX,
diff --git a/drivers/net/wireguard/peer.c b/drivers/net/wireguard/peer.c
index 1acd00ab2fbc..1cb502a932e0 100644
--- a/drivers/net/wireguard/peer.c
+++ b/drivers/net/wireguard/peer.c
@@ -54,8 +54,7 @@ struct wg_peer *wg_peer_create(struct wg_device *wg,
 	skb_queue_head_init(&peer->staged_packet_queue);
 	wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
 	set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state);
-	netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll);
 	napi_enable(&peer->napi);
 	list_add_tail(&peer->peer_list, &wg->peer_list);
 	INIT_LIST_HEAD(&peer->allowedips_list);
diff --git a/drivers/net/wireless/ath/ath10k/bmi.c b/drivers/net/wireless/ath/ath10k/bmi.c
index 4481ed375f55..af6546572df2 100644
--- a/drivers/net/wireless/ath/ath10k/bmi.c
+++ b/drivers/net/wireless/ath/ath10k/bmi.c
@@ -101,7 +101,7 @@ int ath10k_bmi_get_target_info_sdio(struct ath10k *ar,
 	cmd.id = __cpu_to_le32(BMI_GET_TARGET_INFO);
 
 	/* Step 1: Read 4 bytes of the target info and check if it is
-	 * the special sentinal version word or the first word in the
+	 * the special sentinel version word or the first word in the
 	 * version response.
 	 */
 	resplen = sizeof(u32);
@@ -111,7 +111,7 @@ int ath10k_bmi_get_target_info_sdio(struct ath10k *ar,
 		return ret;
 	}
 
-	/* Some SDIO boards have a special sentinal byte before the real
+	/* Some SDIO boards have a special sentinel byte before the real
 	 * version response.
 	 */
 	if (__le32_to_cpu(tmp) == TARGET_VERSION_SENTINAL) {
diff --git a/drivers/net/wireless/ath/ath10k/ce.c b/drivers/net/wireless/ath/ath10k/ce.c
index c45c814fd122..59926227bd49 100644
--- a/drivers/net/wireless/ath/ath10k/ce.c
+++ b/drivers/net/wireless/ath/ath10k/ce.c
@@ -1323,7 +1323,7 @@ EXPORT_SYMBOL(ath10k_ce_per_engine_service);
 /*
  * Handler for per-engine interrupts on ALL active CEs.
  * This is used in cases where the system is sharing a
- * single interrput for all CEs
+ * single interrupt for all CEs
  */
 
 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c
index 276954b70d63..400f332a7ff0 100644
--- a/drivers/net/wireless/ath/ath10k/core.c
+++ b/drivers/net/wireless/ath/ath10k/core.c
@@ -98,6 +98,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = true,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA988X_HW_2_0_VERSION,
@@ -136,6 +137,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = true,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA9887_HW_1_0_VERSION,
@@ -175,6 +177,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA6174_HW_3_2_VERSION,
@@ -209,6 +212,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.supports_peer_stats_info = true,
 		.dynamic_sar_support = true,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA6174_HW_2_1_VERSION,
@@ -247,6 +251,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA6174_HW_2_1_VERSION,
@@ -285,6 +290,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA6174_HW_3_0_VERSION,
@@ -323,6 +329,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA6174_HW_3_2_VERSION,
@@ -365,6 +372,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.supports_peer_stats_info = true,
 		.dynamic_sar_support = true,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA99X0_HW_2_0_DEV_VERSION,
@@ -409,6 +417,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA9984_HW_1_0_DEV_VERSION,
@@ -460,6 +469,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA9888_HW_2_0_DEV_VERSION,
@@ -508,6 +518,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA9377_HW_1_0_DEV_VERSION,
@@ -546,6 +557,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA9377_HW_1_1_DEV_VERSION,
@@ -586,6 +598,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA9377_HW_1_1_DEV_VERSION,
@@ -617,6 +630,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.credit_size_workaround = true,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = QCA4019_HW_1_0_DEV_VERSION,
@@ -662,6 +676,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = false,
 		.hw_restart_disconnect = false,
+		.use_fw_tx_credits = true,
 	},
 	{
 		.id = WCN3990_HW_1_0_DEV_VERSION,
@@ -693,6 +708,7 @@ static const struct ath10k_hw_params ath10k_hw_params_list[] = {
 		.tx_stats_over_pktlog = false,
 		.dynamic_sar_support = true,
 		.hw_restart_disconnect = true,
+		.use_fw_tx_credits = false,
 	},
 };
 
@@ -3080,7 +3096,7 @@ int ath10k_core_start(struct ath10k *ar, enum ath10k_firmware_mode mode,
 		 * enabled always.
 		 *
 		 * We can still enable BTCOEX if firmware has the support
-		 * eventhough btceox_support value is
+		 * even though btceox_support value is
 		 * ATH10K_DT_BTCOEX_NOT_FOUND
 		 */
 
diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h
index d70d7d088a2b..f5de8ce8fb45 100644
--- a/drivers/net/wireless/ath/ath10k/core.h
+++ b/drivers/net/wireless/ath/ath10k/core.h
@@ -76,7 +76,7 @@
 /* The magic used by QCA spec */
 #define ATH10K_SMBIOS_BDF_EXT_MAGIC "BDF_"
 
-/* Default Airtime weight multipler (Tuned for multiclient performance) */
+/* Default Airtime weight multiplier (Tuned for multiclient performance) */
 #define ATH10K_AIRTIME_WEIGHT_MULTIPLIER  4
 
 #define ATH10K_MAX_RETRY_COUNT 30
@@ -857,7 +857,7 @@ enum ath10k_dev_flags {
 	/* Disable HW crypto engine */
 	ATH10K_FLAG_HW_CRYPTO_DISABLED,
 
-	/* Bluetooth coexistance enabled */
+	/* Bluetooth coexistence enabled */
 	ATH10K_FLAG_BTCOEX,
 
 	/* Per Station statistics service */
diff --git a/drivers/net/wireless/ath/ath10k/coredump.c b/drivers/net/wireless/ath/ath10k/coredump.c
index fe6b6f97a916..2d1634a890dd 100644
--- a/drivers/net/wireless/ath/ath10k/coredump.c
+++ b/drivers/net/wireless/ath/ath10k/coredump.c
@@ -531,7 +531,7 @@ static const struct ath10k_mem_section qca6174_hw30_sdio_register_sections[] = {
 
 	{0x40000, 0x400A4},
 
-	/* SI register is skiped here.
+	/* SI register is skipped here.
 	 * Because it will cause bus hang
 	 *
 	 * {0x50000, 0x50018},
diff --git a/drivers/net/wireless/ath/ath10k/coredump.h b/drivers/net/wireless/ath/ath10k/coredump.h
index 240d70515088..437b9759f05d 100644
--- a/drivers/net/wireless/ath/ath10k/coredump.h
+++ b/drivers/net/wireless/ath/ath10k/coredump.h
@@ -125,7 +125,7 @@ enum ath10k_mem_region_type {
  * To minimize the size of the array, the list must obey the format:
  * '{start0,stop0},{start1,stop1},{start2,stop2}....' The values below must
  * also obey to 'start0 < stop0 < start1 < stop1 < start2 < ...', otherwise
- * we may encouter error in the dump processing.
+ * we may encounter error in the dump processing.
  */
 struct ath10k_mem_section {
 	u32 start;
diff --git a/drivers/net/wireless/ath/ath10k/debug.c b/drivers/net/wireless/ath/ath10k/debug.c
index 39378e3f9b2b..c861e66ef6bc 100644
--- a/drivers/net/wireless/ath/ath10k/debug.c
+++ b/drivers/net/wireless/ath/ath10k/debug.c
@@ -1081,7 +1081,7 @@ exit:
  * struct available..
  */
 
-/* This generally cooresponds to the debugfs fw_stats file */
+/* This generally corresponds to the debugfs fw_stats file */
 static const char ath10k_gstrings_stats[][ETH_GSTRING_LEN] = {
 	"tx_pkts_nic",
 	"tx_bytes_nic",
diff --git a/drivers/net/wireless/ath/ath10k/debugfs_sta.c b/drivers/net/wireless/ath/ath10k/debugfs_sta.c
index 367539f2c370..87a3365330ff 100644
--- a/drivers/net/wireless/ath/ath10k/debugfs_sta.c
+++ b/drivers/net/wireless/ath/ath10k/debugfs_sta.c
@@ -498,7 +498,7 @@ static char *get_num_ampdu_subfrm_str(enum ath10k_ampdu_subfrm_num i)
 {
 	switch (i) {
 	case ATH10K_AMPDU_SUBFRM_NUM_10:
-		return "upto 10";
+		return "up to 10";
 	case ATH10K_AMPDU_SUBFRM_NUM_20:
 		return "11-20";
 	case ATH10K_AMPDU_SUBFRM_NUM_30:
diff --git a/drivers/net/wireless/ath/ath10k/htc.c b/drivers/net/wireless/ath/ath10k/htc.c
index fab398046a3f..6d1784f74bea 100644
--- a/drivers/net/wireless/ath/ath10k/htc.c
+++ b/drivers/net/wireless/ath/ath10k/htc.c
@@ -947,13 +947,18 @@ int ath10k_htc_wait_target(struct ath10k_htc *htc)
 		return -ECOMM;
 	}
 
-	htc->total_transmit_credits = __le16_to_cpu(msg->ready.credit_count);
+	if (ar->hw_params.use_fw_tx_credits)
+		htc->total_transmit_credits = __le16_to_cpu(msg->ready.credit_count);
+	else
+		htc->total_transmit_credits = 1;
+
 	htc->target_credit_size = __le16_to_cpu(msg->ready.credit_size);
 
 	ath10k_dbg(ar, ATH10K_DBG_HTC,
-		   "Target ready! transmit resources: %d size:%d\n",
+		   "Target ready! transmit resources: %d size:%d actual credits:%d\n",
 		   htc->total_transmit_credits,
-		   htc->target_credit_size);
+		   htc->target_credit_size,
+		   msg->ready.credit_count);
 
 	if ((htc->total_transmit_credits == 0) ||
 	    (htc->target_credit_size == 0)) {
diff --git a/drivers/net/wireless/ath/ath10k/htt_rx.c b/drivers/net/wireless/ath/ath10k/htt_rx.c
index 8a075a711b71..e76aab973320 100644
--- a/drivers/net/wireless/ath/ath10k/htt_rx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_rx.c
@@ -301,12 +301,16 @@ void ath10k_htt_rx_free(struct ath10k_htt *htt)
 			  ath10k_htt_get_vaddr_ring(htt),
 			  htt->rx_ring.base_paddr);
 
+	ath10k_htt_config_paddrs_ring(htt, NULL);
+
 	dma_free_coherent(htt->ar->dev,
 			  sizeof(*htt->rx_ring.alloc_idx.vaddr),
 			  htt->rx_ring.alloc_idx.vaddr,
 			  htt->rx_ring.alloc_idx.paddr);
+	htt->rx_ring.alloc_idx.vaddr = NULL;
 
 	kfree(htt->rx_ring.netbufs_ring);
+	htt->rx_ring.netbufs_ring = NULL;
 }
 
 static inline struct sk_buff *ath10k_htt_rx_netbuf_pop(struct ath10k_htt *htt)
@@ -846,8 +850,10 @@ err_dma_idx:
 			  ath10k_htt_get_rx_ring_size(htt),
 			  vaddr_ring,
 			  htt->rx_ring.base_paddr);
+	ath10k_htt_config_paddrs_ring(htt, NULL);
 err_dma_ring:
 	kfree(htt->rx_ring.netbufs_ring);
+	htt->rx_ring.netbufs_ring = NULL;
 err_netbuf:
 	return -ENOMEM;
 }
@@ -2496,7 +2502,7 @@ static bool ath10k_htt_rx_proc_rx_ind_hl(struct ath10k_htt *htt,
 
 	/* I have not yet seen any case where num_mpdu_ranges > 1.
 	 * qcacld does not seem handle that case either, so we introduce the
-	 * same limitiation here as well.
+	 * same limitation here as well.
 	 */
 	if (num_mpdu_ranges > 1)
 		ath10k_warn(ar,
diff --git a/drivers/net/wireless/ath/ath10k/htt_tx.c b/drivers/net/wireless/ath/ath10k/htt_tx.c
index a19b0795c86d..bd603feb7953 100644
--- a/drivers/net/wireless/ath/ath10k/htt_tx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_tx.c
@@ -1112,7 +1112,7 @@ int ath10k_htt_tx_fetch_resp(struct ath10k *ar,
 	int len = 0;
 	int ret;
 
-	/* Response IDs are echo-ed back only for host driver convienence
+	/* Response IDs are echo-ed back only for host driver convenience
 	 * purposes. They aren't used for anything in the driver yet so use 0.
 	 */
 
diff --git a/drivers/net/wireless/ath/ath10k/hw.c b/drivers/net/wireless/ath/ath10k/hw.c
index e52e41a70321..6d32b43a4da6 100644
--- a/drivers/net/wireless/ath/ath10k/hw.c
+++ b/drivers/net/wireless/ath/ath10k/hw.c
@@ -84,7 +84,7 @@ const struct ath10k_hw_regs qca99x0_regs = {
 	.ce5_base_address			= 0x0004b400,
 	.ce6_base_address			= 0x0004b800,
 	.ce7_base_address			= 0x0004bc00,
-	/* Note: qca99x0 supports upto 12 Copy Engines. Other than address of
+	/* Note: qca99x0 supports up to 12 Copy Engines. Other than address of
 	 * CE0 and CE1 no other copy engine is directly referred in the code.
 	 * It is not really necessary to assign address for newly supported
 	 * CEs in this address table.
@@ -120,7 +120,7 @@ const struct ath10k_hw_regs qca4019_regs = {
 	.ce5_base_address                       = 0x0004b400,
 	.ce6_base_address                       = 0x0004b800,
 	.ce7_base_address                       = 0x0004bc00,
-	/* qca4019 supports upto 12 copy engines. Since base address
+	/* qca4019 supports up to 12 copy engines. Since base address
 	 * of ce8 to ce11 are not directly referred in the code,
 	 * no need have them in separate members in this table.
 	 *      Copy Engine             Address
@@ -924,7 +924,7 @@ static void ath10k_hw_map_target_mem(struct ath10k *ar, u32 msb)
 	ath10k_hif_write32(ar, address, msb);
 }
 
-/* 1. Write to memory region of target, such as IRAM adn DRAM.
+/* 1. Write to memory region of target, such as IRAM and DRAM.
  * 2. Target address( 0 ~ 00100000 & 0x00400000~0x00500000)
  *    can be written directly. See ath10k_pci_targ_cpu_to_ce_addr() too.
  * 3. In order to access the region other than the above,
diff --git a/drivers/net/wireless/ath/ath10k/hw.h b/drivers/net/wireless/ath/ath10k/hw.h
index 93acf0dd580a..1b99f3a39a11 100644
--- a/drivers/net/wireless/ath/ath10k/hw.h
+++ b/drivers/net/wireless/ath/ath10k/hw.h
@@ -635,6 +635,8 @@ struct ath10k_hw_params {
 	bool dynamic_sar_support;
 
 	bool hw_restart_disconnect;
+
+	bool use_fw_tx_credits;
 };
 
 struct htt_resp;
diff --git a/drivers/net/wireless/ath/ath10k/mac.c b/drivers/net/wireless/ath/ath10k/mac.c
index 9dd3b8fba4b0..ec8d5b29bc72 100644
--- a/drivers/net/wireless/ath/ath10k/mac.c
+++ b/drivers/net/wireless/ath/ath10k/mac.c
@@ -864,11 +864,36 @@ static int ath10k_peer_delete(struct ath10k *ar, u32 vdev_id, const u8 *addr)
 	return 0;
 }
 
+static void ath10k_peer_map_cleanup(struct ath10k *ar, struct ath10k_peer *peer)
+{
+	int peer_id, i;
+
+	lockdep_assert_held(&ar->conf_mutex);
+
+	for_each_set_bit(peer_id, peer->peer_ids,
+			 ATH10K_MAX_NUM_PEER_IDS) {
+		ar->peer_map[peer_id] = NULL;
+	}
+
+	/* Double check that peer is properly un-referenced from
+	 * the peer_map
+	 */
+	for (i = 0; i < ARRAY_SIZE(ar->peer_map); i++) {
+		if (ar->peer_map[i] == peer) {
+			ath10k_warn(ar, "removing stale peer_map entry for %pM (ptr %pK idx %d)\n",
+				    peer->addr, peer, i);
+			ar->peer_map[i] = NULL;
+		}
+	}
+
+	list_del(&peer->list);
+	kfree(peer);
+	ar->num_peers--;
+}
+
 static void ath10k_peer_cleanup(struct ath10k *ar, u32 vdev_id)
 {
 	struct ath10k_peer *peer, *tmp;
-	int peer_id;
-	int i;
 
 	lockdep_assert_held(&ar->conf_mutex);
 
@@ -880,25 +905,7 @@ static void ath10k_peer_cleanup(struct ath10k *ar, u32 vdev_id)
 		ath10k_warn(ar, "removing stale peer %pM from vdev_id %d\n",
 			    peer->addr, vdev_id);
 
-		for_each_set_bit(peer_id, peer->peer_ids,
-				 ATH10K_MAX_NUM_PEER_IDS) {
-			ar->peer_map[peer_id] = NULL;
-		}
-
-		/* Double check that peer is properly un-referenced from
-		 * the peer_map
-		 */
-		for (i = 0; i < ARRAY_SIZE(ar->peer_map); i++) {
-			if (ar->peer_map[i] == peer) {
-				ath10k_warn(ar, "removing stale peer_map entry for %pM (ptr %pK idx %d)\n",
-					    peer->addr, peer, i);
-				ar->peer_map[i] = NULL;
-			}
-		}
-
-		list_del(&peer->list);
-		kfree(peer);
-		ar->num_peers--;
+		ath10k_peer_map_cleanup(ar, peer);
 	}
 	spin_unlock_bh(&ar->data_lock);
 }
@@ -4044,7 +4051,7 @@ static int ath10k_mac_tx(struct ath10k *ar,
 		ath10k_tx_h_seq_no(vif, skb);
 		break;
 	case ATH10K_HW_TXRX_ETHERNET:
-		/* Convert 802.11->802.3 header only if the frame was erlier
+		/* Convert 802.11->802.3 header only if the frame was earlier
 		 * encapsulated to 802.11 by mac80211. Otherwise pass it as is.
 		 */
 		if (!(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP))
@@ -7621,10 +7628,7 @@ static int ath10k_sta_state(struct ieee80211_hw *hw,
 				/* Clean up the peer object as well since we
 				 * must have failed to do this above.
 				 */
-				list_del(&peer->list);
-				ar->peer_map[i] = NULL;
-				kfree(peer);
-				ar->num_peers--;
+				ath10k_peer_map_cleanup(ar, peer);
 			}
 		}
 		spin_unlock_bh(&ar->data_lock);
@@ -8093,7 +8097,7 @@ static void ath10k_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 
 /* TODO: Implement this function properly
  * For now it is needed to reply to Probe Requests in IBSS mode.
- * Propably we need this information from FW.
+ * Probably we need this information from FW.
  */
 static int ath10k_tx_last_beacon(struct ieee80211_hw *hw)
 {
@@ -8516,7 +8520,7 @@ static void ath10k_sta_rc_update(struct ieee80211_hw *hw,
 		   "mac sta rc update for %pM changed %08x bw %d nss %d smps %d\n",
 		   sta->addr, changed, sta->deflink.bandwidth,
 		   sta->deflink.rx_nss,
-		   sta->smps_mode);
+		   sta->deflink.smps_mode);
 
 	if (changed & IEEE80211_RC_BW_CHANGED) {
 		bw = WMI_PEER_CHWIDTH_20MHZ;
@@ -8550,7 +8554,7 @@ static void ath10k_sta_rc_update(struct ieee80211_hw *hw,
 	if (changed & IEEE80211_RC_SMPS_CHANGED) {
 		smps = WMI_PEER_SMPS_PS_NONE;
 
-		switch (sta->smps_mode) {
+		switch (sta->deflink.smps_mode) {
 		case IEEE80211_SMPS_AUTOMATIC:
 		case IEEE80211_SMPS_OFF:
 			smps = WMI_PEER_SMPS_PS_NONE;
@@ -8563,7 +8567,7 @@ static void ath10k_sta_rc_update(struct ieee80211_hw *hw,
 			break;
 		case IEEE80211_SMPS_NUM_MODES:
 			ath10k_warn(ar, "Invalid smps %d in sta rc update for %pM\n",
-				    sta->smps_mode, sta->addr);
+				    sta->deflink.smps_mode, sta->addr);
 			smps = WMI_PEER_SMPS_PS_NONE;
 			break;
 		}
@@ -9682,7 +9686,7 @@ static const struct ieee80211_iface_limit ath10k_tlv_if_limit_ibss[] = {
 	},
 };
 
-/* FIXME: This is not thouroughly tested. These combinations may over- or
+/* FIXME: This is not thoroughly tested. These combinations may over- or
  * underestimate hw/fw capabilities.
  */
 static struct ieee80211_iface_combination ath10k_tlv_if_comb[] = {
@@ -9922,7 +9926,7 @@ int ath10k_mac_register(struct ath10k *ar)
 		WLAN_CIPHER_SUITE_BIP_GMAC_128,
 		WLAN_CIPHER_SUITE_BIP_GMAC_256,
 
-		/* Only QCA99x0 and QCA4019 varients support GCMP-128, GCMP-256
+		/* Only QCA99x0 and QCA4019 variants support GCMP-128, GCMP-256
 		 * and CCMP-256 in hardware.
 		 */
 		WLAN_CIPHER_SUITE_GCMP,
diff --git a/drivers/net/wireless/ath/ath10k/pci.c b/drivers/net/wireless/ath/ath10k/pci.c
index bf1c938be7d0..e56c6a6b1379 100644
--- a/drivers/net/wireless/ath/ath10k/pci.c
+++ b/drivers/net/wireless/ath/ath10k/pci.c
@@ -1244,7 +1244,7 @@ static void ath10k_pci_process_htt_rx_cb(struct ath10k_ce_pipe *ce_state,
 	unsigned int nbytes, max_nbytes, nentries;
 	int orig_len;
 
-	/* No need to aquire ce_lock for CE5, since this is the only place CE5
+	/* No need to acquire ce_lock for CE5, since this is the only place CE5
 	 * is processed other than init and deinit. Before releasing CE5
 	 * buffers, interrupts are disabled. Thus CE5 access is serialized.
 	 */
@@ -3215,8 +3215,7 @@ static void ath10k_pci_free_irq(struct ath10k *ar)
 
 void ath10k_pci_init_napi(struct ath10k *ar)
 {
-	netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_pci_napi_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_pci_napi_poll);
 }
 
 static int ath10k_pci_init_irq(struct ath10k *ar)
diff --git a/drivers/net/wireless/ath/ath10k/pci.h b/drivers/net/wireless/ath/ath10k/pci.h
index cf64898b9447..480cd97ab739 100644
--- a/drivers/net/wireless/ath/ath10k/pci.h
+++ b/drivers/net/wireless/ath/ath10k/pci.h
@@ -81,7 +81,7 @@ struct ath10k_pci_pipe {
 	/* Handle of underlying Copy Engine */
 	struct ath10k_ce_pipe *ce_hdl;
 
-	/* Our pipe number; facilitiates use of pipe_info ptrs. */
+	/* Our pipe number; facilitates use of pipe_info ptrs. */
 	u8 pipe_num;
 
 	/* Convenience back pointer to hif_ce_state. */
diff --git a/drivers/net/wireless/ath/ath10k/qmi.c b/drivers/net/wireless/ath/ath10k/qmi.c
index d7e406916bc8..66cb7a1e628a 100644
--- a/drivers/net/wireless/ath/ath10k/qmi.c
+++ b/drivers/net/wireless/ath/ath10k/qmi.c
@@ -792,7 +792,7 @@ static void ath10k_qmi_event_server_arrive(struct ath10k_qmi *qmi)
 		return;
 
 	/*
-	 * HACK: sleep for a while inbetween receiving the msa info response
+	 * HACK: sleep for a while between receiving the msa info response
 	 * and the XPU update to prevent SDM845 from crashing due to a security
 	 * violation, when running MPSS.AT.4.0.c2-01184-SDM845_GEN_PACK-1.
 	 */
diff --git a/drivers/net/wireless/ath/ath10k/rx_desc.h b/drivers/net/wireless/ath/ath10k/rx_desc.h
index 6ce2a8b1060d..777e53aa69dc 100644
--- a/drivers/net/wireless/ath/ath10k/rx_desc.h
+++ b/drivers/net/wireless/ath/ath10k/rx_desc.h
@@ -448,7 +448,7 @@ struct rx_mpdu_end {
  *     - 4 bytes for WEP
  *     - 8 bytes for TKIP, AES
  *  [padding to 4 bytes]
- *  c) A-MSDU subframe header (14 bytes) if appliable
+ *  c) A-MSDU subframe header (14 bytes) if applicable
  *  d) LLC/SNAP (RFC1042, 8 bytes)
  *
  * In case of A-MSDU only first frame in sequence contains (a) and (b).
diff --git a/drivers/net/wireless/ath/ath10k/sdio.c b/drivers/net/wireless/ath/ath10k/sdio.c
index 24283c02a5ef..79e09c7a82b3 100644
--- a/drivers/net/wireless/ath/ath10k/sdio.c
+++ b/drivers/net/wireless/ath/ath10k/sdio.c
@@ -1057,7 +1057,7 @@ static int ath10k_sdio_mbox_proc_pending_irqs(struct ath10k *ar,
 
 out:
 	/* An optimization to bypass reading the IRQ status registers
-	 * unecessarily which can re-wake the target, if upper layers
+	 * unnecessarily which can re-wake the target, if upper layers
 	 * determine that we are in a low-throughput mode, we can rely on
 	 * taking another interrupt rather than re-checking the status
 	 * registers which can re-wake the target.
@@ -2531,8 +2531,7 @@ static int ath10k_sdio_probe(struct sdio_func *func,
 		return -ENOMEM;
 	}
 
-	netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_sdio_napi_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_sdio_napi_poll);
 
 	ath10k_dbg(ar, ATH10K_DBG_BOOT,
 		   "sdio new func %d vendor 0x%x device 0x%x block 0x%x/0x%x\n",
diff --git a/drivers/net/wireless/ath/ath10k/snoc.c b/drivers/net/wireless/ath/ath10k/snoc.c
index 5576ad9fd116..cfcb759a87de 100644
--- a/drivers/net/wireless/ath/ath10k/snoc.c
+++ b/drivers/net/wireless/ath/ath10k/snoc.c
@@ -1242,8 +1242,7 @@ static int ath10k_snoc_napi_poll(struct napi_struct *ctx, int budget)
 
 static void ath10k_snoc_init_napi(struct ath10k *ar)
 {
-	netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_snoc_napi_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_snoc_napi_poll);
 }
 
 static int ath10k_snoc_request_irq(struct ath10k *ar)
diff --git a/drivers/net/wireless/ath/ath10k/thermal.c b/drivers/net/wireless/ath/ath10k/thermal.c
index 36c9a1364253..cefd97323dfe 100644
--- a/drivers/net/wireless/ath/ath10k/thermal.c
+++ b/drivers/net/wireless/ath/ath10k/thermal.c
@@ -98,7 +98,7 @@ static ssize_t ath10k_thermal_show_temp(struct device *dev,
 	temperature = ar->thermal.temperature;
 	spin_unlock_bh(&ar->data_lock);
 
-	/* display in millidegree celcius */
+	/* display in millidegree celsius */
 	ret = snprintf(buf, PAGE_SIZE, "%d\n", temperature * 1000);
 out:
 	mutex_unlock(&ar->conf_mutex);
diff --git a/drivers/net/wireless/ath/ath10k/thermal.h b/drivers/net/wireless/ath/ath10k/thermal.h
index 5fdb020f4da3..1f4de9fbf2b3 100644
--- a/drivers/net/wireless/ath/ath10k/thermal.h
+++ b/drivers/net/wireless/ath/ath10k/thermal.h
@@ -19,7 +19,7 @@ struct ath10k_thermal {
 	/* protected by conf_mutex */
 	u32 throttle_state;
 	u32 quiet_period;
-	/* temperature value in Celcius degree
+	/* temperature value in Celsius degree
 	 * protected by data_lock
 	 */
 	int temperature;
diff --git a/drivers/net/wireless/ath/ath10k/usb.c b/drivers/net/wireless/ath/ath10k/usb.c
index ad6471b21796..b0067af685b1 100644
--- a/drivers/net/wireless/ath/ath10k/usb.c
+++ b/drivers/net/wireless/ath/ath10k/usb.c
@@ -1014,8 +1014,7 @@ static int ath10k_usb_probe(struct usb_interface *interface,
 		return -ENOMEM;
 	}
 
-	netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_usb_napi_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(&ar->napi_dev, &ar->napi, ath10k_usb_napi_poll);
 
 	usb_get_dev(dev);
 	vendor_id = le16_to_cpu(dev->descriptor.idVendor);
diff --git a/drivers/net/wireless/ath/ath10k/usb.h b/drivers/net/wireless/ath/ath10k/usb.h
index 34d683e8fc18..48e066ba8162 100644
--- a/drivers/net/wireless/ath/ath10k/usb.h
+++ b/drivers/net/wireless/ath/ath10k/usb.h
@@ -26,7 +26,7 @@
 #define ATH10K_USB_EP_ADDR_APP_DATA_MP_OUT      0x03
 #define ATH10K_USB_EP_ADDR_APP_DATA_HP_OUT      0x04
 
-/* diagnostic command defnitions */
+/* diagnostic command definitions */
 #define ATH10K_USB_CONTROL_REQ_SEND_BMI_CMD        1
 #define ATH10K_USB_CONTROL_REQ_RECV_BMI_RESP       2
 #define ATH10K_USB_CONTROL_REQ_DIAG_CMD            3
diff --git a/drivers/net/wireless/ath/ath10k/wmi-tlv.h b/drivers/net/wireless/ath/ath10k/wmi-tlv.h
index b39c9b78b32b..dbb48d70f2e9 100644
--- a/drivers/net/wireless/ath/ath10k/wmi-tlv.h
+++ b/drivers/net/wireless/ath/ath10k/wmi-tlv.h
@@ -1813,7 +1813,7 @@ struct wmi_tlv_pdev_get_temp_cmd {
 
 struct wmi_tlv_pdev_temperature_event {
 	__le32 tlv_hdr;
-	/* temperature value in Celcius degree */
+	/* temperature value in Celsius degree */
 	__le32 temperature;
 	__le32 pdev_id;
 } __packed;
@@ -2548,7 +2548,7 @@ struct nlo_channel_prediction_cfg {
 
 	/* Preconfigured stationary threshold.
 	 * Lesser value means more conservative. Bigger value means more aggressive.
-	 * Maximum is 100 and mininum is 0.
+	 * Maximum is 100 and minimum is 0.
 	 */
 	__le32 stationary_threshold;
 
diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c
index 074d8ba5072a..980d4124fa28 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.c
+++ b/drivers/net/wireless/ath/ath10k/wmi.c
@@ -3555,7 +3555,7 @@ static void ath10k_wmi_update_tim(struct ath10k *ar,
 	__le32 t;
 	u32 v, tim_len;
 
-	/* When FW reports 0 in tim_len, ensure atleast first byte
+	/* When FW reports 0 in tim_len, ensure at least first byte
 	 * in tim_bitmap is considered for pvm calculation.
 	 */
 	tim_len = tim_info->tim_len ? __le32_to_cpu(tim_info->tim_len) : 1;
diff --git a/drivers/net/wireless/ath/ath10k/wmi.h b/drivers/net/wireless/ath/ath10k/wmi.h
index 4abd12e78028..6de3cc4640a0 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.h
+++ b/drivers/net/wireless/ath/ath10k/wmi.h
@@ -3170,7 +3170,7 @@ struct wmi_start_scan_common {
 	/* dwell time in msec on passive channels */
 	__le32 dwell_time_passive;
 	/*
-	 * min time in msec on the BSS channel,only valid if atleast one
+	 * min time in msec on the BSS channel,only valid if at least one
 	 * VDEV is active
 	 */
 	__le32 min_rest_time;
@@ -3196,7 +3196,7 @@ struct wmi_start_scan_common {
 	 * and bssid_list
 	 */
 	__le32 repeat_probe_time;
-	/* time in msec between 2 consequetive probe requests with in a set. */
+	/* time in msec between 2 consecutive probe requests with in a set. */
 	__le32 probe_spacing_time;
 	/*
 	 * data inactivity time in msec on bss channel that will be used by
@@ -4397,7 +4397,7 @@ struct wmi_pdev_stats_tx {
 	/* wal pdev continuous xretry */
 	__le32 pdev_cont_xretry;
 
-	/* wal pdev continous xretry */
+	/* wal pdev continuous xretry */
 	__le32 pdev_tx_timeout;
 
 	/* wal pdev resets  */
@@ -5240,7 +5240,7 @@ enum wmi_vdev_param {
 	 * scheduler.
 	 */
 	WMI_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT,
-	/* enable/dsiable WDS for this VDEV  */
+	/* enable/disable WDS for this VDEV  */
 	WMI_VDEV_PARAM_WDS,
 	/* ATIM Window */
 	WMI_VDEV_PARAM_ATIM_WINDOW,
@@ -5372,7 +5372,7 @@ enum wmi_10x_vdev_param {
 	 * scheduler.
 	 */
 	WMI_10X_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT,
-	/* enable/dsiable WDS for this VDEV  */
+	/* enable/disable WDS for this VDEV  */
 	WMI_10X_VDEV_PARAM_WDS,
 	/* ATIM Window */
 	WMI_10X_VDEV_PARAM_ATIM_WINDOW,
@@ -5904,7 +5904,7 @@ enum wmi_sta_ps_param_tx_wake_threshold {
 enum wmi_sta_ps_param_pspoll_count {
 	WMI_STA_PS_PSPOLL_COUNT_NO_MAX = 0,
 	/*
-	 * Values greater than 0 indicate the maximum numer of PS-Poll frames
+	 * Values greater than 0 indicate the maximum number of PS-Poll frames
 	 * FW will send before waking up.
 	 */
 
@@ -6947,7 +6947,7 @@ struct wmi_echo_ev_arg {
 };
 
 struct wmi_pdev_temperature_event {
-	/* temperature value in Celcius degree */
+	/* temperature value in Celsius degree */
 	__le32 temperature;
 } __packed;
 
diff --git a/drivers/net/wireless/ath/ath11k/ahb.c b/drivers/net/wireless/ath/ath11k/ahb.c
index c47414710138..d34a4d6325b2 100644
--- a/drivers/net/wireless/ath/ath11k/ahb.c
+++ b/drivers/net/wireless/ath/ath11k/ahb.c
@@ -16,6 +16,8 @@
 #include "hif.h"
 #include <linux/remoteproc.h>
 #include "pcic.h"
+#include <linux/soc/qcom/smem.h>
+#include <linux/soc/qcom/smem_state.h>
 
 static const struct of_device_id ath11k_ahb_of_match[] = {
 	/* TODO: Should we change the compatible string to something similar
@@ -359,6 +361,7 @@ static void ath11k_ahb_ext_irq_enable(struct ath11k_base *ab)
 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
 
 		if (!irq_grp->napi_enabled) {
+			dev_set_threaded(&irq_grp->napi_ndev, true);
 			napi_enable(&irq_grp->napi);
 			irq_grp->napi_enabled = true;
 		}
@@ -406,7 +409,8 @@ static int ath11k_ahb_fwreset_from_cold_boot(struct ath11k_base *ab)
 	int timeout;
 
 	if (ath11k_cold_boot_cal == 0 || ab->qmi.cal_done ||
-	    ab->hw_params.cold_boot_calib == 0)
+	    ab->hw_params.cold_boot_calib == 0 ||
+	    ab->hw_params.cbcal_restart_fw == 0)
 		return 0;
 
 	ath11k_dbg(ab, ATH11K_DBG_AHB, "wait for cold boot done\n");
@@ -541,7 +545,7 @@ static int ath11k_ahb_config_ext_irq(struct ath11k_base *ab)
 		irq_grp->grp_id = i;
 		init_dummy_netdev(&irq_grp->napi_ndev);
 		netif_napi_add(&irq_grp->napi_ndev, &irq_grp->napi,
-			       ath11k_ahb_ext_grp_napi_poll, NAPI_POLL_WEIGHT);
+			       ath11k_ahb_ext_grp_napi_poll);
 
 		for (j = 0; j < ATH11K_EXT_IRQ_NUM_MAX; j++) {
 			if (ab->hw_params.ring_mask->tx[i] & BIT(j)) {
@@ -685,11 +689,90 @@ static int ath11k_ahb_map_service_to_pipe(struct ath11k_base *ab, u16 service_id
 	return 0;
 }
 
+static int ath11k_ahb_hif_suspend(struct ath11k_base *ab)
+{
+	struct ath11k_ahb *ab_ahb = ath11k_ahb_priv(ab);
+	u32 wake_irq;
+	u32 value = 0;
+	int ret;
+
+	if (!device_may_wakeup(ab->dev))
+		return -EPERM;
+
+	wake_irq = ab->irq_num[ATH11K_PCI_IRQ_CE0_OFFSET + ATH11K_PCI_CE_WAKE_IRQ];
+
+	ret = enable_irq_wake(wake_irq);
+	if (ret) {
+		ath11k_err(ab, "failed to enable wakeup irq :%d\n", ret);
+		return ret;
+	}
+
+	value = u32_encode_bits(ab_ahb->smp2p_info.seq_no++,
+				ATH11K_AHB_SMP2P_SMEM_SEQ_NO);
+	value |= u32_encode_bits(ATH11K_AHB_POWER_SAVE_ENTER,
+				 ATH11K_AHB_SMP2P_SMEM_MSG);
+
+	ret = qcom_smem_state_update_bits(ab_ahb->smp2p_info.smem_state,
+					  ATH11K_AHB_SMP2P_SMEM_VALUE_MASK, value);
+	if (ret) {
+		ath11k_err(ab, "failed to send smp2p power save enter cmd :%d\n", ret);
+		return ret;
+	}
+
+	ath11k_dbg(ab, ATH11K_DBG_AHB, "ahb device suspended\n");
+
+	return ret;
+}
+
+static int ath11k_ahb_hif_resume(struct ath11k_base *ab)
+{
+	struct ath11k_ahb *ab_ahb = ath11k_ahb_priv(ab);
+	u32 wake_irq;
+	u32 value = 0;
+	int ret;
+
+	if (!device_may_wakeup(ab->dev))
+		return -EPERM;
+
+	wake_irq = ab->irq_num[ATH11K_PCI_IRQ_CE0_OFFSET + ATH11K_PCI_CE_WAKE_IRQ];
+
+	ret = disable_irq_wake(wake_irq);
+	if (ret) {
+		ath11k_err(ab, "failed to disable wakeup irq: %d\n", ret);
+		return ret;
+	}
+
+	reinit_completion(&ab->wow.wakeup_completed);
+
+	value = u32_encode_bits(ab_ahb->smp2p_info.seq_no++,
+				ATH11K_AHB_SMP2P_SMEM_SEQ_NO);
+	value |= u32_encode_bits(ATH11K_AHB_POWER_SAVE_EXIT,
+				 ATH11K_AHB_SMP2P_SMEM_MSG);
+
+	ret = qcom_smem_state_update_bits(ab_ahb->smp2p_info.smem_state,
+					  ATH11K_AHB_SMP2P_SMEM_VALUE_MASK, value);
+	if (ret) {
+		ath11k_err(ab, "failed to send smp2p power save enter cmd :%d\n", ret);
+		return ret;
+	}
+
+	ret = wait_for_completion_timeout(&ab->wow.wakeup_completed, 3 * HZ);
+	if (ret == 0) {
+		ath11k_warn(ab, "timed out while waiting for wow wakeup completion\n");
+		return -ETIMEDOUT;
+	}
+
+	ath11k_dbg(ab, ATH11K_DBG_AHB, "ahb device resumed\n");
+
+	return 0;
+}
+
 static const struct ath11k_hif_ops ath11k_ahb_hif_ops_ipq8074 = {
 	.start = ath11k_ahb_start,
 	.stop = ath11k_ahb_stop,
 	.read32 = ath11k_ahb_read32,
 	.write32 = ath11k_ahb_write32,
+	.read = NULL,
 	.irq_enable = ath11k_ahb_ext_irq_enable,
 	.irq_disable = ath11k_ahb_ext_irq_disable,
 	.map_service_to_pipe = ath11k_ahb_map_service_to_pipe,
@@ -702,6 +785,7 @@ static const struct ath11k_hif_ops ath11k_ahb_hif_ops_wcn6750 = {
 	.stop = ath11k_pcic_stop,
 	.read32 = ath11k_pcic_read32,
 	.write32 = ath11k_pcic_write32,
+	.read = NULL,
 	.irq_enable = ath11k_pcic_ext_irq_enable,
 	.irq_disable = ath11k_pcic_ext_irq_disable,
 	.get_msi_address =  ath11k_pcic_get_msi_address,
@@ -709,6 +793,10 @@ static const struct ath11k_hif_ops ath11k_ahb_hif_ops_wcn6750 = {
 	.map_service_to_pipe = ath11k_pcic_map_service_to_pipe,
 	.power_down = ath11k_ahb_power_down,
 	.power_up = ath11k_ahb_power_up,
+	.suspend = ath11k_ahb_hif_suspend,
+	.resume = ath11k_ahb_hif_resume,
+	.ce_irq_enable = ath11k_pci_enable_ce_irqs_except_wake_irq,
+	.ce_irq_disable = ath11k_pci_disable_ce_irqs_except_wake_irq,
 };
 
 static int ath11k_core_get_rproc(struct ath11k_base *ab)
@@ -783,6 +871,34 @@ static int ath11k_ahb_setup_msi_resources(struct ath11k_base *ab)
 	return 0;
 }
 
+static int ath11k_ahb_setup_smp2p_handle(struct ath11k_base *ab)
+{
+	struct ath11k_ahb *ab_ahb = ath11k_ahb_priv(ab);
+
+	if (!ab->hw_params.smp2p_wow_exit)
+		return 0;
+
+	ab_ahb->smp2p_info.smem_state = qcom_smem_state_get(ab->dev, "wlan-smp2p-out",
+							    &ab_ahb->smp2p_info.smem_bit);
+	if (IS_ERR(ab_ahb->smp2p_info.smem_state)) {
+		ath11k_err(ab, "failed to fetch smem state: %ld\n",
+			   PTR_ERR(ab_ahb->smp2p_info.smem_state));
+		return PTR_ERR(ab_ahb->smp2p_info.smem_state);
+	}
+
+	return 0;
+}
+
+static void ath11k_ahb_release_smp2p_handle(struct ath11k_base *ab)
+{
+	struct ath11k_ahb *ab_ahb = ath11k_ahb_priv(ab);
+
+	if (!ab->hw_params.smp2p_wow_exit)
+		return;
+
+	qcom_smem_state_put(ab_ahb->smp2p_info.smem_state);
+}
+
 static int ath11k_ahb_setup_resources(struct ath11k_base *ab)
 {
 	struct platform_device *pdev = ab->pdev;
@@ -1038,10 +1154,14 @@ static int ath11k_ahb_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_core_free;
 
-	ret = ath11k_hal_srng_init(ab);
+	ret = ath11k_ahb_setup_smp2p_handle(ab);
 	if (ret)
 		goto err_fw_deinit;
 
+	ret = ath11k_hal_srng_init(ab);
+	if (ret)
+		goto err_release_smp2p_handle;
+
 	ret = ath11k_ce_alloc_pipes(ab);
 	if (ret) {
 		ath11k_err(ab, "failed to allocate ce pipes: %d\n", ret);
@@ -1078,6 +1198,9 @@ err_ce_free:
 err_hal_srng_deinit:
 	ath11k_hal_srng_deinit(ab);
 
+err_release_smp2p_handle:
+	ath11k_ahb_release_smp2p_handle(ab);
+
 err_fw_deinit:
 	ath11k_ahb_fw_resource_deinit(ab);
 
@@ -1088,20 +1211,10 @@ err_core_free:
 	return ret;
 }
 
-static int ath11k_ahb_remove(struct platform_device *pdev)
+static void ath11k_ahb_remove_prepare(struct ath11k_base *ab)
 {
-	struct ath11k_base *ab = platform_get_drvdata(pdev);
 	unsigned long left;
 
-	if (test_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags)) {
-		ath11k_ahb_power_down(ab);
-		ath11k_debugfs_soc_destroy(ab);
-		ath11k_qmi_deinit_service(ab);
-		goto qmi_fail;
-	}
-
-	reinit_completion(&ab->driver_recovery);
-
 	if (test_bit(ATH11K_FLAG_RECOVERY, &ab->dev_flags)) {
 		left = wait_for_completion_timeout(&ab->driver_recovery,
 						   ATH11K_AHB_RECOVERY_TIMEOUT);
@@ -1111,19 +1224,61 @@ static int ath11k_ahb_remove(struct platform_device *pdev)
 
 	set_bit(ATH11K_FLAG_UNREGISTERING, &ab->dev_flags);
 	cancel_work_sync(&ab->restart_work);
+	cancel_work_sync(&ab->qmi.event_work);
+}
+
+static void ath11k_ahb_free_resources(struct ath11k_base *ab)
+{
+	struct platform_device *pdev = ab->pdev;
 
-	ath11k_core_deinit(ab);
-qmi_fail:
 	ath11k_ahb_free_irq(ab);
 	ath11k_hal_srng_deinit(ab);
+	ath11k_ahb_release_smp2p_handle(ab);
 	ath11k_ahb_fw_resource_deinit(ab);
 	ath11k_ce_free_pipes(ab);
 	ath11k_core_free(ab);
 	platform_set_drvdata(pdev, NULL);
+}
+
+static int ath11k_ahb_remove(struct platform_device *pdev)
+{
+	struct ath11k_base *ab = platform_get_drvdata(pdev);
+
+	if (test_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags)) {
+		ath11k_ahb_power_down(ab);
+		ath11k_debugfs_soc_destroy(ab);
+		ath11k_qmi_deinit_service(ab);
+		goto qmi_fail;
+	}
+
+	ath11k_ahb_remove_prepare(ab);
+	ath11k_core_deinit(ab);
+
+qmi_fail:
+	ath11k_ahb_free_resources(ab);
 
 	return 0;
 }
 
+static void ath11k_ahb_shutdown(struct platform_device *pdev)
+{
+	struct ath11k_base *ab = platform_get_drvdata(pdev);
+
+	/* platform shutdown() & remove() are mutually exclusive.
+	 * remove() is invoked during rmmod & shutdown() during
+	 * system reboot/shutdown.
+	 */
+	ath11k_ahb_remove_prepare(ab);
+
+	if (!(test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags)))
+		goto free_resources;
+
+	ath11k_core_deinit(ab);
+
+free_resources:
+	ath11k_ahb_free_resources(ab);
+}
+
 static struct platform_driver ath11k_ahb_driver = {
 	.driver         = {
 		.name   = "ath11k",
@@ -1131,6 +1286,7 @@ static struct platform_driver ath11k_ahb_driver = {
 	},
 	.probe  = ath11k_ahb_probe,
 	.remove = ath11k_ahb_remove,
+	.shutdown = ath11k_ahb_shutdown,
 };
 
 static int ath11k_ahb_init(void)
diff --git a/drivers/net/wireless/ath/ath11k/ahb.h b/drivers/net/wireless/ath/ath11k/ahb.h
index 58a945411c5b..415ddfd26654 100644
--- a/drivers/net/wireless/ath/ath11k/ahb.h
+++ b/drivers/net/wireless/ath/ath11k/ahb.h
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
  */
 #ifndef ATH11K_AHB_H
 #define ATH11K_AHB_H
@@ -8,6 +9,16 @@
 #include "core.h"
 
 #define ATH11K_AHB_RECOVERY_TIMEOUT (3 * HZ)
+
+#define ATH11K_AHB_SMP2P_SMEM_MSG		GENMASK(15, 0)
+#define ATH11K_AHB_SMP2P_SMEM_SEQ_NO		GENMASK(31, 16)
+#define ATH11K_AHB_SMP2P_SMEM_VALUE_MASK	0xFFFFFFFF
+
+enum ath11k_ahb_smp2p_msg_id {
+	ATH11K_AHB_POWER_SAVE_ENTER = 1,
+	ATH11K_AHB_POWER_SAVE_EXIT,
+};
+
 struct ath11k_base;
 
 struct ath11k_ahb {
@@ -21,6 +32,11 @@ struct ath11k_ahb {
 		u32 ce_size;
 		bool use_tz;
 	} fw;
+	struct {
+		unsigned short seq_no;
+		unsigned int smem_bit;
+		struct qcom_smem_state *smem_state;
+	} smp2p_info;
 };
 
 static inline struct ath11k_ahb *ath11k_ahb_priv(struct ath11k_base *ab)
diff --git a/drivers/net/wireless/ath/ath11k/ce.c b/drivers/net/wireless/ath/ath11k/ce.c
index c14c51f38709..f2da95fd4253 100644
--- a/drivers/net/wireless/ath/ath11k/ce.c
+++ b/drivers/net/wireless/ath/ath11k/ce.c
@@ -250,7 +250,7 @@ const struct ce_attr ath11k_host_ce_config_qcn9074[] = {
 
 static bool ath11k_ce_need_shadow_fix(int ce_id)
 {
-	/* only ce4 needs shadow workaroud*/
+	/* only ce4 needs shadow workaround */
 	if (ce_id == 4)
 		return true;
 	return false;
@@ -1042,7 +1042,7 @@ int ath11k_ce_alloc_pipes(struct ath11k_base *ab)
 
 		ret = ath11k_ce_alloc_pipe(ab, i);
 		if (ret) {
-			/* Free any parial successful allocation */
+			/* Free any partial successful allocation */
 			ath11k_ce_free_pipes(ab);
 			return ret;
 		}
diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c
index c3e9e4f7bc24..b99180bc8172 100644
--- a/drivers/net/wireless/ath/ath11k/core.c
+++ b/drivers/net/wireless/ath/ath11k/core.c
@@ -70,6 +70,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 			.summary_pad_sz = 0,
 			.fft_hdr_len = 16,
 			.max_fft_bins = 512,
+			.fragment_160mhz = true,
 		},
 
 		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
@@ -81,6 +82,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.idle_ps = false,
 		.supports_sta_ps = false,
 		.cold_boot_calib = true,
+		.cbcal_restart_fw = true,
 		.fw_mem_mode = 0,
 		.num_vdevs = 16 + 1,
 		.num_peers = 512,
@@ -106,6 +108,13 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.hybrid_bus_type = false,
 		.fixed_fw_mem = false,
 		.support_off_channel_tx = false,
+		.supports_multi_bssid = false,
+
+		.sram_dump = {},
+
+		.tcl_ring_retry = true,
+		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
+		.smp2p_wow_exit = false,
 	},
 	{
 		.hw_rev = ATH11K_HW_IPQ6018_HW10,
@@ -141,6 +150,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 			.summary_pad_sz = 0,
 			.fft_hdr_len = 16,
 			.max_fft_bins = 512,
+			.fragment_160mhz = true,
 		},
 
 		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
@@ -152,6 +162,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.idle_ps = false,
 		.supports_sta_ps = false,
 		.cold_boot_calib = true,
+		.cbcal_restart_fw = true,
 		.fw_mem_mode = 0,
 		.num_vdevs = 16 + 1,
 		.num_peers = 512,
@@ -177,6 +188,13 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.hybrid_bus_type = false,
 		.fixed_fw_mem = false,
 		.support_off_channel_tx = false,
+		.supports_multi_bssid = false,
+
+		.sram_dump = {},
+
+		.tcl_ring_retry = true,
+		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
+		.smp2p_wow_exit = false,
 	},
 	{
 		.name = "qca6390 hw2.0",
@@ -212,6 +230,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 			.summary_pad_sz = 0,
 			.fft_hdr_len = 0,
 			.max_fft_bins = 0,
+			.fragment_160mhz = false,
 		},
 
 		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
@@ -222,6 +241,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.idle_ps = true,
 		.supports_sta_ps = true,
 		.cold_boot_calib = false,
+		.cbcal_restart_fw = false,
 		.fw_mem_mode = 0,
 		.num_vdevs = 16 + 1,
 		.num_peers = 512,
@@ -247,6 +267,16 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.hybrid_bus_type = false,
 		.fixed_fw_mem = false,
 		.support_off_channel_tx = true,
+		.supports_multi_bssid = true,
+
+		.sram_dump = {
+			.start = 0x01400000,
+			.end = 0x0171ffff,
+		},
+
+		.tcl_ring_retry = true,
+		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
+		.smp2p_wow_exit = false,
 	},
 	{
 		.name = "qcn9074 hw1.0",
@@ -281,6 +311,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 			.summary_pad_sz = 16,
 			.fft_hdr_len = 24,
 			.max_fft_bins = 1024,
+			.fragment_160mhz = false,
 		},
 
 		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
@@ -292,6 +323,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.idle_ps = false,
 		.supports_sta_ps = false,
 		.cold_boot_calib = false,
+		.cbcal_restart_fw = false,
 		.fw_mem_mode = 2,
 		.num_vdevs = 8,
 		.num_peers = 128,
@@ -317,6 +349,13 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.hybrid_bus_type = false,
 		.fixed_fw_mem = false,
 		.support_off_channel_tx = false,
+		.supports_multi_bssid = false,
+
+		.sram_dump = {},
+
+		.tcl_ring_retry = true,
+		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
+		.smp2p_wow_exit = false,
 	},
 	{
 		.name = "wcn6855 hw2.0",
@@ -352,6 +391,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 			.summary_pad_sz = 0,
 			.fft_hdr_len = 0,
 			.max_fft_bins = 0,
+			.fragment_160mhz = false,
 		},
 
 		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
@@ -362,6 +402,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.idle_ps = true,
 		.supports_sta_ps = true,
 		.cold_boot_calib = false,
+		.cbcal_restart_fw = false,
 		.fw_mem_mode = 0,
 		.num_vdevs = 16 + 1,
 		.num_peers = 512,
@@ -387,6 +428,16 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.hybrid_bus_type = false,
 		.fixed_fw_mem = false,
 		.support_off_channel_tx = true,
+		.supports_multi_bssid = true,
+
+		.sram_dump = {
+			.start = 0x01400000,
+			.end = 0x0177ffff,
+		},
+
+		.tcl_ring_retry = true,
+		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
+		.smp2p_wow_exit = false,
 	},
 	{
 		.name = "wcn6855 hw2.1",
@@ -422,6 +473,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 			.summary_pad_sz = 0,
 			.fft_hdr_len = 0,
 			.max_fft_bins = 0,
+			.fragment_160mhz = false,
 		},
 
 		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
@@ -431,6 +483,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.idle_ps = true,
 		.supports_sta_ps = true,
 		.cold_boot_calib = false,
+		.cbcal_restart_fw = false,
 		.fw_mem_mode = 0,
 		.num_vdevs = 16 + 1,
 		.num_peers = 512,
@@ -456,6 +509,16 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.hybrid_bus_type = false,
 		.fixed_fw_mem = false,
 		.support_off_channel_tx = true,
+		.supports_multi_bssid = true,
+
+		.sram_dump = {
+			.start = 0x01400000,
+			.end = 0x0177ffff,
+		},
+
+		.tcl_ring_retry = true,
+		.tx_ring_size = DP_TCL_DATA_RING_SIZE,
+		.smp2p_wow_exit = false,
 	},
 	{
 		.name = "wcn6750 hw1.0",
@@ -468,7 +531,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.max_radios = 1,
 		.bdf_addr = 0x4B0C0000,
 		.hw_ops = &wcn6750_ops,
-		.ring_mask = &ath11k_hw_ring_mask_qca6390,
+		.ring_mask = &ath11k_hw_ring_mask_wcn6750,
 		.internal_sleep_clock = false,
 		.regs = &wcn6750_regs,
 		.qmi_service_ins_id = ATH11K_QMI_WLFW_SERVICE_INS_ID_V01_WCN6750,
@@ -491,6 +554,7 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 			.summary_pad_sz = 0,
 			.fft_hdr_len = 0,
 			.max_fft_bins = 0,
+			.fragment_160mhz = false,
 		},
 
 		.interface_modes = BIT(NL80211_IFTYPE_STATION) |
@@ -499,7 +563,8 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.supports_shadow_regs = true,
 		.idle_ps = true,
 		.supports_sta_ps = true,
-		.cold_boot_calib = false,
+		.cold_boot_calib = true,
+		.cbcal_restart_fw = false,
 		.fw_mem_mode = 0,
 		.num_vdevs = 16 + 1,
 		.num_peers = 512,
@@ -508,8 +573,8 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.supports_regdb = true,
 		.fix_l1ss = false,
 		.credit_flow = true,
-		.max_tx_ring = DP_TCL_NUM_RING_MAX_QCA6390,
-		.hal_params = &ath11k_hw_hal_params_qca6390,
+		.max_tx_ring = DP_TCL_NUM_RING_MAX,
+		.hal_params = &ath11k_hw_hal_params_wcn6750,
 		.supports_dynamic_smps_6ghz = false,
 		.alloc_cacheable_memory = false,
 		.supports_rssi_stats = true,
@@ -524,7 +589,14 @@ static const struct ath11k_hw_params ath11k_hw_params[] = {
 		.static_window_map = true,
 		.hybrid_bus_type = true,
 		.fixed_fw_mem = true,
-		.support_off_channel_tx = false,
+		.support_off_channel_tx = true,
+		.supports_multi_bssid = true,
+
+		.sram_dump = {},
+
+		.tcl_ring_retry = false,
+		.tx_ring_size = DP_TCL_DATA_RING_SIZE_WCN6750,
+		.smp2p_wow_exit = true,
 	},
 };
 
@@ -535,6 +607,52 @@ static inline struct ath11k_pdev *ath11k_core_get_single_pdev(struct ath11k_base
 	return &ab->pdevs[0];
 }
 
+void ath11k_fw_stats_pdevs_free(struct list_head *head)
+{
+	struct ath11k_fw_stats_pdev *i, *tmp;
+
+	list_for_each_entry_safe(i, tmp, head, list) {
+		list_del(&i->list);
+		kfree(i);
+	}
+}
+
+void ath11k_fw_stats_vdevs_free(struct list_head *head)
+{
+	struct ath11k_fw_stats_vdev *i, *tmp;
+
+	list_for_each_entry_safe(i, tmp, head, list) {
+		list_del(&i->list);
+		kfree(i);
+	}
+}
+
+void ath11k_fw_stats_bcn_free(struct list_head *head)
+{
+	struct ath11k_fw_stats_bcn *i, *tmp;
+
+	list_for_each_entry_safe(i, tmp, head, list) {
+		list_del(&i->list);
+		kfree(i);
+	}
+}
+
+void ath11k_fw_stats_init(struct ath11k *ar)
+{
+	INIT_LIST_HEAD(&ar->fw_stats.pdevs);
+	INIT_LIST_HEAD(&ar->fw_stats.vdevs);
+	INIT_LIST_HEAD(&ar->fw_stats.bcn);
+
+	init_completion(&ar->fw_stats_complete);
+}
+
+void ath11k_fw_stats_free(struct ath11k_fw_stats *stats)
+{
+	ath11k_fw_stats_pdevs_free(&stats->pdevs);
+	ath11k_fw_stats_vdevs_free(&stats->vdevs);
+	ath11k_fw_stats_bcn_free(&stats->bcn);
+}
+
 int ath11k_core_suspend(struct ath11k_base *ab)
 {
 	int ret;
@@ -1544,7 +1662,7 @@ static void ath11k_core_pre_reconfigure_recovery(struct ath11k_base *ab)
 		ar->state_11d = ATH11K_11D_IDLE;
 		complete(&ar->completed_11d_scan);
 		complete(&ar->scan.started);
-		complete(&ar->scan.completed);
+		complete_all(&ar->scan.completed);
 		complete(&ar->scan.on_channel);
 		complete(&ar->peer_assoc_done);
 		complete(&ar->peer_delete_done);
@@ -1563,6 +1681,8 @@ static void ath11k_core_pre_reconfigure_recovery(struct ath11k_base *ab)
 
 	wake_up(&ab->wmi_ab.tx_credits_wq);
 	wake_up(&ab->peer_mapping_wq);
+
+	reinit_completion(&ab->driver_recovery);
 }
 
 static void ath11k_core_post_reconfigure_recovery(struct ath11k_base *ab)
diff --git a/drivers/net/wireless/ath/ath11k/core.h b/drivers/net/wireless/ath/ath11k/core.h
index afad8f55e433..cf2f52cc4e30 100644
--- a/drivers/net/wireless/ath/ath11k/core.h
+++ b/drivers/net/wireless/ath/ath11k/core.h
@@ -498,6 +498,13 @@ struct ath11k_sta {
 
 	bool use_4addr_set;
 	u16 tcl_metadata;
+
+	/* Protected with ar->data_lock */
+	enum ath11k_wmi_peer_ps_state peer_ps_state;
+	u64 ps_start_time;
+	u64 ps_start_jiffies;
+	u64 ps_total_duration;
+	bool peer_current_ps_valid;
 };
 
 #define ATH11K_MIN_5G_FREQ 4150
@@ -545,9 +552,6 @@ struct ath11k_debug {
 	struct dentry *debugfs_pdev;
 	struct ath11k_dbg_htt_stats htt_stats;
 	u32 extd_tx_stats;
-	struct ath11k_fw_stats fw_stats;
-	struct completion fw_stats_complete;
-	bool fw_stats_done;
 	u32 extd_rx_stats;
 	u32 pktlog_filter;
 	u32 pktlog_mode;
@@ -710,6 +714,13 @@ struct ath11k {
 	u8 twt_enabled;
 	bool nlo_enabled;
 	u8 alpha2[REG_ALPHA2_LEN + 1];
+	struct ath11k_fw_stats fw_stats;
+	struct completion fw_stats_complete;
+	bool fw_stats_done;
+
+	/* protected by conf_mutex */
+	bool ps_state_enable;
+	bool ps_timekeeper_enable;
 };
 
 struct ath11k_band_cap {
@@ -887,7 +898,7 @@ struct ath11k_base {
 
 	/* Below regd's are protected by ab->data_lock */
 	/* This is the regd set for every radio
-	 * by the firmware during initializatin
+	 * by the firmware during initialization
 	 */
 	struct ieee80211_regdomain *default_regd[MAX_RADIOS];
 	/* This regd is set during dynamic country setting
@@ -1112,6 +1123,12 @@ struct ath11k_fw_stats_bcn {
 	u32 tx_bcn_outage_cnt;
 };
 
+void ath11k_fw_stats_init(struct ath11k *ar);
+void ath11k_fw_stats_pdevs_free(struct list_head *head);
+void ath11k_fw_stats_vdevs_free(struct list_head *head);
+void ath11k_fw_stats_bcn_free(struct list_head *head);
+void ath11k_fw_stats_free(struct ath11k_fw_stats *stats);
+
 extern const struct ce_pipe_config ath11k_target_ce_config_wlan_ipq8074[];
 extern const struct service_to_pipe ath11k_target_service_to_ce_map_wlan_ipq8074[];
 extern const struct service_to_pipe ath11k_target_service_to_ce_map_wlan_ipq6018[];
diff --git a/drivers/net/wireless/ath/ath11k/debugfs.c b/drivers/net/wireless/ath/ath11k/debugfs.c
index 9648e0017393..ccdf3d5ba1ab 100644
--- a/drivers/net/wireless/ath/ath11k/debugfs.c
+++ b/drivers/net/wireless/ath/ath11k/debugfs.c
@@ -14,6 +14,7 @@
 #include "dp_tx.h"
 #include "debugfs_htt_stats.h"
 #include "peer.h"
+#include "hif.h"
 
 static const char *htt_bp_umac_ring[HTT_SW_UMAC_RING_IDX_MAX] = {
 	"REO2SW1_RING",
@@ -91,91 +92,35 @@ void ath11k_debugfs_add_dbring_entry(struct ath11k *ar,
 	spin_unlock_bh(&dbr_data->lock);
 }
 
-static void ath11k_fw_stats_pdevs_free(struct list_head *head)
-{
-	struct ath11k_fw_stats_pdev *i, *tmp;
-
-	list_for_each_entry_safe(i, tmp, head, list) {
-		list_del(&i->list);
-		kfree(i);
-	}
-}
-
-static void ath11k_fw_stats_vdevs_free(struct list_head *head)
-{
-	struct ath11k_fw_stats_vdev *i, *tmp;
-
-	list_for_each_entry_safe(i, tmp, head, list) {
-		list_del(&i->list);
-		kfree(i);
-	}
-}
-
-static void ath11k_fw_stats_bcn_free(struct list_head *head)
-{
-	struct ath11k_fw_stats_bcn *i, *tmp;
-
-	list_for_each_entry_safe(i, tmp, head, list) {
-		list_del(&i->list);
-		kfree(i);
-	}
-}
-
 static void ath11k_debugfs_fw_stats_reset(struct ath11k *ar)
 {
 	spin_lock_bh(&ar->data_lock);
-	ar->debug.fw_stats_done = false;
-	ath11k_fw_stats_pdevs_free(&ar->debug.fw_stats.pdevs);
-	ath11k_fw_stats_vdevs_free(&ar->debug.fw_stats.vdevs);
+	ar->fw_stats_done = false;
+	ath11k_fw_stats_pdevs_free(&ar->fw_stats.pdevs);
+	ath11k_fw_stats_vdevs_free(&ar->fw_stats.vdevs);
 	spin_unlock_bh(&ar->data_lock);
 }
 
-void ath11k_debugfs_fw_stats_process(struct ath11k_base *ab, struct sk_buff *skb)
+void ath11k_debugfs_fw_stats_process(struct ath11k *ar, struct ath11k_fw_stats *stats)
 {
-	struct ath11k_fw_stats stats = {};
-	struct ath11k *ar;
+	struct ath11k_base *ab = ar->ab;
 	struct ath11k_pdev *pdev;
 	bool is_end;
 	static unsigned int num_vdev, num_bcn;
 	size_t total_vdevs_started = 0;
-	int i, ret;
-
-	INIT_LIST_HEAD(&stats.pdevs);
-	INIT_LIST_HEAD(&stats.vdevs);
-	INIT_LIST_HEAD(&stats.bcn);
-
-	ret = ath11k_wmi_pull_fw_stats(ab, skb, &stats);
-	if (ret) {
-		ath11k_warn(ab, "failed to pull fw stats: %d\n", ret);
-		goto free;
-	}
-
-	rcu_read_lock();
-	ar = ath11k_mac_get_ar_by_pdev_id(ab, stats.pdev_id);
-	if (!ar) {
-		rcu_read_unlock();
-		ath11k_warn(ab, "failed to get ar for pdev_id %d: %d\n",
-			    stats.pdev_id, ret);
-		goto free;
-	}
+	int i;
 
-	spin_lock_bh(&ar->data_lock);
+	/* WMI_REQUEST_PDEV_STAT request has been already processed */
 
-	if (stats.stats_id == WMI_REQUEST_PDEV_STAT) {
-		list_splice_tail_init(&stats.pdevs, &ar->debug.fw_stats.pdevs);
-		ar->debug.fw_stats_done = true;
-		goto complete;
-	}
-
-	if (stats.stats_id == WMI_REQUEST_RSSI_PER_CHAIN_STAT) {
-		ar->debug.fw_stats_done = true;
-		goto complete;
+	if (stats->stats_id == WMI_REQUEST_RSSI_PER_CHAIN_STAT) {
+		ar->fw_stats_done = true;
+		return;
 	}
 
-	if (stats.stats_id == WMI_REQUEST_VDEV_STAT) {
-		if (list_empty(&stats.vdevs)) {
+	if (stats->stats_id == WMI_REQUEST_VDEV_STAT) {
+		if (list_empty(&stats->vdevs)) {
 			ath11k_warn(ab, "empty vdev stats");
-			goto complete;
+			return;
 		}
 		/* FW sends all the active VDEV stats irrespective of PDEV,
 		 * hence limit until the count of all VDEVs started
@@ -188,43 +133,34 @@ void ath11k_debugfs_fw_stats_process(struct ath11k_base *ab, struct sk_buff *skb
 
 		is_end = ((++num_vdev) == total_vdevs_started);
 
-		list_splice_tail_init(&stats.vdevs,
-				      &ar->debug.fw_stats.vdevs);
+		list_splice_tail_init(&stats->vdevs,
+				      &ar->fw_stats.vdevs);
 
 		if (is_end) {
-			ar->debug.fw_stats_done = true;
+			ar->fw_stats_done = true;
 			num_vdev = 0;
 		}
-		goto complete;
+		return;
 	}
 
-	if (stats.stats_id == WMI_REQUEST_BCN_STAT) {
-		if (list_empty(&stats.bcn)) {
+	if (stats->stats_id == WMI_REQUEST_BCN_STAT) {
+		if (list_empty(&stats->bcn)) {
 			ath11k_warn(ab, "empty bcn stats");
-			goto complete;
+			return;
 		}
 		/* Mark end until we reached the count of all started VDEVs
 		 * within the PDEV
 		 */
 		is_end = ((++num_bcn) == ar->num_started_vdevs);
 
-		list_splice_tail_init(&stats.bcn,
-				      &ar->debug.fw_stats.bcn);
+		list_splice_tail_init(&stats->bcn,
+				      &ar->fw_stats.bcn);
 
 		if (is_end) {
-			ar->debug.fw_stats_done = true;
+			ar->fw_stats_done = true;
 			num_bcn = 0;
 		}
 	}
-complete:
-	complete(&ar->debug.fw_stats_complete);
-	rcu_read_unlock();
-	spin_unlock_bh(&ar->data_lock);
-
-free:
-	ath11k_fw_stats_pdevs_free(&stats.pdevs);
-	ath11k_fw_stats_vdevs_free(&stats.vdevs);
-	ath11k_fw_stats_bcn_free(&stats.bcn);
 }
 
 static int ath11k_debugfs_fw_stats_request(struct ath11k *ar,
@@ -245,7 +181,7 @@ static int ath11k_debugfs_fw_stats_request(struct ath11k *ar,
 
 	ath11k_debugfs_fw_stats_reset(ar);
 
-	reinit_completion(&ar->debug.fw_stats_complete);
+	reinit_completion(&ar->fw_stats_complete);
 
 	ret = ath11k_wmi_send_stats_request_cmd(ar, req_param);
 
@@ -255,9 +191,8 @@ static int ath11k_debugfs_fw_stats_request(struct ath11k *ar,
 		return ret;
 	}
 
-	time_left =
-	wait_for_completion_timeout(&ar->debug.fw_stats_complete,
-				    1 * HZ);
+	time_left = wait_for_completion_timeout(&ar->fw_stats_complete, 1 * HZ);
+
 	if (!time_left)
 		return -ETIMEDOUT;
 
@@ -266,7 +201,7 @@ static int ath11k_debugfs_fw_stats_request(struct ath11k *ar,
 			break;
 
 		spin_lock_bh(&ar->data_lock);
-		if (ar->debug.fw_stats_done) {
+		if (ar->fw_stats_done) {
 			spin_unlock_bh(&ar->data_lock);
 			break;
 		}
@@ -338,8 +273,7 @@ static int ath11k_open_pdev_stats(struct inode *inode, struct file *file)
 		goto err_free;
 	}
 
-	ath11k_wmi_fw_stats_fill(ar, &ar->debug.fw_stats, req_param.stats_id,
-				 buf);
+	ath11k_wmi_fw_stats_fill(ar, &ar->fw_stats, req_param.stats_id, buf);
 
 	file->private_data = buf;
 
@@ -410,8 +344,7 @@ static int ath11k_open_vdev_stats(struct inode *inode, struct file *file)
 		goto err_free;
 	}
 
-	ath11k_wmi_fw_stats_fill(ar, &ar->debug.fw_stats, req_param.stats_id,
-				 buf);
+	ath11k_wmi_fw_stats_fill(ar, &ar->fw_stats, req_param.stats_id, buf);
 
 	file->private_data = buf;
 
@@ -488,14 +421,13 @@ static int ath11k_open_bcn_stats(struct inode *inode, struct file *file)
 		}
 	}
 
-	ath11k_wmi_fw_stats_fill(ar, &ar->debug.fw_stats, req_param.stats_id,
-				 buf);
+	ath11k_wmi_fw_stats_fill(ar, &ar->fw_stats, req_param.stats_id, buf);
 
 	/* since beacon stats request is looped for all active VDEVs, saved fw
 	 * stats is not freed for each request until done for all active VDEVs
 	 */
 	spin_lock_bh(&ar->data_lock);
-	ath11k_fw_stats_bcn_free(&ar->debug.fw_stats.bcn);
+	ath11k_fw_stats_bcn_free(&ar->fw_stats.bcn);
 	spin_unlock_bh(&ar->data_lock);
 
 	file->private_data = buf;
@@ -982,6 +914,63 @@ static const struct file_operations fops_fw_dbglog = {
 	.llseek = default_llseek,
 };
 
+static int ath11k_open_sram_dump(struct inode *inode, struct file *file)
+{
+	struct ath11k_base *ab = inode->i_private;
+	u8 *buf;
+	u32 start, end;
+	int ret;
+
+	start = ab->hw_params.sram_dump.start;
+	end = ab->hw_params.sram_dump.end;
+
+	buf = vmalloc(end - start + 1);
+	if (!buf)
+		return -ENOMEM;
+
+	ret = ath11k_hif_read(ab, buf, start, end);
+	if (ret) {
+		ath11k_warn(ab, "failed to dump sram: %d\n", ret);
+		vfree(buf);
+		return ret;
+	}
+
+	file->private_data = buf;
+	return 0;
+}
+
+static ssize_t ath11k_read_sram_dump(struct file *file,
+				     char __user *user_buf,
+				     size_t count, loff_t *ppos)
+{
+	struct ath11k_base *ab = file->f_inode->i_private;
+	const char *buf = file->private_data;
+	int len;
+	u32 start, end;
+
+	start = ab->hw_params.sram_dump.start;
+	end = ab->hw_params.sram_dump.end;
+	len = end - start + 1;
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
+}
+
+static int ath11k_release_sram_dump(struct inode *inode, struct file *file)
+{
+	vfree(file->private_data);
+	file->private_data = NULL;
+
+	return 0;
+}
+
+static const struct file_operations fops_sram_dump = {
+	.open = ath11k_open_sram_dump,
+	.read = ath11k_read_sram_dump,
+	.release = ath11k_release_sram_dump,
+	.owner = THIS_MODULE,
+	.llseek = default_llseek,
+};
+
 int ath11k_debugfs_pdev_create(struct ath11k_base *ab)
 {
 	if (test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags))
@@ -997,6 +986,10 @@ int ath11k_debugfs_pdev_create(struct ath11k_base *ab)
 	debugfs_create_file("soc_dp_stats", 0600, ab->debugfs_soc, ab,
 			    &fops_soc_dp_stats);
 
+	if (ab->hw_params.sram_dump.start != 0)
+		debugfs_create_file("sram", 0400, ab->debugfs_soc, ab,
+				    &fops_sram_dump);
+
 	return 0;
 }
 
@@ -1025,7 +1018,7 @@ void ath11k_debugfs_fw_stats_init(struct ath11k *ar)
 	struct dentry *fwstats_dir = debugfs_create_dir("fw_stats",
 							ar->debug.debugfs_pdev);
 
-	ar->debug.fw_stats.debugfs_fwstats = fwstats_dir;
+	ar->fw_stats.debugfs_fwstats = fwstats_dir;
 
 	/* all stats debugfs files created are under "fw_stats" directory
 	 * created per PDEV
@@ -1036,12 +1029,6 @@ void ath11k_debugfs_fw_stats_init(struct ath11k *ar)
 			    &fops_vdev_stats);
 	debugfs_create_file("beacon_stats", 0600, fwstats_dir, ar,
 			    &fops_bcn_stats);
-
-	INIT_LIST_HEAD(&ar->debug.fw_stats.pdevs);
-	INIT_LIST_HEAD(&ar->debug.fw_stats.vdevs);
-	INIT_LIST_HEAD(&ar->debug.fw_stats.bcn);
-
-	init_completion(&ar->debug.fw_stats_complete);
 }
 
 static ssize_t ath11k_write_pktlog_filter(struct file *file,
@@ -1382,6 +1369,193 @@ static const struct file_operations fops_dbr_debug = {
 	.llseek = default_llseek,
 };
 
+static ssize_t ath11k_write_ps_timekeeper_enable(struct file *file,
+						 const char __user *user_buf,
+						 size_t count, loff_t *ppos)
+{
+	struct ath11k *ar = file->private_data;
+	ssize_t ret;
+	u8 ps_timekeeper_enable;
+
+	if (kstrtou8_from_user(user_buf, count, 0, &ps_timekeeper_enable))
+		return -EINVAL;
+
+	mutex_lock(&ar->conf_mutex);
+
+	if (ar->state != ATH11K_STATE_ON) {
+		ret = -ENETDOWN;
+		goto exit;
+	}
+
+	if (!ar->ps_state_enable) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	ar->ps_timekeeper_enable = !!ps_timekeeper_enable;
+	ret = count;
+exit:
+	mutex_unlock(&ar->conf_mutex);
+
+	return ret;
+}
+
+static ssize_t ath11k_read_ps_timekeeper_enable(struct file *file,
+						char __user *user_buf,
+						size_t count, loff_t *ppos)
+{
+	struct ath11k *ar = file->private_data;
+	char buf[32];
+	int len;
+
+	mutex_lock(&ar->conf_mutex);
+	len = scnprintf(buf, sizeof(buf), "%d\n", ar->ps_timekeeper_enable);
+	mutex_unlock(&ar->conf_mutex);
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
+}
+
+static const struct file_operations fops_ps_timekeeper_enable = {
+	.read = ath11k_read_ps_timekeeper_enable,
+	.write = ath11k_write_ps_timekeeper_enable,
+	.open = simple_open,
+	.owner = THIS_MODULE,
+	.llseek = default_llseek,
+};
+
+static void ath11k_reset_peer_ps_duration(void *data,
+					  struct ieee80211_sta *sta)
+{
+	struct ath11k *ar = data;
+	struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv;
+
+	spin_lock_bh(&ar->data_lock);
+	arsta->ps_total_duration = 0;
+	spin_unlock_bh(&ar->data_lock);
+}
+
+static ssize_t ath11k_write_reset_ps_duration(struct file *file,
+					      const  char __user *user_buf,
+					      size_t count, loff_t *ppos)
+{
+	struct ath11k *ar = file->private_data;
+	int ret;
+	u8 reset_ps_duration;
+
+	if (kstrtou8_from_user(user_buf, count, 0, &reset_ps_duration))
+		return -EINVAL;
+
+	mutex_lock(&ar->conf_mutex);
+
+	if (ar->state != ATH11K_STATE_ON) {
+		ret = -ENETDOWN;
+		goto exit;
+	}
+
+	if (!ar->ps_state_enable) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	ieee80211_iterate_stations_atomic(ar->hw,
+					  ath11k_reset_peer_ps_duration,
+					  ar);
+
+	ret = count;
+exit:
+	mutex_unlock(&ar->conf_mutex);
+	return ret;
+}
+
+static const struct file_operations fops_reset_ps_duration = {
+	.write = ath11k_write_reset_ps_duration,
+	.open = simple_open,
+	.owner = THIS_MODULE,
+	.llseek = default_llseek,
+};
+
+static void ath11k_peer_ps_state_disable(void *data,
+					 struct ieee80211_sta *sta)
+{
+	struct ath11k *ar = data;
+	struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv;
+
+	spin_lock_bh(&ar->data_lock);
+	arsta->peer_ps_state = WMI_PEER_PS_STATE_DISABLED;
+	arsta->ps_start_time = 0;
+	arsta->ps_total_duration = 0;
+	spin_unlock_bh(&ar->data_lock);
+}
+
+static ssize_t ath11k_write_ps_state_enable(struct file *file,
+					    const char __user *user_buf,
+					    size_t count, loff_t *ppos)
+{
+	struct ath11k *ar = file->private_data;
+	struct ath11k_pdev *pdev = ar->pdev;
+	int ret;
+	u32 param;
+	u8 ps_state_enable;
+
+	if (kstrtou8_from_user(user_buf, count, 0, &ps_state_enable))
+		return -EINVAL;
+
+	mutex_lock(&ar->conf_mutex);
+
+	ps_state_enable = !!ps_state_enable;
+
+	if (ar->ps_state_enable == ps_state_enable) {
+		ret = count;
+		goto exit;
+	}
+
+	param = WMI_PDEV_PEER_STA_PS_STATECHG_ENABLE;
+	ret = ath11k_wmi_pdev_set_param(ar, param, ps_state_enable, pdev->pdev_id);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to enable ps_state_enable: %d\n",
+			    ret);
+		goto exit;
+	}
+	ar->ps_state_enable = ps_state_enable;
+
+	if (!ar->ps_state_enable) {
+		ar->ps_timekeeper_enable = false;
+		ieee80211_iterate_stations_atomic(ar->hw,
+						  ath11k_peer_ps_state_disable,
+						  ar);
+	}
+
+	ret = count;
+
+exit:
+	mutex_unlock(&ar->conf_mutex);
+
+	return ret;
+}
+
+static ssize_t ath11k_read_ps_state_enable(struct file *file,
+					   char __user *user_buf,
+					   size_t count, loff_t *ppos)
+{
+	struct ath11k *ar = file->private_data;
+	char buf[32];
+	int len;
+
+	mutex_lock(&ar->conf_mutex);
+	len = scnprintf(buf, sizeof(buf), "%d\n", ar->ps_state_enable);
+	mutex_unlock(&ar->conf_mutex);
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
+}
+
+static const struct file_operations fops_ps_state_enable = {
+	.read = ath11k_read_ps_state_enable,
+	.write = ath11k_write_ps_state_enable,
+	.open = simple_open,
+	.owner = THIS_MODULE,
+	.llseek = default_llseek,
+};
+
 int ath11k_debugfs_register(struct ath11k *ar)
 {
 	struct ath11k_base *ab = ar->ab;
@@ -1428,6 +1602,20 @@ int ath11k_debugfs_register(struct ath11k *ar)
 		debugfs_create_file("enable_dbr_debug", 0200, ar->debug.debugfs_pdev,
 				    ar, &fops_dbr_debug);
 
+	debugfs_create_file("ps_state_enable", 0600, ar->debug.debugfs_pdev, ar,
+			    &fops_ps_state_enable);
+
+	if (test_bit(WMI_TLV_SERVICE_PEER_POWER_SAVE_DURATION_SUPPORT,
+		     ar->ab->wmi_ab.svc_map)) {
+		debugfs_create_file("ps_timekeeper_enable", 0600,
+				    ar->debug.debugfs_pdev, ar,
+				    &fops_ps_timekeeper_enable);
+
+		debugfs_create_file("reset_ps_duration", 0200,
+				    ar->debug.debugfs_pdev, ar,
+				    &fops_reset_ps_duration);
+	}
+
 	return 0;
 }
 
@@ -1456,11 +1644,13 @@ static ssize_t ath11k_write_twt_add_dialog(struct file *file,
 {
 	struct ath11k_vif *arvif = file->private_data;
 	struct wmi_twt_add_dialog_params params = { 0 };
+	struct wmi_twt_enable_params twt_params = {0};
+	struct ath11k *ar = arvif->ar;
 	u8 buf[128] = {0};
 	int ret;
 
-	if (arvif->ar->twt_enabled == 0) {
-		ath11k_err(arvif->ar->ab, "twt support is not enabled\n");
+	if (ar->twt_enabled == 0) {
+		ath11k_err(ar->ab, "twt support is not enabled\n");
 		return -EOPNOTSUPP;
 	}
 
@@ -1490,13 +1680,38 @@ static ssize_t ath11k_write_twt_add_dialog(struct file *file,
 	if (ret != 16)
 		return -EINVAL;
 
+	/* In the case of station vif, TWT is entirely handled by
+	 * the firmware based on the input parameters in the TWT enable
+	 * WMI command that is sent to the target during assoc.
+	 * For manually testing the TWT feature, we need to first disable
+	 * TWT and send enable command again with TWT input parameter
+	 * sta_cong_timer_ms set to 0.
+	 */
+	if (arvif->vif->type == NL80211_IFTYPE_STATION) {
+		ath11k_wmi_send_twt_disable_cmd(ar, ar->pdev->pdev_id);
+
+		ath11k_wmi_fill_default_twt_params(&twt_params);
+		twt_params.sta_cong_timer_ms = 0;
+
+		ath11k_wmi_send_twt_enable_cmd(ar, ar->pdev->pdev_id, &twt_params);
+	}
+
 	params.vdev_id = arvif->vdev_id;
 
 	ret = ath11k_wmi_send_twt_add_dialog_cmd(arvif->ar, &params);
 	if (ret)
-		return ret;
+		goto err_twt_add_dialog;
 
 	return count;
+
+err_twt_add_dialog:
+	if (arvif->vif->type == NL80211_IFTYPE_STATION) {
+		ath11k_wmi_send_twt_disable_cmd(ar, ar->pdev->pdev_id);
+		ath11k_wmi_fill_default_twt_params(&twt_params);
+		ath11k_wmi_send_twt_enable_cmd(ar, ar->pdev->pdev_id, &twt_params);
+	}
+
+	return ret;
 }
 
 static ssize_t ath11k_write_twt_del_dialog(struct file *file,
@@ -1505,11 +1720,13 @@ static ssize_t ath11k_write_twt_del_dialog(struct file *file,
 {
 	struct ath11k_vif *arvif = file->private_data;
 	struct wmi_twt_del_dialog_params params = { 0 };
+	struct wmi_twt_enable_params twt_params = {0};
+	struct ath11k *ar = arvif->ar;
 	u8 buf[64] = {0};
 	int ret;
 
-	if (arvif->ar->twt_enabled == 0) {
-		ath11k_err(arvif->ar->ab, "twt support is not enabled\n");
+	if (ar->twt_enabled == 0) {
+		ath11k_err(ar->ab, "twt support is not enabled\n");
 		return -EOPNOTSUPP;
 	}
 
@@ -1535,6 +1752,12 @@ static ssize_t ath11k_write_twt_del_dialog(struct file *file,
 	if (ret)
 		return ret;
 
+	if (arvif->vif->type == NL80211_IFTYPE_STATION) {
+		ath11k_wmi_send_twt_disable_cmd(ar, ar->pdev->pdev_id);
+		ath11k_wmi_fill_default_twt_params(&twt_params);
+		ath11k_wmi_send_twt_enable_cmd(ar, ar->pdev->pdev_id, &twt_params);
+	}
+
 	return count;
 }
 
@@ -1638,36 +1861,35 @@ static const struct file_operations ath11k_fops_twt_resume_dialog = {
 	.open = simple_open
 };
 
-int ath11k_debugfs_add_interface(struct ath11k_vif *arvif)
+void ath11k_debugfs_add_interface(struct ath11k_vif *arvif)
 {
-	if (arvif->vif->type == NL80211_IFTYPE_AP && !arvif->debugfs_twt) {
-		arvif->debugfs_twt = debugfs_create_dir("twt",
-							arvif->vif->debugfs_dir);
-		if (!arvif->debugfs_twt || IS_ERR(arvif->debugfs_twt)) {
-			ath11k_warn(arvif->ar->ab,
-				    "failed to create directory %p\n",
-				    arvif->debugfs_twt);
-			arvif->debugfs_twt = NULL;
-			return -1;
-		}
+	struct ath11k_base *ab = arvif->ar->ab;
 
-		debugfs_create_file("add_dialog", 0200, arvif->debugfs_twt,
-				    arvif, &ath11k_fops_twt_add_dialog);
+	if (arvif->vif->type != NL80211_IFTYPE_AP &&
+	    !(arvif->vif->type == NL80211_IFTYPE_STATION &&
+	      test_bit(WMI_TLV_SERVICE_STA_TWT, ab->wmi_ab.svc_map)))
+		return;
 
-		debugfs_create_file("del_dialog", 0200, arvif->debugfs_twt,
-				    arvif, &ath11k_fops_twt_del_dialog);
+	arvif->debugfs_twt = debugfs_create_dir("twt",
+						arvif->vif->debugfs_dir);
+	debugfs_create_file("add_dialog", 0200, arvif->debugfs_twt,
+			    arvif, &ath11k_fops_twt_add_dialog);
 
-		debugfs_create_file("pause_dialog", 0200, arvif->debugfs_twt,
-				    arvif, &ath11k_fops_twt_pause_dialog);
+	debugfs_create_file("del_dialog", 0200, arvif->debugfs_twt,
+			    arvif, &ath11k_fops_twt_del_dialog);
 
-		debugfs_create_file("resume_dialog", 0200, arvif->debugfs_twt,
-				    arvif, &ath11k_fops_twt_resume_dialog);
-	}
-	return 0;
+	debugfs_create_file("pause_dialog", 0200, arvif->debugfs_twt,
+			    arvif, &ath11k_fops_twt_pause_dialog);
+
+	debugfs_create_file("resume_dialog", 0200, arvif->debugfs_twt,
+			    arvif, &ath11k_fops_twt_resume_dialog);
 }
 
 void ath11k_debugfs_remove_interface(struct ath11k_vif *arvif)
 {
+	if (!arvif->debugfs_twt)
+		return;
+
 	debugfs_remove_recursive(arvif->debugfs_twt);
 	arvif->debugfs_twt = NULL;
 }
diff --git a/drivers/net/wireless/ath/ath11k/debugfs.h b/drivers/net/wireless/ath/ath11k/debugfs.h
index 30c00cb28311..3af0169f6cf2 100644
--- a/drivers/net/wireless/ath/ath11k/debugfs.h
+++ b/drivers/net/wireless/ath/ath11k/debugfs.h
@@ -269,7 +269,7 @@ int ath11k_debugfs_pdev_create(struct ath11k_base *ab);
 void ath11k_debugfs_pdev_destroy(struct ath11k_base *ab);
 int ath11k_debugfs_register(struct ath11k *ar);
 void ath11k_debugfs_unregister(struct ath11k *ar);
-void ath11k_debugfs_fw_stats_process(struct ath11k_base *ab, struct sk_buff *skb);
+void ath11k_debugfs_fw_stats_process(struct ath11k *ar, struct ath11k_fw_stats *stats);
 
 void ath11k_debugfs_fw_stats_init(struct ath11k *ar);
 int ath11k_debugfs_get_fw_stats(struct ath11k *ar, u32 pdev_id,
@@ -306,7 +306,7 @@ static inline int ath11k_debugfs_rx_filter(struct ath11k *ar)
 	return ar->debug.rx_filter;
 }
 
-int ath11k_debugfs_add_interface(struct ath11k_vif *arvif);
+void ath11k_debugfs_add_interface(struct ath11k_vif *arvif);
 void ath11k_debugfs_remove_interface(struct ath11k_vif *arvif);
 void ath11k_debugfs_add_dbring_entry(struct ath11k *ar,
 				     enum wmi_direct_buffer_module id,
@@ -341,8 +341,8 @@ static inline void ath11k_debugfs_unregister(struct ath11k *ar)
 {
 }
 
-static inline void ath11k_debugfs_fw_stats_process(struct ath11k_base *ab,
-						   struct sk_buff *skb)
+static inline void ath11k_debugfs_fw_stats_process(struct ath11k *ar,
+						   struct ath11k_fw_stats *stats)
 {
 }
 
@@ -386,9 +386,8 @@ static inline int ath11k_debugfs_get_fw_stats(struct ath11k *ar,
 	return 0;
 }
 
-static inline int ath11k_debugfs_add_interface(struct ath11k_vif *arvif)
+static inline void ath11k_debugfs_add_interface(struct ath11k_vif *arvif)
 {
-	return 0;
 }
 
 static inline void ath11k_debugfs_remove_interface(struct ath11k_vif *arvif)
diff --git a/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h b/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h
index 5d722b51b125..2b97cbbd28cb 100644
--- a/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h
+++ b/drivers/net/wireless/ath/ath11k/debugfs_htt_stats.h
@@ -630,7 +630,7 @@ struct htt_tx_hwq_tried_mpdu_cnt_hist_tlv_v {
  * completing the burst, we identify the txop used in the burst and
  * incr the corresponding bin.
  * Each bin represents 1ms & we have 10 bins in this histogram.
- * they are deined in FW using the following macros
+ * they are defined in FW using the following macros
  * #define WAL_MAX_TXOP_USED_CNT_HISTOGRAM 10
  * #define WAL_TXOP_USED_HISTOGRAM_INTERVAL 1000 ( 1 ms )
  */
@@ -1897,7 +1897,7 @@ struct htt_phy_counters_tlv {
 	u32 phytx_abort_cnt;
 	/* number of times rx abort initiated by phy */
 	u32 phyrx_abort_cnt;
-	/* number of rx defered count initiated by phy */
+	/* number of rx deferred count initiated by phy */
 	u32 phyrx_defer_abort_cnt;
 	/* number of sizing events generated at LSTF */
 	u32 rx_gain_adj_lstf_event_cnt;
diff --git a/drivers/net/wireless/ath/ath11k/debugfs_sta.c b/drivers/net/wireless/ath/ath11k/debugfs_sta.c
index 1b1acbdf837a..9cc4ef28e751 100644
--- a/drivers/net/wireless/ath/ath11k/debugfs_sta.c
+++ b/drivers/net/wireless/ath/ath11k/debugfs_sta.c
@@ -751,6 +751,102 @@ static const struct file_operations fops_htt_peer_stats_reset = {
 	.llseek = default_llseek,
 };
 
+static ssize_t ath11k_dbg_sta_read_peer_ps_state(struct file *file,
+						 char __user *user_buf,
+						 size_t count, loff_t *ppos)
+{
+	struct ieee80211_sta *sta = file->private_data;
+	struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv;
+	struct ath11k *ar = arsta->arvif->ar;
+	char buf[20];
+	int len;
+
+	spin_lock_bh(&ar->data_lock);
+
+	len = scnprintf(buf, sizeof(buf), "%d\n", arsta->peer_ps_state);
+
+	spin_unlock_bh(&ar->data_lock);
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
+}
+
+static const struct file_operations fops_peer_ps_state = {
+	.open = simple_open,
+	.read = ath11k_dbg_sta_read_peer_ps_state,
+	.owner = THIS_MODULE,
+	.llseek = default_llseek,
+};
+
+static ssize_t ath11k_dbg_sta_read_current_ps_duration(struct file *file,
+						       char __user *user_buf,
+						       size_t count,
+						       loff_t *ppos)
+{
+	struct ieee80211_sta *sta = file->private_data;
+	struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv;
+	struct ath11k *ar = arsta->arvif->ar;
+	u64 time_since_station_in_power_save;
+	char buf[20];
+	int len;
+
+	spin_lock_bh(&ar->data_lock);
+
+	if (arsta->peer_ps_state == WMI_PEER_PS_STATE_ON &&
+	    arsta->peer_current_ps_valid)
+		time_since_station_in_power_save = jiffies_to_msecs(jiffies
+						- arsta->ps_start_jiffies);
+	else
+		time_since_station_in_power_save = 0;
+
+	len = scnprintf(buf, sizeof(buf), "%llu\n",
+			time_since_station_in_power_save);
+	spin_unlock_bh(&ar->data_lock);
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
+}
+
+static const struct file_operations fops_current_ps_duration = {
+	.open = simple_open,
+	.read = ath11k_dbg_sta_read_current_ps_duration,
+	.owner = THIS_MODULE,
+	.llseek = default_llseek,
+};
+
+static ssize_t ath11k_dbg_sta_read_total_ps_duration(struct file *file,
+						     char __user *user_buf,
+						     size_t count, loff_t *ppos)
+{
+	struct ieee80211_sta *sta = file->private_data;
+	struct ath11k_sta *arsta = (struct ath11k_sta *)sta->drv_priv;
+	struct ath11k *ar = arsta->arvif->ar;
+	char buf[20];
+	u64 power_save_duration;
+	int len;
+
+	spin_lock_bh(&ar->data_lock);
+
+	if (arsta->peer_ps_state == WMI_PEER_PS_STATE_ON &&
+	    arsta->peer_current_ps_valid)
+		power_save_duration = jiffies_to_msecs(jiffies
+						- arsta->ps_start_jiffies)
+						+ arsta->ps_total_duration;
+	else
+		power_save_duration = arsta->ps_total_duration;
+
+	len = scnprintf(buf, sizeof(buf), "%llu\n", power_save_duration);
+
+	spin_unlock_bh(&ar->data_lock);
+
+	return simple_read_from_buffer(user_buf, count, ppos, buf, len);
+}
+
+static const struct file_operations fops_total_ps_duration = {
+	.open = simple_open,
+	.read = ath11k_dbg_sta_read_total_ps_duration,
+	.owner = THIS_MODULE,
+	.llseek = default_llseek,
+};
+
 void ath11k_debugfs_sta_op_add(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			       struct ieee80211_sta *sta, struct dentry *dir)
 {
@@ -778,4 +874,15 @@ void ath11k_debugfs_sta_op_add(struct ieee80211_hw *hw, struct ieee80211_vif *vi
 		     ar->ab->wmi_ab.svc_map))
 		debugfs_create_file("htt_peer_stats_reset", 0600, dir, sta,
 				    &fops_htt_peer_stats_reset);
+
+	debugfs_create_file("peer_ps_state", 0400, dir, sta,
+			    &fops_peer_ps_state);
+
+	if (test_bit(WMI_TLV_SERVICE_PEER_POWER_SAVE_DURATION_SUPPORT,
+		     ar->ab->wmi_ab.svc_map)) {
+		debugfs_create_file("current_ps_duration", 0440, dir, sta,
+				    &fops_current_ps_duration);
+		debugfs_create_file("total_ps_duration", 0440, dir, sta,
+				    &fops_total_ps_duration);
+	}
 }
diff --git a/drivers/net/wireless/ath/ath11k/dp.c b/drivers/net/wireless/ath/ath11k/dp.c
index 8b790ce72e5d..f5156a7fbdd7 100644
--- a/drivers/net/wireless/ath/ath11k/dp.c
+++ b/drivers/net/wireless/ath/ath11k/dp.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include <crypto/hash.h>
@@ -131,13 +132,11 @@ static int ath11k_dp_srng_calculate_msi_group(struct ath11k_base *ab,
 
 	switch (type) {
 	case HAL_WBM2SW_RELEASE:
-		if (ring_num < 3) {
-			grp_mask = &ab->hw_params.ring_mask->tx[0];
-		} else if (ring_num == 3) {
+		if (ring_num == DP_RX_RELEASE_RING_NUM) {
 			grp_mask = &ab->hw_params.ring_mask->rx_wbm_rel[0];
 			ring_num = 0;
 		} else {
-			return -ENOENT;
+			grp_mask = &ab->hw_params.ring_mask->tx[0];
 		}
 		break;
 	case HAL_REO_EXCEPTION:
@@ -371,6 +370,7 @@ static int ath11k_dp_srng_common_setup(struct ath11k_base *ab)
 	struct ath11k_dp *dp = &ab->dp;
 	struct hal_srng *srng;
 	int i, ret;
+	u8 tcl_num, wbm_num;
 
 	ret = ath11k_dp_srng_setup(ab, &dp->wbm_desc_rel_ring,
 				   HAL_SW2WBM_RELEASE, 0, 0,
@@ -396,9 +396,12 @@ static int ath11k_dp_srng_common_setup(struct ath11k_base *ab)
 	}
 
 	for (i = 0; i < ab->hw_params.max_tx_ring; i++) {
+		tcl_num = ab->hw_params.hal_params->tcl2wbm_rbm_map[i].tcl_ring_num;
+		wbm_num = ab->hw_params.hal_params->tcl2wbm_rbm_map[i].wbm_ring_num;
+
 		ret = ath11k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_data_ring,
-					   HAL_TCL_DATA, i, 0,
-					   DP_TCL_DATA_RING_SIZE);
+					   HAL_TCL_DATA, tcl_num, 0,
+					   ab->hw_params.tx_ring_size);
 		if (ret) {
 			ath11k_warn(ab, "failed to set up tcl_data ring (%d) :%d\n",
 				    i, ret);
@@ -406,7 +409,7 @@ static int ath11k_dp_srng_common_setup(struct ath11k_base *ab)
 		}
 
 		ret = ath11k_dp_srng_setup(ab, &dp->tx_ring[i].tcl_comp_ring,
-					   HAL_WBM2SW_RELEASE, i, 0,
+					   HAL_WBM2SW_RELEASE, wbm_num, 0,
 					   DP_TX_COMP_RING_SIZE);
 		if (ret) {
 			ath11k_warn(ab, "failed to set up tcl_comp ring (%d) :%d\n",
@@ -431,7 +434,7 @@ static int ath11k_dp_srng_common_setup(struct ath11k_base *ab)
 	}
 
 	ret = ath11k_dp_srng_setup(ab, &dp->rx_rel_ring, HAL_WBM2SW_RELEASE,
-				   3, 0, DP_RX_RELEASE_RING_SIZE);
+				   DP_RX_RELEASE_RING_NUM, 0, DP_RX_RELEASE_RING_SIZE);
 	if (ret) {
 		ath11k_warn(ab, "failed to set up rx_rel ring :%d\n", ret);
 		goto err;
@@ -774,9 +777,10 @@ int ath11k_dp_service_srng(struct ath11k_base *ab,
 	int i, j;
 	int tot_work_done = 0;
 
-	if (ab->hw_params.ring_mask->tx[grp_id]) {
-		i = __fls(ab->hw_params.ring_mask->tx[grp_id]);
-		ath11k_dp_tx_completion_handler(ab, i);
+	for (i = 0; i < ab->hw_params.max_tx_ring; i++) {
+		if (BIT(ab->hw_params.hal_params->tcl2wbm_rbm_map[i].wbm_ring_num) &
+		    ab->hw_params.ring_mask->tx[grp_id])
+			ath11k_dp_tx_completion_handler(ab, i);
 	}
 
 	if (ab->hw_params.ring_mask->rx_err[grp_id]) {
@@ -963,7 +967,7 @@ static void ath11k_dp_update_vdev_search(struct ath11k_vif *arvif)
 {
 	 /* When v2_map_support is true:for STA mode, enable address
 	  * search index, tcl uses ast_hash value in the descriptor.
-	  * When v2_map_support is false: for STA mode, dont' enable
+	  * When v2_map_support is false: for STA mode, don't enable
 	  * address search index.
 	  */
 	switch (arvif->vdev_type) {
diff --git a/drivers/net/wireless/ath/ath11k/dp.h b/drivers/net/wireless/ath/ath11k/dp.h
index e9dfa209098b..be9eafc872b3 100644
--- a/drivers/net/wireless/ath/ath11k/dp.h
+++ b/drivers/net/wireless/ath/ath11k/dp.h
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifndef ATH11K_DP_H
@@ -203,6 +204,7 @@ struct ath11k_pdev_dp {
 
 #define DP_WBM_RELEASE_RING_SIZE	64
 #define DP_TCL_DATA_RING_SIZE		512
+#define DP_TCL_DATA_RING_SIZE_WCN6750	2048
 #define DP_TX_COMP_RING_SIZE		32768
 #define DP_TX_IDR_SIZE			DP_TX_COMP_RING_SIZE
 #define DP_TCL_CMD_RING_SIZE		32
@@ -222,6 +224,8 @@ struct ath11k_pdev_dp {
 #define DP_RXDMA_MONITOR_DST_RING_SIZE	2048
 #define DP_RXDMA_MONITOR_DESC_RING_SIZE	4096
 
+#define DP_RX_RELEASE_RING_NUM	3
+
 #define DP_RX_BUFFER_SIZE	2048
 #define	DP_RX_BUFFER_SIZE_LITE  1024
 #define DP_RX_BUFFER_ALIGN_SIZE	128
@@ -299,7 +303,7 @@ struct ath11k_dp {
 
 #define HTT_TX_WBM_COMP_STATUS_OFFSET 8
 
-/* HTT tx completion is overlayed in wbm_release_ring */
+/* HTT tx completion is overlaid in wbm_release_ring */
 #define HTT_TX_WBM_COMP_INFO0_STATUS		GENMASK(12, 9)
 #define HTT_TX_WBM_COMP_INFO0_REINJECT_REASON	GENMASK(16, 13)
 #define HTT_TX_WBM_COMP_INFO0_REINJECT_REASON	GENMASK(16, 13)
@@ -466,7 +470,7 @@ enum htt_srng_ring_id {
  *                     3'b010: 4 usec
  *                     3'b011: 8 usec (default)
  *                     3'b100: 16 usec
- *                     Others: Reserverd
+ *                     Others: Reserved
  *           b'19    - response_required:
  *                     Host needs HTT_T2H_MSG_TYPE_SRING_SETUP_DONE as response
  *           b'20:31 - reserved:  reserved for future use
@@ -993,8 +997,7 @@ struct htt_rx_ring_tlv_filter {
 #define HTT_RX_FULL_MON_MODE_CFG_CMD_CFG_NON_ZERO_MPDUS_END	BIT(2)
 #define HTT_RX_FULL_MON_MODE_CFG_CMD_CFG_RELEASE_RING		GENMASK(10, 3)
 
-/**
- * Enumeration for full monitor mode destination ring select
+/* Enumeration for full monitor mode destination ring select
  * 0 - REO destination ring select
  * 1 - FW destination ring select
  * 2 - SW destination ring select
@@ -1391,8 +1394,7 @@ struct htt_ppdu_stats_info {
 	struct list_head list;
 };
 
-/**
- * @brief target -> host packet log message
+/* @brief target -> host packet log message
  *
  * @details
  * The following field definitions describe the format of the packet log
@@ -1430,8 +1432,7 @@ struct htt_pktlog_msg {
 	u8 payload[];
 };
 
-/**
- * @brief host -> target FW extended statistics retrieve
+/* @brief host -> target FW extended statistics retrieve
  *
  * @details
  * The following field definitions describe the format of the HTT host
@@ -1566,8 +1567,7 @@ struct htt_ext_stats_cfg_params {
 	u32 cfg3;
 };
 
-/**
- * @brief target -> host extended statistics upload
+/* @brief target -> host extended statistics upload
  *
  * @details
  * The following field definitions describe the format of the HTT target
diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c
index 2148acf37071..c5a4c34d7749 100644
--- a/drivers/net/wireless/ath/ath11k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath11k/dp_rx.c
@@ -2499,7 +2499,7 @@ static void ath11k_dp_rx_deliver_msdu(struct ath11k *ar, struct napi_struct *nap
 
 	/* PN for multicast packets are not validate in HW,
 	 * so skip 802.3 rx path
-	 * Also, fast_rx expectes the STA to be authorized, hence
+	 * Also, fast_rx expects the STA to be authorized, hence
 	 * eapol packets are sent in slow path.
 	 */
 	if (decap == DP_RX_DECAP_TYPE_ETHERNET2_DIX && !is_eapol &&
@@ -5197,7 +5197,8 @@ int ath11k_dp_rx_process_mon_status(struct ath11k_base *ab, int mac_id,
 		if (log_type != ATH11K_PKTLOG_TYPE_INVALID)
 			trace_ath11k_htt_rxdesc(ar, skb->data, log_type, rx_buf_sz);
 
-		memset(ppdu_info, 0, sizeof(struct hal_rx_mon_ppdu_info));
+		memset(ppdu_info, 0, sizeof(*ppdu_info));
+		ppdu_info->peer_id = HAL_INVALID_PEERID;
 		hal_status = ath11k_hal_rx_parse_mon_status(ab, ppdu_info, skb);
 
 		if (test_bit(ATH11K_FLAG_MONITOR_STARTED, &ar->monitor_flags) &&
diff --git a/drivers/net/wireless/ath/ath11k/dp_tx.c b/drivers/net/wireless/ath/ath11k/dp_tx.c
index c17a2620aad7..8afbba236935 100644
--- a/drivers/net/wireless/ath/ath11k/dp_tx.c
+++ b/drivers/net/wireless/ath/ath11k/dp_tx.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include "core.h"
@@ -93,7 +94,8 @@ int ath11k_dp_tx(struct ath11k *ar, struct ath11k_vif *arvif,
 	u8 pool_id;
 	u8 hal_ring_id;
 	int ret;
-	u8 ring_selector = 0, ring_map = 0;
+	u32 ring_selector = 0;
+	u8 ring_map = 0;
 	bool tcl_ring_retry;
 
 	if (unlikely(test_bit(ATH11K_FLAG_CRASH_FLUSH, &ar->ab->dev_flags)))
@@ -105,19 +107,13 @@ int ath11k_dp_tx(struct ath11k *ar, struct ath11k_vif *arvif,
 
 	pool_id = skb_get_queue_mapping(skb) & (ATH11K_HW_MAX_QUEUES - 1);
 
-	/* Let the default ring selection be based on current processor
-	 * number, where one of the 3 tcl rings are selected based on
-	 * the smp_processor_id(). In case that ring
-	 * is full/busy, we resort to other available rings.
-	 * If all rings are full, we drop the packet.
-	 * //TODO Add throttling logic when all rings are full
-	 */
-	ring_selector = smp_processor_id();
+	ring_selector = ab->hw_params.hw_ops->get_ring_selector(skb);
 
 tcl_ring_sel:
 	tcl_ring_retry = false;
 
 	ti.ring_id = ring_selector % ab->hw_params.max_tx_ring;
+	ti.rbm_id = ab->hw_params.hal_params->tcl2wbm_rbm_map[ti.ring_id].rbm_id;
 
 	ring_map |= BIT(ti.ring_id);
 
@@ -129,7 +125,8 @@ tcl_ring_sel:
 	spin_unlock_bh(&tx_ring->tx_idr_lock);
 
 	if (unlikely(ret < 0)) {
-		if (ring_map == (BIT(ab->hw_params.max_tx_ring) - 1)) {
+		if (ring_map == (BIT(ab->hw_params.max_tx_ring) - 1) ||
+		    !ab->hw_params.tcl_ring_retry) {
 			atomic_inc(&ab->soc_stats.tx_err.misc_fail);
 			return -ENOSPC;
 		}
@@ -247,7 +244,7 @@ tcl_ring_sel:
 		 * Restart ring selection if some rings are not checked yet.
 		 */
 		if (unlikely(ring_map != (BIT(ab->hw_params.max_tx_ring)) - 1) &&
-		    ab->hw_params.max_tx_ring > 1) {
+		    ab->hw_params.tcl_ring_retry && ab->hw_params.max_tx_ring > 1) {
 			tcl_ring_retry = true;
 			ring_selector++;
 		}
@@ -755,7 +752,7 @@ int ath11k_dp_tx_send_reo_cmd(struct ath11k_base *ab, struct dp_rx_tid *rx_tid,
 		return 0;
 
 	/* Can this be optimized so that we keep the pending command list only
-	 * for tid delete command to free up the resoruce on the command status
+	 * for tid delete command to free up the resource on the command status
 	 * indication?
 	 */
 	dp_cmd = kzalloc(sizeof(*dp_cmd), GFP_ATOMIC);
diff --git a/drivers/net/wireless/ath/ath11k/hal.c b/drivers/net/wireless/ath/ath11k/hal.c
index bda71ab5a1f2..2fd224480d45 100644
--- a/drivers/net/wireless/ath/ath11k/hal.c
+++ b/drivers/net/wireless/ath/ath11k/hal.c
@@ -126,7 +126,7 @@ static const struct hal_srng_config hw_srng_config_template[] = {
 	},
 	{ /* WBM2SW_RELEASE */
 		.start_ring_id = HAL_SRNG_RING_ID_WBM2SW0_RELEASE,
-		.max_rings = 4,
+		.max_rings = 5,
 		.entry_size = sizeof(struct hal_wbm_release_ring) >> 2,
 		.lmac_ring = false,
 		.ring_dir = HAL_SRNG_DIR_DST,
@@ -1164,7 +1164,7 @@ void ath11k_hal_srng_shadow_update_hp_tp(struct ath11k_base *ab,
 {
 	lockdep_assert_held(&srng->lock);
 
-	/* check whether the ring is emptry. Update the shadow
+	/* check whether the ring is empty. Update the shadow
 	 * HP only when then ring isn't empty.
 	 */
 	if (srng->ring_dir == HAL_SRNG_DIR_SRC &&
diff --git a/drivers/net/wireless/ath/ath11k/hal.h b/drivers/net/wireless/ath/ath11k/hal.h
index 110c337ddf33..6a1f78ee6eb6 100644
--- a/drivers/net/wireless/ath/ath11k/hal.h
+++ b/drivers/net/wireless/ath/ath11k/hal.h
@@ -243,7 +243,7 @@ struct ath11k_base;
 #define HAL_WBM0_RELEASE_RING_HP		0x000030c0
 #define HAL_WBM1_RELEASE_RING_HP		0x000030c8
 
-/* TCL ring feild mask and offset */
+/* TCL ring field mask and offset */
 #define HAL_TCL1_RING_BASE_MSB_RING_SIZE		GENMASK(27, 8)
 #define HAL_TCL1_RING_BASE_MSB_RING_BASE_ADDR_MSB	GENMASK(7, 0)
 #define HAL_TCL1_RING_ID_ENTRY_SIZE			GENMASK(7, 0)
@@ -268,7 +268,7 @@ struct ath11k_base;
 #define HAL_TCL1_RING_FIELD_DSCP_TID_MAP6		GENMASK(20, 18)
 #define HAL_TCL1_RING_FIELD_DSCP_TID_MAP7		GENMASK(23, 21)
 
-/* REO ring feild mask and offset */
+/* REO ring field mask and offset */
 #define HAL_REO1_RING_BASE_MSB_RING_SIZE		GENMASK(27, 8)
 #define HAL_REO1_RING_BASE_MSB_RING_BASE_ADDR_MSB	GENMASK(7, 0)
 #define HAL_REO1_RING_ID_RING_ID			GENMASK(15, 8)
@@ -389,6 +389,7 @@ enum hal_srng_ring_id {
 	HAL_SRNG_RING_ID_WBM2SW1_RELEASE,
 	HAL_SRNG_RING_ID_WBM2SW2_RELEASE,
 	HAL_SRNG_RING_ID_WBM2SW3_RELEASE,
+	HAL_SRNG_RING_ID_WBM2SW4_RELEASE,
 
 	HAL_SRNG_RING_ID_UMAC_ID_END = 127,
 	HAL_SRNG_RING_ID_LMAC1_ID_START,
@@ -450,13 +451,13 @@ enum hal_ring_type {
 
 /**
  * enum hal_reo_cmd_type: Enum for REO command type
- * @CMD_GET_QUEUE_STATS: Get REO queue status/stats
- * @CMD_FLUSH_QUEUE: Flush all frames in REO queue
- * @CMD_FLUSH_CACHE: Flush descriptor entries in the cache
- * @CMD_UNBLOCK_CACHE: Unblock a descriptor's address that was blocked
+ * @HAL_REO_CMD_GET_QUEUE_STATS: Get REO queue status/stats
+ * @HAL_REO_CMD_FLUSH_QUEUE: Flush all frames in REO queue
+ * @HAL_REO_CMD_FLUSH_CACHE: Flush descriptor entries in the cache
+ * @HAL_REO_CMD_UNBLOCK_CACHE: Unblock a descriptor's address that was blocked
  *      earlier with a 'REO_FLUSH_CACHE' command
- * @CMD_FLUSH_TIMEOUT_LIST: Flush buffers/descriptors from timeout list
- * @CMD_UPDATE_RX_REO_QUEUE: Update REO queue settings
+ * @HAL_REO_CMD_FLUSH_TIMEOUT_LIST: Flush buffers/descriptors from timeout list
+ * @HAL_REO_CMD_UPDATE_RX_QUEUE: Update REO queue settings
  */
 enum hal_reo_cmd_type {
 	HAL_REO_CMD_GET_QUEUE_STATS     = 0,
@@ -635,7 +636,7 @@ struct hal_srng {
 	} u;
 };
 
-/* Interrupt mitigation - Batch threshold in terms of numer of frames */
+/* Interrupt mitigation - Batch threshold in terms of number of frames */
 #define HAL_SRNG_INT_BATCH_THRESHOLD_TX 256
 #define HAL_SRNG_INT_BATCH_THRESHOLD_RX 128
 #define HAL_SRNG_INT_BATCH_THRESHOLD_OTHER 1
@@ -678,6 +679,7 @@ enum hal_rx_buf_return_buf_manager {
 	HAL_RX_BUF_RBM_SW1_BM,
 	HAL_RX_BUF_RBM_SW2_BM,
 	HAL_RX_BUF_RBM_SW3_BM,
+	HAL_RX_BUF_RBM_SW4_BM,
 };
 
 #define HAL_SRNG_DESC_LOOP_CNT		0xf0000000
@@ -873,8 +875,7 @@ struct hal_reo_status {
 	} u;
 };
 
-/**
- * HAL context to be used to access SRNG APIs (currently used by data path
+/* HAL context to be used to access SRNG APIs (currently used by data path
  * and transport (CE) modules)
  */
 struct ath11k_hal {
diff --git a/drivers/net/wireless/ath/ath11k/hal_desc.h b/drivers/net/wireless/ath/ath11k/hal_desc.h
index 24e72e75a8c7..d895ea878d9f 100644
--- a/drivers/net/wireless/ath/ath11k/hal_desc.h
+++ b/drivers/net/wireless/ath/ath11k/hal_desc.h
@@ -607,7 +607,7 @@ struct rx_msdu_desc {
  *
  * msdu_continuation
  *		When set, this MSDU buffer was not able to hold the entire MSDU.
- *		The next buffer will therefor contain additional information
+ *		The next buffer will therefore contain additional information
  *		related to this MSDU.
  *
  * msdu_length
@@ -643,7 +643,7 @@ struct rx_msdu_desc {
  *
  * da_idx_timeout
  *		Indicates, an unsuccessful MAC destination address search due
- *		to the expiration of search timer fot this MSDU.
+ *		to the expiration of search timer for this MSDU.
  */
 
 enum hal_reo_dest_ring_buffer_type {
@@ -1678,7 +1678,7 @@ struct hal_wbm_release_ring {
  *	Producer: SW/TQM/RXDMA/REO/SWITCH
  *	Consumer: WBM/SW/FW
  *
- * HTT tx status is overlayed on wbm_release ring on 4-byte words 2, 3, 4 and 5
+ * HTT tx status is overlaid on wbm_release ring on 4-byte words 2, 3, 4 and 5
  * for software based completions.
  *
  * buf_addr_info
@@ -2159,7 +2159,7 @@ struct hal_reo_status_hdr {
  *		commands.
  *
  * execution_time (in us)
- *		The amount of time REO took to excecute the command. Note that
+ *		The amount of time REO took to execute the command. Note that
  *		this time does not include the duration of the command waiting
  *		in the command ring, before the execution started.
  *
diff --git a/drivers/net/wireless/ath/ath11k/hal_tx.c b/drivers/net/wireless/ath/ath11k/hal_tx.c
index c8929de8ce6c..d1b0e36e04a9 100644
--- a/drivers/net/wireless/ath/ath11k/hal_tx.c
+++ b/drivers/net/wireless/ath/ath11k/hal_tx.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include "hal_desc.h"
@@ -44,8 +45,7 @@ void ath11k_hal_tx_cmd_desc_setup(struct ath11k_base *ab, void *cmd,
 		FIELD_PREP(BUFFER_ADDR_INFO1_ADDR,
 			   ((uint64_t)ti->paddr >> HAL_ADDR_MSB_REG_SHIFT));
 	tcl_cmd->buf_addr_info.info1 |=
-		FIELD_PREP(BUFFER_ADDR_INFO1_RET_BUF_MGR,
-			   (ti->ring_id + HAL_RX_BUF_RBM_SW0_BM)) |
+		FIELD_PREP(BUFFER_ADDR_INFO1_RET_BUF_MGR, ti->rbm_id) |
 		FIELD_PREP(BUFFER_ADDR_INFO1_SW_COOKIE, ti->desc_id);
 
 	tcl_cmd->info0 =
diff --git a/drivers/net/wireless/ath/ath11k/hal_tx.h b/drivers/net/wireless/ath/ath11k/hal_tx.h
index 36f4f6f6cbc2..c5e88364afe5 100644
--- a/drivers/net/wireless/ath/ath11k/hal_tx.h
+++ b/drivers/net/wireless/ath/ath11k/hal_tx.h
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifndef ATH11K_HAL_TX_H
@@ -35,6 +36,7 @@ struct hal_tx_info {
 	u8 lmac_id;
 	u8 dscp_tid_tbl_idx;
 	bool enable_mesh;
+	u8 rbm_id;
 };
 
 /* TODO: Check if the actual desc macros can be used instead */
diff --git a/drivers/net/wireless/ath/ath11k/hif.h b/drivers/net/wireless/ath/ath11k/hif.h
index e9366f786fbb..659b80d2abd4 100644
--- a/drivers/net/wireless/ath/ath11k/hif.h
+++ b/drivers/net/wireless/ath/ath11k/hif.h
@@ -11,6 +11,7 @@
 struct ath11k_hif_ops {
 	u32 (*read32)(struct ath11k_base *sc, u32 address);
 	void (*write32)(struct ath11k_base *sc, u32 address, u32 data);
+	int (*read)(struct ath11k_base *ab, void *buf, u32 start, u32 end);
 	void (*irq_enable)(struct ath11k_base *sc);
 	void (*irq_disable)(struct ath11k_base *sc);
 	int (*start)(struct ath11k_base *sc);
@@ -99,6 +100,15 @@ static inline void ath11k_hif_write32(struct ath11k_base *sc, u32 address, u32 d
 	sc->hif.ops->write32(sc, address, data);
 }
 
+static inline int ath11k_hif_read(struct ath11k_base *ab, void *buf,
+				  u32 start, u32 end)
+{
+	if (!ab->hif.ops->read)
+		return -EOPNOTSUPP;
+
+	return ab->hif.ops->read(ab, buf, start, end);
+}
+
 static inline int ath11k_hif_map_service_to_pipe(struct ath11k_base *sc, u16 service_id,
 						 u8 *ul_pipe, u8 *dl_pipe)
 {
@@ -134,4 +144,5 @@ static inline void ath11k_get_ce_msi_idx(struct ath11k_base *ab, u32 ce_id,
 	else
 		*msi_data_idx = ce_id;
 }
+
 #endif /* _HIF_H_ */
diff --git a/drivers/net/wireless/ath/ath11k/hw.c b/drivers/net/wireless/ath/ath11k/hw.c
index 96db85c55585..dbcc0c4035b6 100644
--- a/drivers/net/wireless/ath/ath11k/hw.c
+++ b/drivers/net/wireless/ath/ath11k/hw.c
@@ -820,6 +820,30 @@ static bool ath11k_hw_wcn6855_rx_desc_get_ldpc_support(struct hal_rx_desc *desc)
 			 __le32_to_cpu(desc->u.wcn6855.msdu_start.info2));
 }
 
+static u32 ath11k_hw_ipq8074_get_tcl_ring_selector(struct sk_buff *skb)
+{
+	/* Let the default ring selection be based on current processor
+	 * number, where one of the 3 tcl rings are selected based on
+	 * the smp_processor_id(). In case that ring
+	 * is full/busy, we resort to other available rings.
+	 * If all rings are full, we drop the packet.
+	 *
+	 * TODO: Add throttling logic when all rings are full
+	 */
+	return smp_processor_id();
+}
+
+static u32 ath11k_hw_wcn6750_get_tcl_ring_selector(struct sk_buff *skb)
+{
+	/* Select the TCL ring based on the flow hash of the SKB instead
+	 * of CPU ID. Since applications pumping the traffic can be scheduled
+	 * on multiple CPUs, there is a chance that packets of the same flow
+	 * could end on different TCL rings, this could sometimes results in
+	 * an out of order arrival of the packets at the receiver.
+	 */
+	return skb_get_hash(skb);
+}
+
 const struct ath11k_hw_ops ipq8074_ops = {
 	.get_hw_mac_from_pdev_id = ath11k_hw_ipq8074_mac_from_pdev_id,
 	.wmi_init_config = ath11k_init_wmi_config_ipq8074,
@@ -857,6 +881,7 @@ const struct ath11k_hw_ops ipq8074_ops = {
 	.mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid,
 	.rx_desc_mac_addr2_valid = ath11k_hw_ipq8074_rx_desc_mac_addr2_valid,
 	.rx_desc_mpdu_start_addr2 = ath11k_hw_ipq8074_rx_desc_mpdu_start_addr2,
+	.get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector,
 };
 
 const struct ath11k_hw_ops ipq6018_ops = {
@@ -896,6 +921,7 @@ const struct ath11k_hw_ops ipq6018_ops = {
 	.mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid,
 	.rx_desc_mac_addr2_valid = ath11k_hw_ipq8074_rx_desc_mac_addr2_valid,
 	.rx_desc_mpdu_start_addr2 = ath11k_hw_ipq8074_rx_desc_mpdu_start_addr2,
+	.get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector,
 };
 
 const struct ath11k_hw_ops qca6390_ops = {
@@ -935,6 +961,7 @@ const struct ath11k_hw_ops qca6390_ops = {
 	.mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid,
 	.rx_desc_mac_addr2_valid = ath11k_hw_ipq8074_rx_desc_mac_addr2_valid,
 	.rx_desc_mpdu_start_addr2 = ath11k_hw_ipq8074_rx_desc_mpdu_start_addr2,
+	.get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector,
 };
 
 const struct ath11k_hw_ops qcn9074_ops = {
@@ -974,6 +1001,7 @@ const struct ath11k_hw_ops qcn9074_ops = {
 	.mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid,
 	.rx_desc_mac_addr2_valid = ath11k_hw_ipq9074_rx_desc_mac_addr2_valid,
 	.rx_desc_mpdu_start_addr2 = ath11k_hw_ipq9074_rx_desc_mpdu_start_addr2,
+	.get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector,
 };
 
 const struct ath11k_hw_ops wcn6855_ops = {
@@ -1013,6 +1041,7 @@ const struct ath11k_hw_ops wcn6855_ops = {
 	.mpdu_info_get_peerid = ath11k_hw_wcn6855_mpdu_info_get_peerid,
 	.rx_desc_mac_addr2_valid = ath11k_hw_wcn6855_rx_desc_mac_addr2_valid,
 	.rx_desc_mpdu_start_addr2 = ath11k_hw_wcn6855_rx_desc_mpdu_start_addr2,
+	.get_ring_selector = ath11k_hw_ipq8074_get_tcl_ring_selector,
 };
 
 const struct ath11k_hw_ops wcn6750_ops = {
@@ -1052,11 +1081,14 @@ const struct ath11k_hw_ops wcn6750_ops = {
 	.mpdu_info_get_peerid = ath11k_hw_ipq8074_mpdu_info_get_peerid,
 	.rx_desc_mac_addr2_valid = ath11k_hw_ipq9074_rx_desc_mac_addr2_valid,
 	.rx_desc_mpdu_start_addr2 = ath11k_hw_ipq9074_rx_desc_mpdu_start_addr2,
+	.get_ring_selector = ath11k_hw_wcn6750_get_tcl_ring_selector,
 };
 
-#define ATH11K_TX_RING_MASK_0 0x1
-#define ATH11K_TX_RING_MASK_1 0x2
-#define ATH11K_TX_RING_MASK_2 0x4
+#define ATH11K_TX_RING_MASK_0 BIT(0)
+#define ATH11K_TX_RING_MASK_1 BIT(1)
+#define ATH11K_TX_RING_MASK_2 BIT(2)
+#define ATH11K_TX_RING_MASK_3 BIT(3)
+#define ATH11K_TX_RING_MASK_4 BIT(4)
 
 #define ATH11K_RX_RING_MASK_0 0x1
 #define ATH11K_RX_RING_MASK_1 0x2
@@ -1903,6 +1935,43 @@ const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_qcn9074 = {
 	},
 };
 
+const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_wcn6750 = {
+	.tx  = {
+		ATH11K_TX_RING_MASK_0,
+		0,
+		ATH11K_TX_RING_MASK_2,
+		0,
+		ATH11K_TX_RING_MASK_4,
+	},
+	.rx_mon_status = {
+		0, 0, 0, 0, 0, 0,
+		ATH11K_RX_MON_STATUS_RING_MASK_0,
+	},
+	.rx = {
+		0, 0, 0, 0, 0, 0, 0,
+		ATH11K_RX_RING_MASK_0,
+		ATH11K_RX_RING_MASK_1,
+		ATH11K_RX_RING_MASK_2,
+		ATH11K_RX_RING_MASK_3,
+	},
+	.rx_err = {
+		0, ATH11K_RX_ERR_RING_MASK_0,
+	},
+	.rx_wbm_rel = {
+		0, ATH11K_RX_WBM_REL_RING_MASK_0,
+	},
+	.reo_status = {
+		0, ATH11K_REO_STATUS_RING_MASK_0,
+	},
+	.rxdma2host = {
+		ATH11K_RXDMA2HOST_RING_MASK_0,
+		ATH11K_RXDMA2HOST_RING_MASK_1,
+		ATH11K_RXDMA2HOST_RING_MASK_2,
+	},
+	.host2rxdma = {
+	},
+};
+
 const struct ath11k_hw_regs ipq8074_regs = {
 	/* SW2TCL(x) R0 ring configuration address */
 	.hal_tcl1_ring_base_lsb = 0x00000510,
@@ -2332,12 +2401,55 @@ const struct ath11k_hw_regs wcn6750_regs = {
 	.hal_reo1_misc_ctl = 0x000005d8,
 };
 
+static const struct ath11k_hw_tcl2wbm_rbm_map ath11k_hw_tcl2wbm_rbm_map_ipq8074[] = {
+	{
+		.tcl_ring_num = 0,
+		.wbm_ring_num = 0,
+		.rbm_id = HAL_RX_BUF_RBM_SW0_BM,
+	},
+	{
+		.tcl_ring_num = 1,
+		.wbm_ring_num = 1,
+		.rbm_id = HAL_RX_BUF_RBM_SW1_BM,
+	},
+	{
+		.tcl_ring_num = 2,
+		.wbm_ring_num = 2,
+		.rbm_id = HAL_RX_BUF_RBM_SW2_BM,
+	},
+};
+
+static const struct ath11k_hw_tcl2wbm_rbm_map ath11k_hw_tcl2wbm_rbm_map_wcn6750[] = {
+	{
+		.tcl_ring_num = 0,
+		.wbm_ring_num = 0,
+		.rbm_id = HAL_RX_BUF_RBM_SW0_BM,
+	},
+	{
+		.tcl_ring_num = 1,
+		.wbm_ring_num = 4,
+		.rbm_id = HAL_RX_BUF_RBM_SW4_BM,
+	},
+	{
+		.tcl_ring_num = 2,
+		.wbm_ring_num = 2,
+		.rbm_id = HAL_RX_BUF_RBM_SW2_BM,
+	},
+};
+
 const struct ath11k_hw_hal_params ath11k_hw_hal_params_ipq8074 = {
 	.rx_buf_rbm = HAL_RX_BUF_RBM_SW3_BM,
+	.tcl2wbm_rbm_map = ath11k_hw_tcl2wbm_rbm_map_ipq8074,
 };
 
 const struct ath11k_hw_hal_params ath11k_hw_hal_params_qca6390 = {
 	.rx_buf_rbm = HAL_RX_BUF_RBM_SW1_BM,
+	.tcl2wbm_rbm_map = ath11k_hw_tcl2wbm_rbm_map_ipq8074,
+};
+
+const struct ath11k_hw_hal_params ath11k_hw_hal_params_wcn6750 = {
+	.rx_buf_rbm = HAL_RX_BUF_RBM_SW1_BM,
+	.tcl2wbm_rbm_map = ath11k_hw_tcl2wbm_rbm_map_wcn6750,
 };
 
 static const struct cfg80211_sar_freq_ranges ath11k_hw_sar_freq_ranges_wcn6855[] = {
diff --git a/drivers/net/wireless/ath/ath11k/hw.h b/drivers/net/wireless/ath/ath11k/hw.h
index bb5ac940e470..8a3f24862edc 100644
--- a/drivers/net/wireless/ath/ath11k/hw.h
+++ b/drivers/net/wireless/ath/ath11k/hw.h
@@ -122,8 +122,15 @@ struct ath11k_hw_ring_mask {
 	u8 host2rxdma[ATH11K_EXT_IRQ_GRP_NUM_MAX];
 };
 
+struct ath11k_hw_tcl2wbm_rbm_map {
+	u8 tcl_ring_num;
+	u8 wbm_ring_num;
+	u8 rbm_id;
+};
+
 struct ath11k_hw_hal_params {
 	enum hal_rx_buf_return_buf_manager rx_buf_rbm;
+	const struct ath11k_hw_tcl2wbm_rbm_map *tcl2wbm_rbm_map;
 };
 
 struct ath11k_hw_params {
@@ -166,6 +173,7 @@ struct ath11k_hw_params {
 		u8 summary_pad_sz;
 		u8 fft_hdr_len;
 		u16 max_fft_bins;
+		bool fragment_160mhz;
 	} spectral;
 
 	u16 interface_modes;
@@ -175,6 +183,7 @@ struct ath11k_hw_params {
 	bool idle_ps;
 	bool supports_sta_ps;
 	bool cold_boot_calib;
+	bool cbcal_restart_fw;
 	int fw_mem_mode;
 	u32 num_vdevs;
 	u32 num_peers;
@@ -200,6 +209,16 @@ struct ath11k_hw_params {
 	bool hybrid_bus_type;
 	bool fixed_fw_mem;
 	bool support_off_channel_tx;
+	bool supports_multi_bssid;
+
+	struct {
+		u32 start;
+		u32 end;
+	} sram_dump;
+
+	bool tcl_ring_retry;
+	u32 tx_ring_size;
+	bool smp2p_wow_exit;
 };
 
 struct ath11k_hw_ops {
@@ -242,6 +261,7 @@ struct ath11k_hw_ops {
 	u16 (*mpdu_info_get_peerid)(u8 *tlv_data);
 	bool (*rx_desc_mac_addr2_valid)(struct hal_rx_desc *desc);
 	u8* (*rx_desc_mpdu_start_addr2)(struct hal_rx_desc *desc);
+	u32 (*get_ring_selector)(struct sk_buff *skb);
 };
 
 extern const struct ath11k_hw_ops ipq8074_ops;
@@ -254,9 +274,11 @@ extern const struct ath11k_hw_ops wcn6750_ops;
 extern const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_ipq8074;
 extern const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_qca6390;
 extern const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_qcn9074;
+extern const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_wcn6750;
 
 extern const struct ath11k_hw_hal_params ath11k_hw_hal_params_ipq8074;
 extern const struct ath11k_hw_hal_params ath11k_hw_hal_params_qca6390;
+extern const struct ath11k_hw_hal_params ath11k_hw_hal_params_wcn6750;
 
 static inline
 int ath11k_hw_get_mac_from_pdev_id(struct ath11k_hw_params *hw,
@@ -397,4 +419,5 @@ static inline const char *ath11k_bd_ie_type_str(enum ath11k_bd_ie_type type)
 }
 
 extern const struct cfg80211_sar_capa ath11k_hw_sar_capa_wcn6855;
+
 #endif
diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c
index 7e91e347c9ff..84d956ad4093 100644
--- a/drivers/net/wireless/ath/ath11k/mac.c
+++ b/drivers/net/wireless/ath/ath11k/mac.c
@@ -3059,7 +3059,7 @@ static int ath11k_mac_config_obss_pd(struct ath11k *ar,
 		return ret;
 	}
 
-	/* Enable all patial BSSID mask for SRG */
+	/* Enable all partial BSSID mask for SRG */
 	ret = ath11k_wmi_pdev_srg_obss_bssid_enable_bitmap(ar, bitmap);
 	if (ret) {
 		ath11k_warn(ar->ab,
@@ -3077,7 +3077,7 @@ static int ath11k_mac_config_obss_pd(struct ath11k *ar,
 		return ret;
 	}
 
-	/* Enable all patial BSSID mask for non-SRG */
+	/* Enable all partial BSSID mask for non-SRG */
 	ret = ath11k_wmi_pdev_non_srg_obss_bssid_enable_bitmap(ar, bitmap);
 	if (ret) {
 		ath11k_warn(ar->ab,
@@ -3350,10 +3350,15 @@ static void ath11k_mac_op_bss_info_changed(struct ieee80211_hw *hw,
 		ath11k_recalculate_mgmt_rate(ar, vif, &def);
 
 	if (changed & BSS_CHANGED_TWT) {
-		if (info->twt_requester || info->twt_responder)
-			ath11k_wmi_send_twt_enable_cmd(ar, ar->pdev->pdev_id);
-		else
+		struct wmi_twt_enable_params twt_params = {0};
+
+		if (info->twt_requester || info->twt_responder) {
+			ath11k_wmi_fill_default_twt_params(&twt_params);
+			ath11k_wmi_send_twt_enable_cmd(ar, ar->pdev->pdev_id,
+						       &twt_params);
+		} else {
 			ath11k_wmi_send_twt_disable_cmd(ar, ar->pdev->pdev_id);
+		}
 	}
 
 	if (changed & BSS_CHANGED_HE_OBSS_PD)
@@ -3451,7 +3456,7 @@ void __ath11k_mac_scan_finish(struct ath11k *ar)
 		ar->scan_channel = NULL;
 		ar->scan.roc_freq = 0;
 		cancel_delayed_work(&ar->scan.timeout);
-		complete(&ar->scan.completed);
+		complete_all(&ar->scan.completed);
 		break;
 	}
 }
@@ -4524,6 +4529,7 @@ static int ath11k_mac_op_sta_state(struct ieee80211_hw *hw,
 	    new_state == IEEE80211_STA_NONE) {
 		memset(arsta, 0, sizeof(*arsta));
 		arsta->arvif = arvif;
+		arsta->peer_ps_state = WMI_PEER_PS_STATE_DISABLED;
 		INIT_WORK(&arsta->update_wk, ath11k_sta_rc_update_wk);
 		INIT_WORK(&arsta->set_4addr_wk, ath11k_sta_set_4addr_wk);
 
@@ -4701,7 +4707,7 @@ static void ath11k_mac_op_sta_rc_update(struct ieee80211_hw *hw,
 		   "mac sta rc update for %pM changed %08x bw %d nss %d smps %d\n",
 		   sta->addr, changed, sta->deflink.bandwidth,
 		   sta->deflink.rx_nss,
-		   sta->smps_mode);
+		   sta->deflink.smps_mode);
 
 	spin_lock_bh(&ar->data_lock);
 
@@ -4737,7 +4743,7 @@ static void ath11k_mac_op_sta_rc_update(struct ieee80211_hw *hw,
 	if (changed & IEEE80211_RC_SMPS_CHANGED) {
 		smps = WMI_PEER_SMPS_PS_NONE;
 
-		switch (sta->smps_mode) {
+		switch (sta->deflink.smps_mode) {
 		case IEEE80211_SMPS_AUTOMATIC:
 		case IEEE80211_SMPS_OFF:
 			smps = WMI_PEER_SMPS_PS_NONE;
@@ -4750,7 +4756,7 @@ static void ath11k_mac_op_sta_rc_update(struct ieee80211_hw *hw,
 			break;
 		default:
 			ath11k_warn(ar->ab, "Invalid smps %d in sta rc update for %pM\n",
-				    sta->smps_mode, sta->addr);
+				    sta->deflink.smps_mode, sta->addr);
 			smps = WMI_PEER_SMPS_PS_NONE;
 			break;
 		}
@@ -4954,6 +4960,8 @@ static int ath11k_mac_set_txbf_conf(struct ath11k_vif *arvif)
 	if (vht_cap & (IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE)) {
 		nsts = vht_cap & IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK;
 		nsts >>= IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT;
+		if (nsts > (ar->num_rx_chains - 1))
+			nsts = ar->num_rx_chains - 1;
 		value |= SM(nsts, WMI_TXBF_STS_CAP_OFFSET);
 	}
 
@@ -4994,7 +5002,7 @@ static int ath11k_mac_set_txbf_conf(struct ath11k_vif *arvif)
 static void ath11k_set_vht_txbf_cap(struct ath11k *ar, u32 *vht_cap)
 {
 	bool subfer, subfee;
-	int sound_dim = 0;
+	int sound_dim = 0, nsts = 0;
 
 	subfer = !!(*vht_cap & (IEEE80211_VHT_CAP_SU_BEAMFORMER_CAPABLE));
 	subfee = !!(*vht_cap & (IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE));
@@ -5004,6 +5012,11 @@ static void ath11k_set_vht_txbf_cap(struct ath11k *ar, u32 *vht_cap)
 		subfer = false;
 	}
 
+	if (ar->num_rx_chains < 2) {
+		*vht_cap &= ~(IEEE80211_VHT_CAP_SU_BEAMFORMEE_CAPABLE);
+		subfee = false;
+	}
+
 	/* If SU Beaformer is not set, then disable MU Beamformer Capability */
 	if (!subfer)
 		*vht_cap &= ~(IEEE80211_VHT_CAP_MU_BEAMFORMER_CAPABLE);
@@ -5016,7 +5029,9 @@ static void ath11k_set_vht_txbf_cap(struct ath11k *ar, u32 *vht_cap)
 	sound_dim >>= IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_SHIFT;
 	*vht_cap &= ~IEEE80211_VHT_CAP_SOUNDING_DIMENSIONS_MASK;
 
-	/* TODO: Need to check invalid STS and Sound_dim values set by FW? */
+	nsts = (*vht_cap & IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK);
+	nsts >>= IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT;
+	*vht_cap &= ~IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK;
 
 	/* Enable Sounding Dimension Field only if SU BF is enabled */
 	if (subfer) {
@@ -5028,9 +5043,15 @@ static void ath11k_set_vht_txbf_cap(struct ath11k *ar, u32 *vht_cap)
 		*vht_cap |= sound_dim;
 	}
 
-	/* Use the STS advertised by FW unless SU Beamformee is not supported*/
-	if (!subfee)
-		*vht_cap &= ~(IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK);
+	/* Enable Beamformee STS Field only if SU BF is enabled */
+	if (subfee) {
+		if (nsts > (ar->num_rx_chains - 1))
+			nsts = ar->num_rx_chains - 1;
+
+		nsts <<= IEEE80211_VHT_CAP_BEAMFORMEE_STS_SHIFT;
+		nsts &=  IEEE80211_VHT_CAP_BEAMFORMEE_STS_MASK;
+		*vht_cap |= nsts;
+	}
 }
 
 static struct ieee80211_sta_vht_cap
@@ -6173,6 +6194,13 @@ static int ath11k_mac_op_add_interface(struct ieee80211_hw *hw,
 		goto err;
 	}
 
+	/* In the case of hardware recovery, debugfs files are
+	 * not deleted since ieee80211_ops.remove_interface() is
+	 * not invoked. In such cases, try to delete the files.
+	 * These will be re-created later.
+	 */
+	ath11k_debugfs_remove_interface(arvif);
+
 	memset(arvif, 0, sizeof(*arvif));
 
 	arvif->ar = ar;
@@ -6354,9 +6382,7 @@ static int ath11k_mac_op_add_interface(struct ieee80211_hw *hw,
 		}
 	}
 
-	ret = ath11k_debugfs_add_interface(arvif);
-	if (ret)
-		goto err_peer_del;
+	ath11k_debugfs_add_interface(arvif);
 
 	mutex_unlock(&ar->conf_mutex);
 
@@ -8421,6 +8447,95 @@ exit:
 	return ret;
 }
 
+static int ath11k_fw_stats_request(struct ath11k *ar,
+				   struct stats_request_params *req_param)
+{
+	struct ath11k_base *ab = ar->ab;
+	unsigned long time_left;
+	int ret;
+
+	lockdep_assert_held(&ar->conf_mutex);
+
+	spin_lock_bh(&ar->data_lock);
+	ar->fw_stats_done = false;
+	ath11k_fw_stats_pdevs_free(&ar->fw_stats.pdevs);
+	spin_unlock_bh(&ar->data_lock);
+
+	reinit_completion(&ar->fw_stats_complete);
+
+	ret = ath11k_wmi_send_stats_request_cmd(ar, req_param);
+	if (ret) {
+		ath11k_warn(ab, "could not request fw stats (%d)\n",
+			    ret);
+		return ret;
+	}
+
+	time_left = wait_for_completion_timeout(&ar->fw_stats_complete,
+						1 * HZ);
+
+	if (!time_left)
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+static int ath11k_mac_op_get_txpower(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     int *dbm)
+{
+	struct ath11k *ar = hw->priv;
+	struct ath11k_base *ab = ar->ab;
+	struct stats_request_params req_param = {0};
+	struct ath11k_fw_stats_pdev *pdev;
+	int ret;
+
+	/* Final Tx power is minimum of Target Power, CTL power, Regulatory
+	 * Power, PSD EIRP Power. We just know the Regulatory power from the
+	 * regulatory rules obtained. FW knows all these power and sets the min
+	 * of these. Hence, we request the FW pdev stats in which FW reports
+	 * the minimum of all vdev's channel Tx power.
+	 */
+	mutex_lock(&ar->conf_mutex);
+
+	if (ar->state != ATH11K_STATE_ON)
+		goto err_fallback;
+
+	req_param.pdev_id = ar->pdev->pdev_id;
+	req_param.stats_id = WMI_REQUEST_PDEV_STAT;
+
+	ret = ath11k_fw_stats_request(ar, &req_param);
+	if (ret) {
+		ath11k_warn(ab, "failed to request fw pdev stats: %d\n", ret);
+		goto err_fallback;
+	}
+
+	spin_lock_bh(&ar->data_lock);
+	pdev = list_first_entry_or_null(&ar->fw_stats.pdevs,
+					struct ath11k_fw_stats_pdev, list);
+	if (!pdev) {
+		spin_unlock_bh(&ar->data_lock);
+		goto err_fallback;
+	}
+
+	/* tx power is set as 2 units per dBm in FW. */
+	*dbm = pdev->chan_tx_power / 2;
+
+	spin_unlock_bh(&ar->data_lock);
+	mutex_unlock(&ar->conf_mutex);
+
+	ath11k_dbg(ar->ab, ATH11K_DBG_MAC, "txpower from firmware %d, reported %d dBm\n",
+		   pdev->chan_tx_power, *dbm);
+	return 0;
+
+err_fallback:
+	mutex_unlock(&ar->conf_mutex);
+	/* We didn't get txpower from FW. Hence, relying on vif->bss_conf.txpower */
+	*dbm = vif->bss_conf.txpower;
+	ath11k_dbg(ar->ab, ATH11K_DBG_MAC, "txpower from firmware NaN, reported %d dBm\n",
+		   *dbm);
+	return 0;
+}
+
 static const struct ieee80211_ops ath11k_ops = {
 	.tx				= ath11k_mac_op_tx,
 	.start                          = ath11k_mac_op_start,
@@ -8471,6 +8586,7 @@ static const struct ieee80211_ops ath11k_ops = {
 #if IS_ENABLED(CONFIG_IPV6)
 	.ipv6_addr_change = ath11k_mac_op_ipv6_changed,
 #endif
+	.get_txpower                    = ath11k_mac_op_get_txpower,
 
 	.set_sar_specs			= ath11k_mac_op_set_bios_sar_specs,
 	.remain_on_channel		= ath11k_mac_op_remain_on_channel,
@@ -8777,6 +8893,11 @@ static int __ath11k_mac_register(struct ath11k *ar)
 	if (ab->hw_params.single_pdev_only && ar->supports_6ghz)
 		ieee80211_hw_set(ar->hw, SINGLE_SCAN_ON_ALL_BANDS);
 
+	if (ab->hw_params.supports_multi_bssid) {
+		ieee80211_hw_set(ar->hw, SUPPORTS_MULTI_BSSID);
+		ieee80211_hw_set(ar->hw, SUPPORTS_ONLY_HE_MULTI_BSSID);
+	}
+
 	ieee80211_hw_set(ar->hw, SIGNAL_DBM);
 	ieee80211_hw_set(ar->hw, SUPPORTS_PS);
 	ieee80211_hw_set(ar->hw, SUPPORTS_DYNAMIC_PS);
@@ -8967,6 +9088,7 @@ int ath11k_mac_register(struct ath11k_base *ab)
 	struct ath11k_pdev *pdev;
 	int i;
 	int ret;
+	u8 mac_addr[ETH_ALEN] = {0};
 
 	if (test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags))
 		return 0;
@@ -8979,13 +9101,18 @@ int ath11k_mac_register(struct ath11k_base *ab)
 	if (ret)
 		return ret;
 
+	device_get_mac_address(ab->dev, mac_addr);
+
 	for (i = 0; i < ab->num_radios; i++) {
 		pdev = &ab->pdevs[i];
 		ar = pdev->ar;
 		if (ab->pdevs_macaddr_valid) {
 			ether_addr_copy(ar->mac_addr, pdev->mac_addr);
 		} else {
-			ether_addr_copy(ar->mac_addr, ab->mac_addr);
+			if (is_zero_ether_addr(mac_addr))
+				ether_addr_copy(ar->mac_addr, ab->mac_addr);
+			else
+				ether_addr_copy(ar->mac_addr, mac_addr);
 			ar->mac_addr[4] += i;
 		}
 
@@ -9079,6 +9206,8 @@ int ath11k_mac_allocate(struct ath11k_base *ab)
 		clear_bit(ATH11K_FLAG_MONITOR_VDEV_CREATED, &ar->monitor_flags);
 		ar->vdev_id_11d_scan = ATH11K_11D_INVALID_VDEV_ID;
 		init_completion(&ar->completed_11d_scan);
+
+		ath11k_fw_stats_init(ar);
 	}
 
 	return 0;
diff --git a/drivers/net/wireless/ath/ath11k/mhi.c b/drivers/net/wireless/ath/ath11k/mhi.c
index c44df17719f6..86995e8dc913 100644
--- a/drivers/net/wireless/ath/ath11k/mhi.c
+++ b/drivers/net/wireless/ath/ath11k/mhi.c
@@ -402,8 +402,7 @@ int ath11k_mhi_register(struct ath11k_pci *ab_pci)
 	ret = ath11k_mhi_get_msi(ab_pci);
 	if (ret) {
 		ath11k_err(ab, "failed to get msi for mhi\n");
-		mhi_free_controller(mhi_ctrl);
-		return ret;
+		goto free_controller;
 	}
 
 	if (!test_bit(ATH11K_FLAG_MULTI_MSI_VECTORS, &ab->dev_flags))
@@ -412,7 +411,7 @@ int ath11k_mhi_register(struct ath11k_pci *ab_pci)
 	if (test_bit(ATH11K_FLAG_FIXED_MEM_RGN, &ab->dev_flags)) {
 		ret = ath11k_mhi_read_addr_from_dt(mhi_ctrl);
 		if (ret < 0)
-			return ret;
+			goto free_controller;
 	} else {
 		mhi_ctrl->iova_start = 0;
 		mhi_ctrl->iova_stop = 0xFFFFFFFF;
@@ -440,18 +439,22 @@ int ath11k_mhi_register(struct ath11k_pci *ab_pci)
 	default:
 		ath11k_err(ab, "failed assign mhi_config for unknown hw rev %d\n",
 			   ab->hw_rev);
-		mhi_free_controller(mhi_ctrl);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto free_controller;
 	}
 
 	ret = mhi_register_controller(mhi_ctrl, ath11k_mhi_config);
 	if (ret) {
 		ath11k_err(ab, "failed to register to mhi bus, err = %d\n", ret);
-		mhi_free_controller(mhi_ctrl);
-		return ret;
+		goto free_controller;
 	}
 
 	return 0;
+
+free_controller:
+	mhi_free_controller(mhi_ctrl);
+	ab_pci->mhi_ctrl = NULL;
+	return ret;
 }
 
 void ath11k_mhi_unregister(struct ath11k_pci *ab_pci)
diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c
index 5bd34a6273d9..99cf3357c66e 100644
--- a/drivers/net/wireless/ath/ath11k/pci.c
+++ b/drivers/net/wireless/ath/ath11k/pci.c
@@ -685,6 +685,7 @@ static const struct ath11k_hif_ops ath11k_pci_hif_ops = {
 	.stop = ath11k_pcic_stop,
 	.read32 = ath11k_pcic_read32,
 	.write32 = ath11k_pcic_write32,
+	.read = ath11k_pcic_read,
 	.power_down = ath11k_pci_power_down,
 	.power_up = ath11k_pci_power_up,
 	.suspend = ath11k_pci_hif_suspend,
diff --git a/drivers/net/wireless/ath/ath11k/pcic.c b/drivers/net/wireless/ath/ath11k/pcic.c
index 1adf20ebef27..380f9d37b644 100644
--- a/drivers/net/wireless/ath/ath11k/pcic.c
+++ b/drivers/net/wireless/ath/ath11k/pcic.c
@@ -140,55 +140,100 @@ int ath11k_pcic_init_msi_config(struct ath11k_base *ab)
 }
 EXPORT_SYMBOL(ath11k_pcic_init_msi_config);
 
+static void __ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value)
+{
+	if (offset < ATH11K_PCI_WINDOW_START)
+		iowrite32(value, ab->mem  + offset);
+	else
+		ab->pci.ops->window_write32(ab, offset, value);
+}
+
 void ath11k_pcic_write32(struct ath11k_base *ab, u32 offset, u32 value)
 {
 	int ret = 0;
+	bool wakeup_required;
 
 	/* for offset beyond BAR + 4K - 32, may
 	 * need to wakeup the device to access.
 	 */
-	if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
-	    offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->wakeup)
+	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
+			  offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
+	if (wakeup_required && ab->pci.ops->wakeup)
 		ret = ab->pci.ops->wakeup(ab);
 
-	if (offset < ATH11K_PCI_WINDOW_START)
-		iowrite32(value, ab->mem  + offset);
-	else
-		ab->pci.ops->window_write32(ab, offset, value);
+	__ath11k_pcic_write32(ab, offset, value);
 
-	if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
-	    offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->release &&
-	    !ret)
+	if (wakeup_required && !ret && ab->pci.ops->release)
 		ab->pci.ops->release(ab);
 }
 EXPORT_SYMBOL(ath11k_pcic_write32);
 
+static u32 __ath11k_pcic_read32(struct ath11k_base *ab, u32 offset)
+{
+	u32 val;
+
+	if (offset < ATH11K_PCI_WINDOW_START)
+		val = ioread32(ab->mem + offset);
+	else
+		val = ab->pci.ops->window_read32(ab, offset);
+
+	return val;
+}
+
 u32 ath11k_pcic_read32(struct ath11k_base *ab, u32 offset)
 {
 	int ret = 0;
 	u32 val;
+	bool wakeup_required;
 
 	/* for offset beyond BAR + 4K - 32, may
 	 * need to wakeup the device to access.
 	 */
-	if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
-	    offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->wakeup)
+	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
+			  offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
+	if (wakeup_required && ab->pci.ops->wakeup)
 		ret = ab->pci.ops->wakeup(ab);
 
-	if (offset < ATH11K_PCI_WINDOW_START)
-		val = ioread32(ab->mem + offset);
-	else
-		val = ab->pci.ops->window_read32(ab, offset);
+	val = __ath11k_pcic_read32(ab, offset);
 
-	if (test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
-	    offset >= ATH11K_PCI_ACCESS_ALWAYS_OFF && ab->pci.ops->release &&
-	    !ret)
+	if (wakeup_required && !ret && ab->pci.ops->release)
 		ab->pci.ops->release(ab);
 
 	return val;
 }
 EXPORT_SYMBOL(ath11k_pcic_read32);
 
+int ath11k_pcic_read(struct ath11k_base *ab, void *buf, u32 start, u32 end)
+{
+	int ret = 0;
+	bool wakeup_required;
+	u32 *data = buf;
+	u32 i;
+
+	/* for offset beyond BAR + 4K - 32, may
+	 * need to wakeup the device to access.
+	 */
+	wakeup_required = test_bit(ATH11K_FLAG_DEVICE_INIT_DONE, &ab->dev_flags) &&
+			  end >= ATH11K_PCI_ACCESS_ALWAYS_OFF;
+	if (wakeup_required && ab->pci.ops->wakeup) {
+		ret = ab->pci.ops->wakeup(ab);
+		if (ret) {
+			ath11k_warn(ab, "failed to wakeup for read from 0x%x: %d\n",
+				    start, ret);
+			return ret;
+		}
+	}
+
+	for (i = start; i < end + 1; i += 4)
+		*data++ = __ath11k_pcic_read32(ab, i);
+
+	if (wakeup_required && ab->pci.ops->release)
+		ab->pci.ops->release(ab);
+
+	return 0;
+}
+EXPORT_SYMBOL(ath11k_pcic_read);
+
 void ath11k_pcic_get_msi_address(struct ath11k_base *ab, u32 *msi_addr_lo,
 				 u32 *msi_addr_hi)
 {
@@ -414,6 +459,7 @@ void ath11k_pcic_ext_irq_enable(struct ath11k_base *ab)
 		struct ath11k_ext_irq_grp *irq_grp = &ab->ext_irq_grp[i];
 
 		if (!irq_grp->napi_enabled) {
+			dev_set_threaded(&irq_grp->napi_ndev, true);
 			napi_enable(&irq_grp->napi);
 			irq_grp->napi_enabled = true;
 		}
@@ -517,7 +563,7 @@ static int ath11k_pcic_ext_irq_config(struct ath11k_base *ab)
 		irq_grp->grp_id = i;
 		init_dummy_netdev(&irq_grp->napi_ndev);
 		netif_napi_add(&irq_grp->napi_ndev, &irq_grp->napi,
-			       ath11k_pcic_ext_grp_napi_poll, NAPI_POLL_WEIGHT);
+			       ath11k_pcic_ext_grp_napi_poll);
 
 		if (ab->hw_params.ring_mask->tx[i] ||
 		    ab->hw_params.ring_mask->rx[i] ||
@@ -731,3 +777,37 @@ int ath11k_pcic_register_pci_ops(struct ath11k_base *ab,
 	return 0;
 }
 EXPORT_SYMBOL(ath11k_pcic_register_pci_ops);
+
+void ath11k_pci_enable_ce_irqs_except_wake_irq(struct ath11k_base *ab)
+{
+	int i;
+
+	for (i = 0; i < ab->hw_params.ce_count; i++) {
+		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR ||
+		    i == ATH11K_PCI_CE_WAKE_IRQ)
+			continue;
+		ath11k_pcic_ce_irq_enable(ab, i);
+	}
+}
+EXPORT_SYMBOL(ath11k_pci_enable_ce_irqs_except_wake_irq);
+
+void ath11k_pci_disable_ce_irqs_except_wake_irq(struct ath11k_base *ab)
+{
+	int i;
+	int irq_idx;
+	struct ath11k_ce_pipe *ce_pipe;
+
+	for (i = 0; i < ab->hw_params.ce_count; i++) {
+		ce_pipe = &ab->ce.ce_pipe[i];
+		irq_idx = ATH11K_PCI_IRQ_CE0_OFFSET + i;
+
+		if (ath11k_ce_get_attr_flags(ab, i) & CE_ATTR_DIS_INTR ||
+		    i == ATH11K_PCI_CE_WAKE_IRQ)
+			continue;
+
+		disable_irq_nosync(ab->irq_num[irq_idx]);
+		synchronize_irq(ab->irq_num[irq_idx]);
+		tasklet_kill(&ce_pipe->intr_tq);
+	}
+}
+EXPORT_SYMBOL(ath11k_pci_disable_ce_irqs_except_wake_irq);
diff --git a/drivers/net/wireless/ath/ath11k/pcic.h b/drivers/net/wireless/ath/ath11k/pcic.h
index 0afbb34510db..ac012e88bf6d 100644
--- a/drivers/net/wireless/ath/ath11k/pcic.h
+++ b/drivers/net/wireless/ath/ath11k/pcic.h
@@ -12,6 +12,8 @@
 #define ATH11K_PCI_IRQ_CE0_OFFSET	3
 #define ATH11K_PCI_IRQ_DP_OFFSET	14
 
+#define ATH11K_PCI_CE_WAKE_IRQ	2
+
 #define ATH11K_PCI_WINDOW_ENABLE_BIT		0x40000000
 #define ATH11K_PCI_WINDOW_REG_ADDRESS		0x310c
 #define ATH11K_PCI_WINDOW_VALUE_MASK		GENMASK(24, 19)
@@ -45,4 +47,8 @@ void ath11k_pcic_ce_irq_disable_sync(struct ath11k_base *ab);
 int ath11k_pcic_init_msi_config(struct ath11k_base *ab);
 int ath11k_pcic_register_pci_ops(struct ath11k_base *ab,
 				 const struct ath11k_pci_ops *pci_ops);
+int ath11k_pcic_read(struct ath11k_base *ab, void *buf, u32 start, u32 end);
+void ath11k_pci_enable_ce_irqs_except_wake_irq(struct ath11k_base *ab);
+void ath11k_pci_disable_ce_irqs_except_wake_irq(struct ath11k_base *ab);
+
 #endif
diff --git a/drivers/net/wireless/ath/ath11k/peer.c b/drivers/net/wireless/ath/ath11k/peer.c
index 9e22aaf34b88..1ae7af02c364 100644
--- a/drivers/net/wireless/ath/ath11k/peer.c
+++ b/drivers/net/wireless/ath/ath11k/peer.c
@@ -302,6 +302,21 @@ static int __ath11k_peer_delete(struct ath11k *ar, u32 vdev_id, const u8 *addr)
 	spin_lock_bh(&ab->base_lock);
 
 	peer = ath11k_peer_find_by_addr(ab, addr);
+	/* Check if the found peer is what we want to remove.
+	 * While the sta is transitioning to another band we may
+	 * have 2 peer with the same addr assigned to different
+	 * vdev_id. Make sure we are deleting the correct peer.
+	 */
+	if (peer && peer->vdev_id == vdev_id)
+		ath11k_peer_rhash_delete(ab, peer);
+
+	/* Fallback to peer list search if the correct peer can't be found.
+	 * Skip the deletion of the peer from the rhash since it has already
+	 * been deleted in peer add.
+	 */
+	if (!peer)
+		peer = ath11k_peer_find(ab, vdev_id, addr);
+
 	if (!peer) {
 		spin_unlock_bh(&ab->base_lock);
 		mutex_unlock(&ab->tbl_mtx_lock);
@@ -312,8 +327,6 @@ static int __ath11k_peer_delete(struct ath11k *ar, u32 vdev_id, const u8 *addr)
 		return -EINVAL;
 	}
 
-	ath11k_peer_rhash_delete(ab, peer);
-
 	spin_unlock_bh(&ab->base_lock);
 	mutex_unlock(&ab->tbl_mtx_lock);
 
@@ -372,8 +385,17 @@ int ath11k_peer_create(struct ath11k *ar, struct ath11k_vif *arvif,
 	spin_lock_bh(&ar->ab->base_lock);
 	peer = ath11k_peer_find_by_addr(ar->ab, param->peer_addr);
 	if (peer) {
-		spin_unlock_bh(&ar->ab->base_lock);
-		return -EINVAL;
+		if (peer->vdev_id == param->vdev_id) {
+			spin_unlock_bh(&ar->ab->base_lock);
+			return -EINVAL;
+		}
+
+		/* Assume sta is transitioning to another band.
+		 * Remove here the peer from rhash.
+		 */
+		mutex_lock(&ar->ab->tbl_mtx_lock);
+		ath11k_peer_rhash_delete(ar->ab, peer);
+		mutex_unlock(&ar->ab->tbl_mtx_lock);
 	}
 	spin_unlock_bh(&ar->ab->base_lock);
 
diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c
index 00136601cb7d..51de2208b789 100644
--- a/drivers/net/wireless/ath/ath11k/qmi.c
+++ b/drivers/net/wireless/ath/ath11k/qmi.c
@@ -1696,6 +1696,13 @@ static struct qmi_elem_info qmi_wlanfw_wlan_ini_resp_msg_v01_ei[] = {
 	},
 };
 
+static struct qmi_elem_info qmi_wlfw_fw_init_done_ind_msg_v01_ei[] = {
+	{
+		.data_type = QMI_EOTI,
+		.array_type = NO_ARRAY,
+	},
+};
+
 static int ath11k_qmi_host_cap_send(struct ath11k_base *ab)
 {
 	struct qmi_wlanfw_host_cap_req_msg_v01 req;
@@ -1872,7 +1879,7 @@ static int ath11k_qmi_respond_fw_mem_request(struct ath11k_base *ab)
 
 	/* For QCA6390 by default FW requests a block of ~4M contiguous
 	 * DMA memory, it's hard to allocate from OS. So host returns
-	 * failure to FW and FW will then request mulitple blocks of small
+	 * failure to FW and FW will then request multiple blocks of small
 	 * chunk size memory.
 	 */
 	if (!(ab->hw_params.fixed_mem_region ||
@@ -3006,6 +3013,12 @@ static void ath11k_qmi_msg_fw_ready_cb(struct qmi_handle *qmi_hdl,
 	struct ath11k_base *ab = qmi->ab;
 
 	ath11k_dbg(ab, ATH11K_DBG_QMI, "qmi firmware ready\n");
+
+	if (!ab->qmi.cal_done) {
+		ab->qmi.cal_done = 1;
+		wake_up(&ab->qmi.cold_boot_waitq);
+	}
+
 	ath11k_qmi_driver_event_post(qmi, ATH11K_QMI_EVENT_FW_READY, NULL);
 }
 
@@ -3023,6 +3036,19 @@ static void ath11k_qmi_msg_cold_boot_cal_done_cb(struct qmi_handle *qmi_hdl,
 	ath11k_dbg(ab, ATH11K_DBG_QMI, "qmi cold boot calibration done\n");
 }
 
+static void ath11k_qmi_msg_fw_init_done_cb(struct qmi_handle *qmi_hdl,
+					   struct sockaddr_qrtr *sq,
+					   struct qmi_txn *txn,
+					   const void *decoded)
+{
+	struct ath11k_qmi *qmi = container_of(qmi_hdl,
+					      struct ath11k_qmi, handle);
+	struct ath11k_base *ab = qmi->ab;
+
+	ath11k_qmi_driver_event_post(qmi, ATH11K_QMI_EVENT_FW_INIT_DONE, NULL);
+	ath11k_dbg(ab, ATH11K_DBG_QMI, "qmi firmware init done\n");
+}
+
 static const struct qmi_msg_handler ath11k_qmi_msg_handlers[] = {
 	{
 		.type = QMI_INDICATION,
@@ -3053,6 +3079,14 @@ static const struct qmi_msg_handler ath11k_qmi_msg_handlers[] = {
 			sizeof(struct qmi_wlanfw_fw_cold_cal_done_ind_msg_v01),
 		.fn = ath11k_qmi_msg_cold_boot_cal_done_cb,
 	},
+	{
+		.type = QMI_INDICATION,
+		.msg_id = QMI_WLFW_FW_INIT_DONE_IND_V01,
+		.ei = qmi_wlfw_fw_init_done_ind_msg_v01_ei,
+		.decoded_size =
+			sizeof(struct qmi_wlfw_fw_init_done_ind_msg_v01),
+		.fn = ath11k_qmi_msg_fw_init_done_cb,
+	},
 };
 
 static int ath11k_qmi_ops_new_server(struct qmi_handle *qmi_hdl,
@@ -3145,7 +3179,7 @@ static void ath11k_qmi_driver_event_work(struct work_struct *work)
 			}
 
 			break;
-		case ATH11K_QMI_EVENT_FW_READY:
+		case ATH11K_QMI_EVENT_FW_INIT_DONE:
 			clear_bit(ATH11K_FLAG_QMI_FAIL, &ab->dev_flags);
 			if (test_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags)) {
 				ath11k_hal_dump_srng_stats(ab);
@@ -3169,6 +3203,22 @@ static void ath11k_qmi_driver_event_work(struct work_struct *work)
 			}
 
 			break;
+		case ATH11K_QMI_EVENT_FW_READY:
+			/* For targets requiring a FW restart upon cold
+			 * boot completion, there is no need to process
+			 * FW ready; such targets will receive FW init
+			 * done message after FW restart.
+			 */
+			if (ab->hw_params.cbcal_restart_fw)
+				break;
+
+			clear_bit(ATH11K_FLAG_CRASH_FLUSH,
+				  &ab->dev_flags);
+			clear_bit(ATH11K_FLAG_RECOVERY, &ab->dev_flags);
+			ath11k_core_qmi_firmware_ready(ab);
+			set_bit(ATH11K_FLAG_REGISTERED, &ab->dev_flags);
+
+			break;
 		case ATH11K_QMI_EVENT_COLD_BOOT_CAL_DONE:
 			break;
 		default:
diff --git a/drivers/net/wireless/ath/ath11k/qmi.h b/drivers/net/wireless/ath/ath11k/qmi.h
index c83cf822be81..2ec56a34fa81 100644
--- a/drivers/net/wireless/ath/ath11k/qmi.h
+++ b/drivers/net/wireless/ath/ath11k/qmi.h
@@ -31,8 +31,9 @@
 
 #define QMI_WLFW_REQUEST_MEM_IND_V01		0x0035
 #define QMI_WLFW_FW_MEM_READY_IND_V01		0x0037
-#define QMI_WLFW_COLD_BOOT_CAL_DONE_IND_V01	0x0021
-#define QMI_WLFW_FW_READY_IND_V01		0x0038
+#define QMI_WLFW_COLD_BOOT_CAL_DONE_IND_V01	0x003E
+#define QMI_WLFW_FW_READY_IND_V01		0x0021
+#define QMI_WLFW_FW_INIT_DONE_IND_V01		0x0038
 
 #define QMI_WLANFW_MAX_DATA_SIZE_V01		6144
 #define ATH11K_FIRMWARE_MODE_OFF		4
@@ -69,6 +70,7 @@ enum ath11k_qmi_event_type {
 	ATH11K_QMI_EVENT_FORCE_FW_ASSERT,
 	ATH11K_QMI_EVENT_POWER_UP,
 	ATH11K_QMI_EVENT_POWER_DOWN,
+	ATH11K_QMI_EVENT_FW_INIT_DONE,
 	ATH11K_QMI_EVENT_MAX,
 };
 
@@ -291,6 +293,10 @@ struct qmi_wlanfw_fw_cold_cal_done_ind_msg_v01 {
 	char placeholder;
 };
 
+struct qmi_wlfw_fw_init_done_ind_msg_v01 {
+	char placeholder;
+};
+
 #define QMI_WLANFW_CAP_REQ_MSG_V01_MAX_LEN		0
 #define QMI_WLANFW_CAP_RESP_MSG_V01_MAX_LEN		235
 #define QMI_WLANFW_CAP_REQ_V01				0x0024
diff --git a/drivers/net/wireless/ath/ath11k/rx_desc.h b/drivers/net/wireless/ath/ath11k/rx_desc.h
index 26ecc1bcd9d5..786d5f36f5e5 100644
--- a/drivers/net/wireless/ath/ath11k/rx_desc.h
+++ b/drivers/net/wireless/ath/ath11k/rx_desc.h
@@ -877,7 +877,7 @@ struct rx_msdu_start_wcn6855 {
  *
  * l4_offset
  *		Depending upon mode bit, this field either indicates the
- *		L4 offset nin bytes from the start of RX_HEADER (only valid
+ *		L4 offset in bytes from the start of RX_HEADER (only valid
  *		if either ipv4_proto or ipv6_proto is set to 1) or indicates
  *		the offset in bytes to the start of TCP or UDP header from
  *		the start of the IP header after decapsulation (Only valid if
diff --git a/drivers/net/wireless/ath/ath11k/spectral.c b/drivers/net/wireless/ath/ath11k/spectral.c
index 516a7b4cd180..705868198df4 100644
--- a/drivers/net/wireless/ath/ath11k/spectral.c
+++ b/drivers/net/wireless/ath/ath11k/spectral.c
@@ -30,6 +30,7 @@
 #define ATH11K_SPECTRAL_20MHZ			20
 #define ATH11K_SPECTRAL_40MHZ			40
 #define ATH11K_SPECTRAL_80MHZ			80
+#define ATH11K_SPECTRAL_160MHZ			160
 
 #define ATH11K_SPECTRAL_SIGNATURE		0xFA
 
@@ -183,6 +184,8 @@ static int ath11k_spectral_scan_trigger(struct ath11k *ar)
 	if (ar->spectral.mode == ATH11K_SPECTRAL_DISABLED)
 		return 0;
 
+	ar->spectral.is_primary = true;
+
 	ret = ath11k_wmi_vdev_spectral_enable(ar, arvif->vdev_id,
 					      ATH11K_WMI_SPECTRAL_TRIGGER_CMD_CLEAR,
 					      ATH11K_WMI_SPECTRAL_ENABLE_CMD_ENABLE);
@@ -585,6 +588,7 @@ int ath11k_spectral_process_fft(struct ath11k *ar,
 	u8 chan_width_mhz, bin_sz;
 	int ret;
 	u32 check_length;
+	bool fragment_sample = false;
 
 	lockdep_assert_held(&ar->spectral.lock);
 
@@ -639,6 +643,13 @@ int ath11k_spectral_process_fft(struct ath11k *ar,
 	case ATH11K_SPECTRAL_80MHZ:
 		fft_sample->chan_width_mhz = chan_width_mhz;
 		break;
+	case ATH11K_SPECTRAL_160MHZ:
+		if (ab->hw_params.spectral.fragment_160mhz) {
+			chan_width_mhz /= 2;
+			fragment_sample = true;
+		}
+		fft_sample->chan_width_mhz = chan_width_mhz;
+		break;
 	default:
 		ath11k_warn(ab, "invalid channel width %d\n", chan_width_mhz);
 		return -EINVAL;
@@ -663,6 +674,17 @@ int ath11k_spectral_process_fft(struct ath11k *ar,
 	freq = summary->meta.freq2;
 	fft_sample->freq2 = __cpu_to_be16(freq);
 
+	/* If freq2 is available then the spectral scan results are fragmented
+	 * as primary and secondary
+	 */
+	if (fragment_sample && freq) {
+		if (!ar->spectral.is_primary)
+			fft_sample->freq1 = cpu_to_be16(freq);
+
+		/* We have to toggle the is_primary to handle the next report */
+		ar->spectral.is_primary = !ar->spectral.is_primary;
+	}
+
 	ath11k_spectral_parse_fft(fft_sample->data, fft_report->bins, num_bins,
 				  ab->hw_params.spectral.fft_sz);
 
diff --git a/drivers/net/wireless/ath/ath11k/spectral.h b/drivers/net/wireless/ath/ath11k/spectral.h
index 081744265f2a..96bfa16e18e9 100644
--- a/drivers/net/wireless/ath/ath11k/spectral.h
+++ b/drivers/net/wireless/ath/ath11k/spectral.h
@@ -35,6 +35,7 @@ struct ath11k_spectral {
 	u16 count;
 	u8 fft_size;
 	bool enabled;
+	bool is_primary;
 };
 
 #ifdef CONFIG_ATH11K_SPECTRAL
diff --git a/drivers/net/wireless/ath/ath11k/thermal.c b/drivers/net/wireless/ath/ath11k/thermal.c
index c96b26f39a25..23ed01bd44f9 100644
--- a/drivers/net/wireless/ath/ath11k/thermal.c
+++ b/drivers/net/wireless/ath/ath11k/thermal.c
@@ -99,7 +99,7 @@ static ssize_t ath11k_thermal_show_temp(struct device *dev,
 	temperature = ar->thermal.temperature;
 	spin_unlock_bh(&ar->data_lock);
 
-	/* display in millidegree celcius */
+	/* display in millidegree Celsius */
 	ret = snprintf(buf, PAGE_SIZE, "%d\n", temperature * 1000);
 out:
 	mutex_unlock(&ar->conf_mutex);
diff --git a/drivers/net/wireless/ath/ath11k/thermal.h b/drivers/net/wireless/ath/ath11k/thermal.h
index f9af55f3682d..3e39675ef7f5 100644
--- a/drivers/net/wireless/ath/ath11k/thermal.h
+++ b/drivers/net/wireless/ath/ath11k/thermal.h
@@ -19,7 +19,7 @@ struct ath11k_thermal {
 
 	/* protected by conf_mutex */
 	u32 throttle_state;
-	/* temperature value in Celcius degree
+	/* temperature value in Celsius degree
 	 * protected by data_lock
 	 */
 	int temperature;
diff --git a/drivers/net/wireless/ath/ath11k/trace.h b/drivers/net/wireless/ath/ath11k/trace.h
index 76560587bea0..9535745fe026 100644
--- a/drivers/net/wireless/ath/ath11k/trace.h
+++ b/drivers/net/wireless/ath/ath11k/trace.h
@@ -305,6 +305,34 @@ TRACE_EVENT(ath11k_wmi_diag,
 	)
 );
 
+TRACE_EVENT(ath11k_ps_timekeeper,
+	    TP_PROTO(struct ath11k *ar, const void *peer_addr,
+		     u32 peer_ps_timestamp, u8 peer_ps_state),
+	TP_ARGS(ar, peer_addr, peer_ps_timestamp, peer_ps_state),
+
+	TP_STRUCT__entry(__string(device, dev_name(ar->ab->dev))
+			 __string(driver, dev_driver_string(ar->ab->dev))
+			 __dynamic_array(u8, peer_addr, ETH_ALEN)
+			 __field(u8, peer_ps_state)
+			 __field(u32, peer_ps_timestamp)
+	),
+
+	TP_fast_assign(__assign_str(device, dev_name(ar->ab->dev));
+		       __assign_str(driver, dev_driver_string(ar->ab->dev));
+		       memcpy(__get_dynamic_array(peer_addr), peer_addr,
+			      ETH_ALEN);
+		       __entry->peer_ps_state = peer_ps_state;
+		       __entry->peer_ps_timestamp = peer_ps_timestamp;
+	),
+
+	TP_printk("%s %s %u %u",
+		  __get_str(driver),
+		  __get_str(device),
+		  __entry->peer_ps_state,
+		  __entry->peer_ps_timestamp
+	)
+);
+
 #endif /* _TRACE_H_ || TRACE_HEADER_MULTI_READ*/
 
 /* we don't want to use include/trace/events */
diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c
index 88ee4f9d19da..fad9f8d308a2 100644
--- a/drivers/net/wireless/ath/ath11k/wmi.c
+++ b/drivers/net/wireless/ath/ath11k/wmi.c
@@ -416,7 +416,7 @@ ath11k_pull_mac_phy_cap_svc_ready_ext(struct ath11k_pdev_wmi *wmi_handle,
 
 	/* tx/rx chainmask reported from fw depends on the actual hw chains used,
 	 * For example, for 4x4 capable macphys, first 4 chains can be used for first
-	 * mac and the remaing 4 chains can be used for the second mac or vice-versa.
+	 * mac and the remaining 4 chains can be used for the second mac or vice-versa.
 	 * In this case, tx/rx chainmask 0xf will be advertised for first mac and 0xf0
 	 * will be advertised for second mac or vice-versa. Compute the shift value
 	 * for tx/rx chainmask which will be used to advertise supported ht/vht rates to
@@ -991,9 +991,13 @@ int ath11k_wmi_vdev_up(struct ath11k *ar, u32 vdev_id, u32 aid, const u8 *bssid)
 {
 	struct ath11k_pdev_wmi *wmi = ar->wmi;
 	struct wmi_vdev_up_cmd *cmd;
+	struct ieee80211_bss_conf *bss_conf;
+	struct ath11k_vif *arvif;
 	struct sk_buff *skb;
 	int ret;
 
+	arvif = ath11k_mac_get_arvif(ar, vdev_id);
+
 	skb = ath11k_wmi_alloc_skb(wmi->wmi_ab, sizeof(*cmd));
 	if (!skb)
 		return -ENOMEM;
@@ -1007,6 +1011,17 @@ int ath11k_wmi_vdev_up(struct ath11k *ar, u32 vdev_id, u32 aid, const u8 *bssid)
 
 	ether_addr_copy(cmd->vdev_bssid.addr, bssid);
 
+	if (arvif && arvif->vif->type == NL80211_IFTYPE_STATION) {
+		bss_conf = &arvif->vif->bss_conf;
+
+		if (bss_conf->nontransmitted) {
+			ether_addr_copy(cmd->trans_bssid.addr,
+					bss_conf->transmitter_bssid);
+			cmd->profile_idx = bss_conf->bssid_index;
+			cmd->profile_num = bss_conf->bssid_indicator;
+		}
+	}
+
 	ret = ath11k_wmi_cmd_send(wmi, skb, WMI_VDEV_UP_CMDID);
 	if (ret) {
 		ath11k_warn(ar->ab, "failed to submit WMI_VDEV_UP cmd\n");
@@ -3064,8 +3079,34 @@ int ath11k_wmi_pdev_pktlog_disable(struct ath11k *ar)
 	return ret;
 }
 
-int
-ath11k_wmi_send_twt_enable_cmd(struct ath11k *ar, u32 pdev_id)
+void ath11k_wmi_fill_default_twt_params(struct wmi_twt_enable_params *twt_params)
+{
+	twt_params->sta_cong_timer_ms = ATH11K_TWT_DEF_STA_CONG_TIMER_MS;
+	twt_params->default_slot_size = ATH11K_TWT_DEF_DEFAULT_SLOT_SIZE;
+	twt_params->congestion_thresh_setup = ATH11K_TWT_DEF_CONGESTION_THRESH_SETUP;
+	twt_params->congestion_thresh_teardown =
+		ATH11K_TWT_DEF_CONGESTION_THRESH_TEARDOWN;
+	twt_params->congestion_thresh_critical =
+		ATH11K_TWT_DEF_CONGESTION_THRESH_CRITICAL;
+	twt_params->interference_thresh_teardown =
+		ATH11K_TWT_DEF_INTERFERENCE_THRESH_TEARDOWN;
+	twt_params->interference_thresh_setup =
+		ATH11K_TWT_DEF_INTERFERENCE_THRESH_SETUP;
+	twt_params->min_no_sta_setup = ATH11K_TWT_DEF_MIN_NO_STA_SETUP;
+	twt_params->min_no_sta_teardown = ATH11K_TWT_DEF_MIN_NO_STA_TEARDOWN;
+	twt_params->no_of_bcast_mcast_slots = ATH11K_TWT_DEF_NO_OF_BCAST_MCAST_SLOTS;
+	twt_params->min_no_twt_slots = ATH11K_TWT_DEF_MIN_NO_TWT_SLOTS;
+	twt_params->max_no_sta_twt = ATH11K_TWT_DEF_MAX_NO_STA_TWT;
+	twt_params->mode_check_interval = ATH11K_TWT_DEF_MODE_CHECK_INTERVAL;
+	twt_params->add_sta_slot_interval = ATH11K_TWT_DEF_ADD_STA_SLOT_INTERVAL;
+	twt_params->remove_sta_slot_interval =
+		ATH11K_TWT_DEF_REMOVE_STA_SLOT_INTERVAL;
+	/* TODO add MBSSID support */
+	twt_params->mbss_support = 0;
+}
+
+int ath11k_wmi_send_twt_enable_cmd(struct ath11k *ar, u32 pdev_id,
+				   struct wmi_twt_enable_params *params)
 {
 	struct ath11k_pdev_wmi *wmi = ar->wmi;
 	struct ath11k_base *ab = wmi->wmi_ab->ab;
@@ -3083,28 +3124,22 @@ ath11k_wmi_send_twt_enable_cmd(struct ath11k *ar, u32 pdev_id)
 	cmd->tlv_header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_TWT_ENABLE_CMD) |
 			  FIELD_PREP(WMI_TLV_LEN, len - TLV_HDR_SIZE);
 	cmd->pdev_id = pdev_id;
-	cmd->sta_cong_timer_ms = ATH11K_TWT_DEF_STA_CONG_TIMER_MS;
-	cmd->default_slot_size = ATH11K_TWT_DEF_DEFAULT_SLOT_SIZE;
-	cmd->congestion_thresh_setup = ATH11K_TWT_DEF_CONGESTION_THRESH_SETUP;
-	cmd->congestion_thresh_teardown =
-		ATH11K_TWT_DEF_CONGESTION_THRESH_TEARDOWN;
-	cmd->congestion_thresh_critical =
-		ATH11K_TWT_DEF_CONGESTION_THRESH_CRITICAL;
-	cmd->interference_thresh_teardown =
-		ATH11K_TWT_DEF_INTERFERENCE_THRESH_TEARDOWN;
-	cmd->interference_thresh_setup =
-		ATH11K_TWT_DEF_INTERFERENCE_THRESH_SETUP;
-	cmd->min_no_sta_setup = ATH11K_TWT_DEF_MIN_NO_STA_SETUP;
-	cmd->min_no_sta_teardown = ATH11K_TWT_DEF_MIN_NO_STA_TEARDOWN;
-	cmd->no_of_bcast_mcast_slots = ATH11K_TWT_DEF_NO_OF_BCAST_MCAST_SLOTS;
-	cmd->min_no_twt_slots = ATH11K_TWT_DEF_MIN_NO_TWT_SLOTS;
-	cmd->max_no_sta_twt = ATH11K_TWT_DEF_MAX_NO_STA_TWT;
-	cmd->mode_check_interval = ATH11K_TWT_DEF_MODE_CHECK_INTERVAL;
-	cmd->add_sta_slot_interval = ATH11K_TWT_DEF_ADD_STA_SLOT_INTERVAL;
-	cmd->remove_sta_slot_interval =
-		ATH11K_TWT_DEF_REMOVE_STA_SLOT_INTERVAL;
-	/* TODO add MBSSID support */
-	cmd->mbss_support = 0;
+	cmd->sta_cong_timer_ms = params->sta_cong_timer_ms;
+	cmd->default_slot_size = params->default_slot_size;
+	cmd->congestion_thresh_setup = params->congestion_thresh_setup;
+	cmd->congestion_thresh_teardown = params->congestion_thresh_teardown;
+	cmd->congestion_thresh_critical = params->congestion_thresh_critical;
+	cmd->interference_thresh_teardown = params->interference_thresh_teardown;
+	cmd->interference_thresh_setup = params->interference_thresh_setup;
+	cmd->min_no_sta_setup = params->min_no_sta_setup;
+	cmd->min_no_sta_teardown = params->min_no_sta_teardown;
+	cmd->no_of_bcast_mcast_slots = params->no_of_bcast_mcast_slots;
+	cmd->min_no_twt_slots = params->min_no_twt_slots;
+	cmd->max_no_sta_twt = params->max_no_sta_twt;
+	cmd->mode_check_interval = params->mode_check_interval;
+	cmd->add_sta_slot_interval = params->add_sta_slot_interval;
+	cmd->remove_sta_slot_interval = params->remove_sta_slot_interval;
+	cmd->mbss_support = params->mbss_support;
 
 	ret = ath11k_wmi_cmd_send(wmi, skb, WMI_TWT_ENABLE_CMDID);
 	if (ret) {
@@ -6767,6 +6802,107 @@ static void ath11k_bcn_tx_status_event(struct ath11k_base *ab, struct sk_buff *s
 	rcu_read_unlock();
 }
 
+static void ath11k_wmi_event_peer_sta_ps_state_chg(struct ath11k_base *ab,
+						   struct sk_buff *skb)
+{
+	const struct wmi_peer_sta_ps_state_chg_event *ev;
+	struct ieee80211_sta *sta;
+	struct ath11k_peer *peer;
+	struct ath11k *ar;
+	struct ath11k_sta *arsta;
+	const void **tb;
+	enum ath11k_wmi_peer_ps_state peer_previous_ps_state;
+	int ret;
+
+	tb = ath11k_wmi_tlv_parse_alloc(ab, skb->data, skb->len, GFP_ATOMIC);
+	if (IS_ERR(tb)) {
+		ret = PTR_ERR(tb);
+		ath11k_warn(ab, "failed to parse tlv: %d\n", ret);
+		return;
+	}
+
+	ev = tb[WMI_TAG_PEER_STA_PS_STATECHANGE_EVENT];
+	if (!ev) {
+		ath11k_warn(ab, "failed to fetch sta ps change ev");
+		kfree(tb);
+		return;
+	}
+
+	ath11k_dbg(ab, ATH11K_DBG_WMI,
+		   "peer sta ps chnange ev addr %pM state %u sup_bitmap %x ps_valid %u ts %u\n",
+		   ev->peer_macaddr.addr, ev->peer_ps_state,
+		   ev->ps_supported_bitmap, ev->peer_ps_valid,
+		   ev->peer_ps_timestamp);
+
+	rcu_read_lock();
+
+	spin_lock_bh(&ab->base_lock);
+
+	peer = ath11k_peer_find_by_addr(ab, ev->peer_macaddr.addr);
+
+	if (!peer) {
+		spin_unlock_bh(&ab->base_lock);
+		ath11k_warn(ab, "peer not found %pM\n", ev->peer_macaddr.addr);
+		goto exit;
+	}
+
+	ar = ath11k_mac_get_ar_by_vdev_id(ab, peer->vdev_id);
+
+	if (!ar) {
+		spin_unlock_bh(&ab->base_lock);
+		ath11k_warn(ab, "invalid vdev id in peer sta ps state change ev %d",
+			    peer->vdev_id);
+
+		goto exit;
+	}
+
+	sta = peer->sta;
+
+	spin_unlock_bh(&ab->base_lock);
+
+	if (!sta) {
+		ath11k_warn(ab, "failed to find station entry %pM\n",
+			    ev->peer_macaddr.addr);
+		goto exit;
+	}
+
+	arsta = (struct ath11k_sta *)sta->drv_priv;
+
+	spin_lock_bh(&ar->data_lock);
+
+	peer_previous_ps_state = arsta->peer_ps_state;
+	arsta->peer_ps_state = ev->peer_ps_state;
+	arsta->peer_current_ps_valid = !!ev->peer_ps_valid;
+
+	if (test_bit(WMI_TLV_SERVICE_PEER_POWER_SAVE_DURATION_SUPPORT,
+		     ar->ab->wmi_ab.svc_map)) {
+		if (!(ev->ps_supported_bitmap & WMI_PEER_PS_VALID) ||
+		    !(ev->ps_supported_bitmap & WMI_PEER_PS_STATE_TIMESTAMP) ||
+		    !ev->peer_ps_valid)
+			goto out;
+
+		if (arsta->peer_ps_state == WMI_PEER_PS_STATE_ON) {
+			arsta->ps_start_time = ev->peer_ps_timestamp;
+			arsta->ps_start_jiffies = jiffies;
+		} else if (arsta->peer_ps_state == WMI_PEER_PS_STATE_OFF &&
+			   peer_previous_ps_state == WMI_PEER_PS_STATE_ON) {
+			arsta->ps_total_duration = arsta->ps_total_duration +
+					(ev->peer_ps_timestamp - arsta->ps_start_time);
+		}
+
+		if (ar->ps_timekeeper_enable)
+			trace_ath11k_ps_timekeeper(ar, ev->peer_macaddr.addr,
+						   ev->peer_ps_timestamp,
+						   arsta->peer_ps_state);
+	}
+
+out:
+	spin_unlock_bh(&ar->data_lock);
+exit:
+	rcu_read_unlock();
+	kfree(tb);
+}
+
 static void ath11k_vdev_stopped_event(struct ath11k_base *ab, struct sk_buff *skb)
 {
 	struct ath11k *ar;
@@ -7409,7 +7545,53 @@ static void ath11k_peer_assoc_conf_event(struct ath11k_base *ab, struct sk_buff
 
 static void ath11k_update_stats_event(struct ath11k_base *ab, struct sk_buff *skb)
 {
-	ath11k_debugfs_fw_stats_process(ab, skb);
+	struct ath11k_fw_stats stats = {};
+	struct ath11k *ar;
+	int ret;
+
+	INIT_LIST_HEAD(&stats.pdevs);
+	INIT_LIST_HEAD(&stats.vdevs);
+	INIT_LIST_HEAD(&stats.bcn);
+
+	ret = ath11k_wmi_pull_fw_stats(ab, skb, &stats);
+	if (ret) {
+		ath11k_warn(ab, "failed to pull fw stats: %d\n", ret);
+		goto free;
+	}
+
+	rcu_read_lock();
+	ar = ath11k_mac_get_ar_by_pdev_id(ab, stats.pdev_id);
+	if (!ar) {
+		rcu_read_unlock();
+		ath11k_warn(ab, "failed to get ar for pdev_id %d: %d\n",
+			    stats.pdev_id, ret);
+		goto free;
+	}
+
+	spin_lock_bh(&ar->data_lock);
+
+	/* WMI_REQUEST_PDEV_STAT can be requested via .get_txpower mac ops or via
+	 * debugfs fw stats. Therefore, processing it separately.
+	 */
+	if (stats.stats_id == WMI_REQUEST_PDEV_STAT) {
+		list_splice_tail_init(&stats.pdevs, &ar->fw_stats.pdevs);
+		ar->fw_stats_done = true;
+		goto complete;
+	}
+
+	/* WMI_REQUEST_VDEV_STAT, WMI_REQUEST_BCN_STAT and WMI_REQUEST_RSSI_PER_CHAIN_STAT
+	 * are currently requested only via debugfs fw stats. Hence, processing these
+	 * in debugfs context
+	 */
+	ath11k_debugfs_fw_stats_process(ar, &stats);
+
+complete:
+	complete(&ar->fw_stats_complete);
+	rcu_read_unlock();
+	spin_unlock_bh(&ar->data_lock);
+
+free:
+	ath11k_fw_stats_free(&stats);
 }
 
 /* PDEV_CTL_FAILSAFE_CHECK_EVENT is received from FW when the frequency scanned
@@ -7960,6 +8142,9 @@ static void ath11k_wmi_tlv_op_rx(struct ath11k_base *ab, struct sk_buff *skb)
 	case WMI_DIAG_EVENTID:
 		ath11k_wmi_diag_event(ab, skb);
 		break;
+	case WMI_PEER_STA_PS_STATECHG_EVENTID:
+		ath11k_wmi_event_peer_sta_ps_state_chg(ab, skb);
+		break;
 	case WMI_GTK_OFFLOAD_STATUS_EVENTID:
 		ath11k_wmi_gtk_offload_status_event(ab, skb);
 		break;
@@ -8962,12 +9147,13 @@ int ath11k_wmi_sta_keepalive(struct ath11k *ar,
 	cmd->interval = arg->interval;
 	cmd->method = arg->method;
 
+	arp = (struct wmi_sta_keepalive_arp_resp *)(cmd + 1);
+	arp->tlv_header = FIELD_PREP(WMI_TLV_TAG,
+				     WMI_TAG_STA_KEEPALIVE_ARP_RESPONSE) |
+			 FIELD_PREP(WMI_TLV_LEN, sizeof(*arp) - TLV_HDR_SIZE);
+
 	if (arg->method == WMI_STA_KEEPALIVE_METHOD_UNSOLICITED_ARP_RESPONSE ||
 	    arg->method == WMI_STA_KEEPALIVE_METHOD_GRATUITOUS_ARP_REQUEST) {
-		arp = (struct wmi_sta_keepalive_arp_resp *)(cmd + 1);
-		arp->tlv_header = FIELD_PREP(WMI_TLV_TAG,
-					     WMI_TAG_STA_KEEPALVE_ARP_RESPONSE) |
-				 FIELD_PREP(WMI_TLV_LEN, sizeof(*arp) - TLV_HDR_SIZE);
 		arp->src_ip4_addr = arg->src_ip4_addr;
 		arp->dest_ip4_addr = arg->dest_ip4_addr;
 		ether_addr_copy(arp->dest_mac_addr.addr, arg->dest_mac_addr);
diff --git a/drivers/net/wireless/ath/ath11k/wmi.h b/drivers/net/wireless/ath/ath11k/wmi.h
index 4da248ffa318..8f2c07d70a4a 100644
--- a/drivers/net/wireless/ath/ath11k/wmi.h
+++ b/drivers/net/wireless/ath/ath11k/wmi.h
@@ -17,7 +17,7 @@ struct ath11k_vif;
 
 #define PSOC_HOST_MAX_NUM_SS (8)
 
-/* defines to set Packet extension values whic can be 0 us, 8 usec or 16 usec */
+/* defines to set Packet extension values which can be 0 us, 8 usec or 16 usec */
 #define MAX_HE_NSS               8
 #define MAX_HE_MODULATION        8
 #define MAX_HE_RU                4
@@ -1214,7 +1214,7 @@ enum wmi_tlv_tag {
 	WMI_TAG_NS_OFFLOAD_TUPLE,
 	WMI_TAG_FTM_INTG_CMD,
 	WMI_TAG_STA_KEEPALIVE_CMD,
-	WMI_TAG_STA_KEEPALVE_ARP_RESPONSE,
+	WMI_TAG_STA_KEEPALIVE_ARP_RESPONSE,
 	WMI_TAG_P2P_SET_VENDOR_IE_DATA_CMD,
 	WMI_TAG_AP_PS_PEER_CMD,
 	WMI_TAG_PEER_RATE_RETRY_SCHED_CMD,
@@ -2090,6 +2090,7 @@ enum wmi_tlv_service {
 	WMI_TLV_SERVICE_PER_PEER_HTT_STATS_RESET = 213,
 	WMI_TLV_SERVICE_FREQINFO_IN_METADATA = 219,
 	WMI_TLV_SERVICE_EXT2_MSG = 220,
+	WMI_TLV_SERVICE_PEER_POWER_SAVE_DURATION_SUPPORT = 246,
 	WMI_TLV_SERVICE_SRG_SRP_SPATIAL_REUSE_SUPPORT = 249,
 
 	/* The second 128 bits */
@@ -4482,7 +4483,7 @@ struct wmi_pdev_radar_ev {
 } __packed;
 
 struct wmi_pdev_temperature_event {
-	/* temperature value in Celcius degree */
+	/* temperature value in Celsius degree */
 	s32 temp;
 	u32 pdev_id;
 } __packed;
@@ -4708,7 +4709,7 @@ enum wmi_sta_ps_param_tx_wake_threshold {
  */
 enum wmi_sta_ps_param_pspoll_count {
 	WMI_STA_PS_PSPOLL_COUNT_NO_MAX = 0,
-	/* Values greater than 0 indicate the maximum numer of PS-Poll frames
+	/* Values greater than 0 indicate the maximum number of PS-Poll frames
 	 * FW will send before waking up.
 	 */
 };
@@ -4820,9 +4821,9 @@ enum wmi_rate_preamble {
 
 /**
  * enum wmi_rtscts_prot_mode - Enable/Disable RTS/CTS and CTS2Self Protection.
- * @WMI_RTS_CTS_DISABLED : RTS/CTS protection is disabled.
- * @WMI_USE_RTS_CTS : RTS/CTS Enabled.
- * @WMI_USE_CTS2SELF : CTS to self protection Enabled.
+ * @WMI_RTS_CTS_DISABLED: RTS/CTS protection is disabled.
+ * @WMI_USE_RTS_CTS: RTS/CTS Enabled.
+ * @WMI_USE_CTS2SELF: CTS to self protection Enabled.
  */
 enum wmi_rtscts_prot_mode {
 	WMI_RTS_CTS_DISABLED = 0,
@@ -4833,13 +4834,13 @@ enum wmi_rtscts_prot_mode {
 /**
  * enum wmi_rtscts_profile - Selection of RTS CTS profile along with enabling
  *                           protection mode.
- * @WMI_RTSCTS_FOR_NO_RATESERIES - Neither of rate-series should use RTS-CTS
- * @WMI_RTSCTS_FOR_SECOND_RATESERIES - Only second rate-series will use RTS-CTS
- * @WMI_RTSCTS_ACROSS_SW_RETRIES - Only the second rate-series will use RTS-CTS,
- *                                 but if there's a sw retry, both the rate
- *                                 series will use RTS-CTS.
- * @WMI_RTSCTS_ERP - RTS/CTS used for ERP protection for every PPDU.
- * @WMI_RTSCTS_FOR_ALL_RATESERIES - Enable RTS-CTS for all rate series.
+ * @WMI_RTSCTS_FOR_NO_RATESERIES: Neither of rate-series should use RTS-CTS
+ * @WMI_RTSCTS_FOR_SECOND_RATESERIES: Only second rate-series will use RTS-CTS
+ * @WMI_RTSCTS_ACROSS_SW_RETRIES: Only the second rate-series will use RTS-CTS,
+ *                                but if there's a sw retry, both the rate
+ *                                series will use RTS-CTS.
+ * @WMI_RTSCTS_ERP: RTS/CTS used for ERP protection for every PPDU.
+ * @WMI_RTSCTS_FOR_ALL_RATESERIES: Enable RTS-CTS for all rate series.
  */
 enum wmi_rtscts_profile {
 	WMI_RTSCTS_FOR_NO_RATESERIES = 0,
@@ -4933,6 +4934,25 @@ struct wmi_wmm_params_all_arg {
 #define ATH11K_TWT_DEF_ADD_STA_SLOT_INTERVAL		1000
 #define ATH11K_TWT_DEF_REMOVE_STA_SLOT_INTERVAL		5000
 
+struct wmi_twt_enable_params {
+	u32 sta_cong_timer_ms;
+	u32 mbss_support;
+	u32 default_slot_size;
+	u32 congestion_thresh_setup;
+	u32 congestion_thresh_teardown;
+	u32 congestion_thresh_critical;
+	u32 interference_thresh_teardown;
+	u32 interference_thresh_setup;
+	u32 min_no_sta_setup;
+	u32 min_no_sta_teardown;
+	u32 no_of_bcast_mcast_slots;
+	u32 min_no_twt_slots;
+	u32 max_no_sta_twt;
+	u32 mode_check_interval;
+	u32 add_sta_slot_interval;
+	u32 remove_sta_slot_interval;
+};
+
 struct wmi_twt_enable_params_cmd {
 	u32 tlv_header;
 	u32 pdev_id;
@@ -5350,6 +5370,26 @@ struct wmi_debug_log_config_cmd_fixed_param {
 #define WMI_SERVICE_READY_TIMEOUT_HZ (5 * HZ)
 #define WMI_SEND_TIMEOUT_HZ (3 * HZ)
 
+enum ath11k_wmi_peer_ps_state {
+	WMI_PEER_PS_STATE_OFF,
+	WMI_PEER_PS_STATE_ON,
+	WMI_PEER_PS_STATE_DISABLED,
+};
+
+enum wmi_peer_ps_supported_bitmap {
+	/* Used to indicate that power save state change is valid */
+	WMI_PEER_PS_VALID = 0x1,
+	WMI_PEER_PS_STATE_TIMESTAMP = 0x2,
+};
+
+struct wmi_peer_sta_ps_state_chg_event {
+	struct wmi_mac_addr peer_macaddr;
+	u32 peer_ps_state;
+	u32 ps_supported_bitmap;
+	u32 peer_ps_valid;
+	u32 peer_ps_timestamp;
+} __packed;
+
 struct ath11k_wmi_base {
 	struct ath11k_base *ab;
 	struct ath11k_pdev_wmi wmi[MAX_RADIOS];
@@ -6039,7 +6079,9 @@ void ath11k_wmi_fw_stats_fill(struct ath11k *ar,
 			      struct ath11k_fw_stats *fw_stats, u32 stats_id,
 			      char *buf);
 int ath11k_wmi_simulate_radar(struct ath11k *ar);
-int ath11k_wmi_send_twt_enable_cmd(struct ath11k *ar, u32 pdev_id);
+void ath11k_wmi_fill_default_twt_params(struct wmi_twt_enable_params *twt_params);
+int ath11k_wmi_send_twt_enable_cmd(struct ath11k *ar, u32 pdev_id,
+				   struct wmi_twt_enable_params *params);
 int ath11k_wmi_send_twt_disable_cmd(struct ath11k *ar, u32 pdev_id);
 int ath11k_wmi_send_twt_add_dialog_cmd(struct ath11k *ar,
 				       struct wmi_twt_add_dialog_params *params);
diff --git a/drivers/net/wireless/ath/ath11k/wow.c b/drivers/net/wireless/ath/ath11k/wow.c
index b3e65cd13d83..1dec23b0699c 100644
--- a/drivers/net/wireless/ath/ath11k/wow.c
+++ b/drivers/net/wireless/ath/ath11k/wow.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2020 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2022, Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #include <linux/delay.h>
@@ -67,6 +68,13 @@ int ath11k_wow_wakeup(struct ath11k_base *ab)
 	struct ath11k *ar = ath11k_ab_to_ar(ab, 0);
 	int ret;
 
+	/* In the case of WCN6750, WoW wakeup is done
+	 * by sending SMP2P power save exit message
+	 * to the target processor.
+	 */
+	if (ab->hw_params.smp2p_wow_exit)
+		return 0;
+
 	reinit_completion(&ab->wow.wakeup_completed);
 
 	ret = ath11k_wmi_wow_host_wakeup_ind(ar);
@@ -664,6 +672,12 @@ int ath11k_wow_op_suspend(struct ieee80211_hw *hw,
 	struct ath11k *ar = hw->priv;
 	int ret;
 
+	ret = ath11k_mac_wait_tx_complete(ar);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to wait tx complete: %d\n", ret);
+		return ret;
+	}
+
 	mutex_lock(&ar->conf_mutex);
 
 	ret = ath11k_dp_rx_pktlog_stop(ar->ab, true);
@@ -695,13 +709,6 @@ int ath11k_wow_op_suspend(struct ieee80211_hw *hw,
 		goto cleanup;
 	}
 
-	ath11k_mac_drain_tx(ar);
-	ret = ath11k_mac_wait_tx_complete(ar);
-	if (ret) {
-		ath11k_warn(ar->ab, "failed to wait tx complete: %d\n", ret);
-		goto cleanup;
-	}
-
 	ret = ath11k_wow_set_hw_filter(ar);
 	if (ret) {
 		ath11k_warn(ar->ab, "failed to set hw filter: %d\n",
diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c
index e11c7e9accc0..a20e0aeae284 100644
--- a/drivers/net/wireless/ath/ath6kl/cfg80211.c
+++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c
@@ -1124,7 +1124,7 @@ void ath6kl_cfg80211_ch_switch_notify(struct ath6kl_vif *vif, int freq,
 }
 
 static int ath6kl_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev,
-				   u8 key_index, bool pairwise,
+				   int link_id, u8 key_index, bool pairwise,
 				   const u8 *mac_addr,
 				   struct key_params *params)
 {
@@ -1249,7 +1249,7 @@ static int ath6kl_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static int ath6kl_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev,
-				   u8 key_index, bool pairwise,
+				   int link_id, u8 key_index, bool pairwise,
 				   const u8 *mac_addr)
 {
 	struct ath6kl *ar = ath6kl_priv(ndev);
@@ -1279,7 +1279,7 @@ static int ath6kl_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static int ath6kl_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev,
-				   u8 key_index, bool pairwise,
+				   int link_id, u8 key_index, bool pairwise,
 				   const u8 *mac_addr, void *cookie,
 				   void (*callback) (void *cookie,
 						     struct key_params *))
@@ -1314,7 +1314,7 @@ static int ath6kl_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static int ath6kl_cfg80211_set_default_key(struct wiphy *wiphy,
-					   struct net_device *ndev,
+					   struct net_device *ndev, int link_id,
 					   u8 key_index, bool unicast,
 					   bool multicast)
 {
diff --git a/drivers/net/wireless/ath/ath6kl/init.c b/drivers/net/wireless/ath/ath6kl/init.c
index 9b5c7d8f2b95..201e45554070 100644
--- a/drivers/net/wireless/ath/ath6kl/init.c
+++ b/drivers/net/wireless/ath/ath6kl/init.c
@@ -1014,7 +1014,7 @@ static int ath6kl_fetch_fw_apin(struct ath6kl *ar, const char *name)
 
 		switch (ie_id) {
 		case ATH6KL_FW_IE_FW_VERSION:
-			strlcpy(ar->wiphy->fw_version, data,
+			strscpy(ar->wiphy->fw_version, data,
 				min(sizeof(ar->wiphy->fw_version), ie_len+1));
 
 			ath6kl_dbg(ATH6KL_DBG_BOOT,
diff --git a/drivers/net/wireless/ath/ath9k/ar9003_phy.c b/drivers/net/wireless/ath/ath9k/ar9003_phy.c
index dc0e5ea25673..090ff0600c81 100644
--- a/drivers/net/wireless/ath/ath9k/ar9003_phy.c
+++ b/drivers/net/wireless/ath/ath9k/ar9003_phy.c
@@ -1744,7 +1744,7 @@ static void ar9003_hw_spectral_scan_config(struct ath_hw *ah,
 	REG_SET_BIT(ah, AR_PHY_RADAR_0, AR_PHY_RADAR_0_FFT_ENA);
 	REG_SET_BIT(ah, AR_PHY_SPECTRAL_SCAN, AR_PHY_SPECTRAL_SCAN_ENABLE);
 
-	/* on AR93xx and newer, count = 0 will make the the chip send
+	/* on AR93xx and newer, count = 0 will make the chip send
 	 * spectral samples endlessly. Check if this really was intended,
 	 * and fix otherwise.
 	 */
diff --git a/drivers/net/wireless/ath/ath9k/channel.c b/drivers/net/wireless/ath/ath9k/channel.c
index 6cf087522157..571062f2e82a 100644
--- a/drivers/net/wireless/ath/ath9k/channel.c
+++ b/drivers/net/wireless/ath/ath9k/channel.c
@@ -1113,7 +1113,7 @@ ath_chanctx_send_vif_ps_frame(struct ath_softc *sc, struct ath_vif *avp,
 		if (!avp->assoc)
 			return false;
 
-		skb = ieee80211_nullfunc_get(sc->hw, vif, false);
+		skb = ieee80211_nullfunc_get(sc->hw, vif, -1, false);
 		if (!skb)
 			return false;
 
diff --git a/drivers/net/wireless/ath/ath9k/htc_hst.c b/drivers/net/wireless/ath/ath9k/htc_hst.c
index 994ec48b2f66..ca05b07a45e6 100644
--- a/drivers/net/wireless/ath/ath9k/htc_hst.c
+++ b/drivers/net/wireless/ath/ath9k/htc_hst.c
@@ -364,33 +364,27 @@ ret:
 }
 
 static void ath9k_htc_fw_panic_report(struct htc_target *htc_handle,
-				      struct sk_buff *skb)
+				      struct sk_buff *skb, u32 len)
 {
 	uint32_t *pattern = (uint32_t *)skb->data;
 
-	switch (*pattern) {
-	case 0x33221199:
-		{
+	if (*pattern == 0x33221199 && len >= sizeof(struct htc_panic_bad_vaddr)) {
 		struct htc_panic_bad_vaddr *htc_panic;
 		htc_panic = (struct htc_panic_bad_vaddr *) skb->data;
 		dev_err(htc_handle->dev, "ath: firmware panic! "
 			"exccause: 0x%08x; pc: 0x%08x; badvaddr: 0x%08x.\n",
 			htc_panic->exccause, htc_panic->pc,
 			htc_panic->badvaddr);
-		break;
-		}
-	case 0x33221299:
-		{
+		return;
+	}
+	if (*pattern == 0x33221299) {
 		struct htc_panic_bad_epid *htc_panic;
 		htc_panic = (struct htc_panic_bad_epid *) skb->data;
 		dev_err(htc_handle->dev, "ath: firmware panic! "
 			"bad epid: 0x%08x\n", htc_panic->epid);
-		break;
-		}
-	default:
-		dev_err(htc_handle->dev, "ath: unknown panic pattern!\n");
-		break;
+		return;
 	}
+	dev_err(htc_handle->dev, "ath: unknown panic pattern!\n");
 }
 
 /*
@@ -411,16 +405,26 @@ void ath9k_htc_rx_msg(struct htc_target *htc_handle,
 	if (!htc_handle || !skb)
 		return;
 
+	/* A valid message requires len >= 8.
+	 *
+	 *   sizeof(struct htc_frame_hdr) == 8
+	 *   sizeof(struct htc_ready_msg) == 8
+	 *   sizeof(struct htc_panic_bad_vaddr) == 16
+	 *   sizeof(struct htc_panic_bad_epid) == 8
+	 */
+	if (unlikely(len < sizeof(struct htc_frame_hdr)))
+		goto invalid;
 	htc_hdr = (struct htc_frame_hdr *) skb->data;
 	epid = htc_hdr->endpoint_id;
 
 	if (epid == 0x99) {
-		ath9k_htc_fw_panic_report(htc_handle, skb);
+		ath9k_htc_fw_panic_report(htc_handle, skb, len);
 		kfree_skb(skb);
 		return;
 	}
 
 	if (epid < 0 || epid >= ENDPOINT_MAX) {
+invalid:
 		if (pipe_id != USB_REG_IN_PIPE)
 			dev_kfree_skb_any(skb);
 		else
@@ -432,21 +436,30 @@ void ath9k_htc_rx_msg(struct htc_target *htc_handle,
 
 		/* Handle trailer */
 		if (htc_hdr->flags & HTC_FLAGS_RECV_TRAILER) {
-			if (be32_to_cpu(*(__be32 *) skb->data) == 0x00C60000)
+			if (be32_to_cpu(*(__be32 *) skb->data) == 0x00C60000) {
 				/* Move past the Watchdog pattern */
 				htc_hdr = (struct htc_frame_hdr *)(skb->data + 4);
+				len -= 4;
+			}
 		}
 
 		/* Get the message ID */
+		if (unlikely(len < sizeof(struct htc_frame_hdr) + sizeof(__be16)))
+			goto invalid;
 		msg_id = (__be16 *) ((void *) htc_hdr +
 				     sizeof(struct htc_frame_hdr));
 
 		/* Now process HTC messages */
 		switch (be16_to_cpu(*msg_id)) {
 		case HTC_MSG_READY_ID:
+			if (unlikely(len < sizeof(struct htc_ready_msg)))
+				goto invalid;
 			htc_process_target_rdy(htc_handle, htc_hdr);
 			break;
 		case HTC_MSG_CONNECT_SERVICE_RESPONSE_ID:
+			if (unlikely(len < sizeof(struct htc_frame_hdr) +
+				     sizeof(struct htc_conn_svc_rspmsg)))
+				goto invalid;
 			htc_process_conn_rsp(htc_handle, htc_hdr);
 			break;
 		default:
diff --git a/drivers/net/wireless/ath/ath9k/hw.h b/drivers/net/wireless/ath/ath9k/hw.h
index 096a206f49ed..450ab19b1d4e 100644
--- a/drivers/net/wireless/ath/ath9k/hw.h
+++ b/drivers/net/wireless/ath/ath9k/hw.h
@@ -710,7 +710,7 @@ struct ath_spec_scan {
 /**
  * struct ath_hw_ops - callbacks used by hardware code and driver code
  *
- * This structure contains callbacks designed to to be used internally by
+ * This structure contains callbacks designed to be used internally by
  * hardware code and also by the lower level driver.
  *
  * @config_pci_powersave:
diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c
index ba16a7f3e23d..ba271a10d4ab 100644
--- a/drivers/net/wireless/ath/ath9k/xmit.c
+++ b/drivers/net/wireless/ath/ath9k/xmit.c
@@ -2160,7 +2160,7 @@ static void setup_frame_info(struct ieee80211_hw *hw,
 		fi->keyix = an->ps_key;
 	else
 		fi->keyix = ATH9K_TXKEYIX_INVALID;
-	fi->dyn_smps = sta && sta->smps_mode == IEEE80211_SMPS_DYNAMIC;
+	fi->dyn_smps = sta && sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC;
 	fi->keytype = keytype;
 	fi->framelen = framelen;
 	fi->tx_power = txpower;
diff --git a/drivers/net/wireless/ath/carl9170/fw.c b/drivers/net/wireless/ath/carl9170/fw.c
index 1ab09e1c9ec5..4c1aecd1163c 100644
--- a/drivers/net/wireless/ath/carl9170/fw.c
+++ b/drivers/net/wireless/ath/carl9170/fw.c
@@ -105,7 +105,7 @@ static void carl9170_fw_info(struct ar9170 *ar)
 			 CARL9170FW_GET_MONTH(fw_date),
 			 CARL9170FW_GET_DAY(fw_date));
 
-		strlcpy(ar->hw->wiphy->fw_version, motd_desc->release,
+		strscpy(ar->hw->wiphy->fw_version, motd_desc->release,
 			sizeof(ar->hw->wiphy->fw_version));
 	}
 }
diff --git a/drivers/net/wireless/ath/wcn36xx/hal.h b/drivers/net/wireless/ath/wcn36xx/hal.h
index f1a43fd1d957..d3a9d00e65e1 100644
--- a/drivers/net/wireless/ath/wcn36xx/hal.h
+++ b/drivers/net/wireless/ath/wcn36xx/hal.h
@@ -2677,7 +2677,7 @@ struct ani_global_security_stats {
 	 * management information base (MIB) object is enabled */
 	u32 rx_wep_unencrypted_frm_cnt;
 
-	/* The number of received MSDU packets that that the 802.11 station
+	/* The number of received MSDU packets that the 802.11 station
 	 * discarded because of MIC failures */
 	u32 rx_mic_fail_cnt;
 
diff --git a/drivers/net/wireless/ath/wcn36xx/txrx.c b/drivers/net/wireless/ath/wcn36xx/txrx.c
index 8da3955995b6..0802ed728824 100644
--- a/drivers/net/wireless/ath/wcn36xx/txrx.c
+++ b/drivers/net/wireless/ath/wcn36xx/txrx.c
@@ -16,6 +16,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/random.h>
 #include "txrx.h"
 
 static inline int get_rssi0(struct wcn36xx_rx_bd *bd)
@@ -278,6 +279,7 @@ static void wcn36xx_update_survey(struct wcn36xx *wcn, int rssi, int snr,
 	struct ieee80211_supported_band *sband;
 	int idx;
 	int i;
+	u8 snr_sample = snr & 0xff;
 
 	idx = 0;
 	if (band == NL80211_BAND_5GHZ)
@@ -297,6 +299,8 @@ static void wcn36xx_update_survey(struct wcn36xx *wcn, int rssi, int snr,
 	wcn->chan_survey[idx].rssi = rssi;
 	wcn->chan_survey[idx].snr = snr;
 	spin_unlock(&wcn->survey_lock);
+
+	add_device_randomness(&snr_sample, sizeof(snr_sample));
 }
 
 int wcn36xx_rx_skb(struct wcn36xx *wcn, struct sk_buff *skb)
diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
index f93bdffa4d1d..40f9a7ef8980 100644
--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
+++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
@@ -1620,7 +1620,7 @@ static void wil_del_rx_key(u8 key_index, enum wmi_key_usage key_usage,
 }
 
 static int wil_cfg80211_add_key(struct wiphy *wiphy,
-				struct net_device *ndev,
+				struct net_device *ndev, int link_id,
 				u8 key_index, bool pairwise,
 				const u8 *mac_addr,
 				struct key_params *params)
@@ -1696,7 +1696,7 @@ static int wil_cfg80211_add_key(struct wiphy *wiphy,
 }
 
 static int wil_cfg80211_del_key(struct wiphy *wiphy,
-				struct net_device *ndev,
+				struct net_device *ndev, int link_id,
 				u8 key_index, bool pairwise,
 				const u8 *mac_addr)
 {
@@ -1723,7 +1723,7 @@ static int wil_cfg80211_del_key(struct wiphy *wiphy,
 
 /* Need to be present or wiphy_new() will WARN */
 static int wil_cfg80211_set_default_key(struct wiphy *wiphy,
-					struct net_device *ndev,
+					struct net_device *ndev, int link_id,
 					u8 key_index, bool unicast,
 					bool multicast)
 {
@@ -2072,8 +2072,8 @@ void wil_cfg80211_ap_recovery(struct wil6210_priv *wil)
 		key_params.key = vif->gtk;
 		key_params.key_len = vif->gtk_len;
 		key_params.seq_len = IEEE80211_GCMP_PN_LEN;
-		rc = wil_cfg80211_add_key(wiphy, ndev, vif->gtk_index, false,
-					  NULL, &key_params);
+		rc = wil_cfg80211_add_key(wiphy, ndev, -1, vif->gtk_index,
+					  false, NULL, &key_params);
 		if (rc)
 			wil_err(wil, "vif %d recovery add key failed (%d)\n",
 				i, rc);
diff --git a/drivers/net/wireless/ath/wil6210/main.c b/drivers/net/wireless/ath/wil6210/main.c
index 7da87c9f363f..94e61dbe94f8 100644
--- a/drivers/net/wireless/ath/wil6210/main.c
+++ b/drivers/net/wireless/ath/wil6210/main.c
@@ -1305,7 +1305,7 @@ void wil_get_board_file(struct wil6210_priv *wil, char *buf, size_t len)
 			board_file = WIL_BOARD_FILE_NAME;
 	}
 
-	strlcpy(buf, board_file, len);
+	strscpy(buf, board_file, len);
 }
 
 static int wil_get_bl_info(struct wil6210_priv *wil)
diff --git a/drivers/net/wireless/ath/wil6210/netdev.c b/drivers/net/wireless/ath/wil6210/netdev.c
index 87a88f26233e..ee7d7e9c2718 100644
--- a/drivers/net/wireless/ath/wil6210/netdev.c
+++ b/drivers/net/wireless/ath/wil6210/netdev.c
@@ -445,7 +445,7 @@ int wil_if_add(struct wil6210_priv *wil)
 
 	wil_dbg_misc(wil, "entered");
 
-	strlcpy(wiphy->fw_version, wil->fw_version, sizeof(wiphy->fw_version));
+	strscpy(wiphy->fw_version, wil->fw_version, sizeof(wiphy->fw_version));
 
 	rc = wiphy_register(wiphy);
 	if (rc < 0) {
@@ -456,14 +456,12 @@ int wil_if_add(struct wil6210_priv *wil)
 	init_dummy_netdev(&wil->napi_ndev);
 	if (wil->use_enhanced_dma_hw) {
 		netif_napi_add(&wil->napi_ndev, &wil->napi_rx,
-			       wil6210_netdev_poll_rx_edma,
-			       NAPI_POLL_WEIGHT);
+			       wil6210_netdev_poll_rx_edma);
 		netif_napi_add_tx(&wil->napi_ndev,
 				  &wil->napi_tx, wil6210_netdev_poll_tx_edma);
 	} else {
 		netif_napi_add(&wil->napi_ndev, &wil->napi_rx,
-			       wil6210_netdev_poll_rx,
-			       NAPI_POLL_WEIGHT);
+			       wil6210_netdev_poll_rx);
 		netif_napi_add_tx(&wil->napi_ndev,
 				  &wil->napi_tx, wil6210_netdev_poll_tx);
 	}
diff --git a/drivers/net/wireless/ath/wil6210/wmi.c b/drivers/net/wireless/ath/wil6210/wmi.c
index ea7bd403e706..6a5976a2944c 100644
--- a/drivers/net/wireless/ath/wil6210/wmi.c
+++ b/drivers/net/wireless/ath/wil6210/wmi.c
@@ -780,7 +780,7 @@ static void wmi_evt_ready(struct wil6210_vif *vif, int id, void *d, int len)
 		return; /* FW load will fail after timeout */
 	}
 	/* ignore MAC address, we already have it from the boot loader */
-	strlcpy(wiphy->fw_version, wil->fw_version, sizeof(wiphy->fw_version));
+	strscpy(wiphy->fw_version, wil->fw_version, sizeof(wiphy->fw_version));
 
 	if (len > offsetof(struct wmi_ready_event, rfc_read_calib_result)) {
 		wil_dbg_wmi(wil, "rfc calibration result %d\n",
diff --git a/drivers/net/wireless/atmel/atmel.c b/drivers/net/wireless/atmel/atmel.c
index 0361c8eb2008..45d079b93384 100644
--- a/drivers/net/wireless/atmel/atmel.c
+++ b/drivers/net/wireless/atmel/atmel.c
@@ -1518,7 +1518,7 @@ struct net_device *init_atmel_card(unsigned short irq, unsigned long port,
 	priv->firmware = NULL;
 	priv->firmware_type = fw_type;
 	if (firmware) /* module parameter */
-		strlcpy(priv->firmware_id, firmware, sizeof(priv->firmware_id));
+		strscpy(priv->firmware_id, firmware, sizeof(priv->firmware_id));
 	priv->bus_type = card_present ? BUS_TYPE_PCCARD : BUS_TYPE_PCI;
 	priv->station_state = STATION_STATE_DOWN;
 	priv->do_rx_crc = 0;
diff --git a/drivers/net/wireless/broadcom/b43/leds.c b/drivers/net/wireless/broadcom/b43/leds.c
index 982a772a9d87..bfe1be345844 100644
--- a/drivers/net/wireless/broadcom/b43/leds.c
+++ b/drivers/net/wireless/broadcom/b43/leds.c
@@ -118,7 +118,7 @@ static int b43_register_led(struct b43_wldev *dev, struct b43_led *led,
 	led->wl = dev->wl;
 	led->index = led_index;
 	led->activelow = activelow;
-	strlcpy(led->name, name, sizeof(led->name));
+	strscpy(led->name, name, sizeof(led->name));
 	atomic_set(&led->state, 0);
 
 	led->led_dev.name = led->name;
diff --git a/drivers/net/wireless/broadcom/b43/phy_n.c b/drivers/net/wireless/broadcom/b43/phy_n.c
index aa5c99465674..2c0c019a815d 100644
--- a/drivers/net/wireless/broadcom/b43/phy_n.c
+++ b/drivers/net/wireless/broadcom/b43/phy_n.c
@@ -2479,11 +2479,7 @@ static void b43_nphy_gain_ctl_workarounds_rev19(struct b43_wldev *dev)
 
 static void b43_nphy_gain_ctl_workarounds_rev7(struct b43_wldev *dev)
 {
-	struct b43_phy *phy = &dev->phy;
-
-	switch (phy->rev) {
-	/* TODO */
-	}
+	/* TODO - should depend on phy->rev */
 }
 
 static void b43_nphy_gain_ctl_workarounds_rev3(struct b43_wldev *dev)
diff --git a/drivers/net/wireless/broadcom/b43legacy/leds.c b/drivers/net/wireless/broadcom/b43legacy/leds.c
index 38b5be3a84e2..79e6fd205bfb 100644
--- a/drivers/net/wireless/broadcom/b43legacy/leds.c
+++ b/drivers/net/wireless/broadcom/b43legacy/leds.c
@@ -88,7 +88,7 @@ static int b43legacy_register_led(struct b43legacy_wldev *dev,
 	led->dev = dev;
 	led->index = led_index;
 	led->activelow = activelow;
-	strlcpy(led->name, name, sizeof(led->name));
+	strscpy(led->name, name, sizeof(led->name));
 
 	led->led_dev.name = led->name;
 	led->led_dev.default_trigger = default_trigger;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c
index 2c95a08a5871..9ec0c60b6da1 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcdc.c
@@ -87,6 +87,8 @@ struct brcmf_proto_bcdc_header {
 					 * plus any space that might be needed
 					 * for bus alignment padding.
 					 */
+#define ROUND_UP_MARGIN 2048
+
 struct brcmf_bcdc {
 	u16 reqid;
 	u8 bus_header[BUS_HEADER_LEN];
@@ -368,8 +370,7 @@ brcmf_proto_bcdc_txcomplete(struct device *dev, struct sk_buff *txp,
 
 	/* await txstatus signal for firmware if active */
 	if (brcmf_fws_fc_active(bcdc->fws)) {
-		if (!success)
-			brcmf_fws_bustxfail(bcdc->fws, txp);
+		brcmf_fws_bustxcomplete(bcdc->fws, txp, success);
 	} else {
 		if (brcmf_proto_bcdc_hdrpull(bus_if->drvr, false, txp, &ifp))
 			brcmu_pkt_buf_free_skb(txp);
@@ -471,7 +472,7 @@ int brcmf_proto_bcdc_attach(struct brcmf_pub *drvr)
 
 	drvr->hdrlen += BCDC_HEADER_LEN + BRCMF_PROT_FW_SIGNAL_MAX_TXBYTES;
 	drvr->bus_if->maxctl = BRCMF_DCMD_MAXLEN +
-			sizeof(struct brcmf_proto_bcdc_dcmd);
+			sizeof(struct brcmf_proto_bcdc_dcmd) + ROUND_UP_MARGIN;
 	return 0;
 
 fail:
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
index d639bb8b51ae..d0daef674e72 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bcmsdh.c
@@ -983,6 +983,7 @@ static const struct sdio_device_id brcmf_sdmmc_ids[] = {
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_4359),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_4373),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_43012),
+	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_43439),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_43752),
 	BRCMF_SDIO_DEVICE(SDIO_DEVICE_ID_BROADCOM_CYPRESS_89359),
 	{ /* end: all zeroes */ }
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h
index ae5af76e2568..2208ab3aa795 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/bus.h
@@ -6,6 +6,8 @@
 #ifndef BRCMFMAC_BUS_H
 #define BRCMFMAC_BUS_H
 
+#include <linux/kernel.h>
+#include <linux/firmware.h>
 #include "debug.h"
 
 /* IDs of the 6 default common rings of msgbuf protocol */
@@ -34,6 +36,11 @@ enum brcmf_bus_protocol_type {
 	BRCMF_PROTO_MSGBUF
 };
 
+/* Firmware blobs that may be available */
+enum brcmf_blob_type {
+	BRCMF_BLOB_CLM,
+};
+
 struct brcmf_mp_device;
 
 struct brcmf_bus_dcmd {
@@ -60,7 +67,7 @@ struct brcmf_bus_dcmd {
  * @wowl_config: specify if dongle is configured for wowl when going to suspend
  * @get_ramsize: obtain size of device memory.
  * @get_memdump: obtain device memory dump in provided buffer.
- * @get_fwname: obtain firmware name.
+ * @get_blob: obtain a firmware blob.
  *
  * This structure provides an abstract interface towards the
  * bus specific driver. For control messages to common driver
@@ -77,8 +84,8 @@ struct brcmf_bus_ops {
 	void (*wowl_config)(struct device *dev, bool enabled);
 	size_t (*get_ramsize)(struct device *dev);
 	int (*get_memdump)(struct device *dev, void *data, size_t len);
-	int (*get_fwname)(struct device *dev, const char *ext,
-			  unsigned char *fw_name);
+	int (*get_blob)(struct device *dev, const struct firmware **fw,
+			enum brcmf_blob_type type);
 	void (*debugfs_create)(struct device *dev);
 	int (*reset)(struct device *dev);
 };
@@ -220,10 +227,10 @@ int brcmf_bus_get_memdump(struct brcmf_bus *bus, void *data, size_t len)
 }
 
 static inline
-int brcmf_bus_get_fwname(struct brcmf_bus *bus, const char *ext,
-			 unsigned char *fw_name)
+int brcmf_bus_get_blob(struct brcmf_bus *bus, const struct firmware **fw,
+		       enum brcmf_blob_type type)
 {
-	return bus->ops->get_fwname(bus->dev, ext, fw_name);
+	return bus->ops->get_blob(bus->dev, fw, type);
 }
 
 static inline
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
index db45da33adfd..dfcfb3333369 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c
@@ -2361,7 +2361,8 @@ done:
 
 static s32
 brcmf_cfg80211_config_default_key(struct wiphy *wiphy, struct net_device *ndev,
-				  u8 key_idx, bool unicast, bool multicast)
+				  int link_id, u8 key_idx, bool unicast,
+				  bool multicast)
 {
 	struct brcmf_if *ifp = netdev_priv(ndev);
 	struct brcmf_pub *drvr = ifp->drvr;
@@ -2395,7 +2396,8 @@ done:
 
 static s32
 brcmf_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev,
-		       u8 key_idx, bool pairwise, const u8 *mac_addr)
+		       int link_id, u8 key_idx, bool pairwise,
+		       const u8 *mac_addr)
 {
 	struct brcmf_if *ifp = netdev_priv(ndev);
 	struct brcmf_wsec_key *key;
@@ -2432,8 +2434,8 @@ brcmf_cfg80211_del_key(struct wiphy *wiphy, struct net_device *ndev,
 
 static s32
 brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev,
-		       u8 key_idx, bool pairwise, const u8 *mac_addr,
-		       struct key_params *params)
+		       int link_id, u8 key_idx, bool pairwise,
+		       const u8 *mac_addr, struct key_params *params)
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
 	struct brcmf_if *ifp = netdev_priv(ndev);
@@ -2457,8 +2459,8 @@ brcmf_cfg80211_add_key(struct wiphy *wiphy, struct net_device *ndev,
 	}
 
 	if (params->key_len == 0)
-		return brcmf_cfg80211_del_key(wiphy, ndev, key_idx, pairwise,
-					      mac_addr);
+		return brcmf_cfg80211_del_key(wiphy, ndev, -1, key_idx,
+					      pairwise, mac_addr);
 
 	if (params->key_len > sizeof(key->data)) {
 		bphy_err(drvr, "Too long key length (%u)\n", params->key_len);
@@ -2553,8 +2555,9 @@ done:
 }
 
 static s32
-brcmf_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev, u8 key_idx,
-		       bool pairwise, const u8 *mac_addr, void *cookie,
+brcmf_cfg80211_get_key(struct wiphy *wiphy, struct net_device *ndev,
+		       int link_id, u8 key_idx, bool pairwise,
+		       const u8 *mac_addr, void *cookie,
 		       void (*callback)(void *cookie,
 					struct key_params *params))
 {
@@ -2610,7 +2613,8 @@ done:
 
 static s32
 brcmf_cfg80211_config_default_mgmt_key(struct wiphy *wiphy,
-				       struct net_device *ndev, u8 key_idx)
+				       struct net_device *ndev, int link_id,
+				       u8 key_idx)
 {
 	struct brcmf_if *ifp = netdev_priv(ndev);
 
@@ -3160,10 +3164,7 @@ static s32 brcmf_update_bss_info(struct brcmf_cfg80211_info *cfg,
 				 struct brcmf_if *ifp)
 {
 	struct brcmf_pub *drvr = cfg->pub;
-	struct brcmf_bss_info_le *bi;
-	const struct brcmf_tlv *tim;
-	size_t ie_len;
-	u8 *ie;
+	struct brcmf_bss_info_le *bi = NULL;
 	s32 err = 0;
 
 	brcmf_dbg(TRACE, "Enter\n");
@@ -3177,29 +3178,8 @@ static s32 brcmf_update_bss_info(struct brcmf_cfg80211_info *cfg,
 		bphy_err(drvr, "Could not get bss info %d\n", err);
 		goto update_bss_info_out;
 	}
-
 	bi = (struct brcmf_bss_info_le *)(cfg->extra_buf + 4);
 	err = brcmf_inform_single_bss(cfg, bi);
-	if (err)
-		goto update_bss_info_out;
-
-	ie = ((u8 *)bi) + le16_to_cpu(bi->ie_offset);
-	ie_len = le32_to_cpu(bi->ie_length);
-
-	tim = brcmf_parse_tlvs(ie, ie_len, WLAN_EID_TIM);
-	if (!tim) {
-		/*
-		* active scan was done so we could not get dtim
-		* information out of probe response.
-		* so we speficially query dtim information to dongle.
-		*/
-		u32 var;
-		err = brcmf_fil_iovar_int_get(ifp, "dtim_assoc", &var);
-		if (err) {
-			bphy_err(drvr, "wl dtim_assoc failed (%d)\n", err);
-			goto update_bss_info_out;
-		}
-	}
 
 update_bss_info_out:
 	brcmf_dbg(TRACE, "Exit");
@@ -3984,7 +3964,6 @@ brcmf_update_pmklist(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp)
 	struct brcmf_pmk_list_le *pmk_list;
 	int i;
 	u32 npmk;
-	s32 err;
 
 	pmk_list = &cfg->pmk_list;
 	npmk = le32_to_cpu(pmk_list->npmk);
@@ -3993,10 +3972,8 @@ brcmf_update_pmklist(struct brcmf_cfg80211_info *cfg, struct brcmf_if *ifp)
 	for (i = 0; i < npmk; i++)
 		brcmf_dbg(CONN, "PMK[%d]: %pM\n", i, &pmk_list->pmk[i].bssid);
 
-	err = brcmf_fil_iovar_data_set(ifp, "pmkid_info", pmk_list,
-				       sizeof(*pmk_list));
-
-	return err;
+	return brcmf_fil_iovar_data_set(ifp, "pmkid_info", pmk_list,
+			sizeof(*pmk_list));
 }
 
 static s32
@@ -5042,13 +5019,10 @@ brcmf_cfg80211_change_beacon(struct wiphy *wiphy, struct net_device *ndev,
 			     struct cfg80211_beacon_data *info)
 {
 	struct brcmf_if *ifp = netdev_priv(ndev);
-	s32 err;
 
 	brcmf_dbg(TRACE, "Enter\n");
 
-	err = brcmf_config_ap_mgmt_ie(ifp->vif, info);
-
-	return err;
+	return brcmf_config_ap_mgmt_ie(ifp->vif, info);
 }
 
 static int
@@ -6431,6 +6405,7 @@ static void wl_deinit_priv(struct brcmf_cfg80211_info *cfg)
 	cfg->dongle_up = false;	/* dongle down */
 	brcmf_abort_scanning(cfg);
 	brcmf_deinit_priv_mem(cfg);
+	brcmf_clear_assoc_ies(cfg);
 }
 
 static void init_vif_event(struct brcmf_cfg80211_vif_event *event)
@@ -7485,6 +7460,7 @@ static bool brmcf_use_iso3166_ccode_fallback(struct brcmf_pub *drvr)
 		return true;
 
 	switch (drvr->bus_if->chip) {
+	case BRCM_CC_43430_CHIP_ID:
 	case BRCM_CC_4345_CHIP_ID:
 	case BRCM_CC_43602_CHIP_ID:
 		return true;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
index 4ec7773b6906..121893bbaa1d 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/chip.c
@@ -641,6 +641,7 @@ static void brcmf_chip_socram_ramsize(struct brcmf_core_priv *sr, u32 *ramsize,
 			*srsize = (32 * 1024);
 		break;
 	case BRCM_CC_43430_CHIP_ID:
+	case CY_CC_43439_CHIP_ID:
 		/* assume sr for now as we can not check
 		 * firmware sr capability at this point.
 		 */
@@ -732,6 +733,10 @@ static u32 brcmf_chip_tcm_rambase(struct brcmf_chip_priv *ci)
 		return 0x160000;
 	case CY_CC_43752_CHIP_ID:
 		return 0x170000;
+	case BRCM_CC_4378_CHIP_ID:
+		return 0x352000;
+	case CY_CC_89459_CHIP_ID:
+		return ((ci->pub.chiprev < 9) ? 0x180000 : 0x160000);
 	default:
 		brcmf_err("unknown chip: %s\n", ci->pub.name);
 		break;
@@ -1258,7 +1263,8 @@ brcmf_chip_cm3_set_passive(struct brcmf_chip_priv *chip)
 	brcmf_chip_resetcore(core, 0, 0, 0);
 
 	/* disable bank #3 remap for this device */
-	if (chip->pub.chip == BRCM_CC_43430_CHIP_ID) {
+	if (chip->pub.chip == BRCM_CC_43430_CHIP_ID ||
+	    chip->pub.chip == CY_CC_43439_CHIP_ID) {
 		sr = container_of(core, struct brcmf_core_priv, pub);
 		brcmf_chip_core_write32(sr, SOCRAMREGOFFS(bankidx), 3);
 		brcmf_chip_core_write32(sr, SOCRAMREGOFFS(bankpda), 0);
@@ -1416,10 +1422,12 @@ bool brcmf_chip_sr_capable(struct brcmf_chip *pub)
 		reg = chip->ops->read32(chip->ctx, addr);
 		return (reg & pmu_cc3_mask) != 0;
 	case BRCM_CC_43430_CHIP_ID:
+	case CY_CC_43439_CHIP_ID:
 		addr = CORE_CC_REG(base, sr_control1);
 		reg = chip->ops->read32(chip->ctx, addr);
 		return reg != 0;
 	case CY_CC_4373_CHIP_ID:
+	case CY_CC_89459_CHIP_ID:
 		/* explicitly check SR engine enable bit */
 		addr = CORE_CC_REG(base, sr_control0);
 		reg = chip->ops->read32(chip->ctx, addr);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
index 7485e784be2a..74020fa10065 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.c
@@ -123,7 +123,6 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp)
 	struct brcmf_bus *bus = drvr->bus_if;
 	struct brcmf_dload_data_le *chunk_buf;
 	const struct firmware *clm = NULL;
-	u8 clm_name[BRCMF_FW_NAME_LEN];
 	u32 chunk_len;
 	u32 datalen;
 	u32 cumulative_len;
@@ -133,15 +132,8 @@ static int brcmf_c_process_clm_blob(struct brcmf_if *ifp)
 
 	brcmf_dbg(TRACE, "Enter\n");
 
-	memset(clm_name, 0, sizeof(clm_name));
-	err = brcmf_bus_get_fwname(bus, ".clm_blob", clm_name);
-	if (err) {
-		bphy_err(drvr, "get CLM blob file name failed (%d)\n", err);
-		return err;
-	}
-
-	err = firmware_request_nowarn(&clm, clm_name, bus->dev);
-	if (err) {
+	err = brcmf_bus_get_blob(bus, &clm, BRCMF_BLOB_CLM);
+	if (err || !clm) {
 		brcmf_info("no clm_blob available (err=%d), device may have limited channels available\n",
 			   err);
 		return 0;
@@ -261,7 +253,7 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp)
 				     &revinfo, sizeof(revinfo));
 	if (err < 0) {
 		bphy_err(drvr, "retrieving revision info failed, %d\n", err);
-		strlcpy(ri->chipname, "UNKNOWN", sizeof(ri->chipname));
+		strscpy(ri->chipname, "UNKNOWN", sizeof(ri->chipname));
 	} else {
 		ri->vendorid = le32_to_cpu(revinfo.vendorid);
 		ri->deviceid = le32_to_cpu(revinfo.deviceid);
@@ -314,7 +306,7 @@ int brcmf_c_preinit_dcmds(struct brcmf_if *ifp)
 
 	/* locate firmware version number for ethtool */
 	ptr = strrchr(buf, ' ') + 1;
-	strlcpy(ifp->drvr->fwver, ptr, sizeof(ifp->drvr->fwver));
+	strscpy(ifp->drvr->fwver, ptr, sizeof(ifp->drvr->fwver));
 
 	/* Query for 'clmver' to get CLM version info from firmware */
 	memset(buf, 0, sizeof(buf));
@@ -424,11 +416,11 @@ static void brcmf_mp_attach(void)
 	 * if not set then if available use the platform data version. To make
 	 * sure it gets initialized at all, always copy the module param version
 	 */
-	strlcpy(brcmf_mp_global.firmware_path, brcmf_firmware_path,
+	strscpy(brcmf_mp_global.firmware_path, brcmf_firmware_path,
 		BRCMF_FW_ALTPATH_LEN);
 	if ((brcmfmac_pdata) && (brcmfmac_pdata->fw_alternative_path) &&
 	    (brcmf_mp_global.firmware_path[0] == '\0')) {
-		strlcpy(brcmf_mp_global.firmware_path,
+		strscpy(brcmf_mp_global.firmware_path,
 			brcmfmac_pdata->fw_alternative_path,
 			BRCMF_FW_ALTPATH_LEN);
 	}
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
index 6c5a22a32a96..aa25abffcc7d 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/common.h
@@ -53,6 +53,7 @@ struct brcmf_mp_device {
 	struct brcmfmac_pd_cc *country_codes;
 	const char	*board_type;
 	unsigned char	mac[ETH_ALEN];
+	const char	*antenna_sku;
 	union {
 		struct brcmfmac_sdio_pd sdio;
 	} bus;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
index bd164a0821f9..595ae3ae561e 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/core.c
@@ -292,6 +292,7 @@ static netdev_tx_t brcmf_netdev_start_xmit(struct sk_buff *skb,
 	struct brcmf_pub *drvr = ifp->drvr;
 	struct ethhdr *eh;
 	int head_delta;
+	unsigned int tx_bytes = skb->len;
 
 	brcmf_dbg(DATA, "Enter, bsscfgidx=%d\n", ifp->bsscfgidx);
 
@@ -366,7 +367,7 @@ done:
 		ndev->stats.tx_dropped++;
 	} else {
 		ndev->stats.tx_packets++;
-		ndev->stats.tx_bytes += skb->len;
+		ndev->stats.tx_bytes += tx_bytes;
 	}
 
 	/* Return ok: we always eat the packet */
@@ -561,10 +562,10 @@ static void brcmf_ethtool_get_drvinfo(struct net_device *ndev,
 
 	if (drvr->revinfo.result == 0)
 		brcmu_dotrev_str(drvr->revinfo.driverrev, drev);
-	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
-	strlcpy(info->version, drev, sizeof(info->version));
-	strlcpy(info->fw_version, drvr->fwver, sizeof(info->fw_version));
-	strlcpy(info->bus_info, dev_name(drvr->bus_if->dev),
+	strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
+	strscpy(info->version, drev, sizeof(info->version));
+	strscpy(info->fw_version, drvr->fwver, sizeof(info->fw_version));
+	strscpy(info->bus_info, dev_name(drvr->bus_if->dev),
 		sizeof(info->bus_info));
 }
 
@@ -1480,8 +1481,10 @@ int brcmf_netdev_wait_pend8021x(struct brcmf_if *ifp)
 				 !brcmf_get_pend_8021x_cnt(ifp),
 				 MAX_WAIT_FOR_8021X_TX);
 
-	if (!err)
+	if (!err) {
 		bphy_err(drvr, "Timed out waiting for no pending 802.1x packets\n");
+		atomic_set(&ifp->pend_8021x_cnt, 0);
+	}
 
 	return !err;
 }
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c
index 0af452dca766..86ff174936a9 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/dmi.c
@@ -24,6 +24,13 @@ static const struct brcmf_dmi_data acepc_t8_data = {
 	BRCM_CC_4345_CHIP_ID, 6, "acepc-t8"
 };
 
+/* The Chuwi Hi8 Pro uses the same Ampak AP6212 module as the Chuwi Vi8 Plus
+ * and the nvram for the Vi8 Plus is already in linux-firmware, so use that.
+ */
+static const struct brcmf_dmi_data chuwi_hi8_pro_data = {
+	BRCM_CC_43430_CHIP_ID, 0, "ilife-S806"
+};
+
 static const struct brcmf_dmi_data gpd_win_pocket_data = {
 	BRCM_CC_4356_CHIP_ID, 2, "gpd-win-pocket"
 };
@@ -76,6 +83,17 @@ static const struct dmi_system_id dmi_platform_data[] = {
 		.driver_data = (void *)&acepc_t8_data,
 	},
 	{
+		/* Chuwi Hi8 Pro with D2D3_Hi8Pro.233 BIOS */
+		.matches = {
+			DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "Hampoo"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "Cherry Trail CR"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "MRD"),
+			/* Above strings are too generic, also match on BIOS date */
+			DMI_MATCH(DMI_BIOS_DATE, "05/10/2016"),
+		},
+		.driver_data = (void *)&chuwi_hi8_pro_data,
+	},
+	{
 		/* Cyberbook T116 rugged tablet */
 		.matches = {
 			DMI_EXACT_MATCH(DMI_BOARD_VENDOR, "Default string"),
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
index d2ac844e1e9f..2c2f3e026c13 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/feature.c
@@ -249,7 +249,8 @@ void brcmf_feat_attach(struct brcmf_pub *drvr)
 	memset(&gscan_cfg, 0, sizeof(gscan_cfg));
 	if (drvr->bus_if->chip != BRCM_CC_43430_CHIP_ID &&
 	    drvr->bus_if->chip != BRCM_CC_4345_CHIP_ID &&
-	    drvr->bus_if->chip != BRCM_CC_43454_CHIP_ID)
+	    drvr->bus_if->chip != BRCM_CC_43454_CHIP_ID &&
+	    drvr->bus_if->chip != CY_CC_43439_CHIP_ID)
 		brcmf_feat_iovar_data_set(ifp, BRCMF_FEAT_GSCAN,
 					  "pfn_gscan_cfg",
 					  &gscan_cfg, sizeof(gscan_cfg));
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
index b8379e4034a4..f2207793f6e2 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.c
@@ -21,6 +21,8 @@
 #define BRCMF_FW_NVRAM_DEVPATH_LEN		19	/* devpath0=pcie/1/4/ */
 #define BRCMF_FW_NVRAM_PCIEDEV_LEN		10	/* pcie/1/4/ + \0 */
 #define BRCMF_FW_DEFAULT_BOARDREV		"boardrev=0xff"
+#define BRCMF_FW_MACADDR_FMT			"macaddr=%pM"
+#define BRCMF_FW_MACADDR_LEN			(7 + ETH_ALEN * 3)
 
 enum nvram_parser_state {
 	IDLE,
@@ -44,6 +46,7 @@ enum nvram_parser_state {
  * @multi_dev_v1: detect pcie multi device v1 (compressed).
  * @multi_dev_v2: detect pcie multi device v2.
  * @boardrev_found: nvram contains boardrev information.
+ * @strip_mac: strip the MAC address.
  */
 struct nvram_parser {
 	enum nvram_parser_state state;
@@ -57,6 +60,7 @@ struct nvram_parser {
 	bool multi_dev_v1;
 	bool multi_dev_v2;
 	bool boardrev_found;
+	bool strip_mac;
 };
 
 /*
@@ -121,6 +125,10 @@ static enum nvram_parser_state brcmf_nvram_handle_key(struct nvram_parser *nvp)
 			nvp->multi_dev_v2 = true;
 		if (strncmp(&nvp->data[nvp->entry], "boardrev", 8) == 0)
 			nvp->boardrev_found = true;
+		/* strip macaddr if platform MAC overrides */
+		if (nvp->strip_mac &&
+		    strncmp(&nvp->data[nvp->entry], "macaddr", 7) == 0)
+			st = COMMENT;
 	} else if (!is_nvram_char(c) || c == ' ') {
 		brcmf_dbg(INFO, "warning: ln=%d:col=%d: '=' expected, skip invalid key entry\n",
 			  nvp->line, nvp->column);
@@ -209,6 +217,7 @@ static int brcmf_init_nvram_parser(struct nvram_parser *nvp,
 		size = data_len;
 	/* Add space for properties we may add */
 	size += strlen(BRCMF_FW_DEFAULT_BOARDREV) + 1;
+	size += BRCMF_FW_MACADDR_LEN + 1;
 	/* Alloc for extra 0 byte + roundup by 4 + length field */
 	size += 1 + 3 + sizeof(u32);
 	nvp->nvram = kzalloc(size, GFP_KERNEL);
@@ -368,22 +377,37 @@ static void brcmf_fw_add_defaults(struct nvram_parser *nvp)
 	nvp->nvram_len++;
 }
 
+static void brcmf_fw_add_macaddr(struct nvram_parser *nvp, u8 *mac)
+{
+	int len;
+
+	len = scnprintf(&nvp->nvram[nvp->nvram_len], BRCMF_FW_MACADDR_LEN + 1,
+			BRCMF_FW_MACADDR_FMT, mac);
+	WARN_ON(len != BRCMF_FW_MACADDR_LEN);
+	nvp->nvram_len += len + 1;
+}
+
 /* brcmf_nvram_strip :Takes a buffer of "<var>=<value>\n" lines read from a fil
  * and ending in a NUL. Removes carriage returns, empty lines, comment lines,
  * and converts newlines to NULs. Shortens buffer as needed and pads with NULs.
  * End of buffer is completed with token identifying length of buffer.
  */
 static void *brcmf_fw_nvram_strip(const u8 *data, size_t data_len,
-				  u32 *new_length, u16 domain_nr, u16 bus_nr)
+				  u32 *new_length, u16 domain_nr, u16 bus_nr,
+				  struct device *dev)
 {
 	struct nvram_parser nvp;
 	u32 pad;
 	u32 token;
 	__le32 token_le;
+	u8 mac[ETH_ALEN];
 
 	if (brcmf_init_nvram_parser(&nvp, data, data_len) < 0)
 		return NULL;
 
+	if (eth_platform_get_mac_address(dev, mac) == 0)
+		nvp.strip_mac = true;
+
 	while (nvp.pos < data_len) {
 		nvp.state = nv_parser_states[nvp.state](&nvp);
 		if (nvp.state == END)
@@ -404,6 +428,9 @@ static void *brcmf_fw_nvram_strip(const u8 *data, size_t data_len,
 
 	brcmf_fw_add_defaults(&nvp);
 
+	if (nvp.strip_mac)
+		brcmf_fw_add_macaddr(&nvp, mac);
+
 	pad = nvp.nvram_len;
 	*new_length = roundup(nvp.nvram_len + 1, 4);
 	while (pad != *new_length) {
@@ -430,6 +457,7 @@ struct brcmf_fw {
 	struct device *dev;
 	struct brcmf_fw_request *req;
 	u32 curpos;
+	unsigned int board_index;
 	void (*done)(struct device *dev, int err, struct brcmf_fw_request *req);
 };
 
@@ -537,7 +565,8 @@ static int brcmf_fw_request_nvram_done(const struct firmware *fw, void *ctx)
 	if (data)
 		nvram = brcmf_fw_nvram_strip(data, data_len, &nvram_length,
 					     fwctx->req->domain_nr,
-					     fwctx->req->bus_nr);
+					     fwctx->req->bus_nr,
+					     fwctx->dev);
 
 	if (free_bcm47xx_nvram)
 		bcm47xx_nvram_release_contents(data);
@@ -587,39 +616,50 @@ static int brcmf_fw_complete_request(const struct firmware *fw,
 
 static char *brcm_alt_fw_path(const char *path, const char *board_type)
 {
-	char alt_path[BRCMF_FW_NAME_LEN];
-	char suffix[5];
+	char base[BRCMF_FW_NAME_LEN];
+	const char *suffix;
+	char *ret;
+
+	if (!board_type)
+		return NULL;
 
-	strscpy(alt_path, path, BRCMF_FW_NAME_LEN);
-	/* At least one character + suffix */
-	if (strlen(alt_path) < 5)
+	suffix = strrchr(path, '.');
+	if (!suffix || suffix == path)
 		return NULL;
 
-	/* strip .txt or .bin at the end */
-	strscpy(suffix, alt_path + strlen(alt_path) - 4, 5);
-	alt_path[strlen(alt_path) - 4] = 0;
-	strlcat(alt_path, ".", BRCMF_FW_NAME_LEN);
-	strlcat(alt_path, board_type, BRCMF_FW_NAME_LEN);
-	strlcat(alt_path, suffix, BRCMF_FW_NAME_LEN);
+	/* strip extension at the end */
+	strscpy(base, path, BRCMF_FW_NAME_LEN);
+	base[suffix - path] = 0;
 
-	return kstrdup(alt_path, GFP_KERNEL);
+	ret = kasprintf(GFP_KERNEL, "%s.%s%s", base, board_type, suffix);
+	if (!ret)
+		brcmf_err("out of memory allocating firmware path for '%s'\n",
+			  path);
+
+	brcmf_dbg(TRACE, "FW alt path: %s\n", ret);
+
+	return ret;
 }
 
 static int brcmf_fw_request_firmware(const struct firmware **fw,
 				     struct brcmf_fw *fwctx)
 {
 	struct brcmf_fw_item *cur = &fwctx->req->items[fwctx->curpos];
+	unsigned int i;
 	int ret;
 
-	/* Files can be board-specific, first try a board-specific path */
-	if (cur->type == BRCMF_FW_TYPE_NVRAM && fwctx->req->board_type) {
+	/* Files can be board-specific, first try board-specific paths */
+	for (i = 0; i < ARRAY_SIZE(fwctx->req->board_types); i++) {
 		char *alt_path;
 
-		alt_path = brcm_alt_fw_path(cur->path, fwctx->req->board_type);
+		if (!fwctx->req->board_types[i])
+			goto fallback;
+		alt_path = brcm_alt_fw_path(cur->path,
+					    fwctx->req->board_types[i]);
 		if (!alt_path)
 			goto fallback;
 
-		ret = request_firmware(fw, alt_path, fwctx->dev);
+		ret = firmware_request_nowarn(fw, alt_path, fwctx->dev);
 		kfree(alt_path);
 		if (ret == 0)
 			return ret;
@@ -653,15 +693,40 @@ static void brcmf_fw_request_done_alt_path(const struct firmware *fw, void *ctx)
 {
 	struct brcmf_fw *fwctx = ctx;
 	struct brcmf_fw_item *first = &fwctx->req->items[0];
+	const char *board_type, *alt_path;
 	int ret = 0;
 
-	/* Fall back to canonical path if board firmware not found */
-	if (!fw)
-		ret = request_firmware_nowait(THIS_MODULE, true, first->path,
+	if (fw) {
+		brcmf_fw_request_done(fw, ctx);
+		return;
+	}
+
+	/* Try next board firmware */
+	if (fwctx->board_index < ARRAY_SIZE(fwctx->req->board_types)) {
+		board_type = fwctx->req->board_types[fwctx->board_index++];
+		if (!board_type)
+			goto fallback;
+		alt_path = brcm_alt_fw_path(first->path, board_type);
+		if (!alt_path)
+			goto fallback;
+
+		ret = request_firmware_nowait(THIS_MODULE, true, alt_path,
 					      fwctx->dev, GFP_KERNEL, fwctx,
-					      brcmf_fw_request_done);
+					      brcmf_fw_request_done_alt_path);
+		kfree(alt_path);
+
+		if (ret < 0)
+			brcmf_fw_request_done(fw, ctx);
+		return;
+	}
 
-	if (fw || ret < 0)
+fallback:
+	/* Fall back to canonical path if board firmware not found */
+	ret = request_firmware_nowait(THIS_MODULE, true, first->path,
+				      fwctx->dev, GFP_KERNEL, fwctx,
+				      brcmf_fw_request_done);
+
+	if (ret < 0)
 		brcmf_fw_request_done(fw, ctx);
 }
 
@@ -705,10 +770,11 @@ int brcmf_fw_get_firmwares(struct device *dev, struct brcmf_fw_request *req,
 	fwctx->done = fw_cb;
 
 	/* First try alternative board-specific path if any */
-	if (fwctx->req->board_type)
+	if (fwctx->req->board_types[0])
 		alt_path = brcm_alt_fw_path(first->path,
-					    fwctx->req->board_type);
+					    fwctx->req->board_types[0]);
 	if (alt_path) {
+		fwctx->board_index++;
 		ret = request_firmware_nowait(THIS_MODULE, true, alt_path,
 					      fwctx->dev, GFP_KERNEL, fwctx,
 					      brcmf_fw_request_done_alt_path);
@@ -769,7 +835,7 @@ brcmf_fw_alloc_request(u32 chip, u32 chiprev,
 		fwnames[j].path[0] = '\0';
 		/* check if firmware path is provided by module parameter */
 		if (brcmf_mp_global.firmware_path[0] != '\0') {
-			strlcpy(fwnames[j].path, mp_path,
+			strscpy(fwnames[j].path, mp_path,
 				BRCMF_FW_NAME_LEN);
 
 			if (end != '/') {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h
index e290dec9c53d..1266cbaee072 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/firmware.h
@@ -11,6 +11,8 @@
 
 #define BRCMF_FW_DEFAULT_PATH		"brcm/"
 
+#define BRCMF_FW_MAX_BOARD_TYPES	8
+
 /**
  * struct brcmf_firmware_mapping - Used to map chipid/revmask to firmware
  *	filename and nvram filename. Each bus type implementation should create
@@ -66,7 +68,7 @@ struct brcmf_fw_request {
 	u16 domain_nr;
 	u16 bus_nr;
 	u32 n_items;
-	const char *board_type;
+	const char *board_types[BRCMF_FW_MAX_BOARD_TYPES];
 	struct brcmf_fw_item items[];
 };
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c
index 096f6b969dd8..e1127d7e086d 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/flowring.c
@@ -419,7 +419,6 @@ void brcmf_flowring_configure_addr_mode(struct brcmf_flowring *flow, int ifidx,
 				flowid = flow->hash[i].flowid;
 				if (flow->rings[flowid]->status != RING_OPEN)
 					continue;
-				flow->rings[flowid]->status = RING_CLOSING;
 				brcmf_msgbuf_delete_flowring(drvr, flowid);
 			}
 		}
@@ -458,10 +457,8 @@ void brcmf_flowring_delete_peer(struct brcmf_flowring *flow, int ifidx,
 		if ((sta || (memcmp(hash[i].mac, peer, ETH_ALEN) == 0)) &&
 		    (hash[i].ifidx == ifidx)) {
 			flowid = flow->hash[i].flowid;
-			if (flow->rings[flowid]->status == RING_OPEN) {
-				flow->rings[flowid]->status = RING_CLOSING;
+			if (flow->rings[flowid]->status == RING_OPEN)
 				brcmf_msgbuf_delete_flowring(drvr, flowid);
-			}
 		}
 	}
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h
index c87b829adb0d..f518e025d6e4 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h
@@ -135,7 +135,7 @@
 /* Link Down indication in WoWL mode: */
 #define BRCMF_WOWL_LINKDOWN		(1 << 31)
 
-#define BRCMF_WOWL_MAXPATTERNS		8
+#define BRCMF_WOWL_MAXPATTERNS		16
 #define BRCMF_WOWL_MAXPATTERNSIZE	128
 
 #define BRCMF_COUNTRY_BUF_SZ		4
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c
index d58525ebe618..36af81975855 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.c
@@ -688,7 +688,7 @@ static void brcmf_fws_macdesc_set_name(struct brcmf_fws_info *fws,
 				       struct brcmf_fws_mac_descriptor *desc)
 {
 	if (desc == &fws->desc.other)
-		strlcpy(desc->name, "MAC-OTHER", sizeof(desc->name));
+		strscpy(desc->name, "MAC-OTHER", sizeof(desc->name));
 	else if (desc->mac_handle)
 		scnprintf(desc->name, sizeof(desc->name), "MAC-%d:%d",
 			  desc->mac_handle, desc->interface_id);
@@ -2475,7 +2475,8 @@ bool brcmf_fws_fc_active(struct brcmf_fws_info *fws)
 	return fws->fcmode != BRCMF_FWS_FCMODE_NONE;
 }
 
-void brcmf_fws_bustxfail(struct brcmf_fws_info *fws, struct sk_buff *skb)
+void brcmf_fws_bustxcomplete(struct brcmf_fws_info *fws, struct sk_buff *skb,
+			     bool success)
 {
 	u32 hslot;
 
@@ -2483,11 +2484,14 @@ void brcmf_fws_bustxfail(struct brcmf_fws_info *fws, struct sk_buff *skb)
 		brcmu_pkt_buf_free_skb(skb);
 		return;
 	}
-	brcmf_fws_lock(fws);
-	hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT);
-	brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, hslot, 0, 0,
-			      1);
-	brcmf_fws_unlock(fws);
+
+	if (!success) {
+		brcmf_fws_lock(fws);
+		hslot = brcmf_skb_htod_tag_get_field(skb, HSLOT);
+		brcmf_fws_txs_process(fws, BRCMF_FWS_TXSTATUS_HOST_TOSSED, hslot,
+				      0, 0, 1);
+		brcmf_fws_unlock(fws);
+	}
 }
 
 void brcmf_fws_bus_blocked(struct brcmf_pub *drvr, bool flow_blocked)
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h
index b16a9d1c0508..f9c36cd8f1de 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwsignal.h
@@ -40,7 +40,8 @@ int brcmf_fws_process_skb(struct brcmf_if *ifp, struct sk_buff *skb);
 void brcmf_fws_reset_interface(struct brcmf_if *ifp);
 void brcmf_fws_add_interface(struct brcmf_if *ifp);
 void brcmf_fws_del_interface(struct brcmf_if *ifp);
-void brcmf_fws_bustxfail(struct brcmf_fws_info *fws, struct sk_buff *skb);
+void brcmf_fws_bustxcomplete(struct brcmf_fws_info *fws, struct sk_buff *skb,
+			     bool success);
 void brcmf_fws_bus_blocked(struct brcmf_pub *drvr, bool flow_blocked);
 void brcmf_fws_rxreorder(struct brcmf_if *ifp, struct sk_buff *skb);
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c
index b2d0f7570aa9..cec53f934940 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.c
@@ -71,6 +71,7 @@
 #define BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS	32
 #define BRCMF_MSGBUF_UPDATE_RX_PTR_THRS		48
 
+#define BRCMF_MAX_TXSTATUS_WAIT_RETRIES		10
 
 struct msgbuf_common_hdr {
 	u8				msgtype;
@@ -806,8 +807,12 @@ static int brcmf_msgbuf_tx_queue_data(struct brcmf_pub *drvr, int ifidx,
 	flowid = brcmf_flowring_lookup(flow, eh->h_dest, skb->priority, ifidx);
 	if (flowid == BRCMF_FLOWRING_INVALID_ID) {
 		flowid = brcmf_msgbuf_flowring_create(msgbuf, ifidx, skb);
-		if (flowid == BRCMF_FLOWRING_INVALID_ID)
+		if (flowid == BRCMF_FLOWRING_INVALID_ID) {
 			return -ENOMEM;
+		} else {
+			brcmf_flowring_enqueue(flow, flowid, skb);
+			return 0;
+		}
 	}
 	queue_count = brcmf_flowring_enqueue(flow, flowid, skb);
 	force = ((queue_count % BRCMF_MSGBUF_TRICKLE_TXWORKER_THRS) == 0);
@@ -1395,9 +1400,27 @@ void brcmf_msgbuf_delete_flowring(struct brcmf_pub *drvr, u16 flowid)
 	struct brcmf_msgbuf *msgbuf = (struct brcmf_msgbuf *)drvr->proto->pd;
 	struct msgbuf_tx_flowring_delete_req *delete;
 	struct brcmf_commonring *commonring;
+	struct brcmf_commonring *commonring_del = msgbuf->flowrings[flowid];
+	struct brcmf_flowring *flow = msgbuf->flow;
 	void *ret_ptr;
 	u8 ifidx;
 	int err;
+	int retry = BRCMF_MAX_TXSTATUS_WAIT_RETRIES;
+
+	/* make sure it is not in txflow */
+	brcmf_commonring_lock(commonring_del);
+	flow->rings[flowid]->status = RING_CLOSING;
+	brcmf_commonring_unlock(commonring_del);
+
+	/* wait for commonring txflow finished */
+	while (retry && atomic_read(&commonring_del->outstanding_tx)) {
+		usleep_range(5000, 10000);
+		retry--;
+	}
+	if (!retry) {
+		brcmf_err("timed out waiting for txstatus\n");
+		atomic_set(&commonring_del->outstanding_tx, 0);
+	}
 
 	/* no need to submit if firmware can not be reached */
 	if (drvr->bus_if->state != BRCMF_BUS_UP) {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h
index 2e322edbb907..6a849f4a94dd 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/msgbuf.h
@@ -8,10 +8,10 @@
 #ifdef CONFIG_BRCMFMAC_PROTO_MSGBUF
 
 #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_MAX_ITEM	64
-#define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_MAX_ITEM	512
+#define BRCMF_H2D_MSGRING_RXPOST_SUBMIT_MAX_ITEM	1024
 #define BRCMF_D2H_MSGRING_CONTROL_COMPLETE_MAX_ITEM	64
 #define BRCMF_D2H_MSGRING_TX_COMPLETE_MAX_ITEM		1024
-#define BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM		512
+#define BRCMF_D2H_MSGRING_RX_COMPLETE_MAX_ITEM		1024
 #define BRCMF_H2D_TXFLOWRING_MAX_ITEM			512
 
 #define BRCMF_H2D_MSGRING_CONTROL_SUBMIT_ITEMSIZE	40
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c
index 79388d49c256..a83699de01ec 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/of.c
@@ -70,14 +70,24 @@ void brcmf_of_probe(struct device *dev, enum brcmf_bus_type bus_type,
 {
 	struct brcmfmac_sdio_pd *sdio = &settings->bus.sdio;
 	struct device_node *root, *np = dev->of_node;
+	const char *prop;
 	int irq;
 	int err;
 	u32 irqf;
 	u32 val;
 
+	/* Apple ARM64 platforms have their own idea of board type, passed in
+	 * via the device tree. They also have an antenna SKU parameter
+	 */
+	if (!of_property_read_string(np, "brcm,board-type", &prop))
+		settings->board_type = prop;
+
+	if (!of_property_read_string(np, "apple,antenna-sku", &prop))
+		settings->antenna_sku = prop;
+
 	/* Set board-type to the first string of the machine compatible prop */
 	root = of_find_node_by_path("/");
-	if (root) {
+	if (root && !settings->board_type) {
 		char *board_type;
 		const char *tmp;
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
index 97f0f13dfe50..80083f9ea311 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
@@ -59,6 +59,8 @@ BRCMF_FW_DEF(4365C, "brcmfmac4365c-pcie");
 BRCMF_FW_DEF(4366B, "brcmfmac4366b-pcie");
 BRCMF_FW_DEF(4366C, "brcmfmac4366c-pcie");
 BRCMF_FW_DEF(4371, "brcmfmac4371-pcie");
+BRCMF_FW_CLM_DEF(4378B1, "brcmfmac4378b1-pcie");
+BRCMF_FW_DEF(4355, "brcmfmac89459-pcie");
 
 /* firmware config files */
 MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.txt");
@@ -66,6 +68,7 @@ MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.txt");
 
 /* per-board firmware binaries */
 MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.bin");
+MODULE_FIRMWARE(BRCMF_FW_DEFAULT_PATH "brcmfmac*-pcie.*.clm_blob");
 
 static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43602_CHIP_ID, 0xFFFFFFFF, 43602),
@@ -87,6 +90,8 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_43664_CHIP_ID, 0xFFFFFFF0, 4366C),
 	BRCMF_FW_ENTRY(BRCM_CC_43666_CHIP_ID, 0xFFFFFFF0, 4366C),
 	BRCMF_FW_ENTRY(BRCM_CC_4371_CHIP_ID, 0xFFFFFFFF, 4371),
+	BRCMF_FW_ENTRY(BRCM_CC_4378_CHIP_ID, 0xFFFFFFFF, 4378B1), /* revision ID 3 */
+	BRCMF_FW_ENTRY(CY_CC_89459_CHIP_ID, 0xFFFFFFFF, 4355),
 };
 
 #define BRCMF_PCIE_FW_UP_TIMEOUT		5000 /* msec */
@@ -118,6 +123,12 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 #define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0	0x140
 #define BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1	0x144
 
+#define BRCMF_PCIE_64_PCIE2REG_INTMASK		0xC14
+#define BRCMF_PCIE_64_PCIE2REG_MAILBOXINT	0xC30
+#define BRCMF_PCIE_64_PCIE2REG_MAILBOXMASK	0xC34
+#define BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0	0xA20
+#define BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1	0xA24
+
 #define BRCMF_PCIE2_INTA			0x01
 #define BRCMF_PCIE2_INTB			0x02
 
@@ -137,6 +148,8 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 #define	BRCMF_PCIE_MB_INT_D2H3_DB0		0x400000
 #define	BRCMF_PCIE_MB_INT_D2H3_DB1		0x800000
 
+#define BRCMF_PCIE_MB_INT_FN0			(BRCMF_PCIE_MB_INT_FN0_0 | \
+						 BRCMF_PCIE_MB_INT_FN0_1)
 #define BRCMF_PCIE_MB_INT_D2H_DB		(BRCMF_PCIE_MB_INT_D2H0_DB0 | \
 						 BRCMF_PCIE_MB_INT_D2H0_DB1 | \
 						 BRCMF_PCIE_MB_INT_D2H1_DB0 | \
@@ -146,6 +159,40 @@ static const struct brcmf_firmware_mapping brcmf_pcie_fwnames[] = {
 						 BRCMF_PCIE_MB_INT_D2H3_DB0 | \
 						 BRCMF_PCIE_MB_INT_D2H3_DB1)
 
+#define	BRCMF_PCIE_64_MB_INT_D2H0_DB0		0x1
+#define	BRCMF_PCIE_64_MB_INT_D2H0_DB1		0x2
+#define	BRCMF_PCIE_64_MB_INT_D2H1_DB0		0x4
+#define	BRCMF_PCIE_64_MB_INT_D2H1_DB1		0x8
+#define	BRCMF_PCIE_64_MB_INT_D2H2_DB0		0x10
+#define	BRCMF_PCIE_64_MB_INT_D2H2_DB1		0x20
+#define	BRCMF_PCIE_64_MB_INT_D2H3_DB0		0x40
+#define	BRCMF_PCIE_64_MB_INT_D2H3_DB1		0x80
+#define	BRCMF_PCIE_64_MB_INT_D2H4_DB0		0x100
+#define	BRCMF_PCIE_64_MB_INT_D2H4_DB1		0x200
+#define	BRCMF_PCIE_64_MB_INT_D2H5_DB0		0x400
+#define	BRCMF_PCIE_64_MB_INT_D2H5_DB1		0x800
+#define	BRCMF_PCIE_64_MB_INT_D2H6_DB0		0x1000
+#define	BRCMF_PCIE_64_MB_INT_D2H6_DB1		0x2000
+#define	BRCMF_PCIE_64_MB_INT_D2H7_DB0		0x4000
+#define	BRCMF_PCIE_64_MB_INT_D2H7_DB1		0x8000
+
+#define BRCMF_PCIE_64_MB_INT_D2H_DB		(BRCMF_PCIE_64_MB_INT_D2H0_DB0 | \
+						 BRCMF_PCIE_64_MB_INT_D2H0_DB1 | \
+						 BRCMF_PCIE_64_MB_INT_D2H1_DB0 | \
+						 BRCMF_PCIE_64_MB_INT_D2H1_DB1 | \
+						 BRCMF_PCIE_64_MB_INT_D2H2_DB0 | \
+						 BRCMF_PCIE_64_MB_INT_D2H2_DB1 | \
+						 BRCMF_PCIE_64_MB_INT_D2H3_DB0 | \
+						 BRCMF_PCIE_64_MB_INT_D2H3_DB1 | \
+						 BRCMF_PCIE_64_MB_INT_D2H4_DB0 | \
+						 BRCMF_PCIE_64_MB_INT_D2H4_DB1 | \
+						 BRCMF_PCIE_64_MB_INT_D2H5_DB0 | \
+						 BRCMF_PCIE_64_MB_INT_D2H5_DB1 | \
+						 BRCMF_PCIE_64_MB_INT_D2H6_DB0 | \
+						 BRCMF_PCIE_64_MB_INT_D2H6_DB1 | \
+						 BRCMF_PCIE_64_MB_INT_D2H7_DB0 | \
+						 BRCMF_PCIE_64_MB_INT_D2H7_DB1)
+
 #define BRCMF_PCIE_SHARED_VERSION_7		7
 #define BRCMF_PCIE_MIN_SHARED_VERSION		5
 #define BRCMF_PCIE_MAX_SHARED_VERSION		BRCMF_PCIE_SHARED_VERSION_7
@@ -255,12 +302,24 @@ struct brcmf_pcie_core_info {
 	u32 wrapbase;
 };
 
+#define BRCMF_OTP_MAX_PARAM_LEN 16
+
+struct brcmf_otp_params {
+	char module[BRCMF_OTP_MAX_PARAM_LEN];
+	char vendor[BRCMF_OTP_MAX_PARAM_LEN];
+	char version[BRCMF_OTP_MAX_PARAM_LEN];
+	bool valid;
+};
+
 struct brcmf_pciedev_info {
 	enum brcmf_pcie_state state;
 	bool in_irq;
 	struct pci_dev *pdev;
 	char fw_name[BRCMF_FW_NAME_LEN];
 	char nvram_name[BRCMF_FW_NAME_LEN];
+	char clm_name[BRCMF_FW_NAME_LEN];
+	const struct firmware *clm_fw;
+	const struct brcmf_pcie_reginfo *reginfo;
 	void __iomem *regs;
 	void __iomem *tcm;
 	u32 ram_base;
@@ -280,6 +339,7 @@ struct brcmf_pciedev_info {
 	void (*write_ptr)(struct brcmf_pciedev_info *devinfo, u32 mem_offset,
 			  u16 value);
 	struct brcmf_mp_device *settings;
+	struct brcmf_otp_params otp;
 };
 
 struct brcmf_pcie_ringbuf {
@@ -346,11 +406,49 @@ static const u32 brcmf_ring_itemsize[BRCMF_NROF_COMMON_MSGRINGS] = {
 	BRCMF_D2H_MSGRING_RX_COMPLETE_ITEMSIZE
 };
 
+struct brcmf_pcie_reginfo {
+	u32 intmask;
+	u32 mailboxint;
+	u32 mailboxmask;
+	u32 h2d_mailbox_0;
+	u32 h2d_mailbox_1;
+	u32 int_d2h_db;
+	u32 int_fn0;
+};
+
+static const struct brcmf_pcie_reginfo brcmf_reginfo_default = {
+	.intmask = BRCMF_PCIE_PCIE2REG_INTMASK,
+	.mailboxint = BRCMF_PCIE_PCIE2REG_MAILBOXINT,
+	.mailboxmask = BRCMF_PCIE_PCIE2REG_MAILBOXMASK,
+	.h2d_mailbox_0 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0,
+	.h2d_mailbox_1 = BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1,
+	.int_d2h_db = BRCMF_PCIE_MB_INT_D2H_DB,
+	.int_fn0 = BRCMF_PCIE_MB_INT_FN0,
+};
+
+static const struct brcmf_pcie_reginfo brcmf_reginfo_64 = {
+	.intmask = BRCMF_PCIE_64_PCIE2REG_INTMASK,
+	.mailboxint = BRCMF_PCIE_64_PCIE2REG_MAILBOXINT,
+	.mailboxmask = BRCMF_PCIE_64_PCIE2REG_MAILBOXMASK,
+	.h2d_mailbox_0 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_0,
+	.h2d_mailbox_1 = BRCMF_PCIE_64_PCIE2REG_H2D_MAILBOX_1,
+	.int_d2h_db = BRCMF_PCIE_64_MB_INT_D2H_DB,
+	.int_fn0 = 0,
+};
+
 static void brcmf_pcie_setup(struct device *dev, int ret,
 			     struct brcmf_fw_request *fwreq);
 static struct brcmf_fw_request *
 brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo);
 
+static u16
+brcmf_pcie_read_reg16(struct brcmf_pciedev_info *devinfo, u32 reg_offset)
+{
+	void __iomem *address = devinfo->regs + reg_offset;
+
+	return ioread16(address);
+}
+
 static u32
 brcmf_pcie_read_reg32(struct brcmf_pciedev_info *devinfo, u32 reg_offset)
 {
@@ -496,6 +594,8 @@ brcmf_pcie_copy_dev_tomem(struct brcmf_pciedev_info *devinfo, u32 mem_offset,
 }
 
 
+#define READCC32(devinfo, reg) brcmf_pcie_read_reg32(devinfo, \
+		CHIPCREGOFFS(reg))
 #define WRITECC32(devinfo, reg, value) brcmf_pcie_write_reg32(devinfo, \
 		CHIPCREGOFFS(reg), value)
 
@@ -779,30 +879,29 @@ static void brcmf_pcie_bus_console_read(struct brcmf_pciedev_info *devinfo,
 
 static void brcmf_pcie_intr_disable(struct brcmf_pciedev_info *devinfo)
 {
-	brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXMASK, 0);
+	brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask, 0);
 }
 
 
 static void brcmf_pcie_intr_enable(struct brcmf_pciedev_info *devinfo)
 {
-	brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXMASK,
-			       BRCMF_PCIE_MB_INT_D2H_DB |
-			       BRCMF_PCIE_MB_INT_FN0_0 |
-			       BRCMF_PCIE_MB_INT_FN0_1);
+	brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxmask,
+			       devinfo->reginfo->int_d2h_db |
+			       devinfo->reginfo->int_fn0);
 }
 
 static void brcmf_pcie_hostready(struct brcmf_pciedev_info *devinfo)
 {
 	if (devinfo->shared.flags & BRCMF_PCIE_SHARED_HOSTRDY_DB1)
 		brcmf_pcie_write_reg32(devinfo,
-				       BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_1, 1);
+				       devinfo->reginfo->h2d_mailbox_1, 1);
 }
 
 static irqreturn_t brcmf_pcie_quick_check_isr(int irq, void *arg)
 {
 	struct brcmf_pciedev_info *devinfo = (struct brcmf_pciedev_info *)arg;
 
-	if (brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT)) {
+	if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint)) {
 		brcmf_pcie_intr_disable(devinfo);
 		brcmf_dbg(PCIE, "Enter\n");
 		return IRQ_WAKE_THREAD;
@@ -817,15 +916,14 @@ static irqreturn_t brcmf_pcie_isr_thread(int irq, void *arg)
 	u32 status;
 
 	devinfo->in_irq = true;
-	status = brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT);
+	status = brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint);
 	brcmf_dbg(PCIE, "Enter %x\n", status);
 	if (status) {
-		brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT,
+		brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint,
 				       status);
-		if (status & (BRCMF_PCIE_MB_INT_FN0_0 |
-			      BRCMF_PCIE_MB_INT_FN0_1))
+		if (status & devinfo->reginfo->int_fn0)
 			brcmf_pcie_handle_mb_data(devinfo);
-		if (status & BRCMF_PCIE_MB_INT_D2H_DB) {
+		if (status & devinfo->reginfo->int_d2h_db) {
 			if (devinfo->state == BRCMFMAC_PCIE_STATE_UP)
 				brcmf_proto_msgbuf_rx_trigger(
 							&devinfo->pdev->dev);
@@ -884,8 +982,8 @@ static void brcmf_pcie_release_irq(struct brcmf_pciedev_info *devinfo)
 	if (devinfo->in_irq)
 		brcmf_err(bus, "Still in IRQ (processing) !!!\n");
 
-	status = brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT);
-	brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT, status);
+	status = brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->mailboxint);
+	brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->mailboxint, status);
 
 	devinfo->irq_allocated = false;
 }
@@ -937,7 +1035,7 @@ static int brcmf_pcie_ring_mb_ring_bell(void *ctx)
 
 	brcmf_dbg(PCIE, "RING !\n");
 	/* Any arbitrary value will do, lets use 1 */
-	brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_H2D_MAILBOX_0, 1);
+	brcmf_pcie_write_reg32(devinfo, devinfo->reginfo->h2d_mailbox_0, 1);
 
 	return 0;
 }
@@ -1382,23 +1480,25 @@ static int brcmf_pcie_get_memdump(struct device *dev, void *data, size_t len)
 	return 0;
 }
 
-static
-int brcmf_pcie_get_fwname(struct device *dev, const char *ext, u8 *fw_name)
+static int brcmf_pcie_get_blob(struct device *dev, const struct firmware **fw,
+			       enum brcmf_blob_type type)
 {
 	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
-	struct brcmf_fw_request *fwreq;
-	struct brcmf_fw_name fwnames[] = {
-		{ ext, fw_name },
-	};
+	struct brcmf_pciedev *buspub = bus_if->bus_priv.pcie;
+	struct brcmf_pciedev_info *devinfo = buspub->devinfo;
 
-	fwreq = brcmf_fw_alloc_request(bus_if->chip, bus_if->chiprev,
-				       brcmf_pcie_fwnames,
-				       ARRAY_SIZE(brcmf_pcie_fwnames),
-				       fwnames, ARRAY_SIZE(fwnames));
-	if (!fwreq)
-		return -ENOMEM;
+	switch (type) {
+	case BRCMF_BLOB_CLM:
+		*fw = devinfo->clm_fw;
+		devinfo->clm_fw = NULL;
+		break;
+	default:
+		return -ENOENT;
+	}
+
+	if (!*fw)
+		return -ENOENT;
 
-	kfree(fwreq);
 	return 0;
 }
 
@@ -1445,7 +1545,7 @@ static const struct brcmf_bus_ops brcmf_pcie_bus_ops = {
 	.wowl_config = brcmf_pcie_wowl_config,
 	.get_ramsize = brcmf_pcie_get_ramsize,
 	.get_memdump = brcmf_pcie_get_memdump,
-	.get_fwname = brcmf_pcie_get_fwname,
+	.get_blob = brcmf_pcie_get_blob,
 	.reset = brcmf_pcie_reset,
 };
 
@@ -1698,15 +1798,22 @@ static int brcmf_pcie_buscoreprep(void *ctx)
 static int brcmf_pcie_buscore_reset(void *ctx, struct brcmf_chip *chip)
 {
 	struct brcmf_pciedev_info *devinfo = (struct brcmf_pciedev_info *)ctx;
-	u32 val;
+	struct brcmf_core *core;
+	u32 val, reg;
 
 	devinfo->ci = chip;
 	brcmf_pcie_reset_device(devinfo);
 
-	val = brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT);
+	/* reginfo is not ready yet */
+	core = brcmf_chip_get_core(chip, BCMA_CORE_PCIE2);
+	if (core->rev >= 64)
+		reg = BRCMF_PCIE_64_PCIE2REG_MAILBOXINT;
+	else
+		reg = BRCMF_PCIE_PCIE2REG_MAILBOXINT;
+
+	val = brcmf_pcie_read_reg32(devinfo, reg);
 	if (val != 0xffffffff)
-		brcmf_pcie_write_reg32(devinfo, BRCMF_PCIE_PCIE2REG_MAILBOXINT,
-				       val);
+		brcmf_pcie_write_reg32(devinfo, reg, val);
 
 	return 0;
 }
@@ -1729,8 +1836,206 @@ static const struct brcmf_buscore_ops brcmf_pcie_buscore_ops = {
 	.write32 = brcmf_pcie_buscore_write32,
 };
 
+#define BRCMF_OTP_SYS_VENDOR	0x15
+#define BRCMF_OTP_BRCM_CIS	0x80
+
+#define BRCMF_OTP_VENDOR_HDR	0x00000008
+
+static int
+brcmf_pcie_parse_otp_sys_vendor(struct brcmf_pciedev_info *devinfo,
+				u8 *data, size_t size)
+{
+	int idx = 4;
+	const char *chip_params;
+	const char *board_params;
+	const char *p;
+
+	/* 4-byte header and two empty strings */
+	if (size < 6)
+		return -EINVAL;
+
+	if (get_unaligned_le32(data) != BRCMF_OTP_VENDOR_HDR)
+		return -EINVAL;
+
+	chip_params = &data[idx];
+
+	/* Skip first string, including terminator */
+	idx += strnlen(chip_params, size - idx) + 1;
+	if (idx >= size)
+		return -EINVAL;
+
+	board_params = &data[idx];
+
+	/* Skip to terminator of second string */
+	idx += strnlen(board_params, size - idx);
+	if (idx >= size)
+		return -EINVAL;
+
+	/* At this point both strings are guaranteed NUL-terminated */
+	brcmf_dbg(PCIE, "OTP: chip_params='%s' board_params='%s'\n",
+		  chip_params, board_params);
+
+	p = skip_spaces(board_params);
+	while (*p) {
+		char tag = *p++;
+		const char *end;
+		size_t len;
+
+		if (*p++ != '=') /* implicit NUL check */
+			return -EINVAL;
+
+		/* *p might be NUL here, if so end == p and len == 0 */
+		end = strchrnul(p, ' ');
+		len = end - p;
+
+		/* leave 1 byte for NUL in destination string */
+		if (len > (BRCMF_OTP_MAX_PARAM_LEN - 1))
+			return -EINVAL;
+
+		/* Copy len characters plus a NUL terminator */
+		switch (tag) {
+		case 'M':
+			strscpy(devinfo->otp.module, p, len + 1);
+			break;
+		case 'V':
+			strscpy(devinfo->otp.vendor, p, len + 1);
+			break;
+		case 'm':
+			strscpy(devinfo->otp.version, p, len + 1);
+			break;
+		}
+
+		/* Skip to next arg, if any */
+		p = skip_spaces(end);
+	}
+
+	brcmf_dbg(PCIE, "OTP: module=%s vendor=%s version=%s\n",
+		  devinfo->otp.module, devinfo->otp.vendor,
+		  devinfo->otp.version);
+
+	if (!devinfo->otp.module[0] ||
+	    !devinfo->otp.vendor[0] ||
+	    !devinfo->otp.version[0])
+		return -EINVAL;
+
+	devinfo->otp.valid = true;
+	return 0;
+}
+
+static int
+brcmf_pcie_parse_otp(struct brcmf_pciedev_info *devinfo, u8 *otp, size_t size)
+{
+	int p = 0;
+	int ret = -EINVAL;
+
+	brcmf_dbg(PCIE, "parse_otp size=%zd\n", size);
+
+	while (p < (size - 1)) {
+		u8 type = otp[p];
+		u8 length = otp[p + 1];
+
+		if (type == 0)
+			break;
+
+		if ((p + 2 + length) > size)
+			break;
+
+		switch (type) {
+		case BRCMF_OTP_SYS_VENDOR:
+			brcmf_dbg(PCIE, "OTP @ 0x%x (%d): SYS_VENDOR\n",
+				  p, length);
+			ret = brcmf_pcie_parse_otp_sys_vendor(devinfo,
+							      &otp[p + 2],
+							      length);
+			break;
+		case BRCMF_OTP_BRCM_CIS:
+			brcmf_dbg(PCIE, "OTP @ 0x%x (%d): BRCM_CIS\n",
+				  p, length);
+			break;
+		default:
+			brcmf_dbg(PCIE, "OTP @ 0x%x (%d): Unknown type 0x%x\n",
+				  p, length, type);
+			break;
+		}
+
+		p += 2 + length;
+	}
+
+	return ret;
+}
+
+static int brcmf_pcie_read_otp(struct brcmf_pciedev_info *devinfo)
+{
+	const struct pci_dev *pdev = devinfo->pdev;
+	struct brcmf_bus *bus = dev_get_drvdata(&pdev->dev);
+	u32 coreid, base, words, idx, sromctl;
+	u16 *otp;
+	struct brcmf_core *core;
+	int ret;
+
+	switch (devinfo->ci->chip) {
+	case BRCM_CC_4378_CHIP_ID:
+		coreid = BCMA_CORE_GCI;
+		base = 0x1120;
+		words = 0x170;
+		break;
+	default:
+		/* OTP not supported on this chip */
+		return 0;
+	}
+
+	core = brcmf_chip_get_core(devinfo->ci, coreid);
+	if (!core) {
+		brcmf_err(bus, "No OTP core\n");
+		return -ENODEV;
+	}
+
+	if (coreid == BCMA_CORE_CHIPCOMMON) {
+		/* Chips with OTP accessed via ChipCommon need additional
+		 * handling to access the OTP
+		 */
+		brcmf_pcie_select_core(devinfo, coreid);
+		sromctl = READCC32(devinfo, sromcontrol);
+
+		if (!(sromctl & BCMA_CC_SROM_CONTROL_OTP_PRESENT)) {
+			/* Chip lacks OTP, try without it... */
+			brcmf_err(bus,
+				  "OTP unavailable, using default firmware\n");
+			return 0;
+		}
+
+		/* Map OTP to shadow area */
+		WRITECC32(devinfo, sromcontrol,
+			  sromctl | BCMA_CC_SROM_CONTROL_OTPSEL);
+	}
+
+	otp = kcalloc(words, sizeof(u16), GFP_KERNEL);
+	if (!otp)
+		return -ENOMEM;
+
+	/* Map bus window to SROM/OTP shadow area in core */
+	base = brcmf_pcie_buscore_prep_addr(devinfo->pdev, base + core->base);
+
+	brcmf_dbg(PCIE, "OTP data:\n");
+	for (idx = 0; idx < words; idx++) {
+		otp[idx] = brcmf_pcie_read_reg16(devinfo, base + 2 * idx);
+		brcmf_dbg(PCIE, "[%8x] 0x%04x\n", base + 2 * idx, otp[idx]);
+	}
+
+	if (coreid == BCMA_CORE_CHIPCOMMON) {
+		brcmf_pcie_select_core(devinfo, coreid);
+		WRITECC32(devinfo, sromcontrol, sromctl);
+	}
+
+	ret = brcmf_pcie_parse_otp(devinfo, (u8 *)otp, 2 * words);
+	kfree(otp);
+
+	return ret;
+}
+
 #define BRCMF_PCIE_FW_CODE	0
 #define BRCMF_PCIE_FW_NVRAM	1
+#define BRCMF_PCIE_FW_CLM	2
 
 static void brcmf_pcie_setup(struct device *dev, int ret,
 			     struct brcmf_fw_request *fwreq)
@@ -1755,6 +2060,7 @@ static void brcmf_pcie_setup(struct device *dev, int ret,
 	fw = fwreq->items[BRCMF_PCIE_FW_CODE].binary;
 	nvram = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.data;
 	nvram_len = fwreq->items[BRCMF_PCIE_FW_NVRAM].nv_data.len;
+	devinfo->clm_fw = fwreq->items[BRCMF_PCIE_FW_CLM].binary;
 	kfree(fwreq);
 
 	ret = brcmf_chip_get_raminfo(devinfo->ci);
@@ -1830,6 +2136,7 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo)
 	struct brcmf_fw_name fwnames[] = {
 		{ ".bin", devinfo->fw_name },
 		{ ".txt", devinfo->nvram_name },
+		{ ".clm_blob", devinfo->clm_name },
 	};
 
 	fwreq = brcmf_fw_alloc_request(devinfo->ci->chip, devinfo->ci->chiprev,
@@ -1842,11 +2149,51 @@ brcmf_pcie_prepare_fw_request(struct brcmf_pciedev_info *devinfo)
 	fwreq->items[BRCMF_PCIE_FW_CODE].type = BRCMF_FW_TYPE_BINARY;
 	fwreq->items[BRCMF_PCIE_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM;
 	fwreq->items[BRCMF_PCIE_FW_NVRAM].flags = BRCMF_FW_REQF_OPTIONAL;
-	fwreq->board_type = devinfo->settings->board_type;
+	fwreq->items[BRCMF_PCIE_FW_CLM].type = BRCMF_FW_TYPE_BINARY;
+	fwreq->items[BRCMF_PCIE_FW_CLM].flags = BRCMF_FW_REQF_OPTIONAL;
 	/* NVRAM reserves PCI domain 0 for Broadcom's SDK faked bus */
 	fwreq->domain_nr = pci_domain_nr(devinfo->pdev->bus) + 1;
 	fwreq->bus_nr = devinfo->pdev->bus->number;
 
+	/* Apple platforms with fancy firmware/NVRAM selection */
+	if (devinfo->settings->board_type &&
+	    devinfo->settings->antenna_sku &&
+	    devinfo->otp.valid) {
+		const struct brcmf_otp_params *otp = &devinfo->otp;
+		struct device *dev = &devinfo->pdev->dev;
+		const char **bt = fwreq->board_types;
+
+		brcmf_dbg(PCIE, "Apple board: %s\n",
+			  devinfo->settings->board_type);
+
+		/* Example: apple,shikoku-RASP-m-6.11-X3 */
+		bt[0] = devm_kasprintf(dev, GFP_KERNEL, "%s-%s-%s-%s-%s",
+				       devinfo->settings->board_type,
+				       otp->module, otp->vendor, otp->version,
+				       devinfo->settings->antenna_sku);
+		bt[1] = devm_kasprintf(dev, GFP_KERNEL, "%s-%s-%s-%s",
+				       devinfo->settings->board_type,
+				       otp->module, otp->vendor, otp->version);
+		bt[2] = devm_kasprintf(dev, GFP_KERNEL, "%s-%s-%s",
+				       devinfo->settings->board_type,
+				       otp->module, otp->vendor);
+		bt[3] = devm_kasprintf(dev, GFP_KERNEL, "%s-%s",
+				       devinfo->settings->board_type,
+				       otp->module);
+		bt[4] = devm_kasprintf(dev, GFP_KERNEL, "%s-%s",
+				       devinfo->settings->board_type,
+				       devinfo->settings->antenna_sku);
+		bt[5] = devinfo->settings->board_type;
+
+		if (!bt[0] || !bt[1] || !bt[2] || !bt[3] || !bt[4]) {
+			kfree(fwreq);
+			return NULL;
+		}
+	} else {
+		brcmf_dbg(PCIE, "Board: %s\n", devinfo->settings->board_type);
+		fwreq->board_types[0] = devinfo->settings->board_type;
+	}
+
 	return fwreq;
 }
 
@@ -1857,6 +2204,7 @@ brcmf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	struct brcmf_fw_request *fwreq;
 	struct brcmf_pciedev_info *devinfo;
 	struct brcmf_pciedev *pcie_bus_dev;
+	struct brcmf_core *core;
 	struct brcmf_bus *bus;
 
 	brcmf_dbg(PCIE, "Enter %x:%x\n", pdev->vendor, pdev->device);
@@ -1876,6 +2224,12 @@ brcmf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto fail;
 	}
 
+	core = brcmf_chip_get_core(devinfo->ci, BCMA_CORE_PCIE2);
+	if (core->rev >= 64)
+		devinfo->reginfo = &brcmf_reginfo_64;
+	else
+		devinfo->reginfo = &brcmf_reginfo_default;
+
 	pcie_bus_dev = kzalloc(sizeof(*pcie_bus_dev), GFP_KERNEL);
 	if (pcie_bus_dev == NULL) {
 		ret = -ENOMEM;
@@ -1918,6 +2272,12 @@ brcmf_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (ret)
 		goto fail_bus;
 
+	ret = brcmf_pcie_read_otp(devinfo);
+	if (ret) {
+		brcmf_err(bus, "failed to parse OTP\n");
+		goto fail_brcmf;
+	}
+
 	fwreq = brcmf_pcie_prepare_fw_request(devinfo);
 	if (!fwreq) {
 		ret = -ENOMEM;
@@ -1981,6 +2341,7 @@ brcmf_pcie_remove(struct pci_dev *pdev)
 	brcmf_pcie_release_ringbuffers(devinfo);
 	brcmf_pcie_reset_device(devinfo);
 	brcmf_pcie_release_resource(devinfo);
+	release_firmware(devinfo->clm_fw);
 
 	if (devinfo->ci)
 		brcmf_chip_detach(devinfo->ci);
@@ -2038,7 +2399,7 @@ static int brcmf_pcie_pm_leave_D3(struct device *dev)
 	brcmf_dbg(PCIE, "Enter, dev=%p, bus=%p\n", dev, bus);
 
 	/* Check if device is still up and running, if so we are ready */
-	if (brcmf_pcie_read_reg32(devinfo, BRCMF_PCIE_PCIE2REG_INTMASK) != 0) {
+	if (brcmf_pcie_read_reg32(devinfo, devinfo->reginfo->intmask) != 0) {
 		brcmf_dbg(PCIE, "Try to wakeup device....\n");
 		if (brcmf_pcie_send_mb_data(devinfo, BRCMF_H2D_HOST_D0_INFORM))
 			goto cleanup;
@@ -2105,6 +2466,9 @@ static const struct pci_device_id brcmf_pcie_devid_table[] = {
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_2G_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4366_5G_DEVICE_ID),
 	BRCMF_PCIE_DEVICE(BRCM_PCIE_4371_DEVICE_ID),
+	BRCMF_PCIE_DEVICE(BRCM_PCIE_4378_DEVICE_ID),
+	BRCMF_PCIE_DEVICE(CY_PCIE_89459_DEVICE_ID),
+	BRCMF_PCIE_DEVICE(CY_PCIE_89459_RAW_DEVICE_ID),
 	{ /* end: all zeroes */ }
 };
 
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c
index fabfbb0b40b0..d0a7465be586 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pno.c
@@ -158,12 +158,12 @@ static int brcmf_pno_set_random(struct brcmf_if *ifp, struct brcmf_pno_info *pi)
 	struct brcmf_pno_macaddr_le pfn_mac;
 	u8 *mac_addr = NULL;
 	u8 *mac_mask = NULL;
-	int err, i;
+	int err, i, ri;
 
-	for (i = 0; i < pi->n_reqs; i++)
-		if (pi->reqs[i]->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) {
-			mac_addr = pi->reqs[i]->mac_addr;
-			mac_mask = pi->reqs[i]->mac_addr_mask;
+	for (ri = 0; ri < pi->n_reqs; ri++)
+		if (pi->reqs[ri]->flags & NL80211_SCAN_FLAG_RANDOM_ADDR) {
+			mac_addr = pi->reqs[ri]->mac_addr;
+			mac_mask = pi->reqs[ri]->mac_addr_mask;
 			break;
 		}
 
@@ -185,7 +185,7 @@ static int brcmf_pno_set_random(struct brcmf_if *ifp, struct brcmf_pno_info *pi)
 	pfn_mac.mac[0] |= 0x02;
 
 	brcmf_dbg(SCAN, "enabling random mac: reqid=%llu mac=%pM\n",
-		  pi->reqs[i]->reqid, pfn_mac.mac);
+		  pi->reqs[ri]->reqid, pfn_mac.mac);
 	err = brcmf_fil_iovar_data_set(ifp, "pfn_macaddr", &pfn_mac,
 				       sizeof(pfn_mac));
 	if (err)
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
index 8968809399c7..465d95d83759 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.c
@@ -618,6 +618,7 @@ BRCMF_FW_DEF(43430A0, "brcmfmac43430a0-sdio");
 /* Note the names are not postfixed with a1 for backward compatibility */
 BRCMF_FW_CLM_DEF(43430A1, "brcmfmac43430-sdio");
 BRCMF_FW_DEF(43430B0, "brcmfmac43430b0-sdio");
+BRCMF_FW_CLM_DEF(43439, "brcmfmac43439-sdio");
 BRCMF_FW_CLM_DEF(43455, "brcmfmac43455-sdio");
 BRCMF_FW_DEF(43456, "brcmfmac43456-sdio");
 BRCMF_FW_CLM_DEF(4354, "brcmfmac4354-sdio");
@@ -657,6 +658,7 @@ static const struct brcmf_firmware_mapping brcmf_sdio_fwnames[] = {
 	BRCMF_FW_ENTRY(BRCM_CC_4359_CHIP_ID, 0xFFFFFFFF, 4359),
 	BRCMF_FW_ENTRY(CY_CC_4373_CHIP_ID, 0xFFFFFFFF, 4373),
 	BRCMF_FW_ENTRY(CY_CC_43012_CHIP_ID, 0xFFFFFFFF, 43012),
+	BRCMF_FW_ENTRY(CY_CC_43439_CHIP_ID, 0xFFFFFFFF, 43439),
 	BRCMF_FW_ENTRY(CY_CC_43752_CHIP_ID, 0xFFFFFFFF, 43752)
 };
 
@@ -4129,23 +4131,24 @@ brcmf_sdio_watchdog(struct timer_list *t)
 	}
 }
 
-static
-int brcmf_sdio_get_fwname(struct device *dev, const char *ext, u8 *fw_name)
+static int brcmf_sdio_get_blob(struct device *dev, const struct firmware **fw,
+			       enum brcmf_blob_type type)
 {
 	struct brcmf_bus *bus_if = dev_get_drvdata(dev);
-	struct brcmf_fw_request *fwreq;
-	struct brcmf_fw_name fwnames[] = {
-		{ ext, fw_name },
-	};
+	struct brcmf_sdio_dev *sdiodev = bus_if->bus_priv.sdio;
 
-	fwreq = brcmf_fw_alloc_request(bus_if->chip, bus_if->chiprev,
-				       brcmf_sdio_fwnames,
-				       ARRAY_SIZE(brcmf_sdio_fwnames),
-				       fwnames, ARRAY_SIZE(fwnames));
-	if (!fwreq)
-		return -ENOMEM;
+	switch (type) {
+	case BRCMF_BLOB_CLM:
+		*fw = sdiodev->clm_fw;
+		sdiodev->clm_fw = NULL;
+		break;
+	default:
+		return -ENOENT;
+	}
+
+	if (!*fw)
+		return -ENOENT;
 
-	kfree(fwreq);
 	return 0;
 }
 
@@ -4180,13 +4183,14 @@ static const struct brcmf_bus_ops brcmf_sdio_bus_ops = {
 	.wowl_config = brcmf_sdio_wowl_config,
 	.get_ramsize = brcmf_sdio_bus_get_ramsize,
 	.get_memdump = brcmf_sdio_bus_get_memdump,
-	.get_fwname = brcmf_sdio_get_fwname,
+	.get_blob = brcmf_sdio_get_blob,
 	.debugfs_create = brcmf_sdio_debugfs_create,
 	.reset = brcmf_sdio_bus_reset
 };
 
 #define BRCMF_SDIO_FW_CODE	0
 #define BRCMF_SDIO_FW_NVRAM	1
+#define BRCMF_SDIO_FW_CLM	2
 
 static void brcmf_sdio_firmware_callback(struct device *dev, int err,
 					 struct brcmf_fw_request *fwreq)
@@ -4209,6 +4213,7 @@ static void brcmf_sdio_firmware_callback(struct device *dev, int err,
 	code = fwreq->items[BRCMF_SDIO_FW_CODE].binary;
 	nvram = fwreq->items[BRCMF_SDIO_FW_NVRAM].nv_data.data;
 	nvram_len = fwreq->items[BRCMF_SDIO_FW_NVRAM].nv_data.len;
+	sdiod->clm_fw = fwreq->items[BRCMF_SDIO_FW_CLM].binary;
 	kfree(fwreq);
 
 	/* try to download image and nvram to the dongle */
@@ -4407,6 +4412,7 @@ brcmf_sdio_prepare_fw_request(struct brcmf_sdio *bus)
 	struct brcmf_fw_name fwnames[] = {
 		{ ".bin", bus->sdiodev->fw_name },
 		{ ".txt", bus->sdiodev->nvram_name },
+		{ ".clm_blob", bus->sdiodev->clm_name },
 	};
 
 	fwreq = brcmf_fw_alloc_request(bus->ci->chip, bus->ci->chiprev,
@@ -4418,7 +4424,9 @@ brcmf_sdio_prepare_fw_request(struct brcmf_sdio *bus)
 
 	fwreq->items[BRCMF_SDIO_FW_CODE].type = BRCMF_FW_TYPE_BINARY;
 	fwreq->items[BRCMF_SDIO_FW_NVRAM].type = BRCMF_FW_TYPE_NVRAM;
-	fwreq->board_type = bus->sdiodev->settings->board_type;
+	fwreq->items[BRCMF_SDIO_FW_CLM].type = BRCMF_FW_TYPE_BINARY;
+	fwreq->items[BRCMF_SDIO_FW_CLM].flags = BRCMF_FW_REQF_OPTIONAL;
+	fwreq->board_types[0] = bus->sdiodev->settings->board_type;
 
 	return fwreq;
 }
@@ -4574,6 +4582,8 @@ void brcmf_sdio_remove(struct brcmf_sdio *bus)
 		if (bus->sdiodev->settings)
 			brcmf_release_module_param(bus->sdiodev->settings);
 
+		release_firmware(bus->sdiodev->clm_fw);
+		bus->sdiodev->clm_fw = NULL;
 		kfree(bus->rxbuf);
 		kfree(bus->hdrbuf);
 		kfree(bus);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h
index 47351ff458ca..b76d34d36bde 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/sdio.h
@@ -186,9 +186,11 @@ struct brcmf_sdio_dev {
 	struct sg_table sgtable;
 	char fw_name[BRCMF_FW_NAME_LEN];
 	char nvram_name[BRCMF_FW_NAME_LEN];
+	char clm_name[BRCMF_FW_NAME_LEN];
 	bool wowl_enabled;
 	enum brcmf_sdiod_state state;
 	struct brcmf_sdiod_freezer *freezer;
+	const struct firmware *clm_fw;
 };
 
 /* sdio core registers */
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
index 9fb68c2dc7e3..85e18fb9c497 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/usb.c
@@ -1154,24 +1154,11 @@ error:
 	return NULL;
 }
 
-static
-int brcmf_usb_get_fwname(struct device *dev, const char *ext, u8 *fw_name)
+static int brcmf_usb_get_blob(struct device *dev, const struct firmware **fw,
+			      enum brcmf_blob_type type)
 {
-	struct brcmf_bus *bus = dev_get_drvdata(dev);
-	struct brcmf_fw_request *fwreq;
-	struct brcmf_fw_name fwnames[] = {
-		{ ext, fw_name },
-	};
-
-	fwreq = brcmf_fw_alloc_request(bus->chip, bus->chiprev,
-				       brcmf_usb_fwnames,
-				       ARRAY_SIZE(brcmf_usb_fwnames),
-				       fwnames, ARRAY_SIZE(fwnames));
-	if (!fwreq)
-		return -ENOMEM;
-
-	kfree(fwreq);
-	return 0;
+	/* No blobs for USB devices... */
+	return -ENOENT;
 }
 
 static const struct brcmf_bus_ops brcmf_usb_bus_ops = {
@@ -1180,7 +1167,7 @@ static const struct brcmf_bus_ops brcmf_usb_bus_ops = {
 	.txdata = brcmf_usb_tx,
 	.txctl = brcmf_usb_tx_ctlpkt,
 	.rxctl = brcmf_usb_rx_ctlpkt,
-	.get_fwname = brcmf_usb_get_fwname,
+	.get_blob = brcmf_usb_get_blob,
 };
 
 #define BRCMF_USB_FW_CODE	0
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/types.h b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/types.h
index ae1f3ad40d45..2b0df07ced74 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/types.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/types.h
@@ -123,7 +123,7 @@
 					 */
 
 /********************************************************************
- * Phy/Core Configuration.  Defines macros to to check core phy/rev *
+ * Phy/Core Configuration.  Defines macros to check core phy/rev *
  * compile-time configuration.  Defines default core support.       *
  * ******************************************************************
  */
diff --git a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
index ed0b707f0cdf..f4939cf62767 100644
--- a/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
+++ b/drivers/net/wireless/broadcom/brcm80211/include/brcm_hw_ids.h
@@ -51,9 +51,12 @@
 #define BRCM_CC_43664_CHIP_ID		43664
 #define BRCM_CC_43666_CHIP_ID		43666
 #define BRCM_CC_4371_CHIP_ID		0x4371
+#define BRCM_CC_4378_CHIP_ID		0x4378
 #define CY_CC_4373_CHIP_ID		0x4373
 #define CY_CC_43012_CHIP_ID		43012
+#define CY_CC_43439_CHIP_ID		43439
 #define CY_CC_43752_CHIP_ID		43752
+#define CY_CC_89459_CHIP_ID		0x4355
 
 /* USB Device IDs */
 #define BRCM_USB_43143_DEVICE_ID	0xbd1e
@@ -87,7 +90,9 @@
 #define BRCM_PCIE_4366_2G_DEVICE_ID	0x43c4
 #define BRCM_PCIE_4366_5G_DEVICE_ID	0x43c5
 #define BRCM_PCIE_4371_DEVICE_ID	0x440d
-
+#define BRCM_PCIE_4378_DEVICE_ID	0x4425
+#define CY_PCIE_89459_DEVICE_ID         0x4415
+#define CY_PCIE_89459_RAW_DEVICE_ID     0x4355
 
 /* brcmsmac IDs */
 #define BCM4313_D11N2G_ID	0x4727	/* 4313 802.11n 2.4G device */
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2100.c b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
index 5234511dac78..b0f23cf1a621 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2100.c
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2100.c
@@ -5907,8 +5907,8 @@ static void ipw_ethtool_get_drvinfo(struct net_device *dev,
 	struct ipw2100_priv *priv = libipw_priv(dev);
 	char fw_ver[64], ucode_ver[64];
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 
 	ipw2100_get_fwversion(priv, fw_ver, sizeof(fw_ver));
 	ipw2100_get_ucodeversion(priv, ucode_ver, sizeof(ucode_ver));
@@ -5916,7 +5916,7 @@ static void ipw_ethtool_get_drvinfo(struct net_device *dev,
 	snprintf(info->fw_version, sizeof(info->fw_version), "%s:%d:%s",
 		 fw_ver, priv->eeprom_version, ucode_ver);
 
-	strlcpy(info->bus_info, pci_name(priv->pci_dev),
+	strscpy(info->bus_info, pci_name(priv->pci_dev),
 		sizeof(info->bus_info));
 }
 
@@ -6529,7 +6529,7 @@ static struct pci_driver ipw2100_pci_driver = {
 	.shutdown = ipw2100_shutdown,
 };
 
-/**
+/*
  * Initialize the ipw2100 driver/module
  *
  * @returns 0 if ok, < 0 errno node con error.
@@ -6561,7 +6561,7 @@ out:
 	return ret;
 }
 
-/**
+/*
  * Cleanup ipw2100 driver registration
  */
 static void __exit ipw2100_exit(void)
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.c b/drivers/net/wireless/intel/ipw2x00/ipw2200.c
index 029dacebe751..5b483de18c81 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2200.c
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.c
@@ -10424,8 +10424,8 @@ static void ipw_ethtool_get_drvinfo(struct net_device *dev,
 	char date[32];
 	u32 len;
 
-	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
+	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
+	strscpy(info->version, DRV_VERSION, sizeof(info->version));
 
 	len = sizeof(vers);
 	ipw_get_ordinal(p, IPW_ORD_STAT_FW_VERSION, vers, &len);
@@ -10434,7 +10434,7 @@ static void ipw_ethtool_get_drvinfo(struct net_device *dev,
 
 	snprintf(info->fw_version, sizeof(info->fw_version), "%s (%s)",
 		 vers, date);
-	strlcpy(info->bus_info, pci_name(p->pci_dev),
+	strscpy(info->bus_info, pci_name(p->pci_dev),
 		sizeof(info->bus_info));
 }
 
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.h b/drivers/net/wireless/intel/ipw2x00/ipw2200.h
index 55cac934f4ee..09ddd21608d4 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2200.h
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.h
@@ -651,7 +651,7 @@ struct ipw_rx_notification {
 		struct notif_link_deterioration link_deterioration;
 		struct notif_calibration calibration;
 		struct notif_noise noise;
-		u8 raw[0];
+		DECLARE_FLEX_ARRAY(u8, raw);
 	} u;
 } __packed;
 
diff --git a/drivers/net/wireless/intel/ipw2x00/libipw.h b/drivers/net/wireless/intel/ipw2x00/libipw.h
index 7964ef7d15f0..bec7bc273748 100644
--- a/drivers/net/wireless/intel/ipw2x00/libipw.h
+++ b/drivers/net/wireless/intel/ipw2x00/libipw.h
@@ -405,7 +405,7 @@ struct libipw_auth {
 	__le16 transaction;
 	__le16 status;
 	/* challenge */
-	struct libipw_info_element info_element[];
+	u8 variable[];
 } __packed;
 
 struct libipw_channel_switch {
@@ -423,7 +423,6 @@ struct libipw_action {
 	union {
 		struct libipw_action_exchange {
 			u8 token;
-			struct libipw_info_element info_element[0];
 		} exchange;
 		struct libipw_channel_switch channel_switch;
 
@@ -441,7 +440,7 @@ struct libipw_disassoc {
 struct libipw_probe_request {
 	struct libipw_hdr_3addr header;
 	/* SSID, supported rates */
-	struct libipw_info_element info_element[];
+	u8 variable[];
 } __packed;
 
 struct libipw_probe_response {
@@ -451,7 +450,7 @@ struct libipw_probe_response {
 	__le16 capability;
 	/* SSID, supported rates, FH params, DS params,
 	 * CF params, IBSS params, TIM (if beacon), RSN */
-	struct libipw_info_element info_element[];
+	u8 variable[];
 } __packed;
 
 /* Alias beacon for probe_response */
@@ -462,7 +461,7 @@ struct libipw_assoc_request {
 	__le16 capability;
 	__le16 listen_interval;
 	/* SSID, supported rates, RSN */
-	struct libipw_info_element info_element[];
+	u8 variable[];
 } __packed;
 
 struct libipw_reassoc_request {
@@ -470,7 +469,7 @@ struct libipw_reassoc_request {
 	__le16 capability;
 	__le16 listen_interval;
 	u8 current_ap[ETH_ALEN];
-	struct libipw_info_element info_element[];
+	u8 variable[];
 } __packed;
 
 struct libipw_assoc_response {
@@ -479,7 +478,7 @@ struct libipw_assoc_response {
 	__le16 status;
 	__le16 aid;
 	/* supported rates */
-	struct libipw_info_element info_element[];
+	u8 variable[];
 } __packed;
 
 struct libipw_txb {
diff --git a/drivers/net/wireless/intel/ipw2x00/libipw_rx.c b/drivers/net/wireless/intel/ipw2x00/libipw_rx.c
index 7a684b76f39b..48d6870bbf4e 100644
--- a/drivers/net/wireless/intel/ipw2x00/libipw_rx.c
+++ b/drivers/net/wireless/intel/ipw2x00/libipw_rx.c
@@ -1329,8 +1329,8 @@ static int libipw_handle_assoc_resp(struct libipw_device *ieee, struct libipw_as
 	network->wpa_ie_len = 0;
 	network->rsn_ie_len = 0;
 
-	if (libipw_parse_info_param
-	    (frame->info_element, stats->len - sizeof(*frame), network))
+	if (libipw_parse_info_param((void *)frame->variable,
+				    stats->len - sizeof(*frame), network))
 		return 1;
 
 	network->mode = 0;
@@ -1389,8 +1389,8 @@ static int libipw_network_init(struct libipw_device *ieee, struct libipw_probe_r
 	network->wpa_ie_len = 0;
 	network->rsn_ie_len = 0;
 
-	if (libipw_parse_info_param
-	    (beacon->info_element, stats->len - sizeof(*beacon), network))
+	if (libipw_parse_info_param((void *)beacon->variable,
+				    stats->len - sizeof(*beacon), network))
 		return 1;
 
 	network->mode = 0;
@@ -1510,7 +1510,7 @@ static void libipw_process_probe_response(struct libipw_device
 	struct libipw_network *target;
 	struct libipw_network *oldest = NULL;
 #ifdef CONFIG_LIBIPW_DEBUG
-	struct libipw_info_element *info_element = beacon->info_element;
+	struct libipw_info_element *info_element = (void *)beacon->variable;
 #endif
 	unsigned long flags;
 
diff --git a/drivers/net/wireless/intel/iwlegacy/3945-mac.c b/drivers/net/wireless/intel/iwlegacy/3945-mac.c
index 846138d6e33d..7352d5b2095f 100644
--- a/drivers/net/wireless/intel/iwlegacy/3945-mac.c
+++ b/drivers/net/wireless/intel/iwlegacy/3945-mac.c
@@ -3254,7 +3254,7 @@ il3945_store_measurement(struct device *d, struct device_attribute *attr,
 
 	if (count) {
 		char *p = buffer;
-		strlcpy(buffer, buf, sizeof(buffer));
+		strscpy(buffer, buf, sizeof(buffer));
 		channel = simple_strtoul(p, NULL, 0);
 		if (channel)
 			params.channel = channel;
diff --git a/drivers/net/wireless/intel/iwlegacy/4965-rs.c b/drivers/net/wireless/intel/iwlegacy/4965-rs.c
index d8a5dbf89a02..718efb1aa1b0 100644
--- a/drivers/net/wireless/intel/iwlegacy/4965-rs.c
+++ b/drivers/net/wireless/intel/iwlegacy/4965-rs.c
@@ -1167,7 +1167,7 @@ il4965_rs_switch_to_mimo2(struct il_priv *il, struct il_lq_sta *lq_sta,
 	if (!conf_is_ht(conf) || !sta->deflink.ht_cap.ht_supported)
 		return -1;
 
-	if (sta->smps_mode == IEEE80211_SMPS_STATIC)
+	if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC)
 		return -1;
 
 	/* Need both Tx chains/antennas to support MIMO */
diff --git a/drivers/net/wireless/intel/iwlegacy/commands.h b/drivers/net/wireless/intel/iwlegacy/commands.h
index 4a97310f8fee..28cf4e832152 100644
--- a/drivers/net/wireless/intel/iwlegacy/commands.h
+++ b/drivers/net/wireless/intel/iwlegacy/commands.h
@@ -1710,7 +1710,7 @@ struct il4965_tx_resp {
 	 */
 	union {
 		__le32 status;
-		struct agg_tx_status agg_status[0];	/* for each agg frame */
+		DECLARE_FLEX_ARRAY(struct agg_tx_status, agg_status);	/* for each agg frame */
 	} u;
 } __packed;
 
@@ -3365,7 +3365,7 @@ struct il_rx_pkt {
 		struct il_compressed_ba_resp compressed_ba;
 		struct il_missed_beacon_notif missed_beacon;
 		__le32 status;
-		u8 raw[0];
+		DECLARE_FLEX_ARRAY(u8, raw);
 	} u;
 } __packed;
 
diff --git a/drivers/net/wireless/intel/iwlegacy/common.c b/drivers/net/wireless/intel/iwlegacy/common.c
index 04d27a26260b..341c17fe2af4 100644
--- a/drivers/net/wireless/intel/iwlegacy/common.c
+++ b/drivers/net/wireless/intel/iwlegacy/common.c
@@ -1870,15 +1870,15 @@ il_set_ht_add_station(struct il_priv *il, u8 idx, struct ieee80211_sta *sta)
 		goto done;
 
 	D_ASSOC("spatial multiplexing power save mode: %s\n",
-		(sta->smps_mode == IEEE80211_SMPS_STATIC) ? "static" :
-		(sta->smps_mode == IEEE80211_SMPS_DYNAMIC) ? "dynamic" :
+		(sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) ? "static" :
+		(sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC) ? "dynamic" :
 		"disabled");
 
 	sta_flags = il->stations[idx].sta.station_flags;
 
 	sta_flags &= ~(STA_FLG_RTS_MIMO_PROT_MSK | STA_FLG_MIMO_DIS_MSK);
 
-	switch (sta->smps_mode) {
+	switch (sta->deflink.smps_mode) {
 	case IEEE80211_SMPS_STATIC:
 		sta_flags |= STA_FLG_MIMO_DIS_MSK;
 		break;
@@ -1888,7 +1888,7 @@ il_set_ht_add_station(struct il_priv *il, u8 idx, struct ieee80211_sta *sta)
 	case IEEE80211_SMPS_OFF:
 		break;
 	default:
-		IL_WARN("Invalid MIMO PS mode %d\n", sta->smps_mode);
+		IL_WARN("Invalid MIMO PS mode %d\n", sta->deflink.smps_mode);
 		break;
 	}
 
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
index 8ff967edc8f0..110fda65bd21 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
@@ -56,13 +56,16 @@
 #define IWL_BZ_A_GF4_A_FW_PRE		"iwlwifi-bz-a0-gf4-a0-"
 #define IWL_BZ_A_MR_A_FW_PRE		"iwlwifi-bz-a0-mr-a0-"
 #define IWL_BZ_A_FM_A_FW_PRE		"iwlwifi-bz-a0-fm-a0-"
+#define IWL_BZ_A_FM4_A_FW_PRE		"iwlwifi-bz-a0-fm4-a0-"
 #define IWL_GL_A_FM_A_FW_PRE		"iwlwifi-gl-a0-fm-a0-"
+#define IWL_GL_B_FM_B_FW_PRE		"iwlwifi-gl-b0-fm-b0-"
 #define IWL_BZ_Z_GF_A_FW_PRE		"iwlwifi-bz-z0-gf-a0-"
 #define IWL_BNJ_A_FM_A_FW_PRE		"iwlwifi-BzBnj-a0-fm-a0-"
 #define IWL_BNJ_A_FM4_A_FW_PRE		"iwlwifi-BzBnj-a0-fm4-a0-"
 #define IWL_BNJ_A_GF_A_FW_PRE		"iwlwifi-BzBnj-a0-gf-a0-"
 #define IWL_BNJ_A_GF4_A_FW_PRE		"iwlwifi-BzBnj-a0-gf4-a0-"
 #define IWL_BNJ_A_HR_B_FW_PRE		"iwlwifi-BzBnj-a0-hr-b0-"
+#define IWL_BNJ_B_FM_B_FW_PRE		"iwlwifi-BzBnj-b0-fm-b0-"
 
 
 #define IWL_QU_B_HR_B_MODULE_FIRMWARE(api) \
@@ -119,8 +122,12 @@
 	IWL_BZ_A_MR_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_BZ_A_FM_A_MODULE_FIRMWARE(api) \
 		IWL_BZ_A_FM_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_BZ_A_FM4_A_MODULE_FIRMWARE(api) \
+		IWL_BZ_A_FM4_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_GL_A_FM_A_MODULE_FIRMWARE(api) \
 		IWL_GL_A_FM_A_FW_PRE __stringify(api) ".ucode"
+#define IWL_GL_B_FM_B_MODULE_FIRMWARE(api) \
+		IWL_GL_B_FM_B_FW_PRE __stringify(api) ".ucode"
 #define IWL_BNJ_A_FM_A_MODULE_FIRMWARE(api) \
 	IWL_BNJ_A_FM_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_BNJ_A_FM4_A_MODULE_FIRMWARE(api) \
@@ -131,6 +138,8 @@
 	IWL_BNJ_A_GF4_A_FW_PRE __stringify(api) ".ucode"
 #define IWL_BNJ_A_HR_B_MODULE_FIRMWARE(api) \
 	IWL_BNJ_A_HR_B_FW_PRE __stringify(api) ".ucode"
+#define IWL_BNJ_B_FM_B_MODULE_FIRMWARE(api) \
+	IWL_BNJ_B_FM_B_FW_PRE __stringify(api) ".ucode"
 
 static const struct iwl_base_params iwl_22000_base_params = {
 	.eeprom_size = OTP_LOW_IMAGE_SIZE_32K,
@@ -240,7 +249,7 @@ static const struct iwl_ht_params iwl_22000_ht_params = {
 		},							\
 	}
 
-#define IWL_DEVICE_BZ_COMMON						\
+#define IWL_DEVICE_BZ							\
 	.ucode_api_max = IWL_22000_UCODE_API_MAX,			\
 	.ucode_api_min = IWL_22000_UCODE_API_MIN,			\
 	.led_mode = IWL_LED_RF_STATE,					\
@@ -276,16 +285,13 @@ static const struct iwl_ht_params iwl_22000_ht_params = {
 			.addr = LDBG_M2S_BUF_WRAP_CNT,			\
 			.mask = LDBG_M2S_BUF_WRAP_CNT_VAL_MSK,		\
 		},							\
-	}
-
-#define IWL_DEVICE_BZ							\
-	IWL_DEVICE_BZ_COMMON,						\
+	},								\
 	.trans.umac_prph_offset = 0x300000,				\
 	.trans.device_family = IWL_DEVICE_FAMILY_BZ,			\
 	.trans.base_params = &iwl_ax210_base_params,			\
 	.min_txq_size = 128,						\
 	.gp2_reg_addr = 0xd02c68,					\
-	.min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_EHT,		\
+	.min_ba_txq_size = IWL_DEFAULT_QUEUE_SIZE_EHT,			\
 	.mon_dram_regs = {						\
 		.write_ptr = {						\
 			.addr = DBGC_CUR_DBGBUF_STATUS,			\
@@ -926,6 +932,13 @@ const struct iwl_cfg iwl_cfg_bz_a0_fm_a0 = {
 	.num_rbds = IWL_NUM_RBDS_AX210_HE,
 };
 
+const struct iwl_cfg iwl_cfg_bz_a0_fm4_a0 = {
+	.fw_name_pre = IWL_BZ_A_FM4_A_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
 const struct iwl_cfg iwl_cfg_gl_a0_fm_a0 = {
 	.fw_name_pre = IWL_GL_A_FM_A_FW_PRE,
 	.uhb_supported = true,
@@ -933,6 +946,13 @@ const struct iwl_cfg iwl_cfg_gl_a0_fm_a0 = {
 	.num_rbds = IWL_NUM_RBDS_AX210_HE,
 };
 
+const struct iwl_cfg iwl_cfg_gl_b0_fm_b0 = {
+	.fw_name_pre = IWL_GL_B_FM_B_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
+
 const struct iwl_cfg iwl_cfg_bz_z0_gf_a0 = {
 	.fw_name_pre = IWL_BZ_Z_GF_A_FW_PRE,
 	.uhb_supported = true,
@@ -974,6 +994,13 @@ const struct iwl_cfg iwl_cfg_bnj_a0_hr_b0 = {
 	IWL_DEVICE_BZ,
 	.num_rbds = IWL_NUM_RBDS_AX210_HE,
 };
+
+const struct iwl_cfg iwl_cfg_bnj_b0_fm_b0 = {
+	.fw_name_pre = IWL_BNJ_B_FM_B_FW_PRE,
+	.uhb_supported = true,
+	IWL_DEVICE_BZ,
+	.num_rbds = IWL_NUM_RBDS_AX210_HE,
+};
 MODULE_FIRMWARE(IWL_QU_B_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_QNJ_B_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_QU_C_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
@@ -1007,3 +1034,6 @@ MODULE_FIRMWARE(IWL_BNJ_A_FM4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BNJ_A_GF_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BNJ_A_GF4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BNJ_A_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BZ_A_FM4_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_GL_B_FM_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_BNJ_B_FM_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/agn.h b/drivers/net/wireless/intel/iwlwifi/dvm/agn.h
index 411a6f6638b4..fefaa414272b 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/agn.h
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/agn.h
@@ -112,7 +112,7 @@ int iwl_load_ucode_wait_alive(struct iwl_priv *priv,
 			      enum iwl_ucode_type ucode_type);
 int iwl_send_calib_results(struct iwl_priv *priv);
 int iwl_calib_set(struct iwl_priv *priv,
-		  const struct iwl_calib_hdr *cmd, int len);
+		  const struct iwl_calib_cmd *cmd, size_t len);
 void iwl_calib_free_results(struct iwl_priv *priv);
 int iwl_dump_nic_event_log(struct iwl_priv *priv, bool full_log,
 			    char **buf);
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/calib.c b/drivers/net/wireless/intel/iwlwifi/dvm/calib.c
index a11884fa254b..f488620d2844 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/calib.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/calib.c
@@ -19,8 +19,7 @@
 struct iwl_calib_result {
 	struct list_head list;
 	size_t cmd_len;
-	struct iwl_calib_hdr hdr;
-	/* data follows */
+	struct iwl_calib_cmd cmd;
 };
 
 struct statistics_general_data {
@@ -43,12 +42,12 @@ int iwl_send_calib_results(struct iwl_priv *priv)
 		int ret;
 
 		hcmd.len[0] = res->cmd_len;
-		hcmd.data[0] = &res->hdr;
+		hcmd.data[0] = &res->cmd;
 		hcmd.dataflags[0] = IWL_HCMD_DFL_NOCOPY;
 		ret = iwl_dvm_send_cmd(priv, &hcmd);
 		if (ret) {
 			IWL_ERR(priv, "Error %d on calib cmd %d\n",
-				ret, res->hdr.op_code);
+				ret, res->cmd.hdr.op_code);
 			return ret;
 		}
 	}
@@ -57,19 +56,22 @@ int iwl_send_calib_results(struct iwl_priv *priv)
 }
 
 int iwl_calib_set(struct iwl_priv *priv,
-		  const struct iwl_calib_hdr *cmd, int len)
+		  const struct iwl_calib_cmd *cmd, size_t len)
 {
 	struct iwl_calib_result *res, *tmp;
 
-	res = kmalloc(sizeof(*res) + len - sizeof(struct iwl_calib_hdr),
-		      GFP_ATOMIC);
+	if (check_sub_overflow(len, sizeof(*cmd), &len))
+		return -ENOMEM;
+
+	res = kmalloc(struct_size(res, cmd.data, len), GFP_ATOMIC);
 	if (!res)
 		return -ENOMEM;
-	memcpy(&res->hdr, cmd, len);
-	res->cmd_len = len;
+	res->cmd = *cmd;
+	memcpy(res->cmd.data, cmd->data, len);
+	res->cmd_len = struct_size(cmd, data, len);
 
 	list_for_each_entry(tmp, &priv->calib_results, list) {
-		if (tmp->hdr.op_code == res->hdr.op_code) {
+		if (tmp->cmd.hdr.op_code == res->cmd.hdr.op_code) {
 			list_replace(&tmp->list, &res->list);
 			kfree(tmp);
 			return 0;
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/dev.h b/drivers/net/wireless/intel/iwlwifi/dvm/dev.h
index bbd574091201..1a9eadace188 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/dev.h
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/dev.h
@@ -696,6 +696,7 @@ struct iwl_priv {
 	/* Scan related variables */
 	unsigned long scan_start;
 	unsigned long scan_start_tsf;
+	size_t scan_cmd_size;
 	void *scan_cmd;
 	enum nl80211_band scan_band;
 	struct cfg80211_scan_request *scan_request;
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/rs.c b/drivers/net/wireless/intel/iwlwifi/dvm/rs.c
index baffa1cbe8fc..687c906a9d72 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/rs.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/rs.c
@@ -2,7 +2,7 @@
 /******************************************************************************
  *
  * Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
- * Copyright (C) 2019 - 2020 Intel Corporation
+ * Copyright (C) 2019 - 2020, 2022 Intel Corporation
  *****************************************************************************/
 #include <linux/kernel.h>
 #include <linux/skbuff.h>
@@ -1242,7 +1242,7 @@ static int rs_switch_to_mimo2(struct iwl_priv *priv,
 	if (!conf_is_ht(conf) || !sta->deflink.ht_cap.ht_supported)
 		return -1;
 
-	if (sta->smps_mode == IEEE80211_SMPS_STATIC)
+	if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC)
 		return -1;
 
 	/* Need both Tx chains/antennas to support MIMO */
@@ -1297,7 +1297,7 @@ static int rs_switch_to_mimo3(struct iwl_priv *priv,
 	if (!conf_is_ht(conf) || !sta->deflink.ht_cap.ht_supported)
 		return -1;
 
-	if (sta->smps_mode == IEEE80211_SMPS_STATIC)
+	if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC)
 		return -1;
 
 	/* Need both Tx chains/antennas to support MIMO */
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/scan.c b/drivers/net/wireless/intel/iwlwifi/dvm/scan.c
index 2d38227dfdd2..a7e85c5c8c72 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/scan.c
@@ -626,7 +626,7 @@ static int iwlagn_request_scan(struct iwl_priv *priv, struct ieee80211_vif *vif)
 	u8 active_chains;
 	u8 scan_tx_antennas = priv->nvm_data->valid_tx_ant;
 	int ret;
-	int scan_cmd_size = sizeof(struct iwl_scan_cmd) +
+	size_t scan_cmd_size = sizeof(struct iwl_scan_cmd) +
 			    MAX_SCAN_CHANNEL * sizeof(struct iwl_scan_channel) +
 			    priv->fw->ucode_capa.max_probe_length;
 	const u8 *ssid = NULL;
@@ -649,9 +649,15 @@ static int iwlagn_request_scan(struct iwl_priv *priv, struct ieee80211_vif *vif)
 				       "fail to allocate memory for scan\n");
 			return -ENOMEM;
 		}
+		priv->scan_cmd_size = scan_cmd_size;
+	}
+	if (priv->scan_cmd_size < scan_cmd_size) {
+		IWL_DEBUG_SCAN(priv,
+			       "memory needed for scan grew unexpectedly\n");
+		return -ENOMEM;
 	}
 	scan = priv->scan_cmd;
-	memset(scan, 0, scan_cmd_size);
+	memset(scan, 0, priv->scan_cmd_size);
 
 	scan->quiet_plcp_th = IWL_PLCP_QUIET_THRESH;
 	scan->quiet_time = IWL_ACTIVE_QUIET_TIME;
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/sta.c b/drivers/net/wireless/intel/iwlwifi/dvm/sta.c
index 476068c0abb7..cef43cf80620 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/sta.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /******************************************************************************
  *
- * Copyright(c) 2003 - 2014 Intel Corporation. All rights reserved.
+ * Copyright(c) 2003 - 2014, 2022 Intel Corporation. All rights reserved.
  *
  * Portions of this file are derived from the ipw3945 project, as well
  * as portions of the ieee80211 subsystem header files.
@@ -161,12 +161,12 @@ static void iwl_sta_calc_ht_flags(struct iwl_priv *priv,
 
 	IWL_DEBUG_INFO(priv, "STA %pM SM PS mode: %s\n",
 			sta->addr,
-			(sta->smps_mode == IEEE80211_SMPS_STATIC) ?
+			(sta->deflink.smps_mode == IEEE80211_SMPS_STATIC) ?
 			"static" :
-			(sta->smps_mode == IEEE80211_SMPS_DYNAMIC) ?
+			(sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC) ?
 			"dynamic" : "disabled");
 
-	switch (sta->smps_mode) {
+	switch (sta->deflink.smps_mode) {
 	case IEEE80211_SMPS_STATIC:
 		*flags |= STA_FLG_MIMO_DIS_MSK;
 		break;
@@ -176,7 +176,7 @@ static void iwl_sta_calc_ht_flags(struct iwl_priv *priv,
 	case IEEE80211_SMPS_OFF:
 		break;
 	default:
-		IWL_WARN(priv, "Invalid MIMO PS mode %d\n", sta->smps_mode);
+		IWL_WARN(priv, "Invalid MIMO PS mode %d\n", sta->deflink.smps_mode);
 		break;
 	}
 
diff --git a/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c b/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c
index 4b27a53d0bb4..bb13ca5d666c 100644
--- a/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c
+++ b/drivers/net/wireless/intel/iwlwifi/dvm/ucode.c
@@ -356,18 +356,18 @@ static bool iwlagn_wait_calib(struct iwl_notif_wait_data *notif_wait,
 			      struct iwl_rx_packet *pkt, void *data)
 {
 	struct iwl_priv *priv = data;
-	struct iwl_calib_hdr *hdr;
+	struct iwl_calib_cmd *cmd;
 
 	if (pkt->hdr.cmd != CALIBRATION_RES_NOTIFICATION) {
 		WARN_ON(pkt->hdr.cmd != CALIBRATION_COMPLETE_NOTIFICATION);
 		return true;
 	}
 
-	hdr = (struct iwl_calib_hdr *)pkt->data;
+	cmd = (struct iwl_calib_cmd *)pkt->data;
 
-	if (iwl_calib_set(priv, hdr, iwl_rx_packet_payload_len(pkt)))
+	if (iwl_calib_set(priv, cmd, iwl_rx_packet_payload_len(pkt)))
 		IWL_ERR(priv, "Failed to record calibration data %d\n",
-			hdr->op_code);
+			cmd->hdr.op_code);
 
 	return false;
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
index c78d2f1c722c..0b052c2e563a 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
@@ -2,7 +2,7 @@
 /*
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2020 Intel Corporation
+ * Copyright (C) 2018-2022 Intel Corporation
  */
 #ifndef __iwl_fw_api_commands_h__
 #define __iwl_fw_api_commands_h__
@@ -20,6 +20,8 @@
  *	&enum iwl_phy_ops_subcmd_ids
  * @DATA_PATH_GROUP: data path group, uses command IDs from
  *	&enum iwl_data_path_subcmd_ids
+ * @SCAN_GROUP: scan group, uses command IDs from
+ *	&enum iwl_scan_subcmd_ids
  * @NAN_GROUP: NAN group, uses command IDs from &enum iwl_nan_subcmd_ids
  * @LOCATION_GROUP: location group, uses command IDs from
  *	&enum iwl_location_subcmd_ids
@@ -36,6 +38,7 @@ enum iwl_mvm_command_groups {
 	MAC_CONF_GROUP = 0x3,
 	PHY_OPS_GROUP = 0x4,
 	DATA_PATH_GROUP = 0x5,
+	SCAN_GROUP = 0x6,
 	NAN_GROUP = 0x7,
 	LOCATION_GROUP = 0x8,
 	PROT_OFFLOAD_GROUP = 0xb,
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h b/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h
index 4cd9ab23954e..df0833890e55 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/d3.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2021 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -766,6 +766,65 @@ struct iwl_wowlan_status_v12 {
 	u8 wake_packet[]; /* can be truncated from _length to _bufsize */
 } __packed; /* WOWLAN_STATUSES_RSP_API_S_VER_12 */
 
+/**
+ * struct iwl_wowlan_info_notif - WoWLAN information notification
+ * @gtk: GTK data
+ * @igtk: IGTK data
+ * @replay_ctr: GTK rekey replay counter
+ * @pattern_number: number of the matched patterns
+ * @reserved1: reserved
+ * @qos_seq_ctr: QoS sequence counters to use next
+ * @wakeup_reasons: wakeup reasons, see &enum iwl_wowlan_wakeup_reason
+ * @num_of_gtk_rekeys: number of GTK rekeys
+ * @transmitted_ndps: number of transmitted neighbor discovery packets
+ * @received_beacons: number of received beacons
+ * @wake_packet_length: wakeup packet length
+ * @wake_packet_bufsize: wakeup packet buffer size
+ * @tid_tear_down: bit mask of tids whose BA sessions were closed
+ *	in suspend state
+ * @station_id: station id
+ * @reserved2: reserved
+ */
+struct iwl_wowlan_info_notif {
+	struct iwl_wowlan_gtk_status_v3 gtk[WOWLAN_GTK_KEYS_NUM];
+	struct iwl_wowlan_igtk_status igtk[WOWLAN_IGTK_KEYS_NUM];
+	__le64 replay_ctr;
+	__le16 pattern_number;
+	__le16 reserved1;
+	__le16 qos_seq_ctr[8];
+	__le32 wakeup_reasons;
+	__le32 num_of_gtk_rekeys;
+	__le32 transmitted_ndps;
+	__le32 received_beacons;
+	__le32 wake_packet_length;
+	__le32 wake_packet_bufsize;
+	u8 tid_tear_down;
+	u8 station_id;
+	u8 reserved2[2];
+} __packed; /* WOWLAN_INFO_NTFY_API_S_VER_1 */
+
+/**
+ * struct iwl_wowlan_wake_pkt_notif - WoWLAN wake packet notification
+ * @wake_packet_length: wakeup packet length
+ * @station_id: station id
+ * @reserved: unused
+ * @wake_packet: wakeup packet
+ */
+struct iwl_wowlan_wake_pkt_notif {
+	__le32 wake_packet_length;
+	u8 station_id;
+	u8 reserved[3];
+	u8 wake_packet[1];
+} __packed; /* WOWLAN_WAKE_PKT_NTFY_API_S_VER_1 */
+
+/**
+ * struct iwl_mvm_d3_end_notif -  d3 end notification
+ * @flags: See &enum iwl_d0i3_flags
+ */
+struct iwl_mvm_d3_end_notif {
+	__le32 flags;
+} __packed;
+
 /* TODO: NetDetect API */
 
 #endif /* __iwl_fw_api_d3_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h b/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h
index 5204aa94e72a..a0123f81f5d8 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/offload.h
@@ -3,7 +3,7 @@
  * Copyright (C) 2012-2014 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
- * Copyright (C) 2021 Intel Corporation
+ * Copyright (C) 2021-2022 Intel Corporation
  */
 #ifndef __iwl_fw_api_offload_h__
 #define __iwl_fw_api_offload_h__
@@ -13,6 +13,21 @@
  */
 enum iwl_prot_offload_subcmd_ids {
 	/**
+	 * @WOWLAN_WAKE_PKT_NOTIFICATION: Notification in &struct iwl_wowlan_wake_pkt_notif
+	 */
+	WOWLAN_WAKE_PKT_NOTIFICATION = 0xFC,
+
+	/**
+	 * @WOWLAN_INFO_NOTIFICATION: Notification in &struct iwl_wowlan_info_notif
+	 */
+	WOWLAN_INFO_NOTIFICATION = 0xFD,
+
+	/**
+	 * @D3_END_NOTIFICATION: End D3 state notification
+	 */
+	D3_END_NOTIFICATION = 0xFE,
+
+	/**
 	 * @STORED_BEACON_NTF: &struct iwl_stored_beacon_notif
 	 */
 	STORED_BEACON_NTF = 0xFF,
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
index 1989b270862b..74a01888715b 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2021 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -668,7 +668,7 @@ struct iwl_rx_no_data {
 	__le32 phy_info[2];
 	__le32 rx_vec[2];
 } __packed; /* RX_NO_DATA_NTFY_API_S_VER_1,
-	       TX_NO_DATA_NTFY_API_S_VER_2 */
+	       RX_NO_DATA_NTFY_API_S_VER_2 */
 
 struct iwl_frame_release {
 	u8 baid;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
index 5543d9cb74c8..7ba0e3409199 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/scan.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2021 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -9,6 +9,16 @@
 
 /* Scan Commands, Responses, Notifications */
 
+/**
+ * enum iwl_scan_subcmd_ids - scan commands
+ */
+enum iwl_scan_subcmd_ids {
+	/**
+	 * @OFFLOAD_MATCH_INFO_NOTIF: &struct iwl_scan_offload_match_info
+	 */
+	OFFLOAD_MATCH_INFO_NOTIF = 0xFC,
+};
+
 /* Max number of IEs for direct SSID scans in a command */
 #define PROBE_OPTION_MAX		20
 
@@ -1188,7 +1198,7 @@ struct iwl_scan_offload_profile_match {
 } __packed; /* SCAN_OFFLOAD_PROFILE_MATCH_RESULTS_S_VER_2 */
 
 /**
- * struct iwl_scan_offload_profiles_query - match results query response
+ * struct iwl_scan_offload_match_info - match results information
  * @matched_profiles: bitmap of matched profiles, referencing the
  *	matches passed in the scan offload request
  * @last_scan_age: age of the last offloaded scan
@@ -1200,7 +1210,7 @@ struct iwl_scan_offload_profile_match {
  * @reserved: reserved
  * @matches: array of match information, one for each match
  */
-struct iwl_scan_offload_profiles_query {
+struct iwl_scan_offload_match_info {
 	__le32 matched_profiles;
 	__le32 last_scan_age;
 	__le32 n_scans_done;
@@ -1210,7 +1220,9 @@ struct iwl_scan_offload_profiles_query {
 	u8 self_recovery;
 	__le16 reserved;
 	struct iwl_scan_offload_profile_match matches[];
-} __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_3 */
+} __packed; /* SCAN_OFFLOAD_PROFILES_QUERY_RSP_S_VER_3 and
+	     * SCAN_OFFLOAD_MATCH_INFO_NOTIFICATION_S_VER_1
+	     */
 
 /**
  * struct iwl_umac_scan_iter_complete_notif - notifies end of scanning iteration
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
index f5b556a103e8..cfa5e1b3c3f6 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
@@ -649,13 +649,16 @@ extern const struct iwl_cfg iwl_cfg_bz_a0_gf_a0;
 extern const struct iwl_cfg iwl_cfg_bz_a0_gf4_a0;
 extern const struct iwl_cfg iwl_cfg_bz_a0_mr_a0;
 extern const struct iwl_cfg iwl_cfg_bz_a0_fm_a0;
+extern const struct iwl_cfg iwl_cfg_bz_a0_fm4_a0;
 extern const struct iwl_cfg iwl_cfg_gl_a0_fm_a0;
+extern const struct iwl_cfg iwl_cfg_gl_b0_fm_b0;
 extern const struct iwl_cfg iwl_cfg_bz_z0_gf_a0;
 extern const struct iwl_cfg iwl_cfg_bnj_a0_fm_a0;
 extern const struct iwl_cfg iwl_cfg_bnj_a0_fm4_a0;
 extern const struct iwl_cfg iwl_cfg_bnj_a0_gf_a0;
 extern const struct iwl_cfg iwl_cfg_bnj_a0_gf4_a0;
 extern const struct iwl_cfg iwl_cfg_bnj_a0_hr_b0;
+extern const struct iwl_cfg iwl_cfg_bnj_b0_fm_b0;
 #endif /* CONFIG_IWLMVM */
 
 #endif /* __IWL_CONFIG_H__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
index aeb0015b73d2..919b1f478b4c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/d3.c
@@ -1427,7 +1427,7 @@ struct iwl_wowlan_status_data {
 		u8 flags;
 	} igtk;
 
-	u8 wake_packet[];
+	u8 *wake_packet;
 };
 
 static void iwl_mvm_report_wakeup_reasons(struct iwl_mvm *mvm,
@@ -1480,7 +1480,7 @@ static void iwl_mvm_report_wakeup_reasons(struct iwl_mvm *mvm,
 	if (reasons & IWL_WOWLAN_WAKEUP_BY_REM_WAKE_WAKEUP_PACKET)
 		wakeup.tcp_match = true;
 
-	if (status->wake_packet_bufsize) {
+	if (status->wake_packet) {
 		int pktsize = status->wake_packet_bufsize;
 		int pktlen = status->wake_packet_length;
 		const u8 *pktdata = status->wake_packet;
@@ -1944,57 +1944,6 @@ out:
 	return true;
 }
 
-/* Occasionally, templates would be nice. This is one of those times ... */
-#define iwl_mvm_parse_wowlan_status_common(_ver)			\
-static struct iwl_wowlan_status_data *					\
-iwl_mvm_parse_wowlan_status_common_ ## _ver(struct iwl_mvm *mvm,	\
-					    struct iwl_wowlan_status_ ##_ver *data,\
-					    int len)			\
-{									\
-	struct iwl_wowlan_status_data *status;				\
-	int data_size, i;						\
-									\
-	if (len < sizeof(*data)) {					\
-		IWL_ERR(mvm, "Invalid WoWLAN status response!\n");	\
-		return NULL;						\
-	}								\
-									\
-	data_size = ALIGN(le32_to_cpu(data->wake_packet_bufsize), 4);	\
-	if (len != sizeof(*data) + data_size) {				\
-		IWL_ERR(mvm, "Invalid WoWLAN status response!\n");	\
-		return NULL;						\
-	}								\
-									\
-	status = kzalloc(sizeof(*status) + data_size, GFP_KERNEL);	\
-	if (!status)							\
-		return NULL;						\
-									\
-	/* copy all the common fields */				\
-	status->replay_ctr = le64_to_cpu(data->replay_ctr);		\
-	status->pattern_number = le16_to_cpu(data->pattern_number);	\
-	status->non_qos_seq_ctr = le16_to_cpu(data->non_qos_seq_ctr);	\
-	for (i = 0; i < 8; i++)						\
-		status->qos_seq_ctr[i] =				\
-			le16_to_cpu(data->qos_seq_ctr[i]);		\
-	status->wakeup_reasons = le32_to_cpu(data->wakeup_reasons);	\
-	status->num_of_gtk_rekeys =					\
-		le32_to_cpu(data->num_of_gtk_rekeys);			\
-	status->received_beacons = le32_to_cpu(data->received_beacons);	\
-	status->wake_packet_length =					\
-		le32_to_cpu(data->wake_packet_length);			\
-	status->wake_packet_bufsize =					\
-		le32_to_cpu(data->wake_packet_bufsize);			\
-	memcpy(status->wake_packet, data->wake_packet,			\
-	       status->wake_packet_bufsize);				\
-									\
-	return status;							\
-}
-
-iwl_mvm_parse_wowlan_status_common(v6)
-iwl_mvm_parse_wowlan_status_common(v7)
-iwl_mvm_parse_wowlan_status_common(v9)
-iwl_mvm_parse_wowlan_status_common(v12)
-
 static void iwl_mvm_convert_gtk_v2(struct iwl_wowlan_status_data *status,
 				   struct iwl_wowlan_gtk_status_v2 *data)
 {
@@ -2054,6 +2003,96 @@ static void iwl_mvm_convert_igtk(struct iwl_wowlan_status_data *status,
 			   ((u64)ipn[0] << 40);
 }
 
+static void iwl_mvm_parse_wowlan_info_notif(struct iwl_mvm *mvm,
+					    struct iwl_wowlan_info_notif *data,
+					    struct iwl_wowlan_status_data *status,
+					    u32 len)
+{
+	u32 i;
+
+	if (len < sizeof(*data)) {
+		IWL_ERR(mvm, "Invalid WoWLAN info notification!\n");
+		status = NULL;
+		return;
+	}
+
+	iwl_mvm_convert_key_counters_v5(status, &data->gtk[0].sc);
+	iwl_mvm_convert_gtk_v3(status, &data->gtk[0]);
+	iwl_mvm_convert_igtk(status, &data->igtk[0]);
+
+	status->replay_ctr = le64_to_cpu(data->replay_ctr);
+	status->pattern_number = le16_to_cpu(data->pattern_number);
+	for (i = 0; i < IWL_MAX_TID_COUNT; i++)
+		status->qos_seq_ctr[i] =
+			le16_to_cpu(data->qos_seq_ctr[i]);
+	status->wakeup_reasons = le32_to_cpu(data->wakeup_reasons);
+	status->num_of_gtk_rekeys =
+		le32_to_cpu(data->num_of_gtk_rekeys);
+	status->received_beacons = le32_to_cpu(data->received_beacons);
+	status->tid_tear_down = data->tid_tear_down;
+}
+
+/* Occasionally, templates would be nice. This is one of those times ... */
+#define iwl_mvm_parse_wowlan_status_common(_ver)			\
+static struct iwl_wowlan_status_data *					\
+iwl_mvm_parse_wowlan_status_common_ ## _ver(struct iwl_mvm *mvm,	\
+					    struct iwl_wowlan_status_ ##_ver *data,\
+					    int len)			\
+{									\
+	struct iwl_wowlan_status_data *status;				\
+	int data_size, i;						\
+									\
+	if (len < sizeof(*data)) {					\
+		IWL_ERR(mvm, "Invalid WoWLAN status response!\n");	\
+		return NULL;						\
+	}								\
+									\
+	data_size = ALIGN(le32_to_cpu(data->wake_packet_bufsize), 4);	\
+	if (len != sizeof(*data) + data_size) {				\
+		IWL_ERR(mvm, "Invalid WoWLAN status response!\n");	\
+		return NULL;						\
+	}								\
+									\
+	status = kzalloc(sizeof(*status), GFP_KERNEL);			\
+	if (!status)							\
+		return NULL;						\
+									\
+	/* copy all the common fields */				\
+	status->replay_ctr = le64_to_cpu(data->replay_ctr);		\
+	status->pattern_number = le16_to_cpu(data->pattern_number);	\
+	status->non_qos_seq_ctr = le16_to_cpu(data->non_qos_seq_ctr);	\
+	for (i = 0; i < 8; i++)						\
+		status->qos_seq_ctr[i] =				\
+			le16_to_cpu(data->qos_seq_ctr[i]);		\
+	status->wakeup_reasons = le32_to_cpu(data->wakeup_reasons);	\
+	status->num_of_gtk_rekeys =					\
+		le32_to_cpu(data->num_of_gtk_rekeys);			\
+	status->received_beacons = le32_to_cpu(data->received_beacons);	\
+	status->wake_packet_length =					\
+		le32_to_cpu(data->wake_packet_length);			\
+	status->wake_packet_bufsize =					\
+		le32_to_cpu(data->wake_packet_bufsize);			\
+	if (status->wake_packet_bufsize) {				\
+		status->wake_packet =					\
+			kmemdup(data->wake_packet,			\
+				status->wake_packet_bufsize,		\
+				GFP_KERNEL);				\
+		if (!status->wake_packet) {				\
+			kfree(status);					\
+			return NULL;					\
+		}							\
+	} else {							\
+		status->wake_packet = NULL;				\
+	}								\
+									\
+	return status;							\
+}
+
+iwl_mvm_parse_wowlan_status_common(v6)
+iwl_mvm_parse_wowlan_status_common(v7)
+iwl_mvm_parse_wowlan_status_common(v9)
+iwl_mvm_parse_wowlan_status_common(v12)
+
 static struct iwl_wowlan_status_data *
 iwl_mvm_send_wowlan_get_status(struct iwl_mvm *mvm, u8 sta_id)
 {
@@ -2173,36 +2212,15 @@ out_free_resp:
 	return status;
 }
 
-static struct iwl_wowlan_status_data *
-iwl_mvm_get_wakeup_status(struct iwl_mvm *mvm, u8 sta_id)
-{
-	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw, OFFLOADS_QUERY_CMD,
-					   IWL_FW_CMD_VER_UNKNOWN);
-	__le32 station_id = cpu_to_le32(sta_id);
-	u32 cmd_size = cmd_ver != IWL_FW_CMD_VER_UNKNOWN ? sizeof(station_id) : 0;
-
-	if (!mvm->net_detect) {
-		/* only for tracing for now */
-		int ret = iwl_mvm_send_cmd_pdu(mvm, OFFLOADS_QUERY_CMD, 0,
-					       cmd_size, &station_id);
-		if (ret)
-			IWL_ERR(mvm, "failed to query offload statistics (%d)\n", ret);
-	}
-
-	return iwl_mvm_send_wowlan_get_status(mvm, sta_id);
-}
-
 /* releases the MVM mutex */
 static bool iwl_mvm_query_wakeup_reasons(struct iwl_mvm *mvm,
-					 struct ieee80211_vif *vif)
+					 struct ieee80211_vif *vif,
+					 struct iwl_wowlan_status_data *status)
 {
-	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_wowlan_status_data *status;
 	int i;
 	bool keep;
 	struct iwl_mvm_sta *mvm_ap_sta;
 
-	status = iwl_mvm_get_wakeup_status(mvm, mvmvif->ap_sta_id);
 	if (!status)
 		goto out_unlock;
 
@@ -2212,7 +2230,7 @@ static bool iwl_mvm_query_wakeup_reasons(struct iwl_mvm *mvm,
 	/* still at hard-coded place 0 for D3 image */
 	mvm_ap_sta = iwl_mvm_sta_from_staid_protected(mvm, 0);
 	if (!mvm_ap_sta)
-		goto out_free;
+		goto out_unlock;
 
 	for (i = 0; i < IWL_MAX_TID_COUNT; i++) {
 		u16 seq = status->qos_seq_ctr[i];
@@ -2235,11 +2253,8 @@ static bool iwl_mvm_query_wakeup_reasons(struct iwl_mvm *mvm,
 
 	keep = iwl_mvm_setup_connection_keep(mvm, vif, status);
 
-	kfree(status);
 	return keep;
 
-out_free:
-	kfree(status);
 out_unlock:
 	mutex_unlock(&mvm->mutex);
 	return false;
@@ -2248,16 +2263,16 @@ out_unlock:
 #define ND_QUERY_BUF_LEN (sizeof(struct iwl_scan_offload_profile_match) * \
 			  IWL_SCAN_MAX_PROFILES)
 
-struct iwl_mvm_nd_query_results {
+struct iwl_mvm_nd_results {
 	u32 matched_profiles;
 	u8 matches[ND_QUERY_BUF_LEN];
 };
 
 static int
 iwl_mvm_netdetect_query_results(struct iwl_mvm *mvm,
-				struct iwl_mvm_nd_query_results *results)
+				struct iwl_mvm_nd_results *results)
 {
-	struct iwl_scan_offload_profiles_query *query;
+	struct iwl_scan_offload_match_info *query;
 	struct iwl_host_cmd cmd = {
 		.id = SCAN_OFFLOAD_PROFILES_QUERY_CMD,
 		.flags = CMD_WANT_SKB,
@@ -2274,7 +2289,7 @@ iwl_mvm_netdetect_query_results(struct iwl_mvm *mvm,
 
 	if (fw_has_api(&mvm->fw->ucode_capa,
 		       IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
-		query_len = sizeof(struct iwl_scan_offload_profiles_query);
+		query_len = sizeof(struct iwl_scan_offload_match_info);
 		matches_len = sizeof(struct iwl_scan_offload_profile_match) *
 			max_profiles;
 	} else {
@@ -2305,7 +2320,7 @@ out_free_resp:
 }
 
 static int iwl_mvm_query_num_match_chans(struct iwl_mvm *mvm,
-					 struct iwl_mvm_nd_query_results *query,
+					 struct iwl_mvm_nd_results *results,
 					 int idx)
 {
 	int n_chans = 0, i;
@@ -2313,13 +2328,13 @@ static int iwl_mvm_query_num_match_chans(struct iwl_mvm *mvm,
 	if (fw_has_api(&mvm->fw->ucode_capa,
 		       IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
 		struct iwl_scan_offload_profile_match *matches =
-			(struct iwl_scan_offload_profile_match *)query->matches;
+			(void *)results->matches;
 
 		for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN; i++)
 			n_chans += hweight8(matches[idx].matching_channels[i]);
 	} else {
 		struct iwl_scan_offload_profile_match_v1 *matches =
-			(struct iwl_scan_offload_profile_match_v1 *)query->matches;
+			(void *)results->matches;
 
 		for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN_V1; i++)
 			n_chans += hweight8(matches[idx].matching_channels[i]);
@@ -2329,7 +2344,7 @@ static int iwl_mvm_query_num_match_chans(struct iwl_mvm *mvm,
 }
 
 static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm,
-				    struct iwl_mvm_nd_query_results *query,
+				    struct iwl_mvm_nd_results *results,
 				    struct cfg80211_wowlan_nd_match *match,
 				    int idx)
 {
@@ -2338,7 +2353,7 @@ static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm,
 	if (fw_has_api(&mvm->fw->ucode_capa,
 		       IWL_UCODE_TLV_API_SCAN_OFFLOAD_CHANS)) {
 		struct iwl_scan_offload_profile_match *matches =
-			(struct iwl_scan_offload_profile_match *)query->matches;
+			 (void *)results->matches;
 
 		for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN * 8; i++)
 			if (matches[idx].matching_channels[i / 8] & (BIT(i % 8)))
@@ -2346,7 +2361,7 @@ static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm,
 					mvm->nd_channels[i]->center_freq;
 	} else {
 		struct iwl_scan_offload_profile_match_v1 *matches =
-			(struct iwl_scan_offload_profile_match_v1 *)query->matches;
+			 (void *)results->matches;
 
 		for (i = 0; i < SCAN_OFFLOAD_MATCHING_CHANNELS_LEN_V1 * 8; i++)
 			if (matches[idx].matching_channels[i / 8] & (BIT(i % 8)))
@@ -2355,25 +2370,50 @@ static void iwl_mvm_query_set_freqs(struct iwl_mvm *mvm,
 	}
 }
 
+/**
+ * enum iwl_d3_notif - d3 notifications
+ * @IWL_D3_NOTIF_WOWLAN_INFO: WOWLAN_INFO_NOTIF was received
+ * @IWL_D3_NOTIF_WOWLAN_WAKE_PKT: WOWLAN_WAKE_PKT_NOTIF was received
+ * @IWL_D3_NOTIF_PROT_OFFLOAD: PROT_OFFLOAD_NOTIF was received
+ * @IWL_D3_ND_MATCH_INFO: OFFLOAD_MATCH_INFO_NOTIF was received
+ * @IWL_D3_NOTIF_D3_END_NOTIF: D3_END_NOTIF was received
+ */
+enum iwl_d3_notif {
+	IWL_D3_NOTIF_WOWLAN_INFO =	BIT(0),
+	IWL_D3_NOTIF_WOWLAN_WAKE_PKT =	BIT(1),
+	IWL_D3_NOTIF_PROT_OFFLOAD =	BIT(2),
+	IWL_D3_ND_MATCH_INFO      =     BIT(3),
+	IWL_D3_NOTIF_D3_END_NOTIF =	BIT(4)
+};
+
+/* manage d3 resume data */
+struct iwl_d3_data {
+	struct iwl_wowlan_status_data *status;
+	bool test;
+	u32 d3_end_flags;
+	u32 notif_expected;	/* bitmap - see &enum iwl_d3_notif */
+	u32 notif_received;	/* bitmap - see &enum iwl_d3_notif */
+	struct iwl_mvm_nd_results *nd_results;
+	bool nd_results_valid;
+};
+
 static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm,
-					    struct ieee80211_vif *vif)
+					    struct ieee80211_vif *vif,
+					    struct iwl_d3_data *d3_data)
 {
 	struct cfg80211_wowlan_nd_info *net_detect = NULL;
 	struct cfg80211_wowlan_wakeup wakeup = {
 		.pattern_idx = -1,
 	};
 	struct cfg80211_wowlan_wakeup *wakeup_report = &wakeup;
-	struct iwl_wowlan_status_data *status;
-	struct iwl_mvm_nd_query_results query;
 	unsigned long matched_profiles;
 	u32 reasons = 0;
 	int i, n_matches, ret;
 
-	status = iwl_mvm_get_wakeup_status(mvm, IWL_MVM_INVALID_STA);
-	if (status) {
-		reasons = status->wakeup_reasons;
-		kfree(status);
-	}
+	if (WARN_ON(!d3_data || !d3_data->status))
+		goto out;
+
+	reasons = d3_data->status->wakeup_reasons;
 
 	if (reasons & IWL_WOWLAN_WAKEUP_BY_RFKILL_DEASSERTED)
 		wakeup.rfkill_release = true;
@@ -2381,13 +2421,22 @@ static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm,
 	if (reasons != IWL_WOWLAN_WAKEUP_BY_NON_WIRELESS)
 		goto out;
 
-	ret = iwl_mvm_netdetect_query_results(mvm, &query);
-	if (ret || !query.matched_profiles) {
+	if (!iwl_fw_lookup_notif_ver(mvm->fw, PROT_OFFLOAD_GROUP,
+				     WOWLAN_INFO_NOTIFICATION, 0)) {
+		IWL_INFO(mvm, "Query FW for ND results\n");
+		ret = iwl_mvm_netdetect_query_results(mvm, d3_data->nd_results);
+
+	} else {
+		IWL_INFO(mvm, "Notification based ND results\n");
+		ret = d3_data->nd_results_valid ? 0 : -1;
+	}
+
+	if (ret || !d3_data->nd_results->matched_profiles) {
 		wakeup_report = NULL;
 		goto out;
 	}
 
-	matched_profiles = query.matched_profiles;
+	matched_profiles = d3_data->nd_results->matched_profiles;
 	if (mvm->n_nd_match_sets) {
 		n_matches = hweight_long(matched_profiles);
 	} else {
@@ -2404,7 +2453,9 @@ static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm,
 		struct cfg80211_wowlan_nd_match *match;
 		int idx, n_channels = 0;
 
-		n_channels = iwl_mvm_query_num_match_chans(mvm, &query, i);
+		n_channels = iwl_mvm_query_num_match_chans(mvm,
+							   d3_data->nd_results,
+							   i);
 
 		match = kzalloc(struct_size(match, channels, n_channels),
 				GFP_KERNEL);
@@ -2424,7 +2475,7 @@ static void iwl_mvm_query_netdetect_reasons(struct iwl_mvm *mvm,
 		if (mvm->n_nd_channels < n_channels)
 			continue;
 
-		iwl_mvm_query_set_freqs(mvm, &query, match, i);
+		iwl_mvm_query_set_freqs(mvm, d3_data->nd_results, match, i);
 	}
 
 out_report_nd:
@@ -2504,16 +2555,317 @@ static bool iwl_mvm_check_rt_status(struct iwl_mvm *mvm,
 	return false;
 }
 
+/*
+ * This function assumes:
+ *	1. The mutex is already held.
+ *	2. The callee functions unlock the mutex.
+ */
+static bool
+iwl_mvm_choose_query_wakeup_reasons(struct iwl_mvm *mvm,
+				    struct ieee80211_vif *vif,
+				    struct iwl_d3_data *d3_data)
+{
+	lockdep_assert_held(&mvm->mutex);
+
+	/* if FW uses status notification, status shouldn't be NULL here */
+	if (!d3_data->status) {
+		struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
+		u8 sta_id = mvm->net_detect ? IWL_MVM_INVALID_STA : mvmvif->ap_sta_id;
+
+		d3_data->status = iwl_mvm_send_wowlan_get_status(mvm, sta_id);
+	}
+
+	if (mvm->net_detect) {
+		iwl_mvm_query_netdetect_reasons(mvm, vif, d3_data);
+	} else {
+		bool keep = iwl_mvm_query_wakeup_reasons(mvm, vif,
+							 d3_data->status);
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+		if (keep)
+			mvm->keep_vif = vif;
+#endif
+
+		return keep;
+	}
+	return false;
+}
+
+#define IWL_WOWLAN_WAKEUP_REASON_HAS_WAKEUP_PKT (IWL_WOWLAN_WAKEUP_BY_MAGIC_PACKET | \
+						 IWL_WOWLAN_WAKEUP_BY_PATTERN | \
+						 IWL_WAKEUP_BY_PATTERN_IPV4_TCP_SYN |\
+						 IWL_WAKEUP_BY_PATTERN_IPV4_TCP_SYN_WILDCARD |\
+						 IWL_WAKEUP_BY_PATTERN_IPV6_TCP_SYN |\
+						 IWL_WAKEUP_BY_PATTERN_IPV6_TCP_SYN_WILDCARD)
+
+static int iwl_mvm_wowlan_store_wake_pkt(struct iwl_mvm *mvm,
+					 struct iwl_wowlan_wake_pkt_notif *notif,
+					 struct iwl_wowlan_status_data *status,
+					 u32 len)
+{
+	u32 data_size, packet_len = le32_to_cpu(notif->wake_packet_length);
+
+	if (len < sizeof(*notif)) {
+		IWL_ERR(mvm, "Invalid WoWLAN wake packet notification!\n");
+		return -EIO;
+	}
+
+	if (WARN_ON(!status)) {
+		IWL_ERR(mvm, "Got wake packet notification but wowlan status data is NULL\n");
+		return -EIO;
+	}
+
+	if (WARN_ON(!(status->wakeup_reasons &
+		      IWL_WOWLAN_WAKEUP_REASON_HAS_WAKEUP_PKT))) {
+		IWL_ERR(mvm, "Got wakeup packet but wakeup reason is %x\n",
+			status->wakeup_reasons);
+		return -EIO;
+	}
+
+	data_size = len - offsetof(struct iwl_wowlan_wake_pkt_notif, wake_packet);
+
+	/* data_size got the padding from the notification, remove it. */
+	if (packet_len < data_size)
+		data_size = packet_len;
+
+	status->wake_packet = kmemdup(notif->wake_packet, data_size,
+				      GFP_ATOMIC);
+
+	if (!status->wake_packet)
+		return -ENOMEM;
+
+	status->wake_packet_length = packet_len;
+	status->wake_packet_bufsize = data_size;
+
+	return 0;
+}
+
+static void iwl_mvm_nd_match_info_handler(struct iwl_mvm *mvm,
+					  struct iwl_d3_data *d3_data,
+					  struct iwl_scan_offload_match_info *notif,
+					  u32 len)
+{
+	struct iwl_wowlan_status_data *status = d3_data->status;
+	struct ieee80211_vif *vif = iwl_mvm_get_bss_vif(mvm);
+	struct iwl_mvm_nd_results *results = d3_data->nd_results;
+	size_t i, matches_len = sizeof(struct iwl_scan_offload_profile_match) *
+		iwl_umac_scan_get_max_profiles(mvm->fw);
+
+	if (IS_ERR_OR_NULL(vif))
+		return;
+
+	if (len < sizeof(struct iwl_scan_offload_match_info)) {
+		IWL_ERR(mvm, "Invalid scan match info notification\n");
+		return;
+	}
+
+	if (!mvm->net_detect) {
+		IWL_ERR(mvm, "Unexpected scan match info notification\n");
+		return;
+	}
+
+	if (!status || status->wakeup_reasons != IWL_WOWLAN_WAKEUP_BY_NON_WIRELESS) {
+		IWL_ERR(mvm,
+			"Ignore scan match info notification: no reason\n");
+		return;
+	}
+
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	mvm->last_netdetect_scans = le32_to_cpu(notif->n_scans_done);
+#endif
+
+	results->matched_profiles = le32_to_cpu(notif->matched_profiles);
+	IWL_INFO(mvm, "number of matched profiles=%u\n",
+		 results->matched_profiles);
+
+	if (results->matched_profiles) {
+		memcpy(results->matches, notif->matches, matches_len);
+		d3_data->nd_results_valid = TRUE;
+	}
+
+	/* no scan should be active at this point */
+	mvm->scan_status = 0;
+	for (i = 0; i < mvm->max_scans; i++)
+		mvm->scan_uid_status[i] = 0;
+}
+
+static bool iwl_mvm_wait_d3_notif(struct iwl_notif_wait_data *notif_wait,
+				  struct iwl_rx_packet *pkt, void *data)
+{
+	struct iwl_mvm *mvm =
+		container_of(notif_wait, struct iwl_mvm, notif_wait);
+	struct iwl_d3_data *d3_data = data;
+	u32 len;
+	int ret;
+
+	switch (WIDE_ID(pkt->hdr.group_id, pkt->hdr.cmd)) {
+	case WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_INFO_NOTIFICATION): {
+		struct iwl_wowlan_info_notif *notif = (void *)pkt->data;
+
+		if (d3_data->notif_received & IWL_D3_NOTIF_WOWLAN_INFO) {
+			/* We might get two notifications due to dual bss */
+			IWL_DEBUG_WOWLAN(mvm,
+					 "Got additional wowlan info notification\n");
+			break;
+		}
+
+		d3_data->notif_received |= IWL_D3_NOTIF_WOWLAN_INFO;
+		len = iwl_rx_packet_payload_len(pkt);
+		iwl_mvm_parse_wowlan_info_notif(mvm, notif, d3_data->status,
+						len);
+		if (d3_data->status &&
+		    d3_data->status->wakeup_reasons & IWL_WOWLAN_WAKEUP_REASON_HAS_WAKEUP_PKT)
+			/* We are supposed to get also wake packet notif */
+			d3_data->notif_expected |= IWL_D3_NOTIF_WOWLAN_WAKE_PKT;
+
+		break;
+	}
+	case WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_WAKE_PKT_NOTIFICATION): {
+		struct iwl_wowlan_wake_pkt_notif *notif = (void *)pkt->data;
+
+		if (d3_data->notif_received & IWL_D3_NOTIF_WOWLAN_WAKE_PKT) {
+			/* We shouldn't get two wake packet notifications */
+			IWL_ERR(mvm,
+				"Got additional wowlan wake packet notification\n");
+		} else {
+			d3_data->notif_received |= IWL_D3_NOTIF_WOWLAN_WAKE_PKT;
+			len =  iwl_rx_packet_payload_len(pkt);
+			ret = iwl_mvm_wowlan_store_wake_pkt(mvm, notif,
+							    d3_data->status,
+							    len);
+			if (ret)
+				IWL_ERR(mvm,
+					"Can't parse WOWLAN_WAKE_PKT_NOTIFICATION\n");
+		}
+
+		break;
+	}
+	case WIDE_ID(SCAN_GROUP, OFFLOAD_MATCH_INFO_NOTIF): {
+		struct iwl_scan_offload_match_info *notif = (void *)pkt->data;
+
+		if (d3_data->notif_received & IWL_D3_ND_MATCH_INFO) {
+			IWL_ERR(mvm,
+				"Got additional netdetect match info\n");
+			break;
+		}
+
+		d3_data->notif_received |= IWL_D3_ND_MATCH_INFO;
+
+		/* explicitly set this in the 'expected' as well */
+		d3_data->notif_expected |= IWL_D3_ND_MATCH_INFO;
+
+		len = iwl_rx_packet_payload_len(pkt);
+		iwl_mvm_nd_match_info_handler(mvm, d3_data, notif, len);
+		break;
+	}
+	case WIDE_ID(PROT_OFFLOAD_GROUP, D3_END_NOTIFICATION): {
+		struct iwl_mvm_d3_end_notif *notif = (void *)pkt->data;
+
+		d3_data->d3_end_flags = __le32_to_cpu(notif->flags);
+		d3_data->notif_received |= IWL_D3_NOTIF_D3_END_NOTIF;
+
+		break;
+	}
+	default:
+		WARN_ON(1);
+	}
+
+	return d3_data->notif_received == d3_data->notif_expected;
+}
+
+static int iwl_mvm_resume_firmware(struct iwl_mvm *mvm, bool test)
+{
+	int ret;
+	enum iwl_d3_status d3_status;
+	struct iwl_host_cmd cmd = {
+			.id = D0I3_END_CMD,
+			.flags = CMD_WANT_SKB | CMD_SEND_IN_D3,
+		};
+	bool reset = fw_has_capa(&mvm->fw->ucode_capa,
+				 IWL_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG);
+
+	ret = iwl_trans_d3_resume(mvm->trans, &d3_status, test, !reset);
+	if (ret)
+		return ret;
+
+	if (d3_status != IWL_D3_STATUS_ALIVE) {
+		IWL_INFO(mvm, "Device was reset during suspend\n");
+		return -ENOENT;
+	}
+
+	/*
+	 * We should trigger resume flow using command only for 22000 family
+	 * AX210 and above don't need the command since they have
+	 * the doorbell interrupt.
+	 */
+	if (mvm->trans->trans_cfg->device_family <= IWL_DEVICE_FAMILY_22000 &&
+	    fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_D0I3_END_FIRST)) {
+		ret = iwl_mvm_send_cmd(mvm, &cmd);
+		if (ret < 0)
+			IWL_ERR(mvm, "Failed to send D0I3_END_CMD first (%d)\n",
+				ret);
+	}
+
+	return ret;
+}
+
+#define IWL_MVM_D3_NOTIF_TIMEOUT (HZ / 5)
+
+static int iwl_mvm_d3_notif_wait(struct iwl_mvm *mvm,
+				 struct iwl_d3_data *d3_data)
+{
+	static const u16 d3_resume_notif[] = {
+		WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_INFO_NOTIFICATION),
+		WIDE_ID(PROT_OFFLOAD_GROUP, WOWLAN_WAKE_PKT_NOTIFICATION),
+		WIDE_ID(SCAN_GROUP, OFFLOAD_MATCH_INFO_NOTIF),
+		WIDE_ID(PROT_OFFLOAD_GROUP, D3_END_NOTIFICATION)
+	};
+	struct iwl_notification_wait wait_d3_notif;
+	int ret;
+
+	iwl_init_notification_wait(&mvm->notif_wait, &wait_d3_notif,
+				   d3_resume_notif, ARRAY_SIZE(d3_resume_notif),
+				   iwl_mvm_wait_d3_notif, d3_data);
+
+	ret = iwl_mvm_resume_firmware(mvm, d3_data->test);
+	if (ret) {
+		iwl_remove_notification(&mvm->notif_wait, &wait_d3_notif);
+		return ret;
+	}
+
+	return iwl_wait_notification(&mvm->notif_wait, &wait_d3_notif,
+				     IWL_MVM_D3_NOTIF_TIMEOUT);
+}
+
+static inline bool iwl_mvm_d3_resume_notif_based(struct iwl_mvm *mvm)
+{
+	return iwl_fw_lookup_notif_ver(mvm->fw, PROT_OFFLOAD_GROUP,
+				       WOWLAN_INFO_NOTIFICATION, 0) &&
+		iwl_fw_lookup_notif_ver(mvm->fw, PROT_OFFLOAD_GROUP,
+					WOWLAN_WAKE_PKT_NOTIFICATION, 0) &&
+		iwl_fw_lookup_notif_ver(mvm->fw, PROT_OFFLOAD_GROUP,
+					D3_END_NOTIFICATION, 0);
+}
+
 static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test)
 {
 	struct ieee80211_vif *vif = NULL;
 	int ret = 1;
-	enum iwl_d3_status d3_status;
-	bool keep = false;
+	struct iwl_mvm_nd_results results = {};
+	struct iwl_d3_data d3_data = {
+		.test = test,
+		.notif_expected =
+			IWL_D3_NOTIF_WOWLAN_INFO |
+			IWL_D3_NOTIF_D3_END_NOTIF,
+		.nd_results_valid = false,
+		.nd_results = &results,
+	};
 	bool unified_image = fw_has_capa(&mvm->fw->ucode_capa,
 					 IWL_UCODE_TLV_CAPA_CNSLDTD_D3_D0_IMG);
 	bool d0i3_first = fw_has_capa(&mvm->fw->ucode_capa,
 				      IWL_UCODE_TLV_CAPA_D0I3_END_FIRST);
+	bool resume_notif_based = iwl_mvm_d3_resume_notif_based(mvm);
+	bool keep = false;
 
 	mutex_lock(&mvm->mutex);
 
@@ -2537,54 +2889,30 @@ static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test)
 		goto err;
 	}
 
-	ret = iwl_trans_d3_resume(mvm->trans, &d3_status, test, !unified_image);
-	if (ret)
-		goto err;
-
-	if (d3_status != IWL_D3_STATUS_ALIVE) {
-		IWL_INFO(mvm, "Device was reset during suspend\n");
-		goto err;
-	}
-
-	if (d0i3_first) {
-		struct iwl_host_cmd cmd = {
-			.id = D0I3_END_CMD,
-			.flags = CMD_WANT_SKB | CMD_SEND_IN_D3,
-		};
-		int len;
-
-		ret = iwl_mvm_send_cmd(mvm, &cmd);
-		if (ret < 0) {
-			IWL_ERR(mvm, "Failed to send D0I3_END_CMD first (%d)\n",
-				ret);
+	if (resume_notif_based) {
+		d3_data.status = kzalloc(sizeof(*d3_data.status), GFP_KERNEL);
+		if (!d3_data.status) {
+			IWL_ERR(mvm, "Failed to allocate wowlan status\n");
+			ret = -ENOMEM;
 			goto err;
 		}
-		switch (mvm->cmd_ver.d0i3_resp) {
-		case 0:
-			break;
-		case 1:
-			len = iwl_rx_packet_payload_len(cmd.resp_pkt);
-			if (len != sizeof(u32)) {
-				IWL_ERR(mvm,
-					"Error with D0I3_END_CMD response size (%d)\n",
-					len);
-				goto err;
-			}
-			if (IWL_D0I3_RESET_REQUIRE &
-			    le32_to_cpu(*(__le32 *)cmd.resp_pkt->data)) {
-				iwl_write32(mvm->trans, CSR_RESET,
-					    CSR_RESET_REG_FLAG_FORCE_NMI);
-				iwl_free_resp(&cmd);
-			}
-			break;
-		default:
-			WARN_ON(1);
-		}
+
+		ret = iwl_mvm_d3_notif_wait(mvm, &d3_data);
+		if (ret)
+			goto err;
+	} else {
+		ret = iwl_mvm_resume_firmware(mvm, test);
+		if (ret < 0)
+			goto err;
 	}
 
 	/* after the successful handshake, we're out of D3 */
 	mvm->trans->system_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
 
+	/* when reset is required we can't send these following commands */
+	if (d3_data.d3_end_flags & IWL_D0I3_RESET_REQUIRE)
+		goto query_wakeup_reasons;
+
 	/*
 	 * Query the current location and source from the D3 firmware so we
 	 * can play it back when we re-intiailize the D0 firmware
@@ -2598,41 +2926,36 @@ static int __iwl_mvm_resume(struct iwl_mvm *mvm, bool test)
 		/*  Re-configure default SAR profile */
 		iwl_mvm_sar_select_profile(mvm, 1, 1);
 
-	if (mvm->net_detect) {
+	if (mvm->net_detect && unified_image) {
 		/* If this is a non-unified image, we restart the FW,
 		 * so no need to stop the netdetect scan.  If that
 		 * fails, continue and try to get the wake-up reasons,
 		 * but trigger a HW restart by keeping a failure code
 		 * in ret.
 		 */
-		if (unified_image)
-			ret = iwl_mvm_scan_stop(mvm, IWL_MVM_SCAN_NETDETECT,
-						false);
-
-		iwl_mvm_query_netdetect_reasons(mvm, vif);
-		/* has unlocked the mutex, so skip that */
-		goto out;
-	} else {
-		keep = iwl_mvm_query_wakeup_reasons(mvm, vif);
-#ifdef CONFIG_IWLWIFI_DEBUGFS
-		if (keep)
-			mvm->keep_vif = vif;
-#endif
-		/* has unlocked the mutex, so skip that */
-		goto out_iterate;
+		ret = iwl_mvm_scan_stop(mvm, IWL_MVM_SCAN_NETDETECT,
+					false);
 	}
 
+query_wakeup_reasons:
+	keep = iwl_mvm_choose_query_wakeup_reasons(mvm, vif, &d3_data);
+	/* has unlocked the mutex, so skip that */
+	goto out;
+
 err:
-	iwl_mvm_free_nd(mvm);
 	mutex_unlock(&mvm->mutex);
+out:
+	if (d3_data.status)
+		kfree(d3_data.status->wake_packet);
+	kfree(d3_data.status);
+	iwl_mvm_free_nd(mvm);
 
-out_iterate:
-	if (!test)
+	if (!d3_data.test && !mvm->net_detect)
 		ieee80211_iterate_active_interfaces_mtx(mvm->hw,
-			IEEE80211_IFACE_ITER_NORMAL,
-			iwl_mvm_d3_disconnect_iter, keep ? vif : NULL);
+							IEEE80211_IFACE_ITER_NORMAL,
+							iwl_mvm_d3_disconnect_iter,
+							keep ? vif : NULL);
 
-out:
 	clear_bit(IWL_MVM_STATUS_IN_D3, &mvm->status);
 
 	/* no need to reset the device in unified images, if successful */
@@ -2641,9 +2964,14 @@ out:
 		if (d0i3_first)
 			return 0;
 
-		ret = iwl_mvm_send_cmd_pdu(mvm, D0I3_END_CMD, 0, 0, NULL);
-		if (!ret)
+		if (!iwl_fw_lookup_notif_ver(mvm->fw, PROT_OFFLOAD_GROUP,
+					     D3_END_NOTIFICATION, 0)) {
+			ret = iwl_mvm_send_cmd_pdu(mvm, D0I3_END_CMD, 0, 0, NULL);
+			if (!ret)
+				return 0;
+		} else if (!(d3_data.d3_end_flags & IWL_D0I3_RESET_REQUIRE)) {
 			return 0;
+		}
 	}
 
 	/*
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
index c0bd697b080a..1e8123140973 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/debugfs.c
@@ -430,14 +430,16 @@ static ssize_t iwl_dbgfs_amsdu_len_write(struct ieee80211_sta *sta,
 		return -EBUSY;
 
 	if (amsdu_len) {
-		mvmsta->orig_amsdu_len = sta->max_amsdu_len;
-		sta->max_amsdu_len = amsdu_len;
-		for (i = 0; i < ARRAY_SIZE(sta->max_tid_amsdu_len); i++)
-			sta->max_tid_amsdu_len[i] = amsdu_len;
+		mvmsta->orig_amsdu_len = sta->cur->max_amsdu_len;
+		sta->deflink.agg.max_amsdu_len = amsdu_len;
+		sta->deflink.agg.max_amsdu_len = amsdu_len;
+		for (i = 0; i < ARRAY_SIZE(sta->deflink.agg.max_tid_amsdu_len); i++)
+			sta->deflink.agg.max_tid_amsdu_len[i] = amsdu_len;
 	} else {
-		sta->max_amsdu_len = mvmsta->orig_amsdu_len;
+		sta->deflink.agg.max_amsdu_len = mvmsta->orig_amsdu_len;
 		mvmsta->orig_amsdu_len = 0;
 	}
+
 	return count;
 }
 
@@ -451,7 +453,7 @@ static ssize_t iwl_dbgfs_amsdu_len_read(struct file *file,
 	char buf[32];
 	int pos;
 
-	pos = scnprintf(buf, sizeof(buf), "current %d ", sta->max_amsdu_len);
+	pos = scnprintf(buf, sizeof(buf), "current %d ", sta->cur->max_amsdu_len);
 	pos += scnprintf(buf + pos, sizeof(buf) - pos, "stored %d\n",
 			 mvmsta->orig_amsdu_len);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
index 11536f115198..8464c9b7baf1 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
@@ -3193,7 +3193,7 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw,
 						   NL80211_TDLS_SETUP);
 		}
 
-		sta->max_rc_amsdu_len = 1;
+		sta->deflink.agg.max_rc_amsdu_len = 1;
 	} else if (old_state == IEEE80211_STA_NONE &&
 		   new_state == IEEE80211_STA_AUTH) {
 		/*
@@ -4949,6 +4949,7 @@ static int iwl_mvm_mac_get_survey(struct ieee80211_hw *hw, int idx,
 static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo)
 {
 	u32 format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
+	u32 gi_ltf;
 
 	switch (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) {
 	case RATE_MCS_CHAN_WIDTH_20:
@@ -5019,9 +5020,12 @@ static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo)
 		RATE_HT_MCS_INDEX(rate_n_flags) :
 		u32_get_bits(rate_n_flags, RATE_MCS_CODE_MSK);
 
-	if (format == RATE_MCS_HE_MSK) {
-		u32 gi_ltf = u32_get_bits(rate_n_flags,
-					  RATE_MCS_HE_GI_LTF_MSK);
+	if (rate_n_flags & RATE_MCS_SGI_MSK)
+		rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI;
+
+	switch (format) {
+	case RATE_MCS_HE_MSK:
+		gi_ltf = u32_get_bits(rate_n_flags, RATE_MCS_HE_GI_LTF_MSK);
 
 		rinfo->flags |= RATE_INFO_FLAGS_HE_MCS;
 
@@ -5060,19 +5064,14 @@ static void iwl_mvm_set_sta_rate(u32 rate_n_flags, struct rate_info *rinfo)
 
 		if (rate_n_flags & RATE_HE_DUAL_CARRIER_MODE_MSK)
 			rinfo->he_dcm = 1;
-		return;
-	}
-
-	if (rate_n_flags & RATE_MCS_SGI_MSK)
-		rinfo->flags |= RATE_INFO_FLAGS_SHORT_GI;
-
-	if (format == RATE_MCS_HT_MSK) {
+		break;
+	case RATE_MCS_HT_MSK:
 		rinfo->flags |= RATE_INFO_FLAGS_MCS;
-
-	} else if (format == RATE_MCS_VHT_MSK) {
+		break;
+	case RATE_MCS_VHT_MSK:
 		rinfo->flags |= RATE_INFO_FLAGS_VHT_MCS;
+		break;
 	}
-
 }
 
 static void iwl_mvm_mac_sta_statistics(struct ieee80211_hw *hw,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index bf35e130c876..97cba526e465 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -860,6 +860,7 @@ struct iwl_mvm {
 
 	/* Scan status, cmd (pre-allocated) and auxiliary station */
 	unsigned int scan_status;
+	size_t scan_cmd_size;
 	void *scan_cmd;
 	struct iwl_mcast_filter_cmd *mcast_filter_cmd;
 	/* For CDB this is low band scan type, for non-CDB - type. */
@@ -1079,7 +1080,6 @@ struct iwl_mvm {
 	struct list_head resp_pasn_list;
 
 	struct {
-		u8 d0i3_resp;
 		u8 range_resp;
 	} cmd_ver;
 
@@ -1705,7 +1705,7 @@ int iwl_mvm_update_quotas(struct iwl_mvm *mvm, bool force_upload,
 int iwl_mvm_reg_scan_start(struct iwl_mvm *mvm, struct ieee80211_vif *vif,
 			   struct cfg80211_scan_request *req,
 			   struct ieee80211_scan_ies *ies);
-int iwl_mvm_scan_size(struct iwl_mvm *mvm);
+size_t iwl_mvm_scan_size(struct iwl_mvm *mvm);
 int iwl_mvm_scan_stop(struct iwl_mvm *mvm, int type, bool notify);
 int iwl_mvm_max_scan_ie_len(struct iwl_mvm *mvm);
 void iwl_mvm_report_scan_aborted(struct iwl_mvm *mvm);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
index db43c8a83a31..d2d42cd48af2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
@@ -557,6 +557,13 @@ static const struct iwl_hcmd_names iwl_mvm_data_path_names[] = {
 /* Please keep this array *SORTED* by hex value.
  * Access is done through binary search
  */
+static const struct iwl_hcmd_names iwl_mvm_scan_names[] = {
+	HCMD_NAME(OFFLOAD_MATCH_INFO_NOTIF),
+};
+
+/* Please keep this array *SORTED* by hex value.
+ * Access is done through binary search
+ */
 static const struct iwl_hcmd_names iwl_mvm_location_names[] = {
 	HCMD_NAME(TOF_RANGE_REQ_CMD),
 	HCMD_NAME(TOF_CONFIG_CMD),
@@ -574,6 +581,9 @@ static const struct iwl_hcmd_names iwl_mvm_location_names[] = {
  * Access is done through binary search
  */
 static const struct iwl_hcmd_names iwl_mvm_prot_offload_names[] = {
+	HCMD_NAME(WOWLAN_WAKE_PKT_NOTIFICATION),
+	HCMD_NAME(WOWLAN_INFO_NOTIFICATION),
+	HCMD_NAME(D3_END_NOTIFICATION),
 	HCMD_NAME(STORED_BEACON_NTF),
 };
 
@@ -593,6 +603,7 @@ static const struct iwl_hcmd_arr iwl_mvm_groups[] = {
 	[MAC_CONF_GROUP] = HCMD_ARR(iwl_mvm_mac_conf_names),
 	[PHY_OPS_GROUP] = HCMD_ARR(iwl_mvm_phy_names),
 	[DATA_PATH_GROUP] = HCMD_ARR(iwl_mvm_data_path_names),
+	[SCAN_GROUP] = HCMD_ARR(iwl_mvm_scan_names),
 	[LOCATION_GROUP] = HCMD_ARR(iwl_mvm_location_names),
 	[PROT_OFFLOAD_GROUP] = HCMD_ARR(iwl_mvm_prot_offload_names),
 	[REGULATORY_AND_NVM_GROUP] =
@@ -1065,7 +1076,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	static const u8 no_reclaim_cmds[] = {
 		TX_CMD,
 	};
-	int scan_size;
+	size_t scan_size;
 	u32 min_backoff;
 	struct iwl_mvm_csme_conn_info *csme_conn_info __maybe_unused;
 
@@ -1188,13 +1199,6 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 
 	INIT_DELAYED_WORK(&mvm->cs_tx_unblock_dwork, iwl_mvm_tx_unblock_dwork);
 
-	mvm->cmd_ver.d0i3_resp =
-		iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP, D0I3_END_CMD,
-					0);
-	/* we only support version 1 */
-	if (WARN_ON_ONCE(mvm->cmd_ver.d0i3_resp > 1))
-		goto out_free;
-
 	mvm->cmd_ver.range_resp =
 		iwl_fw_lookup_notif_ver(mvm->fw, LOCATION_GROUP,
 					TOF_RANGE_RESPONSE_NOTIF, 5);
@@ -1299,6 +1303,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg,
 	mvm->scan_cmd = kmalloc(scan_size, GFP_KERNEL);
 	if (!mvm->scan_cmd)
 		goto out_free;
+	mvm->scan_cmd_size = scan_size;
 
 	/* invalidate ids to prevent accidental removal of sta_id 0 */
 	mvm->aux_sta.sta_id = IWL_MVM_INVALID_STA;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
index d8c3d7ff4f44..2e9081cb6627 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs-fw.c
@@ -143,7 +143,7 @@ rs_fw_vht_set_enabled_rates(const struct ieee80211_sta *sta,
 	};
 
 	/* the station support only a single receive chain */
-	if (sta->smps_mode == IEEE80211_SMPS_STATIC)
+	if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC)
 		max_nss = 1;
 
 	for (i = 0; i < max_nss && i < IWL_TLC_NSS_MAX; i++) {
@@ -205,7 +205,7 @@ rs_fw_he_set_enabled_rates(const struct ieee80211_sta *sta,
 	u8 nss = sta->deflink.rx_nss;
 
 	/* the station support only a single receive chain */
-	if (sta->smps_mode == IEEE80211_SMPS_STATIC)
+	if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC)
 		nss = 1;
 
 	for (i = 0; i < nss && i < IWL_TLC_NSS_MAX; i++) {
@@ -270,7 +270,7 @@ static void rs_fw_set_supp_rates(struct ieee80211_sta *sta,
 			cpu_to_le16(ht_cap->mcs.rx_mask[0]);
 
 		/* the station support only a single receive chain */
-		if (sta->smps_mode == IEEE80211_SMPS_STATIC)
+		if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC)
 			cmd->ht_rates[IWL_TLC_NSS_2][IWL_TLC_MCS_PER_BW_80] =
 				0;
 		else
@@ -340,9 +340,9 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
 		u16 size = le32_to_cpu(notif->amsdu_size);
 		int i;
 
-		if (sta->max_amsdu_len < size) {
+		if (sta->deflink.agg.max_amsdu_len < size) {
 			/*
-			 * In debug sta->max_amsdu_len < size
+			 * In debug sta->deflink.agg.max_amsdu_len < size
 			 * so also check with orig_amsdu_len which holds the
 			 * original data before debugfs changed the value
 			 */
@@ -352,18 +352,18 @@ void iwl_mvm_tlc_update_notif(struct iwl_mvm *mvm,
 
 		mvmsta->amsdu_enabled = le32_to_cpu(notif->amsdu_enabled);
 		mvmsta->max_amsdu_len = size;
-		sta->max_rc_amsdu_len = mvmsta->max_amsdu_len;
+		sta->deflink.agg.max_rc_amsdu_len = mvmsta->max_amsdu_len;
 
 		for (i = 0; i < IWL_MAX_TID_COUNT; i++) {
 			if (mvmsta->amsdu_enabled & BIT(i))
-				sta->max_tid_amsdu_len[i] =
+				sta->deflink.agg.max_tid_amsdu_len[i] =
 					iwl_mvm_max_amsdu_size(mvm, sta, i);
 			else
 				/*
 				 * Not so elegant, but this will effectively
 				 * prevent AMSDU on this TID
 				 */
-				sta->max_tid_amsdu_len[i] = 1;
+				sta->deflink.agg.max_tid_amsdu_len[i] = 1;
 		}
 
 		IWL_DEBUG_RATE(mvm,
@@ -450,7 +450,7 @@ void rs_fw_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	 * since TLC offload works with one mode we can assume
 	 * that only vht/ht is used and also set it as station max amsdu
 	 */
-	sta->max_amsdu_len = max_amsdu_len;
+	sta->deflink.agg.max_amsdu_len = max_amsdu_len;
 
 	cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw,
 					WIDE_ID(DATA_PATH_GROUP,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
index a79043f30775..0b50b816684a 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rs.c
@@ -138,7 +138,7 @@ static bool rs_mimo_allow(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	if (!sta->deflink.ht_cap.ht_supported)
 		return false;
 
-	if (sta->smps_mode == IEEE80211_SMPS_STATIC)
+	if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC)
 		return false;
 
 	if (num_of_ant(iwl_mvm_get_valid_tx_ant(mvm)) < 2)
@@ -1491,7 +1491,7 @@ static void rs_set_amsdu_len(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 	struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 	int i;
 
-	sta->max_amsdu_len = rs_fw_get_max_amsdu_len(sta);
+	sta->deflink.agg.max_amsdu_len = rs_fw_get_max_amsdu_len(sta);
 
 	/*
 	 * In case TLC offload is not active amsdu_enabled is either 0xFFFF
@@ -1506,22 +1506,23 @@ static void rs_set_amsdu_len(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 
 	if (mvmsta->vif->bss_conf.he_support &&
 	    !iwlwifi_mod_params.disable_11ax)
-		mvmsta->max_amsdu_len = sta->max_amsdu_len;
+		mvmsta->max_amsdu_len = sta->deflink.agg.max_amsdu_len;
 	else
-		mvmsta->max_amsdu_len = min_t(int, sta->max_amsdu_len, 8500);
+		mvmsta->max_amsdu_len =
+			min_t(int, sta->deflink.agg.max_amsdu_len, 8500);
 
-	sta->max_rc_amsdu_len = mvmsta->max_amsdu_len;
+	sta->deflink.agg.max_rc_amsdu_len = mvmsta->max_amsdu_len;
 
 	for (i = 0; i < IWL_MAX_TID_COUNT; i++) {
 		if (mvmsta->amsdu_enabled)
-			sta->max_tid_amsdu_len[i] =
+			sta->deflink.agg.max_tid_amsdu_len[i] =
 				iwl_mvm_max_amsdu_size(mvm, sta, i);
 		else
 			/*
 			 * Not so elegant, but this will effectively
 			 * prevent AMSDU on this TID
 			 */
-			sta->max_tid_amsdu_len[i] = 1;
+			sta->deflink.agg.max_tid_amsdu_len[i] = 1;
 	}
 }
 
@@ -2933,7 +2934,7 @@ static void rs_drv_rate_init(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 
 	lq_sta->lq.sta_id = mvmsta->sta_id;
 	mvmsta->amsdu_enabled = 0;
-	mvmsta->max_amsdu_len = sta->max_amsdu_len;
+	mvmsta->max_amsdu_len = sta->cur->max_amsdu_len;
 
 	for (j = 0; j < LQ_SIZE; j++)
 		rs_rate_scale_clear_tbl_windows(mvm, &lq_sta->lq_info[j]);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
index 2c43a9989783..1aadccd8841f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rxmq.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2021 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -1191,16 +1191,22 @@ struct iwl_mvm_rx_phy_data {
 	enum iwl_rx_phy_info_type info_type;
 	__le32 d0, d1, d2, d3;
 	__le16 d4;
+
+	u32 rate_n_flags;
+	u32 gp2_on_air_rise;
+	u16 phy_info;
+	u8 energy_a, energy_b;
+	u8 channel;
 };
 
 static void iwl_mvm_decode_he_mu_ext(struct iwl_mvm *mvm,
 				     struct iwl_mvm_rx_phy_data *phy_data,
-				     u32 rate_n_flags,
 				     struct ieee80211_radiotap_he_mu *he_mu)
 {
 	u32 phy_data2 = le32_to_cpu(phy_data->d2);
 	u32 phy_data3 = le32_to_cpu(phy_data->d3);
 	u16 phy_data4 = le16_to_cpu(phy_data->d4);
+	u32 rate_n_flags = phy_data->rate_n_flags;
 
 	if (FIELD_GET(IWL_RX_PHY_DATA4_HE_MU_EXT_CH1_CRC_OK, phy_data4)) {
 		he_mu->flags1 |=
@@ -1246,7 +1252,6 @@ static void iwl_mvm_decode_he_mu_ext(struct iwl_mvm *mvm,
 
 static void
 iwl_mvm_decode_he_phy_ru_alloc(struct iwl_mvm_rx_phy_data *phy_data,
-			       u32 rate_n_flags,
 			       struct ieee80211_radiotap_he *he,
 			       struct ieee80211_radiotap_he_mu *he_mu,
 			       struct ieee80211_rx_status *rx_status)
@@ -1260,6 +1265,7 @@ iwl_mvm_decode_he_phy_ru_alloc(struct iwl_mvm_rx_phy_data *phy_data,
 	 * the TSF/timers are not be transmitted in HE-MU.
 	 */
 	u8 ru = le32_get_bits(phy_data->d1, IWL_RX_PHY_DATA1_HE_RU_ALLOC_MASK);
+	u32 rate_n_flags = phy_data->rate_n_flags;
 	u32 he_type = rate_n_flags & RATE_MCS_HE_TYPE_MSK_V1;
 	u8 offs = 0;
 
@@ -1331,7 +1337,7 @@ static void iwl_mvm_decode_he_phy_data(struct iwl_mvm *mvm,
 				       struct ieee80211_radiotap_he *he,
 				       struct ieee80211_radiotap_he_mu *he_mu,
 				       struct ieee80211_rx_status *rx_status,
-				       u32 rate_n_flags, int queue)
+				       int queue)
 {
 	switch (phy_data->info_type) {
 	case IWL_RX_PHY_INFO_TYPE_NONE:
@@ -1430,7 +1436,7 @@ static void iwl_mvm_decode_he_phy_data(struct iwl_mvm *mvm,
 			le16_encode_bits(le16_get_bits(phy_data->d4,
 						       IWL_RX_PHY_DATA4_HE_MU_EXT_PREAMBLE_PUNC_TYPE_MASK),
 					 IEEE80211_RADIOTAP_HE_MU_FLAGS2_PUNC_FROM_SIG_A_BW);
-		iwl_mvm_decode_he_mu_ext(mvm, phy_data, rate_n_flags, he_mu);
+		iwl_mvm_decode_he_mu_ext(mvm, phy_data, he_mu);
 		fallthrough;
 	case IWL_RX_PHY_INFO_TYPE_HE_MU:
 		he_mu->flags2 |=
@@ -1444,8 +1450,7 @@ static void iwl_mvm_decode_he_phy_data(struct iwl_mvm *mvm,
 		fallthrough;
 	case IWL_RX_PHY_INFO_TYPE_HE_TB:
 	case IWL_RX_PHY_INFO_TYPE_HE_TB_EXT:
-		iwl_mvm_decode_he_phy_ru_alloc(phy_data, rate_n_flags,
-					       he, he_mu, rx_status);
+		iwl_mvm_decode_he_phy_ru_alloc(phy_data, he, he_mu, rx_status);
 		break;
 	case IWL_RX_PHY_INFO_TYPE_HE_SU:
 		he->data1 |= cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_BEAM_CHANGE_KNOWN);
@@ -1461,13 +1466,14 @@ static void iwl_mvm_decode_he_phy_data(struct iwl_mvm *mvm,
 
 static void iwl_mvm_rx_he(struct iwl_mvm *mvm, struct sk_buff *skb,
 			  struct iwl_mvm_rx_phy_data *phy_data,
-			  u32 rate_n_flags, u16 phy_info, int queue)
+			  int queue)
 {
 	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_radiotap_he *he = NULL;
 	struct ieee80211_radiotap_he_mu *he_mu = NULL;
+	u32 rate_n_flags = phy_data->rate_n_flags;
 	u32 he_type = rate_n_flags & RATE_MCS_HE_TYPE_MSK;
-	u8 stbc, ltf;
+	u8 ltf;
 	static const struct ieee80211_radiotap_he known = {
 		.data1 = cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_DATA_MCS_KNOWN |
 				     IEEE80211_RADIOTAP_HE_DATA1_DATA_DCM_KNOWN |
@@ -1484,6 +1490,7 @@ static void iwl_mvm_rx_he(struct iwl_mvm *mvm, struct sk_buff *skb,
 		.flags2 = cpu_to_le16(IEEE80211_RADIOTAP_HE_MU_FLAGS2_PUNC_FROM_SIG_A_BW_KNOWN |
 				      IEEE80211_RADIOTAP_HE_MU_FLAGS2_BW_FROM_SIG_A_BW_KNOWN),
 	};
+	u16 phy_info = phy_data->phy_info;
 
 	he = skb_put_data(skb, &known, sizeof(known));
 	rx_status->flag |= RX_FLAG_RADIOTAP_HE;
@@ -1504,7 +1511,7 @@ static void iwl_mvm_rx_he(struct iwl_mvm *mvm, struct sk_buff *skb,
 
 	if (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD)
 		iwl_mvm_decode_he_phy_data(mvm, phy_data, he, he_mu, rx_status,
-					   rate_n_flags, queue);
+					   queue);
 
 	/* update aggregation data for monitor sake on default queue */
 	if (!queue && (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD) &&
@@ -1531,19 +1538,6 @@ static void iwl_mvm_rx_he(struct iwl_mvm *mvm, struct sk_buff *skb,
 		he->data1 |=
 			cpu_to_le16(IEEE80211_RADIOTAP_HE_DATA1_BW_RU_ALLOC_KNOWN);
 
-	stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >> RATE_MCS_STBC_POS;
-	rx_status->nss =
-		((rate_n_flags & RATE_MCS_NSS_MSK) >>
-		 RATE_MCS_NSS_POS) + 1;
-	rx_status->rate_idx = rate_n_flags & RATE_MCS_CODE_MSK;
-	rx_status->encoding = RX_ENC_HE;
-	rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
-	if (rate_n_flags & RATE_MCS_BF_MSK)
-		rx_status->enc_flags |= RX_ENC_FLAG_BF;
-
-	rx_status->he_dcm =
-		!!(rate_n_flags & RATE_HE_DUAL_CARRIER_MODE_MSK);
-
 #define CHECK_TYPE(F)							\
 	BUILD_BUG_ON(IEEE80211_RADIOTAP_HE_DATA1_FORMAT_ ## F !=	\
 		     (RATE_MCS_HE_TYPE_ ## F >> RATE_MCS_HE_TYPE_POS))
@@ -1661,6 +1655,107 @@ static void iwl_mvm_rx_get_sta_block_tx(void *data, struct ieee80211_sta *sta)
 		rx_sta_csa->all_sta_unblocked = false;
 }
 
+/*
+ * Note: requires also rx_status->band to be prefilled, as well
+ * as phy_data (apart from phy_data->info_type)
+ */
+static void iwl_mvm_rx_fill_status(struct iwl_mvm *mvm,
+				   struct sk_buff *skb,
+				   struct iwl_mvm_rx_phy_data *phy_data,
+				   int queue)
+{
+	struct ieee80211_rx_status *rx_status = IEEE80211_SKB_RXCB(skb);
+	u32 rate_n_flags = phy_data->rate_n_flags;
+	u8 stbc = u32_get_bits(rate_n_flags, RATE_MCS_STBC_MSK);
+	u32 format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
+	bool is_sgi;
+
+	phy_data->info_type = IWL_RX_PHY_INFO_TYPE_NONE;
+
+	if (phy_data->phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD)
+		phy_data->info_type =
+			le32_get_bits(phy_data->d1,
+				      IWL_RX_PHY_DATA1_INFO_TYPE_MASK);
+
+	/* This may be overridden by iwl_mvm_rx_he() to HE_RU */
+	switch (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) {
+	case RATE_MCS_CHAN_WIDTH_20:
+		break;
+	case RATE_MCS_CHAN_WIDTH_40:
+		rx_status->bw = RATE_INFO_BW_40;
+		break;
+	case RATE_MCS_CHAN_WIDTH_80:
+		rx_status->bw = RATE_INFO_BW_80;
+		break;
+	case RATE_MCS_CHAN_WIDTH_160:
+		rx_status->bw = RATE_INFO_BW_160;
+		break;
+	}
+
+	/* must be before L-SIG data */
+	if (format == RATE_MCS_HE_MSK)
+		iwl_mvm_rx_he(mvm, skb, phy_data, queue);
+
+	iwl_mvm_decode_lsig(skb, phy_data);
+
+	rx_status->device_timestamp = phy_data->gp2_on_air_rise;
+	rx_status->freq = ieee80211_channel_to_frequency(phy_data->channel,
+							 rx_status->band);
+	iwl_mvm_get_signal_strength(mvm, rx_status, rate_n_flags,
+				    phy_data->energy_a, phy_data->energy_b);
+
+	if (unlikely(mvm->monitor_on))
+		iwl_mvm_add_rtap_sniffer_config(mvm, skb);
+
+	is_sgi = format == RATE_MCS_HE_MSK ?
+		iwl_he_is_sgi(rate_n_flags) :
+		rate_n_flags & RATE_MCS_SGI_MSK;
+
+	if (!(format == RATE_MCS_CCK_MSK) && is_sgi)
+		rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
+
+	if (rate_n_flags & RATE_MCS_LDPC_MSK)
+		rx_status->enc_flags |= RX_ENC_FLAG_LDPC;
+
+	switch (format) {
+	case RATE_MCS_VHT_MSK:
+		rx_status->encoding = RX_ENC_VHT;
+		break;
+	case RATE_MCS_HE_MSK:
+		rx_status->encoding = RX_ENC_HE;
+		rx_status->he_dcm =
+			!!(rate_n_flags & RATE_HE_DUAL_CARRIER_MODE_MSK);
+		break;
+	}
+
+	switch (format) {
+	case RATE_MCS_HT_MSK:
+		rx_status->encoding = RX_ENC_HT;
+		rx_status->rate_idx = RATE_HT_MCS_INDEX(rate_n_flags);
+		rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
+		break;
+	case RATE_MCS_VHT_MSK:
+	case RATE_MCS_HE_MSK:
+		rx_status->nss =
+			u32_get_bits(rate_n_flags, RATE_MCS_NSS_MSK) + 1;
+		rx_status->rate_idx = rate_n_flags & RATE_MCS_CODE_MSK;
+		rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
+		break;
+	default: {
+		int rate = iwl_mvm_legacy_hw_idx_to_mac80211_idx(rate_n_flags,
+								 rx_status->band);
+
+		rx_status->rate_idx = rate;
+
+		if (WARN_ONCE(rate < 0 || rate > 0xFF,
+			      "Invalid rate flags 0x%x, band %d,\n",
+			      rate_n_flags, rx_status->band))
+			rx_status->rate_idx = 0;
+		break;
+		}
+	}
+}
+
 void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 			struct iwl_rx_cmd_buffer *rxb, int queue)
 {
@@ -1670,17 +1765,12 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 	struct ieee80211_hdr *hdr;
 	u32 len;
 	u32 pkt_len = iwl_rx_packet_payload_len(pkt);
-	u32 rate_n_flags, gp2_on_air_rise;
-	u16 phy_info;
 	struct ieee80211_sta *sta = NULL;
 	struct sk_buff *skb;
-	u8 crypt_len = 0, channel, energy_a, energy_b;
+	u8 crypt_len = 0;
 	size_t desc_size;
-	struct iwl_mvm_rx_phy_data phy_data = {
-		.info_type = IWL_RX_PHY_INFO_TYPE_NONE,
-	};
+	struct iwl_mvm_rx_phy_data phy_data = {};
 	u32 format;
-	bool is_sgi;
 
 	if (unlikely(test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)))
 		return;
@@ -1696,35 +1786,37 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 	}
 
 	if (mvm->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
-		rate_n_flags = le32_to_cpu(desc->v3.rate_n_flags);
-		channel = desc->v3.channel;
-		gp2_on_air_rise = le32_to_cpu(desc->v3.gp2_on_air_rise);
-		energy_a = desc->v3.energy_a;
-		energy_b = desc->v3.energy_b;
+		phy_data.rate_n_flags = le32_to_cpu(desc->v3.rate_n_flags);
+		phy_data.channel = desc->v3.channel;
+		phy_data.gp2_on_air_rise = le32_to_cpu(desc->v3.gp2_on_air_rise);
+		phy_data.energy_a = desc->v3.energy_a;
+		phy_data.energy_b = desc->v3.energy_b;
 
 		phy_data.d0 = desc->v3.phy_data0;
 		phy_data.d1 = desc->v3.phy_data1;
 		phy_data.d2 = desc->v3.phy_data2;
 		phy_data.d3 = desc->v3.phy_data3;
 	} else {
-		rate_n_flags = le32_to_cpu(desc->v1.rate_n_flags);
-		channel = desc->v1.channel;
-		gp2_on_air_rise = le32_to_cpu(desc->v1.gp2_on_air_rise);
-		energy_a = desc->v1.energy_a;
-		energy_b = desc->v1.energy_b;
+		phy_data.rate_n_flags = le32_to_cpu(desc->v1.rate_n_flags);
+		phy_data.channel = desc->v1.channel;
+		phy_data.gp2_on_air_rise = le32_to_cpu(desc->v1.gp2_on_air_rise);
+		phy_data.energy_a = desc->v1.energy_a;
+		phy_data.energy_b = desc->v1.energy_b;
 
 		phy_data.d0 = desc->v1.phy_data0;
 		phy_data.d1 = desc->v1.phy_data1;
 		phy_data.d2 = desc->v1.phy_data2;
 		phy_data.d3 = desc->v1.phy_data3;
 	}
+
 	if (iwl_fw_lookup_notif_ver(mvm->fw, LEGACY_GROUP,
 				    REPLY_RX_MPDU_CMD, 0) < 4) {
-		rate_n_flags = iwl_new_rate_from_v1(rate_n_flags);
+		phy_data.rate_n_flags = iwl_new_rate_from_v1(phy_data.rate_n_flags);
 		IWL_DEBUG_DROP(mvm, "Got old format rate, converting. New rate: 0x%x\n",
-			       rate_n_flags);
+			       phy_data.rate_n_flags);
 	}
-	format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
+
+	format = phy_data.rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
 
 	len = le16_to_cpu(desc->mpdu_len);
 
@@ -1733,14 +1825,9 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		return;
 	}
 
-	phy_info = le16_to_cpu(desc->phy_info);
+	phy_data.phy_info = le16_to_cpu(desc->phy_info);
 	phy_data.d4 = desc->phy_data4;
 
-	if (phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD)
-		phy_data.info_type =
-			le32_get_bits(phy_data.d1,
-				      IWL_RX_PHY_DATA1_INFO_TYPE_MASK);
-
 	hdr = (void *)(pkt->data + desc_size);
 	/* Dont use dev_alloc_skb(), we'll have enough headroom once
 	 * ieee80211_hdr pulled.
@@ -1763,27 +1850,6 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 
 	rx_status = IEEE80211_SKB_RXCB(skb);
 
-	/* This may be overridden by iwl_mvm_rx_he() to HE_RU */
-	switch (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) {
-	case RATE_MCS_CHAN_WIDTH_20:
-		break;
-	case RATE_MCS_CHAN_WIDTH_40:
-		rx_status->bw = RATE_INFO_BW_40;
-		break;
-	case RATE_MCS_CHAN_WIDTH_80:
-		rx_status->bw = RATE_INFO_BW_80;
-		break;
-	case RATE_MCS_CHAN_WIDTH_160:
-		rx_status->bw = RATE_INFO_BW_160;
-		break;
-	}
-
-	if (format == RATE_MCS_HE_MSK)
-		iwl_mvm_rx_he(mvm, skb, &phy_data, rate_n_flags,
-			      phy_info, queue);
-
-	iwl_mvm_decode_lsig(skb, &phy_data);
-
 	/*
 	 * Keep packets with CRC errors (and with overrun) for monitor mode
 	 * (otherwise the firmware discards them) but mark them as bad.
@@ -1794,12 +1860,13 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 			     le32_to_cpu(desc->status));
 		rx_status->flag |= RX_FLAG_FAILED_FCS_CRC;
 	}
+
 	/* set the preamble flag if appropriate */
 	if (format == RATE_MCS_CCK_MSK &&
-	    phy_info & IWL_RX_MPDU_PHY_SHORT_PREAMBLE)
+	    phy_data.phy_info & IWL_RX_MPDU_PHY_SHORT_PREAMBLE)
 		rx_status->enc_flags |= RX_ENC_FLAG_SHORTPRE;
 
-	if (likely(!(phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD))) {
+	if (likely(!(phy_data.phy_info & IWL_RX_MPDU_PHY_TSF_OVERLOAD))) {
 		u64 tsf_on_air_rise;
 
 		if (mvm->trans->trans_cfg->device_family >=
@@ -1813,24 +1880,20 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		rx_status->flag |= RX_FLAG_MACTIME_PLCP_START;
 	}
 
-	rx_status->device_timestamp = gp2_on_air_rise;
 	if (iwl_mvm_is_band_in_rx_supported(mvm)) {
 		u8 band = BAND_IN_RX_STATUS(desc->mac_phy_idx);
 
 		rx_status->band = iwl_mvm_nl80211_band_from_rx_msdu(band);
 	} else {
-		rx_status->band = channel > 14 ? NL80211_BAND_5GHZ :
+		rx_status->band = phy_data.channel > 14 ? NL80211_BAND_5GHZ :
 			NL80211_BAND_2GHZ;
 	}
-	rx_status->freq = ieee80211_channel_to_frequency(channel,
-							 rx_status->band);
-	iwl_mvm_get_signal_strength(mvm, rx_status, rate_n_flags, energy_a,
-				    energy_b);
 
 	/* update aggregation data for monitor sake on default queue */
-	if (!queue && (phy_info & IWL_RX_MPDU_PHY_AMPDU)) {
-		bool toggle_bit = phy_info & IWL_RX_MPDU_PHY_AMPDU_TOGGLE;
+	if (!queue && (phy_data.phy_info & IWL_RX_MPDU_PHY_AMPDU)) {
+		bool toggle_bit;
 
+		toggle_bit = phy_data.phy_info & IWL_RX_MPDU_PHY_AMPDU_TOGGLE;
 		rx_status->flag |= RX_FLAG_AMPDU_DETAILS;
 		/*
 		 * Toggle is switched whenever new aggregation starts. Make
@@ -1846,9 +1909,6 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		rx_status->ampdu_reference = mvm->ampdu_ref;
 	}
 
-	if (unlikely(mvm->monitor_on))
-		iwl_mvm_add_rtap_sniffer_config(mvm, skb);
-
 	rcu_read_lock();
 
 	if (desc->status & cpu_to_le32(IWL_RX_MPDU_STATUS_SRC_STA_FOUND)) {
@@ -1867,13 +1927,15 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		sta = ieee80211_find_sta_by_ifaddr(mvm->hw, hdr->addr2, NULL);
 	}
 
-	if (iwl_mvm_rx_crypto(mvm, sta, hdr, rx_status, phy_info, desc,
+	if (iwl_mvm_rx_crypto(mvm, sta, hdr, rx_status, phy_data.phy_info, desc,
 			      le32_to_cpu(pkt->len_n_flags), queue,
 			      &crypt_len)) {
 		kfree_skb(skb);
 		goto out;
 	}
 
+	iwl_mvm_rx_fill_status(mvm, skb, &phy_data, queue);
+
 	if (sta) {
 		struct iwl_mvm_sta *mvmsta = iwl_mvm_sta_from_mac80211(sta);
 		struct ieee80211_vif *tx_blocked_vif =
@@ -1971,43 +2033,6 @@ void iwl_mvm_rx_mpdu_mq(struct iwl_mvm *mvm, struct napi_struct *napi,
 		}
 	}
 
-	is_sgi = format == RATE_MCS_HE_MSK ?
-		iwl_he_is_sgi(rate_n_flags) :
-		rate_n_flags & RATE_MCS_SGI_MSK;
-
-	if (!(format == RATE_MCS_CCK_MSK) && is_sgi)
-		rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
-	if (rate_n_flags & RATE_MCS_LDPC_MSK)
-		rx_status->enc_flags |= RX_ENC_FLAG_LDPC;
-	if (format == RATE_MCS_HT_MSK) {
-		u8 stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >>
-			RATE_MCS_STBC_POS;
-		rx_status->encoding = RX_ENC_HT;
-		rx_status->rate_idx = RATE_HT_MCS_INDEX(rate_n_flags);
-		rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
-	} else if (format == RATE_MCS_VHT_MSK) {
-		u8 stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >>
-			RATE_MCS_STBC_POS;
-		rx_status->nss = ((rate_n_flags & RATE_MCS_NSS_MSK) >>
-			RATE_MCS_NSS_POS) + 1;
-		rx_status->rate_idx = rate_n_flags & RATE_MCS_CODE_MSK;
-		rx_status->encoding = RX_ENC_VHT;
-		rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
-		if (rate_n_flags & RATE_MCS_BF_MSK)
-			rx_status->enc_flags |= RX_ENC_FLAG_BF;
-	} else if (!(format == RATE_MCS_HE_MSK)) {
-		int rate = iwl_mvm_legacy_hw_idx_to_mac80211_idx(rate_n_flags,
-								 rx_status->band);
-
-		if (WARN(rate < 0 || rate > 0xFF,
-			 "Invalid rate flags 0x%x, band %d,\n",
-			 rate_n_flags, rx_status->band)) {
-			kfree_skb(skb);
-			goto out;
-		}
-		rx_status->rate_idx = rate;
-	}
-
 	/* management stuff on default queue */
 	if (!queue) {
 		if (unlikely((ieee80211_is_beacon(hdr->frame_control) ||
@@ -2039,32 +2064,32 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 	struct ieee80211_rx_status *rx_status;
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
 	struct iwl_rx_no_data *desc = (void *)pkt->data;
-	u32 rate_n_flags = le32_to_cpu(desc->rate);
-	u32 gp2_on_air_rise = le32_to_cpu(desc->on_air_rise_time);
 	u32 rssi = le32_to_cpu(desc->rssi);
 	u32 info_type = le32_to_cpu(desc->info) & RX_NO_DATA_INFO_TYPE_MSK;
-	u16 phy_info = IWL_RX_MPDU_PHY_TSF_OVERLOAD;
 	struct ieee80211_sta *sta = NULL;
 	struct sk_buff *skb;
-	u8 channel, energy_a, energy_b;
-	u32 format;
 	struct iwl_mvm_rx_phy_data phy_data = {
-		.info_type = le32_get_bits(desc->phy_info[1],
-					   IWL_RX_PHY_DATA1_INFO_TYPE_MASK),
 		.d0 = desc->phy_info[0],
 		.d1 = desc->phy_info[1],
+		.phy_info = IWL_RX_MPDU_PHY_TSF_OVERLOAD,
+		.gp2_on_air_rise = le32_to_cpu(desc->on_air_rise_time),
+		.rate_n_flags = le32_to_cpu(desc->rate),
+		.energy_a = u32_get_bits(rssi, RX_NO_DATA_CHAIN_A_MSK),
+		.energy_b = u32_get_bits(rssi, RX_NO_DATA_CHAIN_B_MSK),
+		.channel = u32_get_bits(rssi, RX_NO_DATA_CHANNEL_MSK),
 	};
-	bool is_sgi;
+	u32 format;
 
 	if (iwl_fw_lookup_notif_ver(mvm->fw, DATA_PATH_GROUP,
 				    RX_NO_DATA_NOTIF, 0) < 2) {
 		IWL_DEBUG_DROP(mvm, "Got an old rate format. Old rate: 0x%x\n",
-			       rate_n_flags);
-		rate_n_flags = iwl_new_rate_from_v1(rate_n_flags);
+			       phy_data.rate_n_flags);
+		phy_data.rate_n_flags = iwl_new_rate_from_v1(phy_data.rate_n_flags);
 		IWL_DEBUG_DROP(mvm, " Rate after conversion to the new format: 0x%x\n",
-			       rate_n_flags);
+			       phy_data.rate_n_flags);
 	}
-	format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
+
+	format = phy_data.rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
 
 	if (unlikely(iwl_rx_packet_payload_len(pkt) < sizeof(*desc)))
 		return;
@@ -2072,10 +2097,6 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 	if (unlikely(test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)))
 		return;
 
-	energy_a = (rssi & RX_NO_DATA_CHAIN_A_MSK) >> RX_NO_DATA_CHAIN_A_POS;
-	energy_b = (rssi & RX_NO_DATA_CHAIN_B_MSK) >> RX_NO_DATA_CHAIN_B_POS;
-	channel = (rssi & RX_NO_DATA_CHANNEL_MSK) >> RX_NO_DATA_CHANNEL_POS;
-
 	/* Dont use dev_alloc_skb(), we'll have enough headroom once
 	 * ieee80211_hdr pulled.
 	 */
@@ -2106,86 +2127,31 @@ void iwl_mvm_rx_monitor_no_data(struct iwl_mvm *mvm, struct napi_struct *napi,
 		break;
 	}
 
-	/* This may be overridden by iwl_mvm_rx_he() to HE_RU */
-	switch (rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK) {
-	case RATE_MCS_CHAN_WIDTH_20:
-		break;
-	case RATE_MCS_CHAN_WIDTH_40:
-		rx_status->bw = RATE_INFO_BW_40;
-		break;
-	case RATE_MCS_CHAN_WIDTH_80:
-		rx_status->bw = RATE_INFO_BW_80;
-		break;
-	case RATE_MCS_CHAN_WIDTH_160:
-		rx_status->bw = RATE_INFO_BW_160;
-		break;
-	}
-
-	if (format == RATE_MCS_HE_MSK)
-		iwl_mvm_rx_he(mvm, skb, &phy_data, rate_n_flags,
-			      phy_info, queue);
-
-	iwl_mvm_decode_lsig(skb, &phy_data);
-
-	rx_status->device_timestamp = gp2_on_air_rise;
-	rx_status->band = channel > 14 ? NL80211_BAND_5GHZ :
+	rx_status->band = phy_data.channel > 14 ? NL80211_BAND_5GHZ :
 		NL80211_BAND_2GHZ;
-	rx_status->freq = ieee80211_channel_to_frequency(channel,
-							 rx_status->band);
-	iwl_mvm_get_signal_strength(mvm, rx_status, rate_n_flags, energy_a,
-				    energy_b);
 
-	rcu_read_lock();
+	iwl_mvm_rx_fill_status(mvm, skb, &phy_data, queue);
 
-	is_sgi = format == RATE_MCS_HE_MSK ?
-		iwl_he_is_sgi(rate_n_flags) :
-		rate_n_flags & RATE_MCS_SGI_MSK;
-
-	if (!(format == RATE_MCS_CCK_MSK) && is_sgi)
-		rx_status->enc_flags |= RX_ENC_FLAG_SHORT_GI;
-	if (rate_n_flags & RATE_MCS_LDPC_MSK)
-		rx_status->enc_flags |= RX_ENC_FLAG_LDPC;
-	if (format == RATE_MCS_HT_MSK) {
-		u8 stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >>
-				RATE_MCS_STBC_POS;
-		rx_status->encoding = RX_ENC_HT;
-		rx_status->rate_idx = RATE_HT_MCS_INDEX(rate_n_flags);
-		rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
-	} else if (format == RATE_MCS_VHT_MSK) {
-		u8 stbc = (rate_n_flags & RATE_MCS_STBC_MSK) >>
-				RATE_MCS_STBC_POS;
-		rx_status->rate_idx = rate_n_flags & RATE_MCS_CODE_MSK;
-		rx_status->encoding = RX_ENC_VHT;
-		rx_status->enc_flags |= stbc << RX_ENC_FLAG_STBC_SHIFT;
-		if (rate_n_flags & RATE_MCS_BF_MSK)
-			rx_status->enc_flags |= RX_ENC_FLAG_BF;
-		/*
-		 * take the nss from the rx_vec since the rate_n_flags has
-		 * only 2 bits for the nss which gives a max of 4 ss but
-		 * there may be up to 8 spatial streams
-		 */
+	/*
+	 * Override the nss from the rx_vec since the rate_n_flags has
+	 * only 2 bits for the nss which gives a max of 4 ss but there
+	 * may be up to 8 spatial streams.
+	 */
+	switch (format) {
+	case RATE_MCS_VHT_MSK:
 		rx_status->nss =
 			le32_get_bits(desc->rx_vec[0],
 				      RX_NO_DATA_RX_VEC0_VHT_NSTS_MSK) + 1;
-	} else if (format == RATE_MCS_HE_MSK) {
+		break;
+	case RATE_MCS_HE_MSK:
 		rx_status->nss =
 			le32_get_bits(desc->rx_vec[0],
 				      RX_NO_DATA_RX_VEC0_HE_NSTS_MSK) + 1;
-	} else {
-		int rate = iwl_mvm_legacy_hw_idx_to_mac80211_idx(rate_n_flags,
-							       rx_status->band);
-
-		if (WARN(rate < 0 || rate > 0xFF,
-			 "Invalid rate flags 0x%x, band %d,\n",
-			 rate_n_flags, rx_status->band)) {
-			kfree_skb(skb);
-			goto out;
-		}
-		rx_status->rate_idx = rate;
+		break;
 	}
 
+	rcu_read_lock();
 	ieee80211_rx_napi(mvm->hw, sta, skb, napi);
-out:
 	rcu_read_unlock();
 }
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
index 582a95ffc7ab..acd8803dbcdd 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/scan.c
@@ -2626,7 +2626,7 @@ static int iwl_mvm_build_scan_cmd(struct iwl_mvm *mvm,
 	u8 scan_ver;
 
 	lockdep_assert_held(&mvm->mutex);
-	memset(mvm->scan_cmd, 0, ksize(mvm->scan_cmd));
+	memset(mvm->scan_cmd, 0, mvm->scan_cmd_size);
 
 	if (!fw_has_capa(&mvm->fw->ucode_capa, IWL_UCODE_TLV_CAPA_UMAC_SCAN)) {
 		hcmd->id = SCAN_OFFLOAD_REQUEST_CMD;
@@ -3091,7 +3091,7 @@ static int iwl_mvm_scan_stop_wait(struct iwl_mvm *mvm, int type)
 				     1 * HZ);
 }
 
-static int iwl_scan_req_umac_get_size(u8 scan_ver)
+static size_t iwl_scan_req_umac_get_size(u8 scan_ver)
 {
 	switch (scan_ver) {
 	case 12:
@@ -3104,7 +3104,7 @@ static int iwl_scan_req_umac_get_size(u8 scan_ver)
 	return 0;
 }
 
-int iwl_mvm_scan_size(struct iwl_mvm *mvm)
+size_t iwl_mvm_scan_size(struct iwl_mvm *mvm)
 {
 	int base_size, tail_size;
 	u8 scan_ver = iwl_fw_lookup_cmd_ver(mvm->fw, SCAN_REQ_UMAC,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
index ff0d3b3df140..cc92706b3d16 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.c
@@ -116,7 +116,7 @@ int iwl_mvm_sta_send_to_fw(struct iwl_mvm *mvm, struct ieee80211_sta *sta,
 		break;
 	}
 
-	switch (sta->smps_mode) {
+	switch (sta->deflink.smps_mode) {
 	case IEEE80211_SMPS_AUTOMATIC:
 	case IEEE80211_SMPS_NUM_MODES:
 		WARN_ON(1);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
index f9e08b339e0c..86d20e13bf47 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c
@@ -926,7 +926,7 @@ static int iwl_mvm_tx_tso(struct iwl_mvm *mvm, struct sk_buff *skb,
 	 * Take the min of ieee80211 station and mvm station
 	 */
 	max_amsdu_len =
-		min_t(unsigned int, sta->max_amsdu_len,
+		min_t(unsigned int, sta->cur->max_amsdu_len,
 		      iwl_mvm_max_amsdu_size(mvm, sta, tid));
 
 	/*
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index b16d4ae182d1..4f699862e7f7 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -1155,10 +1155,20 @@ static const struct iwl_dev_info iwl_dev_info_table[] = {
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_ANY,
 		      iwl_cfg_bz_a0_fm_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_BZ, IWL_CFG_ANY,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_NO_JACKET,
+		      iwl_cfg_bz_a0_fm4_a0, iwl_bz_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_GL, SILICON_A_STEP,
 		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_NO_JACKET,
 		      iwl_cfg_gl_a0_fm_a0, iwl_bz_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_NO_JACKET,
+		      iwl_cfg_gl_b0_fm_b0, iwl_bz_name),
 
 /* BZ Z step */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
@@ -1169,11 +1179,16 @@ static const struct iwl_dev_info iwl_dev_info_table[] = {
 
 /* BNJ */
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
-		      IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_GL, SILICON_A_STEP,
 		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_IS_JACKET,
 		      iwl_cfg_bnj_a0_fm_a0, iwl_bz_name),
 	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
+		      IWL_CFG_MAC_TYPE_GL, SILICON_B_STEP,
+		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
+		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_NO_CDB, IWL_CFG_IS_JACKET,
+		      iwl_cfg_bnj_b0_fm_b0, iwl_bz_name),
+	_IWL_DEV_INFO(IWL_CFG_ANY, IWL_CFG_ANY,
 		      IWL_CFG_MAC_TYPE_GL, IWL_CFG_ANY,
 		      IWL_CFG_RF_TYPE_FM, IWL_CFG_ANY,
 		      IWL_CFG_ANY, IWL_CFG_ANY, IWL_CFG_CDB, IWL_CFG_IS_JACKET,
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
index 68a4572cee53..9c9f87fe8377 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/rx.c
@@ -1110,7 +1110,7 @@ static int _iwl_pcie_rx_init(struct iwl_trans *trans)
 				poll = iwl_pcie_napi_poll_msix;
 
 			netif_napi_add(&trans_pcie->napi_dev, &rxq->napi,
-				       poll, NAPI_POLL_WEIGHT);
+				       poll);
 			napi_enable(&rxq->napi);
 		}
 
diff --git a/drivers/net/wireless/intersil/hostap/hostap_ioctl.c b/drivers/net/wireless/intersil/hostap/hostap_ioctl.c
index 0a376f112db9..4e0a0c881697 100644
--- a/drivers/net/wireless/intersil/hostap/hostap_ioctl.c
+++ b/drivers/net/wireless/intersil/hostap/hostap_ioctl.c
@@ -3848,7 +3848,7 @@ static void prism2_get_drvinfo(struct net_device *dev,
 	iface = netdev_priv(dev);
 	local = iface->local;
 
-	strlcpy(info->driver, "hostap", sizeof(info->driver));
+	strscpy(info->driver, "hostap", sizeof(info->driver));
 	snprintf(info->fw_version, sizeof(info->fw_version),
 		 "%d.%d.%d", (local->sta_fw_ver >> 16) & 0xff,
 		 (local->sta_fw_ver >> 8) & 0xff,
diff --git a/drivers/net/wireless/intersil/p54/main.c b/drivers/net/wireless/intersil/p54/main.c
index b925e327e091..e127453ab51a 100644
--- a/drivers/net/wireless/intersil/p54/main.c
+++ b/drivers/net/wireless/intersil/p54/main.c
@@ -635,7 +635,7 @@ static int p54_get_survey(struct ieee80211_hw *dev, int idx,
 				/*
 				 * hw/fw has not accumulated enough sample sets.
 				 * Wait for 100ms, this ought to be enough to
-				 * to get at least one non-null set of channel
+				 * get at least one non-null set of channel
 				 * usage statistics.
 				 */
 				msleep(100);
diff --git a/drivers/net/wireless/mac80211_hwsim.c b/drivers/net/wireless/mac80211_hwsim.c
index 1f301a5fb396..df51b5b1f171 100644
--- a/drivers/net/wireless/mac80211_hwsim.c
+++ b/drivers/net/wireless/mac80211_hwsim.c
@@ -229,6 +229,7 @@ static inline void hwsim_clear_magic(struct ieee80211_vif *vif)
 struct hwsim_sta_priv {
 	u32 magic;
 	unsigned int last_link;
+	u16 active_links_rx;
 };
 
 #define HWSIM_STA_MAGIC	0x6d537749
@@ -652,7 +653,6 @@ struct mac80211_hwsim_data {
 	u32 ciphers[ARRAY_SIZE(hwsim_ciphers)];
 
 	struct mac_address addresses[2];
-	struct ieee80211_chanctx_conf *chanctx;
 	int channels, idx;
 	bool use_chanctx;
 	bool destroy_on_close;
@@ -1299,6 +1299,8 @@ static void mac80211_hwsim_config_mac_nl(struct ieee80211_hw *hw,
 	struct sk_buff *skb;
 	void *msg_head;
 
+	WARN_ON(!is_valid_ether_addr(addr));
+
 	if (!_portid && !hwsim_virtio_enabled)
 		return;
 
@@ -1561,6 +1563,42 @@ static void mac80211_hwsim_add_vendor_rtap(struct sk_buff *skb)
 #endif
 }
 
+static void mac80211_hwsim_rx(struct mac80211_hwsim_data *data,
+			      struct ieee80211_rx_status *rx_status,
+			      struct sk_buff *skb)
+{
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+
+	if (!ieee80211_has_morefrags(hdr->frame_control) &&
+	    !is_multicast_ether_addr(hdr->addr1) &&
+	    (ieee80211_is_mgmt(hdr->frame_control) ||
+	     ieee80211_is_data(hdr->frame_control))) {
+		struct ieee80211_sta *sta;
+		unsigned int link_id;
+
+		rcu_read_lock();
+		sta = ieee80211_find_sta_by_link_addrs(data->hw, hdr->addr2,
+						       hdr->addr1, &link_id);
+		if (sta) {
+			struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+
+			if (ieee80211_has_pm(hdr->frame_control))
+				sp->active_links_rx &= ~BIT(link_id);
+			else
+				sp->active_links_rx |= BIT(link_id);
+		}
+		rcu_read_unlock();
+	}
+
+	memcpy(IEEE80211_SKB_RXCB(skb), rx_status, sizeof(*rx_status));
+
+	mac80211_hwsim_add_vendor_rtap(skb);
+
+	data->rx_pkts++;
+	data->rx_bytes += skb->len;
+	ieee80211_rx_irqsafe(data->hw, skb);
+}
+
 static bool mac80211_hwsim_tx_frame_no_nl(struct ieee80211_hw *hw,
 					  struct sk_buff *skb,
 					  struct ieee80211_channel *chan)
@@ -1688,13 +1726,7 @@ static bool mac80211_hwsim_tx_frame_no_nl(struct ieee80211_hw *hw,
 
 		rx_status.mactime = now + data2->tsf_offset;
 
-		memcpy(IEEE80211_SKB_RXCB(nskb), &rx_status, sizeof(rx_status));
-
-		mac80211_hwsim_add_vendor_rtap(nskb);
-
-		data2->rx_pkts++;
-		data2->rx_bytes += nskb->len;
-		ieee80211_rx_irqsafe(data2->hw, nskb);
+		mac80211_hwsim_rx(data2, &rx_status, nskb);
 	}
 	spin_unlock(&hwsim_radio_lock);
 
@@ -1714,12 +1746,7 @@ mac80211_hwsim_select_tx_link(struct mac80211_hwsim_data *data,
 	if (!vif->valid_links)
 		return &vif->bss_conf;
 
-	/* FIXME: handle multicast TX properly */
-	if (is_multicast_ether_addr(hdr->addr1) || WARN_ON_ONCE(!sta)) {
-		unsigned int first_link = ffs(vif->valid_links) - 1;
-
-		return rcu_dereference(vif->link_conf[first_link]);
-	}
+	WARN_ON(is_multicast_ether_addr(hdr->addr1));
 
 	if (WARN_ON_ONCE(!sta->valid_links))
 		return &vif->bss_conf;
@@ -1731,6 +1758,12 @@ mac80211_hwsim_select_tx_link(struct mac80211_hwsim_data *data,
 		/* round-robin the available link IDs */
 		link_id = (sp->last_link + i + 1) % ARRAY_SIZE(vif->link_conf);
 
+		if (!(vif->active_links & BIT(link_id)))
+			continue;
+
+		if (!(sp->active_links_rx & BIT(link_id)))
+			continue;
+
 		*link_sta = rcu_dereference(sta->link[link_id]);
 		if (!*link_sta)
 			continue;
@@ -1739,6 +1772,10 @@ mac80211_hwsim_select_tx_link(struct mac80211_hwsim_data *data,
 		if (WARN_ON_ONCE(!bss_conf))
 			continue;
 
+		/* can happen while switching links */
+		if (!rcu_access_pointer(bss_conf->chanctx_conf))
+			continue;
+
 		sp->last_link = link_id;
 		return bss_conf;
 	}
@@ -2401,10 +2438,19 @@ static int mac80211_hwsim_sta_add(struct ieee80211_hw *hw,
 				  struct ieee80211_vif *vif,
 				  struct ieee80211_sta *sta)
 {
+	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+
 	hwsim_check_magic(vif);
 	hwsim_set_sta_magic(sta);
 	mac80211_hwsim_sta_rc_update(hw, vif, sta, 0);
 
+	if (sta->valid_links) {
+		WARN(hweight16(sta->valid_links) > 1,
+		     "expect to add STA with single link, have 0x%x\n",
+		     sta->valid_links);
+		sp->active_links_rx = sta->valid_links;
+	}
+
 	return 0;
 }
 
@@ -2430,6 +2476,14 @@ static int mac80211_hwsim_sta_state(struct ieee80211_hw *hw,
 	if (old_state == IEEE80211_STA_NOTEXIST)
 		return mac80211_hwsim_sta_add(hw, vif, sta);
 
+	/*
+	 * when client is authorized (AP station marked as such),
+	 * enable all links
+	 */
+	if (vif->type == NL80211_IFTYPE_STATION &&
+	    new_state == IEEE80211_STA_AUTHORIZED && !sta->tdls)
+		ieee80211_set_active_links_async(vif, vif->valid_links);
+
 	return 0;
 }
 
@@ -2866,11 +2920,6 @@ static int mac80211_hwsim_croc(struct ieee80211_hw *hw,
 static int mac80211_hwsim_add_chanctx(struct ieee80211_hw *hw,
 				      struct ieee80211_chanctx_conf *ctx)
 {
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-
-	mutex_lock(&hwsim->mutex);
-	hwsim->chanctx = ctx;
-	mutex_unlock(&hwsim->mutex);
 	hwsim_set_chanctx_magic(ctx);
 	wiphy_dbg(hw->wiphy,
 		  "add channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n",
@@ -2882,11 +2931,6 @@ static int mac80211_hwsim_add_chanctx(struct ieee80211_hw *hw,
 static void mac80211_hwsim_remove_chanctx(struct ieee80211_hw *hw,
 					  struct ieee80211_chanctx_conf *ctx)
 {
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-
-	mutex_lock(&hwsim->mutex);
-	hwsim->chanctx = NULL;
-	mutex_unlock(&hwsim->mutex);
 	wiphy_dbg(hw->wiphy,
 		  "remove channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n",
 		  ctx->def.chan->center_freq, ctx->def.width,
@@ -2899,11 +2943,6 @@ static void mac80211_hwsim_change_chanctx(struct ieee80211_hw *hw,
 					  struct ieee80211_chanctx_conf *ctx,
 					  u32 changed)
 {
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-
-	mutex_lock(&hwsim->mutex);
-	hwsim->chanctx = ctx;
-	mutex_unlock(&hwsim->mutex);
 	hwsim_check_chanctx_magic(ctx);
 	wiphy_dbg(hw->wiphy,
 		  "change channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n",
@@ -2919,6 +2958,18 @@ static int mac80211_hwsim_assign_vif_chanctx(struct ieee80211_hw *hw,
 	hwsim_check_magic(vif);
 	hwsim_check_chanctx_magic(ctx);
 
+	/* if we activate a link while already associated wake it up */
+	if (vif->type == NL80211_IFTYPE_STATION && vif->cfg.assoc) {
+		struct sk_buff *skb;
+
+		skb = ieee80211_nullfunc_get(hw, vif, link_conf->link_id, true);
+		if (skb) {
+			local_bh_disable();
+			mac80211_hwsim_tx_frame(hw, skb, ctx->def.chan);
+			local_bh_enable();
+		}
+	}
+
 	return 0;
 }
 
@@ -2929,6 +2980,22 @@ static void mac80211_hwsim_unassign_vif_chanctx(struct ieee80211_hw *hw,
 {
 	hwsim_check_magic(vif);
 	hwsim_check_chanctx_magic(ctx);
+
+	/* if we deactivate a link while associated suspend it first */
+	if (vif->type == NL80211_IFTYPE_STATION && vif->cfg.assoc) {
+		struct sk_buff *skb;
+
+		skb = ieee80211_nullfunc_get(hw, vif, link_conf->link_id, true);
+		if (skb) {
+			struct ieee80211_hdr *hdr = (void *)skb->data;
+
+			hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
+
+			local_bh_disable();
+			mac80211_hwsim_tx_frame(hw, skb, ctx->def.chan);
+			local_bh_enable();
+		}
+	}
 }
 
 static const char mac80211_hwsim_gstrings_stats[][ETH_GSTRING_LEN] = {
@@ -2995,18 +3062,22 @@ static int mac80211_hwsim_change_vif_links(struct ieee80211_hw *hw,
 					   u16 old_links, u16 new_links,
 					   struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS])
 {
-	unsigned long rem = old_links & ~new_links ?: BIT(0);
+	unsigned long rem = old_links & ~new_links;
 	unsigned long add = new_links & ~old_links;
 	int i;
 
+	if (!old_links)
+		rem |= BIT(0);
+	if (!new_links)
+		add |= BIT(0);
+
 	for_each_set_bit(i, &rem, IEEE80211_MLD_MAX_NUM_LINKS)
 		mac80211_hwsim_config_mac_nl(hw, old[i]->addr, false);
 
 	for_each_set_bit(i, &add, IEEE80211_MLD_MAX_NUM_LINKS) {
 		struct ieee80211_bss_conf *link_conf;
 
-		/* FIXME: figure out how to get the locking here */
-		link_conf = rcu_dereference_protected(vif->link_conf[i], 1);
+		link_conf = link_conf_dereference_protected(vif, i);
 		if (WARN_ON(!link_conf))
 			continue;
 
@@ -3021,6 +3092,13 @@ static int mac80211_hwsim_change_sta_links(struct ieee80211_hw *hw,
 					   struct ieee80211_sta *sta,
 					   u16 old_links, u16 new_links)
 {
+	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+
+	hwsim_check_sta_magic(sta);
+
+	if (vif->type == NL80211_IFTYPE_STATION)
+		sp->active_links_rx = new_links;
+
 	return 0;
 }
 
@@ -3208,8 +3286,112 @@ out_err:
 
 static const struct ieee80211_sband_iftype_data sband_capa_2ghz[] = {
 	{
-		.types_mask = BIT(NL80211_IFTYPE_STATION) |
-			      BIT(NL80211_IFTYPE_AP),
+		.types_mask = BIT(NL80211_IFTYPE_STATION),
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_BSR |
+					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] =
+					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
+					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
+					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xffff),
+				.tx_mcs_160 = cpu_to_le16(0xffff),
+				.rx_mcs_80p80 = cpu_to_le16(0xffff),
+				.tx_mcs_80p80 = cpu_to_le16(0xffff),
+			},
+		},
+		.eht_cap = {
+			.has_eht = true,
+			.eht_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
+					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
+				.phy_cap_info[0] =
+					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
+					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
+					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE,
+				.phy_cap_info[3] =
+					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
+				.phy_cap_info[4] =
+					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
+					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
+					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
+					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
+				.phy_cap_info[5] =
+					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
+					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
+					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
+				.phy_cap_info[6] =
+					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
+					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK,
+				.phy_cap_info[7] =
+					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW,
+			},
+
+			/* For all MCS and bandwidth, set 8 NSS for both Tx and
+			 * Rx
+			 */
+			.eht_mcs_nss_supp = {
+				/*
+				 * Since B0, B1, B2 and B3 are not set in
+				 * the supported channel width set field in the
+				 * HE PHY capabilities information field the
+				 * device is a 20MHz only device on 2.4GHz band.
+				 */
+				.only_20mhz = {
+					.rx_tx_mcs7_max_nss = 0x88,
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+			},
+			/* PPE threshold information is not supported */
+		},
+	},
+	{
+		.types_mask = BIT(NL80211_IFTYPE_AP),
 		.he_cap = {
 			.has_he = true,
 			.he_cap_elem = {
@@ -3356,9 +3538,132 @@ static const struct ieee80211_sband_iftype_data sband_capa_2ghz[] = {
 
 static const struct ieee80211_sband_iftype_data sband_capa_5ghz[] = {
 	{
-		/* TODO: should we support other types, e.g., P2P?*/
-		.types_mask = BIT(NL80211_IFTYPE_STATION) |
-			      BIT(NL80211_IFTYPE_AP),
+		/* TODO: should we support other types, e.g., P2P? */
+		.types_mask = BIT(NL80211_IFTYPE_STATION),
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_BSR |
+					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] =
+					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
+					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
+					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xfffa),
+				.tx_mcs_160 = cpu_to_le16(0xfffa),
+				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
+			},
+		},
+		.eht_cap = {
+			.has_eht = true,
+			.eht_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
+					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
+				.phy_cap_info[0] =
+					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
+					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
+					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
+					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
+				.phy_cap_info[1] =
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK,
+				.phy_cap_info[2] =
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK,
+				.phy_cap_info[3] =
+					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
+				.phy_cap_info[4] =
+					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
+					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
+					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
+					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
+				.phy_cap_info[5] =
+					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
+					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
+					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
+				.phy_cap_info[6] =
+					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
+					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK,
+				.phy_cap_info[7] =
+					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ,
+			},
+
+			/* For all MCS and bandwidth, set 8 NSS for both Tx and
+			 * Rx
+			 */
+			.eht_mcs_nss_supp = {
+				/*
+				 * As B1 and B2 are set in the supported
+				 * channel width set field in the HE PHY
+				 * capabilities information field include all
+				 * the following MCS/NSS.
+				 */
+				.bw._80 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._160 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+			},
+			/* PPE threshold information is not supported */
+		},
+	},
+	{
+		.types_mask = BIT(NL80211_IFTYPE_AP),
 		.he_cap = {
 			.has_he = true,
 			.he_cap_elem = {
@@ -3529,9 +3834,153 @@ static const struct ieee80211_sband_iftype_data sband_capa_5ghz[] = {
 
 static const struct ieee80211_sband_iftype_data sband_capa_6ghz[] = {
 	{
-		/* TODO: should we support other types, e.g., P2P?*/
-		.types_mask = BIT(NL80211_IFTYPE_STATION) |
-			      BIT(NL80211_IFTYPE_AP),
+		/* TODO: should we support other types, e.g., P2P? */
+		.types_mask = BIT(NL80211_IFTYPE_STATION),
+		.he_6ghz_capa = {
+			.capa = cpu_to_le16(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START |
+					    IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP |
+					    IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN |
+					    IEEE80211_HE_6GHZ_CAP_SM_PS |
+					    IEEE80211_HE_6GHZ_CAP_RD_RESPONDER |
+					    IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS |
+					    IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS),
+		},
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_BSR |
+					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] =
+					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
+					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
+					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xfffa),
+				.tx_mcs_160 = cpu_to_le16(0xfffa),
+				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
+			},
+		},
+		.eht_cap = {
+			.has_eht = true,
+			.eht_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
+					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
+				.phy_cap_info[0] =
+					IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ |
+					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
+					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
+					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
+					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
+				.phy_cap_info[1] =
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK,
+				.phy_cap_info[2] =
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK,
+				.phy_cap_info[3] =
+					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
+				.phy_cap_info[4] =
+					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
+					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
+					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
+					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
+				.phy_cap_info[5] =
+					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
+					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
+					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
+				.phy_cap_info[6] =
+					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
+					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK |
+					IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP,
+				.phy_cap_info[7] =
+					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_320MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_320MHZ,
+			},
+
+			/* For all MCS and bandwidth, set 8 NSS for both Tx and
+			 * Rx
+			 */
+			.eht_mcs_nss_supp = {
+				/*
+				 * As B1 and B2 are set in the supported
+				 * channel width set field in the HE PHY
+				 * capabilities information field and 320MHz in
+				 * 6GHz is supported include all the following
+				 * MCS/NSS.
+				 */
+				.bw._80 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._160 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._320 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+			},
+			/* PPE threshold information is not supported */
+		},
+	},
+	{
+		.types_mask = BIT(NL80211_IFTYPE_AP),
 		.he_6ghz_capa = {
 			.capa = cpu_to_le16(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START |
 					    IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP |
@@ -3896,7 +4345,6 @@ static int mac80211_hwsim_new_radio(struct genl_info *info,
 		hw->wiphy->max_remain_on_channel_duration = 1000;
 		data->if_combination.radar_detect_widths = 0;
 		data->if_combination.num_different_channels = data->channels;
-		data->chanctx = NULL;
 	} else {
 		data->if_combination.num_different_channels = 1;
 		data->if_combination.radar_detect_widths =
@@ -4471,13 +4919,9 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2,
 	if (data2->use_chanctx) {
 		if (data2->tmp_chan)
 			channel = data2->tmp_chan;
-		else if (data2->chanctx)
-			channel = data2->chanctx->def.chan;
 	} else {
 		channel = data2->channel;
 	}
-	if (!channel)
-		goto out;
 
 	if (!hwsim_virtio_enabled) {
 		if (hwsim_net_get_netgroup(genl_info_net(info)) !=
@@ -4508,6 +4952,7 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2,
 							  rx_status.freq);
 		if (!iter_data.channel)
 			goto out;
+		rx_status.band = iter_data.channel->band;
 
 		mutex_lock(&data2->mutex);
 		if (!hwsim_chans_compat(iter_data.channel, channel)) {
@@ -4520,11 +4965,13 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2,
 			}
 		}
 		mutex_unlock(&data2->mutex);
+	} else if (!channel) {
+		goto out;
 	} else {
 		rx_status.freq = channel->center_freq;
+		rx_status.band = channel->band;
 	}
 
-	rx_status.band = channel->band;
 	rx_status.rate_idx = nla_get_u32(info->attrs[HWSIM_ATTR_RX_RATE]);
 	rx_status.signal = nla_get_u32(info->attrs[HWSIM_ATTR_SIGNAL]);
 
@@ -4534,10 +4981,7 @@ static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2,
 	    ieee80211_is_probe_resp(hdr->frame_control))
 		rx_status.boottime_ns = ktime_get_boottime_ns();
 
-	memcpy(IEEE80211_SKB_RXCB(skb), &rx_status, sizeof(rx_status));
-	data2->rx_pkts++;
-	data2->rx_bytes += skb->len;
-	ieee80211_rx_irqsafe(data2->hw, skb);
+	mac80211_hwsim_rx(data2, &rx_status, skb);
 
 	return 0;
 err:
@@ -4912,6 +5356,7 @@ static struct genl_family hwsim_genl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.small_ops = hwsim_ops,
 	.n_small_ops = ARRAY_SIZE(hwsim_ops),
+	.resv_start_op = HWSIM_CMD_DEL_MAC_ADDR + 1,
 	.mcgrps = hwsim_mcgrps,
 	.n_mcgrps = ARRAY_SIZE(hwsim_mcgrps),
 };
diff --git a/drivers/net/wireless/marvell/libertas/cfg.c b/drivers/net/wireless/marvell/libertas/cfg.c
index b0b3f59dabc6..3e065cbb0af9 100644
--- a/drivers/net/wireless/marvell/libertas/cfg.c
+++ b/drivers/net/wireless/marvell/libertas/cfg.c
@@ -546,7 +546,7 @@ static int lbs_ret_scan(struct lbs_private *priv, unsigned long dummy,
 	pos = scanresp->bssdesc_and_tlvbuffer;
 
 	lbs_deb_hex(LBS_DEB_SCAN, "SCAN_RSP", scanresp->bssdesc_and_tlvbuffer,
-			scanresp->bssdescriptsize);
+		    bsssize);
 
 	tsfdesc = pos + bsssize;
 	tsfsize = 4 + 8 * scanresp->nr_sets;
@@ -1435,7 +1435,7 @@ static int lbs_cfg_disconnect(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int lbs_cfg_set_default_key(struct wiphy *wiphy,
-				   struct net_device *netdev,
+				   struct net_device *netdev, int link_id,
 				   u8 key_index, bool unicast,
 				   bool multicast)
 {
@@ -1455,8 +1455,8 @@ static int lbs_cfg_set_default_key(struct wiphy *wiphy,
 
 
 static int lbs_cfg_add_key(struct wiphy *wiphy, struct net_device *netdev,
-			   u8 idx, bool pairwise, const u8 *mac_addr,
-			   struct key_params *params)
+			   int link_id, u8 idx, bool pairwise,
+			   const u8 *mac_addr, struct key_params *params)
 {
 	struct lbs_private *priv = wiphy_priv(wiphy);
 	u16 key_info;
@@ -1516,7 +1516,8 @@ static int lbs_cfg_add_key(struct wiphy *wiphy, struct net_device *netdev,
 
 
 static int lbs_cfg_del_key(struct wiphy *wiphy, struct net_device *netdev,
-			   u8 key_index, bool pairwise, const u8 *mac_addr)
+			   int link_id, u8 key_index, bool pairwise,
+			   const u8 *mac_addr)
 {
 
 	lbs_deb_assoc("del_key: key_idx %d, mac_addr %pM\n",
diff --git a/drivers/net/wireless/marvell/libertas/ethtool.c b/drivers/net/wireless/marvell/libertas/ethtool.c
index d8e4f29b690d..9f53308a9935 100644
--- a/drivers/net/wireless/marvell/libertas/ethtool.c
+++ b/drivers/net/wireless/marvell/libertas/ethtool.c
@@ -20,8 +20,8 @@ static void lbs_ethtool_get_drvinfo(struct net_device *dev,
 		priv->fwrelease >> 16 & 0xff,
 		priv->fwrelease >>  8 & 0xff,
 		priv->fwrelease       & 0xff);
-	strlcpy(info->driver, "libertas", sizeof(info->driver));
-	strlcpy(info->version, lbs_driver_version, sizeof(info->version));
+	strscpy(info->driver, "libertas", sizeof(info->driver));
+	strscpy(info->version, lbs_driver_version, sizeof(info->version));
 }
 
 /*
diff --git a/drivers/net/wireless/marvell/libertas/main.c b/drivers/net/wireless/marvell/libertas/main.c
index 5c9f295536ea..8f5220cee112 100644
--- a/drivers/net/wireless/marvell/libertas/main.c
+++ b/drivers/net/wireless/marvell/libertas/main.c
@@ -39,8 +39,7 @@ unsigned int lbs_debug;
 EXPORT_SYMBOL_GPL(lbs_debug);
 module_param_named(libertas_debug, lbs_debug, int, 0644);
 
-unsigned int lbs_disablemesh;
-EXPORT_SYMBOL_GPL(lbs_disablemesh);
+static unsigned int lbs_disablemesh;
 module_param_named(libertas_disablemesh, lbs_disablemesh, int, 0644);
 
 
diff --git a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c
index bd835288ce57..a04b66284af4 100644
--- a/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c
+++ b/drivers/net/wireless/marvell/mwifiex/11n_rxreorder.c
@@ -335,7 +335,7 @@ mwifiex_11n_create_rx_reorder_tbl(struct mwifiex_private *priv, u8 *ta,
 	struct mwifiex_sta_node *node;
 
 	/*
-	 * If we get a TID, ta pair which is already present dispatch all the
+	 * If we get a TID, ta pair which is already present dispatch all
 	 * the packets and move the window size until the ssn
 	 */
 	tbl = mwifiex_11n_get_rx_reorder_tbl(priv, tid, ta);
diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
index 134114ac1ac0..535995e8279f 100644
--- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
@@ -142,7 +142,8 @@ static void *mwifiex_cfg80211_get_adapter(struct wiphy *wiphy)
  */
 static int
 mwifiex_cfg80211_del_key(struct wiphy *wiphy, struct net_device *netdev,
-			 u8 key_index, bool pairwise, const u8 *mac_addr)
+			 int link_id, u8 key_index, bool pairwise,
+			 const u8 *mac_addr)
 {
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev);
 	static const u8 bc_mac[] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
@@ -431,7 +432,7 @@ mwifiex_cfg80211_set_power_mgmt(struct wiphy *wiphy,
  */
 static int
 mwifiex_cfg80211_set_default_key(struct wiphy *wiphy, struct net_device *netdev,
-				 u8 key_index, bool unicast,
+				 int link_id, u8 key_index, bool unicast,
 				 bool multicast)
 {
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev);
@@ -456,8 +457,8 @@ mwifiex_cfg80211_set_default_key(struct wiphy *wiphy, struct net_device *netdev,
  */
 static int
 mwifiex_cfg80211_add_key(struct wiphy *wiphy, struct net_device *netdev,
-			 u8 key_index, bool pairwise, const u8 *mac_addr,
-			 struct key_params *params)
+			 int link_id, u8 key_index, bool pairwise,
+			 const u8 *mac_addr, struct key_params *params)
 {
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev);
 	struct mwifiex_wep_key *wep_key;
@@ -494,6 +495,7 @@ mwifiex_cfg80211_add_key(struct wiphy *wiphy, struct net_device *netdev,
 static int
 mwifiex_cfg80211_set_default_mgmt_key(struct wiphy *wiphy,
 				      struct net_device *netdev,
+				      int link_id,
 				      u8 key_index)
 {
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(netdev);
diff --git a/drivers/net/wireless/marvell/mwifiex/fw.h b/drivers/net/wireless/marvell/mwifiex/fw.h
index 26a48d8f49be..b4f945a549f7 100644
--- a/drivers/net/wireless/marvell/mwifiex/fw.h
+++ b/drivers/net/wireless/marvell/mwifiex/fw.h
@@ -2104,7 +2104,7 @@ struct mwifiex_fw_mef_entry {
 struct host_cmd_ds_mef_cfg {
 	__le32 criteria;
 	__le16 num_entries;
-	struct mwifiex_fw_mef_entry mef_entry[];
+	u8 mef_entry_data[];
 } __packed;
 
 #define CONNECTION_TYPE_INFRA   0
@@ -2254,7 +2254,7 @@ struct coalesce_receive_filt_rule {
 struct host_cmd_ds_coalesce_cfg {
 	__le16 action;
 	__le16 num_of_rules;
-	struct coalesce_receive_filt_rule rule[];
+	u8 rule_data[];
 } __packed;
 
 struct host_cmd_ds_multi_chan_policy {
diff --git a/drivers/net/wireless/marvell/mwifiex/init.c b/drivers/net/wireless/marvell/mwifiex/init.c
index fc77489cc511..7dddb4b5dea1 100644
--- a/drivers/net/wireless/marvell/mwifiex/init.c
+++ b/drivers/net/wireless/marvell/mwifiex/init.c
@@ -51,9 +51,10 @@ static void wakeup_timer_fn(struct timer_list *t)
 		adapter->if_ops.card_reset(adapter);
 }
 
-static void fw_dump_timer_fn(struct timer_list *t)
+static void fw_dump_work(struct work_struct *work)
 {
-	struct mwifiex_adapter *adapter = from_timer(adapter, t, devdump_timer);
+	struct mwifiex_adapter *adapter =
+		container_of(work, struct mwifiex_adapter, devdump_work.work);
 
 	mwifiex_upload_device_dump(adapter);
 }
@@ -309,7 +310,7 @@ static void mwifiex_init_adapter(struct mwifiex_adapter *adapter)
 	adapter->active_scan_triggered = false;
 	timer_setup(&adapter->wakeup_timer, wakeup_timer_fn, 0);
 	adapter->devdump_len = 0;
-	timer_setup(&adapter->devdump_timer, fw_dump_timer_fn, 0);
+	INIT_DELAYED_WORK(&adapter->devdump_work, fw_dump_work);
 }
 
 /*
@@ -388,7 +389,7 @@ static void
 mwifiex_adapter_cleanup(struct mwifiex_adapter *adapter)
 {
 	del_timer(&adapter->wakeup_timer);
-	del_timer_sync(&adapter->devdump_timer);
+	cancel_delayed_work_sync(&adapter->devdump_work);
 	mwifiex_cancel_all_pending_cmd(adapter);
 	wake_up_interruptible(&adapter->cmd_wait_q.wait);
 	wake_up_interruptible(&adapter->hs_activate_wait_q);
diff --git a/drivers/net/wireless/marvell/mwifiex/main.h b/drivers/net/wireless/marvell/mwifiex/main.h
index 87729d251fed..63f861e6b28a 100644
--- a/drivers/net/wireless/marvell/mwifiex/main.h
+++ b/drivers/net/wireless/marvell/mwifiex/main.h
@@ -37,6 +37,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/slab.h>
 #include <linux/of_irq.h>
+#include <linux/workqueue.h>
 
 #include "decl.h"
 #include "ioctl.h"
@@ -1043,7 +1044,7 @@ struct mwifiex_adapter {
 	/* Device dump data/length */
 	void *devdump_data;
 	int devdump_len;
-	struct timer_list devdump_timer;
+	struct delayed_work devdump_work;
 
 	bool ignore_btcoex_events;
 };
diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c
index f7f9277602a5..5dcf61761a16 100644
--- a/drivers/net/wireless/marvell/mwifiex/pcie.c
+++ b/drivers/net/wireless/marvell/mwifiex/pcie.c
@@ -644,7 +644,7 @@ static int mwifiex_pm_wakeup_card(struct mwifiex_adapter *adapter)
 {
 	struct pcie_service_card *card = adapter->card;
 	const struct mwifiex_pcie_card_reg *reg = card->pcie.reg;
-	int retval;
+	int retval __maybe_unused;
 
 	mwifiex_dbg(adapter, EVENT,
 		    "event: Wakeup device...\n");
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
index 512b5bb9cf6f..e2800a831c8e 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_cmd.c
@@ -1435,7 +1435,7 @@ mwifiex_cmd_mef_cfg(struct mwifiex_private *priv,
 		mef_entry = (struct mwifiex_fw_mef_entry *)pos;
 		mef_entry->mode = mef->mef_entry[i].mode;
 		mef_entry->action = mef->mef_entry[i].action;
-		pos += sizeof(*mef_cfg->mef_entry);
+		pos += sizeof(*mef_entry);
 
 		if (mwifiex_cmd_append_rpn_expression(priv,
 						      &mef->mef_entry[i], &pos))
@@ -1631,7 +1631,7 @@ mwifiex_cmd_coalesce_cfg(struct mwifiex_private *priv,
 
 	coalesce_cfg->action = cpu_to_le16(cmd_action);
 	coalesce_cfg->num_of_rules = cpu_to_le16(cfg->num_of_rules);
-	rule = coalesce_cfg->rule;
+	rule = (void *)coalesce_cfg->rule_data;
 
 	for (cnt = 0; cnt < cfg->num_of_rules; cnt++) {
 		rule->header.type = cpu_to_le16(TLV_TYPE_COALESCE_RULE);
diff --git a/drivers/net/wireless/marvell/mwifiex/sta_event.c b/drivers/net/wireless/marvell/mwifiex/sta_event.c
index b95e90a7d124..df9cdd10a494 100644
--- a/drivers/net/wireless/marvell/mwifiex/sta_event.c
+++ b/drivers/net/wireless/marvell/mwifiex/sta_event.c
@@ -611,8 +611,8 @@ mwifiex_fw_dump_info_event(struct mwifiex_private *priv,
 		 * transmission event get lost, in this cornel case,
 		 * user would still get partial of the dump.
 		 */
-		mod_timer(&adapter->devdump_timer,
-			  jiffies + msecs_to_jiffies(MWIFIEX_TIMER_10S));
+		schedule_delayed_work(&adapter->devdump_work,
+				      msecs_to_jiffies(MWIFIEX_TIMER_10S));
 	}
 
 	/* Overflow check */
@@ -623,7 +623,7 @@ mwifiex_fw_dump_info_event(struct mwifiex_private *priv,
 		adapter->event_skb->data, event_skb->len);
 	adapter->devdump_len += event_skb->len;
 
-	if (le16_to_cpu(fw_dump_hdr->type == FW_DUMP_INFO_ENDED)) {
+	if (le16_to_cpu(fw_dump_hdr->type) == FW_DUMP_INFO_ENDED) {
 		mwifiex_dbg(adapter, MSG,
 			    "receive end of transmission flag event!\n");
 		goto upload_dump;
@@ -631,7 +631,7 @@ mwifiex_fw_dump_info_event(struct mwifiex_private *priv,
 	return;
 
 upload_dump:
-	del_timer_sync(&adapter->devdump_timer);
+	cancel_delayed_work_sync(&adapter->devdump_work);
 	mwifiex_upload_device_dump(adapter);
 }
 
diff --git a/drivers/net/wireless/marvell/mwifiex/usb.c b/drivers/net/wireless/marvell/mwifiex/usb.c
index c2f2ce2a3f95..d3ab9572e711 100644
--- a/drivers/net/wireless/marvell/mwifiex/usb.c
+++ b/drivers/net/wireless/marvell/mwifiex/usb.c
@@ -911,14 +911,14 @@ static int mwifiex_usb_prepare_tx_aggr_skb(struct mwifiex_adapter *adapter,
 		memcpy(payload, skb_tmp->data, skb_tmp->len);
 		if (skb_queue_empty(&port->tx_aggr.aggr_list)) {
 			/* do not padding for last packet*/
-			*(u16 *)payload = cpu_to_le16(skb_tmp->len);
-			*(u16 *)&payload[2] =
+			*(__le16 *)payload = cpu_to_le16(skb_tmp->len);
+			*(__le16 *)&payload[2] =
 				cpu_to_le16(MWIFIEX_TYPE_AGGR_DATA_V2 | 0x80);
 			skb_trim(skb_aggr, skb_aggr->len - pad);
 		} else {
 			/* add aggregation interface header */
-			*(u16 *)payload = cpu_to_le16(skb_tmp->len + pad);
-			*(u16 *)&payload[2] =
+			*(__le16 *)payload = cpu_to_le16(skb_tmp->len + pad);
+			*(__le16 *)&payload[2] =
 				cpu_to_le16(MWIFIEX_TYPE_AGGR_DATA_V2);
 		}
 
@@ -1097,9 +1097,9 @@ send_aggr_buf:
 		}
 
 		payload = skb->data;
-		*(u16 *)&payload[2] =
+		*(__le16 *)&payload[2] =
 			cpu_to_le16(MWIFIEX_TYPE_AGGR_DATA_V2 | 0x80);
-		*(u16 *)payload = cpu_to_le16(skb->len);
+		*(__le16 *)payload = cpu_to_le16(skb->len);
 		skb_send = skb;
 		context = &port->tx_data_list[port->tx_data_ix++];
 		return mwifiex_usb_construct_send_urb(adapter, port, ep,
diff --git a/drivers/net/wireless/mediatek/mt76/dma.c b/drivers/net/wireless/mediatek/mt76/dma.c
index 40cb91097b2e..4901aa02b4fb 100644
--- a/drivers/net/wireless/mediatek/mt76/dma.c
+++ b/drivers/net/wireless/mediatek/mt76/dma.c
@@ -758,7 +758,7 @@ mt76_dma_init(struct mt76_dev *dev,
 	dev->napi_dev.threaded = 1;
 
 	mt76_for_each_q_rx(dev, i) {
-		netif_napi_add(&dev->napi_dev, &dev->napi[i], poll, 64);
+		netif_napi_add(&dev->napi_dev, &dev->napi[i], poll);
 		mt76_dma_rx_fill(dev, &dev->q_rx[i]);
 		napi_enable(&dev->napi[i]);
 	}
diff --git a/drivers/net/wireless/mediatek/mt76/mt76.h b/drivers/net/wireless/mediatek/mt76/mt76.h
index 4da77d47b0a6..87db9498dea4 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76.h
@@ -252,6 +252,30 @@ struct mt76_queue_ops {
 	void (*reset_q)(struct mt76_dev *dev, struct mt76_queue *q);
 };
 
+enum mt76_phy_type {
+	MT_PHY_TYPE_CCK,
+	MT_PHY_TYPE_OFDM,
+	MT_PHY_TYPE_HT,
+	MT_PHY_TYPE_HT_GF,
+	MT_PHY_TYPE_VHT,
+	MT_PHY_TYPE_HE_SU = 8,
+	MT_PHY_TYPE_HE_EXT_SU,
+	MT_PHY_TYPE_HE_TB,
+	MT_PHY_TYPE_HE_MU,
+	__MT_PHY_TYPE_HE_MAX,
+};
+
+struct mt76_sta_stats {
+	u64 tx_mode[__MT_PHY_TYPE_HE_MAX];
+	u64 tx_bw[4];		/* 20, 40, 80, 160 */
+	u64 tx_nss[4];		/* 1, 2, 3, 4 */
+	u64 tx_mcs[16];		/* mcs idx */
+	u64 tx_bytes;
+	u32 tx_packets;
+	u32 tx_retries;
+	u32 tx_failed;
+};
+
 enum mt76_wcid_flags {
 	MT_WCID_FLAG_CHECK_PS,
 	MT_WCID_FLAG_PS,
@@ -299,6 +323,8 @@ struct mt76_wcid {
 
 	struct list_head list;
 	struct idr pktid;
+
+	struct mt76_sta_stats stats;
 };
 
 struct mt76_txq {
@@ -342,7 +368,8 @@ struct mt76_rx_tid {
 #define MT_PACKET_ID_MASK		GENMASK(6, 0)
 #define MT_PACKET_ID_NO_ACK		0
 #define MT_PACKET_ID_NO_SKB		1
-#define MT_PACKET_ID_FIRST		2
+#define MT_PACKET_ID_WED		2
+#define MT_PACKET_ID_FIRST		3
 #define MT_PACKET_ID_HAS_RATE		BIT(7)
 /* This is timer for when to give up when waiting for TXS callback,
  * with starting time being the time at which the DMA_DONE callback
@@ -527,7 +554,6 @@ struct mt76_usb {
 		struct mt76_reg_pair *rp;
 		int rp_len;
 		u32 base;
-		bool burst;
 	} mcu;
 };
 
@@ -815,26 +841,6 @@ struct mt76_power_limits {
 	s8 ru[7][12];
 };
 
-enum mt76_phy_type {
-	MT_PHY_TYPE_CCK,
-	MT_PHY_TYPE_OFDM,
-	MT_PHY_TYPE_HT,
-	MT_PHY_TYPE_HT_GF,
-	MT_PHY_TYPE_VHT,
-	MT_PHY_TYPE_HE_SU = 8,
-	MT_PHY_TYPE_HE_EXT_SU,
-	MT_PHY_TYPE_HE_TB,
-	MT_PHY_TYPE_HE_MU,
-	__MT_PHY_TYPE_HE_MAX,
-};
-
-struct mt76_sta_stats {
-	u64 tx_mode[__MT_PHY_TYPE_HE_MAX];
-	u64 tx_bw[4];		/* 20, 40, 80, 160 */
-	u64 tx_nss[4];		/* 1, 2, 3, 4 */
-	u64 tx_mcs[16];		/* mcs idx */
-};
-
 struct mt76_ethtool_worker_info {
 	u64 *data;
 	int idx;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7603/main.c b/drivers/net/wireless/mediatek/mt76/mt7603/main.c
index 051715ed90dd..ca50feb0b3a9 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7603/main.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7603/main.c
@@ -658,7 +658,7 @@ mt7603_sta_rate_tbl_update(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	mt7603_wtbl_set_rates(dev, msta, NULL, msta->rates);
 	msta->rate_probe = false;
 	mt7603_wtbl_set_smps(dev, msta,
-			     sta->smps_mode == IEEE80211_SMPS_DYNAMIC);
+			     sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC);
 	spin_unlock_bh(&dev->mt76.lock);
 }
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/main.c b/drivers/net/wireless/mediatek/mt76/mt7615/main.c
index 9bf8545c8c17..8d4733f87cda 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/main.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/main.c
@@ -1195,12 +1195,16 @@ static void mt7615_sta_set_decap_offload(struct ieee80211_hw *hw,
 	struct mt7615_dev *dev = mt7615_hw_dev(hw);
 	struct mt7615_sta *msta = (struct mt7615_sta *)sta->drv_priv;
 
+	mt7615_mutex_acquire(dev);
+
 	if (enabled)
 		set_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags);
 	else
 		clear_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags);
 
 	mt7615_mcu_set_sta_decap_offload(dev, vif, sta);
+
+	mt7615_mutex_release(dev);
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c b/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c
index 49ab3a1f3b9b..304212f5f8da 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/sdio.c
@@ -83,6 +83,7 @@ static int mt7663s_probe(struct sdio_func *func,
 		.tx_complete_skb = mt7663_usb_sdio_tx_complete_skb,
 		.tx_status_data = mt7663_usb_sdio_tx_status_data,
 		.rx_skb = mt7615_queue_rx_skb,
+		.rx_check = mt7615_rx_check,
 		.sta_ps = mt7615_sta_ps,
 		.sta_add = mt7615_mac_sta_add,
 		.sta_remove = mt7615_mac_sta_remove,
@@ -180,7 +181,6 @@ static void mt7663s_remove(struct sdio_func *func)
 	mt76_free_device(&dev->mt76);
 }
 
-#ifdef CONFIG_PM
 static int mt7663s_suspend(struct device *dev)
 {
 	struct sdio_func *func = dev_to_sdio_func(dev);
@@ -235,28 +235,20 @@ static int mt7663s_resume(struct device *dev)
 	return err;
 }
 
-static const struct dev_pm_ops mt7663s_pm_ops = {
-	.suspend = mt7663s_suspend,
-	.resume = mt7663s_resume,
-};
-#endif
-
 MODULE_DEVICE_TABLE(sdio, mt7663s_table);
 MODULE_FIRMWARE(MT7663_OFFLOAD_FIRMWARE_N9);
 MODULE_FIRMWARE(MT7663_OFFLOAD_ROM_PATCH);
 MODULE_FIRMWARE(MT7663_FIRMWARE_N9);
 MODULE_FIRMWARE(MT7663_ROM_PATCH);
 
+static DEFINE_SIMPLE_DEV_PM_OPS(mt7663s_pm_ops, mt7663s_suspend, mt7663s_resume);
+
 static struct sdio_driver mt7663s_driver = {
 	.name		= KBUILD_MODNAME,
 	.probe		= mt7663s_probe,
 	.remove		= mt7663s_remove,
 	.id_table	= mt7663s_table,
-#ifdef CONFIG_PM
-	.drv = {
-		.pm = &mt7663s_pm_ops,
-	}
-#endif
+	.drv.pm		= pm_sleep_ptr(&mt7663s_pm_ops),
 };
 module_sdio_driver(mt7663s_driver);
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7615/usb.c b/drivers/net/wireless/mediatek/mt76/mt7615/usb.c
index 967641aebf5f..f2d651d7adff 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7615/usb.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7615/usb.c
@@ -119,6 +119,7 @@ static int mt7663u_probe(struct usb_interface *usb_intf,
 		.tx_complete_skb = mt7663_usb_sdio_tx_complete_skb,
 		.tx_status_data = mt7663_usb_sdio_tx_status_data,
 		.rx_skb = mt7615_queue_rx_skb,
+		.rx_check = mt7615_rx_check,
 		.sta_ps = mt7615_sta_ps,
 		.sta_add = mt7615_mac_sta_add,
 		.sta_remove = mt7615_mac_sta_remove,
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac.h
index 75afcb469d3c..635192c878cb 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac.h
@@ -63,6 +63,12 @@ enum {
 	REPEATER_BSSID_MAX = 0x3f,
 };
 
+struct mt76_connac_reg_map {
+	u32 phys;
+	u32 maps;
+	u32 size;
+};
+
 struct mt76_connac_pm {
 	bool enable:1;
 	bool enable_user:1;
@@ -348,9 +354,10 @@ void mt76_connac2_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi,
 				 struct sk_buff *skb, struct mt76_wcid *wcid,
 				 struct ieee80211_key_conf *key, int pid,
 				 enum mt76_txq_id qid, u32 changed);
+bool mt76_connac2_mac_fill_txs(struct mt76_dev *dev, struct mt76_wcid *wcid,
+			       __le32 *txs_data);
 bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid,
-				  int pid, __le32 *txs_data,
-				  struct mt76_sta_stats *stats);
+				  int pid, __le32 *txs_data);
 void mt76_connac2_mac_decode_he_radiotap(struct mt76_dev *dev,
 					 struct sk_buff *skb,
 					 __le32 *rxv, u32 mode);
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h b/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h
index 67ce216fb564..f33171bcd343 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac2_mac.h
@@ -158,6 +158,14 @@ enum {
 
 #define MT_TXS4_TIMESTAMP		GENMASK(31, 0)
 
+/* PPDU based TXS */
+#define MT_TXS5_MPDU_TX_BYTE		GENMASK(22, 0)
+#define MT_TXS5_MPDU_TX_CNT		GENMASK(31, 23)
+
+#define MT_TXS6_MPDU_FAIL_CNT		GENMASK(31, 23)
+
+#define MT_TXS7_MPDU_RETRY_CNT		GENMASK(31, 23)
+
 /* RXD DW1 */
 #define MT_RXD1_NORMAL_WLAN_IDX		GENMASK(9, 0)
 #define MT_RXD1_NORMAL_GROUP_1		BIT(11)
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c
index 18dea8e1fb20..34ac3d81a510 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mac.c
@@ -490,6 +490,10 @@ void mt76_connac2_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi,
 		p_fmt = mt76_is_mmio(dev) ? MT_TX_TYPE_CT : MT_TX_TYPE_SF;
 		q_idx = wmm_idx * MT76_CONNAC_MAX_WMM_SETS +
 			mt76_connac_lmac_mapping(skb_get_queue_mapping(skb));
+
+		/* counting non-offloading skbs */
+		wcid->stats.tx_bytes += skb->len;
+		wcid->stats.tx_packets++;
 	}
 
 	val = FIELD_PREP(MT_TXD0_TX_BYTES, skb->len + sz_txd) |
@@ -550,35 +554,29 @@ void mt76_connac2_mac_write_txwi(struct mt76_dev *dev, __le32 *txwi,
 }
 EXPORT_SYMBOL_GPL(mt76_connac2_mac_write_txwi);
 
-bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid,
-				  int pid, __le32 *txs_data,
-				  struct mt76_sta_stats *stats)
+bool mt76_connac2_mac_fill_txs(struct mt76_dev *dev, struct mt76_wcid *wcid,
+			       __le32 *txs_data)
 {
+	struct mt76_sta_stats *stats = &wcid->stats;
 	struct ieee80211_supported_band *sband;
 	struct mt76_phy *mphy;
-	struct ieee80211_tx_info *info;
-	struct sk_buff_head list;
 	struct rate_info rate = {};
-	struct sk_buff *skb;
 	bool cck = false;
 	u32 txrate, txs, mode;
 
-	mt76_tx_status_lock(dev, &list);
-	skb = mt76_tx_status_skb_get(dev, wcid, pid, &list);
-	if (!skb)
-		goto out;
-
 	txs = le32_to_cpu(txs_data[0]);
 
-	info = IEEE80211_SKB_CB(skb);
-	if (!(txs & MT_TXS0_ACK_ERROR_MASK))
-		info->flags |= IEEE80211_TX_STAT_ACK;
-
-	info->status.ampdu_len = 1;
-	info->status.ampdu_ack_len = !!(info->flags &
-					IEEE80211_TX_STAT_ACK);
-
-	info->status.rates[0].idx = -1;
+	/* PPDU based reporting */
+	if (FIELD_GET(MT_TXS0_TXS_FORMAT, txs) > 1) {
+		stats->tx_bytes +=
+			le32_get_bits(txs_data[5], MT_TXS5_MPDU_TX_BYTE);
+		stats->tx_packets +=
+			le32_get_bits(txs_data[5], MT_TXS5_MPDU_TX_CNT);
+		stats->tx_failed +=
+			le32_get_bits(txs_data[6], MT_TXS6_MPDU_FAIL_CNT);
+		stats->tx_retries +=
+			le32_get_bits(txs_data[7], MT_TXS7_MPDU_RETRY_CNT);
+	}
 
 	txrate = FIELD_GET(MT_TXS0_TX_RATE, txs);
 
@@ -613,7 +611,7 @@ bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid,
 	case MT_PHY_TYPE_HT:
 	case MT_PHY_TYPE_HT_GF:
 		if (rate.mcs > 31)
-			goto out;
+			return false;
 
 		rate.flags = RATE_INFO_FLAGS_MCS;
 		if (wcid->rate.flags & RATE_INFO_FLAGS_SHORT_GI)
@@ -621,7 +619,7 @@ bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid,
 		break;
 	case MT_PHY_TYPE_VHT:
 		if (rate.mcs > 9)
-			goto out;
+			return false;
 
 		rate.flags = RATE_INFO_FLAGS_VHT_MCS;
 		break;
@@ -630,14 +628,14 @@ bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid,
 	case MT_PHY_TYPE_HE_TB:
 	case MT_PHY_TYPE_HE_MU:
 		if (rate.mcs > 11)
-			goto out;
+			return false;
 
 		rate.he_gi = wcid->rate.he_gi;
 		rate.he_dcm = FIELD_GET(MT_TX_RATE_DCM, txrate);
 		rate.flags = RATE_INFO_FLAGS_HE_MCS;
 		break;
 	default:
-		goto out;
+		return false;
 	}
 
 	stats->tx_mode[mode]++;
@@ -662,10 +660,34 @@ bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid,
 	}
 	wcid->rate = rate;
 
-out:
-	if (skb)
-		mt76_tx_status_skb_done(dev, skb, &list);
+	return true;
+}
+EXPORT_SYMBOL_GPL(mt76_connac2_mac_fill_txs);
+
+bool mt76_connac2_mac_add_txs_skb(struct mt76_dev *dev, struct mt76_wcid *wcid,
+				  int pid, __le32 *txs_data)
+{
+	struct sk_buff_head list;
+	struct sk_buff *skb;
+
+	mt76_tx_status_lock(dev, &list);
+	skb = mt76_tx_status_skb_get(dev, wcid, pid, &list);
+	if (skb) {
+		struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+		bool noacked = !(info->flags & IEEE80211_TX_STAT_ACK);
+
+		if (!(le32_to_cpu(txs_data[0]) & MT_TXS0_ACK_ERROR_MASK))
+			info->flags |= IEEE80211_TX_STAT_ACK;
+
+		info->status.ampdu_len = 1;
+		info->status.ampdu_ack_len = !noacked;
+		info->status.rates[0].idx = -1;
 
+		wcid->stats.tx_failed += noacked;
+
+		mt76_connac2_mac_fill_txs(dev, wcid, txs_data);
+		mt76_tx_status_skb_done(dev, skb, &list);
+	}
 	mt76_tx_status_unlock(dev, &list);
 
 	return !!skb;
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
index 9b17bd97ec09..011fc9729b38 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.c
@@ -260,8 +260,10 @@ mt76_connac_mcu_add_nested_tlv(struct sk_buff *skb, int tag, int len,
 	ntlv = le16_to_cpu(ntlv_hdr->tlv_num);
 	ntlv_hdr->tlv_num = cpu_to_le16(ntlv + 1);
 
-	if (sta_hdr)
-		le16_add_cpu(&sta_hdr->len, len);
+	if (sta_hdr) {
+		len += le16_to_cpu(sta_hdr->len);
+		sta_hdr->len = cpu_to_le16(len);
+	}
 
 	return ptlv;
 }
@@ -594,14 +596,14 @@ mt76_connac_mcu_sta_amsdu_tlv(struct sk_buff *skb, struct ieee80211_sta *sta,
 	    vif->type != NL80211_IFTYPE_STATION)
 		return;
 
-	if (!sta->max_amsdu_len)
+	if (!sta->deflink.agg.max_amsdu_len)
 		return;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HW_AMSDU, sizeof(*amsdu));
 	amsdu = (struct sta_rec_amsdu *)tlv;
 	amsdu->max_amsdu_num = 8;
 	amsdu->amsdu_en = true;
-	amsdu->max_mpdu_size = sta->max_amsdu_len >=
+	amsdu->max_mpdu_size = sta->deflink.agg.max_amsdu_len >=
 			       IEEE80211_MAX_MPDU_LEN_VHT_7991;
 
 	wcid->amsdu = true;
@@ -896,7 +898,7 @@ void mt76_connac_mcu_wtbl_smps_tlv(struct sk_buff *skb,
 	tlv = mt76_connac_mcu_add_nested_tlv(skb, WTBL_SMPS, sizeof(*smps),
 					     wtbl_tlv, sta_wtbl);
 	smps = (struct wtbl_smps *)tlv;
-	smps->smps = (sta->smps_mode == IEEE80211_SMPS_DYNAMIC);
+	smps->smps = (sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC);
 }
 EXPORT_SYMBOL_GPL(mt76_connac_mcu_wtbl_smps_tlv);
 
@@ -2648,7 +2650,7 @@ int mt76_connac_mcu_add_key(struct mt76_dev *dev, struct ieee80211_vif *vif,
 }
 EXPORT_SYMBOL_GPL(mt76_connac_mcu_add_key);
 
-/* SIFS 20us + 512 byte beacon tranmitted by 1Mbps (3906us) */
+/* SIFS 20us + 512 byte beacon transmitted by 1Mbps (3906us) */
 #define BCN_TX_ESTIMATE_TIME (4096 + 20)
 void mt76_connac_mcu_bss_ext_tlv(struct sk_buff *skb, struct mt76_vif *mvif)
 {
@@ -2886,6 +2888,10 @@ int mt76_connac2_load_ram(struct mt76_dev *dev, const char *fw_wm,
 		goto out;
 	}
 
+	snprintf(dev->hw->wiphy->fw_version,
+		 sizeof(dev->hw->wiphy->fw_version),
+		 "%.10s-%.15s", hdr->fw_ver, hdr->build_date);
+
 	release_firmware(fw);
 
 	if (!fw_wa)
diff --git a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h
index f1d7c05bd794..718f427d8f6b 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h
+++ b/drivers/net/wireless/mediatek/mt76/mt76_connac_mcu.h
@@ -10,6 +10,7 @@
 #define FW_FEATURE_SET_KEY_IDX		GENMASK(2, 1)
 #define FW_FEATURE_ENCRY_MODE		BIT(4)
 #define FW_FEATURE_OVERRIDE_ADDR	BIT(5)
+#define FW_FEATURE_NON_DL		BIT(6)
 
 #define DL_MODE_ENCRYPT			BIT(0)
 #define DL_MODE_KEY_IDX			GENMASK(2, 1)
@@ -33,6 +34,12 @@
 #define PATCH_SEC_ENC_SCRAMBLE_INFO_MASK	GENMASK(15, 0)
 #define PATCH_SEC_ENC_AES_KEY_MASK		GENMASK(7, 0)
 
+enum {
+	FW_TYPE_DEFAULT = 0,
+	FW_TYPE_CLC = 2,
+	FW_TYPE_MAX_NUM = 255
+};
+
 #define MCU_PQ_ID(p, q)		(((p) << 15) | ((q) << 10))
 #define MCU_PKT_ID		0xa0
 
@@ -174,7 +181,8 @@ struct mt76_connac2_fw_region {
 	__le32 addr;
 	__le32 len;
 	u8 feature_set;
-	u8 rsv1[15];
+	u8 type;
+	u8 rsv1[14];
 } __packed;
 
 struct tlv {
@@ -1172,6 +1180,7 @@ enum {
 	MCU_CE_CMD_SET_ROC = 0x1c,
 	MCU_CE_CMD_SET_EDCA_PARMS = 0x1d,
 	MCU_CE_CMD_SET_P2P_OPPPS = 0x33,
+	MCU_CE_CMD_SET_CLC = 0x5c,
 	MCU_CE_CMD_SET_RATE_TX_POWER = 0x5d,
 	MCU_CE_CMD_SCHED_SCAN_ENABLE = 0x61,
 	MCU_CE_CMD_SCHED_SCAN_REQ = 0x62,
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c
index de30cf5e2d2f..93d96739f802 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76x02_mac.c
@@ -404,7 +404,7 @@ void mt76x02_mac_write_txwi(struct mt76x02_dev *dev, struct mt76x02_txwi *txwi,
 		txwi->rate |= cpu_to_le16(MT_RXWI_RATE_LDPC);
 	if ((info->flags & IEEE80211_TX_CTL_STBC) && nss == 1)
 		txwi->rate |= cpu_to_le16(MT_RXWI_RATE_STBC);
-	if (nss > 1 && sta && sta->smps_mode == IEEE80211_SMPS_DYNAMIC)
+	if (nss > 1 && sta && sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC)
 		txwi_flags |= MT_TXWI_FLAGS_MMPS;
 	if (!(info->flags & IEEE80211_TX_CTL_NO_ACK))
 		txwi->ack_ctl |= MT_TXWI_ACK_CTL_REQ;
diff --git a/drivers/net/wireless/mediatek/mt76/mt76x02_usb_mcu.c b/drivers/net/wireless/mediatek/mt76/mt76x02_usb_mcu.c
index c6c16fe8ee85..02da543dfc5c 100644
--- a/drivers/net/wireless/mediatek/mt76/mt76x02_usb_mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt76x02_usb_mcu.c
@@ -21,29 +21,16 @@ static void
 mt76x02u_multiple_mcu_reads(struct mt76_dev *dev, u8 *data, int len)
 {
 	struct mt76_usb *usb = &dev->usb;
-	u32 reg, val;
 	int i;
 
-	if (usb->mcu.burst) {
-		WARN_ON_ONCE(len / 4 != usb->mcu.rp_len);
-
-		reg = usb->mcu.rp[0].reg - usb->mcu.base;
-		for (i = 0; i < usb->mcu.rp_len; i++) {
-			val = get_unaligned_le32(data + 4 * i);
-			usb->mcu.rp[i].reg = reg++;
-			usb->mcu.rp[i].value = val;
-		}
-	} else {
-		WARN_ON_ONCE(len / 8 != usb->mcu.rp_len);
-
-		for (i = 0; i < usb->mcu.rp_len; i++) {
-			reg = get_unaligned_le32(data + 8 * i) -
-			      usb->mcu.base;
-			val = get_unaligned_le32(data + 8 * i + 4);
-
-			WARN_ON_ONCE(usb->mcu.rp[i].reg != reg);
-			usb->mcu.rp[i].value = val;
-		}
+	WARN_ON_ONCE(len / 8 != usb->mcu.rp_len);
+
+	for (i = 0; i < usb->mcu.rp_len; i++) {
+		u32 reg = get_unaligned_le32(data + 8 * i) - usb->mcu.base;
+		u32 val = get_unaligned_le32(data + 8 * i + 4);
+
+		WARN_ON_ONCE(usb->mcu.rp[i].reg != reg);
+		usb->mcu.rp[i].value = val;
 	}
 }
 
@@ -207,7 +194,6 @@ mt76x02u_mcu_rd_rp(struct mt76_dev *dev, u32 base,
 	usb->mcu.rp = data;
 	usb->mcu.rp_len = n;
 	usb->mcu.base = base;
-	usb->mcu.burst = false;
 
 	ret = __mt76x02u_mcu_send_msg(dev, skb, CMD_RANDOM_READ, true);
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c b/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c
index fd76db8f5269..6ef3431cad64 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/debugfs.c
@@ -23,9 +23,9 @@ mt7915_implicit_txbf_set(void *data, u64 val)
 {
 	struct mt7915_dev *dev = data;
 
-	if (test_bit(MT76_STATE_RUNNING, &dev->mphy.state))
-		return -EBUSY;
-
+	/* The existing connected stations shall reconnect to apply
+	 * new implicit txbf configuration.
+	 */
 	dev->ibf = !!val;
 
 	return mt7915_mcu_set_txbf(dev, MT_BF_TYPE_UPDATE);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c
index 60ae834d95a6..be97dede2634 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/mac.c
@@ -176,7 +176,7 @@ static void mt7915_mac_sta_poll(struct mt7915_dev *dev)
 		/*
 		 * We don't support reading GI info from txs packets.
 		 * For accurate tx status reporting and AQL improvement,
-		  we need to make sure that flags match so polling GI
+		 * we need to make sure that flags match so polling GI
 		 * from per-sta counters directly.
 		 */
 		rate = &msta->wcid.rate;
@@ -232,7 +232,7 @@ mt7915_mac_fill_rx(struct mt7915_dev *dev, struct sk_buff *skb)
 	bool unicast, insert_ccmp_hdr = false;
 	u8 remove_pad, amsdu_info;
 	u8 mode = 0, qos_ctl = 0;
-	struct mt7915_sta *msta;
+	struct mt7915_sta *msta = NULL;
 	bool hdr_trans;
 	u16 hdr_gap;
 	u16 seq_ctrl = 0;
@@ -1001,7 +1001,7 @@ static void mt7915_mac_add_txs(struct mt7915_dev *dev, void *data)
 	wcidx = le32_get_bits(txs_data[2], MT_TXS2_WCID);
 	pid = le32_get_bits(txs_data[3], MT_TXS3_PID);
 
-	if (pid < MT_PACKET_ID_FIRST)
+	if (pid < MT_PACKET_ID_WED)
 		return;
 
 	if (wcidx >= mt7915_wtbl_size(dev))
@@ -1015,8 +1015,11 @@ static void mt7915_mac_add_txs(struct mt7915_dev *dev, void *data)
 
 	msta = container_of(wcid, struct mt7915_sta, wcid);
 
-	mt76_connac2_mac_add_txs_skb(&dev->mt76, wcid, pid, txs_data,
-				     &msta->stats);
+	if (pid == MT_PACKET_ID_WED)
+		mt76_connac2_mac_fill_txs(&dev->mt76, wcid, txs_data);
+	else
+		mt76_connac2_mac_add_txs_skb(&dev->mt76, wcid, pid, txs_data);
+
 	if (!wcid->sta)
 		goto out;
 
@@ -1047,7 +1050,7 @@ bool mt7915_rx_check(struct mt76_dev *mdev, void *data, int len)
 		return false;
 	case PKT_TYPE_TXS:
 		for (rxd += 2; rxd + 8 <= end; rxd += 8)
-		    mt7915_mac_add_txs(dev, rxd);
+			mt7915_mac_add_txs(dev, rxd);
 		return false;
 	case PKT_TYPE_RX_FW_MONITOR:
 		mt7915_debugfs_rx_fw_monitor(dev, data, len);
@@ -1084,7 +1087,7 @@ void mt7915_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
 		break;
 	case PKT_TYPE_TXS:
 		for (rxd += 2; rxd + 8 <= end; rxd += 8)
-		    mt7915_mac_add_txs(dev, rxd);
+			mt7915_mac_add_txs(dev, rxd);
 		dev_kfree_skb(skb);
 		break;
 	case PKT_TYPE_RX_FW_MONITOR:
@@ -2071,8 +2074,9 @@ void mt7915_mac_add_twt_setup(struct ieee80211_hw *hw,
 	}
 
 	flowid = ffs(~msta->twt.flowid_mask) - 1;
-	le16p_replace_bits(&twt_agrt->req_type, flowid,
-			   IEEE80211_TWT_REQTYPE_FLOWID);
+	twt_agrt->req_type &= ~cpu_to_le16(IEEE80211_TWT_REQTYPE_FLOWID);
+	twt_agrt->req_type |= le16_encode_bits(flowid,
+					       IEEE80211_TWT_REQTYPE_FLOWID);
 
 	table_id = ffs(~dev->twt.table_mask) - 1;
 	exp = FIELD_GET(IEEE80211_TWT_REQTYPE_WAKE_INT_EXP, req_type);
@@ -2122,8 +2126,9 @@ void mt7915_mac_add_twt_setup(struct ieee80211_hw *hw,
 unlock:
 	mutex_unlock(&dev->mt76.mutex);
 out:
-	le16p_replace_bits(&twt_agrt->req_type, setup_cmd,
-			   IEEE80211_TWT_REQTYPE_SETUP_CMD);
+	twt_agrt->req_type &= ~cpu_to_le16(IEEE80211_TWT_REQTYPE_SETUP_CMD);
+	twt_agrt->req_type |=
+		le16_encode_bits(setup_cmd, IEEE80211_TWT_REQTYPE_SETUP_CMD);
 	twt->control = (twt->control & IEEE80211_TWT_CONTROL_WAKE_DUR_UNIT) |
 		       (twt->control & IEEE80211_TWT_CONTROL_RX_DISABLED);
 }
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/main.c b/drivers/net/wireless/mediatek/mt76/mt7915/main.c
index bd3386bf0f8a..89b519cfd14c 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/main.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/main.c
@@ -1010,6 +1010,23 @@ static void mt7915_sta_statistics(struct ieee80211_hw *hw,
 	}
 	sinfo->txrate.flags = txrate->flags;
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
+
+	/* offloading flows bypass networking stack, so driver counts and
+	 * reports sta statistics via NL80211_STA_INFO when WED is active.
+	 */
+	if (mtk_wed_device_active(&phy->dev->mt76.mmio.wed)) {
+		sinfo->tx_bytes = msta->wcid.stats.tx_bytes;
+		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES64);
+
+		sinfo->tx_packets = msta->wcid.stats.tx_packets;
+		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_PACKETS);
+
+		sinfo->tx_failed = msta->wcid.stats.tx_failed;
+		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
+
+		sinfo->tx_retries = msta->wcid.stats.tx_retries;
+		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES);
+	}
 }
 
 static void mt7915_sta_rc_work(void *data, struct ieee80211_sta *sta)
@@ -1224,7 +1241,7 @@ static void mt7915_ethtool_worker(void *wi_data, struct ieee80211_sta *sta)
 	if (msta->vif->mt76.idx != wi->idx)
 		return;
 
-	mt76_ethtool_worker(wi, &msta->stats);
+	mt76_ethtool_worker(wi, &msta->wcid.stats);
 }
 
 static
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
index f83067961945..8d297e4aa7d4 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/mcu.c
@@ -925,7 +925,7 @@ mt7915_mcu_sta_amsdu_tlv(struct mt7915_dev *dev, struct sk_buff *skb,
 	    vif->type != NL80211_IFTYPE_AP)
 		return;
 
-	if (!sta->max_amsdu_len)
+	if (!sta->deflink.agg.max_amsdu_len)
 	    return;
 
 	tlv = mt76_connac_mcu_add_tlv(skb, STA_REC_HW_AMSDU, sizeof(*amsdu));
@@ -934,7 +934,7 @@ mt7915_mcu_sta_amsdu_tlv(struct mt7915_dev *dev, struct sk_buff *skb,
 	amsdu->amsdu_en = true;
 	msta->wcid.amsdu = true;
 
-	switch (sta->max_amsdu_len) {
+	switch (sta->deflink.agg.max_amsdu_len) {
 	case IEEE80211_MAX_MPDU_LEN_VHT_11454:
 		if (!is_mt7915(&dev->mt76)) {
 			amsdu->max_mpdu_size =
@@ -1304,7 +1304,7 @@ int mt7915_mcu_set_fixed_rate_ctrl(struct mt7915_dev *dev,
 			ra->phy = *phy;
 		break;
 	case RATE_PARAM_MMPS_UPDATE:
-		ra->mmps_mode = mt7915_mcu_get_mmps_mode(sta->smps_mode);
+		ra->mmps_mode = mt7915_mcu_get_mmps_mode(sta->deflink.smps_mode);
 		break;
 	default:
 		break;
@@ -1360,7 +1360,7 @@ mt7915_mcu_add_rate_ctrl_fixed(struct mt7915_dev *dev,
 	struct sta_phy phy = {};
 	int ret, nrates = 0;
 
-#define __sta_phy_bitrate_mask_check(_mcs, _gi, _he)				\
+#define __sta_phy_bitrate_mask_check(_mcs, _gi, _ht, _he)			\
 	do {									\
 		u8 i, gi = mask->control[band]._gi;				\
 		gi = (_he) ? gi : gi == NL80211_TXRATE_FORCE_SGI;		\
@@ -1373,15 +1373,17 @@ mt7915_mcu_add_rate_ctrl_fixed(struct mt7915_dev *dev,
 				continue;					\
 			nrates += hweight16(mask->control[band]._mcs[i]);	\
 			phy.mcs = ffs(mask->control[band]._mcs[i]) - 1;		\
+			if (_ht)						\
+				phy.mcs += 8 * i;				\
 		}								\
 	} while (0)
 
 	if (sta->deflink.he_cap.has_he) {
-		__sta_phy_bitrate_mask_check(he_mcs, he_gi, 1);
+		__sta_phy_bitrate_mask_check(he_mcs, he_gi, 0, 1);
 	} else if (sta->deflink.vht_cap.vht_supported) {
-		__sta_phy_bitrate_mask_check(vht_mcs, gi, 0);
+		__sta_phy_bitrate_mask_check(vht_mcs, gi, 0, 0);
 	} else if (sta->deflink.ht_cap.ht_supported) {
-		__sta_phy_bitrate_mask_check(ht_mcs, gi, 0);
+		__sta_phy_bitrate_mask_check(ht_mcs, gi, 1, 0);
 	} else {
 		nrates = hweight32(mask->control[band].legacy);
 		phy.mcs = ffs(mask->control[band].legacy) - 1;
@@ -1459,7 +1461,7 @@ mt7915_mcu_sta_rate_ctrl_tlv(struct sk_buff *skb, struct mt7915_dev *dev,
 	ra->channel = chandef->chan->hw_value;
 	ra->bw = sta->deflink.bandwidth;
 	ra->phy.bw = sta->deflink.bandwidth;
-	ra->mmps_mode = mt7915_mcu_get_mmps_mode(sta->smps_mode);
+	ra->mmps_mode = mt7915_mcu_get_mmps_mode(sta->deflink.smps_mode);
 
 	if (supp_rate) {
 		supp_rate &= mask->control[band].legacy;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c
index 4499a630e8f1..7bd5f6725d7b 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/mmio.c
@@ -75,6 +75,7 @@ static const u32 mt7915_offs[] = {
 	[AGG_AWSCR0]		= 0x05c,
 	[AGG_PCR0]		= 0x06c,
 	[AGG_ACR0]		= 0x084,
+	[AGG_ACR4]		= 0x08c,
 	[AGG_MRCR]		= 0x098,
 	[AGG_ATCR1]		= 0x0f0,
 	[AGG_ATCR3]		= 0x0f4,
@@ -148,6 +149,7 @@ static const u32 mt7916_offs[] = {
 	[AGG_AWSCR0]		= 0x030,
 	[AGG_PCR0]		= 0x040,
 	[AGG_ACR0]		= 0x054,
+	[AGG_ACR4]		= 0x05c,
 	[AGG_MRCR]		= 0x068,
 	[AGG_ATCR1]		= 0x1a8,
 	[AGG_ATCR3]		= 0x080,
@@ -204,147 +206,147 @@ static const u32 mt7916_offs[] = {
 	[ETBF_PAR_RPT0]		= 0x100,
 };
 
-static const struct __map mt7915_reg_map[] = {
+static const struct mt76_connac_reg_map mt7915_reg_map[] = {
 	{ 0x00400000, 0x80000, 0x10000 }, /* WF_MCU_SYSRAM */
 	{ 0x00410000, 0x90000, 0x10000 }, /* WF_MCU_SYSRAM (configure regs) */
 	{ 0x40000000, 0x70000, 0x10000 }, /* WF_UMAC_SYSRAM */
-	{ 0x54000000, 0x02000, 0x1000 }, /* WFDMA PCIE0 MCU DMA0 */
-	{ 0x55000000, 0x03000, 0x1000 }, /* WFDMA PCIE0 MCU DMA1 */
-	{ 0x58000000, 0x06000, 0x1000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */
-	{ 0x59000000, 0x07000, 0x1000 }, /* WFDMA PCIE1 MCU DMA1 */
+	{ 0x54000000, 0x02000, 0x01000 }, /* WFDMA PCIE0 MCU DMA0 */
+	{ 0x55000000, 0x03000, 0x01000 }, /* WFDMA PCIE0 MCU DMA1 */
+	{ 0x58000000, 0x06000, 0x01000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */
+	{ 0x59000000, 0x07000, 0x01000 }, /* WFDMA PCIE1 MCU DMA1 */
 	{ 0x7c000000, 0xf0000, 0x10000 }, /* CONN_INFRA */
 	{ 0x7c020000, 0xd0000, 0x10000 }, /* CONN_INFRA, WFDMA */
 	{ 0x80020000, 0xb0000, 0x10000 }, /* WF_TOP_MISC_OFF */
 	{ 0x81020000, 0xc0000, 0x10000 }, /* WF_TOP_MISC_ON */
-	{ 0x820c0000, 0x08000, 0x4000 }, /* WF_UMAC_TOP (PLE) */
-	{ 0x820c8000, 0x0c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */
-	{ 0x820cc000, 0x0e000, 0x2000 }, /* WF_UMAC_TOP (PP) */
-	{ 0x820ce000, 0x21c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */
-	{ 0x820cf000, 0x22000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */
+	{ 0x820c0000, 0x08000, 0x04000 }, /* WF_UMAC_TOP (PLE) */
+	{ 0x820c8000, 0x0c000, 0x02000 }, /* WF_UMAC_TOP (PSE) */
+	{ 0x820cc000, 0x0e000, 0x02000 }, /* WF_UMAC_TOP (PP) */
+	{ 0x820ce000, 0x21c00, 0x00200 }, /* WF_LMAC_TOP (WF_SEC) */
+	{ 0x820cf000, 0x22000, 0x01000 }, /* WF_LMAC_TOP (WF_PF) */
 	{ 0x820d0000, 0x30000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */
-	{ 0x820e0000, 0x20000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */
-	{ 0x820e1000, 0x20400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */
-	{ 0x820e2000, 0x20800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */
-	{ 0x820e3000, 0x20c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */
-	{ 0x820e4000, 0x21000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */
-	{ 0x820e5000, 0x21400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */
-	{ 0x820e7000, 0x21e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */
-	{ 0x820e9000, 0x23400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
-	{ 0x820ea000, 0x24000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */
-	{ 0x820eb000, 0x24200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */
-	{ 0x820ec000, 0x24600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */
-	{ 0x820ed000, 0x24800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */
-	{ 0x820f0000, 0xa0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */
-	{ 0x820f1000, 0xa0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */
-	{ 0x820f2000, 0xa0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */
-	{ 0x820f3000, 0xa0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */
-	{ 0x820f4000, 0xa1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */
-	{ 0x820f5000, 0xa1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */
-	{ 0x820f7000, 0xa1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */
-	{ 0x820f9000, 0xa3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
-	{ 0x820fa000, 0xa4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */
-	{ 0x820fb000, 0xa4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */
-	{ 0x820fc000, 0xa4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */
-	{ 0x820fd000, 0xa4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */
+	{ 0x820e0000, 0x20000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */
+	{ 0x820e1000, 0x20400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */
+	{ 0x820e2000, 0x20800, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */
+	{ 0x820e3000, 0x20c00, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */
+	{ 0x820e4000, 0x21000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */
+	{ 0x820e5000, 0x21400, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */
+	{ 0x820e7000, 0x21e00, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */
+	{ 0x820e9000, 0x23400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
+	{ 0x820ea000, 0x24000, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */
+	{ 0x820eb000, 0x24200, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */
+	{ 0x820ec000, 0x24600, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_INT) */
+	{ 0x820ed000, 0x24800, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */
+	{ 0x820f0000, 0xa0000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */
+	{ 0x820f1000, 0xa0600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */
+	{ 0x820f2000, 0xa0800, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */
+	{ 0x820f3000, 0xa0c00, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */
+	{ 0x820f4000, 0xa1000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */
+	{ 0x820f5000, 0xa1400, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */
+	{ 0x820f7000, 0xa1e00, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */
+	{ 0x820f9000, 0xa3400, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
+	{ 0x820fa000, 0xa4000, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */
+	{ 0x820fb000, 0xa4200, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */
+	{ 0x820fc000, 0xa4600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_INT) */
+	{ 0x820fd000, 0xa4800, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */
 	{ 0x0, 0x0, 0x0 }, /* imply end of search */
 };
 
-static const struct __map mt7916_reg_map[] = {
-	{ 0x54000000, 0x02000, 0x1000 }, /* WFDMA_0 (PCIE0 MCU DMA0) */
-	{ 0x55000000, 0x03000, 0x1000 }, /* WFDMA_1 (PCIE0 MCU DMA1) */
-	{ 0x56000000, 0x04000, 0x1000 }, /* WFDMA_2 (Reserved) */
-	{ 0x57000000, 0x05000, 0x1000 }, /* WFDMA_3 (MCU wrap CR) */
-	{ 0x58000000, 0x06000, 0x1000 }, /* WFDMA_4 (PCIE1 MCU DMA0) */
-	{ 0x59000000, 0x07000, 0x1000 }, /* WFDMA_5 (PCIE1 MCU DMA1) */
-	{ 0x820c0000, 0x08000, 0x4000 }, /* WF_UMAC_TOP (PLE) */
-	{ 0x820c8000, 0x0c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */
-	{ 0x820cc000, 0x0e000, 0x2000 }, /* WF_UMAC_TOP (PP) */
-	{ 0x820e0000, 0x20000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */
-	{ 0x820e1000, 0x20400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */
-	{ 0x820e2000, 0x20800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */
-	{ 0x820e3000, 0x20c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */
-	{ 0x820e4000, 0x21000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */
-	{ 0x820e5000, 0x21400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */
-	{ 0x820ce000, 0x21c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */
-	{ 0x820e7000, 0x21e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */
-	{ 0x820cf000, 0x22000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */
-	{ 0x820e9000, 0x23400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
-	{ 0x820ea000, 0x24000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */
-	{ 0x820eb000, 0x24200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */
-	{ 0x820ec000, 0x24600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */
-	{ 0x820ed000, 0x24800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */
-	{ 0x820ca000, 0x26000, 0x2000 }, /* WF_LMAC_TOP BN0 (WF_MUCOP) */
-	{ 0x820d0000, 0x30000, 0x10000}, /* WF_LMAC_TOP (WF_WTBLON) */
-	{ 0x00400000, 0x80000, 0x10000}, /* WF_MCU_SYSRAM */
-	{ 0x00410000, 0x90000, 0x10000}, /* WF_MCU_SYSRAM (configure cr) */
-	{ 0x820f0000, 0xa0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */
-	{ 0x820f1000, 0xa0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */
-	{ 0x820f2000, 0xa0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */
-	{ 0x820f3000, 0xa0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */
-	{ 0x820f4000, 0xa1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */
-	{ 0x820f5000, 0xa1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */
-	{ 0x820f7000, 0xa1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */
-	{ 0x820f9000, 0xa3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
-	{ 0x820fa000, 0xa4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */
-	{ 0x820fb000, 0xa4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */
-	{ 0x820fc000, 0xa4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */
-	{ 0x820fd000, 0xa4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */
-	{ 0x820c4000, 0xa8000, 0x1000 }, /* WF_LMAC_TOP (WF_UWTBL ) */
-	{ 0x820b0000, 0xae000, 0x1000 }, /* [APB2] WFSYS_ON */
-	{ 0x80020000, 0xb0000, 0x10000}, /* WF_TOP_MISC_OFF */
-	{ 0x81020000, 0xc0000, 0x10000}, /* WF_TOP_MISC_ON */
+static const struct mt76_connac_reg_map mt7916_reg_map[] = {
+	{ 0x54000000, 0x02000, 0x01000 }, /* WFDMA_0 (PCIE0 MCU DMA0) */
+	{ 0x55000000, 0x03000, 0x01000 }, /* WFDMA_1 (PCIE0 MCU DMA1) */
+	{ 0x56000000, 0x04000, 0x01000 }, /* WFDMA_2 (Reserved) */
+	{ 0x57000000, 0x05000, 0x01000 }, /* WFDMA_3 (MCU wrap CR) */
+	{ 0x58000000, 0x06000, 0x01000 }, /* WFDMA_4 (PCIE1 MCU DMA0) */
+	{ 0x59000000, 0x07000, 0x01000 }, /* WFDMA_5 (PCIE1 MCU DMA1) */
+	{ 0x820c0000, 0x08000, 0x04000 }, /* WF_UMAC_TOP (PLE) */
+	{ 0x820c8000, 0x0c000, 0x02000 }, /* WF_UMAC_TOP (PSE) */
+	{ 0x820cc000, 0x0e000, 0x02000 }, /* WF_UMAC_TOP (PP) */
+	{ 0x820e0000, 0x20000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */
+	{ 0x820e1000, 0x20400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */
+	{ 0x820e2000, 0x20800, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */
+	{ 0x820e3000, 0x20c00, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */
+	{ 0x820e4000, 0x21000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */
+	{ 0x820e5000, 0x21400, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */
+	{ 0x820ce000, 0x21c00, 0x00200 }, /* WF_LMAC_TOP (WF_SEC) */
+	{ 0x820e7000, 0x21e00, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */
+	{ 0x820cf000, 0x22000, 0x01000 }, /* WF_LMAC_TOP (WF_PF) */
+	{ 0x820e9000, 0x23400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
+	{ 0x820ea000, 0x24000, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */
+	{ 0x820eb000, 0x24200, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */
+	{ 0x820ec000, 0x24600, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_INT) */
+	{ 0x820ed000, 0x24800, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */
+	{ 0x820ca000, 0x26000, 0x02000 }, /* WF_LMAC_TOP BN0 (WF_MUCOP) */
+	{ 0x820d0000, 0x30000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */
+	{ 0x00400000, 0x80000, 0x10000 }, /* WF_MCU_SYSRAM */
+	{ 0x00410000, 0x90000, 0x10000 }, /* WF_MCU_SYSRAM (configure cr) */
+	{ 0x820f0000, 0xa0000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */
+	{ 0x820f1000, 0xa0600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */
+	{ 0x820f2000, 0xa0800, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */
+	{ 0x820f3000, 0xa0c00, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */
+	{ 0x820f4000, 0xa1000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */
+	{ 0x820f5000, 0xa1400, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */
+	{ 0x820f7000, 0xa1e00, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */
+	{ 0x820f9000, 0xa3400, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
+	{ 0x820fa000, 0xa4000, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */
+	{ 0x820fb000, 0xa4200, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */
+	{ 0x820fc000, 0xa4600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_INT) */
+	{ 0x820fd000, 0xa4800, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */
+	{ 0x820c4000, 0xa8000, 0x01000 }, /* WF_LMAC_TOP (WF_UWTBL ) */
+	{ 0x820b0000, 0xae000, 0x01000 }, /* [APB2] WFSYS_ON */
+	{ 0x80020000, 0xb0000, 0x10000 }, /* WF_TOP_MISC_OFF */
+	{ 0x81020000, 0xc0000, 0x10000 }, /* WF_TOP_MISC_ON */
 	{ 0x0, 0x0, 0x0 }, /* imply end of search */
 };
 
-static const struct __map mt7986_reg_map[] = {
-	{ 0x54000000, 0x402000, 0x1000 }, /* WFDMA_0 (PCIE0 MCU DMA0) */
-	{ 0x55000000, 0x403000, 0x1000 }, /* WFDMA_1 (PCIE0 MCU DMA1) */
-	{ 0x56000000, 0x404000, 0x1000 }, /* WFDMA_2 (Reserved) */
-	{ 0x57000000, 0x405000, 0x1000 }, /* WFDMA_3 (MCU wrap CR) */
-	{ 0x58000000, 0x406000, 0x1000 }, /* WFDMA_4 (PCIE1 MCU DMA0) */
-	{ 0x59000000, 0x407000, 0x1000 }, /* WFDMA_5 (PCIE1 MCU DMA1) */
-	{ 0x820c0000, 0x408000, 0x4000 }, /* WF_UMAC_TOP (PLE) */
-	{ 0x820c8000, 0x40c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */
-	{ 0x820cc000, 0x40e000, 0x2000 }, /* WF_UMAC_TOP (PP) */
-	{ 0x820e0000, 0x420000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */
-	{ 0x820e1000, 0x420400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */
-	{ 0x820e2000, 0x420800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */
-	{ 0x820e3000, 0x420c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */
-	{ 0x820e4000, 0x421000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */
-	{ 0x820e5000, 0x421400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */
-	{ 0x820ce000, 0x421c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */
-	{ 0x820e7000, 0x421e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */
-	{ 0x820cf000, 0x422000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */
-	{ 0x820e9000, 0x423400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
-	{ 0x820ea000, 0x424000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */
-	{ 0x820eb000, 0x424200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */
-	{ 0x820ec000, 0x424600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */
-	{ 0x820ed000, 0x424800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */
-	{ 0x820ca000, 0x426000, 0x2000 }, /* WF_LMAC_TOP BN0 (WF_MUCOP) */
-	{ 0x820d0000, 0x430000, 0x10000}, /* WF_LMAC_TOP (WF_WTBLON) */
-	{ 0x00400000, 0x480000, 0x10000}, /* WF_MCU_SYSRAM */
-	{ 0x00410000, 0x490000, 0x10000}, /* WF_MCU_SYSRAM */
-	{ 0x820f0000, 0x4a0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */
-	{ 0x820f1000, 0x4a0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */
-	{ 0x820f2000, 0x4a0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */
-	{ 0x820f3000, 0x4a0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */
-	{ 0x820f4000, 0x4a1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */
-	{ 0x820f5000, 0x4a1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */
-	{ 0x820f7000, 0x4a1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */
-	{ 0x820f9000, 0x4a3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
-	{ 0x820fa000, 0x4a4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */
-	{ 0x820fb000, 0x4a4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */
-	{ 0x820fc000, 0x4a4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */
-	{ 0x820fd000, 0x4a4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */
-	{ 0x820c4000, 0x4a8000, 0x1000 }, /* WF_LMAC_TOP (WF_UWTBL ) */
-	{ 0x820b0000, 0x4ae000, 0x1000 }, /* [APB2] WFSYS_ON */
-	{ 0x80020000, 0x4b0000, 0x10000}, /* WF_TOP_MISC_OFF */
-	{ 0x81020000, 0x4c0000, 0x10000}, /* WF_TOP_MISC_ON */
-	{ 0x89000000, 0x4d0000, 0x1000 }, /* WF_MCU_CFG_ON */
-	{ 0x89010000, 0x4d1000, 0x1000 }, /* WF_MCU_CIRQ */
-	{ 0x89020000, 0x4d2000, 0x1000 }, /* WF_MCU_GPT */
-	{ 0x89030000, 0x4d3000, 0x1000 }, /* WF_MCU_WDT */
-	{ 0x80010000, 0x4d4000, 0x1000 }, /* WF_AXIDMA */
+static const struct mt76_connac_reg_map mt7986_reg_map[] = {
+	{ 0x54000000, 0x402000, 0x01000 }, /* WFDMA_0 (PCIE0 MCU DMA0) */
+	{ 0x55000000, 0x403000, 0x01000 }, /* WFDMA_1 (PCIE0 MCU DMA1) */
+	{ 0x56000000, 0x404000, 0x01000 }, /* WFDMA_2 (Reserved) */
+	{ 0x57000000, 0x405000, 0x01000 }, /* WFDMA_3 (MCU wrap CR) */
+	{ 0x58000000, 0x406000, 0x01000 }, /* WFDMA_4 (PCIE1 MCU DMA0) */
+	{ 0x59000000, 0x407000, 0x01000 }, /* WFDMA_5 (PCIE1 MCU DMA1) */
+	{ 0x820c0000, 0x408000, 0x04000 }, /* WF_UMAC_TOP (PLE) */
+	{ 0x820c8000, 0x40c000, 0x02000 }, /* WF_UMAC_TOP (PSE) */
+	{ 0x820cc000, 0x40e000, 0x02000 }, /* WF_UMAC_TOP (PP) */
+	{ 0x820e0000, 0x420000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */
+	{ 0x820e1000, 0x420400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */
+	{ 0x820e2000, 0x420800, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */
+	{ 0x820e3000, 0x420c00, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */
+	{ 0x820e4000, 0x421000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */
+	{ 0x820e5000, 0x421400, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */
+	{ 0x820ce000, 0x421c00, 0x00200 }, /* WF_LMAC_TOP (WF_SEC) */
+	{ 0x820e7000, 0x421e00, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */
+	{ 0x820cf000, 0x422000, 0x01000 }, /* WF_LMAC_TOP (WF_PF) */
+	{ 0x820e9000, 0x423400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
+	{ 0x820ea000, 0x424000, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */
+	{ 0x820eb000, 0x424200, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */
+	{ 0x820ec000, 0x424600, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_INT) */
+	{ 0x820ed000, 0x424800, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */
+	{ 0x820ca000, 0x426000, 0x02000 }, /* WF_LMAC_TOP BN0 (WF_MUCOP) */
+	{ 0x820d0000, 0x430000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */
+	{ 0x00400000, 0x480000, 0x10000 }, /* WF_MCU_SYSRAM */
+	{ 0x00410000, 0x490000, 0x10000 }, /* WF_MCU_SYSRAM */
+	{ 0x820f0000, 0x4a0000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */
+	{ 0x820f1000, 0x4a0600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */
+	{ 0x820f2000, 0x4a0800, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */
+	{ 0x820f3000, 0x4a0c00, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */
+	{ 0x820f4000, 0x4a1000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */
+	{ 0x820f5000, 0x4a1400, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */
+	{ 0x820f7000, 0x4a1e00, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */
+	{ 0x820f9000, 0x4a3400, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
+	{ 0x820fa000, 0x4a4000, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */
+	{ 0x820fb000, 0x4a4200, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */
+	{ 0x820fc000, 0x4a4600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_INT) */
+	{ 0x820fd000, 0x4a4800, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */
+	{ 0x820c4000, 0x4a8000, 0x01000 }, /* WF_LMAC_TOP (WF_UWTBL ) */
+	{ 0x820b0000, 0x4ae000, 0x01000 }, /* [APB2] WFSYS_ON */
+	{ 0x80020000, 0x4b0000, 0x10000 }, /* WF_TOP_MISC_OFF */
+	{ 0x81020000, 0x4c0000, 0x10000 }, /* WF_TOP_MISC_ON */
+	{ 0x89000000, 0x4d0000, 0x01000 }, /* WF_MCU_CFG_ON */
+	{ 0x89010000, 0x4d1000, 0x01000 }, /* WF_MCU_CIRQ */
+	{ 0x89020000, 0x4d2000, 0x01000 }, /* WF_MCU_GPT */
+	{ 0x89030000, 0x4d3000, 0x01000 }, /* WF_MCU_WDT */
+	{ 0x80010000, 0x4d4000, 0x01000 }, /* WF_AXIDMA */
 	{ 0x0, 0x0, 0x0 }, /* imply end of search */
 };
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h
index 54ef2a12a443..1eb11617a625 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/mt7915.h
@@ -127,8 +127,6 @@ struct mt7915_sta {
 	unsigned long jiffies;
 	unsigned long ampdu_state;
 
-	struct mt76_sta_stats stats;
-
 	struct mt76_connac_sta_key_conf bip;
 
 	struct {
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c
index d74f609775d3..728a879c3b00 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/pci.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/pci.c
@@ -99,6 +99,7 @@ static int mt7915_pci_hif2_probe(struct pci_dev *pdev)
 static int mt7915_wed_offload_enable(struct mtk_wed_device *wed)
 {
 	struct mt7915_dev *dev;
+	struct mt7915_phy *phy;
 	int ret;
 
 	dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed);
@@ -112,18 +113,38 @@ static int mt7915_wed_offload_enable(struct mtk_wed_device *wed)
 	if (!ret)
 		return -EAGAIN;
 
+	phy = &dev->phy;
+	mt76_set(dev, MT_AGG_ACR4(phy->band_idx), MT_AGG_ACR_PPDU_TXS2H);
+
+	phy = dev->mt76.phys[MT_BAND1] ? dev->mt76.phys[MT_BAND1]->priv : NULL;
+	if (phy)
+		mt76_set(dev, MT_AGG_ACR4(phy->band_idx),
+			 MT_AGG_ACR_PPDU_TXS2H);
+
 	return 0;
 }
 
 static void mt7915_wed_offload_disable(struct mtk_wed_device *wed)
 {
 	struct mt7915_dev *dev;
+	struct mt7915_phy *phy;
 
 	dev = container_of(wed, struct mt7915_dev, mt76.mmio.wed);
 
 	spin_lock_bh(&dev->mt76.token_lock);
 	dev->mt76.token_size = MT7915_TOKEN_SIZE;
 	spin_unlock_bh(&dev->mt76.token_lock);
+
+	/* MT_TXD5_TX_STATUS_HOST (MPDU format) has higher priority than
+	 * MT_AGG_ACR_PPDU_TXS2H (PPDU format) even though ACR bit is set.
+	 */
+	phy = &dev->phy;
+	mt76_clear(dev, MT_AGG_ACR4(phy->band_idx), MT_AGG_ACR_PPDU_TXS2H);
+
+	phy = dev->mt76.phys[MT_BAND1] ? dev->mt76.phys[MT_BAND1]->priv : NULL;
+	if (phy)
+		mt76_clear(dev, MT_AGG_ACR4(phy->band_idx),
+			   MT_AGG_ACR_PPDU_TXS2H);
 }
 #endif
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7915/regs.h b/drivers/net/wireless/mediatek/mt76/mt7915/regs.h
index 2493c3ad3c56..5920e705835a 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7915/regs.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7915/regs.h
@@ -4,17 +4,11 @@
 #ifndef __MT7915_REGS_H
 #define __MT7915_REGS_H
 
-struct __map {
-	u32 phys;
-	u32 maps;
-	u32 size;
-};
-
 /* used to differentiate between generations */
 struct mt7915_reg_desc {
 	const u32 *reg_rev;
 	const u32 *offs_rev;
-	const struct __map *map;
+	const struct mt76_connac_reg_map *map;
 	u32 map_size;
 };
 
@@ -52,6 +46,7 @@ enum offs_rev {
 	AGG_AWSCR0,
 	AGG_PCR0,
 	AGG_ACR0,
+	AGG_ACR4,
 	AGG_MRCR,
 	AGG_ATCR1,
 	AGG_ATCR3,
@@ -471,6 +466,9 @@ enum offs_rev {
 #define MT_AGG_ACR_CFEND_RATE		GENMASK(13, 0)
 #define MT_AGG_ACR_BAR_RATE		GENMASK(29, 16)
 
+#define MT_AGG_ACR4(_band)		MT_WF_AGG(_band, __OFFS(AGG_ACR4))
+#define MT_AGG_ACR_PPDU_TXS2H		BIT(1)
+
 #define MT_AGG_MRCR(_band)		MT_WF_AGG(_band, __OFFS(AGG_MRCR))
 #define MT_AGG_MRCR_BAR_CNT_LIMIT		GENMASK(15, 12)
 #define MT_AGG_MRCR_LAST_RTS_CTS_RN		BIT(6)
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.c b/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.c
index be4f07ad3af9..47e034a9b003 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/acpi_sar.c
@@ -13,6 +13,7 @@ mt7921_acpi_read(struct mt7921_dev *dev, u8 *method, u8 **tbl, u32 *len)
 	acpi_handle root, handle;
 	acpi_status status;
 	u32 i = 0;
+	int ret;
 
 	root = ACPI_HANDLE(mdev->dev);
 	if (!root)
@@ -52,9 +53,11 @@ mt7921_acpi_read(struct mt7921_dev *dev, u8 *method, u8 **tbl, u32 *len)
 		*(*tbl + i) = (u8)sar_unit->integer.value;
 	}
 free:
+	ret = (i == sar_root->package.count) ? 0 : -EINVAL;
+
 	kfree(sar_root);
 
-	return (i == sar_root->package.count) ? 0 : -EINVAL;
+	return ret;
 }
 
 /* MTCL : Country List Table for 6G band */
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h b/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h
index 54f30401343c..4b647278eb30 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/eeprom.h
@@ -11,12 +11,15 @@ enum mt7921_eeprom_field {
 	MT_EE_VERSION =		0x002,
 	MT_EE_MAC_ADDR =	0x004,
 	MT_EE_WIFI_CONF =	0x07c,
-	__MT_EE_MAX =		0x3bf
+	MT_EE_HW_TYPE =		0x55b,
+	__MT_EE_MAX =		0x9ff
 };
 
 #define MT_EE_WIFI_CONF_TX_MASK			BIT(0)
 #define MT_EE_WIFI_CONF_BAND_SEL		GENMASK(3, 2)
 
+#define MT_EE_HW_TYPE_ENCAP			BIT(0)
+
 enum mt7921_eeprom_band {
 	MT_EE_NA,
 	MT_EE_5GHZ,
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/init.c b/drivers/net/wireless/mediatek/mt76/mt7921/init.c
index cd960e23770f..dcdb3cf04ac1 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/init.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/init.c
@@ -39,6 +39,7 @@ mt7921_regd_notifier(struct wiphy *wiphy,
 	dev->mt76.region = request->dfs_region;
 
 	mt7921_mutex_acquire(dev);
+	mt7921_mcu_set_clc(dev, request->alpha2, request->country_ie_env);
 	mt76_connac_mcu_set_channel_domain(hw->priv);
 	mt7921_set_tx_sar_pwr(hw, NULL);
 	mt7921_mutex_release(dev);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
index 47f0aa81ab02..e4868c492bc0 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/mac.c
@@ -235,7 +235,7 @@ mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb)
 	u32 rxd2 = le32_to_cpu(rxd[2]);
 	u32 rxd3 = le32_to_cpu(rxd[3]);
 	u32 rxd4 = le32_to_cpu(rxd[4]);
-	struct mt7921_sta *msta;
+	struct mt7921_sta *msta = NULL;
 	u16 seq_ctrl = 0;
 	__le16 fc = 0;
 	u8 mode = 0;
@@ -486,7 +486,7 @@ mt7921_mac_fill_rx(struct mt7921_dev *dev, struct sk_buff *skb)
 	return 0;
 }
 
-void mt7921_tx_check_aggr(struct ieee80211_sta *sta, __le32 *txwi)
+static void mt7921_tx_check_aggr(struct ieee80211_sta *sta, __le32 *txwi)
 {
 	struct mt7921_sta *msta;
 	u16 fc, tid;
@@ -509,7 +509,6 @@ void mt7921_tx_check_aggr(struct ieee80211_sta *sta, __le32 *txwi)
 	if (!test_and_set_bit(tid, &msta->ampdu_state))
 		ieee80211_start_tx_ba_session(sta, tid, 0);
 }
-EXPORT_SYMBOL_GPL(mt7921_tx_check_aggr);
 
 void mt7921_mac_add_txs(struct mt7921_dev *dev, void *data)
 {
@@ -539,8 +538,7 @@ void mt7921_mac_add_txs(struct mt7921_dev *dev, void *data)
 
 	msta = container_of(wcid, struct mt7921_sta, wcid);
 
-	mt76_connac2_mac_add_txs_skb(&dev->mt76, wcid, pid, txs_data,
-				     &msta->stats);
+	mt76_connac2_mac_add_txs_skb(&dev->mt76, wcid, pid, txs_data);
 	if (!wcid->sta)
 		goto out;
 
@@ -552,7 +550,134 @@ void mt7921_mac_add_txs(struct mt7921_dev *dev, void *data)
 out:
 	rcu_read_unlock();
 }
-EXPORT_SYMBOL_GPL(mt7921_mac_add_txs);
+
+void mt7921_txwi_free(struct mt7921_dev *dev, struct mt76_txwi_cache *t,
+		      struct ieee80211_sta *sta, bool clear_status,
+		      struct list_head *free_list)
+{
+	struct mt76_dev *mdev = &dev->mt76;
+	__le32 *txwi;
+	u16 wcid_idx;
+
+	mt76_connac_txp_skb_unmap(mdev, t);
+	if (!t->skb)
+		goto out;
+
+	txwi = (__le32 *)mt76_get_txwi_ptr(mdev, t);
+	if (sta) {
+		struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv;
+
+		if (likely(t->skb->protocol != cpu_to_be16(ETH_P_PAE)))
+			mt7921_tx_check_aggr(sta, txwi);
+
+		wcid_idx = wcid->idx;
+	} else {
+		wcid_idx = le32_get_bits(txwi[1], MT_TXD1_WLAN_IDX);
+	}
+
+	__mt76_tx_complete_skb(mdev, wcid_idx, t->skb, free_list);
+out:
+	t->skb = NULL;
+	mt76_put_txwi(mdev, t);
+}
+EXPORT_SYMBOL_GPL(mt7921_txwi_free);
+
+static void mt7921_mac_tx_free(struct mt7921_dev *dev, void *data, int len)
+{
+	struct mt76_connac_tx_free *free = data;
+	__le32 *tx_info = (__le32 *)(data + sizeof(*free));
+	struct mt76_dev *mdev = &dev->mt76;
+	struct mt76_txwi_cache *txwi;
+	struct ieee80211_sta *sta = NULL;
+	struct sk_buff *skb, *tmp;
+	void *end = data + len;
+	LIST_HEAD(free_list);
+	bool wake = false;
+	u8 i, count;
+
+	/* clean DMA queues and unmap buffers first */
+	mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_PSD], false);
+	mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_BE], false);
+
+	count = le16_get_bits(free->ctrl, MT_TX_FREE_MSDU_CNT);
+	if (WARN_ON_ONCE((void *)&tx_info[count] > end))
+		return;
+
+	for (i = 0; i < count; i++) {
+		u32 msdu, info = le32_to_cpu(tx_info[i]);
+		u8 stat;
+
+		/* 1'b1: new wcid pair.
+		 * 1'b0: msdu_id with the same 'wcid pair' as above.
+		 */
+		if (info & MT_TX_FREE_PAIR) {
+			struct mt7921_sta *msta;
+			struct mt76_wcid *wcid;
+			u16 idx;
+
+			count++;
+			idx = FIELD_GET(MT_TX_FREE_WLAN_ID, info);
+			wcid = rcu_dereference(dev->mt76.wcid[idx]);
+			sta = wcid_to_sta(wcid);
+			if (!sta)
+				continue;
+
+			msta = container_of(wcid, struct mt7921_sta, wcid);
+			spin_lock_bh(&dev->sta_poll_lock);
+			if (list_empty(&msta->poll_list))
+				list_add_tail(&msta->poll_list, &dev->sta_poll_list);
+			spin_unlock_bh(&dev->sta_poll_lock);
+			continue;
+		}
+
+		msdu = FIELD_GET(MT_TX_FREE_MSDU_ID, info);
+		stat = FIELD_GET(MT_TX_FREE_STATUS, info);
+
+		txwi = mt76_token_release(mdev, msdu, &wake);
+		if (!txwi)
+			continue;
+
+		mt7921_txwi_free(dev, txwi, sta, stat, &free_list);
+	}
+
+	if (wake)
+		mt76_set_tx_blocked(&dev->mt76, false);
+
+	list_for_each_entry_safe(skb, tmp, &free_list, list) {
+		skb_list_del_init(skb);
+		napi_consume_skb(skb, 1);
+	}
+
+	rcu_read_lock();
+	mt7921_mac_sta_poll(dev);
+	rcu_read_unlock();
+
+	mt76_worker_schedule(&dev->mt76.tx_worker);
+}
+
+bool mt7921_rx_check(struct mt76_dev *mdev, void *data, int len)
+{
+	struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
+	__le32 *rxd = (__le32 *)data;
+	__le32 *end = (__le32 *)&rxd[len / 4];
+	enum rx_pkt_type type;
+
+	type = le32_get_bits(rxd[0], MT_RXD0_PKT_TYPE);
+
+	switch (type) {
+	case PKT_TYPE_TXRX_NOTIFY:
+		/* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */
+		mt7921_mac_tx_free(dev, data, len); /* mmio */
+		return false;
+	case PKT_TYPE_TXS:
+		for (rxd += 2; rxd + 8 <= end; rxd += 8)
+			mt7921_mac_add_txs(dev, rxd);
+		return false;
+	default:
+		return true;
+	}
+}
+EXPORT_SYMBOL_GPL(mt7921_rx_check);
 
 void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
 			 struct sk_buff *skb)
@@ -570,6 +695,11 @@ void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
 		type = PKT_TYPE_NORMAL_MCU;
 
 	switch (type) {
+	case PKT_TYPE_TXRX_NOTIFY:
+		/* PKT_TYPE_TXRX_NOTIFY can be received only by mmio devices */
+		mt7921_mac_tx_free(dev, skb->data, skb->len);
+		napi_consume_skb(skb, 1);
+		break;
 	case PKT_TYPE_RX_EVENT:
 		mt7921_mcu_rx_event(dev, skb);
 		break;
@@ -780,6 +910,7 @@ void mt7921_mac_reset_work(struct work_struct *work)
 void mt7921_reset(struct mt76_dev *mdev)
 {
 	struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
+	struct mt76_connac_pm *pm = &dev->pm;
 
 	if (!dev->hw_init_done)
 		return;
@@ -787,8 +918,12 @@ void mt7921_reset(struct mt76_dev *mdev)
 	if (dev->hw_full_reset)
 		return;
 
+	if (pm->suspended)
+		return;
+
 	queue_work(dev->mt76.wq, &dev->reset_work);
 }
+EXPORT_SYMBOL_GPL(mt7921_reset);
 
 void mt7921_mac_update_mib_stats(struct mt7921_phy *phy)
 {
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/main.c b/drivers/net/wireless/mediatek/mt76/mt7921/main.c
index 1438a9f8d1fd..7e409ac7d9a8 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/main.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/main.c
@@ -752,6 +752,7 @@ void mt7921_mac_sta_assoc(struct mt76_dev *mdev, struct ieee80211_vif *vif,
 
 	mt7921_mac_wtbl_update(dev, msta->wcid.idx,
 			       MT_WTBL_UPDATE_ADM_COUNT_CLEAR);
+	memset(msta->airtime_ac, 0, sizeof(msta->airtime_ac));
 
 	mt7921_mcu_sta_update(dev, sta, vif, true, MT76_STA_INFO_STATE_ASSOC);
 
@@ -1045,7 +1046,7 @@ mt7921_ethtool_worker(void *wi_data, struct ieee80211_sta *sta)
 	if (msta->vif->mt76.idx != wi->idx)
 		return;
 
-	mt76_ethtool_worker(wi, &msta->stats);
+	mt76_ethtool_worker(wi, &msta->wcid.stats);
 }
 
 static
@@ -1404,6 +1405,8 @@ static void mt7921_sta_set_decap_offload(struct ieee80211_hw *hw,
 	struct mt7921_sta *msta = (struct mt7921_sta *)sta->drv_priv;
 	struct mt7921_dev *dev = mt7921_hw_dev(hw);
 
+	mt7921_mutex_acquire(dev);
+
 	if (enabled)
 		set_bit(MT_WCID_FLAG_HDR_TRANS, &msta->wcid.flags);
 	else
@@ -1411,6 +1414,8 @@ static void mt7921_sta_set_decap_offload(struct ieee80211_hw *hw,
 
 	mt76_connac_mcu_sta_update_hdr_trans(&dev->mt76, vif, &msta->wcid,
 					     MCU_UNI_CMD(STA_REC_UPDATE));
+
+	mt7921_mutex_release(dev);
 }
 
 #if IS_ENABLED(CONFIG_IPV6)
@@ -1526,17 +1531,23 @@ mt7921_start_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	struct mt7921_dev *dev = mt7921_hw_dev(hw);
 	int err;
 
+	mt7921_mutex_acquire(dev);
+
 	err = mt76_connac_mcu_uni_add_bss(phy->mt76, vif, &mvif->sta.wcid,
 					  true);
 	if (err)
-		return err;
+		goto out;
 
 	err = mt7921_mcu_set_bss_pm(dev, vif, true);
 	if (err)
-		return err;
+		goto out;
+
+	err = mt7921_mcu_sta_update(dev, NULL, vif, true,
+				    MT76_STA_INFO_STATE_NONE);
+out:
+	mt7921_mutex_release(dev);
 
-	return mt7921_mcu_sta_update(dev, NULL, vif, true,
-				     MT76_STA_INFO_STATE_NONE);
+	return err;
 }
 
 static void
@@ -1548,11 +1559,16 @@ mt7921_stop_ap(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	struct mt7921_dev *dev = mt7921_hw_dev(hw);
 	int err;
 
+	mt7921_mutex_acquire(dev);
+
 	err = mt7921_mcu_set_bss_pm(dev, vif, false);
 	if (err)
-		return;
+		goto out;
 
 	mt76_connac_mcu_uni_add_bss(phy->mt76, vif, &mvif->sta.wcid, false);
+
+out:
+	mt7921_mutex_release(dev);
 }
 
 const struct ieee80211_ops mt7921_ops = {
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
index da12d0ae0835..67bf92969a7b 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.c
@@ -2,14 +2,20 @@
 /* Copyright (C) 2020 MediaTek Inc. */
 
 #include <linux/fs.h>
+#include <linux/firmware.h>
 #include "mt7921.h"
 #include "mt7921_trace.h"
+#include "eeprom.h"
 #include "mcu.h"
 #include "mac.h"
 
 #define MT_STA_BFER			BIT(0)
 #define MT_STA_BFEE			BIT(1)
 
+static bool mt7921_disable_clc;
+module_param_named(disable_clc, mt7921_disable_clc, bool, 0644);
+MODULE_PARM_DESC(disable_clc, "disable CLC support");
+
 static int
 mt7921_mcu_parse_eeprom(struct mt76_dev *dev, struct sk_buff *skb)
 {
@@ -84,6 +90,27 @@ int mt7921_mcu_parse_response(struct mt76_dev *mdev, int cmd,
 }
 EXPORT_SYMBOL_GPL(mt7921_mcu_parse_response);
 
+static int mt7921_mcu_read_eeprom(struct mt7921_dev *dev, u32 offset, u8 *val)
+{
+	struct mt7921_mcu_eeprom_info *res, req = {
+		.addr = cpu_to_le32(round_down(offset,
+				    MT7921_EEPROM_BLOCK_SIZE)),
+	};
+	struct sk_buff *skb;
+	int ret;
+
+	ret = mt76_mcu_send_and_get_msg(&dev->mt76, MCU_EXT_QUERY(EFUSE_ACCESS),
+					&req, sizeof(req), true, &skb);
+	if (ret)
+		return ret;
+
+	res = (struct mt7921_mcu_eeprom_info *)skb->data;
+	*val = res->data[offset % MT7921_EEPROM_BLOCK_SIZE];
+	dev_kfree_skb(skb);
+
+	return 0;
+}
+
 #ifdef CONFIG_PM
 
 static int
@@ -354,6 +381,90 @@ static char *mt7921_ram_name(struct mt7921_dev *dev)
 	return ret;
 }
 
+static int mt7921_load_clc(struct mt7921_dev *dev, const char *fw_name)
+{
+	const struct mt76_connac2_fw_trailer *hdr;
+	const struct mt76_connac2_fw_region *region;
+	const struct mt7921_clc *clc;
+	struct mt76_dev *mdev = &dev->mt76;
+	struct mt7921_phy *phy = &dev->phy;
+	const struct firmware *fw;
+	int ret, i, len, offset = 0;
+	u8 *clc_base = NULL, hw_encap = 0;
+
+	if (mt7921_disable_clc ||
+	    mt76_is_usb(&dev->mt76))
+		return 0;
+
+	if (mt76_is_mmio(&dev->mt76)) {
+		ret = mt7921_mcu_read_eeprom(dev, MT_EE_HW_TYPE, &hw_encap);
+		if (ret)
+			return ret;
+		hw_encap = u8_get_bits(hw_encap, MT_EE_HW_TYPE_ENCAP);
+	}
+
+	ret = request_firmware(&fw, fw_name, mdev->dev);
+	if (ret)
+		return ret;
+
+	if (!fw || !fw->data || fw->size < sizeof(*hdr)) {
+		dev_err(mdev->dev, "Invalid firmware\n");
+		ret = -EINVAL;
+		goto out;
+	}
+
+	hdr = (const void *)(fw->data + fw->size - sizeof(*hdr));
+	for (i = 0; i < hdr->n_region; i++) {
+		region = (const void *)((const u8 *)hdr -
+					(hdr->n_region - i) * sizeof(*region));
+		len = le32_to_cpu(region->len);
+
+		/* check if we have valid buffer size */
+		if (offset + len > fw->size) {
+			dev_err(mdev->dev, "Invalid firmware region\n");
+			ret = -EINVAL;
+			goto out;
+		}
+
+		if ((region->feature_set & FW_FEATURE_NON_DL) &&
+		    region->type == FW_TYPE_CLC) {
+			clc_base = (u8 *)(fw->data + offset);
+			break;
+		}
+		offset += len;
+	}
+
+	if (!clc_base)
+		goto out;
+
+	for (offset = 0; offset < len; offset += le32_to_cpu(clc->len)) {
+		clc = (const struct mt7921_clc *)(clc_base + offset);
+
+		/* do not init buf again if chip reset triggered */
+		if (phy->clc[clc->idx])
+			continue;
+
+		/* header content sanity */
+		if (clc->idx == MT7921_CLC_POWER &&
+		    u8_get_bits(clc->type, MT_EE_HW_TYPE_ENCAP) != hw_encap)
+			continue;
+
+		phy->clc[clc->idx] = devm_kmemdup(mdev->dev, clc,
+						  le32_to_cpu(clc->len),
+						  GFP_KERNEL);
+
+		if (!phy->clc[clc->idx]) {
+			ret = -ENOMEM;
+			goto out;
+		}
+	}
+	ret = mt7921_mcu_set_clc(dev, "00", ENVIRON_INDOOR);
+out:
+	release_firmware(fw);
+
+	return ret;
+}
+
 static int mt7921_load_firmware(struct mt7921_dev *dev)
 {
 	int ret;
@@ -423,6 +534,10 @@ int mt7921_run_firmware(struct mt7921_dev *dev)
 		return err;
 
 	set_bit(MT76_STATE_MCU_RUNNING, &dev->mphy.state);
+	err = mt7921_load_clc(dev, mt7921_ram_name(dev));
+	if (err)
+		return err;
+
 	return mt7921_mcu_fw_log_2_host(dev, 1);
 }
 EXPORT_SYMBOL_GPL(mt7921_run_firmware);
@@ -930,3 +1045,86 @@ mt7921_mcu_uni_add_beacon_offload(struct mt7921_dev *dev,
 	return mt76_mcu_send_msg(&dev->mt76, MCU_UNI_CMD(BSS_INFO_UPDATE),
 				 &req, sizeof(req), true);
 }
+
+static
+int __mt7921_mcu_set_clc(struct mt7921_dev *dev, u8 *alpha2,
+			 enum environment_cap env_cap,
+			 struct mt7921_clc *clc,
+			 u8 idx)
+{
+	struct sk_buff *skb;
+	struct {
+		u8 ver;
+		u8 pad0;
+		__le16 len;
+		u8 idx;
+		u8 env;
+		u8 pad1[2];
+		u8 alpha2[2];
+		u8 type[2];
+		u8 rsvd[64];
+	} __packed req = {
+		.idx = idx,
+		.env = env_cap,
+	};
+	int ret, valid_cnt = 0;
+	u8 i, *pos;
+
+	if (!clc)
+		return 0;
+
+	pos = clc->data;
+	for (i = 0; i < clc->nr_country; i++) {
+		struct mt7921_clc_rule *rule = (struct mt7921_clc_rule *)pos;
+		u16 len = le16_to_cpu(rule->len);
+
+		pos += len + sizeof(*rule);
+		if (rule->alpha2[0] != alpha2[0] ||
+		    rule->alpha2[1] != alpha2[1])
+			continue;
+
+		memcpy(req.alpha2, rule->alpha2, 2);
+		memcpy(req.type, rule->type, 2);
+
+		req.len = cpu_to_le16(sizeof(req) + len);
+		skb = __mt76_mcu_msg_alloc(&dev->mt76, &req,
+					   le16_to_cpu(req.len),
+					   sizeof(req), GFP_KERNEL);
+		if (!skb)
+			return -ENOMEM;
+		skb_put_data(skb, rule->data, len);
+
+		ret = mt76_mcu_skb_send_msg(&dev->mt76, skb,
+					    MCU_CE_CMD(SET_CLC), false);
+		if (ret < 0)
+			return ret;
+		valid_cnt++;
+	}
+
+	if (!valid_cnt)
+		return -ENOENT;
+
+	return 0;
+}
+
+int mt7921_mcu_set_clc(struct mt7921_dev *dev, u8 *alpha2,
+		       enum environment_cap env_cap)
+{
+	struct mt7921_phy *phy = (struct mt7921_phy *)&dev->phy;
+	int i, ret;
+
+	/* submit all clc config */
+	for (i = 0; i < ARRAY_SIZE(phy->clc); i++) {
+		ret = __mt7921_mcu_set_clc(dev, alpha2, env_cap,
+					   phy->clc[i], i);
+
+		/* If no country found, set "00" as default */
+		if (ret == -ENOENT)
+			ret = __mt7921_mcu_set_clc(dev, "00",
+						   ENVIRON_INDOOR,
+						   phy->clc[i], i);
+		if (ret < 0)
+			return ret;
+	}
+	return 0;
+}
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h
index 0d20f7d8d474..96dc870fd35e 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/mcu.h
@@ -41,7 +41,7 @@ enum {
 struct mt7921_mcu_eeprom_info {
 	__le32 addr;
 	__le32 valid;
-	u8 data[16];
+	u8 data[MT7921_EEPROM_BLOCK_SIZE];
 } __packed;
 
 #define MT_RA_RATE_NSS			GENMASK(8, 6)
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h
index c161031ac62a..eaba114a9c7e 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/mt7921.h
@@ -41,6 +41,8 @@
 #define MT7921_EEPROM_SIZE		3584
 #define MT7921_TOKEN_SIZE		8192
 
+#define MT7921_EEPROM_BLOCK_SIZE	16
+
 #define MT7921_CFEND_RATE_DEFAULT	0x49	/* OFDM 24M */
 #define MT7921_CFEND_RATE_11B		0x03	/* 11B LP, 11M */
 
@@ -100,7 +102,6 @@ struct mt7921_sta {
 
 	unsigned long last_txs;
 	unsigned long ampdu_state;
-	struct mt76_sta_stats stats;
 
 	struct mt76_connac_sta_key_conf bip;
 };
@@ -149,6 +150,29 @@ struct mib_stats {
 	u32 tx_amsdu_cnt;
 };
 
+enum {
+	MT7921_CLC_POWER,
+	MT7921_CLC_CHAN,
+	MT7921_CLC_MAX_NUM,
+};
+
+struct mt7921_clc_rule {
+	u8 alpha2[2];
+	u8 type[2];
+	__le16 len;
+	u8 data[];
+} __packed;
+
+struct mt7921_clc {
+	__le32 len;
+	u8 idx;
+	u8 ver;
+	u8 nr_country;
+	u8 type;
+	u8 rsv[8];
+	u8 data[];
+};
+
 struct mt7921_phy {
 	struct mt76_phy *mt76;
 	struct mt7921_dev *dev;
@@ -174,6 +198,8 @@ struct mt7921_phy {
 #ifdef CONFIG_ACPI
 	struct mt7921_acpi_sar *acpisar;
 #endif
+
+	struct mt7921_clc *clc[MT7921_CLC_MAX_NUM];
 };
 
 #define mt7921_init_reset(dev)		((dev)->hif_ops->init_reset(dev))
@@ -380,6 +406,7 @@ int mt7921e_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr,
 
 void mt7921_tx_worker(struct mt76_worker *w);
 void mt7921_tx_token_put(struct mt7921_dev *dev);
+bool mt7921_rx_check(struct mt76_dev *mdev, void *data, int len);
 void mt7921_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
 			 struct sk_buff *skb);
 void mt7921_sta_ps(struct mt76_dev *mdev, struct ieee80211_sta *sta, bool ps);
@@ -410,14 +437,13 @@ int mt7921_testmode_cmd(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			void *data, int len);
 int mt7921_testmode_dump(struct ieee80211_hw *hw, struct sk_buff *msg,
 			 struct netlink_callback *cb, void *data, int len);
-void mt7921_tx_check_aggr(struct ieee80211_sta *sta, __le32 *txwi);
+void mt7921_txwi_free(struct mt7921_dev *dev, struct mt76_txwi_cache *t,
+		      struct ieee80211_sta *sta, bool clear_status,
+		      struct list_head *free_list);
 void mt7921_mac_sta_poll(struct mt7921_dev *dev);
 int mt7921_mcu_parse_response(struct mt76_dev *mdev, int cmd,
 			      struct sk_buff *skb, int seq);
 
-bool mt7921e_rx_check(struct mt76_dev *mdev, void *data, int len);
-void mt7921e_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
-			  struct sk_buff *skb);
 int mt7921e_driver_own(struct mt7921_dev *dev);
 int mt7921e_mac_reset(struct mt7921_dev *dev);
 int mt7921e_mcu_init(struct mt7921_dev *dev);
@@ -479,4 +505,7 @@ mt7921_init_acpi_sar_power(struct mt7921_phy *phy, bool set_default)
 #endif
 int mt7921_set_tx_sar_pwr(struct ieee80211_hw *hw,
 			  const struct cfg80211_sar_specs *sar);
+
+int mt7921_mcu_set_clc(struct mt7921_dev *dev, u8 *alpha2,
+		       enum environment_cap env_cap);
 #endif
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c
index ea3069d18c35..8a53d8f286db 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/pci.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci.c
@@ -123,54 +123,51 @@ static void mt7921e_unregister_device(struct mt7921_dev *dev)
 
 static u32 __mt7921_reg_addr(struct mt7921_dev *dev, u32 addr)
 {
-	static const struct {
-		u32 phys;
-		u32 mapped;
-		u32 size;
-	} fixed_map[] = {
+	static const struct mt76_connac_reg_map fixed_map[] = {
 		{ 0x820d0000, 0x30000, 0x10000 }, /* WF_LMAC_TOP (WF_WTBLON) */
-		{ 0x820ed000, 0x24800, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */
-		{ 0x820e4000, 0x21000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */
-		{ 0x820e7000, 0x21e00, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */
-		{ 0x820eb000, 0x24200, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */
-		{ 0x820e2000, 0x20800, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */
-		{ 0x820e3000, 0x20c00, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */
-		{ 0x820e5000, 0x21400, 0x0800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */
+		{ 0x820ed000, 0x24800, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_MIB) */
+		{ 0x820e4000, 0x21000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_TMAC) */
+		{ 0x820e7000, 0x21e00, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_DMA) */
+		{ 0x820eb000, 0x24200, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_LPON) */
+		{ 0x820e2000, 0x20800, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_AGG) */
+		{ 0x820e3000, 0x20c00, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_ARB) */
+		{ 0x820e5000, 0x21400, 0x00800 }, /* WF_LMAC_TOP BN0 (WF_RMAC) */
 		{ 0x00400000, 0x80000, 0x10000 }, /* WF_MCU_SYSRAM */
 		{ 0x00410000, 0x90000, 0x10000 }, /* WF_MCU_SYSRAM (configure register) */
 		{ 0x40000000, 0x70000, 0x10000 }, /* WF_UMAC_SYSRAM */
-		{ 0x54000000, 0x02000, 0x1000 }, /* WFDMA PCIE0 MCU DMA0 */
-		{ 0x55000000, 0x03000, 0x1000 }, /* WFDMA PCIE0 MCU DMA1 */
-		{ 0x58000000, 0x06000, 0x1000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */
-		{ 0x59000000, 0x07000, 0x1000 }, /* WFDMA PCIE1 MCU DMA1 */
+		{ 0x54000000, 0x02000, 0x01000 }, /* WFDMA PCIE0 MCU DMA0 */
+		{ 0x55000000, 0x03000, 0x01000 }, /* WFDMA PCIE0 MCU DMA1 */
+		{ 0x58000000, 0x06000, 0x01000 }, /* WFDMA PCIE1 MCU DMA0 (MEM_DMA) */
+		{ 0x59000000, 0x07000, 0x01000 }, /* WFDMA PCIE1 MCU DMA1 */
 		{ 0x7c000000, 0xf0000, 0x10000 }, /* CONN_INFRA */
 		{ 0x7c020000, 0xd0000, 0x10000 }, /* CONN_INFRA, WFDMA */
 		{ 0x7c060000, 0xe0000, 0x10000 }, /* CONN_INFRA, conn_host_csr_top */
 		{ 0x80020000, 0xb0000, 0x10000 }, /* WF_TOP_MISC_OFF */
 		{ 0x81020000, 0xc0000, 0x10000 }, /* WF_TOP_MISC_ON */
-		{ 0x820c0000, 0x08000, 0x4000 }, /* WF_UMAC_TOP (PLE) */
-		{ 0x820c8000, 0x0c000, 0x2000 }, /* WF_UMAC_TOP (PSE) */
-		{ 0x820cc000, 0x0e000, 0x1000 }, /* WF_UMAC_TOP (PP) */
-		{ 0x820cd000, 0x0f000, 0x1000 }, /* WF_MDP_TOP */
-		{ 0x820ce000, 0x21c00, 0x0200 }, /* WF_LMAC_TOP (WF_SEC) */
-		{ 0x820cf000, 0x22000, 0x1000 }, /* WF_LMAC_TOP (WF_PF) */
-		{ 0x820e0000, 0x20000, 0x0400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */
-		{ 0x820e1000, 0x20400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */
-		{ 0x820e9000, 0x23400, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
-		{ 0x820ea000, 0x24000, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */
-		{ 0x820ec000, 0x24600, 0x0200 }, /* WF_LMAC_TOP BN0 (WF_INT) */
-		{ 0x820f0000, 0xa0000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */
-		{ 0x820f1000, 0xa0600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */
-		{ 0x820f2000, 0xa0800, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */
-		{ 0x820f3000, 0xa0c00, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */
-		{ 0x820f4000, 0xa1000, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */
-		{ 0x820f5000, 0xa1400, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */
-		{ 0x820f7000, 0xa1e00, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */
-		{ 0x820f9000, 0xa3400, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
-		{ 0x820fa000, 0xa4000, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */
-		{ 0x820fb000, 0xa4200, 0x0400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */
-		{ 0x820fc000, 0xa4600, 0x0200 }, /* WF_LMAC_TOP BN1 (WF_INT) */
-		{ 0x820fd000, 0xa4800, 0x0800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */
+		{ 0x820c0000, 0x08000, 0x04000 }, /* WF_UMAC_TOP (PLE) */
+		{ 0x820c8000, 0x0c000, 0x02000 }, /* WF_UMAC_TOP (PSE) */
+		{ 0x820cc000, 0x0e000, 0x01000 }, /* WF_UMAC_TOP (PP) */
+		{ 0x820cd000, 0x0f000, 0x01000 }, /* WF_MDP_TOP */
+		{ 0x74030000, 0x10000, 0x10000 }, /* PCIE_MAC_IREG */
+		{ 0x820ce000, 0x21c00, 0x00200 }, /* WF_LMAC_TOP (WF_SEC) */
+		{ 0x820cf000, 0x22000, 0x01000 }, /* WF_LMAC_TOP (WF_PF) */
+		{ 0x820e0000, 0x20000, 0x00400 }, /* WF_LMAC_TOP BN0 (WF_CFG) */
+		{ 0x820e1000, 0x20400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_TRB) */
+		{ 0x820e9000, 0x23400, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_WTBLOFF) */
+		{ 0x820ea000, 0x24000, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_ETBF) */
+		{ 0x820ec000, 0x24600, 0x00200 }, /* WF_LMAC_TOP BN0 (WF_INT) */
+		{ 0x820f0000, 0xa0000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_CFG) */
+		{ 0x820f1000, 0xa0600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_TRB) */
+		{ 0x820f2000, 0xa0800, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_AGG) */
+		{ 0x820f3000, 0xa0c00, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_ARB) */
+		{ 0x820f4000, 0xa1000, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_TMAC) */
+		{ 0x820f5000, 0xa1400, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_RMAC) */
+		{ 0x820f7000, 0xa1e00, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_DMA) */
+		{ 0x820f9000, 0xa3400, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_WTBLOFF) */
+		{ 0x820fa000, 0xa4000, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_ETBF) */
+		{ 0x820fb000, 0xa4200, 0x00400 }, /* WF_LMAC_TOP BN1 (WF_LPON) */
+		{ 0x820fc000, 0xa4600, 0x00200 }, /* WF_LMAC_TOP BN1 (WF_INT) */
+		{ 0x820fd000, 0xa4800, 0x00800 }, /* WF_LMAC_TOP BN1 (WF_MIB) */
 	};
 	int i;
 
@@ -187,7 +184,7 @@ static u32 __mt7921_reg_addr(struct mt7921_dev *dev, u32 addr)
 		if (ofs > fixed_map[i].size)
 			continue;
 
-		return fixed_map[i].mapped + ofs;
+		return fixed_map[i].maps + ofs;
 	}
 
 	if ((addr >= 0x18000000 && addr < 0x18c00000) ||
@@ -238,8 +235,8 @@ static int mt7921_pci_probe(struct pci_dev *pdev,
 		.token_size = MT7921_TOKEN_SIZE,
 		.tx_prepare_skb = mt7921e_tx_prepare_skb,
 		.tx_complete_skb = mt76_connac_tx_complete_skb,
-		.rx_check = mt7921e_rx_check,
-		.rx_skb = mt7921e_queue_rx_skb,
+		.rx_check = mt7921_rx_check,
+		.rx_skb = mt7921_queue_rx_skb,
 		.rx_poll_complete = mt7921_rx_poll_complete,
 		.sta_ps = mt7921_sta_ps,
 		.sta_add = mt7921_mac_sta_add,
@@ -288,6 +285,8 @@ static int mt7921_pci_probe(struct pci_dev *pdev,
 		goto err_free_pci_vec;
 	}
 
+	pci_set_drvdata(pdev, mdev);
+
 	dev = container_of(mdev, struct mt7921_dev, mt76);
 	dev->hif_ops = &mt7921_pcie_ops;
 
@@ -367,6 +366,7 @@ static int mt7921_pci_suspend(struct device *device)
 	int i, err;
 
 	pm->suspended = true;
+	flush_work(&dev->reset_work);
 	cancel_delayed_work_sync(&pm->ps_work);
 	cancel_work_sync(&pm->wake_work);
 
@@ -409,9 +409,6 @@ static int mt7921_pci_suspend(struct device *device)
 	if (err)
 		goto restore_napi;
 
-	if (err)
-		goto restore_napi;
-
 	return 0;
 
 restore_napi:
@@ -428,6 +425,9 @@ restore_napi:
 restore_suspend:
 	pm->suspended = false;
 
+	if (err < 0)
+		mt7921_reset(&dev->mt76);
+
 	return err;
 }
 
@@ -441,7 +441,7 @@ static int mt7921_pci_resume(struct device *device)
 
 	err = mt7921_mcu_drv_pmctrl(dev);
 	if (err < 0)
-		return err;
+		goto failed;
 
 	mt7921_wpdma_reinit_cond(dev);
 
@@ -471,11 +471,12 @@ static int mt7921_pci_resume(struct device *device)
 		mt76_connac_mcu_set_deep_sleep(&dev->mt76, false);
 
 	err = mt76_connac_mcu_set_hif_suspend(mdev, false);
-	if (err)
-		return err;
-
+failed:
 	pm->suspended = false;
 
+	if (err < 0)
+		mt7921_reset(&dev->mt76);
+
 	return err;
 }
 
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c
index 576a0149251b..8dd60408b117 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci_mac.c
@@ -53,154 +53,6 @@ int mt7921e_tx_prepare_skb(struct mt76_dev *mdev, void *txwi_ptr,
 	return 0;
 }
 
-static void
-mt7921_txwi_free(struct mt7921_dev *dev, struct mt76_txwi_cache *t,
-		 struct ieee80211_sta *sta, bool clear_status,
-		 struct list_head *free_list)
-{
-	struct mt76_dev *mdev = &dev->mt76;
-	__le32 *txwi;
-	u16 wcid_idx;
-
-	mt76_connac_txp_skb_unmap(mdev, t);
-	if (!t->skb)
-		goto out;
-
-	txwi = (__le32 *)mt76_get_txwi_ptr(mdev, t);
-	if (sta) {
-		struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv;
-
-		if (likely(t->skb->protocol != cpu_to_be16(ETH_P_PAE)))
-			mt7921_tx_check_aggr(sta, txwi);
-
-		wcid_idx = wcid->idx;
-	} else {
-		wcid_idx = le32_get_bits(txwi[1], MT_TXD1_WLAN_IDX);
-	}
-
-	__mt76_tx_complete_skb(mdev, wcid_idx, t->skb, free_list);
-
-out:
-	t->skb = NULL;
-	mt76_put_txwi(mdev, t);
-}
-
-static void
-mt7921e_mac_tx_free(struct mt7921_dev *dev, void *data, int len)
-{
-	struct mt76_connac_tx_free *free = data;
-	__le32 *tx_info = (__le32 *)(data + sizeof(*free));
-	struct mt76_dev *mdev = &dev->mt76;
-	struct mt76_txwi_cache *txwi;
-	struct ieee80211_sta *sta = NULL;
-	struct sk_buff *skb, *tmp;
-	void *end = data + len;
-	LIST_HEAD(free_list);
-	bool wake = false;
-	u8 i, count;
-
-	/* clean DMA queues and unmap buffers first */
-	mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_PSD], false);
-	mt76_queue_tx_cleanup(dev, dev->mphy.q_tx[MT_TXQ_BE], false);
-
-	count = le16_get_bits(free->ctrl, MT_TX_FREE_MSDU_CNT);
-	if (WARN_ON_ONCE((void *)&tx_info[count] > end))
-		return;
-
-	for (i = 0; i < count; i++) {
-		u32 msdu, info = le32_to_cpu(tx_info[i]);
-		u8 stat;
-
-		/* 1'b1: new wcid pair.
-		 * 1'b0: msdu_id with the same 'wcid pair' as above.
-		 */
-		if (info & MT_TX_FREE_PAIR) {
-			struct mt7921_sta *msta;
-			struct mt76_wcid *wcid;
-			u16 idx;
-
-			count++;
-			idx = FIELD_GET(MT_TX_FREE_WLAN_ID, info);
-			wcid = rcu_dereference(dev->mt76.wcid[idx]);
-			sta = wcid_to_sta(wcid);
-			if (!sta)
-				continue;
-
-			msta = container_of(wcid, struct mt7921_sta, wcid);
-			spin_lock_bh(&dev->sta_poll_lock);
-			if (list_empty(&msta->poll_list))
-				list_add_tail(&msta->poll_list, &dev->sta_poll_list);
-			spin_unlock_bh(&dev->sta_poll_lock);
-			continue;
-		}
-
-		msdu = FIELD_GET(MT_TX_FREE_MSDU_ID, info);
-		stat = FIELD_GET(MT_TX_FREE_STATUS, info);
-
-		txwi = mt76_token_release(mdev, msdu, &wake);
-		if (!txwi)
-			continue;
-
-		mt7921_txwi_free(dev, txwi, sta, stat, &free_list);
-	}
-
-	if (wake)
-		mt76_set_tx_blocked(&dev->mt76, false);
-
-	list_for_each_entry_safe(skb, tmp, &free_list, list) {
-		skb_list_del_init(skb);
-		napi_consume_skb(skb, 1);
-	}
-
-	rcu_read_lock();
-	mt7921_mac_sta_poll(dev);
-	rcu_read_unlock();
-
-	mt76_worker_schedule(&dev->mt76.tx_worker);
-}
-
-bool mt7921e_rx_check(struct mt76_dev *mdev, void *data, int len)
-{
-	struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
-	__le32 *rxd = (__le32 *)data;
-	__le32 *end = (__le32 *)&rxd[len / 4];
-	enum rx_pkt_type type;
-
-	type = le32_get_bits(rxd[0], MT_RXD0_PKT_TYPE);
-
-	switch (type) {
-	case PKT_TYPE_TXRX_NOTIFY:
-		mt7921e_mac_tx_free(dev, data, len);
-		return false;
-	case PKT_TYPE_TXS:
-		for (rxd += 2; rxd + 8 <= end; rxd += 8)
-			mt7921_mac_add_txs(dev, rxd);
-		return false;
-	default:
-		return true;
-	}
-}
-
-void mt7921e_queue_rx_skb(struct mt76_dev *mdev, enum mt76_rxq_id q,
-			  struct sk_buff *skb)
-{
-	struct mt7921_dev *dev = container_of(mdev, struct mt7921_dev, mt76);
-	__le32 *rxd = (__le32 *)skb->data;
-	enum rx_pkt_type type;
-
-	type = le32_get_bits(rxd[0], MT_RXD0_PKT_TYPE);
-
-	switch (type) {
-	case PKT_TYPE_TXRX_NOTIFY:
-		mt7921e_mac_tx_free(dev, skb->data, skb->len);
-		napi_consume_skb(skb, 1);
-		break;
-	default:
-		mt7921_queue_rx_skb(mdev, q, skb);
-		break;
-	}
-}
-
 void mt7921_tx_token_put(struct mt7921_dev *dev)
 {
 	struct mt76_txwi_cache *txwi;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/pci_mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/pci_mcu.c
index 5efda694fb9d..86340d3205c5 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/pci_mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/pci_mcu.c
@@ -30,12 +30,7 @@ mt7921_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb,
 	if (ret)
 		return ret;
 
-	if (cmd == MCU_UNI_CMD(HIF_CTRL) ||
-	    cmd == MCU_UNI_CMD(SUSPEND) ||
-	    cmd == MCU_UNI_CMD(OFFLOAD))
-		mdev->mcu.timeout = HZ;
-	else
-		mdev->mcu.timeout = 3 * HZ;
+	mdev->mcu.timeout = 3 * HZ;
 
 	if (cmd == MCU_CMD(FW_SCATTER))
 		txq = MT_MCUQ_FWDL;
@@ -59,6 +54,8 @@ int mt7921e_mcu_init(struct mt7921_dev *dev)
 	if (err)
 		return err;
 
+	mt76_rmw_field(dev, MT_PCIE_MAC_PM, MT_PCIE_MAC_PM_L0S_DIS, 1);
+
 	err = mt7921_run_firmware(dev);
 
 	mt76_queue_tx_cleanup(dev, dev->mt76.q_mcu[MT_MCUQ_FWDL], false);
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/regs.h b/drivers/net/wireless/mediatek/mt76/mt7921/regs.h
index ea643260ceb6..c65582acfa55 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/regs.h
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/regs.h
@@ -440,6 +440,8 @@
 #define MT_PCIE_MAC_BASE		0x10000
 #define MT_PCIE_MAC(ofs)		(MT_PCIE_MAC_BASE + (ofs))
 #define MT_PCIE_MAC_INT_ENABLE		MT_PCIE_MAC(0x188)
+#define MT_PCIE_MAC_PM			MT_PCIE_MAC(0x194)
+#define MT_PCIE_MAC_PM_L0S_DIS		BIT(8)
 
 #define MT_DMA_SHDL(ofs)		(0x7c026000 + (ofs))
 #define MT_DMASHDL_SW_CONTROL		MT_DMA_SHDL(0x004)
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c b/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c
index 487acd6e2be8..3b25a06fd946 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/sdio.c
@@ -96,6 +96,7 @@ static int mt7921s_probe(struct sdio_func *func,
 		.tx_complete_skb = mt7921_usb_sdio_tx_complete_skb,
 		.tx_status_data = mt7921_usb_sdio_tx_status_data,
 		.rx_skb = mt7921_queue_rx_skb,
+		.rx_check = mt7921_rx_check,
 		.sta_ps = mt7921_sta_ps,
 		.sta_add = mt7921_mac_sta_add,
 		.sta_assoc = mt7921_mac_sta_assoc,
@@ -194,7 +195,6 @@ static void mt7921s_remove(struct sdio_func *func)
 	mt7921s_unregister_device(dev);
 }
 
-#ifdef CONFIG_PM
 static int mt7921s_suspend(struct device *__dev)
 {
 	struct sdio_func *func = dev_to_sdio_func(__dev);
@@ -206,6 +206,7 @@ static int mt7921s_suspend(struct device *__dev)
 	pm->suspended = true;
 	set_bit(MT76_STATE_SUSPEND, &mdev->phy.state);
 
+	flush_work(&dev->reset_work);
 	cancel_delayed_work_sync(&pm->ps_work);
 	cancel_work_sync(&pm->wake_work);
 
@@ -261,6 +262,9 @@ restore_suspend:
 	clear_bit(MT76_STATE_SUSPEND, &mdev->phy.state);
 	pm->suspended = false;
 
+	if (err < 0)
+		mt7921_reset(&dev->mt76);
+
 	return err;
 }
 
@@ -276,7 +280,7 @@ static int mt7921s_resume(struct device *__dev)
 
 	err = mt7921_mcu_drv_pmctrl(dev);
 	if (err < 0)
-		return err;
+		goto failed;
 
 	mt76_worker_enable(&mdev->tx_worker);
 	mt76_worker_enable(&mdev->sdio.txrx_worker);
@@ -288,34 +292,27 @@ static int mt7921s_resume(struct device *__dev)
 		mt76_connac_mcu_set_deep_sleep(mdev, false);
 
 	err = mt76_connac_mcu_set_hif_suspend(mdev, false);
-	if (err)
-		return err;
-
+failed:
 	pm->suspended = false;
 
+	if (err < 0)
+		mt7921_reset(&dev->mt76);
+
 	return err;
 }
 
-static const struct dev_pm_ops mt7921s_pm_ops = {
-	.suspend = mt7921s_suspend,
-	.resume = mt7921s_resume,
-};
-#endif
-
 MODULE_DEVICE_TABLE(sdio, mt7921s_table);
 MODULE_FIRMWARE(MT7921_FIRMWARE_WM);
 MODULE_FIRMWARE(MT7921_ROM_PATCH);
 
+static DEFINE_SIMPLE_DEV_PM_OPS(mt7921s_pm_ops, mt7921s_suspend, mt7921s_resume);
+
 static struct sdio_driver mt7921s_driver = {
 	.name		= KBUILD_MODNAME,
 	.probe		= mt7921s_probe,
 	.remove		= mt7921s_remove,
 	.id_table	= mt7921s_table,
-#ifdef CONFIG_PM
-	.drv = {
-		.pm = &mt7921s_pm_ops,
-	}
-#endif
+	.drv.pm		= pm_sleep_ptr(&mt7921s_pm_ops),
 };
 module_sdio_driver(mt7921s_driver);
 MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c b/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c
index e038d7404323..5c1489766d9f 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/sdio_mcu.c
@@ -33,12 +33,7 @@ mt7921s_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb,
 	if (ret)
 		return ret;
 
-	if (cmd == MCU_UNI_CMD(HIF_CTRL) ||
-	    cmd == MCU_UNI_CMD(SUSPEND) ||
-	    cmd == MCU_UNI_CMD(OFFLOAD))
-		mdev->mcu.timeout = HZ;
-	else
-		mdev->mcu.timeout = 3 * HZ;
+	mdev->mcu.timeout = 3 * HZ;
 
 	if (cmd == MCU_CMD(FW_SCATTER))
 		type = MT7921_SDIO_FWDL;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7921/usb.c b/drivers/net/wireless/mediatek/mt76/mt7921/usb.c
index dd3b8884e162..29c0ee330dbe 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7921/usb.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7921/usb.c
@@ -106,12 +106,7 @@ mt7921u_mcu_send_message(struct mt76_dev *mdev, struct sk_buff *skb,
 	if (ret)
 		return ret;
 
-	if (cmd == MCU_UNI_CMD(HIF_CTRL) ||
-	    cmd == MCU_UNI_CMD(SUSPEND) ||
-	    cmd == MCU_UNI_CMD(OFFLOAD))
-		mdev->mcu.timeout = HZ;
-	else
-		mdev->mcu.timeout = 3 * HZ;
+	mdev->mcu.timeout = 3 * HZ;
 
 	if (cmd != MCU_CMD(FW_SCATTER))
 		ep = MT_EP_OUT_INBAND_CMD;
@@ -183,6 +178,7 @@ static int mt7921u_probe(struct usb_interface *usb_intf,
 		.tx_complete_skb = mt7921_usb_sdio_tx_complete_skb,
 		.tx_status_data = mt7921_usb_sdio_tx_status_data,
 		.rx_skb = mt7921_queue_rx_skb,
+		.rx_check = mt7921_rx_check,
 		.sta_ps = mt7921_sta_ps,
 		.sta_add = mt7921_mac_sta_add,
 		.sta_assoc = mt7921_mac_sta_assoc,
@@ -300,23 +296,34 @@ static void mt7921u_disconnect(struct usb_interface *usb_intf)
 static int mt7921u_suspend(struct usb_interface *intf, pm_message_t state)
 {
 	struct mt7921_dev *dev = usb_get_intfdata(intf);
+	struct mt76_connac_pm *pm = &dev->pm;
 	int err;
 
+	pm->suspended = true;
+	flush_work(&dev->reset_work);
+
 	err = mt76_connac_mcu_set_hif_suspend(&dev->mt76, true);
 	if (err)
-		return err;
+		goto failed;
 
 	mt76u_stop_rx(&dev->mt76);
 	mt76u_stop_tx(&dev->mt76);
 
-	set_bit(MT76_STATE_SUSPEND, &dev->mphy.state);
-
 	return 0;
+
+failed:
+	pm->suspended = false;
+
+	if (err < 0)
+		mt7921_reset(&dev->mt76);
+
+	return err;
 }
 
 static int mt7921u_resume(struct usb_interface *intf)
 {
 	struct mt7921_dev *dev = usb_get_intfdata(intf);
+	struct mt76_connac_pm *pm = &dev->pm;
 	bool reinit = true;
 	int err, i;
 
@@ -338,16 +345,21 @@ static int mt7921u_resume(struct usb_interface *intf)
 	if (reinit || mt7921_dma_need_reinit(dev)) {
 		err = mt7921u_dma_init(dev, true);
 		if (err)
-			return err;
+			goto failed;
 	}
 
-	clear_bit(MT76_STATE_SUSPEND, &dev->mphy.state);
-
 	err = mt76u_resume_rx(&dev->mt76);
 	if (err < 0)
-		return err;
+		goto failed;
+
+	err = mt76_connac_mcu_set_hif_suspend(&dev->mt76, false);
+failed:
+	pm->suspended = false;
+
+	if (err < 0)
+		mt7921_reset(&dev->mt76);
 
-	return mt76_connac_mcu_set_hif_suspend(&dev->mt76, false);
+	return err;
 }
 #endif /* CONFIG_PM */
 
diff --git a/drivers/net/wireless/mediatek/mt76/sdio.c b/drivers/net/wireless/mediatek/mt76/sdio.c
index aba2a9865821..0ec308f99af5 100644
--- a/drivers/net/wireless/mediatek/mt76/sdio.c
+++ b/drivers/net/wireless/mediatek/mt76/sdio.c
@@ -478,14 +478,14 @@ static void mt76s_status_worker(struct mt76_worker *w)
 		if (ndata_frames > 0)
 			resched = true;
 
-		if (dev->drv->tx_status_data &&
+		if (dev->drv->tx_status_data && ndata_frames > 0 &&
 		    !test_and_set_bit(MT76_READING_STATS, &dev->phy.state) &&
 		    !test_bit(MT76_STATE_SUSPEND, &dev->phy.state))
-			queue_work(dev->wq, &dev->sdio.stat_work);
+			ieee80211_queue_work(dev->hw, &dev->sdio.stat_work);
 	} while (nframes > 0);
 
 	if (resched)
-		mt76_worker_schedule(&dev->sdio.txrx_worker);
+		mt76_worker_schedule(&dev->tx_worker);
 }
 
 static void mt76s_tx_status_data(struct work_struct *work)
@@ -508,7 +508,7 @@ static void mt76s_tx_status_data(struct work_struct *work)
 	}
 
 	if (count && test_bit(MT76_STATE_RUNNING, &dev->phy.state))
-		queue_work(dev->wq, &sdio->stat_work);
+		ieee80211_queue_work(dev->hw, &sdio->stat_work);
 	else
 		clear_bit(MT76_READING_STATS, &dev->phy.state);
 }
diff --git a/drivers/net/wireless/mediatek/mt76/sdio_txrx.c b/drivers/net/wireless/mediatek/mt76/sdio_txrx.c
index a2601aa9e7b1..bfc4de50a4d2 100644
--- a/drivers/net/wireless/mediatek/mt76/sdio_txrx.c
+++ b/drivers/net/wireless/mediatek/mt76/sdio_txrx.c
@@ -85,7 +85,7 @@ mt76s_rx_run_queue(struct mt76_dev *dev, enum mt76_rxq_id qid,
 	struct mt76_sdio *sdio = &dev->sdio;
 	int len = 0, err, i;
 	struct page *page;
-	u8 *buf;
+	u8 *buf, *end;
 
 	for (i = 0; i < intr->rx.num[qid]; i++)
 		len += round_up(intr->rx.len[qid][i] + 4, 4);
@@ -112,20 +112,29 @@ mt76s_rx_run_queue(struct mt76_dev *dev, enum mt76_rxq_id qid,
 		return err;
 	}
 
-	for (i = 0; i < intr->rx.num[qid]; i++) {
+	end = buf + len;
+	i = 0;
+
+	while (i < intr->rx.num[qid] && buf < end) {
 		int index = (q->head + i) % q->ndesc;
 		struct mt76_queue_entry *e = &q->entry[index];
 		__le32 *rxd = (__le32 *)buf;
 
 		/* parse rxd to get the actual packet length */
 		len = le32_get_bits(rxd[0], GENMASK(15, 0));
-		e->skb = mt76s_build_rx_skb(buf, len, round_up(len + 4, 4));
-		if (!e->skb)
-			break;
 
+		/* Optimized path for TXS */
+		if (!dev->drv->rx_check || dev->drv->rx_check(dev, buf, len)) {
+			e->skb = mt76s_build_rx_skb(buf, len,
+						    round_up(len + 4, 4));
+			if (!e->skb)
+				break;
+
+			if (q->queued + i + 1 == q->ndesc)
+				break;
+			i++;
+		}
 		buf += round_up(len + 4, 4);
-		if (q->queued + i + 1 == q->ndesc)
-			break;
 	}
 	put_page(page);
 
diff --git a/drivers/net/wireless/mediatek/mt76/testmode.c b/drivers/net/wireless/mediatek/mt76/testmode.c
index 71fd3fbfa7d2..0accc71a91c9 100644
--- a/drivers/net/wireless/mediatek/mt76/testmode.c
+++ b/drivers/net/wireless/mediatek/mt76/testmode.c
@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: ISC
 /* Copyright (C) 2020 Felix Fietkau <nbd@nbd.name> */
+
+#include <linux/random.h>
 #include "mt76.h"
 
 const struct nla_policy mt76_tm_policy[NUM_MT76_TM_ATTRS] = {
@@ -123,12 +125,14 @@ int mt76_testmode_alloc_skb(struct mt76_phy *phy, u32 len)
 	if (!head)
 		return -ENOMEM;
 
-	hdr = __skb_put_zero(head, head_len);
+	hdr = __skb_put_zero(head, sizeof(*hdr));
 	hdr->frame_control = cpu_to_le16(fc);
 	memcpy(hdr->addr1, td->addr[0], ETH_ALEN);
 	memcpy(hdr->addr2, td->addr[1], ETH_ALEN);
 	memcpy(hdr->addr3, td->addr[2], ETH_ALEN);
 	skb_set_queue_mapping(head, IEEE80211_AC_BE);
+	get_random_bytes(__skb_put(head, head_len - sizeof(*hdr)),
+			 head_len - sizeof(*hdr));
 
 	info = IEEE80211_SKB_CB(head);
 	info->flags = IEEE80211_TX_CTL_INJECTED |
@@ -154,7 +158,7 @@ int mt76_testmode_alloc_skb(struct mt76_phy *phy, u32 len)
 			return -ENOMEM;
 		}
 
-		__skb_put_zero(frag, frag_len);
+		get_random_bytes(__skb_put(frag, frag_len), frag_len);
 		head->len += frag->len;
 		head->data_len += frag->len;
 
diff --git a/drivers/net/wireless/mediatek/mt76/usb.c b/drivers/net/wireless/mediatek/mt76/usb.c
index 6b8964c19f50..4c4033bb1bb3 100644
--- a/drivers/net/wireless/mediatek/mt76/usb.c
+++ b/drivers/net/wireless/mediatek/mt76/usb.c
@@ -528,6 +528,11 @@ mt76u_process_rx_entry(struct mt76_dev *dev, struct urb *urb,
 
 	head_room = drv_flags & MT_DRV_RX_DMA_HDR ? 0 : MT_DMA_HDR_LEN;
 	data_len = min_t(int, len, data_len - head_room);
+
+	if (len == data_len &&
+	    dev->drv->rx_check && !dev->drv->rx_check(dev, data, data_len))
+		return 0;
+
 	skb = mt76u_build_rx_skb(dev, data, data_len, buf_size);
 	if (!skb)
 		return 0;
diff --git a/drivers/net/wireless/microchip/wilc1000/cfg80211.c b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
index 3ac373d29d93..b89047965e78 100644
--- a/drivers/net/wireless/microchip/wilc1000/cfg80211.c
+++ b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
@@ -540,8 +540,9 @@ static int wilc_wfi_cfg_copy_wpa_info(struct wilc_wfi_key *key_info,
 	return 0;
 }
 
-static int add_key(struct wiphy *wiphy, struct net_device *netdev, u8 key_index,
-		   bool pairwise, const u8 *mac_addr, struct key_params *params)
+static int add_key(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+		   u8 key_index, bool pairwise, const u8 *mac_addr,
+		   struct key_params *params)
 
 {
 	int ret = 0, keylen = params->key_len;
@@ -644,7 +645,7 @@ static int add_key(struct wiphy *wiphy, struct net_device *netdev, u8 key_index,
 	return ret;
 }
 
-static int del_key(struct wiphy *wiphy, struct net_device *netdev,
+static int del_key(struct wiphy *wiphy, struct net_device *netdev, int link_id,
 		   u8 key_index,
 		   bool pairwise,
 		   const u8 *mac_addr)
@@ -685,8 +686,9 @@ static int del_key(struct wiphy *wiphy, struct net_device *netdev,
 	return 0;
 }
 
-static int get_key(struct wiphy *wiphy, struct net_device *netdev, u8 key_index,
-		   bool pairwise, const u8 *mac_addr, void *cookie,
+static int get_key(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+		   u8 key_index, bool pairwise, const u8 *mac_addr,
+		   void *cookie,
 		   void (*callback)(void *cookie, struct key_params *))
 {
 	struct wilc_vif *vif = netdev_priv(netdev);
@@ -723,13 +725,14 @@ static int get_key(struct wiphy *wiphy, struct net_device *netdev, u8 key_index,
 
 /* wiphy_new_nm() will WARNON if not present */
 static int set_default_key(struct wiphy *wiphy, struct net_device *netdev,
-			   u8 key_index, bool unicast, bool multicast)
+			   int link_id, u8 key_index, bool unicast,
+			   bool multicast)
 {
 	return 0;
 }
 
 static int set_default_mgmt_key(struct wiphy *wiphy, struct net_device *netdev,
-				u8 key_index)
+				int link_id, u8 key_index)
 {
 	struct wilc_vif *vif = netdev_priv(netdev);
 
@@ -994,12 +997,11 @@ bool wilc_wfi_mgmt_frame_rx(struct wilc_vif *vif, u8 *buff, u32 size)
 {
 	struct wilc *wl = vif->wilc;
 	struct wilc_priv *priv = &vif->priv;
-	int freq, ret;
+	int freq;
 
 	freq = ieee80211_channel_to_frequency(wl->op_ch, NL80211_BAND_2GHZ);
-	ret = cfg80211_rx_mgmt(&priv->wdev, freq, 0, buff, size, 0);
 
-	return ret;
+	return cfg80211_rx_mgmt(&priv->wdev, freq, 0, buff, size, 0);
 }
 
 void wilc_wfi_p2p_rx(struct wilc_vif *vif, u8 *buff, u32 size)
diff --git a/drivers/net/wireless/microchip/wilc1000/mon.c b/drivers/net/wireless/microchip/wilc1000/mon.c
index b5a1b65c087c..03b7229a0ff5 100644
--- a/drivers/net/wireless/microchip/wilc1000/mon.c
+++ b/drivers/net/wireless/microchip/wilc1000/mon.c
@@ -229,7 +229,7 @@ struct net_device *wilc_wfi_init_mon_interface(struct wilc *wl,
 		return NULL;
 
 	wl->monitor_dev->type = ARPHRD_IEEE80211_RADIOTAP;
-	strlcpy(wl->monitor_dev->name, name, IFNAMSIZ);
+	strscpy(wl->monitor_dev->name, name, IFNAMSIZ);
 	wl->monitor_dev->netdev_ops = &wilc_wfi_netdev_ops;
 	wl->monitor_dev->needs_free_netdev = true;
 
diff --git a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
index 1593e810b3ca..bfdf03bfa6c5 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/cfg80211.c
@@ -532,8 +532,8 @@ qtnf_dump_station(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int qtnf_add_key(struct wiphy *wiphy, struct net_device *dev,
-			u8 key_index, bool pairwise, const u8 *mac_addr,
-			struct key_params *params)
+			int link_id, u8 key_index, bool pairwise,
+			const u8 *mac_addr, struct key_params *params)
 {
 	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
 	int ret;
@@ -548,7 +548,8 @@ static int qtnf_add_key(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int qtnf_del_key(struct wiphy *wiphy, struct net_device *dev,
-			u8 key_index, bool pairwise, const u8 *mac_addr)
+			int link_id, u8 key_index, bool pairwise,
+			const u8 *mac_addr)
 {
 	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
 	int ret;
@@ -569,7 +570,8 @@ static int qtnf_del_key(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int qtnf_set_default_key(struct wiphy *wiphy, struct net_device *dev,
-				u8 key_index, bool unicast, bool multicast)
+				int link_id, u8 key_index, bool unicast,
+				bool multicast)
 {
 	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
 	int ret;
@@ -585,7 +587,7 @@ static int qtnf_set_default_key(struct wiphy *wiphy, struct net_device *dev,
 
 static int
 qtnf_set_default_mgmt_key(struct wiphy *wiphy, struct net_device *dev,
-			  u8 key_index)
+			  int link_id, u8 key_index)
 {
 	struct qtnf_vif *vif = qtnf_netdev_get_priv(dev);
 	int ret;
@@ -721,9 +723,8 @@ qtnf_disconnect(struct wiphy *wiphy, struct net_device *dev,
 		return -EFAULT;
 	}
 
-	if (vif->wdev.iftype != NL80211_IFTYPE_STATION) {
+	if (vif->wdev.iftype != NL80211_IFTYPE_STATION)
 		return -EOPNOTSUPP;
-	}
 
 	ret = qtnf_cmd_send_disconnect(vif, reason_code);
 	if (ret)
@@ -750,7 +751,6 @@ qtnf_dump_survey(struct wiphy *wiphy, struct net_device *dev,
 	struct ieee80211_channel *chan;
 	int ret;
 
-
 	sband = wiphy->bands[NL80211_BAND_2GHZ];
 	if (sband && idx >= sband->n_channels) {
 		idx -= sband->n_channels;
@@ -1223,7 +1223,7 @@ int qtnf_wiphy_register(struct qtnf_hw_info *hw_info, struct qtnf_wmac *mac)
 			mac->macinfo.extended_capabilities_len;
 	}
 
-	strlcpy(wiphy->fw_version, hw_info->fw_version,
+	strscpy(wiphy->fw_version, hw_info->fw_version,
 		sizeof(wiphy->fw_version));
 	wiphy->hw_version = hw_info->hw_version;
 
diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.c b/drivers/net/wireless/quantenna/qtnfmac/commands.c
index 0fad53693292..b1b73478d89b 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/commands.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/commands.c
@@ -967,7 +967,7 @@ qtnf_cmd_resp_proc_hw_info(struct qtnf_bus *bus,
 		hwinfo->total_rx_chain, hwinfo->total_tx_chain,
 		hwinfo->fw_ver);
 
-	strlcpy(hwinfo->fw_version, bld_label, sizeof(hwinfo->fw_version));
+	strscpy(hwinfo->fw_version, bld_label, sizeof(hwinfo->fw_version));
 	hwinfo->hw_version = hw_ver;
 
 	return 0;
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800.h b/drivers/net/wireless/ralink/rt2x00/rt2800.h
index d758e8874457..de2ee5ffc34e 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800.h
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800.h
@@ -1016,6 +1016,8 @@
  */
 #define MAC_STATUS_CFG			0x1200
 #define MAC_STATUS_CFG_BBP_RF_BUSY	FIELD32(0x00000003)
+#define MAC_STATUS_CFG_BBP_RF_BUSY_TX	FIELD32(0x00000001)
+#define MAC_STATUS_CFG_BBP_RF_BUSY_RX	FIELD32(0x00000002)
 
 /*
  * PWR_PIN_CFG:
@@ -2739,6 +2741,7 @@ enum rt2800_eeprom_word {
 #define EEPROM_NIC_CONF2_RX_STREAM	FIELD16(0x000f)
 #define EEPROM_NIC_CONF2_TX_STREAM	FIELD16(0x00f0)
 #define EEPROM_NIC_CONF2_CRYSTAL	FIELD16(0x0600)
+#define EEPROM_NIC_CONF2_EXTERNAL_PA	FIELD16(0x8000)
 
 /*
  * EEPROM LNA
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
index 18102fbe36d6..cbbb1a4849cf 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.c
@@ -198,6 +198,26 @@ static void rt2800_rfcsr_write_dccal(struct rt2x00_dev *rt2x00dev,
 	rt2800_rfcsr_write_bank(rt2x00dev, 7, reg, value);
 }
 
+static void rt2800_bbp_dcoc_write(struct rt2x00_dev *rt2x00dev,
+				  const u8 reg, const u8 value)
+{
+	rt2800_bbp_write(rt2x00dev, 158, reg);
+	rt2800_bbp_write(rt2x00dev, 159, value);
+}
+
+static u8 rt2800_bbp_dcoc_read(struct rt2x00_dev *rt2x00dev, const u8 reg)
+{
+	rt2800_bbp_write(rt2x00dev, 158, reg);
+	return rt2800_bbp_read(rt2x00dev, 159);
+}
+
+static void rt2800_bbp_glrt_write(struct rt2x00_dev *rt2x00dev,
+				  const u8 reg, const u8 value)
+{
+	rt2800_bbp_write(rt2x00dev, 195, reg);
+	rt2800_bbp_write(rt2x00dev, 196, value);
+}
+
 static u8 rt2800_rfcsr_read(struct rt2x00_dev *rt2x00dev,
 			    const unsigned int word)
 {
@@ -2143,6 +2163,48 @@ void rt2800_config_erp(struct rt2x00_dev *rt2x00dev, struct rt2x00lib_erp *erp,
 }
 EXPORT_SYMBOL_GPL(rt2800_config_erp);
 
+static int rt2800_wait_bbp_rf_ready(struct rt2x00_dev *rt2x00dev,
+				    const struct rt2x00_field32 mask)
+{
+	unsigned int i;
+	u32 reg;
+
+	for (i = 0; i < REGISTER_BUSY_COUNT; i++) {
+		reg = rt2800_register_read(rt2x00dev, MAC_STATUS_CFG);
+		if (!rt2x00_get_field32(reg, mask))
+			return 0;
+
+		udelay(REGISTER_BUSY_DELAY);
+	}
+
+	rt2x00_err(rt2x00dev, "BBP/RF register access failed, aborting\n");
+	return -EACCES;
+}
+
+static int rt2800_wait_bbp_ready(struct rt2x00_dev *rt2x00dev)
+{
+	unsigned int i;
+	u8 value;
+
+	/*
+	 * BBP was enabled after firmware was loaded,
+	 * but we need to reactivate it now.
+	 */
+	rt2800_register_write(rt2x00dev, H2M_BBP_AGENT, 0);
+	rt2800_register_write(rt2x00dev, H2M_MAILBOX_CSR, 0);
+	msleep(1);
+
+	for (i = 0; i < REGISTER_BUSY_COUNT; i++) {
+		value = rt2800_bbp_read(rt2x00dev, 0);
+		if ((value != 0xff) && (value != 0x00))
+			return 0;
+		udelay(REGISTER_BUSY_DELAY);
+	}
+
+	rt2x00_err(rt2x00dev, "BBP register access failed, aborting\n");
+	return -EACCES;
+}
+
 static void rt2800_config_3572bt_ant(struct rt2x00_dev *rt2x00dev)
 {
 	u32 reg;
@@ -3793,16 +3855,23 @@ static void rt2800_config_channel_rf7620(struct rt2x00_dev *rt2x00dev,
 		rfcsr |= tx_agc_fc;
 		rt2800_rfcsr_write_bank(rt2x00dev, 7, 59, rfcsr);
 	}
+
+	if (conf_is_ht40(conf)) {
+		rt2800_bbp_glrt_write(rt2x00dev, 141, 0x10);
+		rt2800_bbp_glrt_write(rt2x00dev, 157, 0x2f);
+	} else {
+		rt2800_bbp_glrt_write(rt2x00dev, 141, 0x1a);
+		rt2800_bbp_glrt_write(rt2x00dev, 157, 0x40);
+	}
 }
 
 static void rt2800_config_alc(struct rt2x00_dev *rt2x00dev,
 			      struct ieee80211_channel *chan,
 			      int power_level) {
 	u16 eeprom, target_power, max_power;
-	u32 mac_sys_ctrl, mac_status;
+	u32 mac_sys_ctrl;
 	u32 reg;
 	u8 bbp;
-	int i;
 
 	/* hardware unit is 0.5dBm, limited to 23.5dBm */
 	power_level *= 2;
@@ -3838,16 +3907,8 @@ static void rt2800_config_alc(struct rt2x00_dev *rt2x00dev,
 	/* Disable Tx/Rx */
 	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, 0);
 	/* Check MAC Tx/Rx idle */
-	for (i = 0; i < 10000; i++) {
-		mac_status = rt2800_register_read(rt2x00dev, MAC_STATUS_CFG);
-		if (mac_status & 0x3)
-			usleep_range(50, 200);
-		else
-			break;
-	}
-
-	if (i == 10000)
-		rt2x00_warn(rt2x00dev, "Wait MAC Status to MAX !!!\n");
+	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY)))
+		rt2x00_warn(rt2x00dev, "RF busy while configuring ALC\n");
 
 	if (chan->center_freq > 2457) {
 		bbp = rt2800_bbp_read(rt2x00dev, 30);
@@ -4164,7 +4225,10 @@ static void rt2800_config_channel(struct rt2x00_dev *rt2x00dev,
 		rt2800_bbp_write(rt2x00dev, 62, 0x37 - rt2x00dev->lna_gain);
 		rt2800_bbp_write(rt2x00dev, 63, 0x37 - rt2x00dev->lna_gain);
 		rt2800_bbp_write(rt2x00dev, 64, 0x37 - rt2x00dev->lna_gain);
-		rt2800_bbp_write(rt2x00dev, 86, 0);
+		if (rt2x00_rt(rt2x00dev, RT6352))
+			rt2800_bbp_write(rt2x00dev, 86, 0x38);
+		else
+			rt2800_bbp_write(rt2x00dev, 86, 0);
 	}
 
 	if (rf->channel <= 14) {
@@ -4365,7 +4429,45 @@ static void rt2800_config_channel(struct rt2x00_dev *rt2x00dev,
 		reg = (rf->channel <= 14 ? 0x1c : 0x24) + 2*rt2x00dev->lna_gain;
 		rt2800_bbp_write_with_rx_chain(rt2x00dev, 66, reg);
 
-		rt2800_iq_calibrate(rt2x00dev, rf->channel);
+		if (rt2x00_rt(rt2x00dev, RT5592))
+			rt2800_iq_calibrate(rt2x00dev, rf->channel);
+	}
+
+	if (rt2x00_rt(rt2x00dev, RT6352)) {
+		if (test_bit(CAPABILITY_EXTERNAL_PA_TX0,
+			     &rt2x00dev->cap_flags)) {
+			reg = rt2800_register_read(rt2x00dev, RF_CONTROL3);
+			reg |= 0x00000101;
+			rt2800_register_write(rt2x00dev, RF_CONTROL3, reg);
+
+			reg = rt2800_register_read(rt2x00dev, RF_BYPASS3);
+			reg |= 0x00000101;
+			rt2800_register_write(rt2x00dev, RF_BYPASS3, reg);
+
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 43, 0x73);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 44, 0x73);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 45, 0x73);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 46, 0x27);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 47, 0xC8);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 48, 0xA4);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 49, 0x05);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 54, 0x27);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 55, 0xC8);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 56, 0xA4);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 57, 0x05);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 58, 0x27);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 59, 0xC8);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 60, 0xA4);
+			rt2800_rfcsr_write_chanreg(rt2x00dev, 61, 0x05);
+			rt2800_rfcsr_write_dccal(rt2x00dev, 05, 0x00);
+
+			rt2800_register_write(rt2x00dev, TX0_RF_GAIN_CORRECT,
+					      0x36303636);
+			rt2800_register_write(rt2x00dev, TX0_RF_GAIN_ATTEN,
+					      0x6C6C6B6C);
+			rt2800_register_write(rt2x00dev, TX1_RF_GAIN_ATTEN,
+					      0x6C6C6B6C);
+		}
 	}
 
 	bbp = rt2800_bbp_read(rt2x00dev, 4);
@@ -5644,7 +5746,8 @@ static inline void rt2800_set_vgc(struct rt2x00_dev *rt2x00dev,
 	if (qual->vgc_level != vgc_level) {
 		if (rt2x00_rt(rt2x00dev, RT3572) ||
 		    rt2x00_rt(rt2x00dev, RT3593) ||
-		    rt2x00_rt(rt2x00dev, RT3883)) {
+		    rt2x00_rt(rt2x00dev, RT3883) ||
+		    rt2x00_rt(rt2x00dev, RT6352)) {
 			rt2800_bbp_write_with_rx_chain(rt2x00dev, 66,
 						       vgc_level);
 		} else if (rt2x00_rt(rt2x00dev, RT5592)) {
@@ -5867,7 +5970,7 @@ static int rt2800_init_registers(struct rt2x00_dev *rt2x00dev)
 		rt2800_register_write(rt2x00dev, TX_SW_CFG0, 0x00000404);
 	} else if (rt2x00_rt(rt2x00dev, RT6352)) {
 		rt2800_register_write(rt2x00dev, TX_SW_CFG0, 0x00000401);
-		rt2800_register_write(rt2x00dev, TX_SW_CFG1, 0x000C0000);
+		rt2800_register_write(rt2x00dev, TX_SW_CFG1, 0x000C0001);
 		rt2800_register_write(rt2x00dev, TX_SW_CFG2, 0x00000000);
 		rt2800_register_write(rt2x00dev, TX_ALC_VGA3, 0x00000000);
 		rt2800_register_write(rt2x00dev, TX0_BB_GAIN_ATTEN, 0x0);
@@ -6129,6 +6232,27 @@ static int rt2800_init_registers(struct rt2x00_dev *rt2x00dev)
 		reg = rt2800_register_read(rt2x00dev, US_CYC_CNT);
 		rt2x00_set_field32(&reg, US_CYC_CNT_CLOCK_CYCLE, 125);
 		rt2800_register_write(rt2x00dev, US_CYC_CNT, reg);
+	} else if (rt2x00_is_soc(rt2x00dev)) {
+		struct clk *clk = clk_get_sys("bus", NULL);
+		int rate;
+
+		if (IS_ERR(clk)) {
+			clk = clk_get_sys("cpu", NULL);
+
+			if (IS_ERR(clk)) {
+				rate = 125;
+			} else {
+				rate = clk_get_rate(clk) / 3000000;
+				clk_put(clk);
+			}
+		} else {
+			rate = clk_get_rate(clk) / 1000000;
+			clk_put(clk);
+		}
+
+		reg = rt2800_register_read(rt2x00dev, US_CYC_CNT);
+		rt2x00_set_field32(&reg, US_CYC_CNT_CLOCK_CYCLE, rate);
+		rt2800_register_write(rt2x00dev, US_CYC_CNT, reg);
 	}
 
 	reg = rt2800_register_read(rt2x00dev, HT_FBK_CFG0);
@@ -6212,46 +6336,6 @@ static int rt2800_init_registers(struct rt2x00_dev *rt2x00dev)
 	return 0;
 }
 
-static int rt2800_wait_bbp_rf_ready(struct rt2x00_dev *rt2x00dev)
-{
-	unsigned int i;
-	u32 reg;
-
-	for (i = 0; i < REGISTER_BUSY_COUNT; i++) {
-		reg = rt2800_register_read(rt2x00dev, MAC_STATUS_CFG);
-		if (!rt2x00_get_field32(reg, MAC_STATUS_CFG_BBP_RF_BUSY))
-			return 0;
-
-		udelay(REGISTER_BUSY_DELAY);
-	}
-
-	rt2x00_err(rt2x00dev, "BBP/RF register access failed, aborting\n");
-	return -EACCES;
-}
-
-static int rt2800_wait_bbp_ready(struct rt2x00_dev *rt2x00dev)
-{
-	unsigned int i;
-	u8 value;
-
-	/*
-	 * BBP was enabled after firmware was loaded,
-	 * but we need to reactivate it now.
-	 */
-	rt2800_register_write(rt2x00dev, H2M_BBP_AGENT, 0);
-	rt2800_register_write(rt2x00dev, H2M_MAILBOX_CSR, 0);
-	msleep(1);
-
-	for (i = 0; i < REGISTER_BUSY_COUNT; i++) {
-		value = rt2800_bbp_read(rt2x00dev, 0);
-		if ((value != 0xff) && (value != 0x00))
-			return 0;
-		udelay(REGISTER_BUSY_DELAY);
-	}
-
-	rt2x00_err(rt2x00dev, "BBP register access failed, aborting\n");
-	return -EACCES;
-}
 
 static void rt2800_bbp4_mac_if_ctrl(struct rt2x00_dev *rt2x00dev)
 {
@@ -6916,26 +7000,6 @@ static void rt2800_init_bbp_5592(struct rt2x00_dev *rt2x00dev)
 		rt2800_bbp_write(rt2x00dev, 103, 0xc0);
 }
 
-static void rt2800_bbp_glrt_write(struct rt2x00_dev *rt2x00dev,
-				  const u8 reg, const u8 value)
-{
-	rt2800_bbp_write(rt2x00dev, 195, reg);
-	rt2800_bbp_write(rt2x00dev, 196, value);
-}
-
-static void rt2800_bbp_dcoc_write(struct rt2x00_dev *rt2x00dev,
-				  const u8 reg, const u8 value)
-{
-	rt2800_bbp_write(rt2x00dev, 158, reg);
-	rt2800_bbp_write(rt2x00dev, 159, value);
-}
-
-static u8 rt2800_bbp_dcoc_read(struct rt2x00_dev *rt2x00dev, const u8 reg)
-{
-	rt2800_bbp_write(rt2x00dev, 158, reg);
-	return rt2800_bbp_read(rt2x00dev, 159);
-}
-
 static void rt2800_init_bbp_6352(struct rt2x00_dev *rt2x00dev)
 {
 	u8 bbp;
@@ -8398,6 +8462,1519 @@ static void rt2800_init_rfcsr_5592(struct rt2x00_dev *rt2x00dev)
 	rt2800_led_open_drain_enable(rt2x00dev);
 }
 
+static void rt2800_rf_self_txdc_cal(struct rt2x00_dev *rt2x00dev)
+{
+	u8 rfb5r1_org, rfb7r1_org, rfvalue;
+	u32 mac0518, mac051c, mac0528, mac052c;
+	u8 i;
+
+	mac0518 = rt2800_register_read(rt2x00dev, RF_CONTROL0);
+	mac051c = rt2800_register_read(rt2x00dev, RF_BYPASS0);
+	mac0528 = rt2800_register_read(rt2x00dev, RF_CONTROL2);
+	mac052c = rt2800_register_read(rt2x00dev, RF_BYPASS2);
+
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x0);
+	rt2800_register_write(rt2x00dev, RF_BYPASS2, 0x0);
+
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0xC);
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x3306);
+	rt2800_register_write(rt2x00dev, RF_CONTROL2, 0x3330);
+	rt2800_register_write(rt2x00dev, RF_BYPASS2, 0xfffff);
+	rfb5r1_org = rt2800_rfcsr_read_bank(rt2x00dev, 5, 1);
+	rfb7r1_org = rt2800_rfcsr_read_bank(rt2x00dev, 7, 1);
+
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 1, 0x4);
+	for (i = 0; i < 100; ++i) {
+		usleep_range(50, 100);
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 1);
+		if ((rfvalue & 0x04) != 0x4)
+			break;
+	}
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 1, rfb5r1_org);
+
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 1, 0x4);
+	for (i = 0; i < 100; ++i) {
+		usleep_range(50, 100);
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 1);
+		if ((rfvalue & 0x04) != 0x4)
+			break;
+	}
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 1, rfb7r1_org);
+
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x0);
+	rt2800_register_write(rt2x00dev, RF_BYPASS2, 0x0);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, mac0518);
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, mac051c);
+	rt2800_register_write(rt2x00dev, RF_CONTROL2, mac0528);
+	rt2800_register_write(rt2x00dev, RF_BYPASS2, mac052c);
+}
+
+static int rt2800_calcrcalibrationcode(struct rt2x00_dev *rt2x00dev, int d1, int d2)
+{
+	int calcode = ((d2 - d1) * 1000) / 43;
+
+	if ((calcode % 10) >= 5)
+		calcode += 10;
+	calcode = (calcode / 10);
+
+	return calcode;
+}
+
+static void rt2800_r_calibration(struct rt2x00_dev *rt2x00dev)
+{
+	u32 savemacsysctrl;
+	u8 saverfb0r1, saverfb0r34, saverfb0r35;
+	u8 saverfb5r4, saverfb5r17, saverfb5r18;
+	u8 saverfb5r19, saverfb5r20;
+	u8 savebbpr22, savebbpr47, savebbpr49;
+	u8 bytevalue = 0;
+	int rcalcode;
+	u8 r_cal_code = 0;
+	char d1 = 0, d2 = 0;
+	u8 rfvalue;
+	u32 MAC_RF_BYPASS0, MAC_RF_CONTROL0, MAC_PWR_PIN_CFG;
+	u32 maccfg;
+
+	saverfb0r1 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 1);
+	saverfb0r34 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 34);
+	saverfb0r35 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 35);
+	saverfb5r4 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 4);
+	saverfb5r17 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 17);
+	saverfb5r18 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 18);
+	saverfb5r19 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 19);
+	saverfb5r20 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 20);
+
+	savebbpr22 = rt2800_bbp_read(rt2x00dev, 22);
+	savebbpr47 = rt2800_bbp_read(rt2x00dev, 47);
+	savebbpr49 = rt2800_bbp_read(rt2x00dev, 49);
+
+	savemacsysctrl = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	MAC_RF_BYPASS0 = rt2800_register_read(rt2x00dev, RF_BYPASS0);
+	MAC_RF_CONTROL0 = rt2800_register_read(rt2x00dev, RF_CONTROL0);
+	MAC_PWR_PIN_CFG = rt2800_register_read(rt2x00dev, PWR_PIN_CFG);
+
+	maccfg = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	maccfg &= (~0x04);
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, maccfg);
+
+	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_TX)))
+		rt2x00_warn(rt2x00dev, "Wait MAC Tx Status to MAX !!!\n");
+
+	maccfg = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	maccfg &= (~0x04);
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, maccfg);
+
+	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_RX)))
+		rt2x00_warn(rt2x00dev, "Wait MAC Rx Status to MAX !!!\n");
+
+	rfvalue = (MAC_RF_BYPASS0 | 0x3004);
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, rfvalue);
+	rfvalue = (MAC_RF_CONTROL0 | (~0x3002));
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, rfvalue);
+
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, 0x27);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 17, 0x80);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 18, 0x83);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 19, 0x00);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 20, 0x20);
+
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, 0x00);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 34, 0x13);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, 0x00);
+
+	rt2800_register_write(rt2x00dev, PWR_PIN_CFG, 0x1);
+
+	rt2800_bbp_write(rt2x00dev, 47, 0x04);
+	rt2800_bbp_write(rt2x00dev, 22, 0x80);
+	usleep_range(100, 200);
+	bytevalue = rt2800_bbp_read(rt2x00dev, 49);
+	if (bytevalue > 128)
+		d1 = bytevalue - 256;
+	else
+		d1 = (char)bytevalue;
+	rt2800_bbp_write(rt2x00dev, 22, 0x0);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, 0x01);
+
+	rt2800_bbp_write(rt2x00dev, 22, 0x80);
+	usleep_range(100, 200);
+	bytevalue = rt2800_bbp_read(rt2x00dev, 49);
+	if (bytevalue > 128)
+		d2 = bytevalue - 256;
+	else
+		d2 = (char)bytevalue;
+	rt2800_bbp_write(rt2x00dev, 22, 0x0);
+
+	rcalcode = rt2800_calcrcalibrationcode(rt2x00dev, d1, d2);
+	if (rcalcode < 0)
+		r_cal_code = 256 + rcalcode;
+	else
+		r_cal_code = (u8)rcalcode;
+
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 7, r_cal_code);
+
+	rt2800_bbp_write(rt2x00dev, 22, 0x0);
+
+	bytevalue = rt2800_bbp_read(rt2x00dev, 21);
+	bytevalue |= 0x1;
+	rt2800_bbp_write(rt2x00dev, 21, bytevalue);
+	bytevalue = rt2800_bbp_read(rt2x00dev, 21);
+	bytevalue &= (~0x1);
+	rt2800_bbp_write(rt2x00dev, 21, bytevalue);
+
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, saverfb0r1);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 34, saverfb0r34);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, saverfb0r35);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, saverfb5r4);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 17, saverfb5r17);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 18, saverfb5r18);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 19, saverfb5r19);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 20, saverfb5r20);
+
+	rt2800_bbp_write(rt2x00dev, 22, savebbpr22);
+	rt2800_bbp_write(rt2x00dev, 47, savebbpr47);
+	rt2800_bbp_write(rt2x00dev, 49, savebbpr49);
+
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, MAC_RF_BYPASS0);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, MAC_RF_CONTROL0);
+
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, savemacsysctrl);
+	rt2800_register_write(rt2x00dev, PWR_PIN_CFG, MAC_PWR_PIN_CFG);
+}
+
+static void rt2800_rxdcoc_calibration(struct rt2x00_dev *rt2x00dev)
+{
+	u8 bbpreg = 0;
+	u32 macvalue = 0;
+	u8 saverfb0r2, saverfb5r4, saverfb7r4, rfvalue;
+	int i;
+
+	saverfb0r2 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 2);
+	rfvalue = saverfb0r2;
+	rfvalue |= 0x03;
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, rfvalue);
+
+	rt2800_bbp_write(rt2x00dev, 158, 141);
+	bbpreg = rt2800_bbp_read(rt2x00dev, 159);
+	bbpreg |= 0x10;
+	rt2800_bbp_write(rt2x00dev, 159, bbpreg);
+
+	macvalue = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, 0x8);
+
+	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_TX)))
+		rt2x00_warn(rt2x00dev, "RF TX busy in RX RXDCOC calibration\n");
+
+	saverfb5r4 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 4);
+	saverfb7r4 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 4);
+	saverfb5r4 = saverfb5r4 & (~0x40);
+	saverfb7r4 = saverfb7r4 & (~0x40);
+	rt2800_rfcsr_write_dccal(rt2x00dev, 4, 0x64);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, saverfb5r4);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 4, saverfb7r4);
+
+	rt2800_bbp_write(rt2x00dev, 158, 141);
+	bbpreg = rt2800_bbp_read(rt2x00dev, 159);
+	bbpreg = bbpreg & (~0x40);
+	rt2800_bbp_write(rt2x00dev, 159, bbpreg);
+	bbpreg |= 0x48;
+	rt2800_bbp_write(rt2x00dev, 159, bbpreg);
+
+	for (i = 0; i < 10000; i++) {
+		bbpreg = rt2800_bbp_read(rt2x00dev, 159);
+		if ((bbpreg & 0x40) == 0)
+			break;
+		usleep_range(50, 100);
+	}
+
+	bbpreg = rt2800_bbp_read(rt2x00dev, 159);
+	bbpreg = bbpreg & (~0x40);
+	rt2800_bbp_write(rt2x00dev, 159, bbpreg);
+
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, macvalue);
+
+	rt2800_bbp_write(rt2x00dev, 158, 141);
+	bbpreg = rt2800_bbp_read(rt2x00dev, 159);
+	bbpreg &= (~0x10);
+	rt2800_bbp_write(rt2x00dev, 159, bbpreg);
+
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, saverfb0r2);
+}
+
+static u32 rt2800_do_sqrt_accumulation(u32 si)
+{
+	u32 root, root_pre, bit;
+	char i;
+
+	bit = 1 << 15;
+	root = 0;
+	for (i = 15; i >= 0; i = i - 1) {
+		root_pre = root + bit;
+		if ((root_pre * root_pre) <= si)
+			root = root_pre;
+		bit = bit >> 1;
+	}
+
+	return root;
+}
+
+static void rt2800_rxiq_calibration(struct rt2x00_dev *rt2x00dev)
+{
+	u8 rfb0r1, rfb0r2, rfb0r42;
+	u8 rfb4r0, rfb4r19;
+	u8 rfb5r3, rfb5r4, rfb5r17, rfb5r18, rfb5r19, rfb5r20;
+	u8 rfb6r0, rfb6r19;
+	u8 rfb7r3, rfb7r4, rfb7r17, rfb7r18, rfb7r19, rfb7r20;
+
+	u8 bbp1, bbp4;
+	u8 bbpr241, bbpr242;
+	u32 i;
+	u8 ch_idx;
+	u8 bbpval;
+	u8 rfval, vga_idx = 0;
+	int mi = 0, mq = 0, si = 0, sq = 0, riq = 0;
+	int sigma_i, sigma_q, r_iq, g_rx;
+	int g_imb;
+	int ph_rx;
+	u32 savemacsysctrl = 0;
+	u32 orig_RF_CONTROL0 = 0;
+	u32 orig_RF_BYPASS0 = 0;
+	u32 orig_RF_CONTROL1 = 0;
+	u32 orig_RF_BYPASS1 = 0;
+	u32 orig_RF_CONTROL3 = 0;
+	u32 orig_RF_BYPASS3 = 0;
+	u32 bbpval1 = 0;
+	static const u8 rf_vga_table[] = {0x20, 0x21, 0x22, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f};
+
+	savemacsysctrl = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	orig_RF_CONTROL0 = rt2800_register_read(rt2x00dev, RF_CONTROL0);
+	orig_RF_BYPASS0 = rt2800_register_read(rt2x00dev, RF_BYPASS0);
+	orig_RF_CONTROL1 = rt2800_register_read(rt2x00dev, RF_CONTROL1);
+	orig_RF_BYPASS1 = rt2800_register_read(rt2x00dev, RF_BYPASS1);
+	orig_RF_CONTROL3 = rt2800_register_read(rt2x00dev, RF_CONTROL3);
+	orig_RF_BYPASS3 = rt2800_register_read(rt2x00dev, RF_BYPASS3);
+
+	bbp1 = rt2800_bbp_read(rt2x00dev, 1);
+	bbp4 = rt2800_bbp_read(rt2x00dev, 4);
+
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, 0x0);
+
+	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY)))
+		rt2x00_warn(rt2x00dev, "Timeout waiting for MAC status in RXIQ calibration\n");
+
+	bbpval = bbp4 & (~0x18);
+	bbpval = bbp4 | 0x00;
+	rt2800_bbp_write(rt2x00dev, 4, bbpval);
+
+	bbpval = rt2800_bbp_read(rt2x00dev, 21);
+	bbpval = bbpval | 1;
+	rt2800_bbp_write(rt2x00dev, 21, bbpval);
+	bbpval = bbpval & 0xfe;
+	rt2800_bbp_write(rt2x00dev, 21, bbpval);
+
+	rt2800_register_write(rt2x00dev, RF_CONTROL1, 0x00000202);
+	rt2800_register_write(rt2x00dev, RF_BYPASS1, 0x00000303);
+	if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags))
+		rt2800_register_write(rt2x00dev, RF_CONTROL3, 0x0101);
+	else
+		rt2800_register_write(rt2x00dev, RF_CONTROL3, 0x0000);
+
+	rt2800_register_write(rt2x00dev, RF_BYPASS3, 0xf1f1);
+
+	rfb0r1 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 1);
+	rfb0r2 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 2);
+	rfb0r42 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 42);
+	rfb4r0 = rt2800_rfcsr_read_bank(rt2x00dev, 4, 0);
+	rfb4r19 = rt2800_rfcsr_read_bank(rt2x00dev, 4, 19);
+	rfb5r3 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 3);
+	rfb5r4 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 4);
+	rfb5r17 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 17);
+	rfb5r18 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 18);
+	rfb5r19 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 19);
+	rfb5r20 = rt2800_rfcsr_read_bank(rt2x00dev, 5, 20);
+
+	rfb6r0 = rt2800_rfcsr_read_bank(rt2x00dev, 6, 0);
+	rfb6r19 = rt2800_rfcsr_read_bank(rt2x00dev, 6, 19);
+	rfb7r3 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 3);
+	rfb7r4 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 4);
+	rfb7r17 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 17);
+	rfb7r18 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 18);
+	rfb7r19 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 19);
+	rfb7r20 = rt2800_rfcsr_read_bank(rt2x00dev, 7, 20);
+
+	rt2800_rfcsr_write_chanreg(rt2x00dev, 0, 0x87);
+	rt2800_rfcsr_write_chanreg(rt2x00dev, 19, 0x27);
+	rt2800_rfcsr_write_dccal(rt2x00dev, 3, 0x38);
+	rt2800_rfcsr_write_dccal(rt2x00dev, 4, 0x38);
+	rt2800_rfcsr_write_dccal(rt2x00dev, 17, 0x80);
+	rt2800_rfcsr_write_dccal(rt2x00dev, 18, 0xC1);
+	rt2800_rfcsr_write_dccal(rt2x00dev, 19, 0x60);
+	rt2800_rfcsr_write_dccal(rt2x00dev, 20, 0x00);
+
+	rt2800_bbp_write(rt2x00dev, 23, 0x0);
+	rt2800_bbp_write(rt2x00dev, 24, 0x0);
+
+	rt2800_bbp_dcoc_write(rt2x00dev, 5, 0x0);
+
+	bbpr241 = rt2800_bbp_read(rt2x00dev, 241);
+	bbpr242 = rt2800_bbp_read(rt2x00dev, 242);
+
+	rt2800_bbp_write(rt2x00dev, 241, 0x10);
+	rt2800_bbp_write(rt2x00dev, 242, 0x84);
+	rt2800_bbp_write(rt2x00dev, 244, 0x31);
+
+	bbpval = rt2800_bbp_dcoc_read(rt2x00dev, 3);
+	bbpval = bbpval & (~0x7);
+	rt2800_bbp_dcoc_write(rt2x00dev, 3, bbpval);
+
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000004);
+	udelay(1);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000006);
+	usleep_range(1, 200);
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x00003376);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00001006);
+	udelay(1);
+	if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) {
+		rt2800_bbp_write(rt2x00dev, 23, 0x06);
+		rt2800_bbp_write(rt2x00dev, 24, 0x06);
+	} else {
+		rt2800_bbp_write(rt2x00dev, 23, 0x02);
+		rt2800_bbp_write(rt2x00dev, 24, 0x02);
+	}
+
+	for (ch_idx = 0; ch_idx < 2; ch_idx = ch_idx + 1) {
+		if (ch_idx == 0) {
+			rfval = rfb0r1 & (~0x3);
+			rfval = rfb0r1 | 0x1;
+			rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, rfval);
+			rfval = rfb0r2 & (~0x33);
+			rfval = rfb0r2 | 0x11;
+			rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, rfval);
+			rfval = rfb0r42 & (~0x50);
+			rfval = rfb0r42 | 0x10;
+			rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, rfval);
+
+			rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00001006);
+			udelay(1);
+
+			bbpval = bbp1 & (~0x18);
+			bbpval = bbpval | 0x00;
+			rt2800_bbp_write(rt2x00dev, 1, bbpval);
+
+			rt2800_bbp_dcoc_write(rt2x00dev, 1, 0x00);
+		} else {
+			rfval = rfb0r1 & (~0x3);
+			rfval = rfb0r1 | 0x2;
+			rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, rfval);
+			rfval = rfb0r2 & (~0x33);
+			rfval = rfb0r2 | 0x22;
+			rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, rfval);
+			rfval = rfb0r42 & (~0x50);
+			rfval = rfb0r42 | 0x40;
+			rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, rfval);
+
+			rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00002006);
+			udelay(1);
+
+			bbpval = bbp1 & (~0x18);
+			bbpval = bbpval | 0x08;
+			rt2800_bbp_write(rt2x00dev, 1, bbpval);
+
+			rt2800_bbp_dcoc_write(rt2x00dev, 1, 0x01);
+		}
+		usleep_range(500, 1500);
+
+		vga_idx = 0;
+		while (vga_idx < 11) {
+			rt2800_rfcsr_write_dccal(rt2x00dev, 3, rf_vga_table[vga_idx]);
+			rt2800_rfcsr_write_dccal(rt2x00dev, 4, rf_vga_table[vga_idx]);
+
+			rt2800_bbp_dcoc_write(rt2x00dev, 0, 0x93);
+
+			for (i = 0; i < 10000; i++) {
+				bbpval = rt2800_bbp_read(rt2x00dev, 159);
+				if ((bbpval & 0xff) == 0x93)
+					usleep_range(50, 100);
+				else
+					break;
+				}
+
+			if ((bbpval & 0xff) == 0x93) {
+				rt2x00_warn(rt2x00dev, "Fatal Error: Calibration doesn't finish");
+				goto restore_value;
+			}
+			for (i = 0; i < 5; i++) {
+				u32 bbptemp = 0;
+				u8 value = 0;
+				int result = 0;
+
+				rt2800_bbp_write(rt2x00dev, 158, 0x1e);
+				rt2800_bbp_write(rt2x00dev, 159, i);
+				rt2800_bbp_write(rt2x00dev, 158, 0x22);
+				value = rt2800_bbp_read(rt2x00dev, 159);
+				bbptemp = bbptemp + (value << 24);
+				rt2800_bbp_write(rt2x00dev, 158, 0x21);
+				value = rt2800_bbp_read(rt2x00dev, 159);
+				bbptemp = bbptemp + (value << 16);
+				rt2800_bbp_write(rt2x00dev, 158, 0x20);
+				value = rt2800_bbp_read(rt2x00dev, 159);
+				bbptemp = bbptemp + (value << 8);
+				rt2800_bbp_write(rt2x00dev, 158, 0x1f);
+				value = rt2800_bbp_read(rt2x00dev, 159);
+				bbptemp = bbptemp + value;
+
+				if (i < 2 && (bbptemp & 0x800000))
+					result = (bbptemp & 0xffffff) - 0x1000000;
+				else if (i == 4)
+					result = bbptemp;
+				else
+					result = bbptemp;
+
+				if (i == 0)
+					mi = result / 4096;
+				else if (i == 1)
+					mq = result / 4096;
+				else if (i == 2)
+					si = bbptemp / 4096;
+				else if (i == 3)
+					sq = bbptemp / 4096;
+				else
+					riq = result / 4096;
+			}
+
+			bbpval1 = si - mi * mi;
+			rt2x00_dbg(rt2x00dev,
+				   "RXIQ si=%d, sq=%d, riq=%d, bbpval %d, vga_idx %d",
+				   si, sq, riq, bbpval1, vga_idx);
+
+			if (bbpval1 >= (100 * 100))
+				break;
+
+			if (bbpval1 <= 100)
+				vga_idx = vga_idx + 9;
+			else if (bbpval1 <= 158)
+				vga_idx = vga_idx + 8;
+			else if (bbpval1 <= 251)
+				vga_idx = vga_idx + 7;
+			else if (bbpval1 <= 398)
+				vga_idx = vga_idx + 6;
+			else if (bbpval1 <= 630)
+				vga_idx = vga_idx + 5;
+			else if (bbpval1 <= 1000)
+				vga_idx = vga_idx + 4;
+			else if (bbpval1 <= 1584)
+				vga_idx = vga_idx + 3;
+			else if (bbpval1 <= 2511)
+				vga_idx = vga_idx + 2;
+			else
+				vga_idx = vga_idx + 1;
+		}
+
+		sigma_i = rt2800_do_sqrt_accumulation(100 * (si - mi * mi));
+		sigma_q = rt2800_do_sqrt_accumulation(100 * (sq - mq * mq));
+		r_iq = 10 * (riq - (mi * mq));
+
+		rt2x00_dbg(rt2x00dev, "Sigma_i=%d, Sigma_q=%d, R_iq=%d", sigma_i, sigma_q, r_iq);
+
+		if (sigma_i <= 1400 && sigma_i >= 1000 &&
+		    (sigma_i - sigma_q) <= 112 &&
+		    (sigma_i - sigma_q) >= -112 &&
+		    mi <= 32 && mi >= -32 &&
+		    mq <= 32 && mq >= -32) {
+			r_iq = 10 * (riq - (mi * mq));
+			rt2x00_dbg(rt2x00dev, "RXIQ Sigma_i=%d, Sigma_q=%d, R_iq=%d\n",
+				   sigma_i, sigma_q, r_iq);
+
+			g_rx = (1000 * sigma_q) / sigma_i;
+			g_imb = ((-2) * 128 * (1000 - g_rx)) / (1000 + g_rx);
+			ph_rx = (r_iq * 2292) / (sigma_i * sigma_q);
+
+			if (ph_rx > 20 || ph_rx < -20) {
+				ph_rx = 0;
+				rt2x00_warn(rt2x00dev, "RXIQ calibration FAIL");
+			}
+
+			if (g_imb > 12 || g_imb < -12) {
+				g_imb = 0;
+				rt2x00_warn(rt2x00dev, "RXIQ calibration FAIL");
+			}
+		} else {
+			g_imb = 0;
+			ph_rx = 0;
+			rt2x00_dbg(rt2x00dev, "RXIQ Sigma_i=%d, Sigma_q=%d, R_iq=%d\n",
+				   sigma_i, sigma_q, r_iq);
+			rt2x00_warn(rt2x00dev, "RXIQ calibration FAIL");
+		}
+
+		if (ch_idx == 0) {
+			rt2800_bbp_write(rt2x00dev, 158, 0x37);
+			rt2800_bbp_write(rt2x00dev, 159, g_imb & 0x3f);
+			rt2800_bbp_write(rt2x00dev, 158, 0x35);
+			rt2800_bbp_write(rt2x00dev, 159, ph_rx & 0x3f);
+		} else {
+			rt2800_bbp_write(rt2x00dev, 158, 0x55);
+			rt2800_bbp_write(rt2x00dev, 159, g_imb & 0x3f);
+			rt2800_bbp_write(rt2x00dev, 158, 0x53);
+			rt2800_bbp_write(rt2x00dev, 159, ph_rx & 0x3f);
+		}
+	}
+
+restore_value:
+	rt2800_bbp_write(rt2x00dev, 158, 0x3);
+	bbpval = rt2800_bbp_read(rt2x00dev, 159);
+	rt2800_bbp_write(rt2x00dev, 159, (bbpval | 0x07));
+
+	rt2800_bbp_write(rt2x00dev, 158, 0x00);
+	rt2800_bbp_write(rt2x00dev, 159, 0x00);
+	rt2800_bbp_write(rt2x00dev, 1, bbp1);
+	rt2800_bbp_write(rt2x00dev, 4, bbp4);
+	rt2800_bbp_write(rt2x00dev, 241, bbpr241);
+	rt2800_bbp_write(rt2x00dev, 242, bbpr242);
+
+	rt2800_bbp_write(rt2x00dev, 244, 0x00);
+	bbpval = rt2800_bbp_read(rt2x00dev, 21);
+	bbpval |= 0x1;
+	rt2800_bbp_write(rt2x00dev, 21, bbpval);
+	usleep_range(10, 200);
+	bbpval &= 0xfe;
+	rt2800_bbp_write(rt2x00dev, 21, bbpval);
+
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, rfb0r1);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, rfb0r2);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, rfb0r42);
+
+	rt2800_rfcsr_write_bank(rt2x00dev, 4, 0, rfb4r0);
+	rt2800_rfcsr_write_bank(rt2x00dev, 4, 19, rfb4r19);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 3, rfb5r3);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, rfb5r4);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 17, rfb5r17);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 18, rfb5r18);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 19, rfb5r19);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 20, rfb5r20);
+
+	rt2800_rfcsr_write_bank(rt2x00dev, 6, 0, rfb6r0);
+	rt2800_rfcsr_write_bank(rt2x00dev, 6, 19, rfb6r19);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 3, rfb7r3);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 4, rfb7r4);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 17, rfb7r17);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 18, rfb7r18);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 19, rfb7r19);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 20, rfb7r20);
+
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000006);
+	udelay(1);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000004);
+	udelay(1);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, orig_RF_CONTROL0);
+	udelay(1);
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, orig_RF_BYPASS0);
+	rt2800_register_write(rt2x00dev, RF_CONTROL1, orig_RF_CONTROL1);
+	rt2800_register_write(rt2x00dev, RF_BYPASS1, orig_RF_BYPASS1);
+	rt2800_register_write(rt2x00dev, RF_CONTROL3, orig_RF_CONTROL3);
+	rt2800_register_write(rt2x00dev, RF_BYPASS3, orig_RF_BYPASS3);
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, savemacsysctrl);
+}
+
+static void rt2800_rf_configstore(struct rt2x00_dev *rt2x00dev,
+				  struct rf_reg_pair rf_reg_record[][13], u8 chain)
+{
+	u8 rfvalue = 0;
+
+	if (chain == CHAIN_0) {
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 1);
+		rf_reg_record[CHAIN_0][0].bank = 0;
+		rf_reg_record[CHAIN_0][0].reg = 1;
+		rf_reg_record[CHAIN_0][0].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 2);
+		rf_reg_record[CHAIN_0][1].bank = 0;
+		rf_reg_record[CHAIN_0][1].reg = 2;
+		rf_reg_record[CHAIN_0][1].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 35);
+		rf_reg_record[CHAIN_0][2].bank = 0;
+		rf_reg_record[CHAIN_0][2].reg = 35;
+		rf_reg_record[CHAIN_0][2].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 42);
+		rf_reg_record[CHAIN_0][3].bank = 0;
+		rf_reg_record[CHAIN_0][3].reg = 42;
+		rf_reg_record[CHAIN_0][3].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 4, 0);
+		rf_reg_record[CHAIN_0][4].bank = 4;
+		rf_reg_record[CHAIN_0][4].reg = 0;
+		rf_reg_record[CHAIN_0][4].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 4, 2);
+		rf_reg_record[CHAIN_0][5].bank = 4;
+		rf_reg_record[CHAIN_0][5].reg = 2;
+		rf_reg_record[CHAIN_0][5].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 4, 34);
+		rf_reg_record[CHAIN_0][6].bank = 4;
+		rf_reg_record[CHAIN_0][6].reg = 34;
+		rf_reg_record[CHAIN_0][6].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 3);
+		rf_reg_record[CHAIN_0][7].bank = 5;
+		rf_reg_record[CHAIN_0][7].reg = 3;
+		rf_reg_record[CHAIN_0][7].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 4);
+		rf_reg_record[CHAIN_0][8].bank = 5;
+		rf_reg_record[CHAIN_0][8].reg = 4;
+		rf_reg_record[CHAIN_0][8].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 17);
+		rf_reg_record[CHAIN_0][9].bank = 5;
+		rf_reg_record[CHAIN_0][9].reg = 17;
+		rf_reg_record[CHAIN_0][9].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 18);
+		rf_reg_record[CHAIN_0][10].bank = 5;
+		rf_reg_record[CHAIN_0][10].reg = 18;
+		rf_reg_record[CHAIN_0][10].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 19);
+		rf_reg_record[CHAIN_0][11].bank = 5;
+		rf_reg_record[CHAIN_0][11].reg = 19;
+		rf_reg_record[CHAIN_0][11].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 5, 20);
+		rf_reg_record[CHAIN_0][12].bank = 5;
+		rf_reg_record[CHAIN_0][12].reg = 20;
+		rf_reg_record[CHAIN_0][12].value = rfvalue;
+	} else if (chain == CHAIN_1) {
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 1);
+		rf_reg_record[CHAIN_1][0].bank = 0;
+		rf_reg_record[CHAIN_1][0].reg = 1;
+		rf_reg_record[CHAIN_1][0].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 2);
+		rf_reg_record[CHAIN_1][1].bank = 0;
+		rf_reg_record[CHAIN_1][1].reg = 2;
+		rf_reg_record[CHAIN_1][1].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 35);
+		rf_reg_record[CHAIN_1][2].bank = 0;
+		rf_reg_record[CHAIN_1][2].reg = 35;
+		rf_reg_record[CHAIN_1][2].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 0, 42);
+		rf_reg_record[CHAIN_1][3].bank = 0;
+		rf_reg_record[CHAIN_1][3].reg = 42;
+		rf_reg_record[CHAIN_1][3].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 6, 0);
+		rf_reg_record[CHAIN_1][4].bank = 6;
+		rf_reg_record[CHAIN_1][4].reg = 0;
+		rf_reg_record[CHAIN_1][4].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 6, 2);
+		rf_reg_record[CHAIN_1][5].bank = 6;
+		rf_reg_record[CHAIN_1][5].reg = 2;
+		rf_reg_record[CHAIN_1][5].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 6, 34);
+		rf_reg_record[CHAIN_1][6].bank = 6;
+		rf_reg_record[CHAIN_1][6].reg = 34;
+		rf_reg_record[CHAIN_1][6].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 3);
+		rf_reg_record[CHAIN_1][7].bank = 7;
+		rf_reg_record[CHAIN_1][7].reg = 3;
+		rf_reg_record[CHAIN_1][7].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 4);
+		rf_reg_record[CHAIN_1][8].bank = 7;
+		rf_reg_record[CHAIN_1][8].reg = 4;
+		rf_reg_record[CHAIN_1][8].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 17);
+		rf_reg_record[CHAIN_1][9].bank = 7;
+		rf_reg_record[CHAIN_1][9].reg = 17;
+		rf_reg_record[CHAIN_1][9].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 18);
+		rf_reg_record[CHAIN_1][10].bank = 7;
+		rf_reg_record[CHAIN_1][10].reg = 18;
+		rf_reg_record[CHAIN_1][10].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 19);
+		rf_reg_record[CHAIN_1][11].bank = 7;
+		rf_reg_record[CHAIN_1][11].reg = 19;
+		rf_reg_record[CHAIN_1][11].value = rfvalue;
+		rfvalue = rt2800_rfcsr_read_bank(rt2x00dev, 7, 20);
+		rf_reg_record[CHAIN_1][12].bank = 7;
+		rf_reg_record[CHAIN_1][12].reg = 20;
+		rf_reg_record[CHAIN_1][12].value = rfvalue;
+	} else {
+		rt2x00_warn(rt2x00dev, "Unknown chain = %u\n", chain);
+	}
+}
+
+static void rt2800_rf_configrecover(struct rt2x00_dev *rt2x00dev,
+				    struct rf_reg_pair rf_record[][13])
+{
+	u8 chain_index = 0, record_index = 0;
+	u8 bank = 0, rf_register = 0, value = 0;
+
+	for (chain_index = 0; chain_index < 2; chain_index++) {
+		for (record_index = 0; record_index < 13; record_index++) {
+			bank = rf_record[chain_index][record_index].bank;
+			rf_register = rf_record[chain_index][record_index].reg;
+			value = rf_record[chain_index][record_index].value;
+			rt2800_rfcsr_write_bank(rt2x00dev, bank, rf_register, value);
+			rt2x00_dbg(rt2x00dev, "bank: %d, rf_register: %d, value: %x\n",
+				   bank, rf_register, value);
+		}
+	}
+}
+
+static void rt2800_setbbptonegenerator(struct rt2x00_dev *rt2x00dev)
+{
+	rt2800_bbp_write(rt2x00dev, 158, 0xAA);
+	rt2800_bbp_write(rt2x00dev, 159, 0x00);
+
+	rt2800_bbp_write(rt2x00dev, 158, 0xAB);
+	rt2800_bbp_write(rt2x00dev, 159, 0x0A);
+
+	rt2800_bbp_write(rt2x00dev, 158, 0xAC);
+	rt2800_bbp_write(rt2x00dev, 159, 0x3F);
+
+	rt2800_bbp_write(rt2x00dev, 158, 0xAD);
+	rt2800_bbp_write(rt2x00dev, 159, 0x3F);
+
+	rt2800_bbp_write(rt2x00dev, 244, 0x40);
+}
+
+static u32 rt2800_do_fft_accumulation(struct rt2x00_dev *rt2x00dev, u8 tidx, u8 read_neg)
+{
+	u32 macvalue = 0;
+	int fftout_i = 0, fftout_q = 0;
+	u32 ptmp = 0, pint = 0;
+	u8 bbp = 0;
+	u8 tidxi;
+
+	rt2800_bbp_write(rt2x00dev, 158, 0x00);
+	rt2800_bbp_write(rt2x00dev, 159, 0x9b);
+
+	bbp = 0x9b;
+
+	while (bbp == 0x9b) {
+		usleep_range(10, 50);
+		bbp = rt2800_bbp_read(rt2x00dev, 159);
+		bbp = bbp & 0xff;
+	}
+
+	rt2800_bbp_write(rt2x00dev, 158, 0xba);
+	rt2800_bbp_write(rt2x00dev, 159, tidx);
+	rt2800_bbp_write(rt2x00dev, 159, tidx);
+	rt2800_bbp_write(rt2x00dev, 159, tidx);
+
+	macvalue = rt2800_register_read(rt2x00dev, 0x057C);
+
+	fftout_i = (macvalue >> 16);
+	fftout_i = (fftout_i & 0x8000) ? (fftout_i - 0x10000) : fftout_i;
+	fftout_q = (macvalue & 0xffff);
+	fftout_q = (fftout_q & 0x8000) ? (fftout_q - 0x10000) : fftout_q;
+	ptmp = (fftout_i * fftout_i);
+	ptmp = ptmp + (fftout_q * fftout_q);
+	pint = ptmp;
+	rt2x00_dbg(rt2x00dev, "I = %d,  Q = %d, power = %x\n", fftout_i, fftout_q, pint);
+	if (read_neg) {
+		pint = pint >> 1;
+		tidxi = 0x40 - tidx;
+		tidxi = tidxi & 0x3f;
+
+		rt2800_bbp_write(rt2x00dev, 158, 0xba);
+		rt2800_bbp_write(rt2x00dev, 159, tidxi);
+		rt2800_bbp_write(rt2x00dev, 159, tidxi);
+		rt2800_bbp_write(rt2x00dev, 159, tidxi);
+
+		macvalue = rt2800_register_read(rt2x00dev, 0x057C);
+
+		fftout_i = (macvalue >> 16);
+		fftout_i = (fftout_i & 0x8000) ? (fftout_i - 0x10000) : fftout_i;
+		fftout_q = (macvalue & 0xffff);
+		fftout_q = (fftout_q & 0x8000) ? (fftout_q - 0x10000) : fftout_q;
+		ptmp = (fftout_i * fftout_i);
+		ptmp = ptmp + (fftout_q * fftout_q);
+		ptmp = ptmp >> 1;
+		pint = pint + ptmp;
+	}
+
+	return pint;
+}
+
+static u32 rt2800_read_fft_accumulation(struct rt2x00_dev *rt2x00dev, u8 tidx)
+{
+	u32 macvalue = 0;
+	int fftout_i = 0, fftout_q = 0;
+	u32 ptmp = 0, pint = 0;
+
+	rt2800_bbp_write(rt2x00dev, 158, 0xBA);
+	rt2800_bbp_write(rt2x00dev, 159, tidx);
+	rt2800_bbp_write(rt2x00dev, 159, tidx);
+	rt2800_bbp_write(rt2x00dev, 159, tidx);
+
+	macvalue = rt2800_register_read(rt2x00dev, 0x057C);
+
+	fftout_i = (macvalue >> 16);
+	fftout_i = (fftout_i & 0x8000) ? (fftout_i - 0x10000) : fftout_i;
+	fftout_q = (macvalue & 0xffff);
+	fftout_q = (fftout_q & 0x8000) ? (fftout_q - 0x10000) : fftout_q;
+	ptmp = (fftout_i * fftout_i);
+	ptmp = ptmp + (fftout_q * fftout_q);
+	pint = ptmp;
+
+	return pint;
+}
+
+static void rt2800_write_dc(struct rt2x00_dev *rt2x00dev, u8 ch_idx, u8 alc, u8 iorq, u8 dc)
+{
+	u8 bbp = 0;
+
+	rt2800_bbp_write(rt2x00dev, 158, 0xb0);
+	bbp = alc | 0x80;
+	rt2800_bbp_write(rt2x00dev, 159, bbp);
+
+	if (ch_idx == 0)
+		bbp = (iorq == 0) ? 0xb1 : 0xb2;
+	else
+		bbp = (iorq == 0) ? 0xb8 : 0xb9;
+
+	rt2800_bbp_write(rt2x00dev, 158, bbp);
+	bbp = dc;
+	rt2800_bbp_write(rt2x00dev, 159, bbp);
+}
+
+static void rt2800_loft_search(struct rt2x00_dev *rt2x00dev, u8 ch_idx,
+			       u8 alc_idx, u8 dc_result[][RF_ALC_NUM][2])
+{
+	u32 p0 = 0, p1 = 0, pf = 0;
+	char idx0 = 0, idx1 = 0;
+	u8 idxf[] = {0x00, 0x00};
+	u8 ibit = 0x20;
+	u8 iorq;
+	char bidx;
+
+	rt2800_bbp_write(rt2x00dev, 158, 0xb0);
+	rt2800_bbp_write(rt2x00dev, 159, 0x80);
+
+	for (bidx = 5; bidx >= 0; bidx--) {
+		for (iorq = 0; iorq <= 1; iorq++) {
+			if (idxf[iorq] == 0x20) {
+				idx0 = 0x20;
+				p0 = pf;
+			} else {
+				idx0 = idxf[iorq] - ibit;
+				idx0 = idx0 & 0x3F;
+				rt2800_write_dc(rt2x00dev, ch_idx, 0, iorq, idx0);
+				p0 = rt2800_do_fft_accumulation(rt2x00dev, 0x0A, 0);
+			}
+
+			idx1 = idxf[iorq] + (bidx == 5 ? 0 : ibit);
+			idx1 = idx1 & 0x3F;
+			rt2800_write_dc(rt2x00dev, ch_idx, 0, iorq, idx1);
+			p1 = rt2800_do_fft_accumulation(rt2x00dev, 0x0A, 0);
+
+			rt2x00_dbg(rt2x00dev, "alc=%u, IorQ=%u, idx_final=%2x\n",
+				   alc_idx, iorq, idxf[iorq]);
+			rt2x00_dbg(rt2x00dev, "p0=%x, p1=%x, pf=%x, idx_0=%x, idx_1=%x, ibit=%x\n",
+				   p0, p1, pf, idx0, idx1, ibit);
+
+			if (bidx != 5 && pf <= p0 && pf < p1) {
+				idxf[iorq] = idxf[iorq];
+			} else if (p0 < p1) {
+				pf = p0;
+				idxf[iorq] = idx0 & 0x3F;
+			} else {
+				pf = p1;
+				idxf[iorq] = idx1 & 0x3F;
+			}
+			rt2x00_dbg(rt2x00dev, "IorQ=%u, idx_final[%u]:%x, pf:%8x\n",
+				   iorq, iorq, idxf[iorq], pf);
+
+			rt2800_write_dc(rt2x00dev, ch_idx, 0, iorq, idxf[iorq]);
+		}
+		ibit = ibit >> 1;
+	}
+	dc_result[ch_idx][alc_idx][0] = idxf[0];
+	dc_result[ch_idx][alc_idx][1] = idxf[1];
+}
+
+static void rt2800_iq_search(struct rt2x00_dev *rt2x00dev, u8 ch_idx, u8 *ges, u8 *pes)
+{
+	u32 p0 = 0, p1 = 0, pf = 0;
+	char perr = 0, gerr = 0, iq_err = 0;
+	char pef = 0, gef = 0;
+	char psta, pend;
+	char gsta, gend;
+
+	u8 ibit = 0x20;
+	u8 first_search = 0x00, touch_neg_max = 0x00;
+	char idx0 = 0, idx1 = 0;
+	u8 gop;
+	u8 bbp = 0;
+	char bidx;
+
+	for (bidx = 5; bidx >= 1; bidx--) {
+		for (gop = 0; gop < 2; gop++) {
+			if (gop == 1 || bidx < 4) {
+				if (gop == 0)
+					iq_err = gerr;
+				else
+					iq_err = perr;
+
+				first_search = (gop == 0) ? (bidx == 3) : (bidx == 5);
+				touch_neg_max = (gop) ? ((iq_err & 0x0F) == 0x08) :
+							((iq_err & 0x3F) == 0x20);
+
+				if (touch_neg_max) {
+					p0 = pf;
+					idx0 = iq_err;
+				} else {
+					idx0 = iq_err - ibit;
+					bbp = (ch_idx == 0) ? ((gop == 0) ? 0x28 : 0x29) :
+							      ((gop == 0) ? 0x46 : 0x47);
+
+					rt2800_bbp_write(rt2x00dev, 158, bbp);
+					rt2800_bbp_write(rt2x00dev, 159, idx0);
+
+					p0 = rt2800_do_fft_accumulation(rt2x00dev, 0x14, 1);
+				}
+
+				idx1 = iq_err + (first_search ? 0 : ibit);
+				idx1 = (gop == 0) ? (idx1 & 0x0F) : (idx1 & 0x3F);
+
+				bbp = (ch_idx == 0) ? (gop == 0) ? 0x28 : 0x29 :
+				      (gop == 0) ? 0x46 : 0x47;
+
+				rt2800_bbp_write(rt2x00dev, 158, bbp);
+				rt2800_bbp_write(rt2x00dev, 159, idx1);
+
+				p1 = rt2800_do_fft_accumulation(rt2x00dev, 0x14, 1);
+
+				rt2x00_dbg(rt2x00dev,
+					   "p0=%x, p1=%x, pwer_final=%x, idx0=%x, idx1=%x, iq_err=%x, gop=%d, ibit=%x\n",
+					   p0, p1, pf, idx0, idx1, iq_err, gop, ibit);
+
+				if (!(!first_search && pf <= p0 && pf < p1)) {
+					if (p0 < p1) {
+						pf = p0;
+						iq_err = idx0;
+					} else {
+						pf = p1;
+						iq_err = idx1;
+					}
+				}
+
+				bbp = (ch_idx == 0) ? (gop == 0) ? 0x28 : 0x29 :
+						      (gop == 0) ? 0x46 : 0x47;
+
+				rt2800_bbp_write(rt2x00dev, 158, bbp);
+				rt2800_bbp_write(rt2x00dev, 159, iq_err);
+
+				if (gop == 0)
+					gerr = iq_err;
+				else
+					perr = iq_err;
+
+				rt2x00_dbg(rt2x00dev, "IQCalibration pf=%8x (%2x, %2x) !\n",
+					   pf, gerr & 0x0F, perr & 0x3F);
+			}
+		}
+
+		if (bidx > 0)
+			ibit = (ibit >> 1);
+	}
+	gerr = (gerr & 0x08) ? (gerr & 0x0F) - 0x10 : (gerr & 0x0F);
+	perr = (perr & 0x20) ? (perr & 0x3F) - 0x40 : (perr & 0x3F);
+
+	gerr = (gerr < -0x07) ? -0x07 : (gerr > 0x05) ? 0x05 : gerr;
+	gsta = gerr - 1;
+	gend = gerr + 2;
+
+	perr = (perr < -0x1f) ? -0x1f : (perr > 0x1d) ? 0x1d : perr;
+	psta = perr - 1;
+	pend = perr + 2;
+
+	for (gef = gsta; gef <= gend; gef = gef + 1)
+		for (pef = psta; pef <= pend; pef = pef + 1) {
+			bbp = (ch_idx == 0) ? 0x28 : 0x46;
+			rt2800_bbp_write(rt2x00dev, 158, bbp);
+			rt2800_bbp_write(rt2x00dev, 159, gef & 0x0F);
+
+			bbp = (ch_idx == 0) ? 0x29 : 0x47;
+			rt2800_bbp_write(rt2x00dev, 158, bbp);
+			rt2800_bbp_write(rt2x00dev, 159, pef & 0x3F);
+
+			p1 = rt2800_do_fft_accumulation(rt2x00dev, 0x14, 1);
+			if (gef == gsta && pef == psta) {
+				pf = p1;
+				gerr = gef;
+				perr = pef;
+			} else if (pf > p1) {
+				pf = p1;
+				gerr = gef;
+				perr = pef;
+			}
+			rt2x00_dbg(rt2x00dev, "Fine IQCalibration p1=%8x pf=%8x (%2x, %2x) !\n",
+				   p1, pf, gef & 0x0F, pef & 0x3F);
+		}
+
+	ges[ch_idx] = gerr & 0x0F;
+	pes[ch_idx] = perr & 0x3F;
+}
+
+static void rt2800_rf_aux_tx0_loopback(struct rt2x00_dev *rt2x00dev)
+{
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, 0x21);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, 0x10);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, 0x00);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, 0x1b);
+	rt2800_rfcsr_write_bank(rt2x00dev, 4, 0, 0x81);
+	rt2800_rfcsr_write_bank(rt2x00dev, 4, 2, 0x81);
+	rt2800_rfcsr_write_bank(rt2x00dev, 4, 34, 0xee);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 3, 0x2d);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 4, 0x2d);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 17, 0x80);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 18, 0xd7);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 19, 0xa2);
+	rt2800_rfcsr_write_bank(rt2x00dev, 5, 20, 0x20);
+}
+
+static void rt2800_rf_aux_tx1_loopback(struct rt2x00_dev *rt2x00dev)
+{
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 1, 0x22);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 2, 0x20);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 35, 0x00);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, 0x4b);
+	rt2800_rfcsr_write_bank(rt2x00dev, 6, 0, 0x81);
+	rt2800_rfcsr_write_bank(rt2x00dev, 6, 2, 0x81);
+	rt2800_rfcsr_write_bank(rt2x00dev, 6, 34, 0xee);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 3, 0x2d);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 4, 0x2d);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 17, 0x80);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 18, 0xd7);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 19, 0xa2);
+	rt2800_rfcsr_write_bank(rt2x00dev, 7, 20, 0x20);
+}
+
+static void rt2800_loft_iq_calibration(struct rt2x00_dev *rt2x00dev)
+{
+	struct rf_reg_pair rf_store[CHAIN_NUM][13];
+	u32 macorg1 = 0;
+	u32 macorg2 = 0;
+	u32 macorg3 = 0;
+	u32 macorg4 = 0;
+	u32 macorg5 = 0;
+	u32 orig528 = 0;
+	u32 orig52c = 0;
+
+	u32 savemacsysctrl = 0;
+	u32 macvalue = 0;
+	u32 mac13b8 = 0;
+	u32 p0 = 0, p1 = 0;
+	u32 p0_idx10 = 0, p1_idx10 = 0;
+
+	u8 rfvalue;
+	u8 loft_dc_search_result[CHAIN_NUM][RF_ALC_NUM][2];
+	u8 ger[CHAIN_NUM], per[CHAIN_NUM];
+
+	u8 vga_gain[] = {14, 14};
+	u8 bbp = 0, ch_idx = 0, rf_alc_idx = 0, idx = 0;
+	u8 bbpr30, rfb0r39, rfb0r42;
+	u8 bbpr1;
+	u8 bbpr4;
+	u8 bbpr241, bbpr242;
+	u8 count_step;
+
+	static const u8 rf_gain[] = {0x00, 0x01, 0x02, 0x04, 0x08, 0x0c};
+	static const u8 rfvga_gain_table[] = {0x24, 0x25, 0x26, 0x27, 0x28, 0x2c, 0x2d, 0x2e, 0x2f, 0x30,
+					      0x31, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3F};
+	static const u8 bbp_2324gain[] = {0x16, 0x14, 0x12, 0x10, 0x0c, 0x08};
+
+	savemacsysctrl = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	macorg1 = rt2800_register_read(rt2x00dev, TX_PIN_CFG);
+	macorg2 = rt2800_register_read(rt2x00dev, RF_CONTROL0);
+	macorg3 = rt2800_register_read(rt2x00dev, RF_BYPASS0);
+	macorg4 = rt2800_register_read(rt2x00dev, RF_CONTROL3);
+	macorg5 = rt2800_register_read(rt2x00dev, RF_BYPASS3);
+	mac13b8 = rt2800_register_read(rt2x00dev, 0x13b8);
+	orig528 = rt2800_register_read(rt2x00dev, RF_CONTROL2);
+	orig52c = rt2800_register_read(rt2x00dev, RF_BYPASS2);
+
+	macvalue = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	macvalue &= (~0x04);
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, macvalue);
+
+	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_TX)))
+		rt2x00_warn(rt2x00dev, "RF TX busy in LOFT IQ calibration\n");
+
+	macvalue = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	macvalue &= (~0x08);
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, macvalue);
+
+	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_RX)))
+		rt2x00_warn(rt2x00dev, "RF RX busy in LOFT IQ calibration\n");
+
+	for (ch_idx = 0; ch_idx < 2; ch_idx++)
+		rt2800_rf_configstore(rt2x00dev, rf_store, ch_idx);
+
+	bbpr30 = rt2800_bbp_read(rt2x00dev, 30);
+	rfb0r39 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 39);
+	rfb0r42 = rt2800_rfcsr_read_bank(rt2x00dev, 0, 42);
+
+	rt2800_bbp_write(rt2x00dev, 30, 0x1F);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 39, 0x80);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, 0x5B);
+
+	rt2800_bbp_write(rt2x00dev, 23, 0x00);
+	rt2800_bbp_write(rt2x00dev, 24, 0x00);
+
+	rt2800_setbbptonegenerator(rt2x00dev);
+
+	for (ch_idx = 0; ch_idx < 2; ch_idx++) {
+		rt2800_bbp_write(rt2x00dev, 23, 0x00);
+		rt2800_bbp_write(rt2x00dev, 24, 0x00);
+		rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, 0x00);
+		rt2800_register_write(rt2x00dev, TX_PIN_CFG, 0x0000000F);
+		rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000004);
+		rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x00003306);
+		rt2800_register_write(rt2x00dev, 0x13b8, 0x10);
+		udelay(1);
+
+		if (ch_idx == 0)
+			rt2800_rf_aux_tx0_loopback(rt2x00dev);
+		else
+			rt2800_rf_aux_tx1_loopback(rt2x00dev);
+
+		udelay(1);
+
+		if (ch_idx == 0)
+			rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00001004);
+		else
+			rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00002004);
+
+		rt2800_bbp_write(rt2x00dev, 158, 0x05);
+		rt2800_bbp_write(rt2x00dev, 159, 0x00);
+
+		rt2800_bbp_write(rt2x00dev, 158, 0x01);
+		if (ch_idx == 0)
+			rt2800_bbp_write(rt2x00dev, 159, 0x00);
+		else
+			rt2800_bbp_write(rt2x00dev, 159, 0x01);
+
+		vga_gain[ch_idx] = 18;
+		for (rf_alc_idx = 0; rf_alc_idx < 3; rf_alc_idx++) {
+			rt2800_bbp_write(rt2x00dev, 23, bbp_2324gain[rf_alc_idx]);
+			rt2800_bbp_write(rt2x00dev, 24, bbp_2324gain[rf_alc_idx]);
+
+			macvalue = rt2800_register_read(rt2x00dev, RF_CONTROL3);
+			macvalue &= (~0x0000F1F1);
+			macvalue |= (rf_gain[rf_alc_idx] << 4);
+			macvalue |= (rf_gain[rf_alc_idx] << 12);
+			rt2800_register_write(rt2x00dev, RF_CONTROL3, macvalue);
+			macvalue = (0x0000F1F1);
+			rt2800_register_write(rt2x00dev, RF_BYPASS3, macvalue);
+
+			if (rf_alc_idx == 0) {
+				rt2800_write_dc(rt2x00dev, ch_idx, 0, 1, 0x21);
+				for (; vga_gain[ch_idx] > 0;
+				     vga_gain[ch_idx] = vga_gain[ch_idx] - 2) {
+					rfvalue = rfvga_gain_table[vga_gain[ch_idx]];
+					rt2800_rfcsr_write_dccal(rt2x00dev, 3, rfvalue);
+					rt2800_rfcsr_write_dccal(rt2x00dev, 4, rfvalue);
+					rt2800_write_dc(rt2x00dev, ch_idx, 0, 1, 0x00);
+					rt2800_write_dc(rt2x00dev, ch_idx, 0, 0, 0x00);
+					p0 = rt2800_do_fft_accumulation(rt2x00dev, 0x0A, 0);
+					rt2800_write_dc(rt2x00dev, ch_idx, 0, 0, 0x21);
+					p1 = rt2800_do_fft_accumulation(rt2x00dev, 0x0A, 0);
+					rt2x00_dbg(rt2x00dev, "LOFT AGC %d %d\n", p0, p1);
+					if ((p0 < 7000 * 7000) && (p1 < (7000 * 7000)))
+						break;
+				}
+
+				rt2800_write_dc(rt2x00dev, ch_idx, 0, 0, 0x00);
+				rt2800_write_dc(rt2x00dev, ch_idx, 0, 1, 0x00);
+
+				rt2x00_dbg(rt2x00dev, "Used VGA %d %x\n", vga_gain[ch_idx],
+					   rfvga_gain_table[vga_gain[ch_idx]]);
+
+				if (vga_gain[ch_idx] < 0)
+					vga_gain[ch_idx] = 0;
+			}
+
+			rfvalue = rfvga_gain_table[vga_gain[ch_idx]];
+
+			rt2800_rfcsr_write_dccal(rt2x00dev, 3, rfvalue);
+			rt2800_rfcsr_write_dccal(rt2x00dev, 4, rfvalue);
+
+			rt2800_loft_search(rt2x00dev, ch_idx, rf_alc_idx, loft_dc_search_result);
+		}
+	}
+
+	for (rf_alc_idx = 0; rf_alc_idx < 3; rf_alc_idx++) {
+		for (idx = 0; idx < 4; idx++) {
+			rt2800_bbp_write(rt2x00dev, 158, 0xB0);
+			bbp = (idx << 2) + rf_alc_idx;
+			rt2800_bbp_write(rt2x00dev, 159, bbp);
+			rt2x00_dbg(rt2x00dev, " ALC %2x,", bbp);
+
+			rt2800_bbp_write(rt2x00dev, 158, 0xb1);
+			bbp = loft_dc_search_result[CHAIN_0][rf_alc_idx][0x00];
+			bbp = bbp & 0x3F;
+			rt2800_bbp_write(rt2x00dev, 159, bbp);
+			rt2x00_dbg(rt2x00dev, " I0 %2x,", bbp);
+
+			rt2800_bbp_write(rt2x00dev, 158, 0xb2);
+			bbp = loft_dc_search_result[CHAIN_0][rf_alc_idx][0x01];
+			bbp = bbp & 0x3F;
+			rt2800_bbp_write(rt2x00dev, 159, bbp);
+			rt2x00_dbg(rt2x00dev, " Q0 %2x,", bbp);
+
+			rt2800_bbp_write(rt2x00dev, 158, 0xb8);
+			bbp = loft_dc_search_result[CHAIN_1][rf_alc_idx][0x00];
+			bbp = bbp & 0x3F;
+			rt2800_bbp_write(rt2x00dev, 159, bbp);
+			rt2x00_dbg(rt2x00dev, " I1 %2x,", bbp);
+
+			rt2800_bbp_write(rt2x00dev, 158, 0xb9);
+			bbp = loft_dc_search_result[CHAIN_1][rf_alc_idx][0x01];
+			bbp = bbp & 0x3F;
+			rt2800_bbp_write(rt2x00dev, 159, bbp);
+			rt2x00_dbg(rt2x00dev, " Q1 %2x\n", bbp);
+		}
+	}
+
+	rt2800_bbp_write(rt2x00dev, 23, 0x00);
+	rt2800_bbp_write(rt2x00dev, 24, 0x00);
+
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x04);
+
+	rt2800_bbp_write(rt2x00dev, 158, 0x00);
+	rt2800_bbp_write(rt2x00dev, 159, 0x00);
+
+	bbp = 0x00;
+	rt2800_bbp_write(rt2x00dev, 244, 0x00);
+
+	rt2800_bbp_write(rt2x00dev, 21, 0x01);
+	udelay(1);
+	rt2800_bbp_write(rt2x00dev, 21, 0x00);
+
+	rt2800_rf_configrecover(rt2x00dev, rf_store);
+
+	rt2800_register_write(rt2x00dev, TX_PIN_CFG, macorg1);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x04);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00);
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x00);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, macorg2);
+	udelay(1);
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, macorg3);
+	rt2800_register_write(rt2x00dev, RF_CONTROL3, macorg4);
+	rt2800_register_write(rt2x00dev, RF_BYPASS3, macorg5);
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, savemacsysctrl);
+	rt2800_register_write(rt2x00dev, RF_CONTROL2, orig528);
+	rt2800_register_write(rt2x00dev, RF_BYPASS2, orig52c);
+	rt2800_register_write(rt2x00dev, 0x13b8, mac13b8);
+
+	savemacsysctrl = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	macorg1 = rt2800_register_read(rt2x00dev, TX_PIN_CFG);
+	macorg2 = rt2800_register_read(rt2x00dev, RF_CONTROL0);
+	macorg3 = rt2800_register_read(rt2x00dev, RF_BYPASS0);
+	macorg4 = rt2800_register_read(rt2x00dev, RF_CONTROL3);
+	macorg5 = rt2800_register_read(rt2x00dev, RF_BYPASS3);
+
+	bbpr1 = rt2800_bbp_read(rt2x00dev, 1);
+	bbpr4 = rt2800_bbp_read(rt2x00dev, 4);
+	bbpr241 = rt2800_bbp_read(rt2x00dev, 241);
+	bbpr242 = rt2800_bbp_read(rt2x00dev, 242);
+	mac13b8 = rt2800_register_read(rt2x00dev, 0x13b8);
+
+	macvalue = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	macvalue &= (~0x04);
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, macvalue);
+
+	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_TX)))
+		rt2x00_warn(rt2x00dev, "RF TX busy in LOFT IQ calibration\n");
+
+	macvalue = rt2800_register_read(rt2x00dev, MAC_SYS_CTRL);
+	macvalue &= (~0x08);
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, macvalue);
+
+	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY_RX)))
+		rt2x00_warn(rt2x00dev, "RF RX busy in LOFT IQ calibration\n");
+
+	if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) {
+		rt2800_register_write(rt2x00dev, RF_CONTROL3, 0x00000101);
+		rt2800_register_write(rt2x00dev, RF_BYPASS3, 0x0000F1F1);
+	}
+
+	rt2800_bbp_write(rt2x00dev, 23, 0x00);
+	rt2800_bbp_write(rt2x00dev, 24, 0x00);
+
+	if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) {
+		rt2800_bbp_write(rt2x00dev, 4, bbpr4 & (~0x18));
+		rt2800_bbp_write(rt2x00dev, 21, 0x01);
+		udelay(1);
+		rt2800_bbp_write(rt2x00dev, 21, 0x00);
+
+		rt2800_bbp_write(rt2x00dev, 241, 0x14);
+		rt2800_bbp_write(rt2x00dev, 242, 0x80);
+		rt2800_bbp_write(rt2x00dev, 244, 0x31);
+	} else {
+		rt2800_setbbptonegenerator(rt2x00dev);
+	}
+
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00000004);
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x00003306);
+	udelay(1);
+
+	rt2800_register_write(rt2x00dev, TX_PIN_CFG, 0x0000000F);
+
+	if (!test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) {
+		rt2800_register_write(rt2x00dev, RF_CONTROL3, 0x00000000);
+		rt2800_register_write(rt2x00dev, RF_BYPASS3, 0x0000F1F1);
+	}
+
+	rt2800_register_write(rt2x00dev, 0x13b8, 0x00000010);
+
+	for (ch_idx = 0; ch_idx < 2; ch_idx++)
+		rt2800_rf_configstore(rt2x00dev, rf_store, ch_idx);
+
+	rt2800_rfcsr_write_dccal(rt2x00dev, 3, 0x3B);
+	rt2800_rfcsr_write_dccal(rt2x00dev, 4, 0x3B);
+
+	rt2800_bbp_write(rt2x00dev, 158, 0x03);
+	rt2800_bbp_write(rt2x00dev, 159, 0x60);
+	rt2800_bbp_write(rt2x00dev, 158, 0xB0);
+	rt2800_bbp_write(rt2x00dev, 159, 0x80);
+
+	for (ch_idx = 0; ch_idx < 2; ch_idx++) {
+		rt2800_bbp_write(rt2x00dev, 23, 0x00);
+		rt2800_bbp_write(rt2x00dev, 24, 0x00);
+
+		if (ch_idx == 0) {
+			rt2800_bbp_write(rt2x00dev, 158, 0x01);
+			rt2800_bbp_write(rt2x00dev, 159, 0x00);
+			if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) {
+				bbp = bbpr1 & (~0x18);
+				bbp = bbp | 0x00;
+				rt2800_bbp_write(rt2x00dev, 1, bbp);
+			}
+			rt2800_rf_aux_tx0_loopback(rt2x00dev);
+			rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00001004);
+		} else {
+			rt2800_bbp_write(rt2x00dev, 158, 0x01);
+			rt2800_bbp_write(rt2x00dev, 159, 0x01);
+			if (test_bit(CAPABILITY_EXTERNAL_PA_TX1, &rt2x00dev->cap_flags)) {
+				bbp = bbpr1 & (~0x18);
+				bbp = bbp | 0x08;
+				rt2800_bbp_write(rt2x00dev, 1, bbp);
+			}
+			rt2800_rf_aux_tx1_loopback(rt2x00dev);
+			rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00002004);
+		}
+
+		rt2800_bbp_write(rt2x00dev, 158, 0x05);
+		rt2800_bbp_write(rt2x00dev, 159, 0x04);
+
+		bbp = (ch_idx == 0) ? 0x28 : 0x46;
+		rt2800_bbp_write(rt2x00dev, 158, bbp);
+		rt2800_bbp_write(rt2x00dev, 159, 0x00);
+
+		if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) {
+			rt2800_bbp_write(rt2x00dev, 23, 0x06);
+			rt2800_bbp_write(rt2x00dev, 24, 0x06);
+			count_step = 1;
+		} else {
+			rt2800_bbp_write(rt2x00dev, 23, 0x1F);
+			rt2800_bbp_write(rt2x00dev, 24, 0x1F);
+			count_step = 2;
+		}
+
+		for (; vga_gain[ch_idx] < 19; vga_gain[ch_idx] = (vga_gain[ch_idx] + count_step)) {
+			rfvalue = rfvga_gain_table[vga_gain[ch_idx]];
+			rt2800_rfcsr_write_dccal(rt2x00dev, 3, rfvalue);
+			rt2800_rfcsr_write_dccal(rt2x00dev, 4, rfvalue);
+
+			bbp = (ch_idx == 0) ? 0x29 : 0x47;
+			rt2800_bbp_write(rt2x00dev, 158, bbp);
+			rt2800_bbp_write(rt2x00dev, 159, 0x00);
+			p0 = rt2800_do_fft_accumulation(rt2x00dev, 0x14, 0);
+			if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags))
+				p0_idx10 = rt2800_read_fft_accumulation(rt2x00dev, 0x0A);
+
+			bbp = (ch_idx == 0) ? 0x29 : 0x47;
+			rt2800_bbp_write(rt2x00dev, 158, bbp);
+			rt2800_bbp_write(rt2x00dev, 159, 0x21);
+			p1 = rt2800_do_fft_accumulation(rt2x00dev, 0x14, 0);
+			if (test_bit(CAPABILITY_EXTERNAL_PA_TX1, &rt2x00dev->cap_flags))
+				p1_idx10 = rt2800_read_fft_accumulation(rt2x00dev, 0x0A);
+
+			rt2x00_dbg(rt2x00dev, "IQ AGC %d %d\n", p0, p1);
+
+			if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) {
+				rt2x00_dbg(rt2x00dev, "IQ AGC IDX 10 %d %d\n", p0_idx10, p1_idx10);
+				if ((p0_idx10 > 7000 * 7000) || (p1_idx10 > 7000 * 7000)) {
+					if (vga_gain[ch_idx] != 0)
+						vga_gain[ch_idx] = vga_gain[ch_idx] - 1;
+					break;
+				}
+			}
+
+			if ((p0 > 2500 * 2500) || (p1 > 2500 * 2500))
+				break;
+		}
+
+		if (vga_gain[ch_idx] > 18)
+			vga_gain[ch_idx] = 18;
+		rt2x00_dbg(rt2x00dev, "Used VGA %d %x\n", vga_gain[ch_idx],
+			   rfvga_gain_table[vga_gain[ch_idx]]);
+
+		bbp = (ch_idx == 0) ? 0x29 : 0x47;
+		rt2800_bbp_write(rt2x00dev, 158, bbp);
+		rt2800_bbp_write(rt2x00dev, 159, 0x00);
+
+		rt2800_iq_search(rt2x00dev, ch_idx, ger, per);
+	}
+
+	rt2800_bbp_write(rt2x00dev, 23, 0x00);
+	rt2800_bbp_write(rt2x00dev, 24, 0x00);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x04);
+
+	rt2800_bbp_write(rt2x00dev, 158, 0x28);
+	bbp = ger[CHAIN_0] & 0x0F;
+	rt2800_bbp_write(rt2x00dev, 159, bbp);
+
+	rt2800_bbp_write(rt2x00dev, 158, 0x29);
+	bbp = per[CHAIN_0] & 0x3F;
+	rt2800_bbp_write(rt2x00dev, 159, bbp);
+
+	rt2800_bbp_write(rt2x00dev, 158, 0x46);
+	bbp = ger[CHAIN_1] & 0x0F;
+	rt2800_bbp_write(rt2x00dev, 159, bbp);
+
+	rt2800_bbp_write(rt2x00dev, 158, 0x47);
+	bbp = per[CHAIN_1] & 0x3F;
+	rt2800_bbp_write(rt2x00dev, 159, bbp);
+
+	if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags)) {
+		rt2800_bbp_write(rt2x00dev, 1, bbpr1);
+		rt2800_bbp_write(rt2x00dev, 241, bbpr241);
+		rt2800_bbp_write(rt2x00dev, 242, bbpr242);
+	}
+	rt2800_bbp_write(rt2x00dev, 244, 0x00);
+
+	rt2800_bbp_write(rt2x00dev, 158, 0x00);
+	rt2800_bbp_write(rt2x00dev, 159, 0x00);
+	rt2800_bbp_write(rt2x00dev, 158, 0xB0);
+	rt2800_bbp_write(rt2x00dev, 159, 0x00);
+
+	rt2800_bbp_write(rt2x00dev, 30, bbpr30);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 39, rfb0r39);
+	rt2800_rfcsr_write_bank(rt2x00dev, 0, 42, rfb0r42);
+
+	if (test_bit(CAPABILITY_EXTERNAL_PA_TX0, &rt2x00dev->cap_flags))
+		rt2800_bbp_write(rt2x00dev, 4, bbpr4);
+
+	rt2800_bbp_write(rt2x00dev, 21, 0x01);
+	udelay(1);
+	rt2800_bbp_write(rt2x00dev, 21, 0x00);
+
+	rt2800_rf_configrecover(rt2x00dev, rf_store);
+
+	rt2800_register_write(rt2x00dev, TX_PIN_CFG, macorg1);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, 0x00);
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, 0x00);
+	rt2800_register_write(rt2x00dev, RF_CONTROL0, macorg2);
+	udelay(1);
+	rt2800_register_write(rt2x00dev, RF_BYPASS0, macorg3);
+	rt2800_register_write(rt2x00dev, RF_CONTROL3, macorg4);
+	rt2800_register_write(rt2x00dev, RF_BYPASS3, macorg5);
+	rt2800_register_write(rt2x00dev, MAC_SYS_CTRL, savemacsysctrl);
+	rt2800_register_write(rt2x00dev, 0x13b8, mac13b8);
+}
+
 static void rt2800_bbp_core_soft_reset(struct rt2x00_dev *rt2x00dev,
 				       bool set_bw, bool is_ht40)
 {
@@ -9005,8 +10582,13 @@ static void rt2800_init_rfcsr_6352(struct rt2x00_dev *rt2x00dev)
 	rt2800_rfcsr_write_dccal(rt2x00dev, 5, 0x00);
 	rt2800_rfcsr_write_dccal(rt2x00dev, 17, 0x7C);
 
+	rt2800_r_calibration(rt2x00dev);
+	rt2800_rf_self_txdc_cal(rt2x00dev);
+	rt2800_rxdcoc_calibration(rt2x00dev);
 	rt2800_bw_filter_calibration(rt2x00dev, true);
 	rt2800_bw_filter_calibration(rt2x00dev, false);
+	rt2800_loft_iq_calibration(rt2x00dev);
+	rt2800_rxiq_calibration(rt2x00dev);
 }
 
 static void rt2800_init_rfcsr(struct rt2x00_dev *rt2x00dev)
@@ -9073,7 +10655,7 @@ int rt2800_enable_radio(struct rt2x00_dev *rt2x00dev)
 	/*
 	 * Wait BBP/RF to wake up.
 	 */
-	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev)))
+	if (unlikely(rt2800_wait_bbp_rf_ready(rt2x00dev, MAC_STATUS_CFG_BBP_RF_BUSY)))
 		return -EIO;
 
 	/*
@@ -9435,6 +11017,8 @@ static int rt2800_init_eeprom(struct rt2x00_dev *rt2x00dev)
 		rf = RF3853;
 	else if (rt2x00_rt(rt2x00dev, RT5350))
 		rf = RF5350;
+	else if (rt2x00_rt(rt2x00dev, RT5592))
+		rf = RF5592;
 	else
 		rf = rt2x00_get_field16(eeprom, EEPROM_NIC_CONF0_RF_TYPE);
 
@@ -9564,7 +11148,8 @@ static int rt2800_init_eeprom(struct rt2x00_dev *rt2x00dev)
 	 */
 	eeprom = rt2800_eeprom_read(rt2x00dev, EEPROM_NIC_CONF1);
 
-	if (rt2x00_rt(rt2x00dev, RT3352)) {
+	if (rt2x00_rt(rt2x00dev, RT3352) ||
+	    rt2x00_rt(rt2x00dev, RT6352)) {
 		if (rt2x00_get_field16(eeprom,
 		    EEPROM_NIC_CONF1_EXTERNAL_TX0_PA_3352))
 		    __set_bit(CAPABILITY_EXTERNAL_PA_TX0,
@@ -9575,6 +11160,18 @@ static int rt2800_init_eeprom(struct rt2x00_dev *rt2x00dev)
 			      &rt2x00dev->cap_flags);
 	}
 
+	eeprom = rt2800_eeprom_read(rt2x00dev, EEPROM_NIC_CONF2);
+
+	if (rt2x00_rt(rt2x00dev, RT6352) && eeprom != 0 && eeprom != 0xffff) {
+		if (!rt2x00_get_field16(eeprom,
+					EEPROM_NIC_CONF2_EXTERNAL_PA)) {
+			__clear_bit(CAPABILITY_EXTERNAL_PA_TX0,
+				    &rt2x00dev->cap_flags);
+			__clear_bit(CAPABILITY_EXTERNAL_PA_TX1,
+				    &rt2x00dev->cap_flags);
+		}
+	}
+
 	return 0;
 }
 
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2800lib.h b/drivers/net/wireless/ralink/rt2x00/rt2800lib.h
index e1761f467b94..3cbef77b4bd3 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2800lib.h
+++ b/drivers/net/wireless/ralink/rt2x00/rt2800lib.h
@@ -17,6 +17,16 @@
 #define WCID_START	33
 #define WCID_END	222
 #define STA_IDS_SIZE	(WCID_END - WCID_START + 2)
+#define CHAIN_0		0x0
+#define CHAIN_1		0x1
+#define RF_ALC_NUM	6
+#define CHAIN_NUM	2
+
+struct rf_reg_pair {
+	u8 bank;
+	u8 reg;
+	u8 value;
+};
 
 /* RT2800 driver data structure */
 struct rt2800_drv_data {
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00.h b/drivers/net/wireless/ralink/rt2x00/rt2x00.h
index 8f5772b98f58..07a6a5a9ce13 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00.h
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00.h
@@ -1309,8 +1309,11 @@ void rt2x00queue_unmap_skb(struct queue_entry *entry);
  */
 static inline struct data_queue *
 rt2x00queue_get_tx_queue(struct rt2x00_dev *rt2x00dev,
-			 const enum data_queue_qid queue)
+			 enum data_queue_qid queue)
 {
+	if (queue >= rt2x00dev->ops->tx_queues && queue < IEEE80211_NUM_ACS)
+		queue = rt2x00dev->ops->tx_queues - 1;
+
 	if (queue < rt2x00dev->ops->tx_queues && rt2x00dev->tx)
 		return &rt2x00dev->tx[queue];
 
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
index e95c101c2711..3a035afcf7f9 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00dev.c
@@ -1093,6 +1093,19 @@ static void rt2x00lib_remove_hw(struct rt2x00_dev *rt2x00dev)
 	kfree(rt2x00dev->spec.channels_info);
 }
 
+static const struct ieee80211_tpt_blink rt2x00_tpt_blink[] = {
+	{ .throughput = 0 * 1024, .blink_time = 334 },
+	{ .throughput = 1 * 1024, .blink_time = 260 },
+	{ .throughput = 2 * 1024, .blink_time = 220 },
+	{ .throughput = 5 * 1024, .blink_time = 190 },
+	{ .throughput = 10 * 1024, .blink_time = 170 },
+	{ .throughput = 25 * 1024, .blink_time = 150 },
+	{ .throughput = 54 * 1024, .blink_time = 130 },
+	{ .throughput = 120 * 1024, .blink_time = 110 },
+	{ .throughput = 265 * 1024, .blink_time = 80 },
+	{ .throughput = 586 * 1024, .blink_time = 50 },
+};
+
 static int rt2x00lib_probe_hw(struct rt2x00_dev *rt2x00dev)
 {
 	struct hw_mode_spec *spec = &rt2x00dev->spec;
@@ -1174,6 +1187,11 @@ static int rt2x00lib_probe_hw(struct rt2x00_dev *rt2x00dev)
 
 #undef RT2X00_TASKLET_INIT
 
+	ieee80211_create_tpt_led_trigger(rt2x00dev->hw,
+					 IEEE80211_TPT_LEDTRIG_FL_RADIO,
+					 rt2x00_tpt_blink,
+					 ARRAY_SIZE(rt2x00_tpt_blink));
+
 	/*
 	 * Register HW.
 	 */
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c b/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c
index 4d06038afd83..98df0aef8168 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00queue.c
@@ -318,7 +318,7 @@ static void rt2x00queue_create_tx_descriptor_ht(struct rt2x00_dev *rt2x00dev,
 		 * when using more then one tx stream (>MCS7).
 		 */
 		if (sta && txdesc->u.ht.mcs > 7 &&
-		    sta->smps_mode == IEEE80211_SMPS_DYNAMIC)
+		    sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC)
 			__set_bit(ENTRY_TXD_HT_MIMO_PS, &txdesc->flags);
 	} else {
 		txdesc->u.ht.mcs = rt2x00_get_rate_mcs(hwrate->mcs);
diff --git a/drivers/net/wireless/realtek/rtl818x/rtl8187/leds.c b/drivers/net/wireless/realtek/rtl818x/rtl8187/leds.c
index 49421d10e22b..f7d95c9624a0 100644
--- a/drivers/net/wireless/realtek/rtl818x/rtl8187/leds.c
+++ b/drivers/net/wireless/realtek/rtl818x/rtl8187/leds.c
@@ -143,7 +143,7 @@ static int rtl8187_register_led(struct ieee80211_hw *dev,
 	led->dev = dev;
 	led->ledpin = ledpin;
 	led->is_radio = is_radio;
-	strlcpy(led->name, name, sizeof(led->name));
+	strscpy(led->name, name, sizeof(led->name));
 
 	led->led_dev.name = led->name;
 	led->led_dev.default_trigger = default_trigger;
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
index 7ddce3c3f0c4..782b089a2e1b 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu.h
@@ -1425,7 +1425,7 @@ struct rtl8xxxu_fileops {
 	void (*set_tx_power) (struct rtl8xxxu_priv *priv, int channel,
 			      bool ht40);
 	void (*update_rate_mask) (struct rtl8xxxu_priv *priv,
-				  u32 ramask, u8 rateid, int sgi);
+				  u32 ramask, u8 rateid, int sgi, int txbw_40mhz);
 	void (*report_connect) (struct rtl8xxxu_priv *priv,
 				u8 macid, bool connect);
 	void (*fill_txdesc) (struct ieee80211_hw *hw, struct ieee80211_hdr *hdr,
@@ -1511,9 +1511,9 @@ void rtl8xxxu_gen2_config_channel(struct ieee80211_hw *hw);
 void rtl8xxxu_gen1_usb_quirks(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_gen2_usb_quirks(struct rtl8xxxu_priv *priv);
 void rtl8xxxu_update_rate_mask(struct rtl8xxxu_priv *priv,
-			       u32 ramask, u8 rateid, int sgi);
+			       u32 ramask, u8 rateid, int sgi, int txbw_40mhz);
 void rtl8xxxu_gen2_update_rate_mask(struct rtl8xxxu_priv *priv,
-				    u32 ramask, u8 rateid, int sgi);
+				    u32 ramask, u8 rateid, int sgi, int txbw_40mhz);
 void rtl8xxxu_gen1_report_connect(struct rtl8xxxu_priv *priv,
 				  u8 macid, bool connect);
 void rtl8xxxu_gen2_report_connect(struct rtl8xxxu_priv *priv,
diff --git a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
index c66f0726b253..ac641a56efb0 100644
--- a/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
+++ b/drivers/net/wireless/realtek/rtl8xxxu/rtl8xxxu_core.c
@@ -1878,13 +1878,6 @@ static int rtl8xxxu_read_efuse(struct rtl8xxxu_priv *priv)
 
 		/* We have 8 bits to indicate validity */
 		map_addr = offset * 8;
-		if (map_addr >= EFUSE_MAP_LEN) {
-			dev_warn(dev, "%s: Illegal map_addr (%04x), "
-				 "efuse corrupt!\n",
-				 __func__, map_addr);
-			ret = -EINVAL;
-			goto exit;
-		}
 		for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++) {
 			/* Check word enable condition in the section */
 			if (word_mask & BIT(i)) {
@@ -1895,6 +1888,13 @@ static int rtl8xxxu_read_efuse(struct rtl8xxxu_priv *priv)
 			ret = rtl8xxxu_read_efuse8(priv, efuse_addr++, &val8);
 			if (ret)
 				goto exit;
+			if (map_addr >= EFUSE_MAP_LEN - 1) {
+				dev_warn(dev, "%s: Illegal map_addr (%04x), "
+					 "efuse corrupt!\n",
+					 __func__, map_addr);
+				ret = -EINVAL;
+				goto exit;
+			}
 			priv->efuse_wifi.raw[map_addr++] = val8;
 
 			ret = rtl8xxxu_read_efuse8(priv, efuse_addr++, &val8);
@@ -2929,12 +2929,12 @@ bool rtl8xxxu_gen2_simularity_compare(struct rtl8xxxu_priv *priv,
 		}
 
 		if (!(simubitmap & 0x30) && priv->tx_paths > 1) {
-			/* path B RX OK */
+			/* path B TX OK */
 			for (i = 4; i < 6; i++)
 				result[3][i] = result[c1][i];
 		}
 
-		if (!(simubitmap & 0x30) && priv->tx_paths > 1) {
+		if (!(simubitmap & 0xc0) && priv->tx_paths > 1) {
 			/* path B RX OK */
 			for (i = 6; i < 8; i++)
 				result[3][i] = result[c1][i];
@@ -4320,7 +4320,7 @@ static void rtl8xxxu_sw_scan_complete(struct ieee80211_hw *hw,
 }
 
 void rtl8xxxu_update_rate_mask(struct rtl8xxxu_priv *priv,
-			       u32 ramask, u8 rateid, int sgi)
+			       u32 ramask, u8 rateid, int sgi, int txbw_40mhz)
 {
 	struct h2c_cmd h2c;
 
@@ -4340,10 +4340,15 @@ void rtl8xxxu_update_rate_mask(struct rtl8xxxu_priv *priv,
 }
 
 void rtl8xxxu_gen2_update_rate_mask(struct rtl8xxxu_priv *priv,
-				    u32 ramask, u8 rateid, int sgi)
+				    u32 ramask, u8 rateid, int sgi, int txbw_40mhz)
 {
 	struct h2c_cmd h2c;
-	u8 bw = RTL8XXXU_CHANNEL_WIDTH_20;
+	u8 bw;
+
+	if (txbw_40mhz)
+		bw = RTL8XXXU_CHANNEL_WIDTH_40;
+	else
+		bw = RTL8XXXU_CHANNEL_WIDTH_20;
 
 	memset(&h2c, 0, sizeof(struct h2c_cmd));
 
@@ -4353,15 +4358,14 @@ void rtl8xxxu_gen2_update_rate_mask(struct rtl8xxxu_priv *priv,
 	h2c.b_macid_cfg.ramask2 = (ramask >> 16) & 0xff;
 	h2c.b_macid_cfg.ramask3 = (ramask >> 24) & 0xff;
 
-	h2c.ramask.arg = 0x80;
 	h2c.b_macid_cfg.data1 = rateid;
 	if (sgi)
 		h2c.b_macid_cfg.data1 |= BIT(7);
 
 	h2c.b_macid_cfg.data2 = bw;
 
-	dev_dbg(&priv->udev->dev, "%s: rate mask %08x, arg %02x, size %zi\n",
-		__func__, ramask, h2c.ramask.arg, sizeof(h2c.b_macid_cfg));
+	dev_dbg(&priv->udev->dev, "%s: rate mask %08x, rateid %02x, sgi %d, size %zi\n",
+		__func__, ramask, rateid, sgi, sizeof(h2c.b_macid_cfg));
 	rtl8xxxu_gen2_h2c_cmd(priv, &h2c, sizeof(h2c.b_macid_cfg));
 }
 
@@ -4556,6 +4560,53 @@ rtl8xxxu_wireless_mode(struct ieee80211_hw *hw, struct ieee80211_sta *sta)
 	return network_type;
 }
 
+static void rtl8xxxu_set_aifs(struct rtl8xxxu_priv *priv, u8 slot_time)
+{
+	u32 reg_edca_param[IEEE80211_NUM_ACS] = {
+		[IEEE80211_AC_VO] = REG_EDCA_VO_PARAM,
+		[IEEE80211_AC_VI] = REG_EDCA_VI_PARAM,
+		[IEEE80211_AC_BE] = REG_EDCA_BE_PARAM,
+		[IEEE80211_AC_BK] = REG_EDCA_BK_PARAM,
+	};
+	u32 val32;
+	u16 wireless_mode = 0;
+	u8 aifs, aifsn, sifs;
+	int i;
+
+	if (priv->vif) {
+		struct ieee80211_sta *sta;
+
+		rcu_read_lock();
+		sta = ieee80211_find_sta(priv->vif, priv->vif->bss_conf.bssid);
+		if (sta)
+			wireless_mode = rtl8xxxu_wireless_mode(priv->hw, sta);
+		rcu_read_unlock();
+	}
+
+	if (priv->hw->conf.chandef.chan->band == NL80211_BAND_5GHZ ||
+	    (wireless_mode & WIRELESS_MODE_N_24G))
+		sifs = 16;
+	else
+		sifs = 10;
+
+	for (i = 0; i < IEEE80211_NUM_ACS; i++) {
+		val32 = rtl8xxxu_read32(priv, reg_edca_param[i]);
+
+		/* It was set in conf_tx. */
+		aifsn = val32 & 0xff;
+
+		/* aifsn not set yet or already fixed */
+		if (aifsn < 2 || aifsn > 15)
+			continue;
+
+		aifs = aifsn * slot_time + sifs;
+
+		val32 &= ~0xff;
+		val32 |= aifs;
+		rtl8xxxu_write32(priv, reg_edca_param[i], val32);
+	}
+}
+
 static void
 rtl8xxxu_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			  struct ieee80211_bss_conf *bss_conf, u64 changed)
@@ -4622,7 +4673,11 @@ rtl8xxxu_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 						RATE_INFO_FLAGS_SHORT_GI;
 				}
 
-				rarpt->txrate.bw |= RATE_INFO_BW_20;
+				if (rtl8xxxu_ht40_2g &&
+				    (sta->deflink.ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40))
+					rarpt->txrate.bw = RATE_INFO_BW_40;
+				else
+					rarpt->txrate.bw = RATE_INFO_BW_20;
 			}
 			bit_rate = cfg80211_calculate_bitrate(&rarpt->txrate);
 			rarpt->bit_rate = bit_rate;
@@ -4631,7 +4686,7 @@ rtl8xxxu_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			priv->vif = vif;
 			priv->rssi_level = RTL8XXXU_RATR_STA_INIT;
 
-			priv->fops->update_rate_mask(priv, ramask, 0, sgi);
+			priv->fops->update_rate_mask(priv, ramask, 0, sgi, rarpt->txrate.bw == RATE_INFO_BW_40);
 
 			rtl8xxxu_write8(priv, REG_BCN_MAX_ERR, 0xff);
 
@@ -4671,6 +4726,8 @@ rtl8xxxu_bss_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		else
 			val8 = 20;
 		rtl8xxxu_write8(priv, REG_SLOT, val8);
+
+		rtl8xxxu_set_aifs(priv, val8);
 	}
 
 	if (changed & BSS_CHANGED_BSSID) {
@@ -4710,9 +4767,8 @@ static u32 rtl8xxxu_80211_to_rtl_queue(u32 queue)
 	return rtlqueue;
 }
 
-static u32 rtl8xxxu_queue_select(struct ieee80211_hw *hw, struct sk_buff *skb)
+static u32 rtl8xxxu_queue_select(struct ieee80211_hdr *hdr, struct sk_buff *skb)
 {
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	u32 queue;
 
 	if (ieee80211_is_mgmt(hdr->frame_control))
@@ -5062,6 +5118,8 @@ static void rtl8xxxu_tx(struct ieee80211_hw *hw,
 	if (control && control->sta)
 		sta = control->sta;
 
+	queue = rtl8xxxu_queue_select(hdr, skb);
+
 	tx_desc = skb_push(skb, tx_desc_size);
 
 	memset(tx_desc, 0, tx_desc_size);
@@ -5074,7 +5132,6 @@ static void rtl8xxxu_tx(struct ieee80211_hw *hw,
 	    is_broadcast_ether_addr(ieee80211_get_DA(hdr)))
 		tx_desc->txdw0 |= TXDESC_BROADMULTICAST;
 
-	queue = rtl8xxxu_queue_select(hw, skb);
 	tx_desc->txdw1 = cpu_to_le32(queue << TXDESC_QUEUE_SHIFT);
 
 	if (tx_info->control.hw_key) {
@@ -6344,7 +6401,7 @@ static void rtl8xxxu_refresh_rate_mask(struct rtl8xxxu_priv *priv,
 		}
 
 		priv->rssi_level = rssi_level;
-		priv->fops->update_rate_mask(priv, rate_bitmap, ratr_idx, sgi);
+		priv->fops->update_rate_mask(priv, rate_bitmap, ratr_idx, sgi, txbw_40mhz);
 	}
 }
 
@@ -6657,7 +6714,6 @@ static int rtl8xxxu_probe(struct usb_interface *interface,
 	hw = ieee80211_alloc_hw(sizeof(struct rtl8xxxu_priv), &rtl8xxxu_ops);
 	if (!hw) {
 		ret = -ENOMEM;
-		priv = NULL;
 		goto err_put_dev;
 	}
 
@@ -6768,11 +6824,9 @@ static int rtl8xxxu_probe(struct usb_interface *interface,
 err_set_intfdata:
 	usb_set_intfdata(interface, NULL);
 
-	if (priv) {
-		kfree(priv->fw_data);
-		mutex_destroy(&priv->usb_buf_mutex);
-		mutex_destroy(&priv->h2c_mutex);
-	}
+	kfree(priv->fw_data);
+	mutex_destroy(&priv->usb_buf_mutex);
+	mutex_destroy(&priv->h2c_mutex);
 
 	ieee80211_free_hw(hw);
 err_put_dev:
diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c
index 15e6a6aded31..d18c092b6142 100644
--- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c
+++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c
@@ -2386,11 +2386,10 @@ void rtl92d_phy_reload_iqk_setting(struct ieee80211_hw *hw, u8 channel)
 			rtl_dbg(rtlpriv, COMP_SCAN, DBG_LOUD,
 				"Just Read IQK Matrix reg for channel:%d....\n",
 				channel);
-			_rtl92d_phy_patha_fill_iqk_matrix(hw, true,
-					rtlphy->iqk_matrix[
-					indexforchannel].value,	0,
-					(rtlphy->iqk_matrix[
-					indexforchannel].value[0][2] == 0));
+			if (rtlphy->iqk_matrix[indexforchannel].value[0][0] != 0)
+				_rtl92d_phy_patha_fill_iqk_matrix(hw, true,
+					rtlphy->iqk_matrix[indexforchannel].value, 0,
+					rtlphy->iqk_matrix[indexforchannel].value[0][2] == 0);
 			if (IS_92D_SINGLEPHY(rtlhal->version)) {
 				if ((rtlphy->iqk_matrix[
 					indexforchannel].value[0][4] != 0)
diff --git a/drivers/net/wireless/realtek/rtw88/bf.c b/drivers/net/wireless/realtek/rtw88/bf.c
index 76c7f3257dd3..038a30b170ef 100644
--- a/drivers/net/wireless/realtek/rtw88/bf.c
+++ b/drivers/net/wireless/realtek/rtw88/bf.c
@@ -30,11 +30,11 @@ void rtw_bf_disassoc(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
 void rtw_bf_assoc(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
 		  struct ieee80211_bss_conf *bss_conf)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct ieee80211_hw *hw = rtwdev->hw;
 	struct rtw_vif *rtwvif = (struct rtw_vif *)vif->drv_priv;
 	struct rtw_bfee *bfee = &rtwvif->bfee;
 	struct rtw_bf_info *bfinfo = &rtwdev->bf_info;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct ieee80211_sta *sta;
 	struct ieee80211_sta_vht_cap *vht_cap;
 	struct ieee80211_sta_vht_cap *ic_vht_cap;
diff --git a/drivers/net/wireless/realtek/rtw88/coex.c b/drivers/net/wireless/realtek/rtw88/coex.c
index cac053f485c3..6276ad624299 100644
--- a/drivers/net/wireless/realtek/rtw88/coex.c
+++ b/drivers/net/wireless/realtek/rtw88/coex.c
@@ -13,7 +13,7 @@
 static u8 rtw_coex_next_rssi_state(struct rtw_dev *rtwdev, u8 pre_state,
 				   u8 rssi, u8 rssi_thresh)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	u8 tol = chip->rssi_tolerance;
 	u8 next_state;
 
@@ -36,7 +36,7 @@ static u8 rtw_coex_next_rssi_state(struct rtw_dev *rtwdev, u8 pre_state,
 static void rtw_coex_limited_tx(struct rtw_dev *rtwdev,
 				bool tx_limit_en, bool ampdu_limit_en)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	u8 num_of_active_port = 1;
@@ -365,7 +365,7 @@ static void rtw_coex_set_wl_pri_mask(struct rtw_dev *rtwdev, u8 bitmap,
 
 void rtw_coex_write_scbd(struct rtw_dev *rtwdev, u16 bitpos, bool set)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	u16 val = 0x2;
@@ -400,7 +400,7 @@ EXPORT_SYMBOL(rtw_coex_write_scbd);
 
 static u16 rtw_coex_read_scbd(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	if (!chip->scbd_support)
 		return 0;
@@ -410,7 +410,7 @@ static u16 rtw_coex_read_scbd(struct rtw_dev *rtwdev)
 
 static void rtw_coex_check_rfk(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_coex_rfe *coex_rfe = &coex->rfe;
@@ -489,7 +489,7 @@ static void rtw_coex_monitor_bt_ctr(struct rtw_dev *rtwdev)
 
 static void rtw_coex_monitor_bt_enable(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
@@ -524,10 +524,10 @@ static void rtw_coex_monitor_bt_enable(struct rtw_dev *rtwdev)
 
 static void rtw_coex_update_wl_link_info(struct rtw_dev *rtwdev, u8 reason)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_traffic_stats *stats = &rtwdev->stats;
 	bool is_5G = false;
 	bool wl_busy = false;
@@ -706,10 +706,10 @@ static const char *rtw_coex_get_bt_status_string(u8 bt_status)
 
 static void rtw_coex_update_bt_link_info(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 i;
 	u8 rssi_state;
 	u8 rssi_step;
@@ -806,7 +806,7 @@ static void rtw_coex_update_bt_link_info(struct rtw_dev *rtwdev)
 
 static void rtw_coex_update_wl_ch_info(struct rtw_dev *rtwdev, u8 type)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 	struct rtw_coex_dm *coex_dm = &rtwdev->coex.dm;
 	struct rtw_coex_stat *coex_stat = &rtwdev->coex.stat;
@@ -933,7 +933,7 @@ EXPORT_SYMBOL(rtw_coex_write_indirect_reg);
 
 static void rtw_coex_coex_ctrl_owner(struct rtw_dev *rtwdev, bool wifi_control)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	const struct rtw_hw_reg *btg_reg = chip->btg_reg;
 
 	if (wifi_control) {
@@ -981,7 +981,7 @@ static void rtw_coex_mimo_ps(struct rtw_dev *rtwdev, bool force, bool state)
 static void rtw_btc_wltoggle_table_a(struct rtw_dev *rtwdev, bool force,
 				     u8 table_case)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 	u8 h2c_para[6] = {0};
 	u32 table_wl = 0x5a5a5a5a;
@@ -1065,9 +1065,9 @@ static void rtw_coex_set_table(struct rtw_dev *rtwdev, bool force, u32 table0,
 
 static void rtw_coex_table(struct rtw_dev *rtwdev, bool force, u8 type)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 
@@ -1135,9 +1135,9 @@ static void rtw_coex_power_save_state(struct rtw_dev *rtwdev, u8 ps_type,
 static void rtw_coex_set_tdma(struct rtw_dev *rtwdev, u8 byte1, u8 byte2,
 			      u8 byte3, u8 byte4, u8 byte5)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	u8 ps_type = COEX_PS_WIFI_NATIVE;
 	bool ap_enable = false;
@@ -1193,10 +1193,10 @@ static void rtw_coex_set_tdma(struct rtw_dev *rtwdev, u8 byte1, u8 byte2,
 
 static void rtw_coex_tdma(struct rtw_dev *rtwdev, bool force, u32 tcase)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 	u8 n, type;
 	bool turn_on;
@@ -1526,8 +1526,8 @@ static u8 rtw_coex_algorithm(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_coex_all_off(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 
 	rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__);
@@ -1549,11 +1549,11 @@ static void rtw_coex_action_coex_all_off(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_freerun(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 level = 0;
 	bool bt_afh_loss = true;
 
@@ -1594,8 +1594,8 @@ static void rtw_coex_action_freerun(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_rf4ce(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 
 	rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__);
@@ -1619,8 +1619,8 @@ static void rtw_coex_action_rf4ce(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_whql_test(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 
 	rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__);
@@ -1644,10 +1644,10 @@ static void rtw_coex_action_bt_whql_test(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_relink(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 	u32 slot_type = 0;
 
@@ -1684,11 +1684,11 @@ static void rtw_coex_action_bt_relink(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_idle(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex_rfe *coex_rfe = &coex->rfe;
 	u8 table_case = 0xff, tdma_case = 0xff;
 
@@ -1753,10 +1753,10 @@ exit:
 
 static void rtw_coex_action_bt_inquiry(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	bool wl_hi_pri = false;
 	u8 table_case, tdma_case;
 	u32 slot_type = 0;
@@ -1853,11 +1853,11 @@ static void rtw_coex_action_bt_inquiry(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_game_hid(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 
 	rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__);
@@ -1901,10 +1901,10 @@ static void rtw_coex_action_bt_game_hid(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_hfp(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 
 	rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__);
@@ -1932,10 +1932,10 @@ static void rtw_coex_action_bt_hfp(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_hid(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 	u32 slot_type = 0;
 	bool bt_multi_link_remain = false, is_toggle_table = false;
@@ -2015,11 +2015,11 @@ static void rtw_coex_action_bt_hid(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_a2dp(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 	u32 slot_type = 0;
 
@@ -2057,10 +2057,10 @@ static void rtw_coex_action_bt_a2dp(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_a2dpsink(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 	bool ap_enable = false;
 
@@ -2096,10 +2096,10 @@ static void rtw_coex_action_bt_a2dpsink(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_pan(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 
 	rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__);
@@ -2133,11 +2133,11 @@ static void rtw_coex_action_bt_pan(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_a2dp_hid(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case, interval = 0;
 	u32 slot_type = 0;
 	bool is_toggle_table = false;
@@ -2190,10 +2190,10 @@ static void rtw_coex_action_bt_a2dp_hid(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_a2dp_pan(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 	bool wl_cpt_test = false, bt_cpt_test = false;
 
@@ -2247,10 +2247,10 @@ static void rtw_coex_action_bt_a2dp_pan(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_pan_hid(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 
 	rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__);
@@ -2282,10 +2282,10 @@ static void rtw_coex_action_bt_pan_hid(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_bt_a2dp_pan_hid(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 
 	rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__);
@@ -2316,9 +2316,9 @@ static void rtw_coex_action_bt_a2dp_pan_hid(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_wl_under5g(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	u8 table_case, tdma_case;
 
@@ -2348,8 +2348,8 @@ static void rtw_coex_action_wl_under5g(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_wl_only(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 
 	rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__);
@@ -2372,9 +2372,9 @@ static void rtw_coex_action_wl_only(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_wl_native_lps(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	u8 table_case, tdma_case;
 
@@ -2411,10 +2411,10 @@ static void rtw_coex_action_wl_native_lps(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_wl_linkscan(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 	u32 slot_type = 0;
 
@@ -2451,8 +2451,8 @@ static void rtw_coex_action_wl_linkscan(struct rtw_dev *rtwdev)
 
 static void rtw_coex_action_wl_not_connected(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 table_case, tdma_case;
 
 	rtw_dbg(rtwdev, RTW_DBG_COEX, "[BTCoex], %s()\n", __func__);
@@ -2528,8 +2528,8 @@ static void rtw_coex_action_wl_connected(struct rtw_dev *rtwdev)
 
 static void rtw_coex_run_coex(struct rtw_dev *rtwdev, u8 reason)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	bool rf4ce_en = false;
@@ -3002,9 +3002,9 @@ void rtw_coex_media_status_notify(struct rtw_dev *rtwdev, u8 type)
 
 void rtw_coex_bt_info_notify(struct rtw_dev *rtwdev, u8 *buf, u8 length)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex_dm *coex_dm = &coex->dm;
 	u32 bt_relink_time;
 	u8 i, rsp_source = 0, type;
@@ -3270,8 +3270,8 @@ static const u8 coex_bt_hidinfo_xb[] = {0x58, 0x62, 0x6f};
 
 void rtw_coex_bt_hid_info_notify(struct rtw_dev *rtwdev, u8 *buf, u8 length)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_coex_hid *hidinfo;
 	struct rtw_coex_hid_info_a *hida;
@@ -3360,8 +3360,8 @@ void rtw_coex_bt_hid_info_notify(struct rtw_dev *rtwdev, u8 *buf, u8 length)
 
 void rtw_coex_query_bt_hid_list(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
 	struct rtw_coex_hid *hidinfo;
 	u8 i, handle;
@@ -3582,7 +3582,7 @@ static const char *rtw_coex_get_reason_string(u8 reason)
 static u8 rtw_coex_get_table_index(struct rtw_dev *rtwdev, u32 wl_reg_6c0,
 				   u32 wl_reg_6c4)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 	u8 ans = 0xFF;
 	u8 n, i;
@@ -3618,8 +3618,8 @@ static u8 rtw_coex_get_table_index(struct rtw_dev *rtwdev, u32 wl_reg_6c0,
 
 static u8 rtw_coex_get_tdma_index(struct rtw_dev *rtwdev, u8 *tdma_para)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u8 ans = 0xFF;
 	u8 n, i, j;
 	u8 load_cur_tab_val;
@@ -3736,7 +3736,7 @@ static int rtw_coex_val_info(struct rtw_dev *rtwdev,
 
 static void rtw_coex_set_coexinfo_hw(struct rtw_dev *rtwdev, struct seq_file *m)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	const struct rtw_reg_domain *reg;
 	char addr_info[INFO_SIZE];
 	int n_addr = 0;
@@ -3910,7 +3910,7 @@ static const char *rtw_coex_get_wl_coex_mode(u8 coex_wl_link_mode)
 
 void rtw_coex_display_coex_info(struct rtw_dev *rtwdev, struct seq_file *m)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
 	struct rtw_coex *coex = &rtwdev->coex;
 	struct rtw_coex_stat *coex_stat = &coex->stat;
diff --git a/drivers/net/wireless/realtek/rtw88/coex.h b/drivers/net/wireless/realtek/rtw88/coex.h
index 07fa7aa34d4b..57cf29da9ea4 100644
--- a/drivers/net/wireless/realtek/rtw88/coex.h
+++ b/drivers/net/wireless/realtek/rtw88/coex.h
@@ -327,7 +327,7 @@ struct coex_rf_para {
 
 static inline void rtw_coex_set_init(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	chip->ops->coex_set_init(rtwdev);
 }
@@ -335,7 +335,7 @@ static inline void rtw_coex_set_init(struct rtw_dev *rtwdev)
 static inline
 void rtw_coex_set_ant_switch(struct rtw_dev *rtwdev, u8 ctrl_type, u8 pos_type)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	if (!chip->ops->coex_set_ant_switch)
 		return;
@@ -345,28 +345,28 @@ void rtw_coex_set_ant_switch(struct rtw_dev *rtwdev, u8 ctrl_type, u8 pos_type)
 
 static inline void rtw_coex_set_gnt_fix(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	chip->ops->coex_set_gnt_fix(rtwdev);
 }
 
 static inline void rtw_coex_set_gnt_debug(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	chip->ops->coex_set_gnt_debug(rtwdev);
 }
 
 static inline  void rtw_coex_set_rfe_type(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	chip->ops->coex_set_rfe_type(rtwdev);
 }
 
 static inline void rtw_coex_set_wl_tx_power(struct rtw_dev *rtwdev, u8 wl_pwr)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	chip->ops->coex_set_wl_tx_power(rtwdev, wl_pwr);
 }
@@ -374,7 +374,7 @@ static inline void rtw_coex_set_wl_tx_power(struct rtw_dev *rtwdev, u8 wl_pwr)
 static inline
 void rtw_coex_set_wl_rx_gain(struct rtw_dev *rtwdev, bool low_gain)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	chip->ops->coex_set_wl_rx_gain(rtwdev, low_gain);
 }
diff --git a/drivers/net/wireless/realtek/rtw88/debug.c b/drivers/net/wireless/realtek/rtw88/debug.c
index 7cde6bcf253b..9ebe544e51d0 100644
--- a/drivers/net/wireless/realtek/rtw88/debug.c
+++ b/drivers/net/wireless/realtek/rtw88/debug.c
@@ -621,11 +621,13 @@ static int rtw_debugfs_get_tx_pwr_tbl(struct seq_file *m, void *v)
 	struct rtw_debugfs_priv *debugfs_priv = m->private;
 	struct rtw_dev *rtwdev = debugfs_priv->rtwdev;
 	struct rtw_hal *hal = &rtwdev->hal;
-	u8 path, rate;
+	u8 path, rate, bw, ch, regd;
 	struct rtw_power_params pwr_param = {0};
-	u8 bw = hal->current_band_width;
-	u8 ch = hal->current_channel;
-	u8 regd = rtw_regd_get(rtwdev);
+
+	mutex_lock(&rtwdev->mutex);
+	bw = hal->current_band_width;
+	ch = hal->current_channel;
+	regd = rtw_regd_get(rtwdev);
 
 	seq_printf(m, "channel: %u\n", ch);
 	seq_printf(m, "bandwidth: %u\n", bw);
@@ -667,6 +669,7 @@ static int rtw_debugfs_get_tx_pwr_tbl(struct seq_file *m, void *v)
 	}
 
 	mutex_unlock(&hal->tx_power_mutex);
+	mutex_unlock(&rtwdev->mutex);
 
 	return 0;
 }
diff --git a/drivers/net/wireless/realtek/rtw88/efuse.c b/drivers/net/wireless/realtek/rtw88/efuse.c
index c266c84ef233..b85075cd68d0 100644
--- a/drivers/net/wireless/realtek/rtw88/efuse.c
+++ b/drivers/net/wireless/realtek/rtw88/efuse.c
@@ -86,7 +86,7 @@ static int rtw_dump_logical_efuse_map(struct rtw_dev *rtwdev, u8 *phy_map,
 
 static int rtw_dump_physical_efuse_map(struct rtw_dev *rtwdev, u8 *map)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	u32 size = rtwdev->efuse.physical_size;
 	u32 efuse_ctl;
 	u32 addr;
@@ -145,7 +145,7 @@ EXPORT_SYMBOL(rtw_read8_physical_efuse);
 
 int rtw_parse_efuse_map(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 	u32 phy_size = efuse->physical_size;
 	u32 log_size = efuse->logical_size;
diff --git a/drivers/net/wireless/realtek/rtw88/fw.c b/drivers/net/wireless/realtek/rtw88/fw.c
index 4fdab0329695..0b5f903c0f36 100644
--- a/drivers/net/wireless/realtek/rtw88/fw.c
+++ b/drivers/net/wireless/realtek/rtw88/fw.c
@@ -14,6 +14,8 @@
 #include "util.h"
 #include "wow.h"
 #include "ps.h"
+#include "phy.h"
+#include "mac.h"
 
 static void rtw_fw_c2h_cmd_handle_ext(struct rtw_dev *rtwdev,
 				      struct sk_buff *skb)
@@ -116,7 +118,7 @@ legacy:
 	si->ra_report.desc_rate = rate;
 	si->ra_report.bit_rate = bit_rate;
 
-	sta->max_rc_amsdu_len = get_max_amsdu_len(bit_rate);
+	sta->deflink.agg.max_rc_amsdu_len = get_max_amsdu_len(bit_rate);
 }
 
 static void rtw_fw_ra_report_handle(struct rtw_dev *rtwdev, u8 *payload,
@@ -904,7 +906,7 @@ void rtw_send_rsvd_page_h2c(struct rtw_dev *rtwdev)
 static struct sk_buff *rtw_nlo_info_get(struct ieee80211_hw *hw)
 {
 	struct rtw_dev *rtwdev = hw->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_pno_request *pno_req = &rtwdev->wow.pno_req;
 	struct rtw_nlo_info_hdr *nlo_hdr;
 	struct cfg80211_ssid *ssid;
@@ -959,7 +961,7 @@ static struct sk_buff *rtw_nlo_info_get(struct ieee80211_hw *hw)
 static struct sk_buff *rtw_cs_channel_info_get(struct ieee80211_hw *hw)
 {
 	struct rtw_dev *rtwdev = hw->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_pno_request *pno_req = &rtwdev->wow.pno_req;
 	struct ieee80211_channel *channels = pno_req->channels;
 	struct sk_buff *skb;
@@ -993,7 +995,7 @@ static struct sk_buff *rtw_cs_channel_info_get(struct ieee80211_hw *hw)
 static struct sk_buff *rtw_lps_pg_dpk_get(struct ieee80211_hw *hw)
 {
 	struct rtw_dev *rtwdev = hw->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_dpk_info *dpk_info = &rtwdev->dm_info.dpk_info;
 	struct rtw_lps_pg_dpk_hdr *dpk_hdr;
 	struct sk_buff *skb;
@@ -1018,7 +1020,7 @@ static struct sk_buff *rtw_lps_pg_dpk_get(struct ieee80211_hw *hw)
 static struct sk_buff *rtw_lps_pg_info_get(struct ieee80211_hw *hw)
 {
 	struct rtw_dev *rtwdev = hw->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_lps_conf *conf = &rtwdev->lps_conf;
 	struct rtw_lps_pg_info_hdr *pg_info_hdr;
 	struct rtw_wow_param *rtw_wow = &rtwdev->wow;
@@ -1080,10 +1082,10 @@ static struct sk_buff *rtw_get_rsvd_page_skb(struct ieee80211_hw *hw,
 		skb_new = ieee80211_proberesp_get(hw, vif);
 		break;
 	case RSVD_NULL:
-		skb_new = ieee80211_nullfunc_get(hw, vif, false);
+		skb_new = ieee80211_nullfunc_get(hw, vif, -1, false);
 		break;
 	case RSVD_QOS_NULL:
-		skb_new = ieee80211_nullfunc_get(hw, vif, true);
+		skb_new = ieee80211_nullfunc_get(hw, vif, -1, true);
 		break;
 	case RSVD_LPS_PG_DPK:
 		skb_new = rtw_lps_pg_dpk_get(hw);
@@ -1122,7 +1124,7 @@ static void rtw_fill_rsvd_page_desc(struct rtw_dev *rtwdev, struct sk_buff *skb,
 				    enum rtw_rsvd_packet_type type)
 {
 	struct rtw_tx_pkt_info pkt_info = {0};
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	u8 *pkt_desc;
 
 	rtw_tx_rsvd_page_pkt_info_update(rtwdev, &pkt_info, skb, type);
@@ -1433,7 +1435,7 @@ static int  __rtw_build_rsvd_page_from_vifs(struct rtw_dev *rtwdev)
 static u8 *rtw_build_rsvd_page(struct rtw_dev *rtwdev, u32 *size)
 {
 	struct ieee80211_hw *hw = rtwdev->hw;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct sk_buff *iter;
 	struct rtw_rsvd_page *rsvd_pkt;
 	u32 page = 0;
@@ -1647,7 +1649,7 @@ out:
 static void rtw_fw_read_fifo(struct rtw_dev *rtwdev, enum rtw_fw_fifo_sel sel,
 			     u32 offset, u32 size, u32 *buf)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	u32 start_pg, residue;
 
 	if (sel >= RTW_FW_FIFO_MAX) {
@@ -1706,7 +1708,7 @@ int rtw_fw_dump_fifo(struct rtw_dev *rtwdev, u8 fifo_sel, u32 addr, u32 size,
 static void __rtw_fw_update_pkt(struct rtw_dev *rtwdev, u8 pkt_id, u16 size,
 				u8 location)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	u8 h2c_pkt[H2C_PKT_SIZE] = {0};
 	u16 total_size = H2C_PKT_HDR_SIZE + H2C_PKT_UPDATE_PKT_LEN;
 
@@ -1818,8 +1820,8 @@ static int rtw_append_probe_req_ie(struct rtw_dev *rtwdev, struct sk_buff *skb,
 				   struct sk_buff_head *list, u8 *bands,
 				   struct rtw_vif *rtwvif)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct ieee80211_scan_ies *ies = rtwvif->scan_ies;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct sk_buff *new;
 	u8 idx;
 
@@ -1841,16 +1843,23 @@ static int rtw_append_probe_req_ie(struct rtw_dev *rtwdev, struct sk_buff *skb,
 static int _rtw_hw_scan_update_probe_req(struct rtw_dev *rtwdev, u8 num_probes,
 					 struct sk_buff_head *probe_req_list)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct sk_buff *skb, *tmp;
 	u8 page_offset = 1, *buf, page_size = chip->page_size;
-	u8 pages = page_offset + num_probes * RTW_PROBE_PG_CNT;
 	u16 pg_addr = rtwdev->fifo.rsvd_h2c_info_addr, loc;
 	u16 buf_offset = page_size * page_offset;
 	u8 tx_desc_sz = chip->tx_pkt_desc_sz;
+	u8 page_cnt, pages;
 	unsigned int pkt_len;
 	int ret;
 
+	if (rtw_fw_feature_ext_check(&rtwdev->fw, FW_FEATURE_EXT_OLD_PAGE_NUM))
+		page_cnt = RTW_OLD_PROBE_PG_CNT;
+	else
+		page_cnt = RTW_PROBE_PG_CNT;
+
+	pages = page_offset + num_probes * page_cnt;
+
 	buf = kzalloc(page_size * pages, GFP_KERNEL);
 	if (!buf)
 		return -ENOMEM;
@@ -1859,7 +1868,7 @@ static int _rtw_hw_scan_update_probe_req(struct rtw_dev *rtwdev, u8 num_probes,
 	skb_queue_walk_safe(probe_req_list, skb, tmp) {
 		skb_unlink(skb, probe_req_list);
 		rtw_fill_rsvd_page_desc(rtwdev, skb, RSVD_PROBE_REQ);
-		if (skb->len > page_size * RTW_PROBE_PG_CNT) {
+		if (skb->len > page_size * page_cnt) {
 			ret = -EINVAL;
 			goto out;
 		}
@@ -1869,8 +1878,8 @@ static int _rtw_hw_scan_update_probe_req(struct rtw_dev *rtwdev, u8 num_probes,
 		loc = pg_addr - rtwdev->fifo.rsvd_boundary + page_offset;
 		__rtw_fw_update_pkt(rtwdev, RTW_PACKET_PROBE_REQ, pkt_len, loc);
 
-		buf_offset += RTW_PROBE_PG_CNT * page_size;
-		page_offset += RTW_PROBE_PG_CNT;
+		buf_offset += page_cnt * page_size;
+		page_offset += page_cnt;
 		kfree_skb(skb);
 	}
 
@@ -2048,6 +2057,9 @@ void rtw_hw_scan_start(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
 	rtwvif->scan_req = req;
 
 	ieee80211_stop_queues(rtwdev->hw);
+	rtw_leave_lps_deep(rtwdev);
+	rtw_hci_flush_all_queues(rtwdev, false);
+	rtw_mac_flush_all_queues(rtwdev, false);
 	if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR)
 		get_random_mask_addr(mac_addr, req->mac_addr,
 				     req->mac_addr_mask);
@@ -2080,10 +2092,9 @@ void rtw_hw_scan_complete(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
 	rtw_core_scan_complete(rtwdev, vif, true);
 
 	rtwvif = (struct rtw_vif *)vif->drv_priv;
-	if (rtwvif->net_type == RTW_NET_MGD_LINKED) {
-		hal->current_channel = chan;
-		hal->current_band_type = chan > 14 ? RTW_BAND_5G : RTW_BAND_2G;
-	}
+	if (chan)
+		rtw_store_op_chan(rtwdev, false);
+	rtw_phy_set_tx_power_level(rtwdev, hal->current_channel);
 	ieee80211_wake_queues(rtwdev->hw);
 	ieee80211_scan_completed(rtwdev->hw, &info);
 
@@ -2124,6 +2135,7 @@ int rtw_hw_scan_offload(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
 			bool enable)
 {
 	struct rtw_vif *rtwvif = vif ? (struct rtw_vif *)vif->drv_priv : NULL;
+	struct rtw_hw_scan_info *scan_info = &rtwdev->scan_info;
 	struct rtw_ch_switch_option cs_option = {0};
 	struct rtw_chan_list chan_list = {0};
 	int ret = 0;
@@ -2132,7 +2144,7 @@ int rtw_hw_scan_offload(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
 		return -EINVAL;
 
 	cs_option.switch_en = enable;
-	cs_option.back_op_en = rtwvif->net_type == RTW_NET_MGD_LINKED;
+	cs_option.back_op_en = scan_info->op_chan != 0;
 	if (enable) {
 		ret = rtw_hw_scan_prehandle(rtwdev, rtwvif, &chan_list);
 		if (ret)
@@ -2171,14 +2183,33 @@ void rtw_hw_scan_status_report(struct rtw_dev *rtwdev, struct sk_buff *skb)
 		rtw_dbg(rtwdev, RTW_DBG_HW_SCAN, "HW scan aborted with code: %d\n", rc);
 }
 
-void rtw_store_op_chan(struct rtw_dev *rtwdev)
+void rtw_store_op_chan(struct rtw_dev *rtwdev, bool backup)
 {
 	struct rtw_hw_scan_info *scan_info = &rtwdev->scan_info;
 	struct rtw_hal *hal = &rtwdev->hal;
+	u8 band;
+
+	if (backup) {
+		scan_info->op_chan = hal->current_channel;
+		scan_info->op_bw = hal->current_band_width;
+		scan_info->op_pri_ch_idx = hal->current_primary_channel_index;
+		scan_info->op_pri_ch = hal->primary_channel;
+	} else {
+		band = scan_info->op_chan > 14 ? RTW_BAND_5G : RTW_BAND_2G;
+		rtw_update_channel(rtwdev, scan_info->op_chan,
+				   scan_info->op_pri_ch,
+				   band, scan_info->op_bw);
+	}
+}
 
-	scan_info->op_chan = hal->current_channel;
-	scan_info->op_bw = hal->current_band_width;
-	scan_info->op_pri_ch_idx = hal->current_primary_channel_index;
+void rtw_clear_op_chan(struct rtw_dev *rtwdev)
+{
+	struct rtw_hw_scan_info *scan_info = &rtwdev->scan_info;
+
+	scan_info->op_chan = 0;
+	scan_info->op_bw = 0;
+	scan_info->op_pri_ch_idx = 0;
+	scan_info->op_pri_ch = 0;
 }
 
 static bool rtw_is_op_chan(struct rtw_dev *rtwdev, u8 channel)
@@ -2193,7 +2224,7 @@ void rtw_hw_scan_chan_switch(struct rtw_dev *rtwdev, struct sk_buff *skb)
 	struct rtw_hal *hal = &rtwdev->hal;
 	struct rtw_c2h_cmd *c2h;
 	enum rtw_scan_notify_id id;
-	u8 chan, status;
+	u8 chan, band, status;
 
 	if (!test_bit(RTW_FLAG_SCANNING, rtwdev->flags))
 		return;
@@ -2204,10 +2235,13 @@ void rtw_hw_scan_chan_switch(struct rtw_dev *rtwdev, struct sk_buff *skb)
 	status = GET_CHAN_SWITCH_STATUS(c2h->payload);
 
 	if (id == RTW_SCAN_NOTIFY_ID_POSTSWITCH) {
-		if (rtw_is_op_chan(rtwdev, chan))
+		band = chan > 14 ? RTW_BAND_5G : RTW_BAND_2G;
+		rtw_update_channel(rtwdev, chan, chan, band,
+				   RTW_CHANNEL_WIDTH_20);
+		if (rtw_is_op_chan(rtwdev, chan)) {
+			rtw_store_op_chan(rtwdev, false);
 			ieee80211_wake_queues(rtwdev->hw);
-		hal->current_channel = chan;
-		hal->current_band_type = chan > 14 ? RTW_BAND_5G : RTW_BAND_2G;
+		}
 	} else if (id == RTW_SCAN_NOTIFY_ID_PRESWITCH) {
 		if (IS_CH_5G_BAND(chan)) {
 			rtw_coex_switchband_notify(rtwdev, COEX_SWITCH_TO_5G);
@@ -2220,7 +2254,12 @@ void rtw_hw_scan_chan_switch(struct rtw_dev *rtwdev, struct sk_buff *skb)
 				chan_type = COEX_SWITCH_TO_24G_NOFORSCAN;
 			rtw_coex_switchband_notify(rtwdev, chan_type);
 		}
-		if (rtw_is_op_chan(rtwdev, chan))
+		/* The channel of C2H RTW_SCAN_NOTIFY_ID_PRESWITCH is next
+		 * channel that hardware will switch. We need to stop queue
+		 * if next channel is non-op channel.
+		 */
+		if (!rtw_is_op_chan(rtwdev, chan) &&
+		    rtw_is_op_chan(rtwdev, hal->current_channel))
 			ieee80211_stop_queues(rtwdev->hw);
 	}
 
diff --git a/drivers/net/wireless/realtek/rtw88/fw.h b/drivers/net/wireless/realtek/rtw88/fw.h
index 7a37675c61e8..a5a965803a3c 100644
--- a/drivers/net/wireless/realtek/rtw88/fw.h
+++ b/drivers/net/wireless/realtek/rtw88/fw.h
@@ -41,7 +41,8 @@
 #define RTW_EX_CH_INFO_HDR_SIZE		2
 #define RTW_SCAN_WIDTH			0
 #define RTW_PRI_CH_IDX			1
-#define RTW_PROBE_PG_CNT		2
+#define RTW_OLD_PROBE_PG_CNT		2
+#define RTW_PROBE_PG_CNT		4
 
 enum rtw_c2h_cmd_id {
 	C2H_CCX_TX_RPT = 0x03,
@@ -120,6 +121,10 @@ enum rtw_fw_feature {
 	FW_FEATURE_MAX = BIT(31),
 };
 
+enum rtw_fw_feature_ext {
+	FW_FEATURE_EXT_OLD_PAGE_NUM = BIT(0),
+};
+
 enum rtw_beacon_filter_offload_mode {
 	BCN_FILTER_OFFLOAD_MODE_0 = 0,
 	BCN_FILTER_OFFLOAD_MODE_1,
@@ -323,6 +328,11 @@ struct rtw_fw_hdr_legacy {
 	__le32 rsvd5;
 } __packed;
 
+#define RTW_FW_VER_CODE(ver, sub_ver, idx)	\
+	(((ver) << 16) | ((sub_ver) << 8) | (idx))
+#define RTW_FW_SUIT_VER_CODE(s)	\
+	RTW_FW_VER_CODE((s).version, (s).sub_version, (s).sub_index)
+
 /* C2H */
 #define GET_CCX_REPORT_SEQNUM_V0(c2h_payload)	(c2h_payload[6] & 0xfc)
 #define GET_CCX_REPORT_STATUS_V0(c2h_payload)	(c2h_payload[0] & 0xc0)
@@ -770,6 +780,12 @@ static inline bool rtw_fw_feature_check(struct rtw_fw_state *fw,
 	return !!(fw->feature & feature);
 }
 
+static inline bool rtw_fw_feature_ext_check(struct rtw_fw_state *fw,
+					    enum rtw_fw_feature_ext feature)
+{
+	return !!(fw->feature_ext & feature);
+}
+
 void rtw_fw_c2h_cmd_rx_irqsafe(struct rtw_dev *rtwdev, u32 pkt_offset,
 			       struct sk_buff *skb);
 void rtw_fw_c2h_cmd_handle(struct rtw_dev *rtwdev, struct sk_buff *skb);
@@ -831,7 +847,8 @@ int rtw_fw_dump_fifo(struct rtw_dev *rtwdev, u8 fifo_sel, u32 addr, u32 size,
 		     u32 *buffer);
 void rtw_fw_scan_notify(struct rtw_dev *rtwdev, bool start);
 void rtw_fw_adaptivity(struct rtw_dev *rtwdev);
-void rtw_store_op_chan(struct rtw_dev *rtwdev);
+void rtw_store_op_chan(struct rtw_dev *rtwdev, bool backup);
+void rtw_clear_op_chan(struct rtw_dev *rtwdev);
 void rtw_hw_scan_start(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
 		       struct ieee80211_scan_request *req);
 void rtw_hw_scan_complete(struct rtw_dev *rtwdev, struct ieee80211_vif *vif,
diff --git a/drivers/net/wireless/realtek/rtw88/mac.c b/drivers/net/wireless/realtek/rtw88/mac.c
index caf2603da2d6..52076e89d59a 100644
--- a/drivers/net/wireless/realtek/rtw88/mac.c
+++ b/drivers/net/wireless/realtek/rtw88/mac.c
@@ -243,7 +243,7 @@ static int rtw_pwr_seq_parser(struct rtw_dev *rtwdev,
 
 static int rtw_mac_power_switch(struct rtw_dev *rtwdev, bool pwr_on)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	const struct rtw_pwr_seq_cmd **pwr_seq;
 	u8 rpwm;
 	bool cur_pwr;
@@ -587,7 +587,7 @@ static int
 download_firmware_to_mem(struct rtw_dev *rtwdev, const u8 *data,
 			 u32 src, u32 dst, u32 size)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	u32 desc_size = chip->tx_pkt_desc_sz;
 	u8 first_part;
 	u32 mem_offset;
@@ -934,7 +934,7 @@ static u32 get_priority_queues(struct rtw_dev *rtwdev, u32 queues)
 static void __rtw_mac_flush_prio_queue(struct rtw_dev *rtwdev,
 				       u32 prio_queue, bool drop)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	const struct rtw_prioq_addr *addr;
 	bool wsize;
 	u16 avail_page, rsvd_page;
@@ -996,7 +996,7 @@ void rtw_mac_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop)
 
 static int txdma_queue_mapping(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	const struct rtw_rqpn *rqpn = NULL;
 	u16 txdma_pq_map = 0;
 
@@ -1037,8 +1037,8 @@ static int txdma_queue_mapping(struct rtw_dev *rtwdev)
 
 static int set_trx_fifo_info(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_fifo_conf *fifo = &rtwdev->fifo;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u16 cur_pg_addr;
 	u8 csi_buf_pg_num = chip->csi_buf_pg_num;
 
@@ -1092,8 +1092,8 @@ static int __priority_queue_cfg(struct rtw_dev *rtwdev,
 				const struct rtw_page_table *pg_tbl,
 				u16 pubq_num)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_fifo_conf *fifo = &rtwdev->fifo;
-	struct rtw_chip_info *chip = rtwdev->chip;
 
 	rtw_write16(rtwdev, REG_FIFOPAGE_INFO_1, pg_tbl->hq_num);
 	rtw_write16(rtwdev, REG_FIFOPAGE_INFO_2, pg_tbl->lq_num);
@@ -1123,8 +1123,8 @@ static int __priority_queue_cfg_legacy(struct rtw_dev *rtwdev,
 				       const struct rtw_page_table *pg_tbl,
 				       u16 pubq_num)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_fifo_conf *fifo = &rtwdev->fifo;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u32 val32;
 
 	val32 = BIT_RQPN_NE(pg_tbl->nq_num, pg_tbl->exq_num);
@@ -1149,8 +1149,8 @@ static int __priority_queue_cfg_legacy(struct rtw_dev *rtwdev,
 
 static int priority_queue_cfg(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_fifo_conf *fifo = &rtwdev->fifo;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	const struct rtw_page_table *pg_tbl = NULL;
 	u16 pubq_num;
 	int ret;
@@ -1277,7 +1277,7 @@ static int rtw_drv_info_cfg(struct rtw_dev *rtwdev)
 
 int rtw_mac_init(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	int ret;
 
 	ret = rtw_init_trx_cfg(rtwdev);
diff --git a/drivers/net/wireless/realtek/rtw88/mac80211.c b/drivers/net/wireless/realtek/rtw88/mac80211.c
index c7b98a0599d5..07578ccc4bab 100644
--- a/drivers/net/wireless/realtek/rtw88/mac80211.c
+++ b/drivers/net/wireless/realtek/rtw88/mac80211.c
@@ -101,7 +101,8 @@ static int rtw_ops_config(struct ieee80211_hw *hw, u32 changed)
 		rtw_set_channel(rtwdev);
 
 	if ((changed & IEEE80211_CONF_CHANGE_IDLE) &&
-	    (hw->conf.flags & IEEE80211_CONF_IDLE))
+	    (hw->conf.flags & IEEE80211_CONF_IDLE) &&
+	    !test_bit(RTW_FLAG_SCANNING, rtwdev->flags))
 		rtw_enter_ips(rtwdev);
 
 out:
@@ -377,7 +378,6 @@ static void rtw_ops_bss_info_changed(struct ieee80211_hw *hw,
 			rtw_coex_media_status_notify(rtwdev, vif->cfg.assoc);
 			if (rtw_bf_support)
 				rtw_bf_assoc(rtwdev, vif, conf);
-			rtw_store_op_chan(rtwdev);
 		} else {
 			rtw_leave_lps(rtwdev);
 			rtw_bf_disassoc(rtwdev, vif, conf);
@@ -395,6 +395,10 @@ static void rtw_ops_bss_info_changed(struct ieee80211_hw *hw,
 	if (changed & BSS_CHANGED_BSSID) {
 		ether_addr_copy(rtwvif->bssid, conf->bssid);
 		config |= PORT_SET_BSSID;
+		if (is_zero_ether_addr(rtwvif->bssid))
+			rtw_clear_op_chan(rtwdev);
+		else
+			rtw_store_op_chan(rtwdev, true);
 	}
 
 	if (changed & BSS_CHANGED_BEACON_INT) {
@@ -434,7 +438,7 @@ static int rtw_ops_start_ap(struct ieee80211_hw *hw,
 			    struct ieee80211_bss_conf *link_conf)
 {
 	struct rtw_dev *rtwdev = hw->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	mutex_lock(&rtwdev->mutex);
 	chip->ops->phy_calibration(rtwdev);
@@ -752,7 +756,7 @@ static int rtw_ops_set_antenna(struct ieee80211_hw *hw,
 			       u32 rx_antenna)
 {
 	struct rtw_dev *rtwdev = hw->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	int ret;
 
 	if (!chip->ops->set_antenna)
@@ -872,7 +876,9 @@ static int rtw_ops_set_sar_specs(struct ieee80211_hw *hw,
 {
 	struct rtw_dev *rtwdev = hw->priv;
 
+	mutex_lock(&rtwdev->mutex);
 	rtw_set_sar_specs(rtwdev, sar);
+	mutex_unlock(&rtwdev->mutex);
 
 	return 0;
 }
diff --git a/drivers/net/wireless/realtek/rtw88/main.c b/drivers/net/wireless/realtek/rtw88/main.c
index 76dc9da88f6c..67151dbf8384 100644
--- a/drivers/net/wireless/realtek/rtw88/main.c
+++ b/drivers/net/wireless/realtek/rtw88/main.c
@@ -353,7 +353,7 @@ struct rtw_fwcd_hdr {
 
 static int rtw_fwcd_prep(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_fwcd_desc *desc = &rtwdev->fw.fwcd_desc;
 	const struct rtw_fwcd_segs *segs = chip->fwcd_segs;
 	u32 prep_size = chip->fw_rxff_size + sizeof(struct rtw_fwcd_hdr);
@@ -675,67 +675,126 @@ void rtw_set_dtim_period(struct rtw_dev *rtwdev, int dtim_period)
 	rtw_write8(rtwdev, REG_DTIM_COUNTER_ROOT, dtim_period - 1);
 }
 
+void rtw_update_channel(struct rtw_dev *rtwdev, u8 center_channel,
+			u8 primary_channel, enum rtw_supported_band band,
+			enum rtw_bandwidth bandwidth)
+{
+	enum nl80211_band nl_band = rtw_hw_to_nl80211_band(band);
+	struct rtw_hal *hal = &rtwdev->hal;
+	u8 *cch_by_bw = hal->cch_by_bw;
+	u32 center_freq, primary_freq;
+	enum rtw_sar_bands sar_band;
+	u8 primary_channel_idx;
+
+	center_freq = ieee80211_channel_to_frequency(center_channel, nl_band);
+	primary_freq = ieee80211_channel_to_frequency(primary_channel, nl_band);
+
+	/* assign the center channel used while 20M bw is selected */
+	cch_by_bw[RTW_CHANNEL_WIDTH_20] = primary_channel;
+
+	/* assign the center channel used while current bw is selected */
+	cch_by_bw[bandwidth] = center_channel;
+
+	switch (bandwidth) {
+	case RTW_CHANNEL_WIDTH_20:
+	default:
+		primary_channel_idx = RTW_SC_DONT_CARE;
+		break;
+	case RTW_CHANNEL_WIDTH_40:
+		if (primary_freq > center_freq)
+			primary_channel_idx = RTW_SC_20_UPPER;
+		else
+			primary_channel_idx = RTW_SC_20_LOWER;
+		break;
+	case RTW_CHANNEL_WIDTH_80:
+		if (primary_freq > center_freq) {
+			if (primary_freq - center_freq == 10)
+				primary_channel_idx = RTW_SC_20_UPPER;
+			else
+				primary_channel_idx = RTW_SC_20_UPMOST;
+
+			/* assign the center channel used
+			 * while 40M bw is selected
+			 */
+			cch_by_bw[RTW_CHANNEL_WIDTH_40] = center_channel + 4;
+		} else {
+			if (center_freq - primary_freq == 10)
+				primary_channel_idx = RTW_SC_20_LOWER;
+			else
+				primary_channel_idx = RTW_SC_20_LOWEST;
+
+			/* assign the center channel used
+			 * while 40M bw is selected
+			 */
+			cch_by_bw[RTW_CHANNEL_WIDTH_40] = center_channel - 4;
+		}
+		break;
+	}
+
+	switch (center_channel) {
+	case 1 ... 14:
+		sar_band = RTW_SAR_BAND_0;
+		break;
+	case 36 ... 64:
+		sar_band = RTW_SAR_BAND_1;
+		break;
+	case 100 ... 144:
+		sar_band = RTW_SAR_BAND_3;
+		break;
+	case 149 ... 177:
+		sar_band = RTW_SAR_BAND_4;
+		break;
+	default:
+		WARN(1, "unknown ch(%u) to SAR band\n", center_channel);
+		sar_band = RTW_SAR_BAND_0;
+		break;
+	}
+
+	hal->current_primary_channel_index = primary_channel_idx;
+	hal->current_band_width = bandwidth;
+	hal->primary_channel = primary_channel;
+	hal->current_channel = center_channel;
+	hal->current_band_type = band;
+	hal->sar_band = sar_band;
+}
+
 void rtw_get_channel_params(struct cfg80211_chan_def *chandef,
 			    struct rtw_channel_params *chan_params)
 {
 	struct ieee80211_channel *channel = chandef->chan;
 	enum nl80211_chan_width width = chandef->width;
-	u8 *cch_by_bw = chan_params->cch_by_bw;
 	u32 primary_freq, center_freq;
 	u8 center_chan;
 	u8 bandwidth = RTW_CHANNEL_WIDTH_20;
-	u8 primary_chan_idx = 0;
-	u8 i;
 
 	center_chan = channel->hw_value;
 	primary_freq = channel->center_freq;
 	center_freq = chandef->center_freq1;
 
-	/* assign the center channel used while 20M bw is selected */
-	cch_by_bw[RTW_CHANNEL_WIDTH_20] = channel->hw_value;
-
 	switch (width) {
 	case NL80211_CHAN_WIDTH_20_NOHT:
 	case NL80211_CHAN_WIDTH_20:
 		bandwidth = RTW_CHANNEL_WIDTH_20;
-		primary_chan_idx = RTW_SC_DONT_CARE;
 		break;
 	case NL80211_CHAN_WIDTH_40:
 		bandwidth = RTW_CHANNEL_WIDTH_40;
-		if (primary_freq > center_freq) {
-			primary_chan_idx = RTW_SC_20_UPPER;
+		if (primary_freq > center_freq)
 			center_chan -= 2;
-		} else {
-			primary_chan_idx = RTW_SC_20_LOWER;
+		else
 			center_chan += 2;
-		}
 		break;
 	case NL80211_CHAN_WIDTH_80:
 		bandwidth = RTW_CHANNEL_WIDTH_80;
 		if (primary_freq > center_freq) {
-			if (primary_freq - center_freq == 10) {
-				primary_chan_idx = RTW_SC_20_UPPER;
+			if (primary_freq - center_freq == 10)
 				center_chan -= 2;
-			} else {
-				primary_chan_idx = RTW_SC_20_UPMOST;
+			else
 				center_chan -= 6;
-			}
-			/* assign the center channel used
-			 * while 40M bw is selected
-			 */
-			cch_by_bw[RTW_CHANNEL_WIDTH_40] = center_chan + 4;
 		} else {
-			if (center_freq - primary_freq == 10) {
-				primary_chan_idx = RTW_SC_20_LOWER;
+			if (center_freq - primary_freq == 10)
 				center_chan += 2;
-			} else {
-				primary_chan_idx = RTW_SC_20_LOWEST;
+			else
 				center_chan += 6;
-			}
-			/* assign the center channel used
-			 * while 40M bw is selected
-			 */
-			cch_by_bw[RTW_CHANNEL_WIDTH_40] = center_chan - 4;
 		}
 		break;
 	default:
@@ -745,60 +804,30 @@ void rtw_get_channel_params(struct cfg80211_chan_def *chandef,
 
 	chan_params->center_chan = center_chan;
 	chan_params->bandwidth = bandwidth;
-	chan_params->primary_chan_idx = primary_chan_idx;
-
-	/* assign the center channel used while current bw is selected */
-	cch_by_bw[bandwidth] = center_chan;
-
-	for (i = bandwidth + 1; i <= RTW_MAX_CHANNEL_WIDTH; i++)
-		cch_by_bw[i] = 0;
+	chan_params->primary_chan = channel->hw_value;
 }
 
 void rtw_set_channel(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct ieee80211_hw *hw = rtwdev->hw;
 	struct rtw_hal *hal = &rtwdev->hal;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_channel_params ch_param;
-	u8 center_chan, bandwidth, primary_chan_idx;
-	u8 i;
+	u8 center_chan, primary_chan, bandwidth, band;
 
 	rtw_get_channel_params(&hw->conf.chandef, &ch_param);
 	if (WARN(ch_param.center_chan == 0, "Invalid channel\n"))
 		return;
 
 	center_chan = ch_param.center_chan;
+	primary_chan = ch_param.primary_chan;
 	bandwidth = ch_param.bandwidth;
-	primary_chan_idx = ch_param.primary_chan_idx;
-
-	hal->current_band_width = bandwidth;
-	hal->current_channel = center_chan;
-	hal->current_primary_channel_index = primary_chan_idx;
-	hal->current_band_type = center_chan > 14 ? RTW_BAND_5G : RTW_BAND_2G;
-
-	switch (center_chan) {
-	case 1 ... 14:
-		hal->sar_band = RTW_SAR_BAND_0;
-		break;
-	case 36 ... 64:
-		hal->sar_band = RTW_SAR_BAND_1;
-		break;
-	case 100 ... 144:
-		hal->sar_band = RTW_SAR_BAND_3;
-		break;
-	case 149 ... 177:
-		hal->sar_band = RTW_SAR_BAND_4;
-		break;
-	default:
-		WARN(1, "unknown ch(%u) to SAR band\n", center_chan);
-		hal->sar_band = RTW_SAR_BAND_0;
-		break;
-	}
+	band = ch_param.center_chan > 14 ? RTW_BAND_5G : RTW_BAND_2G;
 
-	for (i = RTW_CHANNEL_WIDTH_20; i <= RTW_MAX_CHANNEL_WIDTH; i++)
-		hal->cch_by_bw[i] = ch_param.cch_by_bw[i];
+	rtw_update_channel(rtwdev, center_chan, primary_chan, band, bandwidth);
 
-	chip->ops->set_channel(rtwdev, center_chan, bandwidth, primary_chan_idx);
+	chip->ops->set_channel(rtwdev, center_chan, bandwidth,
+			       hal->current_primary_channel_index);
 
 	if (hal->current_band_type == RTW_BAND_5G) {
 		rtw_coex_switchband_notify(rtwdev, COEX_SWITCH_TO_5G);
@@ -821,7 +850,7 @@ void rtw_set_channel(struct rtw_dev *rtwdev)
 
 void rtw_chip_prepare_tx(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	if (rtwdev->need_rfk) {
 		rtwdev->need_rfk = false;
@@ -890,8 +919,8 @@ static u8 hw_bw_cap_to_bitamp(u8 bw_cap)
 
 static void rtw_hw_config_rf_ant_num(struct rtw_dev *rtwdev, u8 hw_ant_num)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_hal *hal = &rtwdev->hal;
-	struct rtw_chip_info *chip = rtwdev->chip;
 
 	if (hw_ant_num == EFUSE_HW_CAP_IGNORE ||
 	    hw_ant_num >= hal->rf_path_num)
@@ -1240,7 +1269,7 @@ void rtw_update_sta_info(struct rtw_dev *rtwdev, struct rtw_sta_info *si,
 
 static int rtw_wait_firmware_completion(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_fw_state *fw;
 
 	fw = &rtwdev->fw;
@@ -1261,7 +1290,7 @@ static int rtw_wait_firmware_completion(struct rtw_dev *rtwdev)
 static enum rtw_lps_deep_mode rtw_update_lps_deep_mode(struct rtw_dev *rtwdev,
 						       struct rtw_fw_state *fw)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	if (rtw_disable_lps_deep_mode || !chip->lps_deep_mode_supported ||
 	    !fw->feature)
@@ -1280,7 +1309,7 @@ static enum rtw_lps_deep_mode rtw_update_lps_deep_mode(struct rtw_dev *rtwdev,
 
 static int rtw_power_on(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_fw_state *fw = &rtwdev->fw;
 	bool wifi_only;
 	int ret;
@@ -1469,8 +1498,8 @@ void rtw_core_stop(struct rtw_dev *rtwdev)
 static void rtw_init_ht_cap(struct rtw_dev *rtwdev,
 			    struct ieee80211_sta_ht_cap *ht_cap)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
-	struct rtw_chip_info *chip = rtwdev->chip;
 
 	ht_cap->ht_supported = true;
 	ht_cap->cap = 0;
@@ -1552,8 +1581,23 @@ static void rtw_init_vht_cap(struct rtw_dev *rtwdev,
 	vht_cap->vht_mcs.tx_highest = highest;
 }
 
+static u16 rtw_get_max_scan_ie_len(struct rtw_dev *rtwdev)
+{
+	u16 len;
+
+	len = rtwdev->chip->max_scan_ie_len;
+
+	if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_SCAN_OFFLOAD) &&
+	    rtwdev->chip->id == RTW_CHIP_TYPE_8822C)
+		len = IEEE80211_MAX_DATA_LEN;
+	else if (rtw_fw_feature_ext_check(&rtwdev->fw, FW_FEATURE_EXT_OLD_PAGE_NUM))
+		len -= RTW_OLD_PROBE_PG_CNT * TX_PAGE_SIZE;
+
+	return len;
+}
+
 static void rtw_set_supported_band(struct ieee80211_hw *hw,
-				   struct rtw_chip_info *chip)
+				   const struct rtw_chip_info *chip)
 {
 	struct rtw_dev *rtwdev = hw->priv;
 	struct ieee80211_supported_band *sband;
@@ -1585,7 +1629,7 @@ err_out:
 }
 
 static void rtw_unset_supported_band(struct ieee80211_hw *hw,
-				     struct rtw_chip_info *chip)
+				     const struct rtw_chip_info *chip)
 {
 	kfree(hw->wiphy->bands[NL80211_BAND_2GHZ]);
 	kfree(hw->wiphy->bands[NL80211_BAND_5GHZ]);
@@ -1607,7 +1651,7 @@ static void rtw_vif_smps_iter(void *data, u8 *mac,
 
 void rtw_set_txrx_1ss(struct rtw_dev *rtwdev, bool txrx_1ss)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_hal *hal = &rtwdev->hal;
 
 	if (!chip->ops->config_txrx_mode || rtwdev->hal.txrx_1ss == txrx_1ss)
@@ -1631,6 +1675,10 @@ static void __update_firmware_feature(struct rtw_dev *rtwdev,
 
 	feature = le32_to_cpu(fw_hdr->feature);
 	fw->feature = feature & FW_FEATURE_SIG ? feature : 0;
+
+	if (rtwdev->chip->id == RTW_CHIP_TYPE_8822C &&
+	    RTW_FW_SUIT_VER_CODE(rtwdev->fw) < RTW_FW_VER_CODE(9, 9, 13))
+		fw->feature_ext |= FW_FEATURE_EXT_OLD_PAGE_NUM;
 }
 
 static void __update_firmware_info(struct rtw_dev *rtwdev,
@@ -1724,7 +1772,7 @@ static int rtw_load_firmware(struct rtw_dev *rtwdev, enum rtw_fw_type type)
 
 static int rtw_chip_parameter_setup(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_hal *hal = &rtwdev->hal;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 
@@ -1982,7 +2030,7 @@ static void rtw_stats_init(struct rtw_dev *rtwdev)
 
 int rtw_core_init(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_coex *coex = &rtwdev->coex;
 	int ret;
 
@@ -2045,7 +2093,7 @@ int rtw_core_init(struct rtw_dev *rtwdev)
 	ret = rtw_load_firmware(rtwdev, RTW_NORMAL_FW);
 	if (ret) {
 		rtw_warn(rtwdev, "no firmware loaded\n");
-		return ret;
+		goto out;
 	}
 
 	if (chip->wow_fw_name) {
@@ -2055,11 +2103,15 @@ int rtw_core_init(struct rtw_dev *rtwdev)
 			wait_for_completion(&rtwdev->fw.completion);
 			if (rtwdev->fw.firmware)
 				release_firmware(rtwdev->fw.firmware);
-			return ret;
+			goto out;
 		}
 	}
 
 	return 0;
+
+out:
+	destroy_workqueue(rtwdev->tx_wq);
+	return ret;
 }
 EXPORT_SYMBOL(rtw_core_init);
 
@@ -2136,7 +2188,7 @@ int rtw_register_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw)
 
 	hw->wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR;
 	hw->wiphy->max_scan_ssids = RTW_SCAN_MAX_SSIDS;
-	hw->wiphy->max_scan_ie_len = RTW_SCAN_MAX_IE_LEN;
+	hw->wiphy->max_scan_ie_len = rtw_get_max_scan_ie_len(rtwdev);
 
 	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
 	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_SCAN_RANDOM_SN);
@@ -2180,7 +2232,7 @@ EXPORT_SYMBOL(rtw_register_hw);
 
 void rtw_unregister_hw(struct rtw_dev *rtwdev, struct ieee80211_hw *hw)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	ieee80211_unregister_hw(hw);
 	rtw_unset_supported_band(hw, chip);
diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h
index 7db627fc26be..bccd7b28f60c 100644
--- a/drivers/net/wireless/realtek/rtw88/main.h
+++ b/drivers/net/wireless/realtek/rtw88/main.h
@@ -22,7 +22,6 @@
 #define MAX_PG_CAM_BACKUP_NUM		8
 
 #define RTW_SCAN_MAX_SSIDS		4
-#define RTW_SCAN_MAX_IE_LEN		128
 
 #define RTW_MAX_PATTERN_NUM		12
 #define RTW_MAX_PATTERN_MASK_SIZE	16
@@ -33,6 +32,7 @@
 #define RFREG_MASK			0xfffff
 #define INV_RF_DATA			0xffffffff
 #define TX_PAGE_SIZE_SHIFT		7
+#define TX_PAGE_SIZE			(1 << TX_PAGE_SIZE_SHIFT)
 
 #define RTW_CHANNEL_WIDTH_MAX		3
 #define RTW_RF_PATH_MAX			4
@@ -510,12 +510,8 @@ struct rtw_timer_list {
 
 struct rtw_channel_params {
 	u8 center_chan;
+	u8 primary_chan;
 	u8 bandwidth;
-	u8 primary_chan_idx;
-	/* center channel by different available bandwidth,
-	 * val of (bw > current bandwidth) is invalid
-	 */
-	u8 cch_by_bw[RTW_MAX_CHANNEL_WIDTH + 1];
 };
 
 struct rtw_hw_reg {
@@ -1232,6 +1228,7 @@ struct rtw_chip_info {
 	const char *wow_fw_name;
 	const struct wiphy_wowlan_support *wowlan_stub;
 	const u8 max_sched_scan_ssids;
+	const u16 max_scan_ie_len;
 
 	/* coex paras */
 	u32 coex_para_ver;
@@ -1853,6 +1850,7 @@ struct rtw_fw_state {
 	u8 sub_index;
 	u16 h2c_version;
 	u32 feature;
+	u32 feature_ext;
 };
 
 enum rtw_sar_sources {
@@ -1896,6 +1894,7 @@ struct rtw_hal {
 	u8 current_primary_channel_index;
 	u8 current_band_width;
 	u8 current_band_type;
+	u8 primary_channel;
 
 	/* center channel for different available bandwidth,
 	 * val of (bw > current_band_width) is invalid
@@ -1967,6 +1966,7 @@ struct rtw_hw_scan_info {
 	struct ieee80211_vif *scanning_vif;
 	u8 probe_pg_size;
 	u8 op_pri_ch_idx;
+	u8 op_pri_ch;
 	u8 op_chan;
 	u8 op_bw;
 };
@@ -1978,7 +1978,7 @@ struct rtw_dev {
 	struct rtw_hci hci;
 
 	struct rtw_hw_scan_info scan_info;
-	struct rtw_chip_info *chip;
+	const struct rtw_chip_info *chip;
 	struct rtw_hal hal;
 	struct rtw_fifo_conf fifo;
 	struct rtw_fw_state fw;
@@ -2132,6 +2132,20 @@ static inline int rtw_chip_dump_fw_crash(struct rtw_dev *rtwdev)
 	return 0;
 }
 
+static inline
+enum nl80211_band rtw_hw_to_nl80211_band(enum rtw_supported_band hw_band)
+{
+	switch (hw_band) {
+	default:
+	case RTW_BAND_2G:
+		return NL80211_BAND_2GHZ;
+	case RTW_BAND_5G:
+		return NL80211_BAND_5GHZ;
+	case RTW_BAND_60G:
+		return NL80211_BAND_60GHZ;
+	}
+}
+
 void rtw_set_rx_freq_band(struct rtw_rx_pkt_stat *pkt_stat, u8 channel);
 void rtw_set_dtim_period(struct rtw_dev *rtwdev, int dtim_period);
 void rtw_get_channel_params(struct cfg80211_chan_def *chandef,
@@ -2173,4 +2187,7 @@ int rtw_dump_fw(struct rtw_dev *rtwdev, const u32 ocp_src, u32 size,
 		u32 fwcd_item);
 int rtw_dump_reg(struct rtw_dev *rtwdev, const u32 addr, const u32 size);
 void rtw_set_txrx_1ss(struct rtw_dev *rtwdev, bool config_1ss);
+void rtw_update_channel(struct rtw_dev *rtwdev, u8 center_channel,
+			u8 primary_channel, enum rtw_supported_band band,
+			enum rtw_bandwidth bandwidth);
 #endif
diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c
index 24d5695363d3..0975d27240e4 100644
--- a/drivers/net/wireless/realtek/rtw88/pci.c
+++ b/drivers/net/wireless/realtek/rtw88/pci.c
@@ -322,7 +322,7 @@ static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev)
 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
 	struct rtw_pci_tx_ring *tx_ring;
 	struct rtw_pci_rx_ring *rx_ring;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0;
 	int tx_desc_size, rx_desc_size;
 	u32 len;
@@ -721,7 +721,7 @@ static void rtw_pci_dma_check(struct rtw_dev *rtwdev,
 			      u32 idx)
 {
 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_pci_rx_buffer_desc *buf_desc;
 	u32 desc_sz = chip->rx_buf_desc_sz;
 	u16 total_pkt_size;
@@ -834,7 +834,7 @@ static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev,
 				 struct sk_buff *skb, u8 queue)
 {
 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_pci_tx_ring *ring;
 	struct rtw_pci_tx_data *tx_data;
 	dma_addr_t dma;
@@ -1073,7 +1073,7 @@ static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev,
 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci,
 			   u8 hw_queue, u32 limit)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct napi_struct *napi = &rtwpci->napi;
 	struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU];
 	struct rtw_rx_pkt_stat pkt_stat;
@@ -1425,7 +1425,7 @@ static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter)
 
 static void rtw_pci_link_cfg(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
 	struct pci_dev *pdev = rtwpci->pdev;
 	u16 link_ctrl;
@@ -1467,7 +1467,7 @@ static void rtw_pci_link_cfg(struct rtw_dev *rtwdev)
 
 static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	switch (chip->id) {
 	case RTW_CHIP_TYPE_8822C:
@@ -1483,7 +1483,7 @@ static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev)
 static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev)
 {
 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct pci_dev *pdev = rtwpci->pdev;
 	const struct rtw_intf_phy_para *para;
 	u16 cut;
@@ -1538,7 +1538,7 @@ static int __maybe_unused rtw_pci_suspend(struct device *dev)
 {
 	struct ieee80211_hw *hw = dev_get_drvdata(dev);
 	struct rtw_dev *rtwdev = hw->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 
 	if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6)
@@ -1550,7 +1550,7 @@ static int __maybe_unused rtw_pci_resume(struct device *dev)
 {
 	struct ieee80211_hw *hw = dev_get_drvdata(dev);
 	struct rtw_dev *rtwdev = hw->priv;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 
 	if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6)
@@ -1717,8 +1717,7 @@ static void rtw_pci_napi_init(struct rtw_dev *rtwdev)
 	struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv;
 
 	init_dummy_netdev(&rtwpci->netdev);
-	netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll,
-		       NAPI_POLL_WEIGHT);
+	netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll);
 }
 
 static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev)
@@ -1848,7 +1847,7 @@ void rtw_pci_shutdown(struct pci_dev *pdev)
 {
 	struct ieee80211_hw *hw = pci_get_drvdata(pdev);
 	struct rtw_dev *rtwdev;
-	struct rtw_chip_info *chip;
+	const struct rtw_chip_info *chip;
 
 	if (!hw)
 		return;
diff --git a/drivers/net/wireless/realtek/rtw88/phy.c b/drivers/net/wireless/realtek/rtw88/phy.c
index 8982e0c98dac..bd7d05e08084 100644
--- a/drivers/net/wireless/realtek/rtw88/phy.c
+++ b/drivers/net/wireless/realtek/rtw88/phy.c
@@ -138,7 +138,7 @@ EXPORT_SYMBOL(rtw_phy_set_edcca_th);
 
 void rtw_phy_adaptivity_set_mode(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
 
 	/* turn off in debugfs for debug usage */
@@ -165,7 +165,7 @@ void rtw_phy_adaptivity_set_mode(struct rtw_dev *rtwdev)
 
 static void rtw_phy_adaptivity_init(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	rtw_phy_adaptivity_set_mode(rtwdev);
 	if (chip->ops->adaptivity_init)
@@ -180,7 +180,7 @@ static void rtw_phy_adaptivity(struct rtw_dev *rtwdev)
 
 static void rtw_phy_cfo_init(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	if (chip->ops->cfo_init)
 		chip->ops->cfo_init(rtwdev);
@@ -199,7 +199,7 @@ static void rtw_phy_tx_path_div_init(struct rtw_dev *rtwdev)
 
 void rtw_phy_init(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
 	u32 addr, mask;
 
@@ -226,7 +226,7 @@ EXPORT_SYMBOL(rtw_phy_init);
 
 void rtw_phy_dig_write(struct rtw_dev *rtwdev, u8 igi)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_hal *hal = &rtwdev->hal;
 	u32 addr, mask;
 	u8 path;
@@ -245,7 +245,7 @@ void rtw_phy_dig_write(struct rtw_dev *rtwdev, u8 igi)
 
 static void rtw_phy_stat_false_alarm(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	chip->ops->false_alarm_statistics(rtwdev);
 }
@@ -603,7 +603,7 @@ static void rtw_phy_rrsr_update(struct rtw_dev *rtwdev)
 
 static void rtw_phy_dpk_track(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	if (chip->ops->dpk_track)
 		chip->ops->dpk_track(rtwdev);
@@ -659,7 +659,7 @@ EXPORT_SYMBOL(rtw_phy_parsing_cfo);
 
 static void rtw_phy_cfo_track(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	if (chip->ops->cfo_track)
 		chip->ops->cfo_track(rtwdev);
@@ -720,8 +720,8 @@ static u8 rtw_phy_cck_pd_lv(struct rtw_dev *rtwdev)
 
 static void rtw_phy_cck_pd(struct rtw_dev *rtwdev)
 {
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_dm_info *dm_info = &rtwdev->dm_info;
-	struct rtw_chip_info *chip = rtwdev->chip;
 	u32 cck_fa = dm_info->cck_fa_cnt;
 	u8 level;
 
@@ -816,23 +816,18 @@ static u8 rtw_phy_linear_2_db(u64 linear)
 	u8 j;
 	u32 dB;
 
-	if (linear >= db_invert_table[11][7])
-		return 96; /* maximum 96 dB */
-
 	for (i = 0; i < 12; i++) {
-		if (i <= 2 && (linear << FRAC_BITS) <= db_invert_table[i][7])
-			break;
-		else if (i > 2 && linear <= db_invert_table[i][7])
-			break;
+		for (j = 0; j < 8; j++) {
+			if (i <= 2 && (linear << FRAC_BITS) <= db_invert_table[i][j])
+				goto cnt;
+			else if (i > 2 && linear <= db_invert_table[i][j])
+				goto cnt;
+		}
 	}
 
-	for (j = 0; j < 8; j++) {
-		if (i <= 2 && (linear << FRAC_BITS) <= db_invert_table[i][j])
-			break;
-		else if (i > 2 && linear <= db_invert_table[i][j])
-			break;
-	}
+	return 96; /* maximum 96 dB */
 
+cnt:
 	if (j == 0 && i == 0)
 		goto end;
 
@@ -900,7 +895,7 @@ u32 rtw_phy_read_rf(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path,
 		    u32 addr, u32 mask)
 {
 	struct rtw_hal *hal = &rtwdev->hal;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	const u32 *base_addr = chip->rf_base_addr;
 	u32 val, direct_addr;
 
@@ -923,7 +918,7 @@ u32 rtw_phy_read_rf_sipi(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path,
 			 u32 addr, u32 mask)
 {
 	struct rtw_hal *hal = &rtwdev->hal;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	const struct rtw_rf_sipi_addr *rf_sipi_addr;
 	const struct rtw_rf_sipi_addr *rf_sipi_addr_a;
 	u32 val32;
@@ -972,8 +967,8 @@ bool rtw_phy_write_rf_reg_sipi(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path,
 			       u32 addr, u32 mask, u32 data)
 {
 	struct rtw_hal *hal = &rtwdev->hal;
-	struct rtw_chip_info *chip = rtwdev->chip;
-	u32 *sipi_addr = chip->rf_sipi_addr;
+	const struct rtw_chip_info *chip = rtwdev->chip;
+	const u32 *sipi_addr = chip->rf_sipi_addr;
 	u32 data_and_addr;
 	u32 old_data = 0;
 	u32 shift;
@@ -1012,7 +1007,7 @@ bool rtw_phy_write_rf_reg(struct rtw_dev *rtwdev, enum rtw_rf_path rf_path,
 			  u32 addr, u32 mask, u32 data)
 {
 	struct rtw_hal *hal = &rtwdev->hal;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	const u32 *base_addr = chip->rf_base_addr;
 	u32 direct_addr;
 
@@ -1747,7 +1742,7 @@ EXPORT_SYMBOL(rtw_phy_cfg_rf);
 
 static void rtw_load_rfk_table(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_dpk_info *dpk_info = &rtwdev->dm_info.dpk_info;
 
 	if (!chip->rfk_init_tbl)
@@ -1766,7 +1761,7 @@ static void rtw_load_rfk_table(struct rtw_dev *rtwdev)
 
 void rtw_phy_load_tables(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	u8 rf_path;
 
 	rtw_load_table(rtwdev, chip->mac_tbl);
@@ -1875,7 +1870,7 @@ static u8 rtw_get_channel_group(u8 channel, u8 rate)
 
 static s8 rtw_phy_get_dis_dpd_by_rate_diff(struct rtw_dev *rtwdev, u16 rate)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	s8 dpd_diff = 0;
 
 	if (!chip->en_dis_dpd)
@@ -1909,7 +1904,7 @@ static u8 rtw_phy_get_2g_tx_power_index(struct rtw_dev *rtwdev,
 					enum rtw_bandwidth bandwidth,
 					u8 rate, u8 group)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	u8 tx_power;
 	bool mcs_rate;
 	bool above_2ss;
@@ -1956,7 +1951,7 @@ static u8 rtw_phy_get_5g_tx_power_index(struct rtw_dev *rtwdev,
 					enum rtw_bandwidth bandwidth,
 					u8 rate, u8 group)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	u8 tx_power;
 	u8 upper, lower;
 	bool mcs_rate;
@@ -2209,7 +2204,7 @@ static void rtw_phy_set_tx_power_level_by_path(struct rtw_dev *rtwdev,
 
 void rtw_phy_set_tx_power_level(struct rtw_dev *rtwdev, u8 channel)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_hal *hal = &rtwdev->hal;
 	u8 path;
 
@@ -2484,7 +2479,7 @@ static void rtw_phy_set_tx_path_by_reg(struct rtw_dev *rtwdev,
 {
 	struct rtw_path_div *path_div = &rtwdev->dm_path_div;
 	enum rtw_bb_path tx_path_sel_cck = tx_path_sel_1ss;
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	if (tx_path_sel_1ss == path_div->current_tx_path)
 		return;
@@ -2539,7 +2534,7 @@ static void rtw_phy_tx_path_diversity_2ss(struct rtw_dev *rtwdev)
 
 void rtw_phy_tx_path_diversity(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 
 	if (!chip->path_div_supported)
 		return;
diff --git a/drivers/net/wireless/realtek/rtw88/phy.h b/drivers/net/wireless/realtek/rtw88/phy.h
index b6c5ae60a462..ccfcbd3ced03 100644
--- a/drivers/net/wireless/realtek/rtw88/phy.h
+++ b/drivers/net/wireless/realtek/rtw88/phy.h
@@ -114,7 +114,7 @@ const struct rtw_table name ## _tbl = {			\
 
 static inline const struct rtw_rfe_def *rtw_get_rfe_def(struct rtw_dev *rtwdev)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct rtw_efuse *efuse = &rtwdev->efuse;
 	const struct rtw_rfe_def *rfe_def = NULL;
 
diff --git a/drivers/net/wireless/realtek/rtw88/ps.c b/drivers/net/wireless/realtek/rtw88/ps.c
index bfa64c038f5f..c93da743681f 100644
--- a/drivers/net/wireless/realtek/rtw88/ps.c
+++ b/drivers/net/wireless/realtek/rtw88/ps.c
@@ -19,14 +19,14 @@ static int rtw_ips_pwr_up(struct rtw_dev *rtwdev)
 		rtw_err(rtwdev, "leave idle state failed\n");
 
 	rtw_set_channel(rtwdev);
-	clear_bit(RTW_FLAG_INACTIVE_PS, rtwdev->flags);
 
 	return ret;
 }
 
 int rtw_enter_ips(struct rtw_dev *rtwdev)
 {
-	set_bit(RTW_FLAG_INACTIVE_PS, rtwdev->flags);
+	if (test_and_set_bit(RTW_FLAG_INACTIVE_PS, rtwdev->flags))
+		return 0;
 
 	rtw_coex_ips_notify(rtwdev, COEX_IPS_ENTER);
 
@@ -50,6 +50,9 @@ int rtw_leave_ips(struct rtw_dev *rtwdev)
 {
 	int ret;
 
+	if (!test_and_clear_bit(RTW_FLAG_INACTIVE_PS, rtwdev->flags))
+		return 0;
+
 	rtw_hci_link_ps(rtwdev, false);
 
 	ret = rtw_ips_pwr_up(rtwdev);
diff --git a/drivers/net/wireless/realtek/rtw88/regd.c b/drivers/net/wireless/realtek/rtw88/regd.c
index 315c2b193e92..2f547cbcf6da 100644
--- a/drivers/net/wireless/realtek/rtw88/regd.c
+++ b/drivers/net/wireless/realtek/rtw88/regd.c
@@ -479,6 +479,7 @@ void rtw_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request)
 	rtw_dbg(rtwdev, RTW_DBG_REGD, "regd state: %d -> %d\n",
 		rtwdev->regd.state, next_regd.state);
 
+	mutex_lock(&rtwdev->mutex);
 	rtwdev->regd = next_regd;
 	rtw_dbg_regd_dump(rtwdev, "get alpha2 %c%c from initiator %d: ",
 			  request->alpha2[0],
@@ -487,6 +488,7 @@ void rtw_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request)
 
 	rtw_phy_adaptivity_set_mode(rtwdev);
 	rtw_phy_set_tx_power_level(rtwdev, hal->current_channel);
+	mutex_unlock(&rtwdev->mutex);
 }
 
 u8 rtw_regd_get(struct rtw_dev *rtwdev)
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8723d.c b/drivers/net/wireless/realtek/rtw88/rtw8723d.c
index 993bd6b1d723..0a4f770fcbb7 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8723d.c
+++ b/drivers/net/wireless/realtek/rtw88/rtw8723d.c
@@ -2720,7 +2720,7 @@ const struct rtw_chip_info rtw8723d_hw_spec = {
 	.max_power_index = 0x3f,
 	.csi_buf_pg_num = 0,
 	.band = RTW_BAND_2G,
-	.page_size = 128,
+	.page_size = TX_PAGE_SIZE,
 	.dig_min = 0x20,
 	.ht_supported = true,
 	.vht_supported = false,
@@ -2748,6 +2748,7 @@ const struct rtw_chip_info rtw8723d_hw_spec = {
 	.pwr_track_tbl = &rtw8723d_rtw_pwr_track_tbl,
 	.iqk_threshold = 8,
 	.ampdu_density = IEEE80211_HT_MPDU_DENSITY_16,
+	.max_scan_ie_len = IEEE80211_MAX_DATA_LEN,
 
 	.coex_para_ver = 0x2007022f,
 	.bt_desired_ver = 0x2f,
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8821c.c b/drivers/net/wireless/realtek/rtw88/rtw8821c.c
index 025262a8970e..9afdc5ce86b4 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8821c.c
+++ b/drivers/net/wireless/realtek/rtw88/rtw8821c.c
@@ -1898,7 +1898,7 @@ const struct rtw_chip_info rtw8821c_hw_spec = {
 	.max_power_index = 0x3f,
 	.csi_buf_pg_num = 0,
 	.band = RTW_BAND_2G | RTW_BAND_5G,
-	.page_size = 128,
+	.page_size = TX_PAGE_SIZE,
 	.dig_min = 0x1c,
 	.ht_supported = true,
 	.vht_supported = true,
@@ -1926,6 +1926,7 @@ const struct rtw_chip_info rtw8821c_hw_spec = {
 	.bfer_su_max_num = 2,
 	.bfer_mu_max_num = 1,
 	.ampdu_density = IEEE80211_HT_MPDU_DENSITY_2,
+	.max_scan_ie_len = IEEE80211_MAX_DATA_LEN,
 
 	.coex_para_ver = 0x19092746,
 	.bt_desired_ver = 0x46,
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822b.c b/drivers/net/wireless/realtek/rtw88/rtw8822b.c
index 321848870561..690e35c98f6e 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8822b.c
+++ b/drivers/net/wireless/realtek/rtw88/rtw8822b.c
@@ -2517,7 +2517,7 @@ const struct rtw_chip_info rtw8822b_hw_spec = {
 	.max_power_index = 0x3f,
 	.csi_buf_pg_num = 0,
 	.band = RTW_BAND_2G | RTW_BAND_5G,
-	.page_size = 128,
+	.page_size = TX_PAGE_SIZE,
 	.dig_min = 0x1c,
 	.ht_supported = true,
 	.vht_supported = true,
@@ -2549,6 +2549,7 @@ const struct rtw_chip_info rtw8822b_hw_spec = {
 	.l2h_th_ini_cs = 10 + EDCCA_IGI_BASE,
 	.l2h_th_ini_ad = -14 + EDCCA_IGI_BASE,
 	.ampdu_density = IEEE80211_HT_MPDU_DENSITY_2,
+	.max_scan_ie_len = IEEE80211_MAX_DATA_LEN,
 
 	.coex_para_ver = 0x20070206,
 	.bt_desired_ver = 0x6,
diff --git a/drivers/net/wireless/realtek/rtw88/rtw8822c.c b/drivers/net/wireless/realtek/rtw88/rtw8822c.c
index 09f9e4adcf34..fccb15dfb959 100644
--- a/drivers/net/wireless/realtek/rtw88/rtw8822c.c
+++ b/drivers/net/wireless/realtek/rtw88/rtw8822c.c
@@ -5330,7 +5330,7 @@ const struct rtw_chip_info rtw8822c_hw_spec = {
 	.max_power_index = 0x7f,
 	.csi_buf_pg_num = 50,
 	.band = RTW_BAND_2G | RTW_BAND_5G,
-	.page_size = 128,
+	.page_size = TX_PAGE_SIZE,
 	.dig_min = 0x20,
 	.default_1ss_tx_path = BB_PATH_A,
 	.path_div_supported = true,
@@ -5375,6 +5375,7 @@ const struct rtw_chip_info rtw8822c_hw_spec = {
 	.wowlan_stub = &rtw_wowlan_stub_8822c,
 	.max_sched_scan_ssids = 4,
 #endif
+	.max_scan_ie_len = (RTW_PROBE_PG_CNT - 1) * TX_PAGE_SIZE,
 	.coex_para_ver = 0x22020720,
 	.bt_desired_ver = 0x20,
 	.scbd_support = true,
diff --git a/drivers/net/wireless/realtek/rtw88/tx.c b/drivers/net/wireless/realtek/rtw88/tx.c
index 60d40a5c2c6a..ab39245e9c2f 100644
--- a/drivers/net/wireless/realtek/rtw88/tx.c
+++ b/drivers/net/wireless/realtek/rtw88/tx.c
@@ -384,7 +384,7 @@ void rtw_tx_pkt_info_update(struct rtw_dev *rtwdev,
 			    struct ieee80211_sta *sta,
 			    struct sk_buff *skb)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	struct rtw_sta_info *si;
@@ -424,7 +424,7 @@ void rtw_tx_rsvd_page_pkt_info_update(struct rtw_dev *rtwdev,
 				      struct sk_buff *skb,
 				      enum rtw_rsvd_packet_type type)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	bool bmc;
 
@@ -475,7 +475,7 @@ rtw_tx_write_data_rsvd_page_get(struct rtw_dev *rtwdev,
 				struct rtw_tx_pkt_info *pkt_info,
 				u8 *buf, u32 size)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct sk_buff *skb;
 	u32 tx_pkt_desc_sz;
 	u32 length;
@@ -501,7 +501,7 @@ rtw_tx_write_data_h2c_get(struct rtw_dev *rtwdev,
 			  struct rtw_tx_pkt_info *pkt_info,
 			  u8 *buf, u32 size)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	struct sk_buff *skb;
 	u32 tx_pkt_desc_sz;
 	u32 length;
diff --git a/drivers/net/wireless/realtek/rtw88/util.c b/drivers/net/wireless/realtek/rtw88/util.c
index 2c515af214e7..cdfd66a85075 100644
--- a/drivers/net/wireless/realtek/rtw88/util.c
+++ b/drivers/net/wireless/realtek/rtw88/util.c
@@ -23,7 +23,7 @@ EXPORT_SYMBOL(check_hw_ready);
 
 bool ltecoex_read_reg(struct rtw_dev *rtwdev, u16 offset, u32 *val)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	const struct rtw_ltecoex_addr *ltecoex = chip->ltecoex_addr;
 
 	if (!check_hw_ready(rtwdev, ltecoex->ctrl, LTECOEX_READY, 1))
@@ -37,7 +37,7 @@ bool ltecoex_read_reg(struct rtw_dev *rtwdev, u16 offset, u32 *val)
 
 bool ltecoex_reg_write(struct rtw_dev *rtwdev, u16 offset, u32 value)
 {
-	struct rtw_chip_info *chip = rtwdev->chip;
+	const struct rtw_chip_info *chip = rtwdev->chip;
 	const struct rtw_ltecoex_addr *ltecoex = chip->ltecoex_addr;
 
 	if (!check_hw_ready(rtwdev, ltecoex->ctrl, LTECOEX_READY, 1))
diff --git a/drivers/net/wireless/realtek/rtw89/Makefile b/drivers/net/wireless/realtek/rtw89/Makefile
index 3006482d25c7..a87f2aff4def 100644
--- a/drivers/net/wireless/realtek/rtw89/Makefile
+++ b/drivers/net/wireless/realtek/rtw89/Makefile
@@ -12,6 +12,7 @@ rtw89_core-y += core.o \
 		sar.o \
 		coex.o \
 		ps.o \
+		chan.o \
 		ser.o
 
 obj-$(CONFIG_RTW89_8852A) += rtw89_8852a.o
diff --git a/drivers/net/wireless/realtek/rtw89/chan.c b/drivers/net/wireless/realtek/rtw89/chan.c
new file mode 100644
index 000000000000..a4f61c2f6512
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw89/chan.c
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2020-2022  Realtek Corporation
+ */
+
+#include "chan.h"
+#include "debug.h"
+
+static enum rtw89_subband rtw89_get_subband_type(enum rtw89_band band,
+						 u8 center_chan)
+{
+	switch (band) {
+	default:
+	case RTW89_BAND_2G:
+		switch (center_chan) {
+		default:
+		case 1 ... 14:
+			return RTW89_CH_2G;
+		}
+	case RTW89_BAND_5G:
+		switch (center_chan) {
+		default:
+		case 36 ... 64:
+			return RTW89_CH_5G_BAND_1;
+		case 100 ... 144:
+			return RTW89_CH_5G_BAND_3;
+		case 149 ... 177:
+			return RTW89_CH_5G_BAND_4;
+		}
+	case RTW89_BAND_6G:
+		switch (center_chan) {
+		default:
+		case 1 ... 29:
+			return RTW89_CH_6G_BAND_IDX0;
+		case 33 ... 61:
+			return RTW89_CH_6G_BAND_IDX1;
+		case 65 ... 93:
+			return RTW89_CH_6G_BAND_IDX2;
+		case 97 ... 125:
+			return RTW89_CH_6G_BAND_IDX3;
+		case 129 ... 157:
+			return RTW89_CH_6G_BAND_IDX4;
+		case 161 ... 189:
+			return RTW89_CH_6G_BAND_IDX5;
+		case 193 ... 221:
+			return RTW89_CH_6G_BAND_IDX6;
+		case 225 ... 253:
+			return RTW89_CH_6G_BAND_IDX7;
+		}
+	}
+}
+
+static enum rtw89_sc_offset rtw89_get_primary_chan_idx(enum rtw89_bandwidth bw,
+						       u32 center_freq,
+						       u32 primary_freq)
+{
+	u8 primary_chan_idx;
+	u32 offset;
+
+	switch (bw) {
+	default:
+	case RTW89_CHANNEL_WIDTH_20:
+		primary_chan_idx = RTW89_SC_DONT_CARE;
+		break;
+	case RTW89_CHANNEL_WIDTH_40:
+		if (primary_freq > center_freq)
+			primary_chan_idx = RTW89_SC_20_UPPER;
+		else
+			primary_chan_idx = RTW89_SC_20_LOWER;
+		break;
+	case RTW89_CHANNEL_WIDTH_80:
+	case RTW89_CHANNEL_WIDTH_160:
+		if (primary_freq > center_freq) {
+			offset = (primary_freq - center_freq - 10) / 20;
+			primary_chan_idx = RTW89_SC_20_UPPER + offset * 2;
+		} else {
+			offset = (center_freq - primary_freq - 10) / 20;
+			primary_chan_idx = RTW89_SC_20_LOWER + offset * 2;
+		}
+		break;
+	}
+
+	return primary_chan_idx;
+}
+
+void rtw89_chan_create(struct rtw89_chan *chan, u8 center_chan, u8 primary_chan,
+		       enum rtw89_band band, enum rtw89_bandwidth bandwidth)
+{
+	enum nl80211_band nl_band = rtw89_hw_to_nl80211_band(band);
+	u32 center_freq, primary_freq;
+
+	memset(chan, 0, sizeof(*chan));
+	chan->channel = center_chan;
+	chan->primary_channel = primary_chan;
+	chan->band_type = band;
+	chan->band_width = bandwidth;
+
+	center_freq = ieee80211_channel_to_frequency(center_chan, nl_band);
+	primary_freq = ieee80211_channel_to_frequency(primary_chan, nl_band);
+
+	chan->freq = center_freq;
+	chan->subband_type = rtw89_get_subband_type(band, center_chan);
+	chan->pri_ch_idx = rtw89_get_primary_chan_idx(bandwidth, center_freq,
+						      primary_freq);
+}
+
+bool rtw89_assign_entity_chan(struct rtw89_dev *rtwdev,
+			      enum rtw89_sub_entity_idx idx,
+			      const struct rtw89_chan *new)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	struct rtw89_chan *chan = &hal->chan[idx];
+	struct rtw89_chan_rcd *rcd = &hal->chan_rcd[idx];
+	bool band_changed;
+
+	rcd->prev_primary_channel = chan->primary_channel;
+	rcd->prev_band_type = chan->band_type;
+	band_changed = new->band_type != chan->band_type;
+
+	*chan = *new;
+	return band_changed;
+}
+
+static void __rtw89_config_entity_chandef(struct rtw89_dev *rtwdev,
+					  enum rtw89_sub_entity_idx idx,
+					  const struct cfg80211_chan_def *chandef,
+					  bool from_stack)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	hal->chandef[idx] = *chandef;
+
+	if (from_stack)
+		set_bit(idx, hal->entity_map);
+}
+
+void rtw89_config_entity_chandef(struct rtw89_dev *rtwdev,
+				 enum rtw89_sub_entity_idx idx,
+				 const struct cfg80211_chan_def *chandef)
+{
+	__rtw89_config_entity_chandef(rtwdev, idx, chandef, true);
+}
+
+static void rtw89_config_default_chandef(struct rtw89_dev *rtwdev)
+{
+	struct cfg80211_chan_def chandef = {0};
+
+	rtw89_get_default_chandef(&chandef);
+	__rtw89_config_entity_chandef(rtwdev, RTW89_SUB_ENTITY_0, &chandef, false);
+}
+
+void rtw89_entity_init(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	bitmap_zero(hal->entity_map, NUM_OF_RTW89_SUB_ENTITY);
+	rtw89_config_default_chandef(rtwdev);
+}
+
+enum rtw89_entity_mode rtw89_entity_recalc(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	enum rtw89_entity_mode mode;
+	u8 weight;
+
+	weight = bitmap_weight(hal->entity_map, NUM_OF_RTW89_SUB_ENTITY);
+	switch (weight) {
+	default:
+		rtw89_warn(rtwdev, "unknown ent chan weight: %d\n", weight);
+		bitmap_zero(hal->entity_map, NUM_OF_RTW89_SUB_ENTITY);
+		fallthrough;
+	case 0:
+		rtw89_config_default_chandef(rtwdev);
+		fallthrough;
+	case 1:
+		mode = RTW89_ENTITY_MODE_SCC;
+		break;
+	}
+
+	rtw89_set_entity_mode(rtwdev, mode);
+	return mode;
+}
+
+int rtw89_chanctx_ops_add(struct rtw89_dev *rtwdev,
+			  struct ieee80211_chanctx_conf *ctx)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	struct rtw89_chanctx_cfg *cfg = (struct rtw89_chanctx_cfg *)ctx->drv_priv;
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	u8 idx;
+
+	idx = find_first_zero_bit(hal->entity_map, NUM_OF_RTW89_SUB_ENTITY);
+	if (idx >= chip->support_chanctx_num)
+		return -ENOENT;
+
+	rtw89_config_entity_chandef(rtwdev, idx, &ctx->def);
+	rtw89_set_channel(rtwdev);
+	cfg->idx = idx;
+	return 0;
+}
+
+void rtw89_chanctx_ops_remove(struct rtw89_dev *rtwdev,
+			      struct ieee80211_chanctx_conf *ctx)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	struct rtw89_chanctx_cfg *cfg = (struct rtw89_chanctx_cfg *)ctx->drv_priv;
+
+	clear_bit(cfg->idx, hal->entity_map);
+	rtw89_set_channel(rtwdev);
+}
+
+void rtw89_chanctx_ops_change(struct rtw89_dev *rtwdev,
+			      struct ieee80211_chanctx_conf *ctx,
+			      u32 changed)
+{
+	struct rtw89_chanctx_cfg *cfg = (struct rtw89_chanctx_cfg *)ctx->drv_priv;
+	u8 idx = cfg->idx;
+
+	if (changed & IEEE80211_CHANCTX_CHANGE_WIDTH) {
+		rtw89_config_entity_chandef(rtwdev, idx, &ctx->def);
+		rtw89_set_channel(rtwdev);
+	}
+}
+
+int rtw89_chanctx_ops_assign_vif(struct rtw89_dev *rtwdev,
+				 struct rtw89_vif *rtwvif,
+				 struct ieee80211_chanctx_conf *ctx)
+{
+	return 0;
+}
+
+void rtw89_chanctx_ops_unassign_vif(struct rtw89_dev *rtwdev,
+				    struct rtw89_vif *rtwvif,
+				    struct ieee80211_chanctx_conf *ctx)
+{
+}
diff --git a/drivers/net/wireless/realtek/rtw89/chan.h b/drivers/net/wireless/realtek/rtw89/chan.h
new file mode 100644
index 000000000000..ecbd4503bead
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw89/chan.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+ * Copyright(c) 2020-2022  Realtek Corporation
+ */
+
+#ifndef __RTW89_CHAN_H__
+#define __RTW89_CHAN_H__
+
+#include "core.h"
+
+static inline bool rtw89_get_entity_state(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	return READ_ONCE(hal->entity_active);
+}
+
+static inline void rtw89_set_entity_state(struct rtw89_dev *rtwdev, bool active)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	WRITE_ONCE(hal->entity_active, active);
+}
+
+static inline
+enum rtw89_entity_mode rtw89_get_entity_mode(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	return READ_ONCE(hal->entity_mode);
+}
+
+static inline void rtw89_set_entity_mode(struct rtw89_dev *rtwdev,
+					 enum rtw89_entity_mode mode)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	WRITE_ONCE(hal->entity_mode, mode);
+}
+
+void rtw89_chan_create(struct rtw89_chan *chan, u8 center_chan, u8 primary_chan,
+		       enum rtw89_band band, enum rtw89_bandwidth bandwidth);
+bool rtw89_assign_entity_chan(struct rtw89_dev *rtwdev,
+			      enum rtw89_sub_entity_idx idx,
+			      const struct rtw89_chan *new);
+void rtw89_config_entity_chandef(struct rtw89_dev *rtwdev,
+				 enum rtw89_sub_entity_idx idx,
+				 const struct cfg80211_chan_def *chandef);
+void rtw89_entity_init(struct rtw89_dev *rtwdev);
+enum rtw89_entity_mode rtw89_entity_recalc(struct rtw89_dev *rtwdev);
+int rtw89_chanctx_ops_add(struct rtw89_dev *rtwdev,
+			  struct ieee80211_chanctx_conf *ctx);
+void rtw89_chanctx_ops_remove(struct rtw89_dev *rtwdev,
+			      struct ieee80211_chanctx_conf *ctx);
+void rtw89_chanctx_ops_change(struct rtw89_dev *rtwdev,
+			      struct ieee80211_chanctx_conf *ctx,
+			      u32 changed);
+int rtw89_chanctx_ops_assign_vif(struct rtw89_dev *rtwdev,
+				 struct rtw89_vif *rtwvif,
+				 struct ieee80211_chanctx_conf *ctx);
+void rtw89_chanctx_ops_unassign_vif(struct rtw89_dev *rtwdev,
+				    struct rtw89_vif *rtwvif,
+				    struct ieee80211_chanctx_conf *ctx);
+
+#endif
diff --git a/drivers/net/wireless/realtek/rtw89/coex.c b/drivers/net/wireless/realtek/rtw89/coex.c
index 683854bba217..bbdfa9ac203c 100644
--- a/drivers/net/wireless/realtek/rtw89/coex.c
+++ b/drivers/net/wireless/realtek/rtw89/coex.c
@@ -9,6 +9,7 @@
 #include "ps.h"
 #include "reg.h"
 
+#define RTW89_COEX_VERSION 0x06030013
 #define FCXDEF_STEP 50 /* MUST <= FCXMAX_STEP and match with wl fw*/
 
 enum btc_fbtc_tdma_template {
@@ -77,21 +78,21 @@ static const struct rtw89_btc_fbtc_tdma t_def[] = {
 
 static const struct rtw89_btc_fbtc_slot s_def[] = {
 	[CXST_OFF]	= __DEF_FBTC_SLOT(100, 0x55555555, SLOT_MIX),
-	[CXST_B2W]	= __DEF_FBTC_SLOT(5,   0x5a5a5a5a, SLOT_ISO),
-	[CXST_W1]	= __DEF_FBTC_SLOT(70,  0x5a5a5a5a, SLOT_ISO),
-	[CXST_W2]	= __DEF_FBTC_SLOT(70,  0x5a5a5aaa, SLOT_ISO),
-	[CXST_W2B]	= __DEF_FBTC_SLOT(15,  0x5a5a5a5a, SLOT_ISO),
-	[CXST_B1]	= __DEF_FBTC_SLOT(100, 0x55555555, SLOT_MIX),
-	[CXST_B2]	= __DEF_FBTC_SLOT(7,   0x6a5a5a5a, SLOT_MIX),
-	[CXST_B3]	= __DEF_FBTC_SLOT(5,   0x55555555, SLOT_MIX),
-	[CXST_B4]	= __DEF_FBTC_SLOT(50,  0x55555555, SLOT_MIX),
-	[CXST_LK]	= __DEF_FBTC_SLOT(20,  0x5a5a5a5a, SLOT_ISO),
+	[CXST_B2W]	= __DEF_FBTC_SLOT(5,   0xea5a5a5a, SLOT_ISO),
+	[CXST_W1]	= __DEF_FBTC_SLOT(70,  0xea5a5a5a, SLOT_ISO),
+	[CXST_W2]	= __DEF_FBTC_SLOT(70,  0xea5a5aaa, SLOT_ISO),
+	[CXST_W2B]	= __DEF_FBTC_SLOT(15,  0xea5a5a5a, SLOT_ISO),
+	[CXST_B1]	= __DEF_FBTC_SLOT(100, 0xe5555555, SLOT_MIX),
+	[CXST_B2]	= __DEF_FBTC_SLOT(7,   0xea5a5a5a, SLOT_MIX),
+	[CXST_B3]	= __DEF_FBTC_SLOT(5,   0xe5555555, SLOT_MIX),
+	[CXST_B4]	= __DEF_FBTC_SLOT(50,  0xe5555555, SLOT_MIX),
+	[CXST_LK]	= __DEF_FBTC_SLOT(20,  0xea5a5a5a, SLOT_ISO),
 	[CXST_BLK]	= __DEF_FBTC_SLOT(250, 0x55555555, SLOT_MIX),
-	[CXST_E2G]	= __DEF_FBTC_SLOT(20,  0x6a5a5a5a, SLOT_MIX),
+	[CXST_E2G]	= __DEF_FBTC_SLOT(20,  0xea5a5a5a, SLOT_MIX),
 	[CXST_E5G]	= __DEF_FBTC_SLOT(20,  0xffffffff, SLOT_MIX),
-	[CXST_EBT]	= __DEF_FBTC_SLOT(20,  0x55555555, SLOT_MIX),
+	[CXST_EBT]	= __DEF_FBTC_SLOT(20,  0xe5555555, SLOT_MIX),
 	[CXST_ENULL]	= __DEF_FBTC_SLOT(7,   0xaaaaaaaa, SLOT_ISO),
-	[CXST_WLK]	= __DEF_FBTC_SLOT(250, 0x6a5a6a5a, SLOT_MIX),
+	[CXST_WLK]	= __DEF_FBTC_SLOT(250, 0xea5a5a5a, SLOT_MIX),
 	[CXST_W1FDD]	= __DEF_FBTC_SLOT(35,  0xfafafafa, SLOT_ISO),
 	[CXST_B1FDD]	= __DEF_FBTC_SLOT(100, 0xffffffff, SLOT_MIX),
 };
@@ -99,13 +100,13 @@ static const struct rtw89_btc_fbtc_slot s_def[] = {
 static const u32 cxtbl[] = {
 	0xffffffff, /* 0 */
 	0xaaaaaaaa, /* 1 */
-	0x55555555, /* 2 */
-	0x66555555, /* 3 */
-	0x66556655, /* 4 */
+	0xe5555555, /* 2 */
+	0xee555555, /* 3 */
+	0xd5555555, /* 4 */
 	0x5a5a5a5a, /* 5 */
-	0x5a5a5aaa, /* 6 */
-	0xaa5a5a5a, /* 7 */
-	0x6a5a5a5a, /* 8 */
+	0xfa5a5a5a, /* 6 */
+	0xda5a5a5a, /* 7 */
+	0xea5a5a5a, /* 8 */
 	0x6a5a5aaa, /* 9 */
 	0x6a5a6a5a, /* 10 */
 	0x6a5a6aaa, /* 11 */
@@ -261,6 +262,12 @@ enum btc_cx_poicy_type {
 	/* TDMA off + pri: WL_Hi-Tx > BT_Hi_Rx, BT_Hi > WL > BT_Lo */
 	BTC_CXP_OFF_BWB1 = (BTC_CXP_OFF << 8) | 7,
 
+	/* TDMA off + pri: WL_Hi-Tx > BT, BT_Hi > other-WL > BT_Lo */
+	BTC_CXP_OFF_BWB2 = (BTC_CXP_OFF << 8) | 8,
+
+	/* TDMA off + pri: WL_Hi-Tx = BT */
+	BTC_CXP_OFF_BWB3 = (BTC_CXP_OFF << 8) | 9,
+
 	/* TDMA off+Bcn-Protect + pri: WL_Hi-Tx > BT_Hi_Rx, BT_Hi > WL > BT_Lo*/
 	BTC_CXP_OFFB_BWB0 = (BTC_CXP_OFFB << 8) | 0,
 
@@ -270,6 +277,21 @@ enum btc_cx_poicy_type {
 	/* TDMA off + Ext-Ctrl + pri: E2G-slot block all BT */
 	BTC_CXP_OFFE_DEF2 = (BTC_CXP_OFFE << 8) | 1,
 
+	/* TDMA off + Ext-Ctrl + pri: default */
+	BTC_CXP_OFFE_2GBWISOB = (BTC_CXP_OFFE << 8) | 2,
+
+	/* TDMA off + Ext-Ctrl + pri: E2G-slot block all BT */
+	BTC_CXP_OFFE_2GISOB = (BTC_CXP_OFFE << 8) | 3,
+
+	/* TDMA off + Ext-Ctrl + pri: E2G-slot WL > BT */
+	BTC_CXP_OFFE_2GBWMIXB = (BTC_CXP_OFFE << 8) | 4,
+
+	/* TDMA off + Ext-Ctrl + pri: E2G/EBT-slot WL > BT */
+	BTC_CXP_OFFE_WL = (BTC_CXP_OFFE << 8) | 5,
+
+	/* TDMA off + Ext-Ctrl + pri: default */
+	BTC_CXP_OFFE_2GBWMIXB2 = (BTC_CXP_OFFE << 8) | 6,
+
 	/* TDMA Fix slot-0: W1:B1 = 30:30 */
 	BTC_CXP_FIX_TD3030 = (BTC_CXP_FIX << 8) | 0,
 
@@ -300,6 +322,9 @@ enum btc_cx_poicy_type {
 	/* TDMA Fix slot-9: W1:B1 = 40:20 */
 	BTC_CXP_FIX_TD4020 = (BTC_CXP_FIX << 8) | 9,
 
+	/* TDMA Fix slot-9: W1:B1 = 40:10 */
+	BTC_CXP_FIX_TD4010ISO = (BTC_CXP_FIX << 8) | 10,
+
 	/* PS-TDMA Fix slot-0: W1:B1 = 30:30 */
 	BTC_CXP_PFIX_TD3030 = (BTC_CXP_PFIX << 8) | 0,
 
@@ -322,25 +347,25 @@ enum btc_cx_poicy_type {
 	BTC_CXP_PFIX_TDW1B1 = (BTC_CXP_PFIX << 8) | 6,
 
 	/* TDMA Auto slot-0: W1:B1 = 50:200 */
-	BTC_CXP_AUTO_TD50200 = (BTC_CXP_AUTO << 8) | 0,
+	BTC_CXP_AUTO_TD50B1 = (BTC_CXP_AUTO << 8) | 0,
 
 	/* TDMA Auto slot-1: W1:B1 = 60:200 */
-	BTC_CXP_AUTO_TD60200 = (BTC_CXP_AUTO << 8) | 1,
+	BTC_CXP_AUTO_TD60B1 = (BTC_CXP_AUTO << 8) | 1,
 
 	/* TDMA Auto slot-2: W1:B1 = 20:200 */
-	BTC_CXP_AUTO_TD20200 = (BTC_CXP_AUTO << 8) | 2,
+	BTC_CXP_AUTO_TD20B1 = (BTC_CXP_AUTO << 8) | 2,
 
 	/* TDMA Auto slot-3: W1:B1 = user-define */
 	BTC_CXP_AUTO_TDW1B1 = (BTC_CXP_AUTO << 8) | 3,
 
 	/* PS-TDMA Auto slot-0: W1:B1 = 50:200 */
-	BTC_CXP_PAUTO_TD50200 = (BTC_CXP_PAUTO << 8) | 0,
+	BTC_CXP_PAUTO_TD50B1 = (BTC_CXP_PAUTO << 8) | 0,
 
 	/* PS-TDMA Auto slot-1: W1:B1 = 60:200 */
-	BTC_CXP_PAUTO_TD60200 = (BTC_CXP_PAUTO << 8) | 1,
+	BTC_CXP_PAUTO_TD60B1 = (BTC_CXP_PAUTO << 8) | 1,
 
 	/* PS-TDMA Auto slot-2: W1:B1 = 20:200 */
-	BTC_CXP_PAUTO_TD20200 = (BTC_CXP_PAUTO << 8) | 2,
+	BTC_CXP_PAUTO_TD20B1 = (BTC_CXP_PAUTO << 8) | 2,
 
 	/* PS-TDMA Auto slot-3: W1:B1 = user-define */
 	BTC_CXP_PAUTO_TDW1B1 = (BTC_CXP_PAUTO << 8) | 3,
@@ -412,7 +437,7 @@ enum btc_w2b_scoreboard {
 	BTC_WSCB_TDMA = BIT(9),
 	BTC_WSCB_FIX2M = BIT(10),
 	BTC_WSCB_WLRFK = BIT(11),
-	BTC_WSCB_BTRFK_GNT = BIT(12), /* not used, use mailbox to inform BT */
+	BTC_WSCB_RXSCAN_PRI = BIT(12),
 	BTC_WSCB_BT_HILNA = BIT(13),
 	BTC_WSCB_BTLOG = BIT(14),
 	BTC_WSCB_ALL = GENMASK(23, 0),
@@ -434,6 +459,16 @@ enum btc_wl_link_mode {
 	BTC_WLINK_MAX
 };
 
+enum btc_wl_mrole_type {
+	BTC_WLMROLE_NONE = 0x0,
+	BTC_WLMROLE_STA_GC,
+	BTC_WLMROLE_STA_GC_NOA,
+	BTC_WLMROLE_STA_GO,
+	BTC_WLMROLE_STA_GO_NOA,
+	BTC_WLMROLE_STA_STA,
+	BTC_WLMROLE_MAX
+};
+
 enum btc_bt_hid_type {
 	BTC_HID_218 = BIT(0),
 	BTC_HID_418 = BIT(1),
@@ -460,6 +495,11 @@ enum btc_gnt_state {
 	BTC_GNT_MAX
 };
 
+enum btc_ctr_path {
+	BTC_CTRL_BY_BT = 0,
+	BTC_CTRL_BY_WL
+};
+
 enum btc_wl_max_tx_time {
 	BTC_MAX_TX_TIME_L1 = 500,
 	BTC_MAX_TX_TIME_L2 = 1000,
@@ -531,6 +571,7 @@ enum btc_reason_and_action {
 #define BTC_FREERUN_ANTISO_MIN 30
 #define BTC_TDMA_BTHID_MAX 2
 #define BTC_BLINK_NOCONNECT 0
+#define BTC_B1_MAX 250 /* unit ms */
 
 static void _run_coex(struct rtw89_dev *rtwdev,
 		      enum btc_reason_and_action reason);
@@ -551,8 +592,10 @@ static void _send_fw_cmd(struct rtw89_dev *rtwdev, u8 h2c_class, u8 h2c_func,
 			    "[BTC], %s(): return by btc not init!!\n", __func__);
 		pfwinfo->cnt_h2c_fail++;
 		return;
-	} else if ((wl->status.map.rf_off_pre == 1 && wl->status.map.rf_off == 1) ||
-		   (wl->status.map.lps_pre == 1 && wl->status.map.lps == 1)) {
+	} else if ((wl->status.map.rf_off_pre == BTC_LPS_RF_OFF &&
+		    wl->status.map.rf_off == BTC_LPS_RF_OFF) ||
+		   (wl->status.map.lps_pre == BTC_LPS_RF_OFF &&
+		    wl->status.map.lps == BTC_LPS_RF_OFF)) {
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
 			    "[BTC], %s(): return by wl off!!\n", __func__);
 		pfwinfo->cnt_h2c_fail++;
@@ -616,8 +659,6 @@ static void _reset_btc_var(struct rtw89_dev *rtwdev, u8 type)
 		memset(&btc->mdinfo, 0, sizeof(btc->mdinfo));
 }
 
-#define BTC_FWINFO_BUF 1024
-
 #define BTC_RPT_HDR_SIZE 3
 #define BTC_CHK_WLSLOT_DRIFT_MAX 15
 #define BTC_CHK_HANG_MAX 3
@@ -869,18 +910,24 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			   struct rtw89_btc_btf_fwinfo *pfwinfo,
 			   u8 *prptbuf, u32 index)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_dm *dm = &btc->dm;
 	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
-	struct rtw89_btc_fbtc_rpt_ctrl *prpt = NULL;
+	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
+	struct rtw89_btc_fbtc_rpt_ctrl *prpt;
+	struct rtw89_btc_fbtc_rpt_ctrl_v1 *prpt_v1;
 	struct rtw89_btc_fbtc_cysta *pcysta_le32 = NULL;
+	struct rtw89_btc_fbtc_cysta_v1 *pcysta_v1 = NULL;
 	struct rtw89_btc_fbtc_cysta_cpu pcysta[1];
 	struct rtw89_btc_prpt *btc_prpt = NULL;
 	struct rtw89_btc_fbtc_slot *rtp_slot = NULL;
-	u8 rpt_type = 0, *rpt_content = NULL, *pfinfo = NULL;
-	u16 wl_slot_set = 0;
+	void *rpt_content = NULL, *pfinfo = NULL;
+	u8 rpt_type = 0;
+	u16 wl_slot_set = 0, wl_slot_real = 0;
 	u32 trace_step = btc->ctrl.trace_step, rpt_len = 0, diff_t;
+	u32 cnt_leak_slot = 0, bt_slot_real = 0, cnt_rx_imr = 0;
 	u8 i;
 
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
@@ -904,100 +951,129 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 	switch (rpt_type) {
 	case BTC_RPT_TYPE_CTRL:
 		pcinfo = &pfwinfo->rpt_ctrl.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_ctrl.finfo);
-		pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo);
-		pcinfo->req_fver = BTCRPT_VER;
+		if (chip->chip_id == RTL8852A) {
+			pfinfo = &pfwinfo->rpt_ctrl.finfo;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo);
+		} else {
+			pfinfo = &pfwinfo->rpt_ctrl.finfo_v1;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_ctrl.finfo_v1);
+		}
+		pcinfo->req_fver = chip->fcxbtcrpt_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_TDMA:
 		pcinfo = &pfwinfo->rpt_fbtc_tdma.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_tdma.finfo);
-		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_tdma.finfo);
-		pcinfo->req_fver = FCXTDMA_VER;
+		if (chip->chip_id == RTL8852A) {
+			pfinfo = &pfwinfo->rpt_fbtc_tdma.finfo;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_tdma.finfo);
+		} else {
+			pfinfo = &pfwinfo->rpt_fbtc_tdma.finfo_v1;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_tdma.finfo_v1);
+		}
+		pcinfo->req_fver = chip->fcxtdma_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_SLOT:
 		pcinfo = &pfwinfo->rpt_fbtc_slots.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_slots.finfo);
+		pfinfo = &pfwinfo->rpt_fbtc_slots.finfo;
 		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_slots.finfo);
-		pcinfo->req_fver = FCXSLOTS_VER;
+		pcinfo->req_fver = chip->fcxslots_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_CYSTA:
 		pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_cysta.finfo);
-		pcysta_le32 = &pfwinfo->rpt_fbtc_cysta.finfo;
-		rtw89_btc_fbtc_cysta_to_cpu(pcysta_le32, pcysta);
-		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo);
-		pcinfo->req_fver = FCXCYSTA_VER;
+		if (chip->chip_id == RTL8852A) {
+			pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo;
+			pcysta_le32 = &pfwinfo->rpt_fbtc_cysta.finfo;
+			rtw89_btc_fbtc_cysta_to_cpu(pcysta_le32, pcysta);
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo);
+		} else {
+			pfinfo = &pfwinfo->rpt_fbtc_cysta.finfo_v1;
+			pcysta_v1 = &pfwinfo->rpt_fbtc_cysta.finfo_v1;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_cysta.finfo_v1);
+		}
+		pcinfo->req_fver = chip->fcxcysta_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_STEP:
 		pcinfo = &pfwinfo->rpt_fbtc_step.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_step.finfo);
-		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_step.finfo.step[0]) *
-				  trace_step + 8;
-		pcinfo->req_fver = FCXSTEP_VER;
+		if (chip->chip_id == RTL8852A) {
+			pfinfo = &pfwinfo->rpt_fbtc_step.finfo;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_step.finfo.step[0]) *
+					  trace_step +
+					  offsetof(struct rtw89_btc_fbtc_steps, step);
+		} else {
+			pfinfo = &pfwinfo->rpt_fbtc_step.finfo_v1;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_step.finfo_v1.step[0]) *
+					  trace_step +
+					  offsetof(struct rtw89_btc_fbtc_steps_v1, step);
+		}
+		pcinfo->req_fver = chip->fcxstep_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_NULLSTA:
 		pcinfo = &pfwinfo->rpt_fbtc_nullsta.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_nullsta.finfo);
-		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_nullsta.finfo);
-		pcinfo->req_fver = FCXNULLSTA_VER;
+		if (chip->chip_id == RTL8852A) {
+			pfinfo = &pfwinfo->rpt_fbtc_nullsta.finfo;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_nullsta.finfo);
+		} else {
+			pfinfo = &pfwinfo->rpt_fbtc_nullsta.finfo_v1;
+			pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_nullsta.finfo_v1);
+		}
+		pcinfo->req_fver = chip->fcxnullsta_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_MREG:
 		pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_mregval.finfo);
+		pfinfo = &pfwinfo->rpt_fbtc_mregval.finfo;
 		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_mregval.finfo);
-		pcinfo->req_fver = FCXMREG_VER;
+		pcinfo->req_fver = chip->fcxmreg_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_GPIO_DBG:
 		pcinfo = &pfwinfo->rpt_fbtc_gpio_dbg.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_gpio_dbg.finfo);
+		pfinfo = &pfwinfo->rpt_fbtc_gpio_dbg.finfo;
 		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_gpio_dbg.finfo);
-		pcinfo->req_fver = FCXGPIODBG_VER;
+		pcinfo->req_fver = chip->fcxgpiodbg_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_BT_VER:
 		pcinfo = &pfwinfo->rpt_fbtc_btver.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btver.finfo);
+		pfinfo = &pfwinfo->rpt_fbtc_btver.finfo;
 		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btver.finfo);
-		pcinfo->req_fver = FCX_BTVER_VER;
+		pcinfo->req_fver = chip->fcxbtver_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_BT_SCAN:
 		pcinfo = &pfwinfo->rpt_fbtc_btscan.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btscan.finfo);
+		pfinfo = &pfwinfo->rpt_fbtc_btscan.finfo;
 		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btscan.finfo);
-		pcinfo->req_fver = FCX_BTSCAN_VER;
+		pcinfo->req_fver = chip->fcxbtscan_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_BT_AFH:
 		pcinfo = &pfwinfo->rpt_fbtc_btafh.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btafh.finfo);
+		pfinfo = &pfwinfo->rpt_fbtc_btafh.finfo;
 		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btafh.finfo);
-		pcinfo->req_fver = FCX_BTAFH_VER;
+		pcinfo->req_fver = chip->fcxbtafh_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
 	case BTC_RPT_TYPE_BT_DEVICE:
 		pcinfo = &pfwinfo->rpt_fbtc_btdev.cinfo;
-		pfinfo = (u8 *)(&pfwinfo->rpt_fbtc_btdev.finfo);
+		pfinfo = &pfwinfo->rpt_fbtc_btdev.finfo;
 		pcinfo->req_len = sizeof(pfwinfo->rpt_fbtc_btdev.finfo);
-		pcinfo->req_fver = FCX_BTDEVINFO_VER;
+		pcinfo->req_fver = chip->fcxbtdevinfo_ver;
 		pcinfo->rx_len = rpt_len;
 		pcinfo->rx_cnt++;
 		break;
@@ -1026,7 +1102,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 	memcpy(pfinfo, rpt_content, pcinfo->req_len);
 	pcinfo->valid = 1;
 
-	if (rpt_type == BTC_RPT_TYPE_TDMA) {
+	if (rpt_type == BTC_RPT_TYPE_TDMA && chip->chip_id == RTL8852A) {
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
 			    "[BTC], %s(): check %d %zu\n", __func__,
 			    BTC_DCNT_TDMA_NONSYNC, sizeof(dm->tdma_now));
@@ -1039,7 +1115,8 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 				    dm->tdma_now.type, dm->tdma_now.rxflctrl,
 				    dm->tdma_now.txpause, dm->tdma_now.wtgle_n,
 				    dm->tdma_now.leak_n, dm->tdma_now.ext_ctrl,
-				    dm->tdma_now.rsvd0, dm->tdma_now.rsvd1);
+				    dm->tdma_now.rxflctrl_role,
+				    dm->tdma_now.option_ctrl);
 
 			rtw89_debug(rtwdev, RTW89_DBG_BTC,
 				    "[BTC], %s(): %d rpt_fbtc_tdma %x %x %x %x %x %x %x %x\n",
@@ -1050,14 +1127,46 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 				    pfwinfo->rpt_fbtc_tdma.finfo.wtgle_n,
 				    pfwinfo->rpt_fbtc_tdma.finfo.leak_n,
 				    pfwinfo->rpt_fbtc_tdma.finfo.ext_ctrl,
-				    pfwinfo->rpt_fbtc_tdma.finfo.rsvd0,
-				    pfwinfo->rpt_fbtc_tdma.finfo.rsvd1);
+				    pfwinfo->rpt_fbtc_tdma.finfo.rxflctrl_role,
+				    pfwinfo->rpt_fbtc_tdma.finfo.option_ctrl);
 		}
 
 		_chk_btc_err(rtwdev, BTC_DCNT_TDMA_NONSYNC,
 			     memcmp(&dm->tdma_now,
 				    &pfwinfo->rpt_fbtc_tdma.finfo,
 				    sizeof(dm->tdma_now)));
+	} else if (rpt_type == BTC_RPT_TYPE_TDMA) {
+		rtw89_debug(rtwdev, RTW89_DBG_BTC,
+			    "[BTC], %s(): check %d %zu\n", __func__,
+			    BTC_DCNT_TDMA_NONSYNC, sizeof(dm->tdma_now));
+
+		if (memcmp(&dm->tdma_now, &pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma,
+			   sizeof(dm->tdma_now)) != 0) {
+			rtw89_debug(rtwdev, RTW89_DBG_BTC,
+				    "[BTC], %s(): %d tdma_now %x %x %x %x %x %x %x %x\n",
+				    __func__, BTC_DCNT_TDMA_NONSYNC,
+				    dm->tdma_now.type, dm->tdma_now.rxflctrl,
+				    dm->tdma_now.txpause, dm->tdma_now.wtgle_n,
+				    dm->tdma_now.leak_n, dm->tdma_now.ext_ctrl,
+				    dm->tdma_now.rxflctrl_role,
+				    dm->tdma_now.option_ctrl);
+			rtw89_debug(rtwdev, RTW89_DBG_BTC,
+				    "[BTC], %s(): %d rpt_fbtc_tdma %x %x %x %x %x %x %x %x\n",
+				    __func__, BTC_DCNT_TDMA_NONSYNC,
+				    pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.type,
+				    pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.rxflctrl,
+				    pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.txpause,
+				    pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.wtgle_n,
+				    pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.leak_n,
+				    pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.ext_ctrl,
+				    pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.rxflctrl_role,
+				    pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma.option_ctrl);
+		}
+
+		_chk_btc_err(rtwdev, BTC_DCNT_TDMA_NONSYNC,
+			     memcmp(&dm->tdma_now,
+				    &pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma,
+				    sizeof(dm->tdma_now)));
 	}
 
 	if (rpt_type == BTC_RPT_TYPE_SLOT) {
@@ -1097,7 +1206,7 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 				    sizeof(dm->slot_now)));
 	}
 
-	if (rpt_type == BTC_RPT_TYPE_CYSTA &&
+	if (rpt_type == BTC_RPT_TYPE_CYSTA && chip->chip_id == RTL8852A &&
 	    pcysta->cycles >= BTC_CYSTA_CHK_PERIOD) {
 		/* Check Leak-AP */
 		if (pcysta->slot_cnt[CXST_LK] != 0 &&
@@ -1120,16 +1229,55 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 		}
 
 		_chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE, pcysta->slot_cnt[CXST_W1]);
-		_chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE, pcysta->slot_cnt[CXST_W1]);
+		_chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE, pcysta->slot_cnt[CXST_B1]);
 		_chk_btc_err(rtwdev, BTC_DCNT_CYCLE_FREEZE, (u32)pcysta->cycles);
+	} else if (rpt_type == BTC_RPT_TYPE_CYSTA && pcysta_v1 &&
+		   le16_to_cpu(pcysta_v1->cycles) >= BTC_CYSTA_CHK_PERIOD) {
+		cnt_leak_slot = le32_to_cpu(pcysta_v1->slot_cnt[CXST_LK]);
+		cnt_rx_imr = le32_to_cpu(pcysta_v1->leak_slot.cnt_rximr);
+		/* Check Leak-AP */
+		if (cnt_leak_slot != 0 && cnt_rx_imr != 0 &&
+		    dm->tdma_now.rxflctrl) {
+			if (cnt_leak_slot < BTC_LEAK_AP_TH * cnt_rx_imr)
+				dm->leak_ap = 1;
+		}
+
+		/* Check diff time between real WL slot and W1 slot */
+		if (dm->tdma_now.type == CXTDMA_OFF) {
+			wl_slot_set = le16_to_cpu(dm->slot_now[CXST_W1].dur);
+			wl_slot_real = le16_to_cpu(pcysta_v1->cycle_time.tavg[CXT_WL]);
+			if (wl_slot_real > wl_slot_set) {
+				diff_t = wl_slot_real - wl_slot_set;
+				_chk_btc_err(rtwdev, BTC_DCNT_WL_SLOT_DRIFT, diff_t);
+			}
+		}
+
+		/* Check diff time between real BT slot and EBT/E5G slot */
+		if (dm->tdma_now.type == CXTDMA_OFF &&
+		    dm->tdma_now.ext_ctrl == CXECTL_EXT &&
+		    btc->bt_req_len != 0) {
+			bt_slot_real = le16_to_cpu(pcysta_v1->cycle_time.tavg[CXT_BT]);
+
+			if (btc->bt_req_len > bt_slot_real) {
+				diff_t = btc->bt_req_len - bt_slot_real;
+				_chk_btc_err(rtwdev, BTC_DCNT_BT_SLOT_DRIFT, diff_t);
+			}
+		}
+
+		_chk_btc_err(rtwdev, BTC_DCNT_W1_FREEZE,
+			     le32_to_cpu(pcysta_v1->slot_cnt[CXST_W1]));
+		_chk_btc_err(rtwdev, BTC_DCNT_B1_FREEZE,
+			     le32_to_cpu(pcysta_v1->slot_cnt[CXST_B1]));
+		_chk_btc_err(rtwdev, BTC_DCNT_CYCLE_FREEZE,
+			     (u32)le16_to_cpu(pcysta_v1->cycles));
 	}
 
-	if (rpt_type == BTC_RPT_TYPE_CTRL) {
+	if (rpt_type == BTC_RPT_TYPE_CTRL && chip->chip_id == RTL8852A) {
 		prpt = &pfwinfo->rpt_ctrl.finfo;
 		btc->fwinfo.rpt_en_map = prpt->rpt_enable;
 		wl->ver_info.fw_coex = prpt->wl_fw_coex_ver;
 		wl->ver_info.fw = prpt->wl_fw_ver;
-		dm->wl_fw_cx_offload = !!(prpt->wl_fw_cx_offload);
+		dm->wl_fw_cx_offload = !!prpt->wl_fw_cx_offload;
 
 		_chk_btc_err(rtwdev, BTC_DCNT_RPT_FREEZE,
 			     pfwinfo->event[BTF_EVNT_RPT]);
@@ -1142,6 +1290,33 @@ static u32 _chk_btc_report(struct rtw89_dev *rtwdev,
 			btc->cx.cnt_bt[BTC_BCNT_POLUT] =
 				rtw89_mac_get_plt_cnt(rtwdev, RTW89_MAC_0);
 		}
+	} else if (rpt_type == BTC_RPT_TYPE_CTRL) {
+		prpt_v1 = &pfwinfo->rpt_ctrl.finfo_v1;
+		btc->fwinfo.rpt_en_map = le32_to_cpu(prpt_v1->rpt_info.en);
+		wl->ver_info.fw_coex = le32_to_cpu(prpt_v1->wl_fw_info.cx_ver);
+		wl->ver_info.fw = le32_to_cpu(prpt_v1->wl_fw_info.fw_ver);
+		dm->wl_fw_cx_offload = !!le32_to_cpu(prpt_v1->wl_fw_info.cx_offload);
+
+		for (i = RTW89_PHY_0; i < RTW89_PHY_MAX; i++)
+			memcpy(&dm->gnt.band[i], &prpt_v1->gnt_val[i],
+			       sizeof(dm->gnt.band[i]));
+
+		btc->cx.cnt_bt[BTC_BCNT_HIPRI_TX] = le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_HI_TX]);
+		btc->cx.cnt_bt[BTC_BCNT_HIPRI_RX] = le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_HI_RX]);
+		btc->cx.cnt_bt[BTC_BCNT_LOPRI_TX] = le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_LO_TX]);
+		btc->cx.cnt_bt[BTC_BCNT_LOPRI_RX] = le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_LO_RX]);
+		btc->cx.cnt_bt[BTC_BCNT_POLUT] = le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_POLLUTED]);
+
+		_chk_btc_err(rtwdev, BTC_DCNT_BTCNT_FREEZE, 0);
+		_chk_btc_err(rtwdev, BTC_DCNT_RPT_FREEZE,
+			     pfwinfo->event[BTF_EVNT_RPT]);
+
+		if (le32_to_cpu(prpt_v1->bt_cnt[BTC_BCNT_RFK_TIMEOUT]) > 0)
+			bt->rfk_info.map.timeout = 1;
+		else
+			bt->rfk_info.map.timeout = 0;
+
+		dm->error.map.bt_rfk_timeout = bt->rfk_info.map.timeout;
 	}
 
 	if (rpt_type >= BTC_RPT_TYPE_BT_VER &&
@@ -1155,6 +1330,7 @@ static void _parse_btc_report(struct rtw89_dev *rtwdev,
 			      struct rtw89_btc_btf_fwinfo *pfwinfo,
 			      u8 *pbuf, u32 buf_len)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc_prpt *btc_prpt = NULL;
 	u32 index = 0, rpt_len = 0;
 
@@ -1164,7 +1340,7 @@ static void _parse_btc_report(struct rtw89_dev *rtwdev,
 
 	while (pbuf) {
 		btc_prpt = (struct rtw89_btc_prpt *)&pbuf[index];
-		if (index + 2 >= BTC_FWINFO_BUF)
+		if (index + 2 >= chip->btc_fwinfo_buf)
 			break;
 		/* At least 3 bytes: type(1) & len(2) */
 		rpt_len = le16_to_cpu(btc_prpt->len);
@@ -1182,10 +1358,12 @@ static void _parse_btc_report(struct rtw89_dev *rtwdev,
 
 static void _append_tdma(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_dm *dm = &btc->dm;
-	struct rtw89_btc_btf_tlv *tlv = NULL;
-	struct rtw89_btc_fbtc_tdma *v = NULL;
+	struct rtw89_btc_btf_tlv *tlv;
+	struct rtw89_btc_fbtc_tdma *v;
+	struct rtw89_btc_fbtc_tdma_v1 *v1;
 	u16 len = btc->policy_len;
 
 	if (!btc->update_policy_force &&
@@ -1197,12 +1375,19 @@ static void _append_tdma(struct rtw89_dev *rtwdev)
 	}
 
 	tlv = (struct rtw89_btc_btf_tlv *)&btc->policy[len];
-	v = (struct rtw89_btc_fbtc_tdma *)&tlv->val[0];
 	tlv->type = CXPOLICY_TDMA;
-	tlv->len = sizeof(*v);
-
-	memcpy(v, &dm->tdma, sizeof(*v));
-	btc->policy_len += BTC_TLV_HDR_LEN  + sizeof(*v);
+	if (chip->chip_id == RTL8852A) {
+		v = (struct rtw89_btc_fbtc_tdma *)&tlv->val[0];
+		tlv->len = sizeof(*v);
+		memcpy(v, &dm->tdma, sizeof(*v));
+		btc->policy_len += BTC_TLV_HDR_LEN  + sizeof(*v);
+	} else {
+		tlv->len = sizeof(*v1);
+		v1 = (struct rtw89_btc_fbtc_tdma_v1 *)&tlv->val[0];
+		v1->fver = chip->fcxtdma_ver;
+		v1->tdma = dm->tdma;
+		btc->policy_len += BTC_TLV_HDR_LEN  + sizeof(*v1);
+	}
 
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
 		    "[BTC], %s(): type:%d, rxflctrl=%d, txpause=%d, wtgle_n=%d, leak_n=%d, ext_ctrl=%d\n",
@@ -1408,12 +1593,17 @@ static void _fw_set_policy(struct rtw89_dev *rtwdev, u16 policy_type,
 
 static void _fw_set_drv_info(struct rtw89_dev *rtwdev, u8 type)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
 	switch (type) {
 	case CXDRVINFO_INIT:
 		rtw89_fw_h2c_cxdrv_init(rtwdev);
 		break;
 	case CXDRVINFO_ROLE:
-		rtw89_fw_h2c_cxdrv_role(rtwdev);
+		if (chip->chip_id == RTL8852A)
+			rtw89_fw_h2c_cxdrv_role(rtwdev);
+		else
+			rtw89_fw_h2c_cxdrv_role_v1(rtwdev);
 		break;
 	case CXDRVINFO_CTRL:
 		rtw89_fw_h2c_cxdrv_ctrl(rtwdev);
@@ -1448,7 +1638,7 @@ void btc_fw_event(struct rtw89_dev *rtwdev, u8 evt_id, void *data, u32 len)
 	}
 }
 
-static void _set_gnt_wl(struct rtw89_dev *rtwdev, u8 phy_map, u8 state)
+static void _set_gnt(struct rtw89_dev *rtwdev, u8 phy_map, u8 wl_state, u8 bt_state)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_dm *dm = &btc->dm;
@@ -1462,7 +1652,7 @@ static void _set_gnt_wl(struct rtw89_dev *rtwdev, u8 phy_map, u8 state)
 		if (!(phy_map & BIT(i)))
 			continue;
 
-		switch (state) {
+		switch (wl_state) {
 		case BTC_GNT_HW:
 			g[i].gnt_wl_sw_en = 0;
 			g[i].gnt_wl = 0;
@@ -1476,6 +1666,21 @@ static void _set_gnt_wl(struct rtw89_dev *rtwdev, u8 phy_map, u8 state)
 			g[i].gnt_wl = 1;
 			break;
 		}
+
+		switch (bt_state) {
+		case BTC_GNT_HW:
+			g[i].gnt_bt_sw_en = 0;
+			g[i].gnt_bt = 0;
+			break;
+		case BTC_GNT_SW_LO:
+			g[i].gnt_bt_sw_en = 1;
+			g[i].gnt_bt = 0;
+			break;
+		case BTC_GNT_SW_HI:
+			g[i].gnt_bt_sw_en = 1;
+			g[i].gnt_bt = 1;
+			break;
+		}
 	}
 
 	rtw89_chip_mac_cfg_gnt(rtwdev, &dm->gnt);
@@ -1534,6 +1739,7 @@ static void _set_wl_tx_power(struct rtw89_dev *rtwdev, u32 level)
 
 static void _set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 
@@ -1546,6 +1752,8 @@ static void _set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level)
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
 		    "[BTC], %s(): level = %d\n",
 		    __func__, level);
+
+	chip->ops->btc_set_wl_rx_gain(rtwdev, level);
 }
 
 static void _set_bt_tx_power(struct rtw89_dev *rtwdev, u8 level)
@@ -1683,28 +1891,45 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
 	struct rtw89_btc_bt_link_info *b = &bt->link_info;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
+	struct rtw89_btc_wl_active_role *r;
+	struct rtw89_btc_wl_active_role_v1 *r1;
 	u8 en = 0, i, ch = 0, bw = 0;
+	u8 mode, connect_cnt;
 
 	if (btc->ctrl.manual || wl->status.map.scan)
 		return;
 
-	/* TODO if include module->ant.type == BTC_ANT_SHARED */
+	if (chip->chip_id == RTL8852A) {
+		mode = wl_rinfo->link_mode;
+		connect_cnt = wl_rinfo->connect_cnt;
+	} else {
+		mode = wl_rinfo_v1->link_mode;
+		connect_cnt = wl_rinfo_v1->connect_cnt;
+	}
+
 	if (wl->status.map.rf_off || bt->whql_test ||
-	    wl_rinfo->link_mode == BTC_WLINK_NOLINK ||
-	    wl_rinfo->link_mode == BTC_WLINK_5G ||
-	    wl_rinfo->connect_cnt > BTC_TDMA_WLROLE_MAX) {
+	    mode == BTC_WLINK_NOLINK || mode == BTC_WLINK_5G ||
+	    connect_cnt > BTC_TDMA_WLROLE_MAX) {
 		en = false;
-	} else if (wl_rinfo->link_mode == BTC_WLINK_2G_MCC ||
-		   wl_rinfo->link_mode == BTC_WLINK_2G_SCC) {
+	} else if (mode == BTC_WLINK_2G_MCC || mode == BTC_WLINK_2G_SCC) {
 		en = true;
 		/* get p2p channel */
 		for (i = 0; i < RTW89_PORT_NUM; i++) {
-			if (wl_rinfo->active_role[i].role ==
-			    RTW89_WIFI_ROLE_P2P_GO ||
-			    wl_rinfo->active_role[i].role ==
-			    RTW89_WIFI_ROLE_P2P_CLIENT) {
-				ch = wl_rinfo->active_role[i].ch;
-				bw = wl_rinfo->active_role[i].bw;
+			r = &wl_rinfo->active_role[i];
+			r1 = &wl_rinfo_v1->active_role_v1[i];
+
+			if (chip->chip_id == RTL8852A &&
+			    (r->role == RTW89_WIFI_ROLE_P2P_GO ||
+			     r->role == RTW89_WIFI_ROLE_P2P_CLIENT)) {
+				ch = r->ch;
+				bw = r->bw;
+				break;
+			} else if (chip->chip_id != RTL8852A &&
+				   (r1->role == RTW89_WIFI_ROLE_P2P_GO ||
+				    r1->role == RTW89_WIFI_ROLE_P2P_CLIENT)) {
+				ch = r1->ch;
+				bw = r1->bw;
 				break;
 			}
 		}
@@ -1712,10 +1937,18 @@ static void _set_bt_afh_info(struct rtw89_dev *rtwdev)
 		en = true;
 		/* get 2g channel  */
 		for (i = 0; i < RTW89_PORT_NUM; i++) {
-			if (wl_rinfo->active_role[i].connected &&
-			    wl_rinfo->active_role[i].band == RTW89_BAND_2G) {
-				ch = wl_rinfo->active_role[i].ch;
-				bw = wl_rinfo->active_role[i].bw;
+			r = &wl_rinfo->active_role[i];
+			r1 = &wl_rinfo_v1->active_role_v1[i];
+
+			if (chip->chip_id == RTL8852A &&
+			    r->connected && r->band == RTW89_BAND_2G) {
+				ch = r->ch;
+				bw = r->bw;
+				break;
+			} else if (chip->chip_id != RTL8852A &&
+				   r1->connected && r1->band == RTW89_BAND_2G) {
+				ch = r1->ch;
+				bw = r1->bw;
 				break;
 			}
 		}
@@ -1768,6 +2001,7 @@ static bool _check_freerun(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
 	struct rtw89_btc_bt_link_info *bt_linfo = &bt->link_info;
 	struct rtw89_btc_bt_hid_desc *hid = &bt_linfo->hid_desc;
 
@@ -1777,7 +2011,8 @@ static bool _check_freerun(struct rtw89_dev *rtwdev)
 	}
 
 	/* The below is dedicated antenna case */
-	if (wl_rinfo->connect_cnt > BTC_TDMA_WLROLE_MAX) {
+	if (wl_rinfo->connect_cnt > BTC_TDMA_WLROLE_MAX ||
+	    wl_rinfo_v1->connect_cnt > BTC_TDMA_WLROLE_MAX) {
 		btc->dm.trx_para_level = 5;
 		return true;
 	}
@@ -1826,6 +2061,7 @@ static bool _check_freerun(struct rtw89_dev *rtwdev)
 }
 
 #define _tdma_set_flctrl(btc, flc) ({(btc)->dm.tdma.rxflctrl = flc; })
+#define _tdma_set_flctrl_role(btc, role) ({(btc)->dm.tdma.rxflctrl_role = role; })
 #define _tdma_set_tog(btc, wtg) ({(btc)->dm.tdma.wtgle_n = wtg; })
 #define _tdma_set_lek(btc, lek) ({(btc)->dm.tdma.leak_n = lek; })
 
@@ -1904,6 +2140,15 @@ union btc_btinfo {
 static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type,
 			enum btc_reason_and_action action)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	chip->ops->btc_set_policy(rtwdev, policy_type);
+	_fw_set_policy(rtwdev, policy_type, action);
+}
+
+#define BTC_B1_MAX 250 /* unit ms */
+void rtw89_btc_set_policy(struct rtw89_dev *rtwdev, u16 policy_type)
+{
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_dm *dm = &btc->dm;
 	struct rtw89_btc_fbtc_tdma *t = &dm->tdma;
@@ -1964,6 +2209,9 @@ static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type,
 		case BTC_CXP_OFF_BWB1:
 			_slot_set_tbl(btc, CXST_OFF, cxtbl[8]);
 			break;
+		case BTC_CXP_OFF_BWB3:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[6]);
+			break;
 		}
 		break;
 	case BTC_CXP_OFFB: /* TDMA off + beacon protect */
@@ -2080,17 +2328,361 @@ static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type,
 		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
 		*t = t_def[CXTD_AUTO];
 		switch (policy_type) {
-		case BTC_CXP_AUTO_TD50200:
+		case BTC_CXP_AUTO_TD50B1:
+			_slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO_TD60B1:
+			_slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO_TD20B1:
+			_slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO_TDW1B1: /* W1:B1 = user-define */
+			_slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			break;
+		}
+		break;
+	case BTC_CXP_PAUTO: /* PS-TDMA Auto-Slot */
+		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
+		*t = t_def[CXTD_PAUTO];
+		switch (policy_type) {
+		case BTC_CXP_PAUTO_TD50B1:
+			_slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO_TD60B1:
+			_slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO_TD20B1:
+			_slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO_TDW1B1:
+			_slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			break;
+		}
+		break;
+	case BTC_CXP_AUTO2: /* TDMA Auto-Slot2 */
+		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
+		*t = t_def[CXTD_AUTO2];
+		switch (policy_type) {
+		case BTC_CXP_AUTO2_TD3050:
+			_slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO2_TD3070:
+			_slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, 70, tbl_b4, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO2_TD5050:
+			_slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO2_TD6060:
+			_slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, 60, tbl_b4, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO2_TD2080:
+			_slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, 80, tbl_b4, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_AUTO2_TDW1B4: /* W1:B1 = user-define */
+			_slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, dm->slot_dur[CXST_B4],
+				  tbl_b4, SLOT_MIX);
+			break;
+		}
+		break;
+	case BTC_CXP_PAUTO2: /* PS-TDMA Auto-Slot2 */
+		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
+		*t = t_def[CXTD_PAUTO2];
+		switch (policy_type) {
+		case BTC_CXP_PAUTO2_TD3050:
+			_slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO2_TD3070:
+			_slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, 70, tbl_b4, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO2_TD5050:
+			_slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, 50, tbl_b4, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO2_TD6060:
+			_slot_set(btc, CXST_W1, 60, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, 60, tbl_b4, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO2_TD2080:
+			_slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, 80, tbl_b4, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PAUTO2_TDW1B4: /* W1:B1 = user-define */
+			_slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B4, dm->slot_dur[CXST_B4],
+				  tbl_b4, SLOT_MIX);
+			break;
+		}
+		break;
+	}
+}
+EXPORT_SYMBOL(rtw89_btc_set_policy);
+
+void rtw89_btc_set_policy_v1(struct rtw89_dev *rtwdev, u16 policy_type)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_dm *dm = &btc->dm;
+	struct rtw89_btc_fbtc_tdma *t = &dm->tdma;
+	struct rtw89_btc_fbtc_slot *s = dm->slot;
+	struct rtw89_btc_wl_role_info_v1 *wl_rinfo = &btc->cx.wl.role_info_v1;
+	struct rtw89_btc_bt_hid_desc *hid = &btc->cx.bt.link_info.hid_desc;
+	struct rtw89_btc_bt_hfp_desc *hfp = &btc->cx.bt.link_info.hfp_desc;
+	u8 type, null_role;
+	u32 tbl_w1, tbl_b1, tbl_b4;
+
+	type = FIELD_GET(BTC_CXP_MASK, policy_type);
+
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) {
+		if (btc->cx.wl.status.map._4way)
+			tbl_w1 = cxtbl[1];
+		else if (hid->exist && hid->type == BTC_HID_218)
+			tbl_w1 = cxtbl[7]; /* Ack/BA no break bt Hi-Pri-rx */
+		else
+			tbl_w1 = cxtbl[8];
+
+		if (dm->leak_ap &&
+		    (type == BTC_CXP_PFIX || type == BTC_CXP_PAUTO2)) {
+			tbl_b1 = cxtbl[3];
+			tbl_b4 = cxtbl[3];
+		} else if (hid->exist && hid->type == BTC_HID_218) {
+			tbl_b1 = cxtbl[4]; /* Ack/BA no break bt Hi-Pri-rx */
+			tbl_b4 = cxtbl[4];
+		} else {
+			tbl_b1 = cxtbl[2];
+			tbl_b4 = cxtbl[2];
+		}
+	} else {
+		tbl_w1 = cxtbl[16];
+		tbl_b1 = cxtbl[17];
+		tbl_b4 = cxtbl[17];
+	}
+
+	btc->bt_req_en = false;
+
+	switch (type) {
+	case BTC_CXP_USERDEF0:
+		btc->update_policy_force = true;
+		*t = t_def[CXTD_OFF];
+		s[CXST_OFF] = s_def[CXST_OFF];
+		_slot_set_tbl(btc, CXST_OFF, cxtbl[2]);
+		break;
+	case BTC_CXP_OFF: /* TDMA off */
+		_write_scbd(rtwdev, BTC_WSCB_TDMA, false);
+		*t = t_def[CXTD_OFF];
+		s[CXST_OFF] = s_def[CXST_OFF];
+
+		switch (policy_type) {
+		case BTC_CXP_OFF_BT:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[2]);
+			break;
+		case BTC_CXP_OFF_WL:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[1]);
+			break;
+		case BTC_CXP_OFF_EQ0:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[0]);
+			break;
+		case BTC_CXP_OFF_EQ1:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[16]);
+			break;
+		case BTC_CXP_OFF_EQ2:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[17]);
+			break;
+		case BTC_CXP_OFF_EQ3:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[18]);
+			break;
+		case BTC_CXP_OFF_BWB0:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[5]);
+			break;
+		case BTC_CXP_OFF_BWB1:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[8]);
+			break;
+		case BTC_CXP_OFF_BWB2:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[7]);
+			break;
+		case BTC_CXP_OFF_BWB3:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[6]);
+			break;
+		default:
+			break;
+		}
+		break;
+	case BTC_CXP_OFFB: /* TDMA off + beacon protect */
+		_write_scbd(rtwdev, BTC_WSCB_TDMA, false);
+		*t = t_def[CXTD_OFF_B2];
+		s[CXST_OFF] = s_def[CXST_OFF];
+
+		switch (policy_type) {
+		case BTC_CXP_OFFB_BWB0:
+			_slot_set_tbl(btc, CXST_OFF, cxtbl[8]);
+			break;
+		default:
+			break;
+		}
+		break;
+	case BTC_CXP_OFFE: /* TDMA off + beacon protect + Ext_control */
+		btc->bt_req_en = true;
+		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
+		*t = t_def[CXTD_OFF_EXT];
+
+		/* To avoid wl-s0 tx break by hid/hfp tx */
+		if (hid->exist || hfp->exist)
+			tbl_w1 = cxtbl[16];
+
+		switch (policy_type) {
+		case BTC_CXP_OFFE_DEF:
+			s[CXST_E2G] = s_def[CXST_E2G];
+			s[CXST_E5G] = s_def[CXST_E5G];
+			s[CXST_EBT] = s_def[CXST_EBT];
+			s[CXST_ENULL] = s_def[CXST_ENULL];
+			break;
+		case BTC_CXP_OFFE_DEF2:
+			_slot_set(btc, CXST_E2G, 20, cxtbl[1], SLOT_ISO);
+			s[CXST_E5G] = s_def[CXST_E5G];
+			s[CXST_EBT] = s_def[CXST_EBT];
+			s[CXST_ENULL] = s_def[CXST_ENULL];
+			break;
+		default:
+			break;
+		}
+		break;
+	case BTC_CXP_FIX: /* TDMA Fix-Slot */
+		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
+		*t = t_def[CXTD_FIX];
+
+		switch (policy_type) {
+		case BTC_CXP_FIX_TD3030:
+			_slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 30, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD5050:
+			_slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 50, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD2030:
+			_slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 30, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD4010:
+			_slot_set(btc, CXST_W1, 40, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 10, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD4010ISO:
+			_slot_set(btc, CXST_W1, 40, cxtbl[1], SLOT_ISO);
+			_slot_set(btc, CXST_B1, 10, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD7010:
+			_slot_set(btc, CXST_W1, 70, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 10, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD2060:
+			_slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 60, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD3060:
+			_slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 60, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TD2080:
+			_slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 80, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_FIX_TDW1B1: /* W1:B1 = user-define */
+			_slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			break;
+		default:
+			break;
+		}
+		break;
+	case BTC_CXP_PFIX: /* PS-TDMA Fix-Slot */
+		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
+		*t = t_def[CXTD_PFIX];
+
+		switch (policy_type) {
+		case BTC_CXP_PFIX_TD3030:
+			_slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 30, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TD5050:
+			_slot_set(btc, CXST_W1, 50, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 50, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TD2030:
+			_slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 30, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TD2060:
+			_slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 60, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TD3070:
+			_slot_set(btc, CXST_W1, 30, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 60, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TD2080:
+			_slot_set(btc, CXST_W1, 20, tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, 80, tbl_b1, SLOT_MIX);
+			break;
+		case BTC_CXP_PFIX_TDW1B1: /* W1:B1 = user-define */
+			_slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1],
+				  tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
+			break;
+		default:
+			break;
+		}
+		break;
+	case BTC_CXP_AUTO: /* TDMA Auto-Slot */
+		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
+		*t = t_def[CXTD_AUTO];
+
+		switch (policy_type) {
+		case BTC_CXP_AUTO_TD50B1:
 			_slot_set(btc, CXST_W1,  50, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			break;
-		case BTC_CXP_AUTO_TD60200:
+		case BTC_CXP_AUTO_TD60B1:
 			_slot_set(btc, CXST_W1,  60, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			break;
-		case BTC_CXP_AUTO_TD20200:
+		case BTC_CXP_AUTO_TD20B1:
 			_slot_set(btc, CXST_W1,  20, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			break;
 		case BTC_CXP_AUTO_TDW1B1: /* W1:B1 = user-define */
 			_slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1],
@@ -2098,23 +2690,26 @@ static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type,
 			_slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1],
 				  tbl_b1, SLOT_MIX);
 			break;
+		default:
+			break;
 		}
 		break;
 	case BTC_CXP_PAUTO: /* PS-TDMA Auto-Slot */
 		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
 		*t = t_def[CXTD_PAUTO];
+
 		switch (policy_type) {
-		case BTC_CXP_PAUTO_TD50200:
+		case BTC_CXP_PAUTO_TD50B1:
 			_slot_set(btc, CXST_W1,  50, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			break;
-		case BTC_CXP_PAUTO_TD60200:
+		case BTC_CXP_PAUTO_TD60B1:
 			_slot_set(btc, CXST_W1,  60, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			break;
-		case BTC_CXP_PAUTO_TD20200:
+		case BTC_CXP_PAUTO_TD20B1:
 			_slot_set(btc, CXST_W1,  20, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			break;
 		case BTC_CXP_PAUTO_TDW1B1:
 			_slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1],
@@ -2122,119 +2717,112 @@ static void _set_policy(struct rtw89_dev *rtwdev, u16 policy_type,
 			_slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1],
 				  tbl_b1, SLOT_MIX);
 			break;
+		default:
+			break;
 		}
 		break;
 	case BTC_CXP_AUTO2: /* TDMA Auto-Slot2 */
 		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
 		*t = t_def[CXTD_AUTO2];
+
 		switch (policy_type) {
 		case BTC_CXP_AUTO2_TD3050:
 			_slot_set(btc, CXST_W1,  30, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4,  50, tbl_b4, SLOT_MIX);
 			break;
 		case BTC_CXP_AUTO2_TD3070:
 			_slot_set(btc, CXST_W1,  30, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4,  70, tbl_b4, SLOT_MIX);
 			break;
 		case BTC_CXP_AUTO2_TD5050:
 			_slot_set(btc, CXST_W1,  50, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4,  50, tbl_b4, SLOT_MIX);
 			break;
 		case BTC_CXP_AUTO2_TD6060:
 			_slot_set(btc, CXST_W1,  60, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4,  60, tbl_b4, SLOT_MIX);
 			break;
 		case BTC_CXP_AUTO2_TD2080:
 			_slot_set(btc, CXST_W1,  20, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4,  80, tbl_b4, SLOT_MIX);
 			break;
 		case BTC_CXP_AUTO2_TDW1B4: /* W1:B1 = user-define */
 			_slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1],
 				  tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4, dm->slot_dur[CXST_B4],
 				  tbl_b4, SLOT_MIX);
 			break;
+		default:
+			break;
 		}
 		break;
 	case BTC_CXP_PAUTO2: /* PS-TDMA Auto-Slot2 */
 		_write_scbd(rtwdev, BTC_WSCB_TDMA, true);
 		*t = t_def[CXTD_PAUTO2];
+
 		switch (policy_type) {
 		case BTC_CXP_PAUTO2_TD3050:
 			_slot_set(btc, CXST_W1,  30, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4,  50, tbl_b4, SLOT_MIX);
 			break;
 		case BTC_CXP_PAUTO2_TD3070:
 			_slot_set(btc, CXST_W1,  30, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4,  70, tbl_b4, SLOT_MIX);
 			break;
 		case BTC_CXP_PAUTO2_TD5050:
 			_slot_set(btc, CXST_W1,  50, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4,  50, tbl_b4, SLOT_MIX);
 			break;
 		case BTC_CXP_PAUTO2_TD6060:
 			_slot_set(btc, CXST_W1,  60, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4,  60, tbl_b4, SLOT_MIX);
 			break;
 		case BTC_CXP_PAUTO2_TD2080:
 			_slot_set(btc, CXST_W1,  20, tbl_w1, SLOT_ISO);
-			_slot_set(btc, CXST_B1, 200, tbl_b1, SLOT_MIX);
+			_slot_set(btc, CXST_B1, BTC_B1_MAX, tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4,  80, tbl_b4, SLOT_MIX);
 			break;
 		case BTC_CXP_PAUTO2_TDW1B4: /* W1:B1 = user-define */
 			_slot_set(btc, CXST_W1, dm->slot_dur[CXST_W1],
 				  tbl_w1, SLOT_ISO);
+			_slot_set(btc, CXST_B1, dm->slot_dur[CXST_B1],
+				  tbl_b1, SLOT_MIX);
 			_slot_set(btc, CXST_B4, dm->slot_dur[CXST_B4],
 				  tbl_b4, SLOT_MIX);
 			break;
+		default:
+			break;
 		}
 		break;
 	}
 
-	_fw_set_policy(rtwdev, policy_type, action);
-}
-
-static void _set_gnt_bt(struct rtw89_dev *rtwdev, u8 phy_map, u8 state)
-{
-	struct rtw89_btc *btc = &rtwdev->btc;
-	struct rtw89_btc_dm *dm = &btc->dm;
-	struct rtw89_mac_ax_gnt *g = dm->gnt.band;
-	u8 i;
-
-	if (phy_map > BTC_PHY_ALL)
-		return;
+	if (wl_rinfo->link_mode == BTC_WLINK_2G_SCC && dm->tdma.rxflctrl) {
+		null_role = FIELD_PREP(0x0f, dm->wl_scc.null_role1) |
+			    FIELD_PREP(0xf0, dm->wl_scc.null_role2);
+		_tdma_set_flctrl_role(btc, null_role);
+	}
 
-	for (i = 0; i < RTW89_PHY_MAX; i++) {
-		if (!(phy_map & BIT(i)))
-			continue;
+	/* enter leak_slot after each null-1 */
+	if (dm->leak_ap && dm->tdma.leak_n > 1)
+		_tdma_set_lek(btc, 1);
 
-		switch (state) {
-		case BTC_GNT_HW:
-			g[i].gnt_bt_sw_en = 0;
-			g[i].gnt_bt = 0;
-			break;
-		case BTC_GNT_SW_LO:
-			g[i].gnt_bt_sw_en = 1;
-			g[i].gnt_bt = 0;
-			break;
-		case BTC_GNT_SW_HI:
-			g[i].gnt_bt_sw_en = 1;
-			g[i].gnt_bt = 1;
-			break;
-		}
+	if (dm->tdma_instant_excute) {
+		btc->dm.tdma.option_ctrl |= BIT(0);
+		btc->update_policy_force = true;
 	}
-
-	rtw89_chip_mac_cfg_gnt(rtwdev, &dm->gnt);
 }
+EXPORT_SYMBOL(rtw89_btc_set_policy_v1);
 
 static void _set_bt_plut(struct rtw89_dev *rtwdev, u8 phy_map,
 			 u8 tx_val, u8 rx_val)
@@ -2300,86 +2888,74 @@ static void _set_ant(struct rtw89_dev *rtwdev, bool force_exec,
 
 	switch (type) {
 	case BTC_ANT_WPOWERON:
-		rtw89_chip_cfg_ctrl_path(rtwdev, false);
+		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_BT);
 		break;
 	case BTC_ANT_WINIT:
-		if (bt->enable.now) {
-			_set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_LO);
-			_set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_HI);
-		} else {
-			_set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_HI);
-			_set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_LO);
-		}
-		rtw89_chip_cfg_ctrl_path(rtwdev, true);
+		if (bt->enable.now)
+			_set_gnt(rtwdev, phy_map, BTC_GNT_SW_LO, BTC_GNT_SW_HI);
+		else
+			_set_gnt(rtwdev, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_LO);
+
+		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL);
 		_set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_BT, BTC_PLT_BT);
 		break;
 	case BTC_ANT_WONLY:
-		_set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_HI);
-		_set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_LO);
-		rtw89_chip_cfg_ctrl_path(rtwdev, true);
+		_set_gnt(rtwdev, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_LO);
+		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL);
 		_set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_NONE, BTC_PLT_NONE);
 		break;
 	case BTC_ANT_WOFF:
-		rtw89_chip_cfg_ctrl_path(rtwdev, false);
+		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_BT);
 		_set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_NONE, BTC_PLT_NONE);
 		break;
 	case BTC_ANT_W2G:
-		rtw89_chip_cfg_ctrl_path(rtwdev, true);
+		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL);
 		if (rtwdev->dbcc_en) {
 			for (i = 0; i < RTW89_PHY_MAX; i++) {
 				b2g = (wl_dinfo->real_band[i] == RTW89_BAND_2G);
 
 				gnt_wl_ctrl = b2g ? BTC_GNT_HW : BTC_GNT_SW_HI;
-				_set_gnt_wl(rtwdev, BIT(i), gnt_wl_ctrl);
-
 				gnt_bt_ctrl = b2g ? BTC_GNT_HW : BTC_GNT_SW_HI;
 				/* BT should control by GNT_BT if WL_2G at S0 */
 				if (i == 1 &&
 				    wl_dinfo->real_band[0] == RTW89_BAND_2G &&
 				    wl_dinfo->real_band[1] == RTW89_BAND_5G)
 					gnt_bt_ctrl = BTC_GNT_HW;
-				_set_gnt_bt(rtwdev, BIT(i), gnt_bt_ctrl);
-
+				_set_gnt(rtwdev, BIT(i), gnt_wl_ctrl, gnt_bt_ctrl);
 				plt_ctrl = b2g ? BTC_PLT_BT : BTC_PLT_NONE;
 				_set_bt_plut(rtwdev, BIT(i),
 					     plt_ctrl, plt_ctrl);
 			}
 		} else {
-			_set_gnt_wl(rtwdev, phy_map, BTC_GNT_HW);
-			_set_gnt_bt(rtwdev, phy_map, BTC_GNT_HW);
+			_set_gnt(rtwdev, phy_map, BTC_GNT_HW, BTC_GNT_HW);
 			_set_bt_plut(rtwdev, BTC_PHY_ALL,
 				     BTC_PLT_BT, BTC_PLT_BT);
 		}
 		break;
 	case BTC_ANT_W5G:
-		rtw89_chip_cfg_ctrl_path(rtwdev, true);
-		_set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_HI);
-		_set_gnt_bt(rtwdev, phy_map, BTC_GNT_HW);
+		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL);
+		_set_gnt(rtwdev, phy_map, BTC_GNT_SW_HI, BTC_GNT_HW);
 		_set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_NONE, BTC_PLT_NONE);
 		break;
 	case BTC_ANT_W25G:
-		rtw89_chip_cfg_ctrl_path(rtwdev, true);
-		_set_gnt_wl(rtwdev, phy_map, BTC_GNT_HW);
-		_set_gnt_bt(rtwdev, phy_map, BTC_GNT_HW);
+		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL);
+		_set_gnt(rtwdev, phy_map, BTC_GNT_HW, BTC_GNT_HW);
 		_set_bt_plut(rtwdev, BTC_PHY_ALL,
 			     BTC_PLT_GNT_WL, BTC_PLT_GNT_WL);
 		break;
 	case BTC_ANT_FREERUN:
-		rtw89_chip_cfg_ctrl_path(rtwdev, true);
-		_set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_HI);
-		_set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_HI);
+		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL);
+		_set_gnt(rtwdev, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_HI);
 		_set_bt_plut(rtwdev, BTC_PHY_ALL, BTC_PLT_NONE, BTC_PLT_NONE);
 		break;
 	case BTC_ANT_WRFK:
-		rtw89_chip_cfg_ctrl_path(rtwdev, true);
-		_set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_HI);
-		_set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_LO);
+		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_WL);
+		_set_gnt(rtwdev, phy_map, BTC_GNT_SW_HI, BTC_GNT_SW_LO);
 		_set_bt_plut(rtwdev, phy_map, BTC_PLT_NONE, BTC_PLT_NONE);
 		break;
 	case BTC_ANT_BRFK:
-		rtw89_chip_cfg_ctrl_path(rtwdev, false);
-		_set_gnt_wl(rtwdev, phy_map, BTC_GNT_SW_LO);
-		_set_gnt_bt(rtwdev, phy_map, BTC_GNT_SW_HI);
+		rtw89_chip_cfg_ctrl_path(rtwdev, BTC_CTRL_BY_BT);
+		_set_gnt(rtwdev, phy_map, BTC_GNT_SW_LO, BTC_GNT_SW_HI);
 		_set_bt_plut(rtwdev, phy_map, BTC_PLT_NONE, BTC_PLT_NONE);
 		break;
 	default:
@@ -2491,14 +3067,19 @@ static void _action_bt_idle(struct rtw89_dev *rtwdev)
 static void _action_bt_hfp(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 
 	_set_ant(rtwdev, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
 
 	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) {
-		if (btc->cx.wl.status.map._4way)
+		if (btc->cx.wl.status.map._4way) {
 			_set_policy(rtwdev, BTC_CXP_OFF_WL, BTC_ACT_BT_HFP);
-		else
-			_set_policy(rtwdev, BTC_CXP_OFF_BWB0, BTC_ACT_BT_HFP);
+		} else if (wl->status.map.traffic_dir & BIT(RTW89_TFC_UL)) {
+			btc->cx.bt.scan_rx_low_pri = true;
+			_set_policy(rtwdev, BTC_CXP_OFF_BWB2, BTC_ACT_BT_HFP);
+		} else {
+			_set_policy(rtwdev, BTC_CXP_OFF_BWB1, BTC_ACT_BT_HFP);
+		}
 	} else {
 		_set_policy(rtwdev, BTC_CXP_OFF_EQ2, BTC_ACT_BT_HFP);
 	}
@@ -2506,17 +3087,37 @@ static void _action_bt_hfp(struct rtw89_dev *rtwdev)
 
 static void _action_bt_hid(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
+	struct rtw89_btc_bt_hid_desc *hid = &bt->link_info.hid_desc;
+	u16 policy_type = BTC_CXP_OFF_BT;
 
 	_set_ant(rtwdev, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
 
-	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) /* shared-antenna */
-		if (btc->cx.wl.status.map._4way)
-			_set_policy(rtwdev, BTC_CXP_OFF_WL, BTC_ACT_BT_HID);
-		else
-			_set_policy(rtwdev, BTC_CXP_OFF_BWB0, BTC_ACT_BT_HID);
-	else /* dedicated-antenna */
-		_set_policy(rtwdev, BTC_CXP_OFF_EQ3, BTC_ACT_BT_HID);
+	if (btc->mdinfo.ant.type == BTC_ANT_SHARED) { /* shared-antenna */
+		if (wl->status.map._4way) {
+			policy_type = BTC_CXP_OFF_WL;
+		} else if (wl->status.map.traffic_dir & BIT(RTW89_TFC_UL)) {
+			btc->cx.bt.scan_rx_low_pri = true;
+			if (hid->type & BTC_HID_BLE)
+				policy_type = BTC_CXP_OFF_BWB0;
+			else
+				policy_type = BTC_CXP_OFF_BWB2;
+		} else if (hid->type == BTC_HID_218) {
+			bt->scan_rx_low_pri = true;
+			policy_type = BTC_CXP_OFF_BWB2;
+		} else if (chip->para_ver == 0x1) {
+			policy_type = BTC_CXP_OFF_BWB3;
+		} else {
+			policy_type = BTC_CXP_OFF_BWB1;
+		}
+	} else { /* dedicated-antenna */
+		policy_type = BTC_CXP_OFF_EQ3;
+	}
+
+	_set_policy(rtwdev, policy_type, BTC_ACT_BT_HID);
 }
 
 static void _action_bt_a2dp(struct rtw89_dev *rtwdev)
@@ -2537,7 +3138,7 @@ static void _action_bt_a2dp(struct rtw89_dev *rtwdev)
 				    BTC_CXP_PAUTO_TDW1B1, BTC_ACT_BT_A2DP);
 		} else {
 			_set_policy(rtwdev,
-				    BTC_CXP_PAUTO_TD50200, BTC_ACT_BT_A2DP);
+				    BTC_CXP_PAUTO_TD50B1, BTC_ACT_BT_A2DP);
 		}
 		break;
 	case BTC_WBUSY_BSCAN: /* wl-busy + bt-inq + bt-A2DP */
@@ -2554,12 +3155,12 @@ static void _action_bt_a2dp(struct rtw89_dev *rtwdev)
 			_set_policy(rtwdev, BTC_CXP_AUTO_TDW1B1,
 				    BTC_ACT_BT_A2DP);
 		} else {
-			_set_policy(rtwdev, BTC_CXP_AUTO_TD50200,
+			_set_policy(rtwdev, BTC_CXP_AUTO_TD50B1,
 				    BTC_ACT_BT_A2DP);
 		}
 		break;
 	case BTC_WIDLE:  /* wl-idle + bt-A2DP */
-		_set_policy(rtwdev, BTC_CXP_AUTO_TD20200, BTC_ACT_BT_A2DP);
+		_set_policy(rtwdev, BTC_CXP_AUTO_TD20B1, BTC_ACT_BT_A2DP);
 		break;
 	}
 }
@@ -2639,7 +3240,7 @@ static void _action_bt_a2dp_hid(struct rtw89_dev *rtwdev)
 				    BTC_CXP_PAUTO_TDW1B1, BTC_ACT_BT_A2DP_HID);
 		} else {
 			_set_policy(rtwdev,
-				    BTC_CXP_PAUTO_TD50200, BTC_ACT_BT_A2DP_HID);
+				    BTC_CXP_PAUTO_TD50B1, BTC_ACT_BT_A2DP_HID);
 		}
 		break;
 	case BTC_WBUSY_BSCAN: /* wl-busy + bt-inq + bt-A2DP+HID */
@@ -2657,7 +3258,7 @@ static void _action_bt_a2dp_hid(struct rtw89_dev *rtwdev)
 			_set_policy(rtwdev, BTC_CXP_AUTO_TDW1B1,
 				    BTC_ACT_BT_A2DP_HID);
 		} else {
-			_set_policy(rtwdev, BTC_CXP_AUTO_TD50200,
+			_set_policy(rtwdev, BTC_CXP_AUTO_TD50B1,
 				    BTC_ACT_BT_A2DP_HID);
 		}
 		break;
@@ -2792,19 +3393,27 @@ static void _action_wl_rfk(struct rtw89_dev *rtwdev)
 
 static void _set_btg_ctrl(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
 	struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
-	bool is_btg = false;
+	bool is_btg;
+	u8 mode;
 
 	if (btc->ctrl.manual)
 		return;
 
+	if (chip->chip_id == RTL8852A)
+		mode = wl_rinfo->link_mode;
+	else
+		mode = wl_rinfo_v1->link_mode;
+
 	/* notify halbb ignore GNT_BT or not for WL BB Rx-AGC control */
-	if (wl_rinfo->link_mode == BTC_WLINK_5G) /* always 0 if 5G */
+	if (mode == BTC_WLINK_5G) /* always 0 if 5G */
 		is_btg = false;
-	else if (wl_rinfo->link_mode == BTC_WLINK_25G_DBCC &&
+	else if (mode == BTC_WLINK_25G_DBCC &&
 		 wl_dinfo->real_band[RTW89_PHY_1] != RTW89_BAND_2G)
 		is_btg = false;
 	else
@@ -2816,7 +3425,7 @@ static void _set_btg_ctrl(struct rtw89_dev *rtwdev)
 
 	btc->dm.wl_btg_rx = is_btg;
 
-	if (wl_rinfo->link_mode == BTC_WLINK_25G_MCC)
+	if (mode == BTC_WLINK_25G_MCC)
 		return;
 
 	rtw89_ctrl_btg(rtwdev, is_btg);
@@ -2889,6 +3498,7 @@ static void rtw89_tx_time_iter(void *data, struct ieee80211_sta *sta)
 
 static void _set_wl_tx_limit(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_cx *cx = &btc->cx;
 	struct rtw89_btc_dm *dm = &btc->dm;
@@ -2898,16 +3508,22 @@ static void _set_wl_tx_limit(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_bt_hfp_desc *hfp = &b->hfp_desc;
 	struct rtw89_btc_bt_hid_desc *hid = &b->hid_desc;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
 	struct rtw89_txtime_data data = {.rtwdev = rtwdev};
-	u8 mode = wl_rinfo->link_mode;
-	u8 tx_retry = 0;
-	u32 tx_time = 0;
-	u16 enable = 0;
+	u8 mode;
+	u8 tx_retry;
+	u32 tx_time;
+	u16 enable;
 	bool reenable = false;
 
 	if (btc->ctrl.manual)
 		return;
 
+	if (chip->chip_id == RTL8852A)
+		mode = wl_rinfo->link_mode;
+	else
+		mode = wl_rinfo_v1->link_mode;
+
 	if (btc->dm.freerun || btc->ctrl.igno_bt || b->profile_cnt.now == 0 ||
 	    mode == BTC_WLINK_5G || mode == BTC_WLINK_NOLINK) {
 		enable = 0;
@@ -2951,13 +3567,21 @@ static void _set_wl_tx_limit(struct rtw89_dev *rtwdev)
 
 static void _set_bt_rx_agc(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
 	bool bt_hi_lna_rx = false;
+	u8 mode;
+
+	if (chip->chip_id == RTL8852A)
+		mode = wl_rinfo->link_mode;
+	else
+		mode = wl_rinfo_v1->link_mode;
 
-	if (wl_rinfo->link_mode != BTC_WLINK_NOLINK && btc->dm.wl_btg_rx)
+	if (mode != BTC_WLINK_NOLINK && btc->dm.wl_btg_rx)
 		bt_hi_lna_rx = true;
 
 	if (bt_hi_lna_rx == bt->hi_lna_rx)
@@ -2966,14 +3590,34 @@ static void _set_bt_rx_agc(struct rtw89_dev *rtwdev)
 	_write_scbd(rtwdev, BTC_WSCB_BT_HILNA, bt_hi_lna_rx);
 }
 
+static void _set_bt_rx_scan_pri(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
+
+	_write_scbd(rtwdev, BTC_WSCB_RXSCAN_PRI, (bool)(!!bt->scan_rx_low_pri));
+}
+
 /* TODO add these functions */
 static void _action_common(struct rtw89_dev *rtwdev)
 {
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+
 	_set_btg_ctrl(rtwdev);
 	_set_wl_tx_limit(rtwdev);
 	_set_bt_afh_info(rtwdev);
 	_set_bt_rx_agc(rtwdev);
 	_set_rf_trx_para(rtwdev);
+	_set_bt_rx_scan_pri(rtwdev);
+
+	if (wl->scbd_change) {
+		rtw89_mac_cfg_sb(rtwdev, wl->scbd);
+		rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], write scbd: 0x%08x\n",
+			    wl->scbd);
+		wl->scbd_change = false;
+		btc->cx.cnt_wl[BTC_WCNT_SCBDUPDATE]++;
+	}
 }
 
 static void _action_by_bt(struct rtw89_dev *rtwdev)
@@ -3145,6 +3789,68 @@ static void _action_wl_2g_scc(struct rtw89_dev *rtwdev)
 	}
 }
 
+static void _action_wl_2g_scc_v1(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
+	struct rtw89_btc_dm *dm = &btc->dm;
+	struct rtw89_btc_wl_role_info_v1 *wl_rinfo = &wl->role_info_v1;
+	u16 policy_type = BTC_CXP_OFF_BT;
+	u32 dur;
+
+	if (btc->mdinfo.ant.type == BTC_ANT_DEDICATED) {
+		policy_type = BTC_CXP_OFF_EQ0;
+	} else {
+		/* shared-antenna */
+		switch (wl_rinfo->mrole_type) {
+		case BTC_WLMROLE_STA_GC:
+			dm->wl_scc.null_role1 = RTW89_WIFI_ROLE_STATION;
+			dm->wl_scc.null_role2 = RTW89_WIFI_ROLE_P2P_CLIENT;
+			dm->wl_scc.ebt_null = 0; /* no ext-slot-control */
+			_action_by_bt(rtwdev);
+			return;
+		case BTC_WLMROLE_STA_STA:
+			dm->wl_scc.null_role1 = RTW89_WIFI_ROLE_STATION;
+			dm->wl_scc.null_role2 = RTW89_WIFI_ROLE_STATION;
+			dm->wl_scc.ebt_null = 0; /* no ext-slot-control */
+			_action_by_bt(rtwdev);
+			return;
+		case BTC_WLMROLE_STA_GC_NOA:
+		case BTC_WLMROLE_STA_GO:
+		case BTC_WLMROLE_STA_GO_NOA:
+			dm->wl_scc.null_role1 = RTW89_WIFI_ROLE_STATION;
+			dm->wl_scc.null_role2 = RTW89_WIFI_ROLE_NONE;
+			dur = wl_rinfo->mrole_noa_duration;
+
+			if (wl->status.map._4way) {
+				dm->wl_scc.ebt_null = 0;
+				policy_type = BTC_CXP_OFFE_WL;
+			} else if (bt->link_info.status.map.connect == 0) {
+				dm->wl_scc.ebt_null = 0;
+				policy_type = BTC_CXP_OFFE_2GISOB;
+			} else if (bt->link_info.a2dp_desc.exist &&
+				   dur < btc->bt_req_len) {
+				dm->wl_scc.ebt_null = 1; /* tx null at EBT */
+				policy_type = BTC_CXP_OFFE_2GBWMIXB2;
+			} else if (bt->link_info.a2dp_desc.exist ||
+				   bt->link_info.pan_desc.exist) {
+				dm->wl_scc.ebt_null = 1; /* tx null at EBT */
+				policy_type = BTC_CXP_OFFE_2GBWISOB;
+			} else {
+				dm->wl_scc.ebt_null = 0;
+				policy_type = BTC_CXP_OFFE_2GBWISOB;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+	_set_ant(rtwdev, NM_EXEC, BTC_PHY_ALL, BTC_ANT_W2G);
+	_set_policy(rtwdev, policy_type, BTC_ACT_WL_2G_SCC);
+}
+
 static void _action_wl_2g_ap(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -3234,20 +3940,20 @@ static void _write_scbd(struct rtw89_dev *rtwdev, u32 val, bool state)
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	u32 scbd_val = 0;
+	u8 force_exec = false;
 
 	if (!chip->scbd)
 		return;
 
 	scbd_val = state ? wl->scbd | val : wl->scbd & ~val;
 
-	if (scbd_val == wl->scbd)
-		return;
-	rtw89_mac_cfg_sb(rtwdev, scbd_val);
-	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], write scbd: 0x%08x\n",
-		    scbd_val);
-	wl->scbd = scbd_val;
+	if (val & BTC_WSCB_ACTIVE || val & BTC_WSCB_ON)
+		force_exec = true;
 
-	btc->cx.cnt_wl[BTC_WCNT_SCBDUPDATE]++;
+	if (scbd_val != wl->scbd || force_exec) {
+		wl->scbd = scbd_val;
+		wl->scbd_change = true;
+	}
 }
 
 static u8
@@ -3428,8 +4134,158 @@ static void _update_wl_info(struct rtw89_dev *rtwdev)
 	}
 
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
-		    "[BTC], cnt_connect = %d, link_mode = %d\n",
-		    cnt_connect, wl_rinfo->link_mode);
+		    "[BTC], cnt_connect = %d, connecting = %d, link_mode = %d\n",
+		    cnt_connect, cnt_connecting, wl_rinfo->link_mode);
+
+	_fw_set_drv_info(rtwdev, CXDRVINFO_ROLE);
+}
+
+static void _update_wl_info_v1(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_wl_link_info *wl_linfo = wl->link_info;
+	struct rtw89_btc_wl_role_info_v1 *wl_rinfo = &wl->role_info_v1;
+	struct rtw89_btc_wl_dbcc_info *wl_dinfo = &wl->dbcc_info;
+	u8 cnt_connect = 0, cnt_connecting = 0, cnt_active = 0;
+	u8 cnt_2g = 0, cnt_5g = 0, phy;
+	u32 wl_2g_ch[2] = {}, wl_5g_ch[2] = {};
+	bool b2g = false, b5g = false, client_joined = false;
+	u8 i;
+
+	memset(wl_rinfo, 0, sizeof(*wl_rinfo));
+
+	for (i = 0; i < RTW89_PORT_NUM; i++) {
+		if (!wl_linfo[i].active)
+			continue;
+
+		cnt_active++;
+		wl_rinfo->active_role_v1[cnt_active - 1].role = wl_linfo[i].role;
+		wl_rinfo->active_role_v1[cnt_active - 1].pid = wl_linfo[i].pid;
+		wl_rinfo->active_role_v1[cnt_active - 1].phy = wl_linfo[i].phy;
+		wl_rinfo->active_role_v1[cnt_active - 1].band = wl_linfo[i].band;
+		wl_rinfo->active_role_v1[cnt_active - 1].noa = (u8)wl_linfo[i].noa;
+		wl_rinfo->active_role_v1[cnt_active - 1].connected = 0;
+
+		wl->port_id[wl_linfo[i].role] = wl_linfo[i].pid;
+
+		phy = wl_linfo[i].phy;
+
+		if (rtwdev->dbcc_en && phy < RTW89_PHY_MAX) {
+			wl_dinfo->role[phy] = wl_linfo[i].role;
+			wl_dinfo->op_band[phy] = wl_linfo[i].band;
+			_update_dbcc_band(rtwdev, phy);
+			_fw_set_drv_info(rtwdev, CXDRVINFO_DBCC);
+		}
+
+		if (wl_linfo[i].connected == MLME_NO_LINK) {
+			continue;
+		} else if (wl_linfo[i].connected == MLME_LINKING) {
+			cnt_connecting++;
+		} else {
+			cnt_connect++;
+			if ((wl_linfo[i].role == RTW89_WIFI_ROLE_P2P_GO ||
+			     wl_linfo[i].role == RTW89_WIFI_ROLE_AP) &&
+			     wl_linfo[i].client_cnt > 1)
+				client_joined = true;
+		}
+
+		wl_rinfo->role_map.val |= BIT(wl_linfo[i].role);
+		wl_rinfo->active_role_v1[cnt_active - 1].ch = wl_linfo[i].ch;
+		wl_rinfo->active_role_v1[cnt_active - 1].bw = wl_linfo[i].bw;
+		wl_rinfo->active_role_v1[cnt_active - 1].connected = 1;
+
+		/* only care 2 roles + BT coex */
+		if (wl_linfo[i].band != RTW89_BAND_2G) {
+			if (cnt_5g <= ARRAY_SIZE(wl_5g_ch) - 1)
+				wl_5g_ch[cnt_5g] = wl_linfo[i].ch;
+			cnt_5g++;
+			b5g = true;
+		} else {
+			if (cnt_2g <= ARRAY_SIZE(wl_2g_ch) - 1)
+				wl_2g_ch[cnt_2g] = wl_linfo[i].ch;
+			cnt_2g++;
+			b2g = true;
+		}
+	}
+
+	wl_rinfo->connect_cnt = cnt_connect;
+
+	/* Be careful to change the following sequence!! */
+	if (cnt_connect == 0) {
+		wl_rinfo->link_mode = BTC_WLINK_NOLINK;
+		wl_rinfo->role_map.role.none = 1;
+	} else if (!b2g && b5g) {
+		wl_rinfo->link_mode = BTC_WLINK_5G;
+	} else if (wl_rinfo->role_map.role.nan) {
+		wl_rinfo->link_mode = BTC_WLINK_2G_NAN;
+	} else if (cnt_connect > BTC_TDMA_WLROLE_MAX) {
+		wl_rinfo->link_mode = BTC_WLINK_OTHER;
+	} else  if (b2g && b5g && cnt_connect == 2) {
+		if (rtwdev->dbcc_en) {
+			switch (wl_dinfo->role[RTW89_PHY_0]) {
+			case RTW89_WIFI_ROLE_STATION:
+				wl_rinfo->link_mode = BTC_WLINK_2G_STA;
+				break;
+			case RTW89_WIFI_ROLE_P2P_GO:
+				wl_rinfo->link_mode = BTC_WLINK_2G_GO;
+				break;
+			case RTW89_WIFI_ROLE_P2P_CLIENT:
+				wl_rinfo->link_mode = BTC_WLINK_2G_GC;
+				break;
+			case RTW89_WIFI_ROLE_AP:
+				wl_rinfo->link_mode = BTC_WLINK_2G_AP;
+				break;
+			default:
+				wl_rinfo->link_mode = BTC_WLINK_OTHER;
+				break;
+			}
+		} else {
+			wl_rinfo->link_mode = BTC_WLINK_25G_MCC;
+		}
+	} else if (!b5g && cnt_connect == 2) {
+		if (wl_rinfo->role_map.role.station &&
+		    (wl_rinfo->role_map.role.p2p_go ||
+		    wl_rinfo->role_map.role.p2p_gc ||
+		    wl_rinfo->role_map.role.ap)) {
+			if (wl_2g_ch[0] == wl_2g_ch[1])
+				wl_rinfo->link_mode = BTC_WLINK_2G_SCC;
+			else
+				wl_rinfo->link_mode = BTC_WLINK_2G_MCC;
+		} else {
+			wl_rinfo->link_mode = BTC_WLINK_2G_MCC;
+		}
+	} else if (!b5g && cnt_connect == 1) {
+		if (wl_rinfo->role_map.role.station)
+			wl_rinfo->link_mode = BTC_WLINK_2G_STA;
+		else if (wl_rinfo->role_map.role.ap)
+			wl_rinfo->link_mode = BTC_WLINK_2G_AP;
+		else if (wl_rinfo->role_map.role.p2p_go)
+			wl_rinfo->link_mode = BTC_WLINK_2G_GO;
+		else if (wl_rinfo->role_map.role.p2p_gc)
+			wl_rinfo->link_mode = BTC_WLINK_2G_GC;
+		else
+			wl_rinfo->link_mode = BTC_WLINK_OTHER;
+	}
+
+	/* if no client_joined, don't care P2P-GO/AP role */
+	if (wl_rinfo->role_map.role.p2p_go || wl_rinfo->role_map.role.ap) {
+		if (!client_joined) {
+			if (wl_rinfo->link_mode == BTC_WLINK_2G_SCC ||
+			    wl_rinfo->link_mode == BTC_WLINK_2G_MCC) {
+				wl_rinfo->link_mode = BTC_WLINK_2G_STA;
+				wl_rinfo->connect_cnt = 1;
+			} else if (wl_rinfo->link_mode == BTC_WLINK_2G_GO ||
+				 wl_rinfo->link_mode == BTC_WLINK_2G_AP) {
+				wl_rinfo->link_mode = BTC_WLINK_NOLINK;
+				wl_rinfo->connect_cnt = 0;
+			}
+		}
+	}
+
+	rtw89_debug(rtwdev, RTW89_DBG_BTC,
+		    "[BTC], cnt_connect = %d, connecting = %d, link_mode = %d\n",
+		    cnt_connect, cnt_connecting, wl_rinfo->link_mode);
 
 	_fw_set_drv_info(rtwdev, CXDRVINFO_ROLE);
 }
@@ -3584,23 +4440,32 @@ static bool _chk_wl_rfk_request(struct rtw89_dev *rtwdev)
 static
 void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_dm *dm = &rtwdev->btc.dm;
 	struct rtw89_btc_cx *cx = &btc->cx;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
-	u8 mode = wl_rinfo->link_mode;
+	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
+	u8 mode;
 
 	lockdep_assert_held(&rtwdev->mutex);
-	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): reason=%d, mode=%d\n",
-		    __func__, reason, mode);
-	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): wl_only=%d, bt_only=%d\n",
-		    __func__, dm->wl_only, dm->bt_only);
 
 	dm->run_reason = reason;
 	_update_dm_step(rtwdev, reason);
 	_update_btc_state_map(rtwdev);
 
+	if (chip->chip_id == RTL8852A)
+		mode = wl_rinfo->link_mode;
+	else
+		mode = wl_rinfo_v1->link_mode;
+
+	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): reason=%d, mode=%d\n",
+		    __func__, reason, mode);
+	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): wl_only=%d, bt_only=%d\n",
+		    __func__, dm->wl_only, dm->bt_only);
+
 	/* Be careful to change the following function sequence!! */
 	if (btc->ctrl.manual) {
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
@@ -3657,6 +4522,7 @@ void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason)
 
 	btc->ctrl.igno_bt = false;
 	dm->freerun = false;
+	bt->scan_rx_low_pri = false;
 
 	if (reason == BTC_RSN_NTFY_INIT) {
 		_action_wl_init(rtwdev);
@@ -3699,21 +4565,30 @@ void _run_coex(struct rtw89_dev *rtwdev, enum btc_reason_and_action reason)
 		_action_wl_2g_sta(rtwdev);
 		break;
 	case BTC_WLINK_2G_AP:
+		bt->scan_rx_low_pri = true;
 		_action_wl_2g_ap(rtwdev);
 		break;
 	case BTC_WLINK_2G_GO:
+		bt->scan_rx_low_pri = true;
 		_action_wl_2g_go(rtwdev);
 		break;
 	case BTC_WLINK_2G_GC:
+		bt->scan_rx_low_pri = true;
 		_action_wl_2g_gc(rtwdev);
 		break;
 	case BTC_WLINK_2G_SCC:
-		_action_wl_2g_scc(rtwdev);
+		bt->scan_rx_low_pri = true;
+		if (chip->chip_id == RTL8852A)
+			_action_wl_2g_scc(rtwdev);
+		else if (chip->chip_id == RTL8852C)
+			_action_wl_2g_scc_v1(rtwdev);
 		break;
 	case BTC_WLINK_2G_MCC:
+		bt->scan_rx_low_pri = true;
 		_action_wl_2g_mcc(rtwdev);
 		break;
 	case BTC_WLINK_25G_MCC:
+		bt->scan_rx_low_pri = true;
 		_action_wl_25g_mcc(rtwdev);
 		break;
 	case BTC_WLINK_5G:
@@ -3743,11 +4618,14 @@ void rtw89_btc_ntfy_poweron(struct rtw89_dev *rtwdev)
 void rtw89_btc_ntfy_poweroff(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): !!\n", __func__);
 	btc->dm.cnt_notify[BTC_NCNT_POWER_OFF]++;
 
 	btc->cx.wl.status.map.rf_off = 1;
+	btc->cx.wl.status.map.busy = 0;
+	wl->status.map.lps = BTC_LPS_OFF;
 
 	_write_scbd(rtwdev, BTC_WSCB_ALL, false);
 	_run_coex(rtwdev, BTC_RSN_NTFY_POWEROFF);
@@ -3807,7 +4685,7 @@ void rtw89_btc_ntfy_init(struct rtw89_dev *rtwdev, u8 mode)
 	_write_scbd(rtwdev,
 		    BTC_WSCB_ACTIVE | BTC_WSCB_ON | BTC_WSCB_BTLOG, true);
 	_update_bt_scbd(rtwdev, true);
-	if (rtw89_mac_get_ctrl_path(rtwdev)) {
+	if (rtw89_mac_get_ctrl_path(rtwdev) && chip->chip_id == RTL8852A) {
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
 			    "[BTC], %s(): PTA owner warning!!\n",
 			    __func__);
@@ -4150,7 +5028,8 @@ enum btc_wl_mode {
 void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 			      struct rtw89_sta *rtwsta, enum btc_role_state state)
 {
-	struct rtw89_hal *hal = &rtwdev->hal;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif);
 	struct ieee80211_sta *sta = rtwsta_to_sta(rtwsta);
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -4165,8 +5044,7 @@ void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif
 		    vif->type == NL80211_IFTYPE_STATION);
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], port=%d\n", rtwvif->port);
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], band=%d ch=%d bw=%d\n",
-		    hal->current_band_type, hal->current_channel,
-		    hal->current_band_width);
+		    chan->band_type, chan->channel, chan->band_width);
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], associated=%d\n",
 		    state == BTC_ROLE_MSTS_STA_CONN_END);
 	rtw89_debug(rtwdev, RTW89_DBG_BTC,
@@ -4205,9 +5083,9 @@ void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif
 	r.connected = MLME_LINKED;
 	r.bcn_period = vif->bss_conf.beacon_int;
 	r.dtim_period = vif->bss_conf.dtim_period;
-	r.band = hal->current_band_type;
-	r.ch = hal->current_channel;
-	r.bw = hal->current_band_width;
+	r.band = chan->band_type;
+	r.ch = chan->channel;
+	r.bw = chan->band_width;
 	ether_addr_copy(r.mac_addr, rtwvif->mac_addr);
 
 	if (rtwsta && vif->type == NL80211_IFTYPE_STATION)
@@ -4218,7 +5096,10 @@ void rtw89_btc_ntfy_role_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif
 	wlinfo = &wl->link_info[r.pid];
 
 	memcpy(wlinfo, &r, sizeof(*wlinfo));
-	_update_wl_info(rtwdev);
+	if (chip->chip_id == RTL8852A)
+		_update_wl_info(rtwdev);
+	else
+		_update_wl_info_v1(rtwdev);
 
 	if (wlinfo->role == RTW89_WIFI_ROLE_STATION &&
 	    wlinfo->connected == MLME_NO_LINK)
@@ -4240,6 +5121,7 @@ void rtw89_btc_ntfy_radio_state(struct rtw89_dev *rtwdev, enum btc_rfctrl rf_sta
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	u32 val;
 
 	rtw89_debug(rtwdev, RTW89_DBG_BTC, "[BTC], %s(): rf_state = %d\n",
 		    __func__, rf_state);
@@ -4249,10 +5131,12 @@ void rtw89_btc_ntfy_radio_state(struct rtw89_dev *rtwdev, enum btc_rfctrl rf_sta
 	case BTC_RFCTRL_WL_OFF:
 		wl->status.map.rf_off = 1;
 		wl->status.map.lps = BTC_LPS_OFF;
+		wl->status.map.busy = 0;
 		break;
 	case BTC_RFCTRL_FW_CTRL:
 		wl->status.map.rf_off = 0;
 		wl->status.map.lps = BTC_LPS_RF_OFF;
+		wl->status.map.busy = 0;
 		break;
 	case BTC_RFCTRL_WL_ON:
 	default:
@@ -4262,14 +5146,17 @@ void rtw89_btc_ntfy_radio_state(struct rtw89_dev *rtwdev, enum btc_rfctrl rf_sta
 	}
 
 	if (rf_state == BTC_RFCTRL_WL_ON) {
+		btc->dm.cnt_dm[BTC_DCNT_BTCNT_FREEZE] = 0;
 		rtw89_btc_fw_en_rpt(rtwdev,
 				    RPT_EN_MREG | RPT_EN_BT_VER_INFO, true);
-		_write_scbd(rtwdev, BTC_WSCB_ACTIVE, true);
+		val = BTC_WSCB_ACTIVE | BTC_WSCB_ON | BTC_WSCB_BTLOG;
+		_write_scbd(rtwdev, val, true);
 		_update_bt_scbd(rtwdev, true);
 		chip->ops->btc_init_cfg(rtwdev);
 	} else {
 		rtw89_btc_fw_en_rpt(rtwdev, RPT_EN_ALL, false);
-		_write_scbd(rtwdev, BTC_WSCB_ACTIVE | BTC_WSCB_WLBUSY, false);
+		if (rf_state == BTC_RFCTRL_WL_OFF)
+			_write_scbd(rtwdev, BTC_WSCB_ALL, false);
 	}
 
 	_run_coex(rtwdev, BTC_RSN_NTFY_RADIO_STATE);
@@ -4609,10 +5496,10 @@ static void _show_cx_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 	seq_printf(m, "========== [BTC COEX INFO (%d)] ==========\n",
 		   chip->chip_id);
 
-	ver_main = FIELD_GET(GENMASK(31, 24), chip->para_ver);
-	ver_sub = FIELD_GET(GENMASK(23, 16), chip->para_ver);
-	ver_hotfix = FIELD_GET(GENMASK(15, 8), chip->para_ver);
-	id_branch = FIELD_GET(GENMASK(7, 0), chip->para_ver);
+	ver_main = FIELD_GET(GENMASK(31, 24), RTW89_COEX_VERSION);
+	ver_sub = FIELD_GET(GENMASK(23, 16), RTW89_COEX_VERSION);
+	ver_hotfix = FIELD_GET(GENMASK(15, 8), RTW89_COEX_VERSION);
+	id_branch = FIELD_GET(GENMASK(7, 0), RTW89_COEX_VERSION);
 	seq_printf(m, " %-15s : Coex:%d.%d.%d(branch:%d), ",
 		   "[coex_version]", ver_main, ver_sub, ver_hotfix, id_branch);
 
@@ -4726,23 +5613,29 @@ static void _show_wl_role_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 
 static void _show_wl_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_cx *cx = &btc->cx;
 	struct rtw89_btc_wl_info *wl = &cx->wl;
 	struct rtw89_btc_wl_role_info *wl_rinfo = &wl->role_info;
+	struct rtw89_btc_wl_role_info_v1 *wl_rinfo_v1 = &wl->role_info_v1;
+	u8 mode;
 
 	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_WL))
 		return;
 
 	seq_puts(m, "========== [WL Status] ==========\n");
 
-	seq_printf(m, " %-15s : link_mode:%d, ",
-		   "[status]", (u32)wl_rinfo->link_mode);
+	if (chip->chip_id == RTL8852A)
+		mode = wl_rinfo->link_mode;
+	else
+		mode = wl_rinfo_v1->link_mode;
+
+	seq_printf(m, " %-15s : link_mode:%d, ", "[status]", mode);
 
 	seq_printf(m,
-		   "rf_off:%s, power_save:%s, scan:%s(band:%d/phy_map:0x%x), ",
-		   wl->status.map.rf_off ? "Y" : "N",
-		   wl->status.map.lps ? "Y" : "N",
+		   "rf_off:%d, power_save:%d, scan:%s(band:%d/phy_map:0x%x), ",
+		   wl->status.map.rf_off, wl->status.map.lps,
 		   wl->status.map.scan ? "Y" : "N",
 		   wl->scan_info.band[RTW89_PHY_0], wl->scan_info.phy_map);
 
@@ -4908,6 +5801,7 @@ static void _show_bt_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 #define CASE_BTC_ACT_STR(e) case BTC_ACT_ ## e | BTC_ACT_EXT_BIT: return #e
 #define CASE_BTC_POLICY_STR(e) \
 	case BTC_CXP_ ## e | BTC_POLICY_EXT_BIT: return #e
+#define CASE_BTC_SLOT_STR(e) case CXST_ ## e: return #e
 
 static const char *steps_to_str(u16 step)
 {
@@ -4969,9 +5863,16 @@ static const char *steps_to_str(u16 step)
 	CASE_BTC_POLICY_STR(OFF_EQ3);
 	CASE_BTC_POLICY_STR(OFF_BWB0);
 	CASE_BTC_POLICY_STR(OFF_BWB1);
+	CASE_BTC_POLICY_STR(OFF_BWB2);
+	CASE_BTC_POLICY_STR(OFF_BWB3);
 	CASE_BTC_POLICY_STR(OFFB_BWB0);
 	CASE_BTC_POLICY_STR(OFFE_DEF);
 	CASE_BTC_POLICY_STR(OFFE_DEF2);
+	CASE_BTC_POLICY_STR(OFFE_2GBWISOB);
+	CASE_BTC_POLICY_STR(OFFE_2GISOB);
+	CASE_BTC_POLICY_STR(OFFE_2GBWMIXB);
+	CASE_BTC_POLICY_STR(OFFE_WL);
+	CASE_BTC_POLICY_STR(OFFE_2GBWMIXB2);
 	CASE_BTC_POLICY_STR(FIX_TD3030);
 	CASE_BTC_POLICY_STR(FIX_TD5050);
 	CASE_BTC_POLICY_STR(FIX_TD2030);
@@ -4982,6 +5883,7 @@ static const char *steps_to_str(u16 step)
 	CASE_BTC_POLICY_STR(FIX_TD2080);
 	CASE_BTC_POLICY_STR(FIX_TDW1B1);
 	CASE_BTC_POLICY_STR(FIX_TD4020);
+	CASE_BTC_POLICY_STR(FIX_TD4010ISO);
 	CASE_BTC_POLICY_STR(PFIX_TD3030);
 	CASE_BTC_POLICY_STR(PFIX_TD5050);
 	CASE_BTC_POLICY_STR(PFIX_TD2030);
@@ -4989,13 +5891,13 @@ static const char *steps_to_str(u16 step)
 	CASE_BTC_POLICY_STR(PFIX_TD3070);
 	CASE_BTC_POLICY_STR(PFIX_TD2080);
 	CASE_BTC_POLICY_STR(PFIX_TDW1B1);
-	CASE_BTC_POLICY_STR(AUTO_TD50200);
-	CASE_BTC_POLICY_STR(AUTO_TD60200);
-	CASE_BTC_POLICY_STR(AUTO_TD20200);
+	CASE_BTC_POLICY_STR(AUTO_TD50B1);
+	CASE_BTC_POLICY_STR(AUTO_TD60B1);
+	CASE_BTC_POLICY_STR(AUTO_TD20B1);
 	CASE_BTC_POLICY_STR(AUTO_TDW1B1);
-	CASE_BTC_POLICY_STR(PAUTO_TD50200);
-	CASE_BTC_POLICY_STR(PAUTO_TD60200);
-	CASE_BTC_POLICY_STR(PAUTO_TD20200);
+	CASE_BTC_POLICY_STR(PAUTO_TD50B1);
+	CASE_BTC_POLICY_STR(PAUTO_TD60B1);
+	CASE_BTC_POLICY_STR(PAUTO_TD20B1);
 	CASE_BTC_POLICY_STR(PAUTO_TDW1B1);
 	CASE_BTC_POLICY_STR(AUTO2_TD3050);
 	CASE_BTC_POLICY_STR(AUTO2_TD3070);
@@ -5014,6 +5916,32 @@ static const char *steps_to_str(u16 step)
 	}
 }
 
+static const char *id_to_slot(u32 id)
+{
+	switch (id) {
+	CASE_BTC_SLOT_STR(OFF);
+	CASE_BTC_SLOT_STR(B2W);
+	CASE_BTC_SLOT_STR(W1);
+	CASE_BTC_SLOT_STR(W2);
+	CASE_BTC_SLOT_STR(W2B);
+	CASE_BTC_SLOT_STR(B1);
+	CASE_BTC_SLOT_STR(B2);
+	CASE_BTC_SLOT_STR(B3);
+	CASE_BTC_SLOT_STR(B4);
+	CASE_BTC_SLOT_STR(LK);
+	CASE_BTC_SLOT_STR(BLK);
+	CASE_BTC_SLOT_STR(E2G);
+	CASE_BTC_SLOT_STR(E5G);
+	CASE_BTC_SLOT_STR(EBT);
+	CASE_BTC_SLOT_STR(ENULL);
+	CASE_BTC_SLOT_STR(WLK);
+	CASE_BTC_SLOT_STR(W1FDD);
+	CASE_BTC_SLOT_STR(B1FDD);
+	default:
+		return "unknown";
+	}
+}
+
 static
 void seq_print_segment(struct seq_file *m, const char *prefix, u16 *data,
 		       u8 len, u8 seg_len, u8 start_idx, u8 ring_len)
@@ -5105,21 +6033,31 @@ static void _show_dm_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 		   (bt->hi_lna_rx ? "Hi" : "Ori"), dm->wl_btg_rx);
 
 	seq_printf(m,
-		   " %-15s : wl_tx_limit[en:%d/max_t:%dus/max_retry:%d], bt_slot_reg:%d-TU\n",
+		   " %-15s : wl_tx_limit[en:%d/max_t:%dus/max_retry:%d], bt_slot_reg:%d-TU, bt_scan_rx_low_pri:%d\n",
 		   "[dm_ctrl]", dm->wl_tx_limit.enable, dm->wl_tx_limit.tx_time,
-		   dm->wl_tx_limit.tx_retry, btc->bt_req_len);
+		   dm->wl_tx_limit.tx_retry, btc->bt_req_len, bt->scan_rx_low_pri);
 }
 
 static void _show_error(struct rtw89_dev *rtwdev, struct seq_file *m)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
-	struct rtw89_btc_fbtc_cysta *pcysta = NULL;
-
-	pcysta = &pfwinfo->rpt_fbtc_cysta.finfo;
+	struct rtw89_btc_fbtc_cysta *pcysta;
+	struct rtw89_btc_fbtc_cysta_v1 *pcysta_v1;
+	u32 except_cnt, exception_map;
+
+	if (chip->chip_id == RTL8852A) {
+		pcysta = &pfwinfo->rpt_fbtc_cysta.finfo;
+		except_cnt = le32_to_cpu(pcysta->except_cnt);
+		exception_map = le32_to_cpu(pcysta->exception);
+	} else {
+		pcysta_v1 = &pfwinfo->rpt_fbtc_cysta.finfo_v1;
+		except_cnt = le32_to_cpu(pcysta_v1->except_cnt);
+		exception_map = le32_to_cpu(pcysta_v1->except_map);
+	}
 
-	if (pfwinfo->event[BTF_EVNT_BUF_OVERFLOW] == 0 &&
-	    pcysta->except_cnt == 0 &&
+	if (pfwinfo->event[BTF_EVNT_BUF_OVERFLOW] == 0 && except_cnt == 0 &&
 	    !pfwinfo->len_mismch && !pfwinfo->fver_mismch)
 		return;
 
@@ -5144,16 +6082,17 @@ static void _show_error(struct rtw89_dev *rtwdev, struct seq_file *m)
 	}
 
 	/* cycle statistics exceptions */
-	if (pcysta->exception || pcysta->except_cnt) {
+	if (exception_map || except_cnt) {
 		seq_printf(m,
 			   "exception-type: 0x%x, exception-cnt = %d",
-			   pcysta->exception, pcysta->except_cnt);
+			   exception_map, except_cnt);
 	}
 	seq_puts(m, "\n");
 }
 
 static void _show_fbtc_tdma(struct rtw89_dev *rtwdev, struct seq_file *m)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
 	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
@@ -5166,7 +6105,10 @@ static void _show_fbtc_tdma(struct rtw89_dev *rtwdev, struct seq_file *m)
 	if (!pcinfo->valid)
 		return;
 
-	t = &pfwinfo->rpt_fbtc_tdma.finfo;
+	if (chip->chip_id == RTL8852A)
+		t = &pfwinfo->rpt_fbtc_tdma.finfo;
+	else
+		t = &pfwinfo->rpt_fbtc_tdma.finfo_v1.tdma;
 
 	seq_printf(m,
 		   " %-15s : ", "[tdma_policy]");
@@ -5369,12 +6311,145 @@ static void _show_fbtc_cysta(struct rtw89_dev *rtwdev, struct seq_file *m)
 	}
 }
 
+static void _show_fbtc_cysta_v1(struct rtw89_dev *rtwdev, struct seq_file *m)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_bt_a2dp_desc *a2dp = &btc->cx.bt.link_info.a2dp_desc;
+	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct rtw89_btc_dm *dm = &btc->dm;
+	struct rtw89_btc_fbtc_a2dp_trx_stat *a2dp_trx;
+	struct rtw89_btc_fbtc_cysta_v1 *pcysta;
+	struct rtw89_btc_rpt_cmn_info *pcinfo;
+	u8 i, cnt = 0, slot_pair, divide_cnt;
+	u16 cycle, c_begin, c_end, store_index;
+
+	pcinfo = &pfwinfo->rpt_fbtc_cysta.cinfo;
+	if (!pcinfo->valid)
+		return;
+
+	pcysta = &pfwinfo->rpt_fbtc_cysta.finfo_v1;
+	seq_printf(m,
+		   " %-15s : cycle:%d, bcn[all:%d/all_ok:%d/bt:%d/bt_ok:%d]",
+		   "[cycle_cnt]",
+		   le16_to_cpu(pcysta->cycles),
+		   le32_to_cpu(pcysta->bcn_cnt[CXBCN_ALL]),
+		   le32_to_cpu(pcysta->bcn_cnt[CXBCN_ALL_OK]),
+		   le32_to_cpu(pcysta->bcn_cnt[CXBCN_BT_SLOT]),
+		   le32_to_cpu(pcysta->bcn_cnt[CXBCN_BT_OK]));
+
+	for (i = 0; i < CXST_MAX; i++) {
+		if (!le32_to_cpu(pcysta->slot_cnt[i]))
+			continue;
+
+		seq_printf(m, ", %s:%d", id_to_slot(i),
+			   le32_to_cpu(pcysta->slot_cnt[i]));
+	}
+
+	if (dm->tdma_now.rxflctrl)
+		seq_printf(m, ", leak_rx:%d", le32_to_cpu(pcysta->leak_slot.cnt_rximr));
+
+	if (le32_to_cpu(pcysta->collision_cnt))
+		seq_printf(m, ", collision:%d", le32_to_cpu(pcysta->collision_cnt));
+
+	if (le32_to_cpu(pcysta->skip_cnt))
+		seq_printf(m, ", skip:%d", le32_to_cpu(pcysta->skip_cnt));
+
+	seq_puts(m, "\n");
+
+	seq_printf(m, " %-15s : avg_t[wl:%d/bt:%d/lk:%d.%03d]",
+		   "[cycle_time]",
+		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_WL]),
+		   le16_to_cpu(pcysta->cycle_time.tavg[CXT_BT]),
+		   le16_to_cpu(pcysta->leak_slot.tavg) / 1000,
+		   le16_to_cpu(pcysta->leak_slot.tavg) % 1000);
+	seq_printf(m,
+		   ", max_t[wl:%d/bt:%d/lk:%d.%03d]",
+		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_WL]),
+		   le16_to_cpu(pcysta->cycle_time.tmax[CXT_BT]),
+		   le16_to_cpu(pcysta->leak_slot.tmax) / 1000,
+		   le16_to_cpu(pcysta->leak_slot.tmax) % 1000);
+	seq_printf(m,
+		   ", maxdiff_t[wl:%d/bt:%d]\n",
+		   le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_WL]),
+		   le16_to_cpu(pcysta->cycle_time.tmaxdiff[CXT_BT]));
+
+	cycle = le16_to_cpu(pcysta->cycles);
+	if (cycle == 0)
+		return;
+
+	/* 1 cycle record 1 wl-slot and 1 bt-slot */
+	slot_pair = BTC_CYCLE_SLOT_MAX / 2;
+
+	if (cycle <= slot_pair)
+		c_begin = 1;
+	else
+		c_begin = cycle - slot_pair + 1;
+
+	c_end = cycle;
+
+	if (a2dp->exist)
+		divide_cnt = 3;
+	else
+		divide_cnt = BTC_CYCLE_SLOT_MAX / 4;
+
+	for (cycle = c_begin; cycle <= c_end; cycle++) {
+		cnt++;
+		store_index = ((cycle - 1) % slot_pair) * 2;
+
+		if (cnt % divide_cnt == 1) {
+			seq_printf(m, "\n\r %-15s : ", "[cycle_step]");
+		} else {
+			seq_printf(m, "->b%02d",
+				   le16_to_cpu(pcysta->slot_step_time[store_index]));
+			if (a2dp->exist) {
+				a2dp_trx = &pcysta->a2dp_trx[store_index];
+				seq_printf(m, "(%d/%d/%dM/%d/%d/%d)",
+					   a2dp_trx->empty_cnt,
+					   a2dp_trx->retry_cnt,
+					   a2dp_trx->tx_rate ? 3 : 2,
+					   a2dp_trx->tx_cnt,
+					   a2dp_trx->ack_cnt,
+					   a2dp_trx->nack_cnt);
+			}
+			seq_printf(m, "->w%02d",
+				   le16_to_cpu(pcysta->slot_step_time[store_index + 1]));
+			if (a2dp->exist) {
+				a2dp_trx = &pcysta->a2dp_trx[store_index + 1];
+				seq_printf(m, "(%d/%d/%dM/%d/%d/%d)",
+					   a2dp_trx->empty_cnt,
+					   a2dp_trx->retry_cnt,
+					   a2dp_trx->tx_rate ? 3 : 2,
+					   a2dp_trx->tx_cnt,
+					   a2dp_trx->ack_cnt,
+					   a2dp_trx->nack_cnt);
+			}
+		}
+		if (cnt % (BTC_CYCLE_SLOT_MAX / 4) == 0 || cnt == c_end)
+			seq_puts(m, "\n");
+	}
+
+	if (a2dp->exist) {
+		seq_printf(m, "%-15s : a2dp_ept:%d, a2dp_late:%d",
+			   "[a2dp_t_sta]",
+			   le16_to_cpu(pcysta->a2dp_ept.cnt),
+			   le16_to_cpu(pcysta->a2dp_ept.cnt_timeout));
+
+		seq_printf(m, ", avg_t:%d, max_t:%d",
+			   le16_to_cpu(pcysta->a2dp_ept.tavg),
+			   le16_to_cpu(pcysta->a2dp_ept.tmax));
+
+		seq_puts(m, "\n");
+	}
+}
+
 static void _show_fbtc_nullsta(struct rtw89_dev *rtwdev, struct seq_file *m)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
-	struct rtw89_btc_rpt_cmn_info *pcinfo = NULL;
-	struct rtw89_btc_fbtc_cynullsta *ns = NULL;
+	struct rtw89_btc_rpt_cmn_info *pcinfo;
+	struct rtw89_btc_fbtc_cynullsta *ns;
+	struct rtw89_btc_fbtc_cynullsta_v1 *ns_v1;
 	u8 i = 0;
 
 	if (!btc->dm.tdma_now.rxflctrl)
@@ -5384,25 +6459,58 @@ static void _show_fbtc_nullsta(struct rtw89_dev *rtwdev, struct seq_file *m)
 	if (!pcinfo->valid)
 		return;
 
-	ns = &pfwinfo->rpt_fbtc_nullsta.finfo;
+	if (chip->chip_id == RTL8852A) {
+		ns = &pfwinfo->rpt_fbtc_nullsta.finfo;
 
-	seq_printf(m, " %-15s : ", "[null_sta]");
+		seq_printf(m, " %-15s : ", "[null_sta]");
 
-	for (i = 0; i < 2; i++) {
-		if (i != 0)
-			seq_printf(m, ", null-%d", i);
-		else
-			seq_printf(m, "null-%d", i);
-		seq_printf(m, "[ok:%d/", le32_to_cpu(ns->result[i][1]));
-		seq_printf(m, "fail:%d/", le32_to_cpu(ns->result[i][0]));
-		seq_printf(m, "on_time:%d/", le32_to_cpu(ns->result[i][2]));
-		seq_printf(m, "retry:%d/", le32_to_cpu(ns->result[i][3]));
-		seq_printf(m, "avg_t:%d.%03d/",
-			   le32_to_cpu(ns->avg_t[i]) / 1000,
-			   le32_to_cpu(ns->avg_t[i]) % 1000);
-		seq_printf(m, "max_t:%d.%03d]",
-			   le32_to_cpu(ns->max_t[i]) / 1000,
-			   le32_to_cpu(ns->max_t[i]) % 1000);
+		for (i = 0; i < 2; i++) {
+			if (i != 0)
+				seq_printf(m, ", null-%d", i);
+			else
+				seq_printf(m, "null-%d", i);
+			seq_printf(m, "[ok:%d/",
+				   le32_to_cpu(ns->result[i][1]));
+			seq_printf(m, "fail:%d/",
+				   le32_to_cpu(ns->result[i][0]));
+			seq_printf(m, "on_time:%d/",
+				   le32_to_cpu(ns->result[i][2]));
+			seq_printf(m, "retry:%d/",
+				   le32_to_cpu(ns->result[i][3]));
+			seq_printf(m, "avg_t:%d.%03d/",
+				   le32_to_cpu(ns->avg_t[i]) / 1000,
+				   le32_to_cpu(ns->avg_t[i]) % 1000);
+			seq_printf(m, "max_t:%d.%03d]",
+				   le32_to_cpu(ns->max_t[i]) / 1000,
+				   le32_to_cpu(ns->max_t[i]) % 1000);
+		}
+	} else {
+		ns_v1 = &pfwinfo->rpt_fbtc_nullsta.finfo_v1;
+
+		seq_printf(m, " %-15s : ", "[null_sta]");
+
+		for (i = 0; i < 2; i++) {
+			if (i != 0)
+				seq_printf(m, ", null-%d", i);
+			else
+				seq_printf(m, "null-%d", i);
+			seq_printf(m, "[Tx:%d/",
+				   le32_to_cpu(ns_v1->result[i][4]));
+			seq_printf(m, "[ok:%d/",
+				   le32_to_cpu(ns_v1->result[i][1]));
+			seq_printf(m, "fail:%d/",
+				   le32_to_cpu(ns_v1->result[i][0]));
+			seq_printf(m, "on_time:%d/",
+				   le32_to_cpu(ns_v1->result[i][2]));
+			seq_printf(m, "retry:%d/",
+				   le32_to_cpu(ns_v1->result[i][3]));
+			seq_printf(m, "avg_t:%d.%03d/",
+				   le32_to_cpu(ns_v1->avg_t[i]) / 1000,
+				   le32_to_cpu(ns_v1->avg_t[i]) % 1000);
+			seq_printf(m, "max_t:%d.%03d]",
+				   le32_to_cpu(ns_v1->max_t[i]) / 1000,
+				   le32_to_cpu(ns_v1->max_t[i]) % 1000);
+		}
 	}
 	seq_puts(m, "\n");
 }
@@ -5478,6 +6586,7 @@ static void _show_fbtc_step(struct rtw89_dev *rtwdev, struct seq_file *m)
 
 static void _show_fw_dm_msg(struct rtw89_dev *rtwdev, struct seq_file *m)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 
 	if (!(btc->dm.coex_info_map & BTC_COEX_INFO_DM))
@@ -5486,11 +6595,57 @@ static void _show_fw_dm_msg(struct rtw89_dev *rtwdev, struct seq_file *m)
 	_show_error(rtwdev, m);
 	_show_fbtc_tdma(rtwdev, m);
 	_show_fbtc_slots(rtwdev, m);
-	_show_fbtc_cysta(rtwdev, m);
+
+	if (chip->chip_id == RTL8852A)
+		_show_fbtc_cysta(rtwdev, m);
+	else
+		_show_fbtc_cysta_v1(rtwdev, m);
+
 	_show_fbtc_nullsta(rtwdev, m);
 	_show_fbtc_step(rtwdev, m);
 }
 
+static void _get_gnt(struct rtw89_dev *rtwdev, struct rtw89_mac_ax_coex_gnt *gnt_cfg)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_mac_ax_gnt *gnt;
+	u32 val, status;
+
+	if (chip->chip_id == RTL8852A || chip->chip_id == RTL8852B) {
+		rtw89_mac_read_lte(rtwdev, R_AX_LTE_SW_CFG_1, &val);
+		rtw89_mac_read_lte(rtwdev, R_AX_GNT_VAL, &status);
+
+		gnt = &gnt_cfg->band[0];
+		gnt->gnt_bt_sw_en = !!(val & B_AX_GNT_BT_RFC_S0_SW_CTRL);
+		gnt->gnt_bt = !!(status & B_AX_GNT_BT_RFC_S0_STA);
+		gnt->gnt_wl_sw_en = !!(val & B_AX_GNT_WL_RFC_S0_SW_CTRL);
+		gnt->gnt_wl = !!(status & B_AX_GNT_WL_RFC_S0_STA);
+
+		gnt = &gnt_cfg->band[1];
+		gnt->gnt_bt_sw_en = !!(val & B_AX_GNT_BT_RFC_S1_SW_CTRL);
+		gnt->gnt_bt = !!(status & B_AX_GNT_BT_RFC_S1_STA);
+		gnt->gnt_wl_sw_en = !!(val & B_AX_GNT_WL_RFC_S1_SW_CTRL);
+		gnt->gnt_wl = !!(status & B_AX_GNT_WL_RFC_S1_STA);
+	} else if (chip->chip_id == RTL8852C) {
+		val = rtw89_read32(rtwdev, R_AX_GNT_SW_CTRL);
+		status = rtw89_read32(rtwdev, R_AX_GNT_VAL_V1);
+
+		gnt = &gnt_cfg->band[0];
+		gnt->gnt_bt_sw_en = !!(val & B_AX_GNT_BT_RFC_S0_SWCTRL);
+		gnt->gnt_bt = !!(status & B_AX_GNT_BT_RFC_S0);
+		gnt->gnt_wl_sw_en = !!(val & B_AX_GNT_WL_RFC_S0_SWCTRL);
+		gnt->gnt_wl = !!(status & B_AX_GNT_WL_RFC_S0);
+
+		gnt = &gnt_cfg->band[1];
+		gnt->gnt_bt_sw_en = !!(val & B_AX_GNT_BT_RFC_S1_SWCTRL);
+		gnt->gnt_bt = !!(status & B_AX_GNT_BT_RFC_S1);
+		gnt->gnt_wl_sw_en = !!(val & B_AX_GNT_WL_RFC_S1_SWCTRL);
+		gnt->gnt_wl = !!(status & B_AX_GNT_WL_RFC_S1);
+	} else {
+		return;
+	}
+}
+
 static void _show_mreg(struct rtw89_dev *rtwdev, struct seq_file *m)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
@@ -5502,7 +6657,8 @@ static void _show_mreg(struct rtw89_dev *rtwdev, struct seq_file *m)
 	struct rtw89_btc_cx *cx = &btc->cx;
 	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
 	struct rtw89_btc_bt_info *bt = &btc->cx.bt;
-	struct rtw89_mac_ax_gnt gnt[2] = {0};
+	struct rtw89_mac_ax_coex_gnt gnt_cfg = {};
+	struct rtw89_mac_ax_gnt gnt;
 	u8 i = 0, type = 0, cnt = 0;
 	u32 val, offset;
 
@@ -5519,45 +6675,28 @@ static void _show_mreg(struct rtw89_dev *rtwdev, struct seq_file *m)
 
 	/* To avoid I/O if WL LPS or power-off  */
 	if (!wl->status.map.lps && !wl->status.map.rf_off) {
-		rtw89_mac_read_lte(rtwdev, R_AX_LTE_SW_CFG_1, &val);
-		if (val & (B_AX_GNT_BT_RFC_S0_SW_VAL |
-		    B_AX_GNT_BT_BB_S0_SW_VAL))
-			gnt[0].gnt_bt = true;
-		if (val & (B_AX_GNT_BT_RFC_S0_SW_CTRL |
-		    B_AX_GNT_BT_BB_S0_SW_CTRL))
-			gnt[0].gnt_bt_sw_en = true;
-		if (val & (B_AX_GNT_WL_RFC_S0_SW_VAL |
-		    B_AX_GNT_WL_BB_S0_SW_VAL))
-			gnt[0].gnt_wl = true;
-		if (val & (B_AX_GNT_WL_RFC_S0_SW_CTRL |
-		    B_AX_GNT_WL_BB_S0_SW_CTRL))
-			gnt[0].gnt_wl_sw_en = true;
-
-		if (val & (B_AX_GNT_BT_RFC_S1_SW_VAL |
-		    B_AX_GNT_BT_BB_S1_SW_VAL))
-			gnt[1].gnt_bt = true;
-		if (val & (B_AX_GNT_BT_RFC_S1_SW_CTRL |
-		    B_AX_GNT_BT_BB_S1_SW_CTRL))
-			gnt[1].gnt_bt_sw_en = true;
-		if (val & (B_AX_GNT_WL_RFC_S1_SW_VAL |
-		    B_AX_GNT_WL_BB_S1_SW_VAL))
-			gnt[1].gnt_wl = true;
-		if (val & (B_AX_GNT_WL_RFC_S1_SW_CTRL |
-		    B_AX_GNT_WL_BB_S1_SW_CTRL))
-			gnt[1].gnt_wl_sw_en = true;
+		if (chip->chip_id == RTL8852A)
+			btc->dm.pta_owner = rtw89_mac_get_ctrl_path(rtwdev);
+		else if (chip->chip_id == RTL8852C)
+			btc->dm.pta_owner = 0;
 
+		_get_gnt(rtwdev, &gnt_cfg);
+		gnt = gnt_cfg.band[0];
 		seq_printf(m,
 			   " %-15s : pta_owner:%s, phy-0[gnt_wl:%s-%d/gnt_bt:%s-%d], ",
 			   "[gnt_status]",
-			   (rtw89_mac_get_ctrl_path(rtwdev) ? "WL" : "BT"),
-			   (gnt[0].gnt_wl_sw_en ? "SW" : "HW"), gnt[0].gnt_wl,
-			   (gnt[0].gnt_bt_sw_en ? "SW" : "HW"), gnt[0].gnt_bt);
+			   chip->chip_id == RTL8852C ? "HW" :
+			   btc->dm.pta_owner == BTC_CTRL_BY_WL ? "WL" : "BT",
+			   gnt.gnt_wl_sw_en ? "SW" : "HW", gnt.gnt_wl,
+			   gnt.gnt_bt_sw_en ? "SW" : "HW", gnt.gnt_bt);
 
+		gnt = gnt_cfg.band[1];
 		seq_printf(m, "phy-1[gnt_wl:%s-%d/gnt_bt:%s-%d]\n",
-			   (gnt[1].gnt_wl_sw_en ? "SW" : "HW"), gnt[1].gnt_wl,
-			   (gnt[1].gnt_bt_sw_en ? "SW" : "HW"), gnt[1].gnt_bt);
+			   gnt.gnt_wl_sw_en ? "SW" : "HW",
+			   gnt.gnt_wl,
+			   gnt.gnt_bt_sw_en ? "SW" : "HW",
+			   gnt.gnt_bt);
 	}
-
 	pcinfo = &pfwinfo->rpt_fbtc_mregval.cinfo;
 	if (!pcinfo->valid) {
 		rtw89_debug(rtwdev, RTW89_DBG_BTC,
@@ -5714,8 +6853,121 @@ static void _show_summary(struct rtw89_dev *rtwdev, struct seq_file *m)
 		   cnt[BTC_NCNT_CUSTOMERIZE]);
 }
 
+static void _show_summary_v1(struct rtw89_dev *rtwdev, struct seq_file *m)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_btf_fwinfo *pfwinfo = &btc->fwinfo;
+	struct rtw89_btc_fbtc_rpt_ctrl_v1 *prptctrl;
+	struct rtw89_btc_rpt_cmn_info *pcinfo;
+	struct rtw89_btc_cx *cx = &btc->cx;
+	struct rtw89_btc_dm *dm = &btc->dm;
+	struct rtw89_btc_wl_info *wl = &cx->wl;
+	struct rtw89_btc_bt_info *bt = &cx->bt;
+	u32 cnt_sum = 0, *cnt = btc->dm.cnt_notify;
+	u8 i;
+
+	if (!(dm->coex_info_map & BTC_COEX_INFO_SUMMARY))
+		return;
+
+	seq_puts(m, "========== [Statistics] ==========\n");
+
+	pcinfo = &pfwinfo->rpt_ctrl.cinfo;
+	if (pcinfo->valid && !wl->status.map.lps && !wl->status.map.rf_off) {
+		prptctrl = &pfwinfo->rpt_ctrl.finfo_v1;
+
+		seq_printf(m,
+			   " %-15s : h2c_cnt=%d(fail:%d, fw_recv:%d), c2h_cnt=%d(fw_send:%d), ",
+			   "[summary]", pfwinfo->cnt_h2c,
+			   pfwinfo->cnt_h2c_fail,
+			   le32_to_cpu(prptctrl->rpt_info.cnt_h2c),
+			   pfwinfo->cnt_c2h,
+			   le32_to_cpu(prptctrl->rpt_info.cnt_c2h));
+
+		seq_printf(m,
+			   "rpt_cnt=%d(fw_send:%d), rpt_map=0x%x, dm_error_map:0x%x",
+			   pfwinfo->event[BTF_EVNT_RPT],
+			   le32_to_cpu(prptctrl->rpt_info.cnt),
+			   le32_to_cpu(prptctrl->rpt_info.en),
+			   dm->error.val);
+
+		if (dm->error.map.wl_fw_hang)
+			seq_puts(m, " (WL FW Hang!!)");
+		seq_puts(m, "\n");
+		seq_printf(m,
+			   " %-15s : send_ok:%d, send_fail:%d, recv:%d, ",
+			   "[mailbox]",
+			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_ok),
+			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_send_fail),
+			   le32_to_cpu(prptctrl->bt_mbx_info.cnt_recv));
+
+		seq_printf(m,
+			   "A2DP_empty:%d(stop:%d, tx:%d, ack:%d, nack:%d)\n",
+			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_empty),
+			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_flowctrl),
+			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_tx),
+			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_ack),
+			   le32_to_cpu(prptctrl->bt_mbx_info.a2dp.cnt_nack));
+
+		seq_printf(m,
+			   " %-15s : wl_rfk[req:%d/go:%d/reject:%d/timeout:%d]",
+			   "[RFK]", cx->cnt_wl[BTC_WCNT_RFK_REQ],
+			   cx->cnt_wl[BTC_WCNT_RFK_GO],
+			   cx->cnt_wl[BTC_WCNT_RFK_REJECT],
+			   cx->cnt_wl[BTC_WCNT_RFK_TIMEOUT]);
+
+		seq_printf(m,
+			   ", bt_rfk[req:%d/go:%d/reject:%d/timeout:%d/fail:%d]\n",
+			   le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REQ]),
+			   le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_GO]),
+			   le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_REJECT]),
+			   le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_TIMEOUT]),
+			   le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_FAIL]));
+
+		if (le32_to_cpu(prptctrl->bt_cnt[BTC_BCNT_RFK_TIMEOUT]) > 0)
+			bt->rfk_info.map.timeout = 1;
+		else
+			bt->rfk_info.map.timeout = 0;
+
+		dm->error.map.wl_rfk_timeout = bt->rfk_info.map.timeout;
+	} else {
+		seq_printf(m,
+			   " %-15s : h2c_cnt=%d(fail:%d), c2h_cnt=%d, rpt_cnt=%d, rpt_map=0x%x",
+			   "[summary]", pfwinfo->cnt_h2c,
+			   pfwinfo->cnt_h2c_fail, pfwinfo->cnt_c2h,
+			   pfwinfo->event[BTF_EVNT_RPT],
+			   btc->fwinfo.rpt_en_map);
+		seq_puts(m, " (WL FW report invalid!!)\n");
+	}
+
+	for (i = 0; i < BTC_NCNT_NUM; i++)
+		cnt_sum += dm->cnt_notify[i];
+
+	seq_printf(m,
+		   " %-15s : total=%d, show_coex_info=%d, power_on=%d, init_coex=%d, ",
+		   "[notify_cnt]", cnt_sum, cnt[BTC_NCNT_SHOW_COEX_INFO],
+		   cnt[BTC_NCNT_POWER_ON], cnt[BTC_NCNT_INIT_COEX]);
+
+	seq_printf(m,
+		   "power_off=%d, radio_state=%d, role_info=%d, wl_rfk=%d, wl_sta=%d\n",
+		   cnt[BTC_NCNT_POWER_OFF], cnt[BTC_NCNT_RADIO_STATE],
+		   cnt[BTC_NCNT_ROLE_INFO], cnt[BTC_NCNT_WL_RFK],
+		   cnt[BTC_NCNT_WL_STA]);
+
+	seq_printf(m,
+		   " %-15s : scan_start=%d, scan_finish=%d, switch_band=%d, special_pkt=%d, ",
+		   "[notify_cnt]", cnt[BTC_NCNT_SCAN_START],
+		   cnt[BTC_NCNT_SCAN_FINISH], cnt[BTC_NCNT_SWITCH_BAND],
+		   cnt[BTC_NCNT_SPECIAL_PACKET]);
+
+	seq_printf(m,
+		   "timer=%d, control=%d, customerize=%d\n",
+		   cnt[BTC_NCNT_TIMER], cnt[BTC_NCNT_CONTROL],
+		   cnt[BTC_NCNT_CUSTOMERIZE]);
+}
+
 void rtw89_btc_dump_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_fw_suit *fw_suit = &rtwdev->fw.normal;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_cx *cx = &btc->cx;
@@ -5746,5 +6998,8 @@ void rtw89_btc_dump_info(struct rtw89_dev *rtwdev, struct seq_file *m)
 	_show_dm_info(rtwdev, m);
 	_show_fw_dm_msg(rtwdev, m);
 	_show_mreg(rtwdev, m);
-	_show_summary(rtwdev, m);
+	if (chip->chip_id == RTL8852A)
+		_show_summary(rtwdev, m);
+	else
+		_show_summary_v1(rtwdev, m);
 }
diff --git a/drivers/net/wireless/realtek/rtw89/coex.h b/drivers/net/wireless/realtek/rtw89/coex.h
index c3a722d259d7..ca16afa97ec0 100644
--- a/drivers/net/wireless/realtek/rtw89/coex.h
+++ b/drivers/net/wireless/realtek/rtw89/coex.h
@@ -162,17 +162,19 @@ void rtw89_coex_act1_work(struct work_struct *work);
 void rtw89_coex_bt_devinfo_work(struct work_struct *work);
 void rtw89_coex_rfk_chk_work(struct work_struct *work);
 void rtw89_coex_power_on(struct rtw89_dev *rtwdev);
+void rtw89_btc_set_policy(struct rtw89_dev *rtwdev, u16 policy_type);
+void rtw89_btc_set_policy_v1(struct rtw89_dev *rtwdev, u16 policy_type);
 
 static inline u8 rtw89_btc_phymap(struct rtw89_dev *rtwdev,
 				  enum rtw89_phy_idx phy_idx,
 				  enum rtw89_rf_path_bit paths)
 {
-	struct rtw89_hal *hal = &rtwdev->hal;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	u8 phy_map;
 
 	phy_map = FIELD_PREP(BTC_RFK_PATH_MAP, paths) |
 		  FIELD_PREP(BTC_RFK_PHY_MAP, BIT(phy_idx)) |
-		  FIELD_PREP(BTC_RFK_BAND_MAP, hal->current_band_type);
+		  FIELD_PREP(BTC_RFK_BAND_MAP, chan->band_type);
 
 	return phy_map;
 }
diff --git a/drivers/net/wireless/realtek/rtw89/core.c b/drivers/net/wireless/realtek/rtw89/core.c
index a5880a54812e..bc2994865372 100644
--- a/drivers/net/wireless/realtek/rtw89/core.c
+++ b/drivers/net/wireless/realtek/rtw89/core.c
@@ -5,6 +5,7 @@
 #include <linux/udp.h>
 
 #include "cam.h"
+#include "chan.h"
 #include "coex.h"
 #include "core.h"
 #include "efuse.h"
@@ -224,18 +225,22 @@ static void rtw89_traffic_stats_accu(struct rtw89_dev *rtwdev,
 	}
 }
 
-static void rtw89_get_channel_params(struct cfg80211_chan_def *chandef,
-				     struct rtw89_channel_params *chan_param)
+void rtw89_get_default_chandef(struct cfg80211_chan_def *chandef)
+{
+	cfg80211_chandef_create(chandef, &rtw89_channels_2ghz[0],
+				NL80211_CHAN_NO_HT);
+}
+
+static void rtw89_get_channel_params(const struct cfg80211_chan_def *chandef,
+				     struct rtw89_chan *chan)
 {
 	struct ieee80211_channel *channel = chandef->chan;
 	enum nl80211_chan_width width = chandef->width;
 	u32 primary_freq, center_freq;
 	u8 center_chan;
 	u8 bandwidth = RTW89_CHANNEL_WIDTH_20;
-	u8 primary_chan_idx = 0;
 	u32 offset;
 	u8 band;
-	u8 subband;
 
 	center_chan = channel->hw_value;
 	primary_freq = channel->center_freq;
@@ -245,15 +250,12 @@ static void rtw89_get_channel_params(struct cfg80211_chan_def *chandef,
 	case NL80211_CHAN_WIDTH_20_NOHT:
 	case NL80211_CHAN_WIDTH_20:
 		bandwidth = RTW89_CHANNEL_WIDTH_20;
-		primary_chan_idx = RTW89_SC_DONT_CARE;
 		break;
 	case NL80211_CHAN_WIDTH_40:
 		bandwidth = RTW89_CHANNEL_WIDTH_40;
 		if (primary_freq > center_freq) {
-			primary_chan_idx = RTW89_SC_20_UPPER;
 			center_chan -= 2;
 		} else {
-			primary_chan_idx = RTW89_SC_20_LOWER;
 			center_chan += 2;
 		}
 		break;
@@ -262,11 +264,9 @@ static void rtw89_get_channel_params(struct cfg80211_chan_def *chandef,
 		bandwidth = nl_to_rtw89_bandwidth(width);
 		if (primary_freq > center_freq) {
 			offset = (primary_freq - center_freq - 10) / 20;
-			primary_chan_idx = RTW89_SC_20_UPPER + offset * 2;
 			center_chan -= 2 + offset * 4;
 		} else {
 			offset = (center_freq - primary_freq - 10) / 20;
-			primary_chan_idx = RTW89_SC_20_LOWER + offset * 2;
 			center_chan += 2 + offset * 4;
 		}
 		break;
@@ -288,110 +288,76 @@ static void rtw89_get_channel_params(struct cfg80211_chan_def *chandef,
 		break;
 	}
 
-	switch (band) {
-	default:
-	case RTW89_BAND_2G:
-		switch (center_chan) {
-		default:
-		case 1 ... 14:
-			subband = RTW89_CH_2G;
-			break;
-		}
-		break;
-	case RTW89_BAND_5G:
-		switch (center_chan) {
-		default:
-		case 36 ... 64:
-			subband = RTW89_CH_5G_BAND_1;
-			break;
-		case 100 ... 144:
-			subband = RTW89_CH_5G_BAND_3;
-			break;
-		case 149 ... 177:
-			subband = RTW89_CH_5G_BAND_4;
-			break;
-		}
-		break;
-	case RTW89_BAND_6G:
-		switch (center_chan) {
-		default:
-		case 1 ... 29:
-			subband = RTW89_CH_6G_BAND_IDX0;
-			break;
-		case 33 ... 61:
-			subband = RTW89_CH_6G_BAND_IDX1;
-			break;
-		case 65 ... 93:
-			subband = RTW89_CH_6G_BAND_IDX2;
-			break;
-		case 97 ... 125:
-			subband = RTW89_CH_6G_BAND_IDX3;
-			break;
-		case 129 ... 157:
-			subband = RTW89_CH_6G_BAND_IDX4;
-			break;
-		case 161 ... 189:
-			subband = RTW89_CH_6G_BAND_IDX5;
-			break;
-		case 193 ... 221:
-			subband = RTW89_CH_6G_BAND_IDX6;
-			break;
-		case 225 ... 253:
-			subband = RTW89_CH_6G_BAND_IDX7;
-			break;
-		}
-		break;
-	}
+	rtw89_chan_create(chan, center_chan, channel->hw_value, band, bandwidth);
+}
 
-	chan_param->center_chan = center_chan;
-	chan_param->center_freq = center_freq;
-	chan_param->primary_chan = channel->hw_value;
-	chan_param->bandwidth = bandwidth;
-	chan_param->pri_ch_idx = primary_chan_idx;
-	chan_param->band_type = band;
-	chan_param->subband_type = subband;
+void rtw89_core_set_chip_txpwr(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	const struct rtw89_chan *chan;
+	enum rtw89_sub_entity_idx sub_entity_idx;
+	enum rtw89_phy_idx phy_idx;
+	enum rtw89_entity_mode mode;
+	bool entity_active;
+
+	entity_active = rtw89_get_entity_state(rtwdev);
+	if (!entity_active)
+		return;
+
+	mode = rtw89_get_entity_mode(rtwdev);
+	if (WARN(mode != RTW89_ENTITY_MODE_SCC, "Invalid ent mode: %d\n", mode))
+		return;
+
+	sub_entity_idx = RTW89_SUB_ENTITY_0;
+	phy_idx = RTW89_PHY_0;
+	chan = rtw89_chan_get(rtwdev, sub_entity_idx);
+	if (chip->ops->set_txpwr)
+		chip->ops->set_txpwr(rtwdev, chan, phy_idx);
 }
 
 void rtw89_set_channel(struct rtw89_dev *rtwdev)
 {
-	struct ieee80211_hw *hw = rtwdev->hw;
 	const struct rtw89_chip_info *chip = rtwdev->chip;
-	struct rtw89_hal *hal = &rtwdev->hal;
-	struct rtw89_channel_params ch_param;
+	const struct cfg80211_chan_def *chandef;
+	enum rtw89_sub_entity_idx sub_entity_idx;
+	enum rtw89_mac_idx mac_idx;
+	enum rtw89_phy_idx phy_idx;
+	struct rtw89_chan chan;
 	struct rtw89_channel_help_params bak;
-	u8 center_chan, bandwidth;
+	enum rtw89_entity_mode mode;
 	bool band_changed;
+	bool entity_active;
 
-	rtw89_get_channel_params(&hw->conf.chandef, &ch_param);
-	if (WARN(ch_param.center_chan == 0, "Invalid channel\n"))
+	entity_active = rtw89_get_entity_state(rtwdev);
+
+	mode = rtw89_entity_recalc(rtwdev);
+	if (WARN(mode != RTW89_ENTITY_MODE_SCC, "Invalid ent mode: %d\n", mode))
 		return;
 
-	center_chan = ch_param.center_chan;
-	bandwidth = ch_param.bandwidth;
-	band_changed = hal->current_band_type != ch_param.band_type ||
-		       hal->current_channel == 0;
+	sub_entity_idx = RTW89_SUB_ENTITY_0;
+	mac_idx = RTW89_MAC_0;
+	phy_idx = RTW89_PHY_0;
+	chandef = rtw89_chandef_get(rtwdev, sub_entity_idx);
+	rtw89_get_channel_params(chandef, &chan);
+	if (WARN(chan.channel == 0, "Invalid channel\n"))
+		return;
 
-	hal->current_band_width = bandwidth;
-	hal->current_channel = center_chan;
-	hal->current_freq = ch_param.center_freq;
-	hal->prev_primary_channel = hal->current_primary_channel;
-	hal->prev_band_type = hal->current_band_type;
-	hal->current_primary_channel = ch_param.primary_chan;
-	hal->current_band_type = ch_param.band_type;
-	hal->current_subband = ch_param.subband_type;
+	band_changed = rtw89_assign_entity_chan(rtwdev, sub_entity_idx, &chan);
 
-	rtw89_chip_set_channel_prepare(rtwdev, &bak);
+	rtw89_chip_set_channel_prepare(rtwdev, &bak, &chan, mac_idx, phy_idx);
 
-	chip->ops->set_channel(rtwdev, &ch_param);
+	chip->ops->set_channel(rtwdev, &chan, mac_idx, phy_idx);
 
-	rtw89_chip_set_txpwr(rtwdev);
+	rtw89_core_set_chip_txpwr(rtwdev);
 
-	rtw89_chip_set_channel_done(rtwdev, &bak);
+	rtw89_chip_set_channel_done(rtwdev, &bak, &chan, mac_idx, phy_idx);
 
-	if (band_changed) {
-		rtw89_btc_ntfy_switch_band(rtwdev, RTW89_PHY_0, hal->current_band_type);
-		rtw89_chip_rfk_band_changed(rtwdev);
+	if (!entity_active || band_changed) {
+		rtw89_btc_ntfy_switch_band(rtwdev, phy_idx, chan.band_type);
+		rtw89_chip_rfk_band_changed(rtwdev, phy_idx);
 	}
+
+	rtw89_set_entity_state(rtwdev, true);
 }
 
 static enum rtw89_core_tx_type
@@ -529,9 +495,15 @@ static u16 rtw89_core_get_mgmt_rate(struct rtw89_dev *rtwdev,
 	struct sk_buff *skb = tx_req->skb;
 	struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb);
 	struct ieee80211_vif *vif = tx_info->control.vif;
-	struct rtw89_hal *hal = &rtwdev->hal;
-	u16 lowest_rate = hal->current_band_type == RTW89_BAND_2G ?
-			  RTW89_HW_RATE_CCK1 : RTW89_HW_RATE_OFDM6;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u16 lowest_rate;
+
+	if (tx_info->flags & IEEE80211_TX_CTL_NO_CCK_RATE || vif->p2p)
+		lowest_rate = RTW89_HW_RATE_OFDM6;
+	else if (chan->band_type == RTW89_BAND_2G)
+		lowest_rate = RTW89_HW_RATE_CCK1;
+	else
+		lowest_rate = RTW89_HW_RATE_OFDM6;
 
 	if (!vif || !vif->bss_conf.basic_rates || !tx_req->sta)
 		return lowest_rate;
@@ -546,6 +518,7 @@ rtw89_core_tx_update_mgmt_info(struct rtw89_dev *rtwdev,
 	struct ieee80211_vif *vif = tx_req->vif;
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	u8 qsel, ch_dma;
 
 	qsel = desc_info->hiq ? RTW89_TX_QSEL_B0_HI : RTW89_TX_QSEL_B0_MGMT;
@@ -564,9 +537,9 @@ rtw89_core_tx_update_mgmt_info(struct rtw89_dev *rtwdev,
 	desc_info->data_rate = rtw89_core_get_mgmt_rate(rtwdev, tx_req);
 
 	rtw89_debug(rtwdev, RTW89_DBG_TXRX,
-		    "tx mgmt frame with rate 0x%x on channel %d (bw %d)\n",
-		    desc_info->data_rate, rtwdev->hal.current_channel,
-		    rtwdev->hal.current_band_width);
+		    "tx mgmt frame with rate 0x%x on channel %d (band %d, bw %d)\n",
+		    desc_info->data_rate, chan->channel, chan->band_type,
+		    chan->band_width);
 }
 
 static void
@@ -591,15 +564,16 @@ static void rtw89_core_get_no_ul_ofdma_htc(struct rtw89_dev *rtwdev, __le32 *htc
 	};
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_hal *hal = &rtwdev->hal;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	u8 om_bandwidth;
 
 	if (!chip->dis_2g_40m_ul_ofdma ||
-	    hal->current_band_type != RTW89_BAND_2G ||
-	    hal->current_band_width != RTW89_CHANNEL_WIDTH_40)
+	    chan->band_type != RTW89_BAND_2G ||
+	    chan->band_width != RTW89_CHANNEL_WIDTH_40)
 		return;
 
-	om_bandwidth = hal->current_band_width < ARRAY_SIZE(rtw89_bandwidth_to_om) ?
-		       rtw89_bandwidth_to_om[hal->current_band_width] : 0;
+	om_bandwidth = chan->band_width < ARRAY_SIZE(rtw89_bandwidth_to_om) ?
+		       rtw89_bandwidth_to_om[chan->band_width] : 0;
 	*htc = le32_encode_bits(RTW89_HTC_VARIANT_HE, RTW89_HTC_MASK_VARIANT) |
 	       le32_encode_bits(RTW89_HTC_VARIANT_HE_CID_OM, RTW89_HTC_MASK_CTL_ID) |
 	       le32_encode_bits(hal->rx_nss - 1, RTW89_HTC_MASK_HTC_OM_RX_NSS) |
@@ -617,6 +591,7 @@ __rtw89_core_tx_check_he_qos_htc(struct rtw89_dev *rtwdev,
 				 enum btc_pkt_type pkt_type)
 {
 	struct ieee80211_sta *sta = tx_req->sta;
+	struct rtw89_sta *rtwsta = sta_to_rtwsta_safe(sta);
 	struct sk_buff *skb = tx_req->skb;
 	struct ieee80211_hdr *hdr = (void *)skb->data;
 	__le16 fc = hdr->frame_control;
@@ -634,6 +609,9 @@ __rtw89_core_tx_check_he_qos_htc(struct rtw89_dev *rtwdev,
 	if (skb_headroom(skb) < IEEE80211_HT_CTL_LEN)
 		return false;
 
+	if (rtwsta && rtwsta->ra_report.might_fallback_legacy)
+		return false;
+
 	return true;
 }
 
@@ -713,7 +691,7 @@ rtw89_core_tx_update_data_info(struct rtw89_dev *rtwdev,
 	struct ieee80211_vif *vif = tx_req->vif;
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	struct rtw89_phy_rate_pattern *rate_pattern = &rtwvif->rate_pattern;
-	struct rtw89_hal *hal = &rtwdev->hal;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
 	struct sk_buff *skb = tx_req->skb;
 	u8 tid, tid_indicate;
@@ -736,9 +714,11 @@ rtw89_core_tx_update_data_info(struct rtw89_dev *rtwdev,
 	if (IEEE80211_SKB_CB(skb)->control.hw_key)
 		rtw89_core_tx_update_sec_key(rtwdev, tx_req);
 
-	if (rate_pattern->enable)
+	if (vif->p2p)
+		desc_info->data_retry_lowest_rate = RTW89_HW_RATE_OFDM6;
+	else if (rate_pattern->enable)
 		desc_info->data_retry_lowest_rate = rate_pattern->rate;
-	else if (hal->current_band_type == RTW89_BAND_2G)
+	else if (chan->band_type == RTW89_BAND_2G)
 		desc_info->data_retry_lowest_rate = RTW89_HW_RATE_CCK1;
 	else
 		desc_info->data_retry_lowest_rate = RTW89_HW_RATE_OFDM6;
@@ -796,13 +776,16 @@ static void
 rtw89_core_tx_wake(struct rtw89_dev *rtwdev,
 		   struct rtw89_core_tx_request *tx_req)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
 	if (!RTW89_CHK_FW_FEATURE(TX_WAKE, &rtwdev->fw))
 		return;
 
 	if (!test_bit(RTW89_FLAG_LOW_POWER_MODE, rtwdev->flags))
 		return;
 
-	if (tx_req->tx_type != RTW89_CORE_TX_TYPE_MGMT)
+	if (chip->chip_id != RTL8852C &&
+	    tx_req->tx_type != RTW89_CORE_TX_TYPE_MGMT)
 		return;
 
 	rtw89_mac_notify_wake(rtwdev);
@@ -872,6 +855,7 @@ int rtw89_h2c_tx(struct rtw89_dev *rtwdev,
 		rtw89_debug(rtwdev, RTW89_DBG_FW,
 			    "ignore h2c due to power is off with firmware state=%d\n",
 			    test_bit(RTW89_FLAG_FW_RDY, rtwdev->flags));
+		dev_kfree_skb(skb);
 		return 0;
 	}
 
@@ -1021,7 +1005,8 @@ static __le32 rtw89_build_txwd_info0(struct rtw89_tx_desc_info *desc_info)
 
 static __le32 rtw89_build_txwd_info0_v1(struct rtw89_tx_desc_info *desc_info)
 {
-	u32 dword = FIELD_PREP(RTW89_TXWD_INFO0_DISDATAFB, desc_info->dis_data_fb);
+	u32 dword = FIELD_PREP(RTW89_TXWD_INFO0_DISDATAFB, desc_info->dis_data_fb) |
+		    FIELD_PREP(RTW89_TXWD_INFO0_MULTIPORT_ID, desc_info->port);
 
 	return cpu_to_le32(dword);
 }
@@ -1171,9 +1156,14 @@ static void rtw89_core_rx_process_phy_ppdu_iter(void *data,
 {
 	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
 	struct rtw89_rx_phy_ppdu *phy_ppdu = (struct rtw89_rx_phy_ppdu *)data;
+	struct rtw89_dev *rtwdev = rtwsta->rtwdev;
+	int i;
 
-	if (rtwsta->mac_id == phy_ppdu->mac_id && phy_ppdu->to_self)
+	if (rtwsta->mac_id == phy_ppdu->mac_id && phy_ppdu->to_self) {
 		ewma_rssi_add(&rtwsta->avg_rssi, phy_ppdu->rssi_avg);
+		for (i = 0; i < rtwdev->chip->rf_path_num; i++)
+			ewma_rssi_add(&rtwsta->rssi[i], phy_ppdu->rssi[i]);
+	}
 }
 
 #define VAR_LEN 0xff
@@ -1229,15 +1219,15 @@ static int rtw89_core_process_phy_status_ie(struct rtw89_dev *rtwdev, u8 *addr,
 
 static void rtw89_core_update_phy_ppdu(struct rtw89_rx_phy_ppdu *phy_ppdu)
 {
-	s8 *rssi = phy_ppdu->rssi;
+	u8 *rssi = phy_ppdu->rssi;
 	u8 *buf = phy_ppdu->buf;
 
 	phy_ppdu->ie = RTW89_GET_PHY_STS_IE_MAP(buf);
 	phy_ppdu->rssi_avg = RTW89_GET_PHY_STS_RSSI_AVG(buf);
-	rssi[RF_PATH_A] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_A(buf));
-	rssi[RF_PATH_B] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_B(buf));
-	rssi[RF_PATH_C] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_C(buf));
-	rssi[RF_PATH_D] = RTW89_RSSI_RAW_TO_DBM(RTW89_GET_PHY_STS_RSSI_D(buf));
+	rssi[RF_PATH_A] = RTW89_GET_PHY_STS_RSSI_A(buf);
+	rssi[RF_PATH_B] = RTW89_GET_PHY_STS_RSSI_B(buf);
+	rssi[RF_PATH_C] = RTW89_GET_PHY_STS_RSSI_C(buf);
+	rssi[RF_PATH_D] = RTW89_GET_PHY_STS_RSSI_D(buf);
 }
 
 static int rtw89_core_rx_process_phy_ppdu(struct rtw89_dev *rtwdev,
@@ -1448,8 +1438,11 @@ static void rtw89_core_rx_stats(struct rtw89_dev *rtwdev,
 static void rtw89_correct_cck_chan(struct rtw89_dev *rtwdev,
 				   struct ieee80211_rx_status *status)
 {
-	u16 chan = rtwdev->hal.prev_primary_channel;
-	u8 band = chan <= 14 ? NL80211_BAND_2GHZ : NL80211_BAND_5GHZ;
+	const struct rtw89_chan_rcd *rcd =
+		rtw89_chan_rcd_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u16 chan = rcd->prev_primary_channel;
+	u8 band = rcd->prev_band_type == RTW89_BAND_2G ?
+		  NL80211_BAND_2GHZ : NL80211_BAND_5GHZ;
 
 	if (status->band != NL80211_BAND_2GHZ &&
 	    status->encoding == RX_ENC_LEGACY &&
@@ -1661,19 +1654,20 @@ static void rtw89_core_update_rx_status(struct rtw89_dev *rtwdev,
 					struct rtw89_rx_desc_info *desc_info,
 					struct ieee80211_rx_status *rx_status)
 {
-	struct ieee80211_hw *hw = rtwdev->hw;
-	struct rtw89_hal *hal = &rtwdev->hal;
+	const struct cfg80211_chan_def *chandef =
+		rtw89_chandef_get(rtwdev, RTW89_SUB_ENTITY_0);
+	const struct rtw89_chan *cur = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	u16 data_rate;
 	u8 data_rate_mode;
 
 	/* currently using single PHY */
-	rx_status->freq = hw->conf.chandef.chan->center_freq;
-	rx_status->band = hw->conf.chandef.chan->band;
+	rx_status->freq = chandef->chan->center_freq;
+	rx_status->band = chandef->chan->band;
 
 	if (rtwdev->scanning &&
 	    RTW89_CHK_FW_FEATURE(SCAN_OFFLOAD, &rtwdev->fw)) {
-		u8 chan = hal->current_primary_channel;
-		u8 band = hal->current_band_type;
+		u8 chan = cur->primary_channel;
+		u8 band = cur->band_type;
 		enum nl80211_band nl_band;
 
 		nl_band = rtw89_hw_to_nl80211_band(band);
@@ -1727,7 +1721,8 @@ static enum rtw89_ps_mode rtw89_update_ps_mode(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 
-	if (rtw89_disable_ps_mode || !chip->ps_mode_supported)
+	if (rtw89_disable_ps_mode || !chip->ps_mode_supported ||
+	    RTW89_CHK_FW_FEATURE(NO_DEEP_PS, &rtwdev->fw))
 		return RTW89_PS_MODE_NONE;
 
 	if (chip->ps_mode_supported & BIT(RTW89_PS_MODE_PWR_GATED))
@@ -1810,7 +1805,7 @@ void rtw89_core_napi_init(struct rtw89_dev *rtwdev)
 {
 	init_dummy_netdev(&rtwdev->netdev);
 	netif_napi_add(&rtwdev->netdev, &rtwdev->napi,
-		       rtwdev->hci.ops->napi_poll, NAPI_POLL_WEIGHT);
+		       rtwdev->hci.ops->napi_poll);
 }
 EXPORT_SYMBOL(rtw89_core_napi_init);
 
@@ -1907,21 +1902,14 @@ static void rtw89_core_stop_tx_ba_session(struct rtw89_dev *rtwdev,
 		return;
 
 	spin_lock_bh(&rtwdev->ba_lock);
-	if (!list_empty(&rtwtxq->list)) {
-		list_del_init(&rtwtxq->list);
-		goto out;
-	}
-
-	set_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags);
+	if (!test_and_set_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags))
+		list_add_tail(&rtwtxq->list, &rtwdev->forbid_ba_list);
+	spin_unlock_bh(&rtwdev->ba_lock);
 
-	list_add_tail(&rtwtxq->list, &rtwdev->forbid_ba_list);
 	ieee80211_stop_tx_ba_session(sta, txq->tid);
 	cancel_delayed_work(&rtwdev->forbid_ba_work);
 	ieee80211_queue_delayed_work(rtwdev->hw, &rtwdev->forbid_ba_work,
 				     RTW89_FORBID_BA_TIMER);
-
-out:
-	spin_unlock_bh(&rtwdev->ba_lock);
 }
 
 static void rtw89_core_txq_check_agg(struct rtw89_dev *rtwdev,
@@ -1933,6 +1921,9 @@ static void rtw89_core_txq_check_agg(struct rtw89_dev *rtwdev,
 	struct ieee80211_sta *sta = txq->sta;
 	struct rtw89_sta *rtwsta = sta ? (struct rtw89_sta *)sta->drv_priv : NULL;
 
+	if (test_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags))
+		return;
+
 	if (unlikely(skb->protocol == cpu_to_be16(ETH_P_PAE))) {
 		rtw89_core_stop_tx_ba_session(rtwdev, rtwtxq);
 		return;
@@ -1941,9 +1932,6 @@ static void rtw89_core_txq_check_agg(struct rtw89_dev *rtwdev,
 	if (unlikely(!sta))
 		return;
 
-	if (test_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags))
-		return;
-
 	if (unlikely(test_bit(RTW89_TXQ_F_BLOCK_BA, &rtwtxq->flags)))
 		return;
 
@@ -2179,12 +2167,13 @@ static bool rtw89_traffic_stats_track(struct rtw89_dev *rtwdev)
 
 static void rtw89_vif_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 {
-	if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION)
+	if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION &&
+	    rtwvif->wifi_role != RTW89_WIFI_ROLE_P2P_CLIENT)
 		return;
 
 	if (rtwvif->stats.tx_tfc_lv == RTW89_TFC_IDLE &&
 	    rtwvif->stats.rx_tfc_lv == RTW89_TFC_IDLE)
-		rtw89_enter_lps(rtwdev, rtwvif->mac_id);
+		rtw89_enter_lps(rtwdev, rtwvif);
 }
 
 static void rtw89_enter_lps_track(struct rtw89_dev *rtwdev)
@@ -2237,6 +2226,7 @@ static void rtw89_track_work(struct work_struct *work)
 	rtw89_chip_rfk_track(rtwdev);
 	rtw89_phy_ra_update(rtwdev);
 	rtw89_phy_cfo_track(rtwdev);
+	rtw89_phy_tx_path_div_track(rtwdev);
 
 	if (rtwdev->lps_enabled && !rtwdev->btc.lps)
 		rtw89_enter_lps_track(rtwdev);
@@ -2266,45 +2256,69 @@ void rtw89_core_release_all_bits_map(unsigned long *addr, unsigned int nbits)
 	bitmap_zero(addr, nbits);
 }
 
-int rtw89_core_acquire_sta_ba_entry(struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx)
+int rtw89_core_acquire_sta_ba_entry(struct rtw89_dev *rtwdev,
+				    struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx)
 {
-	struct rtw89_ba_cam_entry *entry;
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_cam_info *cam_info = &rtwdev->cam_info;
+	struct rtw89_ba_cam_entry *entry = NULL, *tmp;
 	u8 idx;
+	int i;
 
-	idx = rtw89_core_acquire_bit_map(rtwsta->ba_cam_map, RTW89_BA_CAM_NUM);
-	if (idx == RTW89_BA_CAM_NUM) {
-		/* allocate a static BA CAM to tid=0, so replace the existing
+	lockdep_assert_held(&rtwdev->mutex);
+
+	idx = rtw89_core_acquire_bit_map(cam_info->ba_cam_map, chip->bacam_num);
+	if (idx == chip->bacam_num) {
+		/* allocate a static BA CAM to tid=0/5, so replace the existing
 		 * one if BA CAM is full. Hardware will process the original tid
 		 * automatically.
 		 */
-		if (tid != 0)
+		if (tid != 0 && tid != 5)
 			return -ENOSPC;
 
-		idx = 0;
+		for_each_set_bit(i, cam_info->ba_cam_map, chip->bacam_num) {
+			tmp = &cam_info->ba_cam_entry[i];
+			if (tmp->tid == 0 || tmp->tid == 5)
+				continue;
+
+			idx = i;
+			entry = tmp;
+			list_del(&entry->list);
+			break;
+		}
+
+		if (!entry)
+			return -ENOSPC;
+	} else {
+		entry = &cam_info->ba_cam_entry[idx];
 	}
 
-	entry = &rtwsta->ba_cam_entry[idx];
 	entry->tid = tid;
+	list_add_tail(&entry->list, &rtwsta->ba_cam_list);
+
 	*cam_idx = idx;
 
 	return 0;
 }
 
-int rtw89_core_release_sta_ba_entry(struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx)
+int rtw89_core_release_sta_ba_entry(struct rtw89_dev *rtwdev,
+				    struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx)
 {
-	struct rtw89_ba_cam_entry *entry;
-	int i;
+	struct rtw89_cam_info *cam_info = &rtwdev->cam_info;
+	struct rtw89_ba_cam_entry *entry = NULL, *tmp;
+	u8 idx;
 
-	for (i = 0; i < RTW89_BA_CAM_NUM; i++) {
-		if (!test_bit(i, rtwsta->ba_cam_map))
-			continue;
+	lockdep_assert_held(&rtwdev->mutex);
 
-		entry = &rtwsta->ba_cam_entry[i];
+	list_for_each_entry_safe(entry, tmp, &rtwsta->ba_cam_list, list) {
 		if (entry->tid != tid)
 			continue;
 
-		rtw89_core_release_bit_map(rtwsta->ba_cam_map, i);
-		*cam_idx = i;
+		idx = entry - cam_info->ba_cam_entry;
+		list_del(&entry->list);
+
+		rtw89_core_release_bit_map(cam_info->ba_cam_map, idx);
+		*cam_idx = idx;
 		return 0;
 	}
 
@@ -2320,9 +2334,19 @@ void rtw89_vif_type_mapping(struct ieee80211_vif *vif, bool assoc)
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 
 	switch (vif->type) {
+	case NL80211_IFTYPE_STATION:
+		if (vif->p2p)
+			rtwvif->wifi_role = RTW89_WIFI_ROLE_P2P_CLIENT;
+		else
+			rtwvif->wifi_role = RTW89_WIFI_ROLE_STATION;
+		break;
+	case NL80211_IFTYPE_AP:
+		if (vif->p2p)
+			rtwvif->wifi_role = RTW89_WIFI_ROLE_P2P_GO;
+		else
+			rtwvif->wifi_role = RTW89_WIFI_ROLE_AP;
+		break;
 	RTW89_TYPE_MAPPING(ADHOC);
-	RTW89_TYPE_MAPPING(STATION);
-	RTW89_TYPE_MAPPING(AP);
 	RTW89_TYPE_MAPPING(MONITOR);
 	RTW89_TYPE_MAPPING(MESH_POINT);
 	default:
@@ -2365,13 +2389,17 @@ int rtw89_core_sta_add(struct rtw89_dev *rtwdev,
 	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
 	int i;
 
+	rtwsta->rtwdev = rtwdev;
 	rtwsta->rtwvif = rtwvif;
 	rtwsta->prev_rssi = 0;
+	INIT_LIST_HEAD(&rtwsta->ba_cam_list);
 
 	for (i = 0; i < ARRAY_SIZE(sta->txq); i++)
 		rtw89_core_txq_init(rtwdev, sta->txq[i]);
 
 	ewma_rssi_init(&rtwsta->avg_rssi);
+	for (i = 0; i < rtwdev->chip->rf_path_num; i++)
+		ewma_rssi_init(&rtwsta->rssi[i]);
 
 	if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) {
 		/* for station mode, assign the mac_id from itself */
@@ -2541,6 +2569,60 @@ int rtw89_core_sta_remove(struct rtw89_dev *rtwdev,
 	return 0;
 }
 
+static void _rtw89_core_set_tid_config(struct rtw89_dev *rtwdev,
+				       struct ieee80211_sta *sta,
+				       struct cfg80211_tid_cfg *tid_conf)
+{
+	struct ieee80211_txq *txq;
+	struct rtw89_txq *rtwtxq;
+	u32 mask = tid_conf->mask;
+	u8 tids = tid_conf->tids;
+	int tids_nbit = BITS_PER_BYTE;
+	int i;
+
+	for (i = 0; i < tids_nbit; i++, tids >>= 1) {
+		if (!tids)
+			break;
+
+		if (!(tids & BIT(0)))
+			continue;
+
+		txq = sta->txq[i];
+		rtwtxq = (struct rtw89_txq *)txq->drv_priv;
+
+		if (mask & BIT(NL80211_TID_CONFIG_ATTR_AMPDU_CTRL)) {
+			if (tid_conf->ampdu == NL80211_TID_CONFIG_ENABLE) {
+				clear_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags);
+			} else {
+				if (test_bit(RTW89_TXQ_F_AMPDU, &rtwtxq->flags))
+					ieee80211_stop_tx_ba_session(sta, txq->tid);
+				spin_lock_bh(&rtwdev->ba_lock);
+				list_del_init(&rtwtxq->list);
+				set_bit(RTW89_TXQ_F_FORBID_BA, &rtwtxq->flags);
+				spin_unlock_bh(&rtwdev->ba_lock);
+			}
+		}
+
+		if (mask & BIT(NL80211_TID_CONFIG_ATTR_AMSDU_CTRL) && tids == 0xff) {
+			if (tid_conf->amsdu == NL80211_TID_CONFIG_ENABLE)
+				sta->max_amsdu_subframes = 0;
+			else
+				sta->max_amsdu_subframes = 1;
+		}
+	}
+}
+
+void rtw89_core_set_tid_config(struct rtw89_dev *rtwdev,
+			       struct ieee80211_sta *sta,
+			       struct cfg80211_tid_config *tid_config)
+{
+	int i;
+
+	for (i = 0; i < tid_config->n_tid_conf; i++)
+		_rtw89_core_set_tid_config(rtwdev, sta,
+					   &tid_config->tid_conf[i]);
+}
+
 static void rtw89_init_ht_cap(struct rtw89_dev *rtwdev,
 			      struct ieee80211_sta_ht_cap *ht_cap)
 {
@@ -2669,8 +2751,7 @@ static void rtw89_init_he_cap(struct rtw89_dev *rtwdev,
 		phy_cap_info = he_cap->he_cap_elem.phy_cap_info;
 
 		he_cap->has_he = true;
-		if (i == NL80211_IFTYPE_AP)
-			mac_cap_info[0] = IEEE80211_HE_MAC_CAP0_HTC_HE;
+		mac_cap_info[0] = IEEE80211_HE_MAC_CAP0_HTC_HE;
 		if (i == NL80211_IFTYPE_STATION)
 			mac_cap_info[1] = IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US;
 		mac_cap_info[2] = IEEE80211_HE_MAC_CAP2_ALL_ACK |
@@ -2706,6 +2787,8 @@ static void rtw89_init_he_cap(struct rtw89_dev *rtwdev,
 			phy_cap_info[3] |= IEEE80211_HE_PHY_CAP3_RX_PARTIAL_BW_SU_IN_20MHZ_MU;
 		phy_cap_info[4] = IEEE80211_HE_PHY_CAP4_SU_BEAMFORMEE |
 				  IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_UNDER_80MHZ_4;
+		if (chip->support_bw160)
+			phy_cap_info[4] |= IEEE80211_HE_PHY_CAP4_BEAMFORMEE_MAX_STS_ABOVE_80MHZ_4;
 		phy_cap_info[5] = no_ng16 ? 0 :
 				  IEEE80211_HE_PHY_CAP5_NG16_SU_FEEDBACK |
 				  IEEE80211_HE_PHY_CAP5_NG16_MU_FEEDBACK;
@@ -2866,7 +2949,9 @@ int rtw89_core_start(struct rtw89_dev *rtwdev)
 	/* efuse process */
 
 	/* pre-config BB/RF, BB reset/RFC reset */
-	rtw89_chip_disable_bb_rf(rtwdev);
+	ret = rtw89_chip_disable_bb_rf(rtwdev);
+	if (ret)
+		return ret;
 	ret = rtw89_chip_enable_bb_rf(rtwdev);
 	if (ret)
 		return ret;
@@ -2894,6 +2979,7 @@ int rtw89_core_start(struct rtw89_dev *rtwdev)
 
 	rtw89_btc_ntfy_radio_state(rtwdev, BTC_RFCTRL_WL_ON);
 	rtw89_fw_h2c_fw_log(rtwdev, rtwdev->fw.fw_log_enable);
+	rtw89_fw_h2c_init_ba_cam(rtwdev);
 
 	return 0;
 }
@@ -2987,6 +3073,7 @@ int rtw89_core_init(struct rtw89_dev *rtwdev)
 		return ret;
 	}
 	rtw89_ser_init(rtwdev);
+	rtw89_entity_init(rtwdev);
 
 	return 0;
 }
@@ -3007,7 +3094,7 @@ EXPORT_SYMBOL(rtw89_core_deinit);
 void rtw89_core_scan_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 			   const u8 *mac_addr, bool hw_scan)
 {
-	struct rtw89_hal *hal = &rtwdev->hal;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 
 	rtwdev->scanning = true;
 	rtw89_leave_lps(rtwdev);
@@ -3015,7 +3102,7 @@ void rtw89_core_scan_start(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 		rtw89_leave_ips(rtwdev);
 
 	ether_addr_copy(rtwvif->mac_addr, mac_addr);
-	rtw89_btc_ntfy_scan_start(rtwdev, RTW89_PHY_0, hal->current_band_type);
+	rtw89_btc_ntfy_scan_start(rtwdev, RTW89_PHY_0, chan->band_type);
 	rtw89_chip_rfk_scan(rtwdev, true);
 	rtw89_hci_recalc_int_mit(rtwdev);
 
@@ -3141,6 +3228,7 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	hw->vif_data_size = sizeof(struct rtw89_vif);
 	hw->sta_data_size = sizeof(struct rtw89_sta);
 	hw->txq_data_size = sizeof(struct rtw89_txq);
+	hw->chanctx_data_size = sizeof(struct rtw89_chanctx_cfg);
 
 	SET_IEEE80211_PERM_ADDR(hw, efuse->addr);
 
@@ -3148,6 +3236,7 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	hw->queues = IEEE80211_NUM_ACS;
 	hw->max_rx_aggregation_subframes = RTW89_MAX_RX_AGG_NUM;
 	hw->max_tx_aggregation_subframes = RTW89_MAX_TX_AGG_NUM;
+	hw->uapsd_max_sp_len = IEEE80211_WMM_IE_STA_QOSINFO_SP_ALL;
 
 	ieee80211_hw_set(hw, SIGNAL_DBM);
 	ieee80211_hw_set(hw, HAS_RATE_CONTROL);
@@ -3164,17 +3253,26 @@ static int rtw89_core_register_hw(struct rtw89_dev *rtwdev)
 	ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID);
 
 	hw->wiphy->interface_modes = BIT(NL80211_IFTYPE_STATION) |
-				     BIT(NL80211_IFTYPE_AP);
+				     BIT(NL80211_IFTYPE_AP) |
+				     BIT(NL80211_IFTYPE_P2P_CLIENT) |
+				     BIT(NL80211_IFTYPE_P2P_GO);
+
 	hw->wiphy->available_antennas_tx = BIT(rtwdev->chip->rf_path_num) - 1;
 	hw->wiphy->available_antennas_rx = BIT(rtwdev->chip->rf_path_num) - 1;
 
 	hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS |
-			    WIPHY_FLAG_TDLS_EXTERNAL_SETUP;
+			    WIPHY_FLAG_TDLS_EXTERNAL_SETUP |
+			    WIPHY_FLAG_AP_UAPSD;
 	hw->wiphy->features |= NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR;
 
 	hw->wiphy->max_scan_ssids = RTW89_SCANOFLD_MAX_SSID;
 	hw->wiphy->max_scan_ie_len = RTW89_SCANOFLD_MAX_IE_LEN;
 
+	hw->wiphy->tid_config_support.vif |= BIT(NL80211_TID_CONFIG_ATTR_AMPDU_CTRL);
+	hw->wiphy->tid_config_support.peer |= BIT(NL80211_TID_CONFIG_ATTR_AMPDU_CTRL);
+	hw->wiphy->tid_config_support.vif |= BIT(NL80211_TID_CONFIG_ATTR_AMSDU_CTRL);
+	hw->wiphy->tid_config_support.peer |= BIT(NL80211_TID_CONFIG_ATTR_AMSDU_CTRL);
+
 	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
 
 	ret = rtw89_core_set_supported_band(rtwdev);
@@ -3234,6 +3332,63 @@ void rtw89_core_unregister(struct rtw89_dev *rtwdev)
 }
 EXPORT_SYMBOL(rtw89_core_unregister);
 
+struct rtw89_dev *rtw89_alloc_ieee80211_hw(struct device *device,
+					   u32 bus_data_size,
+					   const struct rtw89_chip_info *chip)
+{
+	struct ieee80211_hw *hw;
+	struct rtw89_dev *rtwdev;
+	struct ieee80211_ops *ops;
+	u32 driver_data_size;
+	u32 early_feat_map = 0;
+	bool no_chanctx;
+
+	rtw89_early_fw_feature_recognize(device, chip, &early_feat_map);
+
+	ops = kmemdup(&rtw89_ops, sizeof(rtw89_ops), GFP_KERNEL);
+	if (!ops)
+		goto err;
+
+	no_chanctx = chip->support_chanctx_num == 0 ||
+		     !(early_feat_map & BIT(RTW89_FW_FEATURE_SCAN_OFFLOAD));
+
+	if (no_chanctx) {
+		ops->add_chanctx = NULL;
+		ops->remove_chanctx = NULL;
+		ops->change_chanctx = NULL;
+		ops->assign_vif_chanctx = NULL;
+		ops->unassign_vif_chanctx = NULL;
+	}
+
+	driver_data_size = sizeof(struct rtw89_dev) + bus_data_size;
+	hw = ieee80211_alloc_hw(driver_data_size, ops);
+	if (!hw)
+		goto err;
+
+	rtwdev = hw->priv;
+	rtwdev->hw = hw;
+	rtwdev->dev = device;
+	rtwdev->ops = ops;
+	rtwdev->chip = chip;
+
+	rtw89_debug(rtwdev, RTW89_DBG_FW, "probe driver %s chanctx\n",
+		    no_chanctx ? "without" : "with");
+
+	return rtwdev;
+
+err:
+	kfree(ops);
+	return NULL;
+}
+EXPORT_SYMBOL(rtw89_alloc_ieee80211_hw);
+
+void rtw89_free_ieee80211_hw(struct rtw89_dev *rtwdev)
+{
+	kfree(rtwdev->ops);
+	ieee80211_free_hw(rtwdev->hw);
+}
+EXPORT_SYMBOL(rtw89_free_ieee80211_hw);
+
 MODULE_AUTHOR("Realtek Corporation");
 MODULE_DESCRIPTION("Realtek 802.11ax wireless core module");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h
index 7a9d6f5d8a51..db041b32a8c2 100644
--- a/drivers/net/wireless/realtek/rtw89/core.h
+++ b/drivers/net/wireless/realtek/rtw89/core.h
@@ -34,6 +34,7 @@ extern const struct ieee80211_ops rtw89_ops;
 #define MAX_RSSI 110
 #define RSSI_FACTOR 1
 #define RTW89_RSSI_RAW_TO_DBM(rssi) ((s8)((rssi) >> RSSI_FACTOR) - MAX_RSSI)
+#define RTW89_TX_DIV_RSSI_RAW_TH (2 << RSSI_FACTOR)
 
 #define RTW89_HTC_MASK_VARIANT GENMASK(1, 0)
 #define RTW89_HTC_VARIANT_HE 3
@@ -522,7 +523,7 @@ struct rtw89_rx_phy_ppdu {
 	u8 *buf;
 	u32 len;
 	u8 rssi_avg;
-	s8 rssi[RF_PATH_MAX];
+	u8 rssi[RF_PATH_MAX];
 	u8 mac_id;
 	u8 chan_idx;
 	u8 ie;
@@ -542,6 +543,12 @@ enum rtw89_phy_idx {
 	RTW89_PHY_MAX
 };
 
+enum rtw89_sub_entity_idx {
+	RTW89_SUB_ENTITY_0 = 0,
+
+	NUM_OF_RTW89_SUB_ENTITY,
+};
+
 enum rtw89_rf_path {
 	RF_PATH_A = 0,
 	RF_PATH_B = 1,
@@ -624,14 +631,23 @@ enum rtw89_sc_offset {
 	RTW89_SC_40_LOWER	= 10,
 };
 
-struct rtw89_channel_params {
-	u8 center_chan;
-	u32 center_freq;
-	u8 primary_chan;
-	u8 bandwidth;
-	u8 pri_ch_idx;
-	u8 band_type;
-	u8 subband_type;
+struct rtw89_chan {
+	u8 channel;
+	u8 primary_channel;
+	enum rtw89_band band_type;
+	enum rtw89_bandwidth band_width;
+
+	/* The follow-up are derived from the above. We must ensure that it
+	 * is assigned correctly in rtw89_chan_create() if new one is added.
+	 */
+	u32 freq;
+	enum rtw89_subband subband_type;
+	enum rtw89_sc_offset pri_ch_idx;
+};
+
+struct rtw89_chan_rcd {
+	u8 prev_primary_channel;
+	enum rtw89_band prev_band_type;
 };
 
 struct rtw89_channel_help_params {
@@ -793,7 +809,7 @@ struct rtw89_mac_ax_gnt {
 	u8 gnt_bt;
 	u8 gnt_wl_sw_en;
 	u8 gnt_wl;
-};
+} __packed;
 
 #define RTW89_MAC_AX_COEX_GNT_NR 2
 struct rtw89_mac_ax_coex_gnt {
@@ -848,6 +864,7 @@ enum rtw89_btc_dcnt {
 	BTC_DCNT_SLOT_NONSYNC,
 	BTC_DCNT_BTCNT_FREEZE,
 	BTC_DCNT_WL_SLOT_DRIFT,
+	BTC_DCNT_BT_SLOT_DRIFT,
 	BTC_DCNT_WL_STA_LAST,
 	BTC_DCNT_NUM,
 };
@@ -920,12 +937,12 @@ struct rtw89_btc_wl_smap {
 	u32 roaming: 1;
 	u32 _4way: 1;
 	u32 rf_off: 1;
-	u32 lps: 1;
+	u32 lps: 2;
 	u32 ips: 1;
 	u32 init_ok: 1;
 	u32 traffic_dir : 2;
 	u32 rf_off_pre: 1;
-	u32 lps_pre: 1;
+	u32 lps_pre: 2;
 };
 
 enum rtw89_tfc_lv {
@@ -1108,6 +1125,27 @@ struct rtw89_btc_wl_active_role {
 	u16 rx_rate;
 };
 
+struct rtw89_btc_wl_active_role_v1 {
+	u8 connected: 1;
+	u8 pid: 3;
+	u8 phy: 1;
+	u8 noa: 1;
+	u8 band: 2;
+
+	u8 client_ps: 1;
+	u8 bw: 7;
+
+	u8 role;
+	u8 ch;
+
+	u16 tx_lvl;
+	u16 rx_lvl;
+	u16 tx_rate;
+	u16 rx_rate;
+
+	u32 noa_duration; /* ms */
+};
+
 struct rtw89_btc_wl_role_info_bpos {
 	u16 none: 1;
 	u16 station: 1;
@@ -1123,6 +1161,12 @@ struct rtw89_btc_wl_role_info_bpos {
 	u16 nan: 1;
 };
 
+struct rtw89_btc_wl_scc_ctrl {
+	u8 null_role1;
+	u8 null_role2;
+	u8 ebt_null; /* if tx null at EBT slot */
+};
+
 union rtw89_btc_wl_role_info_map {
 	u16 val;
 	struct rtw89_btc_wl_role_info_bpos role;
@@ -1135,6 +1179,21 @@ struct rtw89_btc_wl_role_info { /* struct size must be n*4 bytes */
 	struct rtw89_btc_wl_active_role active_role[RTW89_PORT_NUM];
 };
 
+struct rtw89_btc_wl_role_info_v1 { /* struct size must be n*4 bytes */
+	u8 connect_cnt;
+	u8 link_mode;
+	union rtw89_btc_wl_role_info_map role_map;
+	struct rtw89_btc_wl_active_role_v1 active_role_v1[RTW89_PORT_NUM];
+	u32 mrole_type; /* btc_wl_mrole_type */
+	u32 mrole_noa_duration; /* ms */
+
+	u32 dbcc_en: 1;
+	u32 dbcc_chg: 1;
+	u32 dbcc_2g_phy: 2; /* which phy operate in 2G, HW_PHY_0 or HW_PHY_1 */
+	u32 link_mode_chg: 1;
+	u32 rsvd: 27;
+};
+
 struct rtw89_btc_wl_ver_info {
 	u32 fw_coex; /* match with which coex_ver */
 	u32 fw;
@@ -1240,6 +1299,7 @@ struct rtw89_btc_wl_info {
 	struct rtw89_btc_wl_ver_info  ver_info;
 	struct rtw89_btc_wl_afh_info afh_info;
 	struct rtw89_btc_wl_role_info role_info;
+	struct rtw89_btc_wl_role_info_v1 role_info_v1;
 	struct rtw89_btc_wl_scan_info scan_info;
 	struct rtw89_btc_wl_dbcc_info dbcc_info;
 	struct rtw89_btc_rf_para rf_para;
@@ -1248,6 +1308,7 @@ struct rtw89_btc_wl_info {
 	u8 port_id[RTW89_WIFI_ROLE_MLME_MAX];
 	u8 rssi_level;
 
+	bool scbd_change;
 	u32 scbd;
 };
 
@@ -1333,7 +1394,8 @@ struct rtw89_btc_bt_info {
 	u32 pag: 1;
 	u32 run_patch_code: 1;
 	u32 hi_lna_rx: 1;
-	u32 rsvd: 22;
+	u32 scan_rx_low_pri: 1;
+	u32 rsvd: 21;
 };
 
 struct rtw89_btc_cx {
@@ -1346,32 +1408,43 @@ struct rtw89_btc_cx {
 };
 
 struct rtw89_btc_fbtc_tdma {
-	u8 type;
+	u8 type; /* chip_info::fcxtdma_ver */
 	u8 rxflctrl;
 	u8 txpause;
 	u8 wtgle_n;
 	u8 leak_n;
 	u8 ext_ctrl;
-	u8 rsvd0;
-	u8 rsvd1;
+	u8 rxflctrl_role;
+	u8 option_ctrl;
+} __packed;
+
+struct rtw89_btc_fbtc_tdma_v1 {
+	u8 fver; /* chip_info::fcxtdma_ver */
+	u8 rsvd;
+	__le16 rsvd1;
+	struct rtw89_btc_fbtc_tdma tdma;
 } __packed;
 
 #define CXMREG_MAX 30
 #define FCXMAX_STEP 255 /*STEP trace record cnt, Max:65535, default:255*/
-#define BTCRPT_VER 1
 #define BTC_CYCLE_SLOT_MAX 48 /* must be even number, non-zero */
 
-enum rtw89_btc_bt_rfk_counter {
+enum rtw89_btc_bt_sta_counter {
 	BTC_BCNT_RFK_REQ = 0,
 	BTC_BCNT_RFK_GO = 1,
 	BTC_BCNT_RFK_REJECT = 2,
 	BTC_BCNT_RFK_FAIL = 3,
 	BTC_BCNT_RFK_TIMEOUT = 4,
-	BTC_BCNT_RFK_MAX
+	BTC_BCNT_HI_TX = 5,
+	BTC_BCNT_HI_RX = 6,
+	BTC_BCNT_LO_TX = 7,
+	BTC_BCNT_LO_RX = 8,
+	BTC_BCNT_POLLUTED = 9,
+	BTC_BCNT_STA_MAX
 };
 
 struct rtw89_btc_fbtc_rpt_ctrl {
-	u16 fver;
+	u16 fver; /* chip_info::fcxbtcrpt_ver */
 	u16 rpt_cnt; /* tmr counters */
 	u32 wl_fw_coex_ver; /* match which driver's coex version */
 	u32 wl_fw_cx_offload;
@@ -1384,11 +1457,56 @@ struct rtw89_btc_fbtc_rpt_ctrl {
 	u32 mb_a2dp_empty_cnt; /* a2dp empty count */
 	u32 mb_a2dp_flct_cnt; /* a2dp empty flow control counter */
 	u32 mb_a2dp_full_cnt; /* a2dp empty full counter */
-	u32 bt_rfk_cnt[BTC_BCNT_RFK_MAX];
+	u32 bt_rfk_cnt[BTC_BCNT_HI_TX];
 	u32 c2h_cnt; /* fw send c2h counter  */
 	u32 h2c_cnt; /* fw recv h2c counter */
 } __packed;
 
+struct rtw89_btc_fbtc_rpt_ctrl_info {
+	__le32 cnt; /* fw report counter */
+	__le32 en; /* report map */
+	__le32 para; /* not used */
+
+	__le32 cnt_c2h; /* fw send c2h counter  */
+	__le32 cnt_h2c; /* fw recv h2c counter */
+	__le32 len_c2h; /* The total length of the last C2H  */
+
+	__le32 cnt_aoac_rf_on;  /* rf-on counter for aoac switch notify */
+	__le32 cnt_aoac_rf_off; /* rf-off counter for aoac switch notify */
+} __packed;
+
+struct rtw89_btc_fbtc_rpt_ctrl_wl_fw_info {
+	__le32 cx_ver; /* match which driver's coex version */
+	__le32 cx_offload;
+	__le32 fw_ver;
+} __packed;
+
+struct rtw89_btc_fbtc_rpt_ctrl_a2dp_empty {
+	__le32 cnt_empty; /* a2dp empty count */
+	__le32 cnt_flowctrl; /* a2dp empty flow control counter */
+	__le32 cnt_tx;
+	__le32 cnt_ack;
+	__le32 cnt_nack;
+} __packed;
+
+struct rtw89_btc_fbtc_rpt_ctrl_bt_mailbox {
+	__le32 cnt_send_ok; /* fw send mailbox ok counter */
+	__le32 cnt_send_fail; /* fw send mailbox fail counter */
+	__le32 cnt_recv; /* fw recv mailbox counter */
+	struct rtw89_btc_fbtc_rpt_ctrl_a2dp_empty a2dp;
+} __packed;
+
+struct rtw89_btc_fbtc_rpt_ctrl_v1 {
+	u8 fver;
+	u8 rsvd;
+	__le16 rsvd1;
+	struct rtw89_btc_fbtc_rpt_ctrl_info rpt_info;
+	struct rtw89_btc_fbtc_rpt_ctrl_wl_fw_info wl_fw_info;
+	struct rtw89_btc_fbtc_rpt_ctrl_bt_mailbox bt_mbx_info;
+	__le32 bt_cnt[BTC_BCNT_STA_MAX];
+	struct rtw89_mac_ax_gnt gnt_val[RTW89_PHY_MAX];
+} __packed;
+
 enum rtw89_fbtc_ext_ctrl_type {
 	CXECTL_OFF = 0x0, /* tdma off */
 	CXECTL_B2 = 0x1, /* allow B2 (beacon-early) */
@@ -1457,10 +1575,9 @@ enum { /* STEP TYPE */
 	CXSTEP_MAX,
 };
 
-#define FCXGPIODBG_VER 1
 #define BTC_DBG_MAX1  32
 struct rtw89_btc_fbtc_gpio_dbg {
-	u8 fver;
+	u8 fver; /* chip_info::fcxgpiodbg_ver */
 	u8 rsvd;
 	u16 rsvd2;
 	u32 en_map; /* which debug signal (see btc_wl_gpio_debug) is enable */
@@ -1468,9 +1585,8 @@ struct rtw89_btc_fbtc_gpio_dbg {
 	u8 gpio_map[BTC_DBG_MAX1]; /*the debug signals to GPIO-Position */
 } __packed;
 
-#define FCXMREG_VER 1
 struct rtw89_btc_fbtc_mreg_val {
-	u8 fver;
+	u8 fver; /* chip_info::fcxmreg_ver */
 	u8 reg_num;
 	__le16 rsvd;
 	__le32 mreg_val[CXMREG_MAX];
@@ -1492,16 +1608,14 @@ struct rtw89_btc_fbtc_slot {
 	__le16 cxtype;
 } __packed;
 
-#define FCXSLOTS_VER 1
 struct rtw89_btc_fbtc_slots {
-	u8 fver;
+	u8 fver; /* chip_info::fcxslots_ver */
 	u8 tbl_num;
 	__le16 rsvd;
 	__le32 update_map;
 	struct rtw89_btc_fbtc_slot slot[CXST_MAX];
 } __packed;
 
-#define FCXSTEP_VER 2
 struct rtw89_btc_fbtc_step {
 	u8 type;
 	u8 val;
@@ -1509,7 +1623,7 @@ struct rtw89_btc_fbtc_step {
 } __packed;
 
 struct rtw89_btc_fbtc_steps {
-	u8 fver;
+	u8 fver; /* chip_info::fcxstep_ver */
 	u8 rsvd;
 	__le16 cnt;
 	__le16 pos_old;
@@ -1517,9 +1631,16 @@ struct rtw89_btc_fbtc_steps {
 	struct rtw89_btc_fbtc_step step[FCXMAX_STEP];
 } __packed;
 
-#define FCXCYSTA_VER 2
-struct rtw89_btc_fbtc_cysta { /* statistics for cycles */
+struct rtw89_btc_fbtc_steps_v1 {
 	u8 fver;
+	u8 en;
+	__le16 rsvd;
+	__le32 cnt;
+	struct rtw89_btc_fbtc_step step[FCXMAX_STEP];
+} __packed;
+
+struct rtw89_btc_fbtc_cysta { /* statistics for cycles */
+	u8 fver; /* chip_info::fcxcysta_ver */
 	u8 rsvd;
 	__le16 cycles; /* total cycle number */
 	__le16 cycles_a2dp[CXT_FLCTRL_MAX];
@@ -1544,19 +1665,80 @@ struct rtw89_btc_fbtc_cysta { /* statistics for cycles */
 	__le16 tslot_cycle[BTC_CYCLE_SLOT_MAX];
 } __packed;
 
-#define FCXNULLSTA_VER 1
-struct rtw89_btc_fbtc_cynullsta { /* cycle null statistics */
+struct rtw89_btc_fbtc_fdd_try_info {
+	__le16 cycles[CXT_FLCTRL_MAX];
+	__le16 tavg[CXT_FLCTRL_MAX]; /* avg try BT-Slot-TDD/BT-slot-FDD time */
+	__le16 tmax[CXT_FLCTRL_MAX]; /* max try BT-Slot-TDD/BT-slot-FDD time */
+} __packed;
+
+struct rtw89_btc_fbtc_cycle_time_info {
+	__le16 tavg[CXT_MAX]; /* avg wl/bt cycle time */
+	__le16 tmax[CXT_MAX]; /* max wl/bt cycle time */
+	__le16 tmaxdiff[CXT_MAX]; /* max wl-wl bt-bt cycle diff time */
+} __packed;
+
+struct rtw89_btc_fbtc_a2dp_trx_stat {
+	u8 empty_cnt;
+	u8 retry_cnt;
+	u8 tx_rate;
+	u8 tx_cnt;
+	u8 ack_cnt;
+	u8 nack_cnt;
+	u8 rsvd1;
+	u8 rsvd2;
+} __packed;
+
+struct rtw89_btc_fbtc_cycle_a2dp_empty_info {
+	__le16 cnt; /* a2dp empty cnt */
+	__le16 cnt_timeout; /* a2dp empty timeout cnt*/
+	__le16 tavg; /* avg a2dp empty time */
+	__le16 tmax; /* max a2dp empty time */
+} __packed;
+
+struct rtw89_btc_fbtc_cycle_leak_info {
+	__le32 cnt_rximr; /* the rximr occur at leak slot  */
+	__le16 tavg; /* avg leak-slot time */
+	__le16 tmax; /* max leak-slot time */
+} __packed;
+
+struct rtw89_btc_fbtc_cysta_v1 { /* statistics for cycles */
 	u8 fver;
 	u8 rsvd;
+	__le16 cycles; /* total cycle number */
+	__le16 slot_step_time[BTC_CYCLE_SLOT_MAX];
+	struct rtw89_btc_fbtc_cycle_time_info cycle_time;
+	struct rtw89_btc_fbtc_fdd_try_info fdd_try;
+	struct rtw89_btc_fbtc_cycle_a2dp_empty_info a2dp_ept;
+	struct rtw89_btc_fbtc_a2dp_trx_stat a2dp_trx[BTC_CYCLE_SLOT_MAX];
+	struct rtw89_btc_fbtc_cycle_leak_info leak_slot;
+	__le32 slot_cnt[CXST_MAX]; /* slot count */
+	__le32 bcn_cnt[CXBCN_MAX];
+	__le32 collision_cnt; /* counter for event/timer occur at the same time */
+	__le32 skip_cnt;
+	__le32 except_cnt;
+	__le32 except_map;
+} __packed;
+
+struct rtw89_btc_fbtc_cynullsta { /* cycle null statistics */
+	u8 fver; /* chip_info::fcxnullsta_ver */
+	u8 rsvd;
 	__le16 rsvd2;
 	__le32 max_t[2]; /* max_t for 0:null0/1:null1 */
 	__le32 avg_t[2]; /* avg_t for 0:null0/1:null1 */
 	__le32 result[2][4]; /* 0:fail, 1:ok, 2:on_time, 3:retry */
 } __packed;
 
-#define FCX_BTVER_VER 1
+struct rtw89_btc_fbtc_cynullsta_v1 { /* cycle null statistics */
+	u8 fver; /* chip_info::fcxnullsta_ver */
+	u8 rsvd;
+	__le16 rsvd2;
+	__le32 max_t[2]; /* max_t for 0:null0/1:null1 */
+	__le32 avg_t[2]; /* avg_t for 0:null0/1:null1 */
+	__le32 result[2][5]; /* 0:fail, 1:ok, 2:on_time, 3:retry, 4:tx */
+} __packed;
+
 struct rtw89_btc_fbtc_btver {
-	u8 fver;
+	u8 fver; /* chip_info::fcxbtver_ver */
 	u8 rsvd;
 	__le16 rsvd2;
 	__le32 coex_ver; /*bit[15:8]->shared, bit[7:0]->non-shared */
@@ -1564,17 +1746,15 @@ struct rtw89_btc_fbtc_btver {
 	__le32 feature;
 } __packed;
 
-#define FCX_BTSCAN_VER 1
 struct rtw89_btc_fbtc_btscan {
-	u8 fver;
+	u8 fver; /* chip_info::fcxbtscan_ver */
 	u8 rsvd;
 	__le16 rsvd2;
 	u8 scan[6];
 } __packed;
 
-#define FCX_BTAFH_VER 1
 struct rtw89_btc_fbtc_btafh {
-	u8 fver;
+	u8 fver; /* chip_info::fcxbtafh_ver */
 	u8 rsvd;
 	__le16 rsvd2;
 	u8 afh_l[4]; /*bit0:2402, bit1: 2403.... bit31:2433 */
@@ -1582,9 +1762,8 @@ struct rtw89_btc_fbtc_btafh {
 	u8 afh_h[4]; /*bit0:2466, bit1:2467......bit14:2480 */
 } __packed;
 
-#define FCX_BTDEVINFO_VER 1
 struct rtw89_btc_fbtc_btdevinfo {
-	u8 fver;
+	u8 fver; /* chip_info::fcxbtdevinfo_ver */
 	u8 rsvd;
 	__le16 vendor_id;
 	__le32 dev_name; /* only 24 bits valid */
@@ -1609,6 +1788,7 @@ struct rtw89_btc_dm {
 	struct rtw89_btc_rf_trx_para rf_trx_para;
 	struct rtw89_btc_wl_tx_limit_para wl_tx_limit;
 	struct rtw89_btc_dm_step dm_step;
+	struct rtw89_btc_wl_scc_ctrl wl_scc;
 	union rtw89_btc_dm_error_map error;
 	u32 cnt_dm[BTC_DCNT_NUM];
 	u32 cnt_notify[BTC_NCNT_NUM];
@@ -1628,7 +1808,9 @@ struct rtw89_btc_dm {
 	u32 wl_btg_rx: 1;
 	u32 trx_para_level: 8;
 	u32 wl_stb_chg: 1;
-	u32 rsvd: 3;
+	u32 pta_owner: 1;
+	u32 tdma_instant_excute: 1;
+	u32 rsvd: 1;
 
 	u16 slot_dur[CXST_MAX];
 
@@ -1650,8 +1832,6 @@ struct rtw89_btc_dbg {
 	u32 rb_val;
 };
 
-#define FCXTDMA_VER 1
-
 enum rtw89_btc_btf_fw_event {
 	BTF_EVNT_RPT = 0,
 	BTF_EVNT_BT_INFO = 1,
@@ -1704,12 +1884,18 @@ struct rtw89_btc_rpt_cmn_info {
 
 struct rtw89_btc_report_ctrl_state {
 	struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */
-	struct rtw89_btc_fbtc_rpt_ctrl finfo; /* info from fw */
+	union {
+		struct rtw89_btc_fbtc_rpt_ctrl finfo; /* info from fw for 52A*/
+		struct rtw89_btc_fbtc_rpt_ctrl_v1 finfo_v1; /* info from fw for 52C*/
+	};
 };
 
 struct rtw89_btc_rpt_fbtc_tdma {
 	struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */
-	struct rtw89_btc_fbtc_tdma finfo; /* info from fw */
+	union {
+		struct rtw89_btc_fbtc_tdma finfo; /* info from fw */
+		struct rtw89_btc_fbtc_tdma_v1 finfo_v1; /* info from fw for 52C*/
+	};
 };
 
 struct rtw89_btc_rpt_fbtc_slots {
@@ -1719,17 +1905,26 @@ struct rtw89_btc_rpt_fbtc_slots {
 
 struct rtw89_btc_rpt_fbtc_cysta {
 	struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */
-	struct rtw89_btc_fbtc_cysta finfo; /* info from fw */
+	union {
+		struct rtw89_btc_fbtc_cysta finfo; /* info from fw for 52A*/
+		struct rtw89_btc_fbtc_cysta_v1 finfo_v1; /* info from fw for 52C*/
+	};
 };
 
 struct rtw89_btc_rpt_fbtc_step {
 	struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */
-	struct rtw89_btc_fbtc_steps finfo; /* info from fw */
+	union {
+		struct rtw89_btc_fbtc_steps finfo; /* info from fw */
+		struct rtw89_btc_fbtc_steps_v1 finfo_v1; /* info from fw */
+	};
 };
 
 struct rtw89_btc_rpt_fbtc_nullsta {
 	struct rtw89_btc_rpt_cmn_info cinfo; /* common info, by driver */
-	struct rtw89_btc_fbtc_cynullsta finfo; /* info from fw */
+	union {
+		struct rtw89_btc_fbtc_cynullsta finfo; /* info from fw */
+		struct rtw89_btc_fbtc_cynullsta_v1 finfo_v1; /* info from fw */
+	};
 };
 
 struct rtw89_btc_rpt_fbtc_mreg {
@@ -1887,7 +2082,9 @@ struct rtw89_ra_info {
 	u8 ra_csi_rate_en:1;
 	u8 fixed_csi_rate_en:1;
 	u8 cr_tbl_sel:1;
-	u8 rsvd2:5;
+	u8 fix_giltf_en:1;
+	u8 fix_giltf:3;
+	u8 rsvd2:1;
 	u8 csi_mcs_ss_idx;
 	u8 csi_mode:2;
 	u8 csi_gi_ltf:3;
@@ -1911,19 +2108,20 @@ struct rtw89_ra_report {
 	struct rate_info txrate;
 	u32 bit_rate;
 	u16 hw_rate;
+	bool might_fallback_legacy;
 };
 
 DECLARE_EWMA(rssi, 10, 16);
 
-#define RTW89_BA_CAM_NUM 2
-
 struct rtw89_ba_cam_entry {
+	struct list_head list;
 	u8 tid;
 };
 
 #define RTW89_MAX_ADDR_CAM_NUM		128
 #define RTW89_MAX_BSSID_CAM_NUM		20
 #define RTW89_MAX_SEC_CAM_NUM		128
+#define RTW89_MAX_BA_CAM_NUM		8
 #define RTW89_SEC_CAM_IN_ADDR_CAM	7
 
 struct rtw89_addr_cam_entry {
@@ -1967,18 +2165,21 @@ struct rtw89_sec_cam_entry {
 struct rtw89_sta {
 	u8 mac_id;
 	bool disassoc;
+	struct rtw89_dev *rtwdev;
 	struct rtw89_vif *rtwvif;
 	struct rtw89_ra_info ra;
 	struct rtw89_ra_report ra_report;
 	int max_agg_wait;
 	u8 prev_rssi;
 	struct ewma_rssi avg_rssi;
+	struct ewma_rssi rssi[RF_PATH_MAX];
 	struct rtw89_ampdu_params ampdu_params[IEEE80211_NUM_TIDS];
 	struct ieee80211_rx_status rx_status;
 	u16 rx_hw_rate;
 	__le32 htc_template;
 	struct rtw89_addr_cam_entry addr_cam; /* AP mode or TDLS peer only */
 	struct rtw89_bssid_cam_entry bssid_cam; /* TDLS peer only */
+	struct list_head ba_cam_list;
 
 	bool use_cfg_mask;
 	struct cfg80211_bitrate_mask mask;
@@ -1987,9 +2188,6 @@ struct rtw89_sta {
 	u32 ampdu_max_time:4;
 	bool cctl_tx_retry_limit;
 	u32 data_tx_cnt_lmt:6;
-
-	DECLARE_BITMAP(ba_cam_map, RTW89_BA_CAM_NUM);
-	struct rtw89_ba_cam_entry ba_cam_entry[RTW89_BA_CAM_NUM];
 };
 
 struct rtw89_efuse {
@@ -2007,6 +2205,8 @@ struct rtw89_phy_rate_pattern {
 	bool enable;
 };
 
+#define RTW89_P2P_MAX_NOA_NUM 2
+
 struct rtw89_vif {
 	struct list_head list;
 	struct rtw89_dev *rtwdev;
@@ -2022,6 +2222,7 @@ struct rtw89_vif {
 	u8 wmm;
 	u8 bcn_hit_cond;
 	u8 hit_rule;
+	u8 last_noa_nr;
 	bool trigger;
 	bool lsig_txop;
 	u8 tgt_ind;
@@ -2091,7 +2292,7 @@ struct rtw89_hci_info {
 
 struct rtw89_chip_ops {
 	int (*enable_bb_rf)(struct rtw89_dev *rtwdev);
-	void (*disable_bb_rf)(struct rtw89_dev *rtwdev);
+	int (*disable_bb_rf)(struct rtw89_dev *rtwdev);
 	void (*bb_reset)(struct rtw89_dev *rtwdev,
 			 enum rtw89_phy_idx phy_idx);
 	void (*bb_sethw)(struct rtw89_dev *rtwdev);
@@ -2100,20 +2301,29 @@ struct rtw89_chip_ops {
 	bool (*write_rf)(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_path,
 			 u32 addr, u32 mask, u32 data);
 	void (*set_channel)(struct rtw89_dev *rtwdev,
-			    struct rtw89_channel_params *param);
+			    const struct rtw89_chan *chan,
+			    enum rtw89_mac_idx mac_idx,
+			    enum rtw89_phy_idx phy_idx);
 	void (*set_channel_help)(struct rtw89_dev *rtwdev, bool enter,
-				 struct rtw89_channel_help_params *p);
+				 struct rtw89_channel_help_params *p,
+				 const struct rtw89_chan *chan,
+				 enum rtw89_mac_idx mac_idx,
+				 enum rtw89_phy_idx phy_idx);
 	int (*read_efuse)(struct rtw89_dev *rtwdev, u8 *log_map);
 	int (*read_phycap)(struct rtw89_dev *rtwdev, u8 *phycap_map);
 	void (*fem_setup)(struct rtw89_dev *rtwdev);
 	void (*rfk_init)(struct rtw89_dev *rtwdev);
 	void (*rfk_channel)(struct rtw89_dev *rtwdev);
-	void (*rfk_band_changed)(struct rtw89_dev *rtwdev);
+	void (*rfk_band_changed)(struct rtw89_dev *rtwdev,
+				 enum rtw89_phy_idx phy_idx);
 	void (*rfk_scan)(struct rtw89_dev *rtwdev, bool start);
 	void (*rfk_track)(struct rtw89_dev *rtwdev);
 	void (*power_trim)(struct rtw89_dev *rtwdev);
-	void (*set_txpwr)(struct rtw89_dev *rtwdev);
-	void (*set_txpwr_ctrl)(struct rtw89_dev *rtwdev);
+	void (*set_txpwr)(struct rtw89_dev *rtwdev,
+			  const struct rtw89_chan *chan,
+			  enum rtw89_phy_idx phy_idx);
+	void (*set_txpwr_ctrl)(struct rtw89_dev *rtwdev,
+			       enum rtw89_phy_idx phy_idx);
 	int (*init_txpwr_unit)(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx);
 	u8 (*get_thermal)(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_path);
 	void (*ctrl_btg)(struct rtw89_dev *rtwdev, bool btg);
@@ -2150,6 +2360,8 @@ struct rtw89_chip_ops {
 	void (*btc_bt_aci_imp)(struct rtw89_dev *rtwdev);
 	void (*btc_update_bt_cnt)(struct rtw89_dev *rtwdev);
 	void (*btc_wl_s1_standby)(struct rtw89_dev *rtwdev, bool state);
+	void (*btc_set_policy)(struct rtw89_dev *rtwdev, u16 policy_type);
+	void (*btc_set_wl_rx_gain)(struct rtw89_dev *rtwdev, u32 level);
 };
 
 enum rtw89_dma_ch {
@@ -2351,6 +2563,7 @@ struct rtw89_imr_info {
 	u32 cpu_disp_imr_set;
 	u32 other_disp_imr_clr;
 	u32 other_disp_imr_set;
+	u32 bbrpt_com_err_imr_reg;
 	u32 bbrpt_chinfo_err_imr_reg;
 	u32 bbrpt_err_imr_set;
 	u32 bbrpt_dfs_err_imr_reg;
@@ -2373,17 +2586,40 @@ struct rtw89_imr_info {
 	u32 tmac_imr_set;
 };
 
+struct rtw89_rrsr_cfgs {
+	struct rtw89_reg3_def ref_rate;
+	struct rtw89_reg3_def rsc;
+};
+
+struct rtw89_dig_regs {
+	u32 seg0_pd_reg;
+	u32 pd_lower_bound_mask;
+	u32 pd_spatial_reuse_en;
+	struct rtw89_reg_def p0_lna_init;
+	struct rtw89_reg_def p1_lna_init;
+	struct rtw89_reg_def p0_tia_init;
+	struct rtw89_reg_def p1_tia_init;
+	struct rtw89_reg_def p0_rxb_init;
+	struct rtw89_reg_def p1_rxb_init;
+	struct rtw89_reg_def p0_p20_pagcugc_en;
+	struct rtw89_reg_def p0_s20_pagcugc_en;
+	struct rtw89_reg_def p1_p20_pagcugc_en;
+	struct rtw89_reg_def p1_s20_pagcugc_en;
+};
+
 struct rtw89_chip_info {
 	enum rtw89_core_chip_id chip_id;
 	const struct rtw89_chip_ops *ops;
 	const char *fw_name;
 	u32 fifo_size;
+	u32 dle_scc_rsvd_size;
 	u16 max_amsdu_limit;
 	bool dis_2g_40m_ul_ofdma;
 	u32 rsvd_ple_ofst;
 	const struct rtw89_hfc_param_ini *hfc_param_ini;
 	const struct rtw89_dle_mem *dle_mem;
 	u32 rf_base_addr[2];
+	u8 support_chanctx_num;
 	u8 support_bands;
 	bool support_bw160;
 	bool hw_sec_hdr;
@@ -2393,6 +2629,9 @@ struct rtw89_chip_info {
 	u8 acam_num;
 	u8 bcam_num;
 	u8 scam_num;
+	u8 bacam_num;
+	u8 bacam_dynamic_num;
+	bool bacam_v1;
 
 	u8 sec_ctrl_efuse_size;
 	u32 physical_efuse_size;
@@ -2411,6 +2650,7 @@ struct rtw89_chip_info {
 	const struct rtw89_phy_table *nctl_table;
 	const struct rtw89_txpwr_table *byr_table;
 	const struct rtw89_phy_dig_gain_table *dig_table;
+	const struct rtw89_dig_regs *dig_regs;
 	const struct rtw89_phy_tssi_dbw_table *tssi_dbw_table;
 	const s8 (*txpwr_lmt_2g)[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
 				[RTW89_RS_LMT_NUM][RTW89_BF_NUM]
@@ -2436,6 +2676,20 @@ struct rtw89_chip_info {
 	u8 btcx_desired;
 	u8 scbd;
 	u8 mailbox;
+	u16 btc_fwinfo_buf;
+
+	u8 fcxbtcrpt_ver;
+	u8 fcxtdma_ver;
+	u8 fcxslots_ver;
+	u8 fcxcysta_ver;
+	u8 fcxstep_ver;
+	u8 fcxnullsta_ver;
+	u8 fcxmreg_ver;
+	u8 fcxgpiodbg_ver;
+	u8 fcxbtver_ver;
+	u8 fcxbtscan_ver;
+	u8 fcxbtafh_ver;
+	u8 fcxbtdevinfo_ver;
 
 	u8 afh_guard_ch;
 	const u8 *wl_rssi_thres;
@@ -2463,6 +2717,8 @@ struct rtw89_chip_info {
 	const struct rtw89_reg_def *dcfo_comp;
 	u8 dcfo_comp_sft;
 	const struct rtw89_imr_info *imr_info;
+	const struct rtw89_rrsr_cfgs *rrsr_cfgs;
+	u32 dma_ch_mask;
 };
 
 union rtw89_bus_info {
@@ -2514,6 +2770,8 @@ enum rtw89_fw_feature {
 	RTW89_FW_FEATURE_SCAN_OFFLOAD,
 	RTW89_FW_FEATURE_TX_WAKE,
 	RTW89_FW_FEATURE_CRASH_TRIGGER,
+	RTW89_FW_FEATURE_PACKET_DROP,
+	RTW89_FW_FEATURE_NO_DEEP_PS,
 };
 
 struct rtw89_fw_suit {
@@ -2536,6 +2794,18 @@ struct rtw89_fw_suit {
 #define RTW89_FW_SUIT_VER_CODE(s)	\
 	RTW89_FW_VER_CODE((s)->major_ver, (s)->minor_ver, (s)->sub_ver, (s)->sub_idex)
 
+#define RTW89_MFW_HDR_VER_CODE(mfw_hdr)		\
+	RTW89_FW_VER_CODE((mfw_hdr)->ver.major,	\
+			  (mfw_hdr)->ver.minor,	\
+			  (mfw_hdr)->ver.sub,	\
+			  (mfw_hdr)->ver.idx)
+
+#define RTW89_FW_HDR_VER_CODE(fw_hdr)				\
+	RTW89_FW_VER_CODE(GET_FW_HDR_MAJOR_VERSION(fw_hdr),	\
+			  GET_FW_HDR_MINOR_VERSION(fw_hdr),	\
+			  GET_FW_HDR_SUBVERSION(fw_hdr),	\
+			  GET_FW_HDR_SUBINDEX(fw_hdr))
+
 struct rtw89_fw_info {
 	const struct firmware *firmware;
 	struct rtw89_dev *rtwdev;
@@ -2558,6 +2828,8 @@ struct rtw89_cam_info {
 	DECLARE_BITMAP(addr_cam_map, RTW89_MAX_ADDR_CAM_NUM);
 	DECLARE_BITMAP(bssid_cam_map, RTW89_MAX_BSSID_CAM_NUM);
 	DECLARE_BITMAP(sec_cam_map, RTW89_MAX_SEC_CAM_NUM);
+	DECLARE_BITMAP(ba_cam_map, RTW89_MAX_BA_CAM_NUM);
+	struct rtw89_ba_cam_entry ba_cam_entry[RTW89_MAX_BA_CAM_NUM];
 };
 
 enum rtw89_sar_sources {
@@ -2599,24 +2871,34 @@ struct rtw89_sar_info {
 	};
 };
 
+struct rtw89_chanctx_cfg {
+	enum rtw89_sub_entity_idx idx;
+};
+
+enum rtw89_entity_mode {
+	RTW89_ENTITY_MODE_SCC,
+};
+
 struct rtw89_hal {
 	u32 rx_fltr;
 	u8 cv;
-	u8 current_channel;
-	u32 current_freq;
-	u8 prev_primary_channel;
-	u8 current_primary_channel;
-	enum rtw89_subband current_subband;
-	u8 current_band_width;
-	u8 prev_band_type;
-	u8 current_band_type;
 	u32 sw_amsdu_max_size;
 	u32 antenna_tx;
 	u32 antenna_rx;
 	u8 tx_nss;
 	u8 rx_nss;
+	bool tx_path_diversity;
 	bool support_cckpd;
 	bool support_igi;
+
+	DECLARE_BITMAP(entity_map, NUM_OF_RTW89_SUB_ENTITY);
+	struct cfg80211_chan_def chandef[NUM_OF_RTW89_SUB_ENTITY];
+
+	bool entity_active;
+	enum rtw89_entity_mode entity_mode;
+
+	struct rtw89_chan chan[NUM_OF_RTW89_SUB_ENTITY];
+	struct rtw89_chan_rcd chan_rcd[NUM_OF_RTW89_SUB_ENTITY];
 };
 
 #define RTW89_MAX_MAC_ID_NUM 128
@@ -2632,11 +2914,37 @@ enum rtw89_flags {
 	RTW89_FLAG_LEISURE_PS,
 	RTW89_FLAG_LOW_POWER_MODE,
 	RTW89_FLAG_INACTIVE_PS,
-	RTW89_FLAG_RESTART_TRIGGER,
+	RTW89_FLAG_CRASH_SIMULATING,
 
 	NUM_OF_RTW89_FLAGS,
 };
 
+enum rtw89_pkt_drop_sel {
+	RTW89_PKT_DROP_SEL_MACID_BE_ONCE,
+	RTW89_PKT_DROP_SEL_MACID_BK_ONCE,
+	RTW89_PKT_DROP_SEL_MACID_VI_ONCE,
+	RTW89_PKT_DROP_SEL_MACID_VO_ONCE,
+	RTW89_PKT_DROP_SEL_MACID_ALL,
+	RTW89_PKT_DROP_SEL_MG0_ONCE,
+	RTW89_PKT_DROP_SEL_HIQ_ONCE,
+	RTW89_PKT_DROP_SEL_HIQ_PORT,
+	RTW89_PKT_DROP_SEL_HIQ_MBSSID,
+	RTW89_PKT_DROP_SEL_BAND,
+	RTW89_PKT_DROP_SEL_BAND_ONCE,
+	RTW89_PKT_DROP_SEL_REL_MACID,
+	RTW89_PKT_DROP_SEL_REL_HIQ_PORT,
+	RTW89_PKT_DROP_SEL_REL_HIQ_MBSSID,
+};
+
+struct rtw89_pkt_drop_params {
+	enum rtw89_pkt_drop_sel sel;
+	enum rtw89_mac_idx mac_band;
+	u8 macid;
+	u8 port;
+	u8 mbssid;
+	bool tf_trs;
+};
+
 struct rtw89_pkt_stat {
 	u16 beacon_nr;
 	u32 rx_rate_cnt[RTW89_HW_RATE_NR];
@@ -3073,6 +3381,7 @@ struct rtw89_hw_scan_info {
 	u8 op_chan;
 	u8 op_bw;
 	u8 op_band;
+	u32 last_chan_idx;
 };
 
 enum rtw89_phy_bb_gain_band {
@@ -3119,6 +3428,7 @@ struct rtw89_phy_efuse_gain {
 struct rtw89_dev {
 	struct ieee80211_hw *hw;
 	struct device *dev;
+	const struct ieee80211_ops *ops;
 
 	bool dbcc_en;
 	struct rtw89_hw_scan_info scan_info;
@@ -3498,6 +3808,16 @@ static inline struct ieee80211_vif *rtwvif_to_vif(struct rtw89_vif *rtwvif)
 	return container_of(p, struct ieee80211_vif, drv_priv);
 }
 
+static inline struct ieee80211_vif *rtwvif_to_vif_safe(struct rtw89_vif *rtwvif)
+{
+	return rtwvif ? rtwvif_to_vif(rtwvif) : NULL;
+}
+
+static inline struct rtw89_vif *vif_to_rtwvif_safe(struct ieee80211_vif *vif)
+{
+	return vif ? (struct rtw89_vif *)vif->drv_priv : NULL;
+}
+
 static inline struct ieee80211_sta *rtwsta_to_sta(struct rtw89_sta *rtwsta)
 {
 	void *p = rtwsta;
@@ -3542,6 +3862,20 @@ enum nl80211_band rtw89_hw_to_nl80211_band(enum rtw89_band hw_band)
 }
 
 static inline
+enum rtw89_band rtw89_nl80211_to_hw_band(enum nl80211_band nl_band)
+{
+	switch (nl_band) {
+	default:
+	case NL80211_BAND_2GHZ:
+		return RTW89_BAND_2G;
+	case NL80211_BAND_5GHZ:
+		return RTW89_BAND_5G;
+	case NL80211_BAND_6GHZ:
+		return RTW89_BAND_6G;
+	}
+}
+
+static inline
 enum rtw89_bandwidth nl_to_rtw89_bandwidth(enum nl80211_chan_width width)
 {
 	switch (width) {
@@ -3588,16 +3922,51 @@ struct rtw89_bssid_cam_entry *rtw89_get_bssid_cam_of(struct rtw89_vif *rtwvif,
 
 static inline
 void rtw89_chip_set_channel_prepare(struct rtw89_dev *rtwdev,
-				    struct rtw89_channel_help_params *p)
+				    struct rtw89_channel_help_params *p,
+				    const struct rtw89_chan *chan,
+				    enum rtw89_mac_idx mac_idx,
+				    enum rtw89_phy_idx phy_idx)
 {
-	rtwdev->chip->ops->set_channel_help(rtwdev, true, p);
+	rtwdev->chip->ops->set_channel_help(rtwdev, true, p, chan,
+					    mac_idx, phy_idx);
 }
 
 static inline
 void rtw89_chip_set_channel_done(struct rtw89_dev *rtwdev,
-				 struct rtw89_channel_help_params *p)
+				 struct rtw89_channel_help_params *p,
+				 const struct rtw89_chan *chan,
+				 enum rtw89_mac_idx mac_idx,
+				 enum rtw89_phy_idx phy_idx)
+{
+	rtwdev->chip->ops->set_channel_help(rtwdev, false, p, chan,
+					    mac_idx, phy_idx);
+}
+
+static inline
+const struct cfg80211_chan_def *rtw89_chandef_get(struct rtw89_dev *rtwdev,
+						  enum rtw89_sub_entity_idx idx)
 {
-	rtwdev->chip->ops->set_channel_help(rtwdev, false, p);
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	return &hal->chandef[idx];
+}
+
+static inline
+const struct rtw89_chan *rtw89_chan_get(struct rtw89_dev *rtwdev,
+					enum rtw89_sub_entity_idx idx)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	return &hal->chan[idx];
+}
+
+static inline
+const struct rtw89_chan_rcd *rtw89_chan_rcd_get(struct rtw89_dev *rtwdev,
+						enum rtw89_sub_entity_idx idx)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+
+	return &hal->chan_rcd[idx];
 }
 
 static inline void rtw89_chip_fem_setup(struct rtw89_dev *rtwdev)
@@ -3632,12 +4001,13 @@ static inline void rtw89_chip_rfk_channel(struct rtw89_dev *rtwdev)
 		chip->ops->rfk_channel(rtwdev);
 }
 
-static inline void rtw89_chip_rfk_band_changed(struct rtw89_dev *rtwdev)
+static inline void rtw89_chip_rfk_band_changed(struct rtw89_dev *rtwdev,
+					       enum rtw89_phy_idx phy_idx)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 
 	if (chip->ops->rfk_band_changed)
-		chip->ops->rfk_band_changed(rtwdev);
+		chip->ops->rfk_band_changed(rtwdev, phy_idx);
 }
 
 static inline void rtw89_chip_rfk_scan(struct rtw89_dev *rtwdev, bool start)
@@ -3661,19 +4031,7 @@ static inline void rtw89_chip_set_txpwr_ctrl(struct rtw89_dev *rtwdev)
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 
 	if (chip->ops->set_txpwr_ctrl)
-		chip->ops->set_txpwr_ctrl(rtwdev);
-}
-
-static inline void rtw89_chip_set_txpwr(struct rtw89_dev *rtwdev)
-{
-	const struct rtw89_chip_info *chip = rtwdev->chip;
-	u8 ch = rtwdev->hal.current_channel;
-
-	if (!ch)
-		return;
-
-	if (chip->ops->set_txpwr)
-		chip->ops->set_txpwr(rtwdev);
+		chip->ops->set_txpwr_ctrl(rtwdev,  RTW89_PHY_0);
 }
 
 static inline void rtw89_chip_power_trim(struct rtw89_dev *rtwdev)
@@ -3902,16 +4260,27 @@ int rtw89_core_sta_disconnect(struct rtw89_dev *rtwdev,
 int rtw89_core_sta_remove(struct rtw89_dev *rtwdev,
 			  struct ieee80211_vif *vif,
 			  struct ieee80211_sta *sta);
+void rtw89_core_set_tid_config(struct rtw89_dev *rtwdev,
+			       struct ieee80211_sta *sta,
+			       struct cfg80211_tid_config *tid_config);
 int rtw89_core_init(struct rtw89_dev *rtwdev);
 void rtw89_core_deinit(struct rtw89_dev *rtwdev);
 int rtw89_core_register(struct rtw89_dev *rtwdev);
 void rtw89_core_unregister(struct rtw89_dev *rtwdev);
+struct rtw89_dev *rtw89_alloc_ieee80211_hw(struct device *device,
+					   u32 bus_data_size,
+					   const struct rtw89_chip_info *chip);
+void rtw89_free_ieee80211_hw(struct rtw89_dev *rtwdev);
+void rtw89_core_set_chip_txpwr(struct rtw89_dev *rtwdev);
+void rtw89_get_default_chandef(struct cfg80211_chan_def *chandef);
 void rtw89_set_channel(struct rtw89_dev *rtwdev);
 u8 rtw89_core_acquire_bit_map(unsigned long *addr, unsigned long size);
 void rtw89_core_release_bit_map(unsigned long *addr, u8 bit);
 void rtw89_core_release_all_bits_map(unsigned long *addr, unsigned int nbits);
-int rtw89_core_acquire_sta_ba_entry(struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx);
-int rtw89_core_release_sta_ba_entry(struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx);
+int rtw89_core_acquire_sta_ba_entry(struct rtw89_dev *rtwdev,
+				    struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx);
+int rtw89_core_release_sta_ba_entry(struct rtw89_dev *rtwdev,
+				    struct rtw89_sta *rtwsta, u8 tid, u8 *cam_idx);
 void rtw89_vif_type_mapping(struct ieee80211_vif *vif, bool assoc);
 int rtw89_chip_info_setup(struct rtw89_dev *rtwdev);
 bool rtw89_ra_report_to_bitrate(struct rtw89_dev *rtwdev, u8 rpt_rate, u16 *bitrate);
diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c
index 829c61da99bb..730e83d54257 100644
--- a/drivers/net/wireless/realtek/rtw89/debug.c
+++ b/drivers/net/wireless/realtek/rtw89/debug.c
@@ -525,7 +525,8 @@ static int __print_txpwr_map(struct seq_file *m, struct rtw89_dev *rtwdev,
 
 static void __print_regd(struct seq_file *m, struct rtw89_dev *rtwdev)
 {
-	u8 band = rtwdev->hal.current_band_type;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 band = chan->band_type;
 	u8 regd = rtw89_regd_get(rtwdev, band);
 
 	switch (regd) {
@@ -2189,6 +2190,37 @@ out:
 	return count;
 }
 
+static int rtw89_dbg_trigger_ctrl_error(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_cpuio_ctrl ctrl_para = {0};
+	u16 pkt_id;
+
+	rtw89_leave_ps_mode(rtwdev);
+
+	pkt_id = rtw89_mac_dle_buf_req(rtwdev, 0x20, true);
+	switch (pkt_id) {
+	case 0xffff:
+		return -ETIMEDOUT;
+	case 0xfff:
+		return -ENOMEM;
+	default:
+		break;
+	}
+
+	/* intentionally, enqueue two pkt, but has only one pkt id */
+	ctrl_para.cmd_type = CPUIO_OP_CMD_ENQ_TO_HEAD;
+	ctrl_para.start_pktid = pkt_id;
+	ctrl_para.end_pktid = pkt_id;
+	ctrl_para.pkt_num = 1; /* start from 0 */
+	ctrl_para.dst_pid = WDE_DLE_PORT_ID_WDRLS;
+	ctrl_para.dst_qid = WDE_DLE_QUEID_NO_REPORT;
+
+	if (rtw89_mac_set_cpuio(rtwdev, &ctrl_para, true))
+		return -EFAULT;
+
+	return 0;
+}
+
 static int
 rtw89_debug_priv_fw_crash_get(struct seq_file *m, void *v)
 {
@@ -2196,10 +2228,15 @@ rtw89_debug_priv_fw_crash_get(struct seq_file *m, void *v)
 	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 
 	seq_printf(m, "%d\n",
-		   test_bit(RTW89_FLAG_RESTART_TRIGGER, rtwdev->flags));
+		   test_bit(RTW89_FLAG_CRASH_SIMULATING, rtwdev->flags));
 	return 0;
 }
 
+enum rtw89_dbg_crash_simulation_type {
+	RTW89_DBG_SIM_CPU_EXCEPTION = 1,
+	RTW89_DBG_SIM_CTRL_ERROR = 2,
+};
+
 static ssize_t
 rtw89_debug_priv_fw_crash_set(struct file *filp, const char __user *user_buf,
 			      size_t count, loff_t *loff)
@@ -2207,22 +2244,30 @@ rtw89_debug_priv_fw_crash_set(struct file *filp, const char __user *user_buf,
 	struct seq_file *m = (struct seq_file *)filp->private_data;
 	struct rtw89_debugfs_priv *debugfs_priv = m->private;
 	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
-	bool fw_crash;
+	int (*sim)(struct rtw89_dev *rtwdev);
+	u8 crash_type;
 	int ret;
 
-	if (!RTW89_CHK_FW_FEATURE(CRASH_TRIGGER, &rtwdev->fw))
-		return -EOPNOTSUPP;
-
-	ret = kstrtobool_from_user(user_buf, count, &fw_crash);
+	ret = kstrtou8_from_user(user_buf, count, 0, &crash_type);
 	if (ret)
 		return -EINVAL;
 
-	if (!fw_crash)
+	switch (crash_type) {
+	case RTW89_DBG_SIM_CPU_EXCEPTION:
+		if (!RTW89_CHK_FW_FEATURE(CRASH_TRIGGER, &rtwdev->fw))
+			return -EOPNOTSUPP;
+		sim = rtw89_fw_h2c_trigger_cpu_exception;
+		break;
+	case RTW89_DBG_SIM_CTRL_ERROR:
+		sim = rtw89_dbg_trigger_ctrl_error;
+		break;
+	default:
 		return -EINVAL;
+	}
 
 	mutex_lock(&rtwdev->mutex);
-	set_bit(RTW89_FLAG_RESTART_TRIGGER, rtwdev->flags);
-	ret = rtw89_fw_h2c_trigger_cpu_exception(rtwdev);
+	set_bit(RTW89_FLAG_CRASH_SIMULATING, rtwdev->flags);
+	ret = sim(rtwdev);
 	mutex_unlock(&rtwdev->mutex);
 
 	if (ret)
@@ -2289,7 +2334,10 @@ static void rtw89_sta_info_get_iter(void *data, struct ieee80211_sta *sta)
 	struct rate_info *rate = &rtwsta->ra_report.txrate;
 	struct ieee80211_rx_status *status = &rtwsta->rx_status;
 	struct seq_file *m = (struct seq_file *)data;
+	struct rtw89_dev *rtwdev = rtwsta->rtwdev;
+	struct rtw89_hal *hal = &rtwdev->hal;
 	u8 rssi;
+	int i;
 
 	seq_printf(m, "TX rate [%d]: ", rtwsta->mac_id);
 
@@ -2305,9 +2353,10 @@ static void rtw89_sta_info_get_iter(void *data, struct ieee80211_sta *sta)
 			   he_gi_str[rate->he_gi] : "N/A");
 	else
 		seq_printf(m, "Legacy %d", rate->legacy);
+	seq_printf(m, "%s", rtwsta->ra_report.might_fallback_legacy ? " FB_G" : "");
 	seq_printf(m, "\t(hw_rate=0x%x)", rtwsta->ra_report.hw_rate);
 	seq_printf(m, "\t==> agg_wait=%d (%d)\n", rtwsta->max_agg_wait,
-		   sta->max_rc_amsdu_len);
+		   sta->deflink.agg.max_rc_amsdu_len);
 
 	seq_printf(m, "RX rate [%d]: ", rtwsta->mac_id);
 
@@ -2333,8 +2382,15 @@ static void rtw89_sta_info_get_iter(void *data, struct ieee80211_sta *sta)
 	seq_printf(m, "\t(hw_rate=0x%x)\n", rtwsta->rx_hw_rate);
 
 	rssi = ewma_rssi_read(&rtwsta->avg_rssi);
-	seq_printf(m, "RSSI: %d dBm (raw=%d, prev=%d)\n",
+	seq_printf(m, "RSSI: %d dBm (raw=%d, prev=%d) [",
 		   RTW89_RSSI_RAW_TO_DBM(rssi), rssi, rtwsta->prev_rssi);
+	for (i = 0; i < rtwdev->chip->rf_path_num; i++) {
+		rssi = ewma_rssi_read(&rtwsta->rssi[i]);
+		seq_printf(m, "%d%s%s", RTW89_RSSI_RAW_TO_DBM(rssi),
+			   hal->tx_path_diversity && (hal->antenna_tx & BIT(i)) ? "*" : "",
+			   i + 1 == rtwdev->chip->rf_path_num ? "" : ", ");
+	}
+	seq_puts(m, "]\n");
 }
 
 static void
@@ -2433,6 +2489,26 @@ void rtw89_vif_ids_get_iter(void *data, u8 *mac, struct ieee80211_vif *vif)
 	rtw89_dump_addr_cam(m, &rtwvif->addr_cam);
 }
 
+static void rtw89_dump_ba_cam(struct seq_file *m, struct rtw89_sta *rtwsta)
+{
+	struct rtw89_vif *rtwvif = rtwsta->rtwvif;
+	struct rtw89_dev *rtwdev = rtwvif->rtwdev;
+	struct rtw89_ba_cam_entry *entry;
+	bool first = true;
+
+	list_for_each_entry(entry, &rtwsta->ba_cam_list, list) {
+		if (first) {
+			seq_puts(m, "\tba_cam ");
+			first = false;
+		} else {
+			seq_puts(m, ", ");
+		}
+		seq_printf(m, "tid[%u]=%d", entry->tid,
+			   (int)(entry - rtwdev->cam_info.ba_cam_entry));
+	}
+	seq_puts(m, "\n");
+}
+
 static void rtw89_sta_ids_get_iter(void *data, struct ieee80211_sta *sta)
 {
 	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
@@ -2441,6 +2517,7 @@ static void rtw89_sta_ids_get_iter(void *data, struct ieee80211_sta *sta)
 	seq_printf(m, "STA [%d] %pM %s\n", rtwsta->mac_id, sta->addr,
 		   sta->tdls ? "(TDLS)" : "");
 	rtw89_dump_addr_cam(m, &rtwsta->addr_cam);
+	rtw89_dump_ba_cam(m, rtwsta);
 }
 
 static int rtw89_debug_priv_stations_get(struct seq_file *m, void *v)
@@ -2449,6 +2526,8 @@ static int rtw89_debug_priv_stations_get(struct seq_file *m, void *v)
 	struct rtw89_dev *rtwdev = debugfs_priv->rtwdev;
 	struct rtw89_cam_info *cam_info = &rtwdev->cam_info;
 
+	mutex_lock(&rtwdev->mutex);
+
 	seq_puts(m, "map:\n");
 	seq_printf(m, "\tmac_id:    %*ph\n", (int)sizeof(rtwdev->mac_id_map),
 		   rtwdev->mac_id_map);
@@ -2458,12 +2537,16 @@ static int rtw89_debug_priv_stations_get(struct seq_file *m, void *v)
 		   cam_info->bssid_cam_map);
 	seq_printf(m, "\tsec_cam:   %*ph\n", (int)sizeof(cam_info->sec_cam_map),
 		   cam_info->sec_cam_map);
+	seq_printf(m, "\tba_cam:    %*ph\n", (int)sizeof(cam_info->ba_cam_map),
+		   cam_info->ba_cam_map);
 
 	ieee80211_iterate_active_interfaces_atomic(rtwdev->hw,
 		IEEE80211_IFACE_ITER_NORMAL, rtw89_vif_ids_get_iter, m);
 
 	ieee80211_iterate_stations_atomic(rtwdev->hw, rtw89_sta_ids_get_iter, m);
 
+	mutex_unlock(&rtwdev->mutex);
+
 	return 0;
 }
 
diff --git a/drivers/net/wireless/realtek/rtw89/debug.h b/drivers/net/wireless/realtek/rtw89/debug.h
index 6176152dbf6b..ee243aadde87 100644
--- a/drivers/net/wireless/realtek/rtw89/debug.h
+++ b/drivers/net/wireless/realtek/rtw89/debug.h
@@ -25,6 +25,7 @@ enum rtw89_debug_mask {
 	RTW89_DBG_BF = BIT(14),
 	RTW89_DBG_HW_SCAN = BIT(15),
 	RTW89_DBG_SAR = BIT(16),
+	RTW89_DBG_STATE = BIT(17),
 
 	RTW89_DBG_UNEXP = BIT(31),
 };
diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c
index 6473015a6b2a..d57e3610fb88 100644
--- a/drivers/net/wireless/realtek/rtw89/fw.c
+++ b/drivers/net/wireless/realtek/rtw89/fw.c
@@ -3,6 +3,7 @@
  */
 
 #include "cam.h"
+#include "chan.h"
 #include "coex.h"
 #include "debug.h"
 #include "fw.h"
@@ -224,6 +225,12 @@ static const struct __fw_feat_cfg fw_feat_tbl[] = {
 	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 35, 0, SCAN_OFFLOAD),
 	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 35, 0, TX_WAKE),
 	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 36, 0, CRASH_TRIGGER),
+	__CFG_FW_FEAT(RTL8852A, ge, 0, 13, 38, 0, PACKET_DROP),
+	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 20, 0, PACKET_DROP),
+	__CFG_FW_FEAT(RTL8852C, le, 0, 27, 33, 0, NO_DEEP_PS),
+	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 34, 0, TX_WAKE),
+	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 36, 0, SCAN_OFFLOAD),
+	__CFG_FW_FEAT(RTL8852C, ge, 0, 27, 40, 0, CRASH_TRIGGER),
 };
 
 static void rtw89_fw_recognize_features(struct rtw89_dev *rtwdev)
@@ -247,6 +254,46 @@ static void rtw89_fw_recognize_features(struct rtw89_dev *rtwdev)
 	}
 }
 
+void rtw89_early_fw_feature_recognize(struct device *device,
+				      const struct rtw89_chip_info *chip,
+				      u32 *early_feat_map)
+{
+	union {
+		struct rtw89_mfw_hdr mfw_hdr;
+		u8 fw_hdr[RTW89_FW_HDR_SIZE];
+	} buf = {};
+	const struct firmware *firmware;
+	u32 ver_code;
+	int ret;
+	int i;
+
+	ret = request_partial_firmware_into_buf(&firmware, chip->fw_name,
+						device, &buf, sizeof(buf), 0);
+	if (ret) {
+		dev_err(device, "failed to early request firmware: %d\n", ret);
+		return;
+	}
+
+	ver_code = buf.mfw_hdr.sig != RTW89_MFW_SIG ?
+		   RTW89_FW_HDR_VER_CODE(&buf.fw_hdr) :
+		   RTW89_MFW_HDR_VER_CODE(&buf.mfw_hdr);
+	if (!ver_code)
+		goto out;
+
+	for (i = 0; i < ARRAY_SIZE(fw_feat_tbl); i++) {
+		const struct __fw_feat_cfg *ent = &fw_feat_tbl[i];
+
+		if (chip->chip_id != ent->chip_id)
+			continue;
+
+		if (ent->cond(ver_code, ent->ver_code))
+			*early_feat_map |= BIT(ent->feature);
+	}
+
+out:
+	release_firmware(firmware);
+}
+
 int rtw89_fw_recognize(struct rtw89_dev *rtwdev)
 {
 	int ret;
@@ -571,6 +618,7 @@ int rtw89_fw_h2c_cam(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 		     struct rtw89_sta *rtwsta, const u8 *scan_mac_addr)
 {
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CAM_LEN);
 	if (!skb) {
@@ -587,7 +635,8 @@ int rtw89_fw_h2c_cam(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 			      H2C_FUNC_MAC_ADDR_CAM_UPD, 0, 1,
 			      H2C_CAM_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -596,7 +645,7 @@ int rtw89_fw_h2c_cam(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_DCTL_SEC_CAM_LEN 68
@@ -605,6 +654,7 @@ int rtw89_fw_h2c_dctl_sec_cam_v1(struct rtw89_dev *rtwdev,
 				 struct rtw89_sta *rtwsta)
 {
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_DCTL_SEC_CAM_LEN);
 	if (!skb) {
@@ -621,7 +671,8 @@ int rtw89_fw_h2c_dctl_sec_cam_v1(struct rtw89_dev *rtwdev,
 			      H2C_FUNC_MAC_DCTLINFO_UD_V1, 0, 0,
 			      H2C_DCTL_SEC_CAM_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -630,7 +681,7 @@ int rtw89_fw_h2c_dctl_sec_cam_v1(struct rtw89_dev *rtwdev,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 EXPORT_SYMBOL(rtw89_fw_h2c_dctl_sec_cam_v1);
 
@@ -638,14 +689,16 @@ EXPORT_SYMBOL(rtw89_fw_h2c_dctl_sec_cam_v1);
 int rtw89_fw_h2c_ba_cam(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta,
 			bool valid, struct ieee80211_ampdu_params *params)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct rtw89_vif *rtwvif = rtwsta->rtwvif;
 	u8 macid = rtwsta->mac_id;
 	struct sk_buff *skb;
 	u8 entry_idx;
 	int ret;
 
 	ret = valid ?
-	      rtw89_core_acquire_sta_ba_entry(rtwsta, params->tid, &entry_idx) :
-	      rtw89_core_release_sta_ba_entry(rtwsta, params->tid, &entry_idx);
+	      rtw89_core_acquire_sta_ba_entry(rtwdev, rtwsta, params->tid, &entry_idx) :
+	      rtw89_core_release_sta_ba_entry(rtwdev, rtwsta, params->tid, &entry_idx);
 	if (ret) {
 		/* it still works even if we don't have static BA CAM, because
 		 * hardware can create dynamic BA CAM automatically.
@@ -663,7 +716,10 @@ int rtw89_fw_h2c_ba_cam(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta,
 	}
 	skb_put(skb, H2C_BA_CAM_LEN);
 	SET_BA_CAM_MACID(skb->data, macid);
-	SET_BA_CAM_ENTRY_IDX(skb->data, entry_idx);
+	if (chip->bacam_v1)
+		SET_BA_CAM_ENTRY_IDX_V1(skb->data, entry_idx);
+	else
+		SET_BA_CAM_ENTRY_IDX(skb->data, entry_idx);
 	if (!valid)
 		goto end;
 	SET_BA_CAM_VALID(skb->data, valid);
@@ -676,6 +732,11 @@ int rtw89_fw_h2c_ba_cam(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta,
 	SET_BA_CAM_INIT_REQ(skb->data, 1);
 	SET_BA_CAM_SSN(skb->data, params->ssn);
 
+	if (chip->bacam_v1) {
+		SET_BA_CAM_STD_EN(skb->data, 1);
+		SET_BA_CAM_BAND(skb->data, rtwvif->mac_idx);
+	}
+
 end:
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_MAC,
@@ -683,7 +744,8 @@ end:
 			      H2C_FUNC_MAC_BA_CAM, 0, 1,
 			      H2C_BA_CAM_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -692,7 +754,59 @@ end:
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
+}
+
+static int rtw89_fw_h2c_init_dynamic_ba_cam_v1(struct rtw89_dev *rtwdev,
+					       u8 entry_idx, u8 uid)
+{
+	struct sk_buff *skb;
+	int ret;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_BA_CAM_LEN);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for dynamic h2c ba cam\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, H2C_BA_CAM_LEN);
+
+	SET_BA_CAM_VALID(skb->data, 1);
+	SET_BA_CAM_ENTRY_IDX_V1(skb->data, entry_idx);
+	SET_BA_CAM_UID(skb->data, uid);
+	SET_BA_CAM_BAND(skb->data, 0);
+	SET_BA_CAM_STD_EN(skb->data, 0);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC,
+			      H2C_CL_BA_CAM,
+			      H2C_FUNC_MAC_BA_CAM, 0, 1,
+			      H2C_BA_CAM_LEN);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
+void rtw89_fw_h2c_init_ba_cam_v1(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	u8 entry_idx = chip->bacam_num;
+	u8 uid = 0;
+	int i;
+
+	for (i = 0; i < chip->bacam_dynamic_num; i++) {
+		rtw89_fw_h2c_init_dynamic_ba_cam_v1(rtwdev, entry_idx, uid);
+		entry_idx++;
+		uid++;
+	}
 }
 
 #define H2C_LOG_CFG_LEN 12
@@ -701,6 +815,7 @@ int rtw89_fw_h2c_fw_log(struct rtw89_dev *rtwdev, bool enable)
 	struct sk_buff *skb;
 	u32 comp = enable ? BIT(RTW89_FW_LOG_COMP_INIT) | BIT(RTW89_FW_LOG_COMP_TASK) |
 			    BIT(RTW89_FW_LOG_COMP_PS) | BIT(RTW89_FW_LOG_COMP_ERROR) : 0;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LOG_CFG_LEN);
 	if (!skb) {
@@ -720,7 +835,8 @@ int rtw89_fw_h2c_fw_log(struct rtw89_dev *rtwdev, bool enable)
 			      H2C_FUNC_LOG_CFG, 0, 0,
 			      H2C_LOG_CFG_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -729,7 +845,7 @@ int rtw89_fw_h2c_fw_log(struct rtw89_dev *rtwdev, bool enable)
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_GENERAL_PKT_LEN 6
@@ -737,6 +853,7 @@ fail:
 int rtw89_fw_h2c_general_pkt(struct rtw89_dev *rtwdev, u8 macid)
 {
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_GENERAL_PKT_LEN);
 	if (!skb) {
@@ -757,7 +874,8 @@ int rtw89_fw_h2c_general_pkt(struct rtw89_dev *rtwdev, u8 macid)
 			      H2C_FUNC_MAC_GENERAL_PKT, 0, 1,
 			      H2C_GENERAL_PKT_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -766,7 +884,7 @@ int rtw89_fw_h2c_general_pkt(struct rtw89_dev *rtwdev, u8 macid)
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_LPS_PARM_LEN 8
@@ -774,6 +892,7 @@ int rtw89_fw_h2c_lps_parm(struct rtw89_dev *rtwdev,
 			  struct rtw89_lps_parm *lps_param)
 {
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LPS_PARM_LEN);
 	if (!skb) {
@@ -799,7 +918,8 @@ int rtw89_fw_h2c_lps_parm(struct rtw89_dev *rtwdev,
 			      H2C_FUNC_MAC_LPS_PARM, 0, 1,
 			      H2C_LPS_PARM_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -808,7 +928,73 @@ int rtw89_fw_h2c_lps_parm(struct rtw89_dev *rtwdev,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
+}
+
+#define H2C_P2P_ACT_LEN 20
+int rtw89_fw_h2c_p2p_act(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
+			 struct ieee80211_p2p_noa_desc *desc,
+			 u8 act, u8 noa_id)
+{
+	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
+	bool p2p_type_gc = rtwvif->wifi_role == RTW89_WIFI_ROLE_P2P_CLIENT;
+	u8 ctwindow_oppps = vif->bss_conf.p2p_noa_attr.oppps_ctwindow;
+	struct sk_buff *skb;
+	u8 *cmd;
+	int ret;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_P2P_ACT_LEN);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c p2p act\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, H2C_P2P_ACT_LEN);
+	cmd = skb->data;
+
+	RTW89_SET_FWCMD_P2P_MACID(cmd, rtwvif->mac_id);
+	RTW89_SET_FWCMD_P2P_P2PID(cmd, 0);
+	RTW89_SET_FWCMD_P2P_NOAID(cmd, noa_id);
+	RTW89_SET_FWCMD_P2P_ACT(cmd, act);
+	RTW89_SET_FWCMD_P2P_TYPE(cmd, p2p_type_gc);
+	RTW89_SET_FWCMD_P2P_ALL_SLEP(cmd, 0);
+	if (desc) {
+		RTW89_SET_FWCMD_NOA_START_TIME(cmd, desc->start_time);
+		RTW89_SET_FWCMD_NOA_INTERVAL(cmd, desc->interval);
+		RTW89_SET_FWCMD_NOA_DURATION(cmd, desc->duration);
+		RTW89_SET_FWCMD_NOA_COUNT(cmd, desc->count);
+		RTW89_SET_FWCMD_NOA_CTWINDOW(cmd, ctwindow_oppps);
+	}
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC, H2C_CL_MAC_PS,
+			      H2C_FUNC_P2P_ACT, 0, 0,
+			      H2C_P2P_ACT_LEN);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
+static void __rtw89_fw_h2c_set_tx_path(struct rtw89_dev *rtwdev,
+				       struct sk_buff *skb)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	u8 ntx_path = hal->antenna_tx ? hal->antenna_tx : RF_B;
+	u8 map_b = hal->antenna_tx == RF_AB ? 1 : 0;
+
+	SET_CMC_TBL_NTX_PATH_EN(skb->data, ntx_path);
+	SET_CMC_TBL_PATH_MAP_A(skb->data, 0);
+	SET_CMC_TBL_PATH_MAP_B(skb->data, map_b);
+	SET_CMC_TBL_PATH_MAP_C(skb->data, 0);
+	SET_CMC_TBL_PATH_MAP_D(skb->data, 0);
 }
 
 #define H2C_CMC_TBL_LEN 68
@@ -816,11 +1002,9 @@ int rtw89_fw_h2c_default_cmac_tbl(struct rtw89_dev *rtwdev,
 				  struct rtw89_vif *rtwvif)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
-	struct rtw89_hal *hal = &rtwdev->hal;
 	struct sk_buff *skb;
-	u8 ntx_path = hal->antenna_tx ? hal->antenna_tx : RF_B;
-	u8 map_b = hal->antenna_tx == RF_AB ? 1 : 0;
 	u8 macid = rtwvif->mac_id;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CMC_TBL_LEN);
 	if (!skb) {
@@ -832,11 +1016,7 @@ int rtw89_fw_h2c_default_cmac_tbl(struct rtw89_dev *rtwdev,
 	SET_CTRL_INFO_OPERATION(skb->data, 1);
 	if (chip->h2c_cctl_func_id == H2C_FUNC_MAC_CCTLINFO_UD) {
 		SET_CMC_TBL_TXPWR_MODE(skb->data, 0);
-		SET_CMC_TBL_NTX_PATH_EN(skb->data, ntx_path);
-		SET_CMC_TBL_PATH_MAP_A(skb->data, 0);
-		SET_CMC_TBL_PATH_MAP_B(skb->data, map_b);
-		SET_CMC_TBL_PATH_MAP_C(skb->data, 0);
-		SET_CMC_TBL_PATH_MAP_D(skb->data, 0);
+		__rtw89_fw_h2c_set_tx_path(rtwdev, skb);
 		SET_CMC_TBL_ANTSEL_A(skb->data, 0);
 		SET_CMC_TBL_ANTSEL_B(skb->data, 0);
 		SET_CMC_TBL_ANTSEL_C(skb->data, 0);
@@ -852,7 +1032,8 @@ int rtw89_fw_h2c_default_cmac_tbl(struct rtw89_dev *rtwdev,
 			      chip->h2c_cctl_func_id, 0, 1,
 			      H2C_CMC_TBL_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -861,7 +1042,7 @@ int rtw89_fw_h2c_default_cmac_tbl(struct rtw89_dev *rtwdev,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 static void __get_sta_he_pkt_padding(struct rtw89_dev *rtwdev,
@@ -926,17 +1107,26 @@ int rtw89_fw_h2c_assoc_cmac_tbl(struct rtw89_dev *rtwdev,
 				struct ieee80211_sta *sta)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
-	struct rtw89_hal *hal = &rtwdev->hal;
 	struct rtw89_sta *rtwsta = sta_to_rtwsta_safe(sta);
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	struct sk_buff *skb;
 	u8 pads[RTW89_PPE_BW_NUM];
 	u8 mac_id = rtwsta ? rtwsta->mac_id : rtwvif->mac_id;
+	u16 lowest_rate;
+	int ret;
 
 	memset(pads, 0, sizeof(pads));
 	if (sta)
 		__get_sta_he_pkt_padding(rtwdev, sta, pads);
 
+	if (vif->p2p)
+		lowest_rate = RTW89_HW_RATE_OFDM6;
+	else if (chan->band_type == RTW89_BAND_2G)
+		lowest_rate = RTW89_HW_RATE_CCK1;
+	else
+		lowest_rate = RTW89_HW_RATE_OFDM6;
+
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CMC_TBL_LEN);
 	if (!skb) {
 		rtw89_err(rtwdev, "failed to alloc skb for fw dl\n");
@@ -947,10 +1137,7 @@ int rtw89_fw_h2c_assoc_cmac_tbl(struct rtw89_dev *rtwdev,
 	SET_CTRL_INFO_OPERATION(skb->data, 1);
 	SET_CMC_TBL_DISRTSFB(skb->data, 1);
 	SET_CMC_TBL_DISDATAFB(skb->data, 1);
-	if (hal->current_band_type == RTW89_BAND_2G)
-		SET_CMC_TBL_RTS_RTY_LOWEST_RATE(skb->data, RTW89_HW_RATE_CCK1);
-	else
-		SET_CMC_TBL_RTS_RTY_LOWEST_RATE(skb->data, RTW89_HW_RATE_OFDM6);
+	SET_CMC_TBL_RTS_RTY_LOWEST_RATE(skb->data, lowest_rate);
 	SET_CMC_TBL_RTS_TXCNT_LMT_SEL(skb->data, 0);
 	SET_CMC_TBL_DATA_TXCNT_LMT_SEL(skb->data, 0);
 	if (vif->type == NL80211_IFTYPE_STATION)
@@ -980,7 +1167,8 @@ int rtw89_fw_h2c_assoc_cmac_tbl(struct rtw89_dev *rtwdev,
 			      chip->h2c_cctl_func_id, 0, 1,
 			      H2C_CMC_TBL_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -989,7 +1177,7 @@ int rtw89_fw_h2c_assoc_cmac_tbl(struct rtw89_dev *rtwdev,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev,
@@ -997,6 +1185,7 @@ int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev,
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CMC_TBL_LEN);
 	if (!skb) {
@@ -1020,7 +1209,47 @@ int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev,
 			      chip->h2c_cctl_func_id, 0, 1,
 			      H2C_CMC_TBL_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
+}
+
+int rtw89_fw_h2c_txpath_cmac_tbl(struct rtw89_dev *rtwdev,
+				 struct rtw89_sta *rtwsta)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	struct sk_buff *skb;
+	int ret;
+
+	if (chip->h2c_cctl_func_id != H2C_FUNC_MAC_CCTLINFO_UD)
+		return 0;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_CMC_TBL_LEN);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for fw dl\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, H2C_CMC_TBL_LEN);
+	SET_CTRL_INFO_MACID(skb->data, rtwsta->mac_id);
+	SET_CTRL_INFO_OPERATION(skb->data, 1);
+
+	__rtw89_fw_h2c_set_tx_path(rtwdev, skb);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC, H2C_CL_MAC_FR_EXCHG,
+			      H2C_FUNC_MAC_CCTLINFO_UD, 0, 1,
+			      H2C_CMC_TBL_LEN);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1029,19 +1258,28 @@ int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_BCN_BASE_LEN 12
 int rtw89_fw_h2c_update_beacon(struct rtw89_dev *rtwdev,
 			       struct rtw89_vif *rtwvif)
 {
-	struct rtw89_hal *hal = &rtwdev->hal;
 	struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif);
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	struct sk_buff *skb;
 	struct sk_buff *skb_beacon;
 	u16 tim_offset;
 	int bcn_total_len;
+	u16 beacon_rate;
+	int ret;
+
+	if (vif->p2p)
+		beacon_rate = RTW89_HW_RATE_OFDM6;
+	else if (chan->band_type == RTW89_BAND_2G)
+		beacon_rate = RTW89_HW_RATE_CCK1;
+	else
+		beacon_rate = RTW89_HW_RATE_OFDM6;
 
 	skb_beacon = ieee80211_beacon_get_tim(rtwdev->hw, vif, &tim_offset,
 					      NULL, 0);
@@ -1066,8 +1304,7 @@ int rtw89_fw_h2c_update_beacon(struct rtw89_dev *rtwdev,
 	SET_BCN_UPD_MACID(skb->data, rtwvif->mac_id);
 	SET_BCN_UPD_SSN_SEL(skb->data, RTW89_MGMT_HW_SSN_SEL);
 	SET_BCN_UPD_SSN_MODE(skb->data, RTW89_MGMT_HW_SEQ_MODE);
-	SET_BCN_UPD_RATE(skb->data, hal->current_band_type == RTW89_BAND_2G ?
-				    RTW89_HW_RATE_CCK1 : RTW89_HW_RATE_OFDM6);
+	SET_BCN_UPD_RATE(skb->data, beacon_rate);
 
 	skb_put_data(skb, skb_beacon->data, skb_beacon->len);
 	dev_kfree_skb_any(skb_beacon);
@@ -1077,10 +1314,11 @@ int rtw89_fw_h2c_update_beacon(struct rtw89_dev *rtwdev,
 			      H2C_FUNC_MAC_BCN_UPD, 0, 1,
 			      bcn_total_len);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		dev_kfree_skb_any(skb);
-		return -EBUSY;
+		return ret;
 	}
 
 	return 0;
@@ -1095,6 +1333,7 @@ int rtw89_fw_h2c_role_maintain(struct rtw89_dev *rtwdev,
 	struct sk_buff *skb;
 	u8 mac_id = rtwsta ? rtwsta->mac_id : rtwvif->mac_id;
 	u8 self_role;
+	int ret;
 
 	if (rtwvif->net_type == RTW89_NET_TYPE_AP_MODE) {
 		if (rtwsta)
@@ -1121,7 +1360,8 @@ int rtw89_fw_h2c_role_maintain(struct rtw89_dev *rtwdev,
 			      H2C_FUNC_MAC_FWROLE_MAINTAIN, 0, 1,
 			      H2C_ROLE_MAINTAIN_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1130,7 +1370,7 @@ int rtw89_fw_h2c_role_maintain(struct rtw89_dev *rtwdev,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_JOIN_INFO_LEN 4
@@ -1141,6 +1381,7 @@ int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 	u8 mac_id = rtwsta ? rtwsta->mac_id : rtwvif->mac_id;
 	u8 self_role = rtwvif->self_role;
 	u8 net_type = rtwvif->net_type;
+	int ret;
 
 	if (net_type == RTW89_NET_TYPE_AP_MODE && rtwsta) {
 		self_role = RTW89_SELF_ROLE_AP_CLIENT;
@@ -1172,7 +1413,8 @@ int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 			      H2C_FUNC_MAC_JOININFO, 0, 1,
 			      H2C_JOIN_INFO_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1181,7 +1423,7 @@ int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp,
@@ -1190,6 +1432,7 @@ int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp,
 	struct rtw89_fw_macid_pause_grp h2c = {{0}};
 	u8 len = sizeof(struct rtw89_fw_macid_pause_grp);
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_JOIN_INFO_LEN);
 	if (!skb) {
@@ -1206,7 +1449,8 @@ int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp,
 			      H2C_FUNC_MAC_MACID_PAUSE, 1, 0,
 			      len);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1215,7 +1459,7 @@ int rtw89_fw_h2c_macid_pause(struct rtw89_dev *rtwdev, u8 sh, u8 grp,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_EDCA_LEN 12
@@ -1223,6 +1467,7 @@ int rtw89_fw_h2c_set_edca(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 			  u8 ac, u32 val)
 {
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_EDCA_LEN);
 	if (!skb) {
@@ -1241,7 +1486,8 @@ int rtw89_fw_h2c_set_edca(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 			      H2C_FUNC_USR_EDCA, 0, 1,
 			      H2C_EDCA_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1250,7 +1496,47 @@ int rtw89_fw_h2c_set_edca(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
+}
+
+#define H2C_TSF32_TOGL_LEN 4
+int rtw89_fw_h2c_tsf32_toggle(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
+			      bool en)
+{
+	struct sk_buff *skb;
+	u16 early_us = en ? 2000 : 0;
+	u8 *cmd;
+	int ret;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_TSF32_TOGL_LEN);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c p2p act\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, H2C_TSF32_TOGL_LEN);
+	cmd = skb->data;
+
+	RTW89_SET_FWCMD_TSF32_TOGL_BAND(cmd, rtwvif->mac_idx);
+	RTW89_SET_FWCMD_TSF32_TOGL_EN(cmd, en);
+	RTW89_SET_FWCMD_TSF32_TOGL_PORT(cmd, rtwvif->port);
+	RTW89_SET_FWCMD_TSF32_TOGL_EARLY(cmd, early_us);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC, H2C_CL_MAC_FW_OFLD,
+			      H2C_FUNC_TSF32_TOGL, 0, 0,
+			      H2C_TSF32_TOGL_LEN);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
 }
 
 #define H2C_OFLD_CFG_LEN 8
@@ -1258,6 +1544,7 @@ int rtw89_fw_h2c_set_ofld_cfg(struct rtw89_dev *rtwdev)
 {
 	static const u8 cfg[] = {0x09, 0x00, 0x00, 0x00, 0x5e, 0x00, 0x00, 0x00};
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_OFLD_CFG_LEN);
 	if (!skb) {
@@ -1271,7 +1558,8 @@ int rtw89_fw_h2c_set_ofld_cfg(struct rtw89_dev *rtwdev)
 			      H2C_FUNC_OFLD_CFG, 0, 1,
 			      H2C_OFLD_CFG_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1280,7 +1568,7 @@ int rtw89_fw_h2c_set_ofld_cfg(struct rtw89_dev *rtwdev)
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_RA_LEN 16
@@ -1288,6 +1576,7 @@ int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi
 {
 	struct sk_buff *skb;
 	u8 *cmd;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_RA_LEN);
 	if (!skb) {
@@ -1318,6 +1607,8 @@ int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi
 	RTW89_SET_FWCMD_RA_MASK_2(cmd, FIELD_GET(MASKBYTE2, ra->ra_mask));
 	RTW89_SET_FWCMD_RA_MASK_3(cmd, FIELD_GET(MASKBYTE3, ra->ra_mask));
 	RTW89_SET_FWCMD_RA_MASK_4(cmd, FIELD_GET(MASKBYTE4, ra->ra_mask));
+	RTW89_SET_FWCMD_RA_FIX_GILTF_EN(cmd, ra->fix_giltf_en);
+	RTW89_SET_FWCMD_RA_FIX_GILTF(cmd, ra->fix_giltf);
 
 	if (csi) {
 		RTW89_SET_FWCMD_RA_BFEE_CSI_CTL(cmd, 1);
@@ -1336,7 +1627,8 @@ int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi
 			      H2C_FUNC_OUTSRC_RA_MACIDCFG, 0, 0,
 			      H2C_RA_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1345,7 +1637,7 @@ int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_LEN_CXDRVHDR 2
@@ -1359,6 +1651,7 @@ int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_ant_info *ant = &module->ant;
 	struct sk_buff *skb;
 	u8 *cmd;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_INIT);
 	if (!skb) {
@@ -1395,7 +1688,8 @@ int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev)
 			      SET_DRV_INFO, 0, 0,
 			      H2C_LEN_CXDRVINFO_INIT);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1404,10 +1698,15 @@ int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev)
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
+#define PORT_DATA_OFFSET 4
+#define H2C_LEN_CXDRVINFO_ROLE_DBCC_LEN 12
 #define H2C_LEN_CXDRVINFO_ROLE (4 + 12 * RTW89_PORT_NUM + H2C_LEN_CXDRVHDR)
+#define H2C_LEN_CXDRVINFO_ROLE_V1 (4 + 16 * RTW89_PORT_NUM + \
+				   H2C_LEN_CXDRVINFO_ROLE_DBCC_LEN + \
+				   H2C_LEN_CXDRVHDR)
 int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_btc *btc = &rtwdev->btc;
@@ -1416,7 +1715,9 @@ int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_wl_role_info_bpos *bpos = &role_info->role_map.role;
 	struct rtw89_btc_wl_active_role *active = role_info->active_role;
 	struct sk_buff *skb;
+	u8 offset = 0;
 	u8 *cmd;
+	int ret;
 	int i;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_ROLE);
@@ -1447,19 +1748,19 @@ int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev)
 	RTW89_SET_FWCMD_CXROLE_ROLE_NAN(cmd, bpos->nan);
 
 	for (i = 0; i < RTW89_PORT_NUM; i++, active++) {
-		RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED(cmd, active->connected, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_PID(cmd, active->pid, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_PHY(cmd, active->phy, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_NOA(cmd, active->noa, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_BAND(cmd, active->band, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS(cmd, active->client_ps, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_BW(cmd, active->bw, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_ROLE(cmd, active->role, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_CH(cmd, active->ch, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_TX_LVL(cmd, active->tx_lvl, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_RX_LVL(cmd, active->rx_lvl, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_TX_RATE(cmd, active->tx_rate, i);
-		RTW89_SET_FWCMD_CXROLE_ACT_RX_RATE(cmd, active->rx_rate, i);
+		RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED(cmd, active->connected, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_PID(cmd, active->pid, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_PHY(cmd, active->phy, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_NOA(cmd, active->noa, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_BAND(cmd, active->band, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS(cmd, active->client_ps, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_BW(cmd, active->bw, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_ROLE(cmd, active->role, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_CH(cmd, active->ch, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_TX_LVL(cmd, active->tx_lvl, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_RX_LVL(cmd, active->rx_lvl, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_TX_RATE(cmd, active->tx_rate, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_RX_RATE(cmd, active->rx_rate, i, offset);
 	}
 
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
@@ -1467,7 +1768,8 @@ int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev)
 			      SET_DRV_INFO, 0, 0,
 			      H2C_LEN_CXDRVINFO_ROLE);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1476,16 +1778,101 @@ int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev)
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
+}
+
+int rtw89_fw_h2c_cxdrv_role_v1(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_btc *btc = &rtwdev->btc;
+	struct rtw89_btc_wl_info *wl = &btc->cx.wl;
+	struct rtw89_btc_wl_role_info_v1 *role_info = &wl->role_info_v1;
+	struct rtw89_btc_wl_role_info_bpos *bpos = &role_info->role_map.role;
+	struct rtw89_btc_wl_active_role_v1 *active = role_info->active_role_v1;
+	struct sk_buff *skb;
+	u8 *cmd, offset;
+	int ret;
+	int i;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_ROLE_V1);
+	if (!skb) {
+		rtw89_err(rtwdev, "failed to alloc skb for h2c cxdrv_role\n");
+		return -ENOMEM;
+	}
+	skb_put(skb, H2C_LEN_CXDRVINFO_ROLE_V1);
+	cmd = skb->data;
+
+	RTW89_SET_FWCMD_CXHDR_TYPE(cmd, CXDRVINFO_ROLE);
+	RTW89_SET_FWCMD_CXHDR_LEN(cmd, H2C_LEN_CXDRVINFO_ROLE_V1 - H2C_LEN_CXDRVHDR);
+
+	RTW89_SET_FWCMD_CXROLE_CONNECT_CNT(cmd, role_info->connect_cnt);
+	RTW89_SET_FWCMD_CXROLE_LINK_MODE(cmd, role_info->link_mode);
+
+	RTW89_SET_FWCMD_CXROLE_ROLE_NONE(cmd, bpos->none);
+	RTW89_SET_FWCMD_CXROLE_ROLE_STA(cmd, bpos->station);
+	RTW89_SET_FWCMD_CXROLE_ROLE_AP(cmd, bpos->ap);
+	RTW89_SET_FWCMD_CXROLE_ROLE_VAP(cmd, bpos->vap);
+	RTW89_SET_FWCMD_CXROLE_ROLE_ADHOC(cmd, bpos->adhoc);
+	RTW89_SET_FWCMD_CXROLE_ROLE_ADHOC_MASTER(cmd, bpos->adhoc_master);
+	RTW89_SET_FWCMD_CXROLE_ROLE_MESH(cmd, bpos->mesh);
+	RTW89_SET_FWCMD_CXROLE_ROLE_MONITOR(cmd, bpos->moniter);
+	RTW89_SET_FWCMD_CXROLE_ROLE_P2P_DEV(cmd, bpos->p2p_device);
+	RTW89_SET_FWCMD_CXROLE_ROLE_P2P_GC(cmd, bpos->p2p_gc);
+	RTW89_SET_FWCMD_CXROLE_ROLE_P2P_GO(cmd, bpos->p2p_go);
+	RTW89_SET_FWCMD_CXROLE_ROLE_NAN(cmd, bpos->nan);
+
+	offset = PORT_DATA_OFFSET;
+	for (i = 0; i < RTW89_PORT_NUM; i++, active++) {
+		RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED(cmd, active->connected, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_PID(cmd, active->pid, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_PHY(cmd, active->phy, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_NOA(cmd, active->noa, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_BAND(cmd, active->band, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS(cmd, active->client_ps, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_BW(cmd, active->bw, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_ROLE(cmd, active->role, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_CH(cmd, active->ch, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_TX_LVL(cmd, active->tx_lvl, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_RX_LVL(cmd, active->rx_lvl, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_TX_RATE(cmd, active->tx_rate, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_RX_RATE(cmd, active->rx_rate, i, offset);
+		RTW89_SET_FWCMD_CXROLE_ACT_NOA_DUR(cmd, active->noa_duration, i, offset);
+	}
+
+	offset = H2C_LEN_CXDRVINFO_ROLE_V1 - H2C_LEN_CXDRVINFO_ROLE_DBCC_LEN;
+	RTW89_SET_FWCMD_CXROLE_MROLE_TYPE(cmd, role_info->mrole_type, offset);
+	RTW89_SET_FWCMD_CXROLE_MROLE_NOA(cmd, role_info->mrole_noa_duration, offset);
+	RTW89_SET_FWCMD_CXROLE_DBCC_EN(cmd, role_info->dbcc_en, offset);
+	RTW89_SET_FWCMD_CXROLE_DBCC_CHG(cmd, role_info->dbcc_chg, offset);
+	RTW89_SET_FWCMD_CXROLE_DBCC_2G_PHY(cmd, role_info->dbcc_2g_phy, offset);
+	RTW89_SET_FWCMD_CXROLE_LINK_MODE_CHG(cmd, role_info->link_mode_chg, offset);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_OUTSRC, BTFC_SET,
+			      SET_DRV_INFO, 0, 0,
+			      H2C_LEN_CXDRVINFO_ROLE_V1);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+fail:
+	dev_kfree_skb_any(skb);
+
+	return ret;
 }
 
 #define H2C_LEN_CXDRVINFO_CTRL (4 + H2C_LEN_CXDRVHDR)
 int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_btc *btc = &rtwdev->btc;
 	struct rtw89_btc_ctrl *ctrl = &btc->ctrl;
 	struct sk_buff *skb;
 	u8 *cmd;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_CTRL);
 	if (!skb) {
@@ -1501,14 +1888,16 @@ int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev)
 	RTW89_SET_FWCMD_CXCTRL_MANUAL(cmd, ctrl->manual);
 	RTW89_SET_FWCMD_CXCTRL_IGNORE_BT(cmd, ctrl->igno_bt);
 	RTW89_SET_FWCMD_CXCTRL_ALWAYS_FREERUN(cmd, ctrl->always_freerun);
-	RTW89_SET_FWCMD_CXCTRL_TRACE_STEP(cmd, ctrl->trace_step);
+	if (chip->chip_id == RTL8852A)
+		RTW89_SET_FWCMD_CXCTRL_TRACE_STEP(cmd, ctrl->trace_step);
 
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_OUTSRC, BTFC_SET,
 			      SET_DRV_INFO, 0, 0,
 			      H2C_LEN_CXDRVINFO_CTRL);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1517,7 +1906,7 @@ int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev)
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_LEN_CXDRVINFO_RFK (4 + H2C_LEN_CXDRVHDR)
@@ -1528,6 +1917,7 @@ int rtw89_fw_h2c_cxdrv_rfk(struct rtw89_dev *rtwdev)
 	struct rtw89_btc_wl_rfk_info *rfk_info = &wl->rfk_info;
 	struct sk_buff *skb;
 	u8 *cmd;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_CXDRVINFO_RFK);
 	if (!skb) {
@@ -1551,7 +1941,8 @@ int rtw89_fw_h2c_cxdrv_rfk(struct rtw89_dev *rtwdev)
 			      SET_DRV_INFO, 0, 0,
 			      H2C_LEN_CXDRVINFO_RFK);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1560,7 +1951,7 @@ int rtw89_fw_h2c_cxdrv_rfk(struct rtw89_dev *rtwdev)
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_LEN_PKT_OFLD 4
@@ -1568,6 +1959,7 @@ int rtw89_fw_h2c_del_pkt_offload(struct rtw89_dev *rtwdev, u8 id)
 {
 	struct sk_buff *skb;
 	u8 *cmd;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_PKT_OFLD);
 	if (!skb) {
@@ -1585,7 +1977,8 @@ int rtw89_fw_h2c_del_pkt_offload(struct rtw89_dev *rtwdev, u8 id)
 			      H2C_FUNC_PACKET_OFLD, 1, 1,
 			      H2C_LEN_PKT_OFLD);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1594,7 +1987,7 @@ int rtw89_fw_h2c_del_pkt_offload(struct rtw89_dev *rtwdev, u8 id)
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 int rtw89_fw_h2c_add_pkt_offload(struct rtw89_dev *rtwdev, u8 *id,
@@ -1603,6 +1996,7 @@ int rtw89_fw_h2c_add_pkt_offload(struct rtw89_dev *rtwdev, u8 *id,
 	struct sk_buff *skb;
 	u8 *cmd;
 	u8 alloc_id;
+	int ret;
 
 	alloc_id = rtw89_core_acquire_bit_map(rtwdev->pkt_offload,
 					      RTW89_MAX_PKT_OFLD_NUM);
@@ -1629,7 +2023,8 @@ int rtw89_fw_h2c_add_pkt_offload(struct rtw89_dev *rtwdev, u8 *id,
 			      H2C_FUNC_PACKET_OFLD, 1, 1,
 			      H2C_LEN_PKT_OFLD + skb_ofld->len);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1638,7 +2033,7 @@ int rtw89_fw_h2c_add_pkt_offload(struct rtw89_dev *rtwdev, u8 *id,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 #define H2C_LEN_SCAN_LIST_OFFLOAD 4
@@ -1649,6 +2044,7 @@ int rtw89_fw_h2c_scan_list_offload(struct rtw89_dev *rtwdev, int len,
 	struct sk_buff *skb;
 	int skb_len = H2C_LEN_SCAN_LIST_OFFLOAD + len * RTW89_MAC_CHINFO_SIZE;
 	u8 *cmd;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, skb_len);
 	if (!skb) {
@@ -1693,7 +2089,8 @@ int rtw89_fw_h2c_scan_list_offload(struct rtw89_dev *rtwdev, int len,
 			      H2C_CAT_MAC, H2C_CL_MAC_FW_OFLD,
 			      H2C_FUNC_ADD_SCANOFLD_CH, 1, 1, skb_len);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1702,10 +2099,10 @@ int rtw89_fw_h2c_scan_list_offload(struct rtw89_dev *rtwdev, int len,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
-#define H2C_LEN_SCAN_OFFLOAD 20
+#define H2C_LEN_SCAN_OFFLOAD 28
 int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev,
 			      struct rtw89_scan_option *option,
 			      struct rtw89_vif *rtwvif)
@@ -1713,6 +2110,7 @@ int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev,
 	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
 	struct sk_buff *skb;
 	u8 *cmd;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_LEN_SCAN_OFFLOAD);
 	if (!skb) {
@@ -1736,6 +2134,8 @@ int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev,
 						       scan_info->op_pri_ch);
 		RTW89_SET_FWCMD_SCANOFLD_TARGET_CENTRAL_CH(cmd,
 							   scan_info->op_chan);
+		RTW89_SET_FWCMD_SCANOFLD_TARGET_CH_BAND(cmd,
+							scan_info->op_band);
 	}
 
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
@@ -1743,7 +2143,8 @@ int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev,
 			      H2C_FUNC_SCANOFLD, 1, 1,
 			      H2C_LEN_SCAN_OFFLOAD);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1752,7 +2153,7 @@ int rtw89_fw_h2c_scan_offload(struct rtw89_dev *rtwdev,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 int rtw89_fw_h2c_rf_reg(struct rtw89_dev *rtwdev,
@@ -1762,6 +2163,7 @@ int rtw89_fw_h2c_rf_reg(struct rtw89_dev *rtwdev,
 	struct sk_buff *skb;
 	u8 class = info->rf_path == RF_PATH_A ?
 		   H2C_CL_OUTSRC_RF_REG_A : H2C_CL_OUTSRC_RF_REG_B;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
 	if (!skb) {
@@ -1774,7 +2176,8 @@ int rtw89_fw_h2c_rf_reg(struct rtw89_dev *rtwdev,
 			      H2C_CAT_OUTSRC, class, page, 0, 0,
 			      len);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1783,14 +2186,16 @@ int rtw89_fw_h2c_rf_reg(struct rtw89_dev *rtwdev,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 int rtw89_fw_h2c_rf_ntfy_mcc(struct rtw89_dev *rtwdev)
 {
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	struct rtw89_mcc_info *mcc_info = &rtwdev->mcc;
 	struct rtw89_fw_h2c_rf_get_mccch *mccch;
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, sizeof(*mccch));
 	if (!skb) {
@@ -1804,15 +2209,16 @@ int rtw89_fw_h2c_rf_ntfy_mcc(struct rtw89_dev *rtwdev)
 	mccch->ch_1 = cpu_to_le32(mcc_info->ch[1]);
 	mccch->band_0 = cpu_to_le32(mcc_info->band[0]);
 	mccch->band_1 = cpu_to_le32(mcc_info->band[1]);
-	mccch->current_channel = cpu_to_le32(rtwdev->hal.current_channel);
-	mccch->current_band_type = cpu_to_le32(rtwdev->hal.current_band_type);
+	mccch->current_channel = cpu_to_le32(chan->channel);
+	mccch->current_band_type = cpu_to_le32(chan->band_type);
 
 	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
 			      H2C_CAT_OUTSRC, H2C_CL_OUTSRC_RF_FW_NOTIFY,
 			      H2C_FUNC_OUTSRC_RF_GET_MCCCH, 0, 0,
 			      sizeof(*mccch));
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1821,7 +2227,7 @@ int rtw89_fw_h2c_rf_ntfy_mcc(struct rtw89_dev *rtwdev)
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 EXPORT_SYMBOL(rtw89_fw_h2c_rf_ntfy_mcc);
 
@@ -1830,6 +2236,7 @@ int rtw89_fw_h2c_raw_with_hdr(struct rtw89_dev *rtwdev,
 			      bool rack, bool dack)
 {
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, len);
 	if (!skb) {
@@ -1842,7 +2249,8 @@ int rtw89_fw_h2c_raw_with_hdr(struct rtw89_dev *rtwdev,
 			      H2C_CAT_OUTSRC, h2c_class, h2c_func, rack, dack,
 			      len);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1851,12 +2259,13 @@ int rtw89_fw_h2c_raw_with_hdr(struct rtw89_dev *rtwdev,
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 int rtw89_fw_h2c_raw(struct rtw89_dev *rtwdev, const u8 *buf, u16 len)
 {
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_no_hdr(rtwdev, len);
 	if (!skb) {
@@ -1865,7 +2274,8 @@ int rtw89_fw_h2c_raw(struct rtw89_dev *rtwdev, const u8 *buf, u16 len)
 	}
 	skb_put_data(skb, buf, len);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -1874,7 +2284,7 @@ int rtw89_fw_h2c_raw(struct rtw89_dev *rtwdev, const u8 *buf, u16 len)
 fail:
 	dev_kfree_skb_any(skb);
 
-	return -EBUSY;
+	return ret;
 }
 
 void rtw89_fw_send_all_early_h2c(struct rtw89_dev *rtwdev)
@@ -2169,7 +2579,7 @@ static void rtw89_hw_scan_add_chan(struct rtw89_dev *rtwdev, int chan_type,
 
 	if (ssid_num) {
 		ch_info->num_pkt = ssid_num;
-		band = ch_info->ch_band;
+		band = rtw89_hw_to_nl80211_band(ch_info->ch_band);
 
 		list_for_each_entry(info, &scan_info->pkt_list[band], list) {
 			ch_info->probe_id = info->id;
@@ -2211,13 +2621,16 @@ static int rtw89_hw_scan_add_chan_list(struct rtw89_dev *rtwdev,
 	struct ieee80211_channel *channel;
 	struct list_head chan_list;
 	bool random_seq = req->flags & NL80211_SCAN_FLAG_RANDOM_SN;
-	int list_len = req->n_channels, off_chan_time = 0;
+	int list_len, off_chan_time = 0;
 	enum rtw89_chan_type type;
-	int ret = 0, i;
+	int ret = 0;
+	u32 idx;
 
 	INIT_LIST_HEAD(&chan_list);
-	for (i = 0; i < req->n_channels; i++) {
-		channel = req->channels[i];
+	for (idx = rtwdev->scan_info.last_chan_idx, list_len = 0;
+	     idx < req->n_channels && list_len < RTW89_SCAN_LIST_LIMIT;
+	     idx++, list_len++) {
+		channel = req->channels[idx];
 		ch_info = kzalloc(sizeof(*ch_info), GFP_KERNEL);
 		if (!ch_info) {
 			ret = -ENOMEM;
@@ -2226,7 +2639,7 @@ static int rtw89_hw_scan_add_chan_list(struct rtw89_dev *rtwdev,
 
 		ch_info->period = req->duration_mandatory ?
 				  req->duration : RTW89_CHANNEL_TIME;
-		ch_info->ch_band = channel->band;
+		ch_info->ch_band = rtw89_nl80211_to_hw_band(channel->band);
 		ch_info->central_ch = channel->hw_value;
 		ch_info->pri_ch = channel->hw_value;
 		ch_info->rand_seq_num = random_seq;
@@ -2258,6 +2671,7 @@ static int rtw89_hw_scan_add_chan_list(struct rtw89_dev *rtwdev,
 		list_add_tail(&ch_info->list, &chan_list);
 		off_chan_time += ch_info->period;
 	}
+	rtwdev->scan_info.last_chan_idx = idx;
 	ret = rtw89_fw_h2c_scan_list_offload(rtwdev, list_len, &chan_list);
 
 out:
@@ -2289,9 +2703,11 @@ void rtw89_hw_scan_start(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 {
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	struct cfg80211_scan_request *req = &scan_req->req;
+	u32 rx_fltr = rtwdev->hal.rx_fltr;
 	u8 mac_addr[ETH_ALEN];
 
 	rtwdev->scan_info.scanning_vif = vif;
+	rtwdev->scan_info.last_chan_idx = 0;
 	rtwvif->scan_ies = &scan_req->ies;
 	rtwvif->scan_req = req;
 	ieee80211_stop_queues(rtwdev->hw);
@@ -2303,13 +2719,13 @@ void rtw89_hw_scan_start(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 		ether_addr_copy(mac_addr, vif->addr);
 	rtw89_core_scan_start(rtwdev, rtwvif, mac_addr, true);
 
-	rtwdev->hal.rx_fltr &= ~B_AX_A_BCN_CHK_EN;
-	rtwdev->hal.rx_fltr &= ~B_AX_A_BC;
-	rtwdev->hal.rx_fltr &= ~B_AX_A_A1_MATCH;
+	rx_fltr &= ~B_AX_A_BCN_CHK_EN;
+	rx_fltr &= ~B_AX_A_BC;
+	rx_fltr &= ~B_AX_A_A1_MATCH;
 	rtw89_write32_mask(rtwdev,
 			   rtw89_mac_reg_by_idx(R_AX_RX_FLTR_OPT, RTW89_MAC_0),
 			   B_AX_RX_FLTR_CFG_MASK,
-			   rtwdev->hal.rx_fltr);
+			   rx_fltr);
 }
 
 void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
@@ -2323,9 +2739,6 @@ void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 	if (!vif)
 		return;
 
-	rtwdev->hal.rx_fltr |= B_AX_A_BCN_CHK_EN;
-	rtwdev->hal.rx_fltr |= B_AX_A_BC;
-	rtwdev->hal.rx_fltr |= B_AX_A_A1_MATCH;
 	rtw89_write32_mask(rtwdev,
 			   rtw89_mac_reg_by_idx(R_AX_RX_FLTR_OPT, RTW89_MAC_0),
 			   B_AX_RX_FLTR_CFG_MASK,
@@ -2339,6 +2752,7 @@ void rtw89_hw_scan_complete(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 	rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	rtwvif->scan_req = NULL;
 	rtwvif->scan_ies = NULL;
+	rtwdev->scan_info.last_chan_idx = 0;
 	rtwdev->scan_info.scanning_vif = NULL;
 
 	if (rtwvif->net_type != RTW89_NET_TYPE_NO_LINK)
@@ -2377,18 +2791,18 @@ out:
 void rtw89_store_op_chan(struct rtw89_dev *rtwdev, bool backup)
 {
 	struct rtw89_hw_scan_info *scan_info = &rtwdev->scan_info;
-	struct rtw89_hal *hal = &rtwdev->hal;
+	const struct rtw89_chan *cur = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	struct rtw89_chan new;
 
 	if (backup) {
-		scan_info->op_pri_ch = hal->current_primary_channel;
-		scan_info->op_chan = hal->current_channel;
-		scan_info->op_bw = hal->current_band_width;
-		scan_info->op_band = hal->current_band_type;
+		scan_info->op_pri_ch = cur->primary_channel;
+		scan_info->op_chan = cur->channel;
+		scan_info->op_bw = cur->band_width;
+		scan_info->op_band = cur->band_type;
 	} else {
-		hal->current_primary_channel = scan_info->op_pri_ch;
-		hal->current_channel = scan_info->op_chan;
-		hal->current_band_width = scan_info->op_bw;
-		hal->current_band_type = scan_info->op_band;
+		rtw89_chan_create(&new, scan_info->op_chan, scan_info->op_pri_ch,
+				  scan_info->op_band, scan_info->op_bw);
+		rtw89_assign_entity_chan(rtwdev, RTW89_SUB_ENTITY_0, &new);
 	}
 }
 
@@ -2397,6 +2811,7 @@ void rtw89_store_op_chan(struct rtw89_dev *rtwdev, bool backup)
 int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev)
 {
 	struct sk_buff *skb;
+	int ret;
 
 	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_FW_CPU_EXCEPTION_LEN);
 	if (!skb) {
@@ -2415,7 +2830,62 @@ int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev)
 			      H2C_FUNC_CPU_EXCEPTION, 0, 0,
 			      H2C_FW_CPU_EXCEPTION_LEN);
 
-	if (rtw89_h2c_tx(rtwdev, skb, false)) {
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
+		rtw89_err(rtwdev, "failed to send h2c\n");
+		goto fail;
+	}
+
+	return 0;
+
+fail:
+	dev_kfree_skb_any(skb);
+	return ret;
+}
+
+#define H2C_PKT_DROP_LEN 24
+int rtw89_fw_h2c_pkt_drop(struct rtw89_dev *rtwdev,
+			  const struct rtw89_pkt_drop_params *params)
+{
+	struct sk_buff *skb;
+	int ret;
+
+	skb = rtw89_fw_h2c_alloc_skb_with_hdr(rtwdev, H2C_PKT_DROP_LEN);
+	if (!skb) {
+		rtw89_err(rtwdev,
+			  "failed to alloc skb for packet drop\n");
+		return -ENOMEM;
+	}
+
+	switch (params->sel) {
+	case RTW89_PKT_DROP_SEL_MACID_BE_ONCE:
+	case RTW89_PKT_DROP_SEL_MACID_BK_ONCE:
+	case RTW89_PKT_DROP_SEL_MACID_VI_ONCE:
+	case RTW89_PKT_DROP_SEL_MACID_VO_ONCE:
+		break;
+	default:
+		rtw89_debug(rtwdev, RTW89_DBG_FW,
+			    "H2C of pkt drop might not fully support sel: %d yet\n",
+			    params->sel);
+		break;
+	}
+
+	skb_put(skb, H2C_PKT_DROP_LEN);
+	RTW89_SET_FWCMD_PKT_DROP_SEL(skb->data, params->sel);
+	RTW89_SET_FWCMD_PKT_DROP_MACID(skb->data, params->macid);
+	RTW89_SET_FWCMD_PKT_DROP_BAND(skb->data, params->mac_band);
+	RTW89_SET_FWCMD_PKT_DROP_PORT(skb->data, params->port);
+	RTW89_SET_FWCMD_PKT_DROP_MBSSID(skb->data, params->mbssid);
+	RTW89_SET_FWCMD_PKT_DROP_ROLE_A_INFO_TF_TRS(skb->data, params->tf_trs);
+
+	rtw89_h2c_pkt_set_hdr(rtwdev, skb, FWCMD_TYPE_H2C,
+			      H2C_CAT_MAC,
+			      H2C_CL_MAC_FW_OFLD,
+			      H2C_FUNC_PKT_DROP, 0, 0,
+			      H2C_PKT_DROP_LEN);
+
+	ret = rtw89_h2c_tx(rtwdev, skb, false);
+	if (ret) {
 		rtw89_err(rtwdev, "failed to send h2c\n");
 		goto fail;
 	}
@@ -2424,5 +2894,5 @@ int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev)
 
 fail:
 	dev_kfree_skb_any(skb);
-	return -EBUSY;
+	return ret;
 }
diff --git a/drivers/net/wireless/realtek/rtw89/fw.h b/drivers/net/wireless/realtek/rtw89/fw.h
index e75ad22aa85d..0047d5d0e9b1 100644
--- a/drivers/net/wireless/realtek/rtw89/fw.h
+++ b/drivers/net/wireless/realtek/rtw89/fw.h
@@ -63,21 +63,32 @@ enum rtw89_mac_c2h_type {
 	RTW89_FWCMD_C2HREG_FUNC_NULL = 0xFF
 };
 
-struct rtw89_c2h_phy_cap {
-	u32 func:7;
-	u32 ack:1;
-	u32 len:4;
-	u32 seq:4;
-	u32 rx_nss:8;
-	u32 bw:8;
-
-	u32 tx_nss:8;
-	u32 prot:8;
-	u32 nic:8;
-	u32 wl_func:8;
-
-	u32 hw_type:8;
-} __packed;
+#define RTW89_GET_C2H_PHYCAP_FUNC(info) \
+	u32_get_bits(*((const u32 *)(info)), GENMASK(6, 0))
+#define RTW89_GET_C2H_PHYCAP_ACK(info) \
+	u32_get_bits(*((const u32 *)(info)), BIT(7))
+#define RTW89_GET_C2H_PHYCAP_LEN(info) \
+	u32_get_bits(*((const u32 *)(info)), GENMASK(11, 8))
+#define RTW89_GET_C2H_PHYCAP_SEQ(info) \
+	u32_get_bits(*((const u32 *)(info)), GENMASK(15, 12))
+#define RTW89_GET_C2H_PHYCAP_RX_NSS(info) \
+	u32_get_bits(*((const u32 *)(info)), GENMASK(23, 16))
+#define RTW89_GET_C2H_PHYCAP_BW(info) \
+	u32_get_bits(*((const u32 *)(info)), GENMASK(31, 24))
+#define RTW89_GET_C2H_PHYCAP_TX_NSS(info) \
+	u32_get_bits(*((const u32 *)(info) + 1), GENMASK(7, 0))
+#define RTW89_GET_C2H_PHYCAP_PROT(info) \
+	u32_get_bits(*((const u32 *)(info) + 1), GENMASK(15, 8))
+#define RTW89_GET_C2H_PHYCAP_NIC(info) \
+	u32_get_bits(*((const u32 *)(info) + 1), GENMASK(23, 16))
+#define RTW89_GET_C2H_PHYCAP_WL_FUNC(info) \
+	u32_get_bits(*((const u32 *)(info) + 1), GENMASK(31, 24))
+#define RTW89_GET_C2H_PHYCAP_HW_TYPE(info) \
+	u32_get_bits(*((const u32 *)(info) + 2), GENMASK(7, 0))
+#define RTW89_GET_C2H_PHYCAP_ANT_TX_NUM(info) \
+	u32_get_bits(*((const u32 *)(info) + 3), GENMASK(15, 8))
+#define RTW89_GET_C2H_PHYCAP_ANT_RX_NUM(info) \
+	u32_get_bits(*((const u32 *)(info) + 3), GENMASK(23, 16))
 
 enum rtw89_fw_c2h_category {
 	RTW89_C2H_CAT_TEST,
@@ -144,6 +155,13 @@ enum rtw89_chan_type {
 	RTW89_CHAN_DFS,
 };
 
+enum rtw89_p2pps_action {
+	RTW89_P2P_ACT_INIT = 0,
+	RTW89_P2P_ACT_UPDATE = 1,
+	RTW89_P2P_ACT_REMOVE = 2,
+	RTW89_P2P_ACT_TERMINATE = 3,
+};
+
 #define FWDL_SECTION_MAX_NUM 10
 #define FWDL_SECTION_CHKSUM_LEN	8
 #define FWDL_SECTION_PER_PKT_LEN 2020
@@ -177,6 +195,7 @@ struct rtw89_h2creg_sch_tx_en {
 	u16 rsvd:15;
 } __packed;
 
+#define RTW89_H2C_MAX_SIZE 2048
 #define RTW89_CHANNEL_TIME 45
 #define RTW89_DFS_CHAN_TIME 105
 #define RTW89_OFF_CHAN_TIME 100
@@ -186,7 +205,10 @@ struct rtw89_h2creg_sch_tx_en {
 #define RTW89_SCANOFLD_MAX_IE_LEN 512
 #define RTW89_SCANOFLD_PKT_NONE 0xFF
 #define RTW89_SCANOFLD_DEBUG_MASK 0x1F
-#define RTW89_MAC_CHINFO_SIZE 20
+#define RTW89_MAC_CHINFO_SIZE 24
+#define RTW89_SCAN_LIST_GUARD 4
+#define RTW89_SCAN_LIST_LIMIT \
+		((RTW89_H2C_MAX_SIZE / RTW89_MAC_CHINFO_SIZE) - RTW89_SCAN_LIST_GUARD)
 
 struct rtw89_mac_chinfo {
 	u8 period;
@@ -346,6 +368,16 @@ static inline void RTW89_SET_FWCMD_RA_CR_TBL_SEL(void *cmd, u32 val)
 	le32p_replace_bits((__le32 *)(cmd) + 0x03, val, BIT(10));
 }
 
+static inline void RTW89_SET_FWCMD_RA_FIX_GILTF_EN(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)(cmd) + 0x03, val, BIT(11));
+}
+
+static inline void RTW89_SET_FWCMD_RA_FIX_GILTF(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)(cmd) + 0x03, val, GENMASK(14, 12));
+}
+
 static inline void RTW89_SET_FWCMD_RA_FIXED_CSI_MCS_SS_IDX(void *cmd, u32 val)
 {
 	le32p_replace_bits((__le32 *)(cmd) + 0x03, val, GENMASK(23, 16));
@@ -1798,6 +1830,36 @@ static inline void RTW89_SET_FWCMD_CPU_EXCEPTION_TYPE(void *cmd, u32 val)
 	le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 0));
 }
 
+static inline void RTW89_SET_FWCMD_PKT_DROP_SEL(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_PKT_DROP_MACID(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, GENMASK(15, 8));
+}
+
+static inline void RTW89_SET_FWCMD_PKT_DROP_BAND(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, GENMASK(23, 16));
+}
+
+static inline void RTW89_SET_FWCMD_PKT_DROP_PORT(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 24));
+}
+
+static inline void RTW89_SET_FWCMD_PKT_DROP_MBSSID(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_PKT_DROP_ROLE_A_INFO_TF_TRS(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd + 1, val, GENMASK(15, 8));
+}
+
 enum rtw89_btc_btf_h2c_class {
 	BTFC_SET = 0x10,
 	BTFC_GET = 0x11,
@@ -2006,69 +2068,104 @@ static inline void RTW89_SET_FWCMD_CXROLE_ROLE_NAN(void *cmd, u16 val)
 	le16p_replace_bits((__le16 *)((u8 *)(cmd) + 4), val, BIT(11));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED(void *cmd, u8 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_CONNECTED(void *cmd, u8 val, int n, u8 offset)
 {
-	u8p_replace_bits((u8 *)(cmd) + (6 + 12 * (n)), val, BIT(0));
+	u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, BIT(0));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_PID(void *cmd, u8 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_PID(void *cmd, u8 val, int n, u8 offset)
 {
-	u8p_replace_bits((u8 *)(cmd) + (6 + 12 * (n)), val, GENMASK(3, 1));
+	u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, GENMASK(3, 1));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_PHY(void *cmd, u8 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_PHY(void *cmd, u8 val, int n, u8 offset)
 {
-	u8p_replace_bits((u8 *)(cmd) + (6 + 12 * (n)), val, BIT(4));
+	u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, BIT(4));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_NOA(void *cmd, u8 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_NOA(void *cmd, u8 val, int n, u8 offset)
 {
-	u8p_replace_bits((u8 *)(cmd) + (6 + 12 * (n)), val, BIT(5));
+	u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, BIT(5));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_BAND(void *cmd, u8 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_BAND(void *cmd, u8 val, int n, u8 offset)
 {
-	u8p_replace_bits((u8 *)(cmd) + (6 + 12 * (n)), val, GENMASK(7, 6));
+	u8p_replace_bits((u8 *)cmd + (6 + (12 + offset) * n), val, GENMASK(7, 6));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS(void *cmd, u8 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_CLIENT_PS(void *cmd, u8 val, int n, u8 offset)
 {
-	u8p_replace_bits((u8 *)(cmd) + (7 + 12 * (n)), val, BIT(0));
+	u8p_replace_bits((u8 *)cmd + (7 + (12 + offset) * n), val, BIT(0));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_BW(void *cmd, u8 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_BW(void *cmd, u8 val, int n, u8 offset)
 {
-	u8p_replace_bits((u8 *)(cmd) + (7 + 12 * (n)), val, GENMASK(7, 1));
+	u8p_replace_bits((u8 *)cmd + (7 + (12 + offset) * n), val, GENMASK(7, 1));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_ROLE(void *cmd, u8 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_ROLE(void *cmd, u8 val, int n, u8 offset)
 {
-	u8p_replace_bits((u8 *)(cmd) + (8 + 12 * (n)), val, GENMASK(7, 0));
+	u8p_replace_bits((u8 *)cmd + (8 + (12 + offset) * n), val, GENMASK(7, 0));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_CH(void *cmd, u8 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_CH(void *cmd, u8 val, int n, u8 offset)
 {
-	u8p_replace_bits((u8 *)(cmd) + (9 + 12 * (n)), val, GENMASK(7, 0));
+	u8p_replace_bits((u8 *)cmd + (9 + (12 + offset) * n), val, GENMASK(7, 0));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_TX_LVL(void *cmd, u16 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_TX_LVL(void *cmd, u16 val, int n, u8 offset)
 {
-	le16p_replace_bits((__le16 *)((u8 *)(cmd) + (10 + 12 * (n))), val, GENMASK(15, 0));
+	le16p_replace_bits((__le16 *)((u8 *)cmd + (10 + (12 + offset) * n)), val, GENMASK(15, 0));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_RX_LVL(void *cmd, u16 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_RX_LVL(void *cmd, u16 val, int n, u8 offset)
 {
-	le16p_replace_bits((__le16 *)((u8 *)(cmd) + (12 + 12 * (n))), val, GENMASK(15, 0));
+	le16p_replace_bits((__le16 *)((u8 *)cmd + (12 + (12 + offset) * n)), val, GENMASK(15, 0));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_TX_RATE(void *cmd, u16 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_TX_RATE(void *cmd, u16 val, int n, u8 offset)
 {
-	le16p_replace_bits((__le16 *)((u8 *)(cmd) + (14 + 12 * (n))), val, GENMASK(15, 0));
+	le16p_replace_bits((__le16 *)((u8 *)cmd + (14 + (12 + offset) * n)), val, GENMASK(15, 0));
 }
 
-static inline void RTW89_SET_FWCMD_CXROLE_ACT_RX_RATE(void *cmd, u16 val, int n)
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_RX_RATE(void *cmd, u16 val, int n, u8 offset)
 {
-	le16p_replace_bits((__le16 *)((u8 *)(cmd) + (16 + 12 * (n))), val, GENMASK(15, 0));
+	le16p_replace_bits((__le16 *)((u8 *)cmd + (16 + (12 + offset) * n)), val, GENMASK(15, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_ACT_NOA_DUR(void *cmd, u32 val, int n, u8 offset)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + (20 + (12 + offset) * n)), val, GENMASK(31, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_MROLE_TYPE(void *cmd, u32 val, u8 offset)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + offset), val, GENMASK(31, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_MROLE_NOA(void *cmd, u32 val, u8 offset)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + offset + 4), val, GENMASK(31, 0));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_DBCC_EN(void *cmd, u32 val, u8 offset)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + offset + 8), val, BIT(0));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_DBCC_CHG(void *cmd, u32 val, u8 offset)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + offset + 8), val, BIT(1));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_DBCC_2G_PHY(void *cmd, u32 val, u8 offset)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + offset + 8), val, GENMASK(3, 2));
+}
+
+static inline void RTW89_SET_FWCMD_CXROLE_LINK_MODE_CHG(void *cmd, u32 val, u8 offset)
+{
+	le32p_replace_bits((__le32 *)((u8 *)cmd + offset + 8), val, BIT(4));
 }
 
 static inline void RTW89_SET_FWCMD_CXCTRL_MANUAL(void *cmd, u32 val)
@@ -2352,6 +2449,86 @@ static inline void RTW89_SET_FWCMD_SCANOFLD_TSF_SLOW(void *cmd, u32 val)
 	le32p_replace_bits((__le32 *)((u8 *)(cmd) + 16), val, GENMASK(31, 0));
 }
 
+static inline void RTW89_SET_FWCMD_P2P_MACID(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_P2P_P2PID(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, GENMASK(11, 8));
+}
+
+static inline void RTW89_SET_FWCMD_P2P_NOAID(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, GENMASK(15, 12));
+}
+
+static inline void RTW89_SET_FWCMD_P2P_ACT(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, GENMASK(19, 16));
+}
+
+static inline void RTW89_SET_FWCMD_P2P_TYPE(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, BIT(20));
+}
+
+static inline void RTW89_SET_FWCMD_P2P_ALL_SLEP(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, BIT(21));
+}
+
+static inline void RTW89_SET_FWCMD_NOA_START_TIME(void *cmd, __le32 val)
+{
+	*((__le32 *)cmd + 1) = val;
+}
+
+static inline void RTW89_SET_FWCMD_NOA_INTERVAL(void *cmd, __le32 val)
+{
+	*((__le32 *)cmd + 2) = val;
+}
+
+static inline void RTW89_SET_FWCMD_NOA_DURATION(void *cmd, __le32 val)
+{
+	*((__le32 *)cmd + 3) = val;
+}
+
+static inline void RTW89_SET_FWCMD_NOA_COUNT(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)(cmd) + 4, val, GENMASK(7, 0));
+}
+
+static inline void RTW89_SET_FWCMD_NOA_CTWINDOW(void *cmd, u32 val)
+{
+	u8 ctwnd;
+
+	if (!(val & IEEE80211_P2P_OPPPS_ENABLE_BIT))
+		return;
+	ctwnd = FIELD_GET(IEEE80211_P2P_OPPPS_CTWINDOW_MASK, val);
+	le32p_replace_bits((__le32 *)(cmd) + 4, ctwnd, GENMASK(23, 8));
+}
+
+static inline void RTW89_SET_FWCMD_TSF32_TOGL_BAND(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, BIT(0));
+}
+
+static inline void RTW89_SET_FWCMD_TSF32_TOGL_EN(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, BIT(1));
+}
+
+static inline void RTW89_SET_FWCMD_TSF32_TOGL_PORT(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, GENMASK(4, 2));
+}
+
+static inline void RTW89_SET_FWCMD_TSF32_TOGL_EARLY(void *cmd, u32 val)
+{
+	le32p_replace_bits((__le32 *)cmd, val, GENMASK(31, 16));
+}
+
 #define RTW89_C2H_HEADER_LEN 8
 
 #define RTW89_GET_C2H_CATEGORY(c2h) \
@@ -2421,6 +2598,8 @@ static inline void RTW89_SET_FWCMD_SCANOFLD_TSF_SLOW(void *cmd, u32 val)
 	le32_get_bits(*((const __le32 *)(c2h) + 2), GENMASK(19, 16))
 #define RTW89_GET_MAC_C2H_SCANOFLD_STATUS(c2h) \
 	le32_get_bits(*((const __le32 *)(c2h) + 2), GENMASK(23, 20))
+#define RTW89_GET_MAC_C2H_ACTUAL_PERIOD(c2h) \
+	le32_get_bits(*((const __le32 *)(c2h) + 2), GENMASK(31, 24))
 #define RTW89_GET_MAC_C2H_SCANOFLD_TX_FAIL(c2h) \
 	le32_get_bits(*((const __le32 *)(c2h) + 5), GENMASK(3, 0))
 #define RTW89_GET_MAC_C2H_SCANOFLD_AIR_DENSITY(c2h) \
@@ -2446,7 +2625,14 @@ struct rtw89_mfw_info {
 struct rtw89_mfw_hdr {
 	u8 sig;	/* RTW89_MFW_SIG */
 	u8 fw_nr;
-	u8 rsvd[14];
+	u8 rsvd0[2];
+	struct {
+		u8 major;
+		u8 minor;
+		u8 sub;
+		u8 idx;
+	} ver;
+	u8 rsvd1[8];
 	struct rtw89_mfw_info info[];
 } __packed;
 
@@ -2493,6 +2679,7 @@ struct rtw89_fw_h2c_rf_reg_info {
 /* CLASS 2 - PS */
 #define H2C_CL_MAC_PS			0x2
 #define H2C_FUNC_MAC_LPS_PARM		0x0
+#define H2C_FUNC_P2P_ACT		0x1
 
 /* CLASS 3 - FW download */
 #define H2C_CL_MAC_FWDL		0x3
@@ -2519,9 +2706,11 @@ struct rtw89_fw_h2c_rf_reg_info {
 #define H2C_FUNC_PACKET_OFLD		0x1
 #define H2C_FUNC_MAC_MACID_PAUSE	0x8
 #define H2C_FUNC_USR_EDCA		0xF
+#define H2C_FUNC_TSF32_TOGL		0x10
 #define H2C_FUNC_OFLD_CFG		0x14
 #define H2C_FUNC_ADD_SCANOFLD_CH	0x16
 #define H2C_FUNC_SCANOFLD		0x17
+#define H2C_FUNC_PKT_DROP		0x1b
 
 /* CLASS 10 - Security CAM */
 #define H2C_CL_MAC_SEC_CAM		0xa
@@ -2552,7 +2741,7 @@ struct rtw89_fw_h2c_rf_get_mccch {
 
 #define RTW89_FW_RSVD_PLE_SIZE 0x800
 
-#define RTW89_WCPU_BASE_ADDR 0xA0000000
+#define RTW89_WCPU_BASE_MASK GENMASK(27, 0)
 
 #define RTW89_FW_BACKTRACE_INFO_SIZE 8
 #define RTW89_VALID_FW_BACKTRACE_SIZE(_size) \
@@ -2563,6 +2752,9 @@ struct rtw89_fw_h2c_rf_get_mccch {
 
 int rtw89_fw_check_rdy(struct rtw89_dev *rtwdev);
 int rtw89_fw_recognize(struct rtw89_dev *rtwdev);
+void rtw89_early_fw_feature_recognize(struct device *device,
+				      const struct rtw89_chip_info *chip,
+				      u32 *early_feat_map);
 int rtw89_fw_download(struct rtw89_dev *rtwdev, enum rtw89_fw_type type);
 int rtw89_load_firmware(struct rtw89_dev *rtwdev);
 void rtw89_unload_firmware(struct rtw89_dev *rtwdev);
@@ -2577,6 +2769,8 @@ int rtw89_fw_h2c_assoc_cmac_tbl(struct rtw89_dev *rtwdev,
 				struct ieee80211_sta *sta);
 int rtw89_fw_h2c_txtime_cmac_tbl(struct rtw89_dev *rtwdev,
 				 struct rtw89_sta *rtwsta);
+int rtw89_fw_h2c_txpath_cmac_tbl(struct rtw89_dev *rtwdev,
+				 struct rtw89_sta *rtwsta);
 int rtw89_fw_h2c_update_beacon(struct rtw89_dev *rtwdev,
 			       struct rtw89_vif *rtwvif);
 int rtw89_fw_h2c_cam(struct rtw89_dev *rtwdev, struct rtw89_vif *vif,
@@ -2600,6 +2794,7 @@ int rtw89_fw_h2c_set_ofld_cfg(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_ra(struct rtw89_dev *rtwdev, struct rtw89_ra_info *ra, bool csi);
 int rtw89_fw_h2c_cxdrv_init(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_cxdrv_role(struct rtw89_dev *rtwdev);
+int rtw89_fw_h2c_cxdrv_role_v1(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_cxdrv_ctrl(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_cxdrv_rfk(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_del_pkt_offload(struct rtw89_dev *rtwdev, u8 id);
@@ -2623,6 +2818,7 @@ void rtw89_fw_free_all_early_h2c(struct rtw89_dev *rtwdev);
 int rtw89_fw_h2c_general_pkt(struct rtw89_dev *rtwdev, u8 macid);
 int rtw89_fw_h2c_ba_cam(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta,
 			bool valid, struct ieee80211_ampdu_params *params);
+void rtw89_fw_h2c_init_ba_cam_v1(struct rtw89_dev *rtwdev);
 
 int rtw89_fw_h2c_lps_parm(struct rtw89_dev *rtwdev,
 			  struct rtw89_lps_parm *lps_param);
@@ -2642,5 +2838,20 @@ int rtw89_hw_scan_offload(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
 			  bool enable);
 void rtw89_hw_scan_abort(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif);
 int rtw89_fw_h2c_trigger_cpu_exception(struct rtw89_dev *rtwdev);
+int rtw89_fw_h2c_pkt_drop(struct rtw89_dev *rtwdev,
+			  const struct rtw89_pkt_drop_params *params);
+int rtw89_fw_h2c_p2p_act(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif,
+			 struct ieee80211_p2p_noa_desc *desc,
+			 u8 act, u8 noa_id);
+int rtw89_fw_h2c_tsf32_toggle(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
+			      bool en);
+
+static inline void rtw89_fw_h2c_init_ba_cam(struct rtw89_dev *rtwdev)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	if (chip->bacam_v1)
+		rtw89_fw_h2c_init_ba_cam_v1(rtwdev);
+}
 
 #endif
diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c
index 93124b815825..0508dfca8edf 100644
--- a/drivers/net/wireless/realtek/rtw89/mac.c
+++ b/drivers/net/wireless/realtek/rtw89/mac.c
@@ -3,6 +3,7 @@
  */
 
 #include "cam.h"
+#include "chan.h"
 #include "debug.h"
 #include "fw.h"
 #include "mac.h"
@@ -826,6 +827,8 @@ static void hfc_func_en(struct rtw89_dev *rtwdev, bool en, bool h2c_en)
 
 static int hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	u32 dma_ch_mask = chip->dma_ch_mask;
 	u8 ch;
 	u32 ret = 0;
 
@@ -847,6 +850,8 @@ static int hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en)
 	}
 
 	for (ch = RTW89_DMA_ACH0; ch < RTW89_DMA_H2C; ch++) {
+		if (dma_ch_mask & BIT(ch))
+			continue;
 		ret = hfc_ch_ctrl(rtwdev, ch);
 		if (ret)
 			return ret;
@@ -862,6 +867,8 @@ static int hfc_init(struct rtw89_dev *rtwdev, bool reset, bool en, bool h2c_en)
 		udelay(10);
 	}
 	for (ch = RTW89_DMA_ACH0; ch < RTW89_DMA_H2C; ch++) {
+		if (dma_ch_mask & BIT(ch))
+			continue;
 		ret = hfc_upd_ch_info(rtwdev, ch);
 		if (ret)
 			return ret;
@@ -1053,18 +1060,29 @@ void rtw89_mac_power_mode_change(struct rtw89_dev *rtwdev, bool enter)
 	enum rtw89_rpwm_req_pwr_state state;
 	unsigned long delay = enter ? 10 : 150;
 	int ret;
+	int i;
 
 	if (enter)
 		state = rtw89_mac_get_req_pwr_state(rtwdev);
 	else
 		state = RTW89_MAC_RPWM_REQ_PWR_STATE_ACTIVE;
 
-	rtw89_mac_send_rpwm(rtwdev, state, false);
-	ret = read_poll_timeout_atomic(rtw89_mac_check_cpwm_state, ret, !ret,
-				       delay, 15000, false, rtwdev, state);
-	if (ret)
-		rtw89_err(rtwdev, "firmware failed to ack for %s ps mode\n",
-			  enter ? "entering" : "leaving");
+	for (i = 0; i < RPWM_TRY_CNT; i++) {
+		rtw89_mac_send_rpwm(rtwdev, state, false);
+		ret = read_poll_timeout_atomic(rtw89_mac_check_cpwm_state, ret,
+					       !ret, delay, 15000, false,
+					       rtwdev, state);
+		if (!ret)
+			break;
+
+		if (i == RPWM_TRY_CNT - 1)
+			rtw89_err(rtwdev, "firmware failed to ack for %s ps mode\n",
+				  enter ? "entering" : "leaving");
+		else
+			rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
+				    "%d time firmware failed to ack for %s ps mode\n",
+				    i + 1, enter ? "entering" : "leaving");
+	}
 }
 
 void rtw89_mac_notify_wake(struct rtw89_dev *rtwdev)
@@ -1081,7 +1099,6 @@ static int rtw89_mac_power_switch(struct rtw89_dev *rtwdev, bool on)
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	const struct rtw89_pwr_cfg * const *cfg_seq;
 	int (*cfg_func)(struct rtw89_dev *rtwdev);
-	struct rtw89_hal *hal = &rtwdev->hal;
 	int ret;
 	u8 val;
 
@@ -1113,7 +1130,7 @@ static int rtw89_mac_power_switch(struct rtw89_dev *rtwdev, bool on)
 		clear_bit(RTW89_FLAG_POWERON, rtwdev->flags);
 		clear_bit(RTW89_FLAG_FW_RDY, rtwdev->flags);
 		rtw89_write8(rtwdev, R_AX_SCOREBOARD + 3, MAC_AX_NOTIFY_PWR_MAJOR);
-		hal->current_channel = 0;
+		rtw89_set_entity_state(rtwdev, false);
 	}
 
 	return 0;
@@ -1207,8 +1224,8 @@ static int chip_func_en(struct rtw89_dev *rtwdev)
 {
 	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 
-	if (chip_id == RTL8852A)
-		rtw89_write32_set(rtwdev, R_AX_SPSLDO_ON_CTRL0,
+	if (chip_id == RTL8852A || chip_id == RTL8852B)
+		rtw89_write32_set(rtwdev, R_AX_SPS_DIG_ON_CTRL0,
 				  B_AX_OCP_L1_MASK);
 
 	return 0;
@@ -1239,6 +1256,10 @@ const struct rtw89_mac_size_set rtw89_mac_size = {
 	.wde_size0 = {RTW89_WDE_PG_64, 4095, 1,},
 	/* DLFW */
 	.wde_size4 = {RTW89_WDE_PG_64, 0, 4096,},
+	/* PCIE 64 */
+	.wde_size6 = {RTW89_WDE_PG_64, 512, 0,},
+	/* DLFW */
+	.wde_size9 = {RTW89_WDE_PG_64, 0, 1024,},
 	/* 8852C DLFW */
 	.wde_size18 = {RTW89_WDE_PG_64, 0, 2048,},
 	/* 8852C PCIE SCC */
@@ -1247,6 +1268,10 @@ const struct rtw89_mac_size_set rtw89_mac_size = {
 	.ple_size0 = {RTW89_PLE_PG_128, 1520, 16,},
 	/* DLFW */
 	.ple_size4 = {RTW89_PLE_PG_128, 64, 1472,},
+	/* PCIE 64 */
+	.ple_size6 = {RTW89_PLE_PG_128, 496, 16,},
+	/* DLFW */
+	.ple_size8 = {RTW89_PLE_PG_128, 64, 960,},
 	/* 8852C DLFW */
 	.ple_size18 = {RTW89_PLE_PG_128, 2544, 16,},
 	/* 8852C PCIE SCC */
@@ -1255,6 +1280,8 @@ const struct rtw89_mac_size_set rtw89_mac_size = {
 	.wde_qt0 = {3792, 196, 0, 107,},
 	/* DLFW */
 	.wde_qt4 = {0, 0, 0, 0,},
+	/* PCIE 64 */
+	.wde_qt6 = {448, 48, 0, 16,},
 	/* 8852C DLFW */
 	.wde_qt17 = {0, 0, 0,  0,},
 	/* 8852C PCIE SCC */
@@ -1265,6 +1292,8 @@ const struct rtw89_mac_size_set rtw89_mac_size = {
 	.ple_qt5 = {264, 0, 32, 20, 64, 13, 1101, 0, 64, 128, 120,},
 	/* DLFW */
 	.ple_qt13 = {0, 0, 16, 48, 0, 0, 0, 0, 0, 0, 0,},
+	/* PCIE 64 */
+	.ple_qt18 = {147, 0, 16, 20, 17, 13, 89, 0, 32, 14, 8, 0,},
 	/* DLFW 52C */
 	.ple_qt44 = {0, 0, 16, 256, 0, 0, 0, 0, 0, 0, 0, 0,},
 	/* DLFW 52C */
@@ -1273,6 +1302,8 @@ const struct rtw89_mac_size_set rtw89_mac_size = {
 	.ple_qt46 = {525, 0, 16, 20, 13, 13, 178, 0, 32, 62, 8, 16,},
 	/* 8852C PCIE SCC */
 	.ple_qt47 = {525, 0, 32, 20, 1034, 13, 1199, 0, 1053, 62, 160, 1037,},
+	/* PCIE 64 */
+	.ple_qt58 = {147, 0, 16, 20, 157, 13, 229, 0, 172, 14, 24, 0,},
 };
 EXPORT_SYMBOL(rtw89_mac_size);
 
@@ -1307,6 +1338,17 @@ static inline u32 dle_used_size(const struct rtw89_dle_size *wde,
 	       ple->pge_size * (ple->lnk_pge_num + ple->unlnk_pge_num);
 }
 
+static u32 dle_expected_used_size(struct rtw89_dev *rtwdev,
+				  enum rtw89_qta_mode mode)
+{
+	u32 size = rtwdev->chip->fifo_size;
+
+	if (mode == RTW89_QTA_SCC)
+		size -= rtwdev->chip->dle_scc_rsvd_size;
+
+	return size;
+}
+
 static void dle_func_en(struct rtw89_dev *rtwdev, bool enable)
 {
 	if (enable)
@@ -1474,7 +1516,8 @@ static int dle_init(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode,
 		ext_wde_min_qt_wcpu = ext_cfg->wde_min_qt->wcpu;
 	}
 
-	if (dle_used_size(cfg->wde_size, cfg->ple_size) != rtwdev->chip->fifo_size) {
+	if (dle_used_size(cfg->wde_size, cfg->ple_size) !=
+	    dle_expected_used_size(rtwdev, mode)) {
 		rtw89_err(rtwdev, "[ERR]wd/dle mem cfg\n");
 		ret = -EINVAL;
 		goto error;
@@ -1734,7 +1777,7 @@ static int addr_cam_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 	rtw89_write32(rtwdev, reg, val);
 
 	ret = read_poll_timeout(rtw89_read16, p_val, !(p_val & B_AX_ADDR_CAM_CLR),
-				1, TRXCFG_WAIT_CNT, false, rtwdev, B_AX_ADDR_CAM_CLR);
+				1, TRXCFG_WAIT_CNT, false, rtwdev, reg);
 	if (ret) {
 		rtw89_err(rtwdev, "[ERR]ADDR_CAM reset\n");
 		return ret;
@@ -1747,13 +1790,19 @@ static int scheduler_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 {
 	u32 ret;
 	u32 reg;
+	u32 val;
 
 	ret = rtw89_mac_check_mac_en(rtwdev, mac_idx, RTW89_CMAC_SEL);
 	if (ret)
 		return ret;
 
 	reg = rtw89_mac_reg_by_idx(R_AX_PREBKF_CFG_1, mac_idx);
-	rtw89_write32_mask(rtwdev, reg, B_AX_SIFS_MACTXEN_T1_MASK, SIFS_MACTXEN_T1);
+	if (rtwdev->chip->chip_id == RTL8852C)
+		rtw89_write32_mask(rtwdev, reg, B_AX_SIFS_MACTXEN_T1_MASK,
+				   SIFS_MACTXEN_T1_V1);
+	else
+		rtw89_write32_mask(rtwdev, reg, B_AX_SIFS_MACTXEN_T1_MASK,
+				   SIFS_MACTXEN_T1);
 
 	if (rtwdev->chip->chip_id == RTL8852B) {
 		reg = rtw89_mac_reg_by_idx(R_AX_SCH_EXT_CTRL, mac_idx);
@@ -1764,7 +1813,16 @@ static int scheduler_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 	rtw89_write32_clr(rtwdev, reg, B_AX_BTCCA_EN);
 
 	reg = rtw89_mac_reg_by_idx(R_AX_PREBKF_CFG_0, mac_idx);
-	rtw89_write32_mask(rtwdev, reg, B_AX_PREBKF_TIME_MASK, SCH_PREBKF_24US);
+	if (rtwdev->chip->chip_id == RTL8852C) {
+		val = rtw89_read32_mask(rtwdev, R_AX_SEC_ENG_CTRL,
+					B_AX_TX_PARTIAL_MODE);
+		if (!val)
+			rtw89_write32_mask(rtwdev, reg, B_AX_PREBKF_TIME_MASK,
+					   SCH_PREBKF_24US);
+	} else {
+		rtw89_write32_mask(rtwdev, reg, B_AX_PREBKF_TIME_MASK,
+				   SCH_PREBKF_24US);
+	}
 
 	return 0;
 }
@@ -1910,7 +1968,7 @@ static int nav_ctrl_init(struct rtw89_dev *rtwdev)
 	rtw89_write32_set(rtwdev, R_AX_WMAC_NAV_CTL, B_AX_WMAC_PLCP_UP_NAV_EN |
 						     B_AX_WMAC_TF_UP_NAV_EN |
 						     B_AX_WMAC_NAV_UPPER_EN);
-	rtw89_write32_mask(rtwdev, R_AX_WMAC_NAV_CTL, B_AX_WMAC_NAV_UPPER_MASK, NAV_12MS);
+	rtw89_write32_mask(rtwdev, R_AX_WMAC_NAV_CTL, B_AX_WMAC_NAV_UPPER_MASK, NAV_25MS);
 
 	return 0;
 }
@@ -1953,6 +2011,8 @@ static int tmac_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 
 static int trxptcl_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 {
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+	const struct rtw89_rrsr_cfgs *rrsr = chip->rrsr_cfgs;
 	u32 reg, val, sifs;
 	int ret;
 
@@ -1983,6 +2043,11 @@ static int trxptcl_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 	reg = rtw89_mac_reg_by_idx(R_AX_RXTRIG_TEST_USER_2, mac_idx);
 	rtw89_write32_set(rtwdev, reg, B_AX_RXTRIG_FCSCHK_EN);
 
+	reg = rtw89_mac_reg_by_idx(rrsr->ref_rate.addr, mac_idx);
+	rtw89_write32_mask(rtwdev, reg, rrsr->ref_rate.mask, rrsr->ref_rate.data);
+	reg = rtw89_mac_reg_by_idx(rrsr->rsc.addr, mac_idx);
+	rtw89_write32_mask(rtwdev, reg, rrsr->rsc.mask, rrsr->rsc.data);
+
 	return 0;
 }
 
@@ -2061,6 +2126,7 @@ static int rmac_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 
 static int cmac_com_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 {
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 	u32 val, reg;
 	int ret;
 
@@ -2075,6 +2141,11 @@ static int cmac_com_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 	val = u32_replace_bits(val, 0, B_AX_TXSC_80M_MASK);
 	rtw89_write32(rtwdev, reg, val);
 
+	if (chip_id == RTL8852A || chip_id == RTL8852B) {
+		reg = rtw89_mac_reg_by_idx(R_AX_PTCL_RRSR1, mac_idx);
+		rtw89_write32_mask(rtwdev, reg, B_AX_RRSR_RATE_EN_MASK, RRSR_OFDM_CCK_EN);
+	}
+
 	return 0;
 }
 
@@ -2134,6 +2205,25 @@ static int ptcl_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 	return 0;
 }
 
+static int cmac_dma_init(struct rtw89_dev *rtwdev, u8 mac_idx)
+{
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
+	u32 reg;
+	int ret;
+
+	if (chip_id != RTL8852A && chip_id != RTL8852B)
+		return 0;
+
+	ret = rtw89_mac_check_mac_en(rtwdev, mac_idx, RTW89_CMAC_SEL);
+	if (ret)
+		return ret;
+
+	reg = rtw89_mac_reg_by_idx(R_AX_RXDMA_CTRL_0, mac_idx);
+	rtw89_write8_clr(rtwdev, reg, RX_FULL_MODE);
+
+	return 0;
+}
+
 static int cmac_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 {
 	int ret;
@@ -2209,6 +2299,12 @@ static int cmac_init(struct rtw89_dev *rtwdev, u8 mac_idx)
 		return ret;
 	}
 
+	ret = cmac_dma_init(rtwdev, mac_idx);
+	if (ret) {
+		rtw89_err(rtwdev, "[ERR]CMAC%d DMA init %d\n", mac_idx, ret);
+		return ret;
+	}
+
 	return ret;
 }
 
@@ -2236,23 +2332,42 @@ int rtw89_mac_setup_phycap(struct rtw89_dev *rtwdev)
 	struct rtw89_hal *hal = &rtwdev->hal;
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 	struct rtw89_mac_c2h_info c2h_info = {0};
-	struct rtw89_c2h_phy_cap *cap =
-		(struct rtw89_c2h_phy_cap *)&c2h_info.c2hreg[0];
+	u8 tx_nss;
+	u8 rx_nss;
+	u8 tx_ant;
+	u8 rx_ant;
 	u32 ret;
 
 	ret = rtw89_mac_read_phycap(rtwdev, &c2h_info);
 	if (ret)
 		return ret;
 
-	hal->tx_nss = cap->tx_nss ?
-		      min_t(u8, cap->tx_nss, chip->tx_nss) : chip->tx_nss;
-	hal->rx_nss = cap->rx_nss ?
-		      min_t(u8, cap->rx_nss, chip->rx_nss) : chip->rx_nss;
+	tx_nss = RTW89_GET_C2H_PHYCAP_TX_NSS(c2h_info.c2hreg);
+	rx_nss = RTW89_GET_C2H_PHYCAP_RX_NSS(c2h_info.c2hreg);
+	tx_ant = RTW89_GET_C2H_PHYCAP_ANT_TX_NUM(c2h_info.c2hreg);
+	rx_ant = RTW89_GET_C2H_PHYCAP_ANT_RX_NUM(c2h_info.c2hreg);
+
+	hal->tx_nss = tx_nss ? min_t(u8, tx_nss, chip->tx_nss) : chip->tx_nss;
+	hal->rx_nss = rx_nss ? min_t(u8, rx_nss, chip->rx_nss) : chip->rx_nss;
+
+	if (tx_ant == 1)
+		hal->antenna_tx = RF_B;
+	if (rx_ant == 1)
+		hal->antenna_rx = RF_B;
+
+	if (tx_nss == 1 && tx_ant == 2 && rx_ant == 2) {
+		hal->antenna_tx = RF_B;
+		hal->tx_path_diversity = true;
+	}
 
 	rtw89_debug(rtwdev, RTW89_DBG_FW,
 		    "phycap hal/phy/chip: tx_nss=0x%x/0x%x/0x%x rx_nss=0x%x/0x%x/0x%x\n",
-		    hal->tx_nss, cap->tx_nss, chip->tx_nss,
-		    hal->rx_nss, cap->rx_nss, chip->rx_nss);
+		    hal->tx_nss, tx_nss, chip->tx_nss,
+		    hal->rx_nss, rx_nss, chip->rx_nss);
+	rtw89_debug(rtwdev, RTW89_DBG_FW,
+		    "ant num/bitmap: tx=%d/0x%x rx=%d/0x%x\n",
+		    tx_ant, hal->antenna_tx, rx_ant, hal->antenna_rx);
+	rtw89_debug(rtwdev, RTW89_DBG_FW, "TX path diversity=%d\n", hal->tx_path_diversity);
 
 	return 0;
 }
@@ -2429,8 +2544,7 @@ int rtw89_mac_resume_sch_tx_v1(struct rtw89_dev *rtwdev, u8 mac_idx, u32 tx_en)
 }
 EXPORT_SYMBOL(rtw89_mac_resume_sch_tx_v1);
 
-static u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len,
-				 bool wd)
+u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd)
 {
 	u32 val, reg;
 	int ret;
@@ -2450,9 +2564,8 @@ static u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len,
 	return FIELD_GET(B_AX_WD_BUF_STAT_PKTID_MASK, val);
 }
 
-static int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev,
-			       struct rtw89_cpuio_ctrl *ctrl_para,
-			       bool wd)
+int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev,
+			struct rtw89_cpuio_ctrl *ctrl_para, bool wd)
 {
 	u32 val, cmd_type, reg;
 	int ret;
@@ -2517,7 +2630,8 @@ static int dle_quota_change(struct rtw89_dev *rtwdev, enum rtw89_qta_mode mode)
 		return -EINVAL;
 	}
 
-	if (dle_used_size(cfg->wde_size, cfg->ple_size) != rtwdev->chip->fifo_size) {
+	if (dle_used_size(cfg->wde_size, cfg->ple_size) !=
+	    dle_expected_used_size(rtwdev, mode)) {
 		rtw89_err(rtwdev, "[ERR]wd/dle mem cfg\n");
 		return -EINVAL;
 	}
@@ -2766,7 +2880,7 @@ static void rtw89_bbrpt_imr_enable(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_imr_info *imr = rtwdev->chip->imr_info;
 
-	rtw89_write32_set(rtwdev, R_AX_BBRPT_COM_ERR_IMR,
+	rtw89_write32_set(rtwdev, imr->bbrpt_com_err_imr_reg,
 			  B_AX_BBRPT_COM_NULL_PLPKTID_ERR_INT_EN);
 	rtw89_write32_clr(rtwdev, imr->bbrpt_chinfo_err_imr_reg,
 			  B_AX_BBRPT_CHINFO_IMR_CLR);
@@ -3026,6 +3140,8 @@ static int rtw89_mac_enable_cpu(struct rtw89_dev *rtwdev, u8 boot_reason,
 
 	rtw89_write32(rtwdev, R_AX_HALT_H2C_CTRL, 0);
 	rtw89_write32(rtwdev, R_AX_HALT_C2H_CTRL, 0);
+	rtw89_write32(rtwdev, R_AX_HALT_H2C, 0);
+	rtw89_write32(rtwdev, R_AX_HALT_C2H, 0);
 
 	rtw89_write32_set(rtwdev, R_AX_SYS_CLK_CTRL, B_AX_CPU_CLK_EN);
 
@@ -3103,14 +3219,6 @@ dle:
 	return ret;
 }
 
-static void rtw89_mac_hci_func_en(struct rtw89_dev *rtwdev)
-{
-	const struct rtw89_chip_info *chip = rtwdev->chip;
-
-	rtw89_write32_set(rtwdev, chip->hci_func_en_addr,
-			  B_AX_HCI_TXDMA_EN | B_AX_HCI_RXDMA_EN);
-}
-
 int rtw89_mac_enable_bb_rf(struct rtw89_dev *rtwdev)
 {
 	rtw89_write8_set(rtwdev, R_AX_SYS_FUNC_EN,
@@ -3124,7 +3232,7 @@ int rtw89_mac_enable_bb_rf(struct rtw89_dev *rtwdev)
 }
 EXPORT_SYMBOL(rtw89_mac_enable_bb_rf);
 
-void rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev)
+int rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev)
 {
 	rtw89_write8_clr(rtwdev, R_AX_SYS_FUNC_EN,
 			 B_AX_FEN_BBRSTB | B_AX_FEN_BB_GLB_RSTN);
@@ -3132,6 +3240,8 @@ void rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev)
 			  B_AX_WLRF1_CTRL_7 | B_AX_WLRF1_CTRL_1 |
 			  B_AX_WLRF_CTRL_7 | B_AX_WLRF_CTRL_1);
 	rtw89_write8_clr(rtwdev, R_AX_PHYREG_SET, PHYREG_SET_ALL_CYCLE);
+
+	return 0;
 }
 EXPORT_SYMBOL(rtw89_mac_disable_bb_rf);
 
@@ -3147,7 +3257,7 @@ int rtw89_mac_partial_init(struct rtw89_dev *rtwdev)
 			return ret;
 	}
 
-	rtw89_mac_hci_func_en(rtwdev);
+	rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
 
 	ret = rtw89_mac_dmac_pre_init(rtwdev);
 	if (ret)
@@ -3524,6 +3634,26 @@ static void rtw89_mac_port_cfg_bcn_early(struct rtw89_dev *rtwdev,
 				BCN_ERLY_DEF);
 }
 
+static void rtw89_mac_port_cfg_tbtt_shift(struct rtw89_dev *rtwdev,
+					  struct rtw89_vif *rtwvif)
+{
+	const struct rtw89_port_reg *p = &rtw_port_base;
+	u16 val;
+
+	if (rtwdev->chip->chip_id != RTL8852C)
+		return;
+
+	if (rtwvif->wifi_role != RTW89_WIFI_ROLE_P2P_CLIENT &&
+	    rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION)
+		return;
+
+	val = FIELD_PREP(B_AX_TBTT_SHIFT_OFST_MAG, 1) |
+			 B_AX_TBTT_SHIFT_OFST_SIGN;
+
+	rtw89_write16_port_mask(rtwdev, rtwvif, p->tbtt_shift,
+				B_AX_TBTT_SHIFT_OFST_MASK, val);
+}
+
 int rtw89_mac_vif_init(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 {
 	int ret;
@@ -3598,6 +3728,7 @@ int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 	rtw89_mac_port_cfg_bcn_hold_time(rtwdev, rtwvif);
 	rtw89_mac_port_cfg_bcn_mask_area(rtwdev, rtwvif);
 	rtw89_mac_port_cfg_tbtt_early(rtwdev, rtwvif);
+	rtw89_mac_port_cfg_tbtt_shift(rtwdev, rtwvif);
 	rtw89_mac_port_cfg_bss_color(rtwdev, rtwvif);
 	rtw89_mac_port_cfg_mbssid(rtwdev, rtwvif);
 	rtw89_mac_port_cfg_func_en(rtwdev, rtwvif);
@@ -3607,6 +3738,50 @@ int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 	return 0;
 }
 
+static void rtw89_mac_check_he_obss_narrow_bw_ru_iter(struct wiphy *wiphy,
+						      struct cfg80211_bss *bss,
+						      void *data)
+{
+	const struct cfg80211_bss_ies *ies;
+	const struct element *elem;
+	bool *tolerated = data;
+
+	rcu_read_lock();
+	ies = rcu_dereference(bss->ies);
+	elem = cfg80211_find_elem(WLAN_EID_EXT_CAPABILITY, ies->data,
+				  ies->len);
+
+	if (!elem || elem->datalen < 10 ||
+	    !(elem->data[10] & WLAN_EXT_CAPA10_OBSS_NARROW_BW_RU_TOLERANCE_SUPPORT))
+		*tolerated = false;
+	rcu_read_unlock();
+}
+
+void rtw89_mac_set_he_obss_narrow_bw_ru(struct rtw89_dev *rtwdev,
+					struct ieee80211_vif *vif)
+{
+	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
+	struct ieee80211_hw *hw = rtwdev->hw;
+	bool tolerated = true;
+	u32 reg;
+
+	if (!vif->bss_conf.he_support || vif->type != NL80211_IFTYPE_STATION)
+		return;
+
+	if (!(vif->bss_conf.chandef.chan->flags & IEEE80211_CHAN_RADAR))
+		return;
+
+	cfg80211_bss_iter(hw->wiphy, &vif->bss_conf.chandef,
+			  rtw89_mac_check_he_obss_narrow_bw_ru_iter,
+			  &tolerated);
+
+	reg = rtw89_mac_reg_by_idx(R_AX_RXTRIG_TEST_USER_2, rtwvif->mac_idx);
+	if (tolerated)
+		rtw89_write32_clr(rtwdev, reg, B_AX_RXTRIG_RU26_DIS);
+	else
+		rtw89_write32_set(rtwdev, reg, B_AX_RXTRIG_RU26_DIS);
+}
+
 int rtw89_mac_add_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 {
 	int ret;
@@ -3655,22 +3830,26 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h,
 			   u32 len)
 {
 	struct ieee80211_vif *vif = rtwdev->scan_info.scanning_vif;
-	struct rtw89_hal *hal = &rtwdev->hal;
-	u8 reason, status, tx_fail, band;
+	struct rtw89_vif *rtwvif = vif_to_rtwvif_safe(vif);
+	struct rtw89_chan new;
+	u8 reason, status, tx_fail, band, actual_period;
+	u32 last_chan = rtwdev->scan_info.last_chan_idx;
 	u16 chan;
+	int ret;
 
 	tx_fail = RTW89_GET_MAC_C2H_SCANOFLD_TX_FAIL(c2h->data);
 	status = RTW89_GET_MAC_C2H_SCANOFLD_STATUS(c2h->data);
 	chan = RTW89_GET_MAC_C2H_SCANOFLD_PRI_CH(c2h->data);
 	reason = RTW89_GET_MAC_C2H_SCANOFLD_RSP(c2h->data);
 	band = RTW89_GET_MAC_C2H_SCANOFLD_BAND(c2h->data);
+	actual_period = RTW89_GET_MAC_C2H_ACTUAL_PERIOD(c2h->data);
 
 	if (!(rtwdev->chip->support_bands & BIT(NL80211_BAND_6GHZ)))
 		band = chan > 14 ? RTW89_BAND_5G : RTW89_BAND_2G;
 
 	rtw89_debug(rtwdev, RTW89_DBG_HW_SCAN,
-		    "band: %d, chan: %d, reason: %d, status: %d, tx_fail: %d\n",
-		    band, chan, reason, status, tx_fail);
+		    "band: %d, chan: %d, reason: %d, status: %d, tx_fail: %d, actual: %d\n",
+		    band, chan, reason, status, tx_fail, actual_period);
 
 	switch (reason) {
 	case RTW89_SCAN_LEAVE_CH_NOTIFY:
@@ -3678,15 +3857,20 @@ rtw89_mac_c2h_scanofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h,
 			ieee80211_stop_queues(rtwdev->hw);
 		return;
 	case RTW89_SCAN_END_SCAN_NOTIFY:
-		rtw89_hw_scan_complete(rtwdev, vif, false);
+		if (rtwvif && rtwvif->scan_req &&
+		    last_chan < rtwvif->scan_req->n_channels) {
+			ret = rtw89_hw_scan_offload(rtwdev, vif, true);
+			if (ret) {
+				rtw89_hw_scan_abort(rtwdev, vif);
+				rtw89_warn(rtwdev, "HW scan failed: %d\n", ret);
+			}
+		} else {
+			rtw89_hw_scan_complete(rtwdev, vif, false);
+		}
 		break;
 	case RTW89_SCAN_ENTER_CH_NOTIFY:
-		hal->prev_band_type = hal->current_band_type;
-		hal->current_band_type = band;
-		hal->prev_primary_channel = hal->current_primary_channel;
-		hal->current_primary_channel = chan;
-		hal->current_channel = chan;
-		hal->current_band_width = RTW89_CHANNEL_WIDTH_20;
+		rtw89_chan_create(&new, chan, chan, band, RTW89_CHANNEL_WIDTH_20);
+		rtw89_assign_entity_chan(rtwdev, RTW89_SUB_ENTITY_0, &new);
 		if (rtw89_is_op_chan(rtwdev, band, chan)) {
 			rtw89_store_op_chan(rtwdev, false);
 			ieee80211_wake_queues(rtwdev->hw);
@@ -3738,6 +3922,12 @@ rtw89_mac_c2h_pkt_ofld_rsp(struct rtw89_dev *rtwdev, struct sk_buff *c2h,
 {
 }
 
+static void
+rtw89_mac_c2h_tsf32_toggle_rpt(struct rtw89_dev *rtwdev, struct sk_buff *c2h,
+			       u32 len)
+{
+}
+
 static
 void (* const rtw89_mac_c2h_ofld_handler[])(struct rtw89_dev *rtwdev,
 					    struct sk_buff *c2h, u32 len) = {
@@ -3747,6 +3937,7 @@ void (* const rtw89_mac_c2h_ofld_handler[])(struct rtw89_dev *rtwdev,
 	[RTW89_MAC_C2H_FUNC_BCN_RESEND] = NULL,
 	[RTW89_MAC_C2H_FUNC_MACID_PAUSE] = rtw89_mac_c2h_macid_pause,
 	[RTW89_MAC_C2H_FUNC_SCANOFLD_RSP] = rtw89_mac_c2h_scanofld_rsp,
+	[RTW89_MAC_C2H_FUNC_TSF32_TOGL_RPT] = rtw89_mac_c2h_tsf32_toggle_rpt,
 };
 
 static
@@ -4628,3 +4819,48 @@ int rtw89_mac_read_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 *val)
 
 	return 0;
 }
+
+static
+void rtw89_mac_pkt_drop_sta(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta)
+{
+	static const enum rtw89_pkt_drop_sel sels[] = {
+		RTW89_PKT_DROP_SEL_MACID_BE_ONCE,
+		RTW89_PKT_DROP_SEL_MACID_BK_ONCE,
+		RTW89_PKT_DROP_SEL_MACID_VI_ONCE,
+		RTW89_PKT_DROP_SEL_MACID_VO_ONCE,
+	};
+	struct rtw89_vif *rtwvif = rtwsta->rtwvif;
+	struct rtw89_pkt_drop_params params = {0};
+	int i;
+
+	params.mac_band = RTW89_MAC_0;
+	params.macid = rtwsta->mac_id;
+	params.port = rtwvif->port;
+	params.mbssid = 0;
+	params.tf_trs = rtwvif->trigger;
+
+	for (i = 0; i < ARRAY_SIZE(sels); i++) {
+		params.sel = sels[i];
+		rtw89_fw_h2c_pkt_drop(rtwdev, &params);
+	}
+}
+
+static void rtw89_mac_pkt_drop_vif_iter(void *data, struct ieee80211_sta *sta)
+{
+	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
+	struct rtw89_vif *rtwvif = rtwsta->rtwvif;
+	struct rtw89_dev *rtwdev = rtwvif->rtwdev;
+	struct rtw89_vif *target = data;
+
+	if (rtwvif != target)
+		return;
+
+	rtw89_mac_pkt_drop_sta(rtwdev, rtwsta);
+}
+
+void rtw89_mac_pkt_drop_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
+{
+	ieee80211_iterate_stations_atomic(rtwdev->hw,
+					  rtw89_mac_pkt_drop_vif_iter,
+					  rtwvif);
+}
diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h
index f66619354734..6f4ada1869a1 100644
--- a/drivers/net/wireless/realtek/rtw89/mac.h
+++ b/drivers/net/wireless/realtek/rtw89/mac.h
@@ -6,11 +6,13 @@
 #define __RTW89_MAC_H__
 
 #include "core.h"
+#include "reg.h"
 
 #define MAC_MEM_DUMP_PAGE_SIZE 0x40000
 #define ADDR_CAM_ENT_SIZE  0x40
 #define BSSID_CAM_ENT_SIZE 0x08
 #define HFC_PAGE_UNIT 64
+#define RPWM_TRY_CNT 3
 
 enum rtw89_mac_hwmod_sel {
 	RTW89_DMAC_SEL = 0,
@@ -304,6 +306,7 @@ enum rtw89_mac_c2h_ofld_func {
 	RTW89_MAC_C2H_FUNC_PKT_OFLD_RSP,
 	RTW89_MAC_C2H_FUNC_BCN_RESEND,
 	RTW89_MAC_C2H_FUNC_MACID_PAUSE,
+	RTW89_MAC_C2H_FUNC_TSF32_TOGL_RPT = 0x6,
 	RTW89_MAC_C2H_FUNC_SCANOFLD_RSP = 0x9,
 	RTW89_MAC_C2H_FUNC_OFLD_MAX,
 };
@@ -688,23 +691,30 @@ struct rtw89_mac_size_set {
 	const struct rtw89_hfc_prec_cfg hfc_preccfg_pcie;
 	const struct rtw89_dle_size wde_size0;
 	const struct rtw89_dle_size wde_size4;
+	const struct rtw89_dle_size wde_size6;
+	const struct rtw89_dle_size wde_size9;
 	const struct rtw89_dle_size wde_size18;
 	const struct rtw89_dle_size wde_size19;
 	const struct rtw89_dle_size ple_size0;
 	const struct rtw89_dle_size ple_size4;
+	const struct rtw89_dle_size ple_size6;
+	const struct rtw89_dle_size ple_size8;
 	const struct rtw89_dle_size ple_size18;
 	const struct rtw89_dle_size ple_size19;
 	const struct rtw89_wde_quota wde_qt0;
 	const struct rtw89_wde_quota wde_qt4;
+	const struct rtw89_wde_quota wde_qt6;
 	const struct rtw89_wde_quota wde_qt17;
 	const struct rtw89_wde_quota wde_qt18;
 	const struct rtw89_ple_quota ple_qt4;
 	const struct rtw89_ple_quota ple_qt5;
 	const struct rtw89_ple_quota ple_qt13;
+	const struct rtw89_ple_quota ple_qt18;
 	const struct rtw89_ple_quota ple_qt44;
 	const struct rtw89_ple_quota ple_qt45;
 	const struct rtw89_ple_quota ple_qt46;
 	const struct rtw89_ple_quota ple_qt47;
+	const struct rtw89_ple_quota ple_qt58;
 };
 
 extern const struct rtw89_mac_size_set rtw89_mac_size;
@@ -798,9 +808,11 @@ int rtw89_mac_write_lte(struct rtw89_dev *rtwdev, const u32 offset, u32 val);
 int rtw89_mac_read_lte(struct rtw89_dev *rtwdev, const u32 offset, u32 *val);
 int rtw89_mac_add_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *vif);
 int rtw89_mac_port_update(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
+void rtw89_mac_set_he_obss_narrow_bw_ru(struct rtw89_dev *rtwdev,
+					struct ieee80211_vif *vif);
 int rtw89_mac_remove_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *vif);
 int rtw89_mac_enable_bb_rf(struct rtw89_dev *rtwdev);
-void rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev);
+int rtw89_mac_disable_bb_rf(struct rtw89_dev *rtwdev);
 
 static inline int rtw89_chip_enable_bb_rf(struct rtw89_dev *rtwdev)
 {
@@ -809,11 +821,11 @@ static inline int rtw89_chip_enable_bb_rf(struct rtw89_dev *rtwdev)
 	return chip->ops->enable_bb_rf(rtwdev);
 }
 
-static inline void rtw89_chip_disable_bb_rf(struct rtw89_dev *rtwdev)
+static inline int rtw89_chip_disable_bb_rf(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
 
-	chip->ops->disable_bb_rf(rtwdev);
+	return chip->ops->disable_bb_rf(rtwdev);
 }
 
 u32 rtw89_mac_get_err_status(struct rtw89_dev *rtwdev);
@@ -911,6 +923,45 @@ static inline int rtw89_mac_txpwr_write32_mask(struct rtw89_dev *rtwdev,
 	return 0;
 }
 
+static inline void rtw89_mac_ctrl_hci_dma_tx(struct rtw89_dev *rtwdev,
+					     bool enable)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	if (enable)
+		rtw89_write32_set(rtwdev, chip->hci_func_en_addr,
+				  B_AX_HCI_TXDMA_EN);
+	else
+		rtw89_write32_clr(rtwdev, chip->hci_func_en_addr,
+				  B_AX_HCI_TXDMA_EN);
+}
+
+static inline void rtw89_mac_ctrl_hci_dma_rx(struct rtw89_dev *rtwdev,
+					     bool enable)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	if (enable)
+		rtw89_write32_set(rtwdev, chip->hci_func_en_addr,
+				  B_AX_HCI_RXDMA_EN);
+	else
+		rtw89_write32_clr(rtwdev, chip->hci_func_en_addr,
+				  B_AX_HCI_RXDMA_EN);
+}
+
+static inline void rtw89_mac_ctrl_hci_dma_trx(struct rtw89_dev *rtwdev,
+					      bool enable)
+{
+	const struct rtw89_chip_info *chip = rtwdev->chip;
+
+	if (enable)
+		rtw89_write32_set(rtwdev, chip->hci_func_en_addr,
+				  B_AX_HCI_TXDMA_EN | B_AX_HCI_RXDMA_EN);
+	else
+		rtw89_write32_clr(rtwdev, chip->hci_func_en_addr,
+				  B_AX_HCI_TXDMA_EN | B_AX_HCI_RXDMA_EN);
+}
+
 int rtw89_mac_set_tx_time(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta,
 			  bool resume, u32 tx_time);
 int rtw89_mac_get_tx_time(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta,
@@ -944,8 +995,10 @@ enum rtw89_mac_xtal_si_offset {
 #define XTAL_SI_HIGH_ADDR_MASK	GENMASK(2, 0)
 	XTAL_SI_READ_VAL = 0x7A,
 	XTAL_SI_WL_RFC_S0 = 0x80,
+#define XTAL_SI_RF00S_EN	GENMASK(2, 0)
 #define XTAL_SI_RF00		BIT(0)
 	XTAL_SI_WL_RFC_S1 = 0x81,
+#define XTAL_SI_RF10S_EN	GENMASK(2, 0)
 #define XTAL_SI_RF10		BIT(0)
 	XTAL_SI_ANAPAR_WL = 0x90,
 #define XTAL_SI_SRAM2RFC	BIT(7)
@@ -962,5 +1015,9 @@ enum rtw89_mac_xtal_si_offset {
 
 int rtw89_mac_write_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 val, u8 mask);
 int rtw89_mac_read_xtal_si(struct rtw89_dev *rtwdev, u8 offset, u8 *val);
+void rtw89_mac_pkt_drop_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
+u16 rtw89_mac_dle_buf_req(struct rtw89_dev *rtwdev, u16 buf_len, bool wd);
+int rtw89_mac_set_cpuio(struct rtw89_dev *rtwdev,
+			struct rtw89_cpuio_ctrl *ctrl_para, bool wd);
 
 #endif
diff --git a/drivers/net/wireless/realtek/rtw89/mac80211.c b/drivers/net/wireless/realtek/rtw89/mac80211.c
index cef27e781ae2..a296bfa8188f 100644
--- a/drivers/net/wireless/realtek/rtw89/mac80211.c
+++ b/drivers/net/wireless/realtek/rtw89/mac80211.c
@@ -3,6 +3,7 @@
  */
 
 #include "cam.h"
+#include "chan.h"
 #include "coex.h"
 #include "debug.h"
 #include "fw.h"
@@ -12,6 +13,7 @@
 #include "reg.h"
 #include "sar.h"
 #include "ser.h"
+#include "util.h"
 
 static void rtw89_ops_tx(struct ieee80211_hw *hw,
 			 struct ieee80211_tx_control *control,
@@ -85,8 +87,11 @@ static int rtw89_ops_config(struct ieee80211_hw *hw, u32 changed)
 		}
 	}
 
-	if (changed & IEEE80211_CONF_CHANGE_CHANNEL)
+	if (changed & IEEE80211_CONF_CHANGE_CHANNEL) {
+		rtw89_config_entity_chandef(rtwdev, RTW89_SUB_ENTITY_0,
+					    &hw->conf.chandef);
 		rtw89_set_channel(rtwdev);
+	}
 
 	if ((changed & IEEE80211_CONF_CHANGE_IDLE) &&
 	    (hw->conf.flags & IEEE80211_CONF_IDLE))
@@ -104,6 +109,9 @@ static int rtw89_ops_add_interface(struct ieee80211_hw *hw,
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	int ret = 0;
 
+	rtw89_debug(rtwdev, RTW89_DBG_STATE, "add vif %pM type %d, p2p %d\n",
+		    vif->addr, vif->type, vif->p2p);
+
 	mutex_lock(&rtwdev->mutex);
 	rtwvif->rtwdev = rtwdev;
 	list_add_tail(&rtwvif->list, &rtwdev->rtwvifs_list);
@@ -146,6 +154,9 @@ static void rtw89_ops_remove_interface(struct ieee80211_hw *hw,
 	struct rtw89_dev *rtwdev = hw->priv;
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 
+	rtw89_debug(rtwdev, RTW89_DBG_STATE, "remove vif %pM type %d p2p %d\n",
+		    vif->addr, vif->type, vif->p2p);
+
 	cancel_work_sync(&rtwvif->update_beacon_work);
 
 	mutex_lock(&rtwdev->mutex);
@@ -157,6 +168,23 @@ static void rtw89_ops_remove_interface(struct ieee80211_hw *hw,
 	mutex_unlock(&rtwdev->mutex);
 }
 
+static int rtw89_ops_change_interface(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      enum nl80211_iftype type, bool p2p)
+{
+	struct rtw89_dev *rtwdev = hw->priv;
+
+	rtw89_debug(rtwdev, RTW89_DBG_STATE, "change vif %pM (%d)->(%d), p2p (%d)->(%d)\n",
+		    vif->addr, vif->type, type, vif->p2p, p2p);
+
+	rtw89_ops_remove_interface(hw, vif);
+
+	vif->type = type;
+	vif->p2p = p2p;
+
+	return rtw89_ops_add_interface(hw, vif);
+}
+
 static void rtw89_ops_configure_filter(struct ieee80211_hw *hw,
 				       unsigned int changed_flags,
 				       unsigned int *new_flags,
@@ -235,11 +263,12 @@ static u8 rtw89_aifsn_to_aifs(struct rtw89_dev *rtwdev,
 			      struct rtw89_vif *rtwvif, u8 aifsn)
 {
 	struct ieee80211_vif *vif = rtwvif_to_vif(rtwvif);
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	u8 slot_time;
 	u8 sifs;
 
 	slot_time = vif->bss_conf.use_short_slot ? 9 : 20;
-	sifs = rtwdev->hal.current_band_type == RTW89_BAND_5G ? 16 : 10;
+	sifs = chan->band_type == RTW89_BAND_5G ? 16 : 10;
 
 	return aifsn * slot_time + sifs;
 }
@@ -350,6 +379,7 @@ static void rtw89_ops_bss_info_changed(struct ieee80211_hw *hw,
 			rtw89_phy_set_bss_color(rtwdev, vif);
 			rtw89_chip_cfg_txpwr_ul_tb_offset(rtwdev, vif);
 			rtw89_mac_port_update(rtwdev, rtwvif);
+			rtw89_mac_set_he_obss_narrow_bw_ru(rtwdev, vif);
 			rtw89_store_op_chan(rtwdev, true);
 		} else {
 			/* Abort ongoing scan if cancel_scan isn't issued
@@ -378,6 +408,9 @@ static void rtw89_ops_bss_info_changed(struct ieee80211_hw *hw,
 	if (changed & BSS_CHANGED_MU_GROUPS)
 		rtw89_mac_bf_set_gid_table(rtwdev, vif, conf);
 
+	if (changed & BSS_CHANGED_P2P_PS)
+		rtw89_process_p2p_ps(rtwdev, vif);
+
 	mutex_unlock(&rtwdev->mutex);
 }
 
@@ -605,6 +638,20 @@ static void rtw89_ops_sta_statistics(struct ieee80211_hw *hw,
 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
 }
 
+static
+void __rtw89_drop_packets(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif)
+{
+	struct rtw89_vif *rtwvif;
+
+	if (vif) {
+		rtwvif = (struct rtw89_vif *)vif->drv_priv;
+		rtw89_mac_pkt_drop_vif(rtwdev, rtwvif);
+	} else {
+		rtw89_for_each_rtwvif(rtwdev, rtwvif)
+			rtw89_mac_pkt_drop_vif(rtwdev, rtwvif);
+	}
+}
+
 static void rtw89_ops_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 			    u32 queues, bool drop)
 {
@@ -613,7 +660,12 @@ static void rtw89_ops_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	mutex_lock(&rtwdev->mutex);
 	rtw89_leave_lps(rtwdev);
 	rtw89_hci_flush_queues(rtwdev, queues, drop);
-	rtw89_mac_flush_txq(rtwdev, queues, drop);
+
+	if (drop && RTW89_CHK_FW_FEATURE(PACKET_DROP, &rtwdev->fw))
+		__rtw89_drop_packets(rtwdev, vif);
+	else
+		rtw89_mac_flush_txq(rtwdev, queues, drop);
+
 	mutex_unlock(&rtwdev->mutex);
 }
 
@@ -629,7 +681,7 @@ static void rtw89_ra_mask_info_update_iter(void *data, struct ieee80211_sta *sta
 	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
 	struct ieee80211_vif *vif = rtwvif_to_vif(rtwsta->rtwvif);
 
-	if (vif != br_data->vif)
+	if (vif != br_data->vif || vif->p2p)
 		return;
 
 	rtwsta->use_cfg_mask = true;
@@ -669,12 +721,13 @@ int rtw89_ops_set_antenna(struct ieee80211_hw *hw, u32 tx_ant, u32 rx_ant)
 	struct rtw89_dev *rtwdev = hw->priv;
 	struct rtw89_hal *hal = &rtwdev->hal;
 
-	if (rx_ant != hw->wiphy->available_antennas_rx)
+	if (rx_ant != hw->wiphy->available_antennas_rx && rx_ant != hal->antenna_rx)
 		return -EINVAL;
 
 	mutex_lock(&rtwdev->mutex);
 	hal->antenna_tx = tx_ant;
 	hal->antenna_rx = rx_ant;
+	hal->tx_path_diversity = false;
 	mutex_unlock(&rtwdev->mutex);
 
 	return 0;
@@ -772,6 +825,97 @@ static void rtw89_ops_sta_rc_update(struct ieee80211_hw *hw,
 	rtw89_phy_ra_updata_sta(rtwdev, sta, changed);
 }
 
+static int rtw89_ops_add_chanctx(struct ieee80211_hw *hw,
+				 struct ieee80211_chanctx_conf *ctx)
+{
+	struct rtw89_dev *rtwdev = hw->priv;
+	int ret;
+
+	mutex_lock(&rtwdev->mutex);
+	ret = rtw89_chanctx_ops_add(rtwdev, ctx);
+	mutex_unlock(&rtwdev->mutex);
+
+	return ret;
+}
+
+static void rtw89_ops_remove_chanctx(struct ieee80211_hw *hw,
+				     struct ieee80211_chanctx_conf *ctx)
+{
+	struct rtw89_dev *rtwdev = hw->priv;
+
+	mutex_lock(&rtwdev->mutex);
+	rtw89_chanctx_ops_remove(rtwdev, ctx);
+	mutex_unlock(&rtwdev->mutex);
+}
+
+static void rtw89_ops_change_chanctx(struct ieee80211_hw *hw,
+				     struct ieee80211_chanctx_conf *ctx,
+				     u32 changed)
+{
+	struct rtw89_dev *rtwdev = hw->priv;
+
+	mutex_lock(&rtwdev->mutex);
+	rtw89_chanctx_ops_change(rtwdev, ctx, changed);
+	mutex_unlock(&rtwdev->mutex);
+}
+
+static int rtw89_ops_assign_vif_chanctx(struct ieee80211_hw *hw,
+					struct ieee80211_vif *vif,
+					struct ieee80211_bss_conf *link_conf,
+					struct ieee80211_chanctx_conf *ctx)
+{
+	struct rtw89_dev *rtwdev = hw->priv;
+	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
+	int ret;
+
+	mutex_lock(&rtwdev->mutex);
+	ret = rtw89_chanctx_ops_assign_vif(rtwdev, rtwvif, ctx);
+	mutex_unlock(&rtwdev->mutex);
+
+	return ret;
+}
+
+static void rtw89_ops_unassign_vif_chanctx(struct ieee80211_hw *hw,
+					   struct ieee80211_vif *vif,
+					   struct ieee80211_bss_conf *link_conf,
+					   struct ieee80211_chanctx_conf *ctx)
+{
+	struct rtw89_dev *rtwdev = hw->priv;
+	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
+
+	mutex_lock(&rtwdev->mutex);
+	rtw89_chanctx_ops_unassign_vif(rtwdev, rtwvif, ctx);
+	mutex_unlock(&rtwdev->mutex);
+}
+
+static void rtw89_set_tid_config_iter(void *data, struct ieee80211_sta *sta)
+{
+	struct cfg80211_tid_config *tid_config = data;
+	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
+	struct rtw89_dev *rtwdev = rtwsta->rtwvif->rtwdev;
+
+	rtw89_core_set_tid_config(rtwdev, sta, tid_config);
+}
+
+static int rtw89_ops_set_tid_config(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_sta *sta,
+				    struct cfg80211_tid_config *tid_config)
+{
+	struct rtw89_dev *rtwdev = hw->priv;
+
+	mutex_lock(&rtwdev->mutex);
+	if (sta)
+		rtw89_core_set_tid_config(rtwdev, sta, tid_config);
+	else
+		ieee80211_iterate_stations_atomic(rtwdev->hw,
+						  rtw89_set_tid_config_iter,
+						  tid_config);
+	mutex_unlock(&rtwdev->mutex);
+
+	return 0;
+}
+
 const struct ieee80211_ops rtw89_ops = {
 	.tx			= rtw89_ops_tx,
 	.wake_tx_queue		= rtw89_ops_wake_tx_queue,
@@ -779,6 +923,7 @@ const struct ieee80211_ops rtw89_ops = {
 	.stop			= rtw89_ops_stop,
 	.config			= rtw89_ops_config,
 	.add_interface		= rtw89_ops_add_interface,
+	.change_interface       = rtw89_ops_change_interface,
 	.remove_interface	= rtw89_ops_remove_interface,
 	.configure_filter	= rtw89_ops_configure_filter,
 	.bss_info_changed	= rtw89_ops_bss_info_changed,
@@ -800,7 +945,13 @@ const struct ieee80211_ops rtw89_ops = {
 	.reconfig_complete	= rtw89_ops_reconfig_complete,
 	.hw_scan		= rtw89_ops_hw_scan,
 	.cancel_hw_scan		= rtw89_ops_cancel_hw_scan,
+	.add_chanctx		= rtw89_ops_add_chanctx,
+	.remove_chanctx		= rtw89_ops_remove_chanctx,
+	.change_chanctx		= rtw89_ops_change_chanctx,
+	.assign_vif_chanctx	= rtw89_ops_assign_vif_chanctx,
+	.unassign_vif_chanctx	= rtw89_ops_unassign_vif_chanctx,
 	.set_sar_specs		= rtw89_ops_set_sar_specs,
 	.sta_rc_update		= rtw89_ops_sta_rc_update,
+	.set_tid_config		= rtw89_ops_set_tid_config,
 };
 EXPORT_SYMBOL(rtw89_ops);
diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c
index c68fec9eb5a6..5f8e19639362 100644
--- a/drivers/net/wireless/realtek/rtw89/pci.c
+++ b/drivers/net/wireless/realtek/rtw89/pci.c
@@ -169,6 +169,23 @@ static int rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,
 	return 0;
 }
 
+static void rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev *rtwdev, bool enable)
+{
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
+	const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
+	const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2;
+
+	if (enable) {
+		rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask);
+		if (dma_stop2->addr)
+			rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask);
+	} else {
+		rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask);
+		if (dma_stop2->addr)
+			rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask);
+	}
+}
+
 static bool
 rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
 		      struct sk_buff *new,
@@ -760,7 +777,8 @@ static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev)
 
 enable_intr:
 	spin_lock_irqsave(&rtwpci->irq_lock, flags);
-	rtw89_chip_enable_intr(rtwdev, rtwpci);
+	if (likely(rtwpci->running))
+		rtw89_chip_enable_intr(rtwdev, rtwpci);
 	spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
 	return IRQ_HANDLED;
 }
@@ -925,10 +943,12 @@ u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,
 {
 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
 	struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
+	struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
 	u32 cnt;
 
 	spin_lock_bh(&rtwpci->trx_lock);
 	cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
+	cnt = min(cnt, wd_ring->curr_num);
 	spin_unlock_bh(&rtwpci->trx_lock);
 
 	return cnt;
@@ -1073,12 +1093,15 @@ static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)
 static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,
 					bool drop)
 {
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	u8 i;
 
 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
 		/* It may be unnecessary to flush FWCMD queue. */
 		if (i == RTW89_TXCH_CH12)
 			continue;
+		if (info->tx_dma_ch_mask & BIT(i))
+			continue;
 
 		if (txchs & BIT(i))
 			__pci_flush_txch(rtwdev, i, drop);
@@ -1357,6 +1380,7 @@ static const struct rtw89_pci_bd_ram bd_ram_table[RTW89_TXCH_NUM] = {
 static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	struct rtw89_pci_tx_ring *tx_ring;
 	struct rtw89_pci_rx_ring *rx_ring;
 	struct rtw89_pci_dma_ring *bd_ring;
@@ -1368,6 +1392,9 @@ static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
 	int i;
 
 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
+		if (info->tx_dma_ch_mask & BIT(i))
+			continue;
+
 		tx_ring = &rtwpci->tx_rings[i];
 		bd_ring = &tx_ring->bd_ring;
 		bd_ram = &bd_ram_table[i];
@@ -1411,12 +1438,15 @@ static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
 static void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	int txch;
 
 	rtw89_pci_reset_trx_rings(rtwdev);
 
 	spin_lock_bh(&rtwpci->trx_lock);
 	for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
+		if (info->tx_dma_ch_mask & BIT(txch))
+			continue;
 		if (txch == RTW89_TXCH_CH12) {
 			rtw89_pci_release_fwcmd(rtwdev, rtwpci,
 						skb_queue_len(&rtwpci->h2c_queue), true);
@@ -1604,33 +1634,41 @@ static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)
 	writel(data, rtwpci->mmap + addr);
 }
 
-static void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
+static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
 {
-	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 	const struct rtw89_pci_info *info = rtwdev->pci_info;
-	u32 txhci_en = info->txhci_en_bit;
-	u32 rxhci_en = info->rxhci_en_bit;
 
-	if (enable) {
-		if (chip_id != RTL8852C)
-			rtw89_write32_clr(rtwdev, info->dma_stop1_reg,
-					  B_AX_STOP_PCIEIO);
-		rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
-				  txhci_en | rxhci_en);
-		if (chip_id == RTL8852C)
-			rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
-					  B_AX_STOP_AXI_MST);
+	if (enable)
+		rtw89_write32_set(rtwdev, info->init_cfg_reg,
+				  info->rxhci_en_bit | info->txhci_en_bit);
+	else
+		rtw89_write32_clr(rtwdev, info->init_cfg_reg,
+				  info->rxhci_en_bit | info->txhci_en_bit);
+}
+
+static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
+{
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
+	u32 reg, mask;
+
+	if (chip_id == RTL8852C) {
+		reg = R_AX_HAXI_INIT_CFG1;
+		mask = B_AX_STOP_AXI_MST;
 	} else {
-		if (chip_id != RTL8852C)
-			rtw89_write32_set(rtwdev, info->dma_stop1_reg,
-					  B_AX_STOP_PCIEIO);
-		else
-			rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
-					  B_AX_STOP_AXI_MST);
-		if (chip_id == RTL8852C)
-			rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
-					  B_AX_STOP_AXI_MST);
+		reg = R_AX_PCIE_DMA_STOP1;
+		mask = B_AX_STOP_PCIEIO;
 	}
+
+	if (enable)
+		rtw89_write32_clr(rtwdev, reg, mask);
+	else
+		rtw89_write32_set(rtwdev, reg, mask);
+}
+
+static void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
+{
+	rtw89_pci_ctrl_dma_io(rtwdev, enable);
+	rtw89_pci_ctrl_dma_trx(rtwdev, enable);
 }
 
 static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)
@@ -1836,6 +1874,18 @@ __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate
 	return 0;
 }
 
+static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
+{
+	int ret;
+
+	if (rtwdev->chip->chip_id != RTL8852B)
+		return 0;
+
+	ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
+				      PCIE_AUTOK_4, PCIE_PHY_GEN1);
+	return ret;
+}
+
 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
 {
 	enum rtw89_pcie_phy phy_rate;
@@ -2049,7 +2099,7 @@ static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
 
 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
 {
-	if (rtwdev->chip->chip_id != RTL8852A)
+	if (rtwdev->chip->chip_id != RTL8852A && rtwdev->chip->chip_id != RTL8852B)
 		return;
 
 	rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
@@ -2234,19 +2284,19 @@ static int rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev *rtwdev)
 {
 	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	u32 ret, check, dma_busy;
-	u32 dma_busy1 = info->dma_busy1_reg;
+	u32 dma_busy1 = info->dma_busy1.addr;
 	u32 dma_busy2 = info->dma_busy2_reg;
 
-	check = B_AX_ACH0_BUSY | B_AX_ACH1_BUSY | B_AX_ACH2_BUSY |
-		B_AX_ACH3_BUSY | B_AX_ACH4_BUSY | B_AX_ACH5_BUSY |
-		B_AX_ACH6_BUSY | B_AX_ACH7_BUSY | B_AX_CH8_BUSY |
-		B_AX_CH9_BUSY | B_AX_CH12_BUSY;
+	check = info->dma_busy1.mask;
 
 	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
 				10, 100, false, rtwdev, dma_busy1);
 	if (ret)
 		return ret;
 
+	if (!dma_busy2)
+		return 0;
+
 	check = B_AX_CH10_BUSY | B_AX_CH11_BUSY;
 
 	ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
@@ -2414,6 +2464,12 @@ static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev)
 	rtw89_pci_hci_ldo(rtwdev);
 	rtw89_pci_dphy_delay(rtwdev);
 
+	ret = rtw89_pci_autok_x(rtwdev);
+	if (ret) {
+		rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret);
+		return ret;
+	}
+
 	ret = rtw89_pci_auto_refclk_cal(rtwdev, false);
 	if (ret) {
 		rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret);
@@ -2432,7 +2488,7 @@ static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev)
 	rtw89_pci_set_dbg(rtwdev);
 	rtw89_pci_set_keep_reg(rtwdev);
 
-	rtw89_write32_set(rtwdev, info->dma_stop1_reg, B_AX_STOP_WPDMA);
+	rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA);
 
 	/* stop DMA activities */
 	rtw89_pci_ctrl_dma_all(rtwdev, false);
@@ -2455,10 +2511,9 @@ static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev)
 		return ret;
 	}
 
-	/* enable FW CMD queue to download firmware */
-	rtw89_write32_set(rtwdev, info->dma_stop1_reg, B_AX_TX_STOP1_ALL);
-	rtw89_write32_clr(rtwdev, info->dma_stop1_reg, B_AX_STOP_CH12);
-	rtw89_write32_set(rtwdev, info->dma_stop2_reg, B_AX_TX_STOP2_ALL);
+	/* disable all channels except to FW CMD channel to download firmware */
+	rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, false);
+	rtw89_write32_clr(rtwdev, info->dma_stop1.addr, B_AX_STOP_CH12);
 
 	/* start DMA activities */
 	rtw89_pci_ctrl_dma_all(rtwdev, true);
@@ -2486,15 +2541,15 @@ int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)
 	if (rtw89_pci_ltr_is_err_reg_val(val))
 		return -EINVAL;
 
-	rtw89_write32_clr(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN);
-	rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_EN);
+	rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN |
+						   B_AX_LTR_WD_NOEMP_CHK);
 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK,
 			   PCI_LTR_SPC_500US);
 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
-			   PCI_LTR_IDLE_TIMER_800US);
+			   PCI_LTR_IDLE_TIMER_3_2MS);
 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
 	rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
-	rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x88e088e0);
+	rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003);
 	rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b);
 
 	return 0;
@@ -2571,11 +2626,10 @@ static int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev)
 	}
 
 	/* enable DMA for all queues */
-	rtw89_write32_clr(rtwdev, info->dma_stop1_reg, B_AX_TX_STOP1_ALL);
-	rtw89_write32_clr(rtwdev, info->dma_stop2_reg, B_AX_TX_STOP2_ALL);
+	rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, true);
 
 	/* Release PCI IO */
-	rtw89_write32_clr(rtwdev, info->dma_stop1_reg,
+	rtw89_write32_clr(rtwdev, info->dma_stop1.addr,
 			  B_AX_STOP_WPDMA | B_AX_STOP_PCIEIO);
 
 	return 0;
@@ -2696,10 +2750,13 @@ static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,
 				    struct pci_dev *pdev)
 {
 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	struct rtw89_pci_tx_ring *tx_ring;
 	int i;
 
 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
+		if (info->tx_dma_ch_mask & BIT(i))
+			continue;
 		tx_ring = &rtwpci->tx_rings[i];
 		rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
 		rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
@@ -2887,6 +2944,7 @@ static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
 				    struct pci_dev *pdev)
 {
 	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+	const struct rtw89_pci_info *info = rtwdev->pci_info;
 	struct rtw89_pci_tx_ring *tx_ring;
 	u32 desc_size;
 	u32 len;
@@ -2894,6 +2952,8 @@ static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
 	int ret;
 
 	for (i = 0; i < RTW89_TXCH_NUM; i++) {
+		if (info->tx_dma_ch_mask & BIT(i))
+			continue;
 		tx_ring = &rtwpci->tx_rings[i];
 		desc_size = sizeof(struct rtw89_pci_tx_bd_32);
 		len = RTW89_PCI_TXBD_NUM_MAX;
@@ -3219,8 +3279,79 @@ static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,
 	pci_free_irq_vectors(pdev);
 }
 
+static u16 gray_code_to_bin(u16 gray_code, u32 bit_num)
+{
+	u16 bin = 0, gray_bit;
+	u32 bit_idx;
+
+	for (bit_idx = 0; bit_idx < bit_num; bit_idx++) {
+		gray_bit = (gray_code >> bit_idx) & 0x1;
+		if (bit_num - bit_idx > 1)
+			gray_bit ^= (gray_code >> (bit_idx + 1)) & 0x1;
+		bin |= (gray_bit << bit_idx);
+	}
+
+	return bin;
+}
+
+static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
+	struct pci_dev *pdev = rtwpci->pdev;
+	u16 val16, filter_out_val;
+	u32 val, phy_offset;
+	int ret;
+
+	if (rtwdev->chip->chip_id != RTL8852C)
+		return 0;
+
+	val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
+	if (val == B_AX_ASPM_CTRL_L1)
+		return 0;
+
+	ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val);
+	if (ret)
+		return ret;
+
+	val = FIELD_GET(RTW89_BCFG_LINK_SPEED_MASK, val);
+	if (val == RTW89_PCIE_GEN1_SPEED) {
+		phy_offset = R_RAC_DIRECT_OFFSET_G1;
+	} else if (val == RTW89_PCIE_GEN2_SPEED) {
+		phy_offset = R_RAC_DIRECT_OFFSET_G2;
+		val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT);
+		rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT,
+				  val16 | B_PCIE_BIT_PINOUT_DIS);
+		rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
+				  val16 & ~B_PCIE_BIT_RD_SEL);
+
+		val16 = rtw89_read16_mask(rtwdev,
+					  phy_offset + RAC_ANA1F * RAC_MULT,
+					  FILTER_OUT_EQ_MASK);
+		val16 = gray_code_to_bin(val16, hweight16(val16));
+		filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 *
+					      RAC_MULT);
+		filter_out_val &= ~REG_FILTER_OUT_MASK;
+		filter_out_val |= FIELD_PREP(REG_FILTER_OUT_MASK, val16);
+
+		rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT,
+			      filter_out_val);
+		rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT,
+				  B_BAC_EQ_SEL);
+		rtw89_write16_set(rtwdev,
+				  R_RAC_DIRECT_OFFSET_G1 + RAC_ANA0C * RAC_MULT,
+				  B_PCIE_BIT_PSAVE);
+	} else {
+		return -EOPNOTSUPP;
+	}
+	rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT,
+			  B_PCIE_BIT_PSAVE);
+
+	return 0;
+}
+
 static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
 {
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 	int ret;
 
 	if (rtw89_pci_disable_clkreq)
@@ -3231,19 +3362,33 @@ static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
 	if (ret)
 		rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
 
-	if (enable)
-		ret = rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_L1_CTRL,
-						RTW89_PCIE_BIT_CLK);
-	else
-		ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1_CTRL,
-						RTW89_PCIE_BIT_CLK);
-	if (ret)
-		rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
-			  enable ? "set" : "unset", ret);
+	if (chip_id == RTL8852A) {
+		if (enable)
+			ret = rtw89_pci_config_byte_set(rtwdev,
+							RTW89_PCIE_L1_CTRL,
+							RTW89_PCIE_BIT_CLK);
+		else
+			ret = rtw89_pci_config_byte_clr(rtwdev,
+							RTW89_PCIE_L1_CTRL,
+							RTW89_PCIE_BIT_CLK);
+		if (ret)
+			rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
+				  enable ? "set" : "unset", ret);
+	} else if (chip_id == RTL8852C) {
+		rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL,
+				  B_AX_CLK_REQ_SEL_OPT | B_AX_CLK_REQ_SEL);
+		if (enable)
+			rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL,
+					  B_AX_CLK_REQ_N);
+		else
+			rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL,
+					  B_AX_CLK_REQ_N);
+	}
 }
 
 static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
 {
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 	u8 value = 0;
 	int ret;
 
@@ -3262,12 +3407,23 @@ static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
 	if (ret)
 		rtw89_err(rtwdev, "failed to read ASPM Delay\n");
 
-	if (enable)
-		ret = rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_L1_CTRL,
-						RTW89_PCIE_BIT_L1);
-	else
-		ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1_CTRL,
-						RTW89_PCIE_BIT_L1);
+	if (chip_id == RTL8852A || chip_id == RTL8852B) {
+		if (enable)
+			ret = rtw89_pci_config_byte_set(rtwdev,
+							RTW89_PCIE_L1_CTRL,
+							RTW89_PCIE_BIT_L1);
+		else
+			ret = rtw89_pci_config_byte_clr(rtwdev,
+							RTW89_PCIE_L1_CTRL,
+							RTW89_PCIE_BIT_L1);
+	} else if (chip_id == RTL8852C) {
+		if (enable)
+			rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
+					  B_AX_ASPM_CTRL_L1);
+		else
+			rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
+					  B_AX_ASPM_CTRL_L1);
+	}
 	if (ret)
 		rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d",
 			  enable ? "set" : "unset", ret);
@@ -3328,17 +3484,34 @@ static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
 
 static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
 {
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 	int ret;
 
-	if (enable)
-		ret = rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_TIMER_CTRL,
-						RTW89_PCIE_BIT_L1SUB);
-	else
-		ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_TIMER_CTRL,
-						RTW89_PCIE_BIT_L1SUB);
-	if (ret)
-		rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
-			  enable ? "set" : "unset", ret);
+	if (chip_id == RTL8852A || chip_id == RTL8852B) {
+		if (enable)
+			ret = rtw89_pci_config_byte_set(rtwdev,
+							RTW89_PCIE_TIMER_CTRL,
+							RTW89_PCIE_BIT_L1SUB);
+		else
+			ret = rtw89_pci_config_byte_clr(rtwdev,
+							RTW89_PCIE_TIMER_CTRL,
+							RTW89_PCIE_BIT_L1SUB);
+		if (ret)
+			rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
+				  enable ? "set" : "unset", ret);
+	} else if (chip_id == RTL8852C) {
+		ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1,
+						RTW89_PCIE_BIT_ASPM_L11 |
+						RTW89_PCIE_BIT_PCI_L11);
+		if (ret)
+			rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret);
+		if (enable)
+			rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
+					  B_AX_L1SUB_DISABLE);
+		else
+			rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
+					  B_AX_L1SUB_DISABLE);
+	}
 }
 
 static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
@@ -3360,26 +3533,6 @@ static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
 		rtw89_pci_l1ss_set(rtwdev, true);
 }
 
-static void rtw89_pci_ctrl_dma_all_pcie(struct rtw89_dev *rtwdev, u8 en)
-{
-	const struct rtw89_pci_info *info = rtwdev->pci_info;
-	u32 val32;
-
-	if (en == MAC_AX_FUNC_EN) {
-		val32 = B_AX_STOP_PCIEIO;
-		rtw89_write32_clr(rtwdev, info->dma_stop1_reg, val32);
-
-		val32 = B_AX_TXHCI_EN | B_AX_RXHCI_EN;
-		rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
-	} else {
-		val32 = B_AX_STOP_PCIEIO;
-		rtw89_write32_set(rtwdev, info->dma_stop1_reg, val32);
-
-		val32 = B_AX_TXHCI_EN | B_AX_RXHCI_EN;
-		rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
-	}
-}
-
 static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev)
 {
 	int ret = 0;
@@ -3399,10 +3552,13 @@ static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev)
 
 static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)
 {
-	u32 val, dma_rst = 0;
+	u32 val;
 	int ret;
 
-	rtw89_pci_ctrl_dma_all_pcie(rtwdev, MAC_AX_FUNC_DIS);
+	if (rtwdev->chip->chip_id == RTL8852C)
+		return 0;
+
+	rtw89_pci_ctrl_dma_all(rtwdev, false);
 	ret = rtw89_pci_poll_io_idle(rtwdev);
 	if (ret) {
 		val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
@@ -3410,12 +3566,10 @@ static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)
 			    "[PCIe] poll_io_idle fail, before 0x%08x: 0x%08x\n",
 			    R_AX_DBG_ERR_FLAG, val);
 		if (val & B_AX_TX_STUCK || val & B_AX_PCIE_TXBD_LEN0)
-			dma_rst |= B_AX_HCI_TXDMA_EN;
+			rtw89_mac_ctrl_hci_dma_tx(rtwdev, false);
 		if (val & B_AX_RX_STUCK)
-			dma_rst |= B_AX_HCI_RXDMA_EN;
-		val = rtw89_read32(rtwdev, R_AX_HCI_FUNC_EN);
-		rtw89_write32(rtwdev, R_AX_HCI_FUNC_EN, val & ~dma_rst);
-		rtw89_write32(rtwdev, R_AX_HCI_FUNC_EN, val | dma_rst);
+			rtw89_mac_ctrl_hci_dma_rx(rtwdev, false);
+		rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
 		ret = rtw89_pci_poll_io_idle(rtwdev);
 		val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
 		rtw89_debug(rtwdev, RTW89_DBG_HCI,
@@ -3426,18 +3580,7 @@ static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev)
 	return ret;
 }
 
-static void rtw89_pci_ctrl_hci_dma_en(struct rtw89_dev *rtwdev, u8 en)
-{
-	u32 val32;
 
-	if (en == MAC_AX_FUNC_EN) {
-		val32 = B_AX_HCI_TXDMA_EN | B_AX_HCI_RXDMA_EN;
-		rtw89_write32_set(rtwdev, R_AX_HCI_FUNC_EN, val32);
-	} else {
-		val32 = B_AX_HCI_TXDMA_EN | B_AX_HCI_RXDMA_EN;
-		rtw89_write32_clr(rtwdev, R_AX_HCI_FUNC_EN, val32);
-	}
-}
 
 static int rtw89_pci_rst_bdram(struct rtw89_dev *rtwdev)
 {
@@ -3457,15 +3600,18 @@ static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev)
 {
 	u32 ret;
 
-	rtw89_pci_ctrl_hci_dma_en(rtwdev, MAC_AX_FUNC_DIS);
-	rtw89_pci_ctrl_hci_dma_en(rtwdev, MAC_AX_FUNC_EN);
+	if (rtwdev->chip->chip_id == RTL8852C)
+		return 0;
+
+	rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
+	rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
 	rtw89_pci_clr_idx_all(rtwdev);
 
 	ret = rtw89_pci_rst_bdram(rtwdev);
 	if (ret)
 		return ret;
 
-	rtw89_pci_ctrl_dma_all_pcie(rtwdev, MAC_AX_FUNC_EN);
+	rtw89_pci_ctrl_dma_all(rtwdev, true);
 	return ret;
 }
 
@@ -3535,14 +3681,20 @@ static int __maybe_unused rtw89_pci_suspend(struct device *dev)
 {
 	struct ieee80211_hw *hw = dev_get_drvdata(dev);
 	struct rtw89_dev *rtwdev = hw->priv;
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 
-	rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
-			  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
-	rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
-			  B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
+	if (chip_id == RTL8852A || chip_id == RTL8852B) {
+		rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
+				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
+		rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
+				  B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
+	} else {
+		rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
+				  B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
+	}
 
 	return 0;
 }
@@ -3563,15 +3715,24 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev)
 {
 	struct ieee80211_hw *hw = dev_get_drvdata(dev);
 	struct rtw89_dev *rtwdev = hw->priv;
+	enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
 
-	rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
-			  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
 	rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
 	rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
-	rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
-			  B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
+	if (chip_id == RTL8852A || chip_id == RTL8852B) {
+		rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
+				  B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
+		rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
+				  B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
+	} else {
+		rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1,
+				  B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
+		rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
+				  B_AX_SEL_REQ_ENTR_L1);
+	}
 	rtw89_pci_l2_hci_ldo(rtwdev);
+	rtw89_pci_filter_out(rtwdev);
 	rtw89_pci_link_cfg(rtwdev);
 	rtw89_pci_l1ss_cfg(rtwdev);
 
@@ -3614,27 +3775,23 @@ static const struct rtw89_hci_ops rtw89_pci_ops = {
 
 int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
-	struct ieee80211_hw *hw;
 	struct rtw89_dev *rtwdev;
 	const struct rtw89_driver_info *info;
 	const struct rtw89_pci_info *pci_info;
-	int driver_data_size;
 	int ret;
 
-	driver_data_size = sizeof(struct rtw89_dev) + sizeof(struct rtw89_pci);
-	hw = ieee80211_alloc_hw(driver_data_size, &rtw89_ops);
-	if (!hw) {
+	info = (const struct rtw89_driver_info *)id->driver_data;
+
+	rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev,
+					  sizeof(struct rtw89_pci),
+					  info->chip);
+	if (!rtwdev) {
 		dev_err(&pdev->dev, "failed to allocate hw\n");
 		return -ENOMEM;
 	}
 
-	info = (const struct rtw89_driver_info *)id->driver_data;
 	pci_info = info->bus.pci;
 
-	rtwdev = hw->priv;
-	rtwdev->hw = hw;
-	rtwdev->dev = &pdev->dev;
-	rtwdev->chip = info->chip;
 	rtwdev->pci_info = info->bus.pci;
 	rtwdev->hci.ops = &rtw89_pci_ops;
 	rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
@@ -3667,6 +3824,7 @@ int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto err_clear_resource;
 	}
 
+	rtw89_pci_filter_out(rtwdev);
 	rtw89_pci_link_cfg(rtwdev);
 	rtw89_pci_l1ss_cfg(rtwdev);
 
@@ -3696,7 +3854,7 @@ err_declaim_pci:
 err_core_deinit:
 	rtw89_core_deinit(rtwdev);
 err_release_hw:
-	ieee80211_free_hw(hw);
+	rtw89_free_ieee80211_hw(rtwdev);
 
 	return ret;
 }
@@ -3715,7 +3873,7 @@ void rtw89_pci_remove(struct pci_dev *pdev)
 	rtw89_pci_clear_resource(rtwdev, pdev);
 	rtw89_pci_declaim_device(rtwdev, pdev);
 	rtw89_core_deinit(rtwdev);
-	ieee80211_free_hw(hw);
+	rtw89_free_ieee80211_hw(rtwdev);
 }
 EXPORT_SYMBOL(rtw89_pci_remove);
 
diff --git a/drivers/net/wireless/realtek/rtw89/pci.h b/drivers/net/wireless/realtek/rtw89/pci.h
index a118647213e3..179740607778 100644
--- a/drivers/net/wireless/realtek/rtw89/pci.h
+++ b/drivers/net/wireless/realtek/rtw89/pci.h
@@ -11,11 +11,21 @@
 #define MDIO_PG1_G1 1
 #define MDIO_PG0_G2 2
 #define MDIO_PG1_G2 3
+#define RAC_CTRL_PPR			0x00
+#define RAC_ANA0A			0x0A
+#define B_BAC_EQ_SEL			BIT(5)
+#define RAC_ANA0C			0x0C
+#define B_PCIE_BIT_PSAVE		BIT(15)
 #define RAC_ANA10			0x10
+#define B_PCIE_BIT_PINOUT_DIS		BIT(3)
 #define RAC_REG_REV2			0x1B
 #define BAC_CMU_EN_DLY_MASK		GENMASK(15, 12)
 #define PCIE_DPHY_DLY_25US		0x1
 #define RAC_ANA19			0x19
+#define B_PCIE_BIT_RD_SEL		BIT(2)
+#define RAC_REG_FLD_0			0x1D
+#define BAC_AUTOK_N_MASK		GENMASK(3, 2)
+#define PCIE_AUTOK_4			0x3
 #define RAC_ANA1F			0x1F
 #define RAC_ANA24			0x24
 #define B_AX_DEGLITCH			GENMASK(11, 8)
@@ -45,9 +55,26 @@
 #define B_AX_SEL_REQ_ENTR_L1		BIT(2)
 #define B_AX_SEL_REQ_EXIT_L1		BIT(0)
 
+#define R_AX_PCIE_MIX_CFG_V1		0x300C
+#define B_AX_ASPM_CTRL_L1		BIT(17)
+#define B_AX_ASPM_CTRL_L0		BIT(16)
+#define B_AX_ASPM_CTRL_MASK		GENMASK(17, 16)
+#define B_AX_XFER_PENDING_FW		BIT(11)
+#define B_AX_XFER_PENDING		BIT(10)
+#define B_AX_REQ_EXIT_L1		BIT(9)
+#define B_AX_REQ_ENTR_L1		BIT(8)
+#define B_AX_L1SUB_DISABLE		BIT(0)
+
+#define R_AX_L1_CLK_CTRL		0x3010
+#define B_AX_CLK_REQ_N			BIT(1)
+
 #define R_AX_PCIE_BG_CLR		0x303C
 #define B_AX_BG_CLR_ASYNC_M3		BIT(4)
 
+#define R_AX_PCIE_LAT_CTRL		0x3044
+#define B_AX_CLK_REQ_SEL_OPT		BIT(1)
+#define B_AX_CLK_REQ_SEL		BIT(0)
+
 #define R_AX_PCIE_IO_RCY_M1 0x3100
 #define B_AX_PCIE_IO_RCY_P_M1 BIT(5)
 #define B_AX_PCIE_IO_RCY_WDT_P_M1 BIT(4)
@@ -88,7 +115,10 @@
 #define B_AX_PCIE_WDT_TIMER_S1_MASK GENMASK(31, 0)
 
 #define R_RAC_DIRECT_OFFSET_G1 0x3800
+#define FILTER_OUT_EQ_MASK GENMASK(14, 10)
 #define R_RAC_DIRECT_OFFSET_G2 0x3880
+#define REG_FILTER_OUT_MASK GENMASK(6, 2)
+#define RAC_MULT 2
 
 #define RTW89_PCI_WR_RETRY_CNT		20
 
@@ -383,6 +413,16 @@
 #define B_AX_STOP_RPQ			BIT(1)
 #define B_AX_STOP_RXQ			BIT(0)
 #define B_AX_TX_STOP1_ALL		GENMASK(18, 8)
+#define B_AX_TX_STOP1_MASK		(B_AX_STOP_ACH0 | B_AX_STOP_ACH1 | \
+					 B_AX_STOP_ACH2 | B_AX_STOP_ACH3 | \
+					 B_AX_STOP_ACH4 | B_AX_STOP_ACH5 | \
+					 B_AX_STOP_ACH6 | B_AX_STOP_ACH7 | \
+					 B_AX_STOP_CH8 | B_AX_STOP_CH9 | \
+					 B_AX_STOP_CH12)
+#define B_AX_TX_STOP1_MASK_V1		(B_AX_STOP_ACH0 | B_AX_STOP_ACH1 | \
+					 B_AX_STOP_ACH2 | B_AX_STOP_ACH3 | \
+					 B_AX_STOP_CH8 | B_AX_STOP_CH9 | \
+					 B_AX_STOP_CH12)
 
 #define R_AX_PCIE_DMA_STOP2	0x1310
 #define B_AX_STOP_CH11			BIT(1)
@@ -431,6 +471,13 @@
 #define B_AX_ACH0_BUSY			BIT(8)
 #define B_AX_RPQ_BUSY			BIT(1)
 #define B_AX_RXQ_BUSY			BIT(0)
+#define DMA_BUSY1_CHECK		(B_AX_ACH0_BUSY | B_AX_ACH1_BUSY | B_AX_ACH2_BUSY | \
+				 B_AX_ACH3_BUSY | B_AX_ACH4_BUSY | B_AX_ACH5_BUSY | \
+				 B_AX_ACH6_BUSY | B_AX_ACH7_BUSY | B_AX_CH8_BUSY | \
+				 B_AX_CH9_BUSY | B_AX_CH12_BUSY)
+#define DMA_BUSY1_CHECK_V1	(B_AX_ACH0_BUSY | B_AX_ACH1_BUSY | B_AX_ACH2_BUSY | \
+				 B_AX_ACH3_BUSY | B_AX_CH8_BUSY | B_AX_CH9_BUSY | \
+				 B_AX_CH12_BUSY)
 
 #define R_AX_PCIE_DMA_BUSY2	0x131C
 #define B_AX_CH11_BUSY			BIT(1)
@@ -505,6 +552,17 @@
 #define RTW89_PCI_MULTITAG		8
 
 /* PCIE CFG register */
+#define RTW89_PCIE_L1_STS_V1		0x80
+#define RTW89_BCFG_LINK_SPEED_MASK	GENMASK(19, 16)
+#define RTW89_PCIE_GEN1_SPEED		0x01
+#define RTW89_PCIE_GEN2_SPEED		0x02
+#define RTW89_PCIE_PHY_RATE		0x82
+#define RTW89_PCIE_PHY_RATE_MASK	GENMASK(1, 0)
+#define RTW89_PCIE_L1SS_STS_V1		0x0168
+#define RTW89_PCIE_BIT_ASPM_L11		BIT(3)
+#define RTW89_PCIE_BIT_ASPM_L12		BIT(2)
+#define RTW89_PCIE_BIT_PCI_L11		BIT(1)
+#define RTW89_PCIE_BIT_PCI_L12		BIT(0)
 #define RTW89_PCIE_ASPM_CTRL		0x070F
 #define RTW89_L1DLY_MASK		GENMASK(5, 3)
 #define RTW89_L0DLY_MASK		GENMASK(2, 0)
@@ -516,8 +574,7 @@
 #define RTW89_PCIE_CLK_CTRL		0x0725
 #define RTW89_PCIE_RST_MSTATE		0x0B48
 #define RTW89_PCIE_BIT_CFG_RST_MSTATE	BIT(0)
-#define RTW89_PCIE_PHY_RATE		0x82
-#define RTW89_PCIE_PHY_RATE_MASK	GENMASK(1, 0)
+
 #define INTF_INTGRA_MINREF_V1	90
 #define INTF_INTGRA_HOSTREF_V1	100
 
@@ -527,11 +584,6 @@ enum rtw89_pcie_phy {
 	PCIE_PHY_GEN1_UNDEFINE = 0x7F,
 };
 
-enum mac_ax_func_sw {
-	MAC_AX_FUNC_DIS,
-	MAC_AX_FUNC_EN,
-};
-
 enum rtw89_pcie_l0sdly {
 	PCIE_L0SDLY_1US = 0,
 	PCIE_L0SDLY_2US = 1,
@@ -710,14 +762,15 @@ struct rtw89_pci_info {
 	u32 max_tag_num_mask;
 	u32 rxbd_rwptr_clr_reg;
 	u32 txbd_rwptr_clr2_reg;
-	u32 dma_stop1_reg;
-	u32 dma_stop2_reg;
-	u32 dma_busy1_reg;
+	struct rtw89_reg_def dma_stop1;
+	struct rtw89_reg_def dma_stop2;
+	struct rtw89_reg_def dma_busy1;
 	u32 dma_busy2_reg;
 	u32 dma_busy3_reg;
 
 	u32 rpwm_addr;
 	u32 cpwm_addr;
+	u32 tx_dma_ch_mask;
 	const struct rtw89_pci_bd_idx_addr *bd_idx_addr_low_power;
 	const struct rtw89_pci_ch_dma_addr_set *dma_addr_set;
 
diff --git a/drivers/net/wireless/realtek/rtw89/phy.c b/drivers/net/wireless/realtek/rtw89/phy.c
index 1532c0a6bbc4..6a6bdc652e09 100644
--- a/drivers/net/wireless/realtek/rtw89/phy.c
+++ b/drivers/net/wireless/realtek/rtw89/phy.c
@@ -14,23 +14,14 @@
 static u16 get_max_amsdu_len(struct rtw89_dev *rtwdev,
 			     const struct rtw89_ra_report *report)
 {
-	const struct rate_info *txrate = &report->txrate;
 	u32 bit_rate = report->bit_rate;
-	u8 mcs;
 
 	/* lower than ofdm, do not aggregate */
 	if (bit_rate < 550)
 		return 1;
 
-	/* prevent hardware rate fallback to G mode rate */
-	if (txrate->flags & RATE_INFO_FLAGS_MCS)
-		mcs = txrate->mcs & 0x07;
-	else if (txrate->flags & (RATE_INFO_FLAGS_VHT_MCS | RATE_INFO_FLAGS_HE_MCS))
-		mcs = txrate->mcs;
-	else
-		mcs = 0;
-
-	if (mcs <= 2)
+	/* avoid AMSDU for legacy rate */
+	if (report->might_fallback_legacy)
 		return 1;
 
 	/* lower than 20M vht 2ss mcs8, make it small */
@@ -142,8 +133,8 @@ static u64 rtw89_phy_ra_mask_recover(u64 ra_mask, u64 ra_mask_bak)
 
 static u64 rtw89_phy_ra_mask_cfg(struct rtw89_dev *rtwdev, struct rtw89_sta *rtwsta)
 {
-	struct rtw89_hal *hal = &rtwdev->hal;
 	struct ieee80211_sta *sta = rtwsta_to_sta(rtwsta);
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	struct cfg80211_bitrate_mask *mask = &rtwsta->mask;
 	enum nl80211_band band;
 	u64 cfg_mask;
@@ -151,7 +142,7 @@ static u64 rtw89_phy_ra_mask_cfg(struct rtw89_dev *rtwdev, struct rtw89_sta *rtw
 	if (!rtwsta->use_cfg_mask)
 		return -1;
 
-	switch (hal->current_band_type) {
+	switch (chan->band_type) {
 	case RTW89_BAND_2G:
 		band = NL80211_BAND_2GHZ;
 		cfg_mask = u64_encode_bits(mask->control[NL80211_BAND_2GHZ].legacy,
@@ -168,7 +159,7 @@ static u64 rtw89_phy_ra_mask_cfg(struct rtw89_dev *rtwdev, struct rtw89_sta *rtw
 					   RA_MASK_OFDM_RATES);
 		break;
 	default:
-		rtw89_warn(rtwdev, "unhandled band type %d\n", hal->current_band_type);
+		rtw89_warn(rtwdev, "unhandled band type %d\n", chan->band_type);
 		return -1;
 	}
 
@@ -202,6 +193,40 @@ static const u64
 rtw89_ra_mask_he_rates[4] = {RA_MASK_HE_1SS_RATES, RA_MASK_HE_2SS_RATES,
 			     RA_MASK_HE_3SS_RATES, RA_MASK_HE_4SS_RATES};
 
+static void rtw89_phy_ra_gi_ltf(struct rtw89_dev *rtwdev,
+				struct rtw89_sta *rtwsta,
+				bool *fix_giltf_en, u8 *fix_giltf)
+{
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	struct cfg80211_bitrate_mask *mask = &rtwsta->mask;
+	u8 band = chan->band_type;
+	enum nl80211_band nl_band = rtw89_hw_to_nl80211_band(band);
+	u8 he_gi = mask->control[nl_band].he_gi;
+	u8 he_ltf = mask->control[nl_band].he_ltf;
+
+	if (!rtwsta->use_cfg_mask)
+		return;
+
+	if (he_ltf == 2 && he_gi == 2) {
+		*fix_giltf = RTW89_GILTF_LGI_4XHE32;
+	} else if (he_ltf == 2 && he_gi == 0) {
+		*fix_giltf = RTW89_GILTF_SGI_4XHE08;
+	} else if (he_ltf == 1 && he_gi == 1) {
+		*fix_giltf = RTW89_GILTF_2XHE16;
+	} else if (he_ltf == 1 && he_gi == 0) {
+		*fix_giltf = RTW89_GILTF_2XHE08;
+	} else if (he_ltf == 0 && he_gi == 1) {
+		*fix_giltf = RTW89_GILTF_1XHE16;
+	} else if (he_ltf == 0 && he_gi == 0) {
+		*fix_giltf = RTW89_GILTF_1XHE08;
+	} else {
+		*fix_giltf_en = false;
+		return;
+	}
+
+	*fix_giltf_en = true;
+}
+
 static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev,
 				    struct ieee80211_sta *sta, bool csi)
 {
@@ -209,6 +234,8 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev,
 	struct rtw89_vif *rtwvif = rtwsta->rtwvif;
 	struct rtw89_phy_rate_pattern *rate_pattern = &rtwvif->rate_pattern;
 	struct rtw89_ra_info *ra = &rtwsta->ra;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	struct ieee80211_vif *vif = rtwvif_to_vif(rtwsta->rtwvif);
 	const u64 *high_rate_masks = rtw89_ra_mask_ht_rates;
 	u8 rssi = ewma_rssi_read(&rtwsta->avg_rssi);
 	u64 ra_mask = 0;
@@ -218,8 +245,10 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev,
 	u8 bw_mode = 0;
 	u8 stbc_en = 0;
 	u8 ldpc_en = 0;
+	u8 fix_giltf = 0;
 	u8 i;
 	bool sgi = false;
+	bool fix_giltf_en = false;
 
 	memset(ra, 0, sizeof(*ra));
 	/* Set the ra mask from sta's capability */
@@ -234,6 +263,7 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev,
 		if (sta->deflink.he_cap.he_cap_elem.phy_cap_info[1] &
 		    IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD)
 			ldpc_en = 1;
+		rtw89_phy_ra_gi_ltf(rtwdev, rtwsta, &fix_giltf_en, &fix_giltf);
 	} else if (sta->deflink.vht_cap.vht_supported) {
 		u16 mcs_map = le16_to_cpu(sta->deflink.vht_cap.vht_mcs.rx_mcs_map);
 
@@ -260,13 +290,13 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev,
 			ldpc_en = 1;
 	}
 
-	switch (rtwdev->hal.current_band_type) {
+	switch (chan->band_type) {
 	case RTW89_BAND_2G:
 		ra_mask |= sta->deflink.supp_rates[NL80211_BAND_2GHZ];
-		if (sta->deflink.supp_rates[NL80211_BAND_2GHZ] <= 0xf)
+		if (sta->deflink.supp_rates[NL80211_BAND_2GHZ] & 0xf)
 			mode |= RTW89_RA_MODE_CCK;
-		else
-			mode |= RTW89_RA_MODE_CCK | RTW89_RA_MODE_OFDM;
+		if (sta->deflink.supp_rates[NL80211_BAND_2GHZ] & 0xff0)
+			mode |= RTW89_RA_MODE_OFDM;
 		break;
 	case RTW89_BAND_5G:
 		ra_mask |= (u64)sta->deflink.supp_rates[NL80211_BAND_5GHZ] << 4;
@@ -329,7 +359,7 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev,
 	    IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_16_QAM)
 		ra->dcm_cap = 1;
 
-	if (rate_pattern->enable) {
+	if (rate_pattern->enable && !vif->p2p) {
 		ra_mask = rtw89_phy_ra_mask_cfg(rtwdev, rtwsta);
 		ra_mask &= rate_pattern->ra_mask;
 		mode = rate_pattern->ra_mode;
@@ -343,6 +373,8 @@ static void rtw89_phy_ra_sta_update(struct rtw89_dev *rtwdev,
 	ra->ss_num = min(sta->deflink.rx_nss, rtwdev->hal.tx_nss) - 1;
 	ra->en_sgi = sgi;
 	ra->ra_mask = ra_mask;
+	ra->fix_giltf_en = fix_giltf_en;
+	ra->fix_giltf = fix_giltf;
 
 	if (!csi)
 		return;
@@ -416,6 +448,7 @@ void rtw89_phy_rate_pattern_vif(struct rtw89_dev *rtwdev,
 	struct ieee80211_supported_band *sband;
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
 	struct rtw89_phy_rate_pattern next_pattern = {0};
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	static const u16 hw_rate_he[] = {RTW89_HW_RATE_HE_NSS1_MCS0,
 					 RTW89_HW_RATE_HE_NSS2_MCS0,
 					 RTW89_HW_RATE_HE_NSS3_MCS0,
@@ -428,7 +461,7 @@ void rtw89_phy_rate_pattern_vif(struct rtw89_dev *rtwdev,
 					 RTW89_HW_RATE_MCS8,
 					 RTW89_HW_RATE_MCS16,
 					 RTW89_HW_RATE_MCS24};
-	u8 band = rtwdev->hal.current_band_type;
+	u8 band = chan->band_type;
 	enum nl80211_band nl_band = rtw89_hw_to_nl80211_band(band);
 	u8 tx_nss = rtwdev->hal.tx_nss;
 	u8 i;
@@ -542,12 +575,12 @@ void rtw89_phy_ra_assoc(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta)
 }
 
 u8 rtw89_phy_get_txsc(struct rtw89_dev *rtwdev,
-		      struct rtw89_channel_params *param,
+		      const struct rtw89_chan *chan,
 		      enum rtw89_bandwidth dbw)
 {
-	enum rtw89_bandwidth cbw = param->bandwidth;
-	u8 pri_ch = param->primary_chan;
-	u8 central_ch = param->center_chan;
+	enum rtw89_bandwidth cbw = chan->band_width;
+	u8 pri_ch = chan->primary_channel;
+	u8 central_ch = chan->channel;
 	u8 txsc_idx = 0;
 	u8 tmp = 0;
 
@@ -1468,10 +1501,9 @@ EXPORT_SYMBOL(rtw89_phy_load_txpwr_byrate);
 	(txpwr_rf) >> (__c->txpwr_factor_rf - __c->txpwr_factor_mac);	\
 })
 
-s8 rtw89_phy_read_txpwr_byrate(struct rtw89_dev *rtwdev,
+s8 rtw89_phy_read_txpwr_byrate(struct rtw89_dev *rtwdev, u8 band,
 			       const struct rtw89_rate_desc *rate_desc)
 {
-	enum rtw89_band band = rtwdev->hal.current_band_type;
 	s8 *byr;
 	u8 idx;
 
@@ -1538,11 +1570,10 @@ static u8 rtw89_channel_to_idx(struct rtw89_dev *rtwdev, u8 band, u8 channel)
 	}
 }
 
-s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev,
+s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, u8 band,
 			      u8 bw, u8 ntx, u8 rs, u8 bf, u8 ch)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
-	u8 band = rtwdev->hal.current_band_type;
 	u8 ch_idx = rtw89_channel_to_idx(rtwdev, band, ch);
 	u8 regd = rtw89_regd_get(rtwdev, band);
 	s8 lmt = 0, sar;
@@ -1578,11 +1609,12 @@ s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev,
 }
 EXPORT_SYMBOL(rtw89_phy_read_txpwr_limit);
 
-#define __fill_txpwr_limit_nonbf_bf(ptr, bw, ntx, rs, ch)		\
+#define __fill_txpwr_limit_nonbf_bf(ptr, band, bw, ntx, rs, ch)		\
 	do {								\
 		u8 __i;							\
 		for (__i = 0; __i < RTW89_BF_NUM; __i++)		\
 			ptr[__i] = rtw89_phy_read_txpwr_limit(rtwdev,	\
+							      band,	\
 							      bw, ntx,	\
 							      rs, __i,	\
 							      (ch));	\
@@ -1590,64 +1622,75 @@ EXPORT_SYMBOL(rtw89_phy_read_txpwr_limit);
 
 static void rtw89_phy_fill_txpwr_limit_20m(struct rtw89_dev *rtwdev,
 					   struct rtw89_txpwr_limit *lmt,
-					   u8 ntx, u8 ch)
+					   u8 band, u8 ntx, u8 ch)
 {
-	__fill_txpwr_limit_nonbf_bf(lmt->cck_20m, RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->cck_20m, band, RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_CCK, ch);
-	__fill_txpwr_limit_nonbf_bf(lmt->cck_40m, RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(lmt->cck_40m, band, RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_CCK, ch);
-	__fill_txpwr_limit_nonbf_bf(lmt->ofdm, RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->ofdm, band, RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_OFDM, ch);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch);
 }
 
 static void rtw89_phy_fill_txpwr_limit_40m(struct rtw89_dev *rtwdev,
 					   struct rtw89_txpwr_limit *lmt,
-					   u8 ntx, u8 ch, u8 pri_ch)
+					   u8 band, u8 ntx, u8 ch, u8 pri_ch)
 {
-	__fill_txpwr_limit_nonbf_bf(lmt->cck_20m, RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->cck_20m, band, RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_CCK, ch - 2);
-	__fill_txpwr_limit_nonbf_bf(lmt->cck_40m, RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(lmt->cck_40m, band, RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_CCK, ch);
-	__fill_txpwr_limit_nonbf_bf(lmt->ofdm, RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->ofdm, band, RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_OFDM, pri_ch);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch - 2);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch + 2);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], band,
+				    RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch);
 }
 
 static void rtw89_phy_fill_txpwr_limit_80m(struct rtw89_dev *rtwdev,
 					   struct rtw89_txpwr_limit *lmt,
-					   u8 ntx, u8 ch, u8 pri_ch)
+					   u8 band, u8 ntx, u8 ch, u8 pri_ch)
 {
 	s8 val_0p5_n[RTW89_BF_NUM];
 	s8 val_0p5_p[RTW89_BF_NUM];
 	u8 i;
 
-	__fill_txpwr_limit_nonbf_bf(lmt->ofdm, RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->ofdm, band, RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_OFDM, pri_ch);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch - 6);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch - 2);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[2], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[2], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch + 2);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[3], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[3], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch + 6);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], band,
+				    RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch - 4);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[1], RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[1], band,
+				    RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch + 4);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[0], RTW89_CHANNEL_WIDTH_80,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[0], band,
+				    RTW89_CHANNEL_WIDTH_80,
 				    ntx, RTW89_RS_MCS, ch);
 
-	__fill_txpwr_limit_nonbf_bf(val_0p5_n, RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(val_0p5_n, band, RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch - 4);
-	__fill_txpwr_limit_nonbf_bf(val_0p5_p, RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(val_0p5_p, band, RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch + 4);
 
 	for (i = 0; i < RTW89_BF_NUM; i++)
@@ -1656,7 +1699,7 @@ static void rtw89_phy_fill_txpwr_limit_80m(struct rtw89_dev *rtwdev,
 
 static void rtw89_phy_fill_txpwr_limit_160m(struct rtw89_dev *rtwdev,
 					    struct rtw89_txpwr_limit *lmt,
-					    u8 ntx, u8 ch, u8 pri_ch)
+					    u8 band, u8 ntx, u8 ch, u8 pri_ch)
 {
 	s8 val_0p5_n[RTW89_BF_NUM];
 	s8 val_0p5_p[RTW89_BF_NUM];
@@ -1665,60 +1708,75 @@ static void rtw89_phy_fill_txpwr_limit_160m(struct rtw89_dev *rtwdev,
 	u8 i;
 
 	/* fill ofdm section */
-	__fill_txpwr_limit_nonbf_bf(lmt->ofdm, RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->ofdm, band, RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_OFDM, pri_ch);
 
 	/* fill mcs 20m section */
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[0], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch - 14);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[1], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch - 10);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[2], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[2], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch - 6);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[3], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[3], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch - 2);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[4], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[4], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch + 2);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[5], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[5], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch + 6);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[6], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[6], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch + 10);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[7], RTW89_CHANNEL_WIDTH_20,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_20m[7], band,
+				    RTW89_CHANNEL_WIDTH_20,
 				    ntx, RTW89_RS_MCS, ch + 14);
 
 	/* fill mcs 40m section */
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[0], band,
+				    RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch - 12);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[1], RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[1], band,
+				    RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch - 4);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[2], RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[2], band,
+				    RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch + 4);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[3], RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_40m[3], band,
+				    RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch + 12);
 
 	/* fill mcs 80m section */
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[0], RTW89_CHANNEL_WIDTH_80,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[0], band,
+				    RTW89_CHANNEL_WIDTH_80,
 				    ntx, RTW89_RS_MCS, ch - 8);
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[1], RTW89_CHANNEL_WIDTH_80,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_80m[1], band,
+				    RTW89_CHANNEL_WIDTH_80,
 				    ntx, RTW89_RS_MCS, ch + 8);
 
 	/* fill mcs 160m section */
-	__fill_txpwr_limit_nonbf_bf(lmt->mcs_160m, RTW89_CHANNEL_WIDTH_160,
+	__fill_txpwr_limit_nonbf_bf(lmt->mcs_160m, band,
+				    RTW89_CHANNEL_WIDTH_160,
 				    ntx, RTW89_RS_MCS, ch);
 
 	/* fill mcs 40m 0p5 section */
-	__fill_txpwr_limit_nonbf_bf(val_0p5_n, RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(val_0p5_n, band, RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch - 4);
-	__fill_txpwr_limit_nonbf_bf(val_0p5_p, RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(val_0p5_p, band, RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch + 4);
 
 	for (i = 0; i < RTW89_BF_NUM; i++)
 		lmt->mcs_40m_0p5[i] = min_t(s8, val_0p5_n[i], val_0p5_p[i]);
 
 	/* fill mcs 40m 2p5 section */
-	__fill_txpwr_limit_nonbf_bf(val_2p5_n, RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(val_2p5_n, band, RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch - 8);
-	__fill_txpwr_limit_nonbf_bf(val_2p5_p, RTW89_CHANNEL_WIDTH_40,
+	__fill_txpwr_limit_nonbf_bf(val_2p5_p, band, RTW89_CHANNEL_WIDTH_40,
 				    ntx, RTW89_RS_MCS, ch + 8);
 
 	for (i = 0; i < RTW89_BF_NUM; i++)
@@ -1726,37 +1784,41 @@ static void rtw89_phy_fill_txpwr_limit_160m(struct rtw89_dev *rtwdev,
 }
 
 void rtw89_phy_fill_txpwr_limit(struct rtw89_dev *rtwdev,
+				const struct rtw89_chan *chan,
 				struct rtw89_txpwr_limit *lmt,
 				u8 ntx)
 {
-	u8 pri_ch = rtwdev->hal.current_primary_channel;
-	u8 ch = rtwdev->hal.current_channel;
-	u8 bw = rtwdev->hal.current_band_width;
+	u8 band = chan->band_type;
+	u8 pri_ch = chan->primary_channel;
+	u8 ch = chan->channel;
+	u8 bw = chan->band_width;
 
 	memset(lmt, 0, sizeof(*lmt));
 
 	switch (bw) {
 	case RTW89_CHANNEL_WIDTH_20:
-		rtw89_phy_fill_txpwr_limit_20m(rtwdev, lmt, ntx, ch);
+		rtw89_phy_fill_txpwr_limit_20m(rtwdev, lmt, band, ntx, ch);
 		break;
 	case RTW89_CHANNEL_WIDTH_40:
-		rtw89_phy_fill_txpwr_limit_40m(rtwdev, lmt, ntx, ch, pri_ch);
+		rtw89_phy_fill_txpwr_limit_40m(rtwdev, lmt, band, ntx, ch,
+					       pri_ch);
 		break;
 	case RTW89_CHANNEL_WIDTH_80:
-		rtw89_phy_fill_txpwr_limit_80m(rtwdev, lmt, ntx, ch, pri_ch);
+		rtw89_phy_fill_txpwr_limit_80m(rtwdev, lmt, band, ntx, ch,
+					       pri_ch);
 		break;
 	case RTW89_CHANNEL_WIDTH_160:
-		rtw89_phy_fill_txpwr_limit_160m(rtwdev, lmt, ntx, ch, pri_ch);
+		rtw89_phy_fill_txpwr_limit_160m(rtwdev, lmt, band, ntx, ch,
+						pri_ch);
 		break;
 	}
 }
 EXPORT_SYMBOL(rtw89_phy_fill_txpwr_limit);
 
-static s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev,
+static s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev, u8 band,
 					u8 ru, u8 ntx, u8 ch)
 {
 	const struct rtw89_chip_info *chip = rtwdev->chip;
-	u8 band = rtwdev->hal.current_band_type;
 	u8 ch_idx = rtw89_channel_to_idx(rtwdev, band, ch);
 	u8 regd = rtw89_regd_get(rtwdev, band);
 	s8 lmt_ru = 0, sar;
@@ -1794,85 +1856,106 @@ static s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev,
 static void
 rtw89_phy_fill_txpwr_limit_ru_20m(struct rtw89_dev *rtwdev,
 				  struct rtw89_txpwr_limit_ru *lmt_ru,
-				  u8 ntx, u8 ch)
+				  u8 band, u8 ntx, u8 ch)
 {
-	lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26,
+	lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU26,
 							ntx, ch);
-	lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52,
+	lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU52,
 							ntx, ch);
-	lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106,
+	lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							 RTW89_RU106,
 							 ntx, ch);
 }
 
 static void
 rtw89_phy_fill_txpwr_limit_ru_40m(struct rtw89_dev *rtwdev,
 				  struct rtw89_txpwr_limit_ru *lmt_ru,
-				  u8 ntx, u8 ch)
+				  u8 band, u8 ntx, u8 ch)
 {
-	lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26,
+	lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU26,
 							ntx, ch - 2);
-	lmt_ru->ru26[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26,
+	lmt_ru->ru26[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU26,
 							ntx, ch + 2);
-	lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52,
+	lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU52,
 							ntx, ch - 2);
-	lmt_ru->ru52[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52,
+	lmt_ru->ru52[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU52,
 							ntx, ch + 2);
-	lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106,
+	lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							 RTW89_RU106,
 							 ntx, ch - 2);
-	lmt_ru->ru106[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106,
+	lmt_ru->ru106[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							 RTW89_RU106,
 							 ntx, ch + 2);
 }
 
 static void
 rtw89_phy_fill_txpwr_limit_ru_80m(struct rtw89_dev *rtwdev,
 				  struct rtw89_txpwr_limit_ru *lmt_ru,
-				  u8 ntx, u8 ch)
+				  u8 band, u8 ntx, u8 ch)
 {
-	lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26,
+	lmt_ru->ru26[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU26,
 							ntx, ch - 6);
-	lmt_ru->ru26[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26,
+	lmt_ru->ru26[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU26,
 							ntx, ch - 2);
-	lmt_ru->ru26[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26,
+	lmt_ru->ru26[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU26,
 							ntx, ch + 2);
-	lmt_ru->ru26[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU26,
+	lmt_ru->ru26[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU26,
 							ntx, ch + 6);
-	lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52,
+	lmt_ru->ru52[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU52,
 							ntx, ch - 6);
-	lmt_ru->ru52[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52,
+	lmt_ru->ru52[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU52,
 							ntx, ch - 2);
-	lmt_ru->ru52[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52,
+	lmt_ru->ru52[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU52,
 							ntx, ch + 2);
-	lmt_ru->ru52[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU52,
+	lmt_ru->ru52[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							RTW89_RU52,
 							ntx, ch + 6);
-	lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106,
+	lmt_ru->ru106[0] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							 RTW89_RU106,
 							 ntx, ch - 6);
-	lmt_ru->ru106[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106,
+	lmt_ru->ru106[1] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							 RTW89_RU106,
 							 ntx, ch - 2);
-	lmt_ru->ru106[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106,
+	lmt_ru->ru106[2] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							 RTW89_RU106,
 							 ntx, ch + 2);
-	lmt_ru->ru106[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, RTW89_RU106,
+	lmt_ru->ru106[3] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
+							 RTW89_RU106,
 							 ntx, ch + 6);
 }
 
 static void
 rtw89_phy_fill_txpwr_limit_ru_160m(struct rtw89_dev *rtwdev,
 				   struct rtw89_txpwr_limit_ru *lmt_ru,
-				   u8 ntx, u8 ch)
+				   u8 band, u8 ntx, u8 ch)
 {
 	static const int ofst[] = { -14, -10, -6, -2, 2, 6, 10, 14 };
 	int i;
 
 	static_assert(ARRAY_SIZE(ofst) == RTW89_RU_SEC_NUM);
 	for (i = 0; i < RTW89_RU_SEC_NUM; i++) {
-		lmt_ru->ru26[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev,
+		lmt_ru->ru26[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
 								RTW89_RU26,
 								ntx,
 								ch + ofst[i]);
-		lmt_ru->ru52[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev,
+		lmt_ru->ru52[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
 								RTW89_RU52,
 								ntx,
 								ch + ofst[i]);
-		lmt_ru->ru106[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev,
+		lmt_ru->ru106[i] = rtw89_phy_read_txpwr_limit_ru(rtwdev, band,
 								 RTW89_RU106,
 								 ntx,
 								 ch + ofst[i]);
@@ -1880,26 +1963,32 @@ rtw89_phy_fill_txpwr_limit_ru_160m(struct rtw89_dev *rtwdev,
 }
 
 void rtw89_phy_fill_txpwr_limit_ru(struct rtw89_dev *rtwdev,
+				   const struct rtw89_chan *chan,
 				   struct rtw89_txpwr_limit_ru *lmt_ru,
 				   u8 ntx)
 {
-	u8 ch = rtwdev->hal.current_channel;
-	u8 bw = rtwdev->hal.current_band_width;
+	u8 band = chan->band_type;
+	u8 ch = chan->channel;
+	u8 bw = chan->band_width;
 
 	memset(lmt_ru, 0, sizeof(*lmt_ru));
 
 	switch (bw) {
 	case RTW89_CHANNEL_WIDTH_20:
-		rtw89_phy_fill_txpwr_limit_ru_20m(rtwdev, lmt_ru, ntx, ch);
+		rtw89_phy_fill_txpwr_limit_ru_20m(rtwdev, lmt_ru, band, ntx,
+						  ch);
 		break;
 	case RTW89_CHANNEL_WIDTH_40:
-		rtw89_phy_fill_txpwr_limit_ru_40m(rtwdev, lmt_ru, ntx, ch);
+		rtw89_phy_fill_txpwr_limit_ru_40m(rtwdev, lmt_ru, band, ntx,
+						  ch);
 		break;
 	case RTW89_CHANNEL_WIDTH_80:
-		rtw89_phy_fill_txpwr_limit_ru_80m(rtwdev, lmt_ru, ntx, ch);
+		rtw89_phy_fill_txpwr_limit_ru_80m(rtwdev, lmt_ru, band, ntx,
+						  ch);
 		break;
 	case RTW89_CHANNEL_WIDTH_160:
-		rtw89_phy_fill_txpwr_limit_ru_160m(rtwdev, lmt_ru, ntx, ch);
+		rtw89_phy_fill_txpwr_limit_ru_160m(rtwdev, lmt_ru, band, ntx,
+						   ch);
 		break;
 	}
 }
@@ -1920,6 +2009,7 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta)
 	u8 mode, rate, bw, giltf, mac_id;
 	u16 legacy_bitrate;
 	bool valid;
+	u8 mcs = 0;
 
 	mac_id = RTW89_GET_PHY_C2H_RA_RPT_MACID(c2h->data);
 	if (mac_id != rtwsta->mac_id)
@@ -1936,7 +2026,7 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta)
 			return;
 	}
 
-	memset(ra_report, 0, sizeof(*ra_report));
+	memset(&ra_report->txrate, 0, sizeof(ra_report->txrate));
 
 	switch (mode) {
 	case RTW89_RA_RPT_MODE_LEGACY:
@@ -1952,6 +2042,7 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta)
 		ra_report->txrate.mcs = rate;
 		if (giltf)
 			ra_report->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI;
+		mcs = ra_report->txrate.mcs & 0x07;
 		break;
 	case RTW89_RA_RPT_MODE_VHT:
 		ra_report->txrate.flags |= RATE_INFO_FLAGS_VHT_MCS;
@@ -1959,6 +2050,7 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta)
 		ra_report->txrate.nss = FIELD_GET(RTW89_RA_RATE_MASK_NSS, rate) + 1;
 		if (giltf)
 			ra_report->txrate.flags |= RATE_INFO_FLAGS_SHORT_GI;
+		mcs = ra_report->txrate.mcs;
 		break;
 	case RTW89_RA_RPT_MODE_HE:
 		ra_report->txrate.flags |= RATE_INFO_FLAGS_HE_MCS;
@@ -1970,6 +2062,7 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta)
 			ra_report->txrate.he_gi = NL80211_RATE_INFO_HE_GI_1_6;
 		else
 			ra_report->txrate.he_gi = NL80211_RATE_INFO_HE_GI_3_2;
+		mcs = ra_report->txrate.mcs;
 		break;
 	}
 
@@ -1977,8 +2070,9 @@ static void rtw89_phy_c2h_ra_rpt_iter(void *data, struct ieee80211_sta *sta)
 	ra_report->bit_rate = cfg80211_calculate_bitrate(&ra_report->txrate);
 	ra_report->hw_rate = FIELD_PREP(RTW89_HW_RATE_MASK_MOD, mode) |
 			     FIELD_PREP(RTW89_HW_RATE_MASK_VAL, rate);
-	sta->max_rc_amsdu_len = get_max_amsdu_len(rtwdev, ra_report);
-	rtwsta->max_agg_wait = sta->max_rc_amsdu_len / 1500 - 1;
+	ra_report->might_fallback_legacy = mcs <= 2;
+	sta->deflink.agg.max_rc_amsdu_len = get_max_amsdu_len(rtwdev, ra_report);
+	rtwsta->max_agg_wait = sta->deflink.agg.max_rc_amsdu_len / 1500 - 1;
 }
 
 static void
@@ -3247,10 +3341,11 @@ static void rtw89_phy_dig_update_rssi_info(struct rtw89_dev *rtwdev)
 static void rtw89_phy_dig_update_para(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_dig_info *dig = &rtwdev->dig;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	bool is_linked = rtwdev->total_sta_assoc > 0;
 	const u16 *fa_th_src = NULL;
 
-	switch (rtwdev->hal.current_band_type) {
+	switch (chan->band_type) {
 	case RTW89_BAND_2G:
 		dig->lna_gain = dig->lna_gain_g;
 		dig->tia_gain = dig->tia_gain_g;
@@ -3410,26 +3505,32 @@ static void rtw89_phy_dig_igi_offset_by_env(struct rtw89_dev *rtwdev)
 
 static void rtw89_phy_dig_set_lna_idx(struct rtw89_dev *rtwdev, u8 lna_idx)
 {
-	rtw89_phy_write32_mask(rtwdev, R_PATH0_LNA_INIT,
-			       B_PATH0_LNA_INIT_IDX_MSK, lna_idx);
-	rtw89_phy_write32_mask(rtwdev, R_PATH1_LNA_INIT,
-			       B_PATH1_LNA_INIT_IDX_MSK, lna_idx);
+	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
+
+	rtw89_phy_write32_mask(rtwdev, dig_regs->p0_lna_init.addr,
+			       dig_regs->p0_lna_init.mask, lna_idx);
+	rtw89_phy_write32_mask(rtwdev, dig_regs->p1_lna_init.addr,
+			       dig_regs->p1_lna_init.mask, lna_idx);
 }
 
 static void rtw89_phy_dig_set_tia_idx(struct rtw89_dev *rtwdev, u8 tia_idx)
 {
-	rtw89_phy_write32_mask(rtwdev, R_PATH0_TIA_INIT,
-			       B_PATH0_TIA_INIT_IDX_MSK, tia_idx);
-	rtw89_phy_write32_mask(rtwdev, R_PATH1_TIA_INIT,
-			       B_PATH1_TIA_INIT_IDX_MSK, tia_idx);
+	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
+
+	rtw89_phy_write32_mask(rtwdev, dig_regs->p0_tia_init.addr,
+			       dig_regs->p0_tia_init.mask, tia_idx);
+	rtw89_phy_write32_mask(rtwdev, dig_regs->p1_tia_init.addr,
+			       dig_regs->p1_tia_init.mask, tia_idx);
 }
 
 static void rtw89_phy_dig_set_rxb_idx(struct rtw89_dev *rtwdev, u8 rxb_idx)
 {
-	rtw89_phy_write32_mask(rtwdev, R_PATH0_RXB_INIT,
-			       B_PATH0_RXB_INIT_IDX_MSK, rxb_idx);
-	rtw89_phy_write32_mask(rtwdev, R_PATH1_RXB_INIT,
-			       B_PATH1_RXB_INIT_IDX_MSK, rxb_idx);
+	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
+
+	rtw89_phy_write32_mask(rtwdev, dig_regs->p0_rxb_init.addr,
+			       dig_regs->p0_rxb_init.mask, rxb_idx);
+	rtw89_phy_write32_mask(rtwdev, dig_regs->p1_rxb_init.addr,
+			       dig_regs->p1_rxb_init.mask, rxb_idx);
 }
 
 static void rtw89_phy_dig_set_igi_cr(struct rtw89_dev *rtwdev,
@@ -3443,21 +3544,19 @@ static void rtw89_phy_dig_set_igi_cr(struct rtw89_dev *rtwdev,
 		    set.lna_idx, set.tia_idx, set.rxb_idx);
 }
 
-static const struct rtw89_reg_def sdagc_config[4] = {
-	{R_PATH0_P20_FOLLOW_BY_PAGCUGC, B_PATH0_P20_FOLLOW_BY_PAGCUGC_EN_MSK},
-	{R_PATH0_S20_FOLLOW_BY_PAGCUGC, B_PATH0_S20_FOLLOW_BY_PAGCUGC_EN_MSK},
-	{R_PATH1_P20_FOLLOW_BY_PAGCUGC, B_PATH1_P20_FOLLOW_BY_PAGCUGC_EN_MSK},
-	{R_PATH1_S20_FOLLOW_BY_PAGCUGC, B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK},
-};
-
 static void rtw89_phy_dig_sdagc_follow_pagc_config(struct rtw89_dev *rtwdev,
 						   bool enable)
 {
-	u8 i = 0;
+	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
 
-	for (i = 0; i < ARRAY_SIZE(sdagc_config); i++)
-		rtw89_phy_write32_mask(rtwdev, sdagc_config[i].addr,
-				       sdagc_config[i].mask, enable);
+	rtw89_phy_write32_mask(rtwdev, dig_regs->p0_p20_pagcugc_en.addr,
+			       dig_regs->p0_p20_pagcugc_en.mask, enable);
+	rtw89_phy_write32_mask(rtwdev, dig_regs->p0_s20_pagcugc_en.addr,
+			       dig_regs->p0_s20_pagcugc_en.mask, enable);
+	rtw89_phy_write32_mask(rtwdev, dig_regs->p1_p20_pagcugc_en.addr,
+			       dig_regs->p1_p20_pagcugc_en.mask, enable);
+	rtw89_phy_write32_mask(rtwdev, dig_regs->p1_s20_pagcugc_en.addr,
+			       dig_regs->p1_s20_pagcugc_en.mask, enable);
 
 	rtw89_debug(rtwdev, RTW89_DBG_DIG, "sdagc_follow_pagc=%d\n", enable);
 }
@@ -3483,7 +3582,9 @@ static void rtw89_phy_dig_config_igi(struct rtw89_dev *rtwdev)
 static void rtw89_phy_dig_dyn_pd_th(struct rtw89_dev *rtwdev, u8 rssi,
 				    bool enable)
 {
-	enum rtw89_bandwidth cbw = rtwdev->hal.current_band_width;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	const struct rtw89_dig_regs *dig_regs = rtwdev->chip->dig_regs;
+	enum rtw89_bandwidth cbw = chan->band_width;
 	struct rtw89_dig_info *dig = &rtwdev->dig;
 	u8 final_rssi = 0, under_region = dig->pd_low_th_ofst;
 	u8 ofdm_cca_th;
@@ -3525,10 +3626,10 @@ static void rtw89_phy_dig_dyn_pd_th(struct rtw89_dev *rtwdev, u8 rssi,
 			    "Dynamic PD th disabled, Set PD_low_bd=0\n");
 	}
 
-	rtw89_phy_write32_mask(rtwdev, R_SEG0R_PD, B_SEG0R_PD_LOWER_BOUND_MSK,
-			       pd_val);
-	rtw89_phy_write32_mask(rtwdev, R_SEG0R_PD,
-			       B_SEG0R_PD_SPATIAL_REUSE_EN_MSK, enable);
+	rtw89_phy_write32_mask(rtwdev, dig_regs->seg0_pd_reg,
+			       dig_regs->pd_lower_bound_mask, pd_val);
+	rtw89_phy_write32_mask(rtwdev, dig_regs->seg0_pd_reg,
+			       dig_regs->pd_spatial_reuse_en, enable);
 
 	if (!rtwdev->hal.support_cckpd)
 		return;
@@ -3604,6 +3705,62 @@ void rtw89_phy_dig(struct rtw89_dev *rtwdev)
 		rtw89_phy_dig_sdagc_follow_pagc_config(rtwdev, false);
 }
 
+static void rtw89_phy_tx_path_div_sta_iter(void *data, struct ieee80211_sta *sta)
+{
+	struct rtw89_sta *rtwsta = (struct rtw89_sta *)sta->drv_priv;
+	struct rtw89_dev *rtwdev = rtwsta->rtwdev;
+	struct rtw89_vif *rtwvif = rtwsta->rtwvif;
+	struct rtw89_hal *hal = &rtwdev->hal;
+	bool *done = data;
+	u8 rssi_a, rssi_b;
+	u32 candidate;
+
+	if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION || sta->tdls)
+		return;
+
+	if (*done)
+		return;
+
+	*done = true;
+
+	rssi_a = ewma_rssi_read(&rtwsta->rssi[RF_PATH_A]);
+	rssi_b = ewma_rssi_read(&rtwsta->rssi[RF_PATH_B]);
+
+	if (rssi_a > rssi_b + RTW89_TX_DIV_RSSI_RAW_TH)
+		candidate = RF_A;
+	else if (rssi_b > rssi_a + RTW89_TX_DIV_RSSI_RAW_TH)
+		candidate = RF_B;
+	else
+		return;
+
+	if (hal->antenna_tx == candidate)
+		return;
+
+	hal->antenna_tx = candidate;
+	rtw89_fw_h2c_txpath_cmac_tbl(rtwdev, rtwsta);
+
+	if (hal->antenna_tx == RF_A) {
+		rtw89_phy_write32_mask(rtwdev, R_P0_RFMODE, B_P0_RFMODE_MUX, 0x12);
+		rtw89_phy_write32_mask(rtwdev, R_P1_RFMODE, B_P1_RFMODE_MUX, 0x11);
+	} else if (hal->antenna_tx == RF_B) {
+		rtw89_phy_write32_mask(rtwdev, R_P0_RFMODE, B_P0_RFMODE_MUX, 0x11);
+		rtw89_phy_write32_mask(rtwdev, R_P1_RFMODE, B_P1_RFMODE_MUX, 0x12);
+	}
+}
+
+void rtw89_phy_tx_path_div_track(struct rtw89_dev *rtwdev)
+{
+	struct rtw89_hal *hal = &rtwdev->hal;
+	bool done = false;
+
+	if (!hal->tx_path_diversity)
+		return;
+
+	ieee80211_iterate_stations_atomic(rtwdev->hw,
+					  rtw89_phy_tx_path_div_sta_iter,
+					  &done);
+}
+
 static void rtw89_phy_env_monitor_init(struct rtw89_dev *rtwdev)
 {
 	rtw89_phy_ccx_top_setting_init(rtwdev);
diff --git a/drivers/net/wireless/realtek/rtw89/phy.h b/drivers/net/wireless/realtek/rtw89/phy.h
index e20636f54b55..ee3bc5e111e1 100644
--- a/drivers/net/wireless/realtek/rtw89/phy.h
+++ b/drivers/net/wireless/realtek/rtw89/phy.h
@@ -56,7 +56,7 @@
 #define CFO_TRK_STOP_TH (2 << 2)
 #define CFO_SW_COMP_FINE_TUNE (2 << 2)
 #define CFO_PERIOD_CNT 15
-#define CFO_BOUND 32
+#define CFO_BOUND 64
 #define CFO_TP_UPPER 100
 #define CFO_TP_LOWER 50
 #define CFO_COMP_PERIOD 250
@@ -439,7 +439,7 @@ rtw89_rfk_parser(struct rtw89_dev *rtwdev, const struct rtw89_rfk_tbl *tbl);
 void rtw89_phy_write_reg3_tbl(struct rtw89_dev *rtwdev,
 			      const struct rtw89_phy_reg3_tbl *tbl);
 u8 rtw89_phy_get_txsc(struct rtw89_dev *rtwdev,
-		      struct rtw89_channel_params *param,
+		      const struct rtw89_chan *chan,
 		      enum rtw89_bandwidth dbw);
 u32 rtw89_phy_read_rf(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_path,
 		      u32 addr, u32 mask);
@@ -460,15 +460,17 @@ void rtw89_phy_write32_idx(struct rtw89_dev *rtwdev, u32 addr, u32 mask,
 			   u32 data, enum rtw89_phy_idx phy_idx);
 void rtw89_phy_load_txpwr_byrate(struct rtw89_dev *rtwdev,
 				 const struct rtw89_txpwr_table *tbl);
-s8 rtw89_phy_read_txpwr_byrate(struct rtw89_dev *rtwdev,
+s8 rtw89_phy_read_txpwr_byrate(struct rtw89_dev *rtwdev, u8 band,
 			       const struct rtw89_rate_desc *rate_desc);
 void rtw89_phy_fill_txpwr_limit(struct rtw89_dev *rtwdev,
+				const struct rtw89_chan *chan,
 				struct rtw89_txpwr_limit *lmt,
 				u8 ntx);
 void rtw89_phy_fill_txpwr_limit_ru(struct rtw89_dev *rtwdev,
+				   const struct rtw89_chan *chan,
 				   struct rtw89_txpwr_limit_ru *lmt_ru,
 				   u8 ntx);
-s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev,
+s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, u8 band,
 			      u8 bw, u8 ntx, u8 rs, u8 bf, u8 ch);
 void rtw89_phy_ra_assoc(struct rtw89_dev *rtwdev, struct ieee80211_sta *sta);
 void rtw89_phy_ra_update(struct rtw89_dev *rtwdev);
@@ -489,6 +491,7 @@ void rtw89_phy_set_phy_regs(struct rtw89_dev *rtwdev, u32 addr, u32 mask,
 			    u32 val);
 void rtw89_phy_dig_reset(struct rtw89_dev *rtwdev);
 void rtw89_phy_dig(struct rtw89_dev *rtwdev);
+void rtw89_phy_tx_path_div_track(struct rtw89_dev *rtwdev);
 void rtw89_phy_set_bss_color(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif);
 void rtw89_phy_tssi_ctrl_set_bandedge_cfg(struct rtw89_dev *rtwdev,
 					  enum rtw89_mac_idx mac_idx,
diff --git a/drivers/net/wireless/realtek/rtw89/ps.c b/drivers/net/wireless/realtek/rtw89/ps.c
index a90b33720588..bf41a1141679 100644
--- a/drivers/net/wireless/realtek/rtw89/ps.c
+++ b/drivers/net/wireless/realtek/rtw89/ps.c
@@ -59,8 +59,11 @@ static void rtw89_ps_power_mode_change(struct rtw89_dev *rtwdev, bool enter)
 		rtw89_mac_power_mode_change(rtwdev, enter);
 }
 
-static void __rtw89_enter_ps_mode(struct rtw89_dev *rtwdev)
+static void __rtw89_enter_ps_mode(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 {
+	if (rtwvif->wifi_role == RTW89_WIFI_ROLE_P2P_CLIENT)
+		return;
+
 	if (!rtwdev->ps_mode)
 		return;
 
@@ -111,23 +114,23 @@ void rtw89_leave_ps_mode(struct rtw89_dev *rtwdev)
 	__rtw89_leave_ps_mode(rtwdev);
 }
 
-void rtw89_enter_lps(struct rtw89_dev *rtwdev, u8 mac_id)
+void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 {
 	lockdep_assert_held(&rtwdev->mutex);
 
 	if (test_and_set_bit(RTW89_FLAG_LEISURE_PS, rtwdev->flags))
 		return;
 
-	__rtw89_enter_lps(rtwdev, mac_id);
-	__rtw89_enter_ps_mode(rtwdev);
+	__rtw89_enter_lps(rtwdev, rtwvif->mac_id);
+	__rtw89_enter_ps_mode(rtwdev, rtwvif);
 }
 
 static void rtw89_leave_lps_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 {
-	if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION)
+	if (rtwvif->wifi_role != RTW89_WIFI_ROLE_STATION &&
+	    rtwvif->wifi_role != RTW89_WIFI_ROLE_P2P_CLIENT)
 		return;
 
-	__rtw89_leave_ps_mode(rtwdev);
 	__rtw89_leave_lps(rtwdev, rtwvif->mac_id);
 }
 
@@ -140,6 +143,8 @@ void rtw89_leave_lps(struct rtw89_dev *rtwdev)
 	if (!test_and_clear_bit(RTW89_FLAG_LEISURE_PS, rtwdev->flags))
 		return;
 
+	__rtw89_leave_ps_mode(rtwdev);
+
 	rtw89_for_each_rtwvif(rtwdev, rtwvif)
 		rtw89_leave_lps_vif(rtwdev, rtwvif);
 }
@@ -178,3 +183,64 @@ void rtw89_set_coex_ctrl_lps(struct rtw89_dev *rtwdev, bool btc_ctrl)
 	if (btc_ctrl)
 		rtw89_leave_lps(rtwdev);
 }
+
+static void rtw89_tsf32_toggle(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif,
+			       enum rtw89_p2pps_action act)
+{
+	if (act == RTW89_P2P_ACT_UPDATE || act == RTW89_P2P_ACT_REMOVE)
+		return;
+
+	if (act == RTW89_P2P_ACT_INIT)
+		rtw89_fw_h2c_tsf32_toggle(rtwdev, rtwvif, true);
+	else if (act == RTW89_P2P_ACT_TERMINATE)
+		rtw89_fw_h2c_tsf32_toggle(rtwdev, rtwvif, false);
+}
+
+static void rtw89_p2p_disable_all_noa(struct rtw89_dev *rtwdev,
+				      struct ieee80211_vif *vif)
+{
+	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
+	enum rtw89_p2pps_action act;
+	u8 noa_id;
+
+	if (rtwvif->last_noa_nr == 0)
+		return;
+
+	for (noa_id = 0; noa_id < rtwvif->last_noa_nr; noa_id++) {
+		if (noa_id == rtwvif->last_noa_nr - 1)
+			act = RTW89_P2P_ACT_TERMINATE;
+		else
+			act = RTW89_P2P_ACT_REMOVE;
+		rtw89_tsf32_toggle(rtwdev, rtwvif, act);
+		rtw89_fw_h2c_p2p_act(rtwdev, vif, NULL, act, noa_id);
+	}
+}
+
+static void rtw89_p2p_update_noa(struct rtw89_dev *rtwdev,
+				 struct ieee80211_vif *vif)
+{
+	struct rtw89_vif *rtwvif = (struct rtw89_vif *)vif->drv_priv;
+	struct ieee80211_p2p_noa_desc *desc;
+	enum rtw89_p2pps_action act;
+	u8 noa_id;
+
+	for (noa_id = 0; noa_id < RTW89_P2P_MAX_NOA_NUM; noa_id++) {
+		desc = &vif->bss_conf.p2p_noa_attr.desc[noa_id];
+		if (!desc->count || !desc->duration)
+			break;
+
+		if (noa_id == 0)
+			act = RTW89_P2P_ACT_INIT;
+		else
+			act = RTW89_P2P_ACT_UPDATE;
+		rtw89_tsf32_toggle(rtwdev, rtwvif, act);
+		rtw89_fw_h2c_p2p_act(rtwdev, vif, desc, act, noa_id);
+	}
+	rtwvif->last_noa_nr = noa_id;
+}
+
+void rtw89_process_p2p_ps(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif)
+{
+	rtw89_p2p_disable_all_noa(rtwdev, vif);
+	rtw89_p2p_update_noa(rtwdev, vif);
+}
diff --git a/drivers/net/wireless/realtek/rtw89/ps.h b/drivers/net/wireless/realtek/rtw89/ps.h
index a184b68994aa..0feae3991623 100644
--- a/drivers/net/wireless/realtek/rtw89/ps.h
+++ b/drivers/net/wireless/realtek/rtw89/ps.h
@@ -5,12 +5,13 @@
 #ifndef __RTW89_PS_H_
 #define __RTW89_PS_H_
 
-void rtw89_enter_lps(struct rtw89_dev *rtwdev, u8 mac_id);
+void rtw89_enter_lps(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif);
 void rtw89_leave_lps(struct rtw89_dev *rtwdev);
 void __rtw89_leave_ps_mode(struct rtw89_dev *rtwdev);
 void rtw89_leave_ps_mode(struct rtw89_dev *rtwdev);
 void rtw89_enter_ips(struct rtw89_dev *rtwdev);
 void rtw89_leave_ips(struct rtw89_dev *rtwdev);
 void rtw89_set_coex_ctrl_lps(struct rtw89_dev *rtwdev, bool btc_ctrl);
+void rtw89_process_p2p_ps(struct rtw89_dev *rtwdev, struct ieee80211_vif *vif);
 
 #endif
diff --git a/drivers/net/wireless/realtek/rtw89/reg.h b/drivers/net/wireless/realtek/rtw89/reg.h
index ebf28719d935..ca20bb024b40 100644
--- a/drivers/net/wireless/realtek/rtw89/reg.h
+++ b/drivers/net/wireless/realtek/rtw89/reg.h
@@ -51,9 +51,6 @@
 #define B_AX_EF_POR BIT(10)
 #define B_AX_EF_CELL_SEL_MASK GENMASK(9, 8)
 
-#define R_AX_SPSLDO_ON_CTRL0 0x0200
-#define B_AX_OCP_L1_MASK GENMASK(15, 13)
-
 #define R_AX_EFUSE_CTRL 0x0030
 #define B_AX_EF_MODE_SEL_MASK GENMASK(31, 30)
 #define B_AX_EF_RDY BIT(29)
@@ -143,6 +140,18 @@
 #define R_AX_PMC_DBG_CTRL2 0x00CC
 #define B_AX_SYSON_DIS_PMCR_AX_WRMSK BIT(2)
 
+#define R_AX_PCIE_MIO_INTF 0x00E4
+#define B_AX_PCIE_MIO_ADDR_PAGE_V1_MASK GENMASK(20, 16)
+#define B_AX_PCIE_MIO_BYIOREG BIT(13)
+#define B_AX_PCIE_MIO_RE BIT(12)
+#define B_AX_PCIE_MIO_WE_MASK GENMASK(11, 8)
+#define MIO_WRITE_BYTE_ALL 0xF
+#define B_AX_PCIE_MIO_ADDR_MASK GENMASK(7, 0)
+#define MIO_ADDR_PAGE_MASK GENMASK(12, 8)
+
+#define R_AX_PCIE_MIO_INTD 0x00E8
+#define B_AX_PCIE_MIO_DATA_MASK GENMASK(31, 0)
+
 #define R_AX_SYS_CFG1 0x00F0
 #define B_AX_CHIP_VER_MASK GENMASK(15, 12)
 
@@ -191,6 +200,12 @@
 #define R_AX_UDM2 0x01F8
 #define R_AX_UDM3 0x01FC
 
+#define R_AX_SPS_DIG_ON_CTRL0 0x0200
+#define B_AX_VREFPFM_L_MASK GENMASK(25, 22)
+#define B_AX_REG_ZCDC_H_MASK GENMASK(18, 17)
+#define B_AX_OCP_L1_MASK GENMASK(15, 13)
+#define B_AX_VOL_L1_MASK GENMASK(3, 0)
+
 #define R_AX_LDO_AON_CTRL0 0x0218
 #define B_AX_PD_REGU_L BIT(16)
 
@@ -383,6 +398,7 @@
 
 #define R_AX_PHYREG_SET 0x8040
 #define PHYREG_SET_ALL_CYCLE 0x8
+#define PHYREG_SET_XYN_CYCLE 0xE
 
 #define R_AX_HD0IMR 0x8110
 #define B_AX_WDT_PTFM_INT_EN BIT(5)
@@ -467,6 +483,7 @@
 #define R_AX_LTR_CTRL_0 0x8410
 #define B_AX_LTR_SPACE_IDX_MASK GENMASK(13, 12)
 #define B_AX_LTR_IDLE_TIMER_IDX_MASK GENMASK(10, 8)
+#define B_AX_LTR_WD_NOEMP_CHK BIT(6)
 #define B_AX_APP_LTR_ACT BIT(5)
 #define B_AX_APP_LTR_IDLE BIT(4)
 #define B_AX_LTR_EN BIT(1)
@@ -1024,15 +1041,13 @@
 			  B_AX_WDE_DATCHN_NULLPG_ERR_INT_EN | \
 			  B_AX_WDE_DATCHN_FRZTO_ERR_INT_EN)
 #define B_AX_WDE_IMR_SET (B_AX_WDE_BUFREQ_QTAID_ERR_INT_EN | \
-			  B_AX_WDE_BUFREQ_SIZE0_INT_EN | \
-			  B_AX_WDE_BUFREQ_SIZELMT_INT_EN | \
-			  B_AX_WDE_BUFREQ_UNAVAL_ERR_INT_EN_V1 | \
-			  B_AX_WDE_BUFRTN_INVLD_PKTID_ERR_INT_EN_V1 | \
-			  B_AX_WDE_BUFRTN_SIZE_ERR_INT_EN_V1 | \
-			  B_AX_WDE_BUFREQ_SRCHTAILPG_ERR_INT_EN_V1 | \
-			  B_AX_WDE_GETNPG_STRPG_ERR_INT_EN_V1 | \
-			  B_AX_WDE_GETNPG_PGOFST_ERR_INT_EN_V1 | \
-			  B_AX_WDE_BUFMGN_FRZTO_ERR_INT_EN_V1 | \
+			  B_AX_WDE_BUFREQ_UNAVAL_ERR_INT_EN | \
+			  B_AX_WDE_BUFRTN_INVLD_PKTID_ERR_INT_EN | \
+			  B_AX_WDE_BUFRTN_SIZE_ERR_INT_EN | \
+			  B_AX_WDE_BUFREQ_SRCHTAILPG_ERR_INT_EN | \
+			  B_AX_WDE_GETNPG_STRPG_ERR_INT_EN | \
+			  B_AX_WDE_GETNPG_PGOFST_ERR_INT_EN | \
+			  B_AX_WDE_BUFMGN_FRZTO_ERR_INT_EN | \
 			  B_AX_WDE_QUE_CMDTYPE_ERR_INT_EN | \
 			  B_AX_WDE_QUE_DSTQUEID_ERR_INT_EN | \
 			  B_AX_WDE_QUE_SRCQUEID_ERR_INT_EN | \
@@ -1043,10 +1058,7 @@
 			  B_AX_WDE_QUEMGN_FRZTO_ERR_INT_EN | \
 			  B_AX_WDE_DATCHN_ARBT_ERR_INT_EN | \
 			  B_AX_WDE_DATCHN_NULLPG_ERR_INT_EN | \
-			  B_AX_WDE_DATCHN_FRZTO_ERR_INT_EN | \
-			  B_AX_WDE_DATCHN_RRDY_ERR_INT_EN | \
-			  B_AX_WDE_DATCHN_ADRERR_ERR_INT_EN | \
-			  B_AX_WDE_DATCHN_CAMREQ_ERR_INT_EN)
+			  B_AX_WDE_DATCHN_FRZTO_ERR_INT_EN)
 
 #define B_AX_WDE_DATCHN_CAMREQ_ERR_INT_EN BIT(29)
 #define B_AX_WDE_DATCHN_ADRERR_ERR_INT_EN BIT(28)
@@ -1826,6 +1838,13 @@
 #define B_AX_TXSC_40M_MASK GENMASK(7, 4)
 #define B_AX_TXSC_20M_MASK GENMASK(3, 0)
 
+#define R_AX_PTCL_RRSR1 0xC090
+#define R_AX_PTCL_RRSR1_C1 0xE090
+#define B_AX_RRSR_RATE_EN_MASK GENMASK(11, 8)
+#define RRSR_OFDM_CCK_EN 3
+#define B_AX_RSC_MASK GENMASK(7, 6)
+#define B_AX_RRSR_CCK_MASK GENMASK(3, 0)
+
 #define R_AX_CMAC_ERR_IMR 0xC160
 #define R_AX_CMAC_ERR_IMR_C1 0xE160
 #define B_AX_WMAC_TX_ERR_IND_EN BIT(7)
@@ -1882,6 +1901,7 @@
 #define B_AX_SIFS_TIMEOUT_T2_MASK GENMASK(14, 8)
 #define B_AX_SIFS_MACTXEN_T1_MASK GENMASK(6, 0)
 #define SIFS_MACTXEN_T1 0x47
+#define SIFS_MACTXEN_T1_V1 0x41
 
 #define R_AX_CCA_CFG_0 0xC340
 #define R_AX_CCA_CFG_0_C1 0xE340
@@ -2098,6 +2118,8 @@
 #define R_AX_TBTT_SHIFT_P3 0xC4E8
 #define R_AX_TBTT_SHIFT_P4 0xC528
 #define B_AX_TBTT_SHIFT_OFST_MASK GENMASK(11, 0)
+#define B_AX_TBTT_SHIFT_OFST_SIGN BIT(11)
+#define B_AX_TBTT_SHIFT_OFST_MAG GENMASK(10, 0)
 
 #define R_AX_BCN_CNT_TMR_P0 0xC434
 #define R_AX_BCN_CNT_TMR_P1 0xC474
@@ -2258,6 +2280,7 @@
 #define B_AX_F2PCMDRPT_FULL_DROP_ERR_INT_EN BIT(8)
 #define B_AX_FSM1_TIMEOUT_ERR_INT_EN BIT(1)
 #define B_AX_FSM_TIMEOUT_ERR_INT_EN BIT(0)
+#define B_AX_PTCL_IMR_CLR_ALL GENMASK(31, 0)
 #define B_AX_PTCL_IMR_CLR (B_AX_FSM_TIMEOUT_ERR_INT_EN | \
 			   B_AX_F2PCMDRPT_FULL_DROP_ERR_INT_EN | \
 			   B_AX_TXPRT_FULL_DROP_ERR_INT_EN | \
@@ -2315,6 +2338,28 @@
 #define B_AX_DLE_IMR_SET (B_AX_RXSTS_FSM_HANG_ERROR_IMR | \
 			  B_AX_RXDATA_FSM_HANG_ERROR_IMR)
 
+#define R_AX_RXDMA_CTRL_0 0xC804
+#define R_AX_RXDMA_CTRL_0_C1 0xE804
+#define B_AX_RXDMA_DBGOUT_EN BIT(31)
+#define B_AX_RXDMA_DBG_SEL_MASK GENMASK(30, 29)
+#define B_AX_RXDMA_FIFO_DBG_SEL_MASK GENMASK(28, 25)
+#define B_AX_RXDMA_DEFAULT_PAGE_MASK GENMASK(22, 21)
+#define B_AX_RXDMA_BUFF_REQ_PRI_MASK GENMASK(20, 19)
+#define B_AX_RXDMA_TGT_QUEID_MASK GENMASK(18, 13)
+#define B_AX_RXDMA_TGT_PRID_MASK GENMASK(12, 10)
+#define B_AX_RXDMA_DIS_CSI_RELEASE BIT(9)
+#define B_AX_RXDMA_DIS_RXSTS_WAIT_PTR_CLR BIT(7)
+#define B_AX_RXDMA_DIS_CSI_WAIT_PTR_CLR BIT(6)
+#define B_AX_RXSTS_PTR_FULL_MODE BIT(5)
+#define B_AX_CSI_PTR_FULL_MODE BIT(4)
+#define B_AX_RU3_PTR_FULL_MODE BIT(3)
+#define B_AX_RU2_PTR_FULL_MODE BIT(2)
+#define B_AX_RU1_PTR_FULL_MODE BIT(1)
+#define B_AX_RU0_PTR_FULL_MODE BIT(0)
+#define RX_FULL_MODE (B_AX_RU0_PTR_FULL_MODE | B_AX_RU1_PTR_FULL_MODE | \
+		      B_AX_RU2_PTR_FULL_MODE | B_AX_RU3_PTR_FULL_MODE | \
+		      B_AX_CSI_PTR_FULL_MODE | B_AX_RXSTS_PTR_FULL_MODE)
+
 #define R_AX_RXDMA_PKT_INFO_0 0xC814
 #define R_AX_RXDMA_PKT_INFO_1 0xC818
 #define R_AX_RXDMA_PKT_INFO_2 0xC81C
@@ -2553,6 +2598,20 @@
 #define WMAC_SPEC_SIFS_OFDM_52C 0x11
 #define WMAC_SPEC_SIFS_CCK	 0xA
 
+#define R_AX_TRXPTCL_RRSR_CTL_0 0xCC08
+#define R_AX_TRXPTCL_RRSR_CTL_0_C1 0xEC08
+#define B_AX_RESP_TX_MACID_CCA_TH_EN BIT(31)
+#define B_AX_RESP_TX_PWRMODE_MASK GENMASK(30, 28)
+#define B_AX_FTM_RRSR_RATE_EN_MASK GENMASK(27, 24)
+#define B_AX_NESS_MASK GENMASK(23, 22)
+#define B_AX_WMAC_RESP_DOPPLEB_AX_EN BIT(21)
+#define B_AX_WMAC_RESP_DCM_EN BIT(20)
+#define B_AX_WMAC_RRSB_AX_CCK_MASK GENMASK(19, 16)
+#define B_AX_WMAC_RESP_RATE_EN_MASK GENMASK(15, 12)
+#define B_AX_WMAC_RESP_RSC_MASK GENMASK(11, 10)
+#define B_AX_WMAC_RESP_REF_RATE_SEL BIT(9)
+#define B_AX_WMAC_RESP_REF_RATE_MASK GENMASK(8, 0)
+
 #define R_AX_MAC_LOOPBACK 0xCC20
 #define R_AX_MAC_LOOPBACK_C1 0xEC20
 #define B_AX_MACLBK_EN BIT(0)
@@ -2565,6 +2624,7 @@
 #define B_AX_WMAC_TF_UP_NAV_EN BIT(16)
 #define B_AX_WMAC_NAV_UPPER_MASK GENMASK(15, 8)
 #define NAV_12MS 0xBC
+#define NAV_25MS 0xC4
 #define B_AX_WMAC_RTS_RST_DUR_MASK GENMASK(7, 0)
 
 #define R_AX_RXTRIG_TEST_USER_2 0xCCB0
@@ -2968,18 +3028,18 @@
 
 #define R_AX_PATH_COM0 0xD800
 #define AX_PATH_COM0_DFVAL 0x00000000
-#define AX_PATH_COM0_PATHA 0x08888880
-#define AX_PATH_COM0_PATHB 0x11111100
+#define AX_PATH_COM0_PATHA 0x08889880
+#define AX_PATH_COM0_PATHB 0x11111900
 #define AX_PATH_COM0_PATHAB 0x19999980
 #define R_AX_PATH_COM1 0xD804
 #define AX_PATH_COM1_DFVAL 0x00000000
-#define AX_PATH_COM1_PATHA 0x11111111
-#define AX_PATH_COM1_PATHB 0x22222222
+#define AX_PATH_COM1_PATHA 0x13111111
+#define AX_PATH_COM1_PATHB 0x23222222
 #define AX_PATH_COM1_PATHAB 0x33333333
 #define R_AX_PATH_COM2 0xD808
 #define AX_PATH_COM2_DFVAL 0x00000000
-#define AX_PATH_COM2_PATHA 0x01209111
-#define AX_PATH_COM2_PATHB 0x01209222
+#define AX_PATH_COM2_PATHA 0x01209313
+#define AX_PATH_COM2_PATHB 0x01209323
 #define AX_PATH_COM2_PATHAB 0x01209333
 #define R_AX_PATH_COM3 0xD80C
 #define AX_PATH_COM3_DFVAL 0x49249249
@@ -3125,6 +3185,18 @@
 #define B_AX_GNT_WL_BB_VAL BIT(1)
 #define B_AX_GNT_WL_BB_SWCTRL BIT(0)
 
+#define R_AX_GNT_VAL 0x0054
+#define B_AX_GNT_BT_RFC_S1_STA BIT(5)
+#define B_AX_GNT_WL_RFC_S1_STA BIT(4)
+#define B_AX_GNT_BT_RFC_S0_STA BIT(3)
+#define B_AX_GNT_WL_RFC_S0_STA BIT(2)
+
+#define R_AX_GNT_VAL_V1 0xDA4C
+#define B_AX_GNT_BT_RFC_S1 BIT(4)
+#define B_AX_GNT_BT_RFC_S0 BIT(3)
+#define B_AX_GNT_WL_RFC_S1 BIT(2)
+#define B_AX_GNT_WL_RFC_S0 BIT(1)
+
 #define R_AX_TDMA_MODE 0xDA4C
 #define R_AX_TDMA_MODE_C1 0xFA4C
 #define B_AX_R_BT_CMD_RPT_MASK GENMASK(31, 16)
@@ -3356,6 +3428,7 @@
 #define RR_DCK_FINE BIT(1)
 #define RR_DCK_LV BIT(0)
 #define RR_DCK1 0x93
+#define RR_DCK1_DONE BIT(5)
 #define RR_DCK1_CLR GENMASK(3, 0)
 #define RR_DCK1_SEL BIT(3)
 #define RR_DCK2 0x94
@@ -3431,8 +3504,9 @@
 #define R_MAC_PIN_SEL 0x0734
 #define B_CH_IDX_SEG0 GENMASK(23, 16)
 #define R_PLCP_HISTOGRAM 0x0738
-#define B_STS_DIS_TRIG_BY_BRK BIT(2)
+#define B_STS_PARSING_TIME GENMASK(19, 16)
 #define B_STS_DIS_TRIG_BY_FAIL BIT(3)
+#define B_STS_DIS_TRIG_BY_BRK BIT(2)
 #define R_PHY_STS_BITMAP_ADDR_START R_PHY_STS_BITMAP_SEARCH_FAIL
 #define B_PHY_STS_BITMAP_ADDR_MASK GENMASK(6, 2)
 #define R_PHY_STS_BITMAP_SEARCH_FAIL 0x073C
@@ -3542,6 +3616,9 @@
 #define B_P0_RXCK_VAL GENMASK(18, 16)
 #define B_P0_TXCK_ON BIT(15)
 #define B_P0_TXCK_VAL GENMASK(14, 12)
+#define R_P0_RFMODE 0x12AC
+#define B_P0_RFMODE_ORI_TXRX_FTM_TX GENMASK(31, 4)
+#define B_P0_RFMODE_MUX GENMASK(11, 4)
 #define R_P0_NRBW 0x12B8
 #define B_P0_NRBW_DBG BIT(30)
 #define R_S0_RXDC 0x12D4
@@ -3648,6 +3725,9 @@
 #define B_P1_EN_SOUND_WO_NDP BIT(1)
 #define R_S1_HW_SI_DIS 0x3200
 #define B_S1_HW_SI_DIS_W_R_TRIG GENMASK(30, 28)
+#define R_P1_RFMODE 0x32AC
+#define B_P1_RFMODE_ORI_TXRX_FTM_TX GENMASK(31, 4)
+#define B_P1_RFMODE_MUX GENMASK(11, 4)
 #define R_P1_DBGMOD 0x32B8
 #define B_P1_DBGMOD_ON BIT(30)
 #define R_S1_RXDC 0x32D4
@@ -3663,6 +3743,8 @@
 #define R_S1_ADDCK 0x3E00
 #define B_S1_ADDCK_I GENMASK(9, 0)
 #define B_S1_ADDCK_Q GENMASK(19, 10)
+#define R_MUIC 0x40F8
+#define B_MUIC_EN BIT(0)
 #define R_DCFO 0x4264
 #define B_DCFO GENMASK(1, 0)
 #define R_SEG0CSI 0x42AC
@@ -3745,15 +3827,22 @@
 #define R_PATH0_RXB_INIT 0x4658
 #define B_PATH0_RXB_INIT_IDX_MSK GENMASK(9, 5)
 #define R_PATH0_LNA_INIT 0x4668
+#define R_PATH0_LNA_INIT_V1 0x472C
 #define B_PATH0_LNA_INIT_IDX_MSK GENMASK(26, 24)
 #define R_PATH0_BTG 0x466C
 #define B_PATH0_BTG_SHEN GENMASK(18, 17)
 #define R_PATH0_TIA_INIT 0x4674
 #define B_PATH0_TIA_INIT_IDX_MSK BIT(17)
 #define R_PATH0_P20_FOLLOW_BY_PAGCUGC 0x46A0
+#define R_PATH0_P20_FOLLOW_BY_PAGCUGC_V1 0x4C24
+#define R_PATH0_P20_FOLLOW_BY_PAGCUGC_V2 0x46E8
 #define B_PATH0_P20_FOLLOW_BY_PAGCUGC_EN_MSK BIT(5)
 #define R_PATH0_S20_FOLLOW_BY_PAGCUGC 0x46A4
+#define R_PATH0_S20_FOLLOW_BY_PAGCUGC_V1 0x4C28
+#define R_PATH0_S20_FOLLOW_BY_PAGCUGC_V2 0x46EC
 #define B_PATH0_S20_FOLLOW_BY_PAGCUGC_EN_MSK BIT(5)
+#define R_PATH0_RXB_INIT_V1 0x46A8
+#define B_PATH0_RXB_INIT_IDX_MSK_V1 GENMASK(14, 10)
 #define R_PATH0_G_LNA6_OP1DB_V1 0x4688
 #define B_PATH0_G_LNA6_OP1DB_V1 GENMASK(31, 24)
 #define R_PATH0_G_TIA0_LNA6_OP1DB_V1 0x4694
@@ -3780,7 +3869,10 @@
 #define R_P0_AGC_CTL 0x4730
 #define B_P0_AGC_EN BIT(31)
 #define R_PATH1_LNA_INIT 0x473C
+#define R_PATH1_LNA_INIT_V1 0x4A80
 #define B_PATH1_LNA_INIT_IDX_MSK GENMASK(26, 24)
+#define R_PATH0_TIA_INIT_V1 0x473C
+#define B_PATH0_TIA_INIT_IDX_MSK_V1 BIT(9)
 #define R_PATH1_TIA_INIT 0x4748
 #define B_PATH1_TIA_INIT_IDX_MSK BIT(17)
 #define R_PATH1_BTG 0x4740
@@ -3790,8 +3882,12 @@
 #define R_PATH1_G_LNA6_OP1DB_V1 0x476C
 #define B_PATH1_G_LNA6_OP1DB_V1 GENMASK(31, 24)
 #define R_PATH1_P20_FOLLOW_BY_PAGCUGC 0x4774
+#define R_PATH1_P20_FOLLOW_BY_PAGCUGC_V1 0x4CE8
+#define R_PATH1_P20_FOLLOW_BY_PAGCUGC_V2 0x47A8
 #define B_PATH1_P20_FOLLOW_BY_PAGCUGC_EN_MSK BIT(5)
 #define R_PATH1_S20_FOLLOW_BY_PAGCUGC 0x4778
+#define R_PATH1_S20_FOLLOW_BY_PAGCUGC_V1 0x4CEC
+#define R_PATH1_S20_FOLLOW_BY_PAGCUGC_V2 0x47AC
 #define B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK BIT(5)
 #define R_PATH1_G_TIA0_LNA6_OP1DB_V1 0x4778
 #define B_PATH1_G_TIA0_LNA6_OP1DB_V1 GENMASK(7, 0)
@@ -3807,6 +3903,8 @@
 #define B_P1_NBIIDX_VAL GENMASK(11, 0)
 #define B_P1_NBIIDX_NOTCH_EN BIT(12)
 #define R_SEG0R_PD 0x481C
+#define R_SEG0R_PD_V1 0x4860
+#define B_SEG0R_PD_SPATIAL_REUSE_EN_MSK_V1 BIT(30)
 #define B_SEG0R_PD_SPATIAL_REUSE_EN_MSK BIT(29)
 #define B_SEG0R_PD_LOWER_BOUND_MSK GENMASK(10, 6)
 #define R_2P4G_BAND 0x4970
@@ -3830,8 +3928,12 @@
 #define B_BK_FC0_INV_MSK_V1 GENMASK(18, 0)
 #define R_CCK_FC0_INV_V1 0x4A20
 #define B_CCK_FC0_INV_MSK_V1 GENMASK(18, 0)
+#define R_PATH1_RXB_INIT_V1 0x4A5C
+#define B_PATH1_RXB_INIT_IDX_MSK_V1 GENMASK(14, 10)
 #define R_P1_AGC_CTL 0x4A9C
 #define B_P1_AGC_EN BIT(31)
+#define R_PATH1_TIA_INIT_V1 0x4AA8
+#define B_PATH1_TIA_INIT_IDX_MSK_V1 BIT(9)
 #define R_PATH0_RXBB_V1 0x4AD4
 #define B_PATH0_RXBB_MSK_V1 GENMASK(31, 0)
 #define R_PATH1_RXBB_V1 0x4AE0
diff --git a/drivers/net/wireless/realtek/rtw89/regd.c b/drivers/net/wireless/realtek/rtw89/regd.c
index 20c7afd3e70f..6e5a740b128f 100644
--- a/drivers/net/wireless/realtek/rtw89/regd.c
+++ b/drivers/net/wireless/realtek/rtw89/regd.c
@@ -346,7 +346,7 @@ void rtw89_regd_notifier(struct wiphy *wiphy, struct regulatory_request *request
 	rtw89_debug_regd(rtwdev, rtwdev->regd, "get from initiator %d, alpha2",
 			 request->initiator);
 
-	rtw89_chip_set_txpwr(rtwdev);
+	rtw89_core_set_chip_txpwr(rtwdev);
 
 exit:
 	mutex_unlock(&rtwdev->mutex);
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a.c b/drivers/net/wireless/realtek/rtw89/rtw8852a.c
index 81bd0c4fe21b..784147680353 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852a.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852a.c
@@ -431,6 +431,7 @@ static const struct rtw89_imr_info rtw8852a_imr_info = {
 	.cpu_disp_imr_set	= B_AX_CPU_DISP_IMR_SET,
 	.other_disp_imr_clr	= B_AX_OTHER_DISP_IMR_CLR,
 	.other_disp_imr_set	= 0,
+	.bbrpt_com_err_imr_reg	= R_AX_BBRPT_COM_ERR_IMR_ISR,
 	.bbrpt_chinfo_err_imr_reg = R_AX_BBRPT_CHINFO_ERR_IMR_ISR,
 	.bbrpt_err_imr_set	= 0,
 	.bbrpt_dfs_err_imr_reg	= R_AX_BBRPT_DFS_ERR_IMR_ISR,
@@ -453,6 +454,31 @@ static const struct rtw89_imr_info rtw8852a_imr_info = {
 	.tmac_imr_set		= B_AX_TMAC_IMR_SET,
 };
 
+static const struct rtw89_rrsr_cfgs rtw8852a_rrsr_cfgs = {
+	.ref_rate = {R_AX_TRXPTCL_RRSR_CTL_0, B_AX_WMAC_RESP_REF_RATE_SEL, 0},
+	.rsc = {R_AX_TRXPTCL_RRSR_CTL_0, B_AX_WMAC_RESP_RSC_MASK, 2},
+};
+
+static const struct rtw89_dig_regs rtw8852a_dig_regs = {
+	.seg0_pd_reg = R_SEG0R_PD,
+	.pd_lower_bound_mask = B_SEG0R_PD_LOWER_BOUND_MSK,
+	.pd_spatial_reuse_en = B_SEG0R_PD_SPATIAL_REUSE_EN_MSK,
+	.p0_lna_init = {R_PATH0_LNA_INIT, B_PATH0_LNA_INIT_IDX_MSK},
+	.p1_lna_init = {R_PATH1_LNA_INIT, B_PATH1_LNA_INIT_IDX_MSK},
+	.p0_tia_init = {R_PATH0_TIA_INIT, B_PATH0_TIA_INIT_IDX_MSK},
+	.p1_tia_init = {R_PATH1_TIA_INIT, B_PATH1_TIA_INIT_IDX_MSK},
+	.p0_rxb_init = {R_PATH0_RXB_INIT, B_PATH0_RXB_INIT_IDX_MSK},
+	.p1_rxb_init = {R_PATH1_RXB_INIT, B_PATH1_RXB_INIT_IDX_MSK},
+	.p0_p20_pagcugc_en = {R_PATH0_P20_FOLLOW_BY_PAGCUGC,
+			      B_PATH0_P20_FOLLOW_BY_PAGCUGC_EN_MSK},
+	.p0_s20_pagcugc_en = {R_PATH0_S20_FOLLOW_BY_PAGCUGC,
+			      B_PATH0_S20_FOLLOW_BY_PAGCUGC_EN_MSK},
+	.p1_p20_pagcugc_en = {R_PATH1_P20_FOLLOW_BY_PAGCUGC,
+			      B_PATH1_P20_FOLLOW_BY_PAGCUGC_EN_MSK},
+	.p1_s20_pagcugc_en = {R_PATH1_S20_FOLLOW_BY_PAGCUGC,
+			      B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK},
+};
+
 static void rtw8852ae_efuse_parsing(struct rtw89_efuse *efuse,
 				    struct rtw8852a_efuse *map)
 {
@@ -660,7 +686,7 @@ static void rtw8852a_power_trim(struct rtw89_dev *rtwdev)
 }
 
 static void rtw8852a_set_channel_mac(struct rtw89_dev *rtwdev,
-				     struct rtw89_channel_params *param,
+				     const struct rtw89_chan *chan,
 				     u8 mac_idx)
 {
 	u32 rf_mod = rtw89_mac_reg_by_idx(R_AX_WMAC_RFMOD, mac_idx);
@@ -669,20 +695,20 @@ static void rtw8852a_set_channel_mac(struct rtw89_dev *rtwdev,
 	u32 chk_rate = rtw89_mac_reg_by_idx(R_AX_TXRATE_CHK, mac_idx);
 	u8 txsc20 = 0, txsc40 = 0;
 
-	switch (param->bandwidth) {
+	switch (chan->band_width) {
 	case RTW89_CHANNEL_WIDTH_80:
-		txsc40 = rtw89_phy_get_txsc(rtwdev, param,
+		txsc40 = rtw89_phy_get_txsc(rtwdev, chan,
 					    RTW89_CHANNEL_WIDTH_40);
 		fallthrough;
 	case RTW89_CHANNEL_WIDTH_40:
-		txsc20 = rtw89_phy_get_txsc(rtwdev, param,
+		txsc20 = rtw89_phy_get_txsc(rtwdev, chan,
 					    RTW89_CHANNEL_WIDTH_20);
 		break;
 	default:
 		break;
 	}
 
-	switch (param->bandwidth) {
+	switch (chan->band_width) {
 	case RTW89_CHANNEL_WIDTH_80:
 		rtw89_write8_mask(rtwdev, rf_mod, B_AX_WMAC_RFMOD_MASK, BIT(1));
 		rtw89_write32(rtwdev, sub_carr, txsc20 | (txsc40 << 4));
@@ -699,7 +725,7 @@ static void rtw8852a_set_channel_mac(struct rtw89_dev *rtwdev,
 		break;
 	}
 
-	if (param->center_chan > 14)
+	if (chan->channel > 14)
 		rtw89_write8_set(rtwdev, chk_rate,
 				 B_AX_CHECK_CCK_EN | B_AX_RTS_LIMIT_IN_OFDM6);
 	else
@@ -1102,11 +1128,12 @@ static void rtw8852a_bb_sethw(struct rtw89_dev *rtwdev)
 	if (rtwdev->hal.cv <= CHIP_CCV) {
 		rtw89_phy_write32_set(rtwdev, R_RSTB_WATCH_DOG, B_P0_RSTB_WATCH_DOG);
 		rtw89_phy_write32(rtwdev, R_BRK_ASYNC_RST_EN_1, 0x864FA000);
-		rtw89_phy_write32(rtwdev, R_BRK_ASYNC_RST_EN_2, 0x3F);
+		rtw89_phy_write32(rtwdev, R_BRK_ASYNC_RST_EN_2, 0x43F);
 		rtw89_phy_write32(rtwdev, R_BRK_ASYNC_RST_EN_3, 0x7FFF);
 		rtw89_phy_write32_set(rtwdev, R_SPOOF_ASYNC_RST, B_SPOOF_ASYNC_RST);
 		rtw89_phy_write32_set(rtwdev, R_P0_TXPW_RSTB, B_P0_TXPW_RSTB_MANON);
 		rtw89_phy_write32_set(rtwdev, R_P1_TXPW_RSTB, B_P1_TXPW_RSTB_MANON);
+		rtw89_phy_write32_set(rtwdev, R_PLCP_HISTOGRAM, B_STS_PARSING_TIME);
 	}
 	rtw89_phy_write32_mask(rtwdev, R_CFO_TRK0, B_CFO_TRK_MSK, 0x1f);
 	rtw89_phy_write32_mask(rtwdev, R_CFO_TRK1, B_CFO_TRK_MSK, 0x0c);
@@ -1130,35 +1157,38 @@ static void rtw8852a_bbrst_for_rfk(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852a_set_channel_bb(struct rtw89_dev *rtwdev,
-				    struct rtw89_channel_params *param,
+				    const struct rtw89_chan *chan,
 				    enum rtw89_phy_idx phy_idx)
 {
-	bool cck_en = param->center_chan <= 14;
-	u8 pri_ch_idx = param->pri_ch_idx;
+	bool cck_en = chan->channel <= 14;
+	u8 pri_ch_idx = chan->pri_ch_idx;
 
 	if (cck_en)
-		rtw8852a_ctrl_sco_cck(rtwdev, param->center_chan,
-				      param->primary_chan, param->bandwidth);
+		rtw8852a_ctrl_sco_cck(rtwdev, chan->channel,
+				      chan->primary_channel,
+				      chan->band_width);
 
-	rtw8852a_ctrl_ch(rtwdev, param->center_chan, phy_idx);
-	rtw8852a_ctrl_bw(rtwdev, pri_ch_idx, param->bandwidth, phy_idx);
+	rtw8852a_ctrl_ch(rtwdev, chan->channel, phy_idx);
+	rtw8852a_ctrl_bw(rtwdev, pri_ch_idx, chan->band_width, phy_idx);
 	if (cck_en) {
 		rtw89_phy_write32_mask(rtwdev, R_RXCCA, B_RXCCA_DIS, 0);
 	} else {
 		rtw89_phy_write32_mask(rtwdev, R_RXCCA, B_RXCCA_DIS, 1);
 		rtw8852a_bbrst_for_rfk(rtwdev, phy_idx);
 	}
-	rtw8852a_spur_elimination(rtwdev, param->center_chan);
+	rtw8852a_spur_elimination(rtwdev, chan->channel);
 	rtw89_phy_write32_mask(rtwdev, R_MAC_PIN_SEL, B_CH_IDX_SEG0,
-			       param->primary_chan);
+			       chan->primary_channel);
 	rtw8852a_bb_reset_all(rtwdev, phy_idx);
 }
 
 static void rtw8852a_set_channel(struct rtw89_dev *rtwdev,
-				 struct rtw89_channel_params *params)
+				 const struct rtw89_chan *chan,
+				 enum rtw89_mac_idx mac_idx,
+				 enum rtw89_phy_idx phy_idx)
 {
-	rtw8852a_set_channel_mac(rtwdev, params, RTW89_MAC_0);
-	rtw8852a_set_channel_bb(rtwdev, params, RTW89_PHY_0);
+	rtw8852a_set_channel_mac(rtwdev, chan, mac_idx);
+	rtw8852a_set_channel_bb(rtwdev, chan, phy_idx);
 }
 
 static void rtw8852a_dfs_en(struct rtw89_dev *rtwdev, bool en)
@@ -1209,25 +1239,27 @@ static void rtw8852a_adc_en(struct rtw89_dev *rtwdev, bool en)
 }
 
 static void rtw8852a_set_channel_help(struct rtw89_dev *rtwdev, bool enter,
-				      struct rtw89_channel_help_params *p)
+				      struct rtw89_channel_help_params *p,
+				      const struct rtw89_chan *chan,
+				      enum rtw89_mac_idx mac_idx,
+				      enum rtw89_phy_idx phy_idx)
 {
-	u8 phy_idx = RTW89_PHY_0;
-
 	if (enter) {
-		rtw89_chip_stop_sch_tx(rtwdev, RTW89_MAC_0, &p->tx_en, RTW89_SCH_TX_SEL_ALL);
-		rtw89_mac_cfg_ppdu_status(rtwdev, RTW89_MAC_0, false);
+		rtw89_chip_stop_sch_tx(rtwdev, mac_idx, &p->tx_en,
+				       RTW89_SCH_TX_SEL_ALL);
+		rtw89_mac_cfg_ppdu_status(rtwdev, mac_idx, false);
 		rtw8852a_dfs_en(rtwdev, false);
-		rtw8852a_tssi_cont_en_phyidx(rtwdev, false, RTW89_PHY_0);
+		rtw8852a_tssi_cont_en_phyidx(rtwdev, false, phy_idx);
 		rtw8852a_adc_en(rtwdev, false);
 		fsleep(40);
 		rtw8852a_bb_reset_en(rtwdev, phy_idx, false);
 	} else {
-		rtw89_mac_cfg_ppdu_status(rtwdev, RTW89_MAC_0, true);
+		rtw89_mac_cfg_ppdu_status(rtwdev, mac_idx, true);
 		rtw8852a_adc_en(rtwdev, true);
 		rtw8852a_dfs_en(rtwdev, true);
-		rtw8852a_tssi_cont_en_phyidx(rtwdev, true, RTW89_PHY_0);
+		rtw8852a_tssi_cont_en_phyidx(rtwdev, true, phy_idx);
 		rtw8852a_bb_reset_en(rtwdev, phy_idx, true);
-		rtw89_chip_resume_sch_tx(rtwdev, RTW89_MAC_0, p->tx_en);
+		rtw89_chip_resume_sch_tx(rtwdev, mac_idx, p->tx_en);
 	}
 }
 
@@ -1277,9 +1309,10 @@ static void rtw8852a_rfk_channel(struct rtw89_dev *rtwdev)
 	rtw8852a_dpk(rtwdev, phy_idx);
 }
 
-static void rtw8852a_rfk_band_changed(struct rtw89_dev *rtwdev)
+static void rtw8852a_rfk_band_changed(struct rtw89_dev *rtwdev,
+				      enum rtw89_phy_idx phy_idx)
 {
-	rtw8852a_tssi_scan(rtwdev, RTW89_PHY_0);
+	rtw8852a_tssi_scan(rtwdev, phy_idx);
 }
 
 static void rtw8852a_rfk_scan(struct rtw89_dev *rtwdev, bool start)
@@ -1378,9 +1411,11 @@ static void rtw8852a_set_txpwr_ref(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852a_set_txpwr_byrate(struct rtw89_dev *rtwdev,
+				      const struct rtw89_chan *chan,
 				      enum rtw89_phy_idx phy_idx)
 {
-	u8 ch = rtwdev->hal.current_channel;
+	u8 band = chan->band_type;
+	u8 ch = chan->channel;
 	static const u8 rs[] = {
 		RTW89_RS_CCK,
 		RTW89_RS_OFDM,
@@ -1406,7 +1441,8 @@ static void rtw8852a_set_txpwr_byrate(struct rtw89_dev *rtwdev,
 			for (j = 0; j < rtw89_rs_idx_max[rs[i]]; j++) {
 				cur.idx = j;
 				shf = (j % 4) * 8;
-				tmp = rtw89_phy_read_txpwr_byrate(rtwdev, &cur);
+				tmp = rtw89_phy_read_txpwr_byrate(rtwdev, band,
+								  &cur);
 				val |= (tmp << shf);
 
 				if ((j + 1) % 4)
@@ -1421,8 +1457,10 @@ static void rtw8852a_set_txpwr_byrate(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852a_set_txpwr_offset(struct rtw89_dev *rtwdev,
+				      const struct rtw89_chan *chan,
 				      enum rtw89_phy_idx phy_idx)
 {
+	u8 band = chan->band_type;
 	struct rtw89_rate_desc desc = {
 		.nss = RTW89_NSS_1,
 		.rs = RTW89_RS_OFFSET,
@@ -1433,7 +1471,7 @@ static void rtw8852a_set_txpwr_offset(struct rtw89_dev *rtwdev,
 	rtw89_debug(rtwdev, RTW89_DBG_TXPWR, "[TXPWR] set txpwr offset\n");
 
 	for (desc.idx = 0; desc.idx < RTW89_RATE_OFFSET_MAX; desc.idx++) {
-		v = rtw89_phy_read_txpwr_byrate(rtwdev, &desc);
+		v = rtw89_phy_read_txpwr_byrate(rtwdev, band, &desc);
 		val |= ((v & 0xf) << (4 * desc.idx));
 	}
 
@@ -1442,29 +1480,31 @@ static void rtw8852a_set_txpwr_offset(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852a_set_txpwr_limit(struct rtw89_dev *rtwdev,
+				     const struct rtw89_chan *chan,
 				     enum rtw89_phy_idx phy_idx)
 {
 #define __MAC_TXPWR_LMT_PAGE_SIZE 40
-	u8 ch = rtwdev->hal.current_channel;
-	u8 bw = rtwdev->hal.current_band_width;
+	u8 ch = chan->channel;
+	u8 bw = chan->band_width;
 	struct rtw89_txpwr_limit lmt[NTX_NUM_8852A];
 	u32 addr, val;
 	const s8 *ptr;
-	u8 i, j, k;
+	u8 i, j;
 
 	rtw89_debug(rtwdev, RTW89_DBG_TXPWR,
 		    "[TXPWR] set txpwr limit with ch=%d bw=%d\n", ch, bw);
 
 	for (i = 0; i < NTX_NUM_8852A; i++) {
-		rtw89_phy_fill_txpwr_limit(rtwdev, &lmt[i], i);
+		rtw89_phy_fill_txpwr_limit(rtwdev, chan, &lmt[i], i);
 
 		for (j = 0; j < __MAC_TXPWR_LMT_PAGE_SIZE; j += 4) {
 			addr = R_AX_PWR_LMT + j + __MAC_TXPWR_LMT_PAGE_SIZE * i;
 			ptr = (s8 *)&lmt[i] + j;
-			val = 0;
 
-			for (k = 0; k < 4; k++)
-				val |= (ptr[k] << (8 * k));
+			val = FIELD_PREP(GENMASK(7, 0), ptr[0]) |
+			      FIELD_PREP(GENMASK(15, 8), ptr[1]) |
+			      FIELD_PREP(GENMASK(23, 16), ptr[2]) |
+			      FIELD_PREP(GENMASK(31, 24), ptr[3]);
 
 			rtw89_mac_txpwr_write32(rtwdev, phy_idx, addr, val);
 		}
@@ -1473,30 +1513,32 @@ static void rtw8852a_set_txpwr_limit(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852a_set_txpwr_limit_ru(struct rtw89_dev *rtwdev,
+					const struct rtw89_chan *chan,
 					enum rtw89_phy_idx phy_idx)
 {
 #define __MAC_TXPWR_LMT_RU_PAGE_SIZE 24
-	u8 ch = rtwdev->hal.current_channel;
-	u8 bw = rtwdev->hal.current_band_width;
+	u8 ch = chan->channel;
+	u8 bw = chan->band_width;
 	struct rtw89_txpwr_limit_ru lmt_ru[NTX_NUM_8852A];
 	u32 addr, val;
 	const s8 *ptr;
-	u8 i, j, k;
+	u8 i, j;
 
 	rtw89_debug(rtwdev, RTW89_DBG_TXPWR,
 		    "[TXPWR] set txpwr limit ru with ch=%d bw=%d\n", ch, bw);
 
 	for (i = 0; i < NTX_NUM_8852A; i++) {
-		rtw89_phy_fill_txpwr_limit_ru(rtwdev, &lmt_ru[i], i);
+		rtw89_phy_fill_txpwr_limit_ru(rtwdev, chan, &lmt_ru[i], i);
 
 		for (j = 0; j < __MAC_TXPWR_LMT_RU_PAGE_SIZE; j += 4) {
 			addr = R_AX_PWR_RU_LMT + j +
 			       __MAC_TXPWR_LMT_RU_PAGE_SIZE * i;
 			ptr = (s8 *)&lmt_ru[i] + j;
-			val = 0;
 
-			for (k = 0; k < 4; k++)
-				val |= (ptr[k] << (8 * k));
+			val = FIELD_PREP(GENMASK(7, 0), ptr[0]) |
+			      FIELD_PREP(GENMASK(15, 8), ptr[1]) |
+			      FIELD_PREP(GENMASK(23, 16), ptr[2]) |
+			      FIELD_PREP(GENMASK(31, 24), ptr[3]);
 
 			rtw89_mac_txpwr_write32(rtwdev, phy_idx, addr, val);
 		}
@@ -1505,17 +1547,20 @@ static void rtw8852a_set_txpwr_limit_ru(struct rtw89_dev *rtwdev,
 #undef __MAC_TXPWR_LMT_RU_PAGE_SIZE
 }
 
-static void rtw8852a_set_txpwr(struct rtw89_dev *rtwdev)
+static void rtw8852a_set_txpwr(struct rtw89_dev *rtwdev,
+			       const struct rtw89_chan *chan,
+			       enum rtw89_phy_idx phy_idx)
 {
-	rtw8852a_set_txpwr_byrate(rtwdev, RTW89_PHY_0);
-	rtw8852a_set_txpwr_limit(rtwdev, RTW89_PHY_0);
-	rtw8852a_set_txpwr_limit_ru(rtwdev, RTW89_PHY_0);
+	rtw8852a_set_txpwr_byrate(rtwdev, chan, phy_idx);
+	rtw8852a_set_txpwr_offset(rtwdev, chan, phy_idx);
+	rtw8852a_set_txpwr_limit(rtwdev, chan, phy_idx);
+	rtw8852a_set_txpwr_limit_ru(rtwdev, chan, phy_idx);
 }
 
-static void rtw8852a_set_txpwr_ctrl(struct rtw89_dev *rtwdev)
+static void rtw8852a_set_txpwr_ctrl(struct rtw89_dev *rtwdev,
+				    enum rtw89_phy_idx phy_idx)
 {
-	rtw8852a_set_txpwr_ref(rtwdev, RTW89_PHY_0);
-	rtw8852a_set_txpwr_offset(rtwdev, RTW89_PHY_0);
+	rtw8852a_set_txpwr_ref(rtwdev, phy_idx);
 }
 
 static int
@@ -1592,10 +1637,12 @@ void rtw8852a_bb_set_pmac_tx(struct rtw89_dev *rtwdev,
 			     struct rtw8852a_bb_pmac_info *tx_info,
 			     enum rtw89_phy_idx idx)
 {
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+
 	if (!tx_info->en_pmac_tx) {
 		rtw8852a_stop_pmac_tx(rtwdev, tx_info, idx);
 		rtw89_phy_write32_idx(rtwdev, R_PD_CTRL, B_PD_HIT_DIS, 0, idx);
-		if (rtwdev->hal.current_band_type == RTW89_BAND_2G)
+		if (chan->band_type == RTW89_BAND_2G)
 			rtw89_phy_write32_clr(rtwdev, R_RXCCA, B_RXCCA_DIS);
 		return;
 	}
@@ -1797,6 +1844,9 @@ static void rtw8852a_btc_init_cfg(struct rtw89_dev *rtwdev)
 				      RF_PATH_A, BTC_BT_SS_GROUP, 0x5ff);
 		rtw8852a_set_trx_mask(rtwdev,
 				      RF_PATH_B, BTC_BT_SS_GROUP, 0x5ff);
+		/* set path-A(S0) Tx/Rx no-mask if GNT_WL=0 && BT_S1=tx group */
+		rtw8852a_set_trx_mask(rtwdev,
+				      RF_PATH_A, BTC_BT_TX_GROUP, 0x5ff);
 	} else { /* set WL Tx stb if GNT_WL = 0 && BT_S1 = ss group for 3-ant */
 		rtw8852a_set_trx_mask(rtwdev,
 				      RF_PATH_A, BTC_BT_SS_GROUP, 0x5df);
@@ -2010,6 +2060,51 @@ void rtw8852a_btc_wl_s1_standby(struct rtw89_dev *rtwdev, bool state)
 	rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0);
 }
 
+static void rtw8852a_set_wl_lna2(struct rtw89_dev *rtwdev, u8 level)
+{
+	/* level=0 Default:    TIA 1/0= (LNA2,TIAN6) = (7,1)/(5,1) = 21dB/12dB
+	 * level=1 Fix LNA2=5: TIA 1/0= (LNA2,TIAN6) = (5,0)/(5,1) = 18dB/12dB
+	 * To improve BT ACI in co-rx
+	 */
+
+	switch (level) {
+	case 0: /* default */
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x1000);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x3);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x17);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x2);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0);
+		break;
+	case 1: /* Fix LNA2=5  */
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x1000);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x3);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x5);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x2);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0);
+		break;
+	}
+}
+
+static void rtw8852a_btc_set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level)
+{
+	switch (level) {
+	case 0: /* original */
+		rtw8852a_bb_ctrl_btc_preagc(rtwdev, false);
+		rtw8852a_set_wl_lna2(rtwdev, 0);
+		break;
+	case 1: /* for FDD free-run */
+		rtw8852a_bb_ctrl_btc_preagc(rtwdev, true);
+		rtw8852a_set_wl_lna2(rtwdev, 0);
+		break;
+	case 2: /* for BTG Co-Rx*/
+		rtw8852a_bb_ctrl_btc_preagc(rtwdev, false);
+		rtw8852a_set_wl_lna2(rtwdev, 1);
+		break;
+	}
+}
+
 static void rtw8852a_fill_freq_with_ppdu(struct rtw89_dev *rtwdev,
 					 struct rtw89_rx_phy_ppdu *phy_ppdu,
 					 struct ieee80211_rx_status *status)
@@ -2030,12 +2125,12 @@ static void rtw8852a_query_ppdu(struct rtw89_dev *rtwdev,
 				struct ieee80211_rx_status *status)
 {
 	u8 path;
-	s8 *rx_power = phy_ppdu->rssi;
+	u8 *rx_power = phy_ppdu->rssi;
 
-	status->signal = max_t(s8, rx_power[RF_PATH_A], rx_power[RF_PATH_B]);
+	status->signal = RTW89_RSSI_RAW_TO_DBM(max(rx_power[RF_PATH_A], rx_power[RF_PATH_B]));
 	for (path = 0; path < rtwdev->chip->rf_path_num; path++) {
 		status->chains |= BIT(path);
-		status->chain_signal[path] = rx_power[path];
+		status->chain_signal[path] = RTW89_RSSI_RAW_TO_DBM(rx_power[path]);
 	}
 	if (phy_ppdu->valid)
 		rtw8852a_fill_freq_with_ppdu(rtwdev, phy_ppdu, status);
@@ -2086,6 +2181,8 @@ static const struct rtw89_chip_ops rtw8852a_chip_ops = {
 	.btc_bt_aci_imp		= rtw8852a_btc_bt_aci_imp,
 	.btc_update_bt_cnt	= rtw8852a_btc_update_bt_cnt,
 	.btc_wl_s1_standby	= rtw8852a_btc_wl_s1_standby,
+	.btc_set_wl_rx_gain	= rtw8852a_btc_set_wl_rx_gain,
+	.btc_set_policy		= rtw89_btc_set_policy,
 };
 
 const struct rtw89_chip_info rtw8852a_chip_info = {
@@ -2093,6 +2190,7 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 	.ops			= &rtw8852a_chip_ops,
 	.fw_name		= "rtw89/rtw8852a_fw.bin",
 	.fifo_size		= 458752,
+	.dle_scc_rsvd_size	= 0,
 	.max_amsdu_limit	= 3500,
 	.dis_2g_40m_ul_ofdma	= true,
 	.rsvd_ple_ofst		= 0x6f800,
@@ -2114,7 +2212,9 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 	.txpwr_factor_rf	= 2,
 	.txpwr_factor_mac	= 1,
 	.dig_table		= &rtw89_8852a_phy_dig_table,
+	.dig_regs		= &rtw8852a_dig_regs,
 	.tssi_dbw_table		= NULL,
+	.support_chanctx_num	= 1,
 	.support_bands		= BIT(NL80211_BAND_2GHZ) |
 				  BIT(NL80211_BAND_5GHZ),
 	.support_bw160		= false,
@@ -2125,6 +2225,9 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 	.acam_num		= 128,
 	.bcam_num		= 10,
 	.scam_num		= 128,
+	.bacam_num		= 2,
+	.bacam_dynamic_num	= 4,
+	.bacam_v1		= false,
 	.sec_ctrl_efuse_size	= 4,
 	.physical_efuse_size	= 1216,
 	.logical_efuse_size	= 1536,
@@ -2133,11 +2236,26 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 	.dav_log_efuse_size	= 0,
 	.phycap_addr		= 0x580,
 	.phycap_size		= 128,
-	.para_ver		= 0x05050864,
-	.wlcx_desired		= 0x05050000,
-	.btcx_desired		= 0x5,
+	.para_ver		= 0x0,
+	.wlcx_desired		= 0x06000000,
+	.btcx_desired		= 0x7,
 	.scbd			= 0x1,
 	.mailbox		= 0x1,
+	.btc_fwinfo_buf		= 1024,
+
+	.fcxbtcrpt_ver		= 1,
+	.fcxtdma_ver		= 1,
+	.fcxslots_ver		= 1,
+	.fcxcysta_ver		= 2,
+	.fcxstep_ver		= 2,
+	.fcxnullsta_ver		= 1,
+	.fcxmreg_ver		= 1,
+	.fcxgpiodbg_ver		= 1,
+	.fcxbtver_ver		= 1,
+	.fcxbtscan_ver		= 1,
+	.fcxbtafh_ver		= 1,
+	.fcxbtdevinfo_ver	= 1,
+
 	.afh_guard_ch		= 6,
 	.wl_rssi_thres		= rtw89_btc_8852a_wl_rssi_thres,
 	.bt_rssi_thres		= rtw89_btc_8852a_bt_rssi_thres,
@@ -2163,7 +2281,9 @@ const struct rtw89_chip_info rtw8852a_chip_info = {
 	.page_regs		= &rtw8852a_page_regs,
 	.dcfo_comp		= &rtw8852a_dcfo_comp,
 	.dcfo_comp_sft		= 3,
-	.imr_info		= &rtw8852a_imr_info
+	.imr_info		= &rtw8852a_imr_info,
+	.rrsr_cfgs		= &rtw8852a_rrsr_cfgs,
+	.dma_ch_mask		= 0,
 };
 EXPORT_SYMBOL(rtw8852a_chip_info);
 
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c b/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c
index 3d60feb78312..582ff0d3a9ea 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c
@@ -1359,7 +1359,7 @@ static void _iqk_get_ch_info(struct rtw89_dev *rtwdev,
 			     enum rtw89_phy_idx phy, u8 path)
 {
 	struct rtw89_iqk_info *iqk_info = &rtwdev->iqk;
-	struct rtw89_hal *hal = &rtwdev->hal;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	u32 reg_rf18 = 0x0, reg_35c = 0x0;
 	u8 idx = 0;
 	u8 get_empty_table = false;
@@ -1380,9 +1380,9 @@ static void _iqk_get_ch_info(struct rtw89_dev *rtwdev,
 	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]cfg ch = %d\n", reg_rf18);
 	reg_35c = rtw89_phy_read32_mask(rtwdev, 0x35c, 0x00000c00);
 
-	iqk_info->iqk_band[path] = hal->current_band_type;
-	iqk_info->iqk_bw[path] = hal->current_band_width;
-	iqk_info->iqk_ch[path] = hal->current_channel;
+	iqk_info->iqk_band[path] = chan->band_type;
+	iqk_info->iqk_bw[path] = chan->band_width;
+	iqk_info->iqk_ch[path] = chan->channel;
 
 	rtw89_debug(rtwdev, RTW89_DBG_RFK,
 		    "[IQK]iqk_info->iqk_band[%x] = 0x%x\n", path,
@@ -1879,13 +1879,12 @@ static void _dpk_information(struct rtw89_dev *rtwdev,
 			     enum rtw89_rf_path path)
 {
 	struct rtw89_dpk_info *dpk = &rtwdev->dpk;
-	struct rtw89_hal *hal = &rtwdev->hal;
-
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	u8 kidx = dpk->cur_idx[path];
 
-	dpk->bp[path][kidx].band = hal->current_band_type;
-	dpk->bp[path][kidx].ch = hal->current_channel;
-	dpk->bp[path][kidx].bw = hal->current_band_width;
+	dpk->bp[path][kidx].band = chan->band_type;
+	dpk->bp[path][kidx].ch = chan->channel;
+	dpk->bp[path][kidx].bw = chan->band_width;
 
 	rtw89_debug(rtwdev, RTW89_DBG_RFK,
 		    "[DPK] S%d[%d] (PHY%d): TSSI %s/ DBCC %s/ %s/ CH%d/ %s\n",
@@ -2358,6 +2357,7 @@ static u8 _dpk_agc(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 #define DPK_RXBB_UPPER 0x1f
 #define DPK_RXBB_LOWER 0
 #define DPK_GL_CRIT 7
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	u8 tmp_txagc, tmp_rxbb = 0, tmp_gl_idx = 0;
 	u8 agc_cnt = 0;
 	bool limited_rxbb = false;
@@ -2404,7 +2404,7 @@ static u8 _dpk_agc(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 				    "[DPK] Adjust RXBB (%d) = 0x%x\n", offset,
 				    tmp_rxbb);
 			if (offset != 0 || agc_cnt == 0) {
-				if (rtwdev->hal.current_band_width < RTW89_CHANNEL_WIDTH_80)
+				if (chan->band_width < RTW89_CHANNEL_WIDTH_80)
 					_dpk_bypass_rxcfir(rtwdev, path, true);
 				else
 					_dpk_lbk_rxiqk(rtwdev, phy, path);
@@ -2548,11 +2548,12 @@ static bool _dpk_reload_check(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 			      enum rtw89_rf_path path)
 {
 	struct rtw89_dpk_info *dpk = &rtwdev->dpk;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	bool is_reload = false;
 	u8 idx, cur_band, cur_ch;
 
-	cur_band = rtwdev->hal.current_band_type;
-	cur_ch = rtwdev->hal.current_channel;
+	cur_band = chan->band_type;
+	cur_ch = chan->channel;
 
 	for (idx = 0; idx < RTW89_DPK_BKUP_NUM; idx++) {
 		if (cur_band != dpk->bp[path][idx].band ||
@@ -2681,12 +2682,13 @@ static void _dpk_cal_select(struct rtw89_dev *rtwdev, bool force,
 static bool _dpk_bypass_check(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy)
 {
 	struct rtw89_fem_info *fem = &rtwdev->fem;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 
-	if (fem->epa_2g && rtwdev->hal.current_band_type == RTW89_BAND_2G) {
+	if (fem->epa_2g && chan->band_type == RTW89_BAND_2G) {
 		rtw89_debug(rtwdev, RTW89_DBG_RFK,
 			    "[DPK] Skip DPK due to 2G_ext_PA exist!!\n");
 		return true;
-	} else if (fem->epa_5g && rtwdev->hal.current_band_type == RTW89_BAND_5G) {
+	} else if (fem->epa_5g && chan->band_type == RTW89_BAND_5G) {
 		rtw89_debug(rtwdev, RTW89_DBG_RFK,
 			    "[DPK] Skip DPK due to 5G_ext_PA exist!!\n");
 		return true;
@@ -2842,7 +2844,8 @@ static void _dpk_track(struct rtw89_dev *rtwdev)
 static void _tssi_rf_setting(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 			     enum rtw89_rf_path path)
 {
-	enum rtw89_band band = rtwdev->hal.current_band_type;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	enum rtw89_band band = chan->band_type;
 
 	if (band == RTW89_BAND_2G)
 		rtw89_write_rf(rtwdev, path, RR_TXPOW, RR_TXPOW_TXG, 0x1);
@@ -2852,7 +2855,8 @@ static void _tssi_rf_setting(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 
 static void _tssi_set_sys(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy)
 {
-	enum rtw89_band band = rtwdev->hal.current_band_type;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	enum rtw89_band band = chan->band_type;
 
 	rtw89_rfk_parser(rtwdev, &rtw8852a_tssi_sys_defs_tbl);
 	rtw89_rfk_parser_by_cond(rtwdev, band == RTW89_BAND_2G,
@@ -2863,7 +2867,8 @@ static void _tssi_set_sys(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy)
 static void _tssi_ini_txpwr_ctrl_bb(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 				    enum rtw89_rf_path path)
 {
-	enum rtw89_band band = rtwdev->hal.current_band_type;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	enum rtw89_band band = chan->band_type;
 
 	rtw89_rfk_parser_by_cond(rtwdev, path == RF_PATH_A,
 				 &rtw8852a_tssi_txpwr_ctrl_bb_defs_a_tbl,
@@ -2905,8 +2910,9 @@ static void _tssi_set_tmeter_tbl(struct rtw89_dev *rtwdev, enum rtw89_phy_idx ph
 	__val;						\
 })
 	struct rtw89_tssi_info *tssi_info = &rtwdev->tssi;
-	u8 ch = rtwdev->hal.current_channel;
-	u8 subband = rtwdev->hal.current_subband;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 ch = chan->channel;
+	u8 subband = chan->subband_type;
 	const s8 *thm_up_a = NULL;
 	const s8 *thm_down_a = NULL;
 	const s8 *thm_up_b = NULL;
@@ -3099,7 +3105,8 @@ static void _tssi_set_txagc_offset_mv_avg(struct rtw89_dev *rtwdev,
 static void _tssi_pak(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 		      enum rtw89_rf_path path)
 {
-	u8 subband = rtwdev->hal.current_subband;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 subband = chan->subband_type;
 
 	switch (subband) {
 	default:
@@ -3275,7 +3282,8 @@ static s8 _tssi_get_ofdm_de(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 			    enum rtw89_rf_path path)
 {
 	struct rtw89_tssi_info *tssi_info = &rtwdev->tssi;
-	u8 ch = rtwdev->hal.current_channel;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 ch = chan->channel;
 	u32 gidx, gidx_1st, gidx_2nd;
 	s8 de_1st = 0;
 	s8 de_2nd = 0;
@@ -3312,7 +3320,8 @@ static s8 _tssi_get_ofdm_trim_de(struct rtw89_dev *rtwdev,
 				 enum rtw89_rf_path path)
 {
 	struct rtw89_tssi_info *tssi_info = &rtwdev->tssi;
-	u8 ch = rtwdev->hal.current_channel;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 ch = chan->channel;
 	u32 tgidx, tgidx_1st, tgidx_2nd;
 	s8 tde_1st = 0;
 	s8 tde_2nd = 0;
@@ -3350,6 +3359,7 @@ static void _tssi_set_efuse_to_de(struct rtw89_dev *rtwdev,
 {
 #define __DE_MASK 0x003ff000
 	struct rtw89_tssi_info *tssi_info = &rtwdev->tssi;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	static const u32 r_cck_long[RF_PATH_NUM_8852A] = {0x5858, 0x7858};
 	static const u32 r_cck_short[RF_PATH_NUM_8852A] = {0x5860, 0x7860};
 	static const u32 r_mcs_20m[RF_PATH_NUM_8852A] = {0x5838, 0x7838};
@@ -3358,7 +3368,7 @@ static void _tssi_set_efuse_to_de(struct rtw89_dev *rtwdev,
 	static const u32 r_mcs_80m_80m[RF_PATH_NUM_8852A] = {0x5850, 0x7850};
 	static const u32 r_mcs_5m[RF_PATH_NUM_8852A] = {0x5828, 0x7828};
 	static const u32 r_mcs_10m[RF_PATH_NUM_8852A] = {0x5830, 0x7830};
-	u8 ch = rtwdev->hal.current_channel;
+	u8 ch = chan->channel;
 	u8 i, gidx;
 	s8 ofdm_de;
 	s8 trim_de;
@@ -3478,9 +3488,11 @@ static void _tssi_track(struct rtw89_dev *rtwdev)
 static void _tssi_high_power(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy)
 {
 	struct rtw89_tssi_info *tssi_info = &rtwdev->tssi;
-	u8 ch = rtwdev->hal.current_channel, ch_tmp;
-	u8 bw = rtwdev->hal.current_band_width;
-	u8 subband = rtwdev->hal.current_subband;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 ch = chan->channel, ch_tmp;
+	u8 bw = chan->band_width;
+	u8 band = chan->band_type;
+	u8 subband = chan->subband_type;
 	s8 power;
 	s32 xdbm;
 
@@ -3491,7 +3503,7 @@ static void _tssi_high_power(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy)
 	else
 		ch_tmp = ch;
 
-	power = rtw89_phy_read_txpwr_limit(rtwdev, bw, RTW89_1TX,
+	power = rtw89_phy_read_txpwr_limit(rtwdev, band, bw, RTW89_1TX,
 					   RTW89_RS_MCS, RTW89_NONBF, ch_tmp);
 
 	xdbm = power * 100 / 4;
@@ -3523,9 +3535,11 @@ static void _tssi_hw_tx(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 static void _tssi_pre_tx(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy)
 {
 	struct rtw89_tssi_info *tssi_info = &rtwdev->tssi;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	const struct rtw89_chip_info *mac_reg = rtwdev->chip;
-	u8 ch = rtwdev->hal.current_channel, ch_tmp;
-	u8 bw = rtwdev->hal.current_band_width;
+	u8 ch = chan->channel, ch_tmp;
+	u8 bw = chan->band_width;
+	u8 band = chan->band_type;
 	u32 tx_en;
 	u8 phy_map = rtw89_btc_phymap(rtwdev, phy, 0);
 	s8 power;
@@ -3539,8 +3553,9 @@ static void _tssi_pre_tx(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy)
 	else
 		ch_tmp = ch;
 
-	power = rtw89_phy_read_txpwr_limit(rtwdev, RTW89_CHANNEL_WIDTH_20, RTW89_1TX,
-					   RTW89_RS_OFDM, RTW89_NONBF, ch_tmp);
+	power = rtw89_phy_read_txpwr_limit(rtwdev, band, RTW89_CHANNEL_WIDTH_20,
+					   RTW89_1TX, RTW89_RS_OFDM,
+					   RTW89_NONBF, ch_tmp);
 
 	xdbm = (power * 100) >> mac_reg->txpwr_factor_mac;
 
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852ae.c b/drivers/net/wireless/realtek/rtw89/rtw8852ae.c
index 190c4aefb02e..0cd8c0c44d19 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852ae.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852ae.c
@@ -33,14 +33,15 @@ static const struct rtw89_pci_info rtw8852a_pci_info = {
 	.max_tag_num_mask	= B_AX_MAX_TAG_NUM,
 	.rxbd_rwptr_clr_reg	= R_AX_RXBD_RWPTR_CLR,
 	.txbd_rwptr_clr2_reg	= R_AX_TXBD_RWPTR_CLR2,
-	.dma_stop1_reg		= R_AX_PCIE_DMA_STOP1,
-	.dma_stop2_reg		= R_AX_PCIE_DMA_STOP2,
-	.dma_busy1_reg		= R_AX_PCIE_DMA_BUSY1,
+	.dma_stop1		= {R_AX_PCIE_DMA_STOP1, B_AX_TX_STOP1_MASK},
+	.dma_stop2		= {R_AX_PCIE_DMA_STOP2, B_AX_TX_STOP2_ALL},
+	.dma_busy1		= {R_AX_PCIE_DMA_BUSY1, DMA_BUSY1_CHECK},
 	.dma_busy2_reg		= R_AX_PCIE_DMA_BUSY2,
 	.dma_busy3_reg		= R_AX_PCIE_DMA_BUSY1,
 
 	.rpwm_addr		= R_AX_PCIE_HRPWM,
 	.cpwm_addr		= R_AX_CPWM,
+	.tx_dma_ch_mask		= 0,
 	.bd_idx_addr_low_power	= NULL,
 	.dma_addr_set		= &rtw89_pci_ch_dma_addr_set,
 
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852b.c b/drivers/net/wireless/realtek/rtw89/rtw8852b.c
new file mode 100644
index 000000000000..9f9908418ee4
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852b.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2019-2022  Realtek Corporation
+ */
+
+#include "core.h"
+#include "mac.h"
+#include "reg.h"
+
+static const struct rtw89_dle_mem rtw8852b_dle_mem_pcie[] = {
+	[RTW89_QTA_SCC] = {RTW89_QTA_SCC, &rtw89_mac_size.wde_size6,
+			   &rtw89_mac_size.ple_size6, &rtw89_mac_size.wde_qt6,
+			   &rtw89_mac_size.wde_qt6, &rtw89_mac_size.ple_qt18,
+			   &rtw89_mac_size.ple_qt58},
+	[RTW89_QTA_DLFW] = {RTW89_QTA_DLFW, &rtw89_mac_size.wde_size9,
+			    &rtw89_mac_size.ple_size8, &rtw89_mac_size.wde_qt4,
+			    &rtw89_mac_size.wde_qt4, &rtw89_mac_size.ple_qt13,
+			    &rtw89_mac_size.ple_qt13},
+	[RTW89_QTA_INVALID] = {RTW89_QTA_INVALID, NULL, NULL, NULL, NULL, NULL,
+			       NULL},
+};
+
+static int rtw8852b_mac_enable_bb_rf(struct rtw89_dev *rtwdev)
+{
+	int ret;
+
+	rtw89_write8_set(rtwdev, R_AX_SYS_FUNC_EN,
+			 B_AX_FEN_BBRSTB | B_AX_FEN_BB_GLB_RSTN);
+	rtw89_write32_mask(rtwdev, R_AX_SPS_DIG_ON_CTRL0, B_AX_REG_ZCDC_H_MASK, 0x1);
+	rtw89_write32_set(rtwdev, R_AX_WLRF_CTRL, B_AX_AFC_AFEDIG);
+	rtw89_write32_clr(rtwdev, R_AX_WLRF_CTRL, B_AX_AFC_AFEDIG);
+	rtw89_write32_set(rtwdev, R_AX_WLRF_CTRL, B_AX_AFC_AFEDIG);
+
+	ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S0, 0xC7,
+				      FULL_BIT_MASK);
+	if (ret)
+		return ret;
+
+	ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S1, 0xC7,
+				      FULL_BIT_MASK);
+	if (ret)
+		return ret;
+
+	rtw89_write8(rtwdev, R_AX_PHYREG_SET, PHYREG_SET_XYN_CYCLE);
+
+	return 0;
+}
+
+static int rtw8852b_mac_disable_bb_rf(struct rtw89_dev *rtwdev)
+{
+	u8 wl_rfc_s0;
+	u8 wl_rfc_s1;
+	int ret;
+
+	rtw89_write8_clr(rtwdev, R_AX_SYS_FUNC_EN,
+			 B_AX_FEN_BBRSTB | B_AX_FEN_BB_GLB_RSTN);
+
+	ret = rtw89_mac_read_xtal_si(rtwdev, XTAL_SI_WL_RFC_S0, &wl_rfc_s0);
+	if (ret)
+		return ret;
+	wl_rfc_s0 &= ~XTAL_SI_RF00S_EN;
+	ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S0, wl_rfc_s0,
+				      FULL_BIT_MASK);
+	if (ret)
+		return ret;
+
+	ret = rtw89_mac_read_xtal_si(rtwdev, XTAL_SI_WL_RFC_S1, &wl_rfc_s1);
+	if (ret)
+		return ret;
+	wl_rfc_s1 &= ~XTAL_SI_RF10S_EN;
+	ret = rtw89_mac_write_xtal_si(rtwdev, XTAL_SI_WL_RFC_S1, wl_rfc_s1,
+				      FULL_BIT_MASK);
+	return ret;
+}
+
+static const struct rtw89_chip_ops rtw8852b_chip_ops = {
+	.enable_bb_rf		= rtw8852b_mac_enable_bb_rf,
+	.disable_bb_rf		= rtw8852b_mac_disable_bb_rf,
+};
+
+const struct rtw89_chip_info rtw8852b_chip_info = {
+	.chip_id		= RTL8852B,
+	.fifo_size		= 196608,
+	.dle_scc_rsvd_size	= 98304,
+	.dle_mem		= rtw8852b_dle_mem_pcie,
+	.dma_ch_mask		= BIT(RTW89_DMA_ACH4) | BIT(RTW89_DMA_ACH5) |
+				  BIT(RTW89_DMA_ACH6) | BIT(RTW89_DMA_ACH7) |
+				  BIT(RTW89_DMA_B1MG) | BIT(RTW89_DMA_B1HI),
+};
+EXPORT_SYMBOL(rtw8852b_chip_info);
+
+MODULE_FIRMWARE("rtw89/rtw8852b_fw.bin");
+MODULE_AUTHOR("Realtek Corporation");
+MODULE_DESCRIPTION("Realtek 802.11ax wireless 8852B driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852be.c b/drivers/net/wireless/realtek/rtw89/rtw8852be.c
new file mode 100644
index 000000000000..7bf95c38d3eb
--- /dev/null
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852be.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/* Copyright(c) 2020-2022  Realtek Corporation
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include "pci.h"
+#include "reg.h"
+
+static const struct rtw89_pci_info rtw8852b_pci_info = {
+	.dma_stop1		= {R_AX_PCIE_DMA_STOP1, B_AX_TX_STOP1_MASK_V1},
+	.dma_stop2		= {0},
+	.dma_busy1		= {R_AX_PCIE_DMA_BUSY1, DMA_BUSY1_CHECK_V1},
+	.dma_busy2_reg		= 0,
+	.dma_busy3_reg		= R_AX_PCIE_DMA_BUSY1,
+
+	.tx_dma_ch_mask		= BIT(RTW89_TXCH_ACH4) | BIT(RTW89_TXCH_ACH5) |
+				  BIT(RTW89_TXCH_ACH6) | BIT(RTW89_TXCH_ACH7) |
+				  BIT(RTW89_TXCH_CH10) | BIT(RTW89_TXCH_CH11),
+};
+
+MODULE_AUTHOR("Realtek Corporation");
+MODULE_DESCRIPTION("Realtek 802.11ax wireless 8852BE driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c.c b/drivers/net/wireless/realtek/rtw89/rtw8852c.c
index b697aef2faf2..67653b3e1a35 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852c.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852c.c
@@ -109,6 +109,7 @@ static const struct rtw89_imr_info rtw8852c_imr_info = {
 	.cpu_disp_imr_set	= B_AX_CPU_DISP_IMR_SET_V1,
 	.other_disp_imr_clr	= B_AX_OTHER_DISP_IMR_CLR_V1,
 	.other_disp_imr_set	= B_AX_OTHER_DISP_IMR_SET_V1,
+	.bbrpt_com_err_imr_reg	= R_AX_BBRPT_COM_ERR_IMR,
 	.bbrpt_chinfo_err_imr_reg = R_AX_BBRPT_CHINFO_ERR_IMR,
 	.bbrpt_err_imr_set	= R_AX_BBRPT_CHINFO_IMR_SET_V1,
 	.bbrpt_dfs_err_imr_reg	= R_AX_BBRPT_DFS_ERR_IMR,
@@ -131,7 +132,34 @@ static const struct rtw89_imr_info rtw8852c_imr_info = {
 	.tmac_imr_set		= B_AX_TMAC_IMR_SET_V1,
 };
 
+static const struct rtw89_rrsr_cfgs rtw8852c_rrsr_cfgs = {
+	.ref_rate = {R_AX_TRXPTCL_RRSR_CTL_0, B_AX_WMAC_RESP_REF_RATE_SEL, 0},
+	.rsc = {R_AX_PTCL_RRSR1, B_AX_RSC_MASK, 2},
+};
+
+static const struct rtw89_dig_regs rtw8852c_dig_regs = {
+	.seg0_pd_reg = R_SEG0R_PD,
+	.pd_lower_bound_mask = B_SEG0R_PD_LOWER_BOUND_MSK,
+	.pd_spatial_reuse_en = B_SEG0R_PD_SPATIAL_REUSE_EN_MSK,
+	.p0_lna_init = {R_PATH0_LNA_INIT_V1, B_PATH0_LNA_INIT_IDX_MSK},
+	.p1_lna_init = {R_PATH1_LNA_INIT_V1, B_PATH1_LNA_INIT_IDX_MSK},
+	.p0_tia_init = {R_PATH0_TIA_INIT_V1, B_PATH0_TIA_INIT_IDX_MSK_V1},
+	.p1_tia_init = {R_PATH1_TIA_INIT_V1, B_PATH1_TIA_INIT_IDX_MSK_V1},
+	.p0_rxb_init = {R_PATH0_RXB_INIT_V1, B_PATH0_RXB_INIT_IDX_MSK_V1},
+	.p1_rxb_init = {R_PATH1_RXB_INIT_V1, B_PATH1_RXB_INIT_IDX_MSK_V1},
+	.p0_p20_pagcugc_en = {R_PATH0_P20_FOLLOW_BY_PAGCUGC_V1,
+			      B_PATH0_P20_FOLLOW_BY_PAGCUGC_EN_MSK},
+	.p0_s20_pagcugc_en = {R_PATH0_S20_FOLLOW_BY_PAGCUGC_V1,
+			      B_PATH0_S20_FOLLOW_BY_PAGCUGC_EN_MSK},
+	.p1_p20_pagcugc_en = {R_PATH1_P20_FOLLOW_BY_PAGCUGC_V1,
+			      B_PATH1_P20_FOLLOW_BY_PAGCUGC_EN_MSK},
+	.p1_s20_pagcugc_en = {R_PATH1_S20_FOLLOW_BY_PAGCUGC_V1,
+			      B_PATH1_S20_FOLLOW_BY_PAGCUGC_EN_MSK},
+};
+
 static void rtw8852c_ctrl_btg(struct rtw89_dev *rtwdev, bool btg);
+static void rtw8852c_ctrl_tx_path_tmac(struct rtw89_dev *rtwdev, u8 tx_path,
+				       enum rtw89_mac_idx mac_idx);
 
 static int rtw8852c_pwr_on_func(struct rtw89_dev *rtwdev)
 {
@@ -567,7 +595,7 @@ static void rtw8852c_power_trim(struct rtw89_dev *rtwdev)
 }
 
 static void rtw8852c_set_channel_mac(struct rtw89_dev *rtwdev,
-				     struct rtw89_channel_params *param,
+				     const struct rtw89_chan *chan,
 				     u8 mac_idx)
 {
 	u32 rf_mod = rtw89_mac_reg_by_idx(R_AX_WMAC_RFMOD, mac_idx);
@@ -578,24 +606,24 @@ static void rtw8852c_set_channel_mac(struct rtw89_dev *rtwdev,
 	u8 rf_mod_val = 0, chk_rate_mask = 0;
 	u32 txsc;
 
-	switch (param->bandwidth) {
+	switch (chan->band_width) {
 	case RTW89_CHANNEL_WIDTH_160:
-		txsc80 = rtw89_phy_get_txsc(rtwdev, param,
+		txsc80 = rtw89_phy_get_txsc(rtwdev, chan,
 					    RTW89_CHANNEL_WIDTH_80);
 		fallthrough;
 	case RTW89_CHANNEL_WIDTH_80:
-		txsc40 = rtw89_phy_get_txsc(rtwdev, param,
+		txsc40 = rtw89_phy_get_txsc(rtwdev, chan,
 					    RTW89_CHANNEL_WIDTH_40);
 		fallthrough;
 	case RTW89_CHANNEL_WIDTH_40:
-		txsc20 = rtw89_phy_get_txsc(rtwdev, param,
+		txsc20 = rtw89_phy_get_txsc(rtwdev, chan,
 					    RTW89_CHANNEL_WIDTH_20);
 		break;
 	default:
 		break;
 	}
 
-	switch (param->bandwidth) {
+	switch (chan->band_width) {
 	case RTW89_CHANNEL_WIDTH_160:
 		rf_mod_val = AX_WMAC_RFMOD_160M;
 		txsc = FIELD_PREP(B_AX_TXSC_20M_MASK, txsc20) |
@@ -620,7 +648,7 @@ static void rtw8852c_set_channel_mac(struct rtw89_dev *rtwdev,
 	rtw89_write8_mask(rtwdev, rf_mod, B_AX_WMAC_RFMOD_MASK, rf_mod_val);
 	rtw89_write32(rtwdev, sub_carr, txsc);
 
-	switch (param->band_type) {
+	switch (chan->band_type) {
 	case RTW89_BAND_2G:
 		chk_rate_mask = B_AX_BAND_MODE;
 		break;
@@ -629,7 +657,7 @@ static void rtw8852c_set_channel_mac(struct rtw89_dev *rtwdev,
 		chk_rate_mask = B_AX_CHECK_CCK_EN | B_AX_RTS_LIMIT_IN_OFDM6;
 		break;
 	default:
-		rtw89_warn(rtwdev, "Invalid band_type:%d\n", param->band_type);
+		rtw89_warn(rtwdev, "Invalid band_type:%d\n", chan->band_type);
 		return;
 	}
 	rtw89_write8_clr(rtwdev, chk_rate, B_AX_BAND_MODE | B_AX_CHECK_CCK_EN |
@@ -920,7 +948,7 @@ static void rtw8852c_decode_chan_idx(struct rtw89_dev *rtwdev, u8 chan_idx,
 }
 
 static void rtw8852c_set_gain_offset(struct rtw89_dev *rtwdev,
-				     const struct rtw89_channel_params *param,
+				     const struct rtw89_chan *chan,
 				     enum rtw89_phy_idx phy_idx,
 				     enum rtw89_rf_path path)
 {
@@ -939,7 +967,7 @@ static void rtw8852c_set_gain_offset(struct rtw89_dev *rtwdev,
 	if (rtwdev->dbcc_en && path == RF_PATH_B)
 		phy_idx = RTW89_PHY_1;
 
-	if (param->band_type == RTW89_BAND_2G) {
+	if (chan->band_type == RTW89_BAND_2G) {
 		offset_q0 = efuse_gain->offset[path][RTW89_GAIN_OFFSET_2G_CCK];
 		offset_base_q4 = efuse_gain->offset_base[phy_idx];
 
@@ -948,7 +976,7 @@ static void rtw8852c_set_gain_offset(struct rtw89_dev *rtwdev,
 		rtw89_phy_write32_mask(rtwdev, R_RPL_OFST, B_RPL_OFST_MASK, tmp & 0x7f);
 	}
 
-	switch (param->subband_type) {
+	switch (chan->subband_type) {
 	default:
 	case RTW89_CH_2G:
 		gain_band = RTW89_GAIN_OFFSET_2G_OFDM;
@@ -977,14 +1005,14 @@ static void rtw8852c_set_gain_offset(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev,
-			     const struct rtw89_channel_params *param,
+			     const struct rtw89_chan *chan,
 			     enum rtw89_phy_idx phy_idx)
 {
 	u8 sco;
-	u16 central_freq = param->center_freq;
-	u8 central_ch = param->center_chan;
-	u8 band = param->band_type;
-	u8 subband = param->subband_type;
+	u16 central_freq = chan->freq;
+	u8 central_ch = chan->channel;
+	u8 band = chan->band_type;
+	u8 subband = chan->subband_type;
 	bool is_2g = band == RTW89_BAND_2G;
 	u8 chan_idx;
 
@@ -996,7 +1024,7 @@ static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev,
 	if (phy_idx == RTW89_PHY_0) {
 		/* Path A */
 		rtw8852c_set_gain_error(rtwdev, subband, RF_PATH_A);
-		rtw8852c_set_gain_offset(rtwdev, param, phy_idx, RF_PATH_A);
+		rtw8852c_set_gain_offset(rtwdev, chan, phy_idx, RF_PATH_A);
 
 		if (is_2g)
 			rtw89_phy_write32_idx(rtwdev, R_PATH0_BAND_SEL_V1,
@@ -1009,7 +1037,7 @@ static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev,
 		/* Path B */
 		if (!rtwdev->dbcc_en) {
 			rtw8852c_set_gain_error(rtwdev, subband, RF_PATH_B);
-			rtw8852c_set_gain_offset(rtwdev, param, phy_idx, RF_PATH_B);
+			rtw8852c_set_gain_offset(rtwdev, chan, phy_idx, RF_PATH_B);
 
 			if (is_2g)
 				rtw89_phy_write32_idx(rtwdev,
@@ -1038,7 +1066,7 @@ static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev,
 	} else {
 		/* Path B */
 		rtw8852c_set_gain_error(rtwdev, subband, RF_PATH_B);
-		rtw8852c_set_gain_offset(rtwdev, param, phy_idx, RF_PATH_B);
+		rtw8852c_set_gain_offset(rtwdev, chan, phy_idx, RF_PATH_B);
 
 		if (is_2g)
 			rtw89_phy_write32_idx(rtwdev, R_PATH1_BAND_SEL_V1,
@@ -1095,7 +1123,7 @@ static void rtw8852c_ctrl_ch(struct rtw89_dev *rtwdev,
 		}
 	}
 
-	chan_idx = rtw8852c_encode_chan_idx(rtwdev, param->primary_chan, band);
+	chan_idx = rtw8852c_encode_chan_idx(rtwdev, chan->primary_channel, band);
 	rtw89_phy_write32_idx(rtwdev, R_MAC_PIN_SEL, B_CH_IDX_SEG0, chan_idx, phy_idx);
 }
 
@@ -1246,12 +1274,12 @@ rtw8852c_ctrl_bw(struct rtw89_dev *rtwdev, u8 pri_ch, u8 bw,
 }
 
 static u32 rtw8852c_spur_freq(struct rtw89_dev *rtwdev,
-			      struct rtw89_channel_params *param)
+			      const struct rtw89_chan *chan)
 {
-	u8 center_chan = param->center_chan;
-	u8 bw = param->bandwidth;
+	u8 center_chan = chan->channel;
+	u8 bw = chan->band_width;
 
-	switch (param->band_type) {
+	switch (chan->band_type) {
 	case RTW89_BAND_2G:
 		if (bw == RTW89_CHANNEL_WIDTH_20) {
 			if (center_chan >= 5 && center_chan <= 8)
@@ -1285,19 +1313,19 @@ static u32 rtw8852c_spur_freq(struct rtw89_dev *rtwdev,
 #define MAX_TONE_NUM 2048
 
 static void rtw8852c_set_csi_tone_idx(struct rtw89_dev *rtwdev,
-				      struct rtw89_channel_params *param,
+				      const struct rtw89_chan *chan,
 				      enum rtw89_phy_idx phy_idx)
 {
 	u32 spur_freq;
 	s32 freq_diff, csi_idx, csi_tone_idx;
 
-	spur_freq = rtw8852c_spur_freq(rtwdev, param);
+	spur_freq = rtw8852c_spur_freq(rtwdev, chan);
 	if (spur_freq == 0) {
 		rtw89_phy_write32_idx(rtwdev, R_SEG0CSI_EN, B_SEG0CSI_EN, 0, phy_idx);
 		return;
 	}
 
-	freq_diff = (spur_freq - param->center_freq) * 1000000;
+	freq_diff = (spur_freq - chan->freq) * 1000000;
 	csi_idx = s32_div_u32_round_closest(freq_diff, CARRIER_SPACING_78_125);
 	s32_div_u32_round_down(csi_idx, MAX_TONE_NUM, &csi_tone_idx);
 
@@ -1325,7 +1353,7 @@ static const struct rtw89_nbi_reg_def rtw8852c_nbi_reg_def[] = {
 };
 
 static void rtw8852c_set_nbi_tone_idx(struct rtw89_dev *rtwdev,
-				      struct rtw89_channel_params *param,
+				      const struct rtw89_chan *chan,
 				      enum rtw89_rf_path path)
 {
 	const struct rtw89_nbi_reg_def *nbi = &rtw8852c_nbi_reg_def[path];
@@ -1335,34 +1363,37 @@ static void rtw8852c_set_nbi_tone_idx(struct rtw89_dev *rtwdev,
 	s32 nbi_frac_idx, nbi_frac_tone_idx;
 	bool notch2_chk = false;
 
-	spur_freq = rtw8852c_spur_freq(rtwdev, param);
+	spur_freq = rtw8852c_spur_freq(rtwdev, chan);
 	if (spur_freq == 0) {
 		rtw89_phy_write32_mask(rtwdev, nbi->notch1_en.addr, nbi->notch1_en.mask, 0);
 		rtw89_phy_write32_mask(rtwdev, nbi->notch1_en.addr, nbi->notch1_en.mask, 0);
 		return;
 	}
 
-	fc = param->center_freq;
-	if (param->bandwidth == RTW89_CHANNEL_WIDTH_160) {
+	fc = chan->freq;
+	if (chan->band_width == RTW89_CHANNEL_WIDTH_160) {
 		fc = (spur_freq > fc) ? fc + 40 : fc - 40;
-		if ((fc > spur_freq && param->center_chan < param->primary_chan) ||
-		    (fc < spur_freq && param->center_chan > param->primary_chan))
+		if ((fc > spur_freq &&
+		     chan->channel < chan->primary_channel) ||
+		    (fc < spur_freq &&
+		     chan->channel > chan->primary_channel))
 			notch2_chk = true;
 	}
 
 	freq_diff = (spur_freq - fc) * 1000000;
 	nbi_idx = s32_div_u32_round_down(freq_diff, CARRIER_SPACING_312_5, &nbi_frac_idx);
 
-	if (param->bandwidth == RTW89_CHANNEL_WIDTH_20) {
+	if (chan->band_width == RTW89_CHANNEL_WIDTH_20) {
 		s32_div_u32_round_down(nbi_idx + 32, 64, &nbi_tone_idx);
 	} else {
-		u16 tone_para = (param->bandwidth == RTW89_CHANNEL_WIDTH_40) ? 128 : 256;
+		u16 tone_para = (chan->band_width == RTW89_CHANNEL_WIDTH_40) ?
+				128 : 256;
 
 		s32_div_u32_round_down(nbi_idx, tone_para, &nbi_tone_idx);
 	}
 	nbi_frac_tone_idx = s32_div_u32_round_closest(nbi_frac_idx, CARRIER_SPACING_78_125);
 
-	if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 && notch2_chk) {
+	if (chan->band_width == RTW89_CHANNEL_WIDTH_160 && notch2_chk) {
 		rtw89_phy_write32_mask(rtwdev, nbi->notch2_idx.addr,
 				       nbi->notch2_idx.mask, nbi_tone_idx);
 		rtw89_phy_write32_mask(rtwdev, nbi->notch2_frac_idx.addr,
@@ -1404,42 +1435,42 @@ static void rtw8852c_spur_notch(struct rtw89_dev *rtwdev, u32 val,
 }
 
 static void rtw8852c_spur_elimination(struct rtw89_dev *rtwdev,
-				      struct rtw89_channel_params *param,
+				      const struct rtw89_chan *chan,
 				      u8 pri_ch_idx,
 				      enum rtw89_phy_idx phy_idx)
 {
-	rtw8852c_set_csi_tone_idx(rtwdev, param, phy_idx);
+	rtw8852c_set_csi_tone_idx(rtwdev, chan, phy_idx);
 
 	if (phy_idx == RTW89_PHY_0) {
-		if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 &&
+		if (chan->band_width == RTW89_CHANNEL_WIDTH_160 &&
 		    (pri_ch_idx == RTW89_SC_20_LOWER ||
 		     pri_ch_idx == RTW89_SC_20_UP3X)) {
 			rtw8852c_spur_notch(rtwdev, 0xe7f, RTW89_PHY_0);
 			if (!rtwdev->dbcc_en)
 				rtw8852c_spur_notch(rtwdev, 0xe7f, RTW89_PHY_1);
-		} else if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 &&
+		} else if (chan->band_width == RTW89_CHANNEL_WIDTH_160 &&
 			   (pri_ch_idx == RTW89_SC_20_UPPER ||
 			    pri_ch_idx == RTW89_SC_20_LOW3X)) {
 			rtw8852c_spur_notch(rtwdev, 0x280, RTW89_PHY_0);
 			if (!rtwdev->dbcc_en)
 				rtw8852c_spur_notch(rtwdev, 0x280, RTW89_PHY_1);
 		} else {
-			rtw8852c_set_nbi_tone_idx(rtwdev, param, RF_PATH_A);
+			rtw8852c_set_nbi_tone_idx(rtwdev, chan, RF_PATH_A);
 			if (!rtwdev->dbcc_en)
-				rtw8852c_set_nbi_tone_idx(rtwdev, param,
+				rtw8852c_set_nbi_tone_idx(rtwdev, chan,
 							  RF_PATH_B);
 		}
 	} else {
-		if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 &&
+		if (chan->band_width == RTW89_CHANNEL_WIDTH_160 &&
 		    (pri_ch_idx == RTW89_SC_20_LOWER ||
 		     pri_ch_idx == RTW89_SC_20_UP3X)) {
 			rtw8852c_spur_notch(rtwdev, 0xe7f, RTW89_PHY_1);
-		} else if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 &&
+		} else if (chan->band_width == RTW89_CHANNEL_WIDTH_160 &&
 			   (pri_ch_idx == RTW89_SC_20_UPPER ||
 			    pri_ch_idx == RTW89_SC_20_LOW3X)) {
 			rtw8852c_spur_notch(rtwdev, 0x280, RTW89_PHY_1);
 		} else {
-			rtw8852c_set_nbi_tone_idx(rtwdev, param, RF_PATH_B);
+			rtw8852c_set_nbi_tone_idx(rtwdev, chan, RF_PATH_B);
 		}
 	}
 
@@ -1450,14 +1481,14 @@ static void rtw8852c_spur_elimination(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852c_5m_mask(struct rtw89_dev *rtwdev,
-			     struct rtw89_channel_params *param,
+			     const struct rtw89_chan *chan,
 			     enum rtw89_phy_idx phy_idx)
 {
-	u8 pri_ch = param->primary_chan;
+	u8 pri_ch = chan->primary_channel;
 	bool mask_5m_low;
 	bool mask_5m_en;
 
-	switch (param->bandwidth) {
+	switch (chan->band_width) {
 	case RTW89_CHANNEL_WIDTH_40:
 		mask_5m_en = true;
 		mask_5m_low = pri_ch == 2;
@@ -1526,11 +1557,9 @@ static void rtw8852c_bb_reset_all(struct rtw89_dev *rtwdev,
 			      phy_idx);
 }
 
-static void rtw8852c_bb_reset_en(struct rtw89_dev *rtwdev,
+static void rtw8852c_bb_reset_en(struct rtw89_dev *rtwdev, enum rtw89_band band,
 				 enum rtw89_phy_idx phy_idx, bool en)
 {
-	struct rtw89_hal *hal = &rtwdev->hal;
-
 	if (en) {
 		rtw89_phy_write32_idx(rtwdev, R_S0_HW_SI_DIS,
 				      B_S0_HW_SI_DIS_W_R_TRIG, 0x0, phy_idx);
@@ -1538,7 +1567,7 @@ static void rtw8852c_bb_reset_en(struct rtw89_dev *rtwdev,
 				      B_S1_HW_SI_DIS_W_R_TRIG, 0x0, phy_idx);
 		rtw89_phy_write32_idx(rtwdev, R_RSTB_ASYNC, B_RSTB_ASYNC_ALL, 1,
 				      phy_idx);
-		if (hal->current_band_type == RTW89_BAND_2G)
+		if (band == RTW89_BAND_2G)
 			rtw89_phy_write32_mask(rtwdev, R_RXCCA_V1, B_RXCCA_DIS_V1, 0x0);
 		rtw89_phy_write32_mask(rtwdev, R_PD_CTRL, B_PD_HIT_DIS, 0x0);
 	} else {
@@ -1690,21 +1719,24 @@ static void rtw8852c_bb_sethw(struct rtw89_dev *rtwdev)
 }
 
 static void rtw8852c_set_channel_bb(struct rtw89_dev *rtwdev,
-				    struct rtw89_channel_params *param,
+				    const struct rtw89_chan *chan,
 				    enum rtw89_phy_idx phy_idx)
 {
-	bool cck_en = param->band_type == RTW89_BAND_2G;
-	u8 pri_ch_idx = param->pri_ch_idx;
+	struct rtw89_hal *hal = &rtwdev->hal;
+	bool cck_en = chan->band_type == RTW89_BAND_2G;
+	u8 pri_ch_idx = chan->pri_ch_idx;
 	u32 mask, reg;
 	u32 ru_alloc_msk[2] = {B_P80_AT_HIGH_FREQ_RU_ALLOC_PHY0,
 			       B_P80_AT_HIGH_FREQ_RU_ALLOC_PHY1};
+	u8 ntx_path;
 
-	if (param->band_type == RTW89_BAND_2G)
-		rtw8852c_ctrl_sco_cck(rtwdev, param->center_chan,
-				      param->primary_chan, param->bandwidth);
+	if (chan->band_type == RTW89_BAND_2G)
+		rtw8852c_ctrl_sco_cck(rtwdev, chan->channel,
+				      chan->primary_channel,
+				      chan->band_width);
 
-	rtw8852c_ctrl_ch(rtwdev, param, phy_idx);
-	rtw8852c_ctrl_bw(rtwdev, pri_ch_idx, param->bandwidth, phy_idx);
+	rtw8852c_ctrl_ch(rtwdev, chan, phy_idx);
+	rtw8852c_ctrl_bw(rtwdev, pri_ch_idx, chan->band_width, phy_idx);
 	if (cck_en) {
 		rtw89_phy_write32_mask(rtwdev, R_UPD_CLK_ADC, B_ENABLE_CCK, 1);
 		rtw89_phy_write32_mask(rtwdev, R_RXCCA_V1, B_RXCCA_DIS_V1, 0);
@@ -1717,17 +1749,17 @@ static void rtw8852c_set_channel_bb(struct rtw89_dev *rtwdev,
 				      B_PD_ARBITER_OFF, 0x1, phy_idx);
 	}
 
-	rtw8852c_spur_elimination(rtwdev, param, pri_ch_idx, phy_idx);
-	rtw8852c_ctrl_btg(rtwdev, param->band_type == RTW89_BAND_2G);
-	rtw8852c_5m_mask(rtwdev, param, phy_idx);
+	rtw8852c_spur_elimination(rtwdev, chan, pri_ch_idx, phy_idx);
+	rtw8852c_ctrl_btg(rtwdev, chan->band_type == RTW89_BAND_2G);
+	rtw8852c_5m_mask(rtwdev, chan, phy_idx);
 
-	if (param->bandwidth == RTW89_CHANNEL_WIDTH_160 &&
+	if (chan->band_width == RTW89_CHANNEL_WIDTH_160 &&
 	    rtwdev->hal.cv != CHIP_CAV) {
 		rtw89_phy_write32_idx(rtwdev, R_P80_AT_HIGH_FREQ,
 				      B_P80_AT_HIGH_FREQ, 0x0, phy_idx);
 		reg = rtw89_mac_reg_by_idx(R_P80_AT_HIGH_FREQ_BB_WRP,
 					   phy_idx);
-		if (param->primary_chan > param->center_chan) {
+		if (chan->primary_channel > chan->channel) {
 			rtw89_phy_write32_mask(rtwdev,
 					       R_P80_AT_HIGH_FREQ_RU_ALLOC,
 					       ru_alloc_msk[phy_idx], 1);
@@ -1742,8 +1774,8 @@ static void rtw8852c_set_channel_bb(struct rtw89_dev *rtwdev,
 		}
 	}
 
-	if (param->band_type == RTW89_BAND_6G &&
-	    param->bandwidth == RTW89_CHANNEL_WIDTH_160)
+	if (chan->band_type == RTW89_BAND_6G &&
+	    chan->band_width == RTW89_CHANNEL_WIDTH_160)
 		rtw89_phy_write32_idx(rtwdev, R_CDD_EVM_CHK_EN,
 				      B_CDD_EVM_CHK_EN, 0, phy_idx);
 	else
@@ -1769,15 +1801,29 @@ static void rtw8852c_set_channel_bb(struct rtw89_dev *rtwdev,
 		}
 	}
 
+	if (chan->band_type == RTW89_BAND_6G)
+		rtw89_phy_write32_set(rtwdev, R_MUIC, B_MUIC_EN);
+	else
+		rtw89_phy_write32_clr(rtwdev, R_MUIC, B_MUIC_EN);
+
+	if (hal->antenna_tx)
+		ntx_path = hal->antenna_tx;
+	else
+		ntx_path = chan->band_type == RTW89_BAND_6G ? RF_B : RF_AB;
+
+	rtw8852c_ctrl_tx_path_tmac(rtwdev, ntx_path, (enum rtw89_mac_idx)phy_idx);
+
 	rtw8852c_bb_reset_all(rtwdev, phy_idx);
 }
 
 static void rtw8852c_set_channel(struct rtw89_dev *rtwdev,
-				 struct rtw89_channel_params *params)
+				 const struct rtw89_chan *chan,
+				 enum rtw89_mac_idx mac_idx,
+				 enum rtw89_phy_idx phy_idx)
 {
-	rtw8852c_set_channel_mac(rtwdev, params, RTW89_MAC_0);
-	rtw8852c_set_channel_bb(rtwdev, params, RTW89_PHY_0);
-	rtw8852c_set_channel_rf(rtwdev, params, RTW89_PHY_0);
+	rtw8852c_set_channel_mac(rtwdev, chan, mac_idx);
+	rtw8852c_set_channel_bb(rtwdev, chan, phy_idx);
+	rtw8852c_set_channel_rf(rtwdev, chan, phy_idx);
 }
 
 static void rtw8852c_dfs_en(struct rtw89_dev *rtwdev, bool en)
@@ -1799,25 +1845,27 @@ static void rtw8852c_adc_en(struct rtw89_dev *rtwdev, bool en)
 }
 
 static void rtw8852c_set_channel_help(struct rtw89_dev *rtwdev, bool enter,
-				      struct rtw89_channel_help_params *p)
+				      struct rtw89_channel_help_params *p,
+				      const struct rtw89_chan *chan,
+				      enum rtw89_mac_idx mac_idx,
+				      enum rtw89_phy_idx phy_idx)
 {
-	u8 phy_idx = RTW89_PHY_0;
-
 	if (enter) {
-		rtw89_chip_stop_sch_tx(rtwdev, RTW89_MAC_0, &p->tx_en, RTW89_SCH_TX_SEL_ALL);
-		rtw89_mac_cfg_ppdu_status(rtwdev, RTW89_MAC_0, false);
+		rtw89_chip_stop_sch_tx(rtwdev, mac_idx, &p->tx_en,
+				       RTW89_SCH_TX_SEL_ALL);
+		rtw89_mac_cfg_ppdu_status(rtwdev, mac_idx, false);
 		rtw8852c_dfs_en(rtwdev, false);
-		rtw8852c_tssi_cont_en_phyidx(rtwdev, false, RTW89_PHY_0);
+		rtw8852c_tssi_cont_en_phyidx(rtwdev, false, phy_idx);
 		rtw8852c_adc_en(rtwdev, false);
 		fsleep(40);
-		rtw8852c_bb_reset_en(rtwdev, phy_idx, false);
+		rtw8852c_bb_reset_en(rtwdev, chan->band_type, phy_idx, false);
 	} else {
-		rtw89_mac_cfg_ppdu_status(rtwdev, RTW89_MAC_0, true);
+		rtw89_mac_cfg_ppdu_status(rtwdev, mac_idx, true);
 		rtw8852c_adc_en(rtwdev, true);
 		rtw8852c_dfs_en(rtwdev, true);
-		rtw8852c_tssi_cont_en_phyidx(rtwdev, true, RTW89_PHY_0);
-		rtw8852c_bb_reset_en(rtwdev, phy_idx, true);
-		rtw89_chip_resume_sch_tx(rtwdev, RTW89_MAC_0, p->tx_en);
+		rtw8852c_tssi_cont_en_phyidx(rtwdev, true, phy_idx);
+		rtw8852c_bb_reset_en(rtwdev, chan->band_type, phy_idx, true);
+		rtw89_chip_resume_sch_tx(rtwdev, mac_idx, p->tx_en);
 	}
 }
 
@@ -1847,9 +1895,10 @@ static void rtw8852c_rfk_channel(struct rtw89_dev *rtwdev)
 	rtw89_fw_h2c_rf_ntfy_mcc(rtwdev);
 }
 
-static void rtw8852c_rfk_band_changed(struct rtw89_dev *rtwdev)
+static void rtw8852c_rfk_band_changed(struct rtw89_dev *rtwdev,
+				      enum rtw89_phy_idx phy_idx)
 {
-	rtw8852c_tssi_scan(rtwdev, RTW89_PHY_0);
+	rtw8852c_tssi_scan(rtwdev, phy_idx);
 }
 
 static void rtw8852c_rfk_scan(struct rtw89_dev *rtwdev, bool start)
@@ -1958,9 +2007,11 @@ static void rtw8852c_set_txpwr_ref(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852c_set_txpwr_byrate(struct rtw89_dev *rtwdev,
+				      const struct rtw89_chan *chan,
 				      enum rtw89_phy_idx phy_idx)
 {
-	u8 ch = rtwdev->hal.current_channel;
+	u8 band = chan->band_type;
+	u8 ch = chan->channel;
 	static const u8 rs[] = {
 		RTW89_RS_CCK,
 		RTW89_RS_OFDM,
@@ -1986,7 +2037,8 @@ static void rtw8852c_set_txpwr_byrate(struct rtw89_dev *rtwdev,
 			for (j = 0; j < rtw89_rs_idx_max[rs[i]]; j++) {
 				cur.idx = j;
 				shf = (j % 4) * 8;
-				tmp = rtw89_phy_read_txpwr_byrate(rtwdev, &cur);
+				tmp = rtw89_phy_read_txpwr_byrate(rtwdev, band,
+								  &cur);
 				val |= (tmp << shf);
 
 				if ((j + 1) % 4)
@@ -2001,8 +2053,10 @@ static void rtw8852c_set_txpwr_byrate(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852c_set_txpwr_offset(struct rtw89_dev *rtwdev,
+				      const struct rtw89_chan *chan,
 				      enum rtw89_phy_idx phy_idx)
 {
+	u8 band = chan->band_type;
 	struct rtw89_rate_desc desc = {
 		.nss = RTW89_NSS_1,
 		.rs = RTW89_RS_OFFSET,
@@ -2013,7 +2067,7 @@ static void rtw8852c_set_txpwr_offset(struct rtw89_dev *rtwdev,
 	rtw89_debug(rtwdev, RTW89_DBG_TXPWR, "[TXPWR] set txpwr offset\n");
 
 	for (desc.idx = 0; desc.idx < RTW89_RATE_OFFSET_MAX; desc.idx++) {
-		v = rtw89_phy_read_txpwr_byrate(rtwdev, &desc);
+		v = rtw89_phy_read_txpwr_byrate(rtwdev, band, &desc);
 		val |= ((v & 0xf) << (4 * desc.idx));
 	}
 
@@ -2045,7 +2099,8 @@ static void rtw8852c_bb_set_tx_shape_dfir(struct rtw89_dev *rtwdev,
 	__DECL_DFIR_ADDR(filter,
 			 0x45BC, 0x45CC, 0x45D0, 0x45D4, 0x45D8, 0x45C0,
 			 0x45C4, 0x45C8);
-	u8 ch = rtwdev->hal.current_channel;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 ch = chan->channel;
 	const u32 *param;
 	int i;
 
@@ -2076,9 +2131,10 @@ static void rtw8852c_bb_set_tx_shape_dfir(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852c_set_tx_shape(struct rtw89_dev *rtwdev,
+				  const struct rtw89_chan *chan,
 				  enum rtw89_phy_idx phy_idx)
 {
-	u8 band = rtwdev->hal.current_band_type;
+	u8 band = chan->band_type;
 	u8 regd = rtw89_regd_get(rtwdev, band);
 	u8 tx_shape_cck = rtw89_8852c_tx_shape[band][RTW89_RS_CCK][regd];
 	u8 tx_shape_ofdm = rtw89_8852c_tx_shape[band][RTW89_RS_OFDM][regd];
@@ -2092,29 +2148,31 @@ static void rtw8852c_set_tx_shape(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852c_set_txpwr_limit(struct rtw89_dev *rtwdev,
+				     const struct rtw89_chan *chan,
 				     enum rtw89_phy_idx phy_idx)
 {
 #define __MAC_TXPWR_LMT_PAGE_SIZE 40
-	u8 ch = rtwdev->hal.current_channel;
-	u8 bw = rtwdev->hal.current_band_width;
+	u8 ch = chan->channel;
+	u8 bw = chan->band_width;
 	struct rtw89_txpwr_limit lmt[NTX_NUM_8852C];
 	u32 addr, val;
 	const s8 *ptr;
-	u8 i, j, k;
+	u8 i, j;
 
 	rtw89_debug(rtwdev, RTW89_DBG_TXPWR,
 		    "[TXPWR] set txpwr limit with ch=%d bw=%d\n", ch, bw);
 
 	for (i = 0; i < NTX_NUM_8852C; i++) {
-		rtw89_phy_fill_txpwr_limit(rtwdev, &lmt[i], i);
+		rtw89_phy_fill_txpwr_limit(rtwdev, chan, &lmt[i], i);
 
 		for (j = 0; j < __MAC_TXPWR_LMT_PAGE_SIZE; j += 4) {
 			addr = R_AX_PWR_LMT + j + __MAC_TXPWR_LMT_PAGE_SIZE * i;
 			ptr = (s8 *)&lmt[i] + j;
-			val = 0;
 
-			for (k = 0; k < 4; k++)
-				val |= (ptr[k] << (8 * k));
+			val = FIELD_PREP(GENMASK(7, 0), ptr[0]) |
+			      FIELD_PREP(GENMASK(15, 8), ptr[1]) |
+			      FIELD_PREP(GENMASK(23, 16), ptr[2]) |
+			      FIELD_PREP(GENMASK(31, 24), ptr[3]);
 
 			rtw89_mac_txpwr_write32(rtwdev, phy_idx, addr, val);
 		}
@@ -2123,30 +2181,32 @@ static void rtw8852c_set_txpwr_limit(struct rtw89_dev *rtwdev,
 }
 
 static void rtw8852c_set_txpwr_limit_ru(struct rtw89_dev *rtwdev,
+					const struct rtw89_chan *chan,
 					enum rtw89_phy_idx phy_idx)
 {
 #define __MAC_TXPWR_LMT_RU_PAGE_SIZE 24
-	u8 ch = rtwdev->hal.current_channel;
-	u8 bw = rtwdev->hal.current_band_width;
+	u8 ch = chan->channel;
+	u8 bw = chan->band_width;
 	struct rtw89_txpwr_limit_ru lmt_ru[NTX_NUM_8852C];
 	u32 addr, val;
 	const s8 *ptr;
-	u8 i, j, k;
+	u8 i, j;
 
 	rtw89_debug(rtwdev, RTW89_DBG_TXPWR,
 		    "[TXPWR] set txpwr limit ru with ch=%d bw=%d\n", ch, bw);
 
 	for (i = 0; i < NTX_NUM_8852C; i++) {
-		rtw89_phy_fill_txpwr_limit_ru(rtwdev, &lmt_ru[i], i);
+		rtw89_phy_fill_txpwr_limit_ru(rtwdev, chan, &lmt_ru[i], i);
 
 		for (j = 0; j < __MAC_TXPWR_LMT_RU_PAGE_SIZE; j += 4) {
 			addr = R_AX_PWR_RU_LMT + j +
 			       __MAC_TXPWR_LMT_RU_PAGE_SIZE * i;
 			ptr = (s8 *)&lmt_ru[i] + j;
-			val = 0;
 
-			for (k = 0; k < 4; k++)
-				val |= (ptr[k] << (8 * k));
+			val = FIELD_PREP(GENMASK(7, 0), ptr[0]) |
+			      FIELD_PREP(GENMASK(15, 8), ptr[1]) |
+			      FIELD_PREP(GENMASK(23, 16), ptr[2]) |
+			      FIELD_PREP(GENMASK(31, 24), ptr[3]);
 
 			rtw89_mac_txpwr_write32(rtwdev, phy_idx, addr, val);
 		}
@@ -2155,18 +2215,21 @@ static void rtw8852c_set_txpwr_limit_ru(struct rtw89_dev *rtwdev,
 #undef __MAC_TXPWR_LMT_RU_PAGE_SIZE
 }
 
-static void rtw8852c_set_txpwr(struct rtw89_dev *rtwdev)
+static void rtw8852c_set_txpwr(struct rtw89_dev *rtwdev,
+			       const struct rtw89_chan *chan,
+			       enum rtw89_phy_idx phy_idx)
 {
-	rtw8852c_set_txpwr_byrate(rtwdev, RTW89_PHY_0);
-	rtw8852c_set_txpwr_offset(rtwdev, RTW89_PHY_0);
-	rtw8852c_set_tx_shape(rtwdev, RTW89_PHY_0);
-	rtw8852c_set_txpwr_limit(rtwdev, RTW89_PHY_0);
-	rtw8852c_set_txpwr_limit_ru(rtwdev, RTW89_PHY_0);
+	rtw8852c_set_txpwr_byrate(rtwdev, chan, phy_idx);
+	rtw8852c_set_txpwr_offset(rtwdev, chan, phy_idx);
+	rtw8852c_set_tx_shape(rtwdev, chan, phy_idx);
+	rtw8852c_set_txpwr_limit(rtwdev, chan, phy_idx);
+	rtw8852c_set_txpwr_limit_ru(rtwdev, chan, phy_idx);
 }
 
-static void rtw8852c_set_txpwr_ctrl(struct rtw89_dev *rtwdev)
+static void rtw8852c_set_txpwr_ctrl(struct rtw89_dev *rtwdev,
+				    enum rtw89_phy_idx phy_idx)
 {
-	rtw8852c_set_txpwr_ref(rtwdev, RTW89_PHY_0);
+	rtw8852c_set_txpwr_ref(rtwdev, phy_idx);
 }
 
 static void
@@ -2222,7 +2285,8 @@ rtw8852c_init_txpwr_unit(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx)
 
 static void rtw8852c_bb_cfg_rx_path(struct rtw89_dev *rtwdev, u8 rx_path)
 {
-	struct rtw89_hal *hal = &rtwdev->hal;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 band = chan->band_type;
 	u32 rst_mask0 = B_P0_TXPW_RSTB_MANON | B_P0_TXPW_RSTB_TSSI;
 	u32 rst_mask1 = B_P1_TXPW_RSTB_MANON | B_P1_TXPW_RSTB_TSSI;
 
@@ -2316,7 +2380,7 @@ static void rtw8852c_bb_cfg_rx_path(struct rtw89_dev *rtwdev, u8 rx_path)
 					       1);
 			rtw89_phy_write32_mask(rtwdev, R_RXHE, B_RXHETB_MAX_NSS,
 					       1);
-			rtw8852c_ctrl_btg(rtwdev, hal->current_band_type == RTW89_BAND_2G);
+			rtw8852c_ctrl_btg(rtwdev, band == RTW89_BAND_2G);
 			rtw89_phy_write32_mask(rtwdev, R_P0_TXPW_RSTB,
 					       rst_mask0, 1);
 			rtw89_phy_write32_mask(rtwdev, R_P0_TXPW_RSTB,
@@ -2458,7 +2522,6 @@ static void rtw8852c_bb_ctrl_btc_preagc(struct rtw89_dev *rtwdev, bool bt_en)
 static void rtw8852c_bb_cfg_txrx_path(struct rtw89_dev *rtwdev)
 {
 	struct rtw89_hal *hal = &rtwdev->hal;
-	u8 ntx_path = hal->antenna_tx ? hal->antenna_tx : RF_AB;
 
 	rtw8852c_bb_cfg_rx_path(rtwdev, RF_PATH_AB);
 
@@ -2473,8 +2536,6 @@ static void rtw8852c_bb_cfg_txrx_path(struct rtw89_dev *rtwdev)
 		rtw89_phy_write32_mask(rtwdev, R_RXHE, B_RXHE_MAX_NSS, 1);
 		rtw89_phy_write32_mask(rtwdev, R_RXHE, B_RXHETB_MAX_NSS, 1);
 	}
-
-	rtw8852c_ctrl_tx_path_tmac(rtwdev, ntx_path, RTW89_MAC_0);
 }
 
 static u8 rtw8852c_get_thermal(struct rtw89_dev *rtwdev, enum rtw89_rf_path rf_path)
@@ -2773,23 +2834,7 @@ void rtw8852c_btc_bt_aci_imp(struct rtw89_dev *rtwdev)
 static
 void rtw8852c_btc_update_bt_cnt(struct rtw89_dev *rtwdev)
 {
-	struct rtw89_btc *btc = &rtwdev->btc;
-	struct rtw89_btc_cx *cx = &btc->cx;
-	u32 val;
-
-	val = rtw89_read32(rtwdev, R_BTC_BT_CNT_HIGH);
-	cx->cnt_bt[BTC_BCNT_HIPRI_TX] = FIELD_GET(B_AX_STATIS_BT_HI_TX_MASK, val);
-	cx->cnt_bt[BTC_BCNT_HIPRI_RX] = FIELD_GET(B_AX_STATIS_BT_HI_RX_MASK, val);
-
-	val = rtw89_read32(rtwdev, R_BTC_BT_CNT_LOW);
-	cx->cnt_bt[BTC_BCNT_LOPRI_TX] = FIELD_GET(B_AX_STATIS_BT_LO_TX_1_MASK, val);
-	cx->cnt_bt[BTC_BCNT_LOPRI_RX] = FIELD_GET(B_AX_STATIS_BT_LO_RX_1_MASK, val);
-
-	/* clock-gate off before reset counter*/
-	rtw89_write32_set(rtwdev, R_AX_BTC_CFG, B_AX_DIS_BTC_CLK_G);
-	rtw89_write32_clr(rtwdev, R_AX_BT_CNT_CFG, B_AX_BT_CNT_RST);
-	rtw89_write32_set(rtwdev, R_AX_BT_CNT_CFG, B_AX_BT_CNT_RST);
-	rtw89_write32_clr(rtwdev, R_AX_BTC_CFG, B_AX_DIS_BTC_CLK_G);
+	/* Feature move to firmware */
 }
 
 static
@@ -2810,6 +2855,59 @@ void rtw8852c_btc_wl_s1_standby(struct rtw89_dev *rtwdev, bool state)
 	rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0);
 }
 
+static void rtw8852c_set_wl_lna2(struct rtw89_dev *rtwdev, u8 level)
+{
+	/* level=0 Default:    TIA 1/0= (LNA2,TIAN6) = (7,1)/(5,1) = 21dB/12dB
+	 * level=1 Fix LNA2=5: TIA 1/0= (LNA2,TIAN6) = (5,0)/(5,1) = 18dB/12dB
+	 * To improve BT ACI in co-rx
+	 */
+
+	switch (level) {
+	case 0: /* default */
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x1000);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x0);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x1);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x17);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x2);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x3);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x17);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0);
+		break;
+	case 1: /* Fix LNA2=5  */
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x1000);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x0);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x1);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x5);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x2);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x15);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWA, RFREG_MASK, 0x3);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWD0, RFREG_MASK, 0x5);
+		rtw89_write_rf(rtwdev, RF_PATH_B, RR_LUTWE, RFREG_MASK, 0x0);
+		break;
+	}
+}
+
+static void rtw8852c_btc_set_wl_rx_gain(struct rtw89_dev *rtwdev, u32 level)
+{
+	switch (level) {
+	case 0: /* original */
+		rtw8852c_bb_ctrl_btc_preagc(rtwdev, false);
+		rtw8852c_set_wl_lna2(rtwdev, 0);
+		break;
+	case 1: /* for FDD free-run */
+		rtw8852c_bb_ctrl_btc_preagc(rtwdev, true);
+		rtw8852c_set_wl_lna2(rtwdev, 0);
+		break;
+	case 2: /* for BTG Co-Rx*/
+		rtw8852c_bb_ctrl_btc_preagc(rtwdev, false);
+		rtw8852c_set_wl_lna2(rtwdev, 1);
+		break;
+	}
+}
+
 static void rtw8852c_fill_freq_with_ppdu(struct rtw89_dev *rtwdev,
 					 struct rtw89_rx_phy_ppdu *phy_ppdu,
 					 struct ieee80211_rx_status *status)
@@ -2831,12 +2929,12 @@ static void rtw8852c_query_ppdu(struct rtw89_dev *rtwdev,
 				struct ieee80211_rx_status *status)
 {
 	u8 path;
-	s8 *rx_power = phy_ppdu->rssi;
+	u8 *rx_power = phy_ppdu->rssi;
 
-	status->signal = max_t(s8, rx_power[RF_PATH_A], rx_power[RF_PATH_B]);
+	status->signal = RTW89_RSSI_RAW_TO_DBM(max(rx_power[RF_PATH_A], rx_power[RF_PATH_B]));
 	for (path = 0; path < rtwdev->chip->rf_path_num; path++) {
 		status->chains |= BIT(path);
-		status->chain_signal[path] = rx_power[path];
+		status->chain_signal[path] = RTW89_RSSI_RAW_TO_DBM(rx_power[path]);
 	}
 	if (phy_ppdu->valid)
 		rtw8852c_fill_freq_with_ppdu(rtwdev, phy_ppdu, status);
@@ -2879,10 +2977,12 @@ static int rtw8852c_mac_enable_bb_rf(struct rtw89_dev *rtwdev)
 	return 0;
 }
 
-static void rtw8852c_mac_disable_bb_rf(struct rtw89_dev *rtwdev)
+static int rtw8852c_mac_disable_bb_rf(struct rtw89_dev *rtwdev)
 {
 	rtw89_write8_clr(rtwdev, R_AX_SYS_FUNC_EN,
 			 B_AX_FEN_BBRSTB | B_AX_FEN_BB_GLB_RSTN);
+
+	return 0;
 }
 
 static const struct rtw89_chip_ops rtw8852c_chip_ops = {
@@ -2930,6 +3030,8 @@ static const struct rtw89_chip_ops rtw8852c_chip_ops = {
 	.btc_bt_aci_imp		= rtw8852c_btc_bt_aci_imp,
 	.btc_update_bt_cnt	= rtw8852c_btc_update_bt_cnt,
 	.btc_wl_s1_standby	= rtw8852c_btc_wl_s1_standby,
+	.btc_set_wl_rx_gain	= rtw8852c_btc_set_wl_rx_gain,
+	.btc_set_policy		= rtw89_btc_set_policy_v1,
 };
 
 const struct rtw89_chip_info rtw8852c_chip_info = {
@@ -2937,6 +3039,7 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 	.ops			= &rtw8852c_chip_ops,
 	.fw_name		= "rtw89/rtw8852c_fw.bin",
 	.fifo_size		= 458752,
+	.dle_scc_rsvd_size	= 0,
 	.max_amsdu_limit	= 8000,
 	.dis_2g_40m_ul_ofdma	= false,
 	.rsvd_ple_ofst		= 0x6f800,
@@ -2960,7 +3063,9 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 	.txpwr_factor_rf	= 2,
 	.txpwr_factor_mac	= 1,
 	.dig_table		= NULL,
+	.dig_regs		= &rtw8852c_dig_regs,
 	.tssi_dbw_table		= &rtw89_8852c_tssi_dbw_table,
+	.support_chanctx_num	= 1,
 	.support_bands		= BIT(NL80211_BAND_2GHZ) |
 				  BIT(NL80211_BAND_5GHZ) |
 				  BIT(NL80211_BAND_6GHZ),
@@ -2972,6 +3077,9 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 	.acam_num		= 128,
 	.bcam_num		= 20,
 	.scam_num		= 128,
+	.bacam_num		= 8,
+	.bacam_dynamic_num	= 8,
+	.bacam_v1		= true,
 	.sec_ctrl_efuse_size	= 4,
 	.physical_efuse_size	= 1216,
 	.logical_efuse_size	= 2048,
@@ -2980,11 +3088,26 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 	.dav_log_efuse_size	= 16,
 	.phycap_addr		= 0x590,
 	.phycap_size		= 0x60,
-	.para_ver		= 0x05050764,
-	.wlcx_desired		= 0x05050000,
-	.btcx_desired		= 0x5,
+	.para_ver		= 0x1,
+	.wlcx_desired		= 0x06000000,
+	.btcx_desired		= 0x7,
 	.scbd			= 0x1,
 	.mailbox		= 0x1,
+	.btc_fwinfo_buf		= 1280,
+
+	.fcxbtcrpt_ver		= 4,
+	.fcxtdma_ver		= 3,
+	.fcxslots_ver		= 1,
+	.fcxcysta_ver		= 3,
+	.fcxstep_ver		= 3,
+	.fcxnullsta_ver		= 2,
+	.fcxmreg_ver		= 1,
+	.fcxgpiodbg_ver		= 1,
+	.fcxbtver_ver		= 1,
+	.fcxbtscan_ver		= 1,
+	.fcxbtafh_ver		= 1,
+	.fcxbtdevinfo_ver	= 1,
+
 	.afh_guard_ch		= 6,
 	.wl_rssi_thres		= rtw89_btc_8852c_wl_rssi_thres,
 	.bt_rssi_thres		= rtw89_btc_8852c_bt_rssi_thres,
@@ -2995,7 +3118,9 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 	.rf_para_ulink		= rtw89_btc_8852c_rf_ul,
 	.rf_para_dlink_num	= ARRAY_SIZE(rtw89_btc_8852c_rf_dl),
 	.rf_para_dlink		= rtw89_btc_8852c_rf_dl,
-	.ps_mode_supported	= 0,
+	.ps_mode_supported	= BIT(RTW89_PS_MODE_RFOFF) |
+				  BIT(RTW89_PS_MODE_CLK_GATED) |
+				  BIT(RTW89_PS_MODE_PWR_GATED),
 	.low_power_hci_modes	= BIT(RTW89_PS_MODE_CLK_GATED) |
 				  BIT(RTW89_PS_MODE_PWR_GATED),
 	.h2c_cctl_func_id	= H2C_FUNC_MAC_CCTLINFO_UD_V1,
@@ -3009,7 +3134,9 @@ const struct rtw89_chip_info rtw8852c_chip_info = {
 	.page_regs		= &rtw8852c_page_regs,
 	.dcfo_comp		= &rtw8852c_dcfo_comp,
 	.dcfo_comp_sft		= 5,
-	.imr_info		= &rtw8852c_imr_info
+	.imr_info		= &rtw8852c_imr_info,
+	.rrsr_cfgs		= &rtw8852c_rrsr_cfgs,
+	.dma_ch_mask		= 0,
 };
 EXPORT_SYMBOL(rtw8852c_chip_info);
 
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c b/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c
index 4186d825d19b..006c2cf93111 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.c
@@ -1294,14 +1294,14 @@ static void _iqk_by_path(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx, u
 static void _iqk_get_ch_info(struct rtw89_dev *rtwdev,
 			     enum rtw89_phy_idx phy, u8 path)
 {
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	struct rtw89_iqk_info *iqk_info = &rtwdev->iqk;
-	struct rtw89_hal *hal = &rtwdev->hal;
 
 	rtw89_debug(rtwdev, RTW89_DBG_RFK, "[IQK]===>%s\n", __func__);
 
-	iqk_info->iqk_band[path] = hal->current_band_type;
-	iqk_info->iqk_bw[path] = hal->current_band_width;
-	iqk_info->iqk_ch[path] = hal->current_channel;
+	iqk_info->iqk_band[path] = chan->band_type;
+	iqk_info->iqk_bw[path] = chan->band_width;
+	iqk_info->iqk_ch[path] = chan->channel;
 
 	rtw89_debug(rtwdev, RTW89_DBG_RFK,
 		    "[IQK]iqk_info->iqk_band[%x] = 0x%x\n", path,
@@ -1546,7 +1546,8 @@ static void _rx_dck_toggle(struct rtw89_dev *rtwdev, u8 path)
 	rtw89_write_rf(rtwdev, path, RR_DCK, RR_DCK_LV, 0x1);
 
 	ret = read_poll_timeout_atomic(rtw89_read_rf, val, val,
-				       2, 1000, false, rtwdev, path, 0x93, BIT(5));
+				       2, 2000, false, rtwdev, path,
+				       RR_DCK1, RR_DCK1_DONE);
 	if (ret)
 		rtw89_warn(rtwdev, "[RX_DCK] S%d RXDCK timeout\n", path);
 	else
@@ -1691,14 +1692,14 @@ static void _dpk_information(struct rtw89_dev *rtwdev,
 			     enum rtw89_phy_idx phy,
 			     enum rtw89_rf_path path)
 {
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	struct rtw89_dpk_info *dpk = &rtwdev->dpk;
-	struct rtw89_hal *hal = &rtwdev->hal;
 
 	u8 kidx = dpk->cur_idx[path];
 
-	dpk->bp[path][kidx].band = hal->current_band_type;
-	dpk->bp[path][kidx].ch = hal->current_channel;
-	dpk->bp[path][kidx].bw = hal->current_band_width;
+	dpk->bp[path][kidx].band = chan->band_type;
+	dpk->bp[path][kidx].ch = chan->channel;
+	dpk->bp[path][kidx].bw = chan->band_width;
 
 	rtw89_debug(rtwdev, RTW89_DBG_RFK,
 		    "[DPK] S%d[%d] (PHY%d): TSSI %s/ DBCC %s/ %s/ CH%d/ %s\n",
@@ -2272,12 +2273,13 @@ static void _dpk_idl_mpa(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 static bool _dpk_reload_check(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 			      enum rtw89_rf_path path)
 {
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	struct rtw89_dpk_info *dpk = &rtwdev->dpk;
 	bool is_reload = false;
 	u8 idx, cur_band, cur_ch;
 
-	cur_band = rtwdev->hal.current_band_type;
-	cur_ch = rtwdev->hal.current_channel;
+	cur_band = chan->band_type;
+	cur_ch = chan->channel;
 
 	for (idx = 0; idx < RTW89_DPK_BKUP_NUM; idx++) {
 		if (cur_band != dpk->bp[path][idx].band ||
@@ -2530,17 +2532,19 @@ static void _dpk_cal_select(struct rtw89_dev *rtwdev, bool force,
 static bool _dpk_bypass_check(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy)
 {
 	struct rtw89_fem_info *fem = &rtwdev->fem;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 band = chan->band_type;
 
-	if (rtwdev->hal.cv == CHIP_CAV && rtwdev->hal.current_band_type != RTW89_BAND_2G) {
+	if (rtwdev->hal.cv == CHIP_CAV && band != RTW89_BAND_2G) {
 		rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] Skip DPK due to CAV & not 2G!!\n");
 		return true;
-	} else if (fem->epa_2g && rtwdev->hal.current_band_type == RTW89_BAND_2G) {
+	} else if (fem->epa_2g && band == RTW89_BAND_2G) {
 		rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] Skip DPK due to 2G_ext_PA exist!!\n");
 		return true;
-	} else if (fem->epa_5g && rtwdev->hal.current_band_type == RTW89_BAND_5G) {
+	} else if (fem->epa_5g && band == RTW89_BAND_5G) {
 		rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] Skip DPK due to 5G_ext_PA exist!!\n");
 		return true;
-	} else if (fem->epa_6g && rtwdev->hal.current_band_type == RTW89_BAND_6G) {
+	} else if (fem->epa_6g && band == RTW89_BAND_6G) {
 		rtw89_debug(rtwdev, RTW89_DBG_RFK, "[DPK] Skip DPK due to 6G_ext_PA exist!!\n");
 		return true;
 	}
@@ -2663,7 +2667,8 @@ static void _dpk_track(struct rtw89_dev *rtwdev)
 static void _tssi_set_sys(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 			  enum rtw89_rf_path path)
 {
-	enum rtw89_band band = rtwdev->hal.current_band_type;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	enum rtw89_band band = chan->band_type;
 
 	rtw89_rfk_parser(rtwdev, &rtw8852c_tssi_sys_defs_tbl);
 
@@ -2697,7 +2702,8 @@ static void _tssi_ini_txpwr_ctrl_bb_he_tb(struct rtw89_dev *rtwdev,
 static void _tssi_set_dck(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 			  enum rtw89_rf_path path)
 {
-	enum rtw89_band band = rtwdev->hal.current_band_type;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	enum rtw89_band band = chan->band_type;
 
 	if (path == RF_PATH_A) {
 		rtw89_rfk_parser(rtwdev, &rtw8852c_tssi_dck_defs_a_tbl);
@@ -2735,8 +2741,9 @@ static void _tssi_set_tmeter_tbl(struct rtw89_dev *rtwdev, enum rtw89_phy_idx ph
 	__val;						\
 })
 	struct rtw89_tssi_info *tssi_info = &rtwdev->tssi;
-	u8 ch = rtwdev->hal.current_channel;
-	u8 subband = rtwdev->hal.current_subband;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 ch = chan->channel;
+	u8 subband = chan->subband_type;
 	const s8 *thm_up_a = NULL;
 	const s8 *thm_down_a = NULL;
 	const s8 *thm_up_b = NULL;
@@ -2908,7 +2915,8 @@ static void _tssi_set_tmeter_tbl(struct rtw89_dev *rtwdev, enum rtw89_phy_idx ph
 static void _tssi_slope_cal_org(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 				enum rtw89_rf_path path)
 {
-	enum rtw89_band band = rtwdev->hal.current_band_type;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	enum rtw89_band band = chan->band_type;
 
 	if (path == RF_PATH_A) {
 		rtw89_rfk_parser_by_cond(rtwdev, band == RTW89_BAND_2G,
@@ -2924,7 +2932,8 @@ static void _tssi_slope_cal_org(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy
 static void _tssi_set_aligk_default(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 				    enum rtw89_rf_path path)
 {
-	enum rtw89_band band = rtwdev->hal.current_band_type;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	enum rtw89_band band = chan->band_type;
 	const struct rtw89_rfk_tbl *tbl;
 
 	if (path == RF_PATH_A) {
@@ -3335,8 +3344,9 @@ static s8 _tssi_get_ofdm_de(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 			    enum rtw89_rf_path path)
 {
 	struct rtw89_tssi_info *tssi_info = &rtwdev->tssi;
-	enum rtw89_band band = rtwdev->hal.current_band_type;
-	u8 ch = rtwdev->hal.current_channel;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	enum rtw89_band band = chan->band_type;
+	u8 ch = chan->channel;
 	u32 gidx, gidx_1st, gidx_2nd;
 	s8 de_1st;
 	s8 de_2nd;
@@ -3398,8 +3408,9 @@ static s8 _tssi_get_ofdm_trim_de(struct rtw89_dev *rtwdev,
 				 enum rtw89_rf_path path)
 {
 	struct rtw89_tssi_info *tssi_info = &rtwdev->tssi;
-	enum rtw89_band band = rtwdev->hal.current_band_type;
-	u8 ch = rtwdev->hal.current_channel;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	enum rtw89_band band = chan->band_type;
+	u8 ch = chan->channel;
 	u32 tgidx, tgidx_1st, tgidx_2nd;
 	s8 tde_1st = 0;
 	s8 tde_2nd = 0;
@@ -3462,7 +3473,8 @@ static void _tssi_set_efuse_to_de(struct rtw89_dev *rtwdev,
 				  enum rtw89_phy_idx phy)
 {
 	struct rtw89_tssi_info *tssi_info = &rtwdev->tssi;
-	u8 ch = rtwdev->hal.current_channel;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	u8 ch = chan->channel;
 	u8 gidx;
 	s8 ofdm_de;
 	s8 trim_de;
@@ -3802,15 +3814,17 @@ void rtw8852c_ctrl_bw_ch(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy,
 }
 
 void rtw8852c_set_channel_rf(struct rtw89_dev *rtwdev,
-			     struct rtw89_channel_params *param,
+			     const struct rtw89_chan *chan,
 			     enum rtw89_phy_idx phy_idx)
 {
-	rtw8852c_ctrl_bw_ch(rtwdev, phy_idx, param->center_chan, param->band_type,
-			    param->bandwidth);
+	rtw8852c_ctrl_bw_ch(rtwdev, phy_idx, chan->channel,
+			    chan->band_type,
+			    chan->band_width);
 }
 
 void rtw8852c_mcc_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_idx)
 {
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
 	struct rtw89_mcc_info *mcc_info = &rtwdev->mcc;
 	u8 idx = mcc_info->table_idx;
 	int i;
@@ -3823,8 +3837,8 @@ void rtw8852c_mcc_get_ch_info(struct rtw89_dev *rtwdev, enum rtw89_phy_idx phy_i
 	}
 
 	mcc_info->table_idx = idx;
-	mcc_info->ch[idx] = rtwdev->hal.current_channel;
-	mcc_info->band[idx] = rtwdev->hal.current_band_type;
+	mcc_info->ch[idx] = chan->channel;
+	mcc_info->band[idx] = chan->band_type;
 }
 
 void rtw8852c_rck(struct rtw89_dev *rtwdev)
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.h b/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.h
index 5118a49da8d3..928a587cdd05 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.h
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852c_rfk.h
@@ -21,7 +21,7 @@ void rtw8852c_tssi_cont_en_phyidx(struct rtw89_dev *rtwdev, bool en, u8 phy_idx)
 void rtw8852c_wifi_scan_notify(struct rtw89_dev *rtwdev, bool scan_start,
 			       enum rtw89_phy_idx phy_idx);
 void rtw8852c_set_channel_rf(struct rtw89_dev *rtwdev,
-			     struct rtw89_channel_params *param,
+			     const struct rtw89_chan *chan,
 			     enum rtw89_phy_idx phy_idx);
 void rtw8852c_lck_init(struct rtw89_dev *rtwdev);
 void rtw8852c_lck_track(struct rtw89_dev *rtwdev);
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c b/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c
index feaa83b16171..11f35e7a7f0e 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852c_table.c
@@ -1767,7 +1767,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_bb_reg_gain[] = {
 	{0x3070103, 0x34343C3C},
 };
 
-static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
+static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] = {
 	{0xF0010000, 0x00000000},
 	{0xF0020000, 0x00000001},
 	{0xF0320000, 0x00000002},
@@ -1777,13 +1777,17 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0xF0360000, 0x00000006},
 	{0xF0010001, 0x00000007},
 	{0xF0020001, 0x00000008},
-	{0xF0320001, 0x00000009},
-	{0xF0330001, 0x0000000A},
-	{0xF0340001, 0x0000000B},
-	{0xF0350001, 0x0000000C},
-	{0xF0360001, 0x0000000D},
-	{0xF03F0001, 0x0000000E},
-	{0xF0400001, 0x0000000F},
+	{0xF0030001, 0x00000009},
+	{0xF0040001, 0x0000000A},
+	{0xF0050001, 0x0000000B},
+	{0xF0070001, 0x0000000C},
+	{0xF0320001, 0x0000000D},
+	{0xF0330001, 0x0000000E},
+	{0xF0340001, 0x0000000F},
+	{0xF0350001, 0x00000010},
+	{0xF0360001, 0x00000011},
+	{0xF03F0001, 0x00000012},
+	{0xF0400001, 0x00000013},
 	{0x005, 0x00000000},
 	{0x10005, 0x00000000},
 	{0x000, 0x00030001},
@@ -1795,7 +1799,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03E, 0x00000620},
 	{0x03F, 0x0000020C},
 	{0x0EF, 0x00000000},
-	{0x05F, 0x00000032},
+	{0x05F, 0x00000038},
 	{0x097, 0x00043200},
 	{0x0A6, 0x00066DB7},
 	{0x0EF, 0x00004000},
@@ -1821,8 +1825,8 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x000, 0x00033C01},
 	{0x10000, 0x00033C00},
 	{0x01A, 0x00040004},
-	{0x0FE, 0x00000000},
 	{0x096, 0x00015200},
+	{0x10055, 0x00080080},
 	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x067, 0x0004D000},
 	{0x0DA, 0x000D4009},
@@ -1850,6 +1854,18 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x067, 0x0000D300},
 	{0x0DA, 0x000D4000},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x067, 0x0000D300},
+	{0x0DA, 0x000D4000},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x067, 0x0000D300},
+	{0x0DA, 0x000D4000},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x067, 0x0000D300},
+	{0x0DA, 0x000D4000},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x067, 0x0000D300},
+	{0x0DA, 0x000D4000},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x067, 0x0000D300},
 	{0x0DA, 0x000D4000},
@@ -1922,6 +1938,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000000CC},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000CC},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000CC},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000CC},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000CC},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000CC},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000CC},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -1958,6 +1982,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000000C4},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000C4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000C4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000C4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000C4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000C4},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000C4},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -1994,6 +2026,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000000BC},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000BC},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000BC},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000BC},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000BC},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000BC},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000BC},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2030,6 +2070,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000000B4},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000B4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000B4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000B4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000B4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000B4},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000B4},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2066,6 +2114,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000000AC},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000AC},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000AC},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000AC},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000AC},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000AC},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000AC},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2102,6 +2158,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000000A4},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000A4},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000A4},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2138,6 +2202,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000009C},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2174,6 +2246,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000094},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000094},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000094},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000094},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000094},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000094},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000094},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2210,6 +2290,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000008C},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000008C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000008C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000008C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000008C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000008C},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000008C},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2246,6 +2334,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000084},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000084},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000084},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000084},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000084},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000084},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000084},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2282,6 +2378,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000000BC},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000BC},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000BC},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000BC},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000BC},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000BC},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000BC},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2318,6 +2422,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000000B4},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000B4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000B4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000B4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000B4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000B4},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000B4},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2354,6 +2466,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000000AC},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000AC},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000AC},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000AC},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000AC},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000AC},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000AC},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2390,6 +2510,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000000A4},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000000A4},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000000A4},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2426,6 +2554,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000009C},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2462,6 +2598,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000094},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000094},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000094},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000094},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000094},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000094},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000094},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2498,6 +2642,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000008C},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000008C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000008C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000008C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000008C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000008C},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000008C},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2534,6 +2686,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000084},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000084},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000084},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000084},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000084},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000084},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000084},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2570,6 +2730,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000003C},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000003C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000003C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000003C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000003C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000003C},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000003C},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2606,6 +2774,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000034},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000034},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000034},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000034},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000034},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000034},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000034},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2642,6 +2818,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000002C},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000002C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000002C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000002C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000002C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000002C},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000002C},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2678,6 +2862,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000024},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000024},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000024},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000024},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000024},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000024},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000024},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2714,6 +2906,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000001C},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000001C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000001C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000001C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000001C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000001C},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000001C},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2750,6 +2950,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000014},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000014},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000014},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000014},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000014},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000014},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000014},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2786,6 +2994,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000000C},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000000C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000000C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000000C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000000C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000000C},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000000C},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2822,6 +3038,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000004},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000004},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000004},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000004},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000004},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000004},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000004},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2871,6 +3095,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x08F, 0x000D1352},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x08F, 0x000D1352},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x08F, 0x000D1352},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x08F, 0x000D1352},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x08F, 0x000D1352},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x08F, 0x000D1352},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x08F, 0x000D1352},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -2905,6 +3137,52 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000015},
 	{0x033, 0x00000001},
 	{0x03F, 0x00000017},
+	{0x033, 0x00000002},
+	{0x03F, 0x00000017},
+	{0x033, 0x00000003},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0xB0000000, 0x00000000},
 	{0x0EF, 0x00000000},
 	{0x0EF, 0x00008000},
 	{0x033, 0x00000020},
@@ -3416,6 +3694,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000EFFF},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000EFFF},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000EFFF},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000EFFF},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000EFFF},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000EFFF},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000EFFF},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -3522,7 +3808,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x00000005},
 	{0x03F, 0x00004344},
 	{0x033, 0x00000006},
-	{0x03F, 0x00004324},
+	{0x03F, 0x00004344},
 	{0x033, 0x00000007},
 	{0x03F, 0x00004344},
 	{0x033, 0x00000008},
@@ -3585,6 +3871,33 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000200},
 	{0x0EF, 0x00000000},
 	{0x0EF, 0x00000010},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x030, 0x000084DC},
 	{0x030, 0x000103C9},
 	{0x030, 0x00018399},
@@ -3597,6 +3910,241 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x030, 0x00050011},
 	{0x030, 0x00058000},
 	{0x030, 0x00060000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0xA0000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0xB0000000, 0x00000000},
 	{0x030, 0x00068000},
 	{0x030, 0x00070000},
 	{0x0EF, 0x00000000},
@@ -3831,6 +4379,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x030, 0x000300FF},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x030, 0x000300FF},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000300FF},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000300FF},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000300FF},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000300FF},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x030, 0x000300FF},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -3901,6 +4457,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x095, 0x00000008},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x095, 0x00000008},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x095, 0x00000008},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x095, 0x00000008},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x095, 0x00000008},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x095, 0x00000008},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x095, 0x00000008},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -3920,101 +4484,2033 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0xB0000000, 0x00000000},
 	{0x0EE, 0x00001000},
 	{0x033, 0x00000020},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000024},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000028},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000002C},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000030},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000034},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E7},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000038},
 	{0x03F, 0x000002E7},
 	{0x033, 0x0000003C},
 	{0x03F, 0x000003E7},
 	{0x033, 0x00000021},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000025},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000029},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000002D},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000031},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000035},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000039},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000003D},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000022},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000026},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000002A},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000002E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000032},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000036},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000003A},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000003E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000060},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000064},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000068},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000006C},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000070},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000074},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000078},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000007C},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000061},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000065},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000069},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000006D},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000071},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000075},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000079},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000007D},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000062},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000066},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000006A},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000006E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000072},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000076},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000007A},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000007E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000063},
 	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
@@ -4034,20 +6530,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x00000152},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000152},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000052},
 	{0xB0000000, 0x00000000},
@@ -4070,20 +6574,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000015A},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000015A},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000005A},
 	{0xB0000000, 0x00000000},
@@ -4106,20 +6618,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x0000019C},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000009C},
 	{0xB0000000, 0x00000000},
@@ -4142,20 +6662,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000001A4},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000019C},
 	{0xB0000000, 0x00000000},
@@ -4178,20 +6706,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000001E6},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x000001A4},
 	{0xB0000000, 0x00000000},
@@ -4214,20 +6750,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x03F, 0x000002E6},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x000001E6},
 	{0xB0000000, 0x00000000},
@@ -5271,131 +7815,131 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x10030, 0x00003C5F},
 	{0x10030, 0x00004059},
 	{0x10030, 0x00004453},
-	{0x10030, 0x000201ED},
-	{0x10030, 0x000205AD},
-	{0x10030, 0x000209A7},
-	{0x10030, 0x00020DA1},
-	{0x10030, 0x0002119B},
-	{0x10030, 0x00021561},
-	{0x10030, 0x0002195B},
-	{0x10030, 0x00021D27},
-	{0x10030, 0x00022121},
-	{0x10030, 0x000224E9},
-	{0x10030, 0x000228E3},
-	{0x10030, 0x00022CA9},
-	{0x10030, 0x000230A3},
-	{0x10030, 0x00023469},
-	{0x10030, 0x00023863},
-	{0x10030, 0x00023C29},
-	{0x10030, 0x00024023},
-	{0x10030, 0x0002441D},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
 	{0x10030, 0x000281EF},
-	{0x10030, 0x000285AF},
-	{0x10030, 0x000289A9},
-	{0x10030, 0x00028DA3},
-	{0x10030, 0x0002919D},
-	{0x10030, 0x00029563},
-	{0x10030, 0x0002995D},
-	{0x10030, 0x00029D25},
-	{0x10030, 0x0002A11F},
-	{0x10030, 0x0002A4E7},
-	{0x10030, 0x0002A8E1},
-	{0x10030, 0x0002ACA7},
-	{0x10030, 0x0002B0A1},
-	{0x10030, 0x0002B467},
-	{0x10030, 0x0002B861},
-	{0x10030, 0x0002BC27},
-	{0x10030, 0x0002C021},
-	{0x10030, 0x0002C41B},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
 	{0x10030, 0x000301EF},
-	{0x10030, 0x000305AF},
-	{0x10030, 0x000309A9},
-	{0x10030, 0x00030DA3},
-	{0x10030, 0x0003119D},
-	{0x10030, 0x00031563},
-	{0x10030, 0x0003195D},
-	{0x10030, 0x00031D25},
-	{0x10030, 0x0003211F},
-	{0x10030, 0x000324E7},
-	{0x10030, 0x000328E1},
-	{0x10030, 0x00032CA7},
-	{0x10030, 0x000330A1},
-	{0x10030, 0x00033467},
-	{0x10030, 0x00033861},
-	{0x10030, 0x00033C27},
-	{0x10030, 0x00034021},
-	{0x10030, 0x0003441B},
-	{0x10030, 0x000601EB},
-	{0x10030, 0x000605AB},
-	{0x10030, 0x000609A5},
-	{0x10030, 0x00060D9F},
-	{0x10030, 0x00061199},
-	{0x10030, 0x00061593},
-	{0x10030, 0x00061959},
-	{0x10030, 0x00061D53},
-	{0x10030, 0x0006211B},
-	{0x10030, 0x00062515},
-	{0x10030, 0x000628DD},
-	{0x10030, 0x00062CD7},
-	{0x10030, 0x0006309D},
-	{0x10030, 0x00063497},
-	{0x10030, 0x0006385D},
-	{0x10030, 0x00063C57},
-	{0x10030, 0x0006401D},
-	{0x10030, 0x00064417},
-	{0x10030, 0x000681E7},
-	{0x10030, 0x000685A7},
-	{0x10030, 0x000689A1},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955F},
-	{0x10030, 0x00069959},
-	{0x10030, 0x00069D21},
-	{0x10030, 0x0006A11B},
-	{0x10030, 0x0006A4E3},
-	{0x10030, 0x0006A8DD},
-	{0x10030, 0x0006ACA5},
-	{0x10030, 0x0006B09F},
-	{0x10030, 0x0006B465},
-	{0x10030, 0x0006B85F},
-	{0x10030, 0x0006BC25},
-	{0x10030, 0x0006C01F},
-	{0x10030, 0x0006C419},
-	{0x10030, 0x000701E7},
-	{0x10030, 0x000705A7},
-	{0x10030, 0x000709A1},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071955},
-	{0x10030, 0x00071D1D},
-	{0x10030, 0x00072117},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072CA1},
-	{0x10030, 0x0007309B},
-	{0x10030, 0x00073461},
-	{0x10030, 0x0007385B},
-	{0x10030, 0x00073C21},
-	{0x10030, 0x0007401B},
-	{0x10030, 0x0007441B},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
 	{0x10030, 0x000781EF},
 	{0x10030, 0x000785E9},
 	{0x10030, 0x000789E3},
-	{0x10030, 0x00078DA3},
-	{0x10030, 0x00079161},
-	{0x10030, 0x0007955B},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
 	{0x10030, 0x00079921},
 	{0x10030, 0x00079D1B},
-	{0x10030, 0x0007A0E1},
-	{0x10030, 0x0007A4DB},
-	{0x10030, 0x0007A8A1},
-	{0x10030, 0x0007AC9B},
-	{0x10030, 0x0007B061},
-	{0x10030, 0x0007B45B},
-	{0x10030, 0x0007B821},
-	{0x10030, 0x0007BC1B},
-	{0x10030, 0x0007C015},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
 	{0x10030, 0x0007C40F},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x000001EF},
@@ -5416,131 +7960,711 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x10030, 0x00003C5F},
 	{0x10030, 0x00004059},
 	{0x10030, 0x00004453},
-	{0x10030, 0x000201ED},
-	{0x10030, 0x000205AD},
-	{0x10030, 0x000209A7},
-	{0x10030, 0x00020DA1},
-	{0x10030, 0x0002119B},
-	{0x10030, 0x00021561},
-	{0x10030, 0x0002195B},
-	{0x10030, 0x00021D27},
-	{0x10030, 0x00022121},
-	{0x10030, 0x000224E9},
-	{0x10030, 0x000228E3},
-	{0x10030, 0x00022CA9},
-	{0x10030, 0x000230A3},
-	{0x10030, 0x00023469},
-	{0x10030, 0x00023863},
-	{0x10030, 0x00023C29},
-	{0x10030, 0x00024023},
-	{0x10030, 0x0002441D},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
 	{0x10030, 0x000281EF},
-	{0x10030, 0x000285AF},
-	{0x10030, 0x000289A9},
-	{0x10030, 0x00028DA3},
-	{0x10030, 0x0002919D},
-	{0x10030, 0x00029563},
-	{0x10030, 0x0002995D},
-	{0x10030, 0x00029D25},
-	{0x10030, 0x0002A11F},
-	{0x10030, 0x0002A4E7},
-	{0x10030, 0x0002A8E1},
-	{0x10030, 0x0002ACA7},
-	{0x10030, 0x0002B0A1},
-	{0x10030, 0x0002B467},
-	{0x10030, 0x0002B861},
-	{0x10030, 0x0002BC27},
-	{0x10030, 0x0002C021},
-	{0x10030, 0x0002C41B},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
 	{0x10030, 0x000301EF},
-	{0x10030, 0x000305AF},
-	{0x10030, 0x000309A9},
-	{0x10030, 0x00030DA3},
-	{0x10030, 0x0003119D},
-	{0x10030, 0x00031563},
-	{0x10030, 0x0003195D},
-	{0x10030, 0x00031D25},
-	{0x10030, 0x0003211F},
-	{0x10030, 0x000324E7},
-	{0x10030, 0x000328E1},
-	{0x10030, 0x00032CA7},
-	{0x10030, 0x000330A1},
-	{0x10030, 0x00033467},
-	{0x10030, 0x00033861},
-	{0x10030, 0x00033C27},
-	{0x10030, 0x00034021},
-	{0x10030, 0x0003441B},
-	{0x10030, 0x000601EB},
-	{0x10030, 0x000605AB},
-	{0x10030, 0x000609A5},
-	{0x10030, 0x00060D9F},
-	{0x10030, 0x00061199},
-	{0x10030, 0x00061593},
-	{0x10030, 0x00061959},
-	{0x10030, 0x00061D53},
-	{0x10030, 0x0006211B},
-	{0x10030, 0x00062515},
-	{0x10030, 0x000628DD},
-	{0x10030, 0x00062CD7},
-	{0x10030, 0x0006309D},
-	{0x10030, 0x00063497},
-	{0x10030, 0x0006385D},
-	{0x10030, 0x00063C57},
-	{0x10030, 0x0006401D},
-	{0x10030, 0x00064417},
-	{0x10030, 0x000681E7},
-	{0x10030, 0x000685A7},
-	{0x10030, 0x000689A1},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955F},
-	{0x10030, 0x00069959},
-	{0x10030, 0x00069D21},
-	{0x10030, 0x0006A11B},
-	{0x10030, 0x0006A4E3},
-	{0x10030, 0x0006A8DD},
-	{0x10030, 0x0006ACA5},
-	{0x10030, 0x0006B09F},
-	{0x10030, 0x0006B465},
-	{0x10030, 0x0006B85F},
-	{0x10030, 0x0006BC25},
-	{0x10030, 0x0006C01F},
-	{0x10030, 0x0006C419},
-	{0x10030, 0x000701E7},
-	{0x10030, 0x000705A7},
-	{0x10030, 0x000709A1},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071955},
-	{0x10030, 0x00071D1D},
-	{0x10030, 0x00072117},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072CA1},
-	{0x10030, 0x0007309B},
-	{0x10030, 0x00073461},
-	{0x10030, 0x0007385B},
-	{0x10030, 0x00073C21},
-	{0x10030, 0x0007401B},
-	{0x10030, 0x0007441B},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
 	{0x10030, 0x000781EF},
 	{0x10030, 0x000785E9},
 	{0x10030, 0x000789E3},
-	{0x10030, 0x00078DA3},
-	{0x10030, 0x00079161},
-	{0x10030, 0x0007955B},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
+	{0x10030, 0x00079921},
+	{0x10030, 0x00079D1B},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
+	{0x10030, 0x000281EF},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
+	{0x10030, 0x000301EF},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781EF},
+	{0x10030, 0x000785E9},
+	{0x10030, 0x000789E3},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
+	{0x10030, 0x00079921},
+	{0x10030, 0x00079D1B},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
+	{0x10030, 0x000281EF},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
+	{0x10030, 0x000301EF},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781EF},
+	{0x10030, 0x000785E9},
+	{0x10030, 0x000789E3},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
+	{0x10030, 0x00079921},
+	{0x10030, 0x00079D1B},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
+	{0x10030, 0x000281EF},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
+	{0x10030, 0x000301EF},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781EF},
+	{0x10030, 0x000785E9},
+	{0x10030, 0x000789E3},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
 	{0x10030, 0x00079921},
 	{0x10030, 0x00079D1B},
-	{0x10030, 0x0007A0E1},
-	{0x10030, 0x0007A4DB},
-	{0x10030, 0x0007A8A1},
-	{0x10030, 0x0007AC9B},
-	{0x10030, 0x0007B061},
-	{0x10030, 0x0007B45B},
-	{0x10030, 0x0007B821},
-	{0x10030, 0x0007BC1B},
-	{0x10030, 0x0007C015},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
+	{0x10030, 0x000281EF},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
+	{0x10030, 0x000301EF},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781EF},
+	{0x10030, 0x000785E9},
+	{0x10030, 0x000789E3},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
+	{0x10030, 0x00079921},
+	{0x10030, 0x00079D1B},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
 	{0x10030, 0x0007C40F},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x000001EF},
@@ -5561,1002 +8685,1002 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x10030, 0x00003C5F},
 	{0x10030, 0x00004059},
 	{0x10030, 0x00004453},
-	{0x10030, 0x000201ED},
-	{0x10030, 0x000205AD},
-	{0x10030, 0x000209A7},
-	{0x10030, 0x00020DA1},
-	{0x10030, 0x0002119B},
-	{0x10030, 0x00021561},
-	{0x10030, 0x0002195B},
-	{0x10030, 0x00021D27},
-	{0x10030, 0x00022121},
-	{0x10030, 0x000224E9},
-	{0x10030, 0x000228E3},
-	{0x10030, 0x00022CA9},
-	{0x10030, 0x000230A3},
-	{0x10030, 0x00023469},
-	{0x10030, 0x00023863},
-	{0x10030, 0x00023C29},
-	{0x10030, 0x00024023},
-	{0x10030, 0x0002441D},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
 	{0x10030, 0x000281EF},
-	{0x10030, 0x000285AF},
-	{0x10030, 0x000289A9},
-	{0x10030, 0x00028DA3},
-	{0x10030, 0x0002919D},
-	{0x10030, 0x00029563},
-	{0x10030, 0x0002995D},
-	{0x10030, 0x00029D25},
-	{0x10030, 0x0002A11F},
-	{0x10030, 0x0002A4E7},
-	{0x10030, 0x0002A8E1},
-	{0x10030, 0x0002ACA7},
-	{0x10030, 0x0002B0A1},
-	{0x10030, 0x0002B467},
-	{0x10030, 0x0002B861},
-	{0x10030, 0x0002BC27},
-	{0x10030, 0x0002C021},
-	{0x10030, 0x0002C41B},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
 	{0x10030, 0x000301EF},
-	{0x10030, 0x000305AF},
-	{0x10030, 0x000309A9},
-	{0x10030, 0x00030DA3},
-	{0x10030, 0x0003119D},
-	{0x10030, 0x00031563},
-	{0x10030, 0x0003195D},
-	{0x10030, 0x00031D25},
-	{0x10030, 0x0003211F},
-	{0x10030, 0x000324E7},
-	{0x10030, 0x000328E1},
-	{0x10030, 0x00032CA7},
-	{0x10030, 0x000330A1},
-	{0x10030, 0x00033467},
-	{0x10030, 0x00033861},
-	{0x10030, 0x00033C27},
-	{0x10030, 0x00034021},
-	{0x10030, 0x0003441B},
-	{0x10030, 0x000601EB},
-	{0x10030, 0x000605AB},
-	{0x10030, 0x000609A5},
-	{0x10030, 0x00060D9F},
-	{0x10030, 0x00061199},
-	{0x10030, 0x00061593},
-	{0x10030, 0x00061959},
-	{0x10030, 0x00061D53},
-	{0x10030, 0x0006211B},
-	{0x10030, 0x00062515},
-	{0x10030, 0x000628DD},
-	{0x10030, 0x00062CD7},
-	{0x10030, 0x0006309D},
-	{0x10030, 0x00063497},
-	{0x10030, 0x0006385D},
-	{0x10030, 0x00063C57},
-	{0x10030, 0x0006401D},
-	{0x10030, 0x00064417},
-	{0x10030, 0x000681E7},
-	{0x10030, 0x000685A7},
-	{0x10030, 0x000689A1},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955F},
-	{0x10030, 0x00069959},
-	{0x10030, 0x00069D21},
-	{0x10030, 0x0006A11B},
-	{0x10030, 0x0006A4E3},
-	{0x10030, 0x0006A8DD},
-	{0x10030, 0x0006ACA5},
-	{0x10030, 0x0006B09F},
-	{0x10030, 0x0006B465},
-	{0x10030, 0x0006B85F},
-	{0x10030, 0x0006BC25},
-	{0x10030, 0x0006C01F},
-	{0x10030, 0x0006C419},
-	{0x10030, 0x000701E7},
-	{0x10030, 0x000705A7},
-	{0x10030, 0x000709A1},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071955},
-	{0x10030, 0x00071D1D},
-	{0x10030, 0x00072117},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072CA1},
-	{0x10030, 0x0007309B},
-	{0x10030, 0x00073461},
-	{0x10030, 0x0007385B},
-	{0x10030, 0x00073C21},
-	{0x10030, 0x0007401B},
-	{0x10030, 0x0007441B},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
 	{0x10030, 0x000781EF},
 	{0x10030, 0x000785E9},
 	{0x10030, 0x000789E3},
-	{0x10030, 0x00078DA3},
-	{0x10030, 0x00079161},
-	{0x10030, 0x0007955B},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
 	{0x10030, 0x00079921},
 	{0x10030, 0x00079D1B},
-	{0x10030, 0x0007A0E1},
-	{0x10030, 0x0007A4DB},
-	{0x10030, 0x0007A8A1},
-	{0x10030, 0x0007AC9B},
-	{0x10030, 0x0007B061},
-	{0x10030, 0x0007B45B},
-	{0x10030, 0x0007B821},
-	{0x10030, 0x0007BC1B},
-	{0x10030, 0x0007C015},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
 	{0x10030, 0x0007C40F},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071CE5},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781EF},
+	{0x10030, 0x000785E9},
+	{0x10030, 0x000789E3},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
+	{0x10030, 0x00079921},
+	{0x10030, 0x00079D1B},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071CE5},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781EF},
+	{0x10030, 0x000785E9},
+	{0x10030, 0x000789E3},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
+	{0x10030, 0x00079921},
+	{0x10030, 0x00079D1B},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071CE5},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781EF},
+	{0x10030, 0x000785E9},
+	{0x10030, 0x000789E3},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
+	{0x10030, 0x00079921},
+	{0x10030, 0x00079D1B},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071CE5},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781EF},
+	{0x10030, 0x000785E9},
+	{0x10030, 0x000789E3},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
+	{0x10030, 0x00079921},
+	{0x10030, 0x00079D1B},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071CE5},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781EF},
+	{0x10030, 0x000785E9},
+	{0x10030, 0x000789E3},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
+	{0x10030, 0x00079921},
+	{0x10030, 0x00079D1B},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701EF},
+	{0x10030, 0x000705E7},
+	{0x10030, 0x000709A7},
+	{0x10030, 0x00070D61},
+	{0x10030, 0x0007115B},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071CE5},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728A1},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781EF},
+	{0x10030, 0x000785E9},
+	{0x10030, 0x000789E3},
+	{0x10030, 0x00078DA1},
+	{0x10030, 0x0007915F},
+	{0x10030, 0x00079559},
+	{0x10030, 0x00079921},
+	{0x10030, 0x00079D1B},
+	{0x10030, 0x0007A0E3},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B823},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0xA0000000, 0x00000000},
 	{0x10030, 0x000001EF},
 	{0x10030, 0x000005E9},
@@ -6724,6 +9848,1150 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x10030, 0x00004017},
 	{0x100EE, 0x00000000},
 	{0x100EE, 0x00002000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0xB0000000, 0x00000000},
+	{0x10030, 0x000600F6},
+	{0x10030, 0x000604F3},
+	{0x10030, 0x000608F0},
+	{0x10030, 0x00060CED},
+	{0x10030, 0x000610EA},
+	{0x10030, 0x000614E7},
+	{0x10030, 0x000618E4},
+	{0x10030, 0x00061CE1},
+	{0x10030, 0x000620DE},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628D8},
+	{0x10030, 0x00062CD5},
+	{0x10030, 0x000630D2},
+	{0x10030, 0x000634CF},
+	{0x10030, 0x000638CC},
+	{0x10030, 0x00063C09},
+	{0x10030, 0x00064006},
+	{0x10030, 0x000680F5},
+	{0x10030, 0x000684F2},
+	{0x10030, 0x000688EF},
+	{0x10030, 0x00068CEC},
+	{0x10030, 0x000690E9},
+	{0x10030, 0x000694E6},
+	{0x10030, 0x000698E3},
+	{0x10030, 0x00069CE0},
+	{0x10030, 0x0006A0DD},
+	{0x10030, 0x0006A4DA},
+	{0x10030, 0x0006A8D7},
+	{0x10030, 0x0006ACD4},
+	{0x10030, 0x0006B0D1},
+	{0x10030, 0x0006B4CE},
+	{0x10030, 0x0006B8CB},
+	{0x10030, 0x0006BC08},
+	{0x10030, 0x0006C005},
+	{0x10030, 0x000700F5},
+	{0x10030, 0x000704F2},
+	{0x10030, 0x000708EF},
+	{0x10030, 0x00070CEC},
+	{0x10030, 0x000710E9},
+	{0x10030, 0x000714E6},
+	{0x10030, 0x000718E3},
+	{0x10030, 0x00071CE0},
+	{0x10030, 0x000720DD},
+	{0x10030, 0x000724DA},
+	{0x10030, 0x000728D7},
+	{0x10030, 0x00072CD4},
+	{0x10030, 0x000730D1},
+	{0x10030, 0x000734CE},
+	{0x10030, 0x000738CB},
+	{0x10030, 0x00073C08},
+	{0x10030, 0x00074005},
 	{0x10030, 0x000780F4},
 	{0x10030, 0x000784F1},
 	{0x10030, 0x000788EE},
@@ -6777,9 +11045,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x00000025},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000026},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000027},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000028},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000029},
@@ -6793,9 +11145,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x0000002D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000002E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000002F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000030},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000031},
@@ -6809,9 +11245,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x00000035},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000036},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000037},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000060},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000061},
@@ -6825,9 +11345,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x00000065},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000066},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000067},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000068},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000069},
@@ -6841,9 +11445,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x0000006D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000006E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000006F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000070},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000071},
@@ -6857,9 +11545,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x00000075},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000076},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000077},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000078},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000079},
@@ -6873,9 +11645,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x0000007D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000007E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000007F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000A0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000A1},
@@ -6889,9 +11745,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000000A5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000A6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000A7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000A8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000A9},
@@ -6905,9 +11845,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000000AD},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000AE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000AF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000B0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000B1},
@@ -6921,9 +11945,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000000B5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000B6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000B7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000E0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000E1},
@@ -6937,9 +12045,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000000E5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000E6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000E7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000E8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000E9},
@@ -6953,9 +12145,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000000ED},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000EE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000EF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000F0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000F1},
@@ -6969,9 +12245,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000000F5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000F6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000F7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000F8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000F9},
@@ -6985,9 +12345,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000000FD},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000FE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000FF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000120},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000121},
@@ -7001,9 +12445,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x00000125},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000126},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000127},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000128},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000129},
@@ -7017,9 +12545,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x0000012D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000012E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000012F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000130},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000131},
@@ -7033,9 +12645,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x00000135},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000136},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000137},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000160},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000161},
@@ -7049,9 +12745,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x00000165},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000166},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000167},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000168},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000169},
@@ -7065,9 +12845,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x0000016D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000016E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000016F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000170},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000171},
@@ -7081,9 +12945,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x00000175},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000176},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000177},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000178},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000179},
@@ -7097,9 +13045,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x0000017D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000017E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000017F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001A0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001A1},
@@ -7113,9 +13145,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000001A5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001A6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001A7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001A8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001A9},
@@ -7129,9 +13245,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000001AD},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001AE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001AF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001B0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001B1},
@@ -7145,9 +13345,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000001B5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001B6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001B7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001E0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001E1},
@@ -7161,9 +13445,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000001E5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001E6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001E7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001E8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001E9},
@@ -7177,9 +13545,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000001ED},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001EE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001EF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001F0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001F1},
@@ -7193,9 +13645,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000001F5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001F6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001F7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001F8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001F9},
@@ -7209,9 +13745,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x033, 0x000001FD},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001FE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001FF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x0EF, 0x00000000},
 	{0x005, 0x00000001},
 	{0x10005, 0x00000001},
@@ -7253,7 +13873,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x10030, 0x00022000},
 	{0x10030, 0x00023000},
 	{0x10030, 0x00024000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x00025000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x00026003},
 	{0x10030, 0x00027003},
 	{0x10030, 0x00028000},
@@ -7261,7 +13923,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x10030, 0x0002A000},
 	{0x10030, 0x0002B000},
 	{0x10030, 0x0002C000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x0002D000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x0002E003},
 	{0x10030, 0x0002F003},
 	{0x10030, 0x00030000},
@@ -7269,7 +13973,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x10030, 0x00032000},
 	{0x10030, 0x00033000},
 	{0x10030, 0x00034000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x00035000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x00036003},
 	{0x10030, 0x00037003},
 	{0x10030, 0x00038000},
@@ -7277,7 +14023,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x10030, 0x0003A000},
 	{0x10030, 0x0003B000},
 	{0x10030, 0x0003C000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x0003D000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x0003E003},
 	{0x10030, 0x0003F003},
 	{0x10030, 0x00060000},
@@ -7285,35 +14073,283 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radioa_regs[] =  {
 	{0x10030, 0x00062000},
 	{0x10030, 0x00063000},
 	{0x10030, 0x00064000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x00065000},
 	{0x10030, 0x00066000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x00067003},
 	{0x10030, 0x00068000},
 	{0x10030, 0x00069000},
 	{0x10030, 0x0006A000},
 	{0x10030, 0x0006B000},
 	{0x10030, 0x0006C000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x0006D000},
 	{0x10030, 0x0006E000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x0006F003},
 	{0x10030, 0x00070000},
 	{0x10030, 0x00071000},
 	{0x10030, 0x00072000},
 	{0x10030, 0x00073000},
 	{0x10030, 0x00074000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0xA0000000, 0x00000000},
 	{0x10030, 0x00075000},
 	{0x10030, 0x00076000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x00077003},
 	{0x10030, 0x00078000},
 	{0x10030, 0x00079000},
 	{0x10030, 0x0007A000},
 	{0x10030, 0x0007B000},
 	{0x10030, 0x0007C000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x0007D000},
 	{0x10030, 0x0007E000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x0007F003},
 	{0x100EE, 0x00000000},
-	{0x0FE, 0x00000031},
+	{0x0FE, 0x00000048},
 };
 
 static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
@@ -7326,13 +14362,17 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0xF0360000, 0x00000006},
 	{0xF0010001, 0x00000007},
 	{0xF0020001, 0x00000008},
-	{0xF0320001, 0x00000009},
-	{0xF0330001, 0x0000000A},
-	{0xF0340001, 0x0000000B},
-	{0xF0350001, 0x0000000C},
-	{0xF0360001, 0x0000000D},
-	{0xF03F0001, 0x0000000E},
-	{0xF0400001, 0x0000000F},
+	{0xF0030001, 0x00000009},
+	{0xF0040001, 0x0000000A},
+	{0xF0050001, 0x0000000B},
+	{0xF0070001, 0x0000000C},
+	{0xF0320001, 0x0000000D},
+	{0xF0330001, 0x0000000E},
+	{0xF0340001, 0x0000000F},
+	{0xF0350001, 0x00000010},
+	{0xF0360001, 0x00000011},
+	{0xF03F0001, 0x00000012},
+	{0xF0400001, 0x00000013},
 	{0x005, 0x00000000},
 	{0x10005, 0x00000000},
 	{0x0B9, 0x00020440},
@@ -7340,42 +14380,69 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x10000, 0x00030000},
 	{0x018, 0x00011124},
 	{0x10018, 0x00011124},
-	{0x05F, 0x00000032},
+	{0x05F, 0x00000038},
 	{0x097, 0x00043200},
 	{0x0A6, 0x00066DB7},
 	{0x0EF, 0x00004000},
 	{0x033, 0x00000005},
 	{0x03E, 0x00000000},
 	{0x03F, 0x00010500},
+	{0x033, 0x00000004},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00000400},
 	{0x033, 0x00000003},
 	{0x03E, 0x00000000},
 	{0x03F, 0x00028B00},
 	{0x033, 0x00000002},
 	{0x03E, 0x00000000},
 	{0x03F, 0x0009AB00},
+	{0x033, 0x00000001},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00001A00},
+	{0x033, 0x00000000},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00002900},
 	{0x033, 0x0000000D},
 	{0x03E, 0x00000000},
 	{0x03F, 0x00010500},
+	{0x033, 0x0000000C},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00000400},
 	{0x033, 0x0000000B},
 	{0x03E, 0x00000000},
 	{0x03F, 0x00028B00},
 	{0x033, 0x0000000A},
 	{0x03E, 0x00000000},
 	{0x03F, 0x0009AB00},
+	{0x033, 0x00000009},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00001A00},
+	{0x033, 0x00000008},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00002900},
 	{0x033, 0x00000015},
 	{0x03E, 0x00000000},
 	{0x03F, 0x00010500},
+	{0x033, 0x00000014},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00000400},
 	{0x033, 0x00000013},
 	{0x03E, 0x00000000},
 	{0x03F, 0x00028B00},
 	{0x033, 0x00000012},
 	{0x03E, 0x00000000},
 	{0x03F, 0x0009AB00},
+	{0x033, 0x00000011},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00001A00},
+	{0x033, 0x00000010},
+	{0x03E, 0x00000000},
+	{0x03F, 0x00002900},
 	{0x0EF, 0x00000000},
+	{0x10055, 0x00080080},
 	{0x000, 0x00033C01},
 	{0x10000, 0x00033C00},
 	{0x01A, 0x00040004},
-	{0x0FE, 0x00000000},
 	{0x096, 0x00015200},
 	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x067, 0x0004D000},
@@ -7404,6 +14471,18 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x067, 0x0000D300},
 	{0x0DA, 0x000D4000},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x067, 0x0000D300},
+	{0x0DA, 0x000D4000},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x067, 0x0000D300},
+	{0x0DA, 0x000D4000},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x067, 0x0000D300},
+	{0x0DA, 0x000D4000},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x067, 0x0000D300},
+	{0x0DA, 0x000D4000},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x067, 0x0000D300},
 	{0x0DA, 0x000D4000},
@@ -7430,7 +14509,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x0DA, 0x000D4009},
 	{0xB0000000, 0x00000000},
 	{0x057, 0x0000D589},
-	{0x05A, 0x0007FFFF},
+	{0x05A, 0x0007F0F8},
 	{0x043, 0x00005000},
 	{0x018, 0x00001001},
 	{0x10018, 0x00001001},
@@ -7462,6 +14541,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x08F, 0x000D1352},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x08F, 0x000D1352},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x08F, 0x000D1352},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x08F, 0x000D1352},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x08F, 0x000D1352},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x08F, 0x000D1352},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x08F, 0x000D1352},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -7496,6 +14583,52 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x03F, 0x00000015},
 	{0x033, 0x00000001},
 	{0x03F, 0x00000017},
+	{0x033, 0x00000004},
+	{0x03F, 0x00000017},
+	{0x033, 0x00000005},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000017},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000007},
+	{0xB0000000, 0x00000000},
 	{0x0EF, 0x00000000},
 	{0x0EF, 0x00008000},
 	{0x033, 0x00000020},
@@ -8007,6 +15140,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x03F, 0x0000EFFF},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000EFFF},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000EFFF},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000EFFF},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000EFFF},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000EFFF},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000EFFF},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -8113,7 +15254,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x00000005},
 	{0x03F, 0x00004344},
 	{0x033, 0x00000006},
-	{0x03F, 0x00004324},
+	{0x03F, 0x00004344},
 	{0x033, 0x00000007},
 	{0x03F, 0x00004344},
 	{0x033, 0x00000008},
@@ -8176,6 +15317,85 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x03F, 0x00000200},
 	{0x0EF, 0x00000000},
 	{0x0EF, 0x00000010},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x030, 0x000084DC},
 	{0x030, 0x000103C9},
 	{0x030, 0x00018399},
@@ -8188,6 +15408,189 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x030, 0x00050011},
 	{0x030, 0x00058000},
 	{0x030, 0x00060000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x030, 0x000085ED},
+	{0x030, 0x000105CC},
+	{0x030, 0x000184AA},
+	{0x030, 0x00020388},
+	{0x030, 0x00028377},
+	{0x030, 0x00030377},
+	{0x030, 0x00038255},
+	{0x030, 0x00040244},
+	{0x030, 0x00048133},
+	{0x030, 0x00050112},
+	{0x030, 0x00058101},
+	{0x030, 0x00060001},
+	{0xA0000000, 0x00000000},
+	{0x030, 0x000084DC},
+	{0x030, 0x000103C9},
+	{0x030, 0x00018399},
+	{0x030, 0x00020287},
+	{0x030, 0x00028277},
+	{0x030, 0x00030165},
+	{0x030, 0x00038144},
+	{0x030, 0x00040044},
+	{0x030, 0x00048022},
+	{0x030, 0x00050011},
+	{0x030, 0x00058000},
+	{0x030, 0x00060000},
+	{0xB0000000, 0x00000000},
 	{0x030, 0x00068000},
 	{0x030, 0x00070000},
 	{0x0EF, 0x00000000},
@@ -8458,6 +15861,14 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x095, 0x00000008},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x095, 0x00000008},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x095, 0x00000008},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x095, 0x00000008},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x095, 0x00000008},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x095, 0x00000008},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x095, 0x00000008},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
@@ -8477,101 +15888,2117 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0xB0000000, 0x00000000},
 	{0x0EE, 0x00001000},
 	{0x033, 0x00000020},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000024},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000028},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000002C},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000030},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000034},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E7},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E7},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000038},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E7},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000003C},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000021},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000025},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000029},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000002D},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000031},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000035},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000039},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000003D},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000022},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000026},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000002A},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000002E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000032},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000036},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000003A},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000003E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000060},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000064},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000068},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000006C},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000070},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000074},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000078},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000007C},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000061},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000065},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000069},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000006D},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000071},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000075},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000079},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000007D},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000062},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000052},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000052},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000066},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000005A},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000005A},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000006A},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000009C},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000009C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000006E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000072},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000076},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000007A},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000007E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000003E6},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E7},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x000003E6},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000063},
 	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000052},
@@ -8591,20 +18018,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x03F, 0x00000152},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000152},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000152},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000152},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x00000152},
+	{0x03F, 0x00000052},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000052},
 	{0xB0000000, 0x00000000},
@@ -8627,20 +18062,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x03F, 0x0000015A},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000015A},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000015A},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000015A},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000015A},
+	{0x03F, 0x0000005A},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000005A},
 	{0xB0000000, 0x00000000},
@@ -8663,20 +18106,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x03F, 0x0000019C},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x0000019C},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x0000019C},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x0000019C},
+	{0x03F, 0x0000009C},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000009C},
 	{0xB0000000, 0x00000000},
@@ -8699,20 +18150,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x03F, 0x000001A4},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001A4},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001A4},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001A4},
+	{0x03F, 0x0000019C},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x0000019C},
 	{0xB0000000, 0x00000000},
@@ -8735,20 +18194,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x03F, 0x000001E6},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000001E6},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000001E6},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000001E6},
+	{0x03F, 0x000001A4},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x000001A4},
 	{0xB0000000, 0x00000000},
@@ -8771,20 +18238,28 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x03F, 0x000002E6},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x000002E6},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x000002E6},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x03F, 0x000002E6},
+	{0x03F, 0x000001E6},
 	{0xA0000000, 0x00000000},
 	{0x03F, 0x000001E6},
 	{0xB0000000, 0x00000000},
@@ -9828,131 +19303,131 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x10030, 0x00003C5F},
 	{0x10030, 0x00004059},
 	{0x10030, 0x00004453},
-	{0x10030, 0x000201ED},
-	{0x10030, 0x000205AD},
-	{0x10030, 0x000209A7},
-	{0x10030, 0x00020DA1},
-	{0x10030, 0x0002119B},
-	{0x10030, 0x00021561},
-	{0x10030, 0x0002195B},
-	{0x10030, 0x00021D27},
-	{0x10030, 0x00022121},
-	{0x10030, 0x000224E9},
-	{0x10030, 0x000228E3},
-	{0x10030, 0x00022CA9},
-	{0x10030, 0x000230A3},
-	{0x10030, 0x00023469},
-	{0x10030, 0x00023863},
-	{0x10030, 0x00023C29},
-	{0x10030, 0x00024023},
-	{0x10030, 0x0002441D},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
 	{0x10030, 0x000281EF},
-	{0x10030, 0x000285AF},
-	{0x10030, 0x000289A9},
-	{0x10030, 0x00028DA3},
-	{0x10030, 0x0002919D},
-	{0x10030, 0x00029563},
-	{0x10030, 0x0002995D},
-	{0x10030, 0x00029D25},
-	{0x10030, 0x0002A11F},
-	{0x10030, 0x0002A4E7},
-	{0x10030, 0x0002A8E1},
-	{0x10030, 0x0002ACA7},
-	{0x10030, 0x0002B0A1},
-	{0x10030, 0x0002B467},
-	{0x10030, 0x0002B861},
-	{0x10030, 0x0002BC27},
-	{0x10030, 0x0002C021},
-	{0x10030, 0x0002C41B},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
 	{0x10030, 0x000301EF},
-	{0x10030, 0x000305AF},
-	{0x10030, 0x000309A9},
-	{0x10030, 0x00030DA3},
-	{0x10030, 0x0003119D},
-	{0x10030, 0x00031563},
-	{0x10030, 0x0003195D},
-	{0x10030, 0x00031D25},
-	{0x10030, 0x0003211F},
-	{0x10030, 0x000324E7},
-	{0x10030, 0x000328E1},
-	{0x10030, 0x00032CA7},
-	{0x10030, 0x000330A1},
-	{0x10030, 0x00033467},
-	{0x10030, 0x00033861},
-	{0x10030, 0x00033C27},
-	{0x10030, 0x00034021},
-	{0x10030, 0x0003441B},
-	{0x10030, 0x000601EB},
-	{0x10030, 0x000605AB},
-	{0x10030, 0x000609A5},
-	{0x10030, 0x00060D9F},
-	{0x10030, 0x00061199},
-	{0x10030, 0x00061593},
-	{0x10030, 0x00061959},
-	{0x10030, 0x00061D53},
-	{0x10030, 0x0006211B},
-	{0x10030, 0x00062515},
-	{0x10030, 0x000628DD},
-	{0x10030, 0x00062CD7},
-	{0x10030, 0x0006309D},
-	{0x10030, 0x00063497},
-	{0x10030, 0x0006385D},
-	{0x10030, 0x00063C57},
-	{0x10030, 0x0006401D},
-	{0x10030, 0x00064417},
-	{0x10030, 0x000681E7},
-	{0x10030, 0x000685A7},
-	{0x10030, 0x000689A1},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955F},
-	{0x10030, 0x00069959},
-	{0x10030, 0x00069D21},
-	{0x10030, 0x0006A11B},
-	{0x10030, 0x0006A4E3},
-	{0x10030, 0x0006A8DD},
-	{0x10030, 0x0006ACA5},
-	{0x10030, 0x0006B09F},
-	{0x10030, 0x0006B465},
-	{0x10030, 0x0006B85F},
-	{0x10030, 0x0006BC25},
-	{0x10030, 0x0006C01F},
-	{0x10030, 0x0006C419},
-	{0x10030, 0x000701E7},
-	{0x10030, 0x000705A7},
-	{0x10030, 0x000709A1},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071955},
-	{0x10030, 0x00071D1D},
-	{0x10030, 0x00072117},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072CA1},
-	{0x10030, 0x0007309B},
-	{0x10030, 0x00073461},
-	{0x10030, 0x0007385B},
-	{0x10030, 0x00073C21},
-	{0x10030, 0x0007401B},
-	{0x10030, 0x0007441B},
-	{0x10030, 0x000781EF},
-	{0x10030, 0x000785E9},
-	{0x10030, 0x000789E3},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x00072111},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
 	{0x10030, 0x00078DA3},
 	{0x10030, 0x00079161},
 	{0x10030, 0x0007955B},
-	{0x10030, 0x00079921},
-	{0x10030, 0x00079D1B},
-	{0x10030, 0x0007A0E1},
-	{0x10030, 0x0007A4DB},
-	{0x10030, 0x0007A8A1},
-	{0x10030, 0x0007AC9B},
-	{0x10030, 0x0007B061},
-	{0x10030, 0x0007B45B},
-	{0x10030, 0x0007B821},
-	{0x10030, 0x0007BC1B},
-	{0x10030, 0x0007C015},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
 	{0x10030, 0x0007C40F},
 	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x000001EF},
@@ -9973,131 +19448,711 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x10030, 0x00003C5F},
 	{0x10030, 0x00004059},
 	{0x10030, 0x00004453},
-	{0x10030, 0x000201ED},
-	{0x10030, 0x000205AD},
-	{0x10030, 0x000209A7},
-	{0x10030, 0x00020DA1},
-	{0x10030, 0x0002119B},
-	{0x10030, 0x00021561},
-	{0x10030, 0x0002195B},
-	{0x10030, 0x00021D27},
-	{0x10030, 0x00022121},
-	{0x10030, 0x000224E9},
-	{0x10030, 0x000228E3},
-	{0x10030, 0x00022CA9},
-	{0x10030, 0x000230A3},
-	{0x10030, 0x00023469},
-	{0x10030, 0x00023863},
-	{0x10030, 0x00023C29},
-	{0x10030, 0x00024023},
-	{0x10030, 0x0002441D},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
 	{0x10030, 0x000281EF},
-	{0x10030, 0x000285AF},
-	{0x10030, 0x000289A9},
-	{0x10030, 0x00028DA3},
-	{0x10030, 0x0002919D},
-	{0x10030, 0x00029563},
-	{0x10030, 0x0002995D},
-	{0x10030, 0x00029D25},
-	{0x10030, 0x0002A11F},
-	{0x10030, 0x0002A4E7},
-	{0x10030, 0x0002A8E1},
-	{0x10030, 0x0002ACA7},
-	{0x10030, 0x0002B0A1},
-	{0x10030, 0x0002B467},
-	{0x10030, 0x0002B861},
-	{0x10030, 0x0002BC27},
-	{0x10030, 0x0002C021},
-	{0x10030, 0x0002C41B},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
 	{0x10030, 0x000301EF},
-	{0x10030, 0x000305AF},
-	{0x10030, 0x000309A9},
-	{0x10030, 0x00030DA3},
-	{0x10030, 0x0003119D},
-	{0x10030, 0x00031563},
-	{0x10030, 0x0003195D},
-	{0x10030, 0x00031D25},
-	{0x10030, 0x0003211F},
-	{0x10030, 0x000324E7},
-	{0x10030, 0x000328E1},
-	{0x10030, 0x00032CA7},
-	{0x10030, 0x000330A1},
-	{0x10030, 0x00033467},
-	{0x10030, 0x00033861},
-	{0x10030, 0x00033C27},
-	{0x10030, 0x00034021},
-	{0x10030, 0x0003441B},
-	{0x10030, 0x000601EB},
-	{0x10030, 0x000605AB},
-	{0x10030, 0x000609A5},
-	{0x10030, 0x00060D9F},
-	{0x10030, 0x00061199},
-	{0x10030, 0x00061593},
-	{0x10030, 0x00061959},
-	{0x10030, 0x00061D53},
-	{0x10030, 0x0006211B},
-	{0x10030, 0x00062515},
-	{0x10030, 0x000628DD},
-	{0x10030, 0x00062CD7},
-	{0x10030, 0x0006309D},
-	{0x10030, 0x00063497},
-	{0x10030, 0x0006385D},
-	{0x10030, 0x00063C57},
-	{0x10030, 0x0006401D},
-	{0x10030, 0x00064417},
-	{0x10030, 0x000681E7},
-	{0x10030, 0x000685A7},
-	{0x10030, 0x000689A1},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955F},
-	{0x10030, 0x00069959},
-	{0x10030, 0x00069D21},
-	{0x10030, 0x0006A11B},
-	{0x10030, 0x0006A4E3},
-	{0x10030, 0x0006A8DD},
-	{0x10030, 0x0006ACA5},
-	{0x10030, 0x0006B09F},
-	{0x10030, 0x0006B465},
-	{0x10030, 0x0006B85F},
-	{0x10030, 0x0006BC25},
-	{0x10030, 0x0006C01F},
-	{0x10030, 0x0006C419},
-	{0x10030, 0x000701E7},
-	{0x10030, 0x000705A7},
-	{0x10030, 0x000709A1},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071955},
-	{0x10030, 0x00071D1D},
-	{0x10030, 0x00072117},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072CA1},
-	{0x10030, 0x0007309B},
-	{0x10030, 0x00073461},
-	{0x10030, 0x0007385B},
-	{0x10030, 0x00073C21},
-	{0x10030, 0x0007401B},
-	{0x10030, 0x0007441B},
-	{0x10030, 0x000781EF},
-	{0x10030, 0x000785E9},
-	{0x10030, 0x000789E3},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x00072111},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
 	{0x10030, 0x00078DA3},
 	{0x10030, 0x00079161},
 	{0x10030, 0x0007955B},
-	{0x10030, 0x00079921},
-	{0x10030, 0x00079D1B},
-	{0x10030, 0x0007A0E1},
-	{0x10030, 0x0007A4DB},
-	{0x10030, 0x0007A8A1},
-	{0x10030, 0x0007AC9B},
-	{0x10030, 0x0007B061},
-	{0x10030, 0x0007B45B},
-	{0x10030, 0x0007B821},
-	{0x10030, 0x0007BC1B},
-	{0x10030, 0x0007C015},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
+	{0x10030, 0x000281EF},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
+	{0x10030, 0x000301EF},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x00072111},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
+	{0x10030, 0x00078DA3},
+	{0x10030, 0x00079161},
+	{0x10030, 0x0007955B},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
+	{0x10030, 0x000281EF},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
+	{0x10030, 0x000301EF},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x00072111},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
+	{0x10030, 0x00078DA3},
+	{0x10030, 0x00079161},
+	{0x10030, 0x0007955B},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
+	{0x10030, 0x000281EF},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
+	{0x10030, 0x000301EF},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x00072111},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
+	{0x10030, 0x00078DA3},
+	{0x10030, 0x00079161},
+	{0x10030, 0x0007955B},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
+	{0x10030, 0x000281EF},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
+	{0x10030, 0x000301EF},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x00072111},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
+	{0x10030, 0x00078DA3},
+	{0x10030, 0x00079161},
+	{0x10030, 0x0007955B},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
 	{0x10030, 0x0007C40F},
 	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x000001EF},
@@ -10118,1002 +20173,1002 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x10030, 0x00003C5F},
 	{0x10030, 0x00004059},
 	{0x10030, 0x00004453},
-	{0x10030, 0x000201ED},
-	{0x10030, 0x000205AD},
-	{0x10030, 0x000209A7},
-	{0x10030, 0x00020DA1},
-	{0x10030, 0x0002119B},
-	{0x10030, 0x00021561},
-	{0x10030, 0x0002195B},
-	{0x10030, 0x00021D27},
-	{0x10030, 0x00022121},
-	{0x10030, 0x000224E9},
-	{0x10030, 0x000228E3},
-	{0x10030, 0x00022CA9},
-	{0x10030, 0x000230A3},
-	{0x10030, 0x00023469},
-	{0x10030, 0x00023863},
-	{0x10030, 0x00023C29},
-	{0x10030, 0x00024023},
-	{0x10030, 0x0002441D},
+	{0x10030, 0x000201EF},
+	{0x10030, 0x000205E9},
+	{0x10030, 0x000209E3},
+	{0x10030, 0x00020DA3},
+	{0x10030, 0x00021161},
+	{0x10030, 0x0002155B},
+	{0x10030, 0x0002191F},
+	{0x10030, 0x00021D19},
+	{0x10030, 0x000220E1},
+	{0x10030, 0x000224DB},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1B},
+	{0x10030, 0x00024015},
+	{0x10030, 0x0002440F},
 	{0x10030, 0x000281EF},
-	{0x10030, 0x000285AF},
-	{0x10030, 0x000289A9},
-	{0x10030, 0x00028DA3},
-	{0x10030, 0x0002919D},
-	{0x10030, 0x00029563},
-	{0x10030, 0x0002995D},
-	{0x10030, 0x00029D25},
-	{0x10030, 0x0002A11F},
-	{0x10030, 0x0002A4E7},
-	{0x10030, 0x0002A8E1},
-	{0x10030, 0x0002ACA7},
-	{0x10030, 0x0002B0A1},
-	{0x10030, 0x0002B467},
-	{0x10030, 0x0002B861},
-	{0x10030, 0x0002BC27},
-	{0x10030, 0x0002C021},
-	{0x10030, 0x0002C41B},
+	{0x10030, 0x000285E7},
+	{0x10030, 0x000289A7},
+	{0x10030, 0x00028D65},
+	{0x10030, 0x0002915F},
+	{0x10030, 0x00029523},
+	{0x10030, 0x0002991D},
+	{0x10030, 0x00029CE5},
+	{0x10030, 0x0002A0DF},
+	{0x10030, 0x0002A4A7},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC67},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B427},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC19},
+	{0x10030, 0x0002C013},
+	{0x10030, 0x0002C40D},
 	{0x10030, 0x000301EF},
-	{0x10030, 0x000305AF},
-	{0x10030, 0x000309A9},
-	{0x10030, 0x00030DA3},
-	{0x10030, 0x0003119D},
-	{0x10030, 0x00031563},
-	{0x10030, 0x0003195D},
-	{0x10030, 0x00031D25},
-	{0x10030, 0x0003211F},
-	{0x10030, 0x000324E7},
-	{0x10030, 0x000328E1},
-	{0x10030, 0x00032CA7},
-	{0x10030, 0x000330A1},
-	{0x10030, 0x00033467},
-	{0x10030, 0x00033861},
-	{0x10030, 0x00033C27},
-	{0x10030, 0x00034021},
-	{0x10030, 0x0003441B},
-	{0x10030, 0x000601EB},
-	{0x10030, 0x000605AB},
-	{0x10030, 0x000609A5},
-	{0x10030, 0x00060D9F},
-	{0x10030, 0x00061199},
-	{0x10030, 0x00061593},
-	{0x10030, 0x00061959},
-	{0x10030, 0x00061D53},
-	{0x10030, 0x0006211B},
-	{0x10030, 0x00062515},
-	{0x10030, 0x000628DD},
-	{0x10030, 0x00062CD7},
-	{0x10030, 0x0006309D},
-	{0x10030, 0x00063497},
-	{0x10030, 0x0006385D},
-	{0x10030, 0x00063C57},
-	{0x10030, 0x0006401D},
-	{0x10030, 0x00064417},
-	{0x10030, 0x000681E7},
-	{0x10030, 0x000685A7},
-	{0x10030, 0x000689A1},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955F},
-	{0x10030, 0x00069959},
-	{0x10030, 0x00069D21},
-	{0x10030, 0x0006A11B},
-	{0x10030, 0x0006A4E3},
-	{0x10030, 0x0006A8DD},
-	{0x10030, 0x0006ACA5},
-	{0x10030, 0x0006B09F},
-	{0x10030, 0x0006B465},
-	{0x10030, 0x0006B85F},
-	{0x10030, 0x0006BC25},
-	{0x10030, 0x0006C01F},
-	{0x10030, 0x0006C419},
-	{0x10030, 0x000701E7},
-	{0x10030, 0x000705A7},
-	{0x10030, 0x000709A1},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071955},
-	{0x10030, 0x00071D1D},
-	{0x10030, 0x00072117},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072CA1},
-	{0x10030, 0x0007309B},
-	{0x10030, 0x00073461},
-	{0x10030, 0x0007385B},
-	{0x10030, 0x00073C21},
-	{0x10030, 0x0007401B},
-	{0x10030, 0x0007441B},
-	{0x10030, 0x000781EF},
-	{0x10030, 0x000785E9},
-	{0x10030, 0x000789E3},
+	{0x10030, 0x000305E7},
+	{0x10030, 0x000309A7},
+	{0x10030, 0x00030D65},
+	{0x10030, 0x0003115F},
+	{0x10030, 0x00031525},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031CE7},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324A9},
+	{0x10030, 0x000328A3},
+	{0x10030, 0x00032C69},
+	{0x10030, 0x00033063},
+	{0x10030, 0x00033429},
+	{0x10030, 0x00033823},
+	{0x10030, 0x00033C1D},
+	{0x10030, 0x00034013},
+	{0x10030, 0x0003440D},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x00072111},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
 	{0x10030, 0x00078DA3},
 	{0x10030, 0x00079161},
 	{0x10030, 0x0007955B},
-	{0x10030, 0x00079921},
-	{0x10030, 0x00079D1B},
-	{0x10030, 0x0007A0E1},
-	{0x10030, 0x0007A4DB},
-	{0x10030, 0x0007A8A1},
-	{0x10030, 0x0007AC9B},
-	{0x10030, 0x0007B061},
-	{0x10030, 0x0007B45B},
-	{0x10030, 0x0007B821},
-	{0x10030, 0x0007BC1B},
-	{0x10030, 0x0007C015},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
 	{0x10030, 0x0007C40F},
 	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
+	{0x10030, 0x00078DA3},
+	{0x10030, 0x00079161},
+	{0x10030, 0x0007955B},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
+	{0x10030, 0x00078DA3},
+	{0x10030, 0x00079161},
+	{0x10030, 0x0007955B},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
+	{0x10030, 0x00078DA3},
+	{0x10030, 0x00079161},
+	{0x10030, 0x0007955B},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
+	{0x10030, 0x00078DA3},
+	{0x10030, 0x00079161},
+	{0x10030, 0x0007955B},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
+	{0x10030, 0x00078DA3},
+	{0x10030, 0x00079161},
+	{0x10030, 0x0007955B},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
-	{0x10030, 0x000201DF},
-	{0x10030, 0x000205D9},
-	{0x10030, 0x000209D3},
-	{0x10030, 0x00020D99},
-	{0x10030, 0x00021193},
-	{0x10030, 0x0002155F},
-	{0x10030, 0x00021959},
-	{0x10030, 0x00021D21},
-	{0x10030, 0x00022119},
-	{0x10030, 0x000224DF},
-	{0x10030, 0x000228D9},
-	{0x10030, 0x00022C9F},
-	{0x10030, 0x00023099},
-	{0x10030, 0x0002345F},
-	{0x10030, 0x00023859},
-	{0x10030, 0x00023C1F},
-	{0x10030, 0x00024019},
-	{0x10030, 0x00024413},
-	{0x10030, 0x000281CD},
-	{0x10030, 0x000285DB},
-	{0x10030, 0x000289D5},
-	{0x10030, 0x00028D9B},
-	{0x10030, 0x0002918D},
-	{0x10030, 0x00029555},
-	{0x10030, 0x00029957},
-	{0x10030, 0x00029D1F},
-	{0x10030, 0x0002A119},
-	{0x10030, 0x0002A4DF},
-	{0x10030, 0x0002A8D9},
-	{0x10030, 0x0002AC9F},
-	{0x10030, 0x0002B099},
-	{0x10030, 0x0002B45F},
-	{0x10030, 0x0002B859},
-	{0x10030, 0x0002BC1F},
-	{0x10030, 0x0002C019},
-	{0x10030, 0x0002C413},
-	{0x10030, 0x000301D9},
-	{0x10030, 0x000305DB},
-	{0x10030, 0x000309D5},
-	{0x10030, 0x00030D9B},
-	{0x10030, 0x00031195},
-	{0x10030, 0x0003155D},
-	{0x10030, 0x00031955},
-	{0x10030, 0x00031D1D},
-	{0x10030, 0x00032119},
-	{0x10030, 0x000324DF},
-	{0x10030, 0x000328D9},
-	{0x10030, 0x00032C9F},
-	{0x10030, 0x00033099},
-	{0x10030, 0x0003345F},
-	{0x10030, 0x00033859},
-	{0x10030, 0x00033C1F},
-	{0x10030, 0x00034019},
-	{0x10030, 0x00034413},
-	{0x10030, 0x000601E1},
-	{0x10030, 0x000605DB},
-	{0x10030, 0x000609D5},
-	{0x10030, 0x00060D9B},
-	{0x10030, 0x00061195},
-	{0x10030, 0x0006155B},
-	{0x10030, 0x00061957},
-	{0x10030, 0x00061D1F},
-	{0x10030, 0x00062119},
-	{0x10030, 0x000624DF},
-	{0x10030, 0x000628D9},
-	{0x10030, 0x00062C9F},
-	{0x10030, 0x00063099},
-	{0x10030, 0x0006345F},
-	{0x10030, 0x00063859},
-	{0x10030, 0x00063C1F},
-	{0x10030, 0x00064019},
-	{0x10030, 0x00064413},
-	{0x10030, 0x000681E1},
-	{0x10030, 0x000685DB},
-	{0x10030, 0x000689D5},
-	{0x10030, 0x00068D9B},
-	{0x10030, 0x00069195},
-	{0x10030, 0x0006955B},
-	{0x10030, 0x00069957},
-	{0x10030, 0x00069D1F},
-	{0x10030, 0x0006A119},
-	{0x10030, 0x0006A4DF},
-	{0x10030, 0x0006A8D9},
-	{0x10030, 0x0006AC9F},
-	{0x10030, 0x0006B099},
-	{0x10030, 0x0006B45F},
-	{0x10030, 0x0006B859},
-	{0x10030, 0x0006BC1F},
-	{0x10030, 0x0006C019},
-	{0x10030, 0x0006C413},
-	{0x10030, 0x000701E1},
-	{0x10030, 0x000705DB},
-	{0x10030, 0x000709D5},
-	{0x10030, 0x00070D9B},
-	{0x10030, 0x00071195},
-	{0x10030, 0x0007155B},
-	{0x10030, 0x00071957},
-	{0x10030, 0x00071D1F},
-	{0x10030, 0x00072119},
-	{0x10030, 0x000724DF},
-	{0x10030, 0x000728D9},
-	{0x10030, 0x00072C9F},
-	{0x10030, 0x00073099},
-	{0x10030, 0x0007345F},
-	{0x10030, 0x00073859},
-	{0x10030, 0x00073C1F},
-	{0x10030, 0x00074019},
-	{0x10030, 0x00074413},
-	{0x10030, 0x000781DF},
-	{0x10030, 0x000785D9},
-	{0x10030, 0x000789D3},
-	{0x10030, 0x00078D99},
-	{0x10030, 0x00079193},
-	{0x10030, 0x0007955F},
-	{0x10030, 0x00079959},
-	{0x10030, 0x00079D21},
-	{0x10030, 0x0007A115},
-	{0x10030, 0x0007A4DF},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007AC9F},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B45F},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC1F},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
-	{0x10030, 0x00000000},
-	{0x10030, 0x000785A9},
-	{0x10030, 0x000789A3},
-	{0x10030, 0x00078D9D},
-	{0x10030, 0x00079197},
-	{0x10030, 0x00079591},
-	{0x10030, 0x00079957},
-	{0x10030, 0x00079D51},
-	{0x10030, 0x0007A119},
-	{0x10030, 0x0007A513},
-	{0x10030, 0x0007A8D9},
-	{0x10030, 0x0007ACD3},
-	{0x10030, 0x0007B099},
-	{0x10030, 0x0007B493},
-	{0x10030, 0x0007B859},
-	{0x10030, 0x0007BC53},
-	{0x10030, 0x0007C019},
-	{0x10030, 0x0007C413},
+	{0x10030, 0x000001EF},
+	{0x10030, 0x000005E9},
+	{0x10030, 0x000009E3},
+	{0x10030, 0x00000DDD},
+	{0x10030, 0x000011D7},
+	{0x10030, 0x0000159F},
+	{0x10030, 0x00001999},
+	{0x10030, 0x00001D5F},
+	{0x10030, 0x00002159},
+	{0x10030, 0x0000251F},
+	{0x10030, 0x00002919},
+	{0x10030, 0x00002CDF},
+	{0x10030, 0x000030D9},
+	{0x10030, 0x0000349F},
+	{0x10030, 0x00003899},
+	{0x10030, 0x00003C5F},
+	{0x10030, 0x00004059},
+	{0x10030, 0x00004453},
+	{0x10030, 0x000201A7},
+	{0x10030, 0x000205A1},
+	{0x10030, 0x0002099B},
+	{0x10030, 0x00020D95},
+	{0x10030, 0x0002115B},
+	{0x10030, 0x00021555},
+	{0x10030, 0x00021921},
+	{0x10030, 0x00021D1B},
+	{0x10030, 0x000220E3},
+	{0x10030, 0x000224DD},
+	{0x10030, 0x000228A3},
+	{0x10030, 0x00022C9D},
+	{0x10030, 0x00023063},
+	{0x10030, 0x0002345D},
+	{0x10030, 0x00023823},
+	{0x10030, 0x00023C1D},
+	{0x10030, 0x00024017},
+	{0x10030, 0x00024411},
+	{0x10030, 0x000281A9},
+	{0x10030, 0x000285A3},
+	{0x10030, 0x0002899D},
+	{0x10030, 0x00028D97},
+	{0x10030, 0x0002915D},
+	{0x10030, 0x00029557},
+	{0x10030, 0x0002991F},
+	{0x10030, 0x00029D19},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DB},
+	{0x10030, 0x0002A8A1},
+	{0x10030, 0x0002AC9B},
+	{0x10030, 0x0002B061},
+	{0x10030, 0x0002B45B},
+	{0x10030, 0x0002B821},
+	{0x10030, 0x0002BC1B},
+	{0x10030, 0x0002C015},
+	{0x10030, 0x0002C40F},
+	{0x10030, 0x000301A9},
+	{0x10030, 0x000305A3},
+	{0x10030, 0x0003099D},
+	{0x10030, 0x00030D97},
+	{0x10030, 0x0003115D},
+	{0x10030, 0x00031557},
+	{0x10030, 0x0003191F},
+	{0x10030, 0x00031D19},
+	{0x10030, 0x000320E1},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328A1},
+	{0x10030, 0x00032C9B},
+	{0x10030, 0x00033061},
+	{0x10030, 0x0003345B},
+	{0x10030, 0x00033821},
+	{0x10030, 0x00033C1B},
+	{0x10030, 0x00034015},
+	{0x10030, 0x0003440F},
+	{0x10030, 0x000601F1},
+	{0x10030, 0x000605E9},
+	{0x10030, 0x000609A9},
+	{0x10030, 0x00060D65},
+	{0x10030, 0x0006115F},
+	{0x10030, 0x00061525},
+	{0x10030, 0x0006191F},
+	{0x10030, 0x00061CE7},
+	{0x10030, 0x000620E1},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628A3},
+	{0x10030, 0x00062C69},
+	{0x10030, 0x00063063},
+	{0x10030, 0x00063429},
+	{0x10030, 0x00063823},
+	{0x10030, 0x00063C1D},
+	{0x10030, 0x00064013},
+	{0x10030, 0x0006440D},
+	{0x10030, 0x000681EF},
+	{0x10030, 0x000685E7},
+	{0x10030, 0x000689A7},
+	{0x10030, 0x00068D61},
+	{0x10030, 0x0006915B},
+	{0x10030, 0x00069523},
+	{0x10030, 0x0006991D},
+	{0x10030, 0x00069CE5},
+	{0x10030, 0x0006A0DF},
+	{0x10030, 0x0006A4A7},
+	{0x10030, 0x0006A8A1},
+	{0x10030, 0x0006AC67},
+	{0x10030, 0x0006B061},
+	{0x10030, 0x0006B429},
+	{0x10030, 0x0006B823},
+	{0x10030, 0x0006BC1D},
+	{0x10030, 0x0006C017},
+	{0x10030, 0x0006C40D},
+	{0x10030, 0x000701F1},
+	{0x10030, 0x000705E9},
+	{0x10030, 0x000709A9},
+	{0x10030, 0x00070D63},
+	{0x10030, 0x0007115D},
+	{0x10030, 0x00071523},
+	{0x10030, 0x0007191D},
+	{0x10030, 0x00071D17},
+	{0x10030, 0x000720DF},
+	{0x10030, 0x000724D9},
+	{0x10030, 0x000728D3},
+	{0x10030, 0x00072C67},
+	{0x10030, 0x00073061},
+	{0x10030, 0x00073427},
+	{0x10030, 0x00073821},
+	{0x10030, 0x00073C1B},
+	{0x10030, 0x00074015},
+	{0x10030, 0x0007440D},
+	{0x10030, 0x000781F1},
+	{0x10030, 0x000785EB},
+	{0x10030, 0x000789E5},
+	{0x10030, 0x00078DA3},
+	{0x10030, 0x00079161},
+	{0x10030, 0x0007955B},
+	{0x10030, 0x00079923},
+	{0x10030, 0x00079D1D},
+	{0x10030, 0x0007A117},
+	{0x10030, 0x0007A4DD},
+	{0x10030, 0x0007A8D7},
+	{0x10030, 0x0007AC9D},
+	{0x10030, 0x0007B063},
+	{0x10030, 0x0007B45D},
+	{0x10030, 0x0007B857},
+	{0x10030, 0x0007BC1D},
+	{0x10030, 0x0007C017},
+	{0x10030, 0x0007C40F},
 	{0xA0000000, 0x00000000},
 	{0x10030, 0x000001EF},
 	{0x10030, 0x000005E9},
@@ -11281,6 +21336,1150 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x10030, 0x00004017},
 	{0x100EE, 0x00000000},
 	{0x100EE, 0x00002000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x000200E8},
+	{0x10030, 0x000204E5},
+	{0x10030, 0x000208E2},
+	{0x10030, 0x00020CDF},
+	{0x10030, 0x000210DC},
+	{0x10030, 0x000214D9},
+	{0x10030, 0x000218D6},
+	{0x10030, 0x00021CD3},
+	{0x10030, 0x000220D0},
+	{0x10030, 0x000224CD},
+	{0x10030, 0x000228CD},
+	{0x10030, 0x00022CCD},
+	{0x10030, 0x000230CD},
+	{0x10030, 0x000234CD},
+	{0x10030, 0x000238CD},
+	{0x10030, 0x00023CCD},
+	{0x10030, 0x000240CD},
+	{0x10030, 0x000280E8},
+	{0x10030, 0x000284E5},
+	{0x10030, 0x000288E2},
+	{0x10030, 0x00028CDF},
+	{0x10030, 0x000290DC},
+	{0x10030, 0x000294D9},
+	{0x10030, 0x000298D6},
+	{0x10030, 0x00029CD3},
+	{0x10030, 0x0002A0D0},
+	{0x10030, 0x0002A4CD},
+	{0x10030, 0x0002A8CD},
+	{0x10030, 0x0002ACCD},
+	{0x10030, 0x0002B0CD},
+	{0x10030, 0x0002B4CD},
+	{0x10030, 0x0002B8CD},
+	{0x10030, 0x0002BCCD},
+	{0x10030, 0x0002C0CD},
+	{0x10030, 0x000300E8},
+	{0x10030, 0x000304E5},
+	{0x10030, 0x000308E2},
+	{0x10030, 0x00030CDF},
+	{0x10030, 0x000310DC},
+	{0x10030, 0x000314D9},
+	{0x10030, 0x000318D6},
+	{0x10030, 0x00031CD3},
+	{0x10030, 0x000320D0},
+	{0x10030, 0x000324CD},
+	{0x10030, 0x000328CD},
+	{0x10030, 0x00032CCD},
+	{0x10030, 0x000330CD},
+	{0x10030, 0x000334CD},
+	{0x10030, 0x000338CD},
+	{0x10030, 0x00033CCD},
+	{0x10030, 0x000340CD},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x000200FA},
+	{0x10030, 0x000204F7},
+	{0x10030, 0x000208F4},
+	{0x10030, 0x00020CF1},
+	{0x10030, 0x000210EE},
+	{0x10030, 0x000214EB},
+	{0x10030, 0x000218E8},
+	{0x10030, 0x00021CE5},
+	{0x10030, 0x000220E2},
+	{0x10030, 0x000224DF},
+	{0x10030, 0x000228DC},
+	{0x10030, 0x00022CD9},
+	{0x10030, 0x000230D6},
+	{0x10030, 0x000234D3},
+	{0x10030, 0x000238D0},
+	{0x10030, 0x00023C0D},
+	{0x10030, 0x0002400A},
+	{0x10030, 0x000280F9},
+	{0x10030, 0x000284F6},
+	{0x10030, 0x000288F3},
+	{0x10030, 0x00028CF0},
+	{0x10030, 0x000290ED},
+	{0x10030, 0x000294EA},
+	{0x10030, 0x000298E7},
+	{0x10030, 0x00029CE4},
+	{0x10030, 0x0002A0E1},
+	{0x10030, 0x0002A4DE},
+	{0x10030, 0x0002A8DB},
+	{0x10030, 0x0002ACD8},
+	{0x10030, 0x0002B0D5},
+	{0x10030, 0x0002B4D2},
+	{0x10030, 0x0002B8CF},
+	{0x10030, 0x0002BC0C},
+	{0x10030, 0x0002C009},
+	{0x10030, 0x000300F6},
+	{0x10030, 0x000304F3},
+	{0x10030, 0x000308F0},
+	{0x10030, 0x00030CED},
+	{0x10030, 0x000310EA},
+	{0x10030, 0x000314E7},
+	{0x10030, 0x000318E4},
+	{0x10030, 0x00031CE1},
+	{0x10030, 0x000320DE},
+	{0x10030, 0x000324DB},
+	{0x10030, 0x000328D8},
+	{0x10030, 0x00032CD5},
+	{0x10030, 0x000330D2},
+	{0x10030, 0x000334CF},
+	{0x10030, 0x000338CC},
+	{0x10030, 0x00033C09},
+	{0x10030, 0x00034006},
+	{0xB0000000, 0x00000000},
+	{0x10030, 0x000600F6},
+	{0x10030, 0x000604F3},
+	{0x10030, 0x000608F0},
+	{0x10030, 0x00060CED},
+	{0x10030, 0x000610EA},
+	{0x10030, 0x000614E7},
+	{0x10030, 0x000618E4},
+	{0x10030, 0x00061CE1},
+	{0x10030, 0x000620DE},
+	{0x10030, 0x000624DB},
+	{0x10030, 0x000628D8},
+	{0x10030, 0x00062CD5},
+	{0x10030, 0x000630D2},
+	{0x10030, 0x000634CF},
+	{0x10030, 0x000638CC},
+	{0x10030, 0x00063C09},
+	{0x10030, 0x00064006},
+	{0x10030, 0x000680F5},
+	{0x10030, 0x000684F2},
+	{0x10030, 0x000688EF},
+	{0x10030, 0x00068CEC},
+	{0x10030, 0x000690E9},
+	{0x10030, 0x000694E6},
+	{0x10030, 0x000698E3},
+	{0x10030, 0x00069CE0},
+	{0x10030, 0x0006A0DD},
+	{0x10030, 0x0006A4DA},
+	{0x10030, 0x0006A8D7},
+	{0x10030, 0x0006ACD4},
+	{0x10030, 0x0006B0D1},
+	{0x10030, 0x0006B4CE},
+	{0x10030, 0x0006B8CB},
+	{0x10030, 0x0006BC08},
+	{0x10030, 0x0006C005},
+	{0x10030, 0x000700F5},
+	{0x10030, 0x000704F2},
+	{0x10030, 0x000708EF},
+	{0x10030, 0x00070CEC},
+	{0x10030, 0x000710E9},
+	{0x10030, 0x000714E6},
+	{0x10030, 0x000718E3},
+	{0x10030, 0x00071CE0},
+	{0x10030, 0x000720DD},
+	{0x10030, 0x000724DA},
+	{0x10030, 0x000728D7},
+	{0x10030, 0x00072CD4},
+	{0x10030, 0x000730D1},
+	{0x10030, 0x000734CE},
+	{0x10030, 0x000738CB},
+	{0x10030, 0x00073C08},
+	{0x10030, 0x00074005},
 	{0x10030, 0x000780F4},
 	{0x10030, 0x000784F1},
 	{0x10030, 0x000788EE},
@@ -11334,9 +22533,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x00000025},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000026},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000027},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000028},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000029},
@@ -11350,9 +22633,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x0000002D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000002E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000002F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000030},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000031},
@@ -11366,9 +22733,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x00000035},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000036},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000037},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000060},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000061},
@@ -11382,9 +22833,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x00000065},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000066},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000067},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000068},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000069},
@@ -11398,9 +22933,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x0000006D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000006E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000006F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000070},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000071},
@@ -11414,9 +23033,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x00000075},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000076},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000077},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000078},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000079},
@@ -11430,9 +23133,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x0000007D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000007E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000007F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000A0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000A1},
@@ -11446,9 +23233,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000000A5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000A6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000A7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000A8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000A9},
@@ -11462,9 +23333,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000000AD},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000AE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000AF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000B0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000B1},
@@ -11478,9 +23433,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000000B5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000B6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000B7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000E0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000E1},
@@ -11494,9 +23533,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000000E5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000E6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000E7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000E8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000E9},
@@ -11510,9 +23633,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000000ED},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000EE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000EF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000F0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000F1},
@@ -11526,9 +23733,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000000F5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000F6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000F7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000F8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000000F9},
@@ -11542,9 +23833,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000000FD},
 	{0x03F, 0x00008002},
 	{0x033, 0x000000FE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000000FF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000120},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000121},
@@ -11558,9 +23933,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x00000125},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000126},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000127},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000128},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000129},
@@ -11574,9 +24033,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x0000012D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000012E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000012F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000130},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000131},
@@ -11590,9 +24133,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x00000135},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000136},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000137},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000160},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000161},
@@ -11606,9 +24233,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x00000165},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000166},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000167},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000168},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000169},
@@ -11622,9 +24333,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x0000016D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000016E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000016F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000170},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000171},
@@ -11638,9 +24433,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x00000175},
 	{0x03F, 0x00008002},
 	{0x033, 0x00000176},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000177},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x00000178},
 	{0x03F, 0x00050002},
 	{0x033, 0x00000179},
@@ -11654,9 +24533,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x0000017D},
 	{0x03F, 0x00008002},
 	{0x033, 0x0000017E},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x0000017F},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001A0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001A1},
@@ -11670,9 +24633,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000001A5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001A6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001A7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001A8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001A9},
@@ -11686,9 +24733,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000001AD},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001AE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001AF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001B0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001B1},
@@ -11702,9 +24833,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000001B5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001B6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001B7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001E0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001E1},
@@ -11718,9 +24933,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000001E5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001E6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001E7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001E8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001E9},
@@ -11734,9 +25033,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000001ED},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001EE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001EF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001F0},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001F1},
@@ -11750,9 +25133,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000001F5},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001F6},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001F7},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001F8},
 	{0x03F, 0x00050002},
 	{0x033, 0x000001F9},
@@ -11766,9 +25233,93 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x033, 0x000001FD},
 	{0x03F, 0x00008002},
 	{0x033, 0x000001FE},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x033, 0x000001FF},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
 	{0x03F, 0x00000003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x03F, 0x00008002},
+	{0xA0000000, 0x00000000},
+	{0x03F, 0x00000003},
+	{0xB0000000, 0x00000000},
 	{0x0EF, 0x00000000},
 	{0x005, 0x00000001},
 	{0x10005, 0x00000001},
@@ -11810,7 +25361,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x10030, 0x00022000},
 	{0x10030, 0x00023000},
 	{0x10030, 0x00024000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x00025000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00025003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x00025000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x00026003},
 	{0x10030, 0x00027003},
 	{0x10030, 0x00028000},
@@ -11818,7 +25411,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x10030, 0x0002A000},
 	{0x10030, 0x0002B000},
 	{0x10030, 0x0002C000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x0002D000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0002D003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x0002D000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x0002E003},
 	{0x10030, 0x0002F003},
 	{0x10030, 0x00030000},
@@ -11826,7 +25461,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x10030, 0x00032000},
 	{0x10030, 0x00033000},
 	{0x10030, 0x00034000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00035003},
+	{0xA0000000, 0x00000000},
 	{0x10030, 0x00035000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x00036003},
 	{0x10030, 0x00037003},
 	{0x10030, 0x00038000},
@@ -11834,7 +25511,49 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x10030, 0x0003A000},
 	{0x10030, 0x0003B000},
 	{0x10030, 0x0003C000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x0003D000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0003D003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x0003D000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x0003E003},
 	{0x10030, 0x0003F003},
 	{0x10030, 0x00060000},
@@ -11842,32 +25561,280 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x10030, 0x00062000},
 	{0x10030, 0x00063000},
 	{0x10030, 0x00064000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x00065000},
 	{0x10030, 0x00066000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00065003},
+	{0x10030, 0x00066003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x00065000},
+	{0x10030, 0x00066000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x00067003},
 	{0x10030, 0x00068000},
 	{0x10030, 0x00069000},
 	{0x10030, 0x0006A000},
 	{0x10030, 0x0006B000},
 	{0x10030, 0x0006C000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x0006D000},
 	{0x10030, 0x0006E000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0006D003},
+	{0x10030, 0x0006E003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x0006D000},
+	{0x10030, 0x0006E000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x0006F003},
 	{0x10030, 0x00070000},
 	{0x10030, 0x00071000},
 	{0x10030, 0x00072000},
 	{0x10030, 0x00073000},
 	{0x10030, 0x00074000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x00075000},
 	{0x10030, 0x00076000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x00075003},
+	{0x10030, 0x00076003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x00075000},
+	{0x10030, 0x00076000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x00077003},
 	{0x10030, 0x00078000},
 	{0x10030, 0x00079000},
 	{0x10030, 0x0007A000},
 	{0x10030, 0x0007B000},
 	{0x10030, 0x0007C000},
+	{0x80010000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90020000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90320000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90330000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90340000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90350000, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0x90360000, 0x00000000}, {0x40000000, 0x00000000},
 	{0x10030, 0x0007D000},
 	{0x10030, 0x0007E000},
+	{0x90010001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90020001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90030001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90040001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90050001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90070001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90320001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90330001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90340001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90350001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90360001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x903f0001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0x90400001, 0x00000000}, {0x40000000, 0x00000000},
+	{0x10030, 0x0007D003},
+	{0x10030, 0x0007E003},
+	{0xA0000000, 0x00000000},
+	{0x10030, 0x0007D000},
+	{0x10030, 0x0007E000},
+	{0xB0000000, 0x00000000},
 	{0x10030, 0x0007F003},
 	{0x0ED, 0x00000010},
 	{0x033, 0x00000001},
@@ -11884,7 +25851,7 @@ static const struct rtw89_reg2_def rtw89_8852c_phy_radiob_regs[] = {
 	{0x03F, 0x0000000A},
 	{0x0ED, 0x00000000},
 	{0x100EE, 0x00000000},
-	{0x0FE, 0x00000031},
+	{0x0FE, 0x00000048},
 };
 
 static const struct rtw89_reg2_def rtw89_8852c_phy_nctl_regs[] = {
@@ -13825,1207 +27792,1722 @@ static const s8 _txpwr_track_delta_swingidx_2g_cck_a_p[] = {
 const u8 rtw89_8852c_tx_shape[RTW89_BAND_MAX][RTW89_RS_TX_SHAPE_NUM]
 			     [RTW89_REGD_NUM] = {
 	[0][0][RTW89_ACMA] = 0,
+	[0][0][RTW89_CN] = 0,
 	[0][0][RTW89_ETSI] = 0,
 	[0][0][RTW89_FCC] = 1,
 	[0][0][RTW89_IC] = 1,
+	[0][0][RTW89_KCC] = 0,
 	[0][0][RTW89_MKK] = 0,
+	[0][0][RTW89_UK] = 0,
 	[0][1][RTW89_ACMA] = 0,
+	[0][1][RTW89_CN] = 0,
 	[0][1][RTW89_ETSI] = 0,
 	[0][1][RTW89_FCC] = 3,
 	[0][1][RTW89_IC] = 3,
+	[0][1][RTW89_KCC] = 0,
 	[0][1][RTW89_MKK] = 0,
+	[0][1][RTW89_UK] = 0,
 	[1][1][RTW89_ACMA] = 0,
+	[1][1][RTW89_CN] = 0,
 	[1][1][RTW89_ETSI] = 0,
 	[1][1][RTW89_FCC] = 3,
 	[1][1][RTW89_IC] = 3,
+	[1][1][RTW89_KCC] = 0,
 	[1][1][RTW89_MKK] = 0,
-	[2][1][RTW89_FCC] = 1,
+	[1][1][RTW89_UK] = 0,
+	[2][1][RTW89_ETSI] = 0,
+	[2][1][RTW89_FCC] = 0,
+	[2][1][RTW89_KCC] = 0,
 };
 
 const s8 rtw89_8852c_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM]
 				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
 				 [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = {
-	[0][0][0][0][RTW89_WW][0] = 60,
-	[0][0][0][0][RTW89_WW][1] = 60,
-	[0][0][0][0][RTW89_WW][2] = 60,
-	[0][0][0][0][RTW89_WW][3] = 60,
-	[0][0][0][0][RTW89_WW][4] = 60,
-	[0][0][0][0][RTW89_WW][5] = 60,
-	[0][0][0][0][RTW89_WW][6] = 60,
-	[0][0][0][0][RTW89_WW][7] = 60,
-	[0][0][0][0][RTW89_WW][8] = 60,
-	[0][0][0][0][RTW89_WW][9] = 60,
-	[0][0][0][0][RTW89_WW][10] = 60,
-	[0][0][0][0][RTW89_WW][11] = 60,
-	[0][0][0][0][RTW89_WW][12] = 48,
+	[0][0][0][0][RTW89_WW][0] = 58,
+	[0][0][0][0][RTW89_WW][1] = 58,
+	[0][0][0][0][RTW89_WW][2] = 58,
+	[0][0][0][0][RTW89_WW][3] = 58,
+	[0][0][0][0][RTW89_WW][4] = 58,
+	[0][0][0][0][RTW89_WW][5] = 58,
+	[0][0][0][0][RTW89_WW][6] = 58,
+	[0][0][0][0][RTW89_WW][7] = 58,
+	[0][0][0][0][RTW89_WW][8] = 58,
+	[0][0][0][0][RTW89_WW][9] = 58,
+	[0][0][0][0][RTW89_WW][10] = 58,
+	[0][0][0][0][RTW89_WW][11] = 58,
+	[0][0][0][0][RTW89_WW][12] = 46,
 	[0][0][0][0][RTW89_WW][13] = 72,
-	[0][1][0][0][RTW89_WW][0] = 48,
-	[0][1][0][0][RTW89_WW][1] = 48,
-	[0][1][0][0][RTW89_WW][2] = 48,
-	[0][1][0][0][RTW89_WW][3] = 48,
-	[0][1][0][0][RTW89_WW][4] = 48,
-	[0][1][0][0][RTW89_WW][5] = 48,
-	[0][1][0][0][RTW89_WW][6] = 48,
-	[0][1][0][0][RTW89_WW][7] = 48,
-	[0][1][0][0][RTW89_WW][8] = 48,
-	[0][1][0][0][RTW89_WW][9] = 48,
-	[0][1][0][0][RTW89_WW][10] = 48,
-	[0][1][0][0][RTW89_WW][11] = 46,
-	[0][1][0][0][RTW89_WW][12] = 34,
+	[0][1][0][0][RTW89_WW][0] = 42,
+	[0][1][0][0][RTW89_WW][1] = 42,
+	[0][1][0][0][RTW89_WW][2] = 42,
+	[0][1][0][0][RTW89_WW][3] = 42,
+	[0][1][0][0][RTW89_WW][4] = 42,
+	[0][1][0][0][RTW89_WW][5] = 42,
+	[0][1][0][0][RTW89_WW][6] = 42,
+	[0][1][0][0][RTW89_WW][7] = 42,
+	[0][1][0][0][RTW89_WW][8] = 42,
+	[0][1][0][0][RTW89_WW][9] = 42,
+	[0][1][0][0][RTW89_WW][10] = 42,
+	[0][1][0][0][RTW89_WW][11] = 42,
+	[0][1][0][0][RTW89_WW][12] = 18,
 	[0][1][0][0][RTW89_WW][13] = 60,
 	[1][0][0][0][RTW89_WW][0] = 0,
 	[1][0][0][0][RTW89_WW][1] = 0,
-	[1][0][0][0][RTW89_WW][2] = 42,
-	[1][0][0][0][RTW89_WW][3] = 42,
-	[1][0][0][0][RTW89_WW][4] = 42,
+	[1][0][0][0][RTW89_WW][2] = 44,
+	[1][0][0][0][RTW89_WW][3] = 58,
+	[1][0][0][0][RTW89_WW][4] = 58,
 	[1][0][0][0][RTW89_WW][5] = 58,
-	[1][0][0][0][RTW89_WW][6] = 42,
-	[1][0][0][0][RTW89_WW][7] = 42,
-	[1][0][0][0][RTW89_WW][8] = 42,
-	[1][0][0][0][RTW89_WW][9] = 34,
-	[1][0][0][0][RTW89_WW][10] = 22,
+	[1][0][0][0][RTW89_WW][6] = 46,
+	[1][0][0][0][RTW89_WW][7] = 46,
+	[1][0][0][0][RTW89_WW][8] = 28,
+	[1][0][0][0][RTW89_WW][9] = 26,
+	[1][0][0][0][RTW89_WW][10] = 26,
 	[1][0][0][0][RTW89_WW][11] = 0,
 	[1][0][0][0][RTW89_WW][12] = 0,
 	[1][0][0][0][RTW89_WW][13] = 0,
 	[1][1][0][0][RTW89_WW][0] = 0,
 	[1][1][0][0][RTW89_WW][1] = 0,
-	[1][1][0][0][RTW89_WW][2] = 38,
-	[1][1][0][0][RTW89_WW][3] = 38,
-	[1][1][0][0][RTW89_WW][4] = 38,
-	[1][1][0][0][RTW89_WW][5] = 48,
-	[1][1][0][0][RTW89_WW][6] = 26,
-	[1][1][0][0][RTW89_WW][7] = 26,
-	[1][1][0][0][RTW89_WW][8] = 26,
-	[1][1][0][0][RTW89_WW][9] = 22,
-	[1][1][0][0][RTW89_WW][10] = 22,
+	[1][1][0][0][RTW89_WW][2] = 46,
+	[1][1][0][0][RTW89_WW][3] = 46,
+	[1][1][0][0][RTW89_WW][4] = 46,
+	[1][1][0][0][RTW89_WW][5] = 46,
+	[1][1][0][0][RTW89_WW][6] = 40,
+	[1][1][0][0][RTW89_WW][7] = 40,
+	[1][1][0][0][RTW89_WW][8] = 14,
+	[1][1][0][0][RTW89_WW][9] = 14,
+	[1][1][0][0][RTW89_WW][10] = 12,
 	[1][1][0][0][RTW89_WW][11] = 0,
 	[1][1][0][0][RTW89_WW][12] = 0,
 	[1][1][0][0][RTW89_WW][13] = 0,
-	[0][0][1][0][RTW89_WW][0] = 60,
-	[0][0][1][0][RTW89_WW][1] = 60,
-	[0][0][1][0][RTW89_WW][2] = 60,
-	[0][0][1][0][RTW89_WW][3] = 60,
-	[0][0][1][0][RTW89_WW][4] = 60,
-	[0][0][1][0][RTW89_WW][5] = 60,
-	[0][0][1][0][RTW89_WW][6] = 60,
-	[0][0][1][0][RTW89_WW][7] = 60,
-	[0][0][1][0][RTW89_WW][8] = 60,
-	[0][0][1][0][RTW89_WW][9] = 60,
-	[0][0][1][0][RTW89_WW][10] = 60,
-	[0][0][1][0][RTW89_WW][11] = 46,
-	[0][0][1][0][RTW89_WW][12] = 42,
+	[0][0][1][0][RTW89_WW][0] = 58,
+	[0][0][1][0][RTW89_WW][1] = 58,
+	[0][0][1][0][RTW89_WW][2] = 58,
+	[0][0][1][0][RTW89_WW][3] = 58,
+	[0][0][1][0][RTW89_WW][4] = 58,
+	[0][0][1][0][RTW89_WW][5] = 58,
+	[0][0][1][0][RTW89_WW][6] = 58,
+	[0][0][1][0][RTW89_WW][7] = 58,
+	[0][0][1][0][RTW89_WW][8] = 58,
+	[0][0][1][0][RTW89_WW][9] = 58,
+	[0][0][1][0][RTW89_WW][10] = 58,
+	[0][0][1][0][RTW89_WW][11] = 58,
+	[0][0][1][0][RTW89_WW][12] = 58,
 	[0][0][1][0][RTW89_WW][13] = 0,
-	[0][1][1][0][RTW89_WW][0] = 48,
-	[0][1][1][0][RTW89_WW][1] = 48,
-	[0][1][1][0][RTW89_WW][2] = 48,
-	[0][1][1][0][RTW89_WW][3] = 48,
-	[0][1][1][0][RTW89_WW][4] = 48,
-	[0][1][1][0][RTW89_WW][5] = 48,
-	[0][1][1][0][RTW89_WW][6] = 48,
-	[0][1][1][0][RTW89_WW][7] = 48,
-	[0][1][1][0][RTW89_WW][8] = 48,
-	[0][1][1][0][RTW89_WW][9] = 48,
-	[0][1][1][0][RTW89_WW][10] = 48,
-	[0][1][1][0][RTW89_WW][11] = 38,
-	[0][1][1][0][RTW89_WW][12] = 34,
+	[0][1][1][0][RTW89_WW][0] = 46,
+	[0][1][1][0][RTW89_WW][1] = 46,
+	[0][1][1][0][RTW89_WW][2] = 46,
+	[0][1][1][0][RTW89_WW][3] = 46,
+	[0][1][1][0][RTW89_WW][4] = 46,
+	[0][1][1][0][RTW89_WW][5] = 46,
+	[0][1][1][0][RTW89_WW][6] = 46,
+	[0][1][1][0][RTW89_WW][7] = 46,
+	[0][1][1][0][RTW89_WW][8] = 46,
+	[0][1][1][0][RTW89_WW][9] = 46,
+	[0][1][1][0][RTW89_WW][10] = 46,
+	[0][1][1][0][RTW89_WW][11] = 46,
+	[0][1][1][0][RTW89_WW][12] = 36,
 	[0][1][1][0][RTW89_WW][13] = 0,
-	[0][0][2][0][RTW89_WW][0] = 60,
-	[0][0][2][0][RTW89_WW][1] = 60,
-	[0][0][2][0][RTW89_WW][2] = 60,
-	[0][0][2][0][RTW89_WW][3] = 60,
-	[0][0][2][0][RTW89_WW][4] = 60,
-	[0][0][2][0][RTW89_WW][5] = 60,
-	[0][0][2][0][RTW89_WW][6] = 60,
-	[0][0][2][0][RTW89_WW][7] = 60,
-	[0][0][2][0][RTW89_WW][8] = 60,
-	[0][0][2][0][RTW89_WW][9] = 60,
-	[0][0][2][0][RTW89_WW][10] = 60,
-	[0][0][2][0][RTW89_WW][11] = 46,
-	[0][0][2][0][RTW89_WW][12] = 42,
+	[0][0][2][0][RTW89_WW][0] = 58,
+	[0][0][2][0][RTW89_WW][1] = 58,
+	[0][0][2][0][RTW89_WW][2] = 58,
+	[0][0][2][0][RTW89_WW][3] = 58,
+	[0][0][2][0][RTW89_WW][4] = 58,
+	[0][0][2][0][RTW89_WW][5] = 58,
+	[0][0][2][0][RTW89_WW][6] = 58,
+	[0][0][2][0][RTW89_WW][7] = 58,
+	[0][0][2][0][RTW89_WW][8] = 58,
+	[0][0][2][0][RTW89_WW][9] = 58,
+	[0][0][2][0][RTW89_WW][10] = 58,
+	[0][0][2][0][RTW89_WW][11] = 58,
+	[0][0][2][0][RTW89_WW][12] = 38,
 	[0][0][2][0][RTW89_WW][13] = 0,
-	[0][1][2][0][RTW89_WW][0] = 48,
-	[0][1][2][0][RTW89_WW][1] = 48,
-	[0][1][2][0][RTW89_WW][2] = 48,
-	[0][1][2][0][RTW89_WW][3] = 48,
-	[0][1][2][0][RTW89_WW][4] = 48,
-	[0][1][2][0][RTW89_WW][5] = 48,
-	[0][1][2][0][RTW89_WW][6] = 48,
-	[0][1][2][0][RTW89_WW][7] = 48,
-	[0][1][2][0][RTW89_WW][8] = 48,
-	[0][1][2][0][RTW89_WW][9] = 48,
-	[0][1][2][0][RTW89_WW][10] = 48,
-	[0][1][2][0][RTW89_WW][11] = 38,
-	[0][1][2][0][RTW89_WW][12] = 34,
+	[0][1][2][0][RTW89_WW][0] = 46,
+	[0][1][2][0][RTW89_WW][1] = 46,
+	[0][1][2][0][RTW89_WW][2] = 46,
+	[0][1][2][0][RTW89_WW][3] = 46,
+	[0][1][2][0][RTW89_WW][4] = 46,
+	[0][1][2][0][RTW89_WW][5] = 46,
+	[0][1][2][0][RTW89_WW][6] = 46,
+	[0][1][2][0][RTW89_WW][7] = 46,
+	[0][1][2][0][RTW89_WW][8] = 46,
+	[0][1][2][0][RTW89_WW][9] = 46,
+	[0][1][2][0][RTW89_WW][10] = 46,
+	[0][1][2][0][RTW89_WW][11] = 46,
+	[0][1][2][0][RTW89_WW][12] = 16,
 	[0][1][2][0][RTW89_WW][13] = 0,
 	[0][1][2][1][RTW89_WW][0] = 36,
-	[0][1][2][1][RTW89_WW][1] = 36,
-	[0][1][2][1][RTW89_WW][2] = 36,
-	[0][1][2][1][RTW89_WW][3] = 36,
-	[0][1][2][1][RTW89_WW][4] = 36,
-	[0][1][2][1][RTW89_WW][5] = 36,
-	[0][1][2][1][RTW89_WW][6] = 36,
-	[0][1][2][1][RTW89_WW][7] = 36,
-	[0][1][2][1][RTW89_WW][8] = 36,
-	[0][1][2][1][RTW89_WW][9] = 36,
-	[0][1][2][1][RTW89_WW][10] = 36,
-	[0][1][2][1][RTW89_WW][11] = 36,
-	[0][1][2][1][RTW89_WW][12] = 34,
+	[0][1][2][1][RTW89_WW][1] = 34,
+	[0][1][2][1][RTW89_WW][2] = 34,
+	[0][1][2][1][RTW89_WW][3] = 34,
+	[0][1][2][1][RTW89_WW][4] = 34,
+	[0][1][2][1][RTW89_WW][5] = 34,
+	[0][1][2][1][RTW89_WW][6] = 34,
+	[0][1][2][1][RTW89_WW][7] = 34,
+	[0][1][2][1][RTW89_WW][8] = 34,
+	[0][1][2][1][RTW89_WW][9] = 34,
+	[0][1][2][1][RTW89_WW][10] = 34,
+	[0][1][2][1][RTW89_WW][11] = 34,
+	[0][1][2][1][RTW89_WW][12] = 16,
 	[0][1][2][1][RTW89_WW][13] = 0,
 	[1][0][2][0][RTW89_WW][0] = 0,
 	[1][0][2][0][RTW89_WW][1] = 0,
-	[1][0][2][0][RTW89_WW][2] = 60,
-	[1][0][2][0][RTW89_WW][3] = 60,
-	[1][0][2][0][RTW89_WW][4] = 60,
-	[1][0][2][0][RTW89_WW][5] = 60,
-	[1][0][2][0][RTW89_WW][6] = 60,
-	[1][0][2][0][RTW89_WW][7] = 60,
-	[1][0][2][0][RTW89_WW][8] = 60,
-	[1][0][2][0][RTW89_WW][9] = 60,
-	[1][0][2][0][RTW89_WW][10] = 58,
+	[1][0][2][0][RTW89_WW][2] = 58,
+	[1][0][2][0][RTW89_WW][3] = 58,
+	[1][0][2][0][RTW89_WW][4] = 58,
+	[1][0][2][0][RTW89_WW][5] = 58,
+	[1][0][2][0][RTW89_WW][6] = 58,
+	[1][0][2][0][RTW89_WW][7] = 58,
+	[1][0][2][0][RTW89_WW][8] = 58,
+	[1][0][2][0][RTW89_WW][9] = 58,
+	[1][0][2][0][RTW89_WW][10] = 56,
 	[1][0][2][0][RTW89_WW][11] = 0,
 	[1][0][2][0][RTW89_WW][12] = 0,
 	[1][0][2][0][RTW89_WW][13] = 0,
 	[1][1][2][0][RTW89_WW][0] = 0,
 	[1][1][2][0][RTW89_WW][1] = 0,
-	[1][1][2][0][RTW89_WW][2] = 46,
-	[1][1][2][0][RTW89_WW][3] = 46,
-	[1][1][2][0][RTW89_WW][4] = 48,
-	[1][1][2][0][RTW89_WW][5] = 48,
-	[1][1][2][0][RTW89_WW][6] = 48,
-	[1][1][2][0][RTW89_WW][7] = 46,
-	[1][1][2][0][RTW89_WW][8] = 46,
+	[1][1][2][0][RTW89_WW][2] = 34,
+	[1][1][2][0][RTW89_WW][3] = 34,
+	[1][1][2][0][RTW89_WW][4] = 34,
+	[1][1][2][0][RTW89_WW][5] = 34,
+	[1][1][2][0][RTW89_WW][6] = 34,
+	[1][1][2][0][RTW89_WW][7] = 34,
+	[1][1][2][0][RTW89_WW][8] = 34,
 	[1][1][2][0][RTW89_WW][9] = 34,
-	[1][1][2][0][RTW89_WW][10] = 30,
+	[1][1][2][0][RTW89_WW][10] = 34,
 	[1][1][2][0][RTW89_WW][11] = 0,
 	[1][1][2][0][RTW89_WW][12] = 0,
 	[1][1][2][0][RTW89_WW][13] = 0,
 	[1][1][2][1][RTW89_WW][0] = 0,
 	[1][1][2][1][RTW89_WW][1] = 0,
-	[1][1][2][1][RTW89_WW][2] = 36,
-	[1][1][2][1][RTW89_WW][3] = 36,
-	[1][1][2][1][RTW89_WW][4] = 36,
-	[1][1][2][1][RTW89_WW][5] = 36,
-	[1][1][2][1][RTW89_WW][6] = 36,
-	[1][1][2][1][RTW89_WW][7] = 36,
-	[1][1][2][1][RTW89_WW][8] = 36,
+	[1][1][2][1][RTW89_WW][2] = 34,
+	[1][1][2][1][RTW89_WW][3] = 34,
+	[1][1][2][1][RTW89_WW][4] = 34,
+	[1][1][2][1][RTW89_WW][5] = 34,
+	[1][1][2][1][RTW89_WW][6] = 34,
+	[1][1][2][1][RTW89_WW][7] = 34,
+	[1][1][2][1][RTW89_WW][8] = 34,
 	[1][1][2][1][RTW89_WW][9] = 34,
-	[1][1][2][1][RTW89_WW][10] = 30,
+	[1][1][2][1][RTW89_WW][10] = 36,
 	[1][1][2][1][RTW89_WW][11] = 0,
 	[1][1][2][1][RTW89_WW][12] = 0,
 	[1][1][2][1][RTW89_WW][13] = 0,
-	[0][0][0][0][RTW89_FCC][0] = 70,
+	[0][0][0][0][RTW89_FCC][0] = 76,
 	[0][0][0][0][RTW89_ETSI][0] = 60,
 	[0][0][0][0][RTW89_MKK][0] = 68,
-	[0][0][0][0][RTW89_IC][0] = 74,
+	[0][0][0][0][RTW89_IC][0] = 76,
+	[0][0][0][0][RTW89_KCC][0] = 68,
 	[0][0][0][0][RTW89_ACMA][0] = 60,
-	[0][0][0][0][RTW89_FCC][1] = 70,
+	[0][0][0][0][RTW89_CN][0] = 58,
+	[0][0][0][0][RTW89_UK][0] = 60,
+	[0][0][0][0][RTW89_FCC][1] = 76,
 	[0][0][0][0][RTW89_ETSI][1] = 60,
 	[0][0][0][0][RTW89_MKK][1] = 68,
-	[0][0][0][0][RTW89_IC][1] = 74,
+	[0][0][0][0][RTW89_IC][1] = 76,
+	[0][0][0][0][RTW89_KCC][1] = 68,
 	[0][0][0][0][RTW89_ACMA][1] = 60,
-	[0][0][0][0][RTW89_FCC][2] = 70,
+	[0][0][0][0][RTW89_CN][1] = 58,
+	[0][0][0][0][RTW89_UK][1] = 60,
+	[0][0][0][0][RTW89_FCC][2] = 76,
 	[0][0][0][0][RTW89_ETSI][2] = 60,
 	[0][0][0][0][RTW89_MKK][2] = 68,
-	[0][0][0][0][RTW89_IC][2] = 74,
+	[0][0][0][0][RTW89_IC][2] = 76,
+	[0][0][0][0][RTW89_KCC][2] = 68,
 	[0][0][0][0][RTW89_ACMA][2] = 60,
-	[0][0][0][0][RTW89_FCC][3] = 70,
+	[0][0][0][0][RTW89_CN][2] = 58,
+	[0][0][0][0][RTW89_UK][2] = 60,
+	[0][0][0][0][RTW89_FCC][3] = 76,
 	[0][0][0][0][RTW89_ETSI][3] = 60,
 	[0][0][0][0][RTW89_MKK][3] = 68,
-	[0][0][0][0][RTW89_IC][3] = 74,
+	[0][0][0][0][RTW89_IC][3] = 76,
+	[0][0][0][0][RTW89_KCC][3] = 68,
 	[0][0][0][0][RTW89_ACMA][3] = 60,
-	[0][0][0][0][RTW89_FCC][4] = 70,
+	[0][0][0][0][RTW89_CN][3] = 58,
+	[0][0][0][0][RTW89_UK][3] = 60,
+	[0][0][0][0][RTW89_FCC][4] = 76,
 	[0][0][0][0][RTW89_ETSI][4] = 60,
 	[0][0][0][0][RTW89_MKK][4] = 68,
-	[0][0][0][0][RTW89_IC][4] = 74,
+	[0][0][0][0][RTW89_IC][4] = 76,
+	[0][0][0][0][RTW89_KCC][4] = 68,
 	[0][0][0][0][RTW89_ACMA][4] = 60,
-	[0][0][0][0][RTW89_FCC][5] = 70,
+	[0][0][0][0][RTW89_CN][4] = 58,
+	[0][0][0][0][RTW89_UK][4] = 60,
+	[0][0][0][0][RTW89_FCC][5] = 76,
 	[0][0][0][0][RTW89_ETSI][5] = 60,
 	[0][0][0][0][RTW89_MKK][5] = 68,
-	[0][0][0][0][RTW89_IC][5] = 74,
+	[0][0][0][0][RTW89_IC][5] = 76,
+	[0][0][0][0][RTW89_KCC][5] = 68,
 	[0][0][0][0][RTW89_ACMA][5] = 60,
-	[0][0][0][0][RTW89_FCC][6] = 70,
+	[0][0][0][0][RTW89_CN][5] = 58,
+	[0][0][0][0][RTW89_UK][5] = 60,
+	[0][0][0][0][RTW89_FCC][6] = 76,
 	[0][0][0][0][RTW89_ETSI][6] = 60,
 	[0][0][0][0][RTW89_MKK][6] = 68,
-	[0][0][0][0][RTW89_IC][6] = 74,
+	[0][0][0][0][RTW89_IC][6] = 76,
+	[0][0][0][0][RTW89_KCC][6] = 68,
 	[0][0][0][0][RTW89_ACMA][6] = 60,
-	[0][0][0][0][RTW89_FCC][7] = 70,
+	[0][0][0][0][RTW89_CN][6] = 58,
+	[0][0][0][0][RTW89_UK][6] = 60,
+	[0][0][0][0][RTW89_FCC][7] = 76,
 	[0][0][0][0][RTW89_ETSI][7] = 60,
 	[0][0][0][0][RTW89_MKK][7] = 68,
-	[0][0][0][0][RTW89_IC][7] = 74,
+	[0][0][0][0][RTW89_IC][7] = 76,
+	[0][0][0][0][RTW89_KCC][7] = 68,
 	[0][0][0][0][RTW89_ACMA][7] = 60,
-	[0][0][0][0][RTW89_FCC][8] = 70,
+	[0][0][0][0][RTW89_CN][7] = 58,
+	[0][0][0][0][RTW89_UK][7] = 60,
+	[0][0][0][0][RTW89_FCC][8] = 76,
 	[0][0][0][0][RTW89_ETSI][8] = 60,
 	[0][0][0][0][RTW89_MKK][8] = 68,
-	[0][0][0][0][RTW89_IC][8] = 74,
+	[0][0][0][0][RTW89_IC][8] = 76,
+	[0][0][0][0][RTW89_KCC][8] = 68,
 	[0][0][0][0][RTW89_ACMA][8] = 60,
-	[0][0][0][0][RTW89_FCC][9] = 70,
+	[0][0][0][0][RTW89_CN][8] = 58,
+	[0][0][0][0][RTW89_UK][8] = 60,
+	[0][0][0][0][RTW89_FCC][9] = 76,
 	[0][0][0][0][RTW89_ETSI][9] = 60,
 	[0][0][0][0][RTW89_MKK][9] = 68,
-	[0][0][0][0][RTW89_IC][9] = 74,
+	[0][0][0][0][RTW89_IC][9] = 76,
+	[0][0][0][0][RTW89_KCC][9] = 70,
 	[0][0][0][0][RTW89_ACMA][9] = 60,
-	[0][0][0][0][RTW89_FCC][10] = 70,
+	[0][0][0][0][RTW89_CN][9] = 58,
+	[0][0][0][0][RTW89_UK][9] = 60,
+	[0][0][0][0][RTW89_FCC][10] = 76,
 	[0][0][0][0][RTW89_ETSI][10] = 60,
 	[0][0][0][0][RTW89_MKK][10] = 68,
-	[0][0][0][0][RTW89_IC][10] = 74,
+	[0][0][0][0][RTW89_IC][10] = 76,
+	[0][0][0][0][RTW89_KCC][10] = 70,
 	[0][0][0][0][RTW89_ACMA][10] = 60,
-	[0][0][0][0][RTW89_FCC][11] = 62,
+	[0][0][0][0][RTW89_CN][10] = 58,
+	[0][0][0][0][RTW89_UK][10] = 60,
+	[0][0][0][0][RTW89_FCC][11] = 58,
 	[0][0][0][0][RTW89_ETSI][11] = 60,
 	[0][0][0][0][RTW89_MKK][11] = 68,
-	[0][0][0][0][RTW89_IC][11] = 72,
+	[0][0][0][0][RTW89_IC][11] = 58,
+	[0][0][0][0][RTW89_KCC][11] = 70,
 	[0][0][0][0][RTW89_ACMA][11] = 60,
-	[0][0][0][0][RTW89_FCC][12] = 48,
+	[0][0][0][0][RTW89_CN][11] = 58,
+	[0][0][0][0][RTW89_UK][11] = 60,
+	[0][0][0][0][RTW89_FCC][12] = 46,
 	[0][0][0][0][RTW89_ETSI][12] = 60,
 	[0][0][0][0][RTW89_MKK][12] = 68,
-	[0][0][0][0][RTW89_IC][12] = 58,
+	[0][0][0][0][RTW89_IC][12] = 46,
+	[0][0][0][0][RTW89_KCC][12] = 70,
 	[0][0][0][0][RTW89_ACMA][12] = 60,
+	[0][0][0][0][RTW89_CN][12] = 58,
+	[0][0][0][0][RTW89_UK][12] = 60,
 	[0][0][0][0][RTW89_FCC][13] = 127,
 	[0][0][0][0][RTW89_ETSI][13] = 127,
 	[0][0][0][0][RTW89_MKK][13] = 72,
 	[0][0][0][0][RTW89_IC][13] = 127,
+	[0][0][0][0][RTW89_KCC][13] = 127,
 	[0][0][0][0][RTW89_ACMA][13] = 127,
-	[0][1][0][0][RTW89_FCC][0] = 66,
+	[0][0][0][0][RTW89_CN][13] = 127,
+	[0][0][0][0][RTW89_UK][13] = 127,
+	[0][1][0][0][RTW89_FCC][0] = 76,
 	[0][1][0][0][RTW89_ETSI][0] = 48,
 	[0][1][0][0][RTW89_MKK][0] = 58,
-	[0][1][0][0][RTW89_IC][0] = 74,
+	[0][1][0][0][RTW89_IC][0] = 76,
+	[0][1][0][0][RTW89_KCC][0] = 56,
 	[0][1][0][0][RTW89_ACMA][0] = 48,
-	[0][1][0][0][RTW89_FCC][1] = 66,
+	[0][1][0][0][RTW89_CN][0] = 42,
+	[0][1][0][0][RTW89_UK][0] = 48,
+	[0][1][0][0][RTW89_FCC][1] = 76,
 	[0][1][0][0][RTW89_ETSI][1] = 48,
 	[0][1][0][0][RTW89_MKK][1] = 58,
-	[0][1][0][0][RTW89_IC][1] = 74,
+	[0][1][0][0][RTW89_IC][1] = 76,
+	[0][1][0][0][RTW89_KCC][1] = 56,
 	[0][1][0][0][RTW89_ACMA][1] = 48,
-	[0][1][0][0][RTW89_FCC][2] = 66,
+	[0][1][0][0][RTW89_CN][1] = 42,
+	[0][1][0][0][RTW89_UK][1] = 48,
+	[0][1][0][0][RTW89_FCC][2] = 76,
 	[0][1][0][0][RTW89_ETSI][2] = 48,
 	[0][1][0][0][RTW89_MKK][2] = 58,
-	[0][1][0][0][RTW89_IC][2] = 74,
+	[0][1][0][0][RTW89_IC][2] = 76,
+	[0][1][0][0][RTW89_KCC][2] = 56,
 	[0][1][0][0][RTW89_ACMA][2] = 48,
-	[0][1][0][0][RTW89_FCC][3] = 66,
+	[0][1][0][0][RTW89_CN][2] = 42,
+	[0][1][0][0][RTW89_UK][2] = 48,
+	[0][1][0][0][RTW89_FCC][3] = 76,
 	[0][1][0][0][RTW89_ETSI][3] = 48,
 	[0][1][0][0][RTW89_MKK][3] = 58,
-	[0][1][0][0][RTW89_IC][3] = 74,
+	[0][1][0][0][RTW89_IC][3] = 76,
+	[0][1][0][0][RTW89_KCC][3] = 56,
 	[0][1][0][0][RTW89_ACMA][3] = 48,
-	[0][1][0][0][RTW89_FCC][4] = 66,
+	[0][1][0][0][RTW89_CN][3] = 42,
+	[0][1][0][0][RTW89_UK][3] = 48,
+	[0][1][0][0][RTW89_FCC][4] = 76,
 	[0][1][0][0][RTW89_ETSI][4] = 48,
 	[0][1][0][0][RTW89_MKK][4] = 58,
-	[0][1][0][0][RTW89_IC][4] = 74,
+	[0][1][0][0][RTW89_IC][4] = 76,
+	[0][1][0][0][RTW89_KCC][4] = 56,
 	[0][1][0][0][RTW89_ACMA][4] = 48,
-	[0][1][0][0][RTW89_FCC][5] = 66,
+	[0][1][0][0][RTW89_CN][4] = 42,
+	[0][1][0][0][RTW89_UK][4] = 48,
+	[0][1][0][0][RTW89_FCC][5] = 76,
 	[0][1][0][0][RTW89_ETSI][5] = 48,
 	[0][1][0][0][RTW89_MKK][5] = 58,
-	[0][1][0][0][RTW89_IC][5] = 74,
+	[0][1][0][0][RTW89_IC][5] = 76,
+	[0][1][0][0][RTW89_KCC][5] = 56,
 	[0][1][0][0][RTW89_ACMA][5] = 48,
-	[0][1][0][0][RTW89_FCC][6] = 66,
+	[0][1][0][0][RTW89_CN][5] = 42,
+	[0][1][0][0][RTW89_UK][5] = 48,
+	[0][1][0][0][RTW89_FCC][6] = 76,
 	[0][1][0][0][RTW89_ETSI][6] = 48,
 	[0][1][0][0][RTW89_MKK][6] = 58,
-	[0][1][0][0][RTW89_IC][6] = 74,
+	[0][1][0][0][RTW89_IC][6] = 76,
+	[0][1][0][0][RTW89_KCC][6] = 56,
 	[0][1][0][0][RTW89_ACMA][6] = 48,
-	[0][1][0][0][RTW89_FCC][7] = 66,
+	[0][1][0][0][RTW89_CN][6] = 42,
+	[0][1][0][0][RTW89_UK][6] = 48,
+	[0][1][0][0][RTW89_FCC][7] = 76,
 	[0][1][0][0][RTW89_ETSI][7] = 48,
 	[0][1][0][0][RTW89_MKK][7] = 58,
-	[0][1][0][0][RTW89_IC][7] = 74,
+	[0][1][0][0][RTW89_IC][7] = 76,
+	[0][1][0][0][RTW89_KCC][7] = 56,
 	[0][1][0][0][RTW89_ACMA][7] = 48,
-	[0][1][0][0][RTW89_FCC][8] = 66,
+	[0][1][0][0][RTW89_CN][7] = 42,
+	[0][1][0][0][RTW89_UK][7] = 48,
+	[0][1][0][0][RTW89_FCC][8] = 76,
 	[0][1][0][0][RTW89_ETSI][8] = 48,
 	[0][1][0][0][RTW89_MKK][8] = 58,
-	[0][1][0][0][RTW89_IC][8] = 74,
+	[0][1][0][0][RTW89_IC][8] = 76,
+	[0][1][0][0][RTW89_KCC][8] = 56,
 	[0][1][0][0][RTW89_ACMA][8] = 48,
-	[0][1][0][0][RTW89_FCC][9] = 66,
+	[0][1][0][0][RTW89_CN][8] = 42,
+	[0][1][0][0][RTW89_UK][8] = 48,
+	[0][1][0][0][RTW89_FCC][9] = 70,
 	[0][1][0][0][RTW89_ETSI][9] = 48,
 	[0][1][0][0][RTW89_MKK][9] = 58,
-	[0][1][0][0][RTW89_IC][9] = 74,
+	[0][1][0][0][RTW89_IC][9] = 70,
+	[0][1][0][0][RTW89_KCC][9] = 56,
 	[0][1][0][0][RTW89_ACMA][9] = 48,
-	[0][1][0][0][RTW89_FCC][10] = 66,
+	[0][1][0][0][RTW89_CN][9] = 42,
+	[0][1][0][0][RTW89_UK][9] = 48,
+	[0][1][0][0][RTW89_FCC][10] = 72,
 	[0][1][0][0][RTW89_ETSI][10] = 48,
 	[0][1][0][0][RTW89_MKK][10] = 58,
-	[0][1][0][0][RTW89_IC][10] = 74,
+	[0][1][0][0][RTW89_IC][10] = 72,
+	[0][1][0][0][RTW89_KCC][10] = 56,
 	[0][1][0][0][RTW89_ACMA][10] = 48,
-	[0][1][0][0][RTW89_FCC][11] = 46,
+	[0][1][0][0][RTW89_CN][10] = 42,
+	[0][1][0][0][RTW89_UK][10] = 48,
+	[0][1][0][0][RTW89_FCC][11] = 44,
 	[0][1][0][0][RTW89_ETSI][11] = 48,
 	[0][1][0][0][RTW89_MKK][11] = 58,
-	[0][1][0][0][RTW89_IC][11] = 56,
+	[0][1][0][0][RTW89_IC][11] = 44,
+	[0][1][0][0][RTW89_KCC][11] = 56,
 	[0][1][0][0][RTW89_ACMA][11] = 48,
-	[0][1][0][0][RTW89_FCC][12] = 34,
+	[0][1][0][0][RTW89_CN][11] = 42,
+	[0][1][0][0][RTW89_UK][11] = 48,
+	[0][1][0][0][RTW89_FCC][12] = 18,
 	[0][1][0][0][RTW89_ETSI][12] = 48,
 	[0][1][0][0][RTW89_MKK][12] = 58,
-	[0][1][0][0][RTW89_IC][12] = 44,
+	[0][1][0][0][RTW89_IC][12] = 18,
+	[0][1][0][0][RTW89_KCC][12] = 56,
 	[0][1][0][0][RTW89_ACMA][12] = 48,
+	[0][1][0][0][RTW89_CN][12] = 42,
+	[0][1][0][0][RTW89_UK][12] = 48,
 	[0][1][0][0][RTW89_FCC][13] = 127,
 	[0][1][0][0][RTW89_ETSI][13] = 127,
 	[0][1][0][0][RTW89_MKK][13] = 60,
 	[0][1][0][0][RTW89_IC][13] = 127,
+	[0][1][0][0][RTW89_KCC][13] = 127,
 	[0][1][0][0][RTW89_ACMA][13] = 127,
+	[0][1][0][0][RTW89_CN][13] = 127,
+	[0][1][0][0][RTW89_UK][13] = 127,
 	[1][0][0][0][RTW89_FCC][0] = 127,
 	[1][0][0][0][RTW89_ETSI][0] = 127,
 	[1][0][0][0][RTW89_MKK][0] = 127,
 	[1][0][0][0][RTW89_IC][0] = 127,
+	[1][0][0][0][RTW89_KCC][0] = 127,
 	[1][0][0][0][RTW89_ACMA][0] = 127,
+	[1][0][0][0][RTW89_CN][0] = 127,
+	[1][0][0][0][RTW89_UK][0] = 127,
 	[1][0][0][0][RTW89_FCC][1] = 127,
 	[1][0][0][0][RTW89_ETSI][1] = 127,
 	[1][0][0][0][RTW89_MKK][1] = 127,
 	[1][0][0][0][RTW89_IC][1] = 127,
+	[1][0][0][0][RTW89_KCC][1] = 127,
 	[1][0][0][0][RTW89_ACMA][1] = 127,
-	[1][0][0][0][RTW89_FCC][2] = 42,
+	[1][0][0][0][RTW89_CN][1] = 127,
+	[1][0][0][0][RTW89_UK][1] = 127,
+	[1][0][0][0][RTW89_FCC][2] = 44,
 	[1][0][0][0][RTW89_ETSI][2] = 60,
 	[1][0][0][0][RTW89_MKK][2] = 66,
-	[1][0][0][0][RTW89_IC][2] = 52,
+	[1][0][0][0][RTW89_IC][2] = 44,
+	[1][0][0][0][RTW89_KCC][2] = 68,
 	[1][0][0][0][RTW89_ACMA][2] = 60,
-	[1][0][0][0][RTW89_FCC][3] = 42,
+	[1][0][0][0][RTW89_CN][2] = 58,
+	[1][0][0][0][RTW89_UK][2] = 60,
+	[1][0][0][0][RTW89_FCC][3] = 60,
 	[1][0][0][0][RTW89_ETSI][3] = 60,
 	[1][0][0][0][RTW89_MKK][3] = 66,
-	[1][0][0][0][RTW89_IC][3] = 52,
+	[1][0][0][0][RTW89_IC][3] = 60,
+	[1][0][0][0][RTW89_KCC][3] = 68,
 	[1][0][0][0][RTW89_ACMA][3] = 60,
-	[1][0][0][0][RTW89_FCC][4] = 42,
+	[1][0][0][0][RTW89_CN][3] = 58,
+	[1][0][0][0][RTW89_UK][3] = 60,
+	[1][0][0][0][RTW89_FCC][4] = 60,
 	[1][0][0][0][RTW89_ETSI][4] = 60,
 	[1][0][0][0][RTW89_MKK][4] = 66,
-	[1][0][0][0][RTW89_IC][4] = 52,
+	[1][0][0][0][RTW89_IC][4] = 60,
+	[1][0][0][0][RTW89_KCC][4] = 68,
 	[1][0][0][0][RTW89_ACMA][4] = 60,
-	[1][0][0][0][RTW89_FCC][5] = 58,
+	[1][0][0][0][RTW89_CN][4] = 58,
+	[1][0][0][0][RTW89_UK][4] = 60,
+	[1][0][0][0][RTW89_FCC][5] = 62,
 	[1][0][0][0][RTW89_ETSI][5] = 60,
 	[1][0][0][0][RTW89_MKK][5] = 66,
-	[1][0][0][0][RTW89_IC][5] = 68,
+	[1][0][0][0][RTW89_IC][5] = 62,
+	[1][0][0][0][RTW89_KCC][5] = 68,
 	[1][0][0][0][RTW89_ACMA][5] = 60,
-	[1][0][0][0][RTW89_FCC][6] = 42,
+	[1][0][0][0][RTW89_CN][5] = 58,
+	[1][0][0][0][RTW89_UK][5] = 60,
+	[1][0][0][0][RTW89_FCC][6] = 46,
 	[1][0][0][0][RTW89_ETSI][6] = 60,
 	[1][0][0][0][RTW89_MKK][6] = 66,
-	[1][0][0][0][RTW89_IC][6] = 52,
+	[1][0][0][0][RTW89_IC][6] = 46,
+	[1][0][0][0][RTW89_KCC][6] = 68,
 	[1][0][0][0][RTW89_ACMA][6] = 60,
-	[1][0][0][0][RTW89_FCC][7] = 42,
+	[1][0][0][0][RTW89_CN][6] = 58,
+	[1][0][0][0][RTW89_UK][6] = 60,
+	[1][0][0][0][RTW89_FCC][7] = 46,
 	[1][0][0][0][RTW89_ETSI][7] = 60,
 	[1][0][0][0][RTW89_MKK][7] = 66,
-	[1][0][0][0][RTW89_IC][7] = 52,
+	[1][0][0][0][RTW89_IC][7] = 46,
+	[1][0][0][0][RTW89_KCC][7] = 68,
 	[1][0][0][0][RTW89_ACMA][7] = 60,
-	[1][0][0][0][RTW89_FCC][8] = 42,
+	[1][0][0][0][RTW89_CN][7] = 58,
+	[1][0][0][0][RTW89_UK][7] = 60,
+	[1][0][0][0][RTW89_FCC][8] = 28,
 	[1][0][0][0][RTW89_ETSI][8] = 60,
 	[1][0][0][0][RTW89_MKK][8] = 66,
-	[1][0][0][0][RTW89_IC][8] = 52,
+	[1][0][0][0][RTW89_IC][8] = 28,
+	[1][0][0][0][RTW89_KCC][8] = 70,
 	[1][0][0][0][RTW89_ACMA][8] = 60,
-	[1][0][0][0][RTW89_FCC][9] = 34,
+	[1][0][0][0][RTW89_CN][8] = 58,
+	[1][0][0][0][RTW89_UK][8] = 60,
+	[1][0][0][0][RTW89_FCC][9] = 26,
 	[1][0][0][0][RTW89_ETSI][9] = 60,
 	[1][0][0][0][RTW89_MKK][9] = 66,
-	[1][0][0][0][RTW89_IC][9] = 44,
+	[1][0][0][0][RTW89_IC][9] = 26,
+	[1][0][0][0][RTW89_KCC][9] = 70,
 	[1][0][0][0][RTW89_ACMA][9] = 60,
-	[1][0][0][0][RTW89_FCC][10] = 22,
+	[1][0][0][0][RTW89_CN][9] = 58,
+	[1][0][0][0][RTW89_UK][9] = 60,
+	[1][0][0][0][RTW89_FCC][10] = 26,
 	[1][0][0][0][RTW89_ETSI][10] = 60,
 	[1][0][0][0][RTW89_MKK][10] = 66,
-	[1][0][0][0][RTW89_IC][10] = 32,
+	[1][0][0][0][RTW89_IC][10] = 26,
+	[1][0][0][0][RTW89_KCC][10] = 70,
 	[1][0][0][0][RTW89_ACMA][10] = 60,
+	[1][0][0][0][RTW89_CN][10] = 58,
+	[1][0][0][0][RTW89_UK][10] = 60,
 	[1][0][0][0][RTW89_FCC][11] = 127,
 	[1][0][0][0][RTW89_ETSI][11] = 127,
 	[1][0][0][0][RTW89_MKK][11] = 127,
 	[1][0][0][0][RTW89_IC][11] = 127,
+	[1][0][0][0][RTW89_KCC][11] = 127,
 	[1][0][0][0][RTW89_ACMA][11] = 127,
+	[1][0][0][0][RTW89_CN][11] = 127,
+	[1][0][0][0][RTW89_UK][11] = 127,
 	[1][0][0][0][RTW89_FCC][12] = 127,
 	[1][0][0][0][RTW89_ETSI][12] = 127,
 	[1][0][0][0][RTW89_MKK][12] = 127,
 	[1][0][0][0][RTW89_IC][12] = 127,
+	[1][0][0][0][RTW89_KCC][12] = 127,
 	[1][0][0][0][RTW89_ACMA][12] = 127,
+	[1][0][0][0][RTW89_CN][12] = 127,
+	[1][0][0][0][RTW89_UK][12] = 127,
 	[1][0][0][0][RTW89_FCC][13] = 127,
 	[1][0][0][0][RTW89_ETSI][13] = 127,
 	[1][0][0][0][RTW89_MKK][13] = 127,
 	[1][0][0][0][RTW89_IC][13] = 127,
+	[1][0][0][0][RTW89_KCC][13] = 127,
 	[1][0][0][0][RTW89_ACMA][13] = 127,
+	[1][0][0][0][RTW89_CN][13] = 127,
+	[1][0][0][0][RTW89_UK][13] = 127,
 	[1][1][0][0][RTW89_FCC][0] = 127,
 	[1][1][0][0][RTW89_ETSI][0] = 127,
 	[1][1][0][0][RTW89_MKK][0] = 127,
 	[1][1][0][0][RTW89_IC][0] = 127,
+	[1][1][0][0][RTW89_KCC][0] = 127,
 	[1][1][0][0][RTW89_ACMA][0] = 127,
+	[1][1][0][0][RTW89_CN][0] = 127,
+	[1][1][0][0][RTW89_UK][0] = 127,
 	[1][1][0][0][RTW89_FCC][1] = 127,
 	[1][1][0][0][RTW89_ETSI][1] = 127,
 	[1][1][0][0][RTW89_MKK][1] = 127,
 	[1][1][0][0][RTW89_IC][1] = 127,
+	[1][1][0][0][RTW89_KCC][1] = 127,
 	[1][1][0][0][RTW89_ACMA][1] = 127,
-	[1][1][0][0][RTW89_FCC][2] = 38,
+	[1][1][0][0][RTW89_CN][1] = 127,
+	[1][1][0][0][RTW89_UK][1] = 127,
+	[1][1][0][0][RTW89_FCC][2] = 46,
 	[1][1][0][0][RTW89_ETSI][2] = 48,
 	[1][1][0][0][RTW89_MKK][2] = 58,
-	[1][1][0][0][RTW89_IC][2] = 48,
+	[1][1][0][0][RTW89_IC][2] = 46,
+	[1][1][0][0][RTW89_KCC][2] = 56,
 	[1][1][0][0][RTW89_ACMA][2] = 48,
-	[1][1][0][0][RTW89_FCC][3] = 38,
+	[1][1][0][0][RTW89_CN][2] = 46,
+	[1][1][0][0][RTW89_UK][2] = 48,
+	[1][1][0][0][RTW89_FCC][3] = 46,
 	[1][1][0][0][RTW89_ETSI][3] = 48,
 	[1][1][0][0][RTW89_MKK][3] = 58,
-	[1][1][0][0][RTW89_IC][3] = 48,
+	[1][1][0][0][RTW89_IC][3] = 46,
+	[1][1][0][0][RTW89_KCC][3] = 56,
 	[1][1][0][0][RTW89_ACMA][3] = 48,
-	[1][1][0][0][RTW89_FCC][4] = 38,
+	[1][1][0][0][RTW89_CN][3] = 46,
+	[1][1][0][0][RTW89_UK][3] = 48,
+	[1][1][0][0][RTW89_FCC][4] = 46,
 	[1][1][0][0][RTW89_ETSI][4] = 48,
 	[1][1][0][0][RTW89_MKK][4] = 58,
-	[1][1][0][0][RTW89_IC][4] = 48,
+	[1][1][0][0][RTW89_IC][4] = 46,
+	[1][1][0][0][RTW89_KCC][4] = 56,
 	[1][1][0][0][RTW89_ACMA][4] = 48,
-	[1][1][0][0][RTW89_FCC][5] = 54,
+	[1][1][0][0][RTW89_CN][4] = 46,
+	[1][1][0][0][RTW89_UK][4] = 48,
+	[1][1][0][0][RTW89_FCC][5] = 48,
 	[1][1][0][0][RTW89_ETSI][5] = 48,
 	[1][1][0][0][RTW89_MKK][5] = 58,
-	[1][1][0][0][RTW89_IC][5] = 64,
+	[1][1][0][0][RTW89_IC][5] = 48,
+	[1][1][0][0][RTW89_KCC][5] = 56,
 	[1][1][0][0][RTW89_ACMA][5] = 48,
-	[1][1][0][0][RTW89_FCC][6] = 26,
+	[1][1][0][0][RTW89_CN][5] = 46,
+	[1][1][0][0][RTW89_UK][5] = 48,
+	[1][1][0][0][RTW89_FCC][6] = 40,
 	[1][1][0][0][RTW89_ETSI][6] = 48,
 	[1][1][0][0][RTW89_MKK][6] = 58,
-	[1][1][0][0][RTW89_IC][6] = 36,
+	[1][1][0][0][RTW89_IC][6] = 40,
+	[1][1][0][0][RTW89_KCC][6] = 56,
 	[1][1][0][0][RTW89_ACMA][6] = 48,
-	[1][1][0][0][RTW89_FCC][7] = 26,
+	[1][1][0][0][RTW89_CN][6] = 46,
+	[1][1][0][0][RTW89_UK][6] = 48,
+	[1][1][0][0][RTW89_FCC][7] = 40,
 	[1][1][0][0][RTW89_ETSI][7] = 48,
 	[1][1][0][0][RTW89_MKK][7] = 58,
-	[1][1][0][0][RTW89_IC][7] = 36,
+	[1][1][0][0][RTW89_IC][7] = 40,
+	[1][1][0][0][RTW89_KCC][7] = 56,
 	[1][1][0][0][RTW89_ACMA][7] = 48,
-	[1][1][0][0][RTW89_FCC][8] = 26,
+	[1][1][0][0][RTW89_CN][7] = 46,
+	[1][1][0][0][RTW89_UK][7] = 48,
+	[1][1][0][0][RTW89_FCC][8] = 14,
 	[1][1][0][0][RTW89_ETSI][8] = 48,
 	[1][1][0][0][RTW89_MKK][8] = 58,
-	[1][1][0][0][RTW89_IC][8] = 36,
+	[1][1][0][0][RTW89_IC][8] = 14,
+	[1][1][0][0][RTW89_KCC][8] = 58,
 	[1][1][0][0][RTW89_ACMA][8] = 48,
-	[1][1][0][0][RTW89_FCC][9] = 22,
+	[1][1][0][0][RTW89_CN][8] = 46,
+	[1][1][0][0][RTW89_UK][8] = 48,
+	[1][1][0][0][RTW89_FCC][9] = 14,
 	[1][1][0][0][RTW89_ETSI][9] = 48,
 	[1][1][0][0][RTW89_MKK][9] = 58,
-	[1][1][0][0][RTW89_IC][9] = 32,
+	[1][1][0][0][RTW89_IC][9] = 14,
+	[1][1][0][0][RTW89_KCC][9] = 58,
 	[1][1][0][0][RTW89_ACMA][9] = 48,
-	[1][1][0][0][RTW89_FCC][10] = 22,
+	[1][1][0][0][RTW89_CN][9] = 46,
+	[1][1][0][0][RTW89_UK][9] = 48,
+	[1][1][0][0][RTW89_FCC][10] = 12,
 	[1][1][0][0][RTW89_ETSI][10] = 48,
 	[1][1][0][0][RTW89_MKK][10] = 56,
-	[1][1][0][0][RTW89_IC][10] = 32,
+	[1][1][0][0][RTW89_IC][10] = 12,
+	[1][1][0][0][RTW89_KCC][10] = 58,
 	[1][1][0][0][RTW89_ACMA][10] = 48,
+	[1][1][0][0][RTW89_CN][10] = 46,
+	[1][1][0][0][RTW89_UK][10] = 48,
 	[1][1][0][0][RTW89_FCC][11] = 127,
 	[1][1][0][0][RTW89_ETSI][11] = 127,
 	[1][1][0][0][RTW89_MKK][11] = 127,
 	[1][1][0][0][RTW89_IC][11] = 127,
+	[1][1][0][0][RTW89_KCC][11] = 127,
 	[1][1][0][0][RTW89_ACMA][11] = 127,
+	[1][1][0][0][RTW89_CN][11] = 127,
+	[1][1][0][0][RTW89_UK][11] = 127,
 	[1][1][0][0][RTW89_FCC][12] = 127,
 	[1][1][0][0][RTW89_ETSI][12] = 127,
 	[1][1][0][0][RTW89_MKK][12] = 127,
 	[1][1][0][0][RTW89_IC][12] = 127,
+	[1][1][0][0][RTW89_KCC][12] = 127,
 	[1][1][0][0][RTW89_ACMA][12] = 127,
+	[1][1][0][0][RTW89_CN][12] = 127,
+	[1][1][0][0][RTW89_UK][12] = 127,
 	[1][1][0][0][RTW89_FCC][13] = 127,
 	[1][1][0][0][RTW89_ETSI][13] = 127,
 	[1][1][0][0][RTW89_MKK][13] = 127,
 	[1][1][0][0][RTW89_IC][13] = 127,
+	[1][1][0][0][RTW89_KCC][13] = 127,
 	[1][1][0][0][RTW89_ACMA][13] = 127,
-	[0][0][1][0][RTW89_FCC][0] = 68,
+	[1][1][0][0][RTW89_CN][13] = 127,
+	[1][1][0][0][RTW89_UK][13] = 127,
+	[0][0][1][0][RTW89_FCC][0] = 66,
 	[0][0][1][0][RTW89_ETSI][0] = 60,
 	[0][0][1][0][RTW89_MKK][0] = 76,
-	[0][0][1][0][RTW89_IC][0] = 78,
+	[0][0][1][0][RTW89_IC][0] = 66,
+	[0][0][1][0][RTW89_KCC][0] = 68,
 	[0][0][1][0][RTW89_ACMA][0] = 60,
+	[0][0][1][0][RTW89_CN][0] = 58,
+	[0][0][1][0][RTW89_UK][0] = 60,
 	[0][0][1][0][RTW89_FCC][1] = 68,
 	[0][0][1][0][RTW89_ETSI][1] = 60,
 	[0][0][1][0][RTW89_MKK][1] = 78,
-	[0][0][1][0][RTW89_IC][1] = 78,
+	[0][0][1][0][RTW89_IC][1] = 68,
+	[0][0][1][0][RTW89_KCC][1] = 68,
 	[0][0][1][0][RTW89_ACMA][1] = 60,
-	[0][0][1][0][RTW89_FCC][2] = 70,
+	[0][0][1][0][RTW89_CN][1] = 58,
+	[0][0][1][0][RTW89_UK][1] = 60,
+	[0][0][1][0][RTW89_FCC][2] = 72,
 	[0][0][1][0][RTW89_ETSI][2] = 60,
 	[0][0][1][0][RTW89_MKK][2] = 78,
-	[0][0][1][0][RTW89_IC][2] = 78,
+	[0][0][1][0][RTW89_IC][2] = 72,
+	[0][0][1][0][RTW89_KCC][2] = 68,
 	[0][0][1][0][RTW89_ACMA][2] = 60,
-	[0][0][1][0][RTW89_FCC][3] = 70,
+	[0][0][1][0][RTW89_CN][2] = 58,
+	[0][0][1][0][RTW89_UK][2] = 60,
+	[0][0][1][0][RTW89_FCC][3] = 76,
 	[0][0][1][0][RTW89_ETSI][3] = 60,
 	[0][0][1][0][RTW89_MKK][3] = 78,
-	[0][0][1][0][RTW89_IC][3] = 78,
+	[0][0][1][0][RTW89_IC][3] = 76,
+	[0][0][1][0][RTW89_KCC][3] = 68,
 	[0][0][1][0][RTW89_ACMA][3] = 60,
-	[0][0][1][0][RTW89_FCC][4] = 70,
+	[0][0][1][0][RTW89_CN][3] = 58,
+	[0][0][1][0][RTW89_UK][3] = 60,
+	[0][0][1][0][RTW89_FCC][4] = 80,
 	[0][0][1][0][RTW89_ETSI][4] = 60,
 	[0][0][1][0][RTW89_MKK][4] = 78,
-	[0][0][1][0][RTW89_IC][4] = 78,
+	[0][0][1][0][RTW89_IC][4] = 80,
+	[0][0][1][0][RTW89_KCC][4] = 76,
 	[0][0][1][0][RTW89_ACMA][4] = 60,
-	[0][0][1][0][RTW89_FCC][5] = 70,
+	[0][0][1][0][RTW89_CN][4] = 58,
+	[0][0][1][0][RTW89_UK][4] = 60,
+	[0][0][1][0][RTW89_FCC][5] = 80,
 	[0][0][1][0][RTW89_ETSI][5] = 60,
 	[0][0][1][0][RTW89_MKK][5] = 78,
-	[0][0][1][0][RTW89_IC][5] = 78,
+	[0][0][1][0][RTW89_IC][5] = 80,
+	[0][0][1][0][RTW89_KCC][5] = 76,
 	[0][0][1][0][RTW89_ACMA][5] = 60,
-	[0][0][1][0][RTW89_FCC][6] = 70,
+	[0][0][1][0][RTW89_CN][5] = 58,
+	[0][0][1][0][RTW89_UK][5] = 60,
+	[0][0][1][0][RTW89_FCC][6] = 80,
 	[0][0][1][0][RTW89_ETSI][6] = 60,
 	[0][0][1][0][RTW89_MKK][6] = 76,
-	[0][0][1][0][RTW89_IC][6] = 78,
+	[0][0][1][0][RTW89_IC][6] = 80,
+	[0][0][1][0][RTW89_KCC][6] = 76,
 	[0][0][1][0][RTW89_ACMA][6] = 60,
-	[0][0][1][0][RTW89_FCC][7] = 70,
+	[0][0][1][0][RTW89_CN][6] = 58,
+	[0][0][1][0][RTW89_UK][6] = 60,
+	[0][0][1][0][RTW89_FCC][7] = 80,
 	[0][0][1][0][RTW89_ETSI][7] = 60,
 	[0][0][1][0][RTW89_MKK][7] = 78,
-	[0][0][1][0][RTW89_IC][7] = 78,
+	[0][0][1][0][RTW89_IC][7] = 80,
+	[0][0][1][0][RTW89_KCC][7] = 76,
 	[0][0][1][0][RTW89_ACMA][7] = 60,
-	[0][0][1][0][RTW89_FCC][8] = 70,
+	[0][0][1][0][RTW89_CN][7] = 58,
+	[0][0][1][0][RTW89_UK][7] = 60,
+	[0][0][1][0][RTW89_FCC][8] = 80,
 	[0][0][1][0][RTW89_ETSI][8] = 60,
 	[0][0][1][0][RTW89_MKK][8] = 78,
-	[0][0][1][0][RTW89_IC][8] = 78,
+	[0][0][1][0][RTW89_IC][8] = 80,
+	[0][0][1][0][RTW89_KCC][8] = 76,
 	[0][0][1][0][RTW89_ACMA][8] = 60,
-	[0][0][1][0][RTW89_FCC][9] = 66,
+	[0][0][1][0][RTW89_CN][8] = 58,
+	[0][0][1][0][RTW89_UK][8] = 60,
+	[0][0][1][0][RTW89_FCC][9] = 76,
 	[0][0][1][0][RTW89_ETSI][9] = 60,
 	[0][0][1][0][RTW89_MKK][9] = 78,
 	[0][0][1][0][RTW89_IC][9] = 76,
+	[0][0][1][0][RTW89_KCC][9] = 70,
 	[0][0][1][0][RTW89_ACMA][9] = 60,
+	[0][0][1][0][RTW89_CN][9] = 58,
+	[0][0][1][0][RTW89_UK][9] = 60,
 	[0][0][1][0][RTW89_FCC][10] = 66,
 	[0][0][1][0][RTW89_ETSI][10] = 60,
 	[0][0][1][0][RTW89_MKK][10] = 78,
-	[0][0][1][0][RTW89_IC][10] = 76,
+	[0][0][1][0][RTW89_IC][10] = 66,
+	[0][0][1][0][RTW89_KCC][10] = 70,
 	[0][0][1][0][RTW89_ACMA][10] = 60,
-	[0][0][1][0][RTW89_FCC][11] = 46,
+	[0][0][1][0][RTW89_CN][10] = 58,
+	[0][0][1][0][RTW89_UK][10] = 60,
+	[0][0][1][0][RTW89_FCC][11] = 62,
 	[0][0][1][0][RTW89_ETSI][11] = 60,
 	[0][0][1][0][RTW89_MKK][11] = 78,
-	[0][0][1][0][RTW89_IC][11] = 56,
+	[0][0][1][0][RTW89_IC][11] = 62,
+	[0][0][1][0][RTW89_KCC][11] = 70,
 	[0][0][1][0][RTW89_ACMA][11] = 60,
-	[0][0][1][0][RTW89_FCC][12] = 42,
+	[0][0][1][0][RTW89_CN][11] = 58,
+	[0][0][1][0][RTW89_UK][11] = 60,
+	[0][0][1][0][RTW89_FCC][12] = 60,
 	[0][0][1][0][RTW89_ETSI][12] = 60,
 	[0][0][1][0][RTW89_MKK][12] = 78,
-	[0][0][1][0][RTW89_IC][12] = 52,
+	[0][0][1][0][RTW89_IC][12] = 60,
+	[0][0][1][0][RTW89_KCC][12] = 70,
 	[0][0][1][0][RTW89_ACMA][12] = 60,
+	[0][0][1][0][RTW89_CN][12] = 58,
+	[0][0][1][0][RTW89_UK][12] = 60,
 	[0][0][1][0][RTW89_FCC][13] = 127,
 	[0][0][1][0][RTW89_ETSI][13] = 127,
 	[0][0][1][0][RTW89_MKK][13] = 127,
 	[0][0][1][0][RTW89_IC][13] = 127,
+	[0][0][1][0][RTW89_KCC][13] = 127,
 	[0][0][1][0][RTW89_ACMA][13] = 127,
-	[0][1][1][0][RTW89_FCC][0] = 54,
+	[0][0][1][0][RTW89_CN][13] = 127,
+	[0][0][1][0][RTW89_UK][13] = 127,
+	[0][1][1][0][RTW89_FCC][0] = 66,
 	[0][1][1][0][RTW89_ETSI][0] = 48,
 	[0][1][1][0][RTW89_MKK][0] = 66,
-	[0][1][1][0][RTW89_IC][0] = 64,
+	[0][1][1][0][RTW89_IC][0] = 66,
+	[0][1][1][0][RTW89_KCC][0] = 64,
 	[0][1][1][0][RTW89_ACMA][0] = 48,
-	[0][1][1][0][RTW89_FCC][1] = 54,
+	[0][1][1][0][RTW89_CN][0] = 46,
+	[0][1][1][0][RTW89_UK][0] = 48,
+	[0][1][1][0][RTW89_FCC][1] = 68,
 	[0][1][1][0][RTW89_ETSI][1] = 48,
 	[0][1][1][0][RTW89_MKK][1] = 66,
-	[0][1][1][0][RTW89_IC][1] = 64,
+	[0][1][1][0][RTW89_IC][1] = 68,
+	[0][1][1][0][RTW89_KCC][1] = 64,
 	[0][1][1][0][RTW89_ACMA][1] = 48,
-	[0][1][1][0][RTW89_FCC][2] = 58,
+	[0][1][1][0][RTW89_CN][1] = 46,
+	[0][1][1][0][RTW89_UK][1] = 48,
+	[0][1][1][0][RTW89_FCC][2] = 72,
 	[0][1][1][0][RTW89_ETSI][2] = 48,
 	[0][1][1][0][RTW89_MKK][2] = 66,
-	[0][1][1][0][RTW89_IC][2] = 68,
+	[0][1][1][0][RTW89_IC][2] = 72,
+	[0][1][1][0][RTW89_KCC][2] = 64,
 	[0][1][1][0][RTW89_ACMA][2] = 48,
-	[0][1][1][0][RTW89_FCC][3] = 62,
+	[0][1][1][0][RTW89_CN][2] = 46,
+	[0][1][1][0][RTW89_UK][2] = 48,
+	[0][1][1][0][RTW89_FCC][3] = 76,
 	[0][1][1][0][RTW89_ETSI][3] = 48,
 	[0][1][1][0][RTW89_MKK][3] = 66,
-	[0][1][1][0][RTW89_IC][3] = 72,
+	[0][1][1][0][RTW89_IC][3] = 76,
+	[0][1][1][0][RTW89_KCC][3] = 64,
 	[0][1][1][0][RTW89_ACMA][3] = 48,
-	[0][1][1][0][RTW89_FCC][4] = 70,
+	[0][1][1][0][RTW89_CN][3] = 46,
+	[0][1][1][0][RTW89_UK][3] = 48,
+	[0][1][1][0][RTW89_FCC][4] = 80,
 	[0][1][1][0][RTW89_ETSI][4] = 48,
 	[0][1][1][0][RTW89_MKK][4] = 66,
-	[0][1][1][0][RTW89_IC][4] = 78,
+	[0][1][1][0][RTW89_IC][4] = 80,
+	[0][1][1][0][RTW89_KCC][4] = 66,
 	[0][1][1][0][RTW89_ACMA][4] = 48,
-	[0][1][1][0][RTW89_FCC][5] = 70,
+	[0][1][1][0][RTW89_CN][4] = 46,
+	[0][1][1][0][RTW89_UK][4] = 48,
+	[0][1][1][0][RTW89_FCC][5] = 80,
 	[0][1][1][0][RTW89_ETSI][5] = 48,
 	[0][1][1][0][RTW89_MKK][5] = 66,
-	[0][1][1][0][RTW89_IC][5] = 78,
+	[0][1][1][0][RTW89_IC][5] = 80,
+	[0][1][1][0][RTW89_KCC][5] = 66,
 	[0][1][1][0][RTW89_ACMA][5] = 48,
-	[0][1][1][0][RTW89_FCC][6] = 70,
+	[0][1][1][0][RTW89_CN][5] = 46,
+	[0][1][1][0][RTW89_UK][5] = 48,
+	[0][1][1][0][RTW89_FCC][6] = 80,
 	[0][1][1][0][RTW89_ETSI][6] = 48,
 	[0][1][1][0][RTW89_MKK][6] = 66,
-	[0][1][1][0][RTW89_IC][6] = 78,
+	[0][1][1][0][RTW89_IC][6] = 80,
+	[0][1][1][0][RTW89_KCC][6] = 66,
 	[0][1][1][0][RTW89_ACMA][6] = 48,
-	[0][1][1][0][RTW89_FCC][7] = 62,
+	[0][1][1][0][RTW89_CN][6] = 46,
+	[0][1][1][0][RTW89_UK][6] = 48,
+	[0][1][1][0][RTW89_FCC][7] = 78,
 	[0][1][1][0][RTW89_ETSI][7] = 48,
 	[0][1][1][0][RTW89_MKK][7] = 66,
-	[0][1][1][0][RTW89_IC][7] = 72,
+	[0][1][1][0][RTW89_IC][7] = 78,
+	[0][1][1][0][RTW89_KCC][7] = 66,
 	[0][1][1][0][RTW89_ACMA][7] = 48,
-	[0][1][1][0][RTW89_FCC][8] = 58,
+	[0][1][1][0][RTW89_CN][7] = 46,
+	[0][1][1][0][RTW89_UK][7] = 48,
+	[0][1][1][0][RTW89_FCC][8] = 74,
 	[0][1][1][0][RTW89_ETSI][8] = 48,
 	[0][1][1][0][RTW89_MKK][8] = 66,
-	[0][1][1][0][RTW89_IC][8] = 68,
+	[0][1][1][0][RTW89_IC][8] = 74,
+	[0][1][1][0][RTW89_KCC][8] = 66,
 	[0][1][1][0][RTW89_ACMA][8] = 48,
-	[0][1][1][0][RTW89_FCC][9] = 54,
+	[0][1][1][0][RTW89_CN][8] = 46,
+	[0][1][1][0][RTW89_UK][8] = 48,
+	[0][1][1][0][RTW89_FCC][9] = 70,
 	[0][1][1][0][RTW89_ETSI][9] = 48,
 	[0][1][1][0][RTW89_MKK][9] = 66,
-	[0][1][1][0][RTW89_IC][9] = 64,
+	[0][1][1][0][RTW89_IC][9] = 70,
+	[0][1][1][0][RTW89_KCC][9] = 64,
 	[0][1][1][0][RTW89_ACMA][9] = 48,
-	[0][1][1][0][RTW89_FCC][10] = 54,
+	[0][1][1][0][RTW89_CN][9] = 46,
+	[0][1][1][0][RTW89_UK][9] = 48,
+	[0][1][1][0][RTW89_FCC][10] = 62,
 	[0][1][1][0][RTW89_ETSI][10] = 48,
 	[0][1][1][0][RTW89_MKK][10] = 66,
-	[0][1][1][0][RTW89_IC][10] = 64,
+	[0][1][1][0][RTW89_IC][10] = 62,
+	[0][1][1][0][RTW89_KCC][10] = 64,
 	[0][1][1][0][RTW89_ACMA][10] = 48,
-	[0][1][1][0][RTW89_FCC][11] = 38,
+	[0][1][1][0][RTW89_CN][10] = 46,
+	[0][1][1][0][RTW89_UK][10] = 48,
+	[0][1][1][0][RTW89_FCC][11] = 60,
 	[0][1][1][0][RTW89_ETSI][11] = 48,
 	[0][1][1][0][RTW89_MKK][11] = 66,
-	[0][1][1][0][RTW89_IC][11] = 48,
+	[0][1][1][0][RTW89_IC][11] = 60,
+	[0][1][1][0][RTW89_KCC][11] = 64,
 	[0][1][1][0][RTW89_ACMA][11] = 48,
-	[0][1][1][0][RTW89_FCC][12] = 34,
+	[0][1][1][0][RTW89_CN][11] = 46,
+	[0][1][1][0][RTW89_UK][11] = 48,
+	[0][1][1][0][RTW89_FCC][12] = 36,
 	[0][1][1][0][RTW89_ETSI][12] = 48,
 	[0][1][1][0][RTW89_MKK][12] = 66,
-	[0][1][1][0][RTW89_IC][12] = 44,
+	[0][1][1][0][RTW89_IC][12] = 36,
+	[0][1][1][0][RTW89_KCC][12] = 64,
 	[0][1][1][0][RTW89_ACMA][12] = 48,
+	[0][1][1][0][RTW89_CN][12] = 46,
+	[0][1][1][0][RTW89_UK][12] = 48,
 	[0][1][1][0][RTW89_FCC][13] = 127,
 	[0][1][1][0][RTW89_ETSI][13] = 127,
 	[0][1][1][0][RTW89_MKK][13] = 127,
 	[0][1][1][0][RTW89_IC][13] = 127,
+	[0][1][1][0][RTW89_KCC][13] = 127,
 	[0][1][1][0][RTW89_ACMA][13] = 127,
-	[0][0][2][0][RTW89_FCC][0] = 68,
+	[0][1][1][0][RTW89_CN][13] = 127,
+	[0][1][1][0][RTW89_UK][13] = 127,
+	[0][0][2][0][RTW89_FCC][0] = 66,
 	[0][0][2][0][RTW89_ETSI][0] = 60,
 	[0][0][2][0][RTW89_MKK][0] = 78,
-	[0][0][2][0][RTW89_IC][0] = 78,
+	[0][0][2][0][RTW89_IC][0] = 66,
+	[0][0][2][0][RTW89_KCC][0] = 70,
 	[0][0][2][0][RTW89_ACMA][0] = 60,
-	[0][0][2][0][RTW89_FCC][1] = 68,
+	[0][0][2][0][RTW89_CN][0] = 58,
+	[0][0][2][0][RTW89_UK][0] = 60,
+	[0][0][2][0][RTW89_FCC][1] = 70,
 	[0][0][2][0][RTW89_ETSI][1] = 60,
 	[0][0][2][0][RTW89_MKK][1] = 78,
-	[0][0][2][0][RTW89_IC][1] = 78,
+	[0][0][2][0][RTW89_IC][1] = 70,
+	[0][0][2][0][RTW89_KCC][1] = 70,
 	[0][0][2][0][RTW89_ACMA][1] = 60,
-	[0][0][2][0][RTW89_FCC][2] = 70,
+	[0][0][2][0][RTW89_CN][1] = 58,
+	[0][0][2][0][RTW89_UK][1] = 60,
+	[0][0][2][0][RTW89_FCC][2] = 74,
 	[0][0][2][0][RTW89_ETSI][2] = 60,
 	[0][0][2][0][RTW89_MKK][2] = 78,
-	[0][0][2][0][RTW89_IC][2] = 78,
+	[0][0][2][0][RTW89_IC][2] = 74,
+	[0][0][2][0][RTW89_KCC][2] = 70,
 	[0][0][2][0][RTW89_ACMA][2] = 60,
-	[0][0][2][0][RTW89_FCC][3] = 70,
+	[0][0][2][0][RTW89_CN][2] = 58,
+	[0][0][2][0][RTW89_UK][2] = 60,
+	[0][0][2][0][RTW89_FCC][3] = 78,
 	[0][0][2][0][RTW89_ETSI][3] = 60,
 	[0][0][2][0][RTW89_MKK][3] = 78,
 	[0][0][2][0][RTW89_IC][3] = 78,
+	[0][0][2][0][RTW89_KCC][3] = 70,
 	[0][0][2][0][RTW89_ACMA][3] = 60,
-	[0][0][2][0][RTW89_FCC][4] = 70,
+	[0][0][2][0][RTW89_CN][3] = 58,
+	[0][0][2][0][RTW89_UK][3] = 60,
+	[0][0][2][0][RTW89_FCC][4] = 80,
 	[0][0][2][0][RTW89_ETSI][4] = 60,
 	[0][0][2][0][RTW89_MKK][4] = 78,
-	[0][0][2][0][RTW89_IC][4] = 78,
+	[0][0][2][0][RTW89_IC][4] = 80,
+	[0][0][2][0][RTW89_KCC][4] = 78,
 	[0][0][2][0][RTW89_ACMA][4] = 60,
-	[0][0][2][0][RTW89_FCC][5] = 70,
+	[0][0][2][0][RTW89_CN][4] = 58,
+	[0][0][2][0][RTW89_UK][4] = 60,
+	[0][0][2][0][RTW89_FCC][5] = 80,
 	[0][0][2][0][RTW89_ETSI][5] = 60,
 	[0][0][2][0][RTW89_MKK][5] = 78,
-	[0][0][2][0][RTW89_IC][5] = 78,
+	[0][0][2][0][RTW89_IC][5] = 80,
+	[0][0][2][0][RTW89_KCC][5] = 78,
 	[0][0][2][0][RTW89_ACMA][5] = 60,
-	[0][0][2][0][RTW89_FCC][6] = 70,
+	[0][0][2][0][RTW89_CN][5] = 58,
+	[0][0][2][0][RTW89_UK][5] = 60,
+	[0][0][2][0][RTW89_FCC][6] = 80,
 	[0][0][2][0][RTW89_ETSI][6] = 60,
 	[0][0][2][0][RTW89_MKK][6] = 78,
-	[0][0][2][0][RTW89_IC][6] = 78,
+	[0][0][2][0][RTW89_IC][6] = 80,
+	[0][0][2][0][RTW89_KCC][6] = 78,
 	[0][0][2][0][RTW89_ACMA][6] = 60,
-	[0][0][2][0][RTW89_FCC][7] = 70,
+	[0][0][2][0][RTW89_CN][6] = 58,
+	[0][0][2][0][RTW89_UK][6] = 60,
+	[0][0][2][0][RTW89_FCC][7] = 80,
 	[0][0][2][0][RTW89_ETSI][7] = 60,
 	[0][0][2][0][RTW89_MKK][7] = 78,
-	[0][0][2][0][RTW89_IC][7] = 78,
+	[0][0][2][0][RTW89_IC][7] = 80,
+	[0][0][2][0][RTW89_KCC][7] = 78,
 	[0][0][2][0][RTW89_ACMA][7] = 60,
-	[0][0][2][0][RTW89_FCC][8] = 68,
+	[0][0][2][0][RTW89_CN][7] = 58,
+	[0][0][2][0][RTW89_UK][7] = 60,
+	[0][0][2][0][RTW89_FCC][8] = 78,
 	[0][0][2][0][RTW89_ETSI][8] = 60,
 	[0][0][2][0][RTW89_MKK][8] = 78,
 	[0][0][2][0][RTW89_IC][8] = 78,
+	[0][0][2][0][RTW89_KCC][8] = 78,
 	[0][0][2][0][RTW89_ACMA][8] = 60,
-	[0][0][2][0][RTW89_FCC][9] = 64,
+	[0][0][2][0][RTW89_CN][8] = 58,
+	[0][0][2][0][RTW89_UK][8] = 60,
+	[0][0][2][0][RTW89_FCC][9] = 74,
 	[0][0][2][0][RTW89_ETSI][9] = 60,
 	[0][0][2][0][RTW89_MKK][9] = 78,
 	[0][0][2][0][RTW89_IC][9] = 74,
+	[0][0][2][0][RTW89_KCC][9] = 66,
 	[0][0][2][0][RTW89_ACMA][9] = 60,
-	[0][0][2][0][RTW89_FCC][10] = 64,
+	[0][0][2][0][RTW89_CN][9] = 58,
+	[0][0][2][0][RTW89_UK][9] = 60,
+	[0][0][2][0][RTW89_FCC][10] = 62,
 	[0][0][2][0][RTW89_ETSI][10] = 60,
 	[0][0][2][0][RTW89_MKK][10] = 78,
-	[0][0][2][0][RTW89_IC][10] = 74,
+	[0][0][2][0][RTW89_IC][10] = 62,
+	[0][0][2][0][RTW89_KCC][10] = 66,
 	[0][0][2][0][RTW89_ACMA][10] = 60,
-	[0][0][2][0][RTW89_FCC][11] = 46,
+	[0][0][2][0][RTW89_CN][10] = 58,
+	[0][0][2][0][RTW89_UK][10] = 60,
+	[0][0][2][0][RTW89_FCC][11] = 60,
 	[0][0][2][0][RTW89_ETSI][11] = 60,
 	[0][0][2][0][RTW89_MKK][11] = 78,
-	[0][0][2][0][RTW89_IC][11] = 56,
+	[0][0][2][0][RTW89_IC][11] = 60,
+	[0][0][2][0][RTW89_KCC][11] = 66,
 	[0][0][2][0][RTW89_ACMA][11] = 60,
-	[0][0][2][0][RTW89_FCC][12] = 42,
+	[0][0][2][0][RTW89_CN][11] = 58,
+	[0][0][2][0][RTW89_UK][11] = 60,
+	[0][0][2][0][RTW89_FCC][12] = 38,
 	[0][0][2][0][RTW89_ETSI][12] = 60,
 	[0][0][2][0][RTW89_MKK][12] = 78,
-	[0][0][2][0][RTW89_IC][12] = 52,
+	[0][0][2][0][RTW89_IC][12] = 38,
+	[0][0][2][0][RTW89_KCC][12] = 66,
 	[0][0][2][0][RTW89_ACMA][12] = 60,
+	[0][0][2][0][RTW89_CN][12] = 58,
+	[0][0][2][0][RTW89_UK][12] = 60,
 	[0][0][2][0][RTW89_FCC][13] = 127,
 	[0][0][2][0][RTW89_ETSI][13] = 127,
 	[0][0][2][0][RTW89_MKK][13] = 127,
 	[0][0][2][0][RTW89_IC][13] = 127,
+	[0][0][2][0][RTW89_KCC][13] = 127,
 	[0][0][2][0][RTW89_ACMA][13] = 127,
-	[0][1][2][0][RTW89_FCC][0] = 50,
+	[0][0][2][0][RTW89_CN][13] = 127,
+	[0][0][2][0][RTW89_UK][13] = 127,
+	[0][1][2][0][RTW89_FCC][0] = 64,
 	[0][1][2][0][RTW89_ETSI][0] = 48,
 	[0][1][2][0][RTW89_MKK][0] = 68,
-	[0][1][2][0][RTW89_IC][0] = 60,
+	[0][1][2][0][RTW89_IC][0] = 64,
+	[0][1][2][0][RTW89_KCC][0] = 66,
 	[0][1][2][0][RTW89_ACMA][0] = 48,
-	[0][1][2][0][RTW89_FCC][1] = 50,
+	[0][1][2][0][RTW89_CN][0] = 46,
+	[0][1][2][0][RTW89_UK][0] = 48,
+	[0][1][2][0][RTW89_FCC][1] = 70,
 	[0][1][2][0][RTW89_ETSI][1] = 48,
 	[0][1][2][0][RTW89_MKK][1] = 68,
-	[0][1][2][0][RTW89_IC][1] = 60,
+	[0][1][2][0][RTW89_IC][1] = 70,
+	[0][1][2][0][RTW89_KCC][1] = 66,
 	[0][1][2][0][RTW89_ACMA][1] = 48,
-	[0][1][2][0][RTW89_FCC][2] = 54,
+	[0][1][2][0][RTW89_CN][1] = 46,
+	[0][1][2][0][RTW89_UK][1] = 48,
+	[0][1][2][0][RTW89_FCC][2] = 74,
 	[0][1][2][0][RTW89_ETSI][2] = 48,
 	[0][1][2][0][RTW89_MKK][2] = 68,
-	[0][1][2][0][RTW89_IC][2] = 64,
+	[0][1][2][0][RTW89_IC][2] = 74,
+	[0][1][2][0][RTW89_KCC][2] = 66,
 	[0][1][2][0][RTW89_ACMA][2] = 48,
-	[0][1][2][0][RTW89_FCC][3] = 58,
+	[0][1][2][0][RTW89_CN][2] = 46,
+	[0][1][2][0][RTW89_UK][2] = 48,
+	[0][1][2][0][RTW89_FCC][3] = 78,
 	[0][1][2][0][RTW89_ETSI][3] = 48,
 	[0][1][2][0][RTW89_MKK][3] = 68,
-	[0][1][2][0][RTW89_IC][3] = 68,
+	[0][1][2][0][RTW89_IC][3] = 78,
+	[0][1][2][0][RTW89_KCC][3] = 66,
 	[0][1][2][0][RTW89_ACMA][3] = 48,
-	[0][1][2][0][RTW89_FCC][4] = 64,
+	[0][1][2][0][RTW89_CN][3] = 46,
+	[0][1][2][0][RTW89_UK][3] = 48,
+	[0][1][2][0][RTW89_FCC][4] = 80,
 	[0][1][2][0][RTW89_ETSI][4] = 48,
 	[0][1][2][0][RTW89_MKK][4] = 68,
-	[0][1][2][0][RTW89_IC][4] = 74,
+	[0][1][2][0][RTW89_IC][4] = 80,
+	[0][1][2][0][RTW89_KCC][4] = 66,
 	[0][1][2][0][RTW89_ACMA][4] = 48,
-	[0][1][2][0][RTW89_FCC][5] = 70,
+	[0][1][2][0][RTW89_CN][4] = 46,
+	[0][1][2][0][RTW89_UK][4] = 48,
+	[0][1][2][0][RTW89_FCC][5] = 80,
 	[0][1][2][0][RTW89_ETSI][5] = 48,
 	[0][1][2][0][RTW89_MKK][5] = 68,
-	[0][1][2][0][RTW89_IC][5] = 78,
+	[0][1][2][0][RTW89_IC][5] = 80,
+	[0][1][2][0][RTW89_KCC][5] = 66,
 	[0][1][2][0][RTW89_ACMA][5] = 48,
-	[0][1][2][0][RTW89_FCC][6] = 66,
+	[0][1][2][0][RTW89_CN][5] = 46,
+	[0][1][2][0][RTW89_UK][5] = 48,
+	[0][1][2][0][RTW89_FCC][6] = 80,
 	[0][1][2][0][RTW89_ETSI][6] = 48,
 	[0][1][2][0][RTW89_MKK][6] = 68,
-	[0][1][2][0][RTW89_IC][6] = 76,
+	[0][1][2][0][RTW89_IC][6] = 80,
+	[0][1][2][0][RTW89_KCC][6] = 66,
 	[0][1][2][0][RTW89_ACMA][6] = 48,
-	[0][1][2][0][RTW89_FCC][7] = 58,
+	[0][1][2][0][RTW89_CN][6] = 46,
+	[0][1][2][0][RTW89_UK][6] = 48,
+	[0][1][2][0][RTW89_FCC][7] = 74,
 	[0][1][2][0][RTW89_ETSI][7] = 48,
 	[0][1][2][0][RTW89_MKK][7] = 68,
-	[0][1][2][0][RTW89_IC][7] = 68,
+	[0][1][2][0][RTW89_IC][7] = 74,
+	[0][1][2][0][RTW89_KCC][7] = 66,
 	[0][1][2][0][RTW89_ACMA][7] = 48,
-	[0][1][2][0][RTW89_FCC][8] = 54,
+	[0][1][2][0][RTW89_CN][7] = 46,
+	[0][1][2][0][RTW89_UK][7] = 48,
+	[0][1][2][0][RTW89_FCC][8] = 70,
 	[0][1][2][0][RTW89_ETSI][8] = 48,
 	[0][1][2][0][RTW89_MKK][8] = 68,
-	[0][1][2][0][RTW89_IC][8] = 64,
+	[0][1][2][0][RTW89_IC][8] = 70,
+	[0][1][2][0][RTW89_KCC][8] = 66,
 	[0][1][2][0][RTW89_ACMA][8] = 48,
-	[0][1][2][0][RTW89_FCC][9] = 50,
+	[0][1][2][0][RTW89_CN][8] = 46,
+	[0][1][2][0][RTW89_UK][8] = 48,
+	[0][1][2][0][RTW89_FCC][9] = 66,
 	[0][1][2][0][RTW89_ETSI][9] = 48,
 	[0][1][2][0][RTW89_MKK][9] = 68,
-	[0][1][2][0][RTW89_IC][9] = 60,
+	[0][1][2][0][RTW89_IC][9] = 66,
+	[0][1][2][0][RTW89_KCC][9] = 64,
 	[0][1][2][0][RTW89_ACMA][9] = 48,
-	[0][1][2][0][RTW89_FCC][10] = 50,
+	[0][1][2][0][RTW89_CN][9] = 46,
+	[0][1][2][0][RTW89_UK][9] = 48,
+	[0][1][2][0][RTW89_FCC][10] = 58,
 	[0][1][2][0][RTW89_ETSI][10] = 48,
 	[0][1][2][0][RTW89_MKK][10] = 68,
-	[0][1][2][0][RTW89_IC][10] = 60,
+	[0][1][2][0][RTW89_IC][10] = 58,
+	[0][1][2][0][RTW89_KCC][10] = 64,
 	[0][1][2][0][RTW89_ACMA][10] = 48,
-	[0][1][2][0][RTW89_FCC][11] = 38,
+	[0][1][2][0][RTW89_CN][10] = 46,
+	[0][1][2][0][RTW89_UK][10] = 48,
+	[0][1][2][0][RTW89_FCC][11] = 58,
 	[0][1][2][0][RTW89_ETSI][11] = 48,
 	[0][1][2][0][RTW89_MKK][11] = 68,
-	[0][1][2][0][RTW89_IC][11] = 48,
+	[0][1][2][0][RTW89_IC][11] = 58,
+	[0][1][2][0][RTW89_KCC][11] = 64,
 	[0][1][2][0][RTW89_ACMA][11] = 48,
-	[0][1][2][0][RTW89_FCC][12] = 34,
+	[0][1][2][0][RTW89_CN][11] = 46,
+	[0][1][2][0][RTW89_UK][11] = 48,
+	[0][1][2][0][RTW89_FCC][12] = 16,
 	[0][1][2][0][RTW89_ETSI][12] = 48,
 	[0][1][2][0][RTW89_MKK][12] = 68,
-	[0][1][2][0][RTW89_IC][12] = 44,
+	[0][1][2][0][RTW89_IC][12] = 16,
+	[0][1][2][0][RTW89_KCC][12] = 64,
 	[0][1][2][0][RTW89_ACMA][12] = 48,
+	[0][1][2][0][RTW89_CN][12] = 46,
+	[0][1][2][0][RTW89_UK][12] = 48,
 	[0][1][2][0][RTW89_FCC][13] = 127,
 	[0][1][2][0][RTW89_ETSI][13] = 127,
 	[0][1][2][0][RTW89_MKK][13] = 127,
 	[0][1][2][0][RTW89_IC][13] = 127,
+	[0][1][2][0][RTW89_KCC][13] = 127,
 	[0][1][2][0][RTW89_ACMA][13] = 127,
-	[0][1][2][1][RTW89_FCC][0] = 50,
+	[0][1][2][0][RTW89_CN][13] = 127,
+	[0][1][2][0][RTW89_UK][13] = 127,
+	[0][1][2][1][RTW89_FCC][0] = 64,
 	[0][1][2][1][RTW89_ETSI][0] = 36,
 	[0][1][2][1][RTW89_MKK][0] = 68,
-	[0][1][2][1][RTW89_IC][0] = 60,
+	[0][1][2][1][RTW89_IC][0] = 64,
+	[0][1][2][1][RTW89_KCC][0] = 66,
 	[0][1][2][1][RTW89_ACMA][0] = 36,
-	[0][1][2][1][RTW89_FCC][1] = 50,
+	[0][1][2][1][RTW89_CN][0] = 36,
+	[0][1][2][1][RTW89_UK][0] = 36,
+	[0][1][2][1][RTW89_FCC][1] = 70,
 	[0][1][2][1][RTW89_ETSI][1] = 36,
 	[0][1][2][1][RTW89_MKK][1] = 68,
-	[0][1][2][1][RTW89_IC][1] = 60,
+	[0][1][2][1][RTW89_IC][1] = 70,
+	[0][1][2][1][RTW89_KCC][1] = 66,
 	[0][1][2][1][RTW89_ACMA][1] = 36,
-	[0][1][2][1][RTW89_FCC][2] = 54,
+	[0][1][2][1][RTW89_CN][1] = 34,
+	[0][1][2][1][RTW89_UK][1] = 36,
+	[0][1][2][1][RTW89_FCC][2] = 74,
 	[0][1][2][1][RTW89_ETSI][2] = 36,
 	[0][1][2][1][RTW89_MKK][2] = 68,
-	[0][1][2][1][RTW89_IC][2] = 64,
+	[0][1][2][1][RTW89_IC][2] = 74,
+	[0][1][2][1][RTW89_KCC][2] = 66,
 	[0][1][2][1][RTW89_ACMA][2] = 36,
-	[0][1][2][1][RTW89_FCC][3] = 58,
+	[0][1][2][1][RTW89_CN][2] = 34,
+	[0][1][2][1][RTW89_UK][2] = 36,
+	[0][1][2][1][RTW89_FCC][3] = 78,
 	[0][1][2][1][RTW89_ETSI][3] = 36,
 	[0][1][2][1][RTW89_MKK][3] = 68,
-	[0][1][2][1][RTW89_IC][3] = 68,
+	[0][1][2][1][RTW89_IC][3] = 78,
+	[0][1][2][1][RTW89_KCC][3] = 66,
 	[0][1][2][1][RTW89_ACMA][3] = 36,
-	[0][1][2][1][RTW89_FCC][4] = 64,
+	[0][1][2][1][RTW89_CN][3] = 34,
+	[0][1][2][1][RTW89_UK][3] = 36,
+	[0][1][2][1][RTW89_FCC][4] = 80,
 	[0][1][2][1][RTW89_ETSI][4] = 36,
 	[0][1][2][1][RTW89_MKK][4] = 68,
-	[0][1][2][1][RTW89_IC][4] = 74,
+	[0][1][2][1][RTW89_IC][4] = 80,
+	[0][1][2][1][RTW89_KCC][4] = 66,
 	[0][1][2][1][RTW89_ACMA][4] = 36,
-	[0][1][2][1][RTW89_FCC][5] = 70,
+	[0][1][2][1][RTW89_CN][4] = 34,
+	[0][1][2][1][RTW89_UK][4] = 36,
+	[0][1][2][1][RTW89_FCC][5] = 80,
 	[0][1][2][1][RTW89_ETSI][5] = 36,
 	[0][1][2][1][RTW89_MKK][5] = 68,
-	[0][1][2][1][RTW89_IC][5] = 78,
+	[0][1][2][1][RTW89_IC][5] = 80,
+	[0][1][2][1][RTW89_KCC][5] = 66,
 	[0][1][2][1][RTW89_ACMA][5] = 36,
-	[0][1][2][1][RTW89_FCC][6] = 66,
+	[0][1][2][1][RTW89_CN][5] = 34,
+	[0][1][2][1][RTW89_UK][5] = 36,
+	[0][1][2][1][RTW89_FCC][6] = 80,
 	[0][1][2][1][RTW89_ETSI][6] = 36,
 	[0][1][2][1][RTW89_MKK][6] = 68,
-	[0][1][2][1][RTW89_IC][6] = 76,
+	[0][1][2][1][RTW89_IC][6] = 80,
+	[0][1][2][1][RTW89_KCC][6] = 66,
 	[0][1][2][1][RTW89_ACMA][6] = 36,
-	[0][1][2][1][RTW89_FCC][7] = 58,
+	[0][1][2][1][RTW89_CN][6] = 34,
+	[0][1][2][1][RTW89_UK][6] = 36,
+	[0][1][2][1][RTW89_FCC][7] = 74,
 	[0][1][2][1][RTW89_ETSI][7] = 36,
 	[0][1][2][1][RTW89_MKK][7] = 68,
-	[0][1][2][1][RTW89_IC][7] = 68,
+	[0][1][2][1][RTW89_IC][7] = 74,
+	[0][1][2][1][RTW89_KCC][7] = 66,
 	[0][1][2][1][RTW89_ACMA][7] = 36,
-	[0][1][2][1][RTW89_FCC][8] = 54,
+	[0][1][2][1][RTW89_CN][7] = 34,
+	[0][1][2][1][RTW89_UK][7] = 36,
+	[0][1][2][1][RTW89_FCC][8] = 70,
 	[0][1][2][1][RTW89_ETSI][8] = 36,
 	[0][1][2][1][RTW89_MKK][8] = 68,
-	[0][1][2][1][RTW89_IC][8] = 64,
+	[0][1][2][1][RTW89_IC][8] = 70,
+	[0][1][2][1][RTW89_KCC][8] = 66,
 	[0][1][2][1][RTW89_ACMA][8] = 36,
-	[0][1][2][1][RTW89_FCC][9] = 50,
+	[0][1][2][1][RTW89_CN][8] = 34,
+	[0][1][2][1][RTW89_UK][8] = 36,
+	[0][1][2][1][RTW89_FCC][9] = 66,
 	[0][1][2][1][RTW89_ETSI][9] = 36,
 	[0][1][2][1][RTW89_MKK][9] = 68,
-	[0][1][2][1][RTW89_IC][9] = 60,
+	[0][1][2][1][RTW89_IC][9] = 66,
+	[0][1][2][1][RTW89_KCC][9] = 64,
 	[0][1][2][1][RTW89_ACMA][9] = 36,
-	[0][1][2][1][RTW89_FCC][10] = 50,
+	[0][1][2][1][RTW89_CN][9] = 34,
+	[0][1][2][1][RTW89_UK][9] = 36,
+	[0][1][2][1][RTW89_FCC][10] = 58,
 	[0][1][2][1][RTW89_ETSI][10] = 36,
 	[0][1][2][1][RTW89_MKK][10] = 68,
-	[0][1][2][1][RTW89_IC][10] = 60,
+	[0][1][2][1][RTW89_IC][10] = 58,
+	[0][1][2][1][RTW89_KCC][10] = 64,
 	[0][1][2][1][RTW89_ACMA][10] = 36,
-	[0][1][2][1][RTW89_FCC][11] = 38,
+	[0][1][2][1][RTW89_CN][10] = 34,
+	[0][1][2][1][RTW89_UK][10] = 36,
+	[0][1][2][1][RTW89_FCC][11] = 58,
 	[0][1][2][1][RTW89_ETSI][11] = 36,
 	[0][1][2][1][RTW89_MKK][11] = 68,
-	[0][1][2][1][RTW89_IC][11] = 48,
+	[0][1][2][1][RTW89_IC][11] = 58,
+	[0][1][2][1][RTW89_KCC][11] = 64,
 	[0][1][2][1][RTW89_ACMA][11] = 36,
-	[0][1][2][1][RTW89_FCC][12] = 34,
+	[0][1][2][1][RTW89_CN][11] = 34,
+	[0][1][2][1][RTW89_UK][11] = 36,
+	[0][1][2][1][RTW89_FCC][12] = 16,
 	[0][1][2][1][RTW89_ETSI][12] = 36,
 	[0][1][2][1][RTW89_MKK][12] = 68,
-	[0][1][2][1][RTW89_IC][12] = 44,
+	[0][1][2][1][RTW89_IC][12] = 16,
+	[0][1][2][1][RTW89_KCC][12] = 64,
 	[0][1][2][1][RTW89_ACMA][12] = 36,
+	[0][1][2][1][RTW89_CN][12] = 34,
+	[0][1][2][1][RTW89_UK][12] = 36,
 	[0][1][2][1][RTW89_FCC][13] = 127,
 	[0][1][2][1][RTW89_ETSI][13] = 127,
 	[0][1][2][1][RTW89_MKK][13] = 127,
 	[0][1][2][1][RTW89_IC][13] = 127,
+	[0][1][2][1][RTW89_KCC][13] = 127,
 	[0][1][2][1][RTW89_ACMA][13] = 127,
+	[0][1][2][1][RTW89_CN][13] = 127,
+	[0][1][2][1][RTW89_UK][13] = 127,
 	[1][0][2][0][RTW89_FCC][0] = 127,
 	[1][0][2][0][RTW89_ETSI][0] = 127,
 	[1][0][2][0][RTW89_MKK][0] = 127,
 	[1][0][2][0][RTW89_IC][0] = 127,
+	[1][0][2][0][RTW89_KCC][0] = 127,
 	[1][0][2][0][RTW89_ACMA][0] = 127,
+	[1][0][2][0][RTW89_CN][0] = 127,
+	[1][0][2][0][RTW89_UK][0] = 127,
 	[1][0][2][0][RTW89_FCC][1] = 127,
 	[1][0][2][0][RTW89_ETSI][1] = 127,
 	[1][0][2][0][RTW89_MKK][1] = 127,
 	[1][0][2][0][RTW89_IC][1] = 127,
+	[1][0][2][0][RTW89_KCC][1] = 127,
 	[1][0][2][0][RTW89_ACMA][1] = 127,
-	[1][0][2][0][RTW89_FCC][2] = 62,
+	[1][0][2][0][RTW89_CN][1] = 127,
+	[1][0][2][0][RTW89_UK][1] = 127,
+	[1][0][2][0][RTW89_FCC][2] = 64,
 	[1][0][2][0][RTW89_ETSI][2] = 60,
 	[1][0][2][0][RTW89_MKK][2] = 74,
-	[1][0][2][0][RTW89_IC][2] = 72,
+	[1][0][2][0][RTW89_IC][2] = 64,
+	[1][0][2][0][RTW89_KCC][2] = 68,
 	[1][0][2][0][RTW89_ACMA][2] = 60,
-	[1][0][2][0][RTW89_FCC][3] = 62,
+	[1][0][2][0][RTW89_CN][2] = 58,
+	[1][0][2][0][RTW89_UK][2] = 60,
+	[1][0][2][0][RTW89_FCC][3] = 64,
 	[1][0][2][0][RTW89_ETSI][3] = 60,
 	[1][0][2][0][RTW89_MKK][3] = 74,
-	[1][0][2][0][RTW89_IC][3] = 72,
+	[1][0][2][0][RTW89_IC][3] = 64,
+	[1][0][2][0][RTW89_KCC][3] = 68,
 	[1][0][2][0][RTW89_ACMA][3] = 60,
-	[1][0][2][0][RTW89_FCC][4] = 64,
+	[1][0][2][0][RTW89_CN][3] = 58,
+	[1][0][2][0][RTW89_UK][3] = 60,
+	[1][0][2][0][RTW89_FCC][4] = 68,
 	[1][0][2][0][RTW89_ETSI][4] = 60,
 	[1][0][2][0][RTW89_MKK][4] = 74,
-	[1][0][2][0][RTW89_IC][4] = 74,
+	[1][0][2][0][RTW89_IC][4] = 68,
+	[1][0][2][0][RTW89_KCC][4] = 68,
 	[1][0][2][0][RTW89_ACMA][4] = 60,
-	[1][0][2][0][RTW89_FCC][5] = 64,
+	[1][0][2][0][RTW89_CN][4] = 58,
+	[1][0][2][0][RTW89_UK][4] = 60,
+	[1][0][2][0][RTW89_FCC][5] = 68,
 	[1][0][2][0][RTW89_ETSI][5] = 60,
 	[1][0][2][0][RTW89_MKK][5] = 74,
-	[1][0][2][0][RTW89_IC][5] = 74,
+	[1][0][2][0][RTW89_IC][5] = 68,
+	[1][0][2][0][RTW89_KCC][5] = 74,
 	[1][0][2][0][RTW89_ACMA][5] = 60,
-	[1][0][2][0][RTW89_FCC][6] = 64,
+	[1][0][2][0][RTW89_CN][5] = 58,
+	[1][0][2][0][RTW89_UK][5] = 60,
+	[1][0][2][0][RTW89_FCC][6] = 66,
 	[1][0][2][0][RTW89_ETSI][6] = 60,
 	[1][0][2][0][RTW89_MKK][6] = 74,
-	[1][0][2][0][RTW89_IC][6] = 74,
+	[1][0][2][0][RTW89_IC][6] = 66,
+	[1][0][2][0][RTW89_KCC][6] = 74,
 	[1][0][2][0][RTW89_ACMA][6] = 60,
-	[1][0][2][0][RTW89_FCC][7] = 60,
+	[1][0][2][0][RTW89_CN][6] = 58,
+	[1][0][2][0][RTW89_UK][6] = 60,
+	[1][0][2][0][RTW89_FCC][7] = 62,
 	[1][0][2][0][RTW89_ETSI][7] = 60,
 	[1][0][2][0][RTW89_MKK][7] = 74,
-	[1][0][2][0][RTW89_IC][7] = 70,
+	[1][0][2][0][RTW89_IC][7] = 62,
+	[1][0][2][0][RTW89_KCC][7] = 74,
 	[1][0][2][0][RTW89_ACMA][7] = 60,
-	[1][0][2][0][RTW89_FCC][8] = 60,
+	[1][0][2][0][RTW89_CN][7] = 58,
+	[1][0][2][0][RTW89_UK][7] = 60,
+	[1][0][2][0][RTW89_FCC][8] = 62,
 	[1][0][2][0][RTW89_ETSI][8] = 60,
 	[1][0][2][0][RTW89_MKK][8] = 74,
-	[1][0][2][0][RTW89_IC][8] = 70,
+	[1][0][2][0][RTW89_IC][8] = 62,
+	[1][0][2][0][RTW89_KCC][8] = 68,
 	[1][0][2][0][RTW89_ACMA][8] = 60,
+	[1][0][2][0][RTW89_CN][8] = 58,
+	[1][0][2][0][RTW89_UK][8] = 60,
 	[1][0][2][0][RTW89_FCC][9] = 60,
 	[1][0][2][0][RTW89_ETSI][9] = 60,
 	[1][0][2][0][RTW89_MKK][9] = 74,
-	[1][0][2][0][RTW89_IC][9] = 70,
+	[1][0][2][0][RTW89_IC][9] = 60,
+	[1][0][2][0][RTW89_KCC][9] = 68,
 	[1][0][2][0][RTW89_ACMA][9] = 60,
-	[1][0][2][0][RTW89_FCC][10] = 58,
+	[1][0][2][0][RTW89_CN][9] = 58,
+	[1][0][2][0][RTW89_UK][9] = 60,
+	[1][0][2][0][RTW89_FCC][10] = 56,
 	[1][0][2][0][RTW89_ETSI][10] = 60,
 	[1][0][2][0][RTW89_MKK][10] = 74,
-	[1][0][2][0][RTW89_IC][10] = 68,
+	[1][0][2][0][RTW89_IC][10] = 56,
+	[1][0][2][0][RTW89_KCC][10] = 68,
 	[1][0][2][0][RTW89_ACMA][10] = 60,
+	[1][0][2][0][RTW89_CN][10] = 58,
+	[1][0][2][0][RTW89_UK][10] = 60,
 	[1][0][2][0][RTW89_FCC][11] = 127,
 	[1][0][2][0][RTW89_ETSI][11] = 127,
 	[1][0][2][0][RTW89_MKK][11] = 127,
 	[1][0][2][0][RTW89_IC][11] = 127,
+	[1][0][2][0][RTW89_KCC][11] = 127,
 	[1][0][2][0][RTW89_ACMA][11] = 127,
+	[1][0][2][0][RTW89_CN][11] = 127,
+	[1][0][2][0][RTW89_UK][11] = 127,
 	[1][0][2][0][RTW89_FCC][12] = 127,
 	[1][0][2][0][RTW89_ETSI][12] = 127,
 	[1][0][2][0][RTW89_MKK][12] = 127,
 	[1][0][2][0][RTW89_IC][12] = 127,
+	[1][0][2][0][RTW89_KCC][12] = 127,
 	[1][0][2][0][RTW89_ACMA][12] = 127,
+	[1][0][2][0][RTW89_CN][12] = 127,
+	[1][0][2][0][RTW89_UK][12] = 127,
 	[1][0][2][0][RTW89_FCC][13] = 127,
 	[1][0][2][0][RTW89_ETSI][13] = 127,
 	[1][0][2][0][RTW89_MKK][13] = 127,
 	[1][0][2][0][RTW89_IC][13] = 127,
+	[1][0][2][0][RTW89_KCC][13] = 127,
 	[1][0][2][0][RTW89_ACMA][13] = 127,
+	[1][0][2][0][RTW89_CN][13] = 127,
+	[1][0][2][0][RTW89_UK][13] = 127,
 	[1][1][2][0][RTW89_FCC][0] = 127,
 	[1][1][2][0][RTW89_ETSI][0] = 127,
 	[1][1][2][0][RTW89_MKK][0] = 127,
 	[1][1][2][0][RTW89_IC][0] = 127,
+	[1][1][2][0][RTW89_KCC][0] = 127,
 	[1][1][2][0][RTW89_ACMA][0] = 127,
+	[1][1][2][0][RTW89_CN][0] = 127,
+	[1][1][2][0][RTW89_UK][0] = 127,
 	[1][1][2][0][RTW89_FCC][1] = 127,
 	[1][1][2][0][RTW89_ETSI][1] = 127,
 	[1][1][2][0][RTW89_MKK][1] = 127,
 	[1][1][2][0][RTW89_IC][1] = 127,
+	[1][1][2][0][RTW89_KCC][1] = 127,
 	[1][1][2][0][RTW89_ACMA][1] = 127,
-	[1][1][2][0][RTW89_FCC][2] = 46,
+	[1][1][2][0][RTW89_CN][1] = 127,
+	[1][1][2][0][RTW89_UK][1] = 127,
+	[1][1][2][0][RTW89_FCC][2] = 60,
 	[1][1][2][0][RTW89_ETSI][2] = 48,
 	[1][1][2][0][RTW89_MKK][2] = 68,
-	[1][1][2][0][RTW89_IC][2] = 56,
+	[1][1][2][0][RTW89_IC][2] = 60,
+	[1][1][2][0][RTW89_KCC][2] = 64,
 	[1][1][2][0][RTW89_ACMA][2] = 48,
-	[1][1][2][0][RTW89_FCC][3] = 46,
+	[1][1][2][0][RTW89_CN][2] = 34,
+	[1][1][2][0][RTW89_UK][2] = 48,
+	[1][1][2][0][RTW89_FCC][3] = 60,
 	[1][1][2][0][RTW89_ETSI][3] = 48,
 	[1][1][2][0][RTW89_MKK][3] = 68,
-	[1][1][2][0][RTW89_IC][3] = 56,
+	[1][1][2][0][RTW89_IC][3] = 60,
+	[1][1][2][0][RTW89_KCC][3] = 64,
 	[1][1][2][0][RTW89_ACMA][3] = 48,
-	[1][1][2][0][RTW89_FCC][4] = 50,
+	[1][1][2][0][RTW89_CN][3] = 34,
+	[1][1][2][0][RTW89_UK][3] = 48,
+	[1][1][2][0][RTW89_FCC][4] = 60,
 	[1][1][2][0][RTW89_ETSI][4] = 48,
 	[1][1][2][0][RTW89_MKK][4] = 68,
 	[1][1][2][0][RTW89_IC][4] = 60,
+	[1][1][2][0][RTW89_KCC][4] = 64,
 	[1][1][2][0][RTW89_ACMA][4] = 48,
-	[1][1][2][0][RTW89_FCC][5] = 58,
+	[1][1][2][0][RTW89_CN][4] = 34,
+	[1][1][2][0][RTW89_UK][4] = 48,
+	[1][1][2][0][RTW89_FCC][5] = 60,
 	[1][1][2][0][RTW89_ETSI][5] = 48,
 	[1][1][2][0][RTW89_MKK][5] = 68,
-	[1][1][2][0][RTW89_IC][5] = 68,
+	[1][1][2][0][RTW89_IC][5] = 60,
+	[1][1][2][0][RTW89_KCC][5] = 66,
 	[1][1][2][0][RTW89_ACMA][5] = 48,
-	[1][1][2][0][RTW89_FCC][6] = 50,
+	[1][1][2][0][RTW89_CN][5] = 34,
+	[1][1][2][0][RTW89_UK][5] = 48,
+	[1][1][2][0][RTW89_FCC][6] = 58,
 	[1][1][2][0][RTW89_ETSI][6] = 48,
 	[1][1][2][0][RTW89_MKK][6] = 68,
-	[1][1][2][0][RTW89_IC][6] = 60,
+	[1][1][2][0][RTW89_IC][6] = 58,
+	[1][1][2][0][RTW89_KCC][6] = 66,
 	[1][1][2][0][RTW89_ACMA][6] = 48,
-	[1][1][2][0][RTW89_FCC][7] = 46,
+	[1][1][2][0][RTW89_CN][6] = 34,
+	[1][1][2][0][RTW89_UK][6] = 48,
+	[1][1][2][0][RTW89_FCC][7] = 54,
 	[1][1][2][0][RTW89_ETSI][7] = 48,
 	[1][1][2][0][RTW89_MKK][7] = 68,
-	[1][1][2][0][RTW89_IC][7] = 56,
+	[1][1][2][0][RTW89_IC][7] = 54,
+	[1][1][2][0][RTW89_KCC][7] = 66,
 	[1][1][2][0][RTW89_ACMA][7] = 48,
-	[1][1][2][0][RTW89_FCC][8] = 46,
+	[1][1][2][0][RTW89_CN][7] = 34,
+	[1][1][2][0][RTW89_UK][7] = 48,
+	[1][1][2][0][RTW89_FCC][8] = 54,
 	[1][1][2][0][RTW89_ETSI][8] = 48,
 	[1][1][2][0][RTW89_MKK][8] = 68,
-	[1][1][2][0][RTW89_IC][8] = 56,
+	[1][1][2][0][RTW89_IC][8] = 54,
+	[1][1][2][0][RTW89_KCC][8] = 64,
 	[1][1][2][0][RTW89_ACMA][8] = 48,
-	[1][1][2][0][RTW89_FCC][9] = 34,
+	[1][1][2][0][RTW89_CN][8] = 34,
+	[1][1][2][0][RTW89_UK][8] = 48,
+	[1][1][2][0][RTW89_FCC][9] = 54,
 	[1][1][2][0][RTW89_ETSI][9] = 48,
 	[1][1][2][0][RTW89_MKK][9] = 68,
-	[1][1][2][0][RTW89_IC][9] = 44,
+	[1][1][2][0][RTW89_IC][9] = 54,
+	[1][1][2][0][RTW89_KCC][9] = 64,
 	[1][1][2][0][RTW89_ACMA][9] = 48,
-	[1][1][2][0][RTW89_FCC][10] = 30,
+	[1][1][2][0][RTW89_CN][9] = 34,
+	[1][1][2][0][RTW89_UK][9] = 48,
+	[1][1][2][0][RTW89_FCC][10] = 46,
 	[1][1][2][0][RTW89_ETSI][10] = 48,
 	[1][1][2][0][RTW89_MKK][10] = 68,
-	[1][1][2][0][RTW89_IC][10] = 40,
+	[1][1][2][0][RTW89_IC][10] = 46,
+	[1][1][2][0][RTW89_KCC][10] = 64,
 	[1][1][2][0][RTW89_ACMA][10] = 48,
+	[1][1][2][0][RTW89_CN][10] = 34,
+	[1][1][2][0][RTW89_UK][10] = 48,
 	[1][1][2][0][RTW89_FCC][11] = 127,
 	[1][1][2][0][RTW89_ETSI][11] = 127,
 	[1][1][2][0][RTW89_MKK][11] = 127,
 	[1][1][2][0][RTW89_IC][11] = 127,
+	[1][1][2][0][RTW89_KCC][11] = 127,
 	[1][1][2][0][RTW89_ACMA][11] = 127,
+	[1][1][2][0][RTW89_CN][11] = 127,
+	[1][1][2][0][RTW89_UK][11] = 127,
 	[1][1][2][0][RTW89_FCC][12] = 127,
 	[1][1][2][0][RTW89_ETSI][12] = 127,
 	[1][1][2][0][RTW89_MKK][12] = 127,
 	[1][1][2][0][RTW89_IC][12] = 127,
+	[1][1][2][0][RTW89_KCC][12] = 127,
 	[1][1][2][0][RTW89_ACMA][12] = 127,
+	[1][1][2][0][RTW89_CN][12] = 127,
+	[1][1][2][0][RTW89_UK][12] = 127,
 	[1][1][2][0][RTW89_FCC][13] = 127,
 	[1][1][2][0][RTW89_ETSI][13] = 127,
 	[1][1][2][0][RTW89_MKK][13] = 127,
 	[1][1][2][0][RTW89_IC][13] = 127,
+	[1][1][2][0][RTW89_KCC][13] = 127,
 	[1][1][2][0][RTW89_ACMA][13] = 127,
+	[1][1][2][0][RTW89_CN][13] = 127,
+	[1][1][2][0][RTW89_UK][13] = 127,
 	[1][1][2][1][RTW89_FCC][0] = 127,
 	[1][1][2][1][RTW89_ETSI][0] = 127,
 	[1][1][2][1][RTW89_MKK][0] = 127,
 	[1][1][2][1][RTW89_IC][0] = 127,
+	[1][1][2][1][RTW89_KCC][0] = 127,
 	[1][1][2][1][RTW89_ACMA][0] = 127,
+	[1][1][2][1][RTW89_CN][0] = 127,
+	[1][1][2][1][RTW89_UK][0] = 127,
 	[1][1][2][1][RTW89_FCC][1] = 127,
 	[1][1][2][1][RTW89_ETSI][1] = 127,
 	[1][1][2][1][RTW89_MKK][1] = 127,
 	[1][1][2][1][RTW89_IC][1] = 127,
+	[1][1][2][1][RTW89_KCC][1] = 127,
 	[1][1][2][1][RTW89_ACMA][1] = 127,
-	[1][1][2][1][RTW89_FCC][2] = 46,
+	[1][1][2][1][RTW89_CN][1] = 127,
+	[1][1][2][1][RTW89_UK][1] = 127,
+	[1][1][2][1][RTW89_FCC][2] = 60,
 	[1][1][2][1][RTW89_ETSI][2] = 36,
 	[1][1][2][1][RTW89_MKK][2] = 68,
-	[1][1][2][1][RTW89_IC][2] = 56,
+	[1][1][2][1][RTW89_IC][2] = 60,
+	[1][1][2][1][RTW89_KCC][2] = 64,
 	[1][1][2][1][RTW89_ACMA][2] = 36,
-	[1][1][2][1][RTW89_FCC][3] = 46,
+	[1][1][2][1][RTW89_CN][2] = 34,
+	[1][1][2][1][RTW89_UK][2] = 36,
+	[1][1][2][1][RTW89_FCC][3] = 60,
 	[1][1][2][1][RTW89_ETSI][3] = 36,
 	[1][1][2][1][RTW89_MKK][3] = 68,
-	[1][1][2][1][RTW89_IC][3] = 56,
+	[1][1][2][1][RTW89_IC][3] = 60,
+	[1][1][2][1][RTW89_KCC][3] = 64,
 	[1][1][2][1][RTW89_ACMA][3] = 36,
-	[1][1][2][1][RTW89_FCC][4] = 50,
+	[1][1][2][1][RTW89_CN][3] = 34,
+	[1][1][2][1][RTW89_UK][3] = 36,
+	[1][1][2][1][RTW89_FCC][4] = 60,
 	[1][1][2][1][RTW89_ETSI][4] = 36,
 	[1][1][2][1][RTW89_MKK][4] = 68,
 	[1][1][2][1][RTW89_IC][4] = 60,
+	[1][1][2][1][RTW89_KCC][4] = 64,
 	[1][1][2][1][RTW89_ACMA][4] = 36,
-	[1][1][2][1][RTW89_FCC][5] = 58,
+	[1][1][2][1][RTW89_CN][4] = 34,
+	[1][1][2][1][RTW89_UK][4] = 36,
+	[1][1][2][1][RTW89_FCC][5] = 60,
 	[1][1][2][1][RTW89_ETSI][5] = 36,
 	[1][1][2][1][RTW89_MKK][5] = 68,
-	[1][1][2][1][RTW89_IC][5] = 68,
+	[1][1][2][1][RTW89_IC][5] = 60,
+	[1][1][2][1][RTW89_KCC][5] = 66,
 	[1][1][2][1][RTW89_ACMA][5] = 36,
-	[1][1][2][1][RTW89_FCC][6] = 50,
+	[1][1][2][1][RTW89_CN][5] = 34,
+	[1][1][2][1][RTW89_UK][5] = 36,
+	[1][1][2][1][RTW89_FCC][6] = 58,
 	[1][1][2][1][RTW89_ETSI][6] = 36,
 	[1][1][2][1][RTW89_MKK][6] = 68,
-	[1][1][2][1][RTW89_IC][6] = 60,
+	[1][1][2][1][RTW89_IC][6] = 58,
+	[1][1][2][1][RTW89_KCC][6] = 66,
 	[1][1][2][1][RTW89_ACMA][6] = 36,
-	[1][1][2][1][RTW89_FCC][7] = 46,
+	[1][1][2][1][RTW89_CN][6] = 34,
+	[1][1][2][1][RTW89_UK][6] = 36,
+	[1][1][2][1][RTW89_FCC][7] = 54,
 	[1][1][2][1][RTW89_ETSI][7] = 36,
 	[1][1][2][1][RTW89_MKK][7] = 68,
-	[1][1][2][1][RTW89_IC][7] = 56,
+	[1][1][2][1][RTW89_IC][7] = 54,
+	[1][1][2][1][RTW89_KCC][7] = 66,
 	[1][1][2][1][RTW89_ACMA][7] = 36,
-	[1][1][2][1][RTW89_FCC][8] = 46,
+	[1][1][2][1][RTW89_CN][7] = 34,
+	[1][1][2][1][RTW89_UK][7] = 36,
+	[1][1][2][1][RTW89_FCC][8] = 54,
 	[1][1][2][1][RTW89_ETSI][8] = 36,
 	[1][1][2][1][RTW89_MKK][8] = 68,
-	[1][1][2][1][RTW89_IC][8] = 56,
+	[1][1][2][1][RTW89_IC][8] = 54,
+	[1][1][2][1][RTW89_KCC][8] = 64,
 	[1][1][2][1][RTW89_ACMA][8] = 36,
-	[1][1][2][1][RTW89_FCC][9] = 34,
+	[1][1][2][1][RTW89_CN][8] = 34,
+	[1][1][2][1][RTW89_UK][8] = 36,
+	[1][1][2][1][RTW89_FCC][9] = 54,
 	[1][1][2][1][RTW89_ETSI][9] = 36,
 	[1][1][2][1][RTW89_MKK][9] = 68,
-	[1][1][2][1][RTW89_IC][9] = 44,
+	[1][1][2][1][RTW89_IC][9] = 54,
+	[1][1][2][1][RTW89_KCC][9] = 64,
 	[1][1][2][1][RTW89_ACMA][9] = 36,
-	[1][1][2][1][RTW89_FCC][10] = 30,
+	[1][1][2][1][RTW89_CN][9] = 34,
+	[1][1][2][1][RTW89_UK][9] = 36,
+	[1][1][2][1][RTW89_FCC][10] = 46,
 	[1][1][2][1][RTW89_ETSI][10] = 36,
 	[1][1][2][1][RTW89_MKK][10] = 68,
-	[1][1][2][1][RTW89_IC][10] = 40,
+	[1][1][2][1][RTW89_IC][10] = 46,
+	[1][1][2][1][RTW89_KCC][10] = 64,
 	[1][1][2][1][RTW89_ACMA][10] = 36,
+	[1][1][2][1][RTW89_CN][10] = 36,
+	[1][1][2][1][RTW89_UK][10] = 36,
 	[1][1][2][1][RTW89_FCC][11] = 127,
 	[1][1][2][1][RTW89_ETSI][11] = 127,
 	[1][1][2][1][RTW89_MKK][11] = 127,
 	[1][1][2][1][RTW89_IC][11] = 127,
+	[1][1][2][1][RTW89_KCC][11] = 127,
 	[1][1][2][1][RTW89_ACMA][11] = 127,
+	[1][1][2][1][RTW89_CN][11] = 127,
+	[1][1][2][1][RTW89_UK][11] = 127,
 	[1][1][2][1][RTW89_FCC][12] = 127,
 	[1][1][2][1][RTW89_ETSI][12] = 127,
 	[1][1][2][1][RTW89_MKK][12] = 127,
 	[1][1][2][1][RTW89_IC][12] = 127,
+	[1][1][2][1][RTW89_KCC][12] = 127,
 	[1][1][2][1][RTW89_ACMA][12] = 127,
+	[1][1][2][1][RTW89_CN][12] = 127,
+	[1][1][2][1][RTW89_UK][12] = 127,
 	[1][1][2][1][RTW89_FCC][13] = 127,
 	[1][1][2][1][RTW89_ETSI][13] = 127,
 	[1][1][2][1][RTW89_MKK][13] = 127,
 	[1][1][2][1][RTW89_IC][13] = 127,
+	[1][1][2][1][RTW89_KCC][13] = 127,
 	[1][1][2][1][RTW89_ACMA][13] = 127,
+	[1][1][2][1][RTW89_CN][13] = 127,
+	[1][1][2][1][RTW89_UK][13] = 127,
 };
 
 const s8 rtw89_8852c_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
 				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
 				 [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
-	[0][0][1][0][RTW89_WW][0] = 60,
-	[0][0][1][0][RTW89_WW][2] = 60,
-	[0][0][1][0][RTW89_WW][4] = 60,
-	[0][0][1][0][RTW89_WW][6] = 60,
-	[0][0][1][0][RTW89_WW][8] = 60,
-	[0][0][1][0][RTW89_WW][10] = 60,
-	[0][0][1][0][RTW89_WW][12] = 60,
-	[0][0][1][0][RTW89_WW][14] = 60,
-	[0][0][1][0][RTW89_WW][15] = 60,
-	[0][0][1][0][RTW89_WW][17] = 60,
-	[0][0][1][0][RTW89_WW][19] = 60,
-	[0][0][1][0][RTW89_WW][21] = 60,
-	[0][0][1][0][RTW89_WW][23] = 60,
+	[0][0][1][0][RTW89_WW][0] = 50,
+	[0][0][1][0][RTW89_WW][2] = 50,
+	[0][0][1][0][RTW89_WW][4] = 50,
+	[0][0][1][0][RTW89_WW][6] = 50,
+	[0][0][1][0][RTW89_WW][8] = 50,
+	[0][0][1][0][RTW89_WW][10] = 50,
+	[0][0][1][0][RTW89_WW][12] = 50,
+	[0][0][1][0][RTW89_WW][14] = 50,
+	[0][0][1][0][RTW89_WW][15] = 66,
+	[0][0][1][0][RTW89_WW][17] = 66,
+	[0][0][1][0][RTW89_WW][19] = 66,
+	[0][0][1][0][RTW89_WW][21] = 66,
+	[0][0][1][0][RTW89_WW][23] = 66,
 	[0][0][1][0][RTW89_WW][25] = 66,
 	[0][0][1][0][RTW89_WW][27] = 66,
 	[0][0][1][0][RTW89_WW][29] = 66,
-	[0][0][1][0][RTW89_WW][31] = 60,
-	[0][0][1][0][RTW89_WW][33] = 60,
+	[0][0][1][0][RTW89_WW][31] = 66,
+	[0][0][1][0][RTW89_WW][33] = 66,
 	[0][0][1][0][RTW89_WW][35] = 60,
-	[0][0][1][0][RTW89_WW][37] = 70,
+	[0][0][1][0][RTW89_WW][37] = 64,
 	[0][0][1][0][RTW89_WW][38] = 30,
 	[0][0][1][0][RTW89_WW][40] = 30,
 	[0][0][1][0][RTW89_WW][42] = 30,
 	[0][0][1][0][RTW89_WW][44] = 30,
 	[0][0][1][0][RTW89_WW][46] = 30,
-	[0][0][1][0][RTW89_WW][48] = 70,
-	[0][0][1][0][RTW89_WW][50] = 70,
-	[0][0][1][0][RTW89_WW][52] = 70,
-	[0][1][1][0][RTW89_WW][0] = 42,
-	[0][1][1][0][RTW89_WW][2] = 42,
-	[0][1][1][0][RTW89_WW][4] = 42,
-	[0][1][1][0][RTW89_WW][6] = 42,
-	[0][1][1][0][RTW89_WW][8] = 48,
-	[0][1][1][0][RTW89_WW][10] = 48,
-	[0][1][1][0][RTW89_WW][12] = 48,
-	[0][1][1][0][RTW89_WW][14] = 48,
-	[0][1][1][0][RTW89_WW][15] = 48,
-	[0][1][1][0][RTW89_WW][17] = 48,
-	[0][1][1][0][RTW89_WW][19] = 48,
-	[0][1][1][0][RTW89_WW][21] = 48,
-	[0][1][1][0][RTW89_WW][23] = 48,
+	[0][0][1][0][RTW89_WW][48] = 72,
+	[0][0][1][0][RTW89_WW][50] = 72,
+	[0][0][1][0][RTW89_WW][52] = 72,
+	[0][1][1][0][RTW89_WW][0] = 34,
+	[0][1][1][0][RTW89_WW][2] = 34,
+	[0][1][1][0][RTW89_WW][4] = 34,
+	[0][1][1][0][RTW89_WW][6] = 36,
+	[0][1][1][0][RTW89_WW][8] = 46,
+	[0][1][1][0][RTW89_WW][10] = 46,
+	[0][1][1][0][RTW89_WW][12] = 46,
+	[0][1][1][0][RTW89_WW][14] = 46,
+	[0][1][1][0][RTW89_WW][15] = 54,
+	[0][1][1][0][RTW89_WW][17] = 54,
+	[0][1][1][0][RTW89_WW][19] = 54,
+	[0][1][1][0][RTW89_WW][21] = 54,
+	[0][1][1][0][RTW89_WW][23] = 54,
 	[0][1][1][0][RTW89_WW][25] = 54,
 	[0][1][1][0][RTW89_WW][27] = 54,
 	[0][1][1][0][RTW89_WW][29] = 54,
-	[0][1][1][0][RTW89_WW][31] = 48,
-	[0][1][1][0][RTW89_WW][33] = 48,
-	[0][1][1][0][RTW89_WW][35] = 48,
-	[0][1][1][0][RTW89_WW][37] = 60,
+	[0][1][1][0][RTW89_WW][31] = 54,
+	[0][1][1][0][RTW89_WW][33] = 54,
+	[0][1][1][0][RTW89_WW][35] = 52,
+	[0][1][1][0][RTW89_WW][37] = 52,
 	[0][1][1][0][RTW89_WW][38] = 18,
-	[0][1][1][0][RTW89_WW][40] = 16,
+	[0][1][1][0][RTW89_WW][40] = 18,
 	[0][1][1][0][RTW89_WW][42] = 18,
-	[0][1][1][0][RTW89_WW][44] = 16,
+	[0][1][1][0][RTW89_WW][44] = 18,
 	[0][1][1][0][RTW89_WW][46] = 18,
 	[0][1][1][0][RTW89_WW][48] = 48,
 	[0][1][1][0][RTW89_WW][50] = 48,
 	[0][1][1][0][RTW89_WW][52] = 48,
-	[0][0][2][0][RTW89_WW][0] = 62,
-	[0][0][2][0][RTW89_WW][2] = 62,
-	[0][0][2][0][RTW89_WW][4] = 62,
-	[0][0][2][0][RTW89_WW][6] = 60,
-	[0][0][2][0][RTW89_WW][8] = 58,
-	[0][0][2][0][RTW89_WW][10] = 62,
-	[0][0][2][0][RTW89_WW][12] = 62,
-	[0][0][2][0][RTW89_WW][14] = 62,
-	[0][0][2][0][RTW89_WW][15] = 62,
-	[0][0][2][0][RTW89_WW][17] = 62,
-	[0][0][2][0][RTW89_WW][19] = 62,
-	[0][0][2][0][RTW89_WW][21] = 62,
-	[0][0][2][0][RTW89_WW][23] = 62,
+	[0][0][2][0][RTW89_WW][0] = 52,
+	[0][0][2][0][RTW89_WW][2] = 52,
+	[0][0][2][0][RTW89_WW][4] = 52,
+	[0][0][2][0][RTW89_WW][6] = 52,
+	[0][0][2][0][RTW89_WW][8] = 52,
+	[0][0][2][0][RTW89_WW][10] = 52,
+	[0][0][2][0][RTW89_WW][12] = 52,
+	[0][0][2][0][RTW89_WW][14] = 52,
+	[0][0][2][0][RTW89_WW][15] = 66,
+	[0][0][2][0][RTW89_WW][17] = 66,
+	[0][0][2][0][RTW89_WW][19] = 66,
+	[0][0][2][0][RTW89_WW][21] = 66,
+	[0][0][2][0][RTW89_WW][23] = 66,
 	[0][0][2][0][RTW89_WW][25] = 66,
 	[0][0][2][0][RTW89_WW][27] = 66,
 	[0][0][2][0][RTW89_WW][29] = 66,
-	[0][0][2][0][RTW89_WW][31] = 62,
-	[0][0][2][0][RTW89_WW][33] = 62,
-	[0][0][2][0][RTW89_WW][35] = 62,
-	[0][0][2][0][RTW89_WW][37] = 70,
+	[0][0][2][0][RTW89_WW][31] = 66,
+	[0][0][2][0][RTW89_WW][33] = 66,
+	[0][0][2][0][RTW89_WW][35] = 56,
+	[0][0][2][0][RTW89_WW][37] = 64,
 	[0][0][2][0][RTW89_WW][38] = 30,
 	[0][0][2][0][RTW89_WW][40] = 30,
 	[0][0][2][0][RTW89_WW][42] = 30,
 	[0][0][2][0][RTW89_WW][44] = 30,
 	[0][0][2][0][RTW89_WW][46] = 30,
-	[0][0][2][0][RTW89_WW][48] = 70,
-	[0][0][2][0][RTW89_WW][50] = 70,
-	[0][0][2][0][RTW89_WW][52] = 70,
-	[0][1][2][0][RTW89_WW][0] = 44,
-	[0][1][2][0][RTW89_WW][2] = 44,
-	[0][1][2][0][RTW89_WW][4] = 44,
-	[0][1][2][0][RTW89_WW][6] = 44,
-	[0][1][2][0][RTW89_WW][8] = 42,
-	[0][1][2][0][RTW89_WW][10] = 50,
-	[0][1][2][0][RTW89_WW][12] = 50,
-	[0][1][2][0][RTW89_WW][14] = 50,
-	[0][1][2][0][RTW89_WW][15] = 50,
-	[0][1][2][0][RTW89_WW][17] = 50,
-	[0][1][2][0][RTW89_WW][19] = 50,
-	[0][1][2][0][RTW89_WW][21] = 50,
-	[0][1][2][0][RTW89_WW][23] = 50,
+	[0][0][2][0][RTW89_WW][48] = 72,
+	[0][0][2][0][RTW89_WW][50] = 72,
+	[0][0][2][0][RTW89_WW][52] = 72,
+	[0][1][2][0][RTW89_WW][0] = 36,
+	[0][1][2][0][RTW89_WW][2] = 36,
+	[0][1][2][0][RTW89_WW][4] = 36,
+	[0][1][2][0][RTW89_WW][6] = 38,
+	[0][1][2][0][RTW89_WW][8] = 40,
+	[0][1][2][0][RTW89_WW][10] = 40,
+	[0][1][2][0][RTW89_WW][12] = 40,
+	[0][1][2][0][RTW89_WW][14] = 40,
+	[0][1][2][0][RTW89_WW][15] = 54,
+	[0][1][2][0][RTW89_WW][17] = 54,
+	[0][1][2][0][RTW89_WW][19] = 54,
+	[0][1][2][0][RTW89_WW][21] = 54,
+	[0][1][2][0][RTW89_WW][23] = 54,
 	[0][1][2][0][RTW89_WW][25] = 54,
 	[0][1][2][0][RTW89_WW][27] = 54,
 	[0][1][2][0][RTW89_WW][29] = 54,
-	[0][1][2][0][RTW89_WW][31] = 50,
-	[0][1][2][0][RTW89_WW][33] = 50,
-	[0][1][2][0][RTW89_WW][35] = 50,
-	[0][1][2][0][RTW89_WW][37] = 62,
+	[0][1][2][0][RTW89_WW][31] = 54,
+	[0][1][2][0][RTW89_WW][33] = 54,
+	[0][1][2][0][RTW89_WW][35] = 46,
+	[0][1][2][0][RTW89_WW][37] = 52,
 	[0][1][2][0][RTW89_WW][38] = 18,
 	[0][1][2][0][RTW89_WW][40] = 18,
 	[0][1][2][0][RTW89_WW][42] = 18,
 	[0][1][2][0][RTW89_WW][44] = 18,
 	[0][1][2][0][RTW89_WW][46] = 18,
-	[0][1][2][0][RTW89_WW][48] = 50,
+	[0][1][2][0][RTW89_WW][48] = 48,
 	[0][1][2][0][RTW89_WW][50] = 50,
-	[0][1][2][0][RTW89_WW][52] = 50,
-	[0][1][2][1][RTW89_WW][0] = 38,
-	[0][1][2][1][RTW89_WW][2] = 38,
-	[0][1][2][1][RTW89_WW][4] = 38,
-	[0][1][2][1][RTW89_WW][6] = 38,
-	[0][1][2][1][RTW89_WW][8] = 38,
-	[0][1][2][1][RTW89_WW][10] = 38,
-	[0][1][2][1][RTW89_WW][12] = 38,
-	[0][1][2][1][RTW89_WW][14] = 38,
-	[0][1][2][1][RTW89_WW][15] = 38,
-	[0][1][2][1][RTW89_WW][17] = 38,
-	[0][1][2][1][RTW89_WW][19] = 38,
-	[0][1][2][1][RTW89_WW][21] = 38,
-	[0][1][2][1][RTW89_WW][23] = 38,
+	[0][1][2][0][RTW89_WW][52] = 48,
+	[0][1][2][1][RTW89_WW][0] = 36,
+	[0][1][2][1][RTW89_WW][2] = 36,
+	[0][1][2][1][RTW89_WW][4] = 36,
+	[0][1][2][1][RTW89_WW][6] = 36,
+	[0][1][2][1][RTW89_WW][8] = 36,
+	[0][1][2][1][RTW89_WW][10] = 36,
+	[0][1][2][1][RTW89_WW][12] = 36,
+	[0][1][2][1][RTW89_WW][14] = 36,
+	[0][1][2][1][RTW89_WW][15] = 40,
+	[0][1][2][1][RTW89_WW][17] = 40,
+	[0][1][2][1][RTW89_WW][19] = 40,
+	[0][1][2][1][RTW89_WW][21] = 40,
+	[0][1][2][1][RTW89_WW][23] = 40,
 	[0][1][2][1][RTW89_WW][25] = 40,
 	[0][1][2][1][RTW89_WW][27] = 40,
 	[0][1][2][1][RTW89_WW][29] = 40,
-	[0][1][2][1][RTW89_WW][31] = 38,
-	[0][1][2][1][RTW89_WW][33] = 38,
-	[0][1][2][1][RTW89_WW][35] = 38,
-	[0][1][2][1][RTW89_WW][37] = 60,
+	[0][1][2][1][RTW89_WW][31] = 40,
+	[0][1][2][1][RTW89_WW][33] = 40,
+	[0][1][2][1][RTW89_WW][35] = 40,
+	[0][1][2][1][RTW89_WW][37] = 40,
 	[0][1][2][1][RTW89_WW][38] = 6,
 	[0][1][2][1][RTW89_WW][40] = 6,
 	[0][1][2][1][RTW89_WW][42] = 6,
 	[0][1][2][1][RTW89_WW][44] = 6,
 	[0][1][2][1][RTW89_WW][46] = 6,
-	[0][1][2][1][RTW89_WW][48] = 50,
+	[0][1][2][1][RTW89_WW][48] = 48,
 	[0][1][2][1][RTW89_WW][50] = 50,
-	[0][1][2][1][RTW89_WW][52] = 50,
-	[1][0][2][0][RTW89_WW][1] = 58,
-	[1][0][2][0][RTW89_WW][5] = 66,
-	[1][0][2][0][RTW89_WW][9] = 66,
-	[1][0][2][0][RTW89_WW][13] = 58,
+	[0][1][2][1][RTW89_WW][52] = 48,
+	[1][0][2][0][RTW89_WW][1] = 54,
+	[1][0][2][0][RTW89_WW][5] = 54,
+	[1][0][2][0][RTW89_WW][9] = 54,
+	[1][0][2][0][RTW89_WW][13] = 52,
 	[1][0][2][0][RTW89_WW][16] = 56,
-	[1][0][2][0][RTW89_WW][20] = 66,
-	[1][0][2][0][RTW89_WW][24] = 66,
+	[1][0][2][0][RTW89_WW][20] = 56,
+	[1][0][2][0][RTW89_WW][24] = 56,
 	[1][0][2][0][RTW89_WW][28] = 66,
-	[1][0][2][0][RTW89_WW][32] = 66,
-	[1][0][2][0][RTW89_WW][36] = 66,
+	[1][0][2][0][RTW89_WW][32] = 62,
+	[1][0][2][0][RTW89_WW][36] = 64,
 	[1][0][2][0][RTW89_WW][39] = 30,
 	[1][0][2][0][RTW89_WW][43] = 30,
 	[1][0][2][0][RTW89_WW][47] = 68,
 	[1][0][2][0][RTW89_WW][51] = 68,
-	[1][1][2][0][RTW89_WW][1] = 48,
-	[1][1][2][0][RTW89_WW][5] = 52,
-	[1][1][2][0][RTW89_WW][9] = 52,
-	[1][1][2][0][RTW89_WW][13] = 52,
-	[1][1][2][0][RTW89_WW][16] = 48,
+	[1][1][2][0][RTW89_WW][1] = 42,
+	[1][1][2][0][RTW89_WW][5] = 42,
+	[1][1][2][0][RTW89_WW][9] = 42,
+	[1][1][2][0][RTW89_WW][13] = 42,
+	[1][1][2][0][RTW89_WW][16] = 54,
 	[1][1][2][0][RTW89_WW][20] = 54,
 	[1][1][2][0][RTW89_WW][24] = 54,
 	[1][1][2][0][RTW89_WW][28] = 54,
 	[1][1][2][0][RTW89_WW][32] = 54,
-	[1][1][2][0][RTW89_WW][36] = 66,
+	[1][1][2][0][RTW89_WW][36] = 52,
 	[1][1][2][0][RTW89_WW][39] = 18,
 	[1][1][2][0][RTW89_WW][43] = 18,
-	[1][1][2][0][RTW89_WW][47] = 60,
-	[1][1][2][0][RTW89_WW][51] = 58,
+	[1][1][2][0][RTW89_WW][47] = 62,
+	[1][1][2][0][RTW89_WW][51] = 60,
 	[1][1][2][1][RTW89_WW][1] = 40,
 	[1][1][2][1][RTW89_WW][5] = 40,
 	[1][1][2][1][RTW89_WW][9] = 40,
@@ -15035,2082 +29517,3694 @@ const s8 rtw89_8852c_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM]
 	[1][1][2][1][RTW89_WW][24] = 40,
 	[1][1][2][1][RTW89_WW][28] = 40,
 	[1][1][2][1][RTW89_WW][32] = 40,
-	[1][1][2][1][RTW89_WW][36] = 60,
+	[1][1][2][1][RTW89_WW][36] = 40,
 	[1][1][2][1][RTW89_WW][39] = 6,
 	[1][1][2][1][RTW89_WW][43] = 6,
-	[1][1][2][1][RTW89_WW][47] = 60,
-	[1][1][2][1][RTW89_WW][51] = 58,
-	[2][0][2][0][RTW89_WW][3] = 56,
-	[2][0][2][0][RTW89_WW][11] = 58,
-	[2][0][2][0][RTW89_WW][18] = 54,
+	[1][1][2][1][RTW89_WW][47] = 62,
+	[1][1][2][1][RTW89_WW][51] = 60,
+	[2][0][2][0][RTW89_WW][3] = 54,
+	[2][0][2][0][RTW89_WW][11] = 50,
+	[2][0][2][0][RTW89_WW][18] = 56,
 	[2][0][2][0][RTW89_WW][26] = 60,
 	[2][0][2][0][RTW89_WW][34] = 60,
 	[2][0][2][0][RTW89_WW][41] = 30,
-	[2][0][2][0][RTW89_WW][49] = 56,
-	[2][1][2][0][RTW89_WW][3] = 48,
-	[2][1][2][0][RTW89_WW][11] = 52,
-	[2][1][2][0][RTW89_WW][18] = 48,
-	[2][1][2][0][RTW89_WW][26] = 54,
-	[2][1][2][0][RTW89_WW][34] = 60,
+	[2][0][2][0][RTW89_WW][49] = 62,
+	[2][1][2][0][RTW89_WW][3] = 46,
+	[2][1][2][0][RTW89_WW][11] = 38,
+	[2][1][2][0][RTW89_WW][18] = 50,
+	[2][1][2][0][RTW89_WW][26] = 52,
+	[2][1][2][0][RTW89_WW][34] = 52,
 	[2][1][2][0][RTW89_WW][41] = 18,
-	[2][1][2][0][RTW89_WW][49] = 50,
+	[2][1][2][0][RTW89_WW][49] = 62,
 	[2][1][2][1][RTW89_WW][3] = 40,
-	[2][1][2][1][RTW89_WW][11] = 40,
+	[2][1][2][1][RTW89_WW][11] = 38,
 	[2][1][2][1][RTW89_WW][18] = 40,
 	[2][1][2][1][RTW89_WW][26] = 42,
-	[2][1][2][1][RTW89_WW][34] = 60,
+	[2][1][2][1][RTW89_WW][34] = 40,
 	[2][1][2][1][RTW89_WW][41] = 6,
-	[2][1][2][1][RTW89_WW][49] = 50,
-	[3][0][2][0][RTW89_WW][7] = 38,
-	[3][0][2][0][RTW89_WW][22] = 50,
-	[3][0][2][0][RTW89_WW][45] = 0,
-	[3][1][2][0][RTW89_WW][7] = 26,
-	[3][1][2][0][RTW89_WW][22] = 42,
-	[3][1][2][0][RTW89_WW][45] = 0,
-	[3][1][2][1][RTW89_WW][7] = 14,
-	[3][1][2][1][RTW89_WW][22] = 30,
-	[3][1][2][1][RTW89_WW][45] = 0,
-	[0][0][1][0][RTW89_FCC][0] = 70,
+	[2][1][2][1][RTW89_WW][49] = 62,
+	[3][0][2][0][RTW89_WW][7] = 40,
+	[3][0][2][0][RTW89_WW][22] = 42,
+	[3][0][2][0][RTW89_WW][45] = 52,
+	[3][1][2][0][RTW89_WW][7] = 32,
+	[3][1][2][0][RTW89_WW][22] = 36,
+	[3][1][2][0][RTW89_WW][45] = 46,
+	[3][1][2][1][RTW89_WW][7] = 32,
+	[3][1][2][1][RTW89_WW][22] = 36,
+	[3][1][2][1][RTW89_WW][45] = 46,
+	[0][0][1][0][RTW89_FCC][0] = 72,
 	[0][0][1][0][RTW89_ETSI][0] = 66,
 	[0][0][1][0][RTW89_MKK][0] = 66,
-	[0][0][1][0][RTW89_IC][0] = 62,
-	[0][0][1][0][RTW89_ACMA][0] = 60,
-	[0][0][1][0][RTW89_FCC][2] = 70,
+	[0][0][1][0][RTW89_IC][0] = 60,
+	[0][0][1][0][RTW89_KCC][0] = 52,
+	[0][0][1][0][RTW89_ACMA][0] = 66,
+	[0][0][1][0][RTW89_CN][0] = 50,
+	[0][0][1][0][RTW89_UK][0] = 66,
+	[0][0][1][0][RTW89_FCC][2] = 72,
 	[0][0][1][0][RTW89_ETSI][2] = 66,
 	[0][0][1][0][RTW89_MKK][2] = 66,
-	[0][0][1][0][RTW89_IC][2] = 62,
-	[0][0][1][0][RTW89_ACMA][2] = 60,
-	[0][0][1][0][RTW89_FCC][4] = 70,
+	[0][0][1][0][RTW89_IC][2] = 60,
+	[0][0][1][0][RTW89_KCC][2] = 52,
+	[0][0][1][0][RTW89_ACMA][2] = 66,
+	[0][0][1][0][RTW89_CN][2] = 50,
+	[0][0][1][0][RTW89_UK][2] = 66,
+	[0][0][1][0][RTW89_FCC][4] = 72,
 	[0][0][1][0][RTW89_ETSI][4] = 66,
 	[0][0][1][0][RTW89_MKK][4] = 66,
-	[0][0][1][0][RTW89_IC][4] = 62,
-	[0][0][1][0][RTW89_ACMA][4] = 60,
-	[0][0][1][0][RTW89_FCC][6] = 70,
+	[0][0][1][0][RTW89_IC][4] = 60,
+	[0][0][1][0][RTW89_KCC][4] = 52,
+	[0][0][1][0][RTW89_ACMA][4] = 66,
+	[0][0][1][0][RTW89_CN][4] = 50,
+	[0][0][1][0][RTW89_UK][4] = 66,
+	[0][0][1][0][RTW89_FCC][6] = 72,
 	[0][0][1][0][RTW89_ETSI][6] = 66,
 	[0][0][1][0][RTW89_MKK][6] = 66,
-	[0][0][1][0][RTW89_IC][6] = 62,
-	[0][0][1][0][RTW89_ACMA][6] = 60,
-	[0][0][1][0][RTW89_FCC][8] = 70,
+	[0][0][1][0][RTW89_IC][6] = 58,
+	[0][0][1][0][RTW89_KCC][6] = 62,
+	[0][0][1][0][RTW89_ACMA][6] = 66,
+	[0][0][1][0][RTW89_CN][6] = 50,
+	[0][0][1][0][RTW89_UK][6] = 66,
+	[0][0][1][0][RTW89_FCC][8] = 72,
 	[0][0][1][0][RTW89_ETSI][8] = 66,
 	[0][0][1][0][RTW89_MKK][8] = 66,
-	[0][0][1][0][RTW89_IC][8] = 66,
-	[0][0][1][0][RTW89_ACMA][8] = 60,
-	[0][0][1][0][RTW89_FCC][10] = 70,
+	[0][0][1][0][RTW89_IC][8] = 64,
+	[0][0][1][0][RTW89_KCC][8] = 70,
+	[0][0][1][0][RTW89_ACMA][8] = 66,
+	[0][0][1][0][RTW89_CN][8] = 50,
+	[0][0][1][0][RTW89_UK][8] = 66,
+	[0][0][1][0][RTW89_FCC][10] = 72,
 	[0][0][1][0][RTW89_ETSI][10] = 66,
 	[0][0][1][0][RTW89_MKK][10] = 66,
-	[0][0][1][0][RTW89_IC][10] = 66,
-	[0][0][1][0][RTW89_ACMA][10] = 60,
-	[0][0][1][0][RTW89_FCC][12] = 70,
+	[0][0][1][0][RTW89_IC][10] = 64,
+	[0][0][1][0][RTW89_KCC][10] = 70,
+	[0][0][1][0][RTW89_ACMA][10] = 66,
+	[0][0][1][0][RTW89_CN][10] = 50,
+	[0][0][1][0][RTW89_UK][10] = 66,
+	[0][0][1][0][RTW89_FCC][12] = 72,
 	[0][0][1][0][RTW89_ETSI][12] = 66,
 	[0][0][1][0][RTW89_MKK][12] = 66,
-	[0][0][1][0][RTW89_IC][12] = 66,
-	[0][0][1][0][RTW89_ACMA][12] = 60,
+	[0][0][1][0][RTW89_IC][12] = 64,
+	[0][0][1][0][RTW89_KCC][12] = 66,
+	[0][0][1][0][RTW89_ACMA][12] = 66,
+	[0][0][1][0][RTW89_CN][12] = 50,
+	[0][0][1][0][RTW89_UK][12] = 66,
 	[0][0][1][0][RTW89_FCC][14] = 70,
 	[0][0][1][0][RTW89_ETSI][14] = 66,
 	[0][0][1][0][RTW89_MKK][14] = 66,
-	[0][0][1][0][RTW89_IC][14] = 66,
-	[0][0][1][0][RTW89_ACMA][14] = 60,
-	[0][0][1][0][RTW89_FCC][15] = 68,
+	[0][0][1][0][RTW89_IC][14] = 64,
+	[0][0][1][0][RTW89_KCC][14] = 66,
+	[0][0][1][0][RTW89_ACMA][14] = 66,
+	[0][0][1][0][RTW89_CN][14] = 50,
+	[0][0][1][0][RTW89_UK][14] = 66,
+	[0][0][1][0][RTW89_FCC][15] = 72,
 	[0][0][1][0][RTW89_ETSI][15] = 66,
 	[0][0][1][0][RTW89_MKK][15] = 70,
-	[0][0][1][0][RTW89_IC][15] = 70,
-	[0][0][1][0][RTW89_ACMA][15] = 60,
-	[0][0][1][0][RTW89_FCC][17] = 70,
+	[0][0][1][0][RTW89_IC][15] = 72,
+	[0][0][1][0][RTW89_KCC][15] = 70,
+	[0][0][1][0][RTW89_ACMA][15] = 66,
+	[0][0][1][0][RTW89_CN][15] = 127,
+	[0][0][1][0][RTW89_UK][15] = 66,
+	[0][0][1][0][RTW89_FCC][17] = 72,
 	[0][0][1][0][RTW89_ETSI][17] = 66,
 	[0][0][1][0][RTW89_MKK][17] = 70,
-	[0][0][1][0][RTW89_IC][17] = 70,
-	[0][0][1][0][RTW89_ACMA][17] = 60,
-	[0][0][1][0][RTW89_FCC][19] = 70,
+	[0][0][1][0][RTW89_IC][17] = 72,
+	[0][0][1][0][RTW89_KCC][17] = 70,
+	[0][0][1][0][RTW89_ACMA][17] = 66,
+	[0][0][1][0][RTW89_CN][17] = 127,
+	[0][0][1][0][RTW89_UK][17] = 66,
+	[0][0][1][0][RTW89_FCC][19] = 72,
 	[0][0][1][0][RTW89_ETSI][19] = 66,
 	[0][0][1][0][RTW89_MKK][19] = 70,
-	[0][0][1][0][RTW89_IC][19] = 70,
-	[0][0][1][0][RTW89_ACMA][19] = 60,
-	[0][0][1][0][RTW89_FCC][21] = 70,
+	[0][0][1][0][RTW89_IC][19] = 72,
+	[0][0][1][0][RTW89_KCC][19] = 70,
+	[0][0][1][0][RTW89_ACMA][19] = 66,
+	[0][0][1][0][RTW89_CN][19] = 127,
+	[0][0][1][0][RTW89_UK][19] = 66,
+	[0][0][1][0][RTW89_FCC][21] = 72,
 	[0][0][1][0][RTW89_ETSI][21] = 66,
 	[0][0][1][0][RTW89_MKK][21] = 70,
-	[0][0][1][0][RTW89_IC][21] = 70,
-	[0][0][1][0][RTW89_ACMA][21] = 60,
-	[0][0][1][0][RTW89_FCC][23] = 70,
+	[0][0][1][0][RTW89_IC][21] = 72,
+	[0][0][1][0][RTW89_KCC][21] = 70,
+	[0][0][1][0][RTW89_ACMA][21] = 66,
+	[0][0][1][0][RTW89_CN][21] = 127,
+	[0][0][1][0][RTW89_UK][21] = 66,
+	[0][0][1][0][RTW89_FCC][23] = 72,
 	[0][0][1][0][RTW89_ETSI][23] = 66,
 	[0][0][1][0][RTW89_MKK][23] = 70,
-	[0][0][1][0][RTW89_IC][23] = 70,
-	[0][0][1][0][RTW89_ACMA][23] = 60,
-	[0][0][1][0][RTW89_FCC][25] = 70,
+	[0][0][1][0][RTW89_IC][23] = 72,
+	[0][0][1][0][RTW89_KCC][23] = 70,
+	[0][0][1][0][RTW89_ACMA][23] = 66,
+	[0][0][1][0][RTW89_CN][23] = 127,
+	[0][0][1][0][RTW89_UK][23] = 66,
+	[0][0][1][0][RTW89_FCC][25] = 72,
 	[0][0][1][0][RTW89_ETSI][25] = 66,
 	[0][0][1][0][RTW89_MKK][25] = 70,
 	[0][0][1][0][RTW89_IC][25] = 127,
+	[0][0][1][0][RTW89_KCC][25] = 70,
 	[0][0][1][0][RTW89_ACMA][25] = 127,
-	[0][0][1][0][RTW89_FCC][27] = 70,
+	[0][0][1][0][RTW89_CN][25] = 127,
+	[0][0][1][0][RTW89_UK][25] = 66,
+	[0][0][1][0][RTW89_FCC][27] = 72,
 	[0][0][1][0][RTW89_ETSI][27] = 66,
 	[0][0][1][0][RTW89_MKK][27] = 70,
 	[0][0][1][0][RTW89_IC][27] = 127,
+	[0][0][1][0][RTW89_KCC][27] = 70,
 	[0][0][1][0][RTW89_ACMA][27] = 127,
-	[0][0][1][0][RTW89_FCC][29] = 70,
+	[0][0][1][0][RTW89_CN][27] = 127,
+	[0][0][1][0][RTW89_UK][27] = 66,
+	[0][0][1][0][RTW89_FCC][29] = 72,
 	[0][0][1][0][RTW89_ETSI][29] = 66,
 	[0][0][1][0][RTW89_MKK][29] = 70,
 	[0][0][1][0][RTW89_IC][29] = 127,
+	[0][0][1][0][RTW89_KCC][29] = 70,
 	[0][0][1][0][RTW89_ACMA][29] = 127,
-	[0][0][1][0][RTW89_FCC][31] = 70,
+	[0][0][1][0][RTW89_CN][29] = 127,
+	[0][0][1][0][RTW89_UK][29] = 66,
+	[0][0][1][0][RTW89_FCC][31] = 72,
 	[0][0][1][0][RTW89_ETSI][31] = 66,
 	[0][0][1][0][RTW89_MKK][31] = 70,
-	[0][0][1][0][RTW89_IC][31] = 70,
-	[0][0][1][0][RTW89_ACMA][31] = 60,
-	[0][0][1][0][RTW89_FCC][33] = 70,
+	[0][0][1][0][RTW89_IC][31] = 72,
+	[0][0][1][0][RTW89_KCC][31] = 70,
+	[0][0][1][0][RTW89_ACMA][31] = 66,
+	[0][0][1][0][RTW89_CN][31] = 127,
+	[0][0][1][0][RTW89_UK][31] = 66,
+	[0][0][1][0][RTW89_FCC][33] = 72,
 	[0][0][1][0][RTW89_ETSI][33] = 66,
 	[0][0][1][0][RTW89_MKK][33] = 70,
-	[0][0][1][0][RTW89_IC][33] = 70,
-	[0][0][1][0][RTW89_ACMA][33] = 60,
-	[0][0][1][0][RTW89_FCC][35] = 62,
+	[0][0][1][0][RTW89_IC][33] = 72,
+	[0][0][1][0][RTW89_KCC][33] = 70,
+	[0][0][1][0][RTW89_ACMA][33] = 66,
+	[0][0][1][0][RTW89_CN][33] = 127,
+	[0][0][1][0][RTW89_UK][33] = 66,
+	[0][0][1][0][RTW89_FCC][35] = 60,
 	[0][0][1][0][RTW89_ETSI][35] = 66,
 	[0][0][1][0][RTW89_MKK][35] = 70,
-	[0][0][1][0][RTW89_IC][35] = 70,
-	[0][0][1][0][RTW89_ACMA][35] = 60,
-	[0][0][1][0][RTW89_FCC][37] = 70,
+	[0][0][1][0][RTW89_IC][35] = 60,
+	[0][0][1][0][RTW89_KCC][35] = 70,
+	[0][0][1][0][RTW89_ACMA][35] = 66,
+	[0][0][1][0][RTW89_CN][35] = 127,
+	[0][0][1][0][RTW89_UK][35] = 66,
+	[0][0][1][0][RTW89_FCC][37] = 72,
 	[0][0][1][0][RTW89_ETSI][37] = 127,
 	[0][0][1][0][RTW89_MKK][37] = 70,
-	[0][0][1][0][RTW89_IC][37] = 70,
+	[0][0][1][0][RTW89_IC][37] = 72,
+	[0][0][1][0][RTW89_KCC][37] = 70,
 	[0][0][1][0][RTW89_ACMA][37] = 70,
-	[0][0][1][0][RTW89_FCC][38] = 70,
+	[0][0][1][0][RTW89_CN][37] = 127,
+	[0][0][1][0][RTW89_UK][37] = 64,
+	[0][0][1][0][RTW89_FCC][38] = 72,
 	[0][0][1][0][RTW89_ETSI][38] = 30,
 	[0][0][1][0][RTW89_MKK][38] = 127,
-	[0][0][1][0][RTW89_IC][38] = 70,
+	[0][0][1][0][RTW89_IC][38] = 72,
+	[0][0][1][0][RTW89_KCC][38] = 62,
 	[0][0][1][0][RTW89_ACMA][38] = 70,
-	[0][0][1][0][RTW89_FCC][40] = 70,
+	[0][0][1][0][RTW89_CN][38] = 68,
+	[0][0][1][0][RTW89_UK][38] = 64,
+	[0][0][1][0][RTW89_FCC][40] = 72,
 	[0][0][1][0][RTW89_ETSI][40] = 30,
 	[0][0][1][0][RTW89_MKK][40] = 127,
-	[0][0][1][0][RTW89_IC][40] = 70,
+	[0][0][1][0][RTW89_IC][40] = 72,
+	[0][0][1][0][RTW89_KCC][40] = 62,
 	[0][0][1][0][RTW89_ACMA][40] = 70,
-	[0][0][1][0][RTW89_FCC][42] = 70,
+	[0][0][1][0][RTW89_CN][40] = 68,
+	[0][0][1][0][RTW89_UK][40] = 64,
+	[0][0][1][0][RTW89_FCC][42] = 72,
 	[0][0][1][0][RTW89_ETSI][42] = 30,
 	[0][0][1][0][RTW89_MKK][42] = 127,
-	[0][0][1][0][RTW89_IC][42] = 70,
+	[0][0][1][0][RTW89_IC][42] = 72,
+	[0][0][1][0][RTW89_KCC][42] = 62,
 	[0][0][1][0][RTW89_ACMA][42] = 70,
-	[0][0][1][0][RTW89_FCC][44] = 70,
+	[0][0][1][0][RTW89_CN][42] = 68,
+	[0][0][1][0][RTW89_UK][42] = 64,
+	[0][0][1][0][RTW89_FCC][44] = 72,
 	[0][0][1][0][RTW89_ETSI][44] = 30,
 	[0][0][1][0][RTW89_MKK][44] = 127,
-	[0][0][1][0][RTW89_IC][44] = 70,
+	[0][0][1][0][RTW89_IC][44] = 72,
+	[0][0][1][0][RTW89_KCC][44] = 62,
 	[0][0][1][0][RTW89_ACMA][44] = 70,
-	[0][0][1][0][RTW89_FCC][46] = 70,
+	[0][0][1][0][RTW89_CN][44] = 68,
+	[0][0][1][0][RTW89_UK][44] = 64,
+	[0][0][1][0][RTW89_FCC][46] = 72,
 	[0][0][1][0][RTW89_ETSI][46] = 30,
 	[0][0][1][0][RTW89_MKK][46] = 127,
-	[0][0][1][0][RTW89_IC][46] = 70,
+	[0][0][1][0][RTW89_IC][46] = 72,
+	[0][0][1][0][RTW89_KCC][46] = 62,
 	[0][0][1][0][RTW89_ACMA][46] = 70,
-	[0][0][1][0][RTW89_FCC][48] = 70,
+	[0][0][1][0][RTW89_CN][46] = 68,
+	[0][0][1][0][RTW89_UK][46] = 64,
+	[0][0][1][0][RTW89_FCC][48] = 72,
 	[0][0][1][0][RTW89_ETSI][48] = 127,
 	[0][0][1][0][RTW89_MKK][48] = 127,
 	[0][0][1][0][RTW89_IC][48] = 127,
+	[0][0][1][0][RTW89_KCC][48] = 127,
 	[0][0][1][0][RTW89_ACMA][48] = 127,
-	[0][0][1][0][RTW89_FCC][50] = 70,
+	[0][0][1][0][RTW89_CN][48] = 127,
+	[0][0][1][0][RTW89_UK][48] = 127,
+	[0][0][1][0][RTW89_FCC][50] = 72,
 	[0][0][1][0][RTW89_ETSI][50] = 127,
 	[0][0][1][0][RTW89_MKK][50] = 127,
 	[0][0][1][0][RTW89_IC][50] = 127,
+	[0][0][1][0][RTW89_KCC][50] = 127,
 	[0][0][1][0][RTW89_ACMA][50] = 127,
-	[0][0][1][0][RTW89_FCC][52] = 70,
+	[0][0][1][0][RTW89_CN][50] = 127,
+	[0][0][1][0][RTW89_UK][50] = 127,
+	[0][0][1][0][RTW89_FCC][52] = 72,
 	[0][0][1][0][RTW89_ETSI][52] = 127,
 	[0][0][1][0][RTW89_MKK][52] = 127,
 	[0][0][1][0][RTW89_IC][52] = 127,
+	[0][0][1][0][RTW89_KCC][52] = 127,
 	[0][0][1][0][RTW89_ACMA][52] = 127,
+	[0][0][1][0][RTW89_CN][52] = 127,
+	[0][0][1][0][RTW89_UK][52] = 127,
 	[0][1][1][0][RTW89_FCC][0] = 60,
 	[0][1][1][0][RTW89_ETSI][0] = 54,
 	[0][1][1][0][RTW89_MKK][0] = 54,
-	[0][1][1][0][RTW89_IC][0] = 42,
-	[0][1][1][0][RTW89_ACMA][0] = 48,
+	[0][1][1][0][RTW89_IC][0] = 34,
+	[0][1][1][0][RTW89_KCC][0] = 40,
+	[0][1][1][0][RTW89_ACMA][0] = 54,
+	[0][1][1][0][RTW89_CN][0] = 46,
+	[0][1][1][0][RTW89_UK][0] = 54,
 	[0][1][1][0][RTW89_FCC][2] = 60,
 	[0][1][1][0][RTW89_ETSI][2] = 54,
 	[0][1][1][0][RTW89_MKK][2] = 54,
-	[0][1][1][0][RTW89_IC][2] = 42,
-	[0][1][1][0][RTW89_ACMA][2] = 48,
+	[0][1][1][0][RTW89_IC][2] = 34,
+	[0][1][1][0][RTW89_KCC][2] = 40,
+	[0][1][1][0][RTW89_ACMA][2] = 54,
+	[0][1][1][0][RTW89_CN][2] = 46,
+	[0][1][1][0][RTW89_UK][2] = 54,
 	[0][1][1][0][RTW89_FCC][4] = 60,
 	[0][1][1][0][RTW89_ETSI][4] = 54,
 	[0][1][1][0][RTW89_MKK][4] = 54,
-	[0][1][1][0][RTW89_IC][4] = 42,
-	[0][1][1][0][RTW89_ACMA][4] = 48,
+	[0][1][1][0][RTW89_IC][4] = 34,
+	[0][1][1][0][RTW89_KCC][4] = 40,
+	[0][1][1][0][RTW89_ACMA][4] = 54,
+	[0][1][1][0][RTW89_CN][4] = 46,
+	[0][1][1][0][RTW89_UK][4] = 54,
 	[0][1][1][0][RTW89_FCC][6] = 60,
 	[0][1][1][0][RTW89_ETSI][6] = 54,
 	[0][1][1][0][RTW89_MKK][6] = 54,
-	[0][1][1][0][RTW89_IC][6] = 42,
-	[0][1][1][0][RTW89_ACMA][6] = 48,
-	[0][1][1][0][RTW89_FCC][8] = 60,
+	[0][1][1][0][RTW89_IC][6] = 36,
+	[0][1][1][0][RTW89_KCC][6] = 60,
+	[0][1][1][0][RTW89_ACMA][6] = 54,
+	[0][1][1][0][RTW89_CN][6] = 46,
+	[0][1][1][0][RTW89_UK][6] = 54,
+	[0][1][1][0][RTW89_FCC][8] = 62,
 	[0][1][1][0][RTW89_ETSI][8] = 54,
 	[0][1][1][0][RTW89_MKK][8] = 52,
-	[0][1][1][0][RTW89_IC][8] = 54,
-	[0][1][1][0][RTW89_ACMA][8] = 48,
-	[0][1][1][0][RTW89_FCC][10] = 60,
+	[0][1][1][0][RTW89_IC][8] = 52,
+	[0][1][1][0][RTW89_KCC][8] = 60,
+	[0][1][1][0][RTW89_ACMA][8] = 54,
+	[0][1][1][0][RTW89_CN][8] = 46,
+	[0][1][1][0][RTW89_UK][8] = 54,
+	[0][1][1][0][RTW89_FCC][10] = 62,
 	[0][1][1][0][RTW89_ETSI][10] = 54,
 	[0][1][1][0][RTW89_MKK][10] = 54,
-	[0][1][1][0][RTW89_IC][10] = 54,
-	[0][1][1][0][RTW89_ACMA][10] = 48,
-	[0][1][1][0][RTW89_FCC][12] = 60,
+	[0][1][1][0][RTW89_IC][10] = 52,
+	[0][1][1][0][RTW89_KCC][10] = 60,
+	[0][1][1][0][RTW89_ACMA][10] = 54,
+	[0][1][1][0][RTW89_CN][10] = 46,
+	[0][1][1][0][RTW89_UK][10] = 54,
+	[0][1][1][0][RTW89_FCC][12] = 62,
 	[0][1][1][0][RTW89_ETSI][12] = 54,
 	[0][1][1][0][RTW89_MKK][12] = 54,
-	[0][1][1][0][RTW89_IC][12] = 54,
-	[0][1][1][0][RTW89_ACMA][12] = 48,
+	[0][1][1][0][RTW89_IC][12] = 52,
+	[0][1][1][0][RTW89_KCC][12] = 60,
+	[0][1][1][0][RTW89_ACMA][12] = 54,
+	[0][1][1][0][RTW89_CN][12] = 46,
+	[0][1][1][0][RTW89_UK][12] = 54,
 	[0][1][1][0][RTW89_FCC][14] = 60,
 	[0][1][1][0][RTW89_ETSI][14] = 54,
 	[0][1][1][0][RTW89_MKK][14] = 54,
-	[0][1][1][0][RTW89_IC][14] = 54,
-	[0][1][1][0][RTW89_ACMA][14] = 48,
-	[0][1][1][0][RTW89_FCC][15] = 58,
+	[0][1][1][0][RTW89_IC][14] = 52,
+	[0][1][1][0][RTW89_KCC][14] = 60,
+	[0][1][1][0][RTW89_ACMA][14] = 54,
+	[0][1][1][0][RTW89_CN][14] = 46,
+	[0][1][1][0][RTW89_UK][14] = 54,
+	[0][1][1][0][RTW89_FCC][15] = 60,
 	[0][1][1][0][RTW89_ETSI][15] = 54,
 	[0][1][1][0][RTW89_MKK][15] = 70,
-	[0][1][1][0][RTW89_IC][15] = 68,
-	[0][1][1][0][RTW89_ACMA][15] = 48,
+	[0][1][1][0][RTW89_IC][15] = 60,
+	[0][1][1][0][RTW89_KCC][15] = 60,
+	[0][1][1][0][RTW89_ACMA][15] = 54,
+	[0][1][1][0][RTW89_CN][15] = 127,
+	[0][1][1][0][RTW89_UK][15] = 54,
 	[0][1][1][0][RTW89_FCC][17] = 60,
 	[0][1][1][0][RTW89_ETSI][17] = 54,
 	[0][1][1][0][RTW89_MKK][17] = 70,
-	[0][1][1][0][RTW89_IC][17] = 70,
-	[0][1][1][0][RTW89_ACMA][17] = 48,
+	[0][1][1][0][RTW89_IC][17] = 60,
+	[0][1][1][0][RTW89_KCC][17] = 60,
+	[0][1][1][0][RTW89_ACMA][17] = 54,
+	[0][1][1][0][RTW89_CN][17] = 127,
+	[0][1][1][0][RTW89_UK][17] = 54,
 	[0][1][1][0][RTW89_FCC][19] = 60,
 	[0][1][1][0][RTW89_ETSI][19] = 54,
 	[0][1][1][0][RTW89_MKK][19] = 70,
-	[0][1][1][0][RTW89_IC][19] = 70,
-	[0][1][1][0][RTW89_ACMA][19] = 48,
+	[0][1][1][0][RTW89_IC][19] = 60,
+	[0][1][1][0][RTW89_KCC][19] = 60,
+	[0][1][1][0][RTW89_ACMA][19] = 54,
+	[0][1][1][0][RTW89_CN][19] = 127,
+	[0][1][1][0][RTW89_UK][19] = 54,
 	[0][1][1][0][RTW89_FCC][21] = 60,
 	[0][1][1][0][RTW89_ETSI][21] = 54,
 	[0][1][1][0][RTW89_MKK][21] = 70,
-	[0][1][1][0][RTW89_IC][21] = 70,
-	[0][1][1][0][RTW89_ACMA][21] = 48,
+	[0][1][1][0][RTW89_IC][21] = 60,
+	[0][1][1][0][RTW89_KCC][21] = 60,
+	[0][1][1][0][RTW89_ACMA][21] = 54,
+	[0][1][1][0][RTW89_CN][21] = 127,
+	[0][1][1][0][RTW89_UK][21] = 54,
 	[0][1][1][0][RTW89_FCC][23] = 60,
 	[0][1][1][0][RTW89_ETSI][23] = 54,
 	[0][1][1][0][RTW89_MKK][23] = 70,
-	[0][1][1][0][RTW89_IC][23] = 70,
-	[0][1][1][0][RTW89_ACMA][23] = 48,
+	[0][1][1][0][RTW89_IC][23] = 60,
+	[0][1][1][0][RTW89_KCC][23] = 60,
+	[0][1][1][0][RTW89_ACMA][23] = 54,
+	[0][1][1][0][RTW89_CN][23] = 127,
+	[0][1][1][0][RTW89_UK][23] = 54,
 	[0][1][1][0][RTW89_FCC][25] = 60,
 	[0][1][1][0][RTW89_ETSI][25] = 54,
 	[0][1][1][0][RTW89_MKK][25] = 70,
 	[0][1][1][0][RTW89_IC][25] = 127,
+	[0][1][1][0][RTW89_KCC][25] = 60,
 	[0][1][1][0][RTW89_ACMA][25] = 127,
+	[0][1][1][0][RTW89_CN][25] = 127,
+	[0][1][1][0][RTW89_UK][25] = 54,
 	[0][1][1][0][RTW89_FCC][27] = 60,
 	[0][1][1][0][RTW89_ETSI][27] = 54,
 	[0][1][1][0][RTW89_MKK][27] = 70,
 	[0][1][1][0][RTW89_IC][27] = 127,
+	[0][1][1][0][RTW89_KCC][27] = 60,
 	[0][1][1][0][RTW89_ACMA][27] = 127,
+	[0][1][1][0][RTW89_CN][27] = 127,
+	[0][1][1][0][RTW89_UK][27] = 54,
 	[0][1][1][0][RTW89_FCC][29] = 60,
 	[0][1][1][0][RTW89_ETSI][29] = 54,
 	[0][1][1][0][RTW89_MKK][29] = 70,
 	[0][1][1][0][RTW89_IC][29] = 127,
+	[0][1][1][0][RTW89_KCC][29] = 60,
 	[0][1][1][0][RTW89_ACMA][29] = 127,
+	[0][1][1][0][RTW89_CN][29] = 127,
+	[0][1][1][0][RTW89_UK][29] = 54,
 	[0][1][1][0][RTW89_FCC][31] = 60,
 	[0][1][1][0][RTW89_ETSI][31] = 54,
 	[0][1][1][0][RTW89_MKK][31] = 70,
-	[0][1][1][0][RTW89_IC][31] = 70,
-	[0][1][1][0][RTW89_ACMA][31] = 48,
+	[0][1][1][0][RTW89_IC][31] = 60,
+	[0][1][1][0][RTW89_KCC][31] = 58,
+	[0][1][1][0][RTW89_ACMA][31] = 54,
+	[0][1][1][0][RTW89_CN][31] = 127,
+	[0][1][1][0][RTW89_UK][31] = 54,
 	[0][1][1][0][RTW89_FCC][33] = 60,
 	[0][1][1][0][RTW89_ETSI][33] = 54,
 	[0][1][1][0][RTW89_MKK][33] = 70,
-	[0][1][1][0][RTW89_IC][33] = 70,
-	[0][1][1][0][RTW89_ACMA][33] = 48,
-	[0][1][1][0][RTW89_FCC][35] = 58,
+	[0][1][1][0][RTW89_IC][33] = 60,
+	[0][1][1][0][RTW89_KCC][33] = 58,
+	[0][1][1][0][RTW89_ACMA][33] = 54,
+	[0][1][1][0][RTW89_CN][33] = 127,
+	[0][1][1][0][RTW89_UK][33] = 54,
+	[0][1][1][0][RTW89_FCC][35] = 52,
 	[0][1][1][0][RTW89_ETSI][35] = 54,
 	[0][1][1][0][RTW89_MKK][35] = 70,
-	[0][1][1][0][RTW89_IC][35] = 68,
-	[0][1][1][0][RTW89_ACMA][35] = 48,
-	[0][1][1][0][RTW89_FCC][37] = 60,
+	[0][1][1][0][RTW89_IC][35] = 52,
+	[0][1][1][0][RTW89_KCC][35] = 58,
+	[0][1][1][0][RTW89_ACMA][35] = 54,
+	[0][1][1][0][RTW89_CN][35] = 127,
+	[0][1][1][0][RTW89_UK][35] = 54,
+	[0][1][1][0][RTW89_FCC][37] = 62,
 	[0][1][1][0][RTW89_ETSI][37] = 127,
 	[0][1][1][0][RTW89_MKK][37] = 70,
-	[0][1][1][0][RTW89_IC][37] = 70,
-	[0][1][1][0][RTW89_ACMA][37] = 70,
-	[0][1][1][0][RTW89_FCC][38] = 70,
+	[0][1][1][0][RTW89_IC][37] = 62,
+	[0][1][1][0][RTW89_KCC][37] = 58,
+	[0][1][1][0][RTW89_ACMA][37] = 64,
+	[0][1][1][0][RTW89_CN][37] = 127,
+	[0][1][1][0][RTW89_UK][37] = 52,
+	[0][1][1][0][RTW89_FCC][38] = 72,
 	[0][1][1][0][RTW89_ETSI][38] = 18,
 	[0][1][1][0][RTW89_MKK][38] = 127,
-	[0][1][1][0][RTW89_IC][38] = 70,
+	[0][1][1][0][RTW89_IC][38] = 72,
+	[0][1][1][0][RTW89_KCC][38] = 60,
 	[0][1][1][0][RTW89_ACMA][38] = 70,
-	[0][1][1][0][RTW89_FCC][40] = 70,
+	[0][1][1][0][RTW89_CN][38] = 64,
+	[0][1][1][0][RTW89_UK][38] = 52,
+	[0][1][1][0][RTW89_FCC][40] = 72,
 	[0][1][1][0][RTW89_ETSI][40] = 18,
 	[0][1][1][0][RTW89_MKK][40] = 127,
-	[0][1][1][0][RTW89_IC][40] = 70,
-	[0][1][1][0][RTW89_ACMA][40] = 16,
-	[0][1][1][0][RTW89_FCC][42] = 70,
+	[0][1][1][0][RTW89_IC][40] = 72,
+	[0][1][1][0][RTW89_KCC][40] = 60,
+	[0][1][1][0][RTW89_ACMA][40] = 70,
+	[0][1][1][0][RTW89_CN][40] = 64,
+	[0][1][1][0][RTW89_UK][40] = 52,
+	[0][1][1][0][RTW89_FCC][42] = 72,
 	[0][1][1][0][RTW89_ETSI][42] = 18,
 	[0][1][1][0][RTW89_MKK][42] = 127,
-	[0][1][1][0][RTW89_IC][42] = 70,
+	[0][1][1][0][RTW89_IC][42] = 72,
+	[0][1][1][0][RTW89_KCC][42] = 60,
 	[0][1][1][0][RTW89_ACMA][42] = 70,
-	[0][1][1][0][RTW89_FCC][44] = 70,
+	[0][1][1][0][RTW89_CN][42] = 64,
+	[0][1][1][0][RTW89_UK][42] = 52,
+	[0][1][1][0][RTW89_FCC][44] = 72,
 	[0][1][1][0][RTW89_ETSI][44] = 18,
 	[0][1][1][0][RTW89_MKK][44] = 127,
-	[0][1][1][0][RTW89_IC][44] = 70,
-	[0][1][1][0][RTW89_ACMA][44] = 16,
-	[0][1][1][0][RTW89_FCC][46] = 70,
+	[0][1][1][0][RTW89_IC][44] = 72,
+	[0][1][1][0][RTW89_KCC][44] = 60,
+	[0][1][1][0][RTW89_ACMA][44] = 70,
+	[0][1][1][0][RTW89_CN][44] = 60,
+	[0][1][1][0][RTW89_UK][44] = 52,
+	[0][1][1][0][RTW89_FCC][46] = 72,
 	[0][1][1][0][RTW89_ETSI][46] = 18,
 	[0][1][1][0][RTW89_MKK][46] = 127,
-	[0][1][1][0][RTW89_IC][46] = 70,
+	[0][1][1][0][RTW89_IC][46] = 72,
+	[0][1][1][0][RTW89_KCC][46] = 60,
 	[0][1][1][0][RTW89_ACMA][46] = 70,
+	[0][1][1][0][RTW89_CN][46] = 60,
+	[0][1][1][0][RTW89_UK][46] = 52,
 	[0][1][1][0][RTW89_FCC][48] = 48,
 	[0][1][1][0][RTW89_ETSI][48] = 127,
 	[0][1][1][0][RTW89_MKK][48] = 127,
 	[0][1][1][0][RTW89_IC][48] = 127,
+	[0][1][1][0][RTW89_KCC][48] = 127,
 	[0][1][1][0][RTW89_ACMA][48] = 127,
+	[0][1][1][0][RTW89_CN][48] = 127,
+	[0][1][1][0][RTW89_UK][48] = 127,
 	[0][1][1][0][RTW89_FCC][50] = 48,
 	[0][1][1][0][RTW89_ETSI][50] = 127,
 	[0][1][1][0][RTW89_MKK][50] = 127,
 	[0][1][1][0][RTW89_IC][50] = 127,
+	[0][1][1][0][RTW89_KCC][50] = 127,
 	[0][1][1][0][RTW89_ACMA][50] = 127,
+	[0][1][1][0][RTW89_CN][50] = 127,
+	[0][1][1][0][RTW89_UK][50] = 127,
 	[0][1][1][0][RTW89_FCC][52] = 48,
 	[0][1][1][0][RTW89_ETSI][52] = 127,
 	[0][1][1][0][RTW89_MKK][52] = 127,
 	[0][1][1][0][RTW89_IC][52] = 127,
+	[0][1][1][0][RTW89_KCC][52] = 127,
 	[0][1][1][0][RTW89_ACMA][52] = 127,
+	[0][1][1][0][RTW89_CN][52] = 127,
+	[0][1][1][0][RTW89_UK][52] = 127,
 	[0][0][2][0][RTW89_FCC][0] = 70,
 	[0][0][2][0][RTW89_ETSI][0] = 66,
 	[0][0][2][0][RTW89_MKK][0] = 68,
-	[0][0][2][0][RTW89_IC][0] = 66,
-	[0][0][2][0][RTW89_ACMA][0] = 62,
-	[0][0][2][0][RTW89_FCC][2] = 70,
+	[0][0][2][0][RTW89_IC][0] = 60,
+	[0][0][2][0][RTW89_KCC][0] = 54,
+	[0][0][2][0][RTW89_ACMA][0] = 66,
+	[0][0][2][0][RTW89_CN][0] = 52,
+	[0][0][2][0][RTW89_UK][0] = 66,
+	[0][0][2][0][RTW89_FCC][2] = 72,
 	[0][0][2][0][RTW89_ETSI][2] = 66,
 	[0][0][2][0][RTW89_MKK][2] = 68,
-	[0][0][2][0][RTW89_IC][2] = 66,
-	[0][0][2][0][RTW89_ACMA][2] = 62,
-	[0][0][2][0][RTW89_FCC][4] = 70,
+	[0][0][2][0][RTW89_IC][2] = 60,
+	[0][0][2][0][RTW89_KCC][2] = 54,
+	[0][0][2][0][RTW89_ACMA][2] = 66,
+	[0][0][2][0][RTW89_CN][2] = 52,
+	[0][0][2][0][RTW89_UK][2] = 66,
+	[0][0][2][0][RTW89_FCC][4] = 72,
 	[0][0][2][0][RTW89_ETSI][4] = 66,
 	[0][0][2][0][RTW89_MKK][4] = 68,
-	[0][0][2][0][RTW89_IC][4] = 66,
-	[0][0][2][0][RTW89_ACMA][4] = 62,
-	[0][0][2][0][RTW89_FCC][6] = 70,
+	[0][0][2][0][RTW89_IC][4] = 60,
+	[0][0][2][0][RTW89_KCC][4] = 54,
+	[0][0][2][0][RTW89_ACMA][4] = 66,
+	[0][0][2][0][RTW89_CN][4] = 52,
+	[0][0][2][0][RTW89_UK][4] = 66,
+	[0][0][2][0][RTW89_FCC][6] = 72,
 	[0][0][2][0][RTW89_ETSI][6] = 66,
 	[0][0][2][0][RTW89_MKK][6] = 60,
-	[0][0][2][0][RTW89_IC][6] = 66,
-	[0][0][2][0][RTW89_ACMA][6] = 62,
-	[0][0][2][0][RTW89_FCC][8] = 70,
+	[0][0][2][0][RTW89_IC][6] = 60,
+	[0][0][2][0][RTW89_KCC][6] = 68,
+	[0][0][2][0][RTW89_ACMA][6] = 66,
+	[0][0][2][0][RTW89_CN][6] = 52,
+	[0][0][2][0][RTW89_UK][6] = 66,
+	[0][0][2][0][RTW89_FCC][8] = 72,
 	[0][0][2][0][RTW89_ETSI][8] = 66,
 	[0][0][2][0][RTW89_MKK][8] = 58,
-	[0][0][2][0][RTW89_IC][8] = 66,
-	[0][0][2][0][RTW89_ACMA][8] = 62,
-	[0][0][2][0][RTW89_FCC][10] = 70,
+	[0][0][2][0][RTW89_IC][8] = 64,
+	[0][0][2][0][RTW89_KCC][8] = 70,
+	[0][0][2][0][RTW89_ACMA][8] = 66,
+	[0][0][2][0][RTW89_CN][8] = 52,
+	[0][0][2][0][RTW89_UK][8] = 66,
+	[0][0][2][0][RTW89_FCC][10] = 72,
 	[0][0][2][0][RTW89_ETSI][10] = 66,
 	[0][0][2][0][RTW89_MKK][10] = 70,
-	[0][0][2][0][RTW89_IC][10] = 66,
-	[0][0][2][0][RTW89_ACMA][10] = 62,
-	[0][0][2][0][RTW89_FCC][12] = 70,
+	[0][0][2][0][RTW89_IC][10] = 64,
+	[0][0][2][0][RTW89_KCC][10] = 70,
+	[0][0][2][0][RTW89_ACMA][10] = 66,
+	[0][0][2][0][RTW89_CN][10] = 52,
+	[0][0][2][0][RTW89_UK][10] = 66,
+	[0][0][2][0][RTW89_FCC][12] = 72,
 	[0][0][2][0][RTW89_ETSI][12] = 66,
 	[0][0][2][0][RTW89_MKK][12] = 70,
-	[0][0][2][0][RTW89_IC][12] = 66,
-	[0][0][2][0][RTW89_ACMA][12] = 62,
-	[0][0][2][0][RTW89_FCC][14] = 70,
+	[0][0][2][0][RTW89_IC][12] = 64,
+	[0][0][2][0][RTW89_KCC][12] = 66,
+	[0][0][2][0][RTW89_ACMA][12] = 66,
+	[0][0][2][0][RTW89_CN][12] = 52,
+	[0][0][2][0][RTW89_UK][12] = 66,
+	[0][0][2][0][RTW89_FCC][14] = 68,
 	[0][0][2][0][RTW89_ETSI][14] = 66,
 	[0][0][2][0][RTW89_MKK][14] = 70,
-	[0][0][2][0][RTW89_IC][14] = 66,
-	[0][0][2][0][RTW89_ACMA][14] = 62,
-	[0][0][2][0][RTW89_FCC][15] = 66,
+	[0][0][2][0][RTW89_IC][14] = 64,
+	[0][0][2][0][RTW89_KCC][14] = 66,
+	[0][0][2][0][RTW89_ACMA][14] = 66,
+	[0][0][2][0][RTW89_CN][14] = 52,
+	[0][0][2][0][RTW89_UK][14] = 66,
+	[0][0][2][0][RTW89_FCC][15] = 70,
 	[0][0][2][0][RTW89_ETSI][15] = 66,
 	[0][0][2][0][RTW89_MKK][15] = 70,
 	[0][0][2][0][RTW89_IC][15] = 70,
-	[0][0][2][0][RTW89_ACMA][15] = 62,
-	[0][0][2][0][RTW89_FCC][17] = 70,
+	[0][0][2][0][RTW89_KCC][15] = 70,
+	[0][0][2][0][RTW89_ACMA][15] = 66,
+	[0][0][2][0][RTW89_CN][15] = 127,
+	[0][0][2][0][RTW89_UK][15] = 66,
+	[0][0][2][0][RTW89_FCC][17] = 72,
 	[0][0][2][0][RTW89_ETSI][17] = 66,
 	[0][0][2][0][RTW89_MKK][17] = 70,
-	[0][0][2][0][RTW89_IC][17] = 70,
-	[0][0][2][0][RTW89_ACMA][17] = 62,
-	[0][0][2][0][RTW89_FCC][19] = 70,
+	[0][0][2][0][RTW89_IC][17] = 72,
+	[0][0][2][0][RTW89_KCC][17] = 70,
+	[0][0][2][0][RTW89_ACMA][17] = 66,
+	[0][0][2][0][RTW89_CN][17] = 127,
+	[0][0][2][0][RTW89_UK][17] = 66,
+	[0][0][2][0][RTW89_FCC][19] = 72,
 	[0][0][2][0][RTW89_ETSI][19] = 66,
 	[0][0][2][0][RTW89_MKK][19] = 70,
-	[0][0][2][0][RTW89_IC][19] = 70,
-	[0][0][2][0][RTW89_ACMA][19] = 62,
-	[0][0][2][0][RTW89_FCC][21] = 70,
+	[0][0][2][0][RTW89_IC][19] = 72,
+	[0][0][2][0][RTW89_KCC][19] = 70,
+	[0][0][2][0][RTW89_ACMA][19] = 66,
+	[0][0][2][0][RTW89_CN][19] = 127,
+	[0][0][2][0][RTW89_UK][19] = 66,
+	[0][0][2][0][RTW89_FCC][21] = 72,
 	[0][0][2][0][RTW89_ETSI][21] = 66,
 	[0][0][2][0][RTW89_MKK][21] = 70,
-	[0][0][2][0][RTW89_IC][21] = 70,
-	[0][0][2][0][RTW89_ACMA][21] = 62,
-	[0][0][2][0][RTW89_FCC][23] = 70,
+	[0][0][2][0][RTW89_IC][21] = 72,
+	[0][0][2][0][RTW89_KCC][21] = 70,
+	[0][0][2][0][RTW89_ACMA][21] = 66,
+	[0][0][2][0][RTW89_CN][21] = 127,
+	[0][0][2][0][RTW89_UK][21] = 66,
+	[0][0][2][0][RTW89_FCC][23] = 72,
 	[0][0][2][0][RTW89_ETSI][23] = 66,
 	[0][0][2][0][RTW89_MKK][23] = 70,
-	[0][0][2][0][RTW89_IC][23] = 70,
-	[0][0][2][0][RTW89_ACMA][23] = 62,
-	[0][0][2][0][RTW89_FCC][25] = 70,
+	[0][0][2][0][RTW89_IC][23] = 72,
+	[0][0][2][0][RTW89_KCC][23] = 70,
+	[0][0][2][0][RTW89_ACMA][23] = 66,
+	[0][0][2][0][RTW89_CN][23] = 127,
+	[0][0][2][0][RTW89_UK][23] = 66,
+	[0][0][2][0][RTW89_FCC][25] = 72,
 	[0][0][2][0][RTW89_ETSI][25] = 66,
 	[0][0][2][0][RTW89_MKK][25] = 70,
 	[0][0][2][0][RTW89_IC][25] = 127,
+	[0][0][2][0][RTW89_KCC][25] = 70,
 	[0][0][2][0][RTW89_ACMA][25] = 127,
-	[0][0][2][0][RTW89_FCC][27] = 70,
+	[0][0][2][0][RTW89_CN][25] = 127,
+	[0][0][2][0][RTW89_UK][25] = 66,
+	[0][0][2][0][RTW89_FCC][27] = 72,
 	[0][0][2][0][RTW89_ETSI][27] = 66,
 	[0][0][2][0][RTW89_MKK][27] = 70,
 	[0][0][2][0][RTW89_IC][27] = 127,
+	[0][0][2][0][RTW89_KCC][27] = 70,
 	[0][0][2][0][RTW89_ACMA][27] = 127,
-	[0][0][2][0][RTW89_FCC][29] = 70,
+	[0][0][2][0][RTW89_CN][27] = 127,
+	[0][0][2][0][RTW89_UK][27] = 66,
+	[0][0][2][0][RTW89_FCC][29] = 72,
 	[0][0][2][0][RTW89_ETSI][29] = 66,
 	[0][0][2][0][RTW89_MKK][29] = 70,
 	[0][0][2][0][RTW89_IC][29] = 127,
+	[0][0][2][0][RTW89_KCC][29] = 70,
 	[0][0][2][0][RTW89_ACMA][29] = 127,
-	[0][0][2][0][RTW89_FCC][31] = 70,
+	[0][0][2][0][RTW89_CN][29] = 127,
+	[0][0][2][0][RTW89_UK][29] = 66,
+	[0][0][2][0][RTW89_FCC][31] = 72,
 	[0][0][2][0][RTW89_ETSI][31] = 66,
 	[0][0][2][0][RTW89_MKK][31] = 70,
-	[0][0][2][0][RTW89_IC][31] = 70,
-	[0][0][2][0][RTW89_ACMA][31] = 62,
-	[0][0][2][0][RTW89_FCC][33] = 70,
+	[0][0][2][0][RTW89_IC][31] = 72,
+	[0][0][2][0][RTW89_KCC][31] = 70,
+	[0][0][2][0][RTW89_ACMA][31] = 66,
+	[0][0][2][0][RTW89_CN][31] = 127,
+	[0][0][2][0][RTW89_UK][31] = 66,
+	[0][0][2][0][RTW89_FCC][33] = 72,
 	[0][0][2][0][RTW89_ETSI][33] = 66,
 	[0][0][2][0][RTW89_MKK][33] = 70,
-	[0][0][2][0][RTW89_IC][33] = 70,
-	[0][0][2][0][RTW89_ACMA][33] = 62,
-	[0][0][2][0][RTW89_FCC][35] = 62,
+	[0][0][2][0][RTW89_IC][33] = 72,
+	[0][0][2][0][RTW89_KCC][33] = 70,
+	[0][0][2][0][RTW89_ACMA][33] = 66,
+	[0][0][2][0][RTW89_CN][33] = 127,
+	[0][0][2][0][RTW89_UK][33] = 66,
+	[0][0][2][0][RTW89_FCC][35] = 56,
 	[0][0][2][0][RTW89_ETSI][35] = 66,
 	[0][0][2][0][RTW89_MKK][35] = 70,
-	[0][0][2][0][RTW89_IC][35] = 70,
-	[0][0][2][0][RTW89_ACMA][35] = 62,
-	[0][0][2][0][RTW89_FCC][37] = 70,
+	[0][0][2][0][RTW89_IC][35] = 56,
+	[0][0][2][0][RTW89_KCC][35] = 70,
+	[0][0][2][0][RTW89_ACMA][35] = 66,
+	[0][0][2][0][RTW89_CN][35] = 127,
+	[0][0][2][0][RTW89_UK][35] = 66,
+	[0][0][2][0][RTW89_FCC][37] = 72,
 	[0][0][2][0][RTW89_ETSI][37] = 127,
 	[0][0][2][0][RTW89_MKK][37] = 70,
-	[0][0][2][0][RTW89_IC][37] = 70,
+	[0][0][2][0][RTW89_IC][37] = 72,
+	[0][0][2][0][RTW89_KCC][37] = 70,
 	[0][0][2][0][RTW89_ACMA][37] = 70,
-	[0][0][2][0][RTW89_FCC][38] = 70,
+	[0][0][2][0][RTW89_CN][37] = 127,
+	[0][0][2][0][RTW89_UK][37] = 64,
+	[0][0][2][0][RTW89_FCC][38] = 72,
 	[0][0][2][0][RTW89_ETSI][38] = 30,
 	[0][0][2][0][RTW89_MKK][38] = 127,
-	[0][0][2][0][RTW89_IC][38] = 70,
+	[0][0][2][0][RTW89_IC][38] = 72,
+	[0][0][2][0][RTW89_KCC][38] = 58,
 	[0][0][2][0][RTW89_ACMA][38] = 70,
-	[0][0][2][0][RTW89_FCC][40] = 70,
+	[0][0][2][0][RTW89_CN][38] = 68,
+	[0][0][2][0][RTW89_UK][38] = 64,
+	[0][0][2][0][RTW89_FCC][40] = 72,
 	[0][0][2][0][RTW89_ETSI][40] = 30,
 	[0][0][2][0][RTW89_MKK][40] = 127,
-	[0][0][2][0][RTW89_IC][40] = 70,
+	[0][0][2][0][RTW89_IC][40] = 72,
+	[0][0][2][0][RTW89_KCC][40] = 58,
 	[0][0][2][0][RTW89_ACMA][40] = 70,
-	[0][0][2][0][RTW89_FCC][42] = 70,
+	[0][0][2][0][RTW89_CN][40] = 68,
+	[0][0][2][0][RTW89_UK][40] = 64,
+	[0][0][2][0][RTW89_FCC][42] = 72,
 	[0][0][2][0][RTW89_ETSI][42] = 30,
 	[0][0][2][0][RTW89_MKK][42] = 127,
-	[0][0][2][0][RTW89_IC][42] = 70,
+	[0][0][2][0][RTW89_IC][42] = 72,
+	[0][0][2][0][RTW89_KCC][42] = 58,
 	[0][0][2][0][RTW89_ACMA][42] = 70,
-	[0][0][2][0][RTW89_FCC][44] = 70,
+	[0][0][2][0][RTW89_CN][42] = 68,
+	[0][0][2][0][RTW89_UK][42] = 64,
+	[0][0][2][0][RTW89_FCC][44] = 72,
 	[0][0][2][0][RTW89_ETSI][44] = 30,
 	[0][0][2][0][RTW89_MKK][44] = 127,
-	[0][0][2][0][RTW89_IC][44] = 70,
+	[0][0][2][0][RTW89_IC][44] = 72,
+	[0][0][2][0][RTW89_KCC][44] = 58,
 	[0][0][2][0][RTW89_ACMA][44] = 70,
-	[0][0][2][0][RTW89_FCC][46] = 70,
+	[0][0][2][0][RTW89_CN][44] = 68,
+	[0][0][2][0][RTW89_UK][44] = 64,
+	[0][0][2][0][RTW89_FCC][46] = 72,
 	[0][0][2][0][RTW89_ETSI][46] = 30,
 	[0][0][2][0][RTW89_MKK][46] = 127,
-	[0][0][2][0][RTW89_IC][46] = 70,
+	[0][0][2][0][RTW89_IC][46] = 72,
+	[0][0][2][0][RTW89_KCC][46] = 58,
 	[0][0][2][0][RTW89_ACMA][46] = 70,
-	[0][0][2][0][RTW89_FCC][48] = 70,
+	[0][0][2][0][RTW89_CN][46] = 68,
+	[0][0][2][0][RTW89_UK][46] = 64,
+	[0][0][2][0][RTW89_FCC][48] = 72,
 	[0][0][2][0][RTW89_ETSI][48] = 127,
 	[0][0][2][0][RTW89_MKK][48] = 127,
 	[0][0][2][0][RTW89_IC][48] = 127,
+	[0][0][2][0][RTW89_KCC][48] = 127,
 	[0][0][2][0][RTW89_ACMA][48] = 127,
-	[0][0][2][0][RTW89_FCC][50] = 70,
+	[0][0][2][0][RTW89_CN][48] = 127,
+	[0][0][2][0][RTW89_UK][48] = 127,
+	[0][0][2][0][RTW89_FCC][50] = 72,
 	[0][0][2][0][RTW89_ETSI][50] = 127,
 	[0][0][2][0][RTW89_MKK][50] = 127,
 	[0][0][2][0][RTW89_IC][50] = 127,
+	[0][0][2][0][RTW89_KCC][50] = 127,
 	[0][0][2][0][RTW89_ACMA][50] = 127,
-	[0][0][2][0][RTW89_FCC][52] = 70,
+	[0][0][2][0][RTW89_CN][50] = 127,
+	[0][0][2][0][RTW89_UK][50] = 127,
+	[0][0][2][0][RTW89_FCC][52] = 72,
 	[0][0][2][0][RTW89_ETSI][52] = 127,
 	[0][0][2][0][RTW89_MKK][52] = 127,
 	[0][0][2][0][RTW89_IC][52] = 127,
+	[0][0][2][0][RTW89_KCC][52] = 127,
 	[0][0][2][0][RTW89_ACMA][52] = 127,
-	[0][1][2][0][RTW89_FCC][0] = 62,
+	[0][0][2][0][RTW89_CN][52] = 127,
+	[0][0][2][0][RTW89_UK][52] = 127,
+	[0][1][2][0][RTW89_FCC][0] = 60,
 	[0][1][2][0][RTW89_ETSI][0] = 54,
 	[0][1][2][0][RTW89_MKK][0] = 54,
-	[0][1][2][0][RTW89_IC][0] = 44,
-	[0][1][2][0][RTW89_ACMA][0] = 50,
+	[0][1][2][0][RTW89_IC][0] = 36,
+	[0][1][2][0][RTW89_KCC][0] = 40,
+	[0][1][2][0][RTW89_ACMA][0] = 54,
+	[0][1][2][0][RTW89_CN][0] = 40,
+	[0][1][2][0][RTW89_UK][0] = 54,
 	[0][1][2][0][RTW89_FCC][2] = 62,
 	[0][1][2][0][RTW89_ETSI][2] = 54,
 	[0][1][2][0][RTW89_MKK][2] = 54,
-	[0][1][2][0][RTW89_IC][2] = 44,
-	[0][1][2][0][RTW89_ACMA][2] = 50,
+	[0][1][2][0][RTW89_IC][2] = 36,
+	[0][1][2][0][RTW89_KCC][2] = 40,
+	[0][1][2][0][RTW89_ACMA][2] = 54,
+	[0][1][2][0][RTW89_CN][2] = 40,
+	[0][1][2][0][RTW89_UK][2] = 54,
 	[0][1][2][0][RTW89_FCC][4] = 62,
 	[0][1][2][0][RTW89_ETSI][4] = 54,
 	[0][1][2][0][RTW89_MKK][4] = 54,
-	[0][1][2][0][RTW89_IC][4] = 44,
-	[0][1][2][0][RTW89_ACMA][4] = 50,
+	[0][1][2][0][RTW89_IC][4] = 36,
+	[0][1][2][0][RTW89_KCC][4] = 40,
+	[0][1][2][0][RTW89_ACMA][4] = 54,
+	[0][1][2][0][RTW89_CN][4] = 40,
+	[0][1][2][0][RTW89_UK][4] = 54,
 	[0][1][2][0][RTW89_FCC][6] = 62,
 	[0][1][2][0][RTW89_ETSI][6] = 54,
 	[0][1][2][0][RTW89_MKK][6] = 50,
-	[0][1][2][0][RTW89_IC][6] = 44,
-	[0][1][2][0][RTW89_ACMA][6] = 50,
+	[0][1][2][0][RTW89_IC][6] = 38,
+	[0][1][2][0][RTW89_KCC][6] = 64,
+	[0][1][2][0][RTW89_ACMA][6] = 54,
+	[0][1][2][0][RTW89_CN][6] = 40,
+	[0][1][2][0][RTW89_UK][6] = 54,
 	[0][1][2][0][RTW89_FCC][8] = 62,
 	[0][1][2][0][RTW89_ETSI][8] = 54,
 	[0][1][2][0][RTW89_MKK][8] = 42,
-	[0][1][2][0][RTW89_IC][8] = 54,
-	[0][1][2][0][RTW89_ACMA][8] = 50,
+	[0][1][2][0][RTW89_IC][8] = 52,
+	[0][1][2][0][RTW89_KCC][8] = 62,
+	[0][1][2][0][RTW89_ACMA][8] = 54,
+	[0][1][2][0][RTW89_CN][8] = 40,
+	[0][1][2][0][RTW89_UK][8] = 54,
 	[0][1][2][0][RTW89_FCC][10] = 62,
 	[0][1][2][0][RTW89_ETSI][10] = 54,
 	[0][1][2][0][RTW89_MKK][10] = 54,
-	[0][1][2][0][RTW89_IC][10] = 54,
-	[0][1][2][0][RTW89_ACMA][10] = 50,
+	[0][1][2][0][RTW89_IC][10] = 52,
+	[0][1][2][0][RTW89_KCC][10] = 62,
+	[0][1][2][0][RTW89_ACMA][10] = 54,
+	[0][1][2][0][RTW89_CN][10] = 40,
+	[0][1][2][0][RTW89_UK][10] = 54,
 	[0][1][2][0][RTW89_FCC][12] = 62,
 	[0][1][2][0][RTW89_ETSI][12] = 54,
 	[0][1][2][0][RTW89_MKK][12] = 54,
-	[0][1][2][0][RTW89_IC][12] = 54,
-	[0][1][2][0][RTW89_ACMA][12] = 50,
+	[0][1][2][0][RTW89_IC][12] = 52,
+	[0][1][2][0][RTW89_KCC][12] = 62,
+	[0][1][2][0][RTW89_ACMA][12] = 54,
+	[0][1][2][0][RTW89_CN][12] = 40,
+	[0][1][2][0][RTW89_UK][12] = 54,
 	[0][1][2][0][RTW89_FCC][14] = 62,
 	[0][1][2][0][RTW89_ETSI][14] = 54,
 	[0][1][2][0][RTW89_MKK][14] = 54,
-	[0][1][2][0][RTW89_IC][14] = 54,
-	[0][1][2][0][RTW89_ACMA][14] = 50,
+	[0][1][2][0][RTW89_IC][14] = 52,
+	[0][1][2][0][RTW89_KCC][14] = 62,
+	[0][1][2][0][RTW89_ACMA][14] = 54,
+	[0][1][2][0][RTW89_CN][14] = 40,
+	[0][1][2][0][RTW89_UK][14] = 54,
 	[0][1][2][0][RTW89_FCC][15] = 60,
 	[0][1][2][0][RTW89_ETSI][15] = 54,
 	[0][1][2][0][RTW89_MKK][15] = 68,
-	[0][1][2][0][RTW89_IC][15] = 70,
-	[0][1][2][0][RTW89_ACMA][15] = 50,
+	[0][1][2][0][RTW89_IC][15] = 60,
+	[0][1][2][0][RTW89_KCC][15] = 64,
+	[0][1][2][0][RTW89_ACMA][15] = 54,
+	[0][1][2][0][RTW89_CN][15] = 127,
+	[0][1][2][0][RTW89_UK][15] = 54,
 	[0][1][2][0][RTW89_FCC][17] = 62,
 	[0][1][2][0][RTW89_ETSI][17] = 54,
 	[0][1][2][0][RTW89_MKK][17] = 68,
-	[0][1][2][0][RTW89_IC][17] = 70,
-	[0][1][2][0][RTW89_ACMA][17] = 50,
+	[0][1][2][0][RTW89_IC][17] = 62,
+	[0][1][2][0][RTW89_KCC][17] = 64,
+	[0][1][2][0][RTW89_ACMA][17] = 54,
+	[0][1][2][0][RTW89_CN][17] = 127,
+	[0][1][2][0][RTW89_UK][17] = 54,
 	[0][1][2][0][RTW89_FCC][19] = 62,
 	[0][1][2][0][RTW89_ETSI][19] = 54,
 	[0][1][2][0][RTW89_MKK][19] = 68,
-	[0][1][2][0][RTW89_IC][19] = 70,
-	[0][1][2][0][RTW89_ACMA][19] = 50,
+	[0][1][2][0][RTW89_IC][19] = 62,
+	[0][1][2][0][RTW89_KCC][19] = 64,
+	[0][1][2][0][RTW89_ACMA][19] = 54,
+	[0][1][2][0][RTW89_CN][19] = 127,
+	[0][1][2][0][RTW89_UK][19] = 54,
 	[0][1][2][0][RTW89_FCC][21] = 62,
 	[0][1][2][0][RTW89_ETSI][21] = 54,
 	[0][1][2][0][RTW89_MKK][21] = 68,
-	[0][1][2][0][RTW89_IC][21] = 70,
-	[0][1][2][0][RTW89_ACMA][21] = 50,
+	[0][1][2][0][RTW89_IC][21] = 62,
+	[0][1][2][0][RTW89_KCC][21] = 64,
+	[0][1][2][0][RTW89_ACMA][21] = 54,
+	[0][1][2][0][RTW89_CN][21] = 127,
+	[0][1][2][0][RTW89_UK][21] = 54,
 	[0][1][2][0][RTW89_FCC][23] = 62,
 	[0][1][2][0][RTW89_ETSI][23] = 54,
 	[0][1][2][0][RTW89_MKK][23] = 68,
-	[0][1][2][0][RTW89_IC][23] = 70,
-	[0][1][2][0][RTW89_ACMA][23] = 50,
+	[0][1][2][0][RTW89_IC][23] = 62,
+	[0][1][2][0][RTW89_KCC][23] = 64,
+	[0][1][2][0][RTW89_ACMA][23] = 54,
+	[0][1][2][0][RTW89_CN][23] = 127,
+	[0][1][2][0][RTW89_UK][23] = 54,
 	[0][1][2][0][RTW89_FCC][25] = 62,
 	[0][1][2][0][RTW89_ETSI][25] = 54,
 	[0][1][2][0][RTW89_MKK][25] = 68,
 	[0][1][2][0][RTW89_IC][25] = 127,
+	[0][1][2][0][RTW89_KCC][25] = 64,
 	[0][1][2][0][RTW89_ACMA][25] = 127,
+	[0][1][2][0][RTW89_CN][25] = 127,
+	[0][1][2][0][RTW89_UK][25] = 54,
 	[0][1][2][0][RTW89_FCC][27] = 62,
 	[0][1][2][0][RTW89_ETSI][27] = 54,
 	[0][1][2][0][RTW89_MKK][27] = 68,
 	[0][1][2][0][RTW89_IC][27] = 127,
+	[0][1][2][0][RTW89_KCC][27] = 64,
 	[0][1][2][0][RTW89_ACMA][27] = 127,
+	[0][1][2][0][RTW89_CN][27] = 127,
+	[0][1][2][0][RTW89_UK][27] = 54,
 	[0][1][2][0][RTW89_FCC][29] = 62,
 	[0][1][2][0][RTW89_ETSI][29] = 54,
 	[0][1][2][0][RTW89_MKK][29] = 68,
 	[0][1][2][0][RTW89_IC][29] = 127,
+	[0][1][2][0][RTW89_KCC][29] = 64,
 	[0][1][2][0][RTW89_ACMA][29] = 127,
+	[0][1][2][0][RTW89_CN][29] = 127,
+	[0][1][2][0][RTW89_UK][29] = 54,
 	[0][1][2][0][RTW89_FCC][31] = 62,
 	[0][1][2][0][RTW89_ETSI][31] = 54,
 	[0][1][2][0][RTW89_MKK][31] = 68,
-	[0][1][2][0][RTW89_IC][31] = 70,
-	[0][1][2][0][RTW89_ACMA][31] = 50,
+	[0][1][2][0][RTW89_IC][31] = 62,
+	[0][1][2][0][RTW89_KCC][31] = 62,
+	[0][1][2][0][RTW89_ACMA][31] = 54,
+	[0][1][2][0][RTW89_CN][31] = 127,
+	[0][1][2][0][RTW89_UK][31] = 54,
 	[0][1][2][0][RTW89_FCC][33] = 62,
 	[0][1][2][0][RTW89_ETSI][33] = 54,
 	[0][1][2][0][RTW89_MKK][33] = 68,
-	[0][1][2][0][RTW89_IC][33] = 70,
-	[0][1][2][0][RTW89_ACMA][33] = 50,
-	[0][1][2][0][RTW89_FCC][35] = 58,
+	[0][1][2][0][RTW89_IC][33] = 62,
+	[0][1][2][0][RTW89_KCC][33] = 62,
+	[0][1][2][0][RTW89_ACMA][33] = 54,
+	[0][1][2][0][RTW89_CN][33] = 127,
+	[0][1][2][0][RTW89_UK][33] = 54,
+	[0][1][2][0][RTW89_FCC][35] = 46,
 	[0][1][2][0][RTW89_ETSI][35] = 54,
 	[0][1][2][0][RTW89_MKK][35] = 68,
-	[0][1][2][0][RTW89_IC][35] = 68,
-	[0][1][2][0][RTW89_ACMA][35] = 50,
-	[0][1][2][0][RTW89_FCC][37] = 62,
+	[0][1][2][0][RTW89_IC][35] = 46,
+	[0][1][2][0][RTW89_KCC][35] = 62,
+	[0][1][2][0][RTW89_ACMA][35] = 54,
+	[0][1][2][0][RTW89_CN][35] = 127,
+	[0][1][2][0][RTW89_UK][35] = 54,
+	[0][1][2][0][RTW89_FCC][37] = 64,
 	[0][1][2][0][RTW89_ETSI][37] = 127,
 	[0][1][2][0][RTW89_MKK][37] = 68,
-	[0][1][2][0][RTW89_IC][37] = 70,
-	[0][1][2][0][RTW89_ACMA][37] = 70,
-	[0][1][2][0][RTW89_FCC][38] = 70,
+	[0][1][2][0][RTW89_IC][37] = 64,
+	[0][1][2][0][RTW89_KCC][37] = 62,
+	[0][1][2][0][RTW89_ACMA][37] = 64,
+	[0][1][2][0][RTW89_CN][37] = 127,
+	[0][1][2][0][RTW89_UK][37] = 52,
+	[0][1][2][0][RTW89_FCC][38] = 72,
 	[0][1][2][0][RTW89_ETSI][38] = 18,
 	[0][1][2][0][RTW89_MKK][38] = 127,
-	[0][1][2][0][RTW89_IC][38] = 70,
+	[0][1][2][0][RTW89_IC][38] = 72,
+	[0][1][2][0][RTW89_KCC][38] = 56,
 	[0][1][2][0][RTW89_ACMA][38] = 70,
-	[0][1][2][0][RTW89_FCC][40] = 70,
+	[0][1][2][0][RTW89_CN][38] = 68,
+	[0][1][2][0][RTW89_UK][38] = 52,
+	[0][1][2][0][RTW89_FCC][40] = 72,
 	[0][1][2][0][RTW89_ETSI][40] = 18,
 	[0][1][2][0][RTW89_MKK][40] = 127,
-	[0][1][2][0][RTW89_IC][40] = 70,
+	[0][1][2][0][RTW89_IC][40] = 72,
+	[0][1][2][0][RTW89_KCC][40] = 56,
 	[0][1][2][0][RTW89_ACMA][40] = 70,
-	[0][1][2][0][RTW89_FCC][42] = 70,
+	[0][1][2][0][RTW89_CN][40] = 68,
+	[0][1][2][0][RTW89_UK][40] = 52,
+	[0][1][2][0][RTW89_FCC][42] = 72,
 	[0][1][2][0][RTW89_ETSI][42] = 18,
 	[0][1][2][0][RTW89_MKK][42] = 127,
-	[0][1][2][0][RTW89_IC][42] = 70,
+	[0][1][2][0][RTW89_IC][42] = 72,
+	[0][1][2][0][RTW89_KCC][42] = 56,
 	[0][1][2][0][RTW89_ACMA][42] = 70,
-	[0][1][2][0][RTW89_FCC][44] = 70,
+	[0][1][2][0][RTW89_CN][42] = 68,
+	[0][1][2][0][RTW89_UK][42] = 52,
+	[0][1][2][0][RTW89_FCC][44] = 72,
 	[0][1][2][0][RTW89_ETSI][44] = 18,
 	[0][1][2][0][RTW89_MKK][44] = 127,
-	[0][1][2][0][RTW89_IC][44] = 70,
+	[0][1][2][0][RTW89_IC][44] = 72,
+	[0][1][2][0][RTW89_KCC][44] = 56,
 	[0][1][2][0][RTW89_ACMA][44] = 70,
-	[0][1][2][0][RTW89_FCC][46] = 70,
+	[0][1][2][0][RTW89_CN][44] = 68,
+	[0][1][2][0][RTW89_UK][44] = 52,
+	[0][1][2][0][RTW89_FCC][46] = 72,
 	[0][1][2][0][RTW89_ETSI][46] = 18,
 	[0][1][2][0][RTW89_MKK][46] = 127,
-	[0][1][2][0][RTW89_IC][46] = 70,
+	[0][1][2][0][RTW89_IC][46] = 72,
+	[0][1][2][0][RTW89_KCC][46] = 56,
 	[0][1][2][0][RTW89_ACMA][46] = 70,
-	[0][1][2][0][RTW89_FCC][48] = 50,
+	[0][1][2][0][RTW89_CN][46] = 68,
+	[0][1][2][0][RTW89_UK][46] = 52,
+	[0][1][2][0][RTW89_FCC][48] = 48,
 	[0][1][2][0][RTW89_ETSI][48] = 127,
 	[0][1][2][0][RTW89_MKK][48] = 127,
 	[0][1][2][0][RTW89_IC][48] = 127,
+	[0][1][2][0][RTW89_KCC][48] = 127,
 	[0][1][2][0][RTW89_ACMA][48] = 127,
+	[0][1][2][0][RTW89_CN][48] = 127,
+	[0][1][2][0][RTW89_UK][48] = 127,
 	[0][1][2][0][RTW89_FCC][50] = 50,
 	[0][1][2][0][RTW89_ETSI][50] = 127,
 	[0][1][2][0][RTW89_MKK][50] = 127,
 	[0][1][2][0][RTW89_IC][50] = 127,
+	[0][1][2][0][RTW89_KCC][50] = 127,
 	[0][1][2][0][RTW89_ACMA][50] = 127,
-	[0][1][2][0][RTW89_FCC][52] = 50,
+	[0][1][2][0][RTW89_CN][50] = 127,
+	[0][1][2][0][RTW89_UK][50] = 127,
+	[0][1][2][0][RTW89_FCC][52] = 48,
 	[0][1][2][0][RTW89_ETSI][52] = 127,
 	[0][1][2][0][RTW89_MKK][52] = 127,
 	[0][1][2][0][RTW89_IC][52] = 127,
+	[0][1][2][0][RTW89_KCC][52] = 127,
 	[0][1][2][0][RTW89_ACMA][52] = 127,
+	[0][1][2][0][RTW89_CN][52] = 127,
+	[0][1][2][0][RTW89_UK][52] = 127,
 	[0][1][2][1][RTW89_FCC][0] = 60,
 	[0][1][2][1][RTW89_ETSI][0] = 40,
 	[0][1][2][1][RTW89_MKK][0] = 54,
-	[0][1][2][1][RTW89_IC][0] = 42,
-	[0][1][2][1][RTW89_ACMA][0] = 38,
-	[0][1][2][1][RTW89_FCC][2] = 60,
+	[0][1][2][1][RTW89_IC][0] = 40,
+	[0][1][2][1][RTW89_KCC][0] = 40,
+	[0][1][2][1][RTW89_ACMA][0] = 40,
+	[0][1][2][1][RTW89_CN][0] = 36,
+	[0][1][2][1][RTW89_UK][0] = 40,
+	[0][1][2][1][RTW89_FCC][2] = 62,
 	[0][1][2][1][RTW89_ETSI][2] = 40,
 	[0][1][2][1][RTW89_MKK][2] = 54,
-	[0][1][2][1][RTW89_IC][2] = 42,
-	[0][1][2][1][RTW89_ACMA][2] = 38,
-	[0][1][2][1][RTW89_FCC][4] = 60,
+	[0][1][2][1][RTW89_IC][2] = 40,
+	[0][1][2][1][RTW89_KCC][2] = 40,
+	[0][1][2][1][RTW89_ACMA][2] = 40,
+	[0][1][2][1][RTW89_CN][2] = 36,
+	[0][1][2][1][RTW89_UK][2] = 40,
+	[0][1][2][1][RTW89_FCC][4] = 62,
 	[0][1][2][1][RTW89_ETSI][4] = 40,
 	[0][1][2][1][RTW89_MKK][4] = 54,
-	[0][1][2][1][RTW89_IC][4] = 42,
-	[0][1][2][1][RTW89_ACMA][4] = 38,
-	[0][1][2][1][RTW89_FCC][6] = 60,
+	[0][1][2][1][RTW89_IC][4] = 40,
+	[0][1][2][1][RTW89_KCC][4] = 40,
+	[0][1][2][1][RTW89_ACMA][4] = 40,
+	[0][1][2][1][RTW89_CN][4] = 36,
+	[0][1][2][1][RTW89_UK][4] = 40,
+	[0][1][2][1][RTW89_FCC][6] = 62,
 	[0][1][2][1][RTW89_ETSI][6] = 40,
 	[0][1][2][1][RTW89_MKK][6] = 50,
-	[0][1][2][1][RTW89_IC][6] = 42,
-	[0][1][2][1][RTW89_ACMA][6] = 38,
-	[0][1][2][1][RTW89_FCC][8] = 60,
+	[0][1][2][1][RTW89_IC][6] = 40,
+	[0][1][2][1][RTW89_KCC][6] = 64,
+	[0][1][2][1][RTW89_ACMA][6] = 40,
+	[0][1][2][1][RTW89_CN][6] = 36,
+	[0][1][2][1][RTW89_UK][6] = 40,
+	[0][1][2][1][RTW89_FCC][8] = 62,
 	[0][1][2][1][RTW89_ETSI][8] = 40,
 	[0][1][2][1][RTW89_MKK][8] = 42,
-	[0][1][2][1][RTW89_IC][8] = 42,
-	[0][1][2][1][RTW89_ACMA][8] = 38,
-	[0][1][2][1][RTW89_FCC][10] = 60,
+	[0][1][2][1][RTW89_IC][8] = 40,
+	[0][1][2][1][RTW89_KCC][8] = 62,
+	[0][1][2][1][RTW89_ACMA][8] = 40,
+	[0][1][2][1][RTW89_CN][8] = 36,
+	[0][1][2][1][RTW89_UK][8] = 40,
+	[0][1][2][1][RTW89_FCC][10] = 62,
 	[0][1][2][1][RTW89_ETSI][10] = 40,
-	[0][1][2][1][RTW89_MKK][10] = 66,
-	[0][1][2][1][RTW89_IC][10] = 42,
-	[0][1][2][1][RTW89_ACMA][10] = 38,
-	[0][1][2][1][RTW89_FCC][12] = 60,
+	[0][1][2][1][RTW89_MKK][10] = 54,
+	[0][1][2][1][RTW89_IC][10] = 40,
+	[0][1][2][1][RTW89_KCC][10] = 62,
+	[0][1][2][1][RTW89_ACMA][10] = 40,
+	[0][1][2][1][RTW89_CN][10] = 36,
+	[0][1][2][1][RTW89_UK][10] = 40,
+	[0][1][2][1][RTW89_FCC][12] = 62,
 	[0][1][2][1][RTW89_ETSI][12] = 40,
-	[0][1][2][1][RTW89_MKK][12] = 66,
-	[0][1][2][1][RTW89_IC][12] = 42,
-	[0][1][2][1][RTW89_ACMA][12] = 38,
-	[0][1][2][1][RTW89_FCC][14] = 60,
+	[0][1][2][1][RTW89_MKK][12] = 54,
+	[0][1][2][1][RTW89_IC][12] = 40,
+	[0][1][2][1][RTW89_KCC][12] = 62,
+	[0][1][2][1][RTW89_ACMA][12] = 40,
+	[0][1][2][1][RTW89_CN][12] = 36,
+	[0][1][2][1][RTW89_UK][12] = 40,
+	[0][1][2][1][RTW89_FCC][14] = 62,
 	[0][1][2][1][RTW89_ETSI][14] = 40,
-	[0][1][2][1][RTW89_MKK][14] = 66,
-	[0][1][2][1][RTW89_IC][14] = 42,
-	[0][1][2][1][RTW89_ACMA][14] = 38,
+	[0][1][2][1][RTW89_MKK][14] = 54,
+	[0][1][2][1][RTW89_IC][14] = 40,
+	[0][1][2][1][RTW89_KCC][14] = 62,
+	[0][1][2][1][RTW89_ACMA][14] = 40,
+	[0][1][2][1][RTW89_CN][14] = 36,
+	[0][1][2][1][RTW89_UK][14] = 40,
 	[0][1][2][1][RTW89_FCC][15] = 60,
 	[0][1][2][1][RTW89_ETSI][15] = 40,
 	[0][1][2][1][RTW89_MKK][15] = 68,
-	[0][1][2][1][RTW89_IC][15] = 70,
-	[0][1][2][1][RTW89_ACMA][15] = 38,
-	[0][1][2][1][RTW89_FCC][17] = 60,
+	[0][1][2][1][RTW89_IC][15] = 60,
+	[0][1][2][1][RTW89_KCC][15] = 64,
+	[0][1][2][1][RTW89_ACMA][15] = 40,
+	[0][1][2][1][RTW89_CN][15] = 127,
+	[0][1][2][1][RTW89_UK][15] = 40,
+	[0][1][2][1][RTW89_FCC][17] = 62,
 	[0][1][2][1][RTW89_ETSI][17] = 40,
 	[0][1][2][1][RTW89_MKK][17] = 68,
-	[0][1][2][1][RTW89_IC][17] = 70,
-	[0][1][2][1][RTW89_ACMA][17] = 38,
-	[0][1][2][1][RTW89_FCC][19] = 60,
+	[0][1][2][1][RTW89_IC][17] = 62,
+	[0][1][2][1][RTW89_KCC][17] = 64,
+	[0][1][2][1][RTW89_ACMA][17] = 40,
+	[0][1][2][1][RTW89_CN][17] = 127,
+	[0][1][2][1][RTW89_UK][17] = 40,
+	[0][1][2][1][RTW89_FCC][19] = 62,
 	[0][1][2][1][RTW89_ETSI][19] = 40,
 	[0][1][2][1][RTW89_MKK][19] = 68,
-	[0][1][2][1][RTW89_IC][19] = 70,
-	[0][1][2][1][RTW89_ACMA][19] = 38,
-	[0][1][2][1][RTW89_FCC][21] = 60,
+	[0][1][2][1][RTW89_IC][19] = 62,
+	[0][1][2][1][RTW89_KCC][19] = 64,
+	[0][1][2][1][RTW89_ACMA][19] = 40,
+	[0][1][2][1][RTW89_CN][19] = 127,
+	[0][1][2][1][RTW89_UK][19] = 40,
+	[0][1][2][1][RTW89_FCC][21] = 62,
 	[0][1][2][1][RTW89_ETSI][21] = 40,
 	[0][1][2][1][RTW89_MKK][21] = 68,
-	[0][1][2][1][RTW89_IC][21] = 70,
-	[0][1][2][1][RTW89_ACMA][21] = 38,
-	[0][1][2][1][RTW89_FCC][23] = 60,
+	[0][1][2][1][RTW89_IC][21] = 62,
+	[0][1][2][1][RTW89_KCC][21] = 64,
+	[0][1][2][1][RTW89_ACMA][21] = 40,
+	[0][1][2][1][RTW89_CN][21] = 127,
+	[0][1][2][1][RTW89_UK][21] = 40,
+	[0][1][2][1][RTW89_FCC][23] = 62,
 	[0][1][2][1][RTW89_ETSI][23] = 40,
 	[0][1][2][1][RTW89_MKK][23] = 68,
-	[0][1][2][1][RTW89_IC][23] = 70,
-	[0][1][2][1][RTW89_ACMA][23] = 38,
-	[0][1][2][1][RTW89_FCC][25] = 58,
+	[0][1][2][1][RTW89_IC][23] = 62,
+	[0][1][2][1][RTW89_KCC][23] = 64,
+	[0][1][2][1][RTW89_ACMA][23] = 40,
+	[0][1][2][1][RTW89_CN][23] = 127,
+	[0][1][2][1][RTW89_UK][23] = 40,
+	[0][1][2][1][RTW89_FCC][25] = 46,
 	[0][1][2][1][RTW89_ETSI][25] = 40,
 	[0][1][2][1][RTW89_MKK][25] = 68,
 	[0][1][2][1][RTW89_IC][25] = 127,
+	[0][1][2][1][RTW89_KCC][25] = 64,
 	[0][1][2][1][RTW89_ACMA][25] = 127,
-	[0][1][2][1][RTW89_FCC][27] = 58,
+	[0][1][2][1][RTW89_CN][25] = 127,
+	[0][1][2][1][RTW89_UK][25] = 40,
+	[0][1][2][1][RTW89_FCC][27] = 46,
 	[0][1][2][1][RTW89_ETSI][27] = 40,
 	[0][1][2][1][RTW89_MKK][27] = 68,
 	[0][1][2][1][RTW89_IC][27] = 127,
+	[0][1][2][1][RTW89_KCC][27] = 64,
 	[0][1][2][1][RTW89_ACMA][27] = 127,
-	[0][1][2][1][RTW89_FCC][29] = 58,
+	[0][1][2][1][RTW89_CN][27] = 127,
+	[0][1][2][1][RTW89_UK][27] = 40,
+	[0][1][2][1][RTW89_FCC][29] = 46,
 	[0][1][2][1][RTW89_ETSI][29] = 40,
 	[0][1][2][1][RTW89_MKK][29] = 68,
 	[0][1][2][1][RTW89_IC][29] = 127,
+	[0][1][2][1][RTW89_KCC][29] = 64,
 	[0][1][2][1][RTW89_ACMA][29] = 127,
-	[0][1][2][1][RTW89_FCC][31] = 58,
+	[0][1][2][1][RTW89_CN][29] = 127,
+	[0][1][2][1][RTW89_UK][29] = 40,
+	[0][1][2][1][RTW89_FCC][31] = 46,
 	[0][1][2][1][RTW89_ETSI][31] = 40,
 	[0][1][2][1][RTW89_MKK][31] = 68,
-	[0][1][2][1][RTW89_IC][31] = 68,
-	[0][1][2][1][RTW89_ACMA][31] = 38,
-	[0][1][2][1][RTW89_FCC][33] = 58,
+	[0][1][2][1][RTW89_IC][31] = 46,
+	[0][1][2][1][RTW89_KCC][31] = 62,
+	[0][1][2][1][RTW89_ACMA][31] = 40,
+	[0][1][2][1][RTW89_CN][31] = 127,
+	[0][1][2][1][RTW89_UK][31] = 40,
+	[0][1][2][1][RTW89_FCC][33] = 46,
 	[0][1][2][1][RTW89_ETSI][33] = 40,
 	[0][1][2][1][RTW89_MKK][33] = 68,
-	[0][1][2][1][RTW89_IC][33] = 68,
-	[0][1][2][1][RTW89_ACMA][33] = 38,
-	[0][1][2][1][RTW89_FCC][35] = 58,
+	[0][1][2][1][RTW89_IC][33] = 46,
+	[0][1][2][1][RTW89_KCC][33] = 62,
+	[0][1][2][1][RTW89_ACMA][33] = 40,
+	[0][1][2][1][RTW89_CN][33] = 127,
+	[0][1][2][1][RTW89_UK][33] = 40,
+	[0][1][2][1][RTW89_FCC][35] = 46,
 	[0][1][2][1][RTW89_ETSI][35] = 40,
 	[0][1][2][1][RTW89_MKK][35] = 68,
-	[0][1][2][1][RTW89_IC][35] = 68,
-	[0][1][2][1][RTW89_ACMA][35] = 38,
-	[0][1][2][1][RTW89_FCC][37] = 60,
+	[0][1][2][1][RTW89_IC][35] = 46,
+	[0][1][2][1][RTW89_KCC][35] = 62,
+	[0][1][2][1][RTW89_ACMA][35] = 40,
+	[0][1][2][1][RTW89_CN][35] = 127,
+	[0][1][2][1][RTW89_UK][35] = 40,
+	[0][1][2][1][RTW89_FCC][37] = 64,
 	[0][1][2][1][RTW89_ETSI][37] = 127,
 	[0][1][2][1][RTW89_MKK][37] = 68,
-	[0][1][2][1][RTW89_IC][37] = 70,
-	[0][1][2][1][RTW89_ACMA][37] = 70,
-	[0][1][2][1][RTW89_FCC][38] = 70,
+	[0][1][2][1][RTW89_IC][37] = 64,
+	[0][1][2][1][RTW89_KCC][37] = 62,
+	[0][1][2][1][RTW89_ACMA][37] = 64,
+	[0][1][2][1][RTW89_CN][37] = 127,
+	[0][1][2][1][RTW89_UK][37] = 40,
+	[0][1][2][1][RTW89_FCC][38] = 72,
 	[0][1][2][1][RTW89_ETSI][38] = 6,
 	[0][1][2][1][RTW89_MKK][38] = 127,
-	[0][1][2][1][RTW89_IC][38] = 70,
+	[0][1][2][1][RTW89_IC][38] = 72,
+	[0][1][2][1][RTW89_KCC][38] = 56,
 	[0][1][2][1][RTW89_ACMA][38] = 70,
-	[0][1][2][1][RTW89_FCC][40] = 70,
+	[0][1][2][1][RTW89_CN][38] = 60,
+	[0][1][2][1][RTW89_UK][38] = 40,
+	[0][1][2][1][RTW89_FCC][40] = 72,
 	[0][1][2][1][RTW89_ETSI][40] = 6,
 	[0][1][2][1][RTW89_MKK][40] = 127,
-	[0][1][2][1][RTW89_IC][40] = 70,
+	[0][1][2][1][RTW89_IC][40] = 72,
+	[0][1][2][1][RTW89_KCC][40] = 56,
 	[0][1][2][1][RTW89_ACMA][40] = 70,
-	[0][1][2][1][RTW89_FCC][42] = 70,
+	[0][1][2][1][RTW89_CN][40] = 60,
+	[0][1][2][1][RTW89_UK][40] = 40,
+	[0][1][2][1][RTW89_FCC][42] = 72,
 	[0][1][2][1][RTW89_ETSI][42] = 6,
 	[0][1][2][1][RTW89_MKK][42] = 127,
-	[0][1][2][1][RTW89_IC][42] = 70,
+	[0][1][2][1][RTW89_IC][42] = 72,
+	[0][1][2][1][RTW89_KCC][42] = 56,
 	[0][1][2][1][RTW89_ACMA][42] = 70,
-	[0][1][2][1][RTW89_FCC][44] = 70,
+	[0][1][2][1][RTW89_CN][42] = 60,
+	[0][1][2][1][RTW89_UK][42] = 40,
+	[0][1][2][1][RTW89_FCC][44] = 72,
 	[0][1][2][1][RTW89_ETSI][44] = 6,
 	[0][1][2][1][RTW89_MKK][44] = 127,
-	[0][1][2][1][RTW89_IC][44] = 70,
+	[0][1][2][1][RTW89_IC][44] = 72,
+	[0][1][2][1][RTW89_KCC][44] = 56,
 	[0][1][2][1][RTW89_ACMA][44] = 70,
-	[0][1][2][1][RTW89_FCC][46] = 70,
+	[0][1][2][1][RTW89_CN][44] = 54,
+	[0][1][2][1][RTW89_UK][44] = 40,
+	[0][1][2][1][RTW89_FCC][46] = 72,
 	[0][1][2][1][RTW89_ETSI][46] = 6,
 	[0][1][2][1][RTW89_MKK][46] = 127,
-	[0][1][2][1][RTW89_IC][46] = 70,
+	[0][1][2][1][RTW89_IC][46] = 72,
+	[0][1][2][1][RTW89_KCC][46] = 56,
 	[0][1][2][1][RTW89_ACMA][46] = 70,
-	[0][1][2][1][RTW89_FCC][48] = 50,
+	[0][1][2][1][RTW89_CN][46] = 54,
+	[0][1][2][1][RTW89_UK][46] = 40,
+	[0][1][2][1][RTW89_FCC][48] = 48,
 	[0][1][2][1][RTW89_ETSI][48] = 127,
 	[0][1][2][1][RTW89_MKK][48] = 127,
 	[0][1][2][1][RTW89_IC][48] = 127,
+	[0][1][2][1][RTW89_KCC][48] = 127,
 	[0][1][2][1][RTW89_ACMA][48] = 127,
+	[0][1][2][1][RTW89_CN][48] = 127,
+	[0][1][2][1][RTW89_UK][48] = 127,
 	[0][1][2][1][RTW89_FCC][50] = 50,
 	[0][1][2][1][RTW89_ETSI][50] = 127,
 	[0][1][2][1][RTW89_MKK][50] = 127,
 	[0][1][2][1][RTW89_IC][50] = 127,
+	[0][1][2][1][RTW89_KCC][50] = 127,
 	[0][1][2][1][RTW89_ACMA][50] = 127,
-	[0][1][2][1][RTW89_FCC][52] = 50,
+	[0][1][2][1][RTW89_CN][50] = 127,
+	[0][1][2][1][RTW89_UK][50] = 127,
+	[0][1][2][1][RTW89_FCC][52] = 48,
 	[0][1][2][1][RTW89_ETSI][52] = 127,
 	[0][1][2][1][RTW89_MKK][52] = 127,
 	[0][1][2][1][RTW89_IC][52] = 127,
+	[0][1][2][1][RTW89_KCC][52] = 127,
 	[0][1][2][1][RTW89_ACMA][52] = 127,
-	[1][0][2][0][RTW89_FCC][1] = 58,
+	[0][1][2][1][RTW89_CN][52] = 127,
+	[0][1][2][1][RTW89_UK][52] = 127,
+	[1][0][2][0][RTW89_FCC][1] = 64,
 	[1][0][2][0][RTW89_ETSI][1] = 66,
 	[1][0][2][0][RTW89_MKK][1] = 66,
-	[1][0][2][0][RTW89_IC][1] = 66,
+	[1][0][2][0][RTW89_IC][1] = 62,
+	[1][0][2][0][RTW89_KCC][1] = 66,
 	[1][0][2][0][RTW89_ACMA][1] = 66,
+	[1][0][2][0][RTW89_CN][1] = 54,
+	[1][0][2][0][RTW89_UK][1] = 66,
 	[1][0][2][0][RTW89_FCC][5] = 68,
 	[1][0][2][0][RTW89_ETSI][5] = 66,
 	[1][0][2][0][RTW89_MKK][5] = 66,
-	[1][0][2][0][RTW89_IC][5] = 66,
+	[1][0][2][0][RTW89_IC][5] = 64,
+	[1][0][2][0][RTW89_KCC][5] = 54,
 	[1][0][2][0][RTW89_ACMA][5] = 66,
+	[1][0][2][0][RTW89_CN][5] = 54,
+	[1][0][2][0][RTW89_UK][5] = 66,
 	[1][0][2][0][RTW89_FCC][9] = 68,
 	[1][0][2][0][RTW89_ETSI][9] = 66,
 	[1][0][2][0][RTW89_MKK][9] = 66,
-	[1][0][2][0][RTW89_IC][9] = 66,
+	[1][0][2][0][RTW89_IC][9] = 64,
+	[1][0][2][0][RTW89_KCC][9] = 66,
 	[1][0][2][0][RTW89_ACMA][9] = 66,
-	[1][0][2][0][RTW89_FCC][13] = 58,
+	[1][0][2][0][RTW89_CN][9] = 54,
+	[1][0][2][0][RTW89_UK][9] = 66,
+	[1][0][2][0][RTW89_FCC][13] = 60,
 	[1][0][2][0][RTW89_ETSI][13] = 66,
 	[1][0][2][0][RTW89_MKK][13] = 66,
-	[1][0][2][0][RTW89_IC][13] = 66,
+	[1][0][2][0][RTW89_IC][13] = 60,
+	[1][0][2][0][RTW89_KCC][13] = 52,
 	[1][0][2][0][RTW89_ACMA][13] = 66,
-	[1][0][2][0][RTW89_FCC][16] = 56,
+	[1][0][2][0][RTW89_CN][13] = 54,
+	[1][0][2][0][RTW89_UK][13] = 66,
+	[1][0][2][0][RTW89_FCC][16] = 64,
 	[1][0][2][0][RTW89_ETSI][16] = 66,
 	[1][0][2][0][RTW89_MKK][16] = 66,
-	[1][0][2][0][RTW89_IC][16] = 66,
+	[1][0][2][0][RTW89_IC][16] = 64,
+	[1][0][2][0][RTW89_KCC][16] = 56,
 	[1][0][2][0][RTW89_ACMA][16] = 66,
+	[1][0][2][0][RTW89_CN][16] = 127,
+	[1][0][2][0][RTW89_UK][16] = 66,
 	[1][0][2][0][RTW89_FCC][20] = 68,
 	[1][0][2][0][RTW89_ETSI][20] = 66,
 	[1][0][2][0][RTW89_MKK][20] = 66,
-	[1][0][2][0][RTW89_IC][20] = 66,
+	[1][0][2][0][RTW89_IC][20] = 68,
+	[1][0][2][0][RTW89_KCC][20] = 56,
 	[1][0][2][0][RTW89_ACMA][20] = 66,
+	[1][0][2][0][RTW89_CN][20] = 127,
+	[1][0][2][0][RTW89_UK][20] = 66,
 	[1][0][2][0][RTW89_FCC][24] = 68,
 	[1][0][2][0][RTW89_ETSI][24] = 66,
 	[1][0][2][0][RTW89_MKK][24] = 66,
 	[1][0][2][0][RTW89_IC][24] = 127,
+	[1][0][2][0][RTW89_KCC][24] = 56,
 	[1][0][2][0][RTW89_ACMA][24] = 127,
+	[1][0][2][0][RTW89_CN][24] = 127,
+	[1][0][2][0][RTW89_UK][24] = 66,
 	[1][0][2][0][RTW89_FCC][28] = 68,
 	[1][0][2][0][RTW89_ETSI][28] = 66,
 	[1][0][2][0][RTW89_MKK][28] = 66,
 	[1][0][2][0][RTW89_IC][28] = 127,
+	[1][0][2][0][RTW89_KCC][28] = 66,
 	[1][0][2][0][RTW89_ACMA][28] = 127,
-	[1][0][2][0][RTW89_FCC][32] = 68,
+	[1][0][2][0][RTW89_CN][28] = 127,
+	[1][0][2][0][RTW89_UK][28] = 66,
+	[1][0][2][0][RTW89_FCC][32] = 62,
 	[1][0][2][0][RTW89_ETSI][32] = 66,
 	[1][0][2][0][RTW89_MKK][32] = 66,
-	[1][0][2][0][RTW89_IC][32] = 66,
+	[1][0][2][0][RTW89_IC][32] = 62,
+	[1][0][2][0][RTW89_KCC][32] = 66,
 	[1][0][2][0][RTW89_ACMA][32] = 66,
+	[1][0][2][0][RTW89_CN][32] = 127,
+	[1][0][2][0][RTW89_UK][32] = 66,
 	[1][0][2][0][RTW89_FCC][36] = 68,
 	[1][0][2][0][RTW89_ETSI][36] = 127,
 	[1][0][2][0][RTW89_MKK][36] = 66,
-	[1][0][2][0][RTW89_IC][36] = 66,
+	[1][0][2][0][RTW89_IC][36] = 68,
+	[1][0][2][0][RTW89_KCC][36] = 66,
 	[1][0][2][0][RTW89_ACMA][36] = 66,
+	[1][0][2][0][RTW89_CN][36] = 127,
+	[1][0][2][0][RTW89_UK][36] = 64,
 	[1][0][2][0][RTW89_FCC][39] = 68,
 	[1][0][2][0][RTW89_ETSI][39] = 30,
 	[1][0][2][0][RTW89_MKK][39] = 127,
-	[1][0][2][0][RTW89_IC][39] = 66,
+	[1][0][2][0][RTW89_IC][39] = 68,
+	[1][0][2][0][RTW89_KCC][39] = 66,
 	[1][0][2][0][RTW89_ACMA][39] = 66,
+	[1][0][2][0][RTW89_CN][39] = 62,
+	[1][0][2][0][RTW89_UK][39] = 64,
 	[1][0][2][0][RTW89_FCC][43] = 68,
 	[1][0][2][0][RTW89_ETSI][43] = 30,
 	[1][0][2][0][RTW89_MKK][43] = 127,
-	[1][0][2][0][RTW89_IC][43] = 66,
+	[1][0][2][0][RTW89_IC][43] = 68,
+	[1][0][2][0][RTW89_KCC][43] = 66,
 	[1][0][2][0][RTW89_ACMA][43] = 66,
+	[1][0][2][0][RTW89_CN][43] = 66,
+	[1][0][2][0][RTW89_UK][43] = 64,
 	[1][0][2][0][RTW89_FCC][47] = 68,
 	[1][0][2][0][RTW89_ETSI][47] = 127,
 	[1][0][2][0][RTW89_MKK][47] = 127,
 	[1][0][2][0][RTW89_IC][47] = 127,
+	[1][0][2][0][RTW89_KCC][47] = 127,
 	[1][0][2][0][RTW89_ACMA][47] = 127,
+	[1][0][2][0][RTW89_CN][47] = 127,
+	[1][0][2][0][RTW89_UK][47] = 127,
 	[1][0][2][0][RTW89_FCC][51] = 68,
 	[1][0][2][0][RTW89_ETSI][51] = 127,
 	[1][0][2][0][RTW89_MKK][51] = 127,
 	[1][0][2][0][RTW89_IC][51] = 127,
+	[1][0][2][0][RTW89_KCC][51] = 127,
 	[1][0][2][0][RTW89_ACMA][51] = 127,
+	[1][0][2][0][RTW89_CN][51] = 127,
+	[1][0][2][0][RTW89_UK][51] = 127,
 	[1][1][2][0][RTW89_FCC][1] = 54,
 	[1][1][2][0][RTW89_ETSI][1] = 54,
 	[1][1][2][0][RTW89_MKK][1] = 48,
-	[1][1][2][0][RTW89_IC][1] = 60,
-	[1][1][2][0][RTW89_ACMA][1] = 60,
+	[1][1][2][0][RTW89_IC][1] = 48,
+	[1][1][2][0][RTW89_KCC][1] = 54,
+	[1][1][2][0][RTW89_ACMA][1] = 54,
+	[1][1][2][0][RTW89_CN][1] = 42,
+	[1][1][2][0][RTW89_UK][1] = 54,
 	[1][1][2][0][RTW89_FCC][5] = 68,
 	[1][1][2][0][RTW89_ETSI][5] = 54,
 	[1][1][2][0][RTW89_MKK][5] = 52,
-	[1][1][2][0][RTW89_IC][5] = 60,
-	[1][1][2][0][RTW89_ACMA][5] = 60,
+	[1][1][2][0][RTW89_IC][5] = 48,
+	[1][1][2][0][RTW89_KCC][5] = 54,
+	[1][1][2][0][RTW89_ACMA][5] = 54,
+	[1][1][2][0][RTW89_CN][5] = 42,
+	[1][1][2][0][RTW89_UK][5] = 54,
 	[1][1][2][0][RTW89_FCC][9] = 68,
 	[1][1][2][0][RTW89_ETSI][9] = 54,
 	[1][1][2][0][RTW89_MKK][9] = 52,
-	[1][1][2][0][RTW89_IC][9] = 60,
-	[1][1][2][0][RTW89_ACMA][9] = 60,
+	[1][1][2][0][RTW89_IC][9] = 52,
+	[1][1][2][0][RTW89_KCC][9] = 64,
+	[1][1][2][0][RTW89_ACMA][9] = 54,
+	[1][1][2][0][RTW89_CN][9] = 42,
+	[1][1][2][0][RTW89_UK][9] = 54,
 	[1][1][2][0][RTW89_FCC][13] = 54,
 	[1][1][2][0][RTW89_ETSI][13] = 54,
 	[1][1][2][0][RTW89_MKK][13] = 52,
-	[1][1][2][0][RTW89_IC][13] = 60,
-	[1][1][2][0][RTW89_ACMA][13] = 60,
-	[1][1][2][0][RTW89_FCC][16] = 48,
+	[1][1][2][0][RTW89_IC][13] = 52,
+	[1][1][2][0][RTW89_KCC][13] = 52,
+	[1][1][2][0][RTW89_ACMA][13] = 54,
+	[1][1][2][0][RTW89_CN][13] = 42,
+	[1][1][2][0][RTW89_UK][13] = 54,
+	[1][1][2][0][RTW89_FCC][16] = 56,
 	[1][1][2][0][RTW89_ETSI][16] = 54,
 	[1][1][2][0][RTW89_MKK][16] = 66,
-	[1][1][2][0][RTW89_IC][16] = 58,
-	[1][1][2][0][RTW89_ACMA][16] = 60,
+	[1][1][2][0][RTW89_IC][16] = 56,
+	[1][1][2][0][RTW89_KCC][16] = 54,
+	[1][1][2][0][RTW89_ACMA][16] = 54,
+	[1][1][2][0][RTW89_CN][16] = 127,
+	[1][1][2][0][RTW89_UK][16] = 54,
 	[1][1][2][0][RTW89_FCC][20] = 68,
 	[1][1][2][0][RTW89_ETSI][20] = 54,
 	[1][1][2][0][RTW89_MKK][20] = 66,
-	[1][1][2][0][RTW89_IC][20] = 66,
-	[1][1][2][0][RTW89_ACMA][20] = 60,
+	[1][1][2][0][RTW89_IC][20] = 68,
+	[1][1][2][0][RTW89_KCC][20] = 54,
+	[1][1][2][0][RTW89_ACMA][20] = 54,
+	[1][1][2][0][RTW89_CN][20] = 127,
+	[1][1][2][0][RTW89_UK][20] = 54,
 	[1][1][2][0][RTW89_FCC][24] = 68,
 	[1][1][2][0][RTW89_ETSI][24] = 54,
 	[1][1][2][0][RTW89_MKK][24] = 66,
 	[1][1][2][0][RTW89_IC][24] = 127,
+	[1][1][2][0][RTW89_KCC][24] = 54,
 	[1][1][2][0][RTW89_ACMA][24] = 127,
+	[1][1][2][0][RTW89_CN][24] = 127,
+	[1][1][2][0][RTW89_UK][24] = 54,
 	[1][1][2][0][RTW89_FCC][28] = 68,
 	[1][1][2][0][RTW89_ETSI][28] = 54,
 	[1][1][2][0][RTW89_MKK][28] = 66,
 	[1][1][2][0][RTW89_IC][28] = 127,
+	[1][1][2][0][RTW89_KCC][28] = 66,
 	[1][1][2][0][RTW89_ACMA][28] = 127,
-	[1][1][2][0][RTW89_FCC][32] = 60,
+	[1][1][2][0][RTW89_CN][28] = 127,
+	[1][1][2][0][RTW89_UK][28] = 54,
+	[1][1][2][0][RTW89_FCC][32] = 56,
 	[1][1][2][0][RTW89_ETSI][32] = 54,
 	[1][1][2][0][RTW89_MKK][32] = 66,
-	[1][1][2][0][RTW89_IC][32] = 66,
+	[1][1][2][0][RTW89_IC][32] = 56,
+	[1][1][2][0][RTW89_KCC][32] = 66,
 	[1][1][2][0][RTW89_ACMA][32] = 54,
+	[1][1][2][0][RTW89_CN][32] = 127,
+	[1][1][2][0][RTW89_UK][32] = 54,
 	[1][1][2][0][RTW89_FCC][36] = 68,
 	[1][1][2][0][RTW89_ETSI][36] = 127,
 	[1][1][2][0][RTW89_MKK][36] = 66,
-	[1][1][2][0][RTW89_IC][36] = 66,
+	[1][1][2][0][RTW89_IC][36] = 68,
+	[1][1][2][0][RTW89_KCC][36] = 66,
 	[1][1][2][0][RTW89_ACMA][36] = 66,
+	[1][1][2][0][RTW89_CN][36] = 127,
+	[1][1][2][0][RTW89_UK][36] = 52,
 	[1][1][2][0][RTW89_FCC][39] = 68,
 	[1][1][2][0][RTW89_ETSI][39] = 18,
 	[1][1][2][0][RTW89_MKK][39] = 127,
-	[1][1][2][0][RTW89_IC][39] = 66,
+	[1][1][2][0][RTW89_IC][39] = 68,
+	[1][1][2][0][RTW89_KCC][39] = 56,
 	[1][1][2][0][RTW89_ACMA][39] = 66,
+	[1][1][2][0][RTW89_CN][39] = 62,
+	[1][1][2][0][RTW89_UK][39] = 52,
 	[1][1][2][0][RTW89_FCC][43] = 68,
 	[1][1][2][0][RTW89_ETSI][43] = 18,
 	[1][1][2][0][RTW89_MKK][43] = 127,
-	[1][1][2][0][RTW89_IC][43] = 66,
+	[1][1][2][0][RTW89_IC][43] = 68,
+	[1][1][2][0][RTW89_KCC][43] = 56,
 	[1][1][2][0][RTW89_ACMA][43] = 66,
-	[1][1][2][0][RTW89_FCC][47] = 60,
+	[1][1][2][0][RTW89_CN][43] = 66,
+	[1][1][2][0][RTW89_UK][43] = 52,
+	[1][1][2][0][RTW89_FCC][47] = 62,
 	[1][1][2][0][RTW89_ETSI][47] = 127,
 	[1][1][2][0][RTW89_MKK][47] = 127,
 	[1][1][2][0][RTW89_IC][47] = 127,
+	[1][1][2][0][RTW89_KCC][47] = 127,
 	[1][1][2][0][RTW89_ACMA][47] = 127,
-	[1][1][2][0][RTW89_FCC][51] = 58,
+	[1][1][2][0][RTW89_CN][47] = 127,
+	[1][1][2][0][RTW89_UK][47] = 127,
+	[1][1][2][0][RTW89_FCC][51] = 60,
 	[1][1][2][0][RTW89_ETSI][51] = 127,
 	[1][1][2][0][RTW89_MKK][51] = 127,
 	[1][1][2][0][RTW89_IC][51] = 127,
+	[1][1][2][0][RTW89_KCC][51] = 127,
 	[1][1][2][0][RTW89_ACMA][51] = 127,
+	[1][1][2][0][RTW89_CN][51] = 127,
+	[1][1][2][0][RTW89_UK][51] = 127,
 	[1][1][2][1][RTW89_FCC][1] = 54,
 	[1][1][2][1][RTW89_ETSI][1] = 40,
 	[1][1][2][1][RTW89_MKK][1] = 48,
-	[1][1][2][1][RTW89_IC][1] = 48,
-	[1][1][2][1][RTW89_ACMA][1] = 48,
-	[1][1][2][1][RTW89_FCC][5] = 60,
+	[1][1][2][1][RTW89_IC][1] = 40,
+	[1][1][2][1][RTW89_KCC][1] = 54,
+	[1][1][2][1][RTW89_ACMA][1] = 40,
+	[1][1][2][1][RTW89_CN][1] = 42,
+	[1][1][2][1][RTW89_UK][1] = 40,
+	[1][1][2][1][RTW89_FCC][5] = 68,
 	[1][1][2][1][RTW89_ETSI][5] = 40,
 	[1][1][2][1][RTW89_MKK][5] = 52,
-	[1][1][2][1][RTW89_IC][5] = 48,
-	[1][1][2][1][RTW89_ACMA][5] = 48,
-	[1][1][2][1][RTW89_FCC][9] = 60,
+	[1][1][2][1][RTW89_IC][5] = 40,
+	[1][1][2][1][RTW89_KCC][5] = 54,
+	[1][1][2][1][RTW89_ACMA][5] = 40,
+	[1][1][2][1][RTW89_CN][5] = 42,
+	[1][1][2][1][RTW89_UK][5] = 40,
+	[1][1][2][1][RTW89_FCC][9] = 68,
 	[1][1][2][1][RTW89_ETSI][9] = 40,
 	[1][1][2][1][RTW89_MKK][9] = 52,
-	[1][1][2][1][RTW89_IC][9] = 48,
-	[1][1][2][1][RTW89_ACMA][9] = 48,
+	[1][1][2][1][RTW89_IC][9] = 40,
+	[1][1][2][1][RTW89_KCC][9] = 64,
+	[1][1][2][1][RTW89_ACMA][9] = 40,
+	[1][1][2][1][RTW89_CN][9] = 42,
+	[1][1][2][1][RTW89_UK][9] = 40,
 	[1][1][2][1][RTW89_FCC][13] = 54,
 	[1][1][2][1][RTW89_ETSI][13] = 40,
 	[1][1][2][1][RTW89_MKK][13] = 52,
-	[1][1][2][1][RTW89_IC][13] = 48,
-	[1][1][2][1][RTW89_ACMA][13] = 48,
-	[1][1][2][1][RTW89_FCC][16] = 48,
+	[1][1][2][1][RTW89_IC][13] = 40,
+	[1][1][2][1][RTW89_KCC][13] = 52,
+	[1][1][2][1][RTW89_ACMA][13] = 40,
+	[1][1][2][1][RTW89_CN][13] = 42,
+	[1][1][2][1][RTW89_UK][13] = 40,
+	[1][1][2][1][RTW89_FCC][16] = 56,
 	[1][1][2][1][RTW89_ETSI][16] = 40,
 	[1][1][2][1][RTW89_MKK][16] = 66,
-	[1][1][2][1][RTW89_IC][16] = 58,
-	[1][1][2][1][RTW89_ACMA][16] = 48,
-	[1][1][2][1][RTW89_FCC][20] = 60,
+	[1][1][2][1][RTW89_IC][16] = 56,
+	[1][1][2][1][RTW89_KCC][16] = 54,
+	[1][1][2][1][RTW89_ACMA][16] = 40,
+	[1][1][2][1][RTW89_CN][16] = 127,
+	[1][1][2][1][RTW89_UK][16] = 40,
+	[1][1][2][1][RTW89_FCC][20] = 68,
 	[1][1][2][1][RTW89_ETSI][20] = 40,
 	[1][1][2][1][RTW89_MKK][20] = 66,
-	[1][1][2][1][RTW89_IC][20] = 66,
-	[1][1][2][1][RTW89_ACMA][20] = 48,
-	[1][1][2][1][RTW89_FCC][24] = 60,
+	[1][1][2][1][RTW89_IC][20] = 68,
+	[1][1][2][1][RTW89_KCC][20] = 54,
+	[1][1][2][1][RTW89_ACMA][20] = 40,
+	[1][1][2][1][RTW89_CN][20] = 127,
+	[1][1][2][1][RTW89_UK][20] = 40,
+	[1][1][2][1][RTW89_FCC][24] = 68,
 	[1][1][2][1][RTW89_ETSI][24] = 40,
 	[1][1][2][1][RTW89_MKK][24] = 66,
 	[1][1][2][1][RTW89_IC][24] = 127,
+	[1][1][2][1][RTW89_KCC][24] = 54,
 	[1][1][2][1][RTW89_ACMA][24] = 127,
-	[1][1][2][1][RTW89_FCC][28] = 60,
+	[1][1][2][1][RTW89_CN][24] = 127,
+	[1][1][2][1][RTW89_UK][24] = 40,
+	[1][1][2][1][RTW89_FCC][28] = 68,
 	[1][1][2][1][RTW89_ETSI][28] = 40,
 	[1][1][2][1][RTW89_MKK][28] = 66,
 	[1][1][2][1][RTW89_IC][28] = 127,
+	[1][1][2][1][RTW89_KCC][28] = 66,
 	[1][1][2][1][RTW89_ACMA][28] = 127,
-	[1][1][2][1][RTW89_FCC][32] = 60,
+	[1][1][2][1][RTW89_CN][28] = 127,
+	[1][1][2][1][RTW89_UK][28] = 40,
+	[1][1][2][1][RTW89_FCC][32] = 56,
 	[1][1][2][1][RTW89_ETSI][32] = 40,
 	[1][1][2][1][RTW89_MKK][32] = 66,
-	[1][1][2][1][RTW89_IC][32] = 66,
-	[1][1][2][1][RTW89_ACMA][32] = 42,
-	[1][1][2][1][RTW89_FCC][36] = 60,
+	[1][1][2][1][RTW89_IC][32] = 56,
+	[1][1][2][1][RTW89_KCC][32] = 66,
+	[1][1][2][1][RTW89_ACMA][32] = 40,
+	[1][1][2][1][RTW89_CN][32] = 127,
+	[1][1][2][1][RTW89_UK][32] = 40,
+	[1][1][2][1][RTW89_FCC][36] = 68,
 	[1][1][2][1][RTW89_ETSI][36] = 127,
 	[1][1][2][1][RTW89_MKK][36] = 66,
-	[1][1][2][1][RTW89_IC][36] = 66,
+	[1][1][2][1][RTW89_IC][36] = 68,
+	[1][1][2][1][RTW89_KCC][36] = 66,
 	[1][1][2][1][RTW89_ACMA][36] = 66,
+	[1][1][2][1][RTW89_CN][36] = 127,
+	[1][1][2][1][RTW89_UK][36] = 40,
 	[1][1][2][1][RTW89_FCC][39] = 68,
 	[1][1][2][1][RTW89_ETSI][39] = 6,
 	[1][1][2][1][RTW89_MKK][39] = 127,
-	[1][1][2][1][RTW89_IC][39] = 66,
+	[1][1][2][1][RTW89_IC][39] = 68,
+	[1][1][2][1][RTW89_KCC][39] = 56,
 	[1][1][2][1][RTW89_ACMA][39] = 66,
+	[1][1][2][1][RTW89_CN][39] = 60,
+	[1][1][2][1][RTW89_UK][39] = 40,
 	[1][1][2][1][RTW89_FCC][43] = 68,
 	[1][1][2][1][RTW89_ETSI][43] = 6,
 	[1][1][2][1][RTW89_MKK][43] = 127,
-	[1][1][2][1][RTW89_IC][43] = 66,
+	[1][1][2][1][RTW89_IC][43] = 68,
+	[1][1][2][1][RTW89_KCC][43] = 56,
 	[1][1][2][1][RTW89_ACMA][43] = 66,
-	[1][1][2][1][RTW89_FCC][47] = 60,
+	[1][1][2][1][RTW89_CN][43] = 52,
+	[1][1][2][1][RTW89_UK][43] = 40,
+	[1][1][2][1][RTW89_FCC][47] = 62,
 	[1][1][2][1][RTW89_ETSI][47] = 127,
 	[1][1][2][1][RTW89_MKK][47] = 127,
 	[1][1][2][1][RTW89_IC][47] = 127,
+	[1][1][2][1][RTW89_KCC][47] = 127,
 	[1][1][2][1][RTW89_ACMA][47] = 127,
-	[1][1][2][1][RTW89_FCC][51] = 58,
+	[1][1][2][1][RTW89_CN][47] = 127,
+	[1][1][2][1][RTW89_UK][47] = 127,
+	[1][1][2][1][RTW89_FCC][51] = 60,
 	[1][1][2][1][RTW89_ETSI][51] = 127,
 	[1][1][2][1][RTW89_MKK][51] = 127,
 	[1][1][2][1][RTW89_IC][51] = 127,
+	[1][1][2][1][RTW89_KCC][51] = 127,
 	[1][1][2][1][RTW89_ACMA][51] = 127,
-	[2][0][2][0][RTW89_FCC][3] = 56,
+	[1][1][2][1][RTW89_CN][51] = 127,
+	[1][1][2][1][RTW89_UK][51] = 127,
+	[2][0][2][0][RTW89_FCC][3] = 58,
 	[2][0][2][0][RTW89_ETSI][3] = 60,
 	[2][0][2][0][RTW89_MKK][3] = 60,
-	[2][0][2][0][RTW89_IC][3] = 60,
+	[2][0][2][0][RTW89_IC][3] = 56,
+	[2][0][2][0][RTW89_KCC][3] = 60,
 	[2][0][2][0][RTW89_ACMA][3] = 60,
-	[2][0][2][0][RTW89_FCC][11] = 58,
+	[2][0][2][0][RTW89_CN][3] = 54,
+	[2][0][2][0][RTW89_UK][3] = 60,
+	[2][0][2][0][RTW89_FCC][11] = 50,
 	[2][0][2][0][RTW89_ETSI][11] = 60,
 	[2][0][2][0][RTW89_MKK][11] = 60,
-	[2][0][2][0][RTW89_IC][11] = 60,
+	[2][0][2][0][RTW89_IC][11] = 50,
+	[2][0][2][0][RTW89_KCC][11] = 58,
 	[2][0][2][0][RTW89_ACMA][11] = 60,
-	[2][0][2][0][RTW89_FCC][18] = 54,
+	[2][0][2][0][RTW89_CN][11] = 54,
+	[2][0][2][0][RTW89_UK][11] = 60,
+	[2][0][2][0][RTW89_FCC][18] = 60,
 	[2][0][2][0][RTW89_ETSI][18] = 60,
 	[2][0][2][0][RTW89_MKK][18] = 60,
 	[2][0][2][0][RTW89_IC][18] = 60,
+	[2][0][2][0][RTW89_KCC][18] = 56,
 	[2][0][2][0][RTW89_ACMA][18] = 60,
+	[2][0][2][0][RTW89_CN][18] = 127,
+	[2][0][2][0][RTW89_UK][18] = 60,
 	[2][0][2][0][RTW89_FCC][26] = 62,
 	[2][0][2][0][RTW89_ETSI][26] = 60,
 	[2][0][2][0][RTW89_MKK][26] = 60,
 	[2][0][2][0][RTW89_IC][26] = 127,
+	[2][0][2][0][RTW89_KCC][26] = 60,
 	[2][0][2][0][RTW89_ACMA][26] = 127,
+	[2][0][2][0][RTW89_CN][26] = 127,
+	[2][0][2][0][RTW89_UK][26] = 60,
 	[2][0][2][0][RTW89_FCC][34] = 62,
 	[2][0][2][0][RTW89_ETSI][34] = 127,
 	[2][0][2][0][RTW89_MKK][34] = 60,
-	[2][0][2][0][RTW89_IC][34] = 60,
+	[2][0][2][0][RTW89_IC][34] = 62,
+	[2][0][2][0][RTW89_KCC][34] = 60,
 	[2][0][2][0][RTW89_ACMA][34] = 60,
+	[2][0][2][0][RTW89_CN][34] = 127,
+	[2][0][2][0][RTW89_UK][34] = 60,
 	[2][0][2][0][RTW89_FCC][41] = 62,
 	[2][0][2][0][RTW89_ETSI][41] = 30,
 	[2][0][2][0][RTW89_MKK][41] = 127,
-	[2][0][2][0][RTW89_IC][41] = 60,
+	[2][0][2][0][RTW89_IC][41] = 62,
+	[2][0][2][0][RTW89_KCC][41] = 58,
 	[2][0][2][0][RTW89_ACMA][41] = 60,
-	[2][0][2][0][RTW89_FCC][49] = 56,
+	[2][0][2][0][RTW89_CN][41] = 62,
+	[2][0][2][0][RTW89_UK][41] = 60,
+	[2][0][2][0][RTW89_FCC][49] = 62,
 	[2][0][2][0][RTW89_ETSI][49] = 127,
 	[2][0][2][0][RTW89_MKK][49] = 127,
 	[2][0][2][0][RTW89_IC][49] = 127,
+	[2][0][2][0][RTW89_KCC][49] = 127,
 	[2][0][2][0][RTW89_ACMA][49] = 127,
+	[2][0][2][0][RTW89_CN][49] = 127,
+	[2][0][2][0][RTW89_UK][49] = 127,
 	[2][1][2][0][RTW89_FCC][3] = 48,
 	[2][1][2][0][RTW89_ETSI][3] = 54,
 	[2][1][2][0][RTW89_MKK][3] = 56,
-	[2][1][2][0][RTW89_IC][3] = 52,
-	[2][1][2][0][RTW89_ACMA][3] = 52,
-	[2][1][2][0][RTW89_FCC][11] = 54,
+	[2][1][2][0][RTW89_IC][3] = 46,
+	[2][1][2][0][RTW89_KCC][3] = 56,
+	[2][1][2][0][RTW89_ACMA][3] = 54,
+	[2][1][2][0][RTW89_CN][3] = 52,
+	[2][1][2][0][RTW89_UK][3] = 54,
+	[2][1][2][0][RTW89_FCC][11] = 38,
 	[2][1][2][0][RTW89_ETSI][11] = 54,
 	[2][1][2][0][RTW89_MKK][11] = 54,
-	[2][1][2][0][RTW89_IC][11] = 52,
-	[2][1][2][0][RTW89_ACMA][11] = 52,
-	[2][1][2][0][RTW89_FCC][18] = 48,
+	[2][1][2][0][RTW89_IC][11] = 38,
+	[2][1][2][0][RTW89_KCC][11] = 52,
+	[2][1][2][0][RTW89_ACMA][11] = 54,
+	[2][1][2][0][RTW89_CN][11] = 52,
+	[2][1][2][0][RTW89_UK][11] = 54,
+	[2][1][2][0][RTW89_FCC][18] = 50,
 	[2][1][2][0][RTW89_ETSI][18] = 54,
 	[2][1][2][0][RTW89_MKK][18] = 60,
-	[2][1][2][0][RTW89_IC][18] = 58,
-	[2][1][2][0][RTW89_ACMA][18] = 52,
-	[2][1][2][0][RTW89_FCC][26] = 62,
+	[2][1][2][0][RTW89_IC][18] = 50,
+	[2][1][2][0][RTW89_KCC][18] = 54,
+	[2][1][2][0][RTW89_ACMA][18] = 54,
+	[2][1][2][0][RTW89_CN][18] = 127,
+	[2][1][2][0][RTW89_UK][18] = 54,
+	[2][1][2][0][RTW89_FCC][26] = 52,
 	[2][1][2][0][RTW89_ETSI][26] = 54,
 	[2][1][2][0][RTW89_MKK][26] = 56,
 	[2][1][2][0][RTW89_IC][26] = 127,
+	[2][1][2][0][RTW89_KCC][26] = 60,
 	[2][1][2][0][RTW89_ACMA][26] = 127,
+	[2][1][2][0][RTW89_CN][26] = 127,
+	[2][1][2][0][RTW89_UK][26] = 54,
 	[2][1][2][0][RTW89_FCC][34] = 62,
 	[2][1][2][0][RTW89_ETSI][34] = 127,
 	[2][1][2][0][RTW89_MKK][34] = 60,
-	[2][1][2][0][RTW89_IC][34] = 60,
+	[2][1][2][0][RTW89_IC][34] = 62,
+	[2][1][2][0][RTW89_KCC][34] = 60,
 	[2][1][2][0][RTW89_ACMA][34] = 60,
-	[2][1][2][0][RTW89_FCC][41] = 62,
+	[2][1][2][0][RTW89_CN][34] = 127,
+	[2][1][2][0][RTW89_UK][34] = 52,
+	[2][1][2][0][RTW89_FCC][41] = 60,
 	[2][1][2][0][RTW89_ETSI][41] = 18,
 	[2][1][2][0][RTW89_MKK][41] = 127,
 	[2][1][2][0][RTW89_IC][41] = 60,
-	[2][1][2][0][RTW89_ACMA][41] = 60,
-	[2][1][2][0][RTW89_FCC][49] = 50,
+	[2][1][2][0][RTW89_KCC][41] = 50,
+	[2][1][2][0][RTW89_ACMA][41] = 58,
+	[2][1][2][0][RTW89_CN][41] = 62,
+	[2][1][2][0][RTW89_UK][41] = 52,
+	[2][1][2][0][RTW89_FCC][49] = 62,
 	[2][1][2][0][RTW89_ETSI][49] = 127,
 	[2][1][2][0][RTW89_MKK][49] = 127,
 	[2][1][2][0][RTW89_IC][49] = 127,
+	[2][1][2][0][RTW89_KCC][49] = 127,
 	[2][1][2][0][RTW89_ACMA][49] = 127,
+	[2][1][2][0][RTW89_CN][49] = 127,
+	[2][1][2][0][RTW89_UK][49] = 127,
 	[2][1][2][1][RTW89_FCC][3] = 48,
 	[2][1][2][1][RTW89_ETSI][3] = 40,
 	[2][1][2][1][RTW89_MKK][3] = 56,
 	[2][1][2][1][RTW89_IC][3] = 40,
+	[2][1][2][1][RTW89_KCC][3] = 56,
 	[2][1][2][1][RTW89_ACMA][3] = 40,
-	[2][1][2][1][RTW89_FCC][11] = 54,
+	[2][1][2][1][RTW89_CN][3] = 42,
+	[2][1][2][1][RTW89_UK][3] = 40,
+	[2][1][2][1][RTW89_FCC][11] = 38,
 	[2][1][2][1][RTW89_ETSI][11] = 40,
 	[2][1][2][1][RTW89_MKK][11] = 54,
-	[2][1][2][1][RTW89_IC][11] = 40,
+	[2][1][2][1][RTW89_IC][11] = 38,
+	[2][1][2][1][RTW89_KCC][11] = 52,
 	[2][1][2][1][RTW89_ACMA][11] = 40,
-	[2][1][2][1][RTW89_FCC][18] = 48,
+	[2][1][2][1][RTW89_CN][11] = 42,
+	[2][1][2][1][RTW89_UK][11] = 40,
+	[2][1][2][1][RTW89_FCC][18] = 50,
 	[2][1][2][1][RTW89_ETSI][18] = 40,
 	[2][1][2][1][RTW89_MKK][18] = 60,
-	[2][1][2][1][RTW89_IC][18] = 58,
+	[2][1][2][1][RTW89_IC][18] = 50,
+	[2][1][2][1][RTW89_KCC][18] = 54,
 	[2][1][2][1][RTW89_ACMA][18] = 40,
-	[2][1][2][1][RTW89_FCC][26] = 60,
+	[2][1][2][1][RTW89_CN][18] = 127,
+	[2][1][2][1][RTW89_UK][18] = 40,
+	[2][1][2][1][RTW89_FCC][26] = 52,
 	[2][1][2][1][RTW89_ETSI][26] = 42,
 	[2][1][2][1][RTW89_MKK][26] = 56,
 	[2][1][2][1][RTW89_IC][26] = 127,
+	[2][1][2][1][RTW89_KCC][26] = 60,
 	[2][1][2][1][RTW89_ACMA][26] = 127,
-	[2][1][2][1][RTW89_FCC][34] = 60,
+	[2][1][2][1][RTW89_CN][26] = 127,
+	[2][1][2][1][RTW89_UK][26] = 42,
+	[2][1][2][1][RTW89_FCC][34] = 62,
 	[2][1][2][1][RTW89_ETSI][34] = 127,
 	[2][1][2][1][RTW89_MKK][34] = 60,
-	[2][1][2][1][RTW89_IC][34] = 60,
+	[2][1][2][1][RTW89_IC][34] = 62,
+	[2][1][2][1][RTW89_KCC][34] = 60,
 	[2][1][2][1][RTW89_ACMA][34] = 60,
-	[2][1][2][1][RTW89_FCC][41] = 62,
+	[2][1][2][1][RTW89_CN][34] = 127,
+	[2][1][2][1][RTW89_UK][34] = 40,
+	[2][1][2][1][RTW89_FCC][41] = 60,
 	[2][1][2][1][RTW89_ETSI][41] = 6,
 	[2][1][2][1][RTW89_MKK][41] = 127,
 	[2][1][2][1][RTW89_IC][41] = 60,
-	[2][1][2][1][RTW89_ACMA][41] = 60,
-	[2][1][2][1][RTW89_FCC][49] = 50,
+	[2][1][2][1][RTW89_KCC][41] = 50,
+	[2][1][2][1][RTW89_ACMA][41] = 58,
+	[2][1][2][1][RTW89_CN][41] = 40,
+	[2][1][2][1][RTW89_UK][41] = 40,
+	[2][1][2][1][RTW89_FCC][49] = 62,
 	[2][1][2][1][RTW89_ETSI][49] = 127,
 	[2][1][2][1][RTW89_MKK][49] = 127,
 	[2][1][2][1][RTW89_IC][49] = 127,
+	[2][1][2][1][RTW89_KCC][49] = 127,
 	[2][1][2][1][RTW89_ACMA][49] = 127,
-	[3][0][2][0][RTW89_FCC][7] = 38,
+	[2][1][2][1][RTW89_CN][49] = 127,
+	[2][1][2][1][RTW89_UK][49] = 127,
+	[3][0][2][0][RTW89_FCC][7] = 40,
 	[3][0][2][0][RTW89_ETSI][7] = 50,
 	[3][0][2][0][RTW89_MKK][7] = 50,
-	[3][0][2][0][RTW89_IC][7] = 50,
-	[3][0][2][0][RTW89_ACMA][7] = 50,
-	[3][0][2][0][RTW89_FCC][22] = 52,
+	[3][0][2][0][RTW89_IC][7] = 40,
+	[3][0][2][0][RTW89_KCC][7] = 44,
+	[3][0][2][0][RTW89_ACMA][7] = 127,
+	[3][0][2][0][RTW89_CN][7] = 66,
+	[3][0][2][0][RTW89_UK][7] = 127,
+	[3][0][2][0][RTW89_FCC][22] = 42,
 	[3][0][2][0][RTW89_ETSI][22] = 50,
 	[3][0][2][0][RTW89_MKK][22] = 50,
-	[3][0][2][0][RTW89_IC][22] = 50,
-	[3][0][2][0][RTW89_ACMA][22] = 50,
-	[3][0][2][0][RTW89_FCC][45] = 127,
+	[3][0][2][0][RTW89_IC][22] = 127,
+	[3][0][2][0][RTW89_KCC][22] = 50,
+	[3][0][2][0][RTW89_ACMA][22] = 127,
+	[3][0][2][0][RTW89_CN][22] = 66,
+	[3][0][2][0][RTW89_UK][22] = 127,
+	[3][0][2][0][RTW89_FCC][45] = 52,
 	[3][0][2][0][RTW89_ETSI][45] = 127,
 	[3][0][2][0][RTW89_MKK][45] = 127,
 	[3][0][2][0][RTW89_IC][45] = 127,
+	[3][0][2][0][RTW89_KCC][45] = 127,
 	[3][0][2][0][RTW89_ACMA][45] = 127,
-	[3][1][2][0][RTW89_FCC][7] = 26,
+	[3][0][2][0][RTW89_CN][45] = 127,
+	[3][0][2][0][RTW89_UK][45] = 127,
+	[3][1][2][0][RTW89_FCC][7] = 32,
 	[3][1][2][0][RTW89_ETSI][7] = 50,
 	[3][1][2][0][RTW89_MKK][7] = 36,
 	[3][1][2][0][RTW89_IC][7] = 44,
-	[3][1][2][0][RTW89_ACMA][7] = 44,
-	[3][1][2][0][RTW89_FCC][22] = 42,
+	[3][1][2][0][RTW89_KCC][7] = 50,
+	[3][1][2][0][RTW89_ACMA][7] = 127,
+	[3][1][2][0][RTW89_CN][7] = 54,
+	[3][1][2][0][RTW89_UK][7] = 127,
+	[3][1][2][0][RTW89_FCC][22] = 36,
 	[3][1][2][0][RTW89_ETSI][22] = 50,
 	[3][1][2][0][RTW89_MKK][22] = 48,
-	[3][1][2][0][RTW89_IC][22] = 44,
-	[3][1][2][0][RTW89_ACMA][22] = 44,
-	[3][1][2][0][RTW89_FCC][45] = 127,
+	[3][1][2][0][RTW89_IC][22] = 127,
+	[3][1][2][0][RTW89_KCC][22] = 50,
+	[3][1][2][0][RTW89_ACMA][22] = 127,
+	[3][1][2][0][RTW89_CN][22] = 54,
+	[3][1][2][0][RTW89_UK][22] = 127,
+	[3][1][2][0][RTW89_FCC][45] = 46,
 	[3][1][2][0][RTW89_ETSI][45] = 127,
 	[3][1][2][0][RTW89_MKK][45] = 127,
 	[3][1][2][0][RTW89_IC][45] = 127,
+	[3][1][2][0][RTW89_KCC][45] = 127,
 	[3][1][2][0][RTW89_ACMA][45] = 127,
-	[3][1][2][1][RTW89_FCC][7] = 14,
+	[3][1][2][0][RTW89_CN][45] = 127,
+	[3][1][2][0][RTW89_UK][45] = 127,
+	[3][1][2][1][RTW89_FCC][7] = 32,
 	[3][1][2][1][RTW89_ETSI][7] = 42,
 	[3][1][2][1][RTW89_MKK][7] = 36,
-	[3][1][2][1][RTW89_IC][7] = 32,
-	[3][1][2][1][RTW89_ACMA][7] = 32,
-	[3][1][2][1][RTW89_FCC][22] = 30,
+	[3][1][2][1][RTW89_IC][7] = 44,
+	[3][1][2][1][RTW89_KCC][7] = 50,
+	[3][1][2][1][RTW89_ACMA][7] = 127,
+	[3][1][2][1][RTW89_CN][7] = 42,
+	[3][1][2][1][RTW89_UK][7] = 127,
+	[3][1][2][1][RTW89_FCC][22] = 36,
 	[3][1][2][1][RTW89_ETSI][22] = 42,
 	[3][1][2][1][RTW89_MKK][22] = 48,
-	[3][1][2][1][RTW89_IC][22] = 32,
-	[3][1][2][1][RTW89_ACMA][22] = 32,
-	[3][1][2][1][RTW89_FCC][45] = 127,
+	[3][1][2][1][RTW89_IC][22] = 127,
+	[3][1][2][1][RTW89_KCC][22] = 50,
+	[3][1][2][1][RTW89_ACMA][22] = 127,
+	[3][1][2][1][RTW89_CN][22] = 42,
+	[3][1][2][1][RTW89_UK][22] = 127,
+	[3][1][2][1][RTW89_FCC][45] = 46,
 	[3][1][2][1][RTW89_ETSI][45] = 127,
 	[3][1][2][1][RTW89_MKK][45] = 127,
 	[3][1][2][1][RTW89_IC][45] = 127,
+	[3][1][2][1][RTW89_KCC][45] = 127,
 	[3][1][2][1][RTW89_ACMA][45] = 127,
+	[3][1][2][1][RTW89_CN][45] = 127,
+	[3][1][2][1][RTW89_UK][45] = 127,
 };
 
 const s8 rtw89_8852c_txpwr_lmt_6g[RTW89_6G_BW_NUM][RTW89_NTX_NUM]
 				 [RTW89_RS_LMT_NUM][RTW89_BF_NUM]
 				 [RTW89_REGD_NUM][RTW89_6G_CH_NUM] = {
-	[0][0][1][0][RTW89_WW][0] = 72,
-	[0][0][1][0][RTW89_WW][2] = 72,
-	[0][0][1][0][RTW89_WW][4] = 72,
-	[0][0][1][0][RTW89_WW][6] = 72,
-	[0][0][1][0][RTW89_WW][8] = 72,
-	[0][0][1][0][RTW89_WW][10] = 72,
-	[0][0][1][0][RTW89_WW][12] = 72,
-	[0][0][1][0][RTW89_WW][14] = 72,
-	[0][0][1][0][RTW89_WW][15] = 72,
-	[0][0][1][0][RTW89_WW][17] = 72,
-	[0][0][1][0][RTW89_WW][19] = 72,
-	[0][0][1][0][RTW89_WW][21] = 72,
-	[0][0][1][0][RTW89_WW][23] = 72,
-	[0][0][1][0][RTW89_WW][25] = 72,
-	[0][0][1][0][RTW89_WW][27] = 72,
-	[0][0][1][0][RTW89_WW][29] = 72,
-	[0][0][1][0][RTW89_WW][30] = 72,
-	[0][0][1][0][RTW89_WW][32] = 72,
-	[0][0][1][0][RTW89_WW][34] = 72,
-	[0][0][1][0][RTW89_WW][36] = 72,
-	[0][0][1][0][RTW89_WW][38] = 72,
-	[0][0][1][0][RTW89_WW][40] = 72,
-	[0][0][1][0][RTW89_WW][42] = 72,
-	[0][0][1][0][RTW89_WW][44] = 72,
-	[0][0][1][0][RTW89_WW][45] = 72,
-	[0][0][1][0][RTW89_WW][47] = 72,
-	[0][0][1][0][RTW89_WW][49] = 72,
-	[0][0][1][0][RTW89_WW][51] = 72,
-	[0][0][1][0][RTW89_WW][53] = 72,
-	[0][0][1][0][RTW89_WW][55] = 72,
-	[0][0][1][0][RTW89_WW][57] = 72,
-	[0][0][1][0][RTW89_WW][59] = 72,
-	[0][0][1][0][RTW89_WW][60] = 72,
-	[0][0][1][0][RTW89_WW][62] = 72,
-	[0][0][1][0][RTW89_WW][64] = 72,
-	[0][0][1][0][RTW89_WW][66] = 72,
-	[0][0][1][0][RTW89_WW][68] = 72,
-	[0][0][1][0][RTW89_WW][70] = 72,
-	[0][0][1][0][RTW89_WW][72] = 72,
-	[0][0][1][0][RTW89_WW][74] = 72,
-	[0][0][1][0][RTW89_WW][75] = 72,
-	[0][0][1][0][RTW89_WW][77] = 72,
-	[0][0][1][0][RTW89_WW][79] = 72,
-	[0][0][1][0][RTW89_WW][81] = 72,
-	[0][0][1][0][RTW89_WW][83] = 72,
-	[0][0][1][0][RTW89_WW][85] = 72,
-	[0][0][1][0][RTW89_WW][87] = 72,
-	[0][0][1][0][RTW89_WW][89] = 72,
-	[0][0][1][0][RTW89_WW][90] = 72,
-	[0][0][1][0][RTW89_WW][92] = 72,
-	[0][0][1][0][RTW89_WW][94] = 72,
-	[0][0][1][0][RTW89_WW][96] = 72,
-	[0][0][1][0][RTW89_WW][98] = 72,
-	[0][0][1][0][RTW89_WW][100] = 72,
-	[0][0][1][0][RTW89_WW][102] = 72,
-	[0][0][1][0][RTW89_WW][104] = 72,
-	[0][0][1][0][RTW89_WW][105] = 72,
-	[0][0][1][0][RTW89_WW][107] = 72,
-	[0][0][1][0][RTW89_WW][109] = 72,
+	[0][0][1][0][RTW89_WW][0] = 24,
+	[0][0][1][0][RTW89_WW][2] = 22,
+	[0][0][1][0][RTW89_WW][4] = 22,
+	[0][0][1][0][RTW89_WW][6] = 22,
+	[0][0][1][0][RTW89_WW][8] = 22,
+	[0][0][1][0][RTW89_WW][10] = 22,
+	[0][0][1][0][RTW89_WW][12] = 22,
+	[0][0][1][0][RTW89_WW][14] = 22,
+	[0][0][1][0][RTW89_WW][15] = 22,
+	[0][0][1][0][RTW89_WW][17] = 22,
+	[0][0][1][0][RTW89_WW][19] = 22,
+	[0][0][1][0][RTW89_WW][21] = 22,
+	[0][0][1][0][RTW89_WW][23] = 22,
+	[0][0][1][0][RTW89_WW][25] = 22,
+	[0][0][1][0][RTW89_WW][27] = 22,
+	[0][0][1][0][RTW89_WW][29] = 22,
+	[0][0][1][0][RTW89_WW][30] = 22,
+	[0][0][1][0][RTW89_WW][32] = 22,
+	[0][0][1][0][RTW89_WW][34] = 22,
+	[0][0][1][0][RTW89_WW][36] = 22,
+	[0][0][1][0][RTW89_WW][38] = 22,
+	[0][0][1][0][RTW89_WW][40] = 22,
+	[0][0][1][0][RTW89_WW][42] = 22,
+	[0][0][1][0][RTW89_WW][44] = 22,
+	[0][0][1][0][RTW89_WW][45] = 22,
+	[0][0][1][0][RTW89_WW][47] = 22,
+	[0][0][1][0][RTW89_WW][49] = 24,
+	[0][0][1][0][RTW89_WW][51] = 22,
+	[0][0][1][0][RTW89_WW][53] = 22,
+	[0][0][1][0][RTW89_WW][55] = 22,
+	[0][0][1][0][RTW89_WW][57] = 22,
+	[0][0][1][0][RTW89_WW][59] = 22,
+	[0][0][1][0][RTW89_WW][60] = 22,
+	[0][0][1][0][RTW89_WW][62] = 22,
+	[0][0][1][0][RTW89_WW][64] = 22,
+	[0][0][1][0][RTW89_WW][66] = 22,
+	[0][0][1][0][RTW89_WW][68] = 22,
+	[0][0][1][0][RTW89_WW][70] = 24,
+	[0][0][1][0][RTW89_WW][72] = 22,
+	[0][0][1][0][RTW89_WW][74] = 22,
+	[0][0][1][0][RTW89_WW][75] = 22,
+	[0][0][1][0][RTW89_WW][77] = 22,
+	[0][0][1][0][RTW89_WW][79] = 22,
+	[0][0][1][0][RTW89_WW][81] = 22,
+	[0][0][1][0][RTW89_WW][83] = 22,
+	[0][0][1][0][RTW89_WW][85] = 22,
+	[0][0][1][0][RTW89_WW][87] = 22,
+	[0][0][1][0][RTW89_WW][89] = 22,
+	[0][0][1][0][RTW89_WW][90] = 22,
+	[0][0][1][0][RTW89_WW][92] = 22,
+	[0][0][1][0][RTW89_WW][94] = 22,
+	[0][0][1][0][RTW89_WW][96] = 22,
+	[0][0][1][0][RTW89_WW][98] = 22,
+	[0][0][1][0][RTW89_WW][100] = 22,
+	[0][0][1][0][RTW89_WW][102] = 22,
+	[0][0][1][0][RTW89_WW][104] = 22,
+	[0][0][1][0][RTW89_WW][105] = 22,
+	[0][0][1][0][RTW89_WW][107] = 24,
+	[0][0][1][0][RTW89_WW][109] = 24,
 	[0][0][1][0][RTW89_WW][111] = 0,
 	[0][0][1][0][RTW89_WW][113] = 0,
 	[0][0][1][0][RTW89_WW][115] = 0,
 	[0][0][1][0][RTW89_WW][117] = 0,
 	[0][0][1][0][RTW89_WW][119] = 0,
-	[0][1][1][0][RTW89_WW][0] = 60,
-	[0][1][1][0][RTW89_WW][2] = 60,
-	[0][1][1][0][RTW89_WW][4] = 60,
-	[0][1][1][0][RTW89_WW][6] = 60,
-	[0][1][1][0][RTW89_WW][8] = 60,
-	[0][1][1][0][RTW89_WW][10] = 60,
-	[0][1][1][0][RTW89_WW][12] = 60,
-	[0][1][1][0][RTW89_WW][14] = 60,
-	[0][1][1][0][RTW89_WW][15] = 60,
-	[0][1][1][0][RTW89_WW][17] = 60,
-	[0][1][1][0][RTW89_WW][19] = 60,
-	[0][1][1][0][RTW89_WW][21] = 60,
-	[0][1][1][0][RTW89_WW][23] = 60,
-	[0][1][1][0][RTW89_WW][25] = 60,
-	[0][1][1][0][RTW89_WW][27] = 60,
-	[0][1][1][0][RTW89_WW][29] = 60,
-	[0][1][1][0][RTW89_WW][30] = 60,
-	[0][1][1][0][RTW89_WW][32] = 60,
-	[0][1][1][0][RTW89_WW][34] = 60,
-	[0][1][1][0][RTW89_WW][36] = 60,
-	[0][1][1][0][RTW89_WW][38] = 60,
-	[0][1][1][0][RTW89_WW][40] = 60,
-	[0][1][1][0][RTW89_WW][42] = 60,
-	[0][1][1][0][RTW89_WW][44] = 60,
-	[0][1][1][0][RTW89_WW][45] = 60,
-	[0][1][1][0][RTW89_WW][47] = 60,
-	[0][1][1][0][RTW89_WW][49] = 60,
-	[0][1][1][0][RTW89_WW][51] = 60,
-	[0][1][1][0][RTW89_WW][53] = 60,
-	[0][1][1][0][RTW89_WW][55] = 60,
-	[0][1][1][0][RTW89_WW][57] = 60,
-	[0][1][1][0][RTW89_WW][59] = 60,
-	[0][1][1][0][RTW89_WW][60] = 60,
-	[0][1][1][0][RTW89_WW][62] = 60,
-	[0][1][1][0][RTW89_WW][64] = 60,
-	[0][1][1][0][RTW89_WW][66] = 60,
-	[0][1][1][0][RTW89_WW][68] = 60,
-	[0][1][1][0][RTW89_WW][70] = 60,
-	[0][1][1][0][RTW89_WW][72] = 60,
-	[0][1][1][0][RTW89_WW][74] = 60,
-	[0][1][1][0][RTW89_WW][75] = 60,
-	[0][1][1][0][RTW89_WW][77] = 60,
-	[0][1][1][0][RTW89_WW][79] = 60,
-	[0][1][1][0][RTW89_WW][81] = 60,
-	[0][1][1][0][RTW89_WW][83] = 60,
-	[0][1][1][0][RTW89_WW][85] = 60,
-	[0][1][1][0][RTW89_WW][87] = 60,
-	[0][1][1][0][RTW89_WW][89] = 60,
-	[0][1][1][0][RTW89_WW][90] = 60,
-	[0][1][1][0][RTW89_WW][92] = 60,
-	[0][1][1][0][RTW89_WW][94] = 60,
-	[0][1][1][0][RTW89_WW][96] = 60,
-	[0][1][1][0][RTW89_WW][98] = 60,
-	[0][1][1][0][RTW89_WW][100] = 60,
-	[0][1][1][0][RTW89_WW][102] = 60,
-	[0][1][1][0][RTW89_WW][104] = 60,
-	[0][1][1][0][RTW89_WW][105] = 60,
-	[0][1][1][0][RTW89_WW][107] = 60,
-	[0][1][1][0][RTW89_WW][109] = 60,
+	[0][1][1][0][RTW89_WW][0] = -2,
+	[0][1][1][0][RTW89_WW][2] = -4,
+	[0][1][1][0][RTW89_WW][4] = -4,
+	[0][1][1][0][RTW89_WW][6] = -4,
+	[0][1][1][0][RTW89_WW][8] = -4,
+	[0][1][1][0][RTW89_WW][10] = -4,
+	[0][1][1][0][RTW89_WW][12] = -4,
+	[0][1][1][0][RTW89_WW][14] = -4,
+	[0][1][1][0][RTW89_WW][15] = -4,
+	[0][1][1][0][RTW89_WW][17] = -4,
+	[0][1][1][0][RTW89_WW][19] = -4,
+	[0][1][1][0][RTW89_WW][21] = -4,
+	[0][1][1][0][RTW89_WW][23] = -4,
+	[0][1][1][0][RTW89_WW][25] = -4,
+	[0][1][1][0][RTW89_WW][27] = -4,
+	[0][1][1][0][RTW89_WW][29] = -4,
+	[0][1][1][0][RTW89_WW][30] = -4,
+	[0][1][1][0][RTW89_WW][32] = -4,
+	[0][1][1][0][RTW89_WW][34] = -4,
+	[0][1][1][0][RTW89_WW][36] = -4,
+	[0][1][1][0][RTW89_WW][38] = -4,
+	[0][1][1][0][RTW89_WW][40] = -4,
+	[0][1][1][0][RTW89_WW][42] = -4,
+	[0][1][1][0][RTW89_WW][44] = -2,
+	[0][1][1][0][RTW89_WW][45] = -2,
+	[0][1][1][0][RTW89_WW][47] = -2,
+	[0][1][1][0][RTW89_WW][49] = -2,
+	[0][1][1][0][RTW89_WW][51] = -2,
+	[0][1][1][0][RTW89_WW][53] = -2,
+	[0][1][1][0][RTW89_WW][55] = -2,
+	[0][1][1][0][RTW89_WW][57] = -2,
+	[0][1][1][0][RTW89_WW][59] = -2,
+	[0][1][1][0][RTW89_WW][60] = -2,
+	[0][1][1][0][RTW89_WW][62] = -2,
+	[0][1][1][0][RTW89_WW][64] = -2,
+	[0][1][1][0][RTW89_WW][66] = -2,
+	[0][1][1][0][RTW89_WW][68] = -2,
+	[0][1][1][0][RTW89_WW][70] = -2,
+	[0][1][1][0][RTW89_WW][72] = -2,
+	[0][1][1][0][RTW89_WW][74] = -2,
+	[0][1][1][0][RTW89_WW][75] = -2,
+	[0][1][1][0][RTW89_WW][77] = -2,
+	[0][1][1][0][RTW89_WW][79] = -2,
+	[0][1][1][0][RTW89_WW][81] = -2,
+	[0][1][1][0][RTW89_WW][83] = -2,
+	[0][1][1][0][RTW89_WW][85] = -2,
+	[0][1][1][0][RTW89_WW][87] = -2,
+	[0][1][1][0][RTW89_WW][89] = -2,
+	[0][1][1][0][RTW89_WW][90] = -2,
+	[0][1][1][0][RTW89_WW][92] = -2,
+	[0][1][1][0][RTW89_WW][94] = -2,
+	[0][1][1][0][RTW89_WW][96] = -2,
+	[0][1][1][0][RTW89_WW][98] = -2,
+	[0][1][1][0][RTW89_WW][100] = -2,
+	[0][1][1][0][RTW89_WW][102] = -2,
+	[0][1][1][0][RTW89_WW][104] = -2,
+	[0][1][1][0][RTW89_WW][105] = -2,
+	[0][1][1][0][RTW89_WW][107] = 1,
+	[0][1][1][0][RTW89_WW][109] = 1,
 	[0][1][1][0][RTW89_WW][111] = 0,
 	[0][1][1][0][RTW89_WW][113] = 0,
 	[0][1][1][0][RTW89_WW][115] = 0,
 	[0][1][1][0][RTW89_WW][117] = 0,
 	[0][1][1][0][RTW89_WW][119] = 0,
-	[0][0][2][0][RTW89_WW][0] = 72,
-	[0][0][2][0][RTW89_WW][2] = 72,
-	[0][0][2][0][RTW89_WW][4] = 72,
-	[0][0][2][0][RTW89_WW][6] = 72,
-	[0][0][2][0][RTW89_WW][8] = 72,
-	[0][0][2][0][RTW89_WW][10] = 72,
-	[0][0][2][0][RTW89_WW][12] = 72,
-	[0][0][2][0][RTW89_WW][14] = 72,
-	[0][0][2][0][RTW89_WW][15] = 72,
-	[0][0][2][0][RTW89_WW][17] = 72,
-	[0][0][2][0][RTW89_WW][19] = 72,
-	[0][0][2][0][RTW89_WW][21] = 72,
-	[0][0][2][0][RTW89_WW][23] = 72,
-	[0][0][2][0][RTW89_WW][25] = 72,
-	[0][0][2][0][RTW89_WW][27] = 72,
-	[0][0][2][0][RTW89_WW][29] = 72,
-	[0][0][2][0][RTW89_WW][30] = 72,
-	[0][0][2][0][RTW89_WW][32] = 72,
-	[0][0][2][0][RTW89_WW][34] = 72,
-	[0][0][2][0][RTW89_WW][36] = 72,
-	[0][0][2][0][RTW89_WW][38] = 72,
-	[0][0][2][0][RTW89_WW][40] = 72,
-	[0][0][2][0][RTW89_WW][42] = 72,
-	[0][0][2][0][RTW89_WW][44] = 72,
-	[0][0][2][0][RTW89_WW][45] = 72,
-	[0][0][2][0][RTW89_WW][47] = 72,
-	[0][0][2][0][RTW89_WW][49] = 72,
-	[0][0][2][0][RTW89_WW][51] = 72,
-	[0][0][2][0][RTW89_WW][53] = 72,
-	[0][0][2][0][RTW89_WW][55] = 72,
-	[0][0][2][0][RTW89_WW][57] = 72,
-	[0][0][2][0][RTW89_WW][59] = 72,
-	[0][0][2][0][RTW89_WW][60] = 72,
-	[0][0][2][0][RTW89_WW][62] = 72,
-	[0][0][2][0][RTW89_WW][64] = 72,
-	[0][0][2][0][RTW89_WW][66] = 72,
-	[0][0][2][0][RTW89_WW][68] = 72,
-	[0][0][2][0][RTW89_WW][70] = 72,
-	[0][0][2][0][RTW89_WW][72] = 72,
-	[0][0][2][0][RTW89_WW][74] = 72,
-	[0][0][2][0][RTW89_WW][75] = 72,
-	[0][0][2][0][RTW89_WW][77] = 72,
-	[0][0][2][0][RTW89_WW][79] = 72,
-	[0][0][2][0][RTW89_WW][81] = 72,
-	[0][0][2][0][RTW89_WW][83] = 72,
-	[0][0][2][0][RTW89_WW][85] = 72,
-	[0][0][2][0][RTW89_WW][87] = 72,
-	[0][0][2][0][RTW89_WW][89] = 72,
-	[0][0][2][0][RTW89_WW][90] = 72,
-	[0][0][2][0][RTW89_WW][92] = 72,
-	[0][0][2][0][RTW89_WW][94] = 72,
-	[0][0][2][0][RTW89_WW][96] = 72,
-	[0][0][2][0][RTW89_WW][98] = 72,
-	[0][0][2][0][RTW89_WW][100] = 72,
-	[0][0][2][0][RTW89_WW][102] = 72,
-	[0][0][2][0][RTW89_WW][104] = 72,
-	[0][0][2][0][RTW89_WW][105] = 72,
-	[0][0][2][0][RTW89_WW][107] = 72,
-	[0][0][2][0][RTW89_WW][109] = 72,
+	[0][0][2][0][RTW89_WW][0] = 24,
+	[0][0][2][0][RTW89_WW][2] = 22,
+	[0][0][2][0][RTW89_WW][4] = 22,
+	[0][0][2][0][RTW89_WW][6] = 22,
+	[0][0][2][0][RTW89_WW][8] = 22,
+	[0][0][2][0][RTW89_WW][10] = 22,
+	[0][0][2][0][RTW89_WW][12] = 22,
+	[0][0][2][0][RTW89_WW][14] = 22,
+	[0][0][2][0][RTW89_WW][15] = 22,
+	[0][0][2][0][RTW89_WW][17] = 22,
+	[0][0][2][0][RTW89_WW][19] = 22,
+	[0][0][2][0][RTW89_WW][21] = 22,
+	[0][0][2][0][RTW89_WW][23] = 22,
+	[0][0][2][0][RTW89_WW][25] = 22,
+	[0][0][2][0][RTW89_WW][27] = 22,
+	[0][0][2][0][RTW89_WW][29] = 22,
+	[0][0][2][0][RTW89_WW][30] = 22,
+	[0][0][2][0][RTW89_WW][32] = 22,
+	[0][0][2][0][RTW89_WW][34] = 22,
+	[0][0][2][0][RTW89_WW][36] = 22,
+	[0][0][2][0][RTW89_WW][38] = 22,
+	[0][0][2][0][RTW89_WW][40] = 22,
+	[0][0][2][0][RTW89_WW][42] = 22,
+	[0][0][2][0][RTW89_WW][44] = 22,
+	[0][0][2][0][RTW89_WW][45] = 22,
+	[0][0][2][0][RTW89_WW][47] = 22,
+	[0][0][2][0][RTW89_WW][49] = 24,
+	[0][0][2][0][RTW89_WW][51] = 22,
+	[0][0][2][0][RTW89_WW][53] = 22,
+	[0][0][2][0][RTW89_WW][55] = 22,
+	[0][0][2][0][RTW89_WW][57] = 22,
+	[0][0][2][0][RTW89_WW][59] = 22,
+	[0][0][2][0][RTW89_WW][60] = 22,
+	[0][0][2][0][RTW89_WW][62] = 22,
+	[0][0][2][0][RTW89_WW][64] = 22,
+	[0][0][2][0][RTW89_WW][66] = 22,
+	[0][0][2][0][RTW89_WW][68] = 22,
+	[0][0][2][0][RTW89_WW][70] = 24,
+	[0][0][2][0][RTW89_WW][72] = 22,
+	[0][0][2][0][RTW89_WW][74] = 22,
+	[0][0][2][0][RTW89_WW][75] = 22,
+	[0][0][2][0][RTW89_WW][77] = 22,
+	[0][0][2][0][RTW89_WW][79] = 22,
+	[0][0][2][0][RTW89_WW][81] = 22,
+	[0][0][2][0][RTW89_WW][83] = 22,
+	[0][0][2][0][RTW89_WW][85] = 22,
+	[0][0][2][0][RTW89_WW][87] = 22,
+	[0][0][2][0][RTW89_WW][89] = 22,
+	[0][0][2][0][RTW89_WW][90] = 22,
+	[0][0][2][0][RTW89_WW][92] = 22,
+	[0][0][2][0][RTW89_WW][94] = 22,
+	[0][0][2][0][RTW89_WW][96] = 22,
+	[0][0][2][0][RTW89_WW][98] = 22,
+	[0][0][2][0][RTW89_WW][100] = 22,
+	[0][0][2][0][RTW89_WW][102] = 22,
+	[0][0][2][0][RTW89_WW][104] = 22,
+	[0][0][2][0][RTW89_WW][105] = 22,
+	[0][0][2][0][RTW89_WW][107] = 24,
+	[0][0][2][0][RTW89_WW][109] = 24,
 	[0][0][2][0][RTW89_WW][111] = 0,
 	[0][0][2][0][RTW89_WW][113] = 0,
 	[0][0][2][0][RTW89_WW][115] = 0,
 	[0][0][2][0][RTW89_WW][117] = 0,
 	[0][0][2][0][RTW89_WW][119] = 0,
-	[0][1][2][0][RTW89_WW][0] = 60,
-	[0][1][2][0][RTW89_WW][2] = 60,
-	[0][1][2][0][RTW89_WW][4] = 60,
-	[0][1][2][0][RTW89_WW][6] = 60,
-	[0][1][2][0][RTW89_WW][8] = 60,
-	[0][1][2][0][RTW89_WW][10] = 60,
-	[0][1][2][0][RTW89_WW][12] = 60,
-	[0][1][2][0][RTW89_WW][14] = 60,
-	[0][1][2][0][RTW89_WW][15] = 60,
-	[0][1][2][0][RTW89_WW][17] = 60,
-	[0][1][2][0][RTW89_WW][19] = 60,
-	[0][1][2][0][RTW89_WW][21] = 60,
-	[0][1][2][0][RTW89_WW][23] = 60,
-	[0][1][2][0][RTW89_WW][25] = 60,
-	[0][1][2][0][RTW89_WW][27] = 60,
-	[0][1][2][0][RTW89_WW][29] = 60,
-	[0][1][2][0][RTW89_WW][30] = 60,
-	[0][1][2][0][RTW89_WW][32] = 60,
-	[0][1][2][0][RTW89_WW][34] = 60,
-	[0][1][2][0][RTW89_WW][36] = 60,
-	[0][1][2][0][RTW89_WW][38] = 60,
-	[0][1][2][0][RTW89_WW][40] = 60,
-	[0][1][2][0][RTW89_WW][42] = 60,
-	[0][1][2][0][RTW89_WW][44] = 60,
-	[0][1][2][0][RTW89_WW][45] = 60,
-	[0][1][2][0][RTW89_WW][47] = 60,
-	[0][1][2][0][RTW89_WW][49] = 60,
-	[0][1][2][0][RTW89_WW][51] = 60,
-	[0][1][2][0][RTW89_WW][53] = 60,
-	[0][1][2][0][RTW89_WW][55] = 60,
-	[0][1][2][0][RTW89_WW][57] = 60,
-	[0][1][2][0][RTW89_WW][59] = 60,
-	[0][1][2][0][RTW89_WW][60] = 60,
-	[0][1][2][0][RTW89_WW][62] = 60,
-	[0][1][2][0][RTW89_WW][64] = 60,
-	[0][1][2][0][RTW89_WW][66] = 60,
-	[0][1][2][0][RTW89_WW][68] = 60,
-	[0][1][2][0][RTW89_WW][70] = 60,
-	[0][1][2][0][RTW89_WW][72] = 60,
-	[0][1][2][0][RTW89_WW][74] = 60,
-	[0][1][2][0][RTW89_WW][75] = 60,
-	[0][1][2][0][RTW89_WW][77] = 60,
-	[0][1][2][0][RTW89_WW][79] = 60,
-	[0][1][2][0][RTW89_WW][81] = 60,
-	[0][1][2][0][RTW89_WW][83] = 60,
-	[0][1][2][0][RTW89_WW][85] = 60,
-	[0][1][2][0][RTW89_WW][87] = 60,
-	[0][1][2][0][RTW89_WW][89] = 60,
-	[0][1][2][0][RTW89_WW][90] = 60,
-	[0][1][2][0][RTW89_WW][92] = 60,
-	[0][1][2][0][RTW89_WW][94] = 60,
-	[0][1][2][0][RTW89_WW][96] = 60,
-	[0][1][2][0][RTW89_WW][98] = 60,
-	[0][1][2][0][RTW89_WW][100] = 60,
-	[0][1][2][0][RTW89_WW][102] = 60,
-	[0][1][2][0][RTW89_WW][104] = 60,
-	[0][1][2][0][RTW89_WW][105] = 60,
-	[0][1][2][0][RTW89_WW][107] = 60,
-	[0][1][2][0][RTW89_WW][109] = 60,
+	[0][1][2][0][RTW89_WW][0] = -2,
+	[0][1][2][0][RTW89_WW][2] = -4,
+	[0][1][2][0][RTW89_WW][4] = -4,
+	[0][1][2][0][RTW89_WW][6] = -4,
+	[0][1][2][0][RTW89_WW][8] = -4,
+	[0][1][2][0][RTW89_WW][10] = -4,
+	[0][1][2][0][RTW89_WW][12] = -4,
+	[0][1][2][0][RTW89_WW][14] = -4,
+	[0][1][2][0][RTW89_WW][15] = -4,
+	[0][1][2][0][RTW89_WW][17] = -4,
+	[0][1][2][0][RTW89_WW][19] = -4,
+	[0][1][2][0][RTW89_WW][21] = -4,
+	[0][1][2][0][RTW89_WW][23] = -4,
+	[0][1][2][0][RTW89_WW][25] = -4,
+	[0][1][2][0][RTW89_WW][27] = -4,
+	[0][1][2][0][RTW89_WW][29] = -4,
+	[0][1][2][0][RTW89_WW][30] = -4,
+	[0][1][2][0][RTW89_WW][32] = -4,
+	[0][1][2][0][RTW89_WW][34] = -4,
+	[0][1][2][0][RTW89_WW][36] = -4,
+	[0][1][2][0][RTW89_WW][38] = -4,
+	[0][1][2][0][RTW89_WW][40] = -4,
+	[0][1][2][0][RTW89_WW][42] = -4,
+	[0][1][2][0][RTW89_WW][44] = -2,
+	[0][1][2][0][RTW89_WW][45] = -2,
+	[0][1][2][0][RTW89_WW][47] = -2,
+	[0][1][2][0][RTW89_WW][49] = -2,
+	[0][1][2][0][RTW89_WW][51] = -2,
+	[0][1][2][0][RTW89_WW][53] = -2,
+	[0][1][2][0][RTW89_WW][55] = -2,
+	[0][1][2][0][RTW89_WW][57] = -2,
+	[0][1][2][0][RTW89_WW][59] = -2,
+	[0][1][2][0][RTW89_WW][60] = -2,
+	[0][1][2][0][RTW89_WW][62] = -2,
+	[0][1][2][0][RTW89_WW][64] = -2,
+	[0][1][2][0][RTW89_WW][66] = -2,
+	[0][1][2][0][RTW89_WW][68] = -2,
+	[0][1][2][0][RTW89_WW][70] = -2,
+	[0][1][2][0][RTW89_WW][72] = -2,
+	[0][1][2][0][RTW89_WW][74] = -2,
+	[0][1][2][0][RTW89_WW][75] = -2,
+	[0][1][2][0][RTW89_WW][77] = -2,
+	[0][1][2][0][RTW89_WW][79] = -2,
+	[0][1][2][0][RTW89_WW][81] = -2,
+	[0][1][2][0][RTW89_WW][83] = -2,
+	[0][1][2][0][RTW89_WW][85] = -2,
+	[0][1][2][0][RTW89_WW][87] = -2,
+	[0][1][2][0][RTW89_WW][89] = -2,
+	[0][1][2][0][RTW89_WW][90] = -2,
+	[0][1][2][0][RTW89_WW][92] = -2,
+	[0][1][2][0][RTW89_WW][94] = -2,
+	[0][1][2][0][RTW89_WW][96] = -2,
+	[0][1][2][0][RTW89_WW][98] = -2,
+	[0][1][2][0][RTW89_WW][100] = -2,
+	[0][1][2][0][RTW89_WW][102] = -2,
+	[0][1][2][0][RTW89_WW][104] = -2,
+	[0][1][2][0][RTW89_WW][105] = -2,
+	[0][1][2][0][RTW89_WW][107] = 1,
+	[0][1][2][0][RTW89_WW][109] = 1,
 	[0][1][2][0][RTW89_WW][111] = 0,
 	[0][1][2][0][RTW89_WW][113] = 0,
 	[0][1][2][0][RTW89_WW][115] = 0,
 	[0][1][2][0][RTW89_WW][117] = 0,
 	[0][1][2][0][RTW89_WW][119] = 0,
-	[0][1][2][1][RTW89_WW][0] = 48,
-	[0][1][2][1][RTW89_WW][2] = 48,
-	[0][1][2][1][RTW89_WW][4] = 48,
-	[0][1][2][1][RTW89_WW][6] = 48,
-	[0][1][2][1][RTW89_WW][8] = 48,
-	[0][1][2][1][RTW89_WW][10] = 48,
-	[0][1][2][1][RTW89_WW][12] = 48,
-	[0][1][2][1][RTW89_WW][14] = 48,
-	[0][1][2][1][RTW89_WW][15] = 48,
-	[0][1][2][1][RTW89_WW][17] = 48,
-	[0][1][2][1][RTW89_WW][19] = 48,
-	[0][1][2][1][RTW89_WW][21] = 48,
-	[0][1][2][1][RTW89_WW][23] = 48,
-	[0][1][2][1][RTW89_WW][25] = 48,
-	[0][1][2][1][RTW89_WW][27] = 48,
-	[0][1][2][1][RTW89_WW][29] = 48,
-	[0][1][2][1][RTW89_WW][30] = 48,
-	[0][1][2][1][RTW89_WW][32] = 48,
-	[0][1][2][1][RTW89_WW][34] = 48,
-	[0][1][2][1][RTW89_WW][36] = 48,
-	[0][1][2][1][RTW89_WW][38] = 48,
-	[0][1][2][1][RTW89_WW][40] = 48,
-	[0][1][2][1][RTW89_WW][42] = 48,
-	[0][1][2][1][RTW89_WW][44] = 48,
-	[0][1][2][1][RTW89_WW][45] = 48,
-	[0][1][2][1][RTW89_WW][47] = 48,
-	[0][1][2][1][RTW89_WW][49] = 48,
-	[0][1][2][1][RTW89_WW][51] = 48,
-	[0][1][2][1][RTW89_WW][53] = 48,
-	[0][1][2][1][RTW89_WW][55] = 48,
-	[0][1][2][1][RTW89_WW][57] = 48,
-	[0][1][2][1][RTW89_WW][59] = 48,
-	[0][1][2][1][RTW89_WW][60] = 48,
-	[0][1][2][1][RTW89_WW][62] = 48,
-	[0][1][2][1][RTW89_WW][64] = 48,
-	[0][1][2][1][RTW89_WW][66] = 48,
-	[0][1][2][1][RTW89_WW][68] = 48,
-	[0][1][2][1][RTW89_WW][70] = 48,
-	[0][1][2][1][RTW89_WW][72] = 48,
-	[0][1][2][1][RTW89_WW][74] = 48,
-	[0][1][2][1][RTW89_WW][75] = 48,
-	[0][1][2][1][RTW89_WW][77] = 48,
-	[0][1][2][1][RTW89_WW][79] = 48,
-	[0][1][2][1][RTW89_WW][81] = 48,
-	[0][1][2][1][RTW89_WW][83] = 48,
-	[0][1][2][1][RTW89_WW][85] = 48,
-	[0][1][2][1][RTW89_WW][87] = 48,
-	[0][1][2][1][RTW89_WW][89] = 48,
-	[0][1][2][1][RTW89_WW][90] = 48,
-	[0][1][2][1][RTW89_WW][92] = 48,
-	[0][1][2][1][RTW89_WW][94] = 48,
-	[0][1][2][1][RTW89_WW][96] = 48,
-	[0][1][2][1][RTW89_WW][98] = 48,
-	[0][1][2][1][RTW89_WW][100] = 48,
-	[0][1][2][1][RTW89_WW][102] = 48,
-	[0][1][2][1][RTW89_WW][104] = 48,
-	[0][1][2][1][RTW89_WW][105] = 48,
-	[0][1][2][1][RTW89_WW][107] = 48,
-	[0][1][2][1][RTW89_WW][109] = 48,
+	[0][1][2][1][RTW89_WW][0] = -2,
+	[0][1][2][1][RTW89_WW][2] = -4,
+	[0][1][2][1][RTW89_WW][4] = -4,
+	[0][1][2][1][RTW89_WW][6] = -4,
+	[0][1][2][1][RTW89_WW][8] = -4,
+	[0][1][2][1][RTW89_WW][10] = -4,
+	[0][1][2][1][RTW89_WW][12] = -4,
+	[0][1][2][1][RTW89_WW][14] = -4,
+	[0][1][2][1][RTW89_WW][15] = -4,
+	[0][1][2][1][RTW89_WW][17] = -4,
+	[0][1][2][1][RTW89_WW][19] = -4,
+	[0][1][2][1][RTW89_WW][21] = -4,
+	[0][1][2][1][RTW89_WW][23] = -4,
+	[0][1][2][1][RTW89_WW][25] = -4,
+	[0][1][2][1][RTW89_WW][27] = -4,
+	[0][1][2][1][RTW89_WW][29] = -4,
+	[0][1][2][1][RTW89_WW][30] = -4,
+	[0][1][2][1][RTW89_WW][32] = -4,
+	[0][1][2][1][RTW89_WW][34] = -4,
+	[0][1][2][1][RTW89_WW][36] = -4,
+	[0][1][2][1][RTW89_WW][38] = -4,
+	[0][1][2][1][RTW89_WW][40] = -4,
+	[0][1][2][1][RTW89_WW][42] = -4,
+	[0][1][2][1][RTW89_WW][44] = -2,
+	[0][1][2][1][RTW89_WW][45] = -2,
+	[0][1][2][1][RTW89_WW][47] = -2,
+	[0][1][2][1][RTW89_WW][49] = -2,
+	[0][1][2][1][RTW89_WW][51] = -2,
+	[0][1][2][1][RTW89_WW][53] = -2,
+	[0][1][2][1][RTW89_WW][55] = -2,
+	[0][1][2][1][RTW89_WW][57] = -2,
+	[0][1][2][1][RTW89_WW][59] = -2,
+	[0][1][2][1][RTW89_WW][60] = -2,
+	[0][1][2][1][RTW89_WW][62] = -2,
+	[0][1][2][1][RTW89_WW][64] = -2,
+	[0][1][2][1][RTW89_WW][66] = -2,
+	[0][1][2][1][RTW89_WW][68] = -2,
+	[0][1][2][1][RTW89_WW][70] = -2,
+	[0][1][2][1][RTW89_WW][72] = -2,
+	[0][1][2][1][RTW89_WW][74] = -2,
+	[0][1][2][1][RTW89_WW][75] = -2,
+	[0][1][2][1][RTW89_WW][77] = -2,
+	[0][1][2][1][RTW89_WW][79] = -2,
+	[0][1][2][1][RTW89_WW][81] = -2,
+	[0][1][2][1][RTW89_WW][83] = -2,
+	[0][1][2][1][RTW89_WW][85] = -2,
+	[0][1][2][1][RTW89_WW][87] = -2,
+	[0][1][2][1][RTW89_WW][89] = -2,
+	[0][1][2][1][RTW89_WW][90] = -2,
+	[0][1][2][1][RTW89_WW][92] = -2,
+	[0][1][2][1][RTW89_WW][94] = -2,
+	[0][1][2][1][RTW89_WW][96] = -2,
+	[0][1][2][1][RTW89_WW][98] = -2,
+	[0][1][2][1][RTW89_WW][100] = -2,
+	[0][1][2][1][RTW89_WW][102] = -2,
+	[0][1][2][1][RTW89_WW][104] = -2,
+	[0][1][2][1][RTW89_WW][105] = -2,
+	[0][1][2][1][RTW89_WW][107] = 1,
+	[0][1][2][1][RTW89_WW][109] = 1,
 	[0][1][2][1][RTW89_WW][111] = 0,
 	[0][1][2][1][RTW89_WW][113] = 0,
 	[0][1][2][1][RTW89_WW][115] = 0,
 	[0][1][2][1][RTW89_WW][117] = 0,
 	[0][1][2][1][RTW89_WW][119] = 0,
-	[1][0][2][0][RTW89_WW][1] = 72,
-	[1][0][2][0][RTW89_WW][5] = 72,
-	[1][0][2][0][RTW89_WW][9] = 72,
-	[1][0][2][0][RTW89_WW][13] = 72,
-	[1][0][2][0][RTW89_WW][16] = 72,
-	[1][0][2][0][RTW89_WW][20] = 72,
-	[1][0][2][0][RTW89_WW][24] = 72,
-	[1][0][2][0][RTW89_WW][28] = 72,
-	[1][0][2][0][RTW89_WW][31] = 72,
-	[1][0][2][0][RTW89_WW][35] = 72,
-	[1][0][2][0][RTW89_WW][39] = 72,
-	[1][0][2][0][RTW89_WW][43] = 72,
-	[1][0][2][0][RTW89_WW][46] = 72,
-	[1][0][2][0][RTW89_WW][50] = 72,
-	[1][0][2][0][RTW89_WW][54] = 72,
-	[1][0][2][0][RTW89_WW][58] = 72,
-	[1][0][2][0][RTW89_WW][61] = 72,
-	[1][0][2][0][RTW89_WW][65] = 72,
-	[1][0][2][0][RTW89_WW][69] = 72,
-	[1][0][2][0][RTW89_WW][73] = 72,
-	[1][0][2][0][RTW89_WW][76] = 72,
-	[1][0][2][0][RTW89_WW][80] = 72,
-	[1][0][2][0][RTW89_WW][84] = 72,
-	[1][0][2][0][RTW89_WW][88] = 72,
-	[1][0][2][0][RTW89_WW][91] = 72,
-	[1][0][2][0][RTW89_WW][95] = 72,
-	[1][0][2][0][RTW89_WW][99] = 72,
-	[1][0][2][0][RTW89_WW][103] = 72,
-	[1][0][2][0][RTW89_WW][106] = 72,
+	[1][0][2][0][RTW89_WW][1] = 34,
+	[1][0][2][0][RTW89_WW][5] = 34,
+	[1][0][2][0][RTW89_WW][9] = 34,
+	[1][0][2][0][RTW89_WW][13] = 34,
+	[1][0][2][0][RTW89_WW][16] = 34,
+	[1][0][2][0][RTW89_WW][20] = 34,
+	[1][0][2][0][RTW89_WW][24] = 36,
+	[1][0][2][0][RTW89_WW][28] = 34,
+	[1][0][2][0][RTW89_WW][31] = 34,
+	[1][0][2][0][RTW89_WW][35] = 34,
+	[1][0][2][0][RTW89_WW][39] = 34,
+	[1][0][2][0][RTW89_WW][43] = 34,
+	[1][0][2][0][RTW89_WW][46] = 34,
+	[1][0][2][0][RTW89_WW][50] = 34,
+	[1][0][2][0][RTW89_WW][54] = 36,
+	[1][0][2][0][RTW89_WW][58] = 36,
+	[1][0][2][0][RTW89_WW][61] = 34,
+	[1][0][2][0][RTW89_WW][65] = 34,
+	[1][0][2][0][RTW89_WW][69] = 34,
+	[1][0][2][0][RTW89_WW][73] = 34,
+	[1][0][2][0][RTW89_WW][76] = 34,
+	[1][0][2][0][RTW89_WW][80] = 34,
+	[1][0][2][0][RTW89_WW][84] = 34,
+	[1][0][2][0][RTW89_WW][88] = 34,
+	[1][0][2][0][RTW89_WW][91] = 36,
+	[1][0][2][0][RTW89_WW][95] = 34,
+	[1][0][2][0][RTW89_WW][99] = 34,
+	[1][0][2][0][RTW89_WW][103] = 34,
+	[1][0][2][0][RTW89_WW][106] = 36,
 	[1][0][2][0][RTW89_WW][110] = 0,
 	[1][0][2][0][RTW89_WW][114] = 0,
 	[1][0][2][0][RTW89_WW][118] = 0,
-	[1][1][2][0][RTW89_WW][1] = 60,
-	[1][1][2][0][RTW89_WW][5] = 60,
-	[1][1][2][0][RTW89_WW][9] = 60,
-	[1][1][2][0][RTW89_WW][13] = 60,
-	[1][1][2][0][RTW89_WW][16] = 60,
-	[1][1][2][0][RTW89_WW][20] = 60,
-	[1][1][2][0][RTW89_WW][24] = 60,
-	[1][1][2][0][RTW89_WW][28] = 60,
-	[1][1][2][0][RTW89_WW][31] = 60,
-	[1][1][2][0][RTW89_WW][35] = 60,
-	[1][1][2][0][RTW89_WW][39] = 60,
-	[1][1][2][0][RTW89_WW][43] = 60,
-	[1][1][2][0][RTW89_WW][46] = 60,
-	[1][1][2][0][RTW89_WW][50] = 60,
-	[1][1][2][0][RTW89_WW][54] = 60,
-	[1][1][2][0][RTW89_WW][58] = 60,
-	[1][1][2][0][RTW89_WW][61] = 60,
-	[1][1][2][0][RTW89_WW][65] = 60,
-	[1][1][2][0][RTW89_WW][69] = 60,
-	[1][1][2][0][RTW89_WW][73] = 60,
-	[1][1][2][0][RTW89_WW][76] = 60,
-	[1][1][2][0][RTW89_WW][80] = 60,
-	[1][1][2][0][RTW89_WW][84] = 60,
-	[1][1][2][0][RTW89_WW][88] = 60,
-	[1][1][2][0][RTW89_WW][91] = 60,
-	[1][1][2][0][RTW89_WW][95] = 60,
-	[1][1][2][0][RTW89_WW][99] = 60,
-	[1][1][2][0][RTW89_WW][103] = 60,
-	[1][1][2][0][RTW89_WW][106] = 60,
+	[1][1][2][0][RTW89_WW][1] = 10,
+	[1][1][2][0][RTW89_WW][5] = 10,
+	[1][1][2][0][RTW89_WW][9] = 10,
+	[1][1][2][0][RTW89_WW][13] = 10,
+	[1][1][2][0][RTW89_WW][16] = 10,
+	[1][1][2][0][RTW89_WW][20] = 10,
+	[1][1][2][0][RTW89_WW][24] = 10,
+	[1][1][2][0][RTW89_WW][28] = 10,
+	[1][1][2][0][RTW89_WW][31] = 10,
+	[1][1][2][0][RTW89_WW][35] = 10,
+	[1][1][2][0][RTW89_WW][39] = 10,
+	[1][1][2][0][RTW89_WW][43] = 10,
+	[1][1][2][0][RTW89_WW][46] = 12,
+	[1][1][2][0][RTW89_WW][50] = 12,
+	[1][1][2][0][RTW89_WW][54] = 10,
+	[1][1][2][0][RTW89_WW][58] = 10,
+	[1][1][2][0][RTW89_WW][61] = 10,
+	[1][1][2][0][RTW89_WW][65] = 10,
+	[1][1][2][0][RTW89_WW][69] = 10,
+	[1][1][2][0][RTW89_WW][73] = 10,
+	[1][1][2][0][RTW89_WW][76] = 10,
+	[1][1][2][0][RTW89_WW][80] = 10,
+	[1][1][2][0][RTW89_WW][84] = 10,
+	[1][1][2][0][RTW89_WW][88] = 10,
+	[1][1][2][0][RTW89_WW][91] = 12,
+	[1][1][2][0][RTW89_WW][95] = 10,
+	[1][1][2][0][RTW89_WW][99] = 10,
+	[1][1][2][0][RTW89_WW][103] = 10,
+	[1][1][2][0][RTW89_WW][106] = 12,
 	[1][1][2][0][RTW89_WW][110] = 0,
 	[1][1][2][0][RTW89_WW][114] = 0,
 	[1][1][2][0][RTW89_WW][118] = 0,
-	[1][1][2][1][RTW89_WW][1] = 48,
-	[1][1][2][1][RTW89_WW][5] = 48,
-	[1][1][2][1][RTW89_WW][9] = 48,
-	[1][1][2][1][RTW89_WW][13] = 48,
-	[1][1][2][1][RTW89_WW][16] = 48,
-	[1][1][2][1][RTW89_WW][20] = 48,
-	[1][1][2][1][RTW89_WW][24] = 48,
-	[1][1][2][1][RTW89_WW][28] = 48,
-	[1][1][2][1][RTW89_WW][31] = 48,
-	[1][1][2][1][RTW89_WW][35] = 48,
-	[1][1][2][1][RTW89_WW][39] = 48,
-	[1][1][2][1][RTW89_WW][43] = 48,
-	[1][1][2][1][RTW89_WW][46] = 48,
-	[1][1][2][1][RTW89_WW][50] = 48,
-	[1][1][2][1][RTW89_WW][54] = 48,
-	[1][1][2][1][RTW89_WW][58] = 48,
-	[1][1][2][1][RTW89_WW][61] = 48,
-	[1][1][2][1][RTW89_WW][65] = 48,
-	[1][1][2][1][RTW89_WW][69] = 48,
-	[1][1][2][1][RTW89_WW][73] = 48,
-	[1][1][2][1][RTW89_WW][76] = 48,
-	[1][1][2][1][RTW89_WW][80] = 48,
-	[1][1][2][1][RTW89_WW][84] = 48,
-	[1][1][2][1][RTW89_WW][88] = 48,
-	[1][1][2][1][RTW89_WW][91] = 48,
-	[1][1][2][1][RTW89_WW][95] = 48,
-	[1][1][2][1][RTW89_WW][99] = 48,
-	[1][1][2][1][RTW89_WW][103] = 48,
-	[1][1][2][1][RTW89_WW][106] = 48,
+	[1][1][2][1][RTW89_WW][1] = 10,
+	[1][1][2][1][RTW89_WW][5] = 10,
+	[1][1][2][1][RTW89_WW][9] = 10,
+	[1][1][2][1][RTW89_WW][13] = 10,
+	[1][1][2][1][RTW89_WW][16] = 10,
+	[1][1][2][1][RTW89_WW][20] = 10,
+	[1][1][2][1][RTW89_WW][24] = 10,
+	[1][1][2][1][RTW89_WW][28] = 10,
+	[1][1][2][1][RTW89_WW][31] = 10,
+	[1][1][2][1][RTW89_WW][35] = 10,
+	[1][1][2][1][RTW89_WW][39] = 10,
+	[1][1][2][1][RTW89_WW][43] = 10,
+	[1][1][2][1][RTW89_WW][46] = 12,
+	[1][1][2][1][RTW89_WW][50] = 12,
+	[1][1][2][1][RTW89_WW][54] = 10,
+	[1][1][2][1][RTW89_WW][58] = 10,
+	[1][1][2][1][RTW89_WW][61] = 10,
+	[1][1][2][1][RTW89_WW][65] = 10,
+	[1][1][2][1][RTW89_WW][69] = 10,
+	[1][1][2][1][RTW89_WW][73] = 10,
+	[1][1][2][1][RTW89_WW][76] = 10,
+	[1][1][2][1][RTW89_WW][80] = 10,
+	[1][1][2][1][RTW89_WW][84] = 10,
+	[1][1][2][1][RTW89_WW][88] = 10,
+	[1][1][2][1][RTW89_WW][91] = 12,
+	[1][1][2][1][RTW89_WW][95] = 10,
+	[1][1][2][1][RTW89_WW][99] = 10,
+	[1][1][2][1][RTW89_WW][103] = 10,
+	[1][1][2][1][RTW89_WW][106] = 12,
 	[1][1][2][1][RTW89_WW][110] = 0,
 	[1][1][2][1][RTW89_WW][114] = 0,
 	[1][1][2][1][RTW89_WW][118] = 0,
-	[2][0][2][0][RTW89_WW][3] = 64,
-	[2][0][2][0][RTW89_WW][11] = 64,
-	[2][0][2][0][RTW89_WW][18] = 64,
-	[2][0][2][0][RTW89_WW][26] = 64,
-	[2][0][2][0][RTW89_WW][33] = 64,
-	[2][0][2][0][RTW89_WW][41] = 64,
-	[2][0][2][0][RTW89_WW][48] = 64,
-	[2][0][2][0][RTW89_WW][56] = 64,
-	[2][0][2][0][RTW89_WW][63] = 64,
-	[2][0][2][0][RTW89_WW][71] = 64,
-	[2][0][2][0][RTW89_WW][78] = 64,
-	[2][0][2][0][RTW89_WW][86] = 64,
-	[2][0][2][0][RTW89_WW][93] = 64,
-	[2][0][2][0][RTW89_WW][101] = 64,
+	[2][0][2][0][RTW89_WW][3] = 46,
+	[2][0][2][0][RTW89_WW][11] = 46,
+	[2][0][2][0][RTW89_WW][18] = 46,
+	[2][0][2][0][RTW89_WW][26] = 46,
+	[2][0][2][0][RTW89_WW][33] = 46,
+	[2][0][2][0][RTW89_WW][41] = 46,
+	[2][0][2][0][RTW89_WW][48] = 46,
+	[2][0][2][0][RTW89_WW][56] = 46,
+	[2][0][2][0][RTW89_WW][63] = 46,
+	[2][0][2][0][RTW89_WW][71] = 46,
+	[2][0][2][0][RTW89_WW][78] = 46,
+	[2][0][2][0][RTW89_WW][86] = 46,
+	[2][0][2][0][RTW89_WW][93] = 46,
+	[2][0][2][0][RTW89_WW][101] = 44,
 	[2][0][2][0][RTW89_WW][108] = 0,
 	[2][0][2][0][RTW89_WW][116] = 0,
-	[2][1][2][0][RTW89_WW][3] = 52,
-	[2][1][2][0][RTW89_WW][11] = 52,
-	[2][1][2][0][RTW89_WW][18] = 52,
-	[2][1][2][0][RTW89_WW][26] = 52,
-	[2][1][2][0][RTW89_WW][33] = 52,
-	[2][1][2][0][RTW89_WW][41] = 52,
-	[2][1][2][0][RTW89_WW][48] = 52,
-	[2][1][2][0][RTW89_WW][56] = 52,
-	[2][1][2][0][RTW89_WW][63] = 52,
-	[2][1][2][0][RTW89_WW][71] = 52,
-	[2][1][2][0][RTW89_WW][78] = 52,
-	[2][1][2][0][RTW89_WW][86] = 52,
-	[2][1][2][0][RTW89_WW][93] = 52,
-	[2][1][2][0][RTW89_WW][101] = 52,
+	[2][1][2][0][RTW89_WW][3] = 22,
+	[2][1][2][0][RTW89_WW][11] = 20,
+	[2][1][2][0][RTW89_WW][18] = 20,
+	[2][1][2][0][RTW89_WW][26] = 20,
+	[2][1][2][0][RTW89_WW][33] = 20,
+	[2][1][2][0][RTW89_WW][41] = 22,
+	[2][1][2][0][RTW89_WW][48] = 22,
+	[2][1][2][0][RTW89_WW][56] = 20,
+	[2][1][2][0][RTW89_WW][63] = 22,
+	[2][1][2][0][RTW89_WW][71] = 20,
+	[2][1][2][0][RTW89_WW][78] = 20,
+	[2][1][2][0][RTW89_WW][86] = 20,
+	[2][1][2][0][RTW89_WW][93] = 22,
+	[2][1][2][0][RTW89_WW][101] = 22,
 	[2][1][2][0][RTW89_WW][108] = 0,
 	[2][1][2][0][RTW89_WW][116] = 0,
-	[2][1][2][1][RTW89_WW][3] = 40,
-	[2][1][2][1][RTW89_WW][11] = 40,
-	[2][1][2][1][RTW89_WW][18] = 40,
-	[2][1][2][1][RTW89_WW][26] = 40,
-	[2][1][2][1][RTW89_WW][33] = 40,
-	[2][1][2][1][RTW89_WW][41] = 40,
-	[2][1][2][1][RTW89_WW][48] = 40,
-	[2][1][2][1][RTW89_WW][56] = 40,
-	[2][1][2][1][RTW89_WW][63] = 40,
-	[2][1][2][1][RTW89_WW][71] = 40,
-	[2][1][2][1][RTW89_WW][78] = 40,
-	[2][1][2][1][RTW89_WW][86] = 40,
-	[2][1][2][1][RTW89_WW][93] = 40,
-	[2][1][2][1][RTW89_WW][101] = 40,
+	[2][1][2][1][RTW89_WW][3] = 22,
+	[2][1][2][1][RTW89_WW][11] = 20,
+	[2][1][2][1][RTW89_WW][18] = 20,
+	[2][1][2][1][RTW89_WW][26] = 20,
+	[2][1][2][1][RTW89_WW][33] = 20,
+	[2][1][2][1][RTW89_WW][41] = 22,
+	[2][1][2][1][RTW89_WW][48] = 22,
+	[2][1][2][1][RTW89_WW][56] = 20,
+	[2][1][2][1][RTW89_WW][63] = 22,
+	[2][1][2][1][RTW89_WW][71] = 20,
+	[2][1][2][1][RTW89_WW][78] = 20,
+	[2][1][2][1][RTW89_WW][86] = 20,
+	[2][1][2][1][RTW89_WW][93] = 22,
+	[2][1][2][1][RTW89_WW][101] = 22,
 	[2][1][2][1][RTW89_WW][108] = 0,
 	[2][1][2][1][RTW89_WW][116] = 0,
-	[3][0][2][0][RTW89_WW][7] = 56,
-	[3][0][2][0][RTW89_WW][22] = 56,
-	[3][0][2][0][RTW89_WW][37] = 56,
-	[3][0][2][0][RTW89_WW][52] = 56,
-	[3][0][2][0][RTW89_WW][67] = 56,
-	[3][0][2][0][RTW89_WW][82] = 56,
-	[3][0][2][0][RTW89_WW][97] = 56,
+	[3][0][2][0][RTW89_WW][7] = 38,
+	[3][0][2][0][RTW89_WW][22] = 38,
+	[3][0][2][0][RTW89_WW][37] = 38,
+	[3][0][2][0][RTW89_WW][52] = 54,
+	[3][0][2][0][RTW89_WW][67] = 54,
+	[3][0][2][0][RTW89_WW][82] = 26,
+	[3][0][2][0][RTW89_WW][97] = 26,
 	[3][0][2][0][RTW89_WW][112] = 0,
-	[3][1][2][0][RTW89_WW][7] = 44,
-	[3][1][2][0][RTW89_WW][22] = 44,
-	[3][1][2][0][RTW89_WW][37] = 44,
-	[3][1][2][0][RTW89_WW][52] = 44,
-	[3][1][2][0][RTW89_WW][67] = 44,
-	[3][1][2][0][RTW89_WW][82] = 44,
-	[3][1][2][0][RTW89_WW][97] = 44,
+	[3][1][2][0][RTW89_WW][7] = 32,
+	[3][1][2][0][RTW89_WW][22] = 30,
+	[3][1][2][0][RTW89_WW][37] = 30,
+	[3][1][2][0][RTW89_WW][52] = 30,
+	[3][1][2][0][RTW89_WW][67] = 32,
+	[3][1][2][0][RTW89_WW][82] = 24,
+	[3][1][2][0][RTW89_WW][97] = 14,
 	[3][1][2][0][RTW89_WW][112] = 0,
 	[3][1][2][1][RTW89_WW][7] = 32,
-	[3][1][2][1][RTW89_WW][22] = 32,
-	[3][1][2][1][RTW89_WW][37] = 32,
-	[3][1][2][1][RTW89_WW][52] = 32,
+	[3][1][2][1][RTW89_WW][22] = 30,
+	[3][1][2][1][RTW89_WW][37] = 30,
+	[3][1][2][1][RTW89_WW][52] = 30,
 	[3][1][2][1][RTW89_WW][67] = 32,
-	[3][1][2][1][RTW89_WW][82] = 32,
-	[3][1][2][1][RTW89_WW][97] = 32,
+	[3][1][2][1][RTW89_WW][82] = 24,
+	[3][1][2][1][RTW89_WW][97] = 14,
 	[3][1][2][1][RTW89_WW][112] = 0,
-	[0][0][1][0][RTW89_FCC][0] = 72,
-	[0][0][1][0][RTW89_FCC][2] = 72,
-	[0][0][1][0][RTW89_FCC][4] = 72,
-	[0][0][1][0][RTW89_FCC][6] = 72,
-	[0][0][1][0][RTW89_FCC][8] = 72,
-	[0][0][1][0][RTW89_FCC][10] = 72,
-	[0][0][1][0][RTW89_FCC][12] = 72,
-	[0][0][1][0][RTW89_FCC][14] = 72,
-	[0][0][1][0][RTW89_FCC][15] = 72,
-	[0][0][1][0][RTW89_FCC][17] = 72,
-	[0][0][1][0][RTW89_FCC][19] = 72,
-	[0][0][1][0][RTW89_FCC][21] = 72,
-	[0][0][1][0][RTW89_FCC][23] = 72,
-	[0][0][1][0][RTW89_FCC][25] = 72,
-	[0][0][1][0][RTW89_FCC][27] = 72,
-	[0][0][1][0][RTW89_FCC][29] = 72,
-	[0][0][1][0][RTW89_FCC][30] = 72,
-	[0][0][1][0][RTW89_FCC][32] = 72,
-	[0][0][1][0][RTW89_FCC][34] = 72,
-	[0][0][1][0][RTW89_FCC][36] = 72,
-	[0][0][1][0][RTW89_FCC][38] = 72,
-	[0][0][1][0][RTW89_FCC][40] = 72,
-	[0][0][1][0][RTW89_FCC][42] = 72,
-	[0][0][1][0][RTW89_FCC][44] = 72,
-	[0][0][1][0][RTW89_FCC][45] = 72,
-	[0][0][1][0][RTW89_FCC][47] = 72,
-	[0][0][1][0][RTW89_FCC][49] = 72,
-	[0][0][1][0][RTW89_FCC][51] = 72,
-	[0][0][1][0][RTW89_FCC][53] = 72,
-	[0][0][1][0][RTW89_FCC][55] = 72,
-	[0][0][1][0][RTW89_FCC][57] = 72,
-	[0][0][1][0][RTW89_FCC][59] = 72,
-	[0][0][1][0][RTW89_FCC][60] = 72,
-	[0][0][1][0][RTW89_FCC][62] = 72,
-	[0][0][1][0][RTW89_FCC][64] = 72,
-	[0][0][1][0][RTW89_FCC][66] = 72,
-	[0][0][1][0][RTW89_FCC][68] = 72,
-	[0][0][1][0][RTW89_FCC][70] = 72,
-	[0][0][1][0][RTW89_FCC][72] = 72,
-	[0][0][1][0][RTW89_FCC][74] = 72,
-	[0][0][1][0][RTW89_FCC][75] = 72,
-	[0][0][1][0][RTW89_FCC][77] = 72,
-	[0][0][1][0][RTW89_FCC][79] = 72,
-	[0][0][1][0][RTW89_FCC][81] = 72,
-	[0][0][1][0][RTW89_FCC][83] = 72,
-	[0][0][1][0][RTW89_FCC][85] = 72,
-	[0][0][1][0][RTW89_FCC][87] = 72,
-	[0][0][1][0][RTW89_FCC][89] = 72,
-	[0][0][1][0][RTW89_FCC][90] = 72,
-	[0][0][1][0][RTW89_FCC][92] = 72,
-	[0][0][1][0][RTW89_FCC][94] = 72,
-	[0][0][1][0][RTW89_FCC][96] = 72,
-	[0][0][1][0][RTW89_FCC][98] = 72,
-	[0][0][1][0][RTW89_FCC][100] = 72,
-	[0][0][1][0][RTW89_FCC][102] = 72,
-	[0][0][1][0][RTW89_FCC][104] = 72,
-	[0][0][1][0][RTW89_FCC][105] = 72,
-	[0][0][1][0][RTW89_FCC][107] = 72,
-	[0][0][1][0][RTW89_FCC][109] = 72,
+	[0][0][1][0][RTW89_FCC][0] = 24,
+	[0][0][1][0][RTW89_ETSI][0] = 66,
+	[0][0][1][0][RTW89_KCC][0] = 24,
+	[0][0][1][0][RTW89_FCC][2] = 22,
+	[0][0][1][0][RTW89_ETSI][2] = 66,
+	[0][0][1][0][RTW89_KCC][2] = 24,
+	[0][0][1][0][RTW89_FCC][4] = 22,
+	[0][0][1][0][RTW89_ETSI][4] = 66,
+	[0][0][1][0][RTW89_KCC][4] = 24,
+	[0][0][1][0][RTW89_FCC][6] = 22,
+	[0][0][1][0][RTW89_ETSI][6] = 66,
+	[0][0][1][0][RTW89_KCC][6] = 24,
+	[0][0][1][0][RTW89_FCC][8] = 22,
+	[0][0][1][0][RTW89_ETSI][8] = 66,
+	[0][0][1][0][RTW89_KCC][8] = 24,
+	[0][0][1][0][RTW89_FCC][10] = 22,
+	[0][0][1][0][RTW89_ETSI][10] = 66,
+	[0][0][1][0][RTW89_KCC][10] = 24,
+	[0][0][1][0][RTW89_FCC][12] = 22,
+	[0][0][1][0][RTW89_ETSI][12] = 66,
+	[0][0][1][0][RTW89_KCC][12] = 24,
+	[0][0][1][0][RTW89_FCC][14] = 22,
+	[0][0][1][0][RTW89_ETSI][14] = 66,
+	[0][0][1][0][RTW89_KCC][14] = 24,
+	[0][0][1][0][RTW89_FCC][15] = 22,
+	[0][0][1][0][RTW89_ETSI][15] = 66,
+	[0][0][1][0][RTW89_KCC][15] = 24,
+	[0][0][1][0][RTW89_FCC][17] = 22,
+	[0][0][1][0][RTW89_ETSI][17] = 66,
+	[0][0][1][0][RTW89_KCC][17] = 24,
+	[0][0][1][0][RTW89_FCC][19] = 22,
+	[0][0][1][0][RTW89_ETSI][19] = 66,
+	[0][0][1][0][RTW89_KCC][19] = 24,
+	[0][0][1][0][RTW89_FCC][21] = 22,
+	[0][0][1][0][RTW89_ETSI][21] = 66,
+	[0][0][1][0][RTW89_KCC][21] = 24,
+	[0][0][1][0][RTW89_FCC][23] = 22,
+	[0][0][1][0][RTW89_ETSI][23] = 66,
+	[0][0][1][0][RTW89_KCC][23] = 24,
+	[0][0][1][0][RTW89_FCC][25] = 22,
+	[0][0][1][0][RTW89_ETSI][25] = 66,
+	[0][0][1][0][RTW89_KCC][25] = 24,
+	[0][0][1][0][RTW89_FCC][27] = 22,
+	[0][0][1][0][RTW89_ETSI][27] = 66,
+	[0][0][1][0][RTW89_KCC][27] = 24,
+	[0][0][1][0][RTW89_FCC][29] = 22,
+	[0][0][1][0][RTW89_ETSI][29] = 66,
+	[0][0][1][0][RTW89_KCC][29] = 24,
+	[0][0][1][0][RTW89_FCC][30] = 22,
+	[0][0][1][0][RTW89_ETSI][30] = 66,
+	[0][0][1][0][RTW89_KCC][30] = 24,
+	[0][0][1][0][RTW89_FCC][32] = 22,
+	[0][0][1][0][RTW89_ETSI][32] = 66,
+	[0][0][1][0][RTW89_KCC][32] = 24,
+	[0][0][1][0][RTW89_FCC][34] = 22,
+	[0][0][1][0][RTW89_ETSI][34] = 66,
+	[0][0][1][0][RTW89_KCC][34] = 24,
+	[0][0][1][0][RTW89_FCC][36] = 22,
+	[0][0][1][0][RTW89_ETSI][36] = 66,
+	[0][0][1][0][RTW89_KCC][36] = 24,
+	[0][0][1][0][RTW89_FCC][38] = 22,
+	[0][0][1][0][RTW89_ETSI][38] = 66,
+	[0][0][1][0][RTW89_KCC][38] = 24,
+	[0][0][1][0][RTW89_FCC][40] = 22,
+	[0][0][1][0][RTW89_ETSI][40] = 66,
+	[0][0][1][0][RTW89_KCC][40] = 24,
+	[0][0][1][0][RTW89_FCC][42] = 22,
+	[0][0][1][0][RTW89_ETSI][42] = 66,
+	[0][0][1][0][RTW89_KCC][42] = 24,
+	[0][0][1][0][RTW89_FCC][44] = 22,
+	[0][0][1][0][RTW89_ETSI][44] = 66,
+	[0][0][1][0][RTW89_KCC][44] = 24,
+	[0][0][1][0][RTW89_FCC][45] = 22,
+	[0][0][1][0][RTW89_ETSI][45] = 127,
+	[0][0][1][0][RTW89_KCC][45] = 24,
+	[0][0][1][0][RTW89_FCC][47] = 22,
+	[0][0][1][0][RTW89_ETSI][47] = 127,
+	[0][0][1][0][RTW89_KCC][47] = 24,
+	[0][0][1][0][RTW89_FCC][49] = 24,
+	[0][0][1][0][RTW89_ETSI][49] = 127,
+	[0][0][1][0][RTW89_KCC][49] = 24,
+	[0][0][1][0][RTW89_FCC][51] = 22,
+	[0][0][1][0][RTW89_ETSI][51] = 127,
+	[0][0][1][0][RTW89_KCC][51] = 24,
+	[0][0][1][0][RTW89_FCC][53] = 22,
+	[0][0][1][0][RTW89_ETSI][53] = 127,
+	[0][0][1][0][RTW89_KCC][53] = 24,
+	[0][0][1][0][RTW89_FCC][55] = 22,
+	[0][0][1][0][RTW89_ETSI][55] = 127,
+	[0][0][1][0][RTW89_KCC][55] = 26,
+	[0][0][1][0][RTW89_FCC][57] = 22,
+	[0][0][1][0][RTW89_ETSI][57] = 127,
+	[0][0][1][0][RTW89_KCC][57] = 26,
+	[0][0][1][0][RTW89_FCC][59] = 22,
+	[0][0][1][0][RTW89_ETSI][59] = 127,
+	[0][0][1][0][RTW89_KCC][59] = 26,
+	[0][0][1][0][RTW89_FCC][60] = 22,
+	[0][0][1][0][RTW89_ETSI][60] = 127,
+	[0][0][1][0][RTW89_KCC][60] = 26,
+	[0][0][1][0][RTW89_FCC][62] = 22,
+	[0][0][1][0][RTW89_ETSI][62] = 127,
+	[0][0][1][0][RTW89_KCC][62] = 26,
+	[0][0][1][0][RTW89_FCC][64] = 22,
+	[0][0][1][0][RTW89_ETSI][64] = 127,
+	[0][0][1][0][RTW89_KCC][64] = 26,
+	[0][0][1][0][RTW89_FCC][66] = 22,
+	[0][0][1][0][RTW89_ETSI][66] = 127,
+	[0][0][1][0][RTW89_KCC][66] = 26,
+	[0][0][1][0][RTW89_FCC][68] = 22,
+	[0][0][1][0][RTW89_ETSI][68] = 127,
+	[0][0][1][0][RTW89_KCC][68] = 26,
+	[0][0][1][0][RTW89_FCC][70] = 24,
+	[0][0][1][0][RTW89_ETSI][70] = 127,
+	[0][0][1][0][RTW89_KCC][70] = 26,
+	[0][0][1][0][RTW89_FCC][72] = 22,
+	[0][0][1][0][RTW89_ETSI][72] = 127,
+	[0][0][1][0][RTW89_KCC][72] = 26,
+	[0][0][1][0][RTW89_FCC][74] = 22,
+	[0][0][1][0][RTW89_ETSI][74] = 127,
+	[0][0][1][0][RTW89_KCC][74] = 26,
+	[0][0][1][0][RTW89_FCC][75] = 22,
+	[0][0][1][0][RTW89_ETSI][75] = 127,
+	[0][0][1][0][RTW89_KCC][75] = 26,
+	[0][0][1][0][RTW89_FCC][77] = 22,
+	[0][0][1][0][RTW89_ETSI][77] = 127,
+	[0][0][1][0][RTW89_KCC][77] = 26,
+	[0][0][1][0][RTW89_FCC][79] = 22,
+	[0][0][1][0][RTW89_ETSI][79] = 127,
+	[0][0][1][0][RTW89_KCC][79] = 26,
+	[0][0][1][0][RTW89_FCC][81] = 22,
+	[0][0][1][0][RTW89_ETSI][81] = 127,
+	[0][0][1][0][RTW89_KCC][81] = 26,
+	[0][0][1][0][RTW89_FCC][83] = 22,
+	[0][0][1][0][RTW89_ETSI][83] = 127,
+	[0][0][1][0][RTW89_KCC][83] = 32,
+	[0][0][1][0][RTW89_FCC][85] = 22,
+	[0][0][1][0][RTW89_ETSI][85] = 127,
+	[0][0][1][0][RTW89_KCC][85] = 32,
+	[0][0][1][0][RTW89_FCC][87] = 22,
+	[0][0][1][0][RTW89_ETSI][87] = 127,
+	[0][0][1][0][RTW89_KCC][87] = 32,
+	[0][0][1][0][RTW89_FCC][89] = 22,
+	[0][0][1][0][RTW89_ETSI][89] = 127,
+	[0][0][1][0][RTW89_KCC][89] = 32,
+	[0][0][1][0][RTW89_FCC][90] = 22,
+	[0][0][1][0][RTW89_ETSI][90] = 127,
+	[0][0][1][0][RTW89_KCC][90] = 32,
+	[0][0][1][0][RTW89_FCC][92] = 22,
+	[0][0][1][0][RTW89_ETSI][92] = 127,
+	[0][0][1][0][RTW89_KCC][92] = 32,
+	[0][0][1][0][RTW89_FCC][94] = 22,
+	[0][0][1][0][RTW89_ETSI][94] = 127,
+	[0][0][1][0][RTW89_KCC][94] = 32,
+	[0][0][1][0][RTW89_FCC][96] = 22,
+	[0][0][1][0][RTW89_ETSI][96] = 127,
+	[0][0][1][0][RTW89_KCC][96] = 32,
+	[0][0][1][0][RTW89_FCC][98] = 22,
+	[0][0][1][0][RTW89_ETSI][98] = 127,
+	[0][0][1][0][RTW89_KCC][98] = 32,
+	[0][0][1][0][RTW89_FCC][100] = 22,
+	[0][0][1][0][RTW89_ETSI][100] = 127,
+	[0][0][1][0][RTW89_KCC][100] = 32,
+	[0][0][1][0][RTW89_FCC][102] = 22,
+	[0][0][1][0][RTW89_ETSI][102] = 127,
+	[0][0][1][0][RTW89_KCC][102] = 32,
+	[0][0][1][0][RTW89_FCC][104] = 22,
+	[0][0][1][0][RTW89_ETSI][104] = 127,
+	[0][0][1][0][RTW89_KCC][104] = 32,
+	[0][0][1][0][RTW89_FCC][105] = 22,
+	[0][0][1][0][RTW89_ETSI][105] = 127,
+	[0][0][1][0][RTW89_KCC][105] = 32,
+	[0][0][1][0][RTW89_FCC][107] = 24,
+	[0][0][1][0][RTW89_ETSI][107] = 127,
+	[0][0][1][0][RTW89_KCC][107] = 32,
+	[0][0][1][0][RTW89_FCC][109] = 24,
+	[0][0][1][0][RTW89_ETSI][109] = 127,
+	[0][0][1][0][RTW89_KCC][109] = 32,
 	[0][0][1][0][RTW89_FCC][111] = 127,
+	[0][0][1][0][RTW89_ETSI][111] = 127,
+	[0][0][1][0][RTW89_KCC][111] = 127,
 	[0][0][1][0][RTW89_FCC][113] = 127,
+	[0][0][1][0][RTW89_ETSI][113] = 127,
+	[0][0][1][0][RTW89_KCC][113] = 127,
 	[0][0][1][0][RTW89_FCC][115] = 127,
+	[0][0][1][0][RTW89_ETSI][115] = 127,
+	[0][0][1][0][RTW89_KCC][115] = 127,
 	[0][0][1][0][RTW89_FCC][117] = 127,
+	[0][0][1][0][RTW89_ETSI][117] = 127,
+	[0][0][1][0][RTW89_KCC][117] = 127,
 	[0][0][1][0][RTW89_FCC][119] = 127,
-	[0][1][1][0][RTW89_FCC][0] = 60,
-	[0][1][1][0][RTW89_FCC][2] = 60,
-	[0][1][1][0][RTW89_FCC][4] = 60,
-	[0][1][1][0][RTW89_FCC][6] = 60,
-	[0][1][1][0][RTW89_FCC][8] = 60,
-	[0][1][1][0][RTW89_FCC][10] = 60,
-	[0][1][1][0][RTW89_FCC][12] = 60,
-	[0][1][1][0][RTW89_FCC][14] = 60,
-	[0][1][1][0][RTW89_FCC][15] = 60,
-	[0][1][1][0][RTW89_FCC][17] = 60,
-	[0][1][1][0][RTW89_FCC][19] = 60,
-	[0][1][1][0][RTW89_FCC][21] = 60,
-	[0][1][1][0][RTW89_FCC][23] = 60,
-	[0][1][1][0][RTW89_FCC][25] = 60,
-	[0][1][1][0][RTW89_FCC][27] = 60,
-	[0][1][1][0][RTW89_FCC][29] = 60,
-	[0][1][1][0][RTW89_FCC][30] = 60,
-	[0][1][1][0][RTW89_FCC][32] = 60,
-	[0][1][1][0][RTW89_FCC][34] = 60,
-	[0][1][1][0][RTW89_FCC][36] = 60,
-	[0][1][1][0][RTW89_FCC][38] = 60,
-	[0][1][1][0][RTW89_FCC][40] = 60,
-	[0][1][1][0][RTW89_FCC][42] = 60,
-	[0][1][1][0][RTW89_FCC][44] = 60,
-	[0][1][1][0][RTW89_FCC][45] = 60,
-	[0][1][1][0][RTW89_FCC][47] = 60,
-	[0][1][1][0][RTW89_FCC][49] = 60,
-	[0][1][1][0][RTW89_FCC][51] = 60,
-	[0][1][1][0][RTW89_FCC][53] = 60,
-	[0][1][1][0][RTW89_FCC][55] = 60,
-	[0][1][1][0][RTW89_FCC][57] = 60,
-	[0][1][1][0][RTW89_FCC][59] = 60,
-	[0][1][1][0][RTW89_FCC][60] = 60,
-	[0][1][1][0][RTW89_FCC][62] = 60,
-	[0][1][1][0][RTW89_FCC][64] = 60,
-	[0][1][1][0][RTW89_FCC][66] = 60,
-	[0][1][1][0][RTW89_FCC][68] = 60,
-	[0][1][1][0][RTW89_FCC][70] = 60,
-	[0][1][1][0][RTW89_FCC][72] = 60,
-	[0][1][1][0][RTW89_FCC][74] = 60,
-	[0][1][1][0][RTW89_FCC][75] = 60,
-	[0][1][1][0][RTW89_FCC][77] = 60,
-	[0][1][1][0][RTW89_FCC][79] = 60,
-	[0][1][1][0][RTW89_FCC][81] = 60,
-	[0][1][1][0][RTW89_FCC][83] = 60,
-	[0][1][1][0][RTW89_FCC][85] = 60,
-	[0][1][1][0][RTW89_FCC][87] = 60,
-	[0][1][1][0][RTW89_FCC][89] = 60,
-	[0][1][1][0][RTW89_FCC][90] = 60,
-	[0][1][1][0][RTW89_FCC][92] = 60,
-	[0][1][1][0][RTW89_FCC][94] = 60,
-	[0][1][1][0][RTW89_FCC][96] = 60,
-	[0][1][1][0][RTW89_FCC][98] = 60,
-	[0][1][1][0][RTW89_FCC][100] = 60,
-	[0][1][1][0][RTW89_FCC][102] = 60,
-	[0][1][1][0][RTW89_FCC][104] = 60,
-	[0][1][1][0][RTW89_FCC][105] = 60,
-	[0][1][1][0][RTW89_FCC][107] = 60,
-	[0][1][1][0][RTW89_FCC][109] = 60,
+	[0][0][1][0][RTW89_ETSI][119] = 127,
+	[0][0][1][0][RTW89_KCC][119] = 127,
+	[0][1][1][0][RTW89_FCC][0] = -2,
+	[0][1][1][0][RTW89_ETSI][0] = 54,
+	[0][1][1][0][RTW89_KCC][0] = 12,
+	[0][1][1][0][RTW89_FCC][2] = -4,
+	[0][1][1][0][RTW89_ETSI][2] = 54,
+	[0][1][1][0][RTW89_KCC][2] = 12,
+	[0][1][1][0][RTW89_FCC][4] = -4,
+	[0][1][1][0][RTW89_ETSI][4] = 54,
+	[0][1][1][0][RTW89_KCC][4] = 12,
+	[0][1][1][0][RTW89_FCC][6] = -4,
+	[0][1][1][0][RTW89_ETSI][6] = 54,
+	[0][1][1][0][RTW89_KCC][6] = 12,
+	[0][1][1][0][RTW89_FCC][8] = -4,
+	[0][1][1][0][RTW89_ETSI][8] = 54,
+	[0][1][1][0][RTW89_KCC][8] = 12,
+	[0][1][1][0][RTW89_FCC][10] = -4,
+	[0][1][1][0][RTW89_ETSI][10] = 54,
+	[0][1][1][0][RTW89_KCC][10] = 12,
+	[0][1][1][0][RTW89_FCC][12] = -4,
+	[0][1][1][0][RTW89_ETSI][12] = 54,
+	[0][1][1][0][RTW89_KCC][12] = 12,
+	[0][1][1][0][RTW89_FCC][14] = -4,
+	[0][1][1][0][RTW89_ETSI][14] = 54,
+	[0][1][1][0][RTW89_KCC][14] = 12,
+	[0][1][1][0][RTW89_FCC][15] = -4,
+	[0][1][1][0][RTW89_ETSI][15] = 54,
+	[0][1][1][0][RTW89_KCC][15] = 12,
+	[0][1][1][0][RTW89_FCC][17] = -4,
+	[0][1][1][0][RTW89_ETSI][17] = 54,
+	[0][1][1][0][RTW89_KCC][17] = 12,
+	[0][1][1][0][RTW89_FCC][19] = -4,
+	[0][1][1][0][RTW89_ETSI][19] = 54,
+	[0][1][1][0][RTW89_KCC][19] = 12,
+	[0][1][1][0][RTW89_FCC][21] = -4,
+	[0][1][1][0][RTW89_ETSI][21] = 54,
+	[0][1][1][0][RTW89_KCC][21] = 12,
+	[0][1][1][0][RTW89_FCC][23] = -4,
+	[0][1][1][0][RTW89_ETSI][23] = 54,
+	[0][1][1][0][RTW89_KCC][23] = 12,
+	[0][1][1][0][RTW89_FCC][25] = -4,
+	[0][1][1][0][RTW89_ETSI][25] = 54,
+	[0][1][1][0][RTW89_KCC][25] = 12,
+	[0][1][1][0][RTW89_FCC][27] = -4,
+	[0][1][1][0][RTW89_ETSI][27] = 54,
+	[0][1][1][0][RTW89_KCC][27] = 12,
+	[0][1][1][0][RTW89_FCC][29] = -4,
+	[0][1][1][0][RTW89_ETSI][29] = 54,
+	[0][1][1][0][RTW89_KCC][29] = 12,
+	[0][1][1][0][RTW89_FCC][30] = -4,
+	[0][1][1][0][RTW89_ETSI][30] = 54,
+	[0][1][1][0][RTW89_KCC][30] = 12,
+	[0][1][1][0][RTW89_FCC][32] = -4,
+	[0][1][1][0][RTW89_ETSI][32] = 54,
+	[0][1][1][0][RTW89_KCC][32] = 12,
+	[0][1][1][0][RTW89_FCC][34] = -4,
+	[0][1][1][0][RTW89_ETSI][34] = 54,
+	[0][1][1][0][RTW89_KCC][34] = 12,
+	[0][1][1][0][RTW89_FCC][36] = -4,
+	[0][1][1][0][RTW89_ETSI][36] = 54,
+	[0][1][1][0][RTW89_KCC][36] = 12,
+	[0][1][1][0][RTW89_FCC][38] = -4,
+	[0][1][1][0][RTW89_ETSI][38] = 54,
+	[0][1][1][0][RTW89_KCC][38] = 12,
+	[0][1][1][0][RTW89_FCC][40] = -4,
+	[0][1][1][0][RTW89_ETSI][40] = 54,
+	[0][1][1][0][RTW89_KCC][40] = 12,
+	[0][1][1][0][RTW89_FCC][42] = -4,
+	[0][1][1][0][RTW89_ETSI][42] = 54,
+	[0][1][1][0][RTW89_KCC][42] = 12,
+	[0][1][1][0][RTW89_FCC][44] = -2,
+	[0][1][1][0][RTW89_ETSI][44] = 54,
+	[0][1][1][0][RTW89_KCC][44] = 12,
+	[0][1][1][0][RTW89_FCC][45] = -2,
+	[0][1][1][0][RTW89_ETSI][45] = 127,
+	[0][1][1][0][RTW89_KCC][45] = 12,
+	[0][1][1][0][RTW89_FCC][47] = -2,
+	[0][1][1][0][RTW89_ETSI][47] = 127,
+	[0][1][1][0][RTW89_KCC][47] = 12,
+	[0][1][1][0][RTW89_FCC][49] = -2,
+	[0][1][1][0][RTW89_ETSI][49] = 127,
+	[0][1][1][0][RTW89_KCC][49] = 12,
+	[0][1][1][0][RTW89_FCC][51] = -2,
+	[0][1][1][0][RTW89_ETSI][51] = 127,
+	[0][1][1][0][RTW89_KCC][51] = 12,
+	[0][1][1][0][RTW89_FCC][53] = -2,
+	[0][1][1][0][RTW89_ETSI][53] = 127,
+	[0][1][1][0][RTW89_KCC][53] = 12,
+	[0][1][1][0][RTW89_FCC][55] = -2,
+	[0][1][1][0][RTW89_ETSI][55] = 127,
+	[0][1][1][0][RTW89_KCC][55] = 12,
+	[0][1][1][0][RTW89_FCC][57] = -2,
+	[0][1][1][0][RTW89_ETSI][57] = 127,
+	[0][1][1][0][RTW89_KCC][57] = 12,
+	[0][1][1][0][RTW89_FCC][59] = -2,
+	[0][1][1][0][RTW89_ETSI][59] = 127,
+	[0][1][1][0][RTW89_KCC][59] = 12,
+	[0][1][1][0][RTW89_FCC][60] = -2,
+	[0][1][1][0][RTW89_ETSI][60] = 127,
+	[0][1][1][0][RTW89_KCC][60] = 12,
+	[0][1][1][0][RTW89_FCC][62] = -2,
+	[0][1][1][0][RTW89_ETSI][62] = 127,
+	[0][1][1][0][RTW89_KCC][62] = 12,
+	[0][1][1][0][RTW89_FCC][64] = -2,
+	[0][1][1][0][RTW89_ETSI][64] = 127,
+	[0][1][1][0][RTW89_KCC][64] = 12,
+	[0][1][1][0][RTW89_FCC][66] = -2,
+	[0][1][1][0][RTW89_ETSI][66] = 127,
+	[0][1][1][0][RTW89_KCC][66] = 12,
+	[0][1][1][0][RTW89_FCC][68] = -2,
+	[0][1][1][0][RTW89_ETSI][68] = 127,
+	[0][1][1][0][RTW89_KCC][68] = 12,
+	[0][1][1][0][RTW89_FCC][70] = -2,
+	[0][1][1][0][RTW89_ETSI][70] = 127,
+	[0][1][1][0][RTW89_KCC][70] = 12,
+	[0][1][1][0][RTW89_FCC][72] = -2,
+	[0][1][1][0][RTW89_ETSI][72] = 127,
+	[0][1][1][0][RTW89_KCC][72] = 12,
+	[0][1][1][0][RTW89_FCC][74] = -2,
+	[0][1][1][0][RTW89_ETSI][74] = 127,
+	[0][1][1][0][RTW89_KCC][74] = 12,
+	[0][1][1][0][RTW89_FCC][75] = -2,
+	[0][1][1][0][RTW89_ETSI][75] = 127,
+	[0][1][1][0][RTW89_KCC][75] = 12,
+	[0][1][1][0][RTW89_FCC][77] = -2,
+	[0][1][1][0][RTW89_ETSI][77] = 127,
+	[0][1][1][0][RTW89_KCC][77] = 12,
+	[0][1][1][0][RTW89_FCC][79] = -2,
+	[0][1][1][0][RTW89_ETSI][79] = 127,
+	[0][1][1][0][RTW89_KCC][79] = 12,
+	[0][1][1][0][RTW89_FCC][81] = -2,
+	[0][1][1][0][RTW89_ETSI][81] = 127,
+	[0][1][1][0][RTW89_KCC][81] = 12,
+	[0][1][1][0][RTW89_FCC][83] = -2,
+	[0][1][1][0][RTW89_ETSI][83] = 127,
+	[0][1][1][0][RTW89_KCC][83] = 20,
+	[0][1][1][0][RTW89_FCC][85] = -2,
+	[0][1][1][0][RTW89_ETSI][85] = 127,
+	[0][1][1][0][RTW89_KCC][85] = 20,
+	[0][1][1][0][RTW89_FCC][87] = -2,
+	[0][1][1][0][RTW89_ETSI][87] = 127,
+	[0][1][1][0][RTW89_KCC][87] = 20,
+	[0][1][1][0][RTW89_FCC][89] = -2,
+	[0][1][1][0][RTW89_ETSI][89] = 127,
+	[0][1][1][0][RTW89_KCC][89] = 20,
+	[0][1][1][0][RTW89_FCC][90] = -2,
+	[0][1][1][0][RTW89_ETSI][90] = 127,
+	[0][1][1][0][RTW89_KCC][90] = 20,
+	[0][1][1][0][RTW89_FCC][92] = -2,
+	[0][1][1][0][RTW89_ETSI][92] = 127,
+	[0][1][1][0][RTW89_KCC][92] = 20,
+	[0][1][1][0][RTW89_FCC][94] = -2,
+	[0][1][1][0][RTW89_ETSI][94] = 127,
+	[0][1][1][0][RTW89_KCC][94] = 20,
+	[0][1][1][0][RTW89_FCC][96] = -2,
+	[0][1][1][0][RTW89_ETSI][96] = 127,
+	[0][1][1][0][RTW89_KCC][96] = 20,
+	[0][1][1][0][RTW89_FCC][98] = -2,
+	[0][1][1][0][RTW89_ETSI][98] = 127,
+	[0][1][1][0][RTW89_KCC][98] = 20,
+	[0][1][1][0][RTW89_FCC][100] = -2,
+	[0][1][1][0][RTW89_ETSI][100] = 127,
+	[0][1][1][0][RTW89_KCC][100] = 20,
+	[0][1][1][0][RTW89_FCC][102] = -2,
+	[0][1][1][0][RTW89_ETSI][102] = 127,
+	[0][1][1][0][RTW89_KCC][102] = 20,
+	[0][1][1][0][RTW89_FCC][104] = -2,
+	[0][1][1][0][RTW89_ETSI][104] = 127,
+	[0][1][1][0][RTW89_KCC][104] = 20,
+	[0][1][1][0][RTW89_FCC][105] = -2,
+	[0][1][1][0][RTW89_ETSI][105] = 127,
+	[0][1][1][0][RTW89_KCC][105] = 20,
+	[0][1][1][0][RTW89_FCC][107] = 0,
+	[0][1][1][0][RTW89_ETSI][107] = 127,
+	[0][1][1][0][RTW89_KCC][107] = 20,
+	[0][1][1][0][RTW89_FCC][109] = 0,
+	[0][1][1][0][RTW89_ETSI][109] = 127,
+	[0][1][1][0][RTW89_KCC][109] = 20,
 	[0][1][1][0][RTW89_FCC][111] = 127,
+	[0][1][1][0][RTW89_ETSI][111] = 127,
+	[0][1][1][0][RTW89_KCC][111] = 127,
 	[0][1][1][0][RTW89_FCC][113] = 127,
+	[0][1][1][0][RTW89_ETSI][113] = 127,
+	[0][1][1][0][RTW89_KCC][113] = 127,
 	[0][1][1][0][RTW89_FCC][115] = 127,
+	[0][1][1][0][RTW89_ETSI][115] = 127,
+	[0][1][1][0][RTW89_KCC][115] = 127,
 	[0][1][1][0][RTW89_FCC][117] = 127,
+	[0][1][1][0][RTW89_ETSI][117] = 127,
+	[0][1][1][0][RTW89_KCC][117] = 127,
 	[0][1][1][0][RTW89_FCC][119] = 127,
-	[0][0][2][0][RTW89_FCC][0] = 72,
-	[0][0][2][0][RTW89_FCC][2] = 72,
-	[0][0][2][0][RTW89_FCC][4] = 72,
-	[0][0][2][0][RTW89_FCC][6] = 72,
-	[0][0][2][0][RTW89_FCC][8] = 72,
-	[0][0][2][0][RTW89_FCC][10] = 72,
-	[0][0][2][0][RTW89_FCC][12] = 72,
-	[0][0][2][0][RTW89_FCC][14] = 72,
-	[0][0][2][0][RTW89_FCC][15] = 72,
-	[0][0][2][0][RTW89_FCC][17] = 72,
-	[0][0][2][0][RTW89_FCC][19] = 72,
-	[0][0][2][0][RTW89_FCC][21] = 72,
-	[0][0][2][0][RTW89_FCC][23] = 72,
-	[0][0][2][0][RTW89_FCC][25] = 72,
-	[0][0][2][0][RTW89_FCC][27] = 72,
-	[0][0][2][0][RTW89_FCC][29] = 72,
-	[0][0][2][0][RTW89_FCC][30] = 72,
-	[0][0][2][0][RTW89_FCC][32] = 72,
-	[0][0][2][0][RTW89_FCC][34] = 72,
-	[0][0][2][0][RTW89_FCC][36] = 72,
-	[0][0][2][0][RTW89_FCC][38] = 72,
-	[0][0][2][0][RTW89_FCC][40] = 72,
-	[0][0][2][0][RTW89_FCC][42] = 72,
-	[0][0][2][0][RTW89_FCC][44] = 72,
-	[0][0][2][0][RTW89_FCC][45] = 72,
-	[0][0][2][0][RTW89_FCC][47] = 72,
-	[0][0][2][0][RTW89_FCC][49] = 72,
-	[0][0][2][0][RTW89_FCC][51] = 72,
-	[0][0][2][0][RTW89_FCC][53] = 72,
-	[0][0][2][0][RTW89_FCC][55] = 72,
-	[0][0][2][0][RTW89_FCC][57] = 72,
-	[0][0][2][0][RTW89_FCC][59] = 72,
-	[0][0][2][0][RTW89_FCC][60] = 72,
-	[0][0][2][0][RTW89_FCC][62] = 72,
-	[0][0][2][0][RTW89_FCC][64] = 72,
-	[0][0][2][0][RTW89_FCC][66] = 72,
-	[0][0][2][0][RTW89_FCC][68] = 72,
-	[0][0][2][0][RTW89_FCC][70] = 72,
-	[0][0][2][0][RTW89_FCC][72] = 72,
-	[0][0][2][0][RTW89_FCC][74] = 72,
-	[0][0][2][0][RTW89_FCC][75] = 72,
-	[0][0][2][0][RTW89_FCC][77] = 72,
-	[0][0][2][0][RTW89_FCC][79] = 72,
-	[0][0][2][0][RTW89_FCC][81] = 72,
-	[0][0][2][0][RTW89_FCC][83] = 72,
-	[0][0][2][0][RTW89_FCC][85] = 72,
-	[0][0][2][0][RTW89_FCC][87] = 72,
-	[0][0][2][0][RTW89_FCC][89] = 72,
-	[0][0][2][0][RTW89_FCC][90] = 72,
-	[0][0][2][0][RTW89_FCC][92] = 72,
-	[0][0][2][0][RTW89_FCC][94] = 72,
-	[0][0][2][0][RTW89_FCC][96] = 72,
-	[0][0][2][0][RTW89_FCC][98] = 72,
-	[0][0][2][0][RTW89_FCC][100] = 72,
-	[0][0][2][0][RTW89_FCC][102] = 72,
-	[0][0][2][0][RTW89_FCC][104] = 72,
-	[0][0][2][0][RTW89_FCC][105] = 72,
-	[0][0][2][0][RTW89_FCC][107] = 72,
-	[0][0][2][0][RTW89_FCC][109] = 72,
+	[0][1][1][0][RTW89_ETSI][119] = 127,
+	[0][1][1][0][RTW89_KCC][119] = 127,
+	[0][0][2][0][RTW89_FCC][0] = 24,
+	[0][0][2][0][RTW89_ETSI][0] = 66,
+	[0][0][2][0][RTW89_KCC][0] = 24,
+	[0][0][2][0][RTW89_FCC][2] = 22,
+	[0][0][2][0][RTW89_ETSI][2] = 66,
+	[0][0][2][0][RTW89_KCC][2] = 24,
+	[0][0][2][0][RTW89_FCC][4] = 22,
+	[0][0][2][0][RTW89_ETSI][4] = 66,
+	[0][0][2][0][RTW89_KCC][4] = 24,
+	[0][0][2][0][RTW89_FCC][6] = 22,
+	[0][0][2][0][RTW89_ETSI][6] = 66,
+	[0][0][2][0][RTW89_KCC][6] = 24,
+	[0][0][2][0][RTW89_FCC][8] = 22,
+	[0][0][2][0][RTW89_ETSI][8] = 66,
+	[0][0][2][0][RTW89_KCC][8] = 24,
+	[0][0][2][0][RTW89_FCC][10] = 22,
+	[0][0][2][0][RTW89_ETSI][10] = 66,
+	[0][0][2][0][RTW89_KCC][10] = 24,
+	[0][0][2][0][RTW89_FCC][12] = 22,
+	[0][0][2][0][RTW89_ETSI][12] = 66,
+	[0][0][2][0][RTW89_KCC][12] = 24,
+	[0][0][2][0][RTW89_FCC][14] = 22,
+	[0][0][2][0][RTW89_ETSI][14] = 66,
+	[0][0][2][0][RTW89_KCC][14] = 24,
+	[0][0][2][0][RTW89_FCC][15] = 22,
+	[0][0][2][0][RTW89_ETSI][15] = 66,
+	[0][0][2][0][RTW89_KCC][15] = 24,
+	[0][0][2][0][RTW89_FCC][17] = 22,
+	[0][0][2][0][RTW89_ETSI][17] = 66,
+	[0][0][2][0][RTW89_KCC][17] = 24,
+	[0][0][2][0][RTW89_FCC][19] = 22,
+	[0][0][2][0][RTW89_ETSI][19] = 66,
+	[0][0][2][0][RTW89_KCC][19] = 24,
+	[0][0][2][0][RTW89_FCC][21] = 22,
+	[0][0][2][0][RTW89_ETSI][21] = 66,
+	[0][0][2][0][RTW89_KCC][21] = 24,
+	[0][0][2][0][RTW89_FCC][23] = 22,
+	[0][0][2][0][RTW89_ETSI][23] = 66,
+	[0][0][2][0][RTW89_KCC][23] = 24,
+	[0][0][2][0][RTW89_FCC][25] = 22,
+	[0][0][2][0][RTW89_ETSI][25] = 66,
+	[0][0][2][0][RTW89_KCC][25] = 24,
+	[0][0][2][0][RTW89_FCC][27] = 22,
+	[0][0][2][0][RTW89_ETSI][27] = 66,
+	[0][0][2][0][RTW89_KCC][27] = 24,
+	[0][0][2][0][RTW89_FCC][29] = 22,
+	[0][0][2][0][RTW89_ETSI][29] = 66,
+	[0][0][2][0][RTW89_KCC][29] = 24,
+	[0][0][2][0][RTW89_FCC][30] = 22,
+	[0][0][2][0][RTW89_ETSI][30] = 66,
+	[0][0][2][0][RTW89_KCC][30] = 24,
+	[0][0][2][0][RTW89_FCC][32] = 22,
+	[0][0][2][0][RTW89_ETSI][32] = 66,
+	[0][0][2][0][RTW89_KCC][32] = 24,
+	[0][0][2][0][RTW89_FCC][34] = 22,
+	[0][0][2][0][RTW89_ETSI][34] = 66,
+	[0][0][2][0][RTW89_KCC][34] = 24,
+	[0][0][2][0][RTW89_FCC][36] = 22,
+	[0][0][2][0][RTW89_ETSI][36] = 66,
+	[0][0][2][0][RTW89_KCC][36] = 24,
+	[0][0][2][0][RTW89_FCC][38] = 22,
+	[0][0][2][0][RTW89_ETSI][38] = 66,
+	[0][0][2][0][RTW89_KCC][38] = 24,
+	[0][0][2][0][RTW89_FCC][40] = 22,
+	[0][0][2][0][RTW89_ETSI][40] = 66,
+	[0][0][2][0][RTW89_KCC][40] = 24,
+	[0][0][2][0][RTW89_FCC][42] = 22,
+	[0][0][2][0][RTW89_ETSI][42] = 66,
+	[0][0][2][0][RTW89_KCC][42] = 24,
+	[0][0][2][0][RTW89_FCC][44] = 22,
+	[0][0][2][0][RTW89_ETSI][44] = 66,
+	[0][0][2][0][RTW89_KCC][44] = 24,
+	[0][0][2][0][RTW89_FCC][45] = 22,
+	[0][0][2][0][RTW89_ETSI][45] = 127,
+	[0][0][2][0][RTW89_KCC][45] = 24,
+	[0][0][2][0][RTW89_FCC][47] = 22,
+	[0][0][2][0][RTW89_ETSI][47] = 127,
+	[0][0][2][0][RTW89_KCC][47] = 24,
+	[0][0][2][0][RTW89_FCC][49] = 24,
+	[0][0][2][0][RTW89_ETSI][49] = 127,
+	[0][0][2][0][RTW89_KCC][49] = 24,
+	[0][0][2][0][RTW89_FCC][51] = 22,
+	[0][0][2][0][RTW89_ETSI][51] = 127,
+	[0][0][2][0][RTW89_KCC][51] = 24,
+	[0][0][2][0][RTW89_FCC][53] = 22,
+	[0][0][2][0][RTW89_ETSI][53] = 127,
+	[0][0][2][0][RTW89_KCC][53] = 24,
+	[0][0][2][0][RTW89_FCC][55] = 22,
+	[0][0][2][0][RTW89_ETSI][55] = 127,
+	[0][0][2][0][RTW89_KCC][55] = 26,
+	[0][0][2][0][RTW89_FCC][57] = 22,
+	[0][0][2][0][RTW89_ETSI][57] = 127,
+	[0][0][2][0][RTW89_KCC][57] = 26,
+	[0][0][2][0][RTW89_FCC][59] = 22,
+	[0][0][2][0][RTW89_ETSI][59] = 127,
+	[0][0][2][0][RTW89_KCC][59] = 26,
+	[0][0][2][0][RTW89_FCC][60] = 22,
+	[0][0][2][0][RTW89_ETSI][60] = 127,
+	[0][0][2][0][RTW89_KCC][60] = 26,
+	[0][0][2][0][RTW89_FCC][62] = 22,
+	[0][0][2][0][RTW89_ETSI][62] = 127,
+	[0][0][2][0][RTW89_KCC][62] = 26,
+	[0][0][2][0][RTW89_FCC][64] = 22,
+	[0][0][2][0][RTW89_ETSI][64] = 127,
+	[0][0][2][0][RTW89_KCC][64] = 26,
+	[0][0][2][0][RTW89_FCC][66] = 22,
+	[0][0][2][0][RTW89_ETSI][66] = 127,
+	[0][0][2][0][RTW89_KCC][66] = 26,
+	[0][0][2][0][RTW89_FCC][68] = 22,
+	[0][0][2][0][RTW89_ETSI][68] = 127,
+	[0][0][2][0][RTW89_KCC][68] = 26,
+	[0][0][2][0][RTW89_FCC][70] = 24,
+	[0][0][2][0][RTW89_ETSI][70] = 127,
+	[0][0][2][0][RTW89_KCC][70] = 26,
+	[0][0][2][0][RTW89_FCC][72] = 22,
+	[0][0][2][0][RTW89_ETSI][72] = 127,
+	[0][0][2][0][RTW89_KCC][72] = 26,
+	[0][0][2][0][RTW89_FCC][74] = 22,
+	[0][0][2][0][RTW89_ETSI][74] = 127,
+	[0][0][2][0][RTW89_KCC][74] = 26,
+	[0][0][2][0][RTW89_FCC][75] = 22,
+	[0][0][2][0][RTW89_ETSI][75] = 127,
+	[0][0][2][0][RTW89_KCC][75] = 26,
+	[0][0][2][0][RTW89_FCC][77] = 22,
+	[0][0][2][0][RTW89_ETSI][77] = 127,
+	[0][0][2][0][RTW89_KCC][77] = 26,
+	[0][0][2][0][RTW89_FCC][79] = 22,
+	[0][0][2][0][RTW89_ETSI][79] = 127,
+	[0][0][2][0][RTW89_KCC][79] = 26,
+	[0][0][2][0][RTW89_FCC][81] = 22,
+	[0][0][2][0][RTW89_ETSI][81] = 127,
+	[0][0][2][0][RTW89_KCC][81] = 26,
+	[0][0][2][0][RTW89_FCC][83] = 22,
+	[0][0][2][0][RTW89_ETSI][83] = 127,
+	[0][0][2][0][RTW89_KCC][83] = 32,
+	[0][0][2][0][RTW89_FCC][85] = 22,
+	[0][0][2][0][RTW89_ETSI][85] = 127,
+	[0][0][2][0][RTW89_KCC][85] = 32,
+	[0][0][2][0][RTW89_FCC][87] = 22,
+	[0][0][2][0][RTW89_ETSI][87] = 127,
+	[0][0][2][0][RTW89_KCC][87] = 32,
+	[0][0][2][0][RTW89_FCC][89] = 22,
+	[0][0][2][0][RTW89_ETSI][89] = 127,
+	[0][0][2][0][RTW89_KCC][89] = 32,
+	[0][0][2][0][RTW89_FCC][90] = 22,
+	[0][0][2][0][RTW89_ETSI][90] = 127,
+	[0][0][2][0][RTW89_KCC][90] = 32,
+	[0][0][2][0][RTW89_FCC][92] = 22,
+	[0][0][2][0][RTW89_ETSI][92] = 127,
+	[0][0][2][0][RTW89_KCC][92] = 32,
+	[0][0][2][0][RTW89_FCC][94] = 22,
+	[0][0][2][0][RTW89_ETSI][94] = 127,
+	[0][0][2][0][RTW89_KCC][94] = 32,
+	[0][0][2][0][RTW89_FCC][96] = 22,
+	[0][0][2][0][RTW89_ETSI][96] = 127,
+	[0][0][2][0][RTW89_KCC][96] = 32,
+	[0][0][2][0][RTW89_FCC][98] = 22,
+	[0][0][2][0][RTW89_ETSI][98] = 127,
+	[0][0][2][0][RTW89_KCC][98] = 32,
+	[0][0][2][0][RTW89_FCC][100] = 22,
+	[0][0][2][0][RTW89_ETSI][100] = 127,
+	[0][0][2][0][RTW89_KCC][100] = 32,
+	[0][0][2][0][RTW89_FCC][102] = 22,
+	[0][0][2][0][RTW89_ETSI][102] = 127,
+	[0][0][2][0][RTW89_KCC][102] = 32,
+	[0][0][2][0][RTW89_FCC][104] = 22,
+	[0][0][2][0][RTW89_ETSI][104] = 127,
+	[0][0][2][0][RTW89_KCC][104] = 32,
+	[0][0][2][0][RTW89_FCC][105] = 22,
+	[0][0][2][0][RTW89_ETSI][105] = 127,
+	[0][0][2][0][RTW89_KCC][105] = 32,
+	[0][0][2][0][RTW89_FCC][107] = 24,
+	[0][0][2][0][RTW89_ETSI][107] = 127,
+	[0][0][2][0][RTW89_KCC][107] = 32,
+	[0][0][2][0][RTW89_FCC][109] = 24,
+	[0][0][2][0][RTW89_ETSI][109] = 127,
+	[0][0][2][0][RTW89_KCC][109] = 32,
 	[0][0][2][0][RTW89_FCC][111] = 127,
+	[0][0][2][0][RTW89_ETSI][111] = 127,
+	[0][0][2][0][RTW89_KCC][111] = 127,
 	[0][0][2][0][RTW89_FCC][113] = 127,
+	[0][0][2][0][RTW89_ETSI][113] = 127,
+	[0][0][2][0][RTW89_KCC][113] = 127,
 	[0][0][2][0][RTW89_FCC][115] = 127,
+	[0][0][2][0][RTW89_ETSI][115] = 127,
+	[0][0][2][0][RTW89_KCC][115] = 127,
 	[0][0][2][0][RTW89_FCC][117] = 127,
+	[0][0][2][0][RTW89_ETSI][117] = 127,
+	[0][0][2][0][RTW89_KCC][117] = 127,
 	[0][0][2][0][RTW89_FCC][119] = 127,
-	[0][1][2][0][RTW89_FCC][0] = 60,
-	[0][1][2][0][RTW89_FCC][2] = 60,
-	[0][1][2][0][RTW89_FCC][4] = 60,
-	[0][1][2][0][RTW89_FCC][6] = 60,
-	[0][1][2][0][RTW89_FCC][8] = 60,
-	[0][1][2][0][RTW89_FCC][10] = 60,
-	[0][1][2][0][RTW89_FCC][12] = 60,
-	[0][1][2][0][RTW89_FCC][14] = 60,
-	[0][1][2][0][RTW89_FCC][15] = 60,
-	[0][1][2][0][RTW89_FCC][17] = 60,
-	[0][1][2][0][RTW89_FCC][19] = 60,
-	[0][1][2][0][RTW89_FCC][21] = 60,
-	[0][1][2][0][RTW89_FCC][23] = 60,
-	[0][1][2][0][RTW89_FCC][25] = 60,
-	[0][1][2][0][RTW89_FCC][27] = 60,
-	[0][1][2][0][RTW89_FCC][29] = 60,
-	[0][1][2][0][RTW89_FCC][30] = 60,
-	[0][1][2][0][RTW89_FCC][32] = 60,
-	[0][1][2][0][RTW89_FCC][34] = 60,
-	[0][1][2][0][RTW89_FCC][36] = 60,
-	[0][1][2][0][RTW89_FCC][38] = 60,
-	[0][1][2][0][RTW89_FCC][40] = 60,
-	[0][1][2][0][RTW89_FCC][42] = 60,
-	[0][1][2][0][RTW89_FCC][44] = 60,
-	[0][1][2][0][RTW89_FCC][45] = 60,
-	[0][1][2][0][RTW89_FCC][47] = 60,
-	[0][1][2][0][RTW89_FCC][49] = 60,
-	[0][1][2][0][RTW89_FCC][51] = 60,
-	[0][1][2][0][RTW89_FCC][53] = 60,
-	[0][1][2][0][RTW89_FCC][55] = 60,
-	[0][1][2][0][RTW89_FCC][57] = 60,
-	[0][1][2][0][RTW89_FCC][59] = 60,
-	[0][1][2][0][RTW89_FCC][60] = 60,
-	[0][1][2][0][RTW89_FCC][62] = 60,
-	[0][1][2][0][RTW89_FCC][64] = 60,
-	[0][1][2][0][RTW89_FCC][66] = 60,
-	[0][1][2][0][RTW89_FCC][68] = 60,
-	[0][1][2][0][RTW89_FCC][70] = 60,
-	[0][1][2][0][RTW89_FCC][72] = 60,
-	[0][1][2][0][RTW89_FCC][74] = 60,
-	[0][1][2][0][RTW89_FCC][75] = 60,
-	[0][1][2][0][RTW89_FCC][77] = 60,
-	[0][1][2][0][RTW89_FCC][79] = 60,
-	[0][1][2][0][RTW89_FCC][81] = 60,
-	[0][1][2][0][RTW89_FCC][83] = 60,
-	[0][1][2][0][RTW89_FCC][85] = 60,
-	[0][1][2][0][RTW89_FCC][87] = 60,
-	[0][1][2][0][RTW89_FCC][89] = 60,
-	[0][1][2][0][RTW89_FCC][90] = 60,
-	[0][1][2][0][RTW89_FCC][92] = 60,
-	[0][1][2][0][RTW89_FCC][94] = 60,
-	[0][1][2][0][RTW89_FCC][96] = 60,
-	[0][1][2][0][RTW89_FCC][98] = 60,
-	[0][1][2][0][RTW89_FCC][100] = 60,
-	[0][1][2][0][RTW89_FCC][102] = 60,
-	[0][1][2][0][RTW89_FCC][104] = 60,
-	[0][1][2][0][RTW89_FCC][105] = 60,
-	[0][1][2][0][RTW89_FCC][107] = 60,
-	[0][1][2][0][RTW89_FCC][109] = 60,
+	[0][0][2][0][RTW89_ETSI][119] = 127,
+	[0][0][2][0][RTW89_KCC][119] = 127,
+	[0][1][2][0][RTW89_FCC][0] = -2,
+	[0][1][2][0][RTW89_ETSI][0] = 54,
+	[0][1][2][0][RTW89_KCC][0] = 12,
+	[0][1][2][0][RTW89_FCC][2] = -4,
+	[0][1][2][0][RTW89_ETSI][2] = 54,
+	[0][1][2][0][RTW89_KCC][2] = 12,
+	[0][1][2][0][RTW89_FCC][4] = -4,
+	[0][1][2][0][RTW89_ETSI][4] = 54,
+	[0][1][2][0][RTW89_KCC][4] = 12,
+	[0][1][2][0][RTW89_FCC][6] = -4,
+	[0][1][2][0][RTW89_ETSI][6] = 54,
+	[0][1][2][0][RTW89_KCC][6] = 12,
+	[0][1][2][0][RTW89_FCC][8] = -4,
+	[0][1][2][0][RTW89_ETSI][8] = 54,
+	[0][1][2][0][RTW89_KCC][8] = 12,
+	[0][1][2][0][RTW89_FCC][10] = -4,
+	[0][1][2][0][RTW89_ETSI][10] = 54,
+	[0][1][2][0][RTW89_KCC][10] = 12,
+	[0][1][2][0][RTW89_FCC][12] = -4,
+	[0][1][2][0][RTW89_ETSI][12] = 54,
+	[0][1][2][0][RTW89_KCC][12] = 12,
+	[0][1][2][0][RTW89_FCC][14] = -4,
+	[0][1][2][0][RTW89_ETSI][14] = 54,
+	[0][1][2][0][RTW89_KCC][14] = 12,
+	[0][1][2][0][RTW89_FCC][15] = -4,
+	[0][1][2][0][RTW89_ETSI][15] = 54,
+	[0][1][2][0][RTW89_KCC][15] = 12,
+	[0][1][2][0][RTW89_FCC][17] = -4,
+	[0][1][2][0][RTW89_ETSI][17] = 54,
+	[0][1][2][0][RTW89_KCC][17] = 12,
+	[0][1][2][0][RTW89_FCC][19] = -4,
+	[0][1][2][0][RTW89_ETSI][19] = 54,
+	[0][1][2][0][RTW89_KCC][19] = 12,
+	[0][1][2][0][RTW89_FCC][21] = -4,
+	[0][1][2][0][RTW89_ETSI][21] = 54,
+	[0][1][2][0][RTW89_KCC][21] = 12,
+	[0][1][2][0][RTW89_FCC][23] = -4,
+	[0][1][2][0][RTW89_ETSI][23] = 54,
+	[0][1][2][0][RTW89_KCC][23] = 12,
+	[0][1][2][0][RTW89_FCC][25] = -4,
+	[0][1][2][0][RTW89_ETSI][25] = 54,
+	[0][1][2][0][RTW89_KCC][25] = 12,
+	[0][1][2][0][RTW89_FCC][27] = -4,
+	[0][1][2][0][RTW89_ETSI][27] = 54,
+	[0][1][2][0][RTW89_KCC][27] = 12,
+	[0][1][2][0][RTW89_FCC][29] = -4,
+	[0][1][2][0][RTW89_ETSI][29] = 54,
+	[0][1][2][0][RTW89_KCC][29] = 12,
+	[0][1][2][0][RTW89_FCC][30] = -4,
+	[0][1][2][0][RTW89_ETSI][30] = 54,
+	[0][1][2][0][RTW89_KCC][30] = 12,
+	[0][1][2][0][RTW89_FCC][32] = -4,
+	[0][1][2][0][RTW89_ETSI][32] = 54,
+	[0][1][2][0][RTW89_KCC][32] = 12,
+	[0][1][2][0][RTW89_FCC][34] = -4,
+	[0][1][2][0][RTW89_ETSI][34] = 54,
+	[0][1][2][0][RTW89_KCC][34] = 12,
+	[0][1][2][0][RTW89_FCC][36] = -4,
+	[0][1][2][0][RTW89_ETSI][36] = 54,
+	[0][1][2][0][RTW89_KCC][36] = 12,
+	[0][1][2][0][RTW89_FCC][38] = -4,
+	[0][1][2][0][RTW89_ETSI][38] = 54,
+	[0][1][2][0][RTW89_KCC][38] = 12,
+	[0][1][2][0][RTW89_FCC][40] = -4,
+	[0][1][2][0][RTW89_ETSI][40] = 54,
+	[0][1][2][0][RTW89_KCC][40] = 12,
+	[0][1][2][0][RTW89_FCC][42] = -4,
+	[0][1][2][0][RTW89_ETSI][42] = 54,
+	[0][1][2][0][RTW89_KCC][42] = 12,
+	[0][1][2][0][RTW89_FCC][44] = -2,
+	[0][1][2][0][RTW89_ETSI][44] = 54,
+	[0][1][2][0][RTW89_KCC][44] = 12,
+	[0][1][2][0][RTW89_FCC][45] = -2,
+	[0][1][2][0][RTW89_ETSI][45] = 127,
+	[0][1][2][0][RTW89_KCC][45] = 12,
+	[0][1][2][0][RTW89_FCC][47] = -2,
+	[0][1][2][0][RTW89_ETSI][47] = 127,
+	[0][1][2][0][RTW89_KCC][47] = 12,
+	[0][1][2][0][RTW89_FCC][49] = -2,
+	[0][1][2][0][RTW89_ETSI][49] = 127,
+	[0][1][2][0][RTW89_KCC][49] = 12,
+	[0][1][2][0][RTW89_FCC][51] = -2,
+	[0][1][2][0][RTW89_ETSI][51] = 127,
+	[0][1][2][0][RTW89_KCC][51] = 12,
+	[0][1][2][0][RTW89_FCC][53] = -2,
+	[0][1][2][0][RTW89_ETSI][53] = 127,
+	[0][1][2][0][RTW89_KCC][53] = 12,
+	[0][1][2][0][RTW89_FCC][55] = -2,
+	[0][1][2][0][RTW89_ETSI][55] = 127,
+	[0][1][2][0][RTW89_KCC][55] = 12,
+	[0][1][2][0][RTW89_FCC][57] = -2,
+	[0][1][2][0][RTW89_ETSI][57] = 127,
+	[0][1][2][0][RTW89_KCC][57] = 12,
+	[0][1][2][0][RTW89_FCC][59] = -2,
+	[0][1][2][0][RTW89_ETSI][59] = 127,
+	[0][1][2][0][RTW89_KCC][59] = 12,
+	[0][1][2][0][RTW89_FCC][60] = -2,
+	[0][1][2][0][RTW89_ETSI][60] = 127,
+	[0][1][2][0][RTW89_KCC][60] = 12,
+	[0][1][2][0][RTW89_FCC][62] = -2,
+	[0][1][2][0][RTW89_ETSI][62] = 127,
+	[0][1][2][0][RTW89_KCC][62] = 12,
+	[0][1][2][0][RTW89_FCC][64] = -2,
+	[0][1][2][0][RTW89_ETSI][64] = 127,
+	[0][1][2][0][RTW89_KCC][64] = 12,
+	[0][1][2][0][RTW89_FCC][66] = -2,
+	[0][1][2][0][RTW89_ETSI][66] = 127,
+	[0][1][2][0][RTW89_KCC][66] = 12,
+	[0][1][2][0][RTW89_FCC][68] = -2,
+	[0][1][2][0][RTW89_ETSI][68] = 127,
+	[0][1][2][0][RTW89_KCC][68] = 12,
+	[0][1][2][0][RTW89_FCC][70] = -2,
+	[0][1][2][0][RTW89_ETSI][70] = 127,
+	[0][1][2][0][RTW89_KCC][70] = 12,
+	[0][1][2][0][RTW89_FCC][72] = -2,
+	[0][1][2][0][RTW89_ETSI][72] = 127,
+	[0][1][2][0][RTW89_KCC][72] = 12,
+	[0][1][2][0][RTW89_FCC][74] = -2,
+	[0][1][2][0][RTW89_ETSI][74] = 127,
+	[0][1][2][0][RTW89_KCC][74] = 12,
+	[0][1][2][0][RTW89_FCC][75] = -2,
+	[0][1][2][0][RTW89_ETSI][75] = 127,
+	[0][1][2][0][RTW89_KCC][75] = 12,
+	[0][1][2][0][RTW89_FCC][77] = -2,
+	[0][1][2][0][RTW89_ETSI][77] = 127,
+	[0][1][2][0][RTW89_KCC][77] = 12,
+	[0][1][2][0][RTW89_FCC][79] = -2,
+	[0][1][2][0][RTW89_ETSI][79] = 127,
+	[0][1][2][0][RTW89_KCC][79] = 12,
+	[0][1][2][0][RTW89_FCC][81] = -2,
+	[0][1][2][0][RTW89_ETSI][81] = 127,
+	[0][1][2][0][RTW89_KCC][81] = 12,
+	[0][1][2][0][RTW89_FCC][83] = -2,
+	[0][1][2][0][RTW89_ETSI][83] = 127,
+	[0][1][2][0][RTW89_KCC][83] = 20,
+	[0][1][2][0][RTW89_FCC][85] = -2,
+	[0][1][2][0][RTW89_ETSI][85] = 127,
+	[0][1][2][0][RTW89_KCC][85] = 20,
+	[0][1][2][0][RTW89_FCC][87] = -2,
+	[0][1][2][0][RTW89_ETSI][87] = 127,
+	[0][1][2][0][RTW89_KCC][87] = 20,
+	[0][1][2][0][RTW89_FCC][89] = -2,
+	[0][1][2][0][RTW89_ETSI][89] = 127,
+	[0][1][2][0][RTW89_KCC][89] = 20,
+	[0][1][2][0][RTW89_FCC][90] = -2,
+	[0][1][2][0][RTW89_ETSI][90] = 127,
+	[0][1][2][0][RTW89_KCC][90] = 20,
+	[0][1][2][0][RTW89_FCC][92] = -2,
+	[0][1][2][0][RTW89_ETSI][92] = 127,
+	[0][1][2][0][RTW89_KCC][92] = 20,
+	[0][1][2][0][RTW89_FCC][94] = -2,
+	[0][1][2][0][RTW89_ETSI][94] = 127,
+	[0][1][2][0][RTW89_KCC][94] = 20,
+	[0][1][2][0][RTW89_FCC][96] = -2,
+	[0][1][2][0][RTW89_ETSI][96] = 127,
+	[0][1][2][0][RTW89_KCC][96] = 20,
+	[0][1][2][0][RTW89_FCC][98] = -2,
+	[0][1][2][0][RTW89_ETSI][98] = 127,
+	[0][1][2][0][RTW89_KCC][98] = 20,
+	[0][1][2][0][RTW89_FCC][100] = -2,
+	[0][1][2][0][RTW89_ETSI][100] = 127,
+	[0][1][2][0][RTW89_KCC][100] = 20,
+	[0][1][2][0][RTW89_FCC][102] = -2,
+	[0][1][2][0][RTW89_ETSI][102] = 127,
+	[0][1][2][0][RTW89_KCC][102] = 20,
+	[0][1][2][0][RTW89_FCC][104] = -2,
+	[0][1][2][0][RTW89_ETSI][104] = 127,
+	[0][1][2][0][RTW89_KCC][104] = 20,
+	[0][1][2][0][RTW89_FCC][105] = -2,
+	[0][1][2][0][RTW89_ETSI][105] = 127,
+	[0][1][2][0][RTW89_KCC][105] = 20,
+	[0][1][2][0][RTW89_FCC][107] = 0,
+	[0][1][2][0][RTW89_ETSI][107] = 127,
+	[0][1][2][0][RTW89_KCC][107] = 20,
+	[0][1][2][0][RTW89_FCC][109] = 0,
+	[0][1][2][0][RTW89_ETSI][109] = 127,
+	[0][1][2][0][RTW89_KCC][109] = 20,
 	[0][1][2][0][RTW89_FCC][111] = 127,
+	[0][1][2][0][RTW89_ETSI][111] = 127,
+	[0][1][2][0][RTW89_KCC][111] = 127,
 	[0][1][2][0][RTW89_FCC][113] = 127,
+	[0][1][2][0][RTW89_ETSI][113] = 127,
+	[0][1][2][0][RTW89_KCC][113] = 127,
 	[0][1][2][0][RTW89_FCC][115] = 127,
+	[0][1][2][0][RTW89_ETSI][115] = 127,
+	[0][1][2][0][RTW89_KCC][115] = 127,
 	[0][1][2][0][RTW89_FCC][117] = 127,
+	[0][1][2][0][RTW89_ETSI][117] = 127,
+	[0][1][2][0][RTW89_KCC][117] = 127,
 	[0][1][2][0][RTW89_FCC][119] = 127,
-	[0][1][2][1][RTW89_FCC][0] = 48,
-	[0][1][2][1][RTW89_FCC][2] = 48,
-	[0][1][2][1][RTW89_FCC][4] = 48,
-	[0][1][2][1][RTW89_FCC][6] = 48,
-	[0][1][2][1][RTW89_FCC][8] = 48,
-	[0][1][2][1][RTW89_FCC][10] = 48,
-	[0][1][2][1][RTW89_FCC][12] = 48,
-	[0][1][2][1][RTW89_FCC][14] = 48,
-	[0][1][2][1][RTW89_FCC][15] = 48,
-	[0][1][2][1][RTW89_FCC][17] = 48,
-	[0][1][2][1][RTW89_FCC][19] = 48,
-	[0][1][2][1][RTW89_FCC][21] = 48,
-	[0][1][2][1][RTW89_FCC][23] = 48,
-	[0][1][2][1][RTW89_FCC][25] = 48,
-	[0][1][2][1][RTW89_FCC][27] = 48,
-	[0][1][2][1][RTW89_FCC][29] = 48,
-	[0][1][2][1][RTW89_FCC][30] = 48,
-	[0][1][2][1][RTW89_FCC][32] = 48,
-	[0][1][2][1][RTW89_FCC][34] = 48,
-	[0][1][2][1][RTW89_FCC][36] = 48,
-	[0][1][2][1][RTW89_FCC][38] = 48,
-	[0][1][2][1][RTW89_FCC][40] = 48,
-	[0][1][2][1][RTW89_FCC][42] = 48,
-	[0][1][2][1][RTW89_FCC][44] = 48,
-	[0][1][2][1][RTW89_FCC][45] = 48,
-	[0][1][2][1][RTW89_FCC][47] = 48,
-	[0][1][2][1][RTW89_FCC][49] = 48,
-	[0][1][2][1][RTW89_FCC][51] = 48,
-	[0][1][2][1][RTW89_FCC][53] = 48,
-	[0][1][2][1][RTW89_FCC][55] = 48,
-	[0][1][2][1][RTW89_FCC][57] = 48,
-	[0][1][2][1][RTW89_FCC][59] = 48,
-	[0][1][2][1][RTW89_FCC][60] = 48,
-	[0][1][2][1][RTW89_FCC][62] = 48,
-	[0][1][2][1][RTW89_FCC][64] = 48,
-	[0][1][2][1][RTW89_FCC][66] = 48,
-	[0][1][2][1][RTW89_FCC][68] = 48,
-	[0][1][2][1][RTW89_FCC][70] = 48,
-	[0][1][2][1][RTW89_FCC][72] = 48,
-	[0][1][2][1][RTW89_FCC][74] = 48,
-	[0][1][2][1][RTW89_FCC][75] = 48,
-	[0][1][2][1][RTW89_FCC][77] = 48,
-	[0][1][2][1][RTW89_FCC][79] = 48,
-	[0][1][2][1][RTW89_FCC][81] = 48,
-	[0][1][2][1][RTW89_FCC][83] = 48,
-	[0][1][2][1][RTW89_FCC][85] = 48,
-	[0][1][2][1][RTW89_FCC][87] = 48,
-	[0][1][2][1][RTW89_FCC][89] = 48,
-	[0][1][2][1][RTW89_FCC][90] = 48,
-	[0][1][2][1][RTW89_FCC][92] = 48,
-	[0][1][2][1][RTW89_FCC][94] = 48,
-	[0][1][2][1][RTW89_FCC][96] = 48,
-	[0][1][2][1][RTW89_FCC][98] = 48,
-	[0][1][2][1][RTW89_FCC][100] = 48,
-	[0][1][2][1][RTW89_FCC][102] = 48,
-	[0][1][2][1][RTW89_FCC][104] = 48,
-	[0][1][2][1][RTW89_FCC][105] = 48,
-	[0][1][2][1][RTW89_FCC][107] = 48,
-	[0][1][2][1][RTW89_FCC][109] = 48,
+	[0][1][2][0][RTW89_ETSI][119] = 127,
+	[0][1][2][0][RTW89_KCC][119] = 127,
+	[0][1][2][1][RTW89_FCC][0] = -2,
+	[0][1][2][1][RTW89_ETSI][0] = 42,
+	[0][1][2][1][RTW89_KCC][0] = 12,
+	[0][1][2][1][RTW89_FCC][2] = -4,
+	[0][1][2][1][RTW89_ETSI][2] = 42,
+	[0][1][2][1][RTW89_KCC][2] = 12,
+	[0][1][2][1][RTW89_FCC][4] = -4,
+	[0][1][2][1][RTW89_ETSI][4] = 42,
+	[0][1][2][1][RTW89_KCC][4] = 12,
+	[0][1][2][1][RTW89_FCC][6] = -4,
+	[0][1][2][1][RTW89_ETSI][6] = 42,
+	[0][1][2][1][RTW89_KCC][6] = 12,
+	[0][1][2][1][RTW89_FCC][8] = -4,
+	[0][1][2][1][RTW89_ETSI][8] = 42,
+	[0][1][2][1][RTW89_KCC][8] = 12,
+	[0][1][2][1][RTW89_FCC][10] = -4,
+	[0][1][2][1][RTW89_ETSI][10] = 42,
+	[0][1][2][1][RTW89_KCC][10] = 12,
+	[0][1][2][1][RTW89_FCC][12] = -4,
+	[0][1][2][1][RTW89_ETSI][12] = 42,
+	[0][1][2][1][RTW89_KCC][12] = 12,
+	[0][1][2][1][RTW89_FCC][14] = -4,
+	[0][1][2][1][RTW89_ETSI][14] = 42,
+	[0][1][2][1][RTW89_KCC][14] = 12,
+	[0][1][2][1][RTW89_FCC][15] = -4,
+	[0][1][2][1][RTW89_ETSI][15] = 42,
+	[0][1][2][1][RTW89_KCC][15] = 12,
+	[0][1][2][1][RTW89_FCC][17] = -4,
+	[0][1][2][1][RTW89_ETSI][17] = 42,
+	[0][1][2][1][RTW89_KCC][17] = 12,
+	[0][1][2][1][RTW89_FCC][19] = -4,
+	[0][1][2][1][RTW89_ETSI][19] = 42,
+	[0][1][2][1][RTW89_KCC][19] = 12,
+	[0][1][2][1][RTW89_FCC][21] = -4,
+	[0][1][2][1][RTW89_ETSI][21] = 42,
+	[0][1][2][1][RTW89_KCC][21] = 12,
+	[0][1][2][1][RTW89_FCC][23] = -4,
+	[0][1][2][1][RTW89_ETSI][23] = 42,
+	[0][1][2][1][RTW89_KCC][23] = 12,
+	[0][1][2][1][RTW89_FCC][25] = -4,
+	[0][1][2][1][RTW89_ETSI][25] = 42,
+	[0][1][2][1][RTW89_KCC][25] = 12,
+	[0][1][2][1][RTW89_FCC][27] = -4,
+	[0][1][2][1][RTW89_ETSI][27] = 42,
+	[0][1][2][1][RTW89_KCC][27] = 12,
+	[0][1][2][1][RTW89_FCC][29] = -4,
+	[0][1][2][1][RTW89_ETSI][29] = 42,
+	[0][1][2][1][RTW89_KCC][29] = 12,
+	[0][1][2][1][RTW89_FCC][30] = -4,
+	[0][1][2][1][RTW89_ETSI][30] = 42,
+	[0][1][2][1][RTW89_KCC][30] = 12,
+	[0][1][2][1][RTW89_FCC][32] = -4,
+	[0][1][2][1][RTW89_ETSI][32] = 42,
+	[0][1][2][1][RTW89_KCC][32] = 12,
+	[0][1][2][1][RTW89_FCC][34] = -4,
+	[0][1][2][1][RTW89_ETSI][34] = 42,
+	[0][1][2][1][RTW89_KCC][34] = 12,
+	[0][1][2][1][RTW89_FCC][36] = -4,
+	[0][1][2][1][RTW89_ETSI][36] = 42,
+	[0][1][2][1][RTW89_KCC][36] = 12,
+	[0][1][2][1][RTW89_FCC][38] = -4,
+	[0][1][2][1][RTW89_ETSI][38] = 42,
+	[0][1][2][1][RTW89_KCC][38] = 12,
+	[0][1][2][1][RTW89_FCC][40] = -4,
+	[0][1][2][1][RTW89_ETSI][40] = 42,
+	[0][1][2][1][RTW89_KCC][40] = 12,
+	[0][1][2][1][RTW89_FCC][42] = -4,
+	[0][1][2][1][RTW89_ETSI][42] = 42,
+	[0][1][2][1][RTW89_KCC][42] = 12,
+	[0][1][2][1][RTW89_FCC][44] = -2,
+	[0][1][2][1][RTW89_ETSI][44] = 42,
+	[0][1][2][1][RTW89_KCC][44] = 12,
+	[0][1][2][1][RTW89_FCC][45] = -2,
+	[0][1][2][1][RTW89_ETSI][45] = 127,
+	[0][1][2][1][RTW89_KCC][45] = 12,
+	[0][1][2][1][RTW89_FCC][47] = -2,
+	[0][1][2][1][RTW89_ETSI][47] = 127,
+	[0][1][2][1][RTW89_KCC][47] = 12,
+	[0][1][2][1][RTW89_FCC][49] = -2,
+	[0][1][2][1][RTW89_ETSI][49] = 127,
+	[0][1][2][1][RTW89_KCC][49] = 12,
+	[0][1][2][1][RTW89_FCC][51] = -2,
+	[0][1][2][1][RTW89_ETSI][51] = 127,
+	[0][1][2][1][RTW89_KCC][51] = 12,
+	[0][1][2][1][RTW89_FCC][53] = -2,
+	[0][1][2][1][RTW89_ETSI][53] = 127,
+	[0][1][2][1][RTW89_KCC][53] = 12,
+	[0][1][2][1][RTW89_FCC][55] = -2,
+	[0][1][2][1][RTW89_ETSI][55] = 127,
+	[0][1][2][1][RTW89_KCC][55] = 12,
+	[0][1][2][1][RTW89_FCC][57] = -2,
+	[0][1][2][1][RTW89_ETSI][57] = 127,
+	[0][1][2][1][RTW89_KCC][57] = 12,
+	[0][1][2][1][RTW89_FCC][59] = -2,
+	[0][1][2][1][RTW89_ETSI][59] = 127,
+	[0][1][2][1][RTW89_KCC][59] = 12,
+	[0][1][2][1][RTW89_FCC][60] = -2,
+	[0][1][2][1][RTW89_ETSI][60] = 127,
+	[0][1][2][1][RTW89_KCC][60] = 12,
+	[0][1][2][1][RTW89_FCC][62] = -2,
+	[0][1][2][1][RTW89_ETSI][62] = 127,
+	[0][1][2][1][RTW89_KCC][62] = 12,
+	[0][1][2][1][RTW89_FCC][64] = -2,
+	[0][1][2][1][RTW89_ETSI][64] = 127,
+	[0][1][2][1][RTW89_KCC][64] = 12,
+	[0][1][2][1][RTW89_FCC][66] = -2,
+	[0][1][2][1][RTW89_ETSI][66] = 127,
+	[0][1][2][1][RTW89_KCC][66] = 12,
+	[0][1][2][1][RTW89_FCC][68] = -2,
+	[0][1][2][1][RTW89_ETSI][68] = 127,
+	[0][1][2][1][RTW89_KCC][68] = 12,
+	[0][1][2][1][RTW89_FCC][70] = -2,
+	[0][1][2][1][RTW89_ETSI][70] = 127,
+	[0][1][2][1][RTW89_KCC][70] = 12,
+	[0][1][2][1][RTW89_FCC][72] = -2,
+	[0][1][2][1][RTW89_ETSI][72] = 127,
+	[0][1][2][1][RTW89_KCC][72] = 12,
+	[0][1][2][1][RTW89_FCC][74] = -2,
+	[0][1][2][1][RTW89_ETSI][74] = 127,
+	[0][1][2][1][RTW89_KCC][74] = 12,
+	[0][1][2][1][RTW89_FCC][75] = -2,
+	[0][1][2][1][RTW89_ETSI][75] = 127,
+	[0][1][2][1][RTW89_KCC][75] = 12,
+	[0][1][2][1][RTW89_FCC][77] = -2,
+	[0][1][2][1][RTW89_ETSI][77] = 127,
+	[0][1][2][1][RTW89_KCC][77] = 12,
+	[0][1][2][1][RTW89_FCC][79] = -2,
+	[0][1][2][1][RTW89_ETSI][79] = 127,
+	[0][1][2][1][RTW89_KCC][79] = 12,
+	[0][1][2][1][RTW89_FCC][81] = -2,
+	[0][1][2][1][RTW89_ETSI][81] = 127,
+	[0][1][2][1][RTW89_KCC][81] = 12,
+	[0][1][2][1][RTW89_FCC][83] = -2,
+	[0][1][2][1][RTW89_ETSI][83] = 127,
+	[0][1][2][1][RTW89_KCC][83] = 20,
+	[0][1][2][1][RTW89_FCC][85] = -2,
+	[0][1][2][1][RTW89_ETSI][85] = 127,
+	[0][1][2][1][RTW89_KCC][85] = 20,
+	[0][1][2][1][RTW89_FCC][87] = -2,
+	[0][1][2][1][RTW89_ETSI][87] = 127,
+	[0][1][2][1][RTW89_KCC][87] = 20,
+	[0][1][2][1][RTW89_FCC][89] = -2,
+	[0][1][2][1][RTW89_ETSI][89] = 127,
+	[0][1][2][1][RTW89_KCC][89] = 20,
+	[0][1][2][1][RTW89_FCC][90] = -2,
+	[0][1][2][1][RTW89_ETSI][90] = 127,
+	[0][1][2][1][RTW89_KCC][90] = 20,
+	[0][1][2][1][RTW89_FCC][92] = -2,
+	[0][1][2][1][RTW89_ETSI][92] = 127,
+	[0][1][2][1][RTW89_KCC][92] = 20,
+	[0][1][2][1][RTW89_FCC][94] = -2,
+	[0][1][2][1][RTW89_ETSI][94] = 127,
+	[0][1][2][1][RTW89_KCC][94] = 20,
+	[0][1][2][1][RTW89_FCC][96] = -2,
+	[0][1][2][1][RTW89_ETSI][96] = 127,
+	[0][1][2][1][RTW89_KCC][96] = 20,
+	[0][1][2][1][RTW89_FCC][98] = -2,
+	[0][1][2][1][RTW89_ETSI][98] = 127,
+	[0][1][2][1][RTW89_KCC][98] = 20,
+	[0][1][2][1][RTW89_FCC][100] = -2,
+	[0][1][2][1][RTW89_ETSI][100] = 127,
+	[0][1][2][1][RTW89_KCC][100] = 20,
+	[0][1][2][1][RTW89_FCC][102] = -2,
+	[0][1][2][1][RTW89_ETSI][102] = 127,
+	[0][1][2][1][RTW89_KCC][102] = 20,
+	[0][1][2][1][RTW89_FCC][104] = -2,
+	[0][1][2][1][RTW89_ETSI][104] = 127,
+	[0][1][2][1][RTW89_KCC][104] = 20,
+	[0][1][2][1][RTW89_FCC][105] = -2,
+	[0][1][2][1][RTW89_ETSI][105] = 127,
+	[0][1][2][1][RTW89_KCC][105] = 20,
+	[0][1][2][1][RTW89_FCC][107] = 0,
+	[0][1][2][1][RTW89_ETSI][107] = 127,
+	[0][1][2][1][RTW89_KCC][107] = 20,
+	[0][1][2][1][RTW89_FCC][109] = 0,
+	[0][1][2][1][RTW89_ETSI][109] = 127,
+	[0][1][2][1][RTW89_KCC][109] = 20,
 	[0][1][2][1][RTW89_FCC][111] = 127,
+	[0][1][2][1][RTW89_ETSI][111] = 127,
+	[0][1][2][1][RTW89_KCC][111] = 127,
 	[0][1][2][1][RTW89_FCC][113] = 127,
+	[0][1][2][1][RTW89_ETSI][113] = 127,
+	[0][1][2][1][RTW89_KCC][113] = 127,
 	[0][1][2][1][RTW89_FCC][115] = 127,
+	[0][1][2][1][RTW89_ETSI][115] = 127,
+	[0][1][2][1][RTW89_KCC][115] = 127,
 	[0][1][2][1][RTW89_FCC][117] = 127,
+	[0][1][2][1][RTW89_ETSI][117] = 127,
+	[0][1][2][1][RTW89_KCC][117] = 127,
 	[0][1][2][1][RTW89_FCC][119] = 127,
-	[1][0][2][0][RTW89_FCC][1] = 72,
-	[1][0][2][0][RTW89_FCC][5] = 72,
-	[1][0][2][0][RTW89_FCC][9] = 72,
-	[1][0][2][0][RTW89_FCC][13] = 72,
-	[1][0][2][0][RTW89_FCC][16] = 72,
-	[1][0][2][0][RTW89_FCC][20] = 72,
-	[1][0][2][0][RTW89_FCC][24] = 72,
-	[1][0][2][0][RTW89_FCC][28] = 72,
-	[1][0][2][0][RTW89_FCC][31] = 72,
-	[1][0][2][0][RTW89_FCC][35] = 72,
-	[1][0][2][0][RTW89_FCC][39] = 72,
-	[1][0][2][0][RTW89_FCC][43] = 72,
-	[1][0][2][0][RTW89_FCC][46] = 72,
-	[1][0][2][0][RTW89_FCC][50] = 72,
-	[1][0][2][0][RTW89_FCC][54] = 72,
-	[1][0][2][0][RTW89_FCC][58] = 72,
-	[1][0][2][0][RTW89_FCC][61] = 72,
-	[1][0][2][0][RTW89_FCC][65] = 72,
-	[1][0][2][0][RTW89_FCC][69] = 72,
-	[1][0][2][0][RTW89_FCC][73] = 72,
-	[1][0][2][0][RTW89_FCC][76] = 72,
-	[1][0][2][0][RTW89_FCC][80] = 72,
-	[1][0][2][0][RTW89_FCC][84] = 72,
-	[1][0][2][0][RTW89_FCC][88] = 72,
-	[1][0][2][0][RTW89_FCC][91] = 72,
-	[1][0][2][0][RTW89_FCC][95] = 72,
-	[1][0][2][0][RTW89_FCC][99] = 72,
-	[1][0][2][0][RTW89_FCC][103] = 72,
-	[1][0][2][0][RTW89_FCC][106] = 72,
+	[0][1][2][1][RTW89_ETSI][119] = 127,
+	[0][1][2][1][RTW89_KCC][119] = 127,
+	[1][0][2][0][RTW89_FCC][1] = 34,
+	[1][0][2][0][RTW89_ETSI][1] = 66,
+	[1][0][2][0][RTW89_KCC][1] = 40,
+	[1][0][2][0][RTW89_FCC][5] = 34,
+	[1][0][2][0][RTW89_ETSI][5] = 66,
+	[1][0][2][0][RTW89_KCC][5] = 40,
+	[1][0][2][0][RTW89_FCC][9] = 34,
+	[1][0][2][0][RTW89_ETSI][9] = 66,
+	[1][0][2][0][RTW89_KCC][9] = 40,
+	[1][0][2][0][RTW89_FCC][13] = 34,
+	[1][0][2][0][RTW89_ETSI][13] = 66,
+	[1][0][2][0][RTW89_KCC][13] = 40,
+	[1][0][2][0][RTW89_FCC][16] = 34,
+	[1][0][2][0][RTW89_ETSI][16] = 66,
+	[1][0][2][0][RTW89_KCC][16] = 40,
+	[1][0][2][0][RTW89_FCC][20] = 34,
+	[1][0][2][0][RTW89_ETSI][20] = 66,
+	[1][0][2][0][RTW89_KCC][20] = 40,
+	[1][0][2][0][RTW89_FCC][24] = 36,
+	[1][0][2][0][RTW89_ETSI][24] = 66,
+	[1][0][2][0][RTW89_KCC][24] = 40,
+	[1][0][2][0][RTW89_FCC][28] = 34,
+	[1][0][2][0][RTW89_ETSI][28] = 66,
+	[1][0][2][0][RTW89_KCC][28] = 40,
+	[1][0][2][0][RTW89_FCC][31] = 34,
+	[1][0][2][0][RTW89_ETSI][31] = 66,
+	[1][0][2][0][RTW89_KCC][31] = 40,
+	[1][0][2][0][RTW89_FCC][35] = 34,
+	[1][0][2][0][RTW89_ETSI][35] = 66,
+	[1][0][2][0][RTW89_KCC][35] = 40,
+	[1][0][2][0][RTW89_FCC][39] = 34,
+	[1][0][2][0][RTW89_ETSI][39] = 66,
+	[1][0][2][0][RTW89_KCC][39] = 40,
+	[1][0][2][0][RTW89_FCC][43] = 34,
+	[1][0][2][0][RTW89_ETSI][43] = 66,
+	[1][0][2][0][RTW89_KCC][43] = 40,
+	[1][0][2][0][RTW89_FCC][46] = 34,
+	[1][0][2][0][RTW89_ETSI][46] = 127,
+	[1][0][2][0][RTW89_KCC][46] = 40,
+	[1][0][2][0][RTW89_FCC][50] = 34,
+	[1][0][2][0][RTW89_ETSI][50] = 127,
+	[1][0][2][0][RTW89_KCC][50] = 40,
+	[1][0][2][0][RTW89_FCC][54] = 36,
+	[1][0][2][0][RTW89_ETSI][54] = 127,
+	[1][0][2][0][RTW89_KCC][54] = 40,
+	[1][0][2][0][RTW89_FCC][58] = 36,
+	[1][0][2][0][RTW89_ETSI][58] = 127,
+	[1][0][2][0][RTW89_KCC][58] = 40,
+	[1][0][2][0][RTW89_FCC][61] = 34,
+	[1][0][2][0][RTW89_ETSI][61] = 127,
+	[1][0][2][0][RTW89_KCC][61] = 40,
+	[1][0][2][0][RTW89_FCC][65] = 34,
+	[1][0][2][0][RTW89_ETSI][65] = 127,
+	[1][0][2][0][RTW89_KCC][65] = 40,
+	[1][0][2][0][RTW89_FCC][69] = 34,
+	[1][0][2][0][RTW89_ETSI][69] = 127,
+	[1][0][2][0][RTW89_KCC][69] = 40,
+	[1][0][2][0][RTW89_FCC][73] = 34,
+	[1][0][2][0][RTW89_ETSI][73] = 127,
+	[1][0][2][0][RTW89_KCC][73] = 40,
+	[1][0][2][0][RTW89_FCC][76] = 34,
+	[1][0][2][0][RTW89_ETSI][76] = 127,
+	[1][0][2][0][RTW89_KCC][76] = 40,
+	[1][0][2][0][RTW89_FCC][80] = 34,
+	[1][0][2][0][RTW89_ETSI][80] = 127,
+	[1][0][2][0][RTW89_KCC][80] = 42,
+	[1][0][2][0][RTW89_FCC][84] = 34,
+	[1][0][2][0][RTW89_ETSI][84] = 127,
+	[1][0][2][0][RTW89_KCC][84] = 42,
+	[1][0][2][0][RTW89_FCC][88] = 34,
+	[1][0][2][0][RTW89_ETSI][88] = 127,
+	[1][0][2][0][RTW89_KCC][88] = 42,
+	[1][0][2][0][RTW89_FCC][91] = 36,
+	[1][0][2][0][RTW89_ETSI][91] = 127,
+	[1][0][2][0][RTW89_KCC][91] = 42,
+	[1][0][2][0][RTW89_FCC][95] = 34,
+	[1][0][2][0][RTW89_ETSI][95] = 127,
+	[1][0][2][0][RTW89_KCC][95] = 42,
+	[1][0][2][0][RTW89_FCC][99] = 34,
+	[1][0][2][0][RTW89_ETSI][99] = 127,
+	[1][0][2][0][RTW89_KCC][99] = 42,
+	[1][0][2][0][RTW89_FCC][103] = 34,
+	[1][0][2][0][RTW89_ETSI][103] = 127,
+	[1][0][2][0][RTW89_KCC][103] = 42,
+	[1][0][2][0][RTW89_FCC][106] = 36,
+	[1][0][2][0][RTW89_ETSI][106] = 127,
+	[1][0][2][0][RTW89_KCC][106] = 42,
 	[1][0][2][0][RTW89_FCC][110] = 127,
+	[1][0][2][0][RTW89_ETSI][110] = 127,
+	[1][0][2][0][RTW89_KCC][110] = 127,
 	[1][0][2][0][RTW89_FCC][114] = 127,
+	[1][0][2][0][RTW89_ETSI][114] = 127,
+	[1][0][2][0][RTW89_KCC][114] = 127,
 	[1][0][2][0][RTW89_FCC][118] = 127,
-	[1][1][2][0][RTW89_FCC][1] = 60,
-	[1][1][2][0][RTW89_FCC][5] = 60,
-	[1][1][2][0][RTW89_FCC][9] = 60,
-	[1][1][2][0][RTW89_FCC][13] = 60,
-	[1][1][2][0][RTW89_FCC][16] = 60,
-	[1][1][2][0][RTW89_FCC][20] = 60,
-	[1][1][2][0][RTW89_FCC][24] = 60,
-	[1][1][2][0][RTW89_FCC][28] = 60,
-	[1][1][2][0][RTW89_FCC][31] = 60,
-	[1][1][2][0][RTW89_FCC][35] = 60,
-	[1][1][2][0][RTW89_FCC][39] = 60,
-	[1][1][2][0][RTW89_FCC][43] = 60,
-	[1][1][2][0][RTW89_FCC][46] = 60,
-	[1][1][2][0][RTW89_FCC][50] = 60,
-	[1][1][2][0][RTW89_FCC][54] = 60,
-	[1][1][2][0][RTW89_FCC][58] = 60,
-	[1][1][2][0][RTW89_FCC][61] = 60,
-	[1][1][2][0][RTW89_FCC][65] = 60,
-	[1][1][2][0][RTW89_FCC][69] = 60,
-	[1][1][2][0][RTW89_FCC][73] = 60,
-	[1][1][2][0][RTW89_FCC][76] = 60,
-	[1][1][2][0][RTW89_FCC][80] = 60,
-	[1][1][2][0][RTW89_FCC][84] = 60,
-	[1][1][2][0][RTW89_FCC][88] = 60,
-	[1][1][2][0][RTW89_FCC][91] = 60,
-	[1][1][2][0][RTW89_FCC][95] = 60,
-	[1][1][2][0][RTW89_FCC][99] = 60,
-	[1][1][2][0][RTW89_FCC][103] = 60,
-	[1][1][2][0][RTW89_FCC][106] = 60,
+	[1][0][2][0][RTW89_ETSI][118] = 127,
+	[1][0][2][0][RTW89_KCC][118] = 127,
+	[1][1][2][0][RTW89_FCC][1] = 10,
+	[1][1][2][0][RTW89_ETSI][1] = 54,
+	[1][1][2][0][RTW89_KCC][1] = 28,
+	[1][1][2][0][RTW89_FCC][5] = 10,
+	[1][1][2][0][RTW89_ETSI][5] = 54,
+	[1][1][2][0][RTW89_KCC][5] = 28,
+	[1][1][2][0][RTW89_FCC][9] = 10,
+	[1][1][2][0][RTW89_ETSI][9] = 54,
+	[1][1][2][0][RTW89_KCC][9] = 28,
+	[1][1][2][0][RTW89_FCC][13] = 10,
+	[1][1][2][0][RTW89_ETSI][13] = 54,
+	[1][1][2][0][RTW89_KCC][13] = 28,
+	[1][1][2][0][RTW89_FCC][16] = 10,
+	[1][1][2][0][RTW89_ETSI][16] = 54,
+	[1][1][2][0][RTW89_KCC][16] = 28,
+	[1][1][2][0][RTW89_FCC][20] = 10,
+	[1][1][2][0][RTW89_ETSI][20] = 54,
+	[1][1][2][0][RTW89_KCC][20] = 28,
+	[1][1][2][0][RTW89_FCC][24] = 10,
+	[1][1][2][0][RTW89_ETSI][24] = 54,
+	[1][1][2][0][RTW89_KCC][24] = 28,
+	[1][1][2][0][RTW89_FCC][28] = 10,
+	[1][1][2][0][RTW89_ETSI][28] = 54,
+	[1][1][2][0][RTW89_KCC][28] = 28,
+	[1][1][2][0][RTW89_FCC][31] = 10,
+	[1][1][2][0][RTW89_ETSI][31] = 54,
+	[1][1][2][0][RTW89_KCC][31] = 28,
+	[1][1][2][0][RTW89_FCC][35] = 10,
+	[1][1][2][0][RTW89_ETSI][35] = 54,
+	[1][1][2][0][RTW89_KCC][35] = 28,
+	[1][1][2][0][RTW89_FCC][39] = 10,
+	[1][1][2][0][RTW89_ETSI][39] = 54,
+	[1][1][2][0][RTW89_KCC][39] = 28,
+	[1][1][2][0][RTW89_FCC][43] = 10,
+	[1][1][2][0][RTW89_ETSI][43] = 54,
+	[1][1][2][0][RTW89_KCC][43] = 28,
+	[1][1][2][0][RTW89_FCC][46] = 12,
+	[1][1][2][0][RTW89_ETSI][46] = 127,
+	[1][1][2][0][RTW89_KCC][46] = 28,
+	[1][1][2][0][RTW89_FCC][50] = 12,
+	[1][1][2][0][RTW89_ETSI][50] = 127,
+	[1][1][2][0][RTW89_KCC][50] = 28,
+	[1][1][2][0][RTW89_FCC][54] = 10,
+	[1][1][2][0][RTW89_ETSI][54] = 127,
+	[1][1][2][0][RTW89_KCC][54] = 28,
+	[1][1][2][0][RTW89_FCC][58] = 10,
+	[1][1][2][0][RTW89_ETSI][58] = 127,
+	[1][1][2][0][RTW89_KCC][58] = 28,
+	[1][1][2][0][RTW89_FCC][61] = 10,
+	[1][1][2][0][RTW89_ETSI][61] = 127,
+	[1][1][2][0][RTW89_KCC][61] = 28,
+	[1][1][2][0][RTW89_FCC][65] = 10,
+	[1][1][2][0][RTW89_ETSI][65] = 127,
+	[1][1][2][0][RTW89_KCC][65] = 28,
+	[1][1][2][0][RTW89_FCC][69] = 10,
+	[1][1][2][0][RTW89_ETSI][69] = 127,
+	[1][1][2][0][RTW89_KCC][69] = 28,
+	[1][1][2][0][RTW89_FCC][73] = 10,
+	[1][1][2][0][RTW89_ETSI][73] = 127,
+	[1][1][2][0][RTW89_KCC][73] = 28,
+	[1][1][2][0][RTW89_FCC][76] = 10,
+	[1][1][2][0][RTW89_ETSI][76] = 127,
+	[1][1][2][0][RTW89_KCC][76] = 28,
+	[1][1][2][0][RTW89_FCC][80] = 10,
+	[1][1][2][0][RTW89_ETSI][80] = 127,
+	[1][1][2][0][RTW89_KCC][80] = 32,
+	[1][1][2][0][RTW89_FCC][84] = 10,
+	[1][1][2][0][RTW89_ETSI][84] = 127,
+	[1][1][2][0][RTW89_KCC][84] = 32,
+	[1][1][2][0][RTW89_FCC][88] = 10,
+	[1][1][2][0][RTW89_ETSI][88] = 127,
+	[1][1][2][0][RTW89_KCC][88] = 32,
+	[1][1][2][0][RTW89_FCC][91] = 12,
+	[1][1][2][0][RTW89_ETSI][91] = 127,
+	[1][1][2][0][RTW89_KCC][91] = 32,
+	[1][1][2][0][RTW89_FCC][95] = 10,
+	[1][1][2][0][RTW89_ETSI][95] = 127,
+	[1][1][2][0][RTW89_KCC][95] = 32,
+	[1][1][2][0][RTW89_FCC][99] = 10,
+	[1][1][2][0][RTW89_ETSI][99] = 127,
+	[1][1][2][0][RTW89_KCC][99] = 32,
+	[1][1][2][0][RTW89_FCC][103] = 10,
+	[1][1][2][0][RTW89_ETSI][103] = 127,
+	[1][1][2][0][RTW89_KCC][103] = 32,
+	[1][1][2][0][RTW89_FCC][106] = 12,
+	[1][1][2][0][RTW89_ETSI][106] = 127,
+	[1][1][2][0][RTW89_KCC][106] = 32,
 	[1][1][2][0][RTW89_FCC][110] = 127,
+	[1][1][2][0][RTW89_ETSI][110] = 127,
+	[1][1][2][0][RTW89_KCC][110] = 127,
 	[1][1][2][0][RTW89_FCC][114] = 127,
+	[1][1][2][0][RTW89_ETSI][114] = 127,
+	[1][1][2][0][RTW89_KCC][114] = 127,
 	[1][1][2][0][RTW89_FCC][118] = 127,
-	[1][1][2][1][RTW89_FCC][1] = 48,
-	[1][1][2][1][RTW89_FCC][5] = 48,
-	[1][1][2][1][RTW89_FCC][9] = 48,
-	[1][1][2][1][RTW89_FCC][13] = 48,
-	[1][1][2][1][RTW89_FCC][16] = 48,
-	[1][1][2][1][RTW89_FCC][20] = 48,
-	[1][1][2][1][RTW89_FCC][24] = 48,
-	[1][1][2][1][RTW89_FCC][28] = 48,
-	[1][1][2][1][RTW89_FCC][31] = 48,
-	[1][1][2][1][RTW89_FCC][35] = 48,
-	[1][1][2][1][RTW89_FCC][39] = 48,
-	[1][1][2][1][RTW89_FCC][43] = 48,
-	[1][1][2][1][RTW89_FCC][46] = 48,
-	[1][1][2][1][RTW89_FCC][50] = 48,
-	[1][1][2][1][RTW89_FCC][54] = 48,
-	[1][1][2][1][RTW89_FCC][58] = 48,
-	[1][1][2][1][RTW89_FCC][61] = 48,
-	[1][1][2][1][RTW89_FCC][65] = 48,
-	[1][1][2][1][RTW89_FCC][69] = 48,
-	[1][1][2][1][RTW89_FCC][73] = 48,
-	[1][1][2][1][RTW89_FCC][76] = 48,
-	[1][1][2][1][RTW89_FCC][80] = 48,
-	[1][1][2][1][RTW89_FCC][84] = 48,
-	[1][1][2][1][RTW89_FCC][88] = 48,
-	[1][1][2][1][RTW89_FCC][91] = 48,
-	[1][1][2][1][RTW89_FCC][95] = 48,
-	[1][1][2][1][RTW89_FCC][99] = 48,
-	[1][1][2][1][RTW89_FCC][103] = 48,
-	[1][1][2][1][RTW89_FCC][106] = 48,
+	[1][1][2][0][RTW89_ETSI][118] = 127,
+	[1][1][2][0][RTW89_KCC][118] = 127,
+	[1][1][2][1][RTW89_FCC][1] = 10,
+	[1][1][2][1][RTW89_ETSI][1] = 42,
+	[1][1][2][1][RTW89_KCC][1] = 28,
+	[1][1][2][1][RTW89_FCC][5] = 10,
+	[1][1][2][1][RTW89_ETSI][5] = 42,
+	[1][1][2][1][RTW89_KCC][5] = 28,
+	[1][1][2][1][RTW89_FCC][9] = 10,
+	[1][1][2][1][RTW89_ETSI][9] = 42,
+	[1][1][2][1][RTW89_KCC][9] = 28,
+	[1][1][2][1][RTW89_FCC][13] = 10,
+	[1][1][2][1][RTW89_ETSI][13] = 42,
+	[1][1][2][1][RTW89_KCC][13] = 28,
+	[1][1][2][1][RTW89_FCC][16] = 10,
+	[1][1][2][1][RTW89_ETSI][16] = 42,
+	[1][1][2][1][RTW89_KCC][16] = 28,
+	[1][1][2][1][RTW89_FCC][20] = 10,
+	[1][1][2][1][RTW89_ETSI][20] = 42,
+	[1][1][2][1][RTW89_KCC][20] = 28,
+	[1][1][2][1][RTW89_FCC][24] = 10,
+	[1][1][2][1][RTW89_ETSI][24] = 42,
+	[1][1][2][1][RTW89_KCC][24] = 28,
+	[1][1][2][1][RTW89_FCC][28] = 10,
+	[1][1][2][1][RTW89_ETSI][28] = 42,
+	[1][1][2][1][RTW89_KCC][28] = 28,
+	[1][1][2][1][RTW89_FCC][31] = 10,
+	[1][1][2][1][RTW89_ETSI][31] = 42,
+	[1][1][2][1][RTW89_KCC][31] = 28,
+	[1][1][2][1][RTW89_FCC][35] = 10,
+	[1][1][2][1][RTW89_ETSI][35] = 42,
+	[1][1][2][1][RTW89_KCC][35] = 28,
+	[1][1][2][1][RTW89_FCC][39] = 10,
+	[1][1][2][1][RTW89_ETSI][39] = 42,
+	[1][1][2][1][RTW89_KCC][39] = 28,
+	[1][1][2][1][RTW89_FCC][43] = 10,
+	[1][1][2][1][RTW89_ETSI][43] = 42,
+	[1][1][2][1][RTW89_KCC][43] = 28,
+	[1][1][2][1][RTW89_FCC][46] = 12,
+	[1][1][2][1][RTW89_ETSI][46] = 127,
+	[1][1][2][1][RTW89_KCC][46] = 28,
+	[1][1][2][1][RTW89_FCC][50] = 12,
+	[1][1][2][1][RTW89_ETSI][50] = 127,
+	[1][1][2][1][RTW89_KCC][50] = 28,
+	[1][1][2][1][RTW89_FCC][54] = 10,
+	[1][1][2][1][RTW89_ETSI][54] = 127,
+	[1][1][2][1][RTW89_KCC][54] = 28,
+	[1][1][2][1][RTW89_FCC][58] = 10,
+	[1][1][2][1][RTW89_ETSI][58] = 127,
+	[1][1][2][1][RTW89_KCC][58] = 28,
+	[1][1][2][1][RTW89_FCC][61] = 10,
+	[1][1][2][1][RTW89_ETSI][61] = 127,
+	[1][1][2][1][RTW89_KCC][61] = 28,
+	[1][1][2][1][RTW89_FCC][65] = 10,
+	[1][1][2][1][RTW89_ETSI][65] = 127,
+	[1][1][2][1][RTW89_KCC][65] = 28,
+	[1][1][2][1][RTW89_FCC][69] = 10,
+	[1][1][2][1][RTW89_ETSI][69] = 127,
+	[1][1][2][1][RTW89_KCC][69] = 28,
+	[1][1][2][1][RTW89_FCC][73] = 10,
+	[1][1][2][1][RTW89_ETSI][73] = 127,
+	[1][1][2][1][RTW89_KCC][73] = 28,
+	[1][1][2][1][RTW89_FCC][76] = 10,
+	[1][1][2][1][RTW89_ETSI][76] = 127,
+	[1][1][2][1][RTW89_KCC][76] = 28,
+	[1][1][2][1][RTW89_FCC][80] = 10,
+	[1][1][2][1][RTW89_ETSI][80] = 127,
+	[1][1][2][1][RTW89_KCC][80] = 32,
+	[1][1][2][1][RTW89_FCC][84] = 10,
+	[1][1][2][1][RTW89_ETSI][84] = 127,
+	[1][1][2][1][RTW89_KCC][84] = 32,
+	[1][1][2][1][RTW89_FCC][88] = 10,
+	[1][1][2][1][RTW89_ETSI][88] = 127,
+	[1][1][2][1][RTW89_KCC][88] = 32,
+	[1][1][2][1][RTW89_FCC][91] = 12,
+	[1][1][2][1][RTW89_ETSI][91] = 127,
+	[1][1][2][1][RTW89_KCC][91] = 32,
+	[1][1][2][1][RTW89_FCC][95] = 10,
+	[1][1][2][1][RTW89_ETSI][95] = 127,
+	[1][1][2][1][RTW89_KCC][95] = 32,
+	[1][1][2][1][RTW89_FCC][99] = 10,
+	[1][1][2][1][RTW89_ETSI][99] = 127,
+	[1][1][2][1][RTW89_KCC][99] = 32,
+	[1][1][2][1][RTW89_FCC][103] = 10,
+	[1][1][2][1][RTW89_ETSI][103] = 127,
+	[1][1][2][1][RTW89_KCC][103] = 32,
+	[1][1][2][1][RTW89_FCC][106] = 12,
+	[1][1][2][1][RTW89_ETSI][106] = 127,
+	[1][1][2][1][RTW89_KCC][106] = 32,
 	[1][1][2][1][RTW89_FCC][110] = 127,
+	[1][1][2][1][RTW89_ETSI][110] = 127,
+	[1][1][2][1][RTW89_KCC][110] = 127,
 	[1][1][2][1][RTW89_FCC][114] = 127,
+	[1][1][2][1][RTW89_ETSI][114] = 127,
+	[1][1][2][1][RTW89_KCC][114] = 127,
 	[1][1][2][1][RTW89_FCC][118] = 127,
-	[2][0][2][0][RTW89_FCC][3] = 64,
-	[2][0][2][0][RTW89_FCC][11] = 64,
-	[2][0][2][0][RTW89_FCC][18] = 64,
-	[2][0][2][0][RTW89_FCC][26] = 64,
-	[2][0][2][0][RTW89_FCC][33] = 64,
-	[2][0][2][0][RTW89_FCC][41] = 64,
-	[2][0][2][0][RTW89_FCC][48] = 64,
-	[2][0][2][0][RTW89_FCC][56] = 64,
-	[2][0][2][0][RTW89_FCC][63] = 64,
-	[2][0][2][0][RTW89_FCC][71] = 64,
-	[2][0][2][0][RTW89_FCC][78] = 64,
-	[2][0][2][0][RTW89_FCC][86] = 64,
-	[2][0][2][0][RTW89_FCC][93] = 64,
-	[2][0][2][0][RTW89_FCC][101] = 64,
+	[1][1][2][1][RTW89_ETSI][118] = 127,
+	[1][1][2][1][RTW89_KCC][118] = 127,
+	[2][0][2][0][RTW89_FCC][3] = 46,
+	[2][0][2][0][RTW89_ETSI][3] = 48,
+	[2][0][2][0][RTW89_KCC][3] = 50,
+	[2][0][2][0][RTW89_FCC][11] = 46,
+	[2][0][2][0][RTW89_ETSI][11] = 48,
+	[2][0][2][0][RTW89_KCC][11] = 50,
+	[2][0][2][0][RTW89_FCC][18] = 46,
+	[2][0][2][0][RTW89_ETSI][18] = 48,
+	[2][0][2][0][RTW89_KCC][18] = 50,
+	[2][0][2][0][RTW89_FCC][26] = 46,
+	[2][0][2][0][RTW89_ETSI][26] = 48,
+	[2][0][2][0][RTW89_KCC][26] = 50,
+	[2][0][2][0][RTW89_FCC][33] = 46,
+	[2][0][2][0][RTW89_ETSI][33] = 48,
+	[2][0][2][0][RTW89_KCC][33] = 50,
+	[2][0][2][0][RTW89_FCC][41] = 46,
+	[2][0][2][0][RTW89_ETSI][41] = 48,
+	[2][0][2][0][RTW89_KCC][41] = 50,
+	[2][0][2][0][RTW89_FCC][48] = 46,
+	[2][0][2][0][RTW89_ETSI][48] = 127,
+	[2][0][2][0][RTW89_KCC][48] = 48,
+	[2][0][2][0][RTW89_FCC][56] = 46,
+	[2][0][2][0][RTW89_ETSI][56] = 127,
+	[2][0][2][0][RTW89_KCC][56] = 48,
+	[2][0][2][0][RTW89_FCC][63] = 46,
+	[2][0][2][0][RTW89_ETSI][63] = 127,
+	[2][0][2][0][RTW89_KCC][63] = 48,
+	[2][0][2][0][RTW89_FCC][71] = 46,
+	[2][0][2][0][RTW89_ETSI][71] = 127,
+	[2][0][2][0][RTW89_KCC][71] = 48,
+	[2][0][2][0][RTW89_FCC][78] = 46,
+	[2][0][2][0][RTW89_ETSI][78] = 127,
+	[2][0][2][0][RTW89_KCC][78] = 52,
+	[2][0][2][0][RTW89_FCC][86] = 46,
+	[2][0][2][0][RTW89_ETSI][86] = 127,
+	[2][0][2][0][RTW89_KCC][86] = 52,
+	[2][0][2][0][RTW89_FCC][93] = 46,
+	[2][0][2][0][RTW89_ETSI][93] = 127,
+	[2][0][2][0][RTW89_KCC][93] = 50,
+	[2][0][2][0][RTW89_FCC][101] = 44,
+	[2][0][2][0][RTW89_ETSI][101] = 127,
+	[2][0][2][0][RTW89_KCC][101] = 50,
 	[2][0][2][0][RTW89_FCC][108] = 127,
+	[2][0][2][0][RTW89_ETSI][108] = 127,
+	[2][0][2][0][RTW89_KCC][108] = 127,
 	[2][0][2][0][RTW89_FCC][116] = 127,
-	[2][1][2][0][RTW89_FCC][3] = 52,
-	[2][1][2][0][RTW89_FCC][11] = 52,
-	[2][1][2][0][RTW89_FCC][18] = 52,
-	[2][1][2][0][RTW89_FCC][26] = 52,
-	[2][1][2][0][RTW89_FCC][33] = 52,
-	[2][1][2][0][RTW89_FCC][41] = 52,
-	[2][1][2][0][RTW89_FCC][48] = 52,
-	[2][1][2][0][RTW89_FCC][56] = 52,
-	[2][1][2][0][RTW89_FCC][63] = 52,
-	[2][1][2][0][RTW89_FCC][71] = 52,
-	[2][1][2][0][RTW89_FCC][78] = 52,
-	[2][1][2][0][RTW89_FCC][86] = 52,
-	[2][1][2][0][RTW89_FCC][93] = 52,
-	[2][1][2][0][RTW89_FCC][101] = 52,
+	[2][0][2][0][RTW89_ETSI][116] = 127,
+	[2][0][2][0][RTW89_KCC][116] = 127,
+	[2][1][2][0][RTW89_FCC][3] = 22,
+	[2][1][2][0][RTW89_ETSI][3] = 48,
+	[2][1][2][0][RTW89_KCC][3] = 38,
+	[2][1][2][0][RTW89_FCC][11] = 20,
+	[2][1][2][0][RTW89_ETSI][11] = 48,
+	[2][1][2][0][RTW89_KCC][11] = 38,
+	[2][1][2][0][RTW89_FCC][18] = 20,
+	[2][1][2][0][RTW89_ETSI][18] = 48,
+	[2][1][2][0][RTW89_KCC][18] = 38,
+	[2][1][2][0][RTW89_FCC][26] = 20,
+	[2][1][2][0][RTW89_ETSI][26] = 48,
+	[2][1][2][0][RTW89_KCC][26] = 38,
+	[2][1][2][0][RTW89_FCC][33] = 20,
+	[2][1][2][0][RTW89_ETSI][33] = 48,
+	[2][1][2][0][RTW89_KCC][33] = 38,
+	[2][1][2][0][RTW89_FCC][41] = 22,
+	[2][1][2][0][RTW89_ETSI][41] = 48,
+	[2][1][2][0][RTW89_KCC][41] = 38,
+	[2][1][2][0][RTW89_FCC][48] = 22,
+	[2][1][2][0][RTW89_ETSI][48] = 127,
+	[2][1][2][0][RTW89_KCC][48] = 38,
+	[2][1][2][0][RTW89_FCC][56] = 20,
+	[2][1][2][0][RTW89_ETSI][56] = 127,
+	[2][1][2][0][RTW89_KCC][56] = 38,
+	[2][1][2][0][RTW89_FCC][63] = 22,
+	[2][1][2][0][RTW89_ETSI][63] = 127,
+	[2][1][2][0][RTW89_KCC][63] = 38,
+	[2][1][2][0][RTW89_FCC][71] = 20,
+	[2][1][2][0][RTW89_ETSI][71] = 127,
+	[2][1][2][0][RTW89_KCC][71] = 38,
+	[2][1][2][0][RTW89_FCC][78] = 20,
+	[2][1][2][0][RTW89_ETSI][78] = 127,
+	[2][1][2][0][RTW89_KCC][78] = 38,
+	[2][1][2][0][RTW89_FCC][86] = 20,
+	[2][1][2][0][RTW89_ETSI][86] = 127,
+	[2][1][2][0][RTW89_KCC][86] = 38,
+	[2][1][2][0][RTW89_FCC][93] = 22,
+	[2][1][2][0][RTW89_ETSI][93] = 127,
+	[2][1][2][0][RTW89_KCC][93] = 38,
+	[2][1][2][0][RTW89_FCC][101] = 22,
+	[2][1][2][0][RTW89_ETSI][101] = 127,
+	[2][1][2][0][RTW89_KCC][101] = 38,
 	[2][1][2][0][RTW89_FCC][108] = 127,
+	[2][1][2][0][RTW89_ETSI][108] = 127,
+	[2][1][2][0][RTW89_KCC][108] = 127,
 	[2][1][2][0][RTW89_FCC][116] = 127,
-	[2][1][2][1][RTW89_FCC][3] = 40,
-	[2][1][2][1][RTW89_FCC][11] = 40,
-	[2][1][2][1][RTW89_FCC][18] = 40,
-	[2][1][2][1][RTW89_FCC][26] = 40,
-	[2][1][2][1][RTW89_FCC][33] = 40,
-	[2][1][2][1][RTW89_FCC][41] = 40,
-	[2][1][2][1][RTW89_FCC][48] = 40,
-	[2][1][2][1][RTW89_FCC][56] = 40,
-	[2][1][2][1][RTW89_FCC][63] = 40,
-	[2][1][2][1][RTW89_FCC][71] = 40,
-	[2][1][2][1][RTW89_FCC][78] = 40,
-	[2][1][2][1][RTW89_FCC][86] = 40,
-	[2][1][2][1][RTW89_FCC][93] = 40,
-	[2][1][2][1][RTW89_FCC][101] = 40,
+	[2][1][2][0][RTW89_ETSI][116] = 127,
+	[2][1][2][0][RTW89_KCC][116] = 127,
+	[2][1][2][1][RTW89_FCC][3] = 22,
+	[2][1][2][1][RTW89_ETSI][3] = 42,
+	[2][1][2][1][RTW89_KCC][3] = 38,
+	[2][1][2][1][RTW89_FCC][11] = 20,
+	[2][1][2][1][RTW89_ETSI][11] = 42,
+	[2][1][2][1][RTW89_KCC][11] = 38,
+	[2][1][2][1][RTW89_FCC][18] = 20,
+	[2][1][2][1][RTW89_ETSI][18] = 42,
+	[2][1][2][1][RTW89_KCC][18] = 38,
+	[2][1][2][1][RTW89_FCC][26] = 20,
+	[2][1][2][1][RTW89_ETSI][26] = 42,
+	[2][1][2][1][RTW89_KCC][26] = 38,
+	[2][1][2][1][RTW89_FCC][33] = 20,
+	[2][1][2][1][RTW89_ETSI][33] = 42,
+	[2][1][2][1][RTW89_KCC][33] = 38,
+	[2][1][2][1][RTW89_FCC][41] = 22,
+	[2][1][2][1][RTW89_ETSI][41] = 42,
+	[2][1][2][1][RTW89_KCC][41] = 38,
+	[2][1][2][1][RTW89_FCC][48] = 22,
+	[2][1][2][1][RTW89_ETSI][48] = 127,
+	[2][1][2][1][RTW89_KCC][48] = 38,
+	[2][1][2][1][RTW89_FCC][56] = 20,
+	[2][1][2][1][RTW89_ETSI][56] = 127,
+	[2][1][2][1][RTW89_KCC][56] = 38,
+	[2][1][2][1][RTW89_FCC][63] = 22,
+	[2][1][2][1][RTW89_ETSI][63] = 127,
+	[2][1][2][1][RTW89_KCC][63] = 38,
+	[2][1][2][1][RTW89_FCC][71] = 20,
+	[2][1][2][1][RTW89_ETSI][71] = 127,
+	[2][1][2][1][RTW89_KCC][71] = 38,
+	[2][1][2][1][RTW89_FCC][78] = 20,
+	[2][1][2][1][RTW89_ETSI][78] = 127,
+	[2][1][2][1][RTW89_KCC][78] = 38,
+	[2][1][2][1][RTW89_FCC][86] = 20,
+	[2][1][2][1][RTW89_ETSI][86] = 127,
+	[2][1][2][1][RTW89_KCC][86] = 38,
+	[2][1][2][1][RTW89_FCC][93] = 22,
+	[2][1][2][1][RTW89_ETSI][93] = 127,
+	[2][1][2][1][RTW89_KCC][93] = 38,
+	[2][1][2][1][RTW89_FCC][101] = 22,
+	[2][1][2][1][RTW89_ETSI][101] = 127,
+	[2][1][2][1][RTW89_KCC][101] = 38,
 	[2][1][2][1][RTW89_FCC][108] = 127,
+	[2][1][2][1][RTW89_ETSI][108] = 127,
+	[2][1][2][1][RTW89_KCC][108] = 127,
 	[2][1][2][1][RTW89_FCC][116] = 127,
-	[3][0][2][0][RTW89_FCC][7] = 56,
-	[3][0][2][0][RTW89_FCC][22] = 56,
-	[3][0][2][0][RTW89_FCC][37] = 56,
-	[3][0][2][0][RTW89_FCC][52] = 56,
-	[3][0][2][0][RTW89_FCC][67] = 56,
-	[3][0][2][0][RTW89_FCC][82] = 56,
-	[3][0][2][0][RTW89_FCC][97] = 56,
+	[2][1][2][1][RTW89_ETSI][116] = 127,
+	[2][1][2][1][RTW89_KCC][116] = 127,
+	[3][0][2][0][RTW89_FCC][7] = 52,
+	[3][0][2][0][RTW89_ETSI][7] = 38,
+	[3][0][2][0][RTW89_KCC][7] = 42,
+	[3][0][2][0][RTW89_FCC][22] = 52,
+	[3][0][2][0][RTW89_ETSI][22] = 38,
+	[3][0][2][0][RTW89_KCC][22] = 42,
+	[3][0][2][0][RTW89_FCC][37] = 52,
+	[3][0][2][0][RTW89_ETSI][37] = 38,
+	[3][0][2][0][RTW89_KCC][37] = 42,
+	[3][0][2][0][RTW89_FCC][52] = 54,
+	[3][0][2][0][RTW89_ETSI][52] = 127,
+	[3][0][2][0][RTW89_KCC][52] = 56,
+	[3][0][2][0][RTW89_FCC][67] = 54,
+	[3][0][2][0][RTW89_ETSI][67] = 127,
+	[3][0][2][0][RTW89_KCC][67] = 54,
+	[3][0][2][0][RTW89_FCC][82] = 54,
+	[3][0][2][0][RTW89_ETSI][82] = 127,
+	[3][0][2][0][RTW89_KCC][82] = 26,
+	[3][0][2][0][RTW89_FCC][97] = 40,
+	[3][0][2][0][RTW89_ETSI][97] = 127,
+	[3][0][2][0][RTW89_KCC][97] = 26,
 	[3][0][2][0][RTW89_FCC][112] = 127,
-	[3][1][2][0][RTW89_FCC][7] = 44,
-	[3][1][2][0][RTW89_FCC][22] = 44,
-	[3][1][2][0][RTW89_FCC][37] = 44,
-	[3][1][2][0][RTW89_FCC][52] = 44,
-	[3][1][2][0][RTW89_FCC][67] = 44,
-	[3][1][2][0][RTW89_FCC][82] = 44,
-	[3][1][2][0][RTW89_FCC][97] = 44,
+	[3][0][2][0][RTW89_ETSI][112] = 127,
+	[3][0][2][0][RTW89_KCC][112] = 127,
+	[3][1][2][0][RTW89_FCC][7] = 32,
+	[3][1][2][0][RTW89_ETSI][7] = 38,
+	[3][1][2][0][RTW89_KCC][7] = 40,
+	[3][1][2][0][RTW89_FCC][22] = 30,
+	[3][1][2][0][RTW89_ETSI][22] = 38,
+	[3][1][2][0][RTW89_KCC][22] = 40,
+	[3][1][2][0][RTW89_FCC][37] = 30,
+	[3][1][2][0][RTW89_ETSI][37] = 38,
+	[3][1][2][0][RTW89_KCC][37] = 40,
+	[3][1][2][0][RTW89_FCC][52] = 30,
+	[3][1][2][0][RTW89_ETSI][52] = 127,
+	[3][1][2][0][RTW89_KCC][52] = 48,
+	[3][1][2][0][RTW89_FCC][67] = 32,
+	[3][1][2][0][RTW89_ETSI][67] = 127,
+	[3][1][2][0][RTW89_KCC][67] = 48,
+	[3][1][2][0][RTW89_FCC][82] = 32,
+	[3][1][2][0][RTW89_ETSI][82] = 127,
+	[3][1][2][0][RTW89_KCC][82] = 24,
+	[3][1][2][0][RTW89_FCC][97] = 14,
+	[3][1][2][0][RTW89_ETSI][97] = 127,
+	[3][1][2][0][RTW89_KCC][97] = 24,
 	[3][1][2][0][RTW89_FCC][112] = 127,
+	[3][1][2][0][RTW89_ETSI][112] = 127,
+	[3][1][2][0][RTW89_KCC][112] = 127,
 	[3][1][2][1][RTW89_FCC][7] = 32,
-	[3][1][2][1][RTW89_FCC][22] = 32,
-	[3][1][2][1][RTW89_FCC][37] = 32,
-	[3][1][2][1][RTW89_FCC][52] = 32,
+	[3][1][2][1][RTW89_ETSI][7] = 38,
+	[3][1][2][1][RTW89_KCC][7] = 40,
+	[3][1][2][1][RTW89_FCC][22] = 30,
+	[3][1][2][1][RTW89_ETSI][22] = 38,
+	[3][1][2][1][RTW89_KCC][22] = 40,
+	[3][1][2][1][RTW89_FCC][37] = 30,
+	[3][1][2][1][RTW89_ETSI][37] = 38,
+	[3][1][2][1][RTW89_KCC][37] = 40,
+	[3][1][2][1][RTW89_FCC][52] = 30,
+	[3][1][2][1][RTW89_ETSI][52] = 127,
+	[3][1][2][1][RTW89_KCC][52] = 48,
 	[3][1][2][1][RTW89_FCC][67] = 32,
+	[3][1][2][1][RTW89_ETSI][67] = 127,
+	[3][1][2][1][RTW89_KCC][67] = 48,
 	[3][1][2][1][RTW89_FCC][82] = 32,
-	[3][1][2][1][RTW89_FCC][97] = 32,
+	[3][1][2][1][RTW89_ETSI][82] = 127,
+	[3][1][2][1][RTW89_KCC][82] = 24,
+	[3][1][2][1][RTW89_FCC][97] = 14,
+	[3][1][2][1][RTW89_ETSI][97] = 127,
+	[3][1][2][1][RTW89_KCC][97] = 24,
 	[3][1][2][1][RTW89_FCC][112] = 127,
+	[3][1][2][1][RTW89_ETSI][112] = 127,
+	[3][1][2][1][RTW89_KCC][112] = 127,
 };
 
 const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
@@ -17126,8 +33220,8 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 	[0][0][RTW89_WW][8] = 32,
 	[0][0][RTW89_WW][9] = 32,
 	[0][0][RTW89_WW][10] = 32,
-	[0][0][RTW89_WW][11] = 32,
-	[0][0][RTW89_WW][12] = 24,
+	[0][0][RTW89_WW][11] = 26,
+	[0][0][RTW89_WW][12] = -20,
 	[0][0][RTW89_WW][13] = 0,
 	[0][1][RTW89_WW][0] = 20,
 	[0][1][RTW89_WW][1] = 22,
@@ -17141,7 +33235,7 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 	[0][1][RTW89_WW][9] = 22,
 	[0][1][RTW89_WW][10] = 22,
 	[0][1][RTW89_WW][11] = 22,
-	[0][1][RTW89_WW][12] = 20,
+	[0][1][RTW89_WW][12] = -30,
 	[0][1][RTW89_WW][13] = 0,
 	[1][0][RTW89_WW][0] = 42,
 	[1][0][RTW89_WW][1] = 44,
@@ -17154,8 +33248,8 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 	[1][0][RTW89_WW][8] = 44,
 	[1][0][RTW89_WW][9] = 44,
 	[1][0][RTW89_WW][10] = 44,
-	[1][0][RTW89_WW][11] = 42,
-	[1][0][RTW89_WW][12] = 30,
+	[1][0][RTW89_WW][11] = 36,
+	[1][0][RTW89_WW][12] = 4,
 	[1][0][RTW89_WW][13] = 0,
 	[1][1][RTW89_WW][0] = 32,
 	[1][1][RTW89_WW][1] = 32,
@@ -17169,7 +33263,7 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 	[1][1][RTW89_WW][9] = 32,
 	[1][1][RTW89_WW][10] = 32,
 	[1][1][RTW89_WW][11] = 30,
-	[1][1][RTW89_WW][12] = 24,
+	[1][1][RTW89_WW][12] = -6,
 	[1][1][RTW89_WW][13] = 0,
 	[2][0][RTW89_WW][0] = 56,
 	[2][0][RTW89_WW][1] = 56,
@@ -17182,8 +33276,8 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 	[2][0][RTW89_WW][8] = 56,
 	[2][0][RTW89_WW][9] = 56,
 	[2][0][RTW89_WW][10] = 56,
-	[2][0][RTW89_WW][11] = 42,
-	[2][0][RTW89_WW][12] = 38,
+	[2][0][RTW89_WW][11] = 48,
+	[2][0][RTW89_WW][12] = 16,
 	[2][0][RTW89_WW][13] = 0,
 	[2][1][RTW89_WW][0] = 44,
 	[2][1][RTW89_WW][1] = 44,
@@ -17196,2213 +33290,3353 @@ const s8 rtw89_8852c_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM]
 	[2][1][RTW89_WW][8] = 44,
 	[2][1][RTW89_WW][9] = 44,
 	[2][1][RTW89_WW][10] = 44,
-	[2][1][RTW89_WW][11] = 30,
-	[2][1][RTW89_WW][12] = 26,
+	[2][1][RTW89_WW][11] = 44,
+	[2][1][RTW89_WW][12] = 6,
 	[2][1][RTW89_WW][13] = 0,
 	[0][0][RTW89_FCC][0] = 60,
 	[0][0][RTW89_ETSI][0] = 34,
 	[0][0][RTW89_MKK][0] = 36,
-	[0][0][RTW89_IC][0] = 68,
-	[0][0][RTW89_ACMA][0] = 32,
+	[0][0][RTW89_IC][0] = 60,
+	[0][0][RTW89_KCC][0] = 42,
+	[0][0][RTW89_ACMA][0] = 34,
+	[0][0][RTW89_CN][0] = 32,
+	[0][0][RTW89_UK][0] = 34,
 	[0][0][RTW89_FCC][1] = 60,
 	[0][0][RTW89_ETSI][1] = 38,
 	[0][0][RTW89_MKK][1] = 40,
-	[0][0][RTW89_IC][1] = 68,
-	[0][0][RTW89_ACMA][1] = 32,
+	[0][0][RTW89_IC][1] = 60,
+	[0][0][RTW89_KCC][1] = 42,
+	[0][0][RTW89_ACMA][1] = 38,
+	[0][0][RTW89_CN][1] = 32,
+	[0][0][RTW89_UK][1] = 38,
 	[0][0][RTW89_FCC][2] = 64,
 	[0][0][RTW89_ETSI][2] = 38,
 	[0][0][RTW89_MKK][2] = 40,
-	[0][0][RTW89_IC][2] = 72,
-	[0][0][RTW89_ACMA][2] = 32,
+	[0][0][RTW89_IC][2] = 64,
+	[0][0][RTW89_KCC][2] = 42,
+	[0][0][RTW89_ACMA][2] = 38,
+	[0][0][RTW89_CN][2] = 32,
+	[0][0][RTW89_UK][2] = 38,
 	[0][0][RTW89_FCC][3] = 68,
 	[0][0][RTW89_ETSI][3] = 38,
 	[0][0][RTW89_MKK][3] = 40,
-	[0][0][RTW89_IC][3] = 76,
-	[0][0][RTW89_ACMA][3] = 32,
+	[0][0][RTW89_IC][3] = 68,
+	[0][0][RTW89_KCC][3] = 42,
+	[0][0][RTW89_ACMA][3] = 38,
+	[0][0][RTW89_CN][3] = 32,
+	[0][0][RTW89_UK][3] = 38,
 	[0][0][RTW89_FCC][4] = 68,
 	[0][0][RTW89_ETSI][4] = 38,
 	[0][0][RTW89_MKK][4] = 40,
-	[0][0][RTW89_IC][4] = 76,
-	[0][0][RTW89_ACMA][4] = 32,
-	[0][0][RTW89_FCC][5] = 76,
+	[0][0][RTW89_IC][4] = 68,
+	[0][0][RTW89_KCC][4] = 42,
+	[0][0][RTW89_ACMA][4] = 38,
+	[0][0][RTW89_CN][4] = 32,
+	[0][0][RTW89_UK][4] = 38,
+	[0][0][RTW89_FCC][5] = 78,
 	[0][0][RTW89_ETSI][5] = 38,
 	[0][0][RTW89_MKK][5] = 40,
-	[0][0][RTW89_IC][5] = 84,
-	[0][0][RTW89_ACMA][5] = 32,
-	[0][0][RTW89_FCC][6] = 66,
+	[0][0][RTW89_IC][5] = 78,
+	[0][0][RTW89_KCC][5] = 42,
+	[0][0][RTW89_ACMA][5] = 38,
+	[0][0][RTW89_CN][5] = 32,
+	[0][0][RTW89_UK][5] = 38,
+	[0][0][RTW89_FCC][6] = 54,
 	[0][0][RTW89_ETSI][6] = 38,
 	[0][0][RTW89_MKK][6] = 40,
-	[0][0][RTW89_IC][6] = 74,
-	[0][0][RTW89_ACMA][6] = 32,
-	[0][0][RTW89_FCC][7] = 66,
+	[0][0][RTW89_IC][6] = 54,
+	[0][0][RTW89_KCC][6] = 42,
+	[0][0][RTW89_ACMA][6] = 38,
+	[0][0][RTW89_CN][6] = 32,
+	[0][0][RTW89_UK][6] = 38,
+	[0][0][RTW89_FCC][7] = 54,
 	[0][0][RTW89_ETSI][7] = 38,
 	[0][0][RTW89_MKK][7] = 40,
-	[0][0][RTW89_IC][7] = 74,
-	[0][0][RTW89_ACMA][7] = 32,
-	[0][0][RTW89_FCC][8] = 62,
+	[0][0][RTW89_IC][7] = 54,
+	[0][0][RTW89_KCC][7] = 42,
+	[0][0][RTW89_ACMA][7] = 38,
+	[0][0][RTW89_CN][7] = 32,
+	[0][0][RTW89_UK][7] = 38,
+	[0][0][RTW89_FCC][8] = 50,
 	[0][0][RTW89_ETSI][8] = 38,
 	[0][0][RTW89_MKK][8] = 40,
-	[0][0][RTW89_IC][8] = 70,
-	[0][0][RTW89_ACMA][8] = 32,
-	[0][0][RTW89_FCC][9] = 58,
+	[0][0][RTW89_IC][8] = 50,
+	[0][0][RTW89_KCC][8] = 42,
+	[0][0][RTW89_ACMA][8] = 38,
+	[0][0][RTW89_CN][8] = 32,
+	[0][0][RTW89_UK][8] = 38,
+	[0][0][RTW89_FCC][9] = 46,
 	[0][0][RTW89_ETSI][9] = 38,
 	[0][0][RTW89_MKK][9] = 40,
-	[0][0][RTW89_IC][9] = 66,
-	[0][0][RTW89_ACMA][9] = 32,
-	[0][0][RTW89_FCC][10] = 58,
+	[0][0][RTW89_IC][9] = 46,
+	[0][0][RTW89_KCC][9] = 40,
+	[0][0][RTW89_ACMA][9] = 38,
+	[0][0][RTW89_CN][9] = 32,
+	[0][0][RTW89_UK][9] = 38,
+	[0][0][RTW89_FCC][10] = 46,
 	[0][0][RTW89_ETSI][10] = 38,
 	[0][0][RTW89_MKK][10] = 40,
-	[0][0][RTW89_IC][10] = 66,
-	[0][0][RTW89_ACMA][10] = 32,
-	[0][0][RTW89_FCC][11] = 42,
+	[0][0][RTW89_IC][10] = 46,
+	[0][0][RTW89_KCC][10] = 40,
+	[0][0][RTW89_ACMA][10] = 38,
+	[0][0][RTW89_CN][10] = 32,
+	[0][0][RTW89_UK][10] = 38,
+	[0][0][RTW89_FCC][11] = 26,
 	[0][0][RTW89_ETSI][11] = 38,
 	[0][0][RTW89_MKK][11] = 40,
-	[0][0][RTW89_IC][11] = 56,
-	[0][0][RTW89_ACMA][11] = 32,
-	[0][0][RTW89_FCC][12] = 24,
+	[0][0][RTW89_IC][11] = 26,
+	[0][0][RTW89_KCC][11] = 40,
+	[0][0][RTW89_ACMA][11] = 38,
+	[0][0][RTW89_CN][11] = 32,
+	[0][0][RTW89_UK][11] = 38,
+	[0][0][RTW89_FCC][12] = -20,
 	[0][0][RTW89_ETSI][12] = 34,
 	[0][0][RTW89_MKK][12] = 36,
-	[0][0][RTW89_IC][12] = 32,
-	[0][0][RTW89_ACMA][12] = 32,
+	[0][0][RTW89_IC][12] = -20,
+	[0][0][RTW89_KCC][12] = 40,
+	[0][0][RTW89_ACMA][12] = 34,
+	[0][0][RTW89_CN][12] = 32,
+	[0][0][RTW89_UK][12] = 34,
 	[0][0][RTW89_FCC][13] = 127,
 	[0][0][RTW89_ETSI][13] = 127,
 	[0][0][RTW89_MKK][13] = 127,
 	[0][0][RTW89_IC][13] = 127,
+	[0][0][RTW89_KCC][13] = 127,
 	[0][0][RTW89_ACMA][13] = 127,
-	[0][1][RTW89_FCC][0] = 46,
+	[0][0][RTW89_CN][13] = 127,
+	[0][0][RTW89_UK][13] = 127,
+	[0][1][RTW89_FCC][0] = 56,
 	[0][1][RTW89_ETSI][0] = 22,
 	[0][1][RTW89_MKK][0] = 24,
-	[0][1][RTW89_IC][0] = 62,
-	[0][1][RTW89_ACMA][0] = 20,
-	[0][1][RTW89_FCC][1] = 46,
+	[0][1][RTW89_IC][0] = 56,
+	[0][1][RTW89_KCC][0] = 30,
+	[0][1][RTW89_ACMA][0] = 22,
+	[0][1][RTW89_CN][0] = 20,
+	[0][1][RTW89_UK][0] = 22,
+	[0][1][RTW89_FCC][1] = 56,
 	[0][1][RTW89_ETSI][1] = 24,
 	[0][1][RTW89_MKK][1] = 30,
-	[0][1][RTW89_IC][1] = 62,
-	[0][1][RTW89_ACMA][1] = 22,
-	[0][1][RTW89_FCC][2] = 50,
+	[0][1][RTW89_IC][1] = 56,
+	[0][1][RTW89_KCC][1] = 30,
+	[0][1][RTW89_ACMA][1] = 24,
+	[0][1][RTW89_CN][1] = 22,
+	[0][1][RTW89_UK][1] = 24,
+	[0][1][RTW89_FCC][2] = 60,
 	[0][1][RTW89_ETSI][2] = 24,
 	[0][1][RTW89_MKK][2] = 30,
-	[0][1][RTW89_IC][2] = 66,
-	[0][1][RTW89_ACMA][2] = 22,
-	[0][1][RTW89_FCC][3] = 54,
+	[0][1][RTW89_IC][2] = 60,
+	[0][1][RTW89_KCC][2] = 30,
+	[0][1][RTW89_ACMA][2] = 24,
+	[0][1][RTW89_CN][2] = 22,
+	[0][1][RTW89_UK][2] = 24,
+	[0][1][RTW89_FCC][3] = 64,
 	[0][1][RTW89_ETSI][3] = 24,
 	[0][1][RTW89_MKK][3] = 30,
-	[0][1][RTW89_IC][3] = 70,
-	[0][1][RTW89_ACMA][3] = 22,
-	[0][1][RTW89_FCC][4] = 58,
+	[0][1][RTW89_IC][3] = 64,
+	[0][1][RTW89_KCC][3] = 30,
+	[0][1][RTW89_ACMA][3] = 24,
+	[0][1][RTW89_CN][3] = 22,
+	[0][1][RTW89_UK][3] = 24,
+	[0][1][RTW89_FCC][4] = 68,
 	[0][1][RTW89_ETSI][4] = 24,
 	[0][1][RTW89_MKK][4] = 30,
-	[0][1][RTW89_IC][4] = 74,
-	[0][1][RTW89_ACMA][4] = 22,
-	[0][1][RTW89_FCC][5] = 66,
+	[0][1][RTW89_IC][4] = 68,
+	[0][1][RTW89_KCC][4] = 28,
+	[0][1][RTW89_ACMA][4] = 24,
+	[0][1][RTW89_CN][4] = 22,
+	[0][1][RTW89_UK][4] = 24,
+	[0][1][RTW89_FCC][5] = 76,
 	[0][1][RTW89_ETSI][5] = 24,
 	[0][1][RTW89_MKK][5] = 30,
-	[0][1][RTW89_IC][5] = 74,
-	[0][1][RTW89_ACMA][5] = 22,
-	[0][1][RTW89_FCC][6] = 58,
+	[0][1][RTW89_IC][5] = 76,
+	[0][1][RTW89_KCC][5] = 28,
+	[0][1][RTW89_ACMA][5] = 24,
+	[0][1][RTW89_CN][5] = 22,
+	[0][1][RTW89_UK][5] = 24,
+	[0][1][RTW89_FCC][6] = 54,
 	[0][1][RTW89_ETSI][6] = 24,
 	[0][1][RTW89_MKK][6] = 30,
-	[0][1][RTW89_IC][6] = 72,
-	[0][1][RTW89_ACMA][6] = 22,
-	[0][1][RTW89_FCC][7] = 54,
+	[0][1][RTW89_IC][6] = 54,
+	[0][1][RTW89_KCC][6] = 28,
+	[0][1][RTW89_ACMA][6] = 24,
+	[0][1][RTW89_CN][6] = 22,
+	[0][1][RTW89_UK][6] = 24,
+	[0][1][RTW89_FCC][7] = 50,
 	[0][1][RTW89_ETSI][7] = 24,
 	[0][1][RTW89_MKK][7] = 30,
-	[0][1][RTW89_IC][7] = 68,
-	[0][1][RTW89_ACMA][7] = 22,
-	[0][1][RTW89_FCC][8] = 50,
+	[0][1][RTW89_IC][7] = 50,
+	[0][1][RTW89_KCC][7] = 28,
+	[0][1][RTW89_ACMA][7] = 24,
+	[0][1][RTW89_CN][7] = 22,
+	[0][1][RTW89_UK][7] = 24,
+	[0][1][RTW89_FCC][8] = 46,
 	[0][1][RTW89_ETSI][8] = 24,
 	[0][1][RTW89_MKK][8] = 30,
-	[0][1][RTW89_IC][8] = 64,
-	[0][1][RTW89_ACMA][8] = 22,
-	[0][1][RTW89_FCC][9] = 46,
+	[0][1][RTW89_IC][8] = 46,
+	[0][1][RTW89_KCC][8] = 28,
+	[0][1][RTW89_ACMA][8] = 24,
+	[0][1][RTW89_CN][8] = 22,
+	[0][1][RTW89_UK][8] = 24,
+	[0][1][RTW89_FCC][9] = 42,
 	[0][1][RTW89_ETSI][9] = 24,
 	[0][1][RTW89_MKK][9] = 30,
-	[0][1][RTW89_IC][9] = 60,
-	[0][1][RTW89_ACMA][9] = 22,
-	[0][1][RTW89_FCC][10] = 46,
+	[0][1][RTW89_IC][9] = 42,
+	[0][1][RTW89_KCC][9] = 28,
+	[0][1][RTW89_ACMA][9] = 24,
+	[0][1][RTW89_CN][9] = 22,
+	[0][1][RTW89_UK][9] = 24,
+	[0][1][RTW89_FCC][10] = 42,
 	[0][1][RTW89_ETSI][10] = 24,
 	[0][1][RTW89_MKK][10] = 30,
-	[0][1][RTW89_IC][10] = 60,
-	[0][1][RTW89_ACMA][10] = 22,
-	[0][1][RTW89_FCC][11] = 30,
+	[0][1][RTW89_IC][10] = 42,
+	[0][1][RTW89_KCC][10] = 28,
+	[0][1][RTW89_ACMA][10] = 24,
+	[0][1][RTW89_CN][10] = 22,
+	[0][1][RTW89_UK][10] = 24,
+	[0][1][RTW89_FCC][11] = 22,
 	[0][1][RTW89_ETSI][11] = 24,
 	[0][1][RTW89_MKK][11] = 30,
-	[0][1][RTW89_IC][11] = 52,
-	[0][1][RTW89_ACMA][11] = 22,
-	[0][1][RTW89_FCC][12] = 22,
+	[0][1][RTW89_IC][11] = 22,
+	[0][1][RTW89_KCC][11] = 28,
+	[0][1][RTW89_ACMA][11] = 24,
+	[0][1][RTW89_CN][11] = 22,
+	[0][1][RTW89_UK][11] = 24,
+	[0][1][RTW89_FCC][12] = -30,
 	[0][1][RTW89_ETSI][12] = 20,
 	[0][1][RTW89_MKK][12] = 24,
-	[0][1][RTW89_IC][12] = 30,
+	[0][1][RTW89_IC][12] = -30,
+	[0][1][RTW89_KCC][12] = 28,
 	[0][1][RTW89_ACMA][12] = 20,
+	[0][1][RTW89_CN][12] = 20,
+	[0][1][RTW89_UK][12] = 20,
 	[0][1][RTW89_FCC][13] = 127,
 	[0][1][RTW89_ETSI][13] = 127,
 	[0][1][RTW89_MKK][13] = 127,
 	[0][1][RTW89_IC][13] = 127,
+	[0][1][RTW89_KCC][13] = 127,
 	[0][1][RTW89_ACMA][13] = 127,
-	[1][0][RTW89_FCC][0] = 64,
+	[0][1][RTW89_CN][13] = 127,
+	[0][1][RTW89_UK][13] = 127,
+	[1][0][RTW89_FCC][0] = 66,
 	[1][0][RTW89_ETSI][0] = 46,
 	[1][0][RTW89_MKK][0] = 48,
-	[1][0][RTW89_IC][0] = 78,
-	[1][0][RTW89_ACMA][0] = 42,
-	[1][0][RTW89_FCC][1] = 64,
+	[1][0][RTW89_IC][0] = 66,
+	[1][0][RTW89_KCC][0] = 50,
+	[1][0][RTW89_ACMA][0] = 46,
+	[1][0][RTW89_CN][0] = 42,
+	[1][0][RTW89_UK][0] = 46,
+	[1][0][RTW89_FCC][1] = 66,
 	[1][0][RTW89_ETSI][1] = 46,
 	[1][0][RTW89_MKK][1] = 48,
-	[1][0][RTW89_IC][1] = 78,
-	[1][0][RTW89_ACMA][1] = 44,
-	[1][0][RTW89_FCC][2] = 68,
+	[1][0][RTW89_IC][1] = 66,
+	[1][0][RTW89_KCC][1] = 50,
+	[1][0][RTW89_ACMA][1] = 46,
+	[1][0][RTW89_CN][1] = 44,
+	[1][0][RTW89_UK][1] = 46,
+	[1][0][RTW89_FCC][2] = 70,
 	[1][0][RTW89_ETSI][2] = 46,
 	[1][0][RTW89_MKK][2] = 48,
-	[1][0][RTW89_IC][2] = 82,
-	[1][0][RTW89_ACMA][2] = 44,
-	[1][0][RTW89_FCC][3] = 70,
+	[1][0][RTW89_IC][2] = 70,
+	[1][0][RTW89_KCC][2] = 50,
+	[1][0][RTW89_ACMA][2] = 46,
+	[1][0][RTW89_CN][2] = 44,
+	[1][0][RTW89_UK][2] = 46,
+	[1][0][RTW89_FCC][3] = 72,
 	[1][0][RTW89_ETSI][3] = 46,
 	[1][0][RTW89_MKK][3] = 48,
-	[1][0][RTW89_IC][3] = 84,
-	[1][0][RTW89_ACMA][3] = 44,
-	[1][0][RTW89_FCC][4] = 70,
+	[1][0][RTW89_IC][3] = 72,
+	[1][0][RTW89_KCC][3] = 50,
+	[1][0][RTW89_ACMA][3] = 46,
+	[1][0][RTW89_CN][3] = 44,
+	[1][0][RTW89_UK][3] = 46,
+	[1][0][RTW89_FCC][4] = 72,
 	[1][0][RTW89_ETSI][4] = 46,
 	[1][0][RTW89_MKK][4] = 48,
-	[1][0][RTW89_IC][4] = 84,
-	[1][0][RTW89_ACMA][4] = 44,
-	[1][0][RTW89_FCC][5] = 76,
+	[1][0][RTW89_IC][4] = 72,
+	[1][0][RTW89_KCC][4] = 50,
+	[1][0][RTW89_ACMA][4] = 46,
+	[1][0][RTW89_CN][4] = 44,
+	[1][0][RTW89_UK][4] = 46,
+	[1][0][RTW89_FCC][5] = 82,
 	[1][0][RTW89_ETSI][5] = 46,
 	[1][0][RTW89_MKK][5] = 48,
-	[1][0][RTW89_IC][5] = 84,
-	[1][0][RTW89_ACMA][5] = 44,
-	[1][0][RTW89_FCC][6] = 64,
+	[1][0][RTW89_IC][5] = 82,
+	[1][0][RTW89_KCC][5] = 50,
+	[1][0][RTW89_ACMA][5] = 46,
+	[1][0][RTW89_CN][5] = 44,
+	[1][0][RTW89_UK][5] = 46,
+	[1][0][RTW89_FCC][6] = 58,
 	[1][0][RTW89_ETSI][6] = 44,
 	[1][0][RTW89_MKK][6] = 48,
-	[1][0][RTW89_IC][6] = 78,
+	[1][0][RTW89_IC][6] = 58,
+	[1][0][RTW89_KCC][6] = 50,
 	[1][0][RTW89_ACMA][6] = 44,
-	[1][0][RTW89_FCC][7] = 64,
+	[1][0][RTW89_CN][6] = 44,
+	[1][0][RTW89_UK][6] = 44,
+	[1][0][RTW89_FCC][7] = 58,
 	[1][0][RTW89_ETSI][7] = 46,
 	[1][0][RTW89_MKK][7] = 48,
-	[1][0][RTW89_IC][7] = 78,
-	[1][0][RTW89_ACMA][7] = 44,
-	[1][0][RTW89_FCC][8] = 64,
+	[1][0][RTW89_IC][7] = 58,
+	[1][0][RTW89_KCC][7] = 50,
+	[1][0][RTW89_ACMA][7] = 46,
+	[1][0][RTW89_CN][7] = 44,
+	[1][0][RTW89_UK][7] = 46,
+	[1][0][RTW89_FCC][8] = 58,
 	[1][0][RTW89_ETSI][8] = 46,
 	[1][0][RTW89_MKK][8] = 48,
-	[1][0][RTW89_IC][8] = 78,
-	[1][0][RTW89_ACMA][8] = 44,
-	[1][0][RTW89_FCC][9] = 60,
+	[1][0][RTW89_IC][8] = 58,
+	[1][0][RTW89_KCC][8] = 50,
+	[1][0][RTW89_ACMA][8] = 46,
+	[1][0][RTW89_CN][8] = 44,
+	[1][0][RTW89_UK][8] = 46,
+	[1][0][RTW89_FCC][9] = 54,
 	[1][0][RTW89_ETSI][9] = 46,
 	[1][0][RTW89_MKK][9] = 48,
-	[1][0][RTW89_IC][9] = 74,
-	[1][0][RTW89_ACMA][9] = 44,
-	[1][0][RTW89_FCC][10] = 60,
+	[1][0][RTW89_IC][9] = 54,
+	[1][0][RTW89_KCC][9] = 50,
+	[1][0][RTW89_ACMA][9] = 46,
+	[1][0][RTW89_CN][9] = 44,
+	[1][0][RTW89_UK][9] = 46,
+	[1][0][RTW89_FCC][10] = 54,
 	[1][0][RTW89_ETSI][10] = 46,
 	[1][0][RTW89_MKK][10] = 48,
-	[1][0][RTW89_IC][10] = 74,
-	[1][0][RTW89_ACMA][10] = 44,
-	[1][0][RTW89_FCC][11] = 42,
+	[1][0][RTW89_IC][10] = 54,
+	[1][0][RTW89_KCC][10] = 50,
+	[1][0][RTW89_ACMA][10] = 46,
+	[1][0][RTW89_CN][10] = 44,
+	[1][0][RTW89_UK][10] = 46,
+	[1][0][RTW89_FCC][11] = 36,
 	[1][0][RTW89_ETSI][11] = 46,
 	[1][0][RTW89_MKK][11] = 48,
-	[1][0][RTW89_IC][11] = 72,
-	[1][0][RTW89_ACMA][11] = 44,
-	[1][0][RTW89_FCC][12] = 30,
+	[1][0][RTW89_IC][11] = 36,
+	[1][0][RTW89_KCC][11] = 50,
+	[1][0][RTW89_ACMA][11] = 46,
+	[1][0][RTW89_CN][11] = 44,
+	[1][0][RTW89_UK][11] = 46,
+	[1][0][RTW89_FCC][12] = 4,
 	[1][0][RTW89_ETSI][12] = 46,
 	[1][0][RTW89_MKK][12] = 46,
-	[1][0][RTW89_IC][12] = 38,
-	[1][0][RTW89_ACMA][12] = 42,
+	[1][0][RTW89_IC][12] = 4,
+	[1][0][RTW89_KCC][12] = 50,
+	[1][0][RTW89_ACMA][12] = 46,
+	[1][0][RTW89_CN][12] = 42,
+	[1][0][RTW89_UK][12] = 46,
 	[1][0][RTW89_FCC][13] = 127,
 	[1][0][RTW89_ETSI][13] = 127,
 	[1][0][RTW89_MKK][13] = 127,
 	[1][0][RTW89_IC][13] = 127,
+	[1][0][RTW89_KCC][13] = 127,
 	[1][0][RTW89_ACMA][13] = 127,
-	[1][1][RTW89_FCC][0] = 46,
+	[1][0][RTW89_CN][13] = 127,
+	[1][0][RTW89_UK][13] = 127,
+	[1][1][RTW89_FCC][0] = 58,
 	[1][1][RTW89_ETSI][0] = 32,
 	[1][1][RTW89_MKK][0] = 34,
-	[1][1][RTW89_IC][0] = 66,
+	[1][1][RTW89_IC][0] = 58,
+	[1][1][RTW89_KCC][0] = 38,
 	[1][1][RTW89_ACMA][0] = 32,
-	[1][1][RTW89_FCC][1] = 46,
+	[1][1][RTW89_CN][0] = 32,
+	[1][1][RTW89_UK][0] = 32,
+	[1][1][RTW89_FCC][1] = 58,
 	[1][1][RTW89_ETSI][1] = 34,
 	[1][1][RTW89_MKK][1] = 34,
-	[1][1][RTW89_IC][1] = 66,
-	[1][1][RTW89_ACMA][1] = 32,
-	[1][1][RTW89_FCC][2] = 50,
+	[1][1][RTW89_IC][1] = 58,
+	[1][1][RTW89_KCC][1] = 38,
+	[1][1][RTW89_ACMA][1] = 34,
+	[1][1][RTW89_CN][1] = 32,
+	[1][1][RTW89_UK][1] = 34,
+	[1][1][RTW89_FCC][2] = 62,
 	[1][1][RTW89_ETSI][2] = 34,
 	[1][1][RTW89_MKK][2] = 34,
-	[1][1][RTW89_IC][2] = 70,
-	[1][1][RTW89_ACMA][2] = 32,
-	[1][1][RTW89_FCC][3] = 54,
+	[1][1][RTW89_IC][2] = 62,
+	[1][1][RTW89_KCC][2] = 38,
+	[1][1][RTW89_ACMA][2] = 34,
+	[1][1][RTW89_CN][2] = 32,
+	[1][1][RTW89_UK][2] = 34,
+	[1][1][RTW89_FCC][3] = 66,
 	[1][1][RTW89_ETSI][3] = 34,
 	[1][1][RTW89_MKK][3] = 34,
-	[1][1][RTW89_IC][3] = 74,
-	[1][1][RTW89_ACMA][3] = 32,
-	[1][1][RTW89_FCC][4] = 58,
+	[1][1][RTW89_IC][3] = 66,
+	[1][1][RTW89_KCC][3] = 38,
+	[1][1][RTW89_ACMA][3] = 34,
+	[1][1][RTW89_CN][3] = 32,
+	[1][1][RTW89_UK][3] = 34,
+	[1][1][RTW89_FCC][4] = 70,
 	[1][1][RTW89_ETSI][4] = 34,
 	[1][1][RTW89_MKK][4] = 34,
-	[1][1][RTW89_IC][4] = 74,
-	[1][1][RTW89_ACMA][4] = 32,
-	[1][1][RTW89_FCC][5] = 66,
+	[1][1][RTW89_IC][4] = 70,
+	[1][1][RTW89_KCC][4] = 38,
+	[1][1][RTW89_ACMA][4] = 34,
+	[1][1][RTW89_CN][4] = 32,
+	[1][1][RTW89_UK][4] = 34,
+	[1][1][RTW89_FCC][5] = 82,
 	[1][1][RTW89_ETSI][5] = 34,
 	[1][1][RTW89_MKK][5] = 34,
-	[1][1][RTW89_IC][5] = 74,
-	[1][1][RTW89_ACMA][5] = 32,
-	[1][1][RTW89_FCC][6] = 58,
+	[1][1][RTW89_IC][5] = 82,
+	[1][1][RTW89_KCC][5] = 38,
+	[1][1][RTW89_ACMA][5] = 34,
+	[1][1][RTW89_CN][5] = 32,
+	[1][1][RTW89_UK][5] = 34,
+	[1][1][RTW89_FCC][6] = 60,
 	[1][1][RTW89_ETSI][6] = 34,
 	[1][1][RTW89_MKK][6] = 34,
-	[1][1][RTW89_IC][6] = 74,
-	[1][1][RTW89_ACMA][6] = 32,
-	[1][1][RTW89_FCC][7] = 54,
+	[1][1][RTW89_IC][6] = 60,
+	[1][1][RTW89_KCC][6] = 38,
+	[1][1][RTW89_ACMA][6] = 34,
+	[1][1][RTW89_CN][6] = 32,
+	[1][1][RTW89_UK][6] = 34,
+	[1][1][RTW89_FCC][7] = 56,
 	[1][1][RTW89_ETSI][7] = 34,
 	[1][1][RTW89_MKK][7] = 34,
-	[1][1][RTW89_IC][7] = 74,
-	[1][1][RTW89_ACMA][7] = 32,
-	[1][1][RTW89_FCC][8] = 50,
+	[1][1][RTW89_IC][7] = 56,
+	[1][1][RTW89_KCC][7] = 38,
+	[1][1][RTW89_ACMA][7] = 34,
+	[1][1][RTW89_CN][7] = 32,
+	[1][1][RTW89_UK][7] = 34,
+	[1][1][RTW89_FCC][8] = 52,
 	[1][1][RTW89_ETSI][8] = 34,
 	[1][1][RTW89_MKK][8] = 34,
-	[1][1][RTW89_IC][8] = 70,
-	[1][1][RTW89_ACMA][8] = 32,
-	[1][1][RTW89_FCC][9] = 46,
+	[1][1][RTW89_IC][8] = 52,
+	[1][1][RTW89_KCC][8] = 38,
+	[1][1][RTW89_ACMA][8] = 34,
+	[1][1][RTW89_CN][8] = 32,
+	[1][1][RTW89_UK][8] = 34,
+	[1][1][RTW89_FCC][9] = 48,
 	[1][1][RTW89_ETSI][9] = 34,
 	[1][1][RTW89_MKK][9] = 34,
-	[1][1][RTW89_IC][9] = 66,
-	[1][1][RTW89_ACMA][9] = 32,
-	[1][1][RTW89_FCC][10] = 46,
+	[1][1][RTW89_IC][9] = 48,
+	[1][1][RTW89_KCC][9] = 38,
+	[1][1][RTW89_ACMA][9] = 34,
+	[1][1][RTW89_CN][9] = 32,
+	[1][1][RTW89_UK][9] = 34,
+	[1][1][RTW89_FCC][10] = 48,
 	[1][1][RTW89_ETSI][10] = 34,
 	[1][1][RTW89_MKK][10] = 34,
-	[1][1][RTW89_IC][10] = 66,
-	[1][1][RTW89_ACMA][10] = 32,
+	[1][1][RTW89_IC][10] = 48,
+	[1][1][RTW89_KCC][10] = 38,
+	[1][1][RTW89_ACMA][10] = 34,
+	[1][1][RTW89_CN][10] = 32,
+	[1][1][RTW89_UK][10] = 34,
 	[1][1][RTW89_FCC][11] = 30,
 	[1][1][RTW89_ETSI][11] = 34,
 	[1][1][RTW89_MKK][11] = 34,
-	[1][1][RTW89_IC][11] = 48,
-	[1][1][RTW89_ACMA][11] = 32,
-	[1][1][RTW89_FCC][12] = 24,
+	[1][1][RTW89_IC][11] = 30,
+	[1][1][RTW89_KCC][11] = 38,
+	[1][1][RTW89_ACMA][11] = 34,
+	[1][1][RTW89_CN][11] = 32,
+	[1][1][RTW89_UK][11] = 34,
+	[1][1][RTW89_FCC][12] = -6,
 	[1][1][RTW89_ETSI][12] = 34,
 	[1][1][RTW89_MKK][12] = 34,
-	[1][1][RTW89_IC][12] = 32,
-	[1][1][RTW89_ACMA][12] = 32,
+	[1][1][RTW89_IC][12] = -6,
+	[1][1][RTW89_KCC][12] = 38,
+	[1][1][RTW89_ACMA][12] = 34,
+	[1][1][RTW89_CN][12] = 32,
+	[1][1][RTW89_UK][12] = 34,
 	[1][1][RTW89_FCC][13] = 127,
 	[1][1][RTW89_ETSI][13] = 127,
 	[1][1][RTW89_MKK][13] = 127,
 	[1][1][RTW89_IC][13] = 127,
+	[1][1][RTW89_KCC][13] = 127,
 	[1][1][RTW89_ACMA][13] = 127,
-	[2][0][RTW89_FCC][0] = 64,
+	[1][1][RTW89_CN][13] = 127,
+	[1][1][RTW89_UK][13] = 127,
+	[2][0][RTW89_FCC][0] = 70,
 	[2][0][RTW89_ETSI][0] = 58,
 	[2][0][RTW89_MKK][0] = 58,
-	[2][0][RTW89_IC][0] = 78,
-	[2][0][RTW89_ACMA][0] = 56,
-	[2][0][RTW89_FCC][1] = 64,
+	[2][0][RTW89_IC][0] = 70,
+	[2][0][RTW89_KCC][0] = 64,
+	[2][0][RTW89_ACMA][0] = 58,
+	[2][0][RTW89_CN][0] = 56,
+	[2][0][RTW89_UK][0] = 58,
+	[2][0][RTW89_FCC][1] = 70,
 	[2][0][RTW89_ETSI][1] = 58,
 	[2][0][RTW89_MKK][1] = 58,
-	[2][0][RTW89_IC][1] = 78,
-	[2][0][RTW89_ACMA][1] = 56,
-	[2][0][RTW89_FCC][2] = 66,
+	[2][0][RTW89_IC][1] = 70,
+	[2][0][RTW89_KCC][1] = 64,
+	[2][0][RTW89_ACMA][1] = 58,
+	[2][0][RTW89_CN][1] = 56,
+	[2][0][RTW89_UK][1] = 58,
+	[2][0][RTW89_FCC][2] = 72,
 	[2][0][RTW89_ETSI][2] = 58,
 	[2][0][RTW89_MKK][2] = 58,
-	[2][0][RTW89_IC][2] = 80,
-	[2][0][RTW89_ACMA][2] = 56,
-	[2][0][RTW89_FCC][3] = 66,
+	[2][0][RTW89_IC][2] = 72,
+	[2][0][RTW89_KCC][2] = 64,
+	[2][0][RTW89_ACMA][2] = 58,
+	[2][0][RTW89_CN][2] = 56,
+	[2][0][RTW89_UK][2] = 58,
+	[2][0][RTW89_FCC][3] = 72,
 	[2][0][RTW89_ETSI][3] = 58,
 	[2][0][RTW89_MKK][3] = 58,
-	[2][0][RTW89_IC][3] = 80,
-	[2][0][RTW89_ACMA][3] = 56,
-	[2][0][RTW89_FCC][4] = 66,
+	[2][0][RTW89_IC][3] = 72,
+	[2][0][RTW89_KCC][3] = 64,
+	[2][0][RTW89_ACMA][3] = 58,
+	[2][0][RTW89_CN][3] = 56,
+	[2][0][RTW89_UK][3] = 58,
+	[2][0][RTW89_FCC][4] = 72,
 	[2][0][RTW89_ETSI][4] = 58,
 	[2][0][RTW89_MKK][4] = 58,
-	[2][0][RTW89_IC][4] = 80,
-	[2][0][RTW89_ACMA][4] = 56,
-	[2][0][RTW89_FCC][5] = 76,
+	[2][0][RTW89_IC][4] = 72,
+	[2][0][RTW89_KCC][4] = 64,
+	[2][0][RTW89_ACMA][4] = 58,
+	[2][0][RTW89_CN][4] = 56,
+	[2][0][RTW89_UK][4] = 58,
+	[2][0][RTW89_FCC][5] = 82,
 	[2][0][RTW89_ETSI][5] = 58,
 	[2][0][RTW89_MKK][5] = 58,
-	[2][0][RTW89_IC][5] = 84,
-	[2][0][RTW89_ACMA][5] = 56,
-	[2][0][RTW89_FCC][6] = 62,
+	[2][0][RTW89_IC][5] = 82,
+	[2][0][RTW89_KCC][5] = 64,
+	[2][0][RTW89_ACMA][5] = 58,
+	[2][0][RTW89_CN][5] = 56,
+	[2][0][RTW89_UK][5] = 58,
+	[2][0][RTW89_FCC][6] = 66,
 	[2][0][RTW89_ETSI][6] = 56,
 	[2][0][RTW89_MKK][6] = 58,
-	[2][0][RTW89_IC][6] = 76,
+	[2][0][RTW89_IC][6] = 66,
+	[2][0][RTW89_KCC][6] = 64,
 	[2][0][RTW89_ACMA][6] = 56,
-	[2][0][RTW89_FCC][7] = 62,
+	[2][0][RTW89_CN][6] = 56,
+	[2][0][RTW89_UK][6] = 56,
+	[2][0][RTW89_FCC][7] = 66,
 	[2][0][RTW89_ETSI][7] = 58,
 	[2][0][RTW89_MKK][7] = 58,
-	[2][0][RTW89_IC][7] = 76,
-	[2][0][RTW89_ACMA][7] = 56,
-	[2][0][RTW89_FCC][8] = 62,
+	[2][0][RTW89_IC][7] = 66,
+	[2][0][RTW89_KCC][7] = 64,
+	[2][0][RTW89_ACMA][7] = 58,
+	[2][0][RTW89_CN][7] = 56,
+	[2][0][RTW89_UK][7] = 58,
+	[2][0][RTW89_FCC][8] = 66,
 	[2][0][RTW89_ETSI][8] = 58,
 	[2][0][RTW89_MKK][8] = 58,
-	[2][0][RTW89_IC][8] = 76,
-	[2][0][RTW89_ACMA][8] = 56,
-	[2][0][RTW89_FCC][9] = 60,
+	[2][0][RTW89_IC][8] = 66,
+	[2][0][RTW89_KCC][8] = 64,
+	[2][0][RTW89_ACMA][8] = 58,
+	[2][0][RTW89_CN][8] = 56,
+	[2][0][RTW89_UK][8] = 58,
+	[2][0][RTW89_FCC][9] = 64,
 	[2][0][RTW89_ETSI][9] = 58,
 	[2][0][RTW89_MKK][9] = 58,
-	[2][0][RTW89_IC][9] = 74,
-	[2][0][RTW89_ACMA][9] = 56,
-	[2][0][RTW89_FCC][10] = 60,
+	[2][0][RTW89_IC][9] = 64,
+	[2][0][RTW89_KCC][9] = 64,
+	[2][0][RTW89_ACMA][9] = 58,
+	[2][0][RTW89_CN][9] = 56,
+	[2][0][RTW89_UK][9] = 58,
+	[2][0][RTW89_FCC][10] = 64,
 	[2][0][RTW89_ETSI][10] = 58,
 	[2][0][RTW89_MKK][10] = 58,
-	[2][0][RTW89_IC][10] = 74,
-	[2][0][RTW89_ACMA][10] = 56,
-	[2][0][RTW89_FCC][11] = 42,
+	[2][0][RTW89_IC][10] = 64,
+	[2][0][RTW89_KCC][10] = 64,
+	[2][0][RTW89_ACMA][10] = 58,
+	[2][0][RTW89_CN][10] = 56,
+	[2][0][RTW89_UK][10] = 58,
+	[2][0][RTW89_FCC][11] = 48,
 	[2][0][RTW89_ETSI][11] = 58,
 	[2][0][RTW89_MKK][11] = 58,
-	[2][0][RTW89_IC][11] = 66,
-	[2][0][RTW89_ACMA][11] = 56,
-	[2][0][RTW89_FCC][12] = 38,
+	[2][0][RTW89_IC][11] = 48,
+	[2][0][RTW89_KCC][11] = 64,
+	[2][0][RTW89_ACMA][11] = 58,
+	[2][0][RTW89_CN][11] = 56,
+	[2][0][RTW89_UK][11] = 58,
+	[2][0][RTW89_FCC][12] = 16,
 	[2][0][RTW89_ETSI][12] = 58,
 	[2][0][RTW89_MKK][12] = 58,
-	[2][0][RTW89_IC][12] = 56,
-	[2][0][RTW89_ACMA][12] = 56,
+	[2][0][RTW89_IC][12] = 16,
+	[2][0][RTW89_KCC][12] = 64,
+	[2][0][RTW89_ACMA][12] = 58,
+	[2][0][RTW89_CN][12] = 56,
+	[2][0][RTW89_UK][12] = 58,
 	[2][0][RTW89_FCC][13] = 127,
 	[2][0][RTW89_ETSI][13] = 127,
 	[2][0][RTW89_MKK][13] = 127,
 	[2][0][RTW89_IC][13] = 127,
+	[2][0][RTW89_KCC][13] = 127,
 	[2][0][RTW89_ACMA][13] = 127,
-	[2][1][RTW89_FCC][0] = 46,
+	[2][0][RTW89_CN][13] = 127,
+	[2][0][RTW89_UK][13] = 127,
+	[2][1][RTW89_FCC][0] = 64,
 	[2][1][RTW89_ETSI][0] = 46,
 	[2][1][RTW89_MKK][0] = 46,
-	[2][1][RTW89_IC][0] = 70,
-	[2][1][RTW89_ACMA][0] = 44,
-	[2][1][RTW89_FCC][1] = 46,
+	[2][1][RTW89_IC][0] = 64,
+	[2][1][RTW89_KCC][0] = 52,
+	[2][1][RTW89_ACMA][0] = 46,
+	[2][1][RTW89_CN][0] = 44,
+	[2][1][RTW89_UK][0] = 46,
+	[2][1][RTW89_FCC][1] = 64,
 	[2][1][RTW89_ETSI][1] = 46,
 	[2][1][RTW89_MKK][1] = 46,
-	[2][1][RTW89_IC][1] = 70,
-	[2][1][RTW89_ACMA][1] = 44,
-	[2][1][RTW89_FCC][2] = 50,
+	[2][1][RTW89_IC][1] = 64,
+	[2][1][RTW89_KCC][1] = 52,
+	[2][1][RTW89_ACMA][1] = 46,
+	[2][1][RTW89_CN][1] = 44,
+	[2][1][RTW89_UK][1] = 46,
+	[2][1][RTW89_FCC][2] = 68,
 	[2][1][RTW89_ETSI][2] = 46,
 	[2][1][RTW89_MKK][2] = 46,
-	[2][1][RTW89_IC][2] = 74,
-	[2][1][RTW89_ACMA][2] = 44,
-	[2][1][RTW89_FCC][3] = 54,
+	[2][1][RTW89_IC][2] = 68,
+	[2][1][RTW89_KCC][2] = 52,
+	[2][1][RTW89_ACMA][2] = 46,
+	[2][1][RTW89_CN][2] = 44,
+	[2][1][RTW89_UK][2] = 46,
+	[2][1][RTW89_FCC][3] = 72,
 	[2][1][RTW89_ETSI][3] = 46,
 	[2][1][RTW89_MKK][3] = 46,
-	[2][1][RTW89_IC][3] = 78,
-	[2][1][RTW89_ACMA][3] = 44,
-	[2][1][RTW89_FCC][4] = 56,
+	[2][1][RTW89_IC][3] = 72,
+	[2][1][RTW89_KCC][3] = 52,
+	[2][1][RTW89_ACMA][3] = 46,
+	[2][1][RTW89_CN][3] = 44,
+	[2][1][RTW89_UK][3] = 46,
+	[2][1][RTW89_FCC][4] = 74,
 	[2][1][RTW89_ETSI][4] = 46,
 	[2][1][RTW89_MKK][4] = 46,
-	[2][1][RTW89_IC][4] = 80,
-	[2][1][RTW89_ACMA][4] = 44,
-	[2][1][RTW89_FCC][5] = 72,
+	[2][1][RTW89_IC][4] = 74,
+	[2][1][RTW89_KCC][4] = 50,
+	[2][1][RTW89_ACMA][4] = 46,
+	[2][1][RTW89_CN][4] = 44,
+	[2][1][RTW89_UK][4] = 46,
+	[2][1][RTW89_FCC][5] = 82,
 	[2][1][RTW89_ETSI][5] = 46,
 	[2][1][RTW89_MKK][5] = 46,
-	[2][1][RTW89_IC][5] = 80,
-	[2][1][RTW89_ACMA][5] = 44,
-	[2][1][RTW89_FCC][6] = 54,
+	[2][1][RTW89_IC][5] = 82,
+	[2][1][RTW89_KCC][5] = 50,
+	[2][1][RTW89_ACMA][5] = 46,
+	[2][1][RTW89_CN][5] = 44,
+	[2][1][RTW89_UK][5] = 46,
+	[2][1][RTW89_FCC][6] = 72,
 	[2][1][RTW89_ETSI][6] = 44,
 	[2][1][RTW89_MKK][6] = 46,
-	[2][1][RTW89_IC][6] = 78,
+	[2][1][RTW89_IC][6] = 72,
+	[2][1][RTW89_KCC][6] = 50,
 	[2][1][RTW89_ACMA][6] = 44,
-	[2][1][RTW89_FCC][7] = 54,
+	[2][1][RTW89_CN][6] = 44,
+	[2][1][RTW89_UK][6] = 44,
+	[2][1][RTW89_FCC][7] = 72,
 	[2][1][RTW89_ETSI][7] = 46,
 	[2][1][RTW89_MKK][7] = 46,
-	[2][1][RTW89_IC][7] = 78,
-	[2][1][RTW89_ACMA][7] = 44,
-	[2][1][RTW89_FCC][8] = 50,
+	[2][1][RTW89_IC][7] = 72,
+	[2][1][RTW89_KCC][7] = 50,
+	[2][1][RTW89_ACMA][7] = 46,
+	[2][1][RTW89_CN][7] = 44,
+	[2][1][RTW89_UK][7] = 46,
+	[2][1][RTW89_FCC][8] = 68,
 	[2][1][RTW89_ETSI][8] = 46,
 	[2][1][RTW89_MKK][8] = 46,
-	[2][1][RTW89_IC][8] = 74,
-	[2][1][RTW89_ACMA][8] = 44,
-	[2][1][RTW89_FCC][9] = 46,
+	[2][1][RTW89_IC][8] = 68,
+	[2][1][RTW89_KCC][8] = 50,
+	[2][1][RTW89_ACMA][8] = 46,
+	[2][1][RTW89_CN][8] = 44,
+	[2][1][RTW89_UK][8] = 46,
+	[2][1][RTW89_FCC][9] = 64,
 	[2][1][RTW89_ETSI][9] = 46,
 	[2][1][RTW89_MKK][9] = 46,
-	[2][1][RTW89_IC][9] = 70,
-	[2][1][RTW89_ACMA][9] = 44,
-	[2][1][RTW89_FCC][10] = 46,
+	[2][1][RTW89_IC][9] = 64,
+	[2][1][RTW89_KCC][9] = 52,
+	[2][1][RTW89_ACMA][9] = 46,
+	[2][1][RTW89_CN][9] = 44,
+	[2][1][RTW89_UK][9] = 46,
+	[2][1][RTW89_FCC][10] = 64,
 	[2][1][RTW89_ETSI][10] = 46,
 	[2][1][RTW89_MKK][10] = 46,
-	[2][1][RTW89_IC][10] = 70,
-	[2][1][RTW89_ACMA][10] = 44,
-	[2][1][RTW89_FCC][11] = 30,
+	[2][1][RTW89_IC][10] = 64,
+	[2][1][RTW89_KCC][10] = 52,
+	[2][1][RTW89_ACMA][10] = 46,
+	[2][1][RTW89_CN][10] = 44,
+	[2][1][RTW89_UK][10] = 46,
+	[2][1][RTW89_FCC][11] = 46,
 	[2][1][RTW89_ETSI][11] = 46,
 	[2][1][RTW89_MKK][11] = 46,
-	[2][1][RTW89_IC][11] = 60,
-	[2][1][RTW89_ACMA][11] = 44,
-	[2][1][RTW89_FCC][12] = 26,
+	[2][1][RTW89_IC][11] = 46,
+	[2][1][RTW89_KCC][11] = 52,
+	[2][1][RTW89_ACMA][11] = 46,
+	[2][1][RTW89_CN][11] = 44,
+	[2][1][RTW89_UK][11] = 46,
+	[2][1][RTW89_FCC][12] = 6,
 	[2][1][RTW89_ETSI][12] = 44,
 	[2][1][RTW89_MKK][12] = 46,
-	[2][1][RTW89_IC][12] = 44,
-	[2][1][RTW89_ACMA][12] = 42,
+	[2][1][RTW89_IC][12] = 6,
+	[2][1][RTW89_KCC][12] = 52,
+	[2][1][RTW89_ACMA][12] = 44,
+	[2][1][RTW89_CN][12] = 42,
+	[2][1][RTW89_UK][12] = 44,
 	[2][1][RTW89_FCC][13] = 127,
 	[2][1][RTW89_ETSI][13] = 127,
 	[2][1][RTW89_MKK][13] = 127,
 	[2][1][RTW89_IC][13] = 127,
+	[2][1][RTW89_KCC][13] = 127,
 	[2][1][RTW89_ACMA][13] = 127,
+	[2][1][RTW89_CN][13] = 127,
+	[2][1][RTW89_UK][13] = 127,
 };
 
 const s8 rtw89_8852c_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM]
 				    [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = {
-	[0][0][RTW89_WW][0] = 24,
-	[0][0][RTW89_WW][2] = 24,
-	[0][0][RTW89_WW][4] = 22,
-	[0][0][RTW89_WW][6] = 22,
-	[0][0][RTW89_WW][8] = 18,
-	[0][0][RTW89_WW][10] = 18,
-	[0][0][RTW89_WW][12] = 24,
-	[0][0][RTW89_WW][14] = 24,
-	[0][0][RTW89_WW][15] = 24,
-	[0][0][RTW89_WW][17] = 24,
-	[0][0][RTW89_WW][19] = 24,
-	[0][0][RTW89_WW][21] = 24,
-	[0][0][RTW89_WW][23] = 24,
+	[0][0][RTW89_WW][0] = 16,
+	[0][0][RTW89_WW][2] = 16,
+	[0][0][RTW89_WW][4] = 16,
+	[0][0][RTW89_WW][6] = 10,
+	[0][0][RTW89_WW][8] = 16,
+	[0][0][RTW89_WW][10] = 16,
+	[0][0][RTW89_WW][12] = 16,
+	[0][0][RTW89_WW][14] = 16,
+	[0][0][RTW89_WW][15] = 30,
+	[0][0][RTW89_WW][17] = 30,
+	[0][0][RTW89_WW][19] = 30,
+	[0][0][RTW89_WW][21] = 30,
+	[0][0][RTW89_WW][23] = 30,
 	[0][0][RTW89_WW][25] = 30,
 	[0][0][RTW89_WW][27] = 30,
 	[0][0][RTW89_WW][29] = 30,
-	[0][0][RTW89_WW][31] = 24,
-	[0][0][RTW89_WW][33] = 24,
-	[0][0][RTW89_WW][35] = 24,
-	[0][0][RTW89_WW][37] = 44,
+	[0][0][RTW89_WW][31] = 30,
+	[0][0][RTW89_WW][33] = 30,
+	[0][0][RTW89_WW][35] = 30,
+	[0][0][RTW89_WW][37] = 30,
 	[0][0][RTW89_WW][38] = 28,
 	[0][0][RTW89_WW][40] = 28,
 	[0][0][RTW89_WW][42] = 28,
 	[0][0][RTW89_WW][44] = 28,
 	[0][0][RTW89_WW][46] = 28,
-	[0][0][RTW89_WW][48] = 24,
-	[0][0][RTW89_WW][50] = 24,
-	[0][0][RTW89_WW][52] = 24,
-	[0][1][RTW89_WW][0] = 0,
+	[0][0][RTW89_WW][48] = 46,
+	[0][0][RTW89_WW][50] = 44,
+	[0][0][RTW89_WW][52] = 34,
+	[0][1][RTW89_WW][0] = 4,
 	[0][1][RTW89_WW][2] = 4,
-	[0][1][RTW89_WW][4] = 0,
-	[0][1][RTW89_WW][6] = 0,
-	[0][1][RTW89_WW][8] = 12,
-	[0][1][RTW89_WW][10] = 12,
-	[0][1][RTW89_WW][12] = 12,
-	[0][1][RTW89_WW][14] = 12,
-	[0][1][RTW89_WW][15] = 12,
-	[0][1][RTW89_WW][17] = 12,
-	[0][1][RTW89_WW][19] = 12,
-	[0][1][RTW89_WW][21] = 12,
-	[0][1][RTW89_WW][23] = 12,
+	[0][1][RTW89_WW][4] = 4,
+	[0][1][RTW89_WW][6] = 1,
+	[0][1][RTW89_WW][8] = 4,
+	[0][1][RTW89_WW][10] = 4,
+	[0][1][RTW89_WW][12] = 4,
+	[0][1][RTW89_WW][14] = 4,
+	[0][1][RTW89_WW][15] = 18,
+	[0][1][RTW89_WW][17] = 18,
+	[0][1][RTW89_WW][19] = 18,
+	[0][1][RTW89_WW][21] = 18,
+	[0][1][RTW89_WW][23] = 18,
 	[0][1][RTW89_WW][25] = 18,
 	[0][1][RTW89_WW][27] = 16,
 	[0][1][RTW89_WW][29] = 16,
-	[0][1][RTW89_WW][31] = 12,
-	[0][1][RTW89_WW][33] = 12,
-	[0][1][RTW89_WW][35] = 12,
-	[0][1][RTW89_WW][37] = 30,
+	[0][1][RTW89_WW][31] = 16,
+	[0][1][RTW89_WW][33] = 16,
+	[0][1][RTW89_WW][35] = 16,
+	[0][1][RTW89_WW][37] = 18,
 	[0][1][RTW89_WW][38] = 16,
 	[0][1][RTW89_WW][40] = 16,
 	[0][1][RTW89_WW][42] = 16,
 	[0][1][RTW89_WW][44] = 16,
 	[0][1][RTW89_WW][46] = 16,
-	[0][1][RTW89_WW][48] = 12,
-	[0][1][RTW89_WW][50] = 12,
-	[0][1][RTW89_WW][52] = 12,
-	[1][0][RTW89_WW][0] = 34,
-	[1][0][RTW89_WW][2] = 34,
-	[1][0][RTW89_WW][4] = 34,
-	[1][0][RTW89_WW][6] = 34,
-	[1][0][RTW89_WW][8] = 34,
-	[1][0][RTW89_WW][10] = 34,
-	[1][0][RTW89_WW][12] = 34,
-	[1][0][RTW89_WW][14] = 34,
-	[1][0][RTW89_WW][15] = 34,
-	[1][0][RTW89_WW][17] = 34,
-	[1][0][RTW89_WW][19] = 34,
-	[1][0][RTW89_WW][21] = 34,
-	[1][0][RTW89_WW][23] = 34,
+	[0][1][RTW89_WW][48] = 20,
+	[0][1][RTW89_WW][50] = 20,
+	[0][1][RTW89_WW][52] = 8,
+	[1][0][RTW89_WW][0] = 26,
+	[1][0][RTW89_WW][2] = 26,
+	[1][0][RTW89_WW][4] = 26,
+	[1][0][RTW89_WW][6] = 24,
+	[1][0][RTW89_WW][8] = 26,
+	[1][0][RTW89_WW][10] = 26,
+	[1][0][RTW89_WW][12] = 26,
+	[1][0][RTW89_WW][14] = 26,
+	[1][0][RTW89_WW][15] = 40,
+	[1][0][RTW89_WW][17] = 40,
+	[1][0][RTW89_WW][19] = 40,
+	[1][0][RTW89_WW][21] = 40,
+	[1][0][RTW89_WW][23] = 40,
 	[1][0][RTW89_WW][25] = 40,
 	[1][0][RTW89_WW][27] = 42,
 	[1][0][RTW89_WW][29] = 42,
-	[1][0][RTW89_WW][31] = 34,
-	[1][0][RTW89_WW][33] = 34,
-	[1][0][RTW89_WW][35] = 34,
-	[1][0][RTW89_WW][37] = 56,
+	[1][0][RTW89_WW][31] = 42,
+	[1][0][RTW89_WW][33] = 42,
+	[1][0][RTW89_WW][35] = 42,
+	[1][0][RTW89_WW][37] = 42,
 	[1][0][RTW89_WW][38] = 28,
 	[1][0][RTW89_WW][40] = 28,
 	[1][0][RTW89_WW][42] = 28,
 	[1][0][RTW89_WW][44] = 28,
 	[1][0][RTW89_WW][46] = 28,
-	[1][0][RTW89_WW][48] = 36,
-	[1][0][RTW89_WW][50] = 36,
-	[1][0][RTW89_WW][52] = 36,
-	[1][1][RTW89_WW][0] = 10,
+	[1][0][RTW89_WW][48] = 56,
+	[1][0][RTW89_WW][50] = 58,
+	[1][0][RTW89_WW][52] = 56,
+	[1][1][RTW89_WW][0] = 14,
 	[1][1][RTW89_WW][2] = 14,
-	[1][1][RTW89_WW][4] = 10,
-	[1][1][RTW89_WW][6] = 10,
-	[1][1][RTW89_WW][8] = 20,
-	[1][1][RTW89_WW][10] = 20,
-	[1][1][RTW89_WW][12] = 22,
-	[1][1][RTW89_WW][14] = 22,
-	[1][1][RTW89_WW][15] = 22,
-	[1][1][RTW89_WW][17] = 22,
-	[1][1][RTW89_WW][19] = 22,
-	[1][1][RTW89_WW][21] = 22,
-	[1][1][RTW89_WW][23] = 22,
+	[1][1][RTW89_WW][4] = 14,
+	[1][1][RTW89_WW][6] = 8,
+	[1][1][RTW89_WW][8] = 14,
+	[1][1][RTW89_WW][10] = 14,
+	[1][1][RTW89_WW][12] = 14,
+	[1][1][RTW89_WW][14] = 14,
+	[1][1][RTW89_WW][15] = 28,
+	[1][1][RTW89_WW][17] = 28,
+	[1][1][RTW89_WW][19] = 28,
+	[1][1][RTW89_WW][21] = 28,
+	[1][1][RTW89_WW][23] = 28,
 	[1][1][RTW89_WW][25] = 28,
 	[1][1][RTW89_WW][27] = 30,
 	[1][1][RTW89_WW][29] = 30,
-	[1][1][RTW89_WW][31] = 22,
-	[1][1][RTW89_WW][33] = 22,
-	[1][1][RTW89_WW][35] = 22,
-	[1][1][RTW89_WW][37] = 40,
+	[1][1][RTW89_WW][31] = 30,
+	[1][1][RTW89_WW][33] = 30,
+	[1][1][RTW89_WW][35] = 30,
+	[1][1][RTW89_WW][37] = 32,
 	[1][1][RTW89_WW][38] = 16,
 	[1][1][RTW89_WW][40] = 16,
 	[1][1][RTW89_WW][42] = 16,
 	[1][1][RTW89_WW][44] = 16,
 	[1][1][RTW89_WW][46] = 16,
-	[1][1][RTW89_WW][48] = 24,
-	[1][1][RTW89_WW][50] = 24,
-	[1][1][RTW89_WW][52] = 24,
-	[2][0][RTW89_WW][0] = 46,
-	[2][0][RTW89_WW][2] = 46,
-	[2][0][RTW89_WW][4] = 46,
-	[2][0][RTW89_WW][6] = 46,
-	[2][0][RTW89_WW][8] = 44,
-	[2][0][RTW89_WW][10] = 44,
-	[2][0][RTW89_WW][12] = 48,
-	[2][0][RTW89_WW][14] = 48,
-	[2][0][RTW89_WW][15] = 48,
-	[2][0][RTW89_WW][17] = 48,
-	[2][0][RTW89_WW][19] = 48,
-	[2][0][RTW89_WW][21] = 48,
-	[2][0][RTW89_WW][23] = 48,
+	[1][1][RTW89_WW][48] = 34,
+	[1][1][RTW89_WW][50] = 34,
+	[1][1][RTW89_WW][52] = 30,
+	[2][0][RTW89_WW][0] = 40,
+	[2][0][RTW89_WW][2] = 40,
+	[2][0][RTW89_WW][4] = 40,
+	[2][0][RTW89_WW][6] = 36,
+	[2][0][RTW89_WW][8] = 40,
+	[2][0][RTW89_WW][10] = 40,
+	[2][0][RTW89_WW][12] = 40,
+	[2][0][RTW89_WW][14] = 40,
+	[2][0][RTW89_WW][15] = 52,
+	[2][0][RTW89_WW][17] = 52,
+	[2][0][RTW89_WW][19] = 52,
+	[2][0][RTW89_WW][21] = 52,
+	[2][0][RTW89_WW][23] = 52,
 	[2][0][RTW89_WW][25] = 52,
 	[2][0][RTW89_WW][27] = 52,
 	[2][0][RTW89_WW][29] = 52,
-	[2][0][RTW89_WW][31] = 48,
-	[2][0][RTW89_WW][33] = 48,
-	[2][0][RTW89_WW][35] = 48,
-	[2][0][RTW89_WW][37] = 62,
+	[2][0][RTW89_WW][31] = 52,
+	[2][0][RTW89_WW][33] = 52,
+	[2][0][RTW89_WW][35] = 52,
+	[2][0][RTW89_WW][37] = 52,
 	[2][0][RTW89_WW][38] = 28,
 	[2][0][RTW89_WW][40] = 28,
 	[2][0][RTW89_WW][42] = 28,
 	[2][0][RTW89_WW][44] = 28,
 	[2][0][RTW89_WW][46] = 28,
-	[2][0][RTW89_WW][48] = 48,
-	[2][0][RTW89_WW][50] = 48,
-	[2][0][RTW89_WW][52] = 48,
-	[2][1][RTW89_WW][0] = 20,
-	[2][1][RTW89_WW][2] = 18,
-	[2][1][RTW89_WW][4] = 22,
-	[2][1][RTW89_WW][6] = 22,
-	[2][1][RTW89_WW][8] = 32,
-	[2][1][RTW89_WW][10] = 32,
-	[2][1][RTW89_WW][12] = 36,
-	[2][1][RTW89_WW][14] = 36,
-	[2][1][RTW89_WW][15] = 36,
-	[2][1][RTW89_WW][17] = 36,
-	[2][1][RTW89_WW][19] = 36,
-	[2][1][RTW89_WW][21] = 36,
-	[2][1][RTW89_WW][23] = 36,
+	[2][0][RTW89_WW][48] = 64,
+	[2][0][RTW89_WW][50] = 64,
+	[2][0][RTW89_WW][52] = 64,
+	[2][1][RTW89_WW][0] = 26,
+	[2][1][RTW89_WW][2] = 26,
+	[2][1][RTW89_WW][4] = 26,
+	[2][1][RTW89_WW][6] = 20,
+	[2][1][RTW89_WW][8] = 28,
+	[2][1][RTW89_WW][10] = 28,
+	[2][1][RTW89_WW][12] = 28,
+	[2][1][RTW89_WW][14] = 28,
+	[2][1][RTW89_WW][15] = 40,
+	[2][1][RTW89_WW][17] = 40,
+	[2][1][RTW89_WW][19] = 40,
+	[2][1][RTW89_WW][21] = 40,
+	[2][1][RTW89_WW][23] = 40,
 	[2][1][RTW89_WW][25] = 40,
 	[2][1][RTW89_WW][27] = 40,
 	[2][1][RTW89_WW][29] = 40,
-	[2][1][RTW89_WW][31] = 36,
-	[2][1][RTW89_WW][33] = 36,
-	[2][1][RTW89_WW][35] = 36,
+	[2][1][RTW89_WW][31] = 40,
+	[2][1][RTW89_WW][33] = 40,
+	[2][1][RTW89_WW][35] = 40,
 	[2][1][RTW89_WW][37] = 42,
 	[2][1][RTW89_WW][38] = 16,
 	[2][1][RTW89_WW][40] = 16,
 	[2][1][RTW89_WW][42] = 16,
 	[2][1][RTW89_WW][44] = 16,
 	[2][1][RTW89_WW][46] = 16,
-	[2][1][RTW89_WW][48] = 36,
-	[2][1][RTW89_WW][50] = 36,
-	[2][1][RTW89_WW][52] = 36,
-	[0][0][RTW89_FCC][0] = 44,
+	[2][1][RTW89_WW][48] = 40,
+	[2][1][RTW89_WW][50] = 40,
+	[2][1][RTW89_WW][52] = 40,
+	[0][0][RTW89_FCC][0] = 50,
 	[0][0][RTW89_ETSI][0] = 30,
 	[0][0][RTW89_MKK][0] = 36,
-	[0][0][RTW89_IC][0] = 24,
-	[0][0][RTW89_ACMA][0] = 24,
-	[0][0][RTW89_FCC][2] = 44,
+	[0][0][RTW89_IC][0] = 32,
+	[0][0][RTW89_KCC][0] = 42,
+	[0][0][RTW89_ACMA][0] = 30,
+	[0][0][RTW89_CN][0] = 16,
+	[0][0][RTW89_UK][0] = 30,
+	[0][0][RTW89_FCC][2] = 50,
 	[0][0][RTW89_ETSI][2] = 30,
 	[0][0][RTW89_MKK][2] = 36,
-	[0][0][RTW89_IC][2] = 24,
-	[0][0][RTW89_ACMA][2] = 24,
-	[0][0][RTW89_FCC][4] = 44,
+	[0][0][RTW89_IC][2] = 32,
+	[0][0][RTW89_KCC][2] = 42,
+	[0][0][RTW89_ACMA][2] = 30,
+	[0][0][RTW89_CN][2] = 16,
+	[0][0][RTW89_UK][2] = 30,
+	[0][0][RTW89_FCC][4] = 50,
 	[0][0][RTW89_ETSI][4] = 30,
 	[0][0][RTW89_MKK][4] = 22,
-	[0][0][RTW89_IC][4] = 24,
-	[0][0][RTW89_ACMA][4] = 24,
-	[0][0][RTW89_FCC][6] = 44,
+	[0][0][RTW89_IC][4] = 32,
+	[0][0][RTW89_KCC][4] = 42,
+	[0][0][RTW89_ACMA][4] = 30,
+	[0][0][RTW89_CN][4] = 16,
+	[0][0][RTW89_UK][4] = 30,
+	[0][0][RTW89_FCC][6] = 50,
 	[0][0][RTW89_ETSI][6] = 30,
 	[0][0][RTW89_MKK][6] = 22,
-	[0][0][RTW89_IC][6] = 24,
-	[0][0][RTW89_ACMA][6] = 24,
-	[0][0][RTW89_FCC][8] = 44,
+	[0][0][RTW89_IC][6] = 32,
+	[0][0][RTW89_KCC][6] = 10,
+	[0][0][RTW89_ACMA][6] = 30,
+	[0][0][RTW89_CN][6] = 16,
+	[0][0][RTW89_UK][6] = 30,
+	[0][0][RTW89_FCC][8] = 52,
 	[0][0][RTW89_ETSI][8] = 28,
 	[0][0][RTW89_MKK][8] = 18,
 	[0][0][RTW89_IC][8] = 52,
-	[0][0][RTW89_ACMA][8] = 24,
-	[0][0][RTW89_FCC][10] = 44,
+	[0][0][RTW89_KCC][8] = 44,
+	[0][0][RTW89_ACMA][8] = 28,
+	[0][0][RTW89_CN][8] = 16,
+	[0][0][RTW89_UK][8] = 28,
+	[0][0][RTW89_FCC][10] = 52,
 	[0][0][RTW89_ETSI][10] = 28,
 	[0][0][RTW89_MKK][10] = 18,
 	[0][0][RTW89_IC][10] = 52,
-	[0][0][RTW89_ACMA][10] = 24,
-	[0][0][RTW89_FCC][12] = 44,
+	[0][0][RTW89_KCC][10] = 44,
+	[0][0][RTW89_ACMA][10] = 28,
+	[0][0][RTW89_CN][10] = 16,
+	[0][0][RTW89_UK][10] = 28,
+	[0][0][RTW89_FCC][12] = 52,
 	[0][0][RTW89_ETSI][12] = 28,
 	[0][0][RTW89_MKK][12] = 34,
 	[0][0][RTW89_IC][12] = 52,
-	[0][0][RTW89_ACMA][12] = 24,
-	[0][0][RTW89_FCC][14] = 44,
+	[0][0][RTW89_KCC][12] = 40,
+	[0][0][RTW89_ACMA][12] = 28,
+	[0][0][RTW89_CN][12] = 16,
+	[0][0][RTW89_UK][12] = 28,
+	[0][0][RTW89_FCC][14] = 52,
 	[0][0][RTW89_ETSI][14] = 28,
 	[0][0][RTW89_MKK][14] = 34,
 	[0][0][RTW89_IC][14] = 52,
-	[0][0][RTW89_ACMA][14] = 24,
-	[0][0][RTW89_FCC][15] = 44,
+	[0][0][RTW89_KCC][14] = 40,
+	[0][0][RTW89_ACMA][14] = 28,
+	[0][0][RTW89_CN][14] = 16,
+	[0][0][RTW89_UK][14] = 28,
+	[0][0][RTW89_FCC][15] = 52,
 	[0][0][RTW89_ETSI][15] = 30,
 	[0][0][RTW89_MKK][15] = 56,
 	[0][0][RTW89_IC][15] = 52,
-	[0][0][RTW89_ACMA][15] = 24,
-	[0][0][RTW89_FCC][17] = 44,
+	[0][0][RTW89_KCC][15] = 42,
+	[0][0][RTW89_ACMA][15] = 30,
+	[0][0][RTW89_CN][15] = 127,
+	[0][0][RTW89_UK][15] = 30,
+	[0][0][RTW89_FCC][17] = 52,
 	[0][0][RTW89_ETSI][17] = 30,
 	[0][0][RTW89_MKK][17] = 58,
 	[0][0][RTW89_IC][17] = 52,
-	[0][0][RTW89_ACMA][17] = 24,
-	[0][0][RTW89_FCC][19] = 44,
+	[0][0][RTW89_KCC][17] = 42,
+	[0][0][RTW89_ACMA][17] = 30,
+	[0][0][RTW89_CN][17] = 127,
+	[0][0][RTW89_UK][17] = 30,
+	[0][0][RTW89_FCC][19] = 52,
 	[0][0][RTW89_ETSI][19] = 30,
 	[0][0][RTW89_MKK][19] = 58,
 	[0][0][RTW89_IC][19] = 52,
-	[0][0][RTW89_ACMA][19] = 24,
-	[0][0][RTW89_FCC][21] = 44,
+	[0][0][RTW89_KCC][19] = 42,
+	[0][0][RTW89_ACMA][19] = 30,
+	[0][0][RTW89_CN][19] = 127,
+	[0][0][RTW89_UK][19] = 30,
+	[0][0][RTW89_FCC][21] = 52,
 	[0][0][RTW89_ETSI][21] = 30,
 	[0][0][RTW89_MKK][21] = 58,
 	[0][0][RTW89_IC][21] = 52,
-	[0][0][RTW89_ACMA][21] = 24,
-	[0][0][RTW89_FCC][23] = 44,
+	[0][0][RTW89_KCC][21] = 42,
+	[0][0][RTW89_ACMA][21] = 30,
+	[0][0][RTW89_CN][21] = 127,
+	[0][0][RTW89_UK][21] = 30,
+	[0][0][RTW89_FCC][23] = 52,
 	[0][0][RTW89_ETSI][23] = 30,
 	[0][0][RTW89_MKK][23] = 58,
 	[0][0][RTW89_IC][23] = 52,
-	[0][0][RTW89_ACMA][23] = 24,
-	[0][0][RTW89_FCC][25] = 44,
+	[0][0][RTW89_KCC][23] = 42,
+	[0][0][RTW89_ACMA][23] = 30,
+	[0][0][RTW89_CN][23] = 127,
+	[0][0][RTW89_UK][23] = 30,
+	[0][0][RTW89_FCC][25] = 52,
 	[0][0][RTW89_ETSI][25] = 30,
 	[0][0][RTW89_MKK][25] = 58,
 	[0][0][RTW89_IC][25] = 127,
+	[0][0][RTW89_KCC][25] = 42,
 	[0][0][RTW89_ACMA][25] = 127,
-	[0][0][RTW89_FCC][27] = 44,
+	[0][0][RTW89_CN][25] = 127,
+	[0][0][RTW89_UK][25] = 30,
+	[0][0][RTW89_FCC][27] = 52,
 	[0][0][RTW89_ETSI][27] = 30,
 	[0][0][RTW89_MKK][27] = 58,
 	[0][0][RTW89_IC][27] = 127,
+	[0][0][RTW89_KCC][27] = 42,
 	[0][0][RTW89_ACMA][27] = 127,
-	[0][0][RTW89_FCC][29] = 44,
+	[0][0][RTW89_CN][27] = 127,
+	[0][0][RTW89_UK][27] = 30,
+	[0][0][RTW89_FCC][29] = 52,
 	[0][0][RTW89_ETSI][29] = 30,
 	[0][0][RTW89_MKK][29] = 58,
 	[0][0][RTW89_IC][29] = 127,
+	[0][0][RTW89_KCC][29] = 42,
 	[0][0][RTW89_ACMA][29] = 127,
-	[0][0][RTW89_FCC][31] = 44,
+	[0][0][RTW89_CN][29] = 127,
+	[0][0][RTW89_UK][29] = 30,
+	[0][0][RTW89_FCC][31] = 52,
 	[0][0][RTW89_ETSI][31] = 30,
 	[0][0][RTW89_MKK][31] = 58,
-	[0][0][RTW89_IC][31] = 52,
-	[0][0][RTW89_ACMA][31] = 24,
+	[0][0][RTW89_IC][31] = 44,
+	[0][0][RTW89_KCC][31] = 42,
+	[0][0][RTW89_ACMA][31] = 30,
+	[0][0][RTW89_CN][31] = 127,
+	[0][0][RTW89_UK][31] = 30,
 	[0][0][RTW89_FCC][33] = 44,
 	[0][0][RTW89_ETSI][33] = 30,
 	[0][0][RTW89_MKK][33] = 58,
-	[0][0][RTW89_IC][33] = 52,
-	[0][0][RTW89_ACMA][33] = 24,
+	[0][0][RTW89_IC][33] = 44,
+	[0][0][RTW89_KCC][33] = 42,
+	[0][0][RTW89_ACMA][33] = 30,
+	[0][0][RTW89_CN][33] = 127,
+	[0][0][RTW89_UK][33] = 30,
 	[0][0][RTW89_FCC][35] = 44,
 	[0][0][RTW89_ETSI][35] = 30,
 	[0][0][RTW89_MKK][35] = 58,
-	[0][0][RTW89_IC][35] = 52,
-	[0][0][RTW89_ACMA][35] = 24,
-	[0][0][RTW89_FCC][37] = 44,
+	[0][0][RTW89_IC][35] = 44,
+	[0][0][RTW89_KCC][35] = 42,
+	[0][0][RTW89_ACMA][35] = 30,
+	[0][0][RTW89_CN][35] = 127,
+	[0][0][RTW89_UK][35] = 30,
+	[0][0][RTW89_FCC][37] = 52,
 	[0][0][RTW89_ETSI][37] = 127,
 	[0][0][RTW89_MKK][37] = 58,
 	[0][0][RTW89_IC][37] = 52,
+	[0][0][RTW89_KCC][37] = 42,
 	[0][0][RTW89_ACMA][37] = 52,
-	[0][0][RTW89_FCC][38] = 76,
+	[0][0][RTW89_CN][37] = 127,
+	[0][0][RTW89_UK][37] = 30,
+	[0][0][RTW89_FCC][38] = 64,
 	[0][0][RTW89_ETSI][38] = 28,
 	[0][0][RTW89_MKK][38] = 127,
-	[0][0][RTW89_IC][38] = 84,
-	[0][0][RTW89_ACMA][38] = 84,
-	[0][0][RTW89_FCC][40] = 76,
+	[0][0][RTW89_IC][38] = 64,
+	[0][0][RTW89_KCC][38] = 42,
+	[0][0][RTW89_ACMA][38] = 64,
+	[0][0][RTW89_CN][38] = 54,
+	[0][0][RTW89_UK][38] = 30,
+	[0][0][RTW89_FCC][40] = 64,
 	[0][0][RTW89_ETSI][40] = 28,
 	[0][0][RTW89_MKK][40] = 127,
-	[0][0][RTW89_IC][40] = 84,
-	[0][0][RTW89_ACMA][40] = 84,
-	[0][0][RTW89_FCC][42] = 76,
+	[0][0][RTW89_IC][40] = 64,
+	[0][0][RTW89_KCC][40] = 42,
+	[0][0][RTW89_ACMA][40] = 64,
+	[0][0][RTW89_CN][40] = 54,
+	[0][0][RTW89_UK][40] = 30,
+	[0][0][RTW89_FCC][42] = 60,
 	[0][0][RTW89_ETSI][42] = 28,
 	[0][0][RTW89_MKK][42] = 127,
-	[0][0][RTW89_IC][42] = 84,
-	[0][0][RTW89_ACMA][42] = 84,
-	[0][0][RTW89_FCC][44] = 76,
+	[0][0][RTW89_IC][42] = 60,
+	[0][0][RTW89_KCC][42] = 42,
+	[0][0][RTW89_ACMA][42] = 60,
+	[0][0][RTW89_CN][42] = 54,
+	[0][0][RTW89_UK][42] = 30,
+	[0][0][RTW89_FCC][44] = 60,
 	[0][0][RTW89_ETSI][44] = 28,
 	[0][0][RTW89_MKK][44] = 127,
-	[0][0][RTW89_IC][44] = 84,
-	[0][0][RTW89_ACMA][44] = 84,
-	[0][0][RTW89_FCC][46] = 76,
+	[0][0][RTW89_IC][44] = 60,
+	[0][0][RTW89_KCC][44] = 42,
+	[0][0][RTW89_ACMA][44] = 60,
+	[0][0][RTW89_CN][44] = 54,
+	[0][0][RTW89_UK][44] = 30,
+	[0][0][RTW89_FCC][46] = 60,
 	[0][0][RTW89_ETSI][46] = 28,
 	[0][0][RTW89_MKK][46] = 127,
-	[0][0][RTW89_IC][46] = 84,
-	[0][0][RTW89_ACMA][46] = 84,
-	[0][0][RTW89_FCC][48] = 24,
+	[0][0][RTW89_IC][46] = 60,
+	[0][0][RTW89_KCC][46] = 42,
+	[0][0][RTW89_ACMA][46] = 60,
+	[0][0][RTW89_CN][46] = 54,
+	[0][0][RTW89_UK][46] = 30,
+	[0][0][RTW89_FCC][48] = 46,
 	[0][0][RTW89_ETSI][48] = 127,
 	[0][0][RTW89_MKK][48] = 127,
 	[0][0][RTW89_IC][48] = 127,
+	[0][0][RTW89_KCC][48] = 127,
 	[0][0][RTW89_ACMA][48] = 127,
-	[0][0][RTW89_FCC][50] = 24,
+	[0][0][RTW89_CN][48] = 127,
+	[0][0][RTW89_UK][48] = 127,
+	[0][0][RTW89_FCC][50] = 44,
 	[0][0][RTW89_ETSI][50] = 127,
 	[0][0][RTW89_MKK][50] = 127,
 	[0][0][RTW89_IC][50] = 127,
+	[0][0][RTW89_KCC][50] = 127,
 	[0][0][RTW89_ACMA][50] = 127,
-	[0][0][RTW89_FCC][52] = 24,
+	[0][0][RTW89_CN][50] = 127,
+	[0][0][RTW89_UK][50] = 127,
+	[0][0][RTW89_FCC][52] = 34,
 	[0][0][RTW89_ETSI][52] = 127,
 	[0][0][RTW89_MKK][52] = 127,
 	[0][0][RTW89_IC][52] = 127,
+	[0][0][RTW89_KCC][52] = 127,
 	[0][0][RTW89_ACMA][52] = 127,
-	[0][1][RTW89_FCC][0] = 26,
+	[0][0][RTW89_CN][52] = 127,
+	[0][0][RTW89_UK][52] = 127,
+	[0][1][RTW89_FCC][0] = 30,
 	[0][1][RTW89_ETSI][0] = 18,
 	[0][1][RTW89_MKK][0] = 20,
-	[0][1][RTW89_IC][0] = 0,
-	[0][1][RTW89_ACMA][0] = 12,
-	[0][1][RTW89_FCC][2] = 30,
+	[0][1][RTW89_IC][0] = 8,
+	[0][1][RTW89_KCC][0] = 26,
+	[0][1][RTW89_ACMA][0] = 18,
+	[0][1][RTW89_CN][0] = 4,
+	[0][1][RTW89_UK][0] = 18,
+	[0][1][RTW89_FCC][2] = 32,
 	[0][1][RTW89_ETSI][2] = 18,
 	[0][1][RTW89_MKK][2] = 20,
-	[0][1][RTW89_IC][2] = 4,
-	[0][1][RTW89_ACMA][2] = 12,
-	[0][1][RTW89_FCC][4] = 26,
+	[0][1][RTW89_IC][2] = 8,
+	[0][1][RTW89_KCC][2] = 26,
+	[0][1][RTW89_ACMA][2] = 18,
+	[0][1][RTW89_CN][2] = 4,
+	[0][1][RTW89_UK][2] = 18,
+	[0][1][RTW89_FCC][4] = 30,
 	[0][1][RTW89_ETSI][4] = 18,
 	[0][1][RTW89_MKK][4] = 8,
-	[0][1][RTW89_IC][4] = 0,
-	[0][1][RTW89_ACMA][4] = 12,
-	[0][1][RTW89_FCC][6] = 26,
+	[0][1][RTW89_IC][4] = 8,
+	[0][1][RTW89_KCC][4] = 26,
+	[0][1][RTW89_ACMA][4] = 18,
+	[0][1][RTW89_CN][4] = 4,
+	[0][1][RTW89_UK][4] = 18,
+	[0][1][RTW89_FCC][6] = 30,
 	[0][1][RTW89_ETSI][6] = 18,
 	[0][1][RTW89_MKK][6] = 8,
-	[0][1][RTW89_IC][6] = 0,
-	[0][1][RTW89_ACMA][6] = 12,
-	[0][1][RTW89_FCC][8] = 26,
+	[0][1][RTW89_IC][6] = 8,
+	[0][1][RTW89_KCC][6] = 0,
+	[0][1][RTW89_ACMA][6] = 18,
+	[0][1][RTW89_CN][6] = 4,
+	[0][1][RTW89_UK][6] = 18,
+	[0][1][RTW89_FCC][8] = 30,
 	[0][1][RTW89_ETSI][8] = 16,
 	[0][1][RTW89_MKK][8] = 20,
-	[0][1][RTW89_IC][8] = 34,
-	[0][1][RTW89_ACMA][8] = 12,
-	[0][1][RTW89_FCC][10] = 26,
+	[0][1][RTW89_IC][8] = 30,
+	[0][1][RTW89_KCC][8] = 28,
+	[0][1][RTW89_ACMA][8] = 16,
+	[0][1][RTW89_CN][8] = 4,
+	[0][1][RTW89_UK][8] = 16,
+	[0][1][RTW89_FCC][10] = 30,
 	[0][1][RTW89_ETSI][10] = 16,
 	[0][1][RTW89_MKK][10] = 20,
-	[0][1][RTW89_IC][10] = 34,
-	[0][1][RTW89_ACMA][10] = 12,
+	[0][1][RTW89_IC][10] = 30,
+	[0][1][RTW89_KCC][10] = 28,
+	[0][1][RTW89_ACMA][10] = 16,
+	[0][1][RTW89_CN][10] = 4,
+	[0][1][RTW89_UK][10] = 16,
 	[0][1][RTW89_FCC][12] = 30,
 	[0][1][RTW89_ETSI][12] = 16,
 	[0][1][RTW89_MKK][12] = 34,
-	[0][1][RTW89_IC][12] = 38,
-	[0][1][RTW89_ACMA][12] = 12,
-	[0][1][RTW89_FCC][14] = 26,
+	[0][1][RTW89_IC][12] = 30,
+	[0][1][RTW89_KCC][12] = 28,
+	[0][1][RTW89_ACMA][12] = 16,
+	[0][1][RTW89_CN][12] = 4,
+	[0][1][RTW89_UK][12] = 16,
+	[0][1][RTW89_FCC][14] = 30,
 	[0][1][RTW89_ETSI][14] = 16,
 	[0][1][RTW89_MKK][14] = 34,
-	[0][1][RTW89_IC][14] = 34,
-	[0][1][RTW89_ACMA][14] = 12,
-	[0][1][RTW89_FCC][15] = 26,
+	[0][1][RTW89_IC][14] = 30,
+	[0][1][RTW89_KCC][14] = 28,
+	[0][1][RTW89_ACMA][14] = 16,
+	[0][1][RTW89_CN][14] = 4,
+	[0][1][RTW89_UK][14] = 16,
+	[0][1][RTW89_FCC][15] = 32,
 	[0][1][RTW89_ETSI][15] = 18,
 	[0][1][RTW89_MKK][15] = 44,
-	[0][1][RTW89_IC][15] = 34,
-	[0][1][RTW89_ACMA][15] = 12,
-	[0][1][RTW89_FCC][17] = 26,
+	[0][1][RTW89_IC][15] = 32,
+	[0][1][RTW89_KCC][15] = 28,
+	[0][1][RTW89_ACMA][15] = 18,
+	[0][1][RTW89_CN][15] = 127,
+	[0][1][RTW89_UK][15] = 18,
+	[0][1][RTW89_FCC][17] = 32,
 	[0][1][RTW89_ETSI][17] = 18,
 	[0][1][RTW89_MKK][17] = 44,
-	[0][1][RTW89_IC][17] = 34,
-	[0][1][RTW89_ACMA][17] = 12,
-	[0][1][RTW89_FCC][19] = 30,
+	[0][1][RTW89_IC][17] = 32,
+	[0][1][RTW89_KCC][17] = 28,
+	[0][1][RTW89_ACMA][17] = 18,
+	[0][1][RTW89_CN][17] = 127,
+	[0][1][RTW89_UK][17] = 18,
+	[0][1][RTW89_FCC][19] = 32,
 	[0][1][RTW89_ETSI][19] = 18,
 	[0][1][RTW89_MKK][19] = 44,
-	[0][1][RTW89_IC][19] = 38,
-	[0][1][RTW89_ACMA][19] = 12,
-	[0][1][RTW89_FCC][21] = 30,
+	[0][1][RTW89_IC][19] = 32,
+	[0][1][RTW89_KCC][19] = 28,
+	[0][1][RTW89_ACMA][19] = 18,
+	[0][1][RTW89_CN][19] = 127,
+	[0][1][RTW89_UK][19] = 18,
+	[0][1][RTW89_FCC][21] = 32,
 	[0][1][RTW89_ETSI][21] = 18,
 	[0][1][RTW89_MKK][21] = 44,
-	[0][1][RTW89_IC][21] = 38,
-	[0][1][RTW89_ACMA][21] = 12,
-	[0][1][RTW89_FCC][23] = 30,
+	[0][1][RTW89_IC][21] = 32,
+	[0][1][RTW89_KCC][21] = 28,
+	[0][1][RTW89_ACMA][21] = 18,
+	[0][1][RTW89_CN][21] = 127,
+	[0][1][RTW89_UK][21] = 18,
+	[0][1][RTW89_FCC][23] = 32,
 	[0][1][RTW89_ETSI][23] = 18,
 	[0][1][RTW89_MKK][23] = 44,
-	[0][1][RTW89_IC][23] = 38,
-	[0][1][RTW89_ACMA][23] = 12,
-	[0][1][RTW89_FCC][25] = 30,
+	[0][1][RTW89_IC][23] = 32,
+	[0][1][RTW89_KCC][23] = 28,
+	[0][1][RTW89_ACMA][23] = 18,
+	[0][1][RTW89_CN][23] = 127,
+	[0][1][RTW89_UK][23] = 18,
+	[0][1][RTW89_FCC][25] = 32,
 	[0][1][RTW89_ETSI][25] = 18,
 	[0][1][RTW89_MKK][25] = 44,
 	[0][1][RTW89_IC][25] = 127,
+	[0][1][RTW89_KCC][25] = 28,
 	[0][1][RTW89_ACMA][25] = 127,
-	[0][1][RTW89_FCC][27] = 30,
+	[0][1][RTW89_CN][25] = 127,
+	[0][1][RTW89_UK][25] = 18,
+	[0][1][RTW89_FCC][27] = 32,
 	[0][1][RTW89_ETSI][27] = 16,
 	[0][1][RTW89_MKK][27] = 44,
 	[0][1][RTW89_IC][27] = 127,
+	[0][1][RTW89_KCC][27] = 28,
 	[0][1][RTW89_ACMA][27] = 127,
-	[0][1][RTW89_FCC][29] = 30,
+	[0][1][RTW89_CN][27] = 127,
+	[0][1][RTW89_UK][27] = 16,
+	[0][1][RTW89_FCC][29] = 32,
 	[0][1][RTW89_ETSI][29] = 16,
 	[0][1][RTW89_MKK][29] = 44,
 	[0][1][RTW89_IC][29] = 127,
+	[0][1][RTW89_KCC][29] = 28,
 	[0][1][RTW89_ACMA][29] = 127,
-	[0][1][RTW89_FCC][31] = 30,
+	[0][1][RTW89_CN][29] = 127,
+	[0][1][RTW89_UK][29] = 16,
+	[0][1][RTW89_FCC][31] = 32,
 	[0][1][RTW89_ETSI][31] = 16,
 	[0][1][RTW89_MKK][31] = 44,
-	[0][1][RTW89_IC][31] = 34,
-	[0][1][RTW89_ACMA][31] = 12,
-	[0][1][RTW89_FCC][33] = 26,
+	[0][1][RTW89_IC][31] = 30,
+	[0][1][RTW89_KCC][31] = 28,
+	[0][1][RTW89_ACMA][31] = 16,
+	[0][1][RTW89_CN][31] = 127,
+	[0][1][RTW89_UK][31] = 16,
+	[0][1][RTW89_FCC][33] = 30,
 	[0][1][RTW89_ETSI][33] = 16,
 	[0][1][RTW89_MKK][33] = 44,
-	[0][1][RTW89_IC][33] = 34,
-	[0][1][RTW89_ACMA][33] = 12,
-	[0][1][RTW89_FCC][35] = 26,
+	[0][1][RTW89_IC][33] = 30,
+	[0][1][RTW89_KCC][33] = 28,
+	[0][1][RTW89_ACMA][33] = 16,
+	[0][1][RTW89_CN][33] = 127,
+	[0][1][RTW89_UK][33] = 16,
+	[0][1][RTW89_FCC][35] = 30,
 	[0][1][RTW89_ETSI][35] = 16,
 	[0][1][RTW89_MKK][35] = 44,
-	[0][1][RTW89_IC][35] = 34,
-	[0][1][RTW89_ACMA][35] = 12,
-	[0][1][RTW89_FCC][37] = 30,
+	[0][1][RTW89_IC][35] = 30,
+	[0][1][RTW89_KCC][35] = 28,
+	[0][1][RTW89_ACMA][35] = 16,
+	[0][1][RTW89_CN][35] = 127,
+	[0][1][RTW89_UK][35] = 16,
+	[0][1][RTW89_FCC][37] = 34,
 	[0][1][RTW89_ETSI][37] = 127,
 	[0][1][RTW89_MKK][37] = 44,
-	[0][1][RTW89_IC][37] = 38,
-	[0][1][RTW89_ACMA][37] = 38,
-	[0][1][RTW89_FCC][38] = 74,
+	[0][1][RTW89_IC][37] = 34,
+	[0][1][RTW89_KCC][37] = 28,
+	[0][1][RTW89_ACMA][37] = 34,
+	[0][1][RTW89_CN][37] = 127,
+	[0][1][RTW89_UK][37] = 18,
+	[0][1][RTW89_FCC][38] = 62,
 	[0][1][RTW89_ETSI][38] = 16,
 	[0][1][RTW89_MKK][38] = 127,
-	[0][1][RTW89_IC][38] = 82,
-	[0][1][RTW89_ACMA][38] = 84,
-	[0][1][RTW89_FCC][40] = 74,
+	[0][1][RTW89_IC][38] = 62,
+	[0][1][RTW89_KCC][38] = 28,
+	[0][1][RTW89_ACMA][38] = 62,
+	[0][1][RTW89_CN][38] = 42,
+	[0][1][RTW89_UK][38] = 18,
+	[0][1][RTW89_FCC][40] = 62,
 	[0][1][RTW89_ETSI][40] = 16,
 	[0][1][RTW89_MKK][40] = 127,
-	[0][1][RTW89_IC][40] = 82,
-	[0][1][RTW89_ACMA][40] = 84,
-	[0][1][RTW89_FCC][42] = 74,
+	[0][1][RTW89_IC][40] = 62,
+	[0][1][RTW89_KCC][40] = 28,
+	[0][1][RTW89_ACMA][40] = 62,
+	[0][1][RTW89_CN][40] = 42,
+	[0][1][RTW89_UK][40] = 18,
+	[0][1][RTW89_FCC][42] = 58,
 	[0][1][RTW89_ETSI][42] = 16,
 	[0][1][RTW89_MKK][42] = 127,
-	[0][1][RTW89_IC][42] = 82,
-	[0][1][RTW89_ACMA][42] = 84,
-	[0][1][RTW89_FCC][44] = 74,
+	[0][1][RTW89_IC][42] = 58,
+	[0][1][RTW89_KCC][42] = 28,
+	[0][1][RTW89_ACMA][42] = 58,
+	[0][1][RTW89_CN][42] = 42,
+	[0][1][RTW89_UK][42] = 18,
+	[0][1][RTW89_FCC][44] = 56,
 	[0][1][RTW89_ETSI][44] = 16,
 	[0][1][RTW89_MKK][44] = 127,
-	[0][1][RTW89_IC][44] = 82,
-	[0][1][RTW89_ACMA][44] = 84,
-	[0][1][RTW89_FCC][46] = 74,
+	[0][1][RTW89_IC][44] = 56,
+	[0][1][RTW89_KCC][44] = 28,
+	[0][1][RTW89_ACMA][44] = 56,
+	[0][1][RTW89_CN][44] = 42,
+	[0][1][RTW89_UK][44] = 18,
+	[0][1][RTW89_FCC][46] = 56,
 	[0][1][RTW89_ETSI][46] = 16,
 	[0][1][RTW89_MKK][46] = 127,
-	[0][1][RTW89_IC][46] = 82,
-	[0][1][RTW89_ACMA][46] = 84,
-	[0][1][RTW89_FCC][48] = 12,
+	[0][1][RTW89_IC][46] = 56,
+	[0][1][RTW89_KCC][46] = 28,
+	[0][1][RTW89_ACMA][46] = 56,
+	[0][1][RTW89_CN][46] = 42,
+	[0][1][RTW89_UK][46] = 18,
+	[0][1][RTW89_FCC][48] = 20,
 	[0][1][RTW89_ETSI][48] = 127,
 	[0][1][RTW89_MKK][48] = 127,
 	[0][1][RTW89_IC][48] = 127,
+	[0][1][RTW89_KCC][48] = 127,
 	[0][1][RTW89_ACMA][48] = 127,
-	[0][1][RTW89_FCC][50] = 12,
+	[0][1][RTW89_CN][48] = 127,
+	[0][1][RTW89_UK][48] = 127,
+	[0][1][RTW89_FCC][50] = 20,
 	[0][1][RTW89_ETSI][50] = 127,
 	[0][1][RTW89_MKK][50] = 127,
 	[0][1][RTW89_IC][50] = 127,
+	[0][1][RTW89_KCC][50] = 127,
 	[0][1][RTW89_ACMA][50] = 127,
-	[0][1][RTW89_FCC][52] = 12,
+	[0][1][RTW89_CN][50] = 127,
+	[0][1][RTW89_UK][50] = 127,
+	[0][1][RTW89_FCC][52] = 8,
 	[0][1][RTW89_ETSI][52] = 127,
 	[0][1][RTW89_MKK][52] = 127,
 	[0][1][RTW89_IC][52] = 127,
+	[0][1][RTW89_KCC][52] = 127,
 	[0][1][RTW89_ACMA][52] = 127,
-	[1][0][RTW89_FCC][0] = 54,
+	[0][1][RTW89_CN][52] = 127,
+	[0][1][RTW89_UK][52] = 127,
+	[1][0][RTW89_FCC][0] = 62,
 	[1][0][RTW89_ETSI][0] = 40,
 	[1][0][RTW89_MKK][0] = 48,
-	[1][0][RTW89_IC][0] = 36,
-	[1][0][RTW89_ACMA][0] = 34,
-	[1][0][RTW89_FCC][2] = 54,
+	[1][0][RTW89_IC][0] = 42,
+	[1][0][RTW89_KCC][0] = 50,
+	[1][0][RTW89_ACMA][0] = 40,
+	[1][0][RTW89_CN][0] = 26,
+	[1][0][RTW89_UK][0] = 40,
+	[1][0][RTW89_FCC][2] = 62,
 	[1][0][RTW89_ETSI][2] = 40,
 	[1][0][RTW89_MKK][2] = 48,
-	[1][0][RTW89_IC][2] = 36,
-	[1][0][RTW89_ACMA][2] = 34,
-	[1][0][RTW89_FCC][4] = 54,
+	[1][0][RTW89_IC][2] = 42,
+	[1][0][RTW89_KCC][2] = 50,
+	[1][0][RTW89_ACMA][2] = 40,
+	[1][0][RTW89_CN][2] = 26,
+	[1][0][RTW89_UK][2] = 40,
+	[1][0][RTW89_FCC][4] = 64,
 	[1][0][RTW89_ETSI][4] = 40,
 	[1][0][RTW89_MKK][4] = 40,
-	[1][0][RTW89_IC][4] = 36,
-	[1][0][RTW89_ACMA][4] = 34,
-	[1][0][RTW89_FCC][6] = 54,
+	[1][0][RTW89_IC][4] = 42,
+	[1][0][RTW89_KCC][4] = 50,
+	[1][0][RTW89_ACMA][4] = 40,
+	[1][0][RTW89_CN][4] = 26,
+	[1][0][RTW89_UK][4] = 40,
+	[1][0][RTW89_FCC][6] = 64,
 	[1][0][RTW89_ETSI][6] = 40,
 	[1][0][RTW89_MKK][6] = 40,
-	[1][0][RTW89_IC][6] = 36,
-	[1][0][RTW89_ACMA][6] = 34,
-	[1][0][RTW89_FCC][8] = 54,
+	[1][0][RTW89_IC][6] = 42,
+	[1][0][RTW89_KCC][6] = 24,
+	[1][0][RTW89_ACMA][6] = 40,
+	[1][0][RTW89_CN][6] = 26,
+	[1][0][RTW89_UK][6] = 40,
+	[1][0][RTW89_FCC][8] = 62,
 	[1][0][RTW89_ETSI][8] = 40,
 	[1][0][RTW89_MKK][8] = 34,
 	[1][0][RTW89_IC][8] = 62,
-	[1][0][RTW89_ACMA][8] = 34,
-	[1][0][RTW89_FCC][10] = 54,
+	[1][0][RTW89_KCC][8] = 52,
+	[1][0][RTW89_ACMA][8] = 40,
+	[1][0][RTW89_CN][8] = 26,
+	[1][0][RTW89_UK][8] = 40,
+	[1][0][RTW89_FCC][10] = 62,
 	[1][0][RTW89_ETSI][10] = 40,
 	[1][0][RTW89_MKK][10] = 34,
 	[1][0][RTW89_IC][10] = 62,
-	[1][0][RTW89_ACMA][10] = 34,
-	[1][0][RTW89_FCC][12] = 56,
+	[1][0][RTW89_KCC][10] = 52,
+	[1][0][RTW89_ACMA][10] = 40,
+	[1][0][RTW89_CN][10] = 26,
+	[1][0][RTW89_UK][10] = 40,
+	[1][0][RTW89_FCC][12] = 62,
 	[1][0][RTW89_ETSI][12] = 40,
 	[1][0][RTW89_MKK][12] = 46,
-	[1][0][RTW89_IC][12] = 64,
-	[1][0][RTW89_ACMA][12] = 34,
-	[1][0][RTW89_FCC][14] = 54,
+	[1][0][RTW89_IC][12] = 62,
+	[1][0][RTW89_KCC][12] = 52,
+	[1][0][RTW89_ACMA][12] = 40,
+	[1][0][RTW89_CN][12] = 26,
+	[1][0][RTW89_UK][12] = 40,
+	[1][0][RTW89_FCC][14] = 62,
 	[1][0][RTW89_ETSI][14] = 40,
 	[1][0][RTW89_MKK][14] = 46,
 	[1][0][RTW89_IC][14] = 62,
-	[1][0][RTW89_ACMA][14] = 34,
-	[1][0][RTW89_FCC][15] = 54,
+	[1][0][RTW89_KCC][14] = 52,
+	[1][0][RTW89_ACMA][14] = 40,
+	[1][0][RTW89_CN][14] = 26,
+	[1][0][RTW89_UK][14] = 40,
+	[1][0][RTW89_FCC][15] = 62,
 	[1][0][RTW89_ETSI][15] = 40,
 	[1][0][RTW89_MKK][15] = 62,
 	[1][0][RTW89_IC][15] = 62,
-	[1][0][RTW89_ACMA][15] = 34,
-	[1][0][RTW89_FCC][17] = 54,
+	[1][0][RTW89_KCC][15] = 52,
+	[1][0][RTW89_ACMA][15] = 40,
+	[1][0][RTW89_CN][15] = 127,
+	[1][0][RTW89_UK][15] = 40,
+	[1][0][RTW89_FCC][17] = 62,
 	[1][0][RTW89_ETSI][17] = 40,
 	[1][0][RTW89_MKK][17] = 68,
 	[1][0][RTW89_IC][17] = 62,
-	[1][0][RTW89_ACMA][17] = 34,
-	[1][0][RTW89_FCC][19] = 54,
+	[1][0][RTW89_KCC][17] = 52,
+	[1][0][RTW89_ACMA][17] = 40,
+	[1][0][RTW89_CN][17] = 127,
+	[1][0][RTW89_UK][17] = 40,
+	[1][0][RTW89_FCC][19] = 64,
 	[1][0][RTW89_ETSI][19] = 40,
 	[1][0][RTW89_MKK][19] = 68,
-	[1][0][RTW89_IC][19] = 62,
-	[1][0][RTW89_ACMA][19] = 34,
-	[1][0][RTW89_FCC][21] = 54,
+	[1][0][RTW89_IC][19] = 64,
+	[1][0][RTW89_KCC][19] = 52,
+	[1][0][RTW89_ACMA][19] = 40,
+	[1][0][RTW89_CN][19] = 127,
+	[1][0][RTW89_UK][19] = 40,
+	[1][0][RTW89_FCC][21] = 64,
 	[1][0][RTW89_ETSI][21] = 40,
 	[1][0][RTW89_MKK][21] = 68,
-	[1][0][RTW89_IC][21] = 62,
-	[1][0][RTW89_ACMA][21] = 34,
-	[1][0][RTW89_FCC][23] = 54,
+	[1][0][RTW89_IC][21] = 64,
+	[1][0][RTW89_KCC][21] = 52,
+	[1][0][RTW89_ACMA][21] = 40,
+	[1][0][RTW89_CN][21] = 127,
+	[1][0][RTW89_UK][21] = 40,
+	[1][0][RTW89_FCC][23] = 64,
 	[1][0][RTW89_ETSI][23] = 40,
 	[1][0][RTW89_MKK][23] = 68,
-	[1][0][RTW89_IC][23] = 62,
-	[1][0][RTW89_ACMA][23] = 34,
-	[1][0][RTW89_FCC][25] = 54,
+	[1][0][RTW89_IC][23] = 64,
+	[1][0][RTW89_KCC][23] = 52,
+	[1][0][RTW89_ACMA][23] = 40,
+	[1][0][RTW89_CN][23] = 127,
+	[1][0][RTW89_UK][23] = 40,
+	[1][0][RTW89_FCC][25] = 64,
 	[1][0][RTW89_ETSI][25] = 40,
 	[1][0][RTW89_MKK][25] = 68,
 	[1][0][RTW89_IC][25] = 127,
+	[1][0][RTW89_KCC][25] = 52,
 	[1][0][RTW89_ACMA][25] = 127,
-	[1][0][RTW89_FCC][27] = 54,
+	[1][0][RTW89_CN][25] = 127,
+	[1][0][RTW89_UK][25] = 40,
+	[1][0][RTW89_FCC][27] = 64,
 	[1][0][RTW89_ETSI][27] = 42,
 	[1][0][RTW89_MKK][27] = 68,
 	[1][0][RTW89_IC][27] = 127,
+	[1][0][RTW89_KCC][27] = 52,
 	[1][0][RTW89_ACMA][27] = 127,
-	[1][0][RTW89_FCC][29] = 54,
+	[1][0][RTW89_CN][27] = 127,
+	[1][0][RTW89_UK][27] = 42,
+	[1][0][RTW89_FCC][29] = 64,
 	[1][0][RTW89_ETSI][29] = 42,
 	[1][0][RTW89_MKK][29] = 68,
 	[1][0][RTW89_IC][29] = 127,
+	[1][0][RTW89_KCC][29] = 52,
 	[1][0][RTW89_ACMA][29] = 127,
-	[1][0][RTW89_FCC][31] = 54,
+	[1][0][RTW89_CN][29] = 127,
+	[1][0][RTW89_UK][29] = 42,
+	[1][0][RTW89_FCC][31] = 64,
 	[1][0][RTW89_ETSI][31] = 42,
 	[1][0][RTW89_MKK][31] = 68,
-	[1][0][RTW89_IC][31] = 62,
-	[1][0][RTW89_ACMA][31] = 34,
-	[1][0][RTW89_FCC][33] = 54,
+	[1][0][RTW89_IC][31] = 56,
+	[1][0][RTW89_KCC][31] = 52,
+	[1][0][RTW89_ACMA][31] = 42,
+	[1][0][RTW89_CN][31] = 127,
+	[1][0][RTW89_UK][31] = 42,
+	[1][0][RTW89_FCC][33] = 56,
 	[1][0][RTW89_ETSI][33] = 42,
 	[1][0][RTW89_MKK][33] = 68,
-	[1][0][RTW89_IC][33] = 62,
-	[1][0][RTW89_ACMA][33] = 34,
-	[1][0][RTW89_FCC][35] = 54,
+	[1][0][RTW89_IC][33] = 56,
+	[1][0][RTW89_KCC][33] = 52,
+	[1][0][RTW89_ACMA][33] = 42,
+	[1][0][RTW89_CN][33] = 127,
+	[1][0][RTW89_UK][33] = 42,
+	[1][0][RTW89_FCC][35] = 56,
 	[1][0][RTW89_ETSI][35] = 42,
 	[1][0][RTW89_MKK][35] = 68,
-	[1][0][RTW89_IC][35] = 62,
-	[1][0][RTW89_ACMA][35] = 34,
-	[1][0][RTW89_FCC][37] = 56,
+	[1][0][RTW89_IC][35] = 56,
+	[1][0][RTW89_KCC][35] = 52,
+	[1][0][RTW89_ACMA][35] = 42,
+	[1][0][RTW89_CN][35] = 127,
+	[1][0][RTW89_UK][35] = 42,
+	[1][0][RTW89_FCC][37] = 66,
 	[1][0][RTW89_ETSI][37] = 127,
 	[1][0][RTW89_MKK][37] = 68,
-	[1][0][RTW89_IC][37] = 64,
-	[1][0][RTW89_ACMA][37] = 64,
+	[1][0][RTW89_IC][37] = 66,
+	[1][0][RTW89_KCC][37] = 52,
+	[1][0][RTW89_ACMA][37] = 66,
+	[1][0][RTW89_CN][37] = 127,
+	[1][0][RTW89_UK][37] = 42,
 	[1][0][RTW89_FCC][38] = 76,
 	[1][0][RTW89_ETSI][38] = 28,
 	[1][0][RTW89_MKK][38] = 127,
-	[1][0][RTW89_IC][38] = 84,
-	[1][0][RTW89_ACMA][38] = 84,
+	[1][0][RTW89_IC][38] = 76,
+	[1][0][RTW89_KCC][38] = 54,
+	[1][0][RTW89_ACMA][38] = 76,
+	[1][0][RTW89_CN][38] = 66,
+	[1][0][RTW89_UK][38] = 44,
 	[1][0][RTW89_FCC][40] = 76,
 	[1][0][RTW89_ETSI][40] = 28,
 	[1][0][RTW89_MKK][40] = 127,
-	[1][0][RTW89_IC][40] = 84,
-	[1][0][RTW89_ACMA][40] = 84,
-	[1][0][RTW89_FCC][42] = 76,
+	[1][0][RTW89_IC][40] = 76,
+	[1][0][RTW89_KCC][40] = 54,
+	[1][0][RTW89_ACMA][40] = 76,
+	[1][0][RTW89_CN][40] = 66,
+	[1][0][RTW89_UK][40] = 44,
+	[1][0][RTW89_FCC][42] = 68,
 	[1][0][RTW89_ETSI][42] = 28,
 	[1][0][RTW89_MKK][42] = 127,
-	[1][0][RTW89_IC][42] = 84,
-	[1][0][RTW89_ACMA][42] = 84,
-	[1][0][RTW89_FCC][44] = 76,
+	[1][0][RTW89_IC][42] = 68,
+	[1][0][RTW89_KCC][42] = 54,
+	[1][0][RTW89_ACMA][42] = 68,
+	[1][0][RTW89_CN][42] = 66,
+	[1][0][RTW89_UK][42] = 44,
+	[1][0][RTW89_FCC][44] = 70,
 	[1][0][RTW89_ETSI][44] = 28,
 	[1][0][RTW89_MKK][44] = 127,
-	[1][0][RTW89_IC][44] = 84,
-	[1][0][RTW89_ACMA][44] = 84,
-	[1][0][RTW89_FCC][46] = 76,
+	[1][0][RTW89_IC][44] = 70,
+	[1][0][RTW89_KCC][44] = 54,
+	[1][0][RTW89_ACMA][44] = 70,
+	[1][0][RTW89_CN][44] = 66,
+	[1][0][RTW89_UK][44] = 42,
+	[1][0][RTW89_FCC][46] = 70,
 	[1][0][RTW89_ETSI][46] = 28,
 	[1][0][RTW89_MKK][46] = 127,
-	[1][0][RTW89_IC][46] = 84,
-	[1][0][RTW89_ACMA][46] = 84,
-	[1][0][RTW89_FCC][48] = 36,
+	[1][0][RTW89_IC][46] = 70,
+	[1][0][RTW89_KCC][46] = 54,
+	[1][0][RTW89_ACMA][46] = 70,
+	[1][0][RTW89_CN][46] = 66,
+	[1][0][RTW89_UK][46] = 42,
+	[1][0][RTW89_FCC][48] = 56,
 	[1][0][RTW89_ETSI][48] = 127,
 	[1][0][RTW89_MKK][48] = 127,
 	[1][0][RTW89_IC][48] = 127,
+	[1][0][RTW89_KCC][48] = 127,
 	[1][0][RTW89_ACMA][48] = 127,
-	[1][0][RTW89_FCC][50] = 36,
+	[1][0][RTW89_CN][48] = 127,
+	[1][0][RTW89_UK][48] = 127,
+	[1][0][RTW89_FCC][50] = 58,
 	[1][0][RTW89_ETSI][50] = 127,
 	[1][0][RTW89_MKK][50] = 127,
 	[1][0][RTW89_IC][50] = 127,
+	[1][0][RTW89_KCC][50] = 127,
 	[1][0][RTW89_ACMA][50] = 127,
-	[1][0][RTW89_FCC][52] = 36,
+	[1][0][RTW89_CN][50] = 127,
+	[1][0][RTW89_UK][50] = 127,
+	[1][0][RTW89_FCC][52] = 56,
 	[1][0][RTW89_ETSI][52] = 127,
 	[1][0][RTW89_MKK][52] = 127,
 	[1][0][RTW89_IC][52] = 127,
+	[1][0][RTW89_KCC][52] = 127,
 	[1][0][RTW89_ACMA][52] = 127,
-	[1][1][RTW89_FCC][0] = 34,
+	[1][0][RTW89_CN][52] = 127,
+	[1][0][RTW89_UK][52] = 127,
+	[1][1][RTW89_FCC][0] = 44,
 	[1][1][RTW89_ETSI][0] = 30,
 	[1][1][RTW89_MKK][0] = 34,
-	[1][1][RTW89_IC][0] = 10,
-	[1][1][RTW89_ACMA][0] = 22,
-	[1][1][RTW89_FCC][2] = 36,
+	[1][1][RTW89_IC][0] = 20,
+	[1][1][RTW89_KCC][0] = 34,
+	[1][1][RTW89_ACMA][0] = 30,
+	[1][1][RTW89_CN][0] = 14,
+	[1][1][RTW89_UK][0] = 30,
+	[1][1][RTW89_FCC][2] = 44,
 	[1][1][RTW89_ETSI][2] = 30,
 	[1][1][RTW89_MKK][2] = 34,
-	[1][1][RTW89_IC][2] = 14,
-	[1][1][RTW89_ACMA][2] = 22,
-	[1][1][RTW89_FCC][4] = 34,
+	[1][1][RTW89_IC][2] = 18,
+	[1][1][RTW89_KCC][2] = 34,
+	[1][1][RTW89_ACMA][2] = 30,
+	[1][1][RTW89_CN][2] = 14,
+	[1][1][RTW89_UK][2] = 30,
+	[1][1][RTW89_FCC][4] = 46,
 	[1][1][RTW89_ETSI][4] = 30,
 	[1][1][RTW89_MKK][4] = 26,
-	[1][1][RTW89_IC][4] = 10,
-	[1][1][RTW89_ACMA][4] = 22,
-	[1][1][RTW89_FCC][6] = 34,
+	[1][1][RTW89_IC][4] = 20,
+	[1][1][RTW89_KCC][4] = 34,
+	[1][1][RTW89_ACMA][4] = 30,
+	[1][1][RTW89_CN][4] = 14,
+	[1][1][RTW89_UK][4] = 30,
+	[1][1][RTW89_FCC][6] = 46,
 	[1][1][RTW89_ETSI][6] = 30,
 	[1][1][RTW89_MKK][6] = 26,
-	[1][1][RTW89_IC][6] = 10,
-	[1][1][RTW89_ACMA][6] = 22,
-	[1][1][RTW89_FCC][8] = 36,
+	[1][1][RTW89_IC][6] = 20,
+	[1][1][RTW89_KCC][6] = 8,
+	[1][1][RTW89_ACMA][6] = 30,
+	[1][1][RTW89_CN][6] = 14,
+	[1][1][RTW89_UK][6] = 30,
+	[1][1][RTW89_FCC][8] = 44,
 	[1][1][RTW89_ETSI][8] = 30,
 	[1][1][RTW89_MKK][8] = 20,
 	[1][1][RTW89_IC][8] = 44,
-	[1][1][RTW89_ACMA][8] = 22,
-	[1][1][RTW89_FCC][10] = 36,
+	[1][1][RTW89_KCC][8] = 34,
+	[1][1][RTW89_ACMA][8] = 30,
+	[1][1][RTW89_CN][8] = 14,
+	[1][1][RTW89_UK][8] = 30,
+	[1][1][RTW89_FCC][10] = 44,
 	[1][1][RTW89_ETSI][10] = 30,
 	[1][1][RTW89_MKK][10] = 20,
 	[1][1][RTW89_IC][10] = 44,
-	[1][1][RTW89_ACMA][10] = 22,
-	[1][1][RTW89_FCC][12] = 38,
+	[1][1][RTW89_KCC][10] = 34,
+	[1][1][RTW89_ACMA][10] = 30,
+	[1][1][RTW89_CN][10] = 14,
+	[1][1][RTW89_UK][10] = 30,
+	[1][1][RTW89_FCC][12] = 44,
 	[1][1][RTW89_ETSI][12] = 30,
 	[1][1][RTW89_MKK][12] = 34,
-	[1][1][RTW89_IC][12] = 46,
-	[1][1][RTW89_ACMA][12] = 22,
-	[1][1][RTW89_FCC][14] = 34,
+	[1][1][RTW89_IC][12] = 44,
+	[1][1][RTW89_KCC][12] = 38,
+	[1][1][RTW89_ACMA][12] = 30,
+	[1][1][RTW89_CN][12] = 14,
+	[1][1][RTW89_UK][12] = 30,
+	[1][1][RTW89_FCC][14] = 44,
 	[1][1][RTW89_ETSI][14] = 30,
 	[1][1][RTW89_MKK][14] = 34,
-	[1][1][RTW89_IC][14] = 40,
-	[1][1][RTW89_ACMA][14] = 22,
-	[1][1][RTW89_FCC][15] = 34,
+	[1][1][RTW89_IC][14] = 44,
+	[1][1][RTW89_KCC][14] = 38,
+	[1][1][RTW89_ACMA][14] = 30,
+	[1][1][RTW89_CN][14] = 14,
+	[1][1][RTW89_UK][14] = 30,
+	[1][1][RTW89_FCC][15] = 44,
 	[1][1][RTW89_ETSI][15] = 28,
 	[1][1][RTW89_MKK][15] = 56,
-	[1][1][RTW89_IC][15] = 42,
-	[1][1][RTW89_ACMA][15] = 22,
-	[1][1][RTW89_FCC][17] = 34,
+	[1][1][RTW89_IC][15] = 44,
+	[1][1][RTW89_KCC][15] = 36,
+	[1][1][RTW89_ACMA][15] = 28,
+	[1][1][RTW89_CN][15] = 127,
+	[1][1][RTW89_UK][15] = 28,
+	[1][1][RTW89_FCC][17] = 44,
 	[1][1][RTW89_ETSI][17] = 28,
 	[1][1][RTW89_MKK][17] = 58,
-	[1][1][RTW89_IC][17] = 42,
-	[1][1][RTW89_ACMA][17] = 22,
-	[1][1][RTW89_FCC][19] = 34,
+	[1][1][RTW89_IC][17] = 44,
+	[1][1][RTW89_KCC][17] = 36,
+	[1][1][RTW89_ACMA][17] = 28,
+	[1][1][RTW89_CN][17] = 127,
+	[1][1][RTW89_UK][17] = 28,
+	[1][1][RTW89_FCC][19] = 44,
 	[1][1][RTW89_ETSI][19] = 28,
 	[1][1][RTW89_MKK][19] = 58,
-	[1][1][RTW89_IC][19] = 42,
-	[1][1][RTW89_ACMA][19] = 22,
-	[1][1][RTW89_FCC][21] = 34,
+	[1][1][RTW89_IC][19] = 44,
+	[1][1][RTW89_KCC][19] = 36,
+	[1][1][RTW89_ACMA][19] = 28,
+	[1][1][RTW89_CN][19] = 127,
+	[1][1][RTW89_UK][19] = 28,
+	[1][1][RTW89_FCC][21] = 44,
 	[1][1][RTW89_ETSI][21] = 28,
 	[1][1][RTW89_MKK][21] = 58,
-	[1][1][RTW89_IC][21] = 42,
-	[1][1][RTW89_ACMA][21] = 22,
-	[1][1][RTW89_FCC][23] = 34,
+	[1][1][RTW89_IC][21] = 44,
+	[1][1][RTW89_KCC][21] = 36,
+	[1][1][RTW89_ACMA][21] = 28,
+	[1][1][RTW89_CN][21] = 127,
+	[1][1][RTW89_UK][21] = 28,
+	[1][1][RTW89_FCC][23] = 44,
 	[1][1][RTW89_ETSI][23] = 28,
 	[1][1][RTW89_MKK][23] = 58,
-	[1][1][RTW89_IC][23] = 42,
-	[1][1][RTW89_ACMA][23] = 22,
-	[1][1][RTW89_FCC][25] = 34,
+	[1][1][RTW89_IC][23] = 44,
+	[1][1][RTW89_KCC][23] = 36,
+	[1][1][RTW89_ACMA][23] = 28,
+	[1][1][RTW89_CN][23] = 127,
+	[1][1][RTW89_UK][23] = 28,
+	[1][1][RTW89_FCC][25] = 44,
 	[1][1][RTW89_ETSI][25] = 28,
 	[1][1][RTW89_MKK][25] = 58,
 	[1][1][RTW89_IC][25] = 127,
+	[1][1][RTW89_KCC][25] = 36,
 	[1][1][RTW89_ACMA][25] = 127,
-	[1][1][RTW89_FCC][27] = 34,
+	[1][1][RTW89_CN][25] = 127,
+	[1][1][RTW89_UK][25] = 28,
+	[1][1][RTW89_FCC][27] = 44,
 	[1][1][RTW89_ETSI][27] = 30,
 	[1][1][RTW89_MKK][27] = 58,
 	[1][1][RTW89_IC][27] = 127,
+	[1][1][RTW89_KCC][27] = 36,
 	[1][1][RTW89_ACMA][27] = 127,
-	[1][1][RTW89_FCC][29] = 34,
+	[1][1][RTW89_CN][27] = 127,
+	[1][1][RTW89_UK][27] = 30,
+	[1][1][RTW89_FCC][29] = 44,
 	[1][1][RTW89_ETSI][29] = 30,
 	[1][1][RTW89_MKK][29] = 58,
 	[1][1][RTW89_IC][29] = 127,
+	[1][1][RTW89_KCC][29] = 36,
 	[1][1][RTW89_ACMA][29] = 127,
-	[1][1][RTW89_FCC][31] = 34,
+	[1][1][RTW89_CN][29] = 127,
+	[1][1][RTW89_UK][29] = 30,
+	[1][1][RTW89_FCC][31] = 44,
 	[1][1][RTW89_ETSI][31] = 30,
 	[1][1][RTW89_MKK][31] = 58,
 	[1][1][RTW89_IC][31] = 38,
-	[1][1][RTW89_ACMA][31] = 22,
-	[1][1][RTW89_FCC][33] = 32,
+	[1][1][RTW89_KCC][31] = 36,
+	[1][1][RTW89_ACMA][31] = 30,
+	[1][1][RTW89_CN][31] = 127,
+	[1][1][RTW89_UK][31] = 30,
+	[1][1][RTW89_FCC][33] = 38,
 	[1][1][RTW89_ETSI][33] = 30,
 	[1][1][RTW89_MKK][33] = 58,
 	[1][1][RTW89_IC][33] = 38,
-	[1][1][RTW89_ACMA][33] = 22,
-	[1][1][RTW89_FCC][35] = 32,
+	[1][1][RTW89_KCC][33] = 36,
+	[1][1][RTW89_ACMA][33] = 30,
+	[1][1][RTW89_CN][33] = 127,
+	[1][1][RTW89_UK][33] = 30,
+	[1][1][RTW89_FCC][35] = 38,
 	[1][1][RTW89_ETSI][35] = 30,
 	[1][1][RTW89_MKK][35] = 58,
 	[1][1][RTW89_IC][35] = 38,
-	[1][1][RTW89_ACMA][35] = 22,
-	[1][1][RTW89_FCC][37] = 40,
+	[1][1][RTW89_KCC][35] = 36,
+	[1][1][RTW89_ACMA][35] = 30,
+	[1][1][RTW89_CN][35] = 127,
+	[1][1][RTW89_UK][35] = 30,
+	[1][1][RTW89_FCC][37] = 46,
 	[1][1][RTW89_ETSI][37] = 127,
 	[1][1][RTW89_MKK][37] = 58,
-	[1][1][RTW89_IC][37] = 48,
-	[1][1][RTW89_ACMA][37] = 48,
-	[1][1][RTW89_FCC][38] = 76,
+	[1][1][RTW89_IC][37] = 46,
+	[1][1][RTW89_KCC][37] = 36,
+	[1][1][RTW89_ACMA][37] = 46,
+	[1][1][RTW89_CN][37] = 127,
+	[1][1][RTW89_UK][37] = 32,
+	[1][1][RTW89_FCC][38] = 74,
 	[1][1][RTW89_ETSI][38] = 16,
 	[1][1][RTW89_MKK][38] = 127,
-	[1][1][RTW89_IC][38] = 84,
-	[1][1][RTW89_ACMA][38] = 82,
-	[1][1][RTW89_FCC][40] = 76,
+	[1][1][RTW89_IC][38] = 74,
+	[1][1][RTW89_KCC][38] = 36,
+	[1][1][RTW89_ACMA][38] = 74,
+	[1][1][RTW89_CN][38] = 54,
+	[1][1][RTW89_UK][38] = 30,
+	[1][1][RTW89_FCC][40] = 74,
 	[1][1][RTW89_ETSI][40] = 16,
 	[1][1][RTW89_MKK][40] = 127,
-	[1][1][RTW89_IC][40] = 84,
-	[1][1][RTW89_ACMA][40] = 82,
-	[1][1][RTW89_FCC][42] = 76,
+	[1][1][RTW89_IC][40] = 74,
+	[1][1][RTW89_KCC][40] = 36,
+	[1][1][RTW89_ACMA][40] = 74,
+	[1][1][RTW89_CN][40] = 54,
+	[1][1][RTW89_UK][40] = 30,
+	[1][1][RTW89_FCC][42] = 74,
 	[1][1][RTW89_ETSI][42] = 16,
 	[1][1][RTW89_MKK][42] = 127,
-	[1][1][RTW89_IC][42] = 84,
-	[1][1][RTW89_ACMA][42] = 84,
-	[1][1][RTW89_FCC][44] = 76,
+	[1][1][RTW89_IC][42] = 74,
+	[1][1][RTW89_KCC][42] = 36,
+	[1][1][RTW89_ACMA][42] = 74,
+	[1][1][RTW89_CN][42] = 54,
+	[1][1][RTW89_UK][42] = 30,
+	[1][1][RTW89_FCC][44] = 74,
 	[1][1][RTW89_ETSI][44] = 16,
 	[1][1][RTW89_MKK][44] = 127,
-	[1][1][RTW89_IC][44] = 84,
-	[1][1][RTW89_ACMA][44] = 84,
-	[1][1][RTW89_FCC][46] = 76,
+	[1][1][RTW89_IC][44] = 74,
+	[1][1][RTW89_KCC][44] = 36,
+	[1][1][RTW89_ACMA][44] = 74,
+	[1][1][RTW89_CN][44] = 54,
+	[1][1][RTW89_UK][44] = 30,
+	[1][1][RTW89_FCC][46] = 74,
 	[1][1][RTW89_ETSI][46] = 16,
 	[1][1][RTW89_MKK][46] = 127,
-	[1][1][RTW89_IC][46] = 84,
-	[1][1][RTW89_ACMA][46] = 84,
-	[1][1][RTW89_FCC][48] = 24,
+	[1][1][RTW89_IC][46] = 74,
+	[1][1][RTW89_KCC][46] = 36,
+	[1][1][RTW89_ACMA][46] = 74,
+	[1][1][RTW89_CN][46] = 54,
+	[1][1][RTW89_UK][46] = 30,
+	[1][1][RTW89_FCC][48] = 34,
 	[1][1][RTW89_ETSI][48] = 127,
 	[1][1][RTW89_MKK][48] = 127,
 	[1][1][RTW89_IC][48] = 127,
+	[1][1][RTW89_KCC][48] = 127,
 	[1][1][RTW89_ACMA][48] = 127,
-	[1][1][RTW89_FCC][50] = 24,
+	[1][1][RTW89_CN][48] = 127,
+	[1][1][RTW89_UK][48] = 127,
+	[1][1][RTW89_FCC][50] = 34,
 	[1][1][RTW89_ETSI][50] = 127,
 	[1][1][RTW89_MKK][50] = 127,
 	[1][1][RTW89_IC][50] = 127,
+	[1][1][RTW89_KCC][50] = 127,
 	[1][1][RTW89_ACMA][50] = 127,
-	[1][1][RTW89_FCC][52] = 24,
+	[1][1][RTW89_CN][50] = 127,
+	[1][1][RTW89_UK][50] = 127,
+	[1][1][RTW89_FCC][52] = 30,
 	[1][1][RTW89_ETSI][52] = 127,
 	[1][1][RTW89_MKK][52] = 127,
 	[1][1][RTW89_IC][52] = 127,
+	[1][1][RTW89_KCC][52] = 127,
 	[1][1][RTW89_ACMA][52] = 127,
-	[2][0][RTW89_FCC][0] = 62,
+	[1][1][RTW89_CN][52] = 127,
+	[1][1][RTW89_UK][52] = 127,
+	[2][0][RTW89_FCC][0] = 68,
 	[2][0][RTW89_ETSI][0] = 52,
 	[2][0][RTW89_MKK][0] = 60,
-	[2][0][RTW89_IC][0] = 46,
-	[2][0][RTW89_ACMA][0] = 48,
-	[2][0][RTW89_FCC][2] = 62,
+	[2][0][RTW89_IC][0] = 52,
+	[2][0][RTW89_KCC][0] = 64,
+	[2][0][RTW89_ACMA][0] = 52,
+	[2][0][RTW89_CN][0] = 40,
+	[2][0][RTW89_UK][0] = 52,
+	[2][0][RTW89_FCC][2] = 64,
 	[2][0][RTW89_ETSI][2] = 52,
 	[2][0][RTW89_MKK][2] = 60,
-	[2][0][RTW89_IC][2] = 46,
-	[2][0][RTW89_ACMA][2] = 48,
-	[2][0][RTW89_FCC][4] = 62,
+	[2][0][RTW89_IC][2] = 50,
+	[2][0][RTW89_KCC][2] = 64,
+	[2][0][RTW89_ACMA][2] = 52,
+	[2][0][RTW89_CN][2] = 40,
+	[2][0][RTW89_UK][2] = 52,
+	[2][0][RTW89_FCC][4] = 68,
 	[2][0][RTW89_ETSI][4] = 52,
 	[2][0][RTW89_MKK][4] = 50,
-	[2][0][RTW89_IC][4] = 46,
-	[2][0][RTW89_ACMA][4] = 48,
-	[2][0][RTW89_FCC][6] = 62,
+	[2][0][RTW89_IC][4] = 50,
+	[2][0][RTW89_KCC][4] = 64,
+	[2][0][RTW89_ACMA][4] = 52,
+	[2][0][RTW89_CN][4] = 40,
+	[2][0][RTW89_UK][4] = 52,
+	[2][0][RTW89_FCC][6] = 68,
 	[2][0][RTW89_ETSI][6] = 52,
 	[2][0][RTW89_MKK][6] = 50,
-	[2][0][RTW89_IC][6] = 46,
-	[2][0][RTW89_ACMA][6] = 48,
-	[2][0][RTW89_FCC][8] = 62,
+	[2][0][RTW89_IC][6] = 50,
+	[2][0][RTW89_KCC][6] = 36,
+	[2][0][RTW89_ACMA][6] = 52,
+	[2][0][RTW89_CN][6] = 40,
+	[2][0][RTW89_UK][6] = 52,
+	[2][0][RTW89_FCC][8] = 68,
 	[2][0][RTW89_ETSI][8] = 52,
 	[2][0][RTW89_MKK][8] = 44,
-	[2][0][RTW89_IC][8] = 66,
-	[2][0][RTW89_ACMA][8] = 48,
-	[2][0][RTW89_FCC][10] = 62,
+	[2][0][RTW89_IC][8] = 64,
+	[2][0][RTW89_KCC][8] = 62,
+	[2][0][RTW89_ACMA][8] = 52,
+	[2][0][RTW89_CN][8] = 40,
+	[2][0][RTW89_UK][8] = 52,
+	[2][0][RTW89_FCC][10] = 68,
 	[2][0][RTW89_ETSI][10] = 52,
 	[2][0][RTW89_MKK][10] = 44,
-	[2][0][RTW89_IC][10] = 66,
-	[2][0][RTW89_ACMA][10] = 48,
-	[2][0][RTW89_FCC][12] = 62,
+	[2][0][RTW89_IC][10] = 64,
+	[2][0][RTW89_KCC][10] = 62,
+	[2][0][RTW89_ACMA][10] = 52,
+	[2][0][RTW89_CN][10] = 40,
+	[2][0][RTW89_UK][10] = 52,
+	[2][0][RTW89_FCC][12] = 68,
 	[2][0][RTW89_ETSI][12] = 52,
 	[2][0][RTW89_MKK][12] = 58,
-	[2][0][RTW89_IC][12] = 66,
-	[2][0][RTW89_ACMA][12] = 48,
-	[2][0][RTW89_FCC][14] = 62,
+	[2][0][RTW89_IC][12] = 64,
+	[2][0][RTW89_KCC][12] = 62,
+	[2][0][RTW89_ACMA][12] = 52,
+	[2][0][RTW89_CN][12] = 40,
+	[2][0][RTW89_UK][12] = 52,
+	[2][0][RTW89_FCC][14] = 68,
 	[2][0][RTW89_ETSI][14] = 52,
 	[2][0][RTW89_MKK][14] = 58,
-	[2][0][RTW89_IC][14] = 66,
-	[2][0][RTW89_ACMA][14] = 48,
-	[2][0][RTW89_FCC][15] = 62,
+	[2][0][RTW89_IC][14] = 64,
+	[2][0][RTW89_KCC][14] = 62,
+	[2][0][RTW89_ACMA][14] = 52,
+	[2][0][RTW89_CN][14] = 40,
+	[2][0][RTW89_UK][14] = 52,
+	[2][0][RTW89_FCC][15] = 68,
 	[2][0][RTW89_ETSI][15] = 52,
 	[2][0][RTW89_MKK][15] = 68,
-	[2][0][RTW89_IC][15] = 70,
-	[2][0][RTW89_ACMA][15] = 48,
-	[2][0][RTW89_FCC][17] = 62,
+	[2][0][RTW89_IC][15] = 68,
+	[2][0][RTW89_KCC][15] = 62,
+	[2][0][RTW89_ACMA][15] = 52,
+	[2][0][RTW89_CN][15] = 127,
+	[2][0][RTW89_UK][15] = 52,
+	[2][0][RTW89_FCC][17] = 68,
 	[2][0][RTW89_ETSI][17] = 52,
 	[2][0][RTW89_MKK][17] = 74,
-	[2][0][RTW89_IC][17] = 70,
-	[2][0][RTW89_ACMA][17] = 48,
-	[2][0][RTW89_FCC][19] = 62,
+	[2][0][RTW89_IC][17] = 68,
+	[2][0][RTW89_KCC][17] = 62,
+	[2][0][RTW89_ACMA][17] = 52,
+	[2][0][RTW89_CN][17] = 127,
+	[2][0][RTW89_UK][17] = 52,
+	[2][0][RTW89_FCC][19] = 70,
 	[2][0][RTW89_ETSI][19] = 52,
 	[2][0][RTW89_MKK][19] = 74,
 	[2][0][RTW89_IC][19] = 70,
-	[2][0][RTW89_ACMA][19] = 48,
-	[2][0][RTW89_FCC][21] = 62,
+	[2][0][RTW89_KCC][19] = 62,
+	[2][0][RTW89_ACMA][19] = 52,
+	[2][0][RTW89_CN][19] = 127,
+	[2][0][RTW89_UK][19] = 52,
+	[2][0][RTW89_FCC][21] = 70,
 	[2][0][RTW89_ETSI][21] = 52,
 	[2][0][RTW89_MKK][21] = 74,
 	[2][0][RTW89_IC][21] = 70,
-	[2][0][RTW89_ACMA][21] = 48,
-	[2][0][RTW89_FCC][23] = 62,
+	[2][0][RTW89_KCC][21] = 62,
+	[2][0][RTW89_ACMA][21] = 52,
+	[2][0][RTW89_CN][21] = 127,
+	[2][0][RTW89_UK][21] = 52,
+	[2][0][RTW89_FCC][23] = 70,
 	[2][0][RTW89_ETSI][23] = 52,
 	[2][0][RTW89_MKK][23] = 74,
 	[2][0][RTW89_IC][23] = 70,
-	[2][0][RTW89_ACMA][23] = 48,
-	[2][0][RTW89_FCC][25] = 62,
+	[2][0][RTW89_KCC][23] = 62,
+	[2][0][RTW89_ACMA][23] = 52,
+	[2][0][RTW89_CN][23] = 127,
+	[2][0][RTW89_UK][23] = 52,
+	[2][0][RTW89_FCC][25] = 70,
 	[2][0][RTW89_ETSI][25] = 52,
 	[2][0][RTW89_MKK][25] = 74,
 	[2][0][RTW89_IC][25] = 127,
+	[2][0][RTW89_KCC][25] = 62,
 	[2][0][RTW89_ACMA][25] = 127,
-	[2][0][RTW89_FCC][27] = 62,
+	[2][0][RTW89_CN][25] = 127,
+	[2][0][RTW89_UK][25] = 52,
+	[2][0][RTW89_FCC][27] = 70,
 	[2][0][RTW89_ETSI][27] = 52,
 	[2][0][RTW89_MKK][27] = 74,
 	[2][0][RTW89_IC][27] = 127,
+	[2][0][RTW89_KCC][27] = 62,
 	[2][0][RTW89_ACMA][27] = 127,
-	[2][0][RTW89_FCC][29] = 62,
+	[2][0][RTW89_CN][27] = 127,
+	[2][0][RTW89_UK][27] = 52,
+	[2][0][RTW89_FCC][29] = 70,
 	[2][0][RTW89_ETSI][29] = 52,
 	[2][0][RTW89_MKK][29] = 74,
 	[2][0][RTW89_IC][29] = 127,
+	[2][0][RTW89_KCC][29] = 62,
 	[2][0][RTW89_ACMA][29] = 127,
-	[2][0][RTW89_FCC][31] = 62,
+	[2][0][RTW89_CN][29] = 127,
+	[2][0][RTW89_UK][29] = 52,
+	[2][0][RTW89_FCC][31] = 70,
 	[2][0][RTW89_ETSI][31] = 52,
 	[2][0][RTW89_MKK][31] = 74,
-	[2][0][RTW89_IC][31] = 72,
-	[2][0][RTW89_ACMA][31] = 48,
-	[2][0][RTW89_FCC][33] = 64,
+	[2][0][RTW89_IC][31] = 62,
+	[2][0][RTW89_KCC][31] = 62,
+	[2][0][RTW89_ACMA][31] = 52,
+	[2][0][RTW89_CN][31] = 127,
+	[2][0][RTW89_UK][31] = 52,
+	[2][0][RTW89_FCC][33] = 62,
 	[2][0][RTW89_ETSI][33] = 52,
 	[2][0][RTW89_MKK][33] = 74,
-	[2][0][RTW89_IC][33] = 72,
-	[2][0][RTW89_ACMA][33] = 48,
-	[2][0][RTW89_FCC][35] = 64,
+	[2][0][RTW89_IC][33] = 62,
+	[2][0][RTW89_KCC][33] = 62,
+	[2][0][RTW89_ACMA][33] = 52,
+	[2][0][RTW89_CN][33] = 127,
+	[2][0][RTW89_UK][33] = 52,
+	[2][0][RTW89_FCC][35] = 62,
 	[2][0][RTW89_ETSI][35] = 52,
 	[2][0][RTW89_MKK][35] = 74,
-	[2][0][RTW89_IC][35] = 72,
-	[2][0][RTW89_ACMA][35] = 48,
-	[2][0][RTW89_FCC][37] = 62,
+	[2][0][RTW89_IC][35] = 62,
+	[2][0][RTW89_KCC][35] = 62,
+	[2][0][RTW89_ACMA][35] = 52,
+	[2][0][RTW89_CN][35] = 127,
+	[2][0][RTW89_UK][35] = 52,
+	[2][0][RTW89_FCC][37] = 70,
 	[2][0][RTW89_ETSI][37] = 127,
 	[2][0][RTW89_MKK][37] = 74,
 	[2][0][RTW89_IC][37] = 70,
-	[2][0][RTW89_ACMA][37] = 76,
-	[2][0][RTW89_FCC][38] = 76,
+	[2][0][RTW89_KCC][37] = 62,
+	[2][0][RTW89_ACMA][37] = 70,
+	[2][0][RTW89_CN][37] = 127,
+	[2][0][RTW89_UK][37] = 52,
+	[2][0][RTW89_FCC][38] = 82,
 	[2][0][RTW89_ETSI][38] = 28,
 	[2][0][RTW89_MKK][38] = 127,
-	[2][0][RTW89_IC][38] = 84,
-	[2][0][RTW89_ACMA][38] = 84,
-	[2][0][RTW89_FCC][40] = 76,
+	[2][0][RTW89_IC][38] = 82,
+	[2][0][RTW89_KCC][38] = 64,
+	[2][0][RTW89_ACMA][38] = 82,
+	[2][0][RTW89_CN][38] = 68,
+	[2][0][RTW89_UK][38] = 54,
+	[2][0][RTW89_FCC][40] = 82,
 	[2][0][RTW89_ETSI][40] = 28,
 	[2][0][RTW89_MKK][40] = 127,
-	[2][0][RTW89_IC][40] = 84,
-	[2][0][RTW89_ACMA][40] = 84,
+	[2][0][RTW89_IC][40] = 82,
+	[2][0][RTW89_KCC][40] = 64,
+	[2][0][RTW89_ACMA][40] = 82,
+	[2][0][RTW89_CN][40] = 68,
+	[2][0][RTW89_UK][40] = 54,
 	[2][0][RTW89_FCC][42] = 76,
 	[2][0][RTW89_ETSI][42] = 28,
 	[2][0][RTW89_MKK][42] = 127,
-	[2][0][RTW89_IC][42] = 84,
-	[2][0][RTW89_ACMA][42] = 84,
-	[2][0][RTW89_FCC][44] = 76,
+	[2][0][RTW89_IC][42] = 76,
+	[2][0][RTW89_KCC][42] = 64,
+	[2][0][RTW89_ACMA][42] = 76,
+	[2][0][RTW89_CN][42] = 68,
+	[2][0][RTW89_UK][42] = 54,
+	[2][0][RTW89_FCC][44] = 80,
 	[2][0][RTW89_ETSI][44] = 28,
 	[2][0][RTW89_MKK][44] = 127,
-	[2][0][RTW89_IC][44] = 84,
-	[2][0][RTW89_ACMA][44] = 84,
-	[2][0][RTW89_FCC][46] = 76,
+	[2][0][RTW89_IC][44] = 80,
+	[2][0][RTW89_KCC][44] = 64,
+	[2][0][RTW89_ACMA][44] = 80,
+	[2][0][RTW89_CN][44] = 68,
+	[2][0][RTW89_UK][44] = 54,
+	[2][0][RTW89_FCC][46] = 80,
 	[2][0][RTW89_ETSI][46] = 28,
 	[2][0][RTW89_MKK][46] = 127,
-	[2][0][RTW89_IC][46] = 84,
-	[2][0][RTW89_ACMA][46] = 84,
-	[2][0][RTW89_FCC][48] = 48,
+	[2][0][RTW89_IC][46] = 80,
+	[2][0][RTW89_KCC][46] = 64,
+	[2][0][RTW89_ACMA][46] = 80,
+	[2][0][RTW89_CN][46] = 68,
+	[2][0][RTW89_UK][46] = 54,
+	[2][0][RTW89_FCC][48] = 64,
 	[2][0][RTW89_ETSI][48] = 127,
 	[2][0][RTW89_MKK][48] = 127,
 	[2][0][RTW89_IC][48] = 127,
+	[2][0][RTW89_KCC][48] = 127,
 	[2][0][RTW89_ACMA][48] = 127,
-	[2][0][RTW89_FCC][50] = 48,
+	[2][0][RTW89_CN][48] = 127,
+	[2][0][RTW89_UK][48] = 127,
+	[2][0][RTW89_FCC][50] = 64,
 	[2][0][RTW89_ETSI][50] = 127,
 	[2][0][RTW89_MKK][50] = 127,
 	[2][0][RTW89_IC][50] = 127,
+	[2][0][RTW89_KCC][50] = 127,
 	[2][0][RTW89_ACMA][50] = 127,
-	[2][0][RTW89_FCC][52] = 48,
+	[2][0][RTW89_CN][50] = 127,
+	[2][0][RTW89_UK][50] = 127,
+	[2][0][RTW89_FCC][52] = 64,
 	[2][0][RTW89_ETSI][52] = 127,
 	[2][0][RTW89_MKK][52] = 127,
 	[2][0][RTW89_IC][52] = 127,
+	[2][0][RTW89_KCC][52] = 127,
 	[2][0][RTW89_ACMA][52] = 127,
-	[2][1][RTW89_FCC][0] = 42,
+	[2][0][RTW89_CN][52] = 127,
+	[2][0][RTW89_UK][52] = 127,
+	[2][1][RTW89_FCC][0] = 50,
 	[2][1][RTW89_ETSI][0] = 40,
 	[2][1][RTW89_MKK][0] = 44,
-	[2][1][RTW89_IC][0] = 20,
-	[2][1][RTW89_ACMA][0] = 36,
-	[2][1][RTW89_FCC][2] = 42,
+	[2][1][RTW89_IC][0] = 26,
+	[2][1][RTW89_KCC][0] = 44,
+	[2][1][RTW89_ACMA][0] = 40,
+	[2][1][RTW89_CN][0] = 28,
+	[2][1][RTW89_UK][0] = 40,
+	[2][1][RTW89_FCC][2] = 50,
 	[2][1][RTW89_ETSI][2] = 40,
 	[2][1][RTW89_MKK][2] = 44,
-	[2][1][RTW89_IC][2] = 18,
-	[2][1][RTW89_ACMA][2] = 36,
-	[2][1][RTW89_FCC][4] = 42,
+	[2][1][RTW89_IC][2] = 26,
+	[2][1][RTW89_KCC][2] = 44,
+	[2][1][RTW89_ACMA][2] = 40,
+	[2][1][RTW89_CN][2] = 28,
+	[2][1][RTW89_UK][2] = 40,
+	[2][1][RTW89_FCC][4] = 50,
 	[2][1][RTW89_ETSI][4] = 40,
 	[2][1][RTW89_MKK][4] = 36,
-	[2][1][RTW89_IC][4] = 22,
-	[2][1][RTW89_ACMA][4] = 36,
-	[2][1][RTW89_FCC][6] = 42,
+	[2][1][RTW89_IC][4] = 26,
+	[2][1][RTW89_KCC][4] = 44,
+	[2][1][RTW89_ACMA][4] = 40,
+	[2][1][RTW89_CN][4] = 28,
+	[2][1][RTW89_UK][4] = 40,
+	[2][1][RTW89_FCC][6] = 50,
 	[2][1][RTW89_ETSI][6] = 40,
 	[2][1][RTW89_MKK][6] = 36,
-	[2][1][RTW89_IC][6] = 22,
-	[2][1][RTW89_ACMA][6] = 36,
-	[2][1][RTW89_FCC][8] = 42,
+	[2][1][RTW89_IC][6] = 26,
+	[2][1][RTW89_KCC][6] = 20,
+	[2][1][RTW89_ACMA][6] = 40,
+	[2][1][RTW89_CN][6] = 28,
+	[2][1][RTW89_UK][6] = 40,
+	[2][1][RTW89_FCC][8] = 50,
 	[2][1][RTW89_ETSI][8] = 40,
 	[2][1][RTW89_MKK][8] = 32,
 	[2][1][RTW89_IC][8] = 50,
-	[2][1][RTW89_ACMA][8] = 36,
-	[2][1][RTW89_FCC][10] = 42,
+	[2][1][RTW89_KCC][8] = 46,
+	[2][1][RTW89_ACMA][8] = 40,
+	[2][1][RTW89_CN][8] = 28,
+	[2][1][RTW89_UK][8] = 40,
+	[2][1][RTW89_FCC][10] = 50,
 	[2][1][RTW89_ETSI][10] = 40,
 	[2][1][RTW89_MKK][10] = 32,
 	[2][1][RTW89_IC][10] = 50,
-	[2][1][RTW89_ACMA][10] = 36,
-	[2][1][RTW89_FCC][12] = 44,
+	[2][1][RTW89_KCC][10] = 46,
+	[2][1][RTW89_ACMA][10] = 40,
+	[2][1][RTW89_CN][10] = 28,
+	[2][1][RTW89_UK][10] = 40,
+	[2][1][RTW89_FCC][12] = 48,
 	[2][1][RTW89_ETSI][12] = 40,
 	[2][1][RTW89_MKK][12] = 44,
-	[2][1][RTW89_IC][12] = 52,
-	[2][1][RTW89_ACMA][12] = 36,
-	[2][1][RTW89_FCC][14] = 44,
+	[2][1][RTW89_IC][12] = 48,
+	[2][1][RTW89_KCC][12] = 46,
+	[2][1][RTW89_ACMA][12] = 40,
+	[2][1][RTW89_CN][12] = 28,
+	[2][1][RTW89_UK][12] = 40,
+	[2][1][RTW89_FCC][14] = 48,
 	[2][1][RTW89_ETSI][14] = 40,
 	[2][1][RTW89_MKK][14] = 44,
-	[2][1][RTW89_IC][14] = 52,
-	[2][1][RTW89_ACMA][14] = 36,
-	[2][1][RTW89_FCC][15] = 42,
+	[2][1][RTW89_IC][14] = 48,
+	[2][1][RTW89_KCC][14] = 46,
+	[2][1][RTW89_ACMA][14] = 40,
+	[2][1][RTW89_CN][14] = 28,
+	[2][1][RTW89_UK][14] = 40,
+	[2][1][RTW89_FCC][15] = 50,
 	[2][1][RTW89_ETSI][15] = 40,
 	[2][1][RTW89_MKK][15] = 66,
 	[2][1][RTW89_IC][15] = 50,
-	[2][1][RTW89_ACMA][15] = 36,
-	[2][1][RTW89_FCC][17] = 42,
+	[2][1][RTW89_KCC][15] = 46,
+	[2][1][RTW89_ACMA][15] = 40,
+	[2][1][RTW89_CN][15] = 127,
+	[2][1][RTW89_UK][15] = 40,
+	[2][1][RTW89_FCC][17] = 50,
 	[2][1][RTW89_ETSI][17] = 40,
 	[2][1][RTW89_MKK][17] = 66,
 	[2][1][RTW89_IC][17] = 50,
-	[2][1][RTW89_ACMA][17] = 36,
-	[2][1][RTW89_FCC][19] = 42,
+	[2][1][RTW89_KCC][17] = 46,
+	[2][1][RTW89_ACMA][17] = 40,
+	[2][1][RTW89_CN][17] = 127,
+	[2][1][RTW89_UK][17] = 40,
+	[2][1][RTW89_FCC][19] = 50,
 	[2][1][RTW89_ETSI][19] = 40,
 	[2][1][RTW89_MKK][19] = 66,
 	[2][1][RTW89_IC][19] = 50,
-	[2][1][RTW89_ACMA][19] = 36,
-	[2][1][RTW89_FCC][21] = 42,
+	[2][1][RTW89_KCC][19] = 46,
+	[2][1][RTW89_ACMA][19] = 40,
+	[2][1][RTW89_CN][19] = 127,
+	[2][1][RTW89_UK][19] = 40,
+	[2][1][RTW89_FCC][21] = 50,
 	[2][1][RTW89_ETSI][21] = 40,
 	[2][1][RTW89_MKK][21] = 66,
 	[2][1][RTW89_IC][21] = 50,
-	[2][1][RTW89_ACMA][21] = 36,
-	[2][1][RTW89_FCC][23] = 42,
+	[2][1][RTW89_KCC][21] = 46,
+	[2][1][RTW89_ACMA][21] = 40,
+	[2][1][RTW89_CN][21] = 127,
+	[2][1][RTW89_UK][21] = 40,
+	[2][1][RTW89_FCC][23] = 50,
 	[2][1][RTW89_ETSI][23] = 40,
 	[2][1][RTW89_MKK][23] = 66,
 	[2][1][RTW89_IC][23] = 50,
-	[2][1][RTW89_ACMA][23] = 36,
-	[2][1][RTW89_FCC][25] = 42,
+	[2][1][RTW89_KCC][23] = 46,
+	[2][1][RTW89_ACMA][23] = 40,
+	[2][1][RTW89_CN][23] = 127,
+	[2][1][RTW89_UK][23] = 40,
+	[2][1][RTW89_FCC][25] = 50,
 	[2][1][RTW89_ETSI][25] = 40,
 	[2][1][RTW89_MKK][25] = 66,
 	[2][1][RTW89_IC][25] = 127,
+	[2][1][RTW89_KCC][25] = 46,
 	[2][1][RTW89_ACMA][25] = 127,
-	[2][1][RTW89_FCC][27] = 42,
+	[2][1][RTW89_CN][25] = 127,
+	[2][1][RTW89_UK][25] = 40,
+	[2][1][RTW89_FCC][27] = 50,
 	[2][1][RTW89_ETSI][27] = 40,
 	[2][1][RTW89_MKK][27] = 66,
 	[2][1][RTW89_IC][27] = 127,
+	[2][1][RTW89_KCC][27] = 46,
 	[2][1][RTW89_ACMA][27] = 127,
-	[2][1][RTW89_FCC][29] = 42,
+	[2][1][RTW89_CN][27] = 127,
+	[2][1][RTW89_UK][27] = 40,
+	[2][1][RTW89_FCC][29] = 50,
 	[2][1][RTW89_ETSI][29] = 40,
 	[2][1][RTW89_MKK][29] = 66,
 	[2][1][RTW89_IC][29] = 127,
+	[2][1][RTW89_KCC][29] = 46,
 	[2][1][RTW89_ACMA][29] = 127,
-	[2][1][RTW89_FCC][31] = 42,
+	[2][1][RTW89_CN][29] = 127,
+	[2][1][RTW89_UK][29] = 40,
+	[2][1][RTW89_FCC][31] = 50,
 	[2][1][RTW89_ETSI][31] = 40,
 	[2][1][RTW89_MKK][31] = 66,
-	[2][1][RTW89_IC][31] = 50,
-	[2][1][RTW89_ACMA][31] = 36,
-	[2][1][RTW89_FCC][33] = 42,
+	[2][1][RTW89_IC][31] = 48,
+	[2][1][RTW89_KCC][31] = 46,
+	[2][1][RTW89_ACMA][31] = 40,
+	[2][1][RTW89_CN][31] = 127,
+	[2][1][RTW89_UK][31] = 40,
+	[2][1][RTW89_FCC][33] = 48,
 	[2][1][RTW89_ETSI][33] = 40,
 	[2][1][RTW89_MKK][33] = 66,
-	[2][1][RTW89_IC][33] = 50,
-	[2][1][RTW89_ACMA][33] = 36,
-	[2][1][RTW89_FCC][35] = 42,
+	[2][1][RTW89_IC][33] = 48,
+	[2][1][RTW89_KCC][33] = 46,
+	[2][1][RTW89_ACMA][33] = 40,
+	[2][1][RTW89_CN][33] = 127,
+	[2][1][RTW89_UK][33] = 40,
+	[2][1][RTW89_FCC][35] = 48,
 	[2][1][RTW89_ETSI][35] = 40,
 	[2][1][RTW89_MKK][35] = 66,
-	[2][1][RTW89_IC][35] = 50,
-	[2][1][RTW89_ACMA][35] = 36,
-	[2][1][RTW89_FCC][37] = 42,
+	[2][1][RTW89_IC][35] = 48,
+	[2][1][RTW89_KCC][35] = 46,
+	[2][1][RTW89_ACMA][35] = 40,
+	[2][1][RTW89_CN][35] = 127,
+	[2][1][RTW89_UK][35] = 40,
+	[2][1][RTW89_FCC][37] = 52,
 	[2][1][RTW89_ETSI][37] = 127,
 	[2][1][RTW89_MKK][37] = 66,
-	[2][1][RTW89_IC][37] = 50,
-	[2][1][RTW89_ACMA][37] = 60,
-	[2][1][RTW89_FCC][38] = 76,
+	[2][1][RTW89_IC][37] = 52,
+	[2][1][RTW89_KCC][37] = 46,
+	[2][1][RTW89_ACMA][37] = 52,
+	[2][1][RTW89_CN][37] = 127,
+	[2][1][RTW89_UK][37] = 42,
+	[2][1][RTW89_FCC][38] = 78,
 	[2][1][RTW89_ETSI][38] = 16,
 	[2][1][RTW89_MKK][38] = 127,
-	[2][1][RTW89_IC][38] = 84,
-	[2][1][RTW89_ACMA][38] = 84,
-	[2][1][RTW89_FCC][40] = 76,
+	[2][1][RTW89_IC][38] = 78,
+	[2][1][RTW89_KCC][38] = 46,
+	[2][1][RTW89_ACMA][38] = 78,
+	[2][1][RTW89_CN][38] = 56,
+	[2][1][RTW89_UK][38] = 42,
+	[2][1][RTW89_FCC][40] = 78,
 	[2][1][RTW89_ETSI][40] = 16,
 	[2][1][RTW89_MKK][40] = 127,
-	[2][1][RTW89_IC][40] = 84,
-	[2][1][RTW89_ACMA][40] = 84,
-	[2][1][RTW89_FCC][42] = 76,
+	[2][1][RTW89_IC][40] = 78,
+	[2][1][RTW89_KCC][40] = 46,
+	[2][1][RTW89_ACMA][40] = 78,
+	[2][1][RTW89_CN][40] = 56,
+	[2][1][RTW89_UK][40] = 42,
+	[2][1][RTW89_FCC][42] = 78,
 	[2][1][RTW89_ETSI][42] = 16,
 	[2][1][RTW89_MKK][42] = 127,
-	[2][1][RTW89_IC][42] = 84,
-	[2][1][RTW89_ACMA][42] = 84,
-	[2][1][RTW89_FCC][44] = 76,
+	[2][1][RTW89_IC][42] = 78,
+	[2][1][RTW89_KCC][42] = 46,
+	[2][1][RTW89_ACMA][42] = 78,
+	[2][1][RTW89_CN][42] = 56,
+	[2][1][RTW89_UK][42] = 42,
+	[2][1][RTW89_FCC][44] = 74,
 	[2][1][RTW89_ETSI][44] = 16,
 	[2][1][RTW89_MKK][44] = 127,
-	[2][1][RTW89_IC][44] = 84,
-	[2][1][RTW89_ACMA][44] = 84,
-	[2][1][RTW89_FCC][46] = 76,
+	[2][1][RTW89_IC][44] = 74,
+	[2][1][RTW89_KCC][44] = 46,
+	[2][1][RTW89_ACMA][44] = 74,
+	[2][1][RTW89_CN][44] = 56,
+	[2][1][RTW89_UK][44] = 42,
+	[2][1][RTW89_FCC][46] = 74,
 	[2][1][RTW89_ETSI][46] = 16,
 	[2][1][RTW89_MKK][46] = 127,
-	[2][1][RTW89_IC][46] = 84,
-	[2][1][RTW89_ACMA][46] = 84,
-	[2][1][RTW89_FCC][48] = 36,
+	[2][1][RTW89_IC][46] = 74,
+	[2][1][RTW89_KCC][46] = 46,
+	[2][1][RTW89_ACMA][46] = 74,
+	[2][1][RTW89_CN][46] = 56,
+	[2][1][RTW89_UK][46] = 42,
+	[2][1][RTW89_FCC][48] = 40,
 	[2][1][RTW89_ETSI][48] = 127,
 	[2][1][RTW89_MKK][48] = 127,
 	[2][1][RTW89_IC][48] = 127,
+	[2][1][RTW89_KCC][48] = 127,
 	[2][1][RTW89_ACMA][48] = 127,
-	[2][1][RTW89_FCC][50] = 36,
+	[2][1][RTW89_CN][48] = 127,
+	[2][1][RTW89_UK][48] = 127,
+	[2][1][RTW89_FCC][50] = 40,
 	[2][1][RTW89_ETSI][50] = 127,
 	[2][1][RTW89_MKK][50] = 127,
 	[2][1][RTW89_IC][50] = 127,
+	[2][1][RTW89_KCC][50] = 127,
 	[2][1][RTW89_ACMA][50] = 127,
-	[2][1][RTW89_FCC][52] = 36,
+	[2][1][RTW89_CN][50] = 127,
+	[2][1][RTW89_UK][50] = 127,
+	[2][1][RTW89_FCC][52] = 40,
 	[2][1][RTW89_ETSI][52] = 127,
 	[2][1][RTW89_MKK][52] = 127,
 	[2][1][RTW89_IC][52] = 127,
+	[2][1][RTW89_KCC][52] = 127,
 	[2][1][RTW89_ACMA][52] = 127,
+	[2][1][RTW89_CN][52] = 127,
+	[2][1][RTW89_UK][52] = 127,
 };
 
 const s8 rtw89_8852c_txpwr_lmt_ru_6g[RTW89_RU_NUM][RTW89_NTX_NUM]
 				    [RTW89_REGD_NUM][RTW89_6G_CH_NUM] = {
-	[0][0][RTW89_WW][0] = 76,
-	[0][0][RTW89_WW][2] = 76,
-	[0][0][RTW89_WW][4] = 76,
-	[0][0][RTW89_WW][6] = 76,
-	[0][0][RTW89_WW][8] = 76,
-	[0][0][RTW89_WW][10] = 76,
-	[0][0][RTW89_WW][12] = 76,
-	[0][0][RTW89_WW][14] = 76,
-	[0][0][RTW89_WW][15] = 76,
-	[0][0][RTW89_WW][17] = 76,
-	[0][0][RTW89_WW][19] = 76,
-	[0][0][RTW89_WW][21] = 76,
-	[0][0][RTW89_WW][23] = 76,
-	[0][0][RTW89_WW][25] = 76,
-	[0][0][RTW89_WW][27] = 76,
-	[0][0][RTW89_WW][29] = 76,
-	[0][0][RTW89_WW][30] = 76,
-	[0][0][RTW89_WW][32] = 76,
-	[0][0][RTW89_WW][34] = 76,
-	[0][0][RTW89_WW][36] = 76,
-	[0][0][RTW89_WW][38] = 76,
-	[0][0][RTW89_WW][40] = 76,
-	[0][0][RTW89_WW][42] = 76,
-	[0][0][RTW89_WW][44] = 76,
-	[0][0][RTW89_WW][45] = 76,
-	[0][0][RTW89_WW][47] = 76,
-	[0][0][RTW89_WW][49] = 76,
-	[0][0][RTW89_WW][51] = 76,
-	[0][0][RTW89_WW][53] = 76,
-	[0][0][RTW89_WW][55] = 76,
-	[0][0][RTW89_WW][57] = 76,
-	[0][0][RTW89_WW][59] = 76,
-	[0][0][RTW89_WW][60] = 76,
-	[0][0][RTW89_WW][62] = 76,
-	[0][0][RTW89_WW][64] = 76,
-	[0][0][RTW89_WW][66] = 76,
-	[0][0][RTW89_WW][68] = 76,
-	[0][0][RTW89_WW][70] = 76,
-	[0][0][RTW89_WW][72] = 76,
-	[0][0][RTW89_WW][74] = 76,
-	[0][0][RTW89_WW][75] = 76,
-	[0][0][RTW89_WW][77] = 76,
-	[0][0][RTW89_WW][79] = 76,
-	[0][0][RTW89_WW][81] = 76,
-	[0][0][RTW89_WW][83] = 76,
-	[0][0][RTW89_WW][85] = 76,
-	[0][0][RTW89_WW][87] = 76,
-	[0][0][RTW89_WW][89] = 76,
-	[0][0][RTW89_WW][90] = 76,
-	[0][0][RTW89_WW][92] = 76,
-	[0][0][RTW89_WW][94] = 76,
-	[0][0][RTW89_WW][96] = 76,
-	[0][0][RTW89_WW][98] = 76,
-	[0][0][RTW89_WW][100] = 76,
-	[0][0][RTW89_WW][102] = 76,
-	[0][0][RTW89_WW][104] = 76,
-	[0][0][RTW89_WW][105] = 76,
-	[0][0][RTW89_WW][107] = 76,
-	[0][0][RTW89_WW][109] = 76,
+	[0][0][RTW89_WW][0] = -16,
+	[0][0][RTW89_WW][2] = -18,
+	[0][0][RTW89_WW][4] = -18,
+	[0][0][RTW89_WW][6] = -18,
+	[0][0][RTW89_WW][8] = -18,
+	[0][0][RTW89_WW][10] = -18,
+	[0][0][RTW89_WW][12] = -18,
+	[0][0][RTW89_WW][14] = -18,
+	[0][0][RTW89_WW][15] = -18,
+	[0][0][RTW89_WW][17] = -18,
+	[0][0][RTW89_WW][19] = -18,
+	[0][0][RTW89_WW][21] = -18,
+	[0][0][RTW89_WW][23] = -18,
+	[0][0][RTW89_WW][25] = -18,
+	[0][0][RTW89_WW][27] = -18,
+	[0][0][RTW89_WW][29] = -18,
+	[0][0][RTW89_WW][30] = -18,
+	[0][0][RTW89_WW][32] = -18,
+	[0][0][RTW89_WW][34] = -18,
+	[0][0][RTW89_WW][36] = -18,
+	[0][0][RTW89_WW][38] = -18,
+	[0][0][RTW89_WW][40] = -18,
+	[0][0][RTW89_WW][42] = -18,
+	[0][0][RTW89_WW][44] = -16,
+	[0][0][RTW89_WW][45] = -16,
+	[0][0][RTW89_WW][47] = -18,
+	[0][0][RTW89_WW][49] = -18,
+	[0][0][RTW89_WW][51] = -18,
+	[0][0][RTW89_WW][53] = -16,
+	[0][0][RTW89_WW][55] = -18,
+	[0][0][RTW89_WW][57] = -18,
+	[0][0][RTW89_WW][59] = -18,
+	[0][0][RTW89_WW][60] = -18,
+	[0][0][RTW89_WW][62] = -18,
+	[0][0][RTW89_WW][64] = -18,
+	[0][0][RTW89_WW][66] = -18,
+	[0][0][RTW89_WW][68] = -18,
+	[0][0][RTW89_WW][70] = -16,
+	[0][0][RTW89_WW][72] = -18,
+	[0][0][RTW89_WW][74] = -18,
+	[0][0][RTW89_WW][75] = -18,
+	[0][0][RTW89_WW][77] = -18,
+	[0][0][RTW89_WW][79] = -18,
+	[0][0][RTW89_WW][81] = -18,
+	[0][0][RTW89_WW][83] = -18,
+	[0][0][RTW89_WW][85] = -18,
+	[0][0][RTW89_WW][87] = -16,
+	[0][0][RTW89_WW][89] = -16,
+	[0][0][RTW89_WW][90] = -16,
+	[0][0][RTW89_WW][92] = -16,
+	[0][0][RTW89_WW][94] = -16,
+	[0][0][RTW89_WW][96] = -16,
+	[0][0][RTW89_WW][98] = -16,
+	[0][0][RTW89_WW][100] = -16,
+	[0][0][RTW89_WW][102] = -16,
+	[0][0][RTW89_WW][104] = -16,
+	[0][0][RTW89_WW][105] = -16,
+	[0][0][RTW89_WW][107] = -12,
+	[0][0][RTW89_WW][109] = -12,
 	[0][0][RTW89_WW][111] = 0,
 	[0][0][RTW89_WW][113] = 0,
 	[0][0][RTW89_WW][115] = 0,
 	[0][0][RTW89_WW][117] = 0,
 	[0][0][RTW89_WW][119] = 0,
-	[0][1][RTW89_WW][0] = 76,
-	[0][1][RTW89_WW][2] = 76,
-	[0][1][RTW89_WW][4] = 76,
-	[0][1][RTW89_WW][6] = 76,
-	[0][1][RTW89_WW][8] = 76,
-	[0][1][RTW89_WW][10] = 76,
-	[0][1][RTW89_WW][12] = 76,
-	[0][1][RTW89_WW][14] = 76,
-	[0][1][RTW89_WW][15] = 76,
-	[0][1][RTW89_WW][17] = 76,
-	[0][1][RTW89_WW][19] = 76,
-	[0][1][RTW89_WW][21] = 76,
-	[0][1][RTW89_WW][23] = 76,
-	[0][1][RTW89_WW][25] = 76,
-	[0][1][RTW89_WW][27] = 76,
-	[0][1][RTW89_WW][29] = 76,
-	[0][1][RTW89_WW][30] = 76,
-	[0][1][RTW89_WW][32] = 76,
-	[0][1][RTW89_WW][34] = 76,
-	[0][1][RTW89_WW][36] = 76,
-	[0][1][RTW89_WW][38] = 76,
-	[0][1][RTW89_WW][40] = 76,
-	[0][1][RTW89_WW][42] = 76,
-	[0][1][RTW89_WW][44] = 76,
-	[0][1][RTW89_WW][45] = 76,
-	[0][1][RTW89_WW][47] = 76,
-	[0][1][RTW89_WW][49] = 76,
-	[0][1][RTW89_WW][51] = 76,
-	[0][1][RTW89_WW][53] = 76,
-	[0][1][RTW89_WW][55] = 76,
-	[0][1][RTW89_WW][57] = 76,
-	[0][1][RTW89_WW][59] = 76,
-	[0][1][RTW89_WW][60] = 76,
-	[0][1][RTW89_WW][62] = 76,
-	[0][1][RTW89_WW][64] = 76,
-	[0][1][RTW89_WW][66] = 76,
-	[0][1][RTW89_WW][68] = 76,
-	[0][1][RTW89_WW][70] = 76,
-	[0][1][RTW89_WW][72] = 76,
-	[0][1][RTW89_WW][74] = 76,
-	[0][1][RTW89_WW][75] = 76,
-	[0][1][RTW89_WW][77] = 76,
-	[0][1][RTW89_WW][79] = 76,
-	[0][1][RTW89_WW][81] = 76,
-	[0][1][RTW89_WW][83] = 76,
-	[0][1][RTW89_WW][85] = 76,
-	[0][1][RTW89_WW][87] = 76,
-	[0][1][RTW89_WW][89] = 76,
-	[0][1][RTW89_WW][90] = 76,
-	[0][1][RTW89_WW][92] = 76,
-	[0][1][RTW89_WW][94] = 76,
-	[0][1][RTW89_WW][96] = 76,
-	[0][1][RTW89_WW][98] = 76,
-	[0][1][RTW89_WW][100] = 76,
-	[0][1][RTW89_WW][102] = 76,
-	[0][1][RTW89_WW][104] = 76,
-	[0][1][RTW89_WW][105] = 76,
-	[0][1][RTW89_WW][107] = 76,
-	[0][1][RTW89_WW][109] = 76,
+	[0][1][RTW89_WW][0] = -40,
+	[0][1][RTW89_WW][2] = -40,
+	[0][1][RTW89_WW][4] = -40,
+	[0][1][RTW89_WW][6] = -40,
+	[0][1][RTW89_WW][8] = -40,
+	[0][1][RTW89_WW][10] = -40,
+	[0][1][RTW89_WW][12] = -40,
+	[0][1][RTW89_WW][14] = -40,
+	[0][1][RTW89_WW][15] = -40,
+	[0][1][RTW89_WW][17] = -40,
+	[0][1][RTW89_WW][19] = -40,
+	[0][1][RTW89_WW][21] = -40,
+	[0][1][RTW89_WW][23] = -40,
+	[0][1][RTW89_WW][25] = -40,
+	[0][1][RTW89_WW][27] = -40,
+	[0][1][RTW89_WW][29] = -40,
+	[0][1][RTW89_WW][30] = -40,
+	[0][1][RTW89_WW][32] = -40,
+	[0][1][RTW89_WW][34] = -40,
+	[0][1][RTW89_WW][36] = -40,
+	[0][1][RTW89_WW][38] = -40,
+	[0][1][RTW89_WW][40] = -40,
+	[0][1][RTW89_WW][42] = -40,
+	[0][1][RTW89_WW][44] = -40,
+	[0][1][RTW89_WW][45] = -40,
+	[0][1][RTW89_WW][47] = -40,
+	[0][1][RTW89_WW][49] = -40,
+	[0][1][RTW89_WW][51] = -40,
+	[0][1][RTW89_WW][53] = -40,
+	[0][1][RTW89_WW][55] = -40,
+	[0][1][RTW89_WW][57] = -40,
+	[0][1][RTW89_WW][59] = -40,
+	[0][1][RTW89_WW][60] = -40,
+	[0][1][RTW89_WW][62] = -40,
+	[0][1][RTW89_WW][64] = -40,
+	[0][1][RTW89_WW][66] = -40,
+	[0][1][RTW89_WW][68] = -40,
+	[0][1][RTW89_WW][70] = -38,
+	[0][1][RTW89_WW][72] = -38,
+	[0][1][RTW89_WW][74] = -38,
+	[0][1][RTW89_WW][75] = -38,
+	[0][1][RTW89_WW][77] = -38,
+	[0][1][RTW89_WW][79] = -38,
+	[0][1][RTW89_WW][81] = -38,
+	[0][1][RTW89_WW][83] = -38,
+	[0][1][RTW89_WW][85] = -38,
+	[0][1][RTW89_WW][87] = -40,
+	[0][1][RTW89_WW][89] = -38,
+	[0][1][RTW89_WW][90] = -38,
+	[0][1][RTW89_WW][92] = -38,
+	[0][1][RTW89_WW][94] = -38,
+	[0][1][RTW89_WW][96] = -38,
+	[0][1][RTW89_WW][98] = -38,
+	[0][1][RTW89_WW][100] = -38,
+	[0][1][RTW89_WW][102] = -38,
+	[0][1][RTW89_WW][104] = -38,
+	[0][1][RTW89_WW][105] = -38,
+	[0][1][RTW89_WW][107] = -34,
+	[0][1][RTW89_WW][109] = -34,
 	[0][1][RTW89_WW][111] = 0,
 	[0][1][RTW89_WW][113] = 0,
 	[0][1][RTW89_WW][115] = 0,
 	[0][1][RTW89_WW][117] = 0,
 	[0][1][RTW89_WW][119] = 0,
-	[1][0][RTW89_WW][0] = 76,
-	[1][0][RTW89_WW][2] = 76,
-	[1][0][RTW89_WW][4] = 76,
-	[1][0][RTW89_WW][6] = 76,
-	[1][0][RTW89_WW][8] = 76,
-	[1][0][RTW89_WW][10] = 76,
-	[1][0][RTW89_WW][12] = 76,
-	[1][0][RTW89_WW][14] = 76,
-	[1][0][RTW89_WW][15] = 76,
-	[1][0][RTW89_WW][17] = 76,
-	[1][0][RTW89_WW][19] = 76,
-	[1][0][RTW89_WW][21] = 76,
-	[1][0][RTW89_WW][23] = 76,
-	[1][0][RTW89_WW][25] = 76,
-	[1][0][RTW89_WW][27] = 76,
-	[1][0][RTW89_WW][29] = 76,
-	[1][0][RTW89_WW][30] = 76,
-	[1][0][RTW89_WW][32] = 76,
-	[1][0][RTW89_WW][34] = 76,
-	[1][0][RTW89_WW][36] = 76,
-	[1][0][RTW89_WW][38] = 76,
-	[1][0][RTW89_WW][40] = 76,
-	[1][0][RTW89_WW][42] = 76,
-	[1][0][RTW89_WW][44] = 76,
-	[1][0][RTW89_WW][45] = 76,
-	[1][0][RTW89_WW][47] = 76,
-	[1][0][RTW89_WW][49] = 76,
-	[1][0][RTW89_WW][51] = 76,
-	[1][0][RTW89_WW][53] = 76,
-	[1][0][RTW89_WW][55] = 76,
-	[1][0][RTW89_WW][57] = 76,
-	[1][0][RTW89_WW][59] = 76,
-	[1][0][RTW89_WW][60] = 76,
-	[1][0][RTW89_WW][62] = 76,
-	[1][0][RTW89_WW][64] = 76,
-	[1][0][RTW89_WW][66] = 76,
-	[1][0][RTW89_WW][68] = 76,
-	[1][0][RTW89_WW][70] = 76,
-	[1][0][RTW89_WW][72] = 76,
-	[1][0][RTW89_WW][74] = 76,
-	[1][0][RTW89_WW][75] = 76,
-	[1][0][RTW89_WW][77] = 76,
-	[1][0][RTW89_WW][79] = 76,
-	[1][0][RTW89_WW][81] = 76,
-	[1][0][RTW89_WW][83] = 76,
-	[1][0][RTW89_WW][85] = 76,
-	[1][0][RTW89_WW][87] = 76,
-	[1][0][RTW89_WW][89] = 76,
-	[1][0][RTW89_WW][90] = 76,
-	[1][0][RTW89_WW][92] = 76,
-	[1][0][RTW89_WW][94] = 76,
-	[1][0][RTW89_WW][96] = 76,
-	[1][0][RTW89_WW][98] = 76,
-	[1][0][RTW89_WW][100] = 76,
-	[1][0][RTW89_WW][102] = 76,
-	[1][0][RTW89_WW][104] = 76,
-	[1][0][RTW89_WW][105] = 76,
-	[1][0][RTW89_WW][107] = 76,
-	[1][0][RTW89_WW][109] = 76,
+	[1][0][RTW89_WW][0] = -4,
+	[1][0][RTW89_WW][2] = -4,
+	[1][0][RTW89_WW][4] = -4,
+	[1][0][RTW89_WW][6] = -4,
+	[1][0][RTW89_WW][8] = -4,
+	[1][0][RTW89_WW][10] = -4,
+	[1][0][RTW89_WW][12] = -4,
+	[1][0][RTW89_WW][14] = -4,
+	[1][0][RTW89_WW][15] = -4,
+	[1][0][RTW89_WW][17] = -4,
+	[1][0][RTW89_WW][19] = -4,
+	[1][0][RTW89_WW][21] = -4,
+	[1][0][RTW89_WW][23] = -4,
+	[1][0][RTW89_WW][25] = -4,
+	[1][0][RTW89_WW][27] = -4,
+	[1][0][RTW89_WW][29] = -4,
+	[1][0][RTW89_WW][30] = -4,
+	[1][0][RTW89_WW][32] = -4,
+	[1][0][RTW89_WW][34] = -4,
+	[1][0][RTW89_WW][36] = -4,
+	[1][0][RTW89_WW][38] = -4,
+	[1][0][RTW89_WW][40] = -4,
+	[1][0][RTW89_WW][42] = -4,
+	[1][0][RTW89_WW][44] = -4,
+	[1][0][RTW89_WW][45] = -4,
+	[1][0][RTW89_WW][47] = -4,
+	[1][0][RTW89_WW][49] = -4,
+	[1][0][RTW89_WW][51] = -4,
+	[1][0][RTW89_WW][53] = -4,
+	[1][0][RTW89_WW][55] = -4,
+	[1][0][RTW89_WW][57] = -4,
+	[1][0][RTW89_WW][59] = -4,
+	[1][0][RTW89_WW][60] = -4,
+	[1][0][RTW89_WW][62] = -4,
+	[1][0][RTW89_WW][64] = -4,
+	[1][0][RTW89_WW][66] = -4,
+	[1][0][RTW89_WW][68] = -4,
+	[1][0][RTW89_WW][70] = -4,
+	[1][0][RTW89_WW][72] = -4,
+	[1][0][RTW89_WW][74] = -4,
+	[1][0][RTW89_WW][75] = -4,
+	[1][0][RTW89_WW][77] = -4,
+	[1][0][RTW89_WW][79] = -4,
+	[1][0][RTW89_WW][81] = -4,
+	[1][0][RTW89_WW][83] = -4,
+	[1][0][RTW89_WW][85] = -4,
+	[1][0][RTW89_WW][87] = -4,
+	[1][0][RTW89_WW][89] = -4,
+	[1][0][RTW89_WW][90] = -4,
+	[1][0][RTW89_WW][92] = -4,
+	[1][0][RTW89_WW][94] = -4,
+	[1][0][RTW89_WW][96] = -4,
+	[1][0][RTW89_WW][98] = -4,
+	[1][0][RTW89_WW][100] = -4,
+	[1][0][RTW89_WW][102] = -4,
+	[1][0][RTW89_WW][104] = -4,
+	[1][0][RTW89_WW][105] = -4,
+	[1][0][RTW89_WW][107] = 1,
+	[1][0][RTW89_WW][109] = 2,
 	[1][0][RTW89_WW][111] = 0,
 	[1][0][RTW89_WW][113] = 0,
 	[1][0][RTW89_WW][115] = 0,
 	[1][0][RTW89_WW][117] = 0,
 	[1][0][RTW89_WW][119] = 0,
-	[1][1][RTW89_WW][0] = 76,
-	[1][1][RTW89_WW][2] = 76,
-	[1][1][RTW89_WW][4] = 76,
-	[1][1][RTW89_WW][6] = 76,
-	[1][1][RTW89_WW][8] = 76,
-	[1][1][RTW89_WW][10] = 76,
-	[1][1][RTW89_WW][12] = 76,
-	[1][1][RTW89_WW][14] = 76,
-	[1][1][RTW89_WW][15] = 76,
-	[1][1][RTW89_WW][17] = 76,
-	[1][1][RTW89_WW][19] = 76,
-	[1][1][RTW89_WW][21] = 76,
-	[1][1][RTW89_WW][23] = 76,
-	[1][1][RTW89_WW][25] = 76,
-	[1][1][RTW89_WW][27] = 76,
-	[1][1][RTW89_WW][29] = 76,
-	[1][1][RTW89_WW][30] = 76,
-	[1][1][RTW89_WW][32] = 76,
-	[1][1][RTW89_WW][34] = 76,
-	[1][1][RTW89_WW][36] = 76,
-	[1][1][RTW89_WW][38] = 76,
-	[1][1][RTW89_WW][40] = 76,
-	[1][1][RTW89_WW][42] = 76,
-	[1][1][RTW89_WW][44] = 76,
-	[1][1][RTW89_WW][45] = 76,
-	[1][1][RTW89_WW][47] = 76,
-	[1][1][RTW89_WW][49] = 76,
-	[1][1][RTW89_WW][51] = 76,
-	[1][1][RTW89_WW][53] = 76,
-	[1][1][RTW89_WW][55] = 76,
-	[1][1][RTW89_WW][57] = 76,
-	[1][1][RTW89_WW][59] = 76,
-	[1][1][RTW89_WW][60] = 76,
-	[1][1][RTW89_WW][62] = 76,
-	[1][1][RTW89_WW][64] = 76,
-	[1][1][RTW89_WW][66] = 76,
-	[1][1][RTW89_WW][68] = 76,
-	[1][1][RTW89_WW][70] = 76,
-	[1][1][RTW89_WW][72] = 76,
-	[1][1][RTW89_WW][74] = 76,
-	[1][1][RTW89_WW][75] = 76,
-	[1][1][RTW89_WW][77] = 76,
-	[1][1][RTW89_WW][79] = 76,
-	[1][1][RTW89_WW][81] = 76,
-	[1][1][RTW89_WW][83] = 76,
-	[1][1][RTW89_WW][85] = 76,
-	[1][1][RTW89_WW][87] = 76,
-	[1][1][RTW89_WW][89] = 76,
-	[1][1][RTW89_WW][90] = 76,
-	[1][1][RTW89_WW][92] = 76,
-	[1][1][RTW89_WW][94] = 76,
-	[1][1][RTW89_WW][96] = 76,
-	[1][1][RTW89_WW][98] = 76,
-	[1][1][RTW89_WW][100] = 76,
-	[1][1][RTW89_WW][102] = 76,
-	[1][1][RTW89_WW][104] = 76,
-	[1][1][RTW89_WW][105] = 76,
-	[1][1][RTW89_WW][107] = 76,
-	[1][1][RTW89_WW][109] = 76,
+	[1][1][RTW89_WW][0] = -26,
+	[1][1][RTW89_WW][2] = -28,
+	[1][1][RTW89_WW][4] = -28,
+	[1][1][RTW89_WW][6] = -28,
+	[1][1][RTW89_WW][8] = -28,
+	[1][1][RTW89_WW][10] = -28,
+	[1][1][RTW89_WW][12] = -28,
+	[1][1][RTW89_WW][14] = -28,
+	[1][1][RTW89_WW][15] = -28,
+	[1][1][RTW89_WW][17] = -28,
+	[1][1][RTW89_WW][19] = -28,
+	[1][1][RTW89_WW][21] = -28,
+	[1][1][RTW89_WW][23] = -28,
+	[1][1][RTW89_WW][25] = -28,
+	[1][1][RTW89_WW][27] = -28,
+	[1][1][RTW89_WW][29] = -28,
+	[1][1][RTW89_WW][30] = -28,
+	[1][1][RTW89_WW][32] = -28,
+	[1][1][RTW89_WW][34] = -28,
+	[1][1][RTW89_WW][36] = -28,
+	[1][1][RTW89_WW][38] = -28,
+	[1][1][RTW89_WW][40] = -28,
+	[1][1][RTW89_WW][42] = -28,
+	[1][1][RTW89_WW][44] = -28,
+	[1][1][RTW89_WW][45] = -26,
+	[1][1][RTW89_WW][47] = -28,
+	[1][1][RTW89_WW][49] = -28,
+	[1][1][RTW89_WW][51] = -28,
+	[1][1][RTW89_WW][53] = -26,
+	[1][1][RTW89_WW][55] = -28,
+	[1][1][RTW89_WW][57] = -28,
+	[1][1][RTW89_WW][59] = -28,
+	[1][1][RTW89_WW][60] = -28,
+	[1][1][RTW89_WW][62] = -28,
+	[1][1][RTW89_WW][64] = -28,
+	[1][1][RTW89_WW][66] = -28,
+	[1][1][RTW89_WW][68] = -28,
+	[1][1][RTW89_WW][70] = -26,
+	[1][1][RTW89_WW][72] = -28,
+	[1][1][RTW89_WW][74] = -28,
+	[1][1][RTW89_WW][75] = -28,
+	[1][1][RTW89_WW][77] = -28,
+	[1][1][RTW89_WW][79] = -28,
+	[1][1][RTW89_WW][81] = -28,
+	[1][1][RTW89_WW][83] = -28,
+	[1][1][RTW89_WW][85] = -28,
+	[1][1][RTW89_WW][87] = -28,
+	[1][1][RTW89_WW][89] = -26,
+	[1][1][RTW89_WW][90] = -26,
+	[1][1][RTW89_WW][92] = -26,
+	[1][1][RTW89_WW][94] = -26,
+	[1][1][RTW89_WW][96] = -26,
+	[1][1][RTW89_WW][98] = -26,
+	[1][1][RTW89_WW][100] = -26,
+	[1][1][RTW89_WW][102] = -26,
+	[1][1][RTW89_WW][104] = -26,
+	[1][1][RTW89_WW][105] = -26,
+	[1][1][RTW89_WW][107] = -22,
+	[1][1][RTW89_WW][109] = -22,
 	[1][1][RTW89_WW][111] = 0,
 	[1][1][RTW89_WW][113] = 0,
 	[1][1][RTW89_WW][115] = 0,
 	[1][1][RTW89_WW][117] = 0,
 	[1][1][RTW89_WW][119] = 0,
-	[2][0][RTW89_WW][0] = 76,
-	[2][0][RTW89_WW][2] = 76,
-	[2][0][RTW89_WW][4] = 76,
-	[2][0][RTW89_WW][6] = 76,
-	[2][0][RTW89_WW][8] = 76,
-	[2][0][RTW89_WW][10] = 76,
-	[2][0][RTW89_WW][12] = 76,
-	[2][0][RTW89_WW][14] = 76,
-	[2][0][RTW89_WW][15] = 76,
-	[2][0][RTW89_WW][17] = 76,
-	[2][0][RTW89_WW][19] = 76,
-	[2][0][RTW89_WW][21] = 76,
-	[2][0][RTW89_WW][23] = 76,
-	[2][0][RTW89_WW][25] = 76,
-	[2][0][RTW89_WW][27] = 76,
-	[2][0][RTW89_WW][29] = 76,
-	[2][0][RTW89_WW][30] = 76,
-	[2][0][RTW89_WW][32] = 76,
-	[2][0][RTW89_WW][34] = 76,
-	[2][0][RTW89_WW][36] = 76,
-	[2][0][RTW89_WW][38] = 76,
-	[2][0][RTW89_WW][40] = 76,
-	[2][0][RTW89_WW][42] = 76,
-	[2][0][RTW89_WW][44] = 76,
-	[2][0][RTW89_WW][45] = 76,
-	[2][0][RTW89_WW][47] = 76,
-	[2][0][RTW89_WW][49] = 76,
-	[2][0][RTW89_WW][51] = 76,
-	[2][0][RTW89_WW][53] = 76,
-	[2][0][RTW89_WW][55] = 76,
-	[2][0][RTW89_WW][57] = 76,
-	[2][0][RTW89_WW][59] = 76,
-	[2][0][RTW89_WW][60] = 76,
-	[2][0][RTW89_WW][62] = 76,
-	[2][0][RTW89_WW][64] = 76,
-	[2][0][RTW89_WW][66] = 76,
-	[2][0][RTW89_WW][68] = 76,
-	[2][0][RTW89_WW][70] = 76,
-	[2][0][RTW89_WW][72] = 76,
-	[2][0][RTW89_WW][74] = 76,
-	[2][0][RTW89_WW][75] = 76,
-	[2][0][RTW89_WW][77] = 76,
-	[2][0][RTW89_WW][79] = 76,
-	[2][0][RTW89_WW][81] = 76,
-	[2][0][RTW89_WW][83] = 76,
-	[2][0][RTW89_WW][85] = 76,
-	[2][0][RTW89_WW][87] = 76,
-	[2][0][RTW89_WW][89] = 76,
-	[2][0][RTW89_WW][90] = 76,
-	[2][0][RTW89_WW][92] = 76,
-	[2][0][RTW89_WW][94] = 76,
-	[2][0][RTW89_WW][96] = 76,
-	[2][0][RTW89_WW][98] = 76,
-	[2][0][RTW89_WW][100] = 76,
-	[2][0][RTW89_WW][102] = 76,
-	[2][0][RTW89_WW][104] = 76,
-	[2][0][RTW89_WW][105] = 76,
-	[2][0][RTW89_WW][107] = 76,
-	[2][0][RTW89_WW][109] = 76,
+	[2][0][RTW89_WW][0] = 8,
+	[2][0][RTW89_WW][2] = 8,
+	[2][0][RTW89_WW][4] = 8,
+	[2][0][RTW89_WW][6] = 8,
+	[2][0][RTW89_WW][8] = 8,
+	[2][0][RTW89_WW][10] = 8,
+	[2][0][RTW89_WW][12] = 8,
+	[2][0][RTW89_WW][14] = 8,
+	[2][0][RTW89_WW][15] = 8,
+	[2][0][RTW89_WW][17] = 8,
+	[2][0][RTW89_WW][19] = 8,
+	[2][0][RTW89_WW][21] = 8,
+	[2][0][RTW89_WW][23] = 8,
+	[2][0][RTW89_WW][25] = 8,
+	[2][0][RTW89_WW][27] = 8,
+	[2][0][RTW89_WW][29] = 8,
+	[2][0][RTW89_WW][30] = 8,
+	[2][0][RTW89_WW][32] = 8,
+	[2][0][RTW89_WW][34] = 8,
+	[2][0][RTW89_WW][36] = 8,
+	[2][0][RTW89_WW][38] = 8,
+	[2][0][RTW89_WW][40] = 8,
+	[2][0][RTW89_WW][42] = 8,
+	[2][0][RTW89_WW][44] = 8,
+	[2][0][RTW89_WW][45] = 8,
+	[2][0][RTW89_WW][47] = 8,
+	[2][0][RTW89_WW][49] = 8,
+	[2][0][RTW89_WW][51] = 8,
+	[2][0][RTW89_WW][53] = 8,
+	[2][0][RTW89_WW][55] = 8,
+	[2][0][RTW89_WW][57] = 8,
+	[2][0][RTW89_WW][59] = 8,
+	[2][0][RTW89_WW][60] = 8,
+	[2][0][RTW89_WW][62] = 8,
+	[2][0][RTW89_WW][64] = 8,
+	[2][0][RTW89_WW][66] = 8,
+	[2][0][RTW89_WW][68] = 8,
+	[2][0][RTW89_WW][70] = 8,
+	[2][0][RTW89_WW][72] = 8,
+	[2][0][RTW89_WW][74] = 8,
+	[2][0][RTW89_WW][75] = 8,
+	[2][0][RTW89_WW][77] = 8,
+	[2][0][RTW89_WW][79] = 8,
+	[2][0][RTW89_WW][81] = 8,
+	[2][0][RTW89_WW][83] = 8,
+	[2][0][RTW89_WW][85] = 8,
+	[2][0][RTW89_WW][87] = 8,
+	[2][0][RTW89_WW][89] = 8,
+	[2][0][RTW89_WW][90] = 8,
+	[2][0][RTW89_WW][92] = 8,
+	[2][0][RTW89_WW][94] = 8,
+	[2][0][RTW89_WW][96] = 8,
+	[2][0][RTW89_WW][98] = 8,
+	[2][0][RTW89_WW][100] = 8,
+	[2][0][RTW89_WW][102] = 8,
+	[2][0][RTW89_WW][104] = 8,
+	[2][0][RTW89_WW][105] = 8,
+	[2][0][RTW89_WW][107] = 10,
+	[2][0][RTW89_WW][109] = 12,
 	[2][0][RTW89_WW][111] = 0,
 	[2][0][RTW89_WW][113] = 0,
 	[2][0][RTW89_WW][115] = 0,
 	[2][0][RTW89_WW][117] = 0,
 	[2][0][RTW89_WW][119] = 0,
-	[2][1][RTW89_WW][0] = 76,
-	[2][1][RTW89_WW][2] = 76,
-	[2][1][RTW89_WW][4] = 76,
-	[2][1][RTW89_WW][6] = 76,
-	[2][1][RTW89_WW][8] = 76,
-	[2][1][RTW89_WW][10] = 76,
-	[2][1][RTW89_WW][12] = 76,
-	[2][1][RTW89_WW][14] = 76,
-	[2][1][RTW89_WW][15] = 76,
-	[2][1][RTW89_WW][17] = 76,
-	[2][1][RTW89_WW][19] = 76,
-	[2][1][RTW89_WW][21] = 76,
-	[2][1][RTW89_WW][23] = 76,
-	[2][1][RTW89_WW][25] = 76,
-	[2][1][RTW89_WW][27] = 76,
-	[2][1][RTW89_WW][29] = 76,
-	[2][1][RTW89_WW][30] = 76,
-	[2][1][RTW89_WW][32] = 76,
-	[2][1][RTW89_WW][34] = 76,
-	[2][1][RTW89_WW][36] = 76,
-	[2][1][RTW89_WW][38] = 76,
-	[2][1][RTW89_WW][40] = 76,
-	[2][1][RTW89_WW][42] = 76,
-	[2][1][RTW89_WW][44] = 76,
-	[2][1][RTW89_WW][45] = 76,
-	[2][1][RTW89_WW][47] = 76,
-	[2][1][RTW89_WW][49] = 76,
-	[2][1][RTW89_WW][51] = 76,
-	[2][1][RTW89_WW][53] = 76,
-	[2][1][RTW89_WW][55] = 76,
-	[2][1][RTW89_WW][57] = 76,
-	[2][1][RTW89_WW][59] = 76,
-	[2][1][RTW89_WW][60] = 76,
-	[2][1][RTW89_WW][62] = 76,
-	[2][1][RTW89_WW][64] = 76,
-	[2][1][RTW89_WW][66] = 76,
-	[2][1][RTW89_WW][68] = 76,
-	[2][1][RTW89_WW][70] = 76,
-	[2][1][RTW89_WW][72] = 76,
-	[2][1][RTW89_WW][74] = 76,
-	[2][1][RTW89_WW][75] = 76,
-	[2][1][RTW89_WW][77] = 76,
-	[2][1][RTW89_WW][79] = 76,
-	[2][1][RTW89_WW][81] = 76,
-	[2][1][RTW89_WW][83] = 76,
-	[2][1][RTW89_WW][85] = 76,
-	[2][1][RTW89_WW][87] = 76,
-	[2][1][RTW89_WW][89] = 76,
-	[2][1][RTW89_WW][90] = 76,
-	[2][1][RTW89_WW][92] = 76,
-	[2][1][RTW89_WW][94] = 76,
-	[2][1][RTW89_WW][96] = 76,
-	[2][1][RTW89_WW][98] = 76,
-	[2][1][RTW89_WW][100] = 76,
-	[2][1][RTW89_WW][102] = 76,
-	[2][1][RTW89_WW][104] = 76,
-	[2][1][RTW89_WW][105] = 76,
-	[2][1][RTW89_WW][107] = 76,
-	[2][1][RTW89_WW][109] = 76,
+	[2][1][RTW89_WW][0] = -16,
+	[2][1][RTW89_WW][2] = -16,
+	[2][1][RTW89_WW][4] = -16,
+	[2][1][RTW89_WW][6] = -16,
+	[2][1][RTW89_WW][8] = -16,
+	[2][1][RTW89_WW][10] = -16,
+	[2][1][RTW89_WW][12] = -16,
+	[2][1][RTW89_WW][14] = -16,
+	[2][1][RTW89_WW][15] = -16,
+	[2][1][RTW89_WW][17] = -16,
+	[2][1][RTW89_WW][19] = -16,
+	[2][1][RTW89_WW][21] = -16,
+	[2][1][RTW89_WW][23] = -16,
+	[2][1][RTW89_WW][25] = -16,
+	[2][1][RTW89_WW][27] = -16,
+	[2][1][RTW89_WW][29] = -16,
+	[2][1][RTW89_WW][30] = -16,
+	[2][1][RTW89_WW][32] = -16,
+	[2][1][RTW89_WW][34] = -16,
+	[2][1][RTW89_WW][36] = -16,
+	[2][1][RTW89_WW][38] = -16,
+	[2][1][RTW89_WW][40] = -16,
+	[2][1][RTW89_WW][42] = -16,
+	[2][1][RTW89_WW][44] = -16,
+	[2][1][RTW89_WW][45] = -16,
+	[2][1][RTW89_WW][47] = -16,
+	[2][1][RTW89_WW][49] = -16,
+	[2][1][RTW89_WW][51] = -16,
+	[2][1][RTW89_WW][53] = -16,
+	[2][1][RTW89_WW][55] = -16,
+	[2][1][RTW89_WW][57] = -16,
+	[2][1][RTW89_WW][59] = -16,
+	[2][1][RTW89_WW][60] = -16,
+	[2][1][RTW89_WW][62] = -16,
+	[2][1][RTW89_WW][64] = -16,
+	[2][1][RTW89_WW][66] = -16,
+	[2][1][RTW89_WW][68] = -16,
+	[2][1][RTW89_WW][70] = -16,
+	[2][1][RTW89_WW][72] = -16,
+	[2][1][RTW89_WW][74] = -16,
+	[2][1][RTW89_WW][75] = -16,
+	[2][1][RTW89_WW][77] = -16,
+	[2][1][RTW89_WW][79] = -16,
+	[2][1][RTW89_WW][81] = -16,
+	[2][1][RTW89_WW][83] = -16,
+	[2][1][RTW89_WW][85] = -18,
+	[2][1][RTW89_WW][87] = -16,
+	[2][1][RTW89_WW][89] = -16,
+	[2][1][RTW89_WW][90] = -16,
+	[2][1][RTW89_WW][92] = -16,
+	[2][1][RTW89_WW][94] = -16,
+	[2][1][RTW89_WW][96] = -16,
+	[2][1][RTW89_WW][98] = -16,
+	[2][1][RTW89_WW][100] = -16,
+	[2][1][RTW89_WW][102] = -16,
+	[2][1][RTW89_WW][104] = -16,
+	[2][1][RTW89_WW][105] = -16,
+	[2][1][RTW89_WW][107] = -12,
+	[2][1][RTW89_WW][109] = -10,
 	[2][1][RTW89_WW][111] = 0,
 	[2][1][RTW89_WW][113] = 0,
 	[2][1][RTW89_WW][115] = 0,
 	[2][1][RTW89_WW][117] = 0,
 	[2][1][RTW89_WW][119] = 0,
-	[0][0][RTW89_FCC][0] = 76,
-	[0][0][RTW89_FCC][2] = 76,
-	[0][0][RTW89_FCC][4] = 76,
-	[0][0][RTW89_FCC][6] = 76,
-	[0][0][RTW89_FCC][8] = 76,
-	[0][0][RTW89_FCC][10] = 76,
-	[0][0][RTW89_FCC][12] = 76,
-	[0][0][RTW89_FCC][14] = 76,
-	[0][0][RTW89_FCC][15] = 76,
-	[0][0][RTW89_FCC][17] = 76,
-	[0][0][RTW89_FCC][19] = 76,
-	[0][0][RTW89_FCC][21] = 76,
-	[0][0][RTW89_FCC][23] = 76,
-	[0][0][RTW89_FCC][25] = 76,
-	[0][0][RTW89_FCC][27] = 76,
-	[0][0][RTW89_FCC][29] = 76,
-	[0][0][RTW89_FCC][30] = 76,
-	[0][0][RTW89_FCC][32] = 76,
-	[0][0][RTW89_FCC][34] = 76,
-	[0][0][RTW89_FCC][36] = 76,
-	[0][0][RTW89_FCC][38] = 76,
-	[0][0][RTW89_FCC][40] = 76,
-	[0][0][RTW89_FCC][42] = 76,
-	[0][0][RTW89_FCC][44] = 76,
-	[0][0][RTW89_FCC][45] = 76,
-	[0][0][RTW89_FCC][47] = 76,
-	[0][0][RTW89_FCC][49] = 76,
-	[0][0][RTW89_FCC][51] = 76,
-	[0][0][RTW89_FCC][53] = 76,
-	[0][0][RTW89_FCC][55] = 76,
-	[0][0][RTW89_FCC][57] = 76,
-	[0][0][RTW89_FCC][59] = 76,
-	[0][0][RTW89_FCC][60] = 76,
-	[0][0][RTW89_FCC][62] = 76,
-	[0][0][RTW89_FCC][64] = 76,
-	[0][0][RTW89_FCC][66] = 76,
-	[0][0][RTW89_FCC][68] = 76,
-	[0][0][RTW89_FCC][70] = 76,
-	[0][0][RTW89_FCC][72] = 76,
-	[0][0][RTW89_FCC][74] = 76,
-	[0][0][RTW89_FCC][75] = 76,
-	[0][0][RTW89_FCC][77] = 76,
-	[0][0][RTW89_FCC][79] = 76,
-	[0][0][RTW89_FCC][81] = 76,
-	[0][0][RTW89_FCC][83] = 76,
-	[0][0][RTW89_FCC][85] = 76,
-	[0][0][RTW89_FCC][87] = 76,
-	[0][0][RTW89_FCC][89] = 76,
-	[0][0][RTW89_FCC][90] = 76,
-	[0][0][RTW89_FCC][92] = 76,
-	[0][0][RTW89_FCC][94] = 76,
-	[0][0][RTW89_FCC][96] = 76,
-	[0][0][RTW89_FCC][98] = 76,
-	[0][0][RTW89_FCC][100] = 76,
-	[0][0][RTW89_FCC][102] = 76,
-	[0][0][RTW89_FCC][104] = 76,
-	[0][0][RTW89_FCC][105] = 76,
-	[0][0][RTW89_FCC][107] = 76,
-	[0][0][RTW89_FCC][109] = 76,
+	[0][0][RTW89_FCC][0] = -16,
+	[0][0][RTW89_ETSI][0] = 32,
+	[0][0][RTW89_FCC][2] = -18,
+	[0][0][RTW89_ETSI][2] = 32,
+	[0][0][RTW89_FCC][4] = -18,
+	[0][0][RTW89_ETSI][4] = 32,
+	[0][0][RTW89_FCC][6] = -18,
+	[0][0][RTW89_ETSI][6] = 32,
+	[0][0][RTW89_FCC][8] = -18,
+	[0][0][RTW89_ETSI][8] = 32,
+	[0][0][RTW89_FCC][10] = -18,
+	[0][0][RTW89_ETSI][10] = 32,
+	[0][0][RTW89_FCC][12] = -18,
+	[0][0][RTW89_ETSI][12] = 32,
+	[0][0][RTW89_FCC][14] = -18,
+	[0][0][RTW89_ETSI][14] = 32,
+	[0][0][RTW89_FCC][15] = -18,
+	[0][0][RTW89_ETSI][15] = 32,
+	[0][0][RTW89_FCC][17] = -18,
+	[0][0][RTW89_ETSI][17] = 32,
+	[0][0][RTW89_FCC][19] = -18,
+	[0][0][RTW89_ETSI][19] = 32,
+	[0][0][RTW89_FCC][21] = -18,
+	[0][0][RTW89_ETSI][21] = 32,
+	[0][0][RTW89_FCC][23] = -18,
+	[0][0][RTW89_ETSI][23] = 32,
+	[0][0][RTW89_FCC][25] = -18,
+	[0][0][RTW89_ETSI][25] = 32,
+	[0][0][RTW89_FCC][27] = -18,
+	[0][0][RTW89_ETSI][27] = 32,
+	[0][0][RTW89_FCC][29] = -18,
+	[0][0][RTW89_ETSI][29] = 32,
+	[0][0][RTW89_FCC][30] = -18,
+	[0][0][RTW89_ETSI][30] = 32,
+	[0][0][RTW89_FCC][32] = -18,
+	[0][0][RTW89_ETSI][32] = 32,
+	[0][0][RTW89_FCC][34] = -18,
+	[0][0][RTW89_ETSI][34] = 32,
+	[0][0][RTW89_FCC][36] = -18,
+	[0][0][RTW89_ETSI][36] = 32,
+	[0][0][RTW89_FCC][38] = -18,
+	[0][0][RTW89_ETSI][38] = 32,
+	[0][0][RTW89_FCC][40] = -18,
+	[0][0][RTW89_ETSI][40] = 32,
+	[0][0][RTW89_FCC][42] = -18,
+	[0][0][RTW89_ETSI][42] = 32,
+	[0][0][RTW89_FCC][44] = -16,
+	[0][0][RTW89_ETSI][44] = 32,
+	[0][0][RTW89_FCC][45] = -16,
+	[0][0][RTW89_ETSI][45] = 127,
+	[0][0][RTW89_FCC][47] = -18,
+	[0][0][RTW89_ETSI][47] = 127,
+	[0][0][RTW89_FCC][49] = -18,
+	[0][0][RTW89_ETSI][49] = 127,
+	[0][0][RTW89_FCC][51] = -18,
+	[0][0][RTW89_ETSI][51] = 127,
+	[0][0][RTW89_FCC][53] = -16,
+	[0][0][RTW89_ETSI][53] = 127,
+	[0][0][RTW89_FCC][55] = -18,
+	[0][0][RTW89_ETSI][55] = 127,
+	[0][0][RTW89_FCC][57] = -18,
+	[0][0][RTW89_ETSI][57] = 127,
+	[0][0][RTW89_FCC][59] = -18,
+	[0][0][RTW89_ETSI][59] = 127,
+	[0][0][RTW89_FCC][60] = -18,
+	[0][0][RTW89_ETSI][60] = 127,
+	[0][0][RTW89_FCC][62] = -18,
+	[0][0][RTW89_ETSI][62] = 127,
+	[0][0][RTW89_FCC][64] = -18,
+	[0][0][RTW89_ETSI][64] = 127,
+	[0][0][RTW89_FCC][66] = -18,
+	[0][0][RTW89_ETSI][66] = 127,
+	[0][0][RTW89_FCC][68] = -18,
+	[0][0][RTW89_ETSI][68] = 127,
+	[0][0][RTW89_FCC][70] = -16,
+	[0][0][RTW89_ETSI][70] = 127,
+	[0][0][RTW89_FCC][72] = -18,
+	[0][0][RTW89_ETSI][72] = 127,
+	[0][0][RTW89_FCC][74] = -18,
+	[0][0][RTW89_ETSI][74] = 127,
+	[0][0][RTW89_FCC][75] = -18,
+	[0][0][RTW89_ETSI][75] = 127,
+	[0][0][RTW89_FCC][77] = -18,
+	[0][0][RTW89_ETSI][77] = 127,
+	[0][0][RTW89_FCC][79] = -18,
+	[0][0][RTW89_ETSI][79] = 127,
+	[0][0][RTW89_FCC][81] = -18,
+	[0][0][RTW89_ETSI][81] = 127,
+	[0][0][RTW89_FCC][83] = -18,
+	[0][0][RTW89_ETSI][83] = 127,
+	[0][0][RTW89_FCC][85] = -18,
+	[0][0][RTW89_ETSI][85] = 127,
+	[0][0][RTW89_FCC][87] = -16,
+	[0][0][RTW89_ETSI][87] = 127,
+	[0][0][RTW89_FCC][89] = -16,
+	[0][0][RTW89_ETSI][89] = 127,
+	[0][0][RTW89_FCC][90] = -16,
+	[0][0][RTW89_ETSI][90] = 127,
+	[0][0][RTW89_FCC][92] = -16,
+	[0][0][RTW89_ETSI][92] = 127,
+	[0][0][RTW89_FCC][94] = -16,
+	[0][0][RTW89_ETSI][94] = 127,
+	[0][0][RTW89_FCC][96] = -16,
+	[0][0][RTW89_ETSI][96] = 127,
+	[0][0][RTW89_FCC][98] = -16,
+	[0][0][RTW89_ETSI][98] = 127,
+	[0][0][RTW89_FCC][100] = -16,
+	[0][0][RTW89_ETSI][100] = 127,
+	[0][0][RTW89_FCC][102] = -16,
+	[0][0][RTW89_ETSI][102] = 127,
+	[0][0][RTW89_FCC][104] = -16,
+	[0][0][RTW89_ETSI][104] = 127,
+	[0][0][RTW89_FCC][105] = -16,
+	[0][0][RTW89_ETSI][105] = 127,
+	[0][0][RTW89_FCC][107] = -12,
+	[0][0][RTW89_ETSI][107] = 127,
+	[0][0][RTW89_FCC][109] = -12,
+	[0][0][RTW89_ETSI][109] = 127,
 	[0][0][RTW89_FCC][111] = 127,
+	[0][0][RTW89_ETSI][111] = 127,
 	[0][0][RTW89_FCC][113] = 127,
+	[0][0][RTW89_ETSI][113] = 127,
 	[0][0][RTW89_FCC][115] = 127,
+	[0][0][RTW89_ETSI][115] = 127,
 	[0][0][RTW89_FCC][117] = 127,
+	[0][0][RTW89_ETSI][117] = 127,
 	[0][0][RTW89_FCC][119] = 127,
-	[0][1][RTW89_FCC][0] = 76,
-	[0][1][RTW89_FCC][2] = 76,
-	[0][1][RTW89_FCC][4] = 76,
-	[0][1][RTW89_FCC][6] = 76,
-	[0][1][RTW89_FCC][8] = 76,
-	[0][1][RTW89_FCC][10] = 76,
-	[0][1][RTW89_FCC][12] = 76,
-	[0][1][RTW89_FCC][14] = 76,
-	[0][1][RTW89_FCC][15] = 76,
-	[0][1][RTW89_FCC][17] = 76,
-	[0][1][RTW89_FCC][19] = 76,
-	[0][1][RTW89_FCC][21] = 76,
-	[0][1][RTW89_FCC][23] = 76,
-	[0][1][RTW89_FCC][25] = 76,
-	[0][1][RTW89_FCC][27] = 76,
-	[0][1][RTW89_FCC][29] = 76,
-	[0][1][RTW89_FCC][30] = 76,
-	[0][1][RTW89_FCC][32] = 76,
-	[0][1][RTW89_FCC][34] = 76,
-	[0][1][RTW89_FCC][36] = 76,
-	[0][1][RTW89_FCC][38] = 76,
-	[0][1][RTW89_FCC][40] = 76,
-	[0][1][RTW89_FCC][42] = 76,
-	[0][1][RTW89_FCC][44] = 76,
-	[0][1][RTW89_FCC][45] = 76,
-	[0][1][RTW89_FCC][47] = 76,
-	[0][1][RTW89_FCC][49] = 76,
-	[0][1][RTW89_FCC][51] = 76,
-	[0][1][RTW89_FCC][53] = 76,
-	[0][1][RTW89_FCC][55] = 76,
-	[0][1][RTW89_FCC][57] = 76,
-	[0][1][RTW89_FCC][59] = 76,
-	[0][1][RTW89_FCC][60] = 76,
-	[0][1][RTW89_FCC][62] = 76,
-	[0][1][RTW89_FCC][64] = 76,
-	[0][1][RTW89_FCC][66] = 76,
-	[0][1][RTW89_FCC][68] = 76,
-	[0][1][RTW89_FCC][70] = 76,
-	[0][1][RTW89_FCC][72] = 76,
-	[0][1][RTW89_FCC][74] = 76,
-	[0][1][RTW89_FCC][75] = 76,
-	[0][1][RTW89_FCC][77] = 76,
-	[0][1][RTW89_FCC][79] = 76,
-	[0][1][RTW89_FCC][81] = 76,
-	[0][1][RTW89_FCC][83] = 76,
-	[0][1][RTW89_FCC][85] = 76,
-	[0][1][RTW89_FCC][87] = 76,
-	[0][1][RTW89_FCC][89] = 76,
-	[0][1][RTW89_FCC][90] = 76,
-	[0][1][RTW89_FCC][92] = 76,
-	[0][1][RTW89_FCC][94] = 76,
-	[0][1][RTW89_FCC][96] = 76,
-	[0][1][RTW89_FCC][98] = 76,
-	[0][1][RTW89_FCC][100] = 76,
-	[0][1][RTW89_FCC][102] = 76,
-	[0][1][RTW89_FCC][104] = 76,
-	[0][1][RTW89_FCC][105] = 76,
-	[0][1][RTW89_FCC][107] = 76,
-	[0][1][RTW89_FCC][109] = 76,
+	[0][0][RTW89_ETSI][119] = 127,
+	[0][1][RTW89_FCC][0] = -40,
+	[0][1][RTW89_ETSI][0] = 20,
+	[0][1][RTW89_FCC][2] = -40,
+	[0][1][RTW89_ETSI][2] = 20,
+	[0][1][RTW89_FCC][4] = -40,
+	[0][1][RTW89_ETSI][4] = 20,
+	[0][1][RTW89_FCC][6] = -40,
+	[0][1][RTW89_ETSI][6] = 20,
+	[0][1][RTW89_FCC][8] = -40,
+	[0][1][RTW89_ETSI][8] = 20,
+	[0][1][RTW89_FCC][10] = -40,
+	[0][1][RTW89_ETSI][10] = 20,
+	[0][1][RTW89_FCC][12] = -40,
+	[0][1][RTW89_ETSI][12] = 20,
+	[0][1][RTW89_FCC][14] = -40,
+	[0][1][RTW89_ETSI][14] = 20,
+	[0][1][RTW89_FCC][15] = -40,
+	[0][1][RTW89_ETSI][15] = 20,
+	[0][1][RTW89_FCC][17] = -40,
+	[0][1][RTW89_ETSI][17] = 20,
+	[0][1][RTW89_FCC][19] = -40,
+	[0][1][RTW89_ETSI][19] = 20,
+	[0][1][RTW89_FCC][21] = -40,
+	[0][1][RTW89_ETSI][21] = 20,
+	[0][1][RTW89_FCC][23] = -40,
+	[0][1][RTW89_ETSI][23] = 20,
+	[0][1][RTW89_FCC][25] = -40,
+	[0][1][RTW89_ETSI][25] = 20,
+	[0][1][RTW89_FCC][27] = -40,
+	[0][1][RTW89_ETSI][27] = 20,
+	[0][1][RTW89_FCC][29] = -40,
+	[0][1][RTW89_ETSI][29] = 20,
+	[0][1][RTW89_FCC][30] = -40,
+	[0][1][RTW89_ETSI][30] = 20,
+	[0][1][RTW89_FCC][32] = -40,
+	[0][1][RTW89_ETSI][32] = 20,
+	[0][1][RTW89_FCC][34] = -40,
+	[0][1][RTW89_ETSI][34] = 20,
+	[0][1][RTW89_FCC][36] = -40,
+	[0][1][RTW89_ETSI][36] = 20,
+	[0][1][RTW89_FCC][38] = -40,
+	[0][1][RTW89_ETSI][38] = 20,
+	[0][1][RTW89_FCC][40] = -40,
+	[0][1][RTW89_ETSI][40] = 20,
+	[0][1][RTW89_FCC][42] = -40,
+	[0][1][RTW89_ETSI][42] = 20,
+	[0][1][RTW89_FCC][44] = -40,
+	[0][1][RTW89_ETSI][44] = 20,
+	[0][1][RTW89_FCC][45] = -40,
+	[0][1][RTW89_ETSI][45] = 127,
+	[0][1][RTW89_FCC][47] = -40,
+	[0][1][RTW89_ETSI][47] = 127,
+	[0][1][RTW89_FCC][49] = -40,
+	[0][1][RTW89_ETSI][49] = 127,
+	[0][1][RTW89_FCC][51] = -40,
+	[0][1][RTW89_ETSI][51] = 127,
+	[0][1][RTW89_FCC][53] = -40,
+	[0][1][RTW89_ETSI][53] = 127,
+	[0][1][RTW89_FCC][55] = -40,
+	[0][1][RTW89_ETSI][55] = 127,
+	[0][1][RTW89_FCC][57] = -40,
+	[0][1][RTW89_ETSI][57] = 127,
+	[0][1][RTW89_FCC][59] = -40,
+	[0][1][RTW89_ETSI][59] = 127,
+	[0][1][RTW89_FCC][60] = -40,
+	[0][1][RTW89_ETSI][60] = 127,
+	[0][1][RTW89_FCC][62] = -40,
+	[0][1][RTW89_ETSI][62] = 127,
+	[0][1][RTW89_FCC][64] = -40,
+	[0][1][RTW89_ETSI][64] = 127,
+	[0][1][RTW89_FCC][66] = -40,
+	[0][1][RTW89_ETSI][66] = 127,
+	[0][1][RTW89_FCC][68] = -40,
+	[0][1][RTW89_ETSI][68] = 127,
+	[0][1][RTW89_FCC][70] = -38,
+	[0][1][RTW89_ETSI][70] = 127,
+	[0][1][RTW89_FCC][72] = -38,
+	[0][1][RTW89_ETSI][72] = 127,
+	[0][1][RTW89_FCC][74] = -38,
+	[0][1][RTW89_ETSI][74] = 127,
+	[0][1][RTW89_FCC][75] = -38,
+	[0][1][RTW89_ETSI][75] = 127,
+	[0][1][RTW89_FCC][77] = -38,
+	[0][1][RTW89_ETSI][77] = 127,
+	[0][1][RTW89_FCC][79] = -38,
+	[0][1][RTW89_ETSI][79] = 127,
+	[0][1][RTW89_FCC][81] = -38,
+	[0][1][RTW89_ETSI][81] = 127,
+	[0][1][RTW89_FCC][83] = -38,
+	[0][1][RTW89_ETSI][83] = 127,
+	[0][1][RTW89_FCC][85] = -38,
+	[0][1][RTW89_ETSI][85] = 127,
+	[0][1][RTW89_FCC][87] = -40,
+	[0][1][RTW89_ETSI][87] = 127,
+	[0][1][RTW89_FCC][89] = -38,
+	[0][1][RTW89_ETSI][89] = 127,
+	[0][1][RTW89_FCC][90] = -38,
+	[0][1][RTW89_ETSI][90] = 127,
+	[0][1][RTW89_FCC][92] = -38,
+	[0][1][RTW89_ETSI][92] = 127,
+	[0][1][RTW89_FCC][94] = -38,
+	[0][1][RTW89_ETSI][94] = 127,
+	[0][1][RTW89_FCC][96] = -38,
+	[0][1][RTW89_ETSI][96] = 127,
+	[0][1][RTW89_FCC][98] = -38,
+	[0][1][RTW89_ETSI][98] = 127,
+	[0][1][RTW89_FCC][100] = -38,
+	[0][1][RTW89_ETSI][100] = 127,
+	[0][1][RTW89_FCC][102] = -38,
+	[0][1][RTW89_ETSI][102] = 127,
+	[0][1][RTW89_FCC][104] = -38,
+	[0][1][RTW89_ETSI][104] = 127,
+	[0][1][RTW89_FCC][105] = -38,
+	[0][1][RTW89_ETSI][105] = 127,
+	[0][1][RTW89_FCC][107] = -34,
+	[0][1][RTW89_ETSI][107] = 127,
+	[0][1][RTW89_FCC][109] = -34,
+	[0][1][RTW89_ETSI][109] = 127,
 	[0][1][RTW89_FCC][111] = 127,
+	[0][1][RTW89_ETSI][111] = 127,
 	[0][1][RTW89_FCC][113] = 127,
+	[0][1][RTW89_ETSI][113] = 127,
 	[0][1][RTW89_FCC][115] = 127,
+	[0][1][RTW89_ETSI][115] = 127,
 	[0][1][RTW89_FCC][117] = 127,
+	[0][1][RTW89_ETSI][117] = 127,
 	[0][1][RTW89_FCC][119] = 127,
-	[1][0][RTW89_FCC][0] = 76,
-	[1][0][RTW89_FCC][2] = 76,
-	[1][0][RTW89_FCC][4] = 76,
-	[1][0][RTW89_FCC][6] = 76,
-	[1][0][RTW89_FCC][8] = 76,
-	[1][0][RTW89_FCC][10] = 76,
-	[1][0][RTW89_FCC][12] = 76,
-	[1][0][RTW89_FCC][14] = 76,
-	[1][0][RTW89_FCC][15] = 76,
-	[1][0][RTW89_FCC][17] = 76,
-	[1][0][RTW89_FCC][19] = 76,
-	[1][0][RTW89_FCC][21] = 76,
-	[1][0][RTW89_FCC][23] = 76,
-	[1][0][RTW89_FCC][25] = 76,
-	[1][0][RTW89_FCC][27] = 76,
-	[1][0][RTW89_FCC][29] = 76,
-	[1][0][RTW89_FCC][30] = 76,
-	[1][0][RTW89_FCC][32] = 76,
-	[1][0][RTW89_FCC][34] = 76,
-	[1][0][RTW89_FCC][36] = 76,
-	[1][0][RTW89_FCC][38] = 76,
-	[1][0][RTW89_FCC][40] = 76,
-	[1][0][RTW89_FCC][42] = 76,
-	[1][0][RTW89_FCC][44] = 76,
-	[1][0][RTW89_FCC][45] = 76,
-	[1][0][RTW89_FCC][47] = 76,
-	[1][0][RTW89_FCC][49] = 76,
-	[1][0][RTW89_FCC][51] = 76,
-	[1][0][RTW89_FCC][53] = 76,
-	[1][0][RTW89_FCC][55] = 76,
-	[1][0][RTW89_FCC][57] = 76,
-	[1][0][RTW89_FCC][59] = 76,
-	[1][0][RTW89_FCC][60] = 76,
-	[1][0][RTW89_FCC][62] = 76,
-	[1][0][RTW89_FCC][64] = 76,
-	[1][0][RTW89_FCC][66] = 76,
-	[1][0][RTW89_FCC][68] = 76,
-	[1][0][RTW89_FCC][70] = 76,
-	[1][0][RTW89_FCC][72] = 76,
-	[1][0][RTW89_FCC][74] = 76,
-	[1][0][RTW89_FCC][75] = 76,
-	[1][0][RTW89_FCC][77] = 76,
-	[1][0][RTW89_FCC][79] = 76,
-	[1][0][RTW89_FCC][81] = 76,
-	[1][0][RTW89_FCC][83] = 76,
-	[1][0][RTW89_FCC][85] = 76,
-	[1][0][RTW89_FCC][87] = 76,
-	[1][0][RTW89_FCC][89] = 76,
-	[1][0][RTW89_FCC][90] = 76,
-	[1][0][RTW89_FCC][92] = 76,
-	[1][0][RTW89_FCC][94] = 76,
-	[1][0][RTW89_FCC][96] = 76,
-	[1][0][RTW89_FCC][98] = 76,
-	[1][0][RTW89_FCC][100] = 76,
-	[1][0][RTW89_FCC][102] = 76,
-	[1][0][RTW89_FCC][104] = 76,
-	[1][0][RTW89_FCC][105] = 76,
-	[1][0][RTW89_FCC][107] = 76,
-	[1][0][RTW89_FCC][109] = 76,
+	[0][1][RTW89_ETSI][119] = 127,
+	[1][0][RTW89_FCC][0] = -4,
+	[1][0][RTW89_ETSI][0] = 46,
+	[1][0][RTW89_FCC][2] = -4,
+	[1][0][RTW89_ETSI][2] = 46,
+	[1][0][RTW89_FCC][4] = -4,
+	[1][0][RTW89_ETSI][4] = 46,
+	[1][0][RTW89_FCC][6] = -4,
+	[1][0][RTW89_ETSI][6] = 46,
+	[1][0][RTW89_FCC][8] = -4,
+	[1][0][RTW89_ETSI][8] = 46,
+	[1][0][RTW89_FCC][10] = -4,
+	[1][0][RTW89_ETSI][10] = 46,
+	[1][0][RTW89_FCC][12] = -4,
+	[1][0][RTW89_ETSI][12] = 46,
+	[1][0][RTW89_FCC][14] = -4,
+	[1][0][RTW89_ETSI][14] = 46,
+	[1][0][RTW89_FCC][15] = -4,
+	[1][0][RTW89_ETSI][15] = 46,
+	[1][0][RTW89_FCC][17] = -4,
+	[1][0][RTW89_ETSI][17] = 46,
+	[1][0][RTW89_FCC][19] = -4,
+	[1][0][RTW89_ETSI][19] = 46,
+	[1][0][RTW89_FCC][21] = -4,
+	[1][0][RTW89_ETSI][21] = 46,
+	[1][0][RTW89_FCC][23] = -4,
+	[1][0][RTW89_ETSI][23] = 46,
+	[1][0][RTW89_FCC][25] = -4,
+	[1][0][RTW89_ETSI][25] = 46,
+	[1][0][RTW89_FCC][27] = -4,
+	[1][0][RTW89_ETSI][27] = 46,
+	[1][0][RTW89_FCC][29] = -4,
+	[1][0][RTW89_ETSI][29] = 46,
+	[1][0][RTW89_FCC][30] = -4,
+	[1][0][RTW89_ETSI][30] = 46,
+	[1][0][RTW89_FCC][32] = -4,
+	[1][0][RTW89_ETSI][32] = 46,
+	[1][0][RTW89_FCC][34] = -4,
+	[1][0][RTW89_ETSI][34] = 46,
+	[1][0][RTW89_FCC][36] = -4,
+	[1][0][RTW89_ETSI][36] = 46,
+	[1][0][RTW89_FCC][38] = -4,
+	[1][0][RTW89_ETSI][38] = 46,
+	[1][0][RTW89_FCC][40] = -4,
+	[1][0][RTW89_ETSI][40] = 46,
+	[1][0][RTW89_FCC][42] = -4,
+	[1][0][RTW89_ETSI][42] = 46,
+	[1][0][RTW89_FCC][44] = -4,
+	[1][0][RTW89_ETSI][44] = 46,
+	[1][0][RTW89_FCC][45] = -4,
+	[1][0][RTW89_ETSI][45] = 127,
+	[1][0][RTW89_FCC][47] = -4,
+	[1][0][RTW89_ETSI][47] = 127,
+	[1][0][RTW89_FCC][49] = -4,
+	[1][0][RTW89_ETSI][49] = 127,
+	[1][0][RTW89_FCC][51] = -4,
+	[1][0][RTW89_ETSI][51] = 127,
+	[1][0][RTW89_FCC][53] = -4,
+	[1][0][RTW89_ETSI][53] = 127,
+	[1][0][RTW89_FCC][55] = -4,
+	[1][0][RTW89_ETSI][55] = 127,
+	[1][0][RTW89_FCC][57] = -4,
+	[1][0][RTW89_ETSI][57] = 127,
+	[1][0][RTW89_FCC][59] = -4,
+	[1][0][RTW89_ETSI][59] = 127,
+	[1][0][RTW89_FCC][60] = -4,
+	[1][0][RTW89_ETSI][60] = 127,
+	[1][0][RTW89_FCC][62] = -4,
+	[1][0][RTW89_ETSI][62] = 127,
+	[1][0][RTW89_FCC][64] = -4,
+	[1][0][RTW89_ETSI][64] = 127,
+	[1][0][RTW89_FCC][66] = -4,
+	[1][0][RTW89_ETSI][66] = 127,
+	[1][0][RTW89_FCC][68] = -4,
+	[1][0][RTW89_ETSI][68] = 127,
+	[1][0][RTW89_FCC][70] = -4,
+	[1][0][RTW89_ETSI][70] = 127,
+	[1][0][RTW89_FCC][72] = -4,
+	[1][0][RTW89_ETSI][72] = 127,
+	[1][0][RTW89_FCC][74] = -4,
+	[1][0][RTW89_ETSI][74] = 127,
+	[1][0][RTW89_FCC][75] = -4,
+	[1][0][RTW89_ETSI][75] = 127,
+	[1][0][RTW89_FCC][77] = -4,
+	[1][0][RTW89_ETSI][77] = 127,
+	[1][0][RTW89_FCC][79] = -4,
+	[1][0][RTW89_ETSI][79] = 127,
+	[1][0][RTW89_FCC][81] = -4,
+	[1][0][RTW89_ETSI][81] = 127,
+	[1][0][RTW89_FCC][83] = -4,
+	[1][0][RTW89_ETSI][83] = 127,
+	[1][0][RTW89_FCC][85] = -4,
+	[1][0][RTW89_ETSI][85] = 127,
+	[1][0][RTW89_FCC][87] = -4,
+	[1][0][RTW89_ETSI][87] = 127,
+	[1][0][RTW89_FCC][89] = -4,
+	[1][0][RTW89_ETSI][89] = 127,
+	[1][0][RTW89_FCC][90] = -4,
+	[1][0][RTW89_ETSI][90] = 127,
+	[1][0][RTW89_FCC][92] = -4,
+	[1][0][RTW89_ETSI][92] = 127,
+	[1][0][RTW89_FCC][94] = -4,
+	[1][0][RTW89_ETSI][94] = 127,
+	[1][0][RTW89_FCC][96] = -4,
+	[1][0][RTW89_ETSI][96] = 127,
+	[1][0][RTW89_FCC][98] = -4,
+	[1][0][RTW89_ETSI][98] = 127,
+	[1][0][RTW89_FCC][100] = -4,
+	[1][0][RTW89_ETSI][100] = 127,
+	[1][0][RTW89_FCC][102] = -4,
+	[1][0][RTW89_ETSI][102] = 127,
+	[1][0][RTW89_FCC][104] = -4,
+	[1][0][RTW89_ETSI][104] = 127,
+	[1][0][RTW89_FCC][105] = -4,
+	[1][0][RTW89_ETSI][105] = 127,
+	[1][0][RTW89_FCC][107] = 0,
+	[1][0][RTW89_ETSI][107] = 127,
+	[1][0][RTW89_FCC][109] = 2,
+	[1][0][RTW89_ETSI][109] = 127,
 	[1][0][RTW89_FCC][111] = 127,
+	[1][0][RTW89_ETSI][111] = 127,
 	[1][0][RTW89_FCC][113] = 127,
+	[1][0][RTW89_ETSI][113] = 127,
 	[1][0][RTW89_FCC][115] = 127,
+	[1][0][RTW89_ETSI][115] = 127,
 	[1][0][RTW89_FCC][117] = 127,
+	[1][0][RTW89_ETSI][117] = 127,
 	[1][0][RTW89_FCC][119] = 127,
-	[1][1][RTW89_FCC][0] = 76,
-	[1][1][RTW89_FCC][2] = 76,
-	[1][1][RTW89_FCC][4] = 76,
-	[1][1][RTW89_FCC][6] = 76,
-	[1][1][RTW89_FCC][8] = 76,
-	[1][1][RTW89_FCC][10] = 76,
-	[1][1][RTW89_FCC][12] = 76,
-	[1][1][RTW89_FCC][14] = 76,
-	[1][1][RTW89_FCC][15] = 76,
-	[1][1][RTW89_FCC][17] = 76,
-	[1][1][RTW89_FCC][19] = 76,
-	[1][1][RTW89_FCC][21] = 76,
-	[1][1][RTW89_FCC][23] = 76,
-	[1][1][RTW89_FCC][25] = 76,
-	[1][1][RTW89_FCC][27] = 76,
-	[1][1][RTW89_FCC][29] = 76,
-	[1][1][RTW89_FCC][30] = 76,
-	[1][1][RTW89_FCC][32] = 76,
-	[1][1][RTW89_FCC][34] = 76,
-	[1][1][RTW89_FCC][36] = 76,
-	[1][1][RTW89_FCC][38] = 76,
-	[1][1][RTW89_FCC][40] = 76,
-	[1][1][RTW89_FCC][42] = 76,
-	[1][1][RTW89_FCC][44] = 76,
-	[1][1][RTW89_FCC][45] = 76,
-	[1][1][RTW89_FCC][47] = 76,
-	[1][1][RTW89_FCC][49] = 76,
-	[1][1][RTW89_FCC][51] = 76,
-	[1][1][RTW89_FCC][53] = 76,
-	[1][1][RTW89_FCC][55] = 76,
-	[1][1][RTW89_FCC][57] = 76,
-	[1][1][RTW89_FCC][59] = 76,
-	[1][1][RTW89_FCC][60] = 76,
-	[1][1][RTW89_FCC][62] = 76,
-	[1][1][RTW89_FCC][64] = 76,
-	[1][1][RTW89_FCC][66] = 76,
-	[1][1][RTW89_FCC][68] = 76,
-	[1][1][RTW89_FCC][70] = 76,
-	[1][1][RTW89_FCC][72] = 76,
-	[1][1][RTW89_FCC][74] = 76,
-	[1][1][RTW89_FCC][75] = 76,
-	[1][1][RTW89_FCC][77] = 76,
-	[1][1][RTW89_FCC][79] = 76,
-	[1][1][RTW89_FCC][81] = 76,
-	[1][1][RTW89_FCC][83] = 76,
-	[1][1][RTW89_FCC][85] = 76,
-	[1][1][RTW89_FCC][87] = 76,
-	[1][1][RTW89_FCC][89] = 76,
-	[1][1][RTW89_FCC][90] = 76,
-	[1][1][RTW89_FCC][92] = 76,
-	[1][1][RTW89_FCC][94] = 76,
-	[1][1][RTW89_FCC][96] = 76,
-	[1][1][RTW89_FCC][98] = 76,
-	[1][1][RTW89_FCC][100] = 76,
-	[1][1][RTW89_FCC][102] = 76,
-	[1][1][RTW89_FCC][104] = 76,
-	[1][1][RTW89_FCC][105] = 76,
-	[1][1][RTW89_FCC][107] = 76,
-	[1][1][RTW89_FCC][109] = 76,
+	[1][0][RTW89_ETSI][119] = 127,
+	[1][1][RTW89_FCC][0] = -26,
+	[1][1][RTW89_ETSI][0] = 32,
+	[1][1][RTW89_FCC][2] = -28,
+	[1][1][RTW89_ETSI][2] = 32,
+	[1][1][RTW89_FCC][4] = -28,
+	[1][1][RTW89_ETSI][4] = 32,
+	[1][1][RTW89_FCC][6] = -28,
+	[1][1][RTW89_ETSI][6] = 32,
+	[1][1][RTW89_FCC][8] = -28,
+	[1][1][RTW89_ETSI][8] = 32,
+	[1][1][RTW89_FCC][10] = -28,
+	[1][1][RTW89_ETSI][10] = 32,
+	[1][1][RTW89_FCC][12] = -28,
+	[1][1][RTW89_ETSI][12] = 32,
+	[1][1][RTW89_FCC][14] = -28,
+	[1][1][RTW89_ETSI][14] = 32,
+	[1][1][RTW89_FCC][15] = -28,
+	[1][1][RTW89_ETSI][15] = 32,
+	[1][1][RTW89_FCC][17] = -28,
+	[1][1][RTW89_ETSI][17] = 32,
+	[1][1][RTW89_FCC][19] = -28,
+	[1][1][RTW89_ETSI][19] = 32,
+	[1][1][RTW89_FCC][21] = -28,
+	[1][1][RTW89_ETSI][21] = 32,
+	[1][1][RTW89_FCC][23] = -28,
+	[1][1][RTW89_ETSI][23] = 32,
+	[1][1][RTW89_FCC][25] = -28,
+	[1][1][RTW89_ETSI][25] = 32,
+	[1][1][RTW89_FCC][27] = -28,
+	[1][1][RTW89_ETSI][27] = 32,
+	[1][1][RTW89_FCC][29] = -28,
+	[1][1][RTW89_ETSI][29] = 32,
+	[1][1][RTW89_FCC][30] = -28,
+	[1][1][RTW89_ETSI][30] = 32,
+	[1][1][RTW89_FCC][32] = -28,
+	[1][1][RTW89_ETSI][32] = 32,
+	[1][1][RTW89_FCC][34] = -28,
+	[1][1][RTW89_ETSI][34] = 32,
+	[1][1][RTW89_FCC][36] = -28,
+	[1][1][RTW89_ETSI][36] = 32,
+	[1][1][RTW89_FCC][38] = -28,
+	[1][1][RTW89_ETSI][38] = 32,
+	[1][1][RTW89_FCC][40] = -28,
+	[1][1][RTW89_ETSI][40] = 32,
+	[1][1][RTW89_FCC][42] = -28,
+	[1][1][RTW89_ETSI][42] = 32,
+	[1][1][RTW89_FCC][44] = -28,
+	[1][1][RTW89_ETSI][44] = 34,
+	[1][1][RTW89_FCC][45] = -26,
+	[1][1][RTW89_ETSI][45] = 127,
+	[1][1][RTW89_FCC][47] = -28,
+	[1][1][RTW89_ETSI][47] = 127,
+	[1][1][RTW89_FCC][49] = -28,
+	[1][1][RTW89_ETSI][49] = 127,
+	[1][1][RTW89_FCC][51] = -28,
+	[1][1][RTW89_ETSI][51] = 127,
+	[1][1][RTW89_FCC][53] = -26,
+	[1][1][RTW89_ETSI][53] = 127,
+	[1][1][RTW89_FCC][55] = -28,
+	[1][1][RTW89_ETSI][55] = 127,
+	[1][1][RTW89_FCC][57] = -28,
+	[1][1][RTW89_ETSI][57] = 127,
+	[1][1][RTW89_FCC][59] = -28,
+	[1][1][RTW89_ETSI][59] = 127,
+	[1][1][RTW89_FCC][60] = -28,
+	[1][1][RTW89_ETSI][60] = 127,
+	[1][1][RTW89_FCC][62] = -28,
+	[1][1][RTW89_ETSI][62] = 127,
+	[1][1][RTW89_FCC][64] = -28,
+	[1][1][RTW89_ETSI][64] = 127,
+	[1][1][RTW89_FCC][66] = -28,
+	[1][1][RTW89_ETSI][66] = 127,
+	[1][1][RTW89_FCC][68] = -28,
+	[1][1][RTW89_ETSI][68] = 127,
+	[1][1][RTW89_FCC][70] = -26,
+	[1][1][RTW89_ETSI][70] = 127,
+	[1][1][RTW89_FCC][72] = -28,
+	[1][1][RTW89_ETSI][72] = 127,
+	[1][1][RTW89_FCC][74] = -28,
+	[1][1][RTW89_ETSI][74] = 127,
+	[1][1][RTW89_FCC][75] = -28,
+	[1][1][RTW89_ETSI][75] = 127,
+	[1][1][RTW89_FCC][77] = -28,
+	[1][1][RTW89_ETSI][77] = 127,
+	[1][1][RTW89_FCC][79] = -28,
+	[1][1][RTW89_ETSI][79] = 127,
+	[1][1][RTW89_FCC][81] = -28,
+	[1][1][RTW89_ETSI][81] = 127,
+	[1][1][RTW89_FCC][83] = -28,
+	[1][1][RTW89_ETSI][83] = 127,
+	[1][1][RTW89_FCC][85] = -28,
+	[1][1][RTW89_ETSI][85] = 127,
+	[1][1][RTW89_FCC][87] = -28,
+	[1][1][RTW89_ETSI][87] = 127,
+	[1][1][RTW89_FCC][89] = -26,
+	[1][1][RTW89_ETSI][89] = 127,
+	[1][1][RTW89_FCC][90] = -26,
+	[1][1][RTW89_ETSI][90] = 127,
+	[1][1][RTW89_FCC][92] = -26,
+	[1][1][RTW89_ETSI][92] = 127,
+	[1][1][RTW89_FCC][94] = -26,
+	[1][1][RTW89_ETSI][94] = 127,
+	[1][1][RTW89_FCC][96] = -26,
+	[1][1][RTW89_ETSI][96] = 127,
+	[1][1][RTW89_FCC][98] = -26,
+	[1][1][RTW89_ETSI][98] = 127,
+	[1][1][RTW89_FCC][100] = -26,
+	[1][1][RTW89_ETSI][100] = 127,
+	[1][1][RTW89_FCC][102] = -26,
+	[1][1][RTW89_ETSI][102] = 127,
+	[1][1][RTW89_FCC][104] = -26,
+	[1][1][RTW89_ETSI][104] = 127,
+	[1][1][RTW89_FCC][105] = -26,
+	[1][1][RTW89_ETSI][105] = 127,
+	[1][1][RTW89_FCC][107] = -22,
+	[1][1][RTW89_ETSI][107] = 127,
+	[1][1][RTW89_FCC][109] = -22,
+	[1][1][RTW89_ETSI][109] = 127,
 	[1][1][RTW89_FCC][111] = 127,
+	[1][1][RTW89_ETSI][111] = 127,
 	[1][1][RTW89_FCC][113] = 127,
+	[1][1][RTW89_ETSI][113] = 127,
 	[1][1][RTW89_FCC][115] = 127,
+	[1][1][RTW89_ETSI][115] = 127,
 	[1][1][RTW89_FCC][117] = 127,
+	[1][1][RTW89_ETSI][117] = 127,
 	[1][1][RTW89_FCC][119] = 127,
-	[2][0][RTW89_FCC][0] = 76,
-	[2][0][RTW89_FCC][2] = 76,
-	[2][0][RTW89_FCC][4] = 76,
-	[2][0][RTW89_FCC][6] = 76,
-	[2][0][RTW89_FCC][8] = 76,
-	[2][0][RTW89_FCC][10] = 76,
-	[2][0][RTW89_FCC][12] = 76,
-	[2][0][RTW89_FCC][14] = 76,
-	[2][0][RTW89_FCC][15] = 76,
-	[2][0][RTW89_FCC][17] = 76,
-	[2][0][RTW89_FCC][19] = 76,
-	[2][0][RTW89_FCC][21] = 76,
-	[2][0][RTW89_FCC][23] = 76,
-	[2][0][RTW89_FCC][25] = 76,
-	[2][0][RTW89_FCC][27] = 76,
-	[2][0][RTW89_FCC][29] = 76,
-	[2][0][RTW89_FCC][30] = 76,
-	[2][0][RTW89_FCC][32] = 76,
-	[2][0][RTW89_FCC][34] = 76,
-	[2][0][RTW89_FCC][36] = 76,
-	[2][0][RTW89_FCC][38] = 76,
-	[2][0][RTW89_FCC][40] = 76,
-	[2][0][RTW89_FCC][42] = 76,
-	[2][0][RTW89_FCC][44] = 76,
-	[2][0][RTW89_FCC][45] = 76,
-	[2][0][RTW89_FCC][47] = 76,
-	[2][0][RTW89_FCC][49] = 76,
-	[2][0][RTW89_FCC][51] = 76,
-	[2][0][RTW89_FCC][53] = 76,
-	[2][0][RTW89_FCC][55] = 76,
-	[2][0][RTW89_FCC][57] = 76,
-	[2][0][RTW89_FCC][59] = 76,
-	[2][0][RTW89_FCC][60] = 76,
-	[2][0][RTW89_FCC][62] = 76,
-	[2][0][RTW89_FCC][64] = 76,
-	[2][0][RTW89_FCC][66] = 76,
-	[2][0][RTW89_FCC][68] = 76,
-	[2][0][RTW89_FCC][70] = 76,
-	[2][0][RTW89_FCC][72] = 76,
-	[2][0][RTW89_FCC][74] = 76,
-	[2][0][RTW89_FCC][75] = 76,
-	[2][0][RTW89_FCC][77] = 76,
-	[2][0][RTW89_FCC][79] = 76,
-	[2][0][RTW89_FCC][81] = 76,
-	[2][0][RTW89_FCC][83] = 76,
-	[2][0][RTW89_FCC][85] = 76,
-	[2][0][RTW89_FCC][87] = 76,
-	[2][0][RTW89_FCC][89] = 76,
-	[2][0][RTW89_FCC][90] = 76,
-	[2][0][RTW89_FCC][92] = 76,
-	[2][0][RTW89_FCC][94] = 76,
-	[2][0][RTW89_FCC][96] = 76,
-	[2][0][RTW89_FCC][98] = 76,
-	[2][0][RTW89_FCC][100] = 76,
-	[2][0][RTW89_FCC][102] = 76,
-	[2][0][RTW89_FCC][104] = 76,
-	[2][0][RTW89_FCC][105] = 76,
-	[2][0][RTW89_FCC][107] = 76,
-	[2][0][RTW89_FCC][109] = 76,
+	[1][1][RTW89_ETSI][119] = 127,
+	[2][0][RTW89_FCC][0] = 8,
+	[2][0][RTW89_ETSI][0] = 56,
+	[2][0][RTW89_FCC][2] = 8,
+	[2][0][RTW89_ETSI][2] = 56,
+	[2][0][RTW89_FCC][4] = 8,
+	[2][0][RTW89_ETSI][4] = 56,
+	[2][0][RTW89_FCC][6] = 8,
+	[2][0][RTW89_ETSI][6] = 56,
+	[2][0][RTW89_FCC][8] = 8,
+	[2][0][RTW89_ETSI][8] = 56,
+	[2][0][RTW89_FCC][10] = 8,
+	[2][0][RTW89_ETSI][10] = 56,
+	[2][0][RTW89_FCC][12] = 8,
+	[2][0][RTW89_ETSI][12] = 56,
+	[2][0][RTW89_FCC][14] = 8,
+	[2][0][RTW89_ETSI][14] = 56,
+	[2][0][RTW89_FCC][15] = 8,
+	[2][0][RTW89_ETSI][15] = 56,
+	[2][0][RTW89_FCC][17] = 8,
+	[2][0][RTW89_ETSI][17] = 56,
+	[2][0][RTW89_FCC][19] = 8,
+	[2][0][RTW89_ETSI][19] = 56,
+	[2][0][RTW89_FCC][21] = 8,
+	[2][0][RTW89_ETSI][21] = 56,
+	[2][0][RTW89_FCC][23] = 8,
+	[2][0][RTW89_ETSI][23] = 56,
+	[2][0][RTW89_FCC][25] = 8,
+	[2][0][RTW89_ETSI][25] = 56,
+	[2][0][RTW89_FCC][27] = 8,
+	[2][0][RTW89_ETSI][27] = 56,
+	[2][0][RTW89_FCC][29] = 8,
+	[2][0][RTW89_ETSI][29] = 56,
+	[2][0][RTW89_FCC][30] = 8,
+	[2][0][RTW89_ETSI][30] = 56,
+	[2][0][RTW89_FCC][32] = 8,
+	[2][0][RTW89_ETSI][32] = 56,
+	[2][0][RTW89_FCC][34] = 8,
+	[2][0][RTW89_ETSI][34] = 56,
+	[2][0][RTW89_FCC][36] = 8,
+	[2][0][RTW89_ETSI][36] = 56,
+	[2][0][RTW89_FCC][38] = 8,
+	[2][0][RTW89_ETSI][38] = 56,
+	[2][0][RTW89_FCC][40] = 8,
+	[2][0][RTW89_ETSI][40] = 56,
+	[2][0][RTW89_FCC][42] = 8,
+	[2][0][RTW89_ETSI][42] = 56,
+	[2][0][RTW89_FCC][44] = 8,
+	[2][0][RTW89_ETSI][44] = 56,
+	[2][0][RTW89_FCC][45] = 8,
+	[2][0][RTW89_ETSI][45] = 127,
+	[2][0][RTW89_FCC][47] = 8,
+	[2][0][RTW89_ETSI][47] = 127,
+	[2][0][RTW89_FCC][49] = 8,
+	[2][0][RTW89_ETSI][49] = 127,
+	[2][0][RTW89_FCC][51] = 8,
+	[2][0][RTW89_ETSI][51] = 127,
+	[2][0][RTW89_FCC][53] = 8,
+	[2][0][RTW89_ETSI][53] = 127,
+	[2][0][RTW89_FCC][55] = 8,
+	[2][0][RTW89_ETSI][55] = 127,
+	[2][0][RTW89_FCC][57] = 8,
+	[2][0][RTW89_ETSI][57] = 127,
+	[2][0][RTW89_FCC][59] = 8,
+	[2][0][RTW89_ETSI][59] = 127,
+	[2][0][RTW89_FCC][60] = 8,
+	[2][0][RTW89_ETSI][60] = 127,
+	[2][0][RTW89_FCC][62] = 8,
+	[2][0][RTW89_ETSI][62] = 127,
+	[2][0][RTW89_FCC][64] = 8,
+	[2][0][RTW89_ETSI][64] = 127,
+	[2][0][RTW89_FCC][66] = 8,
+	[2][0][RTW89_ETSI][66] = 127,
+	[2][0][RTW89_FCC][68] = 8,
+	[2][0][RTW89_ETSI][68] = 127,
+	[2][0][RTW89_FCC][70] = 8,
+	[2][0][RTW89_ETSI][70] = 127,
+	[2][0][RTW89_FCC][72] = 8,
+	[2][0][RTW89_ETSI][72] = 127,
+	[2][0][RTW89_FCC][74] = 8,
+	[2][0][RTW89_ETSI][74] = 127,
+	[2][0][RTW89_FCC][75] = 8,
+	[2][0][RTW89_ETSI][75] = 127,
+	[2][0][RTW89_FCC][77] = 8,
+	[2][0][RTW89_ETSI][77] = 127,
+	[2][0][RTW89_FCC][79] = 8,
+	[2][0][RTW89_ETSI][79] = 127,
+	[2][0][RTW89_FCC][81] = 8,
+	[2][0][RTW89_ETSI][81] = 127,
+	[2][0][RTW89_FCC][83] = 8,
+	[2][0][RTW89_ETSI][83] = 127,
+	[2][0][RTW89_FCC][85] = 8,
+	[2][0][RTW89_ETSI][85] = 127,
+	[2][0][RTW89_FCC][87] = 8,
+	[2][0][RTW89_ETSI][87] = 127,
+	[2][0][RTW89_FCC][89] = 8,
+	[2][0][RTW89_ETSI][89] = 127,
+	[2][0][RTW89_FCC][90] = 8,
+	[2][0][RTW89_ETSI][90] = 127,
+	[2][0][RTW89_FCC][92] = 8,
+	[2][0][RTW89_ETSI][92] = 127,
+	[2][0][RTW89_FCC][94] = 8,
+	[2][0][RTW89_ETSI][94] = 127,
+	[2][0][RTW89_FCC][96] = 8,
+	[2][0][RTW89_ETSI][96] = 127,
+	[2][0][RTW89_FCC][98] = 8,
+	[2][0][RTW89_ETSI][98] = 127,
+	[2][0][RTW89_FCC][100] = 8,
+	[2][0][RTW89_ETSI][100] = 127,
+	[2][0][RTW89_FCC][102] = 8,
+	[2][0][RTW89_ETSI][102] = 127,
+	[2][0][RTW89_FCC][104] = 8,
+	[2][0][RTW89_ETSI][104] = 127,
+	[2][0][RTW89_FCC][105] = 8,
+	[2][0][RTW89_ETSI][105] = 127,
+	[2][0][RTW89_FCC][107] = 10,
+	[2][0][RTW89_ETSI][107] = 127,
+	[2][0][RTW89_FCC][109] = 12,
+	[2][0][RTW89_ETSI][109] = 127,
 	[2][0][RTW89_FCC][111] = 127,
+	[2][0][RTW89_ETSI][111] = 127,
 	[2][0][RTW89_FCC][113] = 127,
+	[2][0][RTW89_ETSI][113] = 127,
 	[2][0][RTW89_FCC][115] = 127,
+	[2][0][RTW89_ETSI][115] = 127,
 	[2][0][RTW89_FCC][117] = 127,
+	[2][0][RTW89_ETSI][117] = 127,
 	[2][0][RTW89_FCC][119] = 127,
-	[2][1][RTW89_FCC][0] = 76,
-	[2][1][RTW89_FCC][2] = 76,
-	[2][1][RTW89_FCC][4] = 76,
-	[2][1][RTW89_FCC][6] = 76,
-	[2][1][RTW89_FCC][8] = 76,
-	[2][1][RTW89_FCC][10] = 76,
-	[2][1][RTW89_FCC][12] = 76,
-	[2][1][RTW89_FCC][14] = 76,
-	[2][1][RTW89_FCC][15] = 76,
-	[2][1][RTW89_FCC][17] = 76,
-	[2][1][RTW89_FCC][19] = 76,
-	[2][1][RTW89_FCC][21] = 76,
-	[2][1][RTW89_FCC][23] = 76,
-	[2][1][RTW89_FCC][25] = 76,
-	[2][1][RTW89_FCC][27] = 76,
-	[2][1][RTW89_FCC][29] = 76,
-	[2][1][RTW89_FCC][30] = 76,
-	[2][1][RTW89_FCC][32] = 76,
-	[2][1][RTW89_FCC][34] = 76,
-	[2][1][RTW89_FCC][36] = 76,
-	[2][1][RTW89_FCC][38] = 76,
-	[2][1][RTW89_FCC][40] = 76,
-	[2][1][RTW89_FCC][42] = 76,
-	[2][1][RTW89_FCC][44] = 76,
-	[2][1][RTW89_FCC][45] = 76,
-	[2][1][RTW89_FCC][47] = 76,
-	[2][1][RTW89_FCC][49] = 76,
-	[2][1][RTW89_FCC][51] = 76,
-	[2][1][RTW89_FCC][53] = 76,
-	[2][1][RTW89_FCC][55] = 76,
-	[2][1][RTW89_FCC][57] = 76,
-	[2][1][RTW89_FCC][59] = 76,
-	[2][1][RTW89_FCC][60] = 76,
-	[2][1][RTW89_FCC][62] = 76,
-	[2][1][RTW89_FCC][64] = 76,
-	[2][1][RTW89_FCC][66] = 76,
-	[2][1][RTW89_FCC][68] = 76,
-	[2][1][RTW89_FCC][70] = 76,
-	[2][1][RTW89_FCC][72] = 76,
-	[2][1][RTW89_FCC][74] = 76,
-	[2][1][RTW89_FCC][75] = 76,
-	[2][1][RTW89_FCC][77] = 76,
-	[2][1][RTW89_FCC][79] = 76,
-	[2][1][RTW89_FCC][81] = 76,
-	[2][1][RTW89_FCC][83] = 76,
-	[2][1][RTW89_FCC][85] = 76,
-	[2][1][RTW89_FCC][87] = 76,
-	[2][1][RTW89_FCC][89] = 76,
-	[2][1][RTW89_FCC][90] = 76,
-	[2][1][RTW89_FCC][92] = 76,
-	[2][1][RTW89_FCC][94] = 76,
-	[2][1][RTW89_FCC][96] = 76,
-	[2][1][RTW89_FCC][98] = 76,
-	[2][1][RTW89_FCC][100] = 76,
-	[2][1][RTW89_FCC][102] = 76,
-	[2][1][RTW89_FCC][104] = 76,
-	[2][1][RTW89_FCC][105] = 76,
-	[2][1][RTW89_FCC][107] = 76,
-	[2][1][RTW89_FCC][109] = 76,
+	[2][0][RTW89_ETSI][119] = 127,
+	[2][1][RTW89_FCC][0] = -16,
+	[2][1][RTW89_ETSI][0] = 44,
+	[2][1][RTW89_FCC][2] = -16,
+	[2][1][RTW89_ETSI][2] = 44,
+	[2][1][RTW89_FCC][4] = -16,
+	[2][1][RTW89_ETSI][4] = 44,
+	[2][1][RTW89_FCC][6] = -16,
+	[2][1][RTW89_ETSI][6] = 44,
+	[2][1][RTW89_FCC][8] = -16,
+	[2][1][RTW89_ETSI][8] = 44,
+	[2][1][RTW89_FCC][10] = -16,
+	[2][1][RTW89_ETSI][10] = 44,
+	[2][1][RTW89_FCC][12] = -16,
+	[2][1][RTW89_ETSI][12] = 44,
+	[2][1][RTW89_FCC][14] = -16,
+	[2][1][RTW89_ETSI][14] = 44,
+	[2][1][RTW89_FCC][15] = -16,
+	[2][1][RTW89_ETSI][15] = 44,
+	[2][1][RTW89_FCC][17] = -16,
+	[2][1][RTW89_ETSI][17] = 44,
+	[2][1][RTW89_FCC][19] = -16,
+	[2][1][RTW89_ETSI][19] = 44,
+	[2][1][RTW89_FCC][21] = -16,
+	[2][1][RTW89_ETSI][21] = 44,
+	[2][1][RTW89_FCC][23] = -16,
+	[2][1][RTW89_ETSI][23] = 44,
+	[2][1][RTW89_FCC][25] = -16,
+	[2][1][RTW89_ETSI][25] = 44,
+	[2][1][RTW89_FCC][27] = -16,
+	[2][1][RTW89_ETSI][27] = 44,
+	[2][1][RTW89_FCC][29] = -16,
+	[2][1][RTW89_ETSI][29] = 44,
+	[2][1][RTW89_FCC][30] = -16,
+	[2][1][RTW89_ETSI][30] = 44,
+	[2][1][RTW89_FCC][32] = -16,
+	[2][1][RTW89_ETSI][32] = 44,
+	[2][1][RTW89_FCC][34] = -16,
+	[2][1][RTW89_ETSI][34] = 44,
+	[2][1][RTW89_FCC][36] = -16,
+	[2][1][RTW89_ETSI][36] = 44,
+	[2][1][RTW89_FCC][38] = -16,
+	[2][1][RTW89_ETSI][38] = 44,
+	[2][1][RTW89_FCC][40] = -16,
+	[2][1][RTW89_ETSI][40] = 44,
+	[2][1][RTW89_FCC][42] = -16,
+	[2][1][RTW89_ETSI][42] = 44,
+	[2][1][RTW89_FCC][44] = -16,
+	[2][1][RTW89_ETSI][44] = 44,
+	[2][1][RTW89_FCC][45] = -16,
+	[2][1][RTW89_ETSI][45] = 127,
+	[2][1][RTW89_FCC][47] = -16,
+	[2][1][RTW89_ETSI][47] = 127,
+	[2][1][RTW89_FCC][49] = -16,
+	[2][1][RTW89_ETSI][49] = 127,
+	[2][1][RTW89_FCC][51] = -16,
+	[2][1][RTW89_ETSI][51] = 127,
+	[2][1][RTW89_FCC][53] = -16,
+	[2][1][RTW89_ETSI][53] = 127,
+	[2][1][RTW89_FCC][55] = -16,
+	[2][1][RTW89_ETSI][55] = 127,
+	[2][1][RTW89_FCC][57] = -16,
+	[2][1][RTW89_ETSI][57] = 127,
+	[2][1][RTW89_FCC][59] = -16,
+	[2][1][RTW89_ETSI][59] = 127,
+	[2][1][RTW89_FCC][60] = -16,
+	[2][1][RTW89_ETSI][60] = 127,
+	[2][1][RTW89_FCC][62] = -16,
+	[2][1][RTW89_ETSI][62] = 127,
+	[2][1][RTW89_FCC][64] = -16,
+	[2][1][RTW89_ETSI][64] = 127,
+	[2][1][RTW89_FCC][66] = -16,
+	[2][1][RTW89_ETSI][66] = 127,
+	[2][1][RTW89_FCC][68] = -16,
+	[2][1][RTW89_ETSI][68] = 127,
+	[2][1][RTW89_FCC][70] = -16,
+	[2][1][RTW89_ETSI][70] = 127,
+	[2][1][RTW89_FCC][72] = -16,
+	[2][1][RTW89_ETSI][72] = 127,
+	[2][1][RTW89_FCC][74] = -16,
+	[2][1][RTW89_ETSI][74] = 127,
+	[2][1][RTW89_FCC][75] = -16,
+	[2][1][RTW89_ETSI][75] = 127,
+	[2][1][RTW89_FCC][77] = -16,
+	[2][1][RTW89_ETSI][77] = 127,
+	[2][1][RTW89_FCC][79] = -16,
+	[2][1][RTW89_ETSI][79] = 127,
+	[2][1][RTW89_FCC][81] = -16,
+	[2][1][RTW89_ETSI][81] = 127,
+	[2][1][RTW89_FCC][83] = -16,
+	[2][1][RTW89_ETSI][83] = 127,
+	[2][1][RTW89_FCC][85] = -18,
+	[2][1][RTW89_ETSI][85] = 127,
+	[2][1][RTW89_FCC][87] = -16,
+	[2][1][RTW89_ETSI][87] = 127,
+	[2][1][RTW89_FCC][89] = -16,
+	[2][1][RTW89_ETSI][89] = 127,
+	[2][1][RTW89_FCC][90] = -16,
+	[2][1][RTW89_ETSI][90] = 127,
+	[2][1][RTW89_FCC][92] = -16,
+	[2][1][RTW89_ETSI][92] = 127,
+	[2][1][RTW89_FCC][94] = -16,
+	[2][1][RTW89_ETSI][94] = 127,
+	[2][1][RTW89_FCC][96] = -16,
+	[2][1][RTW89_ETSI][96] = 127,
+	[2][1][RTW89_FCC][98] = -16,
+	[2][1][RTW89_ETSI][98] = 127,
+	[2][1][RTW89_FCC][100] = -16,
+	[2][1][RTW89_ETSI][100] = 127,
+	[2][1][RTW89_FCC][102] = -16,
+	[2][1][RTW89_ETSI][102] = 127,
+	[2][1][RTW89_FCC][104] = -16,
+	[2][1][RTW89_ETSI][104] = 127,
+	[2][1][RTW89_FCC][105] = -16,
+	[2][1][RTW89_ETSI][105] = 127,
+	[2][1][RTW89_FCC][107] = -12,
+	[2][1][RTW89_ETSI][107] = 127,
+	[2][1][RTW89_FCC][109] = -10,
+	[2][1][RTW89_ETSI][109] = 127,
 	[2][1][RTW89_FCC][111] = 127,
+	[2][1][RTW89_ETSI][111] = 127,
 	[2][1][RTW89_FCC][113] = 127,
+	[2][1][RTW89_ETSI][113] = 127,
 	[2][1][RTW89_FCC][115] = 127,
+	[2][1][RTW89_ETSI][115] = 127,
 	[2][1][RTW89_FCC][117] = 127,
+	[2][1][RTW89_ETSI][117] = 127,
 	[2][1][RTW89_FCC][119] = 127,
+	[2][1][RTW89_ETSI][119] = 127,
 };
 
 const struct rtw89_phy_table rtw89_8852c_phy_bb_table = {
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852ce.c b/drivers/net/wireless/realtek/rtw89/rtw8852ce.c
index fc0394494013..35901f64d17d 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852ce.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852ce.c
@@ -42,14 +42,15 @@ static const struct rtw89_pci_info rtw8852c_pci_info = {
 	.max_tag_num_mask	= B_AX_MAX_TAG_NUM_V1_MASK,
 	.rxbd_rwptr_clr_reg	= R_AX_RXBD_RWPTR_CLR_V1,
 	.txbd_rwptr_clr2_reg	= R_AX_TXBD_RWPTR_CLR2_V1,
-	.dma_stop1_reg		= R_AX_HAXI_DMA_STOP1,
-	.dma_stop2_reg		= R_AX_HAXI_DMA_STOP2,
-	.dma_busy1_reg		= R_AX_HAXI_DMA_BUSY1,
+	.dma_stop1		= {R_AX_HAXI_DMA_STOP1, B_AX_TX_STOP1_MASK},
+	.dma_stop2		= {R_AX_HAXI_DMA_STOP2, B_AX_TX_STOP2_ALL},
+	.dma_busy1		= {R_AX_HAXI_DMA_BUSY1, DMA_BUSY1_CHECK},
 	.dma_busy2_reg		= R_AX_HAXI_DMA_BUSY2,
 	.dma_busy3_reg		= R_AX_HAXI_DMA_BUSY3,
 
 	.rpwm_addr		= R_AX_PCIE_HRPWM_V1,
 	.cpwm_addr		= R_AX_PCIE_CRPWM,
+	.tx_dma_ch_mask		= 0,
 	.bd_idx_addr_low_power	= &rtw8852c_bd_idx_addr_low_power,
 	.dma_addr_set		= &rtw89_pci_ch_dma_addr_set_v1,
 
diff --git a/drivers/net/wireless/realtek/rtw89/sar.c b/drivers/net/wireless/realtek/rtw89/sar.c
index eb2d3ec28775..dfccae81c380 100644
--- a/drivers/net/wireless/realtek/rtw89/sar.c
+++ b/drivers/net/wireless/realtek/rtw89/sar.c
@@ -81,9 +81,9 @@ static const struct rtw89_sar_span rtw89_sar_overlapping_6ghz[] = {
 static int rtw89_query_sar_config_common(struct rtw89_dev *rtwdev, s32 *cfg)
 {
 	struct rtw89_sar_cfg_common *rtwsar = &rtwdev->sar.cfg_common;
-	struct rtw89_hal *hal = &rtwdev->hal;
-	enum rtw89_band band = hal->current_band_type;
-	u32 center_freq = hal->current_freq;
+	const struct rtw89_chan *chan = rtw89_chan_get(rtwdev, RTW89_SUB_ENTITY_0);
+	enum rtw89_band band = chan->band_type;
+	u32 center_freq = chan->freq;
 	const struct rtw89_sar_span *span = NULL;
 	enum rtw89_sar_subband subband_l, subband_h;
 	int idx;
@@ -228,7 +228,7 @@ static int rtw89_apply_sar_common(struct rtw89_dev *rtwdev,
 	}
 
 	rtw89_sar_set_src(rtwdev, RTW89_SAR_SOURCE_COMMON, cfg_common, sar);
-	rtw89_chip_set_txpwr(rtwdev);
+	rtw89_core_set_chip_txpwr(rtwdev);
 
 exit:
 	mutex_unlock(&rtwdev->mutex);
diff --git a/drivers/net/wireless/realtek/rtw89/ser.c b/drivers/net/wireless/realtek/rtw89/ser.c
index 726223f25dc6..c1a4bc1c64d1 100644
--- a/drivers/net/wireless/realtek/rtw89/ser.c
+++ b/drivers/net/wireless/realtek/rtw89/ser.c
@@ -5,6 +5,7 @@
 #include <linux/devcoredump.h>
 
 #include "cam.h"
+#include "chan.h"
 #include "debug.h"
 #include "fw.h"
 #include "mac.h"
@@ -152,7 +153,10 @@ static void ser_state_run(struct rtw89_ser *ser, u8 evt)
 	rtw89_debug(rtwdev, RTW89_DBG_SER, "ser: %s receive %s\n",
 		    ser_st_name(ser), ser_ev_name(ser, evt));
 
+	mutex_lock(&rtwdev->mutex);
 	rtw89_leave_lps(rtwdev);
+	mutex_unlock(&rtwdev->mutex);
+
 	ser->st_tbl[ser->state].st_func(ser, evt);
 }
 
@@ -298,7 +302,7 @@ static void ser_reset_vif(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 	rtwvif->trigger = false;
 }
 
-static void ser_sta_deinit_addr_cam_iter(void *data, struct ieee80211_sta *sta)
+static void ser_sta_deinit_cam_iter(void *data, struct ieee80211_sta *sta)
 {
 	struct rtw89_vif *rtwvif = (struct rtw89_vif *)data;
 	struct rtw89_dev *rtwdev = rtwvif->rtwdev;
@@ -308,15 +312,19 @@ static void ser_sta_deinit_addr_cam_iter(void *data, struct ieee80211_sta *sta)
 		rtw89_cam_deinit_addr_cam(rtwdev, &rtwsta->addr_cam);
 	if (sta->tdls)
 		rtw89_cam_deinit_bssid_cam(rtwdev, &rtwsta->bssid_cam);
+
+	INIT_LIST_HEAD(&rtwsta->ba_cam_list);
 }
 
 static void ser_deinit_cam(struct rtw89_dev *rtwdev, struct rtw89_vif *rtwvif)
 {
 	ieee80211_iterate_stations_atomic(rtwdev->hw,
-					  ser_sta_deinit_addr_cam_iter,
+					  ser_sta_deinit_cam_iter,
 					  rtwvif);
 
 	rtw89_cam_deinit(rtwdev, rtwvif);
+
+	bitmap_zero(rtwdev->cam_info.ba_cam_map, RTW89_MAX_BA_CAM_NUM);
 }
 
 static void ser_reset_mac_binding(struct rtw89_dev *rtwdev)
@@ -388,6 +396,7 @@ static void ser_idle_st_hdl(struct rtw89_ser *ser, u8 evt)
 	switch (evt) {
 	case SER_EV_STATE_IN:
 		rtw89_hci_recovery_complete(rtwdev);
+		clear_bit(RTW89_FLAG_CRASH_SIMULATING, rtwdev->flags);
 		break;
 	case SER_EV_L1_RESET:
 		ser_state_goto(ser, SER_RESET_TRX_ST);
@@ -531,7 +540,7 @@ static int rtw89_ser_fw_backtrace_dump(struct rtw89_dev *rtwdev, u8 *buf,
 				       const struct __fw_backtrace_entry *ent)
 {
 	struct __fw_backtrace_info *ptr = (struct __fw_backtrace_info *)buf;
-	u32 fwbt_addr = ent->wcpu_addr - RTW89_WCPU_BASE_ADDR;
+	u32 fwbt_addr = ent->wcpu_addr & RTW89_WCPU_BASE_MASK;
 	u32 fwbt_size = ent->size;
 	u32 fwbt_key = ent->key;
 	u32 i;
@@ -601,6 +610,7 @@ bottom:
 
 	ser_reset_mac_binding(rtwdev);
 	rtw89_core_stop(rtwdev);
+	rtw89_entity_init(rtwdev);
 	INIT_LIST_HEAD(&rtwdev->rtwvifs_list);
 }
 
@@ -623,7 +633,6 @@ static void ser_l2_reset_st_hdl(struct rtw89_ser *ser, u8 evt)
 		fallthrough;
 	case SER_EV_L2_RECFG_DONE:
 		ser_state_goto(ser, SER_IDLE_ST);
-		clear_bit(RTW89_FLAG_RESTART_TRIGGER, rtwdev->flags);
 		break;
 
 	case SER_EV_STATE_OUT:
diff --git a/drivers/net/wireless/rndis_wlan.c b/drivers/net/wireless/rndis_wlan.c
index 05524291d60c..82a7458e01ae 100644
--- a/drivers/net/wireless/rndis_wlan.c
+++ b/drivers/net/wireless/rndis_wlan.c
@@ -251,7 +251,7 @@ struct ndis_80211_bssid_ex {
 
 struct ndis_80211_bssid_list_ex {
 	__le32 num_items;
-	struct ndis_80211_bssid_ex bssid[];
+	u8 bssid_data[];
 } __packed;
 
 struct ndis_80211_fixed_ies {
@@ -489,14 +489,16 @@ static int rndis_join_ibss(struct wiphy *wiphy, struct net_device *dev,
 static int rndis_leave_ibss(struct wiphy *wiphy, struct net_device *dev);
 
 static int rndis_add_key(struct wiphy *wiphy, struct net_device *netdev,
-			 u8 key_index, bool pairwise, const u8 *mac_addr,
-			 struct key_params *params);
+			 int link_id,  u8 key_index, bool pairwise,
+			 const u8 *mac_addr, struct key_params *params);
 
 static int rndis_del_key(struct wiphy *wiphy, struct net_device *netdev,
-			 u8 key_index, bool pairwise, const u8 *mac_addr);
+			 int link_id, u8 key_index, bool pairwise,
+			 const u8 *mac_addr);
 
 static int rndis_set_default_key(struct wiphy *wiphy, struct net_device *netdev,
-				 u8 key_index, bool unicast, bool multicast);
+				 int link_id, u8 key_index, bool unicast,
+				 bool multicast);
 
 static int rndis_get_station(struct wiphy *wiphy, struct net_device *dev,
 			     const u8 *mac, struct station_info *sinfo);
@@ -2082,7 +2084,8 @@ resize_buf:
 	netdev_dbg(usbdev->net, "%s(): buflen: %d\n", __func__, len);
 
 	bssid_len = 0;
-	bssid = next_bssid_list_item(bssid_list->bssid, &bssid_len, buf, len);
+	bssid = next_bssid_list_item((void *)bssid_list->bssid_data,
+				     &bssid_len, buf, len);
 
 	/* Device returns incorrect 'num_items'. Workaround by ignoring the
 	 * received 'num_items' and walking through full bssid buffer instead.
@@ -2377,8 +2380,8 @@ static int rndis_leave_ibss(struct wiphy *wiphy, struct net_device *dev)
 }
 
 static int rndis_add_key(struct wiphy *wiphy, struct net_device *netdev,
-			 u8 key_index, bool pairwise, const u8 *mac_addr,
-			 struct key_params *params)
+			 int link_id,  u8 key_index, bool pairwise,
+			 const u8 *mac_addr, struct key_params *params)
 {
 	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
 	struct usbnet *usbdev = priv->usbdev;
@@ -2413,7 +2416,8 @@ static int rndis_add_key(struct wiphy *wiphy, struct net_device *netdev,
 }
 
 static int rndis_del_key(struct wiphy *wiphy, struct net_device *netdev,
-			 u8 key_index, bool pairwise, const u8 *mac_addr)
+			 int link_id, u8 key_index, bool pairwise,
+			 const u8 *mac_addr)
 {
 	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
 	struct usbnet *usbdev = priv->usbdev;
@@ -2424,7 +2428,8 @@ static int rndis_del_key(struct wiphy *wiphy, struct net_device *netdev,
 }
 
 static int rndis_set_default_key(struct wiphy *wiphy, struct net_device *netdev,
-				 u8 key_index, bool unicast, bool multicast)
+				 int link_id, u8 key_index, bool unicast,
+				 bool multicast)
 {
 	struct rndis_wlan_private *priv = wiphy_priv(wiphy);
 	struct usbnet *usbdev = priv->usbdev;
diff --git a/drivers/net/wireless/rsi/rsi_91x_mac80211.c b/drivers/net/wireless/rsi/rsi_91x_mac80211.c
index bf39c4bda26f..2fbec51c8f94 100644
--- a/drivers/net/wireless/rsi/rsi_91x_mac80211.c
+++ b/drivers/net/wireless/rsi/rsi_91x_mac80211.c
@@ -889,6 +889,7 @@ static void rsi_mac80211_conf_filter(struct ieee80211_hw *hw,
  *			    for a hardware TX queue.
  * @hw: Pointer to the ieee80211_hw structure
  * @vif: Pointer to the ieee80211_vif structure.
+ * @link_id: the link ID if MLO is used, otherwise 0
  * @queue: Queue number.
  * @params: Pointer to ieee80211_tx_queue_params structure.
  *
diff --git a/drivers/net/wireless/silabs/wfx/main.c b/drivers/net/wireless/silabs/wfx/main.c
index e015bfb8d221..84d82ddded56 100644
--- a/drivers/net/wireless/silabs/wfx/main.c
+++ b/drivers/net/wireless/silabs/wfx/main.c
@@ -181,7 +181,7 @@ int wfx_send_pds(struct wfx_dev *wdev, u8 *buf, size_t len)
 	while (len > 0) {
 		chunk_type = get_unaligned_le16(buf + 0);
 		chunk_len = get_unaligned_le16(buf + 2);
-		if (chunk_len > len) {
+		if (chunk_len < 4 || chunk_len > len) {
 			dev_err(wdev->dev, "PDS:%d: corrupted file\n", chunk_num);
 			return -EINVAL;
 		}
diff --git a/drivers/net/wireless/st/cw1200/queue.c b/drivers/net/wireless/st/cw1200/queue.c
index e06da4b3b0d4..805a3c1bf8fe 100644
--- a/drivers/net/wireless/st/cw1200/queue.c
+++ b/drivers/net/wireless/st/cw1200/queue.c
@@ -91,23 +91,25 @@ static void __cw1200_queue_gc(struct cw1200_queue *queue,
 			      bool unlock)
 {
 	struct cw1200_queue_stats *stats = queue->stats;
-	struct cw1200_queue_item *item = NULL, *tmp;
+	struct cw1200_queue_item *item = NULL, *iter, *tmp;
 	bool wakeup_stats = false;
 
-	list_for_each_entry_safe(item, tmp, &queue->queue, head) {
-		if (time_is_after_jiffies(item->queue_timestamp + queue->ttl))
+	list_for_each_entry_safe(iter, tmp, &queue->queue, head) {
+		if (time_is_after_jiffies(iter->queue_timestamp + queue->ttl)) {
+			item = iter;
 			break;
+		}
 		--queue->num_queued;
-		--queue->link_map_cache[item->txpriv.link_id];
+		--queue->link_map_cache[iter->txpriv.link_id];
 		spin_lock_bh(&stats->lock);
 		--stats->num_queued;
-		if (!--stats->link_map_cache[item->txpriv.link_id])
+		if (!--stats->link_map_cache[iter->txpriv.link_id])
 			wakeup_stats = true;
 		spin_unlock_bh(&stats->lock);
 		cw1200_debug_tx_ttl(stats->priv);
-		cw1200_queue_register_post_gc(head, item);
-		item->skb = NULL;
-		list_move_tail(&item->head, &queue->free_pool);
+		cw1200_queue_register_post_gc(head, iter);
+		iter->skb = NULL;
+		list_move_tail(&iter->head, &queue->free_pool);
 	}
 
 	if (wakeup_stats)
diff --git a/drivers/net/wireless/st/cw1200/sta.c b/drivers/net/wireless/st/cw1200/sta.c
index 26d3614519b1..8ef1d06b9bbd 100644
--- a/drivers/net/wireless/st/cw1200/sta.c
+++ b/drivers/net/wireless/st/cw1200/sta.c
@@ -195,7 +195,7 @@ void __cw1200_cqm_bssloss_sm(struct cw1200_common *priv,
 
 		priv->bss_loss_state++;
 
-		skb = ieee80211_nullfunc_get(priv->hw, priv->vif, false);
+		skb = ieee80211_nullfunc_get(priv->hw, priv->vif, -1, false);
 		WARN_ON(!skb);
 		if (skb)
 			cw1200_tx(priv->hw, NULL, skb);
@@ -2263,7 +2263,7 @@ static int cw1200_upload_null(struct cw1200_common *priv)
 		.rate = 0xFF,
 	};
 
-	frame.skb = ieee80211_nullfunc_get(priv->hw, priv->vif, false);
+	frame.skb = ieee80211_nullfunc_get(priv->hw, priv->vif,-1, false);
 	if (!frame.skb)
 		return -ENOMEM;
 
diff --git a/drivers/net/wireless/st/cw1200/txrx.c b/drivers/net/wireless/st/cw1200/txrx.c
index fde21fca6c5e..6894b919ff94 100644
--- a/drivers/net/wireless/st/cw1200/txrx.c
+++ b/drivers/net/wireless/st/cw1200/txrx.c
@@ -762,8 +762,7 @@ void cw1200_tx(struct ieee80211_hw *dev,
 	if (ret)
 		goto drop;
 
-	rcu_read_lock();
-	sta = rcu_dereference(t.sta);
+	sta = t.sta;
 
 	spin_lock_bh(&priv->ps_state_lock);
 	{
@@ -776,8 +775,6 @@ void cw1200_tx(struct ieee80211_hw *dev,
 	if (tid_update && sta)
 		ieee80211_sta_set_buffered(sta, t.txpriv.tid, true);
 
-	rcu_read_unlock();
-
 	cw1200_bh_wakeup(priv);
 
 	return;
@@ -1145,8 +1142,7 @@ void cw1200_rx_cb(struct cw1200_common *priv,
 
 	/* Remove TSF from the end of frame */
 	if (arg->flags & WSM_RX_STATUS_TSF_INCLUDED) {
-		memcpy(&hdr->mactime, skb->data + skb->len - 8, 8);
-		hdr->mactime = le64_to_cpu(hdr->mactime);
+		hdr->mactime = get_unaligned_le64(skb->data + skb->len - 8);
 		if (skb->len >= 8)
 			skb_trim(skb, skb->len - 8);
 	} else {
diff --git a/drivers/net/wireless/ti/wl1251/main.c b/drivers/net/wireless/ti/wl1251/main.c
index 9144ef5538a8..289371689a8d 100644
--- a/drivers/net/wireless/ti/wl1251/main.c
+++ b/drivers/net/wireless/ti/wl1251/main.c
@@ -546,7 +546,7 @@ static int wl1251_build_null_data(struct wl1251 *wl)
 		size = sizeof(struct wl12xx_null_data_template);
 		ptr = NULL;
 	} else {
-		skb = ieee80211_nullfunc_get(wl->hw, wl->vif, false);
+		skb = ieee80211_nullfunc_get(wl->hw, wl->vif, -1, false);
 		if (!skb)
 			goto out;
 		size = skb->len;
diff --git a/drivers/net/wireless/ti/wl18xx/event.c b/drivers/net/wireless/ti/wl18xx/event.c
index 13d78ada4bb6..34d95f458e1a 100644
--- a/drivers/net/wireless/ti/wl18xx/event.c
+++ b/drivers/net/wireless/ti/wl18xx/event.c
@@ -131,10 +131,10 @@ int wl18xx_process_mailbox_events(struct wl1271 *wl)
 
 	if (vector & TIME_SYNC_EVENT_ID)
 		wlcore_event_time_sync(wl,
-			mbox->time_sync_tsf_high_msb,
-			mbox->time_sync_tsf_high_lsb,
-			mbox->time_sync_tsf_low_msb,
-			mbox->time_sync_tsf_low_lsb);
+			le16_to_cpu(mbox->time_sync_tsf_high_msb),
+			le16_to_cpu(mbox->time_sync_tsf_high_lsb),
+			le16_to_cpu(mbox->time_sync_tsf_low_msb),
+			le16_to_cpu(mbox->time_sync_tsf_low_lsb));
 
 	if (vector & RADAR_DETECTED_EVENT_ID) {
 		wl1271_info("radar event: channel %d type %s",
diff --git a/drivers/net/wireless/ti/wlcore/cmd.c b/drivers/net/wireless/ti/wlcore/cmd.c
index 138edd28b0de..a939fd89a7f5 100644
--- a/drivers/net/wireless/ti/wlcore/cmd.c
+++ b/drivers/net/wireless/ti/wlcore/cmd.c
@@ -1065,7 +1065,7 @@ int wl12xx_cmd_build_null_data(struct wl1271 *wl, struct wl12xx_vif *wlvif)
 	} else {
 		skb = ieee80211_nullfunc_get(wl->hw,
 					     wl12xx_wlvif_to_vif(wlvif),
-					     false);
+					     -1, false);
 		if (!skb)
 			goto out;
 		size = skb->len;
@@ -1092,7 +1092,7 @@ int wl12xx_cmd_build_klv_null_data(struct wl1271 *wl,
 	struct sk_buff *skb = NULL;
 	int ret = -ENOMEM;
 
-	skb = ieee80211_nullfunc_get(wl->hw, vif, false);
+	skb = ieee80211_nullfunc_get(wl->hw, vif,-1, false);
 	if (!skb)
 		goto out;
 
diff --git a/drivers/net/wireless/wl3501_cs.c b/drivers/net/wireless/wl3501_cs.c
index dad38fc04243..1b532e00a56f 100644
--- a/drivers/net/wireless/wl3501_cs.c
+++ b/drivers/net/wireless/wl3501_cs.c
@@ -1441,7 +1441,7 @@ static void wl3501_detach(struct pcmcia_device *link)
 static int wl3501_get_name(struct net_device *dev, struct iw_request_info *info,
 			   union iwreq_data *wrqu, char *extra)
 {
-	strlcpy(wrqu->name, "IEEE 802.11-DS", sizeof(wrqu->name));
+	strscpy(wrqu->name, "IEEE 802.11-DS", sizeof(wrqu->name));
 	return 0;
 }
 
@@ -1652,7 +1652,7 @@ static int wl3501_set_nick(struct net_device *dev, struct iw_request_info *info,
 
 	if (wrqu->data.length > sizeof(this->nick))
 		return -E2BIG;
-	strlcpy(this->nick, extra, wrqu->data.length);
+	strscpy(this->nick, extra, wrqu->data.length);
 	return 0;
 }
 
@@ -1661,7 +1661,7 @@ static int wl3501_get_nick(struct net_device *dev, struct iw_request_info *info,
 {
 	struct wl3501_card *this = netdev_priv(dev);
 
-	strlcpy(extra, this->nick, 32);
+	strscpy(extra, this->nick, 32);
 	wrqu->data.length = strlen(extra);
 	return 0;
 }
@@ -1965,7 +1965,7 @@ static int wl3501_config(struct pcmcia_device *link)
 	this->firmware_date[0]	= '\0';
 	this->rssi		= 255;
 	this->chan		= iw_default_channel(this->reg_domain);
-	strlcpy(this->nick, "Planet WL3501", sizeof(this->nick));
+	strscpy(this->nick, "Planet WL3501", sizeof(this->nick));
 	spin_lock_init(&this->lock);
 	init_waitqueue_head(&this->wait);
 	netif_start_queue(dev);
diff --git a/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c b/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c
index 57304a5adf68..b7f9237dedf7 100644
--- a/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c
+++ b/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c
@@ -590,7 +590,7 @@ int ipc_imem_sys_devlink_write(struct iosm_devlink *ipc_devlink,
 		goto out;
 	}
 
-	memcpy(skb_put(skb, count), buf, count);
+	skb_put_data(skb, buf, count);
 
 	IPC_CB(skb)->op_type = UL_USR_OP_BLOCKED;
 
diff --git a/drivers/net/wwan/iosm/iosm_ipc_wwan.c b/drivers/net/wwan/iosm/iosm_ipc_wwan.c
index 27151148c782..2f1f8b5d5b59 100644
--- a/drivers/net/wwan/iosm/iosm_ipc_wwan.c
+++ b/drivers/net/wwan/iosm/iosm_ipc_wwan.c
@@ -103,8 +103,8 @@ static int ipc_wwan_link_stop(struct net_device *netdev)
 }
 
 /* Transmit a packet */
-static int ipc_wwan_link_transmit(struct sk_buff *skb,
-				  struct net_device *netdev)
+static netdev_tx_t ipc_wwan_link_transmit(struct sk_buff *skb,
+					  struct net_device *netdev)
 {
 	struct iosm_netdev_priv *priv = wwan_netdev_drvpriv(netdev);
 	struct iosm_wwan *ipc_wwan = priv->ipc_wwan;
@@ -323,15 +323,16 @@ struct iosm_wwan *ipc_wwan_init(struct iosm_imem *ipc_imem, struct device *dev)
 	ipc_wwan->dev = dev;
 	ipc_wwan->ipc_imem = ipc_imem;
 
+	mutex_init(&ipc_wwan->if_mutex);
+
 	/* WWAN core will create a netdev for the default IP MUX channel */
 	if (wwan_register_ops(ipc_wwan->dev, &iosm_wwan_ops, ipc_wwan,
 			      IP_MUX_SESSION_DEFAULT)) {
+		mutex_destroy(&ipc_wwan->if_mutex);
 		kfree(ipc_wwan);
 		return NULL;
 	}
 
-	mutex_init(&ipc_wwan->if_mutex);
-
 	return ipc_wwan;
 }
 
diff --git a/drivers/net/wwan/mhi_wwan_ctrl.c b/drivers/net/wwan/mhi_wwan_ctrl.c
index e4d0f696687f..f7ca52353f40 100644
--- a/drivers/net/wwan/mhi_wwan_ctrl.c
+++ b/drivers/net/wwan/mhi_wwan_ctrl.c
@@ -258,6 +258,7 @@ static void mhi_wwan_ctrl_remove(struct mhi_device *mhi_dev)
 
 static const struct mhi_device_id mhi_wwan_ctrl_match_table[] = {
 	{ .chan = "DUN", .driver_data = WWAN_PORT_AT },
+	{ .chan = "DUN2", .driver_data = WWAN_PORT_AT },
 	{ .chan = "MBIM", .driver_data = WWAN_PORT_MBIM },
 	{ .chan = "QMI", .driver_data = WWAN_PORT_QMI },
 	{ .chan = "DIAG", .driver_data = WWAN_PORT_QCDM },
diff --git a/drivers/net/wwan/t7xx/t7xx_netdev.c b/drivers/net/wwan/t7xx/t7xx_netdev.c
index c6b6547f2c6f..f71d3bc3b237 100644
--- a/drivers/net/wwan/t7xx/t7xx_netdev.c
+++ b/drivers/net/wwan/t7xx/t7xx_netdev.c
@@ -74,7 +74,7 @@ static int t7xx_ccmni_send_packet(struct t7xx_ccmni *ccmni, struct sk_buff *skb,
 	return 0;
 }
 
-static int t7xx_ccmni_start_xmit(struct sk_buff *skb, struct net_device *dev)
+static netdev_tx_t t7xx_ccmni_start_xmit(struct sk_buff *skb, struct net_device *dev)
 {
 	struct t7xx_ccmni *ccmni = wwan_netdev_drvpriv(dev);
 	int skb_len = skb->len;
diff --git a/drivers/net/wwan/wwan_hwsim.c b/drivers/net/wwan/wwan_hwsim.c
index fad642f9ffd8..ff09a8cedf93 100644
--- a/drivers/net/wwan/wwan_hwsim.c
+++ b/drivers/net/wwan/wwan_hwsim.c
@@ -157,8 +157,8 @@ static int wwan_hwsim_port_tx(struct wwan_port *wport, struct sk_buff *in)
 			if ((i + 1) < in->len && in->data[i + 1] == '\n')
 				i++;
 			n = i - s + 1;
-			memcpy(skb_put(out, n), &in->data[s], n);/* Echo */
-			memcpy(skb_put(out, 6), "\r\nOK\r\n", 6);
+			skb_put_data(out, &in->data[s], n);/* Echo */
+			skb_put_data(out, "\r\nOK\r\n", 6);
 			s = i + 1;
 			port->pstate = AT_PARSER_WAIT_A;
 		} else if (port->pstate == AT_PARSER_SKIP_LINE) {
@@ -171,7 +171,7 @@ static int wwan_hwsim_port_tx(struct wwan_port *wport, struct sk_buff *in)
 	if (i > s) {
 		/* Echo the processed portion of a not yet completed command */
 		n = i - s;
-		memcpy(skb_put(out, n), &in->data[s], n);
+		skb_put_data(out, &in->data[s], n);
 	}
 
 	consume_skb(in);
diff --git a/drivers/net/xen-netback/common.h b/drivers/net/xen-netback/common.h
index 8174d7b2966c..1545cbee77a4 100644
--- a/drivers/net/xen-netback/common.h
+++ b/drivers/net/xen-netback/common.h
@@ -62,7 +62,7 @@ struct pending_tx_info {
 	 * ubuf_to_vif is a helper which finds the struct xenvif from a pointer
 	 * to this field.
 	 */
-	struct ubuf_info callback_struct;
+	struct ubuf_info_msgzc callback_struct;
 };
 
 #define XEN_NETIF_TX_RING_SIZE __CONST_RING_SIZE(xen_netif_tx, XEN_PAGE_SIZE)
diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c
index fb32ae82d9b0..650fa180220f 100644
--- a/drivers/net/xen-netback/interface.c
+++ b/drivers/net/xen-netback/interface.c
@@ -591,8 +591,8 @@ int xenvif_init_queue(struct xenvif_queue *queue)
 	}
 
 	for (i = 0; i < MAX_PENDING_REQS; i++) {
-		queue->pending_tx_info[i].callback_struct = (struct ubuf_info)
-			{ .callback = xenvif_zerocopy_callback,
+		queue->pending_tx_info[i].callback_struct = (struct ubuf_info_msgzc)
+			{ { .callback = xenvif_zerocopy_callback },
 			  { { .ctx = NULL,
 			      .desc = i } } };
 		queue->grant_tx_handle[i] = NETBACK_INVALID_HANDLE;
@@ -723,8 +723,7 @@ int xenvif_connect_data(struct xenvif_queue *queue,
 	init_waitqueue_head(&queue->dealloc_wq);
 	atomic_set(&queue->inflight_packets, 0);
 
-	netif_napi_add(queue->vif->dev, &queue->napi, xenvif_poll,
-			NAPI_POLL_WEIGHT);
+	netif_napi_add(queue->vif->dev, &queue->napi, xenvif_poll);
 
 	queue->stalled = true;
 
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index a256695fc89e..3d2081bbbc86 100644
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -133,7 +133,7 @@ static inline unsigned long idx_to_kaddr(struct xenvif_queue *queue,
 
 /* Find the containing VIF's structure from a pointer in pending_tx_info array
  */
-static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info *ubuf)
+static inline struct xenvif_queue *ubuf_to_queue(const struct ubuf_info_msgzc *ubuf)
 {
 	u16 pending_idx = ubuf->desc;
 	struct pending_tx_info *temp =
@@ -1228,11 +1228,12 @@ static int xenvif_tx_submit(struct xenvif_queue *queue)
 	return work_done;
 }
 
-void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf,
+void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf_base,
 			      bool zerocopy_success)
 {
 	unsigned long flags;
 	pending_ring_idx_t index;
+	struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base);
 	struct xenvif_queue *queue = ubuf_to_queue(ubuf);
 
 	/* This is the only place where we grab this lock, to protect callbacks
@@ -1241,7 +1242,7 @@ void xenvif_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *ubuf,
 	spin_lock_irqsave(&queue->callback_lock, flags);
 	do {
 		u16 pending_idx = ubuf->desc;
-		ubuf = (struct ubuf_info *) ubuf->ctx;
+		ubuf = (struct ubuf_info_msgzc *) ubuf->ctx;
 		BUG_ON(queue->dealloc_prod - queue->dealloc_cons >=
 			MAX_PENDING_REQS);
 		index = pending_index(queue->dealloc_prod);
diff --git a/drivers/net/xen-netback/xenbus.c b/drivers/net/xen-netback/xenbus.c
index e85b3c5d4acc..c1ba4294f364 100644
--- a/drivers/net/xen-netback/xenbus.c
+++ b/drivers/net/xen-netback/xenbus.c
@@ -865,13 +865,12 @@ static int connect_data_rings(struct backend_info *be,
 	 * queue-N.
 	 */
 	if (num_queues == 1) {
-		xspath = kzalloc(strlen(dev->otherend) + 1, GFP_KERNEL);
+		xspath = kstrdup(dev->otherend, GFP_KERNEL);
 		if (!xspath) {
 			xenbus_dev_fatal(dev, -ENOMEM,
 					 "reading ring references");
 			return -ENOMEM;
 		}
-		strcpy(xspath, dev->otherend);
 	} else {
 		xspathsize = strlen(dev->otherend) + xenstore_path_ext_size;
 		xspath = kzalloc(xspathsize, GFP_KERNEL);
diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c
index 27a11cc08c61..9af2b027c19c 100644
--- a/drivers/net/xen-netfront.c
+++ b/drivers/net/xen-netfront.c
@@ -673,7 +673,7 @@ static int xennet_xdp_xmit(struct net_device *dev, int n,
 	return nxmit;
 }
 
-struct sk_buff *bounce_skb(const struct sk_buff *skb)
+static struct sk_buff *bounce_skb(const struct sk_buff *skb)
 {
 	unsigned int headerlen = skb_headroom(skb);
 	/* Align size to allocate full pages and avoid contiguous data leaks */
@@ -2224,8 +2224,7 @@ static int xennet_create_queues(struct netfront_info *info,
 			return ret;
 		}
 
-		netif_napi_add(queue->info->netdev, &queue->napi,
-			       xennet_poll, 64);
+		netif_napi_add(queue->info->netdev, &queue->napi, xennet_poll);
 		if (netif_running(info->netdev))
 			napi_enable(&queue->napi);
 	}
diff --git a/drivers/nfc/fdp/i2c.c b/drivers/nfc/fdp/i2c.c
index 28a9e1eb9bcf..2d53e0f88d2f 100644
--- a/drivers/nfc/fdp/i2c.c
+++ b/drivers/nfc/fdp/i2c.c
@@ -336,14 +336,12 @@ static int fdp_nci_i2c_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int fdp_nci_i2c_remove(struct i2c_client *client)
+static void fdp_nci_i2c_remove(struct i2c_client *client)
 {
 	struct fdp_i2c_phy *phy = i2c_get_clientdata(client);
 
 	fdp_nci_remove(phy->ndev);
 	fdp_nci_i2c_disable(phy);
-
-	return 0;
 }
 
 static const struct acpi_device_id fdp_nci_i2c_acpi_match[] = {
diff --git a/drivers/nfc/microread/i2c.c b/drivers/nfc/microread/i2c.c
index 067295124eb9..5eaa18f81355 100644
--- a/drivers/nfc/microread/i2c.c
+++ b/drivers/nfc/microread/i2c.c
@@ -268,15 +268,13 @@ err_irq:
 	return r;
 }
 
-static int microread_i2c_remove(struct i2c_client *client)
+static void microread_i2c_remove(struct i2c_client *client)
 {
 	struct microread_i2c_phy *phy = i2c_get_clientdata(client);
 
 	microread_remove(phy->hdev);
 
 	free_irq(client->irq, phy);
-
-	return 0;
 }
 
 static const struct i2c_device_id microread_i2c_id[] = {
diff --git a/drivers/nfc/nfcmrvl/i2c.c b/drivers/nfc/nfcmrvl/i2c.c
index 01329b91d59d..acef0cfd76af 100644
--- a/drivers/nfc/nfcmrvl/i2c.c
+++ b/drivers/nfc/nfcmrvl/i2c.c
@@ -231,13 +231,11 @@ static int nfcmrvl_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int nfcmrvl_i2c_remove(struct i2c_client *client)
+static void nfcmrvl_i2c_remove(struct i2c_client *client)
 {
 	struct nfcmrvl_i2c_drv_data *drv_data = i2c_get_clientdata(client);
 
 	nfcmrvl_nci_unregister_dev(drv_data->priv);
-
-	return 0;
 }
 
 
diff --git a/drivers/nfc/nxp-nci/i2c.c b/drivers/nfc/nxp-nci/i2c.c
index ae2ba08d8ac3..ec6446511984 100644
--- a/drivers/nfc/nxp-nci/i2c.c
+++ b/drivers/nfc/nxp-nci/i2c.c
@@ -314,14 +314,12 @@ static int nxp_nci_i2c_probe(struct i2c_client *client,
 	return r;
 }
 
-static int nxp_nci_i2c_remove(struct i2c_client *client)
+static void nxp_nci_i2c_remove(struct i2c_client *client)
 {
 	struct nxp_nci_i2c_phy *phy = i2c_get_clientdata(client);
 
 	nxp_nci_remove(phy->ndev);
 	free_irq(client->irq, phy);
-
-	return 0;
 }
 
 static const struct i2c_device_id nxp_nci_i2c_id_table[] = {
diff --git a/drivers/nfc/pn533/i2c.c b/drivers/nfc/pn533/i2c.c
index 673eb5e9b887..ddf3db286bad 100644
--- a/drivers/nfc/pn533/i2c.c
+++ b/drivers/nfc/pn533/i2c.c
@@ -227,7 +227,7 @@ nfc_alloc_err:
 	return r;
 }
 
-static int pn533_i2c_remove(struct i2c_client *client)
+static void pn533_i2c_remove(struct i2c_client *client)
 {
 	struct pn533_i2c_phy *phy = i2c_get_clientdata(client);
 
@@ -235,8 +235,6 @@ static int pn533_i2c_remove(struct i2c_client *client)
 
 	pn53x_unregister_nfc(phy->priv);
 	pn53x_common_clean(phy->priv);
-
-	return 0;
 }
 
 static const struct of_device_id of_pn533_i2c_match[] __maybe_unused = {
diff --git a/drivers/nfc/pn544/i2c.c b/drivers/nfc/pn544/i2c.c
index 62a0f1a010cb..9e754abcfa2a 100644
--- a/drivers/nfc/pn544/i2c.c
+++ b/drivers/nfc/pn544/i2c.c
@@ -928,7 +928,7 @@ static int pn544_hci_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int pn544_hci_i2c_remove(struct i2c_client *client)
+static void pn544_hci_i2c_remove(struct i2c_client *client)
 {
 	struct pn544_i2c_phy *phy = i2c_get_clientdata(client);
 
@@ -940,8 +940,6 @@ static int pn544_hci_i2c_remove(struct i2c_client *client)
 
 	if (phy->powered)
 		pn544_hci_i2c_disable(phy);
-
-	return 0;
 }
 
 static const struct of_device_id of_pn544_i2c_match[] __maybe_unused = {
diff --git a/drivers/nfc/s3fwrn5/i2c.c b/drivers/nfc/s3fwrn5/i2c.c
index 4d1cf1bb55b0..f824dc7099ce 100644
--- a/drivers/nfc/s3fwrn5/i2c.c
+++ b/drivers/nfc/s3fwrn5/i2c.c
@@ -246,14 +246,12 @@ disable_clk:
 	return ret;
 }
 
-static int s3fwrn5_i2c_remove(struct i2c_client *client)
+static void s3fwrn5_i2c_remove(struct i2c_client *client)
 {
 	struct s3fwrn5_i2c_phy *phy = i2c_get_clientdata(client);
 
 	s3fwrn5_remove(phy->common.ndev);
 	clk_disable_unprepare(phy->clk);
-
-	return 0;
 }
 
 static const struct i2c_device_id s3fwrn5_i2c_id_table[] = {
diff --git a/drivers/nfc/st-nci/i2c.c b/drivers/nfc/st-nci/i2c.c
index cbd968f013c7..89fa24d71bef 100644
--- a/drivers/nfc/st-nci/i2c.c
+++ b/drivers/nfc/st-nci/i2c.c
@@ -250,13 +250,11 @@ static int st_nci_i2c_probe(struct i2c_client *client,
 	return r;
 }
 
-static int st_nci_i2c_remove(struct i2c_client *client)
+static void st_nci_i2c_remove(struct i2c_client *client)
 {
 	struct st_nci_i2c_phy *phy = i2c_get_clientdata(client);
 
 	ndlc_remove(phy->ndlc);
-
-	return 0;
 }
 
 static const struct i2c_device_id st_nci_i2c_id_table[] = {
diff --git a/drivers/nfc/st21nfca/i2c.c b/drivers/nfc/st21nfca/i2c.c
index 42dc0e5eb161..76b55986bcf8 100644
--- a/drivers/nfc/st21nfca/i2c.c
+++ b/drivers/nfc/st21nfca/i2c.c
@@ -562,7 +562,7 @@ out_free:
 	return r;
 }
 
-static int st21nfca_hci_i2c_remove(struct i2c_client *client)
+static void st21nfca_hci_i2c_remove(struct i2c_client *client)
 {
 	struct st21nfca_i2c_phy *phy = i2c_get_clientdata(client);
 
@@ -571,8 +571,6 @@ static int st21nfca_hci_i2c_remove(struct i2c_client *client)
 	if (phy->powered)
 		st21nfca_hci_i2c_disable(phy);
 	kfree_skb(phy->pending_skb);
-
-	return 0;
 }
 
 static const struct i2c_device_id st21nfca_hci_i2c_id_table[] = {
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index 66446f1e06cf..8d5a7ae19844 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -2162,14 +2162,14 @@ static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
 
 static int nvme_pr_clear(struct block_device *bdev, u64 key)
 {
-	u32 cdw10 = 1 | (key ? 1 << 3 : 0);
+	u32 cdw10 = 1 | (key ? 0 : 1 << 3);
 
-	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
+	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
 }
 
 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
 {
-	u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
+	u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 0 : 1 << 3);
 
 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
 }
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 98864b853eef..67d3335e9cc8 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -3470,6 +3470,10 @@ static const struct pci_device_id nvme_id_table[] = {
 	{ PCI_DEVICE(0x1987, 0x5016),	/* Phison E16 */
 		.driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN |
 				NVME_QUIRK_BOGUS_NID, },
+	{ PCI_DEVICE(0x1987, 0x5019),  /* phison E19 */
+		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
+	{ PCI_DEVICE(0x1987, 0x5021),   /* Phison E21 */
+		.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
 	{ PCI_DEVICE(0x1b4b, 0x1092),	/* Lexar 256 GB SSD */
 		.driver_data = NVME_QUIRK_NO_NS_DESC_LIST |
 				NVME_QUIRK_IGNORE_DEV_SUBNQN, },
diff --git a/drivers/of/base.c b/drivers/of/base.c
index 7fa960bd3df1..42da760e0f45 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -578,6 +578,7 @@ int of_device_compatible_match(struct device_node *device,
 
 	return score;
 }
+EXPORT_SYMBOL_GPL(of_device_compatible_match);
 
 /**
  * of_machine_is_compatible - Test root of device tree for a given compatible value
diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c
index eafa8ffefbd0..f9614552db82 100644
--- a/drivers/of/unittest.c
+++ b/drivers/of/unittest.c
@@ -2524,13 +2524,12 @@ static int unittest_i2c_dev_probe(struct i2c_client *client,
 	return 0;
 };
 
-static int unittest_i2c_dev_remove(struct i2c_client *client)
+static void unittest_i2c_dev_remove(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
 	struct device_node *np = client->dev.of_node;
 
 	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
-	return 0;
 }
 
 static const struct i2c_device_id unittest_i2c_dev_id[] = {
@@ -2601,7 +2600,7 @@ static int unittest_i2c_mux_probe(struct i2c_client *client,
 	return 0;
 };
 
-static int unittest_i2c_mux_remove(struct i2c_client *client)
+static void unittest_i2c_mux_remove(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
 	struct device_node *np = client->dev.of_node;
@@ -2609,7 +2608,6 @@ static int unittest_i2c_mux_remove(struct i2c_client *client)
 
 	dev_dbg(dev, "%s for node @%pOF\n", __func__, np);
 	i2c_mux_del_adapters(muxc);
-	return 0;
 }
 
 static const struct i2c_device_id unittest_i2c_mux_id[] = {
diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c
index 374b9199878d..ecd870087a3d 100644
--- a/drivers/parisc/sba_iommu.c
+++ b/drivers/parisc/sba_iommu.c
@@ -28,6 +28,12 @@
 #include <linux/dma-map-ops.h>
 #include <linux/scatterlist.h>
 #include <linux/iommu-helper.h>
+/*
+ * The semantics of 64 register access on 32bit systems can't be guaranteed
+ * by the C standard, we hope the _lo_hi() macros defining readq and writeq
+ * here will behave as expected.
+ */
+#include <linux/io-64-nonatomic-lo-hi.h>
 
 #include <asm/byteorder.h>
 #include <asm/io.h>
diff --git a/drivers/pci/controller/Kconfig b/drivers/pci/controller/Kconfig
index d1c5fcf00a8a..bfd9bac37e24 100644
--- a/drivers/pci/controller/Kconfig
+++ b/drivers/pci/controller/Kconfig
@@ -274,7 +274,7 @@ config VMD
 
 config PCIE_BRCMSTB
 	tristate "Broadcom Brcmstb PCIe host controller"
-	depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCM4908 || \
+	depends on ARCH_BRCMSTB || ARCH_BCM2835 || ARCH_BCMBCA || \
 		   BMIPS_GENERIC || COMPILE_TEST
 	depends on OF
 	depends on PCI_MSI_IRQ_DOMAIN
diff --git a/drivers/pcmcia/Kconfig b/drivers/pcmcia/Kconfig
index bf495bf0f48a..1525023e49b6 100644
--- a/drivers/pcmcia/Kconfig
+++ b/drivers/pcmcia/Kconfig
@@ -240,10 +240,6 @@ config PCMCIA_PROBE
 	bool
 	default y if ISA && !ARCH_SA1100 && !PARISC
 
-config PCMCIA_VRC4171
-	tristate "NEC VRC4171 Card Controllers support"
-	depends on CPU_VR41XX && ISA && PCMCIA
-
 config OMAP_CF
 	tristate "OMAP CompactFlash Controller"
 	depends on PCMCIA
@@ -252,15 +248,6 @@ config OMAP_CF
 	  Say Y here to support the CompactFlash controller on OMAP.
 	  Note that this doesn't support "True IDE" mode.
 
-config AT91_CF
-	tristate "AT91 CompactFlash Controller"
-	depends on PCI
-	depends on OF
-	depends on PCMCIA && ARCH_AT91
-	help
-	  Say Y here to support the CompactFlash controller on AT91 chips.
-	  Or choose M to compile the driver as a module named "at91_cf".
-
 config ELECTRA_CF
 	tristate "Electra CompactFlash Controller"
 	depends on PCMCIA && PPC_PASEMI
diff --git a/drivers/pcmcia/Makefile b/drivers/pcmcia/Makefile
index c59ddde42007..b3a2accf47af 100644
--- a/drivers/pcmcia/Makefile
+++ b/drivers/pcmcia/Makefile
@@ -29,9 +29,7 @@ obj-$(CONFIG_PCMCIA_SA11XX_BASE)		+= sa11xx_base.o
 obj-$(CONFIG_PCMCIA_SA1100)			+= sa1100_cs.o
 obj-$(CONFIG_PCMCIA_SA1111)			+= sa1111_cs.o
 obj-$(CONFIG_PCMCIA_BCM63XX)			+= bcm63xx_pcmcia.o
-obj-$(CONFIG_PCMCIA_VRC4171)			+= vrc4171_card.o
 obj-$(CONFIG_OMAP_CF)				+= omap_cf.o
-obj-$(CONFIG_AT91_CF)				+= at91_cf.o
 obj-$(CONFIG_ELECTRA_CF)			+= electra_cf.o
 obj-$(CONFIG_PCMCIA_ALCHEMY_DEVBOARD)		+= db1xxx_ss.o
 obj-$(CONFIG_PCMCIA_MAX1600)			+= max1600.o
diff --git a/drivers/pcmcia/at91_cf.c b/drivers/pcmcia/at91_cf.c
deleted file mode 100644
index 92df2c2c5d07..000000000000
--- a/drivers/pcmcia/at91_cf.c
+++ /dev/null
@@ -1,407 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * at91_cf.c -- AT91 CompactFlash controller driver
- *
- * Copyright (C) 2005 David Brownell
- */
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/platform_device.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/interrupt.h>
-#include <linux/slab.h>
-#include <linux/gpio.h>
-#include <linux/io.h>
-#include <linux/sizes.h>
-#include <linux/mfd/syscon.h>
-#include <linux/mfd/syscon/atmel-mc.h>
-#include <linux/of.h>
-#include <linux/of_device.h>
-#include <linux/of_gpio.h>
-#include <linux/pci.h>
-#include <linux/regmap.h>
-
-#include <pcmcia/ss.h>
-
-/*
- * A0..A10 work in each range; A23 indicates I/O space;  A25 is CFRNW;
- * some other bit in {A24,A22..A11} is nREG to flag memory access
- * (vs attributes).  So more than 2KB/region would just be waste.
- * Note: These are offsets from the physical base address.
- */
-#define	CF_ATTR_PHYS	(0)
-#define	CF_IO_PHYS	(1 << 23)
-#define	CF_MEM_PHYS	(0x017ff800)
-
-struct at91_cf_data {
-	int	irq_pin;		/* I/O IRQ */
-	int	det_pin;		/* Card detect */
-	int	vcc_pin;		/* power switching */
-	int	rst_pin;		/* card reset */
-	u8	chipselect;		/* EBI Chip Select number */
-	u8	flags;
-#define AT91_CF_TRUE_IDE	0x01
-#define AT91_IDE_SWAP_A0_A2	0x02
-};
-
-struct regmap *mc;
-
-/*--------------------------------------------------------------------------*/
-
-struct at91_cf_socket {
-	struct pcmcia_socket	socket;
-
-	unsigned		present:1;
-
-	struct platform_device	*pdev;
-	struct at91_cf_data	*board;
-
-	unsigned long		phys_baseaddr;
-};
-
-static inline int at91_cf_present(struct at91_cf_socket *cf)
-{
-	return !gpio_get_value(cf->board->det_pin);
-}
-
-/*--------------------------------------------------------------------------*/
-
-static int at91_cf_ss_init(struct pcmcia_socket *s)
-{
-	return 0;
-}
-
-static irqreturn_t at91_cf_irq(int irq, void *_cf)
-{
-	struct at91_cf_socket *cf = _cf;
-
-	if (irq == gpio_to_irq(cf->board->det_pin)) {
-		unsigned present = at91_cf_present(cf);
-
-		/* kick pccard as needed */
-		if (present != cf->present) {
-			cf->present = present;
-			dev_dbg(&cf->pdev->dev, "card %s\n",
-					present ? "present" : "gone");
-			pcmcia_parse_events(&cf->socket, SS_DETECT);
-		}
-	}
-
-	return IRQ_HANDLED;
-}
-
-static int at91_cf_get_status(struct pcmcia_socket *s, u_int *sp)
-{
-	struct at91_cf_socket	*cf;
-
-	if (!sp)
-		return -EINVAL;
-
-	cf = container_of(s, struct at91_cf_socket, socket);
-
-	/* NOTE: CF is always 3VCARD */
-	if (at91_cf_present(cf)) {
-		int rdy	= gpio_is_valid(cf->board->irq_pin);	/* RDY/nIRQ */
-		int vcc	= gpio_is_valid(cf->board->vcc_pin);
-
-		*sp = SS_DETECT | SS_3VCARD;
-		if (!rdy || gpio_get_value(cf->board->irq_pin))
-			*sp |= SS_READY;
-		if (!vcc || gpio_get_value(cf->board->vcc_pin))
-			*sp |= SS_POWERON;
-	} else
-		*sp = 0;
-
-	return 0;
-}
-
-static int
-at91_cf_set_socket(struct pcmcia_socket *sock, struct socket_state_t *s)
-{
-	struct at91_cf_socket	*cf;
-
-	cf = container_of(sock, struct at91_cf_socket, socket);
-
-	/* switch Vcc if needed and possible */
-	if (gpio_is_valid(cf->board->vcc_pin)) {
-		switch (s->Vcc) {
-		case 0:
-			gpio_set_value(cf->board->vcc_pin, 0);
-			break;
-		case 33:
-			gpio_set_value(cf->board->vcc_pin, 1);
-			break;
-		default:
-			return -EINVAL;
-		}
-	}
-
-	/* toggle reset if needed */
-	gpio_set_value(cf->board->rst_pin, s->flags & SS_RESET);
-
-	dev_dbg(&cf->pdev->dev, "Vcc %d, io_irq %d, flags %04x csc %04x\n",
-				s->Vcc, s->io_irq, s->flags, s->csc_mask);
-
-	return 0;
-}
-
-static int at91_cf_ss_suspend(struct pcmcia_socket *s)
-{
-	return at91_cf_set_socket(s, &dead_socket);
-}
-
-/* we already mapped the I/O region */
-static int at91_cf_set_io_map(struct pcmcia_socket *s, struct pccard_io_map *io)
-{
-	struct at91_cf_socket	*cf;
-	u32			csr;
-
-	cf = container_of(s, struct at91_cf_socket, socket);
-	io->flags &= (MAP_ACTIVE | MAP_16BIT | MAP_AUTOSZ);
-
-	/*
-	 * Use 16 bit accesses unless/until we need 8-bit i/o space.
-	 *
-	 * NOTE: this CF controller ignores IOIS16, so we can't really do
-	 * MAP_AUTOSZ.  The 16bit mode allows single byte access on either
-	 * D0-D7 (even addr) or D8-D15 (odd), so it's close enough for many
-	 * purposes (and handles ide-cs).
-	 *
-	 * The 8bit mode is needed for odd byte access on D0-D7.  It seems
-	 * some cards only like that way to get at the odd byte, despite
-	 * CF 3.0 spec table 35 also giving the D8-D15 option.
-	 */
-	if (!(io->flags & (MAP_16BIT | MAP_AUTOSZ))) {
-		csr = AT91_MC_SMC_DBW_8;
-		dev_dbg(&cf->pdev->dev, "8bit i/o bus\n");
-	} else {
-		csr = AT91_MC_SMC_DBW_16;
-		dev_dbg(&cf->pdev->dev, "16bit i/o bus\n");
-	}
-	regmap_update_bits(mc, AT91_MC_SMC_CSR(cf->board->chipselect),
-			   AT91_MC_SMC_DBW, csr);
-
-	io->start = cf->socket.io_offset;
-	io->stop = io->start + SZ_2K - 1;
-
-	return 0;
-}
-
-/* pcmcia layer maps/unmaps mem regions */
-static int
-at91_cf_set_mem_map(struct pcmcia_socket *s, struct pccard_mem_map *map)
-{
-	struct at91_cf_socket	*cf;
-
-	if (map->card_start)
-		return -EINVAL;
-
-	cf = container_of(s, struct at91_cf_socket, socket);
-
-	map->flags &= (MAP_ACTIVE | MAP_ATTRIB | MAP_16BIT);
-	if (map->flags & MAP_ATTRIB)
-		map->static_start = cf->phys_baseaddr + CF_ATTR_PHYS;
-	else
-		map->static_start = cf->phys_baseaddr + CF_MEM_PHYS;
-
-	return 0;
-}
-
-static struct pccard_operations at91_cf_ops = {
-	.init			= at91_cf_ss_init,
-	.suspend		= at91_cf_ss_suspend,
-	.get_status		= at91_cf_get_status,
-	.set_socket		= at91_cf_set_socket,
-	.set_io_map		= at91_cf_set_io_map,
-	.set_mem_map		= at91_cf_set_mem_map,
-};
-
-/*--------------------------------------------------------------------------*/
-
-static const struct of_device_id at91_cf_dt_ids[] = {
-	{ .compatible = "atmel,at91rm9200-cf" },
-	{ /* sentinel */ }
-};
-MODULE_DEVICE_TABLE(of, at91_cf_dt_ids);
-
-static int at91_cf_probe(struct platform_device *pdev)
-{
-	struct at91_cf_socket	*cf;
-	struct at91_cf_data	*board;
-	struct resource		*io;
-	struct resource		realio;
-	int			status;
-
-	board = devm_kzalloc(&pdev->dev, sizeof(*board), GFP_KERNEL);
-	if (!board)
-		return -ENOMEM;
-
-	board->irq_pin = of_get_gpio(pdev->dev.of_node, 0);
-	board->det_pin = of_get_gpio(pdev->dev.of_node, 1);
-	board->vcc_pin = of_get_gpio(pdev->dev.of_node, 2);
-	board->rst_pin = of_get_gpio(pdev->dev.of_node, 3);
-
-	mc = syscon_regmap_lookup_by_compatible("atmel,at91rm9200-sdramc");
-	if (IS_ERR(mc))
-		return PTR_ERR(mc);
-
-	if (!gpio_is_valid(board->det_pin) || !gpio_is_valid(board->rst_pin))
-		return -ENODEV;
-
-	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!io)
-		return -ENODEV;
-
-	cf = devm_kzalloc(&pdev->dev, sizeof(*cf), GFP_KERNEL);
-	if (!cf)
-		return -ENOMEM;
-
-	cf->board = board;
-	cf->pdev = pdev;
-	cf->phys_baseaddr = io->start;
-	platform_set_drvdata(pdev, cf);
-
-	/* must be a GPIO; ergo must trigger on both edges */
-	status = devm_gpio_request(&pdev->dev, board->det_pin, "cf_det");
-	if (status < 0)
-		return status;
-
-	status = devm_request_irq(&pdev->dev, gpio_to_irq(board->det_pin),
-					at91_cf_irq, 0, "at91_cf detect", cf);
-	if (status < 0)
-		return status;
-
-	device_init_wakeup(&pdev->dev, 1);
-
-	status = devm_gpio_request(&pdev->dev, board->rst_pin, "cf_rst");
-	if (status < 0)
-		goto fail0a;
-
-	if (gpio_is_valid(board->vcc_pin)) {
-		status = devm_gpio_request(&pdev->dev, board->vcc_pin, "cf_vcc");
-		if (status < 0)
-			goto fail0a;
-	}
-
-	/*
-	 * The card driver will request this irq later as needed.
-	 * but it causes lots of "irqNN: nobody cared" messages
-	 * unless we report that we handle everything (sigh).
-	 * (Note:  DK board doesn't wire the IRQ pin...)
-	 */
-	if (gpio_is_valid(board->irq_pin)) {
-		status = devm_gpio_request(&pdev->dev, board->irq_pin, "cf_irq");
-		if (status < 0)
-			goto fail0a;
-
-		status = devm_request_irq(&pdev->dev, gpio_to_irq(board->irq_pin),
-					at91_cf_irq, IRQF_SHARED, "at91_cf", cf);
-		if (status < 0)
-			goto fail0a;
-		cf->socket.pci_irq = gpio_to_irq(board->irq_pin);
-	} else
-		cf->socket.pci_irq = nr_irqs + 1;
-
-	/*
-	 * pcmcia layer only remaps "real" memory not iospace
-	 * io_offset is set to 0x10000 to avoid the check in static_find_io().
-	 * */
-	cf->socket.io_offset = 0x10000;
-	realio.start = cf->socket.io_offset;
-	realio.end = realio.start + SZ_64K - 1;
-	status = pci_remap_iospace(&realio, cf->phys_baseaddr + CF_IO_PHYS);
-	if (status)
-		goto fail0a;
-
-	/* reserve chip-select regions */
-	if (!devm_request_mem_region(&pdev->dev, io->start, resource_size(io), "at91_cf")) {
-		status = -ENXIO;
-		goto fail0a;
-	}
-
-	dev_info(&pdev->dev, "irqs det #%d, io #%d\n",
-		gpio_to_irq(board->det_pin), gpio_to_irq(board->irq_pin));
-
-	cf->socket.owner = THIS_MODULE;
-	cf->socket.dev.parent = &pdev->dev;
-	cf->socket.ops = &at91_cf_ops;
-	cf->socket.resource_ops = &pccard_static_ops;
-	cf->socket.features = SS_CAP_PCCARD | SS_CAP_STATIC_MAP
-				| SS_CAP_MEM_ALIGN;
-	cf->socket.map_size = SZ_2K;
-	cf->socket.io[0].res = io;
-
-	status = pcmcia_register_socket(&cf->socket);
-	if (status < 0)
-		goto fail0a;
-
-	return 0;
-
-fail0a:
-	device_init_wakeup(&pdev->dev, 0);
-	return status;
-}
-
-static int at91_cf_remove(struct platform_device *pdev)
-{
-	struct at91_cf_socket	*cf = platform_get_drvdata(pdev);
-
-	pcmcia_unregister_socket(&cf->socket);
-	device_init_wakeup(&pdev->dev, 0);
-
-	return 0;
-}
-
-#ifdef	CONFIG_PM
-
-static int at91_cf_suspend(struct platform_device *pdev, pm_message_t mesg)
-{
-	struct at91_cf_socket	*cf = platform_get_drvdata(pdev);
-	struct at91_cf_data	*board = cf->board;
-
-	if (device_may_wakeup(&pdev->dev)) {
-		enable_irq_wake(gpio_to_irq(board->det_pin));
-		if (gpio_is_valid(board->irq_pin))
-			enable_irq_wake(gpio_to_irq(board->irq_pin));
-	}
-	return 0;
-}
-
-static int at91_cf_resume(struct platform_device *pdev)
-{
-	struct at91_cf_socket	*cf = platform_get_drvdata(pdev);
-	struct at91_cf_data	*board = cf->board;
-
-	if (device_may_wakeup(&pdev->dev)) {
-		disable_irq_wake(gpio_to_irq(board->det_pin));
-		if (gpio_is_valid(board->irq_pin))
-			disable_irq_wake(gpio_to_irq(board->irq_pin));
-	}
-
-	return 0;
-}
-
-#else
-#define	at91_cf_suspend		NULL
-#define	at91_cf_resume		NULL
-#endif
-
-static struct platform_driver at91_cf_driver = {
-	.driver = {
-		.name		= "at91_cf",
-		.of_match_table = at91_cf_dt_ids,
-	},
-	.probe		= at91_cf_probe,
-	.remove		= at91_cf_remove,
-	.suspend	= at91_cf_suspend,
-	.resume		= at91_cf_resume,
-};
-
-module_platform_driver(at91_cf_driver);
-
-MODULE_DESCRIPTION("AT91 Compact Flash Driver");
-MODULE_AUTHOR("David Brownell");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:at91_cf");
diff --git a/drivers/pcmcia/i82092.c b/drivers/pcmcia/i82092.c
index 192c9049d654..a335748bdef5 100644
--- a/drivers/pcmcia/i82092.c
+++ b/drivers/pcmcia/i82092.c
@@ -661,12 +661,12 @@ static int i82092aa_set_mem_map(struct pcmcia_socket *socket,
 	return 0;
 }
 
-static int i82092aa_module_init(void)
+static int __init i82092aa_module_init(void)
 {
 	return pci_register_driver(&i82092aa_pci_driver);
 }
 
-static void i82092aa_module_exit(void)
+static void __exit i82092aa_module_exit(void)
 {
 	pci_unregister_driver(&i82092aa_pci_driver);
 	if (sockets[0].io_base > 0)
diff --git a/drivers/pcmcia/omap_cf.c b/drivers/pcmcia/omap_cf.c
index 1972a8f6fa8e..d3f827d4224a 100644
--- a/drivers/pcmcia/omap_cf.c
+++ b/drivers/pcmcia/omap_cf.c
@@ -124,8 +124,6 @@ static int omap_cf_get_status(struct pcmcia_socket *s, u_int *sp)
 static int
 omap_cf_set_socket(struct pcmcia_socket *sock, struct socket_state_t *s)
 {
-	u16		control;
-
 	/* REVISIT some non-OSK boards may support power switching */
 	switch (s->Vcc) {
 	case 0:
@@ -135,7 +133,7 @@ omap_cf_set_socket(struct pcmcia_socket *sock, struct socket_state_t *s)
 		return -EINVAL;
 	}
 
-	control = omap_readw(CF_CONTROL);
+	omap_readw(CF_CONTROL);
 	if (s->flags & SS_RESET)
 		omap_writew(CF_CONTROL_RESET, CF_CONTROL);
 	else
diff --git a/drivers/pcmcia/sa1100_generic.c b/drivers/pcmcia/sa1100_generic.c
index 47b060c57418..c2b6e828c2c6 100644
--- a/drivers/pcmcia/sa1100_generic.c
+++ b/drivers/pcmcia/sa1100_generic.c
@@ -125,7 +125,7 @@ static int sa11x0_drv_pcmcia_legacy_probe(struct platform_device *dev)
 	return ret;
 }
 
-static int sa11x0_drv_pcmcia_legacy_remove(struct platform_device *dev)
+static void sa11x0_drv_pcmcia_legacy_remove(struct platform_device *dev)
 {
 	struct skt_dev_info *sinfo = platform_get_drvdata(dev);
 	int i;
@@ -134,8 +134,6 @@ static int sa11x0_drv_pcmcia_legacy_remove(struct platform_device *dev)
 
 	for (i = 0; i < sinfo->nskt; i++)
 		soc_pcmcia_remove_one(&sinfo->skt[i]);
-
-	return 0;
 }
 
 static int sa11x0_drv_pcmcia_probe(struct platform_device *pdev)
@@ -167,8 +165,10 @@ static int sa11x0_drv_pcmcia_remove(struct platform_device *dev)
 {
 	struct soc_pcmcia_socket *skt;
 
-	if (dev->id == -1)
-		return sa11x0_drv_pcmcia_legacy_remove(dev);
+	if (dev->id == -1) {
+		sa11x0_drv_pcmcia_legacy_remove(dev);
+		return 0;
+	}
 
 	skt = platform_get_drvdata(dev);
 
diff --git a/drivers/pcmcia/vrc4171_card.c b/drivers/pcmcia/vrc4171_card.c
deleted file mode 100644
index 177d77892144..000000000000
--- a/drivers/pcmcia/vrc4171_card.c
+++ /dev/null
@@ -1,745 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * vrc4171_card.c, NEC VRC4171 Card Controller driver for Socket Services.
- *
- * Copyright (C) 2003-2005  Yoichi Yuasa <yuasa@linux-mips.org>
- */
-#include <linux/init.h>
-#include <linux/ioport.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/spinlock.h>
-#include <linux/types.h>
-#include <linux/platform_device.h>
-
-#include <asm/io.h>
-
-#include <pcmcia/ss.h>
-
-#include "i82365.h"
-
-MODULE_DESCRIPTION("NEC VRC4171 Card Controllers driver for Socket Services");
-MODULE_AUTHOR("Yoichi Yuasa <yuasa@linux-mips.org>");
-MODULE_LICENSE("GPL");
-
-#define CARD_MAX_SLOTS		2
-#define CARD_SLOTA		0
-#define CARD_SLOTB		1
-#define CARD_SLOTB_OFFSET	0x40
-
-#define CARD_MEM_START		0x10000000
-#define CARD_MEM_END		0x13ffffff
-#define CARD_MAX_MEM_OFFSET	0x3ffffff
-#define CARD_MAX_MEM_SPEED	1000
-
-#define CARD_CONTROLLER_INDEX	0x03e0
-#define CARD_CONTROLLER_DATA	0x03e1
- /* Power register */
-  #define VPP_GET_VCC		0x01
-  #define POWER_ENABLE		0x10
- #define CARD_VOLTAGE_SENSE	0x1f
-  #define VCC_3VORXV_CAPABLE	0x00
-  #define VCC_XV_ONLY		0x01
-  #define VCC_3V_CAPABLE	0x02
-  #define VCC_5V_ONLY		0x03
- #define CARD_VOLTAGE_SELECT	0x2f
-  #define VCC_3V		0x01
-  #define VCC_5V		0x00
-  #define VCC_XV		0x02
-  #define VCC_STATUS_3V		0x02
-  #define VCC_STATUS_5V		0x01
-  #define VCC_STATUS_XV		0x03
- #define GLOBAL_CONTROL		0x1e
-  #define EXWRBK		0x04
-  #define IRQPM_EN		0x08
-  #define CLRPMIRQ		0x10
-
-#define INTERRUPT_STATUS	0x05fa
- #define IRQ_A			0x02
- #define IRQ_B			0x04
-
-#define CONFIGURATION1		0x05fe
- #define SLOTB_CONFIG		0xc000
- #define SLOTB_NONE		0x0000
- #define SLOTB_PCCARD		0x4000
- #define SLOTB_CF		0x8000
- #define SLOTB_FLASHROM		0xc000
-
-#define CARD_CONTROLLER_START	CARD_CONTROLLER_INDEX
-#define CARD_CONTROLLER_END	CARD_CONTROLLER_DATA
-
-#define IO_MAX_MAPS	2
-#define MEM_MAX_MAPS	5
-
-enum vrc4171_slot {
-	SLOT_PROBE = 0,
-	SLOT_NOPROBE_IO,
-	SLOT_NOPROBE_MEM,
-	SLOT_NOPROBE_ALL,
-	SLOT_INITIALIZED,
-};
-
-enum vrc4171_slotb {
-	SLOTB_IS_NONE,
-	SLOTB_IS_PCCARD,
-	SLOTB_IS_CF,
-	SLOTB_IS_FLASHROM,
-};
-
-struct vrc4171_socket {
-	enum vrc4171_slot slot;
-	struct pcmcia_socket pcmcia_socket;
-	char name[24];
-	int csc_irq;
-	int io_irq;
-	spinlock_t lock;
-};
-
-static struct vrc4171_socket vrc4171_sockets[CARD_MAX_SLOTS];
-static enum vrc4171_slotb vrc4171_slotb = SLOTB_IS_NONE;
-static char vrc4171_card_name[] = "NEC VRC4171 Card Controller";
-static unsigned int vrc4171_irq;
-static uint16_t vrc4171_irq_mask = 0xdeb8;
-
-static struct resource vrc4171_card_resource[3] = {
-	{	.name		= vrc4171_card_name,
-		.start		= CARD_CONTROLLER_START,
-		.end		= CARD_CONTROLLER_END,
-		.flags		= IORESOURCE_IO,	},
-	{	.name		= vrc4171_card_name,
-		.start		= INTERRUPT_STATUS,
-		.end		= INTERRUPT_STATUS,
-		.flags		= IORESOURCE_IO,	},
-	{	.name		= vrc4171_card_name,
-		.start		= CONFIGURATION1,
-		.end		= CONFIGURATION1,
-		.flags		= IORESOURCE_IO,	},
-};
-
-static struct platform_device vrc4171_card_device = {
-	.name		= vrc4171_card_name,
-	.id		= 0,
-	.num_resources	= 3,
-	.resource	= vrc4171_card_resource,
-};
-
-static inline uint16_t vrc4171_get_irq_status(void)
-{
-	return inw(INTERRUPT_STATUS);
-}
-
-static inline void vrc4171_set_multifunction_pin(enum vrc4171_slotb config)
-{
-	uint16_t config1;
-
-	config1 = inw(CONFIGURATION1);
-	config1 &= ~SLOTB_CONFIG;
-
-	switch (config) {
-	case SLOTB_IS_NONE:
-		config1 |= SLOTB_NONE;
-		break;
-	case SLOTB_IS_PCCARD:
-		config1 |= SLOTB_PCCARD;
-		break;
-	case SLOTB_IS_CF:
-		config1 |= SLOTB_CF;
-		break;
-	case SLOTB_IS_FLASHROM:
-		config1 |= SLOTB_FLASHROM;
-		break;
-	default:
-		break;
-	}
-
-	outw(config1, CONFIGURATION1);
-}
-
-static inline uint8_t exca_read_byte(int slot, uint8_t index)
-{
-	if (slot == CARD_SLOTB)
-		index += CARD_SLOTB_OFFSET;
-
-	outb(index, CARD_CONTROLLER_INDEX);
-	return inb(CARD_CONTROLLER_DATA);
-}
-
-static inline uint16_t exca_read_word(int slot, uint8_t index)
-{
-	uint16_t data;
-
-	if (slot == CARD_SLOTB)
-		index += CARD_SLOTB_OFFSET;
-
-	outb(index++, CARD_CONTROLLER_INDEX);
-	data = inb(CARD_CONTROLLER_DATA);
-
-	outb(index, CARD_CONTROLLER_INDEX);
-	data |= ((uint16_t)inb(CARD_CONTROLLER_DATA)) << 8;
-
-	return data;
-}
-
-static inline uint8_t exca_write_byte(int slot, uint8_t index, uint8_t data)
-{
-	if (slot == CARD_SLOTB)
-		index += CARD_SLOTB_OFFSET;
-
-	outb(index, CARD_CONTROLLER_INDEX);
-	outb(data, CARD_CONTROLLER_DATA);
-
-	return data;
-}
-
-static inline uint16_t exca_write_word(int slot, uint8_t index, uint16_t data)
-{
-	if (slot == CARD_SLOTB)
-		index += CARD_SLOTB_OFFSET;
-
-	outb(index++, CARD_CONTROLLER_INDEX);
-	outb(data, CARD_CONTROLLER_DATA);
-
-	outb(index, CARD_CONTROLLER_INDEX);
-	outb((uint8_t)(data >> 8), CARD_CONTROLLER_DATA);
-
-	return data;
-}
-
-static inline int search_nonuse_irq(void)
-{
-	int i;
-
-	for (i = 0; i < 16; i++) {
-		if (vrc4171_irq_mask & (1 << i)) {
-			vrc4171_irq_mask &= ~(1 << i);
-			return i;
-		}
-	}
-
-	return -1;
-}
-
-static int pccard_init(struct pcmcia_socket *sock)
-{
-	struct vrc4171_socket *socket;
-	unsigned int slot;
-
-	sock->features |= SS_CAP_PCCARD | SS_CAP_PAGE_REGS;
-	sock->irq_mask = 0;
-	sock->map_size = 0x1000;
-	sock->pci_irq = vrc4171_irq;
-
-	slot = sock->sock;
-	socket = &vrc4171_sockets[slot];
-	socket->csc_irq = search_nonuse_irq();
-	socket->io_irq = search_nonuse_irq();
-	spin_lock_init(&socket->lock);
-
-	return 0;
-}
-
-static int pccard_get_status(struct pcmcia_socket *sock, u_int *value)
-{
-	unsigned int slot;
-	uint8_t status, sense;
-	u_int val = 0;
-
-	if (sock == NULL || sock->sock >= CARD_MAX_SLOTS || value == NULL)
-		return -EINVAL;
-
-	slot = sock->sock;
-
-	status = exca_read_byte(slot, I365_STATUS);
-	if (exca_read_byte(slot, I365_INTCTL) & I365_PC_IOCARD) {
-		if (status & I365_CS_STSCHG)
-			val |= SS_STSCHG;
-	} else {
-		if (!(status & I365_CS_BVD1))
-			val |= SS_BATDEAD;
-		else if ((status & (I365_CS_BVD1 | I365_CS_BVD2)) == I365_CS_BVD1)
-			val |= SS_BATWARN;
-	}
-	if ((status & I365_CS_DETECT) == I365_CS_DETECT)
-		val |= SS_DETECT;
-	if (status & I365_CS_WRPROT)
-		val |= SS_WRPROT;
-	if (status & I365_CS_READY)
-		val |= SS_READY;
-	if (status & I365_CS_POWERON)
-		val |= SS_POWERON;
-
-	sense = exca_read_byte(slot, CARD_VOLTAGE_SENSE);
-	switch (sense) {
-	case VCC_3VORXV_CAPABLE:
-		val |= SS_3VCARD | SS_XVCARD;
-		break;
-	case VCC_XV_ONLY:
-		val |= SS_XVCARD;
-		break;
-	case VCC_3V_CAPABLE:
-		val |= SS_3VCARD;
-		break;
-	default:
-		/* 5V only */
-		break;
-	}
-
-	*value = val;
-
-	return 0;
-}
-
-static inline uint8_t set_Vcc_value(u_char Vcc)
-{
-	switch (Vcc) {
-	case 33:
-		return VCC_3V;
-	case 50:
-		return VCC_5V;
-	}
-
-	/* Small voltage is chosen for safety. */
-	return VCC_3V;
-}
-
-static int pccard_set_socket(struct pcmcia_socket *sock, socket_state_t *state)
-{
-	struct vrc4171_socket *socket;
-	unsigned int slot;
-	uint8_t voltage, power, control, cscint;
-
-	if (sock == NULL || sock->sock >= CARD_MAX_SLOTS ||
-	    (state->Vpp != state->Vcc && state->Vpp != 0) ||
-	    (state->Vcc != 50 && state->Vcc != 33 && state->Vcc != 0))
-		return -EINVAL;
-
-	slot = sock->sock;
-	socket = &vrc4171_sockets[slot];
-
-	spin_lock_irq(&socket->lock);
-
-	voltage = set_Vcc_value(state->Vcc);
-	exca_write_byte(slot, CARD_VOLTAGE_SELECT, voltage);
-
-	power = POWER_ENABLE;
-	if (state->Vpp == state->Vcc)
-		power |= VPP_GET_VCC;
-	if (state->flags & SS_OUTPUT_ENA)
-		power |= I365_PWR_OUT;
-	exca_write_byte(slot, I365_POWER, power);
-
-	control = 0;
-	if (state->io_irq != 0)
-		control |= socket->io_irq;
-	if (state->flags & SS_IOCARD)
-		control |= I365_PC_IOCARD;
-	if (state->flags & SS_RESET)
-		control	&= ~I365_PC_RESET;
-	else
-		control |= I365_PC_RESET;
-	exca_write_byte(slot, I365_INTCTL, control);
-
-        cscint = 0;
-        exca_write_byte(slot, I365_CSCINT, cscint);
-	exca_read_byte(slot, I365_CSC);	/* clear CardStatus change */
-	if (state->csc_mask != 0)
-		cscint |= socket->csc_irq << 8;
-	if (state->flags & SS_IOCARD) {
-		if (state->csc_mask & SS_STSCHG)
-			cscint |= I365_CSC_STSCHG;
-	} else {
-		if (state->csc_mask & SS_BATDEAD)
-			cscint |= I365_CSC_BVD1;
-		if (state->csc_mask & SS_BATWARN)
-			cscint |= I365_CSC_BVD2;
-	}
-	if (state->csc_mask & SS_READY)
-		cscint |= I365_CSC_READY;
-	if (state->csc_mask & SS_DETECT)
-		cscint |= I365_CSC_DETECT;
-        exca_write_byte(slot, I365_CSCINT, cscint);
-
-	spin_unlock_irq(&socket->lock);
-
-	return 0;
-}
-
-static int pccard_set_io_map(struct pcmcia_socket *sock, struct pccard_io_map *io)
-{
-	unsigned int slot;
-	uint8_t ioctl, addrwin;
-	u_char map;
-
-	if (sock == NULL || sock->sock >= CARD_MAX_SLOTS ||
-	    io == NULL || io->map >= IO_MAX_MAPS ||
-	    io->start > 0xffff || io->stop > 0xffff || io->start > io->stop)
-		return -EINVAL;
-
-	slot = sock->sock;
-	map = io->map;
-
-	addrwin = exca_read_byte(slot, I365_ADDRWIN);
-	if (addrwin & I365_ENA_IO(map)) {
-		addrwin &= ~I365_ENA_IO(map);
-		exca_write_byte(slot, I365_ADDRWIN, addrwin);
-	}
-
-	exca_write_word(slot, I365_IO(map)+I365_W_START, io->start);
-	exca_write_word(slot, I365_IO(map)+I365_W_STOP, io->stop);
-
-	ioctl = 0;
-	if (io->speed > 0)
-		ioctl |= I365_IOCTL_WAIT(map);
-	if (io->flags & MAP_16BIT)
-		ioctl |= I365_IOCTL_16BIT(map);
-	if (io->flags & MAP_AUTOSZ)
-		ioctl |= I365_IOCTL_IOCS16(map);
-	if (io->flags & MAP_0WS)
-		ioctl |= I365_IOCTL_0WS(map);
-	exca_write_byte(slot, I365_IOCTL, ioctl);
-
-	if (io->flags & MAP_ACTIVE) {
-		addrwin |= I365_ENA_IO(map);
-		exca_write_byte(slot, I365_ADDRWIN, addrwin);
-	}
-
-	return 0;
-}
-
-static int pccard_set_mem_map(struct pcmcia_socket *sock, struct pccard_mem_map *mem)
-{
-	unsigned int slot;
-	uint16_t start, stop, offset;
-	uint8_t addrwin;
-	u_char map;
-
-	if (sock == NULL || sock->sock >= CARD_MAX_SLOTS ||
-	    mem == NULL || mem->map >= MEM_MAX_MAPS ||
-	    mem->res->start < CARD_MEM_START || mem->res->start > CARD_MEM_END ||
-	    mem->res->end < CARD_MEM_START || mem->res->end > CARD_MEM_END ||
-	    mem->res->start > mem->res->end ||
-	    mem->card_start > CARD_MAX_MEM_OFFSET ||
-	    mem->speed > CARD_MAX_MEM_SPEED)
-		return -EINVAL;
-
-	slot = sock->sock;
-	map = mem->map;
-
-	addrwin = exca_read_byte(slot, I365_ADDRWIN);
-	if (addrwin & I365_ENA_MEM(map)) {
-		addrwin &= ~I365_ENA_MEM(map);
-		exca_write_byte(slot, I365_ADDRWIN, addrwin);
-	}
-
-	start = (mem->res->start >> 12) & 0x3fff;
-	if (mem->flags & MAP_16BIT)
-		start |= I365_MEM_16BIT;
-	exca_write_word(slot, I365_MEM(map)+I365_W_START, start);
-
-	stop = (mem->res->end >> 12) & 0x3fff;
-	switch (mem->speed) {
-	case 0:
-		break;
-	case 1:
-		stop |= I365_MEM_WS0;
-		break;
-	case 2:
-		stop |= I365_MEM_WS1;
-		break;
-	default:
-		stop |= I365_MEM_WS0 | I365_MEM_WS1;
-		break;
-	}
-	exca_write_word(slot, I365_MEM(map)+I365_W_STOP, stop);
-
-	offset = (mem->card_start >> 12) & 0x3fff;
-	if (mem->flags & MAP_ATTRIB)
-		offset |= I365_MEM_REG;
-	if (mem->flags & MAP_WRPROT)
-		offset |= I365_MEM_WRPROT;
-	exca_write_word(slot, I365_MEM(map)+I365_W_OFF, offset);
-
-	if (mem->flags & MAP_ACTIVE) {
-		addrwin |= I365_ENA_MEM(map);
-		exca_write_byte(slot, I365_ADDRWIN, addrwin);
-	}
-
-	return 0;
-}
-
-static struct pccard_operations vrc4171_pccard_operations = {
-	.init			= pccard_init,
-	.get_status		= pccard_get_status,
-	.set_socket		= pccard_set_socket,
-	.set_io_map		= pccard_set_io_map,
-	.set_mem_map		= pccard_set_mem_map,
-};
-
-static inline unsigned int get_events(int slot)
-{
-	unsigned int events = 0;
-	uint8_t status, csc;
-
-	status = exca_read_byte(slot, I365_STATUS);
-	csc = exca_read_byte(slot, I365_CSC);
-
-	if (exca_read_byte(slot, I365_INTCTL) & I365_PC_IOCARD) {
-		if ((csc & I365_CSC_STSCHG) && (status & I365_CS_STSCHG))
-			events |= SS_STSCHG;
-	} else {
-		if (csc & (I365_CSC_BVD1 | I365_CSC_BVD2)) {
-			if (!(status & I365_CS_BVD1))
-				events |= SS_BATDEAD;
-			else if ((status & (I365_CS_BVD1 | I365_CS_BVD2)) == I365_CS_BVD1)
-				events |= SS_BATWARN;
-		}
-	}
-	if ((csc & I365_CSC_READY) && (status & I365_CS_READY))
-		events |= SS_READY;
-	if ((csc & I365_CSC_DETECT) && ((status & I365_CS_DETECT) == I365_CS_DETECT))
-		events |= SS_DETECT;
-
-	return events;
-}
-
-static irqreturn_t pccard_interrupt(int irq, void *dev_id)
-{
-	struct vrc4171_socket *socket;
-	unsigned int events;
-	irqreturn_t retval = IRQ_NONE;
-	uint16_t status;
-
-	status = vrc4171_get_irq_status();
-	if (status & IRQ_A) {
-		socket = &vrc4171_sockets[CARD_SLOTA];
-		if (socket->slot == SLOT_INITIALIZED) {
-			if (status & (1 << socket->csc_irq)) {
-				events = get_events(CARD_SLOTA);
-				if (events != 0) {
-					pcmcia_parse_events(&socket->pcmcia_socket, events);
-					retval = IRQ_HANDLED;
-				}
-			}
-		}
-	}
-
-	if (status & IRQ_B) {
-		socket = &vrc4171_sockets[CARD_SLOTB];
-		if (socket->slot == SLOT_INITIALIZED) {
-			if (status & (1 << socket->csc_irq)) {
-				events = get_events(CARD_SLOTB);
-				if (events != 0) {
-					pcmcia_parse_events(&socket->pcmcia_socket, events);
-					retval = IRQ_HANDLED;
-				}
-			}
-		}
-	}
-
-	return retval;
-}
-
-static inline void reserve_using_irq(int slot)
-{
-	unsigned int irq;
-
-	irq = exca_read_byte(slot, I365_INTCTL);
-	irq &= 0x0f;
-	vrc4171_irq_mask &= ~(1 << irq);
-
-	irq = exca_read_byte(slot, I365_CSCINT);
-	irq = (irq & 0xf0) >> 4;
-	vrc4171_irq_mask &= ~(1 << irq);
-}
-
-static int vrc4171_add_sockets(void)
-{
-	struct vrc4171_socket *socket;
-	int slot, retval;
-
-	for (slot = 0; slot < CARD_MAX_SLOTS; slot++) {
-		if (slot == CARD_SLOTB && vrc4171_slotb == SLOTB_IS_NONE)
-			continue;
-
-		socket = &vrc4171_sockets[slot];
-		if (socket->slot != SLOT_PROBE) {
-			uint8_t addrwin;
-
-			switch (socket->slot) {
-			case SLOT_NOPROBE_MEM:
-				addrwin = exca_read_byte(slot, I365_ADDRWIN);
-				addrwin &= 0x1f;
-				exca_write_byte(slot, I365_ADDRWIN, addrwin);
-				break;
-			case SLOT_NOPROBE_IO:
-				addrwin = exca_read_byte(slot, I365_ADDRWIN);
-				addrwin &= 0xc0;
-				exca_write_byte(slot, I365_ADDRWIN, addrwin);
-				break;
-			default:
-				break;
-			}
-
-			reserve_using_irq(slot);
-			continue;
-		}
-
-		sprintf(socket->name, "NEC VRC4171 Card Slot %1c", 'A' + slot);
-		socket->pcmcia_socket.dev.parent = &vrc4171_card_device.dev;
-		socket->pcmcia_socket.ops = &vrc4171_pccard_operations;
-		socket->pcmcia_socket.owner = THIS_MODULE;
-
-		retval = pcmcia_register_socket(&socket->pcmcia_socket);
-		if (retval < 0)
-			return retval;
-
-		exca_write_byte(slot, I365_ADDRWIN, 0);
-		exca_write_byte(slot, GLOBAL_CONTROL, 0);
-
-		socket->slot = SLOT_INITIALIZED;
-	}
-
-	return 0;
-}
-
-static void vrc4171_remove_sockets(void)
-{
-	struct vrc4171_socket *socket;
-	int slot;
-
-	for (slot = 0; slot < CARD_MAX_SLOTS; slot++) {
-		if (slot == CARD_SLOTB && vrc4171_slotb == SLOTB_IS_NONE)
-			continue;
-
-		socket = &vrc4171_sockets[slot];
-		if (socket->slot == SLOT_INITIALIZED)
-			pcmcia_unregister_socket(&socket->pcmcia_socket);
-
-		socket->slot = SLOT_PROBE;
-	}
-}
-
-static int vrc4171_card_setup(char *options)
-{
-	if (options == NULL || *options == '\0')
-		return 1;
-
-	if (strncmp(options, "irq:", 4) == 0) {
-		int irq;
-		options += 4;
-		irq = simple_strtoul(options, &options, 0);
-		if (irq >= 0 && irq < nr_irqs)
-			vrc4171_irq = irq;
-
-		if (*options != ',')
-			return 1;
-		options++;
-	}
-
-	if (strncmp(options, "slota:", 6) == 0) {
-		options += 6;
-		if (*options != '\0') {
-			if (strncmp(options, "memnoprobe", 10) == 0) {
-				vrc4171_sockets[CARD_SLOTA].slot = SLOT_NOPROBE_MEM;
-				options += 10;
-			} else if (strncmp(options, "ionoprobe", 9) == 0) {
-				vrc4171_sockets[CARD_SLOTA].slot = SLOT_NOPROBE_IO;
-				options += 9;
-			} else if ( strncmp(options, "noprobe", 7) == 0) {
-				vrc4171_sockets[CARD_SLOTA].slot = SLOT_NOPROBE_ALL;
-				options += 7;
-			}
-
-			if (*options != ',')
-				return 1;
-			options++;
-		} else
-			return 1;
-
-	}
-
-	if (strncmp(options, "slotb:", 6) == 0) {
-		options += 6;
-		if (*options != '\0') {
-			if (strncmp(options, "pccard", 6) == 0) {
-				vrc4171_slotb = SLOTB_IS_PCCARD;
-				options += 6;
-			} else if (strncmp(options, "cf", 2) == 0) {
-				vrc4171_slotb = SLOTB_IS_CF;
-				options += 2;
-			} else if (strncmp(options, "flashrom", 8) == 0) {
-				vrc4171_slotb = SLOTB_IS_FLASHROM;
-				options += 8;
-			} else if (strncmp(options, "none", 4) == 0) {
-				vrc4171_slotb = SLOTB_IS_NONE;
-				options += 4;
-			}
-
-			if (*options != ',')
-				return 1;
-			options++;
-
-			if (strncmp(options, "memnoprobe", 10) == 0)
-				vrc4171_sockets[CARD_SLOTB].slot = SLOT_NOPROBE_MEM;
-			if (strncmp(options, "ionoprobe", 9) == 0)
-				vrc4171_sockets[CARD_SLOTB].slot = SLOT_NOPROBE_IO;
-			if (strncmp(options, "noprobe", 7) == 0)
-				vrc4171_sockets[CARD_SLOTB].slot = SLOT_NOPROBE_ALL;
-		}
-	}
-
-	return 1;
-}
-
-__setup("vrc4171_card=", vrc4171_card_setup);
-
-static struct platform_driver vrc4171_card_driver = {
-	.driver = {
-		.name		= vrc4171_card_name,
-	},
-};
-
-static int vrc4171_card_init(void)
-{
-	int retval;
-
-	retval = platform_driver_register(&vrc4171_card_driver);
-	if (retval < 0)
-		return retval;
-
-	retval = platform_device_register(&vrc4171_card_device);
-	if (retval < 0) {
-		platform_driver_unregister(&vrc4171_card_driver);
-		return retval;
-	}
-
-	vrc4171_set_multifunction_pin(vrc4171_slotb);
-
-	retval = vrc4171_add_sockets();
-	if (retval == 0)
-		retval = request_irq(vrc4171_irq, pccard_interrupt, IRQF_SHARED,
-		                     vrc4171_card_name, vrc4171_sockets);
-
-	if (retval < 0) {
-		vrc4171_remove_sockets();
-		platform_device_unregister(&vrc4171_card_device);
-		platform_driver_unregister(&vrc4171_card_driver);
-		return retval;
-	}
-
-	printk(KERN_INFO "%s, connected to IRQ %d\n",
-		vrc4171_card_driver.driver.name, vrc4171_irq);
-
-	return 0;
-}
-
-static void vrc4171_card_exit(void)
-{
-	free_irq(vrc4171_irq, vrc4171_sockets);
-	vrc4171_remove_sockets();
-	platform_device_unregister(&vrc4171_card_device);
-	platform_driver_unregister(&vrc4171_card_driver);
-}
-
-module_init(vrc4171_card_init);
-module_exit(vrc4171_card_exit);
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 1e2d69453771..44c07ea487f4 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -183,6 +183,13 @@ config APPLE_M1_CPU_PMU
 	  Provides support for the non-architectural CPU PMUs present on
 	  the Apple M1 SoCs and derivatives.
 
+config ALIBABA_UNCORE_DRW_PMU
+	tristate "Alibaba T-Head Yitian 710 DDR Sub-system Driveway PMU driver"
+	depends on ARM64 || COMPILE_TEST
+	help
+	  Support for Driveway PMU events monitoring on Yitian 710 DDR
+	  Sub-system.
+
 source "drivers/perf/hisilicon/Kconfig"
 
 config MARVELL_CN10K_DDR_PMU
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index 57a279c61df5..050d04ee19dd 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -20,3 +20,4 @@ obj-$(CONFIG_ARM_DMC620_PMU) += arm_dmc620_pmu.o
 obj-$(CONFIG_MARVELL_CN10K_TAD_PMU) += marvell_cn10k_tad_pmu.o
 obj-$(CONFIG_MARVELL_CN10K_DDR_PMU) += marvell_cn10k_ddr_pmu.o
 obj-$(CONFIG_APPLE_M1_CPU_PMU) += apple_m1_cpu_pmu.o
+obj-$(CONFIG_ALIBABA_UNCORE_DRW_PMU) += alibaba_uncore_drw_pmu.o
diff --git a/drivers/perf/alibaba_uncore_drw_pmu.c b/drivers/perf/alibaba_uncore_drw_pmu.c
new file mode 100644
index 000000000000..82729b874f09
--- /dev/null
+++ b/drivers/perf/alibaba_uncore_drw_pmu.c
@@ -0,0 +1,810 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Alibaba DDR Sub-System Driveway PMU driver
+ *
+ * Copyright (C) 2022 Alibaba Inc
+ */
+
+#define ALI_DRW_PMUNAME		"ali_drw"
+#define ALI_DRW_DRVNAME		ALI_DRW_PMUNAME "_pmu"
+#define pr_fmt(fmt)		ALI_DRW_DRVNAME ": " fmt
+
+#include <linux/acpi.h>
+#include <linux/bitfield.h>
+#include <linux/bitmap.h>
+#include <linux/bitops.h>
+#include <linux/cpuhotplug.h>
+#include <linux/cpumask.h>
+#include <linux/device.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/perf_event.h>
+#include <linux/platform_device.h>
+#include <linux/printk.h>
+#include <linux/rculist.h>
+#include <linux/refcount.h>
+
+
+#define ALI_DRW_PMU_COMMON_MAX_COUNTERS			16
+#define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE	19
+
+#define ALI_DRW_PMU_PA_SHIFT			12
+#define ALI_DRW_PMU_CNT_INIT			0x00000000
+#define ALI_DRW_CNT_MAX_PERIOD			0xffffffff
+#define ALI_DRW_PMU_CYCLE_EVT_ID		0x80
+
+#define ALI_DRW_PMU_CNT_CTRL			0xC00
+#define ALI_DRW_PMU_CNT_RST			BIT(2)
+#define ALI_DRW_PMU_CNT_STOP			BIT(1)
+#define ALI_DRW_PMU_CNT_START			BIT(0)
+
+#define ALI_DRW_PMU_CNT_STATE			0xC04
+#define ALI_DRW_PMU_TEST_CTRL			0xC08
+#define ALI_DRW_PMU_CNT_PRELOAD			0xC0C
+
+#define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK		GENMASK(23, 0)
+#define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK		GENMASK(31, 0)
+#define ALI_DRW_PMU_CYCLE_CNT_HIGH		0xC10
+#define ALI_DRW_PMU_CYCLE_CNT_LOW		0xC14
+
+/* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */
+#define ALI_DRW_PMU_EVENT_SEL0			0xC68
+/* counter 0-3 use sel0, counter 4-7 use sel1...*/
+#define ALI_DRW_PMU_EVENT_SELn(n) \
+	(ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4)
+#define ALI_DRW_PMCOM_CNT_EN			BIT(7)
+#define ALI_DRW_PMCOM_CNT_EVENT_MASK		GENMASK(5, 0)
+#define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \
+	(8 * (n % 4))
+
+/* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */
+#define ALI_DRW_PMU_COMMON_COUNTER0		0xC78
+#define ALI_DRW_PMU_COMMON_COUNTERn(n) \
+	(ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n))
+
+#define ALI_DRW_PMU_OV_INTR_ENABLE_CTL		0xCB8
+#define ALI_DRW_PMU_OV_INTR_DISABLE_CTL		0xCBC
+#define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS	0xCC0
+#define ALI_DRW_PMU_OV_INTR_CLR			0xCC4
+#define ALI_DRW_PMU_OV_INTR_STATUS		0xCC8
+#define ALI_DRW_PMCOM_CNT_OV_INTR_MASK		GENMASK(23, 8)
+#define ALI_DRW_PMBW_CNT_OV_INTR_MASK		GENMASK(7, 0)
+#define ALI_DRW_PMU_OV_INTR_MASK		GENMASK_ULL(63, 0)
+
+static int ali_drw_cpuhp_state_num;
+
+static LIST_HEAD(ali_drw_pmu_irqs);
+static DEFINE_MUTEX(ali_drw_pmu_irqs_lock);
+
+struct ali_drw_pmu_irq {
+	struct hlist_node node;
+	struct list_head irqs_node;
+	struct list_head pmus_node;
+	int irq_num;
+	int cpu;
+	refcount_t refcount;
+};
+
+struct ali_drw_pmu {
+	void __iomem *cfg_base;
+	struct device *dev;
+
+	struct list_head pmus_node;
+	struct ali_drw_pmu_irq *irq;
+	int irq_num;
+	int cpu;
+	DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS);
+	struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
+	int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS];
+
+	struct pmu pmu;
+};
+
+#define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu))
+
+#define DRW_CONFIG_EVENTID		GENMASK(7, 0)
+#define GET_DRW_EVENTID(event)	FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config)
+
+static ssize_t ali_drw_pmu_format_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct dev_ext_attribute *eattr;
+
+	eattr = container_of(attr, struct dev_ext_attribute, attr);
+
+	return sprintf(buf, "%s\n", (char *)eattr->var);
+}
+
+/*
+ * PMU event attributes
+ */
+static ssize_t ali_drw_pmu_event_show(struct device *dev,
+			       struct device_attribute *attr, char *page)
+{
+	struct dev_ext_attribute *eattr;
+
+	eattr = container_of(attr, struct dev_ext_attribute, attr);
+
+	return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var);
+}
+
+#define ALI_DRW_PMU_ATTR(_name, _func, _config)                            \
+		(&((struct dev_ext_attribute[]) {                               \
+				{ __ATTR(_name, 0444, _func, NULL), (void *)_config }   \
+		})[0].attr.attr)
+
+#define ALI_DRW_PMU_FORMAT_ATTR(_name, _config)            \
+	ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config)
+#define ALI_DRW_PMU_EVENT_ATTR(_name, _config)             \
+	ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config)
+
+static struct attribute *ali_drw_pmu_events_attrs[] = {
+	ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr,			0x0),
+	ALI_DRW_PMU_EVENT_ATTR(hif_wr,				0x1),
+	ALI_DRW_PMU_EVENT_ATTR(hif_rd,				0x2),
+	ALI_DRW_PMU_EVENT_ATTR(hif_rmw,				0x3),
+	ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd,			0x4),
+	ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles,		0x7),
+	ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles,		0x8),
+	ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical,		0x9),
+	ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical,		0xA),
+	ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical,		0xB),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_activate,			0xC),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr,			0xD),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate,		0xE),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_rd,			0xF),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_wr,			0x10),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_mwr,			0x11),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_precharge,			0x12),
+	ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr,		0x13),
+	ALI_DRW_PMU_EVENT_ATTR(precharge_for_other,		0x14),
+	ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions,		0x15),
+	ALI_DRW_PMU_EVENT_ATTR(write_combine,			0x16),
+	ALI_DRW_PMU_EVENT_ATTR(war_hazard,			0x17),
+	ALI_DRW_PMU_EVENT_ATTR(raw_hazard,			0x18),
+	ALI_DRW_PMU_EVENT_ATTR(waw_hazard,			0x19),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0,		0x1A),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1,		0x1B),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2,		0x1C),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3,		0x1D),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0,	0x1E),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1,	0x1F),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2,	0x20),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3,	0x21),
+	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0,		0x26),
+	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1,		0x27),
+	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2,		0x28),
+	ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3,		0x29),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_refresh,			0x2A),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref,			0x2B),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode,			0x2D),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl,			0x2E),
+	ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30),
+	ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc,		0x34),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr,		0x35),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr,			0x36),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart,			0x37),
+	ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch,			0x38),
+	ALI_DRW_PMU_EVENT_ATTR(chi_txreq,			0x39),
+	ALI_DRW_PMU_EVENT_ATTR(chi_txdat,			0x3A),
+	ALI_DRW_PMU_EVENT_ATTR(chi_rxdat,			0x3B),
+	ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp,			0x3C),
+	ALI_DRW_PMU_EVENT_ATTR(tsz_vio,				0x3D),
+	ALI_DRW_PMU_EVENT_ATTR(cycle,				0x80),
+	NULL,
+};
+
+static struct attribute_group ali_drw_pmu_events_attr_group = {
+	.name = "events",
+	.attrs = ali_drw_pmu_events_attrs,
+};
+
+static struct attribute *ali_drw_pmu_format_attr[] = {
+	ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"),
+	NULL,
+};
+
+static const struct attribute_group ali_drw_pmu_format_group = {
+	.name = "format",
+	.attrs = ali_drw_pmu_format_attr,
+};
+
+static ssize_t ali_drw_pmu_cpumask_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev));
+
+	return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu));
+}
+
+static struct device_attribute ali_drw_pmu_cpumask_attr =
+		__ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL);
+
+static struct attribute *ali_drw_pmu_cpumask_attrs[] = {
+	&ali_drw_pmu_cpumask_attr.attr,
+	NULL,
+};
+
+static const struct attribute_group ali_drw_pmu_cpumask_attr_group = {
+	.attrs = ali_drw_pmu_cpumask_attrs,
+};
+
+static const struct attribute_group *ali_drw_pmu_attr_groups[] = {
+	&ali_drw_pmu_events_attr_group,
+	&ali_drw_pmu_cpumask_attr_group,
+	&ali_drw_pmu_format_group,
+	NULL,
+};
+
+/* find a counter for event, then in add func, hw.idx will equal to counter */
+static int ali_drw_get_counter_idx(struct perf_event *event)
+{
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
+	int idx;
+
+	for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) {
+		if (!test_and_set_bit(idx, drw_pmu->used_mask))
+			return idx;
+	}
+
+	/* The counters are all in use. */
+	return -EBUSY;
+}
+
+static u64 ali_drw_pmu_read_counter(struct perf_event *event)
+{
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
+	u64 cycle_high, cycle_low;
+
+	if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
+		cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH);
+		cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK;
+		cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW);
+		cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK;
+		return (cycle_high << 32 | cycle_low);
+	}
+
+	return readl(drw_pmu->cfg_base +
+		     ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx));
+}
+
+static void ali_drw_pmu_event_update(struct perf_event *event)
+{
+	struct hw_perf_event *hwc = &event->hw;
+	u64 delta, prev, now;
+
+	do {
+		prev = local64_read(&hwc->prev_count);
+		now = ali_drw_pmu_read_counter(event);
+	} while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
+
+	/* handle overflow. */
+	delta = now - prev;
+	if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID)
+		delta &= ALI_DRW_PMU_OV_INTR_MASK;
+	else
+		delta &= ALI_DRW_CNT_MAX_PERIOD;
+	local64_add(delta, &event->count);
+}
+
+static void ali_drw_pmu_event_set_period(struct perf_event *event)
+{
+	u64 pre_val;
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
+
+	/* set a preload counter for test purpose */
+	writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx,
+	       drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
+
+	/* set conunter initial value */
+	pre_val = ALI_DRW_PMU_CNT_INIT;
+	writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
+	local64_set(&event->hw.prev_count, pre_val);
+
+	/* set sel mode to zero to start test */
+	writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL);
+}
+
+static void ali_drw_pmu_enable_counter(struct perf_event *event)
+{
+	u32 val, subval, reg, shift;
+	int counter = event->hw.idx;
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
+
+	reg = ALI_DRW_PMU_EVENT_SELn(counter);
+	val = readl(drw_pmu->cfg_base + reg);
+	subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) |
+		 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]);
+
+	shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
+	val &= ~(GENMASK(7, 0) << shift);
+	val |= subval << shift;
+
+	writel(val, drw_pmu->cfg_base + reg);
+}
+
+static void ali_drw_pmu_disable_counter(struct perf_event *event)
+{
+	u32 val, reg, subval, shift;
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
+	int counter = event->hw.idx;
+
+	reg = ALI_DRW_PMU_EVENT_SELn(counter);
+	val = readl(drw_pmu->cfg_base + reg);
+	subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) |
+		 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0);
+
+	shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter);
+	val &= ~(GENMASK(7, 0) << shift);
+	val |= subval << shift;
+
+	writel(val, drw_pmu->cfg_base + reg);
+}
+
+static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data)
+{
+	struct ali_drw_pmu_irq *irq = data;
+	struct ali_drw_pmu *drw_pmu;
+	irqreturn_t ret = IRQ_NONE;
+
+	rcu_read_lock();
+	list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) {
+		unsigned long status, clr_status;
+		struct perf_event *event;
+		unsigned int idx;
+
+		for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
+			event = drw_pmu->events[idx];
+			if (!event)
+				continue;
+			ali_drw_pmu_disable_counter(event);
+		}
+
+		/* common counter intr status */
+		status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS);
+		status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status);
+		if (status) {
+			for_each_set_bit(idx, &status,
+					 ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
+				event = drw_pmu->events[idx];
+				if (WARN_ON_ONCE(!event))
+					continue;
+				ali_drw_pmu_event_update(event);
+				ali_drw_pmu_event_set_period(event);
+			}
+
+			/* clear common counter intr status */
+			clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 1);
+			writel(clr_status,
+			       drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
+		}
+
+		for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) {
+			event = drw_pmu->events[idx];
+			if (!event)
+				continue;
+			if (!(event->hw.state & PERF_HES_STOPPED))
+				ali_drw_pmu_enable_counter(event);
+		}
+		if (status)
+			ret = IRQ_HANDLED;
+	}
+	rcu_read_unlock();
+	return ret;
+}
+
+static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device
+						      *pdev, int irq_num)
+{
+	int ret;
+	struct ali_drw_pmu_irq *irq;
+
+	list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) {
+		if (irq->irq_num == irq_num
+		    && refcount_inc_not_zero(&irq->refcount))
+			return irq;
+	}
+
+	irq = kzalloc(sizeof(*irq), GFP_KERNEL);
+	if (!irq)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&irq->pmus_node);
+
+	/* Pick one CPU to be the preferred one to use */
+	irq->cpu = smp_processor_id();
+	refcount_set(&irq->refcount, 1);
+
+	/*
+	 * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same
+	 * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers
+	 * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not
+	 * share with other component.
+	 */
+	ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr,
+			       IRQF_SHARED, dev_name(&pdev->dev), irq);
+	if (ret < 0) {
+		dev_err(&pdev->dev,
+			"Fail to request IRQ:%d ret:%d\n", irq_num, ret);
+		goto out_free;
+	}
+
+	ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu));
+	if (ret)
+		goto out_free;
+
+	ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num,
+					     &irq->node);
+	if (ret)
+		goto out_free;
+
+	irq->irq_num = irq_num;
+	list_add(&irq->irqs_node, &ali_drw_pmu_irqs);
+
+	return irq;
+
+out_free:
+	kfree(irq);
+	return ERR_PTR(ret);
+}
+
+static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu,
+				struct platform_device *pdev)
+{
+	int irq_num;
+	struct ali_drw_pmu_irq *irq;
+
+	/* Read and init IRQ */
+	irq_num = platform_get_irq(pdev, 0);
+	if (irq_num < 0)
+		return irq_num;
+
+	mutex_lock(&ali_drw_pmu_irqs_lock);
+	irq = __ali_drw_pmu_init_irq(pdev, irq_num);
+	mutex_unlock(&ali_drw_pmu_irqs_lock);
+
+	if (IS_ERR(irq))
+		return PTR_ERR(irq);
+
+	drw_pmu->irq = irq;
+
+	mutex_lock(&ali_drw_pmu_irqs_lock);
+	list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node);
+	mutex_unlock(&ali_drw_pmu_irqs_lock);
+
+	return 0;
+}
+
+static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu)
+{
+	struct ali_drw_pmu_irq *irq = drw_pmu->irq;
+
+	mutex_lock(&ali_drw_pmu_irqs_lock);
+	list_del_rcu(&drw_pmu->pmus_node);
+
+	if (!refcount_dec_and_test(&irq->refcount)) {
+		mutex_unlock(&ali_drw_pmu_irqs_lock);
+		return;
+	}
+
+	list_del(&irq->irqs_node);
+	mutex_unlock(&ali_drw_pmu_irqs_lock);
+
+	WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL));
+	cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num,
+					    &irq->node);
+	kfree(irq);
+}
+
+static int ali_drw_pmu_event_init(struct perf_event *event)
+{
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	struct perf_event *sibling;
+	struct device *dev = drw_pmu->pmu.dev;
+
+	if (event->attr.type != event->pmu->type)
+		return -ENOENT;
+
+	if (is_sampling_event(event)) {
+		dev_err(dev, "Sampling not supported!\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (event->attach_state & PERF_ATTACH_TASK) {
+		dev_err(dev, "Per-task counter cannot allocate!\n");
+		return -EOPNOTSUPP;
+	}
+
+	event->cpu = drw_pmu->cpu;
+	if (event->cpu < 0) {
+		dev_err(dev, "Per-task mode not supported!\n");
+		return -EOPNOTSUPP;
+	}
+
+	if (event->group_leader != event &&
+	    !is_software_event(event->group_leader)) {
+		dev_err(dev, "driveway only allow one event!\n");
+		return -EINVAL;
+	}
+
+	for_each_sibling_event(sibling, event->group_leader) {
+		if (sibling != event && !is_software_event(sibling)) {
+			dev_err(dev, "driveway event not allowed!\n");
+			return -EINVAL;
+		}
+	}
+
+	/* reset all the pmu counters */
+	writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
+
+	hwc->idx = -1;
+
+	return 0;
+}
+
+static void ali_drw_pmu_start(struct perf_event *event, int flags)
+{
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
+
+	event->hw.state = 0;
+
+	if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) {
+		writel(ALI_DRW_PMU_CNT_START,
+		       drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
+		return;
+	}
+
+	ali_drw_pmu_event_set_period(event);
+	if (flags & PERF_EF_RELOAD) {
+		unsigned long prev_raw_count =
+		    local64_read(&event->hw.prev_count);
+		writel(prev_raw_count,
+		       drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD);
+	}
+
+	ali_drw_pmu_enable_counter(event);
+
+	writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
+}
+
+static void ali_drw_pmu_stop(struct perf_event *event, int flags)
+{
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
+
+	if (event->hw.state & PERF_HES_STOPPED)
+		return;
+
+	if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID)
+		ali_drw_pmu_disable_counter(event);
+
+	writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
+
+	ali_drw_pmu_event_update(event);
+	event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
+}
+
+static int ali_drw_pmu_add(struct perf_event *event, int flags)
+{
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	int idx = -1;
+	int evtid;
+
+	evtid = GET_DRW_EVENTID(event);
+
+	if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) {
+		idx = ali_drw_get_counter_idx(event);
+		if (idx < 0)
+			return idx;
+		drw_pmu->events[idx] = event;
+		drw_pmu->evtids[idx] = evtid;
+	}
+	hwc->idx = idx;
+
+	hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+
+	if (flags & PERF_EF_START)
+		ali_drw_pmu_start(event, PERF_EF_RELOAD);
+
+	/* Propagate our changes to the userspace mapping. */
+	perf_event_update_userpage(event);
+
+	return 0;
+}
+
+static void ali_drw_pmu_del(struct perf_event *event, int flags)
+{
+	struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu);
+	struct hw_perf_event *hwc = &event->hw;
+	int idx = hwc->idx;
+
+	ali_drw_pmu_stop(event, PERF_EF_UPDATE);
+
+	if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) {
+		drw_pmu->events[idx] = NULL;
+		drw_pmu->evtids[idx] = 0;
+		clear_bit(idx, drw_pmu->used_mask);
+	}
+
+	perf_event_update_userpage(event);
+}
+
+static void ali_drw_pmu_read(struct perf_event *event)
+{
+	ali_drw_pmu_event_update(event);
+}
+
+static int ali_drw_pmu_probe(struct platform_device *pdev)
+{
+	struct ali_drw_pmu *drw_pmu;
+	struct resource *res;
+	char *name;
+	int ret;
+
+	drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL);
+	if (!drw_pmu)
+		return -ENOMEM;
+
+	drw_pmu->dev = &pdev->dev;
+	platform_set_drvdata(pdev, drw_pmu);
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	drw_pmu->cfg_base = devm_ioremap_resource(&pdev->dev, res);
+	if (!drw_pmu->cfg_base)
+		return -ENOMEM;
+
+	name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx",
+			      (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT));
+	if (!name)
+		return -ENOMEM;
+
+	writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL);
+
+	/* enable the generation of interrupt by all common counters */
+	writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
+	       drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL);
+
+	/* clearing interrupt status */
+	writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR);
+
+	drw_pmu->cpu = smp_processor_id();
+
+	ret = ali_drw_pmu_init_irq(drw_pmu, pdev);
+	if (ret)
+		return ret;
+
+	drw_pmu->pmu = (struct pmu) {
+		.module		= THIS_MODULE,
+		.task_ctx_nr	= perf_invalid_context,
+		.event_init	= ali_drw_pmu_event_init,
+		.add		= ali_drw_pmu_add,
+		.del		= ali_drw_pmu_del,
+		.start		= ali_drw_pmu_start,
+		.stop		= ali_drw_pmu_stop,
+		.read		= ali_drw_pmu_read,
+		.attr_groups	= ali_drw_pmu_attr_groups,
+		.capabilities	= PERF_PMU_CAP_NO_EXCLUDE,
+	};
+
+	ret = perf_pmu_register(&drw_pmu->pmu, name, -1);
+	if (ret) {
+		dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n");
+		ali_drw_pmu_uninit_irq(drw_pmu);
+	}
+
+	return ret;
+}
+
+static int ali_drw_pmu_remove(struct platform_device *pdev)
+{
+	struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev);
+
+	/* disable the generation of interrupt by all common counters */
+	writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK,
+	       drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL);
+
+	ali_drw_pmu_uninit_irq(drw_pmu);
+	perf_pmu_unregister(&drw_pmu->pmu);
+
+	return 0;
+}
+
+static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
+{
+	struct ali_drw_pmu_irq *irq;
+	struct ali_drw_pmu *drw_pmu;
+	unsigned int target;
+	int ret;
+	cpumask_t node_online_cpus;
+
+	irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node);
+	if (cpu != irq->cpu)
+		return 0;
+
+	ret = cpumask_and(&node_online_cpus,
+			  cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask);
+	if (ret)
+		target = cpumask_any_but(&node_online_cpus, cpu);
+	else
+		target = cpumask_any_but(cpu_online_mask, cpu);
+
+	if (target >= nr_cpu_ids)
+		return 0;
+
+	/* We're only reading, but this isn't the place to be involving RCU */
+	mutex_lock(&ali_drw_pmu_irqs_lock);
+	list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node)
+		perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target);
+	mutex_unlock(&ali_drw_pmu_irqs_lock);
+
+	WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target)));
+	irq->cpu = target;
+
+	return 0;
+}
+
+/*
+ * Due to historical reasons, the HID used in the production environment is
+ * ARMHD700, so we leave ARMHD700 as Compatible ID.
+ */
+static const struct acpi_device_id ali_drw_acpi_match[] = {
+	{"BABA5000", 0},
+	{"ARMHD700", 0},
+	{}
+};
+
+MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match);
+
+static struct platform_driver ali_drw_pmu_driver = {
+	.driver = {
+		   .name = "ali_drw_pmu",
+		   .acpi_match_table = ali_drw_acpi_match,
+		   },
+	.probe = ali_drw_pmu_probe,
+	.remove = ali_drw_pmu_remove,
+};
+
+static int __init ali_drw_pmu_init(void)
+{
+	int ret;
+
+	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
+				      "ali_drw_pmu:online",
+				      NULL, ali_drw_pmu_offline_cpu);
+
+	if (ret < 0) {
+		pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n",
+		       ret);
+		return ret;
+	}
+	ali_drw_cpuhp_state_num = ret;
+
+	ret = platform_driver_register(&ali_drw_pmu_driver);
+	if (ret)
+		cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
+
+	return ret;
+}
+
+static void __exit ali_drw_pmu_exit(void)
+{
+	platform_driver_unregister(&ali_drw_pmu_driver);
+	cpuhp_remove_multi_state(ali_drw_cpuhp_state_num);
+}
+
+module_init(ali_drw_pmu_init);
+module_exit(ali_drw_pmu_exit);
+
+MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>");
+MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>");
+MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>");
+MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/perf/arm_dsu_pmu.c b/drivers/perf/arm_dsu_pmu.c
index a36698a90d2f..4a15c86f45ef 100644
--- a/drivers/perf/arm_dsu_pmu.c
+++ b/drivers/perf/arm_dsu_pmu.c
@@ -639,6 +639,7 @@ static int dsu_pmu_dt_get_cpus(struct device *dev, cpumask_t *mask)
 static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask)
 {
 #ifdef CONFIG_ACPI
+	struct acpi_device *parent_adev = acpi_dev_parent(ACPI_COMPANION(dev));
 	int cpu;
 
 	/*
@@ -653,8 +654,7 @@ static int dsu_pmu_acpi_get_cpus(struct device *dev, cpumask_t *mask)
 			continue;
 
 		acpi_dev = ACPI_COMPANION(cpu_dev);
-		if (acpi_dev &&
-			acpi_dev->parent == ACPI_COMPANION(dev)->parent)
+		if (acpi_dev && acpi_dev_parent(acpi_dev) == parent_adev)
 			cpumask_set_cpu(cpu, mask);
 	}
 #endif
diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
index 59d3980b8ca2..3f07df5a7e95 100644
--- a/drivers/perf/arm_pmu.c
+++ b/drivers/perf/arm_pmu.c
@@ -894,7 +894,7 @@ static struct arm_pmu *__armpmu_alloc(gfp_t flags)
 		 * pmu::filter_match callback and pmu::event_init group
 		 * validation).
 		 */
-		.capabilities	= PERF_PMU_CAP_HETEROGENEOUS_CPUS,
+		.capabilities	= PERF_PMU_CAP_HETEROGENEOUS_CPUS | PERF_PMU_CAP_EXTENDED_REGS,
 	};
 
 	pmu->attr_groups[ARMPMU_ATTR_GROUP_COMMON] =
diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c
index b65a7d9640e1..6ce05ef4844d 100644
--- a/drivers/perf/arm_spe_pmu.c
+++ b/drivers/perf/arm_spe_pmu.c
@@ -674,9 +674,9 @@ static irqreturn_t arm_spe_pmu_irq_handler(int irq, void *dev)
 static u64 arm_spe_pmsevfr_res0(u16 pmsver)
 {
 	switch (pmsver) {
-	case ID_AA64DFR0_PMSVER_8_2:
+	case ID_AA64DFR0_EL1_PMSVer_IMP:
 		return SYS_PMSEVFR_EL1_RES0_8_2;
-	case ID_AA64DFR0_PMSVER_8_3:
+	case ID_AA64DFR0_EL1_PMSVer_V1P1:
 	/* Return the highest version we support in default */
 	default:
 		return SYS_PMSEVFR_EL1_RES0_8_3;
@@ -958,7 +958,7 @@ static void __arm_spe_pmu_dev_probe(void *info)
 	struct device *dev = &spe_pmu->pdev->dev;
 
 	fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64DFR0_EL1),
-						   ID_AA64DFR0_PMSVER_SHIFT);
+						   ID_AA64DFR0_EL1_PMSVer_SHIFT);
 	if (!fld) {
 		dev_err(dev,
 			"unsupported ID_AA64DFR0_EL1.PMSVer [%d] on CPU %d\n",
diff --git a/drivers/perf/qcom_l2_pmu.c b/drivers/perf/qcom_l2_pmu.c
index 30234c261b05..aaca6db7d8f6 100644
--- a/drivers/perf/qcom_l2_pmu.c
+++ b/drivers/perf/qcom_l2_pmu.c
@@ -840,16 +840,16 @@ static int l2_cache_pmu_probe_cluster(struct device *dev, void *data)
 {
 	struct platform_device *pdev = to_platform_device(dev->parent);
 	struct platform_device *sdev = to_platform_device(dev);
-	struct acpi_device *adev = ACPI_COMPANION(dev);
 	struct l2cache_pmu *l2cache_pmu = data;
 	struct cluster_pmu *cluster;
-	unsigned long fw_cluster_id;
+	u64 fw_cluster_id;
 	int err;
 	int irq;
 
-	if (!adev || kstrtoul(adev->pnp.unique_id, 10, &fw_cluster_id) < 0) {
+	err = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &fw_cluster_id);
+	if (err) {
 		dev_err(&pdev->dev, "unable to read ACPI uid\n");
-		return -ENODEV;
+		return err;
 	}
 
 	cluster = devm_kzalloc(&pdev->dev, sizeof(*cluster), GFP_KERNEL);
@@ -879,7 +879,7 @@ static int l2_cache_pmu_probe_cluster(struct device *dev, void *data)
 	}
 
 	dev_info(&pdev->dev,
-		"Registered L2 cache PMU cluster %ld\n", fw_cluster_id);
+		 "Registered L2 cache PMU cluster %lld\n", fw_cluster_id);
 
 	spin_lock_init(&cluster->pmu_lock);
 
diff --git a/drivers/perf/qcom_l3_pmu.c b/drivers/perf/qcom_l3_pmu.c
index 1ff2ff6582bf..346311a05460 100644
--- a/drivers/perf/qcom_l3_pmu.c
+++ b/drivers/perf/qcom_l3_pmu.c
@@ -742,7 +742,8 @@ static int qcom_l3_cache_pmu_probe(struct platform_device *pdev)
 
 	l3pmu = devm_kzalloc(&pdev->dev, sizeof(*l3pmu), GFP_KERNEL);
 	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "l3cache_%s_%s",
-		      acpi_dev->parent->pnp.unique_id, acpi_dev->pnp.unique_id);
+		      acpi_dev_parent(acpi_dev)->pnp.unique_id,
+		      acpi_dev->pnp.unique_id);
 	if (!l3pmu || !name)
 		return -ENOMEM;
 
diff --git a/drivers/phy/broadcom/Kconfig b/drivers/phy/broadcom/Kconfig
index 93a6a8ee4716..1d89a2fd9b79 100644
--- a/drivers/phy/broadcom/Kconfig
+++ b/drivers/phy/broadcom/Kconfig
@@ -93,11 +93,11 @@ config PHY_BRCM_SATA
 
 config PHY_BRCM_USB
 	tristate "Broadcom STB USB PHY driver"
-	depends on ARCH_BCM4908 || ARCH_BRCMSTB || COMPILE_TEST
+	depends on ARCH_BCMBCA || ARCH_BRCMSTB || COMPILE_TEST
 	depends on OF
 	select GENERIC_PHY
 	select SOC_BRCMSTB if ARCH_BRCMSTB
-	default ARCH_BCM4908 || ARCH_BRCMSTB
+	default ARCH_BCMBCA || ARCH_BRCMSTB
 	help
 	  Enable this to support the Broadcom STB USB PHY.
 	  This driver is required by the USB XHCI, EHCI and OHCI
diff --git a/drivers/phy/microchip/lan966x_serdes.c b/drivers/phy/microchip/lan966x_serdes.c
index e86a879b92b5..d1a50fa81130 100644
--- a/drivers/phy/microchip/lan966x_serdes.c
+++ b/drivers/phy/microchip/lan966x_serdes.c
@@ -401,6 +401,9 @@ static int serdes_set_mode(struct phy *phy, enum phy_mode mode, int submode)
 	    submode == PHY_INTERFACE_MODE_2500BASEX)
 		submode = PHY_INTERFACE_MODE_SGMII;
 
+	if (submode == PHY_INTERFACE_MODE_QUSGMII)
+		submode = PHY_INTERFACE_MODE_QSGMII;
+
 	for (i = 0; i < ARRAY_SIZE(lan966x_serdes_muxes); i++) {
 		if (macro->idx != lan966x_serdes_muxes[i].idx ||
 		    mode != lan966x_serdes_muxes[i].mode ||
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index 1cf74b0c42e5..d768dcf75cf1 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -292,7 +292,7 @@ config PINCTRL_MCP23S08
 	  corresponding interrupt-controller.
 
 config PINCTRL_MICROCHIP_SGPIO
-	bool "Pinctrl driver for Microsemi/Microchip Serial GPIO"
+	tristate "Pinctrl driver for Microsemi/Microchip Serial GPIO"
 	depends on OF
 	depends on HAS_IOMEM
 	select GPIOLIB
@@ -310,6 +310,9 @@ config PINCTRL_MICROCHIP_SGPIO
 	  connect control signals from SFP modules and to act as an
 	  LED controller.
 
+	  If compiled as a module, the module name will be
+	  pinctrl-microchip-sgpio.
+
 config PINCTRL_OCELOT
 	tristate "Pinctrl driver for the Microsemi Ocelot and Jaguar2 SoCs"
 	depends on OF
diff --git a/drivers/pinctrl/bcm/Kconfig b/drivers/pinctrl/bcm/Kconfig
index 8f4d89806fcb..35b51ce4298e 100644
--- a/drivers/pinctrl/bcm/Kconfig
+++ b/drivers/pinctrl/bcm/Kconfig
@@ -31,13 +31,13 @@ config PINCTRL_BCM2835
 
 config PINCTRL_BCM4908
 	tristate "Broadcom BCM4908 pinmux driver"
-	depends on OF && (ARCH_BCM4908 || COMPILE_TEST)
+	depends on OF && (ARCH_BCMBCA || COMPILE_TEST)
 	select PINMUX
 	select PINCONF
 	select GENERIC_PINCONF
 	select GENERIC_PINCTRL_GROUPS
 	select GENERIC_PINMUX_FUNCTIONS
-	default ARCH_BCM4908
+	default ARCH_BCMBCA
 	help
 	  Driver for BCM4908 family SoCs with integrated pin controller.
 
diff --git a/drivers/pinctrl/pinctrl-microchip-sgpio.c b/drivers/pinctrl/pinctrl-microchip-sgpio.c
index 6f55bf7d5e05..2b4167a09b3b 100644
--- a/drivers/pinctrl/pinctrl-microchip-sgpio.c
+++ b/drivers/pinctrl/pinctrl-microchip-sgpio.c
@@ -12,6 +12,7 @@
 #include <linux/clk.h>
 #include <linux/gpio/driver.h>
 #include <linux/io.h>
+#include <linux/mfd/ocelot.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/pinctrl/pinmux.h>
@@ -904,7 +905,6 @@ static int microchip_sgpio_probe(struct platform_device *pdev)
 	struct reset_control *reset;
 	struct sgpio_priv *priv;
 	struct clk *clk;
-	u32 __iomem *regs;
 	u32 val;
 	struct regmap_config regmap_config = {
 		.reg_bits = 32,
@@ -937,11 +937,7 @@ static int microchip_sgpio_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(regs))
-		return PTR_ERR(regs);
-
-	priv->regs = devm_regmap_init_mmio(dev, regs, &regmap_config);
+	priv->regs = ocelot_regmap_from_resource(pdev, 0, &regmap_config);
 	if (IS_ERR(priv->regs))
 		return PTR_ERR(priv->regs);
 
@@ -999,6 +995,7 @@ static const struct of_device_id microchip_sgpio_gpio_of_match[] = {
 		/* sentinel */
 	}
 };
+MODULE_DEVICE_TABLE(of, microchip_sgpio_gpio_of_match);
 
 static struct platform_driver microchip_sgpio_pinctrl_driver = {
 	.driver = {
@@ -1008,4 +1005,7 @@ static struct platform_driver microchip_sgpio_pinctrl_driver = {
 	},
 	.probe = microchip_sgpio_probe,
 };
-builtin_platform_driver(microchip_sgpio_pinctrl_driver);
+module_platform_driver(microchip_sgpio_pinctrl_driver);
+
+MODULE_DESCRIPTION("Microchip SGPIO Pinctrl Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/pinctrl-ocelot.c b/drivers/pinctrl/pinctrl-ocelot.c
index c7df8c5fe585..83464e0bf4e6 100644
--- a/drivers/pinctrl/pinctrl-ocelot.c
+++ b/drivers/pinctrl/pinctrl-ocelot.c
@@ -10,6 +10,7 @@
 #include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/mfd/ocelot.h>
 #include <linux/of_device.h>
 #include <linux/of_irq.h>
 #include <linux/of_platform.h>
@@ -2044,7 +2045,6 @@ static int ocelot_pinctrl_probe(struct platform_device *pdev)
 	struct ocelot_pinctrl *info;
 	struct reset_control *reset;
 	struct regmap *pincfg;
-	void __iomem *base;
 	int ret;
 	struct regmap_config regmap_config = {
 		.reg_bits = 32,
@@ -2077,20 +2077,14 @@ static int ocelot_pinctrl_probe(struct platform_device *pdev)
 				     "Failed to get reset\n");
 	reset_control_reset(reset);
 
-	base = devm_ioremap_resource(dev,
-			platform_get_resource(pdev, IORESOURCE_MEM, 0));
-	if (IS_ERR(base))
-		return PTR_ERR(base);
-
 	info->stride = 1 + (info->desc->npins - 1) / 32;
 
 	regmap_config.max_register = OCELOT_GPIO_SD_MAP * info->stride + 15 * 4;
 
-	info->map = devm_regmap_init_mmio(dev, base, &regmap_config);
-	if (IS_ERR(info->map)) {
-		dev_err(dev, "Failed to create regmap\n");
-		return PTR_ERR(info->map);
-	}
+	info->map = ocelot_regmap_from_resource(pdev, 0, &regmap_config);
+	if (IS_ERR(info->map))
+		return dev_err_probe(dev, PTR_ERR(info->map),
+				     "Failed to create regmap\n");
 	dev_set_drvdata(dev, info);
 	info->dev = dev;
 
diff --git a/drivers/platform/chrome/Kconfig b/drivers/platform/chrome/Kconfig
index c45fb376d653..6b954c5acadb 100644
--- a/drivers/platform/chrome/Kconfig
+++ b/drivers/platform/chrome/Kconfig
@@ -265,6 +265,17 @@ config CHROMEOS_PRIVACY_SCREEN
 	  this should probably always be built into the kernel to avoid or
 	  minimize drm probe deferral.
 
+config CROS_TYPEC_SWITCH
+	tristate "ChromeOS EC Type-C Switch Control"
+	depends on MFD_CROS_EC_DEV && TYPEC && ACPI
+	default MFD_CROS_EC_DEV
+	help
+	  If you say Y here, you get support for configuring the ChromeOS EC Type-C
+	  muxes and retimers.
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called cros_typec_switch.
+
 source "drivers/platform/chrome/wilco_ec/Kconfig"
 
 # Kunit test cases
diff --git a/drivers/platform/chrome/Makefile b/drivers/platform/chrome/Makefile
index f7e74a845afc..2950610101f1 100644
--- a/drivers/platform/chrome/Makefile
+++ b/drivers/platform/chrome/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_CHROMEOS_TBMC)		+= chromeos_tbmc.o
 obj-$(CONFIG_CROS_EC)			+= cros_ec.o
 obj-$(CONFIG_CROS_EC_I2C)		+= cros_ec_i2c.o
 obj-$(CONFIG_CROS_EC_ISHTP)		+= cros_ec_ishtp.o
+obj-$(CONFIG_CROS_TYPEC_SWITCH)		+= cros_typec_switch.o
 obj-$(CONFIG_CROS_EC_RPMSG)		+= cros_ec_rpmsg.o
 obj-$(CONFIG_CROS_EC_SPI)		+= cros_ec_spi.o
 cros_ec_lpcs-objs			:= cros_ec_lpc.o cros_ec_lpc_mec.o
diff --git a/drivers/platform/chrome/chromeos_laptop.c b/drivers/platform/chrome/chromeos_laptop.c
index 4e14b4d6635d..a2cdbfbaeae6 100644
--- a/drivers/platform/chrome/chromeos_laptop.c
+++ b/drivers/platform/chrome/chromeos_laptop.c
@@ -740,6 +740,7 @@ static int __init
 chromeos_laptop_prepare_i2c_peripherals(struct chromeos_laptop *cros_laptop,
 					const struct chromeos_laptop *src)
 {
+	struct i2c_peripheral *i2c_peripherals;
 	struct i2c_peripheral *i2c_dev;
 	struct i2c_board_info *info;
 	int i;
@@ -748,17 +749,15 @@ chromeos_laptop_prepare_i2c_peripherals(struct chromeos_laptop *cros_laptop,
 	if (!src->num_i2c_peripherals)
 		return 0;
 
-	cros_laptop->i2c_peripherals = kmemdup(src->i2c_peripherals,
-					       src->num_i2c_peripherals *
-						sizeof(*src->i2c_peripherals),
-					       GFP_KERNEL);
-	if (!cros_laptop->i2c_peripherals)
+	i2c_peripherals = kmemdup(src->i2c_peripherals,
+					      src->num_i2c_peripherals *
+					  sizeof(*src->i2c_peripherals),
+					  GFP_KERNEL);
+	if (!i2c_peripherals)
 		return -ENOMEM;
 
-	cros_laptop->num_i2c_peripherals = src->num_i2c_peripherals;
-
-	for (i = 0; i < cros_laptop->num_i2c_peripherals; i++) {
-		i2c_dev = &cros_laptop->i2c_peripherals[i];
+	for (i = 0; i < src->num_i2c_peripherals; i++) {
+		i2c_dev = &i2c_peripherals[i];
 		info = &i2c_dev->board_info;
 
 		error = chromeos_laptop_setup_irq(i2c_dev);
@@ -775,16 +774,19 @@ chromeos_laptop_prepare_i2c_peripherals(struct chromeos_laptop *cros_laptop,
 		}
 	}
 
+	cros_laptop->i2c_peripherals = i2c_peripherals;
+	cros_laptop->num_i2c_peripherals = src->num_i2c_peripherals;
+
 	return 0;
 
 err_out:
 	while (--i >= 0) {
-		i2c_dev = &cros_laptop->i2c_peripherals[i];
+		i2c_dev = &i2c_peripherals[i];
 		info = &i2c_dev->board_info;
 		if (!IS_ERR_OR_NULL(info->fwnode))
 			fwnode_remove_software_node(info->fwnode);
 	}
-	kfree(cros_laptop->i2c_peripherals);
+	kfree(i2c_peripherals);
 	return error;
 }
 
diff --git a/drivers/platform/chrome/cros_ec.c b/drivers/platform/chrome/cros_ec.c
index 8aace50d446d..ec733f683f34 100644
--- a/drivers/platform/chrome/cros_ec.c
+++ b/drivers/platform/chrome/cros_ec.c
@@ -115,7 +115,7 @@ static int cros_ec_sleep_event(struct cros_ec_device *ec_dev, u8 sleep_event)
 	if (ec_dev->host_sleep_v1) {
 		buf.u.req1.sleep_event = sleep_event;
 		buf.u.req1.suspend_params.sleep_timeout_ms =
-				EC_HOST_SLEEP_TIMEOUT_DEFAULT;
+				ec_dev->suspend_timeout_ms;
 
 		buf.msg.outsize = sizeof(buf.u.req1);
 		if ((sleep_event == HOST_SLEEP_EVENT_S3_RESUME) ||
@@ -188,6 +188,7 @@ int cros_ec_register(struct cros_ec_device *ec_dev)
 	ec_dev->max_passthru = 0;
 	ec_dev->ec = NULL;
 	ec_dev->pd = NULL;
+	ec_dev->suspend_timeout_ms = EC_HOST_SLEEP_TIMEOUT_DEFAULT;
 
 	ec_dev->din = devm_kzalloc(dev, ec_dev->din_size, GFP_KERNEL);
 	if (!ec_dev->din)
@@ -349,10 +350,16 @@ EXPORT_SYMBOL(cros_ec_suspend);
 
 static void cros_ec_report_events_during_suspend(struct cros_ec_device *ec_dev)
 {
+	bool wake_event;
+
 	while (ec_dev->mkbp_event_supported &&
-	       cros_ec_get_next_event(ec_dev, NULL, NULL) > 0)
+	       cros_ec_get_next_event(ec_dev, &wake_event, NULL) > 0) {
 		blocking_notifier_call_chain(&ec_dev->event_notifier,
 					     1, ec_dev);
+
+		if (wake_event && device_may_wakeup(ec_dev->dev))
+			pm_wakeup_event(ec_dev->dev, 0);
+	}
 }
 
 /**
diff --git a/drivers/platform/chrome/cros_ec_chardev.c b/drivers/platform/chrome/cros_ec_chardev.c
index fd33de546aee..0de7c255254e 100644
--- a/drivers/platform/chrome/cros_ec_chardev.c
+++ b/drivers/platform/chrome/cros_ec_chardev.c
@@ -327,6 +327,9 @@ static long cros_ec_chardev_ioctl_readmem(struct cros_ec_dev *ec,
 	if (copy_from_user(&s_mem, arg, sizeof(s_mem)))
 		return -EFAULT;
 
+	if (s_mem.bytes > sizeof(s_mem.buffer))
+		return -EINVAL;
+
 	num = ec_dev->cmd_readmem(ec_dev, s_mem.offset, s_mem.bytes,
 				  s_mem.buffer);
 	if (num <= 0)
diff --git a/drivers/platform/chrome/cros_ec_debugfs.c b/drivers/platform/chrome/cros_ec_debugfs.c
index 0dbceee87a4b..4e63adf083ea 100644
--- a/drivers/platform/chrome/cros_ec_debugfs.c
+++ b/drivers/platform/chrome/cros_ec_debugfs.c
@@ -470,6 +470,9 @@ static int cros_ec_debugfs_probe(struct platform_device *pd)
 	debugfs_create_x32("last_resume_result", 0444, debug_info->dir,
 			   &ec->ec_dev->last_resume_result);
 
+	debugfs_create_u16("suspend_timeout_ms", 0664, debug_info->dir,
+			   &ec->ec_dev->suspend_timeout_ms);
+
 	ec->debug_info = debug_info;
 
 	dev_set_drvdata(&pd->dev, ec);
diff --git a/drivers/platform/chrome/cros_ec_i2c.c b/drivers/platform/chrome/cros_ec_i2c.c
index 9f5b95763173..b6823c654c3f 100644
--- a/drivers/platform/chrome/cros_ec_i2c.c
+++ b/drivers/platform/chrome/cros_ec_i2c.c
@@ -317,13 +317,11 @@ static int cros_ec_i2c_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int cros_ec_i2c_remove(struct i2c_client *client)
+static void cros_ec_i2c_remove(struct i2c_client *client)
 {
 	struct cros_ec_device *ec_dev = i2c_get_clientdata(client);
 
 	cros_ec_unregister(ec_dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/platform/chrome/cros_ec_proto.c b/drivers/platform/chrome/cros_ec_proto.c
index 05d2e8765a66..475a6dd72db6 100644
--- a/drivers/platform/chrome/cros_ec_proto.c
+++ b/drivers/platform/chrome/cros_ec_proto.c
@@ -773,6 +773,7 @@ int cros_ec_get_next_event(struct cros_ec_device *ec_dev,
 	u8 event_type;
 	u32 host_event;
 	int ret;
+	u32 ver_mask;
 
 	/*
 	 * Default value for wake_event.
@@ -794,6 +795,37 @@ int cros_ec_get_next_event(struct cros_ec_device *ec_dev,
 		return get_keyboard_state_event(ec_dev);
 
 	ret = get_next_event(ec_dev);
+	/*
+	 * -ENOPROTOOPT is returned when EC returns EC_RES_INVALID_VERSION.
+	 * This can occur when EC based device (e.g. Fingerprint MCU) jumps to
+	 * the RO image which doesn't support newer version of the command. In
+	 * this case we will attempt to update maximum supported version of the
+	 * EC_CMD_GET_NEXT_EVENT.
+	 */
+	if (ret == -ENOPROTOOPT) {
+		dev_dbg(ec_dev->dev,
+			"GET_NEXT_EVENT returned invalid version error.\n");
+		ret = cros_ec_get_host_command_version_mask(ec_dev,
+							EC_CMD_GET_NEXT_EVENT,
+							&ver_mask);
+		if (ret < 0 || ver_mask == 0)
+			/*
+			 * Do not change the MKBP supported version if we can't
+			 * obtain supported version correctly. Please note that
+			 * calling EC_CMD_GET_NEXT_EVENT returned
+			 * EC_RES_INVALID_VERSION which means that the command
+			 * is present.
+			 */
+			return -ENOPROTOOPT;
+
+		ec_dev->mkbp_event_supported = fls(ver_mask);
+		dev_dbg(ec_dev->dev, "MKBP support version changed to %u\n",
+			ec_dev->mkbp_event_supported - 1);
+
+		/* Try to get next event with new MKBP support version set. */
+		ret = get_next_event(ec_dev);
+	}
+
 	if (ret <= 0)
 		return ret;
 
diff --git a/drivers/platform/chrome/cros_ec_typec.c b/drivers/platform/chrome/cros_ec_typec.c
index de6ee0f926a6..2a7ff14dc37e 100644
--- a/drivers/platform/chrome/cros_ec_typec.c
+++ b/drivers/platform/chrome/cros_ec_typec.c
@@ -20,12 +20,14 @@
 #include <linux/usb/typec_altmode.h>
 #include <linux/usb/typec_dp.h>
 #include <linux/usb/typec_mux.h>
+#include <linux/usb/typec_retimer.h>
 #include <linux/usb/typec_tbt.h>
 #include <linux/usb/role.h>
 
 #define DRV_NAME "cros-ec-typec"
 
-#define DP_PORT_VDO	(BIT(DP_PIN_ASSIGN_C) | BIT(DP_PIN_ASSIGN_D) | DP_CAP_DFP_D)
+#define DP_PORT_VDO	(DP_CONF_SET_PIN_ASSIGN(BIT(DP_PIN_ASSIGN_C) | BIT(DP_PIN_ASSIGN_D)) | \
+				DP_CAP_DFP_D)
 
 /* Supported alt modes. */
 enum {
@@ -55,6 +57,7 @@ struct cros_typec_port {
 	struct usb_pd_identity c_identity;
 	struct typec_switch *ori_sw;
 	struct typec_mux *mux;
+	struct typec_retimer *retimer;
 	struct usb_role_switch *role_sw;
 
 	/* Variables keeping track of switch state. */
@@ -70,6 +73,11 @@ struct cros_typec_port {
 	struct ec_response_typec_discovery *disc_data;
 	struct list_head partner_mode_list;
 	struct list_head plug_mode_list;
+
+	/* PDO-related structs */
+	struct usb_power_delivery *partner_pd;
+	struct usb_power_delivery_capabilities *partner_src_caps;
+	struct usb_power_delivery_capabilities *partner_sink_caps;
 };
 
 /* Platform-specific data for the Chrome OS EC Type C controller. */
@@ -143,6 +151,12 @@ static int cros_typec_get_switch_handles(struct cros_typec_port *port,
 		goto mux_err;
 	}
 
+	port->retimer = fwnode_typec_retimer_get(fwnode);
+	if (IS_ERR(port->retimer)) {
+		dev_dbg(dev, "Retimer handle not found.\n");
+		goto retimer_sw_err;
+	}
+
 	port->ori_sw = fwnode_typec_switch_get(fwnode);
 	if (IS_ERR(port->ori_sw)) {
 		dev_dbg(dev, "Orientation switch handle not found.\n");
@@ -158,12 +172,12 @@ static int cros_typec_get_switch_handles(struct cros_typec_port *port,
 	return 0;
 
 role_sw_err:
-	usb_role_switch_put(port->role_sw);
-ori_sw_err:
 	typec_switch_put(port->ori_sw);
-mux_err:
+ori_sw_err:
+	typec_retimer_put(port->retimer);
+retimer_sw_err:
 	typec_mux_put(port->mux);
-
+mux_err:
 	return -ENODEV;
 }
 
@@ -206,6 +220,21 @@ static void cros_typec_unregister_altmodes(struct cros_typec_data *typec, int po
 	}
 }
 
+/*
+ * Map the Type-C Mux state to retimer state and call the retimer set function. We need this
+ * because we re-use the Type-C mux state for retimers.
+ */
+static int cros_typec_retimer_set(struct typec_retimer *retimer, struct typec_mux_state state)
+{
+	struct typec_retimer_state rstate = {
+		.alt = state.alt,
+		.mode = state.mode,
+		.data = state.data,
+	};
+
+	return typec_retimer_set(retimer, &rstate);
+}
+
 static int cros_typec_usb_disconnect_state(struct cros_typec_port *port)
 {
 	port->state.alt = NULL;
@@ -214,6 +243,7 @@ static int cros_typec_usb_disconnect_state(struct cros_typec_port *port)
 
 	usb_role_switch_set_role(port->role_sw, USB_ROLE_NONE);
 	typec_switch_set(port->ori_sw, TYPEC_ORIENTATION_NONE);
+	cros_typec_retimer_set(port->retimer, port->state);
 
 	return typec_mux_set(port->mux, &port->state);
 }
@@ -228,6 +258,14 @@ static void cros_typec_remove_partner(struct cros_typec_data *typec,
 
 	cros_typec_unregister_altmodes(typec, port_num, true);
 
+	typec_partner_set_usb_power_delivery(port->partner, NULL);
+	usb_power_delivery_unregister_capabilities(port->partner_sink_caps);
+	port->partner_sink_caps = NULL;
+	usb_power_delivery_unregister_capabilities(port->partner_src_caps);
+	port->partner_src_caps = NULL;
+	usb_power_delivery_unregister(port->partner_pd);
+	port->partner_pd = NULL;
+
 	cros_typec_usb_disconnect_state(port);
 	port->mux_flags = USB_PD_MUX_NONE;
 
@@ -411,9 +449,14 @@ unregister_ports:
 
 static int cros_typec_usb_safe_state(struct cros_typec_port *port)
 {
+	int ret;
 	port->state.mode = TYPEC_STATE_SAFE;
 
-	return typec_mux_set(port->mux, &port->state);
+	ret = cros_typec_retimer_set(port->retimer, port->state);
+	if (!ret)
+		ret = typec_mux_set(port->mux, &port->state);
+
+	return ret;
 }
 
 /*
@@ -510,7 +553,11 @@ static int cros_typec_enable_dp(struct cros_typec_data *typec,
 	port->state.data = &dp_data;
 	port->state.mode = TYPEC_MODAL_STATE(ffs(pd_ctrl->dp_mode));
 
-	return typec_mux_set(port->mux, &port->state);
+	ret = cros_typec_retimer_set(port->retimer, port->state);
+	if (!ret)
+		ret = typec_mux_set(port->mux, &port->state);
+
+	return ret;
 }
 
 static int cros_typec_enable_usb4(struct cros_typec_data *typec,
@@ -599,7 +646,10 @@ static int cros_typec_configure_mux(struct cros_typec_data *typec, int port_num,
 	} else if (port->mux_flags & USB_PD_MUX_USB_ENABLED) {
 		port->state.alt = NULL;
 		port->state.mode = TYPEC_STATE_USB;
-		ret = typec_mux_set(port->mux, &port->state);
+
+		ret = cros_typec_retimer_set(port->retimer, port->state);
+		if (!ret)
+			ret = typec_mux_set(port->mux, &port->state);
 	} else {
 		dev_dbg(typec->dev,
 			"Unrecognized mode requested, mux flags: %x\n",
@@ -697,7 +747,7 @@ static int cros_typec_register_altmodes(struct cros_typec_data *typec, int port_
 		for (j = 0; j < sop_disc->svids[i].mode_count; j++) {
 			memset(&desc, 0, sizeof(desc));
 			desc.svid = sop_disc->svids[i].svid;
-			desc.mode = j;
+			desc.mode = j + 1;
 			desc.vdo = sop_disc->svids[i].mode_vdo[j];
 
 			if (is_partner)
@@ -902,6 +952,46 @@ static int cros_typec_send_clear_event(struct cros_typec_data *typec, int port_n
 			   sizeof(req), NULL, 0);
 }
 
+static void cros_typec_register_partner_pdos(struct cros_typec_data *typec,
+					     struct ec_response_typec_status *resp, int port_num)
+{
+	struct usb_power_delivery_capabilities_desc caps_desc = {};
+	struct usb_power_delivery_desc desc = {
+		.revision = (le16_to_cpu(resp->sop_revision) & 0xff00) >> 4,
+	};
+	struct cros_typec_port *port = typec->ports[port_num];
+
+	if (!port->partner || port->partner_pd)
+		return;
+
+	/* If no caps are available, don't bother creating a device. */
+	if (!resp->source_cap_count && !resp->sink_cap_count)
+		return;
+
+	port->partner_pd = usb_power_delivery_register(NULL, &desc);
+	if (IS_ERR(port->partner_pd)) {
+		dev_warn(typec->dev, "Failed to register partner PD device, port: %d\n", port_num);
+		return;
+	}
+
+	typec_partner_set_usb_power_delivery(port->partner, port->partner_pd);
+
+	memcpy(caps_desc.pdo, resp->source_cap_pdos, sizeof(u32) * resp->source_cap_count);
+	caps_desc.role = TYPEC_SOURCE;
+	port->partner_src_caps = usb_power_delivery_register_capabilities(port->partner_pd,
+									  &caps_desc);
+	if (IS_ERR(port->partner_src_caps))
+		dev_warn(typec->dev, "Failed to register source caps, port: %d\n", port_num);
+
+	memset(&caps_desc, 0, sizeof(caps_desc));
+	memcpy(caps_desc.pdo, resp->sink_cap_pdos, sizeof(u32) * resp->sink_cap_count);
+	caps_desc.role = TYPEC_SINK;
+	port->partner_sink_caps = usb_power_delivery_register_capabilities(port->partner_pd,
+									   &caps_desc);
+	if (IS_ERR(port->partner_sink_caps))
+		dev_warn(typec->dev, "Failed to register sink caps, port: %d\n", port_num);
+}
+
 static void cros_typec_handle_status(struct cros_typec_data *typec, int port_num)
 {
 	struct ec_response_typec_status resp;
@@ -949,6 +1039,8 @@ static void cros_typec_handle_status(struct cros_typec_data *typec, int port_num
 		}
 		if (resp.sop_connected)
 			typec_set_pwr_opmode(typec->ports[port_num]->port, TYPEC_PWR_MODE_PD);
+
+		cros_typec_register_partner_pdos(typec, &resp, port_num);
 	}
 
 	if (resp.events & PD_STATUS_EVENT_SOP_PRIME_DISC_DONE &&
diff --git a/drivers/platform/chrome/cros_typec_switch.c b/drivers/platform/chrome/cros_typec_switch.c
new file mode 100644
index 000000000000..a26219e97c93
--- /dev/null
+++ b/drivers/platform/chrome/cros_typec_switch.c
@@ -0,0 +1,321 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2022 Google LLC
+ *
+ * This driver provides the ability to configure Type-C muxes and retimers which are controlled by
+ * the ChromeOS EC.
+ */
+
+#include <linux/acpi.h>
+#include <linux/delay.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/platform_data/cros_ec_commands.h>
+#include <linux/platform_data/cros_ec_proto.h>
+#include <linux/platform_device.h>
+#include <linux/usb/typec_altmode.h>
+#include <linux/usb/typec_dp.h>
+#include <linux/usb/typec_mux.h>
+#include <linux/usb/typec_retimer.h>
+
+/* Handles and other relevant data required for each port's switches. */
+struct cros_typec_port {
+	int port_num;
+	struct typec_mux_dev *mode_switch;
+	struct typec_retimer *retimer;
+	struct cros_typec_switch_data *sdata;
+};
+
+/* Driver-specific data. */
+struct cros_typec_switch_data {
+	struct device *dev;
+	struct cros_ec_device *ec;
+	struct cros_typec_port *ports[EC_USB_PD_MAX_PORTS];
+};
+
+static int cros_typec_cmd_mux_set(struct cros_typec_switch_data *sdata, int port_num, u8 index,
+				  u8 state)
+{
+	struct ec_params_typec_control req = {
+		.port = port_num,
+		.command = TYPEC_CONTROL_COMMAND_USB_MUX_SET,
+		.mux_params = {
+			.mux_index = index,
+			.mux_flags = state,
+		},
+	};
+
+	return cros_ec_cmd(sdata->ec, 0, EC_CMD_TYPEC_CONTROL, &req, sizeof(req), NULL, 0);
+}
+
+static int cros_typec_get_mux_state(unsigned long mode, struct typec_altmode *alt)
+{
+	int ret = -EOPNOTSUPP;
+
+	if (mode == TYPEC_STATE_SAFE)
+		ret = USB_PD_MUX_SAFE_MODE;
+	else if (mode == TYPEC_STATE_USB)
+		ret = USB_PD_MUX_USB_ENABLED;
+	else if (alt && alt->svid == USB_TYPEC_DP_SID)
+		ret = USB_PD_MUX_DP_ENABLED;
+
+	return ret;
+}
+
+static int cros_typec_send_clear_event(struct cros_typec_switch_data *sdata, int port_num,
+				       u32 events_mask)
+{
+	struct ec_params_typec_control req = {
+		.port = port_num,
+		.command = TYPEC_CONTROL_COMMAND_CLEAR_EVENTS,
+		.clear_events_mask = events_mask,
+	};
+
+	return cros_ec_cmd(sdata->ec, 0, EC_CMD_TYPEC_CONTROL, &req, sizeof(req), NULL, 0);
+}
+
+static bool cros_typec_check_event(struct cros_typec_switch_data *sdata, int port_num, u32 mask)
+{
+	struct ec_response_typec_status resp;
+	struct ec_params_typec_status req = {
+		.port = port_num,
+	};
+	int ret;
+
+	ret = cros_ec_cmd(sdata->ec, 0, EC_CMD_TYPEC_STATUS, &req, sizeof(req),
+			  &resp, sizeof(resp));
+	if (ret < 0) {
+		dev_warn(sdata->dev, "EC_CMD_TYPEC_STATUS failed for port: %d\n", port_num);
+		return false;
+	}
+
+	if (resp.events & mask)
+		return true;
+
+	return false;
+}
+
+/*
+ * The ChromeOS EC treats both mode-switches and retimers as "muxes" for the purposes of the
+ * host command API. This common function configures and verifies the retimer/mode-switch
+ * according to the provided setting.
+ */
+static int cros_typec_configure_mux(struct cros_typec_switch_data *sdata, int port_num, int index,
+				    unsigned long mode, struct typec_altmode *alt)
+{
+	unsigned long end;
+	u32 event_mask;
+	u8 mux_state;
+	int ret;
+
+	ret = cros_typec_get_mux_state(mode, alt);
+	if (ret < 0)
+		return ret;
+	mux_state = (u8)ret;
+
+	/* Clear any old mux set done event. */
+	if (index == 0)
+		event_mask = PD_STATUS_EVENT_MUX_0_SET_DONE;
+	else
+		event_mask = PD_STATUS_EVENT_MUX_1_SET_DONE;
+
+	ret = cros_typec_send_clear_event(sdata, port_num, event_mask);
+	if (ret < 0)
+		return ret;
+
+	/* Send the set command. */
+	ret = cros_typec_cmd_mux_set(sdata, port_num, index, mux_state);
+	if (ret < 0)
+		return ret;
+
+	/* Check for the mux set done event. */
+	end = jiffies + msecs_to_jiffies(1000);
+	do {
+		if (cros_typec_check_event(sdata, port_num, event_mask))
+			return 0;
+
+		usleep_range(500, 1000);
+	} while (time_before(jiffies, end));
+
+	dev_err(sdata->dev, "Timed out waiting for mux set done on index: %d, state: %d\n",
+		index, mux_state);
+
+	return -ETIMEDOUT;
+}
+
+static int cros_typec_mode_switch_set(struct typec_mux_dev *mode_switch,
+				      struct typec_mux_state *state)
+{
+	struct cros_typec_port *port = typec_mux_get_drvdata(mode_switch);
+
+	/* Mode switches have index 0. */
+	return cros_typec_configure_mux(port->sdata, port->port_num, 0, state->mode, state->alt);
+}
+
+static int cros_typec_retimer_set(struct typec_retimer *retimer, struct typec_retimer_state *state)
+{
+	struct cros_typec_port *port = typec_retimer_get_drvdata(retimer);
+
+	/* Retimers have index 1. */
+	return cros_typec_configure_mux(port->sdata, port->port_num, 1, state->mode, state->alt);
+}
+
+static void cros_typec_unregister_switches(struct cros_typec_switch_data *sdata)
+{
+	int i;
+
+	for (i = 0; i < EC_USB_PD_MAX_PORTS; i++) {
+		if (!sdata->ports[i])
+			continue;
+		typec_retimer_unregister(sdata->ports[i]->retimer);
+		typec_mux_unregister(sdata->ports[i]->mode_switch);
+	}
+}
+
+static int cros_typec_register_mode_switch(struct cros_typec_port *port,
+					   struct fwnode_handle *fwnode)
+{
+	struct typec_mux_desc mode_switch_desc = {
+		.fwnode = fwnode,
+		.drvdata = port,
+		.name = fwnode_get_name(fwnode),
+		.set = cros_typec_mode_switch_set,
+	};
+
+	port->mode_switch = typec_mux_register(port->sdata->dev, &mode_switch_desc);
+
+	return PTR_ERR_OR_ZERO(port->mode_switch);
+}
+
+static int cros_typec_register_retimer(struct cros_typec_port *port, struct fwnode_handle *fwnode)
+{
+	struct typec_retimer_desc retimer_desc = {
+		.fwnode = fwnode,
+		.drvdata = port,
+		.name = fwnode_get_name(fwnode),
+		.set = cros_typec_retimer_set,
+	};
+
+	port->retimer = typec_retimer_register(port->sdata->dev, &retimer_desc);
+
+	return PTR_ERR_OR_ZERO(port->retimer);
+}
+
+static int cros_typec_register_switches(struct cros_typec_switch_data *sdata)
+{
+	struct cros_typec_port *port;
+	struct device *dev = sdata->dev;
+	struct fwnode_handle *fwnode;
+	struct acpi_device *adev;
+	unsigned long long index;
+	int nports, ret;
+
+	nports = device_get_child_node_count(dev);
+	if (nports == 0) {
+		dev_err(dev, "No switch devices found.\n");
+		return -ENODEV;
+	}
+
+	device_for_each_child_node(dev, fwnode) {
+		port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
+		if (!port) {
+			ret = -ENOMEM;
+			goto err_switch;
+		}
+
+		adev = to_acpi_device_node(fwnode);
+		if (!adev) {
+			dev_err(fwnode->dev, "Couldn't get ACPI device handle\n");
+			ret = -ENODEV;
+			goto err_switch;
+		}
+
+		ret = acpi_evaluate_integer(adev->handle, "_ADR", NULL, &index);
+		if (ACPI_FAILURE(ret)) {
+			dev_err(fwnode->dev, "_ADR wasn't evaluated\n");
+			ret = -ENODATA;
+			goto err_switch;
+		}
+
+		if (index >= EC_USB_PD_MAX_PORTS) {
+			dev_err(fwnode->dev, "Invalid port index number: %llu\n", index);
+			ret = -EINVAL;
+			goto err_switch;
+		}
+		port->sdata = sdata;
+		port->port_num = index;
+		sdata->ports[index] = port;
+
+		ret = cros_typec_register_retimer(port, fwnode);
+		if (ret) {
+			dev_err(dev, "Retimer switch register failed\n");
+			goto err_switch;
+		}
+
+		dev_dbg(dev, "Retimer switch registered for index %llu\n", index);
+
+		if (!device_property_present(fwnode->dev, "mode-switch"))
+			continue;
+
+		ret = cros_typec_register_mode_switch(port, fwnode);
+		if (ret) {
+			dev_err(dev, "Mode switch register failed\n");
+			goto err_switch;
+		}
+
+		dev_dbg(dev, "Mode switch registered for index %llu\n", index);
+	}
+
+	return 0;
+err_switch:
+	cros_typec_unregister_switches(sdata);
+	return ret;
+}
+
+static int cros_typec_switch_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct cros_typec_switch_data *sdata;
+
+	sdata = devm_kzalloc(dev, sizeof(*sdata), GFP_KERNEL);
+	if (!sdata)
+		return -ENOMEM;
+
+	sdata->dev = dev;
+	sdata->ec = dev_get_drvdata(pdev->dev.parent);
+
+	platform_set_drvdata(pdev, sdata);
+
+	return cros_typec_register_switches(sdata);
+}
+
+static int cros_typec_switch_remove(struct platform_device *pdev)
+{
+	struct cros_typec_switch_data *sdata = platform_get_drvdata(pdev);
+
+	cros_typec_unregister_switches(sdata);
+	return 0;
+}
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id cros_typec_switch_acpi_id[] = {
+	{ "GOOG001A", 0 },
+	{}
+};
+MODULE_DEVICE_TABLE(acpi, cros_typec_switch_acpi_id);
+#endif
+
+static struct platform_driver cros_typec_switch_driver = {
+	.driver	= {
+		.name = "cros-typec-switch",
+		.acpi_match_table = ACPI_PTR(cros_typec_switch_acpi_id),
+	},
+	.probe = cros_typec_switch_probe,
+	.remove = cros_typec_switch_remove,
+};
+
+module_platform_driver(cros_typec_switch_driver);
+
+MODULE_AUTHOR("Prashant Malani <pmalani@chromium.org>");
+MODULE_DESCRIPTION("ChromeOS EC Type-C Switch control");
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/mellanox/mlxreg-lc.c b/drivers/platform/mellanox/mlxreg-lc.c
index 1e071df4c9f5..8d833836a6d3 100644
--- a/drivers/platform/mellanox/mlxreg-lc.c
+++ b/drivers/platform/mellanox/mlxreg-lc.c
@@ -564,10 +564,8 @@ static int mlxreg_lc_event_handler(void *handle, enum mlxreg_hotplug_kind kind,
 		 mlxreg_lc->data->slot, mlxreg_lc->state, kind, action);
 
 	mutex_lock(&mlxreg_lc->lock);
-	if (!(mlxreg_lc->state & MLXREG_LC_INITIALIZED)) {
-		mutex_unlock(&mlxreg_lc->lock);
-		return 0;
-	}
+	if (!(mlxreg_lc->state & MLXREG_LC_INITIALIZED))
+		goto mlxreg_lc_non_initialzed_exit;
 
 	switch (kind) {
 	case MLXREG_HOTPLUG_LC_SYNCED:
@@ -594,8 +592,8 @@ static int mlxreg_lc_event_handler(void *handle, enum mlxreg_hotplug_kind kind,
 				/* In case line card is configured - enable it. */
 				if (mlxreg_lc->state & MLXREG_LC_CONFIGURED)
 					err = mlxreg_lc_enable_disable(mlxreg_lc, 1);
-				mutex_unlock(&mlxreg_lc->lock);
-				return err;
+
+				goto mlxreg_lc_enable_disable_exit;
 			}
 			err = mlxreg_lc_create_static_devices(mlxreg_lc, mlxreg_lc->main_devs,
 							      mlxreg_lc->main_devs_num);
@@ -627,8 +625,10 @@ static int mlxreg_lc_event_handler(void *handle, enum mlxreg_hotplug_kind kind,
 		break;
 	}
 
+mlxreg_lc_enable_disable_exit:
 mlxreg_lc_power_on_off_fail:
 mlxreg_lc_create_static_devices_fail:
+mlxreg_lc_non_initialzed_exit:
 	mutex_unlock(&mlxreg_lc->lock);
 
 	return err;
diff --git a/drivers/platform/surface/surface3_power.c b/drivers/platform/surface/surface3_power.c
index 444ec81ba02d..73961a24c849 100644
--- a/drivers/platform/surface/surface3_power.c
+++ b/drivers/platform/surface/surface3_power.c
@@ -519,7 +519,7 @@ static int mshw0011_probe(struct i2c_client *client)
 	i2c_set_clientdata(client, data);
 
 	memset(&board_info, 0, sizeof(board_info));
-	strlcpy(board_info.type, "MSHW0011-bat0", I2C_NAME_SIZE);
+	strscpy(board_info.type, "MSHW0011-bat0", I2C_NAME_SIZE);
 
 	bat0 = i2c_acpi_new_device(dev, 1, &board_info);
 	if (IS_ERR(bat0))
@@ -554,7 +554,7 @@ out_err:
 	return error;
 }
 
-static int mshw0011_remove(struct i2c_client *client)
+static void mshw0011_remove(struct i2c_client *client)
 {
 	struct mshw0011_data *cdata = i2c_get_clientdata(client);
 
@@ -564,8 +564,6 @@ static int mshw0011_remove(struct i2c_client *client)
 		kthread_stop(cdata->poll_task);
 
 	i2c_unregister_device(cdata->bat0);
-
-	return 0;
 }
 
 static const struct acpi_device_id mshw0011_acpi_match[] = {
diff --git a/drivers/platform/surface/surface_acpi_notify.c b/drivers/platform/surface/surface_acpi_notify.c
index 44e317970557..50500e562963 100644
--- a/drivers/platform/surface/surface_acpi_notify.c
+++ b/drivers/platform/surface/surface_acpi_notify.c
@@ -355,7 +355,8 @@ static u32 san_evt_bat_nf(struct ssam_event_notifier *nf,
 	INIT_DELAYED_WORK(&work->work, san_evt_bat_workfn);
 	work->dev = d->dev;
 
-	memcpy(&work->event, event, sizeof(struct ssam_event) + event->length);
+	work->event = *event;
+	memcpy(work->event.data, event->data, event->length);
 
 	queue_delayed_work(san_wq, &work->work, delay);
 	return SSAM_NOTIF_HANDLED;
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index f2f98e942cf2..f5312f51de19 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -93,6 +93,7 @@ config PEAQ_WMI
 
 config NVIDIA_WMI_EC_BACKLIGHT
 	tristate "EC Backlight Driver for Hybrid Graphics Notebook Systems"
+	depends on ACPI_VIDEO
 	depends on ACPI_WMI
 	depends on BACKLIGHT_CLASS_DEVICE
 	help
@@ -790,6 +791,7 @@ config SAMSUNG_Q10
 config ACPI_TOSHIBA
 	tristate "Toshiba Laptop Extras"
 	depends on ACPI
+	depends on ACPI_BATTERY
 	depends on ACPI_WMI
 	select LEDS_CLASS
 	select NEW_LEDS
@@ -797,6 +799,7 @@ config ACPI_TOSHIBA
 	depends on INPUT
 	depends on SERIO_I8042 || SERIO_I8042 = n
 	depends on ACPI_VIDEO || ACPI_VIDEO = n
+	depends on HWMON || HWMON = n
 	depends on RFKILL || RFKILL = n
 	depends on IIO
 	select INPUT_SPARSEKMAP
diff --git a/drivers/platform/x86/acer-wmi.c b/drivers/platform/x86/acer-wmi.c
index f1259d81d86d..18224f9a5bc0 100644
--- a/drivers/platform/x86/acer-wmi.c
+++ b/drivers/platform/x86/acer-wmi.c
@@ -650,69 +650,6 @@ static const struct dmi_system_id non_acer_quirks[] __initconst = {
 	{}
 };
 
-static int __init
-video_set_backlight_video_vendor(const struct dmi_system_id *d)
-{
-	interface->capability &= ~ACER_CAP_BRIGHTNESS;
-	pr_info("Brightness must be controlled by generic video driver\n");
-	return 0;
-}
-
-static const struct dmi_system_id video_vendor_dmi_table[] __initconst = {
-	{
-		.callback = video_set_backlight_video_vendor,
-		.ident = "Acer TravelMate 4750",
-		.matches = {
-			DMI_MATCH(DMI_BOARD_VENDOR, "Acer"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "TravelMate 4750"),
-		},
-	},
-	{
-		.callback = video_set_backlight_video_vendor,
-		.ident = "Acer Extensa 5235",
-		.matches = {
-			DMI_MATCH(DMI_BOARD_VENDOR, "Acer"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "Extensa 5235"),
-		},
-	},
-	{
-		.callback = video_set_backlight_video_vendor,
-		.ident = "Acer TravelMate 5760",
-		.matches = {
-			DMI_MATCH(DMI_BOARD_VENDOR, "Acer"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "TravelMate 5760"),
-		},
-	},
-	{
-		.callback = video_set_backlight_video_vendor,
-		.ident = "Acer Aspire 5750",
-		.matches = {
-			DMI_MATCH(DMI_BOARD_VENDOR, "Acer"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire 5750"),
-		},
-	},
-	{
-		.callback = video_set_backlight_video_vendor,
-		.ident = "Acer Aspire 5741",
-		.matches = {
-			DMI_MATCH(DMI_BOARD_VENDOR, "Acer"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "Aspire 5741"),
-		},
-	},
-	{
-		/*
-		 * Note no video_set_backlight_video_vendor, we must use the
-		 * acer interface, as there is no native backlight interface.
-		 */
-		.ident = "Acer KAV80",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Acer"),
-			DMI_MATCH(DMI_PRODUCT_NAME, "KAV80"),
-		},
-	},
-	{}
-};
-
 /* Find which quirks are needed for a particular vendor/ model pair */
 static void __init find_quirks(void)
 {
@@ -2484,9 +2421,6 @@ static int __init acer_wmi_init(void)
 
 	set_quirks();
 
-	if (dmi_check_system(video_vendor_dmi_table))
-		acpi_video_set_dmi_backlight_type(acpi_backlight_vendor);
-
 	if (acpi_video_get_backlight_type() != acpi_backlight_vendor)
 		interface->capability &= ~ACER_CAP_BRIGHTNESS;
 
@@ -2529,7 +2463,7 @@ static int __init acer_wmi_init(void)
 		goto error_platform_register;
 	}
 
-	acer_platform_device = platform_device_alloc("acer-wmi", -1);
+	acer_platform_device = platform_device_alloc("acer-wmi", PLATFORM_DEVID_NONE);
 	if (!acer_platform_device) {
 		err = -ENOMEM;
 		goto error_device_alloc;
diff --git a/drivers/platform/x86/acerhdf.c b/drivers/platform/x86/acerhdf.c
index 3463629f8764..d2c0fc38c201 100644
--- a/drivers/platform/x86/acerhdf.c
+++ b/drivers/platform/x86/acerhdf.c
@@ -676,7 +676,7 @@ static int __init acerhdf_register_platform(void)
 	if (err)
 		return err;
 
-	acerhdf_dev = platform_device_alloc("acerhdf", -1);
+	acerhdf_dev = platform_device_alloc("acerhdf", PLATFORM_DEVID_NONE);
 	if (!acerhdf_dev) {
 		err = -ENOMEM;
 		goto err_device_alloc;
diff --git a/drivers/platform/x86/amd/Kconfig b/drivers/platform/x86/amd/Kconfig
index c0d0a3c5170c..a825af8126c8 100644
--- a/drivers/platform/x86/amd/Kconfig
+++ b/drivers/platform/x86/amd/Kconfig
@@ -3,6 +3,8 @@
 # AMD x86 Platform Specific Drivers
 #
 
+source "drivers/platform/x86/amd/pmf/Kconfig"
+
 config AMD_PMC
 	tristate "AMD SoC PMC driver"
 	depends on ACPI && PCI && RTC_CLASS
diff --git a/drivers/platform/x86/amd/Makefile b/drivers/platform/x86/amd/Makefile
index a03fbb08e808..2c229198e24c 100644
--- a/drivers/platform/x86/amd/Makefile
+++ b/drivers/platform/x86/amd/Makefile
@@ -8,3 +8,4 @@ amd-pmc-y			:= pmc.o
 obj-$(CONFIG_AMD_PMC)		+= amd-pmc.o
 amd_hsmp-y			:= hsmp.o
 obj-$(CONFIG_AMD_HSMP)		+= amd_hsmp.o
+obj-$(CONFIG_AMD_PMF)		+= pmf/
diff --git a/drivers/platform/x86/amd/hsmp.c b/drivers/platform/x86/amd/hsmp.c
index a0c54b838c11..521c6a229362 100644
--- a/drivers/platform/x86/amd/hsmp.c
+++ b/drivers/platform/x86/amd/hsmp.c
@@ -392,7 +392,7 @@ static int __init hsmp_plt_init(void)
 	if (ret)
 		return ret;
 
-	amd_hsmp_platdev = platform_device_alloc(DRIVER_NAME, -1);
+	amd_hsmp_platdev = platform_device_alloc(DRIVER_NAME, PLATFORM_DEVID_NONE);
 	if (!amd_hsmp_platdev) {
 		ret = -ENOMEM;
 		goto drv_unregister;
diff --git a/drivers/platform/x86/amd/pmc.c b/drivers/platform/x86/amd/pmc.c
index 700eb19e8450..ce859b300712 100644
--- a/drivers/platform/x86/amd/pmc.c
+++ b/drivers/platform/x86/amd/pmc.c
@@ -39,7 +39,9 @@
 #define AMD_PMC_STB_INDEX_ADDRESS	0xF8
 #define AMD_PMC_STB_INDEX_DATA		0xFC
 #define AMD_PMC_STB_PMI_0		0x03E30600
-#define AMD_PMC_STB_PREDEF		0xC6000001
+#define AMD_PMC_STB_S2IDLE_PREPARE	0xC6000001
+#define AMD_PMC_STB_S2IDLE_RESTORE	0xC6000002
+#define AMD_PMC_STB_S2IDLE_CHECK	0xC6000003
 
 /* STB S2D(Spill to DRAM) has different message port offset */
 #define STB_SPILL_TO_DRAM		0xBE
@@ -151,9 +153,7 @@ struct amd_pmc_dev {
 	struct device *dev;
 	struct pci_dev *rdev;
 	struct mutex lock; /* generic mutex lock */
-#if IS_ENABLED(CONFIG_DEBUG_FS)
 	struct dentry *dbgfs_dir;
-#endif /* CONFIG_DEBUG_FS */
 };
 
 static bool enable_stb;
@@ -369,7 +369,64 @@ static void amd_pmc_validate_deepest(struct amd_pmc_dev *pdev)
 }
 #endif
 
-#ifdef CONFIG_DEBUG_FS
+static int amd_pmc_get_smu_version(struct amd_pmc_dev *dev)
+{
+	int rc;
+	u32 val;
+
+	rc = amd_pmc_send_cmd(dev, 0, &val, SMU_MSG_GETSMUVERSION, 1);
+	if (rc)
+		return rc;
+
+	dev->smu_program = (val >> 24) & GENMASK(7, 0);
+	dev->major = (val >> 16) & GENMASK(7, 0);
+	dev->minor = (val >> 8) & GENMASK(7, 0);
+	dev->rev = (val >> 0) & GENMASK(7, 0);
+
+	dev_dbg(dev->dev, "SMU program %u version is %u.%u.%u\n",
+		dev->smu_program, dev->major, dev->minor, dev->rev);
+
+	return 0;
+}
+
+static ssize_t smu_fw_version_show(struct device *d, struct device_attribute *attr,
+				   char *buf)
+{
+	struct amd_pmc_dev *dev = dev_get_drvdata(d);
+
+	if (!dev->major) {
+		int rc = amd_pmc_get_smu_version(dev);
+
+		if (rc)
+			return rc;
+	}
+	return sysfs_emit(buf, "%u.%u.%u\n", dev->major, dev->minor, dev->rev);
+}
+
+static ssize_t smu_program_show(struct device *d, struct device_attribute *attr,
+				   char *buf)
+{
+	struct amd_pmc_dev *dev = dev_get_drvdata(d);
+
+	if (!dev->major) {
+		int rc = amd_pmc_get_smu_version(dev);
+
+		if (rc)
+			return rc;
+	}
+	return sysfs_emit(buf, "%u\n", dev->smu_program);
+}
+
+static DEVICE_ATTR_RO(smu_fw_version);
+static DEVICE_ATTR_RO(smu_program);
+
+static struct attribute *pmc_attrs[] = {
+	&dev_attr_smu_fw_version.attr,
+	&dev_attr_smu_program.attr,
+	NULL,
+};
+ATTRIBUTE_GROUPS(pmc);
+
 static int smu_fw_info_show(struct seq_file *s, void *unused)
 {
 	struct amd_pmc_dev *dev = s->private;
@@ -435,26 +492,6 @@ static int s0ix_stats_show(struct seq_file *s, void *unused)
 }
 DEFINE_SHOW_ATTRIBUTE(s0ix_stats);
 
-static int amd_pmc_get_smu_version(struct amd_pmc_dev *dev)
-{
-	int rc;
-	u32 val;
-
-	rc = amd_pmc_send_cmd(dev, 0, &val, SMU_MSG_GETSMUVERSION, 1);
-	if (rc)
-		return rc;
-
-	dev->smu_program = (val >> 24) & GENMASK(7, 0);
-	dev->major = (val >> 16) & GENMASK(7, 0);
-	dev->minor = (val >> 8) & GENMASK(7, 0);
-	dev->rev = (val >> 0) & GENMASK(7, 0);
-
-	dev_dbg(dev->dev, "SMU program %u version is %u.%u.%u\n",
-		dev->smu_program, dev->major, dev->minor, dev->rev);
-
-	return 0;
-}
-
 static int amd_pmc_idlemask_show(struct seq_file *s, void *unused)
 {
 	struct amd_pmc_dev *dev = s->private;
@@ -504,15 +541,6 @@ static void amd_pmc_dbgfs_register(struct amd_pmc_dev *dev)
 					    &amd_pmc_stb_debugfs_fops);
 	}
 }
-#else
-static inline void amd_pmc_dbgfs_register(struct amd_pmc_dev *dev)
-{
-}
-
-static inline void amd_pmc_dbgfs_unregister(struct amd_pmc_dev *dev)
-{
-}
-#endif /* CONFIG_DEBUG_FS */
 
 static void amd_pmc_dump_registers(struct amd_pmc_dev *dev)
 {
@@ -691,8 +719,6 @@ static void amd_pmc_s2idle_prepare(void)
 		}
 	}
 
-	/* Dump the IdleMask before we send hint to SMU */
-	amd_pmc_idlemask_read(pdev, pdev->dev, NULL);
 	msg = amd_pmc_get_os_hint(pdev);
 	rc = amd_pmc_send_cmd(pdev, arg, NULL, msg, 0);
 	if (rc) {
@@ -700,11 +726,22 @@ static void amd_pmc_s2idle_prepare(void)
 		return;
 	}
 
-	if (enable_stb) {
-		rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_PREDEF);
-		if (rc)
-			dev_err(pdev->dev, "error writing to STB: %d\n", rc);
-	}
+	rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_S2IDLE_PREPARE);
+	if (rc)
+		dev_err(pdev->dev, "error writing to STB: %d\n", rc);
+}
+
+static void amd_pmc_s2idle_check(void)
+{
+	struct amd_pmc_dev *pdev = &pmc;
+	int rc;
+
+	/* Dump the IdleMask before we add to the STB */
+	amd_pmc_idlemask_read(pdev, pdev->dev, NULL);
+
+	rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_S2IDLE_CHECK);
+	if (rc)
+		dev_err(pdev->dev, "error writing to STB: %d\n", rc);
 }
 
 static void amd_pmc_s2idle_restore(void)
@@ -721,15 +758,9 @@ static void amd_pmc_s2idle_restore(void)
 	/* Let SMU know that we are looking for stats */
 	amd_pmc_send_cmd(pdev, 0, NULL, SMU_MSG_LOG_DUMP_DATA, 0);
 
-	/* Dump the IdleMask to see the blockers */
-	amd_pmc_idlemask_read(pdev, pdev->dev, NULL);
-
-	/* Write data incremented by 1 to distinguish in stb_read */
-	if (enable_stb) {
-		rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_PREDEF + 1);
-		if (rc)
-			dev_err(pdev->dev, "error writing to STB: %d\n", rc);
-	}
+	rc = amd_pmc_write_stb(pdev, AMD_PMC_STB_S2IDLE_RESTORE);
+	if (rc)
+		dev_err(pdev->dev, "error writing to STB: %d\n", rc);
 
 	/* Notify on failed entry */
 	amd_pmc_validate_deepest(pdev);
@@ -737,6 +768,7 @@ static void amd_pmc_s2idle_restore(void)
 
 static struct acpi_s2idle_dev_ops amd_pmc_s2idle_dev_ops = {
 	.prepare = amd_pmc_s2idle_prepare,
+	.check = amd_pmc_s2idle_check,
 	.restore = amd_pmc_s2idle_restore,
 };
 #endif
@@ -935,6 +967,7 @@ static struct platform_driver amd_pmc_driver = {
 	.driver = {
 		.name = "amd_pmc",
 		.acpi_match_table = amd_pmc_acpi_ids,
+		.dev_groups = pmc_groups,
 	},
 	.probe = amd_pmc_probe,
 	.remove = amd_pmc_remove,
diff --git a/drivers/platform/x86/amd/pmf/Kconfig b/drivers/platform/x86/amd/pmf/Kconfig
new file mode 100644
index 000000000000..c375498c4071
--- /dev/null
+++ b/drivers/platform/x86/amd/pmf/Kconfig
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# AMD PMF Driver
+#
+
+config AMD_PMF
+	tristate "AMD Platform Management Framework"
+	depends on ACPI && PCI
+	select ACPI_PLATFORM_PROFILE
+	help
+	  This driver provides support for the AMD Platform Management Framework.
+	  The goal is to enhance end user experience by making AMD PCs smarter,
+	  quiter, power efficient by adapting to user behavior and environment.
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called amd_pmf.
diff --git a/drivers/platform/x86/amd/pmf/Makefile b/drivers/platform/x86/amd/pmf/Makefile
new file mode 100644
index 000000000000..fdededf54392
--- /dev/null
+++ b/drivers/platform/x86/amd/pmf/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for linux/drivers/platform/x86/amd/pmf
+# AMD Platform Management Framework
+#
+
+obj-$(CONFIG_AMD_PMF) += amd-pmf.o
+amd-pmf-objs := core.o acpi.o sps.o \
+		auto-mode.o cnqf.o
diff --git a/drivers/platform/x86/amd/pmf/acpi.c b/drivers/platform/x86/amd/pmf/acpi.c
new file mode 100644
index 000000000000..081e84e116e7
--- /dev/null
+++ b/drivers/platform/x86/amd/pmf/acpi.c
@@ -0,0 +1,304 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD Platform Management Framework Driver
+ *
+ * Copyright (c) 2022, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Author: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+ */
+
+#include <linux/acpi.h>
+#include "pmf.h"
+
+#define APMF_CQL_NOTIFICATION  2
+#define APMF_AMT_NOTIFICATION  3
+
+static union acpi_object *apmf_if_call(struct amd_pmf_dev *pdev, int fn, struct acpi_buffer *param)
+{
+	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
+	acpi_handle ahandle = ACPI_HANDLE(pdev->dev);
+	struct acpi_object_list apmf_if_arg_list;
+	union acpi_object apmf_if_args[2];
+	acpi_status status;
+
+	apmf_if_arg_list.count = 2;
+	apmf_if_arg_list.pointer = &apmf_if_args[0];
+
+	apmf_if_args[0].type = ACPI_TYPE_INTEGER;
+	apmf_if_args[0].integer.value = fn;
+
+	if (param) {
+		apmf_if_args[1].type = ACPI_TYPE_BUFFER;
+		apmf_if_args[1].buffer.length = param->length;
+		apmf_if_args[1].buffer.pointer = param->pointer;
+	} else {
+		apmf_if_args[1].type = ACPI_TYPE_INTEGER;
+		apmf_if_args[1].integer.value = 0;
+	}
+
+	status = acpi_evaluate_object(ahandle, "APMF", &apmf_if_arg_list, &buffer);
+	if (ACPI_FAILURE(status)) {
+		dev_err(pdev->dev, "APMF method:%d call failed\n", fn);
+		kfree(buffer.pointer);
+		return NULL;
+	}
+
+	return buffer.pointer;
+}
+
+static int apmf_if_call_store_buffer(struct amd_pmf_dev *pdev, int fn, void *dest, size_t out_sz)
+{
+	union acpi_object *info;
+	size_t size;
+	int err = 0;
+
+	info = apmf_if_call(pdev, fn, NULL);
+	if (!info)
+		return -EIO;
+
+	if (info->type != ACPI_TYPE_BUFFER) {
+		dev_err(pdev->dev, "object is not a buffer\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	if (info->buffer.length < 2) {
+		dev_err(pdev->dev, "buffer too small\n");
+		err = -EINVAL;
+		goto out;
+	}
+
+	size = *(u16 *)info->buffer.pointer;
+	if (info->buffer.length < size) {
+		dev_err(pdev->dev, "buffer smaller then headersize %u < %zu\n",
+			info->buffer.length, size);
+		err = -EINVAL;
+		goto out;
+	}
+
+	if (size < out_sz) {
+		dev_err(pdev->dev, "buffer too small %zu\n", size);
+		err = -EINVAL;
+		goto out;
+	}
+
+	memcpy(dest, info->buffer.pointer, out_sz);
+
+out:
+	kfree(info);
+	return err;
+}
+
+int is_apmf_func_supported(struct amd_pmf_dev *pdev, unsigned long index)
+{
+	/* If bit-n is set, that indicates function n+1 is supported */
+	return !!(pdev->supported_func & BIT(index - 1));
+}
+
+int apmf_get_static_slider_granular(struct amd_pmf_dev *pdev,
+				    struct apmf_static_slider_granular_output *data)
+{
+	if (!is_apmf_func_supported(pdev, APMF_FUNC_STATIC_SLIDER_GRANULAR))
+		return -EINVAL;
+
+	return apmf_if_call_store_buffer(pdev, APMF_FUNC_STATIC_SLIDER_GRANULAR,
+									 data, sizeof(*data));
+}
+
+static void apmf_sbios_heartbeat_notify(struct work_struct *work)
+{
+	struct amd_pmf_dev *dev = container_of(work, struct amd_pmf_dev, heart_beat.work);
+	union acpi_object *info;
+
+	dev_dbg(dev->dev, "Sending heartbeat to SBIOS\n");
+	info = apmf_if_call(dev, APMF_FUNC_SBIOS_HEARTBEAT, NULL);
+	if (!info)
+		goto out;
+
+	schedule_delayed_work(&dev->heart_beat, msecs_to_jiffies(dev->hb_interval * 1000));
+
+out:
+	kfree(info);
+}
+
+int apmf_update_fan_idx(struct amd_pmf_dev *pdev, bool manual, u32 idx)
+{
+	union acpi_object *info;
+	struct apmf_fan_idx args;
+	struct acpi_buffer params;
+	int err = 0;
+
+	args.size = sizeof(args);
+	args.fan_ctl_mode = manual;
+	args.fan_ctl_idx = idx;
+
+	params.length = sizeof(args);
+	params.pointer = (void *)&args;
+
+	info = apmf_if_call(pdev, APMF_FUNC_SET_FAN_IDX, &params);
+	if (!info) {
+		err = -EIO;
+		goto out;
+	}
+
+out:
+	kfree(info);
+	return err;
+}
+
+int apmf_get_auto_mode_def(struct amd_pmf_dev *pdev, struct apmf_auto_mode *data)
+{
+	return apmf_if_call_store_buffer(pdev, APMF_FUNC_AUTO_MODE, data, sizeof(*data));
+}
+
+int apmf_get_sbios_requests(struct amd_pmf_dev *pdev, struct apmf_sbios_req *req)
+{
+	return apmf_if_call_store_buffer(pdev, APMF_FUNC_SBIOS_REQUESTS,
+									 req, sizeof(*req));
+}
+
+static void apmf_event_handler(acpi_handle handle, u32 event, void *data)
+{
+	struct amd_pmf_dev *pmf_dev = data;
+	struct apmf_sbios_req req;
+	int ret;
+
+	mutex_lock(&pmf_dev->update_mutex);
+	ret = apmf_get_sbios_requests(pmf_dev, &req);
+	if (ret) {
+		dev_err(pmf_dev->dev, "Failed to get SBIOS requests:%d\n", ret);
+		goto out;
+	}
+
+	if (req.pending_req & BIT(APMF_AMT_NOTIFICATION)) {
+		dev_dbg(pmf_dev->dev, "AMT is supported and notifications %s\n",
+			req.amt_event ? "Enabled" : "Disabled");
+		pmf_dev->amt_enabled = !!req.amt_event;
+
+		if (pmf_dev->amt_enabled)
+			amd_pmf_handle_amt(pmf_dev);
+		else
+			amd_pmf_reset_amt(pmf_dev);
+	}
+
+	if (req.pending_req & BIT(APMF_CQL_NOTIFICATION)) {
+		dev_dbg(pmf_dev->dev, "CQL is supported and notifications %s\n",
+			req.cql_event ? "Enabled" : "Disabled");
+
+		/* update the target mode information */
+		if (pmf_dev->amt_enabled)
+			amd_pmf_update_2_cql(pmf_dev, req.cql_event);
+	}
+out:
+	mutex_unlock(&pmf_dev->update_mutex);
+}
+
+static int apmf_if_verify_interface(struct amd_pmf_dev *pdev)
+{
+	struct apmf_verify_interface output;
+	int err;
+
+	err = apmf_if_call_store_buffer(pdev, APMF_FUNC_VERIFY_INTERFACE, &output, sizeof(output));
+	if (err)
+		return err;
+
+	pdev->supported_func = output.supported_functions;
+	dev_dbg(pdev->dev, "supported functions:0x%x notifications:0x%x\n",
+		output.supported_functions, output.notification_mask);
+
+	return 0;
+}
+
+static int apmf_get_system_params(struct amd_pmf_dev *dev)
+{
+	struct apmf_system_params params;
+	int err;
+
+	if (!is_apmf_func_supported(dev, APMF_FUNC_GET_SYS_PARAMS))
+		return -EINVAL;
+
+	err = apmf_if_call_store_buffer(dev, APMF_FUNC_GET_SYS_PARAMS, &params, sizeof(params));
+	if (err)
+		return err;
+
+	dev_dbg(dev->dev, "system params mask:0x%x flags:0x%x cmd_code:0x%x heartbeat:%d\n",
+		params.valid_mask,
+		params.flags,
+		params.command_code,
+		params.heartbeat_int);
+	params.flags = params.flags & params.valid_mask;
+	dev->hb_interval = params.heartbeat_int;
+
+	return 0;
+}
+
+int apmf_get_dyn_slider_def_ac(struct amd_pmf_dev *pdev, struct apmf_dyn_slider_output *data)
+{
+	return apmf_if_call_store_buffer(pdev, APMF_FUNC_DYN_SLIDER_AC, data, sizeof(*data));
+}
+
+int apmf_get_dyn_slider_def_dc(struct amd_pmf_dev *pdev, struct apmf_dyn_slider_output *data)
+{
+	return apmf_if_call_store_buffer(pdev, APMF_FUNC_DYN_SLIDER_DC, data, sizeof(*data));
+}
+
+int apmf_install_handler(struct amd_pmf_dev *pmf_dev)
+{
+	acpi_handle ahandle = ACPI_HANDLE(pmf_dev->dev);
+	acpi_status status;
+
+	/* Install the APMF Notify handler */
+	if (is_apmf_func_supported(pmf_dev, APMF_FUNC_AUTO_MODE) &&
+	    is_apmf_func_supported(pmf_dev, APMF_FUNC_SBIOS_REQUESTS)) {
+		status = acpi_install_notify_handler(ahandle, ACPI_ALL_NOTIFY,
+						     apmf_event_handler, pmf_dev);
+		if (ACPI_FAILURE(status)) {
+			dev_err(pmf_dev->dev, "failed to install notify handler\n");
+			return -ENODEV;
+		}
+
+		/* Call the handler once manually to catch up with possibly missed notifies. */
+		apmf_event_handler(ahandle, 0, pmf_dev);
+	}
+
+	return 0;
+}
+
+void apmf_acpi_deinit(struct amd_pmf_dev *pmf_dev)
+{
+	acpi_handle ahandle = ACPI_HANDLE(pmf_dev->dev);
+
+	if (pmf_dev->hb_interval)
+		cancel_delayed_work_sync(&pmf_dev->heart_beat);
+
+	if (is_apmf_func_supported(pmf_dev, APMF_FUNC_AUTO_MODE) &&
+	    is_apmf_func_supported(pmf_dev, APMF_FUNC_SBIOS_REQUESTS))
+		acpi_remove_notify_handler(ahandle, ACPI_ALL_NOTIFY, apmf_event_handler);
+}
+
+int apmf_acpi_init(struct amd_pmf_dev *pmf_dev)
+{
+	int ret;
+
+	ret = apmf_if_verify_interface(pmf_dev);
+	if (ret) {
+		dev_err(pmf_dev->dev, "APMF verify interface failed :%d\n", ret);
+		goto out;
+	}
+
+	ret = apmf_get_system_params(pmf_dev);
+	if (ret) {
+		dev_err(pmf_dev->dev, "APMF apmf_get_system_params failed :%d\n", ret);
+		goto out;
+	}
+
+	if (pmf_dev->hb_interval) {
+		/* send heartbeats only if the interval is not zero */
+		INIT_DELAYED_WORK(&pmf_dev->heart_beat, apmf_sbios_heartbeat_notify);
+		schedule_delayed_work(&pmf_dev->heart_beat, 0);
+	}
+
+out:
+	return ret;
+}
diff --git a/drivers/platform/x86/amd/pmf/auto-mode.c b/drivers/platform/x86/amd/pmf/auto-mode.c
new file mode 100644
index 000000000000..644af42e07cf
--- /dev/null
+++ b/drivers/platform/x86/amd/pmf/auto-mode.c
@@ -0,0 +1,305 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD Platform Management Framework Driver
+ *
+ * Copyright (c) 2022, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Author: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+ */
+
+#include <linux/acpi.h>
+#include <linux/workqueue.h>
+#include "pmf.h"
+
+static struct auto_mode_mode_config config_store;
+static const char *state_as_str(unsigned int state);
+
+static void amd_pmf_set_automode(struct amd_pmf_dev *dev, int idx,
+				 struct auto_mode_mode_config *table)
+{
+	struct power_table_control *pwr_ctrl = &config_store.mode_set[idx].power_control;
+
+	amd_pmf_send_cmd(dev, SET_SPL, false, pwr_ctrl->spl, NULL);
+	amd_pmf_send_cmd(dev, SET_FPPT, false, pwr_ctrl->fppt, NULL);
+	amd_pmf_send_cmd(dev, SET_SPPT, false, pwr_ctrl->sppt, NULL);
+	amd_pmf_send_cmd(dev, SET_SPPT_APU_ONLY, false, pwr_ctrl->sppt_apu_only, NULL);
+	amd_pmf_send_cmd(dev, SET_STT_MIN_LIMIT, false, pwr_ctrl->stt_min, NULL);
+	amd_pmf_send_cmd(dev, SET_STT_LIMIT_APU, false,
+			 pwr_ctrl->stt_skin_temp[STT_TEMP_APU], NULL);
+	amd_pmf_send_cmd(dev, SET_STT_LIMIT_HS2, false,
+			 pwr_ctrl->stt_skin_temp[STT_TEMP_HS2], NULL);
+
+	if (is_apmf_func_supported(dev, APMF_FUNC_SET_FAN_IDX))
+		apmf_update_fan_idx(dev, config_store.mode_set[idx].fan_control.manual,
+				    config_store.mode_set[idx].fan_control.fan_id);
+}
+
+static int amd_pmf_get_moving_avg(struct amd_pmf_dev *pdev, int socket_power)
+{
+	int i, total = 0;
+
+	if (pdev->socket_power_history_idx == -1) {
+		for (i = 0; i < AVG_SAMPLE_SIZE; i++)
+			pdev->socket_power_history[i] = socket_power;
+	}
+
+	pdev->socket_power_history_idx = (pdev->socket_power_history_idx + 1) % AVG_SAMPLE_SIZE;
+	pdev->socket_power_history[pdev->socket_power_history_idx] = socket_power;
+
+	for (i = 0; i < AVG_SAMPLE_SIZE; i++)
+		total += pdev->socket_power_history[i];
+
+	return total / AVG_SAMPLE_SIZE;
+}
+
+void amd_pmf_trans_automode(struct amd_pmf_dev *dev, int socket_power, ktime_t time_elapsed_ms)
+{
+	int avg_power = 0;
+	bool update = false;
+	int i, j;
+
+	/* Get the average moving average computed by auto mode algorithm */
+	avg_power = amd_pmf_get_moving_avg(dev, socket_power);
+
+	for (i = 0; i < AUTO_TRANSITION_MAX; i++) {
+		if ((config_store.transition[i].shifting_up && avg_power >=
+		     config_store.transition[i].power_threshold) ||
+		    (!config_store.transition[i].shifting_up && avg_power <=
+		     config_store.transition[i].power_threshold)) {
+			if (config_store.transition[i].timer <
+			    config_store.transition[i].time_constant)
+				config_store.transition[i].timer += time_elapsed_ms;
+		} else {
+			config_store.transition[i].timer = 0;
+		}
+
+		if (config_store.transition[i].timer >=
+		    config_store.transition[i].time_constant &&
+		    !config_store.transition[i].applied) {
+			config_store.transition[i].applied = true;
+			update = true;
+		} else if (config_store.transition[i].timer <=
+			   config_store.transition[i].time_constant &&
+			   config_store.transition[i].applied) {
+			config_store.transition[i].applied = false;
+			update = true;
+		}
+	}
+
+	dev_dbg(dev->dev, "[AUTO_MODE] avg power: %u mW mode: %s\n", avg_power,
+		state_as_str(config_store.current_mode));
+
+	if (update) {
+		for (j = 0; j < AUTO_TRANSITION_MAX; j++) {
+			/* Apply the mode with highest priority indentified */
+			if (config_store.transition[j].applied) {
+				if (config_store.current_mode !=
+				    config_store.transition[j].target_mode) {
+					config_store.current_mode =
+							config_store.transition[j].target_mode;
+					dev_dbg(dev->dev, "[AUTO_MODE] moving to mode:%s\n",
+						state_as_str(config_store.current_mode));
+					amd_pmf_set_automode(dev, config_store.current_mode, NULL);
+				}
+				break;
+			}
+		}
+	}
+}
+
+void amd_pmf_update_2_cql(struct amd_pmf_dev *dev, bool is_cql_event)
+{
+	int mode = config_store.current_mode;
+
+	config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].target_mode =
+				   is_cql_event ? AUTO_PERFORMANCE_ON_LAP : AUTO_PERFORMANCE;
+
+	if ((mode == AUTO_PERFORMANCE || mode == AUTO_PERFORMANCE_ON_LAP) &&
+	    mode != config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].target_mode) {
+		mode = config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].target_mode;
+		amd_pmf_set_automode(dev, mode, NULL);
+	}
+	dev_dbg(dev->dev, "updated CQL thermals\n");
+}
+
+static void amd_pmf_get_power_threshold(void)
+{
+	config_store.transition[AUTO_TRANSITION_TO_QUIET].power_threshold =
+				config_store.mode_set[AUTO_BALANCE].power_floor -
+				config_store.transition[AUTO_TRANSITION_TO_QUIET].power_delta;
+
+	config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].power_threshold =
+				config_store.mode_set[AUTO_BALANCE].power_floor -
+				config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].power_delta;
+
+	config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].power_threshold =
+			config_store.mode_set[AUTO_QUIET].power_floor -
+			config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].power_delta;
+
+	config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].power_threshold =
+		config_store.mode_set[AUTO_PERFORMANCE].power_floor -
+		config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].power_delta;
+}
+
+static const char *state_as_str(unsigned int state)
+{
+	switch (state) {
+	case AUTO_QUIET:
+		return "QUIET";
+	case AUTO_BALANCE:
+		return "BALANCED";
+	case AUTO_PERFORMANCE_ON_LAP:
+		return "ON_LAP";
+	case AUTO_PERFORMANCE:
+		return "PERFORMANCE";
+	default:
+		return "Unknown Auto Mode State";
+	}
+}
+
+static void amd_pmf_load_defaults_auto_mode(struct amd_pmf_dev *dev)
+{
+	struct apmf_auto_mode output;
+	struct power_table_control *pwr_ctrl;
+	int i;
+
+	apmf_get_auto_mode_def(dev, &output);
+	/* time constant */
+	config_store.transition[AUTO_TRANSITION_TO_QUIET].time_constant =
+								output.balanced_to_quiet;
+	config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].time_constant =
+								output.balanced_to_perf;
+	config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].time_constant =
+								output.quiet_to_balanced;
+	config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].time_constant =
+								output.perf_to_balanced;
+
+	/* power floor */
+	config_store.mode_set[AUTO_QUIET].power_floor = output.pfloor_quiet;
+	config_store.mode_set[AUTO_BALANCE].power_floor = output.pfloor_balanced;
+	config_store.mode_set[AUTO_PERFORMANCE].power_floor = output.pfloor_perf;
+	config_store.mode_set[AUTO_PERFORMANCE_ON_LAP].power_floor = output.pfloor_perf;
+
+	/* Power delta for mode change */
+	config_store.transition[AUTO_TRANSITION_TO_QUIET].power_delta =
+								output.pd_balanced_to_quiet;
+	config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].power_delta =
+								output.pd_balanced_to_perf;
+	config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].power_delta =
+								output.pd_quiet_to_balanced;
+	config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].power_delta =
+								output.pd_perf_to_balanced;
+
+	/* Power threshold */
+	amd_pmf_get_power_threshold();
+
+	/* skin temperature limits */
+	pwr_ctrl = &config_store.mode_set[AUTO_QUIET].power_control;
+	pwr_ctrl->spl = output.spl_quiet;
+	pwr_ctrl->sppt = output.sppt_quiet;
+	pwr_ctrl->fppt = output.fppt_quiet;
+	pwr_ctrl->sppt_apu_only = output.sppt_apu_only_quiet;
+	pwr_ctrl->stt_min = output.stt_min_limit_quiet;
+	pwr_ctrl->stt_skin_temp[STT_TEMP_APU] = output.stt_apu_quiet;
+	pwr_ctrl->stt_skin_temp[STT_TEMP_HS2] = output.stt_hs2_quiet;
+
+	pwr_ctrl = &config_store.mode_set[AUTO_BALANCE].power_control;
+	pwr_ctrl->spl = output.spl_balanced;
+	pwr_ctrl->sppt = output.sppt_balanced;
+	pwr_ctrl->fppt = output.fppt_balanced;
+	pwr_ctrl->sppt_apu_only = output.sppt_apu_only_balanced;
+	pwr_ctrl->stt_min = output.stt_min_limit_balanced;
+	pwr_ctrl->stt_skin_temp[STT_TEMP_APU] = output.stt_apu_balanced;
+	pwr_ctrl->stt_skin_temp[STT_TEMP_HS2] = output.stt_hs2_balanced;
+
+	pwr_ctrl = &config_store.mode_set[AUTO_PERFORMANCE].power_control;
+	pwr_ctrl->spl = output.spl_perf;
+	pwr_ctrl->sppt = output.sppt_perf;
+	pwr_ctrl->fppt = output.fppt_perf;
+	pwr_ctrl->sppt_apu_only = output.sppt_apu_only_perf;
+	pwr_ctrl->stt_min = output.stt_min_limit_perf;
+	pwr_ctrl->stt_skin_temp[STT_TEMP_APU] = output.stt_apu_perf;
+	pwr_ctrl->stt_skin_temp[STT_TEMP_HS2] = output.stt_hs2_perf;
+
+	pwr_ctrl = &config_store.mode_set[AUTO_PERFORMANCE_ON_LAP].power_control;
+	pwr_ctrl->spl = output.spl_perf_on_lap;
+	pwr_ctrl->sppt = output.sppt_perf_on_lap;
+	pwr_ctrl->fppt = output.fppt_perf_on_lap;
+	pwr_ctrl->sppt_apu_only = output.sppt_apu_only_perf_on_lap;
+	pwr_ctrl->stt_min = output.stt_min_limit_perf_on_lap;
+	pwr_ctrl->stt_skin_temp[STT_TEMP_APU] = output.stt_apu_perf_on_lap;
+	pwr_ctrl->stt_skin_temp[STT_TEMP_HS2] = output.stt_hs2_perf_on_lap;
+
+	/* Fan ID */
+	config_store.mode_set[AUTO_QUIET].fan_control.fan_id = output.fan_id_quiet;
+	config_store.mode_set[AUTO_BALANCE].fan_control.fan_id = output.fan_id_balanced;
+	config_store.mode_set[AUTO_PERFORMANCE].fan_control.fan_id = output.fan_id_perf;
+	config_store.mode_set[AUTO_PERFORMANCE_ON_LAP].fan_control.fan_id =
+									output.fan_id_perf;
+
+	config_store.transition[AUTO_TRANSITION_TO_QUIET].target_mode = AUTO_QUIET;
+	config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].target_mode =
+								AUTO_PERFORMANCE;
+	config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].target_mode =
+									AUTO_BALANCE;
+	config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].target_mode =
+									AUTO_BALANCE;
+
+	config_store.transition[AUTO_TRANSITION_TO_QUIET].shifting_up = false;
+	config_store.transition[AUTO_TRANSITION_TO_PERFORMANCE].shifting_up = true;
+	config_store.transition[AUTO_TRANSITION_FROM_QUIET_TO_BALANCE].shifting_up = true;
+	config_store.transition[AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE].shifting_up =
+										false;
+
+	for (i = 0 ; i < AUTO_MODE_MAX ; i++) {
+		if (config_store.mode_set[i].fan_control.fan_id == FAN_INDEX_AUTO)
+			config_store.mode_set[i].fan_control.manual = false;
+		else
+			config_store.mode_set[i].fan_control.manual = true;
+	}
+
+	/* set to initial default values */
+	config_store.current_mode = AUTO_BALANCE;
+	dev->socket_power_history_idx = -1;
+}
+
+int amd_pmf_reset_amt(struct amd_pmf_dev *dev)
+{
+	/*
+	 * OEM BIOS implementation guide says that if the auto mode is enabled
+	 * the platform_profile registration shall be done by the OEM driver.
+	 * There could be cases where both static slider and auto mode BIOS
+	 * functions are enabled, in that case enable static slider updates
+	 * only if it advertised as supported.
+	 */
+
+	if (is_apmf_func_supported(dev, APMF_FUNC_STATIC_SLIDER_GRANULAR)) {
+		int mode = amd_pmf_get_pprof_modes(dev);
+
+		if (mode < 0)
+			return mode;
+
+		dev_dbg(dev->dev, "resetting AMT thermals\n");
+		amd_pmf_update_slider(dev, SLIDER_OP_SET, mode, NULL);
+	}
+	return 0;
+}
+
+void amd_pmf_handle_amt(struct amd_pmf_dev *dev)
+{
+	amd_pmf_set_automode(dev, config_store.current_mode, NULL);
+}
+
+void amd_pmf_deinit_auto_mode(struct amd_pmf_dev *dev)
+{
+	cancel_delayed_work_sync(&dev->work_buffer);
+}
+
+void amd_pmf_init_auto_mode(struct amd_pmf_dev *dev)
+{
+	amd_pmf_load_defaults_auto_mode(dev);
+	/* update the thermal limits for Automode */
+	amd_pmf_set_automode(dev, config_store.current_mode, NULL);
+	amd_pmf_init_metrics_table(dev);
+}
diff --git a/drivers/platform/x86/amd/pmf/cnqf.c b/drivers/platform/x86/amd/pmf/cnqf.c
new file mode 100644
index 000000000000..668c7c0fea83
--- /dev/null
+++ b/drivers/platform/x86/amd/pmf/cnqf.c
@@ -0,0 +1,395 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD Platform Management Framework Driver
+ *
+ * Copyright (c) 2022, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Author: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+ */
+
+#include <linux/workqueue.h>
+#include "pmf.h"
+
+static struct cnqf_config config_store;
+
+static int amd_pmf_set_cnqf(struct amd_pmf_dev *dev, int src, int idx,
+			    struct cnqf_config *table)
+{
+	struct power_table_control *pc;
+
+	pc = &config_store.mode_set[src][idx].power_control;
+
+	amd_pmf_send_cmd(dev, SET_SPL, false, pc->spl, NULL);
+	amd_pmf_send_cmd(dev, SET_FPPT, false, pc->fppt, NULL);
+	amd_pmf_send_cmd(dev, SET_SPPT, false, pc->sppt, NULL);
+	amd_pmf_send_cmd(dev, SET_SPPT_APU_ONLY, false, pc->sppt_apu_only, NULL);
+	amd_pmf_send_cmd(dev, SET_STT_MIN_LIMIT, false, pc->stt_min, NULL);
+	amd_pmf_send_cmd(dev, SET_STT_LIMIT_APU, false, pc->stt_skin_temp[STT_TEMP_APU],
+			 NULL);
+	amd_pmf_send_cmd(dev, SET_STT_LIMIT_HS2, false, pc->stt_skin_temp[STT_TEMP_HS2],
+			 NULL);
+
+	if (is_apmf_func_supported(dev, APMF_FUNC_SET_FAN_IDX))
+		apmf_update_fan_idx(dev,
+				    config_store.mode_set[src][idx].fan_control.manual,
+				    config_store.mode_set[src][idx].fan_control.fan_id);
+
+	return 0;
+}
+
+static void amd_pmf_update_power_threshold(int src)
+{
+	struct cnqf_mode_settings *ts;
+	struct cnqf_tran_params *tp;
+
+	tp = &config_store.trans_param[src][CNQF_TRANSITION_TO_QUIET];
+	ts = &config_store.mode_set[src][CNQF_MODE_BALANCE];
+	tp->power_threshold = ts->power_floor;
+
+	tp = &config_store.trans_param[src][CNQF_TRANSITION_TO_TURBO];
+	ts = &config_store.mode_set[src][CNQF_MODE_PERFORMANCE];
+	tp->power_threshold = ts->power_floor;
+
+	tp = &config_store.trans_param[src][CNQF_TRANSITION_FROM_BALANCE_TO_PERFORMANCE];
+	ts = &config_store.mode_set[src][CNQF_MODE_BALANCE];
+	tp->power_threshold = ts->power_floor;
+
+	tp = &config_store.trans_param[src][CNQF_TRANSITION_FROM_PERFORMANCE_TO_BALANCE];
+	ts = &config_store.mode_set[src][CNQF_MODE_PERFORMANCE];
+	tp->power_threshold = ts->power_floor;
+
+	tp = &config_store.trans_param[src][CNQF_TRANSITION_FROM_QUIET_TO_BALANCE];
+	ts = &config_store.mode_set[src][CNQF_MODE_QUIET];
+	tp->power_threshold = ts->power_floor;
+
+	tp = &config_store.trans_param[src][CNQF_TRANSITION_FROM_TURBO_TO_PERFORMANCE];
+	ts = &config_store.mode_set[src][CNQF_MODE_TURBO];
+	tp->power_threshold = ts->power_floor;
+}
+
+static const char *state_as_str(unsigned int state)
+{
+	switch (state) {
+	case CNQF_MODE_QUIET:
+		return "QUIET";
+	case CNQF_MODE_BALANCE:
+		return "BALANCED";
+	case CNQF_MODE_TURBO:
+		return "TURBO";
+	case CNQF_MODE_PERFORMANCE:
+		return "PERFORMANCE";
+	default:
+		return "Unknown CnQF mode";
+	}
+}
+
+static int amd_pmf_cnqf_get_power_source(struct amd_pmf_dev *dev)
+{
+	if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_AC) &&
+	    is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_DC))
+		return amd_pmf_get_power_source();
+	else if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_DC))
+		return POWER_SOURCE_DC;
+	else
+		return POWER_SOURCE_AC;
+}
+
+int amd_pmf_trans_cnqf(struct amd_pmf_dev *dev, int socket_power, ktime_t time_lapsed_ms)
+{
+	struct cnqf_tran_params *tp;
+	int src, i, j;
+	u32 avg_power = 0;
+
+	src = amd_pmf_cnqf_get_power_source(dev);
+
+	if (dev->current_profile == PLATFORM_PROFILE_BALANCED) {
+		amd_pmf_set_cnqf(dev, src, config_store.current_mode, NULL);
+	} else {
+		/*
+		 * Return from here if the platform_profile is not balanced
+		 * so that preference is given to user mode selection, rather
+		 * than enforcing CnQF to run all the time (if enabled)
+		 */
+		return -EINVAL;
+	}
+
+	for (i = 0; i < CNQF_TRANSITION_MAX; i++) {
+		config_store.trans_param[src][i].timer += time_lapsed_ms;
+		config_store.trans_param[src][i].total_power += socket_power;
+		config_store.trans_param[src][i].count++;
+
+		tp = &config_store.trans_param[src][i];
+		if (tp->timer >= tp->time_constant && tp->count) {
+			avg_power = tp->total_power / tp->count;
+
+			/* Reset the indices */
+			tp->timer = 0;
+			tp->total_power = 0;
+			tp->count = 0;
+
+			if ((tp->shifting_up && avg_power >= tp->power_threshold) ||
+			    (!tp->shifting_up && avg_power <= tp->power_threshold)) {
+				tp->priority = true;
+			} else {
+				tp->priority = false;
+			}
+		}
+	}
+
+	dev_dbg(dev->dev, "[CNQF] Avg power: %u mW socket power: %u mW mode:%s\n",
+		avg_power, socket_power, state_as_str(config_store.current_mode));
+
+	for (j = 0; j < CNQF_TRANSITION_MAX; j++) {
+		/* apply the highest priority */
+		if (config_store.trans_param[src][j].priority) {
+			if (config_store.current_mode !=
+			    config_store.trans_param[src][j].target_mode) {
+				config_store.current_mode =
+						config_store.trans_param[src][j].target_mode;
+				dev_dbg(dev->dev, "Moving to Mode :%s\n",
+					state_as_str(config_store.current_mode));
+				amd_pmf_set_cnqf(dev, src,
+						 config_store.current_mode, NULL);
+			}
+			break;
+		}
+	}
+	return 0;
+}
+
+static void amd_pmf_update_trans_data(int idx, struct apmf_dyn_slider_output out)
+{
+	struct cnqf_tran_params *tp;
+
+	tp = &config_store.trans_param[idx][CNQF_TRANSITION_TO_QUIET];
+	tp->time_constant = out.t_balanced_to_quiet;
+	tp->target_mode = CNQF_MODE_QUIET;
+	tp->shifting_up = false;
+
+	tp = &config_store.trans_param[idx][CNQF_TRANSITION_FROM_BALANCE_TO_PERFORMANCE];
+	tp->time_constant = out.t_balanced_to_perf;
+	tp->target_mode = CNQF_MODE_PERFORMANCE;
+	tp->shifting_up = true;
+
+	tp = &config_store.trans_param[idx][CNQF_TRANSITION_FROM_QUIET_TO_BALANCE];
+	tp->time_constant = out.t_quiet_to_balanced;
+	tp->target_mode = CNQF_MODE_BALANCE;
+	tp->shifting_up = true;
+
+	tp = &config_store.trans_param[idx][CNQF_TRANSITION_FROM_PERFORMANCE_TO_BALANCE];
+	tp->time_constant = out.t_perf_to_balanced;
+	tp->target_mode = CNQF_MODE_BALANCE;
+	tp->shifting_up = false;
+
+	tp = &config_store.trans_param[idx][CNQF_TRANSITION_FROM_TURBO_TO_PERFORMANCE];
+	tp->time_constant = out.t_turbo_to_perf;
+	tp->target_mode = CNQF_MODE_PERFORMANCE;
+	tp->shifting_up = false;
+
+	tp = &config_store.trans_param[idx][CNQF_TRANSITION_TO_TURBO];
+	tp->time_constant = out.t_perf_to_turbo;
+	tp->target_mode = CNQF_MODE_TURBO;
+	tp->shifting_up = true;
+}
+
+static void amd_pmf_update_mode_set(int idx, struct apmf_dyn_slider_output out)
+{
+	struct cnqf_mode_settings *ms;
+
+	/* Quiet Mode */
+	ms = &config_store.mode_set[idx][CNQF_MODE_QUIET];
+	ms->power_floor = out.ps[APMF_CNQF_QUIET].pfloor;
+	ms->power_control.fppt = out.ps[APMF_CNQF_QUIET].fppt;
+	ms->power_control.sppt = out.ps[APMF_CNQF_QUIET].sppt;
+	ms->power_control.sppt_apu_only = out.ps[APMF_CNQF_QUIET].sppt_apu_only;
+	ms->power_control.spl = out.ps[APMF_CNQF_QUIET].spl;
+	ms->power_control.stt_min = out.ps[APMF_CNQF_QUIET].stt_min_limit;
+	ms->power_control.stt_skin_temp[STT_TEMP_APU] =
+		out.ps[APMF_CNQF_QUIET].stt_skintemp[STT_TEMP_APU];
+	ms->power_control.stt_skin_temp[STT_TEMP_HS2] =
+		out.ps[APMF_CNQF_QUIET].stt_skintemp[STT_TEMP_HS2];
+	ms->fan_control.fan_id = out.ps[APMF_CNQF_QUIET].fan_id;
+
+	/* Balance Mode */
+	ms = &config_store.mode_set[idx][CNQF_MODE_BALANCE];
+	ms->power_floor = out.ps[APMF_CNQF_BALANCE].pfloor;
+	ms->power_control.fppt = out.ps[APMF_CNQF_BALANCE].fppt;
+	ms->power_control.sppt = out.ps[APMF_CNQF_BALANCE].sppt;
+	ms->power_control.sppt_apu_only = out.ps[APMF_CNQF_BALANCE].sppt_apu_only;
+	ms->power_control.spl = out.ps[APMF_CNQF_BALANCE].spl;
+	ms->power_control.stt_min = out.ps[APMF_CNQF_BALANCE].stt_min_limit;
+	ms->power_control.stt_skin_temp[STT_TEMP_APU] =
+		out.ps[APMF_CNQF_BALANCE].stt_skintemp[STT_TEMP_APU];
+	ms->power_control.stt_skin_temp[STT_TEMP_HS2] =
+		out.ps[APMF_CNQF_BALANCE].stt_skintemp[STT_TEMP_HS2];
+	ms->fan_control.fan_id = out.ps[APMF_CNQF_BALANCE].fan_id;
+
+	/* Performance Mode */
+	ms = &config_store.mode_set[idx][CNQF_MODE_PERFORMANCE];
+	ms->power_floor = out.ps[APMF_CNQF_PERFORMANCE].pfloor;
+	ms->power_control.fppt = out.ps[APMF_CNQF_PERFORMANCE].fppt;
+	ms->power_control.sppt = out.ps[APMF_CNQF_PERFORMANCE].sppt;
+	ms->power_control.sppt_apu_only = out.ps[APMF_CNQF_PERFORMANCE].sppt_apu_only;
+	ms->power_control.spl = out.ps[APMF_CNQF_PERFORMANCE].spl;
+	ms->power_control.stt_min = out.ps[APMF_CNQF_PERFORMANCE].stt_min_limit;
+	ms->power_control.stt_skin_temp[STT_TEMP_APU] =
+		out.ps[APMF_CNQF_PERFORMANCE].stt_skintemp[STT_TEMP_APU];
+	ms->power_control.stt_skin_temp[STT_TEMP_HS2] =
+		out.ps[APMF_CNQF_PERFORMANCE].stt_skintemp[STT_TEMP_HS2];
+	ms->fan_control.fan_id = out.ps[APMF_CNQF_PERFORMANCE].fan_id;
+
+	/* Turbo Mode */
+	ms = &config_store.mode_set[idx][CNQF_MODE_TURBO];
+	ms->power_floor = out.ps[APMF_CNQF_TURBO].pfloor;
+	ms->power_control.fppt = out.ps[APMF_CNQF_TURBO].fppt;
+	ms->power_control.sppt = out.ps[APMF_CNQF_TURBO].sppt;
+	ms->power_control.sppt_apu_only = out.ps[APMF_CNQF_TURBO].sppt_apu_only;
+	ms->power_control.spl = out.ps[APMF_CNQF_TURBO].spl;
+	ms->power_control.stt_min = out.ps[APMF_CNQF_TURBO].stt_min_limit;
+	ms->power_control.stt_skin_temp[STT_TEMP_APU] =
+		out.ps[APMF_CNQF_TURBO].stt_skintemp[STT_TEMP_APU];
+	ms->power_control.stt_skin_temp[STT_TEMP_HS2] =
+		out.ps[APMF_CNQF_TURBO].stt_skintemp[STT_TEMP_HS2];
+	ms->fan_control.fan_id = out.ps[APMF_CNQF_TURBO].fan_id;
+}
+
+static int amd_pmf_check_flags(struct amd_pmf_dev *dev)
+{
+	struct apmf_dyn_slider_output out = {};
+
+	if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_AC))
+		apmf_get_dyn_slider_def_ac(dev, &out);
+	else if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_DC))
+		apmf_get_dyn_slider_def_dc(dev, &out);
+
+	return out.flags;
+}
+
+static int amd_pmf_load_defaults_cnqf(struct amd_pmf_dev *dev)
+{
+	struct apmf_dyn_slider_output out;
+	int i, j, ret;
+
+	for (i = 0; i < POWER_SOURCE_MAX; i++) {
+		if (!is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_AC + i))
+			continue;
+
+		if (i == POWER_SOURCE_AC)
+			ret = apmf_get_dyn_slider_def_ac(dev, &out);
+		else
+			ret = apmf_get_dyn_slider_def_dc(dev, &out);
+		if (ret) {
+			dev_err(dev->dev, "APMF apmf_get_dyn_slider_def_dc failed :%d\n", ret);
+			return ret;
+		}
+
+		amd_pmf_update_mode_set(i, out);
+		amd_pmf_update_trans_data(i, out);
+		amd_pmf_update_power_threshold(i);
+
+		for (j = 0; j < CNQF_MODE_MAX; j++) {
+			if (config_store.mode_set[i][j].fan_control.fan_id == FAN_INDEX_AUTO)
+				config_store.mode_set[i][j].fan_control.manual = false;
+			else
+				config_store.mode_set[i][j].fan_control.manual = true;
+		}
+	}
+
+	/* set to initial default values */
+	config_store.current_mode = CNQF_MODE_BALANCE;
+
+	return 0;
+}
+
+static ssize_t cnqf_enable_store(struct device *dev,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	struct amd_pmf_dev *pdev = dev_get_drvdata(dev);
+	int mode, result, src;
+	bool input;
+
+	mode = amd_pmf_get_pprof_modes(pdev);
+	if (mode < 0)
+		return mode;
+
+	result = kstrtobool(buf, &input);
+	if (result)
+		return result;
+
+	src = amd_pmf_cnqf_get_power_source(pdev);
+	pdev->cnqf_enabled = input;
+
+	if (pdev->cnqf_enabled && pdev->current_profile == PLATFORM_PROFILE_BALANCED) {
+		amd_pmf_set_cnqf(pdev, src, config_store.current_mode, NULL);
+	} else {
+		if (is_apmf_func_supported(pdev, APMF_FUNC_STATIC_SLIDER_GRANULAR))
+			amd_pmf_update_slider(pdev, SLIDER_OP_SET, mode, NULL);
+	}
+
+	dev_dbg(pdev->dev, "Received CnQF %s\n", input ? "on" : "off");
+	return count;
+}
+
+static ssize_t cnqf_enable_show(struct device *dev,
+				struct device_attribute *attr,
+				char *buf)
+{
+	struct amd_pmf_dev *pdev = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%s\n", pdev->cnqf_enabled ? "on" : "off");
+}
+
+static DEVICE_ATTR_RW(cnqf_enable);
+
+static umode_t cnqf_feature_is_visible(struct kobject *kobj,
+				       struct attribute *attr, int n)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct amd_pmf_dev *pdev = dev_get_drvdata(dev);
+
+	return pdev->cnqf_supported ? attr->mode : 0;
+}
+
+static struct attribute *cnqf_feature_attrs[] = {
+	&dev_attr_cnqf_enable.attr,
+	NULL
+};
+
+const struct attribute_group cnqf_feature_attribute_group = {
+	.is_visible = cnqf_feature_is_visible,
+	.attrs = cnqf_feature_attrs,
+};
+
+void amd_pmf_deinit_cnqf(struct amd_pmf_dev *dev)
+{
+	cancel_delayed_work_sync(&dev->work_buffer);
+}
+
+int amd_pmf_init_cnqf(struct amd_pmf_dev *dev)
+{
+	int ret, src;
+
+	/*
+	 * Note the caller of this function has already checked that both
+	 * APMF_FUNC_DYN_SLIDER_AC and APMF_FUNC_DYN_SLIDER_DC are supported.
+	 */
+
+	ret = amd_pmf_load_defaults_cnqf(dev);
+	if (ret < 0)
+		return ret;
+
+	amd_pmf_init_metrics_table(dev);
+
+	dev->cnqf_supported = true;
+	dev->cnqf_enabled = amd_pmf_check_flags(dev);
+
+	/* update the thermal for CnQF */
+	if (dev->cnqf_enabled && dev->current_profile == PLATFORM_PROFILE_BALANCED) {
+		src = amd_pmf_cnqf_get_power_source(dev);
+		amd_pmf_set_cnqf(dev, src, config_store.current_mode, NULL);
+	}
+
+	return 0;
+}
diff --git a/drivers/platform/x86/amd/pmf/core.c b/drivers/platform/x86/amd/pmf/core.c
new file mode 100644
index 000000000000..a5f5a4bcff6d
--- /dev/null
+++ b/drivers/platform/x86/amd/pmf/core.c
@@ -0,0 +1,412 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * AMD Platform Management Framework Driver
+ *
+ * Copyright (c) 2022, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Author: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+ */
+
+#include <linux/debugfs.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/power_supply.h>
+#include "pmf.h"
+
+/* PMF-SMU communication registers */
+#define AMD_PMF_REGISTER_MESSAGE	0xA18
+#define AMD_PMF_REGISTER_RESPONSE	0xA78
+#define AMD_PMF_REGISTER_ARGUMENT	0xA58
+
+/* Base address of SMU for mapping physical address to virtual address */
+#define AMD_PMF_SMU_INDEX_ADDRESS	0xB8
+#define AMD_PMF_SMU_INDEX_DATA		0xBC
+#define AMD_PMF_MAPPING_SIZE		0x01000
+#define AMD_PMF_BASE_ADDR_OFFSET	0x10000
+#define AMD_PMF_BASE_ADDR_LO		0x13B102E8
+#define AMD_PMF_BASE_ADDR_HI		0x13B102EC
+#define AMD_PMF_BASE_ADDR_LO_MASK	GENMASK(15, 0)
+#define AMD_PMF_BASE_ADDR_HI_MASK	GENMASK(31, 20)
+
+/* SMU Response Codes */
+#define AMD_PMF_RESULT_OK                    0x01
+#define AMD_PMF_RESULT_CMD_REJECT_BUSY       0xFC
+#define AMD_PMF_RESULT_CMD_REJECT_PREREQ     0xFD
+#define AMD_PMF_RESULT_CMD_UNKNOWN           0xFE
+#define AMD_PMF_RESULT_FAILED                0xFF
+
+/* List of supported CPU ids */
+#define AMD_CPU_ID_RMB			0x14b5
+#define AMD_CPU_ID_PS			0x14e8
+
+#define PMF_MSG_DELAY_MIN_US		50
+#define RESPONSE_REGISTER_LOOP_MAX	20000
+
+#define DELAY_MIN_US	2000
+#define DELAY_MAX_US	3000
+
+/* override Metrics Table sample size time (in ms) */
+static int metrics_table_loop_ms = 1000;
+module_param(metrics_table_loop_ms, int, 0644);
+MODULE_PARM_DESC(metrics_table_loop_ms, "Metrics Table sample size time (default = 1000ms)");
+
+/* Force load on supported older platforms */
+static bool force_load;
+module_param(force_load, bool, 0444);
+MODULE_PARM_DESC(force_load, "Force load this driver on supported older platforms (experimental)");
+
+static int current_power_limits_show(struct seq_file *seq, void *unused)
+{
+	struct amd_pmf_dev *dev = seq->private;
+	struct amd_pmf_static_slider_granular table;
+	int mode, src = 0;
+
+	mode = amd_pmf_get_pprof_modes(dev);
+	if (mode < 0)
+		return mode;
+
+	src = amd_pmf_get_power_source();
+	amd_pmf_update_slider(dev, SLIDER_OP_GET, mode, &table);
+	seq_printf(seq, "spl:%u fppt:%u sppt:%u sppt_apu_only:%u stt_min:%u stt[APU]:%u stt[HS2]: %u\n",
+		   table.prop[src][mode].spl,
+		   table.prop[src][mode].fppt,
+		   table.prop[src][mode].sppt,
+		   table.prop[src][mode].sppt_apu_only,
+		   table.prop[src][mode].stt_min,
+		   table.prop[src][mode].stt_skin_temp[STT_TEMP_APU],
+		   table.prop[src][mode].stt_skin_temp[STT_TEMP_HS2]);
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(current_power_limits);
+
+static void amd_pmf_dbgfs_unregister(struct amd_pmf_dev *dev)
+{
+	debugfs_remove_recursive(dev->dbgfs_dir);
+}
+
+static void amd_pmf_dbgfs_register(struct amd_pmf_dev *dev)
+{
+	dev->dbgfs_dir = debugfs_create_dir("amd_pmf", NULL);
+	debugfs_create_file("current_power_limits", 0644, dev->dbgfs_dir, dev,
+			    &current_power_limits_fops);
+}
+
+int amd_pmf_get_power_source(void)
+{
+	if (power_supply_is_system_supplied() > 0)
+		return POWER_SOURCE_AC;
+	else
+		return POWER_SOURCE_DC;
+}
+
+static void amd_pmf_get_metrics(struct work_struct *work)
+{
+	struct amd_pmf_dev *dev = container_of(work, struct amd_pmf_dev, work_buffer.work);
+	ktime_t time_elapsed_ms;
+	int socket_power;
+
+	mutex_lock(&dev->update_mutex);
+	/* Transfer table contents */
+	memset(dev->buf, 0, sizeof(dev->m_table));
+	amd_pmf_send_cmd(dev, SET_TRANSFER_TABLE, 0, 7, NULL);
+	memcpy(&dev->m_table, dev->buf, sizeof(dev->m_table));
+
+	time_elapsed_ms = ktime_to_ms(ktime_get()) - dev->start_time;
+	/* Calculate the avg SoC power consumption */
+	socket_power = dev->m_table.apu_power + dev->m_table.dgpu_power;
+
+	if (dev->amt_enabled) {
+		/* Apply the Auto Mode transition */
+		amd_pmf_trans_automode(dev, socket_power, time_elapsed_ms);
+	}
+
+	if (dev->cnqf_enabled) {
+		/* Apply the CnQF transition */
+		amd_pmf_trans_cnqf(dev, socket_power, time_elapsed_ms);
+	}
+
+	dev->start_time = ktime_to_ms(ktime_get());
+	schedule_delayed_work(&dev->work_buffer, msecs_to_jiffies(metrics_table_loop_ms));
+	mutex_unlock(&dev->update_mutex);
+}
+
+static inline u32 amd_pmf_reg_read(struct amd_pmf_dev *dev, int reg_offset)
+{
+	return ioread32(dev->regbase + reg_offset);
+}
+
+static inline void amd_pmf_reg_write(struct amd_pmf_dev *dev, int reg_offset, u32 val)
+{
+	iowrite32(val, dev->regbase + reg_offset);
+}
+
+static void __maybe_unused amd_pmf_dump_registers(struct amd_pmf_dev *dev)
+{
+	u32 value;
+
+	value = amd_pmf_reg_read(dev, AMD_PMF_REGISTER_RESPONSE);
+	dev_dbg(dev->dev, "AMD_PMF_REGISTER_RESPONSE:%x\n", value);
+
+	value = amd_pmf_reg_read(dev, AMD_PMF_REGISTER_ARGUMENT);
+	dev_dbg(dev->dev, "AMD_PMF_REGISTER_ARGUMENT:%d\n", value);
+
+	value = amd_pmf_reg_read(dev, AMD_PMF_REGISTER_MESSAGE);
+	dev_dbg(dev->dev, "AMD_PMF_REGISTER_MESSAGE:%x\n", value);
+}
+
+int amd_pmf_send_cmd(struct amd_pmf_dev *dev, u8 message, bool get, u32 arg, u32 *data)
+{
+	int rc;
+	u32 val;
+
+	mutex_lock(&dev->lock);
+
+	/* Wait until we get a valid response */
+	rc = readx_poll_timeout(ioread32, dev->regbase + AMD_PMF_REGISTER_RESPONSE,
+				val, val != 0, PMF_MSG_DELAY_MIN_US,
+				PMF_MSG_DELAY_MIN_US * RESPONSE_REGISTER_LOOP_MAX);
+	if (rc) {
+		dev_err(dev->dev, "failed to talk to SMU\n");
+		goto out_unlock;
+	}
+
+	/* Write zero to response register */
+	amd_pmf_reg_write(dev, AMD_PMF_REGISTER_RESPONSE, 0);
+
+	/* Write argument into argument register */
+	amd_pmf_reg_write(dev, AMD_PMF_REGISTER_ARGUMENT, arg);
+
+	/* Write message ID to message ID register */
+	amd_pmf_reg_write(dev, AMD_PMF_REGISTER_MESSAGE, message);
+
+	/* Wait until we get a valid response */
+	rc = readx_poll_timeout(ioread32, dev->regbase + AMD_PMF_REGISTER_RESPONSE,
+				val, val != 0, PMF_MSG_DELAY_MIN_US,
+				PMF_MSG_DELAY_MIN_US * RESPONSE_REGISTER_LOOP_MAX);
+	if (rc) {
+		dev_err(dev->dev, "SMU response timed out\n");
+		goto out_unlock;
+	}
+
+	switch (val) {
+	case AMD_PMF_RESULT_OK:
+		if (get) {
+			/* PMFW may take longer time to return back the data */
+			usleep_range(DELAY_MIN_US, 10 * DELAY_MAX_US);
+			*data = amd_pmf_reg_read(dev, AMD_PMF_REGISTER_ARGUMENT);
+		}
+		break;
+	case AMD_PMF_RESULT_CMD_REJECT_BUSY:
+		dev_err(dev->dev, "SMU not ready. err: 0x%x\n", val);
+		rc = -EBUSY;
+		goto out_unlock;
+	case AMD_PMF_RESULT_CMD_UNKNOWN:
+		dev_err(dev->dev, "SMU cmd unknown. err: 0x%x\n", val);
+		rc = -EINVAL;
+		goto out_unlock;
+	case AMD_PMF_RESULT_CMD_REJECT_PREREQ:
+	case AMD_PMF_RESULT_FAILED:
+	default:
+		dev_err(dev->dev, "SMU cmd failed. err: 0x%x\n", val);
+		rc = -EIO;
+		goto out_unlock;
+	}
+
+out_unlock:
+	mutex_unlock(&dev->lock);
+	amd_pmf_dump_registers(dev);
+	return rc;
+}
+
+static const struct pci_device_id pmf_pci_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_RMB) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, AMD_CPU_ID_PS) },
+	{ }
+};
+
+int amd_pmf_init_metrics_table(struct amd_pmf_dev *dev)
+{
+	u64 phys_addr;
+	u32 hi, low;
+
+	INIT_DELAYED_WORK(&dev->work_buffer, amd_pmf_get_metrics);
+
+	/* Get Metrics Table Address */
+	dev->buf = kzalloc(sizeof(dev->m_table), GFP_KERNEL);
+	if (!dev->buf)
+		return -ENOMEM;
+
+	phys_addr = virt_to_phys(dev->buf);
+	hi = phys_addr >> 32;
+	low = phys_addr & GENMASK(31, 0);
+
+	amd_pmf_send_cmd(dev, SET_DRAM_ADDR_HIGH, 0, hi, NULL);
+	amd_pmf_send_cmd(dev, SET_DRAM_ADDR_LOW, 0, low, NULL);
+
+	/*
+	 * Start collecting the metrics data after a small delay
+	 * or else, we might end up getting stale values from PMFW.
+	 */
+	schedule_delayed_work(&dev->work_buffer, msecs_to_jiffies(metrics_table_loop_ms * 3));
+
+	return 0;
+}
+
+static void amd_pmf_init_features(struct amd_pmf_dev *dev)
+{
+	int ret;
+
+	/* Enable Static Slider */
+	if (is_apmf_func_supported(dev, APMF_FUNC_STATIC_SLIDER_GRANULAR)) {
+		amd_pmf_init_sps(dev);
+		dev_dbg(dev->dev, "SPS enabled and Platform Profiles registered\n");
+	}
+
+	/* Enable Auto Mode */
+	if (is_apmf_func_supported(dev, APMF_FUNC_AUTO_MODE)) {
+		amd_pmf_init_auto_mode(dev);
+		dev_dbg(dev->dev, "Auto Mode Init done\n");
+	} else if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_AC) ||
+			  is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_DC)) {
+		/* Enable Cool n Quiet Framework (CnQF) */
+		ret = amd_pmf_init_cnqf(dev);
+		if (ret)
+			dev_warn(dev->dev, "CnQF Init failed\n");
+	}
+}
+
+static void amd_pmf_deinit_features(struct amd_pmf_dev *dev)
+{
+	if (is_apmf_func_supported(dev, APMF_FUNC_STATIC_SLIDER_GRANULAR))
+		amd_pmf_deinit_sps(dev);
+
+	if (is_apmf_func_supported(dev, APMF_FUNC_AUTO_MODE)) {
+		amd_pmf_deinit_auto_mode(dev);
+	} else if (is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_AC) ||
+			  is_apmf_func_supported(dev, APMF_FUNC_DYN_SLIDER_DC)) {
+		amd_pmf_deinit_cnqf(dev);
+	}
+}
+
+static const struct acpi_device_id amd_pmf_acpi_ids[] = {
+	{"AMDI0100", 0x100},
+	{"AMDI0102", 0},
+	{ }
+};
+MODULE_DEVICE_TABLE(acpi, amd_pmf_acpi_ids);
+
+static int amd_pmf_probe(struct platform_device *pdev)
+{
+	const struct acpi_device_id *id;
+	struct amd_pmf_dev *dev;
+	struct pci_dev *rdev;
+	u32 base_addr_lo;
+	u32 base_addr_hi;
+	u64 base_addr;
+	u32 val;
+	int err;
+
+	id = acpi_match_device(amd_pmf_acpi_ids, &pdev->dev);
+	if (!id)
+		return -ENODEV;
+
+	if (id->driver_data == 0x100 && !force_load)
+		return -ENODEV;
+
+	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	dev->dev = &pdev->dev;
+
+	rdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0));
+	if (!rdev || !pci_match_id(pmf_pci_ids, rdev)) {
+		pci_dev_put(rdev);
+		return -ENODEV;
+	}
+
+	dev->cpu_id = rdev->device;
+	err = pci_write_config_dword(rdev, AMD_PMF_SMU_INDEX_ADDRESS, AMD_PMF_BASE_ADDR_LO);
+	if (err) {
+		dev_err(dev->dev, "error writing to 0x%x\n", AMD_PMF_SMU_INDEX_ADDRESS);
+		pci_dev_put(rdev);
+		return pcibios_err_to_errno(err);
+	}
+
+	err = pci_read_config_dword(rdev, AMD_PMF_SMU_INDEX_DATA, &val);
+	if (err) {
+		pci_dev_put(rdev);
+		return pcibios_err_to_errno(err);
+	}
+
+	base_addr_lo = val & AMD_PMF_BASE_ADDR_HI_MASK;
+
+	err = pci_write_config_dword(rdev, AMD_PMF_SMU_INDEX_ADDRESS, AMD_PMF_BASE_ADDR_HI);
+	if (err) {
+		dev_err(dev->dev, "error writing to 0x%x\n", AMD_PMF_SMU_INDEX_ADDRESS);
+		pci_dev_put(rdev);
+		return pcibios_err_to_errno(err);
+	}
+
+	err = pci_read_config_dword(rdev, AMD_PMF_SMU_INDEX_DATA, &val);
+	if (err) {
+		pci_dev_put(rdev);
+		return pcibios_err_to_errno(err);
+	}
+
+	base_addr_hi = val & AMD_PMF_BASE_ADDR_LO_MASK;
+	pci_dev_put(rdev);
+	base_addr = ((u64)base_addr_hi << 32 | base_addr_lo);
+
+	dev->regbase = devm_ioremap(dev->dev, base_addr + AMD_PMF_BASE_ADDR_OFFSET,
+				    AMD_PMF_MAPPING_SIZE);
+	if (!dev->regbase)
+		return -ENOMEM;
+
+	apmf_acpi_init(dev);
+	platform_set_drvdata(pdev, dev);
+	amd_pmf_init_features(dev);
+	apmf_install_handler(dev);
+	amd_pmf_dbgfs_register(dev);
+
+	mutex_init(&dev->lock);
+	mutex_init(&dev->update_mutex);
+	dev_info(dev->dev, "registered PMF device successfully\n");
+
+	return 0;
+}
+
+static int amd_pmf_remove(struct platform_device *pdev)
+{
+	struct amd_pmf_dev *dev = platform_get_drvdata(pdev);
+
+	mutex_destroy(&dev->lock);
+	mutex_destroy(&dev->update_mutex);
+	amd_pmf_deinit_features(dev);
+	apmf_acpi_deinit(dev);
+	amd_pmf_dbgfs_unregister(dev);
+	kfree(dev->buf);
+	return 0;
+}
+
+static const struct attribute_group *amd_pmf_driver_groups[] = {
+	&cnqf_feature_attribute_group,
+	NULL,
+};
+
+static struct platform_driver amd_pmf_driver = {
+	.driver = {
+		.name = "amd-pmf",
+		.acpi_match_table = amd_pmf_acpi_ids,
+		.dev_groups = amd_pmf_driver_groups,
+	},
+	.probe = amd_pmf_probe,
+	.remove = amd_pmf_remove,
+};
+module_platform_driver(amd_pmf_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("AMD Platform Management Framework Driver");
diff --git a/drivers/platform/x86/amd/pmf/pmf.h b/drivers/platform/x86/amd/pmf/pmf.h
new file mode 100644
index 000000000000..84bbe2c6ea61
--- /dev/null
+++ b/drivers/platform/x86/amd/pmf/pmf.h
@@ -0,0 +1,417 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * AMD Platform Management Framework Driver
+ *
+ * Copyright (c) 2022, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Author: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+ */
+
+#ifndef PMF_H
+#define PMF_H
+
+#include <linux/acpi.h>
+#include <linux/platform_profile.h>
+
+/* APMF Functions */
+#define APMF_FUNC_VERIFY_INTERFACE			0
+#define APMF_FUNC_GET_SYS_PARAMS			1
+#define APMF_FUNC_SBIOS_REQUESTS			2
+#define APMF_FUNC_SBIOS_HEARTBEAT			4
+#define APMF_FUNC_AUTO_MODE					5
+#define APMF_FUNC_SET_FAN_IDX				7
+#define APMF_FUNC_STATIC_SLIDER_GRANULAR       9
+#define APMF_FUNC_DYN_SLIDER_AC				11
+#define APMF_FUNC_DYN_SLIDER_DC				12
+
+/* Message Definitions */
+#define SET_SPL				0x03 /* SPL: Sustained Power Limit */
+#define SET_SPPT			0x05 /* SPPT: Slow Package Power Tracking */
+#define SET_FPPT			0x07 /* FPPT: Fast Package Power Tracking */
+#define GET_SPL				0x0B
+#define GET_SPPT			0x0D
+#define GET_FPPT			0x0F
+#define SET_DRAM_ADDR_HIGH	0x14
+#define SET_DRAM_ADDR_LOW	0x15
+#define SET_TRANSFER_TABLE	0x16
+#define SET_STT_MIN_LIMIT	0x18 /* STT: Skin Temperature Tracking */
+#define SET_STT_LIMIT_APU	0x19
+#define SET_STT_LIMIT_HS2	0x1A
+#define SET_SPPT_APU_ONLY	0x1D
+#define GET_SPPT_APU_ONLY	0x1E
+#define GET_STT_MIN_LIMIT	0x1F
+#define GET_STT_LIMIT_APU	0x20
+#define GET_STT_LIMIT_HS2	0x21
+
+/* Fan Index for Auto Mode */
+#define FAN_INDEX_AUTO		0xFFFFFFFF
+
+#define ARG_NONE 0
+#define AVG_SAMPLE_SIZE 3
+
+/* AMD PMF BIOS interfaces */
+struct apmf_verify_interface {
+	u16 size;
+	u16 version;
+	u32 notification_mask;
+	u32 supported_functions;
+} __packed;
+
+struct apmf_system_params {
+	u16 size;
+	u32 valid_mask;
+	u32 flags;
+	u8 command_code;
+	u32 heartbeat_int;
+} __packed;
+
+struct apmf_sbios_req {
+	u16 size;
+	u32 pending_req;
+	u8 rsd;
+	u8 cql_event;
+	u8 amt_event;
+	u32 fppt;
+	u32 sppt;
+	u32 fppt_apu_only;
+	u32 spl;
+	u32 stt_min_limit;
+	u8 skin_temp_apu;
+	u8 skin_temp_hs2;
+} __packed;
+
+struct apmf_fan_idx {
+	u16 size;
+	u8 fan_ctl_mode;
+	u32 fan_ctl_idx;
+} __packed;
+
+struct smu_pmf_metrics {
+	u16 gfxclk_freq; /* in MHz */
+	u16 socclk_freq; /* in MHz */
+	u16 vclk_freq; /* in MHz */
+	u16 dclk_freq; /* in MHz */
+	u16 memclk_freq; /* in MHz */
+	u16 spare;
+	u16 gfx_activity; /* in Centi */
+	u16 uvd_activity; /* in Centi */
+	u16 voltage[2]; /* in mV */
+	u16 currents[2]; /* in mA */
+	u16 power[2];/* in mW */
+	u16 core_freq[8]; /* in MHz */
+	u16 core_power[8]; /* in mW */
+	u16 core_temp[8]; /* in centi-Celsius */
+	u16 l3_freq; /* in MHz */
+	u16 l3_temp; /* in centi-Celsius */
+	u16 gfx_temp; /* in centi-Celsius */
+	u16 soc_temp; /* in centi-Celsius */
+	u16 throttler_status;
+	u16 current_socketpower; /* in mW */
+	u16 stapm_orig_limit; /* in W */
+	u16 stapm_cur_limit; /* in W */
+	u32 apu_power; /* in mW */
+	u32 dgpu_power; /* in mW */
+	u16 vdd_tdc_val; /* in mA */
+	u16 soc_tdc_val; /* in mA */
+	u16 vdd_edc_val; /* in mA */
+	u16 soc_edcv_al; /* in mA */
+	u16 infra_cpu_maxfreq; /* in MHz */
+	u16 infra_gfx_maxfreq; /* in MHz */
+	u16 skin_temp; /* in centi-Celsius */
+	u16 device_state;
+} __packed;
+
+enum amd_stt_skin_temp {
+	STT_TEMP_APU,
+	STT_TEMP_HS2,
+	STT_TEMP_COUNT,
+};
+
+enum amd_slider_op {
+	SLIDER_OP_GET,
+	SLIDER_OP_SET,
+};
+
+enum power_source {
+	POWER_SOURCE_AC,
+	POWER_SOURCE_DC,
+	POWER_SOURCE_MAX,
+};
+
+enum power_modes {
+	POWER_MODE_PERFORMANCE,
+	POWER_MODE_BALANCED_POWER,
+	POWER_MODE_POWER_SAVER,
+	POWER_MODE_MAX,
+};
+
+struct amd_pmf_dev {
+	void __iomem *regbase;
+	void __iomem *smu_virt_addr;
+	void *buf;
+	u32 base_addr;
+	u32 cpu_id;
+	struct device *dev;
+	struct mutex lock; /* protects the PMF interface */
+	u32 supported_func;
+	enum platform_profile_option current_profile;
+	struct platform_profile_handler pprof;
+	struct dentry *dbgfs_dir;
+	int hb_interval; /* SBIOS heartbeat interval */
+	struct delayed_work heart_beat;
+	struct smu_pmf_metrics m_table;
+	struct delayed_work work_buffer;
+	ktime_t start_time;
+	int socket_power_history[AVG_SAMPLE_SIZE];
+	int socket_power_history_idx;
+	bool amt_enabled;
+	struct mutex update_mutex; /* protects race between ACPI handler and metrics thread */
+	bool cnqf_enabled;
+	bool cnqf_supported;
+};
+
+struct apmf_sps_prop_granular {
+	u32 fppt;
+	u32 sppt;
+	u32 sppt_apu_only;
+	u32 spl;
+	u32 stt_min;
+	u8 stt_skin_temp[STT_TEMP_COUNT];
+	u32 fan_id;
+} __packed;
+
+/* Static Slider */
+struct apmf_static_slider_granular_output {
+	u16 size;
+	struct apmf_sps_prop_granular prop[POWER_SOURCE_MAX * POWER_MODE_MAX];
+} __packed;
+
+struct amd_pmf_static_slider_granular {
+	u16 size;
+	struct apmf_sps_prop_granular prop[POWER_SOURCE_MAX][POWER_MODE_MAX];
+};
+
+struct fan_table_control {
+	bool manual;
+	unsigned long fan_id;
+};
+
+struct power_table_control {
+	u32 spl;
+	u32 sppt;
+	u32 fppt;
+	u32 sppt_apu_only;
+	u32 stt_min;
+	u32 stt_skin_temp[STT_TEMP_COUNT];
+	u32 reserved[16];
+};
+
+/* Auto Mode Layer */
+enum auto_mode_transition_priority {
+	AUTO_TRANSITION_TO_PERFORMANCE, /* Any other mode to Performance Mode */
+	AUTO_TRANSITION_FROM_QUIET_TO_BALANCE, /* Quiet Mode to Balance Mode */
+	AUTO_TRANSITION_TO_QUIET, /* Any other mode to Quiet Mode */
+	AUTO_TRANSITION_FROM_PERFORMANCE_TO_BALANCE, /* Performance Mode to Balance Mode */
+	AUTO_TRANSITION_MAX,
+};
+
+enum auto_mode_mode {
+	AUTO_QUIET,
+	AUTO_BALANCE,
+	AUTO_PERFORMANCE_ON_LAP,
+	AUTO_PERFORMANCE,
+	AUTO_MODE_MAX,
+};
+
+struct auto_mode_trans_params {
+	u32 time_constant; /* minimum time required to switch to next mode */
+	u32 power_delta; /* delta power to shift mode */
+	u32 power_threshold;
+	u32 timer; /* elapsed time. if timer > TimeThreshold, it will move to next mode */
+	u32 applied;
+	enum auto_mode_mode target_mode;
+	u32 shifting_up;
+};
+
+struct auto_mode_mode_settings {
+	struct power_table_control power_control;
+	struct fan_table_control fan_control;
+	u32 power_floor;
+};
+
+struct auto_mode_mode_config {
+	struct auto_mode_trans_params transition[AUTO_TRANSITION_MAX];
+	struct auto_mode_mode_settings mode_set[AUTO_MODE_MAX];
+	enum auto_mode_mode current_mode;
+};
+
+struct apmf_auto_mode {
+	u16 size;
+	/* time constant */
+	u32 balanced_to_perf;
+	u32 perf_to_balanced;
+	u32 quiet_to_balanced;
+	u32 balanced_to_quiet;
+	/* power floor */
+	u32 pfloor_perf;
+	u32 pfloor_balanced;
+	u32 pfloor_quiet;
+	/* Power delta for mode change */
+	u32 pd_balanced_to_perf;
+	u32 pd_perf_to_balanced;
+	u32 pd_quiet_to_balanced;
+	u32 pd_balanced_to_quiet;
+	/* skin temperature limits */
+	u8 stt_apu_perf_on_lap; /* CQL ON */
+	u8 stt_hs2_perf_on_lap; /* CQL ON */
+	u8 stt_apu_perf;
+	u8 stt_hs2_perf;
+	u8 stt_apu_balanced;
+	u8 stt_hs2_balanced;
+	u8 stt_apu_quiet;
+	u8 stt_hs2_quiet;
+	u32 stt_min_limit_perf_on_lap; /* CQL ON */
+	u32 stt_min_limit_perf;
+	u32 stt_min_limit_balanced;
+	u32 stt_min_limit_quiet;
+	/* SPL based */
+	u32 fppt_perf_on_lap; /* CQL ON */
+	u32 sppt_perf_on_lap; /* CQL ON */
+	u32 spl_perf_on_lap; /* CQL ON */
+	u32 sppt_apu_only_perf_on_lap; /* CQL ON */
+	u32 fppt_perf;
+	u32 sppt_perf;
+	u32 spl_perf;
+	u32 sppt_apu_only_perf;
+	u32 fppt_balanced;
+	u32 sppt_balanced;
+	u32 spl_balanced;
+	u32 sppt_apu_only_balanced;
+	u32 fppt_quiet;
+	u32 sppt_quiet;
+	u32 spl_quiet;
+	u32 sppt_apu_only_quiet;
+	/* Fan ID */
+	u32 fan_id_perf;
+	u32 fan_id_balanced;
+	u32 fan_id_quiet;
+} __packed;
+
+/* CnQF Layer */
+enum cnqf_trans_priority {
+	CNQF_TRANSITION_TO_TURBO, /* Any other mode to Turbo Mode */
+	CNQF_TRANSITION_FROM_BALANCE_TO_PERFORMANCE, /* quiet/balance to Performance Mode */
+	CNQF_TRANSITION_FROM_QUIET_TO_BALANCE, /* Quiet Mode to Balance Mode */
+	CNQF_TRANSITION_TO_QUIET, /* Any other mode to Quiet Mode */
+	CNQF_TRANSITION_FROM_PERFORMANCE_TO_BALANCE, /* Performance/Turbo to Balance Mode */
+	CNQF_TRANSITION_FROM_TURBO_TO_PERFORMANCE, /* Turbo mode to Performance Mode */
+	CNQF_TRANSITION_MAX,
+};
+
+enum cnqf_mode {
+	CNQF_MODE_QUIET,
+	CNQF_MODE_BALANCE,
+	CNQF_MODE_PERFORMANCE,
+	CNQF_MODE_TURBO,
+	CNQF_MODE_MAX,
+};
+
+enum apmf_cnqf_pos {
+	APMF_CNQF_TURBO,
+	APMF_CNQF_PERFORMANCE,
+	APMF_CNQF_BALANCE,
+	APMF_CNQF_QUIET,
+	APMF_CNQF_MAX,
+};
+
+struct cnqf_mode_settings {
+	struct power_table_control power_control;
+	struct fan_table_control fan_control;
+	u32 power_floor;
+};
+
+struct cnqf_tran_params {
+	u32 time_constant; /* minimum time required to switch to next mode */
+	u32 power_threshold;
+	u32 timer; /* elapsed time. if timer > timethreshold, it will move to next mode */
+	u32 total_power;
+	u32 count;
+	bool priority;
+	bool shifting_up;
+	enum cnqf_mode target_mode;
+};
+
+struct cnqf_config {
+	struct cnqf_tran_params trans_param[POWER_SOURCE_MAX][CNQF_TRANSITION_MAX];
+	struct cnqf_mode_settings mode_set[POWER_SOURCE_MAX][CNQF_MODE_MAX];
+	struct power_table_control defaults;
+	enum cnqf_mode current_mode;
+	u32 power_src;
+	u32 avg_power;
+};
+
+struct apmf_cnqf_power_set {
+	u32 pfloor;
+	u32 fppt;
+	u32 sppt;
+	u32 sppt_apu_only;
+	u32 spl;
+	u32 stt_min_limit;
+	u8 stt_skintemp[STT_TEMP_COUNT];
+	u32 fan_id;
+} __packed;
+
+struct apmf_dyn_slider_output {
+	u16 size;
+	u16 flags;
+	u32 t_perf_to_turbo;
+	u32 t_balanced_to_perf;
+	u32 t_quiet_to_balanced;
+	u32 t_balanced_to_quiet;
+	u32 t_perf_to_balanced;
+	u32 t_turbo_to_perf;
+	struct apmf_cnqf_power_set ps[APMF_CNQF_MAX];
+} __packed;
+
+/* Core Layer */
+int apmf_acpi_init(struct amd_pmf_dev *pmf_dev);
+void apmf_acpi_deinit(struct amd_pmf_dev *pmf_dev);
+int is_apmf_func_supported(struct amd_pmf_dev *pdev, unsigned long index);
+int amd_pmf_send_cmd(struct amd_pmf_dev *dev, u8 message, bool get, u32 arg, u32 *data);
+int amd_pmf_init_metrics_table(struct amd_pmf_dev *dev);
+int amd_pmf_get_power_source(void);
+int apmf_install_handler(struct amd_pmf_dev *pmf_dev);
+
+/* SPS Layer */
+int amd_pmf_get_pprof_modes(struct amd_pmf_dev *pmf);
+void amd_pmf_update_slider(struct amd_pmf_dev *dev, bool op, int idx,
+			   struct amd_pmf_static_slider_granular *table);
+int amd_pmf_init_sps(struct amd_pmf_dev *dev);
+void amd_pmf_deinit_sps(struct amd_pmf_dev *dev);
+int apmf_get_static_slider_granular(struct amd_pmf_dev *pdev,
+				    struct apmf_static_slider_granular_output *output);
+
+
+int apmf_update_fan_idx(struct amd_pmf_dev *pdev, bool manual, u32 idx);
+
+/* Auto Mode Layer */
+int apmf_get_auto_mode_def(struct amd_pmf_dev *pdev, struct apmf_auto_mode *data);
+void amd_pmf_init_auto_mode(struct amd_pmf_dev *dev);
+void amd_pmf_deinit_auto_mode(struct amd_pmf_dev *dev);
+void amd_pmf_trans_automode(struct amd_pmf_dev *dev, int socket_power, ktime_t time_elapsed_ms);
+int apmf_get_sbios_requests(struct amd_pmf_dev *pdev, struct apmf_sbios_req *req);
+
+void amd_pmf_update_2_cql(struct amd_pmf_dev *dev, bool is_cql_event);
+int amd_pmf_reset_amt(struct amd_pmf_dev *dev);
+void amd_pmf_handle_amt(struct amd_pmf_dev *dev);
+
+/* CnQF Layer */
+int apmf_get_dyn_slider_def_ac(struct amd_pmf_dev *pdev, struct apmf_dyn_slider_output *data);
+int apmf_get_dyn_slider_def_dc(struct amd_pmf_dev *pdev, struct apmf_dyn_slider_output *data);
+int amd_pmf_init_cnqf(struct amd_pmf_dev *dev);
+void amd_pmf_deinit_cnqf(struct amd_pmf_dev *dev);
+int amd_pmf_trans_cnqf(struct amd_pmf_dev *dev, int socket_power, ktime_t time_lapsed_ms);
+extern const struct attribute_group cnqf_feature_attribute_group;
+
+#endif /* PMF_H */
diff --git a/drivers/platform/x86/amd/pmf/sps.c b/drivers/platform/x86/amd/pmf/sps.c
new file mode 100644
index 000000000000..dba7e36962dc
--- /dev/null
+++ b/drivers/platform/x86/amd/pmf/sps.c
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * AMD Platform Management Framework (PMF) Driver
+ *
+ * Copyright (c) 2022, Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Author: Shyam Sundar S K <Shyam-sundar.S-k@amd.com>
+ */
+
+#include "pmf.h"
+
+static struct amd_pmf_static_slider_granular config_store;
+
+static void amd_pmf_load_defaults_sps(struct amd_pmf_dev *dev)
+{
+	struct apmf_static_slider_granular_output output;
+	int i, j, idx = 0;
+
+	memset(&config_store, 0, sizeof(config_store));
+	apmf_get_static_slider_granular(dev, &output);
+
+	for (i = 0; i < POWER_SOURCE_MAX; i++) {
+		for (j = 0; j < POWER_MODE_MAX; j++) {
+			config_store.prop[i][j].spl = output.prop[idx].spl;
+			config_store.prop[i][j].sppt = output.prop[idx].sppt;
+			config_store.prop[i][j].sppt_apu_only =
+						output.prop[idx].sppt_apu_only;
+			config_store.prop[i][j].fppt = output.prop[idx].fppt;
+			config_store.prop[i][j].stt_min = output.prop[idx].stt_min;
+			config_store.prop[i][j].stt_skin_temp[STT_TEMP_APU] =
+					output.prop[idx].stt_skin_temp[STT_TEMP_APU];
+			config_store.prop[i][j].stt_skin_temp[STT_TEMP_HS2] =
+					output.prop[idx].stt_skin_temp[STT_TEMP_HS2];
+			config_store.prop[i][j].fan_id = output.prop[idx].fan_id;
+			idx++;
+		}
+	}
+}
+
+void amd_pmf_update_slider(struct amd_pmf_dev *dev, bool op, int idx,
+			   struct amd_pmf_static_slider_granular *table)
+{
+	int src = amd_pmf_get_power_source();
+
+	if (op == SLIDER_OP_SET) {
+		amd_pmf_send_cmd(dev, SET_SPL, false, config_store.prop[src][idx].spl, NULL);
+		amd_pmf_send_cmd(dev, SET_FPPT, false, config_store.prop[src][idx].fppt, NULL);
+		amd_pmf_send_cmd(dev, SET_SPPT, false, config_store.prop[src][idx].sppt, NULL);
+		amd_pmf_send_cmd(dev, SET_SPPT_APU_ONLY, false,
+				 config_store.prop[src][idx].sppt_apu_only, NULL);
+		amd_pmf_send_cmd(dev, SET_STT_MIN_LIMIT, false,
+				 config_store.prop[src][idx].stt_min, NULL);
+		amd_pmf_send_cmd(dev, SET_STT_LIMIT_APU, false,
+				 config_store.prop[src][idx].stt_skin_temp[STT_TEMP_APU], NULL);
+		amd_pmf_send_cmd(dev, SET_STT_LIMIT_HS2, false,
+				 config_store.prop[src][idx].stt_skin_temp[STT_TEMP_HS2], NULL);
+	} else if (op == SLIDER_OP_GET) {
+		amd_pmf_send_cmd(dev, GET_SPL, true, ARG_NONE, &table->prop[src][idx].spl);
+		amd_pmf_send_cmd(dev, GET_FPPT, true, ARG_NONE, &table->prop[src][idx].fppt);
+		amd_pmf_send_cmd(dev, GET_SPPT, true, ARG_NONE, &table->prop[src][idx].sppt);
+		amd_pmf_send_cmd(dev, GET_SPPT_APU_ONLY, true, ARG_NONE,
+				 &table->prop[src][idx].sppt_apu_only);
+		amd_pmf_send_cmd(dev, GET_STT_MIN_LIMIT, true, ARG_NONE,
+				 &table->prop[src][idx].stt_min);
+		amd_pmf_send_cmd(dev, GET_STT_LIMIT_APU, true, ARG_NONE,
+				 (u32 *)&table->prop[src][idx].stt_skin_temp[STT_TEMP_APU]);
+		amd_pmf_send_cmd(dev, GET_STT_LIMIT_HS2, true, ARG_NONE,
+				 (u32 *)&table->prop[src][idx].stt_skin_temp[STT_TEMP_HS2]);
+	}
+}
+
+static int amd_pmf_profile_get(struct platform_profile_handler *pprof,
+			       enum platform_profile_option *profile)
+{
+	struct amd_pmf_dev *pmf = container_of(pprof, struct amd_pmf_dev, pprof);
+
+	*profile = pmf->current_profile;
+	return 0;
+}
+
+int amd_pmf_get_pprof_modes(struct amd_pmf_dev *pmf)
+{
+	int mode;
+
+	switch (pmf->current_profile) {
+	case PLATFORM_PROFILE_PERFORMANCE:
+		mode = POWER_MODE_PERFORMANCE;
+		break;
+	case PLATFORM_PROFILE_BALANCED:
+		mode = POWER_MODE_BALANCED_POWER;
+		break;
+	case PLATFORM_PROFILE_LOW_POWER:
+		mode = POWER_MODE_POWER_SAVER;
+		break;
+	default:
+		dev_err(pmf->dev, "Unknown Platform Profile.\n");
+		return -EOPNOTSUPP;
+	}
+
+	return mode;
+}
+
+static int amd_pmf_profile_set(struct platform_profile_handler *pprof,
+			       enum platform_profile_option profile)
+{
+	struct amd_pmf_dev *pmf = container_of(pprof, struct amd_pmf_dev, pprof);
+	int mode;
+
+	pmf->current_profile = profile;
+	mode = amd_pmf_get_pprof_modes(pmf);
+	if (mode < 0)
+		return mode;
+
+	amd_pmf_update_slider(pmf, SLIDER_OP_SET, mode, NULL);
+	return 0;
+}
+
+int amd_pmf_init_sps(struct amd_pmf_dev *dev)
+{
+	int err;
+
+	dev->current_profile = PLATFORM_PROFILE_BALANCED;
+	amd_pmf_load_defaults_sps(dev);
+
+	dev->pprof.profile_get = amd_pmf_profile_get;
+	dev->pprof.profile_set = amd_pmf_profile_set;
+
+	/* Setup supported modes */
+	set_bit(PLATFORM_PROFILE_LOW_POWER, dev->pprof.choices);
+	set_bit(PLATFORM_PROFILE_BALANCED, dev->pprof.choices);
+	set_bit(PLATFORM_PROFILE_PERFORMANCE, dev->pprof.choices);
+
+	/* Create platform_profile structure and register */
+	err = platform_profile_register(&dev->pprof);
+	if (err)
+		dev_err(dev->dev, "Failed to register SPS support, this is most likely an SBIOS bug: %d\n",
+			err);
+
+	return err;
+}
+
+void amd_pmf_deinit_sps(struct amd_pmf_dev *dev)
+{
+	platform_profile_remove();
+}
diff --git a/drivers/platform/x86/amilo-rfkill.c b/drivers/platform/x86/amilo-rfkill.c
index 493e169c8f61..3e313c4d538d 100644
--- a/drivers/platform/x86/amilo-rfkill.c
+++ b/drivers/platform/x86/amilo-rfkill.c
@@ -150,7 +150,8 @@ static int __init amilo_rfkill_init(void)
 	if (rc)
 		return rc;
 
-	amilo_rfkill_pdev = platform_device_register_simple(KBUILD_MODNAME, -1,
+	amilo_rfkill_pdev = platform_device_register_simple(KBUILD_MODNAME,
+							    PLATFORM_DEVID_NONE,
 							    NULL, 0);
 	if (IS_ERR(amilo_rfkill_pdev)) {
 		rc = PTR_ERR(amilo_rfkill_pdev);
diff --git a/drivers/platform/x86/apple-gmux.c b/drivers/platform/x86/apple-gmux.c
index ffe98a18440b..ca33df7ea550 100644
--- a/drivers/platform/x86/apple-gmux.c
+++ b/drivers/platform/x86/apple-gmux.c
@@ -21,7 +21,6 @@
 #include <linux/delay.h>
 #include <linux/pci.h>
 #include <linux/vga_switcheroo.h>
-#include <acpi/video.h>
 #include <asm/io.h>
 
 /**
@@ -694,7 +693,6 @@ static int gmux_probe(struct pnp_dev *pnp, const struct pnp_device_id *id)
 	 * backlight control and supports more levels than other options.
 	 * Disable the other backlight choices.
 	 */
-	acpi_video_set_dmi_backlight_type(acpi_backlight_vendor);
 	apple_bl_unregister();
 
 	gmux_data->power_state = VGA_SWITCHEROO_ON;
@@ -804,7 +802,6 @@ static void gmux_remove(struct pnp_dev *pnp)
 	apple_gmux_data = NULL;
 	kfree(gmux_data);
 
-	acpi_video_register();
 	apple_bl_register();
 }
 
diff --git a/drivers/platform/x86/asus-laptop.c b/drivers/platform/x86/asus-laptop.c
index 4d2d32bfbe2a..47b2f8bb6fb5 100644
--- a/drivers/platform/x86/asus-laptop.c
+++ b/drivers/platform/x86/asus-laptop.c
@@ -1633,7 +1633,7 @@ static int asus_platform_init(struct asus_laptop *asus)
 {
 	int result;
 
-	asus->platform_device = platform_device_alloc(ASUS_LAPTOP_FILE, -1);
+	asus->platform_device = platform_device_alloc(ASUS_LAPTOP_FILE, PLATFORM_DEVID_NONE);
 	if (!asus->platform_device)
 		return -ENOMEM;
 	platform_set_drvdata(asus->platform_device, asus);
diff --git a/drivers/platform/x86/asus-nb-wmi.c b/drivers/platform/x86/asus-nb-wmi.c
index 478dd300b9c9..613c45c9fbe3 100644
--- a/drivers/platform/x86/asus-nb-wmi.c
+++ b/drivers/platform/x86/asus-nb-wmi.c
@@ -43,7 +43,7 @@ MODULE_PARM_DESC(wapf, "WAPF value");
 
 static int tablet_mode_sw = -1;
 module_param(tablet_mode_sw, uint, 0444);
-MODULE_PARM_DESC(tablet_mode_sw, "Tablet mode detect: -1:auto 0:disable 1:kbd-dock 2:lid-flip");
+MODULE_PARM_DESC(tablet_mode_sw, "Tablet mode detect: -1:auto 0:disable 1:kbd-dock 2:lid-flip 3:lid-flip-rog");
 
 static struct quirk_entry *quirks;
 
@@ -79,12 +79,10 @@ static struct quirk_entry quirk_asus_q500a = {
 
 /*
  * For those machines that need software to control bt/wifi status
- * and can't adjust brightness through ACPI interface
  * and have duplicate events(ACPI and WMI) for display toggle
  */
 static struct quirk_entry quirk_asus_x55u = {
 	.wapf = 4,
-	.wmi_backlight_power = true,
 	.wmi_backlight_set_devstate = true,
 	.no_display_toggle = true,
 };
@@ -99,11 +97,6 @@ static struct quirk_entry quirk_asus_x200ca = {
 	.wmi_backlight_set_devstate = true,
 };
 
-static struct quirk_entry quirk_asus_ux303ub = {
-	.wmi_backlight_native = true,
-	.wmi_backlight_set_devstate = true,
-};
-
 static struct quirk_entry quirk_asus_x550lb = {
 	.wmi_backlight_set_devstate = true,
 	.xusb2pr = 0x01D9,
@@ -115,12 +108,17 @@ static struct quirk_entry quirk_asus_forceals = {
 };
 
 static struct quirk_entry quirk_asus_use_kbd_dock_devid = {
-	.use_kbd_dock_devid = true,
+	.tablet_switch_mode = asus_wmi_kbd_dock_devid,
 };
 
 static struct quirk_entry quirk_asus_use_lid_flip_devid = {
 	.wmi_backlight_set_devstate = true,
-	.use_lid_flip_devid = true,
+	.tablet_switch_mode = asus_wmi_lid_flip_devid,
+};
+
+static struct quirk_entry quirk_asus_tablet_mode = {
+	.wmi_backlight_set_devstate = true,
+	.tablet_switch_mode = asus_wmi_lid_flip_rog_devid,
 };
 
 static int dmi_matched(const struct dmi_system_id *dmi)
@@ -147,11 +145,6 @@ static const struct dmi_system_id asus_quirks[] = {
 			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK Computer Inc."),
 			DMI_MATCH(DMI_PRODUCT_NAME, "U32U"),
 		},
-		/*
-		 * Note this machine has a Brazos APU, and most Brazos Asus
-		 * machines need quirk_asus_x55u / wmi_backlight_power but
-		 * here acpi-video seems to work fine for backlight control.
-		 */
 		.driver_data = &quirk_asus_wapf4,
 	},
 	{
@@ -381,15 +374,6 @@ static const struct dmi_system_id asus_quirks[] = {
 	},
 	{
 		.callback = dmi_matched,
-		.ident = "ASUSTeK COMPUTER INC. UX303UB",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "UX303UB"),
-		},
-		.driver_data = &quirk_asus_ux303ub,
-	},
-	{
-		.callback = dmi_matched,
 		.ident = "ASUSTeK COMPUTER INC. UX330UAK",
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
@@ -471,6 +455,15 @@ static const struct dmi_system_id asus_quirks[] = {
 		},
 		.driver_data = &quirk_asus_use_lid_flip_devid,
 	},
+	{
+		.callback = dmi_matched,
+		.ident = "ASUS ROG FLOW X13",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "GV301Q"),
+		},
+		.driver_data = &quirk_asus_tablet_mode,
+	},
 	{},
 };
 
@@ -490,20 +483,8 @@ static void asus_nb_wmi_quirks(struct asus_wmi_driver *driver)
 	else
 		wapf = quirks->wapf;
 
-	switch (tablet_mode_sw) {
-	case 0:
-		quirks->use_kbd_dock_devid = false;
-		quirks->use_lid_flip_devid = false;
-		break;
-	case 1:
-		quirks->use_kbd_dock_devid = true;
-		quirks->use_lid_flip_devid = false;
-		break;
-	case 2:
-		quirks->use_kbd_dock_devid = false;
-		quirks->use_lid_flip_devid = true;
-		break;
-	}
+	if (tablet_mode_sw != -1)
+		quirks->tablet_switch_mode = tablet_mode_sw;
 
 	if (quirks->i8042_filter) {
 		ret = i8042_install_filter(quirks->i8042_filter);
@@ -575,12 +556,14 @@ static const struct key_entry asus_nb_wmi_keymap[] = {
 	{ KE_KEY, 0xA5, { KEY_SWITCHVIDEOMODE } }, /* SDSP LCD + TV + HDMI */
 	{ KE_KEY, 0xA6, { KEY_SWITCHVIDEOMODE } }, /* SDSP CRT + TV + HDMI */
 	{ KE_KEY, 0xA7, { KEY_SWITCHVIDEOMODE } }, /* SDSP LCD + CRT + TV + HDMI */
+	{ KE_KEY, 0xAE, { KEY_FN_F5 } }, /* Fn+F5 fan mode on 2020+ */
 	{ KE_KEY, 0xB3, { KEY_PROG4 } }, /* AURA */
 	{ KE_KEY, 0xB5, { KEY_CALC } },
 	{ KE_KEY, 0xC4, { KEY_KBDILLUMUP } },
 	{ KE_KEY, 0xC5, { KEY_KBDILLUMDOWN } },
 	{ KE_IGNORE, 0xC6, },  /* Ambient Light Sensor notification */
 	{ KE_KEY, 0xFA, { KEY_PROG2 } },           /* Lid flip action */
+	{ KE_KEY, 0xBD, { KEY_PROG2 } },           /* Lid flip action on ROG xflow laptops */
 	{ KE_END, 0},
 };
 
diff --git a/drivers/platform/x86/asus-tf103c-dock.c b/drivers/platform/x86/asus-tf103c-dock.c
index 6fd0c9fea82d..62310e06282b 100644
--- a/drivers/platform/x86/asus-tf103c-dock.c
+++ b/drivers/platform/x86/asus-tf103c-dock.c
@@ -878,14 +878,12 @@ static int tf103c_dock_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int tf103c_dock_remove(struct i2c_client *client)
+static void tf103c_dock_remove(struct i2c_client *client)
 {
 	struct tf103c_dock_data *dock = i2c_get_clientdata(client);
 
 	tf103c_dock_stop_hpd(dock);
 	tf103c_dock_disable(dock);
-
-	return 0;
 }
 
 static int __maybe_unused tf103c_dock_suspend(struct device *dev)
diff --git a/drivers/platform/x86/asus-wmi.c b/drivers/platform/x86/asus-wmi.c
index eec7d0ed7cf2..6e8e093f96b3 100644
--- a/drivers/platform/x86/asus-wmi.c
+++ b/drivers/platform/x86/asus-wmi.c
@@ -68,9 +68,11 @@ module_param(fnlock_default, bool, 0444);
 #define NOTIFY_KBD_FBM			0x99
 #define NOTIFY_KBD_TTP			0xae
 #define NOTIFY_LID_FLIP			0xfa
+#define NOTIFY_LID_FLIP_ROG		0xbd
 
 #define ASUS_WMI_FNLOCK_BIOS_DISABLED	BIT(0)
 
+#define ASUS_GPU_FAN_DESC		"gpu_fan"
 #define ASUS_FAN_DESC			"cpu_fan"
 #define ASUS_FAN_MFUN			0x13
 #define ASUS_FAN_SFUN_READ		0x06
@@ -221,19 +223,25 @@ struct asus_wmi {
 	struct asus_rfkill gps;
 	struct asus_rfkill uwb;
 
+	int tablet_switch_event_code;
+	u32 tablet_switch_dev_id;
+
 	enum fan_type fan_type;
+	enum fan_type gpu_fan_type;
 	int fan_pwm_mode;
+	int gpu_fan_pwm_mode;
 	int agfn_pwm;
 
 	bool fan_boost_mode_available;
 	u8 fan_boost_mode_mask;
 	u8 fan_boost_mode;
 
-	bool egpu_enable_available; // 0 = enable
-	bool egpu_enable;
-
+	bool egpu_enable_available;
 	bool dgpu_disable_available;
-	bool dgpu_disable;
+	bool gpu_mux_mode_available;
+
+	bool kbd_rgb_mode_available;
+	bool kbd_rgb_state_available;
 
 	bool throttle_thermal_policy_available;
 	u8 throttle_thermal_policy_mode;
@@ -249,7 +257,6 @@ struct asus_wmi {
 	bool battery_rsoc_available;
 
 	bool panel_overdrive_available;
-	bool panel_overdrive;
 
 	struct hotplug_slot hotplug_slot;
 	struct mutex hotplug_lock;
@@ -486,10 +493,28 @@ static bool asus_wmi_dev_is_present(struct asus_wmi *asus, u32 dev_id)
 }
 
 /* Input **********************************************************************/
+static void asus_wmi_tablet_sw_init(struct asus_wmi *asus, u32 dev_id, int event_code)
+{
+	struct device *dev = &asus->platform_device->dev;
+	int result;
+
+	result = asus_wmi_get_devstate_simple(asus, dev_id);
+	if (result >= 0) {
+		input_set_capability(asus->inputdev, EV_SW, SW_TABLET_MODE);
+		input_report_switch(asus->inputdev, SW_TABLET_MODE, result);
+		asus->tablet_switch_dev_id = dev_id;
+		asus->tablet_switch_event_code = event_code;
+	} else if (result == -ENODEV) {
+		dev_err(dev, "This device has tablet-mode-switch quirk but got ENODEV checking it. This is a bug.");
+	} else {
+		dev_err(dev, "Error checking for tablet-mode-switch: %d\n", result);
+	}
+}
 
 static int asus_wmi_input_init(struct asus_wmi *asus)
 {
-	int err, result;
+	struct device *dev = &asus->platform_device->dev;
+	int err;
 
 	asus->inputdev = input_allocate_device();
 	if (!asus->inputdev)
@@ -498,35 +523,25 @@ static int asus_wmi_input_init(struct asus_wmi *asus)
 	asus->inputdev->name = asus->driver->input_name;
 	asus->inputdev->phys = asus->driver->input_phys;
 	asus->inputdev->id.bustype = BUS_HOST;
-	asus->inputdev->dev.parent = &asus->platform_device->dev;
+	asus->inputdev->dev.parent = dev;
 	set_bit(EV_REP, asus->inputdev->evbit);
 
 	err = sparse_keymap_setup(asus->inputdev, asus->driver->keymap, NULL);
 	if (err)
 		goto err_free_dev;
 
-	if (asus->driver->quirks->use_kbd_dock_devid) {
-		result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_KBD_DOCK);
-		if (result >= 0) {
-			input_set_capability(asus->inputdev, EV_SW, SW_TABLET_MODE);
-			input_report_switch(asus->inputdev, SW_TABLET_MODE, !result);
-		} else if (result != -ENODEV) {
-			pr_err("Error checking for keyboard-dock: %d\n", result);
-		}
-	}
-
-	if (asus->driver->quirks->use_lid_flip_devid) {
-		result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_LID_FLIP);
-		if (result < 0)
-			asus->driver->quirks->use_lid_flip_devid = 0;
-		if (result >= 0) {
-			input_set_capability(asus->inputdev, EV_SW, SW_TABLET_MODE);
-			input_report_switch(asus->inputdev, SW_TABLET_MODE, result);
-		} else if (result == -ENODEV) {
-			pr_err("This device has lid_flip quirk but got ENODEV checking it. This is a bug.");
-		} else {
-			pr_err("Error checking for lid-flip: %d\n", result);
-		}
+	switch (asus->driver->quirks->tablet_switch_mode) {
+	case asus_wmi_no_tablet_switch:
+		break;
+	case asus_wmi_kbd_dock_devid:
+		asus_wmi_tablet_sw_init(asus, ASUS_WMI_DEVID_KBD_DOCK, NOTIFY_KBD_DOCK_CHANGE);
+		break;
+	case asus_wmi_lid_flip_devid:
+		asus_wmi_tablet_sw_init(asus, ASUS_WMI_DEVID_LID_FLIP, NOTIFY_LID_FLIP);
+		break;
+	case asus_wmi_lid_flip_rog_devid:
+		asus_wmi_tablet_sw_init(asus, ASUS_WMI_DEVID_LID_FLIP_ROG, NOTIFY_LID_FLIP_ROG);
+		break;
 	}
 
 	err = input_register_device(asus->inputdev);
@@ -550,10 +565,14 @@ static void asus_wmi_input_exit(struct asus_wmi *asus)
 
 /* Tablet mode ****************************************************************/
 
-static void lid_flip_tablet_mode_get_state(struct asus_wmi *asus)
+static void asus_wmi_tablet_mode_get_state(struct asus_wmi *asus)
 {
-	int result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_LID_FLIP);
+	int result;
+
+	if (!asus->tablet_switch_dev_id)
+		return;
 
+	result = asus_wmi_get_devstate_simple(asus, asus->tablet_switch_dev_id);
 	if (result >= 0) {
 		input_report_switch(asus->inputdev, SW_TABLET_MODE, result);
 		input_sync(asus->inputdev);
@@ -561,179 +580,267 @@ static void lid_flip_tablet_mode_get_state(struct asus_wmi *asus)
 }
 
 /* dGPU ********************************************************************/
-static int dgpu_disable_check_present(struct asus_wmi *asus)
+static ssize_t dgpu_disable_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
 {
-	u32 result;
-	int err;
-
-	asus->dgpu_disable_available = false;
-
-	err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_DGPU, &result);
-	if (err) {
-		if (err == -ENODEV)
-			return 0;
-		return err;
-	}
+	struct asus_wmi *asus = dev_get_drvdata(dev);
+	int result;
 
-	if (result & ASUS_WMI_DSTS_PRESENCE_BIT) {
-		asus->dgpu_disable_available = true;
-		asus->dgpu_disable = result & ASUS_WMI_DSTS_STATUS_BIT;
-	}
+	result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_DGPU);
+	if (result < 0)
+		return result;
 
-	return 0;
+	return sysfs_emit(buf, "%d\n", result);
 }
 
-static int dgpu_disable_write(struct asus_wmi *asus)
+/*
+ * A user may be required to store the value twice, typcial store first, then
+ * rescan PCI bus to activate power, then store a second time to save correctly.
+ * The reason for this is that an extra code path in the ACPI is enabled when
+ * the device and bus are powered.
+ */
+static ssize_t dgpu_disable_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
 {
-	u32 retval;
-	u8 value;
-	int err;
+	int result, err;
+	u32 disable;
+
+	struct asus_wmi *asus = dev_get_drvdata(dev);
 
-	/* Don't rely on type conversion */
-	value = asus->dgpu_disable ? 1 : 0;
+	result = kstrtou32(buf, 10, &disable);
+	if (result)
+		return result;
+
+	if (disable > 1)
+		return -EINVAL;
 
-	err = asus_wmi_set_devstate(ASUS_WMI_DEVID_DGPU, value, &retval);
+	err = asus_wmi_set_devstate(ASUS_WMI_DEVID_DGPU, disable, &result);
 	if (err) {
 		pr_warn("Failed to set dgpu disable: %d\n", err);
 		return err;
 	}
 
-	if (retval > 1) {
-		pr_warn("Failed to set dgpu disable (retval): 0x%x\n", retval);
+	if (result > 1) {
+		pr_warn("Failed to set dgpu disable (result): 0x%x\n", result);
 		return -EIO;
 	}
 
 	sysfs_notify(&asus->platform_device->dev.kobj, NULL, "dgpu_disable");
 
-	return 0;
+	return count;
 }
+static DEVICE_ATTR_RW(dgpu_disable);
 
-static ssize_t dgpu_disable_show(struct device *dev,
+/* eGPU ********************************************************************/
+static ssize_t egpu_enable_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct asus_wmi *asus = dev_get_drvdata(dev);
-	u8 mode = asus->dgpu_disable;
+	int result;
 
-	return sysfs_emit(buf, "%d\n", mode);
+	result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_EGPU);
+	if (result < 0)
+		return result;
+
+	return sysfs_emit(buf, "%d\n", result);
 }
 
-/*
- * A user may be required to store the value twice, typcial store first, then
- * rescan PCI bus to activate power, then store a second time to save correctly.
- * The reason for this is that an extra code path in the ACPI is enabled when
- * the device and bus are powered.
- */
-static ssize_t dgpu_disable_store(struct device *dev,
+/* The ACPI call to enable the eGPU also disables the internal dGPU */
+static ssize_t egpu_enable_store(struct device *dev,
 				    struct device_attribute *attr,
 				    const char *buf, size_t count)
 {
-	bool disable;
-	int result;
+	int result, err;
+	u32 enable;
 
 	struct asus_wmi *asus = dev_get_drvdata(dev);
 
-	result = kstrtobool(buf, &disable);
-	if (result)
-		return result;
+	err = kstrtou32(buf, 10, &enable);
+	if (err)
+		return err;
 
-	asus->dgpu_disable = disable;
+	if (enable > 1)
+		return -EINVAL;
 
-	result = dgpu_disable_write(asus);
-	if (result)
-		return result;
+	err = asus_wmi_set_devstate(ASUS_WMI_DEVID_EGPU, enable, &result);
+	if (err) {
+		pr_warn("Failed to set egpu disable: %d\n", err);
+		return err;
+	}
+
+	if (result > 1) {
+		pr_warn("Failed to set egpu disable (retval): 0x%x\n", result);
+		return -EIO;
+	}
+
+	sysfs_notify(&asus->platform_device->dev.kobj, NULL, "egpu_enable");
 
 	return count;
 }
+static DEVICE_ATTR_RW(egpu_enable);
 
-static DEVICE_ATTR_RW(dgpu_disable);
+/* gpu mux switch *************************************************************/
+static ssize_t gpu_mux_mode_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	struct asus_wmi *asus = dev_get_drvdata(dev);
+	int result;
 
-/* eGPU ********************************************************************/
-static int egpu_enable_check_present(struct asus_wmi *asus)
+	result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_GPU_MUX);
+	if (result < 0)
+		return result;
+
+	return sysfs_emit(buf, "%d\n", result);
+}
+
+static ssize_t gpu_mux_mode_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t count)
 {
-	u32 result;
-	int err;
+	struct asus_wmi *asus = dev_get_drvdata(dev);
+	int result, err;
+	u32 optimus;
 
-	asus->egpu_enable_available = false;
+	err = kstrtou32(buf, 10, &optimus);
+	if (err)
+		return err;
+
+	if (optimus > 1)
+		return -EINVAL;
 
-	err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_EGPU, &result);
+	err = asus_wmi_set_devstate(ASUS_WMI_DEVID_GPU_MUX, optimus, &result);
 	if (err) {
-		if (err == -ENODEV)
-			return 0;
+		dev_err(dev, "Failed to set GPU MUX mode: %d\n", err);
 		return err;
 	}
-
-	if (result & ASUS_WMI_DSTS_PRESENCE_BIT) {
-		asus->egpu_enable_available = true;
-		asus->egpu_enable = result & ASUS_WMI_DSTS_STATUS_BIT;
+	/* !1 is considered a fail by ASUS */
+	if (result != 1) {
+		dev_warn(dev, "Failed to set GPU MUX mode (result): 0x%x\n", result);
+		return -EIO;
 	}
 
-	return 0;
+	sysfs_notify(&asus->platform_device->dev.kobj, NULL, "gpu_mux_mode");
+
+	return count;
 }
+static DEVICE_ATTR_RW(gpu_mux_mode);
 
-static int egpu_enable_write(struct asus_wmi *asus)
+/* TUF Laptop Keyboard RGB Modes **********************************************/
+static ssize_t kbd_rgb_mode_store(struct device *dev,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count)
 {
-	u32 retval;
-	u8 value;
+	u32 cmd, mode, r, g,  b,  speed;
 	int err;
 
-	/* Don't rely on type conversion */
-	value = asus->egpu_enable ? 1 : 0;
+	if (sscanf(buf, "%d %d %d %d %d %d", &cmd, &mode, &r, &g, &b, &speed) != 6)
+		return -EINVAL;
 
-	err = asus_wmi_set_devstate(ASUS_WMI_DEVID_EGPU, value, &retval);
+	cmd = !!cmd;
 
-	if (err) {
-		pr_warn("Failed to set egpu disable: %d\n", err);
-		return err;
-	}
+	/* These are the known usable modes across all TUF/ROG */
+	if (mode >= 12 || mode == 9)
+		mode = 10;
 
-	if (retval > 1) {
-		pr_warn("Failed to set egpu disable (retval): 0x%x\n", retval);
-		return -EIO;
+	switch (speed) {
+	case 0:
+		speed = 0xe1;
+		break;
+	case 1:
+		speed = 0xeb;
+		break;
+	case 2:
+		speed = 0xf5;
+		break;
+	default:
+		speed = 0xeb;
 	}
 
-	sysfs_notify(&asus->platform_device->dev.kobj, NULL, "egpu_enable");
+	err = asus_wmi_evaluate_method3(ASUS_WMI_METHODID_DEVS, ASUS_WMI_DEVID_TUF_RGB_MODE,
+			cmd | (mode << 8) | (r << 16) | (g << 24), b | (speed << 8), NULL);
+	if (err)
+		return err;
 
-	return 0;
+	return count;
 }
+static DEVICE_ATTR_WO(kbd_rgb_mode);
 
-static ssize_t egpu_enable_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
+static ssize_t kbd_rgb_mode_index_show(struct device *device,
+						 struct device_attribute *attr,
+						 char *buf)
 {
-	struct asus_wmi *asus = dev_get_drvdata(dev);
-	bool mode = asus->egpu_enable;
-
-	return sysfs_emit(buf, "%d\n", mode);
+	return sysfs_emit(buf, "%s\n", "cmd mode red green blue speed");
 }
+static DEVICE_ATTR_RO(kbd_rgb_mode_index);
 
-/* The ACPI call to enable the eGPU also disables the internal dGPU */
-static ssize_t egpu_enable_store(struct device *dev,
-				    struct device_attribute *attr,
-				    const char *buf, size_t count)
-{
-	bool enable;
-	int result;
-
-	struct asus_wmi *asus = dev_get_drvdata(dev);
+static struct attribute *kbd_rgb_mode_attrs[] = {
+	&dev_attr_kbd_rgb_mode.attr,
+	&dev_attr_kbd_rgb_mode_index.attr,
+	NULL,
+};
 
-	result = kstrtobool(buf, &enable);
-	if (result)
-		return result;
+static const struct attribute_group kbd_rgb_mode_group = {
+	.attrs = kbd_rgb_mode_attrs,
+};
 
-	asus->egpu_enable = enable;
+/* TUF Laptop Keyboard RGB State **********************************************/
+static ssize_t kbd_rgb_state_store(struct device *dev,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	u32 flags, cmd, boot, awake, sleep, keyboard;
+	int err;
 
-	result = egpu_enable_write(asus);
-	if (result)
-		return result;
+	if (sscanf(buf, "%d %d %d %d %d", &cmd, &boot, &awake, &sleep, &keyboard) != 5)
+		return -EINVAL;
 
-	/* Ensure that the kernel status of dgpu is updated */
-	result = dgpu_disable_check_present(asus);
-	if (result)
-		return result;
+	if (cmd)
+		cmd = BIT(2);
+
+	flags = 0;
+	if (boot)
+		flags |= BIT(1);
+	if (awake)
+		flags |= BIT(3);
+	if (sleep)
+		flags |= BIT(5);
+	if (keyboard)
+		flags |= BIT(7);
+
+	/* 0xbd is the required default arg0 for the method. Nothing happens otherwise */
+	err = asus_wmi_evaluate_method3(ASUS_WMI_METHODID_DEVS,
+			ASUS_WMI_DEVID_TUF_RGB_STATE, 0xbd | cmd << 8 | (flags << 16), 0, NULL);
+	if (err)
+		return err;
 
 	return count;
 }
+static DEVICE_ATTR_WO(kbd_rgb_state);
 
-static DEVICE_ATTR_RW(egpu_enable);
+static ssize_t kbd_rgb_state_index_show(struct device *device,
+						 struct device_attribute *attr,
+						 char *buf)
+{
+	return sysfs_emit(buf, "%s\n", "cmd boot awake sleep keyboard");
+}
+static DEVICE_ATTR_RO(kbd_rgb_state_index);
+
+static struct attribute *kbd_rgb_state_attrs[] = {
+	&dev_attr_kbd_rgb_state.attr,
+	&dev_attr_kbd_rgb_state_index.attr,
+	NULL,
+};
+
+static const struct attribute_group kbd_rgb_state_group = {
+	.attrs = kbd_rgb_state_attrs,
+};
+
+static const struct attribute_group *kbd_rgb_mode_groups[] = {
+	NULL,
+	NULL,
+	NULL,
+};
 
 /* Battery ********************************************************************/
 
@@ -771,7 +878,7 @@ static ssize_t charge_control_end_threshold_show(struct device *device,
 						 struct device_attribute *attr,
 						 char *buf)
 {
-	return sprintf(buf, "%d\n", charge_end_threshold);
+	return sysfs_emit(buf, "%d\n", charge_end_threshold);
 }
 
 static DEVICE_ATTR_RW(charge_control_end_threshold);
@@ -1053,7 +1160,12 @@ static void asus_wmi_led_exit(struct asus_wmi *asus)
 
 static int asus_wmi_led_init(struct asus_wmi *asus)
 {
-	int rv = 0, led_val;
+	int rv = 0, num_rgb_groups = 0, led_val;
+
+	if (asus->kbd_rgb_mode_available)
+		kbd_rgb_mode_groups[num_rgb_groups++] = &kbd_rgb_mode_group;
+	if (asus->kbd_rgb_state_available)
+		kbd_rgb_mode_groups[num_rgb_groups++] = &kbd_rgb_state_group;
 
 	asus->led_workqueue = create_singlethread_workqueue("led_workqueue");
 	if (!asus->led_workqueue)
@@ -1081,6 +1193,9 @@ static int asus_wmi_led_init(struct asus_wmi *asus)
 		asus->kbd_led.brightness_get = kbd_led_get;
 		asus->kbd_led.max_brightness = 3;
 
+		if (num_rgb_groups != 0)
+			asus->kbd_led.groups = kbd_rgb_mode_groups;
+
 		rv = led_classdev_register(&asus->platform_device->dev,
 					   &asus->kbd_led);
 		if (rv)
@@ -1555,84 +1670,51 @@ exit:
 }
 
 /* Panel Overdrive ************************************************************/
-static int panel_od_check_present(struct asus_wmi *asus)
-{
-	u32 result;
-	int err;
-
-	asus->panel_overdrive_available = false;
-
-	err = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_PANEL_OD, &result);
-	if (err) {
-		if (err == -ENODEV)
-			return 0;
-		return err;
-	}
-
-	if (result & ASUS_WMI_DSTS_PRESENCE_BIT) {
-		asus->panel_overdrive_available = true;
-		asus->panel_overdrive = result & ASUS_WMI_DSTS_STATUS_BIT;
-	}
-
-	return 0;
-}
-
-static int panel_od_write(struct asus_wmi *asus)
-{
-	u32 retval;
-	u8 value;
-	int err;
-
-	/* Don't rely on type conversion */
-	value = asus->panel_overdrive ? 1 : 0;
-
-	err = asus_wmi_set_devstate(ASUS_WMI_DEVID_PANEL_OD, value, &retval);
-
-	if (err) {
-		pr_warn("Failed to set panel overdrive: %d\n", err);
-		return err;
-	}
-
-	if (retval > 1) {
-		pr_warn("Failed to set panel overdrive (retval): 0x%x\n", retval);
-		return -EIO;
-	}
-
-	sysfs_notify(&asus->platform_device->dev.kobj, NULL, "panel_od");
-
-	return 0;
-}
-
 static ssize_t panel_od_show(struct device *dev,
 				   struct device_attribute *attr, char *buf)
 {
 	struct asus_wmi *asus = dev_get_drvdata(dev);
+	int result;
+
+	result = asus_wmi_get_devstate_simple(asus, ASUS_WMI_DEVID_PANEL_OD);
+	if (result < 0)
+		return result;
 
-	return sysfs_emit(buf, "%d\n", asus->panel_overdrive);
+	return sysfs_emit(buf, "%d\n", result);
 }
 
 static ssize_t panel_od_store(struct device *dev,
 				    struct device_attribute *attr,
 				    const char *buf, size_t count)
 {
-	bool overdrive;
-	int result;
+	int result, err;
+	u32 overdrive;
 
 	struct asus_wmi *asus = dev_get_drvdata(dev);
 
-	result = kstrtobool(buf, &overdrive);
+	result = kstrtou32(buf, 10, &overdrive);
 	if (result)
 		return result;
 
-	asus->panel_overdrive = overdrive;
-	result = panel_od_write(asus);
+	if (overdrive > 1)
+		return -EINVAL;
 
-	if (result)
-		return result;
+	err = asus_wmi_set_devstate(ASUS_WMI_DEVID_PANEL_OD, overdrive, &result);
+
+	if (err) {
+		pr_warn("Failed to set panel overdrive: %d\n", err);
+		return err;
+	}
+
+	if (result > 1) {
+		pr_warn("Failed to set panel overdrive (result): 0x%x\n", result);
+		return -EIO;
+	}
+
+	sysfs_notify(&asus->platform_device->dev.kobj, NULL, "panel_od");
 
 	return count;
 }
-
 static DEVICE_ATTR_RW(panel_od);
 
 /* Quirks *********************************************************************/
@@ -1782,6 +1864,18 @@ static int asus_fan_set_auto(struct asus_wmi *asus)
 		return -ENXIO;
 	}
 
+	/*
+	 * Modern models like the G713 also have GPU fan control (this is not AGFN)
+	 */
+	if (asus->gpu_fan_type == FAN_TYPE_SPEC83) {
+		status = asus_wmi_set_devstate(ASUS_WMI_DEVID_GPU_FAN_CTRL,
+					       0, &retval);
+		if (status)
+			return status;
+
+		if (retval != 1)
+			return -EIO;
+	}
 
 	return 0;
 }
@@ -1819,7 +1913,7 @@ static ssize_t pwm1_show(struct device *dev,
 		value = -1;
 	}
 
-	return sprintf(buf, "%d\n", value);
+	return sysfs_emit(buf, "%d\n", value);
 }
 
 static ssize_t pwm1_store(struct device *dev,
@@ -1879,7 +1973,7 @@ static ssize_t fan1_input_show(struct device *dev,
 		return -ENXIO;
 	}
 
-	return sprintf(buf, "%d\n", value < 0 ? -1 : value*100);
+	return sysfs_emit(buf, "%d\n", value < 0 ? -1 : value * 100);
 }
 
 static ssize_t pwm1_enable_show(struct device *dev,
@@ -1897,7 +1991,7 @@ static ssize_t pwm1_enable_show(struct device *dev,
 	 * in practice on X532FL at least (the bit is always 0) and there's
 	 * also nothing in the DSDT to indicate that this behaviour exists.
 	 */
-	return sprintf(buf, "%d\n", asus->fan_pwm_mode);
+	return sysfs_emit(buf, "%d\n", asus->fan_pwm_mode);
 }
 
 static ssize_t pwm1_enable_store(struct device *dev,
@@ -1965,7 +2059,7 @@ static ssize_t fan1_label_show(struct device *dev,
 					  struct device_attribute *attr,
 					  char *buf)
 {
-	return sprintf(buf, "%s\n", ASUS_FAN_DESC);
+	return sysfs_emit(buf, "%s\n", ASUS_FAN_DESC);
 }
 
 static ssize_t asus_hwmon_temp1(struct device *dev,
@@ -1984,11 +2078,86 @@ static ssize_t asus_hwmon_temp1(struct device *dev,
 		       deci_kelvin_to_millicelsius(value & 0xFFFF));
 }
 
+/* GPU fan on modern ROG laptops */
+static ssize_t fan2_input_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct asus_wmi *asus = dev_get_drvdata(dev);
+	int value;
+	int ret;
+
+	ret = asus_wmi_get_devstate(asus, ASUS_WMI_DEVID_GPU_FAN_CTRL, &value);
+	if (ret < 0)
+		return ret;
+
+	value &= 0xffff;
+
+	return sysfs_emit(buf, "%d\n", value * 100);
+}
+
+static ssize_t fan2_label_show(struct device *dev,
+					  struct device_attribute *attr,
+					  char *buf)
+{
+	return sysfs_emit(buf, "%s\n", ASUS_GPU_FAN_DESC);
+}
+
+static ssize_t pwm2_enable_show(struct device *dev,
+				struct device_attribute *attr,
+				char *buf)
+{
+	struct asus_wmi *asus = dev_get_drvdata(dev);
+
+	return sysfs_emit(buf, "%d\n", asus->gpu_fan_pwm_mode);
+}
+
+static ssize_t pwm2_enable_store(struct device *dev,
+				 struct device_attribute *attr,
+				 const char *buf, size_t count)
+{
+	struct asus_wmi *asus = dev_get_drvdata(dev);
+	int state;
+	int value;
+	int ret;
+	u32 retval;
+
+	ret = kstrtouint(buf, 10, &state);
+	if (ret)
+		return ret;
+
+	switch (state) { /* standard documented hwmon values */
+	case ASUS_FAN_CTRL_FULLSPEED:
+		value = 1;
+		break;
+	case ASUS_FAN_CTRL_AUTO:
+		value = 0;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ret = asus_wmi_set_devstate(ASUS_WMI_DEVID_GPU_FAN_CTRL,
+				    value, &retval);
+	if (ret)
+		return ret;
+
+	if (retval != 1)
+		return -EIO;
+
+	asus->gpu_fan_pwm_mode = state;
+	return count;
+}
+
 /* Fan1 */
 static DEVICE_ATTR_RW(pwm1);
 static DEVICE_ATTR_RW(pwm1_enable);
 static DEVICE_ATTR_RO(fan1_input);
 static DEVICE_ATTR_RO(fan1_label);
+/* Fan2 - GPU fan */
+static DEVICE_ATTR_RW(pwm2_enable);
+static DEVICE_ATTR_RO(fan2_input);
+static DEVICE_ATTR_RO(fan2_label);
 
 /* Temperature */
 static DEVICE_ATTR(temp1_input, S_IRUGO, asus_hwmon_temp1, NULL);
@@ -1996,8 +2165,11 @@ static DEVICE_ATTR(temp1_input, S_IRUGO, asus_hwmon_temp1, NULL);
 static struct attribute *hwmon_attributes[] = {
 	&dev_attr_pwm1.attr,
 	&dev_attr_pwm1_enable.attr,
+	&dev_attr_pwm2_enable.attr,
 	&dev_attr_fan1_input.attr,
 	&dev_attr_fan1_label.attr,
+	&dev_attr_fan2_input.attr,
+	&dev_attr_fan2_label.attr,
 
 	&dev_attr_temp1_input.attr,
 	NULL
@@ -2006,7 +2178,7 @@ static struct attribute *hwmon_attributes[] = {
 static umode_t asus_hwmon_sysfs_is_visible(struct kobject *kobj,
 					  struct attribute *attr, int idx)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct asus_wmi *asus = dev_get_drvdata(dev->parent);
 	u32 value = ASUS_WMI_UNSUPPORTED_METHOD;
 
@@ -2018,6 +2190,11 @@ static umode_t asus_hwmon_sysfs_is_visible(struct kobject *kobj,
 	    || attr == &dev_attr_pwm1_enable.attr) {
 		if (asus->fan_type == FAN_TYPE_NONE)
 			return 0;
+	} else if (attr == &dev_attr_fan2_input.attr
+	    || attr == &dev_attr_fan2_label.attr
+	    || attr == &dev_attr_pwm2_enable.attr) {
+		if (asus->gpu_fan_type == FAN_TYPE_NONE)
+			return 0;
 	} else if (attr == &dev_attr_temp1_input.attr) {
 		int err = asus_wmi_get_devstate(asus,
 						ASUS_WMI_DEVID_THERMAL_CTRL,
@@ -2060,6 +2237,7 @@ static int asus_wmi_hwmon_init(struct asus_wmi *asus)
 
 static int asus_wmi_fan_init(struct asus_wmi *asus)
 {
+	asus->gpu_fan_type = FAN_TYPE_NONE;
 	asus->fan_type = FAN_TYPE_NONE;
 	asus->agfn_pwm = -1;
 
@@ -2068,6 +2246,10 @@ static int asus_wmi_fan_init(struct asus_wmi *asus)
 	else if (asus_wmi_has_agfn_fan(asus))
 		asus->fan_type = FAN_TYPE_AGFN;
 
+	/*  Modern models like G713 also have GPU fan control */
+	if (asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_GPU_FAN_CTRL))
+		asus->gpu_fan_type = FAN_TYPE_SPEC83;
+
 	if (asus->fan_type == FAN_TYPE_NONE)
 		return -ENODEV;
 
@@ -2158,7 +2340,7 @@ static ssize_t fan_boost_mode_show(struct device *dev,
 {
 	struct asus_wmi *asus = dev_get_drvdata(dev);
 
-	return scnprintf(buf, PAGE_SIZE, "%d\n", asus->fan_boost_mode);
+	return sysfs_emit(buf, "%d\n", asus->fan_boost_mode);
 }
 
 static ssize_t fan_boost_mode_store(struct device *dev,
@@ -2710,7 +2892,7 @@ static ssize_t throttle_thermal_policy_show(struct device *dev,
 	struct asus_wmi *asus = dev_get_drvdata(dev);
 	u8 mode = asus->throttle_thermal_policy_mode;
 
-	return scnprintf(buf, PAGE_SIZE, "%d\n", mode);
+	return sysfs_emit(buf, "%d\n", mode);
 }
 
 static ssize_t throttle_thermal_policy_store(struct device *dev,
@@ -3062,9 +3244,7 @@ static void asus_wmi_handle_event_code(int code, struct asus_wmi *asus)
 {
 	unsigned int key_value = 1;
 	bool autorelease = 1;
-	int result, orig_code;
-
-	orig_code = code;
+	int orig_code = code;
 
 	if (asus->driver->key_filter) {
 		asus->driver->key_filter(asus->driver, &code, &key_value,
@@ -3107,30 +3287,18 @@ static void asus_wmi_handle_event_code(int code, struct asus_wmi *asus)
 		return;
 	}
 
-	if (asus->driver->quirks->use_kbd_dock_devid && code == NOTIFY_KBD_DOCK_CHANGE) {
-		result = asus_wmi_get_devstate_simple(asus,
-						      ASUS_WMI_DEVID_KBD_DOCK);
-		if (result >= 0) {
-			input_report_switch(asus->inputdev, SW_TABLET_MODE,
-					    !result);
-			input_sync(asus->inputdev);
-		}
-		return;
-	}
-
-	if (asus->driver->quirks->use_lid_flip_devid && code == NOTIFY_LID_FLIP) {
-		lid_flip_tablet_mode_get_state(asus);
+	if (code == asus->tablet_switch_event_code) {
+		asus_wmi_tablet_mode_get_state(asus);
 		return;
 	}
 
-	if (asus->fan_boost_mode_available && code == NOTIFY_KBD_FBM) {
-		fan_boost_mode_switch_next(asus);
+	if (code == NOTIFY_KBD_FBM || code == NOTIFY_KBD_TTP) {
+		if (asus->fan_boost_mode_available)
+			fan_boost_mode_switch_next(asus);
+		if (asus->throttle_thermal_policy_available)
+			throttle_thermal_policy_switch_next(asus);
 		return;
-	}
 
-	if (asus->throttle_thermal_policy_available && code == NOTIFY_KBD_TTP) {
-		throttle_thermal_policy_switch_next(asus);
-		return;
 	}
 
 	if (is_display_toggle(code) && asus->driver->quirks->no_display_toggle)
@@ -3282,6 +3450,7 @@ static struct attribute *platform_attributes[] = {
 	&dev_attr_touchpad.attr,
 	&dev_attr_egpu_enable.attr,
 	&dev_attr_dgpu_disable.attr,
+	&dev_attr_gpu_mux_mode.attr,
 	&dev_attr_lid_resume.attr,
 	&dev_attr_als_enable.attr,
 	&dev_attr_fan_boost_mode.attr,
@@ -3293,7 +3462,7 @@ static struct attribute *platform_attributes[] = {
 static umode_t asus_sysfs_is_visible(struct kobject *kobj,
 				    struct attribute *attr, int idx)
 {
-	struct device *dev = container_of(kobj, struct device, kobj);
+	struct device *dev = kobj_to_dev(kobj);
 	struct asus_wmi *asus = dev_get_drvdata(dev);
 	bool ok = true;
 	int devid = -1;
@@ -3312,6 +3481,8 @@ static umode_t asus_sysfs_is_visible(struct kobject *kobj,
 		ok = asus->egpu_enable_available;
 	else if (attr == &dev_attr_dgpu_disable.attr)
 		ok = asus->dgpu_disable_available;
+	else if (attr == &dev_attr_gpu_mux_mode.attr)
+		ok = asus->gpu_mux_mode_available;
 	else if (attr == &dev_attr_fan_boost_mode.attr)
 		ok = asus->fan_boost_mode_available;
 	else if (attr == &dev_attr_throttle_thermal_policy.attr)
@@ -3552,7 +3723,6 @@ static int asus_wmi_add(struct platform_device *pdev)
 	struct platform_driver *pdrv = to_platform_driver(pdev->dev.driver);
 	struct asus_wmi_driver *wdrv = to_asus_wmi_driver(pdrv);
 	struct asus_wmi *asus;
-	const char *chassis_type;
 	acpi_status status;
 	int err;
 	u32 result;
@@ -3573,13 +3743,12 @@ static int asus_wmi_add(struct platform_device *pdev)
 	if (err)
 		goto fail_platform;
 
-	err = egpu_enable_check_present(asus);
-	if (err)
-		goto fail_egpu_enable;
-
-	err = dgpu_disable_check_present(asus);
-	if (err)
-		goto fail_dgpu_disable;
+	asus->egpu_enable_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_EGPU);
+	asus->dgpu_disable_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_DGPU);
+	asus->gpu_mux_mode_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_GPU_MUX);
+	asus->kbd_rgb_mode_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_MODE);
+	asus->kbd_rgb_state_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_TUF_RGB_STATE);
+	asus->panel_overdrive_available = asus_wmi_dev_is_present(asus, ASUS_WMI_DEVID_PANEL_OD);
 
 	err = fan_boost_mode_check_present(asus);
 	if (err)
@@ -3595,10 +3764,6 @@ static int asus_wmi_add(struct platform_device *pdev)
 	if (err)
 		goto fail_platform_profile_setup;
 
-	err = panel_od_check_present(asus);
-	if (err)
-		goto fail_panel_od;
-
 	err = asus_wmi_sysfs_init(asus->platform_device);
 	if (err)
 		goto fail_sysfs;
@@ -3634,18 +3799,6 @@ static int asus_wmi_add(struct platform_device *pdev)
 	if (asus->driver->quirks->wmi_force_als_set)
 		asus_wmi_set_als();
 
-	/* Some Asus desktop boards export an acpi-video backlight interface,
-	   stop this from showing up */
-	chassis_type = dmi_get_system_info(DMI_CHASSIS_TYPE);
-	if (chassis_type && !strcmp(chassis_type, "3"))
-		acpi_video_set_dmi_backlight_type(acpi_backlight_vendor);
-
-	if (asus->driver->quirks->wmi_backlight_power)
-		acpi_video_set_dmi_backlight_type(acpi_backlight_vendor);
-
-	if (asus->driver->quirks->wmi_backlight_native)
-		acpi_video_set_dmi_backlight_type(acpi_backlight_native);
-
 	if (asus->driver->quirks->xusb2pr)
 		asus_wmi_set_xusb2pr(asus);
 
@@ -3693,10 +3846,7 @@ fail_platform_profile_setup:
 	if (asus->platform_profile_support)
 		platform_profile_remove();
 fail_fan_boost_mode:
-fail_egpu_enable:
-fail_dgpu_disable:
 fail_platform:
-fail_panel_od:
 	kfree(asus);
 	return err;
 }
@@ -3755,9 +3905,7 @@ static int asus_hotk_resume(struct device *device)
 	if (asus_wmi_has_fnlock_key(asus))
 		asus_wmi_fnlock_update(asus);
 
-	if (asus->driver->quirks->use_lid_flip_devid)
-		lid_flip_tablet_mode_get_state(asus);
-
+	asus_wmi_tablet_mode_get_state(asus);
 	return 0;
 }
 
@@ -3797,9 +3945,7 @@ static int asus_hotk_restore(struct device *device)
 	if (asus_wmi_has_fnlock_key(asus))
 		asus_wmi_fnlock_update(asus);
 
-	if (asus->driver->quirks->use_lid_flip_devid)
-		lid_flip_tablet_mode_get_state(asus);
-
+	asus_wmi_tablet_mode_get_state(asus);
 	return 0;
 }
 
diff --git a/drivers/platform/x86/asus-wmi.h b/drivers/platform/x86/asus-wmi.h
index b302415bf1d9..65316998b898 100644
--- a/drivers/platform/x86/asus-wmi.h
+++ b/drivers/platform/x86/asus-wmi.h
@@ -25,16 +25,20 @@ struct module;
 struct key_entry;
 struct asus_wmi;
 
+enum asus_wmi_tablet_switch_mode {
+	asus_wmi_no_tablet_switch,
+	asus_wmi_kbd_dock_devid,
+	asus_wmi_lid_flip_devid,
+	asus_wmi_lid_flip_rog_devid,
+};
+
 struct quirk_entry {
 	bool hotplug_wireless;
 	bool scalar_panel_brightness;
 	bool store_backlight_power;
-	bool wmi_backlight_power;
-	bool wmi_backlight_native;
 	bool wmi_backlight_set_devstate;
 	bool wmi_force_als_set;
-	bool use_kbd_dock_devid;
-	bool use_lid_flip_devid;
+	enum asus_wmi_tablet_switch_mode tablet_switch_mode;
 	int wapf;
 	/*
 	 * For machines with AMD graphic chips, it will send out WMI event
diff --git a/drivers/platform/x86/compal-laptop.c b/drivers/platform/x86/compal-laptop.c
index 0942f50bd793..e10d2f64dfad 100644
--- a/drivers/platform/x86/compal-laptop.c
+++ b/drivers/platform/x86/compal-laptop.c
@@ -721,16 +721,6 @@ static struct attribute *compal_hwmon_attrs[] = {
 };
 ATTRIBUTE_GROUPS(compal_hwmon);
 
-static int compal_probe(struct platform_device *);
-static int compal_remove(struct platform_device *);
-static struct platform_driver compal_driver = {
-	.driver = {
-		.name = DRIVER_NAME,
-	},
-	.probe	= compal_probe,
-	.remove	= compal_remove,
-};
-
 static enum power_supply_property compal_bat_properties[] = {
 	POWER_SUPPLY_PROP_STATUS,
 	POWER_SUPPLY_PROP_HEALTH,
@@ -965,6 +955,80 @@ err_wifi:
 	return ret;
 }
 
+static int compal_probe(struct platform_device *pdev)
+{
+	int err;
+	struct compal_data *data;
+	struct device *hwmon_dev;
+	struct power_supply_config psy_cfg = {};
+
+	if (!extra_features)
+		return 0;
+
+	/* Fan control */
+	data = devm_kzalloc(&pdev->dev, sizeof(struct compal_data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	initialize_fan_control_data(data);
+
+	err = sysfs_create_group(&pdev->dev.kobj, &compal_platform_attr_group);
+	if (err)
+		return err;
+
+	hwmon_dev = devm_hwmon_device_register_with_groups(&pdev->dev,
+							   "compal", data,
+							   compal_hwmon_groups);
+	if (IS_ERR(hwmon_dev)) {
+		err = PTR_ERR(hwmon_dev);
+		goto remove;
+	}
+
+	/* Power supply */
+	initialize_power_supply_data(data);
+	psy_cfg.drv_data = data;
+	data->psy = power_supply_register(&compal_device->dev, &psy_bat_desc,
+					  &psy_cfg);
+	if (IS_ERR(data->psy)) {
+		err = PTR_ERR(data->psy);
+		goto remove;
+	}
+
+	platform_set_drvdata(pdev, data);
+
+	return 0;
+
+remove:
+	sysfs_remove_group(&pdev->dev.kobj, &compal_platform_attr_group);
+	return err;
+}
+
+static int compal_remove(struct platform_device *pdev)
+{
+	struct compal_data *data;
+
+	if (!extra_features)
+		return 0;
+
+	pr_info("Unloading: resetting fan control to motherboard\n");
+	pwm_disable_control();
+
+	data = platform_get_drvdata(pdev);
+	power_supply_unregister(data->psy);
+
+	sysfs_remove_group(&pdev->dev.kobj, &compal_platform_attr_group);
+
+	return 0;
+}
+
+static struct platform_driver compal_driver = {
+	.driver = {
+		.name = DRIVER_NAME,
+	},
+	.probe	= compal_probe,
+	.remove	= compal_remove,
+};
+
 static int __init compal_init(void)
 {
 	int ret;
@@ -996,7 +1060,7 @@ static int __init compal_init(void)
 	if (ret)
 		goto err_backlight;
 
-	compal_device = platform_device_alloc(DRIVER_NAME, -1);
+	compal_device = platform_device_alloc(DRIVER_NAME, PLATFORM_DEVID_NONE);
 	if (!compal_device) {
 		ret = -ENOMEM;
 		goto err_platform_driver;
@@ -1028,54 +1092,6 @@ err_backlight:
 	return ret;
 }
 
-static int compal_probe(struct platform_device *pdev)
-{
-	int err;
-	struct compal_data *data;
-	struct device *hwmon_dev;
-	struct power_supply_config psy_cfg = {};
-
-	if (!extra_features)
-		return 0;
-
-	/* Fan control */
-	data = devm_kzalloc(&pdev->dev, sizeof(struct compal_data), GFP_KERNEL);
-	if (!data)
-		return -ENOMEM;
-
-	initialize_fan_control_data(data);
-
-	err = sysfs_create_group(&pdev->dev.kobj, &compal_platform_attr_group);
-	if (err)
-		return err;
-
-	hwmon_dev = devm_hwmon_device_register_with_groups(&pdev->dev,
-							   "compal", data,
-							   compal_hwmon_groups);
-	if (IS_ERR(hwmon_dev)) {
-		err = PTR_ERR(hwmon_dev);
-		goto remove;
-	}
-
-	/* Power supply */
-	initialize_power_supply_data(data);
-	psy_cfg.drv_data = data;
-	data->psy = power_supply_register(&compal_device->dev, &psy_bat_desc,
-					  &psy_cfg);
-	if (IS_ERR(data->psy)) {
-		err = PTR_ERR(data->psy);
-		goto remove;
-	}
-
-	platform_set_drvdata(pdev, data);
-
-	return 0;
-
-remove:
-	sysfs_remove_group(&pdev->dev.kobj, &compal_platform_attr_group);
-	return err;
-}
-
 static void __exit compal_cleanup(void)
 {
 	platform_device_unregister(compal_device);
@@ -1089,25 +1105,6 @@ static void __exit compal_cleanup(void)
 	pr_info("Driver unloaded\n");
 }
 
-static int compal_remove(struct platform_device *pdev)
-{
-	struct compal_data *data;
-
-	if (!extra_features)
-		return 0;
-
-	pr_info("Unloading: resetting fan control to motherboard\n");
-	pwm_disable_control();
-
-	data = platform_get_drvdata(pdev);
-	power_supply_unregister(data->psy);
-
-	sysfs_remove_group(&pdev->dev.kobj, &compal_platform_attr_group);
-
-	return 0;
-}
-
-
 module_init(compal_init);
 module_exit(compal_cleanup);
 
diff --git a/drivers/platform/x86/dell/alienware-wmi.c b/drivers/platform/x86/dell/alienware-wmi.c
index f21248255529..a34e07ef2c79 100644
--- a/drivers/platform/x86/dell/alienware-wmi.c
+++ b/drivers/platform/x86/dell/alienware-wmi.c
@@ -791,7 +791,7 @@ static int __init alienware_wmi_init(void)
 	ret = platform_driver_register(&platform_driver);
 	if (ret)
 		goto fail_platform_driver;
-	platform_device = platform_device_alloc("alienware-wmi", -1);
+	platform_device = platform_device_alloc("alienware-wmi", PLATFORM_DEVID_NONE);
 	if (!platform_device) {
 		ret = -ENOMEM;
 		goto fail_platform_device1;
diff --git a/drivers/platform/x86/dell/dcdbas.c b/drivers/platform/x86/dell/dcdbas.c
index 42beafbc54b2..0ecb7b164750 100644
--- a/drivers/platform/x86/dell/dcdbas.c
+++ b/drivers/platform/x86/dell/dcdbas.c
@@ -716,7 +716,7 @@ static struct platform_driver dcdbas_driver = {
 
 static const struct platform_device_info dcdbas_dev_info __initconst = {
 	.name		= DRIVER_NAME,
-	.id		= -1,
+	.id		= PLATFORM_DEVID_NONE,
 	.dma_mask	= DMA_BIT_MASK(32),
 };
 
diff --git a/drivers/platform/x86/dell/dell-laptop.c b/drivers/platform/x86/dell/dell-laptop.c
index 1321687d923e..e92c3ad06d69 100644
--- a/drivers/platform/x86/dell/dell-laptop.c
+++ b/drivers/platform/x86/dell/dell-laptop.c
@@ -2193,7 +2193,7 @@ static int __init dell_init(void)
 	ret = platform_driver_register(&platform_driver);
 	if (ret)
 		goto fail_platform_driver;
-	platform_device = platform_device_alloc("dell-laptop", -1);
+	platform_device = platform_device_alloc("dell-laptop", PLATFORM_DEVID_NONE);
 	if (!platform_device) {
 		ret = -ENOMEM;
 		goto fail_platform_device1;
diff --git a/drivers/platform/x86/dell/dell-smbios-base.c b/drivers/platform/x86/dell/dell-smbios-base.c
index fc086b66f70b..e61bfaf8b5c4 100644
--- a/drivers/platform/x86/dell/dell-smbios-base.c
+++ b/drivers/platform/x86/dell/dell-smbios-base.c
@@ -441,7 +441,7 @@ static ssize_t location_show(struct device *dev,
 
 	i = match_attribute(dev, attr);
 	if (i > 0)
-		return scnprintf(buf, PAGE_SIZE, "%08x", da_tokens[i].location);
+		return sysfs_emit(buf, "%08x", da_tokens[i].location);
 	return 0;
 }
 
@@ -455,7 +455,7 @@ static ssize_t value_show(struct device *dev,
 
 	i = match_attribute(dev, attr);
 	if (i > 0)
-		return scnprintf(buf, PAGE_SIZE, "%08x", da_tokens[i].value);
+		return sysfs_emit(buf, "%08x", da_tokens[i].value);
 	return 0;
 }
 
diff --git a/drivers/platform/x86/dell/dell-wmi-base.c b/drivers/platform/x86/dell/dell-wmi-base.c
index e07d3ba85a3f..0a259a27459f 100644
--- a/drivers/platform/x86/dell/dell-wmi-base.c
+++ b/drivers/platform/x86/dell/dell-wmi-base.c
@@ -344,6 +344,9 @@ static const struct key_entry dell_wmi_keymap_type_0011[] = {
  * They are events with extended data
  */
 static const struct key_entry dell_wmi_keymap_type_0012[] = {
+	/* Backlight brightness change event */
+	{ KE_IGNORE, 0x0003, { KEY_RESERVED } },
+
 	/* Ultra-performance mode switch request */
 	{ KE_IGNORE, 0x000d, { KEY_RESERVED } },
 
diff --git a/drivers/platform/x86/dell/dell-wmi-privacy.c b/drivers/platform/x86/dell/dell-wmi-privacy.c
index 074b7e68c227..c82b3d6867c5 100644
--- a/drivers/platform/x86/dell/dell-wmi-privacy.c
+++ b/drivers/platform/x86/dell/dell-wmi-privacy.c
@@ -174,15 +174,12 @@ static ssize_t dell_privacy_current_state_show(struct device *dev,
 static DEVICE_ATTR_RO(dell_privacy_supported_type);
 static DEVICE_ATTR_RO(dell_privacy_current_state);
 
-static struct attribute *privacy_attributes[] = {
+static struct attribute *privacy_attrs[] = {
 	&dev_attr_dell_privacy_supported_type.attr,
 	&dev_attr_dell_privacy_current_state.attr,
 	NULL,
 };
-
-static const struct attribute_group privacy_attribute_group = {
-	.attrs = privacy_attributes
-};
+ATTRIBUTE_GROUPS(privacy);
 
 /*
  * Describes the Device State class exposed by BIOS which can be consumed by
@@ -342,10 +339,6 @@ static int dell_privacy_wmi_probe(struct wmi_device *wdev, const void *context)
 	if (ret)
 		return ret;
 
-	ret = devm_device_add_group(&wdev->dev, &privacy_attribute_group);
-	if (ret)
-		return ret;
-
 	if (priv->features_present & BIT(DELL_PRIVACY_TYPE_AUDIO)) {
 		ret = dell_privacy_leds_setup(&priv->wdev->dev);
 		if (ret)
@@ -374,6 +367,7 @@ static const struct wmi_device_id dell_wmi_privacy_wmi_id_table[] = {
 static struct wmi_driver dell_privacy_wmi_driver = {
 	.driver = {
 		.name = "dell-privacy",
+		.dev_groups = privacy_groups,
 	},
 	.probe = dell_privacy_wmi_probe,
 	.remove = dell_privacy_wmi_remove,
diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c
index 636bdfa83284..0a6411a8a104 100644
--- a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c
+++ b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c
@@ -270,7 +270,7 @@ void strlcpy_attr(char *dest, char *src)
 	size_t len = strlen(src) + 1;
 
 	if (len > 1 && len <= MAX_BUFF)
-		strlcpy(dest, src, len);
+		strscpy(dest, src, len);
 
 	/*len can be zero because any property not-applicable to attribute can
 	 * be empty so check only for too long buffers and log error
diff --git a/drivers/platform/x86/dell/dell_rbu.c b/drivers/platform/x86/dell/dell_rbu.c
index e9f4b30dcafa..9f51e0fcab04 100644
--- a/drivers/platform/x86/dell/dell_rbu.c
+++ b/drivers/platform/x86/dell/dell_rbu.c
@@ -645,7 +645,7 @@ static int __init dcdrbu_init(void)
 	spin_lock_init(&rbu_data.lock);
 
 	init_packet_head();
-	rbu_device = platform_device_register_simple("dell_rbu", -1, NULL, 0);
+	rbu_device = platform_device_register_simple("dell_rbu", PLATFORM_DEVID_NONE, NULL, 0);
 	if (IS_ERR(rbu_device)) {
 		pr_err("platform_device_register_simple failed\n");
 		return PTR_ERR(rbu_device);
diff --git a/drivers/platform/x86/eeepc-laptop.c b/drivers/platform/x86/eeepc-laptop.c
index ba08c9235f76..a388a28b6f2a 100644
--- a/drivers/platform/x86/eeepc-laptop.c
+++ b/drivers/platform/x86/eeepc-laptop.c
@@ -444,7 +444,7 @@ static int eeepc_platform_init(struct eeepc_laptop *eeepc)
 {
 	int result;
 
-	eeepc->platform_device = platform_device_alloc(EEEPC_LAPTOP_FILE, -1);
+	eeepc->platform_device = platform_device_alloc(EEEPC_LAPTOP_FILE, PLATFORM_DEVID_NONE);
 	if (!eeepc->platform_device)
 		return -ENOMEM;
 	platform_set_drvdata(eeepc->platform_device, eeepc);
diff --git a/drivers/platform/x86/eeepc-wmi.c b/drivers/platform/x86/eeepc-wmi.c
index ce86d84ee796..32d9f0ba6be3 100644
--- a/drivers/platform/x86/eeepc-wmi.c
+++ b/drivers/platform/x86/eeepc-wmi.c
@@ -96,11 +96,6 @@ static struct quirk_entry quirk_asus_et2012_type3 = {
 	.store_backlight_power = true,
 };
 
-static struct quirk_entry quirk_asus_x101ch = {
-	/* We need this when ACPI function doesn't do this well */
-	.wmi_backlight_power = true,
-};
-
 static struct quirk_entry *quirks;
 
 static void et2012_quirks(void)
@@ -151,25 +146,7 @@ static const struct dmi_system_id asus_quirks[] = {
 		},
 		.driver_data = &quirk_asus_unknown,
 	},
-	{
-		.callback = dmi_matched,
-		.ident = "ASUSTeK Computer INC. X101CH",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "X101CH"),
-		},
-		.driver_data = &quirk_asus_x101ch,
-	},
-	{
-		.callback = dmi_matched,
-		.ident = "ASUSTeK Computer INC. 1015CX",
-		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."),
-			DMI_MATCH(DMI_PRODUCT_NAME, "1015CX"),
-		},
-		.driver_data = &quirk_asus_x101ch,
-	},
-	{},
+	{}
 };
 
 static void eeepc_wmi_key_filter(struct asus_wmi_driver *asus_wmi, int *code,
diff --git a/drivers/platform/x86/fujitsu-laptop.c b/drivers/platform/x86/fujitsu-laptop.c
index 80929380ec7e..b543d117b12c 100644
--- a/drivers/platform/x86/fujitsu-laptop.c
+++ b/drivers/platform/x86/fujitsu-laptop.c
@@ -543,7 +543,7 @@ static int fujitsu_laptop_platform_add(struct acpi_device *device)
 	struct fujitsu_laptop *priv = acpi_driver_data(device);
 	int ret;
 
-	priv->pf_device = platform_device_alloc("fujitsu-laptop", -1);
+	priv->pf_device = platform_device_alloc("fujitsu-laptop", PLATFORM_DEVID_NONE);
 	if (!priv->pf_device)
 		return -ENOMEM;
 
diff --git a/drivers/platform/x86/hdaps.c b/drivers/platform/x86/hdaps.c
index 9996485f5295..f11f726d2062 100644
--- a/drivers/platform/x86/hdaps.c
+++ b/drivers/platform/x86/hdaps.c
@@ -547,7 +547,7 @@ static int __init hdaps_init(void)
 	if (ret)
 		goto out_region;
 
-	pdev = platform_device_register_simple("hdaps", -1, NULL, 0);
+	pdev = platform_device_register_simple("hdaps", PLATFORM_DEVID_NONE, NULL, 0);
 	if (IS_ERR(pdev)) {
 		ret = PTR_ERR(pdev);
 		goto out_driver;
diff --git a/drivers/platform/x86/hp-wmi.c b/drivers/platform/x86/hp-wmi.c
index bc7020e9df9e..627a6d0eaf83 100644
--- a/drivers/platform/x86/hp-wmi.c
+++ b/drivers/platform/x86/hp-wmi.c
@@ -177,7 +177,8 @@ enum hp_thermal_profile_omen_v1 {
 enum hp_thermal_profile {
 	HP_THERMAL_PROFILE_PERFORMANCE	= 0x00,
 	HP_THERMAL_PROFILE_DEFAULT		= 0x01,
-	HP_THERMAL_PROFILE_COOL			= 0x02
+	HP_THERMAL_PROFILE_COOL			= 0x02,
+	HP_THERMAL_PROFILE_QUIET		= 0x03,
 };
 
 #define IS_HWBLOCKED(x) ((x & HPWMI_POWER_FW_OR_HW) != HPWMI_POWER_FW_OR_HW)
@@ -206,15 +207,17 @@ struct bios_rfkill2_state {
 };
 
 static const struct key_entry hp_wmi_keymap[] = {
-	{ KE_KEY, 0x02,   { KEY_BRIGHTNESSUP } },
-	{ KE_KEY, 0x03,   { KEY_BRIGHTNESSDOWN } },
-	{ KE_KEY, 0x20e6, { KEY_PROG1 } },
-	{ KE_KEY, 0x20e8, { KEY_MEDIA } },
-	{ KE_KEY, 0x2142, { KEY_MEDIA } },
-	{ KE_KEY, 0x213b, { KEY_INFO } },
-	{ KE_KEY, 0x2169, { KEY_ROTATE_DISPLAY } },
-	{ KE_KEY, 0x216a, { KEY_SETUP } },
-	{ KE_KEY, 0x231b, { KEY_HELP } },
+	{ KE_KEY, 0x02,    { KEY_BRIGHTNESSUP } },
+	{ KE_KEY, 0x03,    { KEY_BRIGHTNESSDOWN } },
+	{ KE_KEY, 0x20e6,  { KEY_PROG1 } },
+	{ KE_KEY, 0x20e8,  { KEY_MEDIA } },
+	{ KE_KEY, 0x2142,  { KEY_MEDIA } },
+	{ KE_KEY, 0x213b,  { KEY_INFO } },
+	{ KE_KEY, 0x2169,  { KEY_ROTATE_DISPLAY } },
+	{ KE_KEY, 0x216a,  { KEY_SETUP } },
+	{ KE_KEY, 0x21a9,  { KEY_TOUCHPAD_OFF } },
+	{ KE_KEY, 0x121a9, { KEY_TOUCHPAD_ON } },
+	{ KE_KEY, 0x231b,  { KEY_HELP } },
 	{ KE_END, 0 }
 };
 
@@ -1194,6 +1197,9 @@ static int hp_wmi_platform_profile_get(struct platform_profile_handler *pprof,
 	case HP_THERMAL_PROFILE_COOL:
 		*profile =  PLATFORM_PROFILE_COOL;
 		break;
+	case HP_THERMAL_PROFILE_QUIET:
+		*profile = PLATFORM_PROFILE_QUIET;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -1216,6 +1222,9 @@ static int hp_wmi_platform_profile_set(struct platform_profile_handler *pprof,
 	case PLATFORM_PROFILE_COOL:
 		tp =  HP_THERMAL_PROFILE_COOL;
 		break;
+	case PLATFORM_PROFILE_QUIET:
+		tp = HP_THERMAL_PROFILE_QUIET;
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -1263,6 +1272,8 @@ static int thermal_profile_setup(void)
 
 		platform_profile_handler.profile_get = hp_wmi_platform_profile_get;
 		platform_profile_handler.profile_set = hp_wmi_platform_profile_set;
+
+		set_bit(PLATFORM_PROFILE_QUIET, platform_profile_handler.choices);
 	}
 
 	set_bit(PLATFORM_PROFILE_COOL, platform_profile_handler.choices);
@@ -1508,7 +1519,7 @@ static int __init hp_wmi_init(void)
 
 	if (bios_capable) {
 		hp_wmi_platform_dev =
-			platform_device_register_simple("hp-wmi", -1, NULL, 0);
+			platform_device_register_simple("hp-wmi", PLATFORM_DEVID_NONE, NULL, 0);
 		if (IS_ERR(hp_wmi_platform_dev)) {
 			err = PTR_ERR(hp_wmi_platform_dev);
 			goto err_destroy_input;
diff --git a/drivers/platform/x86/huawei-wmi.c b/drivers/platform/x86/huawei-wmi.c
index eac3e6b4ea11..5873c2663a65 100644
--- a/drivers/platform/x86/huawei-wmi.c
+++ b/drivers/platform/x86/huawei-wmi.c
@@ -871,7 +871,7 @@ static __init int huawei_wmi_init(void)
 	if (err)
 		goto pdrv_err;
 
-	pdev = platform_device_register_simple("huawei-wmi", -1, NULL, 0);
+	pdev = platform_device_register_simple("huawei-wmi", PLATFORM_DEVID_NONE, NULL, 0);
 	if (IS_ERR(pdev)) {
 		err = PTR_ERR(pdev);
 		goto pdev_err;
diff --git a/drivers/platform/x86/intel/chtwc_int33fe.c b/drivers/platform/x86/intel/chtwc_int33fe.c
index c52ac23e2331..2c9a7d52be07 100644
--- a/drivers/platform/x86/intel/chtwc_int33fe.c
+++ b/drivers/platform/x86/intel/chtwc_int33fe.c
@@ -219,7 +219,7 @@ static int cht_int33fe_add_nodes(struct cht_int33fe_data *data)
 
 	/*
 	 * Update node used in "usb-role-switch" property. Note that we
-	 * rely on software_node_register_nodes() to use the original
+	 * rely on software_node_register_node_group() to use the original
 	 * instance of properties instead of copying them.
 	 */
 	fusb302_mux_refs[0].node = mux_ref_node;
@@ -270,7 +270,7 @@ cht_int33fe_register_max17047(struct device *dev, struct cht_int33fe_data *data)
 	}
 
 	memset(&board_info, 0, sizeof(board_info));
-	strlcpy(board_info.type, "max17047", I2C_NAME_SIZE);
+	strscpy(board_info.type, "max17047", I2C_NAME_SIZE);
 	board_info.dev_name = "max17047";
 	board_info.fwnode = fwnode;
 	data->battery_fg = i2c_acpi_new_device(dev, 1, &board_info);
@@ -361,7 +361,7 @@ static int cht_int33fe_typec_probe(struct platform_device *pdev)
 	}
 
 	memset(&board_info, 0, sizeof(board_info));
-	strlcpy(board_info.type, "typec_fusb302", I2C_NAME_SIZE);
+	strscpy(board_info.type, "typec_fusb302", I2C_NAME_SIZE);
 	board_info.dev_name = "fusb302";
 	board_info.fwnode = fwnode;
 	board_info.irq = fusb302_irq;
@@ -381,7 +381,7 @@ static int cht_int33fe_typec_probe(struct platform_device *pdev)
 	memset(&board_info, 0, sizeof(board_info));
 	board_info.dev_name = "pi3usb30532";
 	board_info.fwnode = fwnode;
-	strlcpy(board_info.type, "pi3usb30532", I2C_NAME_SIZE);
+	strscpy(board_info.type, "pi3usb30532", I2C_NAME_SIZE);
 
 	data->pi3usb30532 = i2c_acpi_new_device(dev, 3, &board_info);
 	if (IS_ERR(data->pi3usb30532)) {
diff --git a/drivers/platform/x86/intel/int3472/common.c b/drivers/platform/x86/intel/int3472/common.c
index 77cf058e4168..9db2bb0bbba4 100644
--- a/drivers/platform/x86/intel/int3472/common.c
+++ b/drivers/platform/x86/intel/int3472/common.c
@@ -62,7 +62,7 @@ int skl_int3472_get_sensor_adev_and_name(struct device *dev,
 	struct acpi_device *sensor;
 	int ret = 0;
 
-	sensor = acpi_dev_get_first_consumer_dev(adev);
+	sensor = acpi_dev_get_next_consumer_dev(adev, NULL);
 	if (!sensor) {
 		dev_err(dev, "INT3472 seems to have no dependents.\n");
 		return -ENODEV;
diff --git a/drivers/platform/x86/intel/int3472/discrete.c b/drivers/platform/x86/intel/int3472/discrete.c
index ed4c9d760757..974a132db651 100644
--- a/drivers/platform/x86/intel/int3472/discrete.c
+++ b/drivers/platform/x86/intel/int3472/discrete.c
@@ -331,7 +331,22 @@ static int skl_int3472_parse_crs(struct int3472_discrete_device *int3472)
 	return 0;
 }
 
-static int skl_int3472_discrete_remove(struct platform_device *pdev);
+static int skl_int3472_discrete_remove(struct platform_device *pdev)
+{
+	struct int3472_discrete_device *int3472 = platform_get_drvdata(pdev);
+
+	gpiod_remove_lookup_table(&int3472->gpios);
+
+	if (int3472->clock.cl)
+		skl_int3472_unregister_clock(int3472);
+
+	gpiod_put(int3472->clock.ena_gpio);
+	gpiod_put(int3472->clock.led_gpio);
+
+	skl_int3472_unregister_regulator(int3472);
+
+	return 0;
+}
 
 static int skl_int3472_discrete_probe(struct platform_device *pdev)
 {
@@ -383,23 +398,6 @@ static int skl_int3472_discrete_probe(struct platform_device *pdev)
 	return 0;
 }
 
-static int skl_int3472_discrete_remove(struct platform_device *pdev)
-{
-	struct int3472_discrete_device *int3472 = platform_get_drvdata(pdev);
-
-	gpiod_remove_lookup_table(&int3472->gpios);
-
-	if (int3472->clock.cl)
-		skl_int3472_unregister_clock(int3472);
-
-	gpiod_put(int3472->clock.ena_gpio);
-	gpiod_put(int3472->clock.led_gpio);
-
-	skl_int3472_unregister_regulator(int3472);
-
-	return 0;
-}
-
 static const struct acpi_device_id int3472_device_id[] = {
 	{ "INT3472", 0 },
 	{ }
diff --git a/drivers/platform/x86/intel/int3472/tps68470.c b/drivers/platform/x86/intel/int3472/tps68470.c
index 22f61b47f9e5..f83e9c393f31 100644
--- a/drivers/platform/x86/intel/int3472/tps68470.c
+++ b/drivers/platform/x86/intel/int3472/tps68470.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Author: Dan Scally <djrscally@gmail.com> */
 
+#include <linux/acpi.h>
 #include <linux/i2c.h>
 #include <linux/kernel.h>
 #include <linux/mfd/core.h>
@@ -95,20 +96,65 @@ static int skl_int3472_tps68470_calc_type(struct acpi_device *adev)
 	return DESIGNED_FOR_WINDOWS;
 }
 
+/*
+ * Return the size of the flexible array member, because we'll need that later
+ * on to pass .pdata_size to cells.
+ */
+static int
+skl_int3472_fill_clk_pdata(struct device *dev, struct tps68470_clk_platform_data **clk_pdata)
+{
+	struct acpi_device *adev = ACPI_COMPANION(dev);
+	struct acpi_device *consumer;
+	unsigned int n_consumers = 0;
+	const char *sensor_name;
+	unsigned int i = 0;
+
+	for_each_acpi_consumer_dev(adev, consumer)
+		n_consumers++;
+
+	if (!n_consumers) {
+		dev_err(dev, "INT3472 seems to have no dependents\n");
+		return -ENODEV;
+	}
+
+	*clk_pdata = devm_kzalloc(dev, struct_size(*clk_pdata, consumers, n_consumers),
+				  GFP_KERNEL);
+	if (!*clk_pdata)
+		return -ENOMEM;
+
+	(*clk_pdata)->n_consumers = n_consumers;
+	i = 0;
+
+	for_each_acpi_consumer_dev(adev, consumer) {
+		sensor_name = devm_kasprintf(dev, GFP_KERNEL, I2C_DEV_NAME_FORMAT,
+					     acpi_dev_name(consumer));
+		if (!sensor_name)
+			return -ENOMEM;
+
+		(*clk_pdata)->consumers[i].consumer_dev_name = sensor_name;
+		i++;
+	}
+
+	acpi_dev_put(consumer);
+
+	return n_consumers;
+}
+
 static int skl_int3472_tps68470_probe(struct i2c_client *client)
 {
 	struct acpi_device *adev = ACPI_COMPANION(&client->dev);
 	const struct int3472_tps68470_board_data *board_data;
-	struct tps68470_clk_platform_data clk_pdata = {};
+	struct tps68470_clk_platform_data *clk_pdata;
 	struct mfd_cell *cells;
 	struct regmap *regmap;
+	int n_consumers;
 	int device_type;
 	int ret;
+	int i;
 
-	ret = skl_int3472_get_sensor_adev_and_name(&client->dev, NULL,
-						   &clk_pdata.consumer_dev_name);
-	if (ret)
-		return ret;
+	n_consumers = skl_int3472_fill_clk_pdata(&client->dev, &clk_pdata);
+	if (n_consumers < 0)
+		return n_consumers;
 
 	regmap = devm_regmap_init_i2c(client, &tps68470_regmap_config);
 	if (IS_ERR(regmap)) {
@@ -142,22 +188,25 @@ static int skl_int3472_tps68470_probe(struct i2c_client *client)
 		 * the clk + regulators must be ready when this happens.
 		 */
 		cells[0].name = "tps68470-clk";
-		cells[0].platform_data = &clk_pdata;
-		cells[0].pdata_size = sizeof(clk_pdata);
+		cells[0].platform_data = clk_pdata;
+		cells[0].pdata_size = struct_size(clk_pdata, consumers, n_consumers);
 		cells[1].name = "tps68470-regulator";
 		cells[1].platform_data = (void *)board_data->tps68470_regulator_pdata;
 		cells[1].pdata_size = sizeof(struct tps68470_regulator_platform_data);
 		cells[2].name = "tps68470-gpio";
 
-		gpiod_add_lookup_table(board_data->tps68470_gpio_lookup_table);
+		for (i = 0; i < board_data->n_gpiod_lookups; i++)
+			gpiod_add_lookup_table(board_data->tps68470_gpio_lookup_tables[i]);
 
 		ret = devm_mfd_add_devices(&client->dev, PLATFORM_DEVID_NONE,
 					   cells, TPS68470_WIN_MFD_CELL_COUNT,
 					   NULL, 0, NULL);
 		kfree(cells);
 
-		if (ret)
-			gpiod_remove_lookup_table(board_data->tps68470_gpio_lookup_table);
+		if (ret) {
+			for (i = 0; i < board_data->n_gpiod_lookups; i++)
+				gpiod_remove_lookup_table(board_data->tps68470_gpio_lookup_tables[i]);
+		}
 
 		break;
 	case DESIGNED_FOR_CHROMEOS:
@@ -178,15 +227,16 @@ static int skl_int3472_tps68470_probe(struct i2c_client *client)
 	return ret;
 }
 
-static int skl_int3472_tps68470_remove(struct i2c_client *client)
+static void skl_int3472_tps68470_remove(struct i2c_client *client)
 {
 	const struct int3472_tps68470_board_data *board_data;
+	int i;
 
 	board_data = int3472_tps68470_get_board_data(dev_name(&client->dev));
-	if (board_data)
-		gpiod_remove_lookup_table(board_data->tps68470_gpio_lookup_table);
-
-	return 0;
+	if (board_data) {
+		for (i = 0; i < board_data->n_gpiod_lookups; i++)
+			gpiod_remove_lookup_table(board_data->tps68470_gpio_lookup_tables[i]);
+	}
 }
 
 static const struct acpi_device_id int3472_device_id[] = {
diff --git a/drivers/platform/x86/intel/int3472/tps68470.h b/drivers/platform/x86/intel/int3472/tps68470.h
index cfd33eb62740..35915e701593 100644
--- a/drivers/platform/x86/intel/int3472/tps68470.h
+++ b/drivers/platform/x86/intel/int3472/tps68470.h
@@ -16,8 +16,9 @@ struct tps68470_regulator_platform_data;
 
 struct int3472_tps68470_board_data {
 	const char *dev_name;
-	struct gpiod_lookup_table *tps68470_gpio_lookup_table;
 	const struct tps68470_regulator_platform_data *tps68470_regulator_pdata;
+	unsigned int n_gpiod_lookups;
+	struct gpiod_lookup_table *tps68470_gpio_lookup_tables[];
 };
 
 const struct int3472_tps68470_board_data *int3472_tps68470_get_board_data(const char *dev_name);
diff --git a/drivers/platform/x86/intel/int3472/tps68470_board_data.c b/drivers/platform/x86/intel/int3472/tps68470_board_data.c
index 525f09a3b5ff..309eab9c0558 100644
--- a/drivers/platform/x86/intel/int3472/tps68470_board_data.c
+++ b/drivers/platform/x86/intel/int3472/tps68470_board_data.c
@@ -30,6 +30,15 @@ static struct regulator_consumer_supply int347a_vcm_consumer_supplies[] = {
 static struct regulator_consumer_supply int347a_vsio_consumer_supplies[] = {
 	REGULATOR_SUPPLY("dovdd", "i2c-INT347A:00"),
 	REGULATOR_SUPPLY("vsio", "i2c-INT347A:00-VCM"),
+	REGULATOR_SUPPLY("vddd", "i2c-INT347E:00"),
+};
+
+static struct regulator_consumer_supply int347a_aux1_consumer_supplies[] = {
+	REGULATOR_SUPPLY("vdda", "i2c-INT347E:00"),
+};
+
+static struct regulator_consumer_supply int347a_aux2_consumer_supplies[] = {
+	REGULATOR_SUPPLY("vdddo", "i2c-INT347E:00"),
 };
 
 static const struct regulator_init_data surface_go_tps68470_core_reg_init_data = {
@@ -86,6 +95,28 @@ static const struct regulator_init_data surface_go_tps68470_vsio_reg_init_data =
 	.consumer_supplies = int347a_vsio_consumer_supplies,
 };
 
+static const struct regulator_init_data surface_go_tps68470_aux1_reg_init_data = {
+	.constraints = {
+		.min_uV = 2815200,
+		.max_uV = 2815200,
+		.apply_uV = 1,
+		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
+	},
+	.num_consumer_supplies = ARRAY_SIZE(int347a_aux1_consumer_supplies),
+	.consumer_supplies = int347a_aux1_consumer_supplies,
+};
+
+static const struct regulator_init_data surface_go_tps68470_aux2_reg_init_data = {
+	.constraints = {
+		.min_uV = 1800600,
+		.max_uV = 1800600,
+		.apply_uV = 1,
+		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
+	},
+	.num_consumer_supplies = ARRAY_SIZE(int347a_aux2_consumer_supplies),
+	.consumer_supplies = int347a_aux2_consumer_supplies,
+};
+
 static const struct tps68470_regulator_platform_data surface_go_tps68470_pdata = {
 	.reg_init_data = {
 		[TPS68470_CORE] = &surface_go_tps68470_core_reg_init_data,
@@ -93,10 +124,12 @@ static const struct tps68470_regulator_platform_data surface_go_tps68470_pdata =
 		[TPS68470_VCM]  = &surface_go_tps68470_vcm_reg_init_data,
 		[TPS68470_VIO] = &surface_go_tps68470_vio_reg_init_data,
 		[TPS68470_VSIO] = &surface_go_tps68470_vsio_reg_init_data,
+		[TPS68470_AUX1] = &surface_go_tps68470_aux1_reg_init_data,
+		[TPS68470_AUX2] = &surface_go_tps68470_aux2_reg_init_data,
 	},
 };
 
-static struct gpiod_lookup_table surface_go_tps68470_gpios = {
+static struct gpiod_lookup_table surface_go_int347a_gpios = {
 	.dev_id = "i2c-INT347A:00",
 	.table = {
 		GPIO_LOOKUP("tps68470-gpio", 9, "reset", GPIO_ACTIVE_LOW),
@@ -105,16 +138,31 @@ static struct gpiod_lookup_table surface_go_tps68470_gpios = {
 	}
 };
 
+static struct gpiod_lookup_table surface_go_int347e_gpios = {
+	.dev_id = "i2c-INT347E:00",
+	.table = {
+		GPIO_LOOKUP("tps68470-gpio", 5, "enable", GPIO_ACTIVE_HIGH),
+		{ }
+	}
+};
+
 static const struct int3472_tps68470_board_data surface_go_tps68470_board_data = {
 	.dev_name = "i2c-INT3472:05",
-	.tps68470_gpio_lookup_table = &surface_go_tps68470_gpios,
 	.tps68470_regulator_pdata = &surface_go_tps68470_pdata,
+	.n_gpiod_lookups = 2,
+	.tps68470_gpio_lookup_tables = {
+		&surface_go_int347a_gpios,
+		&surface_go_int347e_gpios,
+	},
 };
 
 static const struct int3472_tps68470_board_data surface_go3_tps68470_board_data = {
 	.dev_name = "i2c-INT3472:01",
-	.tps68470_gpio_lookup_table = &surface_go_tps68470_gpios,
 	.tps68470_regulator_pdata = &surface_go_tps68470_pdata,
+	.n_gpiod_lookups = 1,
+	.tps68470_gpio_lookup_tables = {
+		&surface_go_int347a_gpios
+	},
 };
 
 static const struct dmi_system_id int3472_tps68470_board_data_table[] = {
diff --git a/drivers/platform/x86/intel/oaktrail.c b/drivers/platform/x86/intel/oaktrail.c
index 1a09a75bd16d..7c5c623630c1 100644
--- a/drivers/platform/x86/intel/oaktrail.c
+++ b/drivers/platform/x86/intel/oaktrail.c
@@ -317,7 +317,7 @@ static int __init oaktrail_init(void)
 		goto err_driver_reg;
 	}
 
-	oaktrail_device = platform_device_alloc(DRIVER_NAME, -1);
+	oaktrail_device = platform_device_alloc(DRIVER_NAME, PLATFORM_DEVID_NONE);
 	if (!oaktrail_device) {
 		pr_warn("Unable to allocate platform device\n");
 		ret = -ENOMEM;
diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
index 84eabd6156bb..cb24de9e97dc 100644
--- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
+++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
@@ -113,7 +113,7 @@ show_uncore_perf_status(current_freq_khz);
 		struct uncore_data *data = container_of(attr, struct uncore_data,\
 							  member_name##_dev_attr);\
 									\
-		return scnprintf(buf, PAGE_SIZE, "%u\n",		\
+		return sysfs_emit(buf, "%u\n",				\
 				 data->member_name);			\
 	}								\
 
diff --git a/drivers/platform/x86/intel/wmi/thunderbolt.c b/drivers/platform/x86/intel/wmi/thunderbolt.c
index 4ae87060d18b..fc333ff82d1e 100644
--- a/drivers/platform/x86/intel/wmi/thunderbolt.c
+++ b/drivers/platform/x86/intel/wmi/thunderbolt.c
@@ -51,26 +51,7 @@ static struct attribute *tbt_attrs[] = {
 	&dev_attr_force_power.attr,
 	NULL
 };
-
-static const struct attribute_group tbt_attribute_group = {
-	.attrs = tbt_attrs,
-};
-
-static int intel_wmi_thunderbolt_probe(struct wmi_device *wdev,
-				       const void *context)
-{
-	int ret;
-
-	ret = sysfs_create_group(&wdev->dev.kobj, &tbt_attribute_group);
-	kobject_uevent(&wdev->dev.kobj, KOBJ_CHANGE);
-	return ret;
-}
-
-static void intel_wmi_thunderbolt_remove(struct wmi_device *wdev)
-{
-	sysfs_remove_group(&wdev->dev.kobj, &tbt_attribute_group);
-	kobject_uevent(&wdev->dev.kobj, KOBJ_CHANGE);
-}
+ATTRIBUTE_GROUPS(tbt);
 
 static const struct wmi_device_id intel_wmi_thunderbolt_id_table[] = {
 	{ .guid_string = INTEL_WMI_THUNDERBOLT_GUID },
@@ -80,9 +61,8 @@ static const struct wmi_device_id intel_wmi_thunderbolt_id_table[] = {
 static struct wmi_driver intel_wmi_thunderbolt_driver = {
 	.driver = {
 		.name = "intel-wmi-thunderbolt",
+		.dev_groups = tbt_groups,
 	},
-	.probe = intel_wmi_thunderbolt_probe,
-	.remove = intel_wmi_thunderbolt_remove,
 	.id_table = intel_wmi_thunderbolt_id_table,
 };
 
diff --git a/drivers/platform/x86/mlx-platform.c b/drivers/platform/x86/mlx-platform.c
index 5e072a0666f4..2fac05a17a5c 100644
--- a/drivers/platform/x86/mlx-platform.c
+++ b/drivers/platform/x86/mlx-platform.c
@@ -5181,7 +5181,7 @@ static int __init mlxplat_init(void)
 	if (!dmi_check_system(mlxplat_dmi_table))
 		return -ENODEV;
 
-	mlxplat_dev = platform_device_register_simple(MLX_PLAT_DEVICE_NAME, -1,
+	mlxplat_dev = platform_device_register_simple(MLX_PLAT_DEVICE_NAME, PLATFORM_DEVID_NONE,
 					mlxplat_lpc_resources,
 					ARRAY_SIZE(mlxplat_lpc_resources));
 
diff --git a/drivers/platform/x86/msi-laptop.c b/drivers/platform/x86/msi-laptop.c
index 24ffc8e2d2d1..6b18ec543ac3 100644
--- a/drivers/platform/x86/msi-laptop.c
+++ b/drivers/platform/x86/msi-laptop.c
@@ -53,8 +53,6 @@
 #include <linux/input/sparse-keymap.h>
 #include <acpi/video.h>
 
-#define MSI_DRIVER_VERSION "0.5"
-
 #define MSI_LCD_LEVEL_MAX 9
 
 #define MSI_EC_COMMAND_WIRELESS 0x10
@@ -592,15 +590,22 @@ static int dmi_check_cb(const struct dmi_system_id *dmi)
 	return 1;
 }
 
+static unsigned long msi_work_delay(int msecs)
+{
+	if (quirks->ec_delay)
+		return msecs_to_jiffies(msecs);
+
+	return 0;
+}
+
 static const struct dmi_system_id msi_dmi_table[] __initconst = {
 	{
 		.ident = "MSI S270",
 		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "MICRO-STAR INT'L CO.,LTD"),
+			DMI_MATCH(DMI_SYS_VENDOR, "MICRO-STAR INT"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "MS-1013"),
 			DMI_MATCH(DMI_PRODUCT_VERSION, "0131"),
-			DMI_MATCH(DMI_CHASSIS_VENDOR,
-				  "MICRO-STAR INT'L CO.,LTD")
+			DMI_MATCH(DMI_CHASSIS_VENDOR, "MICRO-STAR INT")
 		},
 		.driver_data = &quirk_old_ec_model,
 		.callback = dmi_check_cb
@@ -633,8 +638,7 @@ static const struct dmi_system_id msi_dmi_table[] __initconst = {
 			DMI_MATCH(DMI_SYS_VENDOR, "NOTEBOOK"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "SAM2000"),
 			DMI_MATCH(DMI_PRODUCT_VERSION, "0131"),
-			DMI_MATCH(DMI_CHASSIS_VENDOR,
-				  "MICRO-STAR INT'L CO.,LTD")
+			DMI_MATCH(DMI_CHASSIS_VENDOR, "MICRO-STAR INT")
 		},
 		.driver_data = &quirk_old_ec_model,
 		.callback = dmi_check_cb
@@ -705,6 +709,7 @@ static const struct dmi_system_id msi_dmi_table[] __initconst = {
 	},
 	{ }
 };
+MODULE_DEVICE_TABLE(dmi, msi_dmi_table);
 
 static int rfkill_bluetooth_set(void *data, bool blocked)
 {
@@ -785,7 +790,6 @@ static void msi_update_rfkill(struct work_struct *ignored)
 		msi_rfkill_set_state(rfk_threeg, !threeg_s);
 }
 static DECLARE_DELAYED_WORK(msi_rfkill_dwork, msi_update_rfkill);
-static DECLARE_WORK(msi_rfkill_work, msi_update_rfkill);
 
 static void msi_send_touchpad_key(struct work_struct *ignored)
 {
@@ -801,7 +805,6 @@ static void msi_send_touchpad_key(struct work_struct *ignored)
 		KEY_TOUCHPAD_ON : KEY_TOUCHPAD_OFF, 1, true);
 }
 static DECLARE_DELAYED_WORK(msi_touchpad_dwork, msi_send_touchpad_key);
-static DECLARE_WORK(msi_touchpad_work, msi_send_touchpad_key);
 
 static bool msi_laptop_i8042_filter(unsigned char data, unsigned char str,
 				struct serio *port)
@@ -819,20 +822,12 @@ static bool msi_laptop_i8042_filter(unsigned char data, unsigned char str,
 		extended = false;
 		switch (data) {
 		case 0xE4:
-			if (quirks->ec_delay) {
-				schedule_delayed_work(&msi_touchpad_dwork,
-					round_jiffies_relative(0.5 * HZ));
-			} else
-				schedule_work(&msi_touchpad_work);
+			schedule_delayed_work(&msi_touchpad_dwork, msi_work_delay(500));
 			break;
 		case 0x54:
 		case 0x62:
 		case 0x76:
-			if (quirks->ec_delay) {
-				schedule_delayed_work(&msi_rfkill_dwork,
-					round_jiffies_relative(0.5 * HZ));
-			} else
-				schedule_work(&msi_rfkill_work);
+			schedule_delayed_work(&msi_rfkill_dwork, msi_work_delay(500));
 			break;
 		}
 	}
@@ -899,12 +894,7 @@ static int rfkill_init(struct platform_device *sdev)
 	}
 
 	/* schedule to run rfkill state initial */
-	if (quirks->ec_delay) {
-		schedule_delayed_work(&msi_rfkill_init,
-			round_jiffies_relative(1 * HZ));
-	} else
-		schedule_work(&msi_rfkill_work);
-
+	schedule_delayed_work(&msi_rfkill_init, msi_work_delay(1000));
 	return 0;
 
 err_threeg:
@@ -921,8 +911,7 @@ err_bluetooth:
 	return retval;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int msi_laptop_resume(struct device *device)
+static int msi_scm_disable_hw_fn_handling(void)
 {
 	u8 data;
 	int result;
@@ -942,6 +931,12 @@ static int msi_laptop_resume(struct device *device)
 
 	return 0;
 }
+
+#ifdef CONFIG_PM_SLEEP
+static int msi_laptop_resume(struct device *device)
+{
+	return msi_scm_disable_hw_fn_handling();
+}
 #endif
 
 static int __init msi_laptop_input_setup(void)
@@ -974,7 +969,6 @@ err_free_dev:
 
 static int __init load_scm_model_init(struct platform_device *sdev)
 {
-	u8 data;
 	int result;
 
 	if (!quirks->ec_read_only) {
@@ -988,12 +982,7 @@ static int __init load_scm_model_init(struct platform_device *sdev)
 	}
 
 	/* disable hardware control by fn key */
-	result = ec_read(MSI_STANDARD_EC_SCM_LOAD_ADDRESS, &data);
-	if (result < 0)
-		return result;
-
-	result = ec_write(MSI_STANDARD_EC_SCM_LOAD_ADDRESS,
-		data | MSI_STANDARD_EC_SCM_LOAD_MASK);
+	result = msi_scm_disable_hw_fn_handling();
 	if (result < 0)
 		return result;
 
@@ -1022,9 +1011,19 @@ fail_input:
 	rfkill_cleanup();
 
 fail_rfkill:
-
 	return result;
+}
+
+static void msi_scm_model_exit(void)
+{
+	if (!quirks->load_scm_model)
+		return;
 
+	i8042_remove_filter(msi_laptop_i8042_filter);
+	cancel_delayed_work_sync(&msi_touchpad_dwork);
+	input_unregister_device(msi_laptop_input_dev);
+	cancel_delayed_work_sync(&msi_rfkill_dwork);
+	rfkill_cleanup();
 }
 
 static int __init msi_init(void)
@@ -1048,8 +1047,7 @@ static int __init msi_init(void)
 		return -EINVAL;
 
 	/* Register backlight stuff */
-
-	if (quirks->old_ec_model ||
+	if (quirks->old_ec_model &&
 	    acpi_video_get_backlight_type() == acpi_backlight_vendor) {
 		struct backlight_properties props;
 		memset(&props, 0, sizeof(struct backlight_properties));
@@ -1068,7 +1066,7 @@ static int __init msi_init(void)
 
 	/* Register platform stuff */
 
-	msipf_device = platform_device_alloc("msi-laptop-pf", -1);
+	msipf_device = platform_device_alloc("msi-laptop-pf", PLATFORM_DEVID_NONE);
 	if (!msipf_device) {
 		ret = -ENOMEM;
 		goto fail_platform_driver;
@@ -1108,19 +1106,12 @@ static int __init msi_init(void)
 			set_auto_brightness(auto_brightness);
 	}
 
-	pr_info("driver " MSI_DRIVER_VERSION " successfully loaded\n");
-
 	return 0;
 
 fail_create_attr:
 	sysfs_remove_group(&msipf_device->dev.kobj, &msipf_attribute_group);
 fail_create_group:
-	if (quirks->load_scm_model) {
-		i8042_remove_filter(msi_laptop_i8042_filter);
-		cancel_delayed_work_sync(&msi_rfkill_dwork);
-		cancel_work_sync(&msi_rfkill_work);
-		rfkill_cleanup();
-	}
+	msi_scm_model_exit();
 fail_scm_model_init:
 	platform_device_del(msipf_device);
 fail_device_add:
@@ -1135,14 +1126,7 @@ fail_backlight:
 
 static void __exit msi_cleanup(void)
 {
-	if (quirks->load_scm_model) {
-		i8042_remove_filter(msi_laptop_i8042_filter);
-		input_unregister_device(msi_laptop_input_dev);
-		cancel_delayed_work_sync(&msi_rfkill_dwork);
-		cancel_work_sync(&msi_rfkill_work);
-		rfkill_cleanup();
-	}
-
+	msi_scm_model_exit();
 	sysfs_remove_group(&msipf_device->dev.kobj, &msipf_attribute_group);
 	if (!quirks->old_ec_model && threeg_exists)
 		device_remove_file(&msipf_device->dev, &dev_attr_threeg);
@@ -1155,8 +1139,6 @@ static void __exit msi_cleanup(void)
 		if (auto_brightness != 2)
 			set_auto_brightness(1);
 	}
-
-	pr_info("driver unloaded\n");
 }
 
 module_init(msi_init);
@@ -1164,16 +1146,4 @@ module_exit(msi_cleanup);
 
 MODULE_AUTHOR("Lennart Poettering");
 MODULE_DESCRIPTION("MSI Laptop Support");
-MODULE_VERSION(MSI_DRIVER_VERSION);
 MODULE_LICENSE("GPL");
-
-MODULE_ALIAS("dmi:*:svnMICRO-STARINT'LCO.,LTD:pnMS-1013:pvr0131*:cvnMICRO-STARINT'LCO.,LTD:ct10:*");
-MODULE_ALIAS("dmi:*:svnMicro-StarInternational:pnMS-1058:pvr0581:rvnMSI:rnMS-1058:*:ct10:*");
-MODULE_ALIAS("dmi:*:svnMicro-StarInternational:pnMS-1412:*:rvnMSI:rnMS-1412:*:cvnMICRO-STARINT'LCO.,LTD:ct10:*");
-MODULE_ALIAS("dmi:*:svnNOTEBOOK:pnSAM2000:pvr0131*:cvnMICRO-STARINT'LCO.,LTD:ct10:*");
-MODULE_ALIAS("dmi:*:svnMICRO-STARINTERNATIONAL*:pnMS-N034:*");
-MODULE_ALIAS("dmi:*:svnMICRO-STARINTERNATIONAL*:pnMS-N051:*");
-MODULE_ALIAS("dmi:*:svnMICRO-STARINTERNATIONAL*:pnMS-N014:*");
-MODULE_ALIAS("dmi:*:svnMicro-StarInternational*:pnCR620:*");
-MODULE_ALIAS("dmi:*:svnMicro-StarInternational*:pnU270series:*");
-MODULE_ALIAS("dmi:*:svnMICRO-STARINTERNATIONAL*:pnU90/U100:*");
diff --git a/drivers/platform/x86/nvidia-wmi-ec-backlight.c b/drivers/platform/x86/nvidia-wmi-ec-backlight.c
index 61e37194df70..baccdf658538 100644
--- a/drivers/platform/x86/nvidia-wmi-ec-backlight.c
+++ b/drivers/platform/x86/nvidia-wmi-ec-backlight.c
@@ -7,73 +7,10 @@
 #include <linux/backlight.h>
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
+#include <linux/platform_data/x86/nvidia-wmi-ec-backlight.h>
 #include <linux/types.h>
 #include <linux/wmi.h>
-
-/**
- * enum wmi_brightness_method - WMI method IDs
- * @WMI_BRIGHTNESS_METHOD_LEVEL:  Get/Set EC brightness level status
- * @WMI_BRIGHTNESS_METHOD_SOURCE: Get/Set EC Brightness Source
- */
-enum wmi_brightness_method {
-	WMI_BRIGHTNESS_METHOD_LEVEL = 1,
-	WMI_BRIGHTNESS_METHOD_SOURCE = 2,
-	WMI_BRIGHTNESS_METHOD_MAX
-};
-
-/**
- * enum wmi_brightness_mode - Operation mode for WMI-wrapped method
- * @WMI_BRIGHTNESS_MODE_GET:            Get the current brightness level/source.
- * @WMI_BRIGHTNESS_MODE_SET:            Set the brightness level.
- * @WMI_BRIGHTNESS_MODE_GET_MAX_LEVEL:  Get the maximum brightness level. This
- *                                      is only valid when the WMI method is
- *                                      %WMI_BRIGHTNESS_METHOD_LEVEL.
- */
-enum wmi_brightness_mode {
-	WMI_BRIGHTNESS_MODE_GET = 0,
-	WMI_BRIGHTNESS_MODE_SET = 1,
-	WMI_BRIGHTNESS_MODE_GET_MAX_LEVEL = 2,
-	WMI_BRIGHTNESS_MODE_MAX
-};
-
-/**
- * enum wmi_brightness_source - Backlight brightness control source selection
- * @WMI_BRIGHTNESS_SOURCE_GPU: Backlight brightness is controlled by the GPU.
- * @WMI_BRIGHTNESS_SOURCE_EC:  Backlight brightness is controlled by the
- *                             system's Embedded Controller (EC).
- * @WMI_BRIGHTNESS_SOURCE_AUX: Backlight brightness is controlled over the
- *                             DisplayPort AUX channel.
- */
-enum wmi_brightness_source {
-	WMI_BRIGHTNESS_SOURCE_GPU = 1,
-	WMI_BRIGHTNESS_SOURCE_EC = 2,
-	WMI_BRIGHTNESS_SOURCE_AUX = 3,
-	WMI_BRIGHTNESS_SOURCE_MAX
-};
-
-/**
- * struct wmi_brightness_args - arguments for the WMI-wrapped ACPI method
- * @mode:    Pass in an &enum wmi_brightness_mode value to select between
- *           getting or setting a value.
- * @val:     In parameter for value to set when using %WMI_BRIGHTNESS_MODE_SET
- *           mode. Not used in conjunction with %WMI_BRIGHTNESS_MODE_GET or
- *           %WMI_BRIGHTNESS_MODE_GET_MAX_LEVEL mode.
- * @ret:     Out parameter returning retrieved value when operating in
- *           %WMI_BRIGHTNESS_MODE_GET or %WMI_BRIGHTNESS_MODE_GET_MAX_LEVEL
- *           mode. Not used in %WMI_BRIGHTNESS_MODE_SET mode.
- * @ignored: Padding; not used. The ACPI method expects a 24 byte params struct.
- *
- * This is the parameters structure for the WmiBrightnessNotify ACPI method as
- * wrapped by WMI. The value passed in to @val or returned by @ret will be a
- * brightness value when the WMI method ID is %WMI_BRIGHTNESS_METHOD_LEVEL, or
- * an &enum wmi_brightness_source value with %WMI_BRIGHTNESS_METHOD_SOURCE.
- */
-struct wmi_brightness_args {
-	u32 mode;
-	u32 val;
-	u32 ret;
-	u32 ignored[3];
-};
+#include <acpi/video.h>
 
 /**
  * wmi_brightness_notify() - helper function for calling WMI-wrapped ACPI method
@@ -151,19 +88,10 @@ static int nvidia_wmi_ec_backlight_probe(struct wmi_device *wdev, const void *ct
 {
 	struct backlight_properties props = {};
 	struct backlight_device *bdev;
-	u32 source;
 	int ret;
 
-	ret = wmi_brightness_notify(wdev, WMI_BRIGHTNESS_METHOD_SOURCE,
-	                           WMI_BRIGHTNESS_MODE_GET, &source);
-	if (ret)
-		return ret;
-
-	/*
-	 * This driver is only to be used when brightness control is handled
-	 * by the EC; otherwise, the GPU driver(s) should control brightness.
-	 */
-	if (source != WMI_BRIGHTNESS_SOURCE_EC)
+	/* drivers/acpi/video_detect.c also checks that SOURCE == EC */
+	if (acpi_video_get_backlight_type() != acpi_backlight_nvidia_wmi_ec)
 		return -ENODEV;
 
 	/*
@@ -191,8 +119,6 @@ static int nvidia_wmi_ec_backlight_probe(struct wmi_device *wdev, const void *ct
 	return PTR_ERR_OR_ZERO(bdev);
 }
 
-#define WMI_BRIGHTNESS_GUID "603E9613-EF25-4338-A3D0-C46177516DB7"
-
 static const struct wmi_device_id nvidia_wmi_ec_backlight_id_table[] = {
 	{ .guid_string = WMI_BRIGHTNESS_GUID },
 	{ }
diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c
index d9a095d2c0eb..ad3083f9946d 100644
--- a/drivers/platform/x86/panasonic-laptop.c
+++ b/drivers/platform/x86/panasonic-laptop.c
@@ -1034,7 +1034,7 @@ static int acpi_pcc_hotkey_add(struct acpi_device *device)
 	/* optical drive initialization */
 	if (ACPI_SUCCESS(check_optd_present())) {
 		pcc->platform = platform_device_register_simple("panasonic",
-			-1, NULL, 0);
+			PLATFORM_DEVID_NONE, NULL, 0);
 		if (IS_ERR(pcc->platform)) {
 			result = PTR_ERR(pcc->platform);
 			goto out_backlight;
diff --git a/drivers/platform/x86/pmc_atom.c b/drivers/platform/x86/pmc_atom.c
index 5c757c7f64de..93a6414c6611 100644
--- a/drivers/platform/x86/pmc_atom.c
+++ b/drivers/platform/x86/pmc_atom.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Intel Atom SOC Power Management Controller Driver
- * Copyright (c) 2014, Intel Corporation.
+ * Intel Atom SoC Power Management Controller Driver
+ * Copyright (c) 2014-2015,2017,2022 Intel Corporation.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -60,7 +60,7 @@ static const struct pmc_clk byt_clks[] = {
 		.freq = 19200000,
 		.parent_name = "xtal",
 	},
-	{},
+	{}
 };
 
 static const struct pmc_clk cht_clks[] = {
@@ -69,7 +69,7 @@ static const struct pmc_clk cht_clks[] = {
 		.freq = 19200000,
 		.parent_name = NULL,
 	},
-	{},
+	{}
 };
 
 static const struct pmc_bit_map d3_sts_0_map[] = {
@@ -105,7 +105,7 @@ static const struct pmc_bit_map d3_sts_0_map[] = {
 	{"LPSS2_F5_I2C5",	BIT_LPSS2_F5_I2C5},
 	{"LPSS2_F6_I2C6",	BIT_LPSS2_F6_I2C6},
 	{"LPSS2_F7_I2C7",	BIT_LPSS2_F7_I2C7},
-	{},
+	{}
 };
 
 static struct pmc_bit_map byt_d3_sts_1_map[] = {
@@ -113,21 +113,21 @@ static struct pmc_bit_map byt_d3_sts_1_map[] = {
 	{"OTG_SS_PHY",		BIT_OTG_SS_PHY},
 	{"USH_SS_PHY",		BIT_USH_SS_PHY},
 	{"DFX",			BIT_DFX},
-	{},
+	{}
 };
 
 static struct pmc_bit_map cht_d3_sts_1_map[] = {
 	{"SMB",			BIT_SMB},
 	{"GMM",			BIT_STS_GMM},
 	{"ISH",			BIT_STS_ISH},
-	{},
+	{}
 };
 
 static struct pmc_bit_map cht_func_dis_2_map[] = {
 	{"SMB",			BIT_SMB},
 	{"GMM",			BIT_FD_GMM},
 	{"ISH",			BIT_FD_ISH},
-	{},
+	{}
 };
 
 static const struct pmc_bit_map byt_pss_map[] = {
@@ -149,7 +149,7 @@ static const struct pmc_bit_map byt_pss_map[] = {
 	{"OTG_VCCA",		PMC_PSS_BIT_OTG_VCCA},
 	{"USB",			PMC_PSS_BIT_USB},
 	{"USB_SUS",		PMC_PSS_BIT_USB_SUS},
-	{},
+	{}
 };
 
 static const struct pmc_bit_map cht_pss_map[] = {
@@ -172,7 +172,7 @@ static const struct pmc_bit_map cht_pss_map[] = {
 	{"DFX_CLUSTER3",	PMC_PSS_BIT_CHT_DFX_CLUSTER3},
 	{"DFX_CLUSTER4",	PMC_PSS_BIT_CHT_DFX_CLUSTER4},
 	{"DFX_CLUSTER5",	PMC_PSS_BIT_CHT_DFX_CLUSTER5},
-	{},
+	{}
 };
 
 static const struct pmc_reg_map byt_reg_map = {
@@ -354,7 +354,7 @@ static bool pmc_clk_is_critical = true;
 
 static int dmi_callback(const struct dmi_system_id *d)
 {
-	pr_info("%s critclks quirk enabled\n", d->ident);
+	pr_info("%s: PMC critical clocks quirk enabled\n", d->ident);
 
 	return 1;
 }
@@ -417,8 +417,7 @@ static const struct dmi_system_id critclk_systems[] = {
 			DMI_MATCH(DMI_SYS_VENDOR, "SIEMENS AG"),
 		},
 	},
-
-	{ /*sentinel*/ }
+	{}
 };
 
 static int pmc_setup_clks(struct pci_dev *pdev, void __iomem *pmc_regmap,
@@ -490,15 +489,11 @@ static int pmc_setup_dev(struct pci_dev *pdev, const struct pci_device_id *ent)
 	return ret;
 }
 
-/*
- * Data for PCI driver interface
- *
- * used by pci_match_id() call below.
- */
+/* Data for PCI driver interface used by pci_match_id() call below */
 static const struct pci_device_id pmc_pci_ids[] = {
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_VLV_PMC), (kernel_ulong_t)&byt_data },
 	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_CHT_PMC), (kernel_ulong_t)&cht_data },
-	{ 0, },
+	{}
 };
 
 static int __init pmc_atom_init(void)
@@ -506,8 +501,9 @@ static int __init pmc_atom_init(void)
 	struct pci_dev *pdev = NULL;
 	const struct pci_device_id *ent;
 
-	/* We look for our device - PCU PMC
-	 * we assume that there is max. one device.
+	/*
+	 * We look for our device - PCU PMC.
+	 * We assume that there is maximum one device.
 	 *
 	 * We can't use plain pci_driver mechanism,
 	 * as the device is really a multiple function device,
@@ -519,7 +515,7 @@ static int __init pmc_atom_init(void)
 		if (ent)
 			return pmc_setup_dev(pdev, ent);
 	}
-	/* Device not found. */
+	/* Device not found */
 	return -ENODEV;
 }
 
@@ -527,6 +523,6 @@ device_initcall(pmc_atom_init);
 
 /*
 MODULE_AUTHOR("Aubrey Li <aubrey.li@linux.intel.com>");
-MODULE_DESCRIPTION("Intel Atom SOC Power Management Controller Interface");
+MODULE_DESCRIPTION("Intel Atom SoC Power Management Controller Interface");
 MODULE_LICENSE("GPL v2");
 */
diff --git a/drivers/platform/x86/samsung-laptop.c b/drivers/platform/x86/samsung-laptop.c
index c187dcdf82f0..b4aa8ba35d2d 100644
--- a/drivers/platform/x86/samsung-laptop.c
+++ b/drivers/platform/x86/samsung-laptop.c
@@ -356,23 +356,13 @@ struct samsung_laptop {
 };
 
 struct samsung_quirks {
-	bool broken_acpi_video;
 	bool four_kbd_backlight_levels;
 	bool enable_kbd_backlight;
-	bool use_native_backlight;
 	bool lid_handling;
 };
 
 static struct samsung_quirks samsung_unknown = {};
 
-static struct samsung_quirks samsung_broken_acpi_video = {
-	.broken_acpi_video = true,
-};
-
-static struct samsung_quirks samsung_use_native_backlight = {
-	.use_native_backlight = true,
-};
-
 static struct samsung_quirks samsung_np740u3e = {
 	.four_kbd_backlight_levels = true,
 	.enable_kbd_backlight = true,
@@ -1484,7 +1474,7 @@ static int __init samsung_platform_init(struct samsung_laptop *samsung)
 {
 	struct platform_device *pdev;
 
-	pdev = platform_device_register_simple("samsung", -1, NULL, 0);
+	pdev = platform_device_register_simple("samsung", PLATFORM_DEVID_NONE, NULL, 0);
 	if (IS_ERR(pdev))
 		return PTR_ERR(pdev);
 
@@ -1542,76 +1532,6 @@ static const struct dmi_system_id samsung_dmi_table[] __initconst = {
 	/* Specific DMI ids for laptop with quirks */
 	{
 	 .callback = samsung_dmi_matched,
-	 .ident = "N150P",
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
-		DMI_MATCH(DMI_PRODUCT_NAME, "N150P"),
-		DMI_MATCH(DMI_BOARD_NAME, "N150P"),
-		},
-	 .driver_data = &samsung_use_native_backlight,
-	},
-	{
-	 .callback = samsung_dmi_matched,
-	 .ident = "N145P/N250P/N260P",
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
-		DMI_MATCH(DMI_PRODUCT_NAME, "N145P/N250P/N260P"),
-		DMI_MATCH(DMI_BOARD_NAME, "N145P/N250P/N260P"),
-		},
-	 .driver_data = &samsung_use_native_backlight,
-	},
-	{
-	 .callback = samsung_dmi_matched,
-	 .ident = "N150/N210/N220",
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
-		DMI_MATCH(DMI_PRODUCT_NAME, "N150/N210/N220"),
-		DMI_MATCH(DMI_BOARD_NAME, "N150/N210/N220"),
-		},
-	 .driver_data = &samsung_broken_acpi_video,
-	},
-	{
-	 .callback = samsung_dmi_matched,
-	 .ident = "NF110/NF210/NF310",
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
-		DMI_MATCH(DMI_PRODUCT_NAME, "NF110/NF210/NF310"),
-		DMI_MATCH(DMI_BOARD_NAME, "NF110/NF210/NF310"),
-		},
-	 .driver_data = &samsung_broken_acpi_video,
-	},
-	{
-	 .callback = samsung_dmi_matched,
-	 .ident = "X360",
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
-		DMI_MATCH(DMI_PRODUCT_NAME, "X360"),
-		DMI_MATCH(DMI_BOARD_NAME, "X360"),
-		},
-	 .driver_data = &samsung_broken_acpi_video,
-	},
-	{
-	 .callback = samsung_dmi_matched,
-	 .ident = "N250P",
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
-		DMI_MATCH(DMI_PRODUCT_NAME, "N250P"),
-		DMI_MATCH(DMI_BOARD_NAME, "N250P"),
-		},
-	 .driver_data = &samsung_use_native_backlight,
-	},
-	{
-	 .callback = samsung_dmi_matched,
-	 .ident = "NC210",
-	 .matches = {
-		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
-		DMI_MATCH(DMI_PRODUCT_NAME, "NC210/NC110"),
-		DMI_MATCH(DMI_BOARD_NAME, "NC210/NC110"),
-		},
-	 .driver_data = &samsung_broken_acpi_video,
-	},
-	{
-	 .callback = samsung_dmi_matched,
 	 .ident = "730U3E/740U3E",
 	 .matches = {
 		DMI_MATCH(DMI_SYS_VENDOR, "SAMSUNG ELECTRONICS CO., LTD."),
@@ -1654,15 +1574,8 @@ static int __init samsung_init(void)
 	samsung->handle_backlight = true;
 	samsung->quirks = quirks;
 
-#ifdef CONFIG_ACPI
-	if (samsung->quirks->broken_acpi_video)
-		acpi_video_set_dmi_backlight_type(acpi_backlight_vendor);
-	if (samsung->quirks->use_native_backlight)
-		acpi_video_set_dmi_backlight_type(acpi_backlight_native);
-
 	if (acpi_video_get_backlight_type() != acpi_backlight_vendor)
 		samsung->handle_backlight = false;
-#endif
 
 	ret = samsung_platform_init(samsung);
 	if (ret)
diff --git a/drivers/platform/x86/simatic-ipc.c b/drivers/platform/x86/simatic-ipc.c
index ca3647b751d5..ca76076fc706 100644
--- a/drivers/platform/x86/simatic-ipc.c
+++ b/drivers/platform/x86/simatic-ipc.c
@@ -41,10 +41,12 @@ static struct {
 	{SIMATIC_IPC_IPC127E, SIMATIC_IPC_DEVICE_127E, SIMATIC_IPC_DEVICE_NONE},
 	{SIMATIC_IPC_IPC227D, SIMATIC_IPC_DEVICE_227D, SIMATIC_IPC_DEVICE_NONE},
 	{SIMATIC_IPC_IPC227E, SIMATIC_IPC_DEVICE_427E, SIMATIC_IPC_DEVICE_227E},
+	{SIMATIC_IPC_IPC227G, SIMATIC_IPC_DEVICE_227G, SIMATIC_IPC_DEVICE_227G},
 	{SIMATIC_IPC_IPC277E, SIMATIC_IPC_DEVICE_NONE, SIMATIC_IPC_DEVICE_227E},
 	{SIMATIC_IPC_IPC427D, SIMATIC_IPC_DEVICE_427E, SIMATIC_IPC_DEVICE_NONE},
 	{SIMATIC_IPC_IPC427E, SIMATIC_IPC_DEVICE_427E, SIMATIC_IPC_DEVICE_427E},
 	{SIMATIC_IPC_IPC477E, SIMATIC_IPC_DEVICE_NONE, SIMATIC_IPC_DEVICE_427E},
+	{SIMATIC_IPC_IPC427G, SIMATIC_IPC_DEVICE_227G, SIMATIC_IPC_DEVICE_227G},
 };
 
 static int register_platform_devices(u32 station_id)
@@ -65,7 +67,8 @@ static int register_platform_devices(u32 station_id)
 	}
 
 	if (ledmode != SIMATIC_IPC_DEVICE_NONE) {
-		if (ledmode == SIMATIC_IPC_DEVICE_127E)
+		if (ledmode == SIMATIC_IPC_DEVICE_127E ||
+		    ledmode == SIMATIC_IPC_DEVICE_227G)
 			pdevname = KBUILD_MODNAME "_leds_gpio";
 		platform_data.devmode = ledmode;
 		ipc_led_platform_device =
@@ -80,6 +83,11 @@ static int register_platform_devices(u32 station_id)
 			 ipc_led_platform_device->name);
 	}
 
+	if (wdtmode == SIMATIC_IPC_DEVICE_227G) {
+		request_module("w83627hf_wdt");
+		return 0;
+	}
+
 	if (wdtmode != SIMATIC_IPC_DEVICE_NONE) {
 		platform_data.devmode = wdtmode;
 		ipc_wdt_platform_device =
diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c
index 07ef05f727a2..765fcaba4d12 100644
--- a/drivers/platform/x86/sony-laptop.c
+++ b/drivers/platform/x86/sony-laptop.c
@@ -584,7 +584,7 @@ static int sony_pf_add(void)
 	if (ret)
 		goto out;
 
-	sony_pf_device = platform_device_alloc("sony-laptop", -1);
+	sony_pf_device = platform_device_alloc("sony-laptop", PLATFORM_DEVID_NONE);
 	if (!sony_pf_device) {
 		ret = -ENOMEM;
 		goto out_platform_registered;
diff --git a/drivers/platform/x86/tc1100-wmi.c b/drivers/platform/x86/tc1100-wmi.c
index 9072eb302618..ded26213c420 100644
--- a/drivers/platform/x86/tc1100-wmi.c
+++ b/drivers/platform/x86/tc1100-wmi.c
@@ -233,7 +233,7 @@ static int __init tc1100_init(void)
 	if (!wmi_has_guid(GUID))
 		return -ENODEV;
 
-	tc1100_device = platform_device_alloc("tc1100-wmi", -1);
+	tc1100_device = platform_device_alloc("tc1100-wmi", PLATFORM_DEVID_NONE);
 	if (!tc1100_device)
 		return -ENOMEM;
 
diff --git a/drivers/platform/x86/thinkpad_acpi.c b/drivers/platform/x86/thinkpad_acpi.c
index 2dbb9fc011a7..6a823b850a77 100644
--- a/drivers/platform/x86/thinkpad_acpi.c
+++ b/drivers/platform/x86/thinkpad_acpi.c
@@ -7623,9 +7623,9 @@ static int __init volume_create_alsa_mixer(void)
 	data = card->private_data;
 	data->card = card;
 
-	strlcpy(card->driver, TPACPI_ALSA_DRVNAME,
+	strscpy(card->driver, TPACPI_ALSA_DRVNAME,
 		sizeof(card->driver));
-	strlcpy(card->shortname, TPACPI_ALSA_SHRTNAME,
+	strscpy(card->shortname, TPACPI_ALSA_SHRTNAME,
 		sizeof(card->shortname));
 	snprintf(card->mixername, sizeof(card->mixername), "ThinkPad EC %s",
 		 (thinkpad_id.ec_version_str) ?
@@ -11715,7 +11715,7 @@ static int __init thinkpad_acpi_module_init(void)
 		tp_features.quirks = dmi_id->driver_data;
 
 	/* Device initialization */
-	tpacpi_pdev = platform_device_register_simple(TPACPI_DRVR_NAME, -1,
+	tpacpi_pdev = platform_device_register_simple(TPACPI_DRVR_NAME, PLATFORM_DEVID_NONE,
 							NULL, 0);
 	if (IS_ERR(tpacpi_pdev)) {
 		ret = PTR_ERR(tpacpi_pdev);
@@ -11726,7 +11726,7 @@ static int __init thinkpad_acpi_module_init(void)
 	}
 	tpacpi_sensors_pdev = platform_device_register_simple(
 						TPACPI_HWMON_DRVR_NAME,
-						-1, NULL, 0);
+						PLATFORM_DEVID_NONE, NULL, 0);
 	if (IS_ERR(tpacpi_sensors_pdev)) {
 		ret = PTR_ERR(tpacpi_sensors_pdev);
 		tpacpi_sensors_pdev = NULL;
diff --git a/drivers/platform/x86/topstar-laptop.c b/drivers/platform/x86/topstar-laptop.c
index f7761d98c0fd..6d18fbf8762b 100644
--- a/drivers/platform/x86/topstar-laptop.c
+++ b/drivers/platform/x86/topstar-laptop.c
@@ -192,7 +192,7 @@ static int topstar_platform_init(struct topstar_laptop *topstar)
 {
 	int err;
 
-	topstar->platform = platform_device_alloc(TOPSTAR_LAPTOP_CLASS, -1);
+	topstar->platform = platform_device_alloc(TOPSTAR_LAPTOP_CLASS, PLATFORM_DEVID_NONE);
 	if (!topstar->platform)
 		return -ENOMEM;
 
diff --git a/drivers/platform/x86/toshiba_acpi.c b/drivers/platform/x86/toshiba_acpi.c
index 0fc9e8b8827b..160abd3b3af8 100644
--- a/drivers/platform/x86/toshiba_acpi.c
+++ b/drivers/platform/x86/toshiba_acpi.c
@@ -23,6 +23,7 @@
 #define PROC_INTERFACE_VERSION	1
 
 #include <linux/compiler.h>
+#include <linux/dmi.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
@@ -38,18 +39,24 @@
 #include <linux/workqueue.h>
 #include <linux/i8042.h>
 #include <linux/acpi.h>
-#include <linux/dmi.h>
 #include <linux/uaccess.h>
 #include <linux/miscdevice.h>
 #include <linux/rfkill.h>
+#include <linux/hwmon.h>
 #include <linux/iio/iio.h>
 #include <linux/toshiba.h>
+#include <acpi/battery.h>
 #include <acpi/video.h>
 
 MODULE_AUTHOR("John Belmonte");
 MODULE_DESCRIPTION("Toshiba Laptop ACPI Extras Driver");
 MODULE_LICENSE("GPL");
 
+static int turn_on_panel_on_resume = -1;
+module_param(turn_on_panel_on_resume, int, 0644);
+MODULE_PARM_DESC(turn_on_panel_on_resume,
+	"Call HCI_PANEL_POWER_ON on resume (-1 = auto, 0 = no, 1 = yes");
+
 #define TOSHIBA_WMI_EVENT_GUID "59142400-C6A3-40FA-BADB-8A2652834100"
 
 /* Scan code for Fn key on TOS1900 models */
@@ -100,18 +107,21 @@ MODULE_LICENSE("GPL");
 #define TOS_NOT_INSTALLED		0x8e00
 
 /* Registers */
+#define HCI_PANEL_POWER_ON		0x0002
 #define HCI_FAN				0x0004
 #define HCI_TR_BACKLIGHT		0x0005
 #define HCI_SYSTEM_EVENT		0x0016
 #define HCI_VIDEO_OUT			0x001c
 #define HCI_HOTKEY_EVENT		0x001e
 #define HCI_LCD_BRIGHTNESS		0x002a
+#define HCI_FAN_RPM			0x0045
 #define HCI_WIRELESS			0x0056
 #define HCI_ACCELEROMETER		0x006d
 #define HCI_COOLING_METHOD		0x007f
 #define HCI_KBD_ILLUMINATION		0x0095
 #define HCI_ECO_MODE			0x0097
 #define HCI_ACCELEROMETER2		0x00a6
+#define HCI_BATTERY_CHARGE_MODE		0x00ba
 #define HCI_SYSTEM_INFO			0xc000
 #define SCI_PANEL_POWER_ON		0x010d
 #define SCI_ILLUMINATION		0x014e
@@ -170,6 +180,9 @@ struct toshiba_acpi_dev {
 	struct miscdevice miscdev;
 	struct rfkill *wwan_rfk;
 	struct iio_dev *indio_dev;
+#if IS_ENABLED(CONFIG_HWMON)
+	struct device *hwmon_device;
+#endif
 
 	int force_fan;
 	int last_key_event;
@@ -185,6 +198,7 @@ struct toshiba_acpi_dev {
 	unsigned int illumination_supported:1;
 	unsigned int video_supported:1;
 	unsigned int fan_supported:1;
+	unsigned int fan_rpm_supported:1;
 	unsigned int system_event_supported:1;
 	unsigned int ntfy_supported:1;
 	unsigned int info_supported:1;
@@ -201,6 +215,7 @@ struct toshiba_acpi_dev {
 	unsigned int usb_three_supported:1;
 	unsigned int wwan_supported:1;
 	unsigned int cooling_method_supported:1;
+	unsigned int battery_charge_mode_supported:1;
 	unsigned int sysfs_created:1;
 	unsigned int special_functions;
 
@@ -272,14 +287,6 @@ static const struct key_entry toshiba_acpi_alt_keymap[] = {
 };
 
 /*
- * List of models which have a broken acpi-video backlight interface and thus
- * need to use the toshiba (vendor) interface instead.
- */
-static const struct dmi_system_id toshiba_vendor_backlight_dmi[] = {
-	{}
-};
-
-/*
  * Utility
  */
 
@@ -675,12 +682,15 @@ static void toshiba_eco_mode_available(struct toshiba_acpi_dev *dev)
 		return;
 	}
 
-	if (out[0] == TOS_INPUT_DATA_ERROR) {
+	if (out[0] == TOS_INPUT_DATA_ERROR || out[0] == TOS_NOT_SUPPORTED) {
 		/*
 		 * If we receive 0x8300 (Input Data Error), it means that the
 		 * LED device is present, but that we just screwed the input
 		 * parameters.
 		 *
+		 * On some laptops 0x8000 (Not supported) is also returned in
+		 * this case, so we need to allow for that as well.
+		 *
 		 * Let's query the status of the LED to see if we really have a
 		 * success response, indicating the actual presense of the LED,
 		 * bail out otherwise.
@@ -1282,6 +1292,69 @@ static int toshiba_cooling_method_set(struct toshiba_acpi_dev *dev, u32 state)
 	return (result == TOS_SUCCESS || result == TOS_SUCCESS2) ? 0 : -EIO;
 }
 
+/* Battery charge control */
+static void toshiba_battery_charge_mode_available(struct toshiba_acpi_dev *dev)
+{
+	u32 in[TCI_WORDS] = { HCI_GET, HCI_BATTERY_CHARGE_MODE, 0, 0, 0, 0 };
+	u32 out[TCI_WORDS];
+	acpi_status status;
+
+	dev->battery_charge_mode_supported = 0;
+
+	status = tci_raw(dev, in, out);
+	if (ACPI_FAILURE(status)) {
+		pr_err("ACPI call to get Battery Charge Mode failed\n");
+		return;
+	}
+
+	if (out[0] != TOS_SUCCESS && out[0] != TOS_SUCCESS2)
+		return;
+
+	dev->battery_charge_mode_supported = 1;
+}
+
+static int toshiba_battery_charge_mode_get(struct toshiba_acpi_dev *dev, u32 *state)
+{
+	u32 in[TCI_WORDS] = { HCI_GET, HCI_BATTERY_CHARGE_MODE, 0, 0, 0, 0x1 };
+	u32 out[TCI_WORDS];
+	int retries = 3;
+
+	do {
+		acpi_status status = tci_raw(dev, in, out);
+
+		if (ACPI_FAILURE(status))
+			pr_err("ACPI call to get Battery Charge Mode failed\n");
+		switch (out[0]) {
+		case TOS_SUCCESS:
+		case TOS_SUCCESS2:
+			*state = out[2];
+			return 0;
+		case TOS_NOT_SUPPORTED:
+			return -ENODEV;
+		case TOS_DATA_NOT_AVAILABLE:
+			retries--;
+			break;
+		default:
+			return -EIO;
+		}
+	} while (retries);
+
+	return -EIO;
+}
+
+static int toshiba_battery_charge_mode_set(struct toshiba_acpi_dev *dev, u32 state)
+{
+	u32 result = hci_write(dev, HCI_BATTERY_CHARGE_MODE, state);
+
+	if (result == TOS_FAILURE)
+		pr_err("ACPI call to set Battery Charge Mode failed\n");
+
+	if (result == TOS_NOT_SUPPORTED)
+		return -ENODEV;
+
+	return (result == TOS_SUCCESS || result == TOS_SUCCESS2) ? 0 : -EIO;
+}
+
 /* Transflective Backlight */
 static int get_tr_backlight_status(struct toshiba_acpi_dev *dev, u32 *status)
 {
@@ -1616,6 +1689,29 @@ static const struct proc_ops fan_proc_ops = {
 	.proc_write	= fan_proc_write,
 };
 
+/* Fan RPM */
+static int get_fan_rpm(struct toshiba_acpi_dev *dev, u32 *rpm)
+{
+	u32 in[TCI_WORDS] = { HCI_GET, HCI_FAN_RPM, 0, 1, 0, 0 };
+	u32 out[TCI_WORDS];
+	acpi_status status = tci_raw(dev, in, out);
+
+	if (ACPI_FAILURE(status)) {
+		pr_err("ACPI call to get Fan speed failed\n");
+		return -EIO;
+	}
+
+	if (out[0] == TOS_NOT_SUPPORTED)
+		return -ENODEV;
+
+	if (out[0] == TOS_SUCCESS) {
+		*rpm = out[2];
+		return 0;
+	}
+
+	return -EIO;
+}
+
 static int keys_proc_show(struct seq_file *m, void *v)
 {
 	struct toshiba_acpi_dev *dev = m->private;
@@ -2786,6 +2882,7 @@ static int toshiba_acpi_setup_keyboard(struct toshiba_acpi_dev *dev)
 	dev->hotkey_dev->name = "Toshiba input device";
 	dev->hotkey_dev->phys = "toshiba_acpi/input0";
 	dev->hotkey_dev->id.bustype = BUS_HOST;
+	dev->hotkey_dev->dev.parent = &dev->acpi_dev->dev;
 
 	if (dev->hotkey_event_type == HCI_SYSTEM_TYPE1 ||
 	    !dev->kbd_function_keys_supported)
@@ -2881,14 +2978,6 @@ static int toshiba_acpi_setup_backlight(struct toshiba_acpi_dev *dev)
 		return 0;
 	}
 
-	/*
-	 * Tell acpi-video-detect code to prefer vendor backlight on all
-	 * systems with transflective backlight and on dmi matched systems.
-	 */
-	if (dev->tr_backlight_supported ||
-	    dmi_check_system(toshiba_vendor_backlight_dmi))
-		acpi_video_set_dmi_backlight_type(acpi_backlight_vendor);
-
 	if (acpi_video_get_backlight_type() != acpi_backlight_vendor)
 		return 0;
 
@@ -2916,6 +3005,139 @@ static int toshiba_acpi_setup_backlight(struct toshiba_acpi_dev *dev)
 	return 0;
 }
 
+/* HWMON support for fan */
+#if IS_ENABLED(CONFIG_HWMON)
+static umode_t toshiba_acpi_hwmon_is_visible(const void *drvdata,
+					     enum hwmon_sensor_types type,
+					     u32 attr, int channel)
+{
+	return 0444;
+}
+
+static int toshiba_acpi_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
+				   u32 attr, int channel, long *val)
+{
+	/*
+	 * There is only a single channel and single attribute (for the
+	 * fan) at this point.
+	 * This can be replaced with more advanced logic in the future,
+	 * should the need arise.
+	 */
+	if (type == hwmon_fan && channel == 0 && attr == hwmon_fan_input) {
+		u32 value;
+		int ret;
+
+		ret = get_fan_rpm(toshiba_acpi, &value);
+		if (ret)
+			return ret;
+
+		*val = value;
+		return 0;
+	}
+	return -EOPNOTSUPP;
+}
+
+static const struct hwmon_channel_info *toshiba_acpi_hwmon_info[] = {
+	HWMON_CHANNEL_INFO(fan, HWMON_F_INPUT),
+	NULL
+};
+
+static const struct hwmon_ops toshiba_acpi_hwmon_ops = {
+	.is_visible = toshiba_acpi_hwmon_is_visible,
+	.read = toshiba_acpi_hwmon_read,
+};
+
+static const struct hwmon_chip_info toshiba_acpi_hwmon_chip_info = {
+	.ops = &toshiba_acpi_hwmon_ops,
+	.info = toshiba_acpi_hwmon_info,
+};
+#endif
+
+/* ACPI battery hooking */
+static ssize_t charge_control_end_threshold_show(struct device *device,
+						 struct device_attribute *attr,
+						 char *buf)
+{
+	u32 state;
+	int status;
+
+	if (toshiba_acpi == NULL) {
+		pr_err("Toshiba ACPI object invalid\n");
+		return -ENODEV;
+	}
+
+	status = toshiba_battery_charge_mode_get(toshiba_acpi, &state);
+
+	if (status != 0)
+		return status;
+
+	if (state == 1)
+		return sprintf(buf, "80\n");
+	else
+		return sprintf(buf, "100\n");
+}
+
+static ssize_t charge_control_end_threshold_store(struct device *dev,
+						  struct device_attribute *attr,
+						  const char *buf,
+						  size_t count)
+{
+	u32 value;
+	int rval;
+
+	if (toshiba_acpi == NULL) {
+		pr_err("Toshiba ACPI object invalid\n");
+		return -ENODEV;
+	}
+
+	rval = kstrtou32(buf, 10, &value);
+	if (rval)
+		return rval;
+
+	if (value < 1 || value > 100)
+		return -EINVAL;
+	rval = toshiba_battery_charge_mode_set(toshiba_acpi,
+					       (value < 90) ? 1 : 0);
+	if (rval < 0)
+		return rval;
+	else
+		return count;
+}
+
+static DEVICE_ATTR_RW(charge_control_end_threshold);
+
+static struct attribute *toshiba_acpi_battery_attrs[] = {
+	&dev_attr_charge_control_end_threshold.attr,
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(toshiba_acpi_battery);
+
+static int toshiba_acpi_battery_add(struct power_supply *battery)
+{
+	if (toshiba_acpi == NULL) {
+		pr_err("Init order issue\n");
+		return -ENODEV;
+	}
+	if (!toshiba_acpi->battery_charge_mode_supported)
+		return -ENODEV;
+	if (device_add_groups(&battery->dev, toshiba_acpi_battery_groups))
+		return -ENODEV;
+	return 0;
+}
+
+static int toshiba_acpi_battery_remove(struct power_supply *battery)
+{
+	device_remove_groups(&battery->dev, toshiba_acpi_battery_groups);
+	return 0;
+}
+
+static struct acpi_battery_hook battery_hook = {
+	.add_battery = toshiba_acpi_battery_add,
+	.remove_battery = toshiba_acpi_battery_remove,
+	.name = "Toshiba Battery Extension",
+};
+
 static void print_supported_features(struct toshiba_acpi_dev *dev)
 {
 	pr_info("Supported laptop features:");
@@ -2928,6 +3150,8 @@ static void print_supported_features(struct toshiba_acpi_dev *dev)
 		pr_cont(" video-out");
 	if (dev->fan_supported)
 		pr_cont(" fan");
+	if (dev->fan_rpm_supported)
+		pr_cont(" fan-rpm");
 	if (dev->tr_backlight_supported)
 		pr_cont(" transflective-backlight");
 	if (dev->illumination_supported)
@@ -2956,6 +3180,8 @@ static void print_supported_features(struct toshiba_acpi_dev *dev)
 		pr_cont(" wwan");
 	if (dev->cooling_method_supported)
 		pr_cont(" cooling-method");
+	if (dev->battery_charge_mode_supported)
+		pr_cont(" battery-charge-mode");
 
 	pr_cont("\n");
 }
@@ -2968,6 +3194,11 @@ static int toshiba_acpi_remove(struct acpi_device *acpi_dev)
 
 	remove_toshiba_proc_entries(dev);
 
+#if IS_ENABLED(CONFIG_HWMON)
+	if (dev->hwmon_device)
+		hwmon_device_unregister(dev->hwmon_device);
+#endif
+
 	if (dev->accelerometer_supported && dev->indio_dev) {
 		iio_device_unregister(dev->indio_dev);
 		iio_device_free(dev->indio_dev);
@@ -2996,6 +3227,9 @@ static int toshiba_acpi_remove(struct acpi_device *acpi_dev)
 		rfkill_destroy(dev->wwan_rfk);
 	}
 
+	if (dev->battery_charge_mode_supported)
+		battery_hook_unregister(&battery_hook);
+
 	if (toshiba_acpi)
 		toshiba_acpi = NULL;
 
@@ -3015,6 +3249,43 @@ static const char *find_hci_method(acpi_handle handle)
 	return NULL;
 }
 
+/*
+ * Some Toshibas have a broken acpi-video interface for brightness control,
+ * these are quirked in drivers/acpi/video_detect.c to use the GPU native
+ * (/sys/class/backlight/intel_backlight) instead.
+ * But these need a HCI_SET call to actually turn the panel back on at resume,
+ * without this call the screen stays black at resume.
+ * Either HCI_LCD_BRIGHTNESS (used by acpi_video's _BCM) or HCI_PANEL_POWER_ON
+ * works. toshiba_acpi_resume() uses HCI_PANEL_POWER_ON to avoid changing
+ * the configured brightness level.
+ */
+static const struct dmi_system_id turn_on_panel_on_resume_dmi_ids[] = {
+	{
+	 /* Toshiba Portégé R700 */
+	 /* https://bugzilla.kernel.org/show_bug.cgi?id=21012 */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "PORTEGE R700"),
+		},
+	},
+	{
+	 /* Toshiba Satellite/Portégé R830 */
+	 /* Portégé: https://bugs.freedesktop.org/show_bug.cgi?id=82634 */
+	 /* Satellite: https://bugzilla.kernel.org/show_bug.cgi?id=21012 */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "R830"),
+		},
+	},
+	{
+	 /* Toshiba Satellite/Portégé Z830 */
+	 .matches = {
+		DMI_MATCH(DMI_SYS_VENDOR, "TOSHIBA"),
+		DMI_MATCH(DMI_PRODUCT_NAME, "Z830"),
+		},
+	},
+};
+
 static int toshiba_acpi_add(struct acpi_device *acpi_dev)
 {
 	struct toshiba_acpi_dev *dev;
@@ -3157,12 +3428,32 @@ iio_error:
 	ret = get_fan_status(dev, &dummy);
 	dev->fan_supported = !ret;
 
+	ret = get_fan_rpm(dev, &dummy);
+	dev->fan_rpm_supported = !ret;
+
+#if IS_ENABLED(CONFIG_HWMON)
+	if (dev->fan_rpm_supported) {
+		dev->hwmon_device = hwmon_device_register_with_info(
+			&dev->acpi_dev->dev, "toshiba_acpi_sensors", NULL,
+			&toshiba_acpi_hwmon_chip_info, NULL);
+		if (IS_ERR(dev->hwmon_device)) {
+			dev->hwmon_device = NULL;
+			pr_warn("unable to register hwmon device, skipping\n");
+		}
+	}
+#endif
+
+	if (turn_on_panel_on_resume == -1)
+		turn_on_panel_on_resume = dmi_check_system(turn_on_panel_on_resume_dmi_ids);
+
 	toshiba_wwan_available(dev);
 	if (dev->wwan_supported)
 		toshiba_acpi_setup_wwan_rfkill(dev);
 
 	toshiba_cooling_method_available(dev);
 
+	toshiba_battery_charge_mode_available(dev);
+
 	print_supported_features(dev);
 
 	ret = sysfs_create_group(&dev->acpi_dev->dev.kobj,
@@ -3177,6 +3468,13 @@ iio_error:
 
 	toshiba_acpi = dev;
 
+	/*
+	 * As the battery hook relies on the static variable toshiba_acpi being
+	 * set, this must be done after toshiba_acpi is assigned.
+	 */
+	if (dev->battery_charge_mode_supported)
+		battery_hook_register(&battery_hook);
+
 	return 0;
 
 error:
@@ -3273,6 +3571,9 @@ static int toshiba_acpi_resume(struct device *device)
 			rfkill_set_hw_state(dev->wwan_rfk, !dev->killswitch);
 	}
 
+	if (turn_on_panel_on_resume)
+		hci_write(dev, HCI_PANEL_POWER_ON, 1);
+
 	return 0;
 }
 #endif
diff --git a/drivers/platform/x86/winmate-fm07-keys.c b/drivers/platform/x86/winmate-fm07-keys.c
index 2c90c5c7eca2..465ffad81a65 100644
--- a/drivers/platform/x86/winmate-fm07-keys.c
+++ b/drivers/platform/x86/winmate-fm07-keys.c
@@ -161,7 +161,7 @@ static int __init fm07keys_init(void)
 		return ret;
 	}
 
-	dev = platform_device_register_simple(DRV_NAME, -1, NULL, 0);
+	dev = platform_device_register_simple(DRV_NAME, PLATFORM_DEVID_NONE, NULL, 0);
 	if (IS_ERR(dev)) {
 		ret = PTR_ERR(dev);
 		pr_err("fm07keys: failed to allocate device, err = %d\n", ret);
diff --git a/drivers/platform/x86/wmi.c b/drivers/platform/x86/wmi.c
index aed293b5af81..223550a10d4d 100644
--- a/drivers/platform/x86/wmi.c
+++ b/drivers/platform/x86/wmi.c
@@ -95,9 +95,6 @@ module_param(debug_dump_wdg, bool, 0444);
 MODULE_PARM_DESC(debug_dump_wdg,
 		 "Dump available WMI interfaces [0/1]");
 
-static int acpi_wmi_remove(struct platform_device *device);
-static int acpi_wmi_probe(struct platform_device *device);
-
 static const struct acpi_device_id wmi_device_ids[] = {
 	{"PNP0C14", 0},
 	{"pnp0c14", 0},
@@ -105,13 +102,10 @@ static const struct acpi_device_id wmi_device_ids[] = {
 };
 MODULE_DEVICE_TABLE(acpi, wmi_device_ids);
 
-static struct platform_driver acpi_wmi_driver = {
-	.driver = {
-		.name = "acpi-wmi",
-		.acpi_match_table = wmi_device_ids,
-	},
-	.probe = acpi_wmi_probe,
-	.remove = acpi_wmi_remove,
+/* allow duplicate GUIDs as these device drivers use struct wmi_driver */
+static const char * const allow_duplicates[] = {
+	"05901221-D566-11D1-B2F0-00A0C9062910",	/* wmi-bmof */
+	NULL
 };
 
 /*
@@ -1073,6 +1067,23 @@ static const struct device_type wmi_type_data = {
 	.release = wmi_dev_release,
 };
 
+/*
+ * _WDG is a static list that is only parsed at startup,
+ * so it's safe to count entries without extra protection.
+ */
+static int guid_count(const guid_t *guid)
+{
+	struct wmi_block *wblock;
+	int count = 0;
+
+	list_for_each_entry(wblock, &wmi_block_list, list) {
+		if (guid_equal(&wblock->gblock.guid, guid))
+			count++;
+	}
+
+	return count;
+}
+
 static int wmi_create_device(struct device *wmi_bus_dev,
 			     struct wmi_block *wblock,
 			     struct acpi_device *device)
@@ -1080,6 +1091,7 @@ static int wmi_create_device(struct device *wmi_bus_dev,
 	struct acpi_device_info *info;
 	char method[WMI_ACPI_METHOD_NAME_SIZE];
 	int result;
+	uint count;
 
 	if (wblock->gblock.flags & ACPI_WMI_EVENT) {
 		wblock->dev.dev.type = &wmi_type_event;
@@ -1134,7 +1146,11 @@ static int wmi_create_device(struct device *wmi_bus_dev,
 	wblock->dev.dev.bus = &wmi_bus_type;
 	wblock->dev.dev.parent = wmi_bus_dev;
 
-	dev_set_name(&wblock->dev.dev, "%pUL", &wblock->gblock.guid);
+	count = guid_count(&wblock->gblock.guid);
+	if (count)
+		dev_set_name(&wblock->dev.dev, "%pUL-%d", &wblock->gblock.guid, count);
+	else
+		dev_set_name(&wblock->dev.dev, "%pUL", &wblock->gblock.guid);
 
 	device_initialize(&wblock->dev.dev);
 
@@ -1154,11 +1170,20 @@ static void wmi_free_devices(struct acpi_device *device)
 	}
 }
 
-static bool guid_already_parsed(struct acpi_device *device, const guid_t *guid)
+static bool guid_already_parsed_for_legacy(struct acpi_device *device, const guid_t *guid)
 {
 	struct wmi_block *wblock;
 
 	list_for_each_entry(wblock, &wmi_block_list, list) {
+		/* skip warning and register if we know the driver will use struct wmi_driver */
+		for (int i = 0; allow_duplicates[i] != NULL; i++) {
+			guid_t tmp;
+
+			if (guid_parse(allow_duplicates[i], &tmp))
+				continue;
+			if (guid_equal(&tmp, guid))
+				return false;
+		}
 		if (guid_equal(&wblock->gblock.guid, guid)) {
 			/*
 			 * Because we historically didn't track the relationship
@@ -1208,13 +1233,7 @@ static int parse_wdg(struct device *wmi_bus_dev, struct acpi_device *device)
 		if (debug_dump_wdg)
 			wmi_dump_wdg(&gblock[i]);
 
-		/*
-		 * Some WMI devices, like those for nVidia hooks, have a
-		 * duplicate GUID. It's not clear what we should do in this
-		 * case yet, so for now, we'll just ignore the duplicate
-		 * for device creation.
-		 */
-		if (guid_already_parsed(device, &gblock[i].guid))
+		if (guid_already_parsed_for_legacy(device, &gblock[i].guid))
 			continue;
 
 		wblock = kzalloc(sizeof(*wblock), GFP_KERNEL);
@@ -1449,6 +1468,15 @@ void wmi_driver_unregister(struct wmi_driver *driver)
 }
 EXPORT_SYMBOL(wmi_driver_unregister);
 
+static struct platform_driver acpi_wmi_driver = {
+	.driver = {
+		.name = "acpi-wmi",
+		.acpi_match_table = wmi_device_ids,
+	},
+	.probe = acpi_wmi_probe,
+	.remove = acpi_wmi_remove,
+};
+
 static int __init acpi_wmi_init(void)
 {
 	int error;
diff --git a/drivers/pnp/pnpbios/pnpbios.h b/drivers/pnp/pnpbios/pnpbios.h
index 2ce739ff9c1a..f3302006842e 100644
--- a/drivers/pnp/pnpbios/pnpbios.h
+++ b/drivers/pnp/pnpbios/pnpbios.h
@@ -153,7 +153,6 @@ extern int  pnpbios_dont_use_current_config;
 extern int pnpbios_parse_data_stream(struct pnp_dev *dev, struct pnp_bios_node * node);
 extern int pnpbios_read_resources_from_node(struct pnp_dev *dev, struct pnp_bios_node *node);
 extern int pnpbios_write_resources_to_node(struct pnp_dev *dev, struct pnp_bios_node *node);
-extern void pnpid32_to_pnpid(u32 id, char *str);
 
 extern void pnpbios_print_status(const char * module, u16 status);
 extern void pnpbios_calls_init(union pnp_bios_install_struct * header);
diff --git a/drivers/power/supply/bq2415x_charger.c b/drivers/power/supply/bq2415x_charger.c
index 5724001e66b9..6b99e1c675b8 100644
--- a/drivers/power/supply/bq2415x_charger.c
+++ b/drivers/power/supply/bq2415x_charger.c
@@ -1696,7 +1696,7 @@ error_1:
 
 /* main bq2415x remove function */
 
-static int bq2415x_remove(struct i2c_client *client)
+static void bq2415x_remove(struct i2c_client *client)
 {
 	struct bq2415x_device *bq = i2c_get_clientdata(client);
 
@@ -1715,8 +1715,6 @@ static int bq2415x_remove(struct i2c_client *client)
 	dev_info(bq->dev, "driver unregistered\n");
 
 	kfree(bq->name);
-
-	return 0;
 }
 
 static const struct i2c_device_id bq2415x_i2c_id_table[] = {
diff --git a/drivers/power/supply/bq24190_charger.c b/drivers/power/supply/bq24190_charger.c
index 27f5c7648617..2274679c5ddd 100644
--- a/drivers/power/supply/bq24190_charger.c
+++ b/drivers/power/supply/bq24190_charger.c
@@ -1901,7 +1901,7 @@ out_pmrt:
 	return ret;
 }
 
-static int bq24190_remove(struct i2c_client *client)
+static void bq24190_remove(struct i2c_client *client)
 {
 	struct bq24190_dev_info *bdi = i2c_get_clientdata(client);
 	int error;
@@ -1918,8 +1918,6 @@ static int bq24190_remove(struct i2c_client *client)
 		pm_runtime_put_sync(bdi->dev);
 	pm_runtime_dont_use_autosuspend(bdi->dev);
 	pm_runtime_disable(bdi->dev);
-
-	return 0;
 }
 
 static void bq24190_shutdown(struct i2c_client *client)
diff --git a/drivers/power/supply/bq24257_charger.c b/drivers/power/supply/bq24257_charger.c
index ecba9ab86faf..a309bbedfe52 100644
--- a/drivers/power/supply/bq24257_charger.c
+++ b/drivers/power/supply/bq24257_charger.c
@@ -1077,7 +1077,7 @@ static int bq24257_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int bq24257_remove(struct i2c_client *client)
+static void bq24257_remove(struct i2c_client *client)
 {
 	struct bq24257_device *bq = i2c_get_clientdata(client);
 
@@ -1085,8 +1085,6 @@ static int bq24257_remove(struct i2c_client *client)
 		cancel_delayed_work_sync(&bq->iilimit_setup_work);
 
 	bq24257_field_write(bq, F_RESET, 1); /* reset to defaults */
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/power/supply/bq25890_charger.c b/drivers/power/supply/bq25890_charger.c
index 852a6fec4339..06ea7399d151 100644
--- a/drivers/power/supply/bq25890_charger.c
+++ b/drivers/power/supply/bq25890_charger.c
@@ -1258,7 +1258,7 @@ err_unregister_usb_notifier:
 	return ret;
 }
 
-static int bq25890_remove(struct i2c_client *client)
+static void bq25890_remove(struct i2c_client *client)
 {
 	struct bq25890_device *bq = i2c_get_clientdata(client);
 
@@ -1269,8 +1269,6 @@ static int bq25890_remove(struct i2c_client *client)
 		/* reset all registers to default values */
 		bq25890_chip_reset(bq);
 	}
-
-	return 0;
 }
 
 static void bq25890_shutdown(struct i2c_client *client)
diff --git a/drivers/power/supply/bq27xxx_battery_i2c.c b/drivers/power/supply/bq27xxx_battery_i2c.c
index cf38cbfe13e9..94b00bb89c17 100644
--- a/drivers/power/supply/bq27xxx_battery_i2c.c
+++ b/drivers/power/supply/bq27xxx_battery_i2c.c
@@ -205,7 +205,7 @@ err_failed:
 	return ret;
 }
 
-static int bq27xxx_battery_i2c_remove(struct i2c_client *client)
+static void bq27xxx_battery_i2c_remove(struct i2c_client *client)
 {
 	struct bq27xxx_device_info *di = i2c_get_clientdata(client);
 
@@ -214,8 +214,6 @@ static int bq27xxx_battery_i2c_remove(struct i2c_client *client)
 	mutex_lock(&battery_mutex);
 	idr_remove(&battery_id, di->id);
 	mutex_unlock(&battery_mutex);
-
-	return 0;
 }
 
 static const struct i2c_device_id bq27xxx_i2c_id_table[] = {
diff --git a/drivers/power/supply/cw2015_battery.c b/drivers/power/supply/cw2015_battery.c
index 728e2a6cc9c3..81e17ad80163 100644
--- a/drivers/power/supply/cw2015_battery.c
+++ b/drivers/power/supply/cw2015_battery.c
@@ -725,13 +725,12 @@ static int __maybe_unused cw_bat_resume(struct device *dev)
 
 static SIMPLE_DEV_PM_OPS(cw_bat_pm_ops, cw_bat_suspend, cw_bat_resume);
 
-static int cw_bat_remove(struct i2c_client *client)
+static void cw_bat_remove(struct i2c_client *client)
 {
 	struct cw_battery *cw_bat = i2c_get_clientdata(client);
 
 	cancel_delayed_work_sync(&cw_bat->battery_delay_work);
 	power_supply_put_battery_info(cw_bat->rk_bat, cw_bat->battery);
-	return 0;
 }
 
 static const struct i2c_device_id cw_bat_id_table[] = {
diff --git a/drivers/power/supply/ds2782_battery.c b/drivers/power/supply/ds2782_battery.c
index 9ae273fde7a2..d78cd05402f6 100644
--- a/drivers/power/supply/ds2782_battery.c
+++ b/drivers/power/supply/ds2782_battery.c
@@ -312,7 +312,7 @@ static void ds278x_power_supply_init(struct power_supply_desc *battery)
 	battery->external_power_changed	= NULL;
 }
 
-static int ds278x_battery_remove(struct i2c_client *client)
+static void ds278x_battery_remove(struct i2c_client *client)
 {
 	struct ds278x_info *info = i2c_get_clientdata(client);
 	int id = info->id;
@@ -325,8 +325,6 @@ static int ds278x_battery_remove(struct i2c_client *client)
 	mutex_lock(&battery_lock);
 	idr_remove(&battery_id, id);
 	mutex_unlock(&battery_lock);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/power/supply/lp8727_charger.c b/drivers/power/supply/lp8727_charger.c
index 9ee54e397754..384a374b52c1 100644
--- a/drivers/power/supply/lp8727_charger.c
+++ b/drivers/power/supply/lp8727_charger.c
@@ -590,13 +590,12 @@ static int lp8727_probe(struct i2c_client *cl, const struct i2c_device_id *id)
 	return 0;
 }
 
-static int lp8727_remove(struct i2c_client *cl)
+static void lp8727_remove(struct i2c_client *cl)
 {
 	struct lp8727_chg *pchg = i2c_get_clientdata(cl);
 
 	lp8727_release_irq(pchg);
 	lp8727_unregister_psy(pchg);
-	return 0;
 }
 
 static const struct of_device_id lp8727_dt_ids[] = {
diff --git a/drivers/power/supply/rt5033_battery.c b/drivers/power/supply/rt5033_battery.c
index 7a23c70f4879..736dec608ff6 100644
--- a/drivers/power/supply/rt5033_battery.c
+++ b/drivers/power/supply/rt5033_battery.c
@@ -149,13 +149,11 @@ static int rt5033_battery_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int rt5033_battery_remove(struct i2c_client *client)
+static void rt5033_battery_remove(struct i2c_client *client)
 {
 	struct rt5033_battery *battery = i2c_get_clientdata(client);
 
 	power_supply_unregister(battery->psy);
-
-	return 0;
 }
 
 static const struct i2c_device_id rt5033_battery_id[] = {
diff --git a/drivers/power/supply/rt9455_charger.c b/drivers/power/supply/rt9455_charger.c
index 74ee54320e6a..72962286d704 100644
--- a/drivers/power/supply/rt9455_charger.c
+++ b/drivers/power/supply/rt9455_charger.c
@@ -1698,7 +1698,7 @@ put_usb_notifier:
 	return ret;
 }
 
-static int rt9455_remove(struct i2c_client *client)
+static void rt9455_remove(struct i2c_client *client)
 {
 	int ret;
 	struct rt9455_info *info = i2c_get_clientdata(client);
@@ -1715,8 +1715,6 @@ static int rt9455_remove(struct i2c_client *client)
 	cancel_delayed_work_sync(&info->pwr_rdy_work);
 	cancel_delayed_work_sync(&info->max_charging_time_work);
 	cancel_delayed_work_sync(&info->batt_presence_work);
-
-	return 0;
 }
 
 static const struct i2c_device_id rt9455_i2c_id_table[] = {
diff --git a/drivers/power/supply/smb347-charger.c b/drivers/power/supply/smb347-charger.c
index 1511f71f937c..996a82f8a2a1 100644
--- a/drivers/power/supply/smb347-charger.c
+++ b/drivers/power/supply/smb347-charger.c
@@ -1595,14 +1595,12 @@ static int smb347_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int smb347_remove(struct i2c_client *client)
+static void smb347_remove(struct i2c_client *client)
 {
 	struct smb347_charger *smb = i2c_get_clientdata(client);
 
 	smb347_usb_vbus_regulator_disable(smb->usb_rdev);
 	smb347_irq_disable(smb);
-
-	return 0;
 }
 
 static void smb347_shutdown(struct i2c_client *client)
diff --git a/drivers/power/supply/z2_battery.c b/drivers/power/supply/z2_battery.c
index 7ed4e4bb26ec..1897c2984860 100644
--- a/drivers/power/supply/z2_battery.c
+++ b/drivers/power/supply/z2_battery.c
@@ -251,7 +251,7 @@ err:
 	return ret;
 }
 
-static int z2_batt_remove(struct i2c_client *client)
+static void z2_batt_remove(struct i2c_client *client)
 {
 	struct z2_charger *charger = i2c_get_clientdata(client);
 
@@ -263,8 +263,6 @@ static int z2_batt_remove(struct i2c_client *client)
 		free_irq(gpiod_to_irq(charger->charge_gpiod), charger);
 
 	kfree(charger);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c
index 21d624f9f5fb..26d00b1853b4 100644
--- a/drivers/powercap/intel_rapl_common.c
+++ b/drivers/powercap/intel_rapl_common.c
@@ -994,6 +994,9 @@ static u64 rapl_compute_time_window_core(struct rapl_package *rp, u64 value,
 		y = value & 0x1f;
 		value = (1 << y) * (4 + f) * rp->time_unit / 4;
 	} else {
+		if (value < rp->time_unit)
+			return 0;
+
 		do_div(value, rp->time_unit);
 		y = ilog2(value);
 		f = div64_u64(4 * (value - (1 << y)), 1 << y);
@@ -1035,7 +1038,6 @@ static const struct rapl_defaults rapl_defaults_spr_server = {
 	.check_unit = rapl_check_unit_core,
 	.set_floor_freq = set_floor_freq_default,
 	.compute_time_window = rapl_compute_time_window_core,
-	.dram_domain_energy_unit = 15300,
 	.psys_domain_energy_unit = 1000000000,
 	.spr_psys_bits = true,
 };
@@ -1110,6 +1112,7 @@ static const struct x86_cpu_id rapl_ids[] __initconst = {
 	X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_N,		&rapl_defaults_core),
 	X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE,		&rapl_defaults_core),
 	X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_P,        &rapl_defaults_core),
+	X86_MATCH_INTEL_FAM6_MODEL(RAPTORLAKE_S,	&rapl_defaults_core),
 	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X,	&rapl_defaults_spr_server),
 	X86_MATCH_INTEL_FAM6_MODEL(LAKEFIELD,		&rapl_defaults_core),
 
diff --git a/drivers/ptp/ptp_clock.c b/drivers/ptp/ptp_clock.c
index 688cde320bb0..51cae72bb6db 100644
--- a/drivers/ptp/ptp_clock.c
+++ b/drivers/ptp/ptp_clock.c
@@ -174,7 +174,7 @@ static void ptp_clock_release(struct device *dev)
 	mutex_destroy(&ptp->tsevq_mux);
 	mutex_destroy(&ptp->pincfg_mux);
 	mutex_destroy(&ptp->n_vclocks_mux);
-	ida_simple_remove(&ptp_clocks_map, ptp->index);
+	ida_free(&ptp_clocks_map, ptp->index);
 	kfree(ptp);
 }
 
@@ -217,7 +217,7 @@ struct ptp_clock *ptp_clock_register(struct ptp_clock_info *info,
 	if (ptp == NULL)
 		goto no_memory;
 
-	index = ida_simple_get(&ptp_clocks_map, 0, MINORMASK + 1, GFP_KERNEL);
+	index = ida_alloc_max(&ptp_clocks_map, MINORMASK, GFP_KERNEL);
 	if (index < 0) {
 		err = index;
 		goto no_slot;
@@ -332,7 +332,7 @@ kworker_err:
 	mutex_destroy(&ptp->tsevq_mux);
 	mutex_destroy(&ptp->pincfg_mux);
 	mutex_destroy(&ptp->n_vclocks_mux);
-	ida_simple_remove(&ptp_clocks_map, index);
+	ida_free(&ptp_clocks_map, index);
 no_slot:
 	kfree(ptp);
 no_memory:
diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c
index e59ea2173aac..d36c3f597f77 100644
--- a/drivers/ptp/ptp_ocp.c
+++ b/drivers/ptp/ptp_ocp.c
@@ -1311,12 +1311,6 @@ fail:
 	goto out;
 }
 
-static int
-ptp_ocp_firstchild(struct device *dev, void *data)
-{
-	return 1;
-}
-
 static struct device *
 ptp_ocp_find_flash(struct ptp_ocp *bp)
 {
@@ -1325,7 +1319,7 @@ ptp_ocp_find_flash(struct ptp_ocp *bp)
 	last = NULL;
 	dev = &bp->spi_flash->dev;
 
-	while ((dev = device_find_child(dev, NULL, ptp_ocp_firstchild))) {
+	while ((dev = device_find_any_child(dev))) {
 		if (!strcmp("mtd", dev_bus_name(dev)))
 			break;
 		put_device(last);
diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c
index 0e042410f6b9..cfe3a0327471 100644
--- a/drivers/pwm/core.c
+++ b/drivers/pwm/core.c
@@ -734,8 +734,8 @@ static struct device_link *pwm_device_link_add(struct device *dev,
  * Returns: A pointer to the requested PWM device or an ERR_PTR()-encoded
  * error code on failure.
  */
-struct pwm_device *of_pwm_get(struct device *dev, struct device_node *np,
-			      const char *con_id)
+static struct pwm_device *of_pwm_get(struct device *dev, struct device_node *np,
+				     const char *con_id)
 {
 	struct pwm_device *pwm = NULL;
 	struct of_phandle_args args;
@@ -797,7 +797,6 @@ put:
 
 	return pwm;
 }
-EXPORT_SYMBOL_GPL(of_pwm_get);
 
 /**
  * acpi_pwm_get() - request a PWM via parsing "pwms" property in ACPI
@@ -1071,36 +1070,6 @@ struct pwm_device *devm_pwm_get(struct device *dev, const char *con_id)
 EXPORT_SYMBOL_GPL(devm_pwm_get);
 
 /**
- * devm_of_pwm_get() - resource managed of_pwm_get()
- * @dev: device for PWM consumer
- * @np: device node to get the PWM from
- * @con_id: consumer name
- *
- * This function performs like of_pwm_get() but the acquired PWM device will
- * automatically be released on driver detach.
- *
- * Returns: A pointer to the requested PWM device or an ERR_PTR()-encoded
- * error code on failure.
- */
-struct pwm_device *devm_of_pwm_get(struct device *dev, struct device_node *np,
-				   const char *con_id)
-{
-	struct pwm_device *pwm;
-	int ret;
-
-	pwm = of_pwm_get(dev, np, con_id);
-	if (IS_ERR(pwm))
-		return pwm;
-
-	ret = devm_add_action_or_reset(dev, devm_pwm_release, pwm);
-	if (ret)
-		return ERR_PTR(ret);
-
-	return pwm;
-}
-EXPORT_SYMBOL_GPL(devm_of_pwm_get);
-
-/**
  * devm_fwnode_pwm_get() - request a resource managed PWM from firmware node
  * @dev: device for PWM consumer
  * @fwnode: firmware node to get the PWM from
diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c
index c91fa7f9e33d..f230c10d28bb 100644
--- a/drivers/pwm/pwm-pca9685.c
+++ b/drivers/pwm/pwm-pca9685.c
@@ -598,7 +598,7 @@ static int pca9685_pwm_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int pca9685_pwm_remove(struct i2c_client *client)
+static void pca9685_pwm_remove(struct i2c_client *client)
 {
 	struct pca9685 *pca = i2c_get_clientdata(client);
 
@@ -610,8 +610,6 @@ static int pca9685_pwm_remove(struct i2c_client *client)
 	}
 
 	pm_runtime_disable(&client->dev);
-
-	return 0;
 }
 
 static int __maybe_unused pca9685_pwm_runtime_suspend(struct device *dev)
diff --git a/drivers/ras/cec.c b/drivers/ras/cec.c
index 42f2fc0bc8a9..321af498ee11 100644
--- a/drivers/ras/cec.c
+++ b/drivers/ras/cec.c
@@ -556,6 +556,14 @@ static int __init cec_init(void)
 	if (ce_arr.disabled)
 		return -ENODEV;
 
+	/*
+	 * Intel systems may avoid uncorrectable errors
+	 * if pages with corrected errors are aggressively
+	 * taken offline.
+	 */
+	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
+		action_threshold = 2;
+
 	ce_arr.array = (void *)get_zeroed_page(GFP_KERNEL);
 	if (!ce_arr.array) {
 		pr_err("Error allocating CE array page!\n");
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index 23e3e4a35cc9..d663ab9670fe 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -787,6 +787,24 @@ config REGULATOR_MT6323
 	  This driver supports the control of different power rails of device
 	  through regulator interface.
 
+config REGULATOR_MT6331
+	tristate "MediaTek MT6331 PMIC"
+	depends on MFD_MT6397
+	help
+	  Say y here to select this option to enable the power regulator of
+	  MediaTek MT6331 PMIC.
+	  This driver supports the control of different power rails of device
+	  through regulator interface
+
+config REGULATOR_MT6332
+	tristate "MediaTek MT6332 PMIC"
+	depends on MFD_MT6397
+	help
+	  Say y here to select this option to enable the power regulator of
+	  MediaTek MT6332 PMIC.
+	  This driver supports the control of different power rails of device
+	  through regulator interface
+
 config REGULATOR_MT6358
 	tristate "MediaTek MT6358 PMIC"
 	depends on MFD_MT6397
@@ -1384,6 +1402,15 @@ config REGULATOR_TPS65218
 	  voltage regulators. It supports software based voltage control
 	  for different voltage domains
 
+config REGULATOR_TPS65219
+	tristate "TI TPS65219 Power regulators"
+	depends on MFD_TPS65219 && OF
+	help
+	  This driver supports TPS65219 voltage regulator chips.
+	  TPS65219 series of PMICs have 3 single phase BUCKs & 4 LDOs
+	  voltage regulators. It supports software based voltage control
+	  for different voltage domains.
+
 config REGULATOR_TPS6524X
 	tristate "TI TPS6524X Power regulators"
 	depends on SPI
diff --git a/drivers/regulator/Makefile b/drivers/regulator/Makefile
index fa49bb6cc544..5962307e1130 100644
--- a/drivers/regulator/Makefile
+++ b/drivers/regulator/Makefile
@@ -95,6 +95,8 @@ obj-$(CONFIG_REGULATOR_MPQ7920) += mpq7920.o
 obj-$(CONFIG_REGULATOR_MT6311) += mt6311-regulator.o
 obj-$(CONFIG_REGULATOR_MT6315) += mt6315-regulator.o
 obj-$(CONFIG_REGULATOR_MT6323)	+= mt6323-regulator.o
+obj-$(CONFIG_REGULATOR_MT6331)	+= mt6331-regulator.o
+obj-$(CONFIG_REGULATOR_MT6332)	+= mt6332-regulator.o
 obj-$(CONFIG_REGULATOR_MT6358)	+= mt6358-regulator.o
 obj-$(CONFIG_REGULATOR_MT6359)	+= mt6359-regulator.o
 obj-$(CONFIG_REGULATOR_MT6360) += mt6360-regulator.o
@@ -162,6 +164,7 @@ obj-$(CONFIG_REGULATOR_TPS65086) += tps65086-regulator.o
 obj-$(CONFIG_REGULATOR_TPS65090) += tps65090-regulator.o
 obj-$(CONFIG_REGULATOR_TPS65217) += tps65217-regulator.o
 obj-$(CONFIG_REGULATOR_TPS65218) += tps65218-regulator.o
+obj-$(CONFIG_REGULATOR_TPS65219) += tps65219-regulator.o
 obj-$(CONFIG_REGULATOR_TPS6524X) += tps6524x-regulator.o
 obj-$(CONFIG_REGULATOR_TPS6586X) += tps6586x-regulator.o
 obj-$(CONFIG_REGULATOR_TPS65910) += tps65910-regulator.o
diff --git a/drivers/regulator/bd71815-regulator.c b/drivers/regulator/bd71815-regulator.c
index acaa6607898e..c2b8b8be7824 100644
--- a/drivers/regulator/bd71815-regulator.c
+++ b/drivers/regulator/bd71815-regulator.c
@@ -571,11 +571,10 @@ static int bd7181x_probe(struct platform_device *pdev)
 		dev_err(&pdev->dev, "No parent regmap\n");
 		return -ENODEV;
 	}
-	ldo4_en = devm_gpiod_get_from_of_node(&pdev->dev,
-					      pdev->dev.parent->of_node,
-						 "rohm,vsel-gpios", 0,
-						 GPIOD_ASIS, "ldo4-en");
 
+	ldo4_en = devm_fwnode_gpiod_get(&pdev->dev,
+					dev_fwnode(pdev->dev.parent),
+					"rohm,vsel", GPIOD_ASIS, "ldo4-en");
 	if (IS_ERR(ldo4_en)) {
 		ret = PTR_ERR(ldo4_en);
 		if (ret != -ENOENT)
diff --git a/drivers/regulator/bd9576-regulator.c b/drivers/regulator/bd9576-regulator.c
index aa42da4d141e..393c8693b327 100644
--- a/drivers/regulator/bd9576-regulator.c
+++ b/drivers/regulator/bd9576-regulator.c
@@ -12,6 +12,7 @@
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
 #include <linux/regulator/driver.h>
 #include <linux/regulator/machine.h>
 #include <linux/regulator/of_regulator.h>
@@ -939,8 +940,8 @@ static int bd957x_probe(struct platform_device *pdev)
 	}
 
 	ic_data->regmap = regmap;
-	vout_mode = of_property_read_bool(pdev->dev.parent->of_node,
-					 "rohm,vout1-en-low");
+	vout_mode = device_property_read_bool(pdev->dev.parent,
+					      "rohm,vout1-en-low");
 	if (vout_mode) {
 		struct gpio_desc *en;
 
@@ -948,10 +949,10 @@ static int bd957x_probe(struct platform_device *pdev)
 
 		/* VOUT1 enable state judged by VOUT1_EN pin */
 		/* See if we have GPIO defined */
-		en = devm_gpiod_get_from_of_node(&pdev->dev,
-						 pdev->dev.parent->of_node,
-						 "rohm,vout1-en-gpios", 0,
-						 GPIOD_OUT_LOW, "vout1-en");
+		en = devm_fwnode_gpiod_get(&pdev->dev,
+					   dev_fwnode(pdev->dev.parent),
+					   "rohm,vout1-en", GPIOD_OUT_LOW,
+					   "vout1-en");
 		if (!IS_ERR(en)) {
 			/* VOUT1_OPS gpio ctrl */
 			/*
@@ -986,8 +987,8 @@ static int bd957x_probe(struct platform_device *pdev)
 	 * like DDR voltage selection.
 	 */
 	platform_set_drvdata(pdev, ic_data);
-	ddr_sel =  of_property_read_bool(pdev->dev.parent->of_node,
-					 "rohm,ddr-sel-low");
+	ddr_sel = device_property_read_bool(pdev->dev.parent,
+					    "rohm,ddr-sel-low");
 	if (ddr_sel)
 		ic_data->regulator_data[2].desc.fixed_uV = 1350000;
 	else
diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c
index d3e8dc32832d..bcccad8f7516 100644
--- a/drivers/regulator/core.c
+++ b/drivers/regulator/core.c
@@ -977,12 +977,27 @@ static int drms_uA_update(struct regulator_dev *rdev)
 			rdev_err(rdev, "failed to set load %d: %pe\n",
 				 current_uA, ERR_PTR(err));
 	} else {
+		/*
+		 * Unfortunately in some cases the constraints->valid_ops has
+		 * REGULATOR_CHANGE_DRMS but there are no valid modes listed.
+		 * That's not really legit but we won't consider it a fatal
+		 * error here. We'll treat it as if REGULATOR_CHANGE_DRMS
+		 * wasn't set.
+		 */
+		if (!rdev->constraints->valid_modes_mask) {
+			rdev_dbg(rdev, "Can change modes; but no valid mode\n");
+			return 0;
+		}
+
 		/* get output voltage */
 		output_uV = regulator_get_voltage_rdev(rdev);
-		if (output_uV <= 0) {
-			rdev_err(rdev, "invalid output voltage found\n");
-			return -EINVAL;
-		}
+
+		/*
+		 * Don't return an error; if regulator driver cares about
+		 * output_uV then it's up to the driver to validate.
+		 */
+		if (output_uV <= 0)
+			rdev_dbg(rdev, "invalid output voltage found\n");
 
 		/* get input voltage */
 		input_uV = 0;
@@ -990,10 +1005,13 @@ static int drms_uA_update(struct regulator_dev *rdev)
 			input_uV = regulator_get_voltage(rdev->supply);
 		if (input_uV <= 0)
 			input_uV = rdev->constraints->input_uV;
-		if (input_uV <= 0) {
-			rdev_err(rdev, "invalid input voltage found\n");
-			return -EINVAL;
-		}
+
+		/*
+		 * Don't return an error; if regulator driver cares about
+		 * input_uV then it's up to the driver to validate.
+		 */
+		if (input_uV <= 0)
+			rdev_dbg(rdev, "invalid input voltage found\n");
 
 		/* now get the optimum mode for our new total regulator load */
 		mode = rdev->desc->ops->get_optimum_mode(rdev, input_uV,
@@ -2681,7 +2699,7 @@ static int _regulator_do_enable(struct regulator_dev *rdev)
 	 * return -ETIMEDOUT.
 	 */
 	if (rdev->desc->poll_enabled_time) {
-		unsigned int time_remaining = delay;
+		int time_remaining = delay;
 
 		while (time_remaining > 0) {
 			_regulator_delay_helper(rdev->desc->poll_enabled_time);
@@ -3502,10 +3520,8 @@ static int _regulator_set_voltage_time(struct regulator_dev *rdev,
 		 (new_uV < old_uV))
 		return rdev->constraints->settling_time_down;
 
-	if (ramp_delay == 0) {
-		rdev_dbg(rdev, "ramp_delay not set\n");
+	if (ramp_delay == 0)
 		return 0;
-	}
 
 	return DIV_ROUND_UP(abs(new_uV - old_uV), ramp_delay);
 }
@@ -5398,6 +5414,7 @@ regulator_register(const struct regulator_desc *regulator_desc,
 	bool dangling_of_gpiod = false;
 	struct device *dev;
 	int ret, i;
+	bool resolved_early = false;
 
 	if (cfg == NULL)
 		return ERR_PTR(-EINVAL);
@@ -5501,24 +5518,10 @@ regulator_register(const struct regulator_desc *regulator_desc,
 	BLOCKING_INIT_NOTIFIER_HEAD(&rdev->notifier);
 	INIT_DELAYED_WORK(&rdev->disable_work, regulator_disable_work);
 
-	/* preform any regulator specific init */
-	if (init_data && init_data->regulator_init) {
-		ret = init_data->regulator_init(rdev->reg_data);
-		if (ret < 0)
-			goto clean;
-	}
-
-	if (config->ena_gpiod) {
-		ret = regulator_ena_gpio_request(rdev, config);
-		if (ret != 0) {
-			rdev_err(rdev, "Failed to request enable GPIO: %pe\n",
-				 ERR_PTR(ret));
-			goto clean;
-		}
-		/* The regulator core took over the GPIO descriptor */
-		dangling_cfg_gpiod = false;
-		dangling_of_gpiod = false;
-	}
+	if (init_data && init_data->supply_regulator)
+		rdev->supply_name = init_data->supply_regulator;
+	else if (regulator_desc->supply_name)
+		rdev->supply_name = regulator_desc->supply_name;
 
 	/* register with sysfs */
 	rdev->dev.class = &regulator_class;
@@ -5540,13 +5543,38 @@ regulator_register(const struct regulator_desc *regulator_desc,
 		goto wash;
 	}
 
-	if (init_data && init_data->supply_regulator)
-		rdev->supply_name = init_data->supply_regulator;
-	else if (regulator_desc->supply_name)
-		rdev->supply_name = regulator_desc->supply_name;
+	if ((rdev->supply_name && !rdev->supply) &&
+		(rdev->constraints->always_on ||
+		 rdev->constraints->boot_on)) {
+		ret = regulator_resolve_supply(rdev);
+		if (ret)
+			rdev_dbg(rdev, "unable to resolve supply early: %pe\n",
+					 ERR_PTR(ret));
+
+		resolved_early = true;
+	}
+
+	/* perform any regulator specific init */
+	if (init_data && init_data->regulator_init) {
+		ret = init_data->regulator_init(rdev->reg_data);
+		if (ret < 0)
+			goto wash;
+	}
+
+	if (config->ena_gpiod) {
+		ret = regulator_ena_gpio_request(rdev, config);
+		if (ret != 0) {
+			rdev_err(rdev, "Failed to request enable GPIO: %pe\n",
+				 ERR_PTR(ret));
+			goto wash;
+		}
+		/* The regulator core took over the GPIO descriptor */
+		dangling_cfg_gpiod = false;
+		dangling_of_gpiod = false;
+	}
 
 	ret = set_machine_constraints(rdev);
-	if (ret == -EPROBE_DEFER) {
+	if (ret == -EPROBE_DEFER && !resolved_early) {
 		/* Regulator might be in bypass mode and so needs its supply
 		 * to set the constraints
 		 */
diff --git a/drivers/regulator/da9121-regulator.c b/drivers/regulator/da9121-regulator.c
index 76e0e23bf598..e4c753b83088 100644
--- a/drivers/regulator/da9121-regulator.c
+++ b/drivers/regulator/da9121-regulator.c
@@ -1164,7 +1164,7 @@ error:
 	return ret;
 }
 
-static int da9121_i2c_remove(struct i2c_client *i2c)
+static void da9121_i2c_remove(struct i2c_client *i2c)
 {
 	struct da9121 *chip = i2c_get_clientdata(i2c);
 	const int mask_all[4] = { 0xFF, 0xFF, 0xFF, 0xFF };
@@ -1176,7 +1176,6 @@ static int da9121_i2c_remove(struct i2c_client *i2c)
 	ret = regmap_bulk_write(chip->regmap, DA9121_REG_SYS_MASK_0, mask_all, 4);
 	if (ret != 0)
 		dev_err(chip->dev, "Failed to set IRQ masks: %d\n", ret);
-	return 0;
 }
 
 static const struct i2c_device_id da9121_i2c_id[] = {
diff --git a/drivers/regulator/devres.c b/drivers/regulator/devres.c
index 32823a87fd40..3265e75e97ab 100644
--- a/drivers/regulator/devres.c
+++ b/drivers/regulator/devres.c
@@ -70,6 +70,65 @@ struct regulator *devm_regulator_get_exclusive(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_regulator_get_exclusive);
 
+static void regulator_action_disable(void *d)
+{
+	struct regulator *r = (struct regulator *)d;
+
+	regulator_disable(r);
+}
+
+static int _devm_regulator_get_enable(struct device *dev, const char *id,
+				      int get_type)
+{
+	struct regulator *r;
+	int ret;
+
+	r = _devm_regulator_get(dev, id, get_type);
+	if (IS_ERR(r))
+		return PTR_ERR(r);
+
+	ret = regulator_enable(r);
+	if (!ret)
+		ret = devm_add_action_or_reset(dev, &regulator_action_disable, r);
+
+	if (ret)
+		devm_regulator_put(r);
+
+	return ret;
+}
+
+/**
+ * devm_regulator_get_enable_optional - Resource managed regulator get and enable
+ * @dev: device to supply
+ * @id:  supply name or regulator ID.
+ *
+ * Get and enable regulator for duration of the device life-time.
+ * regulator_disable() and regulator_put() are automatically called on driver
+ * detach. See regulator_get_optional() and regulator_enable() for more
+ * information.
+ */
+int devm_regulator_get_enable_optional(struct device *dev, const char *id)
+{
+	return _devm_regulator_get_enable(dev, id, OPTIONAL_GET);
+}
+EXPORT_SYMBOL_GPL(devm_regulator_get_enable_optional);
+
+/**
+ * devm_regulator_get_enable - Resource managed regulator get and enable
+ * @dev: device to supply
+ * @id:  supply name or regulator ID.
+ *
+ * Get and enable regulator for duration of the device life-time.
+ * regulator_disable() and regulator_put() are automatically called on driver
+ * detach. See regulator_get() and regulator_enable() for more
+ * information.
+ */
+int devm_regulator_get_enable(struct device *dev, const char *id)
+{
+	return _devm_regulator_get_enable(dev, id, NORMAL_GET);
+}
+EXPORT_SYMBOL_GPL(devm_regulator_get_enable);
+
 /**
  * devm_regulator_get_optional - Resource managed regulator_get_optional()
  * @dev: device to supply
@@ -194,6 +253,111 @@ int devm_regulator_bulk_get_const(struct device *dev, int num_consumers,
 }
 EXPORT_SYMBOL_GPL(devm_regulator_bulk_get_const);
 
+static int devm_regulator_bulk_match(struct device *dev, void *res,
+				     void *data)
+{
+	struct regulator_bulk_devres *match = res;
+	struct regulator_bulk_data *target = data;
+
+	/*
+	 * We check the put uses same consumer list as the get did.
+	 * We _could_ scan all entries in consumer array and check the
+	 * regulators match but ATM I don't see the need. We can change this
+	 * later if needed.
+	 */
+	return match->consumers == target;
+}
+
+/**
+ * devm_regulator_bulk_put - Resource managed regulator_bulk_put()
+ * @consumers: consumers to free
+ *
+ * Deallocate regulators allocated with devm_regulator_bulk_get(). Normally
+ * this function will not need to be called and the resource management
+ * code will ensure that the resource is freed.
+ */
+void devm_regulator_bulk_put(struct regulator_bulk_data *consumers)
+{
+	int rc;
+	struct regulator *regulator = consumers[0].consumer;
+
+	rc = devres_release(regulator->dev, devm_regulator_bulk_release,
+			    devm_regulator_bulk_match, consumers);
+	if (rc != 0)
+		WARN_ON(rc);
+}
+EXPORT_SYMBOL_GPL(devm_regulator_bulk_put);
+
+static void devm_regulator_bulk_disable(void *res)
+{
+	struct regulator_bulk_devres *devres = res;
+	int i;
+
+	for (i = 0; i < devres->num_consumers; i++)
+		regulator_disable(devres->consumers[i].consumer);
+}
+
+/**
+ * devm_regulator_bulk_get_enable - managed get'n enable multiple regulators
+ *
+ * @dev:           device to supply
+ * @num_consumers: number of consumers to register
+ * @id:            list of supply names or regulator IDs
+ *
+ * @return 0 on success, an errno on failure.
+ *
+ * This helper function allows drivers to get several regulator
+ * consumers in one operation with management, the regulators will
+ * automatically be freed when the device is unbound.  If any of the
+ * regulators cannot be acquired then any regulators that were
+ * allocated will be freed before returning to the caller.
+ */
+int devm_regulator_bulk_get_enable(struct device *dev, int num_consumers,
+				   const char * const *id)
+{
+	struct regulator_bulk_devres *devres;
+	struct regulator_bulk_data *consumers;
+	int i, ret;
+
+	devres = devm_kmalloc(dev, sizeof(*devres), GFP_KERNEL);
+	if (!devres)
+		return -ENOMEM;
+
+	devres->consumers = devm_kcalloc(dev, num_consumers, sizeof(*consumers),
+					 GFP_KERNEL);
+	consumers = devres->consumers;
+	if (!consumers)
+		return -ENOMEM;
+
+	devres->num_consumers = num_consumers;
+
+	for (i = 0; i < num_consumers; i++)
+		consumers[i].supply = id[i];
+
+	ret = devm_regulator_bulk_get(dev, num_consumers, consumers);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < num_consumers; i++) {
+		ret = regulator_enable(consumers[i].consumer);
+		if (ret)
+			goto unwind;
+	}
+
+	ret = devm_add_action(dev, devm_regulator_bulk_disable, devres);
+	if (!ret)
+		return 0;
+
+unwind:
+	while (--i >= 0)
+		regulator_disable(consumers[i].consumer);
+
+	devm_regulator_bulk_put(consumers);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(devm_regulator_bulk_get_enable);
+
 static void devm_rdev_release(struct device *dev, void *res)
 {
 	regulator_unregister(*(struct regulator_dev **)res);
diff --git a/drivers/regulator/gpio-regulator.c b/drivers/regulator/gpio-regulator.c
index 5927d4f3eabd..95e61a2f43f5 100644
--- a/drivers/regulator/gpio-regulator.c
+++ b/drivers/regulator/gpio-regulator.c
@@ -220,6 +220,9 @@ of_get_gpio_regulator_config(struct device *dev, struct device_node *np,
 				 regtype);
 	}
 
+	if (of_find_property(np, "vin-supply", NULL))
+		config->input_supply = "vin";
+
 	return config;
 }
 
@@ -259,6 +262,18 @@ static int gpio_regulator_probe(struct platform_device *pdev)
 
 	drvdata->gpiods = devm_kzalloc(dev, sizeof(struct gpio_desc *),
 				       GFP_KERNEL);
+
+	if (config->input_supply) {
+		drvdata->desc.supply_name = devm_kstrdup(&pdev->dev,
+							 config->input_supply,
+							 GFP_KERNEL);
+		if (!drvdata->desc.supply_name) {
+			dev_err(&pdev->dev,
+				"Failed to allocate input supply\n");
+			return -ENOMEM;
+		}
+	}
+
 	if (!drvdata->gpiods)
 		return -ENOMEM;
 	for (i = 0; i < config->ngpios; i++) {
diff --git a/drivers/regulator/lp8755.c b/drivers/regulator/lp8755.c
index 321bec6e3f8d..31b43426d47c 100644
--- a/drivers/regulator/lp8755.c
+++ b/drivers/regulator/lp8755.c
@@ -422,15 +422,13 @@ err:
 	return ret;
 }
 
-static int lp8755_remove(struct i2c_client *client)
+static void lp8755_remove(struct i2c_client *client)
 {
 	int icnt;
 	struct lp8755_chip *pchip = i2c_get_clientdata(client);
 
 	for (icnt = 0; icnt < LP8755_BUCK_MAX; icnt++)
 		regmap_write(pchip->regmap, icnt, 0x00);
-
-	return 0;
 }
 
 static const struct i2c_device_id lp8755_id[] = {
diff --git a/drivers/regulator/max597x-regulator.c b/drivers/regulator/max597x-regulator.c
index 03c6027682d8..39f803ff0a90 100644
--- a/drivers/regulator/max597x-regulator.c
+++ b/drivers/regulator/max597x-regulator.c
@@ -137,7 +137,7 @@ static int max597x_set_ovp(struct regulator_dev *rdev, int lim_uV, int severity,
 static int max597x_set_ocp(struct regulator_dev *rdev, int lim_uA,
 			   int severity, bool enable)
 {
-	int ret, val, reg;
+	int val, reg;
 	unsigned int vthst, vthfst;
 
 	struct max597x_regulator *data = rdev_get_drvdata(rdev);
@@ -183,9 +183,8 @@ static int max597x_set_ocp(struct regulator_dev *rdev, int lim_uA,
 		val = 0xFF;
 
 	reg = MAX5970_REG_DAC_FAST(rdev_id);
-	ret = regmap_write(rdev->regmap, reg, val);
 
-	return ret;
+	return regmap_write(rdev->regmap, reg, val);
 }
 
 static int max597x_get_status(struct regulator_dev *rdev)
diff --git a/drivers/regulator/max8973-regulator.c b/drivers/regulator/max8973-regulator.c
index fdcb0f508984..596cc36aaff6 100644
--- a/drivers/regulator/max8973-regulator.c
+++ b/drivers/regulator/max8973-regulator.c
@@ -434,9 +434,9 @@ static int max8973_init_dcdc(struct max8973_chip *max,
 	return ret;
 }
 
-static int max8973_thermal_read_temp(void *data, int *temp)
+static int max8973_thermal_read_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct max8973_chip *mchip = data;
+	struct max8973_chip *mchip = tz->devdata;
 	unsigned int val;
 	int ret;
 
@@ -465,7 +465,7 @@ static irqreturn_t max8973_thermal_irq(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static const struct thermal_zone_of_device_ops max77621_tz_ops = {
+static const struct thermal_zone_device_ops max77621_tz_ops = {
 	.get_temp = max8973_thermal_read_temp,
 };
 
@@ -479,8 +479,8 @@ static int max8973_thermal_init(struct max8973_chip *mchip)
 	if (mchip->id != MAX77621)
 		return 0;
 
-	tzd = devm_thermal_zone_of_sensor_register(mchip->dev, 0, mchip,
-						   &max77621_tz_ops);
+	tzd = devm_thermal_of_zone_register(mchip->dev, 0, mchip,
+					    &max77621_tz_ops);
 	if (IS_ERR(tzd)) {
 		ret = PTR_ERR(tzd);
 		dev_err(mchip->dev, "Failed to register thermal sensor: %d\n",
diff --git a/drivers/regulator/mt6331-regulator.c b/drivers/regulator/mt6331-regulator.c
new file mode 100644
index 000000000000..56be9a3a84ab
--- /dev/null
+++ b/drivers/regulator/mt6331-regulator.c
@@ -0,0 +1,507 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2022 Collabora Ltd.
+// Author: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
+//
+// Based on mt6323-regulator.c,
+//     Copyright (c) 2016 MediaTek Inc.
+//
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/mfd/mt6397/core.h>
+#include <linux/mfd/mt6331/registers.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/regulator/mt6331-regulator.h>
+#include <linux/regulator/of_regulator.h>
+
+#define MT6331_LDO_MODE_NORMAL	0
+#define MT6331_LDO_MODE_LP	1
+
+/*
+ * MT6331 regulators information
+ *
+ * @desc: standard fields of regulator description.
+ * @qi: Mask for query enable signal status of regulators
+ * @vselon_reg: Register sections for hardware control mode of bucks
+ * @vselctrl_reg: Register for controlling the buck control mode.
+ * @vselctrl_mask: Mask for query buck's voltage control mode.
+ * @status_reg: Register for regulator enable status where qi unavailable
+ * @status_mask: Mask for querying regulator enable status
+ */
+struct mt6331_regulator_info {
+	struct regulator_desc desc;
+	u32 qi;
+	u32 vselon_reg;
+	u32 vselctrl_reg;
+	u32 vselctrl_mask;
+	u32 modeset_reg;
+	u32 modeset_mask;
+	u32 status_reg;
+	u32 status_mask;
+};
+
+#define MT6331_BUCK(match, vreg, min, max, step, volt_ranges, enreg,	\
+		vosel, vosel_mask, voselon, vosel_ctrl)			\
+[MT6331_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6331_volt_range_ops,				\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6331_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = (max - min)/step + 1,			\
+		.linear_ranges = volt_ranges,				\
+		.n_linear_ranges = ARRAY_SIZE(volt_ranges),		\
+		.vsel_reg = vosel,					\
+		.vsel_mask = vosel_mask,				\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(0),					\
+	},								\
+	.qi = BIT(13),							\
+	.vselon_reg = voselon,						\
+	.vselctrl_reg = vosel_ctrl,					\
+	.vselctrl_mask = BIT(1),					\
+	.status_mask = 0,						\
+}
+
+#define MT6331_LDO_AO(match, vreg, ldo_volt_table, vosel, vosel_mask)	\
+[MT6331_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6331_volt_table_ao_ops,			\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6331_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = ARRAY_SIZE(ldo_volt_table),		\
+		.volt_table = ldo_volt_table,				\
+		.vsel_reg = vosel,					\
+		.vsel_mask = vosel_mask,				\
+	},								\
+}
+
+#define MT6331_LDO_S(match, vreg, ldo_volt_table, enreg, enbit, vosel,	\
+		     vosel_mask, _modeset_reg, _modeset_mask,		\
+		     _status_reg, _status_mask)				\
+[MT6331_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6331_volt_table_no_qi_ops,			\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6331_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = ARRAY_SIZE(ldo_volt_table),		\
+		.volt_table = ldo_volt_table,				\
+		.vsel_reg = vosel,					\
+		.vsel_mask = vosel_mask,				\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(enbit),				\
+	},								\
+	.modeset_reg = _modeset_reg,					\
+	.modeset_mask = _modeset_mask,					\
+	.status_reg = _status_reg,					\
+	.status_mask = _status_mask,					\
+}
+
+#define MT6331_LDO(match, vreg, ldo_volt_table, enreg, enbit, vosel,	\
+		   vosel_mask, _modeset_reg, _modeset_mask)		\
+[MT6331_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = (_modeset_reg ?					\
+			&mt6331_volt_table_ops :			\
+			&mt6331_volt_table_no_ms_ops),			\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6331_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = ARRAY_SIZE(ldo_volt_table),		\
+		.volt_table = ldo_volt_table,				\
+		.vsel_reg = vosel,					\
+		.vsel_mask = vosel_mask,				\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(enbit),				\
+	},								\
+	.qi = BIT(15),							\
+	.modeset_reg = _modeset_reg,					\
+	.modeset_mask = _modeset_mask,					\
+}
+
+#define MT6331_REG_FIXED(match, vreg, enreg, enbit, qibit, volt,	\
+			 _modeset_reg, _modeset_mask)			\
+[MT6331_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = (_modeset_reg ?					\
+			&mt6331_volt_fixed_ops :			\
+			&mt6331_volt_fixed_no_ms_ops),			\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6331_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = 1,					\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(enbit),				\
+		.min_uV = volt,						\
+	},								\
+	.qi = BIT(qibit),						\
+	.modeset_reg = _modeset_reg,					\
+	.modeset_mask = _modeset_mask,					\
+}
+
+static const struct linear_range buck_volt_range[] = {
+	REGULATOR_LINEAR_RANGE(700000, 0, 0x7f, 6250),
+};
+
+static const unsigned int ldo_volt_table1[] = {
+	2800000, 3000000, 0, 3200000
+};
+
+static const unsigned int ldo_volt_table2[] = {
+	1500000, 1800000, 2500000, 2800000,
+};
+
+static const unsigned int ldo_volt_table3[] = {
+	1200000, 1300000, 1500000, 1800000, 2000000, 2800000, 3000000, 3300000,
+};
+
+static const unsigned int ldo_volt_table4[] = {
+	0, 0, 1700000, 1800000, 1860000, 2760000, 3000000, 3100000,
+};
+
+static const unsigned int ldo_volt_table5[] = {
+	1800000, 3300000, 1800000, 3300000,
+};
+
+static const unsigned int ldo_volt_table6[] = {
+	3000000, 3300000,
+};
+
+static const unsigned int ldo_volt_table7[] = {
+	1200000, 1600000, 1700000, 1800000, 1900000, 2000000, 2100000, 2200000,
+};
+
+static const unsigned int ldo_volt_table8[] = {
+	900000, 1000000, 1100000, 1220000, 1300000, 1500000, 1500000, 1500000,
+};
+
+static const unsigned int ldo_volt_table9[] = {
+	1000000, 1050000, 1100000, 1150000, 1200000, 1250000, 1300000, 1300000,
+};
+
+static const unsigned int ldo_volt_table10[] = {
+	1200000, 1300000, 1500000, 1800000,
+};
+
+static const unsigned int ldo_volt_table11[] = {
+	1200000, 1300000, 1400000, 1500000, 1600000, 1700000, 1800000, 1800000,
+};
+
+static int mt6331_get_status(struct regulator_dev *rdev)
+{
+	struct mt6331_regulator_info *info = rdev_get_drvdata(rdev);
+	u32 regval;
+	int ret;
+
+	ret = regmap_read(rdev->regmap, info->desc.enable_reg, &regval);
+	if (ret != 0) {
+		dev_err(&rdev->dev, "Failed to get enable reg: %d\n", ret);
+		return ret;
+	}
+
+	return (regval & info->qi) ? REGULATOR_STATUS_ON : REGULATOR_STATUS_OFF;
+}
+
+static int mt6331_ldo_set_mode(struct regulator_dev *rdev, unsigned int mode)
+{
+	struct mt6331_regulator_info *info = rdev_get_drvdata(rdev);
+	int val;
+
+	switch (mode) {
+	case REGULATOR_MODE_STANDBY:
+		val = MT6331_LDO_MODE_LP;
+		break;
+	case REGULATOR_MODE_NORMAL:
+		val = MT6331_LDO_MODE_NORMAL;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	val <<= ffs(info->modeset_mask) - 1;
+
+	return regmap_update_bits(rdev->regmap, info->modeset_reg,
+				  info->modeset_mask, val);
+}
+
+static unsigned int mt6331_ldo_get_mode(struct regulator_dev *rdev)
+{
+	struct mt6331_regulator_info *info = rdev_get_drvdata(rdev);
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(rdev->regmap, info->modeset_reg, &val);
+	if (ret < 0)
+		return ret;
+
+	val &= info->modeset_mask;
+	val >>= ffs(info->modeset_mask) - 1;
+
+	return (val & BIT(0)) ? REGULATOR_MODE_STANDBY : REGULATOR_MODE_NORMAL;
+}
+
+static const struct regulator_ops mt6331_volt_range_ops = {
+	.list_voltage = regulator_list_voltage_linear_range,
+	.map_voltage = regulator_map_voltage_linear_range,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.get_status = mt6331_get_status,
+};
+
+static const struct regulator_ops mt6331_volt_table_no_ms_ops = {
+	.list_voltage = regulator_list_voltage_table,
+	.map_voltage = regulator_map_voltage_iterate,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.get_status = mt6331_get_status,
+};
+
+static const struct regulator_ops mt6331_volt_table_no_qi_ops = {
+	.list_voltage = regulator_list_voltage_table,
+	.map_voltage = regulator_map_voltage_iterate,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.set_mode = mt6331_ldo_set_mode,
+	.get_mode = mt6331_ldo_get_mode,
+};
+
+static const struct regulator_ops mt6331_volt_table_ops = {
+	.list_voltage = regulator_list_voltage_table,
+	.map_voltage = regulator_map_voltage_iterate,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.get_status = mt6331_get_status,
+	.set_mode = mt6331_ldo_set_mode,
+	.get_mode = mt6331_ldo_get_mode,
+};
+
+static const struct regulator_ops mt6331_volt_table_ao_ops = {
+	.list_voltage = regulator_list_voltage_table,
+	.map_voltage = regulator_map_voltage_iterate,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+};
+
+static const struct regulator_ops mt6331_volt_fixed_no_ms_ops = {
+	.list_voltage = regulator_list_voltage_linear,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.get_status = mt6331_get_status,
+};
+
+static const struct regulator_ops mt6331_volt_fixed_ops = {
+	.list_voltage = regulator_list_voltage_linear,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.get_status = mt6331_get_status,
+	.set_mode = mt6331_ldo_set_mode,
+	.get_mode = mt6331_ldo_get_mode,
+};
+
+/* The array is indexed by id(MT6331_ID_XXX) */
+static struct mt6331_regulator_info mt6331_regulators[] = {
+	MT6331_BUCK("buck-vdvfs11", VDVFS11, 700000, 1493750, 6250,
+		    buck_volt_range, MT6331_VDVFS11_CON9,
+		    MT6331_VDVFS11_CON11, GENMASK(6, 0),
+		    MT6331_VDVFS11_CON12, MT6331_VDVFS11_CON7),
+	MT6331_BUCK("buck-vdvfs12", VDVFS12, 700000, 1493750, 6250,
+		    buck_volt_range, MT6331_VDVFS12_CON9,
+		    MT6331_VDVFS12_CON11, GENMASK(6, 0),
+		    MT6331_VDVFS12_CON12, MT6331_VDVFS12_CON7),
+	MT6331_BUCK("buck-vdvfs13", VDVFS13, 700000, 1493750, 6250,
+		    buck_volt_range, MT6331_VDVFS13_CON9,
+		    MT6331_VDVFS13_CON11, GENMASK(6, 0),
+		    MT6331_VDVFS13_CON12, MT6331_VDVFS13_CON7),
+	MT6331_BUCK("buck-vdvfs14", VDVFS14, 700000, 1493750, 6250,
+		    buck_volt_range, MT6331_VDVFS14_CON9,
+		    MT6331_VDVFS14_CON11, GENMASK(6, 0),
+		    MT6331_VDVFS14_CON12, MT6331_VDVFS14_CON7),
+	MT6331_BUCK("buck-vcore2", VCORE2, 700000, 1493750, 6250,
+		    buck_volt_range, MT6331_VCORE2_CON9,
+		    MT6331_VCORE2_CON11, GENMASK(6, 0),
+		    MT6331_VCORE2_CON12, MT6331_VCORE2_CON7),
+	MT6331_REG_FIXED("buck-vio18", VIO18, MT6331_VIO18_CON9, 0, 13, 1800000, 0, 0),
+	MT6331_REG_FIXED("ldo-vrtc", VRTC, MT6331_DIGLDO_CON11, 8, 15, 2800000, 0, 0),
+	MT6331_REG_FIXED("ldo-vtcxo1", VTCXO1, MT6331_ANALDO_CON1, 10, 15, 2800000,
+			 MT6331_ANALDO_CON1, GENMASK(1, 0)),
+	MT6331_REG_FIXED("ldo-vtcxo2", VTCXO2, MT6331_ANALDO_CON2, 10, 15, 2800000,
+			 MT6331_ANALDO_CON2, GENMASK(1, 0)),
+	MT6331_REG_FIXED("ldo-vsram", VSRAM_DVFS1, MT6331_SYSLDO_CON4, 10, 15, 1012500,
+			 MT6331_SYSLDO_CON4, GENMASK(1, 0)),
+	MT6331_REG_FIXED("ldo-vio28", VIO28, MT6331_DIGLDO_CON1, 10, 15, 2800000,
+			 MT6331_DIGLDO_CON1, GENMASK(1, 0)),
+	MT6331_LDO("ldo-avdd32aud", AVDD32_AUD, ldo_volt_table1, MT6331_ANALDO_CON3, 10,
+		   MT6331_ANALDO_CON10, GENMASK(6, 5), MT6331_ANALDO_CON3, GENMASK(1, 0)),
+	MT6331_LDO("ldo-vauxa32", VAUXA32, ldo_volt_table1, MT6331_ANALDO_CON4, 10,
+		   MT6331_ANALDO_CON6, GENMASK(6, 5), MT6331_ANALDO_CON4, GENMASK(1, 0)),
+	MT6331_LDO("ldo-vemc33", VEMC33, ldo_volt_table6, MT6331_DIGLDO_CON5, 10,
+		   MT6331_DIGLDO_CON17, BIT(6), MT6331_DIGLDO_CON5, GENMASK(1, 0)),
+	MT6331_LDO("ldo-vibr", VIBR, ldo_volt_table3, MT6331_DIGLDO_CON12, 10,
+		   MT6331_DIGLDO_CON20, GENMASK(6, 4), MT6331_DIGLDO_CON12, GENMASK(1, 0)),
+	MT6331_LDO("ldo-vmc", VMC, ldo_volt_table5, MT6331_DIGLDO_CON3, 10,
+		   MT6331_DIGLDO_CON15, GENMASK(5, 4), MT6331_DIGLDO_CON3, GENMASK(1, 0)),
+	MT6331_LDO("ldo-vmch", VMCH, ldo_volt_table6, MT6331_DIGLDO_CON4, 10,
+		   MT6331_DIGLDO_CON16, BIT(6), MT6331_DIGLDO_CON4, GENMASK(1, 0)),
+	MT6331_LDO("ldo-vmipi", VMIPI, ldo_volt_table3, MT6331_SYSLDO_CON5, 10,
+		   MT6331_SYSLDO_CON13, GENMASK(5, 3), MT6331_SYSLDO_CON5, GENMASK(1, 0)),
+	MT6331_LDO("ldo-vsim1", VSIM1, ldo_volt_table4, MT6331_DIGLDO_CON8, 10,
+		   MT6331_DIGLDO_CON21, GENMASK(6, 4), MT6331_DIGLDO_CON8, GENMASK(1, 0)),
+	MT6331_LDO("ldo-vsim2", VSIM2, ldo_volt_table4, MT6331_DIGLDO_CON9, 10,
+		   MT6331_DIGLDO_CON22, GENMASK(6, 4), MT6331_DIGLDO_CON9, GENMASK(1, 0)),
+	MT6331_LDO("ldo-vusb10", VUSB10, ldo_volt_table9, MT6331_SYSLDO_CON2, 10,
+		   MT6331_SYSLDO_CON10, GENMASK(5, 3), MT6331_SYSLDO_CON2, GENMASK(1, 0)),
+	MT6331_LDO("ldo-vcama", VCAMA, ldo_volt_table2, MT6331_ANALDO_CON5, 15,
+		   MT6331_ANALDO_CON9, GENMASK(5, 4), 0, 0),
+	MT6331_LDO_S("ldo-vcamaf", VCAM_AF, ldo_volt_table3, MT6331_DIGLDO_CON2, 10,
+		     MT6331_DIGLDO_CON14, GENMASK(6, 4), MT6331_DIGLDO_CON2, GENMASK(1, 0),
+		     MT6331_EN_STATUS1, BIT(0)),
+	MT6331_LDO_S("ldo-vcamd", VCAMD, ldo_volt_table8, MT6331_SYSLDO_CON1, 15,
+		     MT6331_SYSLDO_CON9, GENMASK(6, 4), MT6331_SYSLDO_CON1, GENMASK(1, 0),
+		     MT6331_EN_STATUS1, BIT(11)),
+	MT6331_LDO_S("ldo-vcamio", VCAM_IO,  ldo_volt_table10, MT6331_SYSLDO_CON3, 10,
+		     MT6331_SYSLDO_CON11, GENMASK(4, 3), MT6331_SYSLDO_CON3, GENMASK(1, 0),
+		     MT6331_EN_STATUS1, BIT(13)),
+	MT6331_LDO_S("ldo-vgp1", VGP1, ldo_volt_table3, MT6331_DIGLDO_CON6, 10,
+		     MT6331_DIGLDO_CON19, GENMASK(6, 4), MT6331_DIGLDO_CON6, GENMASK(1, 0),
+		     MT6331_EN_STATUS1, BIT(4)),
+	MT6331_LDO_S("ldo-vgp2", VGP2, ldo_volt_table10, MT6331_SYSLDO_CON6, 10,
+		     MT6331_SYSLDO_CON14, GENMASK(4, 3), MT6331_SYSLDO_CON6, GENMASK(1, 0),
+		     MT6331_EN_STATUS1, BIT(15)),
+	MT6331_LDO_S("ldo-vgp3", VGP3, ldo_volt_table10, MT6331_SYSLDO_CON7, 10,
+		     MT6331_SYSLDO_CON15, GENMASK(4, 3), MT6331_SYSLDO_CON7, GENMASK(1, 0),
+		     MT6331_EN_STATUS2, BIT(0)),
+	MT6331_LDO_S("ldo-vgp4", VGP4, ldo_volt_table7, MT6331_DIGLDO_CON7, 10,
+		     MT6331_DIGLDO_CON18, GENMASK(6, 4), MT6331_DIGLDO_CON7, GENMASK(1, 0),
+		     MT6331_EN_STATUS1, BIT(5)),
+	MT6331_LDO_AO("ldo-vdig18", VDIG18, ldo_volt_table11,
+		      MT6331_DIGLDO_CON28, GENMASK(14, 12)),
+};
+
+static int mt6331_set_buck_vosel_reg(struct platform_device *pdev)
+{
+	struct mt6397_chip *mt6331 = dev_get_drvdata(pdev->dev.parent);
+	int i;
+	u32 regval;
+
+	for (i = 0; i < MT6331_ID_VREG_MAX; i++) {
+		if (mt6331_regulators[i].vselctrl_reg) {
+			if (regmap_read(mt6331->regmap,
+				mt6331_regulators[i].vselctrl_reg,
+				&regval) < 0) {
+				dev_err(&pdev->dev,
+					"Failed to read buck ctrl\n");
+				return -EIO;
+			}
+
+			if (regval & mt6331_regulators[i].vselctrl_mask) {
+				mt6331_regulators[i].desc.vsel_reg =
+				mt6331_regulators[i].vselon_reg;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int mt6331_regulator_probe(struct platform_device *pdev)
+{
+	struct mt6397_chip *mt6331 = dev_get_drvdata(pdev->dev.parent);
+	struct regulator_config config = {};
+	struct regulator_dev *rdev;
+	int i;
+	u32 reg_value;
+
+	/* Query buck controller to select activated voltage register part */
+	if (mt6331_set_buck_vosel_reg(pdev))
+		return -EIO;
+
+	/* Read PMIC chip revision to update constraints and voltage table */
+	if (regmap_read(mt6331->regmap, MT6331_HWCID, &reg_value) < 0) {
+		dev_err(&pdev->dev, "Failed to read Chip ID\n");
+		return -EIO;
+	}
+	reg_value &= GENMASK(7, 0);
+
+	dev_info(&pdev->dev, "Chip ID = 0x%x\n", reg_value);
+
+	/*
+	 * ChipID 0x10 is "MT6331 E1", has a different voltage table and
+	 * it's currently not supported in this driver. Upon detection of
+	 * this ID, refuse to register the regulators, as we will wrongly
+	 * interpret the VSEL for this revision, potentially overvolting
+	 * some device.
+	 */
+	if (reg_value == 0x10) {
+		dev_err(&pdev->dev, "Chip version not supported. Bailing out.\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < MT6331_ID_VREG_MAX; i++) {
+		config.dev = &pdev->dev;
+		config.driver_data = &mt6331_regulators[i];
+		config.regmap = mt6331->regmap;
+		rdev = devm_regulator_register(&pdev->dev,
+				&mt6331_regulators[i].desc, &config);
+		if (IS_ERR(rdev)) {
+			dev_err(&pdev->dev, "failed to register %s\n",
+				mt6331_regulators[i].desc.name);
+			return PTR_ERR(rdev);
+		}
+	}
+	return 0;
+}
+
+static const struct platform_device_id mt6331_platform_ids[] = {
+	{"mt6331-regulator", 0},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(platform, mt6331_platform_ids);
+
+static struct platform_driver mt6331_regulator_driver = {
+	.driver = {
+		.name = "mt6331-regulator",
+	},
+	.probe = mt6331_regulator_probe,
+	.id_table = mt6331_platform_ids,
+};
+
+module_platform_driver(mt6331_regulator_driver);
+
+MODULE_AUTHOR("AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>");
+MODULE_DESCRIPTION("Regulator Driver for MediaTek MT6331 PMIC");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/mt6332-regulator.c b/drivers/regulator/mt6332-regulator.c
new file mode 100644
index 000000000000..77a27d8127a3
--- /dev/null
+++ b/drivers/regulator/mt6332-regulator.c
@@ -0,0 +1,422 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Copyright (c) 2022 Collabora Ltd.
+// Author: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
+//
+// Based on mt6323-regulator.c,
+//     Copyright (c) 2016 MediaTek Inc.
+//
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/mfd/mt6397/core.h>
+#include <linux/mfd/mt6332/registers.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/regulator/mt6332-regulator.h>
+#include <linux/regulator/of_regulator.h>
+
+#define MT6332_LDO_MODE_NORMAL	0
+#define MT6332_LDO_MODE_LP	1
+
+/*
+ * MT6332 regulators information
+ *
+ * @desc: standard fields of regulator description.
+ * @qi: Mask for query enable signal status of regulators
+ * @vselon_reg: Register sections for hardware control mode of bucks
+ * @vselctrl_reg: Register for controlling the buck control mode.
+ * @vselctrl_mask: Mask for query buck's voltage control mode.
+ * @status_reg: Register for regulator enable status where qi unavailable
+ * @status_mask: Mask for querying regulator enable status
+ */
+struct mt6332_regulator_info {
+	struct regulator_desc desc;
+	u32 qi;
+	u32 vselon_reg;
+	u32 vselctrl_reg;
+	u32 vselctrl_mask;
+	u32 modeset_reg;
+	u32 modeset_mask;
+	u32 status_reg;
+	u32 status_mask;
+};
+
+#define MT6332_BUCK(match, vreg, min, max, step, volt_ranges, enreg,	\
+		vosel, vosel_mask, voselon, vosel_ctrl)			\
+[MT6332_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6332_buck_volt_range_ops,			\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6332_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = (max - min)/step + 1,			\
+		.linear_ranges = volt_ranges,				\
+		.n_linear_ranges = ARRAY_SIZE(volt_ranges),		\
+		.vsel_reg = vosel,					\
+		.vsel_mask = vosel_mask,				\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(0),					\
+	},								\
+	.qi = BIT(13),							\
+	.vselon_reg = voselon,						\
+	.vselctrl_reg = vosel_ctrl,					\
+	.vselctrl_mask = BIT(1),					\
+	.status_mask = 0,						\
+}
+
+#define MT6332_LDO_LINEAR(match, vreg, min, max, step, volt_ranges,	\
+			  enreg, vosel, vosel_mask, voselon,		\
+			  vosel_ctrl, _modeset_reg, _modeset_mask)	\
+[MT6332_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6332_ldo_volt_range_ops,			\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6332_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = (max - min)/step + 1,			\
+		.linear_ranges = volt_ranges,				\
+		.n_linear_ranges = ARRAY_SIZE(volt_ranges),		\
+		.vsel_reg = vosel,					\
+		.vsel_mask = vosel_mask,				\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(0),					\
+	},								\
+	.qi = BIT(15),							\
+	.vselon_reg = voselon,						\
+	.vselctrl_reg = vosel_ctrl,					\
+	.vselctrl_mask = BIT(1),					\
+	.modeset_reg = _modeset_reg,					\
+	.modeset_mask = _modeset_mask,					\
+	.status_mask = 0,						\
+}
+
+#define MT6332_LDO_AO(match, vreg, ldo_volt_table, vosel, vosel_mask)	\
+[MT6332_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6332_volt_table_ao_ops,			\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6332_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = ARRAY_SIZE(ldo_volt_table),		\
+		.volt_table = ldo_volt_table,				\
+		.vsel_reg = vosel,					\
+		.vsel_mask = vosel_mask,				\
+	},								\
+}
+
+#define MT6332_LDO(match, vreg, ldo_volt_table, enreg, enbit, vosel,	\
+		   vosel_mask, _modeset_reg, _modeset_mask)		\
+[MT6332_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6332_volt_table_ops,				\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6332_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = ARRAY_SIZE(ldo_volt_table),		\
+		.volt_table = ldo_volt_table,				\
+		.vsel_reg = vosel,					\
+		.vsel_mask = vosel_mask,				\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(enbit),				\
+	},								\
+	.qi = BIT(15),							\
+	.modeset_reg = _modeset_reg,					\
+	.modeset_mask = _modeset_mask,					\
+	.status_mask = 0,						\
+}
+
+#define MT6332_REG_FIXED(match, vreg, enreg, enbit, qibit, volt, stbit)	\
+[MT6332_ID_##vreg] = {							\
+	.desc = {							\
+		.name = #vreg,						\
+		.of_match = of_match_ptr(match),			\
+		.ops = &mt6332_volt_fixed_ops,				\
+		.type = REGULATOR_VOLTAGE,				\
+		.id = MT6332_ID_##vreg,					\
+		.owner = THIS_MODULE,					\
+		.n_voltages = 1,					\
+		.enable_reg = enreg,					\
+		.enable_mask = BIT(enbit),				\
+		.min_uV = volt,						\
+	},								\
+	.qi = BIT(qibit),						\
+	.status_reg = MT6332_EN_STATUS0,				\
+	.status_mask = BIT(stbit),					\
+}
+
+static const struct linear_range boost_volt_range[] = {
+	REGULATOR_LINEAR_RANGE(3500000, 0, 0x7f, 31250),
+};
+
+static const struct linear_range buck_volt_range[] = {
+	REGULATOR_LINEAR_RANGE(700000, 0, 0x7f, 6250),
+};
+
+static const struct linear_range buck_pa_volt_range[] = {
+	REGULATOR_LINEAR_RANGE(500000, 0, 0x3f, 50000),
+};
+
+static const struct linear_range buck_rf_volt_range[] = {
+	REGULATOR_LINEAR_RANGE(1050000, 0, 0x7f, 9375),
+};
+
+static const unsigned int ldo_volt_table1[] = {
+	2800000, 3000000, 0, 3200000
+};
+
+static const unsigned int ldo_volt_table2[] = {
+	1200000, 1300000, 1400000, 1500000, 1600000, 1700000, 1800000, 1800000,
+};
+
+static int mt6332_get_status(struct regulator_dev *rdev)
+{
+	struct mt6332_regulator_info *info = rdev_get_drvdata(rdev);
+	u32 reg, en_mask, regval;
+	int ret;
+
+	if (info->qi > 0) {
+		reg = info->desc.enable_reg;
+		en_mask = info->qi;
+	} else {
+		reg = info->status_reg;
+		en_mask = info->status_mask;
+	}
+
+	ret = regmap_read(rdev->regmap, reg, &regval);
+	if (ret != 0) {
+		dev_err(&rdev->dev, "Failed to get enable reg: %d\n", ret);
+		return ret;
+	}
+
+	return (regval & en_mask) ? REGULATOR_STATUS_ON : REGULATOR_STATUS_OFF;
+}
+
+static int mt6332_ldo_set_mode(struct regulator_dev *rdev, unsigned int mode)
+{
+	struct mt6332_regulator_info *info = rdev_get_drvdata(rdev);
+	int val;
+
+	switch (mode) {
+	case REGULATOR_MODE_STANDBY:
+		val = MT6332_LDO_MODE_LP;
+		break;
+	case REGULATOR_MODE_NORMAL:
+		val = MT6332_LDO_MODE_NORMAL;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	val <<= ffs(info->modeset_mask) - 1;
+
+	return regmap_update_bits(rdev->regmap, info->modeset_reg,
+				  info->modeset_mask, val);
+}
+
+static unsigned int mt6332_ldo_get_mode(struct regulator_dev *rdev)
+{
+	struct mt6332_regulator_info *info = rdev_get_drvdata(rdev);
+	unsigned int val;
+	int ret;
+
+	ret = regmap_read(rdev->regmap, info->modeset_reg, &val);
+	if (ret < 0)
+		return ret;
+
+	val &= info->modeset_mask;
+	val >>= ffs(info->modeset_mask) - 1;
+
+	return (val & BIT(0)) ? REGULATOR_MODE_STANDBY : REGULATOR_MODE_NORMAL;
+}
+
+static const struct regulator_ops mt6332_buck_volt_range_ops = {
+	.list_voltage = regulator_list_voltage_linear_range,
+	.map_voltage = regulator_map_voltage_linear_range,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.get_status = mt6332_get_status,
+};
+
+static const struct regulator_ops mt6332_ldo_volt_range_ops = {
+	.list_voltage = regulator_list_voltage_linear_range,
+	.map_voltage = regulator_map_voltage_linear_range,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.get_status = mt6332_get_status,
+	.set_mode = mt6332_ldo_set_mode,
+	.get_mode = mt6332_ldo_get_mode,
+};
+
+static const struct regulator_ops mt6332_volt_table_ops = {
+	.list_voltage = regulator_list_voltage_table,
+	.map_voltage = regulator_map_voltage_iterate,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.get_status = mt6332_get_status,
+	.set_mode = mt6332_ldo_set_mode,
+	.get_mode = mt6332_ldo_get_mode,
+};
+
+static const struct regulator_ops mt6332_volt_table_ao_ops = {
+	.list_voltage = regulator_list_voltage_table,
+	.map_voltage = regulator_map_voltage_iterate,
+	.set_voltage_sel = regulator_set_voltage_sel_regmap,
+	.get_voltage_sel = regulator_get_voltage_sel_regmap,
+	.set_voltage_time_sel = regulator_set_voltage_time_sel,
+};
+
+static const struct regulator_ops mt6332_volt_fixed_ops = {
+	.list_voltage = regulator_list_voltage_linear,
+	.enable = regulator_enable_regmap,
+	.disable = regulator_disable_regmap,
+	.is_enabled = regulator_is_enabled_regmap,
+	.get_status = mt6332_get_status,
+};
+
+/* The array is indexed by id(MT6332_ID_XXX) */
+static struct mt6332_regulator_info mt6332_regulators[] = {
+	MT6332_BUCK("buck-vdram", VDRAM, 700000, 1493750, 6250, buck_volt_range,
+		    MT6332_EN_STATUS0, MT6332_VDRAM_CON11, GENMASK(6, 0),
+		    MT6332_VDRAM_CON12, MT6332_VDRAM_CON7),
+	MT6332_BUCK("buck-vdvfs2", VDVFS2, 700000, 1312500, 6250, buck_volt_range,
+		    MT6332_VDVFS2_CON9, MT6332_VDVFS2_CON11, GENMASK(6, 0),
+		    MT6332_VDVFS2_CON12, MT6332_VDVFS2_CON7),
+	MT6332_BUCK("buck-vpa", VPA, 500000, 3400000, 50000, buck_pa_volt_range,
+		    MT6332_VPA_CON9, MT6332_VPA_CON11, GENMASK(5, 0),
+		    MT6332_VPA_CON12, MT6332_VPA_CON7),
+	MT6332_BUCK("buck-vrf18a", VRF1, 1050000, 2240625, 9375, buck_rf_volt_range,
+		    MT6332_VRF1_CON9, MT6332_VRF1_CON11, GENMASK(6, 0),
+		    MT6332_VRF1_CON12, MT6332_VRF1_CON7),
+	MT6332_BUCK("buck-vrf18b", VRF2, 1050000, 2240625, 9375, buck_rf_volt_range,
+		    MT6332_VRF2_CON9, MT6332_VRF2_CON11, GENMASK(6, 0),
+		    MT6332_VRF2_CON12, MT6332_VRF2_CON7),
+	MT6332_BUCK("buck-vsbst", VSBST, 3500000, 7468750, 31250, boost_volt_range,
+		    MT6332_VSBST_CON8, MT6332_VSBST_CON12, GENMASK(6, 0),
+		    MT6332_VSBST_CON13, MT6332_VSBST_CON8),
+	MT6332_LDO("ldo-vauxb32", VAUXB32, ldo_volt_table1, MT6332_LDO_CON1, 10,
+		   MT6332_LDO_CON9, GENMASK(6, 5), MT6332_LDO_CON1, GENMASK(1, 0)),
+	MT6332_REG_FIXED("ldo-vbif28", VBIF28, MT6332_LDO_CON2, 10, 0, 2800000, 1),
+	MT6332_REG_FIXED("ldo-vusb33", VUSB33, MT6332_LDO_CON3, 10, 0, 3300000, 2),
+	MT6332_LDO_LINEAR("ldo-vsram", VSRAM_DVFS2, 700000, 1493750, 6250, buck_volt_range,
+			  MT6332_EN_STATUS0, MT6332_LDO_CON8, GENMASK(15, 9),
+			  MT6332_VDVFS2_CON23, MT6332_VDVFS2_CON22,
+			  MT6332_LDO_CON5, GENMASK(1, 0)),
+	MT6332_LDO_AO("ldo-vdig18", VDIG18, ldo_volt_table2, MT6332_LDO_CON12, GENMASK(11, 9)),
+};
+
+static int mt6332_set_buck_vosel_reg(struct platform_device *pdev)
+{
+	struct mt6397_chip *mt6332 = dev_get_drvdata(pdev->dev.parent);
+	int i;
+	u32 regval;
+
+	for (i = 0; i < MT6332_ID_VREG_MAX; i++) {
+		if (mt6332_regulators[i].vselctrl_reg) {
+			if (regmap_read(mt6332->regmap,
+				mt6332_regulators[i].vselctrl_reg,
+				&regval) < 0) {
+				dev_err(&pdev->dev,
+					"Failed to read buck ctrl\n");
+				return -EIO;
+			}
+
+			if (regval & mt6332_regulators[i].vselctrl_mask) {
+				mt6332_regulators[i].desc.vsel_reg =
+				mt6332_regulators[i].vselon_reg;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int mt6332_regulator_probe(struct platform_device *pdev)
+{
+	struct mt6397_chip *mt6332 = dev_get_drvdata(pdev->dev.parent);
+	struct regulator_config config = {};
+	struct regulator_dev *rdev;
+	int i;
+	u32 reg_value;
+
+	/* Query buck controller to select activated voltage register part */
+	if (mt6332_set_buck_vosel_reg(pdev))
+		return -EIO;
+
+	/* Read PMIC chip revision to update constraints and voltage table */
+	if (regmap_read(mt6332->regmap, MT6332_HWCID, &reg_value) < 0) {
+		dev_err(&pdev->dev, "Failed to read Chip ID\n");
+		return -EIO;
+	}
+	reg_value &= GENMASK(7, 0);
+
+	dev_info(&pdev->dev, "Chip ID = 0x%x\n", reg_value);
+
+	/*
+	 * ChipID 0x10 is "MT6332 E1", has a different voltage table and
+	 * it's currently not supported in this driver. Upon detection of
+	 * this ID, refuse to register the regulators, as we will wrongly
+	 * interpret the VSEL for this revision, potentially overvolting
+	 * some device.
+	 */
+	if (reg_value == 0x10) {
+		dev_err(&pdev->dev, "Chip version not supported. Bailing out.\n");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < MT6332_ID_VREG_MAX; i++) {
+		config.dev = &pdev->dev;
+		config.driver_data = &mt6332_regulators[i];
+		config.regmap = mt6332->regmap;
+		rdev = devm_regulator_register(&pdev->dev,
+				&mt6332_regulators[i].desc, &config);
+		if (IS_ERR(rdev)) {
+			dev_err(&pdev->dev, "failed to register %s\n",
+				mt6332_regulators[i].desc.name);
+			return PTR_ERR(rdev);
+		}
+	}
+	return 0;
+}
+
+static const struct platform_device_id mt6332_platform_ids[] = {
+	{"mt6332-regulator", 0},
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(platform, mt6332_platform_ids);
+
+static struct platform_driver mt6332_regulator_driver = {
+	.driver = {
+		.name = "mt6332-regulator",
+	},
+	.probe = mt6332_regulator_probe,
+	.id_table = mt6332_platform_ids,
+};
+
+module_platform_driver(mt6332_regulator_driver);
+
+MODULE_AUTHOR("AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>");
+MODULE_DESCRIPTION("Regulator Driver for MediaTek MT6332 PMIC");
+MODULE_LICENSE("GPL");
diff --git a/drivers/regulator/of_regulator.c b/drivers/regulator/of_regulator.c
index e12b681c72e5..0aff1c2886b5 100644
--- a/drivers/regulator/of_regulator.c
+++ b/drivers/regulator/of_regulator.c
@@ -676,7 +676,7 @@ clean:
 }
 
 /**
- * of_parse_coupled regulator - Get regulator_dev pointer from rdev's property
+ * of_parse_coupled_regulator() - Get regulator_dev pointer from rdev's property
  * @rdev: Pointer to regulator_dev, whose DTS is used as a source to parse
  *	  "regulator-coupled-with" property
  * @index: Index in phandles array
diff --git a/drivers/regulator/qcom-rpmh-regulator.c b/drivers/regulator/qcom-rpmh-regulator.c
index 561de6b2e6e3..4158ff126a67 100644
--- a/drivers/regulator/qcom-rpmh-regulator.c
+++ b/drivers/regulator/qcom-rpmh-regulator.c
@@ -306,9 +306,10 @@ static unsigned int rpmh_regulator_vrm_get_mode(struct regulator_dev *rdev)
 }
 
 /**
- * rpmh_regulator_vrm_set_load() - set the regulator mode based upon the load
- *		current requested
+ * rpmh_regulator_vrm_get_optimum_mode() - get the mode based on the  load
  * @rdev:		Regulator device pointer for the rpmh-regulator
+ * @input_uV:		Input voltage
+ * @output_uV:		Output voltage
  * @load_uA:		Aggregated load current in microamps
  *
  * This function is used in the regulator_ops for VRM type RPMh regulator
@@ -316,17 +317,15 @@ static unsigned int rpmh_regulator_vrm_get_mode(struct regulator_dev *rdev)
  *
  * Return: 0 on success, errno on failure
  */
-static int rpmh_regulator_vrm_set_load(struct regulator_dev *rdev, int load_uA)
+static unsigned int rpmh_regulator_vrm_get_optimum_mode(
+	struct regulator_dev *rdev, int input_uV, int output_uV, int load_uA)
 {
 	struct rpmh_vreg *vreg = rdev_get_drvdata(rdev);
-	unsigned int mode;
 
 	if (load_uA >= vreg->hw_data->hpm_min_load_uA)
-		mode = REGULATOR_MODE_NORMAL;
+		return REGULATOR_MODE_NORMAL;
 	else
-		mode = REGULATOR_MODE_IDLE;
-
-	return rpmh_regulator_vrm_set_mode(rdev, mode);
+		return REGULATOR_MODE_IDLE;
 }
 
 static int rpmh_regulator_vrm_set_bypass(struct regulator_dev *rdev,
@@ -375,7 +374,7 @@ static const struct regulator_ops rpmh_regulator_vrm_drms_ops = {
 	.list_voltage		= regulator_list_voltage_linear_range,
 	.set_mode		= rpmh_regulator_vrm_set_mode,
 	.get_mode		= rpmh_regulator_vrm_get_mode,
-	.set_load		= rpmh_regulator_vrm_set_load,
+	.get_optimum_mode	= rpmh_regulator_vrm_get_optimum_mode,
 };
 
 static const struct regulator_ops rpmh_regulator_vrm_bypass_ops = {
@@ -1199,6 +1198,52 @@ static const struct rpmh_vreg_init_data pmr735a_vreg_data[] = {
 	{}
 };
 
+static const struct rpmh_vreg_init_data pm660_vreg_data[] = {
+	RPMH_VREG("smps1",  "smp%s1",  &pmic4_ftsmps426, "vdd-s1"),
+	RPMH_VREG("smps2",  "smp%s2",  &pmic4_ftsmps426, "vdd-s2"),
+	RPMH_VREG("smps3",  "smp%s3",  &pmic4_ftsmps426, "vdd-s3"),
+	RPMH_VREG("smps4",  "smp%s4",  &pmic4_hfsmps3,   "vdd-s4"),
+	RPMH_VREG("smps5",  "smp%s5",  &pmic4_hfsmps3,   "vdd-s5"),
+	RPMH_VREG("smps6",  "smp%s6",  &pmic4_hfsmps3,   "vdd-s6"),
+	RPMH_VREG("ldo1",   "ldo%s1",  &pmic4_nldo,      "vdd-l1-l6-l7"),
+	RPMH_VREG("ldo2",   "ldo%s2",  &pmic4_nldo,      "vdd-l2-l3"),
+	RPMH_VREG("ldo3",   "ldo%s3",  &pmic4_nldo,      "vdd-l2-l3"),
+	/* ldo4 is inaccessible on PM660 */
+	RPMH_VREG("ldo5",   "ldo%s5",  &pmic4_nldo,      "vdd-l5"),
+	RPMH_VREG("ldo6",   "ldo%s6",  &pmic4_nldo,      "vdd-l1-l6-l7"),
+	RPMH_VREG("ldo7",   "ldo%s7",  &pmic4_nldo,      "vdd-l1-l6-l7"),
+	RPMH_VREG("ldo8",   "ldo%s8",  &pmic4_pldo_lv,   "vdd-l8-l9-l10-l11-l12-l13-l14"),
+	RPMH_VREG("ldo9",   "ldo%s9",  &pmic4_pldo_lv,   "vdd-l8-l9-l10-l11-l12-l13-l14"),
+	RPMH_VREG("ldo10",  "ldo%s10", &pmic4_pldo_lv,   "vdd-l8-l9-l10-l11-l12-l13-l14"),
+	RPMH_VREG("ldo11",  "ldo%s11", &pmic4_pldo_lv,   "vdd-l8-l9-l10-l11-l12-l13-l14"),
+	RPMH_VREG("ldo12",  "ldo%s12", &pmic4_pldo_lv,   "vdd-l8-l9-l10-l11-l12-l13-l14"),
+	RPMH_VREG("ldo13",  "ldo%s13", &pmic4_pldo_lv,   "vdd-l8-l9-l10-l11-l12-l13-l14"),
+	RPMH_VREG("ldo14",  "ldo%s14", &pmic4_pldo_lv,   "vdd-l8-l9-l10-l11-l12-l13-l14"),
+	RPMH_VREG("ldo15",  "ldo%s15", &pmic4_pldo,      "vdd-l15-l16-l17-l18-l19"),
+	RPMH_VREG("ldo16",  "ldo%s16", &pmic4_pldo,      "vdd-l15-l16-l17-l18-l19"),
+	RPMH_VREG("ldo17",  "ldo%s17", &pmic4_pldo,      "vdd-l15-l16-l17-l18-l19"),
+	RPMH_VREG("ldo18",  "ldo%s18", &pmic4_pldo,      "vdd-l15-l16-l17-l18-l19"),
+	RPMH_VREG("ldo19",  "ldo%s19", &pmic4_pldo,      "vdd-l15-l16-l17-l18-l19"),
+	{}
+};
+
+static const struct rpmh_vreg_init_data pm660l_vreg_data[] = {
+	RPMH_VREG("smps1",  "smp%s1",  &pmic4_ftsmps426, "vdd-s1"),
+	RPMH_VREG("smps2",  "smp%s2",  &pmic4_ftsmps426, "vdd-s2"),
+	RPMH_VREG("smps3",  "smp%s3",  &pmic4_ftsmps426, "vdd-s3-s4"),
+	RPMH_VREG("smps5",  "smp%s5",  &pmic4_ftsmps426, "vdd-s5"),
+	RPMH_VREG("ldo1",   "ldo%s1",  &pmic4_nldo,      "vdd-l1-l9-l10"),
+	RPMH_VREG("ldo2",   "ldo%s2",  &pmic4_pldo,      "vdd-l2"),
+	RPMH_VREG("ldo3",   "ldo%s3",  &pmic4_pldo,      "vdd-l3-l5-l7-l8"),
+	RPMH_VREG("ldo4",   "ldo%s4",  &pmic4_pldo,      "vdd-l4-l6"),
+	RPMH_VREG("ldo5",   "ldo%s5",  &pmic4_pldo,      "vdd-l3-l5-l7-l8"),
+	RPMH_VREG("ldo6",   "ldo%s6",  &pmic4_pldo,      "vdd-l4-l6"),
+	RPMH_VREG("ldo7",   "ldo%s7",  &pmic4_pldo,      "vdd-l3-l5-l7-l8"),
+	RPMH_VREG("ldo8",   "ldo%s8",  &pmic4_pldo,      "vdd-l3-l5-l7-l8"),
+	RPMH_VREG("bob",    "bob%s1",  &pmic4_bob,       "vdd-bob"),
+	{}
+};
+
 static int rpmh_regulator_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -1321,6 +1366,14 @@ static const struct of_device_id __maybe_unused rpmh_regulator_match_table[] = {
 		.compatible = "qcom,pmr735a-rpmh-regulators",
 		.data = pmr735a_vreg_data,
 	},
+	{
+		.compatible = "qcom,pm660-rpmh-regulators",
+		.data = pm660_vreg_data,
+	},
+	{
+		.compatible = "qcom,pm660l-rpmh-regulators",
+		.data = pm660l_vreg_data,
+	},
 	{}
 };
 MODULE_DEVICE_TABLE(of, rpmh_regulator_match_table);
diff --git a/drivers/regulator/qcom_rpm-regulator.c b/drivers/regulator/qcom_rpm-regulator.c
index 7f9d66ac37ff..3c41b71a1f52 100644
--- a/drivers/regulator/qcom_rpm-regulator.c
+++ b/drivers/regulator/qcom_rpm-regulator.c
@@ -802,6 +802,12 @@ static const struct rpm_regulator_data rpm_pm8018_regulators[] = {
 };
 
 static const struct rpm_regulator_data rpm_pm8058_regulators[] = {
+	{ "s0",   QCOM_RPM_PM8058_SMPS0,  &pm8058_smps, "vdd_s0" },
+	{ "s1",   QCOM_RPM_PM8058_SMPS1,  &pm8058_smps, "vdd_s1" },
+	{ "s2",   QCOM_RPM_PM8058_SMPS2,  &pm8058_smps, "vdd_s2" },
+	{ "s3",   QCOM_RPM_PM8058_SMPS3,  &pm8058_smps, "vdd_s3" },
+	{ "s4",   QCOM_RPM_PM8058_SMPS4,  &pm8058_smps, "vdd_s4" },
+
 	{ "l0",   QCOM_RPM_PM8058_LDO0,   &pm8058_nldo, "vdd_l0_l1_lvs"	},
 	{ "l1",   QCOM_RPM_PM8058_LDO1,   &pm8058_nldo, "vdd_l0_l1_lvs" },
 	{ "l2",   QCOM_RPM_PM8058_LDO2,   &pm8058_pldo, "vdd_l2_l11_l12" },
@@ -829,12 +835,6 @@ static const struct rpm_regulator_data rpm_pm8058_regulators[] = {
 	{ "l24",  QCOM_RPM_PM8058_LDO24,  &pm8058_nldo, "vdd_l23_l24_l25" },
 	{ "l25",  QCOM_RPM_PM8058_LDO25,  &pm8058_nldo, "vdd_l23_l24_l25" },
 
-	{ "s0",   QCOM_RPM_PM8058_SMPS0,  &pm8058_smps, "vdd_s0" },
-	{ "s1",   QCOM_RPM_PM8058_SMPS1,  &pm8058_smps, "vdd_s1" },
-	{ "s2",   QCOM_RPM_PM8058_SMPS2,  &pm8058_smps, "vdd_s2" },
-	{ "s3",   QCOM_RPM_PM8058_SMPS3,  &pm8058_smps, "vdd_s3" },
-	{ "s4",   QCOM_RPM_PM8058_SMPS4,  &pm8058_smps, "vdd_s4" },
-
 	{ "lvs0", QCOM_RPM_PM8058_LVS0, &pm8058_switch, "vdd_l0_l1_lvs" },
 	{ "lvs1", QCOM_RPM_PM8058_LVS1, &pm8058_switch, "vdd_l0_l1_lvs" },
 
@@ -843,6 +843,12 @@ static const struct rpm_regulator_data rpm_pm8058_regulators[] = {
 };
 
 static const struct rpm_regulator_data rpm_pm8901_regulators[] = {
+	{ "s0",   QCOM_RPM_PM8901_SMPS0, &pm8901_ftsmps, "vdd_s0" },
+	{ "s1",   QCOM_RPM_PM8901_SMPS1, &pm8901_ftsmps, "vdd_s1" },
+	{ "s2",   QCOM_RPM_PM8901_SMPS2, &pm8901_ftsmps, "vdd_s2" },
+	{ "s3",   QCOM_RPM_PM8901_SMPS3, &pm8901_ftsmps, "vdd_s3" },
+	{ "s4",   QCOM_RPM_PM8901_SMPS4, &pm8901_ftsmps, "vdd_s4" },
+
 	{ "l0",   QCOM_RPM_PM8901_LDO0, &pm8901_nldo, "vdd_l0" },
 	{ "l1",   QCOM_RPM_PM8901_LDO1, &pm8901_pldo, "vdd_l1" },
 	{ "l2",   QCOM_RPM_PM8901_LDO2, &pm8901_pldo, "vdd_l2" },
@@ -851,12 +857,6 @@ static const struct rpm_regulator_data rpm_pm8901_regulators[] = {
 	{ "l5",   QCOM_RPM_PM8901_LDO5, &pm8901_pldo, "vdd_l5" },
 	{ "l6",   QCOM_RPM_PM8901_LDO6, &pm8901_pldo, "vdd_l6" },
 
-	{ "s0",   QCOM_RPM_PM8901_SMPS0, &pm8901_ftsmps, "vdd_s0" },
-	{ "s1",   QCOM_RPM_PM8901_SMPS1, &pm8901_ftsmps, "vdd_s1" },
-	{ "s2",   QCOM_RPM_PM8901_SMPS2, &pm8901_ftsmps, "vdd_s2" },
-	{ "s3",   QCOM_RPM_PM8901_SMPS3, &pm8901_ftsmps, "vdd_s3" },
-	{ "s4",   QCOM_RPM_PM8901_SMPS4, &pm8901_ftsmps, "vdd_s4" },
-
 	{ "lvs0", QCOM_RPM_PM8901_LVS0, &pm8901_switch, "lvs0_in" },
 	{ "lvs1", QCOM_RPM_PM8901_LVS1, &pm8901_switch, "lvs1_in" },
 	{ "lvs2", QCOM_RPM_PM8901_LVS2, &pm8901_switch, "lvs2_in" },
diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c
index 59024c639141..f98168d58dce 100644
--- a/drivers/regulator/qcom_smd-regulator.c
+++ b/drivers/regulator/qcom_smd-regulator.c
@@ -668,6 +668,15 @@ static const struct regulator_desc pm660l_bob = {
 	.ops = &rpm_bob_ops,
 };
 
+static const struct regulator_desc pm6125_ftsmps = {
+	.linear_ranges = (struct linear_range[]) {
+		REGULATOR_LINEAR_RANGE(300000, 0, 268, 4000),
+	},
+	.n_linear_ranges = 1,
+	.n_voltages = 269,
+	.ops = &rpm_smps_ldo_ops,
+};
+
 static const struct regulator_desc pms405_hfsmps3 = {
 	.linear_ranges = (struct linear_range[]) {
 		REGULATOR_LINEAR_RANGE(320000, 0, 215, 8000),
@@ -772,6 +781,158 @@ static const struct rpm_regulator_data rpm_mp5496_regulators[] = {
 	{}
 };
 
+static const struct rpm_regulator_data rpm_pm2250_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm2250_lvftsmps, "vdd_s1" },
+	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm2250_lvftsmps, "vdd_s2" },
+	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pm2250_lvftsmps, "vdd_s3" },
+	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pm2250_ftsmps, "vdd_s4" },
+	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
+	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
+	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" },
+	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" },
+	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" },
+	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" },
+	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
+	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
+	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
+	{ "l20", QCOM_SMD_RPM_LDOA, 20, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
+	{ "l21", QCOM_SMD_RPM_LDOA, 21, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
+	{ "l22", QCOM_SMD_RPM_LDOA, 22, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
+	{}
+};
+
+static const struct rpm_regulator_data rpm_pm6125_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm6125_ftsmps, "vdd_s1" },
+	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm6125_ftsmps, "vdd_s2" },
+	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pm6125_ftsmps, "vdd_s3" },
+	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pm6125_ftsmps, "vdd_s4" },
+	{ "s5", QCOM_SMD_RPM_SMPA, 5, &pm8998_hfsmps, "vdd_s5" },
+	{ "s6", QCOM_SMD_RPM_SMPA, 6, &pm8998_hfsmps, "vdd_s6" },
+	{ "s7", QCOM_SMD_RPM_SMPA, 7, &pm8998_hfsmps, "vdd_s7" },
+	{ "s8", QCOM_SMD_RPM_SMPA, 8, &pm6125_ftsmps, "vdd_s8" },
+	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l7_l17_l18" },
+	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_nldo660, "vdd_l2_l3_l4" },
+	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l2_l3_l4" },
+	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pm660_nldo660, "vdd_l2_l3_l4" },
+	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_nldo660, "vdd_l6_l8" },
+	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_nldo660, "vdd_l1_l7_l17_l18" },
+	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_nldo660, "vdd_l6_l8" },
+	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_ht_lvpldo, "vdd_l9_l11" },
+	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_ht_lvpldo, "vdd_l10_l13_l14" },
+	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_ht_lvpldo, "vdd_l9_l11" },
+	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_ht_lvpldo, "vdd_l12_l16" },
+	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l10_l13_l14" },
+	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l10_l13_l14" },
+	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_ht_lvpldo, "vdd_l12_l16" },
+	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_nldo660, "vdd_l1_l7_l17_l18" },
+	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_nldo660, "vdd_l1_l7_l17_l18" },
+	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l20", QCOM_SMD_RPM_LDOA, 20, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l21", QCOM_SMD_RPM_LDOA, 21, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l22", QCOM_SMD_RPM_LDOA, 22, &pm660_pldo660, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l23", QCOM_SMD_RPM_LDOA, 23, &pm660_pldo660, "vdd_l23_l24" },
+	{ "l24", QCOM_SMD_RPM_LDOA, 24, &pm660_pldo660, "vdd_l23_l24" },
+	{ }
+};
+
+static const struct rpm_regulator_data rpm_pm660_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm660_ftsmps, "vdd_s1" },
+	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm660_ftsmps, "vdd_s2" },
+	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pm660_ftsmps, "vdd_s3" },
+	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pm660_hfsmps, "vdd_s4" },
+	{ "s5", QCOM_SMD_RPM_SMPA, 5, &pm660_hfsmps, "vdd_s5" },
+	{ "s6", QCOM_SMD_RPM_SMPA, 6, &pm660_hfsmps, "vdd_s6" },
+	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l6_l7" },
+	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_ht_nldo, "vdd_l2_l3" },
+	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l2_l3" },
+	/* l4 is unaccessible on PM660 */
+	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_ht_nldo, "vdd_l5" },
+	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_ht_nldo, "vdd_l1_l6_l7" },
+	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_ht_nldo, "vdd_l1_l6_l7" },
+	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
+	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
+	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
+	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
+	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
+	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
+	{ }
+};
+
+static const struct rpm_regulator_data rpm_pm660l_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPB, 1, &pm660_ftsmps, "vdd_s1" },
+	{ "s2", QCOM_SMD_RPM_SMPB, 2, &pm660_ftsmps, "vdd_s2" },
+	{ "s3", QCOM_SMD_RPM_RWCX, 0, &pm660_ftsmps, "vdd_s3_s4" },
+	{ "s5", QCOM_SMD_RPM_RWMX, 0, &pm660_ftsmps, "vdd_s5" },
+	{ "l1", QCOM_SMD_RPM_LDOB, 1, &pm660_nldo660, "vdd_l1_l9_l10" },
+	{ "l2", QCOM_SMD_RPM_LDOB, 2, &pm660_pldo660, "vdd_l2" },
+	{ "l3", QCOM_SMD_RPM_LDOB, 3, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
+	{ "l4", QCOM_SMD_RPM_LDOB, 4, &pm660_pldo660, "vdd_l4_l6" },
+	{ "l5", QCOM_SMD_RPM_LDOB, 5, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
+	{ "l6", QCOM_SMD_RPM_LDOB, 6, &pm660_pldo660, "vdd_l4_l6" },
+	{ "l7", QCOM_SMD_RPM_LDOB, 7, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
+	{ "l8", QCOM_SMD_RPM_LDOB, 8, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
+	{ "l9", QCOM_SMD_RPM_RWLC, 0, &pm660_ht_nldo, "vdd_l1_l9_l10" },
+	{ "l10", QCOM_SMD_RPM_RWLM, 0, &pm660_ht_nldo, "vdd_l1_l9_l10" },
+	{ "bob", QCOM_SMD_RPM_BOBB, 1, &pm660l_bob, "vdd_bob", },
+	{ }
+};
+
+static const struct rpm_regulator_data rpm_pm8226_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8226_hfsmps, "vdd_s1" },
+	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8226_ftsmps, "vdd_s2" },
+	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pm8226_hfsmps, "vdd_s3" },
+	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pm8226_hfsmps, "vdd_s4" },
+	{ "s5", QCOM_SMD_RPM_SMPA, 5, &pm8226_hfsmps, "vdd_s5" },
+	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pm8226_nldo, "vdd_l1_l2_l4_l5" },
+	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pm8226_nldo, "vdd_l1_l2_l4_l5" },
+	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pm8226_nldo, "vdd_l3_l24_l26" },
+	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pm8226_nldo, "vdd_l1_l2_l4_l5" },
+	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pm8226_nldo, "vdd_l1_l2_l4_l5" },
+	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" },
+	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" },
+	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" },
+	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" },
+	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pm8226_pldo, "vdd_l10_l11_l13" },
+	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pm8226_pldo, "vdd_l10_l11_l13" },
+	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pm8226_pldo, "vdd_l12_l14" },
+	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pm8226_pldo, "vdd_l10_l11_l13" },
+	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pm8226_pldo, "vdd_l12_l14" },
+	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pm8226_pldo, "vdd_l15_l16_l17_l18" },
+	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pm8226_pldo, "vdd_l15_l16_l17_l18" },
+	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pm8226_pldo, "vdd_l15_l16_l17_l18" },
+	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pm8226_pldo, "vdd_l15_l16_l17_l18" },
+	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
+	{ "l20", QCOM_SMD_RPM_LDOA, 20, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
+	{ "l21", QCOM_SMD_RPM_LDOA, 21, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
+	{ "l22", QCOM_SMD_RPM_LDOA, 22, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
+	{ "l23", QCOM_SMD_RPM_LDOA, 23, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
+	{ "l24", QCOM_SMD_RPM_LDOA, 24, &pm8226_nldo, "vdd_l3_l24_l26" },
+	{ "l25", QCOM_SMD_RPM_LDOA, 25, &pm8226_pldo, "vdd_l25" },
+	{ "l26", QCOM_SMD_RPM_LDOA, 26, &pm8226_nldo, "vdd_l3_l24_l26" },
+	{ "l27", QCOM_SMD_RPM_LDOA, 27, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" },
+	{ "l28", QCOM_SMD_RPM_LDOA, 28, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
+	{ "lvs1", QCOM_SMD_RPM_VSA, 1, &pm8226_switch, "vdd_lvs1" },
+	{}
+};
+
 static const struct rpm_regulator_data rpm_pm8841_regulators[] = {
 	{ "s1", QCOM_SMD_RPM_SMPB, 1, &pm8x41_hfsmps, "vdd_s1" },
 	{ "s2", QCOM_SMD_RPM_SMPB, 2, &pm8841_ftsmps, "vdd_s2" },
@@ -833,44 +994,6 @@ static const struct rpm_regulator_data rpm_pm8916_regulators[] = {
 	{}
 };
 
-static const struct rpm_regulator_data rpm_pm8226_regulators[] = {
-	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8226_hfsmps, "vdd_s1" },
-	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8226_ftsmps, "vdd_s2" },
-	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pm8226_hfsmps, "vdd_s3" },
-	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pm8226_hfsmps, "vdd_s4" },
-	{ "s5", QCOM_SMD_RPM_SMPA, 5, &pm8226_hfsmps, "vdd_s5" },
-	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pm8226_nldo, "vdd_l1_l2_l4_l5" },
-	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pm8226_nldo, "vdd_l1_l2_l4_l5" },
-	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pm8226_nldo, "vdd_l3_l24_l26" },
-	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pm8226_nldo, "vdd_l1_l2_l4_l5" },
-	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pm8226_nldo, "vdd_l1_l2_l4_l5" },
-	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" },
-	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" },
-	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" },
-	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" },
-	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pm8226_pldo, "vdd_l10_l11_l13" },
-	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pm8226_pldo, "vdd_l10_l11_l13" },
-	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pm8226_pldo, "vdd_l12_l14" },
-	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pm8226_pldo, "vdd_l10_l11_l13" },
-	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pm8226_pldo, "vdd_l12_l14" },
-	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pm8226_pldo, "vdd_l15_l16_l17_l18" },
-	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pm8226_pldo, "vdd_l15_l16_l17_l18" },
-	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pm8226_pldo, "vdd_l15_l16_l17_l18" },
-	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pm8226_pldo, "vdd_l15_l16_l17_l18" },
-	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
-	{ "l20", QCOM_SMD_RPM_LDOA, 20, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
-	{ "l21", QCOM_SMD_RPM_LDOA, 21, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
-	{ "l22", QCOM_SMD_RPM_LDOA, 22, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
-	{ "l23", QCOM_SMD_RPM_LDOA, 23, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
-	{ "l24", QCOM_SMD_RPM_LDOA, 24, &pm8226_nldo, "vdd_l3_l24_l26" },
-	{ "l25", QCOM_SMD_RPM_LDOA, 25, &pm8226_pldo, "vdd_l25" },
-	{ "l26", QCOM_SMD_RPM_LDOA, 26, &pm8226_nldo, "vdd_l3_l24_l26" },
-	{ "l27", QCOM_SMD_RPM_LDOA, 27, &pm8226_pldo, "vdd_l6_l7_l8_l9_l27" },
-	{ "l28", QCOM_SMD_RPM_LDOA, 28, &pm8226_pldo, "vdd_l19_l20_l21_l22_l23_l28" },
-	{ "lvs1", QCOM_SMD_RPM_VSA, 1, &pm8226_switch, "vdd_lvs1" },
-	{}
-};
-
 static const struct rpm_regulator_data rpm_pm8941_regulators[] = {
 	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8x41_hfsmps, "vdd_s1" },
 	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8x41_hfsmps, "vdd_s2" },
@@ -912,57 +1035,6 @@ static const struct rpm_regulator_data rpm_pm8941_regulators[] = {
 	{}
 };
 
-static const struct rpm_regulator_data rpm_pma8084_regulators[] = {
-	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pma8084_ftsmps, "vdd_s1" },
-	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pma8084_ftsmps, "vdd_s2" },
-	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pma8084_hfsmps, "vdd_s3" },
-	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pma8084_hfsmps, "vdd_s4" },
-	{ "s5", QCOM_SMD_RPM_SMPA, 5, &pma8084_hfsmps, "vdd_s5" },
-	{ "s6", QCOM_SMD_RPM_SMPA, 6, &pma8084_ftsmps, "vdd_s6" },
-	{ "s7", QCOM_SMD_RPM_SMPA, 7, &pma8084_ftsmps, "vdd_s7" },
-	{ "s8", QCOM_SMD_RPM_SMPA, 8, &pma8084_ftsmps, "vdd_s8" },
-	{ "s9", QCOM_SMD_RPM_SMPA, 9, &pma8084_ftsmps, "vdd_s9" },
-	{ "s10", QCOM_SMD_RPM_SMPA, 10, &pma8084_ftsmps, "vdd_s10" },
-	{ "s11", QCOM_SMD_RPM_SMPA, 11, &pma8084_ftsmps, "vdd_s11" },
-	{ "s12", QCOM_SMD_RPM_SMPA, 12, &pma8084_ftsmps, "vdd_s12" },
-
-	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pma8084_nldo, "vdd_l1_l11" },
-	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
-	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
-	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
-	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pma8084_pldo, "vdd_l5_l7" },
-	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
-	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pma8084_pldo, "vdd_l5_l7" },
-	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pma8084_pldo, "vdd_l8" },
-	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
-	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
-	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pma8084_nldo, "vdd_l1_l11" },
-	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
-	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
-	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
-	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
-	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pma8084_pldo, "vdd_l16_l25" },
-	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pma8084_pldo, "vdd_l17" },
-	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pma8084_pldo, "vdd_l18" },
-	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pma8084_pldo, "vdd_l19" },
-	{ "l20", QCOM_SMD_RPM_LDOA, 20, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
-	{ "l21", QCOM_SMD_RPM_LDOA, 21, &pma8084_pldo, "vdd_l21" },
-	{ "l22", QCOM_SMD_RPM_LDOA, 22, &pma8084_pldo, "vdd_l22" },
-	{ "l23", QCOM_SMD_RPM_LDOA, 23, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
-	{ "l24", QCOM_SMD_RPM_LDOA, 24, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
-	{ "l25", QCOM_SMD_RPM_LDOA, 25, &pma8084_pldo, "vdd_l16_l25" },
-	{ "l26", QCOM_SMD_RPM_LDOA, 26, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
-	{ "l27", QCOM_SMD_RPM_LDOA, 27, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
-
-	{ "lvs1", QCOM_SMD_RPM_VSA, 1, &pma8084_switch },
-	{ "lvs2", QCOM_SMD_RPM_VSA, 2, &pma8084_switch },
-	{ "lvs3", QCOM_SMD_RPM_VSA, 3, &pma8084_switch },
-	{ "lvs4", QCOM_SMD_RPM_VSA, 4, &pma8084_switch },
-	{ "5vs1", QCOM_SMD_RPM_VSA, 5, &pma8084_switch },
-
-	{}
-};
-
 static const struct rpm_regulator_data rpm_pm8950_regulators[] = {
 	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8950_hfsmps, "vdd_s1" },
 	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8950_hfsmps, "vdd_s2" },
@@ -1082,14 +1154,6 @@ static const struct rpm_regulator_data rpm_pm8994_regulators[] = {
 	{}
 };
 
-static const struct rpm_regulator_data rpm_pmi8994_regulators[] = {
-	{ "s1", QCOM_SMD_RPM_SMPB, 1, &pmi8994_ftsmps, "vdd_s1" },
-	{ "s2", QCOM_SMD_RPM_SMPB, 2, &pmi8994_hfsmps, "vdd_s2" },
-	{ "s3", QCOM_SMD_RPM_SMPB, 3, &pmi8994_hfsmps, "vdd_s3" },
-	{ "boost-bypass", QCOM_SMD_RPM_BBYB, 1, &pmi8994_bby, "vdd_bst_byp" },
-	{}
-};
-
 static const struct rpm_regulator_data rpm_pm8998_regulators[] = {
 	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8998_ftsmps, "vdd_s1" },
 	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8998_ftsmps, "vdd_s2" },
@@ -1137,57 +1201,68 @@ static const struct rpm_regulator_data rpm_pm8998_regulators[] = {
 	{}
 };
 
-static const struct rpm_regulator_data rpm_pmi8998_regulators[] = {
-	{ "bob", QCOM_SMD_RPM_BOBB, 1, &pmi8998_bob, "vdd_bob" },
+static const struct rpm_regulator_data rpm_pma8084_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pma8084_ftsmps, "vdd_s1" },
+	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pma8084_ftsmps, "vdd_s2" },
+	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pma8084_hfsmps, "vdd_s3" },
+	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pma8084_hfsmps, "vdd_s4" },
+	{ "s5", QCOM_SMD_RPM_SMPA, 5, &pma8084_hfsmps, "vdd_s5" },
+	{ "s6", QCOM_SMD_RPM_SMPA, 6, &pma8084_ftsmps, "vdd_s6" },
+	{ "s7", QCOM_SMD_RPM_SMPA, 7, &pma8084_ftsmps, "vdd_s7" },
+	{ "s8", QCOM_SMD_RPM_SMPA, 8, &pma8084_ftsmps, "vdd_s8" },
+	{ "s9", QCOM_SMD_RPM_SMPA, 9, &pma8084_ftsmps, "vdd_s9" },
+	{ "s10", QCOM_SMD_RPM_SMPA, 10, &pma8084_ftsmps, "vdd_s10" },
+	{ "s11", QCOM_SMD_RPM_SMPA, 11, &pma8084_ftsmps, "vdd_s11" },
+	{ "s12", QCOM_SMD_RPM_SMPA, 12, &pma8084_ftsmps, "vdd_s12" },
+
+	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pma8084_nldo, "vdd_l1_l11" },
+	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
+	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
+	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
+	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pma8084_pldo, "vdd_l5_l7" },
+	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
+	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pma8084_pldo, "vdd_l5_l7" },
+	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pma8084_pldo, "vdd_l8" },
+	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pma8084_nldo, "vdd_l1_l11" },
+	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
+	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
+	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
+	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pma8084_pldo, "vdd_l16_l25" },
+	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pma8084_pldo, "vdd_l17" },
+	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pma8084_pldo, "vdd_l18" },
+	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pma8084_pldo, "vdd_l19" },
+	{ "l20", QCOM_SMD_RPM_LDOA, 20, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l21", QCOM_SMD_RPM_LDOA, 21, &pma8084_pldo, "vdd_l21" },
+	{ "l22", QCOM_SMD_RPM_LDOA, 22, &pma8084_pldo, "vdd_l22" },
+	{ "l23", QCOM_SMD_RPM_LDOA, 23, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l24", QCOM_SMD_RPM_LDOA, 24, &pma8084_pldo, "vdd_l9_l10_l13_l20_l23_l24" },
+	{ "l25", QCOM_SMD_RPM_LDOA, 25, &pma8084_pldo, "vdd_l16_l25" },
+	{ "l26", QCOM_SMD_RPM_LDOA, 26, &pma8084_pldo, "vdd_l6_l12_l14_l15_l26" },
+	{ "l27", QCOM_SMD_RPM_LDOA, 27, &pma8084_nldo, "vdd_l2_l3_l4_l27" },
+
+	{ "lvs1", QCOM_SMD_RPM_VSA, 1, &pma8084_switch },
+	{ "lvs2", QCOM_SMD_RPM_VSA, 2, &pma8084_switch },
+	{ "lvs3", QCOM_SMD_RPM_VSA, 3, &pma8084_switch },
+	{ "lvs4", QCOM_SMD_RPM_VSA, 4, &pma8084_switch },
+	{ "5vs1", QCOM_SMD_RPM_VSA, 5, &pma8084_switch },
+
 	{}
 };
 
-static const struct rpm_regulator_data rpm_pm660_regulators[] = {
-	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm660_ftsmps, "vdd_s1" },
-	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm660_ftsmps, "vdd_s2" },
-	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pm660_ftsmps, "vdd_s3" },
-	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pm660_hfsmps, "vdd_s4" },
-	{ "s5", QCOM_SMD_RPM_SMPA, 5, &pm660_hfsmps, "vdd_s5" },
-	{ "s6", QCOM_SMD_RPM_SMPA, 6, &pm660_hfsmps, "vdd_s6" },
-	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l6_l7" },
-	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_ht_nldo, "vdd_l2_l3" },
-	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l2_l3" },
-	/* l4 is unaccessible on PM660 */
-	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_ht_nldo, "vdd_l5" },
-	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_ht_nldo, "vdd_l1_l6_l7" },
-	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_ht_nldo, "vdd_l1_l6_l7" },
-	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
-	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
-	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
-	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
-	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
-	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
-	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l8_l9_l10_l11_l12_l13_l14" },
-	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
-	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
-	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
-	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
-	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l15_l16_l17_l18_l19" },
-	{ }
+static const struct rpm_regulator_data rpm_pmi8994_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPB, 1, &pmi8994_ftsmps, "vdd_s1" },
+	{ "s2", QCOM_SMD_RPM_SMPB, 2, &pmi8994_hfsmps, "vdd_s2" },
+	{ "s3", QCOM_SMD_RPM_SMPB, 3, &pmi8994_hfsmps, "vdd_s3" },
+	{ "boost-bypass", QCOM_SMD_RPM_BBYB, 1, &pmi8994_bby, "vdd_bst_byp" },
+	{}
 };
 
-static const struct rpm_regulator_data rpm_pm660l_regulators[] = {
-	{ "s1", QCOM_SMD_RPM_SMPB, 1, &pm660_ftsmps, "vdd_s1" },
-	{ "s2", QCOM_SMD_RPM_SMPB, 2, &pm660_ftsmps, "vdd_s2" },
-	{ "s3", QCOM_SMD_RPM_RWCX, 0, &pm660_ftsmps, "vdd_s3_s4" },
-	{ "s5", QCOM_SMD_RPM_RWMX, 0, &pm660_ftsmps, "vdd_s5" },
-	{ "l1", QCOM_SMD_RPM_LDOB, 1, &pm660_nldo660, "vdd_l1_l9_l10" },
-	{ "l2", QCOM_SMD_RPM_LDOB, 2, &pm660_pldo660, "vdd_l2" },
-	{ "l3", QCOM_SMD_RPM_LDOB, 3, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
-	{ "l4", QCOM_SMD_RPM_LDOB, 4, &pm660_pldo660, "vdd_l4_l6" },
-	{ "l5", QCOM_SMD_RPM_LDOB, 5, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
-	{ "l6", QCOM_SMD_RPM_LDOB, 6, &pm660_pldo660, "vdd_l4_l6" },
-	{ "l7", QCOM_SMD_RPM_LDOB, 7, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
-	{ "l8", QCOM_SMD_RPM_LDOB, 8, &pm660_pldo660, "vdd_l3_l5_l7_l8" },
-	{ "l9", QCOM_SMD_RPM_RWLC, 0, &pm660_ht_nldo, "vdd_l1_l9_l10" },
-	{ "l10", QCOM_SMD_RPM_RWLM, 0, &pm660_ht_nldo, "vdd_l1_l9_l10" },
-	{ "bob", QCOM_SMD_RPM_BOBB, 1, &pm660l_bob, "vdd_bob", },
-	{ }
+static const struct rpm_regulator_data rpm_pmi8998_regulators[] = {
+	{ "bob", QCOM_SMD_RPM_BOBB, 1, &pmi8998_bob, "vdd_bob" },
+	{}
 };
 
 static const struct rpm_regulator_data rpm_pms405_regulators[] = {
@@ -1212,54 +1287,25 @@ static const struct rpm_regulator_data rpm_pms405_regulators[] = {
 	{}
 };
 
-static const struct rpm_regulator_data rpm_pm2250_regulators[] = {
-	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm2250_lvftsmps, "vdd_s1" },
-	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm2250_lvftsmps, "vdd_s2" },
-	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pm2250_lvftsmps, "vdd_s3" },
-	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pm2250_ftsmps, "vdd_s4" },
-	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
-	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_nldo660, "vdd_l1_l2_l3_l5_l6_l7_l8_l9_l10_l11_l12" },
-	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" },
-	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" },
-	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" },
-	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_ht_lvpldo, "vdd_l13_l14_l15_l16" },
-	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
-	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
-	{ "l19", QCOM_SMD_RPM_LDOA, 19, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
-	{ "l20", QCOM_SMD_RPM_LDOA, 20, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
-	{ "l21", QCOM_SMD_RPM_LDOA, 21, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
-	{ "l22", QCOM_SMD_RPM_LDOA, 22, &pm660_pldo660, "vdd_l4_l17_l18_l19_l20_l21_l22" },
-	{}
-};
-
 static const struct of_device_id rpm_of_match[] = {
 	{ .compatible = "qcom,rpm-mp5496-regulators", .data = &rpm_mp5496_regulators },
+	{ .compatible = "qcom,rpm-pm2250-regulators", .data = &rpm_pm2250_regulators },
+	{ .compatible = "qcom,rpm-pm6125-regulators", .data = &rpm_pm6125_regulators },
+	{ .compatible = "qcom,rpm-pm660-regulators", .data = &rpm_pm660_regulators },
+	{ .compatible = "qcom,rpm-pm660l-regulators", .data = &rpm_pm660l_regulators },
+	{ .compatible = "qcom,rpm-pm8226-regulators", .data = &rpm_pm8226_regulators },
 	{ .compatible = "qcom,rpm-pm8841-regulators", .data = &rpm_pm8841_regulators },
 	{ .compatible = "qcom,rpm-pm8909-regulators", .data = &rpm_pm8909_regulators },
 	{ .compatible = "qcom,rpm-pm8916-regulators", .data = &rpm_pm8916_regulators },
-	{ .compatible = "qcom,rpm-pm8226-regulators", .data = &rpm_pm8226_regulators },
 	{ .compatible = "qcom,rpm-pm8941-regulators", .data = &rpm_pm8941_regulators },
 	{ .compatible = "qcom,rpm-pm8950-regulators", .data = &rpm_pm8950_regulators },
 	{ .compatible = "qcom,rpm-pm8953-regulators", .data = &rpm_pm8953_regulators },
 	{ .compatible = "qcom,rpm-pm8994-regulators", .data = &rpm_pm8994_regulators },
 	{ .compatible = "qcom,rpm-pm8998-regulators", .data = &rpm_pm8998_regulators },
-	{ .compatible = "qcom,rpm-pm660-regulators", .data = &rpm_pm660_regulators },
-	{ .compatible = "qcom,rpm-pm660l-regulators", .data = &rpm_pm660l_regulators },
 	{ .compatible = "qcom,rpm-pma8084-regulators", .data = &rpm_pma8084_regulators },
 	{ .compatible = "qcom,rpm-pmi8994-regulators", .data = &rpm_pmi8994_regulators },
 	{ .compatible = "qcom,rpm-pmi8998-regulators", .data = &rpm_pmi8998_regulators },
 	{ .compatible = "qcom,rpm-pms405-regulators", .data = &rpm_pms405_regulators },
-	{ .compatible = "qcom,rpm-pm2250-regulators", .data = &rpm_pm2250_regulators },
 	{}
 };
 MODULE_DEVICE_TABLE(of, rpm_of_match);
diff --git a/drivers/regulator/qcom_spmi-regulator.c b/drivers/regulator/qcom_spmi-regulator.c
index a2d0292a92fd..3e312729741e 100644
--- a/drivers/regulator/qcom_spmi-regulator.c
+++ b/drivers/regulator/qcom_spmi-regulator.c
@@ -99,6 +99,9 @@ enum spmi_regulator_logical_type {
 	SPMI_REGULATOR_LOGICAL_TYPE_ULT_LDO,
 	SPMI_REGULATOR_LOGICAL_TYPE_FTSMPS426,
 	SPMI_REGULATOR_LOGICAL_TYPE_HFS430,
+	SPMI_REGULATOR_LOGICAL_TYPE_FTSMPS3,
+	SPMI_REGULATOR_LOGICAL_TYPE_LDO_510,
+	SPMI_REGULATOR_LOGICAL_TYPE_HFSMPS,
 };
 
 enum spmi_regulator_type {
@@ -166,6 +169,17 @@ enum spmi_regulator_subtype {
 	SPMI_REGULATOR_SUBTYPE_HFS430		= 0x0a,
 	SPMI_REGULATOR_SUBTYPE_HT_P150		= 0x35,
 	SPMI_REGULATOR_SUBTYPE_HT_P600		= 0x3d,
+	SPMI_REGULATOR_SUBTYPE_HFSMPS_510	= 0x0a,
+	SPMI_REGULATOR_SUBTYPE_FTSMPS_510	= 0x0b,
+	SPMI_REGULATOR_SUBTYPE_LV_P150_510	= 0x71,
+	SPMI_REGULATOR_SUBTYPE_LV_P300_510	= 0x72,
+	SPMI_REGULATOR_SUBTYPE_LV_P600_510	= 0x73,
+	SPMI_REGULATOR_SUBTYPE_N300_510		= 0x6a,
+	SPMI_REGULATOR_SUBTYPE_N600_510		= 0x6b,
+	SPMI_REGULATOR_SUBTYPE_N1200_510	= 0x6c,
+	SPMI_REGULATOR_SUBTYPE_MV_P50_510	= 0x7a,
+	SPMI_REGULATOR_SUBTYPE_MV_P150_510	= 0x7b,
+	SPMI_REGULATOR_SUBTYPE_MV_P600_510	= 0x7d,
 };
 
 enum spmi_common_regulator_registers {
@@ -193,6 +207,14 @@ enum spmi_ftsmps426_regulator_registers {
 	SPMI_FTSMPS426_REG_VOLTAGE_ULS_MSB	= 0x69,
 };
 
+/*
+ * Third common register layout
+ */
+enum spmi_hfsmps_regulator_registers {
+	SPMI_HFSMPS_REG_STEP_CTRL		= 0x3c,
+	SPMI_HFSMPS_REG_PULL_DOWN		= 0xa0,
+};
+
 enum spmi_vs_registers {
 	SPMI_VS_REG_OCP				= 0x4a,
 	SPMI_VS_REG_SOFT_START			= 0x4c,
@@ -260,6 +282,15 @@ enum spmi_common_control_register_index {
 
 #define SPMI_FTSMPS426_MODE_MASK		0x07
 
+/* Third common regulator mode register values */
+#define SPMI_HFSMPS_MODE_BYPASS_MASK		2
+#define SPMI_HFSMPS_MODE_RETENTION_MASK		3
+#define SPMI_HFSMPS_MODE_LPM_MASK		4
+#define SPMI_HFSMPS_MODE_AUTO_MASK		6
+#define SPMI_HFSMPS_MODE_HPM_MASK		7
+
+#define SPMI_HFSMPS_MODE_MASK			0x07
+
 /* Common regulator pull down control register layout */
 #define SPMI_COMMON_PULL_DOWN_ENABLE_MASK	0x80
 
@@ -305,6 +336,9 @@ enum spmi_common_control_register_index {
 #define SPMI_FTSMPS_STEP_MARGIN_NUM	4
 #define SPMI_FTSMPS_STEP_MARGIN_DEN	5
 
+/* slew_rate has units of uV/us. */
+#define SPMI_HFSMPS_SLEW_RATE_38p4 38400
+
 #define SPMI_FTSMPS426_STEP_CTRL_DELAY_MASK	0x03
 #define SPMI_FTSMPS426_STEP_CTRL_DELAY_SHIFT	0
 
@@ -554,6 +588,14 @@ static struct spmi_voltage_range ht_p600_ranges[] = {
 	SPMI_VOLTAGE_RANGE(0, 1704000, 1704000, 1896000, 1896000, 8000),
 };
 
+static struct spmi_voltage_range nldo_510_ranges[] = {
+	SPMI_VOLTAGE_RANGE(0, 320000, 320000, 1304000, 1304000, 8000),
+};
+
+static struct spmi_voltage_range ftsmps510_ranges[] = {
+	SPMI_VOLTAGE_RANGE(0, 300000, 300000, 1372000, 1372000, 4000),
+};
+
 static DEFINE_SPMI_SET_POINTS(pldo);
 static DEFINE_SPMI_SET_POINTS(nldo1);
 static DEFINE_SPMI_SET_POINTS(nldo2);
@@ -576,6 +618,8 @@ static DEFINE_SPMI_SET_POINTS(ht_nldo);
 static DEFINE_SPMI_SET_POINTS(hfs430);
 static DEFINE_SPMI_SET_POINTS(ht_p150);
 static DEFINE_SPMI_SET_POINTS(ht_p600);
+static DEFINE_SPMI_SET_POINTS(nldo_510);
+static DEFINE_SPMI_SET_POINTS(ftsmps510);
 
 static inline int spmi_vreg_read(struct spmi_regulator *vreg, u16 addr, u8 *buf,
 				 int len)
@@ -1062,6 +1106,23 @@ static unsigned int spmi_regulator_ftsmps426_get_mode(struct regulator_dev *rdev
 	}
 }
 
+static unsigned int spmi_regulator_hfsmps_get_mode(struct regulator_dev *rdev)
+{
+	struct spmi_regulator *vreg = rdev_get_drvdata(rdev);
+	u8 reg;
+
+	spmi_vreg_read(vreg, SPMI_COMMON_REG_MODE, &reg, 1);
+
+	switch (reg) {
+	case SPMI_HFSMPS_MODE_HPM_MASK:
+		return REGULATOR_MODE_NORMAL;
+	case SPMI_HFSMPS_MODE_AUTO_MASK:
+		return REGULATOR_MODE_FAST;
+	default:
+		return REGULATOR_MODE_IDLE;
+	}
+}
+
 static int
 spmi_regulator_common_set_mode(struct regulator_dev *rdev, unsigned int mode)
 {
@@ -1109,6 +1170,33 @@ spmi_regulator_ftsmps426_set_mode(struct regulator_dev *rdev, unsigned int mode)
 }
 
 static int
+spmi_regulator_hfsmps_set_mode(struct regulator_dev *rdev, unsigned int mode)
+{
+	struct spmi_regulator *vreg = rdev_get_drvdata(rdev);
+	u8 mask = SPMI_HFSMPS_MODE_MASK;
+	u8 val;
+
+	switch (mode) {
+	case REGULATOR_MODE_NORMAL:
+		val = SPMI_HFSMPS_MODE_HPM_MASK;
+		break;
+	case REGULATOR_MODE_FAST:
+		val = SPMI_HFSMPS_MODE_AUTO_MASK;
+		break;
+	case REGULATOR_MODE_IDLE:
+		val = vreg->logical_type ==
+				SPMI_REGULATOR_LOGICAL_TYPE_FTSMPS3 ?
+			SPMI_HFSMPS_MODE_RETENTION_MASK :
+			SPMI_HFSMPS_MODE_LPM_MASK;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return spmi_vreg_update_bits(vreg, SPMI_COMMON_REG_MODE, val, mask);
+}
+
+static int
 spmi_regulator_common_set_load(struct regulator_dev *rdev, int load_uA)
 {
 	struct spmi_regulator *vreg = rdev_get_drvdata(rdev);
@@ -1131,6 +1219,15 @@ static int spmi_regulator_common_set_pull_down(struct regulator_dev *rdev)
 				     mask, mask);
 }
 
+static int spmi_regulator_hfsmps_set_pull_down(struct regulator_dev *rdev)
+{
+	struct spmi_regulator *vreg = rdev_get_drvdata(rdev);
+	unsigned int mask = SPMI_COMMON_PULL_DOWN_ENABLE_MASK;
+
+	return spmi_vreg_update_bits(vreg, SPMI_HFSMPS_REG_PULL_DOWN,
+				     mask, mask);
+}
+
 static int spmi_regulator_common_set_soft_start(struct regulator_dev *rdev)
 {
 	struct spmi_regulator *vreg = rdev_get_drvdata(rdev);
@@ -1465,6 +1562,21 @@ static const struct regulator_ops spmi_hfs430_ops = {
 	.get_mode		= spmi_regulator_ftsmps426_get_mode,
 };
 
+static const struct regulator_ops spmi_hfsmps_ops = {
+	.enable			= regulator_enable_regmap,
+	.disable		= regulator_disable_regmap,
+	.is_enabled		= regulator_is_enabled_regmap,
+	.set_voltage_sel	= spmi_regulator_ftsmps426_set_voltage,
+	.set_voltage_time_sel	= spmi_regulator_set_voltage_time_sel,
+	.get_voltage_sel	= spmi_regulator_ftsmps426_get_voltage,
+	.map_voltage		= spmi_regulator_single_map_voltage,
+	.list_voltage		= spmi_regulator_common_list_voltage,
+	.set_mode		= spmi_regulator_hfsmps_set_mode,
+	.get_mode		= spmi_regulator_hfsmps_get_mode,
+	.set_load		= spmi_regulator_common_set_load,
+	.set_pull_down		= spmi_regulator_hfsmps_set_pull_down,
+};
+
 /* Maximum possible digital major revision value */
 #define INF 0xFF
 
@@ -1473,7 +1585,8 @@ static const struct spmi_regulator_mapping supported_regulators[] = {
 	SPMI_VREG(LDO,   HT_P600,  0, INF, HFS430, hfs430, ht_p600, 10000),
 	SPMI_VREG(LDO,   HT_P150,  0, INF, HFS430, hfs430, ht_p150, 10000),
 	SPMI_VREG(BUCK,  GP_CTL,   0, INF, SMPS,   smps,   smps,   100000),
-	SPMI_VREG(BUCK,  HFS430,   0, INF, HFS430, hfs430, hfs430,  10000),
+	SPMI_VREG(BUCK,  HFS430,   0,   3, HFS430, hfs430, hfs430,  10000),
+	SPMI_VREG(BUCK,  HFSMPS_510, 4, INF, HFSMPS, hfsmps, hfs430, 100000),
 	SPMI_VREG(LDO,   N300,     0, INF, LDO,    ldo,    nldo1,   10000),
 	SPMI_VREG(LDO,   N600,     0,   0, LDO,    ldo,    nldo2,   10000),
 	SPMI_VREG(LDO,   N1200,    0,   0, LDO,    ldo,    nldo2,   10000),
@@ -1549,6 +1662,16 @@ static const struct spmi_regulator_mapping supported_regulators[] = {
 	SPMI_VREG(ULT_LDO, P300,     0, INF, ULT_LDO, ult_ldo, ult_pldo, 10000),
 	SPMI_VREG(ULT_LDO, P150,     0, INF, ULT_LDO, ult_ldo, ult_pldo, 10000),
 	SPMI_VREG(ULT_LDO, P50,     0, INF, ULT_LDO, ult_ldo, ult_pldo, 5000),
+	SPMI_VREG(LDO, LV_P150_510, 0, INF, LDO_510, hfsmps, ht_lvpldo, 10000),
+	SPMI_VREG(LDO, LV_P300_510, 0, INF, LDO_510, hfsmps, ht_lvpldo, 10000),
+	SPMI_VREG(LDO, LV_P600_510, 0, INF, LDO_510, hfsmps, ht_lvpldo, 10000),
+	SPMI_VREG(LDO, MV_P50_510,  0, INF, LDO_510, hfsmps, pldo660, 10000),
+	SPMI_VREG(LDO, MV_P150_510, 0, INF, LDO_510, hfsmps, pldo660, 10000),
+	SPMI_VREG(LDO, MV_P600_510, 0, INF, LDO_510, hfsmps, pldo660, 10000),
+	SPMI_VREG(LDO, N300_510,    0, INF, LDO_510, hfsmps, nldo_510, 10000),
+	SPMI_VREG(LDO, N600_510,    0, INF, LDO_510, hfsmps, nldo_510, 10000),
+	SPMI_VREG(LDO, N1200_510,   0, INF, LDO_510, hfsmps, nldo_510, 10000),
+	SPMI_VREG(FTS, FTSMPS_510,  0, INF, FTSMPS3, hfsmps, ftsmps510, 100000),
 };
 
 static void spmi_calculate_num_voltages(struct spmi_voltage_set_points *points)
@@ -1696,6 +1819,26 @@ static int spmi_regulator_init_slew_rate_ftsmps426(struct spmi_regulator *vreg,
 	return ret;
 }
 
+static int spmi_regulator_init_slew_rate_hfsmps(struct spmi_regulator *vreg)
+{
+	int ret;
+	u8 reg = 0;
+	int delay;
+
+	ret = spmi_vreg_read(vreg, SPMI_HFSMPS_REG_STEP_CTRL, &reg, 1);
+	if (ret) {
+		dev_err(vreg->dev, "spmi read failed, ret=%d\n", ret);
+		return ret;
+	}
+
+	delay = reg & SPMI_FTSMPS426_STEP_CTRL_DELAY_MASK;
+	delay >>= SPMI_FTSMPS426_STEP_CTRL_DELAY_SHIFT;
+
+	vreg->slew_rate = SPMI_HFSMPS_SLEW_RATE_38p4 >> delay;
+
+	return ret;
+}
+
 static int spmi_regulator_init_registers(struct spmi_regulator *vreg,
 				const struct spmi_regulator_init_data *data)
 {
@@ -1846,6 +1989,12 @@ static int spmi_regulator_of_parse(struct device_node *node,
 		if (ret)
 			return ret;
 		break;
+	case SPMI_REGULATOR_LOGICAL_TYPE_HFSMPS:
+	case SPMI_REGULATOR_LOGICAL_TYPE_FTSMPS3:
+		ret = spmi_regulator_init_slew_rate_hfsmps(vreg);
+		if (ret)
+			return ret;
+		break;
 	default:
 		break;
 	}
@@ -1872,40 +2021,100 @@ static int spmi_regulator_of_parse(struct device_node *node,
 	return 0;
 }
 
-static const struct spmi_regulator_data pm8941_regulators[] = {
+static const struct spmi_regulator_data pm6125_regulators[] = {
+	{ "s1", 0x1400, "vdd_s1" },
+	{ "s2", 0x1700, "vdd_s2" },
+	{ "s3", 0x1a00, "vdd_s3" },
+	{ "s4", 0x1d00, "vdd_s4" },
+	{ "s5", 0x2000, "vdd_s5" },
+	{ "s6", 0x2300, "vdd_s6" },
+	{ "s7", 0x2600, "vdd_s7" },
+	{ "s8", 0x2900, "vdd_s8" },
+	{ "l1", 0x4000, "vdd_l1_l7_l17_l18" },
+	{ "l2", 0x4100, "vdd_l2_l3_l4" },
+	{ "l3", 0x4200, "vdd_l2_l3_l4" },
+	{ "l4", 0x4300, "vdd_l2_l3_l4" },
+	{ "l5", 0x4400, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l6", 0x4500, "vdd_l6_l8" },
+	{ "l7", 0x4600, "vdd_l1_l7_l17_l18" },
+	{ "l8", 0x4700, "vdd_l6_l8" },
+	{ "l9", 0x4800, "vdd_l9_l11" },
+	{ "l10", 0x4900, "vdd_l10_l13_l14" },
+	{ "l11", 0x4a00, "vdd_l9_l11" },
+	{ "l12", 0x4b00, "vdd_l12_l16" },
+	{ "l13", 0x4c00, "vdd_l10_l13_l14" },
+	{ "l14", 0x4d00, "vdd_l10_l13_l14" },
+	{ "l15", 0x4e00, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l16", 0x4f00, "vdd_l12_l16" },
+	{ "l17", 0x5000, "vdd_l1_l7_l17_l18" },
+	{ "l18", 0x5100, "vdd_l1_l7_l17_l18" },
+	{ "l19", 0x5200, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l20", 0x5300, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l21", 0x5400, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l22", 0x5500, "vdd_l5_l15_l19_l20_l21_l22" },
+	{ "l23", 0x5600, "vdd_l23_l24" },
+	{ "l24", 0x5700, "vdd_l23_l24" },
+};
+
+static const struct spmi_regulator_data pm660_regulators[] = {
 	{ "s1", 0x1400, "vdd_s1", },
 	{ "s2", 0x1700, "vdd_s2", },
 	{ "s3", 0x1a00, "vdd_s3", },
-	{ "s4", 0xa000, },
-	{ "l1", 0x4000, "vdd_l1_l3", },
-	{ "l2", 0x4100, "vdd_l2_lvs_1_2_3", },
-	{ "l3", 0x4200, "vdd_l1_l3", },
-	{ "l4", 0x4300, "vdd_l4_l11", },
-	{ "l5", 0x4400, "vdd_l5_l7", NULL, 0x0410 },
-	{ "l6", 0x4500, "vdd_l6_l12_l14_l15", },
-	{ "l7", 0x4600, "vdd_l5_l7", NULL, 0x0410 },
-	{ "l8", 0x4700, "vdd_l8_l16_l18_19", },
-	{ "l9", 0x4800, "vdd_l9_l10_l17_l22", },
-	{ "l10", 0x4900, "vdd_l9_l10_l17_l22", },
-	{ "l11", 0x4a00, "vdd_l4_l11", },
-	{ "l12", 0x4b00, "vdd_l6_l12_l14_l15", },
-	{ "l13", 0x4c00, "vdd_l13_l20_l23_l24", },
-	{ "l14", 0x4d00, "vdd_l6_l12_l14_l15", },
-	{ "l15", 0x4e00, "vdd_l6_l12_l14_l15", },
-	{ "l16", 0x4f00, "vdd_l8_l16_l18_19", },
-	{ "l17", 0x5000, "vdd_l9_l10_l17_l22", },
-	{ "l18", 0x5100, "vdd_l8_l16_l18_19", },
-	{ "l19", 0x5200, "vdd_l8_l16_l18_19", },
-	{ "l20", 0x5300, "vdd_l13_l20_l23_l24", },
-	{ "l21", 0x5400, "vdd_l21", },
-	{ "l22", 0x5500, "vdd_l9_l10_l17_l22", },
-	{ "l23", 0x5600, "vdd_l13_l20_l23_l24", },
-	{ "l24", 0x5700, "vdd_l13_l20_l23_l24", },
-	{ "lvs1", 0x8000, "vdd_l2_lvs_1_2_3", },
-	{ "lvs2", 0x8100, "vdd_l2_lvs_1_2_3", },
-	{ "lvs3", 0x8200, "vdd_l2_lvs_1_2_3", },
-	{ "5vs1", 0x8300, "vin_5vs", "ocp-5vs1", },
-	{ "5vs2", 0x8400, "vin_5vs", "ocp-5vs2", },
+	{ "s4", 0x1d00, "vdd_s3", },
+	{ "s5", 0x2000, "vdd_s5", },
+	{ "s6", 0x2300, "vdd_s6", },
+	{ "l1", 0x4000, "vdd_l1_l6_l7", },
+	{ "l2", 0x4100, "vdd_l2_l3", },
+	{ "l3", 0x4200, "vdd_l2_l3", },
+	/* l4 is unaccessible on PM660 */
+	{ "l5", 0x4400, "vdd_l5", },
+	{ "l6", 0x4500, "vdd_l1_l6_l7", },
+	{ "l7", 0x4600, "vdd_l1_l6_l7", },
+	{ "l8", 0x4700, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+	{ "l9", 0x4800, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+	{ "l10", 0x4900, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+	{ "l11", 0x4a00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+	{ "l12", 0x4b00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+	{ "l13", 0x4c00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+	{ "l14", 0x4d00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
+	{ "l15", 0x4e00, "vdd_l15_l16_l17_l18_l19", },
+	{ "l16", 0x4f00, "vdd_l15_l16_l17_l18_l19", },
+	{ "l17", 0x5000, "vdd_l15_l16_l17_l18_l19", },
+	{ "l18", 0x5100, "vdd_l15_l16_l17_l18_l19", },
+	{ "l19", 0x5200, "vdd_l15_l16_l17_l18_l19", },
+	{ }
+};
+
+static const struct spmi_regulator_data pm660l_regulators[] = {
+	{ "s1", 0x1400, "vdd_s1", },
+	{ "s2", 0x1700, "vdd_s2", },
+	{ "s3", 0x1a00, "vdd_s3", },
+	{ "s4", 0x1d00, "vdd_s4", },
+	{ "s5", 0x2000, "vdd_s5", },
+	{ "l1", 0x4000, "vdd_l1_l9_l10", },
+	{ "l2", 0x4100, "vdd_l2", },
+	{ "l3", 0x4200, "vdd_l3_l5_l7_l8", },
+	{ "l4", 0x4300, "vdd_l4_l6", },
+	{ "l5", 0x4400, "vdd_l3_l5_l7_l8", },
+	{ "l6", 0x4500, "vdd_l4_l6", },
+	{ "l7", 0x4600, "vdd_l3_l5_l7_l8", },
+	{ "l8", 0x4700, "vdd_l3_l5_l7_l8", },
+	{ "l9", 0x4800, "vdd_l1_l9_l10", },
+	{ "l10", 0x4900, "vdd_l1_l9_l10", },
+	{ }
+};
+
+static const struct spmi_regulator_data pm8004_regulators[] = {
+	{ "s2", 0x1700, "vdd_s2", },
+	{ "s5", 0x2000, "vdd_s5", },
+	{ }
+};
+
+static const struct spmi_regulator_data pm8005_regulators[] = {
+	{ "s1", 0x1400, "vdd_s1", },
+	{ "s2", 0x1700, "vdd_s2", },
+	{ "s3", 0x1a00, "vdd_s3", },
+	{ "s4", 0x1d00, "vdd_s4", },
 	{ }
 };
 
@@ -1985,6 +2194,43 @@ static const struct spmi_regulator_data pm8916_regulators[] = {
 	{ }
 };
 
+static const struct spmi_regulator_data pm8941_regulators[] = {
+	{ "s1", 0x1400, "vdd_s1", },
+	{ "s2", 0x1700, "vdd_s2", },
+	{ "s3", 0x1a00, "vdd_s3", },
+	{ "s4", 0xa000, },
+	{ "l1", 0x4000, "vdd_l1_l3", },
+	{ "l2", 0x4100, "vdd_l2_lvs_1_2_3", },
+	{ "l3", 0x4200, "vdd_l1_l3", },
+	{ "l4", 0x4300, "vdd_l4_l11", },
+	{ "l5", 0x4400, "vdd_l5_l7", NULL, 0x0410 },
+	{ "l6", 0x4500, "vdd_l6_l12_l14_l15", },
+	{ "l7", 0x4600, "vdd_l5_l7", NULL, 0x0410 },
+	{ "l8", 0x4700, "vdd_l8_l16_l18_19", },
+	{ "l9", 0x4800, "vdd_l9_l10_l17_l22", },
+	{ "l10", 0x4900, "vdd_l9_l10_l17_l22", },
+	{ "l11", 0x4a00, "vdd_l4_l11", },
+	{ "l12", 0x4b00, "vdd_l6_l12_l14_l15", },
+	{ "l13", 0x4c00, "vdd_l13_l20_l23_l24", },
+	{ "l14", 0x4d00, "vdd_l6_l12_l14_l15", },
+	{ "l15", 0x4e00, "vdd_l6_l12_l14_l15", },
+	{ "l16", 0x4f00, "vdd_l8_l16_l18_19", },
+	{ "l17", 0x5000, "vdd_l9_l10_l17_l22", },
+	{ "l18", 0x5100, "vdd_l8_l16_l18_19", },
+	{ "l19", 0x5200, "vdd_l8_l16_l18_19", },
+	{ "l20", 0x5300, "vdd_l13_l20_l23_l24", },
+	{ "l21", 0x5400, "vdd_l21", },
+	{ "l22", 0x5500, "vdd_l9_l10_l17_l22", },
+	{ "l23", 0x5600, "vdd_l13_l20_l23_l24", },
+	{ "l24", 0x5700, "vdd_l13_l20_l23_l24", },
+	{ "lvs1", 0x8000, "vdd_l2_lvs_1_2_3", },
+	{ "lvs2", 0x8100, "vdd_l2_lvs_1_2_3", },
+	{ "lvs3", 0x8200, "vdd_l2_lvs_1_2_3", },
+	{ "5vs1", 0x8300, "vin_5vs", "ocp-5vs1", },
+	{ "5vs2", 0x8400, "vin_5vs", "ocp-5vs2", },
+	{ }
+};
+
 static const struct spmi_regulator_data pm8950_regulators[] = {
 	{ "s1", 0x1400, "vdd_s1", },
 	{ "s2", 0x1700, "vdd_s2", },
@@ -2076,69 +2322,6 @@ static const struct spmi_regulator_data pmi8994_regulators[] = {
 	{ }
 };
 
-static const struct spmi_regulator_data pm660_regulators[] = {
-	{ "s1", 0x1400, "vdd_s1", },
-	{ "s2", 0x1700, "vdd_s2", },
-	{ "s3", 0x1a00, "vdd_s3", },
-	{ "s4", 0x1d00, "vdd_s3", },
-	{ "s5", 0x2000, "vdd_s5", },
-	{ "s6", 0x2300, "vdd_s6", },
-	{ "l1", 0x4000, "vdd_l1_l6_l7", },
-	{ "l2", 0x4100, "vdd_l2_l3", },
-	{ "l3", 0x4200, "vdd_l2_l3", },
-	/* l4 is unaccessible on PM660 */
-	{ "l5", 0x4400, "vdd_l5", },
-	{ "l6", 0x4500, "vdd_l1_l6_l7", },
-	{ "l7", 0x4600, "vdd_l1_l6_l7", },
-	{ "l8", 0x4700, "vdd_l8_l9_l10_l11_l12_l13_l14", },
-	{ "l9", 0x4800, "vdd_l8_l9_l10_l11_l12_l13_l14", },
-	{ "l10", 0x4900, "vdd_l8_l9_l10_l11_l12_l13_l14", },
-	{ "l11", 0x4a00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
-	{ "l12", 0x4b00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
-	{ "l13", 0x4c00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
-	{ "l14", 0x4d00, "vdd_l8_l9_l10_l11_l12_l13_l14", },
-	{ "l15", 0x4e00, "vdd_l15_l16_l17_l18_l19", },
-	{ "l16", 0x4f00, "vdd_l15_l16_l17_l18_l19", },
-	{ "l17", 0x5000, "vdd_l15_l16_l17_l18_l19", },
-	{ "l18", 0x5100, "vdd_l15_l16_l17_l18_l19", },
-	{ "l19", 0x5200, "vdd_l15_l16_l17_l18_l19", },
-	{ }
-};
-
-static const struct spmi_regulator_data pm660l_regulators[] = {
-	{ "s1", 0x1400, "vdd_s1", },
-	{ "s2", 0x1700, "vdd_s2", },
-	{ "s3", 0x1a00, "vdd_s3", },
-	{ "s4", 0x1d00, "vdd_s4", },
-	{ "s5", 0x2000, "vdd_s5", },
-	{ "l1", 0x4000, "vdd_l1_l9_l10", },
-	{ "l2", 0x4100, "vdd_l2", },
-	{ "l3", 0x4200, "vdd_l3_l5_l7_l8", },
-	{ "l4", 0x4300, "vdd_l4_l6", },
-	{ "l5", 0x4400, "vdd_l3_l5_l7_l8", },
-	{ "l6", 0x4500, "vdd_l4_l6", },
-	{ "l7", 0x4600, "vdd_l3_l5_l7_l8", },
-	{ "l8", 0x4700, "vdd_l3_l5_l7_l8", },
-	{ "l9", 0x4800, "vdd_l1_l9_l10", },
-	{ "l10", 0x4900, "vdd_l1_l9_l10", },
-	{ }
-};
-
-
-static const struct spmi_regulator_data pm8004_regulators[] = {
-	{ "s2", 0x1700, "vdd_s2", },
-	{ "s5", 0x2000, "vdd_s5", },
-	{ }
-};
-
-static const struct spmi_regulator_data pm8005_regulators[] = {
-	{ "s1", 0x1400, "vdd_s1", },
-	{ "s2", 0x1700, "vdd_s2", },
-	{ "s3", 0x1a00, "vdd_s3", },
-	{ "s4", 0x1d00, "vdd_s4", },
-	{ }
-};
-
 static const struct spmi_regulator_data pmp8074_regulators[] = {
 	{ "s1", 0x1400, "vdd_s1"},
 	{ "s2", 0x1700, "vdd_s2"},
@@ -2167,6 +2350,9 @@ static const struct spmi_regulator_data pms405_regulators[] = {
 };
 
 static const struct of_device_id qcom_spmi_regulator_match[] = {
+	{ .compatible = "qcom,pm6125-regulators", .data = &pm6125_regulators },
+	{ .compatible = "qcom,pm660-regulators", .data = &pm660_regulators },
+	{ .compatible = "qcom,pm660l-regulators", .data = &pm660l_regulators },
 	{ .compatible = "qcom,pm8004-regulators", .data = &pm8004_regulators },
 	{ .compatible = "qcom,pm8005-regulators", .data = &pm8005_regulators },
 	{ .compatible = "qcom,pm8226-regulators", .data = &pm8226_regulators },
@@ -2176,8 +2362,6 @@ static const struct of_device_id qcom_spmi_regulator_match[] = {
 	{ .compatible = "qcom,pm8950-regulators", .data = &pm8950_regulators },
 	{ .compatible = "qcom,pm8994-regulators", .data = &pm8994_regulators },
 	{ .compatible = "qcom,pmi8994-regulators", .data = &pmi8994_regulators },
-	{ .compatible = "qcom,pm660-regulators", .data = &pm660_regulators },
-	{ .compatible = "qcom,pm660l-regulators", .data = &pm660l_regulators },
 	{ .compatible = "qcom,pmp8074-regulators", .data = &pmp8074_regulators },
 	{ .compatible = "qcom,pms405-regulators", .data = &pms405_regulators },
 	{ }
diff --git a/drivers/regulator/rpi-panel-attiny-regulator.c b/drivers/regulator/rpi-panel-attiny-regulator.c
index 105f694a67e6..308f7972941b 100644
--- a/drivers/regulator/rpi-panel-attiny-regulator.c
+++ b/drivers/regulator/rpi-panel-attiny-regulator.c
@@ -381,13 +381,11 @@ error:
 	return ret;
 }
 
-static int attiny_i2c_remove(struct i2c_client *client)
+static void attiny_i2c_remove(struct i2c_client *client)
 {
 	struct attiny_lcd *state = i2c_get_clientdata(client);
 
 	mutex_destroy(&state->lock);
-
-	return 0;
 }
 
 static const struct of_device_id attiny_dt_ids[] = {
diff --git a/drivers/regulator/ti-abb-regulator.c b/drivers/regulator/ti-abb-regulator.c
index ce00db27589a..115345e9fded 100644
--- a/drivers/regulator/ti-abb-regulator.c
+++ b/drivers/regulator/ti-abb-regulator.c
@@ -151,7 +151,7 @@ static inline void ti_abb_clear_txdone(const struct ti_abb *abb)
 };
 
 /**
- * ti_abb_wait_tranx() - waits for ABB tranxdone event
+ * ti_abb_wait_txdone() - waits for ABB tranxdone event
  * @dev:	device
  * @abb:	pointer to the abb instance
  *
diff --git a/drivers/regulator/tps65219-regulator.c b/drivers/regulator/tps65219-regulator.c
new file mode 100644
index 000000000000..c484c943e467
--- /dev/null
+++ b/drivers/regulator/tps65219-regulator.c
@@ -0,0 +1,411 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// tps65219-regulator.c
+//
+// Regulator driver for TPS65219 PMIC
+//
+// Copyright (C) 2022 BayLibre Incorporated - https://www.baylibre.com/
+//
+// This implementation derived from tps65218 authored by
+// "J Keerthy <j-keerthy@ti.com>"
+//
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/of_device.h>
+#include <linux/regmap.h>
+#include <linux/regulator/of_regulator.h>
+#include <linux/regulator/driver.h>
+#include <linux/regulator/machine.h>
+#include <linux/mfd/tps65219.h>
+
+struct tps65219_regulator_irq_type {
+	const char *irq_name;
+	const char *regulator_name;
+	const char *event_name;
+	unsigned long event;
+};
+
+static struct tps65219_regulator_irq_type tps65219_regulator_irq_types[] = {
+	{ "LDO3_SCG", "LDO3", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT },
+	{ "LDO3_OC", "LDO3", "overcurrent", REGULATOR_EVENT_OVER_CURRENT },
+	{ "LDO3_UV", "LDO3", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE },
+	{ "LDO4_SCG", "LDO4", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT },
+	{ "LDO4_OC", "LDO4", "overcurrent", REGULATOR_EVENT_OVER_CURRENT },
+	{ "LDO4_UV", "LDO4", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE },
+	{ "LDO1_SCG", "LDO1", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT },
+	{ "LDO1_OC", "LDO1", "overcurrent", REGULATOR_EVENT_OVER_CURRENT },
+	{ "LDO1_UV", "LDO1", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE },
+	{ "LDO2_SCG", "LDO2", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT },
+	{ "LDO2_OC", "LDO2", "overcurrent", REGULATOR_EVENT_OVER_CURRENT },
+	{ "LDO2_UV", "LDO2", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE },
+	{ "BUCK3_SCG", "BUCK3", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT },
+	{ "BUCK3_OC", "BUCK3", "overcurrent", REGULATOR_EVENT_OVER_CURRENT },
+	{ "BUCK3_NEG_OC", "BUCK3", "negative overcurrent", REGULATOR_EVENT_OVER_CURRENT },
+	{ "BUCK3_UV", "BUCK3", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE },
+	{ "BUCK1_SCG", "BUCK1", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT },
+	{ "BUCK1_OC", "BUCK1", "overcurrent", REGULATOR_EVENT_OVER_CURRENT },
+	{ "BUCK1_NEG_OC", "BUCK1", "negative overcurrent", REGULATOR_EVENT_OVER_CURRENT },
+	{ "BUCK1_UV", "BUCK1", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE },
+	{ "BUCK2_SCG", "BUCK2", "short circuit to ground", REGULATOR_EVENT_REGULATION_OUT },
+	{ "BUCK2_OC", "BUCK2", "overcurrent", REGULATOR_EVENT_OVER_CURRENT },
+	{ "BUCK2_NEG_OC", "BUCK2", "negative overcurrent", REGULATOR_EVENT_OVER_CURRENT },
+	{ "BUCK2_UV", "BUCK2", "undervoltage", REGULATOR_EVENT_UNDER_VOLTAGE },
+	{ "BUCK1_RV", "BUCK1", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "BUCK2_RV", "BUCK2", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "BUCK3_RV", "BUCK3", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "LDO1_RV", "LDO1", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "LDO2_RV", "LDO2", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "LDO3_RV", "LDO3", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "LDO4_RV", "LDO4", "residual voltage", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "BUCK1_RV_SD", "BUCK1", "residual voltage on shutdown",
+	 REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "BUCK2_RV_SD", "BUCK2", "residual voltage on shutdown",
+	 REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "BUCK3_RV_SD", "BUCK3", "residual voltage on shutdown",
+	 REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "LDO1_RV_SD", "LDO1", "residual voltage on shutdown", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "LDO2_RV_SD", "LDO2", "residual voltage on shutdown", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "LDO3_RV_SD", "LDO3", "residual voltage on shutdown", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "LDO4_RV_SD", "LDO4", "residual voltage on shutdown", REGULATOR_EVENT_OVER_VOLTAGE_WARN },
+	{ "SENSOR_3_WARM", "SENSOR3", "warm temperature", REGULATOR_EVENT_OVER_TEMP_WARN},
+	{ "SENSOR_2_WARM", "SENSOR2", "warm temperature", REGULATOR_EVENT_OVER_TEMP_WARN },
+	{ "SENSOR_1_WARM", "SENSOR1", "warm temperature", REGULATOR_EVENT_OVER_TEMP_WARN },
+	{ "SENSOR_0_WARM", "SENSOR0", "warm temperature", REGULATOR_EVENT_OVER_TEMP_WARN },
+	{ "SENSOR_3_HOT", "SENSOR3", "hot temperature", REGULATOR_EVENT_OVER_TEMP},
+	{ "SENSOR_2_HOT", "SENSOR2", "hot temperature", REGULATOR_EVENT_OVER_TEMP },
+	{ "SENSOR_1_HOT", "SENSOR1", "hot temperature", REGULATOR_EVENT_OVER_TEMP },
+	{ "SENSOR_0_HOT", "SENSOR0", "hot temperature", REGULATOR_EVENT_OVER_TEMP },
+	{ "TIMEOUT", "", "", REGULATOR_EVENT_ABORT_VOLTAGE_CHANGE },
+};
+
+struct tps65219_regulator_irq_data {
+	struct device *dev;
+	struct tps65219_regulator_irq_type *type;
+	struct regulator_dev *rdev;
+};
+
+#define TPS65219_REGULATOR(_name, _of, _id, _type, _ops, _n, _vr, _vm, _er, \
+			   _em, _cr, _cm, _lr, _nlr, _delay, _fuv, \
+			   _ct, _ncl, _bpm) \
+	{								\
+		.name			= _name,			\
+		.of_match		= _of,				\
+		.regulators_node	= of_match_ptr("regulators"),	\
+		.supply_name		= _of,				\
+		.id			= _id,				\
+		.ops			= &(_ops),			\
+		.n_voltages		= _n,				\
+		.type			= _type,			\
+		.owner			= THIS_MODULE,			\
+		.vsel_reg		= _vr,				\
+		.vsel_mask		= _vm,				\
+		.csel_reg		= _cr,				\
+		.csel_mask		= _cm,				\
+		.curr_table		= _ct,				\
+		.n_current_limits	= _ncl,				\
+		.enable_reg		= _er,				\
+		.enable_mask		= _em,				\
+		.volt_table		= NULL,				\
+		.linear_ranges		= _lr,				\
+		.n_linear_ranges	= _nlr,				\
+		.ramp_delay		= _delay,			\
+		.fixed_uV		= _fuv,				\
+		.bypass_reg		= _vr,				\
+		.bypass_mask		= _bpm,				\
+	}								\
+
+static const struct linear_range bucks_ranges[] = {
+	REGULATOR_LINEAR_RANGE(600000, 0x0, 0x1f, 25000),
+	REGULATOR_LINEAR_RANGE(1400000, 0x20, 0x33, 100000),
+	REGULATOR_LINEAR_RANGE(3400000, 0x34, 0x3f, 0),
+};
+
+static const struct linear_range ldos_1_2_ranges[] = {
+	REGULATOR_LINEAR_RANGE(600000, 0x0, 0x37, 50000),
+	REGULATOR_LINEAR_RANGE(3400000, 0x38, 0x3f, 0),
+};
+
+static const struct linear_range ldos_3_4_ranges[] = {
+	REGULATOR_LINEAR_RANGE(1200000, 0x0, 0xC, 0),
+	REGULATOR_LINEAR_RANGE(1250000, 0xD, 0x35, 50000),
+	REGULATOR_LINEAR_RANGE(3300000, 0x36, 0x3F, 0),
+};
+
+static int tps65219_set_mode(struct regulator_dev *dev, unsigned int mode)
+{
+	struct tps65219 *tps = rdev_get_drvdata(dev);
+
+	switch (mode) {
+	case REGULATOR_MODE_NORMAL:
+		return regmap_set_bits(tps->regmap, TPS65219_REG_STBY_1_CONFIG,
+				       dev->desc->enable_mask);
+
+	case REGULATOR_MODE_STANDBY:
+		return regmap_clear_bits(tps->regmap,
+					 TPS65219_REG_STBY_1_CONFIG,
+					 dev->desc->enable_mask);
+	default:
+		return -EINVAL;
+	}
+}
+
+static unsigned int tps65219_get_mode(struct regulator_dev *dev)
+{
+	struct tps65219 *tps = rdev_get_drvdata(dev);
+	unsigned int rid = rdev_get_id(dev);
+	int ret, value = 0;
+
+	ret = regmap_read(tps->regmap, TPS65219_REG_STBY_1_CONFIG, &value);
+	if (ret) {
+		dev_dbg(tps->dev, "%s failed for regulator %s: %d ",
+			__func__, dev->desc->name, ret);
+		return ret;
+	}
+	value = (value & BIT(rid)) >> rid;
+	if (value)
+		return REGULATOR_MODE_STANDBY;
+	else
+		return REGULATOR_MODE_NORMAL;
+}
+
+/*
+ * generic regulator_set_bypass_regmap does not fully match requirements
+ * TPS65219 Requires explicitly that regulator is disabled before switch
+ */
+static int tps65219_set_bypass(struct regulator_dev *dev, bool enable)
+{
+	struct tps65219 *tps = rdev_get_drvdata(dev);
+	unsigned int rid = rdev_get_id(dev);
+
+	if (dev->desc->ops->is_enabled(dev)) {
+		dev_err(tps->dev,
+			"%s LDO%d enabled, must be shut down to set bypass ",
+			__func__, rid);
+		return -EBUSY;
+	}
+	return regulator_set_bypass_regmap(dev, enable);
+}
+
+/* Operations permitted on BUCK1/2/3 */
+static const struct regulator_ops tps65219_bucks_ops = {
+	.is_enabled		= regulator_is_enabled_regmap,
+	.enable			= regulator_enable_regmap,
+	.disable		= regulator_disable_regmap,
+	.set_mode		= tps65219_set_mode,
+	.get_mode		= tps65219_get_mode,
+	.get_voltage_sel	= regulator_get_voltage_sel_regmap,
+	.set_voltage_sel	= regulator_set_voltage_sel_regmap,
+	.list_voltage		= regulator_list_voltage_linear_range,
+	.map_voltage		= regulator_map_voltage_linear_range,
+	.set_voltage_time_sel	= regulator_set_voltage_time_sel,
+
+};
+
+/* Operations permitted on LDO1/2 */
+static const struct regulator_ops tps65219_ldos_1_2_ops = {
+	.is_enabled		= regulator_is_enabled_regmap,
+	.enable			= regulator_enable_regmap,
+	.disable		= regulator_disable_regmap,
+	.set_mode		= tps65219_set_mode,
+	.get_mode		= tps65219_get_mode,
+	.get_voltage_sel	= regulator_get_voltage_sel_regmap,
+	.set_voltage_sel	= regulator_set_voltage_sel_regmap,
+	.list_voltage		= regulator_list_voltage_linear_range,
+	.map_voltage		= regulator_map_voltage_linear_range,
+	.set_bypass		= tps65219_set_bypass,
+	.get_bypass		= regulator_get_bypass_regmap,
+};
+
+/* Operations permitted on LDO3/4 */
+static const struct regulator_ops tps65219_ldos_3_4_ops = {
+	.is_enabled		= regulator_is_enabled_regmap,
+	.enable			= regulator_enable_regmap,
+	.disable		= regulator_disable_regmap,
+	.set_mode		= tps65219_set_mode,
+	.get_mode		= tps65219_get_mode,
+	.get_voltage_sel	= regulator_get_voltage_sel_regmap,
+	.set_voltage_sel	= regulator_set_voltage_sel_regmap,
+	.list_voltage		= regulator_list_voltage_linear_range,
+	.map_voltage		= regulator_map_voltage_linear_range,
+};
+
+static const struct regulator_desc regulators[] = {
+	TPS65219_REGULATOR("BUCK1", "buck1", TPS65219_BUCK_1,
+			   REGULATOR_VOLTAGE, tps65219_bucks_ops, 64,
+			   TPS65219_REG_BUCK1_VOUT,
+			   TPS65219_BUCKS_LDOS_VOUT_VSET_MASK,
+			   TPS65219_REG_ENABLE_CTRL,
+			   TPS65219_ENABLE_BUCK1_EN_MASK, 0, 0, bucks_ranges,
+			   3, 4000, 0, NULL, 0, 0),
+	TPS65219_REGULATOR("BUCK2", "buck2", TPS65219_BUCK_2,
+			   REGULATOR_VOLTAGE, tps65219_bucks_ops, 64,
+			   TPS65219_REG_BUCK2_VOUT,
+			   TPS65219_BUCKS_LDOS_VOUT_VSET_MASK,
+			   TPS65219_REG_ENABLE_CTRL,
+			   TPS65219_ENABLE_BUCK2_EN_MASK, 0, 0, bucks_ranges,
+			   3, 4000, 0, NULL, 0, 0),
+	TPS65219_REGULATOR("BUCK3", "buck3", TPS65219_BUCK_3,
+			   REGULATOR_VOLTAGE, tps65219_bucks_ops, 64,
+			   TPS65219_REG_BUCK3_VOUT,
+			   TPS65219_BUCKS_LDOS_VOUT_VSET_MASK,
+			   TPS65219_REG_ENABLE_CTRL,
+			   TPS65219_ENABLE_BUCK3_EN_MASK, 0, 0, bucks_ranges,
+			   3, 0, 0, NULL, 0, 0),
+	TPS65219_REGULATOR("LDO1", "ldo1", TPS65219_LDO_1,
+			   REGULATOR_VOLTAGE, tps65219_ldos_1_2_ops, 64,
+			   TPS65219_REG_LDO1_VOUT,
+			   TPS65219_BUCKS_LDOS_VOUT_VSET_MASK,
+			   TPS65219_REG_ENABLE_CTRL,
+			   TPS65219_ENABLE_LDO1_EN_MASK, 0, 0, ldos_1_2_ranges,
+			   2, 0, 0, NULL, 0, TPS65219_LDOS_BYP_CONFIG_MASK),
+	TPS65219_REGULATOR("LDO2", "ldo2", TPS65219_LDO_2,
+			   REGULATOR_VOLTAGE, tps65219_ldos_1_2_ops, 64,
+			   TPS65219_REG_LDO2_VOUT,
+			   TPS65219_BUCKS_LDOS_VOUT_VSET_MASK,
+			   TPS65219_REG_ENABLE_CTRL,
+			   TPS65219_ENABLE_LDO2_EN_MASK, 0, 0, ldos_1_2_ranges,
+			   2, 0, 0, NULL, 0, TPS65219_LDOS_BYP_CONFIG_MASK),
+	TPS65219_REGULATOR("LDO3", "ldo3", TPS65219_LDO_3,
+			   REGULATOR_VOLTAGE, tps65219_ldos_3_4_ops, 64,
+			   TPS65219_REG_LDO3_VOUT,
+			   TPS65219_BUCKS_LDOS_VOUT_VSET_MASK,
+			   TPS65219_REG_ENABLE_CTRL,
+			   TPS65219_ENABLE_LDO3_EN_MASK, 0, 0, ldos_3_4_ranges,
+			   3, 0, 0, NULL, 0, 0),
+	TPS65219_REGULATOR("LDO4", "ldo4", TPS65219_LDO_4,
+			   REGULATOR_VOLTAGE, tps65219_ldos_3_4_ops, 64,
+			   TPS65219_REG_LDO4_VOUT,
+			   TPS65219_BUCKS_LDOS_VOUT_VSET_MASK,
+			   TPS65219_REG_ENABLE_CTRL,
+			   TPS65219_ENABLE_LDO4_EN_MASK, 0, 0, ldos_3_4_ranges,
+			   3, 0, 0, NULL, 0, 0),
+};
+
+static irqreturn_t tps65219_regulator_irq_handler(int irq, void *data)
+{
+	struct tps65219_regulator_irq_data *irq_data = data;
+
+	if (irq_data->type->event_name[0] == '\0') {
+		/* This is the timeout interrupt no specific regulator */
+		dev_err(irq_data->dev,
+			"System was put in shutdown due to timeout during an active or standby transition.\n");
+		return IRQ_HANDLED;
+	}
+
+	regulator_notifier_call_chain(irq_data->rdev,
+				      irq_data->type->event, NULL);
+
+	dev_err(irq_data->dev, "Error IRQ trap %s for %s\n",
+		irq_data->type->event_name, irq_data->type->regulator_name);
+	return IRQ_HANDLED;
+}
+
+static int tps65219_get_rdev_by_name(const char *regulator_name,
+				     struct regulator_dev *rdevtbl[7],
+				     struct regulator_dev *dev)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(regulators); i++) {
+		if (strcmp(regulator_name, regulators[i].name) == 0) {
+			dev = rdevtbl[i];
+			return 0;
+		}
+	}
+	return -EINVAL;
+}
+
+static int tps65219_regulator_probe(struct platform_device *pdev)
+{
+	struct tps65219 *tps = dev_get_drvdata(pdev->dev.parent);
+	struct regulator_dev *rdev;
+	struct regulator_config config = { };
+	int i;
+	int error;
+	int irq;
+	struct tps65219_regulator_irq_data *irq_data;
+	struct tps65219_regulator_irq_type *irq_type;
+	struct regulator_dev *rdevtbl[7];
+
+	config.dev = tps->dev;
+	config.driver_data = tps;
+	config.regmap = tps->regmap;
+
+	for (i = 0; i < ARRAY_SIZE(regulators); i++) {
+		dev_dbg(tps->dev, "%s regul i= %d START", __func__, i);
+		rdev = devm_regulator_register(&pdev->dev, &regulators[i],
+					       &config);
+		if (IS_ERR(rdev)) {
+			dev_err(tps->dev, "failed to register %s regulator\n",
+				pdev->name);
+			return PTR_ERR(rdev);
+		}
+		rdevtbl[i] = rdev;
+		dev_dbg(tps->dev, "%s regul i= %d COMPLETED", __func__, i);
+	}
+
+	irq_data = devm_kmalloc(tps->dev,
+				ARRAY_SIZE(tps65219_regulator_irq_types) *
+				sizeof(struct tps65219_regulator_irq_data),
+				GFP_KERNEL);
+	if (!irq_data)
+		return -ENOMEM;
+
+	for (i = 0; i < ARRAY_SIZE(tps65219_regulator_irq_types); ++i) {
+		irq_type = &tps65219_regulator_irq_types[i];
+
+		irq = platform_get_irq_byname(pdev, irq_type->irq_name);
+		if (irq < 0)
+			return -EINVAL;
+
+		irq_data[i].dev = tps->dev;
+		irq_data[i].type = irq_type;
+
+		tps65219_get_rdev_by_name(irq_type->regulator_name, rdevtbl, rdev);
+		if (rdev < 0) {
+			dev_err(tps->dev, "Failed to get rdev for %s\n",
+				irq_type->regulator_name);
+			return -EINVAL;
+		}
+		irq_data[i].rdev = rdev;
+
+		error = devm_request_threaded_irq(tps->dev, irq, NULL,
+						  tps65219_regulator_irq_handler,
+						  IRQF_ONESHOT,
+						  irq_type->irq_name,
+						  &irq_data[i]);
+		if (error) {
+			dev_err(tps->dev, "failed to request %s IRQ %d: %d\n",
+				irq_type->irq_name, irq, error);
+			return error;
+		}
+	}
+
+	return 0;
+}
+
+static const struct platform_device_id tps65219_regulator_id_table[] = {
+	{ "tps65219-regulator", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(platform, tps65219_regulator_id_table);
+
+static struct platform_driver tps65219_regulator_driver = {
+	.driver = {
+		.name = "tps65219-pmic",
+	},
+	.probe = tps65219_regulator_probe,
+	.id_table = tps65219_regulator_id_table,
+};
+
+module_platform_driver(tps65219_regulator_driver);
+
+MODULE_AUTHOR("Jerome Neanne <j-neanne@baylibre.com>");
+MODULE_DESCRIPTION("TPS65219 voltage regulator driver");
+MODULE_ALIAS("platform:tps65219-pmic");
+MODULE_LICENSE("GPL");
diff --git a/drivers/reset/Kconfig b/drivers/reset/Kconfig
index 806773e88832..2a8238eb8794 100644
--- a/drivers/reset/Kconfig
+++ b/drivers/reset/Kconfig
@@ -201,7 +201,7 @@ config RESET_SCMI
 
 config RESET_SIMPLE
 	bool "Simple Reset Controller Driver" if COMPILE_TEST || EXPERT
-	default ARCH_ASPEED || ARCH_BCM4908 || ARCH_BITMAIN || ARCH_REALTEK || ARCH_STM32 || (ARCH_INTEL_SOCFPGA && ARM64) || ARCH_SUNXI || ARC
+	default ARCH_ASPEED || ARCH_BCMBCA || ARCH_BITMAIN || ARCH_REALTEK || ARCH_STM32 || (ARCH_INTEL_SOCFPGA && ARM64) || ARCH_SUNXI || ARC
 	depends on HAS_IOMEM
 	help
 	  This enables a simple reset controller driver for reset lines that
diff --git a/drivers/reset/reset-imx7.c b/drivers/reset/reset-imx7.c
index 185a333df66c..d2408725eb2c 100644
--- a/drivers/reset/reset-imx7.c
+++ b/drivers/reset/reset-imx7.c
@@ -329,6 +329,7 @@ static int imx8mp_reset_set(struct reset_controller_dev *rcdev,
 		break;
 
 	case IMX8MP_RESET_PCIE_CTRL_APPS_EN:
+	case IMX8MP_RESET_PCIEPHY_PERST:
 		value = assert ? 0 : bit;
 		break;
 	}
diff --git a/drivers/reset/reset-microchip-sparx5.c b/drivers/reset/reset-microchip-sparx5.c
index 00b612a0effa..f3528dd1d084 100644
--- a/drivers/reset/reset-microchip-sparx5.c
+++ b/drivers/reset/reset-microchip-sparx5.c
@@ -33,11 +33,8 @@ static struct regmap_config sparx5_reset_regmap_config = {
 	.reg_stride	= 4,
 };
 
-static int sparx5_switch_reset(struct reset_controller_dev *rcdev,
-			       unsigned long id)
+static int sparx5_switch_reset(struct mchp_reset_context *ctx)
 {
-	struct mchp_reset_context *ctx =
-		container_of(rcdev, struct mchp_reset_context, rcdev);
 	u32 val;
 
 	/* Make sure the core is PROTECTED from reset */
@@ -54,8 +51,14 @@ static int sparx5_switch_reset(struct reset_controller_dev *rcdev,
 					1, 100);
 }
 
+static int sparx5_reset_noop(struct reset_controller_dev *rcdev,
+			     unsigned long id)
+{
+	return 0;
+}
+
 static const struct reset_control_ops sparx5_reset_ops = {
-	.reset = sparx5_switch_reset,
+	.reset = sparx5_reset_noop,
 };
 
 static int mchp_sparx5_map_syscon(struct platform_device *pdev, char *name,
@@ -122,6 +125,11 @@ static int mchp_sparx5_reset_probe(struct platform_device *pdev)
 	ctx->rcdev.of_node = dn;
 	ctx->props = device_get_match_data(&pdev->dev);
 
+	/* Issue the reset very early, our actual reset callback is a noop. */
+	err = sparx5_switch_reset(ctx);
+	if (err)
+		return err;
+
 	return devm_reset_controller_register(&pdev->dev, &ctx->rcdev);
 }
 
@@ -163,6 +171,10 @@ static int __init mchp_sparx5_reset_init(void)
 	return platform_driver_register(&mchp_sparx5_reset_driver);
 }
 
+/*
+ * Because this is a global reset, keep this postcore_initcall() to issue the
+ * reset as early as possible during the kernel startup.
+ */
 postcore_initcall(mchp_sparx5_reset_init);
 
 MODULE_DESCRIPTION("Microchip Sparx5 switch reset driver");
diff --git a/drivers/reset/reset-npcm.c b/drivers/reset/reset-npcm.c
index 24c55efa98e5..f2333506b0a6 100644
--- a/drivers/reset/reset-npcm.c
+++ b/drivers/reset/reset-npcm.c
@@ -291,7 +291,7 @@ static void npcm_usb_reset_npcm8xx(struct npcm_rc_data *rc)
 	iprst2 |= ipsrst2_bits;
 	iprst3 |= (ipsrst3_bits | NPCM_IPSRST3_USBPHY1 |
 		   NPCM_IPSRST3_USBPHY2);
-	iprst2 |= ipsrst4_bits;
+	iprst4 |= ipsrst4_bits;
 
 	writel(iprst1, rc->base + NPCM_IPSRST1);
 	writel(iprst2, rc->base + NPCM_IPSRST2);
diff --git a/drivers/rtc/rtc-bq32k.c b/drivers/rtc/rtc-bq32k.c
index e0bbb11d912e..6d6a55efb9cc 100644
--- a/drivers/rtc/rtc-bq32k.c
+++ b/drivers/rtc/rtc-bq32k.c
@@ -297,11 +297,9 @@ static int bq32k_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int bq32k_remove(struct i2c_client *client)
+static void bq32k_remove(struct i2c_client *client)
 {
 	bq32k_sysfs_unregister(&client->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id bq32k_id[] = {
diff --git a/drivers/rtc/rtc-ds1374.c b/drivers/rtc/rtc-ds1374.c
index b19de5100b1a..7f089f066163 100644
--- a/drivers/rtc/rtc-ds1374.c
+++ b/drivers/rtc/rtc-ds1374.c
@@ -530,7 +530,7 @@ static int ds1374_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int ds1374_remove(struct i2c_client *client)
+static void ds1374_remove(struct i2c_client *client)
 {
 	struct ds1374 *ds1374 = i2c_get_clientdata(client);
 
@@ -542,8 +542,6 @@ static int ds1374_remove(struct i2c_client *client)
 		devm_free_irq(&client->dev, client->irq, client);
 		cancel_work_sync(&ds1374->work);
 	}
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/rtc/rtc-isl12026.c b/drivers/rtc/rtc-isl12026.c
index 1fc6627d854d..1bfca39079d4 100644
--- a/drivers/rtc/rtc-isl12026.c
+++ b/drivers/rtc/rtc-isl12026.c
@@ -472,12 +472,11 @@ static int isl12026_probe_new(struct i2c_client *client)
 	return devm_rtc_register_device(priv->rtc);
 }
 
-static int isl12026_remove(struct i2c_client *client)
+static void isl12026_remove(struct i2c_client *client)
 {
 	struct isl12026 *priv = i2c_get_clientdata(client);
 
 	i2c_unregister_device(priv->nvm_client);
-	return 0;
 }
 
 static const struct of_device_id isl12026_dt_match[] = {
diff --git a/drivers/rtc/rtc-m41t80.c b/drivers/rtc/rtc-m41t80.c
index d868458cd40e..e0b4d3794320 100644
--- a/drivers/rtc/rtc-m41t80.c
+++ b/drivers/rtc/rtc-m41t80.c
@@ -989,7 +989,7 @@ static int m41t80_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int m41t80_remove(struct i2c_client *client)
+static void m41t80_remove(struct i2c_client *client)
 {
 #ifdef CONFIG_RTC_DRV_M41T80_WDT
 	struct m41t80_data *clientdata = i2c_get_clientdata(client);
@@ -999,8 +999,6 @@ static int m41t80_remove(struct i2c_client *client)
 		unregister_reboot_notifier(&wdt_notifier);
 	}
 #endif
-
-	return 0;
 }
 
 static struct i2c_driver m41t80_driver = {
diff --git a/drivers/rtc/rtc-rs5c372.c b/drivers/rtc/rtc-rs5c372.c
index cb15983383f5..9562c477e1c9 100644
--- a/drivers/rtc/rtc-rs5c372.c
+++ b/drivers/rtc/rtc-rs5c372.c
@@ -910,10 +910,9 @@ exit:
 	return err;
 }
 
-static int rs5c372_remove(struct i2c_client *client)
+static void rs5c372_remove(struct i2c_client *client)
 {
 	rs5c_sysfs_unregister(&client->dev);
-	return 0;
 }
 
 static struct i2c_driver rs5c372_driver = {
diff --git a/drivers/rtc/rtc-x1205.c b/drivers/rtc/rtc-x1205.c
index ba0d22a5b421..f587afa84357 100644
--- a/drivers/rtc/rtc-x1205.c
+++ b/drivers/rtc/rtc-x1205.c
@@ -657,10 +657,9 @@ static int x1205_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int x1205_remove(struct i2c_client *client)
+static void x1205_remove(struct i2c_client *client)
 {
 	x1205_sysfs_unregister(&client->dev);
-	return 0;
 }
 
 static const struct i2c_device_id x1205_id[] = {
diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c
index 2d4436cbcb47..9dc935886e9f 100644
--- a/drivers/s390/net/qeth_l2_main.c
+++ b/drivers/s390/net/qeth_l2_main.c
@@ -1133,7 +1133,7 @@ static int qeth_l2_setup_netdev(struct qeth_card *card)
 				       PAGE_SIZE * (QDIO_MAX_ELEMENTS_PER_BUFFER - 1));
 	}
 
-	netif_napi_add(card->dev, &card->napi, qeth_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(card->dev, &card->napi, qeth_poll);
 	return register_netdev(card->dev);
 }
 
@@ -1530,8 +1530,8 @@ static void qeth_addr_change_event(struct qeth_card *card,
 	else
 		INIT_DELAYED_WORK(&data->dwork, qeth_l2_dev2br_worker);
 	data->card = card;
-	memcpy(&data->ac_event, hostevs,
-			sizeof(struct qeth_ipacmd_addr_change) + extrasize);
+	data->ac_event = *hostevs;
+	memcpy(data->ac_event.entry, hostevs->entry, extrasize);
 	queue_delayed_work(card->event_wq, &data->dwork, 0);
 }
 
diff --git a/drivers/s390/net/qeth_l3_main.c b/drivers/s390/net/qeth_l3_main.c
index 8d44bce0477a..d8487a10cd55 100644
--- a/drivers/s390/net/qeth_l3_main.c
+++ b/drivers/s390/net/qeth_l3_main.c
@@ -1910,7 +1910,7 @@ static int qeth_l3_setup_netdev(struct qeth_card *card)
 		netif_set_tso_max_size(card->dev,
 				       PAGE_SIZE * (QETH_MAX_BUFFER_ELEMENTS(card) - 1));
 
-	netif_napi_add(card->dev, &card->napi, qeth_poll, NAPI_POLL_WEIGHT);
+	netif_napi_add(card->dev, &card->napi, qeth_poll);
 	return register_netdev(card->dev);
 }
 
diff --git a/drivers/scsi/libsas/sas_scsi_host.c b/drivers/scsi/libsas/sas_scsi_host.c
index 9c82e5dc4fcc..a36fa1c128a8 100644
--- a/drivers/scsi/libsas/sas_scsi_host.c
+++ b/drivers/scsi/libsas/sas_scsi_host.c
@@ -872,7 +872,8 @@ int sas_change_queue_depth(struct scsi_device *sdev, int depth)
 	struct domain_device *dev = sdev_to_domain_dev(sdev);
 
 	if (dev_is_sata(dev))
-		return __ata_change_queue_depth(dev->sata_dev.ap, sdev, depth);
+		return ata_change_queue_depth(dev->sata_dev.ap,
+					      sas_to_ata_dev(dev), sdev, depth);
 
 	if (!sdev->tagged_supported)
 		depth = 1;
diff --git a/drivers/soc/amlogic/meson-ee-pwrc.c b/drivers/soc/amlogic/meson-ee-pwrc.c
index 2be3afe6c2e3..dd5f2a13ceb5 100644
--- a/drivers/soc/amlogic/meson-ee-pwrc.c
+++ b/drivers/soc/amlogic/meson-ee-pwrc.c
@@ -469,6 +469,7 @@ static int meson_ee_pwrc_probe(struct platform_device *pdev)
 {
 	const struct meson_ee_pwrc_domain_data *match;
 	struct regmap *regmap_ao, *regmap_hhi;
+	struct device_node *parent_np;
 	struct meson_ee_pwrc *pwrc;
 	int i, ret;
 
@@ -495,7 +496,9 @@ static int meson_ee_pwrc_probe(struct platform_device *pdev)
 
 	pwrc->xlate.num_domains = match->count;
 
-	regmap_hhi = syscon_node_to_regmap(of_get_parent(pdev->dev.of_node));
+	parent_np = of_get_parent(pdev->dev.of_node);
+	regmap_hhi = syscon_node_to_regmap(parent_np);
+	of_node_put(parent_np);
 	if (IS_ERR(regmap_hhi)) {
 		dev_err(&pdev->dev, "failed to get HHI regmap\n");
 		return PTR_ERR(regmap_hhi);
diff --git a/drivers/soc/amlogic/meson-gx-pwrc-vpu.c b/drivers/soc/amlogic/meson-gx-pwrc-vpu.c
index b4615b288625..312fd9afccb0 100644
--- a/drivers/soc/amlogic/meson-gx-pwrc-vpu.c
+++ b/drivers/soc/amlogic/meson-gx-pwrc-vpu.c
@@ -273,6 +273,7 @@ static int meson_gx_pwrc_vpu_probe(struct platform_device *pdev)
 	const struct meson_gx_pwrc_vpu *vpu_pd_match;
 	struct regmap *regmap_ao, *regmap_hhi;
 	struct meson_gx_pwrc_vpu *vpu_pd;
+	struct device_node *parent_np;
 	struct reset_control *rstc;
 	struct clk *vpu_clk;
 	struct clk *vapb_clk;
@@ -291,7 +292,9 @@ static int meson_gx_pwrc_vpu_probe(struct platform_device *pdev)
 
 	memcpy(vpu_pd, vpu_pd_match, sizeof(*vpu_pd));
 
-	regmap_ao = syscon_node_to_regmap(of_get_parent(pdev->dev.of_node));
+	parent_np = of_get_parent(pdev->dev.of_node);
+	regmap_ao = syscon_node_to_regmap(parent_np);
+	of_node_put(parent_np);
 	if (IS_ERR(regmap_ao)) {
 		dev_err(&pdev->dev, "failed to get regmap\n");
 		return PTR_ERR(regmap_ao);
diff --git a/drivers/soc/apple/rtkit.c b/drivers/soc/apple/rtkit.c
index cf1129e9f76b..031ec4aa06d5 100644
--- a/drivers/soc/apple/rtkit.c
+++ b/drivers/soc/apple/rtkit.c
@@ -660,6 +660,12 @@ int apple_rtkit_send_message_wait(struct apple_rtkit *rtk, u8 ep, u64 message,
 }
 EXPORT_SYMBOL_GPL(apple_rtkit_send_message_wait);
 
+int apple_rtkit_poll(struct apple_rtkit *rtk)
+{
+	return mbox_client_peek_data(rtk->mbox_chan);
+}
+EXPORT_SYMBOL_GPL(apple_rtkit_poll);
+
 int apple_rtkit_start_ep(struct apple_rtkit *rtk, u8 endpoint)
 {
 	u64 msg;
diff --git a/drivers/soc/bcm/bcm63xx/Kconfig b/drivers/soc/bcm/bcm63xx/Kconfig
index 9e501c8ac5ce..355c34482076 100644
--- a/drivers/soc/bcm/bcm63xx/Kconfig
+++ b/drivers/soc/bcm/bcm63xx/Kconfig
@@ -13,8 +13,8 @@ endif # SOC_BCM63XX
 
 config BCM_PMB
 	bool "Broadcom PMB (Power Management Bus) driver"
-	depends on ARCH_BCM4908 || (COMPILE_TEST && OF)
-	default ARCH_BCM4908
+	depends on ARCH_BCMBCA || (COMPILE_TEST && OF)
+	default ARCH_BCMBCA
 	select PM_GENERIC_DOMAINS if PM
 	help
 	  This enables support for the Broadcom's PMB (Power Management Bus) that
diff --git a/drivers/soc/bcm/brcmstb/biuctrl.c b/drivers/soc/bcm/brcmstb/biuctrl.c
index 1467bbd59690..e1d7b4543248 100644
--- a/drivers/soc/bcm/brcmstb/biuctrl.c
+++ b/drivers/soc/bcm/brcmstb/biuctrl.c
@@ -288,7 +288,6 @@ static int __init setup_hifcpubiuctrl_regs(struct device_node *np)
 	if (BRCM_ID(family_id) == 0x7260 && BRCM_REV(family_id) == 0)
 		cpubiuctrl_regs = b53_cpubiuctrl_no_wb_regs;
 out:
-	of_node_put(np);
 	return ret;
 }
 
diff --git a/drivers/soc/bcm/brcmstb/pm/pm-arm.c b/drivers/soc/bcm/brcmstb/pm/pm-arm.c
index 775da69b8efa..d681cd24c6e1 100644
--- a/drivers/soc/bcm/brcmstb/pm/pm-arm.c
+++ b/drivers/soc/bcm/brcmstb/pm/pm-arm.c
@@ -25,7 +25,6 @@
 #include <linux/kernel.h>
 #include <linux/memblock.h>
 #include <linux/module.h>
-#include <linux/notifier.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/panic_notifier.h>
@@ -664,7 +663,20 @@ static void __iomem *brcmstb_ioremap_match(const struct of_device_id *matches,
 
 	return of_io_request_and_map(dn, index, dn->full_name);
 }
-
+/*
+ * The AON is a small domain in the SoC that can retain its state across
+ * various system wide sleep states and specific reset conditions; the
+ * AON DATA RAM is a small RAM of a few words (< 1KB) which can store
+ * persistent information across such events.
+ *
+ * The purpose of the below panic notifier is to help with notifying
+ * the bootloader that a panic occurred and so that it should try its
+ * best to preserve the DRAM contents holding that buffer for recovery
+ * by the kernel as opposed to wiping out DRAM clean again.
+ *
+ * Reference: comment from Florian Fainelli, at
+ * https://lore.kernel.org/lkml/781cafb0-8d06-8b56-907a-5175c2da196a@gmail.com
+ */
 static int brcmstb_pm_panic_notify(struct notifier_block *nb,
 		unsigned long action, void *data)
 {
diff --git a/drivers/soc/fsl/qbman/qman.c b/drivers/soc/fsl/qbman/qman.c
index fde4edd83c14..739e4eee6b75 100644
--- a/drivers/soc/fsl/qbman/qman.c
+++ b/drivers/soc/fsl/qbman/qman.c
@@ -2483,13 +2483,8 @@ out:
 }
 EXPORT_SYMBOL(qman_create_cgr);
 
-int qman_delete_cgr(struct qman_cgr *cgr)
+static struct qman_portal *qman_cgr_get_affine_portal(struct qman_cgr *cgr)
 {
-	unsigned long irqflags;
-	struct qm_mcr_querycgr cgr_state;
-	struct qm_mcc_initcgr local_opts;
-	int ret = 0;
-	struct qman_cgr *i;
 	struct qman_portal *p = get_affine_portal();
 
 	if (cgr->chan != p->config->channel) {
@@ -2497,10 +2492,25 @@ int qman_delete_cgr(struct qman_cgr *cgr)
 		dev_err(p->config->dev, "CGR not owned by current portal");
 		dev_dbg(p->config->dev, " create 0x%x, delete 0x%x\n",
 			cgr->chan, p->config->channel);
-
-		ret = -EINVAL;
-		goto put_portal;
+		put_affine_portal();
+		return NULL;
 	}
+
+	return p;
+}
+
+int qman_delete_cgr(struct qman_cgr *cgr)
+{
+	unsigned long irqflags;
+	struct qm_mcr_querycgr cgr_state;
+	struct qm_mcc_initcgr local_opts;
+	int ret = 0;
+	struct qman_cgr *i;
+	struct qman_portal *p = qman_cgr_get_affine_portal(cgr);
+
+	if (!p)
+		return -EINVAL;
+
 	memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
 	spin_lock_irqsave(&p->cgr_lock, irqflags);
 	list_del(&cgr->node);
@@ -2528,7 +2538,6 @@ int qman_delete_cgr(struct qman_cgr *cgr)
 		list_add(&cgr->node, &p->cgr_cbs);
 release_lock:
 	spin_unlock_irqrestore(&p->cgr_lock, irqflags);
-put_portal:
 	put_affine_portal();
 	return ret;
 }
@@ -2559,6 +2568,54 @@ void qman_delete_cgr_safe(struct qman_cgr *cgr)
 }
 EXPORT_SYMBOL(qman_delete_cgr_safe);
 
+static int qman_update_cgr(struct qman_cgr *cgr, struct qm_mcc_initcgr *opts)
+{
+	int ret;
+	unsigned long irqflags;
+	struct qman_portal *p = qman_cgr_get_affine_portal(cgr);
+
+	if (!p)
+		return -EINVAL;
+
+	spin_lock_irqsave(&p->cgr_lock, irqflags);
+	ret = qm_modify_cgr(cgr, 0, opts);
+	spin_unlock_irqrestore(&p->cgr_lock, irqflags);
+	put_affine_portal();
+	return ret;
+}
+
+struct update_cgr_params {
+	struct qman_cgr *cgr;
+	struct qm_mcc_initcgr *opts;
+	int ret;
+};
+
+static void qman_update_cgr_smp_call(void *p)
+{
+	struct update_cgr_params *params = p;
+
+	params->ret = qman_update_cgr(params->cgr, params->opts);
+}
+
+int qman_update_cgr_safe(struct qman_cgr *cgr, struct qm_mcc_initcgr *opts)
+{
+	struct update_cgr_params params = {
+		.cgr = cgr,
+		.opts = opts,
+	};
+
+	preempt_disable();
+	if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id())
+		smp_call_function_single(qman_cgr_cpus[cgr->cgrid],
+					 qman_update_cgr_smp_call, &params,
+					 true);
+	else
+		params.ret = qman_update_cgr(cgr, opts);
+	preempt_enable();
+	return params.ret;
+}
+EXPORT_SYMBOL(qman_update_cgr_safe);
+
 /* Cleanup FQs */
 
 static int _qm_mr_consume_and_match_verb(struct qm_portal *p, int v)
diff --git a/drivers/soc/imx/Kconfig b/drivers/soc/imx/Kconfig
index a840494e849a..4b906791d6c7 100644
--- a/drivers/soc/imx/Kconfig
+++ b/drivers/soc/imx/Kconfig
@@ -20,4 +20,12 @@ config SOC_IMX8M
 	  support, it will provide the SoC info like SoC family,
 	  ID and revision etc.
 
+config SOC_IMX9
+	tristate "i.MX9 SoC family support"
+	depends on ARCH_MXC || COMPILE_TEST
+	default ARCH_MXC && ARM64
+	select SOC_BUS
+	help
+	  If you say yes here, you get support for the NXP i.MX9 family
+
 endmenu
diff --git a/drivers/soc/imx/Makefile b/drivers/soc/imx/Makefile
index 63cd29f6d4d2..7b4099ceafd6 100644
--- a/drivers/soc/imx/Makefile
+++ b/drivers/soc/imx/Makefile
@@ -7,3 +7,5 @@ obj-$(CONFIG_IMX_GPCV2_PM_DOMAINS) += gpcv2.o
 obj-$(CONFIG_SOC_IMX8M) += soc-imx8m.o
 obj-$(CONFIG_SOC_IMX8M) += imx8m-blk-ctrl.o
 obj-$(CONFIG_SOC_IMX8M) += imx8mp-blk-ctrl.o
+obj-$(CONFIG_SOC_IMX9) += imx93-src.o imx93-pd.o
+obj-$(CONFIG_SOC_IMX9) += imx93-blk-ctrl.o
diff --git a/drivers/soc/imx/imx8m-blk-ctrl.c b/drivers/soc/imx/imx8m-blk-ctrl.c
index 972f289d300a..00879615a701 100644
--- a/drivers/soc/imx/imx8m-blk-ctrl.c
+++ b/drivers/soc/imx/imx8m-blk-ctrl.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/device.h>
+#include <linux/interconnect.h>
 #include <linux/module.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
@@ -37,6 +38,8 @@ struct imx8m_blk_ctrl_domain_data {
 	const char *name;
 	const char * const *clk_names;
 	int num_clks;
+	const char * const *path_names;
+	int num_paths;
 	const char *gpc_name;
 	u32 rst_mask;
 	u32 clk_mask;
@@ -52,13 +55,16 @@ struct imx8m_blk_ctrl_domain_data {
 };
 
 #define DOMAIN_MAX_CLKS 4
+#define DOMAIN_MAX_PATHS 4
 
 struct imx8m_blk_ctrl_domain {
 	struct generic_pm_domain genpd;
 	const struct imx8m_blk_ctrl_domain_data *data;
 	struct clk_bulk_data clks[DOMAIN_MAX_CLKS];
+	struct icc_bulk_data paths[DOMAIN_MAX_PATHS];
 	struct device *power_dev;
 	struct imx8m_blk_ctrl *bc;
+	int num_paths;
 };
 
 struct imx8m_blk_ctrl_data {
@@ -117,6 +123,10 @@ static int imx8m_blk_ctrl_power_on(struct generic_pm_domain *genpd)
 	if (data->mipi_phy_rst_mask)
 		regmap_set_bits(bc->regmap, BLK_MIPI_RESET_DIV, data->mipi_phy_rst_mask);
 
+	ret = icc_bulk_set_bw(domain->num_paths, domain->paths);
+	if (ret)
+		dev_err(bc->dev, "failed to set icc bw\n");
+
 	/* disable upstream clocks */
 	clk_bulk_disable_unprepare(data->num_clks, domain->clks);
 
@@ -152,19 +162,6 @@ static int imx8m_blk_ctrl_power_off(struct generic_pm_domain *genpd)
 	return 0;
 }
 
-static struct generic_pm_domain *
-imx8m_blk_ctrl_xlate(struct of_phandle_args *args, void *data)
-{
-	struct genpd_onecell_data *onecell_data = data;
-	unsigned int index = args->args[0];
-
-	if (args->args_count != 1 ||
-	    index >= onecell_data->num_domains)
-		return ERR_PTR(-EINVAL);
-
-	return onecell_data->domains[index];
-}
-
 static struct lock_class_key blk_ctrl_genpd_lock_class;
 
 static int imx8m_blk_ctrl_probe(struct platform_device *pdev)
@@ -206,7 +203,6 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	bc->onecell_data.num_domains = bc_data->num_domains;
-	bc->onecell_data.xlate = imx8m_blk_ctrl_xlate;
 	bc->onecell_data.domains =
 		devm_kcalloc(dev, bc_data->num_domains,
 			     sizeof(struct generic_pm_domain *), GFP_KERNEL);
@@ -224,10 +220,29 @@ static int imx8m_blk_ctrl_probe(struct platform_device *pdev)
 		int j;
 
 		domain->data = data;
+		domain->num_paths = data->num_paths;
 
 		for (j = 0; j < data->num_clks; j++)
 			domain->clks[j].id = data->clk_names[j];
 
+		for (j = 0; j < data->num_paths; j++) {
+			domain->paths[j].name = data->path_names[j];
+			/* Fake value for now, just let ICC could configure NoC mode/priority */
+			domain->paths[j].avg_bw = 1;
+			domain->paths[j].peak_bw = 1;
+		}
+
+		ret = devm_of_icc_bulk_get(dev, data->num_paths, domain->paths);
+		if (ret) {
+			if (ret != -EPROBE_DEFER) {
+				dev_warn_once(dev, "Could not get interconnect paths, NoC will stay unconfigured!\n");
+				domain->num_paths = 0;
+			} else {
+				dev_err_probe(dev, ret, "failed to get noc entries\n");
+				goto cleanup_pds;
+			}
+		}
+
 		ret = devm_clk_bulk_get(dev, data->num_clks, domain->clks);
 		if (ret) {
 			dev_err_probe(dev, ret, "failed to get clock\n");
@@ -454,6 +469,46 @@ static const struct imx8m_blk_ctrl_data imx8mm_vpu_blk_ctl_dev_data = {
 	.num_domains = ARRAY_SIZE(imx8mm_vpu_blk_ctl_domain_data),
 };
 
+static const struct imx8m_blk_ctrl_domain_data imx8mp_vpu_blk_ctl_domain_data[] = {
+	[IMX8MP_VPUBLK_PD_G1] = {
+		.name = "vpublk-g1",
+		.clk_names = (const char *[]){ "g1", },
+		.num_clks = 1,
+		.gpc_name = "g1",
+		.rst_mask = BIT(1),
+		.clk_mask = BIT(1),
+		.path_names = (const char *[]){"g1"},
+		.num_paths = 1,
+	},
+	[IMX8MP_VPUBLK_PD_G2] = {
+		.name = "vpublk-g2",
+		.clk_names = (const char *[]){ "g2", },
+		.num_clks = 1,
+		.gpc_name = "g2",
+		.rst_mask = BIT(0),
+		.clk_mask = BIT(0),
+		.path_names = (const char *[]){"g2"},
+		.num_paths = 1,
+	},
+	[IMX8MP_VPUBLK_PD_VC8000E] = {
+		.name = "vpublk-vc8000e",
+		.clk_names = (const char *[]){ "vc8000e", },
+		.num_clks = 1,
+		.gpc_name = "vc8000e",
+		.rst_mask = BIT(2),
+		.clk_mask = BIT(2),
+		.path_names = (const char *[]){"vc8000e"},
+		.num_paths = 1,
+	},
+};
+
+static const struct imx8m_blk_ctrl_data imx8mp_vpu_blk_ctl_dev_data = {
+	.max_reg = 0x18,
+	.power_notifier_fn = imx8mm_vpu_power_notifier,
+	.domains = imx8mp_vpu_blk_ctl_domain_data,
+	.num_domains = ARRAY_SIZE(imx8mp_vpu_blk_ctl_domain_data),
+};
+
 static int imx8mm_disp_power_notifier(struct notifier_block *nb,
 				      unsigned long action, void *data)
 {
@@ -649,6 +704,8 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_media_blk_ctl_domain_data[
 		.gpc_name = "lcdif1",
 		.rst_mask = BIT(4) | BIT(5) | BIT(23),
 		.clk_mask = BIT(4) | BIT(5) | BIT(23),
+		.path_names = (const char *[]){"lcdif-rd", "lcdif-wr"},
+		.num_paths = 2,
 	},
 	[IMX8MP_MEDIABLK_PD_ISI] = {
 		.name = "mediablk-isi",
@@ -657,6 +714,8 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_media_blk_ctl_domain_data[
 		.gpc_name = "isi",
 		.rst_mask = BIT(6) | BIT(7),
 		.clk_mask = BIT(6) | BIT(7),
+		.path_names = (const char *[]){"isi0", "isi1", "isi2"},
+		.num_paths = 3,
 	},
 	[IMX8MP_MEDIABLK_PD_MIPI_CSI2_2] = {
 		.name = "mediablk-mipi-csi2-2",
@@ -674,6 +733,8 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_media_blk_ctl_domain_data[
 		.gpc_name = "lcdif2",
 		.rst_mask = BIT(11) | BIT(12) | BIT(24),
 		.clk_mask = BIT(11) | BIT(12) | BIT(24),
+		.path_names = (const char *[]){"lcdif-rd", "lcdif-wr"},
+		.num_paths = 2,
 	},
 	[IMX8MP_MEDIABLK_PD_ISP] = {
 		.name = "mediablk-isp",
@@ -682,6 +743,8 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_media_blk_ctl_domain_data[
 		.gpc_name = "isp",
 		.rst_mask = BIT(16) | BIT(17) | BIT(18),
 		.clk_mask = BIT(16) | BIT(17) | BIT(18),
+		.path_names = (const char *[]){"isp0", "isp1"},
+		.num_paths = 2,
 	},
 	[IMX8MP_MEDIABLK_PD_DWE] = {
 		.name = "mediablk-dwe",
@@ -690,6 +753,8 @@ static const struct imx8m_blk_ctrl_domain_data imx8mp_media_blk_ctl_domain_data[
 		.gpc_name = "dwe",
 		.rst_mask = BIT(19) | BIT(20) | BIT(21),
 		.clk_mask = BIT(19) | BIT(20) | BIT(21),
+		.path_names = (const char *[]){"dwe"},
+		.num_paths = 1,
 	},
 	[IMX8MP_MEDIABLK_PD_MIPI_DSI_2] = {
 		.name = "mediablk-mipi-dsi-2",
@@ -788,6 +853,9 @@ static const struct of_device_id imx8m_blk_ctrl_of_match[] = {
 		.compatible = "fsl,imx8mq-vpu-blk-ctrl",
 		.data = &imx8mq_vpu_blk_ctl_dev_data
 	}, {
+		.compatible = "fsl,imx8mp-vpu-blk-ctrl",
+		.data = &imx8mp_vpu_blk_ctl_dev_data
+	}, {
 		/* Sentinel */
 	}
 };
diff --git a/drivers/soc/imx/imx8mp-blk-ctrl.c b/drivers/soc/imx/imx8mp-blk-ctrl.c
index 4ca2ede6871b..0e3b6ba22f94 100644
--- a/drivers/soc/imx/imx8mp-blk-ctrl.c
+++ b/drivers/soc/imx/imx8mp-blk-ctrl.c
@@ -6,6 +6,7 @@
 
 #include <linux/clk.h>
 #include <linux/device.h>
+#include <linux/interconnect.h>
 #include <linux/module.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
@@ -18,6 +19,8 @@
 #define GPR_REG0		0x0
 #define  PCIE_CLOCK_MODULE_EN	BIT(0)
 #define  USB_CLOCK_MODULE_EN	BIT(1)
+#define  PCIE_PHY_APB_RST	BIT(4)
+#define  PCIE_PHY_INIT_RST	BIT(5)
 
 struct imx8mp_blk_ctrl_domain;
 
@@ -36,17 +39,22 @@ struct imx8mp_blk_ctrl_domain_data {
 	const char *name;
 	const char * const *clk_names;
 	int num_clks;
+	const char * const *path_names;
+	int num_paths;
 	const char *gpc_name;
 };
 
 #define DOMAIN_MAX_CLKS 2
+#define DOMAIN_MAX_PATHS 3
 
 struct imx8mp_blk_ctrl_domain {
 	struct generic_pm_domain genpd;
 	const struct imx8mp_blk_ctrl_domain_data *data;
 	struct clk_bulk_data clks[DOMAIN_MAX_CLKS];
+	struct icc_bulk_data paths[DOMAIN_MAX_PATHS];
 	struct device *power_dev;
 	struct imx8mp_blk_ctrl *bc;
+	int num_paths;
 	int id;
 };
 
@@ -75,6 +83,10 @@ static void imx8mp_hsio_blk_ctrl_power_on(struct imx8mp_blk_ctrl *bc,
 	case IMX8MP_HSIOBLK_PD_PCIE:
 		regmap_set_bits(bc->regmap, GPR_REG0, PCIE_CLOCK_MODULE_EN);
 		break;
+	case IMX8MP_HSIOBLK_PD_PCIE_PHY:
+		regmap_set_bits(bc->regmap, GPR_REG0,
+				PCIE_PHY_APB_RST | PCIE_PHY_INIT_RST);
+		break;
 	default:
 		break;
 	}
@@ -90,6 +102,10 @@ static void imx8mp_hsio_blk_ctrl_power_off(struct imx8mp_blk_ctrl *bc,
 	case IMX8MP_HSIOBLK_PD_PCIE:
 		regmap_clear_bits(bc->regmap, GPR_REG0, PCIE_CLOCK_MODULE_EN);
 		break;
+	case IMX8MP_HSIOBLK_PD_PCIE_PHY:
+		regmap_clear_bits(bc->regmap, GPR_REG0,
+				  PCIE_PHY_APB_RST | PCIE_PHY_INIT_RST);
+		break;
 	default:
 		break;
 	}
@@ -144,6 +160,8 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hsio_domain_data[] = {
 		.clk_names = (const char *[]){ "usb" },
 		.num_clks = 1,
 		.gpc_name = "usb",
+		.path_names = (const char *[]){"usb1", "usb2"},
+		.num_paths = 2,
 	},
 	[IMX8MP_HSIOBLK_PD_USB_PHY1] = {
 		.name = "hsioblk-usb-phy1",
@@ -158,6 +176,8 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hsio_domain_data[] = {
 		.clk_names = (const char *[]){ "pcie" },
 		.num_clks = 1,
 		.gpc_name = "pcie",
+		.path_names = (const char *[]){"noc-pcie", "pcie"},
+		.num_paths = 2,
 	},
 	[IMX8MP_HSIOBLK_PD_PCIE_PHY] = {
 		.name = "hsioblk-pcie-phy",
@@ -225,6 +245,13 @@ static void imx8mp_hdmi_blk_ctrl_power_on(struct imx8mp_blk_ctrl *bc,
 		regmap_set_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(12));
 		regmap_clear_bits(bc->regmap, HDMI_TX_CONTROL0, BIT(3));
 		break;
+	case IMX8MP_HDMIBLK_PD_HDCP:
+		regmap_set_bits(bc->regmap, HDMI_RTX_CLK_CTL0, BIT(11));
+		break;
+	case IMX8MP_HDMIBLK_PD_HRV:
+		regmap_set_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(3) | BIT(4) | BIT(5));
+		regmap_set_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(15));
+		break;
 	default:
 		break;
 	}
@@ -273,6 +300,13 @@ static void imx8mp_hdmi_blk_ctrl_power_off(struct imx8mp_blk_ctrl *bc,
 		regmap_clear_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(12));
 		regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(22) | BIT(24));
 		break;
+	case IMX8MP_HDMIBLK_PD_HDCP:
+		regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL0, BIT(11));
+		break;
+	case IMX8MP_HDMIBLK_PD_HRV:
+		regmap_clear_bits(bc->regmap, HDMI_RTX_RESET_CTL0, BIT(15));
+		regmap_clear_bits(bc->regmap, HDMI_RTX_CLK_CTL1, BIT(3) | BIT(4) | BIT(5));
+		break;
 	default:
 		break;
 	}
@@ -322,6 +356,8 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hdmi_domain_data[] = {
 		.clk_names = (const char *[]){ "axi", "apb" },
 		.num_clks = 2,
 		.gpc_name = "lcdif",
+		.path_names = (const char *[]){"lcdif-hdmi"},
+		.num_paths = 1,
 	},
 	[IMX8MP_HDMIBLK_PD_PAI] = {
 		.name = "hdmiblk-pai",
@@ -353,6 +389,22 @@ static const struct imx8mp_blk_ctrl_domain_data imx8mp_hdmi_domain_data[] = {
 		.num_clks = 2,
 		.gpc_name = "hdmi-tx-phy",
 	},
+	[IMX8MP_HDMIBLK_PD_HRV] = {
+		.name = "hdmiblk-hrv",
+		.clk_names = (const char *[]){ "axi", "apb" },
+		.num_clks = 2,
+		.gpc_name = "hrv",
+		.path_names = (const char *[]){"hrv"},
+		.num_paths = 1,
+	},
+	[IMX8MP_HDMIBLK_PD_HDCP] = {
+		.name = "hdmiblk-hdcp",
+		.clk_names = (const char *[]){ "axi", "apb" },
+		.num_clks = 2,
+		.gpc_name = "hdcp",
+		.path_names = (const char *[]){"hdcp"},
+		.num_paths = 1,
+	},
 };
 
 static const struct imx8mp_blk_ctrl_data imx8mp_hdmi_blk_ctl_dev_data = {
@@ -395,6 +447,10 @@ static int imx8mp_blk_ctrl_power_on(struct generic_pm_domain *genpd)
 		goto clk_disable;
 	}
 
+	ret = icc_bulk_set_bw(domain->num_paths, domain->paths);
+	if (ret)
+		dev_err(bc->dev, "failed to set icc bw\n");
+
 	clk_bulk_disable_unprepare(data->num_clks, domain->clks);
 
 	return 0;
@@ -434,19 +490,6 @@ static int imx8mp_blk_ctrl_power_off(struct generic_pm_domain *genpd)
 	return 0;
 }
 
-static struct generic_pm_domain *
-imx8m_blk_ctrl_xlate(struct of_phandle_args *args, void *data)
-{
-	struct genpd_onecell_data *onecell_data = data;
-	unsigned int index = args->args[0];
-
-	if (args->args_count != 1 ||
-	    index >= onecell_data->num_domains)
-		return ERR_PTR(-EINVAL);
-
-	return onecell_data->domains[index];
-}
-
 static struct lock_class_key blk_ctrl_genpd_lock_class;
 
 static int imx8mp_blk_ctrl_probe(struct platform_device *pdev)
@@ -489,7 +532,6 @@ static int imx8mp_blk_ctrl_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	bc->onecell_data.num_domains = num_domains;
-	bc->onecell_data.xlate = imx8m_blk_ctrl_xlate;
 	bc->onecell_data.domains =
 		devm_kcalloc(dev, num_domains,
 			     sizeof(struct generic_pm_domain *), GFP_KERNEL);
@@ -510,10 +552,29 @@ static int imx8mp_blk_ctrl_probe(struct platform_device *pdev)
 		int j;
 
 		domain->data = data;
+		domain->num_paths = data->num_paths;
 
 		for (j = 0; j < data->num_clks; j++)
 			domain->clks[j].id = data->clk_names[j];
 
+		for (j = 0; j < data->num_paths; j++) {
+			domain->paths[j].name = data->path_names[j];
+			/* Fake value for now, just let ICC could configure NoC mode/priority */
+			domain->paths[j].avg_bw = 1;
+			domain->paths[j].peak_bw = 1;
+		}
+
+		ret = devm_of_icc_bulk_get(dev, data->num_paths, domain->paths);
+		if (ret) {
+			if (ret != -EPROBE_DEFER) {
+				dev_warn_once(dev, "Could not get interconnect paths, NoC will stay unconfigured!\n");
+				domain->num_paths = 0;
+			} else {
+				dev_err_probe(dev, ret, "failed to get noc entries\n");
+				goto cleanup_pds;
+			}
+		}
+
 		ret = devm_clk_bulk_get(dev, data->num_clks, domain->clks);
 		if (ret) {
 			dev_err_probe(dev, ret, "failed to get clock\n");
diff --git a/drivers/soc/imx/imx93-blk-ctrl.c b/drivers/soc/imx/imx93-blk-ctrl.c
new file mode 100644
index 000000000000..2c600329436c
--- /dev/null
+++ b/drivers/soc/imx/imx93-blk-ctrl.c
@@ -0,0 +1,436 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2022 NXP, Peng Fan <peng.fan@nxp.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/sizes.h>
+
+#include <dt-bindings/power/fsl,imx93-power.h>
+
+#define BLK_SFT_RSTN	0x0
+#define BLK_CLK_EN	0x4
+#define BLK_MAX_CLKS	4
+
+#define DOMAIN_MAX_CLKS 4
+
+#define LCDIF_QOS_REG		0xC
+#define LCDIF_DEFAULT_QOS_OFF	12
+#define LCDIF_CFG_QOS_OFF	8
+
+#define PXP_QOS_REG		0x10
+#define PXP_R_DEFAULT_QOS_OFF	28
+#define PXP_R_CFG_QOS_OFF	24
+#define PXP_W_DEFAULT_QOS_OFF	20
+#define PXP_W_CFG_QOS_OFF	16
+
+#define ISI_CACHE_REG		0x14
+
+#define ISI_QOS_REG		0x1C
+#define ISI_V_DEFAULT_QOS_OFF	28
+#define ISI_V_CFG_QOS_OFF	24
+#define ISI_U_DEFAULT_QOS_OFF	20
+#define ISI_U_CFG_QOS_OFF	16
+#define ISI_Y_R_DEFAULT_QOS_OFF	12
+#define ISI_Y_R_CFG_QOS_OFF	8
+#define ISI_Y_W_DEFAULT_QOS_OFF	4
+#define ISI_Y_W_CFG_QOS_OFF	0
+
+#define PRIO_MASK		0xF
+
+#define PRIO(X)			(X)
+
+struct imx93_blk_ctrl_domain;
+
+struct imx93_blk_ctrl {
+	struct device *dev;
+	struct regmap *regmap;
+	int num_clks;
+	struct clk_bulk_data clks[BLK_MAX_CLKS];
+	struct imx93_blk_ctrl_domain *domains;
+	struct genpd_onecell_data onecell_data;
+};
+
+#define DOMAIN_MAX_QOS 4
+
+struct imx93_blk_ctrl_qos {
+	u32 reg;
+	u32 cfg_off;
+	u32 default_prio;
+	u32 cfg_prio;
+};
+
+struct imx93_blk_ctrl_domain_data {
+	const char *name;
+	const char * const *clk_names;
+	int num_clks;
+	u32 rst_mask;
+	u32 clk_mask;
+	int num_qos;
+	struct imx93_blk_ctrl_qos qos[DOMAIN_MAX_QOS];
+};
+
+struct imx93_blk_ctrl_domain {
+	struct generic_pm_domain genpd;
+	const struct imx93_blk_ctrl_domain_data *data;
+	struct clk_bulk_data clks[DOMAIN_MAX_CLKS];
+	struct imx93_blk_ctrl *bc;
+};
+
+struct imx93_blk_ctrl_data {
+	const struct imx93_blk_ctrl_domain_data *domains;
+	int num_domains;
+	const char * const *clk_names;
+	int num_clks;
+	const struct regmap_access_table *reg_access_table;
+};
+
+static inline struct imx93_blk_ctrl_domain *
+to_imx93_blk_ctrl_domain(struct generic_pm_domain *genpd)
+{
+	return container_of(genpd, struct imx93_blk_ctrl_domain, genpd);
+}
+
+static int imx93_blk_ctrl_set_qos(struct imx93_blk_ctrl_domain *domain)
+{
+	const struct imx93_blk_ctrl_domain_data *data = domain->data;
+	struct imx93_blk_ctrl *bc = domain->bc;
+	const struct imx93_blk_ctrl_qos *qos;
+	u32 val, mask;
+	int i;
+
+	for (i = 0; i < data->num_qos; i++) {
+		qos = &data->qos[i];
+
+		mask = PRIO_MASK << qos->cfg_off;
+		mask |= PRIO_MASK << (qos->cfg_off + 4);
+		val = qos->cfg_prio << qos->cfg_off;
+		val |= qos->default_prio << (qos->cfg_off + 4);
+
+		regmap_write_bits(bc->regmap, qos->reg, mask, val);
+
+		dev_dbg(bc->dev, "data->qos[i].reg 0x%x 0x%x\n", qos->reg, val);
+	}
+
+	return 0;
+}
+
+static int imx93_blk_ctrl_power_on(struct generic_pm_domain *genpd)
+{
+	struct imx93_blk_ctrl_domain *domain = to_imx93_blk_ctrl_domain(genpd);
+	const struct imx93_blk_ctrl_domain_data *data = domain->data;
+	struct imx93_blk_ctrl *bc = domain->bc;
+	int ret;
+
+	ret = clk_bulk_prepare_enable(bc->num_clks, bc->clks);
+	if (ret) {
+		dev_err(bc->dev, "failed to enable bus clocks\n");
+		return ret;
+	}
+
+	ret = clk_bulk_prepare_enable(data->num_clks, domain->clks);
+	if (ret) {
+		clk_bulk_disable_unprepare(bc->num_clks, bc->clks);
+		dev_err(bc->dev, "failed to enable clocks\n");
+		return ret;
+	}
+
+	ret = pm_runtime_get_sync(bc->dev);
+	if (ret < 0) {
+		pm_runtime_put_noidle(bc->dev);
+		dev_err(bc->dev, "failed to power up domain\n");
+		goto disable_clk;
+	}
+
+	/* ungate clk */
+	regmap_clear_bits(bc->regmap, BLK_CLK_EN, data->clk_mask);
+
+	/* release reset */
+	regmap_set_bits(bc->regmap, BLK_SFT_RSTN, data->rst_mask);
+
+	dev_dbg(bc->dev, "pd_on: name: %s\n", genpd->name);
+
+	return imx93_blk_ctrl_set_qos(domain);
+
+disable_clk:
+	clk_bulk_disable_unprepare(data->num_clks, domain->clks);
+
+	clk_bulk_disable_unprepare(bc->num_clks, bc->clks);
+
+	return ret;
+}
+
+static int imx93_blk_ctrl_power_off(struct generic_pm_domain *genpd)
+{
+	struct imx93_blk_ctrl_domain *domain = to_imx93_blk_ctrl_domain(genpd);
+	const struct imx93_blk_ctrl_domain_data *data = domain->data;
+	struct imx93_blk_ctrl *bc = domain->bc;
+
+	dev_dbg(bc->dev, "pd_off: name: %s\n", genpd->name);
+
+	regmap_clear_bits(bc->regmap, BLK_SFT_RSTN, data->rst_mask);
+	regmap_set_bits(bc->regmap, BLK_CLK_EN, data->clk_mask);
+
+	pm_runtime_put(bc->dev);
+
+	clk_bulk_disable_unprepare(data->num_clks, domain->clks);
+
+	clk_bulk_disable_unprepare(bc->num_clks, bc->clks);
+
+	return 0;
+}
+
+static int imx93_blk_ctrl_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	const struct imx93_blk_ctrl_data *bc_data = of_device_get_match_data(dev);
+	struct imx93_blk_ctrl *bc;
+	void __iomem *base;
+	int i, ret;
+
+	struct regmap_config regmap_config = {
+		.reg_bits	= 32,
+		.val_bits	= 32,
+		.reg_stride	= 4,
+		.rd_table	= bc_data->reg_access_table,
+		.wr_table	= bc_data->reg_access_table,
+		.max_register   = SZ_4K,
+	};
+
+	bc = devm_kzalloc(dev, sizeof(*bc), GFP_KERNEL);
+	if (!bc)
+		return -ENOMEM;
+
+	bc->dev = dev;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	bc->regmap = devm_regmap_init_mmio(dev, base, &regmap_config);
+	if (IS_ERR(bc->regmap))
+		return dev_err_probe(dev, PTR_ERR(bc->regmap),
+				     "failed to init regmap\n");
+
+	bc->domains = devm_kcalloc(dev, bc_data->num_domains,
+				   sizeof(struct imx93_blk_ctrl_domain),
+				   GFP_KERNEL);
+	if (!bc->domains)
+		return -ENOMEM;
+
+	bc->onecell_data.num_domains = bc_data->num_domains;
+	bc->onecell_data.domains =
+		devm_kcalloc(dev, bc_data->num_domains,
+			     sizeof(struct generic_pm_domain *), GFP_KERNEL);
+	if (!bc->onecell_data.domains)
+		return -ENOMEM;
+
+	for (i = 0; i < bc_data->num_clks; i++)
+		bc->clks[i].id = bc_data->clk_names[i];
+	bc->num_clks = bc_data->num_clks;
+
+	ret = devm_clk_bulk_get(dev, bc->num_clks, bc->clks);
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to get bus clock\n");
+		return ret;
+	}
+
+	for (i = 0; i < bc_data->num_domains; i++) {
+		const struct imx93_blk_ctrl_domain_data *data = &bc_data->domains[i];
+		struct imx93_blk_ctrl_domain *domain = &bc->domains[i];
+		int j;
+
+		domain->data = data;
+
+		for (j = 0; j < data->num_clks; j++)
+			domain->clks[j].id = data->clk_names[j];
+
+		ret = devm_clk_bulk_get(dev, data->num_clks, domain->clks);
+		if (ret) {
+			dev_err_probe(dev, ret, "failed to get clock\n");
+			goto cleanup_pds;
+		}
+
+		domain->genpd.name = data->name;
+		domain->genpd.power_on = imx93_blk_ctrl_power_on;
+		domain->genpd.power_off = imx93_blk_ctrl_power_off;
+		domain->bc = bc;
+
+		ret = pm_genpd_init(&domain->genpd, NULL, true);
+		if (ret) {
+			dev_err_probe(dev, ret, "failed to init power domain\n");
+			goto cleanup_pds;
+		}
+
+		bc->onecell_data.domains[i] = &domain->genpd;
+	}
+
+	pm_runtime_enable(dev);
+
+	ret = of_genpd_add_provider_onecell(dev->of_node, &bc->onecell_data);
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to add power domain provider\n");
+		goto cleanup_pds;
+	}
+
+	dev_set_drvdata(dev, bc);
+
+	return 0;
+
+cleanup_pds:
+	for (i--; i >= 0; i--)
+		pm_genpd_remove(&bc->domains[i].genpd);
+
+	return ret;
+}
+
+static int imx93_blk_ctrl_remove(struct platform_device *pdev)
+{
+	struct imx93_blk_ctrl *bc = dev_get_drvdata(&pdev->dev);
+	int i;
+
+	of_genpd_del_provider(pdev->dev.of_node);
+
+	for (i = 0; bc->onecell_data.num_domains; i++) {
+		struct imx93_blk_ctrl_domain *domain = &bc->domains[i];
+
+		pm_genpd_remove(&domain->genpd);
+	}
+
+	return 0;
+}
+
+static const struct imx93_blk_ctrl_domain_data imx93_media_blk_ctl_domain_data[] = {
+	[IMX93_MEDIABLK_PD_MIPI_DSI] = {
+		.name = "mediablk-mipi-dsi",
+		.clk_names = (const char *[]){ "dsi" },
+		.num_clks = 1,
+		.rst_mask = BIT(11) | BIT(12),
+		.clk_mask = BIT(11) | BIT(12),
+	},
+	[IMX93_MEDIABLK_PD_MIPI_CSI] = {
+		.name = "mediablk-mipi-csi",
+		.clk_names = (const char *[]){ "cam", "csi" },
+		.num_clks = 2,
+		.rst_mask = BIT(9) | BIT(10),
+		.clk_mask = BIT(9) | BIT(10),
+	},
+	[IMX93_MEDIABLK_PD_PXP] = {
+		.name = "mediablk-pxp",
+		.clk_names = (const char *[]){ "pxp" },
+		.num_clks = 1,
+		.rst_mask = BIT(7) | BIT(8),
+		.clk_mask = BIT(7) | BIT(8),
+		.num_qos = 2,
+		.qos = {
+			{
+				.reg = PXP_QOS_REG,
+				.cfg_off = PXP_R_CFG_QOS_OFF,
+				.default_prio = PRIO(3),
+				.cfg_prio = PRIO(6),
+			}, {
+				.reg = PXP_QOS_REG,
+				.cfg_off = PXP_W_CFG_QOS_OFF,
+				.default_prio = PRIO(3),
+				.cfg_prio = PRIO(6),
+			}
+		}
+	},
+	[IMX93_MEDIABLK_PD_LCDIF] = {
+		.name = "mediablk-lcdif",
+		.clk_names = (const char *[]){ "disp", "lcdif" },
+		.num_clks = 2,
+		.rst_mask = BIT(4) | BIT(5) | BIT(6),
+		.clk_mask = BIT(4) | BIT(5) | BIT(6),
+		.num_qos = 1,
+		.qos = {
+			{
+			.reg = LCDIF_QOS_REG,
+			.cfg_off = LCDIF_CFG_QOS_OFF,
+			.default_prio = PRIO(3),
+			.cfg_prio = PRIO(7),
+			}
+		}
+	},
+	[IMX93_MEDIABLK_PD_ISI] = {
+		.name = "mediablk-isi",
+		.clk_names = (const char *[]){ "isi" },
+		.num_clks = 1,
+		.rst_mask = BIT(2) | BIT(3),
+		.clk_mask = BIT(2) | BIT(3),
+		.num_qos = 4,
+		.qos = {
+			{
+				.reg = ISI_QOS_REG,
+				.cfg_off = ISI_Y_W_CFG_QOS_OFF,
+				.default_prio = PRIO(3),
+				.cfg_prio = PRIO(7),
+			}, {
+				.reg = ISI_QOS_REG,
+				.cfg_off = ISI_Y_R_CFG_QOS_OFF,
+				.default_prio = PRIO(3),
+				.cfg_prio = PRIO(7),
+			}, {
+				.reg = ISI_QOS_REG,
+				.cfg_off = ISI_U_CFG_QOS_OFF,
+				.default_prio = PRIO(3),
+				.cfg_prio = PRIO(7),
+			}, {
+				.reg = ISI_QOS_REG,
+				.cfg_off = ISI_V_CFG_QOS_OFF,
+				.default_prio = PRIO(3),
+				.cfg_prio = PRIO(7),
+			}
+		}
+	},
+};
+
+static const struct regmap_range imx93_media_blk_ctl_yes_ranges[] = {
+	regmap_reg_range(BLK_SFT_RSTN, BLK_CLK_EN),
+	regmap_reg_range(LCDIF_QOS_REG, ISI_CACHE_REG),
+	regmap_reg_range(ISI_QOS_REG, ISI_QOS_REG),
+};
+
+static const struct regmap_access_table imx93_media_blk_ctl_access_table = {
+	.yes_ranges = imx93_media_blk_ctl_yes_ranges,
+	.n_yes_ranges = ARRAY_SIZE(imx93_media_blk_ctl_yes_ranges),
+};
+
+static const struct imx93_blk_ctrl_data imx93_media_blk_ctl_dev_data = {
+	.domains = imx93_media_blk_ctl_domain_data,
+	.num_domains = ARRAY_SIZE(imx93_media_blk_ctl_domain_data),
+	.clk_names = (const char *[]){ "axi", "apb", "nic", },
+	.num_clks = 3,
+	.reg_access_table = &imx93_media_blk_ctl_access_table,
+};
+
+static const struct of_device_id imx93_blk_ctrl_of_match[] = {
+	{
+		.compatible = "fsl,imx93-media-blk-ctrl",
+		.data = &imx93_media_blk_ctl_dev_data
+	}, {
+		/* Sentinel */
+	}
+};
+MODULE_DEVICE_TABLE(of, imx93_blk_ctrl_of_match);
+
+static struct platform_driver imx93_blk_ctrl_driver = {
+	.probe = imx93_blk_ctrl_probe,
+	.remove = imx93_blk_ctrl_remove,
+	.driver = {
+		.name = "imx93-blk-ctrl",
+		.of_match_table = imx93_blk_ctrl_of_match,
+	},
+};
+module_platform_driver(imx93_blk_ctrl_driver);
+
+MODULE_AUTHOR("Peng Fan <peng.fan@nxp.com>");
+MODULE_DESCRIPTION("i.MX93 BLK CTRL driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/imx/imx93-pd.c b/drivers/soc/imx/imx93-pd.c
new file mode 100644
index 000000000000..1f3d7039c1de
--- /dev/null
+++ b/drivers/soc/imx/imx93-pd.c
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2022 NXP
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/of_device.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+
+#define MIX_SLICE_SW_CTRL_OFF		0x20
+#define SLICE_SW_CTRL_PSW_CTRL_OFF_MASK	BIT(4)
+#define SLICE_SW_CTRL_PDN_SOFT_MASK	BIT(31)
+
+#define MIX_FUNC_STAT_OFF		0xB4
+
+#define FUNC_STAT_PSW_STAT_MASK		BIT(0)
+#define FUNC_STAT_RST_STAT_MASK		BIT(2)
+#define FUNC_STAT_ISO_STAT_MASK		BIT(4)
+
+struct imx93_power_domain {
+	struct generic_pm_domain genpd;
+	struct device *dev;
+	void __iomem *addr;
+	struct clk_bulk_data *clks;
+	int num_clks;
+	bool init_off;
+};
+
+#define to_imx93_pd(_genpd) container_of(_genpd, struct imx93_power_domain, genpd)
+
+static int imx93_pd_on(struct generic_pm_domain *genpd)
+{
+	struct imx93_power_domain *domain = to_imx93_pd(genpd);
+	void __iomem *addr = domain->addr;
+	u32 val;
+	int ret;
+
+	ret = clk_bulk_prepare_enable(domain->num_clks, domain->clks);
+	if (ret) {
+		dev_err(domain->dev, "failed to enable clocks for domain: %s\n", genpd->name);
+		return ret;
+	}
+
+	val = readl(addr + MIX_SLICE_SW_CTRL_OFF);
+	val &= ~SLICE_SW_CTRL_PDN_SOFT_MASK;
+	writel(val, addr + MIX_SLICE_SW_CTRL_OFF);
+
+	ret = readl_poll_timeout(addr + MIX_FUNC_STAT_OFF, val,
+				 !(val & FUNC_STAT_ISO_STAT_MASK), 1, 10000);
+	if (ret) {
+		dev_err(domain->dev, "pd_on timeout: name: %s, stat: %x\n", genpd->name, val);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int imx93_pd_off(struct generic_pm_domain *genpd)
+{
+	struct imx93_power_domain *domain = to_imx93_pd(genpd);
+	void __iomem *addr = domain->addr;
+	int ret;
+	u32 val;
+
+	/* Power off MIX */
+	val = readl(addr + MIX_SLICE_SW_CTRL_OFF);
+	val |= SLICE_SW_CTRL_PDN_SOFT_MASK;
+	writel(val, addr + MIX_SLICE_SW_CTRL_OFF);
+
+	ret = readl_poll_timeout(addr + MIX_FUNC_STAT_OFF, val,
+				 val & FUNC_STAT_PSW_STAT_MASK, 1, 1000);
+	if (ret) {
+		dev_err(domain->dev, "pd_off timeout: name: %s, stat: %x\n", genpd->name, val);
+		return ret;
+	}
+
+	clk_bulk_disable_unprepare(domain->num_clks, domain->clks);
+
+	return 0;
+};
+
+static int imx93_pd_remove(struct platform_device *pdev)
+{
+	struct imx93_power_domain *domain = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+
+	if (!domain->init_off)
+		clk_bulk_disable_unprepare(domain->num_clks, domain->clks);
+
+	of_genpd_del_provider(np);
+	pm_genpd_remove(&domain->genpd);
+
+	return 0;
+}
+
+static int imx93_pd_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	struct imx93_power_domain *domain;
+	int ret;
+
+	domain = devm_kzalloc(dev, sizeof(*domain), GFP_KERNEL);
+	if (!domain)
+		return -ENOMEM;
+
+	domain->addr = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(domain->addr))
+		return PTR_ERR(domain->addr);
+
+	domain->num_clks = devm_clk_bulk_get_all(dev, &domain->clks);
+	if (domain->num_clks < 0)
+		return dev_err_probe(dev, domain->num_clks, "Failed to get domain's clocks\n");
+
+	domain->genpd.name = dev_name(dev);
+	domain->genpd.power_off = imx93_pd_off;
+	domain->genpd.power_on = imx93_pd_on;
+	domain->dev = dev;
+
+	domain->init_off = readl(domain->addr + MIX_FUNC_STAT_OFF) & FUNC_STAT_ISO_STAT_MASK;
+	/* Just to sync the status of hardware */
+	if (!domain->init_off) {
+		ret = clk_bulk_prepare_enable(domain->num_clks, domain->clks);
+		if (ret) {
+			dev_err(domain->dev, "failed to enable clocks for domain: %s\n",
+				domain->genpd.name);
+			return ret;
+		}
+	}
+
+	ret = pm_genpd_init(&domain->genpd, NULL, domain->init_off);
+	if (ret)
+		return ret;
+
+	platform_set_drvdata(pdev, domain);
+
+	return of_genpd_add_provider_simple(np, &domain->genpd);
+}
+
+static const struct of_device_id imx93_pd_ids[] = {
+	{ .compatible = "fsl,imx93-src-slice" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, imx93_pd_ids);
+
+static struct platform_driver imx93_power_domain_driver = {
+	.driver = {
+		.name	= "imx93_power_domain",
+		.owner	= THIS_MODULE,
+		.of_match_table = imx93_pd_ids,
+	},
+	.probe = imx93_pd_probe,
+	.remove = imx93_pd_remove,
+};
+module_platform_driver(imx93_power_domain_driver);
+
+MODULE_AUTHOR("Peng Fan <peng.fan@nxp.com>");
+MODULE_DESCRIPTION("NXP i.MX93 power domain driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/imx/imx93-src.c b/drivers/soc/imx/imx93-src.c
new file mode 100644
index 000000000000..4d74921cae0f
--- /dev/null
+++ b/drivers/soc/imx/imx93-src.c
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright 2022 NXP
+ */
+
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+
+static int imx93_src_probe(struct platform_device *pdev)
+{
+	return devm_of_platform_populate(&pdev->dev);
+}
+
+static const struct of_device_id imx93_src_ids[] = {
+	{ .compatible = "fsl,imx93-src" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, imx93_src_ids);
+
+static struct platform_driver imx93_src_driver = {
+	.driver = {
+		.name	= "imx93_src",
+		.owner	= THIS_MODULE,
+		.of_match_table = imx93_src_ids,
+	},
+	.probe = imx93_src_probe,
+};
+module_platform_driver(imx93_src_driver);
+
+MODULE_AUTHOR("Peng Fan <peng.fan@nxp.com>");
+MODULE_DESCRIPTION("NXP i.MX93 src driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/mediatek/Kconfig b/drivers/soc/mediatek/Kconfig
index 3c3eedea35f7..73e63920b1b9 100644
--- a/drivers/soc/mediatek/Kconfig
+++ b/drivers/soc/mediatek/Kconfig
@@ -37,6 +37,7 @@ config MTK_INFRACFG
 config MTK_PMIC_WRAP
 	tristate "MediaTek PMIC Wrapper Support"
 	depends on RESET_CONTROLLER
+	depends on OF
 	select REGMAP
 	help
 	  Say yes here to add support for MediaTek PMIC Wrapper found
@@ -46,6 +47,7 @@ config MTK_PMIC_WRAP
 config MTK_SCPSYS
 	bool "MediaTek SCPSYS Support"
 	default ARCH_MEDIATEK
+	depends on OF
 	select REGMAP
 	select MTK_INFRACFG
 	select PM_GENERIC_DOMAINS if PM
diff --git a/drivers/soc/mediatek/mt8186-mmsys.h b/drivers/soc/mediatek/mt8186-mmsys.h
index eb1ad9c37a9c..09b1ccbc0093 100644
--- a/drivers/soc/mediatek/mt8186-mmsys.h
+++ b/drivers/soc/mediatek/mt8186-mmsys.h
@@ -3,6 +3,12 @@
 #ifndef __SOC_MEDIATEK_MT8186_MMSYS_H
 #define __SOC_MEDIATEK_MT8186_MMSYS_H
 
+/* Values for DPI configuration in MMSYS address space */
+#define MT8186_MMSYS_DPI_OUTPUT_FORMAT		0x400
+#define DPI_FORMAT_MASK					0x1
+#define DPI_RGB888_DDR_CON				BIT(0)
+#define DPI_RGB565_SDR_CON				BIT(1)
+
 #define MT8186_MMSYS_OVL_CON			0xF04
 #define MT8186_MMSYS_OVL0_CON_MASK			0x3
 #define MT8186_MMSYS_OVL0_2L_CON_MASK			0xC
diff --git a/drivers/soc/mediatek/mtk-mmsys.c b/drivers/soc/mediatek/mtk-mmsys.c
index 06d8e83a2cb5..d2c7a87aab87 100644
--- a/drivers/soc/mediatek/mtk-mmsys.c
+++ b/drivers/soc/mediatek/mtk-mmsys.c
@@ -227,6 +227,26 @@ void mtk_mmsys_ddp_disconnect(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(mtk_mmsys_ddp_disconnect);
 
+static void mtk_mmsys_update_bits(struct mtk_mmsys *mmsys, u32 offset, u32 mask, u32 val)
+{
+	u32 tmp;
+
+	tmp = readl_relaxed(mmsys->regs + offset);
+	tmp = (tmp & ~mask) | val;
+	writel_relaxed(tmp, mmsys->regs + offset);
+}
+
+void mtk_mmsys_ddp_dpi_fmt_config(struct device *dev, u32 val)
+{
+	if (val)
+		mtk_mmsys_update_bits(dev_get_drvdata(dev), MT8186_MMSYS_DPI_OUTPUT_FORMAT,
+				      DPI_RGB888_DDR_CON, DPI_FORMAT_MASK);
+	else
+		mtk_mmsys_update_bits(dev_get_drvdata(dev), MT8186_MMSYS_DPI_OUTPUT_FORMAT,
+				      DPI_RGB565_SDR_CON, DPI_FORMAT_MASK);
+}
+EXPORT_SYMBOL_GPL(mtk_mmsys_ddp_dpi_fmt_config);
+
 static int mtk_mmsys_reset_update(struct reset_controller_dev *rcdev, unsigned long id,
 				  bool assert)
 {
diff --git a/drivers/soc/mediatek/mtk-mutex.c b/drivers/soc/mediatek/mtk-mutex.c
index 5ea43de4e410..c1a33d52038e 100644
--- a/drivers/soc/mediatek/mtk-mutex.c
+++ b/drivers/soc/mediatek/mtk-mutex.c
@@ -91,6 +91,15 @@
 #define MT8183_MUTEX_MOD_MDP_AAL0		23
 #define MT8183_MUTEX_MOD_MDP_CCORR0		24
 
+#define MT8186_MUTEX_MOD_MDP_RDMA0		0
+#define MT8186_MUTEX_MOD_MDP_AAL0		2
+#define MT8186_MUTEX_MOD_MDP_HDR0		4
+#define MT8186_MUTEX_MOD_MDP_RSZ0		5
+#define MT8186_MUTEX_MOD_MDP_RSZ1		6
+#define MT8186_MUTEX_MOD_MDP_WROT0		7
+#define MT8186_MUTEX_MOD_MDP_TDSHP0		9
+#define MT8186_MUTEX_MOD_MDP_COLOR0		14
+
 #define MT8173_MUTEX_MOD_DISP_OVL0		11
 #define MT8173_MUTEX_MOD_DISP_OVL1		12
 #define MT8173_MUTEX_MOD_DISP_RDMA0		13
@@ -324,6 +333,17 @@ static const unsigned int mt8186_mutex_mod[DDP_COMPONENT_ID_MAX] = {
 	[DDP_COMPONENT_RDMA1] = MT8186_MUTEX_MOD_DISP_RDMA1,
 };
 
+static const unsigned int mt8186_mdp_mutex_table_mod[MUTEX_MOD_IDX_MAX] = {
+	[MUTEX_MOD_IDX_MDP_RDMA0] = MT8186_MUTEX_MOD_MDP_RDMA0,
+	[MUTEX_MOD_IDX_MDP_RSZ0] = MT8186_MUTEX_MOD_MDP_RSZ0,
+	[MUTEX_MOD_IDX_MDP_RSZ1] = MT8186_MUTEX_MOD_MDP_RSZ1,
+	[MUTEX_MOD_IDX_MDP_TDSHP0] = MT8186_MUTEX_MOD_MDP_TDSHP0,
+	[MUTEX_MOD_IDX_MDP_WROT0] = MT8186_MUTEX_MOD_MDP_WROT0,
+	[MUTEX_MOD_IDX_MDP_HDR0] = MT8186_MUTEX_MOD_MDP_HDR0,
+	[MUTEX_MOD_IDX_MDP_AAL0] = MT8186_MUTEX_MOD_MDP_AAL0,
+	[MUTEX_MOD_IDX_MDP_COLOR0] = MT8186_MUTEX_MOD_MDP_COLOR0,
+};
+
 static const unsigned int mt8192_mutex_mod[DDP_COMPONENT_ID_MAX] = {
 	[DDP_COMPONENT_AAL0] = MT8192_MUTEX_MOD_DISP_AAL0,
 	[DDP_COMPONENT_CCORR] = MT8192_MUTEX_MOD_DISP_CCORR0,
@@ -380,6 +400,13 @@ static const unsigned int mt2712_mutex_sof[DDP_MUTEX_SOF_MAX] = {
 	[MUTEX_SOF_DSI3] = MUTEX_SOF_DSI3,
 };
 
+static const unsigned int mt6795_mutex_sof[DDP_MUTEX_SOF_MAX] = {
+	[MUTEX_SOF_SINGLE_MODE] = MUTEX_SOF_SINGLE_MODE,
+	[MUTEX_SOF_DSI0] = MUTEX_SOF_DSI0,
+	[MUTEX_SOF_DSI1] = MUTEX_SOF_DSI1,
+	[MUTEX_SOF_DPI0] = MUTEX_SOF_DPI0,
+};
+
 static const unsigned int mt8167_mutex_sof[DDP_MUTEX_SOF_MAX] = {
 	[MUTEX_SOF_SINGLE_MODE] = MUTEX_SOF_SINGLE_MODE,
 	[MUTEX_SOF_DSI0] = MUTEX_SOF_DSI0,
@@ -434,6 +461,13 @@ static const struct mtk_mutex_data mt2712_mutex_driver_data = {
 	.mutex_sof_reg = MT2701_MUTEX0_SOF0,
 };
 
+static const struct mtk_mutex_data mt6795_mutex_driver_data = {
+	.mutex_mod = mt8173_mutex_mod,
+	.mutex_sof = mt6795_mutex_sof,
+	.mutex_mod_reg = MT2701_MUTEX0_MOD0,
+	.mutex_sof_reg = MT2701_MUTEX0_SOF0,
+};
+
 static const struct mtk_mutex_data mt8167_mutex_driver_data = {
 	.mutex_mod = mt8167_mutex_mod,
 	.mutex_sof = mt8167_mutex_sof,
@@ -458,6 +492,12 @@ static const struct mtk_mutex_data mt8183_mutex_driver_data = {
 	.no_clk = true,
 };
 
+static const struct mtk_mutex_data mt8186_mdp_mutex_driver_data = {
+	.mutex_mod_reg = MT8183_MUTEX0_MOD0,
+	.mutex_sof_reg = MT8183_MUTEX0_SOF0,
+	.mutex_table_mod = mt8186_mdp_mutex_table_mod,
+};
+
 static const struct mtk_mutex_data mt8186_mutex_driver_data = {
 	.mutex_mod = mt8186_mutex_mod,
 	.mutex_sof = mt8186_mutex_sof,
@@ -802,6 +842,8 @@ static const struct of_device_id mutex_driver_dt_match[] = {
 	  .data = &mt2701_mutex_driver_data},
 	{ .compatible = "mediatek,mt2712-disp-mutex",
 	  .data = &mt2712_mutex_driver_data},
+	{ .compatible = "mediatek,mt6795-disp-mutex",
+	  .data = &mt6795_mutex_driver_data},
 	{ .compatible = "mediatek,mt8167-disp-mutex",
 	  .data = &mt8167_mutex_driver_data},
 	{ .compatible = "mediatek,mt8173-disp-mutex",
@@ -810,6 +852,8 @@ static const struct of_device_id mutex_driver_dt_match[] = {
 	  .data = &mt8183_mutex_driver_data},
 	{ .compatible = "mediatek,mt8186-disp-mutex",
 	  .data = &mt8186_mutex_driver_data},
+	{ .compatible = "mediatek,mt8186-mdp3-mutex",
+	  .data = &mt8186_mdp_mutex_driver_data},
 	{ .compatible = "mediatek,mt8192-disp-mutex",
 	  .data = &mt8192_mutex_driver_data},
 	{ .compatible = "mediatek,mt8195-disp-mutex",
diff --git a/drivers/soc/mediatek/mtk-pm-domains.c b/drivers/soc/mediatek/mtk-pm-domains.c
index 9734f1091c69..09e3c38b8466 100644
--- a/drivers/soc/mediatek/mtk-pm-domains.c
+++ b/drivers/soc/mediatek/mtk-pm-domains.c
@@ -393,7 +393,7 @@ generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_no
 		if (IS_ERR(clk)) {
 			ret = PTR_ERR(clk);
 			dev_err_probe(scpsys->dev, ret,
-				      "%pOF: failed to get clk at index %d: %d\n", node, i, ret);
+				      "%pOF: failed to get clk at index %d\n", node, i);
 			goto err_put_clocks;
 		}
 
@@ -405,8 +405,8 @@ generic_pm_domain *scpsys_add_one_domain(struct scpsys *scpsys, struct device_no
 		if (IS_ERR(clk)) {
 			ret = PTR_ERR(clk);
 			dev_err_probe(scpsys->dev, ret,
-				      "%pOF: failed to get clk at index %d: %d\n", node,
-				      i + clk_ind, ret);
+				      "%pOF: failed to get clk at index %d\n", node,
+				      i + clk_ind);
 			goto err_put_subsys_clocks;
 		}
 
diff --git a/drivers/soc/mediatek/mtk-pmic-wrap.c b/drivers/soc/mediatek/mtk-pmic-wrap.c
index d8cb0f833645..eb82ae06697f 100644
--- a/drivers/soc/mediatek/mtk-pmic-wrap.c
+++ b/drivers/soc/mediatek/mtk-pmic-wrap.c
@@ -2316,7 +2316,7 @@ err_out1:
 static struct platform_driver pwrap_drv = {
 	.driver = {
 		.name = "mt-pmic-pwrap",
-		.of_match_table = of_match_ptr(of_pwrap_match_tbl),
+		.of_match_table = of_pwrap_match_tbl,
 	},
 	.probe = pwrap_probe,
 };
diff --git a/drivers/soc/mediatek/mtk-scpsys.c b/drivers/soc/mediatek/mtk-scpsys.c
index ca75b14931ec..7a668888111c 100644
--- a/drivers/soc/mediatek/mtk-scpsys.c
+++ b/drivers/soc/mediatek/mtk-scpsys.c
@@ -1141,7 +1141,7 @@ static struct platform_driver scpsys_drv = {
 		.name = "mtk-scpsys",
 		.suppress_bind_attrs = true,
 		.owner = THIS_MODULE,
-		.of_match_table = of_match_ptr(of_scpsys_match_tbl),
+		.of_match_table = of_scpsys_match_tbl,
 	},
 };
 builtin_platform_driver(scpsys_drv);
diff --git a/drivers/soc/mediatek/mtk-svs.c b/drivers/soc/mediatek/mtk-svs.c
index dee8664a12fd..0469c9dfeb04 100644
--- a/drivers/soc/mediatek/mtk-svs.c
+++ b/drivers/soc/mediatek/mtk-svs.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2022 MediaTek Inc.
  */
 
+#include <linux/bitfield.h>
 #include <linux/bits.h>
 #include <linux/clk.h>
 #include <linux/completion.h>
@@ -53,22 +54,79 @@
 #define SVSB_MON_VOLT_IGNORE		BIT(16)
 #define SVSB_REMOVE_DVTFIXED_VOLT	BIT(24)
 
-/* svs bank register common configuration */
-#define SVSB_DET_MAX			0xffff
+/* svs bank register fields and common configuration */
+#define SVSB_PTPCONFIG_DETMAX		GENMASK(15, 0)
+#define SVSB_DET_MAX			FIELD_PREP(SVSB_PTPCONFIG_DETMAX, 0xffff)
 #define SVSB_DET_WINDOW			0xa28
-#define SVSB_DTHI			0x1
-#define SVSB_DTLO			0xfe
-#define SVSB_EN_INIT01			0x1
-#define SVSB_EN_INIT02			0x5
-#define SVSB_EN_MON			0x2
-#define SVSB_EN_OFF			0x0
-#define SVSB_INTEN_INIT0x		0x00005f01
-#define SVSB_INTEN_MONVOPEN		0x00ff0000
-#define SVSB_INTSTS_CLEAN		0x00ffffff
-#define SVSB_INTSTS_COMPLETE		0x1
-#define SVSB_INTSTS_MONVOP		0x00ff0000
+
+/* DESCHAR */
+#define SVSB_DESCHAR_FLD_MDES		GENMASK(7, 0)
+#define SVSB_DESCHAR_FLD_BDES		GENMASK(15, 8)
+
+/* TEMPCHAR */
+#define SVSB_TEMPCHAR_FLD_DVT_FIXED	GENMASK(7, 0)
+#define SVSB_TEMPCHAR_FLD_MTDES		GENMASK(15, 8)
+#define SVSB_TEMPCHAR_FLD_VCO		GENMASK(23, 16)
+
+/* DETCHAR */
+#define SVSB_DETCHAR_FLD_DCMDET		GENMASK(7, 0)
+#define SVSB_DETCHAR_FLD_DCBDET		GENMASK(15, 8)
+
+/* SVSEN (PTPEN) */
+#define SVSB_PTPEN_INIT01		BIT(0)
+#define SVSB_PTPEN_MON			BIT(1)
+#define SVSB_PTPEN_INIT02		(SVSB_PTPEN_INIT01 | BIT(2))
+#define SVSB_PTPEN_OFF			0x0
+
+/* FREQPCTS */
+#define SVSB_FREQPCTS_FLD_PCT0_4	GENMASK(7, 0)
+#define SVSB_FREQPCTS_FLD_PCT1_5	GENMASK(15, 8)
+#define SVSB_FREQPCTS_FLD_PCT2_6	GENMASK(23, 16)
+#define SVSB_FREQPCTS_FLD_PCT3_7	GENMASK(31, 24)
+
+/* INTSTS */
+#define SVSB_INTSTS_VAL_CLEAN		0x00ffffff
+#define SVSB_INTSTS_F0_COMPLETE		BIT(0)
+#define SVSB_INTSTS_FLD_MONVOP		GENMASK(23, 16)
 #define SVSB_RUNCONFIG_DEFAULT		0x80000000
 
+/* LIMITVALS */
+#define SVSB_LIMITVALS_FLD_DTLO		GENMASK(7, 0)
+#define SVSB_LIMITVALS_FLD_DTHI		GENMASK(15, 8)
+#define SVSB_LIMITVALS_FLD_VMIN		GENMASK(23, 16)
+#define SVSB_LIMITVALS_FLD_VMAX		GENMASK(31, 24)
+#define SVSB_VAL_DTHI			0x1
+#define SVSB_VAL_DTLO			0xfe
+
+/* INTEN */
+#define SVSB_INTEN_F0EN			BIT(0)
+#define SVSB_INTEN_DACK0UPEN		BIT(8)
+#define SVSB_INTEN_DC0EN		BIT(9)
+#define SVSB_INTEN_DC1EN		BIT(10)
+#define SVSB_INTEN_DACK0LOEN		BIT(11)
+#define SVSB_INTEN_INITPROD_OVF_EN	BIT(12)
+#define SVSB_INTEN_INITSUM_OVF_EN	BIT(14)
+#define SVSB_INTEN_MONVOPEN		GENMASK(23, 16)
+#define SVSB_INTEN_INIT0x		(SVSB_INTEN_F0EN | SVSB_INTEN_DACK0UPEN |	\
+					 SVSB_INTEN_DC0EN | SVSB_INTEN_DC1EN |		\
+					 SVSB_INTEN_DACK0LOEN |				\
+					 SVSB_INTEN_INITPROD_OVF_EN |			\
+					 SVSB_INTEN_INITSUM_OVF_EN)
+
+/* TSCALCS */
+#define SVSB_TSCALCS_FLD_MTS		GENMASK(11, 0)
+#define SVSB_TSCALCS_FLD_BTS		GENMASK(23, 12)
+
+/* INIT2VALS */
+#define SVSB_INIT2VALS_FLD_DCVOFFSETIN	GENMASK(15, 0)
+#define SVSB_INIT2VALS_FLD_AGEVOFFSETIN	GENMASK(31, 16)
+
+/* VOPS */
+#define SVSB_VOPS_FLD_VOP0_4		GENMASK(7, 0)
+#define SVSB_VOPS_FLD_VOP1_5		GENMASK(15, 8)
+#define SVSB_VOPS_FLD_VOP2_6		GENMASK(23, 16)
+#define SVSB_VOPS_FLD_VOP3_7		GENMASK(31, 24)
+
 /* svs bank related setting */
 #define BITS8				8
 #define MAX_OPP_ENTRIES			16
@@ -262,7 +320,6 @@ static const u32 svs_regs_v2[] = {
  * @rst: svs platform reset control
  * @efuse_parsing: svs platform efuse parsing function pointer
  * @probe: svs platform probe function pointer
- * @irqflags: svs platform irq settings flags
  * @efuse_max: total number of svs efuse
  * @tefuse_max: total number of thermal efuse
  * @regs: svs platform registers map
@@ -280,7 +337,6 @@ struct svs_platform {
 	struct reset_control *rst;
 	bool (*efuse_parsing)(struct svs_platform *svsp);
 	int (*probe)(struct svs_platform *svsp);
-	unsigned long irqflags;
 	size_t efuse_max;
 	size_t tefuse_max;
 	const u32 *regs;
@@ -294,7 +350,6 @@ struct svs_platform_data {
 	struct svs_bank *banks;
 	bool (*efuse_parsing)(struct svs_platform *svsp);
 	int (*probe)(struct svs_platform *svsp);
-	unsigned long irqflags;
 	const u32 *regs;
 	u32 bank_max;
 };
@@ -668,8 +723,8 @@ static ssize_t svs_enable_debug_write(struct file *filp,
 		svsp->pbank = svsb;
 		svsb->mode_support = SVSB_MODE_ALL_DISABLE;
 		svs_switch_bank(svsp);
-		svs_writel_relaxed(svsp, SVSB_EN_OFF, SVSEN);
-		svs_writel_relaxed(svsp, SVSB_INTSTS_CLEAN, INTSTS);
+		svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN);
+		svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS);
 		spin_unlock_irqrestore(&svs_lock, flags);
 
 		svsb->phase = SVSB_PHASE_ERROR;
@@ -830,7 +885,7 @@ static void svs_get_bank_volts_v3(struct svs_platform *svsp)
 		} else if (svsb->type == SVSB_LOW) {
 			/* volt[turn_pt] + volt[j] ~ volt[opp_count - 1] */
 			j = svsb->opp_count - 7;
-			svsb->volt[turn_pt] = vop30 & GENMASK(7, 0);
+			svsb->volt[turn_pt] = FIELD_GET(SVSB_VOPS_FLD_VOP0_4, vop30);
 			shift_byte++;
 			for (i = j; i < svsb->opp_count; i++) {
 				b_sft = BITS8 * (shift_byte % REG_BYTES);
@@ -852,7 +907,7 @@ static void svs_get_bank_volts_v3(struct svs_platform *svsp)
 		if (svsb->type == SVSB_HIGH) {
 			/* volt[0] + volt[j] ~ volt[turn_pt - 1] */
 			j = turn_pt - 7;
-			svsb->volt[0] = vop30 & GENMASK(7, 0);
+			svsb->volt[0] = FIELD_GET(SVSB_VOPS_FLD_VOP0_4, vop30);
 			shift_byte++;
 			for (i = j; i < turn_pt; i++) {
 				b_sft = BITS8 * (shift_byte % REG_BYTES);
@@ -983,16 +1038,16 @@ static void svs_get_bank_volts_v2(struct svs_platform *svsp)
 	u32 temp, i;
 
 	temp = svs_readl_relaxed(svsp, VOP74);
-	svsb->volt[14] = (temp >> 24) & GENMASK(7, 0);
-	svsb->volt[12] = (temp >> 16) & GENMASK(7, 0);
-	svsb->volt[10] = (temp >> 8)  & GENMASK(7, 0);
-	svsb->volt[8] = (temp & GENMASK(7, 0));
+	svsb->volt[14] = FIELD_GET(SVSB_VOPS_FLD_VOP3_7, temp);
+	svsb->volt[12] = FIELD_GET(SVSB_VOPS_FLD_VOP2_6, temp);
+	svsb->volt[10] = FIELD_GET(SVSB_VOPS_FLD_VOP1_5, temp);
+	svsb->volt[8] = FIELD_GET(SVSB_VOPS_FLD_VOP0_4, temp);
 
 	temp = svs_readl_relaxed(svsp, VOP30);
-	svsb->volt[6] = (temp >> 24) & GENMASK(7, 0);
-	svsb->volt[4] = (temp >> 16) & GENMASK(7, 0);
-	svsb->volt[2] = (temp >> 8)  & GENMASK(7, 0);
-	svsb->volt[0] = (temp & GENMASK(7, 0));
+	svsb->volt[6] = FIELD_GET(SVSB_VOPS_FLD_VOP3_7, temp);
+	svsb->volt[4] = FIELD_GET(SVSB_VOPS_FLD_VOP2_6, temp);
+	svsb->volt[2] = FIELD_GET(SVSB_VOPS_FLD_VOP1_5, temp);
+	svsb->volt[0] = FIELD_GET(SVSB_VOPS_FLD_VOP0_4, temp);
 
 	for (i = 0; i <= 12; i += 2)
 		svsb->volt[i + 1] = interpolate(svsb->freq_pct[i],
@@ -1014,20 +1069,20 @@ static void svs_get_bank_volts_v2(struct svs_platform *svsp)
 static void svs_set_bank_freq_pct_v2(struct svs_platform *svsp)
 {
 	struct svs_bank *svsb = svsp->pbank;
+	u32 freqpct74_val, freqpct30_val;
+
+	freqpct74_val = FIELD_PREP(SVSB_FREQPCTS_FLD_PCT0_4, svsb->freq_pct[8]) |
+			FIELD_PREP(SVSB_FREQPCTS_FLD_PCT1_5, svsb->freq_pct[10]) |
+			FIELD_PREP(SVSB_FREQPCTS_FLD_PCT2_6, svsb->freq_pct[12]) |
+			FIELD_PREP(SVSB_FREQPCTS_FLD_PCT3_7, svsb->freq_pct[14]);
 
-	svs_writel_relaxed(svsp,
-			   (svsb->freq_pct[14] << 24) |
-			   (svsb->freq_pct[12] << 16) |
-			   (svsb->freq_pct[10] << 8) |
-			   svsb->freq_pct[8],
-			   FREQPCT74);
-
-	svs_writel_relaxed(svsp,
-			   (svsb->freq_pct[6] << 24) |
-			   (svsb->freq_pct[4] << 16) |
-			   (svsb->freq_pct[2] << 8) |
-			   svsb->freq_pct[0],
-			   FREQPCT30);
+	freqpct30_val = FIELD_PREP(SVSB_FREQPCTS_FLD_PCT0_4, svsb->freq_pct[0]) |
+			FIELD_PREP(SVSB_FREQPCTS_FLD_PCT1_5, svsb->freq_pct[2]) |
+			FIELD_PREP(SVSB_FREQPCTS_FLD_PCT2_6, svsb->freq_pct[4]) |
+			FIELD_PREP(SVSB_FREQPCTS_FLD_PCT3_7, svsb->freq_pct[6]);
+
+	svs_writel_relaxed(svsp, freqpct74_val, FREQPCT74);
+	svs_writel_relaxed(svsp, freqpct30_val, FREQPCT30);
 }
 
 static void svs_set_bank_phase(struct svs_platform *svsp,
@@ -1038,13 +1093,17 @@ static void svs_set_bank_phase(struct svs_platform *svsp,
 
 	svs_switch_bank(svsp);
 
-	des_char = (svsb->bdes << 8) | svsb->mdes;
+	des_char = FIELD_PREP(SVSB_DESCHAR_FLD_BDES, svsb->bdes) |
+		   FIELD_PREP(SVSB_DESCHAR_FLD_MDES, svsb->mdes);
 	svs_writel_relaxed(svsp, des_char, DESCHAR);
 
-	temp_char = (svsb->vco << 16) | (svsb->mtdes << 8) | svsb->dvt_fixed;
+	temp_char = FIELD_PREP(SVSB_TEMPCHAR_FLD_VCO, svsb->vco) |
+		    FIELD_PREP(SVSB_TEMPCHAR_FLD_MTDES, svsb->mtdes) |
+		    FIELD_PREP(SVSB_TEMPCHAR_FLD_DVT_FIXED, svsb->dvt_fixed);
 	svs_writel_relaxed(svsp, temp_char, TEMPCHAR);
 
-	det_char = (svsb->dcbdet << 8) | svsb->dcmdet;
+	det_char = FIELD_PREP(SVSB_DETCHAR_FLD_DCBDET, svsb->dcbdet) |
+		   FIELD_PREP(SVSB_DETCHAR_FLD_DCMDET, svsb->dcmdet);
 	svs_writel_relaxed(svsp, det_char, DETCHAR);
 
 	svs_writel_relaxed(svsp, svsb->dc_config, DCCONFIG);
@@ -1053,33 +1112,37 @@ static void svs_set_bank_phase(struct svs_platform *svsp,
 
 	svsb->set_freq_pct(svsp);
 
-	limit_vals = (svsb->vmax << 24) | (svsb->vmin << 16) |
-		     (SVSB_DTHI << 8) | SVSB_DTLO;
+	limit_vals = FIELD_PREP(SVSB_LIMITVALS_FLD_DTLO, SVSB_VAL_DTLO) |
+		     FIELD_PREP(SVSB_LIMITVALS_FLD_DTHI, SVSB_VAL_DTHI) |
+		     FIELD_PREP(SVSB_LIMITVALS_FLD_VMIN, svsb->vmin) |
+		     FIELD_PREP(SVSB_LIMITVALS_FLD_VMAX, svsb->vmax);
 	svs_writel_relaxed(svsp, limit_vals, LIMITVALS);
 
 	svs_writel_relaxed(svsp, SVSB_DET_WINDOW, DETWINDOW);
 	svs_writel_relaxed(svsp, SVSB_DET_MAX, CONFIG);
 	svs_writel_relaxed(svsp, svsb->chk_shift, CHKSHIFT);
 	svs_writel_relaxed(svsp, svsb->ctl0, CTL0);
-	svs_writel_relaxed(svsp, SVSB_INTSTS_CLEAN, INTSTS);
+	svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS);
 
 	switch (target_phase) {
 	case SVSB_PHASE_INIT01:
 		svs_writel_relaxed(svsp, svsb->vboot, VBOOT);
 		svs_writel_relaxed(svsp, SVSB_INTEN_INIT0x, INTEN);
-		svs_writel_relaxed(svsp, SVSB_EN_INIT01, SVSEN);
+		svs_writel_relaxed(svsp, SVSB_PTPEN_INIT01, SVSEN);
 		break;
 	case SVSB_PHASE_INIT02:
+		init2vals = FIELD_PREP(SVSB_INIT2VALS_FLD_AGEVOFFSETIN, svsb->age_voffset_in) |
+			    FIELD_PREP(SVSB_INIT2VALS_FLD_DCVOFFSETIN, svsb->dc_voffset_in);
 		svs_writel_relaxed(svsp, SVSB_INTEN_INIT0x, INTEN);
-		init2vals = (svsb->age_voffset_in << 16) | svsb->dc_voffset_in;
 		svs_writel_relaxed(svsp, init2vals, INIT2VALS);
-		svs_writel_relaxed(svsp, SVSB_EN_INIT02, SVSEN);
+		svs_writel_relaxed(svsp, SVSB_PTPEN_INIT02, SVSEN);
 		break;
 	case SVSB_PHASE_MON:
-		ts_calcs = (svsb->bts << 12) | svsb->mts;
+		ts_calcs = FIELD_PREP(SVSB_TSCALCS_FLD_BTS, svsb->bts) |
+			   FIELD_PREP(SVSB_TSCALCS_FLD_MTS, svsb->mts);
 		svs_writel_relaxed(svsp, ts_calcs, TSCALCS);
 		svs_writel_relaxed(svsp, SVSB_INTEN_MONVOPEN, INTEN);
-		svs_writel_relaxed(svsp, SVSB_EN_MON, SVSEN);
+		svs_writel_relaxed(svsp, SVSB_PTPEN_MON, SVSEN);
 		break;
 	default:
 		dev_err(svsb->dev, "requested unknown target phase: %u\n",
@@ -1115,8 +1178,8 @@ static inline void svs_error_isr_handler(struct svs_platform *svsp)
 	svs_save_bank_register_data(svsp, SVSB_PHASE_ERROR);
 
 	svsb->phase = SVSB_PHASE_ERROR;
-	svs_writel_relaxed(svsp, SVSB_EN_OFF, SVSEN);
-	svs_writel_relaxed(svsp, SVSB_INTSTS_CLEAN, INTSTS);
+	svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN);
+	svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS);
 }
 
 static inline void svs_init01_isr_handler(struct svs_platform *svsp)
@@ -1141,8 +1204,8 @@ static inline void svs_init01_isr_handler(struct svs_platform *svsp)
 	svsb->age_voffset_in = svs_readl_relaxed(svsp, AGEVALUES) &
 			       GENMASK(15, 0);
 
-	svs_writel_relaxed(svsp, SVSB_EN_OFF, SVSEN);
-	svs_writel_relaxed(svsp, SVSB_INTSTS_COMPLETE, INTSTS);
+	svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN);
+	svs_writel_relaxed(svsp, SVSB_INTSTS_F0_COMPLETE, INTSTS);
 	svsb->core_sel &= ~SVSB_DET_CLK_EN;
 }
 
@@ -1160,8 +1223,8 @@ static inline void svs_init02_isr_handler(struct svs_platform *svsp)
 	svsb->phase = SVSB_PHASE_INIT02;
 	svsb->get_volts(svsp);
 
-	svs_writel_relaxed(svsp, SVSB_EN_OFF, SVSEN);
-	svs_writel_relaxed(svsp, SVSB_INTSTS_COMPLETE, INTSTS);
+	svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN);
+	svs_writel_relaxed(svsp, SVSB_INTSTS_F0_COMPLETE, INTSTS);
 }
 
 static inline void svs_mon_mode_isr_handler(struct svs_platform *svsp)
@@ -1174,7 +1237,7 @@ static inline void svs_mon_mode_isr_handler(struct svs_platform *svsp)
 	svsb->get_volts(svsp);
 
 	svsb->temp = svs_readl_relaxed(svsp, TEMP) & GENMASK(7, 0);
-	svs_writel_relaxed(svsp, SVSB_INTSTS_MONVOP, INTSTS);
+	svs_writel_relaxed(svsp, SVSB_INTSTS_FLD_MONVOP, INTSTS);
 }
 
 static irqreturn_t svs_isr(int irq, void *data)
@@ -1201,13 +1264,13 @@ static irqreturn_t svs_isr(int irq, void *data)
 		int_sts = svs_readl_relaxed(svsp, INTSTS);
 		svs_en = svs_readl_relaxed(svsp, SVSEN);
 
-		if (int_sts == SVSB_INTSTS_COMPLETE &&
-		    svs_en == SVSB_EN_INIT01)
+		if (int_sts == SVSB_INTSTS_F0_COMPLETE &&
+		    svs_en == SVSB_PTPEN_INIT01)
 			svs_init01_isr_handler(svsp);
-		else if (int_sts == SVSB_INTSTS_COMPLETE &&
-			 svs_en == SVSB_EN_INIT02)
+		else if (int_sts == SVSB_INTSTS_F0_COMPLETE &&
+			 svs_en == SVSB_PTPEN_INIT02)
 			svs_init02_isr_handler(svsp);
-		else if (int_sts & SVSB_INTSTS_MONVOP)
+		else if (int_sts & SVSB_INTSTS_FLD_MONVOP)
 			svs_mon_mode_isr_handler(svsp);
 		else
 			svs_error_isr_handler(svsp);
@@ -1493,8 +1556,8 @@ static int svs_suspend(struct device *dev)
 		spin_lock_irqsave(&svs_lock, flags);
 		svsp->pbank = svsb;
 		svs_switch_bank(svsp);
-		svs_writel_relaxed(svsp, SVSB_EN_OFF, SVSEN);
-		svs_writel_relaxed(svsp, SVSB_INTSTS_CLEAN, INTSTS);
+		svs_writel_relaxed(svsp, SVSB_PTPEN_OFF, SVSEN);
+		svs_writel_relaxed(svsp, SVSB_INTSTS_VAL_CLEAN, INTSTS);
 		spin_unlock_irqrestore(&svs_lock, flags);
 
 		svsb->phase = SVSB_PHASE_ERROR;
@@ -1589,7 +1652,7 @@ static int svs_bank_resource_setup(struct svs_platform *svsp)
 
 		dev_set_drvdata(svsb->dev, svsp);
 
-		ret = dev_pm_opp_of_add_table(svsb->opp_dev);
+		ret = devm_pm_opp_of_add_table(svsb->opp_dev);
 		if (ret) {
 			dev_err(svsb->dev, "add opp table fail: %d\n", ret);
 			return ret;
@@ -1644,11 +1707,36 @@ static int svs_bank_resource_setup(struct svs_platform *svsp)
 	return 0;
 }
 
+static int svs_thermal_efuse_get_data(struct svs_platform *svsp)
+{
+	struct nvmem_cell *cell;
+
+	/* Thermal efuse parsing */
+	cell = nvmem_cell_get(svsp->dev, "t-calibration-data");
+	if (IS_ERR_OR_NULL(cell)) {
+		dev_err(svsp->dev, "no \"t-calibration-data\"? %ld\n", PTR_ERR(cell));
+		return PTR_ERR(cell);
+	}
+
+	svsp->tefuse = nvmem_cell_read(cell, &svsp->tefuse_max);
+	if (IS_ERR(svsp->tefuse)) {
+		dev_err(svsp->dev, "cannot read thermal efuse: %ld\n",
+			PTR_ERR(svsp->tefuse));
+		nvmem_cell_put(cell);
+		return PTR_ERR(svsp->tefuse);
+	}
+
+	svsp->tefuse_max /= sizeof(u32);
+	nvmem_cell_put(cell);
+
+	return 0;
+}
+
 static bool svs_mt8192_efuse_parsing(struct svs_platform *svsp)
 {
 	struct svs_bank *svsb;
-	struct nvmem_cell *cell;
 	u32 idx, i, vmin, golden_temp;
+	int ret;
 
 	for (i = 0; i < svsp->efuse_max; i++)
 		if (svsp->efuse[i])
@@ -1686,24 +1774,9 @@ static bool svs_mt8192_efuse_parsing(struct svs_platform *svsp)
 		svsb->vmax += svsb->dvt_fixed;
 	}
 
-	/* Thermal efuse parsing */
-	cell = nvmem_cell_get(svsp->dev, "t-calibration-data");
-	if (IS_ERR_OR_NULL(cell)) {
-		dev_err(svsp->dev, "no \"t-calibration-data\"? %ld\n",
-			PTR_ERR(cell));
-		return false;
-	}
-
-	svsp->tefuse = nvmem_cell_read(cell, &svsp->tefuse_max);
-	if (IS_ERR(svsp->tefuse)) {
-		dev_err(svsp->dev, "cannot read thermal efuse: %ld\n",
-			PTR_ERR(svsp->tefuse));
-		nvmem_cell_put(cell);
+	ret = svs_thermal_efuse_get_data(svsp);
+	if (ret)
 		return false;
-	}
-
-	svsp->tefuse_max /= sizeof(u32);
-	nvmem_cell_put(cell);
 
 	for (i = 0; i < svsp->tefuse_max; i++)
 		if (svsp->tefuse[i] != 0)
@@ -1726,11 +1799,11 @@ static bool svs_mt8192_efuse_parsing(struct svs_platform *svsp)
 static bool svs_mt8183_efuse_parsing(struct svs_platform *svsp)
 {
 	struct svs_bank *svsb;
-	struct nvmem_cell *cell;
 	int format[6], x_roomt[6], o_vtsmcu[5], o_vtsabb, tb_roomt = 0;
 	int adc_ge_t, adc_oe_t, ge, oe, gain, degc_cali, adc_cali_en_t;
 	int o_slope, o_slope_sign, ts_id;
 	u32 idx, i, ft_pgm, mts, temp0, temp1, temp2;
+	int ret;
 
 	for (i = 0; i < svsp->efuse_max; i++)
 		if (svsp->efuse[i])
@@ -1806,24 +1879,9 @@ static bool svs_mt8183_efuse_parsing(struct svs_platform *svsp)
 		}
 	}
 
-	/* Get thermal efuse by nvmem */
-	cell = nvmem_cell_get(svsp->dev, "t-calibration-data");
-	if (IS_ERR(cell)) {
-		dev_err(svsp->dev, "no \"t-calibration-data\"? %ld\n",
-			PTR_ERR(cell));
-		goto remove_mt8183_svsb_mon_mode;
-	}
-
-	svsp->tefuse = nvmem_cell_read(cell, &svsp->tefuse_max);
-	if (IS_ERR(svsp->tefuse)) {
-		dev_err(svsp->dev, "cannot read thermal efuse: %ld\n",
-			PTR_ERR(svsp->tefuse));
-		nvmem_cell_put(cell);
-		goto remove_mt8183_svsb_mon_mode;
-	}
-
-	svsp->tefuse_max /= sizeof(u32);
-	nvmem_cell_put(cell);
+	ret = svs_thermal_efuse_get_data(svsp);
+	if (ret)
+		return false;
 
 	/* Thermal efuse parsing */
 	adc_ge_t = (svsp->tefuse[1] >> 22) & GENMASK(9, 0);
@@ -2244,7 +2302,6 @@ static const struct svs_platform_data svs_mt8192_platform_data = {
 	.banks = svs_mt8192_banks,
 	.efuse_parsing = svs_mt8192_efuse_parsing,
 	.probe = svs_mt8192_platform_probe,
-	.irqflags = IRQF_TRIGGER_HIGH,
 	.regs = svs_regs_v2,
 	.bank_max = ARRAY_SIZE(svs_mt8192_banks),
 };
@@ -2254,7 +2311,6 @@ static const struct svs_platform_data svs_mt8183_platform_data = {
 	.banks = svs_mt8183_banks,
 	.efuse_parsing = svs_mt8183_efuse_parsing,
 	.probe = svs_mt8183_platform_probe,
-	.irqflags = IRQF_TRIGGER_LOW,
 	.regs = svs_regs_v2,
 	.bank_max = ARRAY_SIZE(svs_mt8183_banks),
 };
@@ -2292,7 +2348,6 @@ static struct svs_platform *svs_platform_probe(struct platform_device *pdev)
 	svsp->banks = svsp_data->banks;
 	svsp->efuse_parsing = svsp_data->efuse_parsing;
 	svsp->probe = svsp_data->probe;
-	svsp->irqflags = svsp_data->irqflags;
 	svsp->regs = svsp_data->regs;
 	svsp->bank_max = svsp_data->bank_max;
 
@@ -2306,8 +2361,7 @@ static struct svs_platform *svs_platform_probe(struct platform_device *pdev)
 static int svs_probe(struct platform_device *pdev)
 {
 	struct svs_platform *svsp;
-	unsigned int svsp_irq;
-	int ret;
+	int svsp_irq, ret;
 
 	svsp = svs_platform_probe(pdev);
 	if (IS_ERR(svsp))
@@ -2325,10 +2379,14 @@ static int svs_probe(struct platform_device *pdev)
 		goto svs_probe_free_resource;
 	}
 
-	svsp_irq = irq_of_parse_and_map(svsp->dev->of_node, 0);
+	svsp_irq = platform_get_irq(pdev, 0);
+	if (svsp_irq < 0) {
+		ret = svsp_irq;
+		goto svs_probe_free_resource;
+	}
+
 	ret = devm_request_threaded_irq(svsp->dev, svsp_irq, NULL, svs_isr,
-					svsp->irqflags | IRQF_ONESHOT,
-					svsp->name, svsp);
+					IRQF_ONESHOT, svsp->name, svsp);
 	if (ret) {
 		dev_err(svsp->dev, "register irq(%d) failed: %d\n",
 			svsp_irq, ret);
@@ -2392,7 +2450,7 @@ static struct platform_driver svs_driver = {
 	.driver	= {
 		.name		= "mtk-svs",
 		.pm		= &svs_pm_ops,
-		.of_match_table	= of_match_ptr(svs_of_match),
+		.of_match_table	= svs_of_match,
 	},
 };
 
diff --git a/drivers/soc/pxa/ssp.c b/drivers/soc/pxa/ssp.c
index 563440315acd..93449fb3519e 100644
--- a/drivers/soc/pxa/ssp.c
+++ b/drivers/soc/pxa/ssp.c
@@ -180,11 +180,7 @@ static int pxa_ssp_probe(struct platform_device *pdev)
 
 static int pxa_ssp_remove(struct platform_device *pdev)
 {
-	struct ssp_device *ssp;
-
-	ssp = platform_get_drvdata(pdev);
-	if (ssp == NULL)
-		return -ENODEV;
+	struct ssp_device *ssp = platform_get_drvdata(pdev);
 
 	mutex_lock(&ssp_lock);
 	list_del(&ssp->node);
diff --git a/drivers/soc/qcom/Kconfig b/drivers/soc/qcom/Kconfig
index e0d7a5459562..024e420f1bb7 100644
--- a/drivers/soc/qcom/Kconfig
+++ b/drivers/soc/qcom/Kconfig
@@ -129,7 +129,7 @@ config QCOM_RPMHPD
 
 config QCOM_RPMPD
 	tristate "Qualcomm RPM Power domain driver"
-	depends on PM
+	depends on PM && OF
 	depends on QCOM_SMD_RPM
 	select PM_GENERIC_DOMAINS
 	select PM_GENERIC_DOMAINS_OF
diff --git a/drivers/soc/qcom/icc-bwmon.c b/drivers/soc/qcom/icc-bwmon.c
index 7f8aca533cd3..d07be3700db6 100644
--- a/drivers/soc/qcom/icc-bwmon.c
+++ b/drivers/soc/qcom/icc-bwmon.c
@@ -5,6 +5,8 @@
  * Author: Krzysztof Kozlowski <krzysztof.kozlowski@linaro.org>, based on
  *         previous work of Thara Gopinath and msm-4.9 downstream sources.
  */
+
+#include <linux/err.h>
 #include <linux/interconnect.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
@@ -13,6 +15,7 @@
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
 #include <linux/pm_opp.h>
+#include <linux/regmap.h>
 #include <linux/sizes.h>
 
 /*
@@ -31,33 +34,44 @@
 /* Internal sampling clock frequency */
 #define HW_TIMER_HZ				19200000
 
-#define BWMON_GLOBAL_IRQ_STATUS			0x0
-#define BWMON_GLOBAL_IRQ_CLEAR			0x8
-#define BWMON_GLOBAL_IRQ_ENABLE			0xc
-#define BWMON_GLOBAL_IRQ_ENABLE_ENABLE		BIT(0)
-
-#define BWMON_IRQ_STATUS			0x100
-#define BWMON_IRQ_STATUS_ZONE_SHIFT		4
-#define BWMON_IRQ_CLEAR				0x108
-#define BWMON_IRQ_ENABLE			0x10c
-#define BWMON_IRQ_ENABLE_ZONE1_SHIFT		5
-#define BWMON_IRQ_ENABLE_ZONE2_SHIFT		6
-#define BWMON_IRQ_ENABLE_ZONE3_SHIFT		7
-#define BWMON_IRQ_ENABLE_MASK			(BIT(BWMON_IRQ_ENABLE_ZONE1_SHIFT) | \
-						 BIT(BWMON_IRQ_ENABLE_ZONE3_SHIFT))
-
-#define BWMON_ENABLE				0x2a0
+#define BWMON_V4_GLOBAL_IRQ_CLEAR		0x008
+#define BWMON_V4_GLOBAL_IRQ_ENABLE		0x00c
+/*
+ * All values here and further are matching regmap fields, so without absolute
+ * register offsets.
+ */
+#define BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE	BIT(0)
+
+#define BWMON_V4_IRQ_STATUS			0x100
+#define BWMON_V4_IRQ_CLEAR			0x108
+
+#define BWMON_V4_IRQ_ENABLE			0x10c
+#define BWMON_IRQ_ENABLE_MASK			(BIT(1) | BIT(3))
+#define BWMON_V5_IRQ_STATUS			0x000
+#define BWMON_V5_IRQ_CLEAR			0x008
+#define BWMON_V5_IRQ_ENABLE			0x00c
+
+#define BWMON_V4_ENABLE				0x2a0
+#define BWMON_V5_ENABLE				0x010
 #define BWMON_ENABLE_ENABLE			BIT(0)
 
-#define BWMON_CLEAR				0x2a4
+#define BWMON_V4_CLEAR				0x2a4
+#define BWMON_V5_CLEAR				0x014
 #define BWMON_CLEAR_CLEAR			BIT(0)
+#define BWMON_CLEAR_CLEAR_ALL			BIT(1)
 
-#define BWMON_SAMPLE_WINDOW			0x2a8
-#define BWMON_THRESHOLD_HIGH			0x2ac
-#define BWMON_THRESHOLD_MED			0x2b0
-#define BWMON_THRESHOLD_LOW			0x2b4
+#define BWMON_V4_SAMPLE_WINDOW			0x2a8
+#define BWMON_V5_SAMPLE_WINDOW			0x020
 
-#define BWMON_ZONE_ACTIONS			0x2b8
+#define BWMON_V4_THRESHOLD_HIGH			0x2ac
+#define BWMON_V4_THRESHOLD_MED			0x2b0
+#define BWMON_V4_THRESHOLD_LOW			0x2b4
+#define BWMON_V5_THRESHOLD_HIGH			0x024
+#define BWMON_V5_THRESHOLD_MED			0x028
+#define BWMON_V5_THRESHOLD_LOW			0x02c
+
+#define BWMON_V4_ZONE_ACTIONS			0x2b8
+#define BWMON_V5_ZONE_ACTIONS			0x030
 /*
  * Actions to perform on some zone 'z' when current zone hits the threshold:
  * Increment counter of zone 'z'
@@ -83,55 +97,244 @@
 						 BWMON_ZONE_ACTIONS_CLEAR(2) | \
 						 BWMON_ZONE_ACTIONS_CLEAR(1) | \
 						 BWMON_ZONE_ACTIONS_CLEAR(0))
-/* Value for BWMON_ZONE_ACTIONS */
-#define BWMON_ZONE_ACTIONS_DEFAULT		(BWMON_ZONE_ACTIONS_ZONE0 | \
-						 BWMON_ZONE_ACTIONS_ZONE1 << 8 | \
-						 BWMON_ZONE_ACTIONS_ZONE2 << 16 | \
-						 BWMON_ZONE_ACTIONS_ZONE3 << 24)
 
 /*
- * There is no clear documentation/explanation of BWMON_THRESHOLD_COUNT
+ * There is no clear documentation/explanation of BWMON_V4_THRESHOLD_COUNT
  * register. Based on observations, this is number of times one threshold has to
  * be reached, to trigger interrupt in given zone.
  *
  * 0xff are maximum values meant to ignore the zones 0 and 2.
  */
-#define BWMON_THRESHOLD_COUNT			0x2bc
-#define BWMON_THRESHOLD_COUNT_ZONE1_SHIFT	8
-#define BWMON_THRESHOLD_COUNT_ZONE2_SHIFT	16
-#define BWMON_THRESHOLD_COUNT_ZONE3_SHIFT	24
+#define BWMON_V4_THRESHOLD_COUNT		0x2bc
+#define BWMON_V5_THRESHOLD_COUNT		0x034
 #define BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT	0xff
 #define BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT	0xff
 
-/* BWMONv4 count registers use count unit of 64 kB */
-#define BWMON_COUNT_UNIT_KB			64
-#define BWMON_ZONE_COUNT			0x2d8
-#define BWMON_ZONE_MAX(zone)			(0x2e0 + 4 * (zone))
+#define BWMON_V4_ZONE_MAX(zone)			(0x2e0 + 4 * (zone))
+#define BWMON_V5_ZONE_MAX(zone)			(0x044 + 4 * (zone))
+
+/* Quirks for specific BWMON types */
+#define BWMON_HAS_GLOBAL_IRQ			BIT(0)
+#define BWMON_NEEDS_FORCE_CLEAR			BIT(1)
+
+enum bwmon_fields {
+	F_GLOBAL_IRQ_CLEAR,
+	F_GLOBAL_IRQ_ENABLE,
+	F_IRQ_STATUS,
+	F_IRQ_CLEAR,
+	F_IRQ_ENABLE,
+	F_ENABLE,
+	F_CLEAR,
+	F_SAMPLE_WINDOW,
+	F_THRESHOLD_HIGH,
+	F_THRESHOLD_MED,
+	F_THRESHOLD_LOW,
+	F_ZONE_ACTIONS_ZONE0,
+	F_ZONE_ACTIONS_ZONE1,
+	F_ZONE_ACTIONS_ZONE2,
+	F_ZONE_ACTIONS_ZONE3,
+	F_THRESHOLD_COUNT_ZONE0,
+	F_THRESHOLD_COUNT_ZONE1,
+	F_THRESHOLD_COUNT_ZONE2,
+	F_THRESHOLD_COUNT_ZONE3,
+	F_ZONE0_MAX,
+	F_ZONE1_MAX,
+	F_ZONE2_MAX,
+	F_ZONE3_MAX,
+
+	F_NUM_FIELDS
+};
 
 struct icc_bwmon_data {
 	unsigned int sample_ms;
+	unsigned int count_unit_kb; /* kbytes */
 	unsigned int default_highbw_kbps;
 	unsigned int default_medbw_kbps;
 	unsigned int default_lowbw_kbps;
 	u8 zone1_thres_count;
 	u8 zone3_thres_count;
+	unsigned int quirks;
+
+	const struct regmap_config *regmap_cfg;
+	const struct reg_field *regmap_fields;
 };
 
 struct icc_bwmon {
 	struct device *dev;
-	void __iomem *base;
+	const struct icc_bwmon_data *data;
 	int irq;
 
-	unsigned int default_lowbw_kbps;
-	unsigned int sample_ms;
+	struct regmap *regmap;
+	struct regmap_field *regs[F_NUM_FIELDS];
+
 	unsigned int max_bw_kbps;
 	unsigned int min_bw_kbps;
 	unsigned int target_kbps;
 	unsigned int current_kbps;
 };
 
-static void bwmon_clear_counters(struct icc_bwmon *bwmon)
+/* BWMON v4 */
+static const struct reg_field msm8998_bwmon_reg_fields[] = {
+	[F_GLOBAL_IRQ_CLEAR]	= REG_FIELD(BWMON_V4_GLOBAL_IRQ_CLEAR, 0, 0),
+	[F_GLOBAL_IRQ_ENABLE]	= REG_FIELD(BWMON_V4_GLOBAL_IRQ_ENABLE, 0, 0),
+	[F_IRQ_STATUS]		= REG_FIELD(BWMON_V4_IRQ_STATUS, 4, 7),
+	[F_IRQ_CLEAR]		= REG_FIELD(BWMON_V4_IRQ_CLEAR, 4, 7),
+	[F_IRQ_ENABLE]		= REG_FIELD(BWMON_V4_IRQ_ENABLE, 4, 7),
+	/* F_ENABLE covers entire register to disable other features */
+	[F_ENABLE]		= REG_FIELD(BWMON_V4_ENABLE, 0, 31),
+	[F_CLEAR]		= REG_FIELD(BWMON_V4_CLEAR, 0, 1),
+	[F_SAMPLE_WINDOW]	= REG_FIELD(BWMON_V4_SAMPLE_WINDOW, 0, 23),
+	[F_THRESHOLD_HIGH]	= REG_FIELD(BWMON_V4_THRESHOLD_HIGH, 0, 11),
+	[F_THRESHOLD_MED]	= REG_FIELD(BWMON_V4_THRESHOLD_MED, 0, 11),
+	[F_THRESHOLD_LOW]	= REG_FIELD(BWMON_V4_THRESHOLD_LOW, 0, 11),
+	[F_ZONE_ACTIONS_ZONE0]	= REG_FIELD(BWMON_V4_ZONE_ACTIONS, 0, 7),
+	[F_ZONE_ACTIONS_ZONE1]	= REG_FIELD(BWMON_V4_ZONE_ACTIONS, 8, 15),
+	[F_ZONE_ACTIONS_ZONE2]	= REG_FIELD(BWMON_V4_ZONE_ACTIONS, 16, 23),
+	[F_ZONE_ACTIONS_ZONE3]	= REG_FIELD(BWMON_V4_ZONE_ACTIONS, 24, 31),
+	[F_THRESHOLD_COUNT_ZONE0]	= REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 0, 7),
+	[F_THRESHOLD_COUNT_ZONE1]	= REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 8, 15),
+	[F_THRESHOLD_COUNT_ZONE2]	= REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 16, 23),
+	[F_THRESHOLD_COUNT_ZONE3]	= REG_FIELD(BWMON_V4_THRESHOLD_COUNT, 24, 31),
+	[F_ZONE0_MAX]		= REG_FIELD(BWMON_V4_ZONE_MAX(0), 0, 11),
+	[F_ZONE1_MAX]		= REG_FIELD(BWMON_V4_ZONE_MAX(1), 0, 11),
+	[F_ZONE2_MAX]		= REG_FIELD(BWMON_V4_ZONE_MAX(2), 0, 11),
+	[F_ZONE3_MAX]		= REG_FIELD(BWMON_V4_ZONE_MAX(3), 0, 11),
+};
+
+static const struct regmap_range msm8998_bwmon_reg_noread_ranges[] = {
+	regmap_reg_range(BWMON_V4_GLOBAL_IRQ_CLEAR, BWMON_V4_GLOBAL_IRQ_CLEAR),
+	regmap_reg_range(BWMON_V4_IRQ_CLEAR, BWMON_V4_IRQ_CLEAR),
+	regmap_reg_range(BWMON_V4_CLEAR, BWMON_V4_CLEAR),
+};
+
+static const struct regmap_access_table msm8998_bwmon_reg_read_table = {
+	.no_ranges	= msm8998_bwmon_reg_noread_ranges,
+	.n_no_ranges	= ARRAY_SIZE(msm8998_bwmon_reg_noread_ranges),
+};
+
+static const struct regmap_range msm8998_bwmon_reg_volatile_ranges[] = {
+	regmap_reg_range(BWMON_V4_IRQ_STATUS, BWMON_V4_IRQ_STATUS),
+	regmap_reg_range(BWMON_V4_ZONE_MAX(0), BWMON_V4_ZONE_MAX(3)),
+};
+
+static const struct regmap_access_table msm8998_bwmon_reg_volatile_table = {
+	.yes_ranges	= msm8998_bwmon_reg_volatile_ranges,
+	.n_yes_ranges	= ARRAY_SIZE(msm8998_bwmon_reg_volatile_ranges),
+};
+
+/*
+ * Fill the cache for non-readable registers only as rest does not really
+ * matter and can be read from the device.
+ */
+static const struct reg_default msm8998_bwmon_reg_defaults[] = {
+	{ BWMON_V4_GLOBAL_IRQ_CLEAR, 0x0 },
+	{ BWMON_V4_IRQ_CLEAR, 0x0 },
+	{ BWMON_V4_CLEAR, 0x0 },
+};
+
+static const struct regmap_config msm8998_bwmon_regmap_cfg = {
+	.reg_bits		= 32,
+	.reg_stride		= 4,
+	.val_bits		= 32,
+	/*
+	 * No concurrent access expected - driver has one interrupt handler,
+	 * regmap is not shared, no driver or user-space API.
+	 */
+	.disable_locking	= true,
+	.rd_table		= &msm8998_bwmon_reg_read_table,
+	.volatile_table		= &msm8998_bwmon_reg_volatile_table,
+	.reg_defaults		= msm8998_bwmon_reg_defaults,
+	.num_reg_defaults	= ARRAY_SIZE(msm8998_bwmon_reg_defaults),
+	/*
+	 * Cache is necessary for using regmap fields with non-readable
+	 * registers.
+	 */
+	.cache_type		= REGCACHE_RBTREE,
+};
+
+/* BWMON v5 */
+static const struct reg_field sdm845_llcc_bwmon_reg_fields[] = {
+	[F_GLOBAL_IRQ_CLEAR]	= {},
+	[F_GLOBAL_IRQ_ENABLE]	= {},
+	[F_IRQ_STATUS]		= REG_FIELD(BWMON_V5_IRQ_STATUS, 0, 3),
+	[F_IRQ_CLEAR]		= REG_FIELD(BWMON_V5_IRQ_CLEAR, 0, 3),
+	[F_IRQ_ENABLE]		= REG_FIELD(BWMON_V5_IRQ_ENABLE, 0, 3),
+	/* F_ENABLE covers entire register to disable other features */
+	[F_ENABLE]		= REG_FIELD(BWMON_V5_ENABLE, 0, 31),
+	[F_CLEAR]		= REG_FIELD(BWMON_V5_CLEAR, 0, 1),
+	[F_SAMPLE_WINDOW]	= REG_FIELD(BWMON_V5_SAMPLE_WINDOW, 0, 19),
+	[F_THRESHOLD_HIGH]	= REG_FIELD(BWMON_V5_THRESHOLD_HIGH, 0, 11),
+	[F_THRESHOLD_MED]	= REG_FIELD(BWMON_V5_THRESHOLD_MED, 0, 11),
+	[F_THRESHOLD_LOW]	= REG_FIELD(BWMON_V5_THRESHOLD_LOW, 0, 11),
+	[F_ZONE_ACTIONS_ZONE0]	= REG_FIELD(BWMON_V5_ZONE_ACTIONS, 0, 7),
+	[F_ZONE_ACTIONS_ZONE1]	= REG_FIELD(BWMON_V5_ZONE_ACTIONS, 8, 15),
+	[F_ZONE_ACTIONS_ZONE2]	= REG_FIELD(BWMON_V5_ZONE_ACTIONS, 16, 23),
+	[F_ZONE_ACTIONS_ZONE3]	= REG_FIELD(BWMON_V5_ZONE_ACTIONS, 24, 31),
+	[F_THRESHOLD_COUNT_ZONE0]	= REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 0, 7),
+	[F_THRESHOLD_COUNT_ZONE1]	= REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 8, 15),
+	[F_THRESHOLD_COUNT_ZONE2]	= REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 16, 23),
+	[F_THRESHOLD_COUNT_ZONE3]	= REG_FIELD(BWMON_V5_THRESHOLD_COUNT, 24, 31),
+	[F_ZONE0_MAX]		= REG_FIELD(BWMON_V5_ZONE_MAX(0), 0, 11),
+	[F_ZONE1_MAX]		= REG_FIELD(BWMON_V5_ZONE_MAX(1), 0, 11),
+	[F_ZONE2_MAX]		= REG_FIELD(BWMON_V5_ZONE_MAX(2), 0, 11),
+	[F_ZONE3_MAX]		= REG_FIELD(BWMON_V5_ZONE_MAX(3), 0, 11),
+};
+
+static const struct regmap_range sdm845_llcc_bwmon_reg_noread_ranges[] = {
+	regmap_reg_range(BWMON_V5_IRQ_CLEAR, BWMON_V5_IRQ_CLEAR),
+	regmap_reg_range(BWMON_V5_CLEAR, BWMON_V5_CLEAR),
+};
+
+static const struct regmap_access_table sdm845_llcc_bwmon_reg_read_table = {
+	.no_ranges	= sdm845_llcc_bwmon_reg_noread_ranges,
+	.n_no_ranges	= ARRAY_SIZE(sdm845_llcc_bwmon_reg_noread_ranges),
+};
+
+static const struct regmap_range sdm845_llcc_bwmon_reg_volatile_ranges[] = {
+	regmap_reg_range(BWMON_V5_IRQ_STATUS, BWMON_V5_IRQ_STATUS),
+	regmap_reg_range(BWMON_V5_ZONE_MAX(0), BWMON_V5_ZONE_MAX(3)),
+};
+
+static const struct regmap_access_table sdm845_llcc_bwmon_reg_volatile_table = {
+	.yes_ranges	= sdm845_llcc_bwmon_reg_volatile_ranges,
+	.n_yes_ranges	= ARRAY_SIZE(sdm845_llcc_bwmon_reg_volatile_ranges),
+};
+
+/*
+ * Fill the cache for non-readable registers only as rest does not really
+ * matter and can be read from the device.
+ */
+static const struct reg_default sdm845_llcc_bwmon_reg_defaults[] = {
+	{ BWMON_V5_IRQ_CLEAR, 0x0 },
+	{ BWMON_V5_CLEAR, 0x0 },
+};
+
+static const struct regmap_config sdm845_llcc_bwmon_regmap_cfg = {
+	.reg_bits		= 32,
+	.reg_stride		= 4,
+	.val_bits		= 32,
+	/*
+	 * No concurrent access expected - driver has one interrupt handler,
+	 * regmap is not shared, no driver or user-space API.
+	 */
+	.disable_locking	= true,
+	.rd_table		= &sdm845_llcc_bwmon_reg_read_table,
+	.volatile_table		= &sdm845_llcc_bwmon_reg_volatile_table,
+	.reg_defaults		= sdm845_llcc_bwmon_reg_defaults,
+	.num_reg_defaults	= ARRAY_SIZE(sdm845_llcc_bwmon_reg_defaults),
+	/*
+	 * Cache is necessary for using regmap fields with non-readable
+	 * registers.
+	 */
+	.cache_type		= REGCACHE_RBTREE,
+};
+
+static void bwmon_clear_counters(struct icc_bwmon *bwmon, bool clear_all)
 {
+	unsigned int val = BWMON_CLEAR_CLEAR;
+
+	if (clear_all)
+		val |= BWMON_CLEAR_CLEAR_ALL;
 	/*
 	 * Clear counters. The order and barriers are
 	 * important. Quoting downstream Qualcomm msm-4.9 tree:
@@ -140,7 +343,9 @@ static void bwmon_clear_counters(struct icc_bwmon *bwmon)
 	 * region. So, we need to make sure the counter clear is completed
 	 * before we try to clear the IRQ or do any other counter operations.
 	 */
-	writel(BWMON_CLEAR_CLEAR, bwmon->base + BWMON_CLEAR);
+	regmap_field_force_write(bwmon->regs[F_CLEAR], val);
+	if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR)
+		regmap_field_force_write(bwmon->regs[F_CLEAR], 0);
 }
 
 static void bwmon_clear_irq(struct icc_bwmon *bwmon)
@@ -161,76 +366,91 @@ static void bwmon_clear_irq(struct icc_bwmon *bwmon)
 	 * clearing here so that local writes don't happen before the
 	 * interrupt is cleared.
 	 */
-	writel(BWMON_IRQ_ENABLE_MASK, bwmon->base + BWMON_IRQ_CLEAR);
-	writel(BIT(0), bwmon->base + BWMON_GLOBAL_IRQ_CLEAR);
+	regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], BWMON_IRQ_ENABLE_MASK);
+	if (bwmon->data->quirks & BWMON_NEEDS_FORCE_CLEAR)
+		regmap_field_force_write(bwmon->regs[F_IRQ_CLEAR], 0);
+	if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)
+		regmap_field_force_write(bwmon->regs[F_GLOBAL_IRQ_CLEAR],
+					 BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE);
 }
 
 static void bwmon_disable(struct icc_bwmon *bwmon)
 {
 	/* Disable interrupts. Strict ordering, see bwmon_clear_irq(). */
-	writel(0x0, bwmon->base + BWMON_GLOBAL_IRQ_ENABLE);
-	writel(0x0, bwmon->base + BWMON_IRQ_ENABLE);
+	if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)
+		regmap_field_write(bwmon->regs[F_GLOBAL_IRQ_ENABLE], 0x0);
+	regmap_field_write(bwmon->regs[F_IRQ_ENABLE], 0x0);
 
 	/*
 	 * Disable bwmon. Must happen before bwmon_clear_irq() to avoid spurious
 	 * IRQ.
 	 */
-	writel(0x0, bwmon->base + BWMON_ENABLE);
+	regmap_field_write(bwmon->regs[F_ENABLE], 0x0);
 }
 
 static void bwmon_enable(struct icc_bwmon *bwmon, unsigned int irq_enable)
 {
 	/* Enable interrupts */
-	writel(BWMON_GLOBAL_IRQ_ENABLE_ENABLE,
-	       bwmon->base + BWMON_GLOBAL_IRQ_ENABLE);
-	writel(irq_enable, bwmon->base + BWMON_IRQ_ENABLE);
+	if (bwmon->data->quirks & BWMON_HAS_GLOBAL_IRQ)
+		regmap_field_write(bwmon->regs[F_GLOBAL_IRQ_ENABLE],
+				   BWMON_V4_GLOBAL_IRQ_ENABLE_ENABLE);
+	regmap_field_write(bwmon->regs[F_IRQ_ENABLE], irq_enable);
 
 	/* Enable bwmon */
-	writel(BWMON_ENABLE_ENABLE, bwmon->base + BWMON_ENABLE);
+	regmap_field_write(bwmon->regs[F_ENABLE], BWMON_ENABLE_ENABLE);
 }
 
-static unsigned int bwmon_kbps_to_count(unsigned int kbps)
+static unsigned int bwmon_kbps_to_count(struct icc_bwmon *bwmon,
+					unsigned int kbps)
 {
-	return kbps / BWMON_COUNT_UNIT_KB;
+	return kbps / bwmon->data->count_unit_kb;
 }
 
-static void bwmon_set_threshold(struct icc_bwmon *bwmon, unsigned int reg,
-				unsigned int kbps)
+static void bwmon_set_threshold(struct icc_bwmon *bwmon,
+				struct regmap_field *reg, unsigned int kbps)
 {
 	unsigned int thres;
 
-	thres = mult_frac(bwmon_kbps_to_count(kbps), bwmon->sample_ms,
-			  MSEC_PER_SEC);
-	writel_relaxed(thres, bwmon->base + reg);
+	thres = mult_frac(bwmon_kbps_to_count(bwmon, kbps),
+			  bwmon->data->sample_ms, MSEC_PER_SEC);
+	regmap_field_write(reg, thres);
 }
 
-static void bwmon_start(struct icc_bwmon *bwmon,
-			const struct icc_bwmon_data *data)
+static void bwmon_start(struct icc_bwmon *bwmon)
 {
-	unsigned int thres_count;
+	const struct icc_bwmon_data *data = bwmon->data;
 	int window;
 
-	bwmon_clear_counters(bwmon);
+	bwmon_clear_counters(bwmon, true);
 
-	window = mult_frac(bwmon->sample_ms, HW_TIMER_HZ, MSEC_PER_SEC);
-	/* Maximum sampling window: 0xfffff */
-	writel_relaxed(window, bwmon->base + BWMON_SAMPLE_WINDOW);
+	window = mult_frac(bwmon->data->sample_ms, HW_TIMER_HZ, MSEC_PER_SEC);
+	/* Maximum sampling window: 0xffffff for v4 and 0xfffff for v5 */
+	regmap_field_write(bwmon->regs[F_SAMPLE_WINDOW], window);
 
-	bwmon_set_threshold(bwmon, BWMON_THRESHOLD_HIGH,
+	bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH],
 			    data->default_highbw_kbps);
-	bwmon_set_threshold(bwmon, BWMON_THRESHOLD_MED,
+	bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED],
 			    data->default_medbw_kbps);
-	bwmon_set_threshold(bwmon, BWMON_THRESHOLD_LOW,
+	bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_LOW],
 			    data->default_lowbw_kbps);
 
-	thres_count = data->zone3_thres_count << BWMON_THRESHOLD_COUNT_ZONE3_SHIFT |
-		      BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT << BWMON_THRESHOLD_COUNT_ZONE2_SHIFT |
-		      data->zone1_thres_count << BWMON_THRESHOLD_COUNT_ZONE1_SHIFT |
-		      BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT;
-	writel_relaxed(thres_count, bwmon->base + BWMON_THRESHOLD_COUNT);
-	writel_relaxed(BWMON_ZONE_ACTIONS_DEFAULT,
-		       bwmon->base + BWMON_ZONE_ACTIONS);
-	/* Write barriers in bwmon_clear_irq() */
+	regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE0],
+			   BWMON_THRESHOLD_COUNT_ZONE0_DEFAULT);
+	regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE1],
+			   data->zone1_thres_count);
+	regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE2],
+			   BWMON_THRESHOLD_COUNT_ZONE2_DEFAULT);
+	regmap_field_write(bwmon->regs[F_THRESHOLD_COUNT_ZONE3],
+			   data->zone3_thres_count);
+
+	regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE0],
+			   BWMON_ZONE_ACTIONS_ZONE0);
+	regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE1],
+			   BWMON_ZONE_ACTIONS_ZONE1);
+	regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE2],
+			   BWMON_ZONE_ACTIONS_ZONE2);
+	regmap_field_write(bwmon->regs[F_ZONE_ACTIONS_ZONE3],
+			   BWMON_ZONE_ACTIONS_ZONE3);
 
 	bwmon_clear_irq(bwmon);
 	bwmon_enable(bwmon, BWMON_IRQ_ENABLE_MASK);
@@ -242,7 +462,9 @@ static irqreturn_t bwmon_intr(int irq, void *dev_id)
 	unsigned int status, max;
 	int zone;
 
-	status = readl(bwmon->base + BWMON_IRQ_STATUS);
+	if (regmap_field_read(bwmon->regs[F_IRQ_STATUS], &status))
+		return IRQ_NONE;
+
 	status &= BWMON_IRQ_ENABLE_MASK;
 	if (!status) {
 		/*
@@ -259,15 +481,18 @@ static irqreturn_t bwmon_intr(int irq, void *dev_id)
 
 	bwmon_disable(bwmon);
 
-	zone = get_bitmask_order(status >> BWMON_IRQ_STATUS_ZONE_SHIFT) - 1;
+	zone = get_bitmask_order(status) - 1;
 	/*
 	 * Zone max bytes count register returns count units within sampling
 	 * window.  Downstream kernel for BWMONv4 (called BWMON type 2 in
 	 * downstream) always increments the max bytes count by one.
 	 */
-	max = readl(bwmon->base + BWMON_ZONE_MAX(zone)) + 1;
-	max *= BWMON_COUNT_UNIT_KB;
-	bwmon->target_kbps = mult_frac(max, MSEC_PER_SEC, bwmon->sample_ms);
+	if (regmap_field_read(bwmon->regs[F_ZONE0_MAX + zone], &max))
+		return IRQ_NONE;
+
+	max += 1;
+	max *= bwmon->data->count_unit_kb;
+	bwmon->target_kbps = mult_frac(max, MSEC_PER_SEC, bwmon->data->sample_ms);
 
 	return IRQ_WAKE_THREAD;
 }
@@ -297,16 +522,17 @@ static irqreturn_t bwmon_intr_thread(int irq, void *dev_id)
 	up_kbps = bwmon->target_kbps + 1;
 
 	if (bwmon->target_kbps >= bwmon->max_bw_kbps)
-		irq_enable = BIT(BWMON_IRQ_ENABLE_ZONE1_SHIFT);
+		irq_enable = BIT(1);
 	else if (bwmon->target_kbps <= bwmon->min_bw_kbps)
-		irq_enable = BIT(BWMON_IRQ_ENABLE_ZONE3_SHIFT);
+		irq_enable = BIT(3);
 	else
 		irq_enable = BWMON_IRQ_ENABLE_MASK;
 
-	bwmon_set_threshold(bwmon, BWMON_THRESHOLD_HIGH, up_kbps);
-	bwmon_set_threshold(bwmon, BWMON_THRESHOLD_MED, down_kbps);
-	/* Write barriers in bwmon_clear_counters() */
-	bwmon_clear_counters(bwmon);
+	bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_HIGH],
+			    up_kbps);
+	bwmon_set_threshold(bwmon, bwmon->regs[F_THRESHOLD_MED],
+			    down_kbps);
+	bwmon_clear_counters(bwmon, false);
 	bwmon_clear_irq(bwmon);
 	bwmon_enable(bwmon, irq_enable);
 
@@ -324,25 +550,47 @@ out:
 	return IRQ_HANDLED;
 }
 
+static int bwmon_init_regmap(struct platform_device *pdev,
+			     struct icc_bwmon *bwmon)
+{
+	struct device *dev = &pdev->dev;
+	void __iomem *base;
+	struct regmap *map;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return dev_err_probe(dev, PTR_ERR(base),
+				     "failed to map bwmon registers\n");
+
+	map = devm_regmap_init_mmio(dev, base, bwmon->data->regmap_cfg);
+	if (IS_ERR(map))
+		return dev_err_probe(dev, PTR_ERR(map),
+				     "failed to initialize regmap\n");
+
+	BUILD_BUG_ON(ARRAY_SIZE(msm8998_bwmon_reg_fields) != F_NUM_FIELDS);
+	BUILD_BUG_ON(ARRAY_SIZE(sdm845_llcc_bwmon_reg_fields) != F_NUM_FIELDS);
+
+	return devm_regmap_field_bulk_alloc(dev, map, bwmon->regs,
+					   bwmon->data->regmap_fields,
+					   F_NUM_FIELDS);
+}
+
 static int bwmon_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct dev_pm_opp *opp;
 	struct icc_bwmon *bwmon;
-	const struct icc_bwmon_data *data;
 	int ret;
 
 	bwmon = devm_kzalloc(dev, sizeof(*bwmon), GFP_KERNEL);
 	if (!bwmon)
 		return -ENOMEM;
 
-	data = of_device_get_match_data(dev);
+	bwmon->data = of_device_get_match_data(dev);
 
-	bwmon->base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(bwmon->base)) {
-		dev_err(dev, "failed to map bwmon registers\n");
-		return PTR_ERR(bwmon->base);
-	}
+	ret = bwmon_init_regmap(pdev, bwmon);
+	if (ret)
+		return ret;
 
 	bwmon->irq = platform_get_irq(pdev, 0);
 	if (bwmon->irq < 0)
@@ -362,8 +610,6 @@ static int bwmon_probe(struct platform_device *pdev)
 	if (IS_ERR(opp))
 		return dev_err_probe(dev, ret, "failed to find min peak bandwidth\n");
 
-	bwmon->sample_ms = data->sample_ms;
-	bwmon->default_lowbw_kbps = data->default_lowbw_kbps;
 	bwmon->dev = dev;
 
 	bwmon_disable(bwmon);
@@ -374,7 +620,7 @@ static int bwmon_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, ret, "failed to request IRQ\n");
 
 	platform_set_drvdata(pdev, bwmon);
-	bwmon_start(bwmon, data);
+	bwmon_start(bwmon);
 
 	return 0;
 }
@@ -388,18 +634,55 @@ static int bwmon_remove(struct platform_device *pdev)
 	return 0;
 }
 
-/* BWMON v4 */
 static const struct icc_bwmon_data msm8998_bwmon_data = {
 	.sample_ms = 4,
+	.count_unit_kb = 64,
 	.default_highbw_kbps = 4800 * 1024, /* 4.8 GBps */
 	.default_medbw_kbps = 512 * 1024, /* 512 MBps */
 	.default_lowbw_kbps = 0,
 	.zone1_thres_count = 16,
 	.zone3_thres_count = 1,
+	.quirks = BWMON_HAS_GLOBAL_IRQ,
+	.regmap_fields = msm8998_bwmon_reg_fields,
+	.regmap_cfg = &msm8998_bwmon_regmap_cfg,
+};
+
+static const struct icc_bwmon_data sdm845_llcc_bwmon_data = {
+	.sample_ms = 4,
+	.count_unit_kb = 1024,
+	.default_highbw_kbps = 800 * 1024, /* 800 MBps */
+	.default_medbw_kbps = 256 * 1024, /* 256 MBps */
+	.default_lowbw_kbps = 0,
+	.zone1_thres_count = 16,
+	.zone3_thres_count = 1,
+	.regmap_fields = sdm845_llcc_bwmon_reg_fields,
+	.regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg,
+};
+
+static const struct icc_bwmon_data sc7280_llcc_bwmon_data = {
+	.sample_ms = 4,
+	.count_unit_kb = 64,
+	.default_highbw_kbps = 800 * 1024, /* 800 MBps */
+	.default_medbw_kbps = 256 * 1024, /* 256 MBps */
+	.default_lowbw_kbps = 0,
+	.zone1_thres_count = 16,
+	.zone3_thres_count = 1,
+	.quirks = BWMON_NEEDS_FORCE_CLEAR,
+	.regmap_fields = sdm845_llcc_bwmon_reg_fields,
+	.regmap_cfg = &sdm845_llcc_bwmon_regmap_cfg,
 };
 
 static const struct of_device_id bwmon_of_match[] = {
-	{ .compatible = "qcom,msm8998-bwmon", .data = &msm8998_bwmon_data },
+	{
+		.compatible = "qcom,msm8998-bwmon",
+		.data = &msm8998_bwmon_data
+	}, {
+		.compatible = "qcom,sdm845-llcc-bwmon",
+		.data = &sdm845_llcc_bwmon_data
+	}, {
+		.compatible = "qcom,sc7280-llcc-bwmon",
+		.data = &sc7280_llcc_bwmon_data
+	},
 	{}
 };
 MODULE_DEVICE_TABLE(of, bwmon_of_match);
diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c
index 38d7296315a2..8b7e8118f3ce 100644
--- a/drivers/soc/qcom/llcc-qcom.c
+++ b/drivers/soc/qcom/llcc-qcom.c
@@ -104,6 +104,7 @@ struct qcom_llcc_config {
 	int size;
 	bool need_llcc_cfg;
 	const u32 *reg_offset;
+	const struct llcc_edac_reg_offset *edac_reg_offset;
 };
 
 enum llcc_reg_offset {
@@ -296,12 +297,68 @@ static const struct llcc_slice_config sm8450_data[] =  {
 	{LLCC_AENPU,     8, 2048, 1, 1, 0xFFFF, 0x0,   0, 0, 0, 0, 0, 0, 0 },
 };
 
-static const u32 llcc_v1_2_reg_offset[] = {
+static const struct llcc_edac_reg_offset llcc_v1_edac_reg_offset = {
+	.trp_ecc_error_status0 = 0x20344,
+	.trp_ecc_error_status1 = 0x20348,
+	.trp_ecc_sb_err_syn0 = 0x2304c,
+	.trp_ecc_db_err_syn0 = 0x20370,
+	.trp_ecc_error_cntr_clear = 0x20440,
+	.trp_interrupt_0_status = 0x20480,
+	.trp_interrupt_0_clear = 0x20484,
+	.trp_interrupt_0_enable = 0x20488,
+
+	/* LLCC Common registers */
+	.cmn_status0 = 0x3000c,
+	.cmn_interrupt_0_enable = 0x3001c,
+	.cmn_interrupt_2_enable = 0x3003c,
+
+	/* LLCC DRP registers */
+	.drp_ecc_error_cfg = 0x40000,
+	.drp_ecc_error_cntr_clear = 0x40004,
+	.drp_interrupt_status = 0x41000,
+	.drp_interrupt_clear = 0x41008,
+	.drp_interrupt_enable = 0x4100c,
+	.drp_ecc_error_status0 = 0x42044,
+	.drp_ecc_error_status1 = 0x42048,
+	.drp_ecc_sb_err_syn0 = 0x4204c,
+	.drp_ecc_db_err_syn0 = 0x42070,
+};
+
+static const struct llcc_edac_reg_offset llcc_v2_1_edac_reg_offset = {
+	.trp_ecc_error_status0 = 0x20344,
+	.trp_ecc_error_status1 = 0x20348,
+	.trp_ecc_sb_err_syn0 = 0x2034c,
+	.trp_ecc_db_err_syn0 = 0x20370,
+	.trp_ecc_error_cntr_clear = 0x20440,
+	.trp_interrupt_0_status = 0x20480,
+	.trp_interrupt_0_clear = 0x20484,
+	.trp_interrupt_0_enable = 0x20488,
+
+	/* LLCC Common registers */
+	.cmn_status0 = 0x3400c,
+	.cmn_interrupt_0_enable = 0x3401c,
+	.cmn_interrupt_2_enable = 0x3403c,
+
+	/* LLCC DRP registers */
+	.drp_ecc_error_cfg = 0x50000,
+	.drp_ecc_error_cntr_clear = 0x50004,
+	.drp_interrupt_status = 0x50020,
+	.drp_interrupt_clear = 0x50028,
+	.drp_interrupt_enable = 0x5002c,
+	.drp_ecc_error_status0 = 0x520f4,
+	.drp_ecc_error_status1 = 0x520f8,
+	.drp_ecc_sb_err_syn0 = 0x520fc,
+	.drp_ecc_db_err_syn0 = 0x52120,
+};
+
+/* LLCC register offset starting from v1.0.0 */
+static const u32 llcc_v1_reg_offset[] = {
 	[LLCC_COMMON_HW_INFO]	= 0x00030000,
 	[LLCC_COMMON_STATUS0]	= 0x0003000c,
 };
 
-static const u32 llcc_v21_reg_offset[] = {
+/* LLCC register offset starting from v2.0.1 */
+static const u32 llcc_v2_1_reg_offset[] = {
 	[LLCC_COMMON_HW_INFO]	= 0x00034000,
 	[LLCC_COMMON_STATUS0]	= 0x0003400c,
 };
@@ -310,70 +367,80 @@ static const struct qcom_llcc_config sc7180_cfg = {
 	.sct_data	= sc7180_data,
 	.size		= ARRAY_SIZE(sc7180_data),
 	.need_llcc_cfg	= true,
-	.reg_offset	= llcc_v1_2_reg_offset,
+	.reg_offset	= llcc_v1_reg_offset,
+	.edac_reg_offset = &llcc_v1_edac_reg_offset,
 };
 
 static const struct qcom_llcc_config sc7280_cfg = {
 	.sct_data	= sc7280_data,
 	.size		= ARRAY_SIZE(sc7280_data),
 	.need_llcc_cfg	= true,
-	.reg_offset	= llcc_v1_2_reg_offset,
+	.reg_offset	= llcc_v1_reg_offset,
+	.edac_reg_offset = &llcc_v1_edac_reg_offset,
 };
 
 static const struct qcom_llcc_config sc8180x_cfg = {
 	.sct_data	= sc8180x_data,
 	.size		= ARRAY_SIZE(sc8180x_data),
 	.need_llcc_cfg	= true,
-	.reg_offset	= llcc_v1_2_reg_offset,
+	.reg_offset	= llcc_v1_reg_offset,
+	.edac_reg_offset = &llcc_v1_edac_reg_offset,
 };
 
 static const struct qcom_llcc_config sc8280xp_cfg = {
 	.sct_data	= sc8280xp_data,
 	.size		= ARRAY_SIZE(sc8280xp_data),
 	.need_llcc_cfg	= true,
-	.reg_offset	= llcc_v1_2_reg_offset,
+	.reg_offset	= llcc_v1_reg_offset,
+	.edac_reg_offset = &llcc_v1_edac_reg_offset,
 };
 
 static const struct qcom_llcc_config sdm845_cfg = {
 	.sct_data	= sdm845_data,
 	.size		= ARRAY_SIZE(sdm845_data),
 	.need_llcc_cfg	= false,
-	.reg_offset	= llcc_v1_2_reg_offset,
+	.reg_offset	= llcc_v1_reg_offset,
+	.edac_reg_offset = &llcc_v1_edac_reg_offset,
 };
 
 static const struct qcom_llcc_config sm6350_cfg = {
 	.sct_data	= sm6350_data,
 	.size		= ARRAY_SIZE(sm6350_data),
 	.need_llcc_cfg	= true,
-	.reg_offset	= llcc_v1_2_reg_offset,
+	.reg_offset	= llcc_v1_reg_offset,
+	.edac_reg_offset = &llcc_v1_edac_reg_offset,
 };
 
 static const struct qcom_llcc_config sm8150_cfg = {
 	.sct_data       = sm8150_data,
 	.size           = ARRAY_SIZE(sm8150_data),
 	.need_llcc_cfg	= true,
-	.reg_offset	= llcc_v1_2_reg_offset,
+	.reg_offset	= llcc_v1_reg_offset,
+	.edac_reg_offset = &llcc_v1_edac_reg_offset,
 };
 
 static const struct qcom_llcc_config sm8250_cfg = {
 	.sct_data       = sm8250_data,
 	.size           = ARRAY_SIZE(sm8250_data),
 	.need_llcc_cfg	= true,
-	.reg_offset	= llcc_v1_2_reg_offset,
+	.reg_offset	= llcc_v1_reg_offset,
+	.edac_reg_offset = &llcc_v1_edac_reg_offset,
 };
 
 static const struct qcom_llcc_config sm8350_cfg = {
 	.sct_data       = sm8350_data,
 	.size           = ARRAY_SIZE(sm8350_data),
 	.need_llcc_cfg	= true,
-	.reg_offset	= llcc_v1_2_reg_offset,
+	.reg_offset	= llcc_v1_reg_offset,
+	.edac_reg_offset = &llcc_v1_edac_reg_offset,
 };
 
 static const struct qcom_llcc_config sm8450_cfg = {
 	.sct_data       = sm8450_data,
 	.size           = ARRAY_SIZE(sm8450_data),
 	.need_llcc_cfg	= true,
-	.reg_offset	= llcc_v21_reg_offset,
+	.reg_offset	= llcc_v2_1_reg_offset,
+	.edac_reg_offset = &llcc_v2_1_edac_reg_offset,
 };
 
 static struct llcc_drv_data *drv_data = (void *) -EPROBE_DEFER;
@@ -774,6 +841,7 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 
 	drv_data->cfg = llcc_cfg;
 	drv_data->cfg_size = sz;
+	drv_data->edac_reg_offset = cfg->edac_reg_offset;
 	mutex_init(&drv_data->lock);
 	platform_set_drvdata(pdev, drv_data);
 
diff --git a/drivers/soc/qcom/qcom_stats.c b/drivers/soc/qcom/qcom_stats.c
index d6bfd1bbdc2a..121ea409fafc 100644
--- a/drivers/soc/qcom/qcom_stats.c
+++ b/drivers/soc/qcom/qcom_stats.c
@@ -246,6 +246,14 @@ static const struct stats_config rpm_data_dba0 = {
 	.subsystem_stats_in_smem = false,
 };
 
+static const struct stats_config rpmh_data_sdm845 = {
+	.stats_offset = 0x48,
+	.num_records = 2,
+	.appended_stats_avail = false,
+	.dynamic_offset = false,
+	.subsystem_stats_in_smem = true,
+};
+
 static const struct stats_config rpmh_data = {
 	.stats_offset = 0x48,
 	.num_records = 3,
@@ -261,6 +269,7 @@ static const struct of_device_id qcom_stats_table[] = {
 	{ .compatible = "qcom,msm8974-rpm-stats", .data = &rpm_data_dba0 },
 	{ .compatible = "qcom,rpm-stats", .data = &rpm_data },
 	{ .compatible = "qcom,rpmh-stats", .data = &rpmh_data },
+	{ .compatible = "qcom,sdm845-rpmh-stats", .data = &rpmh_data_sdm845 },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, qcom_stats_table);
diff --git a/drivers/soc/qcom/qmi_encdec.c b/drivers/soc/qcom/qmi_encdec.c
index 328cc8237191..b7158e3c3a0b 100644
--- a/drivers/soc/qcom/qmi_encdec.c
+++ b/drivers/soc/qcom/qmi_encdec.c
@@ -57,11 +57,11 @@ do { \
 #define TLV_TYPE_SIZE sizeof(u8)
 #define OPTIONAL_TLV_TYPE_START 0x10
 
-static int qmi_encode(struct qmi_elem_info *ei_array, void *out_buf,
+static int qmi_encode(const struct qmi_elem_info *ei_array, void *out_buf,
 		      const void *in_c_struct, u32 out_buf_len,
 		      int enc_level);
 
-static int qmi_decode(struct qmi_elem_info *ei_array, void *out_c_struct,
+static int qmi_decode(const struct qmi_elem_info *ei_array, void *out_c_struct,
 		      const void *in_buf, u32 in_buf_len, int dec_level);
 
 /**
@@ -76,10 +76,10 @@ static int qmi_decode(struct qmi_elem_info *ei_array, void *out_c_struct,
  *
  * Return: struct info of the next element that can be encoded.
  */
-static struct qmi_elem_info *skip_to_next_elem(struct qmi_elem_info *ei_array,
-					       int level)
+static const struct qmi_elem_info *
+skip_to_next_elem(const struct qmi_elem_info *ei_array, int level)
 {
-	struct qmi_elem_info *temp_ei = ei_array;
+	const struct qmi_elem_info *temp_ei = ei_array;
 	u8 tlv_type;
 
 	if (level > 1) {
@@ -101,11 +101,11 @@ static struct qmi_elem_info *skip_to_next_elem(struct qmi_elem_info *ei_array,
  *
  * Return: Expected minimum length of the QMI message or 0 on error.
  */
-static int qmi_calc_min_msg_len(struct qmi_elem_info *ei_array,
+static int qmi_calc_min_msg_len(const struct qmi_elem_info *ei_array,
 				int level)
 {
 	int min_msg_len = 0;
-	struct qmi_elem_info *temp_ei = ei_array;
+	const struct qmi_elem_info *temp_ei = ei_array;
 
 	if (!ei_array)
 		return min_msg_len;
@@ -194,13 +194,13 @@ static int qmi_encode_basic_elem(void *buf_dst, const void *buf_src,
  * Return: The number of bytes of encoded information on success or negative
  * errno on error.
  */
-static int qmi_encode_struct_elem(struct qmi_elem_info *ei_array,
+static int qmi_encode_struct_elem(const struct qmi_elem_info *ei_array,
 				  void *buf_dst, const void *buf_src,
 				  u32 elem_len, u32 out_buf_len,
 				  int enc_level)
 {
 	int i, rc, encoded_bytes = 0;
-	struct qmi_elem_info *temp_ei = ei_array;
+	const struct qmi_elem_info *temp_ei = ei_array;
 
 	for (i = 0; i < elem_len; i++) {
 		rc = qmi_encode(temp_ei->ei_array, buf_dst, buf_src,
@@ -233,13 +233,13 @@ static int qmi_encode_struct_elem(struct qmi_elem_info *ei_array,
  * Return: The number of bytes of encoded information on success or negative
  * errno on error.
  */
-static int qmi_encode_string_elem(struct qmi_elem_info *ei_array,
+static int qmi_encode_string_elem(const struct qmi_elem_info *ei_array,
 				  void *buf_dst, const void *buf_src,
 				  u32 out_buf_len, int enc_level)
 {
 	int rc;
 	int encoded_bytes = 0;
-	struct qmi_elem_info *temp_ei = ei_array;
+	const struct qmi_elem_info *temp_ei = ei_array;
 	u32 string_len = 0;
 	u32 string_len_sz = 0;
 
@@ -289,11 +289,11 @@ static int qmi_encode_string_elem(struct qmi_elem_info *ei_array,
  * Return: The number of bytes of encoded information on success or negative
  * errno on error.
  */
-static int qmi_encode(struct qmi_elem_info *ei_array, void *out_buf,
+static int qmi_encode(const struct qmi_elem_info *ei_array, void *out_buf,
 		      const void *in_c_struct, u32 out_buf_len,
 		      int enc_level)
 {
-	struct qmi_elem_info *temp_ei = ei_array;
+	const struct qmi_elem_info *temp_ei = ei_array;
 	u8 opt_flag_value = 0;
 	u32 data_len_value = 0, data_len_sz;
 	u8 *buf_dst = (u8 *)out_buf;
@@ -468,13 +468,13 @@ static int qmi_decode_basic_elem(void *buf_dst, const void *buf_src,
  * Return: The total size of the decoded data elements on success, negative
  * errno on error.
  */
-static int qmi_decode_struct_elem(struct qmi_elem_info *ei_array,
+static int qmi_decode_struct_elem(const struct qmi_elem_info *ei_array,
 				  void *buf_dst, const void *buf_src,
 				  u32 elem_len, u32 tlv_len,
 				  int dec_level)
 {
 	int i, rc, decoded_bytes = 0;
-	struct qmi_elem_info *temp_ei = ei_array;
+	const struct qmi_elem_info *temp_ei = ei_array;
 
 	for (i = 0; i < elem_len && decoded_bytes < tlv_len; i++) {
 		rc = qmi_decode(temp_ei->ei_array, buf_dst, buf_src,
@@ -514,7 +514,7 @@ static int qmi_decode_struct_elem(struct qmi_elem_info *ei_array,
  * Return: The total size of the decoded data elements on success, negative
  * errno on error.
  */
-static int qmi_decode_string_elem(struct qmi_elem_info *ei_array,
+static int qmi_decode_string_elem(const struct qmi_elem_info *ei_array,
 				  void *buf_dst, const void *buf_src,
 				  u32 tlv_len, int dec_level)
 {
@@ -522,7 +522,7 @@ static int qmi_decode_string_elem(struct qmi_elem_info *ei_array,
 	int decoded_bytes = 0;
 	u32 string_len = 0;
 	u32 string_len_sz = 0;
-	struct qmi_elem_info *temp_ei = ei_array;
+	const struct qmi_elem_info *temp_ei = ei_array;
 
 	if (dec_level == 1) {
 		string_len = tlv_len;
@@ -564,10 +564,10 @@ static int qmi_decode_string_elem(struct qmi_elem_info *ei_array,
  *
  * Return: Pointer to struct info, if found
  */
-static struct qmi_elem_info *find_ei(struct qmi_elem_info *ei_array,
-				     u32 type)
+static const struct qmi_elem_info *find_ei(const struct qmi_elem_info *ei_array,
+					   u32 type)
 {
-	struct qmi_elem_info *temp_ei = ei_array;
+	const struct qmi_elem_info *temp_ei = ei_array;
 
 	while (temp_ei->data_type != QMI_EOTI) {
 		if (temp_ei->tlv_type == (u8)type)
@@ -590,11 +590,11 @@ static struct qmi_elem_info *find_ei(struct qmi_elem_info *ei_array,
  * Return: The number of bytes of decoded information on success, negative
  * errno on error.
  */
-static int qmi_decode(struct qmi_elem_info *ei_array, void *out_c_struct,
+static int qmi_decode(const struct qmi_elem_info *ei_array, void *out_c_struct,
 		      const void *in_buf, u32 in_buf_len,
 		      int dec_level)
 {
-	struct qmi_elem_info *temp_ei = ei_array;
+	const struct qmi_elem_info *temp_ei = ei_array;
 	u8 opt_flag_value = 1;
 	u32 data_len_value = 0, data_len_sz = 0;
 	u8 *buf_dst = out_c_struct;
@@ -713,7 +713,7 @@ static int qmi_decode(struct qmi_elem_info *ei_array, void *out_c_struct,
  * Return: Buffer with encoded message, or negative ERR_PTR() on error
  */
 void *qmi_encode_message(int type, unsigned int msg_id, size_t *len,
-			 unsigned int txn_id, struct qmi_elem_info *ei,
+			 unsigned int txn_id, const struct qmi_elem_info *ei,
 			 const void *c_struct)
 {
 	struct qmi_header *hdr;
@@ -767,7 +767,7 @@ EXPORT_SYMBOL(qmi_encode_message);
  * errno on error.
  */
 int qmi_decode_message(const void *buf, size_t len,
-		       struct qmi_elem_info *ei, void *c_struct)
+		       const struct qmi_elem_info *ei, void *c_struct)
 {
 	if (!ei)
 		return -EINVAL;
@@ -781,7 +781,7 @@ int qmi_decode_message(const void *buf, size_t len,
 EXPORT_SYMBOL(qmi_decode_message);
 
 /* Common header in all QMI responses */
-struct qmi_elem_info qmi_response_type_v01_ei[] = {
+const struct qmi_elem_info qmi_response_type_v01_ei[] = {
 	{
 		.data_type	= QMI_SIGNED_2_BYTE_ENUM,
 		.elem_len	= 1,
diff --git a/drivers/soc/qcom/qmi_interface.c b/drivers/soc/qcom/qmi_interface.c
index c8c4c730b135..57052726299d 100644
--- a/drivers/soc/qcom/qmi_interface.c
+++ b/drivers/soc/qcom/qmi_interface.c
@@ -305,7 +305,7 @@ EXPORT_SYMBOL(qmi_add_server);
  * Return: Transaction id on success, negative errno on failure.
  */
 int qmi_txn_init(struct qmi_handle *qmi, struct qmi_txn *txn,
-		 struct qmi_elem_info *ei, void *c_struct)
+		 const struct qmi_elem_info *ei, void *c_struct)
 {
 	int ret;
 
@@ -736,7 +736,8 @@ EXPORT_SYMBOL(qmi_handle_release);
 static ssize_t qmi_send_message(struct qmi_handle *qmi,
 				struct sockaddr_qrtr *sq, struct qmi_txn *txn,
 				int type, int msg_id, size_t len,
-				struct qmi_elem_info *ei, const void *c_struct)
+				const struct qmi_elem_info *ei,
+				const void *c_struct)
 {
 	struct msghdr msghdr = {};
 	struct kvec iv;
@@ -787,7 +788,7 @@ static ssize_t qmi_send_message(struct qmi_handle *qmi,
  */
 ssize_t qmi_send_request(struct qmi_handle *qmi, struct sockaddr_qrtr *sq,
 			 struct qmi_txn *txn, int msg_id, size_t len,
-			 struct qmi_elem_info *ei, const void *c_struct)
+			 const struct qmi_elem_info *ei, const void *c_struct)
 {
 	return qmi_send_message(qmi, sq, txn, QMI_REQUEST, msg_id, len, ei,
 				c_struct);
@@ -808,7 +809,7 @@ EXPORT_SYMBOL(qmi_send_request);
  */
 ssize_t qmi_send_response(struct qmi_handle *qmi, struct sockaddr_qrtr *sq,
 			  struct qmi_txn *txn, int msg_id, size_t len,
-			  struct qmi_elem_info *ei, const void *c_struct)
+			  const struct qmi_elem_info *ei, const void *c_struct)
 {
 	return qmi_send_message(qmi, sq, txn, QMI_RESPONSE, msg_id, len, ei,
 				c_struct);
@@ -827,7 +828,8 @@ EXPORT_SYMBOL(qmi_send_response);
  * Return: 0 on success, negative errno on failure.
  */
 ssize_t qmi_send_indication(struct qmi_handle *qmi, struct sockaddr_qrtr *sq,
-			    int msg_id, size_t len, struct qmi_elem_info *ei,
+			    int msg_id, size_t len,
+			    const struct qmi_elem_info *ei,
 			    const void *c_struct)
 {
 	struct qmi_txn txn;
diff --git a/drivers/soc/qcom/rpmpd.c b/drivers/soc/qcom/rpmpd.c
index 5803038c744e..337b1ad1cd3b 100644
--- a/drivers/soc/qcom/rpmpd.c
+++ b/drivers/soc/qcom/rpmpd.c
@@ -29,6 +29,7 @@
 #define RPMPD_RWLM 0x6d6c7772
 #define RPMPD_RWSC 0x63737772
 #define RPMPD_RWSM 0x6d737772
+#define RPMPD_RWGX 0x78677772
 
 /* Operation Keys */
 #define KEY_CORNER		0x6e726f63 /* corn */
@@ -433,6 +434,26 @@ static const struct rpmpd_desc sm6125_desc = {
 	.max_state = RPM_SMD_LEVEL_BINNING,
 };
 
+DEFINE_RPMPD_PAIR(sm6375, vddgx, vddgx_ao, RWGX, LEVEL, 0);
+static struct rpmpd *sm6375_rpmpds[] = {
+	[SM6375_VDDCX] = &sm6125_vddcx,
+	[SM6375_VDDCX_AO] = &sm6125_vddcx_ao,
+	[SM6375_VDDCX_VFL] = &sm6125_vddcx_vfl,
+	[SM6375_VDDMX] = &sm6125_vddmx,
+	[SM6375_VDDMX_AO] = &sm6125_vddmx_ao,
+	[SM6375_VDDMX_VFL] = &sm6125_vddmx_vfl,
+	[SM6375_VDDGX] = &sm6375_vddgx,
+	[SM6375_VDDGX_AO] = &sm6375_vddgx_ao,
+	[SM6375_VDD_LPI_CX] = &sm6115_vdd_lpi_cx,
+	[SM6375_VDD_LPI_MX] = &sm6115_vdd_lpi_mx,
+};
+
+static const struct rpmpd_desc sm6375_desc = {
+	.rpmpds = sm6375_rpmpds,
+	.num_pds = ARRAY_SIZE(sm6375_rpmpds),
+	.max_state = RPM_SMD_LEVEL_TURBO_NO_CPR,
+};
+
 static struct rpmpd *qcm2290_rpmpds[] = {
 	[QCM2290_VDDCX] = &sm6115_vddcx,
 	[QCM2290_VDDCX_AO] = &sm6115_vddcx_ao,
@@ -466,6 +487,7 @@ static const struct of_device_id rpmpd_match_table[] = {
 	{ .compatible = "qcom,sdm660-rpmpd", .data = &sdm660_desc },
 	{ .compatible = "qcom,sm6115-rpmpd", .data = &sm6115_desc },
 	{ .compatible = "qcom,sm6125-rpmpd", .data = &sm6125_desc },
+	{ .compatible = "qcom,sm6375-rpmpd", .data = &sm6375_desc },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, rpmpd_match_table);
diff --git a/drivers/soc/qcom/smem_state.c b/drivers/soc/qcom/smem_state.c
index 31faf4aa868e..e848cc9a3cf8 100644
--- a/drivers/soc/qcom/smem_state.c
+++ b/drivers/soc/qcom/smem_state.c
@@ -136,6 +136,7 @@ static void qcom_smem_state_release(struct kref *ref)
 	struct qcom_smem_state *state = container_of(ref, struct qcom_smem_state, refcount);
 
 	list_del(&state->list);
+	of_node_put(state->of_node);
 	kfree(state);
 }
 
@@ -205,7 +206,7 @@ struct qcom_smem_state *qcom_smem_state_register(struct device_node *of_node,
 
 	kref_init(&state->refcount);
 
-	state->of_node = of_node;
+	state->of_node = of_node_get(of_node);
 	state->ops = *ops;
 	state->priv = priv;
 
diff --git a/drivers/soc/qcom/smsm.c b/drivers/soc/qcom/smsm.c
index 9df9bba242f3..3e8994d6110e 100644
--- a/drivers/soc/qcom/smsm.c
+++ b/drivers/soc/qcom/smsm.c
@@ -526,7 +526,7 @@ static int qcom_smsm_probe(struct platform_device *pdev)
 	for (id = 0; id < smsm->num_hosts; id++) {
 		ret = smsm_parse_ipc(smsm, id);
 		if (ret < 0)
-			return ret;
+			goto out_put;
 	}
 
 	/* Acquire the main SMSM state vector */
@@ -534,13 +534,14 @@ static int qcom_smsm_probe(struct platform_device *pdev)
 			      smsm->num_entries * sizeof(u32));
 	if (ret < 0 && ret != -EEXIST) {
 		dev_err(&pdev->dev, "unable to allocate shared state entry\n");
-		return ret;
+		goto out_put;
 	}
 
 	states = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_SHARED_STATE, NULL);
 	if (IS_ERR(states)) {
 		dev_err(&pdev->dev, "Unable to acquire shared state entry\n");
-		return PTR_ERR(states);
+		ret = PTR_ERR(states);
+		goto out_put;
 	}
 
 	/* Acquire the list of interrupt mask vectors */
@@ -548,13 +549,14 @@ static int qcom_smsm_probe(struct platform_device *pdev)
 	ret = qcom_smem_alloc(QCOM_SMEM_HOST_ANY, SMEM_SMSM_CPU_INTR_MASK, size);
 	if (ret < 0 && ret != -EEXIST) {
 		dev_err(&pdev->dev, "unable to allocate smsm interrupt mask\n");
-		return ret;
+		goto out_put;
 	}
 
 	intr_mask = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_SMSM_CPU_INTR_MASK, NULL);
 	if (IS_ERR(intr_mask)) {
 		dev_err(&pdev->dev, "unable to acquire shared memory interrupt mask\n");
-		return PTR_ERR(intr_mask);
+		ret = PTR_ERR(intr_mask);
+		goto out_put;
 	}
 
 	/* Setup the reference to the local state bits */
@@ -565,7 +567,8 @@ static int qcom_smsm_probe(struct platform_device *pdev)
 	smsm->state = qcom_smem_state_register(local_node, &smsm_state_ops, smsm);
 	if (IS_ERR(smsm->state)) {
 		dev_err(smsm->dev, "failed to register qcom_smem_state\n");
-		return PTR_ERR(smsm->state);
+		ret = PTR_ERR(smsm->state);
+		goto out_put;
 	}
 
 	/* Register handlers for remote processor entries of interest. */
@@ -595,16 +598,19 @@ static int qcom_smsm_probe(struct platform_device *pdev)
 	}
 
 	platform_set_drvdata(pdev, smsm);
+	of_node_put(local_node);
 
 	return 0;
 
 unwind_interfaces:
+	of_node_put(node);
 	for (id = 0; id < smsm->num_entries; id++)
 		if (smsm->entries[id].domain)
 			irq_domain_remove(smsm->entries[id].domain);
 
 	qcom_smem_state_unregister(smsm->state);
-
+out_put:
+	of_node_put(local_node);
 	return ret;
 }
 
diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c
index 4554fb8655d3..aa37e1bad095 100644
--- a/drivers/soc/qcom/socinfo.c
+++ b/drivers/soc/qcom/socinfo.c
@@ -104,6 +104,7 @@ static const char *const pmic_models[] = {
 	[36] = "PM8009",
 	[38] = "PM8150C",
 	[41] = "SMB2351",
+	[45] = "PM6125",
 	[47] = "PMK8350",
 	[48] = "PM8350",
 	[49] = "PM8350C",
@@ -334,6 +335,7 @@ static const struct soc_id soc_id[] = {
 	{ 482, "SM8450" },
 	{ 487, "SC7280" },
 	{ 495, "SC7180P" },
+	{ 507, "SM6375" },
 };
 
 static const char *socinfo_machine(struct device *dev, unsigned int id)
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index c50a6ce1b99d..f95a1337450d 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -44,6 +44,7 @@ config ARCH_RZG2L
 	bool
 	select PM
 	select PM_GENERIC_DOMAINS
+	select RENESAS_RZG2L_IRQC
 
 config ARCH_RZN1
 	bool
@@ -332,6 +333,16 @@ config ARCH_R9A09G011
 
 endif # ARM64
 
+if RISCV
+
+config ARCH_R9A07G043
+	bool "RISC-V Platform support for RZ/Five"
+	select ARCH_RZG2L
+	help
+	  This enables support for the Renesas RZ/Five SoC.
+
+endif # RISCV
+
 config RST_RCAR
 	bool "Reset Controller support for R-Car" if COMPILE_TEST
 
diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c
index d171f1b635c7..621ceaa047d4 100644
--- a/drivers/soc/renesas/renesas-soc.c
+++ b/drivers/soc/renesas/renesas-soc.c
@@ -50,6 +50,10 @@ static const struct renesas_family fam_rza2 __initconst __maybe_unused = {
 	.name	= "RZ/A2",
 };
 
+static const struct renesas_family fam_rzfive __initconst __maybe_unused = {
+	.name	= "RZ/Five",
+};
+
 static const struct renesas_family fam_rzg1 __initconst __maybe_unused = {
 	.name	= "RZ/G1",
 	.reg	= 0xff000044,		/* PRR (Product Register) */
@@ -102,6 +106,11 @@ static const struct renesas_soc soc_rmobile_a1 __initconst __maybe_unused = {
 	.id	= 0x40,
 };
 
+static const struct renesas_soc soc_rz_five __initconst __maybe_unused = {
+	.family = &fam_rzfive,
+	.id     = 0x847c447,
+};
+
 static const struct renesas_soc soc_rz_g1h __initconst __maybe_unused = {
 	.family	= &fam_rzg1,
 	.id	= 0x45,
@@ -320,6 +329,7 @@ static const struct of_device_id renesas_socs[] __initconst = {
 	{ .compatible = "renesas,r8a779m0",	.data = &soc_rcar_h3 },
 	{ .compatible = "renesas,r8a779m1",	.data = &soc_rcar_h3 },
 	{ .compatible = "renesas,r8a779m8",	.data = &soc_rcar_h3 },
+	{ .compatible = "renesas,r8a779mb",	.data = &soc_rcar_h3 },
 #endif
 #ifdef CONFIG_ARCH_R8A77960
 	{ .compatible = "renesas,r8a7796",	.data = &soc_rcar_m3_w },
@@ -358,8 +368,12 @@ static const struct of_device_id renesas_socs[] __initconst = {
 	{ .compatible = "renesas,r8a779g0",	.data = &soc_rcar_v4h },
 #endif
 #if defined(CONFIG_ARCH_R9A07G043)
+#ifdef CONFIG_RISCV
+	{ .compatible = "renesas,r9a07g043",	.data = &soc_rz_five },
+#else
 	{ .compatible = "renesas,r9a07g043",	.data = &soc_rz_g2ul },
 #endif
+#endif
 #if defined(CONFIG_ARCH_R9A07G044)
 	{ .compatible = "renesas,r9a07g044",	.data = &soc_rz_g2l },
 #endif
diff --git a/drivers/soc/rockchip/io-domain.c b/drivers/soc/rockchip/io-domain.c
index 9df513d1219b..6619256c2d11 100644
--- a/drivers/soc/rockchip/io-domain.c
+++ b/drivers/soc/rockchip/io-domain.c
@@ -491,6 +491,22 @@ static const struct rockchip_iodomain_soc_data soc_data_rv1108_pmu = {
 	},
 };
 
+static const struct rockchip_iodomain_soc_data soc_data_rv1126_pmu = {
+	.grf_offset = 0x140,
+	.supply_names = {
+		NULL,
+		"vccio1",
+		"vccio2",
+		"vccio3",
+		"vccio4",
+		"vccio5",
+		"vccio6",
+		"vccio7",
+		"pmuio0",
+		"pmuio1",
+	},
+};
+
 static const struct of_device_id rockchip_iodomain_match[] = {
 	{
 		.compatible = "rockchip,px30-io-voltage-domain",
@@ -544,6 +560,10 @@ static const struct of_device_id rockchip_iodomain_match[] = {
 		.compatible = "rockchip,rv1108-pmu-io-voltage-domain",
 		.data = &soc_data_rv1108_pmu
 	},
+	{
+		.compatible = "rockchip,rv1126-pmu-io-voltage-domain",
+		.data = &soc_data_rv1126_pmu
+	},
 	{ /* sentinel */ },
 };
 MODULE_DEVICE_TABLE(of, rockchip_iodomain_match);
diff --git a/drivers/soc/rockchip/pm_domains.c b/drivers/soc/rockchip/pm_domains.c
index 89795abac951..84bc022f9e5b 100644
--- a/drivers/soc/rockchip/pm_domains.c
+++ b/drivers/soc/rockchip/pm_domains.c
@@ -19,6 +19,7 @@
 #include <linux/mfd/syscon.h>
 #include <soc/rockchip/pm_domains.h>
 #include <dt-bindings/power/px30-power.h>
+#include <dt-bindings/power/rockchip,rv1126-power.h>
 #include <dt-bindings/power/rk3036-power.h>
 #include <dt-bindings/power/rk3066-power.h>
 #include <dt-bindings/power/rk3128-power.h>
@@ -30,6 +31,7 @@
 #include <dt-bindings/power/rk3368-power.h>
 #include <dt-bindings/power/rk3399-power.h>
 #include <dt-bindings/power/rk3568-power.h>
+#include <dt-bindings/power/rk3588-power.h>
 
 struct rockchip_domain_info {
 	const char *name;
@@ -41,6 +43,9 @@ struct rockchip_domain_info {
 	bool active_wakeup;
 	int pwr_w_mask;
 	int req_w_mask;
+	int repair_status_mask;
+	u32 pwr_offset;
+	u32 req_offset;
 };
 
 struct rockchip_pmu_info {
@@ -49,6 +54,7 @@ struct rockchip_pmu_info {
 	u32 req_offset;
 	u32 idle_offset;
 	u32 ack_offset;
+	u32 repair_status_offset;
 
 	u32 core_pwrcnt_offset;
 	u32 gpu_pwrcnt_offset;
@@ -113,6 +119,22 @@ struct rockchip_pmu {
 	.active_wakeup = wakeup,			\
 }
 
+#define DOMAIN_M_O_R(_name, p_offset, pwr, status, r_status, r_offset, req, idle, ack, wakeup)	\
+{							\
+	.name = _name,					\
+	.pwr_offset = p_offset,				\
+	.pwr_w_mask = (pwr) << 16,			\
+	.pwr_mask = (pwr),				\
+	.status_mask = (status),			\
+	.repair_status_mask = (r_status),		\
+	.req_offset = r_offset,				\
+	.req_w_mask = (req) << 16,			\
+	.req_mask = (req),				\
+	.idle_mask = (idle),				\
+	.ack_mask = (ack),				\
+	.active_wakeup = wakeup,			\
+}
+
 #define DOMAIN_RK3036(_name, req, ack, idle, wakeup)		\
 {							\
 	.name = _name,				\
@@ -126,6 +148,9 @@ struct rockchip_pmu {
 #define DOMAIN_PX30(name, pwr, status, req, wakeup)		\
 	DOMAIN_M(name, pwr, status, req, (req) << 16, req, wakeup)
 
+#define DOMAIN_RV1126(name, pwr, req, idle, wakeup)		\
+	DOMAIN_M(name, pwr, pwr, req, idle, idle, wakeup)
+
 #define DOMAIN_RK3288(name, pwr, status, req, wakeup)		\
 	DOMAIN(name, pwr, status, req, req, (req) << 16, wakeup)
 
@@ -244,6 +269,9 @@ void rockchip_pmu_unblock(void)
 }
 EXPORT_SYMBOL_GPL(rockchip_pmu_unblock);
 
+#define DOMAIN_RK3588(name, p_offset, pwr, status, r_status, r_offset, req, idle, wakeup)	\
+	DOMAIN_M_O_R(name, p_offset, pwr, status, r_status, r_offset, req, idle, idle, wakeup)
+
 static bool rockchip_pmu_domain_is_idle(struct rockchip_pm_domain *pd)
 {
 	struct rockchip_pmu *pmu = pd->pmu;
@@ -268,6 +296,7 @@ static int rockchip_pmu_set_idle_request(struct rockchip_pm_domain *pd,
 	const struct rockchip_domain_info *pd_info = pd->info;
 	struct generic_pm_domain *genpd = &pd->genpd;
 	struct rockchip_pmu *pmu = pd->pmu;
+	u32 pd_req_offset = pd_info->req_offset;
 	unsigned int target_ack;
 	unsigned int val;
 	bool is_idle;
@@ -276,11 +305,11 @@ static int rockchip_pmu_set_idle_request(struct rockchip_pm_domain *pd,
 	if (pd_info->req_mask == 0)
 		return 0;
 	else if (pd_info->req_w_mask)
-		regmap_write(pmu->regmap, pmu->info->req_offset,
+		regmap_write(pmu->regmap, pmu->info->req_offset + pd_req_offset,
 			     idle ? (pd_info->req_mask | pd_info->req_w_mask) :
 			     pd_info->req_w_mask);
 	else
-		regmap_update_bits(pmu->regmap, pmu->info->req_offset,
+		regmap_update_bits(pmu->regmap, pmu->info->req_offset + pd_req_offset,
 				   pd_info->req_mask, idle ? -1U : 0);
 
 	wmb();
@@ -363,6 +392,12 @@ static bool rockchip_pmu_domain_is_on(struct rockchip_pm_domain *pd)
 	struct rockchip_pmu *pmu = pd->pmu;
 	unsigned int val;
 
+	if (pd->info->repair_status_mask) {
+		regmap_read(pmu->regmap, pmu->info->repair_status_offset, &val);
+		/* 1'b1: power on, 1'b0: power off */
+		return val & pd->info->repair_status_mask;
+	}
+
 	/* check idle status for idle-only domains */
 	if (pd->info->status_mask == 0)
 		return !rockchip_pmu_domain_is_idle(pd);
@@ -378,16 +413,17 @@ static void rockchip_do_pmu_set_power_domain(struct rockchip_pm_domain *pd,
 {
 	struct rockchip_pmu *pmu = pd->pmu;
 	struct generic_pm_domain *genpd = &pd->genpd;
+	u32 pd_pwr_offset = pd->info->pwr_offset;
 	bool is_on;
 
 	if (pd->info->pwr_mask == 0)
 		return;
 	else if (pd->info->pwr_w_mask)
-		regmap_write(pmu->regmap, pmu->info->pwr_offset,
+		regmap_write(pmu->regmap, pmu->info->pwr_offset + pd_pwr_offset,
 			     on ? pd->info->pwr_w_mask :
 			     (pd->info->pwr_mask | pd->info->pwr_w_mask));
 	else
-		regmap_update_bits(pmu->regmap, pmu->info->pwr_offset,
+		regmap_update_bits(pmu->regmap, pmu->info->pwr_offset + pd_pwr_offset,
 				   pd->info->pwr_mask, on ? 0 : -1U);
 
 	wmb();
@@ -514,6 +550,9 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu,
 			node, id);
 		return -EINVAL;
 	}
+	/* RK3588 has domains with two parents (RKVDEC0/RKVDEC1) */
+	if (pmu->genpd_data.domains[id])
+		return 0;
 
 	pd_info = &pmu->info->domain_info[id];
 	if (!pd_info) {
@@ -595,14 +634,6 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu,
 		}
 	}
 
-	error = rockchip_pd_power(pd, true);
-	if (error) {
-		dev_err(pmu->dev,
-			"failed to power on domain '%pOFn': %d\n",
-			node, error);
-		goto err_unprepare_clocks;
-	}
-
 	if (pd->info->name)
 		pd->genpd.name = pd->info->name;
 	else
@@ -614,7 +645,7 @@ static int rockchip_pm_add_one_domain(struct rockchip_pmu *pmu,
 	pd->genpd.flags = GENPD_FLAG_PM_CLK;
 	if (pd_info->active_wakeup)
 		pd->genpd.flags |= GENPD_FLAG_ACTIVE_WAKEUP;
-	pm_genpd_init(&pd->genpd, NULL, false);
+	pm_genpd_init(&pd->genpd, NULL, !rockchip_pmu_domain_is_on(pd));
 
 	pmu->genpd_data.domains[id] = &pd->genpd;
 	return 0;
@@ -855,6 +886,16 @@ static const struct rockchip_domain_info px30_pm_domains[] = {
 	[PX30_PD_GPU]		= DOMAIN_PX30("gpu",      BIT(15), BIT(15), BIT(2),  false),
 };
 
+static const struct rockchip_domain_info rv1126_pm_domains[] = {
+	[RV1126_PD_VEPU]	= DOMAIN_RV1126("vepu", BIT(2),  BIT(9),  BIT(9), false),
+	[RV1126_PD_VI]		= DOMAIN_RV1126("vi", BIT(4),  BIT(6),  BIT(6),  false),
+	[RV1126_PD_ISPP]	= DOMAIN_RV1126("ispp", BIT(1), BIT(8), BIT(8),  false),
+	[RV1126_PD_VDPU]	= DOMAIN_RV1126("vdpu", BIT(3), BIT(10), BIT(10), false),
+	[RV1126_PD_NVM]		= DOMAIN_RV1126("nvm", BIT(7), BIT(11), BIT(11),  false),
+	[RV1126_PD_SDIO]	= DOMAIN_RV1126("sdio", BIT(8), BIT(13), BIT(13),  false),
+	[RV1126_PD_USB]		= DOMAIN_RV1126("usb", BIT(9), BIT(15), BIT(15),  false),
+};
+
 static const struct rockchip_domain_info rk3036_pm_domains[] = {
 	[RK3036_PD_MSCH]	= DOMAIN_RK3036("msch", BIT(14), BIT(23), BIT(30), true),
 	[RK3036_PD_CORE]	= DOMAIN_RK3036("core", BIT(13), BIT(17), BIT(24), false),
@@ -982,6 +1023,38 @@ static const struct rockchip_domain_info rk3568_pm_domains[] = {
 	[RK3568_PD_PIPE]	= DOMAIN_RK3568("pipe", BIT(8), BIT(11), false),
 };
 
+static const struct rockchip_domain_info rk3588_pm_domains[] = {
+	[RK3588_PD_GPU]		= DOMAIN_RK3588("gpu",     0x0, BIT(0),  0,       BIT(1),  0x0, BIT(0),  BIT(0),  false),
+	[RK3588_PD_NPU]		= DOMAIN_RK3588("npu",     0x0, BIT(1),  BIT(1),  0,       0x0, 0,       0,       false),
+	[RK3588_PD_VCODEC]	= DOMAIN_RK3588("vcodec",  0x0, BIT(2),  BIT(2),  0,       0x0, 0,       0,       false),
+	[RK3588_PD_NPUTOP]	= DOMAIN_RK3588("nputop",  0x0, BIT(3),  0,       BIT(2),  0x0, BIT(1),  BIT(1),  false),
+	[RK3588_PD_NPU1]	= DOMAIN_RK3588("npu1",    0x0, BIT(4),  0,       BIT(3),  0x0, BIT(2),  BIT(2),  false),
+	[RK3588_PD_NPU2]	= DOMAIN_RK3588("npu2",    0x0, BIT(5),  0,       BIT(4),  0x0, BIT(3),  BIT(3),  false),
+	[RK3588_PD_VENC0]	= DOMAIN_RK3588("venc0",   0x0, BIT(6),  0,       BIT(5),  0x0, BIT(4),  BIT(4),  false),
+	[RK3588_PD_VENC1]	= DOMAIN_RK3588("venc1",   0x0, BIT(7),  0,       BIT(6),  0x0, BIT(5),  BIT(5),  false),
+	[RK3588_PD_RKVDEC0]	= DOMAIN_RK3588("rkvdec0", 0x0, BIT(8),  0,       BIT(7),  0x0, BIT(6),  BIT(6),  false),
+	[RK3588_PD_RKVDEC1]	= DOMAIN_RK3588("rkvdec1", 0x0, BIT(9),  0,       BIT(8),  0x0, BIT(7),  BIT(7),  false),
+	[RK3588_PD_VDPU]	= DOMAIN_RK3588("vdpu",    0x0, BIT(10), 0,       BIT(9),  0x0, BIT(8),  BIT(8),  false),
+	[RK3588_PD_RGA30]	= DOMAIN_RK3588("rga30",   0x0, BIT(11), 0,       BIT(10), 0x0, 0,       0,       false),
+	[RK3588_PD_AV1]		= DOMAIN_RK3588("av1",     0x0, BIT(12), 0,       BIT(11), 0x0, BIT(9),  BIT(9),  false),
+	[RK3588_PD_VI]		= DOMAIN_RK3588("vi",      0x0, BIT(13), 0,       BIT(12), 0x0, BIT(10), BIT(10), false),
+	[RK3588_PD_FEC]		= DOMAIN_RK3588("fec",     0x0, BIT(14), 0,       BIT(13), 0x0, 0,       0,       false),
+	[RK3588_PD_ISP1]	= DOMAIN_RK3588("isp1",    0x0, BIT(15), 0,       BIT(14), 0x0, BIT(11), BIT(11), false),
+	[RK3588_PD_RGA31]	= DOMAIN_RK3588("rga31",   0x4, BIT(0),  0,       BIT(15), 0x0, BIT(12), BIT(12), false),
+	[RK3588_PD_VOP]		= DOMAIN_RK3588("vop",     0x4, BIT(1),  0,       BIT(16), 0x0, BIT(13) | BIT(14), BIT(13) | BIT(14), false),
+	[RK3588_PD_VO0]		= DOMAIN_RK3588("vo0",     0x4, BIT(2),  0,       BIT(17), 0x0, BIT(15), BIT(15), false),
+	[RK3588_PD_VO1]		= DOMAIN_RK3588("vo1",     0x4, BIT(3),  0,       BIT(18), 0x4, BIT(0),  BIT(16), false),
+	[RK3588_PD_AUDIO]	= DOMAIN_RK3588("audio",   0x4, BIT(4),  0,       BIT(19), 0x4, BIT(1),  BIT(17), false),
+	[RK3588_PD_PHP]		= DOMAIN_RK3588("php",     0x4, BIT(5),  0,       BIT(20), 0x4, BIT(5),  BIT(21), false),
+	[RK3588_PD_GMAC]	= DOMAIN_RK3588("gmac",    0x4, BIT(6),  0,       BIT(21), 0x0, 0,       0,       false),
+	[RK3588_PD_PCIE]	= DOMAIN_RK3588("pcie",    0x4, BIT(7),  0,       BIT(22), 0x0, 0,       0,       true),
+	[RK3588_PD_NVM]		= DOMAIN_RK3588("nvm",     0x4, BIT(8),  BIT(24), 0,       0x4, BIT(2),  BIT(18), false),
+	[RK3588_PD_NVM0]	= DOMAIN_RK3588("nvm0",    0x4, BIT(9),  0,       BIT(23), 0x0, 0,       0,       false),
+	[RK3588_PD_SDIO]	= DOMAIN_RK3588("sdio",    0x4, BIT(10), 0,       BIT(24), 0x4, BIT(3),  BIT(19), false),
+	[RK3588_PD_USB]		= DOMAIN_RK3588("usb",     0x4, BIT(11), 0,       BIT(25), 0x4, BIT(4),  BIT(20), true),
+	[RK3588_PD_SDMMC]	= DOMAIN_RK3588("sdmmc",   0x4, BIT(13), 0,       BIT(26), 0x0, 0,       0,       false),
+};
+
 static const struct rockchip_pmu_info px30_pmu = {
 	.pwr_offset = 0x18,
 	.status_offset = 0x20,
@@ -1128,6 +1201,29 @@ static const struct rockchip_pmu_info rk3568_pmu = {
 	.domain_info = rk3568_pm_domains,
 };
 
+static const struct rockchip_pmu_info rk3588_pmu = {
+	.pwr_offset = 0x14c,
+	.status_offset = 0x180,
+	.req_offset = 0x10c,
+	.idle_offset = 0x120,
+	.ack_offset = 0x118,
+	.repair_status_offset = 0x290,
+
+	.num_domains = ARRAY_SIZE(rk3588_pm_domains),
+	.domain_info = rk3588_pm_domains,
+};
+
+static const struct rockchip_pmu_info rv1126_pmu = {
+	.pwr_offset = 0x110,
+	.status_offset = 0x108,
+	.req_offset = 0xc0,
+	.idle_offset = 0xd8,
+	.ack_offset = 0xd0,
+
+	.num_domains = ARRAY_SIZE(rv1126_pm_domains),
+	.domain_info = rv1126_pm_domains,
+};
+
 static const struct of_device_id rockchip_pm_domain_dt_match[] = {
 	{
 		.compatible = "rockchip,px30-power-controller",
@@ -1177,6 +1273,14 @@ static const struct of_device_id rockchip_pm_domain_dt_match[] = {
 		.compatible = "rockchip,rk3568-power-controller",
 		.data = (void *)&rk3568_pmu,
 	},
+	{
+		.compatible = "rockchip,rk3588-power-controller",
+		.data = (void *)&rk3588_pmu,
+	},
+	{
+		.compatible = "rockchip,rv1126-power-controller",
+		.data = (void *)&rv1126_pmu,
+	},
 	{ /* sentinel */ },
 };
 
diff --git a/drivers/soc/sunxi/sunxi_sram.c b/drivers/soc/sunxi/sunxi_sram.c
index a8f3876963a0..92f9186c1c42 100644
--- a/drivers/soc/sunxi/sunxi_sram.c
+++ b/drivers/soc/sunxi/sunxi_sram.c
@@ -78,8 +78,8 @@ static struct sunxi_sram_desc sun4i_a10_sram_d = {
 
 static struct sunxi_sram_desc sun50i_a64_sram_c = {
 	.data	= SUNXI_SRAM_DATA("C", 0x4, 24, 1,
-				  SUNXI_SRAM_MAP(0, 1, "cpu"),
-				  SUNXI_SRAM_MAP(1, 0, "de2")),
+				  SUNXI_SRAM_MAP(1, 0, "cpu"),
+				  SUNXI_SRAM_MAP(0, 1, "de2")),
 };
 
 static const struct of_device_id sunxi_sram_dt_ids[] = {
@@ -254,36 +254,36 @@ int sunxi_sram_claim(struct device *dev)
 	writel(val | ((device << sram_data->offset) & mask),
 	       base + sram_data->reg);
 
+	sram_desc->claimed = true;
 	spin_unlock(&sram_lock);
 
 	return 0;
 }
 EXPORT_SYMBOL(sunxi_sram_claim);
 
-int sunxi_sram_release(struct device *dev)
+void sunxi_sram_release(struct device *dev)
 {
 	const struct sunxi_sram_data *sram_data;
 	struct sunxi_sram_desc *sram_desc;
 
 	if (!dev || !dev->of_node)
-		return -EINVAL;
+		return;
 
 	sram_data = sunxi_sram_of_parse(dev->of_node, NULL);
 	if (IS_ERR(sram_data))
-		return -EINVAL;
+		return;
 
 	sram_desc = to_sram_desc(sram_data);
 
 	spin_lock(&sram_lock);
 	sram_desc->claimed = false;
 	spin_unlock(&sram_lock);
-
-	return 0;
 }
 EXPORT_SYMBOL(sunxi_sram_release);
 
 struct sunxi_sramc_variant {
 	int num_emac_clocks;
+	bool has_ldo_ctrl;
 };
 
 static const struct sunxi_sramc_variant sun4i_a10_sramc_variant = {
@@ -294,6 +294,11 @@ static const struct sunxi_sramc_variant sun8i_h3_sramc_variant = {
 	.num_emac_clocks = 1,
 };
 
+static const struct sunxi_sramc_variant sun20i_d1_sramc_variant = {
+	.num_emac_clocks = 1,
+	.has_ldo_ctrl = true,
+};
+
 static const struct sunxi_sramc_variant sun50i_a64_sramc_variant = {
 	.num_emac_clocks = 1,
 };
@@ -303,37 +308,38 @@ static const struct sunxi_sramc_variant sun50i_h616_sramc_variant = {
 };
 
 #define SUNXI_SRAM_EMAC_CLOCK_REG	0x30
+#define SUNXI_SYS_LDO_CTRL_REG		0x150
+
 static bool sunxi_sram_regmap_accessible_reg(struct device *dev,
 					     unsigned int reg)
 {
-	const struct sunxi_sramc_variant *variant;
-
-	variant = of_device_get_match_data(dev);
+	const struct sunxi_sramc_variant *variant = dev_get_drvdata(dev);
 
-	if (reg < SUNXI_SRAM_EMAC_CLOCK_REG)
-		return false;
-	if (reg > SUNXI_SRAM_EMAC_CLOCK_REG + variant->num_emac_clocks * 4)
-		return false;
+	if (reg >= SUNXI_SRAM_EMAC_CLOCK_REG &&
+	    reg <  SUNXI_SRAM_EMAC_CLOCK_REG + variant->num_emac_clocks * 4)
+		return true;
+	if (reg == SUNXI_SYS_LDO_CTRL_REG && variant->has_ldo_ctrl)
+		return true;
 
-	return true;
+	return false;
 }
 
-static struct regmap_config sunxi_sram_emac_clock_regmap = {
+static struct regmap_config sunxi_sram_regmap_config = {
 	.reg_bits       = 32,
 	.val_bits       = 32,
 	.reg_stride     = 4,
 	/* last defined register */
-	.max_register   = SUNXI_SRAM_EMAC_CLOCK_REG + 4,
+	.max_register   = SUNXI_SYS_LDO_CTRL_REG,
 	/* other devices have no business accessing other registers */
 	.readable_reg	= sunxi_sram_regmap_accessible_reg,
 	.writeable_reg	= sunxi_sram_regmap_accessible_reg,
 };
 
-static int sunxi_sram_probe(struct platform_device *pdev)
+static int __init sunxi_sram_probe(struct platform_device *pdev)
 {
-	struct dentry *d;
-	struct regmap *emac_clock;
 	const struct sunxi_sramc_variant *variant;
+	struct device *dev = &pdev->dev;
+	struct regmap *regmap;
 
 	sram_dev = &pdev->dev;
 
@@ -341,24 +347,21 @@ static int sunxi_sram_probe(struct platform_device *pdev)
 	if (!variant)
 		return -EINVAL;
 
+	dev_set_drvdata(dev, (struct sunxi_sramc_variant *)variant);
+
 	base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
-	of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev);
-
-	d = debugfs_create_file("sram", S_IRUGO, NULL, NULL,
-				&sunxi_sram_fops);
-	if (!d)
-		return -ENOMEM;
+	if (variant->num_emac_clocks || variant->has_ldo_ctrl) {
+		regmap = devm_regmap_init_mmio(dev, base, &sunxi_sram_regmap_config);
+		if (IS_ERR(regmap))
+			return PTR_ERR(regmap);
+	}
 
-	if (variant->num_emac_clocks > 0) {
-		emac_clock = devm_regmap_init_mmio(&pdev->dev, base,
-						   &sunxi_sram_emac_clock_regmap);
+	of_platform_populate(dev->of_node, NULL, NULL, dev);
 
-		if (IS_ERR(emac_clock))
-			return PTR_ERR(emac_clock);
-	}
+	debugfs_create_file("sram", 0444, NULL, NULL, &sunxi_sram_fops);
 
 	return 0;
 }
@@ -385,6 +388,10 @@ static const struct of_device_id sunxi_sram_dt_match[] = {
 		.data = &sun8i_h3_sramc_variant,
 	},
 	{
+		.compatible = "allwinner,sun20i-d1-system-control",
+		.data = &sun20i_d1_sramc_variant,
+	},
+	{
 		.compatible = "allwinner,sun50i-a64-sram-controller",
 		.data = &sun50i_a64_sramc_variant,
 	},
@@ -409,9 +416,8 @@ static struct platform_driver sunxi_sram_driver = {
 		.name		= "sunxi-sram",
 		.of_match_table	= sunxi_sram_dt_match,
 	},
-	.probe	= sunxi_sram_probe,
 };
-module_platform_driver(sunxi_sram_driver);
+builtin_platform_driver_probe(sunxi_sram_driver, sunxi_sram_probe);
 
 MODULE_AUTHOR("Maxime Ripard <maxime.ripard@free-electrons.com>");
 MODULE_DESCRIPTION("Allwinner sunXi SRAM Controller Driver");
diff --git a/drivers/soc/tegra/Kconfig b/drivers/soc/tegra/Kconfig
index 5725c8ef0406..d1ecadffa1bb 100644
--- a/drivers/soc/tegra/Kconfig
+++ b/drivers/soc/tegra/Kconfig
@@ -136,7 +136,6 @@ config SOC_TEGRA_FUSE
 	def_bool y
 	depends on ARCH_TEGRA
 	select SOC_BUS
-	select TEGRA20_APB_DMA if ARCH_TEGRA_2x_SOC
 
 config SOC_TEGRA_FLOWCTRL
 	bool
@@ -162,3 +161,12 @@ config SOC_TEGRA30_VOLTAGE_COUPLER
 	bool "Voltage scaling support for Tegra30 SoCs"
 	depends on ARCH_TEGRA_3x_SOC || COMPILE_TEST
 	depends on REGULATOR
+
+config SOC_TEGRA_CBB
+	tristate "Tegra driver to handle error from CBB"
+	depends on ARCH_TEGRA_194_SOC || ARCH_TEGRA_234_SOC
+	default y
+	help
+	  Support for handling error from Tegra Control Backbone(CBB).
+	  This driver handles the errors from CBB and prints debug
+	  information about the failed transactions.
diff --git a/drivers/soc/tegra/Makefile b/drivers/soc/tegra/Makefile
index 054e862b63d8..d722f512dc9d 100644
--- a/drivers/soc/tegra/Makefile
+++ b/drivers/soc/tegra/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-y += fuse/
+obj-y += cbb/
 
 obj-y += common.o
 obj-$(CONFIG_SOC_TEGRA_FLOWCTRL) += flowctrl.o
diff --git a/drivers/soc/tegra/cbb/Makefile b/drivers/soc/tegra/cbb/Makefile
new file mode 100644
index 000000000000..e3ac6cdddf5c
--- /dev/null
+++ b/drivers/soc/tegra/cbb/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Control Backbone Driver code.
+#
+ifdef CONFIG_SOC_TEGRA_CBB
+obj-y += tegra-cbb.o
+obj-$(CONFIG_ARCH_TEGRA_194_SOC) += tegra194-cbb.o
+obj-$(CONFIG_ARCH_TEGRA_234_SOC) += tegra234-cbb.o
+endif
diff --git a/drivers/soc/tegra/cbb/tegra-cbb.c b/drivers/soc/tegra/cbb/tegra-cbb.c
new file mode 100644
index 000000000000..d200937353c7
--- /dev/null
+++ b/drivers/soc/tegra/cbb/tegra-cbb.c
@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved
+ */
+
+#include <linux/clk.h>
+#include <linux/cpufeature.h>
+#include <linux/debugfs.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/of_irq.h>
+#include <linux/of_address.h>
+#include <linux/interrupt.h>
+#include <linux/ioport.h>
+#include <linux/version.h>
+#include <soc/tegra/fuse.h>
+#include <soc/tegra/tegra-cbb.h>
+
+void tegra_cbb_print_err(struct seq_file *file, const char *fmt, ...)
+{
+	struct va_format vaf;
+	va_list args;
+
+	va_start(args, fmt);
+
+	if (file) {
+		seq_vprintf(file, fmt, args);
+	} else {
+		vaf.fmt = fmt;
+		vaf.va = &args;
+		pr_crit("%pV", &vaf);
+	}
+
+	va_end(args);
+}
+
+void tegra_cbb_print_cache(struct seq_file *file, u32 cache)
+{
+	const char *buff_str, *mod_str, *rd_str, *wr_str;
+
+	buff_str = (cache & BIT(0)) ? "Bufferable " : "";
+	mod_str = (cache & BIT(1)) ? "Modifiable " : "";
+	rd_str = (cache & BIT(2)) ? "Read-Allocate " : "";
+	wr_str = (cache & BIT(3)) ? "Write-Allocate" : "";
+
+	if (cache == 0x0)
+		buff_str = "Device Non-Bufferable";
+
+	tegra_cbb_print_err(file, "\t  Cache\t\t\t: 0x%x -- %s%s%s%s\n",
+			    cache, buff_str, mod_str, rd_str, wr_str);
+}
+
+void tegra_cbb_print_prot(struct seq_file *file, u32 prot)
+{
+	const char *data_str, *secure_str, *priv_str;
+
+	data_str = (prot & 0x4) ? "Instruction" : "Data";
+	secure_str = (prot & 0x2) ? "Non-Secure" : "Secure";
+	priv_str = (prot & 0x1) ? "Privileged" : "Unprivileged";
+
+	tegra_cbb_print_err(file, "\t  Protection\t\t: 0x%x -- %s, %s, %s Access\n",
+			    prot, priv_str, secure_str, data_str);
+}
+
+static int tegra_cbb_err_show(struct seq_file *file, void *data)
+{
+	struct tegra_cbb *cbb = file->private;
+
+	return cbb->ops->debugfs_show(cbb, file, data);
+}
+
+static int tegra_cbb_err_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, tegra_cbb_err_show, inode->i_private);
+}
+
+static const struct file_operations tegra_cbb_err_fops = {
+	.open = tegra_cbb_err_open,
+	.read = seq_read,
+	.llseek = seq_lseek,
+	.release = single_release
+};
+
+static int tegra_cbb_err_debugfs_init(struct tegra_cbb *cbb)
+{
+	static struct dentry *root;
+
+	if (!root) {
+		root = debugfs_create_file("tegra_cbb_err", 0444, NULL, cbb, &tegra_cbb_err_fops);
+		if (IS_ERR_OR_NULL(root)) {
+			pr_err("%s(): could not create debugfs node\n", __func__);
+			return PTR_ERR(root);
+		}
+	}
+
+	return 0;
+}
+
+void tegra_cbb_stall_enable(struct tegra_cbb *cbb)
+{
+	if (cbb->ops->stall_enable)
+		cbb->ops->stall_enable(cbb);
+}
+
+void tegra_cbb_fault_enable(struct tegra_cbb *cbb)
+{
+	if (cbb->ops->fault_enable)
+		cbb->ops->fault_enable(cbb);
+}
+
+void tegra_cbb_error_clear(struct tegra_cbb *cbb)
+{
+	if (cbb->ops->error_clear)
+		cbb->ops->error_clear(cbb);
+}
+
+u32 tegra_cbb_get_status(struct tegra_cbb *cbb)
+{
+	if (cbb->ops->get_status)
+		return cbb->ops->get_status(cbb);
+
+	return 0;
+}
+
+int tegra_cbb_get_irq(struct platform_device *pdev, unsigned int *nonsec_irq,
+		      unsigned int *sec_irq)
+{
+	unsigned int index = 0;
+	int num_intr = 0, irq;
+
+	num_intr = platform_irq_count(pdev);
+	if (!num_intr)
+		return -EINVAL;
+
+	if (num_intr == 2) {
+		irq = platform_get_irq(pdev, index);
+		if (irq <= 0) {
+			dev_err(&pdev->dev, "failed to get non-secure IRQ: %d\n", irq);
+			return -ENOENT;
+		}
+
+		*nonsec_irq = irq;
+		index++;
+	}
+
+	irq = platform_get_irq(pdev, index);
+	if (irq <= 0) {
+		dev_err(&pdev->dev, "failed to get secure IRQ: %d\n", irq);
+		return -ENOENT;
+	}
+
+	*sec_irq = irq;
+
+	if (num_intr == 1)
+		dev_dbg(&pdev->dev, "secure IRQ: %u\n", *sec_irq);
+
+	if (num_intr == 2)
+		dev_dbg(&pdev->dev, "secure IRQ: %u, non-secure IRQ: %u\n", *sec_irq, *nonsec_irq);
+
+	return 0;
+}
+
+int tegra_cbb_register(struct tegra_cbb *cbb)
+{
+	int ret;
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS)) {
+		ret = tegra_cbb_err_debugfs_init(cbb);
+		if (ret) {
+			dev_err(cbb->dev, "failed to create debugfs\n");
+			return ret;
+		}
+	}
+
+	/* register interrupt handler for errors due to different initiators */
+	ret = cbb->ops->interrupt_enable(cbb);
+	if (ret < 0) {
+		dev_err(cbb->dev, "Failed to register CBB Interrupt ISR");
+		return ret;
+	}
+
+	cbb->ops->error_enable(cbb);
+	dsb(sy);
+
+	return 0;
+}
diff --git a/drivers/soc/tegra/cbb/tegra194-cbb.c b/drivers/soc/tegra/cbb/tegra194-cbb.c
new file mode 100644
index 000000000000..1ae0bd9a1ac1
--- /dev/null
+++ b/drivers/soc/tegra/cbb/tegra194-cbb.c
@@ -0,0 +1,2364 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved
+ *
+ * The driver handles Error's from Control Backbone(CBB) generated due to
+ * illegal accesses. When an error is reported from a NOC within CBB,
+ * the driver checks ErrVld status of all three Error Logger's of that NOC.
+ * It then prints debug information about failed transaction using ErrLog
+ * registers of error logger which has ErrVld set. Currently, SLV, DEC,
+ * TMO, SEC, UNS are the codes which are supported by CBB.
+ */
+
+#include <linux/clk.h>
+#include <linux/cpufeature.h>
+#include <linux/debugfs.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/of_irq.h>
+#include <linux/of_address.h>
+#include <linux/interrupt.h>
+#include <linux/ioport.h>
+#include <linux/version.h>
+#include <soc/tegra/fuse.h>
+#include <soc/tegra/tegra-cbb.h>
+
+#define ERRLOGGER_0_ID_COREID_0     0x00000000
+#define ERRLOGGER_0_ID_REVISIONID_0 0x00000004
+#define ERRLOGGER_0_FAULTEN_0       0x00000008
+#define ERRLOGGER_0_ERRVLD_0        0x0000000c
+#define ERRLOGGER_0_ERRCLR_0        0x00000010
+#define ERRLOGGER_0_ERRLOG0_0       0x00000014
+#define ERRLOGGER_0_ERRLOG1_0       0x00000018
+#define ERRLOGGER_0_RSVD_00_0       0x0000001c
+#define ERRLOGGER_0_ERRLOG3_0       0x00000020
+#define ERRLOGGER_0_ERRLOG4_0       0x00000024
+#define ERRLOGGER_0_ERRLOG5_0       0x00000028
+#define ERRLOGGER_0_STALLEN_0       0x00000038
+
+#define ERRLOGGER_1_ID_COREID_0     0x00000080
+#define ERRLOGGER_1_ID_REVISIONID_0 0x00000084
+#define ERRLOGGER_1_FAULTEN_0       0x00000088
+#define ERRLOGGER_1_ERRVLD_0        0x0000008c
+#define ERRLOGGER_1_ERRCLR_0        0x00000090
+#define ERRLOGGER_1_ERRLOG0_0       0x00000094
+#define ERRLOGGER_1_ERRLOG1_0       0x00000098
+#define ERRLOGGER_1_RSVD_00_0       0x0000009c
+#define ERRLOGGER_1_ERRLOG3_0       0x000000a0
+#define ERRLOGGER_1_ERRLOG4_0       0x000000a4
+#define ERRLOGGER_1_ERRLOG5_0       0x000000a8
+#define ERRLOGGER_1_STALLEN_0       0x000000b8
+
+#define ERRLOGGER_2_ID_COREID_0     0x00000100
+#define ERRLOGGER_2_ID_REVISIONID_0 0x00000104
+#define ERRLOGGER_2_FAULTEN_0       0x00000108
+#define ERRLOGGER_2_ERRVLD_0        0x0000010c
+#define ERRLOGGER_2_ERRCLR_0        0x00000110
+#define ERRLOGGER_2_ERRLOG0_0       0x00000114
+#define ERRLOGGER_2_ERRLOG1_0       0x00000118
+#define ERRLOGGER_2_RSVD_00_0       0x0000011c
+#define ERRLOGGER_2_ERRLOG3_0       0x00000120
+#define ERRLOGGER_2_ERRLOG4_0       0x00000124
+#define ERRLOGGER_2_ERRLOG5_0       0x00000128
+#define ERRLOGGER_2_STALLEN_0       0x00000138
+
+#define CBB_NOC_INITFLOW GENMASK(23, 20)
+#define CBB_NOC_TARGFLOW GENMASK(19, 16)
+#define CBB_NOC_TARG_SUBRANGE GENMASK(15, 9)
+#define CBB_NOC_SEQID GENMASK(8, 0)
+
+#define BPMP_NOC_INITFLOW GENMASK(20, 18)
+#define BPMP_NOC_TARGFLOW GENMASK(17, 13)
+#define BPMP_NOC_TARG_SUBRANGE GENMASK(12, 9)
+#define BPMP_NOC_SEQID GENMASK(8, 0)
+
+#define AON_NOC_INITFLOW GENMASK(22, 21)
+#define AON_NOC_TARGFLOW GENMASK(20, 15)
+#define AON_NOC_TARG_SUBRANGE GENMASK(14, 9)
+#define AON_NOC_SEQID GENMASK(8, 0)
+
+#define SCE_NOC_INITFLOW GENMASK(21, 19)
+#define SCE_NOC_TARGFLOW GENMASK(18, 14)
+#define SCE_NOC_TARG_SUBRANGE GENMASK(13, 9)
+#define SCE_NOC_SEQID GENMASK(8, 0)
+
+#define CBB_NOC_AXCACHE GENMASK(3, 0)
+#define CBB_NOC_NON_MOD GENMASK(4, 4)
+#define CBB_NOC_AXPROT GENMASK(7, 5)
+#define CBB_NOC_FALCONSEC GENMASK(9, 8)
+#define CBB_NOC_GRPSEC GENMASK(16, 10)
+#define CBB_NOC_VQC GENMASK(18, 17)
+#define CBB_NOC_MSTR_ID GENMASK(22, 19)
+#define CBB_NOC_AXI_ID GENMASK(30, 23)
+
+#define CLUSTER_NOC_AXCACHE GENMASK(3, 0)
+#define CLUSTER_NOC_AXPROT GENMASK(6, 4)
+#define CLUSTER_NOC_FALCONSEC GENMASK(8, 7)
+#define CLUSTER_NOC_GRPSEC GENMASK(15, 9)
+#define CLUSTER_NOC_VQC GENMASK(17, 16)
+#define CLUSTER_NOC_MSTR_ID GENMASK(21, 18)
+
+#define USRBITS_MSTR_ID GENMASK(21, 18)
+
+#define CBB_ERR_OPC GENMASK(4, 1)
+#define CBB_ERR_ERRCODE GENMASK(10, 8)
+#define CBB_ERR_LEN1 GENMASK(27, 16)
+
+#define DMAAPB_X_RAW_INTERRUPT_STATUS 0x2ec
+
+struct tegra194_cbb_packet_header {
+	bool lock;   // [0]
+	u8 opc;      // [4:1]
+	u8 errcode;  // [10:8]= RD, RDW, RDL, RDX, WR, WRW, WRC, PRE, URG
+	u16 len1;    // [27:16]
+	bool format; // [31]  = 1 -> FlexNoC versions 2.7 & above
+};
+
+struct tegra194_cbb_aperture {
+	u8 initflow;
+	u8 targflow;
+	u8 targ_subrange;
+	u8 init_mapping;
+	u32 init_localaddress;
+	u8 targ_mapping;
+	u32 targ_localaddress;
+	u16 seqid;
+};
+
+struct tegra194_cbb_userbits {
+	u8 axcache;
+	u8 non_mod;
+	u8 axprot;
+	u8 falconsec;
+	u8 grpsec;
+	u8 vqc;
+	u8 mstr_id;
+	u8 axi_id;
+};
+
+struct tegra194_cbb_noc_data {
+	const char *name;
+	bool erd_mask_inband_err;
+	const char * const *master_id;
+	unsigned int max_aperture;
+	const struct tegra194_cbb_aperture *noc_aperture;
+	const char * const *routeid_initflow;
+	const char * const *routeid_targflow;
+	void (*parse_routeid)(struct tegra194_cbb_aperture *info, u64 routeid);
+	void (*parse_userbits)(struct tegra194_cbb_userbits *usrbits, u32 elog_5);
+};
+
+struct tegra194_axi2apb_bridge {
+	struct resource res;
+	void __iomem *base;
+};
+
+struct tegra194_cbb {
+	struct tegra_cbb base;
+
+	const struct tegra194_cbb_noc_data *noc;
+	struct resource *res;
+
+	void __iomem *regs;
+	unsigned int num_intr;
+	unsigned int sec_irq;
+	unsigned int nonsec_irq;
+	u32 errlog0;
+	u32 errlog1;
+	u32 errlog2;
+	u32 errlog3;
+	u32 errlog4;
+	u32 errlog5;
+
+	struct tegra194_axi2apb_bridge *bridges;
+	unsigned int num_bridges;
+};
+
+static inline struct tegra194_cbb *to_tegra194_cbb(struct tegra_cbb *cbb)
+{
+	return container_of(cbb, struct tegra194_cbb, base);
+}
+
+static LIST_HEAD(cbb_list);
+static DEFINE_SPINLOCK(cbb_lock);
+
+static const char * const tegra194_cbb_trantype[] = {
+	"RD  - Read, Incrementing",
+	"RDW - Read, Wrap",			/* Not Supported */
+	"RDX - Exclusive Read",			/* Not Supported */
+	"RDL - Linked Read",			/* Not Supported */
+	"WR  - Write, Incrementing",
+	"WRW - Write, Wrap",			/* Not Supported */
+	"WRC - Exclusive Write",		/* Not Supported */
+	"PRE - Preamble Sequence for Fixed Accesses"
+};
+
+static const char * const tegra194_axi2apb_error[] = {
+	"SFIFONE - Status FIFO Not Empty interrupt",
+	"SFIFOF - Status FIFO Full interrupt",
+	"TIM - Timer(Timeout) interrupt",
+	"SLV - SLVERR interrupt",
+	"NULL",
+	"ERBF - Early response buffer Full interrupt",
+	"NULL",
+	"RDFIFOF - Read Response FIFO Full interrupt",
+	"WRFIFOF - Write Response FIFO Full interrupt",
+	"CH0DFIFOF - Ch0 Data FIFO Full interrupt",
+	"CH1DFIFOF - Ch1 Data FIFO Full interrupt",
+	"CH2DFIFOF - Ch2 Data FIFO Full interrupt",
+	"UAT - Unsupported alignment type error",
+	"UBS - Unsupported burst size error",
+	"UBE - Unsupported Byte Enable error",
+	"UBT - Unsupported burst type error",
+	"BFS - Block Firewall security error",
+	"ARFS - Address Range Firewall security error",
+	"CH0RFIFOF - Ch0 Request FIFO Full interrupt",
+	"CH1RFIFOF - Ch1 Request FIFO Full interrupt",
+	"CH2RFIFOF - Ch2 Request FIFO Full interrupt"
+};
+
+static const char * const tegra194_master_id[] = {
+	[0x0] = "CCPLEX",
+	[0x1] = "CCPLEX_DPMU",
+	[0x2] = "BPMP",
+	[0x3] = "AON",
+	[0x4] = "SCE",
+	[0x5] = "GPCDMA_PERIPHERAL",
+	[0x6] = "TSECA",
+	[0x7] = "TSECB",
+	[0x8] = "JTAGM_DFT",
+	[0x9] = "CORESIGHT_AXIAP",
+	[0xa] = "APE",
+	[0xb] = "PEATR",
+	[0xc] = "NVDEC",
+	[0xd] = "RCE",
+	[0xe] = "NVDEC1"
+};
+
+static const struct tegra_cbb_error tegra194_cbb_errors[] = {
+	{
+		.code = "SLV",
+		.source = "Target",
+		.desc = "Target error detected by CBB slave"
+	}, {
+		.code = "DEC",
+		.source = "Initiator NIU",
+		.desc = "Address decode error"
+	}, {
+		.code = "UNS",
+		.source = "Target NIU",
+		.desc = "Unsupported request. Not a valid transaction"
+	}, {
+		.code = "DISC", /* Not Supported by CBB */
+		.source = "Power Disconnect",
+		.desc = "Disconnected target or domain"
+	}, {
+		.code = "SEC",
+		.source = "Initiator NIU or Firewall",
+		.desc = "Security violation. Firewall error"
+	}, {
+		.code = "HIDE", /* Not Supported by CBB */
+		.source = "Firewall",
+		.desc = "Hidden security violation, reported as OK to initiator"
+	}, {
+		.code = "TMO",
+		.source = "Target NIU",
+		.desc = "Target time-out error"
+	}, {
+		.code = "RSV",
+		.source = "None",
+		.desc = "Reserved"
+	}
+};
+
+/*
+ * CBB NOC aperture lookup table as per file "cbb_central_noc_Structure.info".
+ */
+static const char * const tegra194_cbbcentralnoc_routeid_initflow[] = {
+	[0x0] = "aon_p2ps/I/aon",
+	[0x1] = "ape_p2ps/I/ape_p2ps",
+	[0x2] = "bpmp_p2ps/I/bpmp_p2ps",
+	[0x3] = "ccroc_p2ps/I/ccroc_p2ps",
+	[0x4] = "csite_p2ps/I/0",
+	[0x5] = "gpcdma_mmio_p2ps/I/0",
+	[0x6] = "jtag_p2ps/I/0",
+	[0x7] = "nvdec1_p2ps/I/0",
+	[0x8] = "nvdec_p2ps/I/0",
+	[0x9] = "rce_p2ps/I/rce_p2ps",
+	[0xa] = "sce_p2ps/I/sce_p2ps",
+	[0xb] = "tseca_p2ps/I/0",
+	[0xc] = "tsecb_p2ps/I/0",
+	[0xd] = "RESERVED",
+	[0xe] = "RESERVED",
+	[0xf] = "RESERVED"
+};
+
+static const char * const tegra194_cbbcentralnoc_routeid_targflow[] = {
+	[0x0] = "SVC/T/intreg",
+	[0x1] = "axis_satellite_axi2apb_p2pm/T/axis_satellite_axi2apb_p2pm",
+	[0x2] = "axis_satellite_grout/T/axis_satellite_grout",
+	[0x3] = "cbb_firewall/T/cbb_firewall",
+	[0x4] = "gpu_p2pm/T/gpu_p2pm",
+	[0x5] = "host1x_p2pm/T/host1x_p2pm",
+	[0x6] = "sapb_3_p2pm/T/sapb_3_p2pm",
+	[0x7] = "smmu0_p2pm/T/smmu0_p2pm",
+	[0x8] = "smmu1_p2pm/T/smmu1_p2pm",
+	[0x9] = "smmu2_p2pm/T/smmu2_p2pm",
+	[0xa] = "stm_p2pm/T/stm_p2pm",
+	[0xb] = "RESERVED",
+	[0xc] = "RESERVED",
+	[0xd] = "RESERVED",
+	[0xe] = "RESERVED",
+	[0xf] = "RESERVED"
+};
+
+/*
+ * Fields of CBB NOC lookup table:
+ * Init flow, Targ flow, Targ subrange, Init mapping, Init localAddress,
+ *                                              Targ mapping, Targ localAddress
+ * ----------------------------------------------------------------------------
+ */
+static const struct tegra194_cbb_aperture tegra194_cbbcentralnoc_apert_lookup[] = {
+	{ 0x0, 0x0, 0x00, 0x0, 0x02300000,  0, 0x00000000 },
+	{ 0x0, 0x1, 0x00, 0x0, 0x02003000,  0, 0x02003000 },
+	{ 0x0, 0x1, 0x01, 0x0, 0x02006000,  2, 0x02006000 },
+	{ 0x0, 0x1, 0x02, 0x0, 0x02016000,  3, 0x02016000 },
+	{ 0x0, 0x1, 0x03, 0x0, 0x0201d000,  4, 0x0201d000 },
+	{ 0x0, 0x1, 0x04, 0x0, 0x0202b000,  6, 0x0202b000 },
+	{ 0x0, 0x1, 0x05, 0x0, 0x02434000, 20, 0x02434000 },
+	{ 0x0, 0x1, 0x06, 0x0, 0x02436000, 21, 0x02436000 },
+	{ 0x0, 0x1, 0x07, 0x0, 0x02438000, 22, 0x02438000 },
+	{ 0x0, 0x1, 0x08, 0x0, 0x02445000, 24, 0x02445000 },
+	{ 0x0, 0x1, 0x09, 0x0, 0x02446000, 25, 0x02446000 },
+	{ 0x0, 0x1, 0x0a, 0x0, 0x02004000,  1, 0x02004000 },
+	{ 0x0, 0x1, 0x0b, 0x0, 0x0201e000,  5, 0x0201e000 },
+	{ 0x0, 0x1, 0x0c, 0x0, 0x0202c000,  7, 0x0202c000 },
+	{ 0x0, 0x1, 0x0d, 0x0, 0x02204000,  8, 0x02204000 },
+	{ 0x0, 0x1, 0x0e, 0x0, 0x02214000,  9, 0x02214000 },
+	{ 0x0, 0x1, 0x0f, 0x0, 0x02224000, 10, 0x02224000 },
+	{ 0x0, 0x1, 0x10, 0x0, 0x02234000, 11, 0x02234000 },
+	{ 0x0, 0x1, 0x11, 0x0, 0x02244000, 12, 0x02244000 },
+	{ 0x0, 0x1, 0x12, 0x0, 0x02254000, 13, 0x02254000 },
+	{ 0x0, 0x1, 0x13, 0x0, 0x02264000, 14, 0x02264000 },
+	{ 0x0, 0x1, 0x14, 0x0, 0x02274000, 15, 0x02274000 },
+	{ 0x0, 0x1, 0x15, 0x0, 0x02284000, 16, 0x02284000 },
+	{ 0x0, 0x1, 0x16, 0x0, 0x0243a000, 23, 0x0243a000 },
+	{ 0x0, 0x1, 0x17, 0x0, 0x02370000, 17, 0x02370000 },
+	{ 0x0, 0x1, 0x18, 0x0, 0x023d0000, 18, 0x023d0000 },
+	{ 0x0, 0x1, 0x19, 0x0, 0x023e0000, 19, 0x023e0000 },
+	{ 0x0, 0x1, 0x1a, 0x0, 0x02450000, 26, 0x02450000 },
+	{ 0x0, 0x1, 0x1b, 0x0, 0x02460000, 27, 0x02460000 },
+	{ 0x0, 0x1, 0x1c, 0x0, 0x02490000, 28, 0x02490000 },
+	{ 0x0, 0x1, 0x1d, 0x0, 0x03130000, 31, 0x03130000 },
+	{ 0x0, 0x1, 0x1e, 0x0, 0x03160000, 32, 0x03160000 },
+	{ 0x0, 0x1, 0x1f, 0x0, 0x03270000, 33, 0x03270000 },
+	{ 0x0, 0x1, 0x20, 0x0, 0x032e0000, 35, 0x032e0000 },
+	{ 0x0, 0x1, 0x21, 0x0, 0x03300000, 36, 0x03300000 },
+	{ 0x0, 0x1, 0x22, 0x0, 0x13090000, 40, 0x13090000 },
+	{ 0x0, 0x1, 0x23, 0x0, 0x20120000, 43, 0x20120000 },
+	{ 0x0, 0x1, 0x24, 0x0, 0x20170000, 44, 0x20170000 },
+	{ 0x0, 0x1, 0x25, 0x0, 0x20190000, 45, 0x20190000 },
+	{ 0x0, 0x1, 0x26, 0x0, 0x201b0000, 46, 0x201b0000 },
+	{ 0x0, 0x1, 0x27, 0x0, 0x20250000, 47, 0x20250000 },
+	{ 0x0, 0x1, 0x28, 0x0, 0x20260000, 48, 0x20260000 },
+	{ 0x0, 0x1, 0x29, 0x0, 0x20420000, 49, 0x20420000 },
+	{ 0x0, 0x1, 0x2a, 0x0, 0x20460000, 50, 0x20460000 },
+	{ 0x0, 0x1, 0x2b, 0x0, 0x204f0000, 51, 0x204f0000 },
+	{ 0x0, 0x1, 0x2c, 0x0, 0x20520000, 52, 0x20520000 },
+	{ 0x0, 0x1, 0x2d, 0x0, 0x20580000, 53, 0x20580000 },
+	{ 0x0, 0x1, 0x2e, 0x0, 0x205a0000, 54, 0x205a0000 },
+	{ 0x0, 0x1, 0x2f, 0x0, 0x205c0000, 55, 0x205c0000 },
+	{ 0x0, 0x1, 0x30, 0x0, 0x20690000, 56, 0x20690000 },
+	{ 0x0, 0x1, 0x31, 0x0, 0x20770000, 57, 0x20770000 },
+	{ 0x0, 0x1, 0x32, 0x0, 0x20790000, 58, 0x20790000 },
+	{ 0x0, 0x1, 0x33, 0x0, 0x20880000, 59, 0x20880000 },
+	{ 0x0, 0x1, 0x34, 0x0, 0x20990000, 62, 0x20990000 },
+	{ 0x0, 0x1, 0x35, 0x0, 0x20e10000, 65, 0x20e10000 },
+	{ 0x0, 0x1, 0x36, 0x0, 0x20e70000, 66, 0x20e70000 },
+	{ 0x0, 0x1, 0x37, 0x0, 0x20e80000, 67, 0x20e80000 },
+	{ 0x0, 0x1, 0x38, 0x0, 0x20f30000, 68, 0x20f30000 },
+	{ 0x0, 0x1, 0x39, 0x0, 0x20f50000, 69, 0x20f50000 },
+	{ 0x0, 0x1, 0x3a, 0x0, 0x20fc0000, 70, 0x20fc0000 },
+	{ 0x0, 0x1, 0x3b, 0x0, 0x21110000, 72, 0x21110000 },
+	{ 0x0, 0x1, 0x3c, 0x0, 0x21270000, 73, 0x21270000 },
+	{ 0x0, 0x1, 0x3d, 0x0, 0x21290000, 74, 0x21290000 },
+	{ 0x0, 0x1, 0x3e, 0x0, 0x21840000, 75, 0x21840000 },
+	{ 0x0, 0x1, 0x3f, 0x0, 0x21880000, 76, 0x21880000 },
+	{ 0x0, 0x1, 0x40, 0x0, 0x218d0000, 77, 0x218d0000 },
+	{ 0x0, 0x1, 0x41, 0x0, 0x21950000, 78, 0x21950000 },
+	{ 0x0, 0x1, 0x42, 0x0, 0x21960000, 79, 0x21960000 },
+	{ 0x0, 0x1, 0x43, 0x0, 0x21a10000, 80, 0x21a10000 },
+	{ 0x0, 0x1, 0x44, 0x0, 0x024a0000, 29, 0x024a0000 },
+	{ 0x0, 0x1, 0x45, 0x0, 0x024c0000, 30, 0x024c0000 },
+	{ 0x0, 0x1, 0x46, 0x0, 0x032c0000, 34, 0x032c0000 },
+	{ 0x0, 0x1, 0x47, 0x0, 0x03400000, 37, 0x03400000 },
+	{ 0x0, 0x1, 0x48, 0x0, 0x130a0000, 41, 0x130a0000 },
+	{ 0x0, 0x1, 0x49, 0x0, 0x130c0000, 42, 0x130c0000 },
+	{ 0x0, 0x1, 0x4a, 0x0, 0x208a0000, 60, 0x208a0000 },
+	{ 0x0, 0x1, 0x4b, 0x0, 0x208c0000, 61, 0x208c0000 },
+	{ 0x0, 0x1, 0x4c, 0x0, 0x209a0000, 63, 0x209a0000 },
+	{ 0x0, 0x1, 0x4d, 0x0, 0x21a40000, 81, 0x21a40000 },
+	{ 0x0, 0x1, 0x4e, 0x0, 0x03440000, 38, 0x03440000 },
+	{ 0x0, 0x1, 0x4f, 0x0, 0x20d00000, 64, 0x20d00000 },
+	{ 0x0, 0x1, 0x50, 0x0, 0x21000000, 71, 0x21000000 },
+	{ 0x0, 0x1, 0x51, 0x0, 0x0b000000, 39, 0x0b000000 },
+	{ 0x0, 0x2, 0x00, 0x0, 0x00000000,  0, 0x00000000 },
+	{ 0x0, 0x3, 0x00, 0x0, 0x02340000,  0, 0x00000000 },
+	{ 0x0, 0x4, 0x00, 0x0, 0x17000000,  0, 0x17000000 },
+	{ 0x0, 0x4, 0x01, 0x0, 0x18000000,  1, 0x18000000 },
+	{ 0x0, 0x5, 0x00, 0x0, 0x13e80000,  1, 0x13e80000 },
+	{ 0x0, 0x5, 0x01, 0x0, 0x15810000, 12, 0x15810000 },
+	{ 0x0, 0x5, 0x02, 0x0, 0x15840000, 14, 0x15840000 },
+	{ 0x0, 0x5, 0x03, 0x0, 0x15a40000, 17, 0x15a40000 },
+	{ 0x0, 0x5, 0x04, 0x0, 0x13f00000,  3, 0x13f00000 },
+	{ 0x0, 0x5, 0x05, 0x0, 0x15820000, 13, 0x15820000 },
+	{ 0x0, 0x5, 0x06, 0x0, 0x13ec0000,  2, 0x13ec0000 },
+	{ 0x0, 0x5, 0x07, 0x0, 0x15200000,  6, 0x15200000 },
+	{ 0x0, 0x5, 0x08, 0x0, 0x15340000,  7, 0x15340000 },
+	{ 0x0, 0x5, 0x09, 0x0, 0x15380000,  8, 0x15380000 },
+	{ 0x0, 0x5, 0x0a, 0x0, 0x15500000, 10, 0x15500000 },
+	{ 0x0, 0x5, 0x0b, 0x0, 0x155c0000, 11, 0x155c0000 },
+	{ 0x0, 0x5, 0x0c, 0x0, 0x15a00000, 16, 0x15a00000 },
+	{ 0x0, 0x5, 0x0d, 0x0, 0x13e00000,  0, 0x13e00000 },
+	{ 0x0, 0x5, 0x0e, 0x0, 0x15100000,  5, 0x15100000 },
+	{ 0x0, 0x5, 0x0f, 0x0, 0x15480000,  9, 0x15480000 },
+	{ 0x0, 0x5, 0x10, 0x0, 0x15880000, 15, 0x15880000 },
+	{ 0x0, 0x5, 0x11, 0x0, 0x15a80000, 18, 0x15a80000 },
+	{ 0x0, 0x5, 0x12, 0x0, 0x15b00000, 19, 0x15b00000 },
+	{ 0x0, 0x5, 0x13, 0x0, 0x14800000,  4, 0x14800000 },
+	{ 0x0, 0x5, 0x14, 0x0, 0x15c00000, 20, 0x15c00000 },
+	{ 0x0, 0x5, 0x15, 0x0, 0x16000000, 21, 0x16000000 },
+	{ 0x0, 0x6, 0x00, 0x0, 0x02000000,  4, 0x02000000 },
+	{ 0x0, 0x6, 0x01, 0x0, 0x02007000,  5, 0x02007000 },
+	{ 0x0, 0x6, 0x02, 0x0, 0x02008000,  6, 0x02008000 },
+	{ 0x0, 0x6, 0x03, 0x0, 0x02013000,  7, 0x02013000 },
+	{ 0x0, 0x6, 0x04, 0x0, 0x0201c000,  8, 0x0201c000 },
+	{ 0x0, 0x6, 0x05, 0x0, 0x02020000,  9, 0x02020000 },
+	{ 0x0, 0x6, 0x06, 0x0, 0x0202a000, 10, 0x0202a000 },
+	{ 0x0, 0x6, 0x07, 0x0, 0x0202e000, 11, 0x0202e000 },
+	{ 0x0, 0x6, 0x08, 0x0, 0x06400000, 33, 0x06400000 },
+	{ 0x0, 0x6, 0x09, 0x0, 0x02038000, 12, 0x02038000 },
+	{ 0x0, 0x6, 0x0a, 0x0, 0x00100000,  0, 0x00100000 },
+	{ 0x0, 0x6, 0x0b, 0x0, 0x023b0000, 13, 0x023b0000 },
+	{ 0x0, 0x6, 0x0c, 0x0, 0x02800000, 16, 0x02800000 },
+	{ 0x0, 0x6, 0x0d, 0x0, 0x030e0000, 22, 0x030e0000 },
+	{ 0x0, 0x6, 0x0e, 0x0, 0x03800000, 23, 0x03800000 },
+	{ 0x0, 0x6, 0x0f, 0x0, 0x03980000, 25, 0x03980000 },
+	{ 0x0, 0x6, 0x10, 0x0, 0x03a60000, 26, 0x03a60000 },
+	{ 0x0, 0x6, 0x11, 0x0, 0x03d80000, 31, 0x03d80000 },
+	{ 0x0, 0x6, 0x12, 0x0, 0x20000000, 36, 0x20000000 },
+	{ 0x0, 0x6, 0x13, 0x0, 0x20050000, 38, 0x20050000 },
+	{ 0x0, 0x6, 0x14, 0x0, 0x201e0000, 40, 0x201e0000 },
+	{ 0x0, 0x6, 0x15, 0x0, 0x20280000, 42, 0x20280000 },
+	{ 0x0, 0x6, 0x16, 0x0, 0x202c0000, 43, 0x202c0000 },
+	{ 0x0, 0x6, 0x17, 0x0, 0x20390000, 44, 0x20390000 },
+	{ 0x0, 0x6, 0x18, 0x0, 0x20430000, 45, 0x20430000 },
+	{ 0x0, 0x6, 0x19, 0x0, 0x20440000, 46, 0x20440000 },
+	{ 0x0, 0x6, 0x1a, 0x0, 0x204e0000, 47, 0x204e0000 },
+	{ 0x0, 0x6, 0x1b, 0x0, 0x20550000, 48, 0x20550000 },
+	{ 0x0, 0x6, 0x1c, 0x0, 0x20570000, 49, 0x20570000 },
+	{ 0x0, 0x6, 0x1d, 0x0, 0x20590000, 50, 0x20590000 },
+	{ 0x0, 0x6, 0x1e, 0x0, 0x20730000, 52, 0x20730000 },
+	{ 0x0, 0x6, 0x1f, 0x0, 0x209f0000, 54, 0x209f0000 },
+	{ 0x0, 0x6, 0x20, 0x0, 0x20e20000, 55, 0x20e20000 },
+	{ 0x0, 0x6, 0x21, 0x0, 0x20ed0000, 56, 0x20ed0000 },
+	{ 0x0, 0x6, 0x22, 0x0, 0x20fd0000, 57, 0x20fd0000 },
+	{ 0x0, 0x6, 0x23, 0x0, 0x21120000, 59, 0x21120000 },
+	{ 0x0, 0x6, 0x24, 0x0, 0x211a0000, 60, 0x211a0000 },
+	{ 0x0, 0x6, 0x25, 0x0, 0x21850000, 61, 0x21850000 },
+	{ 0x0, 0x6, 0x26, 0x0, 0x21860000, 62, 0x21860000 },
+	{ 0x0, 0x6, 0x27, 0x0, 0x21890000, 63, 0x21890000 },
+	{ 0x0, 0x6, 0x28, 0x0, 0x21970000, 64, 0x21970000 },
+	{ 0x0, 0x6, 0x29, 0x0, 0x21990000, 65, 0x21990000 },
+	{ 0x0, 0x6, 0x2a, 0x0, 0x21a00000, 66, 0x21a00000 },
+	{ 0x0, 0x6, 0x2b, 0x0, 0x21a90000, 68, 0x21a90000 },
+	{ 0x0, 0x6, 0x2c, 0x0, 0x21ac0000, 70, 0x21ac0000 },
+	{ 0x0, 0x6, 0x2d, 0x0, 0x01f80000,  3, 0x01f80000 },
+	{ 0x0, 0x6, 0x2e, 0x0, 0x024e0000, 14, 0x024e0000 },
+	{ 0x0, 0x6, 0x2f, 0x0, 0x030c0000, 21, 0x030c0000 },
+	{ 0x0, 0x6, 0x30, 0x0, 0x03820000, 24, 0x03820000 },
+	{ 0x0, 0x6, 0x31, 0x0, 0x03aa0000, 27, 0x03aa0000 },
+	{ 0x0, 0x6, 0x32, 0x0, 0x03c80000, 29, 0x03c80000 },
+	{ 0x0, 0x6, 0x33, 0x0, 0x130e0000, 34, 0x130e0000 },
+	{ 0x0, 0x6, 0x34, 0x0, 0x20020000, 37, 0x20020000 },
+	{ 0x0, 0x6, 0x35, 0x0, 0x20060000, 39, 0x20060000 },
+	{ 0x0, 0x6, 0x36, 0x0, 0x20200000, 41, 0x20200000 },
+	{ 0x0, 0x6, 0x37, 0x0, 0x206a0000, 51, 0x206a0000 },
+	{ 0x0, 0x6, 0x38, 0x0, 0x20740000, 53, 0x20740000 },
+	{ 0x0, 0x6, 0x39, 0x0, 0x20fe0000, 58, 0x20fe0000 },
+	{ 0x0, 0x6, 0x3a, 0x0, 0x21a20000, 67, 0x21a20000 },
+	{ 0x0, 0x6, 0x3b, 0x0, 0x21aa0000, 69, 0x21aa0000 },
+	{ 0x0, 0x6, 0x3c, 0x0, 0x02b80000, 17, 0x02b80000 },
+	{ 0x0, 0x6, 0x3d, 0x0, 0x03080000, 20, 0x03080000 },
+	{ 0x0, 0x6, 0x3e, 0x0, 0x13100000, 35, 0x13100000 },
+	{ 0x0, 0x6, 0x3f, 0x0, 0x01f00000,  2, 0x01f00000 },
+	{ 0x0, 0x6, 0x40, 0x0, 0x03000000, 19, 0x03000000 },
+	{ 0x0, 0x6, 0x41, 0x0, 0x03c00000, 28, 0x03c00000 },
+	{ 0x0, 0x6, 0x42, 0x0, 0x03d00000, 30, 0x03d00000 },
+	{ 0x0, 0x6, 0x43, 0x0, 0x01700000,  1, 0x01700000 },
+	{ 0x0, 0x6, 0x44, 0x0, 0x02c00000, 18, 0x02c00000 },
+	{ 0x0, 0x6, 0x45, 0x0, 0x02600000, 15, 0x02600000 },
+	{ 0x0, 0x6, 0x46, 0x0, 0x06000000, 32, 0x06000000 },
+	{ 0x0, 0x6, 0x47, 0x0, 0x24000000, 71, 0x24000000 },
+	{ 0x0, 0x7, 0x00, 0x0, 0x12000000,  0, 0x12000000 },
+	{ 0x0, 0x8, 0x00, 0x0, 0x11000000,  0, 0x11000000 },
+	{ 0x0, 0x9, 0x00, 0x0, 0x10000000,  0, 0x10000000 },
+	{ 0x0, 0xa, 0x00, 0x0, 0x22000000,  0, 0x22000000 }
+};
+
+/*
+ * BPMP NOC aperture lookup table as per file "BPMP_NOC_Structure.info".
+ */
+static const char * const tegra194_bpmpnoc_routeid_initflow[] = {
+	[0x0] = "cbb_i/I/0",
+	[0x1] = "cpu_m_i/I/0",
+	[0x2] = "cpu_p_i/I/0",
+	[0x3] = "cvc_i/I/0",
+	[0x4] = "dma_m_i/I/0",
+	[0x5] = "dma_p_i/I/0",
+	[0x6] = "RESERVED",
+	[0x7] = "RESERVED"
+};
+
+static const char * const tegra194_bpmpnoc_routeid_targflow[] = {
+	[0x00] = "multiport0_t/T/actmon",
+	[0x01] = "multiport0_t/T/ast_0",
+	[0x02] = "multiport0_t/T/ast_1",
+	[0x03] = "multiport0_t/T/atcm_cfg",
+	[0x04] = "multiport0_t/T/car",
+	[0x05] = "multiport0_t/T/central_pwr_mgr",
+	[0x06] = "multiport0_t/T/central_vtg_ctlr",
+	[0x07] = "multiport0_t/T/cfg",
+	[0x08] = "multiport0_t/T/dma",
+	[0x09] = "multiport0_t/T/err_collator",
+	[0x0a] = "multiport0_t/T/err_collator_car",
+	[0x0b] = "multiport0_t/T/fpga_misc",
+	[0x0c] = "multiport0_t/T/fpga_uart",
+	[0x0d] = "multiport0_t/T/gte",
+	[0x0e] = "multiport0_t/T/hsp",
+	[0x0f] = "multiport0_t/T/misc",
+	[0x10] = "multiport0_t/T/pm",
+	[0x11] = "multiport0_t/T/simon0",
+	[0x12] = "multiport0_t/T/simon1",
+	[0x13] = "multiport0_t/T/simon2",
+	[0x14] = "multiport0_t/T/simon3",
+	[0x15] = "multiport0_t/T/simon4",
+	[0x16] = "multiport0_t/T/soc_therm",
+	[0x17] = "multiport0_t/T/tke",
+	[0x18] = "multiport0_t/T/vic_0",
+	[0x19] = "multiport0_t/T/vic_1",
+	[0x1a] = "ast0_t/T/0",
+	[0x1b] = "ast1_t/T/0",
+	[0x1c] = "bpmp_noc_firewall/T/0",
+	[0x1d] = "cbb_t/T/0",
+	[0x1e] = "cpu_t/T/0",
+	[0x1f] = "svc_t/T/0"
+};
+
+/*
+ * Fields of BPMP NOC lookup table:
+ * Init flow, Targ flow, Targ subrange, Init mapping, Init localAddress,
+ *                                              Targ mapping, Targ localAddress
+ * ----------------------------------------------------------------------------
+ */
+static const struct tegra194_cbb_aperture tegra194_bpmpnoc_apert_lookup[] = {
+	{ 0x0, 0x1c, 0x0, 0x0, 0x0d640000, 0,  0x00000000 },
+	{ 0x0, 0x1e, 0x0, 0x0, 0x0d400000, 0,  0x0d400000 },
+	{ 0x0, 0x00, 0x0, 0x0, 0x0d230000, 0,  0x00000000 },
+	{ 0x0, 0x01, 0x0, 0x0, 0x0d040000, 0,  0x00000000 },
+	{ 0x0, 0x02, 0x0, 0x0, 0x0d050000, 0,  0x00000000 },
+	{ 0x0, 0x03, 0x0, 0x0, 0x0d000000, 0,  0x00000000 },
+	{ 0x0, 0x04, 0x0, 0x0, 0x20ae0000, 3,  0x000e0000 },
+	{ 0x0, 0x04, 0x1, 0x0, 0x20ac0000, 2,  0x000c0000 },
+	{ 0x0, 0x04, 0x2, 0x0, 0x20a80000, 1,  0x00080000 },
+	{ 0x0, 0x04, 0x3, 0x0, 0x20a00000, 0,  0x00000000 },
+	{ 0x0, 0x05, 0x0, 0x0, 0x0d2a0000, 0,  0x00000000 },
+	{ 0x0, 0x06, 0x0, 0x0, 0x0d290000, 0,  0x00000000 },
+	{ 0x0, 0x07, 0x0, 0x0, 0x0d2c0000, 0,  0x00000000 },
+	{ 0x0, 0x08, 0x0, 0x0, 0x0d0e0000, 4,  0x00080000 },
+	{ 0x0, 0x08, 0x1, 0x0, 0x0d060000, 0,  0x00000000 },
+	{ 0x0, 0x08, 0x2, 0x0, 0x0d080000, 1,  0x00020000 },
+	{ 0x0, 0x08, 0x3, 0x0, 0x0d0a0000, 2,  0x00040000 },
+	{ 0x0, 0x08, 0x4, 0x0, 0x0d0c0000, 3,  0x00060000 },
+	{ 0x0, 0x09, 0x0, 0x0, 0x0d650000, 0,  0x00000000 },
+	{ 0x0, 0x0a, 0x0, 0x0, 0x20af0000, 0,  0x00000000 },
+	{ 0x0, 0x0b, 0x0, 0x0, 0x0d3e0000, 0,  0x00000000 },
+	{ 0x0, 0x0c, 0x0, 0x0, 0x0d3d0000, 0,  0x00000000 },
+	{ 0x0, 0x0d, 0x0, 0x0, 0x0d1e0000, 0,  0x00000000 },
+	{ 0x0, 0x0e, 0x0, 0x0, 0x0d150000, 0,  0x00000000 },
+	{ 0x0, 0x0e, 0x1, 0x0, 0x0d160000, 1,  0x00010000 },
+	{ 0x0, 0x0e, 0x2, 0x0, 0x0d170000, 2,  0x00020000 },
+	{ 0x0, 0x0e, 0x3, 0x0, 0x0d180000, 3,  0x00030000 },
+	{ 0x0, 0x0e, 0x4, 0x0, 0x0d190000, 4,  0x00040000 },
+	{ 0x0, 0x0e, 0x5, 0x0, 0x0d1a0000, 5,  0x00050000 },
+	{ 0x0, 0x0e, 0x6, 0x0, 0x0d1b0000, 6,  0x00060000 },
+	{ 0x0, 0x0e, 0x7, 0x0, 0x0d1c0000, 7,  0x00070000 },
+	{ 0x0, 0x0e, 0x8, 0x0, 0x0d1d0000, 8,  0x00080000 },
+	{ 0x0, 0x0f, 0x0, 0x0, 0x0d660000, 0,  0x00000000 },
+	{ 0x0, 0x10, 0x0, 0x0, 0x0d1f0000, 0,  0x00000000 },
+	{ 0x0, 0x10, 0x1, 0x0, 0x0d200000, 1,  0x00010000 },
+	{ 0x0, 0x10, 0x2, 0x0, 0x0d210000, 2,  0x00020000 },
+	{ 0x0, 0x10, 0x3, 0x0, 0x0d220000, 3,  0x00030000 },
+	{ 0x0, 0x11, 0x0, 0x0, 0x0d240000, 0,  0x00000000 },
+	{ 0x0, 0x12, 0x0, 0x0, 0x0d250000, 0,  0x00000000 },
+	{ 0x0, 0x13, 0x0, 0x0, 0x0d260000, 0,  0x00000000 },
+	{ 0x0, 0x14, 0x0, 0x0, 0x0d270000, 0,  0x00000000 },
+	{ 0x0, 0x15, 0x0, 0x0, 0x0d2b0000, 0,  0x00000000 },
+	{ 0x0, 0x16, 0x0, 0x0, 0x0d280000, 0,  0x00000000 },
+	{ 0x0, 0x17, 0x0, 0x0, 0x0d0f0000, 0,  0x00000000 },
+	{ 0x0, 0x17, 0x1, 0x0, 0x0d100000, 1,  0x00010000 },
+	{ 0x0, 0x17, 0x2, 0x0, 0x0d110000, 2,  0x00020000 },
+	{ 0x0, 0x17, 0x3, 0x0, 0x0d120000, 3,  0x00030000 },
+	{ 0x0, 0x17, 0x4, 0x0, 0x0d130000, 4,  0x00040000 },
+	{ 0x0, 0x17, 0x5, 0x0, 0x0d140000, 5,  0x00050000 },
+	{ 0x0, 0x18, 0x0, 0x0, 0x0d020000, 0,  0x00000000 },
+	{ 0x0, 0x19, 0x0, 0x0, 0x0d030000, 0,  0x00000000 },
+	{ 0x0, 0x1f, 0x0, 0x0, 0x0d600000, 0,  0x00000000 },
+	{ 0x0, 0x1f, 0x1, 0x0, 0x00000000, 0,  0x00000000 },
+	{ 0x1, 0x1a, 0x0, 0x0, 0x40000000, 0,  0x40000000 },
+	{ 0x1, 0x1a, 0x1, 0x1, 0x80000000, 1,  0x80000000 },
+	{ 0x1, 0x1a, 0x2, 0x0, 0x00000000, 0,  0x00000000 },
+	{ 0x2, 0x1c, 0x0, 0x0, 0x0d640000, 0,  0x00000000 },
+	{ 0x2, 0x1d, 0x0, 0x0, 0x20b00000, 8,  0x20b00000 },
+	{ 0x2, 0x1d, 0x1, 0x0, 0x20800000, 7,  0x20800000 },
+	{ 0x2, 0x1d, 0x2, 0x0, 0x20c00000, 9,  0x20c00000 },
+	{ 0x2, 0x1d, 0x3, 0x0, 0x0d800000, 3,  0x0d800000 },
+	{ 0x2, 0x1d, 0x4, 0x0, 0x20000000, 6,  0x20000000 },
+	{ 0x2, 0x1d, 0x5, 0x0, 0x0c000000, 2,  0x0c000000 },
+	{ 0x2, 0x1d, 0x6, 0x0, 0x21000000, 10, 0x21000000 },
+	{ 0x2, 0x1d, 0x7, 0x0, 0x0e000000, 4,  0x0e000000 },
+	{ 0x2, 0x1d, 0x8, 0x0, 0x22000000, 11, 0x22000000 },
+	{ 0x2, 0x1d, 0x9, 0x0, 0x08000000, 1,  0x08000000 },
+	{ 0x2, 0x1d, 0xa, 0x0, 0x24000000, 12, 0x24000000 },
+	{ 0x2, 0x1d, 0xb, 0x0, 0x00000000, 0,  0x00000000 },
+	{ 0x2, 0x1d, 0xc, 0x0, 0x28000000, 13, 0x28000000 },
+	{ 0x2, 0x1d, 0xd, 0x0, 0x10000000, 5,  0x10000000 },
+	{ 0x2, 0x1d, 0xe, 0x0, 0x30000000, 14, 0x30000000 },
+	{ 0x2, 0x00, 0x0, 0x0, 0x0d230000, 0,  0x00000000 },
+	{ 0x2, 0x01, 0x0, 0x0, 0x0d040000, 0,  0x00000000 },
+	{ 0x2, 0x02, 0x0, 0x0, 0x0d050000, 0,  0x00000000 },
+	{ 0x2, 0x03, 0x0, 0x0, 0x0d000000, 0,  0x00000000 },
+	{ 0x2, 0x04, 0x0, 0x0, 0x20ae0000, 3,  0x000e0000 },
+	{ 0x2, 0x04, 0x1, 0x0, 0x20ac0000, 2,  0x000c0000 },
+	{ 0x2, 0x04, 0x2, 0x0, 0x20a80000, 1,  0x00080000 },
+	{ 0x2, 0x04, 0x3, 0x0, 0x20a00000, 0,  0x00000000 },
+	{ 0x2, 0x05, 0x0, 0x0, 0x0d2a0000, 0,  0x00000000 },
+	{ 0x2, 0x06, 0x0, 0x0, 0x0d290000, 0,  0x00000000 },
+	{ 0x2, 0x07, 0x0, 0x0, 0x0d2c0000, 0,  0x00000000 },
+	{ 0x2, 0x08, 0x0, 0x0, 0x0d0e0000, 4,  0x00080000 },
+	{ 0x2, 0x08, 0x1, 0x0, 0x0d060000, 0,  0x00000000 },
+	{ 0x2, 0x08, 0x2, 0x0, 0x0d080000, 1,  0x00020000 },
+	{ 0x2, 0x08, 0x3, 0x0, 0x0d0a0000, 2,  0x00040000 },
+	{ 0x2, 0x08, 0x4, 0x0, 0x0d0c0000, 3,  0x00060000 },
+	{ 0x2, 0x09, 0x0, 0x0, 0x0d650000, 0,  0x00000000 },
+	{ 0x2, 0x0a, 0x0, 0x0, 0x20af0000, 0,  0x00000000 },
+	{ 0x2, 0x0b, 0x0, 0x0, 0x0d3e0000, 0,  0x00000000 },
+	{ 0x2, 0x0c, 0x0, 0x0, 0x0d3d0000, 0,  0x00000000 },
+	{ 0x2, 0x0d, 0x0, 0x0, 0x0d1e0000, 0,  0x00000000 },
+	{ 0x2, 0x0e, 0x0, 0x0, 0x0d150000, 0,  0x00000000 },
+	{ 0x2, 0x0e, 0x1, 0x0, 0x0d160000, 1,  0x00010000 },
+	{ 0x2, 0x0e, 0x2, 0x0, 0x0d170000, 2,  0x00020000 },
+	{ 0x2, 0x0e, 0x3, 0x0, 0x0d180000, 3,  0x00030000 },
+	{ 0x2, 0x0e, 0x4, 0x0, 0x0d190000, 4,  0x00040000 },
+	{ 0x2, 0x0e, 0x5, 0x0, 0x0d1a0000, 5,  0x00050000 },
+	{ 0x2, 0x0e, 0x6, 0x0, 0x0d1b0000, 6,  0x00060000 },
+	{ 0x2, 0x0e, 0x7, 0x0, 0x0d1c0000, 7,  0x00070000 },
+	{ 0x2, 0x0e, 0x8, 0x0, 0x0d1d0000, 8,  0x00080000 },
+	{ 0x2, 0x0f, 0x0, 0x0, 0x0d660000, 0,  0x00000000 },
+	{ 0x2, 0x10, 0x0, 0x0, 0x0d1f0000, 0,  0x00000000 },
+	{ 0x2, 0x10, 0x1, 0x0, 0x0d200000, 1,  0x00010000 },
+	{ 0x2, 0x10, 0x2, 0x0, 0x0d210000, 2,  0x00020000 },
+	{ 0x2, 0x10, 0x3, 0x0, 0x0d220000, 3,  0x00030000 },
+	{ 0x2, 0x11, 0x0, 0x0, 0x0d240000, 0,  0x00000000 },
+	{ 0x2, 0x12, 0x0, 0x0, 0x0d250000, 0,  0x00000000 },
+	{ 0x2, 0x13, 0x0, 0x0, 0x0d260000, 0,  0x00000000 },
+	{ 0x2, 0x14, 0x0, 0x0, 0x0d270000, 0,  0x00000000 },
+	{ 0x2, 0x15, 0x0, 0x0, 0x0d2b0000, 0,  0x00000000 },
+	{ 0x2, 0x16, 0x0, 0x0, 0x0d280000, 0,  0x00000000 },
+	{ 0x2, 0x17, 0x0, 0x0, 0x0d0f0000, 0,  0x00000000 },
+	{ 0x2, 0x17, 0x1, 0x0, 0x0d100000, 1,  0x00010000 },
+	{ 0x2, 0x17, 0x2, 0x0, 0x0d110000, 2,  0x00020000 },
+	{ 0x2, 0x17, 0x3, 0x0, 0x0d120000, 3,  0x00030000 },
+	{ 0x2, 0x17, 0x4, 0x0, 0x0d130000, 4,  0x00040000 },
+	{ 0x2, 0x17, 0x5, 0x0, 0x0d140000, 5,  0x00050000 },
+	{ 0x2, 0x18, 0x0, 0x0, 0x0d020000, 0,  0x00000000 },
+	{ 0x2, 0x19, 0x0, 0x0, 0x0d030000, 0,  0x00000000 },
+	{ 0x2, 0x1f, 0x0, 0x0, 0x0d600000, 0,  0x00000000 },
+	{ 0x2, 0x1f, 0x1, 0x0, 0x00000000, 0,  0x00000000 },
+	{ 0x3, 0x1b, 0x0, 0x0, 0x40000000, 0,  0x40000000 },
+	{ 0x3, 0x1b, 0x1, 0x1, 0x80000000, 1,  0x80000000 },
+	{ 0x3, 0x1c, 0x0, 0x2, 0x0d640000, 0,  0x00000000 },
+	{ 0x3, 0x1d, 0x0, 0x2, 0x20b00000, 8,  0x20b00000 },
+	{ 0x3, 0x1d, 0x1, 0x2, 0x20800000, 7,  0x20800000 },
+	{ 0x3, 0x1d, 0x2, 0x2, 0x20c00000, 9,  0x20c00000 },
+	{ 0x3, 0x1d, 0x3, 0x2, 0x0d800000, 3,  0x0d800000 },
+	{ 0x3, 0x1d, 0x4, 0x2, 0x20000000, 6,  0x20000000 },
+	{ 0x3, 0x1d, 0x5, 0x2, 0x0c000000, 2,  0x0c000000 },
+	{ 0x3, 0x1d, 0x6, 0x2, 0x21000000, 10, 0x21000000 },
+	{ 0x3, 0x1d, 0x7, 0x2, 0x0e000000, 4,  0x0e000000 },
+	{ 0x3, 0x1d, 0x8, 0x2, 0x22000000, 11, 0x22000000 },
+	{ 0x3, 0x1d, 0x9, 0x2, 0x08000000, 1,  0x08000000 },
+	{ 0x3, 0x1d, 0xa, 0x2, 0x24000000, 12, 0x24000000 },
+	{ 0x3, 0x1d, 0xb, 0x2, 0x00000000, 0,  0x00000000 },
+	{ 0x3, 0x1d, 0xc, 0x2, 0x28000000, 13, 0x28000000 },
+	{ 0x3, 0x1d, 0xd, 0x2, 0x10000000, 5,  0x10000000 },
+	{ 0x3, 0x1d, 0xe, 0x2, 0x30000000, 14, 0x30000000 },
+	{ 0x3, 0x1e, 0x0, 0x2, 0x0d400000, 0,  0x0d400000 },
+	{ 0x3, 0x00, 0x0, 0x2, 0x0d230000, 0,  0x00000000 },
+	{ 0x3, 0x01, 0x0, 0x2, 0x0d040000, 0,  0x00000000 },
+	{ 0x3, 0x02, 0x0, 0x2, 0x0d050000, 0,  0x00000000 },
+	{ 0x3, 0x03, 0x0, 0x2, 0x0d000000, 0,  0x00000000 },
+	{ 0x3, 0x04, 0x0, 0x2, 0x20ae0000, 3,  0x000e0000 },
+	{ 0x3, 0x04, 0x1, 0x2, 0x20ac0000, 2,  0x000c0000 },
+	{ 0x3, 0x04, 0x2, 0x2, 0x20a80000, 1,  0x00080000 },
+	{ 0x3, 0x04, 0x3, 0x2, 0x20a00000, 0,  0x00000000 },
+	{ 0x3, 0x05, 0x0, 0x2, 0x0d2a0000, 0,  0x00000000 },
+	{ 0x3, 0x06, 0x0, 0x2, 0x0d290000, 0,  0x00000000 },
+	{ 0x3, 0x07, 0x0, 0x2, 0x0d2c0000, 0,  0x00000000 },
+	{ 0x3, 0x08, 0x0, 0x2, 0x0d0e0000, 4,  0x00080000 },
+	{ 0x3, 0x08, 0x1, 0x2, 0x0d060000, 0,  0x00000000 },
+	{ 0x3, 0x08, 0x2, 0x2, 0x0d080000, 1,  0x00020000 },
+	{ 0x3, 0x08, 0x3, 0x2, 0x0d0a0000, 2,  0x00040000 },
+	{ 0x3, 0x08, 0x4, 0x2, 0x0d0c0000, 3,  0x00060000 },
+	{ 0x3, 0x09, 0x0, 0x2, 0x0d650000, 0,  0x00000000 },
+	{ 0x3, 0x0a, 0x0, 0x2, 0x20af0000, 0,  0x00000000 },
+	{ 0x3, 0x0b, 0x0, 0x2, 0x0d3e0000, 0,  0x00000000 },
+	{ 0x3, 0x0c, 0x0, 0x2, 0x0d3d0000, 0,  0x00000000 },
+	{ 0x3, 0x0d, 0x0, 0x2, 0x0d1e0000, 0,  0x00000000 },
+	{ 0x3, 0x0e, 0x0, 0x2, 0x0d150000, 0,  0x00000000 },
+	{ 0x3, 0x0e, 0x1, 0x2, 0x0d160000, 1,  0x00010000 },
+	{ 0x3, 0x0e, 0x2, 0x2, 0x0d170000, 2,  0x00020000 },
+	{ 0x3, 0x0e, 0x3, 0x2, 0x0d180000, 3,  0x00030000 },
+	{ 0x3, 0x0e, 0x4, 0x2, 0x0d190000, 4,  0x00040000 },
+	{ 0x3, 0x0e, 0x5, 0x2, 0x0d1a0000, 5,  0x00050000 },
+	{ 0x3, 0x0e, 0x6, 0x2, 0x0d1b0000, 6,  0x00060000 },
+	{ 0x3, 0x0e, 0x7, 0x2, 0x0d1c0000, 7,  0x00070000 },
+	{ 0x3, 0x0e, 0x8, 0x2, 0x0d1d0000, 8,  0x00080000 },
+	{ 0x3, 0x0f, 0x0, 0x2, 0x0d660000, 0,  0x00000000 },
+	{ 0x3, 0x10, 0x0, 0x2, 0x0d1f0000, 0,  0x00000000 },
+	{ 0x3, 0x10, 0x1, 0x2, 0x0d200000, 1,  0x00010000 },
+	{ 0x3, 0x10, 0x2, 0x2, 0x0d210000, 2,  0x00020000 },
+	{ 0x3, 0x10, 0x3, 0x2, 0x0d220000, 3,  0x00030000 },
+	{ 0x3, 0x11, 0x0, 0x2, 0x0d240000, 0,  0x00000000 },
+	{ 0x3, 0x12, 0x0, 0x2, 0x0d250000, 0,  0x00000000 },
+	{ 0x3, 0x13, 0x0, 0x2, 0x0d260000, 0,  0x00000000 },
+	{ 0x3, 0x14, 0x0, 0x2, 0x0d270000, 0,  0x00000000 },
+	{ 0x3, 0x15, 0x0, 0x2, 0x0d2b0000, 0,  0x00000000 },
+	{ 0x3, 0x16, 0x0, 0x2, 0x0d280000, 0,  0x00000000 },
+	{ 0x3, 0x17, 0x0, 0x2, 0x0d0f0000, 0,  0x00000000 },
+	{ 0x3, 0x17, 0x1, 0x2, 0x0d100000, 1,  0x00010000 },
+	{ 0x3, 0x17, 0x2, 0x2, 0x0d110000, 2,  0x00020000 },
+	{ 0x3, 0x17, 0x3, 0x2, 0x0d120000, 3,  0x00030000 },
+	{ 0x3, 0x17, 0x4, 0x2, 0x0d130000, 4,  0x00040000 },
+	{ 0x3, 0x17, 0x5, 0x2, 0x0d140000, 5,  0x00050000 },
+	{ 0x3, 0x18, 0x0, 0x2, 0x0d020000, 0,  0x00000000 },
+	{ 0x3, 0x19, 0x0, 0x2, 0x0d030000, 0,  0x00000000 },
+	{ 0x3, 0x1f, 0x0, 0x2, 0x0d600000, 0,  0x00000000 },
+	{ 0x3, 0x1f, 0x1, 0x0, 0x00000000, 0,  0x00000000 },
+	{ 0x4, 0x1b, 0x0, 0x0, 0x40000000, 0,  0x40000000 },
+	{ 0x4, 0x1b, 0x1, 0x1, 0x80000000, 1,  0x80000000 },
+	{ 0x4, 0x1e, 0x0, 0x2, 0x0d400000, 0,  0x0d400000 },
+	{ 0x4, 0x1e, 0x1, 0x0, 0x00000000, 0,  0x00000000 },
+	{ 0x5, 0x1c, 0x0, 0x0, 0x0d640000, 0,  0x00000000 },
+	{ 0x5, 0x1d, 0x0, 0x0, 0x20b00000, 8,  0x20b00000 },
+	{ 0x5, 0x1d, 0x1, 0x0, 0x20800000, 7,  0x20800000 },
+	{ 0x5, 0x1d, 0x2, 0x0, 0x20c00000, 9,  0x20c00000 },
+	{ 0x5, 0x1d, 0x3, 0x0, 0x0d800000, 3,  0x0d800000 },
+	{ 0x5, 0x1d, 0x4, 0x0, 0x20000000, 6,  0x20000000 },
+	{ 0x5, 0x1d, 0x5, 0x0, 0x0c000000, 2,  0x0c000000 },
+	{ 0x5, 0x1d, 0x6, 0x0, 0x21000000, 10, 0x21000000 },
+	{ 0x5, 0x1d, 0x7, 0x0, 0x0e000000, 4,  0x0e000000 },
+	{ 0x5, 0x1d, 0x8, 0x0, 0x22000000, 11, 0x22000000 },
+	{ 0x5, 0x1d, 0x9, 0x0, 0x08000000, 1,  0x08000000 },
+	{ 0x5, 0x1d, 0xa, 0x0, 0x24000000, 12, 0x24000000 },
+	{ 0x5, 0x1d, 0xb, 0x0, 0x00000000, 0,  0x00000000 },
+	{ 0x5, 0x1d, 0xc, 0x0, 0x28000000, 13, 0x28000000 },
+	{ 0x5, 0x1d, 0xd, 0x0, 0x10000000, 5,  0x10000000 },
+	{ 0x5, 0x1d, 0xe, 0x0, 0x30000000, 14, 0x30000000 },
+	{ 0x5, 0x00, 0x0, 0x0, 0x0d230000, 0,  0x00000000 },
+	{ 0x5, 0x01, 0x0, 0x0, 0x0d040000, 0,  0x00000000 },
+	{ 0x5, 0x02, 0x0, 0x0, 0x0d050000, 0,  0x00000000 },
+	{ 0x5, 0x03, 0x0, 0x0, 0x0d000000, 0,  0x00000000 },
+	{ 0x5, 0x04, 0x0, 0x0, 0x20ae0000, 3,  0x000e0000 },
+	{ 0x5, 0x04, 0x1, 0x0, 0x20ac0000, 2,  0x000c0000 },
+	{ 0x5, 0x04, 0x2, 0x0, 0x20a80000, 1,  0x00080000 },
+	{ 0x5, 0x04, 0x3, 0x0, 0x20a00000, 0,  0x00000000 },
+	{ 0x5, 0x05, 0x0, 0x0, 0x0d2a0000, 0,  0x00000000 },
+	{ 0x5, 0x06, 0x0, 0x0, 0x0d290000, 0,  0x00000000 },
+	{ 0x5, 0x07, 0x0, 0x0, 0x0d2c0000, 0,  0x00000000 },
+	{ 0x5, 0x08, 0x0, 0x0, 0x0d0e0000, 4,  0x00080000 },
+	{ 0x5, 0x08, 0x1, 0x0, 0x0d060000, 0,  0x00000000 },
+	{ 0x5, 0x08, 0x2, 0x0, 0x0d080000, 1,  0x00020000 },
+	{ 0x5, 0x08, 0x3, 0x0, 0x0d0a0000, 2,  0x00040000 },
+	{ 0x5, 0x08, 0x4, 0x0, 0x0d0c0000, 3,  0x00060000 },
+	{ 0x5, 0x09, 0x0, 0x0, 0x0d650000, 0,  0x00000000 },
+	{ 0x5, 0x0a, 0x0, 0x0, 0x20af0000, 0,  0x00000000 },
+	{ 0x5, 0x0b, 0x0, 0x0, 0x0d3e0000, 0,  0x00000000 },
+	{ 0x5, 0x0c, 0x0, 0x0, 0x0d3d0000, 0,  0x00000000 },
+	{ 0x5, 0x0d, 0x0, 0x0, 0x0d1e0000, 0,  0x00000000 },
+	{ 0x5, 0x0e, 0x0, 0x0, 0x0d150000, 0,  0x00000000 },
+	{ 0x5, 0x0e, 0x1, 0x0, 0x0d160000, 1,  0x00010000 },
+	{ 0x5, 0x0e, 0x2, 0x0, 0x0d170000, 2,  0x00020000 },
+	{ 0x5, 0x0e, 0x3, 0x0, 0x0d180000, 3,  0x00030000 },
+	{ 0x5, 0x0e, 0x4, 0x0, 0x0d190000, 4,  0x00040000 },
+	{ 0x5, 0x0e, 0x5, 0x0, 0x0d1a0000, 5,  0x00050000 },
+	{ 0x5, 0x0e, 0x6, 0x0, 0x0d1b0000, 6,  0x00060000 },
+	{ 0x5, 0x0e, 0x7, 0x0, 0x0d1c0000, 7,  0x00070000 },
+	{ 0x5, 0x0e, 0x8, 0x0, 0x0d1d0000, 8,  0x00080000 },
+	{ 0x5, 0x0f, 0x0, 0x0, 0x0d660000, 0,  0x00000000 },
+	{ 0x5, 0x10, 0x0, 0x0, 0x0d1f0000, 0,  0x00000000 },
+	{ 0x5, 0x10, 0x1, 0x0, 0x0d200000, 1,  0x00010000 },
+	{ 0x5, 0x10, 0x2, 0x0, 0x0d210000, 2,  0x00020000 },
+	{ 0x5, 0x10, 0x3, 0x0, 0x0d220000, 3,  0x00030000 },
+	{ 0x5, 0x11, 0x0, 0x0, 0x0d240000, 0,  0x00000000 },
+	{ 0x5, 0x12, 0x0, 0x0, 0x0d250000, 0,  0x00000000 },
+	{ 0x5, 0x13, 0x0, 0x0, 0x0d260000, 0,  0x00000000 },
+	{ 0x5, 0x14, 0x0, 0x0, 0x0d270000, 0,  0x00000000 },
+	{ 0x5, 0x15, 0x0, 0x0, 0x0d2b0000, 0,  0x00000000 },
+	{ 0x5, 0x16, 0x0, 0x0, 0x0d280000, 0,  0x00000000 },
+	{ 0x5, 0x17, 0x0, 0x0, 0x0d0f0000, 0,  0x00000000 },
+	{ 0x5, 0x17, 0x1, 0x0, 0x0d100000, 1,  0x00010000 },
+	{ 0x5, 0x17, 0x2, 0x0, 0x0d110000, 2,  0x00020000 },
+	{ 0x5, 0x17, 0x3, 0x0, 0x0d120000, 3,  0x00030000 },
+	{ 0x5, 0x17, 0x4, 0x0, 0x0d130000, 4,  0x00040000 },
+	{ 0x5, 0x17, 0x5, 0x0, 0x0d140000, 5,  0x00050000 },
+	{ 0x5, 0x18, 0x0, 0x0, 0x0d020000, 0,  0x00000000 },
+	{ 0x5, 0x19, 0x0, 0x0, 0x0d030000, 0,  0x00000000 },
+	{ 0x5, 0x1f, 0x0, 0x0, 0x0d600000, 0,  0x00000000 },
+	{ 0x5, 0x1f, 0x1, 0x0, 0x00000000, 0,  0x00000000 }
+};
+
+/*
+ * AON NOC aperture lookup table as per file "AON_NOC_Structure.info".
+ */
+static const char * const tegra194_aonnoc_routeid_initflow[] = {
+	[0x0] = "cbb_i/I/0",
+	[0x1] = "cpu_p_i/I/0",
+	[0x2] = "dma_m_i/I/0",
+	[0x3] = "dma_p_i/I/0"
+};
+
+static const char * const tegra194_aonnoc_routeid_targflow[] = {
+	[0x00] = "multiport1_t/T/aon_misc",
+	[0x01] = "multiport1_t/T/avic0",
+	[0x02] = "multiport1_t/T/avic1",
+	[0x03] = "multiport1_t/T/can1",
+	[0x04] = "multiport1_t/T/can2",
+	[0x05] = "multiport1_t/T/dma",
+	[0x06] = "multiport1_t/T/dmic",
+	[0x07] = "multiport1_t/T/err_collator",
+	[0x08] = "multiport1_t/T/fpga_misc",
+	[0x09] = "multiport1_t/T/gte",
+	[0x0a] = "multiport1_t/T/hsp",
+	[0x0b] = "multiport1_t/T/i2c2",
+	[0x0c] = "multiport1_t/T/i2c8",
+	[0x0d] = "multiport1_t/T/pwm",
+	[0x0e] = "multiport1_t/T/spi2",
+	[0x0f] = "multiport1_t/T/tke",
+	[0x10] = "multiport1_t/T/uartg",
+	[0x11] = "RESERVED",
+	[0x12] = "RESERVED",
+	[0x13] = "RESERVED",
+	[0x14] = "RESERVED",
+	[0x15] = "RESERVED",
+	[0x16] = "RESERVED",
+	[0x17] = "RESERVED",
+	[0x18] = "RESERVED",
+	[0x19] = "RESERVED",
+	[0x1a] = "RESERVED",
+	[0x1b] = "RESERVED",
+	[0x1c] = "RESERVED",
+	[0x1d] = "RESERVED",
+	[0x1e] = "RESERVED",
+	[0x1f] = "RESERVED",
+	[0x20] = "multiport0_t/T/aovc",
+	[0x21] = "multiport0_t/T/atcm",
+	[0x22] = "multiport0_t/T/cast",
+	[0x23] = "multiport0_t/T/dast",
+	[0x24] = "multiport0_t/T/err_collator_car",
+	[0x25] = "multiport0_t/T/gpio",
+	[0x26] = "multiport0_t/T/i2c10",
+	[0x27] = "multiport0_t/T/mss",
+	[0x28] = "multiport0_t/T/padctl_a12",
+	[0x29] = "multiport0_t/T/padctl_a14",
+	[0x2a] = "multiport0_t/T/padctl_a15",
+	[0x2b] = "multiport0_t/T/rtc",
+	[0x2c] = "multiport0_t/T/tsc",
+	[0x2d] = "RESERVED",
+	[0x2e] = "RESERVED",
+	[0x2f] = "RESERVED",
+	[0x30] = "multiport2_t/T/aon_vref_ro",
+	[0x31] = "multiport2_t/T/aopm",
+	[0x32] = "multiport2_t/T/car",
+	[0x33] = "multiport2_t/T/pmc",
+	[0x34] = "ast1_t/T/0",
+	[0x35] = "cbb_t/T/0",
+	[0x36] = "cpu_t/T/0",
+	[0x37] = "firewall_t/T/0",
+	[0x38] = "svc_t/T/0",
+	[0x39] = "uartc/T/uartc",
+	[0x3a] = "RESERVED",
+	[0x3b] = "RESERVED",
+	[0x3c] = "RESERVED",
+	[0x3d] = "RESERVED",
+	[0x3e] = "RESERVED",
+	[0x3f] = "RESERVED"
+};
+
+/*
+ * Fields of AON NOC lookup table:
+ * Init flow, Targ flow, Targ subrange, Init mapping, Init localAddress,
+ *                                              Targ mapping, Targ localAddress
+ * ----------------------------------------------------------------------------
+ */
+static const struct tegra194_cbb_aperture tegra194_aonnoc_aperture_lookup[] = {
+	{ 0x0, 0x37, 0x00, 0, 0x0c640000, 0,  0x00000000 },
+	{ 0x0, 0x20, 0x00, 0, 0x0c3b0000, 0,  0x00000000 },
+	{ 0x0, 0x21, 0x00, 0, 0x0c000000, 0,  0x00000000 },
+	{ 0x0, 0x22, 0x00, 0, 0x0c040000, 0,  0x00000000 },
+	{ 0x0, 0x23, 0x00, 0, 0x0c050000, 0,  0x00000000 },
+	{ 0x0, 0x24, 0x00, 0, 0x20cf0000, 0,  0x00000000 },
+	{ 0x0, 0x25, 0x00, 0, 0x0c2f0000, 0,  0x00000000 },
+	{ 0x0, 0x26, 0x00, 0, 0x0c230000, 0,  0x00000000 },
+	{ 0x0, 0x27, 0x00, 0, 0x0c350000, 0,  0x00000000 },
+	{ 0x0, 0x28, 0x00, 0, 0x0c301000, 0,  0x00000000 },
+	{ 0x0, 0x29, 0x00, 0, 0x0c302000, 0,  0x00000000 },
+	{ 0x0, 0x2a, 0x00, 0, 0x0c303000, 0,  0x00000000 },
+	{ 0x0, 0x2b, 0x00, 0, 0x0c2a0000, 0,  0x00000000 },
+	{ 0x0, 0x2c, 0x00, 0, 0x0c2b0000, 0,  0x00000000 },
+	{ 0x0, 0x2c, 0x01, 0, 0x0c2c0000, 1,  0x00010000 },
+	{ 0x0, 0x2c, 0x02, 0, 0x0c2d0000, 2,  0x00020000 },
+	{ 0x0, 0x2c, 0x03, 0, 0x0c2e0000, 3,  0x00030000 },
+	{ 0x0, 0x00, 0x00, 0, 0x0c660000, 0,  0x00000000 },
+	{ 0x0, 0x01, 0x00, 0, 0x0c020000, 0,  0x00000000 },
+	{ 0x0, 0x02, 0x00, 0, 0x0c030000, 0,  0x00000000 },
+	{ 0x0, 0x03, 0x00, 0, 0x0c310000, 0,  0x00000000 },
+	{ 0x0, 0x04, 0x00, 0, 0x0c320000, 0,  0x00000000 },
+	{ 0x0, 0x05, 0x00, 0, 0x0c0a0000, 2,  0x00040000 },
+	{ 0x0, 0x05, 0x01, 0, 0x0c0b0000, 3,  0x00050000 },
+	{ 0x0, 0x05, 0x02, 0, 0x0c0e0000, 5,  0x00080000 },
+	{ 0x0, 0x05, 0x03, 0, 0x0c060000, 0,  0x00000000 },
+	{ 0x0, 0x05, 0x04, 0, 0x0c080000, 1,  0x00020000 },
+	{ 0x0, 0x05, 0x05, 0, 0x0c0c0000, 4,  0x00060000 },
+	{ 0x0, 0x06, 0x00, 0, 0x0c330000, 0,  0x00000000 },
+	{ 0x0, 0x07, 0x00, 0, 0x0c650000, 0,  0x00000000 },
+	{ 0x0, 0x08, 0x00, 0, 0x0c3e0000, 0,  0x00000000 },
+	{ 0x0, 0x09, 0x00, 0, 0x0c1e0000, 0,  0x00000000 },
+	{ 0x0, 0x0a, 0x00, 0, 0x0c150000, 0,  0x00000000 },
+	{ 0x0, 0x0a, 0x01, 0, 0x0c160000, 1,  0x00010000 },
+	{ 0x0, 0x0a, 0x02, 0, 0x0c170000, 2,  0x00020000 },
+	{ 0x0, 0x0a, 0x03, 0, 0x0c180000, 3,  0x00030000 },
+	{ 0x0, 0x0a, 0x04, 0, 0x0c190000, 4,  0x00040000 },
+	{ 0x0, 0x0a, 0x05, 0, 0x0c1a0000, 5,  0x00050000 },
+	{ 0x0, 0x0a, 0x06, 0, 0x0c1b0000, 6,  0x00060000 },
+	{ 0x0, 0x0a, 0x07, 0, 0x0c1c0000, 7,  0x00070000 },
+	{ 0x0, 0x0a, 0x08, 0, 0x0c1d0000, 8,  0x00080000 },
+	{ 0x0, 0x0b, 0x00, 0, 0x0c240000, 0,  0x00000000 },
+	{ 0x0, 0x0c, 0x00, 0, 0x0c250000, 0,  0x00000000 },
+	{ 0x0, 0x0d, 0x00, 0, 0x0c340000, 0,  0x00000000 },
+	{ 0x0, 0x0e, 0x00, 0, 0x0c260000, 0,  0x00000000 },
+	{ 0x0, 0x0f, 0x00, 0, 0x0c0f0000, 0,  0x00000000 },
+	{ 0x0, 0x0f, 0x01, 0, 0x0c100000, 1,  0x00010000 },
+	{ 0x0, 0x0f, 0x02, 0, 0x0c110000, 2,  0x00020000 },
+	{ 0x0, 0x0f, 0x03, 0, 0x0c120000, 3,  0x00030000 },
+	{ 0x0, 0x0f, 0x04, 0, 0x0c130000, 4,  0x00040000 },
+	{ 0x0, 0x0f, 0x05, 0, 0x0c140000, 5,  0x00050000 },
+	{ 0x0, 0x10, 0x00, 0, 0x0c290000, 0,  0x00000000 },
+	{ 0x0, 0x30, 0x00, 0, 0x20ce0000, 0,  0x00000000 },
+	{ 0x0, 0x31, 0x00, 0, 0x0c1f0000, 0,  0x00000000 },
+	{ 0x0, 0x31, 0x01, 0, 0x0c200000, 1,  0x00010000 },
+	{ 0x0, 0x31, 0x02, 0, 0x0c210000, 2,  0x00020000 },
+	{ 0x0, 0x31, 0x03, 0, 0x0c220000, 3,  0x00030000 },
+	{ 0x0, 0x32, 0x00, 0, 0x20cc0000, 3,  0x001c0000 },
+	{ 0x0, 0x32, 0x01, 0, 0x20c80000, 2,  0x00180000 },
+	{ 0x0, 0x32, 0x02, 0, 0x20c00000, 1,  0x00100000 },
+	{ 0x0, 0x32, 0x03, 0, 0x20b00000, 0,  0x00000000 },
+	{ 0x0, 0x33, 0x00, 0, 0x0c360000, 0,  0x00000000 },
+	{ 0x0, 0x33, 0x01, 0, 0x0c370000, 1,  0x00010000 },
+	{ 0x0, 0x33, 0x02, 0, 0x0c3a0000, 3,  0x00040000 },
+	{ 0x0, 0x33, 0x03, 0, 0x0c380000, 2,  0x00020000 },
+	{ 0x0, 0x38, 0x00, 0, 0x0c600000, 0,  0x00000000 },
+	{ 0x0, 0x38, 0x01, 0, 0x00000000, 0,  0x00000000 },
+	{ 0x0, 0x39, 0x00, 0, 0x0c280000, 0,  0x00000000 },
+	{ 0x1, 0x35, 0x00, 0, 0x00000000, 0,  0x00000000 },
+	{ 0x1, 0x35, 0x01, 0, 0x00100000, 1,  0x00100000 },
+	{ 0x1, 0x35, 0x02, 0, 0x05a00000, 11, 0x05a00000 },
+	{ 0x1, 0x35, 0x03, 0, 0x05b00000, 32, 0x05b00000 },
+	{ 0x1, 0x35, 0x04, 0, 0x05c00000, 33, 0x05c00000 },
+	{ 0x1, 0x35, 0x05, 0, 0x05d00000, 12, 0x05d00000 },
+	{ 0x1, 0x35, 0x06, 0, 0x20000000, 19, 0x20000000 },
+	{ 0x1, 0x35, 0x07, 0, 0x20100000, 20, 0x20100000 },
+	{ 0x1, 0x35, 0x08, 0, 0x20a00000, 24, 0x20a00000 },
+	{ 0x1, 0x35, 0x09, 0, 0x20d00000, 25, 0x20d00000 },
+	{ 0x1, 0x35, 0x0a, 0, 0x00200000, 2,  0x00200000 },
+	{ 0x1, 0x35, 0x0b, 0, 0x05800000, 10, 0x05800000 },
+	{ 0x1, 0x35, 0x0c, 0, 0x05e00000, 13, 0x05e00000 },
+	{ 0x1, 0x35, 0x0d, 0, 0x20200000, 21, 0x20200000 },
+	{ 0x1, 0x35, 0x0e, 0, 0x20800000, 23, 0x20800000 },
+	{ 0x1, 0x35, 0x0f, 0, 0x20e00000, 26, 0x20e00000 },
+	{ 0x1, 0x35, 0x10, 0, 0x00400000, 3,  0x00400000 },
+	{ 0x1, 0x35, 0x11, 0, 0x20400000, 22, 0x20400000 },
+	{ 0x1, 0x35, 0x12, 0, 0x00800000, 4,  0x00800000 },
+	{ 0x1, 0x35, 0x13, 0, 0x05000000, 9,  0x05000000 },
+	{ 0x1, 0x35, 0x14, 0, 0x0c800000, 34, 0x0c800000 },
+	{ 0x1, 0x35, 0x15, 0, 0x01000000, 5,  0x01000000 },
+	{ 0x1, 0x35, 0x16, 0, 0x03000000, 7,  0x03000000 },
+	{ 0x1, 0x35, 0x17, 0, 0x04000000, 8,  0x04000000 },
+	{ 0x1, 0x35, 0x18, 0, 0x0d000000, 16, 0x0d000000 },
+	{ 0x1, 0x35, 0x19, 0, 0x21000000, 27, 0x21000000 },
+	{ 0x1, 0x35, 0x1a, 0, 0x02000000, 6,  0x02000000 },
+	{ 0x1, 0x35, 0x1b, 0, 0x06000000, 14, 0x06000000 },
+	{ 0x1, 0x35, 0x1c, 0, 0x0e000000, 17, 0x0e000000 },
+	{ 0x1, 0x35, 0x1d, 0, 0x22000000, 28, 0x22000000 },
+	{ 0x1, 0x35, 0x1e, 0, 0x08000000, 15, 0x08000000 },
+	{ 0x1, 0x35, 0x1f, 0, 0x24000000, 29, 0x24000000 },
+	{ 0x1, 0x35, 0x20, 0, 0x28000000, 30, 0x28000000 },
+	{ 0x1, 0x35, 0x21, 0, 0x10000000, 18, 0x10000000 },
+	{ 0x1, 0x35, 0x22, 0, 0x30000000, 31, 0x30000000 },
+	{ 0x1, 0x37, 0x00, 0, 0x0c640000, 0,  0x00000000 },
+	{ 0x1, 0x20, 0x00, 0, 0x0c3b0000, 0,  0x00000000 },
+	{ 0x1, 0x21, 0x00, 0, 0x0c000000, 0,  0x00000000 },
+	{ 0x1, 0x22, 0x00, 0, 0x0c040000, 0,  0x00000000 },
+	{ 0x1, 0x23, 0x00, 0, 0x0c050000, 0,  0x00000000 },
+	{ 0x1, 0x24, 0x00, 0, 0x20cf0000, 0,  0x00000000 },
+	{ 0x1, 0x25, 0x00, 0, 0x0c2f0000, 0,  0x00000000 },
+	{ 0x1, 0x26, 0x00, 0, 0x0c230000, 0,  0x00000000 },
+	{ 0x1, 0x27, 0x00, 0, 0x0c350000, 0,  0x00000000 },
+	{ 0x1, 0x28, 0x00, 0, 0x0c301000, 0,  0x00000000 },
+	{ 0x1, 0x29, 0x00, 0, 0x0c302000, 0,  0x00000000 },
+	{ 0x1, 0x2a, 0x00, 0, 0x0c303000, 0,  0x00000000 },
+	{ 0x1, 0x2b, 0x00, 0, 0x0c2a0000, 0,  0x00000000 },
+	{ 0x1, 0x2c, 0x00, 0, 0x0c2b0000, 0,  0x00000000 },
+	{ 0x1, 0x2c, 0x01, 0, 0x0c2c0000, 1,  0x00010000 },
+	{ 0x1, 0x2c, 0x02, 0, 0x0c2d0000, 2,  0x00020000 },
+	{ 0x1, 0x2c, 0x03, 0, 0x0c2e0000, 3,  0x00030000 },
+	{ 0x1, 0x00, 0x00, 0, 0x0c660000, 0,  0x00000000 },
+	{ 0x1, 0x01, 0x00, 0, 0x0c020000, 0,  0x00000000 },
+	{ 0x1, 0x02, 0x00, 0, 0x0c030000, 0,  0x00000000 },
+	{ 0x1, 0x03, 0x00, 0, 0x0c310000, 0,  0x00000000 },
+	{ 0x1, 0x04, 0x00, 0, 0x0c320000, 0,  0x00000000 },
+	{ 0x1, 0x05, 0x00, 0, 0x0c0a0000, 2,  0x00040000 },
+	{ 0x1, 0x05, 0x01, 0, 0x0c0b0000, 3,  0x00050000 },
+	{ 0x1, 0x05, 0x02, 0, 0x0c0e0000, 5,  0x00080000 },
+	{ 0x1, 0x05, 0x03, 0, 0x0c060000, 0,  0x00000000 },
+	{ 0x1, 0x05, 0x04, 0, 0x0c080000, 1,  0x00020000 },
+	{ 0x1, 0x05, 0x05, 0, 0x0c0c0000, 4,  0x00060000 },
+	{ 0x1, 0x06, 0x00, 0, 0x0c330000, 0,  0x00000000 },
+	{ 0x1, 0x07, 0x00, 0, 0x0c650000, 0,  0x00000000 },
+	{ 0x1, 0x08, 0x00, 0, 0x0c3e0000, 0,  0x00000000 },
+	{ 0x1, 0x09, 0x00, 0, 0x0c1e0000, 0,  0x00000000 },
+	{ 0x1, 0x0a, 0x00, 0, 0x0c150000, 0,  0x00000000 },
+	{ 0x1, 0x0a, 0x01, 0, 0x0c160000, 1,  0x00010000 },
+	{ 0x1, 0x0a, 0x02, 0, 0x0c170000, 2,  0x00020000 },
+	{ 0x1, 0x0a, 0x03, 0, 0x0c180000, 3,  0x00030000 },
+	{ 0x1, 0x0a, 0x04, 0, 0x0c190000, 4,  0x00040000 },
+	{ 0x1, 0x0a, 0x05, 0, 0x0c1a0000, 5,  0x00050000 },
+	{ 0x1, 0x0a, 0x06, 0, 0x0c1b0000, 6,  0x00060000 },
+	{ 0x1, 0x0a, 0x07, 0, 0x0c1c0000, 7,  0x00070000 },
+	{ 0x1, 0x0a, 0x08, 0, 0x0c1d0000, 8,  0x00080000 },
+	{ 0x1, 0x0b, 0x00, 0, 0x0c240000, 0,  0x00000000 },
+	{ 0x1, 0x0c, 0x00, 0, 0x0c250000, 0,  0x00000000 },
+	{ 0x1, 0x0d, 0x00, 0, 0x0c340000, 0,  0x00000000 },
+	{ 0x1, 0x0e, 0x00, 0, 0x0c260000, 0,  0x00000000 },
+	{ 0x1, 0x0f, 0x00, 0, 0x0c0f0000, 0,  0x00000000 },
+	{ 0x1, 0x0f, 0x01, 0, 0x0c100000, 1,  0x00010000 },
+	{ 0x1, 0x0f, 0x02, 0, 0x0c110000, 2,  0x00020000 },
+	{ 0x1, 0x0f, 0x03, 0, 0x0c120000, 3,  0x00030000 },
+	{ 0x1, 0x0f, 0x04, 0, 0x0c130000, 4,  0x00040000 },
+	{ 0x1, 0x0f, 0x05, 0, 0x0c140000, 5,  0x00050000 },
+	{ 0x1, 0x10, 0x00, 0, 0x0c290000, 0,  0x00000000 },
+	{ 0x1, 0x30, 0x00, 0, 0x20ce0000, 0,  0x00000000 },
+	{ 0x1, 0x31, 0x00, 0, 0x0c1f0000, 0,  0x00000000 },
+	{ 0x1, 0x31, 0x01, 0, 0x0c200000, 1,  0x00010000 },
+	{ 0x1, 0x31, 0x02, 0, 0x0c210000, 2,  0x00020000 },
+	{ 0x1, 0x31, 0x03, 0, 0x0c220000, 3,  0x00030000 },
+	{ 0x1, 0x32, 0x00, 0, 0x20cc0000, 3,  0x001c0000 },
+	{ 0x1, 0x32, 0x01, 0, 0x20c80000, 2,  0x00180000 },
+	{ 0x1, 0x32, 0x02, 0, 0x20c00000, 1,  0x00100000 },
+	{ 0x1, 0x32, 0x03, 0, 0x20b00000, 0,  0x00000000 },
+	{ 0x1, 0x33, 0x00, 0, 0x0c360000, 0,  0x00000000 },
+	{ 0x1, 0x33, 0x01, 0, 0x0c370000, 1,  0x00010000 },
+	{ 0x1, 0x33, 0x02, 0, 0x0c3a0000, 3,  0x00040000 },
+	{ 0x1, 0x33, 0x03, 0, 0x0c380000, 2,  0x00020000 },
+	{ 0x1, 0x38, 0x00, 0, 0x0c600000, 0,  0x00000000 },
+	{ 0x1, 0x38, 0x01, 0, 0x00000000, 0,  0x00000000 },
+	{ 0x1, 0x39, 0x00, 0, 0x0c280000, 0,  0x00000000 },
+	{ 0x2, 0x34, 0x00, 0, 0x40000000, 0,  0x40000000 },
+	{ 0x2, 0x34, 0x01, 0, 0x80000000, 1,  0x80000000 },
+	{ 0x2, 0x36, 0x00, 0, 0x0c400000, 0,  0x0c400000 },
+	{ 0x2, 0x36, 0x01, 0, 0x00000000, 0,  0x00000000 },
+	{ 0x3, 0x35, 0x00, 0, 0x00000000, 0,  0x00000000 },
+	{ 0x3, 0x35, 0x01, 0, 0x00100000, 1,  0x00100000 },
+	{ 0x3, 0x35, 0x02, 0, 0x05a00000, 11, 0x05a00000 },
+	{ 0x3, 0x35, 0x03, 0, 0x05b00000, 32, 0x05b00000 },
+	{ 0x3, 0x35, 0x04, 0, 0x05c00000, 33, 0x05c00000 },
+	{ 0x3, 0x35, 0x05, 0, 0x05d00000, 12, 0x05d00000 },
+	{ 0x3, 0x35, 0x06, 0, 0x20000000, 19, 0x20000000 },
+	{ 0x3, 0x35, 0x07, 0, 0x20100000, 20, 0x20100000 },
+	{ 0x3, 0x35, 0x08, 0, 0x20a00000, 24, 0x20a00000 },
+	{ 0x3, 0x35, 0x09, 0, 0x20d00000, 25, 0x20d00000 },
+	{ 0x3, 0x35, 0x0a, 0, 0x00200000, 2,  0x00200000 },
+	{ 0x3, 0x35, 0x0b, 0, 0x05800000, 10, 0x05800000 },
+	{ 0x3, 0x35, 0x0c, 0, 0x05e00000, 13, 0x05e00000 },
+	{ 0x3, 0x35, 0x0d, 0, 0x20200000, 21, 0x20200000 },
+	{ 0x3, 0x35, 0x0e, 0, 0x20800000, 23, 0x20800000 },
+	{ 0x3, 0x35, 0x0f, 0, 0x20e00000, 26, 0x20e00000 },
+	{ 0x3, 0x35, 0x10, 0, 0x00400000, 3,  0x00400000 },
+	{ 0x3, 0x35, 0x11, 0, 0x20400000, 22, 0x20400000 },
+	{ 0x3, 0x35, 0x12, 0, 0x00800000, 4,  0x00800000 },
+	{ 0x3, 0x35, 0x13, 0, 0x50000000, 9,  0x05000000 },
+	{ 0x3, 0x35, 0x14, 0, 0xc0800000, 34, 0x0c800000 },
+	{ 0x3, 0x35, 0x15, 0, 0x10000000, 5,  0x01000000 },
+	{ 0x3, 0x35, 0x16, 0, 0x30000000, 7,  0x03000000 },
+	{ 0x3, 0x35, 0x17, 0, 0x04000000, 8,  0x04000000 },
+	{ 0x3, 0x35, 0x18, 0, 0x0d000000, 16, 0x0d000000 },
+	{ 0x3, 0x35, 0x19, 0, 0x21000000, 27, 0x21000000 },
+	{ 0x3, 0x35, 0x1a, 0, 0x02000000, 6,  0x02000000 },
+	{ 0x3, 0x35, 0x1b, 0, 0x06000000, 14, 0x06000000 },
+	{ 0x3, 0x35, 0x1c, 0, 0x0e000000, 17, 0x0e000000 },
+	{ 0x3, 0x35, 0x1d, 0, 0x22000000, 28, 0x22000000 },
+	{ 0x3, 0x35, 0x1e, 0, 0x08000000, 15, 0x08000000 },
+	{ 0x3, 0x35, 0x1f, 0, 0x24000000, 29, 0x24000000 },
+	{ 0x3, 0x35, 0x20, 0, 0x28000000, 30, 0x28000000 },
+	{ 0x3, 0x35, 0x21, 0, 0x10000000, 18, 0x10000000 },
+	{ 0x3, 0x35, 0x22, 0, 0x30000000, 31, 0x30000000 },
+	{ 0x3, 0x37, 0x00, 0, 0x0c640000, 0,  0x00000000 },
+	{ 0x3, 0x20, 0x00, 0, 0x0c3b0000, 0,  0x00000000 },
+	{ 0x3, 0x21, 0x00, 0, 0x0c000000, 0,  0x00000000 },
+	{ 0x3, 0x22, 0x00, 0, 0x0c040000, 0,  0x00000000 },
+	{ 0x3, 0x23, 0x00, 0, 0x0c050000, 0,  0x00000000 },
+	{ 0x3, 0x24, 0x00, 0, 0x20cf0000, 0,  0x00000000 },
+	{ 0x3, 0x25, 0x00, 0, 0x0c2f0000, 0,  0x00000000 },
+	{ 0x3, 0x26, 0x00, 0, 0x0c230000, 0,  0x00000000 },
+	{ 0x3, 0x27, 0x00, 0, 0x0c350000, 0,  0x00000000 },
+	{ 0x3, 0x28, 0x00, 0, 0x0c301000, 0,  0x00000000 },
+	{ 0x3, 0x29, 0x00, 0, 0x0c302000, 0,  0x00000000 },
+	{ 0x3, 0x2a, 0x00, 0, 0x0c303000, 0,  0x00000000 },
+	{ 0x3, 0x2b, 0x00, 0, 0x0c2a0000, 0,  0x00000000 },
+	{ 0x3, 0x2c, 0x00, 0, 0x0c2b0000, 0,  0x00000000 },
+	{ 0x3, 0x2c, 0x01, 0, 0x0c2c0000, 1,  0x00010000 },
+	{ 0x3, 0x2c, 0x02, 0, 0x0c2d0000, 2,  0x00020000 },
+	{ 0x3, 0x2c, 0x03, 0, 0x0c2e0000, 3,  0x00030000 },
+	{ 0x3, 0x00, 0x00, 0, 0x0c660000, 0,  0x00000000 },
+	{ 0x3, 0x01, 0x00, 0, 0x0c020000, 0,  0x00000000 },
+	{ 0x3, 0x02, 0x00, 0, 0x0c030000, 0,  0x00000000 },
+	{ 0x3, 0x03, 0x00, 0, 0x0c310000, 0,  0x00000000 },
+	{ 0x3, 0x04, 0x00, 0, 0x0c320000, 0,  0x00000000 },
+	{ 0x3, 0x05, 0x00, 0, 0x0c0a0000, 2,  0x00040000 },
+	{ 0x3, 0x05, 0x01, 0, 0x0c0b0000, 3,  0x00050000 },
+	{ 0x3, 0x05, 0x02, 0, 0x0c0e0000, 5,  0x00080000 },
+	{ 0x3, 0x05, 0x03, 0, 0x0c060000, 0,  0x00000000 },
+	{ 0x3, 0x05, 0x04, 0, 0x0c080000, 1,  0x00020000 },
+	{ 0x3, 0x05, 0x05, 0, 0x0c0c0000, 4,  0x00060000 },
+	{ 0x3, 0x06, 0x00, 0, 0x0c330000, 0,  0x00000000 },
+	{ 0x3, 0x07, 0x00, 0, 0x0c650000, 0,  0x00000000 },
+	{ 0x3, 0x08, 0x00, 0, 0x0c3e0000, 0,  0x00000000 },
+	{ 0x3, 0x09, 0x00, 0, 0x0c1e0000, 0,  0x00000000 },
+	{ 0x3, 0x0a, 0x00, 0, 0x0c150000, 0,  0x00000000 },
+	{ 0x3, 0x0a, 0x01, 0, 0x0c160000, 1,  0x00010000 },
+	{ 0x3, 0x0a, 0x02, 0, 0x0c170000, 2,  0x00020000 },
+	{ 0x3, 0x0a, 0x03, 0, 0x0c180000, 3,  0x00030000 },
+	{ 0x3, 0x0a, 0x04, 0, 0x0c190000, 4,  0x00040000 },
+	{ 0x3, 0x0a, 0x05, 0, 0x0c1a0000, 5,  0x00050000 },
+	{ 0x3, 0x0a, 0x06, 0, 0x0c1b0000, 6,  0x00060000 },
+	{ 0x3, 0x0a, 0x07, 0, 0x0c1c0000, 7,  0x00070000 },
+	{ 0x3, 0x0a, 0x08, 0, 0x0c1d0000, 8,  0x00080000 },
+	{ 0x3, 0x0b, 0x00, 0, 0x0c240000, 0,  0x00000000 },
+	{ 0x3, 0x0c, 0x00, 0, 0x0c250000, 0,  0x00000000 },
+	{ 0x3, 0x0d, 0x00, 0, 0x0c340000, 0,  0x00000000 },
+	{ 0x3, 0x0e, 0x00, 0, 0x0c260000, 0,  0x00000000 },
+	{ 0x3, 0x0f, 0x00, 0, 0x0c0f0000, 0,  0x00000000 },
+	{ 0x3, 0x0f, 0x01, 0, 0x0c100000, 1,  0x00010000 },
+	{ 0x3, 0x0f, 0x02, 0, 0x0c110000, 2,  0x00020000 },
+	{ 0x3, 0x0f, 0x03, 0, 0x0c120000, 3,  0x00030000 },
+	{ 0x3, 0x0f, 0x04, 0, 0x0c130000, 4,  0x00040000 },
+	{ 0x3, 0x0f, 0x05, 0, 0x0c140000, 5,  0x00050000 },
+	{ 0x3, 0x10, 0x00, 0, 0x0c290000, 0,  0x00000000 },
+	{ 0x3, 0x30, 0x00, 0, 0x20ce0000, 0,  0x00000000 },
+	{ 0x3, 0x31, 0x00, 0, 0x0c1f0000, 0,  0x00000000 },
+	{ 0x3, 0x31, 0x01, 0, 0x0c200000, 1,  0x00010000 },
+	{ 0x3, 0x31, 0x02, 0, 0x0c210000, 2,  0x00020000 },
+	{ 0x3, 0x31, 0x03, 0, 0x0c220000, 3,  0x00030000 },
+	{ 0x3, 0x32, 0x00, 0, 0x20cc0000, 3,  0x001c0000 },
+	{ 0x3, 0x32, 0x01, 0, 0x20c80000, 2,  0x00180000 },
+	{ 0x3, 0x32, 0x02, 0, 0x20c00000, 1,  0x00100000 },
+	{ 0x3, 0x32, 0x03, 0, 0x20b00000, 0,  0x00000000 },
+	{ 0x3, 0x33, 0x00, 0, 0x0c360000, 0,  0x00000000 },
+	{ 0x3, 0x33, 0x01, 0, 0x0c370000, 1,  0x00010000 },
+	{ 0x3, 0x33, 0x02, 0, 0x0c3a0000, 3,  0x00040000 },
+	{ 0x3, 0x33, 0x03, 0, 0x0c380000, 2,  0x00020000 },
+	{ 0x3, 0x38, 0x00, 0, 0x0c600000, 0,  0x00000000 },
+	{ 0x3, 0x38, 0x01, 0, 0x00000000, 0,  0x00000000 },
+	{ 0x3, 0x39, 0x00, 0, 0x0c280000, 0,  0x00000000 }
+};
+
+/*
+ * SCE/RCE NOC aperture lookup table as per file "AON_NOC_Structure.info".
+ */
+static const char * const tegra194_scenoc_routeid_initflow[] = {
+	[0x0] = "cbb_i/I/0",
+	[0x1] = "cpu_m_i/I/0",
+	[0x2] = "cpu_p_i/I/0",
+	[0x3] = "dma_m_i/I/0",
+	[0x4] = "dma_p_i/I/0",
+	[0x5] = "RESERVED",
+	[0x6] = "RESERVED",
+	[0x7] = "RESERVED"
+};
+
+static const char * const tegra194_scenoc_routeid_targflow[] = {
+	[0x00] = "multiport0_t/T/atcm_cfg",
+	[0x01] = "multiport0_t/T/car",
+	[0x02] = "multiport0_t/T/cast",
+	[0x03] = "multiport0_t/T/cfg",
+	[0x04] = "multiport0_t/T/dast",
+	[0x05] = "multiport0_t/T/dma",
+	[0x06] = "multiport0_t/T/err_collator",
+	[0x07] = "multiport0_t/T/err_collator_car",
+	[0x08] = "multiport0_t/T/fpga_misc",
+	[0x09] = "multiport0_t/T/fpga_uart",
+	[0x0a] = "multiport0_t/T/gte",
+	[0x0b] = "multiport0_t/T/hsp",
+	[0x0c] = "multiport0_t/T/misc",
+	[0x0d] = "multiport0_t/T/pm",
+	[0x0e] = "multiport0_t/T/tke",
+	[0x0f] = "RESERVED",
+	[0x10] = "multiport1_t/T/hsm",
+	[0x11] = "multiport1_t/T/vic0",
+	[0x12] = "multiport1_t/T/vic1",
+	[0x13] = "ast0_t/T/0",
+	[0x14] = "ast1_t/T/0",
+	[0x15] = "cbb_t/T/0",
+	[0x16] = "cpu_t/T/0",
+	[0x17] = "sce_noc_firewall/T/0",
+	[0x18] = "svc_t/T/0",
+	[0x19] = "RESERVED",
+	[0x1a] = "RESERVED",
+	[0x1b] = "RESERVED",
+	[0x1c] = "RESERVED",
+	[0x1d] = "RESERVED",
+	[0x1e] = "RESERVED",
+	[0x1f] = "RESERVED"
+};
+
+/*
+ * Fields of SCE/RCE NOC lookup table:
+ * Init flow, Targ flow, Targ subrange, Init mapping, Init localAddress,
+ *                                              Targ mapping, Targ localAddress
+ * ----------------------------------------------------------------------------
+ */
+static const struct tegra194_cbb_aperture tegra194_scenoc_apert_lookup[] = {
+	{ 0x0, 0x16, 0x0,  0, 0x0b400000, 0,  0x0b400000 },
+	{ 0x0, 0x16, 0x1,  0, 0x0bc00000, 1,  0x0bc00000 },
+	{ 0x0, 0x0,  0x0,  0, 0x0b000000, 0,  0x00000000 },
+	{ 0x0, 0x0,  0x1,  0, 0x0b800000, 1,  0x00000000 },
+	{ 0x0, 0x1,  0x0,  0, 0x20de0000, 3,  0x000e0000 },
+	{ 0x0, 0x1,  0x1,  0, 0x210e0000, 7,  0x000e0000 },
+	{ 0x0, 0x1,  0x2,  0, 0x20dc0000, 2,  0x000c0000 },
+	{ 0x0, 0x1,  0x3,  0, 0x210c0000, 6,  0x000c0000 },
+	{ 0x0, 0x1,  0x4,  0, 0x20d80000, 1,  0x00080000 },
+	{ 0x0, 0x1,  0x5,  0, 0x21080000, 5,  0x00080000 },
+	{ 0x0, 0x1,  0x6,  0, 0x20d00000, 0,  0x00000000 },
+	{ 0x0, 0x1,  0x7,  0, 0x21000000, 4,  0x00000000 },
+	{ 0x0, 0x2,  0x0,  0, 0x0b040000, 0,  0x00000000 },
+	{ 0x0, 0x2,  0x1,  0, 0x0b840000, 1,  0x00000000 },
+	{ 0x0, 0x3,  0x0,  0, 0x0b230000, 0,  0x00000000 },
+	{ 0x0, 0x3,  0x1,  0, 0x0ba30000, 1,  0x00000000 },
+	{ 0x0, 0x4,  0x0,  0, 0x0b050000, 0,  0x00000000 },
+	{ 0x0, 0x4,  0x1,  0, 0x0b850000, 1,  0x00000000 },
+	{ 0x0, 0x5,  0x0,  0, 0x0b060000, 0,  0x00000000 },
+	{ 0x0, 0x5,  0x1,  0, 0x0b070000, 1,  0x00010000 },
+	{ 0x0, 0x5,  0x2,  0, 0x0b080000, 2,  0x00020000 },
+	{ 0x0, 0x5,  0x3,  0, 0x0b090000, 3,  0x00030000 },
+	{ 0x0, 0x5,  0x4,  0, 0x0b0a0000, 4,  0x00040000 },
+	{ 0x0, 0x5,  0x5,  0, 0x0b0b0000, 5,  0x00050000 },
+	{ 0x0, 0x5,  0x6,  0, 0x0b0c0000, 6,  0x00060000 },
+	{ 0x0, 0x5,  0x7,  0, 0x0b0d0000, 7,  0x00070000 },
+	{ 0x0, 0x5,  0x8,  0, 0x0b0e0000, 8,  0x00080000 },
+	{ 0x0, 0x5,  0x9,  0, 0x0b860000, 9,  0x00000000 },
+	{ 0x0, 0x5,  0xa,  0, 0x0b870000, 10, 0x00010000 },
+	{ 0x0, 0x5,  0xb,  0, 0x0b880000, 11, 0x00020000 },
+	{ 0x0, 0x5,  0xc,  0, 0x0b890000, 12, 0x00030000 },
+	{ 0x0, 0x5,  0xd,  0, 0x0b8a0000, 13, 0x00040000 },
+	{ 0x0, 0x5,  0xe,  0, 0x0b8b0000, 14, 0x00050000 },
+	{ 0x0, 0x5,  0xf,  0, 0x0b8c0000, 15, 0x00060000 },
+	{ 0x0, 0x5,  0x10, 0, 0x0b8d0000, 16, 0x00070000 },
+	{ 0x0, 0x5,  0x11, 0, 0x0b8e0000, 17, 0x00080000 },
+	{ 0x0, 0x6,  0x0,  0, 0x0b650000, 0,  0x00000000 },
+	{ 0x0, 0x6,  0x1,  0, 0x0be50000, 1,  0x00000000 },
+	{ 0x0, 0x7,  0x0,  0, 0x20df0000, 0,  0x00000000 },
+	{ 0x0, 0x7,  0x1,  0, 0x210f0000, 1,  0x00000000 },
+	{ 0x0, 0x8,  0x0,  0, 0x0b3e0000, 0,  0x00000000 },
+	{ 0x0, 0x8,  0x1,  0, 0x0bbe0000, 1,  0x00000000 },
+	{ 0x0, 0x9,  0x0,  0, 0x0b3d0000, 0,  0x00000000 },
+	{ 0x0, 0x9,  0x1,  0, 0x0bbd0000, 1,  0x00000000 },
+	{ 0x0, 0xa,  0x0,  0, 0x0b1e0000, 0,  0x00000000 },
+	{ 0x0, 0xa,  0x1,  0, 0x0b9e0000, 1,  0x00000000 },
+	{ 0x0, 0xb,  0x0,  0, 0x0b150000, 0,  0x00000000 },
+	{ 0x0, 0xb,  0x1,  0, 0x0b160000, 1,  0x00010000 },
+	{ 0x0, 0xb,  0x2,  0, 0x0b170000, 2,  0x00020000 },
+	{ 0x0, 0xb,  0x3,  0, 0x0b180000, 3,  0x00030000 },
+	{ 0x0, 0xb,  0x4,  0, 0x0b190000, 4,  0x00040000 },
+	{ 0x0, 0xb,  0x5,  0, 0x0b1a0000, 5,  0x00050000 },
+	{ 0x0, 0xb,  0x6,  0, 0x0b1b0000, 6,  0x00060000 },
+	{ 0x0, 0xb,  0x7,  0, 0x0b1c0000, 7,  0x00070000 },
+	{ 0x0, 0xb,  0x8,  0, 0x0b1d0000, 8,  0x00080000 },
+	{ 0x0, 0xb,  0x9,  0, 0x0b950000, 9,  0x00000000 },
+	{ 0x0, 0xb,  0xa,  0, 0x0b960000, 10, 0x00010000 },
+	{ 0x0, 0xb,  0xb,  0, 0x0b970000, 11, 0x00020000 },
+	{ 0x0, 0xb,  0xc,  0, 0x0b980000, 12, 0x00030000 },
+	{ 0x0, 0xb,  0xd,  0, 0x0b990000, 13, 0x00040000 },
+	{ 0x0, 0xb,  0xe,  0, 0x0b9a0000, 14, 0x00050000 },
+	{ 0x0, 0xb,  0xf,  0, 0x0b9b0000, 15, 0x00060000 },
+	{ 0x0, 0xb,  0x10, 0, 0x0b9c0000, 16, 0x00070000 },
+	{ 0x0, 0xb,  0x11, 0, 0x0b9d0000, 17, 0x00080000 },
+	{ 0x0, 0xc,  0x0,  0, 0x0b660000, 0,  0x00000000 },
+	{ 0x0, 0xc,  0x1,  0, 0x0be60000, 1,  0x00000000 },
+	{ 0x0, 0xd,  0x0,  0, 0x0b1f0000, 0,  0x00000000 },
+	{ 0x0, 0xd,  0x1,  0, 0x0b200000, 1,  0x00010000 },
+	{ 0x0, 0xd,  0x2,  0, 0x0b210000, 2,  0x00020000 },
+	{ 0x0, 0xd,  0x3,  0, 0x0b220000, 3,  0x00030000 },
+	{ 0x0, 0xd,  0x4,  0, 0x0b9f0000, 4,  0x00000000 },
+	{ 0x0, 0xd,  0x5,  0, 0x0ba00000, 5,  0x00010000 },
+	{ 0x0, 0xd,  0x6,  0, 0x0ba10000, 6,  0x00020000 },
+	{ 0x0, 0xd,  0x7,  0, 0x0ba20000, 7,  0x00030000 },
+	{ 0x0, 0xe,  0x0,  0, 0x0b0f0000, 0,  0x00000000 },
+	{ 0x0, 0xe,  0x1,  0, 0x0b100000, 1,  0x00010000 },
+	{ 0x0, 0xe,  0x2,  0, 0x0b110000, 2,  0x00020000 },
+	{ 0x0, 0xe,  0x3,  0, 0x0b120000, 3,  0x00030000 },
+	{ 0x0, 0xe,  0x4,  0, 0x0b130000, 4,  0x00040000 },
+	{ 0x0, 0xe,  0x5,  0, 0x0b140000, 5,  0x00050000 },
+	{ 0x0, 0xe,  0x6,  0, 0x0b8f0000, 6,  0x00000000 },
+	{ 0x0, 0xe,  0x7,  0, 0x0b900000, 7,  0x00010000 },
+	{ 0x0, 0xe,  0x8,  0, 0x0b910000, 8,  0x00020000 },
+	{ 0x0, 0xe,  0x9,  0, 0x0b920000, 9,  0x00030000 },
+	{ 0x0, 0xe,  0xa,  0, 0x0b930000, 10, 0x00040000 },
+	{ 0x0, 0xe,  0xb,  0, 0x0b940000, 11, 0x00050000 },
+	{ 0x0, 0x10, 0x0,  0, 0x0b240000, 0,  0x00000000 },
+	{ 0x0, 0x10, 0x1,  0, 0x0ba40000, 1,  0x00000000 },
+	{ 0x0, 0x11, 0x0,  0, 0x0b020000, 0,  0x00000000 },
+	{ 0x0, 0x11, 0x1,  0, 0x0b820000, 1,  0x00000000 },
+	{ 0x0, 0x12, 0x0,  0, 0x0b030000, 0,  0x00000000 },
+	{ 0x0, 0x12, 0x1,  0, 0x0b830000, 1,  0x00000000 },
+	{ 0x0, 0x17, 0x0,  0, 0x0b640000, 0,  0x00000000 },
+	{ 0x0, 0x17, 0x1,  0, 0x0be40000, 1,  0x00000000 },
+	{ 0x0, 0x18, 0x0,  0, 0x0b600000, 0,  0x00000000 },
+	{ 0x0, 0x18, 0x1,  0, 0x0be00000, 1,  0x00000000 },
+	{ 0x0, 0x18, 0x2,  0, 0x00000000, 0,  0x00000000 },
+	{ 0x0, 0x18, 0x3,  0, 0x00000000, 0,  0x00000000 },
+	{ 0x1, 0x13, 0x0,  0, 0x40000000, 0,  0x40000000 },
+	{ 0x1, 0x13, 0x1,  1, 0x80000000, 1,  0x80000000 },
+	{ 0x1, 0x13, 0x2,  0, 0x00000000, 0,  0x00000000 },
+	{ 0x2, 0x15, 0x0,  0, 0x20c00000, 8,  0x20c00000 },
+	{ 0x2, 0x15, 0x1,  0, 0x21100000, 22, 0x21100000 },
+	{ 0x2, 0x15, 0x2,  0, 0x20e00000, 9,  0x20e00000 },
+	{ 0x2, 0x15, 0x3,  0, 0x21200000, 23, 0x21200000 },
+	{ 0x2, 0x15, 0x4,  0, 0x20800000, 7,  0x20800000 },
+	{ 0x2, 0x15, 0x5,  0, 0x21400000, 24, 0x21400000 },
+	{ 0x2, 0x15, 0x6,  0, 0x0b000000, 18, 0x0b000000 },
+	{ 0x2, 0x15, 0x7,  0, 0x0b800000, 3,  0x0b800000 },
+	{ 0x2, 0x15, 0x8,  0, 0x20000000, 6,  0x20000000 },
+	{ 0x2, 0x15, 0x9,  0, 0x21800000, 25, 0x21800000 },
+	{ 0x2, 0x15, 0xa,  0, 0x0a000000, 2,  0x0a000000 },
+	{ 0x2, 0x15, 0xb,  0, 0x0a000000, 17, 0x0a000000 },
+	{ 0x2, 0x15, 0xc,  0, 0x20000000, 21, 0x20000000 },
+	{ 0x2, 0x15, 0xd,  0, 0x21000000, 10, 0x21000000 },
+	{ 0x2, 0x15, 0xe,  0, 0x08000000, 1,  0x08000000 },
+	{ 0x2, 0x15, 0xf,  0, 0x08000000, 16, 0x08000000 },
+	{ 0x2, 0x15, 0x10, 0, 0x22000000, 11, 0x22000000 },
+	{ 0x2, 0x15, 0x11, 0, 0x22000000, 26, 0x22000000 },
+	{ 0x2, 0x15, 0x12, 0, 0x0c000000, 4,  0x0c000000 },
+	{ 0x2, 0x15, 0x13, 0, 0x0c000000, 19, 0x0c000000 },
+	{ 0x2, 0x15, 0x14, 0, 0x24000000, 12, 0x24000000 },
+	{ 0x2, 0x15, 0x15, 0, 0x24000000, 27, 0x24000000 },
+	{ 0x2, 0x15, 0x16, 0, 0x00000000, 0,  0x00000000 },
+	{ 0x2, 0x15, 0x17, 0, 0x00000000, 15, 0x00000000 },
+	{ 0x2, 0x15, 0x18, 0, 0x28000000, 13, 0x28000000 },
+	{ 0x2, 0x15, 0x19, 0, 0x28000000, 28, 0x28000000 },
+	{ 0x2, 0x15, 0x1a, 0, 0x10000000, 5,  0x10000000 },
+	{ 0x2, 0x15, 0x1b, 0, 0x10000000, 20, 0x10000000 },
+	{ 0x2, 0x15, 0x1c, 0, 0x30000000, 14, 0x30000000 },
+	{ 0x2, 0x15, 0x1d, 0, 0x30000000, 29, 0x30000000 },
+	{ 0x2, 0x0,  0x0,  0, 0x0b000000, 0,  0x00000000 },
+	{ 0x2, 0x0,  0x1,  0, 0x0b800000, 1,  0x00000000 },
+	{ 0x2, 0x1,  0x0,  0, 0x20de0000, 3,  0x000e0000 },
+	{ 0x2, 0x1,  0x1,  0, 0x210e0000, 7,  0x000e0000 },
+	{ 0x2, 0x1,  0x2,  0, 0x20dc0000, 2,  0x000c0000 },
+	{ 0x2, 0x1,  0x3,  0, 0x210c0000, 6,  0x000c0000 },
+	{ 0x2, 0x1,  0x4,  0, 0x20d80000, 1,  0x00080000 },
+	{ 0x2, 0x1,  0x5,  0, 0x21080000, 5,  0x00080000 },
+	{ 0x2, 0x1,  0x6,  0, 0x20d00000, 0,  0x00000000 },
+	{ 0x2, 0x1,  0x7,  0, 0x21000000, 4,  0x00000000 },
+	{ 0x2, 0x2,  0x0,  0, 0x0b040000, 0,  0x00000000 },
+	{ 0x2, 0x2,  0x1,  0, 0x0b840000, 1,  0x00000000 },
+	{ 0x2, 0x3,  0x0,  0, 0x0b230000, 0,  0x00000000 },
+	{ 0x2, 0x3,  0x1,  0, 0x0ba30000, 1,  0x00000000 },
+	{ 0x2, 0x4,  0x0,  0, 0x0b050000, 0,  0x00000000 },
+	{ 0x2, 0x4,  0x1,  0, 0x0b850000, 1,  0x00000000 },
+	{ 0x2, 0x5,  0x0,  0, 0x0b060000, 0,  0x00000000 },
+	{ 0x2, 0x5,  0x1,  0, 0x0b070000, 1,  0x00010000 },
+	{ 0x2, 0x5,  0x2,  0, 0x0b080000, 2,  0x00020000 },
+	{ 0x2, 0x5,  0x3,  0, 0x0b090000, 3,  0x00030000 },
+	{ 0x2, 0x5,  0x4,  0, 0x0b0a0000, 4,  0x00040000 },
+	{ 0x2, 0x5,  0x5,  0, 0x0b0b0000, 5,  0x00050000 },
+	{ 0x2, 0x5,  0x6,  0, 0x0b0c0000, 6,  0x00060000 },
+	{ 0x2, 0x5,  0x7,  0, 0x0b0d0000, 7,  0x00070000 },
+	{ 0x2, 0x5,  0x8,  0, 0x0b0e0000, 8,  0x00080000 },
+	{ 0x2, 0x5,  0x9,  0, 0x0b860000, 9,  0x00000000 },
+	{ 0x2, 0x5,  0xa,  0, 0x0b870000, 10, 0x00010000 },
+	{ 0x2, 0x5,  0xb,  0, 0x0b880000, 11, 0x00020000 },
+	{ 0x2, 0x5,  0xc,  0, 0x0b890000, 12, 0x00030000 },
+	{ 0x2, 0x5,  0xd,  0, 0x0b8a0000, 13, 0x00040000 },
+	{ 0x2, 0x5,  0xe,  0, 0x0b8b0000, 14, 0x00050000 },
+	{ 0x2, 0x5,  0xf,  0, 0x0b8c0000, 15, 0x00060000 },
+	{ 0x2, 0x5,  0x10, 0, 0x0b8d0000, 16, 0x00070000 },
+	{ 0x2, 0x5,  0x11, 0, 0x0b8e0000, 17, 0x00080000 },
+	{ 0x2, 0x6,  0x0,  0, 0x0b650000, 0,  0x00000000 },
+	{ 0x2, 0x6,  0x1,  0, 0x0be50000, 1,  0x00000000 },
+	{ 0x2, 0x7,  0x0,  0, 0x20df0000, 0,  0x00000000 },
+	{ 0x2, 0x7,  0x1,  0, 0x210f0000, 1,  0x00000000 },
+	{ 0x2, 0x8,  0x0,  0, 0x0b3e0000, 0,  0x00000000 },
+	{ 0x2, 0x8,  0x1,  0, 0x0bbe0000, 1,  0x00000000 },
+	{ 0x2, 0x9,  0x0,  0, 0x0b3d0000, 0,  0x00000000 },
+	{ 0x2, 0x9,  0x1,  0, 0x0bbd0000, 1,  0x00000000 },
+	{ 0x2, 0xa,  0x0,  0, 0x0b1e0000, 0,  0x00000000 },
+	{ 0x2, 0xa,  0x1,  0, 0x0b9e0000, 1,  0x00000000 },
+	{ 0x2, 0xb,  0x0,  0, 0x0b150000, 0,  0x00000000 },
+	{ 0x2, 0xb,  0x1,  0, 0x0b160000, 1,  0x00010000 },
+	{ 0x2, 0xb,  0x2,  0, 0x0b170000, 2,  0x00020000 },
+	{ 0x2, 0xb,  0x3,  0, 0x0b180000, 3,  0x00030000 },
+	{ 0x2, 0xb,  0x4,  0, 0x0b190000, 4,  0x00040000 },
+	{ 0x2, 0xb,  0x5,  0, 0x0b1a0000, 5,  0x00050000 },
+	{ 0x2, 0xb,  0x6,  0, 0x0b1b0000, 6,  0x00060000 },
+	{ 0x2, 0xb,  0x7,  0, 0x0b1c0000, 7,  0x00070000 },
+	{ 0x2, 0xb,  0x8,  0, 0x0b1d0000, 8,  0x00080000 },
+	{ 0x2, 0xb,  0x9,  0, 0x0b950000, 9,  0x00000000 },
+	{ 0x2, 0xb,  0xa,  0, 0x0b960000, 10, 0x00010000 },
+	{ 0x2, 0xb,  0xb,  0, 0x0b970000, 11, 0x00020000 },
+	{ 0x2, 0xb,  0xc,  0, 0x0b980000, 12, 0x00030000 },
+	{ 0x2, 0xb,  0xd,  0, 0x0b990000, 13, 0x00040000 },
+	{ 0x2, 0xb,  0xe,  0, 0x0b9a0000, 14, 0x00050000 },
+	{ 0x2, 0xb,  0xf,  0, 0x0b9b0000, 15, 0x00060000 },
+	{ 0x2, 0xb,  0x10, 0, 0x0b9c0000, 16, 0x00070000 },
+	{ 0x2, 0xb,  0x11, 0, 0x0b9d0000, 17, 0x00080000 },
+	{ 0x2, 0xc,  0x0,  0, 0x0b660000, 0,  0x00000000 },
+	{ 0x2, 0xc,  0x1,  0, 0x0be60000, 1,  0x00000000 },
+	{ 0x2, 0xd,  0x0,  0, 0x0b1f0000, 0,  0x00000000 },
+	{ 0x2, 0xd,  0x1,  0, 0x0b200000, 1,  0x00010000 },
+	{ 0x2, 0xd,  0x2,  0, 0x0b210000, 2,  0x00020000 },
+	{ 0x2, 0xd,  0x3,  0, 0x0b220000, 3,  0x00030000 },
+	{ 0x2, 0xd,  0x4,  0, 0x0b9f0000, 4,  0x00000000 },
+	{ 0x2, 0xd,  0x5,  0, 0x0ba00000, 5,  0x00010000 },
+	{ 0x2, 0xd,  0x6,  0, 0x0ba10000, 6,  0x00020000 },
+	{ 0x2, 0xd,  0x7,  0, 0x0ba20000, 7,  0x00030000 },
+	{ 0x2, 0xe,  0x0,  0, 0x0b0f0000, 0,  0x00000000 },
+	{ 0x2, 0xe,  0x1,  0, 0x0b100000, 1,  0x00010000 },
+	{ 0x2, 0xe,  0x2,  0, 0x0b110000, 2,  0x00020000 },
+	{ 0x2, 0xe,  0x3,  0, 0x0b120000, 3,  0x00030000 },
+	{ 0x2, 0xe,  0x4,  0, 0x0b130000, 4,  0x00040000 },
+	{ 0x2, 0xe,  0x5,  0, 0x0b140000, 5,  0x00050000 },
+	{ 0x2, 0xe,  0x6,  0, 0x0b8f0000, 6,  0x00000000 },
+	{ 0x2, 0xe,  0x7,  0, 0x0b900000, 7,  0x00010000 },
+	{ 0x2, 0xe,  0x8,  0, 0x0b910000, 8,  0x00020000 },
+	{ 0x2, 0xe,  0x9,  0, 0x0b920000, 9,  0x00030000 },
+	{ 0x2, 0xe,  0xa,  0, 0x0b930000, 10, 0x00040000 },
+	{ 0x2, 0xe,  0xb,  0, 0x0b940000, 11, 0x00050000 },
+	{ 0x2, 0x10, 0x0,  0, 0x0b240000, 0,  0x00000000 },
+	{ 0x2, 0x10, 0x1,  0, 0x0ba40000, 1,  0x00000000 },
+	{ 0x2, 0x11, 0x0,  0, 0x0b020000, 0,  0x00000000 },
+	{ 0x2, 0x11, 0x1,  0, 0x0b820000, 1,  0x00000000 },
+	{ 0x2, 0x12, 0x0,  0, 0x0b030000, 0,  0x00000000 },
+	{ 0x2, 0x12, 0x1,  0, 0x0b830000, 1,  0x00000000 },
+	{ 0x2, 0x17, 0x0,  0, 0x0b640000, 0,  0x00000000 },
+	{ 0x2, 0x17, 0x1,  0, 0x0be40000, 1,  0x00000000 },
+	{ 0x2, 0x18, 0x0,  0, 0x0b600000, 0,  0x00000000 },
+	{ 0x2, 0x18, 0x1,  0, 0x0be00000, 1,  0x00000000 },
+	{ 0x2, 0x18, 0x2,  0, 0x00000000, 0,  0x00000000 },
+	{ 0x2, 0x18, 0x3,  0, 0x00000000, 0,  0x00000000 },
+	{ 0x3, 0x14, 0x0,  0, 0x40000000, 0,  0x40000000 },
+	{ 0x3, 0x14, 0x1,  1, 0x80000000, 1,  0x80000000 },
+	{ 0x3, 0x16, 0x0,  2, 0x0b400000, 0,  0x0b400000 },
+	{ 0x3, 0x16, 0x1,  2, 0x0bc00000, 1,  0x0bc00000 },
+	{ 0x3, 0x16, 0x2,  0, 0x00000000, 0,  0x00000000 },
+	{ 0x3, 0x16, 0x3,  0, 0x00000000, 0,  0x00000000 },
+	{ 0x4, 0x15, 0x0,  0, 0x20c00000, 8,  0x20c00000 },
+	{ 0x4, 0x15, 0x1,  0, 0x21100000, 22, 0x21100000 },
+	{ 0x4, 0x15, 0x2,  0, 0x20e00000, 9,  0x20e00000 },
+	{ 0x4, 0x15, 0x3,  0, 0x21200000, 23, 0x21200000 },
+	{ 0x4, 0x15, 0x4,  0, 0x20800000, 7,  0x20800000 },
+	{ 0x4, 0x15, 0x5,  0, 0x21400000, 24, 0x21400000 },
+	{ 0x4, 0x15, 0x6,  0, 0x0b000000, 18, 0x0b000000 },
+	{ 0x4, 0x15, 0x7,  0, 0x0b800000, 3,  0x0b800000 },
+	{ 0x4, 0x15, 0x8,  0, 0x20000000, 6,  0x20000000 },
+	{ 0x4, 0x15, 0x9,  0, 0x21800000, 25, 0x21800000 },
+	{ 0x4, 0x15, 0xa,  0, 0x0a000000, 2,  0x0a000000 },
+	{ 0x4, 0x15, 0xb,  0, 0x0a000000, 17, 0x0a000000 },
+	{ 0x4, 0x15, 0xc,  0, 0x20000000, 21, 0x20000000 },
+	{ 0x4, 0x15, 0xd,  0, 0x21000000, 10, 0x21000000 },
+	{ 0x4, 0x15, 0xe,  0, 0x08000000, 1,  0x08000000 },
+	{ 0x4, 0x15, 0xf,  0, 0x08000000, 16, 0x08000000 },
+	{ 0x4, 0x15, 0x10, 0, 0x22000000, 11, 0x22000000 },
+	{ 0x4, 0x15, 0x11, 0, 0x22000000, 26, 0x22000000 },
+	{ 0x4, 0x15, 0x12, 0, 0x0c000000, 4,  0x0c000000 },
+	{ 0x4, 0x15, 0x13, 0, 0x0c000000, 19, 0x0c000000 },
+	{ 0x4, 0x15, 0x14, 0, 0x24000000, 12, 0x24000000 },
+	{ 0x4, 0x15, 0x15, 0, 0x24000000, 27, 0x24000000 },
+	{ 0x4, 0x15, 0x16, 0, 0x00000000, 0,  0x00000000 },
+	{ 0x4, 0x15, 0x17, 0, 0x00000000, 15, 0x00000000 },
+	{ 0x4, 0x15, 0x18, 0, 0x28000000, 13, 0x28000000 },
+	{ 0x4, 0x15, 0x19, 0, 0x28000000, 28, 0x28000000 },
+	{ 0x4, 0x15, 0x1a, 0, 0x10000000, 5,  0x10000000 },
+	{ 0x4, 0x15, 0x1b, 0, 0x10000000, 20, 0x10000000 },
+	{ 0x4, 0x15, 0x1c, 0, 0x30000000, 14, 0x30000000 },
+	{ 0x4, 0x15, 0x1d, 0, 0x30000000, 29, 0x30000000 },
+	{ 0x4, 0x0,  0x0,  0, 0x0b000000, 0,  0x00000000 },
+	{ 0x4, 0x0,  0x1,  0, 0x0b800000, 1,  0x00000000 },
+	{ 0x4, 0x1,  0x0,  0, 0x20de0000, 3,  0x000e0000 },
+	{ 0x4, 0x1,  0x1,  0, 0x210e0000, 7,  0x000e0000 },
+	{ 0x4, 0x1,  0x2,  0, 0x20dc0000, 2,  0x000c0000 },
+	{ 0x4, 0x1,  0x3,  0, 0x210c0000, 6,  0x000c0000 },
+	{ 0x4, 0x1,  0x4,  0, 0x20d80000, 1,  0x00080000 },
+	{ 0x4, 0x1,  0x5,  0, 0x21080000, 5,  0x00080000 },
+	{ 0x4, 0x1,  0x6,  0, 0x20d00000, 0,  0x00000000 },
+	{ 0x4, 0x1,  0x7,  0, 0x21000000, 4,  0x00000000 },
+	{ 0x4, 0x2,  0x0,  0, 0x0b040000, 0,  0x00000000 },
+	{ 0x4, 0x2,  0x1,  0, 0x0b840000, 1,  0x00000000 },
+	{ 0x4, 0x3,  0x0,  0, 0x0b230000, 0,  0x00000000 },
+	{ 0x4, 0x3,  0x1,  0, 0x0ba30000, 1,  0x00000000 },
+	{ 0x4, 0x4,  0x0,  0, 0x0b050000, 0,  0x00000000 },
+	{ 0x4, 0x4,  0x1,  0, 0x0b850000, 1,  0x00000000 },
+	{ 0x4, 0x5,  0x0,  0, 0x0b060000, 0,  0x00000000 },
+	{ 0x4, 0x5,  0x1,  0, 0x0b070000, 1,  0x00010000 },
+	{ 0x4, 0x5,  0x2,  0, 0x0b080000, 2,  0x00020000 },
+	{ 0x4, 0x5,  0x3,  0, 0x0b090000, 3,  0x00030000 },
+	{ 0x4, 0x5,  0x4,  0, 0x0b0a0000, 4,  0x00040000 },
+	{ 0x4, 0x5,  0x5,  0, 0x0b0b0000, 5,  0x00050000 },
+	{ 0x4, 0x5,  0x6,  0, 0x0b0c0000, 6,  0x00060000 },
+	{ 0x4, 0x5,  0x7,  0, 0x0b0d0000, 7,  0x00070000 },
+	{ 0x4, 0x5,  0x8,  0, 0x0b0e0000, 8,  0x00080000 },
+	{ 0x4, 0x5,  0x9,  0, 0x0b860000, 9,  0x00000000 },
+	{ 0x4, 0x5,  0xa,  0, 0x0b870000, 10, 0x00010000 },
+	{ 0x4, 0x5,  0xb,  0, 0x0b880000, 11, 0x00020000 },
+	{ 0x4, 0x5,  0xc,  0, 0x0b890000, 12, 0x00030000 },
+	{ 0x4, 0x5,  0xd,  0, 0x0b8a0000, 13, 0x00040000 },
+	{ 0x4, 0x5,  0xe,  0, 0x0b8b0000, 14, 0x00050000 },
+	{ 0x4, 0x5,  0xf,  0, 0x0b8c0000, 15, 0x00060000 },
+	{ 0x4, 0x5,  0x10, 0, 0x0b8d0000, 16, 0x00070000 },
+	{ 0x4, 0x5,  0x11, 0, 0x0b8e0000, 17, 0x00080000 },
+	{ 0x4, 0x6,  0x0,  0, 0x0b650000, 0,  0x00000000 },
+	{ 0x4, 0x6,  0x1,  0, 0x0be50000, 1,  0x00000000 },
+	{ 0x4, 0x7,  0x0,  0, 0x20df0000, 0,  0x00000000 },
+	{ 0x4, 0x7,  0x1,  0, 0x210f0000, 1,  0x00000000 },
+	{ 0x4, 0x8,  0x0,  0, 0x0b3e0000, 0,  0x00000000 },
+	{ 0x4, 0x8,  0x1,  0, 0x0bbe0000, 1,  0x00000000 },
+	{ 0x4, 0x9,  0x0,  0, 0x0b3d0000, 0,  0x00000000 },
+	{ 0x4, 0x9,  0x1,  0, 0x0bbd0000, 1,  0x00000000 },
+	{ 0x4, 0xa,  0x0,  0, 0x0b1e0000, 0,  0x00000000 },
+	{ 0x4, 0xa,  0x1,  0, 0x0b9e0000, 1,  0x00000000 },
+	{ 0x4, 0xb,  0x0,  0, 0x0b150000, 0,  0x00000000 },
+	{ 0x4, 0xb,  0x1,  0, 0x0b160000, 1,  0x00010000 },
+	{ 0x4, 0xb,  0x2,  0, 0x0b170000, 2,  0x00020000 },
+	{ 0x4, 0xb,  0x3,  0, 0x0b180000, 3,  0x00030000 },
+	{ 0x4, 0xb,  0x4,  0, 0x0b190000, 4,  0x00040000 },
+	{ 0x4, 0xb,  0x5,  0, 0x0b1a0000, 5,  0x00050000 },
+	{ 0x4, 0xb,  0x6,  0, 0x0b1b0000, 6,  0x00060000 },
+	{ 0x4, 0xb,  0x7,  0, 0x0b1c0000, 7,  0x00070000 },
+	{ 0x4, 0xb,  0x8,  0, 0x0b1d0000, 8,  0x00080000 },
+	{ 0x4, 0xb,  0x9,  0, 0x0b950000, 9,  0x00000000 },
+	{ 0x4, 0xb,  0xa,  0, 0x0b960000, 10, 0x00010000 },
+	{ 0x4, 0xb,  0xb,  0, 0x0b970000, 11, 0x00020000 },
+	{ 0x4, 0xb,  0xc,  0, 0x0b980000, 12, 0x00030000 },
+	{ 0x4, 0xb,  0xd,  0, 0x0b990000, 13, 0x00040000 },
+	{ 0x4, 0xb,  0xe,  0, 0x0b9a0000, 14, 0x00050000 },
+	{ 0x4, 0xb,  0xf,  0, 0x0b9b0000, 15, 0x00060000 },
+	{ 0x4, 0xb,  0x10, 0, 0x0b9c0000, 16, 0x00070000 },
+	{ 0x4, 0xb,  0x11, 0, 0x0b9d0000, 17, 0x00080000 },
+	{ 0x4, 0xc,  0x0,  0, 0x0b660000, 0,  0x00000000 },
+	{ 0x4, 0xc,  0x1,  0, 0x0be60000, 1,  0x00000000 },
+	{ 0x4, 0xd,  0x0,  0, 0x0b1f0000, 0,  0x00000000 },
+	{ 0x4, 0xd,  0x1,  0, 0x0b200000, 1,  0x00010000 },
+	{ 0x4, 0xd,  0x2,  0, 0x0b210000, 2,  0x00020000 },
+	{ 0x4, 0xd,  0x3,  0, 0x0b220000, 3,  0x00030000 },
+	{ 0x4, 0xd,  0x4,  0, 0x0b9f0000, 4,  0x00000000 },
+	{ 0x4, 0xd,  0x5,  0, 0x0ba00000, 5,  0x00010000 },
+	{ 0x4, 0xd,  0x6,  0, 0x0ba10000, 6,  0x00020000 },
+	{ 0x4, 0xd,  0x7,  0, 0x0ba20000, 7,  0x00030000 },
+	{ 0x4, 0xe,  0x0,  0, 0x0b0f0000, 0,  0x00000000 },
+	{ 0x4, 0xe,  0x1,  0, 0x0b100000, 1,  0x00010000 },
+	{ 0x4, 0xe,  0x2,  0, 0x0b110000, 2,  0x00020000 },
+	{ 0x4, 0xe,  0x3,  0, 0x0b120000, 3,  0x00030000 },
+	{ 0x4, 0xe,  0x4,  0, 0x0b130000, 4,  0x00040000 },
+	{ 0x4, 0xe,  0x5,  0, 0x0b140000, 5,  0x00050000 },
+	{ 0x4, 0xe,  0x6,  0, 0x0b8f0000, 6,  0x00000000 },
+	{ 0x4, 0xe,  0x7,  0, 0x0b900000, 7,  0x00010000 },
+	{ 0x4, 0xe,  0x8,  0, 0x0b910000, 8,  0x00020000 },
+	{ 0x4, 0xe,  0x9,  0, 0x0b920000, 9,  0x00030000 },
+	{ 0x4, 0xe,  0xa,  0, 0x0b930000, 10, 0x00040000 },
+	{ 0x4, 0xe,  0xb,  0, 0x0b940000, 11, 0x00050000 },
+	{ 0x4, 0x10, 0x0,  0, 0x0b240000, 0,  0x00000000 },
+	{ 0x4, 0x10, 0x1,  0, 0x0ba40000, 1,  0x00000000 },
+	{ 0x4, 0x11, 0x0,  0, 0x0b020000, 0,  0x00000000 },
+	{ 0x4, 0x11, 0x1,  0, 0x0b820000, 1,  0x00000000 },
+	{ 0x4, 0x12, 0x0,  0, 0x0b030000, 0,  0x00000000 },
+	{ 0x4, 0x12, 0x1,  0, 0x0b830000, 1,  0x00000000 },
+	{ 0x4, 0x17, 0x0,  0, 0x0b640000, 0,  0x00000000 },
+	{ 0x4, 0x17, 0x1,  0, 0x0be40000, 1,  0x00000000 },
+	{ 0x4, 0x18, 0x0,  0, 0x0b600000, 0,  0x00000000 },
+	{ 0x4, 0x18, 0x1,  0, 0x0be00000, 1,  0x00000000 },
+	{ 0x4, 0x18, 0x2,  0, 0x00000000, 0,  0x00000000 },
+	{ 0x4, 0x18, 0x3,  0, 0x00000000, 0,  0x00000000 }
+};
+
+static void cbbcentralnoc_parse_routeid(struct tegra194_cbb_aperture *info, u64 routeid)
+{
+	info->initflow = FIELD_GET(CBB_NOC_INITFLOW, routeid);
+	info->targflow = FIELD_GET(CBB_NOC_TARGFLOW, routeid);
+	info->targ_subrange = FIELD_GET(CBB_NOC_TARG_SUBRANGE, routeid);
+	info->seqid = FIELD_GET(CBB_NOC_SEQID, routeid);
+}
+
+static void bpmpnoc_parse_routeid(struct tegra194_cbb_aperture *info, u64 routeid)
+{
+	info->initflow = FIELD_GET(BPMP_NOC_INITFLOW, routeid);
+	info->targflow = FIELD_GET(BPMP_NOC_TARGFLOW, routeid);
+	info->targ_subrange = FIELD_GET(BPMP_NOC_TARG_SUBRANGE, routeid);
+	info->seqid = FIELD_GET(BPMP_NOC_SEQID, routeid);
+}
+
+static void aonnoc_parse_routeid(struct tegra194_cbb_aperture *info, u64 routeid)
+{
+	info->initflow = FIELD_GET(AON_NOC_INITFLOW, routeid);
+	info->targflow = FIELD_GET(AON_NOC_TARGFLOW, routeid);
+	info->targ_subrange = FIELD_GET(AON_NOC_TARG_SUBRANGE, routeid);
+	info->seqid = FIELD_GET(AON_NOC_SEQID, routeid);
+}
+
+static void scenoc_parse_routeid(struct tegra194_cbb_aperture *info, u64 routeid)
+{
+	info->initflow = FIELD_GET(SCE_NOC_INITFLOW, routeid);
+	info->targflow = FIELD_GET(SCE_NOC_TARGFLOW, routeid);
+	info->targ_subrange = FIELD_GET(SCE_NOC_TARG_SUBRANGE, routeid);
+	info->seqid = FIELD_GET(SCE_NOC_SEQID, routeid);
+}
+
+static void cbbcentralnoc_parse_userbits(struct tegra194_cbb_userbits *usrbits, u32 elog_5)
+{
+	usrbits->axcache = FIELD_GET(CBB_NOC_AXCACHE, elog_5);
+	usrbits->non_mod = FIELD_GET(CBB_NOC_NON_MOD, elog_5);
+	usrbits->axprot = FIELD_GET(CBB_NOC_AXPROT, elog_5);
+	usrbits->falconsec = FIELD_GET(CBB_NOC_FALCONSEC, elog_5);
+	usrbits->grpsec = FIELD_GET(CBB_NOC_GRPSEC, elog_5);
+	usrbits->vqc = FIELD_GET(CBB_NOC_VQC, elog_5);
+	usrbits->mstr_id = FIELD_GET(CBB_NOC_MSTR_ID, elog_5) - 1;
+	usrbits->axi_id = FIELD_GET(CBB_NOC_AXI_ID, elog_5);
+}
+
+static void clusternoc_parse_userbits(struct tegra194_cbb_userbits *usrbits, u32 elog_5)
+{
+	usrbits->axcache = FIELD_GET(CLUSTER_NOC_AXCACHE, elog_5);
+	usrbits->axprot = FIELD_GET(CLUSTER_NOC_AXCACHE, elog_5);
+	usrbits->falconsec = FIELD_GET(CLUSTER_NOC_FALCONSEC, elog_5);
+	usrbits->grpsec = FIELD_GET(CLUSTER_NOC_GRPSEC, elog_5);
+	usrbits->vqc = FIELD_GET(CLUSTER_NOC_VQC, elog_5);
+	usrbits->mstr_id = FIELD_GET(CLUSTER_NOC_MSTR_ID, elog_5) - 1;
+}
+
+static void tegra194_cbb_fault_enable(struct tegra_cbb *cbb)
+{
+	struct tegra194_cbb *priv = to_tegra194_cbb(cbb);
+
+	writel(1, priv->regs + ERRLOGGER_0_FAULTEN_0);
+	writel(1, priv->regs + ERRLOGGER_1_FAULTEN_0);
+	writel(1, priv->regs + ERRLOGGER_2_FAULTEN_0);
+}
+
+static void tegra194_cbb_stall_enable(struct tegra_cbb *cbb)
+{
+	struct tegra194_cbb *priv = to_tegra194_cbb(cbb);
+
+	writel(1, priv->regs + ERRLOGGER_0_STALLEN_0);
+	writel(1, priv->regs + ERRLOGGER_1_STALLEN_0);
+	writel(1, priv->regs + ERRLOGGER_2_STALLEN_0);
+}
+
+static void tegra194_cbb_error_clear(struct tegra_cbb *cbb)
+{
+	struct tegra194_cbb *priv = to_tegra194_cbb(cbb);
+
+	writel(1, priv->regs + ERRLOGGER_0_ERRCLR_0);
+	writel(1, priv->regs + ERRLOGGER_1_ERRCLR_0);
+	writel(1, priv->regs + ERRLOGGER_2_ERRCLR_0);
+	dsb(sy);
+}
+
+static u32 tegra194_cbb_get_status(struct tegra_cbb *cbb)
+{
+	struct tegra194_cbb *priv = to_tegra194_cbb(cbb);
+	u32 value;
+
+	value = readl(priv->regs + ERRLOGGER_0_ERRVLD_0);
+	value |= (readl(priv->regs + ERRLOGGER_1_ERRVLD_0) << 1);
+	value |= (readl(priv->regs + ERRLOGGER_2_ERRVLD_0) << 2);
+
+	dsb(sy);
+	return value;
+}
+
+static u32 tegra194_axi2apb_status(void __iomem *addr)
+{
+	u32 value;
+
+	value = readl(addr + DMAAPB_X_RAW_INTERRUPT_STATUS);
+	writel(0xffffffff, addr + DMAAPB_X_RAW_INTERRUPT_STATUS);
+
+	return value;
+}
+
+static bool tegra194_axi2apb_fatal(struct seq_file *file, unsigned int bridge, u32 status)
+{
+	bool is_fatal = true;
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(tegra194_axi2apb_error); i++) {
+		if (status & BIT(i)) {
+			tegra_cbb_print_err(file, "\t  AXI2APB_%d bridge error: %s\n",
+					    bridge + 1, tegra194_axi2apb_error[i]);
+			if (strstr(tegra194_axi2apb_error[i], "Firewall"))
+				is_fatal = false;
+		}
+	}
+
+	return is_fatal;
+}
+
+/*
+ * Fetch InitlocalAddress from NOC Aperture lookup table
+ * using Targflow, Targsubrange
+ */
+static u32 get_init_localaddress(const struct tegra194_cbb_aperture *info,
+				 const struct tegra194_cbb_aperture *aper, unsigned int max)
+{
+	unsigned int t_f = 0, t_sr = 0;
+	u32 addr = 0;
+
+	for (t_f = 0; t_f < max; t_f++) {
+		if (aper[t_f].targflow == info->targflow) {
+			t_sr = t_f;
+
+			do {
+				if (aper[t_sr].targ_subrange == info->targ_subrange) {
+					addr = aper[t_sr].init_localaddress;
+					return addr;
+				}
+
+				if (t_sr >= max)
+					return 0;
+
+				t_sr++;
+			} while (aper[t_sr].targflow == aper[t_sr - 1].targflow);
+
+			t_f = t_sr;
+		}
+	}
+
+	return addr;
+}
+
+static void print_errlog5(struct seq_file *file, struct tegra194_cbb *cbb)
+{
+	struct tegra194_cbb_userbits userbits;
+
+	cbb->noc->parse_userbits(&userbits, cbb->errlog5);
+
+	if (!strcmp(cbb->noc->name, "cbb-noc")) {
+		tegra_cbb_print_err(file, "\t  Non-Modify\t\t: %#x\n", userbits.non_mod);
+		tegra_cbb_print_err(file, "\t  AXI ID\t\t: %#x\n", userbits.axi_id);
+	}
+
+	tegra_cbb_print_err(file, "\t  Master ID\t\t: %s\n",
+			    cbb->noc->master_id[userbits.mstr_id]);
+	tegra_cbb_print_err(file, "\t  Security Group(GRPSEC): %#x\n", userbits.grpsec);
+	tegra_cbb_print_cache(file, userbits.axcache);
+	tegra_cbb_print_prot(file, userbits.axprot);
+	tegra_cbb_print_err(file, "\t  FALCONSEC\t\t: %#x\n", userbits.falconsec);
+	tegra_cbb_print_err(file, "\t  Virtual Queuing Channel(VQC): %#x\n", userbits.vqc);
+}
+
+/*
+ *  Fetch Base Address/InitlocalAddress from NOC aperture lookup table using TargFlow &
+ *  Targ_subRange extracted from RouteId. Perform address reconstruction as below:
+ *
+ *    Address = Base Address + (ErrLog3 + ErrLog4)
+ */
+static void
+print_errlog3_4(struct seq_file *file, u32 errlog3, u32 errlog4,
+		const struct tegra194_cbb_aperture *info,
+		const struct tegra194_cbb_aperture *aperture, unsigned int max)
+{
+	u64 addr = (u64)errlog4 << 32 | errlog3;
+
+	/*
+	 * If errlog4[7] = "1", then it's a joker entry. Joker entries are a rare phenomenon and
+	 * such addresses are not reliable. Debugging should be done using only the RouteId
+	 * information.
+	 */
+	if (errlog4 & 0x80)
+		tegra_cbb_print_err(file, "\t debug using RouteId alone as below address is a "
+					  "joker entry and not reliable");
+
+	addr += get_init_localaddress(info, aperture, max);
+
+	tegra_cbb_print_err(file, "\t  Address accessed\t: %#llx\n", addr);
+}
+
+/*
+ *  Get RouteId from ErrLog1+ErrLog2 registers and fetch values of
+ *  InitFlow, TargFlow, Targ_subRange and SeqId values from RouteId
+ */
+static void
+print_errlog1_2(struct seq_file *file, struct tegra194_cbb *cbb,
+		struct tegra194_cbb_aperture *info)
+{
+	u64 routeid = (u64)cbb->errlog2 << 32 | cbb->errlog1;
+	u32 seqid = 0;
+
+	tegra_cbb_print_err(file, "\t  RouteId\t\t: %#llx\n", routeid);
+
+	cbb->noc->parse_routeid(info, routeid);
+
+	tegra_cbb_print_err(file, "\t  InitFlow\t\t: %s\n",
+			    cbb->noc->routeid_initflow[info->initflow]);
+
+	tegra_cbb_print_err(file, "\t  Targflow\t\t: %s\n",
+			    cbb->noc->routeid_targflow[info->targflow]);
+
+	tegra_cbb_print_err(file, "\t  TargSubRange\t\t: %d\n", info->targ_subrange);
+	tegra_cbb_print_err(file, "\t  SeqId\t\t\t: %d\n", seqid);
+}
+
+/*
+ * Print transcation type, error code and description from ErrLog0 for all
+ * errors. For NOC slave errors, all relevant error info is printed using
+ * ErrLog0 only. But additional information is printed for errors from
+ * APB slaves because for them:
+ *  - All errors are logged as SLV(slave) errors due to APB having only single
+ *    bit pslverr to report all errors.
+ *  - Exact cause is printed by reading DMAAPB_X_RAW_INTERRUPT_STATUS register.
+ *  - The driver prints information showing AXI2APB bridge and exact error
+ *    only if there is error in any AXI2APB slave.
+ *  - There is still no way to disambiguate a DEC error from SLV error type.
+ */
+static bool print_errlog0(struct seq_file *file, struct tegra194_cbb *cbb)
+{
+	struct tegra194_cbb_packet_header hdr;
+	bool is_fatal = true;
+
+	hdr.lock = cbb->errlog0 & 0x1;
+	hdr.opc = FIELD_GET(CBB_ERR_OPC, cbb->errlog0);
+	hdr.errcode = FIELD_GET(CBB_ERR_ERRCODE, cbb->errlog0);
+	hdr.len1 = FIELD_GET(CBB_ERR_LEN1, cbb->errlog0);
+	hdr.format = (cbb->errlog0 >> 31);
+
+	tegra_cbb_print_err(file, "\t  Transaction Type\t: %s\n",
+			    tegra194_cbb_trantype[hdr.opc]);
+	tegra_cbb_print_err(file, "\t  Error Code\t\t: %s\n",
+			    tegra194_cbb_errors[hdr.errcode].code);
+	tegra_cbb_print_err(file, "\t  Error Source\t\t: %s\n",
+			    tegra194_cbb_errors[hdr.errcode].source);
+	tegra_cbb_print_err(file, "\t  Error Description\t: %s\n",
+			    tegra194_cbb_errors[hdr.errcode].desc);
+
+	/*
+	 * Do not crash system for errors which are only notifications to indicate a transaction
+	 * was not allowed to be attempted.
+	 */
+	if (!strcmp(tegra194_cbb_errors[hdr.errcode].code, "SEC") ||
+	    !strcmp(tegra194_cbb_errors[hdr.errcode].code, "DEC") ||
+	    !strcmp(tegra194_cbb_errors[hdr.errcode].code, "UNS") ||
+	    !strcmp(tegra194_cbb_errors[hdr.errcode].code, "DISC")) {
+		is_fatal = false;
+	} else if (!strcmp(tegra194_cbb_errors[hdr.errcode].code, "SLV") &&
+		   cbb->num_bridges > 0) {
+		unsigned int i;
+		u32 status;
+
+		/* For all SLV errors, read DMAAPB_X_RAW_INTERRUPT_STATUS
+		 * register to get error status for all AXI2APB bridges.
+		 * Print bridge details if a bit is set in a bridge's
+		 * status register due to error in a APB slave connected
+		 * to that bridge. For other NOC slaves, none of the status
+		 * register will be set.
+		 */
+
+		for (i = 0; i < cbb->num_bridges; i++) {
+			status = tegra194_axi2apb_status(cbb->bridges[i].base);
+
+			if (status)
+				is_fatal = tegra194_axi2apb_fatal(file, i, status);
+		}
+	}
+
+	tegra_cbb_print_err(file, "\t  Packet header Lock\t: %d\n", hdr.lock);
+	tegra_cbb_print_err(file, "\t  Packet header Len1\t: %d\n", hdr.len1);
+
+	if (hdr.format)
+		tegra_cbb_print_err(file, "\t  NOC protocol version\t: %s\n",
+				    "version >= 2.7");
+	else
+		tegra_cbb_print_err(file, "\t  NOC protocol version\t: %s\n",
+				    "version < 2.7");
+
+	return is_fatal;
+}
+
+/*
+ * Print debug information about failed transaction using
+ * ErrLog registers of error loggger having ErrVld set
+ */
+static bool print_errloggerX_info(struct seq_file *file, struct tegra194_cbb *cbb,
+				  int errloggerX)
+{
+	struct tegra194_cbb_aperture info = { 0, };
+	bool is_fatal = true;
+
+	tegra_cbb_print_err(file, "\tError Logger\t\t: %d\n", errloggerX);
+
+	if (errloggerX == 0) {
+		cbb->errlog0 = readl(cbb->regs + ERRLOGGER_0_ERRLOG0_0);
+		cbb->errlog1 = readl(cbb->regs + ERRLOGGER_0_ERRLOG1_0);
+		cbb->errlog2 = readl(cbb->regs + ERRLOGGER_0_RSVD_00_0);
+		cbb->errlog3 = readl(cbb->regs + ERRLOGGER_0_ERRLOG3_0);
+		cbb->errlog4 = readl(cbb->regs + ERRLOGGER_0_ERRLOG4_0);
+		cbb->errlog5 = readl(cbb->regs + ERRLOGGER_0_ERRLOG5_0);
+	} else if (errloggerX == 1) {
+		cbb->errlog0 = readl(cbb->regs + ERRLOGGER_1_ERRLOG0_0);
+		cbb->errlog1 = readl(cbb->regs + ERRLOGGER_1_ERRLOG1_0);
+		cbb->errlog2 = readl(cbb->regs + ERRLOGGER_1_RSVD_00_0);
+		cbb->errlog3 = readl(cbb->regs + ERRLOGGER_1_ERRLOG3_0);
+		cbb->errlog4 = readl(cbb->regs + ERRLOGGER_1_ERRLOG4_0);
+		cbb->errlog5 = readl(cbb->regs + ERRLOGGER_1_ERRLOG5_0);
+	} else if (errloggerX == 2) {
+		cbb->errlog0 = readl(cbb->regs + ERRLOGGER_2_ERRLOG0_0);
+		cbb->errlog1 = readl(cbb->regs + ERRLOGGER_2_ERRLOG1_0);
+		cbb->errlog2 = readl(cbb->regs + ERRLOGGER_2_RSVD_00_0);
+		cbb->errlog3 = readl(cbb->regs + ERRLOGGER_2_ERRLOG3_0);
+		cbb->errlog4 = readl(cbb->regs + ERRLOGGER_2_ERRLOG4_0);
+		cbb->errlog5 = readl(cbb->regs + ERRLOGGER_2_ERRLOG5_0);
+	}
+
+	tegra_cbb_print_err(file, "\tErrLog0\t\t\t: %#x\n", cbb->errlog0);
+	is_fatal = print_errlog0(file, cbb);
+
+	tegra_cbb_print_err(file, "\tErrLog1\t\t\t: %#x\n", cbb->errlog1);
+	tegra_cbb_print_err(file, "\tErrLog2\t\t\t: %#x\n", cbb->errlog2);
+	print_errlog1_2(file, cbb, &info);
+
+	tegra_cbb_print_err(file, "\tErrLog3\t\t\t: %#x\n", cbb->errlog3);
+	tegra_cbb_print_err(file, "\tErrLog4\t\t\t: %#x\n", cbb->errlog4);
+	print_errlog3_4(file, cbb->errlog3, cbb->errlog4, &info, cbb->noc->noc_aperture,
+			cbb->noc->max_aperture);
+
+	tegra_cbb_print_err(file, "\tErrLog5\t\t\t: %#x\n", cbb->errlog5);
+
+	if (cbb->errlog5)
+		print_errlog5(file, cbb);
+
+	return is_fatal;
+}
+
+static bool print_errlog(struct seq_file *file, struct tegra194_cbb *cbb, u32 errvld)
+{
+	bool is_fatal = true;
+
+	pr_crit("**************************************\n");
+	pr_crit("CPU:%d, Error:%s\n", smp_processor_id(), cbb->noc->name);
+
+	if (errvld & 0x1)
+		is_fatal = print_errloggerX_info(file, cbb, 0);
+	else if (errvld & 0x2)
+		is_fatal = print_errloggerX_info(file, cbb, 1);
+	else if (errvld & 0x4)
+		is_fatal = print_errloggerX_info(file, cbb, 2);
+
+	tegra_cbb_error_clear(&cbb->base);
+	tegra_cbb_print_err(file, "\t**************************************\n");
+	return is_fatal;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static DEFINE_MUTEX(cbb_err_mutex);
+
+static int tegra194_cbb_debugfs_show(struct tegra_cbb *cbb, struct seq_file *file, void *data)
+{
+	struct tegra_cbb *noc;
+
+	mutex_lock(&cbb_err_mutex);
+
+	list_for_each_entry(noc, &cbb_list, node) {
+		struct tegra194_cbb *priv = to_tegra194_cbb(noc);
+		u32 status;
+
+		status = tegra_cbb_get_status(noc);
+		if (status)
+			print_errlog(file, priv, status);
+	}
+
+	mutex_unlock(&cbb_err_mutex);
+
+	return 0;
+}
+#endif
+
+/*
+ * Handler for CBB errors from different initiators
+ */
+static irqreturn_t tegra194_cbb_err_isr(int irq, void *data)
+{
+	bool is_inband_err = false, is_fatal = false;
+	//struct tegra194_cbb *cbb = data;
+	struct tegra_cbb *noc;
+	unsigned long flags;
+	u8 mstr_id = 0;
+
+	spin_lock_irqsave(&cbb_lock, flags);
+
+	/* XXX only process interrupts for "cbb" instead of iterating over all NOCs? */
+	list_for_each_entry(noc, &cbb_list, node) {
+		struct tegra194_cbb *priv = to_tegra194_cbb(noc);
+		u32 status = 0;
+
+		status = tegra_cbb_get_status(noc);
+
+		if (status && ((irq == priv->sec_irq) || (irq == priv->nonsec_irq))) {
+			tegra_cbb_print_err(NULL, "CPU:%d, Error: %s@%llx, irq=%d\n",
+					    smp_processor_id(), priv->noc->name, priv->res->start,
+					    irq);
+
+			mstr_id =  FIELD_GET(USRBITS_MSTR_ID, priv->errlog5) - 1;
+			is_fatal = print_errlog(NULL, priv, status);
+
+			/*
+			 * If illegal request is from CCPLEX(0x1)
+			 * initiator then call BUG() to crash system.
+			 */
+			if ((mstr_id == 0x1) && priv->noc->erd_mask_inband_err)
+				is_inband_err = 1;
+		}
+	}
+
+	spin_unlock_irqrestore(&cbb_lock, flags);
+
+	if (is_inband_err) {
+		if (is_fatal)
+			BUG();
+		else
+			WARN(true, "Warning due to CBB Error\n");
+	}
+
+	return IRQ_HANDLED;
+}
+
+/*
+ * Register handler for CBB_NONSECURE & CBB_SECURE interrupts
+ * for reporting CBB errors
+ */
+static int tegra194_cbb_interrupt_enable(struct tegra_cbb *cbb)
+{
+	struct tegra194_cbb *priv = to_tegra194_cbb(cbb);
+	struct device *dev = cbb->dev;
+	int err;
+
+	if (priv->sec_irq) {
+		err = devm_request_irq(dev, priv->sec_irq, tegra194_cbb_err_isr, 0, dev_name(dev),
+				       priv);
+		if (err) {
+			dev_err(dev, "failed to register interrupt %u: %d\n", priv->sec_irq, err);
+			return err;
+		}
+	}
+
+	if (priv->nonsec_irq) {
+		err = devm_request_irq(dev, priv->nonsec_irq, tegra194_cbb_err_isr, 0,
+				       dev_name(dev), priv);
+		if (err) {
+			dev_err(dev, "failed to register interrupt %u: %d\n", priv->nonsec_irq,
+				err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static void tegra194_cbb_error_enable(struct tegra_cbb *cbb)
+{
+	/*
+	 * Set “StallEn=1� to enable queuing of error packets till
+	 * first is served & cleared
+	 */
+	tegra_cbb_stall_enable(cbb);
+
+	/* set “FaultEn=1� to enable error reporting signal “Fault� */
+	tegra_cbb_fault_enable(cbb);
+}
+
+static const struct tegra_cbb_ops tegra194_cbb_ops = {
+	.get_status = tegra194_cbb_get_status,
+	.error_clear = tegra194_cbb_error_clear,
+	.fault_enable = tegra194_cbb_fault_enable,
+	.stall_enable = tegra194_cbb_stall_enable,
+	.error_enable = tegra194_cbb_error_enable,
+	.interrupt_enable = tegra194_cbb_interrupt_enable,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_show = tegra194_cbb_debugfs_show,
+#endif
+};
+
+static struct tegra194_cbb_noc_data tegra194_cbb_central_noc_data = {
+	.name = "cbb-noc",
+	.erd_mask_inband_err = true,
+	.master_id = tegra194_master_id,
+	.noc_aperture = tegra194_cbbcentralnoc_apert_lookup,
+	.max_aperture = ARRAY_SIZE(tegra194_cbbcentralnoc_apert_lookup),
+	.routeid_initflow = tegra194_cbbcentralnoc_routeid_initflow,
+	.routeid_targflow = tegra194_cbbcentralnoc_routeid_targflow,
+	.parse_routeid = cbbcentralnoc_parse_routeid,
+	.parse_userbits = cbbcentralnoc_parse_userbits
+};
+
+static struct tegra194_cbb_noc_data tegra194_aon_noc_data = {
+	.name = "aon-noc",
+	.erd_mask_inband_err = false,
+	.master_id = tegra194_master_id,
+	.noc_aperture = tegra194_aonnoc_aperture_lookup,
+	.max_aperture = ARRAY_SIZE(tegra194_aonnoc_aperture_lookup),
+	.routeid_initflow = tegra194_aonnoc_routeid_initflow,
+	.routeid_targflow = tegra194_aonnoc_routeid_targflow,
+	.parse_routeid = aonnoc_parse_routeid,
+	.parse_userbits = clusternoc_parse_userbits
+};
+
+static struct tegra194_cbb_noc_data tegra194_bpmp_noc_data = {
+	.name = "bpmp-noc",
+	.erd_mask_inband_err = false,
+	.master_id = tegra194_master_id,
+	.noc_aperture = tegra194_bpmpnoc_apert_lookup,
+	.max_aperture = ARRAY_SIZE(tegra194_bpmpnoc_apert_lookup),
+	.routeid_initflow = tegra194_bpmpnoc_routeid_initflow,
+	.routeid_targflow = tegra194_bpmpnoc_routeid_targflow,
+	.parse_routeid = bpmpnoc_parse_routeid,
+	.parse_userbits = clusternoc_parse_userbits
+};
+
+static struct tegra194_cbb_noc_data tegra194_rce_noc_data = {
+	.name = "rce-noc",
+	.erd_mask_inband_err = false,
+	.master_id = tegra194_master_id,
+	.noc_aperture = tegra194_scenoc_apert_lookup,
+	.max_aperture = ARRAY_SIZE(tegra194_scenoc_apert_lookup),
+	.routeid_initflow = tegra194_scenoc_routeid_initflow,
+	.routeid_targflow = tegra194_scenoc_routeid_targflow,
+	.parse_routeid = scenoc_parse_routeid,
+	.parse_userbits = clusternoc_parse_userbits
+};
+
+static struct tegra194_cbb_noc_data tegra194_sce_noc_data = {
+	.name = "sce-noc",
+	.erd_mask_inband_err = false,
+	.master_id = tegra194_master_id,
+	.noc_aperture = tegra194_scenoc_apert_lookup,
+	.max_aperture = ARRAY_SIZE(tegra194_scenoc_apert_lookup),
+	.routeid_initflow = tegra194_scenoc_routeid_initflow,
+	.routeid_targflow = tegra194_scenoc_routeid_targflow,
+	.parse_routeid = scenoc_parse_routeid,
+	.parse_userbits = clusternoc_parse_userbits
+};
+
+static const struct of_device_id tegra194_cbb_match[] = {
+	{ .compatible = "nvidia,tegra194-cbb-noc", .data = &tegra194_cbb_central_noc_data },
+	{ .compatible = "nvidia,tegra194-aon-noc", .data = &tegra194_aon_noc_data },
+	{ .compatible = "nvidia,tegra194-bpmp-noc", .data = &tegra194_bpmp_noc_data },
+	{ .compatible = "nvidia,tegra194-rce-noc", .data = &tegra194_rce_noc_data },
+	{ .compatible = "nvidia,tegra194-sce-noc", .data = &tegra194_sce_noc_data },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, tegra194_cbb_match);
+
+static int tegra194_cbb_get_bridges(struct tegra194_cbb *cbb, struct device_node *np)
+{
+	struct tegra_cbb *entry;
+	struct resource res;
+	unsigned long flags;
+	unsigned int i;
+	int err;
+
+	spin_lock_irqsave(&cbb_lock, flags);
+
+	list_for_each_entry(entry, &cbb_list, node) {
+		struct tegra194_cbb *priv = to_tegra194_cbb(entry);
+
+		if (priv->bridges) {
+			cbb->num_bridges = priv->num_bridges;
+			cbb->bridges = priv->bridges;
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&cbb_lock, flags);
+
+	if (!cbb->bridges) {
+		while (of_address_to_resource(np, cbb->num_bridges, &res) == 0)
+			cbb->num_bridges++;
+
+		cbb->bridges = devm_kcalloc(cbb->base.dev, cbb->num_bridges,
+					    sizeof(*cbb->bridges), GFP_KERNEL);
+		if (!cbb->bridges)
+			return -ENOMEM;
+
+		for (i = 0; i < cbb->num_bridges; i++) {
+			err = of_address_to_resource(np, i, &cbb->bridges[i].res);
+			if (err < 0)
+				return err;
+
+			cbb->bridges[i].base = devm_ioremap_resource(cbb->base.dev,
+								     &cbb->bridges[i].res);
+			if (IS_ERR(cbb->bridges[i].base)) {
+				dev_err(cbb->base.dev, "failed to map AXI2APB range\n");
+				return PTR_ERR(cbb->bridges[i].base);
+			}
+		}
+	}
+
+	if (cbb->num_bridges > 0) {
+		dev_dbg(cbb->base.dev, "AXI2APB bridge info present:\n");
+
+		for (i = 0; i < cbb->num_bridges; i++)
+			dev_dbg(cbb->base.dev, "  %u: %pR\n", i, &cbb->bridges[i].res);
+	}
+
+	return 0;
+}
+
+static int tegra194_cbb_probe(struct platform_device *pdev)
+{
+	const struct tegra194_cbb_noc_data *noc;
+	struct tegra194_cbb *cbb;
+	struct device_node *np;
+	unsigned long flags;
+	int err;
+
+	noc = of_device_get_match_data(&pdev->dev);
+
+	if (noc->erd_mask_inband_err) {
+		/*
+		 * Set Error Response Disable(ERD) bit to mask SError/inband
+		 * error and only trigger interrupts for illegal access from
+		 * CCPLEX initiator.
+		 */
+		err = tegra194_miscreg_mask_serror();
+		if (err) {
+			dev_err(&pdev->dev, "couldn't mask inband errors\n");
+			return err;
+		}
+	}
+
+	cbb = devm_kzalloc(&pdev->dev, sizeof(*cbb), GFP_KERNEL);
+	if (!cbb)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&cbb->base.node);
+	cbb->base.ops = &tegra194_cbb_ops;
+	cbb->base.dev = &pdev->dev;
+	cbb->noc = noc;
+
+	cbb->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &cbb->res);
+	if (IS_ERR(cbb->regs))
+		return PTR_ERR(cbb->regs);
+
+	err = tegra_cbb_get_irq(pdev, &cbb->nonsec_irq, &cbb->sec_irq);
+	if (err)
+		return err;
+
+	np = of_parse_phandle(pdev->dev.of_node, "nvidia,axi2apb", 0);
+	if (np) {
+		err = tegra194_cbb_get_bridges(cbb, np);
+		of_node_put(np);
+		if (err < 0)
+			return err;
+	}
+
+	platform_set_drvdata(pdev, cbb);
+
+	spin_lock_irqsave(&cbb_lock, flags);
+	list_add(&cbb->base.node, &cbb_list);
+	spin_unlock_irqrestore(&cbb_lock, flags);
+
+	return tegra_cbb_register(&cbb->base);
+}
+
+static int tegra194_cbb_remove(struct platform_device *pdev)
+{
+	struct tegra194_cbb *cbb = platform_get_drvdata(pdev);
+	struct tegra_cbb *noc, *tmp;
+	unsigned long flags;
+
+	spin_lock_irqsave(&cbb_lock, flags);
+
+	list_for_each_entry_safe(noc, tmp, &cbb_list, node) {
+		struct tegra194_cbb *priv = to_tegra194_cbb(noc);
+
+		if (cbb->res->start == priv->res->start) {
+			list_del(&noc->node);
+			break;
+		}
+	}
+
+	spin_unlock_irqrestore(&cbb_lock, flags);
+
+	return 0;
+}
+
+static int __maybe_unused tegra194_cbb_resume_noirq(struct device *dev)
+{
+	struct tegra194_cbb *cbb = dev_get_drvdata(dev);
+
+	tegra194_cbb_error_enable(&cbb->base);
+	dsb(sy);
+
+	dev_dbg(dev, "%s resumed\n", cbb->noc->name);
+	return 0;
+}
+
+static const struct dev_pm_ops tegra194_cbb_pm = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, tegra194_cbb_resume_noirq)
+};
+
+static struct platform_driver tegra194_cbb_driver = {
+	.probe = tegra194_cbb_probe,
+	.remove = tegra194_cbb_remove,
+	.driver = {
+		.name = "tegra194-cbb",
+		.of_match_table = of_match_ptr(tegra194_cbb_match),
+		.pm = &tegra194_cbb_pm,
+	},
+};
+
+static int __init tegra194_cbb_init(void)
+{
+	return platform_driver_register(&tegra194_cbb_driver);
+}
+pure_initcall(tegra194_cbb_init);
+
+static void __exit tegra194_cbb_exit(void)
+{
+	platform_driver_unregister(&tegra194_cbb_driver);
+}
+module_exit(tegra194_cbb_exit);
+
+MODULE_AUTHOR("Sumit Gupta <sumitg@nvidia.com>");
+MODULE_DESCRIPTION("Control Backbone error handling driver for Tegra194");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/tegra/cbb/tegra234-cbb.c b/drivers/soc/tegra/cbb/tegra234-cbb.c
new file mode 100644
index 000000000000..3528f9e15d5c
--- /dev/null
+++ b/drivers/soc/tegra/cbb/tegra234-cbb.c
@@ -0,0 +1,1113 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved
+ *
+ * The driver handles Error's from Control Backbone(CBB) version 2.0.
+ * generated due to illegal accesses. The driver prints debug information
+ * about failed transaction on receiving interrupt from Error Notifier.
+ * Error types supported by CBB2.0 are:
+ *   UNSUPPORTED_ERR, PWRDOWN_ERR, TIMEOUT_ERR, FIREWALL_ERR, DECODE_ERR,
+ *   SLAVE_ERR
+ */
+
+#include <linux/acpi.h>
+#include <linux/clk.h>
+#include <linux/cpufeature.h>
+#include <linux/debugfs.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/platform_device.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/of_irq.h>
+#include <linux/of_address.h>
+#include <linux/interrupt.h>
+#include <linux/ioport.h>
+#include <linux/version.h>
+#include <soc/tegra/fuse.h>
+#include <soc/tegra/tegra-cbb.h>
+
+#define FABRIC_EN_CFG_INTERRUPT_ENABLE_0_0	0x0
+#define FABRIC_EN_CFG_STATUS_0_0		0x40
+#define FABRIC_EN_CFG_ADDR_INDEX_0_0		0x60
+#define FABRIC_EN_CFG_ADDR_LOW_0		0x80
+#define FABRIC_EN_CFG_ADDR_HI_0			0x84
+
+#define FABRIC_MN_MASTER_ERR_EN_0		0x200
+#define FABRIC_MN_MASTER_ERR_FORCE_0		0x204
+#define FABRIC_MN_MASTER_ERR_STATUS_0		0x208
+#define FABRIC_MN_MASTER_ERR_OVERFLOW_STATUS_0	0x20c
+
+#define FABRIC_MN_MASTER_LOG_ERR_STATUS_0	0x300
+#define FABRIC_MN_MASTER_LOG_ADDR_LOW_0		0x304
+#define FABRIC_MN_MASTER_LOG_ADDR_HIGH_0	0x308
+#define FABRIC_MN_MASTER_LOG_ATTRIBUTES0_0	0x30c
+#define FABRIC_MN_MASTER_LOG_ATTRIBUTES1_0	0x310
+#define FABRIC_MN_MASTER_LOG_ATTRIBUTES2_0	0x314
+#define FABRIC_MN_MASTER_LOG_USER_BITS0_0	0x318
+
+#define AXI_SLV_TIMEOUT_STATUS_0_0		0x8
+#define APB_BLOCK_TMO_STATUS_0			0xc00
+#define APB_BLOCK_NUM_TMO_OFFSET		0x20
+
+#define FAB_EM_EL_MSTRID		GENMASK(29, 24)
+#define FAB_EM_EL_VQC			GENMASK(17, 16)
+#define FAB_EM_EL_GRPSEC		GENMASK(14, 8)
+#define FAB_EM_EL_FALCONSEC		GENMASK(1, 0)
+
+#define FAB_EM_EL_FABID			GENMASK(20, 16)
+#define FAB_EM_EL_SLAVEID		GENMASK(7, 0)
+
+#define FAB_EM_EL_ACCESSID		GENMASK(7, 0)
+
+#define FAB_EM_EL_AXCACHE		GENMASK(27, 24)
+#define FAB_EM_EL_AXPROT		GENMASK(22, 20)
+#define FAB_EM_EL_BURSTLENGTH		GENMASK(19, 12)
+#define FAB_EM_EL_BURSTTYPE		GENMASK(9, 8)
+#define FAB_EM_EL_BEATSIZE		GENMASK(6, 4)
+#define FAB_EM_EL_ACCESSTYPE		GENMASK(0, 0)
+
+#define USRBITS_MSTR_ID			GENMASK(29, 24)
+
+#define REQ_SOCKET_ID			GENMASK(27, 24)
+
+enum tegra234_cbb_fabric_ids {
+	CBB_FAB_ID,
+	SCE_FAB_ID,
+	RCE_FAB_ID,
+	DCE_FAB_ID,
+	AON_FAB_ID,
+	PSC_FAB_ID,
+	BPMP_FAB_ID,
+	FSI_FAB_ID,
+	MAX_FAB_ID,
+};
+
+struct tegra234_slave_lookup {
+	const char *name;
+	unsigned int offset;
+};
+
+struct tegra234_cbb_fabric {
+	const char *name;
+	phys_addr_t off_mask_erd;
+	bool erd_mask_inband_err;
+	const char * const *master_id;
+	unsigned int notifier_offset;
+	const struct tegra_cbb_error *errors;
+	const struct tegra234_slave_lookup *slave_map;
+};
+
+struct tegra234_cbb {
+	struct tegra_cbb base;
+
+	const struct tegra234_cbb_fabric *fabric;
+	struct resource *res;
+	void __iomem *regs;
+
+	int num_intr;
+	int sec_irq;
+
+	/* record */
+	void __iomem *mon;
+	unsigned int type;
+	u32 mask;
+	u64 access;
+	u32 mn_attr0;
+	u32 mn_attr1;
+	u32 mn_attr2;
+	u32 mn_user_bits;
+};
+
+static inline struct tegra234_cbb *to_tegra234_cbb(struct tegra_cbb *cbb)
+{
+	return container_of(cbb, struct tegra234_cbb, base);
+}
+
+static LIST_HEAD(cbb_list);
+static DEFINE_SPINLOCK(cbb_lock);
+
+static void tegra234_cbb_fault_enable(struct tegra_cbb *cbb)
+{
+	struct tegra234_cbb *priv = to_tegra234_cbb(cbb);
+	void __iomem *addr;
+
+	addr = priv->regs + priv->fabric->notifier_offset;
+	writel(0x1ff, addr + FABRIC_EN_CFG_INTERRUPT_ENABLE_0_0);
+	dsb(sy);
+}
+
+static void tegra234_cbb_error_clear(struct tegra_cbb *cbb)
+{
+	struct tegra234_cbb *priv = to_tegra234_cbb(cbb);
+
+	writel(0x3f, priv->mon + FABRIC_MN_MASTER_ERR_STATUS_0);
+	dsb(sy);
+}
+
+static u32 tegra234_cbb_get_status(struct tegra_cbb *cbb)
+{
+	struct tegra234_cbb *priv = to_tegra234_cbb(cbb);
+	void __iomem *addr;
+	u32 value;
+
+	addr = priv->regs + priv->fabric->notifier_offset;
+	value = readl(addr + FABRIC_EN_CFG_STATUS_0_0);
+	dsb(sy);
+
+	return value;
+}
+
+static void tegra234_cbb_mask_serror(struct tegra234_cbb *cbb)
+{
+	writel(0x1, cbb->regs + cbb->fabric->off_mask_erd);
+	dsb(sy);
+}
+
+static u32 tegra234_cbb_get_tmo_slv(void __iomem *addr)
+{
+	u32 timeout;
+
+	timeout = readl(addr);
+	return timeout;
+}
+
+static void tegra234_cbb_tmo_slv(struct seq_file *file, const char *slave, void __iomem *addr,
+				 u32 status)
+{
+	tegra_cbb_print_err(file, "\t  %s : %#x\n", slave, status);
+}
+
+static void tegra234_cbb_lookup_apbslv(struct seq_file *file, const char *slave,
+				       void __iomem *base)
+{
+	unsigned int block = 0;
+	void __iomem *addr;
+	char name[64];
+	u32 status;
+
+	status = tegra234_cbb_get_tmo_slv(base);
+	if (status)
+		tegra_cbb_print_err(file, "\t  %s_BLOCK_TMO_STATUS : %#x\n", slave, status);
+
+	while (status) {
+		if (status & BIT(0)) {
+			u32 timeout, clients, client = 0;
+
+			addr = base + APB_BLOCK_NUM_TMO_OFFSET + (block * 4);
+			timeout = tegra234_cbb_get_tmo_slv(addr);
+			clients = timeout;
+
+			while (timeout) {
+				if (timeout & BIT(0)) {
+					if (clients != 0xffffffff)
+						clients &= BIT(client);
+
+					sprintf(name, "%s_BLOCK%d_TMO", slave, block);
+
+					tegra234_cbb_tmo_slv(file, name, addr, clients);
+				}
+
+				timeout >>= 1;
+				client++;
+			}
+		}
+
+		status >>= 1;
+		block++;
+	}
+}
+
+static void tegra234_lookup_slave_timeout(struct seq_file *file, struct tegra234_cbb *cbb,
+					  u8 slave_id, u8 fab_id)
+{
+	const struct tegra234_slave_lookup *map = cbb->fabric->slave_map;
+	void __iomem *addr;
+
+	/*
+	 * 1) Get slave node name and address mapping using slave_id.
+	 * 2) Check if the timed out slave node is APB or AXI.
+	 * 3) If AXI, then print timeout register and reset axi slave
+	 *    using <FABRIC>_SN_<>_SLV_TIMEOUT_STATUS_0_0 register.
+	 * 4) If APB, then perform an additional lookup to find the client
+	 *    which timed out.
+	 *	a) Get block number from the index of set bit in
+	 *	   <FABRIC>_SN_AXI2APB_<>_BLOCK_TMO_STATUS_0 register.
+	 *	b) Get address of register repective to block number i.e.
+	 *	   <FABRIC>_SN_AXI2APB_<>_BLOCK<index-set-bit>_TMO_0.
+	 *	c) Read the register in above step to get client_id which
+	 *	   timed out as per the set bits.
+	 *      d) Reset the timedout client and print details.
+	 *	e) Goto step-a till all bits are set.
+	 */
+
+	addr = cbb->regs + map[slave_id].offset;
+
+	if (strstr(map[slave_id].name, "AXI2APB")) {
+		addr += APB_BLOCK_TMO_STATUS_0;
+
+		tegra234_cbb_lookup_apbslv(file, map[slave_id].name, addr);
+	} else {
+		char name[64];
+		u32 status;
+
+		addr += AXI_SLV_TIMEOUT_STATUS_0_0;
+
+		status = tegra234_cbb_get_tmo_slv(addr);
+		if (status) {
+			sprintf(name, "%s_SLV_TIMEOUT_STATUS", map[slave_id].name);
+			tegra234_cbb_tmo_slv(file, name, addr, status);
+		}
+	}
+}
+
+static void tegra234_cbb_print_error(struct seq_file *file, struct tegra234_cbb *cbb, u32 status,
+				     u32 overflow)
+{
+	unsigned int type = 0;
+
+	if (status & (status - 1))
+		tegra_cbb_print_err(file, "\t  Multiple type of errors reported\n");
+
+	while (status) {
+		if (status & 0x1)
+			tegra_cbb_print_err(file, "\t  Error Code\t\t: %s\n",
+					    cbb->fabric->errors[type].code);
+
+		status >>= 1;
+		type++;
+	}
+
+	type = 0;
+
+	while (overflow) {
+		if (overflow & 0x1)
+			tegra_cbb_print_err(file, "\t  Overflow\t\t: Multiple %s\n",
+					    cbb->fabric->errors[type].code);
+
+		overflow >>= 1;
+		type++;
+	}
+}
+
+static void print_errlog_err(struct seq_file *file, struct tegra234_cbb *cbb)
+{
+	u8 cache_type, prot_type, burst_length, mstr_id, grpsec, vqc, falconsec, beat_size;
+	u8 access_type, access_id, requester_socket_id, local_socket_id, slave_id, fab_id;
+	char fabric_name[20];
+	bool is_numa = false;
+	u8 burst_type;
+
+	if (num_possible_nodes() > 1)
+		is_numa = true;
+
+	mstr_id = FIELD_GET(FAB_EM_EL_MSTRID, cbb->mn_user_bits);
+	vqc = FIELD_GET(FAB_EM_EL_VQC, cbb->mn_user_bits);
+	grpsec = FIELD_GET(FAB_EM_EL_GRPSEC, cbb->mn_user_bits);
+	falconsec = FIELD_GET(FAB_EM_EL_FALCONSEC, cbb->mn_user_bits);
+
+	/*
+	 * For SOC with multiple NUMA nodes, print cross socket access
+	 * errors only if initiator/master_id is CCPLEX, CPMU or GPU.
+	 */
+	if (is_numa) {
+		local_socket_id = numa_node_id();
+		requester_socket_id = FIELD_GET(REQ_SOCKET_ID, cbb->mn_attr2);
+
+		if (requester_socket_id != local_socket_id) {
+			if ((mstr_id != 0x1) && (mstr_id != 0x2) && (mstr_id != 0xB))
+				return;
+		}
+	}
+
+	fab_id = FIELD_GET(FAB_EM_EL_FABID, cbb->mn_attr2);
+	slave_id = FIELD_GET(FAB_EM_EL_SLAVEID, cbb->mn_attr2);
+
+	access_id = FIELD_GET(FAB_EM_EL_ACCESSID, cbb->mn_attr1);
+
+	cache_type = FIELD_GET(FAB_EM_EL_AXCACHE, cbb->mn_attr0);
+	prot_type = FIELD_GET(FAB_EM_EL_AXPROT, cbb->mn_attr0);
+	burst_length = FIELD_GET(FAB_EM_EL_BURSTLENGTH, cbb->mn_attr0);
+	burst_type = FIELD_GET(FAB_EM_EL_BURSTTYPE, cbb->mn_attr0);
+	beat_size = FIELD_GET(FAB_EM_EL_BEATSIZE, cbb->mn_attr0);
+	access_type = FIELD_GET(FAB_EM_EL_ACCESSTYPE, cbb->mn_attr0);
+
+	tegra_cbb_print_err(file, "\n");
+	tegra_cbb_print_err(file, "\t  Error Code\t\t: %s\n",
+			    cbb->fabric->errors[cbb->type].code);
+
+	tegra_cbb_print_err(file, "\t  MASTER_ID\t\t: %s\n", cbb->fabric->master_id[mstr_id]);
+	tegra_cbb_print_err(file, "\t  Address\t\t: %#llx\n", cbb->access);
+
+	tegra_cbb_print_cache(file, cache_type);
+	tegra_cbb_print_prot(file, prot_type);
+
+	tegra_cbb_print_err(file, "\t  Access_Type\t\t: %s", (access_type) ? "Write\n" : "Read\n");
+	tegra_cbb_print_err(file, "\t  Access_ID\t\t: %#x", access_id);
+
+	if (fab_id == PSC_FAB_ID)
+		strcpy(fabric_name, "psc-fabric");
+	else if (fab_id == FSI_FAB_ID)
+		strcpy(fabric_name, "fsi-fabric");
+	else
+		strcpy(fabric_name, cbb->fabric->name);
+
+	if (is_numa) {
+		tegra_cbb_print_err(file, "\t  Requester_Socket_Id\t: %#x\n",
+				    requester_socket_id);
+		tegra_cbb_print_err(file, "\t  Local_Socket_Id\t: %#x\n",
+				    local_socket_id);
+		tegra_cbb_print_err(file, "\t  No. of NUMA_NODES\t: %#x\n",
+				    num_possible_nodes());
+	}
+
+	tegra_cbb_print_err(file, "\t  Fabric\t\t: %s\n", fabric_name);
+	tegra_cbb_print_err(file, "\t  Slave_Id\t\t: %#x\n", slave_id);
+	tegra_cbb_print_err(file, "\t  Burst_length\t\t: %#x\n", burst_length);
+	tegra_cbb_print_err(file, "\t  Burst_type\t\t: %#x\n", burst_type);
+	tegra_cbb_print_err(file, "\t  Beat_size\t\t: %#x\n", beat_size);
+	tegra_cbb_print_err(file, "\t  VQC\t\t\t: %#x\n", vqc);
+	tegra_cbb_print_err(file, "\t  GRPSEC\t\t: %#x\n", grpsec);
+	tegra_cbb_print_err(file, "\t  FALCONSEC\t\t: %#x\n", falconsec);
+
+	if ((fab_id == PSC_FAB_ID) || (fab_id == FSI_FAB_ID))
+		return;
+
+	if (!strcmp(cbb->fabric->errors[cbb->type].code, "TIMEOUT_ERR")) {
+		tegra234_lookup_slave_timeout(file, cbb, slave_id, fab_id);
+		return;
+	}
+
+	tegra_cbb_print_err(file, "\t  Slave\t\t\t: %s\n", cbb->fabric->slave_map[slave_id].name);
+}
+
+static int print_errmonX_info(struct seq_file *file, struct tegra234_cbb *cbb)
+{
+	u32 overflow, status, error;
+
+	status = readl(cbb->mon + FABRIC_MN_MASTER_ERR_STATUS_0);
+	if (!status) {
+		pr_err("Error Notifier received a spurious notification\n");
+		return -ENODATA;
+	}
+
+	if (status == 0xffffffff) {
+		pr_err("CBB registers returning all 1's which is invalid\n");
+		return -EINVAL;
+	}
+
+	overflow = readl(cbb->mon + FABRIC_MN_MASTER_ERR_OVERFLOW_STATUS_0);
+
+	tegra234_cbb_print_error(file, cbb, status, overflow);
+
+	error = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ERR_STATUS_0);
+	if (!error) {
+		pr_info("Error Monitor doesn't have Error Logger\n");
+		return -EINVAL;
+	}
+
+	cbb->type = 0;
+
+	while (error) {
+		if (error & BIT(0)) {
+			u32 hi, lo;
+
+			hi = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ADDR_HIGH_0);
+			lo = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ADDR_LOW_0);
+
+			cbb->access = (u64)hi << 32 | lo;
+
+			cbb->mn_attr0 = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ATTRIBUTES0_0);
+			cbb->mn_attr1 = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ATTRIBUTES1_0);
+			cbb->mn_attr2 = readl(cbb->mon + FABRIC_MN_MASTER_LOG_ATTRIBUTES2_0);
+			cbb->mn_user_bits = readl(cbb->mon + FABRIC_MN_MASTER_LOG_USER_BITS0_0);
+
+			print_errlog_err(file, cbb);
+		}
+
+		cbb->type++;
+		error >>= 1;
+	}
+
+	return 0;
+}
+
+static int print_err_notifier(struct seq_file *file, struct tegra234_cbb *cbb, u32 status)
+{
+	unsigned int index = 0;
+	int err;
+
+	pr_crit("**************************************\n");
+	pr_crit("CPU:%d, Error:%s, Errmon:%d\n", smp_processor_id(),
+		cbb->fabric->name, status);
+
+	while (status) {
+		if (status & BIT(0)) {
+			unsigned int notifier = cbb->fabric->notifier_offset;
+			u32 hi, lo, mask = BIT(index);
+			phys_addr_t addr;
+			u64 offset;
+
+			writel(mask, cbb->regs + notifier + FABRIC_EN_CFG_ADDR_INDEX_0_0);
+			hi = readl(cbb->regs + notifier + FABRIC_EN_CFG_ADDR_HI_0);
+			lo = readl(cbb->regs + notifier + FABRIC_EN_CFG_ADDR_LOW_0);
+
+			addr = (u64)hi << 32 | lo;
+
+			offset = addr - cbb->res->start;
+			cbb->mon = cbb->regs + offset;
+			cbb->mask = BIT(index);
+
+			err = print_errmonX_info(file, cbb);
+			tegra234_cbb_error_clear(&cbb->base);
+			if (err)
+				return err;
+		}
+
+		status >>= 1;
+		index++;
+	}
+
+	tegra_cbb_print_err(file, "\t**************************************\n");
+	return 0;
+}
+
+#ifdef CONFIG_DEBUG_FS
+static DEFINE_MUTEX(cbb_debugfs_mutex);
+
+static int tegra234_cbb_debugfs_show(struct tegra_cbb *cbb, struct seq_file *file, void *data)
+{
+	int err = 0;
+
+	mutex_lock(&cbb_debugfs_mutex);
+
+	list_for_each_entry(cbb, &cbb_list, node) {
+		struct tegra234_cbb *priv = to_tegra234_cbb(cbb);
+		u32 status;
+
+		status = tegra_cbb_get_status(&priv->base);
+		if (status) {
+			err = print_err_notifier(file, priv, status);
+			if (err)
+				break;
+		}
+	}
+
+	mutex_unlock(&cbb_debugfs_mutex);
+	return err;
+}
+#endif
+
+/*
+ * Handler for CBB errors
+ */
+static irqreturn_t tegra234_cbb_isr(int irq, void *data)
+{
+	bool is_inband_err = false;
+	struct tegra_cbb *cbb;
+	unsigned long flags;
+	u8 mstr_id;
+	int err;
+
+	spin_lock_irqsave(&cbb_lock, flags);
+
+	list_for_each_entry(cbb, &cbb_list, node) {
+		struct tegra234_cbb *priv = to_tegra234_cbb(cbb);
+		u32 status = tegra_cbb_get_status(cbb);
+
+		if (status && (irq == priv->sec_irq)) {
+			tegra_cbb_print_err(NULL, "CPU:%d, Error: %s@%llx, irq=%d\n",
+					    smp_processor_id(), priv->fabric->name,
+					    priv->res->start, irq);
+
+			err = print_err_notifier(NULL, priv, status);
+			if (err)
+				goto unlock;
+
+			mstr_id =  FIELD_GET(USRBITS_MSTR_ID, priv->mn_user_bits);
+
+			/*
+			 * If illegal request is from CCPLEX(id:0x1) master then call BUG() to
+			 * crash system.
+			 */
+			if ((mstr_id == 0x1) && priv->fabric->off_mask_erd)
+				is_inband_err = 1;
+		}
+	}
+
+unlock:
+	spin_unlock_irqrestore(&cbb_lock, flags);
+	WARN_ON(is_inband_err);
+	return IRQ_HANDLED;
+}
+
+/*
+ * Register handler for CBB_SECURE interrupt for reporting errors
+ */
+static int tegra234_cbb_interrupt_enable(struct tegra_cbb *cbb)
+{
+	struct tegra234_cbb *priv = to_tegra234_cbb(cbb);
+
+	if (priv->sec_irq) {
+		int err = devm_request_irq(cbb->dev, priv->sec_irq, tegra234_cbb_isr, 0,
+					   dev_name(cbb->dev), priv);
+		if (err) {
+			dev_err(cbb->dev, "failed to register interrupt %u: %d\n", priv->sec_irq,
+				err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static void tegra234_cbb_error_enable(struct tegra_cbb *cbb)
+{
+	tegra_cbb_fault_enable(cbb);
+}
+
+static const struct tegra_cbb_ops tegra234_cbb_ops = {
+	.get_status = tegra234_cbb_get_status,
+	.error_clear = tegra234_cbb_error_clear,
+	.fault_enable = tegra234_cbb_fault_enable,
+	.error_enable = tegra234_cbb_error_enable,
+	.interrupt_enable = tegra234_cbb_interrupt_enable,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_show = tegra234_cbb_debugfs_show,
+#endif
+};
+
+static const char * const tegra234_master_id[] = {
+	[0x00] = "TZ",
+	[0x01] = "CCPLEX",
+	[0x02] = "CCPMU",
+	[0x03] = "BPMP_FW",
+	[0x04] = "AON",
+	[0x05] = "SCE",
+	[0x06] = "GPCDMA_P",
+	[0x07] = "TSECA_NONSECURE",
+	[0x08] = "TSECA_LIGHTSECURE",
+	[0x09] = "TSECA_HEAVYSECURE",
+	[0x0a] = "CORESIGHT",
+	[0x0b] = "APE",
+	[0x0c] = "PEATRANS",
+	[0x0d] = "JTAGM_DFT",
+	[0x0e] = "RCE",
+	[0x0f] = "DCE",
+	[0x10] = "PSC_FW_USER",
+	[0x11] = "PSC_FW_SUPERVISOR",
+	[0x12] = "PSC_FW_MACHINE",
+	[0x13] = "PSC_BOOT",
+	[0x14] = "BPMP_BOOT",
+	[0x15] = "NVDEC_NONSECURE",
+	[0x16] = "NVDEC_LIGHTSECURE",
+	[0x17] = "NVDEC_HEAVYSECURE",
+	[0x18] = "CBB_INTERNAL",
+	[0x19] = "RSVD"
+};
+
+static const struct tegra_cbb_error tegra234_cbb_errors[] = {
+	{
+		.code = "SLAVE_ERR",
+		.desc = "Slave being accessed responded with an error"
+	}, {
+		.code = "DECODE_ERR",
+		.desc = "Attempt to access an address hole"
+	}, {
+		.code = "FIREWALL_ERR",
+		.desc = "Attempt to access a region which is firewall protected"
+	}, {
+		.code = "TIMEOUT_ERR",
+		.desc = "No response returned by slave"
+	}, {
+		.code = "PWRDOWN_ERR",
+		.desc = "Attempt to access a portion of fabric that is powered down"
+	}, {
+		.code = "UNSUPPORTED_ERR",
+		.desc = "Attempt to access a slave through an unsupported access"
+	}
+};
+
+static const struct tegra234_slave_lookup tegra234_aon_slave_map[] = {
+	{ "AXI2APB", 0x00000 },
+	{ "AST",     0x14000 },
+	{ "CBB",     0x15000 },
+	{ "CPU",     0x16000 },
+};
+
+static const struct tegra234_cbb_fabric tegra234_aon_fabric = {
+	.name = "aon-fabric",
+	.master_id = tegra234_master_id,
+	.slave_map = tegra234_aon_slave_map,
+	.errors = tegra234_cbb_errors,
+	.notifier_offset = 0x17000,
+};
+
+static const struct tegra234_slave_lookup tegra234_bpmp_slave_map[] = {
+	{ "AXI2APB", 0x00000 },
+	{ "AST0",    0x15000 },
+	{ "AST1",    0x16000 },
+	{ "CBB",     0x17000 },
+	{ "CPU",     0x18000 },
+};
+
+static const struct tegra234_cbb_fabric tegra234_bpmp_fabric = {
+	.name = "bpmp-fabric",
+	.master_id = tegra234_master_id,
+	.slave_map = tegra234_bpmp_slave_map,
+	.errors = tegra234_cbb_errors,
+	.notifier_offset = 0x19000,
+};
+
+static const struct tegra234_slave_lookup tegra234_cbb_slave_map[] = {
+	{ "AON",        0x40000 },
+	{ "BPMP",       0x41000 },
+	{ "CBB",        0x42000 },
+	{ "HOST1X",     0x43000 },
+	{ "STM",        0x44000 },
+	{ "FSI",        0x45000 },
+	{ "PSC",        0x46000 },
+	{ "PCIE_C1",    0x47000 },
+	{ "PCIE_C2",    0x48000 },
+	{ "PCIE_C3",    0x49000 },
+	{ "PCIE_C0",    0x4a000 },
+	{ "PCIE_C4",    0x4b000 },
+	{ "GPU",        0x4c000 },
+	{ "SMMU0",      0x4d000 },
+	{ "SMMU1",      0x4e000 },
+	{ "SMMU2",      0x4f000 },
+	{ "SMMU3",      0x50000 },
+	{ "SMMU4",      0x51000 },
+	{ "PCIE_C10",   0x52000 },
+	{ "PCIE_C7",    0x53000 },
+	{ "PCIE_C8",    0x54000 },
+	{ "PCIE_C9",    0x55000 },
+	{ "PCIE_C5",    0x56000 },
+	{ "PCIE_C6",    0x57000 },
+	{ "DCE",        0x58000 },
+	{ "RCE",        0x59000 },
+	{ "SCE",        0x5a000 },
+	{ "AXI2APB_1",  0x70000 },
+	{ "AXI2APB_10", 0x71000 },
+	{ "AXI2APB_11", 0x72000 },
+	{ "AXI2APB_12", 0x73000 },
+	{ "AXI2APB_13", 0x74000 },
+	{ "AXI2APB_14", 0x75000 },
+	{ "AXI2APB_15", 0x76000 },
+	{ "AXI2APB_16", 0x77000 },
+	{ "AXI2APB_17", 0x78000 },
+	{ "AXI2APB_18", 0x79000 },
+	{ "AXI2APB_19", 0x7a000 },
+	{ "AXI2APB_2",  0x7b000 },
+	{ "AXI2APB_20", 0x7c000 },
+	{ "AXI2APB_21", 0x7d000 },
+	{ "AXI2APB_22", 0x7e000 },
+	{ "AXI2APB_23", 0x7f000 },
+	{ "AXI2APB_25", 0x80000 },
+	{ "AXI2APB_26", 0x81000 },
+	{ "AXI2APB_27", 0x82000 },
+	{ "AXI2APB_28", 0x83000 },
+	{ "AXI2APB_29", 0x84000 },
+	{ "AXI2APB_30", 0x85000 },
+	{ "AXI2APB_31", 0x86000 },
+	{ "AXI2APB_32", 0x87000 },
+	{ "AXI2APB_33", 0x88000 },
+	{ "AXI2APB_34", 0x89000 },
+	{ "AXI2APB_35", 0x92000 },
+	{ "AXI2APB_4",  0x8b000 },
+	{ "AXI2APB_5",  0x8c000 },
+	{ "AXI2APB_6",  0x8d000 },
+	{ "AXI2APB_7",  0x8e000 },
+	{ "AXI2APB_8",  0x8f000 },
+	{ "AXI2APB_9",  0x90000 },
+	{ "AXI2APB_3",  0x91000 },
+};
+
+static const struct tegra234_cbb_fabric tegra234_cbb_fabric = {
+	.name = "cbb-fabric",
+	.master_id = tegra234_master_id,
+	.slave_map = tegra234_cbb_slave_map,
+	.errors = tegra234_cbb_errors,
+	.notifier_offset = 0x60000,
+	.off_mask_erd = 0x3a004
+};
+
+static const struct tegra234_slave_lookup tegra234_dce_slave_map[] = {
+	{ "AXI2APB", 0x00000 },
+	{ "AST0",    0x15000 },
+	{ "AST1",    0x16000 },
+	{ "CPU",     0x18000 },
+};
+
+static const struct tegra234_cbb_fabric tegra234_dce_fabric = {
+	.name = "dce-fabric",
+	.master_id = tegra234_master_id,
+	.slave_map = tegra234_dce_slave_map,
+	.errors = tegra234_cbb_errors,
+	.notifier_offset = 0x19000,
+};
+
+static const struct tegra234_slave_lookup tegra234_rce_slave_map[] = {
+	{ "AXI2APB", 0x00000 },
+	{ "AST0",    0x15000 },
+	{ "AST1",    0x16000 },
+	{ "CPU",     0x18000 },
+};
+
+static const struct tegra234_cbb_fabric tegra234_rce_fabric = {
+	.name = "rce-fabric",
+	.master_id = tegra234_master_id,
+	.slave_map = tegra234_rce_slave_map,
+	.errors = tegra234_cbb_errors,
+	.notifier_offset = 0x19000,
+};
+
+static const struct tegra234_slave_lookup tegra234_sce_slave_map[] = {
+	{ "AXI2APB", 0x00000 },
+	{ "AST0",    0x15000 },
+	{ "AST1",    0x16000 },
+	{ "CBB",     0x17000 },
+	{ "CPU",     0x18000 },
+};
+
+static const struct tegra234_cbb_fabric tegra234_sce_fabric = {
+	.name = "sce-fabric",
+	.master_id = tegra234_master_id,
+	.slave_map = tegra234_sce_slave_map,
+	.errors = tegra234_cbb_errors,
+	.notifier_offset = 0x19000,
+};
+
+static const char * const tegra241_master_id[] = {
+	[0x0] = "TZ",
+	[0x1] = "CCPLEX",
+	[0x2] = "CCPMU",
+	[0x3] = "BPMP_FW",
+	[0x4] = "PSC_FW_USER",
+	[0x5] = "PSC_FW_SUPERVISOR",
+	[0x6] = "PSC_FW_MACHINE",
+	[0x7] = "PSC_BOOT",
+	[0x8] = "BPMP_BOOT",
+	[0x9] = "JTAGM_DFT",
+	[0xa] = "CORESIGHT",
+	[0xb] = "GPU",
+	[0xc] = "PEATRANS",
+	[0xd ... 0x3f] = "RSVD"
+};
+
+/*
+ * Possible causes for Slave and Timeout errors.
+ * SLAVE_ERR:
+ * Slave being accessed responded with an error. Slave could return
+ * an error for various cases :
+ *   Unsupported access, clamp setting when power gated, register
+ *   level firewall(SCR), address hole within the slave, etc
+ *
+ * TIMEOUT_ERR:
+ * No response returned by slave. Can be due to slave being clock
+ * gated, under reset, powered down or slave inability to respond
+ * for an internal slave issue
+ */
+static const struct tegra_cbb_error tegra241_cbb_errors[] = {
+	{
+		.code = "SLAVE_ERR",
+		.desc = "Slave being accessed responded with an error."
+	}, {
+		.code = "DECODE_ERR",
+		.desc = "Attempt to access an address hole or Reserved region of memory."
+	}, {
+		.code = "FIREWALL_ERR",
+		.desc = "Attempt to access a region which is firewalled."
+	}, {
+		.code = "TIMEOUT_ERR",
+		.desc = "No response returned by slave."
+	}, {
+		.code = "PWRDOWN_ERR",
+		.desc = "Attempt to access a portion of the fabric that is powered down."
+	}, {
+		.code = "UNSUPPORTED_ERR",
+		.desc = "Attempt to access a slave through an unsupported access."
+	}, {
+		.code = "POISON_ERR",
+		.desc = "Slave responds with poison error to indicate error in data."
+	}, {
+		.code = "RSVD"
+	}, {
+		.code = "RSVD"
+	}, {
+		.code = "RSVD"
+	}, {
+		.code = "RSVD"
+	}, {
+		.code = "RSVD"
+	}, {
+		.code = "RSVD"
+	}, {
+		.code = "RSVD"
+	}, {
+		.code = "RSVD"
+	}, {
+		.code = "RSVD"
+	}, {
+		.code = "NO_SUCH_ADDRESS_ERR",
+		.desc = "The address belongs to the pri_target range but there is no register "
+			"implemented at the address."
+	}, {
+		.code = "TASK_ERR",
+		.desc = "Attempt to update a PRI task when the current task has still not "
+			"completed."
+	}, {
+		.code = "EXTERNAL_ERR",
+		.desc = "Indicates that an external PRI register access met with an error due to "
+			"any issue in the unit."
+	}, {
+		.code = "INDEX_ERR",
+		.desc = "Applicable to PRI index aperture pair, when the programmed index is "
+			"outside the range defined in the manual."
+	}, {
+		.code = "RESET_ERR",
+		.desc = "Target in Reset Error: Attempt to access a SubPri or external PRI "
+			"register but they are in reset."
+	}, {
+		.code = "REGISTER_RST_ERR",
+		.desc = "Attempt to access a PRI register but the register is partial or "
+			"completely in reset."
+	}, {
+		.code = "POWER_GATED_ERR",
+		.desc = "Returned by external PRI client when the external access goes to a power "
+			"gated domain."
+	}, {
+		.code = "SUBPRI_FS_ERR",
+		.desc = "Subpri is floorswept: Attempt to access a subpri through the main pri "
+			"target but subPri logic is floorswept."
+	}, {
+		.code = "SUBPRI_CLK_OFF_ERR",
+		.desc = "Subpri clock is off: Attempt to access a subpri through the main pri "
+			"target but subPris clock is gated/off."
+	},
+};
+
+static const struct tegra234_slave_lookup tegra241_cbb_slave_map[] = {
+	{ "CCPLEX",     0x50000 },
+	{ "PCIE_C8",    0x51000 },
+	{ "PCIE_C9",    0x52000 },
+	{ "RSVD",       0x00000 },
+	{ "RSVD",       0x00000 },
+	{ "RSVD",       0x00000 },
+	{ "RSVD",       0x00000 },
+	{ "RSVD",       0x00000 },
+	{ "RSVD",       0x00000 },
+	{ "RSVD",       0x00000 },
+	{ "RSVD",       0x00000 },
+	{ "AON",        0x5b000 },
+	{ "BPMP",       0x5c000 },
+	{ "RSVD",       0x00000 },
+	{ "RSVD",       0x00000 },
+	{ "PSC",        0x5d000 },
+	{ "STM",        0x5e000 },
+	{ "AXI2APB_1",  0x70000 },
+	{ "AXI2APB_10", 0x71000 },
+	{ "AXI2APB_11", 0x72000 },
+	{ "AXI2APB_12", 0x73000 },
+	{ "AXI2APB_13", 0x74000 },
+	{ "AXI2APB_14", 0x75000 },
+	{ "AXI2APB_15", 0x76000 },
+	{ "AXI2APB_16", 0x77000 },
+	{ "AXI2APB_17", 0x78000 },
+	{ "AXI2APB_18", 0x79000 },
+	{ "AXI2APB_19", 0x7a000 },
+	{ "AXI2APB_2",  0x7b000 },
+	{ "AXI2APB_20", 0x7c000 },
+	{ "AXI2APB_4",  0x87000 },
+	{ "AXI2APB_5",  0x88000 },
+	{ "AXI2APB_6",  0x89000 },
+	{ "AXI2APB_7",  0x8a000 },
+	{ "AXI2APB_8",  0x8b000 },
+	{ "AXI2APB_9",  0x8c000 },
+	{ "AXI2APB_3",  0x8d000 },
+	{ "AXI2APB_21", 0x7d000 },
+	{ "AXI2APB_22", 0x7e000 },
+	{ "AXI2APB_23", 0x7f000 },
+	{ "AXI2APB_24", 0x80000 },
+	{ "AXI2APB_25", 0x81000 },
+	{ "AXI2APB_26", 0x82000 },
+	{ "AXI2APB_27", 0x83000 },
+	{ "AXI2APB_28", 0x84000 },
+	{ "PCIE_C4",    0x53000 },
+	{ "PCIE_C5",    0x54000 },
+	{ "PCIE_C6",    0x55000 },
+	{ "PCIE_C7",    0x56000 },
+	{ "PCIE_C2",    0x57000 },
+	{ "PCIE_C3",    0x58000 },
+	{ "PCIE_C0",    0x59000 },
+	{ "PCIE_C1",    0x5a000 },
+	{ "AXI2APB_29", 0x85000 },
+	{ "AXI2APB_30", 0x86000 },
+};
+
+static const struct tegra234_cbb_fabric tegra241_cbb_fabric = {
+	.name = "cbb-fabric",
+	.master_id = tegra241_master_id,
+	.slave_map = tegra241_cbb_slave_map,
+	.errors = tegra241_cbb_errors,
+	.notifier_offset = 0x60000,
+	.off_mask_erd = 0x40004,
+};
+
+static const struct tegra234_slave_lookup tegra241_bpmp_slave_map[] = {
+	{ "RSVD",    0x00000 },
+	{ "RSVD",    0x00000 },
+	{ "CBB",     0x15000 },
+	{ "CPU",     0x16000 },
+	{ "AXI2APB", 0x00000 },
+	{ "DBB0",    0x17000 },
+	{ "DBB1",    0x18000 },
+};
+
+static const struct tegra234_cbb_fabric tegra241_bpmp_fabric = {
+	.name = "bpmp-fabric",
+	.master_id = tegra241_master_id,
+	.slave_map = tegra241_bpmp_slave_map,
+	.errors = tegra241_cbb_errors,
+	.notifier_offset = 0x19000,
+};
+
+static const struct of_device_id tegra234_cbb_dt_ids[] = {
+	{ .compatible = "nvidia,tegra234-cbb-fabric", .data = &tegra234_cbb_fabric },
+	{ .compatible = "nvidia,tegra234-aon-fabric", .data = &tegra234_aon_fabric },
+	{ .compatible = "nvidia,tegra234-bpmp-fabric", .data = &tegra234_bpmp_fabric },
+	{ .compatible = "nvidia,tegra234-dce-fabric", .data = &tegra234_dce_fabric },
+	{ .compatible = "nvidia,tegra234-rce-fabric", .data = &tegra234_rce_fabric },
+	{ .compatible = "nvidia,tegra234-sce-fabric", .data = &tegra234_sce_fabric },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, tegra234_cbb_dt_ids);
+
+struct tegra234_cbb_acpi_uid {
+	const char *hid;
+	const char *uid;
+	const struct tegra234_cbb_fabric *fabric;
+};
+
+static const struct tegra234_cbb_acpi_uid tegra234_cbb_acpi_uids[] = {
+	{ "NVDA1070", "1", &tegra241_cbb_fabric },
+	{ "NVDA1070", "2", &tegra241_bpmp_fabric },
+	{ },
+};
+
+static const struct
+tegra234_cbb_fabric *tegra234_cbb_acpi_get_fabric(struct acpi_device *adev)
+{
+	const struct tegra234_cbb_acpi_uid *entry;
+
+	for (entry = tegra234_cbb_acpi_uids; entry->hid; entry++) {
+		if (acpi_dev_hid_uid_match(adev, entry->hid, entry->uid))
+			return entry->fabric;
+	}
+
+	return NULL;
+}
+
+static const struct acpi_device_id tegra241_cbb_acpi_ids[] = {
+	{ "NVDA1070" },
+	{ },
+};
+MODULE_DEVICE_TABLE(acpi, tegra241_cbb_acpi_ids);
+
+static int tegra234_cbb_probe(struct platform_device *pdev)
+{
+	const struct tegra234_cbb_fabric *fabric;
+	struct tegra234_cbb *cbb;
+	unsigned long flags = 0;
+	int err;
+
+	if (pdev->dev.of_node) {
+		fabric = of_device_get_match_data(&pdev->dev);
+	} else {
+		struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+		if (!device)
+			return -ENODEV;
+
+		fabric = tegra234_cbb_acpi_get_fabric(device);
+		if (!fabric) {
+			dev_err(&pdev->dev, "no device match found\n");
+			return -ENODEV;
+		}
+	}
+
+	cbb = devm_kzalloc(&pdev->dev, sizeof(*cbb), GFP_KERNEL);
+	if (!cbb)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&cbb->base.node);
+	cbb->base.ops = &tegra234_cbb_ops;
+	cbb->base.dev = &pdev->dev;
+	cbb->fabric = fabric;
+
+	cbb->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &cbb->res);
+	if (IS_ERR(cbb->regs))
+		return PTR_ERR(cbb->regs);
+
+	err = tegra_cbb_get_irq(pdev, NULL, &cbb->sec_irq);
+	if (err)
+		return err;
+
+	platform_set_drvdata(pdev, cbb);
+
+	spin_lock_irqsave(&cbb_lock, flags);
+	list_add(&cbb->base.node, &cbb_list);
+	spin_unlock_irqrestore(&cbb_lock, flags);
+
+	/* set ERD bit to mask SError and generate interrupt to report error */
+	if (cbb->fabric->off_mask_erd)
+		tegra234_cbb_mask_serror(cbb);
+
+	return tegra_cbb_register(&cbb->base);
+}
+
+static int tegra234_cbb_remove(struct platform_device *pdev)
+{
+	return 0;
+}
+
+static int __maybe_unused tegra234_cbb_resume_noirq(struct device *dev)
+{
+	struct tegra234_cbb *cbb = dev_get_drvdata(dev);
+
+	tegra234_cbb_error_enable(&cbb->base);
+
+	dev_dbg(dev, "%s resumed\n", cbb->fabric->name);
+
+	return 0;
+}
+
+static const struct dev_pm_ops tegra234_cbb_pm = {
+	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, tegra234_cbb_resume_noirq)
+};
+
+static struct platform_driver tegra234_cbb_driver = {
+	.probe = tegra234_cbb_probe,
+	.remove = tegra234_cbb_remove,
+	.driver = {
+		.name = "tegra234-cbb",
+		.of_match_table = tegra234_cbb_dt_ids,
+		.acpi_match_table = tegra241_cbb_acpi_ids,
+		.pm = &tegra234_cbb_pm,
+	},
+};
+
+static int __init tegra234_cbb_init(void)
+{
+	return platform_driver_register(&tegra234_cbb_driver);
+}
+pure_initcall(tegra234_cbb_init);
+
+static void __exit tegra234_cbb_exit(void)
+{
+	platform_driver_unregister(&tegra234_cbb_driver);
+}
+module_exit(tegra234_cbb_exit);
+
+MODULE_DESCRIPTION("Control Backbone 2.0 error handling driver for Tegra234");
+MODULE_LICENSE("GPL");
diff --git a/drivers/soc/tegra/fuse/fuse-tegra.c b/drivers/soc/tegra/fuse/fuse-tegra.c
index b0a8405dbdb1..6542267a224d 100644
--- a/drivers/soc/tegra/fuse/fuse-tegra.c
+++ b/drivers/soc/tegra/fuse/fuse-tegra.c
@@ -568,6 +568,7 @@ static int __init tegra_init_fuse(void)
 	np = of_find_matching_node(NULL, car_match);
 	if (np) {
 		void __iomem *base = of_iomap(np, 0);
+		of_node_put(np);
 		if (base) {
 			tegra_enable_fuse_clk(base);
 			iounmap(base);
diff --git a/drivers/soc/tegra/fuse/tegra-apbmisc.c b/drivers/soc/tegra/fuse/tegra-apbmisc.c
index 590c862538d0..3351bd872ab2 100644
--- a/drivers/soc/tegra/fuse/tegra-apbmisc.c
+++ b/drivers/soc/tegra/fuse/tegra-apbmisc.c
@@ -16,12 +16,16 @@
 
 #define FUSE_SKU_INFO	0x10
 
+#define ERD_ERR_CONFIG 0x120c
+#define ERD_MASK_INBAND_ERR 0x1
+
 #define PMC_STRAPPING_OPT_A_RAM_CODE_SHIFT	4
 #define PMC_STRAPPING_OPT_A_RAM_CODE_MASK_LONG	\
 	(0xf << PMC_STRAPPING_OPT_A_RAM_CODE_SHIFT)
 #define PMC_STRAPPING_OPT_A_RAM_CODE_MASK_SHORT	\
 	(0x3 << PMC_STRAPPING_OPT_A_RAM_CODE_SHIFT)
 
+static void __iomem *apbmisc_base;
 static bool long_ram_code;
 static u32 strapping;
 static u32 chipid;
@@ -93,6 +97,28 @@ u32 tegra_read_ram_code(void)
 }
 EXPORT_SYMBOL_GPL(tegra_read_ram_code);
 
+/*
+ * The function sets ERD(Error Response Disable) bit.
+ * This allows to mask inband errors and always send an
+ * OKAY response from CBB to the master which caused error.
+ */
+int tegra194_miscreg_mask_serror(void)
+{
+	if (!apbmisc_base)
+		return -EPROBE_DEFER;
+
+	if (!of_machine_is_compatible("nvidia,tegra194")) {
+		WARN(1, "Only supported for Tegra194 devices!\n");
+		return -EOPNOTSUPP;
+	}
+
+	writel_relaxed(ERD_MASK_INBAND_ERR,
+		       apbmisc_base + ERD_ERR_CONFIG);
+
+	return 0;
+}
+EXPORT_SYMBOL(tegra194_miscreg_mask_serror);
+
 static const struct of_device_id apbmisc_match[] __initconst = {
 	{ .compatible = "nvidia,tegra20-apbmisc", },
 	{ .compatible = "nvidia,tegra186-misc", },
@@ -134,7 +160,7 @@ void __init tegra_init_revision(void)
 
 void __init tegra_init_apbmisc(void)
 {
-	void __iomem *apbmisc_base, *strapping_base;
+	void __iomem *strapping_base;
 	struct resource apbmisc, straps;
 	struct device_node *np;
 
@@ -182,12 +208,12 @@ void __init tegra_init_apbmisc(void)
 		 */
 		if (of_address_to_resource(np, 0, &apbmisc) < 0) {
 			pr_err("failed to get APBMISC registers\n");
-			return;
+			goto put;
 		}
 
 		if (of_address_to_resource(np, 1, &straps) < 0) {
 			pr_err("failed to get strapping options registers\n");
-			return;
+			goto put;
 		}
 	}
 
@@ -196,7 +222,6 @@ void __init tegra_init_apbmisc(void)
 		pr_err("failed to map APBMISC registers\n");
 	} else {
 		chipid = readl_relaxed(apbmisc_base + 4);
-		iounmap(apbmisc_base);
 	}
 
 	strapping_base = ioremap(straps.start, resource_size(&straps));
@@ -208,4 +233,7 @@ void __init tegra_init_apbmisc(void)
 	}
 
 	long_ram_code = of_property_read_bool(np, "nvidia,long-ram-code");
+
+put:
+	of_node_put(np);
 }
diff --git a/drivers/soc/tegra/pmc.c b/drivers/soc/tegra/pmc.c
index 6a4b8f7e7948..678e8bc8a45d 100644
--- a/drivers/soc/tegra/pmc.c
+++ b/drivers/soc/tegra/pmc.c
@@ -296,6 +296,17 @@ struct tegra_wake_event {
 	} gpio;
 };
 
+#define TEGRA_WAKE_SIMPLE(_name, _id)			\
+	{						\
+		.name = _name,				\
+		.id = _id,				\
+		.irq = 0,				\
+		.gpio = {				\
+			.instance = UINT_MAX,		\
+			.pin = UINT_MAX,		\
+		},					\
+	}
+
 #define TEGRA_WAKE_IRQ(_name, _id, _irq)		\
 	{						\
 		.name = _name,				\
@@ -2239,6 +2250,7 @@ static int tegra_pmc_irq_alloc(struct irq_domain *domain, unsigned int virq,
 	for (i = 0; i < soc->num_wake_events; i++) {
 		const struct tegra_wake_event *event = &soc->wake_events[i];
 
+		/* IRQ and simple wake events */
 		if (fwspec->param_count == 2) {
 			struct irq_fwspec spec;
 
@@ -2251,6 +2263,12 @@ static int tegra_pmc_irq_alloc(struct irq_domain *domain, unsigned int virq,
 			if (err < 0)
 				break;
 
+			/* simple hierarchies stop at the PMC level */
+			if (event->irq == 0) {
+				err = irq_domain_disconnect_hierarchy(domain->parent, virq);
+				break;
+			}
+
 			spec.fwnode = &pmc->dev->of_node->fwnode;
 			spec.param_count = 3;
 			spec.param[0] = GIC_SPI;
@@ -2263,6 +2281,7 @@ static int tegra_pmc_irq_alloc(struct irq_domain *domain, unsigned int virq,
 			break;
 		}
 
+		/* GPIO wake events */
 		if (fwspec->param_count == 3) {
 			if (event->gpio.instance != fwspec->param[0] ||
 			    event->gpio.pin != fwspec->param[1])
@@ -2274,7 +2293,7 @@ static int tegra_pmc_irq_alloc(struct irq_domain *domain, unsigned int virq,
 
 			/* GPIO hierarchies stop at the PMC level */
 			if (!err && domain->parent)
- 				err = irq_domain_disconnect_hierarchy(domain->parent,
+				err = irq_domain_disconnect_hierarchy(domain->parent,
 								      virq);
 			break;
 		}
@@ -2885,17 +2904,10 @@ static int tegra_pmc_probe(struct platform_device *pdev)
 		pmc->scratch = base;
 	}
 
-	pmc->clk = devm_clk_get(&pdev->dev, "pclk");
-	if (IS_ERR(pmc->clk)) {
-		err = PTR_ERR(pmc->clk);
-
-		if (err != -ENOENT) {
-			dev_err(&pdev->dev, "failed to get pclk: %d\n", err);
-			return err;
-		}
-
-		pmc->clk = NULL;
-	}
+	pmc->clk = devm_clk_get_optional(&pdev->dev, "pclk");
+	if (IS_ERR(pmc->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(pmc->clk),
+				     "failed to get pclk\n");
 
 	/*
 	 * PMC should be last resort for restarting since it soft-resets
@@ -3757,6 +3769,13 @@ static const struct tegra_wake_event tegra194_wake_events[] = {
 	TEGRA_WAKE_IRQ("pmu", 24, 209),
 	TEGRA_WAKE_GPIO("power", 29, 1, TEGRA194_AON_GPIO(EE, 4)),
 	TEGRA_WAKE_IRQ("rtc", 73, 10),
+	TEGRA_WAKE_SIMPLE("usb3-port-0", 76),
+	TEGRA_WAKE_SIMPLE("usb3-port-1", 77),
+	TEGRA_WAKE_SIMPLE("usb3-port-2-3", 78),
+	TEGRA_WAKE_SIMPLE("usb2-port-0", 79),
+	TEGRA_WAKE_SIMPLE("usb2-port-1", 80),
+	TEGRA_WAKE_SIMPLE("usb2-port-2", 81),
+	TEGRA_WAKE_SIMPLE("usb2-port-3", 82),
 };
 
 static const struct tegra_pmc_soc tegra194_pmc_soc = {
@@ -4025,7 +4044,7 @@ static int __init tegra_pmc_early_init(void)
 		return -ENXIO;
 	}
 
-	if (np) {
+	if (of_device_is_available(np)) {
 		pmc->soc = match->data;
 
 		if (pmc->soc->maybe_tz_only)
diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c
index 8d4000664fa3..d95b07896a3e 100644
--- a/drivers/soundwire/bus.c
+++ b/drivers/soundwire/bus.c
@@ -298,6 +298,38 @@ int sdw_transfer(struct sdw_bus *bus, struct sdw_msg *msg)
 }
 
 /**
+ * sdw_show_ping_status() - Direct report of PING status, to be used by Peripheral drivers
+ * @bus: SDW bus
+ * @sync_delay: Delay before reading status
+ */
+void sdw_show_ping_status(struct sdw_bus *bus, bool sync_delay)
+{
+	u32 status;
+
+	if (!bus->ops->read_ping_status)
+		return;
+
+	/*
+	 * wait for peripheral to sync if desired. 10-15ms should be more than
+	 * enough in most cases.
+	 */
+	if (sync_delay)
+		usleep_range(10000, 15000);
+
+	mutex_lock(&bus->msg_lock);
+
+	status = bus->ops->read_ping_status(bus);
+
+	mutex_unlock(&bus->msg_lock);
+
+	if (!status)
+		dev_warn(bus->dev, "%s: no peripherals attached\n", __func__);
+	else
+		dev_dbg(bus->dev, "PING status: %#x\n", status);
+}
+EXPORT_SYMBOL(sdw_show_ping_status);
+
+/**
  * sdw_transfer_defer() - Asynchronously transfer message to a SDW Slave device
  * @bus: SDW bus
  * @msg: SDW message to be xfered
diff --git a/drivers/soundwire/cadence_master.c b/drivers/soundwire/cadence_master.c
index 4fbb19557f5e..615b0b63a3e1 100644
--- a/drivers/soundwire/cadence_master.c
+++ b/drivers/soundwire/cadence_master.c
@@ -756,6 +756,14 @@ cdns_reset_page_addr(struct sdw_bus *bus, unsigned int dev_num)
 }
 EXPORT_SYMBOL(cdns_reset_page_addr);
 
+u32 cdns_read_ping_status(struct sdw_bus *bus)
+{
+	struct sdw_cdns *cdns = bus_to_cdns(bus);
+
+	return cdns_readl(cdns, CDNS_MCP_SLAVE_STAT);
+}
+EXPORT_SYMBOL(cdns_read_ping_status);
+
 /*
  * IRQ handling
  */
diff --git a/drivers/soundwire/cadence_master.h b/drivers/soundwire/cadence_master.h
index 595d72c15d97..ca9e805bab88 100644
--- a/drivers/soundwire/cadence_master.h
+++ b/drivers/soundwire/cadence_master.h
@@ -177,6 +177,8 @@ enum sdw_command_response
 cdns_xfer_msg_defer(struct sdw_bus *bus,
 		    struct sdw_msg *msg, struct sdw_defer *defer);
 
+u32 cdns_read_ping_status(struct sdw_bus *bus);
+
 int cdns_bus_conf(struct sdw_bus *bus, struct sdw_bus_params *params);
 
 int cdns_set_sdw_stream(struct snd_soc_dai *dai,
diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c
index 89d1d0d021fc..a5965e8827b9 100644
--- a/drivers/soundwire/intel.c
+++ b/drivers/soundwire/intel.c
@@ -1262,6 +1262,7 @@ static struct sdw_master_ops sdw_intel_ops = {
 	.set_bus_conf = cdns_bus_conf,
 	.pre_bank_switch = intel_pre_bank_switch,
 	.post_bank_switch = intel_post_bank_switch,
+	.read_ping_status = cdns_read_ping_status,
 };
 
 static int intel_init(struct sdw_intel *sdw)
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index e32f6a2058ae..d1bb62f7368b 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -591,6 +591,15 @@ config SPI_MICROCHIP_CORE
 	  PolarFire SoC.
 	  If built as a module, it will be called spi-microchip-core.
 
+config SPI_MICROCHIP_CORE_QSPI
+	tristate "Microchip FPGA QSPI controllers"
+	depends on SPI_MASTER
+	help
+	  This enables the QSPI driver for Microchip FPGA QSPI controllers.
+	  Say Y or M here if you want to use the QSPI controllers on
+	  PolarFire SoC.
+	  If built as a module, it will be called spi-microchip-core-qspi.
+
 config SPI_MT65XX
 	tristate "MediaTek SPI controller"
 	depends on ARCH_MEDIATEK || COMPILE_TEST
diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile
index 15d2f3835e45..4b34e855c841 100644
--- a/drivers/spi/Makefile
+++ b/drivers/spi/Makefile
@@ -73,6 +73,7 @@ obj-$(CONFIG_SPI_LP8841_RTC)		+= spi-lp8841-rtc.o
 obj-$(CONFIG_SPI_MESON_SPICC)		+= spi-meson-spicc.o
 obj-$(CONFIG_SPI_MESON_SPIFC)		+= spi-meson-spifc.o
 obj-$(CONFIG_SPI_MICROCHIP_CORE)	+= spi-microchip-core.o
+obj-$(CONFIG_SPI_MICROCHIP_CORE_QSPI)	+= spi-microchip-core-qspi.o
 obj-$(CONFIG_SPI_MPC512x_PSC)		+= spi-mpc512x-psc.o
 obj-$(CONFIG_SPI_MPC52xx_PSC)		+= spi-mpc52xx-psc.o
 obj-$(CONFIG_SPI_MPC52xx)		+= spi-mpc52xx.o
diff --git a/drivers/spi/spi-amd.c b/drivers/spi/spi-amd.c
index 08df4f8d0531..e23121456c70 100644
--- a/drivers/spi/spi-amd.c
+++ b/drivers/spi/spi-amd.c
@@ -36,9 +36,17 @@
 #define AMD_SPI_FIFO_SIZE	70
 #define AMD_SPI_MEM_SIZE	200
 
-/* M_CMD OP codes for SPI */
-#define AMD_SPI_XFER_TX		1
-#define AMD_SPI_XFER_RX		2
+#define AMD_SPI_ENA_REG		0x20
+#define AMD_SPI_ALT_SPD_SHIFT	20
+#define AMD_SPI_ALT_SPD_MASK	GENMASK(23, AMD_SPI_ALT_SPD_SHIFT)
+#define AMD_SPI_SPI100_SHIFT	0
+#define AMD_SPI_SPI100_MASK	GENMASK(AMD_SPI_SPI100_SHIFT, AMD_SPI_SPI100_SHIFT)
+#define AMD_SPI_SPEED_REG	0x6C
+#define AMD_SPI_SPD7_SHIFT	8
+#define AMD_SPI_SPD7_MASK	GENMASK(13, AMD_SPI_SPD7_SHIFT)
+
+#define AMD_SPI_MAX_HZ		100000000
+#define AMD_SPI_MIN_HZ		800000
 
 /**
  * enum amd_spi_versions - SPI controller versions
@@ -50,14 +58,41 @@ enum amd_spi_versions {
 	AMD_SPI_V2,
 };
 
+enum amd_spi_speed {
+	F_66_66MHz,
+	F_33_33MHz,
+	F_22_22MHz,
+	F_16_66MHz,
+	F_100MHz,
+	F_800KHz,
+	SPI_SPD7,
+	F_50MHz = 0x4,
+	F_4MHz = 0x32,
+	F_3_17MHz = 0x3F
+};
+
+/**
+ * struct amd_spi_freq - Matches device speed with values to write in regs
+ * @speed_hz: Device frequency
+ * @enable_val: Value to be written to "enable register"
+ * @spd7_val: Some frequencies requires to have a value written at SPISPEED register
+ */
+struct amd_spi_freq {
+	u32 speed_hz;
+	u32 enable_val;
+	u32 spd7_val;
+};
+
 /**
  * struct amd_spi - SPI driver instance
  * @io_remap_addr:	Start address of the SPI controller registers
  * @version:		SPI controller hardware version
+ * @speed_hz:		Device frequency
  */
 struct amd_spi {
 	void __iomem *io_remap_addr;
 	enum amd_spi_versions version;
+	unsigned int speed_hz;
 };
 
 static inline u8 amd_spi_readreg8(struct amd_spi *amd_spi, int idx)
@@ -189,65 +224,125 @@ static int amd_spi_master_setup(struct spi_device *spi)
 	return 0;
 }
 
+static const struct amd_spi_freq amd_spi_freq[] = {
+	{ AMD_SPI_MAX_HZ,   F_100MHz,         0},
+	{       66660000, F_66_66MHz,         0},
+	{       50000000,   SPI_SPD7,   F_50MHz},
+	{       33330000, F_33_33MHz,         0},
+	{       22220000, F_22_22MHz,         0},
+	{       16660000, F_16_66MHz,         0},
+	{        4000000,   SPI_SPD7,    F_4MHz},
+	{        3170000,   SPI_SPD7, F_3_17MHz},
+	{ AMD_SPI_MIN_HZ,   F_800KHz,         0},
+};
+
+static int amd_set_spi_freq(struct amd_spi *amd_spi, u32 speed_hz)
+{
+	unsigned int i, spd7_val, alt_spd;
+
+	if (speed_hz < AMD_SPI_MIN_HZ)
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(amd_spi_freq); i++)
+		if (speed_hz >= amd_spi_freq[i].speed_hz)
+			break;
+
+	if (amd_spi->speed_hz == amd_spi_freq[i].speed_hz)
+		return 0;
+
+	amd_spi->speed_hz = amd_spi_freq[i].speed_hz;
+
+	alt_spd = (amd_spi_freq[i].enable_val << AMD_SPI_ALT_SPD_SHIFT)
+		   & AMD_SPI_ALT_SPD_MASK;
+	amd_spi_setclear_reg32(amd_spi, AMD_SPI_ENA_REG, alt_spd,
+			       AMD_SPI_ALT_SPD_MASK);
+
+	if (amd_spi->speed_hz == AMD_SPI_MAX_HZ)
+		amd_spi_setclear_reg32(amd_spi, AMD_SPI_ENA_REG, 1,
+				       AMD_SPI_SPI100_MASK);
+
+	if (amd_spi_freq[i].spd7_val) {
+		spd7_val = (amd_spi_freq[i].spd7_val << AMD_SPI_SPD7_SHIFT)
+			    & AMD_SPI_SPD7_MASK;
+		amd_spi_setclear_reg32(amd_spi, AMD_SPI_SPEED_REG, spd7_val,
+				       AMD_SPI_SPD7_MASK);
+	}
+
+	return 0;
+}
+
 static inline int amd_spi_fifo_xfer(struct amd_spi *amd_spi,
 				    struct spi_master *master,
 				    struct spi_message *message)
 {
 	struct spi_transfer *xfer = NULL;
-	u8 cmd_opcode;
+	struct spi_device *spi = message->spi;
+	u8 cmd_opcode = 0, fifo_pos = AMD_SPI_FIFO_BASE;
 	u8 *buf = NULL;
-	u32 m_cmd = 0;
 	u32 i = 0;
 	u32 tx_len = 0, rx_len = 0;
 
 	list_for_each_entry(xfer, &message->transfers,
 			    transfer_list) {
-		if (xfer->rx_buf)
-			m_cmd = AMD_SPI_XFER_RX;
-		if (xfer->tx_buf)
-			m_cmd = AMD_SPI_XFER_TX;
+		if (xfer->speed_hz)
+			amd_set_spi_freq(amd_spi, xfer->speed_hz);
+		else
+			amd_set_spi_freq(amd_spi, spi->max_speed_hz);
 
-		if (m_cmd & AMD_SPI_XFER_TX) {
+		if (xfer->tx_buf) {
 			buf = (u8 *)xfer->tx_buf;
-			tx_len = xfer->len - 1;
-			cmd_opcode = *(u8 *)xfer->tx_buf;
-			buf++;
-			amd_spi_set_opcode(amd_spi, cmd_opcode);
+			if (!tx_len) {
+				cmd_opcode = *(u8 *)xfer->tx_buf;
+				buf++;
+				xfer->len--;
+			}
+			tx_len += xfer->len;
 
 			/* Write data into the FIFO. */
-			for (i = 0; i < tx_len; i++) {
-				iowrite8(buf[i], ((u8 __iomem *)amd_spi->io_remap_addr +
-					 AMD_SPI_FIFO_BASE + i));
-			}
+			for (i = 0; i < xfer->len; i++)
+				amd_spi_writereg8(amd_spi, fifo_pos + i, buf[i]);
 
-			amd_spi_set_tx_count(amd_spi, tx_len);
-			amd_spi_clear_fifo_ptr(amd_spi);
-			/* Execute command */
-			amd_spi_execute_opcode(amd_spi);
-		}
-		if (m_cmd & AMD_SPI_XFER_RX) {
-			/*
-			 * Store no. of bytes to be received from
-			 * FIFO
-			 */
-			rx_len = xfer->len;
-			buf = (u8 *)xfer->rx_buf;
-			amd_spi_set_rx_count(amd_spi, rx_len);
-			amd_spi_clear_fifo_ptr(amd_spi);
-			/* Execute command */
-			amd_spi_execute_opcode(amd_spi);
-			amd_spi_busy_wait(amd_spi);
-			/* Read data from FIFO to receive buffer  */
-			for (i = 0; i < rx_len; i++)
-				buf[i] = amd_spi_readreg8(amd_spi, AMD_SPI_FIFO_BASE + tx_len + i);
+			fifo_pos += xfer->len;
 		}
+
+		/* Store no. of bytes to be received from FIFO */
+		if (xfer->rx_buf)
+			rx_len += xfer->len;
+	}
+
+	if (!buf) {
+		message->status = -EINVAL;
+		goto fin_msg;
+	}
+
+	amd_spi_set_opcode(amd_spi, cmd_opcode);
+	amd_spi_set_tx_count(amd_spi, tx_len);
+	amd_spi_set_rx_count(amd_spi, rx_len);
+
+	/* Execute command */
+	message->status = amd_spi_execute_opcode(amd_spi);
+	if (message->status)
+		goto fin_msg;
+
+	if (rx_len) {
+		message->status = amd_spi_busy_wait(amd_spi);
+		if (message->status)
+			goto fin_msg;
+
+		list_for_each_entry(xfer, &message->transfers, transfer_list)
+			if (xfer->rx_buf) {
+				buf = (u8 *)xfer->rx_buf;
+				/* Read data from FIFO to receive buffer */
+				for (i = 0; i < xfer->len; i++)
+					buf[i] = amd_spi_readreg8(amd_spi, fifo_pos + i);
+				fifo_pos += xfer->len;
+			}
 	}
 
 	/* Update statistics */
 	message->actual_length = tx_len + rx_len + 1;
-	/* complete the transaction */
-	message->status = 0;
 
+fin_msg:
 	switch (amd_spi->version) {
 	case AMD_SPI_V1:
 		break;
@@ -260,7 +355,7 @@ static inline int amd_spi_fifo_xfer(struct amd_spi *amd_spi,
 
 	spi_finalize_current_message(master);
 
-	return 0;
+	return message->status;
 }
 
 static int amd_spi_master_transfer(struct spi_master *master,
@@ -275,9 +370,7 @@ static int amd_spi_master_transfer(struct spi_master *master,
 	 * Extract spi_transfers from the spi message and
 	 * program the controller.
 	 */
-	amd_spi_fifo_xfer(amd_spi, master, msg);
-
-	return 0;
+	return amd_spi_fifo_xfer(amd_spi, master, msg);
 }
 
 static size_t amd_spi_max_transfer_size(struct spi_device *spi)
@@ -312,6 +405,8 @@ static int amd_spi_probe(struct platform_device *pdev)
 	master->num_chipselect = 4;
 	master->mode_bits = 0;
 	master->flags = SPI_MASTER_HALF_DUPLEX;
+	master->max_speed_hz = AMD_SPI_MAX_HZ;
+	master->min_speed_hz = AMD_SPI_MIN_HZ;
 	master->setup = amd_spi_master_setup;
 	master->transfer_one_message = amd_spi_master_transfer;
 	master->max_transfer_size = amd_spi_max_transfer_size;
diff --git a/drivers/spi/spi-aspeed-smc.c b/drivers/spi/spi-aspeed-smc.c
index 3e891bf22470..a334e89add86 100644
--- a/drivers/spi/spi-aspeed-smc.c
+++ b/drivers/spi/spi-aspeed-smc.c
@@ -736,10 +736,8 @@ static int aspeed_spi_probe(struct platform_device *pdev)
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	aspi->regs = devm_ioremap_resource(dev, res);
-	if (IS_ERR(aspi->regs)) {
-		dev_err(dev, "missing AHB register window\n");
+	if (IS_ERR(aspi->regs))
 		return PTR_ERR(aspi->regs);
-	}
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	aspi->ahb_base = devm_ioremap_resource(dev, res);
diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c
index e12ab5b43f34..447230547945 100644
--- a/drivers/spi/spi-cadence-quadspi.c
+++ b/drivers/spi/spi-cadence-quadspi.c
@@ -1645,7 +1645,7 @@ static int cqspi_probe(struct platform_device *pdev)
 	pm_runtime_enable(dev);
 	ret = pm_runtime_resume_and_get(dev);
 	if (ret < 0)
-		return ret;
+		goto probe_pm_failed;
 
 	ret = clk_prepare_enable(cqspi->clk);
 	if (ret) {
@@ -1740,6 +1740,7 @@ probe_reset_failed:
 	clk_disable_unprepare(cqspi->clk);
 probe_clk_failed:
 	pm_runtime_put_sync(dev);
+probe_pm_failed:
 	pm_runtime_disable(dev);
 	return ret;
 }
diff --git a/drivers/spi/spi-cadence-xspi.c b/drivers/spi/spi-cadence-xspi.c
index 3ab19be83095..9e187f9c6c95 100644
--- a/drivers/spi/spi-cadence-xspi.c
+++ b/drivers/spi/spi-cadence-xspi.c
@@ -565,10 +565,8 @@ static int cdns_xspi_probe(struct platform_device *pdev)
 
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sdma");
 	cdns_xspi->sdmabase = devm_ioremap_resource(dev, res);
-	if (IS_ERR(cdns_xspi->sdmabase)) {
-		dev_err(dev, "Failed to remap SDMA address\n");
+	if (IS_ERR(cdns_xspi->sdmabase))
 		return PTR_ERR(cdns_xspi->sdmabase);
-	}
 	cdns_xspi->sdmasize = resource_size(res);
 
 	cdns_xspi->auxbase = devm_platform_ioremap_resource_byname(pdev, "aux");
diff --git a/drivers/spi/spi-dw-bt1.c b/drivers/spi/spi-dw-bt1.c
index c06553416123..3fb89dee595e 100644
--- a/drivers/spi/spi-dw-bt1.c
+++ b/drivers/spi/spi-dw-bt1.c
@@ -293,8 +293,10 @@ static int dw_spi_bt1_probe(struct platform_device *pdev)
 	pm_runtime_enable(&pdev->dev);
 
 	ret = dw_spi_add_host(&pdev->dev, dws);
-	if (ret)
+	if (ret) {
+		pm_runtime_disable(&pdev->dev);
 		goto err_disable_clk;
+	}
 
 	platform_set_drvdata(pdev, dwsbt1);
 
diff --git a/drivers/spi/spi-dw-core.c b/drivers/spi/spi-dw-core.c
index f87d97ccd2d6..99edddf9958b 100644
--- a/drivers/spi/spi-dw-core.c
+++ b/drivers/spi/spi-dw-core.c
@@ -955,7 +955,7 @@ int dw_spi_add_host(struct device *dev, struct dw_spi *dws)
 
 	ret = spi_register_controller(master);
 	if (ret) {
-		dev_err(&master->dev, "problem registering spi master\n");
+		dev_err_probe(dev, ret, "problem registering spi master\n");
 		goto err_dma_exit;
 	}
 
diff --git a/drivers/spi/spi-fsl-dspi.c b/drivers/spi/spi-fsl-dspi.c
index fd004c9db9dc..a33e547b7d39 100644
--- a/drivers/spi/spi-fsl-dspi.c
+++ b/drivers/spi/spi-fsl-dspi.c
@@ -1294,8 +1294,7 @@ static int dspi_probe(struct platform_device *pdev)
 	else
 		ctlr->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	base = devm_ioremap_resource(&pdev->dev, res);
+	base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(base)) {
 		ret = PTR_ERR(base);
 		goto out_ctlr_put;
diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c
index 19b1f3d881b0..e8c1c8a4c6c8 100644
--- a/drivers/spi/spi-fsl-lpspi.c
+++ b/drivers/spi/spi-fsl-lpspi.c
@@ -855,8 +855,7 @@ static int fsl_lpspi_probe(struct platform_device *pdev)
 
 	init_completion(&fsl_lpspi->xfer_done);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	fsl_lpspi->base = devm_ioremap_resource(&pdev->dev, res);
+	fsl_lpspi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
 	if (IS_ERR(fsl_lpspi->base)) {
 		ret = PTR_ERR(fsl_lpspi->base);
 		goto out_controller_put;
@@ -912,7 +911,7 @@ static int fsl_lpspi_probe(struct platform_device *pdev)
 
 	ret = devm_spi_register_controller(&pdev->dev, controller);
 	if (ret < 0) {
-		dev_err_probe(&pdev->dev, ret, "spi_register_controller error: %i\n", ret);
+		dev_err_probe(&pdev->dev, ret, "spi_register_controller error\n");
 		goto free_dma;
 	}
 
@@ -947,11 +946,8 @@ static int fsl_lpspi_remove(struct platform_device *pdev)
 
 static int __maybe_unused fsl_lpspi_suspend(struct device *dev)
 {
-	int ret;
-
 	pinctrl_pm_select_sleep_state(dev);
-	ret = pm_runtime_force_suspend(dev);
-	return ret;
+	return pm_runtime_force_suspend(dev);
 }
 
 static int __maybe_unused fsl_lpspi_resume(struct device *dev)
diff --git a/drivers/spi/spi-fsl-qspi.c b/drivers/spi/spi-fsl-qspi.c
index 46ae46a944c5..85cc71ba624a 100644
--- a/drivers/spi/spi-fsl-qspi.c
+++ b/drivers/spi/spi-fsl-qspi.c
@@ -867,8 +867,7 @@ static int fsl_qspi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, q);
 
 	/* find the resources */
-	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "QuadSPI");
-	q->iobase = devm_ioremap_resource(dev, res);
+	q->iobase = devm_platform_ioremap_resource_byname(pdev, "QuadSPI");
 	if (IS_ERR(q->iobase)) {
 		ret = PTR_ERR(q->iobase);
 		goto err_put_ctrl;
diff --git a/drivers/spi/spi-fsl-spi.c b/drivers/spi/spi-fsl-spi.c
index bdf94cc7be1a..731624f157fc 100644
--- a/drivers/spi/spi-fsl-spi.c
+++ b/drivers/spi/spi-fsl-spi.c
@@ -111,32 +111,6 @@ static void fsl_spi_change_mode(struct spi_device *spi)
 	local_irq_restore(flags);
 }
 
-static void fsl_spi_chipselect(struct spi_device *spi, int value)
-{
-	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(spi->master);
-	struct fsl_spi_platform_data *pdata;
-	struct spi_mpc8xxx_cs	*cs = spi->controller_state;
-
-	pdata = spi->dev.parent->parent->platform_data;
-
-	if (value == BITBANG_CS_INACTIVE) {
-		if (pdata->cs_control)
-			pdata->cs_control(spi, false);
-	}
-
-	if (value == BITBANG_CS_ACTIVE) {
-		mpc8xxx_spi->rx_shift = cs->rx_shift;
-		mpc8xxx_spi->tx_shift = cs->tx_shift;
-		mpc8xxx_spi->get_rx = cs->get_rx;
-		mpc8xxx_spi->get_tx = cs->get_tx;
-
-		fsl_spi_change_mode(spi);
-
-		if (pdata->cs_control)
-			pdata->cs_control(spi, true);
-	}
-}
-
 static void fsl_spi_qe_cpu_set_shifts(u32 *rx_shift, u32 *tx_shift,
 				      int bits_per_word, int msb_first)
 {
@@ -354,15 +328,11 @@ static int fsl_spi_bufs(struct spi_device *spi, struct spi_transfer *t,
 	return mpc8xxx_spi->count;
 }
 
-static int fsl_spi_do_one_msg(struct spi_master *master,
-			      struct spi_message *m)
+static int fsl_spi_prepare_message(struct spi_controller *ctlr,
+				   struct spi_message *m)
 {
-	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master);
-	struct spi_device *spi = m->spi;
-	struct spi_transfer *t, *first;
-	unsigned int cs_change;
-	const int nsecs = 50;
-	int status, last_bpw;
+	struct mpc8xxx_spi *mpc8xxx_spi = spi_controller_get_devdata(ctlr);
+	struct spi_transfer *t;
 
 	/*
 	 * In CPU mode, optimize large byte transfers to use larger
@@ -378,62 +348,30 @@ static int fsl_spi_do_one_msg(struct spi_master *master,
 				t->bits_per_word = 16;
 		}
 	}
+	return 0;
+}
 
-	/* Don't allow changes if CS is active */
-	cs_change = 1;
-	list_for_each_entry(t, &m->transfers, transfer_list) {
-		if (cs_change)
-			first = t;
-		cs_change = t->cs_change;
-		if (first->speed_hz != t->speed_hz) {
-			dev_err(&spi->dev,
-				"speed_hz cannot change while CS is active\n");
-			return -EINVAL;
-		}
-	}
-
-	last_bpw = -1;
-	cs_change = 1;
-	status = -EINVAL;
-	list_for_each_entry(t, &m->transfers, transfer_list) {
-		if (cs_change || last_bpw != t->bits_per_word)
-			status = fsl_spi_setup_transfer(spi, t);
-		if (status < 0)
-			break;
-		last_bpw = t->bits_per_word;
-
-		if (cs_change) {
-			fsl_spi_chipselect(spi, BITBANG_CS_ACTIVE);
-			ndelay(nsecs);
-		}
-		cs_change = t->cs_change;
-		if (t->len)
-			status = fsl_spi_bufs(spi, t, m->is_dma_mapped);
-		if (status) {
-			status = -EMSGSIZE;
-			break;
-		}
-		m->actual_length += t->len;
-
-		spi_transfer_delay_exec(t);
-
-		if (cs_change) {
-			ndelay(nsecs);
-			fsl_spi_chipselect(spi, BITBANG_CS_INACTIVE);
-			ndelay(nsecs);
-		}
-	}
+static int fsl_spi_transfer_one(struct spi_controller *controller,
+				struct spi_device *spi,
+				struct spi_transfer *t)
+{
+	int status;
 
-	m->status = status;
+	status = fsl_spi_setup_transfer(spi, t);
+	if (status < 0)
+		return status;
+	if (t->len)
+		status = fsl_spi_bufs(spi, t, !!t->tx_dma || !!t->rx_dma);
+	if (status > 0)
+		return -EMSGSIZE;
 
-	if (status || !cs_change) {
-		ndelay(nsecs);
-		fsl_spi_chipselect(spi, BITBANG_CS_INACTIVE);
-	}
+	return status;
+}
 
-	fsl_spi_setup_transfer(spi, NULL);
-	spi_finalize_current_message(master);
-	return 0;
+static int fsl_spi_unprepare_message(struct spi_controller *controller,
+				     struct spi_message *msg)
+{
+	return fsl_spi_setup_transfer(msg->spi, NULL);
 }
 
 static int fsl_spi_setup(struct spi_device *spi)
@@ -482,9 +420,6 @@ static int fsl_spi_setup(struct spi_device *spi)
 		return retval;
 	}
 
-	/* Initialize chipselect - might be active for SPI_CS_HIGH mode */
-	fsl_spi_chipselect(spi, BITBANG_CS_INACTIVE);
-
 	return 0;
 }
 
@@ -557,9 +492,7 @@ static void fsl_spi_grlib_cs_control(struct spi_device *spi, bool on)
 	u32 slvsel;
 	u16 cs = spi->chip_select;
 
-	if (spi->cs_gpiod) {
-		gpiod_set_value(spi->cs_gpiod, on);
-	} else if (cs < mpc8xxx_spi->native_chipselects) {
+	if (cs < mpc8xxx_spi->native_chipselects) {
 		slvsel = mpc8xxx_spi_read_reg(&reg_base->slvsel);
 		slvsel = on ? (slvsel | (1 << cs)) : (slvsel & ~(1 << cs));
 		mpc8xxx_spi_write_reg(&reg_base->slvsel, slvsel);
@@ -568,7 +501,6 @@ static void fsl_spi_grlib_cs_control(struct spi_device *spi, bool on)
 
 static void fsl_spi_grlib_probe(struct device *dev)
 {
-	struct fsl_spi_platform_data *pdata = dev_get_platdata(dev);
 	struct spi_master *master = dev_get_drvdata(dev);
 	struct mpc8xxx_spi *mpc8xxx_spi = spi_master_get_devdata(master);
 	struct fsl_spi_reg __iomem *reg_base = mpc8xxx_spi->reg_base;
@@ -588,7 +520,18 @@ static void fsl_spi_grlib_probe(struct device *dev)
 		mpc8xxx_spi_write_reg(&reg_base->slvsel, 0xffffffff);
 	}
 	master->num_chipselect = mpc8xxx_spi->native_chipselects;
-	pdata->cs_control = fsl_spi_grlib_cs_control;
+	master->set_cs = fsl_spi_grlib_cs_control;
+}
+
+static void fsl_spi_cs_control(struct spi_device *spi, bool on)
+{
+	struct device *dev = spi->dev.parent->parent;
+	struct fsl_spi_platform_data *pdata = dev_get_platdata(dev);
+	struct mpc8xxx_spi_probe_info *pinfo = to_of_pinfo(pdata);
+
+	if (WARN_ON_ONCE(!pinfo->immr_spi_cs))
+		return;
+	iowrite32be(on ? 0 : SPI_BOOT_SEL_BIT, pinfo->immr_spi_cs);
 }
 
 static struct spi_master *fsl_spi_probe(struct device *dev,
@@ -613,8 +556,11 @@ static struct spi_master *fsl_spi_probe(struct device *dev,
 
 	master->setup = fsl_spi_setup;
 	master->cleanup = fsl_spi_cleanup;
-	master->transfer_one_message = fsl_spi_do_one_msg;
+	master->prepare_message = fsl_spi_prepare_message;
+	master->transfer_one = fsl_spi_transfer_one;
+	master->unprepare_message = fsl_spi_unprepare_message;
 	master->use_gpio_descriptors = true;
+	master->set_cs = fsl_spi_cs_control;
 
 	mpc8xxx_spi = spi_master_get_devdata(master);
 	mpc8xxx_spi->max_bits_per_word = 32;
@@ -688,21 +634,6 @@ err:
 	return ERR_PTR(ret);
 }
 
-static void fsl_spi_cs_control(struct spi_device *spi, bool on)
-{
-	if (spi->cs_gpiod) {
-		gpiod_set_value(spi->cs_gpiod, on);
-	} else {
-		struct device *dev = spi->dev.parent->parent;
-		struct fsl_spi_platform_data *pdata = dev_get_platdata(dev);
-		struct mpc8xxx_spi_probe_info *pinfo = to_of_pinfo(pdata);
-
-		if (WARN_ON_ONCE(!pinfo->immr_spi_cs))
-			return;
-		iowrite32be(on ? 0 : SPI_BOOT_SEL_BIT, pinfo->immr_spi_cs);
-	}
-}
-
 static int of_fsl_spi_probe(struct platform_device *ofdev)
 {
 	struct device *dev = &ofdev->dev;
@@ -744,12 +675,10 @@ static int of_fsl_spi_probe(struct platform_device *ofdev)
 		ret = gpiod_count(dev, "cs");
 		if (ret < 0)
 			ret = 0;
-		if (ret == 0 && !spisel_boot) {
+		if (ret == 0 && !spisel_boot)
 			pdata->max_chipselect = 1;
-		} else {
+		else
 			pdata->max_chipselect = ret + spisel_boot;
-			pdata->cs_control = fsl_spi_cs_control;
-		}
 	}
 
 	ret = of_address_to_resource(np, 0, &mem);
diff --git a/drivers/spi/spi-gxp.c b/drivers/spi/spi-gxp.c
index 9ea355f7d64f..15b110183839 100644
--- a/drivers/spi/spi-gxp.c
+++ b/drivers/spi/spi-gxp.c
@@ -254,7 +254,6 @@ static int gxp_spifi_probe(struct platform_device *pdev)
 	const struct gxp_spi_data *data;
 	struct spi_controller *ctlr;
 	struct gxp_spi *spifi;
-	struct resource *res;
 	int ret;
 
 	data = of_device_get_match_data(&pdev->dev);
@@ -269,18 +268,15 @@ static int gxp_spifi_probe(struct platform_device *pdev)
 	spifi->data = data;
 	spifi->dev = dev;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	spifi->reg_base = devm_ioremap_resource(&pdev->dev, res);
+	spifi->reg_base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(spifi->reg_base))
 		return PTR_ERR(spifi->reg_base);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-	spifi->dat_base = devm_ioremap_resource(&pdev->dev, res);
+	spifi->dat_base = devm_platform_ioremap_resource(pdev, 1);
 	if (IS_ERR(spifi->dat_base))
 		return PTR_ERR(spifi->dat_base);
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 2);
-	spifi->dir_base = devm_ioremap_resource(&pdev->dev, res);
+	spifi->dir_base = devm_platform_ioremap_resource(pdev, 2);
 	if (IS_ERR(spifi->dir_base))
 		return PTR_ERR(spifi->dir_base);
 
diff --git a/drivers/spi/spi-img-spfi.c b/drivers/spi/spi-img-spfi.c
index 71376b6df89d..bfd12247f173 100644
--- a/drivers/spi/spi-img-spfi.c
+++ b/drivers/spi/spi-img-spfi.c
@@ -730,11 +730,9 @@ static int img_spfi_resume(struct device *dev)
 	struct img_spfi *spfi = spi_master_get_devdata(master);
 	int ret;
 
-	ret = pm_runtime_get_sync(dev);
-	if (ret < 0) {
-		pm_runtime_put_noidle(dev);
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret < 0)
 		return ret;
-	}
 	spfi_reset(spfi);
 	pm_runtime_put(dev);
 
diff --git a/drivers/spi/spi-intel.c b/drivers/spi/spi-intel.c
index 66063687ae27..55f4ee2db002 100644
--- a/drivers/spi/spi-intel.c
+++ b/drivers/spi/spi-intel.c
@@ -116,6 +116,22 @@
 #define ERASE_64K_OPCODE_SHIFT		16
 #define ERASE_64K_OPCODE_MASK		(0xff << ERASE_OPCODE_SHIFT)
 
+/* Flash descriptor fields */
+#define FLVALSIG_MAGIC			0x0ff0a55a
+#define FLMAP0_NC_MASK			GENMASK(9, 8)
+#define FLMAP0_NC_SHIFT			8
+#define FLMAP0_FCBA_MASK		GENMASK(7, 0)
+
+#define FLCOMP_C0DEN_MASK		GENMASK(3, 0)
+#define FLCOMP_C0DEN_512K		0x00
+#define FLCOMP_C0DEN_1M			0x01
+#define FLCOMP_C0DEN_2M			0x02
+#define FLCOMP_C0DEN_4M			0x03
+#define FLCOMP_C0DEN_8M			0x04
+#define FLCOMP_C0DEN_16M		0x05
+#define FLCOMP_C0DEN_32M		0x06
+#define FLCOMP_C0DEN_64M		0x07
+
 #define INTEL_SPI_TIMEOUT		5000 /* ms */
 #define INTEL_SPI_FIFO_SZ		64
 
@@ -129,6 +145,7 @@
  * @master: Pointer to the SPI controller structure
  * @nregions: Maximum number of regions
  * @pr_num: Maximum number of protected range registers
+ * @chip0_size: Size of the first flash chip in bytes
  * @locked: Is SPI setting locked
  * @swseq_reg: Use SW sequencer in register reads/writes
  * @swseq_erase: Use SW sequencer in erase operation
@@ -146,6 +163,7 @@ struct intel_spi {
 	struct spi_controller *master;
 	size_t nregions;
 	size_t pr_num;
+	size_t chip0_size;
 	bool locked;
 	bool swseq_reg;
 	bool swseq_erase;
@@ -158,6 +176,7 @@ struct intel_spi_mem_op {
 	struct spi_mem_op mem_op;
 	u32 replacement_op;
 	int (*exec_op)(struct intel_spi *ispi,
+		       const struct spi_mem *mem,
 		       const struct intel_spi_mem_op *iop,
 		       const struct spi_mem_op *op);
 };
@@ -441,7 +460,16 @@ static int intel_spi_sw_cycle(struct intel_spi *ispi, u8 opcode, size_t len,
 	return 0;
 }
 
-static int intel_spi_read_reg(struct intel_spi *ispi,
+static u32 intel_spi_chip_addr(const struct intel_spi *ispi,
+			       const struct spi_mem *mem)
+{
+	/* Pick up the correct start address */
+	if (!mem)
+		return 0;
+	return mem->spi->chip_select == 1 ? ispi->chip0_size : 0;
+}
+
+static int intel_spi_read_reg(struct intel_spi *ispi, const struct spi_mem *mem,
 			      const struct intel_spi_mem_op *iop,
 			      const struct spi_mem_op *op)
 {
@@ -449,8 +477,7 @@ static int intel_spi_read_reg(struct intel_spi *ispi,
 	u8 opcode = op->cmd.opcode;
 	int ret;
 
-	/* Address of the first chip */
-	writel(0, ispi->base + FADDR);
+	writel(intel_spi_chip_addr(ispi, mem), ispi->base + FADDR);
 
 	if (ispi->swseq_reg)
 		ret = intel_spi_sw_cycle(ispi, opcode, nbytes,
@@ -464,7 +491,7 @@ static int intel_spi_read_reg(struct intel_spi *ispi,
 	return intel_spi_read_block(ispi, op->data.buf.in, nbytes);
 }
 
-static int intel_spi_write_reg(struct intel_spi *ispi,
+static int intel_spi_write_reg(struct intel_spi *ispi, const struct spi_mem *mem,
 			       const struct intel_spi_mem_op *iop,
 			       const struct spi_mem_op *op)
 {
@@ -511,7 +538,7 @@ static int intel_spi_write_reg(struct intel_spi *ispi,
 	if (opcode == SPINOR_OP_WRDI)
 		return 0;
 
-	writel(0, ispi->base + FADDR);
+	writel(intel_spi_chip_addr(ispi, mem), ispi->base + FADDR);
 
 	/* Write the value beforehand */
 	ret = intel_spi_write_block(ispi, op->data.buf.out, nbytes);
@@ -524,13 +551,13 @@ static int intel_spi_write_reg(struct intel_spi *ispi,
 	return intel_spi_hw_cycle(ispi, opcode, nbytes);
 }
 
-static int intel_spi_read(struct intel_spi *ispi,
+static int intel_spi_read(struct intel_spi *ispi, const struct spi_mem *mem,
 			  const struct intel_spi_mem_op *iop,
 			  const struct spi_mem_op *op)
 {
-	void *read_buf = op->data.buf.in;
+	u32 addr = intel_spi_chip_addr(ispi, mem) + op->addr.val;
 	size_t block_size, nbytes = op->data.nbytes;
-	u32 addr = op->addr.val;
+	void *read_buf = op->data.buf.in;
 	u32 val, status;
 	int ret;
 
@@ -585,13 +612,13 @@ static int intel_spi_read(struct intel_spi *ispi,
 	return 0;
 }
 
-static int intel_spi_write(struct intel_spi *ispi,
+static int intel_spi_write(struct intel_spi *ispi, const struct spi_mem *mem,
 			   const struct intel_spi_mem_op *iop,
 			   const struct spi_mem_op *op)
 {
+	u32 addr = intel_spi_chip_addr(ispi, mem) + op->addr.val;
 	size_t block_size, nbytes = op->data.nbytes;
 	const void *write_buf = op->data.buf.out;
-	u32 addr = op->addr.val;
 	u32 val, status;
 	int ret;
 
@@ -648,12 +675,12 @@ static int intel_spi_write(struct intel_spi *ispi,
 	return 0;
 }
 
-static int intel_spi_erase(struct intel_spi *ispi,
+static int intel_spi_erase(struct intel_spi *ispi, const struct spi_mem *mem,
 			   const struct intel_spi_mem_op *iop,
 			   const struct spi_mem_op *op)
 {
+	u32 addr = intel_spi_chip_addr(ispi, mem) + op->addr.val;
 	u8 opcode = op->cmd.opcode;
-	u32 addr = op->addr.val;
 	u32 val, status;
 	int ret;
 
@@ -765,7 +792,7 @@ static int intel_spi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *o
 	if (!iop)
 		return -EOPNOTSUPP;
 
-	return iop->exec_op(ispi, iop, op);
+	return iop->exec_op(ispi, mem, iop, op);
 }
 
 static const char *intel_spi_get_name(struct spi_mem *mem)
@@ -805,7 +832,7 @@ static ssize_t intel_spi_dirmap_read(struct spi_mem_dirmap_desc *desc, u64 offs,
 	op.data.nbytes = len;
 	op.data.buf.in = buf;
 
-	ret = iop->exec_op(ispi, iop, &op);
+	ret = iop->exec_op(ispi, desc->mem, iop, &op);
 	return ret ? ret : len;
 }
 
@@ -821,7 +848,7 @@ static ssize_t intel_spi_dirmap_write(struct spi_mem_dirmap_desc *desc, u64 offs
 	op.data.nbytes = len;
 	op.data.buf.out = buf;
 
-	ret = iop->exec_op(ispi, iop, &op);
+	ret = iop->exec_op(ispi, desc->mem, iop, &op);
 	return ret ? ret : len;
 }
 
@@ -1073,6 +1100,7 @@ static int intel_spi_init(struct intel_spi *ispi)
 		ispi->pregs = ispi->base + CNL_PR;
 		ispi->nregions = CNL_FREG_NUM;
 		ispi->pr_num = CNL_PR_NUM;
+		erase_64k = true;
 		break;
 
 	default:
@@ -1226,10 +1254,98 @@ static void intel_spi_fill_partition(struct intel_spi *ispi,
 	}
 }
 
+static int intel_spi_read_desc(struct intel_spi *ispi)
+{
+	struct spi_mem_op op =
+		SPI_MEM_OP(SPI_MEM_OP_CMD(SPINOR_OP_READ, 0),
+			   SPI_MEM_OP_ADDR(3, 0, 0),
+			   SPI_MEM_OP_NO_DUMMY,
+			   SPI_MEM_OP_DATA_IN(0, NULL, 0));
+	u32 buf[2], nc, fcba, flcomp;
+	ssize_t ret;
+
+	op.addr.val = 0x10;
+	op.data.buf.in = buf;
+	op.data.nbytes = sizeof(buf);
+
+	ret = intel_spi_read(ispi, NULL, NULL, &op);
+	if (ret) {
+		dev_warn(ispi->dev, "failed to read descriptor\n");
+		return ret;
+	}
+
+	dev_dbg(ispi->dev, "FLVALSIG=0x%08x\n", buf[0]);
+	dev_dbg(ispi->dev, "FLMAP0=0x%08x\n", buf[1]);
+
+	if (buf[0] != FLVALSIG_MAGIC) {
+		dev_warn(ispi->dev, "descriptor signature not valid\n");
+		return -ENODEV;
+	}
+
+	fcba = (buf[1] & FLMAP0_FCBA_MASK) << 4;
+	dev_dbg(ispi->dev, "FCBA=%#x\n", fcba);
+
+	op.addr.val = fcba;
+	op.data.buf.in = &flcomp;
+	op.data.nbytes = sizeof(flcomp);
+
+	ret = intel_spi_read(ispi, NULL, NULL, &op);
+	if (ret) {
+		dev_warn(ispi->dev, "failed to read FLCOMP\n");
+		return -ENODEV;
+	}
+
+	dev_dbg(ispi->dev, "FLCOMP=0x%08x\n", flcomp);
+
+	switch (flcomp & FLCOMP_C0DEN_MASK) {
+	case FLCOMP_C0DEN_512K:
+		ispi->chip0_size = SZ_512K;
+		break;
+	case FLCOMP_C0DEN_1M:
+		ispi->chip0_size = SZ_1M;
+		break;
+	case FLCOMP_C0DEN_2M:
+		ispi->chip0_size = SZ_2M;
+		break;
+	case FLCOMP_C0DEN_4M:
+		ispi->chip0_size = SZ_4M;
+		break;
+	case FLCOMP_C0DEN_8M:
+		ispi->chip0_size = SZ_8M;
+		break;
+	case FLCOMP_C0DEN_16M:
+		ispi->chip0_size = SZ_16M;
+		break;
+	case FLCOMP_C0DEN_32M:
+		ispi->chip0_size = SZ_32M;
+		break;
+	case FLCOMP_C0DEN_64M:
+		ispi->chip0_size = SZ_64M;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	dev_dbg(ispi->dev, "chip0 size %zd KB\n", ispi->chip0_size / SZ_1K);
+
+	nc = (buf[1] & FLMAP0_NC_MASK) >> FLMAP0_NC_SHIFT;
+	if (!nc)
+		ispi->master->num_chipselect = 1;
+	else if (nc == 1)
+		ispi->master->num_chipselect = 2;
+	else
+		return -EINVAL;
+
+	dev_dbg(ispi->dev, "%u flash components found\n",
+		ispi->master->num_chipselect);
+	return 0;
+}
+
 static int intel_spi_populate_chip(struct intel_spi *ispi)
 {
 	struct flash_platform_data *pdata;
 	struct spi_board_info chip;
+	int ret;
 
 	pdata = devm_kzalloc(ispi->dev, sizeof(*pdata), GFP_KERNEL);
 	if (!pdata)
@@ -1247,7 +1363,23 @@ static int intel_spi_populate_chip(struct intel_spi *ispi)
 	snprintf(chip.modalias, 8, "spi-nor");
 	chip.platform_data = pdata;
 
-	return spi_new_device(ispi->master, &chip) ? 0 : -ENODEV;
+	if (!spi_new_device(ispi->master, &chip))
+		return -ENODEV;
+
+	/* Add the second chip if present */
+	if (ispi->master->num_chipselect < 2)
+		return 0;
+
+	ret = intel_spi_read_desc(ispi);
+	if (ret)
+		return ret;
+
+	chip.platform_data = NULL;
+	chip.chip_select = 1;
+
+	if (!spi_new_device(ispi->master, &chip))
+		return -ENODEV;
+	return 0;
 }
 
 /**
diff --git a/drivers/spi/spi-loopback-test.c b/drivers/spi/spi-loopback-test.c
index 4d4f77a186a9..dd7de8fa37d0 100644
--- a/drivers/spi/spi-loopback-test.c
+++ b/drivers/spi/spi-loopback-test.c
@@ -313,6 +313,33 @@ static struct spi_test spi_tests[] = {
 			},
 		},
 	},
+	{
+		.description	= "three tx+rx transfers with overlapping cache lines",
+		.fill_option	= FILL_COUNT_8,
+		/*
+		 * This should be large enough for the controller driver to
+		 * choose to transfer it with DMA.
+		 */
+		.iterate_len    = { 512, -1 },
+		.iterate_transfer_mask = BIT(1),
+		.transfer_count = 3,
+		.transfers		= {
+			{
+				.len = 1,
+				.tx_buf = TX(0),
+				.rx_buf = RX(0),
+			},
+			{
+				.tx_buf = TX(1),
+				.rx_buf = RX(1),
+			},
+			{
+				.len = 1,
+				.tx_buf = TX(513),
+				.rx_buf = RX(513),
+			},
+		},
+	},
 
 	{ /* end of tests sequence */ }
 };
diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c
index e4cb52e1fe26..bad201510a99 100644
--- a/drivers/spi/spi-meson-spicc.c
+++ b/drivers/spi/spi-meson-spicc.c
@@ -537,7 +537,7 @@ static unsigned long meson_spicc_pow2_recalc_rate(struct clk_hw *hw,
 	struct clk_divider *divider = to_clk_divider(hw);
 	struct meson_spicc_device *spicc = pow2_clk_to_spicc(divider);
 
-	if (!spicc->master->cur_msg || !spicc->master->busy)
+	if (!spicc->master->cur_msg)
 		return 0;
 
 	return clk_divider_ops.recalc_rate(hw, parent_rate);
@@ -549,7 +549,7 @@ static int meson_spicc_pow2_determine_rate(struct clk_hw *hw,
 	struct clk_divider *divider = to_clk_divider(hw);
 	struct meson_spicc_device *spicc = pow2_clk_to_spicc(divider);
 
-	if (!spicc->master->cur_msg || !spicc->master->busy)
+	if (!spicc->master->cur_msg)
 		return -EINVAL;
 
 	return clk_divider_ops.determine_rate(hw, req);
@@ -561,13 +561,13 @@ static int meson_spicc_pow2_set_rate(struct clk_hw *hw, unsigned long rate,
 	struct clk_divider *divider = to_clk_divider(hw);
 	struct meson_spicc_device *spicc = pow2_clk_to_spicc(divider);
 
-	if (!spicc->master->cur_msg || !spicc->master->busy)
+	if (!spicc->master->cur_msg)
 		return -EINVAL;
 
 	return clk_divider_ops.set_rate(hw, rate, parent_rate);
 }
 
-const struct clk_ops meson_spicc_pow2_clk_ops = {
+static const struct clk_ops meson_spicc_pow2_clk_ops = {
 	.recalc_rate = meson_spicc_pow2_recalc_rate,
 	.determine_rate = meson_spicc_pow2_determine_rate,
 	.set_rate = meson_spicc_pow2_set_rate,
diff --git a/drivers/spi/spi-microchip-core-qspi.c b/drivers/spi/spi-microchip-core-qspi.c
new file mode 100644
index 000000000000..19a6a46829f6
--- /dev/null
+++ b/drivers/spi/spi-microchip-core-qspi.c
@@ -0,0 +1,600 @@
+// SPDX-License-Identifier: (GPL-2.0)
+/*
+ * Microchip coreQSPI QSPI controller driver
+ *
+ * Copyright (C) 2018-2022 Microchip Technology Inc. and its subsidiaries
+ *
+ * Author: Naga Sureshkumar Relli <nagasuresh.relli@microchip.com>
+ *
+ */
+
+#include <linux/clk.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/spi/spi.h>
+#include <linux/spi/spi-mem.h>
+
+/*
+ * QSPI Control register mask defines
+ */
+#define CONTROL_ENABLE		BIT(0)
+#define CONTROL_MASTER		BIT(1)
+#define CONTROL_XIP		BIT(2)
+#define CONTROL_XIPADDR		BIT(3)
+#define CONTROL_CLKIDLE		BIT(10)
+#define CONTROL_SAMPLE_MASK	GENMASK(12, 11)
+#define CONTROL_MODE0		BIT(13)
+#define CONTROL_MODE12_MASK	GENMASK(15, 14)
+#define CONTROL_MODE12_EX_RO	BIT(14)
+#define CONTROL_MODE12_EX_RW	BIT(15)
+#define CONTROL_MODE12_FULL	GENMASK(15, 14)
+#define CONTROL_FLAGSX4		BIT(16)
+#define CONTROL_CLKRATE_MASK	GENMASK(27, 24)
+#define CONTROL_CLKRATE_SHIFT	24
+
+/*
+ * QSPI Frames register mask defines
+ */
+#define FRAMES_TOTALBYTES_MASK	GENMASK(15, 0)
+#define FRAMES_CMDBYTES_MASK	GENMASK(24, 16)
+#define FRAMES_CMDBYTES_SHIFT	16
+#define FRAMES_SHIFT		25
+#define FRAMES_IDLE_MASK	GENMASK(29, 26)
+#define FRAMES_IDLE_SHIFT	26
+#define FRAMES_FLAGBYTE		BIT(30)
+#define FRAMES_FLAGWORD		BIT(31)
+
+/*
+ * QSPI Interrupt Enable register mask defines
+ */
+#define IEN_TXDONE		BIT(0)
+#define IEN_RXDONE		BIT(1)
+#define IEN_RXAVAILABLE		BIT(2)
+#define IEN_TXAVAILABLE		BIT(3)
+#define IEN_RXFIFOEMPTY		BIT(4)
+#define IEN_TXFIFOFULL		BIT(5)
+
+/*
+ * QSPI Status register mask defines
+ */
+#define STATUS_TXDONE		BIT(0)
+#define STATUS_RXDONE		BIT(1)
+#define STATUS_RXAVAILABLE	BIT(2)
+#define STATUS_TXAVAILABLE	BIT(3)
+#define STATUS_RXFIFOEMPTY	BIT(4)
+#define STATUS_TXFIFOFULL	BIT(5)
+#define STATUS_READY		BIT(7)
+#define STATUS_FLAGSX4		BIT(8)
+#define STATUS_MASK		GENMASK(8, 0)
+
+#define BYTESUPPER_MASK		GENMASK(31, 16)
+#define BYTESLOWER_MASK		GENMASK(15, 0)
+
+#define MAX_DIVIDER		16
+#define MIN_DIVIDER		0
+#define MAX_DATA_CMD_LEN	256
+
+/* QSPI ready time out value */
+#define TIMEOUT_MS		500
+
+/*
+ * QSPI Register offsets.
+ */
+#define REG_CONTROL		(0x00)
+#define REG_FRAMES		(0x04)
+#define REG_IEN			(0x0c)
+#define REG_STATUS		(0x10)
+#define REG_DIRECT_ACCESS	(0x14)
+#define REG_UPPER_ACCESS	(0x18)
+#define REG_RX_DATA		(0x40)
+#define REG_TX_DATA		(0x44)
+#define REG_X4_RX_DATA		(0x48)
+#define REG_X4_TX_DATA		(0x4c)
+#define REG_FRAMESUP		(0x50)
+
+/**
+ * struct mchp_coreqspi - Defines qspi driver instance
+ * @regs:              Virtual address of the QSPI controller registers
+ * @clk:               QSPI Operating clock
+ * @data_completion:   completion structure
+ * @op_lock:           lock access to the device
+ * @txbuf:             TX buffer
+ * @rxbuf:             RX buffer
+ * @irq:               IRQ number
+ * @tx_len:            Number of bytes left to transfer
+ * @rx_len:            Number of bytes left to receive
+ */
+struct mchp_coreqspi {
+	void __iomem *regs;
+	struct clk *clk;
+	struct completion data_completion;
+	struct mutex op_lock; /* lock access to the device */
+	u8 *txbuf;
+	u8 *rxbuf;
+	int irq;
+	int tx_len;
+	int rx_len;
+};
+
+static int mchp_coreqspi_set_mode(struct mchp_coreqspi *qspi, const struct spi_mem_op *op)
+{
+	u32 control = readl_relaxed(qspi->regs + REG_CONTROL);
+
+	/*
+	 * The operating mode can be configured based on the command that needs to be send.
+	 * bits[15:14]: Sets whether multiple bit SPI operates in normal, extended or full modes.
+	 *		00: Normal (single DQ0 TX and single DQ1 RX lines)
+	 *		01: Extended RO (command and address bytes on DQ0 only)
+	 *		10: Extended RW (command byte on DQ0 only)
+	 *		11: Full. (command and address are on all DQ lines)
+	 * bit[13]:	Sets whether multiple bit SPI uses 2 or 4 bits of data
+	 *		0: 2-bits (BSPI)
+	 *		1: 4-bits (QSPI)
+	 */
+	if (op->data.buswidth == 4 || op->data.buswidth == 2) {
+		control &= ~CONTROL_MODE12_MASK;
+		if (op->cmd.buswidth == 1 && (op->addr.buswidth == 1 || op->addr.buswidth == 0))
+			control |= CONTROL_MODE12_EX_RO;
+		else if (op->cmd.buswidth == 1)
+			control |= CONTROL_MODE12_EX_RW;
+		else
+			control |= CONTROL_MODE12_FULL;
+
+		control |= CONTROL_MODE0;
+	} else {
+		control &= ~(CONTROL_MODE12_MASK |
+			     CONTROL_MODE0);
+	}
+
+	writel_relaxed(control, qspi->regs + REG_CONTROL);
+
+	return 0;
+}
+
+static inline void mchp_coreqspi_read_op(struct mchp_coreqspi *qspi)
+{
+	u32 control, data;
+
+	if (!qspi->rx_len)
+		return;
+
+	control = readl_relaxed(qspi->regs + REG_CONTROL);
+
+	/*
+	 * Read 4-bytes from the SPI FIFO in single transaction and then read
+	 * the reamaining data byte wise.
+	 */
+	control |= CONTROL_FLAGSX4;
+	writel_relaxed(control, qspi->regs + REG_CONTROL);
+
+	while (qspi->rx_len >= 4) {
+		while (readl_relaxed(qspi->regs + REG_STATUS) & STATUS_RXFIFOEMPTY)
+			;
+		data = readl_relaxed(qspi->regs + REG_X4_RX_DATA);
+		*(u32 *)qspi->rxbuf = data;
+		qspi->rxbuf += 4;
+		qspi->rx_len -= 4;
+	}
+
+	control &= ~CONTROL_FLAGSX4;
+	writel_relaxed(control, qspi->regs + REG_CONTROL);
+
+	while (qspi->rx_len--) {
+		while (readl_relaxed(qspi->regs + REG_STATUS) & STATUS_RXFIFOEMPTY)
+			;
+		data = readl_relaxed(qspi->regs + REG_RX_DATA);
+		*qspi->rxbuf++ = (data & 0xFF);
+	}
+}
+
+static inline void mchp_coreqspi_write_op(struct mchp_coreqspi *qspi, bool word)
+{
+	u32 control, data;
+
+	control = readl_relaxed(qspi->regs + REG_CONTROL);
+	control |= CONTROL_FLAGSX4;
+	writel_relaxed(control, qspi->regs + REG_CONTROL);
+
+	while (qspi->tx_len >= 4) {
+		while (readl_relaxed(qspi->regs + REG_STATUS) & STATUS_TXFIFOFULL)
+			;
+		data = *(u32 *)qspi->txbuf;
+		qspi->txbuf += 4;
+		qspi->tx_len -= 4;
+		writel_relaxed(data, qspi->regs + REG_X4_TX_DATA);
+	}
+
+	control &= ~CONTROL_FLAGSX4;
+	writel_relaxed(control, qspi->regs + REG_CONTROL);
+
+	while (qspi->tx_len--) {
+		while (readl_relaxed(qspi->regs + REG_STATUS) & STATUS_TXFIFOFULL)
+			;
+		data =  *qspi->txbuf++;
+		writel_relaxed(data, qspi->regs + REG_TX_DATA);
+	}
+}
+
+static void mchp_coreqspi_enable_ints(struct mchp_coreqspi *qspi)
+{
+	u32 mask = IEN_TXDONE |
+		   IEN_RXDONE |
+		   IEN_RXAVAILABLE;
+
+	writel_relaxed(mask, qspi->regs + REG_IEN);
+}
+
+static void mchp_coreqspi_disable_ints(struct mchp_coreqspi *qspi)
+{
+	writel_relaxed(0, qspi->regs + REG_IEN);
+}
+
+static irqreturn_t mchp_coreqspi_isr(int irq, void *dev_id)
+{
+	struct mchp_coreqspi *qspi = (struct mchp_coreqspi *)dev_id;
+	irqreturn_t ret = IRQ_NONE;
+	int intfield = readl_relaxed(qspi->regs + REG_STATUS) & STATUS_MASK;
+
+	if (intfield == 0)
+		return ret;
+
+	if (intfield & IEN_TXDONE) {
+		writel_relaxed(IEN_TXDONE, qspi->regs + REG_STATUS);
+		ret = IRQ_HANDLED;
+	}
+
+	if (intfield & IEN_RXAVAILABLE) {
+		writel_relaxed(IEN_RXAVAILABLE, qspi->regs + REG_STATUS);
+		mchp_coreqspi_read_op(qspi);
+		ret = IRQ_HANDLED;
+	}
+
+	if (intfield & IEN_RXDONE) {
+		writel_relaxed(IEN_RXDONE, qspi->regs + REG_STATUS);
+		complete(&qspi->data_completion);
+		ret = IRQ_HANDLED;
+	}
+
+	return ret;
+}
+
+static int mchp_coreqspi_setup_clock(struct mchp_coreqspi *qspi, struct spi_device *spi)
+{
+	unsigned long clk_hz;
+	u32 control, baud_rate_val = 0;
+
+	clk_hz = clk_get_rate(qspi->clk);
+	if (!clk_hz)
+		return -EINVAL;
+
+	baud_rate_val = DIV_ROUND_UP(clk_hz, 2 * spi->max_speed_hz);
+	if (baud_rate_val > MAX_DIVIDER || baud_rate_val < MIN_DIVIDER) {
+		dev_err(&spi->dev,
+			"could not configure the clock for spi clock %d Hz & system clock %ld Hz\n",
+			spi->max_speed_hz, clk_hz);
+		return -EINVAL;
+	}
+
+	control = readl_relaxed(qspi->regs + REG_CONTROL);
+	control |= baud_rate_val << CONTROL_CLKRATE_SHIFT;
+	writel_relaxed(control, qspi->regs + REG_CONTROL);
+	control = readl_relaxed(qspi->regs + REG_CONTROL);
+
+	if ((spi->mode & SPI_CPOL) && (spi->mode & SPI_CPHA))
+		control |= CONTROL_CLKIDLE;
+	else
+		control &= ~CONTROL_CLKIDLE;
+
+	writel_relaxed(control, qspi->regs + REG_CONTROL);
+
+	return 0;
+}
+
+static int mchp_coreqspi_setup_op(struct spi_device *spi_dev)
+{
+	struct spi_controller *ctlr = spi_dev->master;
+	struct mchp_coreqspi *qspi = spi_controller_get_devdata(ctlr);
+	u32 control = readl_relaxed(qspi->regs + REG_CONTROL);
+
+	control |= (CONTROL_MASTER | CONTROL_ENABLE);
+	control &= ~CONTROL_CLKIDLE;
+	writel_relaxed(control, qspi->regs + REG_CONTROL);
+
+	return 0;
+}
+
+static inline void mchp_coreqspi_config_op(struct mchp_coreqspi *qspi, const struct spi_mem_op *op)
+{
+	u32 idle_cycles = 0;
+	int total_bytes, cmd_bytes, frames, ctrl;
+
+	cmd_bytes = op->cmd.nbytes + op->addr.nbytes;
+	total_bytes = cmd_bytes + op->data.nbytes;
+
+	/*
+	 * As per the coreQSPI IP spec,the number of command and data bytes are
+	 * controlled by the frames register for each SPI sequence. This supports
+	 * the SPI flash memory read and writes sequences as below. so configure
+	 * the cmd and total bytes accordingly.
+	 * ---------------------------------------------------------------------
+	 * TOTAL BYTES  |  CMD BYTES | What happens                             |
+	 * ______________________________________________________________________
+	 *              |            |                                          |
+	 *     1        |   1        | The SPI core will transmit a single byte |
+	 *              |            | and receive data is discarded            |
+	 *              |            |                                          |
+	 *     1        |   0        | The SPI core will transmit a single byte |
+	 *              |            | and return a single byte                 |
+	 *              |            |                                          |
+	 *     10       |   4        | The SPI core will transmit 4 command     |
+	 *              |            | bytes discarding the receive data and    |
+	 *              |            | transmits 6 dummy bytes returning the 6  |
+	 *              |            | received bytes and return a single byte  |
+	 *              |            |                                          |
+	 *     10       |   10       | The SPI core will transmit 10 command    |
+	 *              |            |                                          |
+	 *     10       |    0       | The SPI core will transmit 10 command    |
+	 *              |            | bytes and returning 10 received bytes    |
+	 * ______________________________________________________________________
+	 */
+	if (!(op->data.dir == SPI_MEM_DATA_IN))
+		cmd_bytes = total_bytes;
+
+	frames = total_bytes & BYTESUPPER_MASK;
+	writel_relaxed(frames, qspi->regs + REG_FRAMESUP);
+	frames = total_bytes & BYTESLOWER_MASK;
+	frames |= cmd_bytes << FRAMES_CMDBYTES_SHIFT;
+
+	if (op->dummy.buswidth)
+		idle_cycles = op->dummy.nbytes * 8 / op->dummy.buswidth;
+
+	frames |= idle_cycles << FRAMES_IDLE_SHIFT;
+	ctrl = readl_relaxed(qspi->regs + REG_CONTROL);
+
+	if (ctrl & CONTROL_MODE12_MASK)
+		frames |= (1 << FRAMES_SHIFT);
+
+	frames |= FRAMES_FLAGWORD;
+	writel_relaxed(frames, qspi->regs + REG_FRAMES);
+}
+
+static int mchp_qspi_wait_for_ready(struct spi_mem *mem)
+{
+	struct mchp_coreqspi *qspi = spi_controller_get_devdata
+				    (mem->spi->master);
+	u32 status;
+	int ret;
+
+	ret = readl_poll_timeout(qspi->regs + REG_STATUS, status,
+				 (status & STATUS_READY), 0,
+				 TIMEOUT_MS);
+	if (ret) {
+		dev_err(&mem->spi->dev,
+			"Timeout waiting on QSPI ready.\n");
+		return -ETIMEDOUT;
+	}
+
+	return ret;
+}
+
+static int mchp_coreqspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
+{
+	struct mchp_coreqspi *qspi = spi_controller_get_devdata
+				    (mem->spi->master);
+	u32 address = op->addr.val;
+	u8 opcode = op->cmd.opcode;
+	u8 opaddr[5];
+	int err, i;
+
+	mutex_lock(&qspi->op_lock);
+	err = mchp_qspi_wait_for_ready(mem);
+	if (err)
+		goto error;
+
+	err = mchp_coreqspi_setup_clock(qspi, mem->spi);
+	if (err)
+		goto error;
+
+	err = mchp_coreqspi_set_mode(qspi, op);
+	if (err)
+		goto error;
+
+	reinit_completion(&qspi->data_completion);
+	mchp_coreqspi_config_op(qspi, op);
+	if (op->cmd.opcode) {
+		qspi->txbuf = &opcode;
+		qspi->rxbuf = NULL;
+		qspi->tx_len = op->cmd.nbytes;
+		qspi->rx_len = 0;
+		mchp_coreqspi_write_op(qspi, false);
+	}
+
+	qspi->txbuf = &opaddr[0];
+	if (op->addr.nbytes) {
+		for (i = 0; i < op->addr.nbytes; i++)
+			qspi->txbuf[i] = address >> (8 * (op->addr.nbytes - i - 1));
+
+		qspi->rxbuf = NULL;
+		qspi->tx_len = op->addr.nbytes;
+		qspi->rx_len = 0;
+		mchp_coreqspi_write_op(qspi, false);
+	}
+
+	if (op->data.nbytes) {
+		if (op->data.dir == SPI_MEM_DATA_OUT) {
+			qspi->txbuf = (u8 *)op->data.buf.out;
+			qspi->rxbuf = NULL;
+			qspi->rx_len = 0;
+			qspi->tx_len = op->data.nbytes;
+			mchp_coreqspi_write_op(qspi, true);
+		} else {
+			qspi->txbuf = NULL;
+			qspi->rxbuf = (u8 *)op->data.buf.in;
+			qspi->rx_len = op->data.nbytes;
+			qspi->tx_len = 0;
+		}
+	}
+
+	mchp_coreqspi_enable_ints(qspi);
+
+	if (!wait_for_completion_timeout(&qspi->data_completion, msecs_to_jiffies(1000)))
+		err = -ETIMEDOUT;
+
+error:
+	mutex_unlock(&qspi->op_lock);
+	mchp_coreqspi_disable_ints(qspi);
+
+	return err;
+}
+
+static bool mchp_coreqspi_supports_op(struct spi_mem *mem, const struct spi_mem_op *op)
+{
+	if (!spi_mem_default_supports_op(mem, op))
+		return false;
+
+	if ((op->data.buswidth == 4 || op->data.buswidth == 2) &&
+	    (op->cmd.buswidth == 1 && (op->addr.buswidth == 1 || op->addr.buswidth == 0))) {
+		/*
+		 * If the command and address are on DQ0 only, then this
+		 * controller doesn't support sending data on dual and
+		 * quad lines. but it supports reading data on dual and
+		 * quad lines with same configuration as command and
+		 * address on DQ0.
+		 * i.e. The control register[15:13] :EX_RO(read only) is
+		 * meant only for the command and address are on DQ0 but
+		 * not to write data, it is just to read.
+		 * Ex: 0x34h is Quad Load Program Data which is not
+		 * supported. Then the spi-mem layer will iterate over
+		 * each command and it will chose the supported one.
+		 */
+		if (op->data.dir == SPI_MEM_DATA_OUT)
+			return false;
+	}
+
+	return true;
+}
+
+static int mchp_coreqspi_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op)
+{
+	if (op->data.dir == SPI_MEM_DATA_OUT || op->data.dir == SPI_MEM_DATA_IN) {
+		if (op->data.nbytes > MAX_DATA_CMD_LEN)
+			op->data.nbytes = MAX_DATA_CMD_LEN;
+	}
+
+	return 0;
+}
+
+static const struct spi_controller_mem_ops mchp_coreqspi_mem_ops = {
+	.adjust_op_size = mchp_coreqspi_adjust_op_size,
+	.supports_op = mchp_coreqspi_supports_op,
+	.exec_op = mchp_coreqspi_exec_op,
+};
+
+static int mchp_coreqspi_probe(struct platform_device *pdev)
+{
+	struct spi_controller *ctlr;
+	struct mchp_coreqspi *qspi;
+	struct device *dev = &pdev->dev;
+	struct device_node *np = dev->of_node;
+	int ret;
+
+	ctlr = devm_spi_alloc_master(&pdev->dev, sizeof(*qspi));
+	if (!ctlr)
+		return dev_err_probe(&pdev->dev, -ENOMEM,
+				     "unable to allocate master for QSPI controller\n");
+
+	qspi = spi_controller_get_devdata(ctlr);
+	platform_set_drvdata(pdev, qspi);
+
+	qspi->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(qspi->regs))
+		return dev_err_probe(&pdev->dev, PTR_ERR(qspi->regs),
+				     "failed to map registers\n");
+
+	qspi->clk = devm_clk_get(&pdev->dev, NULL);
+	if (IS_ERR(qspi->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(qspi->clk),
+				     "could not get clock\n");
+
+	ret = clk_prepare_enable(qspi->clk);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "failed to enable clock\n");
+
+	init_completion(&qspi->data_completion);
+	mutex_init(&qspi->op_lock);
+
+	qspi->irq = platform_get_irq(pdev, 0);
+	if (qspi->irq < 0) {
+		ret = qspi->irq;
+		goto out;
+	}
+
+	ret = devm_request_irq(&pdev->dev, qspi->irq, mchp_coreqspi_isr,
+			       IRQF_SHARED, pdev->name, qspi);
+	if (ret) {
+		dev_err(&pdev->dev, "request_irq failed %d\n", ret);
+		goto out;
+	}
+
+	ctlr->bits_per_word_mask = SPI_BPW_MASK(8);
+	ctlr->mem_ops = &mchp_coreqspi_mem_ops;
+	ctlr->setup = mchp_coreqspi_setup_op;
+	ctlr->mode_bits = SPI_CPOL | SPI_CPHA | SPI_RX_DUAL | SPI_RX_QUAD |
+			  SPI_TX_DUAL | SPI_TX_QUAD;
+	ctlr->dev.of_node = np;
+
+	ret = devm_spi_register_controller(&pdev->dev, ctlr);
+	if (ret) {
+		dev_err_probe(&pdev->dev, ret,
+			      "spi_register_controller failed\n");
+		goto out;
+	}
+
+	return 0;
+
+out:
+	clk_disable_unprepare(qspi->clk);
+
+	return ret;
+}
+
+static int mchp_coreqspi_remove(struct platform_device *pdev)
+{
+	struct mchp_coreqspi *qspi = platform_get_drvdata(pdev);
+	u32 control = readl_relaxed(qspi->regs + REG_CONTROL);
+
+	mchp_coreqspi_disable_ints(qspi);
+	control &= ~CONTROL_ENABLE;
+	writel_relaxed(control, qspi->regs + REG_CONTROL);
+	clk_disable_unprepare(qspi->clk);
+
+	return 0;
+}
+
+static const struct of_device_id mchp_coreqspi_of_match[] = {
+	{ .compatible = "microchip,coreqspi-rtl-v2" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, mchp_coreqspi_of_match);
+
+static struct platform_driver mchp_coreqspi_driver = {
+	.probe = mchp_coreqspi_probe,
+	.driver = {
+		.name = "microchip,coreqspi",
+		.of_match_table = mchp_coreqspi_of_match,
+	},
+	.remove = mchp_coreqspi_remove,
+};
+module_platform_driver(mchp_coreqspi_driver);
+
+MODULE_AUTHOR("Naga Sureshkumar Relli <nagasuresh.relli@microchip.com");
+MODULE_DESCRIPTION("Microchip coreQSPI QSPI controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/spi/spi-microchip-core.c b/drivers/spi/spi-microchip-core.c
index ce4385330b19..d352844c798c 100644
--- a/drivers/spi/spi-microchip-core.c
+++ b/drivers/spi/spi-microchip-core.c
@@ -548,12 +548,12 @@ static int mchp_corespi_probe(struct platform_device *pdev)
 			       IRQF_SHARED, dev_name(&pdev->dev), master);
 	if (ret)
 		return dev_err_probe(&pdev->dev, ret,
-				     "could not request irq: %d\n", ret);
+				     "could not request irq\n");
 
 	spi->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(spi->clk))
 		return dev_err_probe(&pdev->dev, PTR_ERR(spi->clk),
-				     "could not get clk: %d\n", ret);
+				     "could not get clk\n");
 
 	ret = clk_prepare_enable(spi->clk);
 	if (ret)
diff --git a/drivers/spi/spi-mpc52xx.c b/drivers/spi/spi-mpc52xx.c
index bc5e36fd4288..cb075c1acbee 100644
--- a/drivers/spi/spi-mpc52xx.c
+++ b/drivers/spi/spi-mpc52xx.c
@@ -11,13 +11,14 @@
  */
 
 #include <linux/module.h>
+#include <linux/err.h>
 #include <linux/errno.h>
 #include <linux/of_platform.h>
 #include <linux/interrupt.h>
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/spi/spi.h>
 #include <linux/io.h>
-#include <linux/of_gpio.h>
 #include <linux/slab.h>
 #include <linux/of_address.h>
 #include <linux/of_irq.h>
@@ -89,7 +90,7 @@ struct mpc52xx_spi {
 	const u8 *tx_buf;
 	int cs_change;
 	int gpio_cs_count;
-	unsigned int *gpio_cs;
+	struct gpio_desc **gpio_cs;
 };
 
 /*
@@ -101,9 +102,10 @@ static void mpc52xx_spi_chipsel(struct mpc52xx_spi *ms, int value)
 
 	if (ms->gpio_cs_count > 0) {
 		cs = ms->message->spi->chip_select;
-		gpio_set_value(ms->gpio_cs[cs], value ? 0 : 1);
-	} else
+		gpiod_set_value(ms->gpio_cs[cs], value);
+	} else {
 		out_8(ms->regs + SPI_PORTDATA, value ? 0 : 0x08);
+	}
 }
 
 /*
@@ -385,10 +387,10 @@ static int mpc52xx_spi_probe(struct platform_device *op)
 {
 	struct spi_master *master;
 	struct mpc52xx_spi *ms;
+	struct gpio_desc *gpio_cs;
 	void __iomem *regs;
 	u8 ctrl1;
 	int rc, i = 0;
-	int gpio_cs;
 
 	/* MMIO registers */
 	dev_dbg(&op->dev, "probing mpc5200 SPI device\n");
@@ -438,7 +440,7 @@ static int mpc52xx_spi_probe(struct platform_device *op)
 	ms->irq1 = irq_of_parse_and_map(op->dev.of_node, 1);
 	ms->state = mpc52xx_spi_fsmstate_idle;
 	ms->ipb_freq = mpc5xxx_get_bus_frequency(&op->dev);
-	ms->gpio_cs_count = of_gpio_count(op->dev.of_node);
+	ms->gpio_cs_count = gpiod_count(&op->dev, NULL);
 	if (ms->gpio_cs_count > 0) {
 		master->num_chipselect = ms->gpio_cs_count;
 		ms->gpio_cs = kmalloc_array(ms->gpio_cs_count,
@@ -450,23 +452,16 @@ static int mpc52xx_spi_probe(struct platform_device *op)
 		}
 
 		for (i = 0; i < ms->gpio_cs_count; i++) {
-			gpio_cs = of_get_gpio(op->dev.of_node, i);
-			if (!gpio_is_valid(gpio_cs)) {
-				dev_err(&op->dev,
-					"could not parse the gpio field in oftree\n");
-				rc = -ENODEV;
-				goto err_gpio;
-			}
-
-			rc = gpio_request(gpio_cs, dev_name(&op->dev));
+			gpio_cs = gpiod_get_index(&op->dev,
+						  NULL, i, GPIOD_OUT_LOW);
+			rc = PTR_ERR_OR_ZERO(gpio_cs);
 			if (rc) {
 				dev_err(&op->dev,
-					"can't request spi cs gpio #%d on gpio line %d\n",
-					i, gpio_cs);
+					"failed to get spi cs gpio #%d: %d\n",
+					i, rc);
 				goto err_gpio;
 			}
 
-			gpio_direction_output(gpio_cs, 1);
 			ms->gpio_cs[i] = gpio_cs;
 		}
 	}
@@ -507,7 +502,7 @@ static int mpc52xx_spi_probe(struct platform_device *op)
 	dev_err(&ms->master->dev, "initialization failed\n");
  err_gpio:
 	while (i-- > 0)
-		gpio_free(ms->gpio_cs[i]);
+		gpiod_put(ms->gpio_cs[i]);
 
 	kfree(ms->gpio_cs);
  err_alloc_gpio:
@@ -528,7 +523,7 @@ static int mpc52xx_spi_remove(struct platform_device *op)
 	free_irq(ms->irq1, ms);
 
 	for (i = 0; i < ms->gpio_cs_count; i++)
-		gpio_free(ms->gpio_cs[i]);
+		gpiod_put(ms->gpio_cs[i]);
 
 	kfree(ms->gpio_cs);
 	spi_unregister_master(master);
diff --git a/drivers/spi/spi-mt65xx.c b/drivers/spi/spi-mt65xx.c
index 0a3b9f7eed30..11aeae7fe7fc 100644
--- a/drivers/spi/spi-mt65xx.c
+++ b/drivers/spi/spi-mt65xx.c
@@ -1184,6 +1184,11 @@ static int mtk_spi_probe(struct platform_device *pdev)
 	if (!dev->dma_mask)
 		dev->dma_mask = &dev->coherent_dma_mask;
 
+	if (mdata->dev_comp->ipm_design)
+		dma_set_max_seg_size(dev, SZ_16M);
+	else
+		dma_set_max_seg_size(dev, SZ_256K);
+
 	ret = devm_request_irq(dev, irq, mtk_spi_interrupt,
 			       IRQF_TRIGGER_NONE, dev_name(dev), master);
 	if (ret)
diff --git a/drivers/spi/spi-mt7621.c b/drivers/spi/spi-mt7621.c
index b4b9b7309b5e..c4cc8e2f85e2 100644
--- a/drivers/spi/spi-mt7621.c
+++ b/drivers/spi/spi-mt7621.c
@@ -55,7 +55,6 @@ struct mt7621_spi {
 	void __iomem		*base;
 	unsigned int		sys_freq;
 	unsigned int		speed;
-	struct clk		*clk;
 	int			pending_write;
 };
 
@@ -327,7 +326,6 @@ static int mt7621_spi_probe(struct platform_device *pdev)
 	struct spi_controller *master;
 	struct mt7621_spi *rs;
 	void __iomem *base;
-	int status = 0;
 	struct clk *clk;
 	int ret;
 
@@ -339,21 +337,14 @@ static int mt7621_spi_probe(struct platform_device *pdev)
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
-	clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(clk)) {
-		dev_err(&pdev->dev, "unable to get SYS clock, err=%d\n",
-			status);
-		return PTR_ERR(clk);
-	}
-
-	status = clk_prepare_enable(clk);
-	if (status)
-		return status;
+	clk = devm_clk_get_enabled(&pdev->dev, NULL);
+	if (IS_ERR(clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(clk),
+				     "unable to get SYS clock\n");
 
 	master = devm_spi_alloc_master(&pdev->dev, sizeof(*rs));
 	if (!master) {
 		dev_info(&pdev->dev, "master allocation failed\n");
-		clk_disable_unprepare(clk);
 		return -ENOMEM;
 	}
 
@@ -369,38 +360,18 @@ static int mt7621_spi_probe(struct platform_device *pdev)
 
 	rs = spi_controller_get_devdata(master);
 	rs->base = base;
-	rs->clk = clk;
 	rs->master = master;
-	rs->sys_freq = clk_get_rate(rs->clk);
+	rs->sys_freq = clk_get_rate(clk);
 	rs->pending_write = 0;
 	dev_info(&pdev->dev, "sys_freq: %u\n", rs->sys_freq);
 
 	ret = device_reset(&pdev->dev);
 	if (ret) {
 		dev_err(&pdev->dev, "SPI reset failed!\n");
-		clk_disable_unprepare(clk);
 		return ret;
 	}
 
-	ret = spi_register_controller(master);
-	if (ret)
-		clk_disable_unprepare(clk);
-
-	return ret;
-}
-
-static int mt7621_spi_remove(struct platform_device *pdev)
-{
-	struct spi_controller *master;
-	struct mt7621_spi *rs;
-
-	master = dev_get_drvdata(&pdev->dev);
-	rs = spi_controller_get_devdata(master);
-
-	spi_unregister_controller(master);
-	clk_disable_unprepare(rs->clk);
-
-	return 0;
+	return devm_spi_register_controller(&pdev->dev, master);
 }
 
 MODULE_ALIAS("platform:" DRIVER_NAME);
@@ -411,7 +382,6 @@ static struct platform_driver mt7621_spi_driver = {
 		.of_match_table = mt7621_spi_match,
 	},
 	.probe = mt7621_spi_probe,
-	.remove = mt7621_spi_remove,
 };
 
 module_platform_driver(mt7621_spi_driver);
diff --git a/drivers/spi/spi-npcm-pspi.c b/drivers/spi/spi-npcm-pspi.c
index 1668a347e003..7f2e4d1b0d43 100644
--- a/drivers/spi/spi-npcm-pspi.c
+++ b/drivers/spi/spi-npcm-pspi.c
@@ -443,6 +443,7 @@ static int npcm_pspi_remove(struct platform_device *pdev)
 
 static const struct of_device_id npcm_pspi_match[] = {
 	{ .compatible = "nuvoton,npcm750-pspi", .data = NULL },
+	{ .compatible = "nuvoton,npcm845-pspi", .data = NULL },
 	{}
 };
 MODULE_DEVICE_TABLE(of, npcm_pspi_match);
diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c
index 2b0301fc971c..d6a65a989ef8 100644
--- a/drivers/spi/spi-nxp-fspi.c
+++ b/drivers/spi/spi-nxp-fspi.c
@@ -588,7 +588,7 @@ static int nxp_fspi_clk_prep_enable(struct nxp_fspi *f)
 {
 	int ret;
 
-	if (is_acpi_node(f->dev->fwnode))
+	if (is_acpi_node(dev_fwnode(f->dev)))
 		return 0;
 
 	ret = clk_prepare_enable(f->clk_en);
@@ -606,7 +606,7 @@ static int nxp_fspi_clk_prep_enable(struct nxp_fspi *f)
 
 static int nxp_fspi_clk_disable_unprep(struct nxp_fspi *f)
 {
-	if (is_acpi_node(f->dev->fwnode))
+	if (is_acpi_node(dev_fwnode(f->dev)))
 		return 0;
 
 	clk_disable_unprepare(f->clk);
@@ -1100,7 +1100,7 @@ static int nxp_fspi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, f);
 
 	/* find the resources - configuration register address space */
-	if (is_acpi_node(f->dev->fwnode))
+	if (is_acpi_node(dev_fwnode(f->dev)))
 		res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	else
 		res = platform_get_resource_byname(pdev,
@@ -1113,7 +1113,7 @@ static int nxp_fspi_probe(struct platform_device *pdev)
 	}
 
 	/* find the resources - controller memory mapped space */
-	if (is_acpi_node(f->dev->fwnode))
+	if (is_acpi_node(dev_fwnode(f->dev)))
 		res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
 	else
 		res = platform_get_resource_byname(pdev,
diff --git a/drivers/spi/spi-omap-100k.c b/drivers/spi/spi-omap-100k.c
index 20b047172965..061f7394e5b9 100644
--- a/drivers/spi/spi-omap-100k.c
+++ b/drivers/spi/spi-omap-100k.c
@@ -412,6 +412,7 @@ static int omap1_spi100k_probe(struct platform_device *pdev)
 	return status;
 
 err_fck:
+	pm_runtime_disable(&pdev->dev);
 	clk_disable_unprepare(spi100k->fck);
 err_ick:
 	clk_disable_unprepare(spi100k->ick);
diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c
index c42e59df38fe..6ba9b0d7710b 100644
--- a/drivers/spi/spi-omap2-mcspi.c
+++ b/drivers/spi/spi-omap2-mcspi.c
@@ -1509,10 +1509,8 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 	}
 
 	status = platform_get_irq(pdev, 0);
-	if (status == -EPROBE_DEFER)
-		goto free_master;
 	if (status < 0) {
-		dev_err(&pdev->dev, "no irq resource found\n");
+		dev_err_probe(&pdev->dev, status, "no irq resource found\n");
 		goto free_master;
 	}
 	init_completion(&mcspi->txdone);
diff --git a/drivers/spi/spi-pxa2xx.c b/drivers/spi/spi-pxa2xx.c
index 838d12e65144..2bf21c2e7a52 100644
--- a/drivers/spi/spi-pxa2xx.c
+++ b/drivers/spi/spi-pxa2xx.c
@@ -1441,31 +1441,6 @@ static const struct of_device_id pxa2xx_spi_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, pxa2xx_spi_of_match);
 
-#ifdef CONFIG_ACPI
-
-static int pxa2xx_spi_get_port_id(struct device *dev)
-{
-	struct acpi_device *adev;
-	unsigned int devid;
-	int port_id = -1;
-
-	adev = ACPI_COMPANION(dev);
-	if (adev && adev->pnp.unique_id &&
-	    !kstrtouint(adev->pnp.unique_id, 0, &devid))
-		port_id = devid;
-	return port_id;
-}
-
-#else /* !CONFIG_ACPI */
-
-static int pxa2xx_spi_get_port_id(struct device *dev)
-{
-	return -1;
-}
-
-#endif /* CONFIG_ACPI */
-
-
 #ifdef CONFIG_PCI
 
 static bool pxa2xx_spi_idma_filter(struct dma_chan *chan, void *param)
@@ -1479,13 +1454,16 @@ static struct pxa2xx_spi_controller *
 pxa2xx_spi_init_pdata(struct platform_device *pdev)
 {
 	struct pxa2xx_spi_controller *pdata;
+	struct device *dev = &pdev->dev;
+	struct device *parent = dev->parent;
 	struct ssp_device *ssp;
 	struct resource *res;
-	struct device *parent = pdev->dev.parent;
 	struct pci_dev *pcidev = dev_is_pci(parent) ? to_pci_dev(parent) : NULL;
 	const struct pci_device_id *pcidev_id = NULL;
 	enum pxa_ssp_type type;
 	const void *match;
+	int status;
+	u64 uid;
 
 	if (pcidev)
 		pcidev_id = pci_match_id(pxa2xx_spi_pci_compound_match, pcidev);
@@ -1529,7 +1507,12 @@ pxa2xx_spi_init_pdata(struct platform_device *pdev)
 
 	ssp->type = type;
 	ssp->dev = &pdev->dev;
-	ssp->port_id = pxa2xx_spi_get_port_id(&pdev->dev);
+
+	status = acpi_dev_uid_to_integer(ACPI_COMPANION(dev), &uid);
+	if (status)
+		ssp->port_id = -1;
+	else
+		ssp->port_id = uid;
 
 	pdata->is_slave = device_property_read_bool(&pdev->dev, "spi-slave");
 	pdata->num_chipselect = 1;
@@ -1873,10 +1856,8 @@ static int pxa2xx_spi_runtime_suspend(struct device *dev)
 static int pxa2xx_spi_runtime_resume(struct device *dev)
 {
 	struct driver_data *drv_data = dev_get_drvdata(dev);
-	int status;
 
-	status = clk_prepare_enable(drv_data->ssp->clk);
-	return status;
+	return clk_prepare_enable(drv_data->ssp->clk);
 }
 #endif
 
diff --git a/drivers/spi/spi-qup.c b/drivers/spi/spi-qup.c
index 00d6084306b4..7d89510dc3f0 100644
--- a/drivers/spi/spi-qup.c
+++ b/drivers/spi/spi-qup.c
@@ -1198,8 +1198,10 @@ static int spi_qup_pm_resume_runtime(struct device *device)
 		return ret;
 
 	ret = clk_prepare_enable(controller->cclk);
-	if (ret)
+	if (ret) {
+		clk_disable_unprepare(controller->iclk);
 		return ret;
+	}
 
 	/* Disable clocks auto gaiting */
 	config = readl_relaxed(controller->base + QUP_CONFIG);
@@ -1245,14 +1247,25 @@ static int spi_qup_resume(struct device *device)
 		return ret;
 
 	ret = clk_prepare_enable(controller->cclk);
-	if (ret)
+	if (ret) {
+		clk_disable_unprepare(controller->iclk);
 		return ret;
+	}
 
 	ret = spi_qup_set_state(controller, QUP_STATE_RESET);
 	if (ret)
-		return ret;
+		goto disable_clk;
 
-	return spi_master_resume(master);
+	ret = spi_master_resume(master);
+	if (ret)
+		goto disable_clk;
+
+	return 0;
+
+disable_clk:
+	clk_disable_unprepare(controller->cclk);
+	clk_disable_unprepare(controller->iclk);
+	return ret;
 }
 #endif /* CONFIG_PM_SLEEP */
 
diff --git a/drivers/spi/spi-s3c24xx.c b/drivers/spi/spi-s3c24xx.c
index 660aa866af06..ef25b5e93900 100644
--- a/drivers/spi/spi-s3c24xx.c
+++ b/drivers/spi/spi-s3c24xx.c
@@ -449,7 +449,7 @@ static int s3c24xx_spi_probe(struct platform_device *pdev)
 	struct spi_master *master;
 	int err = 0;
 
-	master = spi_alloc_master(&pdev->dev, sizeof(struct s3c24xx_spi));
+	master = devm_spi_alloc_master(&pdev->dev, sizeof(struct s3c24xx_spi));
 	if (master == NULL) {
 		dev_err(&pdev->dev, "No memory for spi_master\n");
 		return -ENOMEM;
@@ -463,8 +463,7 @@ static int s3c24xx_spi_probe(struct platform_device *pdev)
 
 	if (pdata == NULL) {
 		dev_err(&pdev->dev, "No platform data supplied\n");
-		err = -ENOENT;
-		goto err_no_pdata;
+		return -ENOENT;
 	}
 
 	platform_set_drvdata(pdev, hw);
@@ -499,29 +498,24 @@ static int s3c24xx_spi_probe(struct platform_device *pdev)
 
 	/* find and map our resources */
 	hw->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(hw->regs)) {
-		err = PTR_ERR(hw->regs);
-		goto err_no_pdata;
-	}
+	if (IS_ERR(hw->regs))
+		return PTR_ERR(hw->regs);
 
 	hw->irq = platform_get_irq(pdev, 0);
-	if (hw->irq < 0) {
-		err = -ENOENT;
-		goto err_no_pdata;
-	}
+	if (hw->irq < 0)
+		return -ENOENT;
 
 	err = devm_request_irq(&pdev->dev, hw->irq, s3c24xx_spi_irq, 0,
 				pdev->name, hw);
 	if (err) {
 		dev_err(&pdev->dev, "Cannot claim IRQ\n");
-		goto err_no_pdata;
+		return err;
 	}
 
 	hw->clk = devm_clk_get(&pdev->dev, "spi");
 	if (IS_ERR(hw->clk)) {
 		dev_err(&pdev->dev, "No clock for device\n");
-		err = PTR_ERR(hw->clk);
-		goto err_no_pdata;
+		return PTR_ERR(hw->clk);
 	}
 
 	s3c24xx_spi_initialsetup(hw);
@@ -539,8 +533,6 @@ static int s3c24xx_spi_probe(struct platform_device *pdev)
  err_register:
 	clk_disable(hw->clk);
 
- err_no_pdata:
-	spi_master_put(hw->master);
 	return err;
 }
 
diff --git a/drivers/spi/spi-s3c64xx.c b/drivers/spi/spi-s3c64xx.c
index 7f346866614a..71d324ec9a70 100644
--- a/drivers/spi/spi-s3c64xx.c
+++ b/drivers/spi/spi-s3c64xx.c
@@ -84,6 +84,7 @@
 #define S3C64XX_SPI_ST_TX_FIFORDY		(1<<0)
 
 #define S3C64XX_SPI_PACKET_CNT_EN		(1<<16)
+#define S3C64XX_SPI_PACKET_CNT_MASK		GENMASK(15, 0)
 
 #define S3C64XX_SPI_PND_TX_UNDERRUN_CLR		(1<<4)
 #define S3C64XX_SPI_PND_TX_OVERRUN_CLR		(1<<3)
@@ -389,8 +390,8 @@ static int s3c64xx_spi_unprepare_transfer(struct spi_master *spi)
 	if (sdd->rx_dma.ch && sdd->tx_dma.ch) {
 		dma_release_channel(sdd->rx_dma.ch);
 		dma_release_channel(sdd->tx_dma.ch);
-		sdd->rx_dma.ch = 0;
-		sdd->tx_dma.ch = 0;
+		sdd->rx_dma.ch = NULL;
+		sdd->tx_dma.ch = NULL;
 	}
 
 	return 0;
@@ -711,6 +712,13 @@ static int s3c64xx_spi_prepare_message(struct spi_master *master,
 	return 0;
 }
 
+static size_t s3c64xx_spi_max_transfer_size(struct spi_device *spi)
+{
+	struct spi_controller *ctlr = spi->controller;
+
+	return ctlr->can_dma ? S3C64XX_SPI_PACKET_CNT_MASK : SIZE_MAX;
+}
+
 static int s3c64xx_spi_transfer_one(struct spi_master *master,
 				    struct spi_device *spi,
 				    struct spi_transfer *xfer)
@@ -1152,6 +1160,7 @@ static int s3c64xx_spi_probe(struct platform_device *pdev)
 	master->unprepare_transfer_hardware = s3c64xx_spi_unprepare_transfer;
 	master->prepare_message = s3c64xx_spi_prepare_message;
 	master->transfer_one = s3c64xx_spi_transfer_one;
+	master->max_transfer_size = s3c64xx_spi_max_transfer_size;
 	master->num_chipselect = sci->num_cs;
 	master->use_gpio_descriptors = true;
 	master->dma_alignment = 8;
diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c
index d0012b30410c..9bca3d076f05 100644
--- a/drivers/spi/spi-sh-msiof.c
+++ b/drivers/spi/spi-sh-msiof.c
@@ -1085,6 +1085,7 @@ static const struct of_device_id sh_msiof_match[] = {
 	{ .compatible = "renesas,rcar-gen2-msiof", .data = &rcar_gen2_data },
 	{ .compatible = "renesas,msiof-r8a7796",   .data = &rcar_gen3_data },
 	{ .compatible = "renesas,rcar-gen3-msiof", .data = &rcar_gen3_data },
+	{ .compatible = "renesas,rcar-gen4-msiof", .data = &rcar_gen3_data },
 	{ .compatible = "renesas,sh-msiof",        .data = &sh_data }, /* Deprecated */
 	{},
 };
diff --git a/drivers/spi/spi-stm32-qspi.c b/drivers/spi/spi-stm32-qspi.c
index f3fe92300639..9131660c1afb 100644
--- a/drivers/spi/spi-stm32-qspi.c
+++ b/drivers/spi/spi-stm32-qspi.c
@@ -15,6 +15,7 @@
 #include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/of_device.h>
+#include <linux/of_gpio.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/pm_runtime.h>
 #include <linux/platform_device.h>
@@ -355,10 +356,10 @@ static int stm32_qspi_get_mode(u8 buswidth)
 	return buswidth;
 }
 
-static int stm32_qspi_send(struct spi_mem *mem, const struct spi_mem_op *op)
+static int stm32_qspi_send(struct spi_device *spi, const struct spi_mem_op *op)
 {
-	struct stm32_qspi *qspi = spi_controller_get_devdata(mem->spi->master);
-	struct stm32_qspi_flash *flash = &qspi->flash[mem->spi->chip_select];
+	struct stm32_qspi *qspi = spi_controller_get_devdata(spi->master);
+	struct stm32_qspi_flash *flash = &qspi->flash[spi->chip_select];
 	u32 ccr, cr;
 	int timeout, err = 0, err_poll_status = 0;
 
@@ -465,7 +466,7 @@ static int stm32_qspi_poll_status(struct spi_mem *mem, const struct spi_mem_op *
 	qspi->fmode = CCR_FMODE_APM;
 	qspi->status_timeout = timeout_ms;
 
-	ret = stm32_qspi_send(mem, op);
+	ret = stm32_qspi_send(mem->spi, op);
 	mutex_unlock(&qspi->lock);
 
 	pm_runtime_mark_last_busy(qspi->dev);
@@ -489,7 +490,7 @@ static int stm32_qspi_exec_op(struct spi_mem *mem, const struct spi_mem_op *op)
 	else
 		qspi->fmode = CCR_FMODE_INDW;
 
-	ret = stm32_qspi_send(mem, op);
+	ret = stm32_qspi_send(mem->spi, op);
 	mutex_unlock(&qspi->lock);
 
 	pm_runtime_mark_last_busy(qspi->dev);
@@ -545,7 +546,7 @@ static ssize_t stm32_qspi_dirmap_read(struct spi_mem_dirmap_desc *desc,
 	else
 		qspi->fmode = CCR_FMODE_INDR;
 
-	ret = stm32_qspi_send(desc->mem, &op);
+	ret = stm32_qspi_send(desc->mem->spi, &op);
 	mutex_unlock(&qspi->lock);
 
 	pm_runtime_mark_last_busy(qspi->dev);
@@ -554,12 +555,96 @@ static ssize_t stm32_qspi_dirmap_read(struct spi_mem_dirmap_desc *desc,
 	return ret ?: len;
 }
 
+static int stm32_qspi_transfer_one_message(struct spi_controller *ctrl,
+					   struct spi_message *msg)
+{
+	struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl);
+	struct spi_transfer *transfer;
+	struct spi_device *spi = msg->spi;
+	struct spi_mem_op op;
+	int ret = 0;
+
+	if (!spi->cs_gpiod)
+		return -EOPNOTSUPP;
+
+	ret = pm_runtime_resume_and_get(qspi->dev);
+	if (ret < 0)
+		return ret;
+
+	mutex_lock(&qspi->lock);
+
+	gpiod_set_value_cansleep(spi->cs_gpiod, true);
+
+	list_for_each_entry(transfer, &msg->transfers, transfer_list) {
+		u8 dummy_bytes = 0;
+
+		memset(&op, 0, sizeof(op));
+
+		dev_dbg(qspi->dev, "tx_buf:%p tx_nbits:%d rx_buf:%p rx_nbits:%d len:%d dummy_data:%d\n",
+			transfer->tx_buf, transfer->tx_nbits,
+			transfer->rx_buf, transfer->rx_nbits,
+			transfer->len, transfer->dummy_data);
+
+		/*
+		 * QSPI hardware supports dummy bytes transfer.
+		 * If current transfer is dummy byte, merge it with the next
+		 * transfer in order to take into account QSPI block constraint
+		 */
+		if (transfer->dummy_data) {
+			op.dummy.buswidth = transfer->tx_nbits;
+			op.dummy.nbytes = transfer->len;
+			dummy_bytes = transfer->len;
+
+			/* if happens, means that message is not correctly built */
+			if (list_is_last(&transfer->transfer_list, &msg->transfers)) {
+				ret = -EINVAL;
+				goto end_of_transfer;
+			}
+
+			transfer = list_next_entry(transfer, transfer_list);
+		}
+
+		op.data.nbytes = transfer->len;
+
+		if (transfer->rx_buf) {
+			qspi->fmode = CCR_FMODE_INDR;
+			op.data.buswidth = transfer->rx_nbits;
+			op.data.dir = SPI_MEM_DATA_IN;
+			op.data.buf.in = transfer->rx_buf;
+		} else {
+			qspi->fmode = CCR_FMODE_INDW;
+			op.data.buswidth = transfer->tx_nbits;
+			op.data.dir = SPI_MEM_DATA_OUT;
+			op.data.buf.out = transfer->tx_buf;
+		}
+
+		ret = stm32_qspi_send(spi, &op);
+		if (ret)
+			goto end_of_transfer;
+
+		msg->actual_length += transfer->len + dummy_bytes;
+	}
+
+end_of_transfer:
+	gpiod_set_value_cansleep(spi->cs_gpiod, false);
+
+	mutex_unlock(&qspi->lock);
+
+	msg->status = ret;
+	spi_finalize_current_message(ctrl);
+
+	pm_runtime_mark_last_busy(qspi->dev);
+	pm_runtime_put_autosuspend(qspi->dev);
+
+	return ret;
+}
+
 static int stm32_qspi_setup(struct spi_device *spi)
 {
 	struct spi_controller *ctrl = spi->master;
 	struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl);
 	struct stm32_qspi_flash *flash;
-	u32 presc;
+	u32 presc, mode;
 	int ret;
 
 	if (ctrl->busy)
@@ -568,6 +653,16 @@ static int stm32_qspi_setup(struct spi_device *spi)
 	if (!spi->max_speed_hz)
 		return -EINVAL;
 
+	mode = spi->mode & (SPI_TX_OCTAL | SPI_RX_OCTAL);
+	if ((mode == SPI_TX_OCTAL || mode == SPI_RX_OCTAL) ||
+	    ((mode == (SPI_TX_OCTAL | SPI_RX_OCTAL)) &&
+	    gpiod_count(qspi->dev, "cs") == -ENOENT)) {
+		dev_err(qspi->dev, "spi-rx-bus-width\\/spi-tx-bus-width\\/cs-gpios\n");
+		dev_err(qspi->dev, "configuration not supported\n");
+
+		return -EINVAL;
+	}
+
 	ret = pm_runtime_resume_and_get(qspi->dev);
 	if (ret < 0)
 		return ret;
@@ -580,6 +675,16 @@ static int stm32_qspi_setup(struct spi_device *spi)
 
 	mutex_lock(&qspi->lock);
 	qspi->cr_reg = CR_APMS | 3 << CR_FTHRES_SHIFT | CR_SSHIFT | CR_EN;
+
+	/*
+	 * Dual flash mode is only enable in case SPI_TX_OCTAL and SPI_TX_OCTAL
+	 * are both set in spi->mode and "cs-gpios" properties is found in DT
+	 */
+	if (mode == (SPI_TX_OCTAL | SPI_RX_OCTAL)) {
+		qspi->cr_reg |= CR_DFM;
+		dev_dbg(qspi->dev, "Dual flash mode enable");
+	}
+
 	writel_relaxed(qspi->cr_reg, qspi->io_base + QSPI_CR);
 
 	/* set dcr fsize to max address */
@@ -741,11 +846,13 @@ static int stm32_qspi_probe(struct platform_device *pdev)
 
 	mutex_init(&qspi->lock);
 
-	ctrl->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD
-		| SPI_TX_DUAL | SPI_TX_QUAD;
+	ctrl->mode_bits = SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_OCTAL
+		| SPI_TX_DUAL | SPI_TX_QUAD | SPI_RX_OCTAL;
 	ctrl->setup = stm32_qspi_setup;
 	ctrl->bus_num = -1;
 	ctrl->mem_ops = &stm32_qspi_mem_ops;
+	ctrl->use_gpio_descriptors = true;
+	ctrl->transfer_one_message = stm32_qspi_transfer_one_message;
 	ctrl->num_chipselect = STM32_QSPI_MAX_NORCHIP;
 	ctrl->dev.of_node = dev->of_node;
 
diff --git a/drivers/spi/spi-xilinx.c b/drivers/spi/spi-xilinx.c
index 523edfdf5dcd..7377d3b81302 100644
--- a/drivers/spi/spi-xilinx.c
+++ b/drivers/spi/spi-xilinx.c
@@ -421,7 +421,7 @@ static int xilinx_spi_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	master = spi_alloc_master(&pdev->dev, sizeof(struct xilinx_spi));
+	master = devm_spi_alloc_master(&pdev->dev, sizeof(struct xilinx_spi));
 	if (!master)
 		return -ENODEV;
 
@@ -439,10 +439,8 @@ static int xilinx_spi_probe(struct platform_device *pdev)
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	xspi->regs = devm_ioremap_resource(&pdev->dev, res);
-	if (IS_ERR(xspi->regs)) {
-		ret = PTR_ERR(xspi->regs);
-		goto put_master;
-	}
+	if (IS_ERR(xspi->regs))
+		return PTR_ERR(xspi->regs);
 
 	master->bus_num = pdev->id;
 	master->num_chipselect = num_cs;
@@ -472,14 +470,13 @@ static int xilinx_spi_probe(struct platform_device *pdev)
 
 	xspi->irq = platform_get_irq(pdev, 0);
 	if (xspi->irq < 0 && xspi->irq != -ENXIO) {
-		ret = xspi->irq;
-		goto put_master;
+		return xspi->irq;
 	} else if (xspi->irq >= 0) {
 		/* Register for SPI Interrupt */
 		ret = devm_request_irq(&pdev->dev, xspi->irq, xilinx_spi_irq, 0,
 				dev_name(&pdev->dev), xspi);
 		if (ret)
-			goto put_master;
+			return ret;
 	}
 
 	/* SPI controller initializations */
@@ -488,7 +485,7 @@ static int xilinx_spi_probe(struct platform_device *pdev)
 	ret = spi_bitbang_start(&xspi->bitbang);
 	if (ret) {
 		dev_err(&pdev->dev, "spi_bitbang_start FAILED\n");
-		goto put_master;
+		return ret;
 	}
 
 	dev_info(&pdev->dev, "at %pR, irq=%d\n", res, xspi->irq);
@@ -500,11 +497,6 @@ static int xilinx_spi_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, master);
 	return 0;
-
-put_master:
-	spi_master_put(master);
-
-	return ret;
 }
 
 static int xilinx_spi_remove(struct platform_device *pdev)
diff --git a/drivers/spi/spi-xtensa-xtfpga.c b/drivers/spi/spi-xtensa-xtfpga.c
index fc2b5eb7d614..2fa7608f94cd 100644
--- a/drivers/spi/spi-xtensa-xtfpga.c
+++ b/drivers/spi/spi-xtensa-xtfpga.c
@@ -83,7 +83,7 @@ static int xtfpga_spi_probe(struct platform_device *pdev)
 	int ret;
 	struct spi_master *master;
 
-	master = spi_alloc_master(&pdev->dev, sizeof(struct xtfpga_spi));
+	master = devm_spi_alloc_master(&pdev->dev, sizeof(struct xtfpga_spi));
 	if (!master)
 		return -ENOMEM;
 
@@ -97,30 +97,24 @@ static int xtfpga_spi_probe(struct platform_device *pdev)
 	xspi->bitbang.chipselect = xtfpga_spi_chipselect;
 	xspi->bitbang.txrx_word[SPI_MODE_0] = xtfpga_spi_txrx_word;
 	xspi->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(xspi->regs)) {
-		ret = PTR_ERR(xspi->regs);
-		goto err;
-	}
+	if (IS_ERR(xspi->regs))
+		return PTR_ERR(xspi->regs);
 
 	xtfpga_spi_write32(xspi, XTFPGA_SPI_START, 0);
 	usleep_range(1000, 2000);
 	if (xtfpga_spi_read32(xspi, XTFPGA_SPI_BUSY)) {
 		dev_err(&pdev->dev, "Device stuck in busy state\n");
-		ret = -EBUSY;
-		goto err;
+		return -EBUSY;
 	}
 
 	ret = spi_bitbang_start(&xspi->bitbang);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "spi_bitbang_start failed\n");
-		goto err;
+		return ret;
 	}
 
 	platform_set_drvdata(pdev, master);
 	return 0;
-err:
-	spi_master_put(master);
-	return ret;
 }
 
 static int xtfpga_spi_remove(struct platform_device *pdev)
diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c
index 32c01e684af3..5f9aedd1f0b6 100644
--- a/drivers/spi/spi.c
+++ b/drivers/spi/spi.c
@@ -753,7 +753,7 @@ struct spi_device *spi_new_device(struct spi_controller *ctlr,
 	proxy->max_speed_hz = chip->max_speed_hz;
 	proxy->mode = chip->mode;
 	proxy->irq = chip->irq;
-	strlcpy(proxy->modalias, chip->modalias, sizeof(proxy->modalias));
+	strscpy(proxy->modalias, chip->modalias, sizeof(proxy->modalias));
 	proxy->dev.platform_data = (void *) chip->platform_data;
 	proxy->controller_data = chip->controller_data;
 	proxy->controller_state = NULL;
@@ -1010,9 +1010,9 @@ static void spi_set_cs(struct spi_device *spi, bool enable, bool force)
 }
 
 #ifdef CONFIG_HAS_DMA
-int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
-		struct sg_table *sgt, void *buf, size_t len,
-		enum dma_data_direction dir)
+static int spi_map_buf_attrs(struct spi_controller *ctlr, struct device *dev,
+			     struct sg_table *sgt, void *buf, size_t len,
+			     enum dma_data_direction dir, unsigned long attrs)
 {
 	const bool vmalloced_buf = is_vmalloc_addr(buf);
 	unsigned int max_seg_size = dma_get_max_seg_size(dev);
@@ -1078,28 +1078,41 @@ int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
 		sg = sg_next(sg);
 	}
 
-	ret = dma_map_sg(dev, sgt->sgl, sgt->nents, dir);
-	if (!ret)
-		ret = -ENOMEM;
+	ret = dma_map_sgtable(dev, sgt, dir, attrs);
 	if (ret < 0) {
 		sg_free_table(sgt);
 		return ret;
 	}
 
-	sgt->nents = ret;
-
 	return 0;
 }
 
-void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev,
-		   struct sg_table *sgt, enum dma_data_direction dir)
+int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
+		struct sg_table *sgt, void *buf, size_t len,
+		enum dma_data_direction dir)
+{
+	return spi_map_buf_attrs(ctlr, dev, sgt, buf, len, dir, 0);
+}
+
+static void spi_unmap_buf_attrs(struct spi_controller *ctlr,
+				struct device *dev, struct sg_table *sgt,
+				enum dma_data_direction dir,
+				unsigned long attrs)
 {
 	if (sgt->orig_nents) {
-		dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir);
+		dma_unmap_sgtable(dev, sgt, dir, attrs);
 		sg_free_table(sgt);
+		sgt->orig_nents = 0;
+		sgt->nents = 0;
 	}
 }
 
+void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev,
+		   struct sg_table *sgt, enum dma_data_direction dir)
+{
+	spi_unmap_buf_attrs(ctlr, dev, sgt, dir, 0);
+}
+
 static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg)
 {
 	struct device *tx_dev, *rx_dev;
@@ -1124,29 +1137,37 @@ static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg)
 		rx_dev = ctlr->dev.parent;
 
 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
+		/* The sync is done before each transfer. */
+		unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC;
+
 		if (!ctlr->can_dma(ctlr, msg->spi, xfer))
 			continue;
 
 		if (xfer->tx_buf != NULL) {
-			ret = spi_map_buf(ctlr, tx_dev, &xfer->tx_sg,
-					  (void *)xfer->tx_buf, xfer->len,
-					  DMA_TO_DEVICE);
+			ret = spi_map_buf_attrs(ctlr, tx_dev, &xfer->tx_sg,
+						(void *)xfer->tx_buf,
+						xfer->len, DMA_TO_DEVICE,
+						attrs);
 			if (ret != 0)
 				return ret;
 		}
 
 		if (xfer->rx_buf != NULL) {
-			ret = spi_map_buf(ctlr, rx_dev, &xfer->rx_sg,
-					  xfer->rx_buf, xfer->len,
-					  DMA_FROM_DEVICE);
+			ret = spi_map_buf_attrs(ctlr, rx_dev, &xfer->rx_sg,
+						xfer->rx_buf, xfer->len,
+						DMA_FROM_DEVICE, attrs);
 			if (ret != 0) {
-				spi_unmap_buf(ctlr, tx_dev, &xfer->tx_sg,
-					      DMA_TO_DEVICE);
+				spi_unmap_buf_attrs(ctlr, tx_dev,
+						&xfer->tx_sg, DMA_TO_DEVICE,
+						attrs);
+
 				return ret;
 			}
 		}
 	}
 
+	ctlr->cur_rx_dma_dev = rx_dev;
+	ctlr->cur_tx_dma_dev = tx_dev;
 	ctlr->cur_msg_mapped = true;
 
 	return 0;
@@ -1154,38 +1175,60 @@ static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg)
 
 static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg)
 {
+	struct device *rx_dev = ctlr->cur_rx_dma_dev;
+	struct device *tx_dev = ctlr->cur_tx_dma_dev;
 	struct spi_transfer *xfer;
-	struct device *tx_dev, *rx_dev;
 
 	if (!ctlr->cur_msg_mapped || !ctlr->can_dma)
 		return 0;
 
-	if (ctlr->dma_tx)
-		tx_dev = ctlr->dma_tx->device->dev;
-	else if (ctlr->dma_map_dev)
-		tx_dev = ctlr->dma_map_dev;
-	else
-		tx_dev = ctlr->dev.parent;
-
-	if (ctlr->dma_rx)
-		rx_dev = ctlr->dma_rx->device->dev;
-	else if (ctlr->dma_map_dev)
-		rx_dev = ctlr->dma_map_dev;
-	else
-		rx_dev = ctlr->dev.parent;
-
 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {
+		/* The sync has already been done after each transfer. */
+		unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC;
+
 		if (!ctlr->can_dma(ctlr, msg->spi, xfer))
 			continue;
 
-		spi_unmap_buf(ctlr, rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);
-		spi_unmap_buf(ctlr, tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);
+		spi_unmap_buf_attrs(ctlr, rx_dev, &xfer->rx_sg,
+				    DMA_FROM_DEVICE, attrs);
+		spi_unmap_buf_attrs(ctlr, tx_dev, &xfer->tx_sg,
+				    DMA_TO_DEVICE, attrs);
 	}
 
 	ctlr->cur_msg_mapped = false;
 
 	return 0;
 }
+
+static void spi_dma_sync_for_device(struct spi_controller *ctlr,
+				    struct spi_transfer *xfer)
+{
+	struct device *rx_dev = ctlr->cur_rx_dma_dev;
+	struct device *tx_dev = ctlr->cur_tx_dma_dev;
+
+	if (!ctlr->cur_msg_mapped)
+		return;
+
+	if (xfer->tx_sg.orig_nents)
+		dma_sync_sgtable_for_device(tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);
+	if (xfer->rx_sg.orig_nents)
+		dma_sync_sgtable_for_device(rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);
+}
+
+static void spi_dma_sync_for_cpu(struct spi_controller *ctlr,
+				 struct spi_transfer *xfer)
+{
+	struct device *rx_dev = ctlr->cur_rx_dma_dev;
+	struct device *tx_dev = ctlr->cur_tx_dma_dev;
+
+	if (!ctlr->cur_msg_mapped)
+		return;
+
+	if (xfer->rx_sg.orig_nents)
+		dma_sync_sgtable_for_cpu(rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);
+	if (xfer->tx_sg.orig_nents)
+		dma_sync_sgtable_for_cpu(tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);
+}
 #else /* !CONFIG_HAS_DMA */
 static inline int __spi_map_msg(struct spi_controller *ctlr,
 				struct spi_message *msg)
@@ -1198,6 +1241,16 @@ static inline int __spi_unmap_msg(struct spi_controller *ctlr,
 {
 	return 0;
 }
+
+static void spi_dma_sync_for_device(struct spi_controller *ctrl,
+				    struct spi_transfer *xfer)
+{
+}
+
+static void spi_dma_sync_for_cpu(struct spi_controller *ctrl,
+				 struct spi_transfer *xfer)
+{
+}
 #endif /* !CONFIG_HAS_DMA */
 
 static inline int spi_unmap_msg(struct spi_controller *ctlr,
@@ -1435,7 +1488,8 @@ static int spi_transfer_one_message(struct spi_controller *ctlr,
 	struct spi_statistics __percpu *statm = ctlr->pcpu_statistics;
 	struct spi_statistics __percpu *stats = msg->spi->pcpu_statistics;
 
-	spi_set_cs(msg->spi, true, false);
+	xfer = list_first_entry(&msg->transfers, struct spi_transfer, transfer_list);
+	spi_set_cs(msg->spi, !xfer->cs_off, false);
 
 	SPI_STATISTICS_INCREMENT_FIELD(statm, messages);
 	SPI_STATISTICS_INCREMENT_FIELD(stats, messages);
@@ -1455,8 +1509,11 @@ static int spi_transfer_one_message(struct spi_controller *ctlr,
 			reinit_completion(&ctlr->xfer_completion);
 
 fallback_pio:
+			spi_dma_sync_for_device(ctlr, xfer);
 			ret = ctlr->transfer_one(ctlr, msg->spi, xfer);
 			if (ret < 0) {
+				spi_dma_sync_for_cpu(ctlr, xfer);
+
 				if (ctlr->cur_msg_mapped &&
 				   (xfer->error & SPI_TRANS_FAIL_NO_START)) {
 					__spi_unmap_msg(ctlr, msg);
@@ -1479,6 +1536,8 @@ fallback_pio:
 				if (ret < 0)
 					msg->status = ret;
 			}
+
+			spi_dma_sync_for_cpu(ctlr, xfer);
 		} else {
 			if (xfer->len)
 				dev_err(&msg->spi->dev,
@@ -1503,10 +1562,15 @@ fallback_pio:
 					 &msg->transfers)) {
 				keep_cs = true;
 			} else {
-				spi_set_cs(msg->spi, false, false);
+				if (!xfer->cs_off)
+					spi_set_cs(msg->spi, false, false);
 				_spi_transfer_cs_change_delay(msg, xfer);
-				spi_set_cs(msg->spi, true, false);
+				if (!list_next_entry(xfer, transfer_list)->cs_off)
+					spi_set_cs(msg->spi, true, false);
 			}
+		} else if (!list_is_last(&xfer->transfer_list, &msg->transfers) &&
+			   xfer->cs_off != list_next_entry(xfer, transfer_list)->cs_off) {
+			spi_set_cs(msg->spi, xfer->cs_off, false);
 		}
 
 		msg->actual_length += xfer->len;
@@ -1587,6 +1651,15 @@ static int __spi_pump_transfer_message(struct spi_controller *ctlr,
 
 	trace_spi_message_start(msg);
 
+	ret = spi_split_transfers_maxsize(ctlr, msg,
+					  spi_max_transfer_size(msg->spi),
+					  GFP_KERNEL | GFP_DMA);
+	if (ret) {
+		msg->status = ret;
+		spi_finalize_current_message(ctlr);
+		return ret;
+	}
+
 	if (ctlr->prepare_message) {
 		ret = ctlr->prepare_message(ctlr, msg);
 		if (ret) {
@@ -2329,7 +2402,7 @@ struct spi_device *spi_new_ancillary_device(struct spi_device *spi,
 		goto err_out;
 	}
 
-	strlcpy(ancillary->modalias, "dummy", sizeof(ancillary->modalias));
+	strscpy(ancillary->modalias, "dummy", sizeof(ancillary->modalias));
 
 	/* Use provided chip-select for ancillary device */
 	ancillary->chip_select = chip_select;
@@ -2725,7 +2798,7 @@ static ssize_t slave_store(struct device *dev, struct device_attribute *attr,
 		if (!spi)
 			return -ENOMEM;
 
-		strlcpy(spi->modalias, name, sizeof(spi->modalias));
+		strscpy(spi->modalias, name, sizeof(spi->modalias));
 
 		rc = spi_add_device(spi);
 		if (rc) {
@@ -4374,7 +4447,7 @@ static int acpi_spi_notify(struct notifier_block *nb, unsigned long value,
 
 	switch (value) {
 	case ACPI_RECONFIG_DEVICE_ADD:
-		ctlr = acpi_spi_find_controller_by_adev(adev->parent);
+		ctlr = acpi_spi_find_controller_by_adev(acpi_dev_parent(adev));
 		if (!ctlr)
 			break;
 
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
index cbc8b1d91995..783f1b88ebf2 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c
@@ -1194,7 +1194,7 @@ static const struct v4l2_subdev_ops gc0310_ops = {
 	.sensor = &gc0310_sensor_ops,
 };
 
-static int gc0310_remove(struct i2c_client *client)
+static void gc0310_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct gc0310_device *dev = to_gc0310_sensor(sd);
@@ -1207,8 +1207,6 @@ static int gc0310_remove(struct i2c_client *client)
 	media_entity_cleanup(&dev->sd.entity);
 	v4l2_ctrl_handler_free(&dev->ctrl_handler);
 	kfree(dev);
-
-	return 0;
 }
 
 static int gc0310_probe(struct i2c_client *client)
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
index 0e6b2e6100d1..4d5a7e335f85 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
@@ -952,7 +952,7 @@ static const struct v4l2_subdev_ops gc2235_ops = {
 	.sensor = &gc2235_sensor_ops,
 };
 
-static int gc2235_remove(struct i2c_client *client)
+static void gc2235_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct gc2235_device *dev = to_gc2235_sensor(sd);
@@ -965,8 +965,6 @@ static int gc2235_remove(struct i2c_client *client)
 	media_entity_cleanup(&dev->sd.entity);
 	v4l2_ctrl_handler_free(&dev->ctrl_handler);
 	kfree(dev);
-
-	return 0;
 }
 
 static int gc2235_probe(struct i2c_client *client)
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
index e046489cd253..75d16b525294 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c
@@ -910,7 +910,7 @@ free_flash:
 	return err;
 }
 
-static int lm3554_remove(struct i2c_client *client)
+static void lm3554_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct lm3554 *flash = to_lm3554(sd);
@@ -926,8 +926,6 @@ static int lm3554_remove(struct i2c_client *client)
 	lm3554_gpio_uninit(client);
 
 	kfree(flash);
-
-	return 0;
 }
 
 static const struct dev_pm_ops lm3554_pm_ops = {
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c
index 3c81ab73cdae..a0e8e94b2412 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c
@@ -1713,7 +1713,7 @@ static const struct v4l2_subdev_ops mt9m114_ops = {
 	.sensor = &mt9m114_sensor_ops,
 };
 
-static int mt9m114_remove(struct i2c_client *client)
+static void mt9m114_remove(struct i2c_client *client)
 {
 	struct mt9m114_device *dev;
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
@@ -1724,7 +1724,6 @@ static int mt9m114_remove(struct i2c_client *client)
 	media_entity_cleanup(&dev->sd.entity);
 	v4l2_ctrl_handler_free(&dev->ctrl_handler);
 	kfree(dev);
-	return 0;
 }
 
 static int mt9m114_probe(struct i2c_client *client)
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
index 4ba99c660681..8f48b23be3aa 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c
@@ -1135,7 +1135,7 @@ static const struct v4l2_subdev_ops ov2680_ops = {
 	.sensor = &ov2680_sensor_ops,
 };
 
-static int ov2680_remove(struct i2c_client *client)
+static void ov2680_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov2680_device *dev = to_ov2680_sensor(sd);
@@ -1148,8 +1148,6 @@ static int ov2680_remove(struct i2c_client *client)
 	media_entity_cleanup(&dev->sd.entity);
 	v4l2_ctrl_handler_free(&dev->ctrl_handler);
 	kfree(dev);
-
-	return 0;
 }
 
 static int ov2680_probe(struct i2c_client *client)
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
index d5d099ac1b70..887b6f99f6ca 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c
@@ -1090,7 +1090,7 @@ static const struct v4l2_subdev_ops ov2722_ops = {
 	.sensor = &ov2722_sensor_ops,
 };
 
-static int ov2722_remove(struct i2c_client *client)
+static void ov2722_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov2722_device *dev = to_ov2722_sensor(sd);
@@ -1103,8 +1103,6 @@ static int ov2722_remove(struct i2c_client *client)
 
 	media_entity_cleanup(&dev->sd.entity);
 	kfree(dev);
-
-	return 0;
 }
 
 static int __ov2722_init_ctrl_handler(struct ov2722_device *dev)
diff --git a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c
index 6c95f57a52e9..c1cd631455e6 100644
--- a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c
+++ b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c
@@ -1877,7 +1877,7 @@ static const struct v4l2_subdev_ops ov5693_ops = {
 	.pad = &ov5693_pad_ops,
 };
 
-static int ov5693_remove(struct i2c_client *client)
+static void ov5693_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
 	struct ov5693_device *dev = to_ov5693_sensor(sd);
@@ -1893,8 +1893,6 @@ static int ov5693_remove(struct i2c_client *client)
 	media_entity_cleanup(&dev->sd.entity);
 	v4l2_ctrl_handler_free(&dev->ctrl_handler);
 	kfree(dev);
-
-	return 0;
 }
 
 static int ov5693_probe(struct i2c_client *client)
diff --git a/drivers/staging/media/max96712/max96712.c b/drivers/staging/media/max96712/max96712.c
index 6b5abd958bff..99b333b68198 100644
--- a/drivers/staging/media/max96712/max96712.c
+++ b/drivers/staging/media/max96712/max96712.c
@@ -407,15 +407,13 @@ static int max96712_probe(struct i2c_client *client)
 	return max96712_v4l2_register(priv);
 }
 
-static int max96712_remove(struct i2c_client *client)
+static void max96712_remove(struct i2c_client *client)
 {
 	struct max96712_priv *priv = i2c_get_clientdata(client);
 
 	v4l2_async_unregister_subdev(&priv->sd);
 
 	gpiod_set_value_cansleep(priv->gpiod_pwdn, 0);
-
-	return 0;
 }
 
 static const struct of_device_id max96712_of_table[] = {
diff --git a/drivers/staging/media/rkvdec/rkvdec-h264.c b/drivers/staging/media/rkvdec/rkvdec-h264.c
index 4af5a831bde0..4fc167b42cf0 100644
--- a/drivers/staging/media/rkvdec/rkvdec-h264.c
+++ b/drivers/staging/media/rkvdec/rkvdec-h264.c
@@ -1162,8 +1162,8 @@ static int rkvdec_h264_run(struct rkvdec_ctx *ctx)
 
 	schedule_delayed_work(&rkvdec->watchdog_work, msecs_to_jiffies(2000));
 
-	writel(0xffffffff, rkvdec->regs + RKVDEC_REG_STRMD_ERR_EN);
-	writel(0xffffffff, rkvdec->regs + RKVDEC_REG_H264_ERR_E);
+	writel(0, rkvdec->regs + RKVDEC_REG_STRMD_ERR_EN);
+	writel(0, rkvdec->regs + RKVDEC_REG_H264_ERR_E);
 	writel(1, rkvdec->regs + RKVDEC_REG_PREF_LUMA_CACHE_COMMAND);
 	writel(1, rkvdec->regs + RKVDEC_REG_PREF_CHR_CACHE_COMMAND);
 
diff --git a/drivers/staging/most/i2c/i2c.c b/drivers/staging/most/i2c/i2c.c
index 7042f10887bb..285a071f02be 100644
--- a/drivers/staging/most/i2c/i2c.c
+++ b/drivers/staging/most/i2c/i2c.c
@@ -340,14 +340,12 @@ static int i2c_probe(struct i2c_client *client, const struct i2c_device_id *id)
  *
  * Unregister the i2c client device as a MOST interface
  */
-static int i2c_remove(struct i2c_client *client)
+static void i2c_remove(struct i2c_client *client)
 {
 	struct hdm_i2c *dev = i2c_get_clientdata(client);
 
 	most_deregister_interface(&dev->most_iface);
 	kfree(dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id i2c_id[] = {
diff --git a/drivers/staging/olpc_dcon/olpc_dcon.c b/drivers/staging/olpc_dcon/olpc_dcon.c
index 9363c5cfe50f..4fb9b9f10799 100644
--- a/drivers/staging/olpc_dcon/olpc_dcon.c
+++ b/drivers/staging/olpc_dcon/olpc_dcon.c
@@ -668,7 +668,7 @@ static int dcon_probe(struct i2c_client *client, const struct i2c_device_id *id)
 	return rc;
 }
 
-static int dcon_remove(struct i2c_client *client)
+static void dcon_remove(struct i2c_client *client)
 {
 	struct dcon_priv *dcon = i2c_get_clientdata(client);
 
@@ -684,8 +684,6 @@ static int dcon_remove(struct i2c_client *client)
 	cancel_work_sync(&dcon->switch_source);
 
 	kfree(dcon);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/staging/qlge/qlge_main.c b/drivers/staging/qlge/qlge_main.c
index ca6b966f5dd3..1ead7793062a 100644
--- a/drivers/staging/qlge/qlge_main.c
+++ b/drivers/staging/qlge/qlge_main.c
@@ -3041,8 +3041,8 @@ static int qlge_start_rx_ring(struct qlge_adapter *qdev, struct rx_ring *rx_ring
 		/* Inbound completion handling rx_rings run in
 		 * separate NAPI contexts.
 		 */
-		netif_napi_add_weight(qdev->ndev, &rx_ring->napi,
-				      qlge_napi_poll_msix, 64);
+		netif_napi_add(qdev->ndev, &rx_ring->napi,
+			       qlge_napi_poll_msix);
 		cqicb->irq_delay = cpu_to_le16(qdev->rx_coalesce_usecs);
 		cqicb->pkt_delay = cpu_to_le16(qdev->rx_max_coalesced_frames);
 	} else {
diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
index cb6d287f580d..2d09be6425d5 100644
--- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
+++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
@@ -850,8 +850,8 @@ exit:
 }
 
 static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct net_device *ndev,
-				u8 key_index, bool pairwise, const u8 *mac_addr,
-				struct key_params *params)
+				int link_id, u8 key_index, bool pairwise,
+				const u8 *mac_addr, struct key_params *params)
 {
 	char *alg_name;
 	u32 param_len;
@@ -932,8 +932,8 @@ addkey_end:
 }
 
 static int cfg80211_rtw_get_key(struct wiphy *wiphy, struct net_device *ndev,
-				u8 key_index, bool pairwise, const u8 *mac_addr,
-				void *cookie,
+				int link_id, u8 key_index, bool pairwise,
+				const u8 *mac_addr, void *cookie,
 				void (*callback)(void *cookie,
 						 struct key_params*))
 {
@@ -941,7 +941,8 @@ static int cfg80211_rtw_get_key(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static int cfg80211_rtw_del_key(struct wiphy *wiphy, struct net_device *ndev,
-				u8 key_index, bool pairwise, const u8 *mac_addr)
+				int link_id, u8 key_index, bool pairwise,
+				const u8 *mac_addr)
 {
 	struct adapter *padapter = rtw_netdev_priv(ndev);
 	struct security_priv *psecuritypriv = &padapter->securitypriv;
@@ -955,7 +956,7 @@ static int cfg80211_rtw_del_key(struct wiphy *wiphy, struct net_device *ndev,
 }
 
 static int cfg80211_rtw_set_default_key(struct wiphy *wiphy,
-	struct net_device *ndev, u8 key_index
+	struct net_device *ndev, int link_id, u8 key_index
 	, bool unicast, bool multicast
 	)
 {
diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c
index dbd1159a2ef0..ce04c38f6afd 100644
--- a/drivers/staging/sm750fb/sm750.c
+++ b/drivers/staging/sm750fb/sm750.c
@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
+#include <linux/aperture.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/errno.h>
@@ -987,22 +988,16 @@ release_fb:
 
 static int lynxfb_kick_out_firmware_fb(struct pci_dev *pdev)
 {
-	struct apertures_struct *ap;
+	resource_size_t base = pci_resource_start(pdev, 0);
+	resource_size_t size = pci_resource_len(pdev, 0);
 	bool primary = false;
 
-	ap = alloc_apertures(1);
-	if (!ap)
-		return -ENOMEM;
-
-	ap->ranges[0].base = pci_resource_start(pdev, 0);
-	ap->ranges[0].size = pci_resource_len(pdev, 0);
 #ifdef CONFIG_X86
 	primary = pdev->resource[PCI_ROM_RESOURCE].flags &
 					IORESOURCE_ROM_SHADOW;
 #endif
-	remove_conflicting_framebuffers(ap, "sm750_fb1", primary);
-	kfree(ap);
-	return 0;
+
+	return aperture_remove_conflicting_devices(base, size, primary, "sm750_fb1");
 }
 
 static int lynxfb_pci_probe(struct pci_dev *pdev,
diff --git a/drivers/staging/wlan-ng/cfg80211.c b/drivers/staging/wlan-ng/cfg80211.c
index b7b56d8406d1..471bb310176f 100644
--- a/drivers/staging/wlan-ng/cfg80211.c
+++ b/drivers/staging/wlan-ng/cfg80211.c
@@ -143,8 +143,8 @@ exit:
 }
 
 static int prism2_add_key(struct wiphy *wiphy, struct net_device *dev,
-			  u8 key_index, bool pairwise, const u8 *mac_addr,
-			  struct key_params *params)
+			  int link_id, u8 key_index, bool pairwise,
+			  const u8 *mac_addr, struct key_params *params)
 {
 	struct wlandevice *wlandev = dev->ml_priv;
 	u32 did;
@@ -172,7 +172,7 @@ static int prism2_add_key(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int prism2_get_key(struct wiphy *wiphy, struct net_device *dev,
-			  u8 key_index, bool pairwise,
+			  int link_id, u8 key_index, bool pairwise,
 			  const u8 *mac_addr, void *cookie,
 			  void (*callback)(void *cookie, struct key_params*))
 {
@@ -202,7 +202,8 @@ static int prism2_get_key(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int prism2_del_key(struct wiphy *wiphy, struct net_device *dev,
-			  u8 key_index, bool pairwise, const u8 *mac_addr)
+			  int link_id, u8 key_index, bool pairwise,
+			  const u8 *mac_addr)
 {
 	struct wlandevice *wlandev = dev->ml_priv;
 	u32 did;
@@ -227,7 +228,8 @@ static int prism2_del_key(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int prism2_set_default_key(struct wiphy *wiphy, struct net_device *dev,
-				  u8 key_index, bool unicast, bool multicast)
+				  int link_id, u8 key_index, bool unicast,
+				  bool multicast)
 {
 	struct wlandevice *wlandev = dev->ml_priv;
 
diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c
index 3deaeecb712e..2940559c3086 100644
--- a/drivers/target/target_core_user.c
+++ b/drivers/target/target_core_user.c
@@ -486,6 +486,7 @@ static struct genl_family tcmu_genl_family __ro_after_init = {
 	.netnsok = true,
 	.small_ops = tcmu_genl_ops,
 	.n_small_ops = ARRAY_SIZE(tcmu_genl_ops),
+	.resv_start_op = TCMU_CMD_SET_FEATURES + 1,
 };
 
 #define tcmu_cmd_set_dbi_cur(cmd, index) ((cmd)->dbi_cur = (index))
diff --git a/drivers/tee/optee/ffa_abi.c b/drivers/tee/optee/ffa_abi.c
index 7ab31740cff8..0828240f27e6 100644
--- a/drivers/tee/optee/ffa_abi.c
+++ b/drivers/tee/optee/ffa_abi.c
@@ -271,8 +271,8 @@ static int optee_ffa_shm_register(struct tee_context *ctx, struct tee_shm *shm,
 				  unsigned long start)
 {
 	struct optee *optee = tee_get_drvdata(ctx->teedev);
-	const struct ffa_dev_ops *ffa_ops = optee->ffa.ffa_ops;
 	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
+	const struct ffa_mem_ops *mem_ops = ffa_dev->ops->mem_ops;
 	struct ffa_mem_region_attributes mem_attr = {
 		.receiver = ffa_dev->vm_id,
 		.attrs = FFA_MEM_RW,
@@ -294,14 +294,14 @@ static int optee_ffa_shm_register(struct tee_context *ctx, struct tee_shm *shm,
 	if (rc)
 		return rc;
 	args.sg = sgt.sgl;
-	rc = ffa_ops->memory_share(ffa_dev, &args);
+	rc = mem_ops->memory_share(&args);
 	sg_free_table(&sgt);
 	if (rc)
 		return rc;
 
 	rc = optee_shm_add_ffa_handle(optee, shm, args.g_handle);
 	if (rc) {
-		ffa_ops->memory_reclaim(args.g_handle, 0);
+		mem_ops->memory_reclaim(args.g_handle, 0);
 		return rc;
 	}
 
@@ -314,8 +314,9 @@ static int optee_ffa_shm_unregister(struct tee_context *ctx,
 				    struct tee_shm *shm)
 {
 	struct optee *optee = tee_get_drvdata(ctx->teedev);
-	const struct ffa_dev_ops *ffa_ops = optee->ffa.ffa_ops;
 	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
+	const struct ffa_msg_ops *msg_ops = ffa_dev->ops->msg_ops;
+	const struct ffa_mem_ops *mem_ops = ffa_dev->ops->mem_ops;
 	u64 global_handle = shm->sec_world_id;
 	struct ffa_send_direct_data data = {
 		.data0 = OPTEE_FFA_UNREGISTER_SHM,
@@ -327,11 +328,11 @@ static int optee_ffa_shm_unregister(struct tee_context *ctx,
 	optee_shm_rem_ffa_handle(optee, global_handle);
 	shm->sec_world_id = 0;
 
-	rc = ffa_ops->sync_send_receive(ffa_dev, &data);
+	rc = msg_ops->sync_send_receive(ffa_dev, &data);
 	if (rc)
 		pr_err("Unregister SHM id 0x%llx rc %d\n", global_handle, rc);
 
-	rc = ffa_ops->memory_reclaim(global_handle, 0);
+	rc = mem_ops->memory_reclaim(global_handle, 0);
 	if (rc)
 		pr_err("mem_reclaim: 0x%llx %d", global_handle, rc);
 
@@ -342,7 +343,7 @@ static int optee_ffa_shm_unregister_supp(struct tee_context *ctx,
 					 struct tee_shm *shm)
 {
 	struct optee *optee = tee_get_drvdata(ctx->teedev);
-	const struct ffa_dev_ops *ffa_ops = optee->ffa.ffa_ops;
+	const struct ffa_mem_ops *mem_ops;
 	u64 global_handle = shm->sec_world_id;
 	int rc;
 
@@ -353,7 +354,8 @@ static int optee_ffa_shm_unregister_supp(struct tee_context *ctx,
 	 */
 
 	optee_shm_rem_ffa_handle(optee, global_handle);
-	rc = ffa_ops->memory_reclaim(global_handle, 0);
+	mem_ops = optee->ffa.ffa_dev->ops->mem_ops;
+	rc = mem_ops->memory_reclaim(global_handle, 0);
 	if (rc)
 		pr_err("mem_reclaim: 0x%llx %d", global_handle, rc);
 
@@ -529,8 +531,8 @@ static int optee_ffa_yielding_call(struct tee_context *ctx,
 				   struct optee_msg_arg *rpc_arg)
 {
 	struct optee *optee = tee_get_drvdata(ctx->teedev);
-	const struct ffa_dev_ops *ffa_ops = optee->ffa.ffa_ops;
 	struct ffa_device *ffa_dev = optee->ffa.ffa_dev;
+	const struct ffa_msg_ops *msg_ops = ffa_dev->ops->msg_ops;
 	struct optee_call_waiter w;
 	u32 cmd = data->data0;
 	u32 w4 = data->data1;
@@ -541,7 +543,7 @@ static int optee_ffa_yielding_call(struct tee_context *ctx,
 	/* Initialize waiter */
 	optee_cq_wait_init(&optee->call_queue, &w);
 	while (true) {
-		rc = ffa_ops->sync_send_receive(ffa_dev, data);
+		rc = msg_ops->sync_send_receive(ffa_dev, data);
 		if (rc)
 			goto done;
 
@@ -576,7 +578,7 @@ static int optee_ffa_yielding_call(struct tee_context *ctx,
 		 * OP-TEE has returned with a RPC request.
 		 *
 		 * Note that data->data4 (passed in register w7) is already
-		 * filled in by ffa_ops->sync_send_receive() returning
+		 * filled in by ffa_mem_ops->sync_send_receive() returning
 		 * above.
 		 */
 		cond_resched();
@@ -652,14 +654,15 @@ static int optee_ffa_do_call_with_arg(struct tee_context *ctx,
  */
 
 static bool optee_ffa_api_is_compatbile(struct ffa_device *ffa_dev,
-					const struct ffa_dev_ops *ops)
+					const struct ffa_ops *ops)
 {
+	const struct ffa_msg_ops *msg_ops = ops->msg_ops;
 	struct ffa_send_direct_data data = { OPTEE_FFA_GET_API_VERSION };
 	int rc;
 
-	ops->mode_32bit_set(ffa_dev);
+	msg_ops->mode_32bit_set(ffa_dev);
 
-	rc = ops->sync_send_receive(ffa_dev, &data);
+	rc = msg_ops->sync_send_receive(ffa_dev, &data);
 	if (rc) {
 		pr_err("Unexpected error %d\n", rc);
 		return false;
@@ -672,7 +675,7 @@ static bool optee_ffa_api_is_compatbile(struct ffa_device *ffa_dev,
 	}
 
 	data = (struct ffa_send_direct_data){ OPTEE_FFA_GET_OS_VERSION };
-	rc = ops->sync_send_receive(ffa_dev, &data);
+	rc = msg_ops->sync_send_receive(ffa_dev, &data);
 	if (rc) {
 		pr_err("Unexpected error %d\n", rc);
 		return false;
@@ -687,14 +690,14 @@ static bool optee_ffa_api_is_compatbile(struct ffa_device *ffa_dev,
 }
 
 static bool optee_ffa_exchange_caps(struct ffa_device *ffa_dev,
-				    const struct ffa_dev_ops *ops,
+				    const struct ffa_ops *ops,
 				    u32 *sec_caps,
 				    unsigned int *rpc_param_count)
 {
 	struct ffa_send_direct_data data = { OPTEE_FFA_EXCHANGE_CAPABILITIES };
 	int rc;
 
-	rc = ops->sync_send_receive(ffa_dev, &data);
+	rc = ops->msg_ops->sync_send_receive(ffa_dev, &data);
 	if (rc) {
 		pr_err("Unexpected error %d", rc);
 		return false;
@@ -783,7 +786,7 @@ static void optee_ffa_remove(struct ffa_device *ffa_dev)
 
 static int optee_ffa_probe(struct ffa_device *ffa_dev)
 {
-	const struct ffa_dev_ops *ffa_ops;
+	const struct ffa_ops *ffa_ops;
 	unsigned int rpc_param_count;
 	struct tee_shm_pool *pool;
 	struct tee_device *teedev;
@@ -793,11 +796,7 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev)
 	u32 sec_caps;
 	int rc;
 
-	ffa_ops = ffa_dev_ops_get(ffa_dev);
-	if (!ffa_ops) {
-		pr_warn("failed \"method\" init: ffa\n");
-		return -ENOENT;
-	}
+	ffa_ops = ffa_dev->ops;
 
 	if (!optee_ffa_api_is_compatbile(ffa_dev, ffa_ops))
 		return -EINVAL;
@@ -821,7 +820,6 @@ static int optee_ffa_probe(struct ffa_device *ffa_dev)
 
 	optee->ops = &optee_ffa_ops;
 	optee->ffa.ffa_dev = ffa_dev;
-	optee->ffa.ffa_ops = ffa_ops;
 	optee->rpc_param_count = rpc_param_count;
 
 	teedev = tee_device_alloc(&optee_ffa_clnt_desc, NULL, optee->pool,
diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h
index a33d98d17cfd..04ae58892608 100644
--- a/drivers/tee/optee/optee_private.h
+++ b/drivers/tee/optee/optee_private.h
@@ -111,7 +111,6 @@ struct optee_smc {
  */
 struct optee_ffa {
 	struct ffa_device *ffa_dev;
-	const struct ffa_dev_ops *ffa_ops;
 	/* Serializes access to @global_ids */
 	struct mutex mutex;
 	struct rhashtable global_ids;
diff --git a/drivers/thermal/amlogic_thermal.c b/drivers/thermal/amlogic_thermal.c
index e61b91d14ad1..d30cb791e63c 100644
--- a/drivers/thermal/amlogic_thermal.c
+++ b/drivers/thermal/amlogic_thermal.c
@@ -179,12 +179,12 @@ static int amlogic_thermal_disable(struct amlogic_thermal *data)
 	return 0;
 }
 
-static int amlogic_thermal_get_temp(void *data, int *temp)
+static int amlogic_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
 	unsigned int tval;
-	struct amlogic_thermal *pdata = data;
+	struct amlogic_thermal *pdata = tz->devdata;
 
-	if (!data)
+	if (!pdata)
 		return -EINVAL;
 
 	regmap_read(pdata->regmap, TSENSOR_STAT0, &tval);
@@ -195,7 +195,7 @@ static int amlogic_thermal_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops amlogic_thermal_ops = {
+static const struct thermal_zone_device_ops amlogic_thermal_ops = {
 	.get_temp	= amlogic_thermal_get_temp,
 };
 
@@ -276,10 +276,10 @@ static int amlogic_thermal_probe(struct platform_device *pdev)
 		return PTR_ERR(pdata->sec_ao_map);
 	}
 
-	pdata->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev,
-							  0,
-							  pdata,
-							  &amlogic_thermal_ops);
+	pdata->tzd = devm_thermal_of_zone_register(&pdev->dev,
+						   0,
+						   pdata,
+						   &amlogic_thermal_ops);
 	if (IS_ERR(pdata->tzd)) {
 		ret = PTR_ERR(pdata->tzd);
 		dev_err(dev, "Failed to register tsensor: %d\n", ret);
diff --git a/drivers/thermal/armada_thermal.c b/drivers/thermal/armada_thermal.c
index c2ebfb5be4b3..52d63b3997fe 100644
--- a/drivers/thermal/armada_thermal.c
+++ b/drivers/thermal/armada_thermal.c
@@ -420,9 +420,9 @@ static struct thermal_zone_device_ops legacy_ops = {
 	.get_temp = armada_get_temp_legacy,
 };
 
-static int armada_get_temp(void *_sensor, int *temp)
+static int armada_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct armada_thermal_sensor *sensor = _sensor;
+	struct armada_thermal_sensor *sensor = tz->devdata;
 	struct armada_thermal_priv *priv = sensor->priv;
 	int ret;
 
@@ -450,7 +450,7 @@ unlock_mutex:
 	return ret;
 }
 
-static const struct thermal_zone_of_device_ops of_ops = {
+static const struct thermal_zone_device_ops of_ops = {
 	.get_temp = armada_get_temp,
 };
 
@@ -928,9 +928,9 @@ static int armada_thermal_probe(struct platform_device *pdev)
 		/* Register the sensor */
 		sensor->priv = priv;
 		sensor->id = sensor_id;
-		tz = devm_thermal_zone_of_sensor_register(&pdev->dev,
-							  sensor->id, sensor,
-							  &of_ops);
+		tz = devm_thermal_of_zone_register(&pdev->dev,
+						   sensor->id, sensor,
+						   &of_ops);
 		if (IS_ERR(tz)) {
 			dev_info(&pdev->dev, "Thermal sensor %d unavailable\n",
 				 sensor_id);
diff --git a/drivers/thermal/broadcom/bcm2711_thermal.c b/drivers/thermal/broadcom/bcm2711_thermal.c
index e9bef5c3414b..1f8651d15160 100644
--- a/drivers/thermal/broadcom/bcm2711_thermal.c
+++ b/drivers/thermal/broadcom/bcm2711_thermal.c
@@ -31,11 +31,11 @@ struct bcm2711_thermal_priv {
 	struct thermal_zone_device *thermal;
 };
 
-static int bcm2711_get_temp(void *data, int *temp)
+static int bcm2711_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct bcm2711_thermal_priv *priv = data;
-	int slope = thermal_zone_get_slope(priv->thermal);
-	int offset = thermal_zone_get_offset(priv->thermal);
+	struct bcm2711_thermal_priv *priv = tz->devdata;
+	int slope = thermal_zone_get_slope(tz);
+	int offset = thermal_zone_get_offset(tz);
 	u32 val;
 	int ret;
 
@@ -54,7 +54,7 @@ static int bcm2711_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops bcm2711_thermal_of_ops = {
+static const struct thermal_zone_device_ops bcm2711_thermal_of_ops = {
 	.get_temp	= bcm2711_get_temp,
 };
 
@@ -88,8 +88,8 @@ static int bcm2711_thermal_probe(struct platform_device *pdev)
 	}
 	priv->regmap = regmap;
 
-	thermal = devm_thermal_zone_of_sensor_register(dev, 0, priv,
-						       &bcm2711_thermal_of_ops);
+	thermal = devm_thermal_of_zone_register(dev, 0, priv,
+						&bcm2711_thermal_of_ops);
 	if (IS_ERR(thermal)) {
 		ret = PTR_ERR(thermal);
 		dev_err(dev, "could not register sensor: %d\n", ret);
diff --git a/drivers/thermal/broadcom/bcm2835_thermal.c b/drivers/thermal/broadcom/bcm2835_thermal.c
index c8e4344d5a3d..2c67841a1115 100644
--- a/drivers/thermal/broadcom/bcm2835_thermal.c
+++ b/drivers/thermal/broadcom/bcm2835_thermal.c
@@ -88,9 +88,9 @@ static int bcm2835_thermal_temp2adc(int temp, int offset, int slope)
 	return temp;
 }
 
-static int bcm2835_thermal_get_temp(void *d, int *temp)
+static int bcm2835_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct bcm2835_thermal_data *data = d;
+	struct bcm2835_thermal_data *data = tz->devdata;
 	u32 val = readl(data->regs + BCM2835_TS_TSENSSTAT);
 
 	if (!(val & BCM2835_TS_TSENSSTAT_VALID))
@@ -135,7 +135,7 @@ static void bcm2835_thermal_debugfs(struct platform_device *pdev)
 	debugfs_create_regset32("regset", 0444, data->debugfsdir, regset);
 }
 
-static const struct thermal_zone_of_device_ops bcm2835_thermal_ops = {
+static const struct thermal_zone_device_ops bcm2835_thermal_ops = {
 	.get_temp = bcm2835_thermal_get_temp,
 };
 
@@ -206,8 +206,8 @@ static int bcm2835_thermal_probe(struct platform_device *pdev)
 			 data->clk, rate);
 
 	/* register of thermal sensor and get info from DT */
-	tz = thermal_zone_of_sensor_register(&pdev->dev, 0, data,
-					     &bcm2835_thermal_ops);
+	tz = devm_thermal_of_zone_register(&pdev->dev, 0, data,
+					   &bcm2835_thermal_ops);
 	if (IS_ERR(tz)) {
 		err = PTR_ERR(tz);
 		dev_err(&pdev->dev,
@@ -277,7 +277,7 @@ static int bcm2835_thermal_probe(struct platform_device *pdev)
 
 	return 0;
 err_tz:
-	thermal_zone_of_sensor_unregister(&pdev->dev, tz);
+	thermal_of_zone_unregister(tz);
 err_clk:
 	clk_disable_unprepare(data->clk);
 
@@ -290,7 +290,7 @@ static int bcm2835_thermal_remove(struct platform_device *pdev)
 	struct thermal_zone_device *tz = data->tz;
 
 	debugfs_remove_recursive(data->debugfsdir);
-	thermal_zone_of_sensor_unregister(&pdev->dev, tz);
+	thermal_of_zone_unregister(tz);
 	clk_disable_unprepare(data->clk);
 
 	return 0;
diff --git a/drivers/thermal/broadcom/brcmstb_thermal.c b/drivers/thermal/broadcom/brcmstb_thermal.c
index 0cedb8b4f00a..c79c6cfdd74d 100644
--- a/drivers/thermal/broadcom/brcmstb_thermal.c
+++ b/drivers/thermal/broadcom/brcmstb_thermal.c
@@ -105,7 +105,7 @@ static struct avs_tmon_trip avs_tmon_trips[] = {
 struct brcmstb_thermal_params {
 	unsigned int offset;
 	unsigned int mult;
-	const struct thermal_zone_of_device_ops *of_ops;
+	const struct thermal_zone_device_ops *of_ops;
 };
 
 struct brcmstb_thermal_priv {
@@ -150,9 +150,9 @@ static inline u32 avs_tmon_temp_to_code(struct brcmstb_thermal_priv *priv,
 		return (u32)((offset - temp) / mult);
 }
 
-static int brcmstb_get_temp(void *data, int *temp)
+static int brcmstb_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct brcmstb_thermal_priv *priv = data;
+	struct brcmstb_thermal_priv *priv = tz->devdata;
 	u32 val;
 	long t;
 
@@ -260,9 +260,9 @@ static irqreturn_t brcmstb_tmon_irq_thread(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static int brcmstb_set_trips(void *data, int low, int high)
+static int brcmstb_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct brcmstb_thermal_priv *priv = data;
+	struct brcmstb_thermal_priv *priv = tz->devdata;
 
 	dev_dbg(priv->dev, "set trips %d <--> %d\n", low, high);
 
@@ -288,7 +288,7 @@ static int brcmstb_set_trips(void *data, int low, int high)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops brcmstb_16nm_of_ops = {
+static const struct thermal_zone_device_ops brcmstb_16nm_of_ops = {
 	.get_temp	= brcmstb_get_temp,
 };
 
@@ -298,7 +298,7 @@ static const struct brcmstb_thermal_params brcmstb_16nm_params = {
 	.of_ops	= &brcmstb_16nm_of_ops,
 };
 
-static const struct thermal_zone_of_device_ops brcmstb_28nm_of_ops = {
+static const struct thermal_zone_device_ops brcmstb_28nm_of_ops = {
 	.get_temp	= brcmstb_get_temp,
 	.set_trips	= brcmstb_set_trips,
 };
@@ -318,7 +318,7 @@ MODULE_DEVICE_TABLE(of, brcmstb_thermal_id_table);
 
 static int brcmstb_thermal_probe(struct platform_device *pdev)
 {
-	const struct thermal_zone_of_device_ops *of_ops;
+	const struct thermal_zone_device_ops *of_ops;
 	struct thermal_zone_device *thermal;
 	struct brcmstb_thermal_priv *priv;
 	struct resource *res;
@@ -341,8 +341,8 @@ static int brcmstb_thermal_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, priv);
 	of_ops = priv->temp_params->of_ops;
 
-	thermal = devm_thermal_zone_of_sensor_register(&pdev->dev, 0, priv,
-						       of_ops);
+	thermal = devm_thermal_of_zone_register(&pdev->dev, 0, priv,
+						of_ops);
 	if (IS_ERR(thermal)) {
 		ret = PTR_ERR(thermal);
 		dev_err(&pdev->dev, "could not register sensor: %d\n", ret);
diff --git a/drivers/thermal/broadcom/ns-thermal.c b/drivers/thermal/broadcom/ns-thermal.c
index c9468ba9d449..07a8a3f49bd0 100644
--- a/drivers/thermal/broadcom/ns-thermal.c
+++ b/drivers/thermal/broadcom/ns-thermal.c
@@ -14,19 +14,14 @@
 #define PVTMON_CONTROL0_SEL_TEST_MODE			0x0000000e
 #define PVTMON_STATUS					0x08
 
-struct ns_thermal {
-	struct thermal_zone_device *tz;
-	void __iomem *pvtmon;
-};
-
-static int ns_thermal_get_temp(void *data, int *temp)
+static int ns_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct ns_thermal *ns_thermal = data;
-	int offset = thermal_zone_get_offset(ns_thermal->tz);
-	int slope = thermal_zone_get_slope(ns_thermal->tz);
+	void __iomem *pvtmon = tz->devdata;
+	int offset = thermal_zone_get_offset(tz);
+	int slope = thermal_zone_get_slope(tz);
 	u32 val;
 
-	val = readl(ns_thermal->pvtmon + PVTMON_CONTROL0);
+	val = readl(pvtmon + PVTMON_CONTROL0);
 	if ((val & PVTMON_CONTROL0_SEL_MASK) != PVTMON_CONTROL0_SEL_TEMP_MONITOR) {
 		/* Clear current mode selection */
 		val &= ~PVTMON_CONTROL0_SEL_MASK;
@@ -34,50 +29,47 @@ static int ns_thermal_get_temp(void *data, int *temp)
 		/* Set temp monitor mode (it's the default actually) */
 		val |= PVTMON_CONTROL0_SEL_TEMP_MONITOR;
 
-		writel(val, ns_thermal->pvtmon + PVTMON_CONTROL0);
+		writel(val, pvtmon + PVTMON_CONTROL0);
 	}
 
-	val = readl(ns_thermal->pvtmon + PVTMON_STATUS);
+	val = readl(pvtmon + PVTMON_STATUS);
 	*temp = slope * val + offset;
 
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops ns_thermal_ops = {
+static const struct thermal_zone_device_ops ns_thermal_ops = {
 	.get_temp = ns_thermal_get_temp,
 };
 
 static int ns_thermal_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct ns_thermal *ns_thermal;
-
-	ns_thermal = devm_kzalloc(dev, sizeof(*ns_thermal), GFP_KERNEL);
-	if (!ns_thermal)
-		return -ENOMEM;
+	struct thermal_zone_device *tz;
+	void __iomem *pvtmon;
 
-	ns_thermal->pvtmon = of_iomap(dev_of_node(dev), 0);
-	if (WARN_ON(!ns_thermal->pvtmon))
+	pvtmon = of_iomap(dev_of_node(dev), 0);
+	if (WARN_ON(!pvtmon))
 		return -ENOENT;
 
-	ns_thermal->tz = devm_thermal_zone_of_sensor_register(dev, 0,
-							      ns_thermal,
-							      &ns_thermal_ops);
-	if (IS_ERR(ns_thermal->tz)) {
-		iounmap(ns_thermal->pvtmon);
-		return PTR_ERR(ns_thermal->tz);
+	tz = devm_thermal_of_zone_register(dev, 0,
+					   pvtmon,
+					   &ns_thermal_ops);
+	if (IS_ERR(tz)) {
+		iounmap(pvtmon);
+		return PTR_ERR(tz);
 	}
 
-	platform_set_drvdata(pdev, ns_thermal);
+	platform_set_drvdata(pdev, pvtmon);
 
 	return 0;
 }
 
 static int ns_thermal_remove(struct platform_device *pdev)
 {
-	struct ns_thermal *ns_thermal = platform_get_drvdata(pdev);
+	void __iomem *pvtmon = platform_get_drvdata(pdev);
 
-	iounmap(ns_thermal->pvtmon);
+	iounmap(pvtmon);
 
 	return 0;
 }
diff --git a/drivers/thermal/broadcom/sr-thermal.c b/drivers/thermal/broadcom/sr-thermal.c
index 85ab9edd580c..2b93502543ff 100644
--- a/drivers/thermal/broadcom/sr-thermal.c
+++ b/drivers/thermal/broadcom/sr-thermal.c
@@ -19,7 +19,6 @@
 #define SR_TMON_MAX_LIST        6
 
 struct sr_tmon {
-	struct thermal_zone_device *tz;
 	unsigned int crit_temp;
 	unsigned int tmon_id;
 	struct sr_thermal *priv;
@@ -31,9 +30,9 @@ struct sr_thermal {
 	struct sr_tmon tmon[SR_TMON_MAX_LIST];
 };
 
-static int sr_get_temp(void *data, int *temp)
+static int sr_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct sr_tmon *tmon = data;
+	struct sr_tmon *tmon = tz->devdata;
 	struct sr_thermal *sr_thermal = tmon->priv;
 
 	*temp = readl(sr_thermal->regs + SR_TMON_TEMP_BASE(tmon->tmon_id));
@@ -41,13 +40,14 @@ static int sr_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops sr_tz_ops = {
+static const struct thermal_zone_device_ops sr_tz_ops = {
 	.get_temp = sr_get_temp,
 };
 
 static int sr_thermal_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	struct thermal_zone_device *tz;
 	struct sr_thermal *sr_thermal;
 	struct sr_tmon *tmon;
 	struct resource *res;
@@ -84,10 +84,10 @@ static int sr_thermal_probe(struct platform_device *pdev)
 		writel(0, sr_thermal->regs + SR_TMON_TEMP_BASE(i));
 		tmon->tmon_id = i;
 		tmon->priv = sr_thermal;
-		tmon->tz = devm_thermal_zone_of_sensor_register(dev, i, tmon,
-								&sr_tz_ops);
-		if (IS_ERR(tmon->tz))
-			return PTR_ERR(tmon->tz);
+		tz = devm_thermal_of_zone_register(dev, i, tmon,
+						   &sr_tz_ops);
+		if (IS_ERR(tz))
+			return PTR_ERR(tz);
 
 		dev_dbg(dev, "thermal sensor %d registered\n", i);
 	}
diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c
index b76293cc989c..9f8b438fcf8f 100644
--- a/drivers/thermal/cpufreq_cooling.c
+++ b/drivers/thermal/cpufreq_cooling.c
@@ -475,7 +475,7 @@ static int cpufreq_set_cur_state(struct thermal_cooling_device *cdev,
 
 /**
  * __cpufreq_cooling_register - helper function to create cpufreq cooling device
- * @np: a valid struct device_node to the cooling device device tree node
+ * @np: a valid struct device_node to the cooling device tree node
  * @policy: cpufreq policy
  * Normally this should be same as cpufreq policy->related_cpus.
  * @em: Energy Model of the cpufreq policy
@@ -501,17 +501,17 @@ __cpufreq_cooling_register(struct device_node *np,
 	struct thermal_cooling_device_ops *cooling_ops;
 	char *name;
 
+	if (IS_ERR_OR_NULL(policy)) {
+		pr_err("%s: cpufreq policy isn't valid: %p\n", __func__, policy);
+		return ERR_PTR(-EINVAL);
+	}
+
 	dev = get_cpu_device(policy->cpu);
 	if (unlikely(!dev)) {
 		pr_warn("No cpu device for cpu %d\n", policy->cpu);
 		return ERR_PTR(-ENODEV);
 	}
 
-	if (IS_ERR_OR_NULL(policy)) {
-		pr_err("%s: cpufreq policy isn't valid: %p\n", __func__, policy);
-		return ERR_PTR(-EINVAL);
-	}
-
 	i = cpufreq_table_count_valid_entries(policy);
 	if (!i) {
 		pr_debug("%s: CPUFreq table not found or has no valid entries\n",
diff --git a/drivers/thermal/da9062-thermal.c b/drivers/thermal/da9062-thermal.c
index 180edec34e07..7dcfde7a9f2c 100644
--- a/drivers/thermal/da9062-thermal.c
+++ b/drivers/thermal/da9062-thermal.c
@@ -248,10 +248,9 @@ static int da9062_thermal_probe(struct platform_device *pdev)
 		jiffies_to_msecs(thermal->zone->passive_delay_jiffies));
 
 	ret = platform_get_irq_byname(pdev, "THERMAL");
-	if (ret < 0) {
-		dev_err(&pdev->dev, "Failed to get platform IRQ.\n");
+	if (ret < 0)
 		goto err_zone;
-	}
+
 	thermal->irq = ret;
 
 	ret = request_threaded_irq(thermal->irq, NULL,
diff --git a/drivers/thermal/db8500_thermal.c b/drivers/thermal/db8500_thermal.c
index 121cf853e545..cb10e280681f 100644
--- a/drivers/thermal/db8500_thermal.c
+++ b/drivers/thermal/db8500_thermal.c
@@ -58,9 +58,9 @@ struct db8500_thermal_zone {
 };
 
 /* Callback to get current temperature */
-static int db8500_thermal_get_temp(void *data, int *temp)
+static int db8500_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct db8500_thermal_zone *th = data;
+	struct db8500_thermal_zone *th = tz->devdata;
 
 	/*
 	 * TODO: There is no PRCMU interface to get temperature data currently,
@@ -72,7 +72,7 @@ static int db8500_thermal_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static struct thermal_zone_of_device_ops thdev_ops = {
+static const struct thermal_zone_device_ops thdev_ops = {
 	.get_temp = db8500_thermal_get_temp,
 };
 
@@ -182,7 +182,7 @@ static int db8500_thermal_probe(struct platform_device *pdev)
 	}
 
 	/* register of thermal sensor and get info from DT */
-	th->tz = devm_thermal_zone_of_sensor_register(dev, 0, th, &thdev_ops);
+	th->tz = devm_thermal_of_zone_register(dev, 0, th, &thdev_ops);
 	if (IS_ERR(th->tz)) {
 		dev_err(dev, "register thermal zone sensor failed\n");
 		return PTR_ERR(th->tz);
diff --git a/drivers/thermal/gov_bang_bang.c b/drivers/thermal/gov_bang_bang.c
index 991a1c54296d..a08bbe33be96 100644
--- a/drivers/thermal/gov_bang_bang.c
+++ b/drivers/thermal/gov_bang_bang.c
@@ -31,8 +31,6 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip)
 				trip, trip_temp, tz->temperature,
 				trip_hyst);
 
-	mutex_lock(&tz->lock);
-
 	list_for_each_entry(instance, &tz->thermal_instances, tz_node) {
 		if (instance->trip != trip)
 			continue;
@@ -65,8 +63,6 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip)
 		instance->cdev->updated = false; /* cdev needs update */
 		mutex_unlock(&instance->cdev->lock);
 	}
-
-	mutex_unlock(&tz->lock);
 }
 
 /**
@@ -100,15 +96,13 @@ static int bang_bang_control(struct thermal_zone_device *tz, int trip)
 {
 	struct thermal_instance *instance;
 
-	thermal_zone_trip_update(tz, trip);
+	lockdep_assert_held(&tz->lock);
 
-	mutex_lock(&tz->lock);
+	thermal_zone_trip_update(tz, trip);
 
 	list_for_each_entry(instance, &tz->thermal_instances, tz_node)
 		thermal_cdev_update(instance->cdev);
 
-	mutex_unlock(&tz->lock);
-
 	return 0;
 }
 
diff --git a/drivers/thermal/gov_fair_share.c b/drivers/thermal/gov_fair_share.c
index 6a2abcfc648f..a4ee4661e9cc 100644
--- a/drivers/thermal/gov_fair_share.c
+++ b/drivers/thermal/gov_fair_share.c
@@ -82,7 +82,7 @@ static int fair_share_throttle(struct thermal_zone_device *tz, int trip)
 	int total_instance = 0;
 	int cur_trip_level = get_trip_level(tz);
 
-	mutex_lock(&tz->lock);
+	lockdep_assert_held(&tz->lock);
 
 	list_for_each_entry(instance, &tz->thermal_instances, tz_node) {
 		if (instance->trip != trip)
@@ -112,7 +112,6 @@ static int fair_share_throttle(struct thermal_zone_device *tz, int trip)
 		mutex_unlock(&cdev->lock);
 	}
 
-	mutex_unlock(&tz->lock);
 	return 0;
 }
 
diff --git a/drivers/thermal/gov_power_allocator.c b/drivers/thermal/gov_power_allocator.c
index 1d5052470967..2d1aeaba38a8 100644
--- a/drivers/thermal/gov_power_allocator.c
+++ b/drivers/thermal/gov_power_allocator.c
@@ -392,8 +392,6 @@ static int allocate_power(struct thermal_zone_device *tz,
 	int i, num_actors, total_weight, ret = 0;
 	int trip_max_desired_temperature = params->trip_max_desired_temperature;
 
-	mutex_lock(&tz->lock);
-
 	num_actors = 0;
 	total_weight = 0;
 	list_for_each_entry(instance, &tz->thermal_instances, tz_node) {
@@ -404,10 +402,8 @@ static int allocate_power(struct thermal_zone_device *tz,
 		}
 	}
 
-	if (!num_actors) {
-		ret = -ENODEV;
-		goto unlock;
-	}
+	if (!num_actors)
+		return -ENODEV;
 
 	/*
 	 * We need to allocate five arrays of the same size:
@@ -421,10 +417,8 @@ static int allocate_power(struct thermal_zone_device *tz,
 	BUILD_BUG_ON(sizeof(*req_power) != sizeof(*extra_actor_power));
 	BUILD_BUG_ON(sizeof(*req_power) != sizeof(*weighted_req_power));
 	req_power = kcalloc(num_actors * 5, sizeof(*req_power), GFP_KERNEL);
-	if (!req_power) {
-		ret = -ENOMEM;
-		goto unlock;
-	}
+	if (!req_power)
+		return -ENOMEM;
 
 	max_power = &req_power[num_actors];
 	granted_power = &req_power[2 * num_actors];
@@ -496,8 +490,6 @@ static int allocate_power(struct thermal_zone_device *tz,
 				      control_temp - tz->temperature);
 
 	kfree(req_power);
-unlock:
-	mutex_unlock(&tz->lock);
 
 	return ret;
 }
@@ -576,7 +568,6 @@ static void allow_maximum_power(struct thermal_zone_device *tz, bool update)
 	struct power_allocator_params *params = tz->governor_data;
 	u32 req_power;
 
-	mutex_lock(&tz->lock);
 	list_for_each_entry(instance, &tz->thermal_instances, tz_node) {
 		struct thermal_cooling_device *cdev = instance->cdev;
 
@@ -598,7 +589,6 @@ static void allow_maximum_power(struct thermal_zone_device *tz, bool update)
 
 		mutex_unlock(&instance->cdev->lock);
 	}
-	mutex_unlock(&tz->lock);
 }
 
 /**
@@ -712,6 +702,8 @@ static int power_allocator_throttle(struct thermal_zone_device *tz, int trip)
 	struct power_allocator_params *params = tz->governor_data;
 	bool update;
 
+	lockdep_assert_held(&tz->lock);
+
 	/*
 	 * We get called for every trip point but we only need to do
 	 * our calculations once
diff --git a/drivers/thermal/gov_step_wise.c b/drivers/thermal/gov_step_wise.c
index 9729b46d0258..cdd3354bc27f 100644
--- a/drivers/thermal/gov_step_wise.c
+++ b/drivers/thermal/gov_step_wise.c
@@ -117,8 +117,6 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip)
 	dev_dbg(&tz->device, "Trip%d[type=%d,temp=%d]:trend=%d,throttle=%d\n",
 				trip, trip_type, trip_temp, trend, throttle);
 
-	mutex_lock(&tz->lock);
-
 	list_for_each_entry(instance, &tz->thermal_instances, tz_node) {
 		if (instance->trip != trip)
 			continue;
@@ -145,8 +143,6 @@ static void thermal_zone_trip_update(struct thermal_zone_device *tz, int trip)
 		instance->cdev->updated = false; /* cdev needs update */
 		mutex_unlock(&instance->cdev->lock);
 	}
-
-	mutex_unlock(&tz->lock);
 }
 
 /**
@@ -164,15 +160,13 @@ static int step_wise_throttle(struct thermal_zone_device *tz, int trip)
 {
 	struct thermal_instance *instance;
 
-	thermal_zone_trip_update(tz, trip);
+	lockdep_assert_held(&tz->lock);
 
-	mutex_lock(&tz->lock);
+	thermal_zone_trip_update(tz, trip);
 
 	list_for_each_entry(instance, &tz->thermal_instances, tz_node)
 		thermal_cdev_update(instance->cdev);
 
-	mutex_unlock(&tz->lock);
-
 	return 0;
 }
 
diff --git a/drivers/thermal/gov_user_space.c b/drivers/thermal/gov_user_space.c
index a62a4e90bd3f..8bc1c22aaf03 100644
--- a/drivers/thermal/gov_user_space.c
+++ b/drivers/thermal/gov_user_space.c
@@ -34,7 +34,8 @@ static int notify_user_space(struct thermal_zone_device *tz, int trip)
 	char *thermal_prop[5];
 	int i;
 
-	mutex_lock(&tz->lock);
+	lockdep_assert_held(&tz->lock);
+
 	thermal_prop[0] = kasprintf(GFP_KERNEL, "NAME=%s", tz->type);
 	thermal_prop[1] = kasprintf(GFP_KERNEL, "TEMP=%d", tz->temperature);
 	thermal_prop[2] = kasprintf(GFP_KERNEL, "TRIP=%d", trip);
@@ -43,7 +44,7 @@ static int notify_user_space(struct thermal_zone_device *tz, int trip)
 	kobject_uevent_env(&tz->device.kobj, KOBJ_CHANGE, thermal_prop);
 	for (i = 0; i < 4; ++i)
 		kfree(thermal_prop[i]);
-	mutex_unlock(&tz->lock);
+
 	return 0;
 }
 
diff --git a/drivers/thermal/hisi_thermal.c b/drivers/thermal/hisi_thermal.c
index 19a242c69ce6..d6974db7aaf7 100644
--- a/drivers/thermal/hisi_thermal.c
+++ b/drivers/thermal/hisi_thermal.c
@@ -434,9 +434,9 @@ static int hi3660_thermal_probe(struct hisi_thermal_data *data)
 	return 0;
 }
 
-static int hisi_thermal_get_temp(void *__data, int *temp)
+static int hisi_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct hisi_thermal_sensor *sensor = __data;
+	struct hisi_thermal_sensor *sensor = tz->devdata;
 	struct hisi_thermal_data *data = sensor->data;
 
 	*temp = data->ops->get_temp(sensor);
@@ -447,7 +447,7 @@ static int hisi_thermal_get_temp(void *__data, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops hisi_of_thermal_ops = {
+static const struct thermal_zone_device_ops hisi_of_thermal_ops = {
 	.get_temp = hisi_thermal_get_temp,
 };
 
@@ -459,7 +459,7 @@ static irqreturn_t hisi_thermal_alarm_irq_thread(int irq, void *dev)
 
 	data->ops->irq_handler(sensor);
 
-	hisi_thermal_get_temp(sensor, &temp);
+	temp = data->ops->get_temp(sensor);
 
 	if (temp >= sensor->thres_temp) {
 		dev_crit(&data->pdev->dev,
@@ -484,9 +484,9 @@ static int hisi_thermal_register_sensor(struct platform_device *pdev,
 	int ret, i;
 	const struct thermal_trip *trip;
 
-	sensor->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev,
-							   sensor->id, sensor,
-							   &hisi_of_thermal_ops);
+	sensor->tzd = devm_thermal_of_zone_register(&pdev->dev,
+						    sensor->id, sensor,
+						    &hisi_of_thermal_ops);
 	if (IS_ERR(sensor->tzd)) {
 		ret = PTR_ERR(sensor->tzd);
 		sensor->tzd = NULL;
diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c
index af666bd9e8d4..e2c2673025a7 100644
--- a/drivers/thermal/imx8mm_thermal.c
+++ b/drivers/thermal/imx8mm_thermal.c
@@ -96,15 +96,15 @@ static int imx8mp_tmu_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static int tmu_get_temp(void *data, int *temp)
+static int tmu_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct tmu_sensor *sensor = data;
+	struct tmu_sensor *sensor = tz->devdata;
 	struct imx8mm_tmu *tmu = sensor->priv;
 
-	return tmu->socdata->get_temp(data, temp);
+	return tmu->socdata->get_temp(sensor, temp);
 }
 
-static struct thermal_zone_of_device_ops tmu_tz_ops = {
+static const struct thermal_zone_device_ops tmu_tz_ops = {
 	.get_temp = tmu_get_temp,
 };
 
@@ -165,9 +165,9 @@ static int imx8mm_tmu_probe(struct platform_device *pdev)
 	for (i = 0; i < data->num_sensors; i++) {
 		tmu->sensors[i].priv = tmu;
 		tmu->sensors[i].tzd =
-			devm_thermal_zone_of_sensor_register(&pdev->dev, i,
-							     &tmu->sensors[i],
-							     &tmu_tz_ops);
+			devm_thermal_of_zone_register(&pdev->dev, i,
+						      &tmu->sensors[i],
+						      &tmu_tz_ops);
 		if (IS_ERR(tmu->sensors[i].tzd)) {
 			ret = PTR_ERR(tmu->sensors[i].tzd);
 			dev_err(&pdev->dev,
diff --git a/drivers/thermal/imx_sc_thermal.c b/drivers/thermal/imx_sc_thermal.c
index 331a241eb0ef..10bfa6507eb4 100644
--- a/drivers/thermal/imx_sc_thermal.c
+++ b/drivers/thermal/imx_sc_thermal.c
@@ -43,11 +43,11 @@ struct imx_sc_msg_misc_get_temp {
 	} data;
 } __packed __aligned(4);
 
-static int imx_sc_thermal_get_temp(void *data, int *temp)
+static int imx_sc_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
 	struct imx_sc_msg_misc_get_temp msg;
 	struct imx_sc_rpc_msg *hdr = &msg.hdr;
-	struct imx_sc_sensor *sensor = data;
+	struct imx_sc_sensor *sensor = tz->devdata;
 	int ret;
 
 	msg.data.req.resource_id = sensor->resource_id;
@@ -70,7 +70,7 @@ static int imx_sc_thermal_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops imx_sc_thermal_ops = {
+static const struct thermal_zone_device_ops imx_sc_thermal_ops = {
 	.get_temp = imx_sc_thermal_get_temp,
 };
 
@@ -109,10 +109,10 @@ static int imx_sc_thermal_probe(struct platform_device *pdev)
 			break;
 		}
 
-		sensor->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev,
-								   sensor->resource_id,
-								   sensor,
-								   &imx_sc_thermal_ops);
+		sensor->tzd = devm_thermal_of_zone_register(&pdev->dev,
+							    sensor->resource_id,
+							    sensor,
+							    &imx_sc_thermal_ops);
 		if (IS_ERR(sensor->tzd)) {
 			dev_err(&pdev->dev, "failed to register thermal zone\n");
 			ret = PTR_ERR(sensor->tzd);
diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
index 365489bf4b8c..db8a6f63657d 100644
--- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
+++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c
@@ -614,9 +614,8 @@ static int int3400_thermal_probe(struct platform_device *pdev)
 
 free_sysfs:
 	cleanup_odvp(priv);
-	if (priv->data_vault) {
-		if (!ZERO_OR_NULL_PTR(priv->data_vault))
-			sysfs_remove_group(&pdev->dev.kobj, &data_attribute_group);
+	if (!ZERO_OR_NULL_PTR(priv->data_vault)) {
+		sysfs_remove_group(&pdev->dev.kobj, &data_attribute_group);
 		kfree(priv->data_vault);
 	}
 free_uuid:
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
index c2dc4c158b9d..bf1b1cdfade4 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c
@@ -373,18 +373,7 @@ static struct pci_driver proc_thermal_pci_driver = {
 	.driver.pm	= &proc_thermal_pci_pm,
 };
 
-static int __init proc_thermal_init(void)
-{
-	return pci_register_driver(&proc_thermal_pci_driver);
-}
-
-static void __exit proc_thermal_exit(void)
-{
-	pci_unregister_driver(&proc_thermal_pci_driver);
-}
-
-module_init(proc_thermal_init);
-module_exit(proc_thermal_exit);
+module_pci_driver(proc_thermal_pci_driver);
 
 MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
 MODULE_DESCRIPTION("Processor Thermal Reporting Device Driver");
diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c
index 4571a1a53b84..09e032f822f3 100644
--- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c
+++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci_legacy.c
@@ -151,18 +151,7 @@ static struct pci_driver proc_thermal_pci_driver = {
 	.driver.pm	= &proc_thermal_pci_pm,
 };
 
-static int __init proc_thermal_init(void)
-{
-	return pci_register_driver(&proc_thermal_pci_driver);
-}
-
-static void __exit proc_thermal_exit(void)
-{
-	pci_unregister_driver(&proc_thermal_pci_driver);
-}
-
-module_init(proc_thermal_init);
-module_exit(proc_thermal_exit);
+module_pci_driver(proc_thermal_pci_driver);
 
 MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
 MODULE_DESCRIPTION("Processor Thermal Reporting Device Driver");
diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c
index c841ab37e7c6..2a5570b9799a 100644
--- a/drivers/thermal/intel/intel_powerclamp.c
+++ b/drivers/thermal/intel/intel_powerclamp.c
@@ -62,8 +62,7 @@ static struct dentry *debug_dir;
 static unsigned int set_target_ratio;
 static unsigned int current_ratio;
 static bool should_skip;
-static bool reduce_irq;
-static atomic_t idle_wakeup_counter;
+
 static unsigned int control_cpu; /* The cpu assigned to collect stat and update
 				  * control parameters. default to BSP but BSP
 				  * can be offlined.
@@ -285,9 +284,6 @@ static unsigned int get_compensation(int ratio)
 			cal_data[ratio + 1].steady_comp) / 3;
 	}
 
-	/* REVISIT: simple penalty of double idle injection */
-	if (reduce_irq)
-		comp = ratio;
 	/* do not exceed limit */
 	if (comp + ratio >= MAX_TARGET_RATIO)
 		comp = MAX_TARGET_RATIO - ratio - 1;
@@ -301,13 +297,9 @@ static void adjust_compensation(int target_ratio, unsigned int win)
 	struct powerclamp_calibration_data *d = &cal_data[target_ratio];
 
 	/*
-	 * adjust compensations if confidence level has not been reached or
-	 * there are too many wakeups during the last idle injection period, we
-	 * cannot trust the data for compensation.
+	 * adjust compensations if confidence level has not been reached.
 	 */
-	if (d->confidence >= CONFIDENCE_OK ||
-		atomic_read(&idle_wakeup_counter) >
-		win * num_online_cpus())
+	if (d->confidence >= CONFIDENCE_OK)
 		return;
 
 	delta = set_target_ratio - current_ratio;
@@ -347,14 +339,7 @@ static bool powerclamp_adjust_controls(unsigned int target_ratio,
 	tsc_last = tsc_now;
 
 	adjust_compensation(target_ratio, win);
-	/*
-	 * too many external interrupts, set flag such
-	 * that we can take measure later.
-	 */
-	reduce_irq = atomic_read(&idle_wakeup_counter) >=
-		2 * win * num_online_cpus();
 
-	atomic_set(&idle_wakeup_counter, 0);
 	/* if we are above target+guard, skip */
 	return set_target_ratio + guard <= current_ratio;
 }
@@ -532,8 +517,10 @@ static int start_power_clamp(void)
 
 	/* prefer BSP */
 	control_cpu = 0;
-	if (!cpu_online(control_cpu))
-		control_cpu = smp_processor_id();
+	if (!cpu_online(control_cpu)) {
+		control_cpu = get_cpu();
+		put_cpu();
+	}
 
 	clamping = true;
 	schedule_delayed_work(&poll_pkg_cstate_work, 0);
diff --git a/drivers/thermal/k3_bandgap.c b/drivers/thermal/k3_bandgap.c
index 5d0b3ffc6f46..22c9bcb899c3 100644
--- a/drivers/thermal/k3_bandgap.c
+++ b/drivers/thermal/k3_bandgap.c
@@ -139,9 +139,9 @@ static int k3_bgp_read_temp(struct k3_thermal_data *devdata,
 	return 0;
 }
 
-static int k3_thermal_get_temp(void *devdata, int *temp)
+static int k3_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct k3_thermal_data *data = devdata;
+	struct k3_thermal_data *data = tz->devdata;
 	int ret = 0;
 
 	ret = k3_bgp_read_temp(data, temp);
@@ -151,7 +151,7 @@ static int k3_thermal_get_temp(void *devdata, int *temp)
 	return ret;
 }
 
-static const struct thermal_zone_of_device_ops k3_of_thermal_ops = {
+static const struct thermal_zone_device_ops k3_of_thermal_ops = {
 	.get_temp = k3_thermal_get_temp,
 };
 
@@ -213,9 +213,9 @@ static int k3_bandgap_probe(struct platform_device *pdev)
 		writel(val, data[id].bgp->base + data[id].ctrl_offset);
 
 		data[id].tzd =
-		devm_thermal_zone_of_sensor_register(dev, id,
-						     &data[id],
-						     &k3_of_thermal_ops);
+		devm_thermal_of_zone_register(dev, id,
+					      &data[id],
+					      &k3_of_thermal_ops);
 		if (IS_ERR(data[id].tzd)) {
 			dev_err(dev, "thermal zone device is NULL\n");
 			ret = PTR_ERR(data[id].tzd);
diff --git a/drivers/thermal/k3_j72xx_bandgap.c b/drivers/thermal/k3_j72xx_bandgap.c
index 115a44eb4fbf..16b6bcf1bf4f 100644
--- a/drivers/thermal/k3_j72xx_bandgap.c
+++ b/drivers/thermal/k3_j72xx_bandgap.c
@@ -247,9 +247,9 @@ static inline int k3_bgp_read_temp(struct k3_thermal_data *devdata,
 }
 
 /* Get temperature callback function for thermal zone */
-static int k3_thermal_get_temp(void *devdata, int *temp)
+static int k3_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct k3_thermal_data *data = devdata;
+	struct k3_thermal_data *data = tz->devdata;
 	int ret = 0;
 
 	ret = k3_bgp_read_temp(data, temp);
@@ -259,7 +259,7 @@ static int k3_thermal_get_temp(void *devdata, int *temp)
 	return ret;
 }
 
-static const struct thermal_zone_of_device_ops k3_of_thermal_ops = {
+static const struct thermal_zone_device_ops k3_of_thermal_ops = {
 	.get_temp = k3_thermal_get_temp,
 };
 
@@ -474,10 +474,8 @@ static int k3_j72xx_bandgap_probe(struct platform_device *pdev)
 		writel(val, data[id].bgp->cfg2_base + data[id].ctrl_offset);
 
 		bgp->ts_data[id] = &data[id];
-		ti_thermal =
-		devm_thermal_zone_of_sensor_register(bgp->dev, id,
-						     &data[id],
-						     &k3_of_thermal_ops);
+		ti_thermal = devm_thermal_of_zone_register(bgp->dev, id, &data[id],
+							   &k3_of_thermal_ops);
 		if (IS_ERR(ti_thermal)) {
 			dev_err(bgp->dev, "thermal zone device is NULL\n");
 			ret = PTR_ERR(ti_thermal);
diff --git a/drivers/thermal/max77620_thermal.c b/drivers/thermal/max77620_thermal.c
index 82d06c7411eb..6451a55eb582 100644
--- a/drivers/thermal/max77620_thermal.c
+++ b/drivers/thermal/max77620_thermal.c
@@ -44,9 +44,9 @@ struct max77620_therm_info {
  * Return 0 on success otherwise error number to show reason of failure.
  */
 
-static int max77620_thermal_read_temp(void *data, int *temp)
+static int max77620_thermal_read_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct max77620_therm_info *mtherm = data;
+	struct max77620_therm_info *mtherm = tz->devdata;
 	unsigned int val;
 	int ret;
 
@@ -66,7 +66,7 @@ static int max77620_thermal_read_temp(void *data, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops max77620_thermal_ops = {
+static const struct thermal_zone_device_ops max77620_thermal_ops = {
 	.get_temp = max77620_thermal_read_temp,
 };
 
@@ -114,7 +114,7 @@ static int max77620_thermal_probe(struct platform_device *pdev)
 	 */
 	device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent);
 
-	mtherm->tz_device = devm_thermal_zone_of_sensor_register(&pdev->dev, 0,
+	mtherm->tz_device = devm_thermal_of_zone_register(&pdev->dev, 0,
 				mtherm, &max77620_thermal_ops);
 	if (IS_ERR(mtherm->tz_device)) {
 		ret = PTR_ERR(mtherm->tz_device);
diff --git a/drivers/thermal/mtk_thermal.c b/drivers/thermal/mtk_thermal.c
index ede94eadddda..8440692e3890 100644
--- a/drivers/thermal/mtk_thermal.c
+++ b/drivers/thermal/mtk_thermal.c
@@ -679,9 +679,9 @@ static int mtk_thermal_bank_temperature(struct mtk_thermal_bank *bank)
 	return max;
 }
 
-static int mtk_read_temp(void *data, int *temperature)
+static int mtk_read_temp(struct thermal_zone_device *tz, int *temperature)
 {
-	struct mtk_thermal *mt = data;
+	struct mtk_thermal *mt = tz->devdata;
 	int i;
 	int tempmax = INT_MIN;
 
@@ -700,7 +700,7 @@ static int mtk_read_temp(void *data, int *temperature)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops mtk_thermal_ops = {
+static const struct thermal_zone_device_ops mtk_thermal_ops = {
 	.get_temp = mtk_read_temp,
 };
 
@@ -1082,8 +1082,8 @@ static int mtk_thermal_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, mt);
 
-	tzdev = devm_thermal_zone_of_sensor_register(&pdev->dev, 0, mt,
-						     &mtk_thermal_ops);
+	tzdev = devm_thermal_of_zone_register(&pdev->dev, 0, mt,
+					      &mtk_thermal_ops);
 	if (IS_ERR(tzdev)) {
 		ret = PTR_ERR(tzdev);
 		goto err_disable_clk_peri_therm;
diff --git a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
index 073943cbcc2b..af68adf720cc 100644
--- a/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
+++ b/drivers/thermal/qcom/qcom-spmi-adc-tm5.c
@@ -357,9 +357,9 @@ static irqreturn_t adc_tm5_gen2_isr(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static int adc_tm5_get_temp(void *data, int *temp)
+static int adc_tm5_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct adc_tm5_channel *channel = data;
+	struct adc_tm5_channel *channel = tz->devdata;
 	int ret;
 
 	if (!channel || !channel->iio)
@@ -639,9 +639,9 @@ config_fail:
 	return ret;
 }
 
-static int adc_tm5_set_trips(void *data, int low, int high)
+static int adc_tm5_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct adc_tm5_channel *channel = data;
+	struct adc_tm5_channel *channel = tz->devdata;
 	struct adc_tm5_chip *chip;
 	int ret;
 
@@ -660,7 +660,7 @@ static int adc_tm5_set_trips(void *data, int low, int high)
 	return ret;
 }
 
-static struct thermal_zone_of_device_ops adc_tm5_thermal_ops = {
+static const struct thermal_zone_device_ops adc_tm5_thermal_ops = {
 	.get_temp = adc_tm5_get_temp,
 	.set_trips = adc_tm5_set_trips,
 };
@@ -672,11 +672,10 @@ static int adc_tm5_register_tzd(struct adc_tm5_chip *adc_tm)
 
 	for (i = 0; i < adc_tm->nchannels; i++) {
 		adc_tm->channels[i].chip = adc_tm;
-
-		tzd = devm_thermal_zone_of_sensor_register(adc_tm->dev,
-							   adc_tm->channels[i].channel,
-							   &adc_tm->channels[i],
-							   &adc_tm5_thermal_ops);
+		tzd = devm_thermal_of_zone_register(adc_tm->dev,
+						    adc_tm->channels[i].channel,
+						    &adc_tm->channels[i],
+						    &adc_tm5_thermal_ops);
 		if (IS_ERR(tzd)) {
 			if (PTR_ERR(tzd) == -ENODEV) {
 				dev_warn(adc_tm->dev, "thermal sensor on channel %d is not used\n",
@@ -1026,10 +1025,8 @@ static int adc_tm5_probe(struct platform_device *pdev)
 	adc_tm->base = reg;
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		dev_err(dev, "get_irq failed: %d\n", irq);
+	if (irq < 0)
 		return irq;
-	}
 
 	ret = adc_tm5_get_dt_data(adc_tm, node);
 	if (ret) {
diff --git a/drivers/thermal/qcom/qcom-spmi-temp-alarm.c b/drivers/thermal/qcom/qcom-spmi-temp-alarm.c
index 770f82cc9bca..be785ab37e53 100644
--- a/drivers/thermal/qcom/qcom-spmi-temp-alarm.c
+++ b/drivers/thermal/qcom/qcom-spmi-temp-alarm.c
@@ -186,9 +186,9 @@ static int qpnp_tm_update_temp_no_adc(struct qpnp_tm_chip *chip)
 	return 0;
 }
 
-static int qpnp_tm_get_temp(void *data, int *temp)
+static int qpnp_tm_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct qpnp_tm_chip *chip = data;
+	struct qpnp_tm_chip *chip = tz->devdata;
 	int ret, mili_celsius;
 
 	if (!temp)
@@ -263,9 +263,9 @@ skip:
 	return qpnp_tm_write(chip, QPNP_TM_REG_SHUTDOWN_CTRL1, reg);
 }
 
-static int qpnp_tm_set_trip_temp(void *data, int trip, int temp)
+static int qpnp_tm_set_trip_temp(struct thermal_zone_device *tz, int trip, int temp)
 {
-	struct qpnp_tm_chip *chip = data;
+	struct qpnp_tm_chip *chip = tz->devdata;
 	const struct thermal_trip *trip_points;
 	int ret;
 
@@ -283,7 +283,7 @@ static int qpnp_tm_set_trip_temp(void *data, int trip, int temp)
 	return ret;
 }
 
-static const struct thermal_zone_of_device_ops qpnp_tm_sensor_ops = {
+static const struct thermal_zone_device_ops qpnp_tm_sensor_ops = {
 	.get_temp = qpnp_tm_get_temp,
 	.set_trip_temp = qpnp_tm_set_trip_temp,
 };
@@ -446,7 +446,7 @@ static int qpnp_tm_probe(struct platform_device *pdev)
 	 * read the trip points. get_temp() returns the default temperature
 	 * before the hardware initialization is completed.
 	 */
-	chip->tz_dev = devm_thermal_zone_of_sensor_register(
+	chip->tz_dev = devm_thermal_of_zone_register(
 		&pdev->dev, 0, chip, &qpnp_tm_sensor_ops);
 	if (IS_ERR(chip->tz_dev)) {
 		dev_err(&pdev->dev, "failed to register sensor\n");
diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c
index e49f58e83513..b1b10005fb28 100644
--- a/drivers/thermal/qcom/tsens.c
+++ b/drivers/thermal/qcom/tsens.c
@@ -532,9 +532,9 @@ static irqreturn_t tsens_irq_thread(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
-static int tsens_set_trips(void *_sensor, int low, int high)
+static int tsens_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct tsens_sensor *s = _sensor;
+	struct tsens_sensor *s = tz->devdata;
 	struct tsens_priv *priv = s->priv;
 	struct device *dev = priv->dev;
 	struct tsens_irq_data d;
@@ -925,9 +925,9 @@ err_put_device:
 	return ret;
 }
 
-static int tsens_get_temp(void *data, int *temp)
+static int tsens_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct tsens_sensor *s = data;
+	struct tsens_sensor *s = tz->devdata;
 	struct tsens_priv *priv = s->priv;
 
 	return priv->ops->get_temp(s, temp);
@@ -991,7 +991,7 @@ static const struct of_device_id tsens_table[] = {
 };
 MODULE_DEVICE_TABLE(of, tsens_table);
 
-static const struct thermal_zone_of_device_ops tsens_of_ops = {
+static const struct thermal_zone_device_ops tsens_of_ops = {
 	.get_temp = tsens_get_temp,
 	.set_trips = tsens_set_trips,
 };
@@ -1044,9 +1044,9 @@ static int tsens_register(struct tsens_priv *priv)
 
 	for (i = 0;  i < priv->num_sensors; i++) {
 		priv->sensor[i].priv = priv;
-		tzd = devm_thermal_zone_of_sensor_register(priv->dev, priv->sensor[i].hw_id,
-							   &priv->sensor[i],
-							   &tsens_of_ops);
+		tzd = devm_thermal_of_zone_register(priv->dev, priv->sensor[i].hw_id,
+						    &priv->sensor[i],
+						    &tsens_of_ops);
 		if (IS_ERR(tzd))
 			continue;
 		priv->sensor[i].tzd = tzd;
diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c
index 73049f9bea25..d111e218f362 100644
--- a/drivers/thermal/qoriq_thermal.c
+++ b/drivers/thermal/qoriq_thermal.c
@@ -82,9 +82,9 @@ static struct qoriq_tmu_data *qoriq_sensor_to_data(struct qoriq_sensor *s)
 	return container_of(s, struct qoriq_tmu_data, sensor[s->id]);
 }
 
-static int tmu_get_temp(void *p, int *temp)
+static int tmu_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct qoriq_sensor *qsensor = p;
+	struct qoriq_sensor *qsensor = tz->devdata;
 	struct qoriq_tmu_data *qdata = qoriq_sensor_to_data(qsensor);
 	u32 val;
 	/*
@@ -122,7 +122,7 @@ static int tmu_get_temp(void *p, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops tmu_tz_ops = {
+static const struct thermal_zone_device_ops tmu_tz_ops = {
 	.get_temp = tmu_get_temp,
 };
 
@@ -146,9 +146,9 @@ static int qoriq_tmu_register_tmu_zone(struct device *dev,
 
 		sensor->id = id;
 
-		tzd = devm_thermal_zone_of_sensor_register(dev, id,
-							   sensor,
-							   &tmu_tz_ops);
+		tzd = devm_thermal_of_zone_register(dev, id,
+						    sensor,
+						    &tmu_tz_ops);
 		ret = PTR_ERR_OR_ZERO(tzd);
 		if (ret) {
 			if (ret == -ENODEV)
diff --git a/drivers/thermal/rcar_gen3_thermal.c b/drivers/thermal/rcar_gen3_thermal.c
index cda7c52f2319..4c1c6f89aa2f 100644
--- a/drivers/thermal/rcar_gen3_thermal.c
+++ b/drivers/thermal/rcar_gen3_thermal.c
@@ -164,9 +164,9 @@ static int rcar_gen3_thermal_round(int temp)
 	return result * RCAR3_THERMAL_GRAN;
 }
 
-static int rcar_gen3_thermal_get_temp(void *devdata, int *temp)
+static int rcar_gen3_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct rcar_gen3_thermal_tsc *tsc = devdata;
+	struct rcar_gen3_thermal_tsc *tsc = tz->devdata;
 	int mcelsius, val;
 	int reg;
 
@@ -203,9 +203,9 @@ static int rcar_gen3_thermal_mcelsius_to_temp(struct rcar_gen3_thermal_tsc *tsc,
 	return INT_FIXPT(val);
 }
 
-static int rcar_gen3_thermal_set_trips(void *devdata, int low, int high)
+static int rcar_gen3_thermal_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct rcar_gen3_thermal_tsc *tsc = devdata;
+	struct rcar_gen3_thermal_tsc *tsc = tz->devdata;
 	u32 irqmsk = 0;
 
 	if (low != -INT_MAX) {
@@ -225,7 +225,7 @@ static int rcar_gen3_thermal_set_trips(void *devdata, int low, int high)
 	return 0;
 }
 
-static struct thermal_zone_of_device_ops rcar_gen3_tz_of_ops = {
+static struct thermal_zone_device_ops rcar_gen3_tz_of_ops = {
 	.get_temp	= rcar_gen3_thermal_get_temp,
 	.set_trips	= rcar_gen3_thermal_set_trips,
 };
@@ -508,8 +508,8 @@ static int rcar_gen3_thermal_probe(struct platform_device *pdev)
 	for (i = 0; i < priv->num_tscs; i++) {
 		struct rcar_gen3_thermal_tsc *tsc = priv->tscs[i];
 
-		zone = devm_thermal_zone_of_sensor_register(dev, i, tsc,
-							    &rcar_gen3_tz_of_ops);
+		zone = devm_thermal_of_zone_register(dev, i, tsc,
+						     &rcar_gen3_tz_of_ops);
 		if (IS_ERR(zone)) {
 			dev_err(dev, "Sensor %u: Can't register thermal zone\n", i);
 			ret = PTR_ERR(zone);
@@ -560,7 +560,7 @@ static int __maybe_unused rcar_gen3_thermal_resume(struct device *dev)
 
 		priv->thermal_init(tsc);
 		if (zone->ops->set_trips)
-			rcar_gen3_thermal_set_trips(tsc, zone->prev_low_trip,
+			rcar_gen3_thermal_set_trips(zone, zone->prev_low_trip,
 						    zone->prev_high_trip);
 	}
 
diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c
index 1d729ed4d685..4df42d70d867 100644
--- a/drivers/thermal/rcar_thermal.c
+++ b/drivers/thermal/rcar_thermal.c
@@ -271,13 +271,6 @@ static int rcar_thermal_get_current_temp(struct rcar_thermal_priv *priv,
 	return 0;
 }
 
-static int rcar_thermal_of_get_temp(void *data, int *temp)
-{
-	struct rcar_thermal_priv *priv = data;
-
-	return rcar_thermal_get_current_temp(priv, temp);
-}
-
 static int rcar_thermal_get_temp(struct thermal_zone_device *zone, int *temp)
 {
 	struct rcar_thermal_priv *priv = rcar_zone_to_priv(zone);
@@ -323,8 +316,8 @@ static int rcar_thermal_get_trip_temp(struct thermal_zone_device *zone,
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops rcar_thermal_zone_of_ops = {
-	.get_temp	= rcar_thermal_of_get_temp,
+static struct thermal_zone_device_ops rcar_thermal_zone_of_ops = {
+	.get_temp	= rcar_thermal_get_temp,
 };
 
 static struct thermal_zone_device_ops rcar_thermal_zone_ops = {
@@ -534,7 +527,7 @@ static int rcar_thermal_probe(struct platform_device *pdev)
 			goto error_unregister;
 
 		if (chip->use_of_thermal) {
-			priv->zone = devm_thermal_zone_of_sensor_register(
+			priv->zone = devm_thermal_of_zone_register(
 						dev, i, priv,
 						&rcar_thermal_zone_of_ops);
 		} else {
diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c
index dc3a9c276a09..819e059cde71 100644
--- a/drivers/thermal/rockchip_thermal.c
+++ b/drivers/thermal/rockchip_thermal.c
@@ -1211,9 +1211,9 @@ static irqreturn_t rockchip_thermal_alarm_irq_thread(int irq, void *dev)
 	return IRQ_HANDLED;
 }
 
-static int rockchip_thermal_set_trips(void *_sensor, int low, int high)
+static int rockchip_thermal_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct rockchip_thermal_sensor *sensor = _sensor;
+	struct rockchip_thermal_sensor *sensor = tz->devdata;
 	struct rockchip_thermal_data *thermal = sensor->thermal;
 	const struct rockchip_tsadc_chip *tsadc = thermal->chip;
 
@@ -1224,9 +1224,9 @@ static int rockchip_thermal_set_trips(void *_sensor, int low, int high)
 				     sensor->id, thermal->regs, high);
 }
 
-static int rockchip_thermal_get_temp(void *_sensor, int *out_temp)
+static int rockchip_thermal_get_temp(struct thermal_zone_device *tz, int *out_temp)
 {
-	struct rockchip_thermal_sensor *sensor = _sensor;
+	struct rockchip_thermal_sensor *sensor = tz->devdata;
 	struct rockchip_thermal_data *thermal = sensor->thermal;
 	const struct rockchip_tsadc_chip *tsadc = sensor->thermal->chip;
 	int retval;
@@ -1239,7 +1239,7 @@ static int rockchip_thermal_get_temp(void *_sensor, int *out_temp)
 	return retval;
 }
 
-static const struct thermal_zone_of_device_ops rockchip_of_thermal_ops = {
+static const struct thermal_zone_device_ops rockchip_of_thermal_ops = {
 	.get_temp = rockchip_thermal_get_temp,
 	.set_trips = rockchip_thermal_set_trips,
 };
@@ -1326,8 +1326,8 @@ rockchip_thermal_register_sensor(struct platform_device *pdev,
 
 	sensor->thermal = thermal;
 	sensor->id = id;
-	sensor->tzd = devm_thermal_zone_of_sensor_register(&pdev->dev, id,
-					sensor, &rockchip_of_thermal_ops);
+	sensor->tzd = devm_thermal_of_zone_register(&pdev->dev, id, sensor,
+						    &rockchip_of_thermal_ops);
 	if (IS_ERR(sensor->tzd)) {
 		error = PTR_ERR(sensor->tzd);
 		dev_err(&pdev->dev, "failed to register sensor %d: %d\n",
diff --git a/drivers/thermal/rzg2l_thermal.c b/drivers/thermal/rzg2l_thermal.c
index 51ae80eda6af..2e0649f38506 100644
--- a/drivers/thermal/rzg2l_thermal.c
+++ b/drivers/thermal/rzg2l_thermal.c
@@ -73,9 +73,9 @@ static inline void rzg2l_thermal_write(struct rzg2l_thermal_priv *priv, u32 reg,
 	iowrite32(data, priv->base + reg);
 }
 
-static int rzg2l_thermal_get_temp(void *devdata, int *temp)
+static int rzg2l_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct rzg2l_thermal_priv *priv = devdata;
+	struct rzg2l_thermal_priv *priv = tz->devdata;
 	u32 result = 0, dsensor, ts_code_ave;
 	int val, i;
 
@@ -114,7 +114,7 @@ static int rzg2l_thermal_get_temp(void *devdata, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops rzg2l_tz_of_ops = {
+static const struct thermal_zone_device_ops rzg2l_tz_of_ops = {
 	.get_temp = rzg2l_thermal_get_temp,
 };
 
@@ -207,8 +207,8 @@ static int rzg2l_thermal_probe(struct platform_device *pdev)
 		goto err;
 	}
 
-	zone = devm_thermal_zone_of_sensor_register(dev, 0, priv,
-						    &rzg2l_tz_of_ops);
+	zone = devm_thermal_of_zone_register(dev, 0, priv,
+					     &rzg2l_tz_of_ops);
 	if (IS_ERR(zone)) {
 		dev_err(dev, "Can't register thermal zone");
 		ret = PTR_ERR(zone);
diff --git a/drivers/thermal/samsung/exynos_tmu.c b/drivers/thermal/samsung/exynos_tmu.c
index f4ab4c5b4b62..51874d0a284c 100644
--- a/drivers/thermal/samsung/exynos_tmu.c
+++ b/drivers/thermal/samsung/exynos_tmu.c
@@ -650,9 +650,9 @@ static void exynos7_tmu_control(struct platform_device *pdev, bool on)
 	writel(con, data->base + EXYNOS_TMU_REG_CONTROL);
 }
 
-static int exynos_get_temp(void *p, int *temp)
+static int exynos_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct exynos_tmu_data *data = p;
+	struct exynos_tmu_data *data = tz->devdata;
 	int value, ret = 0;
 
 	if (!data || !data->tmu_read)
@@ -728,9 +728,9 @@ static void exynos4412_tmu_set_emulation(struct exynos_tmu_data *data,
 	writel(val, data->base + emul_con);
 }
 
-static int exynos_tmu_set_emulation(void *drv_data, int temp)
+static int exynos_tmu_set_emulation(struct thermal_zone_device *tz, int temp)
 {
-	struct exynos_tmu_data *data = drv_data;
+	struct exynos_tmu_data *data = tz->devdata;
 	int ret = -EINVAL;
 
 	if (data->soc == SOC_ARCH_EXYNOS4210)
@@ -750,7 +750,7 @@ out:
 }
 #else
 #define exynos4412_tmu_set_emulation NULL
-static int exynos_tmu_set_emulation(void *drv_data, int temp)
+static int exynos_tmu_set_emulation(struct thermal_zone_device *tz, int temp)
 	{ return -EINVAL; }
 #endif /* CONFIG_THERMAL_EMULATION */
 
@@ -997,7 +997,7 @@ static int exynos_map_dt_data(struct platform_device *pdev)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops exynos_sensor_ops = {
+static const struct thermal_zone_device_ops exynos_sensor_ops = {
 	.get_temp = exynos_get_temp,
 	.set_emul_temp = exynos_tmu_set_emulation,
 };
@@ -1091,8 +1091,8 @@ static int exynos_tmu_probe(struct platform_device *pdev)
 	 * data->tzd must be registered before calling exynos_tmu_initialize(),
 	 * requesting irq and calling exynos_tmu_control().
 	 */
-	data->tzd = thermal_zone_of_sensor_register(&pdev->dev, 0, data,
-						    &exynos_sensor_ops);
+	data->tzd = devm_thermal_of_zone_register(&pdev->dev, 0, data,
+						  &exynos_sensor_ops);
 	if (IS_ERR(data->tzd)) {
 		ret = PTR_ERR(data->tzd);
 		if (ret != -EPROBE_DEFER)
@@ -1104,21 +1104,19 @@ static int exynos_tmu_probe(struct platform_device *pdev)
 	ret = exynos_tmu_initialize(pdev);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to initialize TMU\n");
-		goto err_thermal;
+		goto err_sclk;
 	}
 
 	ret = devm_request_irq(&pdev->dev, data->irq, exynos_tmu_irq,
 		IRQF_TRIGGER_RISING | IRQF_SHARED, dev_name(&pdev->dev), data);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to request irq: %d\n", data->irq);
-		goto err_thermal;
+		goto err_sclk;
 	}
 
 	exynos_tmu_control(pdev, true);
 	return 0;
 
-err_thermal:
-	thermal_zone_of_sensor_unregister(&pdev->dev, data->tzd);
 err_sclk:
 	clk_disable_unprepare(data->sclk);
 err_clk:
@@ -1136,9 +1134,7 @@ err_sensor:
 static int exynos_tmu_remove(struct platform_device *pdev)
 {
 	struct exynos_tmu_data *data = platform_get_drvdata(pdev);
-	struct thermal_zone_device *tzd = data->tzd;
 
-	thermal_zone_of_sensor_unregister(&pdev->dev, tzd);
 	exynos_tmu_control(pdev, false);
 
 	clk_disable_unprepare(data->sclk);
diff --git a/drivers/thermal/sprd_thermal.c b/drivers/thermal/sprd_thermal.c
index fff80fc18002..ac884514f116 100644
--- a/drivers/thermal/sprd_thermal.c
+++ b/drivers/thermal/sprd_thermal.c
@@ -204,9 +204,9 @@ static int sprd_thm_temp_to_rawdata(int temp, struct sprd_thermal_sensor *sen)
 	return clamp(val, val, (u32)(SPRD_THM_RAW_DATA_HIGH - 1));
 }
 
-static int sprd_thm_read_temp(void *devdata, int *temp)
+static int sprd_thm_read_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct sprd_thermal_sensor *sen = devdata;
+	struct sprd_thermal_sensor *sen = tz->devdata;
 	u32 data;
 
 	data = readl(sen->data->base + SPRD_THM_TEMP(sen->id)) &
@@ -217,7 +217,7 @@ static int sprd_thm_read_temp(void *devdata, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops sprd_thm_ops = {
+static const struct thermal_zone_device_ops sprd_thm_ops = {
 	.get_temp = sprd_thm_read_temp,
 };
 
@@ -408,10 +408,10 @@ static int sprd_thm_probe(struct platform_device *pdev)
 
 		sprd_thm_sensor_init(thm, sen);
 
-		sen->tzd = devm_thermal_zone_of_sensor_register(sen->dev,
-								sen->id,
-								sen,
-								&sprd_thm_ops);
+		sen->tzd = devm_thermal_of_zone_register(sen->dev,
+							 sen->id,
+							 sen,
+							 &sprd_thm_ops);
 		if (IS_ERR(sen->tzd)) {
 			dev_err(&pdev->dev, "register thermal zone failed %d\n",
 				sen->id);
@@ -523,8 +523,8 @@ static int sprd_thm_remove(struct platform_device *pdev)
 
 	for (i = 0; i < thm->nr_sensors; i++) {
 		sprd_thm_toggle_sensor(thm->sensor[i], false);
-		devm_thermal_zone_of_sensor_unregister(&pdev->dev,
-						       thm->sensor[i]->tzd);
+		devm_thermal_of_zone_unregister(&pdev->dev,
+						thm->sensor[i]->tzd);
 	}
 
 	clk_disable_unprepare(thm->clk);
diff --git a/drivers/thermal/st/stm_thermal.c b/drivers/thermal/st/stm_thermal.c
index 5fd3fb8912a6..78feb802a87d 100644
--- a/drivers/thermal/st/stm_thermal.c
+++ b/drivers/thermal/st/stm_thermal.c
@@ -302,9 +302,9 @@ static int stm_disable_irq(struct stm_thermal_sensor *sensor)
 	return 0;
 }
 
-static int stm_thermal_set_trips(void *data, int low, int high)
+static int stm_thermal_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	struct stm_thermal_sensor *sensor = data;
+	struct stm_thermal_sensor *sensor = tz->devdata;
 	u32 itr1, th;
 	int ret;
 
@@ -350,9 +350,9 @@ static int stm_thermal_set_trips(void *data, int low, int high)
 }
 
 /* Callback to get temperature from HW */
-static int stm_thermal_get_temp(void *data, int *temp)
+static int stm_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct stm_thermal_sensor *sensor = data;
+	struct stm_thermal_sensor *sensor = tz->devdata;
 	u32 periods;
 	int freqM, ret;
 
@@ -474,7 +474,7 @@ static int stm_thermal_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(stm_thermal_pm_ops,
 			 stm_thermal_suspend, stm_thermal_resume);
 
-static const struct thermal_zone_of_device_ops stm_tz_ops = {
+static const struct thermal_zone_device_ops stm_tz_ops = {
 	.get_temp	= stm_thermal_get_temp,
 	.set_trips	= stm_thermal_set_trips,
 };
@@ -539,9 +539,9 @@ static int stm_thermal_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	sensor->th_dev = devm_thermal_zone_of_sensor_register(&pdev->dev, 0,
-							      sensor,
-							      &stm_tz_ops);
+	sensor->th_dev = devm_thermal_of_zone_register(&pdev->dev, 0,
+						       sensor,
+						       &stm_tz_ops);
 
 	if (IS_ERR(sensor->th_dev)) {
 		dev_err(&pdev->dev, "%s: thermal zone sensor registering KO\n",
@@ -572,7 +572,6 @@ static int stm_thermal_probe(struct platform_device *pdev)
 	return 0;
 
 err_tz:
-	thermal_zone_of_sensor_unregister(&pdev->dev, sensor->th_dev);
 	return ret;
 }
 
@@ -582,7 +581,6 @@ static int stm_thermal_remove(struct platform_device *pdev)
 
 	stm_thermal_sensor_off(sensor);
 	thermal_remove_hwmon_sysfs(sensor->th_dev);
-	thermal_zone_of_sensor_unregister(&pdev->dev, sensor->th_dev);
 
 	return 0;
 }
diff --git a/drivers/thermal/sun8i_thermal.c b/drivers/thermal/sun8i_thermal.c
index 212c87e63a66..e64d06d1328c 100644
--- a/drivers/thermal/sun8i_thermal.c
+++ b/drivers/thermal/sun8i_thermal.c
@@ -108,9 +108,9 @@ static int sun50i_h5_calc_temp(struct ths_device *tmdev,
 		return -1590 * reg / 10 + 276000;
 }
 
-static int sun8i_ths_get_temp(void *data, int *temp)
+static int sun8i_ths_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct tsensor *s = data;
+	struct tsensor *s = tz->devdata;
 	struct ths_device *tmdev = s->tmdev;
 	int val = 0;
 
@@ -135,7 +135,7 @@ static int sun8i_ths_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops ths_ops = {
+static const struct thermal_zone_device_ops ths_ops = {
 	.get_temp = sun8i_ths_get_temp,
 };
 
@@ -468,10 +468,10 @@ static int sun8i_ths_register(struct ths_device *tmdev)
 		tmdev->sensor[i].tmdev = tmdev;
 		tmdev->sensor[i].id = i;
 		tmdev->sensor[i].tzd =
-			devm_thermal_zone_of_sensor_register(tmdev->dev,
-							     i,
-							     &tmdev->sensor[i],
-							     &ths_ops);
+			devm_thermal_of_zone_register(tmdev->dev,
+						      i,
+						      &tmdev->sensor[i],
+						      &ths_ops);
 		if (IS_ERR(tmdev->sensor[i].tzd))
 			return PTR_ERR(tmdev->sensor[i].tzd);
 
diff --git a/drivers/thermal/tegra/soctherm.c b/drivers/thermal/tegra/soctherm.c
index 825eab526619..1efe470f31e9 100644
--- a/drivers/thermal/tegra/soctherm.c
+++ b/drivers/thermal/tegra/soctherm.c
@@ -421,9 +421,9 @@ static int translate_temp(u16 val)
 	return t;
 }
 
-static int tegra_thermctl_get_temp(void *data, int *out_temp)
+static int tegra_thermctl_get_temp(struct thermal_zone_device *tz, int *out_temp)
 {
-	struct tegra_thermctl_zone *zone = data;
+	struct tegra_thermctl_zone *zone = tz->devdata;
 	u32 val;
 
 	val = readl(zone->reg);
@@ -582,10 +582,9 @@ static int tsensor_group_thermtrip_get(struct tegra_soctherm *ts, int id)
 	return temp;
 }
 
-static int tegra_thermctl_set_trip_temp(void *data, int trip, int temp)
+static int tegra_thermctl_set_trip_temp(struct thermal_zone_device *tz, int trip, int temp)
 {
-	struct tegra_thermctl_zone *zone = data;
-	struct thermal_zone_device *tz = zone->tz;
+	struct tegra_thermctl_zone *zone = tz->devdata;
 	struct tegra_soctherm *ts = zone->ts;
 	const struct tegra_tsensor_group *sg = zone->sg;
 	struct device *dev = zone->dev;
@@ -657,9 +656,9 @@ static void thermal_irq_disable(struct tegra_thermctl_zone *zn)
 	mutex_unlock(&zn->ts->thermctl_lock);
 }
 
-static int tegra_thermctl_set_trips(void *data, int lo, int hi)
+static int tegra_thermctl_set_trips(struct thermal_zone_device *tz, int lo, int hi)
 {
-	struct tegra_thermctl_zone *zone = data;
+	struct tegra_thermctl_zone *zone = tz->devdata;
 	u32 r;
 
 	thermal_irq_disable(zone);
@@ -682,7 +681,7 @@ static int tegra_thermctl_set_trips(void *data, int lo, int hi)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops tegra_of_thermal_ops = {
+static const struct thermal_zone_device_ops tegra_of_thermal_ops = {
 	.get_temp = tegra_thermctl_get_temp,
 	.set_trip_temp = tegra_thermctl_set_trip_temp,
 	.set_trips = tegra_thermctl_set_trips,
@@ -2194,9 +2193,9 @@ static int tegra_soctherm_probe(struct platform_device *pdev)
 		zone->sg = soc->ttgs[i];
 		zone->ts = tegra;
 
-		z = devm_thermal_zone_of_sensor_register(&pdev->dev,
-							 soc->ttgs[i]->id, zone,
-							 &tegra_of_thermal_ops);
+		z = devm_thermal_of_zone_register(&pdev->dev,
+						  soc->ttgs[i]->id, zone,
+						  &tegra_of_thermal_ops);
 		if (IS_ERR(z)) {
 			err = PTR_ERR(z);
 			dev_err(&pdev->dev, "failed to register sensor: %d\n",
diff --git a/drivers/thermal/tegra/tegra-bpmp-thermal.c b/drivers/thermal/tegra/tegra-bpmp-thermal.c
index 5affc3d196be..eb84f0b9dc7c 100644
--- a/drivers/thermal/tegra/tegra-bpmp-thermal.c
+++ b/drivers/thermal/tegra/tegra-bpmp-thermal.c
@@ -30,9 +30,9 @@ struct tegra_bpmp_thermal {
 	struct tegra_bpmp_thermal_zone **zones;
 };
 
-static int tegra_bpmp_thermal_get_temp(void *data, int *out_temp)
+static int __tegra_bpmp_thermal_get_temp(struct tegra_bpmp_thermal_zone *zone,
+					 int *out_temp)
 {
-	struct tegra_bpmp_thermal_zone *zone = data;
 	struct mrq_thermal_host_to_bpmp_request req;
 	union mrq_thermal_bpmp_to_host_response reply;
 	struct tegra_bpmp_message msg;
@@ -60,9 +60,14 @@ static int tegra_bpmp_thermal_get_temp(void *data, int *out_temp)
 	return 0;
 }
 
-static int tegra_bpmp_thermal_set_trips(void *data, int low, int high)
+static int tegra_bpmp_thermal_get_temp(struct thermal_zone_device *tz, int *out_temp)
 {
-	struct tegra_bpmp_thermal_zone *zone = data;
+	return __tegra_bpmp_thermal_get_temp(tz->devdata, out_temp);
+}
+
+static int tegra_bpmp_thermal_set_trips(struct thermal_zone_device *tz, int low, int high)
+{
+	struct tegra_bpmp_thermal_zone *zone = tz->devdata;
 	struct mrq_thermal_host_to_bpmp_request req;
 	struct tegra_bpmp_message msg;
 	int err;
@@ -157,7 +162,7 @@ static int tegra_bpmp_thermal_get_num_zones(struct tegra_bpmp *bpmp,
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops tegra_bpmp_of_thermal_ops = {
+static const struct thermal_zone_device_ops tegra_bpmp_of_thermal_ops = {
 	.get_temp = tegra_bpmp_thermal_get_temp,
 	.set_trips = tegra_bpmp_thermal_set_trips,
 };
@@ -200,13 +205,13 @@ static int tegra_bpmp_thermal_probe(struct platform_device *pdev)
 		zone->idx = i;
 		zone->tegra = tegra;
 
-		err = tegra_bpmp_thermal_get_temp(zone, &temp);
+		err = __tegra_bpmp_thermal_get_temp(zone, &temp);
 		if (err < 0) {
 			devm_kfree(&pdev->dev, zone);
 			continue;
 		}
 
-		tzd = devm_thermal_zone_of_sensor_register(
+		tzd = devm_thermal_of_zone_register(
 			&pdev->dev, i, zone, &tegra_bpmp_of_thermal_ops);
 		if (IS_ERR(tzd)) {
 			if (PTR_ERR(tzd) == -EPROBE_DEFER)
diff --git a/drivers/thermal/tegra/tegra30-tsensor.c b/drivers/thermal/tegra/tegra30-tsensor.c
index 05886684f429..c34501287e96 100644
--- a/drivers/thermal/tegra/tegra30-tsensor.c
+++ b/drivers/thermal/tegra/tegra30-tsensor.c
@@ -159,9 +159,9 @@ static void devm_tegra_tsensor_hw_disable(void *data)
 	tegra_tsensor_hw_disable(ts);
 }
 
-static int tegra_tsensor_get_temp(void *data, int *temp)
+static int tegra_tsensor_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	const struct tegra_tsensor_channel *tsc = data;
+	const struct tegra_tsensor_channel *tsc = tz->devdata;
 	const struct tegra_tsensor *ts = tsc->ts;
 	int err, c1, c2, c3, c4, counter;
 	u32 val;
@@ -217,9 +217,9 @@ static int tegra_tsensor_temp_to_counter(const struct tegra_tsensor *ts, int tem
 	return DIV_ROUND_CLOSEST(c2 * 1000000 - ts->calib.b, ts->calib.a);
 }
 
-static int tegra_tsensor_set_trips(void *data, int low, int high)
+static int tegra_tsensor_set_trips(struct thermal_zone_device *tz, int low, int high)
 {
-	const struct tegra_tsensor_channel *tsc = data;
+	const struct tegra_tsensor_channel *tsc = tz->devdata;
 	const struct tegra_tsensor *ts = tsc->ts;
 	u32 val;
 
@@ -240,7 +240,7 @@ static int tegra_tsensor_set_trips(void *data, int low, int high)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops ops = {
+static const struct thermal_zone_device_ops ops = {
 	.get_temp = tegra_tsensor_get_temp,
 	.set_trips = tegra_tsensor_set_trips,
 };
@@ -516,7 +516,7 @@ static int tegra_tsensor_register_channel(struct tegra_tsensor *ts,
 	tsc->id = id;
 	tsc->regs = ts->regs + 0x40 * (hw_id + 1);
 
-	tsc->tzd = devm_thermal_zone_of_sensor_register(ts->dev, id, tsc, &ops);
+	tsc->tzd = devm_thermal_of_zone_register(ts->dev, id, tsc, &ops);
 	if (IS_ERR(tsc->tzd)) {
 		if (PTR_ERR(tsc->tzd) != -ENODEV)
 			return dev_err_probe(ts->dev, PTR_ERR(tsc->tzd),
diff --git a/drivers/thermal/thermal-generic-adc.c b/drivers/thermal/thermal-generic-adc.c
index 73665c3ccfe0..323e273e3298 100644
--- a/drivers/thermal/thermal-generic-adc.c
+++ b/drivers/thermal/thermal-generic-adc.c
@@ -52,9 +52,9 @@ static int gadc_thermal_adc_to_temp(struct gadc_thermal_info *gti, int val)
 	return temp;
 }
 
-static int gadc_thermal_get_temp(void *data, int *temp)
+static int gadc_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
-	struct gadc_thermal_info *gti = data;
+	struct gadc_thermal_info *gti = tz->devdata;
 	int val;
 	int ret;
 
@@ -68,7 +68,7 @@ static int gadc_thermal_get_temp(void *data, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops gadc_thermal_ops = {
+static const struct thermal_zone_device_ops gadc_thermal_ops = {
 	.get_temp = gadc_thermal_get_temp,
 };
 
@@ -143,8 +143,8 @@ static int gadc_thermal_probe(struct platform_device *pdev)
 	gti->dev = &pdev->dev;
 	platform_set_drvdata(pdev, gti);
 
-	gti->tz_dev = devm_thermal_zone_of_sensor_register(&pdev->dev, 0, gti,
-							   &gadc_thermal_ops);
+	gti->tz_dev = devm_thermal_of_zone_register(&pdev->dev, 0, gti,
+						    &gadc_thermal_ops);
 	if (IS_ERR(gti->tz_dev)) {
 		ret = PTR_ERR(gti->tz_dev);
 		if (ret != -EPROBE_DEFER)
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index 50d50cec7774..7e669b60a065 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -295,27 +295,14 @@ static void thermal_zone_device_set_polling(struct thermal_zone_device *tz,
 		cancel_delayed_work(&tz->poll_queue);
 }
 
-static inline bool should_stop_polling(struct thermal_zone_device *tz)
-{
-	return !thermal_zone_device_is_enabled(tz);
-}
-
 static void monitor_thermal_zone(struct thermal_zone_device *tz)
 {
-	bool stop;
-
-	stop = should_stop_polling(tz);
-
-	mutex_lock(&tz->lock);
-
-	if (!stop && tz->passive)
+	if (tz->mode != THERMAL_DEVICE_ENABLED)
+		thermal_zone_device_set_polling(tz, 0);
+	else if (tz->passive)
 		thermal_zone_device_set_polling(tz, tz->passive_delay_jiffies);
-	else if (!stop && tz->polling_delay_jiffies)
+	else if (tz->polling_delay_jiffies)
 		thermal_zone_device_set_polling(tz, tz->polling_delay_jiffies);
-	else
-		thermal_zone_device_set_polling(tz, 0);
-
-	mutex_unlock(&tz->lock);
 }
 
 static void handle_non_critical_trips(struct thermal_zone_device *tz, int trip)
@@ -383,18 +370,13 @@ static void handle_thermal_trip(struct thermal_zone_device *tz, int trip)
 		handle_critical_trips(tz, trip, trip_temp, type);
 	else
 		handle_non_critical_trips(tz, trip);
-	/*
-	 * Alright, we handled this trip successfully.
-	 * So, start monitoring again.
-	 */
-	monitor_thermal_zone(tz);
 }
 
 static void update_temperature(struct thermal_zone_device *tz)
 {
 	int temp, ret;
 
-	ret = thermal_zone_get_temp(tz, &temp);
+	ret = __thermal_zone_get_temp(tz, &temp);
 	if (ret) {
 		if (ret != -EAGAIN)
 			dev_warn(&tz->device,
@@ -403,10 +385,8 @@ static void update_temperature(struct thermal_zone_device *tz)
 		return;
 	}
 
-	mutex_lock(&tz->lock);
 	tz->last_temperature = tz->temperature;
 	tz->temperature = temp;
-	mutex_unlock(&tz->lock);
 
 	trace_thermal_temperature(tz);
 
@@ -469,15 +449,9 @@ EXPORT_SYMBOL_GPL(thermal_zone_device_disable);
 
 int thermal_zone_device_is_enabled(struct thermal_zone_device *tz)
 {
-	enum thermal_device_mode mode;
-
-	mutex_lock(&tz->lock);
-
-	mode = tz->mode;
+	lockdep_assert_held(&tz->lock);
 
-	mutex_unlock(&tz->lock);
-
-	return mode == THERMAL_DEVICE_ENABLED;
+	return tz->mode == THERMAL_DEVICE_ENABLED;
 }
 
 void thermal_zone_device_update(struct thermal_zone_device *tz,
@@ -485,9 +459,6 @@ void thermal_zone_device_update(struct thermal_zone_device *tz,
 {
 	int count;
 
-	if (should_stop_polling(tz))
-		return;
-
 	if (atomic_read(&in_suspend))
 		return;
 
@@ -495,14 +466,23 @@ void thermal_zone_device_update(struct thermal_zone_device *tz,
 		      "'get_temp' ops set\n", __func__))
 		return;
 
+	mutex_lock(&tz->lock);
+
+	if (!thermal_zone_device_is_enabled(tz))
+		goto out;
+
 	update_temperature(tz);
 
-	thermal_zone_set_trips(tz);
+	__thermal_zone_set_trips(tz);
 
 	tz->notify_event = event;
 
 	for (count = 0; count < tz->num_trips; count++)
 		handle_thermal_trip(tz, count);
+
+	monitor_thermal_zone(tz);
+out:
+	mutex_unlock(&tz->lock);
 }
 EXPORT_SYMBOL_GPL(thermal_zone_device_update);
 
@@ -1212,7 +1192,20 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t
 		return ERR_PTR(-EINVAL);
 	}
 
-	if (num_trips > THERMAL_MAX_TRIPS || num_trips < 0 || mask >> num_trips) {
+	/*
+	 * Max trip count can't exceed 31 as the "mask >> num_trips" condition.
+	 * For example, shifting by 32 will result in compiler warning:
+	 * warning: right shift count >= width of type [-Wshift-count- overflow]
+	 *
+	 * Also "mask >> num_trips" will always be true with 32 bit shift.
+	 * E.g. mask = 0x80000000 for trip id 31 to be RW. Then
+	 * mask >> 32 = 0x80000000
+	 * This will result in failure for the below condition.
+	 *
+	 * Check will be true when the bit 31 of the mask is set.
+	 * 32 bit shift will cause overflow of 4 byte integer.
+	 */
+	if (num_trips > (BITS_PER_TYPE(int) - 1) || num_trips < 0 || mask >> num_trips) {
 		pr_err("Incorrect number of thermal trips\n");
 		return ERR_PTR(-EINVAL);
 	}
@@ -1239,7 +1232,7 @@ thermal_zone_device_register_with_trips(const char *type, struct thermal_trip *t
 	}
 
 	tz->id = id;
-	strlcpy(tz->type, type, sizeof(tz->type));
+	strscpy(tz->type, type, sizeof(tz->type));
 
 	result = dev_set_name(&tz->device, "thermal_zone%d", tz->id);
 	if (result)
@@ -1458,9 +1451,6 @@ static int thermal_pm_notify(struct notifier_block *nb,
 	case PM_POST_SUSPEND:
 		atomic_set(&in_suspend, 0);
 		list_for_each_entry(tz, &thermal_tz_list, node) {
-			if (!thermal_zone_device_is_enabled(tz))
-				continue;
-
 			thermal_zone_device_init(tz);
 			thermal_zone_device_update(tz,
 						   THERMAL_EVENT_UNSPECIFIED);
@@ -1492,10 +1482,6 @@ static int __init thermal_init(void)
 	if (result)
 		goto unregister_governors;
 
-	result = of_parse_thermal_zones();
-	if (result)
-		goto unregister_class;
-
 	result = register_pm_notifier(&thermal_pm_nb);
 	if (result)
 		pr_warn("Thermal: Can not register suspend notifier, return %d\n",
@@ -1503,8 +1489,6 @@ static int __init thermal_init(void)
 
 	return 0;
 
-unregister_class:
-	class_unregister(&thermal_class);
 unregister_governors:
 	thermal_unregister_governors();
 error:
diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h
index c991bb290512..1571917bd3c8 100644
--- a/drivers/thermal/thermal_core.h
+++ b/drivers/thermal/thermal_core.h
@@ -112,6 +112,8 @@ int thermal_build_list_of_policies(char *buf);
 
 /* Helpers */
 void thermal_zone_set_trips(struct thermal_zone_device *tz);
+void __thermal_zone_set_trips(struct thermal_zone_device *tz);
+int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp);
 
 /* sysfs I/F */
 int thermal_zone_create_device_groups(struct thermal_zone_device *, int);
@@ -135,13 +137,11 @@ thermal_cooling_device_stats_update(struct thermal_cooling_device *cdev,
 
 /* device tree support */
 #ifdef CONFIG_THERMAL_OF
-int of_parse_thermal_zones(void);
 int of_thermal_get_ntrips(struct thermal_zone_device *);
 bool of_thermal_is_trip_valid(struct thermal_zone_device *, int);
 const struct thermal_trip *
 of_thermal_get_trip_points(struct thermal_zone_device *);
 #else
-static inline int of_parse_thermal_zones(void) { return 0; }
 static inline int of_thermal_get_ntrips(struct thermal_zone_device *tz)
 {
 	return 0;
diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c
index 690890f054a3..c65cdce8f856 100644
--- a/drivers/thermal/thermal_helpers.c
+++ b/drivers/thermal/thermal_helpers.c
@@ -64,27 +64,17 @@ get_thermal_instance(struct thermal_zone_device *tz,
 }
 EXPORT_SYMBOL(get_thermal_instance);
 
-/**
- * thermal_zone_get_temp() - returns the temperature of a thermal zone
- * @tz: a valid pointer to a struct thermal_zone_device
- * @temp: a valid pointer to where to store the resulting temperature.
- *
- * When a valid thermal zone reference is passed, it will fetch its
- * temperature and fill @temp.
- *
- * Return: On success returns 0, an error code otherwise
- */
-int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp)
+int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp)
 {
 	int ret = -EINVAL;
 	int count;
 	int crit_temp = INT_MAX;
 	enum thermal_trip_type type;
 
-	if (!tz || IS_ERR(tz) || !tz->ops->get_temp)
-		goto exit;
+	lockdep_assert_held(&tz->lock);
 
-	mutex_lock(&tz->lock);
+	if (!tz || IS_ERR(tz) || !tz->ops->get_temp)
+		return -EINVAL;
 
 	ret = tz->ops->get_temp(tz, temp);
 
@@ -107,35 +97,42 @@ int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp)
 			*temp = tz->emul_temperature;
 	}
 
-	mutex_unlock(&tz->lock);
-exit:
 	return ret;
 }
-EXPORT_SYMBOL_GPL(thermal_zone_get_temp);
 
 /**
- * thermal_zone_set_trips - Computes the next trip points for the driver
- * @tz: a pointer to a thermal zone device structure
+ * thermal_zone_get_temp() - returns the temperature of a thermal zone
+ * @tz: a valid pointer to a struct thermal_zone_device
+ * @temp: a valid pointer to where to store the resulting temperature.
  *
- * The function computes the next temperature boundaries by browsing
- * the trip points. The result is the closer low and high trip points
- * to the current temperature. These values are passed to the backend
- * driver to let it set its own notification mechanism (usually an
- * interrupt).
+ * When a valid thermal zone reference is passed, it will fetch its
+ * temperature and fill @temp.
  *
- * It does not return a value
+ * Return: On success returns 0, an error code otherwise
  */
-void thermal_zone_set_trips(struct thermal_zone_device *tz)
+int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp)
+{
+	int ret;
+
+	mutex_lock(&tz->lock);
+	ret = __thermal_zone_get_temp(tz, temp);
+	mutex_unlock(&tz->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(thermal_zone_get_temp);
+
+void __thermal_zone_set_trips(struct thermal_zone_device *tz)
 {
 	int low = -INT_MAX;
 	int high = INT_MAX;
 	int trip_temp, hysteresis;
 	int i, ret;
 
-	mutex_lock(&tz->lock);
+	lockdep_assert_held(&tz->lock);
 
 	if (!tz->ops->set_trips || !tz->ops->get_trip_hyst)
-		goto exit;
+		return;
 
 	for (i = 0; i < tz->num_trips; i++) {
 		int trip_low;
@@ -154,7 +151,7 @@ void thermal_zone_set_trips(struct thermal_zone_device *tz)
 
 	/* No need to change trip points */
 	if (tz->prev_low_trip == low && tz->prev_high_trip == high)
-		goto exit;
+		return;
 
 	tz->prev_low_trip = low;
 	tz->prev_high_trip = high;
@@ -169,8 +166,24 @@ void thermal_zone_set_trips(struct thermal_zone_device *tz)
 	ret = tz->ops->set_trips(tz, low, high);
 	if (ret)
 		dev_err(&tz->device, "Failed to set trips: %d\n", ret);
+}
 
-exit:
+/**
+ * thermal_zone_set_trips - Computes the next trip points for the driver
+ * @tz: a pointer to a thermal zone device structure
+ *
+ * The function computes the next temperature boundaries by browsing
+ * the trip points. The result is the closer low and high trip points
+ * to the current temperature. These values are passed to the backend
+ * driver to let it set its own notification mechanism (usually an
+ * interrupt).
+ *
+ * It does not return a value
+ */
+void thermal_zone_set_trips(struct thermal_zone_device *tz)
+{
+	mutex_lock(&tz->lock);
+	__thermal_zone_set_trips(tz);
 	mutex_unlock(&tz->lock);
 }
 
diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c
index 09e49ec8b6f4..f53f4ceb6a5d 100644
--- a/drivers/thermal/thermal_hwmon.c
+++ b/drivers/thermal/thermal_hwmon.c
@@ -147,7 +147,7 @@ int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz)
 		return -ENOMEM;
 
 	INIT_LIST_HEAD(&hwmon->tz_list);
-	strlcpy(hwmon->type, tz->type, THERMAL_NAME_LENGTH);
+	strscpy(hwmon->type, tz->type, THERMAL_NAME_LENGTH);
 	strreplace(hwmon->type, '-', '_');
 	hwmon->device = hwmon_device_register_for_thermal(&tz->device,
 							  hwmon->type, hwmon);
diff --git a/drivers/thermal/thermal_mmio.c b/drivers/thermal/thermal_mmio.c
index 360b0dfdc3b0..39c921415989 100644
--- a/drivers/thermal/thermal_mmio.c
+++ b/drivers/thermal/thermal_mmio.c
@@ -20,11 +20,10 @@ static u32 thermal_mmio_readb(void __iomem *mmio_base)
 	return readb(mmio_base);
 }
 
-static int thermal_mmio_get_temperature(void *private, int *temp)
+static int thermal_mmio_get_temperature(struct thermal_zone_device *tz, int *temp)
 {
 	int t;
-	struct thermal_mmio *sensor =
-		(struct thermal_mmio *)private;
+	struct thermal_mmio *sensor = tz->devdata;
 
 	t = sensor->read_mmio(sensor->mmio_base) & sensor->mask;
 	t *= sensor->factor;
@@ -34,7 +33,7 @@ static int thermal_mmio_get_temperature(void *private, int *temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops thermal_mmio_ops = {
+static const struct thermal_zone_device_ops thermal_mmio_ops = {
 	.get_temp = thermal_mmio_get_temperature,
 };
 
@@ -68,10 +67,10 @@ static int thermal_mmio_probe(struct platform_device *pdev)
 		}
 	}
 
-	thermal_zone = devm_thermal_zone_of_sensor_register(&pdev->dev,
-							    0,
-							    sensor,
-							    &thermal_mmio_ops);
+	thermal_zone = devm_thermal_of_zone_register(&pdev->dev,
+						     0,
+						     sensor,
+						     &thermal_mmio_ops);
 	if (IS_ERR(thermal_zone)) {
 		dev_err(&pdev->dev,
 			"failed to register sensor (%ld)\n",
@@ -79,7 +78,7 @@ static int thermal_mmio_probe(struct platform_device *pdev)
 		return PTR_ERR(thermal_zone);
 	}
 
-	thermal_mmio_get_temperature(sensor, &temperature);
+	thermal_mmio_get_temperature(thermal_zone, &temperature);
 	dev_info(&pdev->dev,
 		 "thermal mmio sensor %s registered, current temperature: %d\n",
 		 pdev->name, temperature);
@@ -107,7 +106,7 @@ static struct platform_driver thermal_mmio_driver = {
 	.probe = thermal_mmio_probe,
 	.driver = {
 		.name = "thermal-mmio",
-		.of_match_table = of_match_ptr(thermal_mmio_id_table),
+		.of_match_table = thermal_mmio_id_table,
 	},
 };
 
diff --git a/drivers/thermal/thermal_netlink.c b/drivers/thermal/thermal_netlink.c
index 050d243a5fa1..e2d78a996b5f 100644
--- a/drivers/thermal/thermal_netlink.c
+++ b/drivers/thermal/thermal_netlink.c
@@ -693,6 +693,7 @@ static struct genl_family thermal_gnl_family __ro_after_init = {
 	.policy		= thermal_genl_policy,
 	.small_ops	= thermal_genl_ops,
 	.n_small_ops	= ARRAY_SIZE(thermal_genl_ops),
+	.resv_start_op	= THERMAL_GENL_CMD_CDEV_GET + 1,
 	.mcgrps		= thermal_genl_mcgrps,
 	.n_mcgrps	= ARRAY_SIZE(thermal_genl_mcgrps),
 };
diff --git a/drivers/thermal/thermal_of.c b/drivers/thermal/thermal_of.c
index 802c30b72a92..fd2fb84bf246 100644
--- a/drivers/thermal/thermal_of.c
+++ b/drivers/thermal/thermal_of.c
@@ -19,93 +19,6 @@
 
 #include "thermal_core.h"
 
-/***   Private data structures to represent thermal device tree data ***/
-
-/**
- * struct __thermal_cooling_bind_param - a cooling device for a trip point
- * @cooling_device: a pointer to identify the referred cooling device
- * @min: minimum cooling state used at this trip point
- * @max: maximum cooling state used at this trip point
- */
-
-struct __thermal_cooling_bind_param {
-	struct device_node *cooling_device;
-	unsigned long min;
-	unsigned long max;
-};
-
-/**
- * struct __thermal_bind_params - a match between trip and cooling device
- * @tcbp: a pointer to an array of cooling devices
- * @count: number of elements in array
- * @trip_id: the trip point index
- * @usage: the percentage (from 0 to 100) of cooling contribution
- */
-
-struct __thermal_bind_params {
-	struct __thermal_cooling_bind_param *tcbp;
-	unsigned int count;
-	unsigned int trip_id;
-	unsigned int usage;
-};
-
-/**
- * struct __thermal_zone - internal representation of a thermal zone
- * @passive_delay: polling interval while passive cooling is activated
- * @polling_delay: zone polling interval
- * @slope: slope of the temperature adjustment curve
- * @offset: offset of the temperature adjustment curve
- * @ntrips: number of trip points
- * @trips: an array of trip points (0..ntrips - 1)
- * @num_tbps: number of thermal bind params
- * @tbps: an array of thermal bind params (0..num_tbps - 1)
- * @sensor_data: sensor private data used while reading temperature and trend
- * @ops: set of callbacks to handle the thermal zone based on DT
- */
-
-struct __thermal_zone {
-	int passive_delay;
-	int polling_delay;
-	int slope;
-	int offset;
-
-	/* trip data */
-	int ntrips;
-	struct thermal_trip *trips;
-
-	/* cooling binding data */
-	int num_tbps;
-	struct __thermal_bind_params *tbps;
-
-	/* sensor interface */
-	void *sensor_data;
-	const struct thermal_zone_of_device_ops *ops;
-};
-
-/***   DT thermal zone device callbacks   ***/
-
-static int of_thermal_get_temp(struct thermal_zone_device *tz,
-			       int *temp)
-{
-	struct __thermal_zone *data = tz->devdata;
-
-	if (!data->ops || !data->ops->get_temp)
-		return -EINVAL;
-
-	return data->ops->get_temp(data->sensor_data, temp);
-}
-
-static int of_thermal_set_trips(struct thermal_zone_device *tz,
-				int low, int high)
-{
-	struct __thermal_zone *data = tz->devdata;
-
-	if (!data->ops || !data->ops->set_trips)
-		return -EINVAL;
-
-	return data->ops->set_trips(data->sensor_data, low, high);
-}
-
 /**
  * of_thermal_get_ntrips - function to export number of available trip
  *			   points.
@@ -158,114 +71,6 @@ of_thermal_get_trip_points(struct thermal_zone_device *tz)
 }
 EXPORT_SYMBOL_GPL(of_thermal_get_trip_points);
 
-/**
- * of_thermal_set_emul_temp - function to set emulated temperature
- *
- * @tz:	pointer to a thermal zone
- * @temp:	temperature to set
- *
- * This function gives the ability to set emulated value of temperature,
- * which is handy for debugging
- *
- * Return: zero on success, error code otherwise
- */
-static int of_thermal_set_emul_temp(struct thermal_zone_device *tz,
-				    int temp)
-{
-	struct __thermal_zone *data = tz->devdata;
-
-	if (!data->ops || !data->ops->set_emul_temp)
-		return -EINVAL;
-
-	return data->ops->set_emul_temp(data->sensor_data, temp);
-}
-
-static int of_thermal_get_trend(struct thermal_zone_device *tz, int trip,
-				enum thermal_trend *trend)
-{
-	struct __thermal_zone *data = tz->devdata;
-
-	if (!data->ops || !data->ops->get_trend)
-		return -EINVAL;
-
-	return data->ops->get_trend(data->sensor_data, trip, trend);
-}
-
-static int of_thermal_change_mode(struct thermal_zone_device *tz,
-				enum thermal_device_mode mode)
-{
-	struct __thermal_zone *data = tz->devdata;
-
-	return data->ops->change_mode(data->sensor_data, mode);
-}
-
-static int of_thermal_bind(struct thermal_zone_device *thermal,
-			   struct thermal_cooling_device *cdev)
-{
-	struct __thermal_zone *data = thermal->devdata;
-	struct __thermal_bind_params *tbp;
-	struct __thermal_cooling_bind_param *tcbp;
-	int i, j;
-
-	if (!data || IS_ERR(data))
-		return -ENODEV;
-
-	/* find where to bind */
-	for (i = 0; i < data->num_tbps; i++) {
-		tbp = data->tbps + i;
-
-		for (j = 0; j < tbp->count; j++) {
-			tcbp = tbp->tcbp + j;
-
-			if (tcbp->cooling_device == cdev->np) {
-				int ret;
-
-				ret = thermal_zone_bind_cooling_device(thermal,
-						tbp->trip_id, cdev,
-						tcbp->max,
-						tcbp->min,
-						tbp->usage);
-				if (ret)
-					return ret;
-			}
-		}
-	}
-
-	return 0;
-}
-
-static int of_thermal_unbind(struct thermal_zone_device *thermal,
-			     struct thermal_cooling_device *cdev)
-{
-	struct __thermal_zone *data = thermal->devdata;
-	struct __thermal_bind_params *tbp;
-	struct __thermal_cooling_bind_param *tcbp;
-	int i, j;
-
-	if (!data || IS_ERR(data))
-		return -ENODEV;
-
-	/* find where to unbind */
-	for (i = 0; i < data->num_tbps; i++) {
-		tbp = data->tbps + i;
-
-		for (j = 0; j < tbp->count; j++) {
-			tcbp = tbp->tcbp + j;
-
-			if (tcbp->cooling_device == cdev->np) {
-				int ret;
-
-				ret = thermal_zone_unbind_cooling_device(thermal,
-							tbp->trip_id, cdev);
-				if (ret)
-					return ret;
-			}
-		}
-	}
-
-	return 0;
-}
-
 static int of_thermal_get_trip_type(struct thermal_zone_device *tz, int trip,
 				    enum thermal_trip_type *type)
 {
@@ -288,28 +93,6 @@ static int of_thermal_get_trip_temp(struct thermal_zone_device *tz, int trip,
 	return 0;
 }
 
-static int of_thermal_set_trip_temp(struct thermal_zone_device *tz, int trip,
-				    int temp)
-{
-	struct __thermal_zone *data = tz->devdata;
-
-	if (trip >= tz->num_trips || trip < 0)
-		return -EDOM;
-
-	if (data->ops && data->ops->set_trip_temp) {
-		int ret;
-
-		ret = data->ops->set_trip_temp(data->sensor_data, trip, temp);
-		if (ret)
-			return ret;
-	}
-
-	/* thermal framework should take care of data->mask & (1 << trip) */
-	tz->trips[trip].temperature = temp;
-
-	return 0;
-}
-
 static int of_thermal_get_trip_hyst(struct thermal_zone_device *tz, int trip,
 				    int *hyst)
 {
@@ -347,62 +130,6 @@ static int of_thermal_get_crit_temp(struct thermal_zone_device *tz,
 	return -EINVAL;
 }
 
-static struct thermal_zone_device_ops of_thermal_ops = {
-	.get_trip_type = of_thermal_get_trip_type,
-	.get_trip_temp = of_thermal_get_trip_temp,
-	.set_trip_temp = of_thermal_set_trip_temp,
-	.get_trip_hyst = of_thermal_get_trip_hyst,
-	.set_trip_hyst = of_thermal_set_trip_hyst,
-	.get_crit_temp = of_thermal_get_crit_temp,
-
-	.bind = of_thermal_bind,
-	.unbind = of_thermal_unbind,
-};
-
-/***   sensor API   ***/
-
-static struct thermal_zone_device *
-thermal_zone_of_add_sensor(struct device_node *zone,
-			   struct device_node *sensor, void *data,
-			   const struct thermal_zone_of_device_ops *ops)
-{
-	struct thermal_zone_device *tzd;
-	struct __thermal_zone *tz;
-
-	tzd = thermal_zone_get_zone_by_name(zone->name);
-	if (IS_ERR(tzd))
-		return ERR_PTR(-EPROBE_DEFER);
-
-	tz = tzd->devdata;
-
-	if (!ops)
-		return ERR_PTR(-EINVAL);
-
-	mutex_lock(&tzd->lock);
-	tz->ops = ops;
-	tz->sensor_data = data;
-
-	tzd->ops->get_temp = of_thermal_get_temp;
-	tzd->ops->get_trend = of_thermal_get_trend;
-
-	/*
-	 * The thermal zone core will calculate the window if they have set the
-	 * optional set_trips pointer.
-	 */
-	if (ops->set_trips)
-		tzd->ops->set_trips = of_thermal_set_trips;
-
-	if (ops->set_emul_temp)
-		tzd->ops->set_emul_temp = of_thermal_set_emul_temp;
-
-	if (ops->change_mode)
-		tzd->ops->change_mode = of_thermal_change_mode;
-
-	mutex_unlock(&tzd->lock);
-
-	return tzd;
-}
-
 /**
  * thermal_zone_of_get_sensor_id - get sensor ID from a DT thermal zone
  * @tz_np: a valid thermal zone device node.
@@ -447,207 +174,6 @@ int thermal_zone_of_get_sensor_id(struct device_node *tz_np,
 }
 EXPORT_SYMBOL_GPL(thermal_zone_of_get_sensor_id);
 
-/**
- * thermal_zone_of_sensor_register - registers a sensor to a DT thermal zone
- * @dev: a valid struct device pointer of a sensor device. Must contain
- *       a valid .of_node, for the sensor node.
- * @sensor_id: a sensor identifier, in case the sensor IP has more
- *             than one sensors
- * @data: a private pointer (owned by the caller) that will be passed
- *        back, when a temperature reading is needed.
- * @ops: struct thermal_zone_of_device_ops *. Must contain at least .get_temp.
- *
- * This function will search the list of thermal zones described in device
- * tree and look for the zone that refer to the sensor device pointed by
- * @dev->of_node as temperature providers. For the zone pointing to the
- * sensor node, the sensor will be added to the DT thermal zone device.
- *
- * The thermal zone temperature is provided by the @get_temp function
- * pointer. When called, it will have the private pointer @data back.
- *
- * The thermal zone temperature trend is provided by the @get_trend function
- * pointer. When called, it will have the private pointer @data back.
- *
- * TODO:
- * 01 - This function must enqueue the new sensor instead of using
- * it as the only source of temperature values.
- *
- * 02 - There must be a way to match the sensor with all thermal zones
- * that refer to it.
- *
- * Return: On success returns a valid struct thermal_zone_device,
- * otherwise, it returns a corresponding ERR_PTR(). Caller must
- * check the return value with help of IS_ERR() helper.
- */
-struct thermal_zone_device *
-thermal_zone_of_sensor_register(struct device *dev, int sensor_id, void *data,
-				const struct thermal_zone_of_device_ops *ops)
-{
-	struct device_node *np, *child, *sensor_np;
-	struct thermal_zone_device *tzd = ERR_PTR(-ENODEV);
-
-	np = of_find_node_by_name(NULL, "thermal-zones");
-	if (!np)
-		return ERR_PTR(-ENODEV);
-
-	if (!dev || !dev->of_node) {
-		of_node_put(np);
-		return ERR_PTR(-ENODEV);
-	}
-
-	sensor_np = of_node_get(dev->of_node);
-
-	for_each_available_child_of_node(np, child) {
-		int ret, id;
-
-		/* For now, thermal framework supports only 1 sensor per zone */
-		ret = thermal_zone_of_get_sensor_id(child, sensor_np, &id);
-		if (ret)
-			continue;
-
-		if (id == sensor_id) {
-			tzd = thermal_zone_of_add_sensor(child, sensor_np,
-							 data, ops);
-			if (!IS_ERR(tzd))
-				thermal_zone_device_enable(tzd);
-
-			of_node_put(child);
-			goto exit;
-		}
-	}
-exit:
-	of_node_put(sensor_np);
-	of_node_put(np);
-
-	return tzd;
-}
-EXPORT_SYMBOL_GPL(thermal_zone_of_sensor_register);
-
-/**
- * thermal_zone_of_sensor_unregister - unregisters a sensor from a DT thermal zone
- * @dev: a valid struct device pointer of a sensor device. Must contain
- *       a valid .of_node, for the sensor node.
- * @tzd: a pointer to struct thermal_zone_device where the sensor is registered.
- *
- * This function removes the sensor callbacks and private data from the
- * thermal zone device registered with thermal_zone_of_sensor_register()
- * API. It will also silent the zone by remove the .get_temp() and .get_trend()
- * thermal zone device callbacks.
- *
- * TODO: When the support to several sensors per zone is added, this
- * function must search the sensor list based on @dev parameter.
- *
- */
-void thermal_zone_of_sensor_unregister(struct device *dev,
-				       struct thermal_zone_device *tzd)
-{
-	struct __thermal_zone *tz;
-
-	if (!dev || !tzd || !tzd->devdata)
-		return;
-
-	tz = tzd->devdata;
-
-	/* no __thermal_zone, nothing to be done */
-	if (!tz)
-		return;
-
-	/* stop temperature polling */
-	thermal_zone_device_disable(tzd);
-
-	mutex_lock(&tzd->lock);
-	tzd->ops->get_temp = NULL;
-	tzd->ops->get_trend = NULL;
-	tzd->ops->set_emul_temp = NULL;
-	tzd->ops->change_mode = NULL;
-
-	tz->ops = NULL;
-	tz->sensor_data = NULL;
-	mutex_unlock(&tzd->lock);
-}
-EXPORT_SYMBOL_GPL(thermal_zone_of_sensor_unregister);
-
-static void devm_thermal_zone_of_sensor_release(struct device *dev, void *res)
-{
-	thermal_zone_of_sensor_unregister(dev,
-					  *(struct thermal_zone_device **)res);
-}
-
-static int devm_thermal_zone_of_sensor_match(struct device *dev, void *res,
-					     void *data)
-{
-	struct thermal_zone_device **r = res;
-
-	if (WARN_ON(!r || !*r))
-		return 0;
-
-	return *r == data;
-}
-
-/**
- * devm_thermal_zone_of_sensor_register - Resource managed version of
- *				thermal_zone_of_sensor_register()
- * @dev: a valid struct device pointer of a sensor device. Must contain
- *       a valid .of_node, for the sensor node.
- * @sensor_id: a sensor identifier, in case the sensor IP has more
- *	       than one sensors
- * @data: a private pointer (owned by the caller) that will be passed
- *	  back, when a temperature reading is needed.
- * @ops: struct thermal_zone_of_device_ops *. Must contain at least .get_temp.
- *
- * Refer thermal_zone_of_sensor_register() for more details.
- *
- * Return: On success returns a valid struct thermal_zone_device,
- * otherwise, it returns a corresponding ERR_PTR(). Caller must
- * check the return value with help of IS_ERR() helper.
- * Registered thermal_zone_device device will automatically be
- * released when device is unbounded.
- */
-struct thermal_zone_device *devm_thermal_zone_of_sensor_register(
-	struct device *dev, int sensor_id,
-	void *data, const struct thermal_zone_of_device_ops *ops)
-{
-	struct thermal_zone_device **ptr, *tzd;
-
-	ptr = devres_alloc(devm_thermal_zone_of_sensor_release, sizeof(*ptr),
-			   GFP_KERNEL);
-	if (!ptr)
-		return ERR_PTR(-ENOMEM);
-
-	tzd = thermal_zone_of_sensor_register(dev, sensor_id, data, ops);
-	if (IS_ERR(tzd)) {
-		devres_free(ptr);
-		return tzd;
-	}
-
-	*ptr = tzd;
-	devres_add(dev, ptr);
-
-	return tzd;
-}
-EXPORT_SYMBOL_GPL(devm_thermal_zone_of_sensor_register);
-
-/**
- * devm_thermal_zone_of_sensor_unregister - Resource managed version of
- *				thermal_zone_of_sensor_unregister().
- * @dev: Device for which which resource was allocated.
- * @tzd: a pointer to struct thermal_zone_device where the sensor is registered.
- *
- * This function removes the sensor callbacks and private data from the
- * thermal zone device registered with devm_thermal_zone_of_sensor_register()
- * API. It will also silent the zone by remove the .get_temp() and .get_trend()
- * thermal zone device callbacks.
- * Normally this function will not need to be called and the resource
- * management code will ensure that the resource is freed.
- */
-void devm_thermal_zone_of_sensor_unregister(struct device *dev,
-					    struct thermal_zone_device *tzd)
-{
-	WARN_ON(devres_release(dev, devm_thermal_zone_of_sensor_release,
-			       devm_thermal_zone_of_sensor_match, tzd));
-}
-EXPORT_SYMBOL_GPL(devm_thermal_zone_of_sensor_unregister);
-
 /***   functions parsing device tree nodes   ***/
 
 static int of_find_trip_id(struct device_node *np, struct device_node *trip)
@@ -679,98 +205,6 @@ out:
 	return i;
 }
 
-/**
- * thermal_of_populate_bind_params - parse and fill cooling map data
- * @np: DT node containing a cooling-map node
- * @__tbp: data structure to be filled with cooling map info
- * @trips: array of thermal zone trip points
- * @ntrips: number of trip points inside trips.
- *
- * This function parses a cooling-map type of node represented by
- * @np parameter and fills the read data into @__tbp data structure.
- * It needs the already parsed array of trip points of the thermal zone
- * in consideration.
- *
- * Return: 0 on success, proper error code otherwise
- */
-static int thermal_of_populate_bind_params(struct device_node *tz_np,
-					   struct device_node *np,
-					   struct __thermal_bind_params *__tbp)
-{
-	struct of_phandle_args cooling_spec;
-	struct __thermal_cooling_bind_param *__tcbp;
-	struct device_node *trip;
-	int ret, i, count;
-	int trip_id;
-	u32 prop;
-
-	/* Default weight. Usage is optional */
-	__tbp->usage = THERMAL_WEIGHT_DEFAULT;
-	ret = of_property_read_u32(np, "contribution", &prop);
-	if (ret == 0)
-		__tbp->usage = prop;
-
-	trip = of_parse_phandle(np, "trip", 0);
-	if (!trip) {
-		pr_err("missing trip property\n");
-		return -ENODEV;
-	}
-
-	trip_id = of_find_trip_id(tz_np, trip);
-	if (trip_id < 0) {
-		ret = trip_id;
-		goto end;
-	}
-
-	__tbp->trip_id = trip_id;
-
-	count = of_count_phandle_with_args(np, "cooling-device",
-					   "#cooling-cells");
-	if (count <= 0) {
-		pr_err("Add a cooling_device property with at least one device\n");
-		ret = -ENOENT;
-		goto end;
-	}
-
-	__tcbp = kcalloc(count, sizeof(*__tcbp), GFP_KERNEL);
-	if (!__tcbp) {
-		ret = -ENOMEM;
-		goto end;
-	}
-
-	for (i = 0; i < count; i++) {
-		ret = of_parse_phandle_with_args(np, "cooling-device",
-				"#cooling-cells", i, &cooling_spec);
-		if (ret < 0) {
-			pr_err("Invalid cooling-device entry\n");
-			goto free_tcbp;
-		}
-
-		__tcbp[i].cooling_device = cooling_spec.np;
-
-		if (cooling_spec.args_count >= 2) { /* at least min and max */
-			__tcbp[i].min = cooling_spec.args[0];
-			__tcbp[i].max = cooling_spec.args[1];
-		} else {
-			pr_err("wrong reference to cooling device, missing limits\n");
-		}
-	}
-
-	__tbp->tcbp = __tcbp;
-	__tbp->count = count;
-
-	goto end;
-
-free_tcbp:
-	for (i = i - 1; i >= 0; i--)
-		of_node_put(__tcbp[i].cooling_device);
-	kfree(__tcbp);
-end:
-	of_node_put(trip);
-
-	return ret;
-}
-
 /*
  * It maps 'enum thermal_trip_type' found in include/linux/thermal.h
  * into the device tree binding of 'trip', property type.
@@ -811,16 +245,6 @@ static int thermal_of_get_trip_type(struct device_node *np,
 	return -ENODEV;
 }
 
-/**
- * thermal_of_populate_trip - parse and fill one trip point data
- * @np: DT node containing a trip point node
- * @trip: trip point data structure to be filled up
- *
- * This function parses a trip point type of node represented by
- * @np parameter and fills the read data into @trip data structure.
- *
- * Return: 0 on success, proper error code otherwise
- */
 static int thermal_of_populate_trip(struct device_node *np,
 				    struct thermal_trip *trip)
 {
@@ -897,258 +321,458 @@ out_of_node_put:
 	return ERR_PTR(ret);
 }
 
-/**
- * thermal_of_build_thermal_zone - parse and fill one thermal zone data
- * @np: DT node containing a thermal zone node
- *
- * This function parses a thermal zone type of node represented by
- * @np parameter and fills the read data into a __thermal_zone data structure
- * and return this pointer.
- *
- * TODO: Missing properties to parse: thermal-sensor-names
- *
- * Return: On success returns a valid struct __thermal_zone,
- * otherwise, it returns a corresponding ERR_PTR(). Caller must
- * check the return value with help of IS_ERR() helper.
- */
-static struct __thermal_zone
-__init *thermal_of_build_thermal_zone(struct device_node *np)
+static struct device_node *of_thermal_zone_find(struct device_node *sensor, int id)
 {
-	struct device_node *child = NULL, *gchild;
-	struct __thermal_zone *tz;
-	int ret, i;
-	u32 prop, coef[2];
+	struct device_node *np, *tz;
+	struct of_phandle_args sensor_specs;
 
+	np = of_find_node_by_name(NULL, "thermal-zones");
 	if (!np) {
-		pr_err("no thermal zone np\n");
-		return ERR_PTR(-EINVAL);
+		pr_debug("No thermal zones description\n");
+		return ERR_PTR(-ENODEV);
 	}
 
-	tz = kzalloc(sizeof(*tz), GFP_KERNEL);
-	if (!tz)
-		return ERR_PTR(-ENOMEM);
+	/*
+	 * Search for each thermal zone, a defined sensor
+	 * corresponding to the one passed as parameter
+	 */
+	for_each_available_child_of_node(np, tz) {
+
+		int count, i;
+
+		count = of_count_phandle_with_args(tz, "thermal-sensors",
+						   "#thermal-sensor-cells");
+		if (count <= 0) {
+			pr_err("%pOFn: missing thermal sensor\n", tz);
+			tz = ERR_PTR(-EINVAL);
+			goto out;
+		}
+
+		for (i = 0; i < count; i++) {
+
+			int ret;
 
-	ret = of_property_read_u32(np, "polling-delay-passive", &prop);
+			ret = of_parse_phandle_with_args(tz, "thermal-sensors",
+							 "#thermal-sensor-cells",
+							 i, &sensor_specs);
+			if (ret < 0) {
+				pr_err("%pOFn: Failed to read thermal-sensors cells: %d\n", tz, ret);
+				tz = ERR_PTR(ret);
+				goto out;
+			}
+
+			if ((sensor == sensor_specs.np) && id == (sensor_specs.args_count ?
+								  sensor_specs.args[0] : 0)) {
+				pr_debug("sensor %pOFn id=%d belongs to %pOFn\n", sensor, id, tz);
+				goto out;
+			}
+		}
+	}
+	tz = ERR_PTR(-ENODEV);
+out:
+	of_node_put(np);
+	return tz;
+}
+
+static int thermal_of_monitor_init(struct device_node *np, int *delay, int *pdelay)
+{
+	int ret;
+
+	ret = of_property_read_u32(np, "polling-delay-passive", pdelay);
 	if (ret < 0) {
 		pr_err("%pOFn: missing polling-delay-passive property\n", np);
-		goto free_tz;
+		return ret;
 	}
-	tz->passive_delay = prop;
 
-	ret = of_property_read_u32(np, "polling-delay", &prop);
+	ret = of_property_read_u32(np, "polling-delay", delay);
 	if (ret < 0) {
 		pr_err("%pOFn: missing polling-delay property\n", np);
-		goto free_tz;
+		return ret;
 	}
-	tz->polling_delay = prop;
+
+	return 0;
+}
+
+static struct thermal_zone_params *thermal_of_parameters_init(struct device_node *np)
+{
+	struct thermal_zone_params *tzp;
+	int coef[2];
+	int ncoef = ARRAY_SIZE(coef);
+	int prop, ret;
+
+	tzp = kzalloc(sizeof(*tzp), GFP_KERNEL);
+	if (!tzp)
+		return ERR_PTR(-ENOMEM);
+
+	tzp->no_hwmon = true;
+
+	if (!of_property_read_u32(np, "sustainable-power", &prop))
+		tzp->sustainable_power = prop;
 
 	/*
-	 * REVIST: for now, the thermal framework supports only
-	 * one sensor per thermal zone. Thus, we are considering
-	 * only the first two values as slope and offset.
+	 * For now, the thermal framework supports only one sensor per
+	 * thermal zone. Thus, we are considering only the first two
+	 * values as slope and offset.
 	 */
-	ret = of_property_read_u32_array(np, "coefficients", coef, 2);
-	if (ret == 0) {
-		tz->slope = coef[0];
-		tz->offset = coef[1];
-	} else {
-		tz->slope = 1;
-		tz->offset = 0;
+	ret = of_property_read_u32_array(np, "coefficients", coef, ncoef);
+	if (ret) {
+		coef[0] = 1;
+		coef[1] = 0;
+	}
+
+	tzp->slope = coef[0];
+	tzp->offset = coef[1];
+
+	return tzp;
+}
+
+static struct device_node *thermal_of_zone_get_by_name(struct thermal_zone_device *tz)
+{
+	struct device_node *np, *tz_np;
+
+	np = of_find_node_by_name(NULL, "thermal-zones");
+	if (!np)
+		return ERR_PTR(-ENODEV);
+
+	tz_np = of_get_child_by_name(np, tz->type);
+
+	of_node_put(np);
+
+	if (!tz_np)
+		return ERR_PTR(-ENODEV);
+
+	return tz_np;
+}
+
+static int __thermal_of_unbind(struct device_node *map_np, int index, int trip_id,
+			       struct thermal_zone_device *tz, struct thermal_cooling_device *cdev)
+{
+	struct of_phandle_args cooling_spec;
+	int ret;
+
+	ret = of_parse_phandle_with_args(map_np, "cooling-device", "#cooling-cells",
+					 index, &cooling_spec);
+
+	of_node_put(cooling_spec.np);
+
+	if (ret < 0) {
+		pr_err("Invalid cooling-device entry\n");
+		return ret;
 	}
 
-	tz->trips = thermal_of_trips_init(np, &tz->ntrips);
-	if (IS_ERR(tz->trips)) {
-		ret = PTR_ERR(tz->trips);
-		goto finish;
+	if (cooling_spec.args_count < 2) {
+		pr_err("wrong reference to cooling device, missing limits\n");
+		return -EINVAL;
 	}
 
-	/* cooling-maps */
-	child = of_get_child_by_name(np, "cooling-maps");
+	if (cooling_spec.np != cdev->np)
+		return 0;
 
-	/* cooling-maps not provided */
-	if (!child)
-		goto finish;
+	ret = thermal_zone_unbind_cooling_device(tz, trip_id, cdev);
+	if (ret)
+		pr_err("Failed to unbind '%s' with '%s': %d\n", tz->type, cdev->type, ret);
 
-	tz->num_tbps = of_get_child_count(child);
-	if (tz->num_tbps == 0)
-		goto finish;
+	return ret;
+}
 
-	tz->tbps = kcalloc(tz->num_tbps, sizeof(*tz->tbps), GFP_KERNEL);
-	if (!tz->tbps) {
-		ret = -ENOMEM;
-		goto free_trips;
+static int __thermal_of_bind(struct device_node *map_np, int index, int trip_id,
+			     struct thermal_zone_device *tz, struct thermal_cooling_device *cdev)
+{
+	struct of_phandle_args cooling_spec;
+	int ret, weight = THERMAL_WEIGHT_DEFAULT;
+
+	of_property_read_u32(map_np, "contribution", &weight);
+
+	ret = of_parse_phandle_with_args(map_np, "cooling-device", "#cooling-cells",
+					 index, &cooling_spec);
+
+	of_node_put(cooling_spec.np);
+
+	if (ret < 0) {
+		pr_err("Invalid cooling-device entry\n");
+		return ret;
 	}
 
-	i = 0;
-	for_each_child_of_node(child, gchild) {
-		ret = thermal_of_populate_bind_params(np, gchild, &tz->tbps[i++]);
-		if (ret) {
-			of_node_put(gchild);
-			goto free_tbps;
-		}
+	if (cooling_spec.args_count < 2) {
+		pr_err("wrong reference to cooling device, missing limits\n");
+		return -EINVAL;
 	}
 
-finish:
-	of_node_put(child);
+	if (cooling_spec.np != cdev->np)
+		return 0;
+
+	ret = thermal_zone_bind_cooling_device(tz, trip_id, cdev, cooling_spec.args[1],
+					       cooling_spec.args[0],
+					       weight);
+	if (ret)
+		pr_err("Failed to bind '%s' with '%s': %d\n", tz->type, cdev->type, ret);
 
-	return tz;
+	return ret;
+}
 
-free_tbps:
-	for (i = i - 1; i >= 0; i--) {
-		struct __thermal_bind_params *tbp = tz->tbps + i;
-		int j;
+static int thermal_of_for_each_cooling_device(struct device_node *tz_np, struct device_node *map_np,
+					      struct thermal_zone_device *tz, struct thermal_cooling_device *cdev,
+					      int (*action)(struct device_node *, int, int,
+							    struct thermal_zone_device *, struct thermal_cooling_device *))
+{
+	struct device_node *tr_np;
+	int count, i, trip_id;
 
-		for (j = 0; j < tbp->count; j++)
-			of_node_put(tbp->tcbp[j].cooling_device);
+	tr_np = of_parse_phandle(map_np, "trip", 0);
+	if (!tr_np)
+		return -ENODEV;
 
-		kfree(tbp->tcbp);
+	trip_id = of_find_trip_id(tz_np, tr_np);
+	if (trip_id < 0)
+		return trip_id;
+
+	count = of_count_phandle_with_args(map_np, "cooling-device", "#cooling-cells");
+	if (count <= 0) {
+		pr_err("Add a cooling_device property with at least one device\n");
+		return -ENOENT;
 	}
 
-	kfree(tz->tbps);
-free_trips:
-	kfree(tz->trips);
-free_tz:
-	kfree(tz);
-	of_node_put(child);
+	/*
+	 * At this point, we don't want to bail out when there is an
+	 * error, we will try to bind/unbind as many as possible
+	 * cooling devices
+	 */
+	for (i = 0; i < count; i++)
+		action(map_np, i, trip_id, tz, cdev);
 
-	return ERR_PTR(ret);
+	return 0;
 }
 
-static __init void of_thermal_free_zone(struct __thermal_zone *tz)
+static int thermal_of_for_each_cooling_maps(struct thermal_zone_device *tz,
+					    struct thermal_cooling_device *cdev,
+					    int (*action)(struct device_node *, int, int,
+							  struct thermal_zone_device *, struct thermal_cooling_device *))
 {
-	struct __thermal_bind_params *tbp;
-	int i, j;
+	struct device_node *tz_np, *cm_np, *child;
+	int ret = 0;
 
-	for (i = 0; i < tz->num_tbps; i++) {
-		tbp = tz->tbps + i;
+	tz_np = thermal_of_zone_get_by_name(tz);
+	if (IS_ERR(tz_np)) {
+		pr_err("Failed to get node tz by name\n");
+		return PTR_ERR(tz_np);
+	}
 
-		for (j = 0; j < tbp->count; j++)
-			of_node_put(tbp->tcbp[j].cooling_device);
+	cm_np = of_get_child_by_name(tz_np, "cooling-maps");
+	if (!cm_np)
+		goto out;
 
-		kfree(tbp->tcbp);
+	for_each_child_of_node(cm_np, child) {
+		ret = thermal_of_for_each_cooling_device(tz_np, child, tz, cdev, action);
+		if (ret)
+			break;
 	}
 
-	kfree(tz->tbps);
-	kfree(tz->trips);
-	kfree(tz);
+	of_node_put(cm_np);
+out:
+	of_node_put(tz_np);
+
+	return ret;
+}
+
+static int thermal_of_bind(struct thermal_zone_device *tz,
+			   struct thermal_cooling_device *cdev)
+{
+	return thermal_of_for_each_cooling_maps(tz, cdev, __thermal_of_bind);
+}
+
+static int thermal_of_unbind(struct thermal_zone_device *tz,
+			     struct thermal_cooling_device *cdev)
+{
+	return thermal_of_for_each_cooling_maps(tz, cdev, __thermal_of_unbind);
 }
 
 /**
- * of_thermal_destroy_zones - remove all zones parsed and allocated resources
+ * thermal_of_zone_unregister - Cleanup the specific allocated ressources
  *
- * Finds all zones parsed and added to the thermal framework and remove them
- * from the system, together with their resources.
+ * This function disables the thermal zone and frees the different
+ * ressources allocated specific to the thermal OF.
  *
+ * @tz: a pointer to the thermal zone structure
  */
-static __init void of_thermal_destroy_zones(void)
+void thermal_of_zone_unregister(struct thermal_zone_device *tz)
 {
-	struct device_node *np, *child;
+	struct thermal_trip *trips = tz->trips;
+	struct thermal_zone_params *tzp = tz->tzp;
+	struct thermal_zone_device_ops *ops = tz->ops;
 
-	np = of_find_node_by_name(NULL, "thermal-zones");
-	if (!np) {
-		pr_debug("unable to find thermal zones\n");
-		return;
+	thermal_zone_device_disable(tz);
+	thermal_zone_device_unregister(tz);
+	kfree(trips);
+	kfree(tzp);
+	kfree(ops);
+}
+EXPORT_SYMBOL_GPL(thermal_of_zone_unregister);
+
+/**
+ * thermal_of_zone_register - Register a thermal zone with device node
+ * sensor
+ *
+ * The thermal_of_zone_register() parses a device tree given a device
+ * node sensor and identifier. It searches for the thermal zone
+ * associated to the couple sensor/id and retrieves all the thermal
+ * zone properties and registers new thermal zone with those
+ * properties.
+ *
+ * @sensor: A device node pointer corresponding to the sensor in the device tree
+ * @id: An integer as sensor identifier
+ * @data: A private data to be stored in the thermal zone dedicated private area
+ * @ops: A set of thermal sensor ops
+ *
+ * Return: a valid thermal zone structure pointer on success.
+ * 	- EINVAL: if the device tree thermal description is malformed
+ *	- ENOMEM: if one structure can not be allocated
+ *	- Other negative errors are returned by the underlying called functions
+ */
+struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data,
+						     const struct thermal_zone_device_ops *ops)
+{
+	struct thermal_zone_device *tz;
+	struct thermal_trip *trips;
+	struct thermal_zone_params *tzp;
+	struct thermal_zone_device_ops *of_ops;
+	struct device_node *np;
+	int delay, pdelay;
+	int ntrips, mask;
+	int ret;
+
+	of_ops = kmemdup(ops, sizeof(*ops), GFP_KERNEL);
+	if (!of_ops)
+		return ERR_PTR(-ENOMEM);
+
+	np = of_thermal_zone_find(sensor, id);
+	if (IS_ERR(np)) {
+		if (PTR_ERR(np) != -ENODEV)
+			pr_err("Failed to find thermal zone for %pOFn id=%d\n", sensor, id);
+		return ERR_CAST(np);
 	}
 
-	for_each_available_child_of_node(np, child) {
-		struct thermal_zone_device *zone;
+	trips = thermal_of_trips_init(np, &ntrips);
+	if (IS_ERR(trips)) {
+		pr_err("Failed to find trip points for %pOFn id=%d\n", sensor, id);
+		return ERR_CAST(trips);
+	}
 
-		zone = thermal_zone_get_zone_by_name(child->name);
-		if (IS_ERR(zone))
-			continue;
+	ret = thermal_of_monitor_init(np, &delay, &pdelay);
+	if (ret) {
+		pr_err("Failed to initialize monitoring delays from %pOFn\n", np);
+		goto out_kfree_trips;
+	}
 
-		thermal_zone_device_unregister(zone);
-		kfree(zone->tzp);
-		kfree(zone->ops);
-		of_thermal_free_zone(zone->devdata);
+	tzp = thermal_of_parameters_init(np);
+	if (IS_ERR(tzp)) {
+		ret = PTR_ERR(tzp);
+		pr_err("Failed to initialize parameter from %pOFn: %d\n", np, ret);
+		goto out_kfree_trips;
 	}
-	of_node_put(np);
+
+	of_ops->get_trip_type = of_ops->get_trip_type ? : of_thermal_get_trip_type;
+	of_ops->get_trip_temp = of_ops->get_trip_temp ? : of_thermal_get_trip_temp;
+	of_ops->get_trip_hyst = of_ops->get_trip_hyst ? : of_thermal_get_trip_hyst;
+	of_ops->set_trip_hyst = of_ops->set_trip_hyst ? : of_thermal_set_trip_hyst;
+	of_ops->get_crit_temp = of_ops->get_crit_temp ? : of_thermal_get_crit_temp;
+	of_ops->bind = thermal_of_bind;
+	of_ops->unbind = thermal_of_unbind;
+
+	mask = GENMASK_ULL((ntrips) - 1, 0);
+
+	tz = thermal_zone_device_register_with_trips(np->name, trips, ntrips,
+						     mask, data, of_ops, tzp,
+						     pdelay, delay);
+	if (IS_ERR(tz)) {
+		ret = PTR_ERR(tz);
+		pr_err("Failed to register thermal zone %pOFn: %d\n", np, ret);
+		goto out_kfree_tzp;
+	}
+
+	ret = thermal_zone_device_enable(tz);
+	if (ret) {
+		pr_err("Failed to enabled thermal zone '%s', id=%d: %d\n",
+		       tz->type, tz->id, ret);
+		thermal_of_zone_unregister(tz);
+		return ERR_PTR(ret);
+	}
+
+	return tz;
+
+out_kfree_tzp:
+	kfree(tzp);
+out_kfree_trips:
+	kfree(trips);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(thermal_of_zone_register);
+
+static void devm_thermal_of_zone_release(struct device *dev, void *res)
+{
+	thermal_of_zone_unregister(*(struct thermal_zone_device **)res);
+}
+
+static int devm_thermal_of_zone_match(struct device *dev, void *res,
+				      void *data)
+{
+	struct thermal_zone_device **r = res;
+
+	if (WARN_ON(!r || !*r))
+		return 0;
+
+	return *r == data;
 }
 
 /**
- * of_parse_thermal_zones - parse device tree thermal data
+ * devm_thermal_of_zone_register - register a thermal tied with the sensor life cycle
  *
- * Initialization function that can be called by machine initialization
- * code to parse thermal data and populate the thermal framework
- * with hardware thermal zones info. This function only parses thermal zones.
- * Cooling devices and sensor devices nodes are supposed to be parsed
- * by their respective drivers.
- *
- * Return: 0 on success, proper error code otherwise
+ * This function is the device version of the thermal_of_zone_register() function.
  *
+ * @dev: a device structure pointer to sensor to be tied with the thermal zone OF life cycle
+ * @sensor_id: the sensor identifier
+ * @data: a pointer to a private data to be stored in the thermal zone 'devdata' field
+ * @ops: a pointer to the ops structure associated with the sensor
  */
-int __init of_parse_thermal_zones(void)
+struct thermal_zone_device *devm_thermal_of_zone_register(struct device *dev, int sensor_id, void *data,
+							  const struct thermal_zone_device_ops *ops)
 {
-	struct device_node *np, *child;
-	struct __thermal_zone *tz;
-	struct thermal_zone_device_ops *ops;
-
-	np = of_find_node_by_name(NULL, "thermal-zones");
-	if (!np) {
-		pr_debug("unable to find thermal zones\n");
-		return 0; /* Run successfully on systems without thermal DT */
-	}
-
-	for_each_available_child_of_node(np, child) {
-		struct thermal_zone_device *zone;
-		struct thermal_zone_params *tzp;
-		int i, mask = 0;
-		u32 prop;
-
-		tz = thermal_of_build_thermal_zone(child);
-		if (IS_ERR(tz)) {
-			pr_err("failed to build thermal zone %pOFn: %ld\n",
-			       child,
-			       PTR_ERR(tz));
-			continue;
-		}
-
-		ops = kmemdup(&of_thermal_ops, sizeof(*ops), GFP_KERNEL);
-		if (!ops)
-			goto exit_free;
+	struct thermal_zone_device **ptr, *tzd;
 
-		tzp = kzalloc(sizeof(*tzp), GFP_KERNEL);
-		if (!tzp) {
-			kfree(ops);
-			goto exit_free;
-		}
+	ptr = devres_alloc(devm_thermal_of_zone_release, sizeof(*ptr),
+			   GFP_KERNEL);
+	if (!ptr)
+		return ERR_PTR(-ENOMEM);
 
-		/* No hwmon because there might be hwmon drivers registering */
-		tzp->no_hwmon = true;
-
-		if (!of_property_read_u32(child, "sustainable-power", &prop))
-			tzp->sustainable_power = prop;
-
-		for (i = 0; i < tz->ntrips; i++)
-			mask |= 1 << i;
-
-		/* these two are left for temperature drivers to use */
-		tzp->slope = tz->slope;
-		tzp->offset = tz->offset;
-
-		zone = thermal_zone_device_register_with_trips(child->name, tz->trips, tz->ntrips,
-							       mask, tz, ops, tzp, tz->passive_delay,
-							       tz->polling_delay);
-		if (IS_ERR(zone)) {
-			pr_err("Failed to build %pOFn zone %ld\n", child,
-			       PTR_ERR(zone));
-			kfree(tzp);
-			kfree(ops);
-			of_thermal_free_zone(tz);
-			/* attempting to build remaining zones still */
-		}
+	tzd = thermal_of_zone_register(dev->of_node, sensor_id, data, ops);
+	if (IS_ERR(tzd)) {
+		devres_free(ptr);
+		return tzd;
 	}
-	of_node_put(np);
-
-	return 0;
 
-exit_free:
-	of_node_put(child);
-	of_node_put(np);
-	of_thermal_free_zone(tz);
+	*ptr = tzd;
+	devres_add(dev, ptr);
 
-	/* no memory available, so free what we have built */
-	of_thermal_destroy_zones();
+	return tzd;
+}
+EXPORT_SYMBOL_GPL(devm_thermal_of_zone_register);
 
-	return -ENOMEM;
+/**
+ * devm_thermal_of_zone_unregister - Resource managed version of
+ *				thermal_of_zone_unregister().
+ * @dev: Device for which which resource was allocated.
+ * @tz: a pointer to struct thermal_zone where the sensor is registered.
+ *
+ * This function removes the sensor callbacks and private data from the
+ * thermal zone device registered with devm_thermal_zone_of_sensor_register()
+ * API. It will also silent the zone by remove the .get_temp() and .get_trend()
+ * thermal zone device callbacks.
+ * Normally this function will not need to be called and the resource
+ * management code will ensure that the resource is freed.
+ */
+void devm_thermal_of_zone_unregister(struct device *dev, struct thermal_zone_device *tz)
+{
+	WARN_ON(devres_release(dev, devm_thermal_of_zone_release,
+			       devm_thermal_of_zone_match, tz));
 }
+EXPORT_SYMBOL_GPL(devm_thermal_of_zone_unregister);
diff --git a/drivers/thermal/thermal_sysfs.c b/drivers/thermal/thermal_sysfs.c
index 3a8d6e747c25..78c5841bdfae 100644
--- a/drivers/thermal/thermal_sysfs.c
+++ b/drivers/thermal/thermal_sysfs.c
@@ -49,7 +49,11 @@ static ssize_t
 mode_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct thermal_zone_device *tz = to_thermal_zone(dev);
-	int enabled = thermal_zone_device_is_enabled(tz);
+	int enabled;
+
+	mutex_lock(&tz->lock);
+	enabled = thermal_zone_device_is_enabled(tz);
+	mutex_unlock(&tz->lock);
 
 	return sprintf(buf, "%s\n", enabled ? "enabled" : "disabled");
 }
@@ -115,7 +119,7 @@ trip_point_temp_store(struct device *dev, struct device_attribute *attr,
 	int temperature, hyst = 0;
 	enum thermal_trip_type type;
 
-	if (!tz->ops->set_trip_temp)
+	if (!tz->ops->set_trip_temp && !tz->trips)
 		return -EPERM;
 
 	if (sscanf(attr->attr.name, "trip_point_%d_temp", &trip) != 1)
@@ -128,6 +132,9 @@ trip_point_temp_store(struct device *dev, struct device_attribute *attr,
 	if (ret)
 		return ret;
 
+	if (tz->trips)
+		tz->trips[trip].temperature = temperature;
+
 	if (tz->ops->get_trip_hyst) {
 		ret = tz->ops->get_trip_hyst(tz, trip, &hyst);
 		if (ret)
diff --git a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
index 703039d8b937..8a9055bd376e 100644
--- a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
+++ b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c
@@ -65,10 +65,10 @@ static inline int ti_thermal_hotspot_temperature(int t, int s, int c)
 
 /* thermal zone ops */
 /* Get temperature callback function for thermal zone */
-static inline int __ti_thermal_get_temp(void *devdata, int *temp)
+static inline int __ti_thermal_get_temp(struct thermal_zone_device *tz, int *temp)
 {
 	struct thermal_zone_device *pcb_tz = NULL;
-	struct ti_thermal_data *data = devdata;
+	struct ti_thermal_data *data = tz->devdata;
 	struct ti_bandgap *bgp;
 	const struct ti_temp_sensor *s;
 	int ret, tmp, slope, constant;
@@ -85,8 +85,8 @@ static inline int __ti_thermal_get_temp(void *devdata, int *temp)
 		return ret;
 
 	/* Default constants */
-	slope = thermal_zone_get_slope(data->ti_thermal);
-	constant = thermal_zone_get_offset(data->ti_thermal);
+	slope = thermal_zone_get_slope(tz);
+	constant = thermal_zone_get_offset(tz);
 
 	pcb_tz = data->pcb_tz;
 	/* In case pcb zone is available, use the extrapolation rule with it */
@@ -107,9 +107,9 @@ static inline int __ti_thermal_get_temp(void *devdata, int *temp)
 	return ret;
 }
 
-static int __ti_thermal_get_trend(void *p, int trip, enum thermal_trend *trend)
+static int __ti_thermal_get_trend(struct thermal_zone_device *tz, int trip, enum thermal_trend *trend)
 {
-	struct ti_thermal_data *data = p;
+	struct ti_thermal_data *data = tz->devdata;
 	struct ti_bandgap *bgp;
 	int id, tr, ret = 0;
 
@@ -130,7 +130,7 @@ static int __ti_thermal_get_trend(void *p, int trip, enum thermal_trend *trend)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops ti_of_thermal_ops = {
+static const struct thermal_zone_device_ops ti_of_thermal_ops = {
 	.get_temp = __ti_thermal_get_temp,
 	.get_trend = __ti_thermal_get_trend,
 };
@@ -170,7 +170,7 @@ int ti_thermal_expose_sensor(struct ti_bandgap *bgp, int id,
 		return -EINVAL;
 
 	/* in case this is specified by DT */
-	data->ti_thermal = devm_thermal_zone_of_sensor_register(bgp->dev, id,
+	data->ti_thermal = devm_thermal_of_zone_register(bgp->dev, id,
 					data, &ti_of_thermal_ops);
 	if (IS_ERR(data->ti_thermal)) {
 		dev_err(bgp->dev, "thermal zone device is NULL\n");
diff --git a/drivers/thermal/uniphier_thermal.c b/drivers/thermal/uniphier_thermal.c
index 4cae5561a2a3..4111d99ef50e 100644
--- a/drivers/thermal/uniphier_thermal.c
+++ b/drivers/thermal/uniphier_thermal.c
@@ -187,9 +187,9 @@ static void uniphier_tm_disable_sensor(struct uniphier_tm_dev *tdev)
 	usleep_range(1000, 2000);	/* The spec note says at least 1ms */
 }
 
-static int uniphier_tm_get_temp(void *data, int *out_temp)
+static int uniphier_tm_get_temp(struct thermal_zone_device *tz, int *out_temp)
 {
-	struct uniphier_tm_dev *tdev = data;
+	struct uniphier_tm_dev *tdev = tz->devdata;
 	struct regmap *map = tdev->regmap;
 	int ret;
 	u32 temp;
@@ -204,7 +204,7 @@ static int uniphier_tm_get_temp(void *data, int *out_temp)
 	return 0;
 }
 
-static const struct thermal_zone_of_device_ops uniphier_of_thermal_ops = {
+static const struct thermal_zone_device_ops uniphier_of_thermal_ops = {
 	.get_temp = uniphier_tm_get_temp,
 };
 
@@ -289,8 +289,8 @@ static int uniphier_tm_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, tdev);
 
-	tdev->tz_dev = devm_thermal_zone_of_sensor_register(dev, 0, tdev,
-						&uniphier_of_thermal_ops);
+	tdev->tz_dev = devm_thermal_of_zone_register(dev, 0, tdev,
+						     &uniphier_of_thermal_ops);
 	if (IS_ERR(tdev->tz_dev)) {
 		dev_err(dev, "failed to register sensor device\n");
 		return PTR_ERR(tdev->tz_dev);
diff --git a/drivers/thunderbolt/acpi.c b/drivers/thunderbolt/acpi.c
index b1f0dc8df47c..7a8adf5ad5a0 100644
--- a/drivers/thunderbolt/acpi.c
+++ b/drivers/thunderbolt/acpi.c
@@ -42,7 +42,7 @@ static acpi_status tb_acpi_add_link(acpi_handle handle, u32 level, void *data,
 	 */
 	dev = acpi_get_first_physical_node(adev);
 	while (!dev) {
-		adev = adev->parent;
+		adev = acpi_dev_parent(adev);
 		if (!adev)
 			break;
 		dev = acpi_get_first_physical_node(adev);
diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c
index cb8c9c4ae93a..b5cd9673e15d 100644
--- a/drivers/thunderbolt/nhi.c
+++ b/drivers/thunderbolt/nhi.c
@@ -28,7 +28,11 @@
 #define RING_TYPE(ring) ((ring)->is_tx ? "TX ring" : "RX ring")
 
 #define RING_FIRST_USABLE_HOPID	1
-
+/*
+ * Used with QUIRK_E2E to specify an unused HopID the Rx credits are
+ * transferred.
+ */
+#define RING_E2E_RESERVED_HOPID	RING_FIRST_USABLE_HOPID
 /*
  * Minimal number of vectors when we use MSI-X. Two for control channel
  * Rx/Tx and the rest four are for cross domain DMA paths.
@@ -38,7 +42,9 @@
 
 #define NHI_MAILBOX_TIMEOUT	500 /* ms */
 
+/* Host interface quirks */
 #define QUIRK_AUTO_CLEAR_INT	BIT(0)
+#define QUIRK_E2E		BIT(1)
 
 static int ring_interrupt_index(struct tb_ring *ring)
 {
@@ -458,8 +464,18 @@ static void ring_release_msix(struct tb_ring *ring)
 
 static int nhi_alloc_hop(struct tb_nhi *nhi, struct tb_ring *ring)
 {
+	unsigned int start_hop = RING_FIRST_USABLE_HOPID;
 	int ret = 0;
 
+	if (nhi->quirks & QUIRK_E2E) {
+		start_hop = RING_FIRST_USABLE_HOPID + 1;
+		if (ring->flags & RING_FLAG_E2E && !ring->is_tx) {
+			dev_dbg(&nhi->pdev->dev, "quirking E2E TX HopID %u -> %u\n",
+				ring->e2e_tx_hop, RING_E2E_RESERVED_HOPID);
+			ring->e2e_tx_hop = RING_E2E_RESERVED_HOPID;
+		}
+	}
+
 	spin_lock_irq(&nhi->lock);
 
 	if (ring->hop < 0) {
@@ -469,7 +485,7 @@ static int nhi_alloc_hop(struct tb_nhi *nhi, struct tb_ring *ring)
 		 * Automatically allocate HopID from the non-reserved
 		 * range 1 .. hop_count - 1.
 		 */
-		for (i = RING_FIRST_USABLE_HOPID; i < nhi->hop_count; i++) {
+		for (i = start_hop; i < nhi->hop_count; i++) {
 			if (ring->is_tx) {
 				if (!nhi->tx_rings[i]) {
 					ring->hop = i;
@@ -484,6 +500,11 @@ static int nhi_alloc_hop(struct tb_nhi *nhi, struct tb_ring *ring)
 		}
 	}
 
+	if (ring->hop > 0 && ring->hop < start_hop) {
+		dev_warn(&nhi->pdev->dev, "invalid hop: %d\n", ring->hop);
+		ret = -EINVAL;
+		goto err_unlock;
+	}
 	if (ring->hop < 0 || ring->hop >= nhi->hop_count) {
 		dev_warn(&nhi->pdev->dev, "invalid hop: %d\n", ring->hop);
 		ret = -EINVAL;
@@ -1097,12 +1118,26 @@ static void nhi_shutdown(struct tb_nhi *nhi)
 
 static void nhi_check_quirks(struct tb_nhi *nhi)
 {
-	/*
-	 * Intel hardware supports auto clear of the interrupt status
-	 * reqister right after interrupt is being issued.
-	 */
-	if (nhi->pdev->vendor == PCI_VENDOR_ID_INTEL)
+	if (nhi->pdev->vendor == PCI_VENDOR_ID_INTEL) {
+		/*
+		 * Intel hardware supports auto clear of the interrupt
+		 * status register right after interrupt is being
+		 * issued.
+		 */
 		nhi->quirks |= QUIRK_AUTO_CLEAR_INT;
+
+		switch (nhi->pdev->device) {
+		case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
+		case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
+			/*
+			 * Falcon Ridge controller needs the end-to-end
+			 * flow control workaround to avoid losing Rx
+			 * packets when RING_FLAG_E2E is set.
+			 */
+			nhi->quirks |= QUIRK_E2E;
+			break;
+		}
+	}
 }
 
 static int nhi_check_iommu_pdev(struct pci_dev *pdev, void *data)
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index c63c1f4ff9dc..77d7f07ca075 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -2413,6 +2413,7 @@ int tb_switch_configure(struct tb_switch *sw)
 		 * additional capabilities.
 		 */
 		sw->config.cmuv = USB4_VERSION_1_0;
+		sw->config.plug_events_delay = 0xa;
 
 		/* Enumerate the switch */
 		ret = tb_sw_write(sw, (u32 *)&sw->config + 1, TB_CFG_SWITCH,
diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c
index 9853f6c7e81d..9a277078338c 100644
--- a/drivers/thunderbolt/tb.c
+++ b/drivers/thunderbolt/tb.c
@@ -174,10 +174,10 @@ static void tb_discover_tunnels(struct tb *tb)
 	}
 }
 
-static int tb_port_configure_xdomain(struct tb_port *port)
+static int tb_port_configure_xdomain(struct tb_port *port, struct tb_xdomain *xd)
 {
 	if (tb_switch_is_usb4(port->sw))
-		return usb4_port_configure_xdomain(port);
+		return usb4_port_configure_xdomain(port, xd);
 	return tb_lc_configure_xdomain(port);
 }
 
@@ -212,7 +212,7 @@ static void tb_scan_xdomain(struct tb_port *port)
 			      NULL);
 	if (xd) {
 		tb_port_at(route, sw)->xdomain = xd;
-		tb_port_configure_xdomain(port);
+		tb_port_configure_xdomain(port, xd);
 		tb_xdomain_add(xd);
 	}
 }
@@ -1516,7 +1516,7 @@ static void tb_restore_children(struct tb_switch *sw)
 
 			tb_restore_children(port->remote->sw);
 		} else if (port->xdomain) {
-			tb_port_configure_xdomain(port);
+			tb_port_configure_xdomain(port, port->xdomain);
 		}
 	}
 }
diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h
index 5db76de40cc1..0f067c06cba6 100644
--- a/drivers/thunderbolt/tb.h
+++ b/drivers/thunderbolt/tb.h
@@ -1176,7 +1176,7 @@ void usb4_switch_remove_ports(struct tb_switch *sw);
 int usb4_port_unlock(struct tb_port *port);
 int usb4_port_configure(struct tb_port *port);
 void usb4_port_unconfigure(struct tb_port *port);
-int usb4_port_configure_xdomain(struct tb_port *port);
+int usb4_port_configure_xdomain(struct tb_port *port, struct tb_xdomain *xd);
 void usb4_port_unconfigure_xdomain(struct tb_port *port);
 int usb4_port_router_offline(struct tb_port *port);
 int usb4_port_router_online(struct tb_port *port);
diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c
index 3a2e7126db9d..a386228a44ee 100644
--- a/drivers/thunderbolt/usb4.c
+++ b/drivers/thunderbolt/usb4.c
@@ -1115,12 +1115,14 @@ static int usb4_set_xdomain_configured(struct tb_port *port, bool configured)
 /**
  * usb4_port_configure_xdomain() - Configure port for XDomain
  * @port: USB4 port connected to another host
+ * @xd: XDomain that is connected to the port
  *
- * Marks the USB4 port as being connected to another host. Returns %0 in
- * success and negative errno in failure.
+ * Marks the USB4 port as being connected to another host and updates
+ * the link type. Returns %0 in success and negative errno in failure.
  */
-int usb4_port_configure_xdomain(struct tb_port *port)
+int usb4_port_configure_xdomain(struct tb_port *port, struct tb_xdomain *xd)
 {
+	xd->link_usb4 = link_is_usb4(port);
 	return usb4_set_xdomain_configured(port, true);
 }
 
diff --git a/drivers/thunderbolt/usb4_port.c b/drivers/thunderbolt/usb4_port.c
index 6b02945624ee..1a30c0a23286 100644
--- a/drivers/thunderbolt/usb4_port.c
+++ b/drivers/thunderbolt/usb4_port.c
@@ -53,6 +53,8 @@ static ssize_t link_show(struct device *dev, struct device_attribute *attr,
 		link = port->sw->link_usb4 ? "usb4" : "tbt";
 	else if (tb_port_has_remote(port))
 		link = port->remote->sw->link_usb4 ? "usb4" : "tbt";
+	else if (port->xdomain)
+		link = port->xdomain->link_usb4 ? "usb4" : "tbt";
 	else
 		link = "none";
 
diff --git a/drivers/tty/serial/21285.c b/drivers/tty/serial/21285.c
index 7520cc02fd4d..65d6af755567 100644
--- a/drivers/tty/serial/21285.c
+++ b/drivers/tty/serial/21285.c
@@ -461,9 +461,6 @@ static int __init serial21285_console_setup(struct console *co, char *options)
 	int parity = 'n';
 	int flow = 'n';
 
-	if (machine_is_personal_server())
-		baud = 57600;
-
 	/*
 	 * Check whether an invalid uart number has been specified, and
 	 * if so, search for the first available port that does have
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index 877173907c53..ba4b63fd511e 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -1083,8 +1083,8 @@ config SERIAL_TIMBERDALE
 config SERIAL_BCM63XX
 	tristate "Broadcom BCM63xx/BCM33xx UART support"
 	select SERIAL_CORE
-	depends on ARCH_BCM4908 || ARCH_BCMBCA || BCM63XX || BMIPS_GENERIC || COMPILE_TEST
-	default ARCH_BCM4908 || ARCH_BCMBCA || BCM63XX || BMIPS_GENERIC
+	depends on ARCH_BCMBCA || BCM63XX || BMIPS_GENERIC || COMPILE_TEST
+	default ARCH_BCMBCA || BCM63XX || BMIPS_GENERIC
 	help
 	  This enables the driver for the onchip UART core found on
 	  the following chipsets:
diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c
index ab10ca4a45b5..7cf81f692ac4 100644
--- a/drivers/tty/serial/max310x.c
+++ b/drivers/tty/serial/max310x.c
@@ -1616,11 +1616,9 @@ static int max310x_i2c_probe(struct i2c_client *client)
 			     regmaps, client->irq);
 }
 
-static int max310x_i2c_remove(struct i2c_client *client)
+static void max310x_i2c_remove(struct i2c_client *client)
 {
 	max310x_remove(&client->dev);
-
-	return 0;
 }
 
 static struct i2c_driver max310x_i2c_driver = {
diff --git a/drivers/tty/serial/sc16is7xx.c b/drivers/tty/serial/sc16is7xx.c
index 259e08cc347c..d983692c59e0 100644
--- a/drivers/tty/serial/sc16is7xx.c
+++ b/drivers/tty/serial/sc16is7xx.c
@@ -1689,11 +1689,9 @@ static int sc16is7xx_i2c_probe(struct i2c_client *i2c,
 	return sc16is7xx_probe(&i2c->dev, devtype, regmap, i2c->irq);
 }
 
-static int sc16is7xx_i2c_remove(struct i2c_client *client)
+static void sc16is7xx_i2c_remove(struct i2c_client *client)
 {
 	sc16is7xx_remove(&client->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id sc16is7xx_i2c_id_table[] = {
diff --git a/drivers/usb/dwc3/dwc3-st.c b/drivers/usb/dwc3/dwc3-st.c
index 166b5bde45cb..6c14a79279f9 100644
--- a/drivers/usb/dwc3/dwc3-st.c
+++ b/drivers/usb/dwc3/dwc3-st.c
@@ -251,7 +251,7 @@ static int st_dwc3_probe(struct platform_device *pdev)
 	/* Manage SoftReset */
 	reset_control_deassert(dwc3_data->rstc_rst);
 
-	child = of_get_child_by_name(node, "dwc3");
+	child = of_get_child_by_name(node, "usb");
 	if (!child) {
 		dev_err(&pdev->dev, "failed to find dwc3 core node\n");
 		ret = -ENODEV;
diff --git a/drivers/usb/gadget/function/u_uac1_legacy.c b/drivers/usb/gadget/function/u_uac1_legacy.c
index 60ae8b2d3f6a..dd21c251542c 100644
--- a/drivers/usb/gadget/function/u_uac1_legacy.c
+++ b/drivers/usb/gadget/function/u_uac1_legacy.c
@@ -158,8 +158,8 @@ size_t u_audio_playback(struct gaudio *card, void *buf, size_t count)
 	snd_pcm_sframes_t frames;
 
 try_again:
-	if (runtime->status->state == SNDRV_PCM_STATE_XRUN ||
-		runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) {
+	if (runtime->state == SNDRV_PCM_STATE_XRUN ||
+		runtime->state == SNDRV_PCM_STATE_SUSPENDED) {
 		result = snd_pcm_kernel_ioctl(substream,
 				SNDRV_PCM_IOCTL_PREPARE, NULL);
 		if (result < 0) {
diff --git a/drivers/usb/misc/usb3503.c b/drivers/usb/misc/usb3503.c
index 330f494cd158..3c9fa663475f 100644
--- a/drivers/usb/misc/usb3503.c
+++ b/drivers/usb/misc/usb3503.c
@@ -289,14 +289,12 @@ static int usb3503_i2c_probe(struct i2c_client *i2c,
 	return usb3503_probe(hub);
 }
 
-static int usb3503_i2c_remove(struct i2c_client *i2c)
+static void usb3503_i2c_remove(struct i2c_client *i2c)
 {
 	struct usb3503 *hub;
 
 	hub = i2c_get_clientdata(i2c);
 	clk_disable_unprepare(hub->clk);
-
-	return 0;
 }
 
 static int usb3503_platform_probe(struct platform_device *pdev)
diff --git a/drivers/usb/phy/phy-isp1301-omap.c b/drivers/usb/phy/phy-isp1301-omap.c
index f8bd93fe69cd..e5d3f206097c 100644
--- a/drivers/usb/phy/phy-isp1301-omap.c
+++ b/drivers/usb/phy/phy-isp1301-omap.c
@@ -1196,7 +1196,7 @@ static void isp1301_release(struct device *dev)
 
 static struct isp1301 *the_transceiver;
 
-static int isp1301_remove(struct i2c_client *i2c)
+static void isp1301_remove(struct i2c_client *i2c)
 {
 	struct isp1301	*isp;
 
@@ -1214,8 +1214,6 @@ static int isp1301_remove(struct i2c_client *i2c)
 
 	put_device(&i2c->dev);
 	the_transceiver = NULL;
-
-	return 0;
 }
 
 /*-------------------------------------------------------------------------*/
diff --git a/drivers/usb/phy/phy-isp1301.c b/drivers/usb/phy/phy-isp1301.c
index ad3d57f1c273..c2777a5c1f4e 100644
--- a/drivers/usb/phy/phy-isp1301.c
+++ b/drivers/usb/phy/phy-isp1301.c
@@ -120,14 +120,12 @@ static int isp1301_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int isp1301_remove(struct i2c_client *client)
+static void isp1301_remove(struct i2c_client *client)
 {
 	struct isp1301 *isp = i2c_get_clientdata(client);
 
 	usb_remove_phy(&isp->phy);
 	isp1301_i2c_client = NULL;
-
-	return 0;
 }
 
 static struct i2c_driver isp1301_driver = {
diff --git a/drivers/usb/storage/unusual_uas.h b/drivers/usb/storage/unusual_uas.h
index 23ab3b048d9b..251778d14e2d 100644
--- a/drivers/usb/storage/unusual_uas.h
+++ b/drivers/usb/storage/unusual_uas.h
@@ -52,6 +52,13 @@ UNUSUAL_DEV(0x059f, 0x1061, 0x0000, 0x9999,
 		USB_SC_DEVICE, USB_PR_DEVICE, NULL,
 		US_FL_NO_REPORT_OPCODES | US_FL_NO_SAME),
 
+/* Reported-by: Hongling Zeng <zenghongling@kylinos.cn> */
+UNUSUAL_DEV(0x090c, 0x2000, 0x0000, 0x9999,
+		"Hiksemi",
+		"External HDD",
+		USB_SC_DEVICE, USB_PR_DEVICE, NULL,
+		US_FL_IGNORE_UAS),
+
 /*
  * Apricorn USB3 dongle sometimes returns "USBSUSBSUSBS" in response to SCSI
  * commands in UAS mode.  Observed with the 1.28 firmware; are there others?
@@ -76,6 +83,13 @@ UNUSUAL_DEV(0x0bc2, 0x331a, 0x0000, 0x9999,
 		USB_SC_DEVICE, USB_PR_DEVICE, NULL,
 		US_FL_NO_REPORT_LUNS),
 
+/* Reported-by: Hongling Zeng <zenghongling@kylinos.cn> */
+UNUSUAL_DEV(0x0bda, 0x9210, 0x0000, 0x9999,
+		"Hiksemi",
+		"External HDD",
+		USB_SC_DEVICE, USB_PR_DEVICE, NULL,
+		US_FL_IGNORE_UAS),
+
 /* Reported-by: Benjamin Tissoires <benjamin.tissoires@redhat.com> */
 UNUSUAL_DEV(0x13fd, 0x3940, 0x0000, 0x9999,
 		"Initio Corporation",
@@ -118,6 +132,13 @@ UNUSUAL_DEV(0x154b, 0xf00d, 0x0000, 0x9999,
 		USB_SC_DEVICE, USB_PR_DEVICE, NULL,
 		US_FL_NO_ATA_1X),
 
+/* Reported-by: Hongling Zeng <zenghongling@kylinos.cn> */
+UNUSUAL_DEV(0x17ef, 0x3899, 0x0000, 0x9999,
+		"Thinkplus",
+		"External HDD",
+		USB_SC_DEVICE, USB_PR_DEVICE, NULL,
+		US_FL_IGNORE_UAS),
+
 /* Reported-by: Hans de Goede <hdegoede@redhat.com> */
 UNUSUAL_DEV(0x2109, 0x0711, 0x0000, 0x9999,
 		"VIA",
diff --git a/drivers/usb/typec/anx7411.c b/drivers/usb/typec/anx7411.c
index c0f0842d443c..e205f409589a 100644
--- a/drivers/usb/typec/anx7411.c
+++ b/drivers/usb/typec/anx7411.c
@@ -1541,7 +1541,7 @@ free_i2c_dummy:
 	return ret;
 }
 
-static int anx7411_i2c_remove(struct i2c_client *client)
+static void anx7411_i2c_remove(struct i2c_client *client)
 {
 	struct anx7411_data *plat = i2c_get_clientdata(client);
 
@@ -1565,8 +1565,6 @@ static int anx7411_i2c_remove(struct i2c_client *client)
 		typec_unregister_port(plat->typec.port);
 
 	anx7411_port_unregister_altmodes(plat->typec.port_amode);
-
-	return 0;
 }
 
 static const struct i2c_device_id anx7411_id[] = {
diff --git a/drivers/usb/typec/hd3ss3220.c b/drivers/usb/typec/hd3ss3220.c
index cd47c3597e19..2a58185fb14c 100644
--- a/drivers/usb/typec/hd3ss3220.c
+++ b/drivers/usb/typec/hd3ss3220.c
@@ -245,14 +245,12 @@ err_put_fwnode:
 	return ret;
 }
 
-static int hd3ss3220_remove(struct i2c_client *client)
+static void hd3ss3220_remove(struct i2c_client *client)
 {
 	struct hd3ss3220 *hd3ss3220 = i2c_get_clientdata(client);
 
 	typec_unregister_port(hd3ss3220->port);
 	usb_role_switch_put(hd3ss3220->role_sw);
-
-	return 0;
 }
 
 static const struct of_device_id dev_ids[] = {
diff --git a/drivers/usb/typec/mux/fsa4480.c b/drivers/usb/typec/mux/fsa4480.c
index 6184f5367190..d6495e533e58 100644
--- a/drivers/usb/typec/mux/fsa4480.c
+++ b/drivers/usb/typec/mux/fsa4480.c
@@ -181,14 +181,12 @@ static int fsa4480_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int fsa4480_remove(struct i2c_client *client)
+static void fsa4480_remove(struct i2c_client *client)
 {
 	struct fsa4480 *fsa = i2c_get_clientdata(client);
 
 	typec_mux_unregister(fsa->mux);
 	typec_switch_unregister(fsa->sw);
-
-	return 0;
 }
 
 static const struct i2c_device_id fsa4480_table[] = {
diff --git a/drivers/usb/typec/mux/intel_pmc_mux.c b/drivers/usb/typec/mux/intel_pmc_mux.c
index a8e273fe204a..e1f4df7238bf 100644
--- a/drivers/usb/typec/mux/intel_pmc_mux.c
+++ b/drivers/usb/typec/mux/intel_pmc_mux.c
@@ -569,15 +569,6 @@ err_unregister_switch:
 	return ret;
 }
 
-static int is_memory(struct acpi_resource *res, void *data)
-{
-	struct resource_win win = {};
-	struct resource *r = &win.res;
-
-	return !(acpi_dev_resource_memory(res, r) ||
-		 acpi_dev_resource_address_space(res, &win));
-}
-
 /* IOM ACPI IDs and IOM_PORT_STATUS_OFFSET */
 static const struct acpi_device_id iom_acpi_ids[] = {
 	/* TigerLake */
@@ -611,7 +602,7 @@ static int pmc_usb_probe_iom(struct pmc_usb *pmc)
 		return -ENODEV;
 
 	INIT_LIST_HEAD(&resource_list);
-	ret = acpi_dev_get_resources(adev, &resource_list, is_memory, NULL);
+	ret = acpi_dev_get_memory_resources(adev, &resource_list);
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/usb/typec/mux/pi3usb30532.c b/drivers/usb/typec/mux/pi3usb30532.c
index 6ce9f282594e..1cd388b55c30 100644
--- a/drivers/usb/typec/mux/pi3usb30532.c
+++ b/drivers/usb/typec/mux/pi3usb30532.c
@@ -160,13 +160,12 @@ static int pi3usb30532_probe(struct i2c_client *client)
 	return 0;
 }
 
-static int pi3usb30532_remove(struct i2c_client *client)
+static void pi3usb30532_remove(struct i2c_client *client)
 {
 	struct pi3usb30532 *pi = i2c_get_clientdata(client);
 
 	typec_mux_unregister(pi->mux);
 	typec_switch_unregister(pi->sw);
-	return 0;
 }
 
 static const struct i2c_device_id pi3usb30532_table[] = {
diff --git a/drivers/usb/typec/rt1719.c b/drivers/usb/typec/rt1719.c
index f1b698edd7eb..ea8b700b0ceb 100644
--- a/drivers/usb/typec/rt1719.c
+++ b/drivers/usb/typec/rt1719.c
@@ -930,14 +930,12 @@ err_fwnode_put:
 	return ret;
 }
 
-static int rt1719_remove(struct i2c_client *i2c)
+static void rt1719_remove(struct i2c_client *i2c)
 {
 	struct rt1719_data *data = i2c_get_clientdata(i2c);
 
 	typec_unregister_port(data->port);
 	usb_role_switch_put(data->role_sw);
-
-	return 0;
 }
 
 static const struct of_device_id __maybe_unused rt1719_device_table[] = {
diff --git a/drivers/usb/typec/stusb160x.c b/drivers/usb/typec/stusb160x.c
index e7745d1c2a5c..8638f1d39896 100644
--- a/drivers/usb/typec/stusb160x.c
+++ b/drivers/usb/typec/stusb160x.c
@@ -801,7 +801,7 @@ fwnode_put:
 	return ret;
 }
 
-static int stusb160x_remove(struct i2c_client *client)
+static void stusb160x_remove(struct i2c_client *client)
 {
 	struct stusb160x *chip = i2c_get_clientdata(client);
 
@@ -823,8 +823,6 @@ static int stusb160x_remove(struct i2c_client *client)
 
 	if (chip->main_supply)
 		regulator_disable(chip->main_supply);
-
-	return 0;
 }
 
 static int __maybe_unused stusb160x_suspend(struct device *dev)
diff --git a/drivers/usb/typec/tcpm/fusb302.c b/drivers/usb/typec/tcpm/fusb302.c
index 96c55eaf3f80..5e9348f28d50 100644
--- a/drivers/usb/typec/tcpm/fusb302.c
+++ b/drivers/usb/typec/tcpm/fusb302.c
@@ -1771,7 +1771,7 @@ destroy_workqueue:
 	return ret;
 }
 
-static int fusb302_remove(struct i2c_client *client)
+static void fusb302_remove(struct i2c_client *client)
 {
 	struct fusb302_chip *chip = i2c_get_clientdata(client);
 
@@ -1783,8 +1783,6 @@ static int fusb302_remove(struct i2c_client *client)
 	fwnode_handle_put(chip->tcpc_dev.fwnode);
 	destroy_workqueue(chip->wq);
 	fusb302_debugfs_exit(chip);
-
-	return 0;
 }
 
 static int fusb302_pm_suspend(struct device *dev)
diff --git a/drivers/usb/typec/tcpm/tcpci.c b/drivers/usb/typec/tcpm/tcpci.c
index 812784702d53..f00810d198a8 100644
--- a/drivers/usb/typec/tcpm/tcpci.c
+++ b/drivers/usb/typec/tcpm/tcpci.c
@@ -868,7 +868,7 @@ static int tcpci_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int tcpci_remove(struct i2c_client *client)
+static void tcpci_remove(struct i2c_client *client)
 {
 	struct tcpci_chip *chip = i2c_get_clientdata(client);
 	int err;
@@ -879,8 +879,6 @@ static int tcpci_remove(struct i2c_client *client)
 		dev_warn(&client->dev, "Failed to disable irqs (%pe)\n", ERR_PTR(err));
 
 	tcpci_unregister_port(chip->tcpci);
-
-	return 0;
 }
 
 static const struct i2c_device_id tcpci_id[] = {
diff --git a/drivers/usb/typec/tcpm/tcpci_maxim.c b/drivers/usb/typec/tcpm/tcpci_maxim.c
index 4b6705f3d7b7..03f89e6f1a78 100644
--- a/drivers/usb/typec/tcpm/tcpci_maxim.c
+++ b/drivers/usb/typec/tcpm/tcpci_maxim.c
@@ -492,14 +492,12 @@ unreg_port:
 	return ret;
 }
 
-static int max_tcpci_remove(struct i2c_client *client)
+static void max_tcpci_remove(struct i2c_client *client)
 {
 	struct max_tcpci_chip *chip = i2c_get_clientdata(client);
 
 	if (!IS_ERR_OR_NULL(chip->tcpci))
 		tcpci_unregister_port(chip->tcpci);
-
-	return 0;
 }
 
 static const struct i2c_device_id max_tcpci_id[] = {
diff --git a/drivers/usb/typec/tcpm/tcpci_rt1711h.c b/drivers/usb/typec/tcpm/tcpci_rt1711h.c
index 3291ca4948da..c1327713f06d 100644
--- a/drivers/usb/typec/tcpm/tcpci_rt1711h.c
+++ b/drivers/usb/typec/tcpm/tcpci_rt1711h.c
@@ -263,12 +263,11 @@ static int rt1711h_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int rt1711h_remove(struct i2c_client *client)
+static void rt1711h_remove(struct i2c_client *client)
 {
 	struct rt1711h_chip *chip = i2c_get_clientdata(client);
 
 	tcpci_unregister_port(chip->tcpci);
-	return 0;
 }
 
 static const struct i2c_device_id rt1711h_id[] = {
diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c
index dfbba5ae9487..b637e8b378b3 100644
--- a/drivers/usb/typec/tipd/core.c
+++ b/drivers/usb/typec/tipd/core.c
@@ -857,15 +857,13 @@ err_clear_mask:
 	return ret;
 }
 
-static int tps6598x_remove(struct i2c_client *client)
+static void tps6598x_remove(struct i2c_client *client)
 {
 	struct tps6598x *tps = i2c_get_clientdata(client);
 
 	tps6598x_disconnect(tps, 0);
 	typec_unregister_port(tps->port);
 	usb_role_switch_put(tps->role_sw);
-
-	return 0;
 }
 
 static const struct of_device_id tps6598x_of_match[] = {
diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
index 7f2624f42724..6364f0d467ea 100644
--- a/drivers/usb/typec/ucsi/ucsi.c
+++ b/drivers/usb/typec/ucsi/ucsi.c
@@ -588,8 +588,6 @@ static int ucsi_get_pdos(struct ucsi_connector *con, int is_partner,
 				num_pdos * sizeof(u32));
 	if (ret < 0 && ret != -ETIMEDOUT)
 		dev_err(ucsi->dev, "UCSI_GET_PDOS failed (%d)\n", ret);
-	if (ret == 0 && offset == 0)
-		dev_warn(ucsi->dev, "UCSI_GET_PDOS returned 0 bytes\n");
 
 	return ret;
 }
diff --git a/drivers/usb/typec/ucsi/ucsi_ccg.c b/drivers/usb/typec/ucsi/ucsi_ccg.c
index 5c0bf48be766..349756335362 100644
--- a/drivers/usb/typec/ucsi/ucsi_ccg.c
+++ b/drivers/usb/typec/ucsi/ucsi_ccg.c
@@ -1403,7 +1403,7 @@ out_ucsi_destroy:
 	return status;
 }
 
-static int ucsi_ccg_remove(struct i2c_client *client)
+static void ucsi_ccg_remove(struct i2c_client *client)
 {
 	struct ucsi_ccg *uc = i2c_get_clientdata(client);
 
@@ -1413,8 +1413,6 @@ static int ucsi_ccg_remove(struct i2c_client *client)
 	ucsi_unregister(uc->ucsi);
 	ucsi_destroy(uc->ucsi);
 	free_irq(uc->irq, uc);
-
-	return 0;
 }
 
 static const struct i2c_device_id ucsi_ccg_device_id[] = {
diff --git a/drivers/usb/typec/ucsi/ucsi_stm32g0.c b/drivers/usb/typec/ucsi/ucsi_stm32g0.c
index 061551d464f1..16289ff583b4 100644
--- a/drivers/usb/typec/ucsi/ucsi_stm32g0.c
+++ b/drivers/usb/typec/ucsi/ucsi_stm32g0.c
@@ -688,7 +688,7 @@ destroy:
 	return ret;
 }
 
-static int ucsi_stm32g0_remove(struct i2c_client *client)
+static void ucsi_stm32g0_remove(struct i2c_client *client)
 {
 	struct ucsi_stm32g0 *g0 = i2c_get_clientdata(client);
 
@@ -697,8 +697,6 @@ static int ucsi_stm32g0_remove(struct i2c_client *client)
 	if (g0->fw_name)
 		i2c_unregister_device(g0->i2c_bl);
 	ucsi_destroy(g0->ucsi);
-
-	return 0;
 }
 
 static int ucsi_stm32g0_suspend(struct device *dev)
diff --git a/drivers/usb/typec/wusb3801.c b/drivers/usb/typec/wusb3801.c
index e63509f8b01e..3cc7a15ecbd3 100644
--- a/drivers/usb/typec/wusb3801.c
+++ b/drivers/usb/typec/wusb3801.c
@@ -399,7 +399,7 @@ err_put_connector:
 	return ret;
 }
 
-static int wusb3801_remove(struct i2c_client *client)
+static void wusb3801_remove(struct i2c_client *client)
 {
 	struct wusb3801 *wusb3801 = i2c_get_clientdata(client);
 
@@ -411,8 +411,6 @@ static int wusb3801_remove(struct i2c_client *client)
 
 	if (wusb3801->vbus_on)
 		regulator_disable(wusb3801->vbus_supply);
-
-	return 0;
 }
 
 static const struct of_device_id wusb3801_of_match[] = {
diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
index 75a703b803a2..3e4486bfa0b7 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.c
+++ b/drivers/vdpa/ifcvf/ifcvf_base.c
@@ -323,7 +323,7 @@ u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid)
 	u32 q_pair_id;
 
 	ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg;
-	q_pair_id = qid / hw->nr_vring;
+	q_pair_id = qid / 2;
 	avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
 	last_avail_idx = vp_ioread16(avail_idx_addr);
 
@@ -337,7 +337,7 @@ int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num)
 	u32 q_pair_id;
 
 	ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg;
-	q_pair_id = qid / hw->nr_vring;
+	q_pair_id = qid / 2;
 	avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
 	hw->vring[qid].last_avail_idx = num;
 	vp_iowrite16(num, avail_idx_addr);
diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c
index ed100a35e596..90913365def4 100644
--- a/drivers/vdpa/mlx5/net/mlx5_vnet.c
+++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c
@@ -1320,6 +1320,8 @@ static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *
 
 static int create_rqt(struct mlx5_vdpa_net *ndev)
 {
+	int rqt_table_size = roundup_pow_of_two(ndev->rqt_size);
+	int act_sz = roundup_pow_of_two(ndev->cur_num_vqs / 2);
 	__be32 *list;
 	void *rqtc;
 	int inlen;
@@ -1327,7 +1329,7 @@ static int create_rqt(struct mlx5_vdpa_net *ndev)
 	int i, j;
 	int err;
 
-	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + ndev->rqt_size * MLX5_ST_SZ_BYTES(rq_num);
+	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + rqt_table_size * MLX5_ST_SZ_BYTES(rq_num);
 	in = kzalloc(inlen, GFP_KERNEL);
 	if (!in)
 		return -ENOMEM;
@@ -1336,12 +1338,12 @@ static int create_rqt(struct mlx5_vdpa_net *ndev)
 	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
 
 	MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q);
-	MLX5_SET(rqtc, rqtc, rqt_max_size, ndev->rqt_size);
+	MLX5_SET(rqtc, rqtc, rqt_max_size, rqt_table_size);
 	list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]);
-	for (i = 0, j = 0; i < ndev->rqt_size; i++, j += 2)
+	for (i = 0, j = 0; i < act_sz; i++, j += 2)
 		list[i] = cpu_to_be32(ndev->vqs[j % ndev->cur_num_vqs].virtq_id);
 
-	MLX5_SET(rqtc, rqtc, rqt_actual_size, ndev->rqt_size);
+	MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz);
 	err = mlx5_vdpa_create_rqt(&ndev->mvdev, in, inlen, &ndev->res.rqtn);
 	kfree(in);
 	if (err)
@@ -1354,6 +1356,7 @@ static int create_rqt(struct mlx5_vdpa_net *ndev)
 
 static int modify_rqt(struct mlx5_vdpa_net *ndev, int num)
 {
+	int act_sz = roundup_pow_of_two(num / 2);
 	__be32 *list;
 	void *rqtc;
 	int inlen;
@@ -1361,7 +1364,7 @@ static int modify_rqt(struct mlx5_vdpa_net *ndev, int num)
 	int i, j;
 	int err;
 
-	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + ndev->rqt_size * MLX5_ST_SZ_BYTES(rq_num);
+	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + act_sz * MLX5_ST_SZ_BYTES(rq_num);
 	in = kzalloc(inlen, GFP_KERNEL);
 	if (!in)
 		return -ENOMEM;
@@ -1372,10 +1375,10 @@ static int modify_rqt(struct mlx5_vdpa_net *ndev, int num)
 	MLX5_SET(rqtc, rqtc, list_q_type, MLX5_RQTC_LIST_Q_TYPE_VIRTIO_NET_Q);
 
 	list = MLX5_ADDR_OF(rqtc, rqtc, rq_num[0]);
-	for (i = 0, j = 0; i < ndev->rqt_size; i++, j += 2)
+	for (i = 0, j = 0; i < act_sz; i++, j = j + 2)
 		list[i] = cpu_to_be32(ndev->vqs[j % num].virtq_id);
 
-	MLX5_SET(rqtc, rqtc, rqt_actual_size, ndev->rqt_size);
+	MLX5_SET(rqtc, rqtc, rqt_actual_size, act_sz);
 	err = mlx5_vdpa_modify_rqt(&ndev->mvdev, in, inlen, ndev->res.rqtn);
 	kfree(in);
 	if (err)
diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c
index c06c02704461..7badf5777597 100644
--- a/drivers/vdpa/vdpa.c
+++ b/drivers/vdpa/vdpa.c
@@ -1183,6 +1183,7 @@ static struct genl_family vdpa_nl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.ops = vdpa_nl_ops,
 	.n_ops = ARRAY_SIZE(vdpa_nl_ops),
+	.resv_start_op = VDPA_CMD_DEV_VSTATS_GET + 1,
 };
 
 static int vdpa_init(void)
diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index 41c0b29739f1..35dceee3ed56 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -673,10 +673,15 @@ static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
 {
 	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 
-	if (offset > dev->config_size ||
-	    len > dev->config_size - offset)
+	/* Initialize the buffer in case of partial copy. */
+	memset(buf, 0, len);
+
+	if (offset > dev->config_size)
 		return;
 
+	if (len > dev->config_size - offset)
+		len = dev->config_size - offset;
+
 	memcpy(buf, dev->config + offset, len);
 }
 
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index 68e4ecd1cc0e..d7a04d573988 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -118,7 +118,7 @@ struct vhost_net_virtqueue {
 	/* Number of XDP frames batched */
 	int batched_xdp;
 	/* an array of userspace buffers info */
-	struct ubuf_info *ubuf_info;
+	struct ubuf_info_msgzc *ubuf_info;
 	/* Reference counting for outstanding ubufs.
 	 * Protected by vq mutex. Writers must also take device mutex. */
 	struct vhost_net_ubuf_ref *ubufs;
@@ -382,8 +382,9 @@ static void vhost_zerocopy_signal_used(struct vhost_net *net,
 }
 
 static void vhost_zerocopy_callback(struct sk_buff *skb,
-				    struct ubuf_info *ubuf, bool success)
+				    struct ubuf_info *ubuf_base, bool success)
 {
+	struct ubuf_info_msgzc *ubuf = uarg_to_msgzc(ubuf_base);
 	struct vhost_net_ubuf_ref *ubufs = ubuf->ctx;
 	struct vhost_virtqueue *vq = ubufs->vq;
 	int cnt;
@@ -871,7 +872,7 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
 	size_t len, total_len = 0;
 	int err;
 	struct vhost_net_ubuf_ref *ubufs;
-	struct ubuf_info *ubuf;
+	struct ubuf_info_msgzc *ubuf;
 	bool zcopy_used;
 	int sent_pkts = 0;
 
@@ -907,14 +908,14 @@ static void handle_tx_zerocopy(struct vhost_net *net, struct socket *sock)
 			ubuf = nvq->ubuf_info + nvq->upend_idx;
 			vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
 			vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
-			ubuf->callback = vhost_zerocopy_callback;
 			ubuf->ctx = nvq->ubufs;
 			ubuf->desc = nvq->upend_idx;
-			ubuf->flags = SKBFL_ZEROCOPY_FRAG;
-			refcount_set(&ubuf->refcnt, 1);
+			ubuf->ubuf.callback = vhost_zerocopy_callback;
+			ubuf->ubuf.flags = SKBFL_ZEROCOPY_FRAG;
+			refcount_set(&ubuf->ubuf.refcnt, 1);
 			msg.msg_control = &ctl;
 			ctl.type = TUN_MSG_UBUF;
-			ctl.ptr = ubuf;
+			ctl.ptr = &ubuf->ubuf;
 			msg.msg_controllen = sizeof(ctl);
 			ubufs = nvq->ubufs;
 			atomic_inc(&ubufs->refcount);
diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
index 368330417bde..5703775af129 100644
--- a/drivers/vhost/vsock.c
+++ b/drivers/vhost/vsock.c
@@ -393,7 +393,7 @@ vhost_vsock_alloc_pkt(struct vhost_virtqueue *vq,
 		return NULL;
 	}
 
-	pkt->buf = kmalloc(pkt->len, GFP_KERNEL);
+	pkt->buf = kvmalloc(pkt->len, GFP_KERNEL);
 	if (!pkt->buf) {
 		kfree(pkt);
 		return NULL;
diff --git a/drivers/video/aperture.c b/drivers/video/aperture.c
index 538f2d40acda..9e6bcc03a1a4 100644
--- a/drivers/video/aperture.c
+++ b/drivers/video/aperture.c
@@ -2,15 +2,17 @@
 
 #include <linux/aperture.h>
 #include <linux/device.h>
-#include <linux/fb.h> /* for old fbdev helpers */
 #include <linux/list.h>
 #include <linux/mutex.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
+#include <linux/sysfb.h>
 #include <linux/types.h>
 #include <linux/vgaarb.h>
 
+#include <video/vga.h>
+
 /**
  * DOC: overview
  *
@@ -283,26 +285,27 @@ static void aperture_detach_devices(resource_size_t base, resource_size_t size)
 int aperture_remove_conflicting_devices(resource_size_t base, resource_size_t size,
 					bool primary, const char *name)
 {
-#if IS_REACHABLE(CONFIG_FB)
-	struct apertures_struct *a;
-	int ret;
-
-	a = alloc_apertures(1);
-	if (!a)
-		return -ENOMEM;
-
-	a->ranges[0].base = base;
-	a->ranges[0].size = size;
-
-	ret = remove_conflicting_framebuffers(a, name, primary);
-	kfree(a);
-
-	if (ret)
-		return ret;
-#endif
+	/*
+	 * If a driver asked to unregister a platform device registered by
+	 * sysfb, then can be assumed that this is a driver for a display
+	 * that is set up by the system firmware and has a generic driver.
+	 *
+	 * Drivers for devices that don't have a generic driver will never
+	 * ask for this, so let's assume that a real driver for the display
+	 * was already probed and prevent sysfb to register devices later.
+	 */
+	sysfb_disable();
 
 	aperture_detach_devices(base, size);
 
+	/*
+	 * If this is the primary adapter, there could be a VGA device
+	 * that consumes the VGA framebuffer I/O range. Remove this device
+	 * as well.
+	 */
+	if (primary)
+		aperture_detach_devices(VGA_FB_PHYS_BASE, VGA_FB_PHYS_SIZE);
+
 	return 0;
 }
 EXPORT_SYMBOL(aperture_remove_conflicting_devices);
@@ -321,30 +324,36 @@ EXPORT_SYMBOL(aperture_remove_conflicting_devices);
  */
 int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *name)
 {
+	bool primary = false;
 	resource_size_t base, size;
 	int bar, ret;
 
-	/*
-	 * WARNING: Apparently we must kick fbdev drivers before vgacon,
-	 * otherwise the vga fbdev driver falls over.
-	 */
-#if IS_REACHABLE(CONFIG_FB)
-	ret = remove_conflicting_pci_framebuffers(pdev, name);
-	if (ret)
-		return ret;
+#ifdef CONFIG_X86
+	primary = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
 #endif
-	ret = vga_remove_vgacon(pdev);
-	if (ret)
-		return ret;
 
 	for (bar = 0; bar < PCI_STD_NUM_BARS; ++bar) {
 		if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM))
 			continue;
+
 		base = pci_resource_start(pdev, bar);
 		size = pci_resource_len(pdev, bar);
-		aperture_detach_devices(base, size);
+		ret = aperture_remove_conflicting_devices(base, size, primary, name);
+		if (ret)
+			break;
 	}
 
+	if (ret)
+		return ret;
+
+	/*
+	 * WARNING: Apparently we must kick fbdev drivers before vgacon,
+	 * otherwise the vga fbdev driver falls over.
+	 */
+	ret = vga_remove_vgacon(pdev);
+	if (ret)
+		return ret;
+
 	return 0;
 
 }
diff --git a/drivers/video/backlight/Kconfig b/drivers/video/backlight/Kconfig
index a003e02e13ce..936ba1e4d35e 100644
--- a/drivers/video/backlight/Kconfig
+++ b/drivers/video/backlight/Kconfig
@@ -268,6 +268,19 @@ config BACKLIGHT_MAX8925
 	  If you have a LCD backlight connected to the WLED output of MAX8925
 	  WLED output, say Y here to enable this driver.
 
+config BACKLIGHT_MT6370
+	tristate "MediaTek MT6370 Backlight Driver"
+	depends on MFD_MT6370
+	help
+	  This enables support for Mediatek MT6370 Backlight driver.
+	  It's commonly used to drive the display WLED. There are 4 channels
+	  inside, and each channel supports up to 30mA of current capability
+	  with 2048 current steps (only for MT6370/MT6371) or 16384 current
+	  steps (only for MT6372) in exponential or linear mapping curves.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called "mt6370-backlight".
+
 config BACKLIGHT_APPLE
 	tristate "Apple Backlight Driver"
 	depends on X86 && ACPI
diff --git a/drivers/video/backlight/Makefile b/drivers/video/backlight/Makefile
index cae2c83422ae..e815f3f1deff 100644
--- a/drivers/video/backlight/Makefile
+++ b/drivers/video/backlight/Makefile
@@ -44,6 +44,7 @@ obj-$(CONFIG_BACKLIGHT_LP855X)		+= lp855x_bl.o
 obj-$(CONFIG_BACKLIGHT_LP8788)		+= lp8788_bl.o
 obj-$(CONFIG_BACKLIGHT_LV5207LP)	+= lv5207lp.o
 obj-$(CONFIG_BACKLIGHT_MAX8925)		+= max8925_bl.o
+obj-$(CONFIG_BACKLIGHT_MT6370)		+= mt6370-backlight.o
 obj-$(CONFIG_BACKLIGHT_OMAP1)		+= omap1_bl.o
 obj-$(CONFIG_BACKLIGHT_PANDORA)		+= pandora_bl.o
 obj-$(CONFIG_BACKLIGHT_PCF50633)	+= pcf50633-backlight.o
diff --git a/drivers/video/backlight/adp8860_bl.c b/drivers/video/backlight/adp8860_bl.c
index 8ec19425671f..b0fe02273e87 100644
--- a/drivers/video/backlight/adp8860_bl.c
+++ b/drivers/video/backlight/adp8860_bl.c
@@ -753,7 +753,7 @@ out:
 	return ret;
 }
 
-static int adp8860_remove(struct i2c_client *client)
+static void adp8860_remove(struct i2c_client *client)
 {
 	struct adp8860_bl *data = i2c_get_clientdata(client);
 
@@ -765,8 +765,6 @@ static int adp8860_remove(struct i2c_client *client)
 	if (data->en_ambl_sens)
 		sysfs_remove_group(&data->bl->dev.kobj,
 			&adp8860_bl_attr_group);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/video/backlight/adp8870_bl.c b/drivers/video/backlight/adp8870_bl.c
index 8b5213a39527..5becace3fd0f 100644
--- a/drivers/video/backlight/adp8870_bl.c
+++ b/drivers/video/backlight/adp8870_bl.c
@@ -925,7 +925,7 @@ out:
 	return ret;
 }
 
-static int adp8870_remove(struct i2c_client *client)
+static void adp8870_remove(struct i2c_client *client)
 {
 	struct adp8870_bl *data = i2c_get_clientdata(client);
 
@@ -937,8 +937,6 @@ static int adp8870_remove(struct i2c_client *client)
 	if (data->pdata->en_ambl_sens)
 		sysfs_remove_group(&data->bl->dev.kobj,
 			&adp8870_bl_attr_group);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/video/backlight/arcxcnn_bl.c b/drivers/video/backlight/arcxcnn_bl.c
index 7b1c0a0e6cad..060c0eef6a52 100644
--- a/drivers/video/backlight/arcxcnn_bl.c
+++ b/drivers/video/backlight/arcxcnn_bl.c
@@ -362,7 +362,7 @@ probe_err:
 	return ret;
 }
 
-static int arcxcnn_remove(struct i2c_client *cl)
+static void arcxcnn_remove(struct i2c_client *cl)
 {
 	struct arcxcnn *lp = i2c_get_clientdata(cl);
 
@@ -376,8 +376,6 @@ static int arcxcnn_remove(struct i2c_client *cl)
 	lp->bl->props.brightness = 0;
 
 	backlight_update_status(lp->bl);
-
-	return 0;
 }
 
 static const struct of_device_id arcxcnn_dt_ids[] = {
diff --git a/drivers/video/backlight/bd6107.c b/drivers/video/backlight/bd6107.c
index 515184fbe33a..a506872d4396 100644
--- a/drivers/video/backlight/bd6107.c
+++ b/drivers/video/backlight/bd6107.c
@@ -175,14 +175,12 @@ static int bd6107_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int bd6107_remove(struct i2c_client *client)
+static void bd6107_remove(struct i2c_client *client)
 {
 	struct backlight_device *backlight = i2c_get_clientdata(client);
 
 	backlight->props.brightness = 0;
 	backlight_update_status(backlight);
-
-	return 0;
 }
 
 static const struct i2c_device_id bd6107_ids[] = {
diff --git a/drivers/video/backlight/lm3630a_bl.c b/drivers/video/backlight/lm3630a_bl.c
index 1d17c439430e..475f35635bf6 100644
--- a/drivers/video/backlight/lm3630a_bl.c
+++ b/drivers/video/backlight/lm3630a_bl.c
@@ -579,7 +579,7 @@ static int lm3630a_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lm3630a_remove(struct i2c_client *client)
+static void lm3630a_remove(struct i2c_client *client)
 {
 	int rval;
 	struct lm3630a_chip *pchip = i2c_get_clientdata(client);
@@ -596,7 +596,6 @@ static int lm3630a_remove(struct i2c_client *client)
 		free_irq(pchip->irq, pchip);
 		destroy_workqueue(pchip->irqthread);
 	}
-	return 0;
 }
 
 static const struct i2c_device_id lm3630a_id[] = {
diff --git a/drivers/video/backlight/lm3639_bl.c b/drivers/video/backlight/lm3639_bl.c
index 48c04155a5f9..6580911671a3 100644
--- a/drivers/video/backlight/lm3639_bl.c
+++ b/drivers/video/backlight/lm3639_bl.c
@@ -390,7 +390,7 @@ err_out:
 	return ret;
 }
 
-static int lm3639_remove(struct i2c_client *client)
+static void lm3639_remove(struct i2c_client *client)
 {
 	struct lm3639_chip_data *pchip = i2c_get_clientdata(client);
 
@@ -400,7 +400,6 @@ static int lm3639_remove(struct i2c_client *client)
 	led_classdev_unregister(&pchip->cdev_flash);
 	if (pchip->bled)
 		device_remove_file(&(pchip->bled->dev), &dev_attr_bled_mode);
-	return 0;
 }
 
 static const struct i2c_device_id lm3639_id[] = {
diff --git a/drivers/video/backlight/lp855x_bl.c b/drivers/video/backlight/lp855x_bl.c
index fc02c5c16055..bd0bdeae23a4 100644
--- a/drivers/video/backlight/lp855x_bl.c
+++ b/drivers/video/backlight/lp855x_bl.c
@@ -534,7 +534,7 @@ disable_supply:
 	return ret;
 }
 
-static int lp855x_remove(struct i2c_client *cl)
+static void lp855x_remove(struct i2c_client *cl)
 {
 	struct lp855x *lp = i2c_get_clientdata(cl);
 
@@ -545,8 +545,6 @@ static int lp855x_remove(struct i2c_client *cl)
 	if (lp->supply)
 		regulator_disable(lp->supply);
 	sysfs_remove_group(&lp->dev->kobj, &lp855x_attr_group);
-
-	return 0;
 }
 
 static const struct of_device_id lp855x_dt_ids[] = {
diff --git a/drivers/video/backlight/lv5207lp.c b/drivers/video/backlight/lv5207lp.c
index 1842ae9a55f8..767b800d79fa 100644
--- a/drivers/video/backlight/lv5207lp.c
+++ b/drivers/video/backlight/lv5207lp.c
@@ -124,14 +124,12 @@ static int lv5207lp_probe(struct i2c_client *client,
 	return 0;
 }
 
-static int lv5207lp_remove(struct i2c_client *client)
+static void lv5207lp_remove(struct i2c_client *client)
 {
 	struct backlight_device *backlight = i2c_get_clientdata(client);
 
 	backlight->props.brightness = 0;
 	backlight_update_status(backlight);
-
-	return 0;
 }
 
 static const struct i2c_device_id lv5207lp_ids[] = {
diff --git a/drivers/video/backlight/mt6370-backlight.c b/drivers/video/backlight/mt6370-backlight.c
new file mode 100644
index 000000000000..623d4f2baca2
--- /dev/null
+++ b/drivers/video/backlight/mt6370-backlight.c
@@ -0,0 +1,351 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2022 Richtek Technology Corp.
+ *
+ * Author: ChiaEn Wu <chiaen_wu@richtek.com>
+ */
+
+#include <linux/backlight.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/gpio/consumer.h>
+#include <linux/kernel.h>
+#include <linux/minmax.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+
+#define MT6370_REG_DEV_INFO		0x100
+#define MT6370_REG_BL_EN		0x1A0
+#define MT6370_REG_BL_BSTCTRL		0x1A1
+#define MT6370_REG_BL_PWM		0x1A2
+#define MT6370_REG_BL_DIM2		0x1A4
+
+#define MT6370_VENID_MASK		GENMASK(7, 4)
+#define MT6370_BL_EXT_EN_MASK		BIT(7)
+#define MT6370_BL_EN_MASK		BIT(6)
+#define MT6370_BL_CODE_MASK		BIT(0)
+#define MT6370_BL_CH_MASK		GENMASK(5, 2)
+#define MT6370_BL_CH_SHIFT		2
+#define MT6370_BL_DIM2_COMMON_MASK	GENMASK(2, 0)
+#define MT6370_BL_DIM2_COMMON_SHIFT	3
+#define MT6370_BL_DIM2_6372_MASK	GENMASK(5, 0)
+#define MT6370_BL_DIM2_6372_SHIFT	6
+#define MT6370_BL_PWM_EN_MASK		BIT(7)
+#define MT6370_BL_PWM_HYS_EN_MASK	BIT(2)
+#define MT6370_BL_PWM_HYS_SEL_MASK	GENMASK(1, 0)
+#define MT6370_BL_OVP_EN_MASK		BIT(7)
+#define MT6370_BL_OVP_SEL_MASK		GENMASK(6, 5)
+#define MT6370_BL_OVP_SEL_SHIFT		5
+#define MT6370_BL_OC_EN_MASK		BIT(3)
+#define MT6370_BL_OC_SEL_MASK		GENMASK(2, 1)
+#define MT6370_BL_OC_SEL_SHIFT		1
+
+#define MT6370_BL_PWM_HYS_TH_MIN_STEP	1
+#define MT6370_BL_PWM_HYS_TH_MAX_STEP	64
+#define MT6370_BL_OVP_MIN_UV		17000000
+#define MT6370_BL_OVP_MAX_UV		29000000
+#define MT6370_BL_OVP_STEP_UV		4000000
+#define MT6370_BL_OCP_MIN_UA		900000
+#define MT6370_BL_OCP_MAX_UA		1800000
+#define MT6370_BL_OCP_STEP_UA		300000
+#define MT6370_BL_MAX_COMMON_BRIGHTNESS	2048
+#define MT6370_BL_MAX_6372_BRIGHTNESS	16384
+#define MT6370_BL_MAX_CH		15
+
+enum {
+	MT6370_VID_COMMON = 1,
+	MT6370_VID_6372,
+};
+
+struct mt6370_priv {
+	u8 dim2_mask;
+	u8 dim2_shift;
+	int def_max_brightness;
+	struct backlight_device *bl;
+	struct device *dev;
+	struct gpio_desc *enable_gpio;
+	struct regmap *regmap;
+};
+
+static int mt6370_bl_update_status(struct backlight_device *bl_dev)
+{
+	struct mt6370_priv *priv = bl_get_data(bl_dev);
+	int brightness = backlight_get_brightness(bl_dev);
+	unsigned int enable_val;
+	u8 brightness_val[2];
+	int ret;
+
+	if (brightness) {
+		brightness_val[0] = (brightness - 1) & priv->dim2_mask;
+		brightness_val[1] = (brightness - 1) >> priv->dim2_shift;
+
+		ret = regmap_raw_write(priv->regmap, MT6370_REG_BL_DIM2,
+				       brightness_val, sizeof(brightness_val));
+		if (ret)
+			return ret;
+	}
+
+	gpiod_set_value(priv->enable_gpio, !!brightness);
+
+	enable_val = brightness ? MT6370_BL_EN_MASK : 0;
+	return regmap_update_bits(priv->regmap, MT6370_REG_BL_EN,
+				  MT6370_BL_EN_MASK, enable_val);
+}
+
+static int mt6370_bl_get_brightness(struct backlight_device *bl_dev)
+{
+	struct mt6370_priv *priv = bl_get_data(bl_dev);
+	unsigned int enable;
+	u8 brightness_val[2];
+	int brightness, ret;
+
+	ret = regmap_read(priv->regmap, MT6370_REG_BL_EN, &enable);
+	if (ret)
+		return ret;
+
+	if (!(enable & MT6370_BL_EN_MASK))
+		return 0;
+
+	ret = regmap_raw_read(priv->regmap, MT6370_REG_BL_DIM2,
+			      brightness_val, sizeof(brightness_val));
+	if (ret)
+		return ret;
+
+	brightness = brightness_val[1] << priv->dim2_shift;
+	brightness += brightness_val[0] & priv->dim2_mask;
+
+	return brightness + 1;
+}
+
+static const struct backlight_ops mt6370_bl_ops = {
+	.options = BL_CORE_SUSPENDRESUME,
+	.update_status = mt6370_bl_update_status,
+	.get_brightness = mt6370_bl_get_brightness,
+};
+
+static int mt6370_init_backlight_properties(struct mt6370_priv *priv,
+					    struct backlight_properties *props)
+{
+	struct device *dev = priv->dev;
+	u8 prop_val;
+	u32 brightness, ovp_uV, ocp_uA;
+	unsigned int mask, val;
+	int ret;
+
+	/* Vendor optional properties */
+	val = 0;
+	if (device_property_read_bool(dev, "mediatek,bled-pwm-enable"))
+		val |= MT6370_BL_PWM_EN_MASK;
+
+	if (device_property_read_bool(dev, "mediatek,bled-pwm-hys-enable"))
+		val |= MT6370_BL_PWM_HYS_EN_MASK;
+
+	ret = device_property_read_u8(dev,
+				      "mediatek,bled-pwm-hys-input-th-steps",
+				      &prop_val);
+	if (!ret) {
+		prop_val = clamp_val(prop_val,
+				     MT6370_BL_PWM_HYS_TH_MIN_STEP,
+				     MT6370_BL_PWM_HYS_TH_MAX_STEP);
+		prop_val = prop_val <= 1 ? 0 :
+			   prop_val <= 4 ? 1 :
+			   prop_val <= 16 ? 2 : 3;
+		val |= prop_val;
+	}
+
+	ret = regmap_update_bits(priv->regmap, MT6370_REG_BL_PWM,
+				 val, val);
+	if (ret)
+		return ret;
+
+	val = 0;
+	if (device_property_read_bool(dev, "mediatek,bled-ovp-shutdown"))
+		val |= MT6370_BL_OVP_EN_MASK;
+
+	ret = device_property_read_u32(dev, "mediatek,bled-ovp-microvolt",
+				       &ovp_uV);
+	if (!ret) {
+		ovp_uV = clamp_val(ovp_uV, MT6370_BL_OVP_MIN_UV,
+				   MT6370_BL_OVP_MAX_UV);
+		ovp_uV = DIV_ROUND_UP(ovp_uV - MT6370_BL_OVP_MIN_UV,
+				      MT6370_BL_OVP_STEP_UV);
+		val |= ovp_uV << MT6370_BL_OVP_SEL_SHIFT;
+	}
+
+	if (device_property_read_bool(dev, "mediatek,bled-ocp-shutdown"))
+		val |= MT6370_BL_OC_EN_MASK;
+
+	ret = device_property_read_u32(dev, "mediatek,bled-ocp-microamp",
+				       &ocp_uA);
+	if (!ret) {
+		ocp_uA = clamp_val(ocp_uA, MT6370_BL_OCP_MIN_UA,
+				   MT6370_BL_OCP_MAX_UA);
+		ocp_uA = DIV_ROUND_UP(ocp_uA - MT6370_BL_OCP_MIN_UA,
+				      MT6370_BL_OCP_STEP_UA);
+		val |= ocp_uA << MT6370_BL_OC_SEL_SHIFT;
+	}
+
+	ret = regmap_update_bits(priv->regmap, MT6370_REG_BL_BSTCTRL,
+				 val, val);
+	if (ret)
+		return ret;
+
+	/* Common properties */
+	ret = device_property_read_u32(dev, "max-brightness", &brightness);
+	if (ret)
+		brightness = priv->def_max_brightness;
+
+	props->max_brightness = min_t(u32, brightness, priv->def_max_brightness);
+
+	ret = device_property_read_u32(dev, "default-brightness", &brightness);
+	if (ret)
+		brightness = props->max_brightness;
+
+	props->brightness = min_t(u32, brightness, props->max_brightness);
+
+	val = 0;
+	if (device_property_read_bool(dev, "mediatek,bled-exponential-mode-enable")) {
+		val |= MT6370_BL_CODE_MASK;
+		props->scale = BACKLIGHT_SCALE_NON_LINEAR;
+	} else
+		props->scale = BACKLIGHT_SCALE_LINEAR;
+
+	ret = device_property_read_u8(dev, "mediatek,bled-channel-use",
+				      &prop_val);
+	if (ret) {
+		dev_err(dev, "mediatek,bled-channel-use DT property missing\n");
+		return ret;
+	}
+
+	if (!prop_val || prop_val > MT6370_BL_MAX_CH) {
+		dev_err(dev,
+			"No channel specified or over than upper bound (%d)\n",
+			prop_val);
+		return -EINVAL;
+	}
+
+	mask = MT6370_BL_EXT_EN_MASK | MT6370_BL_CH_MASK;
+	val |= prop_val << MT6370_BL_CH_SHIFT;
+
+	if (priv->enable_gpio)
+		val |= MT6370_BL_EXT_EN_MASK;
+
+	return regmap_update_bits(priv->regmap, MT6370_REG_BL_EN, mask, val);
+}
+
+static int mt6370_check_vendor_info(struct mt6370_priv *priv)
+{
+	/*
+	 * Because MT6372 uses 14 bits to control the brightness,
+	 * MT6370 and MT6371 use 11 bits. This function is used
+	 * to check the vendor's ID and set the relative hardware
+	 * mask, shift and default maximum brightness value that
+	 * should be used.
+	 */
+	unsigned int dev_info, hw_vid, of_vid;
+	int ret;
+
+	ret = regmap_read(priv->regmap, MT6370_REG_DEV_INFO, &dev_info);
+	if (ret)
+		return ret;
+
+	of_vid = (uintptr_t)device_get_match_data(priv->dev);
+	hw_vid = FIELD_GET(MT6370_VENID_MASK, dev_info);
+	hw_vid = (hw_vid == 0x9 || hw_vid == 0xb) ? MT6370_VID_6372 : MT6370_VID_COMMON;
+	if (hw_vid != of_vid)
+		return dev_err_probe(priv->dev, -EINVAL,
+				     "Buggy DT, wrong compatible string\n");
+
+	if (hw_vid == MT6370_VID_6372) {
+		priv->dim2_mask = MT6370_BL_DIM2_6372_MASK;
+		priv->dim2_shift = MT6370_BL_DIM2_6372_SHIFT;
+		priv->def_max_brightness = MT6370_BL_MAX_6372_BRIGHTNESS;
+	} else {
+		priv->dim2_mask = MT6370_BL_DIM2_COMMON_MASK;
+		priv->dim2_shift = MT6370_BL_DIM2_COMMON_SHIFT;
+		priv->def_max_brightness = MT6370_BL_MAX_COMMON_BRIGHTNESS;
+	}
+
+	return 0;
+}
+
+static int mt6370_bl_probe(struct platform_device *pdev)
+{
+	struct backlight_properties props = {
+		.type = BACKLIGHT_RAW,
+	};
+	struct device *dev = &pdev->dev;
+	struct mt6370_priv *priv;
+	int ret;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+
+	priv->regmap = dev_get_regmap(dev->parent, NULL);
+	if (!priv->regmap)
+		return dev_err_probe(dev, -ENODEV, "Failed to get regmap\n");
+
+	ret = mt6370_check_vendor_info(priv);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to check vendor info\n");
+
+	priv->enable_gpio = devm_gpiod_get_optional(dev, "enable",
+						    GPIOD_OUT_HIGH);
+	if (IS_ERR(priv->enable_gpio))
+		return dev_err_probe(dev, PTR_ERR(priv->enable_gpio),
+				     "Failed to get 'enable' gpio\n");
+
+	ret = mt6370_init_backlight_properties(priv, &props);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to init backlight properties\n");
+
+	priv->bl = devm_backlight_device_register(dev, pdev->name, dev, priv,
+						  &mt6370_bl_ops, &props);
+	if (IS_ERR(priv->bl))
+		return dev_err_probe(dev, PTR_ERR(priv->bl),
+				     "Failed to register backlight\n");
+
+	backlight_update_status(priv->bl);
+	platform_set_drvdata(pdev, priv);
+
+	return 0;
+}
+
+static int mt6370_bl_remove(struct platform_device *pdev)
+{
+	struct mt6370_priv *priv = platform_get_drvdata(pdev);
+	struct backlight_device *bl_dev = priv->bl;
+
+	bl_dev->props.brightness = 0;
+	backlight_update_status(priv->bl);
+
+	return 0;
+}
+
+static const struct of_device_id mt6370_bl_of_match[] = {
+	{ .compatible = "mediatek,mt6370-backlight", .data = (void *)MT6370_VID_COMMON },
+	{ .compatible = "mediatek,mt6372-backlight", .data = (void *)MT6370_VID_6372 },
+	{}
+};
+MODULE_DEVICE_TABLE(of, mt6370_bl_of_match);
+
+static struct platform_driver mt6370_bl_driver = {
+	.driver = {
+		.name = "mt6370-backlight",
+		.of_match_table = mt6370_bl_of_match,
+	},
+	.probe = mt6370_bl_probe,
+	.remove = mt6370_bl_remove,
+};
+module_platform_driver(mt6370_bl_driver);
+
+MODULE_AUTHOR("ChiaEn Wu <chiaen_wu@richtek.com>");
+MODULE_DESCRIPTION("MediaTek MT6370 Backlight Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/video/backlight/tosa_bl.c b/drivers/video/backlight/tosa_bl.c
index 6df6fcd132e3..f55b3d616a87 100644
--- a/drivers/video/backlight/tosa_bl.c
+++ b/drivers/video/backlight/tosa_bl.c
@@ -121,12 +121,11 @@ err_reg:
 	return ret;
 }
 
-static int tosa_bl_remove(struct i2c_client *client)
+static void tosa_bl_remove(struct i2c_client *client)
 {
 	struct tosa_bl_data *data = i2c_get_clientdata(client);
 
 	data->bl = NULL;
-	return 0;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/video/fbdev/arkfb.c b/drivers/video/fbdev/arkfb.c
index a2a381631628..a317d9fe1d67 100644
--- a/drivers/video/fbdev/arkfb.c
+++ b/drivers/video/fbdev/arkfb.c
@@ -11,6 +11,7 @@
  *  Code is based on s3fb
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -956,6 +957,10 @@ static int ark_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	int rc;
 	u8 regval;
 
+	rc = aperture_remove_conflicting_pci_devices(dev, "arkfb");
+	if (rc < 0)
+		return rc;
+
 	/* Ignore secondary VGA device because there is no VGA arbitration */
 	if (! svga_primary_device(dev)) {
 		dev_info(&(dev->dev), "ignoring secondary device\n");
diff --git a/drivers/video/fbdev/asiliantfb.c b/drivers/video/fbdev/asiliantfb.c
index f8ef62542f7f..3818437a8f69 100644
--- a/drivers/video/fbdev/asiliantfb.c
+++ b/drivers/video/fbdev/asiliantfb.c
@@ -29,6 +29,7 @@
  *  more details.
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -545,6 +546,10 @@ static int asiliantfb_pci_init(struct pci_dev *dp,
 	struct fb_info *p;
 	int err;
 
+	err = aperture_remove_conflicting_pci_devices(dp, "asiliantfb");
+	if (err)
+		return err;
+
 	if ((dp->resource[0].flags & IORESOURCE_MEM) == 0)
 		return -ENODEV;
 	addr = pci_resource_start(dp, 0);
diff --git a/drivers/video/fbdev/aty/aty128fb.c b/drivers/video/fbdev/aty/aty128fb.c
index b26c81233b6b..57e398fe7a81 100644
--- a/drivers/video/fbdev/aty/aty128fb.c
+++ b/drivers/video/fbdev/aty/aty128fb.c
@@ -23,7 +23,7 @@
  *			- Convert to new framebuffer API,
  *			  fix colormap setting at 16 bits/pixel (565)
  *
- *		  Paul Mundt 
+ *		  Paul Mundt
  *		  	- PCI hotplug
  *
  *		  Jon Smirl <jonsmirl@yahoo.com>
@@ -47,6 +47,7 @@
  */
 
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/kernel.h>
@@ -520,13 +521,13 @@ static const struct fb_ops aty128fb_ops = {
      *	- endian conversions may possibly be avoided by
      *    using the other register aperture. TODO.
      */
-static inline u32 _aty_ld_le32(volatile unsigned int regindex, 
+static inline u32 _aty_ld_le32(volatile unsigned int regindex,
 			       const struct aty128fb_par *par)
 {
 	return readl (par->regbase + regindex);
 }
 
-static inline void _aty_st_le32(volatile unsigned int regindex, u32 val, 
+static inline void _aty_st_le32(volatile unsigned int regindex, u32 val,
 				const struct aty128fb_par *par)
 {
 	writel (val, par->regbase + regindex);
@@ -559,12 +560,12 @@ static inline void _aty_st_8(unsigned int regindex, u8 val,
 
 static u32 _aty_ld_pll(unsigned int pll_index,
 		       const struct aty128fb_par *par)
-{       
+{
 	aty_st_8(CLOCK_CNTL_INDEX, pll_index & 0x3F);
 	return aty_ld_le32(CLOCK_CNTL_DATA);
 }
 
-    
+
 static void _aty_st_pll(unsigned int pll_index, u32 val,
 			const struct aty128fb_par *par)
 {
@@ -619,7 +620,7 @@ static int register_test(const struct aty128fb_par *par)
 		aty_st_le32(BIOS_0_SCRATCH, 0xAAAAAAAA);
 
 		if (aty_ld_le32(BIOS_0_SCRATCH) == 0xAAAAAAAA)
-			flag = 1; 
+			flag = 1;
 	}
 
 	aty_st_le32(BIOS_0_SCRATCH, val);	// restore value
@@ -901,7 +902,7 @@ static void aty128_get_pllinfo(struct aty128fb_par *par,
 
 	bios_hdr = BIOS_IN16(0x48);
 	bios_pll = BIOS_IN16(bios_hdr + 0x30);
-	
+
 	par->constants.ppll_max = BIOS_IN32(bios_pll + 0x16);
 	par->constants.ppll_min = BIOS_IN32(bios_pll + 0x12);
 	par->constants.xclk = BIOS_IN16(bios_pll + 0x08);
@@ -913,7 +914,7 @@ static void aty128_get_pllinfo(struct aty128fb_par *par,
 			par->constants.xclk, par->constants.ref_divider,
 			par->constants.ref_clk);
 
-}           
+}
 
 #ifdef CONFIG_X86
 static void __iomem *aty128_find_mem_vbios(struct aty128fb_par *par)
@@ -925,7 +926,7 @@ static void __iomem *aty128_find_mem_vbios(struct aty128fb_par *par)
 	 */
         u32  segstart;
         unsigned char __iomem *rom_base = NULL;
-                                                
+
         for (segstart=0x000c0000; segstart<0x000f0000; segstart+=0x00001000) {
                 rom_base = ioremap(segstart, 0x10000);
 		if (rom_base == NULL)
@@ -1118,12 +1119,12 @@ static int aty128_var_to_crtc(const struct fb_var_screeninfo *var,
 		v_sync_wid = 1;
 	else if (v_sync_wid > 0x1f)        /* 0x1f = max vwidth */
 		v_sync_wid = 0x1f;
-    
+
 	v_sync_strt = v_disp + lower;
 
 	h_sync_pol = sync & FB_SYNC_HOR_HIGH_ACT ? 0 : 1;
 	v_sync_pol = sync & FB_SYNC_VERT_HIGH_ACT ? 0 : 1;
-    
+
 	c_sync = sync & FB_SYNC_COMP_HIGH_ACT ? (1 << 4) : 0;
 
 	crtc->gen_cntl = 0x3000000L | c_sync | (dst << 8);
@@ -1301,11 +1302,11 @@ static void aty128_set_lcd_enable(struct aty128fb_par *par, int on)
 		aty_st_le32(LVDS_GEN_CNTL, reg);
 #ifdef CONFIG_FB_ATY128_BACKLIGHT
 		aty128_bl_set_power(info, FB_BLANK_UNBLANK);
-#endif	
+#endif
 	} else {
 #ifdef CONFIG_FB_ATY128_BACKLIGHT
 		aty128_bl_set_power(info, FB_BLANK_POWERDOWN);
-#endif	
+#endif
 		reg = aty_ld_le32(LVDS_GEN_CNTL);
 		reg |= LVDS_DISPLAY_DIS;
 		aty_st_le32(LVDS_GEN_CNTL, reg);
@@ -1481,7 +1482,7 @@ static int aty128_ddafifo(struct aty128_ddafifo *dsp,
  * This actually sets the video mode.
  */
 static int aty128fb_set_par(struct fb_info *info)
-{ 
+{
 	struct aty128fb_par *par = info->par;
 	u32 config;
 	int err;
@@ -1595,7 +1596,7 @@ static int aty128_encode_var(struct fb_var_screeninfo *var,
 	var->accel_flags = par->accel_flags;
 
 	return 0;
-}           
+}
 
 
 static int aty128fb_check_var(struct fb_var_screeninfo *var,
@@ -1979,12 +1980,12 @@ static int aty128_init(struct pci_dev *pdev, const struct pci_device_id *ent)
 			/* PowerBook Titanium */
 			if (of_machine_is_compatible("PowerBook3,2"))
 				default_vmode = VMODE_1152_768_60;
-	
-			if (default_cmode > 16) 
+
+			if (default_cmode > 16)
 				default_cmode = CMODE_32;
-			else if (default_cmode > 8) 
+			else if (default_cmode > 8)
 				default_cmode = CMODE_16;
-			else 
+			else
 				default_cmode = CMODE_8;
 
 			if (mac_vmode_to_var(default_vmode, default_cmode, &var))
@@ -1994,7 +1995,7 @@ static int aty128_init(struct pci_dev *pdev, const struct pci_device_id *ent)
 #endif /* CONFIG_PPC_PMAC */
 	{
 		if (mode_option)
-			if (fb_find_mode(&var, info, mode_option, NULL, 
+			if (fb_find_mode(&var, info, mode_option, NULL,
 					 0, &defaultmode, 8) == 0)
 				var = default_var;
 	}
@@ -2055,6 +2056,10 @@ static int aty128_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	void __iomem *bios = NULL;
 #endif
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "aty128fb");
+	if (err)
+		return err;
+
 	/* Enable device in PCI config */
 	if ((err = pci_enable_device(pdev))) {
 		printk(KERN_ERR "aty128fb: Cannot enable PCI device: %d\n",
@@ -2301,7 +2306,7 @@ static int aty128fb_ioctl(struct fb_info *info, u_int cmd, u_long arg)
 	struct aty128fb_par *par = info->par;
 	u32 value;
 	int rc;
-    
+
 	switch (cmd) {
 	case FBIO_ATY128_SET_MIRROR:
 		if (par->chip_gen != rage_M3)
@@ -2313,8 +2318,8 @@ static int aty128fb_ioctl(struct fb_info *info, u_int cmd, u_long arg)
 		par->crt_on = (value & 0x02) != 0;
 		if (!par->crt_on && !par->lcd_on)
 			par->lcd_on = 1;
-		aty128_set_crt_enable(par, par->crt_on);	
-		aty128_set_lcd_enable(par, par->lcd_on);	
+		aty128_set_crt_enable(par, par->crt_on);
+		aty128_set_lcd_enable(par, par->lcd_on);
 		return 0;
 	case FBIO_ATY128_GET_MIRROR:
 		if (par->chip_gen != rage_M3)
@@ -2331,7 +2336,7 @@ static void aty128_set_suspend(struct aty128fb_par *par, int suspend)
 
 	if (!par->pdev->pm_cap)
 		return;
-		
+
 	/* Set the chip into the appropriate suspend mode (we use D2,
 	 * D3 would require a complete re-initialisation of the chip,
 	 * including PCI config registers, clocks, AGP configuration, ...)
@@ -2376,12 +2381,12 @@ static int aty128_pci_suspend_late(struct device *dev, pm_message_t state)
 	 */
 	return 0;
 #endif /* CONFIG_PPC_PMAC */
-	 
+
 	if (state.event == pdev->dev.power.power_state.event)
 		return 0;
 
 	printk(KERN_DEBUG "aty128fb: suspending...\n");
-	
+
 	console_lock();
 
 	fb_set_suspend(info, 1);
diff --git a/drivers/video/fbdev/aty/atyfb_base.c b/drivers/video/fbdev/aty/atyfb_base.c
index 14eb718bd67c..b3463d137152 100644
--- a/drivers/video/fbdev/aty/atyfb_base.c
+++ b/drivers/video/fbdev/aty/atyfb_base.c
@@ -48,6 +48,7 @@
 
 ******************************************************************************/
 
+#include <linux/aperture.h>
 #include <linux/compat.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
@@ -3533,7 +3534,11 @@ static int atyfb_pci_probe(struct pci_dev *pdev,
 	struct fb_info *info;
 	struct resource *rp;
 	struct atyfb_par *par;
-	int rc = -ENOMEM;
+	int rc;
+
+	rc = aperture_remove_conflicting_pci_devices(pdev, "atyfb");
+	if (rc)
+		return rc;
 
 	/* Enable device in PCI config */
 	if (pci_enable_device(pdev)) {
diff --git a/drivers/video/fbdev/aty/radeon_base.c b/drivers/video/fbdev/aty/radeon_base.c
index a14a8d73035c..8b28c9bddd97 100644
--- a/drivers/video/fbdev/aty/radeon_base.c
+++ b/drivers/video/fbdev/aty/radeon_base.c
@@ -7,7 +7,7 @@
  *	Copyright 2000	Ani Joshi <ajoshi@kernel.crashing.org>
  *
  *	i2c bits from Luca Tettamanti <kronos@kronoz.cjb.net>
- *	
+ *
  *	Special thanks to ATI DevRel team for their hardware donations.
  *
  *	...Insert GPL boilerplate here...
@@ -54,6 +54,7 @@
 
 #include "radeonfb.h"
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/kernel.h>
@@ -110,7 +111,7 @@ static const struct pci_device_id radeonfb_pci_table[] = {
 	/* Radeon IGP320M (U1) */
 	CHIP_DEF(PCI_CHIP_RS100_4336,	RS100,	CHIP_HAS_CRTC2 | CHIP_IS_IGP | CHIP_IS_MOBILITY),
 	/* Radeon IGP320 (A3) */
-	CHIP_DEF(PCI_CHIP_RS100_4136,	RS100,	CHIP_HAS_CRTC2 | CHIP_IS_IGP), 
+	CHIP_DEF(PCI_CHIP_RS100_4136,	RS100,	CHIP_HAS_CRTC2 | CHIP_IS_IGP),
 	/* IGP330M/340M/350M (U2) */
 	CHIP_DEF(PCI_CHIP_RS200_4337,	RS200,	CHIP_HAS_CRTC2 | CHIP_IS_IGP | CHIP_IS_MOBILITY),
 	/* IGP330/340/350 (A4) */
@@ -240,7 +241,7 @@ typedef struct {
  * interfere with anything
  */
 static reg_val common_regs[] = {
-	{ OVR_CLR, 0 },	
+	{ OVR_CLR, 0 },
 	{ OVR_WID_LEFT_RIGHT, 0 },
 	{ OVR_WID_TOP_BOTTOM, 0 },
 	{ OV0_SCALE_CNTL, 0 },
@@ -255,7 +256,7 @@ static reg_val common_regs[] = {
 /*
  * globals
  */
-        
+
 static char *mode_option;
 static char *monitor_layout;
 static bool noaccel = 0;
@@ -422,7 +423,7 @@ static int radeon_map_ROM(struct radeonfb_info *rinfo, struct pci_dev *dev)
 	 * ROM somewhere in the first meg. We will just ignore the copy
 	 * and use the ROM directly.
 	 */
-    
+
     	/* Fix from ATI for problem with Radeon hardware not leaving ROM enabled */
     	unsigned int temp;
 	temp = INREG(MPP_TB_CONFIG);
@@ -430,14 +431,14 @@ static int radeon_map_ROM(struct radeonfb_info *rinfo, struct pci_dev *dev)
 	temp |= 0x04 << 24;
 	OUTREG(MPP_TB_CONFIG, temp);
 	temp = INREG(MPP_TB_CONFIG);
-                                                                                                          
+
 	rom = pci_map_rom(dev, &rom_size);
 	if (!rom) {
 		printk(KERN_ERR "radeonfb (%s): ROM failed to map\n",
 		       pci_name(rinfo->pdev));
 		return -ENOMEM;
 	}
-	
+
 	rinfo->bios_seg = rom;
 
 	/* Very simple test to make sure it appeared */
@@ -515,7 +516,7 @@ static int  radeon_find_mem_vbios(struct radeonfb_info *rinfo)
 	 */
         u32  segstart;
 	void __iomem *rom_base = NULL;
-                                                
+
         for(segstart=0x000c0000; segstart<0x000f0000; segstart+=0x00001000) {
                 rom_base = ioremap(segstart, 0x10000);
 		if (rom_base == NULL)
@@ -605,16 +606,16 @@ static int radeon_probe_pll_params(struct radeonfb_info *rinfo)
 	for(i=0; i<1000000; i++)
 		if (((INREG(CRTC_VLINE_CRNT_VLINE) >> 16) & 0x3ff) == 0)
 			break;
-	
+
 	stop_time = ktime_get();
-	
+
 	local_irq_enable();
 
 	total_usecs = ktime_us_delta(stop_time, start_time);
 	if (total_usecs >= 10 * USEC_PER_SEC || total_usecs == 0)
 		return -1;
 	hz = USEC_PER_SEC/(u32)total_usecs;
- 
+
 	hTotal = ((INREG(CRTC_H_TOTAL_DISP) & 0x1ff) + 1) * 8;
 	vTotal = ((INREG(CRTC_V_TOTAL_DISP) & 0x3ff) + 1);
 	vclk = (long long)hTotal * (long long)vTotal * hz;
@@ -662,7 +663,7 @@ static int radeon_probe_pll_params(struct radeonfb_info *rinfo)
 		denom *= 3;
 		break;
 	case 6:
-		denom *= 6;   
+		denom *= 6;
 		break;
 	case 7:
 		denom *= 12;
@@ -878,7 +879,7 @@ static int radeonfb_check_var (struct fb_var_screeninfo *var, struct fb_info *in
                         v.green.length = 6;
                         v.blue.length = 5;
                         v.transp.offset = v.transp.length = 0;
-                        break;                          
+                        break;
                 case 24:
                         nom = 4;
                         den = 1;
@@ -908,7 +909,7 @@ static int radeonfb_check_var (struct fb_var_screeninfo *var, struct fb_info *in
 		v.yres_virtual = v.yres;
 	if (v.xres_virtual < v.xres)
 		v.xres_virtual = v.xres;
-                
+
 
 	/* XXX I'm adjusting xres_virtual to the pitch, that may help XFree
 	 * with some panels, though I don't quite like this solution
@@ -929,14 +930,14 @@ static int radeonfb_check_var (struct fb_var_screeninfo *var, struct fb_info *in
 
         if (v.xoffset > v.xres_virtual - v.xres)
                 v.xoffset = v.xres_virtual - v.xres - 1;
-                        
+
         if (v.yoffset > v.yres_virtual - v.yres)
                 v.yoffset = v.yres_virtual - v.yres - 1;
-         
+
         v.red.msb_right = v.green.msb_right = v.blue.msb_right =
                           v.transp.offset = v.transp.length =
                           v.transp.msb_right = 0;
-	
+
         memcpy(var, &v, sizeof(v));
 
         return 0;
@@ -951,7 +952,7 @@ static int radeonfb_pan_display (struct fb_var_screeninfo *var,
 	if ((var->xoffset + info->var.xres > info->var.xres_virtual)
 	    || (var->yoffset + info->var.yres > info->var.yres_virtual))
 		return -EINVAL;
-                
+
         if (rinfo->asleep)
         	return 0;
 
@@ -1151,7 +1152,7 @@ static int radeonfb_blank (int blank, struct fb_info *info)
 
 	if (rinfo->asleep)
 		return 0;
-		
+
 	return radeon_screen_blank(rinfo, blank, 0);
 }
 
@@ -1401,7 +1402,7 @@ static void radeon_write_pll_regs(struct radeonfb_info *rinfo, struct radeon_reg
 		} else {
 			/* R300 uses ref_div_acc field as real ref divider */
 			OUTPLLP(PPLL_REF_DIV,
-				(mode->ppll_ref_div << R300_PPLL_REF_DIV_ACC_SHIFT), 
+				(mode->ppll_ref_div << R300_PPLL_REF_DIV_ACC_SHIFT),
 				~R300_PPLL_REF_DIV_ACC_MASK);
 		}
 	} else
@@ -1423,7 +1424,7 @@ static void radeon_write_pll_regs(struct radeonfb_info *rinfo, struct radeon_reg
 	   workaround shouldn't have any effect on them. */
 	for (i = 0; (i < 10000 && INPLL(PPLL_REF_DIV) & PPLL_ATOMIC_UPDATE_R); i++)
 		;
-	
+
 	OUTPLL(HTOTAL_CNTL, 0);
 
 	/* Clear reset & atomic update */
@@ -1510,7 +1511,7 @@ void radeon_write_mode (struct radeonfb_info *rinfo, struct radeon_regs *mode,
 
 	radeon_fifo_wait(2);
 	OUTPLL(VCLK_ECP_CNTL, mode->vclk_ecp_cntl);
-	
+
 	return;
 }
 
@@ -1735,7 +1736,7 @@ static int radeonfb_set_par(struct fb_info *info)
 	/* Clear auto-center etc... */
 	newmode->crtc_more_cntl = rinfo->init_state.crtc_more_cntl;
 	newmode->crtc_more_cntl &= 0xfffffff0;
-	
+
 	if ((primary_mon == MT_DFP) || (primary_mon == MT_LCD)) {
 		newmode->crtc_ext_cntl = VGA_ATI_LINEAR | XCRT_CNT_EN;
 		if (mirror)
@@ -1793,7 +1794,7 @@ static int radeonfb_set_par(struct fb_info *info)
 			newmode->surface_cntl |= NONSURF_AP0_SWP_16BPP;
 			newmode->surface_cntl |= NONSURF_AP1_SWP_16BPP;
 			break;
-		case 24:	
+		case 24:
 		case 32:
 			newmode->surface_cntl |= NONSURF_AP0_SWP_32BPP;
 			newmode->surface_cntl |= NONSURF_AP1_SWP_32BPP;
@@ -2028,7 +2029,7 @@ static void fixup_memory_mappings(struct radeonfb_info *rinfo)
 	}
 	save_crtc_gen_cntl = INREG(CRTC_GEN_CNTL);
 	save_crtc_ext_cntl = INREG(CRTC_EXT_CNTL);
-	
+
 	OUTREG(CRTC_EXT_CNTL, save_crtc_ext_cntl | CRTC_DISPLAY_DIS);
 	OUTREG(CRTC_GEN_CNTL, save_crtc_gen_cntl | CRTC_DISP_REQ_EN_B);
 	mdelay(100);
@@ -2038,7 +2039,7 @@ static void fixup_memory_mappings(struct radeonfb_info *rinfo)
 
 #ifdef SET_MC_FB_FROM_APERTURE
 	/* Set framebuffer to be at the same address as set in PCI BAR */
-	OUTREG(MC_FB_LOCATION, 
+	OUTREG(MC_FB_LOCATION,
 		((aper_base + aper_size - 1) & 0xffff0000) | (aper_base >> 16));
 	rinfo->fb_local_base = aper_base;
 #else
@@ -2079,7 +2080,7 @@ static void fixup_memory_mappings(struct radeonfb_info *rinfo)
 	OUTREG(CRTC_GEN_CNTL, save_crtc_gen_cntl);
 	OUTREG(CRTC_EXT_CNTL, save_crtc_ext_cntl);
 	if (rinfo->has_CRTC2)
-		OUTREG(CRTC2_GEN_CNTL, save_crtc2_gen_cntl);	
+		OUTREG(CRTC2_GEN_CNTL, save_crtc2_gen_cntl);
 
 	pr_debug("aper_base: %08x MC_FB_LOC to: %08x, MC_AGP_LOC to: %08x\n",
 		aper_base,
@@ -2239,20 +2240,10 @@ static const struct bin_attribute edid2_attr = {
 
 static int radeon_kick_out_firmware_fb(struct pci_dev *pdev)
 {
-	struct apertures_struct *ap;
-
-	ap = alloc_apertures(1);
-	if (!ap)
-		return -ENOMEM;
-
-	ap->ranges[0].base = pci_resource_start(pdev, 0);
-	ap->ranges[0].size = pci_resource_len(pdev, 0);
+	resource_size_t base = pci_resource_start(pdev, 0);
+	resource_size_t size = pci_resource_len(pdev, 0);
 
-	remove_conflicting_framebuffers(ap, KBUILD_MODNAME, false);
-
-	kfree(ap);
-
-	return 0;
+	return aperture_remove_conflicting_devices(base, size, false, KBUILD_MODNAME);
 }
 
 static int radeonfb_pci_register(struct pci_dev *pdev,
@@ -2265,7 +2256,7 @@ static int radeonfb_pci_register(struct pci_dev *pdev,
 	int err = 0;
 
 	pr_debug("radeonfb_pci_register BEGIN\n");
-	
+
 	/* Enable device in PCI config */
 	ret = pci_enable_device(pdev);
 	if (ret < 0) {
@@ -2280,9 +2271,9 @@ static int radeonfb_pci_register(struct pci_dev *pdev,
 		goto err_disable;
 	}
 	rinfo = info->par;
-	rinfo->info = info;	
+	rinfo->info = info;
 	rinfo->pdev = pdev;
-	
+
 	spin_lock_init(&rinfo->reg_lock);
 	timer_setup(&rinfo->lvds_timer, radeon_lvds_timer_func, 0);
 
@@ -2521,7 +2512,7 @@ static void radeonfb_pci_unregister(struct pci_dev *pdev)
 {
         struct fb_info *info = pci_get_drvdata(pdev);
         struct radeonfb_info *rinfo = info->par;
- 
+
         if (!rinfo)
                 return;
 
@@ -2540,7 +2531,7 @@ static void radeonfb_pci_unregister(struct pci_dev *pdev)
 
         iounmap(rinfo->mmio_base);
         iounmap(rinfo->fb_base);
- 
+
 	pci_release_region(pdev, 2);
 	pci_release_region(pdev, 0);
 
@@ -2550,7 +2541,7 @@ static void radeonfb_pci_unregister(struct pci_dev *pdev)
 		fb_destroy_modedb(rinfo->mon1_modedb);
 #ifdef CONFIG_FB_RADEON_I2C
 	radeon_delete_i2c_busses(rinfo);
-#endif        
+#endif
 	fb_dealloc_cmap(&info->cmap);
         framebuffer_release(info);
 }
diff --git a/drivers/video/fbdev/carminefb.c b/drivers/video/fbdev/carminefb.c
index 3a1c2e0739a1..4651b48a87f9 100644
--- a/drivers/video/fbdev/carminefb.c
+++ b/drivers/video/fbdev/carminefb.c
@@ -7,6 +7,7 @@
  * - FB1 is display 1 with unique memory area
  * - both display use 32 bit colors
  */
+#include <linux/aperture.h>
 #include <linux/delay.h>
 #include <linux/errno.h>
 #include <linux/fb.h>
@@ -614,6 +615,10 @@ static int carminefb_probe(struct pci_dev *dev, const struct pci_device_id *ent)
 	struct fb_info *info;
 	int ret;
 
+	ret = aperture_remove_conflicting_pci_devices(dev, "carminefb");
+	if (ret)
+		return ret;
+
 	ret = pci_enable_device(dev);
 	if (ret)
 		return ret;
diff --git a/drivers/video/fbdev/chipsfb.c b/drivers/video/fbdev/chipsfb.c
index 2b00a9d554fc..f1c1c95c1fdf 100644
--- a/drivers/video/fbdev/chipsfb.c
+++ b/drivers/video/fbdev/chipsfb.c
@@ -14,6 +14,7 @@
  *  more details.
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -122,7 +123,7 @@ static int chipsfb_set_par(struct fb_info *info)
 		info->var.blue.offset = 0;
 		info->var.red.length = info->var.green.length =
 			info->var.blue.length = 5;
-		
+
 	} else {
 		/* p->var.bits_per_pixel == 8 */
 		write_cr(0x13, 100);		// Set line length (doublewords)
@@ -131,13 +132,13 @@ static int chipsfb_set_par(struct fb_info *info)
 		write_xr(0x20, 0x00);		// 8 bit blitter mode
 
 		info->fix.line_length = 800;
-		info->fix.visual = FB_VISUAL_PSEUDOCOLOR;		
+		info->fix.visual = FB_VISUAL_PSEUDOCOLOR;
 
  		info->var.red.offset = info->var.green.offset =
 			info->var.blue.offset = 0;
 		info->var.red.length = info->var.green.length =
 			info->var.blue.length = 8;
-		
+
 	}
 	return 0;
 }
@@ -351,7 +352,11 @@ static int chipsfb_pci_init(struct pci_dev *dp, const struct pci_device_id *ent)
 	struct fb_info *p;
 	unsigned long addr;
 	unsigned short cmd;
-	int rc = -ENODEV;
+	int rc;
+
+	rc = aperture_remove_conflicting_pci_devices(dp, "chipsfb");
+	if (rc)
+		return rc;
 
 	if (pci_enable_device(dp) < 0) {
 		dev_err(&dp->dev, "Cannot enable PCI device\n");
diff --git a/drivers/video/fbdev/cirrusfb.c b/drivers/video/fbdev/cirrusfb.c
index 2a9fa06881b5..b08bee43779a 100644
--- a/drivers/video/fbdev/cirrusfb.c
+++ b/drivers/video/fbdev/cirrusfb.c
@@ -34,6 +34,7 @@
  *
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -2085,6 +2086,10 @@ static int cirrusfb_pci_register(struct pci_dev *pdev,
 	unsigned long board_addr, board_size;
 	int ret;
 
+	ret = aperture_remove_conflicting_pci_devices(pdev, "cirrusfb");
+	if (ret)
+		return ret;
+
 	ret = pci_enable_device(pdev);
 	if (ret < 0) {
 		printk(KERN_ERR "cirrusfb: Cannot enable PCI device\n");
diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c
index 02b0cf2cfafe..1e70d8c67653 100644
--- a/drivers/video/fbdev/core/fbmem.c
+++ b/drivers/video/fbdev/core/fbmem.c
@@ -13,13 +13,13 @@
 
 #include <linux/module.h>
 
+#include <linux/aperture.h>
 #include <linux/compat.h>
 #include <linux/types.h>
 #include <linux/errno.h>
 #include <linux/kernel.h>
 #include <linux/major.h>
 #include <linux/slab.h>
-#include <linux/sysfb.h>
 #include <linux/mm.h>
 #include <linux/mman.h>
 #include <linux/vt.h>
@@ -40,6 +40,7 @@
 
 #include <asm/fb.h>
 
+#include <video/vga.h>
 
     /*
      *  Frame buffer device initialization and setup routines
@@ -50,10 +51,10 @@
 static DEFINE_MUTEX(registration_lock);
 
 struct fb_info *registered_fb[FB_MAX] __read_mostly;
-EXPORT_SYMBOL(registered_fb);
-
 int num_registered_fb __read_mostly;
-EXPORT_SYMBOL(num_registered_fb);
+#define for_each_registered_fb(i)		\
+	for (i = 0; i < FB_MAX; i++)		\
+		if (!registered_fb[i]) {} else
 
 bool fb_center_logo __read_mostly;
 
@@ -1525,103 +1526,6 @@ static int fb_check_foreignness(struct fb_info *fi)
 	return 0;
 }
 
-static bool apertures_overlap(struct aperture *gen, struct aperture *hw)
-{
-	/* is the generic aperture base the same as the HW one */
-	if (gen->base == hw->base)
-		return true;
-	/* is the generic aperture base inside the hw base->hw base+size */
-	if (gen->base > hw->base && gen->base < hw->base + hw->size)
-		return true;
-	return false;
-}
-
-static bool fb_do_apertures_overlap(struct apertures_struct *gena,
-				    struct apertures_struct *hwa)
-{
-	int i, j;
-	if (!hwa || !gena)
-		return false;
-
-	for (i = 0; i < hwa->count; ++i) {
-		struct aperture *h = &hwa->ranges[i];
-		for (j = 0; j < gena->count; ++j) {
-			struct aperture *g = &gena->ranges[j];
-			printk(KERN_DEBUG "checking generic (%llx %llx) vs hw (%llx %llx)\n",
-				(unsigned long long)g->base,
-				(unsigned long long)g->size,
-				(unsigned long long)h->base,
-				(unsigned long long)h->size);
-			if (apertures_overlap(g, h))
-				return true;
-		}
-	}
-
-	return false;
-}
-
-static void do_unregister_framebuffer(struct fb_info *fb_info);
-
-#define VGA_FB_PHYS 0xA0000
-static void do_remove_conflicting_framebuffers(struct apertures_struct *a,
-					       const char *name, bool primary)
-{
-	int i;
-
-restart_removal:
-	/* check all firmware fbs and kick off if the base addr overlaps */
-	for_each_registered_fb(i) {
-		struct apertures_struct *gen_aper;
-		struct device *device;
-
-		if (!(registered_fb[i]->flags & FBINFO_MISC_FIRMWARE))
-			continue;
-
-		gen_aper = registered_fb[i]->apertures;
-		device = registered_fb[i]->device;
-		if (fb_do_apertures_overlap(gen_aper, a) ||
-			(primary && gen_aper && gen_aper->count &&
-			 gen_aper->ranges[0].base == VGA_FB_PHYS)) {
-
-			printk(KERN_INFO "fb%d: switching to %s from %s\n",
-			       i, name, registered_fb[i]->fix.id);
-
-			/*
-			 * If we kick-out a firmware driver, we also want to remove
-			 * the underlying platform device, such as simple-framebuffer,
-			 * VESA, EFI, etc. A native driver will then be able to
-			 * allocate the memory range.
-			 *
-			 * If it's not a platform device, at least print a warning. A
-			 * fix would add code to remove the device from the system. For
-			 * framebuffers without any Linux device, print a warning as
-			 * well.
-			 */
-			if (!device) {
-				pr_warn("fb%d: no device set\n", i);
-				do_unregister_framebuffer(registered_fb[i]);
-			} else if (dev_is_platform(device)) {
-				/*
-				 * Drop the lock because if the device is unregistered, its
-				 * driver will call to unregister_framebuffer(), that takes
-				 * this lock.
-				 */
-				mutex_unlock(&registration_lock);
-				platform_device_unregister(to_platform_device(device));
-				mutex_lock(&registration_lock);
-			} else {
-				pr_warn("fb%d: cannot remove device\n", i);
-				do_unregister_framebuffer(registered_fb[i]);
-			}
-			/*
-			 * Restart the removal loop now that the device has been
-			 * unregistered and its associated framebuffer gone.
-			 */
-			goto restart_removal;
-		}
-	}
-}
-
 static int do_register_framebuffer(struct fb_info *fb_info)
 {
 	int i;
@@ -1630,10 +1534,6 @@ static int do_register_framebuffer(struct fb_info *fb_info)
 	if (fb_check_foreignness(fb_info))
 		return -ENOSYS;
 
-	do_remove_conflicting_framebuffers(fb_info->apertures,
-					   fb_info->fix.id,
-					   fb_is_primary_device(fb_info));
-
 	if (num_registered_fb == FB_MAX)
 		return -ENXIO;
 
@@ -1752,100 +1652,31 @@ static void do_unregister_framebuffer(struct fb_info *fb_info)
 	put_fb_info(fb_info);
 }
 
-/**
- * remove_conflicting_framebuffers - remove firmware-configured framebuffers
- * @a: memory range, users of which are to be removed
- * @name: requesting driver name
- * @primary: also kick vga16fb if present
- *
- * This function removes framebuffer devices (initialized by firmware/bootloader)
- * which use memory range described by @a. If @a is NULL all such devices are
- * removed.
- */
-int remove_conflicting_framebuffers(struct apertures_struct *a,
-				    const char *name, bool primary)
+static int fb_aperture_acquire_for_platform_device(struct fb_info *fb_info)
 {
-	bool do_free = false;
-
-	if (!a) {
-		a = alloc_apertures(1);
-		if (!a)
-			return -ENOMEM;
-
-		a->ranges[0].base = 0;
-		a->ranges[0].size = ~0;
-		do_free = true;
-	}
-
-	/*
-	 * If a driver asked to unregister a platform device registered by
-	 * sysfb, then can be assumed that this is a driver for a display
-	 * that is set up by the system firmware and has a generic driver.
-	 *
-	 * Drivers for devices that don't have a generic driver will never
-	 * ask for this, so let's assume that a real driver for the display
-	 * was already probed and prevent sysfb to register devices later.
-	 */
-	sysfb_disable();
-
-	mutex_lock(&registration_lock);
-	do_remove_conflicting_framebuffers(a, name, primary);
-	mutex_unlock(&registration_lock);
-
-	if (do_free)
-		kfree(a);
-
-	return 0;
-}
-EXPORT_SYMBOL(remove_conflicting_framebuffers);
+	struct apertures_struct *ap = fb_info->apertures;
+	struct device *dev = fb_info->device;
+	struct platform_device *pdev;
+	unsigned int i;
+	int ret;
 
-/**
- * remove_conflicting_pci_framebuffers - remove firmware-configured framebuffers for PCI devices
- * @pdev: PCI device
- * @name: requesting driver name
- *
- * This function removes framebuffer devices (eg. initialized by firmware)
- * using memory range configured for any of @pdev's memory bars.
- *
- * The function assumes that PCI device with shadowed ROM drives a primary
- * display and so kicks out vga16fb.
- */
-int remove_conflicting_pci_framebuffers(struct pci_dev *pdev, const char *name)
-{
-	struct apertures_struct *ap;
-	bool primary = false;
-	int err, idx, bar;
+	if (!ap)
+		return 0;
 
-	for (idx = 0, bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
-		if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM))
-			continue;
-		idx++;
-	}
+	if (!dev_is_platform(dev))
+		return 0;
 
-	ap = alloc_apertures(idx);
-	if (!ap)
-		return -ENOMEM;
+	pdev = to_platform_device(dev);
 
-	for (idx = 0, bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
-		if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM))
-			continue;
-		ap->ranges[idx].base = pci_resource_start(pdev, bar);
-		ap->ranges[idx].size = pci_resource_len(pdev, bar);
-		pci_dbg(pdev, "%s: bar %d: 0x%lx -> 0x%lx\n", __func__, bar,
-			(unsigned long)pci_resource_start(pdev, bar),
-			(unsigned long)pci_resource_end(pdev, bar));
-		idx++;
+	for (ret = 0, i = 0; i < ap->count; ++i) {
+		ret = devm_aperture_acquire_for_platform_device(pdev, ap->ranges[i].base,
+								ap->ranges[i].size);
+		if (ret)
+			break;
 	}
 
-#ifdef CONFIG_X86
-	primary = pdev->resource[PCI_ROM_RESOURCE].flags &
-					IORESOURCE_ROM_SHADOW;
-#endif
-	err = remove_conflicting_framebuffers(ap, name, primary);
-	kfree(ap);
-	return err;
+	return ret;
 }
-EXPORT_SYMBOL(remove_conflicting_pci_framebuffers);
 
 /**
  *	register_framebuffer - registers a frame buffer device
@@ -1861,6 +1692,12 @@ register_framebuffer(struct fb_info *fb_info)
 {
 	int ret;
 
+	if (fb_info->flags & FBINFO_MISC_FIRMWARE) {
+		ret = fb_aperture_acquire_for_platform_device(fb_info);
+		if (ret)
+			return ret;
+	}
+
 	mutex_lock(&registration_lock);
 	ret = do_register_framebuffer(fb_info);
 	mutex_unlock(&registration_lock);
diff --git a/drivers/video/fbdev/cyber2000fb.c b/drivers/video/fbdev/cyber2000fb.c
index 8f041f9b14c7..585af90a68a5 100644
--- a/drivers/video/fbdev/cyber2000fb.c
+++ b/drivers/video/fbdev/cyber2000fb.c
@@ -33,6 +33,7 @@
  * (which, incidentally, is about the same saving as a 2.5in hard disk
  * entering standby mode.)
  */
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -1720,6 +1721,10 @@ static int cyberpro_pci_probe(struct pci_dev *dev,
 
 	sprintf(name, "CyberPro%4X", id->device);
 
+	err = aperture_remove_conflicting_pci_devices(dev, name);
+	if (err)
+		return err;
+
 	err = pci_enable_device(dev);
 	if (err)
 		return err;
diff --git a/drivers/video/fbdev/geode/gx1fb_core.c b/drivers/video/fbdev/geode/gx1fb_core.c
index e41204ecb0e3..1514c653a84f 100644
--- a/drivers/video/fbdev/geode/gx1fb_core.c
+++ b/drivers/video/fbdev/geode/gx1fb_core.c
@@ -6,6 +6,7 @@
  * Copyright (C) 2005 Arcom Control Systems Ltd.
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -320,6 +321,10 @@ static int gx1fb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	struct fb_info *info;
 	int ret;
 
+	ret = aperture_remove_conflicting_pci_devices(pdev, "gx1fb");
+	if (ret)
+		return ret;
+
 	info = gx1fb_init_fbinfo(&pdev->dev);
 	if (!info)
 		return -ENOMEM;
diff --git a/drivers/video/fbdev/geode/gxfb_core.c b/drivers/video/fbdev/geode/gxfb_core.c
index 44089b331f91..2527bd80ec5f 100644
--- a/drivers/video/fbdev/geode/gxfb_core.c
+++ b/drivers/video/fbdev/geode/gxfb_core.c
@@ -15,6 +15,7 @@
  *
  * 16 MiB of framebuffer memory is assumed to be available.
  */
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -364,6 +365,10 @@ static int gxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	struct fb_videomode *modedb_ptr;
 	unsigned int modedb_size;
 
+	ret = aperture_remove_conflicting_pci_devices(pdev, "gxfb");
+	if (ret)
+		return ret;
+
 	info = gxfb_init_fbinfo(&pdev->dev);
 	if (!info)
 		return -ENOMEM;
diff --git a/drivers/video/fbdev/geode/lxfb_core.c b/drivers/video/fbdev/geode/lxfb_core.c
index 66c81262d18f..9d26592dbfce 100644
--- a/drivers/video/fbdev/geode/lxfb_core.c
+++ b/drivers/video/fbdev/geode/lxfb_core.c
@@ -6,6 +6,7 @@
  * Built from gxfb (which is Copyright (C) 2006 Arcom Control Systems Ltd.)
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -484,6 +485,10 @@ static int lxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	struct fb_videomode *modedb_ptr;
 	unsigned int modedb_size;
 
+	ret = aperture_remove_conflicting_pci_devices(pdev, "lxfb");
+	if (ret)
+		return ret;
+
 	info = lxfb_init_fbinfo(&pdev->dev);
 
 	if (info == NULL)
diff --git a/drivers/video/fbdev/gxt4500.c b/drivers/video/fbdev/gxt4500.c
index 94588b809ebf..0dcef4bec8d7 100644
--- a/drivers/video/fbdev/gxt4500.c
+++ b/drivers/video/fbdev/gxt4500.c
@@ -6,6 +6,7 @@
  * Copyright (C) 2006 Paul Mackerras, IBM Corp. <paulus@samba.org>
  */
 
+#include <linux/aperture.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/fb.h>
@@ -621,6 +622,10 @@ static int gxt4500_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct fb_var_screeninfo var;
 	enum gxt_cards cardtype;
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "gxt4500fb");
+	if (err)
+		return err;
+
 	err = pci_enable_device(pdev);
 	if (err) {
 		dev_err(&pdev->dev, "gxt4500: cannot enable PCI device: %d\n",
diff --git a/drivers/video/fbdev/hyperv_fb.c b/drivers/video/fbdev/hyperv_fb.c
index b58b445bb529..072ce07ba9e0 100644
--- a/drivers/video/fbdev/hyperv_fb.c
+++ b/drivers/video/fbdev/hyperv_fb.c
@@ -45,6 +45,7 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/vmalloc.h>
@@ -1070,8 +1071,9 @@ static int hvfb_getmem(struct hv_device *hdev, struct fb_info *info)
 	info->screen_size = dio_fb_size;
 
 getmem_done:
-	remove_conflicting_framebuffers(info->apertures,
-					KBUILD_MODNAME, false);
+	aperture_remove_conflicting_devices(info->apertures->ranges[0].base,
+					    info->apertures->ranges[0].size,
+					    false, KBUILD_MODNAME);
 
 	if (gen2vm) {
 		/* framebuffer is reallocated, clear screen_info to avoid misuse from kexec */
diff --git a/drivers/video/fbdev/i740fb.c b/drivers/video/fbdev/i740fb.c
index bd30d8314b68..b795f6503cb6 100644
--- a/drivers/video/fbdev/i740fb.c
+++ b/drivers/video/fbdev/i740fb.c
@@ -12,6 +12,7 @@
  *  i740fb by Patrick LERDA, v0.9
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -1013,6 +1014,10 @@ static int i740fb_probe(struct pci_dev *dev, const struct pci_device_id *ent)
 	bool found = false;
 	u8 *edid;
 
+	ret = aperture_remove_conflicting_pci_devices(dev, "i740fb");
+	if (ret)
+		return ret;
+
 	info = framebuffer_alloc(sizeof(struct i740fb_par), &(dev->dev));
 	if (!info)
 		return -ENOMEM;
diff --git a/drivers/video/fbdev/i810/i810_main.c b/drivers/video/fbdev/i810/i810_main.c
index 13bbf7fe13bf..ff09f8c20bfc 100644
--- a/drivers/video/fbdev/i810/i810_main.c
+++ b/drivers/video/fbdev/i810/i810_main.c
@@ -2,12 +2,12 @@
  *  linux/drivers/video/i810_main.c -- Intel 810 frame buffer device
  *
  *      Copyright (C) 2001 Antonino Daplas<adaplas@pol.net>
- *      All Rights Reserved      
+ *      All Rights Reserved
  *
  *      Contributors:
  *         Michael Vogt <mvogt@acm.org> - added support for Intel 815 chipsets
- *                                        and enabling the power-on state of 
- *                                        external VGA connectors for 
+ *                                        and enabling the power-on state of
+ *                                        external VGA connectors for
  *                                        secondary displays
  *
  *         Fredrik Andersson <krueger@shell.linux.se> - alpha testing of
@@ -17,10 +17,10 @@
  *                                        timings support
  *
  *	The code framework is a modification of vfb.c by Geert Uytterhoeven.
- *      DotClock and PLL calculations are partly based on i810_driver.c 
+ *      DotClock and PLL calculations are partly based on i810_driver.c
  *              in xfree86 v4.0.3 by Precision Insight.
- *      Watermark calculation and tables are based on i810_wmark.c 
- *              in xfre86 v4.0.3 by Precision Insight.  Slight modifications 
+ *      Watermark calculation and tables are based on i810_wmark.c
+ *              in xfre86 v4.0.3 by Precision Insight.  Slight modifications
  *              only to allow for integer operations instead of floating point.
  *
  *  This file is subject to the terms and conditions of the GNU General Public
@@ -28,6 +28,7 @@
  *  more details.
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -204,8 +205,8 @@ static void i810_dram_off(u8 __iomem *mmio, u8 mode)
  * @mode: protect/unprotect
  *
  * DESCRIPTION:
- * The IBM VGA standard allows protection of certain VGA registers.  
- * This will  protect or unprotect them. 
+ * The IBM VGA standard allows protection of certain VGA registers.
+ * This will  protect or unprotect them.
  */
 static void i810_protect_regs(u8 __iomem *mmio, int mode)
 {
@@ -215,7 +216,7 @@ static void i810_protect_regs(u8 __iomem *mmio, int mode)
 	reg = i810_readb(CR_DATA_CGA, mmio);
 	reg = (mode == OFF) ? reg & ~0x80 :
 		reg | 0x80;
- 		
+
 	i810_writeb(CR_INDEX_CGA, mmio, CR11);
 	i810_writeb(CR_DATA_CGA, mmio, reg);
 }
@@ -225,18 +226,18 @@ static void i810_protect_regs(u8 __iomem *mmio, int mode)
  * @par: pointer to i810fb_par structure
  *
  * DESCRIPTION:
- * Loads the P, M, and N registers.  
+ * Loads the P, M, and N registers.
  */
 static void i810_load_pll(struct i810fb_par *par)
 {
 	u32 tmp1, tmp2;
 	u8 __iomem *mmio = par->mmio_start_virtual;
-	
+
 	tmp1 = par->regs.M | par->regs.N << 16;
 	tmp2 = i810_readl(DCLK_2D, mmio);
 	tmp2 &= ~MN_MASK;
 	i810_writel(DCLK_2D, mmio, tmp1 | tmp2);
-	
+
 	tmp1 = par->regs.P;
 	tmp2 = i810_readl(DCLK_0DS, mmio);
 	tmp2 &= ~(P_OR << 16);
@@ -254,7 +255,7 @@ static void i810_load_pll(struct i810fb_par *par)
  * Load values to VGA registers
  */
 static void i810_load_vga(struct i810fb_par *par)
-{	
+{
 	u8 __iomem *mmio = par->mmio_start_virtual;
 
 	/* interlace */
@@ -327,7 +328,7 @@ static void i810_load_2d(struct i810fb_par *par)
 	u8 tmp8;
 	u8 __iomem *mmio = par->mmio_start_virtual;
 
-  	i810_writel(FW_BLC, mmio, par->watermark); 
+  	i810_writel(FW_BLC, mmio, par->watermark);
 	tmp = i810_readl(PIXCONF, mmio);
 	tmp |= 1 | 1 << 20;
 	i810_writel(PIXCONF, mmio, tmp);
@@ -339,7 +340,7 @@ static void i810_load_2d(struct i810fb_par *par)
 	tmp8 |= 2;
 	i810_writeb(GR_INDEX, mmio, GR10);
 	i810_writeb(GR_DATA, mmio, tmp8);
-}	
+}
 
 /**
  * i810_hires - enables high resolution mode
@@ -348,7 +349,7 @@ static void i810_load_2d(struct i810fb_par *par)
 static void i810_hires(u8 __iomem *mmio)
 {
 	u8 val;
-	
+
 	i810_writeb(CR_INDEX_CGA, mmio, CR80);
 	val = i810_readb(CR_DATA_CGA, mmio);
 	i810_writeb(CR_INDEX_CGA, mmio, CR80);
@@ -363,13 +364,13 @@ static void i810_hires(u8 __iomem *mmio)
  *
  * DESCRIPTION:
  * Loads the characters per line
- */	
+ */
 static void i810_load_pitch(struct i810fb_par *par)
 {
 	u32 tmp, pitch;
 	u8 val;
 	u8 __iomem *mmio = par->mmio_start_virtual;
-			
+
 	pitch = par->pitch >> 3;
 	i810_writeb(SR_INDEX, mmio, SR01);
 	val = i810_readb(SR_DATA, mmio);
@@ -381,7 +382,7 @@ static void i810_load_pitch(struct i810fb_par *par)
 	tmp = pitch & 0xFF;
 	i810_writeb(CR_INDEX_CGA, mmio, CR13);
 	i810_writeb(CR_DATA_CGA, mmio, (u8) tmp);
-	
+
 	tmp = pitch >> 8;
 	i810_writeb(CR_INDEX_CGA, mmio, CR41);
 	val = i810_readb(CR_DATA_CGA, mmio) & ~0x0F;
@@ -414,7 +415,7 @@ static void i810_load_color(struct i810fb_par *par)
 /**
  * i810_load_regs - loads all registers for the mode
  * @par: pointer to i810fb_par structure
- * 
+ *
  * DESCRIPTION:
  * Loads registers
  */
@@ -428,7 +429,7 @@ static void i810_load_regs(struct i810fb_par *par)
 	i810_load_pll(par);
 	i810_load_vga(par);
 	i810_load_vgax(par);
-	i810_dram_off(mmio, ON);	
+	i810_dram_off(mmio, ON);
 	i810_load_2d(par);
 	i810_hires(mmio);
 	i810_screen_off(mmio, ON);
@@ -443,7 +444,7 @@ static void i810_write_dac(u8 regno, u8 red, u8 green, u8 blue,
 	i810_writeb(CLUT_INDEX_WRITE, mmio, regno);
 	i810_writeb(CLUT_DATA, mmio, red);
 	i810_writeb(CLUT_DATA, mmio, green);
-	i810_writeb(CLUT_DATA, mmio, blue); 	
+	i810_writeb(CLUT_DATA, mmio, blue);
 }
 
 static void i810_read_dac(u8 regno, u8 *red, u8 *green, u8 *blue,
@@ -456,13 +457,13 @@ static void i810_read_dac(u8 regno, u8 *red, u8 *green, u8 *blue,
 }
 
 /************************************************************
- *                   VGA State Restore                      * 
+ *                   VGA State Restore                      *
  ************************************************************/
 static void i810_restore_pll(struct i810fb_par *par)
 {
 	u32 tmp1, tmp2;
 	u8 __iomem *mmio = par->mmio_start_virtual;
-	
+
 	tmp1 = par->hw_state.dclk_2d;
 	tmp2 = i810_readl(DCLK_2D, mmio);
 	tmp1 &= ~MN_MASK;
@@ -494,7 +495,7 @@ static void i810_restore_vgax(struct i810fb_par *par)
 {
 	u8 i, j;
 	u8 __iomem *mmio = par->mmio_start_virtual;
-	
+
 	for (i = 0; i < 4; i++) {
 		i810_writeb(CR_INDEX_CGA, mmio, CR30+i);
 		i810_writeb(CR_DATA_CGA, mmio, *(&(par->hw_state.cr30) + i));
@@ -528,7 +529,7 @@ static void i810_restore_vga(struct i810fb_par *par)
 {
 	u8 i;
 	u8 __iomem *mmio = par->mmio_start_virtual;
-	
+
 	for (i = 0; i < 10; i++) {
 		i810_writeb(CR_INDEX_CGA, mmio, CR00 + i);
 		i810_writeb(CR_DATA_CGA, mmio, *((&par->hw_state.cr00) + i));
@@ -559,10 +560,10 @@ static void i810_restore_2d(struct i810fb_par *par)
 	u8 __iomem *mmio = par->mmio_start_virtual;
 
 	tmp_word = i810_readw(BLTCNTL, mmio);
-	tmp_word &= ~(3 << 4); 
+	tmp_word &= ~(3 << 4);
 	tmp_word |= par->hw_state.bltcntl;
 	i810_writew(BLTCNTL, mmio, tmp_word);
-       
+
 	i810_dram_off(mmio, OFF);
 	i810_writel(PIXCONF, mmio, par->hw_state.pixconf);
 	i810_dram_off(mmio, ON);
@@ -577,7 +578,7 @@ static void i810_restore_2d(struct i810fb_par *par)
 	tmp_long |= par->hw_state.fw_blc;
 	i810_writel(FW_BLC, mmio, tmp_long);
 
-	i810_writel(HWS_PGA, mmio, par->hw_state.hws_pga); 
+	i810_writel(HWS_PGA, mmio, par->hw_state.hws_pga);
 	i810_writew(IER, mmio, par->hw_state.ier);
 	i810_writew(IMR, mmio, par->hw_state.imr);
 	i810_writel(DPLYSTAS, mmio, par->hw_state.dplystas);
@@ -621,7 +622,7 @@ static void i810_save_vgax(struct i810fb_par *par)
 	i810_writeb(CR_INDEX_CGA, mmio, CR41);
 	par->hw_state.cr41 = i810_readb(CR_DATA_CGA, mmio);
 	i810_writeb(CR_INDEX_CGA, mmio, CR70);
-	par->hw_state.cr70 = i810_readb(CR_DATA_CGA, mmio);	
+	par->hw_state.cr70 = i810_readb(CR_DATA_CGA, mmio);
 	par->hw_state.msr = i810_readb(MSR_READ, mmio);
 	i810_writeb(CR_INDEX_CGA, mmio, CR80);
 	par->hw_state.cr80 = i810_readb(CR_DATA_CGA, mmio);
@@ -654,8 +655,8 @@ static void i810_save_2d(struct i810fb_par *par)
 	par->hw_state.pixconf = i810_readl(PIXCONF, mmio);
 	par->hw_state.fw_blc = i810_readl(FW_BLC, mmio);
 	par->hw_state.bltcntl = i810_readw(BLTCNTL, mmio);
-	par->hw_state.hwstam = i810_readw(HWSTAM, mmio); 
-	par->hw_state.hws_pga = i810_readl(HWS_PGA, mmio); 
+	par->hw_state.hwstam = i810_readw(HWSTAM, mmio);
+	par->hw_state.hws_pga = i810_readl(HWS_PGA, mmio);
 	par->hw_state.ier = i810_readw(IER, mmio);
 	par->hw_state.imr = i810_readw(IMR, mmio);
 	par->hw_state.dplystas = i810_readl(DPLYSTAS, mmio);
@@ -669,7 +670,7 @@ static void i810_save_vga_state(struct i810fb_par *par)
 }
 
 /************************************************************
- *                    Helpers                               * 
+ *                    Helpers                               *
  ************************************************************/
 /**
  * get_line_length - calculates buffer pitch in bytes
@@ -678,12 +679,12 @@ static void i810_save_vga_state(struct i810fb_par *par)
  * @bpp: bits per pixel
  *
  * DESCRIPTION:
- * Calculates buffer pitch in bytes.  
+ * Calculates buffer pitch in bytes.
  */
 static u32 get_line_length(struct i810fb_par *par, int xres_virtual, int bpp)
 {
    	u32 length;
-	
+
 	length = xres_virtual*bpp;
 	length = (length+31)&-32;
 	length >>= 3;
@@ -716,17 +717,17 @@ static void i810_calc_dclk(u32 freq, u32 *m, u32 *n, u32 *p)
 	n_target_max = 30;
 
 	/*
-	 * find P such that target freq is 16x reference freq (Hz). 
+	 * find P such that target freq is 16x reference freq (Hz).
 	 */
 	p_divisor = 1;
 	p_target = 0;
-	while(!((1000000 * p_divisor)/(16 * 24 * target_freq)) && 
+	while(!((1000000 * p_divisor)/(16 * 24 * target_freq)) &&
 	      p_divisor <= 32) {
 		p_divisor <<= 1;
 		p_target++;
 	}
 
-	n_reg = m_reg = n_target = 3;	
+	n_reg = m_reg = n_target = 3;
 	while (diff_min && mod_min && (n_target < n_target_max)) {
 		f_out = (p_divisor * n_reg * 1000000)/(4 * 24 * m_reg);
 		mod = (p_divisor * n_reg * 1000000) % (4 * 24 * m_reg);
@@ -744,14 +745,14 @@ static void i810_calc_dclk(u32 freq, u32 *m, u32 *n, u32 *p)
 			diff_min = diff;
 			n_best = n_target;
 			m_best = m_target;
-		}		 
+		}
 
 		if (!diff && mod_min > mod) {
 			mod_min = mod;
 			n_best = n_target;
 			m_best = m_target;
 		}
-	} 
+	}
 	if (m) *m = (m_best - 2) & 0x3FF;
 	if (n) *n = (n_best - 2) & 0x3FF;
 	if (p) *p = (p_target << 4);
@@ -772,7 +773,7 @@ static void i810_calc_dclk(u32 freq, u32 *m, u32 *n, u32 *p)
 static void i810_enable_cursor(u8 __iomem *mmio, int mode)
 {
 	u32 temp;
-	
+
 	temp = i810_readl(PIXCONF, mmio);
 	temp = (mode == ON) ? temp | CURSOR_ENABLE_MASK :
 		temp & ~CURSOR_ENABLE_MASK;
@@ -786,10 +787,10 @@ static void i810_reset_cursor_image(struct i810fb_par *par)
 	int i, j;
 
 	for (i = 64; i--; ) {
-		for (j = 0; j < 8; j++) {             
-			i810_writeb(j, addr, 0xff);   
-			i810_writeb(j+8, addr, 0x00); 
-		}	
+		for (j = 0; j < 8; j++) {
+			i810_writeb(j, addr, 0xff);
+			i810_writeb(j+8, addr, 0x00);
+		}
 		addr +=16;
 	}
 }
@@ -800,9 +801,9 @@ static void i810_load_cursor_image(int width, int height, u8 *data,
 	u8 __iomem *addr = par->cursor_heap.virtual;
 	int i, j, w = width/8;
 	int mod = width % 8, t_mask, d_mask;
-	
+
 	t_mask = 0xff >> mod;
-	d_mask = ~(0xff >> mod); 
+	d_mask = ~(0xff >> mod);
 	for (i = height; i--; ) {
 		for (j = 0; j < w; j++) {
 			i810_writeb(j+0, addr, 0x00);
@@ -854,7 +855,7 @@ static void i810_init_cursor(struct i810fb_par *par)
 	i810_enable_cursor(mmio, OFF);
 	i810_writel(CURBASE, mmio, par->cursor_heap.physical);
 	i810_writew(CURCNTR, mmio, COORD_ACTIVE | CURSOR_MODE_64_XOR);
-}	
+}
 
 /*********************************************************************
  *                    Framebuffer hook helpers                       *
@@ -873,7 +874,7 @@ static void i810_round_off(struct fb_var_screeninfo *var)
 	u32 xres, yres, vxres, vyres;
 
 	/*
-	 *  Presently supports only these configurations 
+	 *  Presently supports only these configurations
 	 */
 
 	xres = var->xres;
@@ -883,20 +884,20 @@ static void i810_round_off(struct fb_var_screeninfo *var)
 
 	var->bits_per_pixel += 7;
 	var->bits_per_pixel &= ~7;
-	
+
 	if (var->bits_per_pixel < 8)
 		var->bits_per_pixel = 8;
-	if (var->bits_per_pixel > 32) 
+	if (var->bits_per_pixel > 32)
 		var->bits_per_pixel = 32;
 
 	round_off_xres(&xres);
 	if (xres < 40)
 		xres = 40;
-	if (xres > 2048) 
+	if (xres > 2048)
 		xres = 2048;
 	xres = (xres + 7) & ~7;
 
-	if (vxres < xres) 
+	if (vxres < xres)
 		vxres = xres;
 
 	round_off_yres(&xres, &yres);
@@ -905,7 +906,7 @@ static void i810_round_off(struct fb_var_screeninfo *var)
 	if (yres >= 2048)
 		yres = 2048;
 
-	if (vyres < yres) 
+	if (vyres < yres)
 		vyres = yres;
 
 	if (var->bits_per_pixel == 32)
@@ -917,30 +918,30 @@ static void i810_round_off(struct fb_var_screeninfo *var)
 	var->hsync_len = (var->hsync_len + 4) & ~7;
 
 	if (var->vmode & FB_VMODE_INTERLACED) {
-		if (!((yres + var->upper_margin + var->vsync_len + 
+		if (!((yres + var->upper_margin + var->vsync_len +
 		       var->lower_margin) & 1))
 			var->upper_margin++;
 	}
-	
+
 	var->xres = xres;
 	var->yres = yres;
 	var->xres_virtual = vxres;
 	var->yres_virtual = vyres;
-}	
+}
 
 /**
  * set_color_bitfields - sets rgba fields
  * @var: pointer to fb_var_screeninfo
  *
  * DESCRIPTION:
- * The length, offset and ordering  for each color field 
- * (red, green, blue)  will be set as specified 
+ * The length, offset and ordering  for each color field
+ * (red, green, blue)  will be set as specified
  * by the hardware
- */  
+ */
 static void set_color_bitfields(struct fb_var_screeninfo *var)
 {
 	switch (var->bits_per_pixel) {
-	case 8:       
+	case 8:
 		var->red.offset = 0;
 		var->red.length = 8;
 		var->green.offset = 0;
@@ -984,11 +985,11 @@ static void set_color_bitfields(struct fb_var_screeninfo *var)
  * @info: pointer to fb_info
  *
  * DESCRIPTION:
- * This will check if the framebuffer size is sufficient 
- * for the current mode and if the user's monitor has the 
+ * This will check if the framebuffer size is sufficient
+ * for the current mode and if the user's monitor has the
  * required specifications to display the current mode.
  */
-static int i810_check_params(struct fb_var_screeninfo *var, 
+static int i810_check_params(struct fb_var_screeninfo *var,
 			     struct fb_info *info)
 {
 	struct i810fb_par *par = info->par;
@@ -1007,14 +1008,14 @@ static int i810_check_params(struct fb_var_screeninfo *var,
 			vyres = info->var.yres;
 			vxres = par->fb.size/vyres;
 			vxres /= var->bits_per_pixel >> 3;
-			line_length = get_line_length(par, vxres, 
+			line_length = get_line_length(par, vxres,
 						      var->bits_per_pixel);
 			vidmem = line_length * info->var.yres;
 			if (vxres < var->xres) {
 				printk("i810fb: required video memory, "
 				       "%d bytes, for %dx%d-%d (virtual) "
-				       "is out of range\n", 
-				       vidmem, vxres, vyres, 
+				       "is out of range\n",
+				       vidmem, vxres, vyres,
 				       var->bits_per_pixel);
 				return -ENOMEM;
 			}
@@ -1074,7 +1075,7 @@ static int i810_check_params(struct fb_var_screeninfo *var,
 	}
 
 	return retval;
-}	
+}
 
 /**
  * encode_fix - fill up fb_fix_screeninfo structure
@@ -1131,9 +1132,9 @@ static int encode_fix(struct fb_fix_screeninfo *fix, struct fb_info *info)
  *
  * DESCRIPTION:
  * Based on the contents of @var, @par will be dynamically filled up.
- * @par contains all information necessary to modify the hardware. 
+ * @par contains all information necessary to modify the hardware.
 */
-static void decode_var(const struct fb_var_screeninfo *var, 
+static void decode_var(const struct fb_var_screeninfo *var,
 		       struct i810fb_par *par)
 {
 	u32 xres, yres, vxres, vyres;
@@ -1175,13 +1176,13 @@ static void decode_var(const struct fb_var_screeninfo *var,
 	if (var->nonstd && var->bits_per_pixel != 8)
 		par->pixconf |= 1 << 27;
 
-	i810_calc_dclk(var->pixclock, &par->regs.M, 
+	i810_calc_dclk(var->pixclock, &par->regs.M,
 		       &par->regs.N, &par->regs.P);
 	i810fb_encode_registers(var, par, xres, yres);
 
 	par->watermark = i810_get_watermark(var, par);
 	par->pitch = get_line_length(par, vxres, var->bits_per_pixel);
-}	
+}
 
 /**
  * i810fb_getcolreg - gets red, green and blue values of the hardware DAC
@@ -1196,7 +1197,7 @@ static void decode_var(const struct fb_var_screeninfo *var,
  * Gets the red, green and blue values of the hardware DAC as pointed by @regno
  * and writes them to @red, @green and @blue respectively
  */
-static int i810fb_getcolreg(u8 regno, u8 *red, u8 *green, u8 *blue, 
+static int i810fb_getcolreg(u8 regno, u8 *red, u8 *green, u8 *blue,
 			    u8 *transp, struct fb_info *info)
 {
 	struct i810fb_par *par = info->par;
@@ -1212,18 +1213,18 @@ static int i810fb_getcolreg(u8 regno, u8 *red, u8 *green, u8 *blue,
 	temp = i810_readb(PIXCONF1, mmio);
 	i810_writeb(PIXCONF1, mmio, temp & ~EXTENDED_PALETTE);
 
-	if (info->fix.visual == FB_VISUAL_DIRECTCOLOR && 
-	    info->var.green.length == 5) 
+	if (info->fix.visual == FB_VISUAL_DIRECTCOLOR &&
+	    info->var.green.length == 5)
 		i810_read_dac(regno * 8, red, green, blue, mmio);
 
-	else if (info->fix.visual == FB_VISUAL_DIRECTCOLOR && 
+	else if (info->fix.visual == FB_VISUAL_DIRECTCOLOR &&
 		 info->var.green.length == 6) {
 		u8 tmp;
 
 		i810_read_dac(regno * 8, red, &tmp, blue, mmio);
 		i810_read_dac(regno * 4, &tmp, green, &tmp, mmio);
 	}
-	else 
+	else
 		i810_read_dac(regno, red, green, blue, mmio);
 
     	*transp = 0;
@@ -1232,7 +1233,7 @@ static int i810fb_getcolreg(u8 regno, u8 *red, u8 *green, u8 *blue,
     	return 0;
 }
 
-/****************************************************************** 
+/******************************************************************
  *           Framebuffer device-specific hooks                    *
  ******************************************************************/
 
@@ -1252,7 +1253,7 @@ static int i810fb_open(struct fb_info *info, int user)
 
 	par->use_count++;
 	mutex_unlock(&par->open_lock);
-	
+
 	return 0;
 }
 
@@ -1273,13 +1274,13 @@ static int i810fb_release(struct fb_info *info, int user)
 
 	par->use_count--;
 	mutex_unlock(&par->open_lock);
-	
+
 	return 0;
 }
 
 
-static int i810fb_setcolreg(unsigned regno, unsigned red, unsigned green, 
-			    unsigned blue, unsigned transp, 
+static int i810fb_setcolreg(unsigned regno, unsigned red, unsigned green,
+			    unsigned blue, unsigned transp,
 			    struct fb_info *info)
 {
 	struct i810fb_par *par = info->par;
@@ -1302,24 +1303,24 @@ static int i810fb_setcolreg(unsigned regno, unsigned red, unsigned green,
 	temp = i810_readb(PIXCONF1, mmio);
 	i810_writeb(PIXCONF1, mmio, temp & ~EXTENDED_PALETTE);
 
-	if (info->fix.visual == FB_VISUAL_DIRECTCOLOR && 
+	if (info->fix.visual == FB_VISUAL_DIRECTCOLOR &&
 	    info->var.green.length == 5) {
-		for (i = 0; i < 8; i++) 
-			i810_write_dac((u8) (regno * 8) + i, (u8) red, 
+		for (i = 0; i < 8; i++)
+			i810_write_dac((u8) (regno * 8) + i, (u8) red,
 				       (u8) green, (u8) blue, mmio);
-	} else if (info->fix.visual == FB_VISUAL_DIRECTCOLOR && 
+	} else if (info->fix.visual == FB_VISUAL_DIRECTCOLOR &&
 		 info->var.green.length == 6) {
 		u8 r, g, b;
 
 		if (regno < 32) {
-			for (i = 0; i < 8; i++) 
+			for (i = 0; i < 8; i++)
 				i810_write_dac((u8) (regno * 8) + i,
-					       (u8) red, (u8) green, 
+					       (u8) red, (u8) green,
 					       (u8) blue, mmio);
 		}
 		i810_read_dac((u8) (regno*4), &r, &g, &b, mmio);
-		for (i = 0; i < 4; i++) 
-			i810_write_dac((u8) (regno*4) + i, r, (u8) green, 
+		for (i = 0; i < 4; i++)
+			i810_write_dac((u8) (regno*4) + i, r, (u8) green,
 				       b, mmio);
 	} else if (info->fix.visual == FB_VISUAL_PSEUDOCOLOR) {
 		i810_write_dac((u8) regno, (u8) red, (u8) green,
@@ -1330,20 +1331,20 @@ static int i810fb_setcolreg(unsigned regno, unsigned red, unsigned green,
 
 	if (regno < 16) {
 		switch (info->var.bits_per_pixel) {
-		case 16:	
+		case 16:
 			if (info->fix.visual == FB_VISUAL_DIRECTCOLOR) {
-				if (info->var.green.length == 5) 
-					((u32 *)info->pseudo_palette)[regno] = 
+				if (info->var.green.length == 5)
+					((u32 *)info->pseudo_palette)[regno] =
 						(regno << 10) | (regno << 5) |
 						regno;
 				else
-					((u32 *)info->pseudo_palette)[regno] = 
+					((u32 *)info->pseudo_palette)[regno] =
 						(regno << 11) | (regno << 5) |
 						regno;
 			} else {
 				if (info->var.green.length == 5) {
 					/* RGB 555 */
-					((u32 *)info->pseudo_palette)[regno] = 
+					((u32 *)info->pseudo_palette)[regno] =
 						((red & 0xf800) >> 1) |
 						((green & 0xf800) >> 6) |
 						((blue & 0xf800) >> 11);
@@ -1358,12 +1359,12 @@ static int i810fb_setcolreg(unsigned regno, unsigned red, unsigned green,
 			break;
 		case 24:	/* RGB 888 */
 		case 32:	/* RGBA 8888 */
-			if (info->fix.visual == FB_VISUAL_DIRECTCOLOR) 
-				((u32 *)info->pseudo_palette)[regno] = 
+			if (info->fix.visual == FB_VISUAL_DIRECTCOLOR)
+				((u32 *)info->pseudo_palette)[regno] =
 					(regno << 16) | (regno << 8) |
 					regno;
-			else 
-				((u32 *)info->pseudo_palette)[regno] = 
+			else
+				((u32 *)info->pseudo_palette)[regno] =
 					((red & 0xff00) << 8) |
 					(green & 0xff00) |
 					((blue & 0xff00) >> 8);
@@ -1373,13 +1374,13 @@ static int i810fb_setcolreg(unsigned regno, unsigned red, unsigned green,
 	return 0;
 }
 
-static int i810fb_pan_display(struct fb_var_screeninfo *var, 
+static int i810fb_pan_display(struct fb_var_screeninfo *var,
 			      struct fb_info *info)
 {
 	struct i810fb_par *par = info->par;
 	u32 total;
-	
-	total = var->xoffset * par->depth + 
+
+	total = var->xoffset * par->depth +
 		var->yoffset * info->fix.line_length;
 	i810fb_load_front(total, info);
 
@@ -1391,7 +1392,7 @@ static int i810fb_blank (int blank_mode, struct fb_info *info)
 	struct i810fb_par *par = info->par;
 	u8 __iomem *mmio = par->mmio_start_virtual;
 	int mode = 0, pwr, scr_off = 0;
-	
+
 	pwr = i810_readl(PWR_CLKC, mmio);
 
 	switch (blank_mode) {
@@ -1421,7 +1422,7 @@ static int i810fb_blank (int blank_mode, struct fb_info *info)
 		scr_off = OFF;
 		break;
 	default:
-		return -EINVAL; 
+		return -EINVAL;
 	}
 
 	i810_screen_off(mmio, scr_off);
@@ -1452,7 +1453,7 @@ static int i810fb_set_par(struct fb_info *info)
 	return 0;
 }
 
-static int i810fb_check_var(struct fb_var_screeninfo *var, 
+static int i810fb_check_var(struct fb_var_screeninfo *var,
 			    struct fb_info *info)
 {
 	int err;
@@ -1550,7 +1551,7 @@ static const struct fb_ops i810fb_ops = {
 	.fb_set_par =        i810fb_set_par,
 	.fb_setcolreg =      i810fb_setcolreg,
 	.fb_blank =          i810fb_blank,
-	.fb_pan_display =    i810fb_pan_display, 
+	.fb_pan_display =    i810fb_pan_display,
 	.fb_fillrect =       i810fb_fillrect,
 	.fb_copyarea =       i810fb_copyarea,
 	.fb_imageblit =      i810fb_imageblit,
@@ -1593,7 +1594,7 @@ static int i810fb_suspend(struct pci_dev *dev, pm_message_t mesg)
 	return 0;
 }
 
-static int i810fb_resume(struct pci_dev *dev) 
+static int i810fb_resume(struct pci_dev *dev)
 {
 	struct fb_info *info = pci_get_drvdata(dev);
 	struct i810fb_par *par = info->par;
@@ -1628,14 +1629,14 @@ fail:
 /***********************************************************************
  *                  AGP resource allocation                            *
  ***********************************************************************/
-  
+
 static void i810_fix_pointers(struct i810fb_par *par)
 {
       	par->fb.physical = par->aperture.physical+(par->fb.offset << 12);
 	par->fb.virtual = par->aperture.virtual+(par->fb.offset << 12);
-	par->iring.physical = par->aperture.physical + 
+	par->iring.physical = par->aperture.physical +
 		(par->iring.offset << 12);
-	par->iring.virtual = par->aperture.virtual + 
+	par->iring.virtual = par->aperture.virtual +
 		(par->iring.offset << 12);
 	par->cursor_heap.virtual = par->aperture.virtual+
 		(par->cursor_heap.offset << 12);
@@ -1666,7 +1667,7 @@ static int i810_alloc_agp_mem(struct fb_info *info)
 	struct i810fb_par *par = info->par;
 	int size;
 	struct agp_bridge_data *bridge;
-	
+
 	i810_fix_offsets(par);
 	size = par->fb.size + par->iring.size;
 
@@ -1674,7 +1675,7 @@ static int i810_alloc_agp_mem(struct fb_info *info)
 		printk("i810fb_alloc_fbmem: cannot acquire agpgart\n");
 		return -ENODEV;
 	}
-	if (!(par->i810_gtt.i810_fb_memory = 
+	if (!(par->i810_gtt.i810_fb_memory =
 	      agp_allocate_memory(bridge, size >> 12, AGP_NORMAL_MEMORY))) {
 		printk("i810fb_alloc_fbmem: can't allocate framebuffer "
 		       "memory\n");
@@ -1686,9 +1687,9 @@ static int i810_alloc_agp_mem(struct fb_info *info)
 		printk("i810fb_alloc_fbmem: can't bind framebuffer memory\n");
 		agp_backend_release(bridge);
 		return -EBUSY;
-	}	
-	
-	if (!(par->i810_gtt.i810_cursor_memory = 
+	}
+
+	if (!(par->i810_gtt.i810_cursor_memory =
 	      agp_allocate_memory(bridge, par->cursor_heap.size >> 12,
 				  AGP_PHYSICAL_MEMORY))) {
 		printk("i810fb_alloc_cursormem:  can't allocate "
@@ -1701,7 +1702,7 @@ static int i810_alloc_agp_mem(struct fb_info *info)
 		printk("i810fb_alloc_cursormem: cannot bind cursor memory\n");
 		agp_backend_release(bridge);
 		return -EBUSY;
-	}	
+	}
 
 	par->cursor_heap.physical = par->i810_gtt.i810_cursor_memory->physical;
 
@@ -1712,8 +1713,8 @@ static int i810_alloc_agp_mem(struct fb_info *info)
 	return 0;
 }
 
-/*************************************************************** 
- *                    Initialization                           * 
+/***************************************************************
+ *                    Initialization                           *
  ***************************************************************/
 
 /**
@@ -1728,7 +1729,7 @@ static void i810_init_monspecs(struct fb_info *info)
 {
 	if (!hsync1)
 		hsync1 = HFMIN;
-	if (!hsync2) 
+	if (!hsync2)
 		hsync2 = HFMAX;
 	if (!info->monspecs.hfmax)
 		info->monspecs.hfmax = hsync2;
@@ -1739,7 +1740,7 @@ static void i810_init_monspecs(struct fb_info *info)
 
 	if (!vsync1)
 		vsync1 = VFMIN;
-	if (!vsync2) 
+	if (!vsync2)
 		vsync2 = VFMAX;
 	if (IS_DVT && vsync1 < 60)
 		vsync1 = 60;
@@ -1747,7 +1748,7 @@ static void i810_init_monspecs(struct fb_info *info)
 		info->monspecs.vfmax = vsync2;
 	if (!info->monspecs.vfmin)
 		info->monspecs.vfmin = vsync1;
-	if (vsync2 < vsync1) 
+	if (vsync2 < vsync1)
 		info->monspecs.vfmin = vsync2;
 }
 
@@ -1760,27 +1761,27 @@ static void i810_init_defaults(struct i810fb_par *par, struct fb_info *info)
 {
 	mutex_init(&par->open_lock);
 
-	if (voffset) 
+	if (voffset)
 		v_offset_default = voffset;
 	else if (par->aperture.size > 32 * 1024 * 1024)
 		v_offset_default = 16;
 	else
 		v_offset_default = 8;
 
-	if (!vram) 
+	if (!vram)
 		vram = 1;
 
-	if (accel) 
+	if (accel)
 		par->dev_flags |= HAS_ACCELERATION;
 
-	if (sync) 
+	if (sync)
 		par->dev_flags |= ALWAYS_SYNC;
 
 	par->ddc_num = (ddc3 ? 3 : 2);
 
 	if (bpp < 8)
 		bpp = 8;
-	
+
 	par->i810fb_ops = i810fb_ops;
 
 	if (xres)
@@ -1793,7 +1794,7 @@ static void i810_init_defaults(struct i810fb_par *par, struct fb_info *info)
 	else
 		info->var.yres = 480;
 
-	if (!vyres) 
+	if (!vyres)
 		vyres = (vram << 20)/(info->var.xres*bpp >> 3);
 
 	info->var.yres_virtual = vyres;
@@ -1802,12 +1803,12 @@ static void i810_init_defaults(struct i810fb_par *par, struct fb_info *info)
 	if (dcolor)
 		info->var.nonstd = 1;
 
-	if (par->dev_flags & HAS_ACCELERATION) 
+	if (par->dev_flags & HAS_ACCELERATION)
 		info->var.accel_flags = 1;
 
 	i810_init_monspecs(info);
 }
-	
+
 /**
  * i810_init_device - initialize device
  * @par: pointer to i810fb_par structure
@@ -1840,9 +1841,9 @@ static int i810_allocate_pci_resource(struct i810fb_par *par,
 {
 	int err;
 
-	if ((err = pci_enable_device(par->dev))) { 
+	if ((err = pci_enable_device(par->dev))) {
 		printk("i810fb_init: cannot enable device\n");
-		return err;		
+		return err;
 	}
 	par->res_flags |= PCI_DEVICE_ENABLED;
 
@@ -1860,8 +1861,8 @@ static int i810_allocate_pci_resource(struct i810fb_par *par,
 		return -ENOMEM;
 	}
 
-	if (!request_mem_region(par->aperture.physical, 
-				par->aperture.size, 
+	if (!request_mem_region(par->aperture.physical,
+				par->aperture.size,
 				i810_pci_list[entry->driver_data])) {
 		printk("i810fb_init: cannot request framebuffer region\n");
 		return -ENODEV;
@@ -1874,16 +1875,16 @@ static int i810_allocate_pci_resource(struct i810fb_par *par,
 		printk("i810fb_init: cannot remap framebuffer region\n");
 		return -ENODEV;
 	}
-  
-	if (!request_mem_region(par->mmio_start_phys, 
-				MMIO_SIZE, 
+
+	if (!request_mem_region(par->mmio_start_phys,
+				MMIO_SIZE,
 				i810_pci_list[entry->driver_data])) {
 		printk("i810fb_init: cannot request mmio region\n");
 		return -ENODEV;
 	}
 	par->res_flags |= MMIO_REQ;
 
-	par->mmio_start_virtual = ioremap(par->mmio_start_phys, 
+	par->mmio_start_virtual = ioremap(par->mmio_start_phys,
 						  MMIO_SIZE);
 	if (!par->mmio_start_virtual) {
 		printk("i810fb_init: cannot remap mmio region\n");
@@ -1963,7 +1964,7 @@ static int i810fb_setup(char *options)
 
 	if (!options || !*options)
 		return 0;
-	
+
 	while ((this_opt = strsep(&options, ",")) != NULL) {
 		if (!strncmp(this_opt, "mtrr", 4))
 			mtrr = true;
@@ -1987,13 +1988,13 @@ static int i810fb_setup(char *options)
 			bpp = simple_strtoul(this_opt+4, NULL, 0);
 		else if (!strncmp(this_opt, "hsync1:", 7)) {
 			hsync1 = simple_strtoul(this_opt+7, &suffix, 0);
-			if (strncmp(suffix, "H", 1)) 
+			if (strncmp(suffix, "H", 1))
 				hsync1 *= 1000;
 		} else if (!strncmp(this_opt, "hsync2:", 7)) {
 			hsync2 = simple_strtoul(this_opt+7, &suffix, 0);
-			if (strncmp(suffix, "H", 1)) 
+			if (strncmp(suffix, "H", 1))
 				hsync2 *= 1000;
-		} else if (!strncmp(this_opt, "vsync1:", 7)) 
+		} else if (!strncmp(this_opt, "vsync1:", 7))
 			vsync1 = simple_strtoul(this_opt+7, NULL, 0);
 		else if (!strncmp(this_opt, "vsync2:", 7))
 			vsync2 = simple_strtoul(this_opt+7, NULL, 0);
@@ -2016,6 +2017,10 @@ static int i810fb_init_pci(struct pci_dev *dev,
 	struct fb_videomode mode;
 	int err = -1, vfreq, hfreq, pixclock;
 
+	err = aperture_remove_conflicting_pci_devices(dev, "i810fb");
+	if (err)
+		return err;
+
 	info = framebuffer_alloc(sizeof(struct i810fb_par), &dev->dev);
 	if (!info)
 		return -ENOMEM;
@@ -2044,7 +2049,7 @@ static int i810fb_init_pci(struct pci_dev *dev,
 		return err;
 	}
 
-	i810_init_device(par);        
+	i810_init_device(par);
 
 	info->screen_base = par->fb.virtual;
 	info->fbops = &par->i810fb_ops;
@@ -2064,21 +2069,21 @@ static int i810fb_init_pci(struct pci_dev *dev,
 	err = register_framebuffer(info);
 
 	if (err < 0) {
-    		i810fb_release_resource(info, par); 
+    		i810fb_release_resource(info, par);
 		printk("i810fb_init: cannot register framebuffer device\n");
-    		return err;  
-    	}   
+    		return err;
+    	}
 
 	pci_set_drvdata(dev, info);
 	pixclock = 1000000000/(info->var.pixclock);
 	pixclock *= 1000;
-	hfreq = pixclock/(info->var.xres + info->var.left_margin + 
+	hfreq = pixclock/(info->var.xres + info->var.left_margin +
 			  info->var.hsync_len + info->var.right_margin);
 	vfreq = hfreq/(info->var.yres + info->var.upper_margin +
 		       info->var.vsync_len + info->var.lower_margin);
 
       	printk("I810FB: fb%d         : %s v%d.%d.%d%s\n"
-      	       "I810FB: Video RAM   : %dK\n" 
+      	       "I810FB: Video RAM   : %dK\n"
 	       "I810FB: Monitor     : H: %d-%d KHz V: %d-%d Hz\n"
 	       "I810FB: Mode        : %dx%d-%dbpp@%dHz\n",
 	       info->node,
@@ -2086,7 +2091,7 @@ static int i810fb_init_pci(struct pci_dev *dev,
 	       VERSION_MAJOR, VERSION_MINOR, VERSION_TEENIE, BRANCH_VERSION,
 	       (int) par->fb.size>>10, info->monspecs.hfmin/1000,
 	       info->monspecs.hfmax/1000, info->monspecs.vfmin,
-	       info->monspecs.vfmax, info->var.xres, 
+	       info->monspecs.vfmax, info->var.xres,
 	       info->var.yres, info->var.bits_per_pixel, vfreq);
 	return 0;
 }
@@ -2095,7 +2100,7 @@ static int i810fb_init_pci(struct pci_dev *dev,
  *                     De-initialization                        *
  ***************************************************************/
 
-static void i810fb_release_resource(struct fb_info *info, 
+static void i810fb_release_resource(struct fb_info *info,
 				    struct i810fb_par *par)
 {
 	struct gtt_data *gtt = &par->i810_gtt;
@@ -2128,10 +2133,10 @@ static void i810fb_remove_pci(struct pci_dev *dev)
 	struct fb_info *info = pci_get_drvdata(dev);
 	struct i810fb_par *par = info->par;
 
-	unregister_framebuffer(info);  
+	unregister_framebuffer(info);
 	i810fb_release_resource(info, par);
 	printk("cleanup_module:  unloaded i810 framebuffer device\n");
-}                                                	
+}
 
 #ifndef MODULE
 static int i810fb_init(void)
@@ -2144,7 +2149,7 @@ static int i810fb_init(void)
 
 	return pci_register_driver(&i810fb_driver);
 }
-#endif 
+#endif
 
 /*********************************************************************
  *                          Modularization                           *
@@ -2161,7 +2166,7 @@ static int i810fb_init(void)
 }
 
 module_param(vram, int, 0);
-MODULE_PARM_DESC(vram, "System RAM to allocate to framebuffer in MiB" 
+MODULE_PARM_DESC(vram, "System RAM to allocate to framebuffer in MiB"
 		 " (default=4)");
 module_param(voffset, int, 0);
 MODULE_PARM_DESC(voffset, "at what offset to place start of framebuffer "
@@ -2186,7 +2191,7 @@ module_param(vsync1, int, 0);
 MODULE_PARM_DESC(vsync1, "Minimum vertical frequency of monitor in Hz"
 		 " (default = 50)");
 module_param(vsync2, int, 0);
-MODULE_PARM_DESC(vsync2, "Maximum vertical frequency of monitor in Hz" 
+MODULE_PARM_DESC(vsync2, "Maximum vertical frequency of monitor in Hz"
 		 " (default = 60)");
 module_param(accel, bool, 0);
 MODULE_PARM_DESC(accel, "Use Acceleration (BLIT) engine (default = 0)");
@@ -2208,7 +2213,7 @@ MODULE_PARM_DESC(mode_option, "Specify initial video mode");
 MODULE_AUTHOR("Tony A. Daplas");
 MODULE_DESCRIPTION("Framebuffer device for the Intel 810/815 and"
 		   " compatible cards");
-MODULE_LICENSE("GPL"); 
+MODULE_LICENSE("GPL");
 
 static void __exit i810fb_exit(void)
 {
diff --git a/drivers/video/fbdev/imsttfb.c b/drivers/video/fbdev/imsttfb.c
index 16f272a50811..d7edb9c5d3a3 100644
--- a/drivers/video/fbdev/imsttfb.c
+++ b/drivers/video/fbdev/imsttfb.c
@@ -16,6 +16,7 @@
  *  more details.
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -86,7 +87,7 @@ enum {
 	SSTATUS	= 36, /* 0x90 */
 	PRC	= 37, /* 0x94 */
 
-#if 0	
+#if 0
 	/* PCI Registers */
 	DVID	= 0x00000000L,
 	SC	= 0x00000004L,
@@ -103,8 +104,8 @@ enum {
 	PDATA	= 0x04,
 	PPMASK	= 0x08,
 	PADDRR	= 0x0c,
-	PIDXLO	= 0x10,	
-	PIDXHI	= 0x14,	
+	PIDXLO	= 0x10,
+	PIDXHI	= 0x14,
 	PIDXDATA= 0x18,
 	PIDXCTL	= 0x1c
 };
@@ -131,7 +132,7 @@ enum {
 	SYSCLKC		= 0x18,	/* () System Clock C */
 	/*
 	 * Dot clock rate is 20MHz * (m + 1) / ((n + 1) * (p ? 2 * p : 1)
-	 * c is charge pump bias which depends on the VCO frequency  
+	 * c is charge pump bias which depends on the VCO frequency
 	 */
 	PIXM0		= 0x20,	/* () Pixel M 0 */
 	PIXN0		= 0x21,	/* () Pixel N 0 */
@@ -320,7 +321,7 @@ struct imstt_par {
 	__u32 ramdac;
 	__u32 palette[16];
 };
- 
+
 enum {
 	IBM = 0,
 	TVP = 1
@@ -373,7 +374,7 @@ static struct imstt_regvals tvp_reg_init_17 = {
 
 static struct imstt_regvals tvp_reg_init_18 = {
 	1152,
-  	0x0009, 0x0011, 0x059, 0x5b, 0x0003, 0x0031, 0x0397, 0x039a, 0x0000, 
+  	0x0009, 0x0011, 0x059, 0x5b, 0x0003, 0x0031, 0x0397, 0x039a, 0x0000,
 	0xfd, 0x3a, 0xf1,
 	{ 0x39, 0x38, 0x38 }, { 0xf3, 0xf3, 0xf2 }
 };
@@ -856,10 +857,10 @@ imsttfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 }
 
 static int
-imsttfb_set_par(struct fb_info *info) 
+imsttfb_set_par(struct fb_info *info)
 {
 	struct imstt_par *par = info->par;
-		
+
 	if (!compute_imstt_regvals(par, info->var.xres, info->var.yres))
 		return -EINVAL;
 
@@ -930,7 +931,7 @@ imsttfb_pan_display(struct fb_var_screeninfo *var, struct fb_info *info)
 	return 0;
 }
 
-static int 
+static int
 imsttfb_blank(int blank, struct fb_info *info)
 {
 	struct imstt_par *par = info->par;
@@ -986,7 +987,7 @@ imsttfb_blank(int blank, struct fb_info *info)
 
 static void
 imsttfb_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
-{ 
+{
 	struct imstt_par *par = info->par;
 	__u32 Bpp, line_pitch, bgc, dx, dy, width, height;
 
@@ -1192,7 +1193,7 @@ imstt_set_cursor(struct imstt_par *par, struct fb_image *d, int on)
 	}
 }
 
-static int 
+static int
 imsttfb_cursor(struct fb_info *info, struct fb_cursor *cursor)
 {
 	struct imstt_par *par = info->par;
@@ -1200,7 +1201,7 @@ imsttfb_cursor(struct fb_info *info, struct fb_cursor *cursor)
 
 	if (cursor->dest == NULL && cursor->rop == ROP_XOR)
 		return 1;
-	
+
 	imstt_set_cursor(info, cursor, 0);
 
 	if (flags & FB_CUR_SETPOS) {
@@ -1469,8 +1470,13 @@ static int imsttfb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	struct imstt_par *par;
 	struct fb_info *info;
 	struct device_node *dp;
-	int ret = -ENOMEM;
-	
+	int ret;
+
+	ret = aperture_remove_conflicting_pci_devices(pdev, "imsttfb");
+	if (ret)
+		return ret;
+	ret = -ENOMEM;
+
 	dp = pci_device_to_OF_node(pdev);
 	if(dp)
 		printk(KERN_INFO "%s: OF name %pOFn\n",__func__, dp);
@@ -1619,7 +1625,7 @@ static int __init imsttfb_init(void)
 #endif
 	return pci_register_driver(&imsttfb_pci_driver);
 }
- 
+
 static void __exit imsttfb_exit(void)
 {
 	pci_unregister_driver(&imsttfb_pci_driver);
diff --git a/drivers/video/fbdev/intelfb/intelfbdrv.c b/drivers/video/fbdev/intelfb/intelfbdrv.c
index 5647fca8c49a..d4a2891a9a7a 100644
--- a/drivers/video/fbdev/intelfb/intelfbdrv.c
+++ b/drivers/video/fbdev/intelfb/intelfbdrv.c
@@ -107,6 +107,7 @@
  *              Add support for 945GME. (Phil Endecott <spam_from_intelfb@chezphil.org>)
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -483,6 +484,10 @@ static int intelfb_pci_register(struct pci_dev *pdev,
 
 	DBG_MSG("intelfb_pci_register\n");
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "intelfb");
+	if (err)
+		return err;
+
 	num_registered++;
 	if (num_registered != 1) {
 		ERR_MSG("Attempted to register %d devices "
diff --git a/drivers/video/fbdev/kyro/fbdev.c b/drivers/video/fbdev/kyro/fbdev.c
index d57772f96ad2..b4b93054c520 100644
--- a/drivers/video/fbdev/kyro/fbdev.c
+++ b/drivers/video/fbdev/kyro/fbdev.c
@@ -9,6 +9,7 @@
  * for more details.
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/types.h>
 #include <linux/kernel.h>
@@ -676,6 +677,10 @@ static int kyrofb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	unsigned long size;
 	int err;
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "kyrofb");
+	if (err)
+		return err;
+
 	if ((err = pci_enable_device(pdev))) {
 		printk(KERN_WARNING "kyrofb: Can't enable pdev: %d\n", err);
 		return err;
diff --git a/drivers/video/fbdev/matrox/matroxfb_base.c b/drivers/video/fbdev/matrox/matroxfb_base.c
index 68bba2688f4c..775d34115e2d 100644
--- a/drivers/video/fbdev/matrox/matroxfb_base.c
+++ b/drivers/video/fbdev/matrox/matroxfb_base.c
@@ -100,6 +100,7 @@
  *
  */
 
+#include <linux/aperture.h>
 #include <linux/version.h>
 
 #include "matroxfb_base.h"
@@ -2044,6 +2045,10 @@ static int matroxfb_probe(struct pci_dev* pdev, const struct pci_device_id* dumm
 	u_int32_t cmd;
 	DBG(__func__)
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "matroxfb");
+	if (err)
+		return err;
+
 	svid = pdev->subsystem_vendor;
 	sid = pdev->subsystem_device;
 	for (b = dev_list; b->vendor; b++) {
diff --git a/drivers/video/fbdev/matrox/matroxfb_maven.c b/drivers/video/fbdev/matrox/matroxfb_maven.c
index 9a98c4a6ba33..f2e02958673d 100644
--- a/drivers/video/fbdev/matrox/matroxfb_maven.c
+++ b/drivers/video/fbdev/matrox/matroxfb_maven.c
@@ -1276,11 +1276,10 @@ ERROR0:;
 	return err;
 }
 
-static int maven_remove(struct i2c_client *client)
+static void maven_remove(struct i2c_client *client)
 {
 	maven_shutdown_client(client);
 	kfree(i2c_get_clientdata(client));
-	return 0;
 }
 
 static const struct i2c_device_id maven_id[] = {
diff --git a/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c b/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c
index a7508f5be343..96800c9c9cd9 100644
--- a/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c
+++ b/drivers/video/fbdev/mb862xx/mb862xxfbdrv.c
@@ -10,6 +10,7 @@
 
 #undef DEBUG
 
+#include <linux/aperture.h>
 #include <linux/fb.h>
 #include <linux/delay.h>
 #include <linux/uaccess.h>
@@ -999,6 +1000,10 @@ static int mb862xx_pci_probe(struct pci_dev *pdev,
 	struct device *dev = &pdev->dev;
 	int ret;
 
+	ret = aperture_remove_conflicting_pci_devices(pdev, "mb862xxfb");
+	if (ret)
+		return ret;
+
 	ret = pci_enable_device(pdev);
 	if (ret < 0) {
 		dev_err(dev, "Cannot enable PCI device\n");
diff --git a/drivers/video/fbdev/neofb.c b/drivers/video/fbdev/neofb.c
index 28d32cbf496b..93a2d2d1abe8 100644
--- a/drivers/video/fbdev/neofb.c
+++ b/drivers/video/fbdev/neofb.c
@@ -23,9 +23,9 @@
  *
  * 0.3.3
  *  - Porting over to new fbdev api. (jsimmons)
- *  
+ *
  * 0.3.2
- *  - got rid of all floating point (dok) 
+ *  - got rid of all floating point (dok)
  *
  * 0.3.1
  *  - added module license (dok)
@@ -54,6 +54,7 @@
  *
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -1154,14 +1155,14 @@ static int neofb_set_par(struct fb_info *info)
 
 	switch (info->fix.accel) {
 		case FB_ACCEL_NEOMAGIC_NM2200:
-		case FB_ACCEL_NEOMAGIC_NM2230: 
-		case FB_ACCEL_NEOMAGIC_NM2360: 
-		case FB_ACCEL_NEOMAGIC_NM2380: 
+		case FB_ACCEL_NEOMAGIC_NM2230:
+		case FB_ACCEL_NEOMAGIC_NM2360:
+		case FB_ACCEL_NEOMAGIC_NM2380:
 			neo2200_accel_init(info, &info->var);
 			break;
 		default:
 			break;
-	}	
+	}
 	return 0;
 }
 
@@ -1493,15 +1494,15 @@ neofb_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
 	switch (info->fix.accel) {
 		case FB_ACCEL_NEOMAGIC_NM2200:
-		case FB_ACCEL_NEOMAGIC_NM2230: 
-		case FB_ACCEL_NEOMAGIC_NM2360: 
+		case FB_ACCEL_NEOMAGIC_NM2230:
+		case FB_ACCEL_NEOMAGIC_NM2360:
 		case FB_ACCEL_NEOMAGIC_NM2380:
 			neo2200_fillrect(info, rect);
 			break;
 		default:
 			cfb_fillrect(info, rect);
 			break;
-	}	
+	}
 }
 
 static void
@@ -1509,15 +1510,15 @@ neofb_copyarea(struct fb_info *info, const struct fb_copyarea *area)
 {
 	switch (info->fix.accel) {
 		case FB_ACCEL_NEOMAGIC_NM2200:
-		case FB_ACCEL_NEOMAGIC_NM2230: 
-		case FB_ACCEL_NEOMAGIC_NM2360: 
-		case FB_ACCEL_NEOMAGIC_NM2380: 
+		case FB_ACCEL_NEOMAGIC_NM2230:
+		case FB_ACCEL_NEOMAGIC_NM2360:
+		case FB_ACCEL_NEOMAGIC_NM2380:
 			neo2200_copyarea(info, area);
 			break;
 		default:
 			cfb_copyarea(info, area);
 			break;
-	}	
+	}
 }
 
 static void
@@ -1536,20 +1537,20 @@ neofb_imageblit(struct fb_info *info, const struct fb_image *image)
 	}
 }
 
-static int 
+static int
 neofb_sync(struct fb_info *info)
 {
 	switch (info->fix.accel) {
 		case FB_ACCEL_NEOMAGIC_NM2200:
-		case FB_ACCEL_NEOMAGIC_NM2230: 
-		case FB_ACCEL_NEOMAGIC_NM2360: 
-		case FB_ACCEL_NEOMAGIC_NM2380: 
+		case FB_ACCEL_NEOMAGIC_NM2230:
+		case FB_ACCEL_NEOMAGIC_NM2360:
+		case FB_ACCEL_NEOMAGIC_NM2380:
 			neo2200_sync(info);
 			break;
 		default:
 			break;
 	}
-	return 0;		
+	return 0;
 }
 
 /*
@@ -2029,6 +2030,10 @@ static int neofb_probe(struct pci_dev *dev, const struct pci_device_id *id)
 
 	DBG("neofb_probe");
 
+	err = aperture_remove_conflicting_pci_devices(dev, "neofb");
+	if (err)
+		return err;
+
 	err = pci_enable_device(dev);
 	if (err)
 		return err;
diff --git a/drivers/video/fbdev/nvidia/nvidia.c b/drivers/video/fbdev/nvidia/nvidia.c
index a372a183c1f0..329e2e8133c6 100644
--- a/drivers/video/fbdev/nvidia/nvidia.c
+++ b/drivers/video/fbdev/nvidia/nvidia.c
@@ -9,6 +9,7 @@
  *
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -1276,11 +1277,15 @@ static int nvidiafb_probe(struct pci_dev *pd, const struct pci_device_id *ent)
 	struct nvidia_par *par;
 	struct fb_info *info;
 	unsigned short cmd;
-
+	int ret;
 
 	NVTRACE_ENTER();
 	assert(pd != NULL);
 
+	ret = aperture_remove_conflicting_pci_devices(pd, "nvidiafb");
+	if (ret)
+		return ret;
+
 	info = framebuffer_alloc(sizeof(struct nvidia_par), &pd->dev);
 
 	if (!info)
diff --git a/drivers/video/fbdev/pm2fb.c b/drivers/video/fbdev/pm2fb.c
index 8fd79deb1e2a..7da715d31a93 100644
--- a/drivers/video/fbdev/pm2fb.c
+++ b/drivers/video/fbdev/pm2fb.c
@@ -27,6 +27,7 @@
  *
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/kernel.h>
@@ -1521,6 +1522,10 @@ static int pm2fb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	int err;
 	int retval = -ENXIO;
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "pm2fb");
+	if (err)
+		return err;
+
 	err = pci_enable_device(pdev);
 	if (err) {
 		printk(KERN_WARNING "pm2fb: Can't enable pdev: %d\n", err);
diff --git a/drivers/video/fbdev/pm3fb.c b/drivers/video/fbdev/pm3fb.c
index a8faf46adeb1..ba69846d444f 100644
--- a/drivers/video/fbdev/pm3fb.c
+++ b/drivers/video/fbdev/pm3fb.c
@@ -22,6 +22,7 @@
  *
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -1315,6 +1316,10 @@ static int pm3fb_probe(struct pci_dev *dev, const struct pci_device_id *ent)
 	int err;
 	int retval = -ENXIO;
 
+	err = aperture_remove_conflicting_pci_devices(dev, "pm3fb");
+	if (err)
+		return err;
+
 	err = pci_enable_device(dev);
 	if (err) {
 		printk(KERN_WARNING "pm3fb: Can't enable PCI dev: %d\n", err);
diff --git a/drivers/video/fbdev/pvr2fb.c b/drivers/video/fbdev/pvr2fb.c
index f4add36cb5f4..b73ad14efa20 100644
--- a/drivers/video/fbdev/pvr2fb.c
+++ b/drivers/video/fbdev/pvr2fb.c
@@ -45,6 +45,7 @@
 
 #undef DEBUG
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -942,6 +943,10 @@ static int pvr2fb_pci_probe(struct pci_dev *pdev,
 {
 	int ret;
 
+	ret = aperture_remove_conflicting_pci_devices(pdev, "pvrfb");
+	if (ret)
+		return ret;
+
 	ret = pci_enable_device(pdev);
 	if (ret) {
 		printk(KERN_ERR "pvr2fb: PCI enable failed\n");
diff --git a/drivers/video/fbdev/riva/fbdev.c b/drivers/video/fbdev/riva/fbdev.c
index 84d5e23ad7d3..0ea74e28f915 100644
--- a/drivers/video/fbdev/riva/fbdev.c
+++ b/drivers/video/fbdev/riva/fbdev.c
@@ -29,6 +29,7 @@
  *	doublescan modes are broken
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -474,7 +475,7 @@ static inline void reverse_order(u32 *l)
  * DESCRIPTiON:
  * Loads cursor image based on a monochrome source and mask bitmap.  The
  * image bits determines the color of the pixel, 0 for background, 1 for
- * foreground.  Only the affected region (as determined by @w and @h 
+ * foreground.  Only the affected region (as determined by @w and @h
  * parameters) will be updated.
  *
  * CALLED FROM:
@@ -494,7 +495,7 @@ static void rivafb_load_cursor_image(struct riva_par *par, u8 *data8,
 	for (i = 0; i < h; i++) {
 		b = *data++;
 		reverse_order(&b);
-		
+
 		for (j = 0; j < w/2; j++) {
 			tmp = 0;
 #if defined (__BIG_ENDIAN)
@@ -562,7 +563,7 @@ static void riva_rclut(RIVA_HW_INST *chip,
 		       unsigned char regnum, unsigned char *red,
 		       unsigned char *green, unsigned char *blue)
 {
-	
+
 	VGA_WR08(chip->PDIO, 0x3c7, regnum);
 	*red = VGA_RD08(chip->PDIO, 0x3c9);
 	*green = VGA_RD08(chip->PDIO, 0x3c9);
@@ -673,7 +674,7 @@ static int riva_load_video_mode(struct fb_info *info)
 	int rc;
 	struct riva_par *par = info->par;
 	struct riva_regs newmode;
-	
+
 	NVTRACE_ENTER();
 	/* time to calculate */
 	rivafb_blank(FB_BLANK_NORMAL, info);
@@ -717,7 +718,7 @@ static int riva_load_video_mode(struct fb_info *info)
 		hBlankEnd = hTotal + 4;
 	}
 
-	newmode.crtc[0x0] = Set8Bits (hTotal); 
+	newmode.crtc[0x0] = Set8Bits (hTotal);
 	newmode.crtc[0x1] = Set8Bits (hDisplay);
 	newmode.crtc[0x2] = Set8Bits (hBlankStart);
 	newmode.crtc[0x3] = SetBitField (hBlankEnd, 4: 0, 4:0) | SetBit (7);
@@ -748,20 +749,20 @@ static int riva_load_video_mode(struct fb_info *info)
 		| SetBitField(vStart,10:10,2:2)
 		| SetBitField(vDisplay,10:10,1:1)
 		| SetBitField(vTotal,10:10,0:0);
-	newmode.ext.horiz  = SetBitField(hTotal,8:8,0:0) 
+	newmode.ext.horiz  = SetBitField(hTotal,8:8,0:0)
 		| SetBitField(hDisplay,8:8,1:1)
 		| SetBitField(hBlankStart,8:8,2:2)
 		| SetBitField(hStart,8:8,3:3);
 	newmode.ext.extra  = SetBitField(vTotal,11:11,0:0)
 		| SetBitField(vDisplay,11:11,2:2)
 		| SetBitField(vStart,11:11,4:4)
-		| SetBitField(vBlankStart,11:11,6:6); 
+		| SetBitField(vBlankStart,11:11,6:6);
 
 	if ((info->var.vmode & FB_VMODE_MASK) == FB_VMODE_INTERLACED) {
 		int tmp = (hTotal >> 1) & ~1;
 		newmode.ext.interlace = Set8Bits(tmp);
 		newmode.ext.horiz |= SetBitField(tmp, 8:8,4:4);
-	} else 
+	} else
 		newmode.ext.interlace = 0xff; /* interlace off */
 
 	if (par->riva.Architecture >= NV_ARCH_10)
@@ -774,7 +775,7 @@ static int riva_load_video_mode(struct fb_info *info)
 	if (info->var.sync & FB_SYNC_VERT_HIGH_ACT)
 		newmode.misc_output &= ~0x80;
 	else
-		newmode.misc_output |= 0x80;	
+		newmode.misc_output |= 0x80;
 
 	rc = CalcStateExt(&par->riva, &newmode.ext, par->pdev, bpp, width,
 			  hDisplaySize, height, dotClock);
@@ -841,7 +842,7 @@ static void riva_update_var(struct fb_var_screeninfo *var,
 }
 
 /**
- * rivafb_do_maximize - 
+ * rivafb_do_maximize -
  * @info: pointer to fb_info object containing info for current riva board
  * @var: standard kernel fb changeable data
  * @nom: nom
@@ -852,7 +853,7 @@ static void riva_update_var(struct fb_var_screeninfo *var,
  *
  * RETURNS:
  * -EINVAL on failure, 0 on success
- * 
+ *
  *
  * CALLED FROM:
  * rivafb_check_var()
@@ -916,14 +917,14 @@ static int rivafb_do_maximize(struct fb_info *info,
 			return -EINVAL;
 		}
 	}
-	
+
 	if (var->xres_virtual * nom / den >= 8192) {
 		printk(KERN_WARNING PFX
 		       "virtual X resolution (%d) is too high, lowering to %d\n",
 		       var->xres_virtual, 8192 * den / nom - 16);
 		var->xres_virtual = 8192 * den / nom - 16;
 	}
-	
+
 	if (var->xres_virtual < var->xres) {
 		printk(KERN_ERR PFX
 		       "virtual X resolution (%d) is smaller than real\n", var->xres_virtual);
@@ -1010,7 +1011,7 @@ static int riva_get_cmap_len(const struct fb_var_screeninfo *var)
 		break;
 	case 6:
 		rc = 64;	/* 64 entries (2^6), 16 bpp, RGB565 */
-		break;		
+		break;
 	default:
 		/* should not occur */
 		break;
@@ -1042,7 +1043,7 @@ static int rivafb_open(struct fb_info *info, int user)
 		/* vgaHWunlock() + riva unlock (0x7F) */
 		CRTCout(par, 0x11, 0xFF);
 		par->riva.LockUnlock(&par->riva, 0);
-	
+
 		riva_save_state(par, &par->initial_state);
 	}
 	par->ref_count++;
@@ -1082,7 +1083,7 @@ static int rivafb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 	struct riva_par *par = info->par;
 	int nom, den;		/* translating from pixels->bytes */
 	int mode_valid = 0;
-	
+
 	NVTRACE_ENTER();
 	if (!var->pixclock)
 		return -EINVAL;
@@ -1176,7 +1177,7 @@ static int rivafb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 	if (var->yoffset > var->yres_virtual - var->yres)
 		var->yoffset = var->yres_virtual - var->yres - 1;
 
-	var->red.msb_right = 
+	var->red.msb_right =
 	    var->green.msb_right =
 	    var->blue.msb_right =
 	    var->transp.offset = var->transp.length = var->transp.msb_right = 0;
@@ -1198,7 +1199,7 @@ static int rivafb_set_par(struct fb_info *info)
 		goto out;
 	if(!(info->flags & FBINFO_HWACCEL_DISABLED))
 		riva_setup_accel(info);
-	
+
 	par->cursor_reset = 1;
 	info->fix.line_length = (info->var.xres_virtual * (info->var.bits_per_pixel >> 3));
 	info->fix.visual = (info->var.bits_per_pixel == 8) ?
@@ -1486,7 +1487,7 @@ static inline void convert_bgcolor_16(u32 *col)
  * CALLED FROM:
  * framebuffer hook
  */
-static void rivafb_imageblit(struct fb_info *info, 
+static void rivafb_imageblit(struct fb_info *info,
 			     const struct fb_image *image)
 {
 	struct riva_par *par = info->par;
@@ -1515,7 +1516,7 @@ static void rivafb_imageblit(struct fb_info *info,
 			bgx = par->palette[image->bg_color];
 		}
 		if (info->var.green.length == 6)
-			convert_bgcolor_16(&bgx);	
+			convert_bgcolor_16(&bgx);
 		break;
 	}
 
@@ -1612,7 +1613,7 @@ static int rivafb_cursor(struct fb_info *info, struct fb_cursor *cursor)
 		u8 *dat = (u8 *) cursor->image.data;
 		u8 *msk = (u8 *) cursor->mask;
 		u8 *src;
-		
+
 		src = kmalloc_array(s_pitch, cursor->image.height, GFP_ATOMIC);
 
 		if (src) {
@@ -1683,7 +1684,7 @@ static const struct fb_ops riva_fb_ops = {
 	.fb_fillrect 	= rivafb_fillrect,
 	.fb_copyarea 	= rivafb_copyarea,
 	.fb_imageblit 	= rivafb_imageblit,
-	.fb_cursor	= rivafb_cursor,	
+	.fb_cursor	= rivafb_cursor,
 	.fb_sync 	= rivafb_sync,
 };
 
@@ -1713,7 +1714,7 @@ static int riva_set_fbinfo(struct fb_info *info)
 	info->pseudo_palette = par->pseudo_palette;
 
 	cmap_len = riva_get_cmap_len(&info->var);
-	fb_alloc_cmap(&info->cmap, cmap_len, 0);	
+	fb_alloc_cmap(&info->cmap, cmap_len, 0);
 
 	info->pixmap.size = 8 * 1024;
 	info->pixmap.buf_align = 4;
@@ -1898,6 +1899,10 @@ static int rivafb_probe(struct pci_dev *pd, const struct pci_device_id *ent)
 	NVTRACE_ENTER();
 	assert(pd != NULL);
 
+	ret = aperture_remove_conflicting_pci_devices(pd, "rivafb");
+	if (ret)
+		return ret;
+
 	info = framebuffer_alloc(sizeof(struct riva_par), &pd->dev);
 	if (!info) {
 		ret = -ENOMEM;
@@ -1929,7 +1934,7 @@ static int rivafb_probe(struct pci_dev *pd, const struct pci_device_id *ent)
 
 	default_par->Chipset = (pd->vendor << 16) | pd->device;
 	printk(KERN_INFO PFX "nVidia device/chipset %X\n",default_par->Chipset);
-	
+
 	if(default_par->riva.Architecture == 0) {
 		printk(KERN_ERR PFX "unknown NV_ARCH\n");
 		ret=-ENODEV;
@@ -1947,7 +1952,7 @@ static int rivafb_probe(struct pci_dev *pd, const struct pci_device_id *ent)
 	if (flatpanel == 1)
 		printk(KERN_INFO PFX "flatpanel support enabled\n");
 	default_par->forceCRTC = forceCRTC;
-	
+
 	rivafb_fix.mmio_len = pci_resource_len(pd, 0);
 	rivafb_fix.smem_len = pci_resource_len(pd, 1);
 
@@ -1959,7 +1964,7 @@ static int rivafb_probe(struct pci_dev *pd, const struct pci_device_id *ent)
 		cmd |= (PCI_COMMAND_IO | PCI_COMMAND_MEMORY);
 		pci_write_config_word(pd, PCI_COMMAND, cmd);
 	}
-	
+
 	rivafb_fix.mmio_start = pci_resource_start(pd, 0);
 	rivafb_fix.smem_start = pci_resource_start(pd, 1);
 
@@ -2058,7 +2063,7 @@ err_iounmap_screen_base:
 #endif
 	iounmap(info->screen_base);
 err_iounmap_pramin:
-	if (default_par->riva.Architecture == NV_ARCH_03) 
+	if (default_par->riva.Architecture == NV_ARCH_03)
 		iounmap(default_par->riva.PRAMIN);
 err_iounmap_ctrl_base:
 	iounmap(default_par->ctrl_base);
@@ -2077,7 +2082,7 @@ static void rivafb_remove(struct pci_dev *pd)
 {
 	struct fb_info *info = pci_get_drvdata(pd);
 	struct riva_par *par = info->par;
-	
+
 	NVTRACE_ENTER();
 
 #ifdef CONFIG_FB_RIVA_I2C
@@ -2117,11 +2122,11 @@ static int rivafb_setup(char *options)
 	while ((this_opt = strsep(&options, ",")) != NULL) {
 		if (!strncmp(this_opt, "forceCRTC", 9)) {
 			char *p;
-			
+
 			p = this_opt + 9;
-			if (!*p || !*(++p)) continue; 
+			if (!*p || !*(++p)) continue;
 			forceCRTC = *p - '0';
-			if (forceCRTC < 0 || forceCRTC > 1) 
+			if (forceCRTC < 0 || forceCRTC > 1)
 				forceCRTC = -1;
 		} else if (!strncmp(this_opt, "flatpanel", 9)) {
 			flatpanel = 1;
diff --git a/drivers/video/fbdev/s3fb.c b/drivers/video/fbdev/s3fb.c
index 67b63a753cb2..7713274bd04c 100644
--- a/drivers/video/fbdev/s3fb.c
+++ b/drivers/video/fbdev/s3fb.c
@@ -11,6 +11,7 @@
  * which is based on the code of neofb.
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -1131,6 +1132,10 @@ static int s3_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 		return -ENODEV;
 	}
 
+	rc = aperture_remove_conflicting_pci_devices(dev, "s3fb");
+	if (rc)
+		return rc;
+
 	/* Allocate and fill driver data structure */
 	info = framebuffer_alloc(sizeof(struct s3fb_info), &(dev->dev));
 	if (!info)
diff --git a/drivers/video/fbdev/savage/savagefb_driver.c b/drivers/video/fbdev/savage/savagefb_driver.c
index 8114c921ceb8..b7818b652698 100644
--- a/drivers/video/fbdev/savage/savagefb_driver.c
+++ b/drivers/video/fbdev/savage/savagefb_driver.c
@@ -41,6 +41,7 @@
  *
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -2176,6 +2177,10 @@ static int savagefb_probe(struct pci_dev *dev, const struct pci_device_id *id)
 
 	DBG("savagefb_probe");
 
+	err = aperture_remove_conflicting_pci_devices(dev, "savagefb");
+	if (err)
+		return err;
+
 	info = framebuffer_alloc(sizeof(struct savagefb_par), &dev->dev);
 	if (!info)
 		return -ENOMEM;
diff --git a/drivers/video/fbdev/sis/sis_main.c b/drivers/video/fbdev/sis/sis_main.c
index c9e77429dfa3..1c197c3f9538 100644
--- a/drivers/video/fbdev/sis/sis_main.c
+++ b/drivers/video/fbdev/sis/sis_main.c
@@ -19,6 +19,7 @@
  * which is (c) 1998 Gerd Knorr <kraxel@goldbach.in-berlin.de>
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/kernel.h>
@@ -5861,6 +5862,10 @@ static int sisfb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if(sisfb_off)
 		return -ENXIO;
 
+	ret = aperture_remove_conflicting_pci_devices(pdev, "sisfb");
+	if (ret)
+		return ret;
+
 	sis_fb_info = framebuffer_alloc(sizeof(*ivideo), &pdev->dev);
 	if(!sis_fb_info)
 		return -ENOMEM;
diff --git a/drivers/video/fbdev/skeletonfb.c b/drivers/video/fbdev/skeletonfb.c
index 8ab9a3fbd281..a10f1057293b 100644
--- a/drivers/video/fbdev/skeletonfb.c
+++ b/drivers/video/fbdev/skeletonfb.c
@@ -10,38 +10,39 @@
  *  The primary goal is to remove the console code from fbdev and place it
  *  into fbcon.c. This reduces the code and makes writing a new fbdev driver
  *  easy since the author doesn't need to worry about console internals. It
- *  also allows the ability to run fbdev without a console/tty system on top 
- *  of it. 
+ *  also allows the ability to run fbdev without a console/tty system on top
+ *  of it.
  *
  *  First the roles of struct fb_info and struct display have changed. Struct
  *  display will go away. The way the new framebuffer console code will
- *  work is that it will act to translate data about the tty/console in 
+ *  work is that it will act to translate data about the tty/console in
  *  struct vc_data to data in a device independent way in struct fb_info. Then
- *  various functions in struct fb_ops will be called to store the device 
- *  dependent state in the par field in struct fb_info and to change the 
+ *  various functions in struct fb_ops will be called to store the device
+ *  dependent state in the par field in struct fb_info and to change the
  *  hardware to that state. This allows a very clean separation of the fbdev
  *  layer from the console layer. It also allows one to use fbdev on its own
- *  which is a bounus for embedded devices. The reason this approach works is  
+ *  which is a bounus for embedded devices. The reason this approach works is
  *  for each framebuffer device when used as a tty/console device is allocated
- *  a set of virtual terminals to it. Only one virtual terminal can be active 
- *  per framebuffer device. We already have all the data we need in struct 
+ *  a set of virtual terminals to it. Only one virtual terminal can be active
+ *  per framebuffer device. We already have all the data we need in struct
  *  vc_data so why store a bunch of colormaps and other fbdev specific data
- *  per virtual terminal. 
+ *  per virtual terminal.
  *
  *  As you can see doing this makes the con parameter pretty much useless
- *  for struct fb_ops functions, as it should be. Also having struct  
- *  fb_var_screeninfo and other data in fb_info pretty much eliminates the 
+ *  for struct fb_ops functions, as it should be. Also having struct
+ *  fb_var_screeninfo and other data in fb_info pretty much eliminates the
  *  need for get_fix and get_var. Once all drivers use the fix, var, and cmap
  *  fbcon can be written around these fields. This will also eliminate the
  *  need to regenerate struct fb_var_screeninfo, struct fb_fix_screeninfo
  *  struct fb_cmap every time get_var, get_fix, get_cmap functions are called
- *  as many drivers do now. 
+ *  as many drivers do now.
  *
  *  This file is subject to the terms and conditions of the GNU General Public
  *  License. See the file COPYING in the main directory of this archive for
  *  more details.
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -66,68 +67,68 @@
 static char *mode_option;
 
 /*
- *  If your driver supports multiple boards, you should make the  
- *  below data types arrays, or allocate them dynamically (using kmalloc()). 
- */ 
+ *  If your driver supports multiple boards, you should make the
+ *  below data types arrays, or allocate them dynamically (using kmalloc()).
+ */
 
-/* 
+/*
  * This structure defines the hardware state of the graphics card. Normally
  * you place this in a header file in linux/include/video. This file usually
  * also includes register information. That allows other driver subsystems
- * and userland applications the ability to use the same header file to 
- * avoid duplicate work and easy porting of software. 
+ * and userland applications the ability to use the same header file to
+ * avoid duplicate work and easy porting of software.
  */
 struct xxx_par;
 
 /*
  * Here we define the default structs fb_fix_screeninfo and fb_var_screeninfo
  * if we don't use modedb. If we do use modedb see xxxfb_init how to use it
- * to get a fb_var_screeninfo. Otherwise define a default var as well. 
+ * to get a fb_var_screeninfo. Otherwise define a default var as well.
  */
 static const struct fb_fix_screeninfo xxxfb_fix = {
-	.id =		"FB's name", 
+	.id =		"FB's name",
 	.type =		FB_TYPE_PACKED_PIXELS,
 	.visual =	FB_VISUAL_PSEUDOCOLOR,
 	.xpanstep =	1,
 	.ypanstep =	1,
-	.ywrapstep =	1, 
+	.ywrapstep =	1,
 	.accel =	FB_ACCEL_NONE,
 };
 
     /*
-     * 	Modern graphical hardware not only supports pipelines but some 
+     * 	Modern graphical hardware not only supports pipelines but some
      *  also support multiple monitors where each display can have
-     *  its own unique data. In this case each display could be  
-     *  represented by a separate framebuffer device thus a separate 
+     *  its own unique data. In this case each display could be
+     *  represented by a separate framebuffer device thus a separate
      *  struct fb_info. Now the struct xxx_par represents the graphics
-     *  hardware state thus only one exist per card. In this case the 
-     *  struct xxx_par for each graphics card would be shared between 
-     *  every struct fb_info that represents a framebuffer on that card. 
-     *  This allows when one display changes it video resolution (info->var) 
+     *  hardware state thus only one exist per card. In this case the
+     *  struct xxx_par for each graphics card would be shared between
+     *  every struct fb_info that represents a framebuffer on that card.
+     *  This allows when one display changes it video resolution (info->var)
      *  the other displays know instantly. Each display can always be
      *  aware of the entire hardware state that affects it because they share
      *  the same xxx_par struct. The other side of the coin is multiple
      *  graphics cards that pass data around until it is finally displayed
      *  on one monitor. Such examples are the voodoo 1 cards and high end
      *  NUMA graphics servers. For this case we have a bunch of pars, each
-     *  one that represents a graphics state, that belong to one struct 
+     *  one that represents a graphics state, that belong to one struct
      *  fb_info. Their you would want to have *par point to a array of device
-     *  states and have each struct fb_ops function deal with all those 
+     *  states and have each struct fb_ops function deal with all those
      *  states. I hope this covers every possible hardware design. If not
-     *  feel free to send your ideas at jsimmons@users.sf.net 
+     *  feel free to send your ideas at jsimmons@users.sf.net
      */
 
     /*
-     *  If your driver supports multiple boards or it supports multiple 
-     *  framebuffers, you should make these arrays, or allocate them 
+     *  If your driver supports multiple boards or it supports multiple
+     *  framebuffers, you should make these arrays, or allocate them
      *  dynamically using framebuffer_alloc() and free them with
      *  framebuffer_release().
-     */ 
+     */
 static struct fb_info info;
 
-    /* 
+    /*
      * Each one represents the state of the hardware. Most hardware have
-     * just one hardware state. These here represent the default state(s). 
+     * just one hardware state. These here represent the default state(s).
      */
 static struct xxx_par __initdata current_par;
 
@@ -136,12 +137,12 @@ static struct xxx_par __initdata current_par;
  *		     first accessed.
  *	@info: frame buffer structure that represents a single frame buffer
  *	@user: tell us if the userland (value=1) or the console is accessing
- *	       the framebuffer. 
+ *	       the framebuffer.
  *
  *	This function is the first function called in the framebuffer api.
- *	Usually you don't need to provide this function. The case where it 
+ *	Usually you don't need to provide this function. The case where it
  *	is used is to change from a text mode hardware state to a graphics
- * 	mode state. 
+ * 	mode state.
  *
  *	Returns negative errno on error, or zero on success.
  */
@@ -151,13 +152,13 @@ static int xxxfb_open(struct fb_info *info, int user)
 }
 
 /**
- *	xxxfb_release - Optional function. Called when the framebuffer 
- *			device is closed. 
+ *	xxxfb_release - Optional function. Called when the framebuffer
+ *			device is closed.
  *	@info: frame buffer structure that represents a single frame buffer
  *	@user: tell us if the userland (value=1) or the console is accessing
- *	       the framebuffer. 
- *	
- *	Thus function is called when we close /dev/fb or the framebuffer 
+ *	       the framebuffer.
+ *
+ *	Thus function is called when we close /dev/fb or the framebuffer
  *	console system is released. Usually you don't need this function.
  *	The case where it is usually used is to go from a graphics state
  *	to a text mode state.
@@ -170,17 +171,17 @@ static int xxxfb_release(struct fb_info *info, int user)
 }
 
 /**
- *      xxxfb_check_var - Optional function. Validates a var passed in. 
+ *      xxxfb_check_var - Optional function. Validates a var passed in.
  *      @var: frame buffer variable screen structure
- *      @info: frame buffer structure that represents a single frame buffer 
+ *      @info: frame buffer structure that represents a single frame buffer
  *
  *	Checks to see if the hardware supports the state requested by
- *	var passed in. This function does not alter the hardware state!!! 
- *	This means the data stored in struct fb_info and struct xxx_par do 
- *      not change. This includes the var inside of struct fb_info. 
+ *	var passed in. This function does not alter the hardware state!!!
+ *	This means the data stored in struct fb_info and struct xxx_par do
+ *      not change. This includes the var inside of struct fb_info.
  *	Do NOT change these. This function can be called on its own if we
- *	intent to only test a mode and not actually set it. The stuff in 
- *	modedb.c is a example of this. If the var passed in is slightly 
+ *	intent to only test a mode and not actually set it. The stuff in
+ *	modedb.c is a example of this. If the var passed in is slightly
  *	off by what the hardware can support then we alter the var PASSED in
  *	to what we can do.
  *
@@ -208,7 +209,7 @@ static int xxxfb_release(struct fb_info *info, int user)
 static int xxxfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
 {
     /* ... */
-    return 0;	   	
+    return 0;
 }
 
 /**
@@ -217,9 +218,9 @@ static int xxxfb_check_var(struct fb_var_screeninfo *var, struct fb_info *info)
  *
  *	Using the fb_var_screeninfo in fb_info we set the resolution of the
  *	this particular framebuffer. This function alters the par AND the
- *	fb_fix_screeninfo stored in fb_info. It doesn't not alter var in 
+ *	fb_fix_screeninfo stored in fb_info. It doesn't not alter var in
  *	fb_info since we are using that data. This means we depend on the
- *	data in var inside fb_info to be supported by the hardware. 
+ *	data in var inside fb_info to be supported by the hardware.
  *
  *      This function is also used to recover/restore the hardware to a
  *      known working state.
@@ -254,20 +255,20 @@ static int xxxfb_set_par(struct fb_info *info)
 {
     struct xxx_par *par = info->par;
     /* ... */
-    return 0;	
+    return 0;
 }
 
 /**
  *  	xxxfb_setcolreg - Optional function. Sets a color register.
- *      @regno: Which register in the CLUT we are programming 
- *      @red: The red value which can be up to 16 bits wide 
- *	@green: The green value which can be up to 16 bits wide 
+ *      @regno: Which register in the CLUT we are programming
+ *      @red: The red value which can be up to 16 bits wide
+ *	@green: The green value which can be up to 16 bits wide
  *	@blue:  The blue value which can be up to 16 bits wide.
  *	@transp: If supported, the alpha value which can be up to 16 bits wide.
  *      @info: frame buffer info structure
- * 
+ *
  *  	Set a single color register. The values supplied have a 16 bit
- *  	magnitude which needs to be scaled in this function for the hardware. 
+ *  	magnitude which needs to be scaled in this function for the hardware.
  *	Things to take into consideration are how many color registers, if
  *	any, are supported with the current color visual. With truecolor mode
  *	no color palettes are supported. Here a pseudo palette is created
@@ -275,8 +276,8 @@ static int xxxfb_set_par(struct fb_info *info)
  *	pseudocolor mode we have a limited color palette. To deal with this
  *	we can program what color is displayed for a particular pixel value.
  *	DirectColor is similar in that we can program each color field. If
- *	we have a static colormap we don't need to implement this function. 
- * 
+ *	we have a static colormap we don't need to implement this function.
+ *
  *	Returns negative errno on error, or zero on success.
  */
 static int xxxfb_setcolreg(unsigned regno, unsigned red, unsigned green,
@@ -440,7 +441,7 @@ static int xxxfb_pan_display(struct fb_var_screeninfo *var,
 
 /**
  *      xxxfb_blank - NOT a required function. Blanks the display.
- *      @blank_mode: the blank mode we want. 
+ *      @blank_mode: the blank mode we want.
  *      @info: frame buffer structure that represents a single frame buffer
  *
  *      Blank the screen if blank_mode != FB_BLANK_UNBLANK, else unblank.
@@ -469,22 +470,22 @@ static int xxxfb_blank(int blank_mode, struct fb_info *info)
 
 /*
  * We provide our own functions if we have hardware acceleration
- * or non packed pixel format layouts. If we have no hardware 
+ * or non packed pixel format layouts. If we have no hardware
  * acceleration, we can use a generic unaccelerated function. If using
- * a pack pixel format just use the functions in cfb_*.c. Each file 
+ * a pack pixel format just use the functions in cfb_*.c. Each file
  * has one of the three different accel functions we support.
  */
 
 /**
- *      xxxfb_fillrect - REQUIRED function. Can use generic routines if 
+ *      xxxfb_fillrect - REQUIRED function. Can use generic routines if
  *		 	 non acclerated hardware and packed pixel based.
- *			 Draws a rectangle on the screen.		
+ *			 Draws a rectangle on the screen.
  *
  *      @info: frame buffer structure that represents a single frame buffer
- *	@region: The structure representing the rectangular region we 
+ *	@region: The structure representing the rectangular region we
  *		 wish to draw to.
  *
- *	This drawing operation places/removes a retangle on the screen 
+ *	This drawing operation places/removes a retangle on the screen
  *	depending on the rastering operation with the value of color which
  *	is in the current color depth format.
  */
@@ -492,13 +493,13 @@ void xxxfb_fillrect(struct fb_info *p, const struct fb_fillrect *region)
 {
 /*	Meaning of struct fb_fillrect
  *
- *	@dx: The x and y corrdinates of the upper left hand corner of the 
- *	@dy: area we want to draw to. 
+ *	@dx: The x and y corrdinates of the upper left hand corner of the
+ *	@dy: area we want to draw to.
  *	@width: How wide the rectangle is we want to draw.
  *	@height: How tall the rectangle is we want to draw.
- *	@color:	The color to fill in the rectangle with. 
+ *	@color:	The color to fill in the rectangle with.
  *	@rop: The raster operation. We can draw the rectangle with a COPY
- *	      of XOR which provides erasing effect. 
+ *	      of XOR which provides erasing effect.
  */
 }
 
@@ -514,7 +515,7 @@ void xxxfb_fillrect(struct fb_info *p, const struct fb_fillrect *region)
  *      This drawing operation copies a rectangular area from one area of the
  *	screen to another area.
  */
-void xxxfb_copyarea(struct fb_info *p, const struct fb_copyarea *area) 
+void xxxfb_copyarea(struct fb_info *p, const struct fb_copyarea *area)
 {
 /*
  *      @dx: The x and y coordinates of the upper left hand corner of the
@@ -530,28 +531,28 @@ void xxxfb_copyarea(struct fb_info *p, const struct fb_copyarea *area)
 /**
  *      xxxfb_imageblit - REQUIRED function. Can use generic routines if
  *                        non acclerated hardware and packed pixel based.
- *                        Copies a image from system memory to the screen. 
+ *                        Copies a image from system memory to the screen.
  *
  *      @info: frame buffer structure that represents a single frame buffer
  *	@image:	structure defining the image.
  *
- *      This drawing operation draws a image on the screen. It can be a 
+ *      This drawing operation draws a image on the screen. It can be a
  *	mono image (needed for font handling) or a color image (needed for
- *	tux). 
+ *	tux).
  */
-void xxxfb_imageblit(struct fb_info *p, const struct fb_image *image) 
+void xxxfb_imageblit(struct fb_info *p, const struct fb_image *image)
 {
 /*
  *      @dx: The x and y coordinates of the upper left hand corner of the
  *	@dy: destination area to place the image on the screen.
  *      @width: How wide the image is we want to copy.
  *      @height: How tall the image is we want to copy.
- *      @fg_color: For mono bitmap images this is color data for     
+ *      @fg_color: For mono bitmap images this is color data for
  *      @bg_color: the foreground and background of the image to
  *		   write directly to the frmaebuffer.
  *	@depth:	How many bits represent a single pixel for this image.
  *	@data: The actual data used to construct the image on the display.
- *	@cmap: The colormap used for color images.   
+ *	@cmap: The colormap used for color images.
  */
 
 /*
@@ -578,13 +579,13 @@ void xxxfb_imageblit(struct fb_info *p, const struct fb_image *image)
 int xxxfb_cursor(struct fb_info *info, struct fb_cursor *cursor)
 {
 /*
- *      @set: 	Which fields we are altering in struct fb_cursor 
- *	@enable: Disable or enable the cursor 
- *      @rop: 	The bit operation we want to do. 
- *      @mask:  This is the cursor mask bitmap. 
+ *      @set: 	Which fields we are altering in struct fb_cursor
+ *	@enable: Disable or enable the cursor
+ *      @rop: 	The bit operation we want to do.
+ *      @mask:  This is the cursor mask bitmap.
  *      @dest:  A image of the area we are going to display the cursor.
- *		Used internally by the driver.	 
- *      @hot:	The hot spot. 
+ *		Used internally by the driver.
+ *      @hot:	The hot spot.
  *	@image:	The actual data for the cursor image.
  *
  *      NOTES ON FLAGS (cursor->set):
@@ -612,11 +613,11 @@ int xxxfb_cursor(struct fb_info *info, struct fb_cursor *cursor)
 }
 
 /**
- *	xxxfb_sync - NOT a required function. Normally the accel engine 
+ *	xxxfb_sync - NOT a required function. Normally the accel engine
  *		     for a graphics card take a specific amount of time.
  *		     Often we have to wait for the accelerator to finish
  *		     its operation before we can write to the framebuffer
- *		     so we can have consistent display output. 
+ *		     so we can have consistent display output.
  *
  *      @info: frame buffer structure that represents a single frame buffer
  *
@@ -664,8 +665,15 @@ static int xxxfb_probe(struct pci_dev *dev, const struct pci_device_id *ent)
     struct fb_info *info;
     struct xxx_par *par;
     struct device *device = &dev->dev; /* or &pdev->dev */
-    int cmap_len, retval;	
-   
+    int cmap_len, retval;
+
+    /*
+     * Remove firmware-based drivers that create resource conflicts.
+     */
+    retval = aperture_remove_conflicting_pci_devices(pdev, "xxxfb");
+    if (retval)
+	    return retval;
+
     /*
      * Dynamically allocate info and par
      */
@@ -677,11 +685,11 @@ static int xxxfb_probe(struct pci_dev *dev, const struct pci_device_id *ent)
 
     par = info->par;
 
-    /* 
+    /*
      * Here we set the screen_base to the virtual memory address
      * for the framebuffer. Usually we obtain the resource address
      * from the bus layer and then translate it to virtual memory
-     * space via ioremap. Consult ioport.h. 
+     * space via ioremap. Consult ioport.h.
      */
     info->screen_base = framebuffer_virtual_memory;
     info->fbops = &xxxfb_ops;
@@ -765,24 +773,24 @@ static int xxxfb_probe(struct pci_dev *dev, const struct pci_device_id *ent)
 
     /*
      * This should give a reasonable default video mode. The following is
-     * done when we can set a video mode. 
+     * done when we can set a video mode.
      */
     if (!mode_option)
-	mode_option = "640x480@60";	 	
+	mode_option = "640x480@60";
 
     retval = fb_find_mode(&info->var, info, mode_option, NULL, 0, NULL, 8);
-  
+
     if (!retval || retval == 4)
-	return -EINVAL;			
+	return -EINVAL;
 
     /* This has to be done! */
     if (fb_alloc_cmap(&info->cmap, cmap_len, 0))
 	return -ENOMEM;
-	
-    /* 
-     * The following is done in the case of having hardware with a static 
-     * mode. If we are setting the mode ourselves we don't call this. 
-     */	
+
+    /*
+     * The following is done in the case of having hardware with a static
+     * mode. If we are setting the mode ourselves we don't call this.
+     */
     info->var = xxxfb_var;
 
     /*
diff --git a/drivers/video/fbdev/sm712fb.c b/drivers/video/fbdev/sm712fb.c
index 092a1caa1208..3baf33635e65 100644
--- a/drivers/video/fbdev/sm712fb.c
+++ b/drivers/video/fbdev/sm712fb.c
@@ -18,6 +18,7 @@
  * Framebuffer driver for Silicon Motion SM710, SM712, SM721 and SM722 chips
  */
 
+#include <linux/aperture.h>
 #include <linux/io.h>
 #include <linux/fb.h>
 #include <linux/pci.h>
@@ -1502,6 +1503,10 @@ static int smtcfb_pci_probe(struct pci_dev *pdev,
 
 	dev_info(&pdev->dev, "Silicon Motion display driver.\n");
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "smtcfb");
+	if (err)
+		return err;
+
 	err = pci_enable_device(pdev);	/* enable SMTC chip */
 	if (err)
 		return err;
diff --git a/drivers/video/fbdev/ssd1307fb.c b/drivers/video/fbdev/ssd1307fb.c
index 52e4ed9da78c..5c891aa00d59 100644
--- a/drivers/video/fbdev/ssd1307fb.c
+++ b/drivers/video/fbdev/ssd1307fb.c
@@ -817,7 +817,7 @@ fb_alloc_error:
 	return ret;
 }
 
-static int ssd1307fb_remove(struct i2c_client *client)
+static void ssd1307fb_remove(struct i2c_client *client)
 {
 	struct fb_info *info = i2c_get_clientdata(client);
 	struct ssd1307fb_par *par = info->par;
@@ -836,8 +836,6 @@ static int ssd1307fb_remove(struct i2c_client *client)
 	fb_deferred_io_cleanup(info);
 	__free_pages(__va(info->fix.smem_start), get_order(info->fix.smem_len));
 	framebuffer_release(info);
-
-	return 0;
 }
 
 static const struct i2c_device_id ssd1307fb_i2c_id[] = {
diff --git a/drivers/video/fbdev/sstfb.c b/drivers/video/fbdev/sstfb.c
index cd4d640f9477..a56b24288566 100644
--- a/drivers/video/fbdev/sstfb.c
+++ b/drivers/video/fbdev/sstfb.c
@@ -80,6 +80,7 @@
  * Includes
  */
 
+#include <linux/aperture.h>
 #include <linux/string.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
@@ -364,7 +365,7 @@ static int sstfb_check_var(struct fb_var_screeninfo *var,
 		return -EINVAL;
 	}
 	var->pixclock = KHZ2PICOS(freq);
-	
+
 	if (var->vmode & FB_VMODE_INTERLACED)
 		vBackPorch += (vBackPorch % 2);
 	if (var->vmode & FB_VMODE_DOUBLE) {
@@ -382,7 +383,7 @@ static int sstfb_check_var(struct fb_var_screeninfo *var,
 		printk(KERN_ERR "sstfb: Unsupported bpp %d\n", var->bits_per_pixel);
 		return -EINVAL;
 	}
-	
+
 	/* validity tests */
 	if (var->xres <= 1 || yDim <= 0 || var->hsync_len <= 1  ||
 	    hSyncOff <= 1  || var->left_margin <= 2  || vSyncOn <= 0 ||
@@ -392,7 +393,7 @@ static int sstfb_check_var(struct fb_var_screeninfo *var,
 
 	if (IS_VOODOO2(par)) {
 		/* Voodoo 2 limits */
-		tiles_in_X = (var->xres + 63 ) / 64 * 2;		
+		tiles_in_X = (var->xres + 63 ) / 64 * 2;
 
 		if (var->xres  > POW2(11) || yDim >= POW2(11)) {
 			printk(KERN_ERR "sstfb: Unsupported resolution %dx%d\n",
@@ -631,7 +632,7 @@ static int sstfb_set_par(struct fb_info *info)
 	lfbmode |= ( LFB_WORD_SWIZZLE_WR | LFB_BYTE_SWIZZLE_WR |
 		     LFB_WORD_SWIZZLE_RD | LFB_BYTE_SWIZZLE_RD );
 #endif
-	
+
 	if (clipping) {
 		sst_write(LFBMODE, lfbmode | EN_PXL_PIPELINE);
 	/*
@@ -684,7 +685,7 @@ static int sstfb_setcolreg(u_int regno, u_int red, u_int green, u_int blue,
 	    | (green << info->var.green.offset)
 	    | (blue  << info->var.blue.offset)
 	    | (transp << info->var.transp.offset);
-	
+
 	par->palette[regno] = col;
 
 	return 0;
@@ -773,7 +774,7 @@ static void sstfb_copyarea(struct fb_info *info, const struct fb_copyarea *area)
 {
 	struct sstfb_par *par = info->par;
 	u32 stride = info->fix.line_length;
-   
+
 	if (!IS_VOODOO2(par))
 		return;
 
@@ -795,17 +796,17 @@ static void sstfb_copyarea(struct fb_info *info, const struct fb_copyarea *area)
  * FillRect 2D command (solidfill or invert (via ROP_XOR)) - Voodoo2 only
  */
 #if 0
-static void sstfb_fillrect(struct fb_info *info, const struct fb_fillrect *rect) 
+static void sstfb_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 {
 	struct sstfb_par *par = info->par;
 	u32 stride = info->fix.line_length;
 
 	if (!IS_VOODOO2(par))
 		return;
-   	
+
 	sst_write(BLTCLIPX, info->var.xres);
 	sst_write(BLTCLIPY, info->var.yres);
-	
+
 	sst_write(BLTDSTBASEADDR, 0);
 	sst_write(BLTCOLOR, rect->color);
 	sst_write(BLTROP, rect->rop == ROP_COPY ? BLTROP_COPY : BLTROP_XOR);
@@ -820,8 +821,8 @@ static void sstfb_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 
 
 
-/* 
- * get lfb size 
+/*
+ * get lfb size
  */
 static int sst_get_memsize(struct fb_info *info, __u32 *memsize)
 {
@@ -859,8 +860,8 @@ static int sst_get_memsize(struct fb_info *info, __u32 *memsize)
 }
 
 
-/* 
- * DAC detection routines 
+/*
+ * DAC detection routines
  */
 
 /* fbi should be idle, and fifo emty and mem disabled */
@@ -963,7 +964,7 @@ static int sst_detect_ics(struct fb_info *info)
  * see detect_dac
  */
 
-static int sst_set_pll_att_ti(struct fb_info *info, 
+static int sst_set_pll_att_ti(struct fb_info *info,
 		const struct pll_timing *t, const int clock)
 {
 	struct sstfb_par *par = info->par;
@@ -1326,6 +1327,10 @@ static int sstfb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	struct sst_spec *spec;
 	int err;
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "sstfb");
+	if (err)
+		return err;
+
 	/* Enable device in PCI config. */
 	if ((err=pci_enable_device(pdev))) {
 		printk(KERN_ERR "cannot enable device\n");
@@ -1338,10 +1343,10 @@ static int sstfb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		return -ENOMEM;
 
 	pci_set_drvdata(pdev, info);
-	
+
 	par  = info->par;
 	fix  = &info->fix;
-	
+
 	par->type = id->driver_data;
 	spec = &voodoo_spec[par->type];
 	f_ddprintk("found device : %s\n", spec->name);
@@ -1407,7 +1412,7 @@ static int sstfb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	 * fact dithered to 16bit).
 	 */
 	fix->line_length = 2048; /* default value, for 24 or 32bit: 4096 */
-	
+
 	fb_find_mode(&info->var, info, mode_option, NULL, 0, NULL, 16);
 
 	if (sstfb_check_var(&info->var, info)) {
@@ -1419,7 +1424,7 @@ static int sstfb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		printk(KERN_ERR "sstfb: can't set default video mode.\n");
 		goto fail;
 	}
-	
+
 	if (fb_alloc_cmap(&info->cmap, 256, 0)) {
 		printk(KERN_ERR "sstfb: can't alloc cmap memory.\n");
 		goto fail;
@@ -1465,7 +1470,7 @@ static void sstfb_remove(struct pci_dev *pdev)
 
 	info = pci_get_drvdata(pdev);
 	par = info->par;
-	
+
 	device_remove_file(info->dev, &device_attrs[0]);
 	sst_shutdown(info);
 	iounmap(info->screen_base);
diff --git a/drivers/video/fbdev/sunxvr2500.c b/drivers/video/fbdev/sunxvr2500.c
index 1279b02234f8..f4059529c602 100644
--- a/drivers/video/fbdev/sunxvr2500.c
+++ b/drivers/video/fbdev/sunxvr2500.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
  */
 
+#include <linux/aperture.h>
 #include <linux/kernel.h>
 #include <linux/fb.h>
 #include <linux/pci.h>
@@ -123,6 +124,10 @@ static int s3d_pci_register(struct pci_dev *pdev,
 	struct s3d_info *sp;
 	int err;
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "s3dfb");
+	if (err)
+		return err;
+
 	err = pci_enable_device(pdev);
 	if (err < 0) {
 		printk(KERN_ERR "s3d: Cannot enable PCI device %s\n",
diff --git a/drivers/video/fbdev/sunxvr500.c b/drivers/video/fbdev/sunxvr500.c
index f7b463633ba0..b0c8cf0c535a 100644
--- a/drivers/video/fbdev/sunxvr500.c
+++ b/drivers/video/fbdev/sunxvr500.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2007 David S. Miller (davem@davemloft.net)
  */
 
+#include <linux/aperture.h>
 #include <linux/kernel.h>
 #include <linux/fb.h>
 #include <linux/pci.h>
@@ -249,6 +250,10 @@ static int e3d_pci_register(struct pci_dev *pdev,
 	unsigned int line_length;
 	int err;
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "e3dfb");
+	if (err)
+		return err;
+
 	of_node = pci_device_to_OF_node(pdev);
 	if (!of_node) {
 		printk(KERN_ERR "e3d: Cannot find OF node of %s\n",
diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c
index 8a8122f8bfeb..592a913d0718 100644
--- a/drivers/video/fbdev/tdfxfb.c
+++ b/drivers/video/fbdev/tdfxfb.c
@@ -64,6 +64,7 @@
  *
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -1376,6 +1377,10 @@ static int tdfxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	struct fb_monspecs *specs;
 	bool found;
 
+	err = aperture_remove_conflicting_pci_devices(pdev, "tdfxfb");
+	if (err)
+		return err;
+
 	err = pci_enable_device(pdev);
 	if (err) {
 		printk(KERN_ERR "tdfxfb: Can't enable pdev: %d\n", err);
diff --git a/drivers/video/fbdev/tgafb.c b/drivers/video/fbdev/tgafb.c
index 1fff5fd7ab51..251dbd282f5e 100644
--- a/drivers/video/fbdev/tgafb.c
+++ b/drivers/video/fbdev/tgafb.c
@@ -12,6 +12,7 @@
  *  more details.
  */
 
+#include <linux/aperture.h>
 #include <linux/bitrev.h>
 #include <linux/compiler.h>
 #include <linux/delay.h>
@@ -106,6 +107,12 @@ static struct pci_driver tgafb_pci_driver = {
 static int tgafb_pci_register(struct pci_dev *pdev,
 			      const struct pci_device_id *ent)
 {
+	int ret;
+
+	ret = aperture_remove_conflicting_pci_devices(pdev, "tgafb");
+	if (ret)
+		return ret;
+
 	return tgafb_register(&pdev->dev);
 }
 
@@ -729,7 +736,7 @@ tgafb_mono_imageblit(struct fb_info *info, const struct fb_image *image)
 
 		/* Handle another common case in which accel_putcs
 		   generates a large bitmap, which happens to be aligned.
-		   Allow the tail to be misaligned.  This case is 
+		   Allow the tail to be misaligned.  This case is
 		   interesting because we've not got to hold partial
 		   bytes across the words being written.  */
 
@@ -908,9 +915,9 @@ tgafb_imageblit(struct fb_info *info, const struct fb_image *image)
 }
 
 /**
- *      tgafb_fillrect - REQUIRED function. Can use generic routines if 
+ *      tgafb_fillrect - REQUIRED function. Can use generic routines if
  *                       non acclerated hardware and packed pixel based.
- *                       Draws a rectangle on the screen.               
+ *                       Draws a rectangle on the screen.
  *
  *      @info: frame buffer structure that represents a single frame buffer
  *      @rect: structure defining the rectagle and operation.
@@ -1044,7 +1051,7 @@ tgafb_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
 
 /* Handle the special case of copying entire lines, e.g. during scrolling.
    We can avoid a lot of needless computation in this case.  In the 8bpp
-   case we need to use the COPY64 registers instead of mask writes into 
+   case we need to use the COPY64 registers instead of mask writes into
    the frame buffer to achieve maximum performance.  */
 
 static inline void
@@ -1251,7 +1258,7 @@ copyarea_8bpp(struct fb_info *info, u32 dx, u32 dy, u32 sx, u32 sy,
 }
 
 static void
-tgafb_copyarea(struct fb_info *info, const struct fb_copyarea *area) 
+tgafb_copyarea(struct fb_info *info, const struct fb_copyarea *area)
 {
 	unsigned long dx, dy, width, height, sx, sy, vxres, vyres;
 	unsigned long line_length, bpp;
diff --git a/drivers/video/fbdev/tridentfb.c b/drivers/video/fbdev/tridentfb.c
index cda095420ee8..f9c3b1d38fc2 100644
--- a/drivers/video/fbdev/tridentfb.c
+++ b/drivers/video/fbdev/tridentfb.c
@@ -16,6 +16,7 @@
  *	timing value tweaking so it looks good on every monitor in every mode
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/fb.h>
 #include <linux/init.h>
@@ -1470,6 +1471,10 @@ static int trident_pci_probe(struct pci_dev *dev,
 	int chip_id;
 	bool found = false;
 
+	err = aperture_remove_conflicting_pci_devices(dev, "tridentfb");
+	if (err)
+		return err;
+
 	err = pci_enable_device(dev);
 	if (err)
 		return err;
diff --git a/drivers/video/fbdev/vermilion/vermilion.c b/drivers/video/fbdev/vermilion/vermilion.c
index ff61605b8764..82b36dbb5b1a 100644
--- a/drivers/video/fbdev/vermilion/vermilion.c
+++ b/drivers/video/fbdev/vermilion/vermilion.c
@@ -14,6 +14,7 @@
  *   Alan Hourihane <alanh-at-tungstengraphics-dot-com>
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -442,7 +443,11 @@ static int vml_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 	struct vml_info *vinfo;
 	struct fb_info *info;
 	struct vml_par *par;
-	int err = 0;
+	int err;
+
+	err = aperture_remove_conflicting_pci_devices(dev, "vmlfb");
+	if (err)
+		return err;
 
 	par = kzalloc(sizeof(*par), GFP_KERNEL);
 	if (par == NULL)
diff --git a/drivers/video/fbdev/vga16fb.c b/drivers/video/fbdev/vga16fb.c
index d21f68f3ee44..35cf51ae3292 100644
--- a/drivers/video/fbdev/vga16fb.c
+++ b/drivers/video/fbdev/vga16fb.c
@@ -1,13 +1,13 @@
 /*
  * linux/drivers/video/vga16.c -- VGA 16-color framebuffer driver
- * 
+ *
  * Copyright 1999 Ben Pfaff <pfaffben@debian.org> and Petr Vandrovec <VANDROVE@vc.cvut.cz>
  * Based on VGA info at http://www.goodnet.com/~tinara/FreeVGA/home.htm
  * Based on VESA framebuffer (c) 1998 Gerd Knorr <kraxel@goldbach.in-berlin.de>
  *
  * This file is subject to the terms and conditions of the GNU General
  * Public License.  See the file COPYING in the main directory of this
- * archive for more details.  
+ * archive for more details.
  */
 
 #include <linux/module.h>
@@ -25,9 +25,6 @@
 #include <asm/io.h>
 #include <video/vga.h>
 
-#define VGA_FB_PHYS 0xA0000
-#define VGA_FB_PHYS_LEN 65536
-
 #define MODE_SKIP4	1
 #define MODE_8BPP	2
 #define MODE_CFB	4
@@ -70,7 +67,7 @@ static struct fb_var_screeninfo vga16fb_defined = {
 	.yres		= 480,
 	.xres_virtual	= 640,
 	.yres_virtual	= 480,
-	.bits_per_pixel	= 4,	
+	.bits_per_pixel	= 4,
 	.activate	= FB_ACTIVATE_TEST,
 	.height		= -1,
 	.width		= -1,
@@ -87,8 +84,8 @@ static struct fb_var_screeninfo vga16fb_defined = {
 /* name should not depend on EGA/VGA */
 static const struct fb_fix_screeninfo vga16fb_fix = {
 	.id		= "VGA16 VGA",
-	.smem_start	= VGA_FB_PHYS,
-	.smem_len	= VGA_FB_PHYS_LEN,
+	.smem_start	= VGA_FB_PHYS_BASE,
+	.smem_len	= VGA_FB_PHYS_SIZE,
 	.type		= FB_TYPE_VGA_PLANES,
 	.type_aux	= FB_AUX_VGA_PLANES_VGA4,
 	.visual		= FB_VISUAL_PSEUDOCOLOR,
@@ -120,7 +117,7 @@ static inline void rmw(volatile char __iomem *p)
 static inline int setmode(int mode)
 {
 	int oldmode;
-	
+
 	oldmode = vga_io_rgfx(VGA_GFX_MODE);
 	vga_io_w(VGA_GFX_D, mode);
 	return oldmode;
@@ -139,19 +136,19 @@ static inline void setmask(int mask)
 	vga_io_w(VGA_GFX_D, mask);
 }
 
-/* Set the Data Rotate Register and return its old value. 
+/* Set the Data Rotate Register and return its old value.
    Bits 0-2 are rotate count, bits 3-4 are logical operation
    (0=NOP, 1=AND, 2=OR, 3=XOR). */
 static inline int setop(int op)
 {
 	int oldop;
-	
+
 	oldop = vga_io_rgfx(VGA_GFX_DATA_ROTATE);
 	vga_io_w(VGA_GFX_D, op);
 	return oldop;
 }
 
-/* Set the Enable Set/Reset Register and return its old value.  
+/* Set the Enable Set/Reset Register and return its old value.
    The code here always uses value 0xf for this register. */
 static inline int setsr(int sr)
 {
@@ -185,25 +182,25 @@ static inline void setindex(int index)
 }
 
 /* Check if the video mode is supported by the driver */
-static inline int check_mode_supported(void)
+static inline int check_mode_supported(const struct screen_info *si)
 {
 	/* non-x86 architectures treat orig_video_isVGA as a boolean flag */
 #if defined(CONFIG_X86)
 	/* only EGA and VGA in 16 color graphic mode are supported */
-	if (screen_info.orig_video_isVGA != VIDEO_TYPE_EGAC &&
-	    screen_info.orig_video_isVGA != VIDEO_TYPE_VGAC)
+	if (si->orig_video_isVGA != VIDEO_TYPE_EGAC &&
+	    si->orig_video_isVGA != VIDEO_TYPE_VGAC)
 		return -ENODEV;
 
-	if (screen_info.orig_video_mode != 0x0D &&	/* 320x200/4 (EGA) */
-	    screen_info.orig_video_mode != 0x0E &&	/* 640x200/4 (EGA) */
-	    screen_info.orig_video_mode != 0x10 &&	/* 640x350/4 (EGA) */
-	    screen_info.orig_video_mode != 0x12)	/* 640x480/4 (VGA) */
+	if (si->orig_video_mode != 0x0D &&	/* 320x200/4 (EGA) */
+	    si->orig_video_mode != 0x0E &&	/* 640x200/4 (EGA) */
+	    si->orig_video_mode != 0x10 &&	/* 640x350/4 (EGA) */
+	    si->orig_video_mode != 0x12)	/* 640x480/4 (VGA) */
 		return -ENODEV;
 #endif
 	return 0;
 }
 
-static void vga16fb_pan_var(struct fb_info *info, 
+static void vga16fb_pan_var(struct fb_info *info,
 			    struct fb_var_screeninfo *var)
 {
 	struct vga16fb_par *par = info->par;
@@ -296,7 +293,7 @@ static void vga16fb_clock_chip(struct vga16fb_par *par,
 	par->clkdiv = best->seq_clock_mode;
 	*pixclock = (best->pixclock * div) / mul;
 }
-			       
+
 #define FAIL(X) return -EINVAL
 
 static int vga16fb_open(struct fb_info *info, int user)
@@ -511,7 +508,7 @@ static int vga16fb_check_var(struct fb_var_screeninfo *var,
 		par->misc &= ~0x40;
 	if (var->sync & FB_SYNC_VERT_HIGH_ACT)
 		par->misc &= ~0x80;
-	
+
 	par->mode = mode;
 
 	if (mode & MODE_8BPP)
@@ -520,8 +517,8 @@ static int vga16fb_check_var(struct fb_var_screeninfo *var,
 	else
 		/* pixel clock == vga clock */
 		vga16fb_clock_chip(par, &var->pixclock, info, 1, 1);
-	
-	var->red.offset = var->green.offset = var->blue.offset = 
+
+	var->red.offset = var->green.offset = var->blue.offset =
 	var->transp.offset = 0;
 	var->red.length = var->green.length = var->blue.length =
 		(par->isVGA) ? 6 : 2;
@@ -588,10 +585,10 @@ static int vga16fb_set_par(struct fb_info *info)
 	else
 		atc[VGA_ATC_PEL] = info->var.xoffset & 7;
 	atc[VGA_ATC_COLOR_PAGE] = 0x00;
-	
+
 	if (par->mode & MODE_TEXT) {
-		fh = 16; // FIXME !!! Fudge font height. 
-		par->crtc[VGA_CRTC_MAX_SCAN] = (par->crtc[VGA_CRTC_MAX_SCAN] 
+		fh = 16; // FIXME !!! Fudge font height.
+		par->crtc[VGA_CRTC_MAX_SCAN] = (par->crtc[VGA_CRTC_MAX_SCAN]
 					       & ~0x1F) | (fh - 1);
 	}
 
@@ -602,10 +599,10 @@ static int vga16fb_set_par(struct fb_info *info)
 		vga_io_w(EGA_GFX_E0, 0x00);
 		vga_io_w(EGA_GFX_E1, 0x01);
 	}
-	
+
 	/* update misc output register */
 	vga_io_w(VGA_MIS_W, par->misc);
-	
+
 	/* synchronous reset on */
 	vga_io_wseq(0x00, 0x01);
 
@@ -617,7 +614,7 @@ static int vga16fb_set_par(struct fb_info *info)
 	for (i = 2; i < VGA_SEQ_C; i++) {
 		vga_io_wseq(i, seq[i]);
 	}
-	
+
 	/* synchronous reset off */
 	vga_io_wseq(0x00, 0x03);
 
@@ -628,12 +625,12 @@ static int vga16fb_set_par(struct fb_info *info)
 	for (i = 0; i < VGA_CRTC_REGS; i++) {
 		vga_io_wcrt(i, par->crtc[i]);
 	}
-	
+
 	/* write graphics controller registers */
 	for (i = 0; i < VGA_GFX_C; i++) {
 		vga_io_wgfx(i, gdc[i]);
 	}
-	
+
 	/* write attribute controller registers */
 	for (i = 0; i < VGA_ATT_C; i++) {
 		vga_io_r(VGA_IS1_RC);		/* reset flip-flop */
@@ -656,7 +653,7 @@ static void ega16_setpalette(int regno, unsigned red, unsigned green, unsigned b
 {
 	static const unsigned char map[] = { 000, 001, 010, 011 };
 	int val;
-	
+
 	if (regno >= 16)
 		return;
 	val = map[red>>14] | ((map[green>>14]) << 1) | ((map[blue>>14]) << 2);
@@ -687,17 +684,17 @@ static int vga16fb_setcolreg(unsigned regno, unsigned red, unsigned green,
 	 *  (according to the entries in the `var' structure). Return
 	 *  != 0 for invalid regno.
 	 */
-	
+
 	if (regno >= 256)
 		return 1;
 
 	gray = info->var.grayscale;
-	
+
 	if (gray) {
 		/* gray = 0.30*R + 0.59*G + 0.11*B */
 		red = green = blue = (red * 77 + green * 151 + blue * 28) >> 8;
 	}
-	if (par->isVGA) 
+	if (par->isVGA)
 		vga16_setpalette(regno,red,green,blue);
 	else
 		ega16_setpalette(regno,red,green,blue);
@@ -705,7 +702,7 @@ static int vga16fb_setcolreg(unsigned regno, unsigned red, unsigned green,
 }
 
 static int vga16fb_pan_display(struct fb_var_screeninfo *var,
-			       struct fb_info *info) 
+			       struct fb_info *info)
 {
 	vga16fb_pan_var(info, var);
 	return 0;
@@ -720,7 +717,7 @@ static void vga_vesa_blank(struct vga16fb_par *par, int mode)
 {
 	unsigned char SeqCtrlIndex = vga_io_r(VGA_SEQ_I);
 	unsigned char CrtCtrlIndex = vga_io_r(VGA_CRT_IC);
-	
+
 	/* save original values of VGA controller registers */
 	if(!par->vesa_blanked) {
 		par->vga_state.CrtMiscIO = vga_io_r(VGA_MIS_R);
@@ -776,7 +773,7 @@ static void vga_vesa_unblank(struct vga16fb_par *par)
 {
 	unsigned char SeqCtrlIndex = vga_io_r(VGA_SEQ_I);
 	unsigned char CrtCtrlIndex = vga_io_r(VGA_CRT_IC);
-	
+
 	/* restore original values of VGA controller registers */
 	vga_io_w(VGA_MIS_W, par->vga_state.CrtMiscIO);
 
@@ -962,7 +959,7 @@ static void vga16fb_fillrect(struct fb_info *info, const struct fb_fillrect *rec
 				}
 				break;
 			}
-		} else 
+		} else
 			vga_8planes_fillrect(info, rect);
 		break;
 	case FB_TYPE_PACKED_PIXELS:
@@ -1029,7 +1026,7 @@ static void vga_8planes_copyarea(struct fb_info *info, const struct fb_copyarea
 
 static void vga16fb_copyarea(struct fb_info *info, const struct fb_copyarea *area)
 {
-	u32 dx = area->dx, dy = area->dy, sx = area->sx, sy = area->sy; 
+	u32 dx = area->dx, dy = area->dy, sx = area->sx, sy = area->sy;
 	int x, x2, y2, old_dx, old_dy, vxres, vyres;
 	int height, width, line_ofs;
 	char __iomem *dst = NULL;
@@ -1094,9 +1091,9 @@ static void vga16fb_copyarea(struct fb_info *info, const struct fb_copyarea *are
 					dst += line_ofs;
 				}
 			} else {
-				dst = info->screen_base + (dx/8) + width + 
+				dst = info->screen_base + (dx/8) + width +
 					(dy + height - 1) * info->fix.line_length;
-				src = info->screen_base + (sx/8) + width + 
+				src = info->screen_base + (sx/8) + width +
 					(sy + height  - 1) * info->fix.line_length;
 				while (height--) {
 					for (x = 0; x < width; x++) {
@@ -1109,7 +1106,7 @@ static void vga16fb_copyarea(struct fb_info *info, const struct fb_copyarea *are
 					dst -= line_ofs;
 				}
 			}
-		} else 
+		} else
 			vga_8planes_copyarea(info, area);
 		break;
 	case FB_TYPE_PACKED_PIXELS:
@@ -1182,7 +1179,7 @@ static void vga_imageblit_expand(struct fb_info *info, const struct fb_image *im
 				setsr(0xf);
 				setcolor(image->fg_color);
 				selectmask();
-				
+
 				setmask(0xff);
 				writeb(image->bg_color, where);
 				rmb();
@@ -1191,7 +1188,7 @@ static void vga_imageblit_expand(struct fb_info *info, const struct fb_image *im
 				wmb();
 				for (y = 0; y < image->height; y++) {
 					dst = where;
-					for (x = image->width/8; x--;) 
+					for (x = image->width/8; x--;)
 						writeb(*cdat++, dst++);
 					where += info->fix.line_length;
 				}
@@ -1202,7 +1199,7 @@ static void vga_imageblit_expand(struct fb_info *info, const struct fb_image *im
 				setsr(0xf);
 				setcolor(image->bg_color);
 				selectmask();
-				
+
 				setmask(0xff);
 				for (y = 0; y < image->height; y++) {
 					dst = where;
@@ -1218,7 +1215,7 @@ static void vga_imageblit_expand(struct fb_info *info, const struct fb_image *im
 					where += info->fix.line_length;
 				}
 			}
-		} else 
+		} else
 			vga_8planes_imageblit(info, image);
 		break;
 	case FB_TYPE_PACKED_PIXELS:
@@ -1231,7 +1228,7 @@ static void vga_imageblit_expand(struct fb_info *info, const struct fb_image *im
 static void vga_imageblit_color(struct fb_info *info, const struct fb_image *image)
 {
 	/*
-	 * Draw logo 
+	 * Draw logo
 	 */
 	struct vga16fb_par *par = info->par;
 	char __iomem *where =
@@ -1248,7 +1245,7 @@ static void vga_imageblit_color(struct fb_info *info, const struct fb_image *ima
 			setsr(0xf);
 			setop(0);
 			setmode(0);
-			
+
 			for (y = 0; y < image->height; y++) {
 				for (x = 0; x < image->width; x++) {
 					dst = where + x/8;
@@ -1272,7 +1269,7 @@ static void vga_imageblit_color(struct fb_info *info, const struct fb_image *ima
 		break;
 	}
 }
-				
+
 static void vga16fb_imageblit(struct fb_info *info, const struct fb_image *image)
 {
 	if (image->depth == 1)
@@ -1304,28 +1301,22 @@ static const struct fb_ops vga16fb_ops = {
 	.fb_imageblit	= vga16fb_imageblit,
 };
 
-#ifndef MODULE
-static int __init vga16fb_setup(char *options)
-{
-	char *this_opt;
-	
-	if (!options || !*options)
-		return 0;
-	
-	while ((this_opt = strsep(&options, ",")) != NULL) {
-		if (!*this_opt) continue;
-	}
-	return 0;
-}
-#endif
-
 static int vga16fb_probe(struct platform_device *dev)
 {
+	struct screen_info *si;
 	struct fb_info *info;
 	struct vga16fb_par *par;
 	int i;
 	int ret = 0;
 
+	si = dev_get_platdata(&dev->dev);
+	if (!si)
+		return -ENODEV;
+
+	ret = check_mode_supported(si);
+	if (ret)
+		return ret;
+
 	printk(KERN_DEBUG "vga16fb: initializing\n");
 	info = framebuffer_alloc(sizeof(struct vga16fb_par), &dev->dev);
 
@@ -1339,8 +1330,8 @@ static int vga16fb_probe(struct platform_device *dev)
 		goto err_ioremap;
 	}
 
-	/* XXX share VGA_FB_PHYS and I/O region with vgacon and others */
-	info->screen_base = (void __iomem *)VGA_MAP_MEM(VGA_FB_PHYS, 0);
+	/* XXX share VGA_FB_PHYS_BASE and I/O region with vgacon and others */
+	info->screen_base = (void __iomem *)VGA_MAP_MEM(VGA_FB_PHYS_BASE, 0);
 
 	if (!info->screen_base) {
 		printk(KERN_ERR "vga16fb: unable to map device\n");
@@ -1352,19 +1343,19 @@ static int vga16fb_probe(struct platform_device *dev)
 	par = info->par;
 
 #if defined(CONFIG_X86)
-	par->isVGA = screen_info.orig_video_isVGA == VIDEO_TYPE_VGAC;
+	par->isVGA = si->orig_video_isVGA == VIDEO_TYPE_VGAC;
 #else
 	/* non-x86 architectures treat orig_video_isVGA as a boolean flag */
-	par->isVGA = screen_info.orig_video_isVGA;
+	par->isVGA = si->orig_video_isVGA;
 #endif
 	par->palette_blanked = 0;
 	par->vesa_blanked = 0;
 
 	i = par->isVGA? 6 : 2;
-	
+
 	vga16fb_defined.red.length   = i;
 	vga16fb_defined.green.length = i;
-	vga16fb_defined.blue.length  = i;	
+	vga16fb_defined.blue.length  = i;
 
 	/* name should not depend on EGA/VGA */
 	info->fbops = &vga16fb_ops;
@@ -1391,8 +1382,8 @@ static int vga16fb_probe(struct platform_device *dev)
 
 	vga16fb_update_fix(info);
 
-	info->apertures->ranges[0].base = VGA_FB_PHYS;
-	info->apertures->ranges[0].size = VGA_FB_PHYS_LEN;
+	info->apertures->ranges[0].base = VGA_FB_PHYS_BASE;
+	info->apertures->ranges[0].size = VGA_FB_PHYS_SIZE;
 
 	if (register_framebuffer(info) < 0) {
 		printk(KERN_ERR "vga16fb: unable to register framebuffer\n");
@@ -1425,58 +1416,22 @@ static int vga16fb_remove(struct platform_device *dev)
 	return 0;
 }
 
+static const struct platform_device_id vga16fb_driver_id_table[] = {
+	{"ega-framebuffer", 0},
+	{"vga-framebuffer", 0},
+	{ }
+};
+
 static struct platform_driver vga16fb_driver = {
 	.probe = vga16fb_probe,
 	.remove = vga16fb_remove,
 	.driver = {
 		.name = "vga16fb",
 	},
+	.id_table = vga16fb_driver_id_table,
 };
 
-static struct platform_device *vga16fb_device;
-
-static int __init vga16fb_init(void)
-{
-	int ret;
-#ifndef MODULE
-	char *option = NULL;
-
-	if (fb_get_options("vga16fb", &option))
-		return -ENODEV;
-
-	vga16fb_setup(option);
-#endif
-
-	ret = check_mode_supported();
-	if (ret)
-		return ret;
-
-	ret = platform_driver_register(&vga16fb_driver);
-
-	if (!ret) {
-		vga16fb_device = platform_device_alloc("vga16fb", 0);
-
-		if (vga16fb_device)
-			ret = platform_device_add(vga16fb_device);
-		else
-			ret = -ENOMEM;
-
-		if (ret) {
-			platform_device_put(vga16fb_device);
-			platform_driver_unregister(&vga16fb_driver);
-		}
-	}
-
-	return ret;
-}
-
-static void __exit vga16fb_exit(void)
-{
-	platform_device_unregister(vga16fb_device);
-	platform_driver_unregister(&vga16fb_driver);
-}
+module_platform_driver(vga16fb_driver);
 
 MODULE_DESCRIPTION("Legacy VGA framebuffer device driver");
 MODULE_LICENSE("GPL");
-module_init(vga16fb_init);
-module_exit(vga16fb_exit);
diff --git a/drivers/video/fbdev/via/via-core.c b/drivers/video/fbdev/via/via-core.c
index 89d75079b730..2ee8fcae08df 100644
--- a/drivers/video/fbdev/via/via-core.c
+++ b/drivers/video/fbdev/via/via-core.c
@@ -8,6 +8,7 @@
 /*
  * Core code for the Via multifunction framebuffer device.
  */
+#include <linux/aperture.h>
 #include <linux/via-core.h>
 #include <linux/via_i2c.h>
 #include <linux/via-gpio.h>
@@ -617,6 +618,10 @@ static int via_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	int ret;
 
+	ret = aperture_remove_conflicting_pci_devices(pdev, "viafb");
+	if (ret)
+		return ret;
+
 	ret = pci_enable_device(pdev);
 	if (ret)
 		return ret;
diff --git a/drivers/video/fbdev/vt8623fb.c b/drivers/video/fbdev/vt8623fb.c
index 4274c6efb249..49b9f148d3a1 100644
--- a/drivers/video/fbdev/vt8623fb.c
+++ b/drivers/video/fbdev/vt8623fb.c
@@ -12,6 +12,7 @@
  * (http://davesdomain.org.uk/viafb/)
  */
 
+#include <linux/aperture.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
@@ -672,6 +673,10 @@ static int vt8623_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 		return -ENODEV;
 	}
 
+	rc = aperture_remove_conflicting_pci_devices(dev, "vt8623fb");
+	if (rc)
+		return rc;
+
 	/* Allocate and fill driver data structure */
 	info = framebuffer_alloc(sizeof(struct vt8623fb_info), &(dev->dev));
 	if (!info)
diff --git a/drivers/video/hdmi.c b/drivers/video/hdmi.c
index 947be761dfa4..03c7f27dde49 100644
--- a/drivers/video/hdmi.c
+++ b/drivers/video/hdmi.c
@@ -21,6 +21,7 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#include <drm/display/drm_dp.h>
 #include <linux/bitops.h>
 #include <linux/bug.h>
 #include <linux/errno.h>
@@ -381,12 +382,34 @@ static int hdmi_audio_infoframe_check_only(const struct hdmi_audio_infoframe *fr
  *
  * Returns 0 on success or a negative error code on failure.
  */
-int hdmi_audio_infoframe_check(struct hdmi_audio_infoframe *frame)
+int hdmi_audio_infoframe_check(const struct hdmi_audio_infoframe *frame)
 {
 	return hdmi_audio_infoframe_check_only(frame);
 }
 EXPORT_SYMBOL(hdmi_audio_infoframe_check);
 
+static void
+hdmi_audio_infoframe_pack_payload(const struct hdmi_audio_infoframe *frame,
+				  u8 *buffer)
+{
+	u8 channels;
+
+	if (frame->channels >= 2)
+		channels = frame->channels - 1;
+	else
+		channels = 0;
+
+	buffer[0] = ((frame->coding_type & 0xf) << 4) | (channels & 0x7);
+	buffer[1] = ((frame->sample_frequency & 0x7) << 2) |
+		 (frame->sample_size & 0x3);
+	buffer[2] = frame->coding_type_ext & 0x1f;
+	buffer[3] = frame->channel_allocation;
+	buffer[4] = (frame->level_shift_value & 0xf) << 3;
+
+	if (frame->downmix_inhibit)
+		buffer[4] |= BIT(7);
+}
+
 /**
  * hdmi_audio_infoframe_pack_only() - write HDMI audio infoframe to binary buffer
  * @frame: HDMI audio infoframe
@@ -404,7 +427,6 @@ EXPORT_SYMBOL(hdmi_audio_infoframe_check);
 ssize_t hdmi_audio_infoframe_pack_only(const struct hdmi_audio_infoframe *frame,
 				       void *buffer, size_t size)
 {
-	unsigned char channels;
 	u8 *ptr = buffer;
 	size_t length;
 	int ret;
@@ -420,28 +442,13 @@ ssize_t hdmi_audio_infoframe_pack_only(const struct hdmi_audio_infoframe *frame,
 
 	memset(buffer, 0, size);
 
-	if (frame->channels >= 2)
-		channels = frame->channels - 1;
-	else
-		channels = 0;
-
 	ptr[0] = frame->type;
 	ptr[1] = frame->version;
 	ptr[2] = frame->length;
 	ptr[3] = 0; /* checksum */
 
-	/* start infoframe payload */
-	ptr += HDMI_INFOFRAME_HEADER_SIZE;
-
-	ptr[0] = ((frame->coding_type & 0xf) << 4) | (channels & 0x7);
-	ptr[1] = ((frame->sample_frequency & 0x7) << 2) |
-		 (frame->sample_size & 0x3);
-	ptr[2] = frame->coding_type_ext & 0x1f;
-	ptr[3] = frame->channel_allocation;
-	ptr[4] = (frame->level_shift_value & 0xf) << 3;
-
-	if (frame->downmix_inhibit)
-		ptr[4] |= BIT(7);
+	hdmi_audio_infoframe_pack_payload(frame,
+					  ptr + HDMI_INFOFRAME_HEADER_SIZE);
 
 	hdmi_infoframe_set_checksum(buffer, length);
 
@@ -480,6 +487,43 @@ ssize_t hdmi_audio_infoframe_pack(struct hdmi_audio_infoframe *frame,
 EXPORT_SYMBOL(hdmi_audio_infoframe_pack);
 
 /**
+ * hdmi_audio_infoframe_pack_for_dp - Pack a HDMI Audio infoframe for DisplayPort
+ *
+ * @frame:      HDMI Audio infoframe
+ * @sdp:        Secondary data packet for DisplayPort.
+ * @dp_version: DisplayPort version to be encoded in the header
+ *
+ * Packs a HDMI Audio Infoframe to be sent over DisplayPort. This function
+ * fills the secondary data packet to be used for DisplayPort.
+ *
+ * Return: Number of total written bytes or a negative errno on failure.
+ */
+ssize_t
+hdmi_audio_infoframe_pack_for_dp(const struct hdmi_audio_infoframe *frame,
+				 struct dp_sdp *sdp, u8 dp_version)
+{
+	int ret;
+
+	ret = hdmi_audio_infoframe_check(frame);
+	if (ret)
+		return ret;
+
+	memset(sdp->db, 0, sizeof(sdp->db));
+
+	/* Secondary-data packet header */
+	sdp->sdp_header.HB0 = 0;
+	sdp->sdp_header.HB1 = frame->type;
+	sdp->sdp_header.HB2 = DP_SDP_AUDIO_INFOFRAME_HB2;
+	sdp->sdp_header.HB3 = (dp_version & 0x3f) << 2;
+
+	hdmi_audio_infoframe_pack_payload(frame, sdp->db);
+
+	/* Return size =  frame length + four HB for sdp_header */
+	return frame->length + 4;
+}
+EXPORT_SYMBOL(hdmi_audio_infoframe_pack_for_dp);
+
+/**
  * hdmi_vendor_infoframe_init() - initialize an HDMI vendor infoframe
  * @frame: HDMI vendor infoframe
  *
diff --git a/drivers/w1/masters/ds2482.c b/drivers/w1/masters/ds2482.c
index 6c962e88501c..62c44616d8a9 100644
--- a/drivers/w1/masters/ds2482.c
+++ b/drivers/w1/masters/ds2482.c
@@ -525,7 +525,7 @@ exit:
 	return err;
 }
 
-static int ds2482_remove(struct i2c_client *client)
+static void ds2482_remove(struct i2c_client *client)
 {
 	struct ds2482_data   *data = i2c_get_clientdata(client);
 	int idx;
@@ -538,7 +538,6 @@ static int ds2482_remove(struct i2c_client *client)
 
 	/* Free the memory */
 	kfree(data);
-	return 0;
 }
 
 /*
diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig
index 9295492d24f7..688922fc4edb 100644
--- a/drivers/watchdog/Kconfig
+++ b/drivers/watchdog/Kconfig
@@ -1799,7 +1799,7 @@ config BCM7038_WDT
 	tristate "BCM63xx/BCM7038 Watchdog"
 	select WATCHDOG_CORE
 	depends on HAS_IOMEM
-	depends on ARCH_BCM4908 || ARCH_BRCMSTB || BMIPS_GENERIC || BCM63XX || COMPILE_TEST
+	depends on ARCH_BCMBCA || ARCH_BRCMSTB || BMIPS_GENERIC || BCM63XX || COMPILE_TEST
 	help
 	  Watchdog driver for the built-in hardware in Broadcom 7038 and
 	  later SoCs used in set-top boxes.  BCM7038 was made public
diff --git a/drivers/watchdog/ziirave_wdt.c b/drivers/watchdog/ziirave_wdt.c
index c5a9b820d43a..d0e88875443a 100644
--- a/drivers/watchdog/ziirave_wdt.c
+++ b/drivers/watchdog/ziirave_wdt.c
@@ -708,13 +708,11 @@ static int ziirave_wdt_probe(struct i2c_client *client,
 	return ret;
 }
 
-static int ziirave_wdt_remove(struct i2c_client *client)
+static void ziirave_wdt_remove(struct i2c_client *client)
 {
 	struct ziirave_wdt_data *w_priv = i2c_get_clientdata(client);
 
 	watchdog_unregister_device(&w_priv->wdd);
-
-	return 0;
 }
 
 static const struct i2c_device_id ziirave_wdt_id[] = {
diff --git a/fs/Kconfig.binfmt b/fs/Kconfig.binfmt
index 21e154516bf2..f14478643b91 100644
--- a/fs/Kconfig.binfmt
+++ b/fs/Kconfig.binfmt
@@ -142,39 +142,6 @@ config BINFMT_ZFLAT
 	help
 	  Support FLAT format compressed binaries
 
-config HAVE_AOUT
-       def_bool n
-
-config BINFMT_AOUT
-	tristate "Kernel support for a.out and ECOFF binaries"
-	depends on HAVE_AOUT
-	help
-	  A.out (Assembler.OUTput) is a set of formats for libraries and
-	  executables used in the earliest versions of UNIX.  Linux used
-	  the a.out formats QMAGIC and ZMAGIC until they were replaced
-	  with the ELF format.
-
-	  The conversion to ELF started in 1995.  This option is primarily
-	  provided for historical interest and for the benefit of those
-	  who need to run binaries from that era.
-
-	  Most people should answer N here.  If you think you may have
-	  occasional use for this format, enable module support above
-	  and answer M here to compile this support as a module called
-	  binfmt_aout.
-
-	  If any crucial components of your system (such as /sbin/init
-	  or /lib/ld.so) are still in a.out format, you will have to
-	  say Y here.
-
-config OSF4_COMPAT
-	bool "OSF/1 v4 readv/writev compatibility"
-	depends on ALPHA && BINFMT_AOUT
-	help
-	  Say Y if you are using OSF/1 binaries (like Netscape and Acrobat)
-	  with v4 shared libraries freely available from Compaq. If you're
-	  going to use shared libraries from Tru64 version 5.0 or later, say N.
-
 config BINFMT_MISC
 	tristate "Kernel support for MISC binaries"
 	help
diff --git a/fs/Makefile b/fs/Makefile
index 93b80529f8e8..4dea17840761 100644
--- a/fs/Makefile
+++ b/fs/Makefile
@@ -38,7 +38,6 @@ obj-$(CONFIG_FS_DAX)		+= dax.o
 obj-$(CONFIG_FS_ENCRYPTION)	+= crypto/
 obj-$(CONFIG_FS_VERITY)		+= verity/
 obj-$(CONFIG_FILE_LOCKING)      += locks.o
-obj-$(CONFIG_BINFMT_AOUT)	+= binfmt_aout.o
 obj-$(CONFIG_BINFMT_MISC)	+= binfmt_misc.o
 obj-$(CONFIG_BINFMT_SCRIPT)	+= binfmt_script.o
 obj-$(CONFIG_BINFMT_ELF)	+= binfmt_elf.o
diff --git a/fs/afs/dir.c b/fs/afs/dir.c
index 56ae5cd5184f..230c2d19116d 100644
--- a/fs/afs/dir.c
+++ b/fs/afs/dir.c
@@ -24,9 +24,9 @@ static int afs_readdir(struct file *file, struct dir_context *ctx);
 static int afs_d_revalidate(struct dentry *dentry, unsigned int flags);
 static int afs_d_delete(const struct dentry *dentry);
 static void afs_d_iput(struct dentry *dentry, struct inode *inode);
-static int afs_lookup_one_filldir(struct dir_context *ctx, const char *name, int nlen,
+static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name, int nlen,
 				  loff_t fpos, u64 ino, unsigned dtype);
-static int afs_lookup_filldir(struct dir_context *ctx, const char *name, int nlen,
+static bool afs_lookup_filldir(struct dir_context *ctx, const char *name, int nlen,
 			      loff_t fpos, u64 ino, unsigned dtype);
 static int afs_create(struct user_namespace *mnt_userns, struct inode *dir,
 		      struct dentry *dentry, umode_t mode, bool excl);
@@ -568,7 +568,7 @@ static int afs_readdir(struct file *file, struct dir_context *ctx)
  * - if afs_dir_iterate_block() spots this function, it'll pass the FID
  *   uniquifier through dtype
  */
-static int afs_lookup_one_filldir(struct dir_context *ctx, const char *name,
+static bool afs_lookup_one_filldir(struct dir_context *ctx, const char *name,
 				  int nlen, loff_t fpos, u64 ino, unsigned dtype)
 {
 	struct afs_lookup_one_cookie *cookie =
@@ -584,16 +584,16 @@ static int afs_lookup_one_filldir(struct dir_context *ctx, const char *name,
 
 	if (cookie->name.len != nlen ||
 	    memcmp(cookie->name.name, name, nlen) != 0) {
-		_leave(" = 0 [no]");
-		return 0;
+		_leave(" = true [keep looking]");
+		return true;
 	}
 
 	cookie->fid.vnode = ino;
 	cookie->fid.unique = dtype;
 	cookie->found = 1;
 
-	_leave(" = -1 [found]");
-	return -1;
+	_leave(" = false [found]");
+	return false;
 }
 
 /*
@@ -636,12 +636,11 @@ static int afs_do_lookup_one(struct inode *dir, struct dentry *dentry,
  * - if afs_dir_iterate_block() spots this function, it'll pass the FID
  *   uniquifier through dtype
  */
-static int afs_lookup_filldir(struct dir_context *ctx, const char *name,
+static bool afs_lookup_filldir(struct dir_context *ctx, const char *name,
 			      int nlen, loff_t fpos, u64 ino, unsigned dtype)
 {
 	struct afs_lookup_cookie *cookie =
 		container_of(ctx, struct afs_lookup_cookie, ctx);
-	int ret;
 
 	_enter("{%s,%u},%s,%u,,%llu,%u",
 	       cookie->name.name, cookie->name.len, name, nlen,
@@ -663,12 +662,10 @@ static int afs_lookup_filldir(struct dir_context *ctx, const char *name,
 		cookie->fids[1].unique	= dtype;
 		cookie->found = 1;
 		if (cookie->one_only)
-			return -1;
+			return false;
 	}
 
-	ret = cookie->nr_fids >= 50 ? -1 : 0;
-	_leave(" = %d", ret);
-	return ret;
+	return cookie->nr_fids < 50;
 }
 
 /*
diff --git a/fs/anon_inodes.c b/fs/anon_inodes.c
index e0c3e33c4177..24192a7667ed 100644
--- a/fs/anon_inodes.c
+++ b/fs/anon_inodes.c
@@ -32,7 +32,7 @@ static struct inode *anon_inode_inode;
  */
 static char *anon_inodefs_dname(struct dentry *dentry, char *buffer, int buflen)
 {
-	return dynamic_dname(dentry, buffer, buflen, "anon_inode:%s",
+	return dynamic_dname(buffer, buflen, "anon_inode:%s",
 				dentry->d_name.name);
 }
 
diff --git a/fs/binfmt_aout.c b/fs/binfmt_aout.c
deleted file mode 100644
index 0dcfc691e7e2..000000000000
--- a/fs/binfmt_aout.c
+++ /dev/null
@@ -1,342 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- *  linux/fs/binfmt_aout.c
- *
- *  Copyright (C) 1991, 1992, 1996  Linus Torvalds
- */
-
-#include <linux/module.h>
-
-#include <linux/time.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/mman.h>
-#include <linux/a.out.h>
-#include <linux/errno.h>
-#include <linux/signal.h>
-#include <linux/string.h>
-#include <linux/fs.h>
-#include <linux/file.h>
-#include <linux/stat.h>
-#include <linux/fcntl.h>
-#include <linux/ptrace.h>
-#include <linux/user.h>
-#include <linux/binfmts.h>
-#include <linux/personality.h>
-#include <linux/init.h>
-#include <linux/coredump.h>
-#include <linux/slab.h>
-#include <linux/sched/task_stack.h>
-
-#include <linux/uaccess.h>
-#include <asm/cacheflush.h>
-
-static int load_aout_binary(struct linux_binprm *);
-static int load_aout_library(struct file*);
-
-static struct linux_binfmt aout_format = {
-	.module		= THIS_MODULE,
-	.load_binary	= load_aout_binary,
-	.load_shlib	= load_aout_library,
-};
-
-#define BAD_ADDR(x)	((unsigned long)(x) >= TASK_SIZE)
-
-static int set_brk(unsigned long start, unsigned long end)
-{
-	start = PAGE_ALIGN(start);
-	end = PAGE_ALIGN(end);
-	if (end > start)
-		return vm_brk(start, end - start);
-	return 0;
-}
-
-/*
- * create_aout_tables() parses the env- and arg-strings in new user
- * memory and creates the pointer tables from them, and puts their
- * addresses on the "stack", returning the new stack pointer value.
- */
-static unsigned long __user *create_aout_tables(char __user *p, struct linux_binprm * bprm)
-{
-	char __user * __user *argv;
-	char __user * __user *envp;
-	unsigned long __user *sp;
-	int argc = bprm->argc;
-	int envc = bprm->envc;
-
-	sp = (void __user *)((-(unsigned long)sizeof(char *)) & (unsigned long) p);
-#ifdef __alpha__
-/* whee.. test-programs are so much fun. */
-	put_user(0, --sp);
-	put_user(0, --sp);
-	if (bprm->loader) {
-		put_user(0, --sp);
-		put_user(1003, --sp);
-		put_user(bprm->loader, --sp);
-		put_user(1002, --sp);
-	}
-	put_user(bprm->exec, --sp);
-	put_user(1001, --sp);
-#endif
-	sp -= envc+1;
-	envp = (char __user * __user *) sp;
-	sp -= argc+1;
-	argv = (char __user * __user *) sp;
-#ifndef __alpha__
-	put_user((unsigned long) envp,--sp);
-	put_user((unsigned long) argv,--sp);
-#endif
-	put_user(argc,--sp);
-	current->mm->arg_start = (unsigned long) p;
-	while (argc-->0) {
-		char c;
-		put_user(p,argv++);
-		do {
-			get_user(c,p++);
-		} while (c);
-	}
-	put_user(NULL,argv);
-	current->mm->arg_end = current->mm->env_start = (unsigned long) p;
-	while (envc-->0) {
-		char c;
-		put_user(p,envp++);
-		do {
-			get_user(c,p++);
-		} while (c);
-	}
-	put_user(NULL,envp);
-	current->mm->env_end = (unsigned long) p;
-	return sp;
-}
-
-/*
- * These are the functions used to load a.out style executables and shared
- * libraries.  There is no binary dependent code anywhere else.
- */
-
-static int load_aout_binary(struct linux_binprm * bprm)
-{
-	struct pt_regs *regs = current_pt_regs();
-	struct exec ex;
-	unsigned long error;
-	unsigned long fd_offset;
-	unsigned long rlim;
-	int retval;
-
-	ex = *((struct exec *) bprm->buf);		/* exec-header */
-	if ((N_MAGIC(ex) != ZMAGIC && N_MAGIC(ex) != OMAGIC &&
-	     N_MAGIC(ex) != QMAGIC && N_MAGIC(ex) != NMAGIC) ||
-	    N_TRSIZE(ex) || N_DRSIZE(ex) ||
-	    i_size_read(file_inode(bprm->file)) < ex.a_text+ex.a_data+N_SYMSIZE(ex)+N_TXTOFF(ex)) {
-		return -ENOEXEC;
-	}
-
-	/*
-	 * Requires a mmap handler. This prevents people from using a.out
-	 * as part of an exploit attack against /proc-related vulnerabilities.
-	 */
-	if (!bprm->file->f_op->mmap)
-		return -ENOEXEC;
-
-	fd_offset = N_TXTOFF(ex);
-
-	/* Check initial limits. This avoids letting people circumvent
-	 * size limits imposed on them by creating programs with large
-	 * arrays in the data or bss.
-	 */
-	rlim = rlimit(RLIMIT_DATA);
-	if (rlim >= RLIM_INFINITY)
-		rlim = ~0;
-	if (ex.a_data + ex.a_bss > rlim)
-		return -ENOMEM;
-
-	/* Flush all traces of the currently running executable */
-	retval = begin_new_exec(bprm);
-	if (retval)
-		return retval;
-
-	/* OK, This is the point of no return */
-#ifdef __alpha__
-	SET_AOUT_PERSONALITY(bprm, ex);
-#else
-	set_personality(PER_LINUX);
-#endif
-	setup_new_exec(bprm);
-
-	current->mm->end_code = ex.a_text +
-		(current->mm->start_code = N_TXTADDR(ex));
-	current->mm->end_data = ex.a_data +
-		(current->mm->start_data = N_DATADDR(ex));
-	current->mm->brk = ex.a_bss +
-		(current->mm->start_brk = N_BSSADDR(ex));
-
-	retval = setup_arg_pages(bprm, STACK_TOP, EXSTACK_DEFAULT);
-	if (retval < 0)
-		return retval;
-
-
-	if (N_MAGIC(ex) == OMAGIC) {
-		unsigned long text_addr, map_size;
-		loff_t pos;
-
-		text_addr = N_TXTADDR(ex);
-
-#ifdef __alpha__
-		pos = fd_offset;
-		map_size = ex.a_text+ex.a_data + PAGE_SIZE - 1;
-#else
-		pos = 32;
-		map_size = ex.a_text+ex.a_data;
-#endif
-		error = vm_brk(text_addr & PAGE_MASK, map_size);
-		if (error)
-			return error;
-
-		error = read_code(bprm->file, text_addr, pos,
-				  ex.a_text+ex.a_data);
-		if ((signed long)error < 0)
-			return error;
-	} else {
-		if ((ex.a_text & 0xfff || ex.a_data & 0xfff) &&
-		    (N_MAGIC(ex) != NMAGIC) && printk_ratelimit())
-		{
-			printk(KERN_NOTICE "executable not page aligned\n");
-		}
-
-		if ((fd_offset & ~PAGE_MASK) != 0 && printk_ratelimit())
-		{
-			printk(KERN_WARNING 
-			       "fd_offset is not page aligned. Please convert program: %pD\n",
-			       bprm->file);
-		}
-
-		if (!bprm->file->f_op->mmap||((fd_offset & ~PAGE_MASK) != 0)) {
-			error = vm_brk(N_TXTADDR(ex), ex.a_text+ex.a_data);
-			if (error)
-				return error;
-
-			read_code(bprm->file, N_TXTADDR(ex), fd_offset,
-				  ex.a_text + ex.a_data);
-			goto beyond_if;
-		}
-
-		error = vm_mmap(bprm->file, N_TXTADDR(ex), ex.a_text,
-			PROT_READ | PROT_EXEC, MAP_FIXED | MAP_PRIVATE,
-			fd_offset);
-
-		if (error != N_TXTADDR(ex))
-			return error;
-
-		error = vm_mmap(bprm->file, N_DATADDR(ex), ex.a_data,
-				PROT_READ | PROT_WRITE | PROT_EXEC,
-				MAP_FIXED | MAP_PRIVATE,
-				fd_offset + ex.a_text);
-		if (error != N_DATADDR(ex))
-			return error;
-	}
-beyond_if:
-	set_binfmt(&aout_format);
-
-	retval = set_brk(current->mm->start_brk, current->mm->brk);
-	if (retval < 0)
-		return retval;
-
-	current->mm->start_stack =
-		(unsigned long) create_aout_tables((char __user *) bprm->p, bprm);
-#ifdef __alpha__
-	regs->gp = ex.a_gpvalue;
-#endif
-	finalize_exec(bprm);
-	start_thread(regs, ex.a_entry, current->mm->start_stack);
-	return 0;
-}
-
-static int load_aout_library(struct file *file)
-{
-	struct inode * inode;
-	unsigned long bss, start_addr, len;
-	unsigned long error;
-	int retval;
-	struct exec ex;
-	loff_t pos = 0;
-
-	inode = file_inode(file);
-
-	retval = -ENOEXEC;
-	error = kernel_read(file, &ex, sizeof(ex), &pos);
-	if (error != sizeof(ex))
-		goto out;
-
-	/* We come in here for the regular a.out style of shared libraries */
-	if ((N_MAGIC(ex) != ZMAGIC && N_MAGIC(ex) != QMAGIC) || N_TRSIZE(ex) ||
-	    N_DRSIZE(ex) || ((ex.a_entry & 0xfff) && N_MAGIC(ex) == ZMAGIC) ||
-	    i_size_read(inode) < ex.a_text+ex.a_data+N_SYMSIZE(ex)+N_TXTOFF(ex)) {
-		goto out;
-	}
-
-	/*
-	 * Requires a mmap handler. This prevents people from using a.out
-	 * as part of an exploit attack against /proc-related vulnerabilities.
-	 */
-	if (!file->f_op->mmap)
-		goto out;
-
-	if (N_FLAGS(ex))
-		goto out;
-
-	/* For  QMAGIC, the starting address is 0x20 into the page.  We mask
-	   this off to get the starting address for the page */
-
-	start_addr =  ex.a_entry & 0xfffff000;
-
-	if ((N_TXTOFF(ex) & ~PAGE_MASK) != 0) {
-		if (printk_ratelimit())
-		{
-			printk(KERN_WARNING 
-			       "N_TXTOFF is not page aligned. Please convert library: %pD\n",
-			       file);
-		}
-		retval = vm_brk(start_addr, ex.a_text + ex.a_data + ex.a_bss);
-		if (retval)
-			goto out;
-
-		read_code(file, start_addr, N_TXTOFF(ex),
-			  ex.a_text + ex.a_data);
-		retval = 0;
-		goto out;
-	}
-	/* Now use mmap to map the library into memory. */
-	error = vm_mmap(file, start_addr, ex.a_text + ex.a_data,
-			PROT_READ | PROT_WRITE | PROT_EXEC,
-			MAP_FIXED | MAP_PRIVATE,
-			N_TXTOFF(ex));
-	retval = error;
-	if (error != start_addr)
-		goto out;
-
-	len = PAGE_ALIGN(ex.a_text + ex.a_data);
-	bss = ex.a_text + ex.a_data + ex.a_bss;
-	if (bss > len) {
-		retval = vm_brk(start_addr + len, bss - len);
-		if (retval)
-			goto out;
-	}
-	retval = 0;
-out:
-	return retval;
-}
-
-static int __init init_aout_binfmt(void)
-{
-	register_binfmt(&aout_format);
-	return 0;
-}
-
-static void __exit exit_aout_binfmt(void)
-{
-	unregister_binfmt(&aout_format);
-}
-
-core_initcall(init_aout_binfmt);
-module_exit(exit_aout_binfmt);
-MODULE_LICENSE("GPL");
diff --git a/fs/cifs/netlink.c b/fs/cifs/netlink.c
index 291cb606f149..147d9409252c 100644
--- a/fs/cifs/netlink.c
+++ b/fs/cifs/netlink.c
@@ -51,6 +51,7 @@ struct genl_family cifs_genl_family = {
 	.policy		= cifs_genl_policy,
 	.ops		= cifs_genl_ops,
 	.n_ops		= ARRAY_SIZE(cifs_genl_ops),
+	.resv_start_op	= CIFS_GENL_CMD_SWN_NOTIFY + 1,
 	.mcgrps		= cifs_genl_mcgrps,
 	.n_mcgrps	= ARRAY_SIZE(cifs_genl_mcgrps),
 };
diff --git a/fs/coredump.c b/fs/coredump.c
index 9f4aae202109..3538f3a63965 100644
--- a/fs/coredump.c
+++ b/fs/coredump.c
@@ -832,6 +832,39 @@ static int __dump_skip(struct coredump_params *cprm, size_t nr)
 	}
 }
 
+static int dump_emit_page(struct coredump_params *cprm, struct page *page)
+{
+	struct bio_vec bvec = {
+		.bv_page	= page,
+		.bv_offset	= 0,
+		.bv_len		= PAGE_SIZE,
+	};
+	struct iov_iter iter;
+	struct file *file = cprm->file;
+	loff_t pos;
+	ssize_t n;
+
+	if (cprm->to_skip) {
+		if (!__dump_skip(cprm, cprm->to_skip))
+			return 0;
+		cprm->to_skip = 0;
+	}
+	if (cprm->written + PAGE_SIZE > cprm->limit)
+		return 0;
+	if (dump_interrupted())
+		return 0;
+	pos = file->f_pos;
+	iov_iter_bvec(&iter, WRITE, &bvec, 1, PAGE_SIZE);
+	n = __kernel_write_iter(cprm->file, &iter, &pos);
+	if (n != PAGE_SIZE)
+		return 0;
+	file->f_pos = pos;
+	cprm->written += PAGE_SIZE;
+	cprm->pos += PAGE_SIZE;
+
+	return 1;
+}
+
 int dump_emit(struct coredump_params *cprm, const void *addr, int nr)
 {
 	if (cprm->to_skip) {
@@ -863,7 +896,6 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start,
 
 	for (addr = start; addr < start + len; addr += PAGE_SIZE) {
 		struct page *page;
-		int stop;
 
 		/*
 		 * To avoid having to allocate page tables for virtual address
@@ -874,10 +906,7 @@ int dump_user_range(struct coredump_params *cprm, unsigned long start,
 		 */
 		page = get_dump_page(addr);
 		if (page) {
-			void *kaddr = kmap_local_page(page);
-
-			stop = !dump_emit(cprm, kaddr, PAGE_SIZE);
-			kunmap_local(kaddr);
+			int stop = !dump_emit_page(cprm, page);
 			put_page(page);
 			if (stop)
 				return 0;
diff --git a/fs/crypto/bio.c b/fs/crypto/bio.c
index 2217fe5ece6f..1b4403136d05 100644
--- a/fs/crypto/bio.c
+++ b/fs/crypto/bio.c
@@ -25,21 +25,25 @@
  * then this function isn't applicable.  This function may sleep, so it must be
  * called from a workqueue rather than from the bio's bi_end_io callback.
  *
- * This function sets PG_error on any pages that contain any blocks that failed
- * to be decrypted.  The filesystem must not mark such pages uptodate.
+ * Return: %true on success; %false on failure.  On failure, bio->bi_status is
+ *	   also set to an error status.
  */
-void fscrypt_decrypt_bio(struct bio *bio)
+bool fscrypt_decrypt_bio(struct bio *bio)
 {
 	struct bio_vec *bv;
 	struct bvec_iter_all iter_all;
 
 	bio_for_each_segment_all(bv, bio, iter_all) {
 		struct page *page = bv->bv_page;
-		int ret = fscrypt_decrypt_pagecache_blocks(page, bv->bv_len,
+		int err = fscrypt_decrypt_pagecache_blocks(page, bv->bv_len,
 							   bv->bv_offset);
-		if (ret)
-			SetPageError(page);
+
+		if (err) {
+			bio->bi_status = errno_to_blk_status(err);
+			return false;
+		}
 	}
+	return true;
 }
 EXPORT_SYMBOL(fscrypt_decrypt_bio);
 
diff --git a/fs/crypto/fscrypt_private.h b/fs/crypto/fscrypt_private.h
index 3afdaa084773..d5f68a0c5d15 100644
--- a/fs/crypto/fscrypt_private.h
+++ b/fs/crypto/fscrypt_private.h
@@ -184,7 +184,7 @@ struct fscrypt_symlink_data {
 struct fscrypt_prepared_key {
 	struct crypto_skcipher *tfm;
 #ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT
-	struct fscrypt_blk_crypto_key *blk_key;
+	struct blk_crypto_key *blk_key;
 #endif
 };
 
@@ -225,7 +225,7 @@ struct fscrypt_info {
 	 * will be NULL if the master key was found in a process-subscribed
 	 * keyring rather than in the filesystem-level keyring.
 	 */
-	struct key *ci_master_key;
+	struct fscrypt_master_key *ci_master_key;
 
 	/*
 	 * Link in list of inodes that were unlocked with the master key.
@@ -344,7 +344,8 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
 				     const u8 *raw_key,
 				     const struct fscrypt_info *ci);
 
-void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key);
+void fscrypt_destroy_inline_crypt_key(struct super_block *sb,
+				      struct fscrypt_prepared_key *prep_key);
 
 /*
  * Check whether the crypto transform or blk-crypto key has been allocated in
@@ -390,7 +391,8 @@ fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
 }
 
 static inline void
-fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key)
+fscrypt_destroy_inline_crypt_key(struct super_block *sb,
+				 struct fscrypt_prepared_key *prep_key)
 {
 }
 
@@ -437,6 +439,40 @@ struct fscrypt_master_key_secret {
 struct fscrypt_master_key {
 
 	/*
+	 * Back-pointer to the super_block of the filesystem to which this
+	 * master key has been added.  Only valid if ->mk_active_refs > 0.
+	 */
+	struct super_block			*mk_sb;
+
+	/*
+	 * Link in ->mk_sb->s_master_keys->key_hashtable.
+	 * Only valid if ->mk_active_refs > 0.
+	 */
+	struct hlist_node			mk_node;
+
+	/* Semaphore that protects ->mk_secret and ->mk_users */
+	struct rw_semaphore			mk_sem;
+
+	/*
+	 * Active and structural reference counts.  An active ref guarantees
+	 * that the struct continues to exist, continues to be in the keyring
+	 * ->mk_sb->s_master_keys, and that any embedded subkeys (e.g.
+	 * ->mk_direct_keys) that have been prepared continue to exist.
+	 * A structural ref only guarantees that the struct continues to exist.
+	 *
+	 * There is one active ref associated with ->mk_secret being present,
+	 * and one active ref for each inode in ->mk_decrypted_inodes.
+	 *
+	 * There is one structural ref associated with the active refcount being
+	 * nonzero.  Finding a key in the keyring also takes a structural ref,
+	 * which is then held temporarily while the key is operated on.
+	 */
+	refcount_t				mk_active_refs;
+	refcount_t				mk_struct_refs;
+
+	struct rcu_head				mk_rcu_head;
+
+	/*
 	 * The secret key material.  After FS_IOC_REMOVE_ENCRYPTION_KEY is
 	 * executed, this is wiped and no new inodes can be unlocked with this
 	 * key; however, there may still be inodes in ->mk_decrypted_inodes
@@ -444,7 +480,10 @@ struct fscrypt_master_key {
 	 * FS_IOC_REMOVE_ENCRYPTION_KEY can be retried, or
 	 * FS_IOC_ADD_ENCRYPTION_KEY can add the secret again.
 	 *
-	 * Locking: protected by this master key's key->sem.
+	 * While ->mk_secret is present, one ref in ->mk_active_refs is held.
+	 *
+	 * Locking: protected by ->mk_sem.  The manipulation of ->mk_active_refs
+	 *	    associated with this field is protected by ->mk_sem as well.
 	 */
 	struct fscrypt_master_key_secret	mk_secret;
 
@@ -465,23 +504,13 @@ struct fscrypt_master_key {
 	 *
 	 * This is NULL for v1 policy keys; those can only be added by root.
 	 *
-	 * Locking: in addition to this keyring's own semaphore, this is
-	 * protected by this master key's key->sem, so we can do atomic
-	 * search+insert.  It can also be searched without taking any locks, but
-	 * in that case the returned key may have already been removed.
+	 * Locking: protected by ->mk_sem.  (We don't just rely on the keyrings
+	 * subsystem semaphore ->mk_users->sem, as we need support for atomic
+	 * search+insert along with proper synchronization with ->mk_secret.)
 	 */
 	struct key		*mk_users;
 
 	/*
-	 * Length of ->mk_decrypted_inodes, plus one if mk_secret is present.
-	 * Once this goes to 0, the master key is removed from ->s_master_keys.
-	 * The 'struct fscrypt_master_key' will continue to live as long as the
-	 * 'struct key' whose payload it is, but we won't let this reference
-	 * count rise again.
-	 */
-	refcount_t		mk_refcount;
-
-	/*
 	 * List of inodes that were unlocked using this key.  This allows the
 	 * inodes to be evicted efficiently if the key is removed.
 	 */
@@ -506,10 +535,10 @@ static inline bool
 is_master_key_secret_present(const struct fscrypt_master_key_secret *secret)
 {
 	/*
-	 * The READ_ONCE() is only necessary for fscrypt_drop_inode() and
-	 * fscrypt_key_describe().  These run in atomic context, so they can't
-	 * take the key semaphore and thus 'secret' can change concurrently
-	 * which would be a data race.  But they only need to know whether the
+	 * The READ_ONCE() is only necessary for fscrypt_drop_inode().
+	 * fscrypt_drop_inode() runs in atomic context, so it can't take the key
+	 * semaphore and thus 'secret' can change concurrently which would be a
+	 * data race.  But fscrypt_drop_inode() only need to know whether the
 	 * secret *was* present at the time of check, so READ_ONCE() suffices.
 	 */
 	return READ_ONCE(secret->size) != 0;
@@ -538,7 +567,11 @@ static inline int master_key_spec_len(const struct fscrypt_key_specifier *spec)
 	return 0;
 }
 
-struct key *
+void fscrypt_put_master_key(struct fscrypt_master_key *mk);
+
+void fscrypt_put_master_key_activeref(struct fscrypt_master_key *mk);
+
+struct fscrypt_master_key *
 fscrypt_find_master_key(struct super_block *sb,
 			const struct fscrypt_key_specifier *mk_spec);
 
@@ -569,7 +602,8 @@ extern struct fscrypt_mode fscrypt_modes[];
 int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key,
 			const u8 *raw_key, const struct fscrypt_info *ci);
 
-void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key);
+void fscrypt_destroy_prepared_key(struct super_block *sb,
+				  struct fscrypt_prepared_key *prep_key);
 
 int fscrypt_set_per_file_enc_key(struct fscrypt_info *ci, const u8 *raw_key);
 
diff --git a/fs/crypto/hooks.c b/fs/crypto/hooks.c
index 7c01025879b3..7b8c5a1104b5 100644
--- a/fs/crypto/hooks.c
+++ b/fs/crypto/hooks.c
@@ -5,8 +5,6 @@
  * Encryption hooks for higher-level filesystem operations.
  */
 
-#include <linux/key.h>
-
 #include "fscrypt_private.h"
 
 /**
@@ -142,7 +140,6 @@ int fscrypt_prepare_setflags(struct inode *inode,
 			     unsigned int oldflags, unsigned int flags)
 {
 	struct fscrypt_info *ci;
-	struct key *key;
 	struct fscrypt_master_key *mk;
 	int err;
 
@@ -158,14 +155,13 @@ int fscrypt_prepare_setflags(struct inode *inode,
 		ci = inode->i_crypt_info;
 		if (ci->ci_policy.version != FSCRYPT_POLICY_V2)
 			return -EINVAL;
-		key = ci->ci_master_key;
-		mk = key->payload.data[0];
-		down_read(&key->sem);
+		mk = ci->ci_master_key;
+		down_read(&mk->mk_sem);
 		if (is_master_key_secret_present(&mk->mk_secret))
 			err = fscrypt_derive_dirhash_key(ci, mk);
 		else
 			err = -ENOKEY;
-		up_read(&key->sem);
+		up_read(&mk->mk_sem);
 		return err;
 	}
 	return 0;
diff --git a/fs/crypto/inline_crypt.c b/fs/crypto/inline_crypt.c
index 90f3e68f166e..cea8b14007e6 100644
--- a/fs/crypto/inline_crypt.c
+++ b/fs/crypto/inline_crypt.c
@@ -21,26 +21,22 @@
 
 #include "fscrypt_private.h"
 
-struct fscrypt_blk_crypto_key {
-	struct blk_crypto_key base;
-	int num_devs;
-	struct request_queue *devs[];
-};
-
-static int fscrypt_get_num_devices(struct super_block *sb)
+static struct block_device **fscrypt_get_devices(struct super_block *sb,
+						 unsigned int *num_devs)
 {
-	if (sb->s_cop->get_num_devices)
-		return sb->s_cop->get_num_devices(sb);
-	return 1;
-}
+	struct block_device **devs;
 
-static void fscrypt_get_devices(struct super_block *sb, int num_devs,
-				struct request_queue **devs)
-{
-	if (num_devs == 1)
-		devs[0] = bdev_get_queue(sb->s_bdev);
-	else
-		sb->s_cop->get_devices(sb, devs);
+	if (sb->s_cop->get_devices) {
+		devs = sb->s_cop->get_devices(sb, num_devs);
+		if (devs)
+			return devs;
+	}
+	devs = kmalloc(sizeof(*devs), GFP_KERNEL);
+	if (!devs)
+		return ERR_PTR(-ENOMEM);
+	devs[0] = sb->s_bdev;
+	*num_devs = 1;
+	return devs;
 }
 
 static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_info *ci)
@@ -74,15 +70,17 @@ static unsigned int fscrypt_get_dun_bytes(const struct fscrypt_info *ci)
  * helpful for debugging problems where the "wrong" implementation is used.
  */
 static void fscrypt_log_blk_crypto_impl(struct fscrypt_mode *mode,
-					struct request_queue **devs,
-					int num_devs,
+					struct block_device **devs,
+					unsigned int num_devs,
 					const struct blk_crypto_config *cfg)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < num_devs; i++) {
+		struct request_queue *q = bdev_get_queue(devs[i]);
+
 		if (!IS_ENABLED(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) ||
-		    __blk_crypto_cfg_supported(devs[i]->crypto_profile, cfg)) {
+		    __blk_crypto_cfg_supported(q->crypto_profile, cfg)) {
 			if (!xchg(&mode->logged_blk_crypto_native, 1))
 				pr_info("fscrypt: %s using blk-crypto (native)\n",
 					mode->friendly_name);
@@ -99,9 +97,9 @@ int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
 	const struct inode *inode = ci->ci_inode;
 	struct super_block *sb = inode->i_sb;
 	struct blk_crypto_config crypto_cfg;
-	int num_devs;
-	struct request_queue **devs;
-	int i;
+	struct block_device **devs;
+	unsigned int num_devs;
+	unsigned int i;
 
 	/* The file must need contents encryption, not filenames encryption */
 	if (!S_ISREG(inode->i_mode))
@@ -129,20 +127,20 @@ int fscrypt_select_encryption_impl(struct fscrypt_info *ci)
 		return 0;
 
 	/*
-	 * On all the filesystem's devices, blk-crypto must support the crypto
-	 * configuration that the file would use.
+	 * On all the filesystem's block devices, blk-crypto must support the
+	 * crypto configuration that the file would use.
 	 */
 	crypto_cfg.crypto_mode = ci->ci_mode->blk_crypto_mode;
 	crypto_cfg.data_unit_size = sb->s_blocksize;
 	crypto_cfg.dun_bytes = fscrypt_get_dun_bytes(ci);
-	num_devs = fscrypt_get_num_devices(sb);
-	devs = kmalloc_array(num_devs, sizeof(*devs), GFP_KERNEL);
-	if (!devs)
-		return -ENOMEM;
-	fscrypt_get_devices(sb, num_devs, devs);
+
+	devs = fscrypt_get_devices(sb, &num_devs);
+	if (IS_ERR(devs))
+		return PTR_ERR(devs);
 
 	for (i = 0; i < num_devs; i++) {
-		if (!blk_crypto_config_supported(devs[i], &crypto_cfg))
+		if (!blk_crypto_config_supported(bdev_get_queue(devs[i]),
+						 &crypto_cfg))
 			goto out_free_devs;
 	}
 
@@ -162,49 +160,41 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
 	const struct inode *inode = ci->ci_inode;
 	struct super_block *sb = inode->i_sb;
 	enum blk_crypto_mode_num crypto_mode = ci->ci_mode->blk_crypto_mode;
-	int num_devs = fscrypt_get_num_devices(sb);
-	int queue_refs = 0;
-	struct fscrypt_blk_crypto_key *blk_key;
+	struct blk_crypto_key *blk_key;
+	struct block_device **devs;
+	unsigned int num_devs;
+	unsigned int i;
 	int err;
-	int i;
 
-	blk_key = kzalloc(struct_size(blk_key, devs, num_devs), GFP_KERNEL);
+	blk_key = kmalloc(sizeof(*blk_key), GFP_KERNEL);
 	if (!blk_key)
 		return -ENOMEM;
 
-	blk_key->num_devs = num_devs;
-	fscrypt_get_devices(sb, num_devs, blk_key->devs);
-
-	err = blk_crypto_init_key(&blk_key->base, raw_key, crypto_mode,
+	err = blk_crypto_init_key(blk_key, raw_key, crypto_mode,
 				  fscrypt_get_dun_bytes(ci), sb->s_blocksize);
 	if (err) {
 		fscrypt_err(inode, "error %d initializing blk-crypto key", err);
 		goto fail;
 	}
 
-	/*
-	 * We have to start using blk-crypto on all the filesystem's devices.
-	 * We also have to save all the request_queue's for later so that the
-	 * key can be evicted from them.  This is needed because some keys
-	 * aren't destroyed until after the filesystem was already unmounted
-	 * (namely, the per-mode keys in struct fscrypt_master_key).
-	 */
+	/* Start using blk-crypto on all the filesystem's block devices. */
+	devs = fscrypt_get_devices(sb, &num_devs);
+	if (IS_ERR(devs)) {
+		err = PTR_ERR(devs);
+		goto fail;
+	}
 	for (i = 0; i < num_devs; i++) {
-		if (!blk_get_queue(blk_key->devs[i])) {
-			fscrypt_err(inode, "couldn't get request_queue");
-			err = -EAGAIN;
-			goto fail;
-		}
-		queue_refs++;
-
-		err = blk_crypto_start_using_key(&blk_key->base,
-						 blk_key->devs[i]);
-		if (err) {
-			fscrypt_err(inode,
-				    "error %d starting to use blk-crypto", err);
-			goto fail;
-		}
+		err = blk_crypto_start_using_key(blk_key,
+						 bdev_get_queue(devs[i]));
+		if (err)
+			break;
 	}
+	kfree(devs);
+	if (err) {
+		fscrypt_err(inode, "error %d starting to use blk-crypto", err);
+		goto fail;
+	}
+
 	/*
 	 * Pairs with the smp_load_acquire() in fscrypt_is_key_prepared().
 	 * I.e., here we publish ->blk_key with a RELEASE barrier so that
@@ -215,24 +205,29 @@ int fscrypt_prepare_inline_crypt_key(struct fscrypt_prepared_key *prep_key,
 	return 0;
 
 fail:
-	for (i = 0; i < queue_refs; i++)
-		blk_put_queue(blk_key->devs[i]);
 	kfree_sensitive(blk_key);
 	return err;
 }
 
-void fscrypt_destroy_inline_crypt_key(struct fscrypt_prepared_key *prep_key)
+void fscrypt_destroy_inline_crypt_key(struct super_block *sb,
+				      struct fscrypt_prepared_key *prep_key)
 {
-	struct fscrypt_blk_crypto_key *blk_key = prep_key->blk_key;
-	int i;
+	struct blk_crypto_key *blk_key = prep_key->blk_key;
+	struct block_device **devs;
+	unsigned int num_devs;
+	unsigned int i;
 
-	if (blk_key) {
-		for (i = 0; i < blk_key->num_devs; i++) {
-			blk_crypto_evict_key(blk_key->devs[i], &blk_key->base);
-			blk_put_queue(blk_key->devs[i]);
-		}
-		kfree_sensitive(blk_key);
+	if (!blk_key)
+		return;
+
+	/* Evict the key from all the filesystem's block devices. */
+	devs = fscrypt_get_devices(sb, &num_devs);
+	if (!IS_ERR(devs)) {
+		for (i = 0; i < num_devs; i++)
+			blk_crypto_evict_key(bdev_get_queue(devs[i]), blk_key);
+		kfree(devs);
 	}
+	kfree_sensitive(blk_key);
 }
 
 bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode)
@@ -282,7 +277,7 @@ void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode,
 	ci = inode->i_crypt_info;
 
 	fscrypt_generate_dun(ci, first_lblk, dun);
-	bio_crypt_set_ctx(bio, &ci->ci_enc_key.blk_key->base, dun, gfp_mask);
+	bio_crypt_set_ctx(bio, ci->ci_enc_key.blk_key, dun, gfp_mask);
 }
 EXPORT_SYMBOL_GPL(fscrypt_set_bio_crypt_ctx);
 
@@ -369,7 +364,7 @@ bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode,
 	 * uses the same pointer.  I.e., there's currently no need to support
 	 * merging requests where the keys are the same but the pointers differ.
 	 */
-	if (bc->bc_key != &inode->i_crypt_info->ci_enc_key.blk_key->base)
+	if (bc->bc_key != inode->i_crypt_info->ci_enc_key.blk_key)
 		return false;
 
 	fscrypt_generate_dun(inode->i_crypt_info, next_lblk, next_dun);
@@ -401,46 +396,45 @@ bool fscrypt_mergeable_bio_bh(struct bio *bio,
 EXPORT_SYMBOL_GPL(fscrypt_mergeable_bio_bh);
 
 /**
- * fscrypt_dio_supported() - check whether a DIO (direct I/O) request is
- *			     supported as far as encryption is concerned
- * @iocb: the file and position the I/O is targeting
- * @iter: the I/O data segment(s)
+ * fscrypt_dio_supported() - check whether DIO (direct I/O) is supported on an
+ *			     inode, as far as encryption is concerned
+ * @inode: the inode in question
  *
  * Return: %true if there are no encryption constraints that prevent DIO from
  *	   being supported; %false if DIO is unsupported.  (Note that in the
  *	   %true case, the filesystem might have other, non-encryption-related
- *	   constraints that prevent DIO from actually being supported.)
+ *	   constraints that prevent DIO from actually being supported.  Also, on
+ *	   encrypted files the filesystem is still responsible for only allowing
+ *	   DIO when requests are filesystem-block-aligned.)
  */
-bool fscrypt_dio_supported(struct kiocb *iocb, struct iov_iter *iter)
+bool fscrypt_dio_supported(struct inode *inode)
 {
-	const struct inode *inode = file_inode(iocb->ki_filp);
-	const unsigned int blocksize = i_blocksize(inode);
+	int err;
 
 	/* If the file is unencrypted, no veto from us. */
 	if (!fscrypt_needs_contents_encryption(inode))
 		return true;
 
-	/* We only support DIO with inline crypto, not fs-layer crypto. */
-	if (!fscrypt_inode_uses_inline_crypto(inode))
-		return false;
-
 	/*
-	 * Since the granularity of encryption is filesystem blocks, the file
-	 * position and total I/O length must be aligned to the filesystem block
-	 * size -- not just to the block device's logical block size as is
-	 * traditionally the case for DIO on many filesystems.
+	 * We only support DIO with inline crypto, not fs-layer crypto.
 	 *
-	 * We require that the user-provided memory buffers be filesystem block
-	 * aligned too.  It is simpler to have a single alignment value required
-	 * for all properties of the I/O, as is normally the case for DIO.
-	 * Also, allowing less aligned buffers would imply that data units could
-	 * cross bvecs, which would greatly complicate the I/O stack, which
-	 * assumes that bios can be split at any bvec boundary.
+	 * To determine whether the inode is using inline crypto, we have to set
+	 * up the key if it wasn't already done.  This is because in the current
+	 * design of fscrypt, the decision of whether to use inline crypto or
+	 * not isn't made until the inode's encryption key is being set up.  In
+	 * the DIO read/write case, the key will always be set up already, since
+	 * the file will be open.  But in the case of statx(), the key might not
+	 * be set up yet, as the file might not have been opened yet.
 	 */
-	if (!IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), blocksize))
+	err = fscrypt_require_key(inode);
+	if (err) {
+		/*
+		 * Key unavailable or couldn't be set up.  This edge case isn't
+		 * worth worrying about; just report that DIO is unsupported.
+		 */
 		return false;
-
-	return true;
+	}
+	return fscrypt_inode_uses_inline_crypto(inode);
 }
 EXPORT_SYMBOL_GPL(fscrypt_dio_supported);
 
diff --git a/fs/crypto/keyring.c b/fs/crypto/keyring.c
index caee9f8620dd..1cca09aa43f8 100644
--- a/fs/crypto/keyring.c
+++ b/fs/crypto/keyring.c
@@ -18,6 +18,7 @@
  * information about these ioctls.
  */
 
+#include <asm/unaligned.h>
 #include <crypto/skcipher.h>
 #include <linux/key-type.h>
 #include <linux/random.h>
@@ -25,6 +26,18 @@
 
 #include "fscrypt_private.h"
 
+/* The master encryption keys for a filesystem (->s_master_keys) */
+struct fscrypt_keyring {
+	/*
+	 * Lock that protects ->key_hashtable.  It does *not* protect the
+	 * fscrypt_master_key structs themselves.
+	 */
+	spinlock_t lock;
+
+	/* Hash table that maps fscrypt_key_specifier to fscrypt_master_key */
+	struct hlist_head key_hashtable[128];
+};
+
 static void wipe_master_key_secret(struct fscrypt_master_key_secret *secret)
 {
 	fscrypt_destroy_hkdf(&secret->hkdf);
@@ -38,66 +51,81 @@ static void move_master_key_secret(struct fscrypt_master_key_secret *dst,
 	memzero_explicit(src, sizeof(*src));
 }
 
-static void free_master_key(struct fscrypt_master_key *mk)
+static void fscrypt_free_master_key(struct rcu_head *head)
 {
-	size_t i;
-
-	wipe_master_key_secret(&mk->mk_secret);
-
-	for (i = 0; i <= FSCRYPT_MODE_MAX; i++) {
-		fscrypt_destroy_prepared_key(&mk->mk_direct_keys[i]);
-		fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_64_keys[i]);
-		fscrypt_destroy_prepared_key(&mk->mk_iv_ino_lblk_32_keys[i]);
-	}
-
-	key_put(mk->mk_users);
+	struct fscrypt_master_key *mk =
+		container_of(head, struct fscrypt_master_key, mk_rcu_head);
+	/*
+	 * The master key secret and any embedded subkeys should have already
+	 * been wiped when the last active reference to the fscrypt_master_key
+	 * struct was dropped; doing it here would be unnecessarily late.
+	 * Nevertheless, use kfree_sensitive() in case anything was missed.
+	 */
 	kfree_sensitive(mk);
 }
 
-static inline bool valid_key_spec(const struct fscrypt_key_specifier *spec)
+void fscrypt_put_master_key(struct fscrypt_master_key *mk)
 {
-	if (spec->__reserved)
-		return false;
-	return master_key_spec_len(spec) != 0;
+	if (!refcount_dec_and_test(&mk->mk_struct_refs))
+		return;
+	/*
+	 * No structural references left, so free ->mk_users, and also free the
+	 * fscrypt_master_key struct itself after an RCU grace period ensures
+	 * that concurrent keyring lookups can no longer find it.
+	 */
+	WARN_ON(refcount_read(&mk->mk_active_refs) != 0);
+	key_put(mk->mk_users);
+	mk->mk_users = NULL;
+	call_rcu(&mk->mk_rcu_head, fscrypt_free_master_key);
 }
 
-static int fscrypt_key_instantiate(struct key *key,
-				   struct key_preparsed_payload *prep)
+void fscrypt_put_master_key_activeref(struct fscrypt_master_key *mk)
 {
-	key->payload.data[0] = (struct fscrypt_master_key *)prep->data;
-	return 0;
-}
+	struct super_block *sb = mk->mk_sb;
+	struct fscrypt_keyring *keyring = sb->s_master_keys;
+	size_t i;
 
-static void fscrypt_key_destroy(struct key *key)
-{
-	free_master_key(key->payload.data[0]);
-}
+	if (!refcount_dec_and_test(&mk->mk_active_refs))
+		return;
+	/*
+	 * No active references left, so complete the full removal of this
+	 * fscrypt_master_key struct by removing it from the keyring and
+	 * destroying any subkeys embedded in it.
+	 */
 
-static void fscrypt_key_describe(const struct key *key, struct seq_file *m)
-{
-	seq_puts(m, key->description);
+	spin_lock(&keyring->lock);
+	hlist_del_rcu(&mk->mk_node);
+	spin_unlock(&keyring->lock);
 
-	if (key_is_positive(key)) {
-		const struct fscrypt_master_key *mk = key->payload.data[0];
+	/*
+	 * ->mk_active_refs == 0 implies that ->mk_secret is not present and
+	 * that ->mk_decrypted_inodes is empty.
+	 */
+	WARN_ON(is_master_key_secret_present(&mk->mk_secret));
+	WARN_ON(!list_empty(&mk->mk_decrypted_inodes));
 
-		if (!is_master_key_secret_present(&mk->mk_secret))
-			seq_puts(m, ": secret removed");
+	for (i = 0; i <= FSCRYPT_MODE_MAX; i++) {
+		fscrypt_destroy_prepared_key(
+				sb, &mk->mk_direct_keys[i]);
+		fscrypt_destroy_prepared_key(
+				sb, &mk->mk_iv_ino_lblk_64_keys[i]);
+		fscrypt_destroy_prepared_key(
+				sb, &mk->mk_iv_ino_lblk_32_keys[i]);
 	}
+	memzero_explicit(&mk->mk_ino_hash_key,
+			 sizeof(mk->mk_ino_hash_key));
+	mk->mk_ino_hash_key_initialized = false;
+
+	/* Drop the structural ref associated with the active refs. */
+	fscrypt_put_master_key(mk);
 }
 
-/*
- * Type of key in ->s_master_keys.  Each key of this type represents a master
- * key which has been added to the filesystem.  Its payload is a
- * 'struct fscrypt_master_key'.  The "." prefix in the key type name prevents
- * users from adding keys of this type via the keyrings syscalls rather than via
- * the intended method of FS_IOC_ADD_ENCRYPTION_KEY.
- */
-static struct key_type key_type_fscrypt = {
-	.name			= "._fscrypt",
-	.instantiate		= fscrypt_key_instantiate,
-	.destroy		= fscrypt_key_destroy,
-	.describe		= fscrypt_key_describe,
-};
+static inline bool valid_key_spec(const struct fscrypt_key_specifier *spec)
+{
+	if (spec->__reserved)
+		return false;
+	return master_key_spec_len(spec) != 0;
+}
 
 static int fscrypt_user_key_instantiate(struct key *key,
 					struct key_preparsed_payload *prep)
@@ -131,32 +159,6 @@ static struct key_type key_type_fscrypt_user = {
 	.describe		= fscrypt_user_key_describe,
 };
 
-/* Search ->s_master_keys or ->mk_users */
-static struct key *search_fscrypt_keyring(struct key *keyring,
-					  struct key_type *type,
-					  const char *description)
-{
-	/*
-	 * We need to mark the keyring reference as "possessed" so that we
-	 * acquire permission to search it, via the KEY_POS_SEARCH permission.
-	 */
-	key_ref_t keyref = make_key_ref(keyring, true /* possessed */);
-
-	keyref = keyring_search(keyref, type, description, false);
-	if (IS_ERR(keyref)) {
-		if (PTR_ERR(keyref) == -EAGAIN || /* not found */
-		    PTR_ERR(keyref) == -EKEYREVOKED) /* recently invalidated */
-			keyref = ERR_PTR(-ENOKEY);
-		return ERR_CAST(keyref);
-	}
-	return key_ref_to_ptr(keyref);
-}
-
-#define FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE	\
-	(CONST_STRLEN("fscrypt-") + sizeof_field(struct super_block, s_id))
-
-#define FSCRYPT_MK_DESCRIPTION_SIZE	(2 * FSCRYPT_KEY_IDENTIFIER_SIZE + 1)
-
 #define FSCRYPT_MK_USERS_DESCRIPTION_SIZE	\
 	(CONST_STRLEN("fscrypt-") + 2 * FSCRYPT_KEY_IDENTIFIER_SIZE + \
 	 CONST_STRLEN("-users") + 1)
@@ -164,21 +166,6 @@ static struct key *search_fscrypt_keyring(struct key *keyring,
 #define FSCRYPT_MK_USER_DESCRIPTION_SIZE	\
 	(2 * FSCRYPT_KEY_IDENTIFIER_SIZE + CONST_STRLEN(".uid.") + 10 + 1)
 
-static void format_fs_keyring_description(
-			char description[FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE],
-			const struct super_block *sb)
-{
-	sprintf(description, "fscrypt-%s", sb->s_id);
-}
-
-static void format_mk_description(
-			char description[FSCRYPT_MK_DESCRIPTION_SIZE],
-			const struct fscrypt_key_specifier *mk_spec)
-{
-	sprintf(description, "%*phN",
-		master_key_spec_len(mk_spec), (u8 *)&mk_spec->u);
-}
-
 static void format_mk_users_keyring_description(
 			char description[FSCRYPT_MK_USERS_DESCRIPTION_SIZE],
 			const u8 mk_identifier[FSCRYPT_KEY_IDENTIFIER_SIZE])
@@ -199,20 +186,15 @@ static void format_mk_user_description(
 /* Create ->s_master_keys if needed.  Synchronized by fscrypt_add_key_mutex. */
 static int allocate_filesystem_keyring(struct super_block *sb)
 {
-	char description[FSCRYPT_FS_KEYRING_DESCRIPTION_SIZE];
-	struct key *keyring;
+	struct fscrypt_keyring *keyring;
 
 	if (sb->s_master_keys)
 		return 0;
 
-	format_fs_keyring_description(description, sb);
-	keyring = keyring_alloc(description, GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
-				current_cred(), KEY_POS_SEARCH |
-				  KEY_USR_SEARCH | KEY_USR_READ | KEY_USR_VIEW,
-				KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL);
-	if (IS_ERR(keyring))
-		return PTR_ERR(keyring);
-
+	keyring = kzalloc(sizeof(*keyring), GFP_KERNEL);
+	if (!keyring)
+		return -ENOMEM;
+	spin_lock_init(&keyring->lock);
 	/*
 	 * Pairs with the smp_load_acquire() in fscrypt_find_master_key().
 	 * I.e., here we publish ->s_master_keys with a RELEASE barrier so that
@@ -222,21 +204,75 @@ static int allocate_filesystem_keyring(struct super_block *sb)
 	return 0;
 }
 
-void fscrypt_sb_free(struct super_block *sb)
+/*
+ * This is called at unmount time to release all encryption keys that have been
+ * added to the filesystem, along with the keyring that contains them.
+ *
+ * Note that besides clearing and freeing memory, this might need to evict keys
+ * from the keyslots of an inline crypto engine.  Therefore, this must be called
+ * while the filesystem's underlying block device(s) are still available.
+ */
+void fscrypt_sb_delete(struct super_block *sb)
 {
-	key_put(sb->s_master_keys);
+	struct fscrypt_keyring *keyring = sb->s_master_keys;
+	size_t i;
+
+	if (!keyring)
+		return;
+
+	for (i = 0; i < ARRAY_SIZE(keyring->key_hashtable); i++) {
+		struct hlist_head *bucket = &keyring->key_hashtable[i];
+		struct fscrypt_master_key *mk;
+		struct hlist_node *tmp;
+
+		hlist_for_each_entry_safe(mk, tmp, bucket, mk_node) {
+			/*
+			 * Since all inodes were already evicted, every key
+			 * remaining in the keyring should have an empty inode
+			 * list, and should only still be in the keyring due to
+			 * the single active ref associated with ->mk_secret.
+			 * There should be no structural refs beyond the one
+			 * associated with the active ref.
+			 */
+			WARN_ON(refcount_read(&mk->mk_active_refs) != 1);
+			WARN_ON(refcount_read(&mk->mk_struct_refs) != 1);
+			WARN_ON(!is_master_key_secret_present(&mk->mk_secret));
+			wipe_master_key_secret(&mk->mk_secret);
+			fscrypt_put_master_key_activeref(mk);
+		}
+	}
+	kfree_sensitive(keyring);
 	sb->s_master_keys = NULL;
 }
 
+static struct hlist_head *
+fscrypt_mk_hash_bucket(struct fscrypt_keyring *keyring,
+		       const struct fscrypt_key_specifier *mk_spec)
+{
+	/*
+	 * Since key specifiers should be "random" values, it is sufficient to
+	 * use a trivial hash function that just takes the first several bits of
+	 * the key specifier.
+	 */
+	unsigned long i = get_unaligned((unsigned long *)&mk_spec->u);
+
+	return &keyring->key_hashtable[i % ARRAY_SIZE(keyring->key_hashtable)];
+}
+
 /*
- * Find the specified master key in ->s_master_keys.
- * Returns ERR_PTR(-ENOKEY) if not found.
+ * Find the specified master key struct in ->s_master_keys and take a structural
+ * ref to it.  The structural ref guarantees that the key struct continues to
+ * exist, but it does *not* guarantee that ->s_master_keys continues to contain
+ * the key struct.  The structural ref needs to be dropped by
+ * fscrypt_put_master_key().  Returns NULL if the key struct is not found.
  */
-struct key *fscrypt_find_master_key(struct super_block *sb,
-				    const struct fscrypt_key_specifier *mk_spec)
+struct fscrypt_master_key *
+fscrypt_find_master_key(struct super_block *sb,
+			const struct fscrypt_key_specifier *mk_spec)
 {
-	struct key *keyring;
-	char description[FSCRYPT_MK_DESCRIPTION_SIZE];
+	struct fscrypt_keyring *keyring;
+	struct hlist_head *bucket;
+	struct fscrypt_master_key *mk;
 
 	/*
 	 * Pairs with the smp_store_release() in allocate_filesystem_keyring().
@@ -246,10 +282,38 @@ struct key *fscrypt_find_master_key(struct super_block *sb,
 	 */
 	keyring = smp_load_acquire(&sb->s_master_keys);
 	if (keyring == NULL)
-		return ERR_PTR(-ENOKEY); /* No keyring yet, so no keys yet. */
-
-	format_mk_description(description, mk_spec);
-	return search_fscrypt_keyring(keyring, &key_type_fscrypt, description);
+		return NULL; /* No keyring yet, so no keys yet. */
+
+	bucket = fscrypt_mk_hash_bucket(keyring, mk_spec);
+	rcu_read_lock();
+	switch (mk_spec->type) {
+	case FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR:
+		hlist_for_each_entry_rcu(mk, bucket, mk_node) {
+			if (mk->mk_spec.type ==
+				FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR &&
+			    memcmp(mk->mk_spec.u.descriptor,
+				   mk_spec->u.descriptor,
+				   FSCRYPT_KEY_DESCRIPTOR_SIZE) == 0 &&
+			    refcount_inc_not_zero(&mk->mk_struct_refs))
+				goto out;
+		}
+		break;
+	case FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER:
+		hlist_for_each_entry_rcu(mk, bucket, mk_node) {
+			if (mk->mk_spec.type ==
+				FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER &&
+			    memcmp(mk->mk_spec.u.identifier,
+				   mk_spec->u.identifier,
+				   FSCRYPT_KEY_IDENTIFIER_SIZE) == 0 &&
+			    refcount_inc_not_zero(&mk->mk_struct_refs))
+				goto out;
+		}
+		break;
+	}
+	mk = NULL;
+out:
+	rcu_read_unlock();
+	return mk;
 }
 
 static int allocate_master_key_users_keyring(struct fscrypt_master_key *mk)
@@ -277,17 +341,30 @@ static int allocate_master_key_users_keyring(struct fscrypt_master_key *mk)
 static struct key *find_master_key_user(struct fscrypt_master_key *mk)
 {
 	char description[FSCRYPT_MK_USER_DESCRIPTION_SIZE];
+	key_ref_t keyref;
 
 	format_mk_user_description(description, mk->mk_spec.u.identifier);
-	return search_fscrypt_keyring(mk->mk_users, &key_type_fscrypt_user,
-				      description);
+
+	/*
+	 * We need to mark the keyring reference as "possessed" so that we
+	 * acquire permission to search it, via the KEY_POS_SEARCH permission.
+	 */
+	keyref = keyring_search(make_key_ref(mk->mk_users, true /*possessed*/),
+				&key_type_fscrypt_user, description, false);
+	if (IS_ERR(keyref)) {
+		if (PTR_ERR(keyref) == -EAGAIN || /* not found */
+		    PTR_ERR(keyref) == -EKEYREVOKED) /* recently invalidated */
+			keyref = ERR_PTR(-ENOKEY);
+		return ERR_CAST(keyref);
+	}
+	return key_ref_to_ptr(keyref);
 }
 
 /*
  * Give the current user a "key" in ->mk_users.  This charges the user's quota
  * and marks the master key as added by the current user, so that it cannot be
- * removed by another user with the key.  Either the master key's key->sem must
- * be held for write, or the master key must be still undergoing initialization.
+ * removed by another user with the key.  Either ->mk_sem must be held for
+ * write, or the master key must be still undergoing initialization.
  */
 static int add_master_key_user(struct fscrypt_master_key *mk)
 {
@@ -309,7 +386,7 @@ static int add_master_key_user(struct fscrypt_master_key *mk)
 
 /*
  * Remove the current user's "key" from ->mk_users.
- * The master key's key->sem must be held for write.
+ * ->mk_sem must be held for write.
  *
  * Returns 0 if removed, -ENOKEY if not found, or another -errno code.
  */
@@ -327,63 +404,49 @@ static int remove_master_key_user(struct fscrypt_master_key *mk)
 }
 
 /*
- * Allocate a new fscrypt_master_key which contains the given secret, set it as
- * the payload of a new 'struct key' of type fscrypt, and link the 'struct key'
- * into the given keyring.  Synchronized by fscrypt_add_key_mutex.
+ * Allocate a new fscrypt_master_key, transfer the given secret over to it, and
+ * insert it into sb->s_master_keys.
  */
-static int add_new_master_key(struct fscrypt_master_key_secret *secret,
-			      const struct fscrypt_key_specifier *mk_spec,
-			      struct key *keyring)
+static int add_new_master_key(struct super_block *sb,
+			      struct fscrypt_master_key_secret *secret,
+			      const struct fscrypt_key_specifier *mk_spec)
 {
+	struct fscrypt_keyring *keyring = sb->s_master_keys;
 	struct fscrypt_master_key *mk;
-	char description[FSCRYPT_MK_DESCRIPTION_SIZE];
-	struct key *key;
 	int err;
 
 	mk = kzalloc(sizeof(*mk), GFP_KERNEL);
 	if (!mk)
 		return -ENOMEM;
 
+	mk->mk_sb = sb;
+	init_rwsem(&mk->mk_sem);
+	refcount_set(&mk->mk_struct_refs, 1);
 	mk->mk_spec = *mk_spec;
 
-	move_master_key_secret(&mk->mk_secret, secret);
-
-	refcount_set(&mk->mk_refcount, 1); /* secret is present */
 	INIT_LIST_HEAD(&mk->mk_decrypted_inodes);
 	spin_lock_init(&mk->mk_decrypted_inodes_lock);
 
 	if (mk_spec->type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) {
 		err = allocate_master_key_users_keyring(mk);
 		if (err)
-			goto out_free_mk;
+			goto out_put;
 		err = add_master_key_user(mk);
 		if (err)
-			goto out_free_mk;
+			goto out_put;
 	}
 
-	/*
-	 * Note that we don't charge this key to anyone's quota, since when
-	 * ->mk_users is in use those keys are charged instead, and otherwise
-	 * (when ->mk_users isn't in use) only root can add these keys.
-	 */
-	format_mk_description(description, mk_spec);
-	key = key_alloc(&key_type_fscrypt, description,
-			GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, current_cred(),
-			KEY_POS_SEARCH | KEY_USR_SEARCH | KEY_USR_VIEW,
-			KEY_ALLOC_NOT_IN_QUOTA, NULL);
-	if (IS_ERR(key)) {
-		err = PTR_ERR(key);
-		goto out_free_mk;
-	}
-	err = key_instantiate_and_link(key, mk, sizeof(*mk), keyring, NULL);
-	key_put(key);
-	if (err)
-		goto out_free_mk;
+	move_master_key_secret(&mk->mk_secret, secret);
+	refcount_set(&mk->mk_active_refs, 1); /* ->mk_secret is present */
 
+	spin_lock(&keyring->lock);
+	hlist_add_head_rcu(&mk->mk_node,
+			   fscrypt_mk_hash_bucket(keyring, mk_spec));
+	spin_unlock(&keyring->lock);
 	return 0;
 
-out_free_mk:
-	free_master_key(mk);
+out_put:
+	fscrypt_put_master_key(mk);
 	return err;
 }
 
@@ -392,42 +455,34 @@ out_free_mk:
 static int add_existing_master_key(struct fscrypt_master_key *mk,
 				   struct fscrypt_master_key_secret *secret)
 {
-	struct key *mk_user;
-	bool rekey;
 	int err;
 
 	/*
 	 * If the current user is already in ->mk_users, then there's nothing to
-	 * do.  (Not applicable for v1 policy keys, which have NULL ->mk_users.)
+	 * do.  Otherwise, we need to add the user to ->mk_users.  (Neither is
+	 * applicable for v1 policy keys, which have NULL ->mk_users.)
 	 */
 	if (mk->mk_users) {
-		mk_user = find_master_key_user(mk);
+		struct key *mk_user = find_master_key_user(mk);
+
 		if (mk_user != ERR_PTR(-ENOKEY)) {
 			if (IS_ERR(mk_user))
 				return PTR_ERR(mk_user);
 			key_put(mk_user);
 			return 0;
 		}
-	}
-
-	/* If we'll be re-adding ->mk_secret, try to take the reference. */
-	rekey = !is_master_key_secret_present(&mk->mk_secret);
-	if (rekey && !refcount_inc_not_zero(&mk->mk_refcount))
-		return KEY_DEAD;
-
-	/* Add the current user to ->mk_users, if applicable. */
-	if (mk->mk_users) {
 		err = add_master_key_user(mk);
-		if (err) {
-			if (rekey && refcount_dec_and_test(&mk->mk_refcount))
-				return KEY_DEAD;
+		if (err)
 			return err;
-		}
 	}
 
 	/* Re-add the secret if needed. */
-	if (rekey)
+	if (!is_master_key_secret_present(&mk->mk_secret)) {
+		if (!refcount_inc_not_zero(&mk->mk_active_refs))
+			return KEY_DEAD;
 		move_master_key_secret(&mk->mk_secret, secret);
+	}
+
 	return 0;
 }
 
@@ -436,38 +491,36 @@ static int do_add_master_key(struct super_block *sb,
 			     const struct fscrypt_key_specifier *mk_spec)
 {
 	static DEFINE_MUTEX(fscrypt_add_key_mutex);
-	struct key *key;
+	struct fscrypt_master_key *mk;
 	int err;
 
 	mutex_lock(&fscrypt_add_key_mutex); /* serialize find + link */
-retry:
-	key = fscrypt_find_master_key(sb, mk_spec);
-	if (IS_ERR(key)) {
-		err = PTR_ERR(key);
-		if (err != -ENOKEY)
-			goto out_unlock;
+
+	mk = fscrypt_find_master_key(sb, mk_spec);
+	if (!mk) {
 		/* Didn't find the key in ->s_master_keys.  Add it. */
 		err = allocate_filesystem_keyring(sb);
-		if (err)
-			goto out_unlock;
-		err = add_new_master_key(secret, mk_spec, sb->s_master_keys);
+		if (!err)
+			err = add_new_master_key(sb, secret, mk_spec);
 	} else {
 		/*
 		 * Found the key in ->s_master_keys.  Re-add the secret if
 		 * needed, and add the user to ->mk_users if needed.
 		 */
-		down_write(&key->sem);
-		err = add_existing_master_key(key->payload.data[0], secret);
-		up_write(&key->sem);
+		down_write(&mk->mk_sem);
+		err = add_existing_master_key(mk, secret);
+		up_write(&mk->mk_sem);
 		if (err == KEY_DEAD) {
-			/* Key being removed or needs to be removed */
-			key_invalidate(key);
-			key_put(key);
-			goto retry;
+			/*
+			 * We found a key struct, but it's already been fully
+			 * removed.  Ignore the old struct and add a new one.
+			 * fscrypt_add_key_mutex means we don't need to worry
+			 * about concurrent adds.
+			 */
+			err = add_new_master_key(sb, secret, mk_spec);
 		}
-		key_put(key);
+		fscrypt_put_master_key(mk);
 	}
-out_unlock:
 	mutex_unlock(&fscrypt_add_key_mutex);
 	return err;
 }
@@ -771,19 +824,19 @@ int fscrypt_verify_key_added(struct super_block *sb,
 			     const u8 identifier[FSCRYPT_KEY_IDENTIFIER_SIZE])
 {
 	struct fscrypt_key_specifier mk_spec;
-	struct key *key, *mk_user;
 	struct fscrypt_master_key *mk;
+	struct key *mk_user;
 	int err;
 
 	mk_spec.type = FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER;
 	memcpy(mk_spec.u.identifier, identifier, FSCRYPT_KEY_IDENTIFIER_SIZE);
 
-	key = fscrypt_find_master_key(sb, &mk_spec);
-	if (IS_ERR(key)) {
-		err = PTR_ERR(key);
+	mk = fscrypt_find_master_key(sb, &mk_spec);
+	if (!mk) {
+		err = -ENOKEY;
 		goto out;
 	}
-	mk = key->payload.data[0];
+	down_read(&mk->mk_sem);
 	mk_user = find_master_key_user(mk);
 	if (IS_ERR(mk_user)) {
 		err = PTR_ERR(mk_user);
@@ -791,7 +844,8 @@ int fscrypt_verify_key_added(struct super_block *sb,
 		key_put(mk_user);
 		err = 0;
 	}
-	key_put(key);
+	up_read(&mk->mk_sem);
+	fscrypt_put_master_key(mk);
 out:
 	if (err == -ENOKEY && capable(CAP_FOWNER))
 		err = 0;
@@ -953,11 +1007,10 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users)
 	struct super_block *sb = file_inode(filp)->i_sb;
 	struct fscrypt_remove_key_arg __user *uarg = _uarg;
 	struct fscrypt_remove_key_arg arg;
-	struct key *key;
 	struct fscrypt_master_key *mk;
 	u32 status_flags = 0;
 	int err;
-	bool dead;
+	bool inodes_remain;
 
 	if (copy_from_user(&arg, uarg, sizeof(arg)))
 		return -EFAULT;
@@ -977,12 +1030,10 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users)
 		return -EACCES;
 
 	/* Find the key being removed. */
-	key = fscrypt_find_master_key(sb, &arg.key_spec);
-	if (IS_ERR(key))
-		return PTR_ERR(key);
-	mk = key->payload.data[0];
-
-	down_write(&key->sem);
+	mk = fscrypt_find_master_key(sb, &arg.key_spec);
+	if (!mk)
+		return -ENOKEY;
+	down_write(&mk->mk_sem);
 
 	/* If relevant, remove current user's (or all users) claim to the key */
 	if (mk->mk_users && mk->mk_users->keys.nr_leaves_on_tree != 0) {
@@ -991,7 +1042,7 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users)
 		else
 			err = remove_master_key_user(mk);
 		if (err) {
-			up_write(&key->sem);
+			up_write(&mk->mk_sem);
 			goto out_put_key;
 		}
 		if (mk->mk_users->keys.nr_leaves_on_tree != 0) {
@@ -1003,26 +1054,22 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users)
 			status_flags |=
 				FSCRYPT_KEY_REMOVAL_STATUS_FLAG_OTHER_USERS;
 			err = 0;
-			up_write(&key->sem);
+			up_write(&mk->mk_sem);
 			goto out_put_key;
 		}
 	}
 
 	/* No user claims remaining.  Go ahead and wipe the secret. */
-	dead = false;
+	err = -ENOKEY;
 	if (is_master_key_secret_present(&mk->mk_secret)) {
 		wipe_master_key_secret(&mk->mk_secret);
-		dead = refcount_dec_and_test(&mk->mk_refcount);
-	}
-	up_write(&key->sem);
-	if (dead) {
-		/*
-		 * No inodes reference the key, and we wiped the secret, so the
-		 * key object is free to be removed from the keyring.
-		 */
-		key_invalidate(key);
+		fscrypt_put_master_key_activeref(mk);
 		err = 0;
-	} else {
+	}
+	inodes_remain = refcount_read(&mk->mk_active_refs) > 0;
+	up_write(&mk->mk_sem);
+
+	if (inodes_remain) {
 		/* Some inodes still reference this key; try to evict them. */
 		err = try_to_lock_encrypted_files(sb, mk);
 		if (err == -EBUSY) {
@@ -1038,7 +1085,7 @@ static int do_remove_key(struct file *filp, void __user *_uarg, bool all_users)
 	 * has been fully removed including all files locked.
 	 */
 out_put_key:
-	key_put(key);
+	fscrypt_put_master_key(mk);
 	if (err == 0)
 		err = put_user(status_flags, &uarg->removal_status_flags);
 	return err;
@@ -1085,7 +1132,6 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg)
 {
 	struct super_block *sb = file_inode(filp)->i_sb;
 	struct fscrypt_get_key_status_arg arg;
-	struct key *key;
 	struct fscrypt_master_key *mk;
 	int err;
 
@@ -1102,19 +1148,18 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg)
 	arg.user_count = 0;
 	memset(arg.__out_reserved, 0, sizeof(arg.__out_reserved));
 
-	key = fscrypt_find_master_key(sb, &arg.key_spec);
-	if (IS_ERR(key)) {
-		if (key != ERR_PTR(-ENOKEY))
-			return PTR_ERR(key);
+	mk = fscrypt_find_master_key(sb, &arg.key_spec);
+	if (!mk) {
 		arg.status = FSCRYPT_KEY_STATUS_ABSENT;
 		err = 0;
 		goto out;
 	}
-	mk = key->payload.data[0];
-	down_read(&key->sem);
+	down_read(&mk->mk_sem);
 
 	if (!is_master_key_secret_present(&mk->mk_secret)) {
-		arg.status = FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED;
+		arg.status = refcount_read(&mk->mk_active_refs) > 0 ?
+			FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED :
+			FSCRYPT_KEY_STATUS_ABSENT /* raced with full removal */;
 		err = 0;
 		goto out_release_key;
 	}
@@ -1136,8 +1181,8 @@ int fscrypt_ioctl_get_key_status(struct file *filp, void __user *uarg)
 	}
 	err = 0;
 out_release_key:
-	up_read(&key->sem);
-	key_put(key);
+	up_read(&mk->mk_sem);
+	fscrypt_put_master_key(mk);
 out:
 	if (!err && copy_to_user(uarg, &arg, sizeof(arg)))
 		err = -EFAULT;
@@ -1149,13 +1194,9 @@ int __init fscrypt_init_keyring(void)
 {
 	int err;
 
-	err = register_key_type(&key_type_fscrypt);
-	if (err)
-		return err;
-
 	err = register_key_type(&key_type_fscrypt_user);
 	if (err)
-		goto err_unregister_fscrypt;
+		return err;
 
 	err = register_key_type(&key_type_fscrypt_provisioning);
 	if (err)
@@ -1165,7 +1206,5 @@ int __init fscrypt_init_keyring(void)
 
 err_unregister_fscrypt_user:
 	unregister_key_type(&key_type_fscrypt_user);
-err_unregister_fscrypt:
-	unregister_key_type(&key_type_fscrypt);
 	return err;
 }
diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c
index fbc71abdabe3..f7407071a952 100644
--- a/fs/crypto/keysetup.c
+++ b/fs/crypto/keysetup.c
@@ -9,7 +9,6 @@
  */
 
 #include <crypto/skcipher.h>
-#include <linux/key.h>
 #include <linux/random.h>
 
 #include "fscrypt_private.h"
@@ -155,10 +154,12 @@ int fscrypt_prepare_key(struct fscrypt_prepared_key *prep_key,
 }
 
 /* Destroy a crypto transform object and/or blk-crypto key. */
-void fscrypt_destroy_prepared_key(struct fscrypt_prepared_key *prep_key)
+void fscrypt_destroy_prepared_key(struct super_block *sb,
+				  struct fscrypt_prepared_key *prep_key)
 {
 	crypto_free_skcipher(prep_key->tfm);
-	fscrypt_destroy_inline_crypt_key(prep_key);
+	fscrypt_destroy_inline_crypt_key(sb, prep_key);
+	memzero_explicit(prep_key, sizeof(*prep_key));
 }
 
 /* Given a per-file encryption key, set up the file's crypto transform object */
@@ -412,20 +413,18 @@ static bool fscrypt_valid_master_key_size(const struct fscrypt_master_key *mk,
 /*
  * Find the master key, then set up the inode's actual encryption key.
  *
- * If the master key is found in the filesystem-level keyring, then the
- * corresponding 'struct key' is returned in *master_key_ret with its semaphore
- * read-locked.  This is needed to ensure that only one task links the
- * fscrypt_info into ->mk_decrypted_inodes (as multiple tasks may race to create
- * an fscrypt_info for the same inode), and to synchronize the master key being
- * removed with a new inode starting to use it.
+ * If the master key is found in the filesystem-level keyring, then it is
+ * returned in *mk_ret with its semaphore read-locked.  This is needed to ensure
+ * that only one task links the fscrypt_info into ->mk_decrypted_inodes (as
+ * multiple tasks may race to create an fscrypt_info for the same inode), and to
+ * synchronize the master key being removed with a new inode starting to use it.
  */
 static int setup_file_encryption_key(struct fscrypt_info *ci,
 				     bool need_dirhash_key,
-				     struct key **master_key_ret)
+				     struct fscrypt_master_key **mk_ret)
 {
-	struct key *key;
-	struct fscrypt_master_key *mk = NULL;
 	struct fscrypt_key_specifier mk_spec;
+	struct fscrypt_master_key *mk;
 	int err;
 
 	err = fscrypt_select_encryption_impl(ci);
@@ -436,11 +435,10 @@ static int setup_file_encryption_key(struct fscrypt_info *ci,
 	if (err)
 		return err;
 
-	key = fscrypt_find_master_key(ci->ci_inode->i_sb, &mk_spec);
-	if (IS_ERR(key)) {
-		if (key != ERR_PTR(-ENOKEY) ||
-		    ci->ci_policy.version != FSCRYPT_POLICY_V1)
-			return PTR_ERR(key);
+	mk = fscrypt_find_master_key(ci->ci_inode->i_sb, &mk_spec);
+	if (!mk) {
+		if (ci->ci_policy.version != FSCRYPT_POLICY_V1)
+			return -ENOKEY;
 
 		/*
 		 * As a legacy fallback for v1 policies, search for the key in
@@ -450,9 +448,7 @@ static int setup_file_encryption_key(struct fscrypt_info *ci,
 		 */
 		return fscrypt_setup_v1_file_key_via_subscribed_keyrings(ci);
 	}
-
-	mk = key->payload.data[0];
-	down_read(&key->sem);
+	down_read(&mk->mk_sem);
 
 	/* Has the secret been removed (via FS_IOC_REMOVE_ENCRYPTION_KEY)? */
 	if (!is_master_key_secret_present(&mk->mk_secret)) {
@@ -480,18 +476,18 @@ static int setup_file_encryption_key(struct fscrypt_info *ci,
 	if (err)
 		goto out_release_key;
 
-	*master_key_ret = key;
+	*mk_ret = mk;
 	return 0;
 
 out_release_key:
-	up_read(&key->sem);
-	key_put(key);
+	up_read(&mk->mk_sem);
+	fscrypt_put_master_key(mk);
 	return err;
 }
 
 static void put_crypt_info(struct fscrypt_info *ci)
 {
-	struct key *key;
+	struct fscrypt_master_key *mk;
 
 	if (!ci)
 		return;
@@ -499,26 +495,21 @@ static void put_crypt_info(struct fscrypt_info *ci)
 	if (ci->ci_direct_key)
 		fscrypt_put_direct_key(ci->ci_direct_key);
 	else if (ci->ci_owns_key)
-		fscrypt_destroy_prepared_key(&ci->ci_enc_key);
-
-	key = ci->ci_master_key;
-	if (key) {
-		struct fscrypt_master_key *mk = key->payload.data[0];
+		fscrypt_destroy_prepared_key(ci->ci_inode->i_sb,
+					     &ci->ci_enc_key);
 
+	mk = ci->ci_master_key;
+	if (mk) {
 		/*
 		 * Remove this inode from the list of inodes that were unlocked
-		 * with the master key.
-		 *
-		 * In addition, if we're removing the last inode from a key that
-		 * already had its secret removed, invalidate the key so that it
-		 * gets removed from ->s_master_keys.
+		 * with the master key.  In addition, if we're removing the last
+		 * inode from a master key struct that already had its secret
+		 * removed, then complete the full removal of the struct.
 		 */
 		spin_lock(&mk->mk_decrypted_inodes_lock);
 		list_del(&ci->ci_master_key_link);
 		spin_unlock(&mk->mk_decrypted_inodes_lock);
-		if (refcount_dec_and_test(&mk->mk_refcount))
-			key_invalidate(key);
-		key_put(key);
+		fscrypt_put_master_key_activeref(mk);
 	}
 	memzero_explicit(ci, sizeof(*ci));
 	kmem_cache_free(fscrypt_info_cachep, ci);
@@ -532,7 +523,7 @@ fscrypt_setup_encryption_info(struct inode *inode,
 {
 	struct fscrypt_info *crypt_info;
 	struct fscrypt_mode *mode;
-	struct key *master_key = NULL;
+	struct fscrypt_master_key *mk = NULL;
 	int res;
 
 	res = fscrypt_initialize(inode->i_sb->s_cop->flags);
@@ -555,8 +546,7 @@ fscrypt_setup_encryption_info(struct inode *inode,
 	WARN_ON(mode->ivsize > FSCRYPT_MAX_IV_SIZE);
 	crypt_info->ci_mode = mode;
 
-	res = setup_file_encryption_key(crypt_info, need_dirhash_key,
-					&master_key);
+	res = setup_file_encryption_key(crypt_info, need_dirhash_key, &mk);
 	if (res)
 		goto out;
 
@@ -571,12 +561,9 @@ fscrypt_setup_encryption_info(struct inode *inode,
 		 * We won the race and set ->i_crypt_info to our crypt_info.
 		 * Now link it into the master key's inode list.
 		 */
-		if (master_key) {
-			struct fscrypt_master_key *mk =
-				master_key->payload.data[0];
-
-			refcount_inc(&mk->mk_refcount);
-			crypt_info->ci_master_key = key_get(master_key);
+		if (mk) {
+			crypt_info->ci_master_key = mk;
+			refcount_inc(&mk->mk_active_refs);
 			spin_lock(&mk->mk_decrypted_inodes_lock);
 			list_add(&crypt_info->ci_master_key_link,
 				 &mk->mk_decrypted_inodes);
@@ -586,9 +573,9 @@ fscrypt_setup_encryption_info(struct inode *inode,
 	}
 	res = 0;
 out:
-	if (master_key) {
-		up_read(&master_key->sem);
-		key_put(master_key);
+	if (mk) {
+		up_read(&mk->mk_sem);
+		fscrypt_put_master_key(mk);
 	}
 	put_crypt_info(crypt_info);
 	return res;
@@ -753,7 +740,6 @@ EXPORT_SYMBOL(fscrypt_free_inode);
 int fscrypt_drop_inode(struct inode *inode)
 {
 	const struct fscrypt_info *ci = fscrypt_get_info(inode);
-	const struct fscrypt_master_key *mk;
 
 	/*
 	 * If ci is NULL, then the inode doesn't have an encryption key set up
@@ -763,7 +749,6 @@ int fscrypt_drop_inode(struct inode *inode)
 	 */
 	if (!ci || !ci->ci_master_key)
 		return 0;
-	mk = ci->ci_master_key->payload.data[0];
 
 	/*
 	 * With proper, non-racy use of FS_IOC_REMOVE_ENCRYPTION_KEY, all inodes
@@ -782,6 +767,6 @@ int fscrypt_drop_inode(struct inode *inode)
 	 * then the thread removing the key will either evict the inode itself
 	 * or will correctly detect that it wasn't evicted due to the race.
 	 */
-	return !is_master_key_secret_present(&mk->mk_secret);
+	return !is_master_key_secret_present(&ci->ci_master_key->mk_secret);
 }
 EXPORT_SYMBOL_GPL(fscrypt_drop_inode);
diff --git a/fs/crypto/keysetup_v1.c b/fs/crypto/keysetup_v1.c
index 2762c5350432..75dabd9b27f9 100644
--- a/fs/crypto/keysetup_v1.c
+++ b/fs/crypto/keysetup_v1.c
@@ -143,6 +143,7 @@ invalid:
 
 /* Master key referenced by DIRECT_KEY policy */
 struct fscrypt_direct_key {
+	struct super_block		*dk_sb;
 	struct hlist_node		dk_node;
 	refcount_t			dk_refcount;
 	const struct fscrypt_mode	*dk_mode;
@@ -154,7 +155,7 @@ struct fscrypt_direct_key {
 static void free_direct_key(struct fscrypt_direct_key *dk)
 {
 	if (dk) {
-		fscrypt_destroy_prepared_key(&dk->dk_key);
+		fscrypt_destroy_prepared_key(dk->dk_sb, &dk->dk_key);
 		kfree_sensitive(dk);
 	}
 }
@@ -231,6 +232,7 @@ fscrypt_get_direct_key(const struct fscrypt_info *ci, const u8 *raw_key)
 	dk = kzalloc(sizeof(*dk), GFP_KERNEL);
 	if (!dk)
 		return ERR_PTR(-ENOMEM);
+	dk->dk_sb = ci->ci_inode->i_sb;
 	refcount_set(&dk->dk_refcount, 1);
 	dk->dk_mode = ci->ci_mode;
 	err = fscrypt_prepare_key(&dk->dk_key, raw_key, ci);
diff --git a/fs/crypto/policy.c b/fs/crypto/policy.c
index 80b8ca0f340b..46757c3052ef 100644
--- a/fs/crypto/policy.c
+++ b/fs/crypto/policy.c
@@ -744,12 +744,8 @@ int fscrypt_set_context(struct inode *inode, void *fs_data)
 	 * delayed key setup that requires the inode number.
 	 */
 	if (ci->ci_policy.version == FSCRYPT_POLICY_V2 &&
-	    (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32)) {
-		const struct fscrypt_master_key *mk =
-			ci->ci_master_key->payload.data[0];
-
-		fscrypt_hash_inode_number(ci, mk);
-	}
+	    (ci->ci_policy.v2.flags & FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32))
+		fscrypt_hash_inode_number(ci, ci->ci_master_key);
 
 	return inode->i_sb->s_cop->set_context(inode, &ctx, ctxsize, fs_data);
 }
@@ -833,19 +829,6 @@ bool fscrypt_dummy_policies_equal(const struct fscrypt_dummy_policy *p1,
 }
 EXPORT_SYMBOL_GPL(fscrypt_dummy_policies_equal);
 
-/* Deprecated, do not use */
-int fscrypt_set_test_dummy_encryption(struct super_block *sb, const char *arg,
-				      struct fscrypt_dummy_policy *dummy_policy)
-{
-	struct fs_parameter param = {
-		.type = fs_value_is_string,
-		.string = arg ? (char *)arg : "",
-	};
-	return fscrypt_parse_test_dummy_encryption(&param, dummy_policy) ?:
-		fscrypt_add_test_dummy_key(sb, dummy_policy);
-}
-EXPORT_SYMBOL_GPL(fscrypt_set_test_dummy_encryption);
-
 /**
  * fscrypt_show_test_dummy_encryption() - show '-o test_dummy_encryption'
  * @seq: the seq_file to print the option to
diff --git a/fs/d_path.c b/fs/d_path.c
index e4e0ebad1f15..56a6ee4c6331 100644
--- a/fs/d_path.c
+++ b/fs/d_path.c
@@ -34,7 +34,7 @@ static bool prepend_char(struct prepend_buffer *p, unsigned char c)
 }
 
 /*
- * The source of the prepend data can be an optimistoc load
+ * The source of the prepend data can be an optimistic load
  * of a dentry name and length. And because we don't hold any
  * locks, the length and the pointer to the name may not be
  * in sync if a concurrent rename happens, and the kernel
@@ -297,8 +297,7 @@ EXPORT_SYMBOL(d_path);
 /*
  * Helper function for dentry_operations.d_dname() members
  */
-char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen,
-			const char *fmt, ...)
+char *dynamic_dname(char *buffer, int buflen, const char *fmt, ...)
 {
 	va_list args;
 	char temp[64];
diff --git a/fs/dlm/ast.c b/fs/dlm/ast.c
index 19ef136f9e4f..d60a8d8f109d 100644
--- a/fs/dlm/ast.c
+++ b/fs/dlm/ast.c
@@ -200,13 +200,13 @@ void dlm_add_cb(struct dlm_lkb *lkb, uint32_t flags, int mode, int status,
 	if (!prev_seq) {
 		kref_get(&lkb->lkb_ref);
 
+		mutex_lock(&ls->ls_cb_mutex);
 		if (test_bit(LSFL_CB_DELAY, &ls->ls_flags)) {
-			mutex_lock(&ls->ls_cb_mutex);
 			list_add(&lkb->lkb_cb_list, &ls->ls_cb_delay);
-			mutex_unlock(&ls->ls_cb_mutex);
 		} else {
 			queue_work(ls->ls_callback_wq, &lkb->lkb_cb_work);
 		}
+		mutex_unlock(&ls->ls_cb_mutex);
 	}
  out:
 	mutex_unlock(&lkb->lkb_cb_mutex);
@@ -288,10 +288,13 @@ void dlm_callback_stop(struct dlm_ls *ls)
 
 void dlm_callback_suspend(struct dlm_ls *ls)
 {
-	set_bit(LSFL_CB_DELAY, &ls->ls_flags);
+	if (ls->ls_callback_wq) {
+		mutex_lock(&ls->ls_cb_mutex);
+		set_bit(LSFL_CB_DELAY, &ls->ls_flags);
+		mutex_unlock(&ls->ls_cb_mutex);
 
-	if (ls->ls_callback_wq)
 		flush_workqueue(ls->ls_callback_wq);
+	}
 }
 
 #define MAX_CB_QUEUE 25
@@ -302,11 +305,11 @@ void dlm_callback_resume(struct dlm_ls *ls)
 	int count = 0, sum = 0;
 	bool empty;
 
-	clear_bit(LSFL_CB_DELAY, &ls->ls_flags);
-
 	if (!ls->ls_callback_wq)
 		return;
 
+	clear_bit(LSFL_CB_DELAY, &ls->ls_flags);
+
 more:
 	mutex_lock(&ls->ls_cb_mutex);
 	list_for_each_entry_safe(lkb, safe, &ls->ls_cb_delay, lkb_cb_list) {
diff --git a/fs/dlm/ast.h b/fs/dlm/ast.h
index 181ad7d20c4d..e5e05fcc5813 100644
--- a/fs/dlm/ast.h
+++ b/fs/dlm/ast.h
@@ -11,7 +11,6 @@
 #ifndef __ASTD_DOT_H__
 #define __ASTD_DOT_H__
 
-void dlm_del_ast(struct dlm_lkb *lkb);
 int dlm_add_lkb_callback(struct dlm_lkb *lkb, uint32_t flags, int mode,
                          int status, uint32_t sbflags, uint64_t seq);
 int dlm_rem_lkb_callback(struct dlm_ls *ls, struct dlm_lkb *lkb,
diff --git a/fs/dlm/dlm_internal.h b/fs/dlm/dlm_internal.h
index 8aca8085d24e..e34c3d2639a5 100644
--- a/fs/dlm/dlm_internal.h
+++ b/fs/dlm/dlm_internal.h
@@ -661,7 +661,7 @@ struct dlm_ls {
 	spinlock_t		ls_recover_idr_lock;
 	wait_queue_head_t	ls_wait_general;
 	wait_queue_head_t	ls_recover_lock_wait;
-	struct mutex		ls_clear_proc_locks;
+	spinlock_t		ls_clear_proc_locks;
 
 	struct list_head	ls_root_list;	/* root resources */
 	struct rw_semaphore	ls_root_sem;	/* protect root_list */
diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c
index dac7eb75dba9..94a72ede5764 100644
--- a/fs/dlm/lock.c
+++ b/fs/dlm/lock.c
@@ -401,7 +401,7 @@ static int pre_rsb_struct(struct dlm_ls *ls)
    unlock any spinlocks, go back and call pre_rsb_struct again.
    Otherwise, take an rsb off the list and return it. */
 
-static int get_rsb_struct(struct dlm_ls *ls, char *name, int len,
+static int get_rsb_struct(struct dlm_ls *ls, const void *name, int len,
 			  struct dlm_rsb **r_ret)
 {
 	struct dlm_rsb *r;
@@ -412,7 +412,8 @@ static int get_rsb_struct(struct dlm_ls *ls, char *name, int len,
 		count = ls->ls_new_rsb_count;
 		spin_unlock(&ls->ls_new_rsb_spin);
 		log_debug(ls, "find_rsb retry %d %d %s",
-			  count, dlm_config.ci_new_rsb_count, name);
+			  count, dlm_config.ci_new_rsb_count,
+			  (const char *)name);
 		return -EAGAIN;
 	}
 
@@ -448,7 +449,7 @@ static int rsb_cmp(struct dlm_rsb *r, const char *name, int nlen)
 	return memcmp(r->res_name, maxname, DLM_RESNAME_MAXLEN);
 }
 
-int dlm_search_rsb_tree(struct rb_root *tree, char *name, int len,
+int dlm_search_rsb_tree(struct rb_root *tree, const void *name, int len,
 			struct dlm_rsb **r_ret)
 {
 	struct rb_node *node = tree->rb_node;
@@ -546,7 +547,7 @@ static int rsb_insert(struct dlm_rsb *rsb, struct rb_root *tree)
  * while that rsb has a potentially stale master.)
  */
 
-static int find_rsb_dir(struct dlm_ls *ls, char *name, int len,
+static int find_rsb_dir(struct dlm_ls *ls, const void *name, int len,
 			uint32_t hash, uint32_t b,
 			int dir_nodeid, int from_nodeid,
 			unsigned int flags, struct dlm_rsb **r_ret)
@@ -724,7 +725,7 @@ static int find_rsb_dir(struct dlm_ls *ls, char *name, int len,
    dlm_recover_locks) before we've made ourself master (in
    dlm_recover_masters). */
 
-static int find_rsb_nodir(struct dlm_ls *ls, char *name, int len,
+static int find_rsb_nodir(struct dlm_ls *ls, const void *name, int len,
 			  uint32_t hash, uint32_t b,
 			  int dir_nodeid, int from_nodeid,
 			  unsigned int flags, struct dlm_rsb **r_ret)
@@ -818,8 +819,9 @@ static int find_rsb_nodir(struct dlm_ls *ls, char *name, int len,
 	return error;
 }
 
-static int find_rsb(struct dlm_ls *ls, char *name, int len, int from_nodeid,
-		    unsigned int flags, struct dlm_rsb **r_ret)
+static int find_rsb(struct dlm_ls *ls, const void *name, int len,
+		    int from_nodeid, unsigned int flags,
+		    struct dlm_rsb **r_ret)
 {
 	uint32_t hash, b;
 	int dir_nodeid;
@@ -2864,17 +2866,9 @@ static int set_unlock_args(uint32_t flags, void *astarg, struct dlm_args *args)
 static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
 			      struct dlm_args *args)
 {
-	int rv = -EINVAL;
+	int rv = -EBUSY;
 
 	if (args->flags & DLM_LKF_CONVERT) {
-		if (lkb->lkb_flags & DLM_IFL_MSTCPY)
-			goto out;
-
-		if (args->flags & DLM_LKF_QUECVT &&
-		    !__quecvt_compat_matrix[lkb->lkb_grmode+1][args->mode+1])
-			goto out;
-
-		rv = -EBUSY;
 		if (lkb->lkb_status != DLM_LKSTS_GRANTED)
 			goto out;
 
@@ -2884,6 +2878,14 @@ static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
 
 		if (is_overlap(lkb))
 			goto out;
+
+		rv = -EINVAL;
+		if (lkb->lkb_flags & DLM_IFL_MSTCPY)
+			goto out;
+
+		if (args->flags & DLM_LKF_QUECVT &&
+		    !__quecvt_compat_matrix[lkb->lkb_grmode+1][args->mode+1])
+			goto out;
 	}
 
 	lkb->lkb_exflags = args->flags;
@@ -2900,11 +2902,25 @@ static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
 #endif
 	rv = 0;
  out:
-	if (rv)
-		log_debug(ls, "validate_lock_args %d %x %x %x %d %d %s",
+	switch (rv) {
+	case 0:
+		break;
+	case -EINVAL:
+		/* annoy the user because dlm usage is wrong */
+		WARN_ON(1);
+		log_error(ls, "%s %d %x %x %x %d %d %s", __func__,
+			  rv, lkb->lkb_id, lkb->lkb_flags, args->flags,
+			  lkb->lkb_status, lkb->lkb_wait_type,
+			  lkb->lkb_resource->res_name);
+		break;
+	default:
+		log_debug(ls, "%s %d %x %x %x %d %d %s", __func__,
 			  rv, lkb->lkb_id, lkb->lkb_flags, args->flags,
 			  lkb->lkb_status, lkb->lkb_wait_type,
 			  lkb->lkb_resource->res_name);
+		break;
+	}
+
 	return rv;
 }
 
@@ -2918,23 +2934,12 @@ static int validate_lock_args(struct dlm_ls *ls, struct dlm_lkb *lkb,
 static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args)
 {
 	struct dlm_ls *ls = lkb->lkb_resource->res_ls;
-	int rv = -EINVAL;
-
-	if (lkb->lkb_flags & DLM_IFL_MSTCPY) {
-		log_error(ls, "unlock on MSTCPY %x", lkb->lkb_id);
-		dlm_print_lkb(lkb);
-		goto out;
-	}
-
-	/* an lkb may still exist even though the lock is EOL'ed due to a
-	   cancel, unlock or failed noqueue request; an app can't use these
-	   locks; return same error as if the lkid had not been found at all */
+	int rv = -EBUSY;
 
-	if (lkb->lkb_flags & DLM_IFL_ENDOFLIFE) {
-		log_debug(ls, "unlock on ENDOFLIFE %x", lkb->lkb_id);
-		rv = -ENOENT;
+	/* normal unlock not allowed if there's any op in progress */
+	if (!(args->flags & (DLM_LKF_CANCEL | DLM_LKF_FORCEUNLOCK)) &&
+	    (lkb->lkb_wait_type || lkb->lkb_wait_count))
 		goto out;
-	}
 
 	/* an lkb may be waiting for an rsb lookup to complete where the
 	   lookup was initiated by another lock */
@@ -2949,7 +2954,24 @@ static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args)
 			unhold_lkb(lkb); /* undoes create_lkb() */
 		}
 		/* caller changes -EBUSY to 0 for CANCEL and FORCEUNLOCK */
-		rv = -EBUSY;
+		goto out;
+	}
+
+	rv = -EINVAL;
+	if (lkb->lkb_flags & DLM_IFL_MSTCPY) {
+		log_error(ls, "unlock on MSTCPY %x", lkb->lkb_id);
+		dlm_print_lkb(lkb);
+		goto out;
+	}
+
+	/* an lkb may still exist even though the lock is EOL'ed due to a
+	 * cancel, unlock or failed noqueue request; an app can't use these
+	 * locks; return same error as if the lkid had not been found at all
+	 */
+
+	if (lkb->lkb_flags & DLM_IFL_ENDOFLIFE) {
+		log_debug(ls, "unlock on ENDOFLIFE %x", lkb->lkb_id);
+		rv = -ENOENT;
 		goto out;
 	}
 
@@ -3022,14 +3044,8 @@ static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args)
 			goto out;
 		}
 		/* add_to_waiters() will set OVERLAP_UNLOCK */
-		goto out_ok;
 	}
 
-	/* normal unlock not allowed if there's any op in progress */
-	rv = -EBUSY;
-	if (lkb->lkb_wait_type || lkb->lkb_wait_count)
-		goto out;
-
  out_ok:
 	/* an overlapping op shouldn't blow away exflags from other op */
 	lkb->lkb_exflags |= args->flags;
@@ -3037,11 +3053,25 @@ static int validate_unlock_args(struct dlm_lkb *lkb, struct dlm_args *args)
 	lkb->lkb_astparam = args->astparam;
 	rv = 0;
  out:
-	if (rv)
-		log_debug(ls, "validate_unlock_args %d %x %x %x %x %d %s", rv,
+	switch (rv) {
+	case 0:
+		break;
+	case -EINVAL:
+		/* annoy the user because dlm usage is wrong */
+		WARN_ON(1);
+		log_error(ls, "%s %d %x %x %x %x %d %s", __func__, rv,
+			  lkb->lkb_id, lkb->lkb_flags, lkb->lkb_exflags,
+			  args->flags, lkb->lkb_wait_type,
+			  lkb->lkb_resource->res_name);
+		break;
+	default:
+		log_debug(ls, "%s %d %x %x %x %x %d %s", __func__, rv,
 			  lkb->lkb_id, lkb->lkb_flags, lkb->lkb_exflags,
 			  args->flags, lkb->lkb_wait_type,
 			  lkb->lkb_resource->res_name);
+		break;
+	}
+
 	return rv;
 }
 
@@ -3292,8 +3322,9 @@ static int _cancel_lock(struct dlm_rsb *r, struct dlm_lkb *lkb)
  * request_lock(), convert_lock(), unlock_lock(), cancel_lock()
  */
 
-static int request_lock(struct dlm_ls *ls, struct dlm_lkb *lkb, char *name,
-			int len, struct dlm_args *args)
+static int request_lock(struct dlm_ls *ls, struct dlm_lkb *lkb,
+			const void *name, int len,
+			struct dlm_args *args)
 {
 	struct dlm_rsb *r;
 	int error;
@@ -3392,7 +3423,7 @@ int dlm_lock(dlm_lockspace_t *lockspace,
 	     int mode,
 	     struct dlm_lksb *lksb,
 	     uint32_t flags,
-	     void *name,
+	     const void *name,
 	     unsigned int namelen,
 	     uint32_t parent_lkid,
 	     void (*ast) (void *astarg),
@@ -3438,7 +3469,7 @@ int dlm_lock(dlm_lockspace_t *lockspace,
 	if (error == -EINPROGRESS)
 		error = 0;
  out_put:
-	trace_dlm_lock_end(ls, lkb, name, namelen, mode, flags, error);
+	trace_dlm_lock_end(ls, lkb, name, namelen, mode, flags, error, true);
 
 	if (convert || error)
 		__put_lkb(ls, lkb);
@@ -3623,7 +3654,7 @@ static void send_args(struct dlm_rsb *r, struct dlm_lkb *lkb,
 	case cpu_to_le32(DLM_MSG_REQUEST_REPLY):
 	case cpu_to_le32(DLM_MSG_CONVERT_REPLY):
 	case cpu_to_le32(DLM_MSG_GRANT):
-		if (!lkb->lkb_lvbptr)
+		if (!lkb->lkb_lvbptr || !(lkb->lkb_exflags & DLM_LKF_VALBLK))
 			break;
 		memcpy(ms->m_extra, lkb->lkb_lvbptr, r->res_ls->ls_lvblen);
 		break;
@@ -5080,8 +5111,11 @@ void dlm_receive_buffer(union dlm_packet *p, int nodeid)
 	down_read(&ls->ls_recv_active);
 	if (hd->h_cmd == DLM_MSG)
 		dlm_receive_message(ls, &p->message, nodeid);
-	else
+	else if (hd->h_cmd == DLM_RCOM)
 		dlm_receive_rcom(ls, &p->rcom, nodeid);
+	else
+		log_error(ls, "invalid h_cmd %d from %d lockspace %x",
+			  hd->h_cmd, nodeid, le32_to_cpu(hd->u.h_lockspace));
 	up_read(&ls->ls_recv_active);
 
 	dlm_put_lockspace(ls);
@@ -5801,6 +5835,7 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua,
 {
 	struct dlm_lkb *lkb;
 	struct dlm_args args;
+	bool do_put = true;
 	int error;
 
 	dlm_lock_recovery(ls);
@@ -5811,13 +5846,14 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua,
 		goto out;
 	}
 
+	trace_dlm_lock_start(ls, lkb, name, namelen, mode, flags);
+
 	if (flags & DLM_LKF_VALBLK) {
 		ua->lksb.sb_lvbptr = kzalloc(DLM_USER_LVB_LEN, GFP_NOFS);
 		if (!ua->lksb.sb_lvbptr) {
 			kfree(ua);
-			__put_lkb(ls, lkb);
 			error = -ENOMEM;
-			goto out;
+			goto out_put;
 		}
 	}
 #ifdef CONFIG_DLM_DEPRECATED_API
@@ -5831,8 +5867,7 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua,
 		kfree(ua->lksb.sb_lvbptr);
 		ua->lksb.sb_lvbptr = NULL;
 		kfree(ua);
-		__put_lkb(ls, lkb);
-		goto out;
+		goto out_put;
 	}
 
 	/* After ua is attached to lkb it will be freed by dlm_free_lkb().
@@ -5851,8 +5886,7 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua,
 		error = 0;
 		fallthrough;
 	default:
-		__put_lkb(ls, lkb);
-		goto out;
+		goto out_put;
 	}
 
 	/* add this new lkb to the per-process list of locks */
@@ -5860,6 +5894,11 @@ int dlm_user_request(struct dlm_ls *ls, struct dlm_user_args *ua,
 	hold_lkb(lkb);
 	list_add_tail(&lkb->lkb_ownqueue, &ua->proc->locks);
 	spin_unlock(&ua->proc->locks_spin);
+	do_put = false;
+ out_put:
+	trace_dlm_lock_end(ls, lkb, name, namelen, mode, flags, error, false);
+	if (do_put)
+		__put_lkb(ls, lkb);
  out:
 	dlm_unlock_recovery(ls);
 	return error;
@@ -5885,6 +5924,8 @@ int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
 	if (error)
 		goto out;
 
+	trace_dlm_lock_start(ls, lkb, NULL, 0, mode, flags);
+
 	/* user can change the params on its lock when it converts it, or
 	   add an lvb that didn't exist before */
 
@@ -5922,6 +5963,7 @@ int dlm_user_convert(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
 	if (error == -EINPROGRESS || error == -EAGAIN || error == -EDEADLK)
 		error = 0;
  out_put:
+	trace_dlm_lock_end(ls, lkb, NULL, 0, mode, flags, error, false);
 	dlm_put_lkb(lkb);
  out:
 	dlm_unlock_recovery(ls);
@@ -6014,6 +6056,8 @@ int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
 	if (error)
 		goto out;
 
+	trace_dlm_unlock_start(ls, lkb, flags);
+
 	ua = lkb->lkb_ua;
 
 	if (lvb_in && ua->lksb.sb_lvbptr)
@@ -6042,6 +6086,7 @@ int dlm_user_unlock(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
 		list_move(&lkb->lkb_ownqueue, &ua->proc->unlocking);
 	spin_unlock(&ua->proc->locks_spin);
  out_put:
+	trace_dlm_unlock_end(ls, lkb, flags, error);
 	dlm_put_lkb(lkb);
  out:
 	dlm_unlock_recovery(ls);
@@ -6063,6 +6108,8 @@ int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
 	if (error)
 		goto out;
 
+	trace_dlm_unlock_start(ls, lkb, flags);
+
 	ua = lkb->lkb_ua;
 	if (ua_tmp->castparam)
 		ua->castparam = ua_tmp->castparam;
@@ -6080,6 +6127,7 @@ int dlm_user_cancel(struct dlm_ls *ls, struct dlm_user_args *ua_tmp,
 	if (error == -EBUSY)
 		error = 0;
  out_put:
+	trace_dlm_unlock_end(ls, lkb, flags, error);
 	dlm_put_lkb(lkb);
  out:
 	dlm_unlock_recovery(ls);
@@ -6101,6 +6149,8 @@ int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid)
 	if (error)
 		goto out;
 
+	trace_dlm_unlock_start(ls, lkb, flags);
+
 	ua = lkb->lkb_ua;
 
 	error = set_unlock_args(flags, ua, &args);
@@ -6129,6 +6179,7 @@ int dlm_user_deadlock(struct dlm_ls *ls, uint32_t flags, uint32_t lkid)
 	if (error == -EBUSY)
 		error = 0;
  out_put:
+	trace_dlm_unlock_end(ls, lkb, flags, error);
 	dlm_put_lkb(lkb);
  out:
 	dlm_unlock_recovery(ls);
@@ -6184,7 +6235,7 @@ static struct dlm_lkb *del_proc_lock(struct dlm_ls *ls,
 {
 	struct dlm_lkb *lkb = NULL;
 
-	mutex_lock(&ls->ls_clear_proc_locks);
+	spin_lock(&ls->ls_clear_proc_locks);
 	if (list_empty(&proc->locks))
 		goto out;
 
@@ -6196,7 +6247,7 @@ static struct dlm_lkb *del_proc_lock(struct dlm_ls *ls,
 	else
 		lkb->lkb_flags |= DLM_IFL_DEAD;
  out:
-	mutex_unlock(&ls->ls_clear_proc_locks);
+	spin_unlock(&ls->ls_clear_proc_locks);
 	return lkb;
 }
 
@@ -6233,7 +6284,7 @@ void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc)
 		dlm_put_lkb(lkb);
 	}
 
-	mutex_lock(&ls->ls_clear_proc_locks);
+	spin_lock(&ls->ls_clear_proc_locks);
 
 	/* in-progress unlocks */
 	list_for_each_entry_safe(lkb, safe, &proc->unlocking, lkb_ownqueue) {
@@ -6249,7 +6300,7 @@ void dlm_clear_proc_locks(struct dlm_ls *ls, struct dlm_user_proc *proc)
 		dlm_put_lkb(lkb);
 	}
 
-	mutex_unlock(&ls->ls_clear_proc_locks);
+	spin_unlock(&ls->ls_clear_proc_locks);
 	dlm_unlock_recovery(ls);
 }
 
diff --git a/fs/dlm/lock.h b/fs/dlm/lock.h
index a7b6474f009d..40c76b5544da 100644
--- a/fs/dlm/lock.h
+++ b/fs/dlm/lock.h
@@ -36,7 +36,7 @@ static inline void dlm_adjust_timeouts(struct dlm_ls *ls) { }
 int dlm_master_lookup(struct dlm_ls *ls, int nodeid, char *name, int len,
 		      unsigned int flags, int *r_nodeid, int *result);
 
-int dlm_search_rsb_tree(struct rb_root *tree, char *name, int len,
+int dlm_search_rsb_tree(struct rb_root *tree, const void *name, int len,
 			struct dlm_rsb **r_ret);
 
 void dlm_recover_purge(struct dlm_ls *ls);
diff --git a/fs/dlm/lockspace.c b/fs/dlm/lockspace.c
index 3972f4d86c75..bae050df7abf 100644
--- a/fs/dlm/lockspace.c
+++ b/fs/dlm/lockspace.c
@@ -416,7 +416,7 @@ static int new_lockspace(const char *name, const char *cluster,
 	if (namelen > DLM_LOCKSPACE_LEN || namelen == 0)
 		return -EINVAL;
 
-	if (!lvblen || (lvblen % 8))
+	if (lvblen % 8)
 		return -EINVAL;
 
 	if (!try_module_get(THIS_MODULE))
@@ -584,7 +584,7 @@ static int new_lockspace(const char *name, const char *cluster,
 	atomic_set(&ls->ls_requestqueue_cnt, 0);
 	init_waitqueue_head(&ls->ls_requestqueue_wait);
 	mutex_init(&ls->ls_requestqueue_mutex);
-	mutex_init(&ls->ls_clear_proc_locks);
+	spin_lock_init(&ls->ls_clear_proc_locks);
 
 	/* Due backwards compatibility with 3.1 we need to use maximum
 	 * possible dlm message size to be sure the message will fit and
@@ -703,10 +703,11 @@ static int new_lockspace(const char *name, const char *cluster,
 	return error;
 }
 
-int dlm_new_lockspace(const char *name, const char *cluster,
-		      uint32_t flags, int lvblen,
-		      const struct dlm_lockspace_ops *ops, void *ops_arg,
-		      int *ops_result, dlm_lockspace_t **lockspace)
+static int __dlm_new_lockspace(const char *name, const char *cluster,
+			       uint32_t flags, int lvblen,
+			       const struct dlm_lockspace_ops *ops,
+			       void *ops_arg, int *ops_result,
+			       dlm_lockspace_t **lockspace)
 {
 	int error = 0;
 
@@ -732,6 +733,25 @@ int dlm_new_lockspace(const char *name, const char *cluster,
 	return error;
 }
 
+int dlm_new_lockspace(const char *name, const char *cluster, uint32_t flags,
+		      int lvblen, const struct dlm_lockspace_ops *ops,
+		      void *ops_arg, int *ops_result,
+		      dlm_lockspace_t **lockspace)
+{
+	return __dlm_new_lockspace(name, cluster, flags | DLM_LSFL_FS, lvblen,
+				   ops, ops_arg, ops_result, lockspace);
+}
+
+int dlm_new_user_lockspace(const char *name, const char *cluster,
+			   uint32_t flags, int lvblen,
+			   const struct dlm_lockspace_ops *ops,
+			   void *ops_arg, int *ops_result,
+			   dlm_lockspace_t **lockspace)
+{
+	return __dlm_new_lockspace(name, cluster, flags, lvblen, ops,
+				   ops_arg, ops_result, lockspace);
+}
+
 static int lkb_idr_is_local(int id, void *p, void *data)
 {
 	struct dlm_lkb *lkb = p;
diff --git a/fs/dlm/lockspace.h b/fs/dlm/lockspace.h
index 306fc4f4ea15..03f4a4a3a871 100644
--- a/fs/dlm/lockspace.h
+++ b/fs/dlm/lockspace.h
@@ -12,6 +12,14 @@
 #ifndef __LOCKSPACE_DOT_H__
 #define __LOCKSPACE_DOT_H__
 
+/* DLM_LSFL_FS
+ *   The lockspace user is in the kernel (i.e. filesystem).  Enables
+ *   direct bast/cast callbacks.
+ *
+ * internal lockspace flag - will be removed in future
+ */
+#define DLM_LSFL_FS	0x00000004
+
 int dlm_lockspace_init(void);
 void dlm_lockspace_exit(void);
 struct dlm_ls *dlm_find_lockspace_global(uint32_t id);
@@ -20,6 +28,11 @@ struct dlm_ls *dlm_find_lockspace_device(int minor);
 void dlm_put_lockspace(struct dlm_ls *ls);
 void dlm_stop_lockspaces(void);
 void dlm_stop_lockspaces_check(void);
+int dlm_new_user_lockspace(const char *name, const char *cluster,
+			   uint32_t flags, int lvblen,
+			   const struct dlm_lockspace_ops *ops,
+			   void *ops_arg, int *ops_result,
+			   dlm_lockspace_t **lockspace);
 
 #endif				/* __LOCKSPACE_DOT_H__ */
 
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index a4e84e8d94c8..59f64c596233 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -1336,6 +1336,8 @@ struct dlm_msg *dlm_lowcomms_new_msg(int nodeid, int len, gfp_t allocation,
 		return NULL;
 	}
 
+	/* for dlm_lowcomms_commit_msg() */
+	kref_get(&msg->ref);
 	/* we assume if successful commit must called */
 	msg->idx = idx;
 	return msg;
@@ -1375,6 +1377,8 @@ void dlm_lowcomms_commit_msg(struct dlm_msg *msg)
 {
 	_dlm_lowcomms_commit_msg(msg);
 	srcu_read_unlock(&connections_srcu, msg->idx);
+	/* because dlm_lowcomms_new_msg() */
+	kref_put(&msg->ref, dlm_msg_release);
 }
 #endif
 
diff --git a/fs/dlm/netlink.c b/fs/dlm/netlink.c
index 67f68d48d60c..4de4b8651c6c 100644
--- a/fs/dlm/netlink.c
+++ b/fs/dlm/netlink.c
@@ -75,6 +75,7 @@ static struct genl_family family __ro_after_init = {
 	.version	= DLM_GENL_VERSION,
 	.small_ops	= dlm_nl_ops,
 	.n_small_ops	= ARRAY_SIZE(dlm_nl_ops),
+	.resv_start_op	= DLM_CMD_HELLO + 1,
 	.module		= THIS_MODULE,
 };
 
diff --git a/fs/dlm/user.c b/fs/dlm/user.c
index 99e8f0744513..c5d27bccc3dc 100644
--- a/fs/dlm/user.c
+++ b/fs/dlm/user.c
@@ -16,6 +16,8 @@
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
 
+#include <trace/events/dlm.h>
+
 #include "dlm_internal.h"
 #include "lockspace.h"
 #include "lock.h"
@@ -184,7 +186,7 @@ void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode,
 		return;
 
 	ls = lkb->lkb_resource->res_ls;
-	mutex_lock(&ls->ls_clear_proc_locks);
+	spin_lock(&ls->ls_clear_proc_locks);
 
 	/* If ORPHAN/DEAD flag is set, it means the process is dead so an ast
 	   can't be delivered.  For ORPHAN's, dlm_clear_proc_locks() freed
@@ -230,7 +232,7 @@ void dlm_user_add_ast(struct dlm_lkb *lkb, uint32_t flags, int mode,
 		spin_unlock(&proc->locks_spin);
 	}
  out:
-	mutex_unlock(&ls->ls_clear_proc_locks);
+	spin_unlock(&ls->ls_clear_proc_locks);
 }
 
 static int device_user_lock(struct dlm_user_proc *proc,
@@ -421,9 +423,9 @@ static int device_create_lockspace(struct dlm_lspace_params *params)
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
 
-	error = dlm_new_lockspace(params->name, dlm_config.ci_cluster_name, params->flags,
-				  DLM_USER_LVB_LEN, NULL, NULL, NULL,
-				  &lockspace);
+	error = dlm_new_user_lockspace(params->name, dlm_config.ci_cluster_name,
+				       params->flags, DLM_USER_LVB_LEN, NULL,
+				       NULL, NULL, &lockspace);
 	if (error)
 		return error;
 
@@ -882,7 +884,9 @@ static ssize_t device_read(struct file *file, char __user *buf, size_t count,
 		goto try_another;
 	}
 
-	if (cb.flags & DLM_CB_CAST) {
+	if (cb.flags & DLM_CB_BAST) {
+		trace_dlm_bast(lkb->lkb_resource->res_ls, lkb, cb.mode);
+	} else if (cb.flags & DLM_CB_CAST) {
 		new_mode = cb.mode;
 
 		if (!cb.sb_status && lkb->lkb_lksb->sb_lvbptr &&
@@ -891,6 +895,7 @@ static ssize_t device_read(struct file *file, char __user *buf, size_t count,
 
 		lkb->lkb_lksb->sb_status = cb.sb_status;
 		lkb->lkb_lksb->sb_flags = cb.sb_flags;
+		trace_dlm_ast(lkb->lkb_resource->res_ls, lkb);
 	}
 
 	rv = copy_result_to_user(lkb->lkb_ua,
diff --git a/fs/ecryptfs/ecryptfs_kernel.h b/fs/ecryptfs/ecryptfs_kernel.h
index 5f2b49e13731..f2ed0c0266cb 100644
--- a/fs/ecryptfs/ecryptfs_kernel.h
+++ b/fs/ecryptfs/ecryptfs_kernel.h
@@ -506,7 +506,7 @@ ecryptfs_dentry_to_lower(struct dentry *dentry)
 	return ((struct ecryptfs_dentry_info *)dentry->d_fsdata)->lower_path.dentry;
 }
 
-static inline struct path *
+static inline const struct path *
 ecryptfs_dentry_to_lower_path(struct dentry *dentry)
 {
 	return &((struct ecryptfs_dentry_info *)dentry->d_fsdata)->lower_path;
diff --git a/fs/ecryptfs/file.c b/fs/ecryptfs/file.c
index 18d5b91cb573..268b74499c28 100644
--- a/fs/ecryptfs/file.c
+++ b/fs/ecryptfs/file.c
@@ -33,7 +33,7 @@ static ssize_t ecryptfs_read_update_atime(struct kiocb *iocb,
 				struct iov_iter *to)
 {
 	ssize_t rc;
-	struct path *path;
+	const struct path *path;
 	struct file *file = iocb->ki_filp;
 
 	rc = generic_file_read_iter(iocb, to);
@@ -53,7 +53,7 @@ struct ecryptfs_getdents_callback {
 };
 
 /* Inspired by generic filldir in fs/readdir.c */
-static int
+static bool
 ecryptfs_filldir(struct dir_context *ctx, const char *lower_name,
 		 int lower_namelen, loff_t offset, u64 ino, unsigned int d_type)
 {
@@ -61,18 +61,19 @@ ecryptfs_filldir(struct dir_context *ctx, const char *lower_name,
 		container_of(ctx, struct ecryptfs_getdents_callback, ctx);
 	size_t name_size;
 	char *name;
-	int rc;
+	int err;
+	bool res;
 
 	buf->filldir_called++;
-	rc = ecryptfs_decode_and_decrypt_filename(&name, &name_size,
-						  buf->sb, lower_name,
-						  lower_namelen);
-	if (rc) {
-		if (rc != -EINVAL) {
+	err = ecryptfs_decode_and_decrypt_filename(&name, &name_size,
+						   buf->sb, lower_name,
+						   lower_namelen);
+	if (err) {
+		if (err != -EINVAL) {
 			ecryptfs_printk(KERN_DEBUG,
 					"%s: Error attempting to decode and decrypt filename [%s]; rc = [%d]\n",
-					__func__, lower_name, rc);
-			return rc;
+					__func__, lower_name, err);
+			return false;
 		}
 
 		/* Mask -EINVAL errors as these are most likely due a plaintext
@@ -81,16 +82,15 @@ ecryptfs_filldir(struct dir_context *ctx, const char *lower_name,
 		 * the "lost+found" dentry in the root directory of an Ext4
 		 * filesystem.
 		 */
-		return 0;
+		return true;
 	}
 
 	buf->caller->pos = buf->ctx.pos;
-	rc = !dir_emit(buf->caller, name, name_size, ino, d_type);
+	res = dir_emit(buf->caller, name, name_size, ino, d_type);
 	kfree(name);
-	if (!rc)
+	if (res)
 		buf->entries_written++;
-
-	return rc;
+	return res;
 }
 
 /**
@@ -111,14 +111,8 @@ static int ecryptfs_readdir(struct file *file, struct dir_context *ctx)
 	lower_file = ecryptfs_file_to_lower(file);
 	rc = iterate_dir(lower_file, &buf.ctx);
 	ctx->pos = buf.ctx.pos;
-	if (rc < 0)
-		goto out;
-	if (buf.filldir_called && !buf.entries_written)
-		goto out;
-	if (rc >= 0)
-		fsstack_copy_attr_atime(inode,
-					file_inode(lower_file));
-out:
+	if (rc >= 0 && (buf.entries_written || !buf.filldir_called))
+		fsstack_copy_attr_atime(inode, file_inode(lower_file));
 	return rc;
 }
 
diff --git a/fs/ecryptfs/inode.c b/fs/ecryptfs/inode.c
index 16d50dface59..c214fe0981bd 100644
--- a/fs/ecryptfs/inode.c
+++ b/fs/ecryptfs/inode.c
@@ -317,7 +317,7 @@ static int ecryptfs_i_size_read(struct dentry *dentry, struct inode *inode)
 static struct dentry *ecryptfs_lookup_interpose(struct dentry *dentry,
 				     struct dentry *lower_dentry)
 {
-	struct path *path = ecryptfs_dentry_to_lower_path(dentry->d_parent);
+	const struct path *path = ecryptfs_dentry_to_lower_path(dentry->d_parent);
 	struct inode *inode, *lower_inode;
 	struct ecryptfs_dentry_info *dentry_info;
 	int rc = 0;
diff --git a/fs/ecryptfs/main.c b/fs/ecryptfs/main.c
index 2dd23a82e0de..2dc927ba067f 100644
--- a/fs/ecryptfs/main.c
+++ b/fs/ecryptfs/main.c
@@ -105,7 +105,7 @@ static int ecryptfs_init_lower_file(struct dentry *dentry,
 				    struct file **lower_file)
 {
 	const struct cred *cred = current_cred();
-	struct path *path = ecryptfs_dentry_to_lower_path(dentry);
+	const struct path *path = ecryptfs_dentry_to_lower_path(dentry);
 	int rc;
 
 	rc = ecryptfs_privileged_open(lower_file, path->dentry, path->mnt,
diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c
index 2d55569f96ac..51b7ac7166d9 100644
--- a/fs/erofs/decompressor.c
+++ b/fs/erofs/decompressor.c
@@ -317,52 +317,61 @@ dstmap_out:
 	return ret;
 }
 
-static int z_erofs_shifted_transform(struct z_erofs_decompress_req *rq,
-				     struct page **pagepool)
+static int z_erofs_transform_plain(struct z_erofs_decompress_req *rq,
+				   struct page **pagepool)
 {
-	const unsigned int nrpages_out =
+	const unsigned int inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
+	const unsigned int outpages =
 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
 	const unsigned int righthalf = min_t(unsigned int, rq->outputsize,
 					     PAGE_SIZE - rq->pageofs_out);
 	const unsigned int lefthalf = rq->outputsize - righthalf;
+	const unsigned int interlaced_offset =
+		rq->alg == Z_EROFS_COMPRESSION_SHIFTED ? 0 : rq->pageofs_out;
 	unsigned char *src, *dst;
 
-	if (nrpages_out > 2) {
+	if (outpages > 2 && rq->alg == Z_EROFS_COMPRESSION_SHIFTED) {
 		DBG_BUGON(1);
-		return -EIO;
+		return -EFSCORRUPTED;
 	}
 
 	if (rq->out[0] == *rq->in) {
-		DBG_BUGON(nrpages_out != 1);
+		DBG_BUGON(rq->pageofs_out);
 		return 0;
 	}
 
-	src = kmap_atomic(*rq->in) + rq->pageofs_in;
+	src = kmap_local_page(rq->in[inpages - 1]) + rq->pageofs_in;
 	if (rq->out[0]) {
-		dst = kmap_atomic(rq->out[0]);
-		memcpy(dst + rq->pageofs_out, src, righthalf);
-		kunmap_atomic(dst);
+		dst = kmap_local_page(rq->out[0]);
+		memcpy(dst + rq->pageofs_out, src + interlaced_offset,
+		       righthalf);
+		kunmap_local(dst);
 	}
 
-	if (nrpages_out == 2) {
-		DBG_BUGON(!rq->out[1]);
-		if (rq->out[1] == *rq->in) {
+	if (outpages > inpages) {
+		DBG_BUGON(!rq->out[outpages - 1]);
+		if (rq->out[outpages - 1] != rq->in[inpages - 1]) {
+			dst = kmap_local_page(rq->out[outpages - 1]);
+			memcpy(dst, interlaced_offset ? src :
+					(src + righthalf), lefthalf);
+			kunmap_local(dst);
+		} else if (!interlaced_offset) {
 			memmove(src, src + righthalf, lefthalf);
-		} else {
-			dst = kmap_atomic(rq->out[1]);
-			memcpy(dst, src + righthalf, lefthalf);
-			kunmap_atomic(dst);
 		}
 	}
-	kunmap_atomic(src);
+	kunmap_local(src);
 	return 0;
 }
 
 static struct z_erofs_decompressor decompressors[] = {
 	[Z_EROFS_COMPRESSION_SHIFTED] = {
-		.decompress = z_erofs_shifted_transform,
+		.decompress = z_erofs_transform_plain,
 		.name = "shifted"
 	},
+	[Z_EROFS_COMPRESSION_INTERLACED] = {
+		.decompress = z_erofs_transform_plain,
+		.name = "interlaced"
+	},
 	[Z_EROFS_COMPRESSION_LZ4] = {
 		.decompress = z_erofs_lz4_decompress,
 		.name = "lz4"
diff --git a/fs/erofs/decompressor_lzma.c b/fs/erofs/decompressor_lzma.c
index 5e59b3f523eb..091fd5adf818 100644
--- a/fs/erofs/decompressor_lzma.c
+++ b/fs/erofs/decompressor_lzma.c
@@ -217,6 +217,9 @@ again:
 			strm->buf.out_size = min_t(u32, outlen,
 						   PAGE_SIZE - pageofs);
 			outlen -= strm->buf.out_size;
+			if (!rq->out[no] && rq->fillgaps)	/* deduped */
+				rq->out[no] = erofs_allocpage(pagepool,
+						GFP_KERNEL | __GFP_NOFAIL);
 			if (rq->out[no])
 				strm->buf.out = kmap(rq->out[no]) + pageofs;
 			pageofs = 0;
diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h
index 2b48373f690b..dbcd24371002 100644
--- a/fs/erofs/erofs_fs.h
+++ b/fs/erofs/erofs_fs.h
@@ -25,6 +25,8 @@
 #define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE	0x00000008
 #define EROFS_FEATURE_INCOMPAT_COMPR_HEAD2	0x00000008
 #define EROFS_FEATURE_INCOMPAT_ZTAILPACKING	0x00000010
+#define EROFS_FEATURE_INCOMPAT_FRAGMENTS	0x00000020
+#define EROFS_FEATURE_INCOMPAT_DEDUPE		0x00000020
 #define EROFS_ALL_FEATURE_INCOMPAT		\
 	(EROFS_FEATURE_INCOMPAT_ZERO_PADDING | \
 	 EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \
@@ -32,7 +34,9 @@
 	 EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \
 	 EROFS_FEATURE_INCOMPAT_DEVICE_TABLE | \
 	 EROFS_FEATURE_INCOMPAT_COMPR_HEAD2 | \
-	 EROFS_FEATURE_INCOMPAT_ZTAILPACKING)
+	 EROFS_FEATURE_INCOMPAT_ZTAILPACKING | \
+	 EROFS_FEATURE_INCOMPAT_FRAGMENTS | \
+	 EROFS_FEATURE_INCOMPAT_DEDUPE)
 
 #define EROFS_SB_EXTSLOT_SIZE	16
 
@@ -71,7 +75,9 @@ struct erofs_super_block {
 	} __packed u1;
 	__le16 extra_devices;	/* # of devices besides the primary device */
 	__le16 devt_slotoff;	/* startoff = devt_slotoff * devt_slotsize */
-	__u8 reserved2[38];
+	__u8 reserved[6];
+	__le64 packed_nid;	/* nid of the special packed inode */
+	__u8 reserved2[24];
 };
 
 /*
@@ -295,16 +301,27 @@ struct z_erofs_lzma_cfgs {
  * bit 1 : HEAD1 big pcluster (0 - off; 1 - on)
  * bit 2 : HEAD2 big pcluster (0 - off; 1 - on)
  * bit 3 : tailpacking inline pcluster (0 - off; 1 - on)
+ * bit 4 : interlaced plain pcluster (0 - off; 1 - on)
+ * bit 5 : fragment pcluster (0 - off; 1 - on)
  */
 #define Z_EROFS_ADVISE_COMPACTED_2B		0x0001
 #define Z_EROFS_ADVISE_BIG_PCLUSTER_1		0x0002
 #define Z_EROFS_ADVISE_BIG_PCLUSTER_2		0x0004
 #define Z_EROFS_ADVISE_INLINE_PCLUSTER		0x0008
+#define Z_EROFS_ADVISE_INTERLACED_PCLUSTER	0x0010
+#define Z_EROFS_ADVISE_FRAGMENT_PCLUSTER	0x0020
 
+#define Z_EROFS_FRAGMENT_INODE_BIT              7
 struct z_erofs_map_header {
-	__le16	h_reserved1;
-	/* indicates the encoded size of tailpacking data */
-	__le16  h_idata_size;
+	union {
+		/* fragment data offset in the packed inode */
+		__le32  h_fragmentoff;
+		struct {
+			__le16  h_reserved1;
+			/* indicates the encoded size of tailpacking data */
+			__le16  h_idata_size;
+		};
+	};
 	__le16	h_advise;
 	/*
 	 * bit 0-3 : algorithm type of head 1 (logical cluster type 01);
@@ -313,7 +330,8 @@ struct z_erofs_map_header {
 	__u8	h_algorithmtype;
 	/*
 	 * bit 0-2 : logical cluster bits - 12, e.g. 0 for 4096;
-	 * bit 3-7 : reserved.
+	 * bit 3-6 : reserved;
+	 * bit 7   : move the whole file into packed inode or not.
 	 */
 	__u8	h_clusterbits;
 };
@@ -355,6 +373,9 @@ enum {
 #define Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS        2
 #define Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT         0
 
+/* (noncompact only, HEAD) This pcluster refers to partial decompressed data */
+#define Z_EROFS_VLE_DI_PARTIAL_REF		(1 << 15)
+
 /*
  * D0_CBLKCNT will be marked _only_ at the 1st non-head lcluster to store the
  * compressed block count of a compressed extent (in logical clusters, aka.
@@ -402,6 +423,10 @@ struct erofs_dirent {
 /* check the EROFS on-disk layout strictly at compile time */
 static inline void erofs_check_ondisk_layout_definitions(void)
 {
+	const __le64 fmh = *(__le64 *)&(struct z_erofs_map_header) {
+		.h_clusterbits = 1 << Z_EROFS_FRAGMENT_INODE_BIT
+	};
+
 	BUILD_BUG_ON(sizeof(struct erofs_super_block) != 128);
 	BUILD_BUG_ON(sizeof(struct erofs_inode_compact) != 32);
 	BUILD_BUG_ON(sizeof(struct erofs_inode_extended) != 64);
@@ -419,6 +444,9 @@ static inline void erofs_check_ondisk_layout_definitions(void)
 
 	BUILD_BUG_ON(BIT(Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) <
 		     Z_EROFS_VLE_CLUSTER_TYPE_MAX - 1);
+	/* exclude old compiler versions like gcc 7.5.0 */
+	BUILD_BUG_ON(__builtin_constant_p(fmh) ?
+		     fmh != cpu_to_le64(1ULL << 63) : 0);
 }
 
 #endif
diff --git a/fs/erofs/fscache.c b/fs/erofs/fscache.c
index b5fd9d71e67f..998cd26a1b3b 100644
--- a/fs/erofs/fscache.c
+++ b/fs/erofs/fscache.c
@@ -1,10 +1,16 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
  * Copyright (C) 2022, Alibaba Cloud
+ * Copyright (C) 2022, Bytedance Inc. All rights reserved.
  */
 #include <linux/fscache.h>
 #include "internal.h"
 
+static DEFINE_MUTEX(erofs_domain_list_lock);
+static DEFINE_MUTEX(erofs_domain_cookies_lock);
+static LIST_HEAD(erofs_domain_list);
+static struct vfsmount *erofs_pseudo_mnt;
+
 static struct netfs_io_request *erofs_fscache_alloc_request(struct address_space *mapping,
 					     loff_t start, size_t len)
 {
@@ -234,113 +240,111 @@ out:
 	return ret;
 }
 
-static int erofs_fscache_read_folio_inline(struct folio *folio,
-					 struct erofs_map_blocks *map)
-{
-	struct super_block *sb = folio_mapping(folio)->host->i_sb;
-	struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
-	erofs_blk_t blknr;
-	size_t offset, len;
-	void *src, *dst;
-
-	/* For tail packing layout, the offset may be non-zero. */
-	offset = erofs_blkoff(map->m_pa);
-	blknr = erofs_blknr(map->m_pa);
-	len = map->m_llen;
-
-	src = erofs_read_metabuf(&buf, sb, blknr, EROFS_KMAP);
-	if (IS_ERR(src))
-		return PTR_ERR(src);
-
-	dst = kmap_local_folio(folio, 0);
-	memcpy(dst, src + offset, len);
-	memset(dst + len, 0, PAGE_SIZE - len);
-	kunmap_local(dst);
-
-	erofs_put_metabuf(&buf);
-	return 0;
-}
-
-static int erofs_fscache_read_folio(struct file *file, struct folio *folio)
+/*
+ * Read into page cache in the range described by (@pos, @len).
+ *
+ * On return, the caller is responsible for page unlocking if the output @unlock
+ * is true, or the callee will take this responsibility through netfs_io_request
+ * interface.
+ *
+ * The return value is the number of bytes successfully handled, or negative
+ * error code on failure. The only exception is that, the length of the range
+ * instead of the error code is returned on failure after netfs_io_request is
+ * allocated, so that .readahead() could advance rac accordingly.
+ */
+static int erofs_fscache_data_read(struct address_space *mapping,
+				   loff_t pos, size_t len, bool *unlock)
 {
-	struct inode *inode = folio_mapping(folio)->host;
+	struct inode *inode = mapping->host;
 	struct super_block *sb = inode->i_sb;
+	struct netfs_io_request *rreq;
 	struct erofs_map_blocks map;
 	struct erofs_map_dev mdev;
-	struct netfs_io_request *rreq;
-	erofs_off_t pos;
-	loff_t pstart;
+	struct iov_iter iter;
+	size_t count;
 	int ret;
 
-	DBG_BUGON(folio_size(folio) != EROFS_BLKSIZ);
+	*unlock = true;
 
-	pos = folio_pos(folio);
 	map.m_la = pos;
-
 	ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
 	if (ret)
-		goto out_unlock;
+		return ret;
 
-	if (!(map.m_flags & EROFS_MAP_MAPPED)) {
-		folio_zero_range(folio, 0, folio_size(folio));
-		goto out_uptodate;
+	if (map.m_flags & EROFS_MAP_META) {
+		struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
+		erofs_blk_t blknr;
+		size_t offset, size;
+		void *src;
+
+		/* For tail packing layout, the offset may be non-zero. */
+		offset = erofs_blkoff(map.m_pa);
+		blknr = erofs_blknr(map.m_pa);
+		size = map.m_llen;
+
+		src = erofs_read_metabuf(&buf, sb, blknr, EROFS_KMAP);
+		if (IS_ERR(src))
+			return PTR_ERR(src);
+
+		iov_iter_xarray(&iter, READ, &mapping->i_pages, pos, PAGE_SIZE);
+		if (copy_to_iter(src + offset, size, &iter) != size)
+			return -EFAULT;
+		iov_iter_zero(PAGE_SIZE - size, &iter);
+		erofs_put_metabuf(&buf);
+		return PAGE_SIZE;
 	}
 
-	if (map.m_flags & EROFS_MAP_META) {
-		ret = erofs_fscache_read_folio_inline(folio, &map);
-		goto out_uptodate;
+	count = min_t(size_t, map.m_llen - (pos - map.m_la), len);
+	DBG_BUGON(!count || count % PAGE_SIZE);
+
+	if (!(map.m_flags & EROFS_MAP_MAPPED)) {
+		iov_iter_xarray(&iter, READ, &mapping->i_pages, pos, count);
+		iov_iter_zero(count, &iter);
+		return count;
 	}
 
 	mdev = (struct erofs_map_dev) {
 		.m_deviceid = map.m_deviceid,
 		.m_pa = map.m_pa,
 	};
-
 	ret = erofs_map_dev(sb, &mdev);
 	if (ret)
-		goto out_unlock;
+		return ret;
 
+	rreq = erofs_fscache_alloc_request(mapping, pos, count);
+	if (IS_ERR(rreq))
+		return PTR_ERR(rreq);
 
-	rreq = erofs_fscache_alloc_request(folio_mapping(folio),
-				folio_pos(folio), folio_size(folio));
-	if (IS_ERR(rreq)) {
-		ret = PTR_ERR(rreq);
-		goto out_unlock;
-	}
-
-	pstart = mdev.m_pa + (pos - map.m_la);
-	return erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
-				rreq, pstart);
-
-out_uptodate:
-	if (!ret)
-		folio_mark_uptodate(folio);
-out_unlock:
-	folio_unlock(folio);
-	return ret;
+	*unlock = false;
+	erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
+			rreq, mdev.m_pa + (pos - map.m_la));
+	return count;
 }
 
-static void erofs_fscache_advance_folios(struct readahead_control *rac,
-					 size_t len, bool unlock)
+static int erofs_fscache_read_folio(struct file *file, struct folio *folio)
 {
-	while (len) {
-		struct folio *folio = readahead_folio(rac);
-		len -= folio_size(folio);
-		if (unlock) {
+	bool unlock;
+	int ret;
+
+	DBG_BUGON(folio_size(folio) != EROFS_BLKSIZ);
+
+	ret = erofs_fscache_data_read(folio_mapping(folio), folio_pos(folio),
+				      folio_size(folio), &unlock);
+	if (unlock) {
+		if (ret > 0)
 			folio_mark_uptodate(folio);
-			folio_unlock(folio);
-		}
+		folio_unlock(folio);
 	}
+	return ret < 0 ? ret : 0;
 }
 
 static void erofs_fscache_readahead(struct readahead_control *rac)
 {
-	struct inode *inode = rac->mapping->host;
-	struct super_block *sb = inode->i_sb;
-	size_t len, count, done = 0;
-	erofs_off_t pos;
-	loff_t start, offset;
-	int ret;
+	struct folio *folio;
+	size_t len, done = 0;
+	loff_t start, pos;
+	bool unlock;
+	int ret, size;
 
 	if (!readahead_count(rac))
 		return;
@@ -349,67 +353,22 @@ static void erofs_fscache_readahead(struct readahead_control *rac)
 	len = readahead_length(rac);
 
 	do {
-		struct erofs_map_blocks map;
-		struct erofs_map_dev mdev;
-		struct netfs_io_request *rreq;
-
 		pos = start + done;
-		map.m_la = pos;
-
-		ret = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW);
-		if (ret)
+		ret = erofs_fscache_data_read(rac->mapping, pos,
+					      len - done, &unlock);
+		if (ret <= 0)
 			return;
 
-		offset = start + done;
-		count = min_t(size_t, map.m_llen - (pos - map.m_la),
-			      len - done);
-
-		if (!(map.m_flags & EROFS_MAP_MAPPED)) {
-			struct iov_iter iter;
-
-			iov_iter_xarray(&iter, READ, &rac->mapping->i_pages,
-					offset, count);
-			iov_iter_zero(count, &iter);
-
-			erofs_fscache_advance_folios(rac, count, true);
-			ret = count;
-			continue;
-		}
-
-		if (map.m_flags & EROFS_MAP_META) {
-			struct folio *folio = readahead_folio(rac);
-
-			ret = erofs_fscache_read_folio_inline(folio, &map);
-			if (!ret) {
+		size = ret;
+		while (size) {
+			folio = readahead_folio(rac);
+			size -= folio_size(folio);
+			if (unlock) {
 				folio_mark_uptodate(folio);
-				ret = folio_size(folio);
+				folio_unlock(folio);
 			}
-
-			folio_unlock(folio);
-			continue;
 		}
-
-		mdev = (struct erofs_map_dev) {
-			.m_deviceid = map.m_deviceid,
-			.m_pa = map.m_pa,
-		};
-		ret = erofs_map_dev(sb, &mdev);
-		if (ret)
-			return;
-
-		rreq = erofs_fscache_alloc_request(rac->mapping, offset, count);
-		if (IS_ERR(rreq))
-			return;
-		/*
-		 * Drop the ref of folios here. Unlock them in
-		 * rreq_unlock_folios() when rreq complete.
-		 */
-		erofs_fscache_advance_folios(rac, count, false);
-		ret = erofs_fscache_read_folios_async(mdev.m_fscache->cookie,
-					rreq, mdev.m_pa + (pos - map.m_la));
-		if (!ret)
-			ret = count;
-	} while (ret > 0 && ((done += ret) < len));
+	} while ((done += ret) < len);
 }
 
 static const struct address_space_operations erofs_fscache_meta_aops = {
@@ -421,9 +380,114 @@ const struct address_space_operations erofs_fscache_access_aops = {
 	.readahead = erofs_fscache_readahead,
 };
 
-int erofs_fscache_register_cookie(struct super_block *sb,
-				  struct erofs_fscache **fscache,
-				  char *name, bool need_inode)
+static void erofs_fscache_domain_put(struct erofs_domain *domain)
+{
+	if (!domain)
+		return;
+	mutex_lock(&erofs_domain_list_lock);
+	if (refcount_dec_and_test(&domain->ref)) {
+		list_del(&domain->list);
+		if (list_empty(&erofs_domain_list)) {
+			kern_unmount(erofs_pseudo_mnt);
+			erofs_pseudo_mnt = NULL;
+		}
+		mutex_unlock(&erofs_domain_list_lock);
+		fscache_relinquish_volume(domain->volume, NULL, false);
+		kfree(domain->domain_id);
+		kfree(domain);
+		return;
+	}
+	mutex_unlock(&erofs_domain_list_lock);
+}
+
+static int erofs_fscache_register_volume(struct super_block *sb)
+{
+	struct erofs_sb_info *sbi = EROFS_SB(sb);
+	char *domain_id = sbi->opt.domain_id;
+	struct fscache_volume *volume;
+	char *name;
+	int ret = 0;
+
+	name = kasprintf(GFP_KERNEL, "erofs,%s",
+			 domain_id ? domain_id : sbi->opt.fsid);
+	if (!name)
+		return -ENOMEM;
+
+	volume = fscache_acquire_volume(name, NULL, NULL, 0);
+	if (IS_ERR_OR_NULL(volume)) {
+		erofs_err(sb, "failed to register volume for %s", name);
+		ret = volume ? PTR_ERR(volume) : -EOPNOTSUPP;
+		volume = NULL;
+	}
+
+	sbi->volume = volume;
+	kfree(name);
+	return ret;
+}
+
+static int erofs_fscache_init_domain(struct super_block *sb)
+{
+	int err;
+	struct erofs_domain *domain;
+	struct erofs_sb_info *sbi = EROFS_SB(sb);
+
+	domain = kzalloc(sizeof(struct erofs_domain), GFP_KERNEL);
+	if (!domain)
+		return -ENOMEM;
+
+	domain->domain_id = kstrdup(sbi->opt.domain_id, GFP_KERNEL);
+	if (!domain->domain_id) {
+		kfree(domain);
+		return -ENOMEM;
+	}
+
+	err = erofs_fscache_register_volume(sb);
+	if (err)
+		goto out;
+
+	if (!erofs_pseudo_mnt) {
+		erofs_pseudo_mnt = kern_mount(&erofs_fs_type);
+		if (IS_ERR(erofs_pseudo_mnt)) {
+			err = PTR_ERR(erofs_pseudo_mnt);
+			goto out;
+		}
+	}
+
+	domain->volume = sbi->volume;
+	refcount_set(&domain->ref, 1);
+	list_add(&domain->list, &erofs_domain_list);
+	sbi->domain = domain;
+	return 0;
+out:
+	kfree(domain->domain_id);
+	kfree(domain);
+	return err;
+}
+
+static int erofs_fscache_register_domain(struct super_block *sb)
+{
+	int err;
+	struct erofs_domain *domain;
+	struct erofs_sb_info *sbi = EROFS_SB(sb);
+
+	mutex_lock(&erofs_domain_list_lock);
+	list_for_each_entry(domain, &erofs_domain_list, list) {
+		if (!strcmp(domain->domain_id, sbi->opt.domain_id)) {
+			sbi->domain = domain;
+			sbi->volume = domain->volume;
+			refcount_inc(&domain->ref);
+			mutex_unlock(&erofs_domain_list_lock);
+			return 0;
+		}
+	}
+	err = erofs_fscache_init_domain(sb);
+	mutex_unlock(&erofs_domain_list_lock);
+	return err;
+}
+
+static
+struct erofs_fscache *erofs_fscache_acquire_cookie(struct super_block *sb,
+						    char *name, bool need_inode)
 {
 	struct fscache_volume *volume = EROFS_SB(sb)->volume;
 	struct erofs_fscache *ctx;
@@ -432,7 +496,7 @@ int erofs_fscache_register_cookie(struct super_block *sb,
 
 	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 
 	cookie = fscache_acquire_cookie(volume, FSCACHE_ADV_WANT_CACHE_SIZE,
 					name, strlen(name), NULL, 0, 0);
@@ -462,63 +526,146 @@ int erofs_fscache_register_cookie(struct super_block *sb,
 		ctx->inode = inode;
 	}
 
-	*fscache = ctx;
-	return 0;
+	return ctx;
 
 err_cookie:
 	fscache_unuse_cookie(ctx->cookie, NULL, NULL);
 	fscache_relinquish_cookie(ctx->cookie, false);
-	ctx->cookie = NULL;
 err:
 	kfree(ctx);
-	return ret;
+	return ERR_PTR(ret);
 }
 
-void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache)
+static void erofs_fscache_relinquish_cookie(struct erofs_fscache *ctx)
 {
-	struct erofs_fscache *ctx = *fscache;
-
-	if (!ctx)
-		return;
-
 	fscache_unuse_cookie(ctx->cookie, NULL, NULL);
 	fscache_relinquish_cookie(ctx->cookie, false);
-	ctx->cookie = NULL;
-
 	iput(ctx->inode);
-	ctx->inode = NULL;
-
+	kfree(ctx->name);
 	kfree(ctx);
-	*fscache = NULL;
+}
+
+static
+struct erofs_fscache *erofs_fscache_domain_init_cookie(struct super_block *sb,
+		char *name, bool need_inode)
+{
+	int err;
+	struct inode *inode;
+	struct erofs_fscache *ctx;
+	struct erofs_domain *domain = EROFS_SB(sb)->domain;
+
+	ctx = erofs_fscache_acquire_cookie(sb, name, need_inode);
+	if (IS_ERR(ctx))
+		return ctx;
+
+	ctx->name = kstrdup(name, GFP_KERNEL);
+	if (!ctx->name) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	inode = new_inode(erofs_pseudo_mnt->mnt_sb);
+	if (!inode) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	ctx->domain = domain;
+	ctx->anon_inode = inode;
+	inode->i_private = ctx;
+	refcount_inc(&domain->ref);
+	return ctx;
+out:
+	erofs_fscache_relinquish_cookie(ctx);
+	return ERR_PTR(err);
+}
+
+static
+struct erofs_fscache *erofs_domain_register_cookie(struct super_block *sb,
+						   char *name, bool need_inode)
+{
+	struct inode *inode;
+	struct erofs_fscache *ctx;
+	struct erofs_domain *domain = EROFS_SB(sb)->domain;
+	struct super_block *psb = erofs_pseudo_mnt->mnt_sb;
+
+	mutex_lock(&erofs_domain_cookies_lock);
+	list_for_each_entry(inode, &psb->s_inodes, i_sb_list) {
+		ctx = inode->i_private;
+		if (!ctx || ctx->domain != domain || strcmp(ctx->name, name))
+			continue;
+		igrab(inode);
+		mutex_unlock(&erofs_domain_cookies_lock);
+		return ctx;
+	}
+	ctx = erofs_fscache_domain_init_cookie(sb, name, need_inode);
+	mutex_unlock(&erofs_domain_cookies_lock);
+	return ctx;
+}
+
+struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb,
+						    char *name, bool need_inode)
+{
+	if (EROFS_SB(sb)->opt.domain_id)
+		return erofs_domain_register_cookie(sb, name, need_inode);
+	return erofs_fscache_acquire_cookie(sb, name, need_inode);
+}
+
+void erofs_fscache_unregister_cookie(struct erofs_fscache *ctx)
+{
+	bool drop;
+	struct erofs_domain *domain;
+
+	if (!ctx)
+		return;
+	domain = ctx->domain;
+	if (domain) {
+		mutex_lock(&erofs_domain_cookies_lock);
+		drop = atomic_read(&ctx->anon_inode->i_count) == 1;
+		iput(ctx->anon_inode);
+		mutex_unlock(&erofs_domain_cookies_lock);
+		if (!drop)
+			return;
+	}
+
+	erofs_fscache_relinquish_cookie(ctx);
+	erofs_fscache_domain_put(domain);
 }
 
 int erofs_fscache_register_fs(struct super_block *sb)
 {
+	int ret;
 	struct erofs_sb_info *sbi = EROFS_SB(sb);
-	struct fscache_volume *volume;
-	char *name;
-	int ret = 0;
+	struct erofs_fscache *fscache;
 
-	name = kasprintf(GFP_KERNEL, "erofs,%s", sbi->opt.fsid);
-	if (!name)
-		return -ENOMEM;
+	if (sbi->opt.domain_id)
+		ret = erofs_fscache_register_domain(sb);
+	else
+		ret = erofs_fscache_register_volume(sb);
+	if (ret)
+		return ret;
 
-	volume = fscache_acquire_volume(name, NULL, NULL, 0);
-	if (IS_ERR_OR_NULL(volume)) {
-		erofs_err(sb, "failed to register volume for %s", name);
-		ret = volume ? PTR_ERR(volume) : -EOPNOTSUPP;
-		volume = NULL;
-	}
+	/* acquired domain/volume will be relinquished in kill_sb() on error */
+	fscache = erofs_fscache_register_cookie(sb, sbi->opt.fsid, true);
+	if (IS_ERR(fscache))
+		return PTR_ERR(fscache);
 
-	sbi->volume = volume;
-	kfree(name);
-	return ret;
+	sbi->s_fscache = fscache;
+	return 0;
 }
 
 void erofs_fscache_unregister_fs(struct super_block *sb)
 {
 	struct erofs_sb_info *sbi = EROFS_SB(sb);
 
-	fscache_relinquish_volume(sbi->volume, NULL, false);
+	erofs_fscache_unregister_cookie(sbi->s_fscache);
+
+	if (sbi->domain)
+		erofs_fscache_domain_put(sbi->domain);
+	else
+		fscache_relinquish_volume(sbi->volume, NULL, false);
+
+	sbi->s_fscache = NULL;
 	sbi->volume = NULL;
+	sbi->domain = NULL;
 }
diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c
index 95a403720e8c..ad2a82f2eb4c 100644
--- a/fs/erofs/inode.c
+++ b/fs/erofs/inode.c
@@ -214,7 +214,7 @@ static int erofs_fill_symlink(struct inode *inode, void *kaddr,
 
 	/* if it cannot be handled with fast symlink scheme */
 	if (vi->datalayout != EROFS_INODE_FLAT_INLINE ||
-	    inode->i_size >= EROFS_BLKSIZ) {
+	    inode->i_size >= EROFS_BLKSIZ || inode->i_size < 0) {
 		inode->i_op = &erofs_symlink_iops;
 		return 0;
 	}
@@ -241,7 +241,7 @@ static int erofs_fill_symlink(struct inode *inode, void *kaddr,
 	return 0;
 }
 
-static int erofs_fill_inode(struct inode *inode, int isdir)
+static int erofs_fill_inode(struct inode *inode)
 {
 	struct erofs_inode *vi = EROFS_I(inode);
 	struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
@@ -249,7 +249,7 @@ static int erofs_fill_inode(struct inode *inode, int isdir)
 	unsigned int ofs;
 	int err = 0;
 
-	trace_erofs_fill_inode(inode, isdir);
+	trace_erofs_fill_inode(inode);
 
 	/* read inode base data from disk */
 	kaddr = erofs_read_inode(&buf, inode, &ofs);
@@ -324,21 +324,13 @@ static int erofs_iget_set_actor(struct inode *inode, void *opaque)
 	return 0;
 }
 
-static inline struct inode *erofs_iget_locked(struct super_block *sb,
-					      erofs_nid_t nid)
+struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid)
 {
 	const unsigned long hashval = erofs_inode_hash(nid);
+	struct inode *inode;
 
-	return iget5_locked(sb, hashval, erofs_ilookup_test_actor,
+	inode = iget5_locked(sb, hashval, erofs_ilookup_test_actor,
 		erofs_iget_set_actor, &nid);
-}
-
-struct inode *erofs_iget(struct super_block *sb,
-			 erofs_nid_t nid,
-			 bool isdir)
-{
-	struct inode *inode = erofs_iget_locked(sb, nid);
-
 	if (!inode)
 		return ERR_PTR(-ENOMEM);
 
@@ -348,10 +340,10 @@ struct inode *erofs_iget(struct super_block *sb,
 
 		vi->nid = nid;
 
-		err = erofs_fill_inode(inode, isdir);
-		if (!err)
+		err = erofs_fill_inode(inode);
+		if (!err) {
 			unlock_new_inode(inode);
-		else {
+		} else {
 			iget_failed(inode);
 			inode = ERR_PTR(err);
 		}
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index a01cc82795a2..1701df48c446 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -76,6 +76,7 @@ struct erofs_mount_opts {
 #endif
 	unsigned int mount_opt;
 	char *fsid;
+	char *domain_id;
 };
 
 struct erofs_dev_context {
@@ -98,9 +99,19 @@ struct erofs_sb_lz4_info {
 	u16 max_pclusterblks;
 };
 
+struct erofs_domain {
+	refcount_t ref;
+	struct list_head list;
+	struct fscache_volume *volume;
+	char *domain_id;
+};
+
 struct erofs_fscache {
 	struct fscache_cookie *cookie;
 	struct inode *inode;
+	struct inode *anon_inode;
+	struct erofs_domain *domain;
+	char *name;
 };
 
 struct erofs_sb_info {
@@ -120,6 +131,7 @@ struct erofs_sb_info {
 	struct inode *managed_cache;
 
 	struct erofs_sb_lz4_info lz4;
+	struct inode *packed_inode;
 #endif	/* CONFIG_EROFS_FS_ZIP */
 	struct erofs_dev_context *devs;
 	struct dax_device *dax_dev;
@@ -157,6 +169,7 @@ struct erofs_sb_info {
 	/* fscache support */
 	struct fscache_volume *volume;
 	struct erofs_fscache *s_fscache;
+	struct erofs_domain *domain;
 };
 
 #define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info)
@@ -183,7 +196,6 @@ enum {
 	EROFS_ZIP_CACHE_READAROUND
 };
 
-#ifdef CONFIG_EROFS_FS_ZIP
 #define EROFS_LOCKED_MAGIC     (INT_MIN | 0xE0F510CCL)
 
 /* basic unit of the workstation of a super_block */
@@ -223,7 +235,6 @@ static inline int erofs_wait_on_workgroup_freezed(struct erofs_workgroup *grp)
 	return atomic_cond_read_relaxed(&grp->refcount,
 					VAL != EROFS_LOCKED_MAGIC);
 }
-#endif	/* !CONFIG_EROFS_FS_ZIP */
 
 /* we strictly follow PAGE_SIZE and no buffer head yet */
 #define LOG_BLOCK_SIZE		PAGE_SHIFT
@@ -277,6 +288,8 @@ EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE)
 EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE)
 EROFS_FEATURE_FUNCS(compr_head2, incompat, INCOMPAT_COMPR_HEAD2)
 EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING)
+EROFS_FEATURE_FUNCS(fragments, incompat, INCOMPAT_FRAGMENTS)
+EROFS_FEATURE_FUNCS(dedupe, incompat, INCOMPAT_DEDUPE)
 EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM)
 
 /* atomic flag definitions */
@@ -312,8 +325,13 @@ struct erofs_inode {
 			unsigned char  z_algorithmtype[2];
 			unsigned char  z_logical_clusterbits;
 			unsigned long  z_tailextent_headlcn;
-			erofs_off_t    z_idataoff;
-			unsigned short z_idata_size;
+			union {
+				struct {
+					erofs_off_t    z_idataoff;
+					unsigned short z_idata_size;
+				};
+				erofs_off_t z_fragmentoff;
+			};
 		};
 #endif	/* CONFIG_EROFS_FS_ZIP */
 	};
@@ -364,6 +382,7 @@ struct page *erofs_grab_cache_page_nowait(struct address_space *mapping,
 }
 
 extern const struct super_operations erofs_sops;
+extern struct file_system_type erofs_fs_type;
 
 extern const struct address_space_operations erofs_raw_access_aops;
 extern const struct address_space_operations z_erofs_aops;
@@ -371,6 +390,8 @@ extern const struct address_space_operations z_erofs_aops;
 enum {
 	BH_Encoded = BH_PrivateStart,
 	BH_FullMapped,
+	BH_Fragment,
+	BH_Partialref,
 };
 
 /* Has a disk mapping */
@@ -381,6 +402,10 @@ enum {
 #define EROFS_MAP_ENCODED	(1 << BH_Encoded)
 /* The length of extent is full */
 #define EROFS_MAP_FULL_MAPPED	(1 << BH_FullMapped)
+/* Located in the special packed inode */
+#define EROFS_MAP_FRAGMENT	(1 << BH_Fragment)
+/* The extent refers to partial decompressed data */
+#define EROFS_MAP_PARTIAL_REF	(1 << BH_Partialref)
 
 struct erofs_map_blocks {
 	struct erofs_buf buf;
@@ -402,11 +427,12 @@ struct erofs_map_blocks {
 #define EROFS_GET_BLOCKS_FIEMAP	0x0002
 /* Used to map the whole extent if non-negligible data is requested for LZMA */
 #define EROFS_GET_BLOCKS_READMORE	0x0004
-/* Used to map tail extent for tailpacking inline pcluster */
+/* Used to map tail extent for tailpacking inline or fragment pcluster */
 #define EROFS_GET_BLOCKS_FINDTAIL	0x0008
 
 enum {
 	Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX,
+	Z_EROFS_COMPRESSION_INTERLACED,
 	Z_EROFS_COMPRESSION_RUNTIME_MAX
 };
 
@@ -466,7 +492,7 @@ extern const struct inode_operations erofs_generic_iops;
 extern const struct inode_operations erofs_symlink_iops;
 extern const struct inode_operations erofs_fast_symlink_iops;
 
-struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid, bool dir);
+struct inode *erofs_iget(struct super_block *sb, erofs_nid_t nid);
 int erofs_getattr(struct user_namespace *mnt_userns, const struct path *path,
 		  struct kstat *stat, u32 request_mask,
 		  unsigned int query_flags);
@@ -581,27 +607,26 @@ static inline int z_erofs_load_lzma_config(struct super_block *sb,
 int erofs_fscache_register_fs(struct super_block *sb);
 void erofs_fscache_unregister_fs(struct super_block *sb);
 
-int erofs_fscache_register_cookie(struct super_block *sb,
-				  struct erofs_fscache **fscache,
-				  char *name, bool need_inode);
-void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache);
+struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb,
+						     char *name, bool need_inode);
+void erofs_fscache_unregister_cookie(struct erofs_fscache *fscache);
 
 extern const struct address_space_operations erofs_fscache_access_aops;
 #else
 static inline int erofs_fscache_register_fs(struct super_block *sb)
 {
-	return 0;
+	return -EOPNOTSUPP;
 }
 static inline void erofs_fscache_unregister_fs(struct super_block *sb) {}
 
-static inline int erofs_fscache_register_cookie(struct super_block *sb,
-						struct erofs_fscache **fscache,
-						char *name, bool need_inode)
+static inline
+struct erofs_fscache *erofs_fscache_register_cookie(struct super_block *sb,
+						     char *name, bool need_inode)
 {
-	return -EOPNOTSUPP;
+	return ERR_PTR(-EOPNOTSUPP);
 }
 
-static inline void erofs_fscache_unregister_cookie(struct erofs_fscache **fscache)
+static inline void erofs_fscache_unregister_cookie(struct erofs_fscache *fscache)
 {
 }
 #endif
diff --git a/fs/erofs/namei.c b/fs/erofs/namei.c
index fd75506799c4..0dc34721080c 100644
--- a/fs/erofs/namei.c
+++ b/fs/erofs/namei.c
@@ -185,7 +185,6 @@ int erofs_namei(struct inode *dir, const struct qstr *name, erofs_nid_t *nid,
 	if (IS_ERR(de))
 		return PTR_ERR(de);
 
-	/* the target page has been mapped */
 	if (ndirents)
 		de = find_target_dirent(&qn, (u8 *)de, EROFS_BLKSIZ, ndirents);
 
@@ -197,9 +196,7 @@ int erofs_namei(struct inode *dir, const struct qstr *name, erofs_nid_t *nid,
 	return PTR_ERR_OR_ZERO(de);
 }
 
-/* NOTE: i_mutex is already held by vfs */
-static struct dentry *erofs_lookup(struct inode *dir,
-				   struct dentry *dentry,
+static struct dentry *erofs_lookup(struct inode *dir, struct dentry *dentry,
 				   unsigned int flags)
 {
 	int err;
@@ -207,17 +204,11 @@ static struct dentry *erofs_lookup(struct inode *dir,
 	unsigned int d_type;
 	struct inode *inode;
 
-	DBG_BUGON(!d_really_is_negative(dentry));
-	/* dentry must be unhashed in lookup, no need to worry about */
-	DBG_BUGON(!d_unhashed(dentry));
-
 	trace_erofs_lookup(dir, dentry, flags);
 
-	/* file name exceeds fs limit */
 	if (dentry->d_name.len > EROFS_NAME_LEN)
 		return ERR_PTR(-ENAMETOOLONG);
 
-	/* false uninitialized warnings on gcc 4.8.x */
 	err = erofs_namei(dir, &dentry->d_name, &nid, &d_type);
 
 	if (err == -ENOENT) {
@@ -228,7 +219,7 @@ static struct dentry *erofs_lookup(struct inode *dir,
 	} else {
 		erofs_dbg("%s, %pd (nid %llu) found, d_type %u", __func__,
 			  dentry, nid, d_type);
-		inode = erofs_iget(dir->i_sb, nid, d_type == FT_DIR);
+		inode = erofs_iget(dir->i_sb, nid);
 	}
 	return d_splice_alias(inode, dentry);
 }
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 3173debeaa5a..2cf96ce1c32e 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -224,10 +224,10 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb,
 			     struct erofs_device_info *dif, erofs_off_t *pos)
 {
 	struct erofs_sb_info *sbi = EROFS_SB(sb);
+	struct erofs_fscache *fscache;
 	struct erofs_deviceslot *dis;
 	struct block_device *bdev;
 	void *ptr;
-	int ret;
 
 	ptr = erofs_read_metabuf(buf, sb, erofs_blknr(*pos), EROFS_KMAP);
 	if (IS_ERR(ptr))
@@ -245,10 +245,10 @@ static int erofs_init_device(struct erofs_buf *buf, struct super_block *sb,
 	}
 
 	if (erofs_is_fscache_mode(sb)) {
-		ret = erofs_fscache_register_cookie(sb, &dif->fscache,
-				dif->path, false);
-		if (ret)
-			return ret;
+		fscache = erofs_fscache_register_cookie(sb, dif->path, false);
+		if (IS_ERR(fscache))
+			return PTR_ERR(fscache);
+		dif->fscache = fscache;
 	} else {
 		bdev = blkdev_get_by_path(dif->path, FMODE_READ | FMODE_EXCL,
 					  sb->s_type);
@@ -381,6 +381,17 @@ static int erofs_read_superblock(struct super_block *sb)
 #endif
 	sbi->islotbits = ilog2(sizeof(struct erofs_inode_compact));
 	sbi->root_nid = le16_to_cpu(dsb->root_nid);
+#ifdef CONFIG_EROFS_FS_ZIP
+	sbi->packed_inode = NULL;
+	if (erofs_sb_has_fragments(sbi) && dsb->packed_nid) {
+		sbi->packed_inode =
+			erofs_iget(sb, le64_to_cpu(dsb->packed_nid));
+		if (IS_ERR(sbi->packed_inode)) {
+			ret = PTR_ERR(sbi->packed_inode);
+			goto out;
+		}
+	}
+#endif
 	sbi->inos = le64_to_cpu(dsb->inos);
 
 	sbi->build_time = le64_to_cpu(dsb->build_time);
@@ -411,6 +422,10 @@ static int erofs_read_superblock(struct super_block *sb)
 		erofs_info(sb, "EXPERIMENTAL compressed inline data feature in use. Use at your own risk!");
 	if (erofs_is_fscache_mode(sb))
 		erofs_info(sb, "EXPERIMENTAL fscache-based on-demand read feature in use. Use at your own risk!");
+	if (erofs_sb_has_fragments(sbi))
+		erofs_info(sb, "EXPERIMENTAL compressed fragments feature in use. Use at your own risk!");
+	if (erofs_sb_has_dedupe(sbi))
+		erofs_info(sb, "EXPERIMENTAL global deduplication feature in use. Use at your own risk!");
 out:
 	erofs_put_metabuf(&buf);
 	return ret;
@@ -440,6 +455,7 @@ enum {
 	Opt_dax_enum,
 	Opt_device,
 	Opt_fsid,
+	Opt_domain_id,
 	Opt_err
 };
 
@@ -465,6 +481,7 @@ static const struct fs_parameter_spec erofs_fs_parameters[] = {
 	fsparam_enum("dax",		Opt_dax_enum, erofs_dax_param_enums),
 	fsparam_string("device",	Opt_device),
 	fsparam_string("fsid",		Opt_fsid),
+	fsparam_string("domain_id",	Opt_domain_id),
 	{}
 };
 
@@ -570,6 +587,16 @@ static int erofs_fc_parse_param(struct fs_context *fc,
 		errorfc(fc, "fsid option not supported");
 #endif
 		break;
+	case Opt_domain_id:
+#ifdef CONFIG_EROFS_FS_ONDEMAND
+		kfree(ctx->opt.domain_id);
+		ctx->opt.domain_id = kstrdup(param->string, GFP_KERNEL);
+		if (!ctx->opt.domain_id)
+			return -ENOMEM;
+#else
+		errorfc(fc, "domain_id option not supported");
+#endif
+		break;
 	default:
 		return -ENOPARAM;
 	}
@@ -641,7 +668,7 @@ static int erofs_init_managed_cache(struct super_block *sb) { return 0; }
 static struct inode *erofs_nfs_get_inode(struct super_block *sb,
 					 u64 ino, u32 generation)
 {
-	return erofs_iget(sb, ino, false);
+	return erofs_iget(sb, ino);
 }
 
 static struct dentry *erofs_fh_to_dentry(struct super_block *sb,
@@ -667,7 +694,7 @@ static struct dentry *erofs_get_parent(struct dentry *child)
 	err = erofs_namei(d_inode(child), &dotdot_name, &nid, &d_type);
 	if (err)
 		return ERR_PTR(err);
-	return d_obtain_alias(erofs_iget(child->d_sb, nid, d_type == FT_DIR));
+	return d_obtain_alias(erofs_iget(child->d_sb, nid));
 }
 
 static const struct export_operations erofs_export_ops = {
@@ -676,6 +703,13 @@ static const struct export_operations erofs_export_ops = {
 	.get_parent = erofs_get_parent,
 };
 
+static int erofs_fc_fill_pseudo_super(struct super_block *sb, struct fs_context *fc)
+{
+	static const struct tree_descr empty_descr = {""};
+
+	return simple_fill_super(sb, EROFS_SUPER_MAGIC, &empty_descr);
+}
+
 static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
 {
 	struct inode *inode;
@@ -695,6 +729,7 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
 	sb->s_fs_info = sbi;
 	sbi->opt = ctx->opt;
 	ctx->opt.fsid = NULL;
+	ctx->opt.domain_id = NULL;
 	sbi->devs = ctx->devs;
 	ctx->devs = NULL;
 
@@ -706,11 +741,6 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
 		if (err)
 			return err;
 
-		err = erofs_fscache_register_cookie(sb, &sbi->s_fscache,
-						    sbi->opt.fsid, true);
-		if (err)
-			return err;
-
 		err = super_setup_bdi(sb);
 		if (err)
 			return err;
@@ -752,7 +782,7 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
 #endif
 
 	/* get the root inode */
-	inode = erofs_iget(sb, ROOT_NID(sbi), true);
+	inode = erofs_iget(sb, ROOT_NID(sbi));
 	if (IS_ERR(inode))
 		return PTR_ERR(inode);
 
@@ -781,6 +811,11 @@ static int erofs_fc_fill_super(struct super_block *sb, struct fs_context *fc)
 	return 0;
 }
 
+static int erofs_fc_anon_get_tree(struct fs_context *fc)
+{
+	return get_tree_nodev(fc, erofs_fc_fill_pseudo_super);
+}
+
 static int erofs_fc_get_tree(struct fs_context *fc)
 {
 	struct erofs_fs_context *ctx = fc->fs_private;
@@ -817,7 +852,8 @@ static int erofs_release_device_info(int id, void *ptr, void *data)
 	fs_put_dax(dif->dax_dev, NULL);
 	if (dif->bdev)
 		blkdev_put(dif->bdev, FMODE_READ | FMODE_EXCL);
-	erofs_fscache_unregister_cookie(&dif->fscache);
+	erofs_fscache_unregister_cookie(dif->fscache);
+	dif->fscache = NULL;
 	kfree(dif->path);
 	kfree(dif);
 	return 0;
@@ -838,6 +874,7 @@ static void erofs_fc_free(struct fs_context *fc)
 
 	erofs_free_dev_context(ctx->devs);
 	kfree(ctx->opt.fsid);
+	kfree(ctx->opt.domain_id);
 	kfree(ctx);
 }
 
@@ -848,10 +885,21 @@ static const struct fs_context_operations erofs_context_ops = {
 	.free		= erofs_fc_free,
 };
 
+static const struct fs_context_operations erofs_anon_context_ops = {
+	.get_tree       = erofs_fc_anon_get_tree,
+};
+
 static int erofs_init_fs_context(struct fs_context *fc)
 {
-	struct erofs_fs_context *ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	struct erofs_fs_context *ctx;
+
+	/* pseudo mount for anon inodes */
+	if (fc->sb_flags & SB_KERNMOUNT) {
+		fc->ops = &erofs_anon_context_ops;
+		return 0;
+	}
 
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 	if (!ctx)
 		return -ENOMEM;
 	ctx->devs = kzalloc(sizeof(struct erofs_dev_context), GFP_KERNEL);
@@ -878,8 +926,14 @@ static void erofs_kill_sb(struct super_block *sb)
 
 	WARN_ON(sb->s_magic != EROFS_SUPER_MAGIC);
 
+	/* pseudo mount for anon inodes */
+	if (sb->s_flags & SB_KERNMOUNT) {
+		kill_anon_super(sb);
+		return;
+	}
+
 	if (erofs_is_fscache_mode(sb))
-		generic_shutdown_super(sb);
+		kill_anon_super(sb);
 	else
 		kill_block_super(sb);
 
@@ -889,9 +943,9 @@ static void erofs_kill_sb(struct super_block *sb)
 
 	erofs_free_dev_context(sbi->devs);
 	fs_put_dax(sbi->dax_dev, NULL);
-	erofs_fscache_unregister_cookie(&sbi->s_fscache);
 	erofs_fscache_unregister_fs(sb);
 	kfree(sbi->opt.fsid);
+	kfree(sbi->opt.domain_id);
 	kfree(sbi);
 	sb->s_fs_info = NULL;
 }
@@ -908,11 +962,13 @@ static void erofs_put_super(struct super_block *sb)
 #ifdef CONFIG_EROFS_FS_ZIP
 	iput(sbi->managed_cache);
 	sbi->managed_cache = NULL;
+	iput(sbi->packed_inode);
+	sbi->packed_inode = NULL;
 #endif
-	erofs_fscache_unregister_cookie(&sbi->s_fscache);
+	erofs_fscache_unregister_fs(sb);
 }
 
-static struct file_system_type erofs_fs_type = {
+struct file_system_type erofs_fs_type = {
 	.owner          = THIS_MODULE,
 	.name           = "erofs",
 	.init_fs_context = erofs_init_fs_context,
@@ -1044,6 +1100,8 @@ static int erofs_show_options(struct seq_file *seq, struct dentry *root)
 #ifdef CONFIG_EROFS_FS_ONDEMAND
 	if (opt->fsid)
 		seq_printf(seq, ",fsid=%s", opt->fsid);
+	if (opt->domain_id)
+		seq_printf(seq, ",domain_id=%s", opt->domain_id);
 #endif
 	return 0;
 }
diff --git a/fs/erofs/sysfs.c b/fs/erofs/sysfs.c
index c1383e508bbe..783bb7b21b51 100644
--- a/fs/erofs/sysfs.c
+++ b/fs/erofs/sysfs.c
@@ -76,6 +76,8 @@ EROFS_ATTR_FEATURE(device_table);
 EROFS_ATTR_FEATURE(compr_head2);
 EROFS_ATTR_FEATURE(sb_chksum);
 EROFS_ATTR_FEATURE(ztailpacking);
+EROFS_ATTR_FEATURE(fragments);
+EROFS_ATTR_FEATURE(dedupe);
 
 static struct attribute *erofs_feat_attrs[] = {
 	ATTR_LIST(zero_padding),
@@ -86,6 +88,8 @@ static struct attribute *erofs_feat_attrs[] = {
 	ATTR_LIST(compr_head2),
 	ATTR_LIST(sb_chksum),
 	ATTR_LIST(ztailpacking),
+	ATTR_LIST(fragments),
+	ATTR_LIST(dedupe),
 	NULL,
 };
 ATTRIBUTE_GROUPS(erofs_feat);
@@ -201,12 +205,27 @@ static struct kobject erofs_feat = {
 int erofs_register_sysfs(struct super_block *sb)
 {
 	struct erofs_sb_info *sbi = EROFS_SB(sb);
+	char *name;
+	char *str = NULL;
 	int err;
 
+	if (erofs_is_fscache_mode(sb)) {
+		if (sbi->opt.domain_id) {
+			str = kasprintf(GFP_KERNEL, "%s,%s", sbi->opt.domain_id,
+					sbi->opt.fsid);
+			if (!str)
+				return -ENOMEM;
+			name = str;
+		} else {
+			name = sbi->opt.fsid;
+		}
+	} else {
+		name = sb->s_id;
+	}
 	sbi->s_kobj.kset = &erofs_root;
 	init_completion(&sbi->s_kobj_unregister);
-	err = kobject_init_and_add(&sbi->s_kobj, &erofs_sb_ktype, NULL, "%s",
-			erofs_is_fscache_mode(sb) ? sbi->opt.fsid : sb->s_id);
+	err = kobject_init_and_add(&sbi->s_kobj, &erofs_sb_ktype, NULL, "%s", name);
+	kfree(str);
 	if (err)
 		goto put_sb_kobj;
 	return 0;
diff --git a/fs/erofs/xattr.h b/fs/erofs/xattr.h
index 332462c59f11..0a43c9ee9f8f 100644
--- a/fs/erofs/xattr.h
+++ b/fs/erofs/xattr.h
@@ -39,9 +39,7 @@ static inline unsigned int xattrblock_offset(struct erofs_sb_info *sbi,
 #ifdef CONFIG_EROFS_FS_XATTR
 extern const struct xattr_handler erofs_xattr_user_handler;
 extern const struct xattr_handler erofs_xattr_trusted_handler;
-#ifdef CONFIG_EROFS_FS_SECURITY
 extern const struct xattr_handler erofs_xattr_security_handler;
-#endif
 
 static inline const struct xattr_handler *erofs_xattr_handler(unsigned int idx)
 {
diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
index 5792ca9e0d5e..cce56dde135c 100644
--- a/fs/erofs/zdata.c
+++ b/fs/erofs/zdata.c
@@ -650,6 +650,35 @@ static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe,
 		la < fe->headoffset;
 }
 
+static int z_erofs_read_fragment(struct inode *inode, erofs_off_t pos,
+				 struct page *page, unsigned int pageofs,
+				 unsigned int len)
+{
+	struct inode *packed_inode = EROFS_I_SB(inode)->packed_inode;
+	struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
+	u8 *src, *dst;
+	unsigned int i, cnt;
+
+	pos += EROFS_I(inode)->z_fragmentoff;
+	for (i = 0; i < len; i += cnt) {
+		cnt = min_t(unsigned int, len - i,
+			    EROFS_BLKSIZ - erofs_blkoff(pos));
+		src = erofs_bread(&buf, packed_inode,
+				  erofs_blknr(pos), EROFS_KMAP);
+		if (IS_ERR(src)) {
+			erofs_put_metabuf(&buf);
+			return PTR_ERR(src);
+		}
+
+		dst = kmap_local_page(page);
+		memcpy(dst + pageofs + i, src + erofs_blkoff(pos), cnt);
+		kunmap_local(dst);
+		pos += cnt;
+	}
+	erofs_put_metabuf(&buf);
+	return 0;
+}
+
 static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,
 				struct page *page, struct page **pagepool)
 {
@@ -688,7 +717,8 @@ repeat:
 		/* didn't get a valid pcluster previously (very rare) */
 	}
 
-	if (!(map->m_flags & EROFS_MAP_MAPPED))
+	if (!(map->m_flags & EROFS_MAP_MAPPED) ||
+	    map->m_flags & EROFS_MAP_FRAGMENT)
 		goto hitted;
 
 	err = z_erofs_collector_begin(fe);
@@ -735,6 +765,24 @@ hitted:
 		zero_user_segment(page, cur, end);
 		goto next_part;
 	}
+	if (map->m_flags & EROFS_MAP_FRAGMENT) {
+		unsigned int pageofs, skip, len;
+
+		if (offset > map->m_la) {
+			pageofs = 0;
+			skip = offset - map->m_la;
+		} else {
+			pageofs = map->m_la & ~PAGE_MASK;
+			skip = 0;
+		}
+		len = min_t(unsigned int, map->m_llen - skip, end - cur);
+		err = z_erofs_read_fragment(inode, skip, page, pageofs, len);
+		if (err)
+			goto out;
+		++spiltted;
+		tight = false;
+		goto next_part;
+	}
 
 	exclusive = (!cur && (!spiltted || tight));
 	if (cur)
@@ -766,6 +814,7 @@ retry:
 		fe->pcl->multibases = true;
 
 	if ((map->m_flags & EROFS_MAP_FULL_MAPPED) &&
+	    !(map->m_flags & EROFS_MAP_PARTIAL_REF) &&
 	    fe->pcl->length == map->m_llen)
 		fe->pcl->partial = false;
 	if (fe->pcl->length < offset + end - map->m_la) {
diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c
index d58549ca1df9..44c27ef39c43 100644
--- a/fs/erofs/zmap.c
+++ b/fs/erofs/zmap.c
@@ -17,7 +17,7 @@ int z_erofs_fill_inode(struct inode *inode)
 	struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
 
 	if (!erofs_sb_has_big_pcluster(sbi) &&
-	    !erofs_sb_has_ztailpacking(sbi) &&
+	    !erofs_sb_has_ztailpacking(sbi) && !erofs_sb_has_fragments(sbi) &&
 	    vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
 		vi->z_advise = 0;
 		vi->z_algorithmtype[0] = 0;
@@ -55,10 +55,6 @@ static int z_erofs_fill_inode_lazy(struct inode *inode)
 	if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
 		goto out_unlock;
 
-	DBG_BUGON(!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
-		  !erofs_sb_has_ztailpacking(EROFS_SB(sb)) &&
-		  vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
-
 	pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize +
 		    vi->xattr_isize, 8);
 	kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(pos),
@@ -69,6 +65,16 @@ static int z_erofs_fill_inode_lazy(struct inode *inode)
 	}
 
 	h = kaddr + erofs_blkoff(pos);
+	/*
+	 * if the highest bit of the 8-byte map header is set, the whole file
+	 * is stored in the packed inode. The rest bits keeps z_fragmentoff.
+	 */
+	if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) {
+		vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
+		vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63);
+		vi->z_tailextent_headlcn = 0;
+		goto unmap_done;
+	}
 	vi->z_advise = le16_to_cpu(h->h_advise);
 	vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
 	vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
@@ -123,6 +129,20 @@ unmap_done:
 		if (err < 0)
 			goto out_unlock;
 	}
+
+	if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER &&
+	    !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) {
+		struct erofs_map_blocks map = {
+			.buf = __EROFS_BUF_INITIALIZER
+		};
+
+		vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff);
+		err = z_erofs_do_map_blocks(inode, &map,
+					    EROFS_GET_BLOCKS_FINDTAIL);
+		erofs_put_metabuf(&map.buf);
+		if (err < 0)
+			goto out_unlock;
+	}
 	/* paired with smp_mb() at the beginning of the function */
 	smp_mb();
 	set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
@@ -143,20 +163,9 @@ struct z_erofs_maprecorder {
 	u16 delta[2];
 	erofs_blk_t pblk, compressedblks;
 	erofs_off_t nextpackoff;
+	bool partialref;
 };
 
-static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
-				  erofs_blk_t eblk)
-{
-	struct super_block *const sb = m->inode->i_sb;
-
-	m->kaddr = erofs_read_metabuf(&m->map->buf, sb, eblk,
-				      EROFS_KMAP_ATOMIC);
-	if (IS_ERR(m->kaddr))
-		return PTR_ERR(m->kaddr);
-	return 0;
-}
-
 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
 					 unsigned long lcn)
 {
@@ -169,11 +178,11 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
 		lcn * sizeof(struct z_erofs_vle_decompressed_index);
 	struct z_erofs_vle_decompressed_index *di;
 	unsigned int advise, type;
-	int err;
 
-	err = z_erofs_reload_indexes(m, erofs_blknr(pos));
-	if (err)
-		return err;
+	m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb,
+				      erofs_blknr(pos), EROFS_KMAP_ATOMIC);
+	if (IS_ERR(m->kaddr))
+		return PTR_ERR(m->kaddr);
 
 	m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index);
 	m->lcn = lcn;
@@ -201,6 +210,8 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
 	case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
 	case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
 	case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
+		if (advise & Z_EROFS_VLE_DI_PARTIAL_REF)
+			m->partialref = true;
 		m->clusterofs = le16_to_cpu(di->di_clusterofs);
 		m->pblk = le32_to_cpu(di->di_u.blkaddr);
 		break;
@@ -370,7 +381,6 @@ static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m,
 	unsigned int compacted_4b_initial, compacted_2b;
 	unsigned int amortizedshift;
 	erofs_off_t pos;
-	int err;
 
 	if (lclusterbits != 12)
 		return -EOPNOTSUPP;
@@ -407,9 +417,10 @@ static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m,
 	amortizedshift = 2;
 out:
 	pos += lcn * (1 << amortizedshift);
-	err = z_erofs_reload_indexes(m, erofs_blknr(pos));
-	if (err)
-		return err;
+	m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb,
+				      erofs_blknr(pos), EROFS_KMAP_ATOMIC);
+	if (IS_ERR(m->kaddr))
+		return PTR_ERR(m->kaddr);
 	return unpack_compacted_index(m, amortizedshift, pos, lookahead);
 }
 
@@ -598,6 +609,7 @@ static int z_erofs_do_map_blocks(struct inode *inode,
 {
 	struct erofs_inode *const vi = EROFS_I(inode);
 	bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER;
+	bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
 	struct z_erofs_maprecorder m = {
 		.inode = inode,
 		.map = map,
@@ -663,15 +675,23 @@ static int z_erofs_do_map_blocks(struct inode *inode,
 		err = -EOPNOTSUPP;
 		goto unmap_out;
 	}
-
+	if (m.partialref)
+		map->m_flags |= EROFS_MAP_PARTIAL_REF;
 	map->m_llen = end - map->m_la;
 
-	if (flags & EROFS_GET_BLOCKS_FINDTAIL)
+	if (flags & EROFS_GET_BLOCKS_FINDTAIL) {
 		vi->z_tailextent_headlcn = m.lcn;
+		/* for non-compact indexes, fragmentoff is 64 bits */
+		if (fragment &&
+		    vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
+			vi->z_fragmentoff |= (u64)m.pblk << 32;
+	}
 	if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) {
 		map->m_flags |= EROFS_MAP_META;
 		map->m_pa = vi->z_idataoff;
 		map->m_plen = vi->z_idata_size;
+	} else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
+		map->m_flags |= EROFS_MAP_FRAGMENT;
 	} else {
 		map->m_pa = blknr_to_addr(m.pblk);
 		err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
@@ -679,12 +699,18 @@ static int z_erofs_do_map_blocks(struct inode *inode,
 			goto out;
 	}
 
-	if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN)
-		map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
-	else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2)
+	if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) {
+		if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER)
+			map->m_algorithmformat =
+				Z_EROFS_COMPRESSION_INTERLACED;
+		else
+			map->m_algorithmformat =
+				Z_EROFS_COMPRESSION_SHIFTED;
+	} else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) {
 		map->m_algorithmformat = vi->z_algorithmtype[1];
-	else
+	} else {
 		map->m_algorithmformat = vi->z_algorithmtype[0];
+	}
 
 	if ((flags & EROFS_GET_BLOCKS_FIEMAP) ||
 	    ((flags & EROFS_GET_BLOCKS_READMORE) &&
@@ -705,10 +731,10 @@ out:
 	return err;
 }
 
-int z_erofs_map_blocks_iter(struct inode *inode,
-			    struct erofs_map_blocks *map,
+int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map,
 			    int flags)
 {
+	struct erofs_inode *const vi = EROFS_I(inode);
 	int err = 0;
 
 	trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
@@ -725,6 +751,15 @@ int z_erofs_map_blocks_iter(struct inode *inode,
 	if (err)
 		goto out;
 
+	if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) &&
+	    !vi->z_tailextent_headlcn) {
+		map->m_la = 0;
+		map->m_llen = inode->i_size;
+		map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED |
+				EROFS_MAP_FRAGMENT;
+		goto out;
+	}
+
 	err = z_erofs_do_map_blocks(inode, map, flags);
 out:
 	trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
@@ -751,7 +786,8 @@ static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset,
 	iomap->length = map.m_llen;
 	if (map.m_flags & EROFS_MAP_MAPPED) {
 		iomap->type = IOMAP_MAPPED;
-		iomap->addr = map.m_pa;
+		iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ?
+			      IOMAP_NULL_ADDR : map.m_pa;
 	} else {
 		iomap->type = IOMAP_HOLE;
 		iomap->addr = IOMAP_NULL_ADDR;
diff --git a/fs/exec.c b/fs/exec.c
index d046dbb9cbd0..768843477a49 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -957,8 +957,7 @@ struct file *open_exec(const char *name)
 }
 EXPORT_SYMBOL(open_exec);
 
-#if defined(CONFIG_HAVE_AOUT) || defined(CONFIG_BINFMT_FLAT) || \
-    defined(CONFIG_BINFMT_ELF_FDPIC)
+#if defined(CONFIG_BINFMT_FLAT) || defined(CONFIG_BINFMT_ELF_FDPIC)
 ssize_t read_code(struct file *file, unsigned long addr, loff_t pos, size_t len)
 {
 	ssize_t res = vfs_read(file, (void __user *)addr, len, &pos);
@@ -1588,7 +1587,7 @@ static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file)
 {
 	/* Handle suid and sgid on files */
 	struct user_namespace *mnt_userns;
-	struct inode *inode;
+	struct inode *inode = file_inode(file);
 	unsigned int mode;
 	kuid_t uid;
 	kgid_t gid;
@@ -1599,7 +1598,6 @@ static void bprm_fill_uid(struct linux_binprm *bprm, struct file *file)
 	if (task_no_new_privs(current))
 		return;
 
-	inode = file->f_path.dentry->d_inode;
 	mode = READ_ONCE(inode->i_mode);
 	if (!(mode & (S_ISUID|S_ISGID)))
 		return;
diff --git a/fs/exfat/dir.c b/fs/exfat/dir.c
index a27b55ec060a..0fc08fdcba73 100644
--- a/fs/exfat/dir.c
+++ b/fs/exfat/dir.c
@@ -212,9 +212,9 @@ static void exfat_free_namebuf(struct exfat_dentry_namebuf *nb)
 
 /* skip iterating emit_dots when dir is empty */
 #define ITER_POS_FILLED_DOTS    (2)
-static int exfat_iterate(struct file *filp, struct dir_context *ctx)
+static int exfat_iterate(struct file *file, struct dir_context *ctx)
 {
-	struct inode *inode = filp->f_path.dentry->d_inode;
+	struct inode *inode = file_inode(file);
 	struct super_block *sb = inode->i_sb;
 	struct inode *tmp;
 	struct exfat_dir_entry de;
@@ -228,7 +228,7 @@ static int exfat_iterate(struct file *filp, struct dir_context *ctx)
 	mutex_lock(&EXFAT_SB(sb)->s_lock);
 
 	cpos = ctx->pos;
-	if (!dir_emit_dots(filp, ctx))
+	if (!dir_emit_dots(file, ctx))
 		goto unlock;
 
 	if (ctx->pos == ITER_POS_FILLED_DOTS) {
diff --git a/fs/exportfs/expfs.c b/fs/exportfs/expfs.c
index 3ef80d000e13..c648a493faf2 100644
--- a/fs/exportfs/expfs.c
+++ b/fs/exportfs/expfs.c
@@ -248,21 +248,20 @@ struct getdents_callback {
  * A rather strange filldir function to capture
  * the name matching the specified inode number.
  */
-static int filldir_one(struct dir_context *ctx, const char *name, int len,
+static bool filldir_one(struct dir_context *ctx, const char *name, int len,
 			loff_t pos, u64 ino, unsigned int d_type)
 {
 	struct getdents_callback *buf =
 		container_of(ctx, struct getdents_callback, ctx);
-	int result = 0;
 
 	buf->sequence++;
 	if (buf->ino == ino && len <= NAME_MAX) {
 		memcpy(buf->name, name, len);
 		buf->name[len] = '\0';
 		buf->found = 1;
-		result = -1;
+		return false;	// no more
 	}
-	return result;
+	return true;
 }
 
 /**
diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
index 3bf9a6926798..e5f2f5ca5120 100644
--- a/fs/ext4/ext4.h
+++ b/fs/ext4/ext4.h
@@ -2977,6 +2977,7 @@ extern struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
 extern int  ext4_write_inode(struct inode *, struct writeback_control *);
 extern int  ext4_setattr(struct user_namespace *, struct dentry *,
 			 struct iattr *);
+extern u32  ext4_dio_alignment(struct inode *inode);
 extern int  ext4_getattr(struct user_namespace *, const struct path *,
 			 struct kstat *, u32, unsigned int);
 extern void ext4_evict_inode(struct inode *);
diff --git a/fs/ext4/file.c b/fs/ext4/file.c
index 109d07629f81..8bb1c35fd6dd 100644
--- a/fs/ext4/file.c
+++ b/fs/ext4/file.c
@@ -36,19 +36,34 @@
 #include "acl.h"
 #include "truncate.h"
 
-static bool ext4_dio_supported(struct kiocb *iocb, struct iov_iter *iter)
+/*
+ * Returns %true if the given DIO request should be attempted with DIO, or
+ * %false if it should fall back to buffered I/O.
+ *
+ * DIO isn't well specified; when it's unsupported (either due to the request
+ * being misaligned, or due to the file not supporting DIO at all), filesystems
+ * either fall back to buffered I/O or return EINVAL.  For files that don't use
+ * any special features like encryption or verity, ext4 has traditionally
+ * returned EINVAL for misaligned DIO.  iomap_dio_rw() uses this convention too.
+ * In this case, we should attempt the DIO, *not* fall back to buffered I/O.
+ *
+ * In contrast, in cases where DIO is unsupported due to ext4 features, ext4
+ * traditionally falls back to buffered I/O.
+ *
+ * This function implements the traditional ext4 behavior in all these cases.
+ */
+static bool ext4_should_use_dio(struct kiocb *iocb, struct iov_iter *iter)
 {
 	struct inode *inode = file_inode(iocb->ki_filp);
+	u32 dio_align = ext4_dio_alignment(inode);
 
-	if (!fscrypt_dio_supported(iocb, iter))
-		return false;
-	if (fsverity_active(inode))
+	if (dio_align == 0)
 		return false;
-	if (ext4_should_journal_data(inode))
-		return false;
-	if (ext4_has_inline_data(inode))
-		return false;
-	return true;
+
+	if (dio_align == 1)
+		return true;
+
+	return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align);
 }
 
 static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)
@@ -63,7 +78,7 @@ static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)
 		inode_lock_shared(inode);
 	}
 
-	if (!ext4_dio_supported(iocb, to)) {
+	if (!ext4_should_use_dio(iocb, to)) {
 		inode_unlock_shared(inode);
 		/*
 		 * Fallback to buffered I/O if the operation being performed on
@@ -511,7 +526,7 @@ static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	}
 
 	/* Fallback to buffered I/O if the inode does not support direct I/O. */
-	if (!ext4_dio_supported(iocb, from)) {
+	if (!ext4_should_use_dio(iocb, from)) {
 		if (ilock_shared)
 			inode_unlock_shared(inode);
 		else
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 601214453c3a..364774230d87 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -5550,6 +5550,22 @@ err_out:
 	return error;
 }
 
+u32 ext4_dio_alignment(struct inode *inode)
+{
+	if (fsverity_active(inode))
+		return 0;
+	if (ext4_should_journal_data(inode))
+		return 0;
+	if (ext4_has_inline_data(inode))
+		return 0;
+	if (IS_ENCRYPTED(inode)) {
+		if (!fscrypt_dio_supported(inode))
+			return 0;
+		return i_blocksize(inode);
+	}
+	return 1; /* use the iomap defaults */
+}
+
 int ext4_getattr(struct user_namespace *mnt_userns, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int query_flags)
 {
@@ -5565,6 +5581,27 @@ int ext4_getattr(struct user_namespace *mnt_userns, const struct path *path,
 		stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
 	}
 
+	/*
+	 * Return the DIO alignment restrictions if requested.  We only return
+	 * this information when requested, since on encrypted files it might
+	 * take a fair bit of work to get if the file wasn't opened recently.
+	 */
+	if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) {
+		u32 dio_align = ext4_dio_alignment(inode);
+
+		stat->result_mask |= STATX_DIOALIGN;
+		if (dio_align == 1) {
+			struct block_device *bdev = inode->i_sb->s_bdev;
+
+			/* iomap defaults */
+			stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
+			stat->dio_offset_align = bdev_logical_block_size(bdev);
+		} else {
+			stat->dio_mem_align = dio_align;
+			stat->dio_offset_align = dio_align;
+		}
+	}
+
 	flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
 	if (flags & EXT4_APPEND_FL)
 		stat->attributes |= STATX_ATTR_APPEND;
diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
index 71f5b67d7f28..9dad93059945 100644
--- a/fs/ext4/mballoc.c
+++ b/fs/ext4/mballoc.c
@@ -910,7 +910,7 @@ static void ext4_mb_choose_next_group_cr1(struct ext4_allocation_context *ac,
 		int *new_cr, ext4_group_t *group, ext4_group_t ngroups)
 {
 	struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
-	struct ext4_group_info *grp, *iter;
+	struct ext4_group_info *grp = NULL, *iter;
 	int i;
 
 	if (unlikely(ac->ac_flags & EXT4_MB_CR1_OPTIMIZED)) {
@@ -927,7 +927,6 @@ static void ext4_mb_choose_next_group_cr1(struct ext4_allocation_context *ac,
 			read_unlock(&sbi->s_mb_avg_fragment_size_locks[i]);
 			continue;
 		}
-		grp = NULL;
 		list_for_each_entry(iter, &sbi->s_mb_avg_fragment_size[i],
 				    bb_avg_fragment_size_node) {
 			if (sbi->s_mb_stats)
diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c
index e02a5f14e021..3d21eae267fc 100644
--- a/fs/ext4/readpage.c
+++ b/fs/ext4/readpage.c
@@ -75,7 +75,7 @@ static void __read_end_io(struct bio *bio)
 	bio_for_each_segment_all(bv, bio, iter_all) {
 		page = bv->bv_page;
 
-		/* PG_error was set if any post_read step failed */
+		/* PG_error was set if verity failed. */
 		if (bio->bi_status || PageError(page)) {
 			ClearPageUptodate(page);
 			/* will re-read again later */
@@ -96,10 +96,12 @@ static void decrypt_work(struct work_struct *work)
 {
 	struct bio_post_read_ctx *ctx =
 		container_of(work, struct bio_post_read_ctx, work);
+	struct bio *bio = ctx->bio;
 
-	fscrypt_decrypt_bio(ctx->bio);
-
-	bio_post_read_processing(ctx);
+	if (fscrypt_decrypt_bio(bio))
+		bio_post_read_processing(ctx);
+	else
+		__read_end_io(bio);
 }
 
 static void verity_work(struct work_struct *work)
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index aa3ccddfa037..93cc2ec51c2a 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -139,7 +139,7 @@ static void f2fs_finish_read_bio(struct bio *bio, bool in_task)
 			continue;
 		}
 
-		/* PG_error was set if decryption or verity failed. */
+		/* PG_error was set if verity failed. */
 		if (bio->bi_status || PageError(page)) {
 			ClearPageUptodate(page);
 			/* will re-read again later */
@@ -185,7 +185,7 @@ static void f2fs_verify_bio(struct work_struct *work)
 			struct page *page = bv->bv_page;
 
 			if (!f2fs_is_compressed_page(page) &&
-			    !PageError(page) && !fsverity_verify_page(page))
+			    !fsverity_verify_page(page))
 				SetPageError(page);
 		}
 	} else {
@@ -236,10 +236,9 @@ static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx,
 	bio_for_each_segment_all(bv, ctx->bio, iter_all) {
 		struct page *page = bv->bv_page;
 
-		/* PG_error was set if decryption failed. */
 		if (f2fs_is_compressed_page(page))
-			f2fs_end_read_compressed_page(page, PageError(page),
-						blkaddr, in_task);
+			f2fs_end_read_compressed_page(page, false, blkaddr,
+						      in_task);
 		else
 			all_compressed = false;
 
@@ -259,14 +258,17 @@ static void f2fs_post_read_work(struct work_struct *work)
 {
 	struct bio_post_read_ctx *ctx =
 		container_of(work, struct bio_post_read_ctx, work);
+	struct bio *bio = ctx->bio;
 
-	if (ctx->enabled_steps & STEP_DECRYPT)
-		fscrypt_decrypt_bio(ctx->bio);
+	if ((ctx->enabled_steps & STEP_DECRYPT) && !fscrypt_decrypt_bio(bio)) {
+		f2fs_finish_read_bio(bio, true);
+		return;
+	}
 
 	if (ctx->enabled_steps & STEP_DECOMPRESS)
 		f2fs_handle_step_decompress(ctx, true);
 
-	f2fs_verify_and_finish_bio(ctx->bio, true);
+	f2fs_verify_and_finish_bio(bio, true);
 }
 
 static void f2fs_read_end_io(struct bio *bio)
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 3c7cdb70fe2e..aea816a133a8 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -4471,17 +4471,6 @@ static inline void f2fs_i_compr_blocks_update(struct inode *inode,
 	f2fs_mark_inode_dirty_sync(inode, true);
 }
 
-static inline int block_unaligned_IO(struct inode *inode,
-				struct kiocb *iocb, struct iov_iter *iter)
-{
-	unsigned int i_blkbits = READ_ONCE(inode->i_blkbits);
-	unsigned int blocksize_mask = (1 << i_blkbits) - 1;
-	loff_t offset = iocb->ki_pos;
-	unsigned long align = offset | iov_iter_alignment(iter);
-
-	return align & blocksize_mask;
-}
-
 static inline bool f2fs_allow_multi_device_dio(struct f2fs_sb_info *sbi,
 								int flag)
 {
@@ -4492,35 +4481,6 @@ static inline bool f2fs_allow_multi_device_dio(struct f2fs_sb_info *sbi,
 	return sbi->aligned_blksize;
 }
 
-static inline bool f2fs_force_buffered_io(struct inode *inode,
-				struct kiocb *iocb, struct iov_iter *iter)
-{
-	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
-	int rw = iov_iter_rw(iter);
-
-	if (!fscrypt_dio_supported(iocb, iter))
-		return true;
-	if (fsverity_active(inode))
-		return true;
-	if (f2fs_compressed_file(inode))
-		return true;
-
-	/* disallow direct IO if any of devices has unaligned blksize */
-	if (f2fs_is_multi_device(sbi) && !sbi->aligned_blksize)
-		return true;
-
-	if (f2fs_lfs_mode(sbi) && (rw == WRITE)) {
-		if (block_unaligned_IO(inode, iocb, iter))
-			return true;
-		if (F2FS_IO_ALIGNED(sbi))
-			return true;
-	}
-	if (is_sbi_flag_set(F2FS_I_SB(inode), SBI_CP_DISABLED))
-		return true;
-
-	return false;
-}
-
 static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx)
 {
 	return fsverity_active(inode) &&
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index ce4905a073b3..791770507328 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -808,6 +808,29 @@ int f2fs_truncate(struct inode *inode)
 	return 0;
 }
 
+static bool f2fs_force_buffered_io(struct inode *inode, int rw)
+{
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
+
+	if (!fscrypt_dio_supported(inode))
+		return true;
+	if (fsverity_active(inode))
+		return true;
+	if (f2fs_compressed_file(inode))
+		return true;
+
+	/* disallow direct IO if any of devices has unaligned blksize */
+	if (f2fs_is_multi_device(sbi) && !sbi->aligned_blksize)
+		return true;
+
+	if (f2fs_lfs_mode(sbi) && rw == WRITE && F2FS_IO_ALIGNED(sbi))
+		return true;
+	if (is_sbi_flag_set(sbi, SBI_CP_DISABLED))
+		return true;
+
+	return false;
+}
+
 int f2fs_getattr(struct user_namespace *mnt_userns, const struct path *path,
 		 struct kstat *stat, u32 request_mask, unsigned int query_flags)
 {
@@ -824,6 +847,24 @@ int f2fs_getattr(struct user_namespace *mnt_userns, const struct path *path,
 		stat->btime.tv_nsec = fi->i_crtime.tv_nsec;
 	}
 
+	/*
+	 * Return the DIO alignment restrictions if requested.  We only return
+	 * this information when requested, since on encrypted files it might
+	 * take a fair bit of work to get if the file wasn't opened recently.
+	 *
+	 * f2fs sometimes supports DIO reads but not DIO writes.  STATX_DIOALIGN
+	 * cannot represent that, so in that case we report no DIO support.
+	 */
+	if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) {
+		unsigned int bsize = i_blocksize(inode);
+
+		stat->result_mask |= STATX_DIOALIGN;
+		if (!f2fs_force_buffered_io(inode, WRITE)) {
+			stat->dio_mem_align = bsize;
+			stat->dio_offset_align = bsize;
+		}
+	}
+
 	flags = fi->i_flags;
 	if (flags & F2FS_COMPR_FL)
 		stat->attributes |= STATX_ATTR_COMPRESSED;
@@ -4182,7 +4223,7 @@ static bool f2fs_should_use_dio(struct inode *inode, struct kiocb *iocb,
 	if (!(iocb->ki_flags & IOCB_DIRECT))
 		return false;
 
-	if (f2fs_force_buffered_io(inode, iocb, iter))
+	if (f2fs_force_buffered_io(inode, iov_iter_rw(iter)))
 		return false;
 
 	/*
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index 2451623c05a7..26817b5aeac7 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -3039,23 +3039,24 @@ static void f2fs_get_ino_and_lblk_bits(struct super_block *sb,
 	*lblk_bits_ret = 8 * sizeof(block_t);
 }
 
-static int f2fs_get_num_devices(struct super_block *sb)
+static struct block_device **f2fs_get_devices(struct super_block *sb,
+					      unsigned int *num_devs)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(sb);
+	struct block_device **devs;
+	int i;
 
-	if (f2fs_is_multi_device(sbi))
-		return sbi->s_ndevs;
-	return 1;
-}
+	if (!f2fs_is_multi_device(sbi))
+		return NULL;
 
-static void f2fs_get_devices(struct super_block *sb,
-			     struct request_queue **devs)
-{
-	struct f2fs_sb_info *sbi = F2FS_SB(sb);
-	int i;
+	devs = kmalloc_array(sbi->s_ndevs, sizeof(*devs), GFP_KERNEL);
+	if (!devs)
+		return ERR_PTR(-ENOMEM);
 
 	for (i = 0; i < sbi->s_ndevs; i++)
-		devs[i] = bdev_get_queue(FDEV(i).bdev);
+		devs[i] = FDEV(i).bdev;
+	*num_devs = sbi->s_ndevs;
+	return devs;
 }
 
 static const struct fscrypt_operations f2fs_cryptops = {
@@ -3066,7 +3067,6 @@ static const struct fscrypt_operations f2fs_cryptops = {
 	.empty_dir		= f2fs_empty_dir,
 	.has_stable_inodes	= f2fs_has_stable_inodes,
 	.get_ino_and_lblk_bits	= f2fs_get_ino_and_lblk_bits,
-	.get_num_devices	= f2fs_get_num_devices,
 	.get_devices		= f2fs_get_devices,
 };
 #endif
diff --git a/fs/fat/dir.c b/fs/fat/dir.c
index 249825017da7..00235b8a1823 100644
--- a/fs/fat/dir.c
+++ b/fs/fat/dir.c
@@ -705,7 +705,7 @@ static int fat_readdir(struct file *file, struct dir_context *ctx)
 }
 
 #define FAT_IOCTL_FILLDIR_FUNC(func, dirent_type)			   \
-static int func(struct dir_context *ctx, const char *name, int name_len,   \
+static bool func(struct dir_context *ctx, const char *name, int name_len,  \
 			     loff_t offset, u64 ino, unsigned int d_type)  \
 {									   \
 	struct fat_ioctl_filldir_callback *buf =			   \
@@ -714,7 +714,7 @@ static int func(struct dir_context *ctx, const char *name, int name_len,   \
 	struct dirent_type __user *d2 = d1 + 1;				   \
 									   \
 	if (buf->result)						   \
-		return -EINVAL;						   \
+		return false;						   \
 	buf->result++;							   \
 									   \
 	if (name != NULL) {						   \
@@ -750,10 +750,10 @@ static int func(struct dir_context *ctx, const char *name, int name_len,   \
 		    put_user(short_len, &d1->d_reclen))			   \
 			goto efault;					   \
 	}								   \
-	return 0;							   \
+	return true;							   \
 efault:									   \
 	buf->result = -EFAULT;						   \
-	return -EFAULT;							   \
+	return false;							   \
 }
 
 FAT_IOCTL_FILLDIR_FUNC(fat_ioctl_filldir, __fat_dirent)
diff --git a/fs/fat/file.c b/fs/fat/file.c
index 3e4eb3467cb4..8a6b493b5b5f 100644
--- a/fs/fat/file.c
+++ b/fs/fat/file.c
@@ -461,8 +461,9 @@ static int fat_allow_set_time(struct user_namespace *mnt_userns,
 {
 	umode_t allow_utime = sbi->options.allow_utime;
 
-	if (!uid_eq(current_fsuid(), i_uid_into_mnt(mnt_userns, inode))) {
-		if (in_group_p(i_gid_into_mnt(mnt_userns, inode)))
+	if (!vfsuid_eq_kuid(i_uid_into_vfsuid(mnt_userns, inode),
+			    current_fsuid())) {
+		if (vfsgid_in_group_p(i_gid_into_vfsgid(mnt_userns, inode)))
 			allow_utime >>= 3;
 		if (allow_utime & MAY_WRITE)
 			return 1;
diff --git a/fs/fhandle.c b/fs/fhandle.c
index 6630c69c23a2..f2bc27d1975e 100644
--- a/fs/fhandle.c
+++ b/fs/fhandle.c
@@ -14,7 +14,7 @@
 #include "internal.h"
 #include "mount.h"
 
-static long do_sys_name_to_handle(struct path *path,
+static long do_sys_name_to_handle(const struct path *path,
 				  struct file_handle __user *ufh,
 				  int __user *mnt_id)
 {
diff --git a/fs/file_table.c b/fs/file_table.c
index 99c6796c9f28..dd88701e54a9 100644
--- a/fs/file_table.c
+++ b/fs/file_table.c
@@ -324,12 +324,7 @@ static void __fput(struct file *file)
 	}
 	fops_put(file->f_op);
 	put_pid(file->f_owner.pid);
-	if ((mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ)
-		i_readcount_dec(inode);
-	if (mode & FMODE_WRITER) {
-		put_write_access(inode);
-		__mnt_drop_write(mnt);
-	}
+	put_file_access(file);
 	dput(dentry);
 	if (unlikely(mode & FMODE_NEED_UNMOUNT))
 		dissolve_on_fput(mnt);
diff --git a/fs/gfs2/export.c b/fs/gfs2/export.c
index 756d05779200..cf40895233f5 100644
--- a/fs/gfs2/export.c
+++ b/fs/gfs2/export.c
@@ -66,7 +66,7 @@ struct get_name_filldir {
 	char *name;
 };
 
-static int get_name_filldir(struct dir_context *ctx, const char *name,
+static bool get_name_filldir(struct dir_context *ctx, const char *name,
 			    int length, loff_t offset, u64 inum,
 			    unsigned int type)
 {
@@ -74,12 +74,12 @@ static int get_name_filldir(struct dir_context *ctx, const char *name,
 		container_of(ctx, struct get_name_filldir, ctx);
 
 	if (inum != gnfd->inum.no_addr)
-		return 0;
+		return true;
 
 	memcpy(gnfd->name, name, length);
 	gnfd->name[length] = 0;
 
-	return 1;
+	return false;
 }
 
 static int gfs2_get_name(struct dentry *parent, char *name,
diff --git a/fs/gfs2/lock_dlm.c b/fs/gfs2/lock_dlm.c
index 6ce369b096d4..71911bf9ab34 100644
--- a/fs/gfs2/lock_dlm.c
+++ b/fs/gfs2/lock_dlm.c
@@ -1302,7 +1302,7 @@ static int gdlm_mount(struct gfs2_sbd *sdp, const char *table)
 	memcpy(cluster, table, strlen(table) - strlen(fsname));
 	fsname++;
 
-	flags = DLM_LSFL_FS | DLM_LSFL_NEWEXCL;
+	flags = DLM_LSFL_NEWEXCL;
 
 	/*
 	 * create/join lockspace
diff --git a/fs/inode.c b/fs/inode.c
index ba1de23c13c1..b608528efd3a 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -192,8 +192,6 @@ int inode_init_always(struct super_block *sb, struct inode *inode)
 	inode->i_wb_frn_history = 0;
 #endif
 
-	if (security_inode_alloc(inode))
-		goto out;
 	spin_lock_init(&inode->i_lock);
 	lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key);
 
@@ -228,11 +226,12 @@ int inode_init_always(struct super_block *sb, struct inode *inode)
 	inode->i_fsnotify_mask = 0;
 #endif
 	inode->i_flctx = NULL;
+
+	if (unlikely(security_inode_alloc(inode)))
+		return -ENOMEM;
 	this_cpu_inc(nr_inodes);
 
 	return 0;
-out:
-	return -ENOMEM;
 }
 EXPORT_SYMBOL(inode_init_always);
 
diff --git a/fs/internal.h b/fs/internal.h
index 87e96b9024ce..6f0386b34fae 100644
--- a/fs/internal.h
+++ b/fs/internal.h
@@ -16,6 +16,7 @@ struct shrink_control;
 struct fs_context;
 struct user_namespace;
 struct pipe_inode_info;
+struct iov_iter;
 
 /*
  * block/bdev.c
@@ -62,7 +63,7 @@ extern int vfs_path_lookup(struct dentry *, struct vfsmount *,
 			   const char *, unsigned int, struct path *);
 int do_rmdir(int dfd, struct filename *name);
 int do_unlinkat(int dfd, struct filename *name);
-int may_linkat(struct user_namespace *mnt_userns, struct path *link);
+int may_linkat(struct user_namespace *mnt_userns, const struct path *link);
 int do_renameat2(int olddfd, struct filename *oldname, int newdfd,
 		 struct filename *newname, unsigned int flags);
 int do_mkdirat(int dfd, struct filename *name, umode_t mode);
@@ -101,6 +102,16 @@ extern void chroot_fs_refs(const struct path *, const struct path *);
 extern struct file *alloc_empty_file(int, const struct cred *);
 extern struct file *alloc_empty_file_noaccount(int, const struct cred *);
 
+static inline void put_file_access(struct file *file)
+{
+	if ((file->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) {
+		i_readcount_dec(file->f_inode);
+	} else if (file->f_mode & FMODE_WRITER) {
+		put_write_access(file->f_inode);
+		__mnt_drop_write(file->f_path.mnt);
+	}
+}
+
 /*
  * super.c
  */
@@ -221,3 +232,5 @@ ssize_t do_getxattr(struct user_namespace *mnt_userns,
 int setxattr_copy(const char __user *name, struct xattr_ctx *ctx);
 int do_setxattr(struct user_namespace *mnt_userns, struct dentry *dentry,
 		struct xattr_ctx *ctx);
+
+ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos);
diff --git a/fs/ksmbd/smb2pdu.c b/fs/ksmbd/smb2pdu.c
index 19412ac701a6..bfa6b41d895b 100644
--- a/fs/ksmbd/smb2pdu.c
+++ b/fs/ksmbd/smb2pdu.c
@@ -3776,7 +3776,7 @@ static int reserve_populate_dentry(struct ksmbd_dir_info *d_info,
 	return 0;
 }
 
-static int __query_dir(struct dir_context *ctx, const char *name, int namlen,
+static bool __query_dir(struct dir_context *ctx, const char *name, int namlen,
 		       loff_t offset, u64 ino, unsigned int d_type)
 {
 	struct ksmbd_readdir_data	*buf;
@@ -3790,22 +3790,20 @@ static int __query_dir(struct dir_context *ctx, const char *name, int namlen,
 
 	/* dot and dotdot entries are already reserved */
 	if (!strcmp(".", name) || !strcmp("..", name))
-		return 0;
+		return true;
 	if (ksmbd_share_veto_filename(priv->work->tcon->share_conf, name))
-		return 0;
+		return true;
 	if (!match_pattern(name, namlen, priv->search_pattern))
-		return 0;
+		return true;
 
 	d_info->name		= name;
 	d_info->name_len	= namlen;
 	rc = reserve_populate_dentry(d_info, priv->info_level);
 	if (rc)
-		return rc;
-	if (d_info->flags & SMB2_RETURN_SINGLE_ENTRY) {
+		return false;
+	if (d_info->flags & SMB2_RETURN_SINGLE_ENTRY)
 		d_info->out_buf_len = 0;
-		return 0;
-	}
-	return 0;
+	return true;
 }
 
 static void restart_ctx(struct dir_context *ctx)
diff --git a/fs/ksmbd/transport_ipc.c b/fs/ksmbd/transport_ipc.c
index 7cb0eeb07c80..c9aca21637d5 100644
--- a/fs/ksmbd/transport_ipc.c
+++ b/fs/ksmbd/transport_ipc.c
@@ -197,6 +197,7 @@ static struct genl_family ksmbd_genl_family = {
 	.module		= THIS_MODULE,
 	.ops		= ksmbd_genl_ops,
 	.n_ops		= ARRAY_SIZE(ksmbd_genl_ops),
+	.resv_start_op	= KSMBD_EVENT_SPNEGO_AUTHEN_RESPONSE + 1,
 };
 
 static void ksmbd_nl_init_fixup(void)
diff --git a/fs/ksmbd/vfs.c b/fs/ksmbd/vfs.c
index 78d01033604c..48b2b901f6e5 100644
--- a/fs/ksmbd/vfs.c
+++ b/fs/ksmbd/vfs.c
@@ -1105,7 +1105,7 @@ int ksmbd_vfs_unlink(struct user_namespace *user_ns,
 	return err;
 }
 
-static int __dir_empty(struct dir_context *ctx, const char *name, int namlen,
+static bool __dir_empty(struct dir_context *ctx, const char *name, int namlen,
 		       loff_t offset, u64 ino, unsigned int d_type)
 {
 	struct ksmbd_readdir_data *buf;
@@ -1113,9 +1113,7 @@ static int __dir_empty(struct dir_context *ctx, const char *name, int namlen,
 	buf = container_of(ctx, struct ksmbd_readdir_data, ctx);
 	buf->dirent_count++;
 
-	if (buf->dirent_count > 2)
-		return -ENOTEMPTY;
-	return 0;
+	return buf->dirent_count <= 2;
 }
 
 /**
@@ -1142,7 +1140,7 @@ int ksmbd_vfs_empty_dir(struct ksmbd_file *fp)
 	return err;
 }
 
-static int __caseless_lookup(struct dir_context *ctx, const char *name,
+static bool __caseless_lookup(struct dir_context *ctx, const char *name,
 			     int namlen, loff_t offset, u64 ino,
 			     unsigned int d_type)
 {
@@ -1151,13 +1149,13 @@ static int __caseless_lookup(struct dir_context *ctx, const char *name,
 	buf = container_of(ctx, struct ksmbd_readdir_data, ctx);
 
 	if (buf->used != namlen)
-		return 0;
+		return true;
 	if (!strncasecmp((char *)buf->private, name, namlen)) {
 		memcpy((char *)buf->private, name, namlen);
 		buf->dirent_count = 1;
-		return -EEXIST;
+		return false;
 	}
-	return 0;
+	return true;
 }
 
 /**
diff --git a/fs/lockd/host.c b/fs/lockd/host.c
index f802223e71ab..cdc8e12cdac4 100644
--- a/fs/lockd/host.c
+++ b/fs/lockd/host.c
@@ -164,7 +164,7 @@ static struct nlm_host *nlm_alloc_host(struct nlm_lookup_host_info *ni,
 	host->h_addrbuf    = nsm->sm_addrbuf;
 	host->net	   = ni->net;
 	host->h_cred	   = get_cred(ni->cred);
-	strlcpy(host->nodename, utsname()->nodename, sizeof(host->nodename));
+	strscpy(host->nodename, utsname()->nodename, sizeof(host->nodename));
 
 out:
 	return host;
diff --git a/fs/lockd/svc4proc.c b/fs/lockd/svc4proc.c
index bf274f23969b..284b019cb652 100644
--- a/fs/lockd/svc4proc.c
+++ b/fs/lockd/svc4proc.c
@@ -521,6 +521,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_void,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_void),
+		.pc_argzero = sizeof(struct nlm_void),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "NULL",
@@ -530,6 +531,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_testargs,
 		.pc_encode = nlm4svc_encode_testres,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St+2+No+Rg,
 		.pc_name = "TEST",
@@ -539,6 +541,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_lockargs,
 		.pc_encode = nlm4svc_encode_res,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St,
 		.pc_name = "LOCK",
@@ -548,6 +551,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_cancargs,
 		.pc_encode = nlm4svc_encode_res,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St,
 		.pc_name = "CANCEL",
@@ -557,6 +561,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_unlockargs,
 		.pc_encode = nlm4svc_encode_res,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St,
 		.pc_name = "UNLOCK",
@@ -566,6 +571,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_testargs,
 		.pc_encode = nlm4svc_encode_res,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St,
 		.pc_name = "GRANTED",
@@ -575,6 +581,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_testargs,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "TEST_MSG",
@@ -584,6 +591,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_lockargs,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "LOCK_MSG",
@@ -593,6 +601,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_cancargs,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "CANCEL_MSG",
@@ -602,6 +611,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_unlockargs,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "UNLOCK_MSG",
@@ -611,6 +621,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_testargs,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "GRANTED_MSG",
@@ -620,6 +631,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_void,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_res),
+		.pc_argzero = sizeof(struct nlm_res),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "TEST_RES",
@@ -629,6 +641,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_void,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_res),
+		.pc_argzero = sizeof(struct nlm_res),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "LOCK_RES",
@@ -638,6 +651,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_void,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_res),
+		.pc_argzero = sizeof(struct nlm_res),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "CANCEL_RES",
@@ -647,6 +661,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_void,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_res),
+		.pc_argzero = sizeof(struct nlm_res),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "UNLOCK_RES",
@@ -656,6 +671,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_res,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_res),
+		.pc_argzero = sizeof(struct nlm_res),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "GRANTED_RES",
@@ -665,6 +681,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_reboot,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_reboot),
+		.pc_argzero = sizeof(struct nlm_reboot),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "SM_NOTIFY",
@@ -674,6 +691,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_void,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_void),
+		.pc_argzero = sizeof(struct nlm_void),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = 0,
 		.pc_name = "UNUSED",
@@ -683,6 +701,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_void,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_void),
+		.pc_argzero = sizeof(struct nlm_void),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = 0,
 		.pc_name = "UNUSED",
@@ -692,6 +711,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_void,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_void),
+		.pc_argzero = sizeof(struct nlm_void),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = 0,
 		.pc_name = "UNUSED",
@@ -701,6 +721,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_shareargs,
 		.pc_encode = nlm4svc_encode_shareres,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St+1,
 		.pc_name = "SHARE",
@@ -710,6 +731,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_shareargs,
 		.pc_encode = nlm4svc_encode_shareres,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St+1,
 		.pc_name = "UNSHARE",
@@ -719,6 +741,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_lockargs,
 		.pc_encode = nlm4svc_encode_res,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St,
 		.pc_name = "NM_LOCK",
@@ -728,6 +751,7 @@ const struct svc_procedure nlmsvc_procedures4[24] = {
 		.pc_decode = nlm4svc_decode_notify,
 		.pc_encode = nlm4svc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "FREE_ALL",
diff --git a/fs/lockd/svcproc.c b/fs/lockd/svcproc.c
index b09ca35b527c..e35c05e27806 100644
--- a/fs/lockd/svcproc.c
+++ b/fs/lockd/svcproc.c
@@ -555,6 +555,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_void,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_void),
+		.pc_argzero = sizeof(struct nlm_void),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "NULL",
@@ -564,6 +565,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_testargs,
 		.pc_encode = nlmsvc_encode_testres,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St+2+No+Rg,
 		.pc_name = "TEST",
@@ -573,6 +575,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_lockargs,
 		.pc_encode = nlmsvc_encode_res,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St,
 		.pc_name = "LOCK",
@@ -582,6 +585,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_cancargs,
 		.pc_encode = nlmsvc_encode_res,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St,
 		.pc_name = "CANCEL",
@@ -591,6 +595,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_unlockargs,
 		.pc_encode = nlmsvc_encode_res,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St,
 		.pc_name = "UNLOCK",
@@ -600,6 +605,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_testargs,
 		.pc_encode = nlmsvc_encode_res,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St,
 		.pc_name = "GRANTED",
@@ -609,6 +615,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_testargs,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "TEST_MSG",
@@ -618,6 +625,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_lockargs,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "LOCK_MSG",
@@ -627,6 +635,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_cancargs,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "CANCEL_MSG",
@@ -636,6 +645,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_unlockargs,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "UNLOCK_MSG",
@@ -645,6 +655,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_testargs,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "GRANTED_MSG",
@@ -654,6 +665,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_void,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_res),
+		.pc_argzero = sizeof(struct nlm_res),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "TEST_RES",
@@ -663,6 +675,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_void,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_res),
+		.pc_argzero = sizeof(struct nlm_res),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "LOCK_RES",
@@ -672,6 +685,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_void,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_res),
+		.pc_argzero = sizeof(struct nlm_res),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "CANCEL_RES",
@@ -681,6 +695,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_void,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_res),
+		.pc_argzero = sizeof(struct nlm_res),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "UNLOCK_RES",
@@ -690,6 +705,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_res,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_res),
+		.pc_argzero = sizeof(struct nlm_res),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "GRANTED_RES",
@@ -699,6 +715,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_reboot,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_reboot),
+		.pc_argzero = sizeof(struct nlm_reboot),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "SM_NOTIFY",
@@ -708,6 +725,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_void,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_void),
+		.pc_argzero = sizeof(struct nlm_void),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "UNUSED",
@@ -717,6 +735,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_void,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_void),
+		.pc_argzero = sizeof(struct nlm_void),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "UNUSED",
@@ -726,6 +745,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_void,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_void),
+		.pc_argzero = sizeof(struct nlm_void),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = St,
 		.pc_name = "UNUSED",
@@ -735,6 +755,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_shareargs,
 		.pc_encode = nlmsvc_encode_shareres,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St+1,
 		.pc_name = "SHARE",
@@ -744,6 +765,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_shareargs,
 		.pc_encode = nlmsvc_encode_shareres,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St+1,
 		.pc_name = "UNSHARE",
@@ -753,6 +775,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_lockargs,
 		.pc_encode = nlmsvc_encode_res,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_res),
 		.pc_xdrressize = Ck+St,
 		.pc_name = "NM_LOCK",
@@ -762,6 +785,7 @@ const struct svc_procedure nlmsvc_procedures[24] = {
 		.pc_decode = nlmsvc_decode_notify,
 		.pc_encode = nlmsvc_encode_void,
 		.pc_argsize = sizeof(struct nlm_args),
+		.pc_argzero = sizeof(struct nlm_args),
 		.pc_ressize = sizeof(struct nlm_void),
 		.pc_xdrressize = 0,
 		.pc_name = "FREE_ALL",
diff --git a/fs/namei.c b/fs/namei.c
index 53b4bc094db2..8533087e5dac 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -986,7 +986,7 @@ static int nd_jump_root(struct nameidata *nd)
  * Helper to directly jump to a known parsed path from ->get_link,
  * caller must have taken a reference to path beforehand.
  */
-int nd_jump_link(struct path *path)
+int nd_jump_link(const struct path *path)
 {
 	int error = -ELOOP;
 	struct nameidata *nd = current->nameidata;
@@ -1178,7 +1178,7 @@ static bool safe_hardlink_source(struct user_namespace *mnt_userns,
  *
  * Returns 0 if successful, -ve on error.
  */
-int may_linkat(struct user_namespace *mnt_userns, struct path *link)
+int may_linkat(struct user_namespace *mnt_userns, const struct path *link)
 {
 	struct inode *inode = link->dentry->d_inode;
 
diff --git a/fs/nfs/callback_xdr.c b/fs/nfs/callback_xdr.c
index 8dcb08e1a885..d0cccddb7d08 100644
--- a/fs/nfs/callback_xdr.c
+++ b/fs/nfs/callback_xdr.c
@@ -1065,6 +1065,7 @@ static const struct svc_procedure nfs4_callback_procedures1[] = {
 		.pc_func = nfs4_callback_compound,
 		.pc_encode = nfs4_encode_void,
 		.pc_argsize = 256,
+		.pc_argzero = 256,
 		.pc_ressize = 256,
 		.pc_xdrressize = NFS4_CALLBACK_BUFSIZE,
 		.pc_name = "COMPOUND",
diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index 5d6c2ddc7ea6..58036f657126 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -2022,7 +2022,7 @@ static int nfs_finish_open(struct nfs_open_context *ctx,
 	err = finish_open(file, dentry, do_open);
 	if (err)
 		goto out;
-	if (S_ISREG(file->f_path.dentry->d_inode->i_mode))
+	if (S_ISREG(file_inode(file)->i_mode))
 		nfs_file_set_open_context(file, ctx);
 	else
 		err = -EOPENSTALE;
diff --git a/fs/nfs/nfs42proc.c b/fs/nfs/nfs42proc.c
index 6dab9e408372..d37e4a5401b1 100644
--- a/fs/nfs/nfs42proc.c
+++ b/fs/nfs/nfs42proc.c
@@ -341,7 +341,7 @@ static ssize_t _nfs42_proc_copy(struct file *src,
 			return status;
 		}
 	}
-	status = nfs_filemap_write_and_wait_range(file_inode(src)->i_mapping,
+	status = nfs_filemap_write_and_wait_range(src->f_mapping,
 			pos_src, pos_src + (loff_t)count - 1);
 	if (status)
 		return status;
diff --git a/fs/nfsd/cache.h b/fs/nfsd/cache.h
index 65c331f75e9c..f21259ead64b 100644
--- a/fs/nfsd/cache.h
+++ b/fs/nfsd/cache.h
@@ -84,6 +84,6 @@ int	nfsd_reply_cache_init(struct nfsd_net *);
 void	nfsd_reply_cache_shutdown(struct nfsd_net *);
 int	nfsd_cache_lookup(struct svc_rqst *);
 void	nfsd_cache_update(struct svc_rqst *, int, __be32 *);
-int	nfsd_reply_cache_stats_open(struct inode *, struct file *);
+int	nfsd_reply_cache_stats_show(struct seq_file *m, void *v);
 
 #endif /* NFSCACHE_H */
diff --git a/fs/nfsd/filecache.c b/fs/nfsd/filecache.c
index eeed4ae5b4ad..d5c57360b418 100644
--- a/fs/nfsd/filecache.c
+++ b/fs/nfsd/filecache.c
@@ -1212,7 +1212,7 @@ nfsd_file_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
  * scraping this file for info should test the labels to ensure they're
  * getting the correct field.
  */
-static int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
+int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
 {
 	unsigned long releases = 0, pages_flushed = 0, evictions = 0;
 	unsigned long hits = 0, acquisitions = 0;
@@ -1259,8 +1259,3 @@ static int nfsd_file_cache_stats_show(struct seq_file *m, void *v)
 	seq_printf(m, "pages flushed: %lu\n", pages_flushed);
 	return 0;
 }
-
-int nfsd_file_cache_stats_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, nfsd_file_cache_stats_show, NULL);
-}
diff --git a/fs/nfsd/filecache.h b/fs/nfsd/filecache.h
index 8e8c0c47d67d..357832bac736 100644
--- a/fs/nfsd/filecache.h
+++ b/fs/nfsd/filecache.h
@@ -60,5 +60,5 @@ __be32 nfsd_file_acquire(struct svc_rqst *rqstp, struct svc_fh *fhp,
 		  unsigned int may_flags, struct nfsd_file **nfp);
 __be32 nfsd_file_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
 		  unsigned int may_flags, struct nfsd_file **nfp);
-int	nfsd_file_cache_stats_open(struct inode *, struct file *);
+int nfsd_file_cache_stats_show(struct seq_file *m, void *v);
 #endif /* _FS_NFSD_FILECACHE_H */
diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h
index ffe17743cc74..8c854ba3285b 100644
--- a/fs/nfsd/netns.h
+++ b/fs/nfsd/netns.h
@@ -192,6 +192,10 @@ struct nfsd_net {
 
 	atomic_t		nfs4_client_count;
 	int			nfs4_max_clients;
+
+	atomic_t		nfsd_courtesy_clients;
+	struct shrinker		nfsd_client_shrinker;
+	struct delayed_work	nfsd_shrinker_work;
 };
 
 /* Simple check to find out if a given net was properly initialized */
diff --git a/fs/nfsd/nfs2acl.c b/fs/nfsd/nfs2acl.c
index 9edd3c1a30fb..13e6e6897f6c 100644
--- a/fs/nfsd/nfs2acl.c
+++ b/fs/nfsd/nfs2acl.c
@@ -331,6 +331,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = {
 		.pc_decode = nfssvc_decode_voidarg,
 		.pc_encode = nfssvc_encode_voidres,
 		.pc_argsize = sizeof(struct nfsd_voidargs),
+		.pc_argzero = sizeof(struct nfsd_voidargs),
 		.pc_ressize = sizeof(struct nfsd_voidres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST,
@@ -342,6 +343,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = {
 		.pc_encode = nfsaclsvc_encode_getaclres,
 		.pc_release = nfsaclsvc_release_getacl,
 		.pc_argsize = sizeof(struct nfsd3_getaclargs),
+		.pc_argzero = sizeof(struct nfsd3_getaclargs),
 		.pc_ressize = sizeof(struct nfsd3_getaclres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+1+2*(1+ACL),
@@ -353,6 +355,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = {
 		.pc_encode = nfssvc_encode_attrstatres,
 		.pc_release = nfssvc_release_attrstat,
 		.pc_argsize = sizeof(struct nfsd3_setaclargs),
+		.pc_argzero = sizeof(struct nfsd3_setaclargs),
 		.pc_ressize = sizeof(struct nfsd_attrstat),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+AT,
@@ -364,6 +367,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = {
 		.pc_encode = nfssvc_encode_attrstatres,
 		.pc_release = nfssvc_release_attrstat,
 		.pc_argsize = sizeof(struct nfsd_fhandle),
+		.pc_argzero = sizeof(struct nfsd_fhandle),
 		.pc_ressize = sizeof(struct nfsd_attrstat),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+AT,
@@ -375,6 +379,7 @@ static const struct svc_procedure nfsd_acl_procedures2[5] = {
 		.pc_encode = nfsaclsvc_encode_accessres,
 		.pc_release = nfsaclsvc_release_access,
 		.pc_argsize = sizeof(struct nfsd3_accessargs),
+		.pc_argzero = sizeof(struct nfsd3_accessargs),
 		.pc_ressize = sizeof(struct nfsd3_accessres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+AT+1,
diff --git a/fs/nfsd/nfs3acl.c b/fs/nfsd/nfs3acl.c
index 9446c6743664..2fb9ee356455 100644
--- a/fs/nfsd/nfs3acl.c
+++ b/fs/nfsd/nfs3acl.c
@@ -252,6 +252,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = {
 		.pc_decode = nfssvc_decode_voidarg,
 		.pc_encode = nfssvc_encode_voidres,
 		.pc_argsize = sizeof(struct nfsd_voidargs),
+		.pc_argzero = sizeof(struct nfsd_voidargs),
 		.pc_ressize = sizeof(struct nfsd_voidres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST,
@@ -263,6 +264,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = {
 		.pc_encode = nfs3svc_encode_getaclres,
 		.pc_release = nfs3svc_release_getacl,
 		.pc_argsize = sizeof(struct nfsd3_getaclargs),
+		.pc_argzero = sizeof(struct nfsd3_getaclargs),
 		.pc_ressize = sizeof(struct nfsd3_getaclres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+1+2*(1+ACL),
@@ -274,6 +276,7 @@ static const struct svc_procedure nfsd_acl_procedures3[3] = {
 		.pc_encode = nfs3svc_encode_setaclres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd3_setaclargs),
+		.pc_argzero = sizeof(struct nfsd3_setaclargs),
 		.pc_ressize = sizeof(struct nfsd3_attrstat),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+pAT,
diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c
index a41cca619338..923d9a80df92 100644
--- a/fs/nfsd/nfs3proc.c
+++ b/fs/nfsd/nfs3proc.c
@@ -150,7 +150,6 @@ nfsd3_proc_read(struct svc_rqst *rqstp)
 {
 	struct nfsd3_readargs *argp = rqstp->rq_argp;
 	struct nfsd3_readres *resp = rqstp->rq_resp;
-	u32 max_blocksize = svc_max_payload(rqstp);
 	unsigned int len;
 	int v;
 
@@ -159,7 +158,8 @@ nfsd3_proc_read(struct svc_rqst *rqstp)
 				(unsigned long) argp->count,
 				(unsigned long long) argp->offset);
 
-	argp->count = min_t(u32, argp->count, max_blocksize);
+	argp->count = min_t(u32, argp->count, svc_max_payload(rqstp));
+	argp->count = min_t(u32, argp->count, rqstp->rq_res.buflen);
 	if (argp->offset > (u64)OFFSET_MAX)
 		argp->offset = (u64)OFFSET_MAX;
 	if (argp->offset + argp->count > (u64)OFFSET_MAX)
@@ -563,25 +563,18 @@ static void nfsd3_init_dirlist_pages(struct svc_rqst *rqstp,
 {
 	struct xdr_buf *buf = &resp->dirlist;
 	struct xdr_stream *xdr = &resp->xdr;
-
-	count = clamp(count, (u32)(XDR_UNIT * 2), svc_max_payload(rqstp));
+	unsigned int sendbuf = min_t(unsigned int, rqstp->rq_res.buflen,
+				     svc_max_payload(rqstp));
 
 	memset(buf, 0, sizeof(*buf));
 
 	/* Reserve room for the NULL ptr & eof flag (-2 words) */
-	buf->buflen = count - XDR_UNIT * 2;
+	buf->buflen = clamp(count, (u32)(XDR_UNIT * 2), sendbuf);
+	buf->buflen -= XDR_UNIT * 2;
 	buf->pages = rqstp->rq_next_page;
 	rqstp->rq_next_page += (buf->buflen + PAGE_SIZE - 1) >> PAGE_SHIFT;
 
-	/* This is xdr_init_encode(), but it assumes that
-	 * the head kvec has already been consumed. */
-	xdr_set_scratch_buffer(xdr, NULL, 0);
-	xdr->buf = buf;
-	xdr->page_ptr = buf->pages;
-	xdr->iov = NULL;
-	xdr->p = page_address(*buf->pages);
-	xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE);
-	xdr->rqst = NULL;
+	xdr_init_encode_pages(xdr, buf, buf->pages,  NULL);
 }
 
 /*
@@ -808,6 +801,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_decode = nfssvc_decode_voidarg,
 		.pc_encode = nfssvc_encode_voidres,
 		.pc_argsize = sizeof(struct nfsd_voidargs),
+		.pc_argzero = sizeof(struct nfsd_voidargs),
 		.pc_ressize = sizeof(struct nfsd_voidres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST,
@@ -819,6 +813,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_getattrres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd_fhandle),
+		.pc_argzero = sizeof(struct nfsd_fhandle),
 		.pc_ressize = sizeof(struct nfsd3_attrstatres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+AT,
@@ -830,6 +825,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_wccstatres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd3_sattrargs),
+		.pc_argzero = sizeof(struct nfsd3_sattrargs),
 		.pc_ressize = sizeof(struct nfsd3_wccstatres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+WC,
@@ -841,6 +837,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_lookupres,
 		.pc_release = nfs3svc_release_fhandle2,
 		.pc_argsize = sizeof(struct nfsd3_diropargs),
+		.pc_argzero = sizeof(struct nfsd3_diropargs),
 		.pc_ressize = sizeof(struct nfsd3_diropres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+FH+pAT+pAT,
@@ -852,6 +849,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_accessres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd3_accessargs),
+		.pc_argzero = sizeof(struct nfsd3_accessargs),
 		.pc_ressize = sizeof(struct nfsd3_accessres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+pAT+1,
@@ -863,6 +861,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_readlinkres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd_fhandle),
+		.pc_argzero = sizeof(struct nfsd_fhandle),
 		.pc_ressize = sizeof(struct nfsd3_readlinkres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+pAT+1+NFS3_MAXPATHLEN/4,
@@ -874,6 +873,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_readres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd3_readargs),
+		.pc_argzero = sizeof(struct nfsd3_readargs),
 		.pc_ressize = sizeof(struct nfsd3_readres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+pAT+4+NFSSVC_MAXBLKSIZE/4,
@@ -885,6 +885,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_writeres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd3_writeargs),
+		.pc_argzero = sizeof(struct nfsd3_writeargs),
 		.pc_ressize = sizeof(struct nfsd3_writeres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+WC+4,
@@ -896,6 +897,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_createres,
 		.pc_release = nfs3svc_release_fhandle2,
 		.pc_argsize = sizeof(struct nfsd3_createargs),
+		.pc_argzero = sizeof(struct nfsd3_createargs),
 		.pc_ressize = sizeof(struct nfsd3_createres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+(1+FH+pAT)+WC,
@@ -907,6 +909,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_createres,
 		.pc_release = nfs3svc_release_fhandle2,
 		.pc_argsize = sizeof(struct nfsd3_mkdirargs),
+		.pc_argzero = sizeof(struct nfsd3_mkdirargs),
 		.pc_ressize = sizeof(struct nfsd3_createres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+(1+FH+pAT)+WC,
@@ -918,6 +921,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_createres,
 		.pc_release = nfs3svc_release_fhandle2,
 		.pc_argsize = sizeof(struct nfsd3_symlinkargs),
+		.pc_argzero = sizeof(struct nfsd3_symlinkargs),
 		.pc_ressize = sizeof(struct nfsd3_createres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+(1+FH+pAT)+WC,
@@ -929,6 +933,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_createres,
 		.pc_release = nfs3svc_release_fhandle2,
 		.pc_argsize = sizeof(struct nfsd3_mknodargs),
+		.pc_argzero = sizeof(struct nfsd3_mknodargs),
 		.pc_ressize = sizeof(struct nfsd3_createres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+(1+FH+pAT)+WC,
@@ -940,6 +945,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_wccstatres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd3_diropargs),
+		.pc_argzero = sizeof(struct nfsd3_diropargs),
 		.pc_ressize = sizeof(struct nfsd3_wccstatres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+WC,
@@ -951,6 +957,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_wccstatres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd3_diropargs),
+		.pc_argzero = sizeof(struct nfsd3_diropargs),
 		.pc_ressize = sizeof(struct nfsd3_wccstatres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+WC,
@@ -962,6 +969,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_renameres,
 		.pc_release = nfs3svc_release_fhandle2,
 		.pc_argsize = sizeof(struct nfsd3_renameargs),
+		.pc_argzero = sizeof(struct nfsd3_renameargs),
 		.pc_ressize = sizeof(struct nfsd3_renameres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+WC+WC,
@@ -973,6 +981,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_linkres,
 		.pc_release = nfs3svc_release_fhandle2,
 		.pc_argsize = sizeof(struct nfsd3_linkargs),
+		.pc_argzero = sizeof(struct nfsd3_linkargs),
 		.pc_ressize = sizeof(struct nfsd3_linkres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+pAT+WC,
@@ -984,6 +993,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_readdirres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd3_readdirargs),
+		.pc_argzero = sizeof(struct nfsd3_readdirargs),
 		.pc_ressize = sizeof(struct nfsd3_readdirres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_name = "READDIR",
@@ -994,6 +1004,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_readdirres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd3_readdirplusargs),
+		.pc_argzero = sizeof(struct nfsd3_readdirplusargs),
 		.pc_ressize = sizeof(struct nfsd3_readdirres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_name = "READDIRPLUS",
@@ -1003,6 +1014,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_decode = nfs3svc_decode_fhandleargs,
 		.pc_encode = nfs3svc_encode_fsstatres,
 		.pc_argsize = sizeof(struct nfsd3_fhandleargs),
+		.pc_argzero = sizeof(struct nfsd3_fhandleargs),
 		.pc_ressize = sizeof(struct nfsd3_fsstatres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+pAT+2*6+1,
@@ -1013,6 +1025,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_decode = nfs3svc_decode_fhandleargs,
 		.pc_encode = nfs3svc_encode_fsinfores,
 		.pc_argsize = sizeof(struct nfsd3_fhandleargs),
+		.pc_argzero = sizeof(struct nfsd3_fhandleargs),
 		.pc_ressize = sizeof(struct nfsd3_fsinfores),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+pAT+12,
@@ -1023,6 +1036,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_decode = nfs3svc_decode_fhandleargs,
 		.pc_encode = nfs3svc_encode_pathconfres,
 		.pc_argsize = sizeof(struct nfsd3_fhandleargs),
+		.pc_argzero = sizeof(struct nfsd3_fhandleargs),
 		.pc_ressize = sizeof(struct nfsd3_pathconfres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+pAT+6,
@@ -1034,6 +1048,7 @@ static const struct svc_procedure nfsd_procedures3[22] = {
 		.pc_encode = nfs3svc_encode_commitres,
 		.pc_release = nfs3svc_release_fhandle,
 		.pc_argsize = sizeof(struct nfsd3_commitargs),
+		.pc_argzero = sizeof(struct nfsd3_commitargs),
 		.pc_ressize = sizeof(struct nfsd3_commitres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+WC+2,
diff --git a/fs/nfsd/nfs3xdr.c b/fs/nfsd/nfs3xdr.c
index 0293b8d65f10..3308dd671ef0 100644
--- a/fs/nfsd/nfs3xdr.c
+++ b/fs/nfsd/nfs3xdr.c
@@ -571,10 +571,8 @@ nfs3svc_decode_writeargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
 		args->count = max_blocksize;
 		args->len = max_blocksize;
 	}
-	if (!xdr_stream_subsegment(xdr, &args->payload, args->count))
-		return false;
 
-	return true;
+	return xdr_stream_subsegment(xdr, &args->payload, args->count);
 }
 
 bool
@@ -616,8 +614,6 @@ nfs3svc_decode_symlinkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
 {
 	struct nfsd3_symlinkargs *args = rqstp->rq_argp;
 	struct kvec *head = rqstp->rq_arg.head;
-	struct kvec *tail = rqstp->rq_arg.tail;
-	size_t remaining;
 
 	if (!svcxdr_decode_diropargs3(xdr, &args->ffh, &args->fname, &args->flen))
 		return false;
@@ -626,16 +622,10 @@ nfs3svc_decode_symlinkargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
 	if (xdr_stream_decode_u32(xdr, &args->tlen) < 0)
 		return false;
 
-	/* request sanity */
-	remaining = head->iov_len + rqstp->rq_arg.page_len + tail->iov_len;
-	remaining -= xdr_stream_pos(xdr);
-	if (remaining < xdr_align_size(args->tlen))
-		return false;
-
-	args->first.iov_base = xdr->p;
+	/* symlink_data */
 	args->first.iov_len = head->iov_len - xdr_stream_pos(xdr);
-
-	return true;
+	args->first.iov_base = xdr_inline_decode(xdr, args->tlen);
+	return args->first.iov_base != NULL;
 }
 
 bool
diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c
index 4ce328209f61..f0e69edf5f0f 100644
--- a/fs/nfsd/nfs4callback.c
+++ b/fs/nfsd/nfs4callback.c
@@ -1371,11 +1371,21 @@ void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,
 	cb->cb_holds_slot = false;
 }
 
-void nfsd4_run_cb(struct nfsd4_callback *cb)
+/**
+ * nfsd4_run_cb - queue up a callback job to run
+ * @cb: callback to queue
+ *
+ * Kick off a callback to do its thing. Returns false if it was already
+ * on a queue, true otherwise.
+ */
+bool nfsd4_run_cb(struct nfsd4_callback *cb)
 {
 	struct nfs4_client *clp = cb->cb_clp;
+	bool queued;
 
 	nfsd41_cb_inflight_begin(clp);
-	if (!nfsd4_queue_cb(cb))
+	queued = nfsd4_queue_cb(cb);
+	if (!queued)
 		nfsd41_cb_inflight_end(clp);
+	return queued;
 }
diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c
index f92161ce1f97..e70a1a2999b7 100644
--- a/fs/nfsd/nfs4idmap.c
+++ b/fs/nfsd/nfs4idmap.c
@@ -82,8 +82,8 @@ ent_init(struct cache_head *cnew, struct cache_head *citm)
 	new->id = itm->id;
 	new->type = itm->type;
 
-	strlcpy(new->name, itm->name, sizeof(new->name));
-	strlcpy(new->authname, itm->authname, sizeof(new->authname));
+	strscpy(new->name, itm->name, sizeof(new->name));
+	strscpy(new->authname, itm->authname, sizeof(new->authname));
 }
 
 static void
@@ -548,7 +548,7 @@ idmap_name_to_id(struct svc_rqst *rqstp, int type, const char *name, u32 namelen
 		return nfserr_badowner;
 	memcpy(key.name, name, namelen);
 	key.name[namelen] = '\0';
-	strlcpy(key.authname, rqst_authname(rqstp), sizeof(key.authname));
+	strscpy(key.authname, rqst_authname(rqstp), sizeof(key.authname));
 	ret = idmap_lookup(rqstp, nametoid_lookup, &key, nn->nametoid_cache, &item);
 	if (ret == -ENOENT)
 		return nfserr_badowner;
@@ -584,7 +584,7 @@ static __be32 idmap_id_to_name(struct xdr_stream *xdr,
 	int ret;
 	struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
 
-	strlcpy(key.authname, rqst_authname(rqstp), sizeof(key.authname));
+	strscpy(key.authname, rqst_authname(rqstp), sizeof(key.authname));
 	ret = idmap_lookup(rqstp, idtoname_lookup, &key, nn->idtoname_cache, &item);
 	if (ret == -ENOENT)
 		return encode_ascii_id(xdr, id);
diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
index 2c05692a9abf..3564d1c6f610 100644
--- a/fs/nfsd/nfs4layouts.c
+++ b/fs/nfsd/nfs4layouts.c
@@ -658,7 +658,7 @@ nfsd4_cb_layout_done(struct nfsd4_callback *cb, struct rpc_task *task)
 	ktime_t now, cutoff;
 	const struct nfsd4_layout_ops *ops;
 
-
+	trace_nfsd_cb_layout_done(&ls->ls_stid.sc_stateid, task);
 	switch (task->tk_status) {
 	case 0:
 	case -NFS4ERR_DELAY:
diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index a72ab97f77ef..8beb2bc4c328 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -141,7 +141,6 @@ fh_dup2(struct svc_fh *dst, struct svc_fh *src)
 static __be32
 do_open_permission(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfsd4_open *open, int accmode)
 {
-	__be32 status;
 
 	if (open->op_truncate &&
 		!(open->op_share_access & NFS4_SHARE_ACCESS_WRITE))
@@ -156,9 +155,7 @@ do_open_permission(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfs
 	if (open->op_share_deny & NFS4_SHARE_DENY_READ)
 		accmode |= NFSD_MAY_WRITE;
 
-	status = fh_verify(rqstp, current_fh, S_IFREG, accmode);
-
-	return status;
+	return fh_verify(rqstp, current_fh, S_IFREG, accmode);
 }
 
 static __be32 nfsd_check_obj_isreg(struct svc_fh *fh)
@@ -454,7 +451,6 @@ static __be32
 do_open_fhandle(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, struct nfsd4_open *open)
 {
 	struct svc_fh *current_fh = &cstate->current_fh;
-	__be32 status;
 	int accmode = 0;
 
 	/* We don't know the target directory, and therefore can not
@@ -479,9 +475,7 @@ do_open_fhandle(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, str
 	if (open->op_claim_type == NFS4_OPEN_CLAIM_DELEG_CUR_FH)
 		accmode = NFSD_MAY_OWNER_OVERRIDE;
 
-	status = do_open_permission(rqstp, current_fh, open, accmode);
-
-	return status;
+	return do_open_permission(rqstp, current_fh, open, accmode);
 }
 
 static void
@@ -668,11 +662,9 @@ static __be32
 nfsd4_putrootfh(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		union nfsd4_op_u *u)
 {
-	__be32 status;
-
 	fh_put(&cstate->current_fh);
-	status = exp_pseudoroot(rqstp, &cstate->current_fh);
-	return status;
+
+	return exp_pseudoroot(rqstp, &cstate->current_fh);
 }
 
 static __be32
@@ -1343,7 +1335,7 @@ try_again:
 		return 0;
 	}
 	if (work) {
-		strlcpy(work->nsui_ipaddr, ipaddr, sizeof(work->nsui_ipaddr) - 1);
+		strscpy(work->nsui_ipaddr, ipaddr, sizeof(work->nsui_ipaddr) - 1);
 		refcount_set(&work->nsui_refcnt, 2);
 		work->nsui_busy = true;
 		list_add_tail(&work->nsui_list, &nn->nfsd_ssc_mount_list);
@@ -1621,6 +1613,10 @@ static void nfsd4_cb_offload_release(struct nfsd4_callback *cb)
 static int nfsd4_cb_offload_done(struct nfsd4_callback *cb,
 				 struct rpc_task *task)
 {
+	struct nfsd4_cb_offload *cbo =
+		container_of(cb, struct nfsd4_cb_offload, co_cb);
+
+	trace_nfsd_cb_offload_done(&cbo->co_res.cb_stateid, task);
 	return 1;
 }
 
@@ -1768,7 +1764,13 @@ static int nfsd4_do_async_copy(void *data)
 		filp = nfs42_ssc_open(copy->ss_mnt, &copy->c_fh,
 				      &copy->stateid);
 		if (IS_ERR(filp)) {
-			nfserr = nfserr_offload_denied;
+			switch (PTR_ERR(filp)) {
+			case -EBADF:
+				nfserr = nfserr_wrong_type;
+				break;
+			default:
+				nfserr = nfserr_offload_denied;
+			}
 			nfsd4_interssc_disconnect(copy->ss_mnt);
 			goto do_callback;
 		}
@@ -1826,7 +1828,7 @@ nfsd4_copy(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		if (!nfs4_init_copy_state(nn, copy))
 			goto out_err;
 		refcount_set(&async_copy->refcount, 1);
-		memcpy(&copy->cp_res.cb_stateid, &copy->cp_stateid.stid,
+		memcpy(&copy->cp_res.cb_stateid, &copy->cp_stateid.cs_stid,
 			sizeof(copy->cp_res.cb_stateid));
 		dup_copy_fields(copy, async_copy);
 		async_copy->copy_task = kthread_create(nfsd4_do_async_copy,
@@ -1862,7 +1864,7 @@ find_async_copy(struct nfs4_client *clp, stateid_t *stateid)
 
 	spin_lock(&clp->async_lock);
 	list_for_each_entry(copy, &clp->async_copies, copies) {
-		if (memcmp(&copy->cp_stateid.stid, stateid, NFS4_STATEID_SIZE))
+		if (memcmp(&copy->cp_stateid.cs_stid, stateid, NFS4_STATEID_SIZE))
 			continue;
 		refcount_inc(&copy->refcount);
 		spin_unlock(&clp->async_lock);
@@ -1916,7 +1918,7 @@ nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	cps = nfs4_alloc_init_cpntf_state(nn, stid);
 	if (!cps)
 		goto out;
-	memcpy(&cn->cpn_cnr_stateid, &cps->cp_stateid.stid, sizeof(stateid_t));
+	memcpy(&cn->cpn_cnr_stateid, &cps->cp_stateid.cs_stid, sizeof(stateid_t));
 	memcpy(&cps->cp_p_stateid, &stid->sc_stateid, sizeof(stateid_t));
 	memcpy(&cps->cp_p_clid, &clp->cl_clientid, sizeof(clientid_t));
 
@@ -2633,9 +2635,6 @@ nfsd4_proc_compound(struct svc_rqst *rqstp)
 	status = nfserr_minor_vers_mismatch;
 	if (nfsd_minorversion(nn, args->minorversion, NFSD_TEST) <= 0)
 		goto out;
-	status = nfserr_resource;
-	if (args->opcnt > NFSD_MAX_OPS_PER_COMPOUND)
-		goto out;
 
 	status = nfs41_check_op_ordering(args);
 	if (status) {
@@ -2648,10 +2647,20 @@ nfsd4_proc_compound(struct svc_rqst *rqstp)
 
 	rqstp->rq_lease_breaker = (void **)&cstate->clp;
 
-	trace_nfsd_compound(rqstp, args->opcnt);
+	trace_nfsd_compound(rqstp, args->tag, args->taglen, args->client_opcnt);
 	while (!status && resp->opcnt < args->opcnt) {
 		op = &args->ops[resp->opcnt++];
 
+		if (unlikely(resp->opcnt == NFSD_MAX_OPS_PER_COMPOUND)) {
+			/* If there are still more operations to process,
+			 * stop here and report NFS4ERR_RESOURCE. */
+			if (cstate->minorversion == 0 &&
+			    args->client_opcnt > resp->opcnt) {
+				op->status = nfserr_resource;
+				goto encode_op;
+			}
+		}
+
 		/*
 		 * The XDR decode routines may have pre-set op->status;
 		 * for example, if there is a miscellaneous XDR error
@@ -2727,8 +2736,8 @@ encode_op:
 			status = op->status;
 		}
 
-		trace_nfsd_compound_status(args->opcnt, resp->opcnt, status,
-					   nfsd4_op_name(op->opnum));
+		trace_nfsd_compound_status(args->client_opcnt, resp->opcnt,
+					   status, nfsd4_op_name(op->opnum));
 
 		nfsd4_cstate_clear_replay(cstate);
 		nfsd4_increment_op_stats(op->opnum);
@@ -2762,28 +2771,49 @@ out:
 
 #define op_encode_channel_attrs_maxsz	(6 + 1 + 1)
 
-static inline u32 nfsd4_only_status_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+/*
+ * The _rsize() helpers are invoked by the NFSv4 COMPOUND decoder, which
+ * is called before sunrpc sets rq_res.buflen. Thus we have to compute
+ * the maximum payload size here, based on transport limits and the size
+ * of the remaining space in the rq_pages array.
+ */
+static u32 nfsd4_max_payload(const struct svc_rqst *rqstp)
+{
+	u32 buflen;
+
+	buflen = (rqstp->rq_page_end - rqstp->rq_next_page) * PAGE_SIZE;
+	buflen -= rqstp->rq_auth_slack;
+	buflen -= rqstp->rq_res.head[0].iov_len;
+	return min_t(u32, buflen, svc_max_payload(rqstp));
+}
+
+static u32 nfsd4_only_status_rsize(const struct svc_rqst *rqstp,
+				   const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_status_stateid_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_status_stateid_rsize(const struct svc_rqst *rqstp,
+				      const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + op_encode_stateid_maxsz)* sizeof(__be32);
 }
 
-static inline u32 nfsd4_access_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_access_rsize(const struct svc_rqst *rqstp,
+			      const struct nfsd4_op *op)
 {
 	/* ac_supported, ac_resp_access */
 	return (op_encode_hdr_size + 2)* sizeof(__be32);
 }
 
-static inline u32 nfsd4_commit_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_commit_rsize(const struct svc_rqst *rqstp,
+			      const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + op_encode_verifier_maxsz) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_create_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_create_rsize(const struct svc_rqst *rqstp,
+			      const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + op_encode_change_info_maxsz
 		+ nfs4_fattr_bitmap_maxsz) * sizeof(__be32);
@@ -2794,17 +2824,17 @@ static inline u32 nfsd4_create_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op
  * the op prematurely if the estimate is too large.  We may turn off splice
  * reads unnecessarily.
  */
-static inline u32 nfsd4_getattr_rsize(struct svc_rqst *rqstp,
-				      struct nfsd4_op *op)
+static u32 nfsd4_getattr_rsize(const struct svc_rqst *rqstp,
+			       const struct nfsd4_op *op)
 {
-	u32 *bmap = op->u.getattr.ga_bmval;
+	const u32 *bmap = op->u.getattr.ga_bmval;
 	u32 bmap0 = bmap[0], bmap1 = bmap[1], bmap2 = bmap[2];
 	u32 ret = 0;
 
 	if (bmap0 & FATTR4_WORD0_ACL)
-		return svc_max_payload(rqstp);
+		return nfsd4_max_payload(rqstp);
 	if (bmap0 & FATTR4_WORD0_FS_LOCATIONS)
-		return svc_max_payload(rqstp);
+		return nfsd4_max_payload(rqstp);
 
 	if (bmap1 & FATTR4_WORD1_OWNER) {
 		ret += IDMAP_NAMESZ + 4;
@@ -2832,24 +2862,28 @@ static inline u32 nfsd4_getattr_rsize(struct svc_rqst *rqstp,
 	return ret;
 }
 
-static inline u32 nfsd4_getfh_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_getfh_rsize(const struct svc_rqst *rqstp,
+			     const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + 1) * sizeof(__be32) + NFS4_FHSIZE;
 }
 
-static inline u32 nfsd4_link_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_link_rsize(const struct svc_rqst *rqstp,
+			    const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + op_encode_change_info_maxsz)
 		* sizeof(__be32);
 }
 
-static inline u32 nfsd4_lock_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_lock_rsize(const struct svc_rqst *rqstp,
+			    const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + op_encode_lock_denied_maxsz)
 		* sizeof(__be32);
 }
 
-static inline u32 nfsd4_open_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_open_rsize(const struct svc_rqst *rqstp,
+			    const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + op_encode_stateid_maxsz
 		+ op_encode_change_info_maxsz + 1
@@ -2857,20 +2891,18 @@ static inline u32 nfsd4_open_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 		+ op_encode_delegation_maxsz) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_read_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_read_rsize(const struct svc_rqst *rqstp,
+			    const struct nfsd4_op *op)
 {
-	u32 maxcount = 0, rlen = 0;
-
-	maxcount = svc_max_payload(rqstp);
-	rlen = min(op->u.read.rd_length, maxcount);
+	u32 rlen = min(op->u.read.rd_length, nfsd4_max_payload(rqstp));
 
 	return (op_encode_hdr_size + 2 + XDR_QUADLEN(rlen)) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_read_plus_rsize(const struct svc_rqst *rqstp,
+				 const struct nfsd4_op *op)
 {
-	u32 maxcount = svc_max_payload(rqstp);
-	u32 rlen = min(op->u.read.rd_length, maxcount);
+	u32 rlen = min(op->u.read.rd_length, nfsd4_max_payload(rqstp));
 	/*
 	 * If we detect that the file changed during hole encoding, then we
 	 * recover by encoding the remaining reply as data. This means we need
@@ -2881,70 +2913,77 @@ static inline u32 nfsd4_read_plus_rsize(struct svc_rqst *rqstp, struct nfsd4_op
 	return (op_encode_hdr_size + 2 + seg_len + XDR_QUADLEN(rlen)) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_readdir_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_readdir_rsize(const struct svc_rqst *rqstp,
+			       const struct nfsd4_op *op)
 {
-	u32 maxcount = 0, rlen = 0;
-
-	maxcount = svc_max_payload(rqstp);
-	rlen = min(op->u.readdir.rd_maxcount, maxcount);
+	u32 rlen = min(op->u.readdir.rd_maxcount, nfsd4_max_payload(rqstp));
 
 	return (op_encode_hdr_size + op_encode_verifier_maxsz +
 		XDR_QUADLEN(rlen)) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_readlink_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_readlink_rsize(const struct svc_rqst *rqstp,
+				const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + 1) * sizeof(__be32) + PAGE_SIZE;
 }
 
-static inline u32 nfsd4_remove_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_remove_rsize(const struct svc_rqst *rqstp,
+			      const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + op_encode_change_info_maxsz)
 		* sizeof(__be32);
 }
 
-static inline u32 nfsd4_rename_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_rename_rsize(const struct svc_rqst *rqstp,
+			      const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + op_encode_change_info_maxsz
 		+ op_encode_change_info_maxsz) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_sequence_rsize(struct svc_rqst *rqstp,
-				       struct nfsd4_op *op)
+static u32 nfsd4_sequence_rsize(const struct svc_rqst *rqstp,
+				const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size
 		+ XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + 5) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_test_stateid_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_test_stateid_rsize(const struct svc_rqst *rqstp,
+				    const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + 1 + op->u.test_stateid.ts_num_ids)
 		* sizeof(__be32);
 }
 
-static inline u32 nfsd4_setattr_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_setattr_rsize(const struct svc_rqst *rqstp,
+			       const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + nfs4_fattr_bitmap_maxsz) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_secinfo_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_secinfo_rsize(const struct svc_rqst *rqstp,
+			       const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + RPC_AUTH_MAXFLAVOR *
 		(4 + XDR_QUADLEN(GSS_OID_MAX_LEN))) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_setclientid_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_setclientid_rsize(const struct svc_rqst *rqstp,
+				   const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + 2 + XDR_QUADLEN(NFS4_VERIFIER_SIZE)) *
 								sizeof(__be32);
 }
 
-static inline u32 nfsd4_write_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_write_rsize(const struct svc_rqst *rqstp,
+			     const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + 2 + op_encode_verifier_maxsz) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_exchange_id_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_exchange_id_rsize(const struct svc_rqst *rqstp,
+				   const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + 2 + 1 + /* eir_clientid, eir_sequenceid */\
 		1 + 1 + /* eir_flags, spr_how */\
@@ -2958,14 +2997,16 @@ static inline u32 nfsd4_exchange_id_rsize(struct svc_rqst *rqstp, struct nfsd4_o
 		0 /* ignored eir_server_impl_id contents */) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_bind_conn_to_session_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_bind_conn_to_session_rsize(const struct svc_rqst *rqstp,
+					    const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + \
 		XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + /* bctsr_sessid */\
 		2 /* bctsr_dir, use_conn_in_rdma_mode */) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_create_session_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_create_session_rsize(const struct svc_rqst *rqstp,
+				      const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + \
 		XDR_QUADLEN(NFS4_MAX_SESSIONID_LEN) + /* sessionid */\
@@ -2974,7 +3015,8 @@ static inline u32 nfsd4_create_session_rsize(struct svc_rqst *rqstp, struct nfsd
 		op_encode_channel_attrs_maxsz) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_copy_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_copy_rsize(const struct svc_rqst *rqstp,
+			    const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size +
 		1 /* wr_callback */ +
@@ -2986,16 +3028,16 @@ static inline u32 nfsd4_copy_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
 		1 /* cr_synchronous */) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_offload_status_rsize(struct svc_rqst *rqstp,
-					     struct nfsd4_op *op)
+static u32 nfsd4_offload_status_rsize(const struct svc_rqst *rqstp,
+				      const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size +
 		2 /* osr_count */ +
 		1 /* osr_complete<1> optional 0 for now */) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_copy_notify_rsize(struct svc_rqst *rqstp,
-					struct nfsd4_op *op)
+static u32 nfsd4_copy_notify_rsize(const struct svc_rqst *rqstp,
+				   const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size +
 		3 /* cnr_lease_time */ +
@@ -3010,12 +3052,10 @@ static inline u32 nfsd4_copy_notify_rsize(struct svc_rqst *rqstp,
 }
 
 #ifdef CONFIG_NFSD_PNFS
-static inline u32 nfsd4_getdeviceinfo_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_getdeviceinfo_rsize(const struct svc_rqst *rqstp,
+				     const struct nfsd4_op *op)
 {
-	u32 maxcount = 0, rlen = 0;
-
-	maxcount = svc_max_payload(rqstp);
-	rlen = min(op->u.getdeviceinfo.gd_maxcount, maxcount);
+	u32 rlen = min(op->u.getdeviceinfo.gd_maxcount, nfsd4_max_payload(rqstp));
 
 	return (op_encode_hdr_size +
 		1 /* gd_layout_type*/ +
@@ -3028,7 +3068,8 @@ static inline u32 nfsd4_getdeviceinfo_rsize(struct svc_rqst *rqstp, struct nfsd4
  * so we need to define an arbitrary upper bound here.
  */
 #define MAX_LAYOUT_SIZE		128
-static inline u32 nfsd4_layoutget_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_layoutget_rsize(const struct svc_rqst *rqstp,
+				 const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size +
 		1 /* logr_return_on_close */ +
@@ -3037,14 +3078,16 @@ static inline u32 nfsd4_layoutget_rsize(struct svc_rqst *rqstp, struct nfsd4_op
 		MAX_LAYOUT_SIZE) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_layoutcommit_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_layoutcommit_rsize(const struct svc_rqst *rqstp,
+				    const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size +
 		1 /* locr_newsize */ +
 		2 /* ns_size */) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_layoutreturn_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_layoutreturn_rsize(const struct svc_rqst *rqstp,
+				    const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size +
 		1 /* lrs_stateid */ +
@@ -3053,41 +3096,36 @@ static inline u32 nfsd4_layoutreturn_rsize(struct svc_rqst *rqstp, struct nfsd4_
 #endif /* CONFIG_NFSD_PNFS */
 
 
-static inline u32 nfsd4_seek_rsize(struct svc_rqst *rqstp, struct nfsd4_op *op)
+static u32 nfsd4_seek_rsize(const struct svc_rqst *rqstp,
+			    const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + 3) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_getxattr_rsize(struct svc_rqst *rqstp,
-				       struct nfsd4_op *op)
+static u32 nfsd4_getxattr_rsize(const struct svc_rqst *rqstp,
+				const struct nfsd4_op *op)
 {
-	u32 maxcount, rlen;
-
-	maxcount = svc_max_payload(rqstp);
-	rlen = min_t(u32, XATTR_SIZE_MAX, maxcount);
+	u32 rlen = min_t(u32, XATTR_SIZE_MAX, nfsd4_max_payload(rqstp));
 
 	return (op_encode_hdr_size + 1 + XDR_QUADLEN(rlen)) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_setxattr_rsize(struct svc_rqst *rqstp,
-				       struct nfsd4_op *op)
+static u32 nfsd4_setxattr_rsize(const struct svc_rqst *rqstp,
+				const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + op_encode_change_info_maxsz)
 		* sizeof(__be32);
 }
-static inline u32 nfsd4_listxattrs_rsize(struct svc_rqst *rqstp,
-					 struct nfsd4_op *op)
+static u32 nfsd4_listxattrs_rsize(const struct svc_rqst *rqstp,
+				  const struct nfsd4_op *op)
 {
-	u32 maxcount, rlen;
-
-	maxcount = svc_max_payload(rqstp);
-	rlen = min(op->u.listxattrs.lsxa_maxcount, maxcount);
+	u32 rlen = min(op->u.listxattrs.lsxa_maxcount, nfsd4_max_payload(rqstp));
 
 	return (op_encode_hdr_size + 4 + XDR_QUADLEN(rlen)) * sizeof(__be32);
 }
 
-static inline u32 nfsd4_removexattr_rsize(struct svc_rqst *rqstp,
-					  struct nfsd4_op *op)
+static u32 nfsd4_removexattr_rsize(const struct svc_rqst *rqstp,
+				   const struct nfsd4_op *op)
 {
 	return (op_encode_hdr_size + op_encode_change_info_maxsz)
 		* sizeof(__be32);
@@ -3576,6 +3614,7 @@ static const struct svc_procedure nfsd_procedures4[2] = {
 		.pc_decode = nfssvc_decode_voidarg,
 		.pc_encode = nfssvc_encode_voidres,
 		.pc_argsize = sizeof(struct nfsd_voidargs),
+		.pc_argzero = sizeof(struct nfsd_voidargs),
 		.pc_ressize = sizeof(struct nfsd_voidres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = 1,
@@ -3586,6 +3625,7 @@ static const struct svc_procedure nfsd_procedures4[2] = {
 		.pc_decode = nfs4svc_decode_compoundargs,
 		.pc_encode = nfs4svc_encode_compoundres,
 		.pc_argsize = sizeof(struct nfsd4_compoundargs),
+		.pc_argzero = offsetof(struct nfsd4_compoundargs, iops),
 		.pc_ressize = sizeof(struct nfsd4_compoundres),
 		.pc_release = nfsd4_release_compoundargs,
 		.pc_cachetype = RC_NOCACHE,
diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c
index c634483d85d2..78b8cd9651d5 100644
--- a/fs/nfsd/nfs4recover.c
+++ b/fs/nfsd/nfs4recover.c
@@ -266,7 +266,7 @@ struct nfs4_dir_ctx {
 	struct list_head names;
 };
 
-static int
+static bool
 nfsd4_build_namelist(struct dir_context *__ctx, const char *name, int namlen,
 		loff_t offset, u64 ino, unsigned int d_type)
 {
@@ -275,14 +275,14 @@ nfsd4_build_namelist(struct dir_context *__ctx, const char *name, int namlen,
 	struct name_list *entry;
 
 	if (namlen != HEXDIR_LEN - 1)
-		return 0;
+		return true;
 	entry = kmalloc(sizeof(struct name_list), GFP_KERNEL);
 	if (entry == NULL)
-		return -ENOMEM;
+		return false;
 	memcpy(entry->name, name, HEXDIR_LEN - 1);
 	entry->name[HEXDIR_LEN - 1] = '\0';
 	list_add(&entry->list, &ctx->names);
-	return 0;
+	return true;
 }
 
 static int
@@ -807,16 +807,18 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg,
 			if (get_user(namelen, &ci->cc_name.cn_len))
 				return -EFAULT;
 			name.data = memdup_user(&ci->cc_name.cn_id, namelen);
-			if (IS_ERR_OR_NULL(name.data))
-				return -EFAULT;
+			if (IS_ERR(name.data))
+				return PTR_ERR(name.data);
 			name.len = namelen;
 			get_user(princhashlen, &ci->cc_princhash.cp_len);
 			if (princhashlen > 0) {
 				princhash.data = memdup_user(
 						&ci->cc_princhash.cp_data,
 						princhashlen);
-				if (IS_ERR_OR_NULL(princhash.data))
-					return -EFAULT;
+				if (IS_ERR(princhash.data)) {
+					kfree(name.data);
+					return PTR_ERR(princhash.data);
+				}
 				princhash.len = princhashlen;
 			} else
 				princhash.len = 0;
@@ -827,8 +829,8 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg,
 			if (get_user(namelen, &cnm->cn_len))
 				return -EFAULT;
 			name.data = memdup_user(&cnm->cn_id, namelen);
-			if (IS_ERR_OR_NULL(name.data))
-				return -EFAULT;
+			if (IS_ERR(name.data))
+				return PTR_ERR(name.data);
 			name.len = namelen;
 		}
 		if (name.len > 5 && memcmp(name.data, "hash:", 5) == 0) {
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index c5d199d7e6b4..198d7abf34e4 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -160,6 +160,13 @@ static bool is_client_expired(struct nfs4_client *clp)
 	return clp->cl_time == 0;
 }
 
+static void nfsd4_dec_courtesy_client_count(struct nfsd_net *nn,
+					struct nfs4_client *clp)
+{
+	if (clp->cl_state != NFSD4_ACTIVE)
+		atomic_add_unless(&nn->nfsd_courtesy_clients, -1, 0);
+}
+
 static __be32 get_client_locked(struct nfs4_client *clp)
 {
 	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
@@ -169,6 +176,7 @@ static __be32 get_client_locked(struct nfs4_client *clp)
 	if (is_client_expired(clp))
 		return nfserr_expired;
 	atomic_inc(&clp->cl_rpc_users);
+	nfsd4_dec_courtesy_client_count(nn, clp);
 	clp->cl_state = NFSD4_ACTIVE;
 	return nfs_ok;
 }
@@ -190,6 +198,7 @@ renew_client_locked(struct nfs4_client *clp)
 
 	list_move_tail(&clp->cl_lru, &nn->client_lru);
 	clp->cl_time = ktime_get_boottime_seconds();
+	nfsd4_dec_courtesy_client_count(nn, clp);
 	clp->cl_state = NFSD4_ACTIVE;
 }
 
@@ -357,6 +366,8 @@ nfsd4_cb_notify_lock_prepare(struct nfsd4_callback *cb)
 static int
 nfsd4_cb_notify_lock_done(struct nfsd4_callback *cb, struct rpc_task *task)
 {
+	trace_nfsd_cb_notify_lock_done(&zero_stateid, task);
+
 	/*
 	 * Since this is just an optimization, we don't try very hard if it
 	 * turns out not to succeed. We'll requeue it on NFS4ERR_DELAY, and
@@ -963,19 +974,19 @@ out_free:
  * Create a unique stateid_t to represent each COPY.
  */
 static int nfs4_init_cp_state(struct nfsd_net *nn, copy_stateid_t *stid,
-			      unsigned char sc_type)
+			      unsigned char cs_type)
 {
 	int new_id;
 
-	stid->stid.si_opaque.so_clid.cl_boot = (u32)nn->boot_time;
-	stid->stid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
-	stid->sc_type = sc_type;
+	stid->cs_stid.si_opaque.so_clid.cl_boot = (u32)nn->boot_time;
+	stid->cs_stid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
+	stid->cs_type = cs_type;
 
 	idr_preload(GFP_KERNEL);
 	spin_lock(&nn->s2s_cp_lock);
 	new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, stid, 0, 0, GFP_NOWAIT);
-	stid->stid.si_opaque.so_id = new_id;
-	stid->stid.si_generation = 1;
+	stid->cs_stid.si_opaque.so_id = new_id;
+	stid->cs_stid.si_generation = 1;
 	spin_unlock(&nn->s2s_cp_lock);
 	idr_preload_end();
 	if (new_id < 0)
@@ -997,7 +1008,7 @@ struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn,
 	if (!cps)
 		return NULL;
 	cps->cpntf_time = ktime_get_boottime_seconds();
-	refcount_set(&cps->cp_stateid.sc_count, 1);
+	refcount_set(&cps->cp_stateid.cs_count, 1);
 	if (!nfs4_init_cp_state(nn, &cps->cp_stateid, NFS4_COPYNOTIFY_STID))
 		goto out_free;
 	spin_lock(&nn->s2s_cp_lock);
@@ -1013,11 +1024,11 @@ void nfs4_free_copy_state(struct nfsd4_copy *copy)
 {
 	struct nfsd_net *nn;
 
-	WARN_ON_ONCE(copy->cp_stateid.sc_type != NFS4_COPY_STID);
+	WARN_ON_ONCE(copy->cp_stateid.cs_type != NFS4_COPY_STID);
 	nn = net_generic(copy->cp_clp->net, nfsd_net_id);
 	spin_lock(&nn->s2s_cp_lock);
 	idr_remove(&nn->s2s_cp_stateids,
-		   copy->cp_stateid.stid.si_opaque.so_id);
+		   copy->cp_stateid.cs_stid.si_opaque.so_id);
 	spin_unlock(&nn->s2s_cp_lock);
 }
 
@@ -1049,6 +1060,12 @@ static struct nfs4_ol_stateid * nfs4_alloc_open_stateid(struct nfs4_client *clp)
 
 static void nfs4_free_deleg(struct nfs4_stid *stid)
 {
+	struct nfs4_delegation *dp = delegstateid(stid);
+
+	WARN_ON_ONCE(!list_empty(&stid->sc_cp_list));
+	WARN_ON_ONCE(!list_empty(&dp->dl_perfile));
+	WARN_ON_ONCE(!list_empty(&dp->dl_perclnt));
+	WARN_ON_ONCE(!list_empty(&dp->dl_recall_lru));
 	kmem_cache_free(deleg_slab, stid);
 	atomic_long_dec(&num_delegations);
 }
@@ -1462,6 +1479,7 @@ static void nfs4_free_ol_stateid(struct nfs4_stid *stid)
 	release_all_access(stp);
 	if (stp->st_stateowner)
 		nfs4_put_stateowner(stp->st_stateowner);
+	WARN_ON(!list_empty(&stid->sc_cp_list));
 	kmem_cache_free(stateid_slab, stid);
 }
 
@@ -2233,6 +2251,7 @@ __destroy_client(struct nfs4_client *clp)
 	if (clp->cl_cb_conn.cb_xprt)
 		svc_xprt_put(clp->cl_cb_conn.cb_xprt);
 	atomic_add_unless(&nn->nfs4_client_count, -1, 0);
+	nfsd4_dec_courtesy_client_count(nn, clp);
 	free_client(clp);
 	wake_up_all(&expiry_wq);
 }
@@ -2478,7 +2497,7 @@ static const char *cb_state2str(int state)
 
 static int client_info_show(struct seq_file *m, void *v)
 {
-	struct inode *inode = m->private;
+	struct inode *inode = file_inode(m->file);
 	struct nfs4_client *clp;
 	u64 clid;
 
@@ -2518,17 +2537,7 @@ static int client_info_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int client_info_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, client_info_show, inode);
-}
-
-static const struct file_operations client_info_fops = {
-	.open		= client_info_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(client_info);
 
 static void *states_start(struct seq_file *s, loff_t *pos)
 	__acquires(&clp->cl_lock)
@@ -4337,7 +4346,27 @@ out:
 	return -ENOMEM;
 }
 
-void nfsd4_init_leases_net(struct nfsd_net *nn)
+static unsigned long
+nfsd_courtesy_client_count(struct shrinker *shrink, struct shrink_control *sc)
+{
+	int cnt;
+	struct nfsd_net *nn = container_of(shrink,
+			struct nfsd_net, nfsd_client_shrinker);
+
+	cnt = atomic_read(&nn->nfsd_courtesy_clients);
+	if (cnt > 0)
+		mod_delayed_work(laundry_wq, &nn->nfsd_shrinker_work, 0);
+	return (unsigned long)cnt;
+}
+
+static unsigned long
+nfsd_courtesy_client_scan(struct shrinker *shrink, struct shrink_control *sc)
+{
+	return SHRINK_STOP;
+}
+
+int
+nfsd4_init_leases_net(struct nfsd_net *nn)
 {
 	struct sysinfo si;
 	u64 max_clients;
@@ -4356,6 +4385,18 @@ void nfsd4_init_leases_net(struct nfsd_net *nn)
 	max_clients = (u64)si.totalram * si.mem_unit / (1024 * 1024 * 1024);
 	max_clients *= NFS4_CLIENTS_PER_GB;
 	nn->nfs4_max_clients = max_t(int, max_clients, NFS4_CLIENTS_PER_GB);
+
+	atomic_set(&nn->nfsd_courtesy_clients, 0);
+	nn->nfsd_client_shrinker.scan_objects = nfsd_courtesy_client_scan;
+	nn->nfsd_client_shrinker.count_objects = nfsd_courtesy_client_count;
+	nn->nfsd_client_shrinker.seeks = DEFAULT_SEEKS;
+	return register_shrinker(&nn->nfsd_client_shrinker, "nfsd-client");
+}
+
+void
+nfsd4_leases_net_shutdown(struct nfsd_net *nn)
+{
+	unregister_shrinker(&nn->nfsd_client_shrinker);
 }
 
 static void init_nfs4_replay(struct nfs4_replay *rp)
@@ -4715,6 +4756,35 @@ nfs4_share_conflict(struct svc_fh *current_fh, unsigned int deny_type)
 	return ret;
 }
 
+static bool nfsd4_deleg_present(const struct inode *inode)
+{
+	struct file_lock_context *ctx = smp_load_acquire(&inode->i_flctx);
+
+	return ctx && !list_empty_careful(&ctx->flc_lease);
+}
+
+/**
+ * nfsd_wait_for_delegreturn - wait for delegations to be returned
+ * @rqstp: the RPC transaction being executed
+ * @inode: in-core inode of the file being waited for
+ *
+ * The timeout prevents deadlock if all nfsd threads happen to be
+ * tied up waiting for returning delegations.
+ *
+ * Return values:
+ *   %true: delegation was returned
+ *   %false: timed out waiting for delegreturn
+ */
+bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode)
+{
+	long __maybe_unused timeo;
+
+	timeo = wait_var_event_timeout(inode, !nfsd4_deleg_present(inode),
+				       NFSD_DELEGRETURN_TIMEOUT);
+	trace_nfsd_delegret_wakeup(rqstp, inode, timeo);
+	return timeo > 0;
+}
+
 static void nfsd4_cb_recall_prepare(struct nfsd4_callback *cb)
 {
 	struct nfs4_delegation *dp = cb_to_delegation(cb);
@@ -4743,6 +4813,8 @@ static int nfsd4_cb_recall_done(struct nfsd4_callback *cb,
 {
 	struct nfs4_delegation *dp = cb_to_delegation(cb);
 
+	trace_nfsd_cb_recall_done(&dp->dl_stid.sc_stateid, task);
+
 	if (dp->dl_stid.sc_type == NFS4_CLOSED_DELEG_STID ||
 	    dp->dl_stid.sc_type == NFS4_REVOKED_DELEG_STID)
 	        return 1;
@@ -4788,18 +4860,17 @@ static void nfsd_break_one_deleg(struct nfs4_delegation *dp)
 	 * We're assuming the state code never drops its reference
 	 * without first removing the lease.  Since we're in this lease
 	 * callback (and since the lease code is serialized by the
-	 * i_lock) we know the server hasn't removed the lease yet, and
+	 * flc_lock) we know the server hasn't removed the lease yet, and
 	 * we know it's safe to take a reference.
 	 */
 	refcount_inc(&dp->dl_stid.sc_count);
-	nfsd4_run_cb(&dp->dl_recall);
+	WARN_ON_ONCE(!nfsd4_run_cb(&dp->dl_recall));
 }
 
-/* Called from break_lease() with i_lock held. */
+/* Called from break_lease() with flc_lock held. */
 static bool
 nfsd_break_deleg_cb(struct file_lock *fl)
 {
-	bool ret = false;
 	struct nfs4_delegation *dp = (struct nfs4_delegation *)fl->fl_owner;
 	struct nfs4_file *fp = dp->dl_stid.sc_file;
 	struct nfs4_client *clp = dp->dl_stid.sc_client;
@@ -4825,7 +4896,7 @@ nfsd_break_deleg_cb(struct file_lock *fl)
 	fp->fi_had_conflict = true;
 	nfsd_break_one_deleg(dp);
 	spin_unlock(&fp->fi_lock);
-	return ret;
+	return false;
 }
 
 /**
@@ -5878,8 +5949,11 @@ nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
 			goto exp_client;
 		if (!state_expired(lt, clp->cl_time))
 			break;
-		if (!atomic_read(&clp->cl_rpc_users))
+		if (!atomic_read(&clp->cl_rpc_users)) {
+			if (clp->cl_state == NFSD4_ACTIVE)
+				atomic_inc(&nn->nfsd_courtesy_clients);
 			clp->cl_state = NFSD4_COURTESY;
+		}
 		if (!client_has_state(clp))
 			goto exp_client;
 		if (!nfs4_anylock_blockers(clp))
@@ -5894,10 +5968,49 @@ exp_client:
 	spin_unlock(&nn->client_lock);
 }
 
+static void
+nfs4_get_courtesy_client_reaplist(struct nfsd_net *nn,
+				struct list_head *reaplist)
+{
+	unsigned int maxreap = 0, reapcnt = 0;
+	struct list_head *pos, *next;
+	struct nfs4_client *clp;
+
+	maxreap = NFSD_CLIENT_MAX_TRIM_PER_RUN;
+	INIT_LIST_HEAD(reaplist);
+
+	spin_lock(&nn->client_lock);
+	list_for_each_safe(pos, next, &nn->client_lru) {
+		clp = list_entry(pos, struct nfs4_client, cl_lru);
+		if (clp->cl_state == NFSD4_ACTIVE)
+			break;
+		if (reapcnt >= maxreap)
+			break;
+		if (!mark_client_expired_locked(clp)) {
+			list_add(&clp->cl_lru, reaplist);
+			reapcnt++;
+		}
+	}
+	spin_unlock(&nn->client_lock);
+}
+
+static void
+nfs4_process_client_reaplist(struct list_head *reaplist)
+{
+	struct list_head *pos, *next;
+	struct nfs4_client *clp;
+
+	list_for_each_safe(pos, next, reaplist) {
+		clp = list_entry(pos, struct nfs4_client, cl_lru);
+		trace_nfsd_clid_purged(&clp->cl_clientid);
+		list_del_init(&clp->cl_lru);
+		expire_client(clp);
+	}
+}
+
 static time64_t
 nfs4_laundromat(struct nfsd_net *nn)
 {
-	struct nfs4_client *clp;
 	struct nfs4_openowner *oo;
 	struct nfs4_delegation *dp;
 	struct nfs4_ol_stateid *stp;
@@ -5920,18 +6033,14 @@ nfs4_laundromat(struct nfsd_net *nn)
 	spin_lock(&nn->s2s_cp_lock);
 	idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) {
 		cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid);
-		if (cps->cp_stateid.sc_type == NFS4_COPYNOTIFY_STID &&
+		if (cps->cp_stateid.cs_type == NFS4_COPYNOTIFY_STID &&
 				state_expired(&lt, cps->cpntf_time))
 			_free_cpntf_state_locked(nn, cps);
 	}
 	spin_unlock(&nn->s2s_cp_lock);
 	nfs4_get_client_reaplist(nn, &reaplist, &lt);
-	list_for_each_safe(pos, next, &reaplist) {
-		clp = list_entry(pos, struct nfs4_client, cl_lru);
-		trace_nfsd_clid_purged(&clp->cl_clientid);
-		list_del_init(&clp->cl_lru);
-		expire_client(clp);
-	}
+	nfs4_process_client_reaplist(&reaplist);
+
 	spin_lock(&state_lock);
 	list_for_each_safe(pos, next, &nn->del_recall_lru) {
 		dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
@@ -6014,6 +6123,18 @@ laundromat_main(struct work_struct *laundry)
 	queue_delayed_work(laundry_wq, &nn->laundromat_work, t*HZ);
 }
 
+static void
+courtesy_client_reaper(struct work_struct *reaper)
+{
+	struct list_head reaplist;
+	struct delayed_work *dwork = to_delayed_work(reaper);
+	struct nfsd_net *nn = container_of(dwork, struct nfsd_net,
+					nfsd_shrinker_work);
+
+	nfs4_get_courtesy_client_reaplist(nn, &reaplist);
+	nfs4_process_client_reaplist(&reaplist);
+}
+
 static inline __be32 nfs4_check_fh(struct svc_fh *fhp, struct nfs4_stid *stp)
 {
 	if (!fh_match(&fhp->fh_handle, &stp->sc_file->fi_fhandle))
@@ -6149,6 +6270,7 @@ nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
 		     struct nfs4_stid **s, struct nfsd_net *nn)
 {
 	__be32 status;
+	struct nfs4_stid *stid;
 	bool return_revoked = false;
 
 	/*
@@ -6171,15 +6293,16 @@ nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
 	}
 	if (status)
 		return status;
-	*s = find_stateid_by_type(cstate->clp, stateid, typemask);
-	if (!*s)
+	stid = find_stateid_by_type(cstate->clp, stateid, typemask);
+	if (!stid)
 		return nfserr_bad_stateid;
-	if (((*s)->sc_type == NFS4_REVOKED_DELEG_STID) && !return_revoked) {
-		nfs4_put_stid(*s);
+	if ((stid->sc_type == NFS4_REVOKED_DELEG_STID) && !return_revoked) {
+		nfs4_put_stid(stid);
 		if (cstate->minorversion)
 			return nfserr_deleg_revoked;
 		return nfserr_bad_stateid;
 	}
+	*s = stid;
 	return nfs_ok;
 }
 
@@ -6244,12 +6367,12 @@ out:
 static void
 _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
 {
-	WARN_ON_ONCE(cps->cp_stateid.sc_type != NFS4_COPYNOTIFY_STID);
-	if (!refcount_dec_and_test(&cps->cp_stateid.sc_count))
+	WARN_ON_ONCE(cps->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID);
+	if (!refcount_dec_and_test(&cps->cp_stateid.cs_count))
 		return;
 	list_del(&cps->cp_list);
 	idr_remove(&nn->s2s_cp_stateids,
-		   cps->cp_stateid.stid.si_opaque.so_id);
+		   cps->cp_stateid.cs_stid.si_opaque.so_id);
 	kfree(cps);
 }
 /*
@@ -6271,12 +6394,12 @@ __be32 manage_cpntf_state(struct nfsd_net *nn, stateid_t *st,
 	if (cps_t) {
 		state = container_of(cps_t, struct nfs4_cpntf_state,
 				     cp_stateid);
-		if (state->cp_stateid.sc_type != NFS4_COPYNOTIFY_STID) {
+		if (state->cp_stateid.cs_type != NFS4_COPYNOTIFY_STID) {
 			state = NULL;
 			goto unlock;
 		}
 		if (!clp)
-			refcount_inc(&state->cp_stateid.sc_count);
+			refcount_inc(&state->cp_stateid.cs_count);
 		else
 			_free_cpntf_state_locked(nn, state);
 	}
@@ -6684,6 +6807,7 @@ static void nfsd4_close_open_stateid(struct nfs4_ol_stateid *s)
 	struct nfs4_client *clp = s->st_stid.sc_client;
 	bool unhashed;
 	LIST_HEAD(reaplist);
+	struct nfs4_ol_stateid *stp;
 
 	spin_lock(&clp->cl_lock);
 	unhashed = unhash_open_stateid(s, &reaplist);
@@ -6692,6 +6816,8 @@ static void nfsd4_close_open_stateid(struct nfs4_ol_stateid *s)
 		if (unhashed)
 			put_ol_stateid_locked(s, &reaplist);
 		spin_unlock(&clp->cl_lock);
+		list_for_each_entry(stp, &reaplist, st_locks)
+			nfs4_free_cpntf_statelist(clp->net, &stp->st_stid);
 		free_ol_stateid_reaplist(&reaplist);
 	} else {
 		spin_unlock(&clp->cl_lock);
@@ -6775,6 +6901,7 @@ nfsd4_delegreturn(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 	if (status)
 		goto put_stateid;
 
+	wake_up_var(d_inode(cstate->current_fh.fh_dentry));
 	destroy_delegation(dp);
 put_stateid:
 	nfs4_put_stid(&dp->dl_stid);
@@ -7830,6 +7957,7 @@ static int nfs4_state_create_net(struct net *net)
 	INIT_LIST_HEAD(&nn->blocked_locks_lru);
 
 	INIT_DELAYED_WORK(&nn->laundromat_work, laundromat_main);
+	INIT_DELAYED_WORK(&nn->nfsd_shrinker_work, courtesy_client_reaper);
 	get_net(net);
 
 	return 0;
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 1e9690a061ec..bcfeb1a922c0 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -42,6 +42,8 @@
 #include <linux/sunrpc/svcauth_gss.h>
 #include <linux/sunrpc/addr.h>
 #include <linux/xattr.h>
+#include <linux/vmalloc.h>
+
 #include <uapi/linux/xattr.h>
 
 #include "idmap.h"
@@ -791,6 +793,7 @@ nfsd4_decode_commit(struct nfsd4_compoundargs *argp, struct nfsd4_commit *commit
 		return nfserr_bad_xdr;
 	if (xdr_stream_decode_u32(argp->xdr, &commit->co_count) < 0)
 		return nfserr_bad_xdr;
+	memset(&commit->co_verf, 0, sizeof(commit->co_verf));
 	return nfs_ok;
 }
 
@@ -799,6 +802,7 @@ nfsd4_decode_create(struct nfsd4_compoundargs *argp, struct nfsd4_create *create
 {
 	__be32 *p, status;
 
+	memset(create, 0, sizeof(*create));
 	if (xdr_stream_decode_u32(argp->xdr, &create->cr_type) < 0)
 		return nfserr_bad_xdr;
 	switch (create->cr_type) {
@@ -848,6 +852,7 @@ nfsd4_decode_delegreturn(struct nfsd4_compoundargs *argp, struct nfsd4_delegretu
 static inline __be32
 nfsd4_decode_getattr(struct nfsd4_compoundargs *argp, struct nfsd4_getattr *getattr)
 {
+	memset(getattr, 0, sizeof(*getattr));
 	return nfsd4_decode_bitmap4(argp, getattr->ga_bmval,
 				    ARRAY_SIZE(getattr->ga_bmval));
 }
@@ -855,6 +860,7 @@ nfsd4_decode_getattr(struct nfsd4_compoundargs *argp, struct nfsd4_getattr *geta
 static __be32
 nfsd4_decode_link(struct nfsd4_compoundargs *argp, struct nfsd4_link *link)
 {
+	memset(link, 0, sizeof(*link));
 	return nfsd4_decode_component4(argp, &link->li_name, &link->li_namelen);
 }
 
@@ -903,6 +909,7 @@ nfsd4_decode_locker4(struct nfsd4_compoundargs *argp, struct nfsd4_lock *lock)
 static __be32
 nfsd4_decode_lock(struct nfsd4_compoundargs *argp, struct nfsd4_lock *lock)
 {
+	memset(lock, 0, sizeof(*lock));
 	if (xdr_stream_decode_u32(argp->xdr, &lock->lk_type) < 0)
 		return nfserr_bad_xdr;
 	if ((lock->lk_type < NFS4_READ_LT) || (lock->lk_type > NFS4_WRITEW_LT))
@@ -919,6 +926,7 @@ nfsd4_decode_lock(struct nfsd4_compoundargs *argp, struct nfsd4_lock *lock)
 static __be32
 nfsd4_decode_lockt(struct nfsd4_compoundargs *argp, struct nfsd4_lockt *lockt)
 {
+	memset(lockt, 0, sizeof(*lockt));
 	if (xdr_stream_decode_u32(argp->xdr, &lockt->lt_type) < 0)
 		return nfserr_bad_xdr;
 	if ((lockt->lt_type < NFS4_READ_LT) || (lockt->lt_type > NFS4_WRITEW_LT))
@@ -1140,11 +1148,8 @@ nfsd4_decode_open(struct nfsd4_compoundargs *argp, struct nfsd4_open *open)
 	__be32 status;
 	u32 dummy;
 
-	memset(open->op_bmval, 0, sizeof(open->op_bmval));
-	open->op_iattr.ia_valid = 0;
-	open->op_openowner = NULL;
+	memset(open, 0, sizeof(*open));
 
-	open->op_xdr_error = 0;
 	if (xdr_stream_decode_u32(argp->xdr, &open->op_seqid) < 0)
 		return nfserr_bad_xdr;
 	/* deleg_want is ignored */
@@ -1179,6 +1184,8 @@ nfsd4_decode_open_confirm(struct nfsd4_compoundargs *argp, struct nfsd4_open_con
 	if (xdr_stream_decode_u32(argp->xdr, &open_conf->oc_seqid) < 0)
 		return nfserr_bad_xdr;
 
+	memset(&open_conf->oc_resp_stateid, 0,
+	       sizeof(open_conf->oc_resp_stateid));
 	return nfs_ok;
 }
 
@@ -1187,6 +1194,7 @@ nfsd4_decode_open_downgrade(struct nfsd4_compoundargs *argp, struct nfsd4_open_d
 {
 	__be32 status;
 
+	memset(open_down, 0, sizeof(*open_down));
 	status = nfsd4_decode_stateid4(argp, &open_down->od_stateid);
 	if (status)
 		return status;
@@ -1216,6 +1224,7 @@ nfsd4_decode_putfh(struct nfsd4_compoundargs *argp, struct nfsd4_putfh *putfh)
 	if (!putfh->pf_fhval)
 		return nfserr_jukebox;
 
+	putfh->no_verify = false;
 	return nfs_ok;
 }
 
@@ -1232,6 +1241,7 @@ nfsd4_decode_read(struct nfsd4_compoundargs *argp, struct nfsd4_read *read)
 {
 	__be32 status;
 
+	memset(read, 0, sizeof(*read));
 	status = nfsd4_decode_stateid4(argp, &read->rd_stateid);
 	if (status)
 		return status;
@@ -1248,6 +1258,7 @@ nfsd4_decode_readdir(struct nfsd4_compoundargs *argp, struct nfsd4_readdir *read
 {
 	__be32 status;
 
+	memset(readdir, 0, sizeof(*readdir));
 	if (xdr_stream_decode_u64(argp->xdr, &readdir->rd_cookie) < 0)
 		return nfserr_bad_xdr;
 	status = nfsd4_decode_verifier4(argp, &readdir->rd_verf);
@@ -1267,6 +1278,7 @@ nfsd4_decode_readdir(struct nfsd4_compoundargs *argp, struct nfsd4_readdir *read
 static __be32
 nfsd4_decode_remove(struct nfsd4_compoundargs *argp, struct nfsd4_remove *remove)
 {
+	memset(&remove->rm_cinfo, 0, sizeof(remove->rm_cinfo));
 	return nfsd4_decode_component4(argp, &remove->rm_name, &remove->rm_namelen);
 }
 
@@ -1275,6 +1287,7 @@ nfsd4_decode_rename(struct nfsd4_compoundargs *argp, struct nfsd4_rename *rename
 {
 	__be32 status;
 
+	memset(rename, 0, sizeof(*rename));
 	status = nfsd4_decode_component4(argp, &rename->rn_sname, &rename->rn_snamelen);
 	if (status)
 		return status;
@@ -1291,6 +1304,7 @@ static __be32
 nfsd4_decode_secinfo(struct nfsd4_compoundargs *argp,
 		     struct nfsd4_secinfo *secinfo)
 {
+	secinfo->si_exp = NULL;
 	return nfsd4_decode_component4(argp, &secinfo->si_name, &secinfo->si_namelen);
 }
 
@@ -1299,6 +1313,7 @@ nfsd4_decode_setattr(struct nfsd4_compoundargs *argp, struct nfsd4_setattr *seta
 {
 	__be32 status;
 
+	memset(setattr, 0, sizeof(*setattr));
 	status = nfsd4_decode_stateid4(argp, &setattr->sa_stateid);
 	if (status)
 		return status;
@@ -1313,6 +1328,8 @@ nfsd4_decode_setclientid(struct nfsd4_compoundargs *argp, struct nfsd4_setclient
 {
 	__be32 *p, status;
 
+	memset(setclientid, 0, sizeof(*setclientid));
+
 	if (argp->minorversion >= 1)
 		return nfserr_notsupp;
 
@@ -1369,6 +1386,8 @@ nfsd4_decode_verify(struct nfsd4_compoundargs *argp, struct nfsd4_verify *verify
 {
 	__be32 *p, status;
 
+	memset(verify, 0, sizeof(*verify));
+
 	status = nfsd4_decode_bitmap4(argp, verify->ve_bmval,
 				      ARRAY_SIZE(verify->ve_bmval));
 	if (status)
@@ -1408,6 +1427,9 @@ nfsd4_decode_write(struct nfsd4_compoundargs *argp, struct nfsd4_write *write)
 	if (!xdr_stream_subsegment(argp->xdr, &write->wr_payload, write->wr_buflen))
 		return nfserr_bad_xdr;
 
+	write->wr_bytes_written = 0;
+	write->wr_how_written = 0;
+	memset(&write->wr_verifier, 0, sizeof(write->wr_verifier));
 	return nfs_ok;
 }
 
@@ -1432,6 +1454,7 @@ nfsd4_decode_release_lockowner(struct nfsd4_compoundargs *argp, struct nfsd4_rel
 
 static __be32 nfsd4_decode_backchannel_ctl(struct nfsd4_compoundargs *argp, struct nfsd4_backchannel_ctl *bc)
 {
+	memset(bc, 0, sizeof(*bc));
 	if (xdr_stream_decode_u32(argp->xdr, &bc->bc_cb_program) < 0)
 		return nfserr_bad_xdr;
 	return nfsd4_decode_cb_sec(argp, &bc->bc_cb_sec);
@@ -1442,6 +1465,7 @@ static __be32 nfsd4_decode_bind_conn_to_session(struct nfsd4_compoundargs *argp,
 	u32 use_conn_in_rdma_mode;
 	__be32 status;
 
+	memset(bcts, 0, sizeof(*bcts));
 	status = nfsd4_decode_sessionid4(argp, &bcts->sessionid);
 	if (status)
 		return status;
@@ -1583,6 +1607,7 @@ nfsd4_decode_exchange_id(struct nfsd4_compoundargs *argp,
 {
 	__be32 status;
 
+	memset(exid, 0, sizeof(*exid));
 	status = nfsd4_decode_verifier4(argp, &exid->verifier);
 	if (status)
 		return status;
@@ -1635,6 +1660,7 @@ nfsd4_decode_create_session(struct nfsd4_compoundargs *argp,
 {
 	__be32 status;
 
+	memset(sess, 0, sizeof(*sess));
 	status = nfsd4_decode_clientid4(argp, &sess->clientid);
 	if (status)
 		return status;
@@ -1650,11 +1676,7 @@ nfsd4_decode_create_session(struct nfsd4_compoundargs *argp,
 		return status;
 	if (xdr_stream_decode_u32(argp->xdr, &sess->callback_prog) < 0)
 		return nfserr_bad_xdr;
-	status = nfsd4_decode_cb_sec(argp, &sess->cb_sec);
-	if (status)
-		return status;
-
-	return nfs_ok;
+	return nfsd4_decode_cb_sec(argp, &sess->cb_sec);
 }
 
 static __be32
@@ -1678,6 +1700,7 @@ nfsd4_decode_getdeviceinfo(struct nfsd4_compoundargs *argp,
 {
 	__be32 status;
 
+	memset(gdev, 0, sizeof(*gdev));
 	status = nfsd4_decode_deviceid4(argp, &gdev->gd_devid);
 	if (status)
 		return status;
@@ -1698,6 +1721,7 @@ nfsd4_decode_layoutcommit(struct nfsd4_compoundargs *argp,
 {
 	__be32 *p, status;
 
+	memset(lcp, 0, sizeof(*lcp));
 	if (xdr_stream_decode_u64(argp->xdr, &lcp->lc_seg.offset) < 0)
 		return nfserr_bad_xdr;
 	if (xdr_stream_decode_u64(argp->xdr, &lcp->lc_seg.length) < 0)
@@ -1733,6 +1757,7 @@ nfsd4_decode_layoutget(struct nfsd4_compoundargs *argp,
 {
 	__be32 status;
 
+	memset(lgp, 0, sizeof(*lgp));
 	if (xdr_stream_decode_u32(argp->xdr, &lgp->lg_signal) < 0)
 		return nfserr_bad_xdr;
 	if (xdr_stream_decode_u32(argp->xdr, &lgp->lg_layout_type) < 0)
@@ -1758,6 +1783,7 @@ static __be32
 nfsd4_decode_layoutreturn(struct nfsd4_compoundargs *argp,
 		struct nfsd4_layoutreturn *lrp)
 {
+	memset(lrp, 0, sizeof(*lrp));
 	if (xdr_stream_decode_bool(argp->xdr, &lrp->lr_reclaim) < 0)
 		return nfserr_bad_xdr;
 	if (xdr_stream_decode_u32(argp->xdr, &lrp->lr_layout_type) < 0)
@@ -1773,6 +1799,8 @@ static __be32 nfsd4_decode_secinfo_no_name(struct nfsd4_compoundargs *argp,
 {
 	if (xdr_stream_decode_u32(argp->xdr, &sin->sin_style) < 0)
 		return nfserr_bad_xdr;
+
+	sin->sin_exp = NULL;
 	return nfs_ok;
 }
 
@@ -1793,6 +1821,7 @@ nfsd4_decode_sequence(struct nfsd4_compoundargs *argp,
 	seq->maxslots = be32_to_cpup(p++);
 	seq->cachethis = be32_to_cpup(p);
 
+	seq->status_flags = 0;
 	return nfs_ok;
 }
 
@@ -1803,6 +1832,7 @@ nfsd4_decode_test_stateid(struct nfsd4_compoundargs *argp, struct nfsd4_test_sta
 	__be32 status;
 	u32 i;
 
+	memset(test_stateid, 0, sizeof(*test_stateid));
 	if (xdr_stream_decode_u32(argp->xdr, &test_stateid->ts_num_ids) < 0)
 		return nfserr_bad_xdr;
 
@@ -1900,6 +1930,7 @@ nfsd4_decode_copy(struct nfsd4_compoundargs *argp, struct nfsd4_copy *copy)
 	struct nl4_server *ns_dummy;
 	__be32 status;
 
+	memset(copy, 0, sizeof(*copy));
 	status = nfsd4_decode_stateid4(argp, &copy->cp_src_stateid);
 	if (status)
 		return status;
@@ -1955,6 +1986,7 @@ nfsd4_decode_copy_notify(struct nfsd4_compoundargs *argp,
 {
 	__be32 status;
 
+	memset(cn, 0, sizeof(*cn));
 	cn->cpn_src = svcxdr_tmpalloc(argp, sizeof(*cn->cpn_src));
 	if (cn->cpn_src == NULL)
 		return nfserr_jukebox;
@@ -1972,6 +2004,8 @@ static __be32
 nfsd4_decode_offload_status(struct nfsd4_compoundargs *argp,
 			    struct nfsd4_offload_status *os)
 {
+	os->count = 0;
+	os->status = 0;
 	return nfsd4_decode_stateid4(argp, &os->stateid);
 }
 
@@ -1988,6 +2022,8 @@ nfsd4_decode_seek(struct nfsd4_compoundargs *argp, struct nfsd4_seek *seek)
 	if (xdr_stream_decode_u32(argp->xdr, &seek->seek_whence) < 0)
 		return nfserr_bad_xdr;
 
+	seek->seek_eof = 0;
+	seek->seek_pos = 0;
 	return nfs_ok;
 }
 
@@ -2123,6 +2159,7 @@ nfsd4_decode_getxattr(struct nfsd4_compoundargs *argp,
 	__be32 status;
 	u32 maxcount;
 
+	memset(getxattr, 0, sizeof(*getxattr));
 	status = nfsd4_decode_xattr_name(argp, &getxattr->getxa_name);
 	if (status)
 		return status;
@@ -2131,8 +2168,7 @@ nfsd4_decode_getxattr(struct nfsd4_compoundargs *argp,
 	maxcount = min_t(u32, XATTR_SIZE_MAX, maxcount);
 
 	getxattr->getxa_len = maxcount;
-
-	return status;
+	return nfs_ok;
 }
 
 static __be32
@@ -2142,6 +2178,8 @@ nfsd4_decode_setxattr(struct nfsd4_compoundargs *argp,
 	u32 flags, maxcount, size;
 	__be32 status;
 
+	memset(setxattr, 0, sizeof(*setxattr));
+
 	if (xdr_stream_decode_u32(argp->xdr, &flags) < 0)
 		return nfserr_bad_xdr;
 
@@ -2180,6 +2218,8 @@ nfsd4_decode_listxattrs(struct nfsd4_compoundargs *argp,
 {
 	u32 maxcount;
 
+	memset(listxattrs, 0, sizeof(*listxattrs));
+
 	if (xdr_stream_decode_u64(argp->xdr, &listxattrs->lsxa_cookie) < 0)
 		return nfserr_bad_xdr;
 
@@ -2207,6 +2247,7 @@ static __be32
 nfsd4_decode_removexattr(struct nfsd4_compoundargs *argp,
 			 struct nfsd4_removexattr *removexattr)
 {
+	memset(removexattr, 0, sizeof(*removexattr));
 	return nfsd4_decode_xattr_name(argp, &removexattr->rmxa_name);
 }
 
@@ -2357,22 +2398,15 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp)
 
 	if (xdr_stream_decode_u32(argp->xdr, &argp->minorversion) < 0)
 		return false;
-	if (xdr_stream_decode_u32(argp->xdr, &argp->opcnt) < 0)
+	if (xdr_stream_decode_u32(argp->xdr, &argp->client_opcnt) < 0)
 		return false;
-
-	/*
-	 * NFS4ERR_RESOURCE is a more helpful error than GARBAGE_ARGS
-	 * here, so we return success at the xdr level so that
-	 * nfsd4_proc can handle this is an NFS-level error.
-	 */
-	if (argp->opcnt > NFSD_MAX_OPS_PER_COMPOUND)
-		return true;
+	argp->opcnt = min_t(u32, argp->client_opcnt,
+			    NFSD_MAX_OPS_PER_COMPOUND);
 
 	if (argp->opcnt > ARRAY_SIZE(argp->iops)) {
-		argp->ops = kzalloc(argp->opcnt * sizeof(*argp->ops), GFP_KERNEL);
+		argp->ops = vcalloc(argp->opcnt, sizeof(*argp->ops));
 		if (!argp->ops) {
 			argp->ops = argp->iops;
-			dprintk("nfsd: couldn't allocate room for COMPOUND\n");
 			return false;
 		}
 	}
@@ -2774,9 +2808,10 @@ static __be32 fattr_handle_absent_fs(u32 *bmval0, u32 *bmval1, u32 *bmval2, u32
 }
 
 
-static int get_parent_attributes(struct svc_export *exp, struct kstat *stat)
+static int nfsd4_get_mounted_on_ino(struct svc_export *exp, u64 *pino)
 {
 	struct path path = exp->ex_path;
+	struct kstat stat;
 	int err;
 
 	path_get(&path);
@@ -2784,8 +2819,10 @@ static int get_parent_attributes(struct svc_export *exp, struct kstat *stat)
 		if (path.dentry != path.mnt->mnt_root)
 			break;
 	}
-	err = vfs_getattr(&path, stat, STATX_BASIC_STATS, AT_STATX_SYNC_AS_STAT);
+	err = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT);
 	path_put(&path);
+	if (!err)
+		*pino = stat.ino;
 	return err;
 }
 
@@ -3282,22 +3319,21 @@ out_acl:
 		*p++ = cpu_to_be32(stat.btime.tv_nsec);
 	}
 	if (bmval1 & FATTR4_WORD1_MOUNTED_ON_FILEID) {
-		struct kstat parent_stat;
 		u64 ino = stat.ino;
 
 		p = xdr_reserve_space(xdr, 8);
 		if (!p)
                 	goto out_resource;
 		/*
-		 * Get parent's attributes if not ignoring crossmount
-		 * and this is the root of a cross-mounted filesystem.
+		 * Get ino of mountpoint in parent filesystem, if not ignoring
+		 * crossmount and this is the root of a cross-mounted
+		 * filesystem.
 		 */
 		if (ignore_crossmnt == 0 &&
 		    dentry == exp->ex_path.mnt->mnt_root) {
-			err = get_parent_attributes(exp, &parent_stat);
+			err = nfsd4_get_mounted_on_ino(exp, &ino);
 			if (err)
 				goto out_nfserr;
-			ino = parent_stat.ino;
 		}
 		p = xdr_encode_hyper(p, ino);
 	}
@@ -3994,7 +4030,7 @@ nfsd4_encode_read(struct nfsd4_compoundres *resp, __be32 nfserr,
 	}
 	if (resp->xdr->buf->page_len && splice_ok) {
 		WARN_ON_ONCE(1);
-		return nfserr_resource;
+		return nfserr_serverfault;
 	}
 	xdr_commit_encode(xdr);
 
@@ -5394,7 +5430,7 @@ void nfsd4_release_compoundargs(struct svc_rqst *rqstp)
 	struct nfsd4_compoundargs *args = rqstp->rq_argp;
 
 	if (args->ops != args->iops) {
-		kfree(args->ops);
+		vfree(args->ops);
 		args->ops = args->iops;
 	}
 	while (args->to_free) {
@@ -5423,12 +5459,8 @@ bool
 nfs4svc_encode_compoundres(struct svc_rqst *rqstp, struct xdr_stream *xdr)
 {
 	struct nfsd4_compoundres *resp = rqstp->rq_resp;
-	struct xdr_buf *buf = xdr->buf;
 	__be32 *p;
 
-	WARN_ON_ONCE(buf->len != buf->head[0].iov_len + buf->page_len +
-				 buf->tail[0].iov_len);
-
 	/*
 	 * Send buffer space for the following items is reserved
 	 * at the top of nfsd4_proc_compound().
diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c
index 9b31e1103e7b..3e64a3d50a1c 100644
--- a/fs/nfsd/nfscache.c
+++ b/fs/nfsd/nfscache.c
@@ -604,9 +604,10 @@ nfsd_cache_append(struct svc_rqst *rqstp, struct kvec *data)
  * scraping this file for info should test the labels to ensure they're
  * getting the correct field.
  */
-static int nfsd_reply_cache_stats_show(struct seq_file *m, void *v)
+int nfsd_reply_cache_stats_show(struct seq_file *m, void *v)
 {
-	struct nfsd_net *nn = m->private;
+	struct nfsd_net *nn = net_generic(file_inode(m->file)->i_sb->s_fs_info,
+					  nfsd_net_id);
 
 	seq_printf(m, "max entries:           %u\n", nn->max_drc_entries);
 	seq_printf(m, "num entries:           %u\n",
@@ -626,11 +627,3 @@ static int nfsd_reply_cache_stats_show(struct seq_file *m, void *v)
 	seq_printf(m, "cachesize at longest:  %u\n", nn->longest_chain_cachesize);
 	return 0;
 }
-
-int nfsd_reply_cache_stats_open(struct inode *inode, struct file *file)
-{
-	struct nfsd_net *nn = net_generic(file_inode(file)->i_sb->s_fs_info,
-								nfsd_net_id);
-
-	return single_open(file, nfsd_reply_cache_stats_show, nn);
-}
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 917fa1892fd2..6a29bcfc9390 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -185,17 +185,7 @@ static int export_features_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int export_features_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, export_features_show, NULL);
-}
-
-static const struct file_operations export_features_operations = {
-	.open		= export_features_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(export_features);
 
 #if defined(CONFIG_SUNRPC_GSS) || defined(CONFIG_SUNRPC_GSS_MODULE)
 static int supported_enctypes_show(struct seq_file *m, void *v)
@@ -204,17 +194,7 @@ static int supported_enctypes_show(struct seq_file *m, void *v)
 	return 0;
 }
 
-static int supported_enctypes_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, supported_enctypes_show, NULL);
-}
-
-static const struct file_operations supported_enctypes_ops = {
-	.open		= supported_enctypes_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(supported_enctypes);
 #endif /* CONFIG_SUNRPC_GSS or CONFIG_SUNRPC_GSS_MODULE */
 
 static const struct file_operations pool_stats_operations = {
@@ -224,19 +204,9 @@ static const struct file_operations pool_stats_operations = {
 	.release	= nfsd_pool_stats_release,
 };
 
-static const struct file_operations reply_cache_stats_operations = {
-	.open		= nfsd_reply_cache_stats_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(nfsd_reply_cache_stats);
 
-static const struct file_operations filecache_ops = {
-	.open		= nfsd_file_cache_stats_open,
-	.read		= seq_read,
-	.llseek		= seq_lseek,
-	.release	= single_release,
-};
+DEFINE_SHOW_ATTRIBUTE(nfsd_file_cache_stats);
 
 /*----------------------------------------------------------------------------*/
 /*
@@ -1365,7 +1335,7 @@ static int nfsd_fill_super(struct super_block *sb, struct fs_context *fc)
 		/* Per-export io stats use same ops as exports file */
 		[NFSD_Export_Stats] = {"export_stats", &exports_nfsd_operations, S_IRUGO},
 		[NFSD_Export_features] = {"export_features",
-					&export_features_operations, S_IRUGO},
+					&export_features_fops, S_IRUGO},
 		[NFSD_FO_UnlockIP] = {"unlock_ip",
 					&transaction_ops, S_IWUSR|S_IRUSR},
 		[NFSD_FO_UnlockFS] = {"unlock_filesystem",
@@ -1374,14 +1344,16 @@ static int nfsd_fill_super(struct super_block *sb, struct fs_context *fc)
 		[NFSD_Threads] = {"threads", &transaction_ops, S_IWUSR|S_IRUSR},
 		[NFSD_Pool_Threads] = {"pool_threads", &transaction_ops, S_IWUSR|S_IRUSR},
 		[NFSD_Pool_Stats] = {"pool_stats", &pool_stats_operations, S_IRUGO},
-		[NFSD_Reply_Cache_Stats] = {"reply_cache_stats", &reply_cache_stats_operations, S_IRUGO},
+		[NFSD_Reply_Cache_Stats] = {"reply_cache_stats",
+					&nfsd_reply_cache_stats_fops, S_IRUGO},
 		[NFSD_Versions] = {"versions", &transaction_ops, S_IWUSR|S_IRUSR},
 		[NFSD_Ports] = {"portlist", &transaction_ops, S_IWUSR|S_IRUGO},
 		[NFSD_MaxBlkSize] = {"max_block_size", &transaction_ops, S_IWUSR|S_IRUGO},
 		[NFSD_MaxConnections] = {"max_connections", &transaction_ops, S_IWUSR|S_IRUGO},
-		[NFSD_Filecache] = {"filecache", &filecache_ops, S_IRUGO},
+		[NFSD_Filecache] = {"filecache", &nfsd_file_cache_stats_fops, S_IRUGO},
 #if defined(CONFIG_SUNRPC_GSS) || defined(CONFIG_SUNRPC_GSS_MODULE)
-		[NFSD_SupportedEnctypes] = {"supported_krb5_enctypes", &supported_enctypes_ops, S_IRUGO},
+		[NFSD_SupportedEnctypes] = {"supported_krb5_enctypes",
+					&supported_enctypes_fops, S_IRUGO},
 #endif /* CONFIG_SUNRPC_GSS or CONFIG_SUNRPC_GSS_MODULE */
 #ifdef CONFIG_NFSD_V4
 		[NFSD_Leasetime] = {"nfsv4leasetime", &transaction_ops, S_IWUSR|S_IRUSR},
@@ -1481,11 +1453,12 @@ static __net_init int nfsd_init_net(struct net *net)
 		goto out_idmap_error;
 	nn->nfsd_versions = NULL;
 	nn->nfsd4_minorversions = NULL;
+	retval = nfsd4_init_leases_net(nn);
+	if (retval)
+		goto out_drc_error;
 	retval = nfsd_reply_cache_init(nn);
 	if (retval)
 		goto out_drc_error;
-	nfsd4_init_leases_net(nn);
-
 	get_random_bytes(&nn->siphash_key, sizeof(nn->siphash_key));
 	seqlock_init(&nn->writeverf_lock);
 
@@ -1507,6 +1480,7 @@ static __net_exit void nfsd_exit_net(struct net *net)
 	nfsd_idmap_shutdown(net);
 	nfsd_export_shutdown(net);
 	nfsd_netns_free_versions(net_generic(net, nfsd_net_id));
+	nfsd4_leases_net_shutdown(nn);
 }
 
 static struct pernet_operations nfsd_net_ops = {
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index 57a468ed85c3..09726c5b9a31 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -164,6 +164,7 @@ char * nfs4_recoverydir(void);
 bool nfsd4_spo_must_allow(struct svc_rqst *rqstp);
 int nfsd4_create_laundry_wq(void);
 void nfsd4_destroy_laundry_wq(void);
+bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp, struct inode *inode);
 #else
 static inline int nfsd4_init_slabs(void) { return 0; }
 static inline void nfsd4_free_slabs(void) { }
@@ -179,6 +180,11 @@ static inline bool nfsd4_spo_must_allow(struct svc_rqst *rqstp)
 }
 static inline int nfsd4_create_laundry_wq(void) { return 0; };
 static inline void nfsd4_destroy_laundry_wq(void) {};
+static inline bool nfsd_wait_for_delegreturn(struct svc_rqst *rqstp,
+					      struct inode *inode)
+{
+	return false;
+}
 #endif
 
 /*
@@ -343,6 +349,7 @@ void		nfsd_lockd_shutdown(void);
 #define	NFSD_COURTESY_CLIENT_TIMEOUT	(24 * 60 * 60)	/* seconds */
 #define	NFSD_CLIENT_MAX_TRIM_PER_RUN	128
 #define	NFS4_CLIENTS_PER_GB		1024
+#define NFSD_DELEGRETURN_TIMEOUT	(HZ / 34)	/* 30ms */
 
 /*
  * The following attributes are currently not supported by the NFSv4 server:
@@ -498,7 +505,8 @@ extern void unregister_cld_notifier(void);
 extern void nfsd4_ssc_init_umount_work(struct nfsd_net *nn);
 #endif
 
-extern void nfsd4_init_leases_net(struct nfsd_net *nn);
+extern int nfsd4_init_leases_net(struct nfsd_net *nn);
+extern void nfsd4_leases_net_shutdown(struct nfsd_net *nn);
 
 #else /* CONFIG_NFSD_V4 */
 static inline int nfsd4_is_junction(struct dentry *dentry)
@@ -506,7 +514,8 @@ static inline int nfsd4_is_junction(struct dentry *dentry)
 	return 0;
 }
 
-static inline void nfsd4_init_leases_net(struct nfsd_net *nn) {};
+static inline int nfsd4_init_leases_net(struct nfsd_net *nn) { return 0; };
+static inline void nfsd4_leases_net_shutdown(struct nfsd_net *nn) {};
 
 #define register_cld_notifier() 0
 #define unregister_cld_notifier() do { } while(0)
diff --git a/fs/nfsd/nfsfh.c b/fs/nfsd/nfsfh.c
index a5b71526cee0..d73434200df9 100644
--- a/fs/nfsd/nfsfh.c
+++ b/fs/nfsd/nfsfh.c
@@ -392,13 +392,7 @@ fh_verify(struct svc_rqst *rqstp, struct svc_fh *fhp, umode_t type, int access)
 skip_pseudoflavor_check:
 	/* Finally, check access permissions. */
 	error = nfsd_permission(rqstp, exp, dentry, access);
-
-	if (error) {
-		dprintk("fh_verify: %pd2 permission failure, "
-			"acc=%x, error=%d\n",
-			dentry,
-			access, ntohl(error));
-	}
+	trace_nfsd_fh_verify_err(rqstp, fhp, type, access, error);
 out:
 	if (error == nfserr_stale)
 		nfsd_stats_fh_stale_inc(exp);
diff --git a/fs/nfsd/nfsproc.c b/fs/nfsd/nfsproc.c
index 7381972f1677..82b3ddeacc33 100644
--- a/fs/nfsd/nfsproc.c
+++ b/fs/nfsd/nfsproc.c
@@ -185,6 +185,7 @@ nfsd_proc_read(struct svc_rqst *rqstp)
 		argp->count, argp->offset);
 
 	argp->count = min_t(u32, argp->count, NFSSVC_MAXBLKSIZE_V2);
+	argp->count = min_t(u32, argp->count, rqstp->rq_res.buflen);
 
 	v = 0;
 	len = argp->count;
@@ -390,9 +391,8 @@ nfsd_proc_create(struct svc_rqst *rqstp)
 	resp->status = nfs_ok;
 	if (!inode) {
 		/* File doesn't exist. Create it and set attrs */
-		resp->status = nfsd_create_locked(rqstp, dirfhp, argp->name,
-						  argp->len, &attrs, type, rdev,
-						  newfhp);
+		resp->status = nfsd_create_locked(rqstp, dirfhp, &attrs, type,
+						  rdev, newfhp);
 	} else if (type == S_IFREG) {
 		dprintk("nfsd:   existing %s, valid=%x, size=%ld\n",
 			argp->name, attr->ia_valid, (long) attr->ia_size);
@@ -567,24 +567,15 @@ static void nfsd_init_dirlist_pages(struct svc_rqst *rqstp,
 	struct xdr_buf *buf = &resp->dirlist;
 	struct xdr_stream *xdr = &resp->xdr;
 
-	count = clamp(count, (u32)(XDR_UNIT * 2), svc_max_payload(rqstp));
-
 	memset(buf, 0, sizeof(*buf));
 
 	/* Reserve room for the NULL ptr & eof flag (-2 words) */
-	buf->buflen = count - XDR_UNIT * 2;
+	buf->buflen = clamp(count, (u32)(XDR_UNIT * 2), (u32)PAGE_SIZE);
+	buf->buflen -= XDR_UNIT * 2;
 	buf->pages = rqstp->rq_next_page;
 	rqstp->rq_next_page++;
 
-	/* This is xdr_init_encode(), but it assumes that
-	 * the head kvec has already been consumed. */
-	xdr_set_scratch_buffer(xdr, NULL, 0);
-	xdr->buf = buf;
-	xdr->page_ptr = buf->pages;
-	xdr->iov = NULL;
-	xdr->p = page_address(*buf->pages);
-	xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE);
-	xdr->rqst = NULL;
+	xdr_init_encode_pages(xdr, buf, buf->pages,  NULL);
 }
 
 /*
@@ -646,6 +637,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_voidarg,
 		.pc_encode = nfssvc_encode_voidres,
 		.pc_argsize = sizeof(struct nfsd_voidargs),
+		.pc_argzero = sizeof(struct nfsd_voidargs),
 		.pc_ressize = sizeof(struct nfsd_voidres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = 0,
@@ -657,6 +649,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_encode = nfssvc_encode_attrstatres,
 		.pc_release = nfssvc_release_attrstat,
 		.pc_argsize = sizeof(struct nfsd_fhandle),
+		.pc_argzero = sizeof(struct nfsd_fhandle),
 		.pc_ressize = sizeof(struct nfsd_attrstat),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+AT,
@@ -668,6 +661,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_encode = nfssvc_encode_attrstatres,
 		.pc_release = nfssvc_release_attrstat,
 		.pc_argsize = sizeof(struct nfsd_sattrargs),
+		.pc_argzero = sizeof(struct nfsd_sattrargs),
 		.pc_ressize = sizeof(struct nfsd_attrstat),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+AT,
@@ -678,6 +672,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_voidarg,
 		.pc_encode = nfssvc_encode_voidres,
 		.pc_argsize = sizeof(struct nfsd_voidargs),
+		.pc_argzero = sizeof(struct nfsd_voidargs),
 		.pc_ressize = sizeof(struct nfsd_voidres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = 0,
@@ -689,6 +684,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_encode = nfssvc_encode_diropres,
 		.pc_release = nfssvc_release_diropres,
 		.pc_argsize = sizeof(struct nfsd_diropargs),
+		.pc_argzero = sizeof(struct nfsd_diropargs),
 		.pc_ressize = sizeof(struct nfsd_diropres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+FH+AT,
@@ -699,6 +695,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_fhandleargs,
 		.pc_encode = nfssvc_encode_readlinkres,
 		.pc_argsize = sizeof(struct nfsd_fhandle),
+		.pc_argzero = sizeof(struct nfsd_fhandle),
 		.pc_ressize = sizeof(struct nfsd_readlinkres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+1+NFS_MAXPATHLEN/4,
@@ -710,6 +707,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_encode = nfssvc_encode_readres,
 		.pc_release = nfssvc_release_readres,
 		.pc_argsize = sizeof(struct nfsd_readargs),
+		.pc_argzero = sizeof(struct nfsd_readargs),
 		.pc_ressize = sizeof(struct nfsd_readres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+AT+1+NFSSVC_MAXBLKSIZE_V2/4,
@@ -720,6 +718,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_voidarg,
 		.pc_encode = nfssvc_encode_voidres,
 		.pc_argsize = sizeof(struct nfsd_voidargs),
+		.pc_argzero = sizeof(struct nfsd_voidargs),
 		.pc_ressize = sizeof(struct nfsd_voidres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = 0,
@@ -731,6 +730,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_encode = nfssvc_encode_attrstatres,
 		.pc_release = nfssvc_release_attrstat,
 		.pc_argsize = sizeof(struct nfsd_writeargs),
+		.pc_argzero = sizeof(struct nfsd_writeargs),
 		.pc_ressize = sizeof(struct nfsd_attrstat),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+AT,
@@ -742,6 +742,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_encode = nfssvc_encode_diropres,
 		.pc_release = nfssvc_release_diropres,
 		.pc_argsize = sizeof(struct nfsd_createargs),
+		.pc_argzero = sizeof(struct nfsd_createargs),
 		.pc_ressize = sizeof(struct nfsd_diropres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+FH+AT,
@@ -752,6 +753,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_diropargs,
 		.pc_encode = nfssvc_encode_statres,
 		.pc_argsize = sizeof(struct nfsd_diropargs),
+		.pc_argzero = sizeof(struct nfsd_diropargs),
 		.pc_ressize = sizeof(struct nfsd_stat),
 		.pc_cachetype = RC_REPLSTAT,
 		.pc_xdrressize = ST,
@@ -762,6 +764,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_renameargs,
 		.pc_encode = nfssvc_encode_statres,
 		.pc_argsize = sizeof(struct nfsd_renameargs),
+		.pc_argzero = sizeof(struct nfsd_renameargs),
 		.pc_ressize = sizeof(struct nfsd_stat),
 		.pc_cachetype = RC_REPLSTAT,
 		.pc_xdrressize = ST,
@@ -772,6 +775,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_linkargs,
 		.pc_encode = nfssvc_encode_statres,
 		.pc_argsize = sizeof(struct nfsd_linkargs),
+		.pc_argzero = sizeof(struct nfsd_linkargs),
 		.pc_ressize = sizeof(struct nfsd_stat),
 		.pc_cachetype = RC_REPLSTAT,
 		.pc_xdrressize = ST,
@@ -782,6 +786,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_symlinkargs,
 		.pc_encode = nfssvc_encode_statres,
 		.pc_argsize = sizeof(struct nfsd_symlinkargs),
+		.pc_argzero = sizeof(struct nfsd_symlinkargs),
 		.pc_ressize = sizeof(struct nfsd_stat),
 		.pc_cachetype = RC_REPLSTAT,
 		.pc_xdrressize = ST,
@@ -793,6 +798,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_encode = nfssvc_encode_diropres,
 		.pc_release = nfssvc_release_diropres,
 		.pc_argsize = sizeof(struct nfsd_createargs),
+		.pc_argzero = sizeof(struct nfsd_createargs),
 		.pc_ressize = sizeof(struct nfsd_diropres),
 		.pc_cachetype = RC_REPLBUFF,
 		.pc_xdrressize = ST+FH+AT,
@@ -803,6 +809,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_diropargs,
 		.pc_encode = nfssvc_encode_statres,
 		.pc_argsize = sizeof(struct nfsd_diropargs),
+		.pc_argzero = sizeof(struct nfsd_diropargs),
 		.pc_ressize = sizeof(struct nfsd_stat),
 		.pc_cachetype = RC_REPLSTAT,
 		.pc_xdrressize = ST,
@@ -813,6 +820,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_readdirargs,
 		.pc_encode = nfssvc_encode_readdirres,
 		.pc_argsize = sizeof(struct nfsd_readdirargs),
+		.pc_argzero = sizeof(struct nfsd_readdirargs),
 		.pc_ressize = sizeof(struct nfsd_readdirres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_name = "READDIR",
@@ -822,6 +830,7 @@ static const struct svc_procedure nfsd_procedures2[18] = {
 		.pc_decode = nfssvc_decode_fhandleargs,
 		.pc_encode = nfssvc_encode_statfsres,
 		.pc_argsize = sizeof(struct nfsd_fhandle),
+		.pc_argzero = sizeof(struct nfsd_fhandle),
 		.pc_ressize = sizeof(struct nfsd_statfsres),
 		.pc_cachetype = RC_NOCACHE,
 		.pc_xdrressize = ST+5,
diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c
index 4bb5baa17040..bfbd9f672f59 100644
--- a/fs/nfsd/nfssvc.c
+++ b/fs/nfsd/nfssvc.c
@@ -799,7 +799,7 @@ nfsd_svc(int nrservs, struct net *net, const struct cred *cred)
 	if (nrservs == 0 && nn->nfsd_serv == NULL)
 		goto out;
 
-	strlcpy(nn->nfsd_name, utsname()->nodename,
+	strscpy(nn->nfsd_name, utsname()->nodename,
 		sizeof(nn->nfsd_name));
 
 	error = nfsd_create_serv(net);
diff --git a/fs/nfsd/nfsxdr.c b/fs/nfsd/nfsxdr.c
index aba8520b4b8b..caf6355b18fa 100644
--- a/fs/nfsd/nfsxdr.c
+++ b/fs/nfsd/nfsxdr.c
@@ -338,10 +338,8 @@ nfssvc_decode_writeargs(struct svc_rqst *rqstp, struct xdr_stream *xdr)
 		return false;
 	if (args->len > NFSSVC_MAXBLKSIZE_V2)
 		return false;
-	if (!xdr_stream_subsegment(xdr, &args->payload, args->len))
-		return false;
 
-	return true;
+	return xdr_stream_subsegment(xdr, &args->payload, args->len);
 }
 
 bool
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index ae596dbf8667..e2daef3cc003 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -57,11 +57,11 @@ typedef struct {
 } stateid_t;
 
 typedef struct {
-	stateid_t		stid;
+	stateid_t		cs_stid;
 #define NFS4_COPY_STID 1
 #define NFS4_COPYNOTIFY_STID 2
-	unsigned char		sc_type;
-	refcount_t		sc_count;
+	unsigned char		cs_type;
+	refcount_t		cs_count;
 } copy_stateid_t;
 
 struct nfsd4_callback {
@@ -175,7 +175,7 @@ static inline struct nfs4_delegation *delegstateid(struct nfs4_stid *s)
 /* Maximum number of slots per session. 160 is useful for long haul TCP */
 #define NFSD_MAX_SLOTS_PER_SESSION     160
 /* Maximum number of operations per session compound */
-#define NFSD_MAX_OPS_PER_COMPOUND	16
+#define NFSD_MAX_OPS_PER_COMPOUND	50
 /* Maximum  session per slot cache size */
 #define NFSD_SLOT_CACHE_SIZE		2048
 /* Maximum number of NFSD_SLOT_CACHE_SIZE slots per session */
@@ -692,12 +692,11 @@ extern void nfsd4_probe_callback_sync(struct nfs4_client *clp);
 extern void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *);
 extern void nfsd4_init_cb(struct nfsd4_callback *cb, struct nfs4_client *clp,
 		const struct nfsd4_callback_ops *ops, enum nfsd4_cb_op op);
-extern void nfsd4_run_cb(struct nfsd4_callback *cb);
+extern bool nfsd4_run_cb(struct nfsd4_callback *cb);
 extern int nfsd4_create_callback_queue(void);
 extern void nfsd4_destroy_callback_queue(void);
 extern void nfsd4_shutdown_callback(struct nfs4_client *);
 extern void nfsd4_shutdown_copy(struct nfs4_client *clp);
-extern void nfsd4_prepare_cb_recall(struct nfs4_delegation *dp);
 extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name,
 				struct xdr_netobj princhash, struct nfsd_net *nn);
 extern bool nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn);
diff --git a/fs/nfsd/stats.c b/fs/nfsd/stats.c
index a8c5a02a84f0..777e24e5da33 100644
--- a/fs/nfsd/stats.c
+++ b/fs/nfsd/stats.c
@@ -32,7 +32,7 @@ struct svc_stat		nfsd_svcstats = {
 	.program	= &nfsd_program,
 };
 
-static int nfsd_proc_show(struct seq_file *seq, void *v)
+static int nfsd_show(struct seq_file *seq, void *v)
 {
 	int i;
 
@@ -72,17 +72,7 @@ static int nfsd_proc_show(struct seq_file *seq, void *v)
 	return 0;
 }
 
-static int nfsd_proc_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, nfsd_proc_show, NULL);
-}
-
-static const struct proc_ops nfsd_proc_ops = {
-	.proc_open	= nfsd_proc_open,
-	.proc_read	= seq_read,
-	.proc_lseek	= seq_lseek,
-	.proc_release	= single_release,
-};
+DEFINE_PROC_SHOW_ATTRIBUTE(nfsd);
 
 int nfsd_percpu_counters_init(struct percpu_counter counters[], int num)
 {
diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h
index 9ebd67d461f9..06a96e955bd0 100644
--- a/fs/nfsd/trace.h
+++ b/fs/nfsd/trace.h
@@ -84,19 +84,26 @@ DEFINE_NFSD_XDR_ERR_EVENT(cant_encode);
 		{ NFSD_MAY_64BIT_COOKIE,	"64BIT_COOKIE" })
 
 TRACE_EVENT(nfsd_compound,
-	TP_PROTO(const struct svc_rqst *rqst,
-		 u32 args_opcnt),
-	TP_ARGS(rqst, args_opcnt),
+	TP_PROTO(
+		const struct svc_rqst *rqst,
+		const char *tag,
+		u32 taglen,
+		u32 opcnt
+	),
+	TP_ARGS(rqst, tag, taglen, opcnt),
 	TP_STRUCT__entry(
 		__field(u32, xid)
-		__field(u32, args_opcnt)
+		__field(u32, opcnt)
+		__string_len(tag, tag, taglen)
 	),
 	TP_fast_assign(
 		__entry->xid = be32_to_cpu(rqst->rq_xid);
-		__entry->args_opcnt = args_opcnt;
+		__entry->opcnt = opcnt;
+		__assign_str_len(tag, tag, taglen);
 	),
-	TP_printk("xid=0x%08x opcnt=%u",
-		__entry->xid, __entry->args_opcnt)
+	TP_printk("xid=0x%08x opcnt=%u tag=%s",
+		__entry->xid, __entry->opcnt, __get_str(tag)
+	)
 )
 
 TRACE_EVENT(nfsd_compound_status,
@@ -195,7 +202,7 @@ TRACE_EVENT(nfsd_fh_verify,
 		__sockaddr(client, rqstp->rq_xprt->xpt_remotelen)
 		__field(u32, xid)
 		__field(u32, fh_hash)
-		__field(void *, inode)
+		__field(const void *, inode)
 		__field(unsigned long, type)
 		__field(unsigned long, access)
 	),
@@ -211,13 +218,55 @@ TRACE_EVENT(nfsd_fh_verify,
 		__entry->type = type;
 		__entry->access = access;
 	),
-	TP_printk("xid=0x%08x fh_hash=0x%08x inode=%p type=%s access=%s",
-		__entry->xid, __entry->fh_hash, __entry->inode,
+	TP_printk("xid=0x%08x fh_hash=0x%08x type=%s access=%s",
+		__entry->xid, __entry->fh_hash,
 		show_fs_file_type(__entry->type),
 		show_nfsd_may_flags(__entry->access)
 	)
 );
 
+TRACE_EVENT_CONDITION(nfsd_fh_verify_err,
+	TP_PROTO(
+		const struct svc_rqst *rqstp,
+		const struct svc_fh *fhp,
+		umode_t type,
+		int access,
+		__be32 error
+	),
+	TP_ARGS(rqstp, fhp, type, access, error),
+	TP_CONDITION(error),
+	TP_STRUCT__entry(
+		__field(unsigned int, netns_ino)
+		__sockaddr(server, rqstp->rq_xprt->xpt_remotelen)
+		__sockaddr(client, rqstp->rq_xprt->xpt_remotelen)
+		__field(u32, xid)
+		__field(u32, fh_hash)
+		__field(const void *, inode)
+		__field(unsigned long, type)
+		__field(unsigned long, access)
+		__field(int, error)
+	),
+	TP_fast_assign(
+		__entry->netns_ino = SVC_NET(rqstp)->ns.inum;
+		__assign_sockaddr(server, &rqstp->rq_xprt->xpt_local,
+		       rqstp->rq_xprt->xpt_locallen);
+		__assign_sockaddr(client, &rqstp->rq_xprt->xpt_remote,
+				  rqstp->rq_xprt->xpt_remotelen);
+		__entry->xid = be32_to_cpu(rqstp->rq_xid);
+		__entry->fh_hash = knfsd_fh_hash(&fhp->fh_handle);
+		__entry->inode = d_inode(fhp->fh_dentry);
+		__entry->type = type;
+		__entry->access = access;
+		__entry->error = be32_to_cpu(error);
+	),
+	TP_printk("xid=0x%08x fh_hash=0x%08x type=%s access=%s error=%d",
+		__entry->xid, __entry->fh_hash,
+		show_fs_file_type(__entry->type),
+		show_nfsd_may_flags(__entry->access),
+		__entry->error
+	)
+);
+
 DECLARE_EVENT_CLASS(nfsd_fh_err_class,
 	TP_PROTO(struct svc_rqst *rqstp,
 		 struct svc_fh	*fhp,
@@ -489,6 +538,29 @@ DEFINE_NFSD_COPY_ERR_EVENT(clone_file_range_err);
 #include "filecache.h"
 #include "vfs.h"
 
+TRACE_EVENT(nfsd_delegret_wakeup,
+	TP_PROTO(
+		const struct svc_rqst *rqstp,
+		const struct inode *inode,
+		long timeo
+	),
+	TP_ARGS(rqstp, inode, timeo),
+	TP_STRUCT__entry(
+		__field(u32, xid)
+		__field(const void *, inode)
+		__field(long, timeo)
+	),
+	TP_fast_assign(
+		__entry->xid = be32_to_cpu(rqstp->rq_xid);
+		__entry->inode = inode;
+		__entry->timeo = timeo;
+	),
+	TP_printk("xid=0x%08x inode=%p%s",
+		  __entry->xid, __entry->inode,
+		  __entry->timeo == 0 ? " (timed out)" : ""
+	)
+);
+
 DECLARE_EVENT_CLASS(nfsd_stateid_class,
 	TP_PROTO(stateid_t *stp),
 	TP_ARGS(stp),
@@ -1399,6 +1471,45 @@ TRACE_EVENT(nfsd_cb_offload,
 		__entry->fh_hash, __entry->count, __entry->status)
 );
 
+DECLARE_EVENT_CLASS(nfsd_cb_done_class,
+	TP_PROTO(
+		const stateid_t *stp,
+		const struct rpc_task *task
+	),
+	TP_ARGS(stp, task),
+	TP_STRUCT__entry(
+		__field(u32, cl_boot)
+		__field(u32, cl_id)
+		__field(u32, si_id)
+		__field(u32, si_generation)
+		__field(int, status)
+	),
+	TP_fast_assign(
+		__entry->cl_boot = stp->si_opaque.so_clid.cl_boot;
+		__entry->cl_id = stp->si_opaque.so_clid.cl_id;
+		__entry->si_id = stp->si_opaque.so_id;
+		__entry->si_generation = stp->si_generation;
+		__entry->status = task->tk_status;
+	),
+	TP_printk("client %08x:%08x stateid %08x:%08x status=%d",
+		__entry->cl_boot, __entry->cl_id, __entry->si_id,
+		__entry->si_generation, __entry->status
+	)
+);
+
+#define DEFINE_NFSD_CB_DONE_EVENT(name)			\
+DEFINE_EVENT(nfsd_cb_done_class, name,			\
+	TP_PROTO(					\
+		const stateid_t *stp,			\
+		const struct rpc_task *task		\
+	),						\
+	TP_ARGS(stp, task))
+
+DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_recall_done);
+DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_notify_lock_done);
+DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_layout_done);
+DEFINE_NFSD_CB_DONE_EVENT(nfsd_cb_offload_done);
+
 #endif /* _NFSD_TRACE_H */
 
 #undef TRACE_INCLUDE_PATH
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index fc17b0ac8729..f650afedd67f 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -343,8 +343,61 @@ nfsd_get_write_access(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	return nfserrno(get_write_access(inode));
 }
 
-/*
- * Set various file attributes.  After this call fhp needs an fh_put.
+static int __nfsd_setattr(struct dentry *dentry, struct iattr *iap)
+{
+	int host_err;
+
+	if (iap->ia_valid & ATTR_SIZE) {
+		/*
+		 * RFC5661, Section 18.30.4:
+		 *   Changing the size of a file with SETATTR indirectly
+		 *   changes the time_modify and change attributes.
+		 *
+		 * (and similar for the older RFCs)
+		 */
+		struct iattr size_attr = {
+			.ia_valid	= ATTR_SIZE | ATTR_CTIME | ATTR_MTIME,
+			.ia_size	= iap->ia_size,
+		};
+
+		if (iap->ia_size < 0)
+			return -EFBIG;
+
+		host_err = notify_change(&init_user_ns, dentry, &size_attr, NULL);
+		if (host_err)
+			return host_err;
+		iap->ia_valid &= ~ATTR_SIZE;
+
+		/*
+		 * Avoid the additional setattr call below if the only other
+		 * attribute that the client sends is the mtime, as we update
+		 * it as part of the size change above.
+		 */
+		if ((iap->ia_valid & ~ATTR_MTIME) == 0)
+			return 0;
+	}
+
+	if (!iap->ia_valid)
+		return 0;
+
+	iap->ia_valid |= ATTR_CTIME;
+	return notify_change(&init_user_ns, dentry, iap, NULL);
+}
+
+/**
+ * nfsd_setattr - Set various file attributes.
+ * @rqstp: controlling RPC transaction
+ * @fhp: filehandle of target
+ * @attr: attributes to set
+ * @check_guard: set to 1 if guardtime is a valid timestamp
+ * @guardtime: do not act if ctime.tv_sec does not match this timestamp
+ *
+ * This call may adjust the contents of @attr (in particular, this
+ * call may change the bits in the na_iattr.ia_valid field).
+ *
+ * Returns nfs_ok on success, otherwise an NFS status code is
+ * returned. Caller must release @fhp by calling fh_put in either
+ * case.
  */
 __be32
 nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp,
@@ -357,9 +410,10 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	int		accmode = NFSD_MAY_SATTR;
 	umode_t		ftype = 0;
 	__be32		err;
-	int		host_err = 0;
+	int		host_err;
 	bool		get_write_count;
 	bool		size_change = (iap->ia_valid & ATTR_SIZE);
+	int		retries;
 
 	if (iap->ia_valid & ATTR_SIZE) {
 		accmode |= NFSD_MAY_WRITE|NFSD_MAY_OWNER_OVERRIDE;
@@ -414,43 +468,13 @@ nfsd_setattr(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	}
 
 	inode_lock(inode);
-	if (size_change) {
-		/*
-		 * RFC5661, Section 18.30.4:
-		 *   Changing the size of a file with SETATTR indirectly
-		 *   changes the time_modify and change attributes.
-		 *
-		 * (and similar for the older RFCs)
-		 */
-		struct iattr size_attr = {
-			.ia_valid	= ATTR_SIZE | ATTR_CTIME | ATTR_MTIME,
-			.ia_size	= iap->ia_size,
-		};
-
-		host_err = -EFBIG;
-		if (iap->ia_size < 0)
-			goto out_unlock;
-
-		host_err = notify_change(&init_user_ns, dentry, &size_attr, NULL);
-		if (host_err)
-			goto out_unlock;
-		iap->ia_valid &= ~ATTR_SIZE;
-
-		/*
-		 * Avoid the additional setattr call below if the only other
-		 * attribute that the client sends is the mtime, as we update
-		 * it as part of the size change above.
-		 */
-		if ((iap->ia_valid & ~ATTR_MTIME) == 0)
-			goto out_unlock;
-	}
-
-	if (iap->ia_valid) {
-		iap->ia_valid |= ATTR_CTIME;
-		host_err = notify_change(&init_user_ns, dentry, iap, NULL);
+	for (retries = 1;;) {
+		host_err = __nfsd_setattr(dentry, iap);
+		if (host_err != -EAGAIN || !retries--)
+			break;
+		if (!nfsd_wait_for_delegreturn(rqstp, inode))
+			break;
 	}
-
-out_unlock:
 	if (attr->na_seclabel && attr->na_seclabel->len)
 		attr->na_labelerr = security_inode_setsecctx(dentry,
 			attr->na_seclabel->data, attr->na_seclabel->len);
@@ -1255,7 +1279,7 @@ nfsd_check_ignore_resizing(struct iattr *iap)
 /* The parent directory should already be locked: */
 __be32
 nfsd_create_locked(struct svc_rqst *rqstp, struct svc_fh *fhp,
-		   char *fname, int flen, struct nfsd_attrs *attrs,
+		   struct nfsd_attrs *attrs,
 		   int type, dev_t rdev, struct svc_fh *resfhp)
 {
 	struct dentry	*dentry, *dchild;
@@ -1382,8 +1406,7 @@ nfsd_create(struct svc_rqst *rqstp, struct svc_fh *fhp,
 	if (err)
 		goto out_unlock;
 	fh_fill_pre_attrs(fhp);
-	err = nfsd_create_locked(rqstp, fhp, fname, flen, attrs, type,
-				 rdev, resfhp);
+	err = nfsd_create_locked(rqstp, fhp, attrs, type, rdev, resfhp);
 	fh_fill_post_attrs(fhp);
 out_unlock:
 	inode_unlock(dentry->d_inode);
@@ -1673,7 +1696,15 @@ retry:
 			.new_dir	= tdir,
 			.new_dentry	= ndentry,
 		};
-		host_err = vfs_rename(&rd);
+		int retries;
+
+		for (retries = 1;;) {
+			host_err = vfs_rename(&rd);
+			if (host_err != -EAGAIN || !retries--)
+				break;
+			if (!nfsd_wait_for_delegreturn(rqstp, d_inode(odentry)))
+				break;
+		}
 		if (!host_err) {
 			host_err = commit_metadata(tfhp);
 			if (!host_err)
@@ -1757,9 +1788,18 @@ nfsd_unlink(struct svc_rqst *rqstp, struct svc_fh *fhp, int type,
 
 	fh_fill_pre_attrs(fhp);
 	if (type != S_IFDIR) {
+		int retries;
+
 		if (rdentry->d_sb->s_export_op->flags & EXPORT_OP_CLOSE_BEFORE_UNLINK)
 			nfsd_close_cached_files(rdentry);
-		host_err = vfs_unlink(&init_user_ns, dirp, rdentry, NULL);
+
+		for (retries = 1;;) {
+			host_err = vfs_unlink(&init_user_ns, dirp, rdentry, NULL);
+			if (host_err != -EAGAIN || !retries--)
+				break;
+			if (!nfsd_wait_for_delegreturn(rqstp, rinode))
+				break;
+		}
 	} else {
 		host_err = vfs_rmdir(&init_user_ns, dirp, rdentry);
 	}
@@ -1814,7 +1854,7 @@ struct readdir_data {
 	int		full;
 };
 
-static int nfsd_buffered_filldir(struct dir_context *ctx, const char *name,
+static bool nfsd_buffered_filldir(struct dir_context *ctx, const char *name,
 				 int namlen, loff_t offset, u64 ino,
 				 unsigned int d_type)
 {
@@ -1826,7 +1866,7 @@ static int nfsd_buffered_filldir(struct dir_context *ctx, const char *name,
 	reclen = ALIGN(sizeof(struct buffered_dirent) + namlen, sizeof(u64));
 	if (buf->used + reclen > PAGE_SIZE) {
 		buf->full = 1;
-		return -EINVAL;
+		return false;
 	}
 
 	de->namlen = namlen;
@@ -1836,7 +1876,7 @@ static int nfsd_buffered_filldir(struct dir_context *ctx, const char *name,
 	memcpy(de->name, name, namlen);
 	buf->used += reclen;
 
-	return 0;
+	return true;
 }
 
 static __be32 nfsd_buffered_readdir(struct file *file, struct svc_fh *fhp,
diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h
index c95cd414b4bb..120521bc7b24 100644
--- a/fs/nfsd/vfs.h
+++ b/fs/nfsd/vfs.h
@@ -79,8 +79,8 @@ __be32		nfsd4_clone_file_range(struct svc_rqst *rqstp,
 				       u64 count, bool sync);
 #endif /* CONFIG_NFSD_V4 */
 __be32		nfsd_create_locked(struct svc_rqst *, struct svc_fh *,
-				char *name, int len, struct nfsd_attrs *attrs,
-				int type, dev_t rdev, struct svc_fh *res);
+				struct nfsd_attrs *attrs, int type, dev_t rdev,
+				struct svc_fh *res);
 __be32		nfsd_create(struct svc_rqst *, struct svc_fh *,
 				char *name, int len, struct nfsd_attrs *attrs,
 				int type, dev_t rdev, struct svc_fh *res);
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 96267258e629..0eb00105d845 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -717,13 +717,13 @@ struct nfsd4_compoundargs {
 	struct svcxdr_tmpbuf		*to_free;
 	struct svc_rqst			*rqstp;
 
-	u32				taglen;
 	char *				tag;
+	u32				taglen;
 	u32				minorversion;
+	u32				client_opcnt;
 	u32				opcnt;
 	struct nfsd4_op			*ops;
 	struct nfsd4_op			iops[8];
-	int				cachetype;
 };
 
 struct nfsd4_compoundres {
@@ -732,8 +732,8 @@ struct nfsd4_compoundres {
 	struct svc_rqst *		rqstp;
 
 	__be32				*statusp;
-	u32				taglen;
 	char *				tag;
+	u32				taglen;
 	u32				opcnt;
 
 	struct nfsd4_compound_state	cstate;
@@ -888,7 +888,8 @@ struct nfsd4_operation {
 	u32 op_flags;
 	char *op_name;
 	/* Try to get response size before operation */
-	u32 (*op_rsize_bop)(struct svc_rqst *, struct nfsd4_op *);
+	u32 (*op_rsize_bop)(const struct svc_rqst *rqstp,
+			const struct nfsd4_op *op);
 	void (*op_get_currentstateid)(struct nfsd4_compound_state *,
 			union nfsd4_op_u *);
 	void (*op_set_currentstateid)(struct nfsd4_compound_state *,
diff --git a/fs/notify/fanotify/fanotify.c b/fs/notify/fanotify/fanotify.c
index cd7d09a569ff..a2a15bc4df28 100644
--- a/fs/notify/fanotify/fanotify.c
+++ b/fs/notify/fanotify/fanotify.c
@@ -18,7 +18,7 @@
 
 #include "fanotify.h"
 
-static bool fanotify_path_equal(struct path *p1, struct path *p2)
+static bool fanotify_path_equal(const struct path *p1, const struct path *p2)
 {
 	return p1->mnt == p2->mnt && p1->dentry == p2->dentry;
 }
diff --git a/fs/notify/fanotify/fanotify.h b/fs/notify/fanotify/fanotify.h
index 1d9f11255c64..bf6d4d38afa0 100644
--- a/fs/notify/fanotify/fanotify.h
+++ b/fs/notify/fanotify/fanotify.h
@@ -458,7 +458,7 @@ static inline bool fanotify_event_has_path(struct fanotify_event *event)
 		event->type == FANOTIFY_EVENT_TYPE_PATH_PERM;
 }
 
-static inline struct path *fanotify_event_path(struct fanotify_event *event)
+static inline const struct path *fanotify_event_path(struct fanotify_event *event)
 {
 	if (event->type == FANOTIFY_EVENT_TYPE_PATH)
 		return &FANOTIFY_PE(event)->path;
diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c
index f0e49a406ffa..4546da4a54f9 100644
--- a/fs/notify/fanotify/fanotify_user.c
+++ b/fs/notify/fanotify/fanotify_user.c
@@ -249,7 +249,7 @@ out:
 	return event;
 }
 
-static int create_fd(struct fsnotify_group *group, struct path *path,
+static int create_fd(struct fsnotify_group *group, const struct path *path,
 		     struct file **file)
 {
 	int client_fd;
@@ -619,7 +619,7 @@ static ssize_t copy_event_to_user(struct fsnotify_group *group,
 				  char __user *buf, size_t count)
 {
 	struct fanotify_event_metadata metadata;
-	struct path *path = fanotify_event_path(event);
+	const struct path *path = fanotify_event_path(event);
 	struct fanotify_info *info = fanotify_event_info(event);
 	unsigned int info_mode = FAN_GROUP_FLAG(group, FANOTIFY_INFO_MODES);
 	unsigned int pidfd_mode = info_mode & FAN_REPORT_PIDFD;
@@ -1553,7 +1553,7 @@ static int fanotify_test_fid(struct dentry *dentry)
 }
 
 static int fanotify_events_supported(struct fsnotify_group *group,
-				     struct path *path, __u64 mask,
+				     const struct path *path, __u64 mask,
 				     unsigned int flags)
 {
 	unsigned int mark_type = flags & FANOTIFY_MARK_TYPE_BITS;
diff --git a/fs/nsfs.c b/fs/nsfs.c
index 800c1d0eb0d0..3506f6074288 100644
--- a/fs/nsfs.c
+++ b/fs/nsfs.c
@@ -28,7 +28,7 @@ static char *ns_dname(struct dentry *dentry, char *buffer, int buflen)
 	struct inode *inode = d_inode(dentry);
 	const struct proc_ns_operations *ns_ops = dentry->d_fsdata;
 
-	return dynamic_dname(dentry, buffer, buflen, "%s:[%lu]",
+	return dynamic_dname(buffer, buflen, "%s:[%lu]",
 		ns_ops->name, inode->i_ino);
 }
 
diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c
index 5ae8de09b271..001f4e053c85 100644
--- a/fs/ntfs/super.c
+++ b/fs/ntfs/super.c
@@ -2092,7 +2092,8 @@ get_ctx_vol_failed:
 	// TODO: Initialize security.
 	/* Get the extended system files' directory inode. */
 	vol->extend_ino = ntfs_iget(sb, FILE_Extend);
-	if (IS_ERR(vol->extend_ino) || is_bad_inode(vol->extend_ino)) {
+	if (IS_ERR(vol->extend_ino) || is_bad_inode(vol->extend_ino) ||
+	    !S_ISDIR(vol->extend_ino->i_mode)) {
 		if (!IS_ERR(vol->extend_ino))
 			iput(vol->extend_ino);
 		ntfs_error(sb, "Failed to load $Extend.");
diff --git a/fs/ntfs3/inode.c b/fs/ntfs3/inode.c
index 51363d4e8636..26a76ebfe58f 100644
--- a/fs/ntfs3/inode.c
+++ b/fs/ntfs3/inode.c
@@ -1927,8 +1927,6 @@ const struct inode_operations ntfs_link_inode_operations = {
 	.setattr	= ntfs3_setattr,
 	.listxattr	= ntfs_listxattr,
 	.permission	= ntfs_permission,
-	.get_acl	= ntfs_get_acl,
-	.set_acl	= ntfs_set_acl,
 };
 
 const struct address_space_operations ntfs_aops = {
diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c
index 6ae1f56b7358..7de8718c68a9 100644
--- a/fs/ntfs3/xattr.c
+++ b/fs/ntfs3/xattr.c
@@ -625,67 +625,6 @@ int ntfs_set_acl(struct user_namespace *mnt_userns, struct inode *inode,
 	return ntfs_set_acl_ex(mnt_userns, inode, acl, type, false);
 }
 
-static int ntfs_xattr_get_acl(struct user_namespace *mnt_userns,
-			      struct inode *inode, int type, void *buffer,
-			      size_t size)
-{
-	struct posix_acl *acl;
-	int err;
-
-	if (!(inode->i_sb->s_flags & SB_POSIXACL)) {
-		ntfs_inode_warn(inode, "add mount option \"acl\" to use acl");
-		return -EOPNOTSUPP;
-	}
-
-	acl = ntfs_get_acl(inode, type, false);
-	if (IS_ERR(acl))
-		return PTR_ERR(acl);
-
-	if (!acl)
-		return -ENODATA;
-
-	err = posix_acl_to_xattr(&init_user_ns, acl, buffer, size);
-	posix_acl_release(acl);
-
-	return err;
-}
-
-static int ntfs_xattr_set_acl(struct user_namespace *mnt_userns,
-			      struct inode *inode, int type, const void *value,
-			      size_t size)
-{
-	struct posix_acl *acl;
-	int err;
-
-	if (!(inode->i_sb->s_flags & SB_POSIXACL)) {
-		ntfs_inode_warn(inode, "add mount option \"acl\" to use acl");
-		return -EOPNOTSUPP;
-	}
-
-	if (!inode_owner_or_capable(mnt_userns, inode))
-		return -EPERM;
-
-	if (!value) {
-		acl = NULL;
-	} else {
-		acl = posix_acl_from_xattr(&init_user_ns, value, size);
-		if (IS_ERR(acl))
-			return PTR_ERR(acl);
-
-		if (acl) {
-			err = posix_acl_valid(&init_user_ns, acl);
-			if (err)
-				goto release_and_out;
-		}
-	}
-
-	err = ntfs_set_acl(mnt_userns, inode, acl, type);
-
-release_and_out:
-	posix_acl_release(acl);
-	return err;
-}
-
 /*
  * ntfs_init_acl - Initialize the ACLs of a new inode.
  *
@@ -852,23 +791,6 @@ static int ntfs_getxattr(const struct xattr_handler *handler, struct dentry *de,
 		goto out;
 	}
 
-#ifdef CONFIG_NTFS3_FS_POSIX_ACL
-	if ((name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1 &&
-	     !memcmp(name, XATTR_NAME_POSIX_ACL_ACCESS,
-		     sizeof(XATTR_NAME_POSIX_ACL_ACCESS))) ||
-	    (name_len == sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1 &&
-	     !memcmp(name, XATTR_NAME_POSIX_ACL_DEFAULT,
-		     sizeof(XATTR_NAME_POSIX_ACL_DEFAULT)))) {
-		/* TODO: init_user_ns? */
-		err = ntfs_xattr_get_acl(
-			&init_user_ns, inode,
-			name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1
-				? ACL_TYPE_ACCESS
-				: ACL_TYPE_DEFAULT,
-			buffer, size);
-		goto out;
-	}
-#endif
 	/* Deal with NTFS extended attribute. */
 	err = ntfs_get_ea(inode, name, name_len, buffer, size, NULL);
 
@@ -981,22 +903,6 @@ set_new_fa:
 		goto out;
 	}
 
-#ifdef CONFIG_NTFS3_FS_POSIX_ACL
-	if ((name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1 &&
-	     !memcmp(name, XATTR_NAME_POSIX_ACL_ACCESS,
-		     sizeof(XATTR_NAME_POSIX_ACL_ACCESS))) ||
-	    (name_len == sizeof(XATTR_NAME_POSIX_ACL_DEFAULT) - 1 &&
-	     !memcmp(name, XATTR_NAME_POSIX_ACL_DEFAULT,
-		     sizeof(XATTR_NAME_POSIX_ACL_DEFAULT)))) {
-		err = ntfs_xattr_set_acl(
-			mnt_userns, inode,
-			name_len == sizeof(XATTR_NAME_POSIX_ACL_ACCESS) - 1
-				? ACL_TYPE_ACCESS
-				: ACL_TYPE_DEFAULT,
-			value, size);
-		goto out;
-	}
-#endif
 	/* Deal with NTFS extended attribute. */
 	err = ntfs_set_ea(inode, name, name_len, value, size, flags, 0);
 
@@ -1086,7 +992,7 @@ static bool ntfs_xattr_user_list(struct dentry *dentry)
 }
 
 // clang-format off
-static const struct xattr_handler ntfs_xattr_handler = {
+static const struct xattr_handler ntfs_other_xattr_handler = {
 	.prefix	= "",
 	.get	= ntfs_getxattr,
 	.set	= ntfs_setxattr,
@@ -1094,7 +1000,11 @@ static const struct xattr_handler ntfs_xattr_handler = {
 };
 
 const struct xattr_handler *ntfs_xattr_handlers[] = {
-	&ntfs_xattr_handler,
+#ifdef CONFIG_NTFS3_FS_POSIX_ACL
+	&posix_acl_access_xattr_handler,
+	&posix_acl_default_xattr_handler,
+#endif
+	&ntfs_other_xattr_handler,
 	NULL,
 };
 // clang-format on
diff --git a/fs/ocfs2/dir.c b/fs/ocfs2/dir.c
index 81c3d65d68fe..694471fc46b8 100644
--- a/fs/ocfs2/dir.c
+++ b/fs/ocfs2/dir.c
@@ -2032,7 +2032,7 @@ struct ocfs2_empty_dir_priv {
 	unsigned seen_other;
 	unsigned dx_dir;
 };
-static int ocfs2_empty_dir_filldir(struct dir_context *ctx, const char *name,
+static bool ocfs2_empty_dir_filldir(struct dir_context *ctx, const char *name,
 				   int name_len, loff_t pos, u64 ino,
 				   unsigned type)
 {
@@ -2052,7 +2052,7 @@ static int ocfs2_empty_dir_filldir(struct dir_context *ctx, const char *name,
 	 */
 	if (name_len == 1 && !strncmp(".", name, 1) && pos == 0) {
 		p->seen_dot = 1;
-		return 0;
+		return true;
 	}
 
 	if (name_len == 2 && !strncmp("..", name, 2) &&
@@ -2060,13 +2060,13 @@ static int ocfs2_empty_dir_filldir(struct dir_context *ctx, const char *name,
 		p->seen_dot_dot = 1;
 
 		if (p->dx_dir && p->seen_dot)
-			return 1;
+			return false;
 
-		return 0;
+		return true;
 	}
 
 	p->seen_other = 1;
-	return 1;
+	return false;
 }
 
 static int ocfs2_empty_dir_dx(struct inode *inode,
diff --git a/fs/ocfs2/journal.c b/fs/ocfs2/journal.c
index fa87d89cf754..126671e6caed 100644
--- a/fs/ocfs2/journal.c
+++ b/fs/ocfs2/journal.c
@@ -2057,7 +2057,7 @@ struct ocfs2_orphan_filldir_priv {
 	enum ocfs2_orphan_reco_type orphan_reco_type;
 };
 
-static int ocfs2_orphan_filldir(struct dir_context *ctx, const char *name,
+static bool ocfs2_orphan_filldir(struct dir_context *ctx, const char *name,
 				int name_len, loff_t pos, u64 ino,
 				unsigned type)
 {
@@ -2066,21 +2066,21 @@ static int ocfs2_orphan_filldir(struct dir_context *ctx, const char *name,
 	struct inode *iter;
 
 	if (name_len == 1 && !strncmp(".", name, 1))
-		return 0;
+		return true;
 	if (name_len == 2 && !strncmp("..", name, 2))
-		return 0;
+		return true;
 
 	/* do not include dio entry in case of orphan scan */
 	if ((p->orphan_reco_type == ORPHAN_NO_NEED_TRUNCATE) &&
 			(!strncmp(name, OCFS2_DIO_ORPHAN_PREFIX,
 			OCFS2_DIO_ORPHAN_PREFIX_LEN)))
-		return 0;
+		return true;
 
 	/* Skip bad inodes so that recovery can continue */
 	iter = ocfs2_iget(p->osb, ino,
 			  OCFS2_FI_FLAG_ORPHAN_RECOVERY, 0);
 	if (IS_ERR(iter))
-		return 0;
+		return true;
 
 	if (!strncmp(name, OCFS2_DIO_ORPHAN_PREFIX,
 			OCFS2_DIO_ORPHAN_PREFIX_LEN))
@@ -2090,7 +2090,7 @@ static int ocfs2_orphan_filldir(struct dir_context *ctx, const char *name,
 	 * happen concurrently with unlink/rename */
 	if (OCFS2_I(iter)->ip_next_orphan) {
 		iput(iter);
-		return 0;
+		return true;
 	}
 
 	trace_ocfs2_orphan_filldir((unsigned long long)OCFS2_I(iter)->ip_blkno);
@@ -2099,7 +2099,7 @@ static int ocfs2_orphan_filldir(struct dir_context *ctx, const char *name,
 	OCFS2_I(iter)->ip_next_orphan = p->head;
 	p->head = iter;
 
-	return 0;
+	return true;
 }
 
 static int ocfs2_queue_orphans(struct ocfs2_super *osb,
diff --git a/fs/ocfs2/stack_user.c b/fs/ocfs2/stack_user.c
index a75e2b7d67f5..64e6ddcfe329 100644
--- a/fs/ocfs2/stack_user.c
+++ b/fs/ocfs2/stack_user.c
@@ -991,7 +991,7 @@ static int user_cluster_connect(struct ocfs2_cluster_connection *conn)
 	lc->oc_type = NO_CONTROLD;
 
 	rc = dlm_new_lockspace(conn->cc_name, conn->cc_cluster_name,
-			       DLM_LSFL_FS | DLM_LSFL_NEWEXCL, DLM_LVB_LEN,
+			       DLM_LSFL_NEWEXCL, DLM_LVB_LEN,
 			       &ocfs2_ls_ops, conn, &ops_rv, &fsdlm);
 	if (rc) {
 		if (rc == -EEXIST || rc == -EPROTO)
diff --git a/fs/open.c b/fs/open.c
index cf7e5c350a54..a81319b6177f 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -842,7 +842,9 @@ static int do_dentry_open(struct file *f,
 		return 0;
 	}
 
-	if (f->f_mode & FMODE_WRITE && !special_file(inode->i_mode)) {
+	if ((f->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ) {
+		i_readcount_inc(inode);
+	} else if (f->f_mode & FMODE_WRITE && !special_file(inode->i_mode)) {
 		error = get_write_access(inode);
 		if (unlikely(error))
 			goto cleanup_file;
@@ -882,8 +884,6 @@ static int do_dentry_open(struct file *f,
 			goto cleanup_all;
 	}
 	f->f_mode |= FMODE_OPENED;
-	if ((f->f_mode & (FMODE_READ | FMODE_WRITE)) == FMODE_READ)
-		i_readcount_inc(inode);
 	if ((f->f_mode & FMODE_READ) &&
 	     likely(f->f_op->read || f->f_op->read_iter))
 		f->f_mode |= FMODE_CAN_READ;
@@ -937,10 +937,7 @@ cleanup_all:
 	if (WARN_ON_ONCE(error > 0))
 		error = -EINVAL;
 	fops_put(f->f_op);
-	if (f->f_mode & FMODE_WRITER) {
-		put_write_access(inode);
-		__mnt_drop_write(f->f_path.mnt);
-	}
+	put_file_access(f);
 cleanup_file:
 	path_put(&f->f_path);
 	f->f_path.mnt = NULL;
diff --git a/fs/orangefs/file.c b/fs/orangefs/file.c
index 86810e5d7914..732661aa2680 100644
--- a/fs/orangefs/file.c
+++ b/fs/orangefs/file.c
@@ -417,9 +417,7 @@ static int orangefs_file_release(struct inode *inode, struct file *file)
 	 * readahead cache (if any); this forces an expensive refresh of
 	 * data for the next caller of mmap (or 'get_block' accesses)
 	 */
-	if (file_inode(file) &&
-	    file_inode(file)->i_mapping &&
-	    mapping_nrpages(&file_inode(file)->i_data)) {
+	if (mapping_nrpages(file->f_mapping)) {
 		if (orangefs_features & ORANGEFS_FEATURE_READAHEAD) {
 			gossip_debug(GOSSIP_INODE_DEBUG,
 			    "calling flush_racache on %pU\n",
diff --git a/fs/overlayfs/copy_up.c b/fs/overlayfs/copy_up.c
index fdde6c56cc3d..3ffea291c410 100644
--- a/fs/overlayfs/copy_up.c
+++ b/fs/overlayfs/copy_up.c
@@ -44,7 +44,7 @@ static bool ovl_must_copy_xattr(const char *name)
 	       !strncmp(name, XATTR_SECURITY_PREFIX, XATTR_SECURITY_PREFIX_LEN);
 }
 
-int ovl_copy_xattr(struct super_block *sb, struct path *oldpath, struct dentry *new)
+int ovl_copy_xattr(struct super_block *sb, const struct path *oldpath, struct dentry *new)
 {
 	struct dentry *old = oldpath->dentry;
 	ssize_t list_size, size, value_size = 0;
@@ -132,8 +132,8 @@ out:
 	return error;
 }
 
-static int ovl_copy_fileattr(struct inode *inode, struct path *old,
-			     struct path *new)
+static int ovl_copy_fileattr(struct inode *inode, const struct path *old,
+			     const struct path *new)
 {
 	struct fileattr oldfa = { .flags_valid = true };
 	struct fileattr newfa = { .flags_valid = true };
@@ -193,8 +193,8 @@ static int ovl_copy_fileattr(struct inode *inode, struct path *old,
 	return ovl_real_fileattr_set(new, &newfa);
 }
 
-static int ovl_copy_up_data(struct ovl_fs *ofs, struct path *old,
-			    struct path *new, loff_t len)
+static int ovl_copy_up_data(struct ovl_fs *ofs, const struct path *old,
+			    const struct path *new, loff_t len)
 {
 	struct file *old_file;
 	struct file *new_file;
@@ -872,7 +872,7 @@ static bool ovl_need_meta_copy_up(struct dentry *dentry, umode_t mode,
 	return true;
 }
 
-static ssize_t ovl_getxattr_value(struct path *path, char *name, char **value)
+static ssize_t ovl_getxattr_value(const struct path *path, char *name, char **value)
 {
 	ssize_t res;
 	char *buf;
diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c
index daff601b5c41..a1a22f58ba18 100644
--- a/fs/overlayfs/file.c
+++ b/fs/overlayfs/file.c
@@ -38,7 +38,7 @@ static char ovl_whatisit(struct inode *inode, struct inode *realinode)
 #define OVL_OPEN_FLAGS (O_NOATIME | FMODE_NONOTIFY)
 
 static struct file *ovl_open_realfile(const struct file *file,
-				      struct path *realpath)
+				      const struct path *realpath)
 {
 	struct inode *realinode = d_inode(realpath->dentry);
 	struct inode *inode = file_inode(file);
diff --git a/fs/overlayfs/inode.c b/fs/overlayfs/inode.c
index 0fbcb590af84..9e61511de7a7 100644
--- a/fs/overlayfs/inode.c
+++ b/fs/overlayfs/inode.c
@@ -588,7 +588,7 @@ static int ovl_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
  * Introducing security_inode_fileattr_get/set() hooks would solve this issue
  * properly.
  */
-static int ovl_security_fileattr(struct path *realpath, struct fileattr *fa,
+static int ovl_security_fileattr(const struct path *realpath, struct fileattr *fa,
 				 bool set)
 {
 	struct file *file;
@@ -610,7 +610,7 @@ static int ovl_security_fileattr(struct path *realpath, struct fileattr *fa,
 	return err;
 }
 
-int ovl_real_fileattr_set(struct path *realpath, struct fileattr *fa)
+int ovl_real_fileattr_set(const struct path *realpath, struct fileattr *fa)
 {
 	int err;
 
@@ -685,7 +685,7 @@ static void ovl_fileattr_prot_flags(struct inode *inode, struct fileattr *fa)
 	}
 }
 
-int ovl_real_fileattr_get(struct path *realpath, struct fileattr *fa)
+int ovl_real_fileattr_get(const struct path *realpath, struct fileattr *fa)
 {
 	int err;
 
diff --git a/fs/overlayfs/namei.c b/fs/overlayfs/namei.c
index 69dc577974f8..0fd1d5fdfc72 100644
--- a/fs/overlayfs/namei.c
+++ b/fs/overlayfs/namei.c
@@ -26,7 +26,7 @@ struct ovl_lookup_data {
 	bool metacopy;
 };
 
-static int ovl_check_redirect(struct path *path, struct ovl_lookup_data *d,
+static int ovl_check_redirect(const struct path *path, struct ovl_lookup_data *d,
 			      size_t prelen, const char *post)
 {
 	int res;
@@ -194,7 +194,7 @@ struct dentry *ovl_decode_real_fh(struct ovl_fs *ofs, struct ovl_fh *fh,
 	return real;
 }
 
-static bool ovl_is_opaquedir(struct ovl_fs *ofs, struct path *path)
+static bool ovl_is_opaquedir(struct ovl_fs *ofs, const struct path *path)
 {
 	return ovl_path_check_dir_xattr(ofs, path, OVL_XATTR_OPAQUE);
 }
diff --git a/fs/overlayfs/overlayfs.h b/fs/overlayfs/overlayfs.h
index 87759165d32b..59624521eeb2 100644
--- a/fs/overlayfs/overlayfs.h
+++ b/fs/overlayfs/overlayfs.h
@@ -208,7 +208,7 @@ static inline int ovl_do_symlink(struct ovl_fs *ofs,
 	return err;
 }
 
-static inline ssize_t ovl_do_getxattr(struct path *path, const char *name,
+static inline ssize_t ovl_do_getxattr(const struct path *path, const char *name,
 				      void *value, size_t size)
 {
 	int err, len;
@@ -238,7 +238,7 @@ static inline ssize_t ovl_getxattr_upper(struct ovl_fs *ofs,
 }
 
 static inline ssize_t ovl_path_getxattr(struct ovl_fs *ofs,
-					 struct path *path,
+					 const struct path *path,
 					 enum ovl_xattr ox, void *value,
 					 size_t size)
 {
@@ -250,7 +250,7 @@ static inline int ovl_do_setxattr(struct ovl_fs *ofs, struct dentry *dentry,
 				  size_t size, int flags)
 {
 	int err = vfs_setxattr(ovl_upper_mnt_userns(ofs), dentry, name,
-			       (void *)value, size, flags);
+			       value, size, flags);
 
 	pr_debug("setxattr(%pd2, \"%s\", \"%*pE\", %zu, %d) = %i\n",
 		 dentry, name, min((int)size, 48), value, size, flags, err);
@@ -401,13 +401,13 @@ void ovl_inode_update(struct inode *inode, struct dentry *upperdentry);
 void ovl_dir_modified(struct dentry *dentry, bool impurity);
 u64 ovl_dentry_version_get(struct dentry *dentry);
 bool ovl_is_whiteout(struct dentry *dentry);
-struct file *ovl_path_open(struct path *path, int flags);
+struct file *ovl_path_open(const struct path *path, int flags);
 int ovl_copy_up_start(struct dentry *dentry, int flags);
 void ovl_copy_up_end(struct dentry *dentry);
 bool ovl_already_copied_up(struct dentry *dentry, int flags);
-bool ovl_path_check_dir_xattr(struct ovl_fs *ofs, struct path *path,
+bool ovl_path_check_dir_xattr(struct ovl_fs *ofs, const struct path *path,
 			      enum ovl_xattr ox);
-bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, struct path *path);
+bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, const struct path *path);
 
 static inline bool ovl_check_origin_xattr(struct ovl_fs *ofs,
 					  struct dentry *upperdentry)
@@ -430,9 +430,9 @@ bool ovl_need_index(struct dentry *dentry);
 int ovl_nlink_start(struct dentry *dentry);
 void ovl_nlink_end(struct dentry *dentry);
 int ovl_lock_rename_workdir(struct dentry *workdir, struct dentry *upperdir);
-int ovl_check_metacopy_xattr(struct ovl_fs *ofs, struct path *path);
+int ovl_check_metacopy_xattr(struct ovl_fs *ofs, const struct path *path);
 bool ovl_is_metacopy_dentry(struct dentry *dentry);
-char *ovl_get_redirect_xattr(struct ovl_fs *ofs, struct path *path, int padding);
+char *ovl_get_redirect_xattr(struct ovl_fs *ofs, const struct path *path, int padding);
 int ovl_sync_status(struct ovl_fs *ofs);
 
 static inline void ovl_set_flag(unsigned long flag, struct inode *inode)
@@ -556,7 +556,7 @@ void ovl_cleanup_whiteouts(struct ovl_fs *ofs, struct dentry *upper,
 			   struct list_head *list);
 void ovl_cache_free(struct list_head *list);
 void ovl_dir_cache_free(struct inode *inode);
-int ovl_check_d_type_supported(struct path *realpath);
+int ovl_check_d_type_supported(const struct path *realpath);
 int ovl_workdir_cleanup(struct ovl_fs *ofs, struct inode *dir,
 			struct vfsmount *mnt, struct dentry *dentry, int level);
 int ovl_indexdir_cleanup(struct ovl_fs *ofs);
@@ -673,8 +673,8 @@ struct dentry *ovl_create_temp(struct ovl_fs *ofs, struct dentry *workdir,
 extern const struct file_operations ovl_file_operations;
 int __init ovl_aio_request_cache_init(void);
 void ovl_aio_request_cache_destroy(void);
-int ovl_real_fileattr_get(struct path *realpath, struct fileattr *fa);
-int ovl_real_fileattr_set(struct path *realpath, struct fileattr *fa);
+int ovl_real_fileattr_get(const struct path *realpath, struct fileattr *fa);
+int ovl_real_fileattr_set(const struct path *realpath, struct fileattr *fa);
 int ovl_fileattr_get(struct dentry *dentry, struct fileattr *fa);
 int ovl_fileattr_set(struct user_namespace *mnt_userns,
 		     struct dentry *dentry, struct fileattr *fa);
@@ -683,7 +683,7 @@ int ovl_fileattr_set(struct user_namespace *mnt_userns,
 int ovl_copy_up(struct dentry *dentry);
 int ovl_copy_up_with_data(struct dentry *dentry);
 int ovl_maybe_copy_up(struct dentry *dentry, int flags);
-int ovl_copy_xattr(struct super_block *sb, struct path *path, struct dentry *new);
+int ovl_copy_xattr(struct super_block *sb, const struct path *path, struct dentry *new);
 int ovl_set_attr(struct ovl_fs *ofs, struct dentry *upper, struct kstat *stat);
 struct ovl_fh *ovl_encode_real_fh(struct ovl_fs *ofs, struct dentry *real,
 				  bool is_upper);
diff --git a/fs/overlayfs/readdir.c b/fs/overlayfs/readdir.c
index 78f62cc1797b..2b210640036c 100644
--- a/fs/overlayfs/readdir.c
+++ b/fs/overlayfs/readdir.c
@@ -170,7 +170,7 @@ static struct ovl_cache_entry *ovl_cache_entry_new(struct ovl_readdir_data *rdd,
 	return p;
 }
 
-static int ovl_cache_entry_add_rb(struct ovl_readdir_data *rdd,
+static bool ovl_cache_entry_add_rb(struct ovl_readdir_data *rdd,
 				  const char *name, int len, u64 ino,
 				  unsigned int d_type)
 {
@@ -179,22 +179,22 @@ static int ovl_cache_entry_add_rb(struct ovl_readdir_data *rdd,
 	struct ovl_cache_entry *p;
 
 	if (ovl_cache_entry_find_link(name, len, &newp, &parent))
-		return 0;
+		return true;
 
 	p = ovl_cache_entry_new(rdd, name, len, ino, d_type);
 	if (p == NULL) {
 		rdd->err = -ENOMEM;
-		return -ENOMEM;
+		return false;
 	}
 
 	list_add_tail(&p->l_node, rdd->list);
 	rb_link_node(&p->node, parent, newp);
 	rb_insert_color(&p->node, rdd->root);
 
-	return 0;
+	return true;
 }
 
-static int ovl_fill_lowest(struct ovl_readdir_data *rdd,
+static bool ovl_fill_lowest(struct ovl_readdir_data *rdd,
 			   const char *name, int namelen,
 			   loff_t offset, u64 ino, unsigned int d_type)
 {
@@ -211,7 +211,7 @@ static int ovl_fill_lowest(struct ovl_readdir_data *rdd,
 			list_add_tail(&p->l_node, &rdd->middle);
 	}
 
-	return rdd->err;
+	return rdd->err == 0;
 }
 
 void ovl_cache_free(struct list_head *list)
@@ -250,7 +250,7 @@ static void ovl_cache_put(struct ovl_dir_file *od, struct dentry *dentry)
 	}
 }
 
-static int ovl_fill_merge(struct dir_context *ctx, const char *name,
+static bool ovl_fill_merge(struct dir_context *ctx, const char *name,
 			  int namelen, loff_t offset, u64 ino,
 			  unsigned int d_type)
 {
@@ -264,7 +264,7 @@ static int ovl_fill_merge(struct dir_context *ctx, const char *name,
 		return ovl_fill_lowest(rdd, name, namelen, offset, ino, d_type);
 }
 
-static int ovl_check_whiteouts(struct path *path, struct ovl_readdir_data *rdd)
+static int ovl_check_whiteouts(const struct path *path, struct ovl_readdir_data *rdd)
 {
 	int err;
 	struct ovl_cache_entry *p;
@@ -291,7 +291,7 @@ static int ovl_check_whiteouts(struct path *path, struct ovl_readdir_data *rdd)
 	return err;
 }
 
-static inline int ovl_dir_read(struct path *realpath,
+static inline int ovl_dir_read(const struct path *realpath,
 			       struct ovl_readdir_data *rdd)
 {
 	struct file *realfile;
@@ -455,7 +455,7 @@ static u64 ovl_remap_lower_ino(u64 ino, int xinobits, int fsid,
  * copy up origin, call vfs_getattr() on the overlay entry to make
  * sure that d_ino will be consistent with st_ino from stat(2).
  */
-static int ovl_cache_update_ino(struct path *path, struct ovl_cache_entry *p)
+static int ovl_cache_update_ino(const struct path *path, struct ovl_cache_entry *p)
 
 {
 	struct dentry *dir = path->dentry;
@@ -528,7 +528,7 @@ fail:
 	goto out;
 }
 
-static int ovl_fill_plain(struct dir_context *ctx, const char *name,
+static bool ovl_fill_plain(struct dir_context *ctx, const char *name,
 			  int namelen, loff_t offset, u64 ino,
 			  unsigned int d_type)
 {
@@ -540,14 +540,14 @@ static int ovl_fill_plain(struct dir_context *ctx, const char *name,
 	p = ovl_cache_entry_new(rdd, name, namelen, ino, d_type);
 	if (p == NULL) {
 		rdd->err = -ENOMEM;
-		return -ENOMEM;
+		return false;
 	}
 	list_add_tail(&p->l_node, rdd->list);
 
-	return 0;
+	return true;
 }
 
-static int ovl_dir_read_impure(struct path *path,  struct list_head *list,
+static int ovl_dir_read_impure(const struct path *path,  struct list_head *list,
 			       struct rb_root *root)
 {
 	int err;
@@ -592,7 +592,7 @@ static int ovl_dir_read_impure(struct path *path,  struct list_head *list,
 	return 0;
 }
 
-static struct ovl_dir_cache *ovl_cache_get_impure(struct path *path)
+static struct ovl_dir_cache *ovl_cache_get_impure(const struct path *path)
 {
 	int res;
 	struct dentry *dentry = path->dentry;
@@ -648,7 +648,7 @@ struct ovl_readdir_translate {
 	bool xinowarn;
 };
 
-static int ovl_fill_real(struct dir_context *ctx, const char *name,
+static bool ovl_fill_real(struct dir_context *ctx, const char *name,
 			   int namelen, loff_t offset, u64 ino,
 			   unsigned int d_type)
 {
@@ -834,7 +834,7 @@ out_unlock:
 }
 
 static struct file *ovl_dir_open_realfile(const struct file *file,
-					  struct path *realpath)
+					  const struct path *realpath)
 {
 	struct file *res;
 	const struct cred *old_cred;
@@ -1027,7 +1027,7 @@ void ovl_cleanup_whiteouts(struct ovl_fs *ofs, struct dentry *upper,
 	inode_unlock(upper->d_inode);
 }
 
-static int ovl_check_d_type(struct dir_context *ctx, const char *name,
+static bool ovl_check_d_type(struct dir_context *ctx, const char *name,
 			  int namelen, loff_t offset, u64 ino,
 			  unsigned int d_type)
 {
@@ -1036,19 +1036,19 @@ static int ovl_check_d_type(struct dir_context *ctx, const char *name,
 
 	/* Even if d_type is not supported, DT_DIR is returned for . and .. */
 	if (!strncmp(name, ".", namelen) || !strncmp(name, "..", namelen))
-		return 0;
+		return true;
 
 	if (d_type != DT_UNKNOWN)
 		rdd->d_type_supported = true;
 
-	return 0;
+	return true;
 }
 
 /*
  * Returns 1 if d_type is supported, 0 not supported/unknown. Negative values
  * if error is encountered.
  */
-int ovl_check_d_type_supported(struct path *realpath)
+int ovl_check_d_type_supported(const struct path *realpath)
 {
 	int err;
 	struct ovl_readdir_data rdd = {
@@ -1065,7 +1065,7 @@ int ovl_check_d_type_supported(struct path *realpath)
 
 #define OVL_INCOMPATDIR_NAME "incompat"
 
-static int ovl_workdir_cleanup_recurse(struct ovl_fs *ofs, struct path *path,
+static int ovl_workdir_cleanup_recurse(struct ovl_fs *ofs, const struct path *path,
 				       int level)
 {
 	int err;
diff --git a/fs/overlayfs/super.c b/fs/overlayfs/super.c
index ec746d447f1b..9ca98bea8e18 100644
--- a/fs/overlayfs/super.c
+++ b/fs/overlayfs/super.c
@@ -908,7 +908,7 @@ static int ovl_mount_dir(const char *name, struct path *path)
 	return err;
 }
 
-static int ovl_check_namelen(struct path *path, struct ovl_fs *ofs,
+static int ovl_check_namelen(const struct path *path, struct ovl_fs *ofs,
 			     const char *name)
 {
 	struct kstatfs statfs;
@@ -1022,7 +1022,20 @@ ovl_posix_acl_xattr_set(const struct xattr_handler *handler,
 
 	/* Check that everything is OK before copy-up */
 	if (value) {
-		acl = posix_acl_from_xattr(&init_user_ns, value, size);
+		/* The above comment can be understood in two ways:
+		 *
+		 * 1. We just want to check whether the basic POSIX ACL format
+		 *    is ok. For example, if the header is correct and the size
+		 *    is sane.
+		 * 2. We want to know whether the ACL_{GROUP,USER} entries can
+		 *    be mapped according to the underlying filesystem.
+		 *
+		 * Currently, we only check 1. If we wanted to check 2. we
+		 * would need to pass the mnt_userns and the fs_userns of the
+		 * underlying filesystem. But frankly, I think checking 1. is
+		 * enough to start the copy-up.
+		 */
+		acl = vfs_set_acl_prepare(&init_user_ns, &init_user_ns, value, size);
 		if (IS_ERR(acl))
 			return PTR_ERR(acl);
 	}
@@ -1353,7 +1366,7 @@ static int ovl_create_volatile_dirty(struct ovl_fs *ofs)
 }
 
 static int ovl_make_workdir(struct super_block *sb, struct ovl_fs *ofs,
-			    struct path *workpath)
+			    const struct path *workpath)
 {
 	struct vfsmount *mnt = ovl_upper_mnt(ofs);
 	struct dentry *temp, *workdir;
@@ -1482,7 +1495,7 @@ out:
 }
 
 static int ovl_get_workdir(struct super_block *sb, struct ovl_fs *ofs,
-			   struct path *upperpath)
+			   const struct path *upperpath)
 {
 	int err;
 	struct path workpath = { };
@@ -1525,7 +1538,7 @@ out:
 }
 
 static int ovl_get_indexdir(struct super_block *sb, struct ovl_fs *ofs,
-			    struct ovl_entry *oe, struct path *upperpath)
+			    struct ovl_entry *oe, const struct path *upperpath)
 {
 	struct vfsmount *mnt = ovl_upper_mnt(ofs);
 	struct dentry *indexdir;
diff --git a/fs/overlayfs/util.c b/fs/overlayfs/util.c
index 87f811c089e4..81a57a8d80d9 100644
--- a/fs/overlayfs/util.c
+++ b/fs/overlayfs/util.c
@@ -490,7 +490,7 @@ bool ovl_is_whiteout(struct dentry *dentry)
 	return inode && IS_WHITEOUT(inode);
 }
 
-struct file *ovl_path_open(struct path *path, int flags)
+struct file *ovl_path_open(const struct path *path, int flags)
 {
 	struct inode *inode = d_inode(path->dentry);
 	struct user_namespace *real_mnt_userns = mnt_user_ns(path->mnt);
@@ -578,7 +578,7 @@ void ovl_copy_up_end(struct dentry *dentry)
 	ovl_inode_unlock(d_inode(dentry));
 }
 
-bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, struct path *path)
+bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, const struct path *path)
 {
 	int res;
 
@@ -591,7 +591,7 @@ bool ovl_path_check_origin_xattr(struct ovl_fs *ofs, struct path *path)
 	return false;
 }
 
-bool ovl_path_check_dir_xattr(struct ovl_fs *ofs, struct path *path,
+bool ovl_path_check_dir_xattr(struct ovl_fs *ofs, const struct path *path,
 			       enum ovl_xattr ox)
 {
 	int res;
@@ -971,7 +971,7 @@ err:
 }
 
 /* err < 0, 0 if no metacopy xattr, 1 if metacopy xattr found */
-int ovl_check_metacopy_xattr(struct ovl_fs *ofs, struct path *path)
+int ovl_check_metacopy_xattr(struct ovl_fs *ofs, const struct path *path)
 {
 	int res;
 
@@ -1015,7 +1015,7 @@ bool ovl_is_metacopy_dentry(struct dentry *dentry)
 	return (oe->numlower > 1);
 }
 
-char *ovl_get_redirect_xattr(struct ovl_fs *ofs, struct path *path, int padding)
+char *ovl_get_redirect_xattr(struct ovl_fs *ofs, const struct path *path, int padding)
 {
 	int res;
 	char *s, *next, *buf = NULL;
diff --git a/fs/pipe.c b/fs/pipe.c
index 74ae9fafd25a..42c7ff41c2db 100644
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -860,7 +860,7 @@ static struct vfsmount *pipe_mnt __read_mostly;
  */
 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
 {
-	return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]",
+	return dynamic_dname(buffer, buflen, "pipe:[%lu]",
 				d_inode(dentry)->i_ino);
 }
 
diff --git a/fs/posix_acl.c b/fs/posix_acl.c
index 5af33800743e..b4f109875e79 100644
--- a/fs/posix_acl.c
+++ b/fs/posix_acl.c
@@ -710,9 +710,9 @@ EXPORT_SYMBOL(posix_acl_update_mode);
 /*
  * Fix up the uids and gids in posix acl extended attributes in place.
  */
-static int posix_acl_fix_xattr_common(void *value, size_t size)
+static int posix_acl_fix_xattr_common(const void *value, size_t size)
 {
-	struct posix_acl_xattr_header *header = value;
+	const struct posix_acl_xattr_header *header = value;
 	int count;
 
 	if (!header)
@@ -720,13 +720,13 @@ static int posix_acl_fix_xattr_common(void *value, size_t size)
 	if (size < sizeof(struct posix_acl_xattr_header))
 		return -EINVAL;
 	if (header->a_version != cpu_to_le32(POSIX_ACL_XATTR_VERSION))
-		return -EINVAL;
+		return -EOPNOTSUPP;
 
 	count = posix_acl_xattr_count(size);
 	if (count < 0)
 		return -EINVAL;
 	if (count == 0)
-		return -EINVAL;
+		return 0;
 
 	return count;
 }
@@ -748,7 +748,7 @@ void posix_acl_getxattr_idmapped_mnt(struct user_namespace *mnt_userns,
 		return;
 
 	count = posix_acl_fix_xattr_common(value, size);
-	if (count < 0)
+	if (count <= 0)
 		return;
 
 	for (end = entry + count; entry != end; entry++) {
@@ -771,46 +771,6 @@ void posix_acl_getxattr_idmapped_mnt(struct user_namespace *mnt_userns,
 	}
 }
 
-void posix_acl_setxattr_idmapped_mnt(struct user_namespace *mnt_userns,
-				     const struct inode *inode,
-				     void *value, size_t size)
-{
-	struct posix_acl_xattr_header *header = value;
-	struct posix_acl_xattr_entry *entry = (void *)(header + 1), *end;
-	struct user_namespace *fs_userns = i_user_ns(inode);
-	int count;
-	vfsuid_t vfsuid;
-	vfsgid_t vfsgid;
-	kuid_t uid;
-	kgid_t gid;
-
-	if (no_idmapping(mnt_userns, i_user_ns(inode)))
-		return;
-
-	count = posix_acl_fix_xattr_common(value, size);
-	if (count < 0)
-		return;
-
-	for (end = entry + count; entry != end; entry++) {
-		switch (le16_to_cpu(entry->e_tag)) {
-		case ACL_USER:
-			uid = make_kuid(&init_user_ns, le32_to_cpu(entry->e_id));
-			vfsuid = VFSUIDT_INIT(uid);
-			uid = from_vfsuid(mnt_userns, fs_userns, vfsuid);
-			entry->e_id = cpu_to_le32(from_kuid(&init_user_ns, uid));
-			break;
-		case ACL_GROUP:
-			gid = make_kgid(&init_user_ns, le32_to_cpu(entry->e_id));
-			vfsgid = VFSGIDT_INIT(gid);
-			gid = from_vfsgid(mnt_userns, fs_userns, vfsgid);
-			entry->e_id = cpu_to_le32(from_kgid(&init_user_ns, gid));
-			break;
-		default:
-			break;
-		}
-	}
-}
-
 static void posix_acl_fix_xattr_userns(
 	struct user_namespace *to, struct user_namespace *from,
 	void *value, size_t size)
@@ -822,7 +782,7 @@ static void posix_acl_fix_xattr_userns(
 	kgid_t gid;
 
 	count = posix_acl_fix_xattr_common(value, size);
-	if (count < 0)
+	if (count <= 0)
 		return;
 
 	for (end = entry + count; entry != end; entry++) {
@@ -857,12 +817,32 @@ void posix_acl_fix_xattr_to_user(void *value, size_t size)
 	posix_acl_fix_xattr_userns(user_ns, &init_user_ns, value, size);
 }
 
-/*
- * Convert from extended attribute to in-memory representation.
+/**
+ * make_posix_acl - convert POSIX ACLs from uapi to VFS format using the
+ *                  provided callbacks to map ACL_{GROUP,USER} entries into the
+ *                  appropriate format
+ * @mnt_userns: the mount's idmapping
+ * @fs_userns: the filesystem's idmapping
+ * @value: the uapi representation of POSIX ACLs
+ * @size: the size of @void
+ * @uid_cb: callback to use for mapping the uid stored in ACL_USER entries
+ * @gid_cb: callback to use for mapping the gid stored in ACL_GROUP entries
+ *
+ * The make_posix_acl() helper is an abstraction to translate from uapi format
+ * into the VFS format allowing the caller to specific callbacks to map
+ * ACL_{GROUP,USER} entries into the expected format. This is used in
+ * posix_acl_from_xattr() and vfs_set_acl_prepare() and avoids pointless code
+ * duplication.
+ *
+ * Return: Allocated struct posix_acl on success, NULL for a valid header but
+ *         without actual POSIX ACL entries, or ERR_PTR() encoded error code.
  */
-struct posix_acl *
-posix_acl_from_xattr(struct user_namespace *user_ns,
-		     const void *value, size_t size)
+static struct posix_acl *make_posix_acl(struct user_namespace *mnt_userns,
+	struct user_namespace *fs_userns, const void *value, size_t size,
+	kuid_t (*uid_cb)(struct user_namespace *, struct user_namespace *,
+			 const struct posix_acl_xattr_entry *),
+	kgid_t (*gid_cb)(struct user_namespace *, struct user_namespace *,
+			 const struct posix_acl_xattr_entry *))
 {
 	const struct posix_acl_xattr_header *header = value;
 	const struct posix_acl_xattr_entry *entry = (const void *)(header + 1), *end;
@@ -870,16 +850,9 @@ posix_acl_from_xattr(struct user_namespace *user_ns,
 	struct posix_acl *acl;
 	struct posix_acl_entry *acl_e;
 
-	if (!value)
-		return NULL;
-	if (size < sizeof(struct posix_acl_xattr_header))
-		 return ERR_PTR(-EINVAL);
-	if (header->a_version != cpu_to_le32(POSIX_ACL_XATTR_VERSION))
-		return ERR_PTR(-EOPNOTSUPP);
-
-	count = posix_acl_xattr_count(size);
+	count = posix_acl_fix_xattr_common(value, size);
 	if (count < 0)
-		return ERR_PTR(-EINVAL);
+		return ERR_PTR(count);
 	if (count == 0)
 		return NULL;
 	
@@ -900,16 +873,12 @@ posix_acl_from_xattr(struct user_namespace *user_ns,
 				break;
 
 			case ACL_USER:
-				acl_e->e_uid =
-					make_kuid(user_ns,
-						  le32_to_cpu(entry->e_id));
+				acl_e->e_uid = uid_cb(mnt_userns, fs_userns, entry);
 				if (!uid_valid(acl_e->e_uid))
 					goto fail;
 				break;
 			case ACL_GROUP:
-				acl_e->e_gid =
-					make_kgid(user_ns,
-						  le32_to_cpu(entry->e_id));
+				acl_e->e_gid = gid_cb(mnt_userns, fs_userns, entry);
 				if (!gid_valid(acl_e->e_gid))
 					goto fail;
 				break;
@@ -924,6 +893,181 @@ fail:
 	posix_acl_release(acl);
 	return ERR_PTR(-EINVAL);
 }
+
+/**
+ * vfs_set_acl_prepare_kuid - map ACL_USER uid according to mount- and
+ *                            filesystem idmapping
+ * @mnt_userns: the mount's idmapping
+ * @fs_userns: the filesystem's idmapping
+ * @e: a ACL_USER entry in POSIX ACL uapi format
+ *
+ * The uid stored as ACL_USER entry in @e is a kuid_t stored as a raw {g,u}id
+ * value. The vfs_set_acl_prepare_kuid() will recover the kuid_t through
+ * KUIDT_INIT() and then map it according to the idmapped mount. The resulting
+ * kuid_t is the value which the filesystem can map up into a raw backing store
+ * id in the filesystem's idmapping.
+ *
+ * This is used in vfs_set_acl_prepare() to generate the proper VFS
+ * representation of POSIX ACLs with ACL_USER entries during setxattr().
+ *
+ * Return: A kuid in @fs_userns for the uid stored in @e.
+ */
+static inline kuid_t
+vfs_set_acl_prepare_kuid(struct user_namespace *mnt_userns,
+			 struct user_namespace *fs_userns,
+			 const struct posix_acl_xattr_entry *e)
+{
+	kuid_t kuid = KUIDT_INIT(le32_to_cpu(e->e_id));
+	return from_vfsuid(mnt_userns, fs_userns, VFSUIDT_INIT(kuid));
+}
+
+/**
+ * vfs_set_acl_prepare_kgid - map ACL_GROUP gid according to mount- and
+ *                            filesystem idmapping
+ * @mnt_userns: the mount's idmapping
+ * @fs_userns: the filesystem's idmapping
+ * @e: a ACL_GROUP entry in POSIX ACL uapi format
+ *
+ * The gid stored as ACL_GROUP entry in @e is a kgid_t stored as a raw {g,u}id
+ * value. The vfs_set_acl_prepare_kgid() will recover the kgid_t through
+ * KGIDT_INIT() and then map it according to the idmapped mount. The resulting
+ * kgid_t is the value which the filesystem can map up into a raw backing store
+ * id in the filesystem's idmapping.
+ *
+ * This is used in vfs_set_acl_prepare() to generate the proper VFS
+ * representation of POSIX ACLs with ACL_GROUP entries during setxattr().
+ *
+ * Return: A kgid in @fs_userns for the gid stored in @e.
+ */
+static inline kgid_t
+vfs_set_acl_prepare_kgid(struct user_namespace *mnt_userns,
+			 struct user_namespace *fs_userns,
+			 const struct posix_acl_xattr_entry *e)
+{
+	kgid_t kgid = KGIDT_INIT(le32_to_cpu(e->e_id));
+	return from_vfsgid(mnt_userns, fs_userns, VFSGIDT_INIT(kgid));
+}
+
+/**
+ * vfs_set_acl_prepare - convert POSIX ACLs from uapi to VFS format taking
+ *                       mount and filesystem idmappings into account
+ * @mnt_userns: the mount's idmapping
+ * @fs_userns: the filesystem's idmapping
+ * @value: the uapi representation of POSIX ACLs
+ * @size: the size of @void
+ *
+ * When setting POSIX ACLs with ACL_{GROUP,USER} entries they need to be
+ * mapped according to the relevant mount- and filesystem idmapping. It is
+ * important that the ACL_{GROUP,USER} entries in struct posix_acl will be
+ * mapped into k{g,u}id_t that are supposed to be mapped up in the filesystem
+ * idmapping. This is crucial since the resulting struct posix_acl might be
+ * cached filesystem wide. The vfs_set_acl_prepare() function will take care to
+ * perform all necessary idmappings.
+ *
+ * Note, that since basically forever the {g,u}id values encoded as
+ * ACL_{GROUP,USER} entries in the uapi POSIX ACLs passed via @value contain
+ * values that have been mapped according to the caller's idmapping. In other
+ * words, POSIX ACLs passed in uapi format as @value during setxattr() contain
+ * {g,u}id values in their ACL_{GROUP,USER} entries that should actually have
+ * been stored as k{g,u}id_t.
+ *
+ * This means, vfs_set_acl_prepare() needs to first recover the k{g,u}id_t by
+ * calling K{G,U}IDT_INIT(). Afterwards they can be interpreted as vfs{g,u}id_t
+ * through from_vfs{g,u}id() to account for any idmapped mounts. The
+ * vfs_set_acl_prepare_k{g,u}id() helpers will take care to generate the
+ * correct k{g,u}id_t.
+ *
+ * The filesystem will then receive the POSIX ACLs ready to be cached
+ * filesystem wide and ready to be written to the backing store taking the
+ * filesystem's idmapping into account.
+ *
+ * Return: Allocated struct posix_acl on success, NULL for a valid header but
+ *         without actual POSIX ACL entries, or ERR_PTR() encoded error code.
+ */
+struct posix_acl *vfs_set_acl_prepare(struct user_namespace *mnt_userns,
+				      struct user_namespace *fs_userns,
+				      const void *value, size_t size)
+{
+	return make_posix_acl(mnt_userns, fs_userns, value, size,
+			      vfs_set_acl_prepare_kuid,
+			      vfs_set_acl_prepare_kgid);
+}
+EXPORT_SYMBOL(vfs_set_acl_prepare);
+
+/**
+ * posix_acl_from_xattr_kuid - map ACL_USER uid into filesystem idmapping
+ * @mnt_userns: unused
+ * @fs_userns: the filesystem's idmapping
+ * @e: a ACL_USER entry in POSIX ACL uapi format
+ *
+ * Map the uid stored as ACL_USER entry in @e into the filesystem's idmapping.
+ * This is used in posix_acl_from_xattr() to generate the proper VFS
+ * representation of POSIX ACLs with ACL_USER entries.
+ *
+ * Return: A kuid in @fs_userns for the uid stored in @e.
+ */
+static inline kuid_t
+posix_acl_from_xattr_kuid(struct user_namespace *mnt_userns,
+			  struct user_namespace *fs_userns,
+			  const struct posix_acl_xattr_entry *e)
+{
+	return make_kuid(fs_userns, le32_to_cpu(e->e_id));
+}
+
+/**
+ * posix_acl_from_xattr_kgid - map ACL_GROUP gid into filesystem idmapping
+ * @mnt_userns: unused
+ * @fs_userns: the filesystem's idmapping
+ * @e: a ACL_GROUP entry in POSIX ACL uapi format
+ *
+ * Map the gid stored as ACL_GROUP entry in @e into the filesystem's idmapping.
+ * This is used in posix_acl_from_xattr() to generate the proper VFS
+ * representation of POSIX ACLs with ACL_GROUP entries.
+ *
+ * Return: A kgid in @fs_userns for the gid stored in @e.
+ */
+static inline kgid_t
+posix_acl_from_xattr_kgid(struct user_namespace *mnt_userns,
+			  struct user_namespace *fs_userns,
+			  const struct posix_acl_xattr_entry *e)
+{
+	return make_kgid(fs_userns, le32_to_cpu(e->e_id));
+}
+
+/**
+ * posix_acl_from_xattr - convert POSIX ACLs from backing store to VFS format
+ * @fs_userns: the filesystem's idmapping
+ * @value: the uapi representation of POSIX ACLs
+ * @size: the size of @void
+ *
+ * Filesystems that store POSIX ACLs in the unaltered uapi format should use
+ * posix_acl_from_xattr() when reading them from the backing store and
+ * converting them into the struct posix_acl VFS format. The helper is
+ * specifically intended to be called from the ->get_acl() inode operation.
+ *
+ * The posix_acl_from_xattr() function will map the raw {g,u}id values stored
+ * in ACL_{GROUP,USER} entries into the filesystem idmapping in @fs_userns. The
+ * posix_acl_from_xattr_k{g,u}id() helpers will take care to generate the
+ * correct k{g,u}id_t. The returned struct posix_acl can be cached.
+ *
+ * Note that posix_acl_from_xattr() does not take idmapped mounts into account.
+ * If it did it calling is from the ->get_acl() inode operation would return
+ * POSIX ACLs mapped according to an idmapped mount which would mean that the
+ * value couldn't be cached for the filesystem. Idmapped mounts are taken into
+ * account on the fly during permission checking or right at the VFS -
+ * userspace boundary before reporting them to the user.
+ *
+ * Return: Allocated struct posix_acl on success, NULL for a valid header but
+ *         without actual POSIX ACL entries, or ERR_PTR() encoded error code.
+ */
+struct posix_acl *
+posix_acl_from_xattr(struct user_namespace *fs_userns,
+		     const void *value, size_t size)
+{
+	return make_posix_acl(&init_user_ns, fs_userns, value, size,
+			      posix_acl_from_xattr_kuid,
+			      posix_acl_from_xattr_kgid);
+}
 EXPORT_SYMBOL (posix_acl_from_xattr);
 
 /*
@@ -1027,7 +1171,17 @@ posix_acl_xattr_set(const struct xattr_handler *handler,
 	int ret;
 
 	if (value) {
-		acl = posix_acl_from_xattr(&init_user_ns, value, size);
+		/*
+		 * By the time we end up here the {g,u}ids stored in
+		 * ACL_{GROUP,USER} have already been mapped according to the
+		 * caller's idmapping. The vfs_set_acl_prepare() helper will
+		 * recover them and take idmapped mounts into account. The
+		 * filesystem will receive the POSIX ACLs in the correct
+		 * format ready to be cached or written to the backing store
+		 * taking the filesystem idmapping into account.
+		 */
+		acl = vfs_set_acl_prepare(mnt_userns, i_user_ns(inode),
+					  value, size);
 		if (IS_ERR(acl))
 			return PTR_ERR(acl);
 	}
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 93f7e3d971e4..2d9429bf51fa 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -1761,7 +1761,7 @@ out:
 	return ERR_PTR(error);
 }
 
-static int do_proc_readlink(struct path *path, char __user *buffer, int buflen)
+static int do_proc_readlink(const struct path *path, char __user *buffer, int buflen)
 {
 	char *tmp = kmalloc(PATH_MAX, GFP_KERNEL);
 	char *pathname;
@@ -2728,7 +2728,7 @@ static ssize_t proc_pid_attr_read(struct file * file, char __user * buf,
 		return -ESRCH;
 
 	length = security_getprocattr(task, PROC_I(inode)->op.lsm,
-				      (char*)file->f_path.dentry->d_name.name,
+				      file->f_path.dentry->d_name.name,
 				      &p);
 	put_task_struct(task);
 	if (length > 0)
diff --git a/fs/pstore/platform.c b/fs/pstore/platform.c
index b2fd3c20e7c2..0c034ea39954 100644
--- a/fs/pstore/platform.c
+++ b/fs/pstore/platform.c
@@ -28,14 +28,11 @@
 #include <linux/crypto.h>
 #include <linux/string.h>
 #include <linux/timer.h>
-#include <linux/scatterlist.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/jiffies.h>
 #include <linux/workqueue.h>
 
-#include <crypto/acompress.h>
-
 #include "internal.h"
 
 /*
@@ -93,8 +90,7 @@ module_param(compress, charp, 0444);
 MODULE_PARM_DESC(compress, "compression to use");
 
 /* Compression parameters */
-static struct crypto_acomp *tfm;
-static struct acomp_req *creq;
+static struct crypto_comp *tfm;
 
 struct pstore_zbackend {
 	int (*zbufsize)(size_t size);
@@ -272,21 +268,12 @@ static const struct pstore_zbackend zbackends[] = {
 static int pstore_compress(const void *in, void *out,
 			   unsigned int inlen, unsigned int outlen)
 {
-	struct scatterlist src, dst;
 	int ret;
 
 	if (!IS_ENABLED(CONFIG_PSTORE_COMPRESS))
 		return -EINVAL;
 
-	sg_init_table(&src, 1);
-	sg_set_buf(&src, in, inlen);
-
-	sg_init_table(&dst, 1);
-	sg_set_buf(&dst, out, outlen);
-
-	acomp_request_set_params(creq, &src, &dst, inlen, outlen);
-
-	ret = crypto_acomp_compress(creq);
+	ret = crypto_comp_compress(tfm, in, inlen, out, &outlen);
 	if (ret) {
 		pr_err("crypto_comp_compress failed, ret = %d!\n", ret);
 		return ret;
@@ -297,7 +284,7 @@ static int pstore_compress(const void *in, void *out,
 
 static void allocate_buf_for_compression(void)
 {
-	struct crypto_acomp *acomp;
+	struct crypto_comp *ctx;
 	int size;
 	char *buf;
 
@@ -309,7 +296,7 @@ static void allocate_buf_for_compression(void)
 	if (!psinfo || tfm)
 		return;
 
-	if (!crypto_has_acomp(zbackend->name, 0, CRYPTO_ALG_ASYNC)) {
+	if (!crypto_has_comp(zbackend->name, 0, 0)) {
 		pr_err("Unknown compression: %s\n", zbackend->name);
 		return;
 	}
@@ -328,24 +315,16 @@ static void allocate_buf_for_compression(void)
 		return;
 	}
 
-	acomp = crypto_alloc_acomp(zbackend->name, 0, CRYPTO_ALG_ASYNC);
-	if (IS_ERR_OR_NULL(acomp)) {
+	ctx = crypto_alloc_comp(zbackend->name, 0, 0);
+	if (IS_ERR_OR_NULL(ctx)) {
 		kfree(buf);
 		pr_err("crypto_alloc_comp('%s') failed: %ld\n", zbackend->name,
-		       PTR_ERR(acomp));
-		return;
-	}
-
-	creq = acomp_request_alloc(acomp);
-	if (!creq) {
-		crypto_free_acomp(acomp);
-		kfree(buf);
-		pr_err("acomp_request_alloc('%s') failed\n", zbackend->name);
+		       PTR_ERR(ctx));
 		return;
 	}
 
 	/* A non-NULL big_oops_buf indicates compression is available. */
-	tfm = acomp;
+	tfm = ctx;
 	big_oops_buf_sz = size;
 	big_oops_buf = buf;
 
@@ -355,8 +334,7 @@ static void allocate_buf_for_compression(void)
 static void free_buf_for_compression(void)
 {
 	if (IS_ENABLED(CONFIG_PSTORE_COMPRESS) && tfm) {
-		acomp_request_free(creq);
-		crypto_free_acomp(tfm);
+		crypto_free_comp(tfm);
 		tfm = NULL;
 	}
 	kfree(big_oops_buf);
@@ -693,8 +671,6 @@ static void decompress_record(struct pstore_record *record)
 	int ret;
 	int unzipped_len;
 	char *unzipped, *workspace;
-	struct acomp_req *dreq;
-	struct scatterlist src, dst;
 
 	if (!IS_ENABLED(CONFIG_PSTORE_COMPRESS) || !record->compressed)
 		return;
@@ -718,30 +694,16 @@ static void decompress_record(struct pstore_record *record)
 	if (!workspace)
 		return;
 
-	dreq = acomp_request_alloc(tfm);
-	if (!dreq) {
-		kfree(workspace);
-		return;
-	}
-
-	sg_init_table(&src, 1);
-	sg_set_buf(&src, record->buf, record->size);
-
-	sg_init_table(&dst, 1);
-	sg_set_buf(&dst, workspace, unzipped_len);
-
-	acomp_request_set_params(dreq, &src, &dst, record->size, unzipped_len);
-
 	/* After decompression "unzipped_len" is almost certainly smaller. */
-	ret = crypto_acomp_decompress(dreq);
+	ret = crypto_comp_decompress(tfm, record->buf, record->size,
+					  workspace, &unzipped_len);
 	if (ret) {
-		pr_err("crypto_acomp_decompress failed, ret = %d!\n", ret);
+		pr_err("crypto_comp_decompress failed, ret = %d!\n", ret);
 		kfree(workspace);
 		return;
 	}
 
 	/* Append ECC notice to decompressed buffer. */
-	unzipped_len = dreq->dlen;
 	memcpy(workspace + unzipped_len, record->buf + record->size,
 	       record->ecc_notice_size);
 
@@ -749,7 +711,6 @@ static void decompress_record(struct pstore_record *record)
 	unzipped = kmemdup(workspace, unzipped_len + record->ecc_notice_size,
 			   GFP_KERNEL);
 	kfree(workspace);
-	acomp_request_free(dreq);
 	if (!unzipped)
 		return;
 
diff --git a/fs/read_write.c b/fs/read_write.c
index 1a261dcf1778..328ce8cf9a85 100644
--- a/fs/read_write.c
+++ b/fs/read_write.c
@@ -496,14 +496,9 @@ static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t
 }
 
 /* caller is responsible for file_start_write/file_end_write */
-ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
+ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos)
 {
-	struct kvec iov = {
-		.iov_base	= (void *)buf,
-		.iov_len	= min_t(size_t, count, MAX_RW_COUNT),
-	};
 	struct kiocb kiocb;
-	struct iov_iter iter;
 	ssize_t ret;
 
 	if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE)))
@@ -519,8 +514,7 @@ ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t
 
 	init_sync_kiocb(&kiocb, file);
 	kiocb.ki_pos = pos ? *pos : 0;
-	iov_iter_kvec(&iter, WRITE, &iov, 1, iov.iov_len);
-	ret = file->f_op->write_iter(&kiocb, &iter);
+	ret = file->f_op->write_iter(&kiocb, from);
 	if (ret > 0) {
 		if (pos)
 			*pos = kiocb.ki_pos;
@@ -530,6 +524,18 @@ ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t
 	inc_syscw(current);
 	return ret;
 }
+
+/* caller is responsible for file_start_write/file_end_write */
+ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
+{
+	struct kvec iov = {
+		.iov_base	= (void *)buf,
+		.iov_len	= min_t(size_t, count, MAX_RW_COUNT),
+	};
+	struct iov_iter iter;
+	iov_iter_kvec(&iter, WRITE, &iov, 1, iov.iov_len);
+	return __kernel_write_iter(file, &iter, pos);
+}
 /*
  * This "EXPORT_SYMBOL_GPL()" is more of a "EXPORT_SYMBOL_DONTUSE()",
  * but autofs is one of the few internal kernel users that actually
diff --git a/fs/readdir.c b/fs/readdir.c
index 09e8ed7d4161..9c53edb60c03 100644
--- a/fs/readdir.c
+++ b/fs/readdir.c
@@ -140,7 +140,7 @@ struct readdir_callback {
 	int result;
 };
 
-static int fillonedir(struct dir_context *ctx, const char *name, int namlen,
+static bool fillonedir(struct dir_context *ctx, const char *name, int namlen,
 		      loff_t offset, u64 ino, unsigned int d_type)
 {
 	struct readdir_callback *buf =
@@ -149,14 +149,14 @@ static int fillonedir(struct dir_context *ctx, const char *name, int namlen,
 	unsigned long d_ino;
 
 	if (buf->result)
-		return -EINVAL;
+		return false;
 	buf->result = verify_dirent_name(name, namlen);
-	if (buf->result < 0)
-		return buf->result;
+	if (buf->result)
+		return false;
 	d_ino = ino;
 	if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) {
 		buf->result = -EOVERFLOW;
-		return -EOVERFLOW;
+		return false;
 	}
 	buf->result++;
 	dirent = buf->dirent;
@@ -169,12 +169,12 @@ static int fillonedir(struct dir_context *ctx, const char *name, int namlen,
 	unsafe_put_user(namlen, &dirent->d_namlen, efault_end);
 	unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault_end);
 	user_write_access_end();
-	return 0;
+	return true;
 efault_end:
 	user_write_access_end();
 efault:
 	buf->result = -EFAULT;
-	return -EFAULT;
+	return false;
 }
 
 SYSCALL_DEFINE3(old_readdir, unsigned int, fd,
@@ -219,7 +219,7 @@ struct getdents_callback {
 	int error;
 };
 
-static int filldir(struct dir_context *ctx, const char *name, int namlen,
+static bool filldir(struct dir_context *ctx, const char *name, int namlen,
 		   loff_t offset, u64 ino, unsigned int d_type)
 {
 	struct linux_dirent __user *dirent, *prev;
@@ -232,18 +232,18 @@ static int filldir(struct dir_context *ctx, const char *name, int namlen,
 
 	buf->error = verify_dirent_name(name, namlen);
 	if (unlikely(buf->error))
-		return buf->error;
+		return false;
 	buf->error = -EINVAL;	/* only used if we fail.. */
 	if (reclen > buf->count)
-		return -EINVAL;
+		return false;
 	d_ino = ino;
 	if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) {
 		buf->error = -EOVERFLOW;
-		return -EOVERFLOW;
+		return false;
 	}
 	prev_reclen = buf->prev_reclen;
 	if (prev_reclen && signal_pending(current))
-		return -EINTR;
+		return false;
 	dirent = buf->current_dir;
 	prev = (void __user *) dirent - prev_reclen;
 	if (!user_write_access_begin(prev, reclen + prev_reclen))
@@ -260,12 +260,12 @@ static int filldir(struct dir_context *ctx, const char *name, int namlen,
 	buf->current_dir = (void __user *)dirent + reclen;
 	buf->prev_reclen = reclen;
 	buf->count -= reclen;
-	return 0;
+	return true;
 efault_end:
 	user_write_access_end();
 efault:
 	buf->error = -EFAULT;
-	return -EFAULT;
+	return false;
 }
 
 SYSCALL_DEFINE3(getdents, unsigned int, fd,
@@ -307,7 +307,7 @@ struct getdents_callback64 {
 	int error;
 };
 
-static int filldir64(struct dir_context *ctx, const char *name, int namlen,
+static bool filldir64(struct dir_context *ctx, const char *name, int namlen,
 		     loff_t offset, u64 ino, unsigned int d_type)
 {
 	struct linux_dirent64 __user *dirent, *prev;
@@ -319,13 +319,13 @@ static int filldir64(struct dir_context *ctx, const char *name, int namlen,
 
 	buf->error = verify_dirent_name(name, namlen);
 	if (unlikely(buf->error))
-		return buf->error;
+		return false;
 	buf->error = -EINVAL;	/* only used if we fail.. */
 	if (reclen > buf->count)
-		return -EINVAL;
+		return false;
 	prev_reclen = buf->prev_reclen;
 	if (prev_reclen && signal_pending(current))
-		return -EINTR;
+		return false;
 	dirent = buf->current_dir;
 	prev = (void __user *)dirent - prev_reclen;
 	if (!user_write_access_begin(prev, reclen + prev_reclen))
@@ -342,13 +342,13 @@ static int filldir64(struct dir_context *ctx, const char *name, int namlen,
 	buf->prev_reclen = reclen;
 	buf->current_dir = (void __user *)dirent + reclen;
 	buf->count -= reclen;
-	return 0;
+	return true;
 
 efault_end:
 	user_write_access_end();
 efault:
 	buf->error = -EFAULT;
-	return -EFAULT;
+	return false;
 }
 
 SYSCALL_DEFINE3(getdents64, unsigned int, fd,
@@ -397,7 +397,7 @@ struct compat_readdir_callback {
 	int result;
 };
 
-static int compat_fillonedir(struct dir_context *ctx, const char *name,
+static bool compat_fillonedir(struct dir_context *ctx, const char *name,
 			     int namlen, loff_t offset, u64 ino,
 			     unsigned int d_type)
 {
@@ -407,14 +407,14 @@ static int compat_fillonedir(struct dir_context *ctx, const char *name,
 	compat_ulong_t d_ino;
 
 	if (buf->result)
-		return -EINVAL;
+		return false;
 	buf->result = verify_dirent_name(name, namlen);
-	if (buf->result < 0)
-		return buf->result;
+	if (buf->result)
+		return false;
 	d_ino = ino;
 	if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) {
 		buf->result = -EOVERFLOW;
-		return -EOVERFLOW;
+		return false;
 	}
 	buf->result++;
 	dirent = buf->dirent;
@@ -427,12 +427,12 @@ static int compat_fillonedir(struct dir_context *ctx, const char *name,
 	unsafe_put_user(namlen, &dirent->d_namlen, efault_end);
 	unsafe_copy_dirent_name(dirent->d_name, name, namlen, efault_end);
 	user_write_access_end();
-	return 0;
+	return true;
 efault_end:
 	user_write_access_end();
 efault:
 	buf->result = -EFAULT;
-	return -EFAULT;
+	return false;
 }
 
 COMPAT_SYSCALL_DEFINE3(old_readdir, unsigned int, fd,
@@ -471,7 +471,7 @@ struct compat_getdents_callback {
 	int error;
 };
 
-static int compat_filldir(struct dir_context *ctx, const char *name, int namlen,
+static bool compat_filldir(struct dir_context *ctx, const char *name, int namlen,
 		loff_t offset, u64 ino, unsigned int d_type)
 {
 	struct compat_linux_dirent __user *dirent, *prev;
@@ -484,18 +484,18 @@ static int compat_filldir(struct dir_context *ctx, const char *name, int namlen,
 
 	buf->error = verify_dirent_name(name, namlen);
 	if (unlikely(buf->error))
-		return buf->error;
+		return false;
 	buf->error = -EINVAL;	/* only used if we fail.. */
 	if (reclen > buf->count)
-		return -EINVAL;
+		return false;
 	d_ino = ino;
 	if (sizeof(d_ino) < sizeof(ino) && d_ino != ino) {
 		buf->error = -EOVERFLOW;
-		return -EOVERFLOW;
+		return false;
 	}
 	prev_reclen = buf->prev_reclen;
 	if (prev_reclen && signal_pending(current))
-		return -EINTR;
+		return false;
 	dirent = buf->current_dir;
 	prev = (void __user *) dirent - prev_reclen;
 	if (!user_write_access_begin(prev, reclen + prev_reclen))
@@ -511,12 +511,12 @@ static int compat_filldir(struct dir_context *ctx, const char *name, int namlen,
 	buf->prev_reclen = reclen;
 	buf->current_dir = (void __user *)dirent + reclen;
 	buf->count -= reclen;
-	return 0;
+	return true;
 efault_end:
 	user_write_access_end();
 efault:
 	buf->error = -EFAULT;
-	return -EFAULT;
+	return false;
 }
 
 COMPAT_SYSCALL_DEFINE3(getdents, unsigned int, fd,
diff --git a/fs/reiserfs/xattr.c b/fs/reiserfs/xattr.c
index 436641369283..8b2d52443f41 100644
--- a/fs/reiserfs/xattr.c
+++ b/fs/reiserfs/xattr.c
@@ -189,7 +189,7 @@ struct reiserfs_dentry_buf {
 	struct dentry *dentries[8];
 };
 
-static int
+static bool
 fill_with_dentries(struct dir_context *ctx, const char *name, int namelen,
 		   loff_t offset, u64 ino, unsigned int d_type)
 {
@@ -200,16 +200,16 @@ fill_with_dentries(struct dir_context *ctx, const char *name, int namelen,
 	WARN_ON_ONCE(!inode_is_locked(d_inode(dbuf->xadir)));
 
 	if (dbuf->count == ARRAY_SIZE(dbuf->dentries))
-		return -ENOSPC;
+		return false;
 
 	if (name[0] == '.' && (namelen < 2 ||
 			       (namelen == 2 && name[1] == '.')))
-		return 0;
+		return true;
 
 	dentry = lookup_one_len(name, dbuf->xadir, namelen);
 	if (IS_ERR(dentry)) {
 		dbuf->err = PTR_ERR(dentry);
-		return PTR_ERR(dentry);
+		return false;
 	} else if (d_really_is_negative(dentry)) {
 		/* A directory entry exists, but no file? */
 		reiserfs_error(dentry->d_sb, "xattr-20003",
@@ -218,11 +218,11 @@ fill_with_dentries(struct dir_context *ctx, const char *name, int namelen,
 			       dentry, dbuf->xadir);
 		dput(dentry);
 		dbuf->err = -EIO;
-		return -EIO;
+		return false;
 	}
 
 	dbuf->dentries[dbuf->count++] = dentry;
-	return 0;
+	return true;
 }
 
 static void
@@ -797,7 +797,7 @@ struct listxattr_buf {
 	struct dentry *dentry;
 };
 
-static int listxattr_filler(struct dir_context *ctx, const char *name,
+static bool listxattr_filler(struct dir_context *ctx, const char *name,
 			    int namelen, loff_t offset, u64 ino,
 			    unsigned int d_type)
 {
@@ -813,19 +813,19 @@ static int listxattr_filler(struct dir_context *ctx, const char *name,
 						    name);
 		if (!handler /* Unsupported xattr name */ ||
 		    (handler->list && !handler->list(b->dentry)))
-			return 0;
+			return true;
 		size = namelen + 1;
 		if (b->buf) {
 			if (b->pos + size > b->size) {
 				b->pos = -ERANGE;
-				return -ERANGE;
+				return false;
 			}
 			memcpy(b->buf + b->pos, name, namelen);
 			b->buf[b->pos + namelen] = 0;
 		}
 		b->pos += size;
 	}
-	return 0;
+	return true;
 }
 
 /*
diff --git a/fs/stat.c b/fs/stat.c
index 9ced8860e0f3..ef50573c72a2 100644
--- a/fs/stat.c
+++ b/fs/stat.c
@@ -5,6 +5,7 @@
  *  Copyright (C) 1991, 1992  Linus Torvalds
  */
 
+#include <linux/blkdev.h>
 #include <linux/export.h>
 #include <linux/mm.h>
 #include <linux/errno.h>
@@ -230,11 +231,22 @@ retry:
 		goto out;
 
 	error = vfs_getattr(&path, stat, request_mask, flags);
+
 	stat->mnt_id = real_mount(path.mnt)->mnt_id;
 	stat->result_mask |= STATX_MNT_ID;
+
 	if (path.mnt->mnt_root == path.dentry)
 		stat->attributes |= STATX_ATTR_MOUNT_ROOT;
 	stat->attributes_mask |= STATX_ATTR_MOUNT_ROOT;
+
+	/* Handle STATX_DIOALIGN for block devices. */
+	if (request_mask & STATX_DIOALIGN) {
+		struct inode *inode = d_backing_inode(path.dentry);
+
+		if (S_ISBLK(inode->i_mode))
+			bdev_statx_dioalign(inode, stat);
+	}
+
 	path_put(&path);
 	if (retry_estale(error, lookup_flags)) {
 		lookup_flags |= LOOKUP_REVAL;
@@ -611,6 +623,8 @@ cp_statx(const struct kstat *stat, struct statx __user *buffer)
 	tmp.stx_dev_major = MAJOR(stat->dev);
 	tmp.stx_dev_minor = MINOR(stat->dev);
 	tmp.stx_mnt_id = stat->mnt_id;
+	tmp.stx_dio_mem_align = stat->dio_mem_align;
+	tmp.stx_dio_offset_align = stat->dio_offset_align;
 
 	return copy_to_user(buffer, &tmp, sizeof(tmp)) ? -EFAULT : 0;
 }
diff --git a/fs/super.c b/fs/super.c
index 734ed584a946..6a82660e1adb 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -291,7 +291,6 @@ static void __put_super(struct super_block *s)
 		WARN_ON(s->s_inode_lru.node);
 		WARN_ON(!list_empty(&s->s_mounts));
 		security_sb_free(s);
-		fscrypt_sb_free(s);
 		put_user_ns(s->s_user_ns);
 		kfree(s->s_subtype);
 		call_rcu(&s->rcu, destroy_super_rcu);
@@ -480,6 +479,7 @@ void generic_shutdown_super(struct super_block *sb)
 		evict_inodes(sb);
 		/* only nonzero refcount inodes can have marks */
 		fsnotify_sb_delete(sb);
+		fscrypt_sb_delete(sb);
 		security_sb_delete(sb);
 
 		if (sb->s_dio_done_wq) {
diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 175de70e3adf..0c1d33c4f74c 100644
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -991,7 +991,7 @@ static int resolve_userfault_fork(struct userfaultfd_ctx *new,
 	int fd;
 
 	fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, new,
-			O_RDWR | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);
+			O_RDONLY | (new->flags & UFFD_SHARED_FCNTL_FLAGS), inode);
 	if (fd < 0)
 		return fd;
 
@@ -2094,7 +2094,7 @@ SYSCALL_DEFINE1(userfaultfd, int, flags)
 	mmgrab(ctx->mm);
 
 	fd = anon_inode_getfd_secure("[userfaultfd]", &userfaultfd_fops, ctx,
-			O_RDWR | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL);
+			O_RDONLY | (flags & UFFD_SHARED_FCNTL_FLAGS), NULL);
 	if (fd < 0) {
 		mmdrop(ctx->mm);
 		kmem_cache_free(userfaultfd_ctx_cachep, ctx);
diff --git a/fs/verity/read_metadata.c b/fs/verity/read_metadata.c
index 6ee849dc7bc1..2aefc5565152 100644
--- a/fs/verity/read_metadata.c
+++ b/fs/verity/read_metadata.c
@@ -53,14 +53,14 @@ static int fsverity_read_merkle_tree(struct inode *inode,
 			break;
 		}
 
-		virt = kmap(page);
+		virt = kmap_local_page(page);
 		if (copy_to_user(buf, virt + offs_in_page, bytes_to_copy)) {
-			kunmap(page);
+			kunmap_local(virt);
 			put_page(page);
 			err = -EFAULT;
 			break;
 		}
-		kunmap(page);
+		kunmap_local(virt);
 		put_page(page);
 
 		retval += bytes_to_copy;
diff --git a/fs/verity/verify.c b/fs/verity/verify.c
index 14e2fb49cff5..bde8c9b7d25f 100644
--- a/fs/verity/verify.c
+++ b/fs/verity/verify.c
@@ -39,16 +39,6 @@ static void hash_at_level(const struct merkle_tree_params *params,
 		   (params->log_blocksize - params->log_arity);
 }
 
-/* Extract a hash from a hash page */
-static void extract_hash(struct page *hpage, unsigned int hoffset,
-			 unsigned int hsize, u8 *out)
-{
-	void *virt = kmap_atomic(hpage);
-
-	memcpy(out, virt + hoffset, hsize);
-	kunmap_atomic(virt);
-}
-
 static inline int cmp_hashes(const struct fsverity_info *vi,
 			     const u8 *want_hash, const u8 *real_hash,
 			     pgoff_t index, int level)
@@ -129,7 +119,7 @@ static bool verify_page(struct inode *inode, const struct fsverity_info *vi,
 		}
 
 		if (PageChecked(hpage)) {
-			extract_hash(hpage, hoffset, hsize, _want_hash);
+			memcpy_from_page(_want_hash, hpage, hoffset, hsize);
 			want_hash = _want_hash;
 			put_page(hpage);
 			pr_debug_ratelimited("Hash page already checked, want %s:%*phN\n",
@@ -158,7 +148,7 @@ descend:
 		if (err)
 			goto out;
 		SetPageChecked(hpage);
-		extract_hash(hpage, hoffset, hsize, _want_hash);
+		memcpy_from_page(_want_hash, hpage, hoffset, hsize);
 		want_hash = _want_hash;
 		put_page(hpage);
 		pr_debug("Verified hash page at level %d, now want %s:%*phN\n",
diff --git a/fs/xattr.c b/fs/xattr.c
index a1f4998bc6be..61107b6bbed2 100644
--- a/fs/xattr.c
+++ b/fs/xattr.c
@@ -290,7 +290,7 @@ static inline bool is_posix_acl_xattr(const char *name)
 
 int
 vfs_setxattr(struct user_namespace *mnt_userns, struct dentry *dentry,
-	     const char *name, void *value, size_t size, int flags)
+	     const char *name, const void *value, size_t size, int flags)
 {
 	struct inode *inode = dentry->d_inode;
 	struct inode *delegated_inode = NULL;
@@ -298,16 +298,12 @@ vfs_setxattr(struct user_namespace *mnt_userns, struct dentry *dentry,
 	int error;
 
 	if (size && strcmp(name, XATTR_NAME_CAPS) == 0) {
-		error = cap_convert_nscap(mnt_userns, dentry,
-					  (const void **)&value, size);
+		error = cap_convert_nscap(mnt_userns, dentry, &value, size);
 		if (error < 0)
 			return error;
 		size = error;
 	}
 
-	if (size && is_posix_acl_xattr(name))
-		posix_acl_setxattr_idmapped_mnt(mnt_userns, inode, value, size);
-
 retry_deleg:
 	inode_lock(inode);
 	error = __vfs_setxattr_locked(mnt_userns, dentry, name, value, size,
@@ -587,9 +583,7 @@ int setxattr_copy(const char __user *name, struct xattr_ctx *ctx)
 static void setxattr_convert(struct user_namespace *mnt_userns,
 			     struct dentry *d, struct xattr_ctx *ctx)
 {
-	if (ctx->size &&
-		((strcmp(ctx->kname->name, XATTR_NAME_POSIX_ACL_ACCESS) == 0) ||
-		(strcmp(ctx->kname->name, XATTR_NAME_POSIX_ACL_DEFAULT) == 0)))
+	if (ctx->size && is_posix_acl_xattr(ctx->kname->name))
 		posix_acl_fix_xattr_from_user(ctx->kvalue, ctx->size);
 }
 
@@ -705,8 +699,7 @@ do_getxattr(struct user_namespace *mnt_userns, struct dentry *d,
 
 	error = vfs_getxattr(mnt_userns, d, kname, ctx->kvalue, ctx->size);
 	if (error > 0) {
-		if ((strcmp(kname, XATTR_NAME_POSIX_ACL_ACCESS) == 0) ||
-		    (strcmp(kname, XATTR_NAME_POSIX_ACL_DEFAULT) == 0))
+		if (is_posix_acl_xattr(kname))
 			posix_acl_fix_xattr_to_user(ctx->kvalue, error);
 		if (ctx->size && copy_to_user(ctx->value, ctx->kvalue, error))
 			error = -EFAULT;
diff --git a/fs/xfs/scrub/dir.c b/fs/xfs/scrub/dir.c
index 5abb5fdb71d9..b594f02a52c4 100644
--- a/fs/xfs/scrub/dir.c
+++ b/fs/xfs/scrub/dir.c
@@ -99,7 +99,7 @@ out:
  * we check the inode number to make sure it's sane, then we check that
  * we can look up this filename.  Finally, we check the ftype.
  */
-STATIC int
+STATIC bool
 xchk_dir_actor(
 	struct dir_context	*dir_iter,
 	const char		*name,
@@ -124,7 +124,7 @@ xchk_dir_actor(
 			xfs_dir2_dataptr_to_db(mp->m_dir_geo, pos));
 
 	if (xchk_should_terminate(sdc->sc, &error))
-		return error;
+		return !error;
 
 	/* Does this inode number make sense? */
 	if (!xfs_verify_dir_ino(mp, ino)) {
@@ -191,8 +191,8 @@ out:
 	 * and return zero to xchk_directory.
 	 */
 	if (error == 0 && sdc->sc->sm->sm_flags & XFS_SCRUB_OFLAG_CORRUPT)
-		return -EFSCORRUPTED;
-	return error;
+		return false;
+	return !error;
 }
 
 /* Scrub a directory btree record. */
diff --git a/fs/xfs/scrub/parent.c b/fs/xfs/scrub/parent.c
index ab182a5cd0c0..d8dff3fd8053 100644
--- a/fs/xfs/scrub/parent.c
+++ b/fs/xfs/scrub/parent.c
@@ -38,7 +38,7 @@ struct xchk_parent_ctx {
 };
 
 /* Look for a single entry in a directory pointing to an inode. */
-STATIC int
+STATIC bool
 xchk_parent_actor(
 	struct dir_context	*dc,
 	const char		*name,
@@ -62,7 +62,7 @@ xchk_parent_actor(
 	if (xchk_should_terminate(spc->sc, &error))
 		spc->cancelled = true;
 
-	return error;
+	return !error;
 }
 
 /* Count the number of dentries in the parent dir that point to this inode. */
diff --git a/fs/xfs/xfs_iops.c b/fs/xfs/xfs_iops.c
index 45518b8c613c..f51c60d7e205 100644
--- a/fs/xfs/xfs_iops.c
+++ b/fs/xfs/xfs_iops.c
@@ -604,6 +604,16 @@ xfs_vn_getattr(
 		stat->blksize = BLKDEV_IOSIZE;
 		stat->rdev = inode->i_rdev;
 		break;
+	case S_IFREG:
+		if (request_mask & STATX_DIOALIGN) {
+			struct xfs_buftarg	*target = xfs_inode_buftarg(ip);
+			struct block_device	*bdev = target->bt_bdev;
+
+			stat->result_mask |= STATX_DIOALIGN;
+			stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
+			stat->dio_offset_align = bdev_logical_block_size(bdev);
+		}
+		fallthrough;
 	default:
 		stat->blksize = xfs_stat_blksize(ip);
 		stat->rdev = 0;
diff --git a/fs/xfs/xfs_notify_failure.c b/fs/xfs/xfs_notify_failure.c
index 69d9c83ea4b2..5b1f9a24ed59 100644
--- a/fs/xfs/xfs_notify_failure.c
+++ b/fs/xfs/xfs_notify_failure.c
@@ -175,13 +175,13 @@ xfs_dax_notify_failure(
 	u64			ddev_start;
 	u64			ddev_end;
 
-	if (!(mp->m_sb.sb_flags & SB_BORN)) {
+	if (!(mp->m_super->s_flags & SB_BORN)) {
 		xfs_warn(mp, "filesystem is not ready for notify_failure()!");
 		return -EIO;
 	}
 
 	if (mp->m_rtdev_targp && mp->m_rtdev_targp->bt_daxdev == dax_dev) {
-		xfs_warn(mp,
+		xfs_debug(mp,
 			 "notify_failure() not supported on realtime device!");
 		return -EOPNOTSUPP;
 	}
@@ -194,7 +194,7 @@ xfs_dax_notify_failure(
 	}
 
 	if (!xfs_has_rmapbt(mp)) {
-		xfs_warn(mp, "notify_failure() needs rmapbt enabled!");
+		xfs_debug(mp, "notify_failure() needs rmapbt enabled!");
 		return -EOPNOTSUPP;
 	}
 
diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h
index e7d27373ff71..c09d72986968 100644
--- a/include/acpi/acpi_bus.h
+++ b/include/acpi/acpi_bus.h
@@ -365,7 +365,6 @@ struct acpi_device {
 	int device_type;
 	acpi_handle handle;		/* no handle for fixed hardware */
 	struct fwnode_handle fwnode;
-	struct acpi_device *parent;
 	struct list_head wakeup_list;
 	struct list_head del_list;
 	struct acpi_device_status status;
@@ -458,6 +457,14 @@ static inline void *acpi_driver_data(struct acpi_device *d)
 #define to_acpi_device(d)	container_of(d, struct acpi_device, dev)
 #define to_acpi_driver(d)	container_of(d, struct acpi_driver, drv)
 
+static inline struct acpi_device *acpi_dev_parent(struct acpi_device *adev)
+{
+	if (adev->dev.parent)
+		return to_acpi_device(adev->dev.parent);
+
+	return NULL;
+}
+
 static inline void acpi_set_device_status(struct acpi_device *adev, u32 sta)
 {
 	*((u32 *)&adev->status) = sta;
@@ -512,7 +519,6 @@ extern int unregister_acpi_notifier(struct notifier_block *);
  * External Functions
  */
 
-struct acpi_device *acpi_fetch_acpi_dev(acpi_handle handle);
 acpi_status acpi_bus_get_status_handle(acpi_handle handle,
 				       unsigned long long *sta);
 int acpi_bus_get_status(struct acpi_device *device);
@@ -613,8 +619,7 @@ enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev);
 int acpi_iommu_fwspec_init(struct device *dev, u32 id,
 			   struct fwnode_handle *fwnode,
 			   const struct iommu_ops *ops);
-int acpi_dma_get_range(struct device *dev, u64 *dma_addr, u64 *offset,
-		       u64 *size);
+int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map);
 int acpi_dma_configure_id(struct device *dev, enum dev_dma_attr attr,
 			   const u32 *input_id);
 static inline int acpi_dma_configure(struct device *dev,
@@ -733,10 +738,24 @@ static inline bool acpi_device_can_poweroff(struct acpi_device *adev)
 }
 
 bool acpi_dev_hid_uid_match(struct acpi_device *adev, const char *hid2, const char *uid2);
+int acpi_dev_uid_to_integer(struct acpi_device *adev, u64 *integer);
 
 void acpi_dev_clear_dependencies(struct acpi_device *supplier);
 bool acpi_dev_ready_for_enumeration(const struct acpi_device *device);
-struct acpi_device *acpi_dev_get_first_consumer_dev(struct acpi_device *supplier);
+struct acpi_device *acpi_dev_get_next_consumer_dev(struct acpi_device *supplier,
+						   struct acpi_device *start);
+
+/**
+ * for_each_acpi_consumer_dev - iterate over the consumer ACPI devices for a
+ *				given supplier
+ * @supplier: Pointer to the supplier's ACPI device
+ * @consumer: Pointer to &struct acpi_device to hold the consumer, initially NULL
+ */
+#define for_each_acpi_consumer_dev(supplier, consumer)			\
+	for (consumer = acpi_dev_get_next_consumer_dev(supplier, NULL);	\
+	     consumer;							\
+	     consumer = acpi_dev_get_next_consumer_dev(supplier, consumer))
+
 struct acpi_device *
 acpi_dev_get_next_match_dev(struct acpi_device *adev, const char *hid, const char *uid, s64 hrv);
 struct acpi_device *
@@ -767,9 +786,10 @@ static inline void acpi_dev_put(struct acpi_device *adev)
 		put_device(&adev->dev);
 }
 
-struct acpi_device *acpi_bus_get_acpi_device(acpi_handle handle);
+struct acpi_device *acpi_fetch_acpi_dev(acpi_handle handle);
+struct acpi_device *acpi_get_acpi_dev(acpi_handle handle);
 
-static inline void acpi_bus_put_acpi_device(struct acpi_device *adev)
+static inline void acpi_put_acpi_dev(struct acpi_device *adev)
 {
 	acpi_dev_put(adev);
 }
diff --git a/include/acpi/cppc_acpi.h b/include/acpi/cppc_acpi.h
index f73d357ecdf5..c5614444031f 100644
--- a/include/acpi/cppc_acpi.h
+++ b/include/acpi/cppc_acpi.h
@@ -140,6 +140,7 @@ extern int cppc_get_perf_ctrs(int cpu, struct cppc_perf_fb_ctrs *perf_fb_ctrs);
 extern int cppc_set_perf(int cpu, struct cppc_perf_ctrls *perf_ctrls);
 extern int cppc_set_enable(int cpu, bool enable);
 extern int cppc_get_perf_caps(int cpu, struct cppc_perf_caps *caps);
+extern bool cppc_perf_ctrs_in_pcc(void);
 extern bool acpi_cpc_valid(void);
 extern bool cppc_allow_fast_switch(void);
 extern int acpi_get_psd_map(unsigned int cpu, struct cppc_cpudata *cpu_data);
@@ -173,6 +174,10 @@ static inline int cppc_get_perf_caps(int cpu, struct cppc_perf_caps *caps)
 {
 	return -ENOTSUPP;
 }
+static inline bool cppc_perf_ctrs_in_pcc(void)
+{
+	return false;
+}
 static inline bool acpi_cpc_valid(void)
 {
 	return false;
diff --git a/include/acpi/video.h b/include/acpi/video.h
index db8548ff03ce..a275c35e5249 100644
--- a/include/acpi/video.h
+++ b/include/acpi/video.h
@@ -48,15 +48,18 @@ enum acpi_backlight_type {
 	acpi_backlight_video,
 	acpi_backlight_vendor,
 	acpi_backlight_native,
+	acpi_backlight_nvidia_wmi_ec,
+	acpi_backlight_apple_gmux,
 };
 
 #if IS_ENABLED(CONFIG_ACPI_VIDEO)
 extern int acpi_video_register(void);
 extern void acpi_video_unregister(void);
+extern void acpi_video_register_backlight(void);
 extern int acpi_video_get_edid(struct acpi_device *device, int type,
 			       int device_id, void **edid);
 extern enum acpi_backlight_type acpi_video_get_backlight_type(void);
-extern void acpi_video_set_dmi_backlight_type(enum acpi_backlight_type type);
+extern bool acpi_video_backlight_use_native(void);
 /*
  * Note: The value returned by acpi_video_handles_brightness_key_presses()
  * may change over time and should not be cached.
@@ -68,6 +71,7 @@ extern int acpi_video_get_levels(struct acpi_device *device,
 #else
 static inline int acpi_video_register(void) { return -ENODEV; }
 static inline void acpi_video_unregister(void) { return; }
+static inline void acpi_video_register_backlight(void) { return; }
 static inline int acpi_video_get_edid(struct acpi_device *device, int type,
 				      int device_id, void **edid)
 {
@@ -77,8 +81,9 @@ static inline enum acpi_backlight_type acpi_video_get_backlight_type(void)
 {
 	return acpi_backlight_vendor;
 }
-static inline void acpi_video_set_dmi_backlight_type(enum acpi_backlight_type type)
+static inline bool acpi_video_backlight_use_native(void)
 {
+	return true;
 }
 static inline bool acpi_video_handles_brightness_key_presses(void)
 {
diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h
index ba1f860af38b..4050b191e1a9 100644
--- a/include/asm-generic/bug.h
+++ b/include/asm-generic/bug.h
@@ -220,22 +220,6 @@ extern __printf(1, 2) void __warn_printk(const char *fmt, ...);
 # define WARN_ON_SMP(x)			({0;})
 #endif
 
-/*
- * WARN_ON_FUNCTION_MISMATCH() warns if a value doesn't match a
- * function address, and can be useful for catching issues with
- * callback functions, for example.
- *
- * With CONFIG_CFI_CLANG, the warning is disabled because the
- * compiler replaces function addresses taken in C code with
- * local jump table addresses, which breaks cross-module function
- * address equality.
- */
-#if defined(CONFIG_CFI_CLANG) && defined(CONFIG_MODULES)
-# define WARN_ON_FUNCTION_MISMATCH(x, fn) ({ 0; })
-#else
-# define WARN_ON_FUNCTION_MISMATCH(x, fn) WARN_ON_ONCE((x) != (fn))
-#endif
-
 #endif /* __ASSEMBLY__ */
 
 #endif
diff --git a/include/asm-generic/signal.h b/include/asm-generic/signal.h
index c53984fa9761..663dd6d0795d 100644
--- a/include/asm-generic/signal.h
+++ b/include/asm-generic/signal.h
@@ -5,8 +5,6 @@
 #include <uapi/asm-generic/signal.h>
 
 #ifndef __ASSEMBLY__
-#ifdef SA_RESTORER
-#endif
 
 #include <asm/sigcontext.h>
 #undef __HAVE_ARCH_SIG_BITOPS
diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 7c90b1ab3e00..a232f518d98a 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -154,6 +154,14 @@
 #define MEM_DISCARD(sec) *(.mem##sec)
 #endif
 
+#ifndef CONFIG_HAVE_DYNAMIC_FTRACE_NO_PATCHABLE
+#define KEEP_PATCHABLE		KEEP(*(__patchable_function_entries))
+#define PATCHABLE_DISCARDS
+#else
+#define KEEP_PATCHABLE
+#define PATCHABLE_DISCARDS	*(__patchable_function_entries)
+#endif
+
 #ifdef CONFIG_FTRACE_MCOUNT_RECORD
 /*
  * The ftrace call sites are logged to a section whose name depends on the
@@ -172,7 +180,7 @@
 #define MCOUNT_REC()	. = ALIGN(8);				\
 			__start_mcount_loc = .;			\
 			KEEP(*(__mcount_loc))			\
-			KEEP(*(__patchable_function_entries))	\
+			KEEP_PATCHABLE				\
 			__stop_mcount_loc = .;			\
 			ftrace_stub_graph = ftrace_stub;	\
 			ftrace_ops_list_func = arch_ftrace_ops_list_func;
@@ -422,6 +430,22 @@
 #endif
 
 /*
+ * .kcfi_traps contains a list KCFI trap locations.
+ */
+#ifndef KCFI_TRAPS
+#ifdef CONFIG_ARCH_USES_CFI_TRAPS
+#define KCFI_TRAPS							\
+	__kcfi_traps : AT(ADDR(__kcfi_traps) - LOAD_OFFSET) {		\
+		__start___kcfi_traps = .;				\
+		KEEP(*(.kcfi_traps))					\
+		__stop___kcfi_traps = .;				\
+	}
+#else
+#define KCFI_TRAPS
+#endif
+#endif
+
+/*
  * Read only Data
  */
 #define RO_DATA(align)							\
@@ -529,6 +553,8 @@
 		__stop___modver = .;					\
 	}								\
 									\
+	KCFI_TRAPS							\
+									\
 	RO_EXCEPTION_TABLE						\
 	NOTES								\
 	BTF								\
@@ -538,20 +564,6 @@
 
 
 /*
- * .text..L.cfi.jumptable.* contain Control-Flow Integrity (CFI)
- * jump table entries.
- */
-#ifdef CONFIG_CFI_CLANG
-#define TEXT_CFI_JT							\
-		ALIGN_FUNCTION();					\
-		__cfi_jt_start = .;					\
-		*(.text..L.cfi.jumptable .text..L.cfi.jumptable.*)	\
-		__cfi_jt_end = .;
-#else
-#define TEXT_CFI_JT
-#endif
-
-/*
  * Non-instrumentable text section
  */
 #define NOINSTR_TEXT							\
@@ -578,7 +590,6 @@
 		*(.text..refcount)					\
 		*(.ref.text)						\
 		*(.text.asan.* .text.tsan.*)				\
-		TEXT_CFI_JT						\
 	MEM_KEEP(init.text*)						\
 	MEM_KEEP(exit.text*)						\
 
@@ -1007,8 +1018,7 @@
  * keep any .init_array.* sections.
  * https://bugs.llvm.org/show_bug.cgi?id=46478
  */
-#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) || \
-	defined(CONFIG_CFI_CLANG)
+#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN)
 # ifdef CONFIG_CONSTRUCTORS
 #  define SANITIZER_DISCARDS						\
 	*(.eh_frame)
@@ -1023,6 +1033,7 @@
 
 #define COMMON_DISCARDS							\
 	SANITIZER_DISCARDS						\
+	PATCHABLE_DISCARDS						\
 	*(.discard)							\
 	*(.discard.*)							\
 	*(.modinfo)							\
diff --git a/include/drm/display/drm_dp.h b/include/drm/display/drm_dp.h
index 9e3aff7e68bb..e934aab357be 100644
--- a/include/drm/display/drm_dp.h
+++ b/include/drm/display/drm_dp.h
@@ -239,6 +239,7 @@
 
 #define DP_DSC_SUPPORT                      0x060   /* DP 1.4 */
 # define DP_DSC_DECOMPRESSION_IS_SUPPORTED  (1 << 0)
+# define DP_DSC_PASSTHROUGH_IS_SUPPORTED    (1 << 1)
 
 #define DP_DSC_REV                          0x061
 # define DP_DSC_MAJOR_MASK                  (0xf << 0)
@@ -1536,6 +1537,8 @@ enum drm_dp_phy {
 #define DP_SDP_VSC_EXT_CEA		0x21 /* DP 1.4 */
 /* 0x80+ CEA-861 infoframe types */
 
+#define DP_SDP_AUDIO_INFOFRAME_HB2	0x1b
+
 /**
  * struct dp_sdp_header - DP secondary data packet header
  * @HB0: Secondary Data Packet ID
diff --git a/include/drm/display/drm_dp_helper.h b/include/drm/display/drm_dp_helper.h
index db0fe9f8a612..ab55453f2d2c 100644
--- a/include/drm/display/drm_dp_helper.h
+++ b/include/drm/display/drm_dp_helper.h
@@ -69,6 +69,8 @@ bool drm_dp_128b132b_link_training_failed(const u8 link_status[DP_LINK_STATUS_SI
 u8 drm_dp_link_rate_to_bw_code(int link_rate);
 int drm_dp_bw_code_to_link_rate(u8 link_bw);
 
+const char *drm_dp_phy_name(enum drm_dp_phy dp_phy);
+
 /**
  * struct drm_dp_vsc_sdp - drm DP VSC SDP
  *
diff --git a/include/drm/display/drm_dp_mst_helper.h b/include/drm/display/drm_dp_mst_helper.h
index 10adec068b7f..41fd8352ab65 100644
--- a/include/drm/display/drm_dp_mst_helper.h
+++ b/include/drm/display/drm_dp_mst_helper.h
@@ -49,20 +49,6 @@ struct drm_dp_mst_topology_ref_history {
 struct drm_dp_mst_branch;
 
 /**
- * struct drm_dp_vcpi - Virtual Channel Payload Identifier
- * @vcpi: Virtual channel ID.
- * @pbn: Payload Bandwidth Number for this channel
- * @aligned_pbn: PBN aligned with slot size
- * @num_slots: number of slots for this PBN
- */
-struct drm_dp_vcpi {
-	int vcpi;
-	int pbn;
-	int aligned_pbn;
-	int num_slots;
-};
-
-/**
  * struct drm_dp_mst_port - MST port
  * @port_num: port number
  * @input: if this port is an input port. Protected by
@@ -86,6 +72,8 @@ struct drm_dp_vcpi {
  * @next: link to next port on this branch device
  * @aux: i2c aux transport to talk to device connected to this port, protected
  * by &drm_dp_mst_topology_mgr.base.lock.
+ * @passthrough_aux: parent aux to which DSC pass-through requests should be
+ * sent, only set if DSC pass-through is possible.
  * @parent: branch device parent of this port
  * @vcpi: Virtual Channel Payload info for this port.
  * @connector: DRM connector this port is connected to. Protected by
@@ -140,9 +128,9 @@ struct drm_dp_mst_port {
 	 */
 	struct drm_dp_mst_branch *mstb;
 	struct drm_dp_aux aux; /* i2c bus for this port? */
+	struct drm_dp_aux *passthrough_aux;
 	struct drm_dp_mst_branch *parent;
 
-	struct drm_dp_vcpi vcpi;
 	struct drm_connector *connector;
 	struct drm_dp_mst_topology_mgr *mgr;
 
@@ -527,35 +515,104 @@ struct drm_dp_mst_topology_cbs {
 	void (*poll_hpd_irq)(struct drm_dp_mst_topology_mgr *mgr);
 };
 
-#define DP_MAX_PAYLOAD (sizeof(unsigned long) * 8)
-
-#define DP_PAYLOAD_LOCAL 1
-#define DP_PAYLOAD_REMOTE 2
-#define DP_PAYLOAD_DELETE_LOCAL 3
-
-struct drm_dp_payload {
-	int payload_state;
-	int start_slot;
-	int num_slots;
-	int vcpi;
-};
-
 #define to_dp_mst_topology_state(x) container_of(x, struct drm_dp_mst_topology_state, base)
 
-struct drm_dp_vcpi_allocation {
+/**
+ * struct drm_dp_mst_atomic_payload - Atomic state struct for an MST payload
+ *
+ * The primary atomic state structure for a given MST payload. Stores information like current
+ * bandwidth allocation, intended action for this payload, etc.
+ */
+struct drm_dp_mst_atomic_payload {
+	/** @port: The MST port assigned to this payload */
 	struct drm_dp_mst_port *port;
-	int vcpi;
+
+	/**
+	 * @vc_start_slot: The time slot that this payload starts on. Because payload start slots
+	 * can't be determined ahead of time, the contents of this value are UNDEFINED at atomic
+	 * check time. This shouldn't usually matter, as the start slot should never be relevant for
+	 * atomic state computations.
+	 *
+	 * Since this value is determined at commit time instead of check time, this value is
+	 * protected by the MST helpers ensuring that async commits operating on the given topology
+	 * never run in parallel. In the event that a driver does need to read this value (e.g. to
+	 * inform hardware of the starting timeslot for a payload), the driver may either:
+	 *
+	 * * Read this field during the atomic commit after
+	 *   drm_dp_mst_atomic_wait_for_dependencies() has been called, which will ensure the
+	 *   previous MST states payload start slots have been copied over to the new state. Note
+	 *   that a new start slot won't be assigned/removed from this payload until
+	 *   drm_dp_add_payload_part1()/drm_dp_remove_payload() have been called.
+	 * * Acquire the MST modesetting lock, and then wait for any pending MST-related commits to
+	 *   get committed to hardware by calling drm_crtc_commit_wait() on each of the
+	 *   &drm_crtc_commit structs in &drm_dp_mst_topology_state.commit_deps.
+	 *
+	 * If neither of the two above solutions suffice (e.g. the driver needs to read the start
+	 * slot in the middle of an atomic commit without waiting for some reason), then drivers
+	 * should cache this value themselves after changing payloads.
+	 */
+	s8 vc_start_slot;
+
+	/** @vcpi: The Virtual Channel Payload Identifier */
+	u8 vcpi;
+	/**
+	 * @time_slots:
+	 * The number of timeslots allocated to this payload from the source DP Tx to
+	 * the immediate downstream DP Rx
+	 */
+	int time_slots;
+	/** @pbn: The payload bandwidth for this payload */
 	int pbn;
-	bool dsc_enabled;
+
+	/** @delete: Whether or not we intend to delete this payload during this atomic commit */
+	bool delete : 1;
+	/** @dsc_enabled: Whether or not this payload has DSC enabled */
+	bool dsc_enabled : 1;
+
+	/** @next: The list node for this payload */
 	struct list_head next;
 };
 
+/**
+ * struct drm_dp_mst_topology_state - DisplayPort MST topology atomic state
+ *
+ * This struct represents the atomic state of the toplevel DisplayPort MST manager
+ */
 struct drm_dp_mst_topology_state {
+	/** @base: Base private state for atomic */
 	struct drm_private_state base;
-	struct list_head vcpis;
+
+	/** @mgr: The topology manager */
 	struct drm_dp_mst_topology_mgr *mgr;
+
+	/**
+	 * @pending_crtc_mask: A bitmask of all CRTCs this topology state touches, drivers may
+	 * modify this to add additional dependencies if needed.
+	 */
+	u32 pending_crtc_mask;
+	/**
+	 * @commit_deps: A list of all CRTC commits affecting this topology, this field isn't
+	 * populated until drm_dp_mst_atomic_wait_for_dependencies() is called.
+	 */
+	struct drm_crtc_commit **commit_deps;
+	/** @num_commit_deps: The number of CRTC commits in @commit_deps */
+	size_t num_commit_deps;
+
+	/** @payload_mask: A bitmask of allocated VCPIs, used for VCPI assignments */
+	u32 payload_mask;
+	/** @payloads: The list of payloads being created/destroyed in this state */
+	struct list_head payloads;
+
+	/** @total_avail_slots: The total number of slots this topology can handle (63 or 64) */
 	u8 total_avail_slots;
+	/** @start_slot: The first usable time slot in this topology (1 or 0) */
 	u8 start_slot;
+
+	/**
+	 * @pbn_div: The current PBN divisor for this topology. The driver is expected to fill this
+	 * out itself.
+	 */
+	int pbn_div;
 };
 
 #define to_dp_mst_topology_mgr(x) container_of(x, struct drm_dp_mst_topology_mgr, base)
@@ -596,14 +653,6 @@ struct drm_dp_mst_topology_mgr {
 	 */
 	int max_payloads;
 	/**
-	 * @max_lane_count: maximum number of lanes the GPU can drive.
-	 */
-	int max_lane_count;
-	/**
-	 * @max_link_rate: maximum link rate per lane GPU can output, in kHz.
-	 */
-	int max_link_rate;
-	/**
 	 * @conn_base_id: DRM connector ID this mgr is connected to. Only used
 	 * to build the MST connector path value.
 	 */
@@ -646,6 +695,20 @@ struct drm_dp_mst_topology_mgr {
 	bool payload_id_table_cleared : 1;
 
 	/**
+	 * @payload_count: The number of currently active payloads in hardware. This value is only
+	 * intended to be used internally by MST helpers for payload tracking, and is only safe to
+	 * read/write from the atomic commit (not check) context.
+	 */
+	u8 payload_count;
+
+	/**
+	 * @next_start_slot: The starting timeslot to use for new VC payloads. This value is used
+	 * internally by MST helpers for payload tracking, and is only safe to read/write from the
+	 * atomic commit (not check) context.
+	 */
+	u8 next_start_slot;
+
+	/**
 	 * @mst_primary: Pointer to the primary/first branch device.
 	 */
 	struct drm_dp_mst_branch *mst_primary;
@@ -658,10 +721,6 @@ struct drm_dp_mst_topology_mgr {
 	 * @sink_count: Sink count from DEVICE_SERVICE_IRQ_VECTOR_ESI0.
 	 */
 	u8 sink_count;
-	/**
-	 * @pbn_div: PBN to slots divisor.
-	 */
-	int pbn_div;
 
 	/**
 	 * @funcs: Atomic helper callbacks
@@ -679,32 +738,6 @@ struct drm_dp_mst_topology_mgr {
 	struct list_head tx_msg_downq;
 
 	/**
-	 * @payload_lock: Protect payload information.
-	 */
-	struct mutex payload_lock;
-	/**
-	 * @proposed_vcpis: Array of pointers for the new VCPI allocation. The
-	 * VCPI structure itself is &drm_dp_mst_port.vcpi, and the size of
-	 * this array is determined by @max_payloads.
-	 */
-	struct drm_dp_vcpi **proposed_vcpis;
-	/**
-	 * @payloads: Array of payloads. The size of this array is determined
-	 * by @max_payloads.
-	 */
-	struct drm_dp_payload *payloads;
-	/**
-	 * @payload_mask: Elements of @payloads actually in use. Since
-	 * reallocation of active outputs isn't possible gaps can be created by
-	 * disabling outputs out of order compared to how they've been enabled.
-	 */
-	unsigned long payload_mask;
-	/**
-	 * @vcpi_mask: Similar to @payload_mask, but for @proposed_vcpis.
-	 */
-	unsigned long vcpi_mask;
-
-	/**
 	 * @tx_waitq: Wait to queue stall for the tx worker.
 	 */
 	wait_queue_head_t tx_waitq;
@@ -775,9 +808,7 @@ struct drm_dp_mst_topology_mgr {
 int drm_dp_mst_topology_mgr_init(struct drm_dp_mst_topology_mgr *mgr,
 				 struct drm_device *dev, struct drm_dp_aux *aux,
 				 int max_dpcd_transaction_bytes,
-				 int max_payloads,
-				 int max_lane_count, int max_link_rate,
-				 int conn_base_id);
+				 int max_payloads, int conn_base_id);
 
 void drm_dp_mst_topology_mgr_destroy(struct drm_dp_mst_topology_mgr *mgr);
 
@@ -800,28 +831,17 @@ int drm_dp_get_vc_payload_bw(const struct drm_dp_mst_topology_mgr *mgr,
 
 int drm_dp_calc_pbn_mode(int clock, int bpp, bool dsc);
 
-bool drm_dp_mst_allocate_vcpi(struct drm_dp_mst_topology_mgr *mgr,
-			      struct drm_dp_mst_port *port, int pbn, int slots);
-
-int drm_dp_mst_get_vcpi_slots(struct drm_dp_mst_topology_mgr *mgr, struct drm_dp_mst_port *port);
-
-
-void drm_dp_mst_reset_vcpi_slots(struct drm_dp_mst_topology_mgr *mgr, struct drm_dp_mst_port *port);
-
 void drm_dp_mst_update_slots(struct drm_dp_mst_topology_state *mst_state, uint8_t link_encoding_cap);
 
-void drm_dp_mst_deallocate_vcpi(struct drm_dp_mst_topology_mgr *mgr,
-				struct drm_dp_mst_port *port);
-
-
-int drm_dp_find_vcpi_slots(struct drm_dp_mst_topology_mgr *mgr,
-			   int pbn);
-
-
-int drm_dp_update_payload_part1(struct drm_dp_mst_topology_mgr *mgr, int start_slot);
-
-
-int drm_dp_update_payload_part2(struct drm_dp_mst_topology_mgr *mgr);
+int drm_dp_add_payload_part1(struct drm_dp_mst_topology_mgr *mgr,
+			     struct drm_dp_mst_topology_state *mst_state,
+			     struct drm_dp_mst_atomic_payload *payload);
+int drm_dp_add_payload_part2(struct drm_dp_mst_topology_mgr *mgr,
+			     struct drm_atomic_state *state,
+			     struct drm_dp_mst_atomic_payload *payload);
+void drm_dp_remove_payload(struct drm_dp_mst_topology_mgr *mgr,
+			   struct drm_dp_mst_topology_state *mst_state,
+			   struct drm_dp_mst_atomic_payload *payload);
 
 int drm_dp_check_act_status(struct drm_dp_mst_topology_mgr *mgr);
 
@@ -843,36 +863,51 @@ int drm_dp_mst_connector_late_register(struct drm_connector *connector,
 void drm_dp_mst_connector_early_unregister(struct drm_connector *connector,
 					   struct drm_dp_mst_port *port);
 
-struct drm_dp_mst_topology_state *drm_atomic_get_mst_topology_state(struct drm_atomic_state *state,
-								    struct drm_dp_mst_topology_mgr *mgr);
+struct drm_dp_mst_topology_state *
+drm_atomic_get_mst_topology_state(struct drm_atomic_state *state,
+				  struct drm_dp_mst_topology_mgr *mgr);
+struct drm_dp_mst_topology_state *
+drm_atomic_get_new_mst_topology_state(struct drm_atomic_state *state,
+				      struct drm_dp_mst_topology_mgr *mgr);
+struct drm_dp_mst_atomic_payload *
+drm_atomic_get_mst_payload_state(struct drm_dp_mst_topology_state *state,
+				 struct drm_dp_mst_port *port);
 int __must_check
-drm_dp_atomic_find_vcpi_slots(struct drm_atomic_state *state,
+drm_dp_atomic_find_time_slots(struct drm_atomic_state *state,
 			      struct drm_dp_mst_topology_mgr *mgr,
-			      struct drm_dp_mst_port *port, int pbn,
-			      int pbn_div);
+			      struct drm_dp_mst_port *port, int pbn);
 int drm_dp_mst_atomic_enable_dsc(struct drm_atomic_state *state,
 				 struct drm_dp_mst_port *port,
-				 int pbn, int pbn_div,
-				 bool enable);
+				 int pbn, bool enable);
 int __must_check
 drm_dp_mst_add_affected_dsc_crtcs(struct drm_atomic_state *state,
 				  struct drm_dp_mst_topology_mgr *mgr);
 int __must_check
-drm_dp_atomic_release_vcpi_slots(struct drm_atomic_state *state,
+drm_dp_atomic_release_time_slots(struct drm_atomic_state *state,
 				 struct drm_dp_mst_topology_mgr *mgr,
 				 struct drm_dp_mst_port *port);
+void drm_dp_mst_atomic_wait_for_dependencies(struct drm_atomic_state *state);
+int __must_check drm_dp_mst_atomic_setup_commit(struct drm_atomic_state *state);
 int drm_dp_send_power_updown_phy(struct drm_dp_mst_topology_mgr *mgr,
 				 struct drm_dp_mst_port *port, bool power_up);
 int drm_dp_send_query_stream_enc_status(struct drm_dp_mst_topology_mgr *mgr,
 		struct drm_dp_mst_port *port,
 		struct drm_dp_query_stream_enc_status_ack_reply *status);
 int __must_check drm_dp_mst_atomic_check(struct drm_atomic_state *state);
+int __must_check drm_dp_mst_root_conn_atomic_check(struct drm_connector_state *new_conn_state,
+						   struct drm_dp_mst_topology_mgr *mgr);
 
 void drm_dp_mst_get_port_malloc(struct drm_dp_mst_port *port);
 void drm_dp_mst_put_port_malloc(struct drm_dp_mst_port *port);
 
 struct drm_dp_aux *drm_dp_mst_dsc_aux_for_port(struct drm_dp_mst_port *port);
 
+static inline struct drm_dp_mst_topology_state *
+to_drm_dp_mst_topology_state(struct drm_private_state *state)
+{
+	return container_of(state, struct drm_dp_mst_topology_state, base);
+}
+
 extern const struct drm_private_state_funcs drm_dp_mst_topology_state_funcs;
 
 /**
diff --git a/include/drm/drm_atomic_helper.h b/include/drm/drm_atomic_helper.h
index 2a0b17842402..06d8902a8097 100644
--- a/include/drm/drm_atomic_helper.h
+++ b/include/drm/drm_atomic_helper.h
@@ -34,12 +34,24 @@
 #include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_util.h>
 
+/*
+ * Drivers that don't allow primary plane scaling may pass this macro in place
+ * of the min/max scale parameters of the plane-state checker function.
+ *
+ * Due to src being in 16.16 fixed point and dest being in integer pixels,
+ * 1<<16 represents no scaling.
+ */
+#define DRM_PLANE_NO_SCALING (1<<16)
+
 struct drm_atomic_state;
 struct drm_private_obj;
 struct drm_private_state;
 
 int drm_atomic_helper_check_modeset(struct drm_device *dev,
 				struct drm_atomic_state *state);
+int
+drm_atomic_helper_check_wb_encoder_state(struct drm_encoder *encoder,
+					 struct drm_connector_state *conn_state);
 int drm_atomic_helper_check_plane_state(struct drm_plane_state *plane_state,
 					const struct drm_crtc_state *crtc_state,
 					int min_scale,
diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h
index d434ab416ad4..6b65b0dfb4fb 100644
--- a/include/drm/drm_bridge.h
+++ b/include/drm/drm_bridge.h
@@ -930,6 +930,8 @@ struct drm_bridge *devm_drm_panel_bridge_add(struct device *dev,
 struct drm_bridge *devm_drm_panel_bridge_add_typed(struct device *dev,
 						   struct drm_panel *panel,
 						   u32 connector_type);
+struct drm_bridge *drmm_panel_bridge_add(struct drm_device *drm,
+					     struct drm_panel *panel);
 struct drm_connector *drm_panel_bridge_connector(struct drm_bridge *bridge);
 #else
 static inline bool drm_bridge_is_panel(const struct drm_bridge *bridge)
@@ -947,6 +949,8 @@ static inline int drm_panel_bridge_set_orientation(struct drm_connector *connect
 #if defined(CONFIG_OF) && defined(CONFIG_DRM_PANEL_BRIDGE)
 struct drm_bridge *devm_drm_of_get_bridge(struct device *dev, struct device_node *node,
 					  u32 port, u32 endpoint);
+struct drm_bridge *drmm_of_get_bridge(struct drm_device *drm, struct device_node *node,
+					  u32 port, u32 endpoint);
 #else
 static inline struct drm_bridge *devm_drm_of_get_bridge(struct device *dev,
 							struct device_node *node,
@@ -955,6 +959,14 @@ static inline struct drm_bridge *devm_drm_of_get_bridge(struct device *dev,
 {
 	return ERR_PTR(-ENODEV);
 }
+
+static inline struct drm_bridge *drmm_of_get_bridge(struct drm_device *drm,
+						     struct device_node *node,
+						     u32 port,
+						     u32 endpoint)
+{
+	return ERR_PTR(-ENODEV);
+}
 #endif
 
 #endif
diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h
index 7df7876b2ad5..56aee949c6fa 100644
--- a/include/drm/drm_connector.h
+++ b/include/drm/drm_connector.h
@@ -324,6 +324,22 @@ struct drm_monitor_range_info {
 };
 
 /**
+ * struct drm_luminance_range_info - Panel's luminance range for
+ * &drm_display_info. Calculated using data in EDID
+ *
+ * This struct is used to store a luminance range supported by panel
+ * as calculated using data from EDID's static hdr metadata.
+ *
+ * @min_luminance: This is the min supported luminance value
+ *
+ * @max_luminance: This is the max supported luminance value
+ */
+struct drm_luminance_range_info {
+	u32 min_luminance;
+	u32 max_luminance;
+};
+
+/**
  * enum drm_privacy_screen_status - privacy screen status
  *
  * This enum is used to track and control the state of the integrated privacy
@@ -625,6 +641,11 @@ struct drm_display_info {
 	struct drm_monitor_range_info monitor_range;
 
 	/**
+	 * @luminance_range: Luminance range supported by panel
+	 */
+	struct drm_luminance_range_info luminance_range;
+
+	/**
 	 * @mso_stream_count: eDP Multi-SST Operation (MSO) stream count from
 	 * the DisplayID VESA vendor block. 0 for conventional Single-Stream
 	 * Transport (SST), or 2 or 4 MSO streams.
@@ -1677,6 +1698,11 @@ int drm_connector_init_with_ddc(struct drm_device *dev,
 				const struct drm_connector_funcs *funcs,
 				int connector_type,
 				struct i2c_adapter *ddc);
+int drmm_connector_init(struct drm_device *dev,
+			struct drm_connector *connector,
+			const struct drm_connector_funcs *funcs,
+			int connector_type,
+			struct i2c_adapter *ddc);
 void drm_connector_attach_edid_property(struct drm_connector *connector);
 int drm_connector_register(struct drm_connector *connector);
 void drm_connector_unregister(struct drm_connector *connector);
diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h
index ffc1cde331d3..8e1cbc75143e 100644
--- a/include/drm/drm_crtc.h
+++ b/include/drm/drm_crtc.h
@@ -1216,6 +1216,15 @@ int drm_crtc_init_with_planes(struct drm_device *dev,
 			      struct drm_plane *cursor,
 			      const struct drm_crtc_funcs *funcs,
 			      const char *name, ...);
+
+__printf(6, 7)
+int drmm_crtc_init_with_planes(struct drm_device *dev,
+			       struct drm_crtc *crtc,
+			       struct drm_plane *primary,
+			       struct drm_plane *cursor,
+			       const struct drm_crtc_funcs *funcs,
+			       const char *name, ...);
+
 void drm_crtc_cleanup(struct drm_crtc *crtc);
 
 __printf(7, 8)
diff --git a/include/drm/drm_encoder.h b/include/drm/drm_encoder.h
index 6e91a0280f31..3a09682af685 100644
--- a/include/drm/drm_encoder.h
+++ b/include/drm/drm_encoder.h
@@ -194,6 +194,12 @@ int drm_encoder_init(struct drm_device *dev,
 		     const struct drm_encoder_funcs *funcs,
 		     int encoder_type, const char *name, ...);
 
+__printf(5, 6)
+int drmm_encoder_init(struct drm_device *dev,
+		      struct drm_encoder *encoder,
+		      const struct drm_encoder_funcs *funcs,
+		      int encoder_type, const char *name, ...);
+
 __printf(6, 7)
 void *__drmm_encoder_alloc(struct drm_device *dev,
 			   size_t size, size_t offset,
diff --git a/include/drm/drm_fb_cma_helper.h b/include/drm/drm_fb_dma_helper.h
index 6447e34528f8..d5e036c57801 100644
--- a/include/drm/drm_fb_cma_helper.h
+++ b/include/drm/drm_fb_dma_helper.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __DRM_FB_CMA_HELPER_H__
-#define __DRM_FB_CMA_HELPER_H__
+#ifndef __DRM_FB_DMA_HELPER_H__
+#define __DRM_FB_DMA_HELPER_H__
 
 #include <linux/types.h>
 
@@ -8,14 +8,14 @@ struct drm_device;
 struct drm_framebuffer;
 struct drm_plane_state;
 
-struct drm_gem_cma_object *drm_fb_cma_get_gem_obj(struct drm_framebuffer *fb,
+struct drm_gem_dma_object *drm_fb_dma_get_gem_obj(struct drm_framebuffer *fb,
 	unsigned int plane);
 
-dma_addr_t drm_fb_cma_get_gem_addr(struct drm_framebuffer *fb,
+dma_addr_t drm_fb_dma_get_gem_addr(struct drm_framebuffer *fb,
 				   struct drm_plane_state *state,
 				   unsigned int plane);
 
-void drm_fb_cma_sync_non_coherent(struct drm_device *drm,
+void drm_fb_dma_sync_non_coherent(struct drm_device *drm,
 				  struct drm_plane_state *old_state,
 				  struct drm_plane_state *state);
 
diff --git a/include/drm/drm_file.h b/include/drm/drm_file.h
index e0a73a1e2df7..d780fd151789 100644
--- a/include/drm/drm_file.h
+++ b/include/drm/drm_file.h
@@ -421,13 +421,4 @@ void drm_send_event_timestamp_locked(struct drm_device *dev,
 
 struct file *mock_drm_getfile(struct drm_minor *minor, unsigned int flags);
 
-#ifdef CONFIG_MMU
-struct drm_vma_offset_manager;
-unsigned long drm_get_unmapped_area(struct file *file,
-				    unsigned long uaddr, unsigned long len,
-				    unsigned long pgoff, unsigned long flags,
-				    struct drm_vma_offset_manager *mgr);
-#endif /* CONFIG_MMU */
-
-
 #endif /* _DRM_FILE_H_ */
diff --git a/include/drm/drm_format_helper.h b/include/drm/drm_format_helper.h
index 55145eca0782..eb5c98cf82b8 100644
--- a/include/drm/drm_format_helper.h
+++ b/include/drm/drm_format_helper.h
@@ -6,44 +6,51 @@
 #ifndef __LINUX_DRM_FORMAT_HELPER_H
 #define __LINUX_DRM_FORMAT_HELPER_H
 
+#include <linux/types.h>
+
+struct drm_device;
 struct drm_format_info;
 struct drm_framebuffer;
 struct drm_rect;
 
+struct iosys_map;
+
 unsigned int drm_fb_clip_offset(unsigned int pitch, const struct drm_format_info *format,
 				const struct drm_rect *clip);
 
-void drm_fb_memcpy(void *dst, unsigned int dst_pitch, const void *vaddr,
-		   const struct drm_framebuffer *fb, const struct drm_rect *clip);
-void drm_fb_memcpy_toio(void __iomem *dst, unsigned int dst_pitch, const void *vaddr,
-			const struct drm_framebuffer *fb, const struct drm_rect *clip);
-void drm_fb_swab(void *dst, unsigned int dst_pitch, const void *src,
-		 const struct drm_framebuffer *fb, const struct drm_rect *clip,
-		 bool cached);
-void drm_fb_xrgb8888_to_rgb332(void *dst, unsigned int dst_pitch, const void *vaddr,
-			       const struct drm_framebuffer *fb, const struct drm_rect *clip);
-void drm_fb_xrgb8888_to_rgb565(void *dst, unsigned int dst_pitch, const void *vaddr,
-			       const struct drm_framebuffer *fb, const struct drm_rect *clip,
-			       bool swab);
-void drm_fb_xrgb8888_to_rgb565_toio(void __iomem *dst, unsigned int dst_pitch,
-				    const void *vaddr, const struct drm_framebuffer *fb,
-				    const struct drm_rect *clip, bool swab);
-void drm_fb_xrgb8888_to_rgb888(void *dst, unsigned int dst_pitch, const void *src,
-			       const struct drm_framebuffer *fb, const struct drm_rect *clip);
-void drm_fb_xrgb8888_to_rgb888_toio(void __iomem *dst, unsigned int dst_pitch,
-				    const void *vaddr, const struct drm_framebuffer *fb,
+void drm_fb_memcpy(struct iosys_map *dst, const unsigned int *dst_pitch,
+		   const struct iosys_map *src, const struct drm_framebuffer *fb,
+		   const struct drm_rect *clip);
+void drm_fb_swab(struct iosys_map *dst, const unsigned int *dst_pitch,
+		 const struct iosys_map *src, const struct drm_framebuffer *fb,
+		 const struct drm_rect *clip, bool cached);
+void drm_fb_xrgb8888_to_rgb332(struct iosys_map *dst, const unsigned int *dst_pitch,
+			       const struct iosys_map *src, const struct drm_framebuffer *fb,
+			       const struct drm_rect *clip);
+void drm_fb_xrgb8888_to_rgb565(struct iosys_map *dst, const unsigned int *dst_pitch,
+			       const struct iosys_map *src, const struct drm_framebuffer *fb,
+			       const struct drm_rect *clip, bool swab);
+void drm_fb_xrgb8888_to_rgb888(struct iosys_map *dst, const unsigned int *dst_pitch,
+			       const struct iosys_map *src, const struct drm_framebuffer *fb,
+			       const struct drm_rect *clip);
+void drm_fb_xrgb8888_to_xrgb2101010(struct iosys_map *dst, const unsigned int *dst_pitch,
+				    const struct iosys_map *src, const struct drm_framebuffer *fb,
 				    const struct drm_rect *clip);
-void drm_fb_xrgb8888_to_xrgb2101010_toio(void __iomem *dst, unsigned int dst_pitch,
-					 const void *vaddr, const struct drm_framebuffer *fb,
-					 const struct drm_rect *clip);
-void drm_fb_xrgb8888_to_gray8(void *dst, unsigned int dst_pitch, const void *vaddr,
-			      const struct drm_framebuffer *fb, const struct drm_rect *clip);
-
-int drm_fb_blit_toio(void __iomem *dst, unsigned int dst_pitch, uint32_t dst_format,
-		     const void *vmap, const struct drm_framebuffer *fb,
-		     const struct drm_rect *rect);
-
-void drm_fb_xrgb8888_to_mono(void *dst, unsigned int dst_pitch, const void *src,
-			     const struct drm_framebuffer *fb, const struct drm_rect *clip);
+void drm_fb_xrgb8888_to_gray8(struct iosys_map *dst, const unsigned int *dst_pitch,
+			      const struct iosys_map *src, const struct drm_framebuffer *fb,
+			      const struct drm_rect *clip);
+
+int drm_fb_blit(struct iosys_map *dst, const unsigned int *dst_pitch, uint32_t dst_format,
+		const struct iosys_map *src, const struct drm_framebuffer *fb,
+		const struct drm_rect *rect);
+
+void drm_fb_xrgb8888_to_mono(struct iosys_map *dst, const unsigned int *dst_pitch,
+			     const struct iosys_map *src, const struct drm_framebuffer *fb,
+			     const struct drm_rect *clip);
+
+size_t drm_fb_build_fourcc_list(struct drm_device *dev,
+				const u32 *native_fourccs, size_t native_nfourccs,
+				const u32 *extra_fourccs, size_t extra_nfourccs,
+				u32 *fourccs_out, size_t nfourccs_out);
 
 #endif /* __LINUX_DRM_FORMAT_HELPER_H */
diff --git a/include/drm/drm_fourcc.h b/include/drm/drm_fourcc.h
index 22aa64d07c79..532ae78ca747 100644
--- a/include/drm/drm_fourcc.h
+++ b/include/drm/drm_fourcc.h
@@ -138,6 +138,9 @@ struct drm_format_info {
 
 	/** @is_yuv: Is it a YUV format? */
 	bool is_yuv;
+
+	/** @is_color_indexed: Is it a color-indexed format? */
+	bool is_color_indexed;
 };
 
 /**
@@ -313,6 +316,7 @@ unsigned int drm_format_info_block_width(const struct drm_format_info *info,
 					 int plane);
 unsigned int drm_format_info_block_height(const struct drm_format_info *info,
 					  int plane);
+unsigned int drm_format_info_bpp(const struct drm_format_info *info, int plane);
 uint64_t drm_format_info_min_pitch(const struct drm_format_info *info,
 				   int plane, unsigned int buffer_width);
 
diff --git a/include/drm/drm_framebuffer.h b/include/drm/drm_framebuffer.h
index f67c5b7bcb68..0dcc07b68654 100644
--- a/include/drm/drm_framebuffer.h
+++ b/include/drm/drm_framebuffer.h
@@ -154,10 +154,10 @@ struct drm_framebuffer {
 	 * drm_mode_fb_cmd2.
 	 *
 	 * Note that this is a linear offset and does not take into account
-	 * tiling or buffer laytou per @modifier. It meant to be used when the
-	 * actual pixel data for this framebuffer plane starts at an offset,
-	 * e.g.  when multiple planes are allocated within the same backing
-	 * storage buffer object. For tiled layouts this generally means it
+	 * tiling or buffer layout per @modifier. It is meant to be used when
+	 * the actual pixel data for this framebuffer plane starts at an offset,
+	 * e.g. when multiple planes are allocated within the same backing
+	 * storage buffer object. For tiled layouts this generally means its
 	 * @offsets must at least be tile-size aligned, but hardware often has
 	 * stricter requirements.
 	 *
diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h
index 87cffc9efa85..bd42f25e449c 100644
--- a/include/drm/drm_gem.h
+++ b/include/drm/drm_gem.h
@@ -175,6 +175,41 @@ struct drm_gem_object_funcs {
 };
 
 /**
+ * struct drm_gem_lru - A simple LRU helper
+ *
+ * A helper for tracking GEM objects in a given state, to aid in
+ * driver's shrinker implementation.  Tracks the count of pages
+ * for lockless &shrinker.count_objects, and provides
+ * &drm_gem_lru_scan for driver's &shrinker.scan_objects
+ * implementation.
+ */
+struct drm_gem_lru {
+	/**
+	 * @lock:
+	 *
+	 * Lock protecting movement of GEM objects between LRUs.  All
+	 * LRUs that the object can move between should be protected
+	 * by the same lock.
+	 */
+	struct mutex *lock;
+
+	/**
+	 * @count:
+	 *
+	 * The total number of backing pages of the GEM objects in
+	 * this LRU.
+	 */
+	long count;
+
+	/**
+	 * @list:
+	 *
+	 * The LRU list.
+	 */
+	struct list_head list;
+};
+
+/**
  * struct drm_gem_object - GEM buffer object
  *
  * This structure defines the generic parts for GEM buffer objects, which are
@@ -217,7 +252,7 @@ struct drm_gem_object {
 	 *
 	 * SHMEM file node used as backing storage for swappable buffer objects.
 	 * GEM also supports driver private objects with driver-specific backing
-	 * storage (contiguous CMA memory, special reserved blocks). In this
+	 * storage (contiguous DMA memory, special reserved blocks). In this
 	 * case @filp is NULL.
 	 */
 	struct file *filp;
@@ -312,6 +347,20 @@ struct drm_gem_object {
 	 *
 	 */
 	const struct drm_gem_object_funcs *funcs;
+
+	/**
+	 * @lru_node:
+	 *
+	 * List node in a &drm_gem_lru.
+	 */
+	struct list_head lru_node;
+
+	/**
+	 * @lru:
+	 *
+	 * The current LRU list that the GEM object is on.
+	 */
+	struct drm_gem_lru *lru;
 };
 
 /**
@@ -420,4 +469,10 @@ void drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
 int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
 			    u32 handle, u64 *offset);
 
+void drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock);
+void drm_gem_lru_remove(struct drm_gem_object *obj);
+void drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj);
+unsigned long drm_gem_lru_scan(struct drm_gem_lru *lru, unsigned nr_to_scan,
+			       bool (*shrink)(struct drm_gem_object *obj));
+
 #endif /* __DRM_GEM_H__ */
diff --git a/include/drm/drm_gem_cma_helper.h b/include/drm/drm_gem_dma_helper.h
index fbda4ce5d5fb..8a043235dad8 100644
--- a/include/drm/drm_gem_cma_helper.h
+++ b/include/drm/drm_gem_dma_helper.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __DRM_GEM_CMA_HELPER_H__
-#define __DRM_GEM_CMA_HELPER_H__
+#ifndef __DRM_GEM_DMA_HELPER_H__
+#define __DRM_GEM_DMA_HELPER_H__
 
 #include <drm/drm_file.h>
 #include <drm/drm_ioctl.h>
@@ -9,128 +9,128 @@
 struct drm_mode_create_dumb;
 
 /**
- * struct drm_gem_cma_object - GEM object backed by CMA memory allocations
+ * struct drm_gem_dma_object - GEM object backed by DMA memory allocations
  * @base: base GEM object
- * @paddr: physical address of the backing memory
+ * @dma_addr: DMA address of the backing memory
  * @sgt: scatter/gather table for imported PRIME buffers. The table can have
  *       more than one entry but they are guaranteed to have contiguous
  *       DMA addresses.
  * @vaddr: kernel virtual address of the backing memory
  * @map_noncoherent: if true, the GEM object is backed by non-coherent memory
  */
-struct drm_gem_cma_object {
+struct drm_gem_dma_object {
 	struct drm_gem_object base;
-	dma_addr_t paddr;
+	dma_addr_t dma_addr;
 	struct sg_table *sgt;
 
-	/* For objects with DMA memory allocated by GEM CMA */
+	/* For objects with DMA memory allocated by GEM DMA */
 	void *vaddr;
 
 	bool map_noncoherent;
 };
 
-#define to_drm_gem_cma_obj(gem_obj) \
-	container_of(gem_obj, struct drm_gem_cma_object, base)
+#define to_drm_gem_dma_obj(gem_obj) \
+	container_of(gem_obj, struct drm_gem_dma_object, base)
 
-struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm,
+struct drm_gem_dma_object *drm_gem_dma_create(struct drm_device *drm,
 					      size_t size);
-void drm_gem_cma_free(struct drm_gem_cma_object *cma_obj);
-void drm_gem_cma_print_info(const struct drm_gem_cma_object *cma_obj,
+void drm_gem_dma_free(struct drm_gem_dma_object *dma_obj);
+void drm_gem_dma_print_info(const struct drm_gem_dma_object *dma_obj,
 			    struct drm_printer *p, unsigned int indent);
-struct sg_table *drm_gem_cma_get_sg_table(struct drm_gem_cma_object *cma_obj);
-int drm_gem_cma_vmap(struct drm_gem_cma_object *cma_obj,
+struct sg_table *drm_gem_dma_get_sg_table(struct drm_gem_dma_object *dma_obj);
+int drm_gem_dma_vmap(struct drm_gem_dma_object *dma_obj,
 		     struct iosys_map *map);
-int drm_gem_cma_mmap(struct drm_gem_cma_object *cma_obj, struct vm_area_struct *vma);
+int drm_gem_dma_mmap(struct drm_gem_dma_object *dma_obj, struct vm_area_struct *vma);
 
-extern const struct vm_operations_struct drm_gem_cma_vm_ops;
+extern const struct vm_operations_struct drm_gem_dma_vm_ops;
 
 /*
  * GEM object functions
  */
 
 /**
- * drm_gem_cma_object_free - GEM object function for drm_gem_cma_free()
+ * drm_gem_dma_object_free - GEM object function for drm_gem_dma_free()
  * @obj: GEM object to free
  *
- * This function wraps drm_gem_cma_free_object(). Drivers that employ the CMA helpers
+ * This function wraps drm_gem_dma_free_object(). Drivers that employ the DMA helpers
  * should use it as their &drm_gem_object_funcs.free handler.
  */
-static inline void drm_gem_cma_object_free(struct drm_gem_object *obj)
+static inline void drm_gem_dma_object_free(struct drm_gem_object *obj)
 {
-	struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj);
+	struct drm_gem_dma_object *dma_obj = to_drm_gem_dma_obj(obj);
 
-	drm_gem_cma_free(cma_obj);
+	drm_gem_dma_free(dma_obj);
 }
 
 /**
- * drm_gem_cma_object_print_info() - Print &drm_gem_cma_object info for debugfs
+ * drm_gem_dma_object_print_info() - Print &drm_gem_dma_object info for debugfs
  * @p: DRM printer
  * @indent: Tab indentation level
  * @obj: GEM object
  *
- * This function wraps drm_gem_cma_print_info(). Drivers that employ the CMA helpers
+ * This function wraps drm_gem_dma_print_info(). Drivers that employ the DMA helpers
  * should use this function as their &drm_gem_object_funcs.print_info handler.
  */
-static inline void drm_gem_cma_object_print_info(struct drm_printer *p, unsigned int indent,
+static inline void drm_gem_dma_object_print_info(struct drm_printer *p, unsigned int indent,
 						 const struct drm_gem_object *obj)
 {
-	const struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj);
+	const struct drm_gem_dma_object *dma_obj = to_drm_gem_dma_obj(obj);
 
-	drm_gem_cma_print_info(cma_obj, p, indent);
+	drm_gem_dma_print_info(dma_obj, p, indent);
 }
 
 /**
- * drm_gem_cma_object_get_sg_table - GEM object function for drm_gem_cma_get_sg_table()
+ * drm_gem_dma_object_get_sg_table - GEM object function for drm_gem_dma_get_sg_table()
  * @obj: GEM object
  *
- * This function wraps drm_gem_cma_get_sg_table(). Drivers that employ the CMA helpers should
+ * This function wraps drm_gem_dma_get_sg_table(). Drivers that employ the DMA helpers should
  * use it as their &drm_gem_object_funcs.get_sg_table handler.
  *
  * Returns:
  * A pointer to the scatter/gather table of pinned pages or NULL on failure.
  */
-static inline struct sg_table *drm_gem_cma_object_get_sg_table(struct drm_gem_object *obj)
+static inline struct sg_table *drm_gem_dma_object_get_sg_table(struct drm_gem_object *obj)
 {
-	struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj);
+	struct drm_gem_dma_object *dma_obj = to_drm_gem_dma_obj(obj);
 
-	return drm_gem_cma_get_sg_table(cma_obj);
+	return drm_gem_dma_get_sg_table(dma_obj);
 }
 
 /*
- * drm_gem_cma_object_vmap - GEM object function for drm_gem_cma_vmap()
+ * drm_gem_dma_object_vmap - GEM object function for drm_gem_dma_vmap()
  * @obj: GEM object
- * @map: Returns the kernel virtual address of the CMA GEM object's backing store.
+ * @map: Returns the kernel virtual address of the DMA GEM object's backing store.
  *
- * This function wraps drm_gem_cma_vmap(). Drivers that employ the CMA helpers should
+ * This function wraps drm_gem_dma_vmap(). Drivers that employ the DMA helpers should
  * use it as their &drm_gem_object_funcs.vmap handler.
  *
  * Returns:
  * 0 on success or a negative error code on failure.
  */
-static inline int drm_gem_cma_object_vmap(struct drm_gem_object *obj,
+static inline int drm_gem_dma_object_vmap(struct drm_gem_object *obj,
 					  struct iosys_map *map)
 {
-	struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj);
+	struct drm_gem_dma_object *dma_obj = to_drm_gem_dma_obj(obj);
 
-	return drm_gem_cma_vmap(cma_obj, map);
+	return drm_gem_dma_vmap(dma_obj, map);
 }
 
 /**
- * drm_gem_cma_object_mmap - GEM object function for drm_gem_cma_mmap()
+ * drm_gem_dma_object_mmap - GEM object function for drm_gem_dma_mmap()
  * @obj: GEM object
  * @vma: VMA for the area to be mapped
  *
- * This function wraps drm_gem_cma_mmap(). Drivers that employ the cma helpers should
+ * This function wraps drm_gem_dma_mmap(). Drivers that employ the dma helpers should
  * use it as their &drm_gem_object_funcs.mmap handler.
  *
  * Returns:
  * 0 on success or a negative error code on failure.
  */
-static inline int drm_gem_cma_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+static inline int drm_gem_dma_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
 {
-	struct drm_gem_cma_object *cma_obj = to_drm_gem_cma_obj(obj);
+	struct drm_gem_dma_object *dma_obj = to_drm_gem_dma_obj(obj);
 
-	return drm_gem_cma_mmap(cma_obj, vma);
+	return drm_gem_dma_mmap(dma_obj, vma);
 }
 
 /*
@@ -138,57 +138,57 @@ static inline int drm_gem_cma_object_mmap(struct drm_gem_object *obj, struct vm_
  */
 
 /* create memory region for DRM framebuffer */
-int drm_gem_cma_dumb_create_internal(struct drm_file *file_priv,
+int drm_gem_dma_dumb_create_internal(struct drm_file *file_priv,
 				     struct drm_device *drm,
 				     struct drm_mode_create_dumb *args);
 
 /* create memory region for DRM framebuffer */
-int drm_gem_cma_dumb_create(struct drm_file *file_priv,
+int drm_gem_dma_dumb_create(struct drm_file *file_priv,
 			    struct drm_device *drm,
 			    struct drm_mode_create_dumb *args);
 
 struct drm_gem_object *
-drm_gem_cma_prime_import_sg_table(struct drm_device *dev,
+drm_gem_dma_prime_import_sg_table(struct drm_device *dev,
 				  struct dma_buf_attachment *attach,
 				  struct sg_table *sgt);
 
 /**
- * DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE - CMA GEM driver operations
+ * DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE - DMA GEM driver operations
  * @dumb_create_func: callback function for .dumb_create
  *
  * This macro provides a shortcut for setting the default GEM operations in the
  * &drm_driver structure.
  *
- * This macro is a variant of DRM_GEM_CMA_DRIVER_OPS for drivers that
+ * This macro is a variant of DRM_GEM_DMA_DRIVER_OPS for drivers that
  * override the default implementation of &struct rm_driver.dumb_create. Use
- * DRM_GEM_CMA_DRIVER_OPS if possible. Drivers that require a virtual address
+ * DRM_GEM_DMA_DRIVER_OPS if possible. Drivers that require a virtual address
  * on imported buffers should use
- * DRM_GEM_CMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE() instead.
+ * DRM_GEM_DMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE() instead.
  */
-#define DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(dumb_create_func) \
+#define DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(dumb_create_func) \
 	.dumb_create		= (dumb_create_func), \
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd, \
 	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle, \
-	.gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table, \
+	.gem_prime_import_sg_table = drm_gem_dma_prime_import_sg_table, \
 	.gem_prime_mmap		= drm_gem_prime_mmap
 
 /**
- * DRM_GEM_CMA_DRIVER_OPS - CMA GEM driver operations
+ * DRM_GEM_DMA_DRIVER_OPS - DMA GEM driver operations
  *
  * This macro provides a shortcut for setting the default GEM operations in the
  * &drm_driver structure.
  *
  * Drivers that come with their own implementation of
  * &struct drm_driver.dumb_create should use
- * DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE() instead. Use
- * DRM_GEM_CMA_DRIVER_OPS if possible. Drivers that require a virtual address
- * on imported buffers should use DRM_GEM_CMA_DRIVER_OPS_VMAP instead.
+ * DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE() instead. Use
+ * DRM_GEM_DMA_DRIVER_OPS if possible. Drivers that require a virtual address
+ * on imported buffers should use DRM_GEM_DMA_DRIVER_OPS_VMAP instead.
  */
-#define DRM_GEM_CMA_DRIVER_OPS \
-	DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE(drm_gem_cma_dumb_create)
+#define DRM_GEM_DMA_DRIVER_OPS \
+	DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(drm_gem_dma_dumb_create)
 
 /**
- * DRM_GEM_CMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE - CMA GEM driver operations
+ * DRM_GEM_DMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE - DMA GEM driver operations
  *                                                ensuring a virtual address
  *                                                on the buffer
  * @dumb_create_func: callback function for .dumb_create
@@ -197,21 +197,21 @@ drm_gem_cma_prime_import_sg_table(struct drm_device *dev,
  * &drm_driver structure for drivers that need the virtual address also on
  * imported buffers.
  *
- * This macro is a variant of DRM_GEM_CMA_DRIVER_OPS_VMAP for drivers that
+ * This macro is a variant of DRM_GEM_DMA_DRIVER_OPS_VMAP for drivers that
  * override the default implementation of &struct drm_driver.dumb_create. Use
- * DRM_GEM_CMA_DRIVER_OPS_VMAP if possible. Drivers that do not require a
+ * DRM_GEM_DMA_DRIVER_OPS_VMAP if possible. Drivers that do not require a
  * virtual address on imported buffers should use
- * DRM_GEM_CMA_DRIVER_OPS_WITH_DUMB_CREATE() instead.
+ * DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE() instead.
  */
-#define DRM_GEM_CMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE(dumb_create_func) \
+#define DRM_GEM_DMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE(dumb_create_func) \
 	.dumb_create		= dumb_create_func, \
 	.prime_handle_to_fd	= drm_gem_prime_handle_to_fd, \
 	.prime_fd_to_handle	= drm_gem_prime_fd_to_handle, \
-	.gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table_vmap, \
+	.gem_prime_import_sg_table = drm_gem_dma_prime_import_sg_table_vmap, \
 	.gem_prime_mmap		= drm_gem_prime_mmap
 
 /**
- * DRM_GEM_CMA_DRIVER_OPS_VMAP - CMA GEM driver operations ensuring a virtual
+ * DRM_GEM_DMA_DRIVER_OPS_VMAP - DMA GEM driver operations ensuring a virtual
  *                               address on the buffer
  *
  * This macro provides a shortcut for setting the default GEM operations in the
@@ -220,16 +220,16 @@ drm_gem_cma_prime_import_sg_table(struct drm_device *dev,
  *
  * Drivers that come with their own implementation of
  * &struct drm_driver.dumb_create should use
- * DRM_GEM_CMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE() instead. Use
- * DRM_GEM_CMA_DRIVER_OPS_VMAP if possible. Drivers that do not require a
- * virtual address on imported buffers should use DRM_GEM_CMA_DRIVER_OPS
+ * DRM_GEM_DMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE() instead. Use
+ * DRM_GEM_DMA_DRIVER_OPS_VMAP if possible. Drivers that do not require a
+ * virtual address on imported buffers should use DRM_GEM_DMA_DRIVER_OPS
  * instead.
  */
-#define DRM_GEM_CMA_DRIVER_OPS_VMAP \
-	DRM_GEM_CMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE(drm_gem_cma_dumb_create)
+#define DRM_GEM_DMA_DRIVER_OPS_VMAP \
+	DRM_GEM_DMA_DRIVER_OPS_VMAP_WITH_DUMB_CREATE(drm_gem_dma_dumb_create)
 
 struct drm_gem_object *
-drm_gem_cma_prime_import_sg_table_vmap(struct drm_device *drm,
+drm_gem_dma_prime_import_sg_table_vmap(struct drm_device *drm,
 				       struct dma_buf_attachment *attach,
 				       struct sg_table *sgt);
 
@@ -238,22 +238,22 @@ drm_gem_cma_prime_import_sg_table_vmap(struct drm_device *drm,
  */
 
 #ifndef CONFIG_MMU
-unsigned long drm_gem_cma_get_unmapped_area(struct file *filp,
+unsigned long drm_gem_dma_get_unmapped_area(struct file *filp,
 					    unsigned long addr,
 					    unsigned long len,
 					    unsigned long pgoff,
 					    unsigned long flags);
-#define DRM_GEM_CMA_UNMAPPED_AREA_FOPS \
-	.get_unmapped_area	= drm_gem_cma_get_unmapped_area,
+#define DRM_GEM_DMA_UNMAPPED_AREA_FOPS \
+	.get_unmapped_area	= drm_gem_dma_get_unmapped_area,
 #else
-#define DRM_GEM_CMA_UNMAPPED_AREA_FOPS
+#define DRM_GEM_DMA_UNMAPPED_AREA_FOPS
 #endif
 
 /**
- * DEFINE_DRM_GEM_CMA_FOPS() - macro to generate file operations for CMA drivers
+ * DEFINE_DRM_GEM_DMA_FOPS() - macro to generate file operations for DMA drivers
  * @name: name for the generated structure
  *
- * This macro autogenerates a suitable &struct file_operations for CMA based
+ * This macro autogenerates a suitable &struct file_operations for DMA based
  * drivers, which can be assigned to &drm_driver.fops. Note that this structure
  * cannot be shared between drivers, because it contains a reference to the
  * current module using THIS_MODULE.
@@ -262,7 +262,7 @@ unsigned long drm_gem_cma_get_unmapped_area(struct file *filp,
  * non-static version of this you're probably doing it wrong and will break the
  * THIS_MODULE reference by accident.
  */
-#define DEFINE_DRM_GEM_CMA_FOPS(name) \
+#define DEFINE_DRM_GEM_DMA_FOPS(name) \
 	static const struct file_operations name = {\
 		.owner		= THIS_MODULE,\
 		.open		= drm_open,\
@@ -273,7 +273,7 @@ unsigned long drm_gem_cma_get_unmapped_area(struct file *filp,
 		.read		= drm_read,\
 		.llseek		= noop_llseek,\
 		.mmap		= drm_gem_mmap,\
-		DRM_GEM_CMA_UNMAPPED_AREA_FOPS \
+		DRM_GEM_DMA_UNMAPPED_AREA_FOPS \
 	}
 
-#endif /* __DRM_GEM_CMA_HELPER_H__ */
+#endif /* __DRM_GEM_DMA_HELPER_H__ */
diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h
index d0a57853c188..a2201b2488c5 100644
--- a/include/drm/drm_gem_shmem_helper.h
+++ b/include/drm/drm_gem_shmem_helper.h
@@ -210,7 +210,7 @@ static inline void drm_gem_shmem_object_unpin(struct drm_gem_object *obj)
  * use it as their &drm_gem_object_funcs.get_sg_table handler.
  *
  * Returns:
- * A pointer to the scatter/gather table of pinned pages or NULL on failure.
+ * A pointer to the scatter/gather table of pinned pages or error pointer on failure.
  */
 static inline struct sg_table *drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj)
 {
diff --git a/include/drm/drm_mipi_dbi.h b/include/drm/drm_mipi_dbi.h
index dad2f187b64b..14eaecb1825c 100644
--- a/include/drm/drm_mipi_dbi.h
+++ b/include/drm/drm_mipi_dbi.h
@@ -155,6 +155,8 @@ int mipi_dbi_dev_init_with_formats(struct mipi_dbi_dev *dbidev,
 int mipi_dbi_dev_init(struct mipi_dbi_dev *dbidev,
 		      const struct drm_simple_display_pipe_funcs *funcs,
 		      const struct drm_display_mode *mode, unsigned int rotation);
+enum drm_mode_status mipi_dbi_pipe_mode_valid(struct drm_simple_display_pipe *pipe,
+					      const struct drm_display_mode *mode);
 void mipi_dbi_pipe_update(struct drm_simple_display_pipe *pipe,
 			  struct drm_plane_state *old_state);
 void mipi_dbi_enable_flush(struct mipi_dbi_dev *dbidev,
diff --git a/include/drm/drm_mipi_dsi.h b/include/drm/drm_mipi_dsi.h
index 91a164bdd8f3..20b21b577dea 100644
--- a/include/drm/drm_mipi_dsi.h
+++ b/include/drm/drm_mipi_dsi.h
@@ -179,6 +179,7 @@ struct mipi_dsi_device_info {
  * @lp_rate: maximum lane frequency for low power mode in hertz, this should
  * be set to the real limits of the hardware, zero is only accepted for
  * legacy drivers
+ * @dsc: panel/bridge DSC pps payload to be sent
  */
 struct mipi_dsi_device {
 	struct mipi_dsi_host *host;
@@ -191,6 +192,7 @@ struct mipi_dsi_device {
 	unsigned long mode_flags;
 	unsigned long hs_rate;
 	unsigned long lp_rate;
+	struct drm_dsc_config *dsc;
 };
 
 #define MIPI_DSI_MODULE_PREFIX "mipi-dsi:"
@@ -322,7 +324,7 @@ int mipi_dsi_dcs_get_display_brightness(struct mipi_dsi_device *dsi,
 struct mipi_dsi_driver {
 	struct device_driver driver;
 	int(*probe)(struct mipi_dsi_device *dsi);
-	int(*remove)(struct mipi_dsi_device *dsi);
+	void (*remove)(struct mipi_dsi_device *dsi);
 	void (*shutdown)(struct mipi_dsi_device *dsi);
 };
 
diff --git a/include/drm/drm_modes.h b/include/drm/drm_modes.h
index a80ae9639e96..b0c680e6f670 100644
--- a/include/drm/drm_modes.h
+++ b/include/drm/drm_modes.h
@@ -139,6 +139,35 @@ enum drm_mode_status {
 	.vscan = (vs), .flags = (f)
 
 /**
+ * DRM_MODE_RES_MM - Calculates the display size from resolution and DPI
+ * @res: The resolution in pixel
+ * @dpi: The number of dots per inch
+ */
+#define DRM_MODE_RES_MM(res, dpi)	\
+	(((res) * 254ul) / ((dpi) * 10ul))
+
+#define __DRM_MODE_INIT(pix, hd, vd, hd_mm, vd_mm) \
+	.type = DRM_MODE_TYPE_DRIVER, .clock = (pix), \
+	.hdisplay = (hd), .hsync_start = (hd), .hsync_end = (hd), \
+	.htotal = (hd), .vdisplay = (vd), .vsync_start = (vd), \
+	.vsync_end = (vd), .vtotal = (vd), .width_mm = (hd_mm), \
+	.height_mm = (vd_mm)
+
+/**
+ * DRM_MODE_INIT - Initialize display mode
+ * @hz: Vertical refresh rate in Hertz
+ * @hd: Horizontal resolution, width
+ * @vd: Vertical resolution, height
+ * @hd_mm: Display width in millimeters
+ * @vd_mm: Display height in millimeters
+ *
+ * This macro initializes a &drm_display_mode that contains information about
+ * refresh rate, resolution and physical size.
+ */
+#define DRM_MODE_INIT(hz, hd, vd, hd_mm, vd_mm) \
+	__DRM_MODE_INIT((hd) * (vd) * (hz) / 1000 /* kHz */, hd, vd, hd_mm, vd_mm)
+
+/**
  * DRM_SIMPLE_MODE - Simple display mode
  * @hd: Horizontal resolution, width
  * @vd: Vertical resolution, height
@@ -149,11 +178,7 @@ enum drm_mode_status {
  * resolution and physical size.
  */
 #define DRM_SIMPLE_MODE(hd, vd, hd_mm, vd_mm) \
-	.type = DRM_MODE_TYPE_DRIVER, .clock = 1 /* pass validation */, \
-	.hdisplay = (hd), .hsync_start = (hd), .hsync_end = (hd), \
-	.htotal = (hd), .vdisplay = (vd), .vsync_start = (vd), \
-	.vsync_end = (vd), .vtotal = (vd), .width_mm = (hd_mm), \
-	.height_mm = (vd_mm)
+	__DRM_MODE_INIT(1 /* pass validation */, hd, vd, hd_mm, vd_mm)
 
 #define CRTC_INTERLACE_HALVE_V	(1 << 0) /* halve V values for interlacing */
 #define CRTC_STEREO_DOUBLE	(1 << 1) /* adjust timings for stereo modes */
diff --git a/include/drm/drm_panel.h b/include/drm/drm_panel.h
index 3a271128c078..994bfcdd84c5 100644
--- a/include/drm/drm_panel.h
+++ b/include/drm/drm_panel.h
@@ -188,13 +188,6 @@ struct drm_panel {
 	 * Panel entry in registry.
 	 */
 	struct list_head list;
-
-	/**
-	 * @dsc:
-	 *
-	 * Panel DSC pps payload to be sent
-	 */
-	struct drm_dsc_config *dsc;
 };
 
 void drm_panel_init(struct drm_panel *panel, struct device *dev,
diff --git a/include/drm/drm_plane.h b/include/drm/drm_plane.h
index 89ea54652e87..447e664e49d5 100644
--- a/include/drm/drm_plane.h
+++ b/include/drm/drm_plane.h
@@ -631,7 +631,7 @@ struct drm_plane {
 	unsigned int format_count;
 	/**
 	 * @format_default: driver hasn't supplied supported formats for the
-	 * plane. Used by the drm_plane_init compatibility wrapper only.
+	 * plane. Used by the non-atomic driver compatibility wrapper only.
 	 */
 	bool format_default;
 
@@ -762,12 +762,6 @@ int drm_universal_plane_init(struct drm_device *dev,
 			     const uint64_t *format_modifiers,
 			     enum drm_plane_type type,
 			     const char *name, ...);
-int drm_plane_init(struct drm_device *dev,
-		   struct drm_plane *plane,
-		   uint32_t possible_crtcs,
-		   const struct drm_plane_funcs *funcs,
-		   const uint32_t *formats, unsigned int format_count,
-		   bool is_primary);
 void drm_plane_cleanup(struct drm_plane *plane);
 
 __printf(10, 11)
@@ -815,6 +809,50 @@ void *__drmm_universal_plane_alloc(struct drm_device *dev,
 					      format_count, format_modifiers, \
 					      plane_type, name, ##__VA_ARGS__))
 
+__printf(10, 11)
+void *__drm_universal_plane_alloc(struct drm_device *dev,
+				  size_t size, size_t offset,
+				  uint32_t possible_crtcs,
+				  const struct drm_plane_funcs *funcs,
+				  const uint32_t *formats,
+				  unsigned int format_count,
+				  const uint64_t *format_modifiers,
+				  enum drm_plane_type plane_type,
+				  const char *name, ...);
+
+/**
+ * drm_universal_plane_alloc() - Allocate and initialize an universal plane object
+ * @dev: DRM device
+ * @type: the type of the struct which contains struct &drm_plane
+ * @member: the name of the &drm_plane within @type
+ * @possible_crtcs: bitmask of possible CRTCs
+ * @funcs: callbacks for the new plane
+ * @formats: array of supported formats (DRM_FORMAT\_\*)
+ * @format_count: number of elements in @formats
+ * @format_modifiers: array of struct drm_format modifiers terminated by
+ *                    DRM_FORMAT_MOD_INVALID
+ * @plane_type: type of plane (overlay, primary, cursor)
+ * @name: printf style format string for the plane name, or NULL for default name
+ *
+ * Allocates and initializes a plane object of type @type. The caller
+ * is responsible for releasing the allocated memory with kfree().
+ *
+ * Drivers are encouraged to use drmm_universal_plane_alloc() instead.
+ *
+ * Drivers that only support the DRM_FORMAT_MOD_LINEAR modifier support may set
+ * @format_modifiers to NULL. The plane will advertise the linear modifier.
+ *
+ * Returns:
+ * Pointer to new plane, or ERR_PTR on failure.
+ */
+#define drm_universal_plane_alloc(dev, type, member, possible_crtcs, funcs, formats, \
+				   format_count, format_modifiers, plane_type, name, ...) \
+	((type *)__drm_universal_plane_alloc(dev, sizeof(type), \
+					     offsetof(type, member), \
+					     possible_crtcs, funcs, formats, \
+					     format_count, format_modifiers, \
+					     plane_type, name, ##__VA_ARGS__))
+
 /**
  * drm_plane_index - find the index of a registered plane
  * @plane: plane to find index for
diff --git a/include/drm/drm_plane_helper.h b/include/drm/drm_plane_helper.h
index 331ebd60b3a3..ff83d2621687 100644
--- a/include/drm/drm_plane_helper.h
+++ b/include/drm/drm_plane_helper.h
@@ -24,21 +24,35 @@
 #ifndef DRM_PLANE_HELPER_H
 #define DRM_PLANE_HELPER_H
 
-#include <drm/drm_rect.h>
-#include <drm/drm_crtc.h>
-#include <drm/drm_modeset_helper_vtables.h>
-#include <drm/drm_modeset_helper.h>
+#include <linux/types.h>
 
-/*
- * Drivers that don't allow primary plane scaling may pass this macro in place
- * of the min/max scale parameters of the update checker function.
+struct drm_crtc;
+struct drm_framebuffer;
+struct drm_modeset_acquire_ctx;
+struct drm_plane;
+
+int drm_plane_helper_update_primary(struct drm_plane *plane, struct drm_crtc *crtc,
+				    struct drm_framebuffer *fb,
+				    int crtc_x, int crtc_y,
+				    unsigned int crtc_w, unsigned int crtc_h,
+				    uint32_t src_x, uint32_t src_y,
+				    uint32_t src_w, uint32_t src_h,
+				    struct drm_modeset_acquire_ctx *ctx);
+int drm_plane_helper_disable_primary(struct drm_plane *plane,
+				     struct drm_modeset_acquire_ctx *ctx);
+void drm_plane_helper_destroy(struct drm_plane *plane);
+int drm_plane_helper_atomic_check(struct drm_plane *plane, struct drm_atomic_state *state);
+
+/**
+ * DRM_PLANE_NON_ATOMIC_FUNCS - Default plane functions for non-atomic drivers
  *
- * Due to src being in 16.16 fixed point and dest being in integer pixels,
- * 1<<16 represents no scaling.
+ * This macro initializes plane functions for non-atomic drivers to default
+ * values. Non-atomic interfaces are deprecated and should not be used in new
+ * drivers.
  */
-#define DRM_PLANE_HELPER_NO_SCALING (1<<16)
-
-void drm_primary_helper_destroy(struct drm_plane *plane);
-extern const struct drm_plane_funcs drm_primary_helper_funcs;
+#define DRM_PLANE_NON_ATOMIC_FUNCS \
+	.update_plane = drm_plane_helper_update_primary, \
+	.disable_plane = drm_plane_helper_disable_primary, \
+	.destroy = drm_plane_helper_destroy
 
 #endif
diff --git a/include/drm/drm_probe_helper.h b/include/drm/drm_probe_helper.h
index 8075e02aa865..5880daa14624 100644
--- a/include/drm/drm_probe_helper.h
+++ b/include/drm/drm_probe_helper.h
@@ -3,9 +3,10 @@
 #ifndef __DRM_PROBE_HELPER_H__
 #define __DRM_PROBE_HELPER_H__
 
-#include <linux/types.h>
+#include <drm/drm_modes.h>
 
 struct drm_connector;
+struct drm_crtc;
 struct drm_device;
 struct drm_modeset_acquire_ctx;
 
@@ -26,7 +27,13 @@ void drm_kms_helper_poll_disable(struct drm_device *dev);
 void drm_kms_helper_poll_enable(struct drm_device *dev);
 bool drm_kms_helper_is_poll_worker(void);
 
+enum drm_mode_status drm_crtc_helper_mode_valid_fixed(struct drm_crtc *crtc,
+						      const struct drm_display_mode *mode,
+						      const struct drm_display_mode *fixed_mode);
+
 int drm_connector_helper_get_modes_from_ddc(struct drm_connector *connector);
+int drm_connector_helper_get_modes_fixed(struct drm_connector *connector,
+					 const struct drm_display_mode *fixed_mode);
 int drm_connector_helper_get_modes(struct drm_connector *connector);
 
 #endif
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index addb135eeea6..599855c6a672 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -329,10 +329,10 @@ enum drm_gpu_sched_stat {
 };
 
 /**
- * struct drm_sched_backend_ops
+ * struct drm_sched_backend_ops - Define the backend operations
+ *	called by the scheduler
  *
- * Define the backend operations called by the scheduler,
- * these functions should be implemented in driver side.
+ * These functions should be implemented in the driver side.
  */
 struct drm_sched_backend_ops {
 	/**
@@ -409,7 +409,7 @@ struct drm_sched_backend_ops {
 };
 
 /**
- * struct drm_gpu_scheduler
+ * struct drm_gpu_scheduler - scheduler instance-specific data
  *
  * @ops: backend operations provided by the driver.
  * @hw_submission_limit: the max size of the hardware queue.
@@ -435,6 +435,7 @@ struct drm_sched_backend_ops {
  * @_score: score used when the driver doesn't provide one
  * @ready: marks if the underlying HW is ready to work
  * @free_guilty: A hit to time out handler to free the guilty job.
+ * @dev: system &struct device
  *
  * One scheduler is implemented for each hardware ring.
  */
diff --git a/include/drm/i915_pciids.h b/include/drm/i915_pciids.h
index 278031aa2e84..4a4c190f7698 100644
--- a/include/drm/i915_pciids.h
+++ b/include/drm/i915_pciids.h
@@ -641,6 +641,7 @@
 	INTEL_VGA_DEVICE(0x4682, info), \
 	INTEL_VGA_DEVICE(0x4688, info), \
 	INTEL_VGA_DEVICE(0x468A, info), \
+	INTEL_VGA_DEVICE(0x468B, info), \
 	INTEL_VGA_DEVICE(0x4690, info), \
 	INTEL_VGA_DEVICE(0x4692, info), \
 	INTEL_VGA_DEVICE(0x4693, info)
diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h
index 2d524f8b0802..44a538ee5e2a 100644
--- a/include/drm/ttm/ttm_bo_api.h
+++ b/include/drm/ttm/ttm_bo_api.h
@@ -317,93 +317,16 @@ void ttm_bo_unlock_delayed_workqueue(struct ttm_device *bdev, int resched);
 bool ttm_bo_eviction_valuable(struct ttm_buffer_object *bo,
 			      const struct ttm_place *place);
 
-/**
- * ttm_bo_init_reserved
- *
- * @bdev: Pointer to a ttm_device struct.
- * @bo: Pointer to a ttm_buffer_object to be initialized.
- * @size: Requested size of buffer object.
- * @type: Requested type of buffer object.
- * @placement: Initial placement for buffer object.
- * @page_alignment: Data alignment in pages.
- * @ctx: TTM operation context for memory allocation.
- * @sg: Scatter-gather table.
- * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
- * @destroy: Destroy function. Use NULL for kfree().
- *
- * This function initializes a pre-allocated struct ttm_buffer_object.
- * As this object may be part of a larger structure, this function,
- * together with the @destroy function,
- * enables driver-specific objects derived from a ttm_buffer_object.
- *
- * On successful return, the caller owns an object kref to @bo. The kref and
- * list_kref are usually set to 1, but note that in some situations, other
- * tasks may already be holding references to @bo as well.
- * Furthermore, if resv == NULL, the buffer's reservation lock will be held,
- * and it is the caller's responsibility to call ttm_bo_unreserve.
- *
- * If a failure occurs, the function will call the @destroy function, or
- * kfree() if @destroy is NULL. Thus, after a failure, dereferencing @bo is
- * illegal and will likely cause memory corruption.
- *
- * Returns
- * -ENOMEM: Out of memory.
- * -EINVAL: Invalid placement flags.
- * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
- */
-
-int ttm_bo_init_reserved(struct ttm_device *bdev,
-			 struct ttm_buffer_object *bo,
-			 size_t size, enum ttm_bo_type type,
-			 struct ttm_placement *placement,
-			 uint32_t page_alignment,
-			 struct ttm_operation_ctx *ctx,
+int ttm_bo_init_reserved(struct ttm_device *bdev, struct ttm_buffer_object *bo,
+			 enum ttm_bo_type type, struct ttm_placement *placement,
+			 uint32_t alignment, struct ttm_operation_ctx *ctx,
+			 struct sg_table *sg, struct dma_resv *resv,
+			 void (*destroy) (struct ttm_buffer_object *));
+int ttm_bo_init_validate(struct ttm_device *bdev, struct ttm_buffer_object *bo,
+			 enum ttm_bo_type type, struct ttm_placement *placement,
+			 uint32_t alignment, bool interruptible,
 			 struct sg_table *sg, struct dma_resv *resv,
 			 void (*destroy) (struct ttm_buffer_object *));
-
-/**
- * ttm_bo_init
- *
- * @bdev: Pointer to a ttm_device struct.
- * @bo: Pointer to a ttm_buffer_object to be initialized.
- * @size: Requested size of buffer object.
- * @type: Requested type of buffer object.
- * @placement: Initial placement for buffer object.
- * @page_alignment: Data alignment in pages.
- * @interruptible: If needing to sleep to wait for GPU resources,
- * sleep interruptible.
- * pinned in physical memory. If this behaviour is not desired, this member
- * holds a pointer to a persistent shmem object. Typically, this would
- * point to the shmem object backing a GEM object if TTM is used to back a
- * GEM user interface.
- * @sg: Scatter-gather table.
- * @resv: Pointer to a dma_resv, or NULL to let ttm allocate one.
- * @destroy: Destroy function. Use NULL for kfree().
- *
- * This function initializes a pre-allocated struct ttm_buffer_object.
- * As this object may be part of a larger structure, this function,
- * together with the @destroy function,
- * enables driver-specific objects derived from a ttm_buffer_object.
- *
- * On successful return, the caller owns an object kref to @bo. The kref and
- * list_kref are usually set to 1, but note that in some situations, other
- * tasks may already be holding references to @bo as well.
- *
- * If a failure occurs, the function will call the @destroy function, or
- * kfree() if @destroy is NULL. Thus, after a failure, dereferencing @bo is
- * illegal and will likely cause memory corruption.
- *
- * Returns
- * -ENOMEM: Out of memory.
- * -EINVAL: Invalid placement flags.
- * -ERESTARTSYS: Interrupted by signal while sleeping waiting for resources.
- */
-int ttm_bo_init(struct ttm_device *bdev, struct ttm_buffer_object *bo,
-		size_t size, enum ttm_bo_type type,
-		struct ttm_placement *placement,
-		uint32_t page_alignment, bool interrubtible,
-		struct sg_table *sg, struct dma_resv *resv,
-		void (*destroy) (struct ttm_buffer_object *));
 
 /**
  * ttm_kmap_obj_virtual
diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h
index 897b88f0bd59..1afa891f488a 100644
--- a/include/drm/ttm/ttm_bo_driver.h
+++ b/include/drm/ttm/ttm_bo_driver.h
@@ -106,7 +106,7 @@ static inline int ttm_bo_reserve(struct ttm_buffer_object *bo,
 				 bool interruptible, bool no_wait,
 				 struct ww_acquire_ctx *ticket)
 {
-	int ret = 0;
+	int ret;
 
 	if (no_wait) {
 		bool success;
diff --git a/include/drm/ttm/ttm_resource.h b/include/drm/ttm/ttm_resource.h
index ca89a48c2460..5afc6d664fde 100644
--- a/include/drm/ttm/ttm_resource.h
+++ b/include/drm/ttm/ttm_resource.h
@@ -89,6 +89,38 @@ struct ttm_resource_manager_func {
 		     struct ttm_resource *res);
 
 	/**
+	 * struct ttm_resource_manager_func member intersects
+	 *
+	 * @man: Pointer to a memory type manager.
+	 * @res: Pointer to a struct ttm_resource to be checked.
+	 * @place: Placement to check against.
+	 * @size: Size of the check.
+	 *
+	 * Test if @res intersects with @place + @size. Used to judge if
+	 * evictions are valueable or not.
+	 */
+	bool (*intersects)(struct ttm_resource_manager *man,
+			   struct ttm_resource *res,
+			   const struct ttm_place *place,
+			   size_t size);
+
+	/**
+	 * struct ttm_resource_manager_func member compatible
+	 *
+	 * @man: Pointer to a memory type manager.
+	 * @res: Pointer to a struct ttm_resource to be checked.
+	 * @place: Placement to check against.
+	 * @size: Size of the check.
+	 *
+	 * Test if @res compatible with @place + @size. Used to check of
+	 * the need to move the backing store or not.
+	 */
+	bool (*compatible)(struct ttm_resource_manager *man,
+			   struct ttm_resource *res,
+			   const struct ttm_place *place,
+			   size_t size);
+
+	/**
 	 * struct ttm_resource_manager_func member debug
 	 *
 	 * @man: Pointer to a memory type manager.
@@ -329,6 +361,14 @@ int ttm_resource_alloc(struct ttm_buffer_object *bo,
 		       const struct ttm_place *place,
 		       struct ttm_resource **res);
 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res);
+bool ttm_resource_intersects(struct ttm_device *bdev,
+			     struct ttm_resource *res,
+			     const struct ttm_place *place,
+			     size_t size);
+bool ttm_resource_compatible(struct ttm_device *bdev,
+			     struct ttm_resource *res,
+			     const struct ttm_place *place,
+			     size_t size);
 bool ttm_resource_compat(struct ttm_resource *res,
 			 struct ttm_placement *placement);
 void ttm_resource_set_bo(struct ttm_resource *res,
diff --git a/include/dt-bindings/clock/exynos850.h b/include/dt-bindings/clock/exynos850.h
index 0b6a3c6a7c90..88d5289883d3 100644
--- a/include/dt-bindings/clock/exynos850.h
+++ b/include/dt-bindings/clock/exynos850.h
@@ -58,7 +58,34 @@
 #define CLK_MOUT_CLKCMU_APM_BUS		46
 #define CLK_DOUT_CLKCMU_APM_BUS		47
 #define CLK_GOUT_CLKCMU_APM_BUS		48
-#define TOP_NR_CLK			49
+#define CLK_MOUT_AUD			49
+#define CLK_GOUT_AUD			50
+#define CLK_DOUT_AUD			51
+#define CLK_MOUT_IS_BUS			52
+#define CLK_MOUT_IS_ITP			53
+#define CLK_MOUT_IS_VRA			54
+#define CLK_MOUT_IS_GDC			55
+#define CLK_GOUT_IS_BUS			56
+#define CLK_GOUT_IS_ITP			57
+#define CLK_GOUT_IS_VRA			58
+#define CLK_GOUT_IS_GDC			59
+#define CLK_DOUT_IS_BUS			60
+#define CLK_DOUT_IS_ITP			61
+#define CLK_DOUT_IS_VRA			62
+#define CLK_DOUT_IS_GDC			63
+#define CLK_MOUT_MFCMSCL_MFC		64
+#define CLK_MOUT_MFCMSCL_M2M		65
+#define CLK_MOUT_MFCMSCL_MCSC		66
+#define CLK_MOUT_MFCMSCL_JPEG		67
+#define CLK_GOUT_MFCMSCL_MFC		68
+#define CLK_GOUT_MFCMSCL_M2M		69
+#define CLK_GOUT_MFCMSCL_MCSC		70
+#define CLK_GOUT_MFCMSCL_JPEG		71
+#define CLK_DOUT_MFCMSCL_MFC		72
+#define CLK_DOUT_MFCMSCL_M2M		73
+#define CLK_DOUT_MFCMSCL_MCSC		74
+#define CLK_DOUT_MFCMSCL_JPEG		75
+#define TOP_NR_CLK			76
 
 /* CMU_APM */
 #define CLK_RCO_I3C_PMIC		1
@@ -87,6 +114,69 @@
 #define CLK_GOUT_SYSREG_APM_PCLK	24
 #define APM_NR_CLK			25
 
+/* CMU_AUD */
+#define CLK_DOUT_AUD_AUDIF		1
+#define CLK_DOUT_AUD_BUSD		2
+#define CLK_DOUT_AUD_BUSP		3
+#define CLK_DOUT_AUD_CNT		4
+#define CLK_DOUT_AUD_CPU		5
+#define CLK_DOUT_AUD_CPU_ACLK		6
+#define CLK_DOUT_AUD_CPU_PCLKDBG	7
+#define CLK_DOUT_AUD_FM			8
+#define CLK_DOUT_AUD_FM_SPDY		9
+#define CLK_DOUT_AUD_MCLK		10
+#define CLK_DOUT_AUD_UAIF0		11
+#define CLK_DOUT_AUD_UAIF1		12
+#define CLK_DOUT_AUD_UAIF2		13
+#define CLK_DOUT_AUD_UAIF3		14
+#define CLK_DOUT_AUD_UAIF4		15
+#define CLK_DOUT_AUD_UAIF5		16
+#define CLK_DOUT_AUD_UAIF6		17
+#define CLK_FOUT_AUD_PLL		18
+#define CLK_GOUT_AUD_ABOX_ACLK		19
+#define CLK_GOUT_AUD_ASB_CCLK		20
+#define CLK_GOUT_AUD_CA32_CCLK		21
+#define CLK_GOUT_AUD_CNT_BCLK		22
+#define CLK_GOUT_AUD_CODEC_MCLK		23
+#define CLK_GOUT_AUD_DAP_CCLK		24
+#define CLK_GOUT_AUD_GPIO_PCLK		25
+#define CLK_GOUT_AUD_PPMU_ACLK		26
+#define CLK_GOUT_AUD_PPMU_PCLK		27
+#define CLK_GOUT_AUD_SPDY_BCLK		28
+#define CLK_GOUT_AUD_SYSMMU_CLK		29
+#define CLK_GOUT_AUD_SYSREG_PCLK	30
+#define CLK_GOUT_AUD_TZPC_PCLK		31
+#define CLK_GOUT_AUD_UAIF0_BCLK		32
+#define CLK_GOUT_AUD_UAIF1_BCLK		33
+#define CLK_GOUT_AUD_UAIF2_BCLK		34
+#define CLK_GOUT_AUD_UAIF3_BCLK		35
+#define CLK_GOUT_AUD_UAIF4_BCLK		36
+#define CLK_GOUT_AUD_UAIF5_BCLK		37
+#define CLK_GOUT_AUD_UAIF6_BCLK		38
+#define CLK_GOUT_AUD_WDT_PCLK		39
+#define CLK_MOUT_AUD_CPU		40
+#define CLK_MOUT_AUD_CPU_HCH		41
+#define CLK_MOUT_AUD_CPU_USER		42
+#define CLK_MOUT_AUD_FM			43
+#define CLK_MOUT_AUD_PLL		44
+#define CLK_MOUT_AUD_TICK_USB_USER	45
+#define CLK_MOUT_AUD_UAIF0		46
+#define CLK_MOUT_AUD_UAIF1		47
+#define CLK_MOUT_AUD_UAIF2		48
+#define CLK_MOUT_AUD_UAIF3		49
+#define CLK_MOUT_AUD_UAIF4		50
+#define CLK_MOUT_AUD_UAIF5		51
+#define CLK_MOUT_AUD_UAIF6		52
+#define IOCLK_AUDIOCDCLK0		53
+#define IOCLK_AUDIOCDCLK1		54
+#define IOCLK_AUDIOCDCLK2		55
+#define IOCLK_AUDIOCDCLK3		56
+#define IOCLK_AUDIOCDCLK4		57
+#define IOCLK_AUDIOCDCLK5		58
+#define IOCLK_AUDIOCDCLK6		59
+#define TICK_USB			60
+#define AUD_NR_CLK			61
+
 /* CMU_CMGP */
 #define CLK_RCO_CMGP			1
 #define CLK_MOUT_CMGP_ADC		2
@@ -121,6 +211,50 @@
 #define CLK_GOUT_SYSREG_HSI_PCLK	13
 #define HSI_NR_CLK			14
 
+/* CMU_IS */
+#define CLK_MOUT_IS_BUS_USER		1
+#define CLK_MOUT_IS_ITP_USER		2
+#define CLK_MOUT_IS_VRA_USER		3
+#define CLK_MOUT_IS_GDC_USER		4
+#define CLK_DOUT_IS_BUSP		5
+#define CLK_GOUT_IS_CMU_IS_PCLK		6
+#define CLK_GOUT_IS_CSIS0_ACLK		7
+#define CLK_GOUT_IS_CSIS1_ACLK		8
+#define CLK_GOUT_IS_CSIS2_ACLK		9
+#define CLK_GOUT_IS_TZPC_PCLK		10
+#define CLK_GOUT_IS_CSIS_DMA_CLK	11
+#define CLK_GOUT_IS_GDC_CLK		12
+#define CLK_GOUT_IS_IPP_CLK		13
+#define CLK_GOUT_IS_ITP_CLK		14
+#define CLK_GOUT_IS_MCSC_CLK		15
+#define CLK_GOUT_IS_VRA_CLK		16
+#define CLK_GOUT_IS_PPMU_IS0_ACLK	17
+#define CLK_GOUT_IS_PPMU_IS0_PCLK	18
+#define CLK_GOUT_IS_PPMU_IS1_ACLK	19
+#define CLK_GOUT_IS_PPMU_IS1_PCLK	20
+#define CLK_GOUT_IS_SYSMMU_IS0_CLK	21
+#define CLK_GOUT_IS_SYSMMU_IS1_CLK	22
+#define CLK_GOUT_IS_SYSREG_PCLK		23
+#define IS_NR_CLK			24
+
+/* CMU_MFCMSCL */
+#define CLK_MOUT_MFCMSCL_MFC_USER		1
+#define CLK_MOUT_MFCMSCL_M2M_USER		2
+#define CLK_MOUT_MFCMSCL_MCSC_USER		3
+#define CLK_MOUT_MFCMSCL_JPEG_USER		4
+#define CLK_DOUT_MFCMSCL_BUSP			5
+#define CLK_GOUT_MFCMSCL_CMU_MFCMSCL_PCLK	6
+#define CLK_GOUT_MFCMSCL_TZPC_PCLK		7
+#define CLK_GOUT_MFCMSCL_JPEG_ACLK		8
+#define CLK_GOUT_MFCMSCL_M2M_ACLK		9
+#define CLK_GOUT_MFCMSCL_MCSC_CLK		10
+#define CLK_GOUT_MFCMSCL_MFC_ACLK		11
+#define CLK_GOUT_MFCMSCL_PPMU_ACLK		12
+#define CLK_GOUT_MFCMSCL_PPMU_PCLK		13
+#define CLK_GOUT_MFCMSCL_SYSMMU_CLK		14
+#define CLK_GOUT_MFCMSCL_SYSREG_PCLK		15
+#define MFCMSCL_NR_CLK				16
+
 /* CMU_PERI */
 #define CLK_MOUT_PERI_BUS_USER		1
 #define CLK_MOUT_PERI_UART_USER		2
diff --git a/include/dt-bindings/clock/imx8mm-clock.h b/include/dt-bindings/clock/imx8mm-clock.h
index 47c6f7f9582c..1f768b2eeb1a 100644
--- a/include/dt-bindings/clock/imx8mm-clock.h
+++ b/include/dt-bindings/clock/imx8mm-clock.h
@@ -281,7 +281,6 @@
 #define IMX8MM_CLK_CLKOUT2_DIV			256
 #define IMX8MM_CLK_CLKOUT2			257
 
-
 #define IMX8MM_CLK_END				258
 
 #endif
diff --git a/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h b/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h
index 20ef2ea673f3..22dcd47d4513 100644
--- a/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h
+++ b/include/dt-bindings/clock/qcom,lpassaudiocc-sc7280.h
@@ -24,6 +24,11 @@
 #define LPASS_AUDIO_CC_RX_MCLK_CLK			14
 #define LPASS_AUDIO_CC_RX_MCLK_CLK_SRC			15
 
+/* LPASS AUDIO CC CSR */
+#define LPASS_AUDIO_SWR_RX_CGCR				0
+#define LPASS_AUDIO_SWR_TX_CGCR				1
+#define LPASS_AUDIO_SWR_WSA_CGCR			2
+
 /* LPASS_AON_CC clocks */
 #define LPASS_AON_CC_PLL				0
 #define LPASS_AON_CC_PLL_OUT_EVEN			1
diff --git a/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h b/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h
index 28ed2a07aacc..0324c69ce968 100644
--- a/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h
+++ b/include/dt-bindings/clock/qcom,lpasscorecc-sc7280.h
@@ -19,6 +19,8 @@
 #define LPASS_CORE_CC_LPM_CORE_CLK			9
 #define LPASS_CORE_CC_LPM_MEM0_CORE_CLK			10
 #define LPASS_CORE_CC_SYSNOC_MPORT_CORE_CLK		11
+#define LPASS_CORE_CC_EXT_MCLK0_CLK			12
+#define LPASS_CORE_CC_EXT_MCLK0_CLK_SRC			13
 
 /* LPASS_CORE_CC power domains */
 #define LPASS_CORE_CC_LPASS_CORE_HM_GDSC		0
diff --git a/include/dt-bindings/clock/samsung,exynosautov9.h b/include/dt-bindings/clock/samsung,exynosautov9.h
index ea9f91b4eb1a..42133af6d6b9 100644
--- a/include/dt-bindings/clock/samsung,exynosautov9.h
+++ b/include/dt-bindings/clock/samsung,exynosautov9.h
@@ -185,6 +185,74 @@
 
 #define CORE_NR_CLK			6
 
+/* CMU_FSYS0 */
+#define CLK_MOUT_FSYS0_BUS_USER		1
+#define CLK_MOUT_FSYS0_PCIE_USER	2
+#define CLK_GOUT_FSYS0_BUS_PCLK		3
+
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_REFCLK		4
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_REFCLK		5
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_DBI_ACLK	6
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_MSTR_ACLK	7
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X1_SLV_ACLK	8
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_DBI_ACLK	9
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_MSTR_ACLK	10
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_SLV_ACLK	11
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L0_X2_PIPE_CLK	12
+#define CLK_GOUT_FSYS0_PCIE_GEN3A_2L0_CLK		13
+#define CLK_GOUT_FSYS0_PCIE_GEN3B_2L0_CLK		14
+
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_REFCLK		15
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_REFCLK		16
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_DBI_ACLK	17
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_MSTR_ACLK	18
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X1_SLV_ACLK	19
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_DBI_ACLK	20
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_MSTR_ACLK	21
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_SLV_ACLK	22
+#define CLK_GOUT_FSYS0_PCIE_GEN3_2L1_X2_PIPE_CLK	23
+#define CLK_GOUT_FSYS0_PCIE_GEN3A_2L1_CLK		24
+#define CLK_GOUT_FSYS0_PCIE_GEN3B_2L1_CLK		25
+
+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_REFCLK		26
+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_REFCLK		27
+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_DBI_ACLK		28
+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_MSTR_ACLK	29
+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X2_SLV_ACLK		30
+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_DBI_ACLK		31
+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_MSTR_ACLK	32
+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_SLV_ACLK		33
+#define CLK_GOUT_FSYS0_PCIE_GEN3_4L_X4_PIPE_CLK		34
+#define CLK_GOUT_FSYS0_PCIE_GEN3A_4L_CLK		35
+#define CLK_GOUT_FSYS0_PCIE_GEN3B_4L_CLK		36
+
+#define FSYS0_NR_CLK			37
+
+/* CMU_FSYS1 */
+#define FOUT_MMC_PLL				1
+
+#define CLK_MOUT_FSYS1_BUS_USER			2
+#define CLK_MOUT_FSYS1_MMC_PLL			3
+#define CLK_MOUT_FSYS1_MMC_CARD_USER		4
+#define CLK_MOUT_FSYS1_USBDRD_USER		5
+#define CLK_MOUT_FSYS1_MMC_CARD			6
+
+#define CLK_DOUT_FSYS1_MMC_CARD			7
+
+#define CLK_GOUT_FSYS1_PCLK			8
+#define CLK_GOUT_FSYS1_MMC_CARD_SDCLKIN		9
+#define CLK_GOUT_FSYS1_MMC_CARD_ACLK		10
+#define CLK_GOUT_FSYS1_USB20DRD_0_REFCLK	11
+#define CLK_GOUT_FSYS1_USB20DRD_1_REFCLK	12
+#define CLK_GOUT_FSYS1_USB30DRD_0_REFCLK	13
+#define CLK_GOUT_FSYS1_USB30DRD_1_REFCLK	14
+#define CLK_GOUT_FSYS1_USB20_0_ACLK		15
+#define CLK_GOUT_FSYS1_USB20_1_ACLK		16
+#define CLK_GOUT_FSYS1_USB30_0_ACLK		17
+#define CLK_GOUT_FSYS1_USB30_1_ACLK		18
+
+#define FSYS1_NR_CLK				19
+
 /* CMU_FSYS2 */
 #define CLK_MOUT_FSYS2_BUS_USER		1
 #define CLK_MOUT_FSYS2_UFS_EMBD_USER	2
@@ -226,21 +294,21 @@
 #define CLK_GOUT_PERIC0_IPCLK_8		28
 #define CLK_GOUT_PERIC0_IPCLK_9		29
 #define CLK_GOUT_PERIC0_IPCLK_10	30
-#define CLK_GOUT_PERIC0_IPCLK_11	30
-#define CLK_GOUT_PERIC0_PCLK_0		31
-#define CLK_GOUT_PERIC0_PCLK_1		32
-#define CLK_GOUT_PERIC0_PCLK_2		33
-#define CLK_GOUT_PERIC0_PCLK_3		34
-#define CLK_GOUT_PERIC0_PCLK_4		35
-#define CLK_GOUT_PERIC0_PCLK_5		36
-#define CLK_GOUT_PERIC0_PCLK_6		37
-#define CLK_GOUT_PERIC0_PCLK_7		38
-#define CLK_GOUT_PERIC0_PCLK_8		39
-#define CLK_GOUT_PERIC0_PCLK_9		40
-#define CLK_GOUT_PERIC0_PCLK_10		41
-#define CLK_GOUT_PERIC0_PCLK_11		42
-
-#define PERIC0_NR_CLK			43
+#define CLK_GOUT_PERIC0_IPCLK_11	31
+#define CLK_GOUT_PERIC0_PCLK_0		32
+#define CLK_GOUT_PERIC0_PCLK_1		33
+#define CLK_GOUT_PERIC0_PCLK_2		34
+#define CLK_GOUT_PERIC0_PCLK_3		35
+#define CLK_GOUT_PERIC0_PCLK_4		36
+#define CLK_GOUT_PERIC0_PCLK_5		37
+#define CLK_GOUT_PERIC0_PCLK_6		38
+#define CLK_GOUT_PERIC0_PCLK_7		39
+#define CLK_GOUT_PERIC0_PCLK_8		40
+#define CLK_GOUT_PERIC0_PCLK_9		41
+#define CLK_GOUT_PERIC0_PCLK_10		42
+#define CLK_GOUT_PERIC0_PCLK_11		43
+
+#define PERIC0_NR_CLK			44
 
 /* CMU_PERIC1 */
 #define CLK_MOUT_PERIC1_BUS_USER	1
@@ -272,21 +340,21 @@
 #define CLK_GOUT_PERIC1_IPCLK_8		28
 #define CLK_GOUT_PERIC1_IPCLK_9		29
 #define CLK_GOUT_PERIC1_IPCLK_10	30
-#define CLK_GOUT_PERIC1_IPCLK_11	30
-#define CLK_GOUT_PERIC1_PCLK_0		31
-#define CLK_GOUT_PERIC1_PCLK_1		32
-#define CLK_GOUT_PERIC1_PCLK_2		33
-#define CLK_GOUT_PERIC1_PCLK_3		34
-#define CLK_GOUT_PERIC1_PCLK_4		35
-#define CLK_GOUT_PERIC1_PCLK_5		36
-#define CLK_GOUT_PERIC1_PCLK_6		37
-#define CLK_GOUT_PERIC1_PCLK_7		38
-#define CLK_GOUT_PERIC1_PCLK_8		39
-#define CLK_GOUT_PERIC1_PCLK_9		40
-#define CLK_GOUT_PERIC1_PCLK_10		41
-#define CLK_GOUT_PERIC1_PCLK_11		42
-
-#define PERIC1_NR_CLK			43
+#define CLK_GOUT_PERIC1_IPCLK_11	31
+#define CLK_GOUT_PERIC1_PCLK_0		32
+#define CLK_GOUT_PERIC1_PCLK_1		33
+#define CLK_GOUT_PERIC1_PCLK_2		34
+#define CLK_GOUT_PERIC1_PCLK_3		35
+#define CLK_GOUT_PERIC1_PCLK_4		36
+#define CLK_GOUT_PERIC1_PCLK_5		37
+#define CLK_GOUT_PERIC1_PCLK_6		38
+#define CLK_GOUT_PERIC1_PCLK_7		39
+#define CLK_GOUT_PERIC1_PCLK_8		40
+#define CLK_GOUT_PERIC1_PCLK_9		41
+#define CLK_GOUT_PERIC1_PCLK_10		42
+#define CLK_GOUT_PERIC1_PCLK_11		43
+
+#define PERIC1_NR_CLK			44
 
 /* CMU_PERIS */
 #define CLK_MOUT_PERIS_BUS_USER		1
diff --git a/include/dt-bindings/firmware/imx/rsrc.h b/include/dt-bindings/firmware/imx/rsrc.h
index 43885056557c..1675de05ad33 100644
--- a/include/dt-bindings/firmware/imx/rsrc.h
+++ b/include/dt-bindings/firmware/imx/rsrc.h
@@ -37,10 +37,14 @@
 #define IMX_SC_R_DC_0_BLIT2		21
 #define IMX_SC_R_DC_0_BLIT_OUT		22
 #define IMX_SC_R_PERF			23
+#define IMX_SC_R_USB_1_PHY		24
 #define IMX_SC_R_DC_0_WARP		25
+#define IMX_SC_R_V2X_MU_0		26
+#define IMX_SC_R_V2X_MU_1		27
 #define IMX_SC_R_DC_0_VIDEO0		28
 #define IMX_SC_R_DC_0_VIDEO1		29
 #define IMX_SC_R_DC_0_FRAC0		30
+#define IMX_SC_R_V2X_MU_2		31
 #define IMX_SC_R_DC_0			32
 #define IMX_SC_R_GPU_2_PID0		33
 #define IMX_SC_R_DC_0_PLL_0		34
@@ -49,7 +53,10 @@
 #define IMX_SC_R_DC_1_BLIT1		37
 #define IMX_SC_R_DC_1_BLIT2		38
 #define IMX_SC_R_DC_1_BLIT_OUT		39
+#define IMX_SC_R_V2X_MU_3		40
+#define IMX_SC_R_V2X_MU_4		41
 #define IMX_SC_R_DC_1_WARP		42
+#define IMX_SC_R_SECVIO			44
 #define IMX_SC_R_DC_1_VIDEO0		45
 #define IMX_SC_R_DC_1_VIDEO1		46
 #define IMX_SC_R_DC_1_FRAC0		47
diff --git a/include/dt-bindings/interrupt-controller/irqc-rzg2l.h b/include/dt-bindings/interrupt-controller/irqc-rzg2l.h
new file mode 100644
index 000000000000..34ce778885a1
--- /dev/null
+++ b/include/dt-bindings/interrupt-controller/irqc-rzg2l.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * This header provides constants for Renesas RZ/G2L family IRQC bindings.
+ *
+ * Copyright (C) 2022 Renesas Electronics Corp.
+ *
+ */
+
+#ifndef __DT_BINDINGS_IRQC_RZG2L_H
+#define __DT_BINDINGS_IRQC_RZG2L_H
+
+/* NMI maps to SPI0 */
+#define RZG2L_NMI	0
+
+/* IRQ0-7 map to SPI1-8 */
+#define RZG2L_IRQ0	1
+#define RZG2L_IRQ1	2
+#define RZG2L_IRQ2	3
+#define RZG2L_IRQ3	4
+#define RZG2L_IRQ4	5
+#define RZG2L_IRQ5	6
+#define RZG2L_IRQ6	7
+#define RZG2L_IRQ7	8
+
+#endif /* __DT_BINDINGS_IRQC_RZG2L_H */
diff --git a/include/dt-bindings/memory/tegra234-mc.h b/include/dt-bindings/memory/tegra234-mc.h
index 62987b47ce81..bd71cc1d7990 100644
--- a/include/dt-bindings/memory/tegra234-mc.h
+++ b/include/dt-bindings/memory/tegra234-mc.h
@@ -34,6 +34,16 @@
 #define TEGRA234_SID_HOST1X	0x27
 #define TEGRA234_SID_VIC	0x34
 
+/* Shared stream IDs */
+#define TEGRA234_SID_HOST1X_CTX0	0x35
+#define TEGRA234_SID_HOST1X_CTX1	0x36
+#define TEGRA234_SID_HOST1X_CTX2	0x37
+#define TEGRA234_SID_HOST1X_CTX3	0x38
+#define TEGRA234_SID_HOST1X_CTX4	0x39
+#define TEGRA234_SID_HOST1X_CTX5	0x3a
+#define TEGRA234_SID_HOST1X_CTX6	0x3b
+#define TEGRA234_SID_HOST1X_CTX7	0x3c
+
 /*
  * memory client IDs
  */
diff --git a/include/dt-bindings/pinctrl/k3.h b/include/dt-bindings/pinctrl/k3.h
index a5204ab91d3e..54df633f9bfe 100644
--- a/include/dt-bindings/pinctrl/k3.h
+++ b/include/dt-bindings/pinctrl/k3.h
@@ -29,19 +29,22 @@
 #define PIN_INPUT_PULLUP	(INPUT_EN | PULL_UP)
 #define PIN_INPUT_PULLDOWN	(INPUT_EN | PULL_DOWN)
 
+#define AM62AX_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
+#define AM62AX_MCU_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
+
+#define AM62X_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
+#define AM62X_MCU_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
+
+#define AM64X_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
+#define AM64X_MCU_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
+
 #define AM65X_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
 #define AM65X_WKUP_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
 
 #define J721E_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
 #define J721E_WKUP_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
 
-#define AM64X_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
-#define AM64X_MCU_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
-
 #define J721S2_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
 #define J721S2_WKUP_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
 
-#define AM62X_IOPAD(pa, val, muxmode)		(((pa) & 0x1fff)) ((val) | (muxmode))
-#define AM62X_MCU_IOPAD(pa, val, muxmode)	(((pa) & 0x1fff)) ((val) | (muxmode))
-
 #endif
diff --git a/include/dt-bindings/power/fsl,imx93-power.h b/include/dt-bindings/power/fsl,imx93-power.h
new file mode 100644
index 000000000000..17f9f015bf7d
--- /dev/null
+++ b/include/dt-bindings/power/fsl,imx93-power.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/*
+ *  Copyright 2022 NXP
+ */
+
+#ifndef __DT_BINDINGS_IMX93_POWER_H__
+#define __DT_BINDINGS_IMX93_POWER_H__
+
+#define IMX93_MEDIABLK_PD_MIPI_DSI		0
+#define IMX93_MEDIABLK_PD_MIPI_CSI		1
+#define IMX93_MEDIABLK_PD_PXP			2
+#define IMX93_MEDIABLK_PD_LCDIF			3
+#define IMX93_MEDIABLK_PD_ISI			4
+
+#endif
diff --git a/include/dt-bindings/power/imx8mp-power.h b/include/dt-bindings/power/imx8mp-power.h
index 7789bcca3223..2fe3c2abad13 100644
--- a/include/dt-bindings/power/imx8mp-power.h
+++ b/include/dt-bindings/power/imx8mp-power.h
@@ -49,5 +49,11 @@
 #define IMX8MP_HDMIBLK_PD_TRNG				4
 #define IMX8MP_HDMIBLK_PD_HDMI_TX			5
 #define IMX8MP_HDMIBLK_PD_HDMI_TX_PHY			6
+#define IMX8MP_HDMIBLK_PD_HDCP				7
+#define IMX8MP_HDMIBLK_PD_HRV				8
+
+#define IMX8MP_VPUBLK_PD_G1				0
+#define IMX8MP_VPUBLK_PD_G2				1
+#define IMX8MP_VPUBLK_PD_VC8000E			2
 
 #endif
diff --git a/include/dt-bindings/power/qcom-rpmpd.h b/include/dt-bindings/power/qcom-rpmpd.h
index d81de63ae31c..f5f82dde7399 100644
--- a/include/dt-bindings/power/qcom-rpmpd.h
+++ b/include/dt-bindings/power/qcom-rpmpd.h
@@ -36,6 +36,18 @@
 #define SM6350_MSS	4
 #define SM6350_MX	5
 
+/* SM6350 Power Domain Indexes */
+#define SM6375_VDDCX		0
+#define SM6375_VDDCX_AO	1
+#define SM6375_VDDCX_VFL	2
+#define SM6375_VDDMX		3
+#define SM6375_VDDMX_AO	4
+#define SM6375_VDDMX_VFL	5
+#define SM6375_VDDGX		6
+#define SM6375_VDDGX_AO	7
+#define SM6375_VDD_LPI_CX	8
+#define SM6375_VDD_LPI_MX	9
+
 /* SM8150 Power Domain Indexes */
 #define SM8150_MSS	0
 #define SM8150_EBI	1
diff --git a/include/dt-bindings/power/rk3588-power.h b/include/dt-bindings/power/rk3588-power.h
new file mode 100644
index 000000000000..1b92fec013cb
--- /dev/null
+++ b/include/dt-bindings/power/rk3588-power.h
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: (GPL-2.0 or MIT) */
+#ifndef __DT_BINDINGS_POWER_RK3588_POWER_H__
+#define __DT_BINDINGS_POWER_RK3588_POWER_H__
+
+/* VD_LITDSU */
+#define RK3588_PD_CPU_0		0
+#define RK3588_PD_CPU_1		1
+#define RK3588_PD_CPU_2		2
+#define RK3588_PD_CPU_3		3
+
+/* VD_BIGCORE0 */
+#define RK3588_PD_CPU_4		4
+#define RK3588_PD_CPU_5		5
+
+/* VD_BIGCORE1 */
+#define RK3588_PD_CPU_6		6
+#define RK3588_PD_CPU_7		7
+
+/* VD_NPU */
+#define RK3588_PD_NPU		8
+#define RK3588_PD_NPUTOP	9
+#define RK3588_PD_NPU1		10
+#define RK3588_PD_NPU2		11
+
+/* VD_GPU */
+#define RK3588_PD_GPU		12
+
+/* VD_VCODEC */
+#define RK3588_PD_VCODEC	13
+#define RK3588_PD_RKVDEC0	14
+#define RK3588_PD_RKVDEC1	15
+#define RK3588_PD_VENC0		16
+#define RK3588_PD_VENC1		17
+
+/* VD_DD01 */
+#define RK3588_PD_DDR01		18
+
+/* VD_DD23 */
+#define RK3588_PD_DDR23		19
+
+/* VD_LOGIC */
+#define RK3588_PD_CENTER	20
+#define RK3588_PD_VDPU		21
+#define RK3588_PD_RGA30		22
+#define RK3588_PD_AV1		23
+#define RK3588_PD_VOP		24
+#define RK3588_PD_VO0		25
+#define RK3588_PD_VO1		26
+#define RK3588_PD_VI		27
+#define RK3588_PD_ISP1		28
+#define RK3588_PD_FEC		29
+#define RK3588_PD_RGA31		30
+#define RK3588_PD_USB		31
+#define RK3588_PD_PHP		32
+#define RK3588_PD_GMAC		33
+#define RK3588_PD_PCIE		34
+#define RK3588_PD_NVM		35
+#define RK3588_PD_NVM0		36
+#define RK3588_PD_SDIO		37
+#define RK3588_PD_AUDIO		38
+#define RK3588_PD_SECURE	39
+#define RK3588_PD_SDMMC		40
+#define RK3588_PD_CRYPTO	41
+#define RK3588_PD_BUS		42
+
+/* VD_PMU */
+#define RK3588_PD_PMU1		43
+
+#endif
diff --git a/include/dt-bindings/power/rockchip,rv1126-power.h b/include/dt-bindings/power/rockchip,rv1126-power.h
new file mode 100644
index 000000000000..38a68e000d38
--- /dev/null
+++ b/include/dt-bindings/power/rockchip,rv1126-power.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __DT_BINDINGS_POWER_RV1126_POWER_H__
+#define __DT_BINDINGS_POWER_RV1126_POWER_H__
+
+/* VD_CORE */
+#define RV1126_PD_CPU_0		0
+#define RV1126_PD_CPU_1		1
+#define RV1126_PD_CPU_2		2
+#define RV1126_PD_CPU_3		3
+#define RV1126_PD_CORE_ALIVE	4
+
+/* VD_PMU */
+#define RV1126_PD_PMU		5
+#define RV1126_PD_PMU_ALIVE	6
+
+/* VD_NPU */
+#define RV1126_PD_NPU		7
+
+/* VD_VEPU */
+#define RV1126_PD_VEPU		8
+
+/* VD_LOGIC */
+#define RV1126_PD_VI		9
+#define RV1126_PD_VO		10
+#define RV1126_PD_ISPP		11
+#define RV1126_PD_VDPU		12
+#define RV1126_PD_CRYPTO	13
+#define RV1126_PD_DDR		14
+#define RV1126_PD_NVM		15
+#define RV1126_PD_SDIO		16
+#define RV1126_PD_USB		17
+#define RV1126_PD_LOGIC_ALIVE	18
+
+#endif
diff --git a/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h b/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h
index 0d3276c8fc11..9f7c5103bc82 100644
--- a/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h
+++ b/include/dt-bindings/sound/qcom,q6dsp-lpass-ports.h
@@ -193,6 +193,24 @@
 #define LPASS_CLK_ID_RX_CORE_MCLK	59
 #define LPASS_CLK_ID_RX_CORE_NPL_MCLK	60
 #define LPASS_CLK_ID_VA_CORE_2X_MCLK	61
+/* Clock ID for MCLK for WSA2 core */
+#define LPASS_CLK_ID_WSA2_CORE_MCLK	62
+/* Clock ID for NPL MCLK for WSA2 core */
+#define LPASS_CLK_ID_WSA2_CORE_2X_MCLK	63
+/* Clock ID for RX Core TX MCLK */
+#define LPASS_CLK_ID_RX_CORE_TX_MCLK	64
+/* Clock ID for RX CORE TX 2X MCLK */
+#define LPASS_CLK_ID_RX_CORE_TX_2X_MCLK	65
+/* Clock ID for WSA core TX MCLK */
+#define LPASS_CLK_ID_WSA_CORE_TX_MCLK	66
+/* Clock ID for WSA core TX 2X MCLK */
+#define LPASS_CLK_ID_WSA_CORE_TX_2X_MCLK	67
+/* Clock ID for WSA2 core TX MCLK */
+#define LPASS_CLK_ID_WSA2_CORE_TX_MCLK	68
+/* Clock ID for WSA2 core TX 2X MCLK */
+#define LPASS_CLK_ID_WSA2_CORE_TX_2X_MCLK	69
+/* Clock ID for RX CORE MCLK2 2X  MCLK */
+#define LPASS_CLK_ID_RX_CORE_MCLK2_2X_MCLK	70
 
 #define LPASS_HW_AVTIMER_VOTE		101
 #define LPASS_HW_MACRO_VOTE		102
diff --git a/include/kunit/test.h b/include/kunit/test.h
index 840a2c375065..20cc4770cb3f 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -228,6 +228,8 @@ static inline void kunit_set_failure(struct kunit *test)
 	WRITE_ONCE(test->status, KUNIT_FAILURE);
 }
 
+bool kunit_enabled(void);
+
 void kunit_init_test(struct kunit *test, const char *name, char *log);
 
 int kunit_run_tests(struct kunit_suite *suite);
@@ -251,7 +253,6 @@ static inline int kunit_run_all_tests(void)
 #endif /* IS_BUILTIN(CONFIG_KUNIT) */
 
 #define __kunit_test_suites(unique_array, ...)				       \
-	MODULE_INFO(test, "Y");						       \
 	static struct kunit_suite *unique_array[]			       \
 	__aligned(sizeof(struct kunit_suite *))				       \
 	__used __section(".kunit_test_suites") = { __VA_ARGS__ }
diff --git a/include/linux/a.out.h b/include/linux/a.out.h
deleted file mode 100644
index 600cf45645c6..000000000000
--- a/include/linux/a.out.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __A_OUT_GNU_H__
-#define __A_OUT_GNU_H__
-
-#include <uapi/linux/a.out.h>
-
-#ifndef __ASSEMBLY__
-#ifdef linux
-#include <asm/page.h>
-#if defined(__i386__) || defined(__mc68000__)
-#else
-#ifndef SEGMENT_SIZE
-#define SEGMENT_SIZE	PAGE_SIZE
-#endif
-#endif
-#endif
-#endif /*__ASSEMBLY__ */
-#endif /* __A_OUT_GNU_H__ */
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index 6f64b2f3dc54..729cff1ee3f8 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -279,14 +279,17 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) { }
 
 void acpi_numa_x2apic_affinity_init(struct acpi_srat_x2apic_cpu_affinity *pa);
 
+#if defined(CONFIG_ARM64) || defined(CONFIG_LOONGARCH)
+void acpi_arch_dma_setup(struct device *dev);
+#else
+static inline void acpi_arch_dma_setup(struct device *dev) { }
+#endif
+
 #ifdef CONFIG_ARM64
 void acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa);
-void acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size);
 #else
 static inline void
 acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa) { }
-static inline void
-acpi_arch_dma_setup(struct device *dev, u64 *dma_addr, u64 *dma_size) { }
 #endif
 
 int acpi_numa_memory_affinity_init (struct acpi_srat_mem_affinity *ma);
@@ -506,6 +509,7 @@ int acpi_dev_get_resources(struct acpi_device *adev, struct list_head *list,
 			   void *preproc_data);
 int acpi_dev_get_dma_resources(struct acpi_device *adev,
 			       struct list_head *list);
+int acpi_dev_get_memory_resources(struct acpi_device *adev, struct list_head *list);
 int acpi_dev_filter_resource_type(struct acpi_resource *ares,
 				  unsigned long types);
 
@@ -798,6 +802,11 @@ acpi_dev_hid_uid_match(struct acpi_device *adev, const char *hid2, const char *u
 	return false;
 }
 
+static inline int acpi_dev_uid_to_integer(struct acpi_device *adev, u64 *integer)
+{
+	return -ENODEV;
+}
+
 static inline struct acpi_device *
 acpi_dev_get_first_match_dev(const char *hid, const char *uid, s64 hrv)
 {
@@ -977,8 +986,7 @@ static inline enum dev_dma_attr acpi_get_dma_attr(struct acpi_device *adev)
 	return DEV_DMA_NOT_SUPPORTED;
 }
 
-static inline int acpi_dma_get_range(struct device *dev, u64 *dma_addr,
-				     u64 *offset, u64 *size)
+static inline int acpi_dma_get_range(struct device *dev, const struct bus_dma_region **map)
 {
 	return -ENODEV;
 }
@@ -1075,6 +1083,7 @@ acpi_status acpi_os_prepare_extended_sleep(u8 sleep_state,
 struct acpi_s2idle_dev_ops {
 	struct list_head list_node;
 	void (*prepare)(void);
+	void (*check)(void);
 	void (*restore)(void);
 };
 int acpi_register_lps0_dev(struct acpi_s2idle_dev_ops *arg);
diff --git a/include/linux/amd-pstate.h b/include/linux/amd-pstate.h
new file mode 100644
index 000000000000..1c4b8659f171
--- /dev/null
+++ b/include/linux/amd-pstate.h
@@ -0,0 +1,77 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * linux/include/linux/amd-pstate.h
+ *
+ * Copyright (C) 2022 Advanced Micro Devices, Inc.
+ *
+ * Author: Meng Li <li.meng@amd.com>
+ */
+
+#ifndef _LINUX_AMD_PSTATE_H
+#define _LINUX_AMD_PSTATE_H
+
+#include <linux/pm_qos.h>
+
+/*********************************************************************
+ *                        AMD P-state INTERFACE                       *
+ *********************************************************************/
+/**
+ * struct  amd_aperf_mperf
+ * @aperf: actual performance frequency clock count
+ * @mperf: maximum performance frequency clock count
+ * @tsc:   time stamp counter
+ */
+struct amd_aperf_mperf {
+	u64 aperf;
+	u64 mperf;
+	u64 tsc;
+};
+
+/**
+ * struct amd_cpudata - private CPU data for AMD P-State
+ * @cpu: CPU number
+ * @req: constraint request to apply
+ * @cppc_req_cached: cached performance request hints
+ * @highest_perf: the maximum performance an individual processor may reach,
+ *		  assuming ideal conditions
+ * @nominal_perf: the maximum sustained performance level of the processor,
+ *		  assuming ideal operating conditions
+ * @lowest_nonlinear_perf: the lowest performance level at which nonlinear power
+ *			   savings are achieved
+ * @lowest_perf: the absolute lowest performance level of the processor
+ * @max_freq: the frequency that mapped to highest_perf
+ * @min_freq: the frequency that mapped to lowest_perf
+ * @nominal_freq: the frequency that mapped to nominal_perf
+ * @lowest_nonlinear_freq: the frequency that mapped to lowest_nonlinear_perf
+ * @cur: Difference of Aperf/Mperf/tsc count between last and current sample
+ * @prev: Last Aperf/Mperf/tsc count value read from register
+ * @freq: current cpu frequency value
+ * @boost_supported: check whether the Processor or SBIOS supports boost mode
+ *
+ * The amd_cpudata is key private data for each CPU thread in AMD P-State, and
+ * represents all the attributes and goals that AMD P-State requests at runtime.
+ */
+struct amd_cpudata {
+	int	cpu;
+
+	struct	freq_qos_request req[2];
+	u64	cppc_req_cached;
+
+	u32	highest_perf;
+	u32	nominal_perf;
+	u32	lowest_nonlinear_perf;
+	u32	lowest_perf;
+
+	u32	max_freq;
+	u32	min_freq;
+	u32	nominal_freq;
+	u32	lowest_nonlinear_freq;
+
+	struct amd_aperf_mperf cur;
+	struct amd_aperf_mperf prev;
+
+	u64	freq;
+	bool	boost_supported;
+};
+
+#endif /* _LINUX_AMD_PSTATE_H */
diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h
index e5c76c1ef9ed..5f02d2e6b9d9 100644
--- a/include/linux/arm_ffa.h
+++ b/include/linux/arm_ffa.h
@@ -17,6 +17,7 @@ struct ffa_device {
 	bool mode_32bit;
 	uuid_t uuid;
 	struct device dev;
+	const struct ffa_ops *ops;
 };
 
 #define to_ffa_dev(d) container_of(d, struct ffa_device, dev)
@@ -47,17 +48,18 @@ static inline void *ffa_dev_get_drvdata(struct ffa_device *fdev)
 }
 
 #if IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT)
-struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id);
+struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id,
+				       const struct ffa_ops *ops);
 void ffa_device_unregister(struct ffa_device *ffa_dev);
 int ffa_driver_register(struct ffa_driver *driver, struct module *owner,
 			const char *mod_name);
 void ffa_driver_unregister(struct ffa_driver *driver);
 bool ffa_device_is_valid(struct ffa_device *ffa_dev);
-const struct ffa_dev_ops *ffa_dev_ops_get(struct ffa_device *dev);
 
 #else
 static inline
-struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id)
+struct ffa_device *ffa_device_register(const uuid_t *uuid, int vm_id,
+				       const struct ffa_ops *ops)
 {
 	return NULL;
 }
@@ -76,11 +78,6 @@ static inline void ffa_driver_unregister(struct ffa_driver *driver) {}
 static inline
 bool ffa_device_is_valid(struct ffa_device *ffa_dev) { return false; }
 
-static inline
-const struct ffa_dev_ops *ffa_dev_ops_get(struct ffa_device *dev)
-{
-	return NULL;
-}
 #endif /* CONFIG_ARM_FFA_TRANSPORT */
 
 #define ffa_register(driver) \
@@ -109,7 +106,10 @@ struct ffa_partition_info {
 #define FFA_PARTITION_DIRECT_SEND	BIT(1)
 /* partition can send and receive indirect messages. */
 #define FFA_PARTITION_INDIRECT_MSG	BIT(2)
+/* partition runs in the AArch64 execution state. */
+#define FFA_PARTITION_AARCH64_EXEC	BIT(8)
 	u32 properties;
+	u32 uuid[4];
 };
 
 /* For use with FFA_MSG_SEND_DIRECT_{REQ,RESP} which pass data via registers */
@@ -257,18 +257,28 @@ struct ffa_mem_ops_args {
 	struct ffa_mem_region_attributes *attrs;
 };
 
-struct ffa_dev_ops {
+struct ffa_info_ops {
 	u32 (*api_version_get)(void);
 	int (*partition_info_get)(const char *uuid_str,
 				  struct ffa_partition_info *buffer);
+};
+
+struct ffa_msg_ops {
 	void (*mode_32bit_set)(struct ffa_device *dev);
 	int (*sync_send_receive)(struct ffa_device *dev,
 				 struct ffa_send_direct_data *data);
+};
+
+struct ffa_mem_ops {
 	int (*memory_reclaim)(u64 g_handle, u32 flags);
-	int (*memory_share)(struct ffa_device *dev,
-			    struct ffa_mem_ops_args *args);
-	int (*memory_lend)(struct ffa_device *dev,
-			   struct ffa_mem_ops_args *args);
+	int (*memory_share)(struct ffa_mem_ops_args *args);
+	int (*memory_lend)(struct ffa_mem_ops_args *args);
+};
+
+struct ffa_ops {
+	const struct ffa_info_ops *info_ops;
+	const struct ffa_msg_ops *msg_ops;
+	const struct ffa_mem_ops *mem_ops;
 };
 
 #endif /* _LINUX_ARM_FFA_H */
diff --git a/include/linux/bcma/bcma_driver_chipcommon.h b/include/linux/bcma/bcma_driver_chipcommon.h
index e3314f746bfa..2d94c30ed439 100644
--- a/include/linux/bcma/bcma_driver_chipcommon.h
+++ b/include/linux/bcma/bcma_driver_chipcommon.h
@@ -271,6 +271,7 @@
 #define  BCMA_CC_SROM_CONTROL_OP_WRDIS	0x40000000
 #define  BCMA_CC_SROM_CONTROL_OP_WREN	0x60000000
 #define  BCMA_CC_SROM_CONTROL_OTPSEL	0x00000010
+#define  BCMA_CC_SROM_CONTROL_OTP_PRESENT	0x00000020
 #define  BCMA_CC_SROM_CONTROL_LOCK	0x00000008
 #define  BCMA_CC_SROM_CONTROL_SIZE_MASK	0x00000006
 #define  BCMA_CC_SROM_CONTROL_SIZE_1K	0x00000000
diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h
index 3dc20c4f394c..8d51f69f9f5e 100644
--- a/include/linux/binfmts.h
+++ b/include/linux/binfmts.h
@@ -43,9 +43,6 @@ struct linux_binprm {
 		 * original userspace.
 		 */
 		point_of_no_return:1;
-#ifdef __alpha__
-	unsigned int taso:1;
-#endif
 	struct file *executable; /* Executable to pass to the interpreter */
 	struct file *interpreter;
 	struct file *file;
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 84b13fdd34a7..8038c5fbde40 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1498,6 +1498,7 @@ int sync_blockdev(struct block_device *bdev);
 int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend);
 int sync_blockdev_nowait(struct block_device *bdev);
 void sync_bdevs(bool wait);
+void bdev_statx_dioalign(struct inode *inode, struct kstat *stat);
 void printk_all_partitions(void);
 #else
 static inline void invalidate_bdev(struct block_device *bdev)
@@ -1514,6 +1515,9 @@ static inline int sync_blockdev_nowait(struct block_device *bdev)
 static inline void sync_bdevs(bool wait)
 {
 }
+static inline void bdev_statx_dioalign(struct inode *inode, struct kstat *stat)
+{
+}
 static inline void printk_all_partitions(void)
 {
 }
diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h
index 2bd1b5f8de9b..57e9e109257e 100644
--- a/include/linux/bpf-cgroup.h
+++ b/include/linux/bpf-cgroup.h
@@ -414,6 +414,11 @@ int cgroup_bpf_prog_detach(const union bpf_attr *attr,
 int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
 int cgroup_bpf_prog_query(const union bpf_attr *attr,
 			  union bpf_attr __user *uattr);
+
+const struct bpf_func_proto *
+cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
+const struct bpf_func_proto *
+cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog);
 #else
 
 static inline int cgroup_bpf_inherit(struct cgroup *cgrp) { return 0; }
@@ -444,6 +449,18 @@ static inline int cgroup_bpf_prog_query(const union bpf_attr *attr,
 	return -EINVAL;
 }
 
+static inline const struct bpf_func_proto *
+cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+{
+	return NULL;
+}
+
+static inline const struct bpf_func_proto *
+cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+{
+	return NULL;
+}
+
 static inline int bpf_cgroup_storage_assign(struct bpf_prog_aux *aux,
 					    struct bpf_map *map) { return 0; }
 static inline struct bpf_cgroup_storage *bpf_cgroup_storage_alloc(
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 20c26aed7896..9e7d46d16032 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -48,6 +48,7 @@ struct mem_cgroup;
 struct module;
 struct bpf_func_state;
 struct ftrace_ops;
+struct cgroup;
 
 extern struct idr btf_idr;
 extern spinlock_t btf_idr_lock;
@@ -279,14 +280,33 @@ static inline void check_and_init_map_value(struct bpf_map *map, void *dst)
 	}
 }
 
-/* copy everything but bpf_spin_lock and bpf_timer. There could be one of each. */
-static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
+/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
+ * forced to use 'long' read/writes to try to atomically copy long counters.
+ * Best-effort only.  No barriers here, since it _will_ race with concurrent
+ * updates from BPF programs. Called from bpf syscall and mostly used with
+ * size 8 or 16 bytes, so ask compiler to inline it.
+ */
+static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
+{
+	const long *lsrc = src;
+	long *ldst = dst;
+
+	size /= sizeof(long);
+	while (size--)
+		*ldst++ = *lsrc++;
+}
+
+/* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */
+static inline void __copy_map_value(struct bpf_map *map, void *dst, void *src, bool long_memcpy)
 {
 	u32 curr_off = 0;
 	int i;
 
 	if (likely(!map->off_arr)) {
-		memcpy(dst, src, map->value_size);
+		if (long_memcpy)
+			bpf_long_memcpy(dst, src, round_up(map->value_size, 8));
+		else
+			memcpy(dst, src, map->value_size);
 		return;
 	}
 
@@ -298,6 +318,36 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
 	}
 	memcpy(dst + curr_off, src + curr_off, map->value_size - curr_off);
 }
+
+static inline void copy_map_value(struct bpf_map *map, void *dst, void *src)
+{
+	__copy_map_value(map, dst, src, false);
+}
+
+static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src)
+{
+	__copy_map_value(map, dst, src, true);
+}
+
+static inline void zero_map_value(struct bpf_map *map, void *dst)
+{
+	u32 curr_off = 0;
+	int i;
+
+	if (likely(!map->off_arr)) {
+		memset(dst, 0, map->value_size);
+		return;
+	}
+
+	for (i = 0; i < map->off_arr->cnt; i++) {
+		u32 next_off = map->off_arr->field_off[i];
+
+		memset(dst + curr_off, 0, next_off - curr_off);
+		curr_off += map->off_arr->field_sz[i];
+	}
+	memset(dst + curr_off, 0, map->value_size - curr_off);
+}
+
 void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
 			   bool lock_src);
 void bpf_timer_cancel_and_free(void *timer);
@@ -401,7 +451,7 @@ enum bpf_type_flag {
 	/* DYNPTR points to memory local to the bpf program. */
 	DYNPTR_TYPE_LOCAL	= BIT(8 + BPF_BASE_TYPE_BITS),
 
-	/* DYNPTR points to a ringbuf record. */
+	/* DYNPTR points to a kernel-produced ringbuf record. */
 	DYNPTR_TYPE_RINGBUF	= BIT(9 + BPF_BASE_TYPE_BITS),
 
 	/* Size is known at compile time. */
@@ -606,6 +656,7 @@ enum bpf_reg_type {
 	PTR_TO_MEM,		 /* reg points to valid memory region */
 	PTR_TO_BUF,		 /* reg points to a read/write buffer */
 	PTR_TO_FUNC,		 /* reg points to a bpf program function */
+	PTR_TO_DYNPTR,		 /* reg points to a dynptr */
 	__BPF_REG_TYPE_MAX,
 
 	/* Extended reg_types. */
@@ -726,10 +777,14 @@ enum bpf_cgroup_storage_type {
  */
 #define MAX_BPF_FUNC_REG_ARGS 5
 
+/* The argument is a structure. */
+#define BTF_FMODEL_STRUCT_ARG		BIT(0)
+
 struct btf_func_model {
 	u8 ret_size;
 	u8 nr_args;
 	u8 arg_size[MAX_BPF_FUNC_ARGS];
+	u8 arg_flags[MAX_BPF_FUNC_ARGS];
 };
 
 /* Restore arguments before returning from trampoline to let original function
@@ -809,6 +864,10 @@ u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,
 					struct bpf_tramp_run_ctx *run_ctx);
 void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,
 					struct bpf_tramp_run_ctx *run_ctx);
+u64 notrace __bpf_prog_enter_struct_ops(struct bpf_prog *prog,
+					struct bpf_tramp_run_ctx *run_ctx);
+void notrace __bpf_prog_exit_struct_ops(struct bpf_prog *prog, u64 start,
+					struct bpf_tramp_run_ctx *run_ctx);
 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
 void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
 
@@ -891,6 +950,7 @@ struct bpf_dispatcher {
 	struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX];
 	int num_progs;
 	void *image;
+	void *rw_image;
 	u32 image_off;
 	struct bpf_ksym ksym;
 };
@@ -909,7 +969,7 @@ int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampolin
 struct bpf_trampoline *bpf_trampoline_get(u64 key,
 					  struct bpf_attach_target_info *tgt_info);
 void bpf_trampoline_put(struct bpf_trampoline *tr);
-int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs);
+int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs);
 #define BPF_DISPATCHER_INIT(_name) {				\
 	.mutex = __MUTEX_INITIALIZER(_name.mutex),		\
 	.func = &_name##_func,					\
@@ -923,7 +983,14 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs);
 	},							\
 }
 
+#ifdef CONFIG_X86_64
+#define BPF_DISPATCHER_ATTRIBUTES __attribute__((patchable_function_entry(5)))
+#else
+#define BPF_DISPATCHER_ATTRIBUTES
+#endif
+
 #define DEFINE_BPF_DISPATCHER(name)					\
+	notrace BPF_DISPATCHER_ATTRIBUTES				\
 	noinline __nocfi unsigned int bpf_dispatcher_##name##_func(	\
 		const void *ctx,					\
 		const struct bpf_insn *insnsi,				\
@@ -945,7 +1012,6 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs);
 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
 				struct bpf_prog *to);
 /* Called only from JIT-enabled code, so there's no need for stubs. */
-void *bpf_jit_alloc_exec_page(void);
 void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym);
 void bpf_image_ksym_del(struct bpf_ksym *ksym);
 void bpf_ksym_add(struct bpf_ksym *ksym);
@@ -1333,6 +1399,11 @@ struct bpf_array {
 #define BPF_MAP_CAN_READ	BIT(0)
 #define BPF_MAP_CAN_WRITE	BIT(1)
 
+/* Maximum number of user-producer ring buffer samples that can be drained in
+ * a call to bpf_user_ringbuf_drain().
+ */
+#define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024)
+
 static inline u32 bpf_map_flags_to_cap(struct bpf_map *map)
 {
 	u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG);
@@ -1729,8 +1800,40 @@ int bpf_obj_get_user(const char __user *pathname, int flags);
 	extern int bpf_iter_ ## target(args);			\
 	int __init bpf_iter_ ## target(args) { return 0; }
 
+/*
+ * The task type of iterators.
+ *
+ * For BPF task iterators, they can be parameterized with various
+ * parameters to visit only some of tasks.
+ *
+ * BPF_TASK_ITER_ALL (default)
+ *	Iterate over resources of every task.
+ *
+ * BPF_TASK_ITER_TID
+ *	Iterate over resources of a task/tid.
+ *
+ * BPF_TASK_ITER_TGID
+ *	Iterate over resources of every task of a process / task group.
+ */
+enum bpf_iter_task_type {
+	BPF_TASK_ITER_ALL = 0,
+	BPF_TASK_ITER_TID,
+	BPF_TASK_ITER_TGID,
+};
+
 struct bpf_iter_aux_info {
+	/* for map_elem iter */
 	struct bpf_map *map;
+
+	/* for cgroup iter */
+	struct {
+		struct cgroup *start; /* starting cgroup */
+		enum bpf_cgroup_iter_order order;
+	} cgroup;
+	struct {
+		enum bpf_iter_task_type	type;
+		u32 pid;
+	} task;
 };
 
 typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog,
@@ -1815,22 +1918,6 @@ int bpf_get_file_flag(int flags);
 int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
 			     size_t actual_size);
 
-/* memcpy that is used with 8-byte aligned pointers, power-of-8 size and
- * forced to use 'long' read/writes to try to atomically copy long counters.
- * Best-effort only.  No barriers here, since it _will_ race with concurrent
- * updates from BPF programs. Called from bpf syscall and mostly used with
- * size 8 or 16 bytes, so ask compiler to inline it.
- */
-static inline void bpf_long_memcpy(void *dst, const void *src, u32 size)
-{
-	const long *lsrc = src;
-	long *ldst = dst;
-
-	size /= sizeof(long);
-	while (size--)
-		*ldst++ = *lsrc++;
-}
-
 /* verify correctness of eBPF program */
 int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr);
 
@@ -1932,13 +2019,22 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
 			   const char *func_name,
 			   struct btf_func_model *m);
 
+struct bpf_kfunc_arg_meta {
+	u64 r0_size;
+	bool r0_rdonly;
+	int ref_obj_id;
+	u32 flags;
+};
+
 struct bpf_reg_state;
 int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
 				struct bpf_reg_state *regs);
+int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
+			   struct bpf_reg_state *regs);
 int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
 			      const struct btf *btf, u32 func_id,
 			      struct bpf_reg_state *regs,
-			      u32 kfunc_flags);
+			      struct bpf_kfunc_arg_meta *meta);
 int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
 			  struct bpf_reg_state *reg);
 int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog,
@@ -1966,6 +2062,17 @@ static inline bool unprivileged_ebpf_enabled(void)
 	return !sysctl_unprivileged_bpf_disabled;
 }
 
+/* Not all bpf prog type has the bpf_ctx.
+ * For the bpf prog type that has initialized the bpf_ctx,
+ * this function can be used to decide if a kernel function
+ * is called by a bpf program.
+ */
+static inline bool has_current_bpf_ctx(void)
+{
+	return !!current->bpf_ctx;
+}
+
+void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog);
 #else /* !CONFIG_BPF_SYSCALL */
 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
 {
@@ -2148,6 +2255,15 @@ static inline struct bpf_prog *bpf_prog_by_id(u32 id)
 	return ERR_PTR(-ENOTSUPP);
 }
 
+static inline int btf_struct_access(struct bpf_verifier_log *log,
+				    const struct btf *btf,
+				    const struct btf_type *t, int off, int size,
+				    enum bpf_access_type atype,
+				    u32 *next_btf_id, enum bpf_type_flag *flag)
+{
+	return -EACCES;
+}
+
 static inline const struct bpf_func_proto *
 bpf_base_func_proto(enum bpf_func_id func_id)
 {
@@ -2175,6 +2291,14 @@ static inline bool unprivileged_ebpf_enabled(void)
 	return false;
 }
 
+static inline bool has_current_bpf_ctx(void)
+{
+	return false;
+}
+
+static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog)
+{
+}
 #endif /* CONFIG_BPF_SYSCALL */
 
 void __bpf_free_used_btfs(struct bpf_prog_aux *aux,
@@ -2349,6 +2473,7 @@ extern const struct bpf_func_proto bpf_get_numa_node_id_proto;
 extern const struct bpf_func_proto bpf_tail_call_proto;
 extern const struct bpf_func_proto bpf_ktime_get_ns_proto;
 extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto;
+extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto;
 extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto;
 extern const struct bpf_func_proto bpf_get_current_uid_gid_proto;
 extern const struct bpf_func_proto bpf_get_current_comm_proto;
@@ -2361,6 +2486,7 @@ extern const struct bpf_func_proto bpf_sock_map_update_proto;
 extern const struct bpf_func_proto bpf_sock_hash_update_proto;
 extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto;
 extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto;
+extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto;
 extern const struct bpf_func_proto bpf_msg_redirect_hash_proto;
 extern const struct bpf_func_proto bpf_msg_redirect_map_proto;
 extern const struct bpf_func_proto bpf_sk_redirect_hash_proto;
@@ -2410,6 +2536,7 @@ extern const struct bpf_func_proto bpf_loop_proto;
 extern const struct bpf_func_proto bpf_copy_from_user_task_proto;
 extern const struct bpf_func_proto bpf_set_retval_proto;
 extern const struct bpf_func_proto bpf_get_retval_proto;
+extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto;
 
 const struct bpf_func_proto *tracing_prog_func_proto(
   enum bpf_func_id func_id, const struct bpf_prog *prog);
@@ -2554,7 +2681,7 @@ enum bpf_dynptr_type {
 	BPF_DYNPTR_TYPE_INVALID,
 	/* Points to memory that is local to the bpf program */
 	BPF_DYNPTR_TYPE_LOCAL,
-	/* Underlying data is a ringbuf record */
+	/* Underlying data is a kernel-produced ringbuf record */
 	BPF_DYNPTR_TYPE_RINGBUF,
 };
 
@@ -2562,6 +2689,7 @@ void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
 		     enum bpf_dynptr_type type, u32 offset, u32 size);
 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
 int bpf_dynptr_check_size(u32 size);
+u32 bpf_dynptr_get_size(struct bpf_dynptr_kern *ptr);
 
 #ifdef CONFIG_BPF_LSM
 void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype);
@@ -2571,4 +2699,12 @@ static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {}
 static inline void bpf_cgroup_atype_put(int cgroup_atype) {}
 #endif /* CONFIG_BPF_LSM */
 
+struct key;
+
+#ifdef CONFIG_KEYS
+struct bpf_key {
+	struct key *key;
+	bool has_ref;
+};
+#endif /* CONFIG_KEYS */
 #endif /* _LINUX_BPF_H */
diff --git a/include/linux/bpf_mem_alloc.h b/include/linux/bpf_mem_alloc.h
new file mode 100644
index 000000000000..3e164b8efaa9
--- /dev/null
+++ b/include/linux/bpf_mem_alloc.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
+#ifndef _BPF_MEM_ALLOC_H
+#define _BPF_MEM_ALLOC_H
+#include <linux/compiler_types.h>
+#include <linux/workqueue.h>
+
+struct bpf_mem_cache;
+struct bpf_mem_caches;
+
+struct bpf_mem_alloc {
+	struct bpf_mem_caches __percpu *caches;
+	struct bpf_mem_cache __percpu *cache;
+	struct work_struct work;
+};
+
+int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu);
+void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma);
+
+/* kmalloc/kfree equivalent: */
+void *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size);
+void bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr);
+
+/* kmem_cache_alloc/free equivalent: */
+void *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma);
+void bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr);
+
+#endif /* _BPF_MEM_ALLOC_H */
diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h
index 2b9112b80171..2c6a4f2562a7 100644
--- a/include/linux/bpf_types.h
+++ b/include/linux/bpf_types.h
@@ -126,6 +126,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_STRUCT_OPS, bpf_struct_ops_map_ops)
 #endif
 BPF_MAP_TYPE(BPF_MAP_TYPE_RINGBUF, ringbuf_map_ops)
 BPF_MAP_TYPE(BPF_MAP_TYPE_BLOOM_FILTER, bloom_filter_map_ops)
+BPF_MAP_TYPE(BPF_MAP_TYPE_USER_RINGBUF, user_ringbuf_map_ops)
 
 BPF_LINK_TYPE(BPF_LINK_TYPE_RAW_TRACEPOINT, raw_tracepoint)
 BPF_LINK_TYPE(BPF_LINK_TYPE_TRACING, tracing)
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index 2e3bad8640dc..9e1e6965f407 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -212,6 +212,17 @@ struct bpf_reference_state {
 	 * is used purely to inform the user of a reference leak.
 	 */
 	int insn_idx;
+	/* There can be a case like:
+	 * main (frame 0)
+	 *  cb (frame 1)
+	 *   func (frame 3)
+	 *    cb (frame 4)
+	 * Hence for frame 4, if callback_ref just stored boolean, it would be
+	 * impossible to distinguish nested callback refs. Hence store the
+	 * frameno and compare that to callback_ref in check_reference_leak when
+	 * exiting a callback function.
+	 */
+	int callback_ref;
 };
 
 /* state of the program:
@@ -237,6 +248,7 @@ struct bpf_func_state {
 	 */
 	u32 async_entry_cnt;
 	bool in_callback_fn;
+	struct tnum callback_ret_range;
 	bool in_async_callback_fn;
 
 	/* The following fields should be last. See copy_func_state() */
@@ -337,6 +349,27 @@ struct bpf_verifier_state {
 	     iter < frame->allocated_stack / BPF_REG_SIZE;		\
 	     iter++, reg = bpf_get_spilled_reg(iter, frame))
 
+/* Invoke __expr over regsiters in __vst, setting __state and __reg */
+#define bpf_for_each_reg_in_vstate(__vst, __state, __reg, __expr)   \
+	({                                                               \
+		struct bpf_verifier_state *___vstate = __vst;            \
+		int ___i, ___j;                                          \
+		for (___i = 0; ___i <= ___vstate->curframe; ___i++) {    \
+			struct bpf_reg_state *___regs;                   \
+			__state = ___vstate->frame[___i];                \
+			___regs = __state->regs;                         \
+			for (___j = 0; ___j < MAX_BPF_REG; ___j++) {     \
+				__reg = &___regs[___j];                  \
+				(void)(__expr);                          \
+			}                                                \
+			bpf_for_each_spilled_reg(___j, __state, __reg) { \
+				if (!__reg)                              \
+					continue;                        \
+				(void)(__expr);                          \
+			}                                                \
+		}                                                        \
+	})
+
 /* linked list of verifier states used to prune search */
 struct bpf_verifier_state_list {
 	struct bpf_verifier_state state;
@@ -560,6 +593,11 @@ int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state
 			     u32 regno);
 int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 		   u32 regno, u32 mem_size);
+bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env,
+			      struct bpf_reg_state *reg);
+bool is_dynptr_type_expected(struct bpf_verifier_env *env,
+			     struct bpf_reg_state *reg,
+			     enum bpf_arg_type arg_type);
 
 /* this lives here instead of in bpf.h because it needs to dereference tgt_prog */
 static inline u64 bpf_trampoline_compute_key(const struct bpf_prog *tgt_prog,
@@ -587,6 +625,8 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
 			    struct bpf_attach_target_info *tgt_info);
 void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab);
 
+int mark_chain_precision(struct bpf_verifier_env *env, int regno);
+
 #define BPF_BASE_TYPE_MASK	GENMASK(BPF_BASE_TYPE_BITS - 1, 0)
 
 /* extract base type from bpf_{arg, return, reg}_type. */
diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h
index 6ff567ece34a..9e77165f3ef6 100644
--- a/include/linux/brcmphy.h
+++ b/include/linux/brcmphy.h
@@ -293,6 +293,7 @@
 #define MII_BRCM_FET_SHDW_MC_FAME	0x4000	/* Force Auto MDIX enable */
 
 #define MII_BRCM_FET_SHDW_AUXMODE4	0x1a	/* Auxiliary mode 4 */
+#define MII_BRCM_FET_SHDW_AM4_STANDBY	0x0008	/* Standby enable */
 #define MII_BRCM_FET_SHDW_AM4_LED_MASK	0x0003
 #define MII_BRCM_FET_SHDW_AM4_LED_MODE1 0x0001
 
diff --git a/include/linux/btf.h b/include/linux/btf.h
index cdb376d53238..f9aababc5d78 100644
--- a/include/linux/btf.h
+++ b/include/linux/btf.h
@@ -49,6 +49,17 @@
  * for this case.
  */
 #define KF_TRUSTED_ARGS (1 << 4) /* kfunc only takes trusted pointer arguments */
+#define KF_SLEEPABLE    (1 << 5) /* kfunc may sleep */
+#define KF_DESTRUCTIVE  (1 << 6) /* kfunc performs destructive actions */
+
+/*
+ * Return the name of the passed struct, if exists, or halt the build if for
+ * example the structure gets renamed. In this way, developers have to revisit
+ * the code using that structure name, and update it accordingly.
+ */
+#define stringify_struct(x)			\
+	({ BUILD_BUG_ON(sizeof(struct x) < 0);	\
+	   __stringify(x); })
 
 struct btf;
 struct btf_member;
@@ -439,4 +450,14 @@ static inline int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dt
 }
 #endif
 
+static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t)
+{
+	if (!btf_type_is_ptr(t))
+		return false;
+
+	t = btf_type_skip_modifiers(btf, t->type, NULL);
+
+	return btf_type_is_struct(t);
+}
+
 #endif
diff --git a/include/linux/can/dev.h b/include/linux/can/dev.h
index c3e50e537e39..58f5431a5559 100644
--- a/include/linux/can/dev.h
+++ b/include/linux/can/dev.h
@@ -147,6 +147,11 @@ static inline u32 can_get_static_ctrlmode(struct can_priv *priv)
 	return priv->ctrlmode & ~priv->ctrlmode_supported;
 }
 
+static inline bool can_is_canxl_dev_mtu(unsigned int mtu)
+{
+	return (mtu >= CANXL_MIN_MTU && mtu <= CANXL_MAX_MTU);
+}
+
 void can_setup(struct net_device *dev);
 
 struct net_device *alloc_candev_mqs(int sizeof_priv, unsigned int echo_skb_max,
diff --git a/include/linux/can/skb.h b/include/linux/can/skb.h
index 182749e858b3..1abc25a8d144 100644
--- a/include/linux/can/skb.h
+++ b/include/linux/can/skb.h
@@ -20,7 +20,8 @@ void can_flush_echo_skb(struct net_device *dev);
 int can_put_echo_skb(struct sk_buff *skb, struct net_device *dev,
 		     unsigned int idx, unsigned int frame_len);
 struct sk_buff *__can_get_echo_skb(struct net_device *dev, unsigned int idx,
-				   u8 *len_ptr, unsigned int *frame_len_ptr);
+				   unsigned int *len_ptr,
+				   unsigned int *frame_len_ptr);
 unsigned int __must_check can_get_echo_skb(struct net_device *dev,
 					   unsigned int idx,
 					   unsigned int *frame_len_ptr);
@@ -29,6 +30,9 @@ void can_free_echo_skb(struct net_device *dev, unsigned int idx,
 struct sk_buff *alloc_can_skb(struct net_device *dev, struct can_frame **cf);
 struct sk_buff *alloc_canfd_skb(struct net_device *dev,
 				struct canfd_frame **cfd);
+struct sk_buff *alloc_canxl_skb(struct net_device *dev,
+				struct canxl_frame **cxl,
+				unsigned int data_len);
 struct sk_buff *alloc_can_err_skb(struct net_device *dev,
 				  struct can_frame **cf);
 bool can_dropped_invalid_skb(struct net_device *dev, struct sk_buff *skb);
@@ -97,10 +101,59 @@ static inline struct sk_buff *can_create_echo_skb(struct sk_buff *skb)
 	return nskb;
 }
 
+static inline bool can_is_can_skb(const struct sk_buff *skb)
+{
+	struct can_frame *cf = (struct can_frame *)skb->data;
+
+	/* the CAN specific type of skb is identified by its data length */
+	return (skb->len == CAN_MTU && cf->len <= CAN_MAX_DLEN);
+}
+
 static inline bool can_is_canfd_skb(const struct sk_buff *skb)
 {
+	struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
+
 	/* the CAN specific type of skb is identified by its data length */
-	return skb->len == CANFD_MTU;
+	return (skb->len == CANFD_MTU && cfd->len <= CANFD_MAX_DLEN);
+}
+
+static inline bool can_is_canxl_skb(const struct sk_buff *skb)
+{
+	const struct canxl_frame *cxl = (struct canxl_frame *)skb->data;
+
+	if (skb->len < CANXL_HDR_SIZE + CANXL_MIN_DLEN || skb->len > CANXL_MTU)
+		return false;
+
+	/* this also checks valid CAN XL data length boundaries */
+	if (skb->len != CANXL_HDR_SIZE + cxl->len)
+		return false;
+
+	return cxl->flags & CANXL_XLF;
+}
+
+/* get length element value from can[|fd|xl]_frame structure */
+static inline unsigned int can_skb_get_len_val(struct sk_buff *skb)
+{
+	const struct canxl_frame *cxl = (struct canxl_frame *)skb->data;
+	const struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
+
+	if (can_is_canxl_skb(skb))
+		return cxl->len;
+
+	return cfd->len;
+}
+
+/* get needed data length inside CAN frame for all frame types (RTR aware) */
+static inline unsigned int can_skb_get_data_len(struct sk_buff *skb)
+{
+	unsigned int len = can_skb_get_len_val(skb);
+	const struct can_frame *cf = (struct can_frame *)skb->data;
+
+	/* RTR frames have an actual length of zero */
+	if (can_is_can_skb(skb) && cf->can_id & CAN_RTR_FLAG)
+		return 0;
+
+	return len;
 }
 
 #endif /* !_CAN_SKB_H */
diff --git a/include/linux/cfi.h b/include/linux/cfi.h
index c6dfc1ed0626..5e134f4ce8b7 100644
--- a/include/linux/cfi.h
+++ b/include/linux/cfi.h
@@ -2,49 +2,38 @@
 /*
  * Clang Control Flow Integrity (CFI) support.
  *
- * Copyright (C) 2021 Google LLC
+ * Copyright (C) 2022 Google LLC
  */
 #ifndef _LINUX_CFI_H
 #define _LINUX_CFI_H
 
-#ifdef CONFIG_CFI_CLANG
-typedef void (*cfi_check_fn)(uint64_t id, void *ptr, void *diag);
-
-/* Compiler-generated function in each module, and the kernel */
-extern void __cfi_check(uint64_t id, void *ptr, void *diag);
-
-/*
- * Force the compiler to generate a CFI jump table entry for a function
- * and store the jump table address to __cfi_jt_<function>.
- */
-#define __CFI_ADDRESSABLE(fn, __attr) \
-	const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn
-
-#ifdef CONFIG_CFI_CLANG_SHADOW
-
-extern void cfi_module_add(struct module *mod, unsigned long base_addr);
-extern void cfi_module_remove(struct module *mod, unsigned long base_addr);
-
-#else
+#include <linux/bug.h>
+#include <linux/module.h>
 
-static inline void cfi_module_add(struct module *mod, unsigned long base_addr) {}
-static inline void cfi_module_remove(struct module *mod, unsigned long base_addr) {}
-
-#endif /* CONFIG_CFI_CLANG_SHADOW */
-
-#else /* !CONFIG_CFI_CLANG */
-
-#ifdef CONFIG_X86_KERNEL_IBT
-
-#define __CFI_ADDRESSABLE(fn, __attr) \
-	const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn
+#ifdef CONFIG_CFI_CLANG
+enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr,
+				      unsigned long *target, u32 type);
 
-#endif /* CONFIG_X86_KERNEL_IBT */
+static inline enum bug_trap_type report_cfi_failure_noaddr(struct pt_regs *regs,
+							   unsigned long addr)
+{
+	return report_cfi_failure(regs, addr, NULL, 0);
+}
 
+#ifdef CONFIG_ARCH_USES_CFI_TRAPS
+bool is_cfi_trap(unsigned long addr);
+#endif
 #endif /* CONFIG_CFI_CLANG */
 
-#ifndef __CFI_ADDRESSABLE
-#define __CFI_ADDRESSABLE(fn, __attr)
-#endif
+#ifdef CONFIG_MODULES
+#ifdef CONFIG_ARCH_USES_CFI_TRAPS
+void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
+			 struct module *mod);
+#else
+static inline void module_cfi_finalize(const Elf_Ehdr *hdr,
+				       const Elf_Shdr *sechdrs,
+				       struct module *mod) {}
+#endif /* CONFIG_ARCH_USES_CFI_TRAPS */
+#endif /* CONFIG_MODULES */
 
 #endif /* _LINUX_CFI_H */
diff --git a/include/linux/cfi_types.h b/include/linux/cfi_types.h
new file mode 100644
index 000000000000..6b8713675765
--- /dev/null
+++ b/include/linux/cfi_types.h
@@ -0,0 +1,45 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Clang Control Flow Integrity (CFI) type definitions.
+ */
+#ifndef _LINUX_CFI_TYPES_H
+#define _LINUX_CFI_TYPES_H
+
+#ifdef __ASSEMBLY__
+#include <linux/linkage.h>
+
+#ifdef CONFIG_CFI_CLANG
+/*
+ * Use the __kcfi_typeid_<function> type identifier symbol to
+ * annotate indirectly called assembly functions. The compiler emits
+ * these symbols for all address-taken function declarations in C
+ * code.
+ */
+#ifndef __CFI_TYPE
+#define __CFI_TYPE(name)				\
+	.4byte __kcfi_typeid_##name
+#endif
+
+#define SYM_TYPED_ENTRY(name, linkage, align...)	\
+	linkage(name) ASM_NL				\
+	align ASM_NL					\
+	__CFI_TYPE(name) ASM_NL				\
+	name:
+
+#define SYM_TYPED_START(name, linkage, align...)	\
+	SYM_TYPED_ENTRY(name, linkage, align)
+
+#else /* CONFIG_CFI_CLANG */
+
+#define SYM_TYPED_START(name, linkage, align...)	\
+	SYM_START(name, linkage, align)
+
+#endif /* CONFIG_CFI_CLANG */
+
+#ifndef SYM_TYPED_FUNC_START
+#define SYM_TYPED_FUNC_START(name) 			\
+	SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)
+#endif
+
+#endif /* __ASSEMBLY__ */
+#endif /* _LINUX_CFI_TYPES_H */
diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h
index c84fec767445..42e55579d649 100644
--- a/include/linux/compiler-clang.h
+++ b/include/linux/compiler-clang.h
@@ -66,17 +66,9 @@
 # define __noscs	__attribute__((__no_sanitize__("shadow-call-stack")))
 #endif
 
-#define __nocfi		__attribute__((__no_sanitize__("cfi")))
-#define __cficanonical	__attribute__((__cfi_canonical_jump_table__))
-
-#if defined(CONFIG_CFI_CLANG)
-/*
- * With CONFIG_CFI_CLANG, the compiler replaces function address
- * references with the address of the function's CFI jump table
- * entry. The function_nocfi macro always returns the address of the
- * actual function instead.
- */
-#define function_nocfi(x)	__builtin_function_start(x)
+#if __has_feature(kcfi)
+/* Disable CFI checking inside a function. */
+#define __nocfi		__attribute__((__no_sanitize__("kcfi")))
 #endif
 
 /*
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 7713d7bcdaea..973a1bfd7ef5 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -203,16 +203,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
 	__v;								\
 })
 
-/*
- * With CONFIG_CFI_CLANG, the compiler replaces function addresses in
- * instrumented C code with jump table addresses. Architectures that
- * support CFI can define this macro to return the actual function address
- * when needed.
- */
-#ifndef function_nocfi
-#define function_nocfi(x) (x)
-#endif
-
 #endif /* __KERNEL__ */
 
 /*
@@ -221,9 +211,11 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
  * otherwise, or eliminated entirely due to lack of references that are
  * visible to the compiler.
  */
-#define __ADDRESSABLE(sym) \
-	static void * __section(".discard.addressable") __used \
+#define ___ADDRESSABLE(sym, __attrs) \
+	static void * __used __attrs \
 		__UNIQUE_ID(__PASTE(__addressable_,sym)) = (void *)&sym;
+#define __ADDRESSABLE(sym) \
+	___ADDRESSABLE(sym, __section(".discard.addressable"))
 
 /**
  * offset_to_ptr - convert a relative memory offset to an absolute pointer
diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h
index 445e80517cab..fc93c9488c76 100644
--- a/include/linux/compiler_attributes.h
+++ b/include/linux/compiler_attributes.h
@@ -371,4 +371,11 @@
  */
 #define __weak                          __attribute__((__weak__))
 
+/*
+ * Used by functions that use '__builtin_return_address'. These function
+ * don't want to be splited or made inline, which can make
+ * the '__builtin_return_address' get unexpected address.
+ */
+#define __fix_address noinline __noclone
+
 #endif /* __LINUX_COMPILER_ATTRIBUTES_H */
diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h
index 4f2a819fd60a..d9d98e8a9a3b 100644
--- a/include/linux/compiler_types.h
+++ b/include/linux/compiler_types.h
@@ -4,8 +4,12 @@
 
 #ifndef __ASSEMBLY__
 
+/*
+ * Skipped when running bindgen due to a libclang issue;
+ * see https://github.com/rust-lang/rust-bindgen/issues/2244.
+ */
 #if defined(CONFIG_DEBUG_INFO_BTF) && defined(CONFIG_PAHOLE_HAS_BTF_TAG) && \
-	__has_attribute(btf_type_tag)
+	__has_attribute(btf_type_tag) && !defined(__BINDGEN__)
 # define BTF_TYPE_TAG(value) __attribute__((btf_type_tag(#value)))
 #else
 # define BTF_TYPE_TAG(value) /* nothing */
@@ -265,10 +269,6 @@ struct ftrace_likely_data {
 # define __nocfi
 #endif
 
-#ifndef __cficanonical
-# define __cficanonical
-#endif
-
 /*
  * Any place that could be marked with the "alloc_size" attribute is also
  * a place to be marked with the "malloc" attribute. Do this as part of the
diff --git a/include/linux/dcache.h b/include/linux/dcache.h
index 92c78ed02b54..54d46518c481 100644
--- a/include/linux/dcache.h
+++ b/include/linux/dcache.h
@@ -287,8 +287,8 @@ static inline unsigned d_count(const struct dentry *dentry)
 /*
  * helper function for dentry_operations.d_dname() members
  */
-extern __printf(4, 5)
-char *dynamic_dname(struct dentry *, char *, int, const char *, ...);
+extern __printf(3, 4)
+char *dynamic_dname(char *, int, const char *, ...);
 
 extern char *__d_path(const struct path *, const struct path *, char *, int);
 extern char *d_absolute_path(const struct path *, char *, int);
diff --git a/include/linux/dlm.h b/include/linux/dlm.h
index ff951e9f6f20..c6bc2b5ee7e6 100644
--- a/include/linux/dlm.h
+++ b/include/linux/dlm.h
@@ -56,9 +56,6 @@ struct dlm_lockspace_ops {
  * DLM_LSFL_TIMEWARN
  *   The dlm should emit netlink messages if locks have been waiting
  *   for a configurable amount of time.  (Unused.)
- * DLM_LSFL_FS
- *   The lockspace user is in the kernel (i.e. filesystem).  Enables
- *   direct bast/cast callbacks.
  * DLM_LSFL_NEWEXCL
  *   dlm_new_lockspace() should return -EEXIST if the lockspace exists.
  *
@@ -134,7 +131,7 @@ int dlm_lock(dlm_lockspace_t *lockspace,
 	     int mode,
 	     struct dlm_lksb *lksb,
 	     uint32_t flags,
-	     void *name,
+	     const void *name,
 	     unsigned int namelen,
 	     uint32_t parent_lkid,
 	     void (*lockast) (void *astarg),
diff --git a/include/linux/dma-resv.h b/include/linux/dma-resv.h
index c8ccbc94d5d2..0637659a702c 100644
--- a/include/linux/dma-resv.h
+++ b/include/linux/dma-resv.h
@@ -62,6 +62,11 @@ struct dma_resv_list;
  * For example when asking for WRITE fences then the KERNEL fences are returned
  * as well. Similar when asked for READ fences then both WRITE and KERNEL
  * fences are returned as well.
+ *
+ * Already used fences can be promoted in the sense that a fence with
+ * DMA_RESV_USAGE_BOOKKEEP could become DMA_RESV_USAGE_READ by adding it again
+ * with this usage. But fences can never be degraded in the sense that a fence
+ * with DMA_RESV_USAGE_WRITE could become DMA_RESV_USAGE_READ.
  */
 enum dma_resv_usage {
 	/**
@@ -98,10 +103,15 @@ enum dma_resv_usage {
 	 * @DMA_RESV_USAGE_BOOKKEEP: No implicit sync.
 	 *
 	 * This should be used by submissions which don't want to participate in
-	 * implicit synchronization.
+	 * any implicit synchronization.
+	 *
+	 * The most common case are preemption fences, page table updates, TLB
+	 * flushes as well as explicit synced user submissions.
 	 *
-	 * The most common case are preemption fences as well as page table
-	 * updates and their TLB flushes.
+	 * Explicit synced user user submissions can be promoted to
+	 * DMA_RESV_USAGE_READ or DMA_RESV_USAGE_WRITE as needed using
+	 * dma_buf_import_sync_file() when implicit synchronization should
+	 * become necessary after initial adding of the fence.
 	 */
 	DMA_RESV_USAGE_BOOKKEEP
 };
diff --git a/include/linux/edac.h b/include/linux/edac.h
index e730b3468719..fa4bda2a70f6 100644
--- a/include/linux/edac.h
+++ b/include/linux/edac.h
@@ -231,21 +231,21 @@ enum mem_type {
 #define MEM_FLAG_DDR		BIT(MEM_DDR)
 #define MEM_FLAG_RDDR		BIT(MEM_RDDR)
 #define MEM_FLAG_RMBS		BIT(MEM_RMBS)
-#define MEM_FLAG_DDR2           BIT(MEM_DDR2)
-#define MEM_FLAG_FB_DDR2        BIT(MEM_FB_DDR2)
-#define MEM_FLAG_RDDR2          BIT(MEM_RDDR2)
-#define MEM_FLAG_XDR            BIT(MEM_XDR)
-#define MEM_FLAG_DDR3           BIT(MEM_DDR3)
-#define MEM_FLAG_RDDR3          BIT(MEM_RDDR3)
-#define MEM_FLAG_LPDDR3         BIT(MEM_LPDDR3)
-#define MEM_FLAG_DDR4           BIT(MEM_DDR4)
-#define MEM_FLAG_RDDR4          BIT(MEM_RDDR4)
-#define MEM_FLAG_LRDDR4         BIT(MEM_LRDDR4)
-#define MEM_FLAG_LPDDR4         BIT(MEM_LPDDR4)
-#define MEM_FLAG_DDR5           BIT(MEM_DDR5)
-#define MEM_FLAG_RDDR5          BIT(MEM_RDDR5)
-#define MEM_FLAG_LRDDR5         BIT(MEM_LRDDR5)
-#define MEM_FLAG_NVDIMM         BIT(MEM_NVDIMM)
+#define MEM_FLAG_DDR2		BIT(MEM_DDR2)
+#define MEM_FLAG_FB_DDR2	BIT(MEM_FB_DDR2)
+#define MEM_FLAG_RDDR2		BIT(MEM_RDDR2)
+#define MEM_FLAG_XDR		BIT(MEM_XDR)
+#define MEM_FLAG_DDR3		BIT(MEM_DDR3)
+#define MEM_FLAG_RDDR3		BIT(MEM_RDDR3)
+#define MEM_FLAG_LPDDR3		BIT(MEM_LPDDR3)
+#define MEM_FLAG_DDR4		BIT(MEM_DDR4)
+#define MEM_FLAG_RDDR4		BIT(MEM_RDDR4)
+#define MEM_FLAG_LRDDR4		BIT(MEM_LRDDR4)
+#define MEM_FLAG_LPDDR4		BIT(MEM_LPDDR4)
+#define MEM_FLAG_DDR5		BIT(MEM_DDR5)
+#define MEM_FLAG_RDDR5		BIT(MEM_RDDR5)
+#define MEM_FLAG_LRDDR5		BIT(MEM_LRDDR5)
+#define MEM_FLAG_NVDIMM		BIT(MEM_NVDIMM)
 #define MEM_FLAG_WIO2		BIT(MEM_WIO2)
 #define MEM_FLAG_HBM2		BIT(MEM_HBM2)
 
diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h
index 92b10e67d5f8..a541f0c4f146 100644
--- a/include/linux/etherdevice.h
+++ b/include/linux/etherdevice.h
@@ -428,6 +428,28 @@ static inline bool ether_addr_equal_masked(const u8 *addr1, const u8 *addr2,
 	return true;
 }
 
+static inline bool ether_addr_is_ipv4_mcast(const u8 *addr)
+{
+	u8 base[ETH_ALEN] = { 0x01, 0x00, 0x5e, 0x00, 0x00, 0x00 };
+	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0xff, 0x80, 0x00, 0x00 };
+
+	return ether_addr_equal_masked(addr, base, mask);
+}
+
+static inline bool ether_addr_is_ipv6_mcast(const u8 *addr)
+{
+	u8 base[ETH_ALEN] = { 0x33, 0x33, 0x00, 0x00, 0x00, 0x00 };
+	u8 mask[ETH_ALEN] = { 0xff, 0xff, 0x00, 0x00, 0x00, 0x00 };
+
+	return ether_addr_equal_masked(addr, base, mask);
+}
+
+static inline bool ether_addr_is_ip_mcast(const u8 *addr)
+{
+	return ether_addr_is_ipv4_mcast(addr) ||
+		ether_addr_is_ipv6_mcast(addr);
+}
+
 /**
  * ether_addr_to_u64 - Convert an Ethernet address into a u64 value.
  * @addr: Pointer to a six-byte array containing the Ethernet address
diff --git a/include/linux/fb.h b/include/linux/fb.h
index 07fcd0e56682..0aff76bcbb00 100644
--- a/include/linux/fb.h
+++ b/include/linux/fb.h
@@ -615,10 +615,6 @@ extern ssize_t fb_sys_write(struct fb_info *info, const char __user *buf,
 /* drivers/video/fbmem.c */
 extern int register_framebuffer(struct fb_info *fb_info);
 extern void unregister_framebuffer(struct fb_info *fb_info);
-extern int remove_conflicting_pci_framebuffers(struct pci_dev *pdev,
-					       const char *name);
-extern int remove_conflicting_framebuffers(struct apertures_struct *a,
-					   const char *name, bool primary);
 extern int fb_prepare_logo(struct fb_info *fb_info, int rotate);
 extern int fb_show_logo(struct fb_info *fb_info, int rotate);
 extern char* fb_get_buffer_offset(struct fb_info *info, struct fb_pixmap *buf, u32 size);
@@ -631,16 +627,10 @@ extern int fb_get_color_depth(struct fb_var_screeninfo *var,
 extern int fb_get_options(const char *name, char **option);
 extern int fb_new_modelist(struct fb_info *info);
 
-extern struct fb_info *registered_fb[FB_MAX];
-extern int num_registered_fb;
 extern bool fb_center_logo;
 extern int fb_logo_count;
 extern struct class *fb_class;
 
-#define for_each_registered_fb(i)		\
-	for (i = 0; i < FB_MAX; i++)		\
-		if (!registered_fb[i]) {} else
-
 static inline void lock_fb_info(struct fb_info *info)
 {
 	mutex_lock(&info->lock);
diff --git a/include/linux/filter.h b/include/linux/filter.h
index a5f21dc3c432..efc42a6e3aed 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -567,6 +567,12 @@ struct sk_filter {
 
 DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key);
 
+extern struct mutex nf_conn_btf_access_lock;
+extern int (*nfct_btf_struct_access)(struct bpf_verifier_log *log, const struct btf *btf,
+				     const struct btf_type *t, int off, int size,
+				     enum bpf_access_type atype, u32 *next_btf_id,
+				     enum bpf_type_flag *flag);
+
 typedef unsigned int (*bpf_dispatcher_fn)(const void *ctx,
 					  const struct bpf_insn *insnsi,
 					  unsigned int (*bpf_func)(const void *,
@@ -900,8 +906,7 @@ int sk_reuseport_attach_filter(struct sock_fprog *fprog, struct sock *sk);
 int sk_reuseport_attach_bpf(u32 ufd, struct sock *sk);
 void sk_reuseport_prog_free(struct bpf_prog *prog);
 int sk_detach_filter(struct sock *sk);
-int sk_get_filter(struct sock *sk, struct sock_filter __user *filter,
-		  unsigned int len);
+int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len);
 
 bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
@@ -1018,6 +1023,8 @@ extern long bpf_jit_limit_max;
 
 typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size);
 
+void bpf_jit_fill_hole_with_zero(void *area, unsigned int size);
+
 struct bpf_binary_header *
 bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
 		     unsigned int alignment,
@@ -1030,6 +1037,9 @@ void bpf_jit_free(struct bpf_prog *fp);
 struct bpf_binary_header *
 bpf_jit_binary_pack_hdr(const struct bpf_prog *fp);
 
+void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns);
+void bpf_prog_pack_free(struct bpf_binary_header *hdr);
+
 static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
 {
 	return list_empty(&fp->aux->ksym.lnode) ||
@@ -1100,7 +1110,7 @@ static inline bool bpf_jit_blinding_enabled(struct bpf_prog *prog)
 		return false;
 	if (!bpf_jit_harden)
 		return false;
-	if (bpf_jit_harden == 1 && capable(CAP_SYS_ADMIN))
+	if (bpf_jit_harden == 1 && bpf_capable())
 		return false;
 
 	return true;
diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h
index 9f50dacbf7d6..76d2b3ebad84 100644
--- a/include/linux/firmware/xlnx-zynqmp.h
+++ b/include/linux/firmware/xlnx-zynqmp.h
@@ -153,6 +153,9 @@ enum pm_ioctl_id {
 	/* Runtime feature configuration */
 	IOCTL_SET_FEATURE_CONFIG = 26,
 	IOCTL_GET_FEATURE_CONFIG = 27,
+	/* Dynamic SD/GEM configuration */
+	IOCTL_SET_SD_CONFIG = 30,
+	IOCTL_SET_GEM_CONFIG = 31,
 };
 
 enum pm_query_id {
@@ -400,6 +403,30 @@ enum pm_feature_config_id {
 };
 
 /**
+ * enum pm_sd_config_type - PM SD configuration.
+ * @SD_CONFIG_EMMC_SEL: To set SD_EMMC_SEL in CTRL_REG_SD and SD_SLOTTYPE
+ * @SD_CONFIG_BASECLK: To set SD_BASECLK in SD_CONFIG_REG1
+ * @SD_CONFIG_8BIT: To set SD_8BIT in SD_CONFIG_REG2
+ * @SD_CONFIG_FIXED: To set fixed config registers
+ */
+enum pm_sd_config_type {
+	SD_CONFIG_EMMC_SEL = 1,
+	SD_CONFIG_BASECLK = 2,
+	SD_CONFIG_8BIT = 3,
+	SD_CONFIG_FIXED = 4,
+};
+
+/**
+ * enum pm_gem_config_type - PM GEM configuration.
+ * @GEM_CONFIG_SGMII_MODE: To set GEM_SGMII_MODE in GEM_CLK_CTRL register
+ * @GEM_CONFIG_FIXED: To set fixed config registers
+ */
+enum pm_gem_config_type {
+	GEM_CONFIG_SGMII_MODE = 1,
+	GEM_CONFIG_FIXED = 2,
+};
+
+/**
  * struct zynqmp_pm_query_data - PM query data
  * @qid:	query ID
  * @arg1:	Argument 1 of query data
@@ -475,6 +502,9 @@ int zynqmp_pm_is_function_supported(const u32 api_id, const u32 id);
 int zynqmp_pm_set_feature_config(enum pm_feature_config_id id, u32 value);
 int zynqmp_pm_get_feature_config(enum pm_feature_config_id id, u32 *payload);
 int zynqmp_pm_register_sgi(u32 sgi_num, u32 reset);
+int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value);
+int zynqmp_pm_set_gem_config(u32 node, enum pm_gem_config_type config,
+			     u32 value);
 #else
 static inline int zynqmp_pm_get_api_version(u32 *version)
 {
@@ -745,6 +775,21 @@ static inline int zynqmp_pm_register_sgi(u32 sgi_num, u32 reset)
 {
 	return -ENODEV;
 }
+
+static inline int zynqmp_pm_set_sd_config(u32 node,
+					  enum pm_sd_config_type config,
+					  u32 value)
+{
+	return -ENODEV;
+}
+
+static inline int zynqmp_pm_set_gem_config(u32 node,
+					   enum pm_gem_config_type config,
+					   u32 value)
+{
+	return -ENODEV;
+}
+
 #endif
 
 #endif /* __FIRMWARE_ZYNQMP_H__ */
diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h
index 3b401fa0f374..b62c90cfafaf 100644
--- a/include/linux/fortify-string.h
+++ b/include/linux/fortify-string.h
@@ -2,7 +2,9 @@
 #ifndef _LINUX_FORTIFY_STRING_H_
 #define _LINUX_FORTIFY_STRING_H_
 
+#include <linux/bug.h>
 #include <linux/const.h>
+#include <linux/limits.h>
 
 #define __FORTIFY_INLINE extern __always_inline __gnu_inline __overloadable
 #define __RENAME(x) __asm__(#x)
@@ -17,9 +19,10 @@ void __write_overflow_field(size_t avail, size_t wanted) __compiletime_warning("
 #define __compiletime_strlen(p)					\
 ({								\
 	unsigned char *__p = (unsigned char *)(p);		\
-	size_t __ret = (size_t)-1;				\
-	size_t __p_size = __builtin_object_size(p, 1);		\
-	if (__p_size != (size_t)-1) {				\
+	size_t __ret = SIZE_MAX;				\
+	size_t __p_size = __member_size(p);			\
+	if (__p_size != SIZE_MAX &&				\
+	    __builtin_constant_p(*__p)) {			\
 		size_t __p_len = __p_size - 1;			\
 		if (__builtin_constant_p(__p[__p_len]) &&	\
 		    __p[__p_len] == '\0')			\
@@ -69,20 +72,59 @@ extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size)
 	__underlying_memcpy(dst, src, bytes)
 
 /*
- * Clang's use of __builtin_object_size() within inlines needs hinting via
- * __pass_object_size(). The preference is to only ever use type 1 (member
+ * Clang's use of __builtin_*object_size() within inlines needs hinting via
+ * __pass_*object_size(). The preference is to only ever use type 1 (member
  * size, rather than struct size), but there remain some stragglers using
  * type 0 that will be converted in the future.
  */
-#define POS	__pass_object_size(1)
-#define POS0	__pass_object_size(0)
+#define POS			__pass_object_size(1)
+#define POS0			__pass_object_size(0)
+#define __struct_size(p)	__builtin_object_size(p, 0)
+#define __member_size(p)	__builtin_object_size(p, 1)
 
+#define __compiletime_lessthan(bounds, length)	(	\
+	__builtin_constant_p((bounds) < (length)) &&	\
+	(bounds) < (length)				\
+)
+
+/**
+ * strncpy - Copy a string to memory with non-guaranteed NUL padding
+ *
+ * @p: pointer to destination of copy
+ * @q: pointer to NUL-terminated source string to copy
+ * @size: bytes to write at @p
+ *
+ * If strlen(@q) >= @size, the copy of @q will stop after @size bytes,
+ * and @p will NOT be NUL-terminated
+ *
+ * If strlen(@q) < @size, following the copy of @q, trailing NUL bytes
+ * will be written to @p until @size total bytes have been written.
+ *
+ * Do not use this function. While FORTIFY_SOURCE tries to avoid
+ * over-reads of @q, it cannot defend against writing unterminated
+ * results to @p. Using strncpy() remains ambiguous and fragile.
+ * Instead, please choose an alternative, so that the expectation
+ * of @p's contents is unambiguous:
+ *
+ * +--------------------+-----------------+------------+
+ * | @p needs to be:    | padded to @size | not padded |
+ * +====================+=================+============+
+ * |     NUL-terminated | strscpy_pad()   | strscpy()  |
+ * +--------------------+-----------------+------------+
+ * | not NUL-terminated | strtomem_pad()  | strtomem() |
+ * +--------------------+-----------------+------------+
+ *
+ * Note strscpy*()'s differing return values for detecting truncation,
+ * and strtomem*()'s expectation that the destination is marked with
+ * __nonstring when it is a character array.
+ *
+ */
 __FORTIFY_INLINE __diagnose_as(__builtin_strncpy, 1, 2, 3)
 char *strncpy(char * const POS p, const char *q, __kernel_size_t size)
 {
-	size_t p_size = __builtin_object_size(p, 1);
+	size_t p_size = __member_size(p);
 
-	if (__builtin_constant_p(size) && p_size < size)
+	if (__compiletime_lessthan(p_size, size))
 		__write_overflow();
 	if (p_size < size)
 		fortify_panic(__func__);
@@ -92,9 +134,9 @@ char *strncpy(char * const POS p, const char *q, __kernel_size_t size)
 __FORTIFY_INLINE __diagnose_as(__builtin_strcat, 1, 2)
 char *strcat(char * const POS p, const char *q)
 {
-	size_t p_size = __builtin_object_size(p, 1);
+	size_t p_size = __member_size(p);
 
-	if (p_size == (size_t)-1)
+	if (p_size == SIZE_MAX)
 		return __underlying_strcat(p, q);
 	if (strlcat(p, q, p_size) >= p_size)
 		fortify_panic(__func__);
@@ -104,12 +146,12 @@ char *strcat(char * const POS p, const char *q)
 extern __kernel_size_t __real_strnlen(const char *, __kernel_size_t) __RENAME(strnlen);
 __FORTIFY_INLINE __kernel_size_t strnlen(const char * const POS p, __kernel_size_t maxlen)
 {
-	size_t p_size = __builtin_object_size(p, 1);
+	size_t p_size = __member_size(p);
 	size_t p_len = __compiletime_strlen(p);
 	size_t ret;
 
 	/* We can take compile-time actions when maxlen is const. */
-	if (__builtin_constant_p(maxlen) && p_len != (size_t)-1) {
+	if (__builtin_constant_p(maxlen) && p_len != SIZE_MAX) {
 		/* If p is const, we can use its compile-time-known len. */
 		if (maxlen >= p_size)
 			return p_len;
@@ -134,10 +176,10 @@ __FORTIFY_INLINE __diagnose_as(__builtin_strlen, 1)
 __kernel_size_t __fortify_strlen(const char * const POS p)
 {
 	__kernel_size_t ret;
-	size_t p_size = __builtin_object_size(p, 1);
+	size_t p_size = __member_size(p);
 
 	/* Give up if we don't know how large p is. */
-	if (p_size == (size_t)-1)
+	if (p_size == SIZE_MAX)
 		return __underlying_strlen(p);
 	ret = strnlen(p, p_size);
 	if (p_size <= ret)
@@ -149,12 +191,12 @@ __kernel_size_t __fortify_strlen(const char * const POS p)
 extern size_t __real_strlcpy(char *, const char *, size_t) __RENAME(strlcpy);
 __FORTIFY_INLINE size_t strlcpy(char * const POS p, const char * const POS q, size_t size)
 {
-	size_t p_size = __builtin_object_size(p, 1);
-	size_t q_size = __builtin_object_size(q, 1);
+	size_t p_size = __member_size(p);
+	size_t q_size = __member_size(q);
 	size_t q_len;	/* Full count of source string length. */
 	size_t len;	/* Count of characters going into destination. */
 
-	if (p_size == (size_t)-1 && q_size == (size_t)-1)
+	if (p_size == SIZE_MAX && q_size == SIZE_MAX)
 		return __real_strlcpy(p, q, size);
 	q_len = strlen(q);
 	len = (q_len >= size) ? size - 1 : q_len;
@@ -178,18 +220,18 @@ __FORTIFY_INLINE ssize_t strscpy(char * const POS p, const char * const POS q, s
 {
 	size_t len;
 	/* Use string size rather than possible enclosing struct size. */
-	size_t p_size = __builtin_object_size(p, 1);
-	size_t q_size = __builtin_object_size(q, 1);
+	size_t p_size = __member_size(p);
+	size_t q_size = __member_size(q);
 
 	/* If we cannot get size of p and q default to call strscpy. */
-	if (p_size == (size_t) -1 && q_size == (size_t) -1)
+	if (p_size == SIZE_MAX && q_size == SIZE_MAX)
 		return __real_strscpy(p, q, size);
 
 	/*
 	 * If size can be known at compile time and is greater than
 	 * p_size, generate a compile time write overflow error.
 	 */
-	if (__builtin_constant_p(size) && size > p_size)
+	if (__compiletime_lessthan(p_size, size))
 		__write_overflow();
 
 	/*
@@ -224,10 +266,10 @@ __FORTIFY_INLINE __diagnose_as(__builtin_strncat, 1, 2, 3)
 char *strncat(char * const POS p, const char * const POS q, __kernel_size_t count)
 {
 	size_t p_len, copy_len;
-	size_t p_size = __builtin_object_size(p, 1);
-	size_t q_size = __builtin_object_size(q, 1);
+	size_t p_size = __member_size(p);
+	size_t q_size = __member_size(q);
 
-	if (p_size == (size_t)-1 && q_size == (size_t)-1)
+	if (p_size == SIZE_MAX && q_size == SIZE_MAX)
 		return __underlying_strncat(p, q, count);
 	p_len = strlen(p);
 	copy_len = strnlen(q, count);
@@ -246,15 +288,16 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size,
 		/*
 		 * Length argument is a constant expression, so we
 		 * can perform compile-time bounds checking where
-		 * buffer sizes are known.
+		 * buffer sizes are also known at compile time.
 		 */
 
 		/* Error when size is larger than enclosing struct. */
-		if (p_size > p_size_field && p_size < size)
+		if (__compiletime_lessthan(p_size_field, p_size) &&
+		    __compiletime_lessthan(p_size, size))
 			__write_overflow();
 
 		/* Warn when write size is larger than dest field. */
-		if (p_size_field < size)
+		if (__compiletime_lessthan(p_size_field, size))
 			__write_overflow_field(p_size_field, size);
 	}
 	/*
@@ -268,10 +311,10 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size,
 	/*
 	 * Always stop accesses beyond the struct that contains the
 	 * field, when the buffer's remaining size is known.
-	 * (The -1 test is to optimize away checks where the buffer
+	 * (The SIZE_MAX test is to optimize away checks where the buffer
 	 * lengths are unknown.)
 	 */
-	if (p_size != (size_t)(-1) && p_size < size)
+	if (p_size != SIZE_MAX && p_size < size)
 		fortify_panic("memset");
 }
 
@@ -282,11 +325,11 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size,
 })
 
 /*
- * __builtin_object_size() must be captured here to avoid evaluating argument
- * side-effects further into the macro layers.
+ * __struct_size() vs __member_size() must be captured here to avoid
+ * evaluating argument side-effects further into the macro layers.
  */
 #define memset(p, c, s) __fortify_memset_chk(p, c, s,			\
-		__builtin_object_size(p, 0), __builtin_object_size(p, 1))
+		__struct_size(p), __member_size(p))
 
 /*
  * To make sure the compiler can enforce protection against buffer overflows,
@@ -319,7 +362,7 @@ __FORTIFY_INLINE void fortify_memset_chk(__kernel_size_t size,
  * V = vulnerable to run-time overflow (will need refactoring to solve)
  *
  */
-__FORTIFY_INLINE void fortify_memcpy_chk(__kernel_size_t size,
+__FORTIFY_INLINE bool fortify_memcpy_chk(__kernel_size_t size,
 					 const size_t p_size,
 					 const size_t q_size,
 					 const size_t p_size_field,
@@ -330,25 +373,28 @@ __FORTIFY_INLINE void fortify_memcpy_chk(__kernel_size_t size,
 		/*
 		 * Length argument is a constant expression, so we
 		 * can perform compile-time bounds checking where
-		 * buffer sizes are known.
+		 * buffer sizes are also known at compile time.
 		 */
 
 		/* Error when size is larger than enclosing struct. */
-		if (p_size > p_size_field && p_size < size)
+		if (__compiletime_lessthan(p_size_field, p_size) &&
+		    __compiletime_lessthan(p_size, size))
 			__write_overflow();
-		if (q_size > q_size_field && q_size < size)
+		if (__compiletime_lessthan(q_size_field, q_size) &&
+		    __compiletime_lessthan(q_size, size))
 			__read_overflow2();
 
 		/* Warn when write size argument larger than dest field. */
-		if (p_size_field < size)
+		if (__compiletime_lessthan(p_size_field, size))
 			__write_overflow_field(p_size_field, size);
 		/*
 		 * Warn for source field over-read when building with W=1
 		 * or when an over-write happened, so both can be fixed at
 		 * the same time.
 		 */
-		if ((IS_ENABLED(KBUILD_EXTRA_WARN1) || p_size_field < size) &&
-		    q_size_field < size)
+		if ((IS_ENABLED(KBUILD_EXTRA_WARN1) ||
+		     __compiletime_lessthan(p_size_field, size)) &&
+		    __compiletime_lessthan(q_size_field, size))
 			__read_overflow2_field(q_size_field, size);
 	}
 	/*
@@ -362,41 +408,104 @@ __FORTIFY_INLINE void fortify_memcpy_chk(__kernel_size_t size,
 	/*
 	 * Always stop accesses beyond the struct that contains the
 	 * field, when the buffer's remaining size is known.
-	 * (The -1 test is to optimize away checks where the buffer
+	 * (The SIZE_MAX test is to optimize away checks where the buffer
 	 * lengths are unknown.)
 	 */
-	if ((p_size != (size_t)(-1) && p_size < size) ||
-	    (q_size != (size_t)(-1) && q_size < size))
+	if ((p_size != SIZE_MAX && p_size < size) ||
+	    (q_size != SIZE_MAX && q_size < size))
 		fortify_panic(func);
+
+	/*
+	 * Warn when writing beyond destination field size.
+	 *
+	 * We must ignore p_size_field == 0 for existing 0-element
+	 * fake flexible arrays, until they are all converted to
+	 * proper flexible arrays.
+	 *
+	 * The implementation of __builtin_*object_size() behaves
+	 * like sizeof() when not directly referencing a flexible
+	 * array member, which means there will be many bounds checks
+	 * that will appear at run-time, without a way for them to be
+	 * detected at compile-time (as can be done when the destination
+	 * is specifically the flexible array member).
+	 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101832
+	 */
+	if (p_size_field != 0 && p_size_field != SIZE_MAX &&
+	    p_size != p_size_field && p_size_field < size)
+		return true;
+
+	return false;
 }
 
 #define __fortify_memcpy_chk(p, q, size, p_size, q_size,		\
 			     p_size_field, q_size_field, op) ({		\
 	size_t __fortify_size = (size_t)(size);				\
-	fortify_memcpy_chk(__fortify_size, p_size, q_size,		\
-			   p_size_field, q_size_field, #op);		\
+	WARN_ONCE(fortify_memcpy_chk(__fortify_size, p_size, q_size,	\
+				     p_size_field, q_size_field, #op),	\
+		  #op ": detected field-spanning write (size %zu) of single %s (size %zu)\n", \
+		  __fortify_size,					\
+		  "field \"" #p "\" at " __FILE__ ":" __stringify(__LINE__), \
+		  p_size_field);					\
 	__underlying_##op(p, q, __fortify_size);			\
 })
 
 /*
- * __builtin_object_size() must be captured here to avoid evaluating argument
- * side-effects further into the macro layers.
+ * Notes about compile-time buffer size detection:
+ *
+ * With these types...
+ *
+ *	struct middle {
+ *		u16 a;
+ *		u8 middle_buf[16];
+ *		int b;
+ *	};
+ *	struct end {
+ *		u16 a;
+ *		u8 end_buf[16];
+ *	};
+ *	struct flex {
+ *		int a;
+ *		u8 flex_buf[];
+ *	};
+ *
+ *	void func(TYPE *ptr) { ... }
+ *
+ * Cases where destination size cannot be currently detected:
+ * - the size of ptr's object (seemingly by design, gcc & clang fail):
+ *	__builtin_object_size(ptr, 1) == SIZE_MAX
+ * - the size of flexible arrays in ptr's obj (by design, dynamic size):
+ *	__builtin_object_size(ptr->flex_buf, 1) == SIZE_MAX
+ * - the size of ANY array at the end of ptr's obj (gcc and clang bug):
+ *	__builtin_object_size(ptr->end_buf, 1) == SIZE_MAX
+ *	https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101836
+ *
+ * Cases where destination size is currently detected:
+ * - the size of non-array members within ptr's object:
+ *	__builtin_object_size(ptr->a, 1) == 2
+ * - the size of non-flexible-array in the middle of ptr's obj:
+ *	__builtin_object_size(ptr->middle_buf, 1) == 16
+ *
+ */
+
+/*
+ * __struct_size() vs __member_size() must be captured here to avoid
+ * evaluating argument side-effects further into the macro layers.
  */
 #define memcpy(p, q, s)  __fortify_memcpy_chk(p, q, s,			\
-		__builtin_object_size(p, 0), __builtin_object_size(q, 0), \
-		__builtin_object_size(p, 1), __builtin_object_size(q, 1), \
+		__struct_size(p), __struct_size(q),			\
+		__member_size(p), __member_size(q),			\
 		memcpy)
 #define memmove(p, q, s)  __fortify_memcpy_chk(p, q, s,			\
-		__builtin_object_size(p, 0), __builtin_object_size(q, 0), \
-		__builtin_object_size(p, 1), __builtin_object_size(q, 1), \
+		__struct_size(p), __struct_size(q),			\
+		__member_size(p), __member_size(q),			\
 		memmove)
 
 extern void *__real_memscan(void *, int, __kernel_size_t) __RENAME(memscan);
 __FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size)
 {
-	size_t p_size = __builtin_object_size(p, 0);
+	size_t p_size = __struct_size(p);
 
-	if (__builtin_constant_p(size) && p_size < size)
+	if (__compiletime_lessthan(p_size, size))
 		__read_overflow();
 	if (p_size < size)
 		fortify_panic(__func__);
@@ -406,13 +515,13 @@ __FORTIFY_INLINE void *memscan(void * const POS0 p, int c, __kernel_size_t size)
 __FORTIFY_INLINE __diagnose_as(__builtin_memcmp, 1, 2, 3)
 int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t size)
 {
-	size_t p_size = __builtin_object_size(p, 0);
-	size_t q_size = __builtin_object_size(q, 0);
+	size_t p_size = __struct_size(p);
+	size_t q_size = __struct_size(q);
 
 	if (__builtin_constant_p(size)) {
-		if (p_size < size)
+		if (__compiletime_lessthan(p_size, size))
 			__read_overflow();
-		if (q_size < size)
+		if (__compiletime_lessthan(q_size, size))
 			__read_overflow2();
 	}
 	if (p_size < size || q_size < size)
@@ -423,9 +532,9 @@ int memcmp(const void * const POS0 p, const void * const POS0 q, __kernel_size_t
 __FORTIFY_INLINE __diagnose_as(__builtin_memchr, 1, 2, 3)
 void *memchr(const void * const POS0 p, int c, __kernel_size_t size)
 {
-	size_t p_size = __builtin_object_size(p, 0);
+	size_t p_size = __struct_size(p);
 
-	if (__builtin_constant_p(size) && p_size < size)
+	if (__compiletime_lessthan(p_size, size))
 		__read_overflow();
 	if (p_size < size)
 		fortify_panic(__func__);
@@ -435,9 +544,9 @@ void *memchr(const void * const POS0 p, int c, __kernel_size_t size)
 void *__real_memchr_inv(const void *s, int c, size_t n) __RENAME(memchr_inv);
 __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size)
 {
-	size_t p_size = __builtin_object_size(p, 0);
+	size_t p_size = __struct_size(p);
 
-	if (__builtin_constant_p(size) && p_size < size)
+	if (__compiletime_lessthan(p_size, size))
 		__read_overflow();
 	if (p_size < size)
 		fortify_panic(__func__);
@@ -447,9 +556,9 @@ __FORTIFY_INLINE void *memchr_inv(const void * const POS0 p, int c, size_t size)
 extern void *__real_kmemdup(const void *src, size_t len, gfp_t gfp) __RENAME(kmemdup);
 __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp)
 {
-	size_t p_size = __builtin_object_size(p, 0);
+	size_t p_size = __struct_size(p);
 
-	if (__builtin_constant_p(size) && p_size < size)
+	if (__compiletime_lessthan(p_size, size))
 		__read_overflow();
 	if (p_size < size)
 		fortify_panic(__func__);
@@ -460,16 +569,18 @@ __FORTIFY_INLINE void *kmemdup(const void * const POS0 p, size_t size, gfp_t gfp
 __FORTIFY_INLINE __diagnose_as(__builtin_strcpy, 1, 2)
 char *strcpy(char * const POS p, const char * const POS q)
 {
-	size_t p_size = __builtin_object_size(p, 1);
-	size_t q_size = __builtin_object_size(q, 1);
+	size_t p_size = __member_size(p);
+	size_t q_size = __member_size(q);
 	size_t size;
 
 	/* If neither buffer size is known, immediately give up. */
-	if (p_size == (size_t)-1 && q_size == (size_t)-1)
+	if (__builtin_constant_p(p_size) &&
+	    __builtin_constant_p(q_size) &&
+	    p_size == SIZE_MAX && q_size == SIZE_MAX)
 		return __underlying_strcpy(p, q);
 	size = strlen(q) + 1;
 	/* Compile-time check for const size overflow. */
-	if (__builtin_constant_p(size) && p_size < size)
+	if (__compiletime_lessthan(p_size, size))
 		__write_overflow();
 	/* Run-time check for dynamic size overflow. */
 	if (p_size < size)
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 9eced4cc286e..7591d2d2dcbb 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -1472,7 +1472,7 @@ struct super_block {
 	const struct xattr_handler **s_xattr;
 #ifdef CONFIG_FS_ENCRYPTION
 	const struct fscrypt_operations	*s_cop;
-	struct key		*s_master_keys; /* master crypto keys in use */
+	struct fscrypt_keyring	*s_master_keys; /* master crypto keys in use */
 #endif
 #ifdef CONFIG_FS_VERITY
 	const struct fsverity_operations *s_vop;
@@ -2038,9 +2038,10 @@ umode_t mode_strip_sgid(struct user_namespace *mnt_userns,
  * the kernel specify what kind of dirent layout it wants to have.
  * This allows the kernel to read directories into kernel space or
  * to have different dirent layouts depending on the binary type.
+ * Return 'true' to keep going and 'false' if there are no more entries.
  */
 struct dir_context;
-typedef int (*filldir_t)(struct dir_context *, const char *, int, loff_t, u64,
+typedef bool (*filldir_t)(struct dir_context *, const char *, int, loff_t, u64,
 			 unsigned);
 
 struct dir_context {
@@ -3540,17 +3541,17 @@ static inline bool dir_emit(struct dir_context *ctx,
 			    const char *name, int namelen,
 			    u64 ino, unsigned type)
 {
-	return ctx->actor(ctx, name, namelen, ctx->pos, ino, type) == 0;
+	return ctx->actor(ctx, name, namelen, ctx->pos, ino, type);
 }
 static inline bool dir_emit_dot(struct file *file, struct dir_context *ctx)
 {
 	return ctx->actor(ctx, ".", 1, ctx->pos,
-			  file->f_path.dentry->d_inode->i_ino, DT_DIR) == 0;
+			  file->f_path.dentry->d_inode->i_ino, DT_DIR);
 }
 static inline bool dir_emit_dotdot(struct file *file, struct dir_context *ctx)
 {
 	return ctx->actor(ctx, "..", 2, ctx->pos,
-			  parent_ino(file->f_path.dentry), DT_DIR) == 0;
+			  parent_ino(file->f_path.dentry), DT_DIR);
 }
 static inline bool dir_emit_dots(struct file *file, struct dir_context *ctx)
 {
diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h
index 7d2f1e0f23b1..cad78b569c7e 100644
--- a/include/linux/fscrypt.h
+++ b/include/linux/fscrypt.h
@@ -161,24 +161,21 @@ struct fscrypt_operations {
 				      int *ino_bits_ret, int *lblk_bits_ret);
 
 	/*
-	 * Return the number of block devices to which the filesystem may write
-	 * encrypted file contents.
+	 * Return an array of pointers to the block devices to which the
+	 * filesystem may write encrypted file contents, NULL if the filesystem
+	 * only has a single such block device, or an ERR_PTR() on error.
+	 *
+	 * On successful non-NULL return, *num_devs is set to the number of
+	 * devices in the returned array.  The caller must free the returned
+	 * array using kfree().
 	 *
 	 * If the filesystem can use multiple block devices (other than block
 	 * devices that aren't used for encrypted file contents, such as
 	 * external journal devices), and wants to support inline encryption,
 	 * then it must implement this function.  Otherwise it's not needed.
 	 */
-	int (*get_num_devices)(struct super_block *sb);
-
-	/*
-	 * If ->get_num_devices() returns a value greater than 1, then this
-	 * function is called to get the array of request_queues that the
-	 * filesystem is using -- one per block device.  (There may be duplicate
-	 * entries in this array, as block devices can share a request_queue.)
-	 */
-	void (*get_devices)(struct super_block *sb,
-			    struct request_queue **devs);
+	struct block_device **(*get_devices)(struct super_block *sb,
+					     unsigned int *num_devs);
 };
 
 static inline struct fscrypt_info *fscrypt_get_info(const struct inode *inode)
@@ -295,8 +292,6 @@ int fscrypt_parse_test_dummy_encryption(const struct fs_parameter *param,
 				    struct fscrypt_dummy_policy *dummy_policy);
 bool fscrypt_dummy_policies_equal(const struct fscrypt_dummy_policy *p1,
 				  const struct fscrypt_dummy_policy *p2);
-int fscrypt_set_test_dummy_encryption(struct super_block *sb, const char *arg,
-				struct fscrypt_dummy_policy *dummy_policy);
 void fscrypt_show_test_dummy_encryption(struct seq_file *seq, char sep,
 					struct super_block *sb);
 static inline bool
@@ -312,7 +307,7 @@ fscrypt_free_dummy_policy(struct fscrypt_dummy_policy *dummy_policy)
 }
 
 /* keyring.c */
-void fscrypt_sb_free(struct super_block *sb);
+void fscrypt_sb_delete(struct super_block *sb);
 int fscrypt_ioctl_add_key(struct file *filp, void __user *arg);
 int fscrypt_add_test_dummy_key(struct super_block *sb,
 			       const struct fscrypt_dummy_policy *dummy_policy);
@@ -353,7 +348,7 @@ u64 fscrypt_fname_siphash(const struct inode *dir, const struct qstr *name);
 int fscrypt_d_revalidate(struct dentry *dentry, unsigned int flags);
 
 /* bio.c */
-void fscrypt_decrypt_bio(struct bio *bio);
+bool fscrypt_decrypt_bio(struct bio *bio);
 int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,
 			  sector_t pblk, unsigned int len);
 
@@ -526,7 +521,7 @@ fscrypt_free_dummy_policy(struct fscrypt_dummy_policy *dummy_policy)
 }
 
 /* keyring.c */
-static inline void fscrypt_sb_free(struct super_block *sb)
+static inline void fscrypt_sb_delete(struct super_block *sb)
 {
 }
 
@@ -646,8 +641,9 @@ static inline int fscrypt_d_revalidate(struct dentry *dentry,
 }
 
 /* bio.c */
-static inline void fscrypt_decrypt_bio(struct bio *bio)
+static inline bool fscrypt_decrypt_bio(struct bio *bio)
 {
+	return true;
 }
 
 static inline int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk,
@@ -768,7 +764,7 @@ bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode,
 bool fscrypt_mergeable_bio_bh(struct bio *bio,
 			      const struct buffer_head *next_bh);
 
-bool fscrypt_dio_supported(struct kiocb *iocb, struct iov_iter *iter);
+bool fscrypt_dio_supported(struct inode *inode);
 
 u64 fscrypt_limit_io_blocks(const struct inode *inode, u64 lblk, u64 nr_blocks);
 
@@ -801,11 +797,8 @@ static inline bool fscrypt_mergeable_bio_bh(struct bio *bio,
 	return true;
 }
 
-static inline bool fscrypt_dio_supported(struct kiocb *iocb,
-					 struct iov_iter *iter)
+static inline bool fscrypt_dio_supported(struct inode *inode)
 {
-	const struct inode *inode = file_inode(iocb->ki_filp);
-
 	return !fscrypt_needs_contents_encryption(inode);
 }
 
diff --git a/include/linux/genl_magic_func.h b/include/linux/genl_magic_func.h
index 939b1a8f571b..4a4b387181ad 100644
--- a/include/linux/genl_magic_func.h
+++ b/include/linux/genl_magic_func.h
@@ -294,6 +294,7 @@ static struct genl_family ZZZ_genl_family __ro_after_init = {
 	.ops = ZZZ_genl_ops,
 	.n_ops = ARRAY_SIZE(ZZZ_genl_ops),
 	.mcgrps = ZZZ_genl_mcgrps,
+	.resv_start_op = 42, /* drbd is currently the only user */
 	.n_mcgrps = ARRAY_SIZE(ZZZ_genl_mcgrps),
 	.module = THIS_MODULE,
 };
diff --git a/include/linux/hdmi.h b/include/linux/hdmi.h
index c8ec982ff498..2f4dcc8d060e 100644
--- a/include/linux/hdmi.h
+++ b/include/linux/hdmi.h
@@ -336,7 +336,12 @@ ssize_t hdmi_audio_infoframe_pack(struct hdmi_audio_infoframe *frame,
 				  void *buffer, size_t size);
 ssize_t hdmi_audio_infoframe_pack_only(const struct hdmi_audio_infoframe *frame,
 				       void *buffer, size_t size);
-int hdmi_audio_infoframe_check(struct hdmi_audio_infoframe *frame);
+int hdmi_audio_infoframe_check(const struct hdmi_audio_infoframe *frame);
+
+struct dp_sdp;
+ssize_t
+hdmi_audio_infoframe_pack_for_dp(const struct hdmi_audio_infoframe *frame,
+				 struct dp_sdp *sdp, u8 dp_version);
 
 enum hdmi_3d_structure {
 	HDMI_3D_STRUCTURE_INVALID = -1,
diff --git a/include/linux/i2c.h b/include/linux/i2c.h
index 8eab5017bff3..f7c49bbdb8a1 100644
--- a/include/linux/i2c.h
+++ b/include/linux/i2c.h
@@ -273,7 +273,7 @@ struct i2c_driver {
 
 	/* Standard driver model interfaces */
 	int (*probe)(struct i2c_client *client, const struct i2c_device_id *id);
-	int (*remove)(struct i2c_client *client);
+	void (*remove)(struct i2c_client *client);
 
 	/* New driver model interface to aid the seamless removal of the
 	 * current probe()'s, more commonly unused than used second parameter.
diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h
index b6e6d5b40774..79690938d9a2 100644
--- a/include/linux/ieee80211.h
+++ b/include/linux/ieee80211.h
@@ -2888,7 +2888,8 @@ ieee80211_he_spr_size(const u8 *he_spr_ie)
 /* Calculate 802.11be EHT capabilities IE Tx/Rx EHT MCS NSS Support Field size */
 static inline u8
 ieee80211_eht_mcs_nss_size(const struct ieee80211_he_cap_elem *he_cap,
-			   const struct ieee80211_eht_cap_elem_fixed *eht_cap)
+			   const struct ieee80211_eht_cap_elem_fixed *eht_cap,
+			   bool from_ap)
 {
 	u8 count = 0;
 
@@ -2909,7 +2910,10 @@ ieee80211_eht_mcs_nss_size(const struct ieee80211_he_cap_elem *he_cap,
 	if (eht_cap->phy_cap_info[0] & IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ)
 		count += 3;
 
-	return count ? count : 4;
+	if (count)
+		return count;
+
+	return from_ap ? 3 : 4;
 }
 
 /* 802.11be EHT PPE Thresholds */
@@ -2945,7 +2949,8 @@ ieee80211_eht_ppe_size(u16 ppe_thres_hdr, const u8 *phy_cap_info)
 }
 
 static inline bool
-ieee80211_eht_capa_size_ok(const u8 *he_capa, const u8 *data, u8 len)
+ieee80211_eht_capa_size_ok(const u8 *he_capa, const u8 *data, u8 len,
+			   bool from_ap)
 {
 	const struct ieee80211_eht_cap_elem_fixed *elem = (const void *)data;
 	u8 needed = sizeof(struct ieee80211_eht_cap_elem_fixed);
@@ -2954,7 +2959,8 @@ ieee80211_eht_capa_size_ok(const u8 *he_capa, const u8 *data, u8 len)
 		return false;
 
 	needed += ieee80211_eht_mcs_nss_size((const void *)he_capa,
-					     (const void *)data);
+					     (const void *)data,
+					     from_ap);
 	if (len < needed)
 		return false;
 
diff --git a/include/linux/if_pppol2tp.h b/include/linux/if_pppol2tp.h
index 96d40942e5a3..c87efd333faa 100644
--- a/include/linux/if_pppol2tp.h
+++ b/include/linux/if_pppol2tp.h
@@ -4,8 +4,6 @@
  *
  * This file supplies definitions required by the PPP over L2TP driver
  * (l2tp_ppp.c).  All version information wrt this file is located in l2tp_ppp.c
- *
- * License:
  */
 #ifndef __LINUX_IF_PPPOL2TP_H
 #define __LINUX_IF_PPPOL2TP_H
diff --git a/include/linux/if_pppox.h b/include/linux/if_pppox.h
index 69e813bcb947..ff3beda1312c 100644
--- a/include/linux/if_pppox.h
+++ b/include/linux/if_pppox.h
@@ -5,8 +5,6 @@
  *
  * This file supplies definitions required by the PPP over Ethernet driver
  * (pppox.c).  All version information wrt this file is located in pppox.c
- *
- * License:
  */
 #ifndef __LINUX_IF_PPPOX_H
 #define __LINUX_IF_PPPOX_H
diff --git a/include/linux/igmp.h b/include/linux/igmp.h
index 93c262ecbdc9..78890143f079 100644
--- a/include/linux/igmp.h
+++ b/include/linux/igmp.h
@@ -118,9 +118,9 @@ extern int ip_mc_source(int add, int omode, struct sock *sk,
 		struct ip_mreq_source *mreqs, int ifindex);
 extern int ip_mc_msfilter(struct sock *sk, struct ip_msfilter *msf,int ifindex);
 extern int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf,
-		struct ip_msfilter __user *optval, int __user *optlen);
+			sockptr_t optval, sockptr_t optlen);
 extern int ip_mc_gsfget(struct sock *sk, struct group_filter *gsf,
-			struct sockaddr_storage __user *p);
+			sockptr_t optval, size_t offset);
 extern int ip_mc_sf_allow(struct sock *sk, __be32 local, __be32 rmt,
 			  int dif, int sdif);
 extern void ip_mc_init_dev(struct in_device *);
diff --git a/include/linux/init.h b/include/linux/init.h
index baf0b29a7010..a0a90cd73ebe 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -47,7 +47,7 @@
 
 /* These are for everybody (although not all archs will actually
    discard it in modules) */
-#define __init		__section(".init.text") __cold  __latent_entropy __noinitretpoline __nocfi
+#define __init		__section(".init.text") __cold  __latent_entropy __noinitretpoline
 #define __initdata	__section(".init.data")
 #define __initconst	__section(".init.rodata")
 #define __exitdata	__section(".exit.data")
@@ -220,8 +220,8 @@ extern bool initcall_debug;
 	__initcall_name(initstub, __iid, id)
 
 #define __define_initcall_stub(__stub, fn)			\
-	int __init __cficanonical __stub(void);			\
-	int __init __cficanonical __stub(void)			\
+	int __init __stub(void);				\
+	int __init __stub(void)					\
 	{ 							\
 		return fn();					\
 	}							\
diff --git a/include/linux/ioport.h b/include/linux/ioport.h
index 616b683563a9..8a76dca9deee 100644
--- a/include/linux/ioport.h
+++ b/include/linux/ioport.h
@@ -172,6 +172,11 @@ enum {
 #define DEFINE_RES_MEM(_start, _size)					\
 	DEFINE_RES_MEM_NAMED((_start), (_size), NULL)
 
+#define DEFINE_RES_REG_NAMED(_start, _size, _name)			\
+	DEFINE_RES_NAMED((_start), (_size), (_name), IORESOURCE_REG)
+#define DEFINE_RES_REG(_start, _size)					\
+	DEFINE_RES_REG_NAMED((_start), (_size), NULL)
+
 #define DEFINE_RES_IRQ_NAMED(_irq, _name)				\
 	DEFINE_RES_NAMED((_irq), 1, (_name), IORESOURCE_IRQ)
 #define DEFINE_RES_IRQ(_irq)						\
diff --git a/include/linux/iosys-map.h b/include/linux/iosys-map.h
index a533cae189d7..cb71aa616bd3 100644
--- a/include/linux/iosys-map.h
+++ b/include/linux/iosys-map.h
@@ -46,10 +46,13 @@
  *
  *	iosys_map_set_vaddr(&map, 0xdeadbeaf);
  *
- * To set an address in I/O memory, use iosys_map_set_vaddr_iomem().
+ * To set an address in I/O memory, use IOSYS_MAP_INIT_VADDR_IOMEM() or
+ * iosys_map_set_vaddr_iomem().
  *
  * .. code-block:: c
  *
+ *	struct iosys_map map = IOSYS_MAP_INIT_VADDR_IOMEM(0xdeadbeaf);
+ *
  *	iosys_map_set_vaddr_iomem(&map, 0xdeadbeaf);
  *
  * Instances of struct iosys_map do not have to be cleaned up, but
@@ -122,6 +125,16 @@ struct iosys_map {
 	}
 
 /**
+ * IOSYS_MAP_INIT_VADDR_IOMEM - Initializes struct iosys_map to an address in I/O memory
+ * @vaddr_iomem_:	An I/O-memory address
+ */
+#define IOSYS_MAP_INIT_VADDR_IOMEM(vaddr_iomem_)	\
+	{						\
+		.vaddr_iomem = (vaddr_iomem_),		\
+		.is_iomem = true,			\
+	}
+
+/**
  * IOSYS_MAP_INIT_OFFSET - Initializes struct iosys_map from another iosys_map
  * @map_:	The dma-buf mapping structure to copy from
  * @offset_:	Offset to add to the other mapping
diff --git a/include/linux/kallsyms.h b/include/linux/kallsyms.h
index ad39636e0c3f..649faac31ddb 100644
--- a/include/linux/kallsyms.h
+++ b/include/linux/kallsyms.h
@@ -15,7 +15,7 @@
 
 #include <asm/sections.h>
 
-#define KSYM_NAME_LEN 128
+#define KSYM_NAME_LEN 512
 #define KSYM_SYMBOL_LEN (sizeof("%s+%#lx/%#lx [%s %s]") + \
 			(KSYM_NAME_LEN - 1) + \
 			2*(BITS_PER_LONG*3/10) + (MODULE_NAME_LEN - 1) + \
diff --git a/include/linux/key.h b/include/linux/key.h
index 7febc4881363..d27477faf00d 100644
--- a/include/linux/key.h
+++ b/include/linux/key.h
@@ -88,6 +88,12 @@ enum key_need_perm {
 	KEY_DEFER_PERM_CHECK,	/* Special: permission check is deferred */
 };
 
+enum key_lookup_flag {
+	KEY_LOOKUP_CREATE = 0x01,
+	KEY_LOOKUP_PARTIAL = 0x02,
+	KEY_LOOKUP_ALL = (KEY_LOOKUP_CREATE | KEY_LOOKUP_PARTIAL),
+};
+
 struct seq_file;
 struct user_struct;
 struct signal_struct;
diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h
index 55041d2f884d..a0b92be98984 100644
--- a/include/linux/kprobes.h
+++ b/include/linux/kprobes.h
@@ -103,6 +103,7 @@ struct kprobe {
 				   * this flag is only for optimized_kprobe.
 				   */
 #define KPROBE_FLAG_FTRACE	8 /* probe is using ftrace */
+#define KPROBE_FLAG_ON_FUNC_ENTRY	16 /* probe is on the function entry */
 
 /* Has this kprobe gone ? */
 static inline bool kprobe_gone(struct kprobe *p)
diff --git a/include/linux/libata.h b/include/linux/libata.h
index 698032e5ef2d..20765d1c5f80 100644
--- a/include/linux/libata.h
+++ b/include/linux/libata.h
@@ -1136,8 +1136,8 @@ extern int ata_scsi_slave_config(struct scsi_device *sdev);
 extern void ata_scsi_slave_destroy(struct scsi_device *sdev);
 extern int ata_scsi_change_queue_depth(struct scsi_device *sdev,
 				       int queue_depth);
-extern int __ata_change_queue_depth(struct ata_port *ap, struct scsi_device *sdev,
-				    int queue_depth);
+extern int ata_change_queue_depth(struct ata_port *ap, struct ata_device *dev,
+				  struct scsi_device *sdev, int queue_depth);
 extern struct ata_device *ata_dev_pair(struct ata_device *adev);
 extern int ata_do_set_mode(struct ata_link *link, struct ata_device **r_failed_dev);
 extern void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap);
diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h
index 60fff133c0b1..ec119da1d89b 100644
--- a/include/linux/lsm_hook_defs.h
+++ b/include/linux/lsm_hook_defs.h
@@ -224,6 +224,7 @@ LSM_HOOK(int, -ENOSYS, task_prctl, int option, unsigned long arg2,
 	 unsigned long arg3, unsigned long arg4, unsigned long arg5)
 LSM_HOOK(void, LSM_RET_VOID, task_to_inode, struct task_struct *p,
 	 struct inode *inode)
+LSM_HOOK(int, 0, userns_create, const struct cred *cred)
 LSM_HOOK(int, 0, ipc_permission, struct kern_ipc_perm *ipcp, short flag)
 LSM_HOOK(void, LSM_RET_VOID, ipc_getsecid, struct kern_ipc_perm *ipcp,
 	 u32 *secid)
@@ -253,7 +254,7 @@ LSM_HOOK(int, 0, sem_semop, struct kern_ipc_perm *perm, struct sembuf *sops,
 LSM_HOOK(int, 0, netlink_send, struct sock *sk, struct sk_buff *skb)
 LSM_HOOK(void, LSM_RET_VOID, d_instantiate, struct dentry *dentry,
 	 struct inode *inode)
-LSM_HOOK(int, -EINVAL, getprocattr, struct task_struct *p, char *name,
+LSM_HOOK(int, -EINVAL, getprocattr, struct task_struct *p, const char *name,
 	 char **value)
 LSM_HOOK(int, -EINVAL, setprocattr, const char *name, void *value, size_t size)
 LSM_HOOK(int, 0, ismaclabel, const char *name)
diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h
index 3aa6030302f5..4ec80b96c22e 100644
--- a/include/linux/lsm_hooks.h
+++ b/include/linux/lsm_hooks.h
@@ -806,6 +806,10 @@
  *	security attributes, e.g. for /proc/pid inodes.
  *	@p contains the task_struct for the task.
  *	@inode contains the inode structure for the inode.
+ * @userns_create:
+ *	Check permission prior to creating a new user namespace.
+ *	@cred points to prepared creds.
+ *	Return 0 if successful, otherwise < 0 error code.
  *
  * Security hooks for Netlink messaging.
  *
diff --git a/include/linux/mdio/mdio-i2c.h b/include/linux/mdio/mdio-i2c.h
index b1d27f7cd23f..65b550a6fc32 100644
--- a/include/linux/mdio/mdio-i2c.h
+++ b/include/linux/mdio/mdio-i2c.h
@@ -11,6 +11,14 @@ struct device;
 struct i2c_adapter;
 struct mii_bus;
 
-struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c);
+enum mdio_i2c_proto {
+	MDIO_I2C_NONE,
+	MDIO_I2C_MARVELL_C22,
+	MDIO_I2C_C45,
+	MDIO_I2C_ROLLBALL,
+};
+
+struct mii_bus *mdio_i2c_alloc(struct device *parent, struct i2c_adapter *i2c,
+			       enum mdio_i2c_proto protocol);
 
 #endif
diff --git a/include/linux/mei_aux.h b/include/linux/mei_aux.h
index 587f25128848..506912ad363b 100644
--- a/include/linux/mei_aux.h
+++ b/include/linux/mei_aux.h
@@ -7,10 +7,22 @@
 
 #include <linux/auxiliary_bus.h>
 
+/**
+ * struct mei_aux_device - mei auxiliary device
+ * @aux_dev: - auxiliary device object
+ * @irq: interrupt driving the mei auxiliary device
+ * @bar: mmio resource bar reserved to mei auxiliary device
+ * @ext_op_mem: resource for extend operational memory
+ *              used in graphics PXP mode.
+ * @slow_firmware: The device has slow underlying firmware.
+ *                 Such firmware will require to use larger operation timeouts.
+ */
 struct mei_aux_device {
 	struct auxiliary_device aux_dev;
 	int irq;
 	struct resource bar;
+	struct resource ext_op_mem;
+	bool slow_firmware;
 };
 
 #define auxiliary_dev_to_mei_aux_dev(auxiliary_dev) \
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 6257867fbf95..567f12323f55 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -1788,42 +1788,6 @@ static inline void count_objcg_event(struct obj_cgroup *objcg,
 	rcu_read_unlock();
 }
 
-/**
- * get_mem_cgroup_from_obj - get a memcg associated with passed kernel object.
- * @p: pointer to object from which memcg should be extracted. It can be NULL.
- *
- * Retrieves the memory group into which the memory of the pointed kernel
- * object is accounted. If memcg is found, its reference is taken.
- * If a passed kernel object is uncharged, or if proper memcg cannot be found,
- * as well as if mem_cgroup is disabled, NULL is returned.
- *
- * Return: valid memcg pointer with taken reference or NULL.
- */
-static inline struct mem_cgroup *get_mem_cgroup_from_obj(void *p)
-{
-	struct mem_cgroup *memcg;
-
-	rcu_read_lock();
-	do {
-		memcg = mem_cgroup_from_obj(p);
-	} while (memcg && !css_tryget(&memcg->css));
-	rcu_read_unlock();
-	return memcg;
-}
-
-/**
- * mem_cgroup_or_root - always returns a pointer to a valid memory cgroup.
- * @memcg: pointer to a valid memory cgroup or NULL.
- *
- * If passed argument is not NULL, returns it without any additional checks
- * and changes. Otherwise, root_mem_cgroup is returned.
- *
- * NOTE: root_mem_cgroup can be NULL during early boot.
- */
-static inline struct mem_cgroup *mem_cgroup_or_root(struct mem_cgroup *memcg)
-{
-	return memcg ? memcg : root_mem_cgroup;
-}
 #else
 static inline bool mem_cgroup_kmem_disabled(void)
 {
@@ -1880,15 +1844,6 @@ static inline void count_objcg_event(struct obj_cgroup *objcg,
 {
 }
 
-static inline struct mem_cgroup *get_mem_cgroup_from_obj(void *p)
-{
-	return NULL;
-}
-
-static inline struct mem_cgroup *mem_cgroup_or_root(struct mem_cgroup *memcg)
-{
-	return NULL;
-}
 #endif /* CONFIG_MEMCG_KMEM */
 
 #if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_ZSWAP)
diff --git a/include/linux/memremap.h b/include/linux/memremap.h
index 19010491a603..c3b4cc84877b 100644
--- a/include/linux/memremap.h
+++ b/include/linux/memremap.h
@@ -139,6 +139,11 @@ struct dev_pagemap {
 	};
 };
 
+static inline bool pgmap_has_memory_failure(struct dev_pagemap *pgmap)
+{
+	return pgmap->ops && pgmap->ops->memory_failure;
+}
+
 static inline struct vmem_altmap *pgmap_altmap(struct dev_pagemap *pgmap)
 {
 	if (pgmap->flags & PGMAP_ALTMAP_VALID)
diff --git a/include/linux/mfd/ocelot.h b/include/linux/mfd/ocelot.h
new file mode 100644
index 000000000000..dd72073d2d4f
--- /dev/null
+++ b/include/linux/mfd/ocelot.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright 2022 Innovative Advantage Inc. */
+
+#ifndef _LINUX_MFD_OCELOT_H
+#define _LINUX_MFD_OCELOT_H
+
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/ioport.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/types.h>
+
+struct resource;
+
+static inline struct regmap *
+ocelot_regmap_from_resource_optional(struct platform_device *pdev,
+				     unsigned int index,
+				     const struct regmap_config *config)
+{
+	struct device *dev = &pdev->dev;
+	struct resource *res;
+	void __iomem *regs;
+
+	/*
+	 * Don't use _get_and_ioremap_resource() here, since that will invoke
+	 * prints of "invalid resource" which will simply add confusion.
+	 */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, index);
+	if (res) {
+		regs = devm_ioremap_resource(dev, res);
+		if (IS_ERR(regs))
+			return ERR_CAST(regs);
+		return devm_regmap_init_mmio(dev, regs, config);
+	}
+
+	/*
+	 * Fall back to using REG and getting the resource from the parent
+	 * device, which is possible in an MFD configuration
+	 */
+	if (dev->parent) {
+		res = platform_get_resource(pdev, IORESOURCE_REG, index);
+		if (!res)
+			return NULL;
+
+		return dev_get_regmap(dev->parent, res->name);
+	}
+
+	return NULL;
+}
+
+static inline struct regmap *
+ocelot_regmap_from_resource(struct platform_device *pdev, unsigned int index,
+			    const struct regmap_config *config)
+{
+	struct regmap *map;
+
+	map = ocelot_regmap_from_resource_optional(pdev, index, config);
+	return map ?: ERR_PTR(-ENOENT);
+}
+
+#endif
diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h
index b5f58fd37a0f..1ff91cb79ded 100644
--- a/include/linux/mlx5/device.h
+++ b/include/linux/mlx5/device.h
@@ -325,6 +325,7 @@ enum mlx5_event {
 	MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10,
 	MLX5_EVENT_TYPE_WQ_ACCESS_ERROR	   = 0x11,
 	MLX5_EVENT_TYPE_SRQ_CATAS_ERROR	   = 0x12,
+	MLX5_EVENT_TYPE_OBJECT_CHANGE	   = 0x27,
 
 	MLX5_EVENT_TYPE_INTERNAL_ERROR	   = 0x08,
 	MLX5_EVENT_TYPE_PORT_CHANGE	   = 0x09,
@@ -699,6 +700,12 @@ struct mlx5_eqe_temp_warning {
 	__be64 sensor_warning_lsb;
 } __packed;
 
+struct mlx5_eqe_obj_change {
+	u8      rsvd0[2];
+	__be16  obj_type;
+	__be32  obj_id;
+} __packed;
+
 #define SYNC_RST_STATE_MASK    0xf
 
 enum sync_rst_state_type {
@@ -737,6 +744,7 @@ union ev_data {
 	struct mlx5_eqe_xrq_err		xrq_err;
 	struct mlx5_eqe_sync_fw_update	sync_fw_update;
 	struct mlx5_eqe_vhca_state	vhca_state;
+	struct mlx5_eqe_obj_change	obj_change;
 } __packed;
 
 struct mlx5_eqe {
@@ -874,12 +882,6 @@ static inline u8 get_cqe_opcode(struct mlx5_cqe64 *cqe)
 	return cqe->op_own >> 4;
 }
 
-static inline u8 get_cqe_enhanced_num_mini_cqes(struct mlx5_cqe64 *cqe)
-{
-	/* num_of_mini_cqes is zero based */
-	return get_cqe_opcode(cqe) + 1;
-}
-
 static inline u8 get_cqe_lro_tcppsh(struct mlx5_cqe64 *cqe)
 {
 	return (cqe->lro.tcppsh_abort_dupack >> 6) & 1;
@@ -890,11 +892,6 @@ static inline u8 get_cqe_l4_hdr_type(struct mlx5_cqe64 *cqe)
 	return (cqe->l4_l3_hdr_type >> 4) & 0x7;
 }
 
-static inline u8 get_cqe_l3_hdr_type(struct mlx5_cqe64 *cqe)
-{
-	return (cqe->l4_l3_hdr_type >> 2) & 0x3;
-}
-
 static inline bool cqe_is_tunneled(struct mlx5_cqe64 *cqe)
 {
 	return cqe->tls_outer_l3_tunneled & 0x1;
@@ -1198,8 +1195,10 @@ enum mlx5_cap_type {
 	MLX5_CAP_DEV_EVENT = 0x14,
 	MLX5_CAP_IPSEC,
 	MLX5_CAP_DEV_SHAMPO = 0x1d,
+	MLX5_CAP_MACSEC = 0x1f,
 	MLX5_CAP_GENERAL_2 = 0x20,
 	MLX5_CAP_PORT_SELECTION = 0x25,
+	MLX5_CAP_ADV_VIRTUALIZATION = 0x26,
 	/* NUM OF CAP Types */
 	MLX5_CAP_NUM
 };
@@ -1365,6 +1364,14 @@ enum mlx5_qcam_feature_groups {
 	MLX5_GET(port_selection_cap, \
 		 mdev->caps.hca[MLX5_CAP_PORT_SELECTION]->max, cap)
 
+#define MLX5_CAP_ADV_VIRTUALIZATION(mdev, cap) \
+	MLX5_GET(adv_virtualization_cap, \
+		 mdev->caps.hca[MLX5_CAP_ADV_VIRTUALIZATION]->cur, cap)
+
+#define MLX5_CAP_ADV_VIRTUALIZATION_MAX(mdev, cap) \
+	MLX5_GET(adv_virtualization_cap, \
+		 mdev->caps.hca[MLX5_CAP_ADV_VIRTUALIZATION]->max, cap)
+
 #define MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) \
 	MLX5_CAP_PORT_SELECTION(mdev, flow_table_properties_port_selection.cap)
 
@@ -1446,6 +1453,9 @@ enum mlx5_qcam_feature_groups {
 #define MLX5_CAP_DEV_SHAMPO(mdev, cap)\
 	MLX5_GET(shampo_cap, mdev->caps.hca_cur[MLX5_CAP_DEV_SHAMPO], cap)
 
+#define MLX5_CAP_MACSEC(mdev, cap)\
+	MLX5_GET(macsec_cap, (mdev)->caps.hca[MLX5_CAP_MACSEC]->cur, cap)
+
 enum {
 	MLX5_CMD_STAT_OK			= 0x0,
 	MLX5_CMD_STAT_INT_ERR			= 0x1,
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index c32de987fa71..a12929bc31b2 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -698,6 +698,8 @@ struct mlx5_pps {
 	struct work_struct         out_work;
 	u64                        start[MAX_PIN_NUM];
 	u8                         enabled;
+	u64                        min_npps_period;
+	u64                        min_out_pulse_duration_ns;
 };
 
 struct mlx5_timer {
@@ -855,11 +857,6 @@ struct mlx5_cmd_work_ent {
 	refcount_t              refcnt;
 };
 
-struct mlx5_pas {
-	u64	pa;
-	u8	log_sz;
-};
-
 enum phy_port_state {
 	MLX5_AAA_111
 };
@@ -1016,11 +1013,11 @@ int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
 bool mlx5_cmd_is_down(struct mlx5_core_dev *dev);
 
 int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type);
-void mlx5_health_flush(struct mlx5_core_dev *dev);
 void mlx5_health_cleanup(struct mlx5_core_dev *dev);
 int mlx5_health_init(struct mlx5_core_dev *dev);
 void mlx5_start_health_poll(struct mlx5_core_dev *dev);
 void mlx5_stop_health_poll(struct mlx5_core_dev *dev, bool disable_health);
+void mlx5_start_health_fw_log_up(struct mlx5_core_dev *dev);
 void mlx5_drain_health_wq(struct mlx5_core_dev *dev);
 void mlx5_trigger_health_work(struct mlx5_core_dev *dev);
 int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size,
@@ -1085,8 +1082,6 @@ int mlx5_core_destroy_psv(struct mlx5_core_dev *dev, int psv_num);
 void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common);
 int mlx5_query_odp_caps(struct mlx5_core_dev *dev,
 			struct mlx5_odp_caps *odp_caps);
-int mlx5_core_query_ib_ppcnt(struct mlx5_core_dev *dev,
-			     u8 port_num, void *out, size_t sz);
 
 int mlx5_init_rl_table(struct mlx5_core_dev *dev);
 void mlx5_cleanup_rl_table(struct mlx5_core_dev *dev);
@@ -1153,6 +1148,7 @@ int mlx5_cmd_destroy_vport_lag(struct mlx5_core_dev *dev);
 bool mlx5_lag_is_roce(struct mlx5_core_dev *dev);
 bool mlx5_lag_is_sriov(struct mlx5_core_dev *dev);
 bool mlx5_lag_is_active(struct mlx5_core_dev *dev);
+bool mlx5_lag_mode_is_hash(struct mlx5_core_dev *dev);
 bool mlx5_lag_is_master(struct mlx5_core_dev *dev);
 bool mlx5_lag_is_shared_fdb(struct mlx5_core_dev *dev);
 struct net_device *mlx5_lag_get_roce_netdev(struct mlx5_core_dev *dev);
@@ -1293,4 +1289,8 @@ static inline bool mlx5_get_roce_state(struct mlx5_core_dev *dev)
 	return mlx5_is_roce_on(dev);
 }
 
+enum {
+	MLX5_OCTWORD = 16,
+};
+
 #endif /* MLX5_DRIVER_H */
diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h
index 8e73c377da2c..c7a91981cd5a 100644
--- a/include/linux/mlx5/fs.h
+++ b/include/linux/mlx5/fs.h
@@ -79,6 +79,7 @@ static inline void build_leftovers_ft_param(int *priority,
 
 enum mlx5_flow_namespace_type {
 	MLX5_FLOW_NAMESPACE_BYPASS,
+	MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC,
 	MLX5_FLOW_NAMESPACE_LAG,
 	MLX5_FLOW_NAMESPACE_OFFLOADS,
 	MLX5_FLOW_NAMESPACE_ETHTOOL,
@@ -92,7 +93,8 @@ enum mlx5_flow_namespace_type {
 	MLX5_FLOW_NAMESPACE_SNIFFER_RX,
 	MLX5_FLOW_NAMESPACE_SNIFFER_TX,
 	MLX5_FLOW_NAMESPACE_EGRESS,
-	MLX5_FLOW_NAMESPACE_EGRESS_KERNEL,
+	MLX5_FLOW_NAMESPACE_EGRESS_IPSEC,
+	MLX5_FLOW_NAMESPACE_EGRESS_MACSEC,
 	MLX5_FLOW_NAMESPACE_RDMA_RX,
 	MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL,
 	MLX5_FLOW_NAMESPACE_RDMA_TX,
@@ -243,10 +245,10 @@ struct mlx5_flow_act {
 	u32 action;
 	struct mlx5_modify_hdr  *modify_hdr;
 	struct mlx5_pkt_reformat *pkt_reformat;
-	union {
-		u32 ipsec_obj_id;
-		uintptr_t esp_id;
-	};
+	struct mlx5_flow_act_crypto_params {
+		u8 type;
+		u32 obj_id;
+	} crypto;
 	u32 flags;
 	struct mlx5_fs_vlan vlan[MLX5_FS_VLAN_DEPTH];
 	struct ib_counters *counters;
diff --git a/include/linux/mlx5/fs_helpers.h b/include/linux/mlx5/fs_helpers.h
index 9db21cd0e92c..bc5125bc0561 100644
--- a/include/linux/mlx5/fs_helpers.h
+++ b/include/linux/mlx5/fs_helpers.h
@@ -38,46 +38,6 @@
 #define MLX5_FS_IPV4_VERSION 4
 #define MLX5_FS_IPV6_VERSION 6
 
-static inline bool mlx5_fs_is_ipsec_flow(const u32 *match_c)
-{
-	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
-					   misc_parameters);
-
-	return MLX5_GET(fte_match_set_misc, misc_params_c, outer_esp_spi);
-}
-
-static inline bool _mlx5_fs_is_outer_ipproto_flow(const u32 *match_c,
-						  const u32 *match_v, u8 match)
-{
-	const void *headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
-					     outer_headers);
-	const void *headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
-					     outer_headers);
-
-	return MLX5_GET(fte_match_set_lyr_2_4, headers_c, ip_protocol) == 0xff &&
-		MLX5_GET(fte_match_set_lyr_2_4, headers_v, ip_protocol) == match;
-}
-
-static inline bool mlx5_fs_is_outer_tcp_flow(const u32 *match_c,
-					     const u32 *match_v)
-{
-	return _mlx5_fs_is_outer_ipproto_flow(match_c, match_v, IPPROTO_TCP);
-}
-
-static inline bool mlx5_fs_is_outer_udp_flow(const u32 *match_c,
-					     const u32 *match_v)
-{
-	return _mlx5_fs_is_outer_ipproto_flow(match_c, match_v, IPPROTO_UDP);
-}
-
-static inline bool mlx5_fs_is_vxlan_flow(const u32 *match_c)
-{
-	void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
-					   misc_parameters);
-
-	return MLX5_GET(fte_match_set_misc, misc_params_c, vxlan_vni);
-}
-
 static inline bool _mlx5_fs_is_outer_ipv_flow(struct mlx5_core_dev *mdev,
 					      const u32 *match_c,
 					      const u32 *match_v, int version)
@@ -131,12 +91,4 @@ mlx5_fs_is_outer_ipv6_flow(struct mlx5_core_dev *mdev, const u32 *match_c,
 					  MLX5_FS_IPV6_VERSION);
 }
 
-static inline bool mlx5_fs_is_outer_ipsec_flow(const u32 *match_c)
-{
-	void *misc_params_c =
-			MLX5_ADDR_OF(fte_match_param, match_c, misc_parameters);
-
-	return MLX5_GET(fte_match_set_misc, misc_params_c, outer_esp_spi);
-}
-
 #endif
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index 4acd5610e96b..06574d430ff5 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -68,6 +68,7 @@ enum {
 	MLX5_SET_HCA_CAP_OP_MOD_ODP                   = 0x2,
 	MLX5_SET_HCA_CAP_OP_MOD_ATOMIC                = 0x3,
 	MLX5_SET_HCA_CAP_OP_MOD_ROCE                  = 0x4,
+	MLX5_SET_HCA_CAP_OP_MODE_PORT_SELECTION       = 0x25,
 };
 
 enum {
@@ -82,6 +83,7 @@ enum {
 	MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM = (1ULL << MLX5_OBJ_TYPE_SW_ICM),
 	MLX5_GENERAL_OBJ_TYPES_CAP_GENEVE_TLV_OPT = (1ULL << 11),
 	MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q = (1ULL << 13),
+	MLX5_GENERAL_OBJ_TYPES_CAP_MACSEC_OFFLOAD = (1ULL << 39),
 };
 
 enum {
@@ -89,6 +91,7 @@ enum {
 	MLX5_OBJ_TYPE_VIRTIO_NET_Q = 0x000d,
 	MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS = 0x001c,
 	MLX5_OBJ_TYPE_MATCH_DEFINER = 0x0018,
+	MLX5_OBJ_TYPE_PAGE_TRACK = 0x46,
 	MLX5_OBJ_TYPE_MKEY = 0xff01,
 	MLX5_OBJ_TYPE_QP = 0xff02,
 	MLX5_OBJ_TYPE_PSV = 0xff03,
@@ -449,7 +452,12 @@ struct mlx5_ifc_flow_table_prop_layout_bits {
 	u8         reserved_at_60[0x2];
 	u8         reformat_insert[0x1];
 	u8         reformat_remove[0x1];
-	u8         reserver_at_64[0x14];
+	u8         macsec_encrypt[0x1];
+	u8         macsec_decrypt[0x1];
+	u8         reserved_at_66[0x2];
+	u8         reformat_add_macsec[0x1];
+	u8         reformat_remove_macsec[0x1];
+	u8         reserved_at_6a[0xe];
 	u8         log_max_ft_num[0x8];
 
 	u8         reserved_at_80[0x10];
@@ -476,6 +484,22 @@ struct mlx5_ifc_odp_per_transport_service_cap_bits {
 	u8         reserved_at_6[0x1a];
 };
 
+struct mlx5_ifc_ipv4_layout_bits {
+	u8         reserved_at_0[0x60];
+
+	u8         ipv4[0x20];
+};
+
+struct mlx5_ifc_ipv6_layout_bits {
+	u8         ipv6[16][0x8];
+};
+
+union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits {
+	struct mlx5_ifc_ipv6_layout_bits ipv6_layout;
+	struct mlx5_ifc_ipv4_layout_bits ipv4_layout;
+	u8         reserved_at_0[0x80];
+};
+
 struct mlx5_ifc_fte_match_set_lyr_2_4_bits {
 	u8         smac_47_16[0x20];
 
@@ -611,7 +635,11 @@ struct mlx5_ifc_fte_match_set_misc2_bits {
 
 	u8         metadata_reg_a[0x20];
 
-	u8         reserved_at_1a0[0x60];
+	u8         reserved_at_1a0[0x8];
+
+	u8         macsec_syndrome[0x8];
+
+	u8         reserved_at_1b0[0x50];
 };
 
 struct mlx5_ifc_fte_match_set_misc3_bits {
@@ -813,7 +841,9 @@ struct mlx5_ifc_flow_table_nic_cap_bits {
 struct mlx5_ifc_port_selection_cap_bits {
 	u8         reserved_at_0[0x10];
 	u8         port_select_flow_table[0x1];
-	u8         reserved_at_11[0xf];
+	u8         reserved_at_11[0x1];
+	u8         port_select_flow_table_bypass[0x1];
+	u8         reserved_at_13[0xd];
 
 	u8         reserved_at_20[0x1e0];
 
@@ -1276,6 +1306,24 @@ struct mlx5_ifc_ipsec_cap_bits {
 	u8         reserved_at_30[0x7d0];
 };
 
+struct mlx5_ifc_macsec_cap_bits {
+	u8    macsec_epn[0x1];
+	u8    reserved_at_1[0x2];
+	u8    macsec_crypto_esp_aes_gcm_256_encrypt[0x1];
+	u8    macsec_crypto_esp_aes_gcm_128_encrypt[0x1];
+	u8    macsec_crypto_esp_aes_gcm_256_decrypt[0x1];
+	u8    macsec_crypto_esp_aes_gcm_128_decrypt[0x1];
+	u8    reserved_at_7[0x4];
+	u8    log_max_macsec_offload[0x5];
+	u8    reserved_at_10[0x10];
+
+	u8    min_log_macsec_full_replay_window[0x8];
+	u8    max_log_macsec_full_replay_window[0x8];
+	u8    reserved_at_30[0x10];
+
+	u8    reserved_at_40[0x7c0];
+};
+
 enum {
 	MLX5_WQ_TYPE_LINKED_LIST  = 0x0,
 	MLX5_WQ_TYPE_CYCLIC       = 0x1,
@@ -1443,7 +1491,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 
 	u8         reserved_at_120[0xa];
 	u8         log_max_ra_req_dc[0x6];
-	u8         reserved_at_130[0x9];
+	u8         reserved_at_130[0x2];
+	u8         eth_wqe_too_small[0x1];
+	u8         reserved_at_133[0x6];
 	u8         vnic_env_cq_overrun[0x1];
 	u8         log_max_ra_res_dc[0x6];
 
@@ -1733,7 +1783,9 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         max_geneve_tlv_options[0x8];
 	u8         reserved_at_568[0x3];
 	u8         max_geneve_tlv_option_data_len[0x5];
-	u8         reserved_at_570[0x10];
+	u8         reserved_at_570[0x9];
+	u8         adv_virtualization[0x1];
+	u8         reserved_at_57a[0x6];
 
 	u8	   reserved_at_580[0xb];
 	u8	   log_max_dci_stream_channels[0x5];
@@ -1828,7 +1880,13 @@ struct mlx5_ifc_cmd_hca_cap_2_bits {
 	u8	   max_reformat_remove_size[0x8];
 	u8	   max_reformat_remove_offset[0x8];
 
-	u8	   reserved_at_c0[0x160];
+	u8	   reserved_at_c0[0xe0];
+
+	u8	   reserved_at_1a0[0xb];
+	u8	   log_min_mkey_entity_size[0x5];
+	u8	   reserved_at_1b0[0x10];
+
+	u8	   reserved_at_1c0[0x60];
 
 	u8	   reserved_at_220[0x1];
 	u8	   sw_vhca_id_valid[0x1];
@@ -3295,6 +3353,7 @@ union mlx5_ifc_hca_cap_union_bits {
 	struct mlx5_ifc_device_mem_cap_bits device_mem_cap;
 	struct mlx5_ifc_virtio_emulation_cap_bits virtio_emulation_cap;
 	struct mlx5_ifc_shampo_cap_bits shampo_cap;
+	struct mlx5_ifc_macsec_cap_bits macsec_cap;
 	u8         reserved_at_0[0x8000];
 };
 
@@ -3310,8 +3369,8 @@ enum {
 	MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH = 0x100,
 	MLX5_FLOW_CONTEXT_ACTION_VLAN_POP_2  = 0x400,
 	MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2 = 0x800,
-	MLX5_FLOW_CONTEXT_ACTION_IPSEC_DECRYPT = 0x1000,
-	MLX5_FLOW_CONTEXT_ACTION_IPSEC_ENCRYPT = 0x2000,
+	MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT = 0x1000,
+	MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT = 0x2000,
 	MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO = 0x4000,
 };
 
@@ -3321,6 +3380,11 @@ enum {
 	MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT       = 0x2,
 };
 
+enum {
+	MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC   = 0x0,
+	MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC  = 0x1,
+};
+
 struct mlx5_ifc_vlan_bits {
 	u8         ethtype[0x10];
 	u8         prio[0x3];
@@ -3374,7 +3438,7 @@ struct mlx5_ifc_flow_context_bits {
 	u8         extended_destination[0x1];
 	u8         reserved_at_81[0x1];
 	u8         flow_source[0x2];
-	u8         reserved_at_84[0x4];
+	u8         encrypt_decrypt_type[0x4];
 	u8         destination_list_size[0x18];
 
 	u8         reserved_at_a0[0x8];
@@ -3386,7 +3450,7 @@ struct mlx5_ifc_flow_context_bits {
 
 	struct mlx5_ifc_vlan_bits push_vlan_2;
 
-	u8         ipsec_obj_id[0x20];
+	u8         encrypt_decrypt_obj_id[0x20];
 	u8         reserved_at_140[0xc0];
 
 	struct mlx5_ifc_fte_match_param_bits match_value;
@@ -3475,7 +3539,9 @@ struct mlx5_ifc_vnic_diagnostic_statistics_bits {
 
 	u8         cq_overrun[0x20];
 
-	u8         reserved_at_220[0xde0];
+	u8         eth_wqe_too_small[0x20];
+
+	u8         reserved_at_220[0xdc0];
 };
 
 struct mlx5_ifc_traffic_counter_bits {
@@ -6316,6 +6382,8 @@ enum mlx5_reformat_ctx_type {
 	MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x4,
 	MLX5_REFORMAT_TYPE_INSERT_HDR = 0xf,
 	MLX5_REFORMAT_TYPE_REMOVE_HDR = 0x10,
+	MLX5_REFORMAT_TYPE_ADD_MACSEC = 0x11,
+	MLX5_REFORMAT_TYPE_DEL_MACSEC = 0x12,
 };
 
 struct mlx5_ifc_alloc_packet_reformat_context_in_bits {
@@ -9789,7 +9857,9 @@ struct mlx5_ifc_pcam_reg_bits {
 struct mlx5_ifc_mcam_enhanced_features_bits {
 	u8         reserved_at_0[0x5d];
 	u8         mcia_32dwords[0x1];
-	u8         reserved_at_5e[0xc];
+	u8         out_pulse_duration_ns[0x1];
+	u8         npps_period[0x1];
+	u8         reserved_at_60[0xa];
 	u8         reset_state[0x1];
 	u8         ptpcyc2realtime_modify[0x1];
 	u8         reserved_at_6c[0x2];
@@ -10289,7 +10359,12 @@ struct mlx5_ifc_mtpps_reg_bits {
 	u8         reserved_at_18[0x4];
 	u8         cap_max_num_of_pps_out_pins[0x4];
 
-	u8         reserved_at_20[0x24];
+	u8         reserved_at_20[0x13];
+	u8         cap_log_min_npps_period[0x5];
+	u8         reserved_at_38[0x3];
+	u8         cap_log_min_out_pulse_duration_ns[0x5];
+
+	u8         reserved_at_40[0x4];
 	u8         cap_pin_3_mode[0x4];
 	u8         reserved_at_48[0x4];
 	u8         cap_pin_2_mode[0x4];
@@ -10308,7 +10383,9 @@ struct mlx5_ifc_mtpps_reg_bits {
 	u8         cap_pin_4_mode[0x4];
 
 	u8         field_select[0x20];
-	u8         reserved_at_a0[0x60];
+	u8         reserved_at_a0[0x20];
+
+	u8         npps_period[0x40];
 
 	u8         enable[0x1];
 	u8         reserved_at_101[0xb];
@@ -10317,7 +10394,8 @@ struct mlx5_ifc_mtpps_reg_bits {
 	u8         pin_mode[0x4];
 	u8         pin[0x8];
 
-	u8         reserved_at_120[0x20];
+	u8         reserved_at_120[0x2];
+	u8         out_pulse_duration_ns[0x1e];
 
 	u8         time_stamp[0x40];
 
@@ -10920,7 +10998,9 @@ struct mlx5_ifc_lagc_bits {
 	u8         reserved_at_18[0x5];
 	u8         lag_state[0x3];
 
-	u8         reserved_at_20[0x14];
+	u8         reserved_at_20[0xc];
+	u8         active_port[0x4];
+	u8         reserved_at_30[0x4];
 	u8         tx_remap_affinity_2[0x4];
 	u8         reserved_at_38[0x4];
 	u8         tx_remap_affinity_1[0x4];
@@ -11471,6 +11551,7 @@ enum {
 	MLX5_GENERAL_OBJECT_TYPES_IPSEC = 0x13,
 	MLX5_GENERAL_OBJECT_TYPES_SAMPLER = 0x20,
 	MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO = 0x24,
+	MLX5_GENERAL_OBJECT_TYPES_MACSEC = 0x27,
 };
 
 enum {
@@ -11521,6 +11602,96 @@ struct mlx5_ifc_modify_ipsec_obj_in_bits {
 	struct mlx5_ifc_ipsec_obj_bits ipsec_object;
 };
 
+enum {
+	MLX5_MACSEC_ASO_REPLAY_PROTECTION = 0x1,
+};
+
+enum {
+	MLX5_MACSEC_ASO_REPLAY_WIN_32BIT  = 0x0,
+	MLX5_MACSEC_ASO_REPLAY_WIN_64BIT  = 0x1,
+	MLX5_MACSEC_ASO_REPLAY_WIN_128BIT = 0x2,
+	MLX5_MACSEC_ASO_REPLAY_WIN_256BIT = 0x3,
+};
+
+#define MLX5_MACSEC_ASO_INC_SN  0x2
+#define MLX5_MACSEC_ASO_REG_C_4_5 0x2
+
+struct mlx5_ifc_macsec_aso_bits {
+	u8    valid[0x1];
+	u8    reserved_at_1[0x1];
+	u8    mode[0x2];
+	u8    window_size[0x2];
+	u8    soft_lifetime_arm[0x1];
+	u8    hard_lifetime_arm[0x1];
+	u8    remove_flow_enable[0x1];
+	u8    epn_event_arm[0x1];
+	u8    reserved_at_a[0x16];
+
+	u8    remove_flow_packet_count[0x20];
+
+	u8    remove_flow_soft_lifetime[0x20];
+
+	u8    reserved_at_60[0x80];
+
+	u8    mode_parameter[0x20];
+
+	u8    replay_protection_window[8][0x20];
+};
+
+struct mlx5_ifc_macsec_offload_obj_bits {
+	u8    modify_field_select[0x40];
+
+	u8    confidentiality_en[0x1];
+	u8    reserved_at_41[0x1];
+	u8    epn_en[0x1];
+	u8    epn_overlap[0x1];
+	u8    reserved_at_44[0x2];
+	u8    confidentiality_offset[0x2];
+	u8    reserved_at_48[0x4];
+	u8    aso_return_reg[0x4];
+	u8    reserved_at_50[0x10];
+
+	u8    epn_msb[0x20];
+
+	u8    reserved_at_80[0x8];
+	u8    dekn[0x18];
+
+	u8    reserved_at_a0[0x20];
+
+	u8    sci[0x40];
+
+	u8    reserved_at_100[0x8];
+	u8    macsec_aso_access_pd[0x18];
+
+	u8    reserved_at_120[0x60];
+
+	u8    salt[3][0x20];
+
+	u8    reserved_at_1e0[0x20];
+
+	struct mlx5_ifc_macsec_aso_bits macsec_aso;
+};
+
+struct mlx5_ifc_create_macsec_obj_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
+	struct mlx5_ifc_macsec_offload_obj_bits macsec_object;
+};
+
+struct mlx5_ifc_modify_macsec_obj_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
+	struct mlx5_ifc_macsec_offload_obj_bits macsec_object;
+};
+
+enum {
+	MLX5_MODIFY_MACSEC_BITMASK_EPN_OVERLAP = BIT(0),
+	MLX5_MODIFY_MACSEC_BITMASK_EPN_MSB = BIT(1),
+};
+
+struct mlx5_ifc_query_macsec_obj_out_bits {
+	struct mlx5_ifc_general_obj_out_cmd_hdr_bits general_obj_out_cmd_hdr;
+	struct mlx5_ifc_macsec_offload_obj_bits macsec_object;
+};
+
 struct mlx5_ifc_encryption_key_obj_bits {
 	u8         modify_field_select[0x40];
 
@@ -11638,6 +11809,7 @@ enum {
 enum {
 	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS = 0x1,
 	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_IPSEC = 0x2,
+	MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_MACSEC = 0x4,
 };
 
 struct mlx5_ifc_tls_static_params_bits {
@@ -11818,4 +11990,82 @@ struct mlx5_ifc_load_vhca_state_out_bits {
 	u8         reserved_at_40[0x40];
 };
 
+struct mlx5_ifc_adv_virtualization_cap_bits {
+	u8         reserved_at_0[0x3];
+	u8         pg_track_log_max_num[0x5];
+	u8         pg_track_max_num_range[0x8];
+	u8         pg_track_log_min_addr_space[0x8];
+	u8         pg_track_log_max_addr_space[0x8];
+
+	u8         reserved_at_20[0x3];
+	u8         pg_track_log_min_msg_size[0x5];
+	u8         reserved_at_28[0x3];
+	u8         pg_track_log_max_msg_size[0x5];
+	u8         reserved_at_30[0x3];
+	u8         pg_track_log_min_page_size[0x5];
+	u8         reserved_at_38[0x3];
+	u8         pg_track_log_max_page_size[0x5];
+
+	u8         reserved_at_40[0x7c0];
+};
+
+struct mlx5_ifc_page_track_report_entry_bits {
+	u8         dirty_address_high[0x20];
+
+	u8         dirty_address_low[0x20];
+};
+
+enum {
+	MLX5_PAGE_TRACK_STATE_TRACKING,
+	MLX5_PAGE_TRACK_STATE_REPORTING,
+	MLX5_PAGE_TRACK_STATE_ERROR,
+};
+
+struct mlx5_ifc_page_track_range_bits {
+	u8         start_address[0x40];
+
+	u8         length[0x40];
+};
+
+struct mlx5_ifc_page_track_bits {
+	u8         modify_field_select[0x40];
+
+	u8         reserved_at_40[0x10];
+	u8         vhca_id[0x10];
+
+	u8         reserved_at_60[0x20];
+
+	u8         state[0x4];
+	u8         track_type[0x4];
+	u8         log_addr_space_size[0x8];
+	u8         reserved_at_90[0x3];
+	u8         log_page_size[0x5];
+	u8         reserved_at_98[0x3];
+	u8         log_msg_size[0x5];
+
+	u8         reserved_at_a0[0x8];
+	u8         reporting_qpn[0x18];
+
+	u8         reserved_at_c0[0x18];
+	u8         num_ranges[0x8];
+
+	u8         reserved_at_e0[0x20];
+
+	u8         range_start_address[0x40];
+
+	u8         length[0x40];
+
+	struct     mlx5_ifc_page_track_range_bits track_range[0];
+};
+
+struct mlx5_ifc_create_page_track_obj_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
+	struct mlx5_ifc_page_track_bits obj_context;
+};
+
+struct mlx5_ifc_modify_page_track_obj_in_bits {
+	struct mlx5_ifc_general_obj_in_cmd_hdr_bits general_obj_in_cmd_hdr;
+	struct mlx5_ifc_page_track_bits obj_context;
+};
+
 #endif /* MLX5_IFC_H */
diff --git a/include/linux/mlx5/mlx5_ifc_fpga.h b/include/linux/mlx5/mlx5_ifc_fpga.h
index 45c7c0d67635..0596472923ad 100644
--- a/include/linux/mlx5/mlx5_ifc_fpga.h
+++ b/include/linux/mlx5/mlx5_ifc_fpga.h
@@ -32,30 +32,6 @@
 #ifndef MLX5_IFC_FPGA_H
 #define MLX5_IFC_FPGA_H
 
-struct mlx5_ifc_ipv4_layout_bits {
-	u8         reserved_at_0[0x60];
-
-	u8         ipv4[0x20];
-};
-
-struct mlx5_ifc_ipv6_layout_bits {
-	u8         ipv6[16][0x8];
-};
-
-union mlx5_ifc_ipv6_layout_ipv4_layout_auto_bits {
-	struct mlx5_ifc_ipv6_layout_bits ipv6_layout;
-	struct mlx5_ifc_ipv4_layout_bits ipv4_layout;
-	u8         reserved_at_0[0x80];
-};
-
-enum {
-	MLX5_FPGA_CAP_SANDBOX_VENDOR_ID_MLNX = 0x2c9,
-};
-
-enum {
-	MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_IPSEC    = 0x2,
-};
-
 struct mlx5_ifc_fpga_shell_caps_bits {
 	u8         max_num_qps[0x10];
 	u8         reserved_at_10[0x8];
diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h
index 8bda3ba5b109..4657d5c54abe 100644
--- a/include/linux/mlx5/qp.h
+++ b/include/linux/mlx5/qp.h
@@ -162,6 +162,8 @@ enum {
 	MLX5_SEND_WQE_MAX_WQEBBS	= 16,
 };
 
+#define MLX5_SEND_WQE_MAX_SIZE (MLX5_SEND_WQE_MAX_WQEBBS * MLX5_SEND_WQE_BB)
+
 enum {
 	MLX5_WQE_FMR_PERM_LOCAL_READ	= 1 << 27,
 	MLX5_WQE_FMR_PERM_LOCAL_WRITE	= 1 << 28,
@@ -252,6 +254,7 @@ enum {
 
 enum {
 	MLX5_ETH_WQE_FT_META_IPSEC = BIT(0),
+	MLX5_ETH_WQE_FT_META_MACSEC = BIT(1),
 };
 
 struct mlx5_wqe_eth_seg {
@@ -475,6 +478,12 @@ struct mlx5_klm {
 	__be64		va;
 };
 
+struct mlx5_ksm {
+	__be32		reserved;
+	__be32		key;
+	__be64		va;
+};
+
 struct mlx5_stride_block_entry {
 	__be16		stride;
 	__be16		bcount;
diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h
index eb8bc5b9b0b7..8fdd3cf971a3 100644
--- a/include/linux/mmc/host.h
+++ b/include/linux/mmc/host.h
@@ -476,7 +476,7 @@ struct mmc_host {
 
 	unsigned int		sdio_irqs;
 	struct task_struct	*sdio_irq_thread;
-	struct delayed_work	sdio_irq_work;
+	struct work_struct	sdio_irq_work;
 	bool			sdio_irq_pending;
 	atomic_t		sdio_irq_thread_abort;
 
diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h
index 53f0efa0bccf..74f9d9a6d330 100644
--- a/include/linux/mmc/sdio_ids.h
+++ b/include/linux/mmc/sdio_ids.h
@@ -74,6 +74,7 @@
 #define SDIO_DEVICE_ID_BROADCOM_43362		0xa962
 #define SDIO_DEVICE_ID_BROADCOM_43364		0xa9a4
 #define SDIO_DEVICE_ID_BROADCOM_43430		0xa9a6
+#define SDIO_DEVICE_ID_BROADCOM_CYPRESS_43439	0xa9af
 #define SDIO_DEVICE_ID_BROADCOM_43455		0xa9bf
 #define SDIO_DEVICE_ID_BROADCOM_CYPRESS_43752	0xaae8
 
diff --git a/include/linux/module.h b/include/linux/module.h
index 518296ea7f73..ec61fb53979a 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -27,7 +27,6 @@
 #include <linux/tracepoint-defs.h>
 #include <linux/srcu.h>
 #include <linux/static_call_types.h>
-#include <linux/cfi.h>
 
 #include <linux/percpu.h>
 #include <asm/module.h>
@@ -132,7 +131,7 @@ extern void cleanup_module(void);
 	{ return initfn; }					\
 	int init_module(void) __copy(initfn)			\
 		__attribute__((alias(#initfn)));		\
-	__CFI_ADDRESSABLE(init_module, __initdata);
+	___ADDRESSABLE(init_module, __initdata);
 
 /* This is only required if you want to be unloadable. */
 #define module_exit(exitfn)					\
@@ -140,7 +139,7 @@ extern void cleanup_module(void);
 	{ return exitfn; }					\
 	void cleanup_module(void) __copy(exitfn)		\
 		__attribute__((alias(#exitfn)));		\
-	__CFI_ADDRESSABLE(cleanup_module, __exitdata);
+	___ADDRESSABLE(cleanup_module, __exitdata);
 
 #endif
 
@@ -387,8 +386,9 @@ struct module {
 	const s32 *crcs;
 	unsigned int num_syms;
 
-#ifdef CONFIG_CFI_CLANG
-	cfi_check_fn cfi_check;
+#ifdef CONFIG_ARCH_USES_CFI_TRAPS
+	s32 *kcfi_traps;
+	s32 *kcfi_traps_end;
 #endif
 
 	/* Kernel parameters. */
diff --git a/include/linux/mroute.h b/include/linux/mroute.h
index 6cbbfe94348c..80b8400ab8b2 100644
--- a/include/linux/mroute.h
+++ b/include/linux/mroute.h
@@ -17,7 +17,7 @@ static inline int ip_mroute_opt(int opt)
 }
 
 int ip_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int);
-int ip_mroute_getsockopt(struct sock *, int, char __user *, int __user *);
+int ip_mroute_getsockopt(struct sock *, int, sockptr_t, sockptr_t);
 int ipmr_ioctl(struct sock *sk, int cmd, void __user *arg);
 int ipmr_compat_ioctl(struct sock *sk, unsigned int cmd, void __user *arg);
 int ip_mr_init(void);
@@ -29,8 +29,8 @@ static inline int ip_mroute_setsockopt(struct sock *sock, int optname,
 	return -ENOPROTOOPT;
 }
 
-static inline int ip_mroute_getsockopt(struct sock *sock, int optname,
-				       char __user *optval, int __user *optlen)
+static inline int ip_mroute_getsockopt(struct sock *sk, int optname,
+				       sockptr_t optval, sockptr_t optlen)
 {
 	return -ENOPROTOOPT;
 }
diff --git a/include/linux/mroute6.h b/include/linux/mroute6.h
index bc351a85ce9b..8f2b307fb124 100644
--- a/include/linux/mroute6.h
+++ b/include/linux/mroute6.h
@@ -27,7 +27,7 @@ struct sock;
 
 #ifdef CONFIG_IPV6_MROUTE
 extern int ip6_mroute_setsockopt(struct sock *, int, sockptr_t, unsigned int);
-extern int ip6_mroute_getsockopt(struct sock *, int, char __user *, int __user *);
+extern int ip6_mroute_getsockopt(struct sock *, int, sockptr_t, sockptr_t);
 extern int ip6_mr_input(struct sk_buff *skb);
 extern int ip6mr_ioctl(struct sock *sk, int cmd, void __user *arg);
 extern int ip6mr_compat_ioctl(struct sock *sk, unsigned int cmd, void __user *arg);
@@ -42,7 +42,7 @@ static inline int ip6_mroute_setsockopt(struct sock *sock, int optname,
 
 static inline
 int ip6_mroute_getsockopt(struct sock *sock,
-			  int optname, char __user *optval, int __user *optlen)
+			  int optname, sockptr_t optval, sockptr_t optlen)
 {
 	return -ENOPROTOOPT;
 }
diff --git a/include/linux/namei.h b/include/linux/namei.h
index caeb08a98536..00fee52df842 100644
--- a/include/linux/namei.h
+++ b/include/linux/namei.h
@@ -83,7 +83,7 @@ extern int follow_up(struct path *);
 extern struct dentry *lock_rename(struct dentry *, struct dentry *);
 extern void unlock_rename(struct dentry *, struct dentry *);
 
-extern int __must_check nd_jump_link(struct path *path);
+extern int __must_check nd_jump_link(const struct path *path);
 
 static inline void nd_terminate_link(void *name, size_t len, size_t maxlen)
 {
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 05d6f3facd5a..eddf8ee270e7 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -253,11 +253,17 @@ struct netdev_hw_addr_list {
 #define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
 #define netdev_for_each_uc_addr(ha, dev) \
 	netdev_hw_addr_list_for_each(ha, &(dev)->uc)
+#define netdev_for_each_synced_uc_addr(_ha, _dev) \
+	netdev_for_each_uc_addr((_ha), (_dev)) \
+		if ((_ha)->sync_cnt)
 
 #define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
 #define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
 #define netdev_for_each_mc_addr(ha, dev) \
 	netdev_hw_addr_list_for_each(ha, &(dev)->mc)
+#define netdev_for_each_synced_mc_addr(_ha, _dev) \
+	netdev_for_each_mc_addr((_ha), (_dev)) \
+		if ((_ha)->sync_cnt)
 
 struct hh_cache {
 	unsigned int	hh_len;
@@ -546,8 +552,8 @@ static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n)
 {
 	unsigned long val, new;
 
+	val = READ_ONCE(n->state);
 	do {
-		val = READ_ONCE(n->state);
 		if (val & NAPIF_STATE_DISABLE)
 			return true;
 
@@ -555,7 +561,7 @@ static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n)
 			return false;
 
 		new = val | NAPIF_STATE_MISSED;
-	} while (cmpxchg(&n->state, val, new) != val);
+	} while (!try_cmpxchg(&n->state, &val, new));
 
 	return true;
 }
@@ -934,6 +940,7 @@ struct net_device_path_ctx {
 };
 
 enum tc_setup_type {
+	TC_QUERY_CAPS,
 	TC_SETUP_QDISC_MQPRIO,
 	TC_SETUP_CLSU32,
 	TC_SETUP_CLSFLOWER,
@@ -1851,7 +1858,6 @@ enum netdev_ml_priv_type {
  *	@tipc_ptr:	TIPC specific data
  *	@atalk_ptr:	AppleTalk link
  *	@ip_ptr:	IPv4 specific data
- *	@dn_ptr:	DECnet specific data
  *	@ip6_ptr:	IPv6 specific data
  *	@ax25_ptr:	AX.25 specific data
  *	@ieee80211_ptr:	IEEE 802.11 specific data, assign before registering
@@ -2147,9 +2153,6 @@ struct net_device {
 #if IS_ENABLED(CONFIG_ATALK)
 	void 			*atalk_ptr;
 #endif
-#if IS_ENABLED(CONFIG_DECNET)
-	struct dn_dev __rcu     *dn_ptr;
-#endif
 #if IS_ENABLED(CONFIG_AX25)
 	void			*ax25_ptr;
 #endif
@@ -2551,16 +2554,15 @@ void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
  * @dev:  network device
  * @napi: NAPI context
  * @poll: polling function
- * @weight: default weight
  *
  * netif_napi_add() must be used to initialize a NAPI context prior to calling
  * *any* of the other NAPI-related functions.
  */
 static inline void
 netif_napi_add(struct net_device *dev, struct napi_struct *napi,
-	       int (*poll)(struct napi_struct *, int), int weight)
+	       int (*poll)(struct napi_struct *, int))
 {
-	netif_napi_add_weight(dev, napi, poll, weight);
+	netif_napi_add_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
 }
 
 static inline void
@@ -2573,8 +2575,6 @@ netif_napi_add_tx_weight(struct net_device *dev,
 	netif_napi_add_weight(dev, napi, poll, weight);
 }
 
-#define netif_tx_napi_add netif_napi_add_tx_weight
-
 /**
  * netif_napi_add_tx() - initialize a NAPI context to be used for Tx only
  * @dev:  network device
@@ -3357,6 +3357,16 @@ static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_qu
 #endif
 }
 
+/**
+ *	netdev_tx_sent_queue - report the number of bytes queued to a given tx queue
+ *	@dev_queue: network device queue
+ *	@bytes: number of bytes queued to the device queue
+ *
+ *	Report the number of bytes queued for sending/completion to the network
+ *	device hardware queue. @bytes should be a good approximation and should
+ *	exactly match netdev_completed_queue() @bytes.
+ *	This is typically called once per packet, from ndo_start_xmit().
+ */
 static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
 					unsigned int bytes)
 {
@@ -3402,13 +3412,14 @@ static inline bool __netdev_tx_sent_queue(struct netdev_queue *dev_queue,
 }
 
 /**
- * 	netdev_sent_queue - report the number of bytes queued to hardware
- * 	@dev: network device
- * 	@bytes: number of bytes queued to the hardware device queue
+ *	netdev_sent_queue - report the number of bytes queued to hardware
+ *	@dev: network device
+ *	@bytes: number of bytes queued to the hardware device queue
  *
- * 	Report the number of bytes queued for sending/completion to the network
- * 	device hardware queue. @bytes should be a good approximation and should
- * 	exactly match netdev_completed_queue() @bytes
+ *	Report the number of bytes queued for sending/completion to the network
+ *	device hardware queue#0. @bytes should be a good approximation and should
+ *	exactly match netdev_completed_queue() @bytes.
+ *	This is typically called once per packet, from ndo_start_xmit().
  */
 static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
 {
@@ -3423,6 +3434,15 @@ static inline bool __netdev_sent_queue(struct net_device *dev,
 				      xmit_more);
 }
 
+/**
+ *	netdev_tx_completed_queue - report number of packets/bytes at TX completion.
+ *	@dev_queue: network device queue
+ *	@pkts: number of packets (currently ignored)
+ *	@bytes: number of bytes dequeued from the device queue
+ *
+ *	Must be called at most once per TX completion round (and not per
+ *	individual packet), so that BQL can adjust its limits appropriately.
+ */
 static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
 					     unsigned int pkts, unsigned int bytes)
 {
@@ -3802,6 +3822,7 @@ void netif_receive_skb_list(struct list_head *head);
 gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb);
 void napi_gro_flush(struct napi_struct *napi, bool flush_old);
 struct sk_buff *napi_get_frags(struct napi_struct *napi);
+void napi_get_frags_check(struct napi_struct *napi);
 gro_result_t napi_gro_frags(struct napi_struct *napi);
 struct packet_offload *gro_find_receive_by_type(__be16 type);
 struct packet_offload *gro_find_complete_by_type(__be16 type);
diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h
index c2c6f332fb90..d8817d381c14 100644
--- a/include/linux/netfilter.h
+++ b/include/linux/netfilter.h
@@ -243,11 +243,6 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
 		hook_head = rcu_dereference(net->nf.hooks_bridge[hook]);
 #endif
 		break;
-#if IS_ENABLED(CONFIG_DECNET)
-	case NFPROTO_DECNET:
-		hook_head = rcu_dereference(net->nf.hooks_decnet[hook]);
-		break;
-#endif
 	default:
 		WARN_ON_ONCE(1);
 		break;
diff --git a/include/linux/netfilter_defs.h b/include/linux/netfilter_defs.h
index 8dddfb151f00..a5f7bef1b3a4 100644
--- a/include/linux/netfilter_defs.h
+++ b/include/linux/netfilter_defs.h
@@ -7,14 +7,6 @@
 /* in/out/forward only */
 #define NF_ARP_NUMHOOKS 3
 
-/* max hook is NF_DN_ROUTE (6), also see uapi/linux/netfilter_decnet.h */
-#define NF_DN_NUMHOOKS 7
-
-#if IS_ENABLED(CONFIG_DECNET)
-/* Largest hook number + 1, see uapi/linux/netfilter_decnet.h */
-#define NF_MAX_HOOKS	NF_DN_NUMHOOKS
-#else
 #define NF_MAX_HOOKS	NF_INET_NUMHOOKS
-#endif
 
 #endif
diff --git a/include/linux/netlink.h b/include/linux/netlink.h
index bda1c385cffb..d51e041d2242 100644
--- a/include/linux/netlink.h
+++ b/include/linux/netlink.h
@@ -71,6 +71,8 @@ netlink_kernel_create(struct net *net, int unit, struct netlink_kernel_cfg *cfg)
  *	%NL_SET_ERR_MSG
  * @bad_attr: attribute with error
  * @policy: policy for a bad attribute
+ * @miss_type: attribute type which was missing
+ * @miss_nest: nest missing an attribute (%NULL if missing top level attr)
  * @cookie: cookie data to return to userspace (for success)
  * @cookie_len: actual cookie data length
  */
@@ -78,6 +80,8 @@ struct netlink_ext_ack {
 	const char *_msg;
 	const struct nlattr *bad_attr;
 	const struct nla_policy *policy;
+	const struct nlattr *miss_nest;
+	u16 miss_type;
 	u8 cookie[NETLINK_MAX_COOKIE_LEN];
 	u8 cookie_len;
 };
@@ -126,6 +130,26 @@ struct netlink_ext_ack {
 #define NL_SET_ERR_MSG_ATTR(extack, attr, msg)		\
 	NL_SET_ERR_MSG_ATTR_POL(extack, attr, NULL, msg)
 
+#define NL_SET_ERR_ATTR_MISS(extack, nest, type)  do {	\
+	struct netlink_ext_ack *__extack = (extack);	\
+							\
+	if (__extack) {					\
+		__extack->miss_nest = (nest);		\
+		__extack->miss_type = (type);		\
+	}						\
+} while (0)
+
+#define NL_REQ_ATTR_CHECK(extack, nest, tb, type) ({		\
+	struct nlattr **__tb = (tb);				\
+	u32 __attr = (type);					\
+	int __retval;						\
+								\
+	__retval = !__tb[__attr];				\
+	if (__retval)						\
+		NL_SET_ERR_ATTR_MISS((extack), (nest), __attr);	\
+	__retval;						\
+})
+
 static inline void nl_set_extack_cookie_u64(struct netlink_ext_ack *extack,
 					    u64 cookie)
 {
diff --git a/include/linux/once.h b/include/linux/once.h
index b14d8b309d52..bc714d414448 100644
--- a/include/linux/once.h
+++ b/include/linux/once.h
@@ -5,10 +5,18 @@
 #include <linux/types.h>
 #include <linux/jump_label.h>
 
+/* Helpers used from arbitrary contexts.
+ * Hard irqs are blocked, be cautious.
+ */
 bool __do_once_start(bool *done, unsigned long *flags);
 void __do_once_done(bool *done, struct static_key_true *once_key,
 		    unsigned long *flags, struct module *mod);
 
+/* Variant for process contexts only. */
+bool __do_once_sleepable_start(bool *done);
+void __do_once_sleepable_done(bool *done, struct static_key_true *once_key,
+			      struct module *mod);
+
 /* Call a function exactly once. The idea of DO_ONCE() is to perform
  * a function call such as initialization of random seeds, etc, only
  * once, where DO_ONCE() can live in the fast-path. After @func has
@@ -52,7 +60,27 @@ void __do_once_done(bool *done, struct static_key_true *once_key,
 		___ret;							     \
 	})
 
+/* Variant of DO_ONCE() for process/sleepable contexts. */
+#define DO_ONCE_SLEEPABLE(func, ...)						\
+	({									\
+		bool ___ret = false;						\
+		static bool __section(".data.once") ___done = false;		\
+		static DEFINE_STATIC_KEY_TRUE(___once_key);			\
+		if (static_branch_unlikely(&___once_key)) {			\
+			___ret = __do_once_sleepable_start(&___done);		\
+			if (unlikely(___ret)) {					\
+				func(__VA_ARGS__);				\
+				__do_once_sleepable_done(&___done, &___once_key,\
+						    THIS_MODULE);		\
+			}							\
+		}								\
+		___ret;								\
+	})
+
 #define get_random_once(buf, nbytes)					     \
 	DO_ONCE(get_random_bytes, (buf), (nbytes))
 
+#define get_random_sleepable_once(buf, nbytes)				     \
+	DO_ONCE_SLEEPABLE(get_random_bytes, (buf), (nbytes))
+
 #endif /* _LINUX_ONCE_H */
diff --git a/include/linux/overflow.h b/include/linux/overflow.h
index 0eb3b192f07a..19dfdd74835e 100644
--- a/include/linux/overflow.h
+++ b/include/linux/overflow.h
@@ -51,40 +51,50 @@ static inline bool __must_check __must_check_overflow(bool overflow)
 	return unlikely(overflow);
 }
 
-/*
- * For simplicity and code hygiene, the fallback code below insists on
- * a, b and *d having the same type (similar to the min() and max()
- * macros), whereas gcc's type-generic overflow checkers accept
- * different types. Hence we don't just make check_add_overflow an
- * alias for __builtin_add_overflow, but add type checks similar to
- * below.
+/** check_add_overflow() - Calculate addition with overflow checking
+ *
+ * @a: first addend
+ * @b: second addend
+ * @d: pointer to store sum
+ *
+ * Returns 0 on success.
+ *
+ * *@d holds the results of the attempted addition, but is not considered
+ * "safe for use" on a non-zero return value, which indicates that the
+ * sum has overflowed or been truncated.
  */
-#define check_add_overflow(a, b, d) __must_check_overflow(({	\
-	typeof(a) __a = (a);			\
-	typeof(b) __b = (b);			\
-	typeof(d) __d = (d);			\
-	(void) (&__a == &__b);			\
-	(void) (&__a == __d);			\
-	__builtin_add_overflow(__a, __b, __d);	\
-}))
+#define check_add_overflow(a, b, d)	\
+	__must_check_overflow(__builtin_add_overflow(a, b, d))
 
-#define check_sub_overflow(a, b, d) __must_check_overflow(({	\
-	typeof(a) __a = (a);			\
-	typeof(b) __b = (b);			\
-	typeof(d) __d = (d);			\
-	(void) (&__a == &__b);			\
-	(void) (&__a == __d);			\
-	__builtin_sub_overflow(__a, __b, __d);	\
-}))
+/** check_sub_overflow() - Calculate subtraction with overflow checking
+ *
+ * @a: minuend; value to subtract from
+ * @b: subtrahend; value to subtract from @a
+ * @d: pointer to store difference
+ *
+ * Returns 0 on success.
+ *
+ * *@d holds the results of the attempted subtraction, but is not considered
+ * "safe for use" on a non-zero return value, which indicates that the
+ * difference has underflowed or been truncated.
+ */
+#define check_sub_overflow(a, b, d)	\
+	__must_check_overflow(__builtin_sub_overflow(a, b, d))
 
-#define check_mul_overflow(a, b, d) __must_check_overflow(({	\
-	typeof(a) __a = (a);			\
-	typeof(b) __b = (b);			\
-	typeof(d) __d = (d);			\
-	(void) (&__a == &__b);			\
-	(void) (&__a == __d);			\
-	__builtin_mul_overflow(__a, __b, __d);	\
-}))
+/** check_mul_overflow() - Calculate multiplication with overflow checking
+ *
+ * @a: first factor
+ * @b: second factor
+ * @d: pointer to store product
+ *
+ * Returns 0 on success.
+ *
+ * *@d holds the results of the attempted multiplication, but is not
+ * considered "safe for use" on a non-zero return value, which indicates
+ * that the product has overflowed or been truncated.
+ */
+#define check_mul_overflow(a, b, d)	\
+	__must_check_overflow(__builtin_mul_overflow(a, b, d))
 
 /** check_shl_overflow() - Calculate a left-shifted value and check overflow
  *
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 060af91bafcd..5da0846aa3c1 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -2019,8 +2019,8 @@ enum pci_fixup_pass {
 #ifdef CONFIG_LTO_CLANG
 #define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class,	\
 				  class_shift, hook, stub)		\
-	void __cficanonical stub(struct pci_dev *dev);			\
-	void __cficanonical stub(struct pci_dev *dev)			\
+	void stub(struct pci_dev *dev);					\
+	void stub(struct pci_dev *dev)					\
 	{ 								\
 		hook(dev); 						\
 	}								\
diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h
index 15b49e655ce3..b362d90eb9b0 100644
--- a/include/linux/pci_ids.h
+++ b/include/linux/pci_ids.h
@@ -75,6 +75,9 @@
 #define PCI_CLASS_COMMUNICATION_MODEM	0x0703
 #define PCI_CLASS_COMMUNICATION_OTHER	0x0780
 
+/* Interface for SERIAL/MODEM */
+#define PCI_SERIAL_16550_COMPATIBLE	0x02
+
 #define PCI_BASE_CLASS_SYSTEM		0x08
 #define PCI_CLASS_SYSTEM_PIC		0x0800
 #define PCI_CLASS_SYSTEM_PIC_IOAPIC	0x080010
diff --git a/include/linux/pcs-altera-tse.h b/include/linux/pcs-altera-tse.h
new file mode 100644
index 000000000000..92ab9f08e835
--- /dev/null
+++ b/include/linux/pcs-altera-tse.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2022 Bootlin
+ *
+ * Maxime Chevallier <maxime.chevallier@bootlin.com>
+ */
+
+#ifndef __LINUX_PCS_ALTERA_TSE_H
+#define __LINUX_PCS_ALTERA_TSE_H
+
+struct phylink_pcs;
+struct net_device;
+
+struct phylink_pcs *alt_tse_pcs_create(struct net_device *ndev,
+				       void __iomem *pcs_base, int reg_width);
+
+#endif /* __LINUX_PCS_ALTERA_TSE_H */
diff --git a/include/linux/phy.h b/include/linux/phy.h
index 87638c55d844..ddf66198f751 100644
--- a/include/linux/phy.h
+++ b/include/linux/phy.h
@@ -115,6 +115,8 @@ extern const int phy_10gbit_features_array[1];
  * @PHY_INTERFACE_MODE_25GBASER: 25G BaseR
  * @PHY_INTERFACE_MODE_USXGMII:  Universal Serial 10GE MII
  * @PHY_INTERFACE_MODE_10GKR: 10GBASE-KR - with Clause 73 AN
+ * @PHY_INTERFACE_MODE_QUSGMII: Quad Universal SGMII
+ * @PHY_INTERFACE_MODE_1000BASEKX: 1000Base-KX - with Clause 73 AN
  * @PHY_INTERFACE_MODE_MAX: Book keeping
  *
  * Describes the interface between the MAC and PHY.
@@ -152,6 +154,8 @@ typedef enum {
 	PHY_INTERFACE_MODE_USXGMII,
 	/* 10GBASE-KR - with Clause 73 AN */
 	PHY_INTERFACE_MODE_10GKR,
+	PHY_INTERFACE_MODE_QUSGMII,
+	PHY_INTERFACE_MODE_1000BASEKX,
 	PHY_INTERFACE_MODE_MAX,
 } phy_interface_t;
 
@@ -249,6 +253,8 @@ static inline const char *phy_modes(phy_interface_t interface)
 		return "trgmii";
 	case PHY_INTERFACE_MODE_1000BASEX:
 		return "1000base-x";
+	case PHY_INTERFACE_MODE_1000BASEKX:
+		return "1000base-kx";
 	case PHY_INTERFACE_MODE_2500BASEX:
 		return "2500base-x";
 	case PHY_INTERFACE_MODE_5GBASER:
@@ -267,12 +273,13 @@ static inline const char *phy_modes(phy_interface_t interface)
 		return "10gbase-kr";
 	case PHY_INTERFACE_MODE_100BASEX:
 		return "100base-x";
+	case PHY_INTERFACE_MODE_QUSGMII:
+		return "qusgmii";
 	default:
 		return "unknown";
 	}
 }
 
-
 #define PHY_INIT_TIMEOUT	100000
 #define PHY_FORCE_TIMEOUT	10
 
@@ -564,8 +571,10 @@ struct macsec_ops;
  * @advertising: Currently advertised linkmodes
  * @adv_old: Saved advertised while power saving for WoL
  * @lp_advertising: Current link partner advertised linkmodes
+ * @host_interfaces: PHY interface modes supported by host
  * @eee_broken_modes: Energy efficient ethernet modes which should be prohibited
  * @autoneg: Flag autoneg being used
+ * @rate_matching: Current rate matching mode
  * @link: Current link state
  * @autoneg_complete: Flag auto negotiation of the link has completed
  * @mdix: Current crossover
@@ -588,6 +597,7 @@ struct macsec_ops;
  * @master_slave_get: Current master/slave advertisement
  * @master_slave_state: Current master/slave configuration
  * @mii_ts: Pointer to time stamper callbacks
+ * @psec: Pointer to Power Sourcing Equipment control struct
  * @lock:  Mutex for serialization access to PHY
  * @state_queue: Work queue for state machine
  * @shared: Pointer to private data shared by phys in one package
@@ -633,6 +643,8 @@ struct phy_device {
 	unsigned irq_suspended:1;
 	unsigned irq_rerun:1;
 
+	int rate_matching;
+
 	enum phy_state state;
 
 	u32 dev_flags;
@@ -660,6 +672,9 @@ struct phy_device {
 	/* used with phy_speed_down */
 	__ETHTOOL_DECLARE_LINK_MODE_MASK(adv_old);
 
+	/* Host supported PHY interface types. Should be ignored if empty. */
+	DECLARE_PHY_INTERFACE_MASK(host_interfaces);
+
 	/* Energy efficient ethernet modes which should be prohibited */
 	u32 eee_broken_modes;
 
@@ -701,6 +716,7 @@ struct phy_device {
 	struct phylink *phylink;
 	struct net_device *attached_dev;
 	struct mii_timestamper *mii_ts;
+	struct pse_control *psec;
 
 	u8 mdix;
 	u8 mdix_ctrl;
@@ -797,6 +813,21 @@ struct phy_driver {
 	 */
 	int (*get_features)(struct phy_device *phydev);
 
+	/**
+	 * @get_rate_matching: Get the supported type of rate matching for a
+	 * particular phy interface. This is used by phy consumers to determine
+	 * whether to advertise lower-speed modes for that interface. It is
+	 * assumed that if a rate matching mode is supported on an interface,
+	 * then that interface's rate can be adapted to all slower link speeds
+	 * supported by the phy. If iface is %PHY_INTERFACE_MODE_NA, and the phy
+	 * supports any kind of rate matching for any interface, then it must
+	 * return that rate matching mode (preferring %RATE_MATCH_PAUSE to
+	 * %RATE_MATCH_CRS). If the interface is not supported, this should
+	 * return %RATE_MATCH_NONE.
+	 */
+	int (*get_rate_matching)(struct phy_device *phydev,
+				   phy_interface_t iface);
+
 	/* PHY Power Management */
 	/** @suspend: Suspend the hardware, saving state if needed */
 	int (*suspend)(struct phy_device *phydev);
@@ -963,6 +994,9 @@ struct phy_fixup {
 
 const char *phy_speed_to_str(int speed);
 const char *phy_duplex_to_str(unsigned int duplex);
+const char *phy_rate_matching_to_str(int rate_matching);
+
+int phy_interface_num_ports(phy_interface_t interface);
 
 /* A structure for mapping a particular speed and duplex
  * combination to a particular SUPPORTED and ADVERTISED value
@@ -1677,6 +1711,8 @@ int phy_disable_interrupts(struct phy_device *phydev);
 void phy_request_interrupt(struct phy_device *phydev);
 void phy_free_interrupt(struct phy_device *phydev);
 void phy_print_status(struct phy_device *phydev);
+int phy_get_rate_matching(struct phy_device *phydev,
+			    phy_interface_t iface);
 void phy_set_max_speed(struct phy_device *phydev, u32 max_speed);
 void phy_remove_link_mode(struct phy_device *phydev, u32 link_mode);
 void phy_advertise_supported(struct phy_device *phydev);
diff --git a/include/linux/phylink.h b/include/linux/phylink.h
index 6d06896fc20d..664dd409feb9 100644
--- a/include/linux/phylink.h
+++ b/include/linux/phylink.h
@@ -21,6 +21,35 @@ enum {
 	MLO_AN_FIXED,	/* Fixed-link mode */
 	MLO_AN_INBAND,	/* In-band protocol */
 
+	/* MAC_SYM_PAUSE and MAC_ASYM_PAUSE are used when configuring our
+	 * autonegotiation advertisement. They correspond to the PAUSE and
+	 * ASM_DIR bits defined by 802.3, respectively.
+	 *
+	 * The following table lists the values of tx_pause and rx_pause which
+	 * might be requested in mac_link_up. The exact values depend on either
+	 * the results of autonegotation (if MLO_PAUSE_AN is set) or user
+	 * configuration (if MLO_PAUSE_AN is not set).
+	 *
+	 * MAC_SYM_PAUSE MAC_ASYM_PAUSE MLO_PAUSE_AN tx_pause/rx_pause
+	 * ============= ============== ============ ==================
+	 *             0              0            0 0/0
+	 *             0              0            1 0/0
+	 *             0              1            0 0/0, 0/1, 1/0, 1/1
+	 *             0              1            1 0/0,      1/0
+	 *             1              0            0 0/0,           1/1
+	 *             1              0            1 0/0,           1/1
+	 *             1              1            0 0/0, 0/1, 1/0, 1/1
+	 *             1              1            1 0/0, 0/1,      1/1
+	 *
+	 * If you set MAC_ASYM_PAUSE, the user may request any combination of
+	 * tx_pause and rx_pause. You do not have to support these
+	 * combinations.
+	 *
+	 * However, you should support combinations of tx_pause and rx_pause
+	 * which might be the result of autonegotation. For example, don't set
+	 * MAC_SYM_PAUSE unless your device can support tx_pause and rx_pause
+	 * at the same time.
+	 */
 	MAC_SYM_PAUSE	= BIT(0),
 	MAC_ASYM_PAUSE	= BIT(1),
 	MAC_10HD	= BIT(2),
@@ -59,6 +88,10 @@ static inline bool phylink_autoneg_inband(unsigned int mode)
  * @speed: link speed, one of the SPEED_* constants.
  * @duplex: link duplex mode, one of DUPLEX_* constants.
  * @pause: link pause state, described by MLO_PAUSE_* constants.
+ * @rate_matching: rate matching being performed, one of the RATE_MATCH_*
+ *   constants. If rate matching is taking place, then the speed/duplex of
+ *   the medium link mode (@speed and @duplex) and the speed/duplex of the phy
+ *   interface mode (@interface) are different.
  * @link: true if the link is up.
  * @an_enabled: true if autonegotiation is enabled/desired.
  * @an_complete: true if autonegotiation has completed.
@@ -70,6 +103,7 @@ struct phylink_link_state {
 	int speed;
 	int duplex;
 	int pause;
+	int rate_matching;
 	unsigned int link:1;
 	unsigned int an_enabled:1;
 	unsigned int an_complete:1;
@@ -518,8 +552,10 @@ void pcs_link_up(struct phylink_pcs *pcs, unsigned int mode,
 		 phy_interface_t interface, int speed, int duplex);
 #endif
 
-void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface,
-			   unsigned long mac_capabilities);
+void phylink_caps_to_linkmodes(unsigned long *linkmodes, unsigned long caps);
+unsigned long phylink_get_capabilities(phy_interface_t interface,
+				       unsigned long mac_capabilities,
+				       int rate_matching);
 void phylink_generic_validate(struct phylink_config *config,
 			      unsigned long *supported,
 			      struct phylink_link_state *state);
diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h
index 8b1b795867a1..5744a2d746aa 100644
--- a/include/linux/platform_data/cros_ec_commands.h
+++ b/include/linux/platform_data/cros_ec_commands.h
@@ -5724,8 +5724,21 @@ enum typec_control_command {
 	TYPEC_CONTROL_COMMAND_EXIT_MODES,
 	TYPEC_CONTROL_COMMAND_CLEAR_EVENTS,
 	TYPEC_CONTROL_COMMAND_ENTER_MODE,
+	TYPEC_CONTROL_COMMAND_TBT_UFP_REPLY,
+	TYPEC_CONTROL_COMMAND_USB_MUX_SET,
 };
 
+/* Replies the AP may specify to the TBT EnterMode command as a UFP */
+enum typec_tbt_ufp_reply {
+	TYPEC_TBT_UFP_REPLY_NAK,
+	TYPEC_TBT_UFP_REPLY_ACK,
+};
+
+struct typec_usb_mux_set {
+	uint8_t mux_index;	/* Index of the mux to set in the chain */
+	uint8_t mux_flags;	/* USB_PD_MUX_*-encoded USB mux state to set */
+} __ec_align1;
+
 struct ec_params_typec_control {
 	uint8_t port;
 	uint8_t command;	/* enum typec_control_command */
@@ -5739,6 +5752,8 @@ struct ec_params_typec_control {
 	union {
 		uint32_t clear_events_mask;
 		uint8_t mode_to_enter;      /* enum typec_mode */
+		uint8_t tbt_ufp_reply;      /* enum typec_tbt_ufp_reply */
+		struct typec_usb_mux_set mux_params;
 		uint8_t placeholder[128];
 	};
 } __ec_align1;
@@ -5817,6 +5832,9 @@ enum tcpc_cc_polarity {
 #define PD_STATUS_EVENT_SOP_DISC_DONE		BIT(0)
 #define PD_STATUS_EVENT_SOP_PRIME_DISC_DONE	BIT(1)
 #define PD_STATUS_EVENT_HARD_RESET		BIT(2)
+#define PD_STATUS_EVENT_DISCONNECTED		BIT(3)
+#define PD_STATUS_EVENT_MUX_0_SET_DONE		BIT(4)
+#define PD_STATUS_EVENT_MUX_1_SET_DONE		BIT(5)
 
 struct ec_params_typec_status {
 	uint8_t port;
diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h
index 408b29ca4004..e43107e0bee1 100644
--- a/include/linux/platform_data/cros_ec_proto.h
+++ b/include/linux/platform_data/cros_ec_proto.h
@@ -169,6 +169,7 @@ struct cros_ec_device {
 	int event_size;
 	u32 host_event_wake_mask;
 	u32 last_resume_result;
+	u16 suspend_timeout_ms;
 	ktime_t last_event_time;
 	struct notifier_block notifier_ready;
 
diff --git a/include/linux/platform_data/emc2305.h b/include/linux/platform_data/emc2305.h
new file mode 100644
index 000000000000..54d672dd6f7d
--- /dev/null
+++ b/include/linux/platform_data/emc2305.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __LINUX_PLATFORM_DATA_EMC2305__
+#define __LINUX_PLATFORM_DATA_EMC2305__
+
+#define EMC2305_PWM_MAX	5
+
+/**
+ * struct emc2305_platform_data - EMC2305 driver platform data
+ * @max_state: maximum cooling state of the cooling device;
+ * @pwm_num: number of active channels;
+ * @pwm_separate: separate PWM settings for every channel;
+ * @pwm_min: array of minimum PWM per channel;
+ */
+struct emc2305_platform_data {
+	u8 max_state;
+	u8 pwm_num;
+	bool pwm_separate;
+	u8 pwm_min[EMC2305_PWM_MAX];
+};
+
+#endif
diff --git a/include/linux/platform_data/pca953x.h b/include/linux/platform_data/pca953x.h
index 4eb53e023997..96c1a14ab365 100644
--- a/include/linux/platform_data/pca953x.h
+++ b/include/linux/platform_data/pca953x.h
@@ -22,7 +22,7 @@ struct pca953x_platform_data {
 	int		(*setup)(struct i2c_client *client,
 				unsigned gpio, unsigned ngpio,
 				void *context);
-	int		(*teardown)(struct i2c_client *client,
+	void		(*teardown)(struct i2c_client *client,
 				unsigned gpio, unsigned ngpio,
 				void *context);
 	const char	*const *names;
diff --git a/include/linux/platform_data/ssm2518.h b/include/linux/platform_data/ssm2518.h
deleted file mode 100644
index 3f9e632d6f63..000000000000
--- a/include/linux/platform_data/ssm2518.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * SSM2518 amplifier audio driver
- *
- * Copyright 2013 Analog Devices Inc.
- *  Author: Lars-Peter Clausen <lars@metafoo.de>
- */
-
-#ifndef __LINUX_PLATFORM_DATA_SSM2518_H__
-#define __LINUX_PLATFORM_DATA_SSM2518_H__
-
-/**
- * struct ssm2518_platform_data - Platform data for the ssm2518 driver
- * @enable_gpio: GPIO connected to the nSD pin. Set to -1 if the nSD pin is
- *            hardwired.
- */
-struct ssm2518_platform_data {
-	int enable_gpio;
-};
-
-#endif
diff --git a/include/linux/platform_data/tps68470.h b/include/linux/platform_data/tps68470.h
index 126d082c3f2e..e605a2cab07f 100644
--- a/include/linux/platform_data/tps68470.h
+++ b/include/linux/platform_data/tps68470.h
@@ -27,9 +27,14 @@ struct tps68470_regulator_platform_data {
 	const struct regulator_init_data *reg_init_data[TPS68470_NUM_REGULATORS];
 };
 
-struct tps68470_clk_platform_data {
+struct tps68470_clk_consumer {
 	const char *consumer_dev_name;
 	const char *consumer_con_id;
 };
 
+struct tps68470_clk_platform_data {
+	unsigned int n_consumers;
+	struct tps68470_clk_consumer consumers[];
+};
+
 #endif
diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h
index 98f2b2f20f3e..28234dc9fa6a 100644
--- a/include/linux/platform_data/x86/asus-wmi.h
+++ b/include/linux/platform_data/x86/asus-wmi.h
@@ -65,6 +65,7 @@
 #define ASUS_WMI_DEVID_PANEL_OD		0x00050019
 #define ASUS_WMI_DEVID_CAMERA		0x00060013
 #define ASUS_WMI_DEVID_LID_FLIP		0x00060062
+#define ASUS_WMI_DEVID_LID_FLIP_ROG	0x00060077
 
 /* Storage */
 #define ASUS_WMI_DEVID_CARDREADER	0x00080013
@@ -78,6 +79,7 @@
 #define ASUS_WMI_DEVID_THERMAL_CTRL	0x00110011
 #define ASUS_WMI_DEVID_FAN_CTRL		0x00110012 /* deprecated */
 #define ASUS_WMI_DEVID_CPU_FAN_CTRL	0x00110013
+#define ASUS_WMI_DEVID_GPU_FAN_CTRL	0x00110014
 #define ASUS_WMI_DEVID_CPU_FAN_CURVE	0x00110024
 #define ASUS_WMI_DEVID_GPU_FAN_CURVE	0x00110025
 
@@ -99,6 +101,15 @@
 /* dgpu on/off */
 #define ASUS_WMI_DEVID_DGPU		0x00090020
 
+/* gpu mux switch, 0 = dGPU, 1 = Optimus */
+#define ASUS_WMI_DEVID_GPU_MUX		0x00090016
+
+/* TUF laptop RGB modes/colours */
+#define ASUS_WMI_DEVID_TUF_RGB_MODE	0x00100056
+
+/* TUF laptop RGB power/state */
+#define ASUS_WMI_DEVID_TUF_RGB_STATE	0x00100057
+
 /* DSTS masks */
 #define ASUS_WMI_DSTS_STATUS_BIT	0x00000001
 #define ASUS_WMI_DSTS_UNKNOWN_BIT	0x00000002
diff --git a/include/linux/platform_data/x86/nvidia-wmi-ec-backlight.h b/include/linux/platform_data/x86/nvidia-wmi-ec-backlight.h
new file mode 100644
index 000000000000..23d60130272c
--- /dev/null
+++ b/include/linux/platform_data/x86/nvidia-wmi-ec-backlight.h
@@ -0,0 +1,76 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+ */
+
+#ifndef __PLATFORM_DATA_X86_NVIDIA_WMI_EC_BACKLIGHT_H
+#define __PLATFORM_DATA_X86_NVIDIA_WMI_EC_BACKLIGHT_H
+
+#define WMI_BRIGHTNESS_GUID "603E9613-EF25-4338-A3D0-C46177516DB7"
+
+/**
+ * enum wmi_brightness_method - WMI method IDs
+ * @WMI_BRIGHTNESS_METHOD_LEVEL:  Get/Set EC brightness level status
+ * @WMI_BRIGHTNESS_METHOD_SOURCE: Get/Set EC Brightness Source
+ */
+enum wmi_brightness_method {
+	WMI_BRIGHTNESS_METHOD_LEVEL = 1,
+	WMI_BRIGHTNESS_METHOD_SOURCE = 2,
+	WMI_BRIGHTNESS_METHOD_MAX
+};
+
+/**
+ * enum wmi_brightness_mode - Operation mode for WMI-wrapped method
+ * @WMI_BRIGHTNESS_MODE_GET:            Get the current brightness level/source.
+ * @WMI_BRIGHTNESS_MODE_SET:            Set the brightness level.
+ * @WMI_BRIGHTNESS_MODE_GET_MAX_LEVEL:  Get the maximum brightness level. This
+ *                                      is only valid when the WMI method is
+ *                                      %WMI_BRIGHTNESS_METHOD_LEVEL.
+ */
+enum wmi_brightness_mode {
+	WMI_BRIGHTNESS_MODE_GET = 0,
+	WMI_BRIGHTNESS_MODE_SET = 1,
+	WMI_BRIGHTNESS_MODE_GET_MAX_LEVEL = 2,
+	WMI_BRIGHTNESS_MODE_MAX
+};
+
+/**
+ * enum wmi_brightness_source - Backlight brightness control source selection
+ * @WMI_BRIGHTNESS_SOURCE_GPU: Backlight brightness is controlled by the GPU.
+ * @WMI_BRIGHTNESS_SOURCE_EC:  Backlight brightness is controlled by the
+ *                             system's Embedded Controller (EC).
+ * @WMI_BRIGHTNESS_SOURCE_AUX: Backlight brightness is controlled over the
+ *                             DisplayPort AUX channel.
+ */
+enum wmi_brightness_source {
+	WMI_BRIGHTNESS_SOURCE_GPU = 1,
+	WMI_BRIGHTNESS_SOURCE_EC = 2,
+	WMI_BRIGHTNESS_SOURCE_AUX = 3,
+	WMI_BRIGHTNESS_SOURCE_MAX
+};
+
+/**
+ * struct wmi_brightness_args - arguments for the WMI-wrapped ACPI method
+ * @mode:    Pass in an &enum wmi_brightness_mode value to select between
+ *           getting or setting a value.
+ * @val:     In parameter for value to set when using %WMI_BRIGHTNESS_MODE_SET
+ *           mode. Not used in conjunction with %WMI_BRIGHTNESS_MODE_GET or
+ *           %WMI_BRIGHTNESS_MODE_GET_MAX_LEVEL mode.
+ * @ret:     Out parameter returning retrieved value when operating in
+ *           %WMI_BRIGHTNESS_MODE_GET or %WMI_BRIGHTNESS_MODE_GET_MAX_LEVEL
+ *           mode. Not used in %WMI_BRIGHTNESS_MODE_SET mode.
+ * @ignored: Padding; not used. The ACPI method expects a 24 byte params struct.
+ *
+ * This is the parameters structure for the WmiBrightnessNotify ACPI method as
+ * wrapped by WMI. The value passed in to @val or returned by @ret will be a
+ * brightness value when the WMI method ID is %WMI_BRIGHTNESS_METHOD_LEVEL, or
+ * an &enum wmi_brightness_source value with %WMI_BRIGHTNESS_METHOD_SOURCE.
+ */
+struct wmi_brightness_args {
+	u32 mode;
+	u32 val;
+	u32 ret;
+	u32 ignored[3];
+};
+
+#endif
diff --git a/include/linux/platform_data/x86/pmc_atom.h b/include/linux/platform_data/x86/pmc_atom.h
index dd81f510e4cf..b8a701c77fd0 100644
--- a/include/linux/platform_data/x86/pmc_atom.h
+++ b/include/linux/platform_data/x86/pmc_atom.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Intel Atom SOC Power Management Controller Header File
- * Copyright (c) 2014, Intel Corporation.
+ * Intel Atom SoC Power Management Controller Header File
+ * Copyright (c) 2014-2015,2022 Intel Corporation.
  */
 
 #ifndef PMC_ATOM_H
diff --git a/include/linux/platform_data/x86/simatic-ipc-base.h b/include/linux/platform_data/x86/simatic-ipc-base.h
index 39fefd48cf4d..57d6a10dfc9e 100644
--- a/include/linux/platform_data/x86/simatic-ipc-base.h
+++ b/include/linux/platform_data/x86/simatic-ipc-base.h
@@ -19,6 +19,7 @@
 #define SIMATIC_IPC_DEVICE_427E 2
 #define SIMATIC_IPC_DEVICE_127E 3
 #define SIMATIC_IPC_DEVICE_227E 4
+#define SIMATIC_IPC_DEVICE_227G 5
 
 struct simatic_ipc_platform {
 	u8	devmode;
diff --git a/include/linux/platform_data/x86/simatic-ipc.h b/include/linux/platform_data/x86/simatic-ipc.h
index f3b76b39776b..632320ec8f08 100644
--- a/include/linux/platform_data/x86/simatic-ipc.h
+++ b/include/linux/platform_data/x86/simatic-ipc.h
@@ -31,6 +31,8 @@ enum simatic_ipc_station_ids {
 	SIMATIC_IPC_IPC427E = 0x00000A01,
 	SIMATIC_IPC_IPC477E = 0x00000A02,
 	SIMATIC_IPC_IPC127E = 0x00000D01,
+	SIMATIC_IPC_IPC227G = 0x00000F01,
+	SIMATIC_IPC_IPC427G = 0x00001001,
 };
 
 static inline u32 simatic_ipc_get_station_id(u8 *data, int max_len)
diff --git a/include/linux/poison.h b/include/linux/poison.h
index d62ef5a6b4e9..2d3249eb0e62 100644
--- a/include/linux/poison.h
+++ b/include/linux/poison.h
@@ -81,4 +81,7 @@
 /********** net/core/page_pool.c **********/
 #define PP_SIGNATURE		(0x40 + POISON_POINTER_DELTA)
 
+/********** kernel/bpf/ **********/
+#define BPF_PTR_POISON ((void *)(0xeB9FUL + POISON_POINTER_DELTA))
+
 #endif
diff --git a/include/linux/posix_acl_xattr.h b/include/linux/posix_acl_xattr.h
index b6bd3eac2bcc..8163dd48c430 100644
--- a/include/linux/posix_acl_xattr.h
+++ b/include/linux/posix_acl_xattr.h
@@ -38,9 +38,6 @@ void posix_acl_fix_xattr_to_user(void *value, size_t size);
 void posix_acl_getxattr_idmapped_mnt(struct user_namespace *mnt_userns,
 				     const struct inode *inode,
 				     void *value, size_t size);
-void posix_acl_setxattr_idmapped_mnt(struct user_namespace *mnt_userns,
-				     const struct inode *inode,
-				     void *value, size_t size);
 #else
 static inline void posix_acl_fix_xattr_from_user(void *value, size_t size)
 {
@@ -54,18 +51,15 @@ posix_acl_getxattr_idmapped_mnt(struct user_namespace *mnt_userns,
 				size_t size)
 {
 }
-static inline void
-posix_acl_setxattr_idmapped_mnt(struct user_namespace *mnt_userns,
-				const struct inode *inode, void *value,
-				size_t size)
-{
-}
 #endif
 
 struct posix_acl *posix_acl_from_xattr(struct user_namespace *user_ns, 
 				       const void *value, size_t size);
 int posix_acl_to_xattr(struct user_namespace *user_ns,
 		       const struct posix_acl *acl, void *buffer, size_t size);
+struct posix_acl *vfs_set_acl_prepare(struct user_namespace *mnt_userns,
+				      struct user_namespace *fs_userns,
+				      const void *value, size_t size);
 
 extern const struct xattr_handler posix_acl_access_xattr_handler;
 extern const struct xattr_handler posix_acl_default_xattr_handler;
diff --git a/include/linux/pse-pd/pse.h b/include/linux/pse-pd/pse.h
new file mode 100644
index 000000000000..fb724c65c77b
--- /dev/null
+++ b/include/linux/pse-pd/pse.h
@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+// Copyright (c) 2022 Pengutronix, Oleksij Rempel <kernel@pengutronix.de>
+ */
+#ifndef _LINUX_PSE_CONTROLLER_H
+#define _LINUX_PSE_CONTROLLER_H
+
+#include <linux/ethtool.h>
+#include <linux/list.h>
+#include <uapi/linux/ethtool.h>
+
+struct phy_device;
+struct pse_controller_dev;
+
+/**
+ * struct pse_control_config - PSE control/channel configuration.
+ *
+ * @admin_cotrol: set PoDL PSE admin control as described in
+ *	IEEE 802.3-2018 30.15.1.2.1 acPoDLPSEAdminControl
+ */
+struct pse_control_config {
+	enum ethtool_podl_pse_admin_state admin_cotrol;
+};
+
+/**
+ * struct pse_control_status - PSE control/channel status.
+ *
+ * @podl_admin_state: operational state of the PoDL PSE
+ *	functions. IEEE 802.3-2018 30.15.1.1.2 aPoDLPSEAdminState
+ * @podl_pw_status: power detection status of the PoDL PSE.
+ *	IEEE 802.3-2018 30.15.1.1.3 aPoDLPSEPowerDetectionStatus:
+ */
+struct pse_control_status {
+	enum ethtool_podl_pse_admin_state podl_admin_state;
+	enum ethtool_podl_pse_pw_d_status podl_pw_status;
+};
+
+/**
+ * struct pse_controller_ops - PSE controller driver callbacks
+ *
+ * @ethtool_get_status: get PSE control status for ethtool interface
+ * @ethtool_set_config: set PSE control configuration over ethtool interface
+ */
+struct pse_controller_ops {
+	int (*ethtool_get_status)(struct pse_controller_dev *pcdev,
+		unsigned long id, struct netlink_ext_ack *extack,
+		struct pse_control_status *status);
+	int (*ethtool_set_config)(struct pse_controller_dev *pcdev,
+		unsigned long id, struct netlink_ext_ack *extack,
+		const struct pse_control_config *config);
+};
+
+struct module;
+struct device_node;
+struct of_phandle_args;
+struct pse_control;
+
+/**
+ * struct pse_controller_dev - PSE controller entity that might
+ *                             provide multiple PSE controls
+ * @ops: a pointer to device specific struct pse_controller_ops
+ * @owner: kernel module of the PSE controller driver
+ * @list: internal list of PSE controller devices
+ * @pse_control_head: head of internal list of requested PSE controls
+ * @dev: corresponding driver model device struct
+ * @of_pse_n_cells: number of cells in PSE line specifiers
+ * @of_xlate: translation function to translate from specifier as found in the
+ *            device tree to id as given to the PSE control ops
+ * @nr_lines: number of PSE controls in this controller device
+ * @lock: Mutex for serialization access to the PSE controller
+ */
+struct pse_controller_dev {
+	const struct pse_controller_ops *ops;
+	struct module *owner;
+	struct list_head list;
+	struct list_head pse_control_head;
+	struct device *dev;
+	int of_pse_n_cells;
+	int (*of_xlate)(struct pse_controller_dev *pcdev,
+			const struct of_phandle_args *pse_spec);
+	unsigned int nr_lines;
+	struct mutex lock;
+};
+
+#if IS_ENABLED(CONFIG_PSE_CONTROLLER)
+int pse_controller_register(struct pse_controller_dev *pcdev);
+void pse_controller_unregister(struct pse_controller_dev *pcdev);
+struct device;
+int devm_pse_controller_register(struct device *dev,
+				 struct pse_controller_dev *pcdev);
+
+struct pse_control *of_pse_control_get(struct device_node *node);
+void pse_control_put(struct pse_control *psec);
+
+int pse_ethtool_get_status(struct pse_control *psec,
+			   struct netlink_ext_ack *extack,
+			   struct pse_control_status *status);
+int pse_ethtool_set_config(struct pse_control *psec,
+			   struct netlink_ext_ack *extack,
+			   const struct pse_control_config *config);
+
+#else
+
+static inline struct pse_control *of_pse_control_get(struct device_node *node)
+{
+	return ERR_PTR(-ENOENT);
+}
+
+static inline void pse_control_put(struct pse_control *psec)
+{
+}
+
+static inline int pse_ethtool_get_status(struct pse_control *psec,
+					 struct netlink_ext_ack *extack,
+					 struct pse_control_status *status)
+{
+	return -ENOTSUPP;
+}
+
+static inline int pse_ethtool_set_config(struct pse_control *psec,
+					 struct netlink_ext_ack *extack,
+					 const struct pse_control_config *config)
+{
+	return -ENOTSUPP;
+}
+
+#endif
+
+#endif
diff --git a/include/linux/pwm.h b/include/linux/pwm.h
index 9429930c5566..d70c6e5a839d 100644
--- a/include/linux/pwm.h
+++ b/include/linux/pwm.h
@@ -403,13 +403,9 @@ struct pwm_device *of_pwm_single_xlate(struct pwm_chip *pc,
 				       const struct of_phandle_args *args);
 
 struct pwm_device *pwm_get(struct device *dev, const char *con_id);
-struct pwm_device *of_pwm_get(struct device *dev, struct device_node *np,
-			      const char *con_id);
 void pwm_put(struct pwm_device *pwm);
 
 struct pwm_device *devm_pwm_get(struct device *dev, const char *con_id);
-struct pwm_device *devm_of_pwm_get(struct device *dev, struct device_node *np,
-				   const char *con_id);
 struct pwm_device *devm_fwnode_pwm_get(struct device *dev,
 				       struct fwnode_handle *fwnode,
 				       const char *con_id);
@@ -497,14 +493,6 @@ static inline struct pwm_device *pwm_get(struct device *dev,
 	return ERR_PTR(-ENODEV);
 }
 
-static inline struct pwm_device *of_pwm_get(struct device *dev,
-					    struct device_node *np,
-					    const char *con_id)
-{
-	might_sleep();
-	return ERR_PTR(-ENODEV);
-}
-
 static inline void pwm_put(struct pwm_device *pwm)
 {
 	might_sleep();
@@ -517,14 +505,6 @@ static inline struct pwm_device *devm_pwm_get(struct device *dev,
 	return ERR_PTR(-ENODEV);
 }
 
-static inline struct pwm_device *devm_of_pwm_get(struct device *dev,
-						 struct device_node *np,
-						 const char *con_id)
-{
-	might_sleep();
-	return ERR_PTR(-ENODEV);
-}
-
 static inline struct pwm_device *
 devm_fwnode_pwm_get(struct device *dev, struct fwnode_handle *fwnode,
 		    const char *con_id)
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index f527f27e6438..08605ce7379d 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -42,7 +42,31 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func);
 void rcu_barrier_tasks(void);
 void rcu_barrier_tasks_rude(void);
 void synchronize_rcu(void);
+
+struct rcu_gp_oldstate;
 unsigned long get_completed_synchronize_rcu(void);
+void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
+
+// Maximum number of unsigned long values corresponding to
+// not-yet-completed RCU grace periods.
+#define NUM_ACTIVE_RCU_POLL_OLDSTATE 2
+
+/**
+ * same_state_synchronize_rcu - Are two old-state values identical?
+ * @oldstate1: First old-state value.
+ * @oldstate2: Second old-state value.
+ *
+ * The two old-state values must have been obtained from either
+ * get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or
+ * get_completed_synchronize_rcu().  Returns @true if the two values are
+ * identical and @false otherwise.  This allows structures whose lifetimes
+ * are tracked by old-state values to push these values to a list header,
+ * allowing those structures to be slightly smaller.
+ */
+static inline bool same_state_synchronize_rcu(unsigned long oldstate1, unsigned long oldstate2)
+{
+	return oldstate1 == oldstate2;
+}
 
 #ifdef CONFIG_PREEMPT_RCU
 
@@ -496,13 +520,21 @@ do {									      \
  * against NULL.  Although rcu_access_pointer() may also be used in cases
  * where update-side locks prevent the value of the pointer from changing,
  * you should instead use rcu_dereference_protected() for this use case.
+ * Within an RCU read-side critical section, there is little reason to
+ * use rcu_access_pointer().
+ *
+ * It is usually best to test the rcu_access_pointer() return value
+ * directly in order to avoid accidental dereferences being introduced
+ * by later inattentive changes.  In other words, assigning the
+ * rcu_access_pointer() return value to a local variable results in an
+ * accident waiting to happen.
  *
  * It is also permissible to use rcu_access_pointer() when read-side
- * access to the pointer was removed at least one grace period ago, as
- * is the case in the context of the RCU callback that is freeing up
- * the data, or after a synchronize_rcu() returns.  This can be useful
- * when tearing down multi-linked structures after a grace period
- * has elapsed.
+ * access to the pointer was removed at least one grace period ago, as is
+ * the case in the context of the RCU callback that is freeing up the data,
+ * or after a synchronize_rcu() returns.  This can be useful when tearing
+ * down multi-linked structures after a grace period has elapsed.  However,
+ * rcu_dereference_protected() is normally preferred for this use case.
  */
 #define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu)
 
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 62815c0a2dce..768196a5f39d 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -14,25 +14,75 @@
 
 #include <asm/param.h> /* for HZ */
 
+struct rcu_gp_oldstate {
+	unsigned long rgos_norm;
+};
+
+// Maximum number of rcu_gp_oldstate values corresponding to
+// not-yet-completed RCU grace periods.
+#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 2
+
+/*
+ * Are the two oldstate values the same?  See the Tree RCU version for
+ * docbook header.
+ */
+static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1,
+						   struct rcu_gp_oldstate *rgosp2)
+{
+	return rgosp1->rgos_norm == rgosp2->rgos_norm;
+}
+
 unsigned long get_state_synchronize_rcu(void);
+
+static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+	rgosp->rgos_norm = get_state_synchronize_rcu();
+}
+
 unsigned long start_poll_synchronize_rcu(void);
+
+static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+	rgosp->rgos_norm = start_poll_synchronize_rcu();
+}
+
 bool poll_state_synchronize_rcu(unsigned long oldstate);
 
+static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+	return poll_state_synchronize_rcu(rgosp->rgos_norm);
+}
+
 static inline void cond_synchronize_rcu(unsigned long oldstate)
 {
 	might_sleep();
 }
 
+static inline void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+	cond_synchronize_rcu(rgosp->rgos_norm);
+}
+
 static inline unsigned long start_poll_synchronize_rcu_expedited(void)
 {
 	return start_poll_synchronize_rcu();
 }
 
+static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)
+{
+	rgosp->rgos_norm = start_poll_synchronize_rcu_expedited();
+}
+
 static inline void cond_synchronize_rcu_expedited(unsigned long oldstate)
 {
 	cond_synchronize_rcu(oldstate);
 }
 
+static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)
+{
+	cond_synchronize_rcu_expedited(rgosp->rgos_norm);
+}
+
 extern void rcu_barrier(void);
 
 static inline void synchronize_rcu_expedited(void)
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 47eaa4cb0df7..5efb51486e8a 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -40,12 +40,52 @@ bool rcu_eqs_special_set(int cpu);
 void rcu_momentary_dyntick_idle(void);
 void kfree_rcu_scheduler_running(void);
 bool rcu_gp_might_be_stalled(void);
+
+struct rcu_gp_oldstate {
+	unsigned long rgos_norm;
+	unsigned long rgos_exp;
+};
+
+// Maximum number of rcu_gp_oldstate values corresponding to
+// not-yet-completed RCU grace periods.
+#define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4
+
+/**
+ * same_state_synchronize_rcu_full - Are two old-state values identical?
+ * @rgosp1: First old-state value.
+ * @rgosp2: Second old-state value.
+ *
+ * The two old-state values must have been obtained from either
+ * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(),
+ * or get_completed_synchronize_rcu_full().  Returns @true if the two
+ * values are identical and @false otherwise.  This allows structures
+ * whose lifetimes are tracked by old-state values to push these values
+ * to a list header, allowing those structures to be slightly smaller.
+ *
+ * Note that equality is judged on a bitwise basis, so that an
+ * @rcu_gp_oldstate structure with an already-completed state in one field
+ * will compare not-equal to a structure with an already-completed state
+ * in the other field.  After all, the @rcu_gp_oldstate structure is opaque
+ * so how did such a situation come to pass in the first place?
+ */
+static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1,
+						   struct rcu_gp_oldstate *rgosp2)
+{
+	return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp;
+}
+
 unsigned long start_poll_synchronize_rcu_expedited(void);
+void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);
 void cond_synchronize_rcu_expedited(unsigned long oldstate);
+void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp);
 unsigned long get_state_synchronize_rcu(void);
+void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
 unsigned long start_poll_synchronize_rcu(void);
+void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
 bool poll_state_synchronize_rcu(unsigned long oldstate);
+bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
 void cond_synchronize_rcu(unsigned long oldstate);
+void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp);
 
 bool rcu_is_idle_cpu(int cpu);
 
diff --git a/include/linux/regmap.h b/include/linux/regmap.h
index 7cf2157134ac..ca3434dca3a0 100644
--- a/include/linux/regmap.h
+++ b/include/linux/regmap.h
@@ -311,6 +311,8 @@ typedef void (*regmap_unlock)(void *);
  *		  This field is a duplicate of a similar file in
  *		  'struct regmap_bus' and serves exact same purpose.
  *		   Use it only for "no-bus" cases.
+ * @io_port:	  Support IO port accessors. Makes sense only when MMIO vs. IO port
+ *		  access can be distinguished.
  * @max_register: Optional, specifies the maximum valid register address.
  * @wr_table:     Optional, points to a struct regmap_access_table specifying
  *                valid ranges for write access.
@@ -399,6 +401,7 @@ struct regmap_config {
 	size_t max_raw_write;
 
 	bool fast_io;
+	bool io_port;
 
 	unsigned int max_register;
 	const struct regmap_access_table *wr_table;
@@ -489,8 +492,12 @@ typedef int (*regmap_hw_read)(void *context,
 			      void *val_buf, size_t val_size);
 typedef int (*regmap_hw_reg_read)(void *context, unsigned int reg,
 				  unsigned int *val);
+typedef int (*regmap_hw_reg_noinc_read)(void *context, unsigned int reg,
+					void *val, size_t val_count);
 typedef int (*regmap_hw_reg_write)(void *context, unsigned int reg,
 				   unsigned int val);
+typedef int (*regmap_hw_reg_noinc_write)(void *context, unsigned int reg,
+					 const void *val, size_t val_count);
 typedef int (*regmap_hw_reg_update_bits)(void *context, unsigned int reg,
 					 unsigned int mask, unsigned int val);
 typedef struct regmap_async *(*regmap_hw_async_alloc)(void);
@@ -511,6 +518,8 @@ typedef void (*regmap_hw_free_context)(void *context);
  *               must serialise with respect to non-async I/O.
  * @reg_write: Write a single register value to the given register address. This
  *             write operation has to complete when returning from the function.
+ * @reg_write_noinc: Write multiple register value to the same register. This
+ *             write operation has to complete when returning from the function.
  * @reg_update_bits: Update bits operation to be used against volatile
  *                   registers, intended for devices supporting some mechanism
  *                   for setting clearing bits without having to
@@ -538,9 +547,11 @@ struct regmap_bus {
 	regmap_hw_gather_write gather_write;
 	regmap_hw_async_write async_write;
 	regmap_hw_reg_write reg_write;
+	regmap_hw_reg_noinc_write reg_noinc_write;
 	regmap_hw_reg_update_bits reg_update_bits;
 	regmap_hw_read read;
 	regmap_hw_reg_read reg_read;
+	regmap_hw_reg_noinc_read reg_noinc_read;
 	regmap_hw_free_context free_context;
 	regmap_hw_async_alloc async_alloc;
 	u8 read_flag_mask;
diff --git a/include/linux/regulator/consumer.h b/include/linux/regulator/consumer.h
index bc6cda706d1f..ee3b4a014611 100644
--- a/include/linux/regulator/consumer.h
+++ b/include/linux/regulator/consumer.h
@@ -207,6 +207,8 @@ struct regulator *__must_check regulator_get_optional(struct device *dev,
 						      const char *id);
 struct regulator *__must_check devm_regulator_get_optional(struct device *dev,
 							   const char *id);
+int devm_regulator_get_enable(struct device *dev, const char *id);
+int devm_regulator_get_enable_optional(struct device *dev, const char *id);
 void regulator_put(struct regulator *regulator);
 void devm_regulator_put(struct regulator *regulator);
 
@@ -244,12 +246,15 @@ int __must_check regulator_bulk_get(struct device *dev, int num_consumers,
 				    struct regulator_bulk_data *consumers);
 int __must_check devm_regulator_bulk_get(struct device *dev, int num_consumers,
 					 struct regulator_bulk_data *consumers);
+void devm_regulator_bulk_put(struct regulator_bulk_data *consumers);
 int __must_check devm_regulator_bulk_get_const(
 	struct device *dev, int num_consumers,
 	const struct regulator_bulk_data *in_consumers,
 	struct regulator_bulk_data **out_consumers);
 int __must_check regulator_bulk_enable(int num_consumers,
 				       struct regulator_bulk_data *consumers);
+int devm_regulator_bulk_get_enable(struct device *dev, int num_consumers,
+				   const char * const *id);
 int regulator_bulk_disable(int num_consumers,
 			   struct regulator_bulk_data *consumers);
 int regulator_bulk_force_disable(int num_consumers,
@@ -354,6 +359,17 @@ devm_regulator_get_exclusive(struct device *dev, const char *id)
 	return ERR_PTR(-ENODEV);
 }
 
+static inline int devm_regulator_get_enable(struct device *dev, const char *id)
+{
+	return -ENODEV;
+}
+
+static inline int devm_regulator_get_enable_optional(struct device *dev,
+						     const char *id)
+{
+	return -ENODEV;
+}
+
 static inline struct regulator *__must_check
 regulator_get_optional(struct device *dev, const char *id)
 {
@@ -375,6 +391,10 @@ static inline void devm_regulator_put(struct regulator *regulator)
 {
 }
 
+static inline void devm_regulator_bulk_put(struct regulator_bulk_data *consumers)
+{
+}
+
 static inline int regulator_register_supply_alias(struct device *dev,
 						  const char *id,
 						  struct device *alias_dev,
@@ -465,6 +485,13 @@ static inline int regulator_bulk_enable(int num_consumers,
 	return 0;
 }
 
+static inline int devm_regulator_bulk_get_enable(struct device *dev,
+						 int num_consumers,
+						 const char * const *id)
+{
+	return 0;
+}
+
 static inline int regulator_bulk_disable(int num_consumers,
 					 struct regulator_bulk_data *consumers)
 {
diff --git a/include/linux/regulator/gpio-regulator.h b/include/linux/regulator/gpio-regulator.h
index fdeb312cdabd..c223e50ff9f7 100644
--- a/include/linux/regulator/gpio-regulator.h
+++ b/include/linux/regulator/gpio-regulator.h
@@ -42,6 +42,7 @@ struct gpio_regulator_state {
 /**
  * struct gpio_regulator_config - config structure
  * @supply_name:	Name of the regulator supply
+ * @input_supply:	Name of the input regulator supply
  * @enabled_at_boot:	Whether regulator has been enabled at
  *			boot or not. 1 = Yes, 0 = No
  *			This is used to keep the regulator at
@@ -62,6 +63,7 @@ struct gpio_regulator_state {
  */
 struct gpio_regulator_config {
 	const char *supply_name;
+	const char *input_supply;
 
 	unsigned enabled_at_boot:1;
 	unsigned startup_delay;
diff --git a/include/linux/regulator/mt6331-regulator.h b/include/linux/regulator/mt6331-regulator.h
new file mode 100644
index 000000000000..2801a9879c14
--- /dev/null
+++ b/include/linux/regulator/mt6331-regulator.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022 Collabora Ltd.
+ * Author: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
+ */
+
+#ifndef __LINUX_REGULATOR_MT6331_H
+#define __LINUX_REGULATOR_MT6331_H
+
+enum {
+	/* BUCK */
+	MT6331_ID_VDVFS11 = 0,
+	MT6331_ID_VDVFS12,
+	MT6331_ID_VDVFS13,
+	MT6331_ID_VDVFS14,
+	MT6331_ID_VCORE2,
+	MT6331_ID_VIO18,
+	/* LDO */
+	MT6331_ID_VTCXO1,
+	MT6331_ID_VTCXO2,
+	MT6331_ID_AVDD32_AUD,
+	MT6331_ID_VAUXA32,
+	MT6331_ID_VCAMA,
+	MT6331_ID_VIO28,
+	MT6331_ID_VCAM_AF,
+	MT6331_ID_VMC,
+	MT6331_ID_VMCH,
+	MT6331_ID_VEMC33,
+	MT6331_ID_VGP1,
+	MT6331_ID_VSIM1,
+	MT6331_ID_VSIM2,
+	MT6331_ID_VMIPI,
+	MT6331_ID_VIBR,
+	MT6331_ID_VGP4,
+	MT6331_ID_VCAMD,
+	MT6331_ID_VUSB10,
+	MT6331_ID_VCAM_IO,
+	MT6331_ID_VSRAM_DVFS1,
+	MT6331_ID_VGP2,
+	MT6331_ID_VGP3,
+	MT6331_ID_VRTC,
+	MT6331_ID_VDIG18,
+	MT6331_ID_VREG_MAX
+};
+
+#endif /* __LINUX_REGULATOR_MT6331_H */
diff --git a/include/linux/regulator/mt6332-regulator.h b/include/linux/regulator/mt6332-regulator.h
new file mode 100644
index 000000000000..af5e3ed31029
--- /dev/null
+++ b/include/linux/regulator/mt6332-regulator.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022 Collabora Ltd.
+ * Author: AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>
+ */
+
+#ifndef __LINUX_REGULATOR_MT6332_H
+#define __LINUX_REGULATOR_MT6332_H
+
+enum {
+	/* BUCK */
+	MT6332_ID_VDRAM = 0,
+	MT6332_ID_VDVFS2,
+	MT6332_ID_VPA,
+	MT6332_ID_VRF1,
+	MT6332_ID_VRF2,
+	MT6332_ID_VSBST,
+	/* LDO */
+	MT6332_ID_VAUXB32,
+	MT6332_ID_VBIF28,
+	MT6332_ID_VDIG18,
+	MT6332_ID_VSRAM_DVFS2,
+	MT6332_ID_VUSB33,
+	MT6332_ID_VREG_MAX
+};
+
+#endif /* __LINUX_REGULATOR_MT6332_H */
diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h
index 21deb5212bbd..0cf5b20c6ddf 100644
--- a/include/linux/resctrl.h
+++ b/include/linux/resctrl.h
@@ -15,6 +15,9 @@ int proc_resctrl_show(struct seq_file *m,
 
 #endif
 
+/* max value for struct rdt_domain's mbps_val */
+#define MBA_MAX_MBPS   U32_MAX
+
 /**
  * enum resctrl_conf_type - The type of configuration.
  * @CDP_NONE:	No prioritisation, both code and data are controlled or monitored.
@@ -29,6 +32,16 @@ enum resctrl_conf_type {
 
 #define CDP_NUM_TYPES	(CDP_DATA + 1)
 
+/*
+ * Event IDs, the values match those used to program IA32_QM_EVTSEL before
+ * reading IA32_QM_CTR on RDT systems.
+ */
+enum resctrl_event_id {
+	QOS_L3_OCCUP_EVENT_ID		= 0x01,
+	QOS_L3_MBM_TOTAL_EVENT_ID	= 0x02,
+	QOS_L3_MBM_LOCAL_EVENT_ID	= 0x03,
+};
+
 /**
  * struct resctrl_staged_config - parsed configuration to be applied
  * @new_ctrl:		new ctrl value to be loaded
@@ -53,6 +66,9 @@ struct resctrl_staged_config {
  * @cqm_work_cpu:	worker CPU for CQM h/w counters
  * @plr:		pseudo-locked region (if any) associated with domain
  * @staged_config:	parsed configuration to be applied
+ * @mbps_val:		When mba_sc is enabled, this holds the array of user
+ *			specified control values for mba_sc in MBps, indexed
+ *			by closid
  */
 struct rdt_domain {
 	struct list_head		list;
@@ -67,6 +83,7 @@ struct rdt_domain {
 	int				cqm_work_cpu;
 	struct pseudo_lock_region	*plr;
 	struct resctrl_staged_config	staged_config[CDP_NUM_TYPES];
+	u32				*mbps_val;
 };
 
 /**
@@ -130,8 +147,6 @@ struct resctrl_schema;
 /**
  * struct rdt_resource - attributes of a resctrl resource
  * @rid:		The index of the resource
- * @alloc_enabled:	Is allocation enabled on this machine
- * @mon_enabled:	Is monitoring enabled for this feature
  * @alloc_capable:	Is allocation available on this machine
  * @mon_capable:	Is monitor feature available on this machine
  * @num_rmid:		Number of RMIDs available
@@ -150,8 +165,6 @@ struct resctrl_schema;
  */
 struct rdt_resource {
 	int			rid;
-	bool			alloc_enabled;
-	bool			mon_enabled;
 	bool			alloc_capable;
 	bool			mon_capable;
 	int			num_rmid;
@@ -194,7 +207,50 @@ struct resctrl_schema {
 /* The number of closid supported by this resource regardless of CDP */
 u32 resctrl_arch_get_num_closid(struct rdt_resource *r);
 int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid);
+
+/*
+ * Update the ctrl_val and apply this config right now.
+ * Must be called on one of the domain's CPUs.
+ */
+int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_domain *d,
+			    u32 closid, enum resctrl_conf_type t, u32 cfg_val);
+
 u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_domain *d,
 			    u32 closid, enum resctrl_conf_type type);
+int resctrl_online_domain(struct rdt_resource *r, struct rdt_domain *d);
+void resctrl_offline_domain(struct rdt_resource *r, struct rdt_domain *d);
+
+/**
+ * resctrl_arch_rmid_read() - Read the eventid counter corresponding to rmid
+ *			      for this resource and domain.
+ * @r:			resource that the counter should be read from.
+ * @d:			domain that the counter should be read from.
+ * @rmid:		rmid of the counter to read.
+ * @eventid:		eventid to read, e.g. L3 occupancy.
+ * @val:		result of the counter read in bytes.
+ *
+ * Call from process context on a CPU that belongs to domain @d.
+ *
+ * Return:
+ * 0 on success, or -EIO, -EINVAL etc on error.
+ */
+int resctrl_arch_rmid_read(struct rdt_resource *r, struct rdt_domain *d,
+			   u32 rmid, enum resctrl_event_id eventid, u64 *val);
+
+/**
+ * resctrl_arch_reset_rmid() - Reset any private state associated with rmid
+ *			       and eventid.
+ * @r:		The domain's resource.
+ * @d:		The rmid's domain.
+ * @rmid:	The rmid whose counter values should be reset.
+ * @eventid:	The eventid whose counter values should be reset.
+ *
+ * This can be called from any CPU.
+ */
+void resctrl_arch_reset_rmid(struct rdt_resource *r, struct rdt_domain *d,
+			     u32 rmid, enum resctrl_event_id eventid);
+
+extern unsigned int resctrl_rmid_realloc_threshold;
+extern unsigned int resctrl_rmid_realloc_limit;
 
 #endif /* _RESCTRL_H */
diff --git a/include/linux/security.h b/include/linux/security.h
index 7bd0c490703d..87fac3af6dad 100644
--- a/include/linux/security.h
+++ b/include/linux/security.h
@@ -437,6 +437,7 @@ int security_task_kill(struct task_struct *p, struct kernel_siginfo *info,
 int security_task_prctl(int option, unsigned long arg2, unsigned long arg3,
 			unsigned long arg4, unsigned long arg5);
 void security_task_to_inode(struct task_struct *p, struct inode *inode);
+int security_create_user_ns(const struct cred *cred);
 int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag);
 void security_ipc_getsecid(struct kern_ipc_perm *ipcp, u32 *secid);
 int security_msg_msg_alloc(struct msg_msg *msg);
@@ -461,7 +462,7 @@ int security_sem_semctl(struct kern_ipc_perm *sma, int cmd);
 int security_sem_semop(struct kern_ipc_perm *sma, struct sembuf *sops,
 			unsigned nsops, int alter);
 void security_d_instantiate(struct dentry *dentry, struct inode *inode);
-int security_getprocattr(struct task_struct *p, const char *lsm, char *name,
+int security_getprocattr(struct task_struct *p, const char *lsm, const char *name,
 			 char **value);
 int security_setprocattr(const char *lsm, const char *name, void *value,
 			 size_t size);
@@ -1194,6 +1195,11 @@ static inline int security_task_prctl(int option, unsigned long arg2,
 static inline void security_task_to_inode(struct task_struct *p, struct inode *inode)
 { }
 
+static inline int security_create_user_ns(const struct cred *cred)
+{
+	return 0;
+}
+
 static inline int security_ipc_permission(struct kern_ipc_perm *ipcp,
 					  short flag)
 {
@@ -1301,7 +1307,7 @@ static inline void security_d_instantiate(struct dentry *dentry,
 { }
 
 static inline int security_getprocattr(struct task_struct *p, const char *lsm,
-				       char *name, char **value)
+				       const char *name, char **value)
 {
 	return -EINVAL;
 }
diff --git a/include/linux/sfp.h b/include/linux/sfp.h
index 302094b855fb..d1f343853b6c 100644
--- a/include/linux/sfp.h
+++ b/include/linux/sfp.h
@@ -535,7 +535,7 @@ int sfp_parse_port(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
 		   unsigned long *support);
 bool sfp_may_have_phy(struct sfp_bus *bus, const struct sfp_eeprom_id *id);
 void sfp_parse_support(struct sfp_bus *bus, const struct sfp_eeprom_id *id,
-		       unsigned long *support);
+		       unsigned long *support, unsigned long *interfaces);
 phy_interface_t sfp_select_interface(struct sfp_bus *bus,
 				     unsigned long *link_modes);
 
@@ -568,7 +568,8 @@ static inline bool sfp_may_have_phy(struct sfp_bus *bus,
 
 static inline void sfp_parse_support(struct sfp_bus *bus,
 				     const struct sfp_eeprom_id *id,
-				     unsigned long *support)
+				     unsigned long *support,
+				     unsigned long *interfaces)
 {
 }
 
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 18e163a3460d..9fcf534f2d92 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -533,6 +533,13 @@ enum {
 struct ubuf_info {
 	void (*callback)(struct sk_buff *, struct ubuf_info *,
 			 bool zerocopy_success);
+	refcount_t refcnt;
+	u8 flags;
+};
+
+struct ubuf_info_msgzc {
+	struct ubuf_info ubuf;
+
 	union {
 		struct {
 			unsigned long desc;
@@ -545,8 +552,6 @@ struct ubuf_info {
 			u32 bytelen;
 		};
 	};
-	refcount_t refcnt;
-	u8 flags;
 
 	struct mmpin {
 		struct user_struct *user;
@@ -555,6 +560,8 @@ struct ubuf_info {
 };
 
 #define skb_uarg(SKB)	((struct ubuf_info *)(skb_shinfo(SKB)->destructor_arg))
+#define uarg_to_msgzc(ubuf_ptr)	container_of((ubuf_ptr), struct ubuf_info_msgzc, \
+					     ubuf)
 
 int mm_account_pinned_pages(struct mmpin *mmp, size_t size);
 void mm_unaccount_pinned_pages(struct mmpin *mmp);
@@ -1195,7 +1202,8 @@ static inline bool skb_unref(struct sk_buff *skb)
 	return true;
 }
 
-void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason);
+void __fix_address
+kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason);
 
 /**
  *	kfree_skb - free an sk_buff with 'NOT_SPECIFIED' reason
@@ -1460,8 +1468,8 @@ void skb_flow_dissector_init(struct flow_dissector *flow_dissector,
 			     unsigned int key_count);
 
 struct bpf_flow_dissector;
-bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
-		      __be16 proto, int nhoff, int hlen, unsigned int flags);
+u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
+		     __be16 proto, int nhoff, int hlen, unsigned int flags);
 
 bool __skb_flow_dissect(const struct net *net,
 			const struct sk_buff *skb,
@@ -2608,20 +2616,6 @@ void *skb_pull_data(struct sk_buff *skb, size_t len);
 
 void *__pskb_pull_tail(struct sk_buff *skb, int delta);
 
-static inline void *__pskb_pull(struct sk_buff *skb, unsigned int len)
-{
-	if (len > skb_headlen(skb) &&
-	    !__pskb_pull_tail(skb, len - skb_headlen(skb)))
-		return NULL;
-	skb->len -= len;
-	return skb->data += len;
-}
-
-static inline void *pskb_pull(struct sk_buff *skb, unsigned int len)
-{
-	return unlikely(len > skb->len) ? NULL : __pskb_pull(skb, len);
-}
-
 static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len)
 {
 	if (likely(len <= skb_headlen(skb)))
@@ -2631,6 +2625,15 @@ static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len)
 	return __pskb_pull_tail(skb, len - skb_headlen(skb)) != NULL;
 }
 
+static inline void *pskb_pull(struct sk_buff *skb, unsigned int len)
+{
+	if (!pskb_may_pull(skb, len))
+		return NULL;
+
+	skb->len -= len;
+	return skb->data += len;
+}
+
 void skb_condense(struct sk_buff *skb);
 
 /**
diff --git a/include/linux/soc/apple/rtkit.h b/include/linux/soc/apple/rtkit.h
index 88eb832eac7b..c9cabb679cd1 100644
--- a/include/linux/soc/apple/rtkit.h
+++ b/include/linux/soc/apple/rtkit.h
@@ -152,4 +152,16 @@ int apple_rtkit_send_message(struct apple_rtkit *rtk, u8 ep, u64 message,
 int apple_rtkit_send_message_wait(struct apple_rtkit *rtk, u8 ep, u64 message,
 				  unsigned long timeout, bool atomic);
 
+/*
+ * Process incoming messages in atomic context.
+ * This only guarantees that messages arrive as far as the recv_message_early
+ * callback; drivers expecting to handle incoming messages synchronously
+ * by calling this function must do it that way.
+ * Will return 1 if some data was processed, 0 if none was, or a
+ * negative error code on failure.
+ *
+ * @rtk:            RTKit reference
+ */
+int apple_rtkit_poll(struct apple_rtkit *rtk);
+
 #endif /* _LINUX_APPLE_RTKIT_H_ */
diff --git a/include/linux/soc/mediatek/mtk-mmsys.h b/include/linux/soc/mediatek/mtk-mmsys.h
index 59117d970daf..d2b02bb43768 100644
--- a/include/linux/soc/mediatek/mtk-mmsys.h
+++ b/include/linux/soc/mediatek/mtk-mmsys.h
@@ -65,4 +65,6 @@ void mtk_mmsys_ddp_disconnect(struct device *dev,
 			      enum mtk_ddp_comp_id cur,
 			      enum mtk_ddp_comp_id next);
 
+void mtk_mmsys_ddp_dpi_fmt_config(struct device *dev, u32 val);
+
 #endif /* __MTK_MMSYS_H */
diff --git a/include/linux/soc/mediatek/mtk-mutex.h b/include/linux/soc/mediatek/mtk-mutex.h
index a0f4f51a3b45..b335c2837cd8 100644
--- a/include/linux/soc/mediatek/mtk-mutex.h
+++ b/include/linux/soc/mediatek/mtk-mutex.h
@@ -20,6 +20,8 @@ enum mtk_mutex_mod_index {
 	MUTEX_MOD_IDX_MDP_WDMA,
 	MUTEX_MOD_IDX_MDP_AAL0,
 	MUTEX_MOD_IDX_MDP_CCORR0,
+	MUTEX_MOD_IDX_MDP_HDR0,
+	MUTEX_MOD_IDX_MDP_COLOR0,
 
 	MUTEX_MOD_IDX_MAX		/* ALWAYS keep at the end */
 };
diff --git a/include/linux/soc/mediatek/mtk_sip_svc.h b/include/linux/soc/mediatek/mtk_sip_svc.h
index 082398e0cfb1..0761128b4354 100644
--- a/include/linux/soc/mediatek/mtk_sip_svc.h
+++ b/include/linux/soc/mediatek/mtk_sip_svc.h
@@ -22,4 +22,7 @@
 	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, MTK_SIP_SMC_CONVENTION, \
 			   ARM_SMCCC_OWNER_SIP, fn_id)
 
+/* IOMMU related SMC call */
+#define MTK_SIP_KERNEL_IOMMU_CONTROL	MTK_SIP_SMC_CMD(0x514)
+
 #endif
diff --git a/include/linux/soc/mediatek/mtk_wed.h b/include/linux/soc/mediatek/mtk_wed.h
index 7e00cca06709..4450c8b7a1cb 100644
--- a/include/linux/soc/mediatek/mtk_wed.h
+++ b/include/linux/soc/mediatek/mtk_wed.h
@@ -11,9 +11,15 @@
 struct mtk_wed_hw;
 struct mtk_wdma_desc;
 
+enum mtk_wed_bus_tye {
+	MTK_WED_BUS_PCIE,
+	MTK_WED_BUS_AXI,
+};
+
 struct mtk_wed_ring {
 	struct mtk_wdma_desc *desc;
 	dma_addr_t desc_phys;
+	u32 desc_size;
 	int size;
 
 	u32 reg_base;
@@ -42,13 +48,24 @@ struct mtk_wed_device {
 
 	/* filled by driver: */
 	struct {
-		struct pci_dev *pci_dev;
+		union {
+			struct platform_device *platform_dev;
+			struct pci_dev *pci_dev;
+		};
+		enum mtk_wed_bus_tye bus_type;
 
 		u32 wpdma_phys;
+		u32 wpdma_int;
+		u32 wpdma_mask;
+		u32 wpdma_tx;
+		u32 wpdma_txfree;
 
 		u16 token_start;
 		unsigned int nbuf;
 
+		u8 tx_tbit[MTK_WED_TX_QUEUES];
+		u8 txfree_tbit;
+
 		u32 (*init_buf)(void *ptr, dma_addr_t phys, int token_id);
 		int (*offload_enable)(struct mtk_wed_device *wed);
 		void (*offload_disable)(struct mtk_wed_device *wed);
diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h
index 9ed5384c5ca1..bc2fb8343a94 100644
--- a/include/linux/soc/qcom/llcc-qcom.h
+++ b/include/linux/soc/qcom/llcc-qcom.h
@@ -78,11 +78,40 @@ struct llcc_edac_reg_data {
 	u8  ways_shift;
 };
 
+struct llcc_edac_reg_offset {
+	/* LLCC TRP registers */
+	u32 trp_ecc_error_status0;
+	u32 trp_ecc_error_status1;
+	u32 trp_ecc_sb_err_syn0;
+	u32 trp_ecc_db_err_syn0;
+	u32 trp_ecc_error_cntr_clear;
+	u32 trp_interrupt_0_status;
+	u32 trp_interrupt_0_clear;
+	u32 trp_interrupt_0_enable;
+
+	/* LLCC Common registers */
+	u32 cmn_status0;
+	u32 cmn_interrupt_0_enable;
+	u32 cmn_interrupt_2_enable;
+
+	/* LLCC DRP registers */
+	u32 drp_ecc_error_cfg;
+	u32 drp_ecc_error_cntr_clear;
+	u32 drp_interrupt_status;
+	u32 drp_interrupt_clear;
+	u32 drp_interrupt_enable;
+	u32 drp_ecc_error_status0;
+	u32 drp_ecc_error_status1;
+	u32 drp_ecc_sb_err_syn0;
+	u32 drp_ecc_db_err_syn0;
+};
+
 /**
  * struct llcc_drv_data - Data associated with the llcc driver
  * @regmap: regmap associated with the llcc device
  * @bcast_regmap: regmap associated with llcc broadcast offset
  * @cfg: pointer to the data structure for slice configuration
+ * @edac_reg_offset: Offset of the LLCC EDAC registers
  * @lock: mutex associated with each slice
  * @cfg_size: size of the config data table
  * @max_slices: max slices as read from device tree
@@ -96,6 +125,7 @@ struct llcc_drv_data {
 	struct regmap *regmap;
 	struct regmap *bcast_regmap;
 	const struct llcc_slice_config *cfg;
+	const struct llcc_edac_reg_offset *edac_reg_offset;
 	struct mutex lock;
 	u32 cfg_size;
 	u32 max_slices;
diff --git a/include/linux/soc/qcom/qmi.h b/include/linux/soc/qcom/qmi.h
index b1f80e756d2a..469e02d2aa0d 100644
--- a/include/linux/soc/qcom/qmi.h
+++ b/include/linux/soc/qcom/qmi.h
@@ -75,7 +75,7 @@ struct qmi_elem_info {
 	enum qmi_array_type array_type;
 	u8 tlv_type;
 	u32 offset;
-	struct qmi_elem_info *ei_array;
+	const struct qmi_elem_info *ei_array;
 };
 
 #define QMI_RESULT_SUCCESS_V01			0
@@ -102,7 +102,7 @@ struct qmi_response_type_v01 {
 	u16 error;
 };
 
-extern struct qmi_elem_info qmi_response_type_v01_ei[];
+extern const struct qmi_elem_info qmi_response_type_v01_ei[];
 
 /**
  * struct qmi_service - context to track lookup-results
@@ -173,7 +173,7 @@ struct qmi_txn {
 	struct completion completion;
 	int result;
 
-	struct qmi_elem_info *ei;
+	const struct qmi_elem_info *ei;
 	void *dest;
 };
 
@@ -189,7 +189,7 @@ struct qmi_msg_handler {
 	unsigned int type;
 	unsigned int msg_id;
 
-	struct qmi_elem_info *ei;
+	const struct qmi_elem_info *ei;
 
 	size_t decoded_size;
 	void (*fn)(struct qmi_handle *qmi, struct sockaddr_qrtr *sq,
@@ -249,23 +249,23 @@ void qmi_handle_release(struct qmi_handle *qmi);
 
 ssize_t qmi_send_request(struct qmi_handle *qmi, struct sockaddr_qrtr *sq,
 			 struct qmi_txn *txn, int msg_id, size_t len,
-			 struct qmi_elem_info *ei, const void *c_struct);
+			 const struct qmi_elem_info *ei, const void *c_struct);
 ssize_t qmi_send_response(struct qmi_handle *qmi, struct sockaddr_qrtr *sq,
 			  struct qmi_txn *txn, int msg_id, size_t len,
-			  struct qmi_elem_info *ei, const void *c_struct);
+			  const struct qmi_elem_info *ei, const void *c_struct);
 ssize_t qmi_send_indication(struct qmi_handle *qmi, struct sockaddr_qrtr *sq,
-			    int msg_id, size_t len, struct qmi_elem_info *ei,
+			    int msg_id, size_t len, const struct qmi_elem_info *ei,
 			    const void *c_struct);
 
 void *qmi_encode_message(int type, unsigned int msg_id, size_t *len,
-			 unsigned int txn_id, struct qmi_elem_info *ei,
+			 unsigned int txn_id, const struct qmi_elem_info *ei,
 			 const void *c_struct);
 
 int qmi_decode_message(const void *buf, size_t len,
-		       struct qmi_elem_info *ei, void *c_struct);
+		       const struct qmi_elem_info *ei, void *c_struct);
 
 int qmi_txn_init(struct qmi_handle *qmi, struct qmi_txn *txn,
-		 struct qmi_elem_info *ei, void *c_struct);
+		 const struct qmi_elem_info *ei, void *c_struct);
 int qmi_txn_wait(struct qmi_txn *txn, unsigned long timeout);
 void qmi_txn_cancel(struct qmi_txn *txn);
 
diff --git a/include/linux/soc/sunxi/sunxi_sram.h b/include/linux/soc/sunxi/sunxi_sram.h
index c5f663bba9c2..60e274d1b821 100644
--- a/include/linux/soc/sunxi/sunxi_sram.h
+++ b/include/linux/soc/sunxi/sunxi_sram.h
@@ -14,6 +14,6 @@
 #define _SUNXI_SRAM_H_
 
 int sunxi_sram_claim(struct device *dev);
-int sunxi_sram_release(struct device *dev);
+void sunxi_sram_release(struct device *dev);
 
 #endif /* _SUNXI_SRAM_H_ */
diff --git a/include/linux/sockptr.h b/include/linux/sockptr.h
index d45902fb4cad..bae5e2369b4f 100644
--- a/include/linux/sockptr.h
+++ b/include/linux/sockptr.h
@@ -64,6 +64,11 @@ static inline int copy_to_sockptr_offset(sockptr_t dst, size_t offset,
 	return 0;
 }
 
+static inline int copy_to_sockptr(sockptr_t dst, const void *src, size_t size)
+{
+	return copy_to_sockptr_offset(dst, 0, src, size);
+}
+
 static inline void *memdup_sockptr(sockptr_t src, size_t len)
 {
 	void *p = kmalloc_track_caller(len, GFP_USER | __GFP_NOWARN);
diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h
index 39058c841469..822599957b35 100644
--- a/include/linux/soundwire/sdw.h
+++ b/include/linux/soundwire/sdw.h
@@ -839,6 +839,8 @@ struct sdw_defer {
  * @set_bus_conf: Set the bus configuration
  * @pre_bank_switch: Callback for pre bank switch
  * @post_bank_switch: Callback for post bank switch
+ * @read_ping_status: Read status from PING frames, reported with two bits per Device.
+ * Bits 31:24 are reserved.
  */
 struct sdw_master_ops {
 	int (*read_prop)(struct sdw_bus *bus);
@@ -855,6 +857,7 @@ struct sdw_master_ops {
 			struct sdw_bus_params *params);
 	int (*pre_bank_switch)(struct sdw_bus *bus);
 	int (*post_bank_switch)(struct sdw_bus *bus);
+	u32 (*read_ping_status)(struct sdw_bus *bus);
 
 };
 
@@ -919,6 +922,8 @@ int sdw_bus_master_add(struct sdw_bus *bus, struct device *parent,
 		       struct fwnode_handle *fwnode);
 void sdw_bus_master_delete(struct sdw_bus *bus);
 
+void sdw_show_ping_status(struct sdw_bus *bus, bool sync_delay);
+
 /**
  * sdw_port_config: Master or Slave Port configuration
  *
diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h
index f089ee1ead58..fbf8c0d95968 100644
--- a/include/linux/spi/spi.h
+++ b/include/linux/spi/spi.h
@@ -378,6 +378,8 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch
  * @cleanup: frees controller-specific state
  * @can_dma: determine whether this controller supports DMA
  * @dma_map_dev: device which can be used for DMA mapping
+ * @cur_rx_dma_dev: device which is currently used for RX DMA mapping
+ * @cur_tx_dma_dev: device which is currently used for TX DMA mapping
  * @queued: whether this controller is providing an internal message queue
  * @kworker: pointer to thread struct for message pump
  * @pump_messages: work struct for scheduling work to the message pump
@@ -610,6 +612,8 @@ struct spi_controller {
 					   struct spi_device *spi,
 					   struct spi_transfer *xfer);
 	struct device *dma_map_dev;
+	struct device *cur_rx_dma_dev;
+	struct device *cur_tx_dma_dev;
 
 	/*
 	 * These hooks are for drivers that want to use the generic
@@ -848,6 +852,7 @@ struct spi_res {
  * @bits_per_word: select a bits_per_word other than the device default
  *      for this transfer. If 0 the default (from @spi_device) is used.
  * @dummy_data: indicates transfer is dummy bytes transfer.
+ * @cs_off: performs the transfer with chipselect off.
  * @cs_change: affects chipselect after this transfer completes
  * @cs_change_delay: delay between cs deassert and assert when
  *      @cs_change is set and @spi_transfer is not the last in @spi_message
@@ -958,6 +963,7 @@ struct spi_transfer {
 	struct sg_table rx_sg;
 
 	unsigned	dummy_data:1;
+	unsigned	cs_off:1;
 	unsigned	cs_change:1;
 	unsigned	tx_nbits:3;
 	unsigned	rx_nbits:3;
diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h
index 6cfaa0a9a9b9..5aa5e0faf6a1 100644
--- a/include/linux/srcutiny.h
+++ b/include/linux/srcutiny.h
@@ -15,10 +15,10 @@
 
 struct srcu_struct {
 	short srcu_lock_nesting[2];	/* srcu_read_lock() nesting depth. */
-	unsigned short srcu_idx;	/* Current reader array element in bit 0x2. */
-	unsigned short srcu_idx_max;	/* Furthest future srcu_idx request. */
 	u8 srcu_gp_running;		/* GP workqueue running? */
 	u8 srcu_gp_waiting;		/* GP waiting for readers? */
+	unsigned long srcu_idx;		/* Current reader array element in bit 0x2. */
+	unsigned long srcu_idx_max;	/* Furthest future srcu_idx request. */
 	struct swait_queue_head srcu_wq;
 					/* Last srcu_read_unlock() wakes GP. */
 	struct rcu_head *srcu_cb_head;	/* Pending callbacks: Head. */
@@ -82,10 +82,12 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
 	int idx;
 
 	idx = ((data_race(READ_ONCE(ssp->srcu_idx)) + 1) & 0x2) >> 1;
-	pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n",
+	pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd) gp: %lu->%lu\n",
 		 tt, tf, idx,
 		 data_race(READ_ONCE(ssp->srcu_lock_nesting[!idx])),
-		 data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])));
+		 data_race(READ_ONCE(ssp->srcu_lock_nesting[idx])),
+		 data_race(READ_ONCE(ssp->srcu_idx)),
+		 data_race(READ_ONCE(ssp->srcu_idx_max)));
 }
 
 #endif
diff --git a/include/linux/stat.h b/include/linux/stat.h
index 7df06931f25d..ff277ced50e9 100644
--- a/include/linux/stat.h
+++ b/include/linux/stat.h
@@ -50,6 +50,8 @@ struct kstat {
 	struct timespec64 btime;			/* File creation time */
 	u64		blocks;
 	u64		mnt_id;
+	u32		dio_mem_align;
+	u32		dio_offset_align;
 };
 
 #endif
diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h
index 8df475db88c0..fb2e88614f5d 100644
--- a/include/linux/stmmac.h
+++ b/include/linux/stmmac.h
@@ -257,7 +257,6 @@ struct plat_stmmacenet_data {
 	u8 vlan_fail_q;
 	unsigned int eee_usecs_rate;
 	struct pci_dev *pdev;
-	bool has_crossts;
 	int int_snapshot_num;
 	int ext_snapshot_num;
 	bool int_snapshot_en;
diff --git a/include/linux/string.h b/include/linux/string.h
index 61ec7e4f6311..cf7607b32102 100644
--- a/include/linux/string.h
+++ b/include/linux/string.h
@@ -261,6 +261,49 @@ void memcpy_and_pad(void *dest, size_t dest_len, const void *src, size_t count,
 		    int pad);
 
 /**
+ * strtomem_pad - Copy NUL-terminated string to non-NUL-terminated buffer
+ *
+ * @dest: Pointer of destination character array (marked as __nonstring)
+ * @src: Pointer to NUL-terminated string
+ * @pad: Padding character to fill any remaining bytes of @dest after copy
+ *
+ * This is a replacement for strncpy() uses where the destination is not
+ * a NUL-terminated string, but with bounds checking on the source size, and
+ * an explicit padding character. If padding is not required, use strtomem().
+ *
+ * Note that the size of @dest is not an argument, as the length of @dest
+ * must be discoverable by the compiler.
+ */
+#define strtomem_pad(dest, src, pad)	do {				\
+	const size_t _dest_len = __builtin_object_size(dest, 1);	\
+									\
+	BUILD_BUG_ON(!__builtin_constant_p(_dest_len) ||		\
+		     _dest_len == (size_t)-1);				\
+	memcpy_and_pad(dest, _dest_len, src, strnlen(src, _dest_len), pad); \
+} while (0)
+
+/**
+ * strtomem - Copy NUL-terminated string to non-NUL-terminated buffer
+ *
+ * @dest: Pointer of destination character array (marked as __nonstring)
+ * @src: Pointer to NUL-terminated string
+ *
+ * This is a replacement for strncpy() uses where the destination is not
+ * a NUL-terminated string, but with bounds checking on the source size, and
+ * without trailing padding. If padding is required, use strtomem_pad().
+ *
+ * Note that the size of @dest is not an argument, as the length of @dest
+ * must be discoverable by the compiler.
+ */
+#define strtomem(dest, src)	do {					\
+	const size_t _dest_len = __builtin_object_size(dest, 1);	\
+									\
+	BUILD_BUG_ON(!__builtin_constant_p(_dest_len) ||		\
+		     _dest_len == (size_t)-1);				\
+	memcpy(dest, src, min(_dest_len, strnlen(src, _dest_len)));	\
+} while (0)
+
+/**
  * memset_after - Set a value after a struct member to the end of a struct
  *
  * @obj: Address of target struct instance
diff --git a/include/linux/string_helpers.h b/include/linux/string_helpers.h
index 4d72258d42fd..dc2e726fd820 100644
--- a/include/linux/string_helpers.h
+++ b/include/linux/string_helpers.h
@@ -21,6 +21,8 @@ enum string_size_units {
 void string_get_size(u64 size, u64 blk_size, enum string_size_units units,
 		     char *buf, int len);
 
+int parse_int_array_user(const char __user *from, size_t count, int **array);
+
 #define UNESCAPE_SPACE		BIT(0)
 #define UNESCAPE_OCTAL		BIT(1)
 #define UNESCAPE_HEX		BIT(2)
diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h
index daecb009c05b..88de45491376 100644
--- a/include/linux/sunrpc/svc.h
+++ b/include/linux/sunrpc/svc.h
@@ -472,6 +472,7 @@ struct svc_procedure {
 	/* XDR free result: */
 	void			(*pc_release)(struct svc_rqst *);
 	unsigned int		pc_argsize;	/* argument struct size */
+	unsigned int		pc_argzero;	/* how much of argument to clear */
 	unsigned int		pc_ressize;	/* result struct size */
 	unsigned int		pc_cachetype;	/* cache info (NFS) */
 	unsigned int		pc_xdrressize;	/* maximum size of XDR reply */
@@ -544,16 +545,27 @@ static inline void svc_reserve_auth(struct svc_rqst *rqstp, int space)
 }
 
 /**
- * svcxdr_init_decode - Prepare an xdr_stream for svc Call decoding
+ * svcxdr_init_decode - Prepare an xdr_stream for Call decoding
  * @rqstp: controlling server RPC transaction context
  *
+ * This function currently assumes the RPC header in rq_arg has
+ * already been decoded. Upon return, xdr->p points to the
+ * location of the upper layer header.
  */
 static inline void svcxdr_init_decode(struct svc_rqst *rqstp)
 {
 	struct xdr_stream *xdr = &rqstp->rq_arg_stream;
-	struct kvec *argv = rqstp->rq_arg.head;
+	struct xdr_buf *buf = &rqstp->rq_arg;
+	struct kvec *argv = buf->head;
 
-	xdr_init_decode(xdr, &rqstp->rq_arg, argv->iov_base, NULL);
+	/*
+	 * svc_getnl() and friends do not keep the xdr_buf's ::len
+	 * field up to date. Refresh that field before initializing
+	 * the argument decoding stream.
+	 */
+	buf->len = buf->head->iov_len + buf->page_len + buf->tail->iov_len;
+
+	xdr_init_decode(xdr, buf, argv->iov_base, NULL);
 	xdr_set_scratch_page(xdr, rqstp->rq_scratch_page);
 }
 
@@ -576,7 +588,7 @@ static inline void svcxdr_init_encode(struct svc_rqst *rqstp)
 	xdr->end = resv->iov_base + PAGE_SIZE - rqstp->rq_auth_slack;
 	buf->len = resv->iov_len;
 	xdr->page_ptr = buf->pages - 1;
-	buf->buflen = PAGE_SIZE * (1 + rqstp->rq_page_end - buf->pages);
+	buf->buflen = PAGE_SIZE * (rqstp->rq_page_end - buf->pages);
 	buf->buflen -= rqstp->rq_auth_slack;
 	xdr->rqst = NULL;
 }
diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h
index 69175029abbb..f84e2a1358e1 100644
--- a/include/linux/sunrpc/xdr.h
+++ b/include/linux/sunrpc/xdr.h
@@ -240,6 +240,8 @@ typedef int	(*kxdrdproc_t)(struct rpc_rqst *rqstp, struct xdr_stream *xdr,
 
 extern void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf,
 			    __be32 *p, struct rpc_rqst *rqst);
+extern void xdr_init_encode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,
+			   struct page **pages, struct rpc_rqst *rqst);
 extern __be32 *xdr_reserve_space(struct xdr_stream *xdr, size_t nbytes);
 extern int xdr_reserve_space_vec(struct xdr_stream *xdr, struct kvec *vec,
 		size_t nbytes);
diff --git a/include/linux/suspend.h b/include/linux/suspend.h
index 70f2921e2e70..be7737262d8f 100644
--- a/include/linux/suspend.h
+++ b/include/linux/suspend.h
@@ -75,7 +75,7 @@ extern struct suspend_stats suspend_stats;
 
 static inline void dpm_save_failed_dev(const char *name)
 {
-	strlcpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev],
+	strscpy(suspend_stats.failed_devs[suspend_stats.last_failed_dev],
 		name,
 		sizeof(suspend_stats.failed_devs[0]));
 	suspend_stats.last_failed_dev++;
@@ -191,6 +191,7 @@ struct platform_s2idle_ops {
 	int (*begin)(void);
 	int (*prepare)(void);
 	int (*prepare_late)(void);
+	void (*check)(void);
 	bool (*wake)(void);
 	void (*restore_early)(void);
 	void (*restore)(void);
diff --git a/include/linux/swab.h b/include/linux/swab.h
index bcff5149861a..9b804dbb0d79 100644
--- a/include/linux/swab.h
+++ b/include/linux/swab.h
@@ -20,4 +20,29 @@
 # define swab64s __swab64s
 # define swahw32s __swahw32s
 # define swahb32s __swahb32s
+
+static inline void swab16_array(u16 *buf, unsigned int words)
+{
+	while (words--) {
+		swab16s(buf);
+		buf++;
+	}
+}
+
+static inline void swab32_array(u32 *buf, unsigned int words)
+{
+	while (words--) {
+		swab32s(buf);
+		buf++;
+	}
+}
+
+static inline void swab64_array(u64 *buf, unsigned int words)
+{
+	while (words--) {
+		swab64s(buf);
+		buf++;
+	}
+}
+
 #endif /* _LINUX_SWAB_H */
diff --git a/include/linux/tcp.h b/include/linux/tcp.h
index a9fbe22732c3..41b1da621a45 100644
--- a/include/linux/tcp.h
+++ b/include/linux/tcp.h
@@ -295,7 +295,7 @@ struct tcp_sock {
 	u32	packets_out;	/* Packets which are "in flight"	*/
 	u32	retrans_out;	/* Retransmitted packets out		*/
 	u32	max_packets_out;  /* max packets_out in last window */
-	u32	max_packets_seq;  /* right edge of max_packets_out flight */
+	u32	cwnd_usage_seq;  /* right edge of cwnd usage tracking flight */
 
 	u16	urg_data;	/* Saved octet of OOB data and control flags */
 	u8	ecn_flags;	/* ECN status bits.			*/
@@ -388,6 +388,12 @@ struct tcp_sock {
 	u8	bpf_sock_ops_cb_flags;  /* Control calling BPF programs
 					 * values defined in uapi/linux/tcp.h
 					 */
+	u8	bpf_chg_cc_inprogress:1; /* In the middle of
+					  * bpf_setsockopt(TCP_CONGESTION),
+					  * it is to avoid the bpf_tcp_cc->init()
+					  * to recur itself by calling
+					  * bpf_setsockopt(TCP_CONGESTION, "itself").
+					  */
 #define BPF_SOCK_OPS_TEST_FLAG(TP, ARG) (TP->bpf_sock_ops_cb_flags & ARG)
 #else
 #define BPF_SOCK_OPS_TEST_FLAG(TP, ARG) 0
diff --git a/include/linux/thermal.h b/include/linux/thermal.h
index 1386c713885d..6f1ec4fb7ef8 100644
--- a/include/linux/thermal.h
+++ b/include/linux/thermal.h
@@ -17,8 +17,6 @@
 #include <linux/workqueue.h>
 #include <uapi/linux/thermal.h>
 
-#define THERMAL_MAX_TRIPS	12
-
 /* invalid cooling state */
 #define THERMAL_CSTATE_INVALID -1UL
 
@@ -296,82 +294,53 @@ struct thermal_zone_params {
 	int offset;
 };
 
-/**
- * struct thermal_zone_of_device_ops - callbacks for handling DT based zones
- *
- * Mandatory:
- * @get_temp: a pointer to a function that reads the sensor temperature.
- *
- * Optional:
- * @get_trend: a pointer to a function that reads the sensor temperature trend.
- * @set_trips: a pointer to a function that sets a temperature window. When
- *	       this window is left the driver must inform the thermal core via
- *	       thermal_zone_device_update.
- * @set_emul_temp: a pointer to a function that sets sensor emulated
- *		   temperature.
- * @set_trip_temp: a pointer to a function that sets the trip temperature on
- *		   hardware.
- * @change_mode: a pointer to a function that notifies the thermal zone
- *		   mode change.
- */
-struct thermal_zone_of_device_ops {
-	int (*get_temp)(void *, int *);
-	int (*get_trend)(void *, int, enum thermal_trend *);
-	int (*set_trips)(void *, int, int);
-	int (*set_emul_temp)(void *, int);
-	int (*set_trip_temp)(void *, int, int);
-	int (*change_mode) (void *, enum thermal_device_mode);
-};
-
 /* Function declarations */
 #ifdef CONFIG_THERMAL_OF
+struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data,
+						     const struct thermal_zone_device_ops *ops);
+
+struct thermal_zone_device *devm_thermal_of_zone_register(struct device *dev, int id, void *data,
+							  const struct thermal_zone_device_ops *ops);
+
+void thermal_of_zone_unregister(struct thermal_zone_device *tz);
+
+void devm_thermal_of_zone_unregister(struct device *dev, struct thermal_zone_device *tz);
+
+void thermal_of_zone_unregister(struct thermal_zone_device *tz);
+
 int thermal_zone_of_get_sensor_id(struct device_node *tz_np,
 				  struct device_node *sensor_np,
 				  u32 *id);
-struct thermal_zone_device *
-thermal_zone_of_sensor_register(struct device *dev, int id, void *data,
-				const struct thermal_zone_of_device_ops *ops);
-void thermal_zone_of_sensor_unregister(struct device *dev,
-				       struct thermal_zone_device *tz);
-struct thermal_zone_device *devm_thermal_zone_of_sensor_register(
-		struct device *dev, int id, void *data,
-		const struct thermal_zone_of_device_ops *ops);
-void devm_thermal_zone_of_sensor_unregister(struct device *dev,
-					    struct thermal_zone_device *tz);
 #else
-
-static inline int thermal_zone_of_get_sensor_id(struct device_node *tz_np,
-					 struct device_node *sensor_np,
-					 u32 *id)
-{
-	return -ENOENT;
-}
-static inline struct thermal_zone_device *
-thermal_zone_of_sensor_register(struct device *dev, int id, void *data,
-				const struct thermal_zone_of_device_ops *ops)
+static inline
+struct thermal_zone_device *thermal_of_zone_register(struct device_node *sensor, int id, void *data,
+						     const struct thermal_zone_device_ops *ops)
 {
-	return ERR_PTR(-ENODEV);
+	return ERR_PTR(-ENOTSUPP);
 }
 
 static inline
-void thermal_zone_of_sensor_unregister(struct device *dev,
-				       struct thermal_zone_device *tz)
+struct thermal_zone_device *devm_thermal_of_zone_register(struct device *dev, int id, void *data,
+							  const struct thermal_zone_device_ops *ops)
 {
+	return ERR_PTR(-ENOTSUPP);
 }
 
-static inline struct thermal_zone_device *devm_thermal_zone_of_sensor_register(
-		struct device *dev, int id, void *data,
-		const struct thermal_zone_of_device_ops *ops)
+static inline void thermal_of_zone_unregister(struct thermal_zone_device *tz)
 {
-	return ERR_PTR(-ENODEV);
 }
 
-static inline
-void devm_thermal_zone_of_sensor_unregister(struct device *dev,
-					    struct thermal_zone_device *tz)
+static inline void devm_thermal_of_zone_unregister(struct device *dev,
+						   struct thermal_zone_device *tz)
 {
 }
 
+static inline int thermal_zone_of_get_sensor_id(struct device_node *tz_np,
+						struct device_node *sensor_np,
+						u32 *id)
+{
+	return -ENOENT;
+}
 #endif
 
 #ifdef CONFIG_THERMAL
diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h
index 9f442d73f3df..90cd08ab2f5d 100644
--- a/include/linux/thunderbolt.h
+++ b/include/linux/thunderbolt.h
@@ -187,6 +187,7 @@ void tb_unregister_property_dir(const char *key, struct tb_property_dir *dir);
  * @device_name: Name of the device (or %NULL if not known)
  * @link_speed: Speed of the link in Gb/s
  * @link_width: Width of the link (1 or 2)
+ * @link_usb4: Downstream link is USB4
  * @is_unplugged: The XDomain is unplugged
  * @needs_uuid: If the XDomain does not have @remote_uuid it will be
  *		queried first
@@ -234,6 +235,7 @@ struct tb_xdomain {
 	const char *device_name;
 	unsigned int link_speed;
 	unsigned int link_width;
+	bool link_usb4;
 	bool is_unplugged;
 	bool needs_uuid;
 	struct ida service_ids;
diff --git a/include/linux/tnum.h b/include/linux/tnum.h
index 498dbcedb451..1c3948a1d6ad 100644
--- a/include/linux/tnum.h
+++ b/include/linux/tnum.h
@@ -21,7 +21,12 @@ struct tnum {
 struct tnum tnum_const(u64 value);
 /* A completely unknown value */
 extern const struct tnum tnum_unknown;
-/* A value that's unknown except that @min <= value <= @max */
+/* An unknown value that is a superset of @min <= value <= @max.
+ *
+ * Could include values outside the range of [@min, @max].
+ * For example tnum_range(0, 2) is represented by {0, 1, 2, *3*},
+ * rather than the intended set of {0, 1, 2}.
+ */
 struct tnum tnum_range(u64 min, u64 max);
 
 /* Arithmetic and logical ops */
@@ -73,7 +78,18 @@ static inline bool tnum_is_unknown(struct tnum a)
  */
 bool tnum_is_aligned(struct tnum a, u64 size);
 
-/* Returns true if @b represents a subset of @a. */
+/* Returns true if @b represents a subset of @a.
+ *
+ * Note that using tnum_range() as @a requires extra cautions as tnum_in() may
+ * return true unexpectedly due to tnum limited ability to represent tight
+ * range, e.g.
+ *
+ *   tnum_in(tnum_range(0, 2), tnum_const(3)) == true
+ *
+ * As a rule of thumb, if @a is explicitly coded rather than coming from
+ * reg->var_off, it should be in form of tnum_const(), tnum_range(0, 2**n - 1),
+ * or tnum_range(2**n, 2**(n+1) - 1).
+ */
 bool tnum_in(struct tnum a, struct tnum b);
 
 /* Formatting functions.  These have snprintf-like semantics: they will write
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 5896af36199c..2e3134b14ffd 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -298,7 +298,7 @@ iov_iter_npages_cap(struct iov_iter *i, int maxpages, size_t max_bytes)
 		shorted = iov_iter_count(i) - max_bytes;
 		iov_iter_truncate(i, max_bytes);
 	}
-	npages = iov_iter_npages(i, INT_MAX);
+	npages = iov_iter_npages(i, maxpages);
 	if (shorted)
 		iov_iter_reexpand(i, iov_iter_count(i) + shorted);
 
diff --git a/include/linux/verification.h b/include/linux/verification.h
index a655923335ae..f34e50ebcf60 100644
--- a/include/linux/verification.h
+++ b/include/linux/verification.h
@@ -17,6 +17,14 @@
 #define VERIFY_USE_SECONDARY_KEYRING ((struct key *)1UL)
 #define VERIFY_USE_PLATFORM_KEYRING  ((struct key *)2UL)
 
+static inline int system_keyring_id_check(u64 id)
+{
+	if (id > (unsigned long)VERIFY_USE_PLATFORM_KEYRING)
+		return -EINVAL;
+
+	return 0;
+}
+
 /*
  * The use to which an asymmetric key is being put.
  */
diff --git a/include/linux/xattr.h b/include/linux/xattr.h
index 979a9d3e5bfb..4c379d23ec6e 100644
--- a/include/linux/xattr.h
+++ b/include/linux/xattr.h
@@ -61,7 +61,7 @@ int __vfs_setxattr_locked(struct user_namespace *, struct dentry *,
 			  const char *, const void *, size_t, int,
 			  struct inode **);
 int vfs_setxattr(struct user_namespace *, struct dentry *, const char *,
-		 void *, size_t, int);
+		 const void *, size_t, int);
 int __vfs_removexattr(struct user_namespace *, struct dentry *, const char *);
 int __vfs_removexattr_locked(struct user_namespace *, struct dentry *,
 			     const char *, struct inode **);
diff --git a/include/media/vsp1.h b/include/media/vsp1.h
index cc1b0d42ce95..48f4a5023d81 100644
--- a/include/media/vsp1.h
+++ b/include/media/vsp1.h
@@ -51,6 +51,7 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
  * @dst: destination rectangle on the display (integer coordinates)
  * @alpha: alpha value (0: fully transparent, 255: fully opaque)
  * @zpos: Z position of the plane (from 0 to number of planes minus 1)
+ * @premult: true for premultiplied alpha
  */
 struct vsp1_du_atomic_config {
 	u32 pixelformat;
@@ -60,6 +61,7 @@ struct vsp1_du_atomic_config {
 	struct v4l2_rect dst;
 	unsigned int alpha;
 	unsigned int zpos;
+	bool premult;
 };
 
 /**
diff --git a/include/net/act_api.h b/include/net/act_api.h
index 9cf6870b526e..61f2ceb3939e 100644
--- a/include/net/act_api.h
+++ b/include/net/act_api.h
@@ -111,6 +111,7 @@ struct tc_action_ops {
 	struct list_head head;
 	char    kind[IFNAMSIZ];
 	enum tca_id  id; /* identifier should match kind */
+	unsigned int	net_id;
 	size_t	size;
 	struct module		*owner;
 	int     (*act)(struct sk_buff *, const struct tc_action *,
diff --git a/include/net/af_vsock.h b/include/net/af_vsock.h
index 1c53c4c4d88f..568a87c5e0d0 100644
--- a/include/net/af_vsock.h
+++ b/include/net/af_vsock.h
@@ -78,6 +78,7 @@ struct vsock_sock {
 s64 vsock_stream_has_data(struct vsock_sock *vsk);
 s64 vsock_stream_has_space(struct vsock_sock *vsk);
 struct sock *vsock_create_connected(struct sock *parent);
+void vsock_data_ready(struct sock *sk);
 
 /**** TRANSPORT ****/
 
@@ -135,6 +136,7 @@ struct vsock_transport {
 	u64 (*stream_rcvhiwat)(struct vsock_sock *);
 	bool (*stream_is_active)(struct vsock_sock *);
 	bool (*stream_allow)(u32 cid, u32 port);
+	int (*set_rcvlowat)(struct vsock_sock *vsk, int val);
 
 	/* SEQ_PACKET. */
 	ssize_t (*seqpacket_dequeue)(struct vsock_sock *vsk, struct msghdr *msg,
diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h
index e72f3b247b5e..bcc5a4cd2c17 100644
--- a/include/net/bluetooth/bluetooth.h
+++ b/include/net/bluetooth/bluetooth.h
@@ -627,6 +627,7 @@ static inline bool iso_enabled(void)
 
 int mgmt_init(void);
 void mgmt_exit(void);
+void mgmt_cleanup(struct sock *sk);
 
 void bt_sock_reclassify_lock(struct sock *sk, int proto);
 
diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h
index cf29511b25a8..e004ba04a9ae 100644
--- a/include/net/bluetooth/hci.h
+++ b/include/net/bluetooth/hci.h
@@ -354,6 +354,10 @@ enum {
 	HCI_LE_SIMULTANEOUS_ROLES,
 	HCI_CMD_DRAIN_WORKQUEUE,
 
+	HCI_MESH_EXPERIMENTAL,
+	HCI_MESH,
+	HCI_MESH_SENDING,
+
 	__HCI_NUM_FLAGS,
 };
 
diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h
index e7862903187d..c54bc71254af 100644
--- a/include/net/bluetooth/hci_core.h
+++ b/include/net/bluetooth/hci_core.h
@@ -238,6 +238,7 @@ struct adv_info {
 	bool	enabled;
 	bool	pending;
 	bool	periodic;
+	__u8	mesh;
 	__u8	instance;
 	__u32	flags;
 	__u16	timeout;
@@ -372,6 +373,8 @@ struct hci_dev {
 	__u8		le_resolv_list_size;
 	__u8		le_num_of_adv_sets;
 	__u8		le_states[8];
+	__u8		mesh_ad_types[16];
+	__u8		mesh_send_ref;
 	__u8		commands[64];
 	__u8		hci_ver;
 	__u16		hci_rev;
@@ -511,6 +514,7 @@ struct hci_dev {
 	struct list_head	cmd_sync_work_list;
 	struct mutex		cmd_sync_work_lock;
 	struct work_struct	cmd_sync_cancel_work;
+	struct work_struct	reenable_adv_work;
 
 	__u16			discov_timeout;
 	struct delayed_work	discov_off;
@@ -561,6 +565,7 @@ struct hci_dev {
 
 	struct hci_conn_hash	conn_hash;
 
+	struct list_head	mesh_pending;
 	struct list_head	mgmt_pending;
 	struct list_head	reject_list;
 	struct list_head	accept_list;
@@ -614,6 +619,8 @@ struct hci_dev {
 	struct delayed_work	rpa_expired;
 	bdaddr_t		rpa;
 
+	struct delayed_work	mesh_send_done;
+
 	enum {
 		INTERLEAVE_SCAN_NONE,
 		INTERLEAVE_SCAN_NO_FILTER,
@@ -1576,7 +1583,8 @@ struct adv_info *hci_add_adv_instance(struct hci_dev *hdev, u8 instance,
 				      u32 flags, u16 adv_data_len, u8 *adv_data,
 				      u16 scan_rsp_len, u8 *scan_rsp_data,
 				      u16 timeout, u16 duration, s8 tx_power,
-				      u32 min_interval, u32 max_interval);
+				      u32 min_interval, u32 max_interval,
+				      u8 mesh_handle);
 struct adv_info *hci_add_per_instance(struct hci_dev *hdev, u8 instance,
 				      u32 flags, u8 data_len, u8 *data,
 				      u32 min_interval, u32 max_interval);
@@ -1997,6 +2005,9 @@ void hci_mgmt_chan_unregister(struct hci_mgmt_chan *c);
 #define DISCOV_LE_FAST_ADV_INT_MAX	0x00F0	/* 150 msec */
 #define DISCOV_LE_PER_ADV_INT_MIN	0x00A0	/* 200 msec */
 #define DISCOV_LE_PER_ADV_INT_MAX	0x00A0	/* 200 msec */
+#define DISCOV_LE_ADV_MESH_MIN		0x00A0  /* 100 msec */
+#define DISCOV_LE_ADV_MESH_MAX		0x00A0  /* 100 msec */
+#define INTERVAL_TO_MS(x)		(((x) * 10) / 0x10)
 
 #define NAME_RESOLVE_DURATION		msecs_to_jiffies(10240)	/* 10.24 sec */
 
@@ -2048,7 +2059,8 @@ void mgmt_start_discovery_complete(struct hci_dev *hdev, u8 status);
 void mgmt_stop_discovery_complete(struct hci_dev *hdev, u8 status);
 void mgmt_device_found(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 link_type,
 		       u8 addr_type, u8 *dev_class, s8 rssi, u32 flags,
-		       u8 *eir, u16 eir_len, u8 *scan_rsp, u8 scan_rsp_len);
+		       u8 *eir, u16 eir_len, u8 *scan_rsp, u8 scan_rsp_len,
+		       u64 instant);
 void mgmt_remote_name(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 link_type,
 		      u8 addr_type, s8 rssi, u8 *name, u8 name_len);
 void mgmt_discovering(struct hci_dev *hdev, u8 discovering);
@@ -2075,6 +2087,7 @@ int mgmt_phy_configuration_changed(struct hci_dev *hdev, struct sock *skip);
 void mgmt_adv_monitor_device_lost(struct hci_dev *hdev, u16 handle,
 				  bdaddr_t *bdaddr, u8 addr_type);
 
+int hci_abort_conn(struct hci_conn *conn, u8 reason);
 u8 hci_le_conn_update(struct hci_conn *conn, u16 min, u16 max, u16 latency,
 		      u16 to_multiplier);
 void hci_le_start_enc(struct hci_conn *conn, __le16 ediv, __le64 rand,
diff --git a/include/net/bluetooth/hci_sync.h b/include/net/bluetooth/hci_sync.h
index 3843f5060c73..17f5a4c32f36 100644
--- a/include/net/bluetooth/hci_sync.h
+++ b/include/net/bluetooth/hci_sync.h
@@ -16,6 +16,7 @@ struct hci_cmd_sync_work_entry {
 	hci_cmd_sync_work_destroy_t destroy;
 };
 
+struct adv_info;
 /* Function with sync suffix shall not be called with hdev->lock held as they
  * wait the command to complete and in the meantime an event could be received
  * which could attempt to acquire hdev->lock causing a deadlock.
@@ -51,11 +52,16 @@ int hci_update_class_sync(struct hci_dev *hdev);
 int hci_update_name_sync(struct hci_dev *hdev);
 int hci_write_ssp_mode_sync(struct hci_dev *hdev, u8 mode);
 
+int hci_get_random_address(struct hci_dev *hdev, bool require_privacy,
+			   bool use_rpa, struct adv_info *adv_instance,
+			   u8 *own_addr_type, bdaddr_t *rand_addr);
+
 int hci_update_random_address_sync(struct hci_dev *hdev, bool require_privacy,
 				   bool rpa, u8 *own_addr_type);
 
 int hci_update_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance);
 int hci_update_adv_data_sync(struct hci_dev *hdev, u8 instance);
+int hci_update_adv_data(struct hci_dev *hdev, u8 instance);
 int hci_schedule_adv_instance_sync(struct hci_dev *hdev, u8 instance,
 				   bool force);
 
@@ -72,7 +78,8 @@ int hci_start_per_adv_sync(struct hci_dev *hdev, u8 instance, u8 data_len,
 int hci_remove_advertising_sync(struct hci_dev *hdev, struct sock *sk,
 				u8 instance, bool force);
 int hci_disable_advertising_sync(struct hci_dev *hdev);
-
+int hci_clear_adv_instance_sync(struct hci_dev *hdev, struct sock *sk,
+				u8 instance, bool force);
 int hci_update_passive_scan_sync(struct hci_dev *hdev);
 int hci_update_passive_scan(struct hci_dev *hdev);
 int hci_read_rssi_sync(struct hci_dev *hdev, __le16 handle);
diff --git a/include/net/bluetooth/mgmt.h b/include/net/bluetooth/mgmt.h
index 7c1ad0f6fcec..743f6f59dff8 100644
--- a/include/net/bluetooth/mgmt.h
+++ b/include/net/bluetooth/mgmt.h
@@ -837,6 +837,42 @@ struct mgmt_cp_add_adv_patterns_monitor_rssi {
 	struct mgmt_adv_pattern patterns[];
 } __packed;
 #define MGMT_ADD_ADV_PATTERNS_MONITOR_RSSI_SIZE	8
+#define MGMT_OP_SET_MESH_RECEIVER		0x0057
+struct mgmt_cp_set_mesh {
+	__u8   enable;
+	__le16 window;
+	__le16 period;
+	__u8   num_ad_types;
+	__u8   ad_types[];
+} __packed;
+#define MGMT_SET_MESH_RECEIVER_SIZE	6
+
+#define MGMT_OP_MESH_READ_FEATURES	0x0058
+#define MGMT_MESH_READ_FEATURES_SIZE	0
+#define MESH_HANDLES_MAX	3
+struct mgmt_rp_mesh_read_features {
+	__le16	index;
+	__u8   max_handles;
+	__u8   used_handles;
+	__u8   handles[MESH_HANDLES_MAX];
+} __packed;
+
+#define MGMT_OP_MESH_SEND		0x0059
+struct mgmt_cp_mesh_send {
+	struct mgmt_addr_info addr;
+	__le64  instant;
+	__le16  delay;
+	__u8   cnt;
+	__u8   adv_data_len;
+	__u8   adv_data[];
+} __packed;
+#define MGMT_MESH_SEND_SIZE		19
+
+#define MGMT_OP_MESH_SEND_CANCEL	0x005A
+struct mgmt_cp_mesh_send_cancel {
+	__u8  handle;
+} __packed;
+#define MGMT_MESH_SEND_CANCEL_SIZE	1
 
 #define MGMT_EV_CMD_COMPLETE		0x0001
 struct mgmt_ev_cmd_complete {
@@ -1120,3 +1156,19 @@ struct mgmt_ev_adv_monitor_device_lost {
 	__le16 monitor_handle;
 	struct mgmt_addr_info addr;
 } __packed;
+
+#define MGMT_EV_MESH_DEVICE_FOUND	0x0031
+struct mgmt_ev_mesh_device_found {
+	struct mgmt_addr_info addr;
+	__s8	rssi;
+	__le64	instant;
+	__le32	flags;
+	__le16	eir_len;
+	__u8	eir[];
+} __packed;
+
+
+#define MGMT_EV_MESH_PACKET_CMPLT		0x0032
+struct mgmt_ev_mesh_pkt_cmplt {
+	__u8	handle;
+} __packed;
diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h
index 908d58393484..e09ff87146c1 100644
--- a/include/net/cfg80211.h
+++ b/include/net/cfg80211.h
@@ -2316,6 +2316,7 @@ struct ocb_setup {
  * @cwmax: Maximum contention window [a value of the form 2^n-1 in the range
  *	1..32767]
  * @aifs: Arbitration interframe space [0..255]
+ * @link_id: link_id or -1 for non-MLD
  */
 struct ieee80211_txq_params {
 	enum nl80211_ac ac;
@@ -2323,6 +2324,7 @@ struct ieee80211_txq_params {
 	u16 cwmin;
 	u16 cwmax;
 	u8 aifs;
+	int link_id;
 };
 
 /**
@@ -3929,22 +3931,33 @@ struct mgmt_frame_regs {
  * @del_intf_link: Remove an MLO link from the given interface.
  *
  * @add_key: add a key with the given parameters. @mac_addr will be %NULL
- *	when adding a group key.
+ *	when adding a group key. @link_id will be -1 for non-MLO connection.
+ *	For MLO connection, @link_id will be >= 0 for group key and -1 for
+ *	pairwise key, @mac_addr will be peer's MLD address for MLO pairwise key.
  *
  * @get_key: get information about the key with the given parameters.
  *	@mac_addr will be %NULL when requesting information for a group
  *	key. All pointers given to the @callback function need not be valid
  *	after it returns. This function should return an error if it is
  *	not possible to retrieve the key, -ENOENT if it doesn't exist.
+ *	@link_id will be -1 for non-MLO connection. For MLO connection,
+ *	@link_id will be >= 0 for group key and -1 for pairwise key, @mac_addr
+ *	will be peer's MLD address for MLO pairwise key.
  *
  * @del_key: remove a key given the @mac_addr (%NULL for a group key)
- *	and @key_index, return -ENOENT if the key doesn't exist.
+ *	and @key_index, return -ENOENT if the key doesn't exist. @link_id will
+ *	be -1 for non-MLO connection. For MLO connection, @link_id will be >= 0
+ *	for group key and -1 for pairwise key, @mac_addr will be peer's MLD
+ *	address for MLO pairwise key.
  *
- * @set_default_key: set the default key on an interface
+ * @set_default_key: set the default key on an interface. @link_id will be >= 0
+ *	for MLO connection and -1 for non-MLO connection.
  *
- * @set_default_mgmt_key: set the default management frame key on an interface
+ * @set_default_mgmt_key: set the default management frame key on an interface.
+ *	@link_id will be >= 0 for MLO connection and -1 for non-MLO connection.
  *
- * @set_default_beacon_key: set the default Beacon frame key on an interface
+ * @set_default_beacon_key: set the default Beacon frame key on an interface.
+ *	@link_id will be >= 0 for MLO connection and -1 for non-MLO connection.
  *
  * @set_rekey_data: give the data necessary for GTK rekeying to the driver
  *
@@ -4293,22 +4306,24 @@ struct cfg80211_ops {
 				 unsigned int link_id);
 
 	int	(*add_key)(struct wiphy *wiphy, struct net_device *netdev,
-			   u8 key_index, bool pairwise, const u8 *mac_addr,
-			   struct key_params *params);
+			   int link_id, u8 key_index, bool pairwise,
+			   const u8 *mac_addr, struct key_params *params);
 	int	(*get_key)(struct wiphy *wiphy, struct net_device *netdev,
-			   u8 key_index, bool pairwise, const u8 *mac_addr,
-			   void *cookie,
+			   int link_id, u8 key_index, bool pairwise,
+			   const u8 *mac_addr, void *cookie,
 			   void (*callback)(void *cookie, struct key_params*));
 	int	(*del_key)(struct wiphy *wiphy, struct net_device *netdev,
-			   u8 key_index, bool pairwise, const u8 *mac_addr);
+			   int link_id, u8 key_index, bool pairwise,
+			   const u8 *mac_addr);
 	int	(*set_default_key)(struct wiphy *wiphy,
-				   struct net_device *netdev,
+				   struct net_device *netdev, int link_id,
 				   u8 key_index, bool unicast, bool multicast);
 	int	(*set_default_mgmt_key)(struct wiphy *wiphy,
-					struct net_device *netdev,
+					struct net_device *netdev, int link_id,
 					u8 key_index);
 	int	(*set_default_beacon_key)(struct wiphy *wiphy,
 					  struct net_device *netdev,
+					  int link_id,
 					  u8 key_index);
 
 	int	(*start_ap)(struct wiphy *wiphy, struct net_device *dev,
@@ -8266,6 +8281,7 @@ void cfg80211_ch_switch_notify(struct net_device *dev,
  * cfg80211_ch_switch_started_notify - notify channel switch start
  * @dev: the device on which the channel switch started
  * @chandef: the future channel definition
+ * @link_id: the link ID for MLO, must be 0 for non-MLO
  * @count: the number of TBTTs until the channel switch happens
  * @quiet: whether or not immediate quiet was requested by the AP
  *
@@ -8275,7 +8291,8 @@ void cfg80211_ch_switch_notify(struct net_device *dev,
  */
 void cfg80211_ch_switch_started_notify(struct net_device *dev,
 				       struct cfg80211_chan_def *chandef,
-				       u8 count, bool quiet);
+				       unsigned int link_id, u8 count,
+				       bool quiet);
 
 /**
  * ieee80211_operating_class_to_band - convert operating class to band
diff --git a/include/net/devlink.h b/include/net/devlink.h
index 119ed1ffb988..ba6b8b094943 100644
--- a/include/net/devlink.h
+++ b/include/net/devlink.h
@@ -118,7 +118,6 @@ struct devlink_rate {
 
 struct devlink_port {
 	struct list_head list;
-	struct list_head param_list;
 	struct list_head region_list;
 	struct devlink *devlink;
 	unsigned int index;
@@ -130,7 +129,9 @@ struct devlink_port {
 	void *type_dev;
 	struct devlink_port_attrs attrs;
 	u8 attrs_set:1,
-	   switch_port:1;
+	   switch_port:1,
+	   registered:1,
+	   initialized:1;
 	struct delayed_work type_warn_dw;
 	struct list_head reporter_list;
 	struct mutex reporters_lock; /* Protects reporter_list */
@@ -624,8 +625,7 @@ struct devlink_flash_update_params {
 	u32 overwrite_mask;
 };
 
-#define DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT		BIT(0)
-#define DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK	BIT(1)
+#define DEVLINK_SUPPORT_FLASH_UPDATE_OVERWRITE_MASK	BIT(0)
 
 struct devlink_region;
 struct devlink_info_req;
@@ -1564,6 +1564,9 @@ void devlink_set_features(struct devlink *devlink, u64 features);
 void devlink_register(struct devlink *devlink);
 void devlink_unregister(struct devlink *devlink);
 void devlink_free(struct devlink *devlink);
+void devlink_port_init(struct devlink *devlink,
+		       struct devlink_port *devlink_port);
+void devlink_port_fini(struct devlink_port *devlink_port);
 int devl_port_register(struct devlink *devlink,
 		       struct devlink_port *devlink_port,
 		       unsigned int port_index);
@@ -1714,15 +1717,31 @@ int devlink_info_driver_name_put(struct devlink_info_req *req,
 				 const char *name);
 int devlink_info_board_serial_number_put(struct devlink_info_req *req,
 					 const char *bsn);
+
+enum devlink_info_version_type {
+	DEVLINK_INFO_VERSION_TYPE_NONE,
+	DEVLINK_INFO_VERSION_TYPE_COMPONENT, /* May be used as flash update
+					      * component by name.
+					      */
+};
+
 int devlink_info_version_fixed_put(struct devlink_info_req *req,
 				   const char *version_name,
 				   const char *version_value);
 int devlink_info_version_stored_put(struct devlink_info_req *req,
 				    const char *version_name,
 				    const char *version_value);
+int devlink_info_version_stored_put_ext(struct devlink_info_req *req,
+					const char *version_name,
+					const char *version_value,
+					enum devlink_info_version_type version_type);
 int devlink_info_version_running_put(struct devlink_info_req *req,
 				     const char *version_name,
 				     const char *version_value);
+int devlink_info_version_running_put_ext(struct devlink_info_req *req,
+					 const char *version_name,
+					 const char *version_value,
+					 enum devlink_info_version_type version_type);
 
 int devlink_fmsg_obj_nest_start(struct devlink_fmsg *fmsg);
 int devlink_fmsg_obj_nest_end(struct devlink_fmsg *fmsg);
diff --git a/include/net/dn.h b/include/net/dn.h
deleted file mode 100644
index ba9655b0098a..000000000000
--- a/include/net/dn.h
+++ /dev/null
@@ -1,231 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _NET_DN_H
-#define _NET_DN_H
-
-#include <linux/dn.h>
-#include <net/sock.h>
-#include <net/flow.h>
-#include <asm/byteorder.h>
-#include <asm/unaligned.h>
-
-struct dn_scp                                   /* Session Control Port */
-{
-        unsigned char           state;
-#define DN_O     1                      /* Open                 */
-#define DN_CR    2                      /* Connect Receive      */
-#define DN_DR    3                      /* Disconnect Reject    */
-#define DN_DRC   4                      /* Discon. Rej. Complete*/
-#define DN_CC    5                      /* Connect Confirm      */
-#define DN_CI    6                      /* Connect Initiate     */
-#define DN_NR    7                      /* No resources         */
-#define DN_NC    8                      /* No communication     */
-#define DN_CD    9                      /* Connect Delivery     */
-#define DN_RJ    10                     /* Rejected             */
-#define DN_RUN   11                     /* Running              */
-#define DN_DI    12                     /* Disconnect Initiate  */
-#define DN_DIC   13                     /* Disconnect Complete  */
-#define DN_DN    14                     /* Disconnect Notificat */
-#define DN_CL    15                     /* Closed               */
-#define DN_CN    16                     /* Closed Notification  */
-
-        __le16          addrloc;
-        __le16          addrrem;
-        __u16          numdat;
-        __u16          numoth;
-        __u16          numoth_rcv;
-        __u16          numdat_rcv;
-        __u16          ackxmt_dat;
-        __u16          ackxmt_oth;
-        __u16          ackrcv_dat;
-        __u16          ackrcv_oth;
-        __u8           flowrem_sw;
-	__u8           flowloc_sw;
-#define DN_SEND         2
-#define DN_DONTSEND     1
-#define DN_NOCHANGE     0
-	__u16		flowrem_dat;
-	__u16		flowrem_oth;
-	__u16		flowloc_dat;
-	__u16		flowloc_oth;
-	__u8		services_rem;
-	__u8		services_loc;
-	__u8		info_rem;
-	__u8		info_loc;
-
-	__u16		segsize_rem;
-	__u16		segsize_loc;
-
-	__u8		nonagle;
-	__u8		multi_ireq;
-	__u8		accept_mode;
-	unsigned long		seg_total; /* Running total of current segment */
-
-	struct optdata_dn     conndata_in;
-	struct optdata_dn     conndata_out;
-	struct optdata_dn     discdata_in;
-	struct optdata_dn     discdata_out;
-        struct accessdata_dn  accessdata;
-
-        struct sockaddr_dn addr; /* Local address  */
-	struct sockaddr_dn peer; /* Remote address */
-
-	/*
-	 * In this case the RTT estimation is not specified in the
-	 * docs, nor is any back off algorithm. Here we follow well
-	 * known tcp algorithms with a few small variations.
-	 *
-	 * snd_window: Max number of packets we send before we wait for
-	 *             an ack to come back. This will become part of a
-	 *             more complicated scheme when we support flow
-	 *             control.
-	 *
-	 * nsp_srtt:   Round-Trip-Time (x8) in jiffies. This is a rolling
-	 *             average.
-	 * nsp_rttvar: Round-Trip-Time-Varience (x4) in jiffies. This is the
-	 *             varience of the smoothed average (but calculated in
-	 *             a simpler way than for normal statistical varience
-	 *             calculations).
-	 *
-	 * nsp_rxtshift: Backoff counter. Value is zero normally, each time
-	 *               a packet is lost is increases by one until an ack
-	 *               is received. Its used to index an array of backoff
-	 *               multipliers.
-	 */
-#define NSP_MIN_WINDOW 1
-#define NSP_MAX_WINDOW (0x07fe)
-	unsigned long max_window;
-	unsigned long snd_window;
-#define NSP_INITIAL_SRTT (HZ)
-	unsigned long nsp_srtt;
-#define NSP_INITIAL_RTTVAR (HZ*3)
-	unsigned long nsp_rttvar;
-#define NSP_MAXRXTSHIFT 12
-	unsigned long nsp_rxtshift;
-
-	/*
-	 * Output queues, one for data, one for otherdata/linkservice
-	 */
-	struct sk_buff_head data_xmit_queue;
-	struct sk_buff_head other_xmit_queue;
-
-	/*
-	 * Input queue for other data
-	 */
-	struct sk_buff_head other_receive_queue;
-	int other_report;
-
-	/*
-	 * Stuff to do with the slow timer
-	 */
-	unsigned long stamp;          /* time of last transmit */
-	unsigned long persist;
-	int (*persist_fxn)(struct sock *sk);
-	unsigned long keepalive;
-	void (*keepalive_fxn)(struct sock *sk);
-
-};
-
-static inline struct dn_scp *DN_SK(struct sock *sk)
-{
-	return (struct dn_scp *)(sk + 1);
-}
-
-/*
- * src,dst : Source and Destination DECnet addresses
- * hops : Number of hops through the network
- * dst_port, src_port : NSP port numbers
- * services, info : Useful data extracted from conninit messages
- * rt_flags : Routing flags byte
- * nsp_flags : NSP layer flags byte
- * segsize : Size of segment
- * segnum : Number, for data, otherdata and linkservice
- * xmit_count : Number of times we've transmitted this skb
- * stamp : Time stamp of most recent transmission, used in RTT calculations
- * iif: Input interface number
- *
- * As a general policy, this structure keeps all addresses in network
- * byte order, and all else in host byte order. Thus dst, src, dst_port
- * and src_port are in network order. All else is in host order.
- * 
- */
-#define DN_SKB_CB(skb) ((struct dn_skb_cb *)(skb)->cb)
-struct dn_skb_cb {
-	__le16 dst;
-	__le16 src;
-	__u16 hops;
-	__le16 dst_port;
-	__le16 src_port;
-	__u8 services;
-	__u8 info;
-	__u8 rt_flags;
-	__u8 nsp_flags;
-	__u16 segsize;
-	__u16 segnum;
-	__u16 xmit_count;
-	unsigned long stamp;
-	int iif;
-};
-
-static inline __le16 dn_eth2dn(const unsigned char *ethaddr)
-{
-	return get_unaligned((__le16 *)(ethaddr + 4));
-}
-
-static inline __le16 dn_saddr2dn(struct sockaddr_dn *saddr)
-{
-	return *(__le16 *)saddr->sdn_nodeaddr;
-}
-
-static inline void dn_dn2eth(unsigned char *ethaddr, __le16 addr)
-{
-	__u16 a = le16_to_cpu(addr);
-	ethaddr[0] = 0xAA;
-	ethaddr[1] = 0x00;
-	ethaddr[2] = 0x04;
-	ethaddr[3] = 0x00;
-	ethaddr[4] = (__u8)(a & 0xff);
-	ethaddr[5] = (__u8)(a >> 8);
-}
-
-static inline void dn_sk_ports_copy(struct flowidn *fld, struct dn_scp *scp)
-{
-	fld->fld_sport = scp->addrloc;
-	fld->fld_dport = scp->addrrem;
-}
-
-unsigned int dn_mss_from_pmtu(struct net_device *dev, int mtu);
-void dn_register_sysctl(void);
-void dn_unregister_sysctl(void);
-
-#define DN_MENUVER_ACC 0x01
-#define DN_MENUVER_USR 0x02
-#define DN_MENUVER_PRX 0x04
-#define DN_MENUVER_UIC 0x08
-
-struct sock *dn_sklist_find_listener(struct sockaddr_dn *addr);
-struct sock *dn_find_by_skb(struct sk_buff *skb);
-#define DN_ASCBUF_LEN 9
-char *dn_addr2asc(__u16, char *);
-int dn_destroy_timer(struct sock *sk);
-
-int dn_sockaddr2username(struct sockaddr_dn *addr, unsigned char *buf,
-			 unsigned char type);
-int dn_username2sockaddr(unsigned char *data, int len, struct sockaddr_dn *addr,
-			 unsigned char *type);
-
-void dn_start_slow_timer(struct sock *sk);
-void dn_stop_slow_timer(struct sock *sk);
-
-extern __le16 decnet_address;
-extern int decnet_debug_level;
-extern int decnet_time_wait;
-extern int decnet_dn_count;
-extern int decnet_di_count;
-extern int decnet_dr_count;
-extern int decnet_no_fc_max_cwnd;
-
-extern long sysctl_decnet_mem[3];
-extern int sysctl_decnet_wmem[3];
-extern int sysctl_decnet_rmem[3];
-
-#endif /* _NET_DN_H */
diff --git a/include/net/dn_dev.h b/include/net/dn_dev.h
deleted file mode 100644
index bec303ea8367..000000000000
--- a/include/net/dn_dev.h
+++ /dev/null
@@ -1,200 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _NET_DN_DEV_H
-#define _NET_DN_DEV_H
-
-#include <linux/netdevice.h>
-
-struct dn_dev;
-
-struct dn_ifaddr {
-	struct dn_ifaddr __rcu *ifa_next;
-	struct dn_dev    *ifa_dev;
-	__le16            ifa_local;
-	__le16            ifa_address;
-	__u32             ifa_flags;
-	__u8              ifa_scope;
-	char              ifa_label[IFNAMSIZ];
-	struct rcu_head   rcu;
-};
-
-#define DN_DEV_S_RU  0 /* Run - working normally   */
-#define DN_DEV_S_CR  1 /* Circuit Rejected         */
-#define DN_DEV_S_DS  2 /* Data Link Start          */
-#define DN_DEV_S_RI  3 /* Routing Layer Initialize */
-#define DN_DEV_S_RV  4 /* Routing Layer Verify     */
-#define DN_DEV_S_RC  5 /* Routing Layer Complete   */
-#define DN_DEV_S_OF  6 /* Off                      */
-#define DN_DEV_S_HA  7 /* Halt                     */
-
-
-/*
- * The dn_dev_parms structure contains the set of parameters
- * for each device (hence inclusion in the dn_dev structure)
- * and an array is used to store the default types of supported
- * device (in dn_dev.c).
- *
- * The type field matches the ARPHRD_ constants and is used in
- * searching the list for supported devices when new devices
- * come up.
- *
- * The mode field is used to find out if a device is broadcast,
- * multipoint, or pointopoint. Please note that DECnet thinks
- * different ways about devices to the rest of the kernel
- * so the normal IFF_xxx flags are invalid here. For devices
- * which can be any combination of the previously mentioned
- * attributes, you can set this on a per device basis by
- * installing an up() routine.
- *
- * The device state field, defines the initial state in which the
- * device will come up. In the dn_dev structure, it is the actual
- * state.
- *
- * Things have changed here. I've killed timer1 since it's a user space
- * issue for a user space routing deamon to sort out. The kernel does
- * not need to be bothered with it.
- *
- * Timers:
- * t2 - Rate limit timer, min time between routing and hello messages
- * t3 - Hello timer, send hello messages when it expires
- *
- * Callbacks:
- * up() - Called to initialize device, return value can veto use of
- *        device with DECnet.
- * down() - Called to turn device off when it goes down
- * timer3() - Called once for each ifaddr when timer 3 goes off
- * 
- * sysctl - Hook for sysctl things
- *
- */
-struct dn_dev_parms {
-	int type;	          /* ARPHRD_xxx                         */
-	int mode;	          /* Broadcast, Unicast, Mulitpoint     */
-#define DN_DEV_BCAST  1
-#define DN_DEV_UCAST  2
-#define DN_DEV_MPOINT 4
-	int state;                /* Initial state                      */
-	int forwarding;	          /* 0=EndNode, 1=L1Router, 2=L2Router  */
-	unsigned long t2;         /* Default value of t2                */
-	unsigned long t3;         /* Default value of t3                */
-	int priority;             /* Priority to be a router            */
-	char *name;               /* Name for sysctl                    */
-	int  (*up)(struct net_device *);
-	void (*down)(struct net_device *);
-	void (*timer3)(struct net_device *, struct dn_ifaddr *ifa);
-	void *sysctl;
-};
-
-
-struct dn_dev {
-	struct dn_ifaddr __rcu *ifa_list;
-	struct net_device *dev;
-	struct dn_dev_parms parms;
-	char use_long;
-	struct timer_list timer;
-	unsigned long t3;
-	struct neigh_parms *neigh_parms;
-	__u8 addr[ETH_ALEN];
-	struct neighbour *router; /* Default router on circuit */
-	struct neighbour *peer;   /* Peer on pointopoint links */
-	unsigned long uptime;     /* Time device went up in jiffies */
-};
-
-struct dn_short_packet {
-	__u8    msgflg;
-	__le16 dstnode;
-	__le16 srcnode;
-	__u8   forward;
-} __packed;
-
-struct dn_long_packet {
-	__u8   msgflg;
-	__u8   d_area;
-	__u8   d_subarea;
-	__u8   d_id[6];
-	__u8   s_area;
-	__u8   s_subarea;
-	__u8   s_id[6];
-	__u8   nl2;
-	__u8   visit_ct;
-	__u8   s_class;
-	__u8   pt;
-} __packed;
-
-/*------------------------- DRP - Routing messages ---------------------*/
-
-struct endnode_hello_message {
-	__u8   msgflg;
-	__u8   tiver[3];
-	__u8   id[6];
-	__u8   iinfo;
-	__le16 blksize;
-	__u8   area;
-	__u8   seed[8];
-	__u8   neighbor[6];
-	__le16 timer;
-	__u8   mpd;
-	__u8   datalen;
-	__u8   data[2];
-} __packed;
-
-struct rtnode_hello_message {
-	__u8   msgflg;
-	__u8   tiver[3];
-	__u8   id[6];
-	__u8   iinfo;
-	__le16  blksize;
-	__u8   priority;
-	__u8   area;
-	__le16  timer;
-	__u8   mpd;
-} __packed;
-
-
-void dn_dev_init(void);
-void dn_dev_cleanup(void);
-
-int dn_dev_ioctl(unsigned int cmd, void __user *arg);
-
-void dn_dev_devices_off(void);
-void dn_dev_devices_on(void);
-
-void dn_dev_init_pkt(struct sk_buff *skb);
-void dn_dev_veri_pkt(struct sk_buff *skb);
-void dn_dev_hello(struct sk_buff *skb);
-
-void dn_dev_up(struct net_device *);
-void dn_dev_down(struct net_device *);
-
-int dn_dev_set_default(struct net_device *dev, int force);
-struct net_device *dn_dev_get_default(void);
-int dn_dev_bind_default(__le16 *addr);
-
-int register_dnaddr_notifier(struct notifier_block *nb);
-int unregister_dnaddr_notifier(struct notifier_block *nb);
-
-static inline int dn_dev_islocal(struct net_device *dev, __le16 addr)
-{
-	struct dn_dev *dn_db;
-	struct dn_ifaddr *ifa;
-	int res = 0;
-
-	rcu_read_lock();
-	dn_db = rcu_dereference(dev->dn_ptr);
-	if (dn_db == NULL) {
-		printk(KERN_DEBUG "dn_dev_islocal: Called for non DECnet device\n");
-		goto out;
-	}
-
-	for (ifa = rcu_dereference(dn_db->ifa_list);
-	     ifa != NULL;
-	     ifa = rcu_dereference(ifa->ifa_next))
-		if ((addr ^ ifa->ifa_local) == 0) {
-			res = 1;
-			break;
-		}
-out:
-	rcu_read_unlock();
-	return res;
-}
-
-#endif /* _NET_DN_DEV_H */
diff --git a/include/net/dn_fib.h b/include/net/dn_fib.h
deleted file mode 100644
index 1929a3cd5ebe..000000000000
--- a/include/net/dn_fib.h
+++ /dev/null
@@ -1,169 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _NET_DN_FIB_H
-#define _NET_DN_FIB_H
-
-#include <linux/netlink.h>
-#include <linux/refcount.h>
-#include <linux/rtnetlink.h>
-#include <net/fib_rules.h>
-
-extern const struct nla_policy rtm_dn_policy[];
-
-struct dn_fib_res {
-	struct fib_rule *r;
-	struct dn_fib_info *fi;
-	unsigned char prefixlen;
-	unsigned char nh_sel;
-	unsigned char type;
-	unsigned char scope;
-};
-
-struct dn_fib_nh {
-	struct net_device	*nh_dev;
-	unsigned int		nh_flags;
-	unsigned char		nh_scope;
-	int			nh_weight;
-	int			nh_power;
-	int			nh_oif;
-	__le16			nh_gw;
-};
-
-struct dn_fib_info {
-	struct dn_fib_info	*fib_next;
-	struct dn_fib_info	*fib_prev;
-	refcount_t		fib_treeref;
-	refcount_t		fib_clntref;
-	int			fib_dead;
-	unsigned int		fib_flags;
-	int			fib_protocol;
-	__le16			fib_prefsrc;
-	__u32			fib_priority;
-	__u32			fib_metrics[RTAX_MAX];
-	int			fib_nhs;
-	int			fib_power;
-	struct dn_fib_nh	fib_nh[0];
-#define dn_fib_dev		fib_nh[0].nh_dev
-};
-
-
-#define DN_FIB_RES_RESET(res)	((res).nh_sel = 0)
-#define DN_FIB_RES_NH(res)	((res).fi->fib_nh[(res).nh_sel])
-
-#define DN_FIB_RES_PREFSRC(res)	((res).fi->fib_prefsrc ? : __dn_fib_res_prefsrc(&res))
-#define DN_FIB_RES_GW(res)	(DN_FIB_RES_NH(res).nh_gw)
-#define DN_FIB_RES_DEV(res)	(DN_FIB_RES_NH(res).nh_dev)
-#define DN_FIB_RES_OIF(res)	(DN_FIB_RES_NH(res).nh_oif)
-
-typedef struct {
-	__le16	datum;
-} dn_fib_key_t;
-
-typedef struct {
-	__le16	datum;
-} dn_fib_hash_t;
-
-typedef struct {
-	__u16	datum;
-} dn_fib_idx_t;
-
-struct dn_fib_node {
-	struct dn_fib_node *fn_next;
-	struct dn_fib_info *fn_info;
-#define DN_FIB_INFO(f) ((f)->fn_info)
-	dn_fib_key_t	fn_key;
-	u8		fn_type;
-	u8		fn_scope;
-	u8		fn_state;
-};
-
-
-struct dn_fib_table {
-	struct hlist_node hlist;
-	u32 n;
-
-	int (*insert)(struct dn_fib_table *t, struct rtmsg *r, 
-			struct nlattr *attrs[], struct nlmsghdr *n,
-			struct netlink_skb_parms *req);
-	int (*delete)(struct dn_fib_table *t, struct rtmsg *r,
-			struct nlattr *attrs[], struct nlmsghdr *n,
-			struct netlink_skb_parms *req);
-	int (*lookup)(struct dn_fib_table *t, const struct flowidn *fld,
-			struct dn_fib_res *res);
-	int (*flush)(struct dn_fib_table *t);
-	int (*dump)(struct dn_fib_table *t, struct sk_buff *skb, struct netlink_callback *cb);
-
-	unsigned char data[];
-};
-
-#ifdef CONFIG_DECNET_ROUTER
-/*
- * dn_fib.c
- */
-void dn_fib_init(void);
-void dn_fib_cleanup(void);
-
-int dn_fib_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg);
-struct dn_fib_info *dn_fib_create_info(const struct rtmsg *r,
-				       struct nlattr *attrs[],
-				       const struct nlmsghdr *nlh, int *errp);
-int dn_fib_semantic_match(int type, struct dn_fib_info *fi,
-			  const struct flowidn *fld, struct dn_fib_res *res);
-void dn_fib_release_info(struct dn_fib_info *fi);
-void dn_fib_flush(void);
-void dn_fib_select_multipath(const struct flowidn *fld, struct dn_fib_res *res);
-
-/*
- * dn_tables.c
- */
-struct dn_fib_table *dn_fib_get_table(u32 n, int creat);
-struct dn_fib_table *dn_fib_empty_table(void);
-void dn_fib_table_init(void);
-void dn_fib_table_cleanup(void);
-
-/*
- * dn_rules.c
- */
-void dn_fib_rules_init(void);
-void dn_fib_rules_cleanup(void);
-unsigned int dnet_addr_type(__le16 addr);
-int dn_fib_lookup(struct flowidn *fld, struct dn_fib_res *res);
-
-int dn_fib_dump(struct sk_buff *skb, struct netlink_callback *cb);
-
-void dn_fib_free_info(struct dn_fib_info *fi);
-
-static inline void dn_fib_info_put(struct dn_fib_info *fi)
-{
-	if (refcount_dec_and_test(&fi->fib_clntref))
-		dn_fib_free_info(fi);
-}
-
-static inline void dn_fib_res_put(struct dn_fib_res *res)
-{
-	if (res->fi)
-		dn_fib_info_put(res->fi);
-	if (res->r)
-		fib_rule_put(res->r);
-}
-
-#else /* Endnode */
-
-#define dn_fib_init()  do { } while(0)
-#define dn_fib_cleanup() do { } while(0)
-
-#define dn_fib_lookup(fl, res) (-ESRCH)
-#define dn_fib_info_put(fi) do { } while(0)
-#define dn_fib_select_multipath(fl, res) do { } while(0)
-#define dn_fib_rules_policy(saddr,res,flags) (0)
-#define dn_fib_res_put(res) do { } while(0)
-
-#endif /* CONFIG_DECNET_ROUTER */
-
-static inline __le16 dnet_make_mask(int n)
-{
-	if (n)
-		return cpu_to_le16(~((1 << (16 - n)) - 1));
-	return cpu_to_le16(0);
-}
-
-#endif /* _NET_DN_FIB_H */
diff --git a/include/net/dn_neigh.h b/include/net/dn_neigh.h
deleted file mode 100644
index 1f7df98bfc33..000000000000
--- a/include/net/dn_neigh.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _NET_DN_NEIGH_H
-#define _NET_DN_NEIGH_H
-
-#include <net/neighbour.h>
-
-/*
- * The position of the first two fields of
- * this structure are critical - SJW
- */
-struct dn_neigh {
-        struct neighbour n;
-	__le16 addr;
-        unsigned long flags;
-#define DN_NDFLAG_R1    0x0001 /* Router L1      */
-#define DN_NDFLAG_R2    0x0002 /* Router L2      */
-#define DN_NDFLAG_P3    0x0004 /* Phase III Node */
-        unsigned long blksize;
-	__u8 priority;
-};
-
-void dn_neigh_init(void);
-void dn_neigh_cleanup(void);
-int dn_neigh_router_hello(struct net *net, struct sock *sk, struct sk_buff *skb);
-int dn_neigh_endnode_hello(struct net *net, struct sock *sk, struct sk_buff *skb);
-void dn_neigh_pointopoint_hello(struct sk_buff *skb);
-int dn_neigh_elist(struct net_device *dev, unsigned char *ptr, int n);
-int dn_to_neigh_output(struct net *net, struct sock *sk, struct sk_buff *skb);
-
-extern struct neigh_table dn_neigh_table;
-
-#endif /* _NET_DN_NEIGH_H */
diff --git a/include/net/dn_nsp.h b/include/net/dn_nsp.h
deleted file mode 100644
index a4a18fee0b7c..000000000000
--- a/include/net/dn_nsp.h
+++ /dev/null
@@ -1,201 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-#ifndef _NET_DN_NSP_H
-#define _NET_DN_NSP_H
-/******************************************************************************
-    (c) 1995-1998 E.M. Serrat		emserrat@geocities.com
-    
-*******************************************************************************/
-/* dn_nsp.c functions prototyping */
-#include <linux/atomic.h>
-#include <linux/types.h>
-#include <net/sock.h>
-
-struct sk_buff;
-struct sk_buff_head;
-
-void dn_nsp_send_data_ack(struct sock *sk);
-void dn_nsp_send_oth_ack(struct sock *sk);
-void dn_send_conn_ack(struct sock *sk);
-void dn_send_conn_conf(struct sock *sk, gfp_t gfp);
-void dn_nsp_send_disc(struct sock *sk, unsigned char type,
-		      unsigned short reason, gfp_t gfp);
-void dn_nsp_return_disc(struct sk_buff *skb, unsigned char type,
-			unsigned short reason);
-void dn_nsp_send_link(struct sock *sk, unsigned char lsflags, char fcval);
-void dn_nsp_send_conninit(struct sock *sk, unsigned char flags);
-
-void dn_nsp_output(struct sock *sk);
-int dn_nsp_check_xmit_queue(struct sock *sk, struct sk_buff *skb,
-			    struct sk_buff_head *q, unsigned short acknum);
-void dn_nsp_queue_xmit(struct sock *sk, struct sk_buff *skb, gfp_t gfp,
-		       int oob);
-unsigned long dn_nsp_persist(struct sock *sk);
-int dn_nsp_xmit_timeout(struct sock *sk);
-
-int dn_nsp_rx(struct sk_buff *);
-int dn_nsp_backlog_rcv(struct sock *sk, struct sk_buff *skb);
-
-struct sk_buff *dn_alloc_skb(struct sock *sk, int size, gfp_t pri);
-struct sk_buff *dn_alloc_send_skb(struct sock *sk, size_t *size, int noblock,
-				  long timeo, int *err);
-
-#define NSP_REASON_OK 0		/* No error */
-#define NSP_REASON_NR 1		/* No resources */
-#define NSP_REASON_UN 2		/* Unrecognised node name */
-#define NSP_REASON_SD 3		/* Node shutting down */
-#define NSP_REASON_ID 4		/* Invalid destination end user */
-#define NSP_REASON_ER 5		/* End user lacks resources */
-#define NSP_REASON_OB 6		/* Object too busy */
-#define NSP_REASON_US 7		/* Unspecified error */
-#define NSP_REASON_TP 8		/* Third-Party abort */
-#define NSP_REASON_EA 9		/* End user has aborted the link */
-#define NSP_REASON_IF 10	/* Invalid node name format */
-#define NSP_REASON_LS 11	/* Local node shutdown */
-#define NSP_REASON_LL 32	/* Node lacks logical-link resources */
-#define NSP_REASON_LE 33	/* End user lacks logical-link resources */
-#define NSP_REASON_UR 34	/* Unacceptable RQSTRID or PASSWORD field */
-#define NSP_REASON_UA 36	/* Unacceptable ACCOUNT field */
-#define NSP_REASON_TM 38	/* End user timed out logical link */
-#define NSP_REASON_NU 39	/* Node unreachable */
-#define NSP_REASON_NL 41	/* No-link message */
-#define NSP_REASON_DC 42	/* Disconnect confirm */
-#define NSP_REASON_IO 43	/* Image data field overflow */
-
-#define NSP_DISCINIT 0x38
-#define NSP_DISCCONF 0x48
-
-/*------------------------- NSP - messages ------------------------------*/
-/* Data Messages */
-/*---------------*/
-
-/* Data Messages    (data segment/interrupt/link service)               */
-
-struct nsp_data_seg_msg {
-	__u8   msgflg;
-	__le16 dstaddr;
-	__le16 srcaddr;
-} __packed;
-
-struct nsp_data_opt_msg {
-	__le16 acknum;
-	__le16 segnum;
-	__le16 lsflgs;
-} __packed;
-
-struct nsp_data_opt_msg1 {
-	__le16 acknum;
-	__le16 segnum;
-} __packed;
-
-
-/* Acknowledgment Message (data/other data)                             */
-struct nsp_data_ack_msg {
-	__u8   msgflg;
-	__le16 dstaddr;
-	__le16 srcaddr;
-	__le16 acknum;
-} __packed;
-
-/* Connect Acknowledgment Message */
-struct  nsp_conn_ack_msg {
-	__u8 msgflg;
-	__le16 dstaddr;
-} __packed;
-
-
-/* Connect Initiate/Retransmit Initiate/Connect Confirm */
-struct  nsp_conn_init_msg {
-	__u8   msgflg;
-#define NSP_CI      0x18            /* Connect Initiate     */
-#define NSP_RCI     0x68            /* Retrans. Conn Init   */
-	__le16 dstaddr;
-	__le16 srcaddr;
-	__u8   services;
-#define NSP_FC_NONE   0x00            /* Flow Control None    */
-#define NSP_FC_SRC    0x04            /* Seg Req. Count       */
-#define NSP_FC_SCMC   0x08            /* Sess. Control Mess   */
-#define NSP_FC_MASK   0x0c            /* FC type mask         */
-	__u8   info;
-	__le16 segsize;
-} __packed;
-
-/* Disconnect Initiate/Disconnect Confirm */
-struct  nsp_disconn_init_msg {
-	__u8   msgflg;
-	__le16 dstaddr;
-	__le16 srcaddr;
-	__le16 reason;
-} __packed;
-
-
-
-struct  srcobj_fmt {
-	__u8   format;
-	__u8   task;
-	__le16 grpcode;
-	__le16 usrcode;
-	__u8   dlen;
-} __packed;
-
-/*
- * A collection of functions for manipulating the sequence
- * numbers used in NSP. Similar in operation to the functions
- * of the same name in TCP.
- */
-static __inline__ int dn_before(__u16 seq1, __u16 seq2)
-{
-        seq1 &= 0x0fff;
-        seq2 &= 0x0fff;
-
-        return (int)((seq1 - seq2) & 0x0fff) > 2048;
-}
-
-
-static __inline__ int dn_after(__u16 seq1, __u16 seq2)
-{
-        seq1 &= 0x0fff;
-        seq2 &= 0x0fff;
-
-        return (int)((seq2 - seq1) & 0x0fff) > 2048;
-}
-
-static __inline__ int dn_equal(__u16 seq1, __u16 seq2)
-{
-        return ((seq1 ^ seq2) & 0x0fff) == 0;
-}
-
-static __inline__ int dn_before_or_equal(__u16 seq1, __u16 seq2)
-{
-	return (dn_before(seq1, seq2) || dn_equal(seq1, seq2));
-}
-
-static __inline__ void seq_add(__u16 *seq, __u16 off)
-{
-        (*seq) += off;
-        (*seq) &= 0x0fff;
-}
-
-static __inline__ int seq_next(__u16 seq1, __u16 seq2)
-{
-	return dn_equal(seq1 + 1, seq2);
-}
-
-/*
- * Can we delay the ack ?
- */
-static __inline__ int sendack(__u16 seq)
-{
-        return (int)((seq & 0x1000) ? 0 : 1);
-}
-
-/*
- * Is socket congested ?
- */
-static __inline__ int dn_congested(struct sock *sk)
-{
-        return atomic_read(&sk->sk_rmem_alloc) > (sk->sk_rcvbuf >> 1);
-}
-
-#define DN_MAX_NSP_DATA_HEADER (11)
-
-#endif /* _NET_DN_NSP_H */
diff --git a/include/net/dn_route.h b/include/net/dn_route.h
deleted file mode 100644
index 88c0300236cc..000000000000
--- a/include/net/dn_route.h
+++ /dev/null
@@ -1,118 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-#ifndef _NET_DN_ROUTE_H
-#define _NET_DN_ROUTE_H
-
-/******************************************************************************
-    (c) 1995-1998 E.M. Serrat		emserrat@geocities.com
-    
-*******************************************************************************/
-
-#include <linux/types.h>
-#include <net/dst.h>
-
-struct sk_buff *dn_alloc_skb(struct sock *sk, int size, gfp_t pri);
-int dn_route_output_sock(struct dst_entry __rcu **pprt, struct flowidn *,
-			 struct sock *sk, int flags);
-int dn_cache_dump(struct sk_buff *skb, struct netlink_callback *cb);
-void dn_rt_cache_flush(int delay);
-int dn_route_rcv(struct sk_buff *skb, struct net_device *dev,
-		 struct packet_type *pt, struct net_device *orig_dev);
-
-/* Masks for flags field */
-#define DN_RT_F_PID 0x07 /* Mask for packet type                      */
-#define DN_RT_F_PF  0x80 /* Padding Follows                           */
-#define DN_RT_F_VER 0x40 /* Version =0 discard packet if ==1          */
-#define DN_RT_F_IE  0x20 /* Intra Ethernet, Reserved in short pkt     */
-#define DN_RT_F_RTS 0x10 /* Packet is being returned to sender        */
-#define DN_RT_F_RQR 0x08 /* Return packet to sender upon non-delivery */
-
-/* Mask for types of routing packets */
-#define DN_RT_PKT_MSK   0x06
-/* Types of routing packets */
-#define DN_RT_PKT_SHORT 0x02 /* Short routing packet */
-#define DN_RT_PKT_LONG  0x06 /* Long routing packet  */
-
-/* Mask for control/routing selection */
-#define DN_RT_PKT_CNTL  0x01 /* Set to 1 if a control packet  */
-/* Types of control packets */
-#define DN_RT_CNTL_MSK  0x0f /* Mask for control packets      */
-#define DN_RT_PKT_INIT  0x01 /* Initialisation packet         */
-#define DN_RT_PKT_VERI  0x03 /* Verification Message          */
-#define DN_RT_PKT_HELO  0x05 /* Hello and Test Message        */
-#define DN_RT_PKT_L1RT  0x07 /* Level 1 Routing Message       */
-#define DN_RT_PKT_L2RT  0x09 /* Level 2 Routing Message       */
-#define DN_RT_PKT_ERTH  0x0b /* Ethernet Router Hello         */
-#define DN_RT_PKT_EEDH  0x0d /* Ethernet EndNode Hello        */
-
-/* Values for info field in hello message */
-#define DN_RT_INFO_TYPE 0x03 /* Type mask                     */
-#define DN_RT_INFO_L1RT 0x02 /* L1 Router                     */
-#define DN_RT_INFO_L2RT 0x01 /* L2 Router                     */
-#define DN_RT_INFO_ENDN 0x03 /* EndNode                       */
-#define DN_RT_INFO_VERI 0x04 /* Verification Reqd.            */
-#define DN_RT_INFO_RJCT 0x08 /* Reject Flag, Reserved         */
-#define DN_RT_INFO_VFLD 0x10 /* Verification Failed, Reserved */
-#define DN_RT_INFO_NOML 0x20 /* No Multicast traffic accepted */
-#define DN_RT_INFO_BLKR 0x40 /* Blocking Requested            */
-
-/*
- * The fl structure is what we used to look up the route.
- * The rt_saddr & rt_daddr entries are the same as key.saddr & key.daddr
- * except for local input routes, where the rt_saddr = fl.fld_dst and
- * rt_daddr = fl.fld_src to allow the route to be used for returning
- * packets to the originating host.
- */
-struct dn_route {
-	struct dst_entry dst;
-	struct dn_route __rcu *dn_next;
-
-	struct neighbour *n;
-
-	struct flowidn fld;
-
-	__le16 rt_saddr;
-	__le16 rt_daddr;
-	__le16 rt_gateway;
-	__le16 rt_local_src;	/* Source used for forwarding packets */
-	__le16 rt_src_map;
-	__le16 rt_dst_map;
-
-	unsigned int rt_flags;
-	unsigned int rt_type;
-};
-
-static inline bool dn_is_input_route(struct dn_route *rt)
-{
-	return rt->fld.flowidn_iif != 0;
-}
-
-static inline bool dn_is_output_route(struct dn_route *rt)
-{
-	return rt->fld.flowidn_iif == 0;
-}
-
-void dn_route_init(void);
-void dn_route_cleanup(void);
-
-#include <net/sock.h>
-#include <linux/if_arp.h>
-
-static inline void dn_rt_send(struct sk_buff *skb)
-{
-	dev_queue_xmit(skb);
-}
-
-static inline void dn_rt_finish_output(struct sk_buff *skb, char *dst, char *src)
-{
-	struct net_device *dev = skb->dev;
-
-	if ((dev->type != ARPHRD_ETHER) && (dev->type != ARPHRD_LOOPBACK))
-		dst = NULL;
-
-	if (dev_hard_header(skb, dev, ETH_P_DNA_RT, dst, src, skb->len) >= 0)
-		dn_rt_send(skb);
-	else
-		kfree_skb(skb);
-}
-
-#endif /* _NET_DN_ROUTE_H */
diff --git a/include/net/dsa.h b/include/net/dsa.h
index b902b31bebce..ee369670e20e 100644
--- a/include/net/dsa.h
+++ b/include/net/dsa.h
@@ -294,12 +294,13 @@ struct dsa_port {
 
 	u8			lag_tx_enabled:1;
 
-	u8			devlink_port_setup:1;
-
 	/* Master state bits, valid only on CPU ports */
 	u8			master_admin_up:1;
 	u8			master_oper_up:1;
 
+	/* Valid only on user ports */
+	u8			cpu_port_in_lag:1;
+
 	u8			setup:1;
 
 	struct device_node	*dn;
@@ -559,6 +560,14 @@ static inline bool dsa_is_user_port(struct dsa_switch *ds, int p)
 	list_for_each_entry((_dp), &(_dst)->ports, list) \
 		if (dsa_port_is_user((_dp)))
 
+#define dsa_tree_for_each_user_port_continue_reverse(_dp, _dst) \
+	list_for_each_entry_continue_reverse((_dp), &(_dst)->ports, list) \
+		if (dsa_port_is_user((_dp)))
+
+#define dsa_tree_for_each_cpu_port(_dp, _dst) \
+	list_for_each_entry((_dp), &(_dst)->ports, list) \
+		if (dsa_port_is_cpu((_dp)))
+
 #define dsa_switch_for_each_port(_dp, _ds) \
 	list_for_each_entry((_dp), &(_ds)->dst->ports, list) \
 		if ((_dp)->ds == (_ds))
@@ -714,6 +723,14 @@ static inline bool dsa_port_offloads_lag(struct dsa_port *dp,
 	return dsa_port_lag_dev_get(dp) == lag->dev;
 }
 
+static inline struct net_device *dsa_port_to_master(const struct dsa_port *dp)
+{
+	if (dp->cpu_port_in_lag)
+		return dsa_port_lag_dev_get(dp->cpu_dp);
+
+	return dp->cpu_dp->master;
+}
+
 static inline
 struct net_device *dsa_port_to_bridge_port(const struct dsa_port *dp)
 {
@@ -798,6 +815,12 @@ dsa_tree_offloads_bridge_dev(struct dsa_switch_tree *dst,
 	return false;
 }
 
+static inline bool dsa_port_tree_same(const struct dsa_port *a,
+				      const struct dsa_port *b)
+{
+	return a->ds->dst == b->ds->dst;
+}
+
 typedef int dsa_fdb_dump_cb_t(const unsigned char *addr, u16 vid,
 			      bool is_static, void *data);
 struct dsa_switch_ops {
@@ -821,6 +844,10 @@ struct dsa_switch_ops {
 	int	(*connect_tag_protocol)(struct dsa_switch *ds,
 					enum dsa_tag_protocol proto);
 
+	int	(*port_change_master)(struct dsa_switch *ds, int port,
+				      struct net_device *master,
+				      struct netlink_ext_ack *extack);
+
 	/* Optional switch-wide initialization and destruction methods */
 	int	(*setup)(struct dsa_switch *ds);
 	void	(*teardown)(struct dsa_switch *ds);
@@ -1077,7 +1104,8 @@ struct dsa_switch_ops {
 					int port);
 	int	(*crosschip_lag_join)(struct dsa_switch *ds, int sw_index,
 				      int port, struct dsa_lag lag,
-				      struct netdev_lag_upper_info *info);
+				      struct netdev_lag_upper_info *info,
+				      struct netlink_ext_ack *extack);
 	int	(*crosschip_lag_leave)(struct dsa_switch *ds, int sw_index,
 				       int port, struct dsa_lag lag);
 
@@ -1152,7 +1180,8 @@ struct dsa_switch_ops {
 	int	(*port_lag_change)(struct dsa_switch *ds, int port);
 	int	(*port_lag_join)(struct dsa_switch *ds, int port,
 				 struct dsa_lag lag,
-				 struct netdev_lag_upper_info *info);
+				 struct netdev_lag_upper_info *info,
+				 struct netlink_ext_ack *extack);
 	int	(*port_lag_leave)(struct dsa_switch *ds, int port,
 				  struct dsa_lag lag);
 
diff --git a/include/net/dst.h b/include/net/dst.h
index 6aa252c3fc55..00b479ce6b99 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -239,12 +239,6 @@ static inline void dst_use_noref(struct dst_entry *dst, unsigned long time)
 	}
 }
 
-static inline void dst_hold_and_use(struct dst_entry *dst, unsigned long time)
-{
-	dst_hold(dst);
-	dst_use_noref(dst, time);
-}
-
 static inline struct dst_entry *dst_clone(struct dst_entry *dst)
 {
 	if (dst)
diff --git a/include/net/dst_metadata.h b/include/net/dst_metadata.h
index adab27ba1ecb..a454cf4327fe 100644
--- a/include/net/dst_metadata.h
+++ b/include/net/dst_metadata.h
@@ -4,11 +4,14 @@
 
 #include <linux/skbuff.h>
 #include <net/ip_tunnels.h>
+#include <net/macsec.h>
 #include <net/dst.h>
 
 enum metadata_type {
 	METADATA_IP_TUNNEL,
 	METADATA_HW_PORT_MUX,
+	METADATA_MACSEC,
+	METADATA_XFRM,
 };
 
 struct hw_port_info {
@@ -16,12 +19,23 @@ struct hw_port_info {
 	u32 port_id;
 };
 
+struct macsec_info {
+	sci_t sci;
+};
+
+struct xfrm_md_info {
+	u32 if_id;
+	int link;
+};
+
 struct metadata_dst {
 	struct dst_entry		dst;
 	enum metadata_type		type;
 	union {
 		struct ip_tunnel_info	tun_info;
 		struct hw_port_info	port_info;
+		struct macsec_info	macsec_info;
+		struct xfrm_md_info	xfrm_info;
 	} u;
 };
 
@@ -53,6 +67,27 @@ skb_tunnel_info(const struct sk_buff *skb)
 	return NULL;
 }
 
+static inline struct xfrm_md_info *lwt_xfrm_info(struct lwtunnel_state *lwt)
+{
+	return (struct xfrm_md_info *)lwt->data;
+}
+
+static inline struct xfrm_md_info *skb_xfrm_md_info(const struct sk_buff *skb)
+{
+	struct metadata_dst *md_dst = skb_metadata_dst(skb);
+	struct dst_entry *dst;
+
+	if (md_dst && md_dst->type == METADATA_XFRM)
+		return &md_dst->u.xfrm_info;
+
+	dst = skb_dst(skb);
+	if (dst && dst->lwtstate &&
+	    dst->lwtstate->type == LWTUNNEL_ENCAP_XFRM)
+		return lwt_xfrm_info(dst->lwtstate);
+
+	return NULL;
+}
+
 static inline bool skb_valid_dst(const struct sk_buff *skb)
 {
 	struct dst_entry *dst = skb_dst(skb);
@@ -82,6 +117,12 @@ static inline int skb_metadata_dst_cmp(const struct sk_buff *skb_a,
 		return memcmp(&a->u.tun_info, &b->u.tun_info,
 			      sizeof(a->u.tun_info) +
 					 a->u.tun_info.options_len);
+	case METADATA_MACSEC:
+		return memcmp(&a->u.macsec_info, &b->u.macsec_info,
+			      sizeof(a->u.macsec_info));
+	case METADATA_XFRM:
+		return memcmp(&a->u.xfrm_info, &b->u.xfrm_info,
+			      sizeof(a->u.xfrm_info));
 	default:
 		return 1;
 	}
diff --git a/include/net/flow.h b/include/net/flow.h
index 987bd511d652..2f0da4f0318b 100644
--- a/include/net/flow.h
+++ b/include/net/flow.h
@@ -54,11 +54,6 @@ union flowi_uli {
 		__u8	code;
 	} icmpt;
 
-	struct {
-		__le16	dport;
-		__le16	sport;
-	} dnports;
-
 	__be32		gre_key;
 
 	struct {
@@ -156,27 +151,11 @@ struct flowi6 {
 	__u32			mp_hash;
 } __attribute__((__aligned__(BITS_PER_LONG/8)));
 
-struct flowidn {
-	struct flowi_common	__fl_common;
-#define flowidn_oif		__fl_common.flowic_oif
-#define flowidn_iif		__fl_common.flowic_iif
-#define flowidn_mark		__fl_common.flowic_mark
-#define flowidn_scope		__fl_common.flowic_scope
-#define flowidn_proto		__fl_common.flowic_proto
-#define flowidn_flags		__fl_common.flowic_flags
-	__le16			daddr;
-	__le16			saddr;
-	union flowi_uli		uli;
-#define fld_sport		uli.ports.sport
-#define fld_dport		uli.ports.dport
-} __attribute__((__aligned__(BITS_PER_LONG/8)));
-
 struct flowi {
 	union {
 		struct flowi_common	__fl_common;
 		struct flowi4		ip4;
 		struct flowi6		ip6;
-		struct flowidn		dn;
 	} u;
 #define flowi_oif	u.__fl_common.flowic_oif
 #define flowi_iif	u.__fl_common.flowic_iif
@@ -211,11 +190,6 @@ static inline struct flowi_common *flowi6_to_flowi_common(struct flowi6 *fl6)
 	return &(fl6->__fl_common);
 }
 
-static inline struct flowi *flowidn_to_flowi(struct flowidn *fldn)
-{
-	return container_of(fldn, struct flowi, u.dn);
-}
-
 __u32 __get_hash_from_flowi6(const struct flowi6 *fl6, struct flow_keys *keys);
 
 #endif
diff --git a/include/net/flow_dissector.h b/include/net/flow_dissector.h
index 6c74812d64b2..5ccf52ef8809 100644
--- a/include/net/flow_dissector.h
+++ b/include/net/flow_dissector.h
@@ -289,6 +289,14 @@ struct flow_dissector_key_pppoe {
 	__be16 type;
 };
 
+/**
+ * struct flow_dissector_key_l2tpv3:
+ * @session_id: identifier for a l2tp session
+ */
+struct flow_dissector_key_l2tpv3 {
+	__be32 session_id;
+};
+
 enum flow_dissector_key_id {
 	FLOW_DISSECTOR_KEY_CONTROL, /* struct flow_dissector_key_control */
 	FLOW_DISSECTOR_KEY_BASIC, /* struct flow_dissector_key_basic */
@@ -320,6 +328,7 @@ enum flow_dissector_key_id {
 	FLOW_DISSECTOR_KEY_HASH, /* struct flow_dissector_key_hash */
 	FLOW_DISSECTOR_KEY_NUM_OF_VLANS, /* struct flow_dissector_key_num_of_vlans */
 	FLOW_DISSECTOR_KEY_PPPOE, /* struct flow_dissector_key_pppoe */
+	FLOW_DISSECTOR_KEY_L2TPV3, /* struct flow_dissector_key_l2tpv3 */
 
 	FLOW_DISSECTOR_KEY_MAX,
 };
diff --git a/include/net/flow_offload.h b/include/net/flow_offload.h
index 2a9a9e42e7fd..e343f9f8363e 100644
--- a/include/net/flow_offload.h
+++ b/include/net/flow_offload.h
@@ -80,6 +80,10 @@ struct flow_match_pppoe {
 	struct flow_dissector_key_pppoe *key, *mask;
 };
 
+struct flow_match_l2tpv3 {
+	struct flow_dissector_key_l2tpv3 *key, *mask;
+};
+
 struct flow_rule;
 
 void flow_rule_match_meta(const struct flow_rule *rule,
@@ -128,6 +132,8 @@ void flow_rule_match_ct(const struct flow_rule *rule,
 			struct flow_match_ct *out);
 void flow_rule_match_pppoe(const struct flow_rule *rule,
 			   struct flow_match_pppoe *out);
+void flow_rule_match_l2tpv3(const struct flow_rule *rule,
+			    struct flow_match_l2tpv3 *out);
 
 enum flow_action_id {
 	FLOW_ACTION_ACCEPT		= 0,
diff --git a/include/net/genetlink.h b/include/net/genetlink.h
index 56a50e1c51b9..8f780170e2f8 100644
--- a/include/net/genetlink.h
+++ b/include/net/genetlink.h
@@ -39,6 +39,8 @@ struct genl_info;
  *	undo operations done by pre_doit, for example release locks
  * @mcgrps: multicast groups used by this family
  * @n_mcgrps: number of multicast groups
+ * @resv_start_op: first operation for which reserved fields of the header
+ *	can be validated, new families should leave this field at zero
  * @mcgrp_offset: starting number of multicast group IDs in this family
  *	(private)
  * @ops: the operations supported by this family
@@ -58,6 +60,7 @@ struct genl_family {
 	u8			n_ops;
 	u8			n_small_ops;
 	u8			n_mcgrps;
+	u8			resv_start_op;
 	const struct nla_policy *policy;
 	int			(*pre_doit)(const struct genl_ops *ops,
 					    struct sk_buff *skb,
@@ -107,6 +110,13 @@ static inline void genl_info_net_set(struct genl_info *info, struct net *net)
 
 #define GENL_SET_ERR_MSG(info, msg) NL_SET_ERR_MSG((info)->extack, msg)
 
+/* Report that a root attribute is missing */
+#define GENL_REQ_ATTR_CHECK(info, attr) ({				\
+	struct genl_info *__info = (info);				\
+									\
+	NL_REQ_ATTR_CHECK(__info->extack, NULL, __info->attrs, (attr)); \
+})
+
 enum genl_validate_flags {
 	GENL_DONT_VALIDATE_STRICT		= BIT(0),
 	GENL_DONT_VALIDATE_DUMP			= BIT(1),
diff --git a/include/net/gro.h b/include/net/gro.h
index 24003dea8fa4..a4fab706240d 100644
--- a/include/net/gro.h
+++ b/include/net/gro.h
@@ -160,6 +160,17 @@ static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
 	return skb->data + offset;
 }
 
+static inline void *skb_gro_header(struct sk_buff *skb,
+					unsigned int hlen, unsigned int offset)
+{
+	void *ptr;
+
+	ptr = skb_gro_header_fast(skb, offset);
+	if (skb_gro_header_hard(skb, hlen))
+		ptr = skb_gro_header_slow(skb, hlen, offset);
+	return ptr;
+}
+
 static inline void *skb_gro_network_header(struct sk_buff *skb)
 {
 	return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +
@@ -301,12 +312,9 @@ static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr,
 		return ptr;
 	}
 
-	ptr = skb_gro_header_fast(skb, off);
-	if (skb_gro_header_hard(skb, off + plen)) {
-		ptr = skb_gro_header_slow(skb, off + plen, off);
-		if (!ptr)
-			return NULL;
-	}
+	ptr = skb_gro_header(skb, off + plen, off);
+	if (!ptr)
+		return NULL;
 
 	delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,
 			       start, offset);
@@ -329,12 +337,9 @@ static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb,
 	if (!grc->delta)
 		return;
 
-	ptr = skb_gro_header_fast(skb, grc->offset);
-	if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) {
-		ptr = skb_gro_header_slow(skb, plen, grc->offset);
-		if (!ptr)
-			return;
-	}
+	ptr = skb_gro_header(skb, plen, grc->offset);
+	if (!ptr)
+		return;
 
 	remcsum_unadjust((__sum16 *)ptr, grc->delta);
 }
@@ -405,9 +410,7 @@ static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)
 
 	off  = skb_gro_offset(skb);
 	hlen = off + sizeof(*uh);
-	uh   = skb_gro_header_fast(skb, off);
-	if (skb_gro_header_hard(skb, hlen))
-		uh = skb_gro_header_slow(skb, hlen, off);
+	uh   = skb_gro_header(skb, hlen, off);
 
 	return uh;
 }
diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h
index ee88f0f1350f..c2b15f7e5516 100644
--- a/include/net/inet_connection_sock.h
+++ b/include/net/inet_connection_sock.h
@@ -25,6 +25,7 @@
 #undef INET_CSK_CLEAR_TIMERS
 
 struct inet_bind_bucket;
+struct inet_bind2_bucket;
 struct tcp_congestion_ops;
 
 /*
@@ -57,6 +58,7 @@ struct inet_connection_sock_af_ops {
  *
  * @icsk_accept_queue:	   FIFO of established children
  * @icsk_bind_hash:	   Bind node
+ * @icsk_bind2_hash:	   Bind node in the bhash2 table
  * @icsk_timeout:	   Timeout
  * @icsk_retransmit_timer: Resend (no ack)
  * @icsk_rto:		   Retransmit timeout
@@ -83,6 +85,7 @@ struct inet_connection_sock {
 	struct inet_sock	  icsk_inet;
 	struct request_sock_queue icsk_accept_queue;
 	struct inet_bind_bucket	  *icsk_bind_hash;
+	struct inet_bind2_bucket  *icsk_bind2_hash;
 	unsigned long		  icsk_timeout;
  	struct timer_list	  icsk_retransmit_timer;
  	struct timer_list	  icsk_delack_timer;
diff --git a/include/net/inet_hashtables.h b/include/net/inet_hashtables.h
index e9cf2157ed8a..3af1e927247d 100644
--- a/include/net/inet_hashtables.h
+++ b/include/net/inet_hashtables.h
@@ -23,6 +23,7 @@
 
 #include <net/inet_connection_sock.h>
 #include <net/inet_sock.h>
+#include <net/ip.h>
 #include <net/sock.h>
 #include <net/route.h>
 #include <net/tcp_states.h>
@@ -90,7 +91,31 @@ struct inet_bind_bucket {
 	struct hlist_head	owners;
 };
 
-static inline struct net *ib_net(struct inet_bind_bucket *ib)
+struct inet_bind2_bucket {
+	possible_net_t		ib_net;
+	int			l3mdev;
+	unsigned short		port;
+#if IS_ENABLED(CONFIG_IPV6)
+	unsigned short		family;
+#endif
+	union {
+#if IS_ENABLED(CONFIG_IPV6)
+		struct in6_addr		v6_rcv_saddr;
+#endif
+		__be32			rcv_saddr;
+	};
+	/* Node in the bhash2 inet_bind_hashbucket chain */
+	struct hlist_node	node;
+	/* List of sockets hashed to this bucket */
+	struct hlist_head	owners;
+};
+
+static inline struct net *ib_net(const struct inet_bind_bucket *ib)
+{
+	return read_pnet(&ib->ib_net);
+}
+
+static inline struct net *ib2_net(const struct inet_bind2_bucket *ib)
 {
 	return read_pnet(&ib->ib_net);
 }
@@ -133,14 +158,33 @@ struct inet_hashinfo {
 	 * TCP hash as well as the others for fast bind/connect.
 	 */
 	struct kmem_cache		*bind_bucket_cachep;
+	/* This bind table is hashed by local port */
 	struct inet_bind_hashbucket	*bhash;
+	struct kmem_cache		*bind2_bucket_cachep;
+	/* This bind table is hashed by local port and sk->sk_rcv_saddr (ipv4)
+	 * or sk->sk_v6_rcv_saddr (ipv6). This 2nd bind table is used
+	 * primarily for expediting bind conflict resolution.
+	 */
+	struct inet_bind_hashbucket	*bhash2;
 	unsigned int			bhash_size;
 
 	/* The 2nd listener table hashed by local port and address */
 	unsigned int			lhash2_mask;
 	struct inet_listen_hashbucket	*lhash2;
+
+	bool				pernet;
 };
 
+static inline struct inet_hashinfo *tcp_or_dccp_get_hashinfo(const struct sock *sk)
+{
+#if IS_ENABLED(CONFIG_IP_DCCP)
+	return sk->sk_prot->h.hashinfo ? :
+		sock_net(sk)->ipv4.tcp_death_row.hashinfo;
+#else
+	return sock_net(sk)->ipv4.tcp_death_row.hashinfo;
+#endif
+}
+
 static inline struct inet_listen_hashbucket *
 inet_lhash2_bucket(struct inet_hashinfo *h, u32 hash)
 {
@@ -175,6 +219,10 @@ static inline void inet_ehash_locks_free(struct inet_hashinfo *hashinfo)
 	hashinfo->ehash_locks = NULL;
 }
 
+struct inet_hashinfo *inet_pernet_hashinfo_alloc(struct inet_hashinfo *hashinfo,
+						 unsigned int ehash_entries);
+void inet_pernet_hashinfo_free(struct inet_hashinfo *hashinfo);
+
 struct inet_bind_bucket *
 inet_bind_bucket_create(struct kmem_cache *cachep, struct net *net,
 			struct inet_bind_hashbucket *head,
@@ -182,14 +230,61 @@ inet_bind_bucket_create(struct kmem_cache *cachep, struct net *net,
 void inet_bind_bucket_destroy(struct kmem_cache *cachep,
 			      struct inet_bind_bucket *tb);
 
+bool inet_bind_bucket_match(const struct inet_bind_bucket *tb,
+			    const struct net *net, unsigned short port,
+			    int l3mdev);
+
+struct inet_bind2_bucket *
+inet_bind2_bucket_create(struct kmem_cache *cachep, struct net *net,
+			 struct inet_bind_hashbucket *head,
+			 unsigned short port, int l3mdev,
+			 const struct sock *sk);
+
+void inet_bind2_bucket_destroy(struct kmem_cache *cachep,
+			       struct inet_bind2_bucket *tb);
+
+struct inet_bind2_bucket *
+inet_bind2_bucket_find(const struct inet_bind_hashbucket *head,
+		       const struct net *net,
+		       unsigned short port, int l3mdev,
+		       const struct sock *sk);
+
+bool inet_bind2_bucket_match_addr_any(const struct inet_bind2_bucket *tb,
+				      const struct net *net, unsigned short port,
+				      int l3mdev, const struct sock *sk);
+
 static inline u32 inet_bhashfn(const struct net *net, const __u16 lport,
 			       const u32 bhash_size)
 {
 	return (lport + net_hash_mix(net)) & (bhash_size - 1);
 }
 
+static inline struct inet_bind_hashbucket *
+inet_bhashfn_portaddr(const struct inet_hashinfo *hinfo, const struct sock *sk,
+		      const struct net *net, unsigned short port)
+{
+	u32 hash;
+
+#if IS_ENABLED(CONFIG_IPV6)
+	if (sk->sk_family == AF_INET6)
+		hash = ipv6_portaddr_hash(net, &sk->sk_v6_rcv_saddr, port);
+	else
+#endif
+		hash = ipv4_portaddr_hash(net, sk->sk_rcv_saddr, port);
+	return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)];
+}
+
+struct inet_bind_hashbucket *
+inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port);
+
+/* This should be called whenever a socket's sk_rcv_saddr (ipv4) or
+ * sk_v6_rcv_saddr (ipv6) changes after it has been binded. The socket's
+ * rcv_saddr field should already have been updated when this is called.
+ */
+int inet_bhash2_update_saddr(struct inet_bind_hashbucket *prev_saddr, struct sock *sk);
+
 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
-		    const unsigned short snum);
+		    struct inet_bind2_bucket *tb2, unsigned short port);
 
 /* Caller must disable local BH processing. */
 int __inet_inherit_port(const struct sock *sk, struct sock *child);
diff --git a/include/net/ip.h b/include/net/ip.h
index 1c979fd1904c..038097c2a152 100644
--- a/include/net/ip.h
+++ b/include/net/ip.h
@@ -743,8 +743,12 @@ void ip_cmsg_recv_offset(struct msghdr *msg, struct sock *sk,
 int ip_cmsg_send(struct sock *sk, struct msghdr *msg,
 		 struct ipcm_cookie *ipc, bool allow_ipv6);
 DECLARE_STATIC_KEY_FALSE(ip4_min_ttl);
+int do_ip_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
+		     unsigned int optlen);
 int ip_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
 		  unsigned int optlen);
+int do_ip_getsockopt(struct sock *sk, int level, int optname,
+		     sockptr_t optval, sockptr_t optlen);
 int ip_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
 		  int __user *optlen);
 int ip_ra_control(struct sock *sk, unsigned char on,
diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h
index ced80e2f8b58..fca357679816 100644
--- a/include/net/ip_tunnels.h
+++ b/include/net/ip_tunnels.h
@@ -302,6 +302,12 @@ int ip_tunnel_newlink(struct net_device *dev, struct nlattr *tb[],
 		      struct ip_tunnel_parm *p, __u32 fwmark);
 void ip_tunnel_setup(struct net_device *dev, unsigned int net_id);
 
+bool ip_tunnel_netlink_encap_parms(struct nlattr *data[],
+				   struct ip_tunnel_encap *encap);
+
+void ip_tunnel_netlink_parms(struct nlattr *data[],
+			     struct ip_tunnel_parm *parms);
+
 extern const struct header_ops ip_tunnel_header_ops;
 __be16 ip_tunnel_parse_protocol(const struct sk_buff *skb);
 
diff --git a/include/net/ipcomp.h b/include/net/ipcomp.h
index c31108295079..8660a2a6d1fc 100644
--- a/include/net/ipcomp.h
+++ b/include/net/ipcomp.h
@@ -22,7 +22,7 @@ struct xfrm_state;
 int ipcomp_input(struct xfrm_state *x, struct sk_buff *skb);
 int ipcomp_output(struct xfrm_state *x, struct sk_buff *skb);
 void ipcomp_destroy(struct xfrm_state *x);
-int ipcomp_init_state(struct xfrm_state *x);
+int ipcomp_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack);
 
 static inline struct ip_comp_hdr *ip_comp_hdr(const struct sk_buff *skb)
 {
diff --git a/include/net/ipv6.h b/include/net/ipv6.h
index de9dcc5652c4..d664ba5812d8 100644
--- a/include/net/ipv6.h
+++ b/include/net/ipv6.h
@@ -1156,8 +1156,12 @@ struct in6_addr *fl6_update_dst(struct flowi6 *fl6,
  */
 DECLARE_STATIC_KEY_FALSE(ip6_min_hopcount);
 
+int do_ipv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
+		       unsigned int optlen);
 int ipv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
 		    unsigned int optlen);
+int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
+		       sockptr_t optval, sockptr_t optlen);
 int ipv6_getsockopt(struct sock *sk, int level, int optname,
 		    char __user *optval, int __user *optlen);
 
@@ -1207,7 +1211,7 @@ int ip6_mc_source(int add, int omode, struct sock *sk,
 int ip6_mc_msfilter(struct sock *sk, struct group_filter *gsf,
 		  struct sockaddr_storage *list);
 int ip6_mc_msfget(struct sock *sk, struct group_filter *gsf,
-		  struct sockaddr_storage __user *p);
+		  sockptr_t optval, size_t ss_offset);
 
 #ifdef CONFIG_PROC_FS
 int ac6_proc_init(struct net *net);
diff --git a/include/net/ipv6_stubs.h b/include/net/ipv6_stubs.h
index 45e0339be6fa..c48186bf4737 100644
--- a/include/net/ipv6_stubs.h
+++ b/include/net/ipv6_stubs.h
@@ -81,6 +81,10 @@ struct ipv6_bpf_stub {
 				     const struct in6_addr *daddr, __be16 dport,
 				     int dif, int sdif, struct udp_table *tbl,
 				     struct sk_buff *skb);
+	int (*ipv6_setsockopt)(struct sock *sk, int level, int optname,
+			       sockptr_t optval, unsigned int optlen);
+	int (*ipv6_getsockopt)(struct sock *sk, int level, int optname,
+			       sockptr_t optval, sockptr_t optlen);
 };
 extern const struct ipv6_bpf_stub *ipv6_bpf_stub __read_mostly;
 
diff --git a/include/net/mac80211.h b/include/net/mac80211.h
index f198af600b5e..ac2bad57933f 100644
--- a/include/net/mac80211.h
+++ b/include/net/mac80211.h
@@ -18,6 +18,7 @@
 #include <linux/if_ether.h>
 #include <linux/skbuff.h>
 #include <linux/ieee80211.h>
+#include <linux/lockdep.h>
 #include <net/cfg80211.h>
 #include <net/codel.h>
 #include <net/ieee80211_radiotap.h>
@@ -1480,6 +1481,10 @@ enum mac80211_rx_encoding {
  *	each A-MPDU but the same for each subframe within one A-MPDU
  * @ampdu_delimiter_crc: A-MPDU delimiter CRC
  * @zero_length_psdu_type: radiotap type of the 0-length PSDU
+ * @link_valid: if the link which is identified by @link_id is valid. This flag
+ *	is set only when connection is MLO.
+ * @link_id: id of the link used to receive the packet. This is used along with
+ *	@link_valid.
  */
 struct ieee80211_rx_status {
 	u64 mactime;
@@ -1504,6 +1509,7 @@ struct ieee80211_rx_status {
 	s8 chain_signal[IEEE80211_MAX_CHAINS];
 	u8 ampdu_delimiter_crc;
 	u8 zero_length_psdu_type;
+	u8 link_valid:1, link_id:4;
 };
 
 static inline u32
@@ -1794,6 +1800,9 @@ struct ieee80211_vif_cfg {
  * @link_conf: in case of MLD, the per-link BSS configuration,
  *	indexed by link ID
  * @valid_links: bitmap of valid links, or 0 for non-MLO.
+ * @active_links: The bitmap of active links, or 0 for non-MLO.
+ *	The driver shouldn't change this directly, but use the
+ *	API calls meant for that purpose.
  * @addr: address of this interface
  * @p2p: indicates whether this AP or STA interface is a p2p
  *	interface, i.e. a GO or p2p-sta respectively
@@ -1829,7 +1838,7 @@ struct ieee80211_vif {
 	struct ieee80211_vif_cfg cfg;
 	struct ieee80211_bss_conf bss_conf;
 	struct ieee80211_bss_conf __rcu *link_conf[IEEE80211_MLD_MAX_NUM_LINKS];
-	u16 valid_links;
+	u16 valid_links, active_links;
 	u8 addr[ETH_ALEN] __aligned(2);
 	bool p2p;
 
@@ -1856,12 +1865,11 @@ struct ieee80211_vif {
 	u8 drv_priv[] __aligned(sizeof(void *));
 };
 
-/* FIXME: for now loop over all the available links; later will be changed
- * to loop only over the active links.
- */
-#define for_each_vif_active_link(vif, link, link_id)			     \
-	for (link_id = 0; link_id < ARRAY_SIZE((vif)->link_conf); link_id++) \
-		if ((link = rcu_dereference((vif)->link_conf[link_id])))
+#define for_each_vif_active_link(vif, link, link_id)				\
+	for (link_id = 0; link_id < ARRAY_SIZE((vif)->link_conf); link_id++)	\
+		if ((!(vif)->active_links ||					\
+		     (vif)->active_links & BIT(link_id)) &&			\
+		    (link = rcu_dereference((vif)->link_conf[link_id])))
 
 static inline bool ieee80211_vif_is_mesh(struct ieee80211_vif *vif)
 {
@@ -1895,6 +1903,19 @@ struct ieee80211_vif *wdev_to_ieee80211_vif(struct wireless_dev *wdev);
 struct wireless_dev *ieee80211_vif_to_wdev(struct ieee80211_vif *vif);
 
 /**
+ * lockdep_vif_mutex_held - for lockdep checks on link poiners
+ * @vif: the interface to check
+ */
+static inline bool lockdep_vif_mutex_held(struct ieee80211_vif *vif)
+{
+	return lockdep_is_held(&ieee80211_vif_to_wdev(vif)->mtx);
+}
+
+#define link_conf_dereference_protected(vif, link_id)		\
+	rcu_dereference_protected((vif)->link_conf[link_id],	\
+				  lockdep_vif_mutex_held(vif))
+
+/**
  * enum ieee80211_key_flags - key flags
  *
  * These flags are used for communication about keys between the driver
@@ -1975,6 +1996,7 @@ enum ieee80211_key_flags {
  * 	- Temporal Authenticator Rx MIC Key (64 bits)
  * @icv_len: The ICV length for this key type
  * @iv_len: The IV length for this key type
+ * @link_id: the link ID for MLO, or -1 for non-MLO or pairwise keys
  */
 struct ieee80211_key_conf {
 	atomic64_t tx_pn;
@@ -1984,6 +2006,7 @@ struct ieee80211_key_conf {
 	u8 hw_key_idx;
 	s8 keyidx;
 	u16 flags;
+	s8 link_id;
 	u8 keylen;
 	u8 key[];
 };
@@ -2121,6 +2144,34 @@ struct ieee80211_sta_txpwr {
 };
 
 /**
+ * struct ieee80211_sta_aggregates - info that is aggregated from active links
+ *
+ * Used for any per-link data that needs to be aggregated and updated in the
+ * main &struct ieee80211_sta when updated or the active links change.
+ *
+ * @max_amsdu_len: indicates the maximal length of an A-MSDU in bytes.
+ *	This field is always valid for packets with a VHT preamble.
+ *	For packets with a HT preamble, additional limits apply:
+ *
+ *	* If the skb is transmitted as part of a BA agreement, the
+ *	  A-MSDU maximal size is min(max_amsdu_len, 4065) bytes.
+ *	* If the skb is not part of a BA agreement, the A-MSDU maximal
+ *	  size is min(max_amsdu_len, 7935) bytes.
+ *
+ * Both additional HT limits must be enforced by the low level
+ * driver. This is defined by the spec (IEEE 802.11-2012 section
+ * 8.3.2.2 NOTE 2).
+ * @max_rc_amsdu_len: Maximum A-MSDU size in bytes recommended by rate control.
+ * @max_tid_amsdu_len: Maximum A-MSDU size in bytes for this TID
+ */
+struct ieee80211_sta_aggregates {
+	u16 max_amsdu_len;
+
+	u16 max_rc_amsdu_len;
+	u16 max_tid_amsdu_len[IEEE80211_NUM_TIDS];
+};
+
+/**
  * struct ieee80211_link_sta - station Link specific info
  * All link specific info for a STA link for a non MLD STA(single)
  * or a MLD STA(multiple entries) are stored here.
@@ -2128,6 +2179,8 @@ struct ieee80211_sta_txpwr {
  * @addr: MAC address of the Link STA. For non-MLO STA this is same as the addr
  *	in ieee80211_sta. For MLO Link STA this addr can be same or different
  *	from addr in ieee80211_sta (representing MLD STA addr)
+ * @link_id: the link ID for this link STA (0 for deflink)
+ * @smps_mode: current SMPS mode (off, static or dynamic)
  * @supp_rates: Bitmap of supported rates
  * @ht_cap: HT capabilities of this STA; restricted to our own capabilities
  * @vht_cap: VHT capabilities of this STA; restricted to our own capabilities
@@ -2144,6 +2197,8 @@ struct ieee80211_sta_txpwr {
  */
 struct ieee80211_link_sta {
 	u8 addr[ETH_ALEN];
+	u8 link_id;
+	enum ieee80211_smps_mode smps_mode;
 
 	u32 supp_rates[NUM_NL80211_BANDS];
 	struct ieee80211_sta_ht_cap ht_cap;
@@ -2152,6 +2207,8 @@ struct ieee80211_link_sta {
 	struct ieee80211_he_6ghz_capa he_6ghz_capa;
 	struct ieee80211_sta_eht_cap eht_cap;
 
+	struct ieee80211_sta_aggregates agg;
+
 	u8 rx_nss;
 	enum ieee80211_sta_rx_bandwidth bandwidth;
 	struct ieee80211_sta_txpwr txpwr;
@@ -2182,7 +2239,6 @@ struct ieee80211_link_sta {
  *	if wme is supported. The bits order is like in
  *	IEEE80211_WMM_IE_STA_QOSINFO_AC_*.
  * @max_sp: max Service Period. Only valid if wme is supported.
- * @smps_mode: current SMPS mode (off, static or dynamic)
  * @rates: rate control selection table
  * @tdls: indicates whether the STA is a TDLS peer
  * @tdls_initiator: indicates the STA is an initiator of the TDLS link. Only
@@ -2192,9 +2248,10 @@ struct ieee80211_link_sta {
  * @max_amsdu_subframes: indicates the maximal number of MSDUs in a single
  *	A-MSDU. Taken from the Extended Capabilities element. 0 means
  *	unlimited.
+ * @cur: currently valid data as aggregated from the active links
+ *	For non MLO STA it will point to the deflink data. For MLO STA
+ *	ieee80211_sta_recalc_aggregates() must be called to update it.
  * @support_p2p_ps: indicates whether the STA supports P2P PS mechanism or not.
- * @max_rc_amsdu_len: Maximum A-MSDU size in bytes recommended by rate control.
- * @max_tid_amsdu_len: Maximum A-MSDU size in bytes for this TID
  * @txq: per-TID data TX queues (if driver uses the TXQ abstraction); note that
  *	the last entry (%IEEE80211_NUM_TIDS) is used for non-data frames
  * @deflink: This holds the default link STA information, for non MLO STA all link
@@ -2217,7 +2274,6 @@ struct ieee80211_sta {
 	bool wme;
 	u8 uapsd_queues;
 	u8 max_sp;
-	enum ieee80211_smps_mode smps_mode;
 	struct ieee80211_sta_rates __rcu *rates;
 	bool tdls;
 	bool tdls_initiator;
@@ -2225,25 +2281,9 @@ struct ieee80211_sta {
 	bool mlo;
 	u8 max_amsdu_subframes;
 
-	/**
-	 * @max_amsdu_len:
-	 * indicates the maximal length of an A-MSDU in bytes.
-	 * This field is always valid for packets with a VHT preamble.
-	 * For packets with a HT preamble, additional limits apply:
-	 *
-	 * * If the skb is transmitted as part of a BA agreement, the
-	 *   A-MSDU maximal size is min(max_amsdu_len, 4065) bytes.
-	 * * If the skb is not part of a BA agreement, the A-MSDU maximal
-	 *   size is min(max_amsdu_len, 7935) bytes.
-	 *
-	 * Both additional HT limits must be enforced by the low level
-	 * driver. This is defined by the spec (IEEE 802.11-2012 section
-	 * 8.3.2.2 NOTE 2).
-	 */
-	u16 max_amsdu_len;
+	struct ieee80211_sta_aggregates *cur;
+
 	bool support_p2p_ps;
-	u16 max_rc_amsdu_len;
-	u16 max_tid_amsdu_len[IEEE80211_NUM_TIDS];
 
 	struct ieee80211_txq *txq[IEEE80211_NUM_TIDS + 1];
 
@@ -2255,13 +2295,24 @@ struct ieee80211_sta {
 	u8 drv_priv[] __aligned(sizeof(void *));
 };
 
-/* FIXME: need to loop only over links which are active and check the actual
- * lock
- */
-#define for_each_sta_active_link(sta, link_sta, link_id)		         \
-	for (link_id = 0; link_id < ARRAY_SIZE((sta)->link); link_id++)	         \
-		if (((link_sta) = rcu_dereference_protected((sta)->link[link_id],\
-							    1)))	         \
+#ifdef CONFIG_LOCKDEP
+bool lockdep_sta_mutex_held(struct ieee80211_sta *pubsta);
+#else
+static inline bool lockdep_sta_mutex_held(struct ieee80211_sta *pubsta)
+{
+	return true;
+}
+#endif
+
+#define link_sta_dereference_protected(sta, link_id)		\
+	rcu_dereference_protected((sta)->link[link_id],		\
+				  lockdep_sta_mutex_held(sta))
+
+#define for_each_sta_active_link(vif, sta, link_sta, link_id)			\
+	for (link_id = 0; link_id < ARRAY_SIZE((sta)->link); link_id++)		\
+		if ((!(vif)->active_links ||					\
+		     (vif)->active_links & BIT(link_id)) &&			\
+		    ((link_sta) = link_sta_dereference_protected(sta, link_id)))
 
 /**
  * enum sta_notify_cmd - sta notify command
@@ -5287,6 +5338,9 @@ struct sk_buff *ieee80211_pspoll_get(struct ieee80211_hw *hw,
  * ieee80211_nullfunc_get - retrieve a nullfunc template
  * @hw: pointer obtained from ieee80211_alloc_hw().
  * @vif: &struct ieee80211_vif pointer from the add_interface callback.
+ * @link_id: If the vif is an MLD, get a frame with the link addresses
+ *	for the given link ID. For a link_id < 0 you get a frame with
+ *	MLD addresses, however useful that might be.
  * @qos_ok: QoS NDP is acceptable to the caller, this should be set
  *	if at all possible
  *
@@ -5304,7 +5358,7 @@ struct sk_buff *ieee80211_pspoll_get(struct ieee80211_hw *hw,
  */
 struct sk_buff *ieee80211_nullfunc_get(struct ieee80211_hw *hw,
 				       struct ieee80211_vif *vif,
-				       bool qos_ok);
+				       int link_id, bool qos_ok);
 
 /**
  * ieee80211_probereq_get - retrieve a Probe Request template
@@ -5976,6 +6030,22 @@ struct ieee80211_sta *ieee80211_find_sta_by_ifaddr(struct ieee80211_hw *hw,
 					       const u8 *localaddr);
 
 /**
+ * ieee80211_find_sta_by_link_addrs - find STA by link addresses
+ * @hw: pointer as obtained from ieee80211_alloc_hw()
+ * @addr: remote station's link address
+ * @localaddr: local link address, use %NULL for any (but avoid that)
+ * @link_id: pointer to obtain the link ID if the STA is found,
+ *	may be %NULL if the link ID is not needed
+ *
+ * Obtain the STA by link address, must use RCU protection.
+ */
+struct ieee80211_sta *
+ieee80211_find_sta_by_link_addrs(struct ieee80211_hw *hw,
+				 const u8 *addr,
+				 const u8 *localaddr,
+				 unsigned int *link_id);
+
+/**
  * ieee80211_sta_block_awake - block station from waking up
  * @hw: the hardware
  * @pubsta: the station
@@ -6051,6 +6121,19 @@ void ieee80211_sta_eosp(struct ieee80211_sta *pubsta);
 void ieee80211_send_eosp_nullfunc(struct ieee80211_sta *pubsta, int tid);
 
 /**
+ * ieee80211_sta_recalc_aggregates - recalculate aggregate data after a change
+ * @pubsta: the station
+ *
+ * Call this function after changing a per-link aggregate data as referenced in
+ * &struct ieee80211_sta_aggregates by accessing the agg field of
+ * &struct ieee80211_link_sta.
+ *
+ * With non MLO the data in deflink will be referenced directly. In that case
+ * there is no need to call this function.
+ */
+void ieee80211_sta_recalc_aggregates(struct ieee80211_sta *pubsta);
+
+/**
  * ieee80211_sta_register_airtime - register airtime usage for a sta/tid
  *
  * Register airtime usage for a given sta on a given tid. The driver must call
@@ -7101,4 +7184,45 @@ static inline bool ieee80211_is_tx_data(struct sk_buff *skb)
 	       ieee80211_is_data(hdr->frame_control);
 }
 
+/**
+ * ieee80211_set_active_links - set active links in client mode
+ * @vif: interface to set active links on
+ * @active_links: the new active links bitmap
+ *
+ * This changes the active links on an interface. The interface
+ * must be in client mode (in AP mode, all links are always active),
+ * and @active_links must be a subset of the vif's valid_links.
+ *
+ * If a link is switched off and another is switched on at the same
+ * time (e.g. active_links going from 0x1 to 0x10) then you will get
+ * a sequence of calls like
+ *  - change_vif_links(0x11)
+ *  - unassign_vif_chanctx(link_id=0)
+ *  - change_sta_links(0x11) for each affected STA (the AP)
+ *    (TDLS connections on now inactive links should be torn down)
+ *  - remove group keys on the old link (link_id 0)
+ *  - add new group keys (GTK/IGTK/BIGTK) on the new link (link_id 4)
+ *  - change_sta_links(0x10) for each affected STA (the AP)
+ *  - assign_vif_chanctx(link_id=4)
+ *  - change_vif_links(0x10)
+ *
+ * Note: This function acquires some mac80211 locks and must not
+ *	 be called with any driver locks held that could cause a
+ *	 lock dependency inversion. Best call it without locks.
+ */
+int ieee80211_set_active_links(struct ieee80211_vif *vif, u16 active_links);
+
+/**
+ * ieee80211_set_active_links_async - asynchronously set active links
+ * @vif: interface to set active links on
+ * @active_links: the new active links bitmap
+ *
+ * See ieee80211_set_active_links() for more information, the only
+ * difference here is that the link change is triggered async and
+ * can be called in any context, but the link switch will only be
+ * completed after it returns.
+ */
+void ieee80211_set_active_links_async(struct ieee80211_vif *vif,
+				      u16 active_links);
+
 #endif /* MAC80211_H */
diff --git a/include/net/macsec.h b/include/net/macsec.h
index d6fa6b97f6ef..5b9c61c4d3a6 100644
--- a/include/net/macsec.h
+++ b/include/net/macsec.h
@@ -14,12 +14,27 @@
 #define MACSEC_DEFAULT_PN_LEN 4
 #define MACSEC_XPN_PN_LEN 8
 
-#define MACSEC_SALT_LEN 12
 #define MACSEC_NUM_AN 4 /* 2 bits for the association number */
 
+#define MACSEC_SCI_LEN 8
+#define MACSEC_PORT_ES (htons(0x0001))
+
+#define MACSEC_TCI_VERSION 0x80
+#define MACSEC_TCI_ES      0x40 /* end station */
+#define MACSEC_TCI_SC      0x20 /* SCI present */
+#define MACSEC_TCI_SCB     0x10 /* epon */
+#define MACSEC_TCI_E       0x08 /* encryption */
+#define MACSEC_TCI_C       0x04 /* changed text */
+#define MACSEC_AN_MASK     0x03 /* association number */
+#define MACSEC_TCI_CONFID  (MACSEC_TCI_E | MACSEC_TCI_C)
+
+#define MACSEC_DEFAULT_ICV_LEN 16
+
 typedef u64 __bitwise sci_t;
 typedef u32 __bitwise ssci_t;
 
+struct metadata_dst;
+
 typedef union salt {
 	struct {
 		u32 ssci;
@@ -183,6 +198,7 @@ struct macsec_tx_sa {
  * @scb: single copy broadcast flag
  * @sa: array of secure associations
  * @stats: stats for this TXSC
+ * @md_dst: MACsec offload metadata dst
  */
 struct macsec_tx_sc {
 	bool active;
@@ -193,6 +209,7 @@ struct macsec_tx_sc {
 	bool scb;
 	struct macsec_tx_sa __rcu *sa[MACSEC_NUM_AN];
 	struct pcpu_tx_sc_stats __percpu *stats;
+	struct metadata_dst *md_dst;
 };
 
 /**
@@ -254,8 +271,6 @@ struct macsec_context {
 		struct macsec_rx_sa_stats *rx_sa_stats;
 		struct macsec_dev_stats  *dev_stats;
 	} stats;
-
-	u8 prepare:1;
 };
 
 /**
@@ -289,5 +304,12 @@ struct macsec_ops {
 };
 
 void macsec_pn_wrapped(struct macsec_secy *secy, struct macsec_tx_sa *tx_sa);
+static inline bool macsec_send_sci(const struct macsec_secy *secy)
+{
+	const struct macsec_tx_sc *tx_sc = &secy->tx_sc;
+
+	return tx_sc->send_sci ||
+		(secy->n_rx_sc > 1 && !tx_sc->end_station && !tx_sc->scb);
+}
 
 #endif /* _NET_MACSEC_H_ */
diff --git a/include/net/neighbour.h b/include/net/neighbour.h
index 3827a6b395fd..20745cf7ae1a 100644
--- a/include/net/neighbour.h
+++ b/include/net/neighbour.h
@@ -276,11 +276,6 @@ static inline void *neighbour_priv(const struct neighbour *n)
 
 extern const struct nla_policy nda_policy[];
 
-static inline bool neigh_key_eq16(const struct neighbour *n, const void *pkey)
-{
-	return *(const u16 *)n->primary_key == *(const u16 *)pkey;
-}
-
 static inline bool neigh_key_eq32(const struct neighbour *n, const void *pkey)
 {
 	return *(const u32 *)n->primary_key == *(const u32 *)pkey;
diff --git a/include/net/netfilter/nf_conntrack_bpf.h b/include/net/netfilter/nf_conntrack_bpf.h
index a473b56842c5..2d0da478c8e0 100644
--- a/include/net/netfilter/nf_conntrack_bpf.h
+++ b/include/net/netfilter/nf_conntrack_bpf.h
@@ -3,13 +3,18 @@
 #ifndef _NF_CONNTRACK_BPF_H
 #define _NF_CONNTRACK_BPF_H
 
-#include <linux/btf.h>
 #include <linux/kconfig.h>
+#include <net/netfilter/nf_conntrack.h>
+
+struct nf_conn___init {
+	struct nf_conn ct;
+};
 
 #if (IS_BUILTIN(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \
     (IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES))
 
 extern int register_nf_conntrack_bpf(void);
+extern void cleanup_nf_conntrack_bpf(void);
 
 #else
 
@@ -18,6 +23,24 @@ static inline int register_nf_conntrack_bpf(void)
 	return 0;
 }
 
+static inline void cleanup_nf_conntrack_bpf(void)
+{
+}
+
+#endif
+
+#if (IS_BUILTIN(CONFIG_NF_NAT) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \
+    (IS_MODULE(CONFIG_NF_NAT) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES))
+
+extern int register_nf_nat_bpf(void);
+
+#else
+
+static inline int register_nf_nat_bpf(void)
+{
+	return 0;
+}
+
 #endif
 
 #endif /* _NF_CONNTRACK_BPF_H */
diff --git a/include/net/netfilter/nf_conntrack_core.h b/include/net/netfilter/nf_conntrack_core.h
index 3cd3a6e631aa..b2b9de70d9f4 100644
--- a/include/net/netfilter/nf_conntrack_core.h
+++ b/include/net/netfilter/nf_conntrack_core.h
@@ -86,10 +86,6 @@ extern spinlock_t nf_conntrack_expect_lock;
 
 /* ctnetlink code shared by both ctnetlink and nf_conntrack_bpf */
 
-#if (IS_BUILTIN(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \
-    (IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES) || \
-    IS_ENABLED(CONFIG_NF_CT_NETLINK))
-
 static inline void __nf_ct_set_timeout(struct nf_conn *ct, u64 timeout)
 {
 	if (timeout > INT_MAX)
@@ -101,6 +97,4 @@ int __nf_ct_change_timeout(struct nf_conn *ct, u64 cta_timeout);
 void __nf_ct_change_status(struct nf_conn *ct, unsigned long on, unsigned long off);
 int nf_ct_change_status_common(struct nf_conn *ct, unsigned int status);
 
-#endif
-
 #endif /* _NF_CONNTRACK_CORE_H */
diff --git a/include/net/netfilter/nf_nat_helper.h b/include/net/netfilter/nf_nat_helper.h
index efae84646353..44c421b9be85 100644
--- a/include/net/netfilter/nf_nat_helper.h
+++ b/include/net/netfilter/nf_nat_helper.h
@@ -38,4 +38,5 @@ bool nf_nat_mangle_udp_packet(struct sk_buff *skb, struct nf_conn *ct,
  * to port ct->master->saved_proto. */
 void nf_nat_follow_master(struct nf_conn *ct, struct nf_conntrack_expect *this);
 
+u16 nf_nat_exp_find_port(struct nf_conntrack_expect *exp, u16 port);
 #endif
diff --git a/include/net/netlink.h b/include/net/netlink.h
index 7a2a9d3144ba..4418b1981e31 100644
--- a/include/net/netlink.h
+++ b/include/net/netlink.h
@@ -325,6 +325,7 @@ struct nla_policy {
 		struct netlink_range_validation_signed *range_signed;
 		struct {
 			s16 min, max;
+			u8 network_byte_order:1;
 		};
 		int (*validate)(const struct nlattr *attr,
 				struct netlink_ext_ack *extack);
@@ -418,6 +419,14 @@ struct nla_policy {
 	.type = NLA_ENSURE_INT_OR_BINARY_TYPE(tp),	\
 	.validation_type = NLA_VALIDATE_MAX,		\
 	.max = _max,					\
+	.network_byte_order = 0,			\
+}
+
+#define NLA_POLICY_MAX_BE(tp, _max) {			\
+	.type = NLA_ENSURE_UINT_TYPE(tp),		\
+	.validation_type = NLA_VALIDATE_MAX,		\
+	.max = _max,					\
+	.network_byte_order = 1,			\
 }
 
 #define NLA_POLICY_MASK(tp, _mask) {			\
@@ -741,6 +750,7 @@ static inline int __nlmsg_parse(const struct nlmsghdr *nlh, int hdrlen,
  * @hdrlen: length of family specific header
  * @tb: destination array with maxtype+1 elements
  * @maxtype: maximum attribute type to be expected
+ * @policy: validation policy
  * @extack: extended ACK report struct
  *
  * See nla_parse()
@@ -760,6 +770,7 @@ static inline int nlmsg_parse(const struct nlmsghdr *nlh, int hdrlen,
  * @hdrlen: length of family specific header
  * @tb: destination array with maxtype+1 elements
  * @maxtype: maximum attribute type to be expected
+ * @policy: validation policy
  * @extack: extended ACK report struct
  *
  * See nla_parse_deprecated()
@@ -779,6 +790,7 @@ static inline int nlmsg_parse_deprecated(const struct nlmsghdr *nlh, int hdrlen,
  * @hdrlen: length of family specific header
  * @tb: destination array with maxtype+1 elements
  * @maxtype: maximum attribute type to be expected
+ * @policy: validation policy
  * @extack: extended ACK report struct
  *
  * See nla_parse_deprecated_strict()
@@ -814,7 +826,6 @@ static inline struct nlattr *nlmsg_find_attr(const struct nlmsghdr *nlh,
  * @len: length of attribute stream
  * @maxtype: maximum attribute type to be expected
  * @policy: validation policy
- * @validate: validation strictness
  * @extack: extended ACK report struct
  *
  * Validates all attributes in the specified attribute stream against the
diff --git a/include/net/netns/generic.h b/include/net/netns/generic.h
index 7ce68183f6e1..00c399edeed1 100644
--- a/include/net/netns/generic.h
+++ b/include/net/netns/generic.h
@@ -33,7 +33,7 @@ struct net_generic {
 			struct rcu_head rcu;
 		} s;
 
-		void *ptr[0];
+		DECLARE_FLEX_ARRAY(void *, ptr);
 	};
 };
 
diff --git a/include/net/netns/ipv4.h b/include/net/netns/ipv4.h
index 6320a76cefdc..1b8004679445 100644
--- a/include/net/netns/ipv4.h
+++ b/include/net/netns/ipv4.h
@@ -34,6 +34,7 @@ struct inet_hashinfo;
 struct inet_timewait_death_row {
 	refcount_t		tw_refcount;
 
+	/* Padding to avoid false sharing, tw_refcount can be often written */
 	struct inet_hashinfo 	*hashinfo ____cacheline_aligned_in_smp;
 	int			sysctl_max_tw_buckets;
 };
@@ -41,7 +42,7 @@ struct inet_timewait_death_row {
 struct tcp_fastopen_context;
 
 struct netns_ipv4 {
-	struct inet_timewait_death_row *tcp_death_row;
+	struct inet_timewait_death_row tcp_death_row;
 
 #ifdef CONFIG_SYSCTL
 	struct ctl_table_header	*forw_hdr;
@@ -170,6 +171,7 @@ struct netns_ipv4 {
 	int sysctl_tcp_pacing_ca_ratio;
 	int sysctl_tcp_wmem[3];
 	int sysctl_tcp_rmem[3];
+	unsigned int sysctl_tcp_child_ehash_entries;
 	unsigned long sysctl_tcp_comp_sack_delay_ns;
 	unsigned long sysctl_tcp_comp_sack_slack_ns;
 	int sysctl_max_syn_backlog;
diff --git a/include/net/netns/netfilter.h b/include/net/netns/netfilter.h
index b593f95e9991..02bbdc577f8e 100644
--- a/include/net/netns/netfilter.h
+++ b/include/net/netns/netfilter.h
@@ -24,9 +24,6 @@ struct netns_nf {
 #ifdef CONFIG_NETFILTER_FAMILY_BRIDGE
 	struct nf_hook_entries __rcu *hooks_bridge[NF_INET_NUMHOOKS];
 #endif
-#if IS_ENABLED(CONFIG_DECNET)
-	struct nf_hook_entries __rcu *hooks_decnet[NF_DN_NUMHOOKS];
-#endif
 #if IS_ENABLED(CONFIG_NF_DEFRAG_IPV4)
 	unsigned int defrag_ipv4_users;
 #endif
diff --git a/include/net/netns/smc.h b/include/net/netns/smc.h
index 2adbe2b245df..582212ada3ba 100644
--- a/include/net/netns/smc.h
+++ b/include/net/netns/smc.h
@@ -19,5 +19,8 @@ struct netns_smc {
 #endif
 	unsigned int			sysctl_autocorking_size;
 	unsigned int			sysctl_smcr_buf_type;
+	int				sysctl_smcr_testlink_time;
+	int				sysctl_wmem;
+	int				sysctl_rmem;
 };
 #endif
diff --git a/include/net/nl802154.h b/include/net/nl802154.h
index 145acb8f2509..f5850b569c52 100644
--- a/include/net/nl802154.h
+++ b/include/net/nl802154.h
@@ -58,9 +58,6 @@ enum nl802154_commands {
 
 	NL802154_CMD_SET_WPAN_PHY_NETNS,
 
-	/* add new commands above here */
-
-#ifdef CONFIG_IEEE802154_NL802154_EXPERIMENTAL
 	NL802154_CMD_SET_SEC_PARAMS,
 	NL802154_CMD_GET_SEC_KEY,		/* can dump */
 	NL802154_CMD_NEW_SEC_KEY,
@@ -74,7 +71,8 @@ enum nl802154_commands {
 	NL802154_CMD_GET_SEC_LEVEL,		/* can dump */
 	NL802154_CMD_NEW_SEC_LEVEL,
 	NL802154_CMD_DEL_SEC_LEVEL,
-#endif /* CONFIG_IEEE802154_NL802154_EXPERIMENTAL */
+
+	/* add new commands above here */
 
 	/* used to define NL802154_CMD_MAX below */
 	__NL802154_CMD_AFTER_LAST,
diff --git a/include/net/pkt_cls.h b/include/net/pkt_cls.h
index d9d90e6925e1..4cabb32a2ad9 100644
--- a/include/net/pkt_cls.h
+++ b/include/net/pkt_cls.h
@@ -81,6 +81,19 @@ int tcf_classify(struct sk_buff *skb,
 		 const struct tcf_proto *tp, struct tcf_result *res,
 		 bool compat_mode);
 
+static inline bool tc_cls_stats_dump(struct tcf_proto *tp,
+				     struct tcf_walker *arg,
+				     void *filter)
+{
+	if (arg->count >= arg->skip && arg->fn(tp, filter, arg) < 0) {
+		arg->stop = 1;
+		return false;
+	}
+
+	arg->count++;
+	return true;
+}
+
 #else
 static inline bool tcf_block_shared(struct tcf_block *block)
 {
@@ -197,6 +210,18 @@ tcf_unbind_filter(struct tcf_proto *tp, struct tcf_result *r)
 	__tcf_unbind_filter(q, r);
 }
 
+static inline void tc_cls_bind_class(u32 classid, unsigned long cl,
+				     void *q, struct tcf_result *res,
+				     unsigned long base)
+{
+	if (res->classid == classid) {
+		if (cl)
+			__tcf_bind_filter(q, res, base);
+		else
+			__tcf_unbind_filter(q, res);
+	}
+}
+
 struct tcf_exts {
 #ifdef CONFIG_NET_CLS_ACT
 	__u32	type; /* for backward compat(TCA_OLD_COMPAT) */
diff --git a/include/net/pkt_sched.h b/include/net/pkt_sched.h
index 3372a1f67cf4..38207873eda6 100644
--- a/include/net/pkt_sched.h
+++ b/include/net/pkt_sched.h
@@ -100,7 +100,7 @@ struct Qdisc *fifo_create_dflt(struct Qdisc *sch, struct Qdisc_ops *ops,
 			       struct netlink_ext_ack *extack);
 
 int register_qdisc(struct Qdisc_ops *qops);
-int unregister_qdisc(struct Qdisc_ops *qops);
+void unregister_qdisc(struct Qdisc_ops *qops);
 void qdisc_get_default(char *id, size_t len);
 int qdisc_set_default(const char *id);
 
@@ -141,6 +141,11 @@ static inline struct net *qdisc_net(struct Qdisc *q)
 	return dev_net(q->dev_queue->dev);
 }
 
+struct tc_query_caps_base {
+	enum tc_setup_type type;
+	void *caps;
+};
+
 struct tc_cbs_qopt_offload {
 	u8 enable;
 	s32 queue;
@@ -155,6 +160,10 @@ struct tc_etf_qopt_offload {
 	s32 queue;
 };
 
+struct tc_taprio_caps {
+	bool supports_queue_max_sdu:1;
+};
+
 struct tc_taprio_sched_entry {
 	u8 command; /* TC_TAPRIO_CMD_* */
 
@@ -168,6 +177,7 @@ struct tc_taprio_qopt_offload {
 	ktime_t base_time;
 	u64 cycle_time;
 	u64 cycle_time_extension;
+	u32 max_sdu[TC_MAX_QUEUE];
 
 	size_t num_entries;
 	struct tc_taprio_sched_entry entries[];
@@ -222,4 +232,17 @@ static inline struct tc_skb_cb *tc_skb_cb(const struct sk_buff *skb)
 	return cb;
 }
 
+static inline bool tc_qdisc_stats_dump(struct Qdisc *sch,
+				       unsigned long cl,
+				       struct qdisc_walker *arg)
+{
+	if (arg->count >= arg->skip && arg->fn(sch, cl, arg) < 0) {
+		arg->stop = 1;
+		return false;
+	}
+
+	arg->count++;
+	return true;
+}
+
 #endif
diff --git a/include/net/red.h b/include/net/red.h
index be11dbd26492..454ac2b65d8c 100644
--- a/include/net/red.h
+++ b/include/net/red.h
@@ -122,7 +122,6 @@ struct red_stats {
 	u32		forced_drop;	/* Forced drops, qavg > max_thresh */
 	u32		forced_mark;	/* Forced marks, qavg > max_thresh */
 	u32		pdrop;          /* Drops due to queue limits */
-	u32		other;          /* Drops due to drop() calls */
 };
 
 struct red_parms {
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index ec693fe7c553..d5517719af4e 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -326,11 +326,6 @@ struct tcf_result {
 		};
 		const struct tcf_proto *goto_tp;
 
-		/* used in the skb_tc_reinsert function */
-		struct {
-			bool		ingress;
-			struct gnet_stats_queue *qstats;
-		};
 	};
 };
 
@@ -682,6 +677,9 @@ qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
 {
 }
 #endif
+void qdisc_offload_query_caps(struct net_device *dev,
+			      enum tc_setup_type type,
+			      void *caps, size_t caps_len);
 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
 			  const struct Qdisc_ops *ops,
 			  struct netlink_ext_ack *extack);
@@ -940,13 +938,6 @@ static inline void qdisc_purge_queue(struct Qdisc *sch)
 	qdisc_tree_reduce_backlog(sch, qlen, backlog);
 }
 
-static inline void qdisc_skb_head_init(struct qdisc_skb_head *qh)
-{
-	qh->head = NULL;
-	qh->tail = NULL;
-	qh->qlen = 0;
-}
-
 static inline void __qdisc_enqueue_tail(struct sk_buff *skb,
 					struct qdisc_skb_head *qh)
 {
@@ -1137,7 +1128,6 @@ static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh)
 static inline void qdisc_reset_queue(struct Qdisc *sch)
 {
 	__qdisc_reset_queue(&sch->q);
-	sch->qstats.backlog = 0;
 }
 
 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new,
diff --git a/include/net/sock.h b/include/net/sock.h
index d08cfe190a78..08038a385ef2 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -348,6 +348,7 @@ struct sk_filter;
   *	@sk_txtime_report_errors: set report errors mode for SO_TXTIME
   *	@sk_txtime_unused: unused txtime flags
   *	@ns_tracker: tracker for netns reference
+  *	@sk_bind2_node: bind node in the bhash2 table
   */
 struct sock {
 	/*
@@ -537,6 +538,7 @@ struct sock {
 #endif
 	struct rcu_head		sk_rcu;
 	netns_tracker		ns_tracker;
+	struct hlist_node	sk_bind2_node;
 };
 
 enum sk_pacing {
@@ -742,11 +744,6 @@ static inline void sk_node_init(struct hlist_node *node)
 	node->pprev = NULL;
 }
 
-static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
-{
-	node->pprev = NULL;
-}
-
 static inline void __sk_del_node(struct sock *sk)
 {
 	__hlist_del(&sk->sk_node);
@@ -870,6 +867,16 @@ static inline void sk_add_bind_node(struct sock *sk,
 	hlist_add_head(&sk->sk_bind_node, list);
 }
 
+static inline void __sk_del_bind2_node(struct sock *sk)
+{
+	__hlist_del(&sk->sk_bind2_node);
+}
+
+static inline void sk_add_bind2_node(struct sock *sk, struct hlist_head *list)
+{
+	hlist_add_head(&sk->sk_bind2_node, list);
+}
+
 #define sk_for_each(__sk, list) \
 	hlist_for_each_entry(__sk, list, sk_node)
 #define sk_for_each_rcu(__sk, list) \
@@ -887,6 +894,8 @@ static inline void sk_add_bind_node(struct sock *sk,
 	hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
 #define sk_for_each_bound(__sk, list) \
 	hlist_for_each_entry(__sk, list, sk_bind_node)
+#define sk_for_each_bound_bhash2(__sk, list) \
+	hlist_for_each_entry(__sk, list, sk_bind2_node)
 
 /**
  * sk_for_each_entry_offset_rcu - iterate over a list at a given struct offset
@@ -1774,6 +1783,11 @@ static inline void unlock_sock_fast(struct sock *sk, bool slow)
 	}
 }
 
+void sockopt_lock_sock(struct sock *sk);
+void sockopt_release_sock(struct sock *sk);
+bool sockopt_ns_capable(struct user_namespace *ns, int cap);
+bool sockopt_capable(int cap);
+
 /* Used by processes to "lock" a socket state, so that
  * interrupts and bottom half handlers won't change it
  * from under us. It essentially blocks any incoming
@@ -1848,9 +1862,13 @@ void sock_pfree(struct sk_buff *skb);
 #define sock_edemux sock_efree
 #endif
 
+int sk_setsockopt(struct sock *sk, int level, int optname,
+		  sockptr_t optval, unsigned int optlen);
 int sock_setsockopt(struct socket *sock, int level, int op,
 		    sockptr_t optval, unsigned int optlen);
 
+int sk_getsockopt(struct sock *sk, int level, int optname,
+		  sockptr_t optval, sockptr_t optlen);
 int sock_getsockopt(struct socket *sock, int level, int op,
 		    char __user *optval, int __user *optlen);
 int sock_gettstamp(struct socket *sock, void __user *userstamp,
diff --git a/include/net/tcp.h b/include/net/tcp.h
index d10962b9f0d0..14d45661a84d 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -327,6 +327,8 @@ void tcp_remove_empty_skb(struct sock *sk);
 int tcp_v4_tw_remember_stamp(struct inet_timewait_sock *tw);
 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
+int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
+			 size_t size, struct ubuf_info *uarg);
 int tcp_sendpage(struct sock *sk, struct page *page, int offset, size_t size,
 		 int flags);
 int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
@@ -346,6 +348,7 @@ void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
 void tcp_rcv_space_adjust(struct sock *sk);
 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
 void tcp_twsk_destructor(struct sock *sk);
+void tcp_twsk_purge(struct list_head *net_exit_list, int family);
 ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
 			struct pipe_inode_info *pipe, size_t len,
 			unsigned int flags);
@@ -402,9 +405,13 @@ void tcp_init_sock(struct sock *sk);
 void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb);
 __poll_t tcp_poll(struct file *file, struct socket *sock,
 		      struct poll_table_struct *wait);
+int do_tcp_getsockopt(struct sock *sk, int level,
+		      int optname, sockptr_t optval, sockptr_t optlen);
 int tcp_getsockopt(struct sock *sk, int level, int optname,
 		   char __user *optval, int __user *optlen);
 bool tcp_bpf_bypass_getsockopt(int level, int optname);
+int do_tcp_setsockopt(struct sock *sk, int level, int optname,
+		      sockptr_t optval, unsigned int optlen);
 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
 		   unsigned int optlen);
 void tcp_set_keepalive(struct sock *sk, int val);
@@ -1295,11 +1302,14 @@ static inline bool tcp_is_cwnd_limited(const struct sock *sk)
 {
 	const struct tcp_sock *tp = tcp_sk(sk);
 
+	if (tp->is_cwnd_limited)
+		return true;
+
 	/* If in slow start, ensure cwnd grows to twice what was ACKed. */
 	if (tcp_in_slow_start(tp))
 		return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out;
 
-	return tp->is_cwnd_limited;
+	return false;
 }
 
 /* BBR congestion control needs pacing.
diff --git a/include/net/tls.h b/include/net/tls.h
index cb205f9d9473..154949c7b0c8 100644
--- a/include/net/tls.h
+++ b/include/net/tls.h
@@ -51,6 +51,16 @@
 
 struct tls_rec;
 
+struct tls_cipher_size_desc {
+	unsigned int iv;
+	unsigned int key;
+	unsigned int salt;
+	unsigned int tag;
+	unsigned int rec_seq;
+};
+
+extern const struct tls_cipher_size_desc tls_cipher_size_desc[];
+
 /* Maximum data size carried in a TLS record */
 #define TLS_MAX_PAYLOAD_SIZE		((size_t)1 << 14)
 
diff --git a/include/net/xdp.h b/include/net/xdp.h
index 04c852c7a77f..55dbc68bfffc 100644
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -164,13 +164,13 @@ struct xdp_frame {
 	void *data;
 	u16 len;
 	u16 headroom;
-	u32 metasize:8;
-	u32 frame_sz:24;
+	u32 metasize; /* uses lower 8-bits */
 	/* Lifetime of xdp_rxq_info is limited to NAPI/enqueue time,
 	 * while mem info is valid on remote CPU.
 	 */
 	struct xdp_mem_info mem;
 	struct net_device *dev_rx; /* used by cpumap */
+	u32 frame_sz;
 	u32 flags; /* supported values defined in xdp_buff_flags */
 };
 
diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
index 0e58c38ce0c1..9c0d860609ba 100644
--- a/include/net/xdp_sock_drv.h
+++ b/include/net/xdp_sock_drv.h
@@ -9,6 +9,9 @@
 #include <net/xdp_sock.h>
 #include <net/xsk_buff_pool.h>
 
+#define XDP_UMEM_MIN_CHUNK_SHIFT 11
+#define XDP_UMEM_MIN_CHUNK_SIZE (1 << XDP_UMEM_MIN_CHUNK_SHIFT)
+
 #ifdef CONFIG_XDP_SOCKETS
 
 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
@@ -104,13 +107,6 @@ static inline void xsk_buff_free(struct xdp_buff *xdp)
 	xp_free(xskb);
 }
 
-static inline void xsk_buff_discard(struct xdp_buff *xdp)
-{
-	struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp);
-
-	xp_release(xskb);
-}
-
 static inline void xsk_buff_set_size(struct xdp_buff *xdp, u32 size)
 {
 	xdp->data = xdp->data_hard_start + XDP_PACKET_HEADROOM;
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 6e8fa98f786f..dbc81f5eb553 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -312,9 +312,15 @@ struct km_event {
 	struct net *net;
 };
 
+struct xfrm_if_decode_session_result {
+	struct net *net;
+	u32 if_id;
+};
+
 struct xfrm_if_cb {
-	struct xfrm_if	*(*decode_session)(struct sk_buff *skb,
-					   unsigned short family);
+	bool (*decode_session)(struct sk_buff *skb,
+			       unsigned short family,
+			       struct xfrm_if_decode_session_result *res);
 };
 
 void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb);
@@ -399,7 +405,8 @@ struct xfrm_type {
 #define XFRM_TYPE_LOCAL_COADDR	4
 #define XFRM_TYPE_REMOTE_COADDR	8
 
-	int			(*init_state)(struct xfrm_state *x);
+	int			(*init_state)(struct xfrm_state *x,
+					      struct netlink_ext_ack *extack);
 	void			(*destructor)(struct xfrm_state *);
 	int			(*input)(struct xfrm_state *, struct sk_buff *skb);
 	int			(*output)(struct xfrm_state *, struct sk_buff *pskb);
@@ -985,6 +992,7 @@ void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev);
 struct xfrm_if_parms {
 	int link;		/* ifindex of underlying L2 interface */
 	u32 if_id;		/* interface identifyer */
+	bool collect_md;
 };
 
 struct xfrm_if {
@@ -1573,9 +1581,10 @@ int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_vali
 void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si);
 void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si);
 u32 xfrm_replay_seqhi(struct xfrm_state *x, __be32 net_seq);
-int xfrm_init_replay(struct xfrm_state *x);
+int xfrm_init_replay(struct xfrm_state *x, struct netlink_ext_ack *extack);
 u32 xfrm_state_mtu(struct xfrm_state *x, int mtu);
-int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload);
+int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload,
+		      struct netlink_ext_ack *extack);
 int xfrm_init_state(struct xfrm_state *x);
 int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type);
 int xfrm_input_resume(struct sk_buff *skb, int nexthdr);
@@ -1879,7 +1888,8 @@ void xfrm_dev_resume(struct sk_buff *skb);
 void xfrm_dev_backlog(struct softnet_data *sd);
 struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again);
 int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
-		       struct xfrm_user_offload *xuo);
+		       struct xfrm_user_offload *xuo,
+		       struct netlink_ext_ack *extack);
 bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x);
 
 static inline void xfrm_dev_state_advance_esn(struct xfrm_state *x)
@@ -1942,7 +1952,7 @@ static inline struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_fea
 	return skb;
 }
 
-static inline int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_user_offload *xuo)
+static inline int xfrm_dev_state_add(struct net *net, struct xfrm_state *x, struct xfrm_user_offload *xuo, struct netlink_ext_ack *extack)
 {
 	return 0;
 }
diff --git a/include/net/xsk_buff_pool.h b/include/net/xsk_buff_pool.h
index 647722e847b4..f787c3f524b0 100644
--- a/include/net/xsk_buff_pool.h
+++ b/include/net/xsk_buff_pool.h
@@ -95,7 +95,7 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
 						struct xdp_umem *umem);
 int xp_assign_dev(struct xsk_buff_pool *pool, struct net_device *dev,
 		  u16 queue_id, u16 flags);
-int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem,
+int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs,
 			 struct net_device *dev, u16 queue_id);
 int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs);
 void xp_destroy(struct xsk_buff_pool *pool);
diff --git a/include/soc/fsl/qman.h b/include/soc/fsl/qman.h
index 59eeba31c192..0d3d6beb7fdb 100644
--- a/include/soc/fsl/qman.h
+++ b/include/soc/fsl/qman.h
@@ -1172,6 +1172,15 @@ int qman_delete_cgr(struct qman_cgr *cgr);
 void qman_delete_cgr_safe(struct qman_cgr *cgr);
 
 /**
+ * qman_update_cgr_safe - Modifies a congestion group object from any CPU
+ * @cgr: the 'cgr' object to modify
+ * @opts: state of the CGR settings
+ *
+ * This will select the proper CPU and modify the CGR settings.
+ */
+int qman_update_cgr_safe(struct qman_cgr *cgr, struct qm_mcc_initcgr *opts);
+
+/**
  * qman_query_cgr_congested - Queries CGR's congestion status
  * @cgr: the 'cgr' object to query
  * @result: returns 'cgr's congestion status, 1 (true) if congested
diff --git a/include/soc/mediatek/smi.h b/include/soc/mediatek/smi.h
index 11f7d6b59642..dfd8efca5e60 100644
--- a/include/soc/mediatek/smi.h
+++ b/include/soc/mediatek/smi.h
@@ -11,6 +11,11 @@
 
 #if IS_ENABLED(CONFIG_MTK_SMI)
 
+enum iommu_atf_cmd {
+	IOMMU_ATF_CMD_CONFIG_SMI_LARB,		/* For mm master to en/disable iommu */
+	IOMMU_ATF_CMD_MAX,
+};
+
 #define MTK_SMI_MMU_EN(port)	BIT(port)
 
 struct mtk_smi_larb_iommu {
diff --git a/include/soc/mscc/ocelot.h b/include/soc/mscc/ocelot.h
index 2edea901bbd5..967ba30ea636 100644
--- a/include/soc/mscc/ocelot.h
+++ b/include/soc/mscc/ocelot.h
@@ -392,7 +392,7 @@ enum ocelot_reg {
 	SYS_COUNT_TX_GREEN_PRIO_5,
 	SYS_COUNT_TX_GREEN_PRIO_6,
 	SYS_COUNT_TX_GREEN_PRIO_7,
-	SYS_COUNT_TX_AGING,
+	SYS_COUNT_TX_AGED,
 	SYS_COUNT_DROP_LOCAL,
 	SYS_COUNT_DROP_TAIL,
 	SYS_COUNT_DROP_YELLOW_PRIO_0,
@@ -411,6 +411,10 @@ enum ocelot_reg {
 	SYS_COUNT_DROP_GREEN_PRIO_5,
 	SYS_COUNT_DROP_GREEN_PRIO_6,
 	SYS_COUNT_DROP_GREEN_PRIO_7,
+	SYS_COUNT_SF_MATCHING_FRAMES,
+	SYS_COUNT_SF_NOT_PASSING_FRAMES,
+	SYS_COUNT_SF_NOT_PASSING_SDU,
+	SYS_COUNT_SF_RED_FRAMES,
 	SYS_RESET_CFG,
 	SYS_SR_ETYPE_CFG,
 	SYS_VLAN_ETYPE_CFG,
@@ -433,7 +437,6 @@ enum ocelot_reg {
 	SYS_MMGT_FAST,
 	SYS_EVENTS_DIF,
 	SYS_EVENTS_CORE,
-	SYS_CNT,
 	SYS_PTP_STATUS,
 	SYS_PTP_TXSTAMP,
 	SYS_PTP_NXT,
@@ -695,6 +698,112 @@ struct ocelot_stat_layout {
 	char name[ETH_GSTRING_LEN];
 };
 
+/* 32-bit counter checked for wraparound by ocelot_port_update_stats()
+ * and copied to ocelot->stats.
+ */
+#define OCELOT_STAT(kind) \
+	[OCELOT_STAT_ ## kind] = { .reg = SYS_COUNT_ ## kind }
+/* Same as above, except also exported to ethtool -S. Standard counters should
+ * only be exposed to more specific interfaces rather than by their string name.
+ */
+#define OCELOT_STAT_ETHTOOL(kind, ethtool_name) \
+	[OCELOT_STAT_ ## kind] = { .reg = SYS_COUNT_ ## kind, .name = ethtool_name }
+
+#define OCELOT_COMMON_STATS \
+	OCELOT_STAT_ETHTOOL(RX_OCTETS, "rx_octets"), \
+	OCELOT_STAT_ETHTOOL(RX_UNICAST, "rx_unicast"), \
+	OCELOT_STAT_ETHTOOL(RX_MULTICAST, "rx_multicast"), \
+	OCELOT_STAT_ETHTOOL(RX_BROADCAST, "rx_broadcast"), \
+	OCELOT_STAT_ETHTOOL(RX_SHORTS, "rx_shorts"), \
+	OCELOT_STAT_ETHTOOL(RX_FRAGMENTS, "rx_fragments"), \
+	OCELOT_STAT_ETHTOOL(RX_JABBERS, "rx_jabbers"), \
+	OCELOT_STAT_ETHTOOL(RX_CRC_ALIGN_ERRS, "rx_crc_align_errs"), \
+	OCELOT_STAT_ETHTOOL(RX_SYM_ERRS, "rx_sym_errs"), \
+	OCELOT_STAT_ETHTOOL(RX_64, "rx_frames_below_65_octets"), \
+	OCELOT_STAT_ETHTOOL(RX_65_127, "rx_frames_65_to_127_octets"), \
+	OCELOT_STAT_ETHTOOL(RX_128_255, "rx_frames_128_to_255_octets"), \
+	OCELOT_STAT_ETHTOOL(RX_256_511, "rx_frames_256_to_511_octets"), \
+	OCELOT_STAT_ETHTOOL(RX_512_1023, "rx_frames_512_to_1023_octets"), \
+	OCELOT_STAT_ETHTOOL(RX_1024_1526, "rx_frames_1024_to_1526_octets"), \
+	OCELOT_STAT_ETHTOOL(RX_1527_MAX, "rx_frames_over_1526_octets"), \
+	OCELOT_STAT_ETHTOOL(RX_PAUSE, "rx_pause"), \
+	OCELOT_STAT_ETHTOOL(RX_CONTROL, "rx_control"), \
+	OCELOT_STAT_ETHTOOL(RX_LONGS, "rx_longs"), \
+	OCELOT_STAT_ETHTOOL(RX_CLASSIFIED_DROPS, "rx_classified_drops"), \
+	OCELOT_STAT_ETHTOOL(RX_RED_PRIO_0, "rx_red_prio_0"), \
+	OCELOT_STAT_ETHTOOL(RX_RED_PRIO_1, "rx_red_prio_1"), \
+	OCELOT_STAT_ETHTOOL(RX_RED_PRIO_2, "rx_red_prio_2"), \
+	OCELOT_STAT_ETHTOOL(RX_RED_PRIO_3, "rx_red_prio_3"), \
+	OCELOT_STAT_ETHTOOL(RX_RED_PRIO_4, "rx_red_prio_4"), \
+	OCELOT_STAT_ETHTOOL(RX_RED_PRIO_5, "rx_red_prio_5"), \
+	OCELOT_STAT_ETHTOOL(RX_RED_PRIO_6, "rx_red_prio_6"), \
+	OCELOT_STAT_ETHTOOL(RX_RED_PRIO_7, "rx_red_prio_7"), \
+	OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_0, "rx_yellow_prio_0"), \
+	OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_1, "rx_yellow_prio_1"), \
+	OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_2, "rx_yellow_prio_2"), \
+	OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_3, "rx_yellow_prio_3"), \
+	OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_4, "rx_yellow_prio_4"), \
+	OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_5, "rx_yellow_prio_5"), \
+	OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_6, "rx_yellow_prio_6"), \
+	OCELOT_STAT_ETHTOOL(RX_YELLOW_PRIO_7, "rx_yellow_prio_7"), \
+	OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_0, "rx_green_prio_0"), \
+	OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_1, "rx_green_prio_1"), \
+	OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_2, "rx_green_prio_2"), \
+	OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_3, "rx_green_prio_3"), \
+	OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_4, "rx_green_prio_4"), \
+	OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_5, "rx_green_prio_5"), \
+	OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_6, "rx_green_prio_6"), \
+	OCELOT_STAT_ETHTOOL(RX_GREEN_PRIO_7, "rx_green_prio_7"), \
+	OCELOT_STAT_ETHTOOL(TX_OCTETS, "tx_octets"), \
+	OCELOT_STAT_ETHTOOL(TX_UNICAST, "tx_unicast"), \
+	OCELOT_STAT_ETHTOOL(TX_MULTICAST, "tx_multicast"), \
+	OCELOT_STAT_ETHTOOL(TX_BROADCAST, "tx_broadcast"), \
+	OCELOT_STAT_ETHTOOL(TX_COLLISION, "tx_collision"), \
+	OCELOT_STAT_ETHTOOL(TX_DROPS, "tx_drops"), \
+	OCELOT_STAT_ETHTOOL(TX_PAUSE, "tx_pause"), \
+	OCELOT_STAT_ETHTOOL(TX_64, "tx_frames_below_65_octets"), \
+	OCELOT_STAT_ETHTOOL(TX_65_127, "tx_frames_65_to_127_octets"), \
+	OCELOT_STAT_ETHTOOL(TX_128_255, "tx_frames_128_255_octets"), \
+	OCELOT_STAT_ETHTOOL(TX_256_511, "tx_frames_256_511_octets"), \
+	OCELOT_STAT_ETHTOOL(TX_512_1023, "tx_frames_512_1023_octets"), \
+	OCELOT_STAT_ETHTOOL(TX_1024_1526, "tx_frames_1024_1526_octets"), \
+	OCELOT_STAT_ETHTOOL(TX_1527_MAX, "tx_frames_over_1526_octets"), \
+	OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_0, "tx_yellow_prio_0"), \
+	OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_1, "tx_yellow_prio_1"), \
+	OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_2, "tx_yellow_prio_2"), \
+	OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_3, "tx_yellow_prio_3"), \
+	OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_4, "tx_yellow_prio_4"), \
+	OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_5, "tx_yellow_prio_5"), \
+	OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_6, "tx_yellow_prio_6"), \
+	OCELOT_STAT_ETHTOOL(TX_YELLOW_PRIO_7, "tx_yellow_prio_7"), \
+	OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_0, "tx_green_prio_0"), \
+	OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_1, "tx_green_prio_1"), \
+	OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_2, "tx_green_prio_2"), \
+	OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_3, "tx_green_prio_3"), \
+	OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_4, "tx_green_prio_4"), \
+	OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_5, "tx_green_prio_5"), \
+	OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_6, "tx_green_prio_6"), \
+	OCELOT_STAT_ETHTOOL(TX_GREEN_PRIO_7, "tx_green_prio_7"), \
+	OCELOT_STAT_ETHTOOL(TX_AGED, "tx_aged"), \
+	OCELOT_STAT_ETHTOOL(DROP_LOCAL, "drop_local"), \
+	OCELOT_STAT_ETHTOOL(DROP_TAIL, "drop_tail"), \
+	OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_0, "drop_yellow_prio_0"), \
+	OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_1, "drop_yellow_prio_1"), \
+	OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_2, "drop_yellow_prio_2"), \
+	OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_3, "drop_yellow_prio_3"), \
+	OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_4, "drop_yellow_prio_4"), \
+	OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_5, "drop_yellow_prio_5"), \
+	OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_6, "drop_yellow_prio_6"), \
+	OCELOT_STAT_ETHTOOL(DROP_YELLOW_PRIO_7, "drop_yellow_prio_7"), \
+	OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_0, "drop_green_prio_0"), \
+	OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_1, "drop_green_prio_1"), \
+	OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_2, "drop_green_prio_2"), \
+	OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_3, "drop_green_prio_3"), \
+	OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_4, "drop_green_prio_4"), \
+	OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_5, "drop_green_prio_5"), \
+	OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_6, "drop_green_prio_6"), \
+	OCELOT_STAT_ETHTOOL(DROP_GREEN_PRIO_7, "drop_green_prio_7")
+
 struct ocelot_stats_region {
 	struct list_head node;
 	u32 base;
@@ -726,6 +835,7 @@ struct ocelot_ops {
 			      struct flow_stats *stats);
 	void (*cut_through_fwd)(struct ocelot *ocelot);
 	void (*tas_clock_adjust)(struct ocelot *ocelot);
+	void (*update_stats)(struct ocelot *ocelot);
 };
 
 struct ocelot_vcap_policer {
@@ -763,6 +873,8 @@ struct ocelot_psfp_list {
 	struct list_head stream_list;
 	struct list_head sfi_list;
 	struct list_head sgi_list;
+	/* Serialize access to the lists */
+	struct mutex lock;
 };
 
 enum ocelot_sb {
@@ -898,12 +1010,15 @@ struct ocelot {
 
 	struct ocelot_psfp_list		psfp;
 
-	/* Workqueue to check statistics for overflow with its lock */
-	spinlock_t			stats_lock;
-	u64				*stats;
+	/* Workqueue to check statistics for overflow */
 	struct delayed_work		stats_work;
 	struct workqueue_struct		*stats_queue;
+	/* Lock for serializing access to the statistics array */
+	spinlock_t			stats_lock;
+	u64				*stats;
 
+	/* Lock for serializing indirect access to STAT_VIEW registers */
+	struct mutex			stat_view_lock;
 	/* Lock for serializing access to the MAC table */
 	struct mutex			mact_lock;
 	/* Lock for serializing forwarding domain changes */
@@ -1024,6 +1139,8 @@ void ocelot_deinit(struct ocelot *ocelot);
 void ocelot_init_port(struct ocelot *ocelot, int port);
 void ocelot_deinit_port(struct ocelot *ocelot, int port);
 
+void ocelot_port_setup_dsa_8021q_cpu(struct ocelot *ocelot, int cpu);
+void ocelot_port_teardown_dsa_8021q_cpu(struct ocelot *ocelot, int cpu);
 void ocelot_port_assign_dsa_8021q_cpu(struct ocelot *ocelot, int port, int cpu);
 void ocelot_port_unassign_dsa_8021q_cpu(struct ocelot *ocelot, int port);
 u32 ocelot_port_assigned_dsa_8021q_cpu_mask(struct ocelot *ocelot, int port);
@@ -1032,6 +1149,19 @@ u32 ocelot_port_assigned_dsa_8021q_cpu_mask(struct ocelot *ocelot, int port);
 void ocelot_get_strings(struct ocelot *ocelot, int port, u32 sset, u8 *data);
 void ocelot_get_ethtool_stats(struct ocelot *ocelot, int port, u64 *data);
 int ocelot_get_sset_count(struct ocelot *ocelot, int port, int sset);
+void ocelot_port_get_stats64(struct ocelot *ocelot, int port,
+			     struct rtnl_link_stats64 *stats);
+void ocelot_port_get_pause_stats(struct ocelot *ocelot, int port,
+				 struct ethtool_pause_stats *pause_stats);
+void ocelot_port_get_rmon_stats(struct ocelot *ocelot, int port,
+				struct ethtool_rmon_stats *rmon_stats,
+				const struct ethtool_rmon_hist_range **ranges);
+void ocelot_port_get_eth_ctrl_stats(struct ocelot *ocelot, int port,
+				    struct ethtool_eth_ctrl_stats *ctrl_stats);
+void ocelot_port_get_eth_mac_stats(struct ocelot *ocelot, int port,
+				   struct ethtool_eth_mac_stats *mac_stats);
+void ocelot_port_get_eth_phy_stats(struct ocelot *ocelot, int port,
+				   struct ethtool_eth_phy_stats *phy_stats);
 int ocelot_get_ts_info(struct ocelot *ocelot, int port,
 		       struct ethtool_ts_info *info);
 void ocelot_set_ageing_time(struct ocelot *ocelot, unsigned int msecs);
@@ -1099,10 +1229,12 @@ int ocelot_port_mdb_del(struct ocelot *ocelot, int port,
 			const struct net_device *bridge);
 int ocelot_port_lag_join(struct ocelot *ocelot, int port,
 			 struct net_device *bond,
-			 struct netdev_lag_upper_info *info);
+			 struct netdev_lag_upper_info *info,
+			 struct netlink_ext_ack *extack);
 void ocelot_port_lag_leave(struct ocelot *ocelot, int port,
 			   struct net_device *bond);
 void ocelot_port_lag_change(struct ocelot *ocelot, int port, bool lag_tx_active);
+int ocelot_bond_get_id(struct ocelot *ocelot, struct net_device *bond);
 
 int ocelot_devlink_sb_register(struct ocelot *ocelot);
 void ocelot_devlink_sb_unregister(struct ocelot *ocelot);
diff --git a/include/soc/tegra/fuse.h b/include/soc/tegra/fuse.h
index 67d2bc856fbc..977c334136e9 100644
--- a/include/soc/tegra/fuse.h
+++ b/include/soc/tegra/fuse.h
@@ -58,6 +58,7 @@ u32 tegra_read_chipid(void);
 u8 tegra_get_chip_id(void);
 u8 tegra_get_platform(void);
 bool tegra_is_silicon(void);
+int tegra194_miscreg_mask_serror(void);
 #else
 static struct tegra_sku_info tegra_sku_info __maybe_unused;
 
@@ -95,6 +96,11 @@ static inline bool tegra_is_silicon(void)
 {
 	return false;
 }
+
+static inline int tegra194_miscreg_mask_serror(void)
+{
+	return false;
+}
 #endif
 
 struct device *tegra_soc_device_register(void);
diff --git a/include/soc/tegra/tegra-cbb.h b/include/soc/tegra/tegra-cbb.h
new file mode 100644
index 000000000000..e864c2ebe794
--- /dev/null
+++ b/include/soc/tegra/tegra-cbb.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved
+ */
+
+#ifndef TEGRA_CBB_H
+#define TEGRA_CBB_H
+
+#include <linux/list.h>
+
+struct tegra_cbb_error {
+	const char *code;
+	const char *source;
+	const char *desc;
+};
+
+struct tegra_cbb {
+	struct device *dev;
+	const struct tegra_cbb_ops *ops;
+	struct list_head node;
+};
+
+struct tegra_cbb_ops {
+	int (*debugfs_show)(struct tegra_cbb *cbb, struct seq_file *s, void *v);
+	int (*interrupt_enable)(struct tegra_cbb *cbb);
+	void (*error_enable)(struct tegra_cbb *cbb);
+	void (*fault_enable)(struct tegra_cbb *cbb);
+	void (*stall_enable)(struct tegra_cbb *cbb);
+	void (*error_clear)(struct tegra_cbb *cbb);
+	u32 (*get_status)(struct tegra_cbb *cbb);
+};
+
+int tegra_cbb_get_irq(struct platform_device *pdev, unsigned int *nonsec_irq,
+		      unsigned int *sec_irq);
+__printf(2, 3)
+void tegra_cbb_print_err(struct seq_file *file, const char *fmt, ...);
+
+void tegra_cbb_print_cache(struct seq_file *file, u32 cache);
+void tegra_cbb_print_prot(struct seq_file *file, u32 prot);
+int tegra_cbb_register(struct tegra_cbb *cbb);
+
+void tegra_cbb_fault_enable(struct tegra_cbb *cbb);
+void tegra_cbb_stall_enable(struct tegra_cbb *cbb);
+void tegra_cbb_error_clear(struct tegra_cbb *cbb);
+u32 tegra_cbb_get_status(struct tegra_cbb *cbb);
+
+#endif /* TEGRA_CBB_H */
diff --git a/include/sound/acp62_chip_offset_byte.h b/include/sound/acp62_chip_offset_byte.h
new file mode 100644
index 000000000000..f03992f81168
--- /dev/null
+++ b/include/sound/acp62_chip_offset_byte.h
@@ -0,0 +1,444 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * AMD ACP 6.2 Register Documentation
+ *
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _acp_ip_OFFSET_HEADER
+#define _acp_ip_OFFSET_HEADER
+
+/* Registers from ACP_DMA block */
+#define ACP_DMA_CNTL_0                                0x0000000
+#define ACP_DMA_CNTL_1                                0x0000004
+#define ACP_DMA_CNTL_2                                0x0000008
+#define ACP_DMA_CNTL_3                                0x000000C
+#define ACP_DMA_CNTL_4                                0x0000010
+#define ACP_DMA_CNTL_5                                0x0000014
+#define ACP_DMA_CNTL_6                                0x0000018
+#define ACP_DMA_CNTL_7                                0x000001C
+#define ACP_DMA_DSCR_STRT_IDX_0                       0x0000020
+#define ACP_DMA_DSCR_STRT_IDX_1                       0x0000024
+#define ACP_DMA_DSCR_STRT_IDX_2                       0x0000028
+#define ACP_DMA_DSCR_STRT_IDX_3                       0x000002C
+#define ACP_DMA_DSCR_STRT_IDX_4                       0x0000030
+#define ACP_DMA_DSCR_STRT_IDX_5                       0x0000034
+#define ACP_DMA_DSCR_STRT_IDX_6                       0x0000038
+#define ACP_DMA_DSCR_STRT_IDX_7                       0x000003C
+#define ACP_DMA_DSCR_CNT_0                            0x0000040
+#define ACP_DMA_DSCR_CNT_1                            0x0000044
+#define ACP_DMA_DSCR_CNT_2                            0x0000048
+#define ACP_DMA_DSCR_CNT_3                            0x000004C
+#define ACP_DMA_DSCR_CNT_4                            0x0000050
+#define ACP_DMA_DSCR_CNT_5                            0x0000054
+#define ACP_DMA_DSCR_CNT_6                            0x0000058
+#define ACP_DMA_DSCR_CNT_7                            0x000005C
+#define ACP_DMA_PRIO_0                                0x0000060
+#define ACP_DMA_PRIO_1                                0x0000064
+#define ACP_DMA_PRIO_2                                0x0000068
+#define ACP_DMA_PRIO_3                                0x000006C
+#define ACP_DMA_PRIO_4                                0x0000070
+#define ACP_DMA_PRIO_5                                0x0000074
+#define ACP_DMA_PRIO_6                                0x0000078
+#define ACP_DMA_PRIO_7                                0x000007C
+#define ACP_DMA_CUR_DSCR_0                            0x0000080
+#define ACP_DMA_CUR_DSCR_1                            0x0000084
+#define ACP_DMA_CUR_DSCR_2                            0x0000088
+#define ACP_DMA_CUR_DSCR_3                            0x000008C
+#define ACP_DMA_CUR_DSCR_4                            0x0000090
+#define ACP_DMA_CUR_DSCR_5                            0x0000094
+#define ACP_DMA_CUR_DSCR_6                            0x0000098
+#define ACP_DMA_CUR_DSCR_7                            0x000009C
+#define ACP_DMA_CUR_TRANS_CNT_0                       0x00000A0
+#define ACP_DMA_CUR_TRANS_CNT_1                       0x00000A4
+#define ACP_DMA_CUR_TRANS_CNT_2                       0x00000A8
+#define ACP_DMA_CUR_TRANS_CNT_3                       0x00000AC
+#define ACP_DMA_CUR_TRANS_CNT_4                       0x00000B0
+#define ACP_DMA_CUR_TRANS_CNT_5                       0x00000B4
+#define ACP_DMA_CUR_TRANS_CNT_6                       0x00000B8
+#define ACP_DMA_CUR_TRANS_CNT_7                       0x00000BC
+#define ACP_DMA_ERR_STS_0                             0x00000C0
+#define ACP_DMA_ERR_STS_1                             0x00000C4
+#define ACP_DMA_ERR_STS_2                             0x00000C8
+#define ACP_DMA_ERR_STS_3                             0x00000CC
+#define ACP_DMA_ERR_STS_4                             0x00000D0
+#define ACP_DMA_ERR_STS_5                             0x00000D4
+#define ACP_DMA_ERR_STS_6                             0x00000D8
+#define ACP_DMA_ERR_STS_7                             0x00000DC
+#define ACP_DMA_DESC_BASE_ADDR                        0x00000E0
+#define ACP_DMA_DESC_MAX_NUM_DSCR                     0x00000E4
+#define ACP_DMA_CH_STS                                0x00000E8
+#define ACP_DMA_CH_GROUP                              0x00000EC
+#define ACP_DMA_CH_RST_STS                            0x00000F0
+
+/* Registers from ACP_AXI2AXIATU block */
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_1                0x0000C00
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_1                0x0000C04
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_2                0x0000C08
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_2                0x0000C0C
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_3                0x0000C10
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_3                0x0000C14
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_4                0x0000C18
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_4                0x0000C1C
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_5                0x0000C20
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_5                0x0000C24
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_6                0x0000C28
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_6                0x0000C2C
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_7                0x0000C30
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_7                0x0000C34
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_8                0x0000C38
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_8                0x0000C3C
+#define ACPAXI2AXI_ATU_CTRL                           0x0000C40
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_9                0x0000C44
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_9                0x0000C48
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_10               0x0000C4C
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_10               0x0000C50
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_11               0x0000C54
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_11               0x0000C58
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_12               0x0000C5C
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_12               0x0000C60
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_13               0x0000C64
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_13               0x0000C68
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_14               0x0000C6C
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_14               0x0000C70
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_15               0x0000C74
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_15               0x0000C78
+#define ACPAXI2AXI_ATU_PAGE_SIZE_GRP_16               0x0000C7C
+#define ACPAXI2AXI_ATU_BASE_ADDR_GRP_16               0x0000C80
+
+/* Registers from ACP_CLKRST block */
+#define ACP_SOFT_RESET                                0x0001000
+#define ACP_CONTROL                                   0x0001004
+#define ACP_STATUS                                    0x0001008
+#define ACP_DYNAMIC_CG_MASTER_CONTROL                 0x0001010
+#define ACP_ZSC_DSP_CTRL                              0x0001014
+#define ACP_ZSC_STS                                   0x0001018
+#define ACP_PGFSM_CONTROL                             0x0001024
+#define ACP_PGFSM_STATUS                              0x0001028
+#define ACP_CLKMUX_SEL                                0x000102C
+
+/* Registers from ACP_AON block */
+#define ACP_PME_EN                                    0x0001400
+#define ACP_DEVICE_STATE                              0x0001404
+#define AZ_DEVICE_STATE                               0x0001408
+#define ACP_PIN_CONFIG                                0x0001440
+#define ACP_PAD_PULLUP_CTRL                           0x0001444
+#define ACP_PAD_PULLDOWN_CTRL                         0x0001448
+#define ACP_PAD_DRIVE_STRENGTH_CTRL                   0x000144C
+#define ACP_PAD_SCHMEN_CTRL                           0x0001450
+#define ACP_SW_PAD_KEEPER_EN                          0x0001454
+#define ACP_SW_WAKE_EN                                0x0001458
+#define ACP_I2S_WAKE_EN                               0x000145C
+#define ACP_SW1_WAKE_EN                               0x0001460
+
+/* Registers from ACP_P1_MISC block */
+#define ACP_EXTERNAL_INTR_ENB                         0x0001A00
+#define ACP_EXTERNAL_INTR_CNTL                        0x0001A04
+#define ACP_EXTERNAL_INTR_CNTL1                       0x0001A08
+#define ACP_EXTERNAL_INTR_STAT                        0x0001A0C
+#define ACP_EXTERNAL_INTR_STAT1                       0x0001A10
+#define ACP_ERROR_STATUS                              0x0001A4C
+#define ACP_P1_SW_I2S_ERROR_REASON                    0x0001A50
+#define ACP_P1_SW_POS_TRACK_I2S_TX_CTRL               0x0001A6C
+#define ACP_P1_SW_I2S_TX_DMA_POS                      0x0001A70
+#define ACP_P1_SW_POS_TRACK_I2S_RX_CTRL               0x0001A74
+#define ACP_P1_SW_I2S_RX_DMA_POS                      0x0001A78
+#define ACP_P1_DMIC_I2S_GPIO_INTR_CTRL                0x0001A7C
+#define ACP_P1_DMIC_I2S_GPIO_INTR_STATUS              0x0001A80
+#define ACP_SCRATCH_REG_BASE_ADDR                     0x0001A84
+#define ACP_P1_SW_POS_TRACK_BT_TX_CTRL                0x0001A88
+#define ACP_P1_SW_BT_TX_DMA_POS                       0x0001A8C
+#define ACP_P1_SW_POS_TRACK_HS_TX_CTRL                0x0001A90
+#define ACP_P1_SW_HS_TX_DMA_POS                       0x0001A94
+#define ACP_P1_SW_POS_TRACK_BT_RX_CTRL                0x0001A98
+#define ACP_P1_SW_BT_RX_DMA_POS                       0x0001A9C
+#define ACP_P1_SW_POS_TRACK_HS_RX_CTRL                0x0001AA0
+#define ACP_P1_SW_HS_RX_DMA_POS                       0x0001AA4
+
+/* Registers from ACP_AUDIO_BUFFERS block */
+#define ACP_I2S_RX_RINGBUFADDR                        0x0002000
+#define ACP_I2S_RX_RINGBUFSIZE                        0x0002004
+#define ACP_I2S_RX_LINKPOSITIONCNTR                   0x0002008
+#define ACP_I2S_RX_FIFOADDR                           0x000200C
+#define ACP_I2S_RX_FIFOSIZE                           0x0002010
+#define ACP_I2S_RX_DMA_SIZE                           0x0002014
+#define ACP_I2S_RX_LINEARPOSITIONCNTR_HIGH            0x0002018
+#define ACP_I2S_RX_LINEARPOSITIONCNTR_LOW             0x000201C
+#define ACP_I2S_RX_INTR_WATERMARK_SIZE                0x0002020
+#define ACP_I2S_TX_RINGBUFADDR                        0x0002024
+#define ACP_I2S_TX_RINGBUFSIZE                        0x0002028
+#define ACP_I2S_TX_LINKPOSITIONCNTR                   0x000202C
+#define ACP_I2S_TX_FIFOADDR                           0x0002030
+#define ACP_I2S_TX_FIFOSIZE                           0x0002034
+#define ACP_I2S_TX_DMA_SIZE                           0x0002038
+#define ACP_I2S_TX_LINEARPOSITIONCNTR_HIGH            0x000203C
+#define ACP_I2S_TX_LINEARPOSITIONCNTR_LOW             0x0002040
+#define ACP_I2S_TX_INTR_WATERMARK_SIZE                0x0002044
+#define ACP_BT_RX_RINGBUFADDR                         0x0002048
+#define ACP_BT_RX_RINGBUFSIZE                         0x000204C
+#define ACP_BT_RX_LINKPOSITIONCNTR                    0x0002050
+#define ACP_BT_RX_FIFOADDR                            0x0002054
+#define ACP_BT_RX_FIFOSIZE                            0x0002058
+#define ACP_BT_RX_DMA_SIZE                            0x000205C
+#define ACP_BT_RX_LINEARPOSITIONCNTR_HIGH             0x0002060
+#define ACP_BT_RX_LINEARPOSITIONCNTR_LOW              0x0002064
+#define ACP_BT_RX_INTR_WATERMARK_SIZE                 0x0002068
+#define ACP_BT_TX_RINGBUFADDR                         0x000206C
+#define ACP_BT_TX_RINGBUFSIZE                         0x0002070
+#define ACP_BT_TX_LINKPOSITIONCNTR                    0x0002074
+#define ACP_BT_TX_FIFOADDR                            0x0002078
+#define ACP_BT_TX_FIFOSIZE                            0x000207C
+#define ACP_BT_TX_DMA_SIZE                            0x0002080
+#define ACP_BT_TX_LINEARPOSITIONCNTR_HIGH             0x0002084
+#define ACP_BT_TX_LINEARPOSITIONCNTR_LOW              0x0002088
+#define ACP_BT_TX_INTR_WATERMARK_SIZE                 0x000208C
+#define ACP_HS_RX_RINGBUFADDR                         0x0002090
+#define ACP_HS_RX_RINGBUFSIZE                         0x0002094
+#define ACP_HS_RX_LINKPOSITIONCNTR                    0x0002098
+#define ACP_HS_RX_FIFOADDR                            0x000209C
+#define ACP_HS_RX_FIFOSIZE                            0x00020A0
+#define ACP_HS_RX_DMA_SIZE                            0x00020A4
+#define ACP_HS_RX_LINEARPOSITIONCNTR_HIGH             0x00020A8
+#define ACP_HS_RX_LINEARPOSITIONCNTR_LOW              0x00020AC
+#define ACP_HS_RX_INTR_WATERMARK_SIZE                 0x00020B0
+#define ACP_HS_TX_RINGBUFADDR                         0x00020B4
+#define ACP_HS_TX_RINGBUFSIZE                         0x00020B8
+#define ACP_HS_TX_LINKPOSITIONCNTR                    0x00020BC
+#define ACP_HS_TX_FIFOADDR                            0x00020C0
+#define ACP_HS_TX_FIFOSIZE                            0x00020C4
+#define ACP_HS_TX_DMA_SIZE                            0x00020C8
+#define ACP_HS_TX_LINEARPOSITIONCNTR_HIGH             0x00020CC
+#define ACP_HS_TX_LINEARPOSITIONCNTR_LOW              0x00020D0
+#define ACP_HS_TX_INTR_WATERMARK_SIZE                 0x00020D4
+
+/* Registers from ACP_I2S_TDM block */
+#define ACP_I2STDM_IER                                0x0002400
+#define ACP_I2STDM_IRER                               0x0002404
+#define ACP_I2STDM_RXFRMT                             0x0002408
+#define ACP_I2STDM_ITER                               0x000240C
+#define ACP_I2STDM_TXFRMT                             0x0002410
+#define ACP_I2STDM0_MSTRCLKGEN                        0x0002414
+#define ACP_I2STDM1_MSTRCLKGEN                        0x0002418
+#define ACP_I2STDM2_MSTRCLKGEN                        0x000241C
+#define ACP_I2STDM_REFCLKGEN                          0x0002420
+
+/* Registers from ACP_BT_TDM block */
+#define ACP_BTTDM_IER                                 0x0002800
+#define ACP_BTTDM_IRER                                0x0002804
+#define ACP_BTTDM_RXFRMT                              0x0002808
+#define ACP_BTTDM_ITER                                0x000280C
+#define ACP_BTTDM_TXFRMT                              0x0002810
+#define ACP_HSTDM_IER                                 0x0002814
+#define ACP_HSTDM_IRER                                0x0002818
+#define ACP_HSTDM_RXFRMT                              0x000281C
+#define ACP_HSTDM_ITER                                0x0002820
+#define ACP_HSTDM_TXFRMT                              0x0002824
+
+/* Registers from ACP_WOV block */
+#define ACP_WOV_PDM_ENABLE                            0x0002C04
+#define ACP_WOV_PDM_DMA_ENABLE                        0x0002C08
+#define ACP_WOV_RX_RINGBUFADDR                        0x0002C0C
+#define ACP_WOV_RX_RINGBUFSIZE                        0x0002C10
+#define ACP_WOV_RX_LINKPOSITIONCNTR                   0x0002C14
+#define ACP_WOV_RX_LINEARPOSITIONCNTR_HIGH            0x0002C18
+#define ACP_WOV_RX_LINEARPOSITIONCNTR_LOW             0x0002C1C
+#define ACP_WOV_RX_INTR_WATERMARK_SIZE                0x0002C20
+#define ACP_WOV_PDM_FIFO_FLUSH                        0x0002C24
+#define ACP_WOV_PDM_NO_OF_CHANNELS                    0x0002C28
+#define ACP_WOV_PDM_DECIMATION_FACTOR                 0x0002C2C
+#define ACP_WOV_PDM_VAD_CTRL                          0x0002C30
+#define ACP_WOV_WAKE                                  0x0002C54
+#define ACP_WOV_BUFFER_STATUS                         0x0002C58
+#define ACP_WOV_MISC_CTRL                             0x0002C5C
+#define ACP_WOV_CLK_CTRL                              0x0002C60
+#define ACP_PDM_VAD_DYNAMIC_CLK_GATING_EN             0x0002C64
+#define ACP_WOV_ERROR_STATUS_REGISTER                 0x0002C68
+#define ACP_PDM_CLKDIV                                0x0002C6C
+
+/* Registers from ACP_P1_AUDIO_BUFFERS block */
+#define ACP_P1_I2S_RX_RINGBUFADDR                     0x0003A00
+#define ACP_P1_I2S_RX_RINGBUFSIZE                     0x0003A04
+#define ACP_P1_I2S_RX_LINKPOSITIONCNTR                0x0003A08
+#define ACP_P1_I2S_RX_FIFOADDR                        0x0003A0C
+#define ACP_P1_I2S_RX_FIFOSIZE                        0x0003A10
+#define ACP_P1_I2S_RX_DMA_SIZE                        0x0003A14
+#define ACP_P1_I2S_RX_LINEARPOSITIONCNTR_HIGH         0x0003A18
+#define ACP_P1_I2S_RX_LINEARPOSITIONCNTR_LOW          0x0003A1C
+#define ACP_P1_I2S_RX_INTR_WATERMARK_SIZE             0x0003A20
+#define ACP_P1_I2S_TX_RINGBUFADDR                     0x0003A24
+#define ACP_P1_I2S_TX_RINGBUFSIZE                     0x0003A28
+#define ACP_P1_I2S_TX_LINKPOSITIONCNTR                0x0003A2C
+#define ACP_P1_I2S_TX_FIFOADDR                        0x0003A30
+#define ACP_P1_I2S_TX_FIFOSIZE                        0x0003A34
+#define ACP_P1_I2S_TX_DMA_SIZE                        0x0003A38
+#define ACP_P1_I2S_TX_LINEARPOSITIONCNTR_HIGH         0x0003A3C
+#define ACP_P1_I2S_TX_LINEARPOSITIONCNTR_LOW          0x0003A40
+#define ACP_P1_I2S_TX_INTR_WATERMARK_SIZE             0x0003A44
+#define ACP_P1_BT_RX_RINGBUFADDR                      0x0003A48
+#define ACP_P1_BT_RX_RINGBUFSIZE                      0x0003A4C
+#define ACP_P1_BT_RX_LINKPOSITIONCNTR                 0x0003A50
+#define ACP_P1_BT_RX_FIFOADDR                         0x0003A54
+#define ACP_P1_BT_RX_FIFOSIZE                         0x0003A58
+#define ACP_P1_BT_RX_DMA_SIZE                         0x0003A5C
+#define ACP_P1_BT_RX_LINEARPOSITIONCNTR_HIGH          0x0003A60
+#define ACP_P1_BT_RX_LINEARPOSITIONCNTR_LOW           0x0003A64
+#define ACP_P1_BT_RX_INTR_WATERMARK_SIZE              0x0003A68
+#define ACP_P1_BT_TX_RINGBUFADDR                      0x0003A6C
+#define ACP_P1_BT_TX_RINGBUFSIZE                      0x0003A70
+#define ACP_P1_BT_TX_LINKPOSITIONCNTR                 0x0003A74
+#define ACP_P1_BT_TX_FIFOADDR                         0x0003A78
+#define ACP_P1_BT_TX_FIFOSIZE                         0x0003A7C
+#define ACP_P1_BT_TX_DMA_SIZE                         0x0003A80
+#define ACP_P1_BT_TX_LINEARPOSITIONCNTR_HIGH          0x0003A84
+#define ACP_P1_BT_TX_LINEARPOSITIONCNTR_LOW           0x0003A88
+#define ACP_P1_BT_TX_INTR_WATERMARK_SIZE              0x0003A8C
+#define ACP_P1_HS_RX_RINGBUFADDR                      0x0003A90
+#define ACP_P1_HS_RX_RINGBUFSIZE                      0x0003A94
+#define ACP_P1_HS_RX_LINKPOSITIONCNTR                 0x0003A98
+#define ACP_P1_HS_RX_FIFOADDR                         0x0003A9C
+#define ACP_P1_HS_RX_FIFOSIZE                         0x0003AA0
+#define ACP_P1_HS_RX_DMA_SIZE                         0x0003AA4
+#define ACP_P1_HS_RX_LINEARPOSITIONCNTR_HIGH          0x0003AA8
+#define ACP_P1_HS_RX_LINEARPOSITIONCNTR_LOW           0x0003AAC
+#define ACP_P1_HS_RX_INTR_WATERMARK_SIZE              0x0003AB0
+#define ACP_P1_HS_TX_RINGBUFADDR                      0x0003AB4
+#define ACP_P1_HS_TX_RINGBUFSIZE                      0x0003AB8
+#define ACP_P1_HS_TX_LINKPOSITIONCNTR                 0x0003ABC
+#define ACP_P1_HS_TX_FIFOADDR                         0x0003AC0
+#define ACP_P1_HS_TX_FIFOSIZE                         0x0003AC4
+#define ACP_P1_HS_TX_DMA_SIZE                         0x0003AC8
+#define ACP_P1_HS_TX_LINEARPOSITIONCNTR_HIGH          0x0003ACC
+#define ACP_P1_HS_TX_LINEARPOSITIONCNTR_LOW           0x0003AD0
+#define ACP_P1_HS_TX_INTR_WATERMARK_SIZE              0x0003AD4
+
+/* Registers from ACP_SCRATCH block */
+#define ACP_SCRATCH_REG_0                             0x0010000
+#define ACP_SCRATCH_REG_1                             0x0010004
+#define ACP_SCRATCH_REG_2                             0x0010008
+#define ACP_SCRATCH_REG_3                             0x001000C
+#define ACP_SCRATCH_REG_4                             0x0010010
+#define ACP_SCRATCH_REG_5                             0x0010014
+#define ACP_SCRATCH_REG_6                             0x0010018
+#define ACP_SCRATCH_REG_7                             0x001001C
+#define ACP_SCRATCH_REG_8                             0x0010020
+#define ACP_SCRATCH_REG_9                             0x0010024
+#define ACP_SCRATCH_REG_10                            0x0010028
+#define ACP_SCRATCH_REG_11                            0x001002C
+#define ACP_SCRATCH_REG_12                            0x0010030
+#define ACP_SCRATCH_REG_13                            0x0010034
+#define ACP_SCRATCH_REG_14                            0x0010038
+#define ACP_SCRATCH_REG_15                            0x001003C
+#define ACP_SCRATCH_REG_16                            0x0010040
+#define ACP_SCRATCH_REG_17                            0x0010044
+#define ACP_SCRATCH_REG_18                            0x0010048
+#define ACP_SCRATCH_REG_19                            0x001004C
+#define ACP_SCRATCH_REG_20                            0x0010050
+#define ACP_SCRATCH_REG_21                            0x0010054
+#define ACP_SCRATCH_REG_22                            0x0010058
+#define ACP_SCRATCH_REG_23                            0x001005C
+#define ACP_SCRATCH_REG_24                            0x0010060
+#define ACP_SCRATCH_REG_25                            0x0010064
+#define ACP_SCRATCH_REG_26                            0x0010068
+#define ACP_SCRATCH_REG_27                            0x001006C
+#define ACP_SCRATCH_REG_28                            0x0010070
+#define ACP_SCRATCH_REG_29                            0x0010074
+#define ACP_SCRATCH_REG_30                            0x0010078
+#define ACP_SCRATCH_REG_31                            0x001007C
+#define ACP_SCRATCH_REG_32                            0x0010080
+#define ACP_SCRATCH_REG_33                            0x0010084
+#define ACP_SCRATCH_REG_34                            0x0010088
+#define ACP_SCRATCH_REG_35                            0x001008C
+#define ACP_SCRATCH_REG_36                            0x0010090
+#define ACP_SCRATCH_REG_37                            0x0010094
+#define ACP_SCRATCH_REG_38                            0x0010098
+#define ACP_SCRATCH_REG_39                            0x001009C
+#define ACP_SCRATCH_REG_40                            0x00100A0
+#define ACP_SCRATCH_REG_41                            0x00100A4
+#define ACP_SCRATCH_REG_42                            0x00100A8
+#define ACP_SCRATCH_REG_43                            0x00100AC
+#define ACP_SCRATCH_REG_44                            0x00100B0
+#define ACP_SCRATCH_REG_45                            0x00100B4
+#define ACP_SCRATCH_REG_46                            0x00100B8
+#define ACP_SCRATCH_REG_47                            0x00100BC
+#define ACP_SCRATCH_REG_48                            0x00100C0
+#define ACP_SCRATCH_REG_49                            0x00100C4
+#define ACP_SCRATCH_REG_50                            0x00100C8
+#define ACP_SCRATCH_REG_51                            0x00100CC
+#define ACP_SCRATCH_REG_52                            0x00100D0
+#define ACP_SCRATCH_REG_53                            0x00100D4
+#define ACP_SCRATCH_REG_54                            0x00100D8
+#define ACP_SCRATCH_REG_55                            0x00100DC
+#define ACP_SCRATCH_REG_56                            0x00100E0
+#define ACP_SCRATCH_REG_57                            0x00100E4
+#define ACP_SCRATCH_REG_58                            0x00100E8
+#define ACP_SCRATCH_REG_59                            0x00100EC
+#define ACP_SCRATCH_REG_60                            0x00100F0
+#define ACP_SCRATCH_REG_61                            0x00100F4
+#define ACP_SCRATCH_REG_62                            0x00100F8
+#define ACP_SCRATCH_REG_63                            0x00100FC
+#define ACP_SCRATCH_REG_64                            0x0010100
+#define ACP_SCRATCH_REG_65                            0x0010104
+#define ACP_SCRATCH_REG_66                            0x0010108
+#define ACP_SCRATCH_REG_67                            0x001010C
+#define ACP_SCRATCH_REG_68                            0x0010110
+#define ACP_SCRATCH_REG_69                            0x0010114
+#define ACP_SCRATCH_REG_70                            0x0010118
+#define ACP_SCRATCH_REG_71                            0x001011C
+#define ACP_SCRATCH_REG_72                            0x0010120
+#define ACP_SCRATCH_REG_73                            0x0010124
+#define ACP_SCRATCH_REG_74                            0x0010128
+#define ACP_SCRATCH_REG_75                            0x001012C
+#define ACP_SCRATCH_REG_76                            0x0010130
+#define ACP_SCRATCH_REG_77                            0x0010134
+#define ACP_SCRATCH_REG_78                            0x0010138
+#define ACP_SCRATCH_REG_79                            0x001013C
+#define ACP_SCRATCH_REG_80                            0x0010140
+#define ACP_SCRATCH_REG_81                            0x0010144
+#define ACP_SCRATCH_REG_82                            0x0010148
+#define ACP_SCRATCH_REG_83                            0x001014C
+#define ACP_SCRATCH_REG_84                            0x0010150
+#define ACP_SCRATCH_REG_85                            0x0010154
+#define ACP_SCRATCH_REG_86                            0x0010158
+#define ACP_SCRATCH_REG_87                            0x001015C
+#define ACP_SCRATCH_REG_88                            0x0010160
+#define ACP_SCRATCH_REG_89                            0x0010164
+#define ACP_SCRATCH_REG_90                            0x0010168
+#define ACP_SCRATCH_REG_91                            0x001016C
+#define ACP_SCRATCH_REG_92                            0x0010170
+#define ACP_SCRATCH_REG_93                            0x0010174
+#define ACP_SCRATCH_REG_94                            0x0010178
+#define ACP_SCRATCH_REG_95                            0x001017C
+#define ACP_SCRATCH_REG_96                            0x0010180
+#define ACP_SCRATCH_REG_97                            0x0010184
+#define ACP_SCRATCH_REG_98                            0x0010188
+#define ACP_SCRATCH_REG_99                            0x001018C
+#define ACP_SCRATCH_REG_100                           0x0010190
+#define ACP_SCRATCH_REG_101                           0x0010194
+#define ACP_SCRATCH_REG_102                           0x0010198
+#define ACP_SCRATCH_REG_103                           0x001019C
+#define ACP_SCRATCH_REG_104                           0x00101A0
+#define ACP_SCRATCH_REG_105                           0x00101A4
+#define ACP_SCRATCH_REG_106                           0x00101A8
+#define ACP_SCRATCH_REG_107                           0x00101AC
+#define ACP_SCRATCH_REG_108                           0x00101B0
+#define ACP_SCRATCH_REG_109                           0x00101B4
+#define ACP_SCRATCH_REG_110                           0x00101B8
+#define ACP_SCRATCH_REG_111                           0x00101BC
+#define ACP_SCRATCH_REG_112                           0x00101C0
+#define ACP_SCRATCH_REG_113                           0x00101C4
+#define ACP_SCRATCH_REG_114                           0x00101C8
+#define ACP_SCRATCH_REG_115                           0x00101CC
+#define ACP_SCRATCH_REG_116                           0x00101D0
+#define ACP_SCRATCH_REG_117                           0x00101D4
+#define ACP_SCRATCH_REG_118                           0x00101D8
+#define ACP_SCRATCH_REG_119                           0x00101DC
+#define ACP_SCRATCH_REG_120                           0x00101E0
+#define ACP_SCRATCH_REG_121                           0x00101E4
+#define ACP_SCRATCH_REG_122                           0x00101E8
+#define ACP_SCRATCH_REG_123                           0x00101EC
+#define ACP_SCRATCH_REG_124                           0x00101F0
+#define ACP_SCRATCH_REG_125                           0x00101F4
+#define ACP_SCRATCH_REG_126                           0x00101F8
+#define ACP_SCRATCH_REG_127                           0x00101FC
+#define ACP_SCRATCH_REG_128                           0x0010200
+#endif
diff --git a/include/sound/cs42l42.h b/include/sound/cs42l42.h
index a55d522f1772..1d1c24fdd0ca 100644
--- a/include/sound/cs42l42.h
+++ b/include/sound/cs42l42.h
@@ -40,6 +40,7 @@
 #define CS42L42_PAGE_30		0x3000
 
 #define CS42L42_CHIP_ID		0x42A42
+#define CS42L83_CHIP_ID		0x42A83
 
 /* Page 0x10 Global Registers */
 #define CS42L42_DEVID_AB		(CS42L42_PAGE_10 + 0x01)
diff --git a/include/sound/hda_codec.h b/include/sound/hda_codec.h
index 6d3c82c4b6ac..25ec8c181688 100644
--- a/include/sound/hda_codec.h
+++ b/include/sound/hda_codec.h
@@ -258,7 +258,6 @@ struct hda_codec {
 	unsigned int link_down_at_suspend:1; /* link down at runtime suspend */
 	unsigned int relaxed_resume:1;	/* don't resume forcibly for jack */
 	unsigned int forced_resume:1; /* forced resume for jack */
-	unsigned int mst_no_extra_pcms:1; /* no backup PCMs for DP-MST */
 
 #ifdef CONFIG_PM
 	unsigned long power_on_acct;
@@ -293,8 +292,6 @@ struct hda_codec {
 #define dev_to_hda_codec(_dev)	container_of(_dev, struct hda_codec, core.dev)
 #define hda_codec_dev(_dev)	(&(_dev)->core.dev)
 
-#define hdac_to_hda_priv(_hdac) \
-			container_of(_hdac, struct hdac_hda_priv, codec.core)
 #define hdac_to_hda_codec(_hdac) container_of(_hdac, struct hda_codec, core)
 
 #define list_for_each_codec(c, bus) \
diff --git a/include/sound/hda_register.h b/include/sound/hda_register.h
index ad8b71b1dbb6..d37cf43546eb 100644
--- a/include/sound/hda_register.h
+++ b/include/sound/hda_register.h
@@ -260,7 +260,18 @@ enum { SDI0, SDI1, SDI2, SDI3, SDO0, SDO1, SDO2, SDO3 };
 
 #define AZX_REG_ML_LCAP			0x00
 #define AZX_REG_ML_LCTL			0x04
+
+#define AZX_ML_LCTL_CPA			BIT(23)
+#define AZX_ML_LCTL_CPA_SHIFT		23
+#define AZX_ML_LCTL_SPA			BIT(16)
+#define AZX_ML_LCTL_SPA_SHIFT		16
+#define AZX_ML_LCTL_SCF			GENMASK(3, 0)
+
 #define AZX_REG_ML_LOSIDV		0x08
+
+/* bit0 is reserved, with BIT(1) mapping to stream1 */
+#define AZX_ML_LOSIDV_STREAM_MASK	0xFFFE
+
 #define AZX_REG_ML_LSDIID		0x0C
 #define AZX_REG_ML_LPSOO		0x10
 #define AZX_REG_ML_LPSIO		0x12
@@ -268,15 +279,6 @@ enum { SDI0, SDI1, SDI2, SDI3, SDO0, SDO1, SDO2, SDO3 };
 #define AZX_REG_ML_LOUTPAY		0x20
 #define AZX_REG_ML_LINPAY		0x30
 
-/* bit0 is reserved, with BIT(1) mapping to stream1 */
-#define ML_LOSIDV_STREAM_MASK		0xFFFE
-
-#define ML_LCTL_SCF_MASK			0xF
-#define AZX_MLCTL_SPA				(0x1 << 16)
-#define AZX_MLCTL_CPA				(0x1 << 23)
-#define AZX_MLCTL_SPA_SHIFT			16
-#define AZX_MLCTL_CPA_SHIFT			23
-
 /* registers for DMA Resume Capability Structure */
 #define AZX_DRSM_CAP_ID			0x5
 #define AZX_REG_DRSM_CTL		0x4
diff --git a/include/sound/hdaudio.h b/include/sound/hdaudio.h
index 797bf67a164d..ddff03e546e9 100644
--- a/include/sound/hdaudio.h
+++ b/include/sound/hdaudio.h
@@ -10,6 +10,7 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/iopoll.h>
 #include <linux/pm_runtime.h>
 #include <linux/timecounter.h>
 #include <sound/core.h>
@@ -550,6 +551,7 @@ void snd_hdac_stream_init(struct hdac_bus *bus, struct hdac_stream *azx_dev,
 			  int idx, int direction, int tag);
 struct hdac_stream *snd_hdac_stream_assign(struct hdac_bus *bus,
 					   struct snd_pcm_substream *substream);
+void snd_hdac_stream_release_locked(struct hdac_stream *azx_dev);
 void snd_hdac_stream_release(struct hdac_stream *azx_dev);
 struct hdac_stream *snd_hdac_get_stream(struct hdac_bus *bus,
 					int dir, int stream_tag);
@@ -560,8 +562,8 @@ int snd_hdac_stream_setup_periods(struct hdac_stream *azx_dev);
 int snd_hdac_stream_set_params(struct hdac_stream *azx_dev,
 				unsigned int format_val);
 void snd_hdac_stream_start(struct hdac_stream *azx_dev, bool fresh_start);
-void snd_hdac_stream_clear(struct hdac_stream *azx_dev);
 void snd_hdac_stream_stop(struct hdac_stream *azx_dev);
+void snd_hdac_stop_streams(struct hdac_bus *bus);
 void snd_hdac_stop_streams_and_chip(struct hdac_bus *bus);
 void snd_hdac_stream_reset(struct hdac_stream *azx_dev);
 void snd_hdac_stream_sync_trigger(struct hdac_stream *azx_dev, bool set,
@@ -589,6 +591,12 @@ int snd_hdac_get_stream_stripe_ctl(struct hdac_bus *bus,
 	snd_hdac_reg_readw((dev)->bus, (dev)->sd_addr + AZX_REG_ ## reg)
 #define snd_hdac_stream_readb(dev, reg) \
 	snd_hdac_reg_readb((dev)->bus, (dev)->sd_addr + AZX_REG_ ## reg)
+#define snd_hdac_stream_readb_poll(dev, reg, val, cond, delay_us, timeout_us) \
+	readb_poll_timeout((dev)->sd_addr + AZX_REG_ ## reg, val, cond, \
+			   delay_us, timeout_us)
+#define snd_hdac_stream_readl_poll(dev, reg, val, cond, delay_us, timeout_us) \
+	readl_poll_timeout((dev)->sd_addr + AZX_REG_ ## reg, val, cond, \
+			   delay_us, timeout_us)
 
 /* update a register, pass without AZX_REG_ prefix */
 #define snd_hdac_stream_updatel(dev, reg, mask, val) \
diff --git a/include/sound/hdaudio_ext.h b/include/sound/hdaudio_ext.h
index d26234f9ee46..83aed26ab143 100644
--- a/include/sound/hdaudio_ext.h
+++ b/include/sound/hdaudio_ext.h
@@ -11,9 +11,6 @@ int snd_hdac_ext_bus_init(struct hdac_bus *bus, struct device *dev,
 		      const struct hdac_ext_bus_ops *ext_ops);
 
 void snd_hdac_ext_bus_exit(struct hdac_bus *bus);
-int snd_hdac_ext_bus_device_init(struct hdac_bus *bus, int addr,
-				struct hdac_device *hdev, int type);
-void snd_hdac_ext_bus_device_exit(struct hdac_device *hdev);
 void snd_hdac_ext_bus_device_remove(struct hdac_bus *bus);
 
 #define HDA_CODEC_REV_EXT_ENTRY(_vid, _rev, _name, drv_data) \
@@ -80,12 +77,9 @@ struct hdac_ext_stream {
 #define stream_to_hdac_ext_stream(s) \
 	container_of(s, struct hdac_ext_stream, hstream)
 
-void snd_hdac_ext_stream_init(struct hdac_bus *bus,
-			      struct hdac_ext_stream *hext_stream, int idx,
-			      int direction, int tag);
 int snd_hdac_ext_stream_init_all(struct hdac_bus *bus, int start_idx,
 				 int num_stream, int dir);
-void snd_hdac_stream_free_all(struct hdac_bus *bus);
+void snd_hdac_ext_stream_free_all(struct hdac_bus *bus);
 void snd_hdac_link_free_all(struct hdac_bus *bus);
 struct hdac_ext_stream *snd_hdac_ext_stream_assign(struct hdac_bus *bus,
 					   struct snd_pcm_substream *substream,
@@ -188,12 +182,6 @@ void snd_hdac_ext_bus_link_power(struct hdac_device *codec, bool enable);
 #define snd_hdac_adsp_readq_poll(chip, reg, val, cond, delay_us, timeout_us) \
 	readq_poll_timeout((chip)->dsp_ba + (reg), val, cond, \
 			   delay_us, timeout_us)
-#define snd_hdac_stream_readb_poll(strm, reg, val, cond, delay_us, timeout_us) \
-	readb_poll_timeout((strm)->sd_addr + AZX_REG_ ## reg, val, cond, \
-			   delay_us, timeout_us)
-#define snd_hdac_stream_readl_poll(strm, reg, val, cond, delay_us, timeout_us) \
-	readl_poll_timeout((strm)->sd_addr + AZX_REG_ ## reg, val, cond, \
-			   delay_us, timeout_us)
 
 struct hdac_ext_device;
 
diff --git a/include/sound/intel-nhlt.h b/include/sound/intel-nhlt.h
index 3d5cf201cd80..53470d6a28d6 100644
--- a/include/sound/intel-nhlt.h
+++ b/include/sound/intel-nhlt.h
@@ -136,6 +136,8 @@ bool intel_nhlt_has_endpoint_type(struct nhlt_acpi_table *nhlt, u8 link_type);
 
 int intel_nhlt_ssp_endpoint_mask(struct nhlt_acpi_table *nhlt, u8 device_type);
 
+int intel_nhlt_ssp_mclk_mask(struct nhlt_acpi_table *nhlt, int ssp_num);
+
 struct nhlt_specific_cfg *
 intel_nhlt_get_endpoint_blob(struct device *dev, struct nhlt_acpi_table *nhlt,
 			     u32 bus_id, u8 link_type, u8 vbps, u8 bps,
@@ -169,6 +171,11 @@ static inline int intel_nhlt_ssp_endpoint_mask(struct nhlt_acpi_table *nhlt, u8
 	return 0;
 }
 
+static inline int intel_nhlt_ssp_mclk_mask(struct nhlt_acpi_table *nhlt, int ssp_num)
+{
+	return 0;
+}
+
 static inline struct nhlt_specific_cfg *
 intel_nhlt_get_endpoint_blob(struct device *dev, struct nhlt_acpi_table *nhlt,
 			     u32 bus_id, u8 link_type, u8 vbps, u8 bps,
diff --git a/include/sound/memalloc.h b/include/sound/memalloc.h
index 8d79cebf95f3..43d524580bd2 100644
--- a/include/sound/memalloc.h
+++ b/include/sound/memalloc.h
@@ -26,9 +26,6 @@ struct snd_dma_device {
 	struct device *dev;		/* generic device */
 };
 
-#define snd_dma_continuous_data(x)	((struct device *)(__force unsigned long)(x))
-
-
 /*
  * buffer types
  */
diff --git a/include/sound/pcm.h b/include/sound/pcm.h
index 8c48a5bce88c..7b1a022910e8 100644
--- a/include/sound/pcm.h
+++ b/include/sound/pcm.h
@@ -346,6 +346,8 @@ static inline void snd_pcm_pack_audio_tstamp_report(__u32 *data, __u32 *accuracy
 
 struct snd_pcm_runtime {
 	/* -- Status -- */
+	snd_pcm_state_t state;		/* stream state */
+	snd_pcm_state_t suspended_state; /* suspended stream state */
 	struct snd_pcm_substream *trigger_master;
 	struct timespec64 trigger_tstamp;	/* trigger timestamp */
 	bool trigger_tstamp_latched;     /* trigger timestamp latched in low-level driver/hardware */
@@ -678,12 +680,26 @@ void snd_pcm_stream_unlock_irqrestore(struct snd_pcm_substream *substream,
  */
 static inline int snd_pcm_running(struct snd_pcm_substream *substream)
 {
-	return (substream->runtime->status->state == SNDRV_PCM_STATE_RUNNING ||
-		(substream->runtime->status->state == SNDRV_PCM_STATE_DRAINING &&
+	return (substream->runtime->state == SNDRV_PCM_STATE_RUNNING ||
+		(substream->runtime->state == SNDRV_PCM_STATE_DRAINING &&
 		 substream->stream == SNDRV_PCM_STREAM_PLAYBACK));
 }
 
 /**
+ * __snd_pcm_set_state - Change the current PCM state
+ * @runtime: PCM runtime to set
+ * @state: the current state to set
+ *
+ * Call within the stream lock
+ */
+static inline void __snd_pcm_set_state(struct snd_pcm_runtime *runtime,
+				       snd_pcm_state_t state)
+{
+	runtime->state = state;
+	runtime->status->state = state; /* copy for mmap */
+}
+
+/**
  * bytes_to_samples - Unit conversion of the size from bytes to samples
  * @runtime: PCM runtime instance
  * @size: size in bytes
diff --git a/include/sound/simple_card_utils.h b/include/sound/simple_card_utils.h
index ab55f40896e0..a0b827f0c2f6 100644
--- a/include/sound/simple_card_utils.h
+++ b/include/sound/simple_card_utils.h
@@ -39,6 +39,7 @@ struct asoc_simple_dai {
 struct asoc_simple_data {
 	u32 convert_rate;
 	u32 convert_channels;
+	const char *convert_sample_format;
 };
 
 struct asoc_simple_jack {
diff --git a/include/sound/soc-acpi-intel-match.h b/include/sound/soc-acpi-intel-match.h
index bc7fd46ec2bc..82a7db23db69 100644
--- a/include/sound/soc-acpi-intel-match.h
+++ b/include/sound/soc-acpi-intel-match.h
@@ -14,7 +14,6 @@
  * these tables are not constants, some fields can be used for
  * pdata or machine ops
  */
-extern struct snd_soc_acpi_mach snd_soc_acpi_intel_haswell_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_broadwell_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_baytrail_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_cherrytrail_machines[];
@@ -30,6 +29,7 @@ extern struct snd_soc_acpi_mach snd_soc_acpi_intel_tgl_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_ehl_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_jsl_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_adl_machines[];
+extern struct snd_soc_acpi_mach snd_soc_acpi_intel_rpl_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_mtl_machines[];
 
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_cnl_sdw_machines[];
@@ -38,6 +38,7 @@ extern struct snd_soc_acpi_mach snd_soc_acpi_intel_cml_sdw_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_icl_sdw_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_tgl_sdw_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_adl_sdw_machines[];
+extern struct snd_soc_acpi_mach snd_soc_acpi_intel_rpl_sdw_machines[];
 extern struct snd_soc_acpi_mach snd_soc_acpi_intel_mtl_sdw_machines[];
 
 /*
diff --git a/include/sound/soc.h b/include/sound/soc.h
index aad24a1d3276..37bbfc8b45cb 100644
--- a/include/sound/soc.h
+++ b/include/sound/soc.h
@@ -31,31 +31,31 @@
 #define SOC_DOUBLE_VALUE(xreg, shift_left, shift_right, xmax, xinvert, xautodisable) \
 	((unsigned long)&(struct soc_mixer_control) \
 	{.reg = xreg, .rreg = xreg, .shift = shift_left, \
-	.rshift = shift_right, .max = xmax, .platform_max = xmax, \
+	.rshift = shift_right, .max = xmax, \
 	.invert = xinvert, .autodisable = xautodisable})
 #define SOC_DOUBLE_S_VALUE(xreg, shift_left, shift_right, xmin, xmax, xsign_bit, xinvert, xautodisable) \
 	((unsigned long)&(struct soc_mixer_control) \
 	{.reg = xreg, .rreg = xreg, .shift = shift_left, \
-	.rshift = shift_right, .min = xmin, .max = xmax, .platform_max = xmax, \
+	.rshift = shift_right, .min = xmin, .max = xmax, \
 	.sign_bit = xsign_bit, .invert = xinvert, .autodisable = xautodisable})
 #define SOC_SINGLE_VALUE(xreg, xshift, xmax, xinvert, xautodisable) \
 	SOC_DOUBLE_VALUE(xreg, xshift, xshift, xmax, xinvert, xautodisable)
 #define SOC_SINGLE_VALUE_EXT(xreg, xmax, xinvert) \
 	((unsigned long)&(struct soc_mixer_control) \
-	{.reg = xreg, .max = xmax, .platform_max = xmax, .invert = xinvert})
+	{.reg = xreg, .max = xmax, .invert = xinvert})
 #define SOC_DOUBLE_R_VALUE(xlreg, xrreg, xshift, xmax, xinvert) \
 	((unsigned long)&(struct soc_mixer_control) \
 	{.reg = xlreg, .rreg = xrreg, .shift = xshift, .rshift = xshift, \
-	.max = xmax, .platform_max = xmax, .invert = xinvert})
+	.max = xmax, .invert = xinvert})
 #define SOC_DOUBLE_R_S_VALUE(xlreg, xrreg, xshift, xmin, xmax, xsign_bit, xinvert) \
 	((unsigned long)&(struct soc_mixer_control) \
 	{.reg = xlreg, .rreg = xrreg, .shift = xshift, .rshift = xshift, \
-	.max = xmax, .min = xmin, .platform_max = xmax, .sign_bit = xsign_bit, \
+	.max = xmax, .min = xmin, .sign_bit = xsign_bit, \
 	.invert = xinvert})
 #define SOC_DOUBLE_R_RANGE_VALUE(xlreg, xrreg, xshift, xmin, xmax, xinvert) \
 	((unsigned long)&(struct soc_mixer_control) \
 	{.reg = xlreg, .rreg = xrreg, .shift = xshift, .rshift = xshift, \
-	.min = xmin, .max = xmax, .platform_max = xmax, .invert = xinvert})
+	.min = xmin, .max = xmax, .invert = xinvert})
 #define SOC_SINGLE(xname, reg, shift, max, invert) \
 {	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
 	.info = snd_soc_info_volsw, .get = snd_soc_get_volsw,\
@@ -68,7 +68,7 @@
 	.private_value = (unsigned long)&(struct soc_mixer_control) \
 		{.reg = xreg, .rreg = xreg, .shift = xshift, \
 		 .rshift = xshift,  .min = xmin, .max = xmax, \
-		 .platform_max = xmax, .invert = xinvert} }
+		 .invert = xinvert} }
 #define SOC_SINGLE_TLV(xname, reg, shift, max, invert, tlv_array) \
 {	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = xname, \
 	.access = SNDRV_CTL_ELEM_ACCESS_TLV_READ |\
@@ -99,7 +99,7 @@
 	.private_value = (unsigned long)&(struct soc_mixer_control) \
 		{.reg = xreg, .rreg = xreg, .shift = xshift, \
 		 .rshift = xshift, .min = xmin, .max = xmax, \
-		 .platform_max = xmax, .invert = xinvert} }
+		 .invert = xinvert} }
 #define SOC_DOUBLE(xname, reg, shift_left, shift_right, max, invert) \
 {	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = (xname),\
 	.info = snd_soc_info_volsw, .get = snd_soc_get_volsw, \
@@ -199,7 +199,7 @@
 	.put = snd_soc_put_volsw, \
 	.private_value = (unsigned long)&(struct soc_mixer_control) \
 	{.reg = xreg, .rreg = xreg,  \
-	 .min = xmin, .max = xmax, .platform_max = xmax, \
+	 .min = xmin, .max = xmax, \
 	.sign_bit = 7,} }
 #define SOC_DOUBLE_S8_TLV(xname, xreg, xmin, xmax, tlv_array) \
 {	.iface  = SNDRV_CTL_ELEM_IFACE_MIXER, .name = (xname), \
@@ -273,7 +273,7 @@
 	.private_value = (unsigned long)&(struct soc_mixer_control) \
 		{.reg = xreg, .rreg = xreg, .shift = xshift, \
 		 .rshift = xshift, .min = xmin, .max = xmax, \
-		 .platform_max = xmax, .invert = xinvert} }
+		 .invert = xinvert} }
 #define SOC_DOUBLE_EXT_TLV(xname, xreg, shift_left, shift_right, xmax, xinvert,\
 	 xhandler_get, xhandler_put, tlv_array) \
 {	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = (xname), \
@@ -1062,7 +1062,8 @@ struct snd_soc_pcm_runtime {
 	unsigned int params_select; /* currently selected param for dai link */
 
 	/* Dynamic PCM BE runtime data */
-	struct snd_soc_dpcm_runtime dpcm[2];
+	struct snd_soc_dpcm_runtime dpcm[SNDRV_PCM_STREAM_LAST + 1];
+	struct snd_soc_dapm_widget *c2c_widget[SNDRV_PCM_STREAM_LAST + 1];
 
 	long pmdown_time;
 
@@ -1078,11 +1079,6 @@ struct snd_soc_pcm_runtime {
 	 *	asoc_rtd_to_codec()
 	 */
 	struct snd_soc_dai **dais;
-	unsigned int num_codecs;
-	unsigned int num_cpus;
-
-	struct snd_soc_dapm_widget *playback_widget;
-	struct snd_soc_dapm_widget *capture_widget;
 
 	struct delayed_work delayed_work;
 	void (*close_delayed_work_func)(struct snd_soc_pcm_runtime *rtd);
@@ -1108,7 +1104,7 @@ struct snd_soc_pcm_runtime {
 };
 /* see soc_new_pcm_runtime()  */
 #define asoc_rtd_to_cpu(rtd, n)   (rtd)->dais[n]
-#define asoc_rtd_to_codec(rtd, n) (rtd)->dais[n + (rtd)->num_cpus]
+#define asoc_rtd_to_codec(rtd, n) (rtd)->dais[n + (rtd)->dai_link->num_cpus]
 #define asoc_substream_to_rtd(substream) \
 	(struct snd_soc_pcm_runtime *)snd_pcm_substream_chip(substream)
 
@@ -1118,15 +1114,15 @@ struct snd_soc_pcm_runtime {
 	     (i)++)
 #define for_each_rtd_cpu_dais(rtd, i, dai)				\
 	for ((i) = 0;							\
-	     ((i) < rtd->num_cpus) && ((dai) = asoc_rtd_to_cpu(rtd, i)); \
+	     ((i) < rtd->dai_link->num_cpus) && ((dai) = asoc_rtd_to_cpu(rtd, i)); \
 	     (i)++)
 #define for_each_rtd_codec_dais(rtd, i, dai)				\
 	for ((i) = 0;							\
-	     ((i) < rtd->num_codecs) && ((dai) = asoc_rtd_to_codec(rtd, i)); \
+	     ((i) < rtd->dai_link->num_codecs) && ((dai) = asoc_rtd_to_codec(rtd, i)); \
 	     (i)++)
 #define for_each_rtd_dais(rtd, i, dai)					\
 	for ((i) = 0;							\
-	     ((i) < (rtd)->num_cpus + (rtd)->num_codecs) &&		\
+	     ((i) < (rtd)->dai_link->num_cpus + (rtd)->dai_link->num_codecs) &&	\
 		     ((dai) = (rtd)->dais[i]);				\
 	     (i)++)
 
diff --git a/include/sound/sof.h b/include/sound/sof.h
index 367dccfea7ad..341fef19e612 100644
--- a/include/sound/sof.h
+++ b/include/sound/sof.h
@@ -89,6 +89,7 @@ struct snd_sof_pdata {
 	/* machine */
 	struct platform_device *pdev_mach;
 	const struct snd_soc_acpi_mach *machine;
+	const struct snd_sof_of_mach *of_machine;
 
 	void *hw_pdata;
 
@@ -102,6 +103,7 @@ struct snd_sof_pdata {
 struct sof_dev_desc {
 	/* list of machines using this configuration */
 	struct snd_soc_acpi_mach *machines;
+	struct snd_sof_of_mach *of_machines;
 
 	/* alternate list of machines using this configuration */
 	struct snd_soc_acpi_mach *alt_machines;
diff --git a/include/sound/sof/control.h b/include/sound/sof/control.h
index 7379a33d7247..983d374fe511 100644
--- a/include/sound/sof/control.h
+++ b/include/sound/sof/control.h
@@ -117,11 +117,11 @@ struct sof_ipc_ctrl_data {
 	/* control data - add new types if needed */
 	union {
 		/* channel values can be used by volume type controls */
-		struct sof_ipc_ctrl_value_chan chanv[0];
+		DECLARE_FLEX_ARRAY(struct sof_ipc_ctrl_value_chan, chanv);
 		/* component values used by routing controls like mux, mixer */
-		struct sof_ipc_ctrl_value_comp compv[0];
+		DECLARE_FLEX_ARRAY(struct sof_ipc_ctrl_value_comp, compv);
 		/* data can be used by binary controls */
-		struct sof_abi_hdr data[0];
+		DECLARE_FLEX_ARRAY(struct sof_abi_hdr, data);
 	};
 } __packed;
 
diff --git a/include/sound/sof/dai.h b/include/sound/sof/dai.h
index 21d98f31a9ca..83fd81c82e4c 100644
--- a/include/sound/sof/dai.h
+++ b/include/sound/sof/dai.h
@@ -84,6 +84,7 @@ enum sof_ipc_dai_type {
 	SOF_DAI_AMD_BT,			/**< AMD ACP BT*/
 	SOF_DAI_AMD_SP,			/**< AMD ACP SP */
 	SOF_DAI_AMD_DMIC,		/**< AMD ACP DMIC */
+	SOF_DAI_AMD_HS,			/**< Amd HS */
 	SOF_DAI_MEDIATEK_AFE,		/**< Mediatek AFE */
 };
 
@@ -112,6 +113,7 @@ struct sof_ipc_dai_config {
 		struct sof_ipc_dai_acp_params acpbt;
 		struct sof_ipc_dai_acp_params acpsp;
 		struct sof_ipc_dai_acpdmic_params acpdmic;
+		struct sof_ipc_dai_acp_params acphs;
 		struct sof_ipc_dai_mtk_afe_params afe;
 	};
 } __packed;
diff --git a/include/sound/sof/ipc4/header.h b/include/sound/sof/ipc4/header.h
index a795deacc2ea..99efe0ef1784 100644
--- a/include/sound/sof/ipc4/header.h
+++ b/include/sound/sof/ipc4/header.h
@@ -427,6 +427,11 @@ struct sof_ipc4_dx_state_info {
 #define SOF_IPC4_NOTIFICATION_TYPE_GET(x)	(((x) & SOF_IPC4_NOTIFICATION_TYPE_MASK) >> \
 						 SOF_IPC4_NOTIFICATION_TYPE_SHIFT)
 
+#define SOF_IPC4_LOG_CORE_SHIFT			12
+#define SOF_IPC4_LOG_CORE_MASK			GENMASK(15, 12)
+#define SOF_IPC4_LOG_CORE_GET(x)		(((x) & SOF_IPC4_LOG_CORE_MASK) >> \
+						 SOF_IPC4_LOG_CORE_SHIFT)
+
 /* Value of notification type field - must fit into 8 bits */
 enum sof_ipc4_notification_type {
 	/* Phrase detected (notification from WoV module) */
diff --git a/include/trace/events/dlm.h b/include/trace/events/dlm.h
index bad21222130e..da0eaae98fa3 100644
--- a/include/trace/events/dlm.h
+++ b/include/trace/events/dlm.h
@@ -49,7 +49,7 @@
 /* note: we begin tracing dlm_lock_start() only if ls and lkb are found */
 TRACE_EVENT(dlm_lock_start,
 
-	TP_PROTO(struct dlm_ls *ls, struct dlm_lkb *lkb, void *name,
+	TP_PROTO(struct dlm_ls *ls, struct dlm_lkb *lkb, const void *name,
 		 unsigned int namelen, int mode, __u32 flags),
 
 	TP_ARGS(ls, lkb, name, namelen, mode, flags),
@@ -91,10 +91,11 @@ TRACE_EVENT(dlm_lock_start,
 
 TRACE_EVENT(dlm_lock_end,
 
-	TP_PROTO(struct dlm_ls *ls, struct dlm_lkb *lkb, void *name,
-		 unsigned int namelen, int mode, __u32 flags, int error),
+	TP_PROTO(struct dlm_ls *ls, struct dlm_lkb *lkb, const void *name,
+		 unsigned int namelen, int mode, __u32 flags, int error,
+		 bool kernel_lock),
 
-	TP_ARGS(ls, lkb, name, namelen, mode, flags, error),
+	TP_ARGS(ls, lkb, name, namelen, mode, flags, error, kernel_lock),
 
 	TP_STRUCT__entry(
 		__field(__u32, ls_id)
@@ -113,6 +114,7 @@ TRACE_EVENT(dlm_lock_end,
 		__entry->lkb_id = lkb->lkb_id;
 		__entry->mode = mode;
 		__entry->flags = flags;
+		__entry->error = error;
 
 		r = lkb->lkb_resource;
 		if (r)
@@ -122,14 +124,14 @@ TRACE_EVENT(dlm_lock_end,
 			memcpy(__get_dynamic_array(res_name), name,
 			       __get_dynamic_array_len(res_name));
 
-		/* return value will be zeroed in those cases by dlm_lock()
-		 * we do it here again to not introduce more overhead if
-		 * trace isn't running and error reflects the return value.
-		 */
-		if (error == -EAGAIN || error == -EDEADLK)
-			__entry->error = 0;
-		else
-			__entry->error = error;
+		if (kernel_lock) {
+			/* return value will be zeroed in those cases by dlm_lock()
+			 * we do it here again to not introduce more overhead if
+			 * trace isn't running and error reflects the return value.
+			 */
+			if (error == -EAGAIN || error == -EDEADLK)
+				__entry->error = 0;
+		}
 
 	),
 
diff --git a/include/trace/events/erofs.h b/include/trace/events/erofs.h
index 57de057bd503..4f4c44ea3a65 100644
--- a/include/trace/events/erofs.h
+++ b/include/trace/events/erofs.h
@@ -53,15 +53,14 @@ TRACE_EVENT(erofs_lookup,
 );
 
 TRACE_EVENT(erofs_fill_inode,
-	TP_PROTO(struct inode *inode, int isdir),
-	TP_ARGS(inode, isdir),
+	TP_PROTO(struct inode *inode),
+	TP_ARGS(inode),
 
 	TP_STRUCT__entry(
 		__field(dev_t,		dev	)
 		__field(erofs_nid_t,	nid	)
 		__field(erofs_blk_t,	blkaddr )
 		__field(unsigned int,	ofs	)
-		__field(int,		isdir	)
 	),
 
 	TP_fast_assign(
@@ -69,13 +68,11 @@ TRACE_EVENT(erofs_fill_inode,
 		__entry->nid		= EROFS_I(inode)->nid;
 		__entry->blkaddr	= erofs_blknr(iloc(EROFS_I_SB(inode), __entry->nid));
 		__entry->ofs		= erofs_blkoff(iloc(EROFS_I_SB(inode), __entry->nid));
-		__entry->isdir		= isdir;
 	),
 
-	TP_printk("dev = (%d,%d), nid = %llu, blkaddr %u ofs %u, isdir %d",
+	TP_printk("dev = (%d,%d), nid = %llu, blkaddr %u ofs %u",
 		  show_dev_nid(__entry),
-		  __entry->blkaddr, __entry->ofs,
-		  __entry->isdir)
+		  __entry->blkaddr, __entry->ofs)
 );
 
 TRACE_EVENT(erofs_readpage,
diff --git a/include/trace/events/sof.h b/include/trace/events/sof.h
new file mode 100644
index 000000000000..21c2a1efb9f6
--- /dev/null
+++ b/include/trace/events/sof.h
@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright(c) 2022 Intel Corporation. All rights reserved.
+ *
+ * Author: Noah Klayman <noah.klayman@intel.com>
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM sof
+
+#if !defined(_TRACE_SOF_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_SOF_H
+#include <linux/tracepoint.h>
+#include <linux/types.h>
+#include <sound/sof/stream.h>
+#include "../../../sound/soc/sof/sof-audio.h"
+
+DECLARE_EVENT_CLASS(sof_widget_template,
+	TP_PROTO(struct snd_sof_widget *swidget),
+	TP_ARGS(swidget),
+	TP_STRUCT__entry(
+		__string(name, swidget->widget->name)
+		__field(int, use_count)
+	),
+	TP_fast_assign(
+		__assign_str(name, swidget->widget->name);
+		__entry->use_count = swidget->use_count;
+	),
+	TP_printk("name=%s use_count=%d", __get_str(name), __entry->use_count)
+);
+
+DEFINE_EVENT(sof_widget_template, sof_widget_setup,
+	TP_PROTO(struct snd_sof_widget *swidget),
+	TP_ARGS(swidget)
+);
+
+DEFINE_EVENT(sof_widget_template, sof_widget_free,
+	TP_PROTO(struct snd_sof_widget *swidget),
+	TP_ARGS(swidget)
+);
+
+TRACE_EVENT(sof_ipc3_period_elapsed_position,
+	TP_PROTO(struct snd_sof_dev *sdev, struct sof_ipc_stream_posn *posn),
+	TP_ARGS(sdev, posn),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(sdev->dev))
+		__field(u64, host_posn)
+		__field(u64, dai_posn)
+		__field(u64, wallclock)
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(sdev->dev));
+		__entry->host_posn = posn->host_posn;
+		__entry->dai_posn = posn->dai_posn;
+		__entry->wallclock = posn->wallclock;
+	),
+	TP_printk("device_name=%s host_posn=%#llx dai_posn=%#llx wallclock=%#llx",
+		  __get_str(device_name), __entry->host_posn, __entry->dai_posn,
+		  __entry->wallclock)
+);
+
+TRACE_EVENT(sof_pcm_pointer_position,
+	TP_PROTO(struct snd_sof_dev *sdev,
+		struct snd_sof_pcm *spcm,
+		struct snd_pcm_substream *substream,
+		snd_pcm_uframes_t dma_posn,
+		snd_pcm_uframes_t dai_posn
+	),
+	TP_ARGS(sdev, spcm, substream, dma_posn, dai_posn),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(sdev->dev))
+		__field(u32, pcm_id)
+		__field(int, stream)
+		__field(unsigned long, dma_posn)
+		__field(unsigned long, dai_posn)
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(sdev->dev));
+		__entry->pcm_id = le32_to_cpu(spcm->pcm.pcm_id);
+		__entry->stream = substream->stream;
+		__entry->dma_posn = dma_posn;
+		__entry->dai_posn = dai_posn;
+	),
+	TP_printk("device_name=%s pcm_id=%d stream=%d dma_posn=%lu dai_posn=%lu",
+		  __get_str(device_name), __entry->pcm_id, __entry->stream,
+		  __entry->dma_posn, __entry->dai_posn)
+);
+
+TRACE_EVENT(sof_stream_position_ipc_rx,
+	TP_PROTO(struct device *dev),
+	TP_ARGS(dev),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(dev))
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(dev));
+	),
+	TP_printk("device_name=%s", __get_str(device_name))
+);
+
+TRACE_EVENT(sof_ipc4_fw_config,
+	TP_PROTO(struct snd_sof_dev *sdev, char *key, u32 value),
+	TP_ARGS(sdev, key, value),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(sdev->dev))
+		__string(key, key)
+		__field(u32, value)
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(sdev->dev));
+		__assign_str(key, key);
+		__entry->value = value;
+	),
+	TP_printk("device_name=%s key=%s value=%d",
+		  __get_str(device_name), __get_str(key), __entry->value)
+);
+
+#endif /* _TRACE_SOF_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/events/sof_intel.h b/include/trace/events/sof_intel.h
new file mode 100644
index 000000000000..2a77f9d26c0b
--- /dev/null
+++ b/include/trace/events/sof_intel.h
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright(c) 2022 Intel Corporation. All rights reserved.
+ *
+ * Author: Noah Klayman <noah.klayman@intel.com>
+ */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM sof_intel
+
+#if !defined(_TRACE_SOF_INTEL_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_SOF_INTEL_H
+#include <linux/tracepoint.h>
+#include <sound/hdaudio.h>
+#include "../../../sound/soc/sof/sof-audio.h"
+
+TRACE_EVENT(sof_intel_hda_irq,
+	TP_PROTO(struct snd_sof_dev *sdev, char *source),
+	TP_ARGS(sdev, source),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(sdev->dev))
+		__string(source, source)
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(sdev->dev));
+		__assign_str(source, source);
+	),
+	TP_printk("device_name=%s source=%s",
+		  __get_str(device_name), __get_str(source))
+);
+
+DECLARE_EVENT_CLASS(sof_intel_ipc_firmware_template,
+	TP_ARGS(struct snd_sof_dev *sdev, u32 msg, u32 msg_ext),
+	TP_PROTO(sdev, msg, msg_ext),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(sdev->dev))
+		__field(u32, msg)
+		__field(u32, msg_ext)
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(sdev->dev));
+		__entry->msg = msg;
+		__entry->msg_ext = msg_ext;
+	),
+	TP_printk("device_name=%s msg=%#x msg_ext=%#x",
+		  __get_str(device_name), __entry->msg, __entry->msg_ext)
+);
+
+DEFINE_EVENT(sof_intel_ipc_firmware_template, sof_intel_ipc_firmware_response,
+	TP_PROTO(struct snd_sof_dev *sdev, u32 msg, u32 msg_ext),
+	TP_ARGS(sdev, msg, msg_ext)
+);
+
+DEFINE_EVENT(sof_intel_ipc_firmware_template, sof_intel_ipc_firmware_initiated,
+	TP_PROTO(struct snd_sof_dev *sdev, u32 msg, u32 msg_ext),
+	TP_ARGS(sdev, msg, msg_ext)
+);
+
+TRACE_EVENT(sof_intel_D0I3C_updated,
+	TP_PROTO(struct snd_sof_dev *sdev, u8 reg),
+	TP_ARGS(sdev, reg),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(sdev->dev))
+		__field(u8, reg)
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(sdev->dev));
+		__entry->reg = reg;
+	),
+	TP_printk("device_name=%s register=%#x",
+		  __get_str(device_name), __entry->reg)
+);
+
+TRACE_EVENT(sof_intel_hda_irq_ipc_check,
+	TP_PROTO(struct snd_sof_dev *sdev, u32 irq_status),
+	TP_ARGS(sdev, irq_status),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(sdev->dev))
+		__field(u32, irq_status)
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(sdev->dev));
+		__entry->irq_status = irq_status;
+	),
+	TP_printk("device_name=%s irq_status=%#x",
+		  __get_str(device_name), __entry->irq_status)
+);
+
+TRACE_EVENT(sof_intel_hda_dsp_pcm,
+	TP_PROTO(struct snd_sof_dev *sdev,
+		struct hdac_stream *hstream,
+		struct snd_pcm_substream *substream,
+		snd_pcm_uframes_t pos
+	),
+	TP_ARGS(sdev, hstream, substream, pos),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(sdev->dev))
+		__field(u32, hstream_index)
+		__field(u32, substream)
+		__field(unsigned long, pos)
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(sdev->dev));
+		__entry->hstream_index = hstream->index;
+		__entry->substream = substream->stream;
+		__entry->pos = pos;
+	),
+	TP_printk("device_name=%s hstream_index=%d substream=%d pos=%lu",
+		  __get_str(device_name), __entry->hstream_index,
+		  __entry->substream, __entry->pos)
+);
+
+TRACE_EVENT(sof_intel_hda_dsp_stream_status,
+	TP_PROTO(struct device *dev, struct hdac_stream *s, u32 status),
+	TP_ARGS(dev, s, status),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(dev))
+		__field(u32, stream)
+		__field(u32, status)
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(dev));
+		__entry->stream = s->index;
+		__entry->status = status;
+	),
+	TP_printk("device_name=%s stream=%d status=%#x",
+		  __get_str(device_name), __entry->stream, __entry->status)
+);
+
+TRACE_EVENT(sof_intel_hda_dsp_check_stream_irq,
+	TP_PROTO(struct snd_sof_dev *sdev, u32 status),
+	TP_ARGS(sdev, status),
+	TP_STRUCT__entry(
+		__string(device_name, dev_name(sdev->dev))
+		__field(u32, status)
+	),
+	TP_fast_assign(
+		__assign_str(device_name, dev_name(sdev->dev));
+		__entry->status = status;
+	),
+	TP_printk("device_name=%s status=%#x",
+		  __get_str(device_name), __entry->status)
+);
+
+#endif /* _TRACE_SOF_INTEL_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h
index c2c9c674a223..7ee65c0b4f70 100644
--- a/include/uapi/drm/amdgpu_drm.h
+++ b/include/uapi/drm/amdgpu_drm.h
@@ -755,6 +755,14 @@ struct drm_amdgpu_cs_chunk_data {
 	#define AMDGPU_INFO_FW_TOC		0x15
 	/* Subquery id: Query CAP firmware version */
 	#define AMDGPU_INFO_FW_CAP		0x16
+	/* Subquery id: Query GFX RLCP firmware version */
+	#define AMDGPU_INFO_FW_GFX_RLCP		0x17
+	/* Subquery id: Query GFX RLCV firmware version */
+	#define AMDGPU_INFO_FW_GFX_RLCV		0x18
+	/* Subquery id: Query MES_KIQ firmware version */
+	#define AMDGPU_INFO_FW_MES_KIQ		0x19
+	/* Subquery id: Query MES firmware version */
+	#define AMDGPU_INFO_FW_MES		0x1a
 
 /* number of bytes moved for TTM migration */
 #define AMDGPU_INFO_NUM_BYTES_MOVED		0x0f
diff --git a/include/uapi/drm/drm_fourcc.h b/include/uapi/drm/drm_fourcc.h
index 0206f812c569..868d6909b718 100644
--- a/include/uapi/drm/drm_fourcc.h
+++ b/include/uapi/drm/drm_fourcc.h
@@ -99,18 +99,42 @@ extern "C" {
 #define DRM_FORMAT_INVALID	0
 
 /* color index */
+#define DRM_FORMAT_C1		fourcc_code('C', '1', ' ', ' ') /* [7:0] C0:C1:C2:C3:C4:C5:C6:C7 1:1:1:1:1:1:1:1 eight pixels/byte */
+#define DRM_FORMAT_C2		fourcc_code('C', '2', ' ', ' ') /* [7:0] C0:C1:C2:C3 2:2:2:2 four pixels/byte */
+#define DRM_FORMAT_C4		fourcc_code('C', '4', ' ', ' ') /* [7:0] C0:C1 4:4 two pixels/byte */
 #define DRM_FORMAT_C8		fourcc_code('C', '8', ' ', ' ') /* [7:0] C */
 
-/* 8 bpp Red */
+/* 1 bpp Darkness (inverse relationship between channel value and brightness) */
+#define DRM_FORMAT_D1		fourcc_code('D', '1', ' ', ' ') /* [7:0] D0:D1:D2:D3:D4:D5:D6:D7 1:1:1:1:1:1:1:1 eight pixels/byte */
+
+/* 2 bpp Darkness (inverse relationship between channel value and brightness) */
+#define DRM_FORMAT_D2		fourcc_code('D', '2', ' ', ' ') /* [7:0] D0:D1:D2:D3 2:2:2:2 four pixels/byte */
+
+/* 4 bpp Darkness (inverse relationship between channel value and brightness) */
+#define DRM_FORMAT_D4		fourcc_code('D', '4', ' ', ' ') /* [7:0] D0:D1 4:4 two pixels/byte */
+
+/* 8 bpp Darkness (inverse relationship between channel value and brightness) */
+#define DRM_FORMAT_D8		fourcc_code('D', '8', ' ', ' ') /* [7:0] D */
+
+/* 1 bpp Red (direct relationship between channel value and brightness) */
+#define DRM_FORMAT_R1		fourcc_code('R', '1', ' ', ' ') /* [7:0] R0:R1:R2:R3:R4:R5:R6:R7 1:1:1:1:1:1:1:1 eight pixels/byte */
+
+/* 2 bpp Red (direct relationship between channel value and brightness) */
+#define DRM_FORMAT_R2		fourcc_code('R', '2', ' ', ' ') /* [7:0] R0:R1:R2:R3 2:2:2:2 four pixels/byte */
+
+/* 4 bpp Red (direct relationship between channel value and brightness) */
+#define DRM_FORMAT_R4		fourcc_code('R', '4', ' ', ' ') /* [7:0] R0:R1 4:4 two pixels/byte */
+
+/* 8 bpp Red (direct relationship between channel value and brightness) */
 #define DRM_FORMAT_R8		fourcc_code('R', '8', ' ', ' ') /* [7:0] R */
 
-/* 10 bpp Red */
+/* 10 bpp Red (direct relationship between channel value and brightness) */
 #define DRM_FORMAT_R10		fourcc_code('R', '1', '0', ' ') /* [15:0] x:R 6:10 little endian */
 
-/* 12 bpp Red */
+/* 12 bpp Red (direct relationship between channel value and brightness) */
 #define DRM_FORMAT_R12		fourcc_code('R', '1', '2', ' ') /* [15:0] x:R 4:12 little endian */
 
-/* 16 bpp Red */
+/* 16 bpp Red (direct relationship between channel value and brightness) */
 #define DRM_FORMAT_R16		fourcc_code('R', '1', '6', ' ') /* [15:0] R little endian */
 
 /* 16 bpp RG */
@@ -205,7 +229,9 @@ extern "C" {
 #define DRM_FORMAT_VYUY		fourcc_code('V', 'Y', 'U', 'Y') /* [31:0] Y1:Cb0:Y0:Cr0 8:8:8:8 little endian */
 
 #define DRM_FORMAT_AYUV		fourcc_code('A', 'Y', 'U', 'V') /* [31:0] A:Y:Cb:Cr 8:8:8:8 little endian */
+#define DRM_FORMAT_AVUY8888	fourcc_code('A', 'V', 'U', 'Y') /* [31:0] A:Cr:Cb:Y 8:8:8:8 little endian */
 #define DRM_FORMAT_XYUV8888	fourcc_code('X', 'Y', 'U', 'V') /* [31:0] X:Y:Cb:Cr 8:8:8:8 little endian */
+#define DRM_FORMAT_XVUY8888	fourcc_code('X', 'V', 'U', 'Y') /* [31:0] X:Cr:Cb:Y 8:8:8:8 little endian */
 #define DRM_FORMAT_VUY888	fourcc_code('V', 'U', '2', '4') /* [23:0] Cr:Cb:Y 8:8:8 little endian */
 #define DRM_FORMAT_VUY101010	fourcc_code('V', 'U', '3', '0') /* Y followed by U then V, 10:10:10. Non-linear modifier only */
 
diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h
index 0a0d56a6158e..fa953309d9ce 100644
--- a/include/uapi/drm/drm_mode.h
+++ b/include/uapi/drm/drm_mode.h
@@ -675,11 +675,11 @@ struct drm_mode_fb_cmd {
  *   fetch metadata about an existing frame-buffer.
  *
  * In case of planar formats, this struct allows up to 4 buffer objects with
- * offsets and pitches per plane. The pitch and offset order is dictated by the
- * format FourCC as defined by ``drm_fourcc.h``, e.g. NV12 is described as:
+ * offsets and pitches per plane. The pitch and offset order are dictated by
+ * the format FourCC as defined by ``drm_fourcc.h``, e.g. NV12 is described as:
  *
- *     YUV 4:2:0 image with a plane of 8 bit Y samples followed by an
- *     interleaved U/V plane containing 8 bit 2x2 subsampled colour difference
+ *     YUV 4:2:0 image with a plane of 8-bit Y samples followed by an
+ *     interleaved U/V plane containing 8-bit 2x2 subsampled colour difference
  *     samples.
  *
  * So it would consist of a Y plane at ``offsets[0]`` and a UV plane at
diff --git a/include/uapi/drm/panfrost_drm.h b/include/uapi/drm/panfrost_drm.h
index 9e40277d8185..eac87310b348 100644
--- a/include/uapi/drm/panfrost_drm.h
+++ b/include/uapi/drm/panfrost_drm.h
@@ -224,6 +224,53 @@ struct drm_panfrost_madvise {
 	__u32 retained;       /* out, whether backing store still exists */
 };
 
+/* Definitions for coredump decoding in user space */
+#define PANFROSTDUMP_MAJOR 1
+#define PANFROSTDUMP_MINOR 0
+
+#define PANFROSTDUMP_MAGIC 0x464E4150 /* PANF */
+
+#define PANFROSTDUMP_BUF_REG 0
+#define PANFROSTDUMP_BUF_BOMAP (PANFROSTDUMP_BUF_REG + 1)
+#define PANFROSTDUMP_BUF_BO (PANFROSTDUMP_BUF_BOMAP + 1)
+#define PANFROSTDUMP_BUF_TRAILER (PANFROSTDUMP_BUF_BO + 1)
+
+struct panfrost_dump_object_header {
+	__le32 magic;
+	__le32 type;
+	__le32 file_size;
+	__le32 file_offset;
+
+	union {
+		struct pan_reg_hdr {
+			__le64 jc;
+			__le32 gpu_id;
+			__le32 major;
+			__le32 minor;
+			__le64 nbos;
+		} reghdr;
+
+		struct pan_bomap_hdr {
+			__le32 valid;
+			__le64 iova;
+			__le32 data[2];
+		} bomap;
+
+		/*
+		 * Force same size in case we want to expand the header
+		 * with new fields and also keep it 512-byte aligned
+		 */
+
+		__le32 sizer[496];
+	};
+};
+
+/* Registers object, an array of these */
+struct panfrost_dump_registers {
+	__le32 reg;
+	__le32 value;
+};
+
 #if defined(__cplusplus)
 }
 #endif
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 59a217ca2dfd..51b9aa640ad2 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -87,10 +87,35 @@ struct bpf_cgroup_storage_key {
 	__u32	attach_type;		/* program attach type (enum bpf_attach_type) */
 };
 
+enum bpf_cgroup_iter_order {
+	BPF_CGROUP_ITER_ORDER_UNSPEC = 0,
+	BPF_CGROUP_ITER_SELF_ONLY,		/* process only a single object. */
+	BPF_CGROUP_ITER_DESCENDANTS_PRE,	/* walk descendants in pre-order. */
+	BPF_CGROUP_ITER_DESCENDANTS_POST,	/* walk descendants in post-order. */
+	BPF_CGROUP_ITER_ANCESTORS_UP,		/* walk ancestors upward. */
+};
+
 union bpf_iter_link_info {
 	struct {
 		__u32	map_fd;
 	} map;
+	struct {
+		enum bpf_cgroup_iter_order order;
+
+		/* At most one of cgroup_fd and cgroup_id can be non-zero. If
+		 * both are zero, the walk starts from the default cgroup v2
+		 * root. For walking v1 hierarchy, one should always explicitly
+		 * specify cgroup_fd.
+		 */
+		__u32	cgroup_fd;
+		__u64	cgroup_id;
+	} cgroup;
+	/* Parameters of task iterators. */
+	struct {
+		__u32	tid;
+		__u32	pid;
+		__u32	pid_fd;
+	} task;
 };
 
 /* BPF syscall commands, see bpf(2) man-page for more details. */
@@ -909,6 +934,7 @@ enum bpf_map_type {
 	BPF_MAP_TYPE_INODE_STORAGE,
 	BPF_MAP_TYPE_TASK_STORAGE,
 	BPF_MAP_TYPE_BLOOM_FILTER,
+	BPF_MAP_TYPE_USER_RINGBUF,
 };
 
 /* Note that tracing related programs such as
@@ -1233,7 +1259,7 @@ enum {
 
 /* Query effective (directly attached + inherited from ancestor cgroups)
  * programs that will be executed for events within a cgroup.
- * attach_flags with this flag are returned only for directly attached programs.
+ * attach_flags with this flag are always returned 0.
  */
 #define BPF_F_QUERY_EFFECTIVE	(1U << 0)
 
@@ -1432,7 +1458,10 @@ union bpf_attr {
 		__u32		attach_flags;
 		__aligned_u64	prog_ids;
 		__u32		prog_cnt;
-		__aligned_u64	prog_attach_flags; /* output: per-program attach_flags */
+		/* output: per-program attach_flags.
+		 * not allowed to be set during effective query.
+		 */
+		__aligned_u64	prog_attach_flags;
 	} query;
 
 	struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */
@@ -2573,10 +2602,12 @@ union bpf_attr {
  *		There are two supported modes at this time:
  *
  *		* **BPF_ADJ_ROOM_MAC**: Adjust room at the mac layer
- *		  (room space is added or removed below the layer 2 header).
+ * 		  (room space is added or removed between the layer 2 and
+ * 		  layer 3 headers).
  *
  * 		* **BPF_ADJ_ROOM_NET**: Adjust room at the network layer
- * 		  (room space is added or removed below the layer 3 header).
+ * 		  (room space is added or removed between the layer 3 and
+ * 		  layer 4 headers).
  *
  *		The following flags are supported at this time:
  *
@@ -3008,8 +3039,18 @@ union bpf_attr {
  * 		**BPF_F_USER_STACK**
  * 			Collect a user space stack instead of a kernel stack.
  * 		**BPF_F_USER_BUILD_ID**
- * 			Collect buildid+offset instead of ips for user stack,
- * 			only valid if **BPF_F_USER_STACK** is also specified.
+ * 			Collect (build_id, file_offset) instead of ips for user
+ * 			stack, only valid if **BPF_F_USER_STACK** is also
+ * 			specified.
+ *
+ * 			*file_offset* is an offset relative to the beginning
+ * 			of the executable or shared object file backing the vma
+ * 			which the *ip* falls in. It is *not* an offset relative
+ * 			to that object's base address. Accordingly, it must be
+ * 			adjusted by adding (sh_addr - sh_offset), where
+ * 			sh_{addr,offset} correspond to the executable section
+ * 			containing *file_offset* in the object, for comparisons
+ * 			to symbols' st_value to be valid.
  *
  * 		**bpf_get_stack**\ () can collect up to
  * 		**PERF_MAX_STACK_DEPTH** both kernel and user frames, subject
@@ -4425,7 +4466,7 @@ union bpf_attr {
  *
  *		**-EEXIST** if the option already exists.
  *
- *		**-EFAULT** on failrue to parse the existing header options.
+ *		**-EFAULT** on failure to parse the existing header options.
  *
  *		**-EPERM** if the helper cannot be used under the current
  *		*skops*\ **->op**.
@@ -4634,7 +4675,7 @@ union bpf_attr {
  *		a *map* with *task* as the **key**.  From this
  *		perspective,  the usage is not much different from
  *		**bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this
- *		helper enforces the key must be an task_struct and the map must also
+ *		helper enforces the key must be a task_struct and the map must also
  *		be a **BPF_MAP_TYPE_TASK_STORAGE**.
  *
  *		Underneath, the value is stored locally at *task* instead of
@@ -4692,7 +4733,7 @@ union bpf_attr {
  *
  * long bpf_ima_inode_hash(struct inode *inode, void *dst, u32 size)
  *	Description
- *		Returns the stored IMA hash of the *inode* (if it's avaialable).
+ *		Returns the stored IMA hash of the *inode* (if it's available).
  *		If the hash is larger than *size*, then only *size*
  *		bytes will be copied to *dst*
  *	Return
@@ -4716,12 +4757,12 @@ union bpf_attr {
  *
  *		The argument *len_diff* can be used for querying with a planned
  *		size change. This allows to check MTU prior to changing packet
- *		ctx. Providing an *len_diff* adjustment that is larger than the
+ *		ctx. Providing a *len_diff* adjustment that is larger than the
  *		actual packet size (resulting in negative packet size) will in
- *		principle not exceed the MTU, why it is not considered a
- *		failure.  Other BPF-helpers are needed for performing the
- *		planned size change, why the responsability for catch a negative
- *		packet size belong in those helpers.
+ *		principle not exceed the MTU, which is why it is not considered
+ *		a failure.  Other BPF helpers are needed for performing the
+ *		planned size change; therefore the responsibility for catching
+ *		a negative packet size belongs in those helpers.
  *
  *		Specifying *ifindex* zero means the MTU check is performed
  *		against the current net device.  This is practical if this isn't
@@ -4919,6 +4960,7 @@ union bpf_attr {
  * 		Get address of the traced function (for tracing and kprobe programs).
  * 	Return
  * 		Address of the traced function.
+ * 		0 for kprobes placed within the function (not at the entry).
  *
  * u64 bpf_get_attach_cookie(void *ctx)
  * 	Description
@@ -5048,12 +5090,12 @@ union bpf_attr {
  *
  * long bpf_get_func_arg(void *ctx, u32 n, u64 *value)
  *	Description
- *		Get **n**-th argument (zero based) of the traced function (for tracing programs)
+ *		Get **n**-th argument register (zero based) of the traced function (for tracing programs)
  *		returned in **value**.
  *
  *	Return
  *		0 on success.
- *		**-EINVAL** if n >= arguments count of traced function.
+ *		**-EINVAL** if n >= argument register count of traced function.
  *
  * long bpf_get_func_ret(void *ctx, u64 *value)
  *	Description
@@ -5066,24 +5108,37 @@ union bpf_attr {
  *
  * long bpf_get_func_arg_cnt(void *ctx)
  *	Description
- *		Get number of arguments of the traced function (for tracing programs).
+ *		Get number of registers of the traced function (for tracing programs) where
+ *		function arguments are stored in these registers.
  *
  *	Return
- *		The number of arguments of the traced function.
+ *		The number of argument registers of the traced function.
  *
  * int bpf_get_retval(void)
  *	Description
- *		Get the syscall's return value that will be returned to userspace.
+ *		Get the BPF program's return value that will be returned to the upper layers.
  *
- *		This helper is currently supported by cgroup programs only.
+ *		This helper is currently supported by cgroup programs and only by the hooks
+ *		where BPF program's return value is returned to the userspace via errno.
  *	Return
- *		The syscall's return value.
+ *		The BPF program's return value.
  *
  * int bpf_set_retval(int retval)
  *	Description
- *		Set the syscall's return value that will be returned to userspace.
+ *		Set the BPF program's return value that will be returned to the upper layers.
+ *
+ *		This helper is currently supported by cgroup programs and only by the hooks
+ *		where BPF program's return value is returned to the userspace via errno.
+ *
+ *		Note that there is the following corner case where the program exports an error
+ *		via bpf_set_retval but signals success via 'return 1':
+ *
+ *			bpf_set_retval(-EPERM);
+ *			return 1;
+ *
+ *		In this case, the BPF program's return value will use helper's -EPERM. This
+ *		still holds true for cgroup/bind{4,6} which supports extra 'return 3' success case.
  *
- *		This helper is currently supported by cgroup programs only.
  *	Return
  *		0 on success, or a negative error in case of failure.
  *
@@ -5331,6 +5386,55 @@ union bpf_attr {
  *		**-EACCES** if the SYN cookie is not valid.
  *
  *		**-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin.
+ *
+ * u64 bpf_ktime_get_tai_ns(void)
+ *	Description
+ *		A nonsettable system-wide clock derived from wall-clock time but
+ *		ignoring leap seconds.  This clock does not experience
+ *		discontinuities and backwards jumps caused by NTP inserting leap
+ *		seconds as CLOCK_REALTIME does.
+ *
+ *		See: **clock_gettime**\ (**CLOCK_TAI**)
+ *	Return
+ *		Current *ktime*.
+ *
+ * long bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void *ctx, u64 flags)
+ *	Description
+ *		Drain samples from the specified user ring buffer, and invoke
+ *		the provided callback for each such sample:
+ *
+ *		long (\*callback_fn)(struct bpf_dynptr \*dynptr, void \*ctx);
+ *
+ *		If **callback_fn** returns 0, the helper will continue to try
+ *		and drain the next sample, up to a maximum of
+ *		BPF_MAX_USER_RINGBUF_SAMPLES samples. If the return value is 1,
+ *		the helper will skip the rest of the samples and return. Other
+ *		return values are not used now, and will be rejected by the
+ *		verifier.
+ *	Return
+ *		The number of drained samples if no error was encountered while
+ *		draining samples, or 0 if no samples were present in the ring
+ *		buffer. If a user-space producer was epoll-waiting on this map,
+ *		and at least one sample was drained, they will receive an event
+ *		notification notifying them of available space in the ring
+ *		buffer. If the BPF_RB_NO_WAKEUP flag is passed to this
+ *		function, no wakeup notification will be sent. If the
+ *		BPF_RB_FORCE_WAKEUP flag is passed, a wakeup notification will
+ *		be sent even if no sample was drained.
+ *
+ *		On failure, the returned value is one of the following:
+ *
+ *		**-EBUSY** if the ring buffer is contended, and another calling
+ *		context was concurrently draining the ring buffer.
+ *
+ *		**-EINVAL** if user-space is not properly tracking the ring
+ *		buffer due to the producer position not being aligned to 8
+ *		bytes, a sample not being aligned to 8 bytes, or the producer
+ *		position not matching the advertised length of a sample.
+ *
+ *		**-E2BIG** if user-space has tried to publish a sample which is
+ *		larger than the size of the ring buffer, or which cannot fit
+ *		within a struct bpf_dynptr.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5541,6 +5645,8 @@ union bpf_attr {
 	FN(tcp_raw_gen_syncookie_ipv6),	\
 	FN(tcp_raw_check_syncookie_ipv4),	\
 	FN(tcp_raw_check_syncookie_ipv6),	\
+	FN(ktime_get_tai_ns),		\
+	FN(user_ringbuf_drain),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -5603,6 +5709,11 @@ enum {
 	BPF_F_SEQ_NUMBER		= (1ULL << 3),
 };
 
+/* BPF_FUNC_skb_get_tunnel_key flags. */
+enum {
+	BPF_F_TUNINFO_FLAGS		= (1ULL << 4),
+};
+
 /* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and
  * BPF_FUNC_perf_event_read_value flags.
  */
@@ -5792,7 +5903,10 @@ struct bpf_tunnel_key {
 	};
 	__u8 tunnel_tos;
 	__u8 tunnel_ttl;
-	__u16 tunnel_ext;	/* Padding, future use. */
+	union {
+		__u16 tunnel_ext;	/* compat */
+		__be16 tunnel_flags;
+	};
 	__u32 tunnel_label;
 	union {
 		__u32 local_ipv4;
@@ -5836,6 +5950,11 @@ enum bpf_ret_code {
 	 *    represented by BPF_REDIRECT above).
 	 */
 	BPF_LWT_REROUTE = 128,
+	/* BPF_FLOW_DISSECTOR_CONTINUE: used by BPF_PROG_TYPE_FLOW_DISSECTOR
+	 *   to indicate that no custom dissection was performed, and
+	 *   fallback to standard dissector is requested.
+	 */
+	BPF_FLOW_DISSECTOR_CONTINUE = 129,
 };
 
 struct bpf_sock {
@@ -6134,11 +6253,26 @@ struct bpf_link_info {
 		struct {
 			__aligned_u64 target_name; /* in/out: target_name buffer ptr */
 			__u32 target_name_len;	   /* in/out: target_name buffer len */
+
+			/* If the iter specific field is 32 bits, it can be put
+			 * in the first or second union. Otherwise it should be
+			 * put in the second union.
+			 */
 			union {
 				struct {
 					__u32 map_id;
 				} map;
 			};
+			union {
+				struct {
+					__u64 cgroup_id;
+					__u32 order;
+				} cgroup;
+				struct {
+					__u32 tid;
+					__u32 pid;
+				} task;
+			};
 		} iter;
 		struct  {
 			__u32 netns_ino;
diff --git a/include/uapi/linux/can.h b/include/uapi/linux/can.h
index 90801ada2bbe..dd645ea72306 100644
--- a/include/uapi/linux/can.h
+++ b/include/uapi/linux/can.h
@@ -48,6 +48,7 @@
 
 #include <linux/types.h>
 #include <linux/socket.h>
+#include <linux/stddef.h> /* for offsetof */
 
 /* controller area network (CAN) kernel definitions */
 
@@ -60,6 +61,7 @@
 #define CAN_SFF_MASK 0x000007FFU /* standard frame format (SFF) */
 #define CAN_EFF_MASK 0x1FFFFFFFU /* extended frame format (EFF) */
 #define CAN_ERR_MASK 0x1FFFFFFFU /* omit EFF, RTR, ERR flags */
+#define CANXL_PRIO_MASK CAN_SFF_MASK /* 11 bit priority mask */
 
 /*
  * Controller Area Network Identifier structure
@@ -73,6 +75,7 @@ typedef __u32 canid_t;
 
 #define CAN_SFF_ID_BITS		11
 #define CAN_EFF_ID_BITS		29
+#define CANXL_PRIO_BITS		CAN_SFF_ID_BITS
 
 /*
  * Controller Area Network Error Message Frame Mask structure
@@ -91,6 +94,16 @@ typedef __u32 can_err_mask_t;
 #define CANFD_MAX_DLC 15
 #define CANFD_MAX_DLEN 64
 
+/*
+ * CAN XL payload length and DLC definitions according to ISO 11898-1
+ * CAN XL DLC ranges from 0 .. 2047 => data length from 1 .. 2048 byte
+ */
+#define CANXL_MIN_DLC 0
+#define CANXL_MAX_DLC 2047
+#define CANXL_MAX_DLC_MASK 0x07FF
+#define CANXL_MIN_DLEN 1
+#define CANXL_MAX_DLEN 2048
+
 /**
  * struct can_frame - Classical CAN frame structure (aka CAN 2.0B)
  * @can_id:   CAN ID of the frame and CAN_*_FLAG flags, see canid_t definition
@@ -141,8 +154,8 @@ struct can_frame {
  * When this is done the former differentiation via CAN_MTU / CANFD_MTU gets
  * lost. CANFD_FDF allows programmers to mark CAN FD frames in the case of
  * using struct canfd_frame for mixed CAN / CAN FD content (dual use).
- * N.B. the Kernel APIs do NOT provide mixed CAN / CAN FD content inside of
- * struct canfd_frame therefore the CANFD_FDF flag is disregarded by Linux.
+ * Since the introduction of CAN XL the CANFD_FDF flag is set in all CAN FD
+ * frame structures provided by the CAN subsystem of the Linux kernel.
  */
 #define CANFD_BRS 0x01 /* bit rate switch (second bitrate for payload data) */
 #define CANFD_ESI 0x02 /* error state indicator of the transmitting node */
@@ -166,8 +179,46 @@ struct canfd_frame {
 	__u8    data[CANFD_MAX_DLEN] __attribute__((aligned(8)));
 };
 
+/*
+ * defined bits for canxl_frame.flags
+ *
+ * The canxl_frame.flags element contains two bits CANXL_XLF and CANXL_SEC
+ * and shares the relative position of the struct can[fd]_frame.len element.
+ * The CANXL_XLF bit ALWAYS needs to be set to indicate a valid CAN XL frame.
+ * As a side effect setting this bit intentionally breaks the length checks
+ * for Classical CAN and CAN FD frames.
+ *
+ * Undefined bits in canxl_frame.flags are reserved and shall be set to zero.
+ */
+#define CANXL_XLF 0x80 /* mandatory CAN XL frame flag (must always be set!) */
+#define CANXL_SEC 0x01 /* Simple Extended Content (security/segmentation) */
+
+/**
+ * struct canxl_frame - CAN with e'X'tended frame 'L'ength frame structure
+ * @prio:  11 bit arbitration priority with zero'ed CAN_*_FLAG flags
+ * @flags: additional flags for CAN XL
+ * @sdt:   SDU (service data unit) type
+ * @len:   frame payload length in byte (CANXL_MIN_DLEN .. CANXL_MAX_DLEN)
+ * @af:    acceptance field
+ * @data:  CAN XL frame payload (CANXL_MIN_DLEN .. CANXL_MAX_DLEN byte)
+ *
+ * @prio shares the same position as @can_id from struct can[fd]_frame.
+ */
+struct canxl_frame {
+	canid_t prio;  /* 11 bit priority for arbitration (canid_t) */
+	__u8    flags; /* additional flags for CAN XL */
+	__u8    sdt;   /* SDU (service data unit) type */
+	__u16   len;   /* frame payload length in byte */
+	__u32   af;    /* acceptance field */
+	__u8    data[CANXL_MAX_DLEN];
+};
+
 #define CAN_MTU		(sizeof(struct can_frame))
 #define CANFD_MTU	(sizeof(struct canfd_frame))
+#define CANXL_MTU	(sizeof(struct canxl_frame))
+#define CANXL_HDR_SIZE	(offsetof(struct canxl_frame, data))
+#define CANXL_MIN_MTU	(CANXL_HDR_SIZE + 64)
+#define CANXL_MAX_MTU	CANXL_MTU
 
 /* particular protocols of the protocol family PF_CAN */
 #define CAN_RAW		1 /* RAW sockets */
diff --git a/include/uapi/linux/can/raw.h b/include/uapi/linux/can/raw.h
index 3386aa81fdf2..ff12f525c37c 100644
--- a/include/uapi/linux/can/raw.h
+++ b/include/uapi/linux/can/raw.h
@@ -62,6 +62,7 @@ enum {
 	CAN_RAW_RECV_OWN_MSGS,	/* receive my own msgs (default:off) */
 	CAN_RAW_FD_FRAMES,	/* allow CAN FD frames (default:off) */
 	CAN_RAW_JOIN_FILTERS,	/* all filters must match to trigger */
+	CAN_RAW_XL_FRAMES,	/* allow CAN XL frames (default:off) */
 };
 
 #endif /* !_UAPI_CAN_RAW_H */
diff --git a/include/uapi/linux/dlm.h b/include/uapi/linux/dlm.h
index 0d2eca287567..1923f4f3b05e 100644
--- a/include/uapi/linux/dlm.h
+++ b/include/uapi/linux/dlm.h
@@ -69,7 +69,6 @@ struct dlm_lksb {
 /* dlm_new_lockspace() flags */
 
 #define DLM_LSFL_TIMEWARN	0x00000002
-#define DLM_LSFL_FS     	0x00000004
 #define DLM_LSFL_NEWEXCL     	0x00000008
 
 
diff --git a/include/uapi/linux/dn.h b/include/uapi/linux/dn.h
deleted file mode 100644
index 36ca71bd8bbe..000000000000
--- a/include/uapi/linux/dn.h
+++ /dev/null
@@ -1,149 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef _LINUX_DN_H
-#define _LINUX_DN_H
-
-#include <linux/ioctl.h>
-#include <linux/types.h>
-#include <linux/if_ether.h>
-
-/*
-
-	DECnet Data Structures and Constants
-
-*/
-
-/* 
- * DNPROTO_NSP can't be the same as SOL_SOCKET, 
- * so increment each by one (compared to ULTRIX)
- */
-#define DNPROTO_NSP     2                       /* NSP protocol number       */
-#define DNPROTO_ROU     3                       /* Routing protocol number   */
-#define DNPROTO_NML     4                       /* Net mgt protocol number   */
-#define DNPROTO_EVL     5                       /* Evl protocol number (usr) */
-#define DNPROTO_EVR     6                       /* Evl protocol number (evl) */
-#define DNPROTO_NSPT    7                       /* NSP trace protocol number */
-
-
-#define DN_ADDL		2
-#define DN_MAXADDL	2 /* ULTRIX headers have 20 here, but pathworks has 2 */
-#define DN_MAXOPTL	16
-#define DN_MAXOBJL	16
-#define DN_MAXACCL	40
-#define DN_MAXALIASL	128
-#define DN_MAXNODEL	256
-#define DNBUFSIZE	65023
-
-/* 
- * SET/GET Socket options  - must match the DSO_ numbers below
- */
-#define SO_CONDATA      1
-#define SO_CONACCESS    2
-#define SO_PROXYUSR     3
-#define SO_LINKINFO     7
-
-#define DSO_CONDATA     1        /* Set/Get connect data                */
-#define DSO_DISDATA     10       /* Set/Get disconnect data             */
-#define DSO_CONACCESS   2        /* Set/Get connect access data         */
-#define DSO_ACCEPTMODE  4        /* Set/Get accept mode                 */
-#define DSO_CONACCEPT   5        /* Accept deferred connection          */
-#define DSO_CONREJECT   6        /* Reject deferred connection          */
-#define DSO_LINKINFO    7        /* Set/Get link information            */
-#define DSO_STREAM      8        /* Set socket type to stream           */
-#define DSO_SEQPACKET   9        /* Set socket type to sequenced packet */
-#define DSO_MAXWINDOW   11       /* Maximum window size allowed         */
-#define DSO_NODELAY	12       /* Turn off nagle                      */
-#define DSO_CORK        13       /* Wait for more data!                 */
-#define DSO_SERVICES	14       /* NSP Services field                  */
-#define DSO_INFO	15       /* NSP Info field                      */
-#define DSO_MAX         15       /* Maximum option number               */
-
-
-/* LINK States */
-#define LL_INACTIVE	0
-#define LL_CONNECTING	1
-#define LL_RUNNING	2
-#define LL_DISCONNECTING 3
-
-#define ACC_IMMED 0
-#define ACC_DEFER 1
-
-#define SDF_WILD        1                  /* Wild card object          */
-#define SDF_PROXY       2                  /* Addr eligible for proxy   */
-#define SDF_UICPROXY    4                  /* Use uic-based proxy       */
-
-/* Structures */
-
-
-struct dn_naddr {
-	__le16		a_len;
-	__u8 a_addr[DN_MAXADDL]; /* Two bytes little endian */
-};
-
-struct sockaddr_dn {
-	__u16		sdn_family;
-	__u8		sdn_flags;
-	__u8		sdn_objnum;
-	__le16		sdn_objnamel;
-	__u8		sdn_objname[DN_MAXOBJL];
-	struct   dn_naddr	sdn_add;
-};
-#define sdn_nodeaddrl   sdn_add.a_len   /* Node address length  */
-#define sdn_nodeaddr    sdn_add.a_addr  /* Node address         */
-
-
-
-/*
- * DECnet set/get DSO_CONDATA, DSO_DISDATA (optional data) structure
- */
-struct optdata_dn {
-        __le16  opt_status;     /* Extended status return */
-#define opt_sts opt_status
-        __le16  opt_optl;       /* Length of user data    */
-        __u8   opt_data[16];   /* User data              */
-};
-
-struct accessdata_dn {
-	__u8		acc_accl;
-	__u8		acc_acc[DN_MAXACCL];
-	__u8 		acc_passl;
-	__u8		acc_pass[DN_MAXACCL];
-	__u8 		acc_userl;
-	__u8		acc_user[DN_MAXACCL];
-};
-
-/*
- * DECnet logical link information structure
- */
-struct linkinfo_dn {
-        __u16  idn_segsize;    /* Segment size for link */
-        __u8   idn_linkstate;  /* Logical link state    */
-};
-
-/*
- * Ethernet address format (for DECnet)
- */
-union etheraddress {
-        __u8 dne_addr[ETH_ALEN];      /* Full ethernet address */
-  struct {
-                __u8 dne_hiord[4];    /* DECnet HIORD prefix   */
-                __u8 dne_nodeaddr[2]; /* DECnet node address   */
-  } dne_remote;
-};
-
-
-/*
- * DECnet physical socket address format
- */
-struct dn_addr {
-        __le16 dna_family;      /* AF_DECnet               */
-        union etheraddress dna_netaddr; /* DECnet ethernet address */
-};
-
-#define DECNET_IOCTL_BASE 0x89 /* PROTOPRIVATE range */
-
-#define SIOCSNETADDR  _IOW(DECNET_IOCTL_BASE, 0xe0, struct dn_naddr)
-#define SIOCGNETADDR  _IOR(DECNET_IOCTL_BASE, 0xe1, struct dn_naddr)
-#define OSIOCSNETADDR _IOW(DECNET_IOCTL_BASE, 0xe0, int)
-#define OSIOCGNETADDR _IOR(DECNET_IOCTL_BASE, 0xe1, int)
-
-#endif /* _LINUX_DN_H */
diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h
index 2d5741fd44bb..dc2aa3d75b39 100644
--- a/include/uapi/linux/ethtool.h
+++ b/include/uapi/linux/ethtool.h
@@ -737,6 +737,51 @@ enum ethtool_module_power_mode {
 };
 
 /**
+ * enum ethtool_podl_pse_admin_state - operational state of the PoDL PSE
+ *	functions. IEEE 802.3-2018 30.15.1.1.2 aPoDLPSEAdminState
+ * @ETHTOOL_PODL_PSE_ADMIN_STATE_UNKNOWN: state of PoDL PSE functions are
+ * 	unknown
+ * @ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED: PoDL PSE functions are disabled
+ * @ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED: PoDL PSE functions are enabled
+ */
+enum ethtool_podl_pse_admin_state {
+	ETHTOOL_PODL_PSE_ADMIN_STATE_UNKNOWN = 1,
+	ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED,
+	ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED,
+};
+
+/**
+ * enum ethtool_podl_pse_pw_d_status - power detection status of the PoDL PSE.
+ *	IEEE 802.3-2018 30.15.1.1.3 aPoDLPSEPowerDetectionStatus:
+ * @ETHTOOL_PODL_PSE_PW_D_STATUS_UNKNOWN: PoDL PSE
+ * @ETHTOOL_PODL_PSE_PW_D_STATUS_DISABLED: "The enumeration “disabled� is
+ *	asserted true when the PoDL PSE state diagram variable mr_pse_enable is
+ *	false"
+ * @ETHTOOL_PODL_PSE_PW_D_STATUS_SEARCHING: "The enumeration “searching� is
+ *	asserted true when either of the PSE state diagram variables
+ *	pi_detecting or pi_classifying is true."
+ * @ETHTOOL_PODL_PSE_PW_D_STATUS_DELIVERING: "The enumeration “deliveringPower�
+ *	is asserted true when the PoDL PSE state diagram variable pi_powered is
+ *	true."
+ * @ETHTOOL_PODL_PSE_PW_D_STATUS_SLEEP: "The enumeration “sleep� is asserted
+ *	true when the PoDL PSE state diagram variable pi_sleeping is true."
+ * @ETHTOOL_PODL_PSE_PW_D_STATUS_IDLE: "The enumeration “idle� is asserted true
+ *	when the logical combination of the PoDL PSE state diagram variables
+ *	pi_prebiased*!pi_sleeping is true."
+ * @ETHTOOL_PODL_PSE_PW_D_STATUS_ERROR: "The enumeration “error� is asserted
+ *	true when the PoDL PSE state diagram variable overload_held is true."
+ */
+enum ethtool_podl_pse_pw_d_status {
+	ETHTOOL_PODL_PSE_PW_D_STATUS_UNKNOWN = 1,
+	ETHTOOL_PODL_PSE_PW_D_STATUS_DISABLED,
+	ETHTOOL_PODL_PSE_PW_D_STATUS_SEARCHING,
+	ETHTOOL_PODL_PSE_PW_D_STATUS_DELIVERING,
+	ETHTOOL_PODL_PSE_PW_D_STATUS_SLEEP,
+	ETHTOOL_PODL_PSE_PW_D_STATUS_IDLE,
+	ETHTOOL_PODL_PSE_PW_D_STATUS_ERROR,
+};
+
+/**
  * struct ethtool_gstrings - string set for data tagging
  * @cmd: Command number = %ETHTOOL_GSTRINGS
  * @string_set: String set ID; one of &enum ethtool_stringset
@@ -1840,6 +1885,20 @@ static inline int ethtool_validate_duplex(__u8 duplex)
 #define MASTER_SLAVE_STATE_SLAVE		3
 #define MASTER_SLAVE_STATE_ERR			4
 
+/* These are used to throttle the rate of data on the phy interface when the
+ * native speed of the interface is higher than the link speed. These should
+ * not be used for phy interfaces which natively support multiple speeds (e.g.
+ * MII or SGMII).
+ */
+/* No rate matching performed. */
+#define RATE_MATCH_NONE		0
+/* The phy sends pause frames to throttle the MAC. */
+#define RATE_MATCH_PAUSE	1
+/* The phy asserts CRS to prevent the MAC from transmitting. */
+#define RATE_MATCH_CRS		2
+/* The MAC is programmed with a sufficiently-large IPG. */
+#define RATE_MATCH_OPEN_LOOP	3
+
 /* Which connector port. */
 #define PORT_TP			0x00
 #define PORT_AUI		0x01
@@ -2033,8 +2092,8 @@ enum ethtool_reset_flags {
  *	reported consistently by PHYLIB.  Read-only.
  * @master_slave_cfg: Master/slave port mode.
  * @master_slave_state: Master/slave port state.
+ * @rate_matching: Rate adaptation performed by the PHY
  * @reserved: Reserved for future use; see the note on reserved space.
- * @reserved1: Reserved for future use; see the note on reserved space.
  * @link_mode_masks: Variable length bitmaps.
  *
  * If autonegotiation is disabled, the speed and @duplex represent the
@@ -2085,7 +2144,7 @@ struct ethtool_link_settings {
 	__u8	transceiver;
 	__u8	master_slave_cfg;
 	__u8	master_slave_state;
-	__u8	reserved1[1];
+	__u8	rate_matching;
 	__u32	reserved[7];
 	__u32	link_mode_masks[];
 	/* layout of link_mode_masks fields:
diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h
index d2fb4f7be61b..bb57084ac524 100644
--- a/include/uapi/linux/ethtool_netlink.h
+++ b/include/uapi/linux/ethtool_netlink.h
@@ -49,6 +49,8 @@ enum {
 	ETHTOOL_MSG_PHC_VCLOCKS_GET,
 	ETHTOOL_MSG_MODULE_GET,
 	ETHTOOL_MSG_MODULE_SET,
+	ETHTOOL_MSG_PSE_GET,
+	ETHTOOL_MSG_PSE_SET,
 
 	/* add new constants above here */
 	__ETHTOOL_MSG_USER_CNT,
@@ -94,6 +96,7 @@ enum {
 	ETHTOOL_MSG_PHC_VCLOCKS_GET_REPLY,
 	ETHTOOL_MSG_MODULE_GET_REPLY,
 	ETHTOOL_MSG_MODULE_NTF,
+	ETHTOOL_MSG_PSE_GET_REPLY,
 
 	/* add new constants above here */
 	__ETHTOOL_MSG_KERNEL_CNT,
@@ -242,6 +245,7 @@ enum {
 	ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG,	/* u8 */
 	ETHTOOL_A_LINKMODES_MASTER_SLAVE_STATE,	/* u8 */
 	ETHTOOL_A_LINKMODES_LANES,		/* u32 */
+	ETHTOOL_A_LINKMODES_RATE_MATCHING,	/* u8 */
 
 	/* add new constants above here */
 	__ETHTOOL_A_LINKMODES_CNT,
@@ -862,6 +866,19 @@ enum {
 	ETHTOOL_A_MODULE_MAX = (__ETHTOOL_A_MODULE_CNT - 1)
 };
 
+/* Power Sourcing Equipment */
+enum {
+	ETHTOOL_A_PSE_UNSPEC,
+	ETHTOOL_A_PSE_HEADER,			/* nest - _A_HEADER_* */
+	ETHTOOL_A_PODL_PSE_ADMIN_STATE,		/* u32 */
+	ETHTOOL_A_PODL_PSE_ADMIN_CONTROL,	/* u32 */
+	ETHTOOL_A_PODL_PSE_PW_D_STATUS,		/* u32 */
+
+	/* add new constants above here */
+	__ETHTOOL_A_PSE_CNT,
+	ETHTOOL_A_PSE_MAX = (__ETHTOOL_A_PSE_CNT - 1)
+};
+
 /* generic netlink info */
 #define ETHTOOL_GENL_NAME "ethtool"
 #define ETHTOOL_GENL_VERSION 1
diff --git a/include/uapi/linux/if_ether.h b/include/uapi/linux/if_ether.h
index d370165bc621..69e0457eb200 100644
--- a/include/uapi/linux/if_ether.h
+++ b/include/uapi/linux/if_ether.h
@@ -138,6 +138,7 @@
 #define ETH_P_LOCALTALK 0x0009		/* Localtalk pseudo type 	*/
 #define ETH_P_CAN	0x000C		/* CAN: Controller Area Network */
 #define ETH_P_CANFD	0x000D		/* CANFD: CAN flexible data rate*/
+#define ETH_P_CANXL	0x000E		/* CANXL: eXtended frame Length */
 #define ETH_P_PPPTALK	0x0010		/* Dummy type for Atalk over PPP*/
 #define ETH_P_TR_802_2	0x0011		/* 802.2 frames 		*/
 #define ETH_P_MOBITEX	0x0015		/* Mobitex (kaz@cafe.net)	*/
diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index e36d9d2c65a7..5e7a1041df3a 100644
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -370,6 +370,7 @@ enum {
 	IFLA_GRO_MAX_SIZE,
 	IFLA_TSO_MAX_SIZE,
 	IFLA_TSO_MAX_SEGS,
+	IFLA_ALLMULTI,		/* Allmulti count: > 0 means acts ALLMULTI */
 
 	__IFLA_MAX
 };
@@ -694,6 +695,7 @@ enum {
 	IFLA_XFRM_UNSPEC,
 	IFLA_XFRM_LINK,
 	IFLA_XFRM_IF_ID,
+	IFLA_XFRM_COLLECT_METADATA,
 	__IFLA_XFRM_MAX
 };
 
@@ -1374,4 +1376,14 @@ enum {
 
 #define IFLA_MCTP_MAX (__IFLA_MCTP_MAX - 1)
 
+/* DSA section */
+
+enum {
+	IFLA_DSA_UNSPEC,
+	IFLA_DSA_MASTER,
+	__IFLA_DSA_MAX,
+};
+
+#define IFLA_DSA_MAX	(__IFLA_DSA_MAX - 1)
+
 #endif /* _UAPI_LINUX_IF_LINK_H */
diff --git a/include/uapi/linux/if_macsec.h b/include/uapi/linux/if_macsec.h
index 3af2aa069a36..d5b6d1f37353 100644
--- a/include/uapi/linux/if_macsec.h
+++ b/include/uapi/linux/if_macsec.h
@@ -22,6 +22,8 @@
 
 #define MACSEC_KEYID_LEN 16
 
+#define MACSEC_SALT_LEN 12
+
 /* cipher IDs as per IEEE802.1AE-2018 (Table 14-1) */
 #define MACSEC_CIPHER_ID_GCM_AES_128 0x0080C20001000001ULL
 #define MACSEC_CIPHER_ID_GCM_AES_256 0x0080C20001000002ULL
diff --git a/include/uapi/linux/if_tun.h b/include/uapi/linux/if_tun.h
index 2ec07de1d73b..b6d7b868f290 100644
--- a/include/uapi/linux/if_tun.h
+++ b/include/uapi/linux/if_tun.h
@@ -67,6 +67,8 @@
 #define IFF_TAP		0x0002
 #define IFF_NAPI	0x0010
 #define IFF_NAPI_FRAGS	0x0020
+/* Used in TUNSETIFF to bring up tun/tap without carrier */
+#define IFF_NO_CARRIER	0x0040
 #define IFF_NO_PI	0x1000
 /* This flag has no real effect */
 #define IFF_ONE_QUEUE	0x2000
diff --git a/include/uapi/linux/in.h b/include/uapi/linux/in.h
index 14168225cecd..f243ce665f74 100644
--- a/include/uapi/linux/in.h
+++ b/include/uapi/linux/in.h
@@ -68,6 +68,8 @@ enum {
 #define IPPROTO_PIM		IPPROTO_PIM
   IPPROTO_COMP = 108,		/* Compression Header Protocol		*/
 #define IPPROTO_COMP		IPPROTO_COMP
+  IPPROTO_L2TP = 115,		/* Layer 2 Tunnelling Protocol		*/
+#define IPPROTO_L2TP		IPPROTO_L2TP
   IPPROTO_SCTP = 132,		/* Stream Control Transport Protocol	*/
 #define IPPROTO_SCTP		IPPROTO_SCTP
   IPPROTO_UDPLITE = 136,	/* UDP-Lite (RFC 3828)			*/
@@ -188,21 +190,13 @@ struct ip_mreq_source {
 };
 
 struct ip_msfilter {
+	__be32		imsf_multiaddr;
+	__be32		imsf_interface;
+	__u32		imsf_fmode;
+	__u32		imsf_numsrc;
 	union {
-		struct {
-			__be32		imsf_multiaddr_aux;
-			__be32		imsf_interface_aux;
-			__u32		imsf_fmode_aux;
-			__u32		imsf_numsrc_aux;
-			__be32		imsf_slist[1];
-		};
-		struct {
-			__be32		imsf_multiaddr;
-			__be32		imsf_interface;
-			__u32		imsf_fmode;
-			__u32		imsf_numsrc;
-			__be32		imsf_slist_flex[];
-		};
+		__be32		imsf_slist[1];
+		__DECLARE_FLEX_ARRAY(__be32, imsf_slist_flex);
 	};
 };
 
diff --git a/include/uapi/linux/l2tp.h b/include/uapi/linux/l2tp.h
index bab8c9708611..7d81c3e1ec29 100644
--- a/include/uapi/linux/l2tp.h
+++ b/include/uapi/linux/l2tp.h
@@ -13,8 +13,6 @@
 #include <linux/in.h>
 #include <linux/in6.h>
 
-#define IPPROTO_L2TP		115
-
 /**
  * struct sockaddr_l2tpip - the sockaddr structure for L2TP-over-IP sockets
  * @l2tp_family:  address family number AF_L2TPIP.
diff --git a/include/uapi/linux/landlock.h b/include/uapi/linux/landlock.h
index 23df4e0e8ace..9c4bcc37a455 100644
--- a/include/uapi/linux/landlock.h
+++ b/include/uapi/linux/landlock.h
@@ -26,7 +26,7 @@ struct landlock_ruleset_attr {
 	 * Landlock filesystem access rights that are not part of
 	 * handled_access_fs are allowed.  This is needed for backward
 	 * compatibility reasons.  One exception is the
-	 * LANDLOCK_ACCESS_FS_REFER access right, which is always implicitly
+	 * %LANDLOCK_ACCESS_FS_REFER access right, which is always implicitly
 	 * handled, but must still be explicitly handled to add new rules with
 	 * this access right.
 	 */
@@ -128,11 +128,11 @@ struct landlock_path_beneath_attr {
  *   hierarchy must also always have the same or a superset of restrictions of
  *   the source hierarchy.  If it is not the case, or if the domain doesn't
  *   handle this access right, such actions are denied by default with errno
- *   set to EXDEV.  Linking also requires a LANDLOCK_ACCESS_FS_MAKE_* access
- *   right on the destination directory, and renaming also requires a
- *   LANDLOCK_ACCESS_FS_REMOVE_* access right on the source's (file or
+ *   set to ``EXDEV``.  Linking also requires a ``LANDLOCK_ACCESS_FS_MAKE_*``
+ *   access right on the destination directory, and renaming also requires a
+ *   ``LANDLOCK_ACCESS_FS_REMOVE_*`` access right on the source's (file or
  *   directory) parent.  Otherwise, such actions are denied with errno set to
- *   EACCES.  The EACCES errno prevails over EXDEV to let user space
+ *   ``EACCES``.  The ``EACCES`` errno prevails over ``EXDEV`` to let user space
  *   efficiently deal with an unrecoverable error.
  *
  * .. warning::
diff --git a/include/uapi/linux/lwtunnel.h b/include/uapi/linux/lwtunnel.h
index 2e206919125c..229655ef792f 100644
--- a/include/uapi/linux/lwtunnel.h
+++ b/include/uapi/linux/lwtunnel.h
@@ -15,6 +15,7 @@ enum lwtunnel_encap_types {
 	LWTUNNEL_ENCAP_SEG6_LOCAL,
 	LWTUNNEL_ENCAP_RPL,
 	LWTUNNEL_ENCAP_IOAM6,
+	LWTUNNEL_ENCAP_XFRM,
 	__LWTUNNEL_ENCAP_MAX,
 };
 
@@ -111,4 +112,13 @@ enum {
 
 #define LWT_BPF_MAX_HEADROOM 256
 
+enum {
+	LWT_XFRM_UNSPEC,
+	LWT_XFRM_IF_ID,
+	LWT_XFRM_LINK,
+	__LWT_XFRM_MAX,
+};
+
+#define LWT_XFRM_MAX (__LWT_XFRM_MAX - 1)
+
 #endif /* _UAPI_LWTUNNEL_H_ */
diff --git a/include/uapi/linux/netfilter.h b/include/uapi/linux/netfilter.h
index 53411ccc69db..5a79ccb76701 100644
--- a/include/uapi/linux/netfilter.h
+++ b/include/uapi/linux/netfilter.h
@@ -63,7 +63,9 @@ enum {
 	NFPROTO_NETDEV =  5,
 	NFPROTO_BRIDGE =  7,
 	NFPROTO_IPV6   = 10,
+#ifndef __KERNEL__ /* no longer supported by kernel */
 	NFPROTO_DECNET = 12,
+#endif
 	NFPROTO_NUMPROTO,
 };
 
diff --git a/include/uapi/linux/netfilter/ipset/ip_set.h b/include/uapi/linux/netfilter/ipset/ip_set.h
index 6397d75899bc..79e5d68b87af 100644
--- a/include/uapi/linux/netfilter/ipset/ip_set.h
+++ b/include/uapi/linux/netfilter/ipset/ip_set.h
@@ -3,10 +3,6 @@
  *                         Patrick Schaaf <bof@bof.de>
  *                         Martin Josefsson <gandalf@wlug.westbo.se>
  * Copyright (C) 2003-2011 Jozsef Kadlecsik <kadlec@netfilter.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 #ifndef _UAPI_IP_SET_H
 #define _UAPI_IP_SET_H
diff --git a/include/uapi/linux/netfilter/xt_AUDIT.h b/include/uapi/linux/netfilter/xt_AUDIT.h
index 1b314e2f84ac..56a3f6092e0c 100644
--- a/include/uapi/linux/netfilter/xt_AUDIT.h
+++ b/include/uapi/linux/netfilter/xt_AUDIT.h
@@ -4,10 +4,6 @@
  *
  * (C) 2010-2011 Thomas Graf <tgraf@redhat.com>
  * (C) 2010-2011 Red Hat, Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
  */
 
 #ifndef _XT_AUDIT_TARGET_H
diff --git a/include/uapi/linux/netfilter/xt_connmark.h b/include/uapi/linux/netfilter/xt_connmark.h
index f01c19b83a2b..41b578ccd03b 100644
--- a/include/uapi/linux/netfilter/xt_connmark.h
+++ b/include/uapi/linux/netfilter/xt_connmark.h
@@ -1,18 +1,13 @@
 /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
+/* Copyright (C) 2002,2004 MARA Systems AB <https://www.marasystems.com>
+ * by Henrik Nordstrom <hno@marasystems.com>
+ */
+
 #ifndef _XT_CONNMARK_H
 #define _XT_CONNMARK_H
 
 #include <linux/types.h>
 
-/* Copyright (C) 2002,2004 MARA Systems AB <https://www.marasystems.com>
- * by Henrik Nordstrom <hno@marasystems.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
 enum {
 	XT_CONNMARK_SET = 0,
 	XT_CONNMARK_SAVE,
diff --git a/include/uapi/linux/netfilter/xt_osf.h b/include/uapi/linux/netfilter/xt_osf.h
index 6e466236ca4b..f1f097896bdf 100644
--- a/include/uapi/linux/netfilter/xt_osf.h
+++ b/include/uapi/linux/netfilter/xt_osf.h
@@ -1,20 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
 /*
  * Copyright (c) 2003+ Evgeniy Polyakov <johnpol@2ka.mxt.ru>
- *
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef _XT_OSF_H
diff --git a/include/uapi/linux/netfilter_decnet.h b/include/uapi/linux/netfilter_decnet.h
deleted file mode 100644
index 3c77f54560f2..000000000000
--- a/include/uapi/linux/netfilter_decnet.h
+++ /dev/null
@@ -1,72 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
-#ifndef __LINUX_DECNET_NETFILTER_H
-#define __LINUX_DECNET_NETFILTER_H
-
-/* DECnet-specific defines for netfilter. 
- * This file (C) Steve Whitehouse 1999 derived from the
- * ipv4 netfilter header file which is
- * (C)1998 Rusty Russell -- This code is GPL.
- */
-
-#include <linux/netfilter.h>
-
-/* only for userspace compatibility */
-#ifndef __KERNEL__
-
-#include <limits.h> /* for INT_MIN, INT_MAX */
-
-/* kernel define is in netfilter_defs.h */
-#define NF_DN_NUMHOOKS		7
-#endif /* ! __KERNEL__ */
-
-/* DECnet Hooks */
-/* After promisc drops, checksum checks. */
-#define NF_DN_PRE_ROUTING	0
-/* If the packet is destined for this box. */
-#define NF_DN_LOCAL_IN		1
-/* If the packet is destined for another interface. */
-#define NF_DN_FORWARD		2
-/* Packets coming from a local process. */
-#define NF_DN_LOCAL_OUT		3
-/* Packets about to hit the wire. */
-#define NF_DN_POST_ROUTING	4
-/* Input Hello Packets */
-#define NF_DN_HELLO		5
-/* Input Routing Packets */
-#define NF_DN_ROUTE		6
-
-enum nf_dn_hook_priorities {
-	NF_DN_PRI_FIRST = INT_MIN,
-	NF_DN_PRI_CONNTRACK = -200,
-	NF_DN_PRI_MANGLE = -150,
-	NF_DN_PRI_NAT_DST = -100,
-	NF_DN_PRI_FILTER = 0,
-	NF_DN_PRI_NAT_SRC = 100,
-	NF_DN_PRI_DNRTMSG = 200,
-	NF_DN_PRI_LAST = INT_MAX,
-};
-
-struct nf_dn_rtmsg {
-	int nfdn_ifindex;
-};
-
-#define NFDN_RTMSG(r) ((unsigned char *)(r) + NLMSG_ALIGN(sizeof(struct nf_dn_rtmsg)))
-
-#ifndef __KERNEL__
-/* backwards compatibility for userspace */
-#define DNRMG_L1_GROUP 0x01
-#define DNRMG_L2_GROUP 0x02
-#endif
-
-enum {
-	DNRNG_NLGRP_NONE,
-#define DNRNG_NLGRP_NONE	DNRNG_NLGRP_NONE
-	DNRNG_NLGRP_L1,
-#define DNRNG_NLGRP_L1		DNRNG_NLGRP_L1
-	DNRNG_NLGRP_L2,
-#define DNRNG_NLGRP_L2		DNRNG_NLGRP_L2
-	__DNRNG_NLGRP_MAX
-};
-#define DNRNG_NLGRP_MAX	(__DNRNG_NLGRP_MAX - 1)
-
-#endif /*__LINUX_DECNET_NETFILTER_H*/
diff --git a/include/uapi/linux/netlink.h b/include/uapi/linux/netlink.h
index 855dffb4c1c3..e2ae82e3f9f7 100644
--- a/include/uapi/linux/netlink.h
+++ b/include/uapi/linux/netlink.h
@@ -20,7 +20,7 @@
 #define NETLINK_CONNECTOR	11
 #define NETLINK_NETFILTER	12	/* netfilter subsystem */
 #define NETLINK_IP6_FW		13
-#define NETLINK_DNRTMSG		14	/* DECnet routing messages */
+#define NETLINK_DNRTMSG		14	/* DECnet routing messages (obsolete) */
 #define NETLINK_KOBJECT_UEVENT	15	/* Kernel messages to userspace */
 #define NETLINK_GENERIC		16
 /* leave room for NETLINK_DM (DM Events) */
@@ -41,12 +41,20 @@ struct sockaddr_nl {
        	__u32		nl_groups;	/* multicast groups mask */
 };
 
+/**
+ * struct nlmsghdr - fixed format metadata header of Netlink messages
+ * @nlmsg_len:   Length of message including header
+ * @nlmsg_type:  Message content type
+ * @nlmsg_flags: Additional flags
+ * @nlmsg_seq:   Sequence number
+ * @nlmsg_pid:   Sending process port ID
+ */
 struct nlmsghdr {
-	__u32		nlmsg_len;	/* Length of message including header */
-	__u16		nlmsg_type;	/* Message content */
-	__u16		nlmsg_flags;	/* Additional flags */
-	__u32		nlmsg_seq;	/* Sequence number */
-	__u32		nlmsg_pid;	/* Sending process port ID */
+	__u32		nlmsg_len;
+	__u16		nlmsg_type;
+	__u16		nlmsg_flags;
+	__u32		nlmsg_seq;
+	__u32		nlmsg_pid;
 };
 
 /* Flags values */
@@ -54,7 +62,7 @@ struct nlmsghdr {
 #define NLM_F_REQUEST		0x01	/* It is request message. 	*/
 #define NLM_F_MULTI		0x02	/* Multipart message, terminated by NLMSG_DONE */
 #define NLM_F_ACK		0x04	/* Reply with ack, with zero or error code */
-#define NLM_F_ECHO		0x08	/* Echo this request 		*/
+#define NLM_F_ECHO		0x08	/* Receive resulting notifications */
 #define NLM_F_DUMP_INTR		0x10	/* Dump was inconsistent due to sequence change */
 #define NLM_F_DUMP_FILTERED	0x20	/* Dump was filtered as requested */
 
@@ -132,6 +140,10 @@ struct nlmsgerr {
  *	be used - in the success case - to identify a created
  *	object or operation or similar (binary)
  * @NLMSGERR_ATTR_POLICY: policy for a rejected attribute
+ * @NLMSGERR_ATTR_MISS_TYPE: type of a missing required attribute,
+ *	%NLMSGERR_ATTR_MISS_NEST will not be present if the attribute was
+ *	missing at the message level
+ * @NLMSGERR_ATTR_MISS_NEST: offset of the nest where attribute was missing
  * @__NLMSGERR_ATTR_MAX: number of attributes
  * @NLMSGERR_ATTR_MAX: highest attribute number
  */
@@ -141,6 +153,8 @@ enum nlmsgerr_attrs {
 	NLMSGERR_ATTR_OFFS,
 	NLMSGERR_ATTR_COOKIE,
 	NLMSGERR_ATTR_POLICY,
+	NLMSGERR_ATTR_MISS_TYPE,
+	NLMSGERR_ATTR_MISS_NEST,
 
 	__NLMSGERR_ATTR_MAX,
 	NLMSGERR_ATTR_MAX = __NLMSGERR_ATTR_MAX - 1
@@ -337,6 +351,9 @@ enum netlink_attribute_type {
  *	bitfield32 type (U32)
  * @NL_POLICY_TYPE_ATTR_MASK: mask of valid bits for unsigned integers (U64)
  * @NL_POLICY_TYPE_ATTR_PAD: pad attribute for 64-bit alignment
+ *
+ * @__NL_POLICY_TYPE_ATTR_MAX: number of attributes
+ * @NL_POLICY_TYPE_ATTR_MAX: highest attribute number
  */
 enum netlink_policy_type_attr {
 	NL_POLICY_TYPE_ATTR_UNSPEC,
diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h
index ffb7c573e299..c32e7616a366 100644
--- a/include/uapi/linux/nl80211.h
+++ b/include/uapi/linux/nl80211.h
@@ -377,14 +377,22 @@
  *	the non-transmitting interfaces are deleted as well.
  *
  * @NL80211_CMD_GET_KEY: Get sequence counter information for a key specified
- *	by %NL80211_ATTR_KEY_IDX and/or %NL80211_ATTR_MAC.
+ *	by %NL80211_ATTR_KEY_IDX and/or %NL80211_ATTR_MAC. %NL80211_ATTR_MAC
+ *	represents peer's MLD address for MLO pairwise key. For MLO group key,
+ *	the link is identified by %NL80211_ATTR_MLO_LINK_ID.
  * @NL80211_CMD_SET_KEY: Set key attributes %NL80211_ATTR_KEY_DEFAULT,
  *	%NL80211_ATTR_KEY_DEFAULT_MGMT, or %NL80211_ATTR_KEY_THRESHOLD.
+ *	For MLO connection, the link to set default key is identified by
+ *	%NL80211_ATTR_MLO_LINK_ID.
  * @NL80211_CMD_NEW_KEY: add a key with given %NL80211_ATTR_KEY_DATA,
  *	%NL80211_ATTR_KEY_IDX, %NL80211_ATTR_MAC, %NL80211_ATTR_KEY_CIPHER,
- *	and %NL80211_ATTR_KEY_SEQ attributes.
+ *	and %NL80211_ATTR_KEY_SEQ attributes. %NL80211_ATTR_MAC represents
+ *	peer's MLD address for MLO pairwise key. The link to add MLO
+ *	group key is identified by %NL80211_ATTR_MLO_LINK_ID.
  * @NL80211_CMD_DEL_KEY: delete a key identified by %NL80211_ATTR_KEY_IDX
- *	or %NL80211_ATTR_MAC.
+ *	or %NL80211_ATTR_MAC. %NL80211_ATTR_MAC represents peer's MLD address
+ *	for MLO pairwise key. The link to delete group key is identified by
+ *	%NL80211_ATTR_MLO_LINK_ID.
  *
  * @NL80211_CMD_GET_BEACON: (not used)
  * @NL80211_CMD_SET_BEACON: change the beacon on an access point interface
@@ -4951,6 +4959,7 @@ enum nl80211_bss_scan_width {
  *	using the nesting index as the antenna number.
  * @NL80211_BSS_FREQUENCY_OFFSET: frequency offset in KHz
  * @NL80211_BSS_MLO_LINK_ID: MLO link ID of the BSS (u8).
+ * @NL80211_BSS_MLD_ADDR: MLD address of this BSS if connected to it.
  * @__NL80211_BSS_AFTER_LAST: internal
  * @NL80211_BSS_MAX: highest BSS attribute
  */
@@ -4977,6 +4986,7 @@ enum nl80211_bss {
 	NL80211_BSS_CHAIN_SIGNAL,
 	NL80211_BSS_FREQUENCY_OFFSET,
 	NL80211_BSS_MLO_LINK_ID,
+	NL80211_BSS_MLD_ADDR,
 
 	/* keep last */
 	__NL80211_BSS_AFTER_LAST,
@@ -6273,6 +6283,14 @@ enum nl80211_feature_flags {
  * @NL80211_EXT_FEATURE_RADAR_BACKGROUND: Device supports background radar/CAC
  *	detection.
  *
+ * @NL80211_EXT_FEATURE_POWERED_ADDR_CHANGE: Device can perform a MAC address
+ *	change without having to bring the underlying network device down
+ *	first. For example, in station mode this can be used to vary the
+ *	origin MAC address prior to a connection to a new AP for privacy
+ *	or other reasons. Note that certain driver specific restrictions
+ *	might apply, e.g. no scans in progress, no offchannel operations
+ *	in progress, and no active connections.
+ *
  * @NUM_NL80211_EXT_FEATURES: number of extended features.
  * @MAX_NL80211_EXT_FEATURES: highest extended feature index.
  */
@@ -6340,6 +6358,7 @@ enum nl80211_ext_feature_index {
 	NL80211_EXT_FEATURE_BSS_COLOR,
 	NL80211_EXT_FEATURE_FILS_CRYPTO_OFFLOAD,
 	NL80211_EXT_FEATURE_RADAR_BACKGROUND,
+	NL80211_EXT_FEATURE_POWERED_ADDR_CHANGE,
 
 	/* add new features before the definition below */
 	NUM_NL80211_EXT_FEATURES,
diff --git a/include/uapi/linux/openvswitch.h b/include/uapi/linux/openvswitch.h
index ce3e1738d427..94066f87e9ee 100644
--- a/include/uapi/linux/openvswitch.h
+++ b/include/uapi/linux/openvswitch.h
@@ -76,6 +76,8 @@ enum ovs_datapath_cmd {
  * datapath.  Always present in notifications.
  * @OVS_DP_ATTR_MEGAFLOW_STATS: Statistics about mega flow masks usage for the
  * datapath. Always present in notifications.
+ * @OVS_DP_ATTR_IFINDEX: Interface index for a new datapath netdev. Only
+ * valid for %OVS_DP_CMD_NEW requests.
  *
  * These attributes follow the &struct ovs_header within the Generic Netlink
  * payload for %OVS_DP_* commands.
@@ -92,6 +94,7 @@ enum ovs_datapath_attr {
 	OVS_DP_ATTR_PER_CPU_PIDS,   /* Netlink PIDS to receive upcalls in
 				     * per-cpu dispatch mode
 				     */
+	OVS_DP_ATTR_IFINDEX,
 	__OVS_DP_ATTR_MAX
 };
 
diff --git a/include/uapi/linux/pkt_cls.h b/include/uapi/linux/pkt_cls.h
index 877309d6ca3c..648a82f32666 100644
--- a/include/uapi/linux/pkt_cls.h
+++ b/include/uapi/linux/pkt_cls.h
@@ -592,6 +592,8 @@ enum {
 	TCA_FLOWER_KEY_PPPOE_SID,	/* be16 */
 	TCA_FLOWER_KEY_PPP_PROTO,	/* be16 */
 
+	TCA_FLOWER_KEY_L2TPV3_SID,	/* be32 */
+
 	__TCA_FLOWER_MAX,
 };
 
diff --git a/include/uapi/linux/pkt_sched.h b/include/uapi/linux/pkt_sched.h
index f292b467b27f..000eec106856 100644
--- a/include/uapi/linux/pkt_sched.h
+++ b/include/uapi/linux/pkt_sched.h
@@ -1233,6 +1233,16 @@ enum {
 #define TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD	_BITUL(1)
 
 enum {
+	TCA_TAPRIO_TC_ENTRY_UNSPEC,
+	TCA_TAPRIO_TC_ENTRY_INDEX,		/* u32 */
+	TCA_TAPRIO_TC_ENTRY_MAX_SDU,		/* u32 */
+
+	/* add new constants above here */
+	__TCA_TAPRIO_TC_ENTRY_CNT,
+	TCA_TAPRIO_TC_ENTRY_MAX = (__TCA_TAPRIO_TC_ENTRY_CNT - 1)
+};
+
+enum {
 	TCA_TAPRIO_ATTR_UNSPEC,
 	TCA_TAPRIO_ATTR_PRIOMAP, /* struct tc_mqprio_qopt */
 	TCA_TAPRIO_ATTR_SCHED_ENTRY_LIST, /* nested of entry */
@@ -1245,6 +1255,7 @@ enum {
 	TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION, /* s64 */
 	TCA_TAPRIO_ATTR_FLAGS, /* u32 */
 	TCA_TAPRIO_ATTR_TXTIME_DELAY, /* u32 */
+	TCA_TAPRIO_ATTR_TC_ENTRY, /* nest */
 	__TCA_TAPRIO_ATTR_MAX,
 };
 
diff --git a/include/uapi/linux/psci.h b/include/uapi/linux/psci.h
index 2bf93c0d6354..3511095c2702 100644
--- a/include/uapi/linux/psci.h
+++ b/include/uapi/linux/psci.h
@@ -48,12 +48,26 @@
 #define PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU	PSCI_0_2_FN64(7)
 
 #define PSCI_1_0_FN_PSCI_FEATURES		PSCI_0_2_FN(10)
+#define PSCI_1_0_FN_CPU_FREEZE			PSCI_0_2_FN(11)
+#define PSCI_1_0_FN_CPU_DEFAULT_SUSPEND		PSCI_0_2_FN(12)
+#define PSCI_1_0_FN_NODE_HW_STATE		PSCI_0_2_FN(13)
 #define PSCI_1_0_FN_SYSTEM_SUSPEND		PSCI_0_2_FN(14)
 #define PSCI_1_0_FN_SET_SUSPEND_MODE		PSCI_0_2_FN(15)
+#define PSCI_1_0_FN_STAT_RESIDENCY		PSCI_0_2_FN(16)
+#define PSCI_1_0_FN_STAT_COUNT			PSCI_0_2_FN(17)
+
 #define PSCI_1_1_FN_SYSTEM_RESET2		PSCI_0_2_FN(18)
+#define PSCI_1_1_FN_MEM_PROTECT			PSCI_0_2_FN(19)
+#define PSCI_1_1_FN_MEM_PROTECT_CHECK_RANGE	PSCI_0_2_FN(19)
 
+#define PSCI_1_0_FN64_CPU_DEFAULT_SUSPEND	PSCI_0_2_FN64(12)
+#define PSCI_1_0_FN64_NODE_HW_STATE		PSCI_0_2_FN64(13)
 #define PSCI_1_0_FN64_SYSTEM_SUSPEND		PSCI_0_2_FN64(14)
+#define PSCI_1_0_FN64_STAT_RESIDENCY		PSCI_0_2_FN64(16)
+#define PSCI_1_0_FN64_STAT_COUNT		PSCI_0_2_FN64(17)
+
 #define PSCI_1_1_FN64_SYSTEM_RESET2		PSCI_0_2_FN64(18)
+#define PSCI_1_1_FN64_MEM_PROTECT_CHECK_RANGE	PSCI_0_2_FN64(19)
 
 /* PSCI v0.2 power state encoding for CPU_SUSPEND function */
 #define PSCI_0_2_POWER_STATE_ID_MASK		0xffff
diff --git a/include/uapi/linux/seg6_local.h b/include/uapi/linux/seg6_local.h
index 332b18f318f8..4fdc424c9cb3 100644
--- a/include/uapi/linux/seg6_local.h
+++ b/include/uapi/linux/seg6_local.h
@@ -28,6 +28,7 @@ enum {
 	SEG6_LOCAL_BPF,
 	SEG6_LOCAL_VRFTABLE,
 	SEG6_LOCAL_COUNTERS,
+	SEG6_LOCAL_FLAVORS,
 	__SEG6_LOCAL_MAX,
 };
 #define SEG6_LOCAL_MAX (__SEG6_LOCAL_MAX - 1)
@@ -110,4 +111,27 @@ enum {
 
 #define SEG6_LOCAL_CNT_MAX (__SEG6_LOCAL_CNT_MAX - 1)
 
+/* SRv6 End* Flavor attributes */
+enum {
+	SEG6_LOCAL_FLV_UNSPEC,
+	SEG6_LOCAL_FLV_OPERATION,
+	SEG6_LOCAL_FLV_LCBLOCK_BITS,
+	SEG6_LOCAL_FLV_LCNODE_FN_BITS,
+	__SEG6_LOCAL_FLV_MAX,
+};
+
+#define SEG6_LOCAL_FLV_MAX (__SEG6_LOCAL_FLV_MAX - 1)
+
+/* Designed flavor operations for SRv6 End* Behavior */
+enum {
+	SEG6_LOCAL_FLV_OP_UNSPEC,
+	SEG6_LOCAL_FLV_OP_PSP,
+	SEG6_LOCAL_FLV_OP_USP,
+	SEG6_LOCAL_FLV_OP_USD,
+	SEG6_LOCAL_FLV_OP_NEXT_CSID,
+	__SEG6_LOCAL_FLV_OP_MAX
+};
+
+#define SEG6_LOCAL_FLV_OP_MAX (__SEG6_LOCAL_FLV_OP_MAX - 1)
+
 #endif
diff --git a/include/uapi/linux/stat.h b/include/uapi/linux/stat.h
index 1500a0f58041..7cab2c65d3d7 100644
--- a/include/uapi/linux/stat.h
+++ b/include/uapi/linux/stat.h
@@ -124,7 +124,8 @@ struct statx {
 	__u32	stx_dev_minor;
 	/* 0x90 */
 	__u64	stx_mnt_id;
-	__u64	__spare2;
+	__u32	stx_dio_mem_align;	/* Memory buffer alignment for direct I/O */
+	__u32	stx_dio_offset_align;	/* File offset alignment for direct I/O */
 	/* 0xa0 */
 	__u64	__spare3[12];	/* Spare space for future expansion */
 	/* 0x100 */
@@ -152,6 +153,7 @@ struct statx {
 #define STATX_BASIC_STATS	0x000007ffU	/* The stuff in the normal stat struct */
 #define STATX_BTIME		0x00000800U	/* Want/got stx_btime */
 #define STATX_MNT_ID		0x00001000U	/* Got stx_mnt_id */
+#define STATX_DIOALIGN		0x00002000U	/* Want/got direct I/O alignment info */
 
 #define STATX__RESERVED		0x80000000U	/* Reserved for future struct statx expansion */
 
diff --git a/include/uapi/linux/tc_act/tc_bpf.h b/include/uapi/linux/tc_act/tc_bpf.h
index 653c4f94f76e..fe6c8f8f3e8c 100644
--- a/include/uapi/linux/tc_act/tc_bpf.h
+++ b/include/uapi/linux/tc_act/tc_bpf.h
@@ -1,11 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
 /*
  * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #ifndef __LINUX_TC_BPF_H
diff --git a/include/uapi/linux/tc_act/tc_skbedit.h b/include/uapi/linux/tc_act/tc_skbedit.h
index 6cb6101208d0..64032513cc4c 100644
--- a/include/uapi/linux/tc_act/tc_skbedit.h
+++ b/include/uapi/linux/tc_act/tc_skbedit.h
@@ -2,19 +2,6 @@
 /*
  * Copyright (c) 2008, Intel Corporation.
  *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along with
- * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
- * Place - Suite 330, Boston, MA 02111-1307 USA.
- *
  * Author: Alexander Duyck <alexander.h.duyck@intel.com>
  */
 
diff --git a/include/uapi/linux/tc_act/tc_skbmod.h b/include/uapi/linux/tc_act/tc_skbmod.h
index af6ef2cfbf3d..ac62c9a993ea 100644
--- a/include/uapi/linux/tc_act/tc_skbmod.h
+++ b/include/uapi/linux/tc_act/tc_skbmod.h
@@ -1,12 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
 /*
  * Copyright (c) 2016, Jamal Hadi Salim
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
-*/
+ */
 
 #ifndef __LINUX_TC_SKBMOD_H
 #define __LINUX_TC_SKBMOD_H
diff --git a/include/uapi/linux/tc_act/tc_tunnel_key.h b/include/uapi/linux/tc_act/tc_tunnel_key.h
index 3f10dc4e7a4b..49ad4033951b 100644
--- a/include/uapi/linux/tc_act/tc_tunnel_key.h
+++ b/include/uapi/linux/tc_act/tc_tunnel_key.h
@@ -2,11 +2,6 @@
 /*
  * Copyright (c) 2016, Amir Vadai <amir@vadai.me>
  * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #ifndef __LINUX_TC_TUNNEL_KEY_H
diff --git a/include/uapi/linux/tc_act/tc_vlan.h b/include/uapi/linux/tc_act/tc_vlan.h
index 5b306fe815cc..3e1f8e57cdd2 100644
--- a/include/uapi/linux/tc_act/tc_vlan.h
+++ b/include/uapi/linux/tc_act/tc_vlan.h
@@ -1,11 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
 /*
  * Copyright (c) 2014 Jiri Pirko <jiri@resnulli.us>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #ifndef __LINUX_TC_VLAN_H
diff --git a/include/uapi/linux/tls.h b/include/uapi/linux/tls.h
index f1157d8f4acd..b66a800389cc 100644
--- a/include/uapi/linux/tls.h
+++ b/include/uapi/linux/tls.h
@@ -100,6 +100,20 @@
 #define TLS_CIPHER_SM4_CCM_TAG_SIZE		16
 #define TLS_CIPHER_SM4_CCM_REC_SEQ_SIZE		8
 
+#define TLS_CIPHER_ARIA_GCM_128				57
+#define TLS_CIPHER_ARIA_GCM_128_IV_SIZE			8
+#define TLS_CIPHER_ARIA_GCM_128_KEY_SIZE		16
+#define TLS_CIPHER_ARIA_GCM_128_SALT_SIZE		4
+#define TLS_CIPHER_ARIA_GCM_128_TAG_SIZE		16
+#define TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE		8
+
+#define TLS_CIPHER_ARIA_GCM_256				58
+#define TLS_CIPHER_ARIA_GCM_256_IV_SIZE			8
+#define TLS_CIPHER_ARIA_GCM_256_KEY_SIZE		32
+#define TLS_CIPHER_ARIA_GCM_256_SALT_SIZE		4
+#define TLS_CIPHER_ARIA_GCM_256_TAG_SIZE		16
+#define TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE		8
+
 #define TLS_SET_RECORD_TYPE	1
 #define TLS_GET_RECORD_TYPE	2
 
@@ -156,6 +170,22 @@ struct tls12_crypto_info_sm4_ccm {
 	unsigned char rec_seq[TLS_CIPHER_SM4_CCM_REC_SEQ_SIZE];
 };
 
+struct tls12_crypto_info_aria_gcm_128 {
+	struct tls_crypto_info info;
+	unsigned char iv[TLS_CIPHER_ARIA_GCM_128_IV_SIZE];
+	unsigned char key[TLS_CIPHER_ARIA_GCM_128_KEY_SIZE];
+	unsigned char salt[TLS_CIPHER_ARIA_GCM_128_SALT_SIZE];
+	unsigned char rec_seq[TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE];
+};
+
+struct tls12_crypto_info_aria_gcm_256 {
+	struct tls_crypto_info info;
+	unsigned char iv[TLS_CIPHER_ARIA_GCM_256_IV_SIZE];
+	unsigned char key[TLS_CIPHER_ARIA_GCM_256_KEY_SIZE];
+	unsigned char salt[TLS_CIPHER_ARIA_GCM_256_SALT_SIZE];
+	unsigned char rec_seq[TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE];
+};
+
 enum {
 	TLS_INFO_UNSPEC,
 	TLS_INFO_VERSION,
diff --git a/include/uapi/sound/asequencer.h b/include/uapi/sound/asequencer.h
index a75e14edc957..6d4a2c60808d 100644
--- a/include/uapi/sound/asequencer.h
+++ b/include/uapi/sound/asequencer.h
@@ -3,22 +3,6 @@
  *  Main header file for the ALSA sequencer
  *  Copyright (c) 1998-1999 by Frank van de Pol <fvdpol@coil.demon.nl>
  *            (c) 1998-1999 by Jaroslav Kysela <perex@perex.cz>
- *
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation; either version 2 of the License, or
- *   (at your option) any later version.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program; if not, write to the Free Software
- *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
- *
  */
 #ifndef _UAPI__SOUND_ASEQUENCER_H
 #define _UAPI__SOUND_ASEQUENCER_H
diff --git a/include/uapi/sound/asoc.h b/include/uapi/sound/asoc.h
index 053949287ce8..9f35bedafcff 100644
--- a/include/uapi/sound/asoc.h
+++ b/include/uapi/sound/asoc.h
@@ -5,10 +5,6 @@
  * Copyright (C) 2012 Texas Instruments Inc.
  * Copyright (C) 2015 Intel Corporation.
  *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- *
  * Simple file API to load FW that includes mixers, coefficients, DAPM graphs,
  * algorithms, equalisers, DAIs, widgets etc.
 */
diff --git a/include/uapi/sound/asound.h b/include/uapi/sound/asound.h
index 3974a2a911cc..de6810e94abe 100644
--- a/include/uapi/sound/asound.h
+++ b/include/uapi/sound/asound.h
@@ -3,22 +3,6 @@
  *  Advanced Linux Sound Architecture - ALSA - Driver
  *  Copyright (c) 1994-2003 by Jaroslav Kysela <perex@perex.cz>,
  *                             Abramo Bagnara <abramo@alsa-project.org>
- *
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation; either version 2 of the License, or
- *   (at your option) any later version.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program; if not, write to the Free Software
- *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
- *
  */
 
 #ifndef _UAPI__SOUND_ASOUND_H
diff --git a/include/uapi/sound/asound_fm.h b/include/uapi/sound/asound_fm.h
index 8471f404ff0b..25ec5e38af5c 100644
--- a/include/uapi/sound/asound_fm.h
+++ b/include/uapi/sound/asound_fm.h
@@ -10,21 +10,6 @@
  *                           4Front Technologies
  *
  *  Direct FM control
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation; either version 2 of the License, or
- *   (at your option) any later version.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program; if not, write to the Free Software
- *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
- *
  */
 
 #define SNDRV_DM_FM_MODE_OPL2	0x00
diff --git a/include/uapi/sound/compress_offload.h b/include/uapi/sound/compress_offload.h
index 3aef123dbd7f..d185957f3fe0 100644
--- a/include/uapi/sound/compress_offload.h
+++ b/include/uapi/sound/compress_offload.h
@@ -5,23 +5,6 @@
  *  Copyright (C) 2011 Intel Corporation
  *  Authors:	Vinod Koul <vinod.koul@linux.intel.com>
  *		Pierre-Louis Bossart <pierre-louis.bossart@linux.intel.com>
- *  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License along
- *  with this program; if not, write to the Free Software Foundation, Inc.,
- *  59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
- *
- * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- *
  */
 #ifndef __COMPRESS_OFFLOAD_H
 #define __COMPRESS_OFFLOAD_H
diff --git a/include/uapi/sound/compress_params.h b/include/uapi/sound/compress_params.h
index 726361716919..ddc77322d571 100644
--- a/include/uapi/sound/compress_params.h
+++ b/include/uapi/sound/compress_params.h
@@ -7,47 +7,13 @@
  *  Authors:	Pierre-Louis Bossart <pierre-louis.bossart@linux.intel.com>
  *              Vinod Koul <vinod.koul@linux.intel.com>
  *
- *  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License as published by
- *  the Free Software Foundation; version 2 of the License.
- *
- *  This program is distributed in the hope that it will be useful, but
- *  WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  General Public License for more details.
- *
- *  You should have received a copy of the GNU General Public License along
- *  with this program; if not, write to the Free Software Foundation, Inc.,
- *  59 Temple Place, Suite 330, Boston, MA 02111-1307 USA.
- *
  * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  *
  * The definitions in this file are derived from the OpenMAX AL version 1.1
- * and OpenMAX IL v 1.1.2 header files which contain the copyright notice below.
+ * and OpenMAX IL v 1.1.2 header files which contain the copyright notice below
+ * and are licensed under the MIT license.
  *
  * Copyright (c) 2007-2010 The Khronos Group Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and/or associated documentation files (the
- * "Materials "), to deal in the Materials without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Materials, and to
- * permit persons to whom the Materials are furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice shall be included
- * in all copies or substantial portions of the Materials.
- *
- * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
- *
  */
 #ifndef __SND_COMPRESS_PARAMS_H
 #define __SND_COMPRESS_PARAMS_H
diff --git a/include/uapi/sound/emu10k1.h b/include/uapi/sound/emu10k1.h
index 88609cc0524c..1c1f1dd44611 100644
--- a/include/uapi/sound/emu10k1.h
+++ b/include/uapi/sound/emu10k1.h
@@ -3,22 +3,6 @@
  *  Copyright (c) by Jaroslav Kysela <perex@perex.cz>,
  *		     Creative Labs, Inc.
  *  Definitions for EMU10K1 (SB Live!) chips
- *
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation; either version 2 of the License, or
- *   (at your option) any later version.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program; if not, write to the Free Software
- *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
- *
  */
 #ifndef _UAPI__SOUND_EMU10K1_H
 #define _UAPI__SOUND_EMU10K1_H
diff --git a/include/uapi/sound/hdsp.h b/include/uapi/sound/hdsp.h
index b8df62b60f4d..0961954658d6 100644
--- a/include/uapi/sound/hdsp.h
+++ b/include/uapi/sound/hdsp.h
@@ -4,20 +4,6 @@
 
 /*
  *   Copyright (C) 2003 Thomas Charbonnel (thomas@undata.org)
- *    
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation; either version 2 of the License, or
- *   (at your option) any later version.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program; if not, write to the Free Software
- *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
 #ifdef __linux__
diff --git a/include/uapi/sound/hdspm.h b/include/uapi/sound/hdspm.h
index 14af3d00ea3f..7043bb3d435a 100644
--- a/include/uapi/sound/hdspm.h
+++ b/include/uapi/sound/hdspm.h
@@ -4,21 +4,6 @@
 /*
  *   Copyright (C) 2003 Winfried Ritsch (IEM)
  *   based on hdsp.h from Thomas Charbonnel (thomas@undata.org)
- *
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation; either version 2 of the License, or
- *   (at your option) any later version.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program; if not, write to the Free Software
- *   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
 #ifdef __linux__
diff --git a/include/uapi/sound/sb16_csp.h b/include/uapi/sound/sb16_csp.h
index e64851481d88..5a80f5ec02ee 100644
--- a/include/uapi/sound/sb16_csp.h
+++ b/include/uapi/sound/sb16_csp.h
@@ -4,21 +4,6 @@
  *                        Takashi Iwai <tiwai@suse.de>
  *
  *  SB16ASP/AWE32 CSP control
- *
- *   This program is free software; you can redistribute it and/or modify 
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation; either version 2 of the License, or
- *   (at your option) any later version.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program; if not, write to the Free Software
- *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
- *
  */
 #ifndef _UAPI__SOUND_SB16_CSP_H
 #define _UAPI__SOUND_SB16_CSP_H
diff --git a/include/uapi/sound/sfnt_info.h b/include/uapi/sound/sfnt_info.h
index c9a810a6ef48..f2b5e13fb5a7 100644
--- a/include/uapi/sound/sfnt_info.h
+++ b/include/uapi/sound/sfnt_info.h
@@ -6,21 +6,6 @@
  *  Patch record compatible with AWE driver on OSS
  *
  *  Copyright (C) 1999-2000 Takashi Iwai
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation; either version 2 of the License, or
- *   (at your option) any later version.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- *
- *   You should have received a copy of the GNU General Public License
- *   along with this program; if not, write to the Free Software
- *   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
- *
  */
 
 #include <sound/asound.h>
diff --git a/include/uapi/sound/snd_sst_tokens.h b/include/uapi/sound/snd_sst_tokens.h
index ff3748e9308a..defeb0c6ed20 100644
--- a/include/uapi/sound/snd_sst_tokens.h
+++ b/include/uapi/sound/snd_sst_tokens.h
@@ -4,16 +4,6 @@
  *
  * Copyright (C) 2016 Intel Corp
  * Author: Shreyas NC <shreyas.nc@intel.com>
- * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as version 2, as
- * published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
  */
 #ifndef __SND_SST_TOKENS_H__
 #define __SND_SST_TOKENS_H__
diff --git a/include/uapi/sound/tlv.h b/include/uapi/sound/tlv.h
index 7d6d65f60a42..b99a2414b53d 100644
--- a/include/uapi/sound/tlv.h
+++ b/include/uapi/sound/tlv.h
@@ -1,15 +1,4 @@
 /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
-/*
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation; either version 2 of the License, or
- *   (at your option) any later version.
- *
- *   This program is distributed in the hope that it will be useful,
- *   but WITHOUT ANY WARRANTY; without even the implied warranty of
- *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- *   GNU General Public License for more details.
- */
 
 #ifndef __UAPI_SOUND_TLV_H
 #define __UAPI_SOUND_TLV_H
diff --git a/include/uapi/sound/usb_stream.h b/include/uapi/sound/usb_stream.h
index ffdd3ea1e31d..50609016185a 100644
--- a/include/uapi/sound/usb_stream.h
+++ b/include/uapi/sound/usb_stream.h
@@ -1,20 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
 /*
  * Copyright (C) 2007, 2008 Karsten Wiese <fzu@wemgehoertderstaat.de>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation; either version 2 of the License, or (at your
- * option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software Foundation,
- * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
 #ifndef _UAPI__SOUND_USB_STREAM_H
diff --git a/include/video/vga.h b/include/video/vga.h
index d334e64c1c19..947c0abd04ef 100644
--- a/include/video/vga.h
+++ b/include/video/vga.h
@@ -2,15 +2,15 @@
  * linux/include/video/vga.h -- standard VGA chipset interaction
  *
  * Copyright 1999 Jeff Garzik <jgarzik@pobox.com>
- * 
+ *
  * Copyright history from vga16fb.c:
  *	Copyright 1999 Ben Pfaff and Petr Vandrovec
- *	Based on VGA info at http://www.osdever.net/FreeVGA/home.htm 
+ *	Based on VGA info at http://www.osdever.net/FreeVGA/home.htm
  *	Based on VESA framebuffer (c) 1998 Gerd Knorr
  *
  * This file is subject to the terms and conditions of the GNU General
  * Public License.  See the file COPYING in the main directory of this
- * archive for more details.  
+ * archive for more details.
  *
  */
 
@@ -22,6 +22,8 @@
 #include <asm/vga.h>
 #include <asm/byteorder.h>
 
+#define VGA_FB_PHYS_BASE	0xA0000 /* VGA framebuffer I/O base */
+#define VGA_FB_PHYS_SIZE	65536	/* VGA framebuffer I/O size */
 
 /* Some of the code below is taken from SVGAlib.  The original,
    unmodified copyright notice for that code is below. */
@@ -190,7 +192,7 @@ struct vgastate {
 	__u32 num_gfx;		/* number of gfx registers, 0 for default  */
 	__u32 num_seq;		/* number of seq registers, 0 for default  */
 	void *vidstate;
-};	
+};
 
 extern int save_vga(struct vgastate *state);
 extern int restore_vga(struct vgastate *state);
@@ -198,7 +200,7 @@ extern int restore_vga(struct vgastate *state);
 /*
  * generic VGA port read/write
  */
- 
+
 static inline unsigned char vga_io_r (unsigned short port)
 {
 	return inb_p(port);
@@ -261,7 +263,7 @@ static inline void vga_w_fast (void __iomem *regbase, unsigned short port,
 /*
  * VGA CRTC register read/write
  */
- 
+
 static inline unsigned char vga_rcrt (void __iomem *regbase, unsigned char reg)
 {
         vga_w (regbase, VGA_CRT_IC, reg);
@@ -314,7 +316,7 @@ static inline void vga_mm_wcrt (void __iomem *regbase, unsigned char reg, unsign
 /*
  * VGA sequencer register read/write
  */
- 
+
 static inline unsigned char vga_rseq (void __iomem *regbase, unsigned char reg)
 {
         vga_w (regbase, VGA_SEQ_I, reg);
@@ -366,7 +368,7 @@ static inline void vga_mm_wseq (void __iomem *regbase, unsigned char reg, unsign
 /*
  * VGA graphics controller register read/write
  */
- 
+
 static inline unsigned char vga_rgfx (void __iomem *regbase, unsigned char reg)
 {
         vga_w (regbase, VGA_GFX_I, reg);
@@ -419,7 +421,7 @@ static inline void vga_mm_wgfx (void __iomem *regbase, unsigned char reg, unsign
 /*
  * VGA attribute controller register read/write
  */
- 
+
 static inline unsigned char vga_rattr (void __iomem *regbase, unsigned char reg)
 {
         vga_w (regbase, VGA_ATT_IW, reg);
diff --git a/init/Kconfig b/init/Kconfig
index 532362fcfe31..a078cb026523 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -60,6 +60,17 @@ config LLD_VERSION
 	default $(ld-version) if LD_IS_LLD
 	default 0
 
+config RUST_IS_AVAILABLE
+	def_bool $(success,$(srctree)/scripts/rust_is_available.sh)
+	help
+	  This shows whether a suitable Rust toolchain is available (found).
+
+	  Please see Documentation/rust/quick-start.rst for instructions on how
+	  to satify the build requirements of Rust support.
+
+	  In particular, the Makefile target 'rustavailable' is useful to check
+	  why the Rust toolchain is not being detected.
+
 config CC_CAN_LINK
 	bool
 	default $(success,$(srctree)/scripts/cc-can-link.sh $(CC) $(CLANG_FLAGS) $(USERCFLAGS) $(USERLDFLAGS) $(m64-flag)) if 64BIT
@@ -147,7 +158,8 @@ config WERROR
 	default COMPILE_TEST
 	help
 	  A kernel build should not cause any compiler warnings, and this
-	  enables the '-Werror' flag to enforce that rule by default.
+	  enables the '-Werror' (for C) and '-Dwarnings' (for Rust) flags
+	  to enforce that rule by default.
 
 	  However, if you have a new (or very old) compiler with odd and
 	  unusual warnings, or you have some architecture with problems,
@@ -1899,6 +1911,38 @@ config PROFILING
 	  Say Y here to enable the extended profiling support mechanisms used
 	  by profilers.
 
+config RUST
+	bool "Rust support"
+	depends on HAVE_RUST
+	depends on RUST_IS_AVAILABLE
+	depends on !MODVERSIONS
+	depends on !GCC_PLUGINS
+	depends on !RANDSTRUCT
+	depends on !DEBUG_INFO_BTF
+	select CONSTRUCTORS
+	help
+	  Enables Rust support in the kernel.
+
+	  This allows other Rust-related options, like drivers written in Rust,
+	  to be selected.
+
+	  It is also required to be able to load external kernel modules
+	  written in Rust.
+
+	  See Documentation/rust/ for more information.
+
+	  If unsure, say N.
+
+config RUSTC_VERSION_TEXT
+	string
+	depends on RUST
+	default $(shell,command -v $(RUSTC) >/dev/null 2>&1 && $(RUSTC) --version || echo n)
+
+config BINDGEN_VERSION_TEXT
+	string
+	depends on RUST
+	default $(shell,command -v $(BINDGEN) >/dev/null 2>&1 && $(BINDGEN) --version || echo n)
+
 #
 # Place an empty function call at each tracepoint site. Can be
 # dynamically changed for a probe function.
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 2965b354efc8..242d896c00f3 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -3357,6 +3357,10 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
 		goto err;
 	}
 
+	if (ctx->flags & IORING_SETUP_SINGLE_ISSUER
+	    && !(ctx->flags & IORING_SETUP_R_DISABLED))
+		ctx->submitter_task = get_task_struct(current);
+
 	file = io_uring_get_file(ctx);
 	if (IS_ERR(file)) {
 		ret = PTR_ERR(file);
@@ -3548,6 +3552,9 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx)
 	if (!(ctx->flags & IORING_SETUP_R_DISABLED))
 		return -EBADFD;
 
+	if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task)
+		ctx->submitter_task = get_task_struct(current);
+
 	if (ctx->restrictions.registered)
 		ctx->restricted = 1;
 
diff --git a/io_uring/poll.c b/io_uring/poll.c
index d5bad0bea6e4..0d9f49c575e0 100644
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -857,7 +857,7 @@ int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	if (sqe->buf_index || sqe->off || sqe->addr)
 		return -EINVAL;
 	flags = READ_ONCE(sqe->len);
-	if (flags & ~(IORING_POLL_ADD_MULTI|IORING_POLL_ADD_LEVEL))
+	if (flags & ~IORING_POLL_ADD_MULTI)
 		return -EINVAL;
 	if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP))
 		return -EINVAL;
diff --git a/io_uring/xattr.c b/io_uring/xattr.c
index 84180afd090b..99df641594d7 100644
--- a/io_uring/xattr.c
+++ b/io_uring/xattr.c
@@ -206,7 +206,7 @@ int io_fsetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 }
 
 static int __io_setxattr(struct io_kiocb *req, unsigned int issue_flags,
-			struct path *path)
+			const struct path *path)
 {
 	struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr);
 	int ret;
diff --git a/kernel/audit.c b/kernel/audit.c
index a75978ae38ad..9bc0b0301198 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -321,7 +321,6 @@ static inline int audit_rate_check(void)
 	static DEFINE_SPINLOCK(lock);
 	unsigned long		flags;
 	unsigned long		now;
-	unsigned long		elapsed;
 	int			retval	   = 0;
 
 	if (!audit_rate_limit) return 1;
@@ -330,9 +329,8 @@ static inline int audit_rate_check(void)
 	if (++messages < audit_rate_limit) {
 		retval = 1;
 	} else {
-		now     = jiffies;
-		elapsed = now - last_check;
-		if (elapsed > HZ) {
+		now = jiffies;
+		if (time_after(now, last_check + HZ)) {
 			last_check = now;
 			messages   = 0;
 			retval     = 1;
@@ -366,7 +364,7 @@ void audit_log_lost(const char *message)
 	if (!print) {
 		spin_lock_irqsave(&lock, flags);
 		now = jiffies;
-		if (now - last_msg > HZ) {
+		if (time_after(now, last_msg + HZ)) {
 			print = 1;
 			last_msg = now;
 		}
diff --git a/kernel/audit.h b/kernel/audit.h
index 58b66543b4d5..c57b008b9914 100644
--- a/kernel/audit.h
+++ b/kernel/audit.h
@@ -133,7 +133,7 @@ struct audit_context {
 	struct sockaddr_storage *sockaddr;
 	size_t sockaddr_len;
 				/* Save things to print about task_struct */
-	pid_t		    pid, ppid;
+	pid_t		    ppid;
 	kuid_t		    uid, euid, suid, fsuid;
 	kgid_t		    gid, egid, sgid, fsgid;
 	unsigned long	    personality;
@@ -245,8 +245,6 @@ struct audit_netlink_list {
 
 int audit_send_list_thread(void *_dest);
 
-extern int selinux_audit_rule_update(void);
-
 extern struct mutex audit_filter_mutex;
 extern int audit_del_rule(struct audit_entry *entry);
 extern void audit_free_rule_rcu(struct rcu_head *head);
diff --git a/kernel/audit_watch.c b/kernel/audit_watch.c
index 4b0957aa2cd4..65075f1e4ac8 100644
--- a/kernel/audit_watch.c
+++ b/kernel/audit_watch.c
@@ -133,7 +133,7 @@ int audit_watch_compare(struct audit_watch *watch, unsigned long ino, dev_t dev)
 }
 
 /* Initialize a parent watch entry. */
-static struct audit_parent *audit_init_parent(struct path *path)
+static struct audit_parent *audit_init_parent(const struct path *path)
 {
 	struct inode *inode = d_backing_inode(path->dentry);
 	struct audit_parent *parent;
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index 79a5da1bc5bb..9f8c05228d6d 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -965,7 +965,7 @@ static void audit_reset_context(struct audit_context *ctx)
 	if (!ctx)
 		return;
 
-	/* if ctx is non-null, reset the "ctx->state" regardless */
+	/* if ctx is non-null, reset the "ctx->context" regardless */
 	ctx->context = AUDIT_CTX_UNUSED;
 	if (ctx->dummy)
 		return;
@@ -1002,7 +1002,7 @@ static void audit_reset_context(struct audit_context *ctx)
 	kfree(ctx->sockaddr);
 	ctx->sockaddr = NULL;
 	ctx->sockaddr_len = 0;
-	ctx->pid = ctx->ppid = 0;
+	ctx->ppid = 0;
 	ctx->uid = ctx->euid = ctx->suid = ctx->fsuid = KUIDT_INIT(0);
 	ctx->gid = ctx->egid = ctx->sgid = ctx->fsgid = KGIDT_INIT(0);
 	ctx->personality = 0;
@@ -1016,7 +1016,6 @@ static void audit_reset_context(struct audit_context *ctx)
 	WARN_ON(!list_empty(&ctx->killed_trees));
 	audit_free_module(ctx);
 	ctx->fds[0] = -1;
-	audit_proctitle_free(ctx);
 	ctx->type = 0; /* reset last for audit_free_*() */
 }
 
@@ -1077,6 +1076,7 @@ static inline void audit_free_context(struct audit_context *context)
 {
 	/* resetting is extra work, but it is likely just noise */
 	audit_reset_context(context);
+	audit_proctitle_free(context);
 	free_tree_refs(context);
 	kfree(context->filterkey);
 	kfree(context);
@@ -1833,7 +1833,7 @@ void __audit_free(struct task_struct *tsk)
 
 	/* We are called either by do_exit() or the fork() error handling code;
 	 * in the former case tsk == current and in the latter tsk is a
-	 * random task_struct that doesn't doesn't have any meaningful data we
+	 * random task_struct that doesn't have any meaningful data we
 	 * need to log via audit_log_exit().
 	 */
 	if (tsk == current && !context->dummy) {
@@ -2069,7 +2069,7 @@ void __audit_syscall_exit(int success, long return_code)
 	/* run through both filters to ensure we set the filterkey properly */
 	audit_filter_syscall(current, context);
 	audit_filter_inodes(current, context);
-	if (context->current_state < AUDIT_STATE_RECORD)
+	if (context->current_state != AUDIT_STATE_RECORD)
 		goto out;
 
 	audit_log_exit();
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index 057ba8e01e70..341c94f208f4 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -13,7 +13,7 @@ obj-$(CONFIG_BPF_SYSCALL) += bpf_local_storage.o bpf_task_storage.o
 obj-${CONFIG_BPF_LSM}	  += bpf_inode_storage.o
 obj-$(CONFIG_BPF_SYSCALL) += disasm.o
 obj-$(CONFIG_BPF_JIT) += trampoline.o
-obj-$(CONFIG_BPF_SYSCALL) += btf.o
+obj-$(CONFIG_BPF_SYSCALL) += btf.o memalloc.o
 obj-$(CONFIG_BPF_JIT) += dispatcher.o
 ifeq ($(CONFIG_NET),y)
 obj-$(CONFIG_BPF_SYSCALL) += devmap.o
@@ -24,6 +24,9 @@ endif
 ifeq ($(CONFIG_PERF_EVENTS),y)
 obj-$(CONFIG_BPF_SYSCALL) += stackmap.o
 endif
+ifeq ($(CONFIG_CGROUPS),y)
+obj-$(CONFIG_BPF_SYSCALL) += cgroup_iter.o
+endif
 obj-$(CONFIG_CGROUP_BPF) += cgroup.o
 ifeq ($(CONFIG_INET),y)
 obj-$(CONFIG_BPF_SYSCALL) += reuseport_array.o
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 624527401d4d..832b2659e96e 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -279,7 +279,8 @@ int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value)
 	rcu_read_lock();
 	pptr = array->pptrs[index & array->index_mask];
 	for_each_possible_cpu(cpu) {
-		bpf_long_memcpy(value + off, per_cpu_ptr(pptr, cpu), size);
+		copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu));
+		check_and_init_map_value(map, value + off);
 		off += size;
 	}
 	rcu_read_unlock();
@@ -338,8 +339,9 @@ static int array_map_update_elem(struct bpf_map *map, void *key, void *value,
 		return -EINVAL;
 
 	if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
-		memcpy(this_cpu_ptr(array->pptrs[index & array->index_mask]),
-		       value, map->value_size);
+		val = this_cpu_ptr(array->pptrs[index & array->index_mask]);
+		copy_map_value(map, val, value);
+		check_and_free_fields(array, val);
 	} else {
 		val = array->value +
 			(u64)array->elem_size * (index & array->index_mask);
@@ -383,7 +385,8 @@ int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value,
 	rcu_read_lock();
 	pptr = array->pptrs[index & array->index_mask];
 	for_each_possible_cpu(cpu) {
-		bpf_long_memcpy(per_cpu_ptr(pptr, cpu), value + off, size);
+		copy_map_value_long(map, per_cpu_ptr(pptr, cpu), value + off);
+		check_and_free_fields(array, per_cpu_ptr(pptr, cpu));
 		off += size;
 	}
 	rcu_read_unlock();
@@ -421,8 +424,20 @@ static void array_map_free(struct bpf_map *map)
 	int i;
 
 	if (map_value_has_kptrs(map)) {
-		for (i = 0; i < array->map.max_entries; i++)
-			bpf_map_free_kptrs(map, array_map_elem_ptr(array, i));
+		if (array->map.map_type == BPF_MAP_TYPE_PERCPU_ARRAY) {
+			for (i = 0; i < array->map.max_entries; i++) {
+				void __percpu *pptr = array->pptrs[i & array->index_mask];
+				int cpu;
+
+				for_each_possible_cpu(cpu) {
+					bpf_map_free_kptrs(map, per_cpu_ptr(pptr, cpu));
+					cond_resched();
+				}
+			}
+		} else {
+			for (i = 0; i < array->map.max_entries; i++)
+				bpf_map_free_kptrs(map, array_map_elem_ptr(array, i));
+		}
 		bpf_map_free_kptr_off_tab(map);
 	}
 
@@ -608,9 +623,9 @@ static int __bpf_array_map_seq_show(struct seq_file *seq, void *v)
 			pptr = v;
 			size = array->elem_size;
 			for_each_possible_cpu(cpu) {
-				bpf_long_memcpy(info->percpu_value_buf + off,
-						per_cpu_ptr(pptr, cpu),
-						size);
+				copy_map_value_long(map, info->percpu_value_buf + off,
+						    per_cpu_ptr(pptr, cpu));
+				check_and_init_map_value(map, info->percpu_value_buf + off);
 				off += size;
 			}
 			ctx.value = info->percpu_value_buf;
diff --git a/kernel/bpf/bpf_iter.c b/kernel/bpf/bpf_iter.c
index 24b755eca0b3..5dc307bdeaeb 100644
--- a/kernel/bpf/bpf_iter.c
+++ b/kernel/bpf/bpf_iter.c
@@ -202,6 +202,11 @@ static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
 	}
 stop:
 	offs = seq->count;
+	if (IS_ERR(p)) {
+		seq->op->stop(seq, NULL);
+		err = PTR_ERR(p);
+		goto done;
+	}
 	/* bpf program called if !p */
 	seq->op->stop(seq, p);
 	if (!p) {
@@ -689,19 +694,24 @@ struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
 
 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
 {
+	struct bpf_run_ctx run_ctx, *old_run_ctx;
 	int ret;
 
 	if (prog->aux->sleepable) {
 		rcu_read_lock_trace();
 		migrate_disable();
 		might_fault();
+		old_run_ctx = bpf_set_run_ctx(&run_ctx);
 		ret = bpf_prog_run(prog, ctx);
+		bpf_reset_run_ctx(old_run_ctx);
 		migrate_enable();
 		rcu_read_unlock_trace();
 	} else {
 		rcu_read_lock();
 		migrate_disable();
+		old_run_ctx = bpf_set_run_ctx(&run_ctx);
 		ret = bpf_prog_run(prog, ctx);
+		bpf_reset_run_ctx(old_run_ctx);
 		migrate_enable();
 		rcu_read_unlock();
 	}
diff --git a/kernel/bpf/bpf_local_storage.c b/kernel/bpf/bpf_local_storage.c
index 8ce40fd869f6..802fc15b0d73 100644
--- a/kernel/bpf/bpf_local_storage.c
+++ b/kernel/bpf/bpf_local_storage.c
@@ -555,11 +555,11 @@ void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,
 				struct bpf_local_storage_elem, map_node))) {
 			if (busy_counter) {
 				migrate_disable();
-				__this_cpu_inc(*busy_counter);
+				this_cpu_inc(*busy_counter);
 			}
 			bpf_selem_unlink(selem, false);
 			if (busy_counter) {
-				__this_cpu_dec(*busy_counter);
+				this_cpu_dec(*busy_counter);
 				migrate_enable();
 			}
 			cond_resched_rcu();
@@ -582,7 +582,7 @@ void bpf_local_storage_map_free(struct bpf_local_storage_map *smap,
 	synchronize_rcu();
 
 	kvfree(smap->buckets);
-	kfree(smap);
+	bpf_map_area_free(smap);
 }
 
 int bpf_local_storage_map_alloc_check(union bpf_attr *attr)
@@ -610,7 +610,7 @@ struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr)
 	unsigned int i;
 	u32 nbuckets;
 
-	smap = kzalloc(sizeof(*smap), GFP_USER | __GFP_NOWARN | __GFP_ACCOUNT);
+	smap = bpf_map_area_alloc(sizeof(*smap), NUMA_NO_NODE);
 	if (!smap)
 		return ERR_PTR(-ENOMEM);
 	bpf_map_init_from_attr(&smap->map, attr);
@@ -623,7 +623,7 @@ struct bpf_local_storage_map *bpf_local_storage_map_alloc(union bpf_attr *attr)
 	smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets,
 				 GFP_USER | __GFP_NOWARN | __GFP_ACCOUNT);
 	if (!smap->buckets) {
-		kfree(smap);
+		bpf_map_area_free(smap);
 		return ERR_PTR(-ENOMEM);
 	}
 
diff --git a/kernel/bpf/bpf_lsm.c b/kernel/bpf/bpf_lsm.c
index fa71d58b7ded..d6c9b3705f24 100644
--- a/kernel/bpf/bpf_lsm.c
+++ b/kernel/bpf/bpf_lsm.c
@@ -41,17 +41,21 @@ BTF_SET_END(bpf_lsm_hooks)
  */
 BTF_SET_START(bpf_lsm_current_hooks)
 /* operate on freshly allocated sk without any cgroup association */
+#ifdef CONFIG_SECURITY_NETWORK
 BTF_ID(func, bpf_lsm_sk_alloc_security)
 BTF_ID(func, bpf_lsm_sk_free_security)
+#endif
 BTF_SET_END(bpf_lsm_current_hooks)
 
 /* List of LSM hooks that trigger while the socket is properly locked.
  */
 BTF_SET_START(bpf_lsm_locked_sockopt_hooks)
+#ifdef CONFIG_SECURITY_NETWORK
 BTF_ID(func, bpf_lsm_socket_sock_rcv_skb)
 BTF_ID(func, bpf_lsm_sock_graft)
 BTF_ID(func, bpf_lsm_inet_csk_clone)
 BTF_ID(func, bpf_lsm_inet_conn_established)
+#endif
 BTF_SET_END(bpf_lsm_locked_sockopt_hooks)
 
 /* List of LSM hooks that trigger while the socket is _not_ locked,
@@ -59,8 +63,10 @@ BTF_SET_END(bpf_lsm_locked_sockopt_hooks)
  * in the early init phase.
  */
 BTF_SET_START(bpf_lsm_unlocked_sockopt_hooks)
+#ifdef CONFIG_SECURITY_NETWORK
 BTF_ID(func, bpf_lsm_socket_post_create)
 BTF_ID(func, bpf_lsm_socket_socketpair)
+#endif
 BTF_SET_END(bpf_lsm_unlocked_sockopt_hooks)
 
 #ifdef CONFIG_CGROUP_BPF
@@ -189,6 +195,14 @@ static const struct bpf_func_proto bpf_get_attach_cookie_proto = {
 static const struct bpf_func_proto *
 bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
+	const struct bpf_func_proto *func_proto;
+
+	if (prog->expected_attach_type == BPF_LSM_CGROUP) {
+		func_proto = cgroup_common_func_proto(func_id, prog);
+		if (func_proto)
+			return func_proto;
+	}
+
 	switch (func_id) {
 	case BPF_FUNC_inode_storage_get:
 		return &bpf_inode_storage_get_proto;
@@ -212,15 +226,6 @@ bpf_lsm_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return prog->aux->sleepable ? &bpf_ima_file_hash_proto : NULL;
 	case BPF_FUNC_get_attach_cookie:
 		return bpf_prog_has_trampoline(prog) ? &bpf_get_attach_cookie_proto : NULL;
-	case BPF_FUNC_get_local_storage:
-		return prog->expected_attach_type == BPF_LSM_CGROUP ?
-			&bpf_get_local_storage_proto : NULL;
-	case BPF_FUNC_set_retval:
-		return prog->expected_attach_type == BPF_LSM_CGROUP ?
-			&bpf_set_retval_proto : NULL;
-	case BPF_FUNC_get_retval:
-		return prog->expected_attach_type == BPF_LSM_CGROUP ?
-			&bpf_get_retval_proto : NULL;
 #ifdef CONFIG_NET
 	case BPF_FUNC_setsockopt:
 		if (prog->expected_attach_type != BPF_LSM_CGROUP)
@@ -335,6 +340,7 @@ BTF_ID(func, bpf_lsm_task_getsecid_obj)
 BTF_ID(func, bpf_lsm_task_prctl)
 BTF_ID(func, bpf_lsm_task_setscheduler)
 BTF_ID(func, bpf_lsm_task_to_inode)
+BTF_ID(func, bpf_lsm_userns_create)
 BTF_SET_END(sleepable_lsm_hooks)
 
 bool bpf_lsm_is_sleepable_hook(u32 btf_id)
diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c
index e9014dc62682..6f290623347e 100644
--- a/kernel/bpf/bpf_task_storage.c
+++ b/kernel/bpf/bpf_task_storage.c
@@ -26,20 +26,20 @@ static DEFINE_PER_CPU(int, bpf_task_storage_busy);
 static void bpf_task_storage_lock(void)
 {
 	migrate_disable();
-	__this_cpu_inc(bpf_task_storage_busy);
+	this_cpu_inc(bpf_task_storage_busy);
 }
 
 static void bpf_task_storage_unlock(void)
 {
-	__this_cpu_dec(bpf_task_storage_busy);
+	this_cpu_dec(bpf_task_storage_busy);
 	migrate_enable();
 }
 
 static bool bpf_task_storage_trylock(void)
 {
 	migrate_disable();
-	if (unlikely(__this_cpu_inc_return(bpf_task_storage_busy) != 1)) {
-		__this_cpu_dec(bpf_task_storage_busy);
+	if (unlikely(this_cpu_inc_return(bpf_task_storage_busy) != 1)) {
+		this_cpu_dec(bpf_task_storage_busy);
 		migrate_enable();
 		return false;
 	}
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 7e64447659f3..eba603cec2c5 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -208,7 +208,7 @@ enum btf_kfunc_hook {
 };
 
 enum {
-	BTF_KFUNC_SET_MAX_CNT = 32,
+	BTF_KFUNC_SET_MAX_CNT = 256,
 	BTF_DTOR_KFUNC_MAX_CNT = 256,
 };
 
@@ -818,6 +818,7 @@ const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id)
 		return NULL;
 	return btf->types[type_id];
 }
+EXPORT_SYMBOL_GPL(btf_type_by_id);
 
 /*
  * Regular int is not a bit field and it must be either
@@ -1396,7 +1397,6 @@ __printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,
 						   const char *fmt, ...)
 {
 	struct bpf_verifier_log *log = &env->log;
-	u8 kind = BTF_INFO_KIND(t->info);
 	struct btf *btf = env->btf;
 	va_list args;
 
@@ -1412,7 +1412,7 @@ __printf(4, 5) static void __btf_verifier_log_type(struct btf_verifier_env *env,
 
 	__btf_verifier_log(log, "[%u] %s %s%s",
 			   env->log_type_id,
-			   btf_kind_str[kind],
+			   btf_type_str(t),
 			   __btf_name_by_offset(btf, t->name_off),
 			   log_details ? " " : "");
 
@@ -3128,7 +3128,7 @@ static int btf_struct_resolve(struct btf_verifier_env *env,
 	if (v->next_member) {
 		const struct btf_type *last_member_type;
 		const struct btf_member *last_member;
-		u16 last_member_type_id;
+		u32 last_member_type_id;
 
 		last_member = btf_type_member(v->t) + v->next_member - 1;
 		last_member_type_id = last_member->type;
@@ -4854,7 +4854,6 @@ static int btf_parse_hdr(struct btf_verifier_env *env)
 	u32 hdr_len, hdr_copy, btf_data_size;
 	const struct btf_header *hdr;
 	struct btf *btf;
-	int err;
 
 	btf = env->btf;
 	btf_data_size = btf->data_size;
@@ -4911,11 +4910,7 @@ static int btf_parse_hdr(struct btf_verifier_env *env)
 		return -EINVAL;
 	}
 
-	err = btf_check_sec_info(env, btf_data_size);
-	if (err)
-		return err;
-
-	return 0;
+	return btf_check_sec_info(env, btf_data_size);
 }
 
 static int btf_check_type_tags(struct btf_verifier_env *env,
@@ -5328,6 +5323,34 @@ static bool is_int_ptr(struct btf *btf, const struct btf_type *t)
 	return btf_type_is_int(t);
 }
 
+static u32 get_ctx_arg_idx(struct btf *btf, const struct btf_type *func_proto,
+			   int off)
+{
+	const struct btf_param *args;
+	const struct btf_type *t;
+	u32 offset = 0, nr_args;
+	int i;
+
+	if (!func_proto)
+		return off / 8;
+
+	nr_args = btf_type_vlen(func_proto);
+	args = (const struct btf_param *)(func_proto + 1);
+	for (i = 0; i < nr_args; i++) {
+		t = btf_type_skip_modifiers(btf, args[i].type, NULL);
+		offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8);
+		if (off < offset)
+			return i;
+	}
+
+	t = btf_type_skip_modifiers(btf, func_proto->type, NULL);
+	offset += btf_type_is_ptr(t) ? 8 : roundup(t->size, 8);
+	if (off < offset)
+		return nr_args;
+
+	return nr_args + 1;
+}
+
 bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 		    const struct bpf_prog *prog,
 		    struct bpf_insn_access_aux *info)
@@ -5347,7 +5370,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 			tname, off);
 		return false;
 	}
-	arg = off / 8;
+	arg = get_ctx_arg_idx(btf, t, off);
 	args = (const struct btf_param *)(t + 1);
 	/* if (t == NULL) Fall back to default BPF prog with
 	 * MAX_BPF_FUNC_REG_ARGS u64 arguments.
@@ -5398,7 +5421,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 			if (!btf_type_is_small_int(t)) {
 				bpf_log(log,
 					"ret type %s not allowed for fmod_ret\n",
-					btf_kind_str[BTF_INFO_KIND(t->info)]);
+					btf_type_str(t));
 				return false;
 			}
 			break;
@@ -5417,7 +5440,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 	/* skip modifiers */
 	while (btf_type_is_modifier(t))
 		t = btf_type_by_id(btf, t->type);
-	if (btf_type_is_small_int(t) || btf_is_any_enum(t))
+	if (btf_type_is_small_int(t) || btf_is_any_enum(t) || __btf_type_is_struct(t))
 		/* accessing a scalar */
 		return true;
 	if (!btf_type_is_ptr(t)) {
@@ -5425,7 +5448,7 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 			"func '%s' arg%d '%s' has type %s. Only pointer access is allowed\n",
 			tname, arg,
 			__btf_name_by_offset(btf, t->name_off),
-			btf_kind_str[BTF_INFO_KIND(t->info)]);
+			btf_type_str(t));
 		return false;
 	}
 
@@ -5509,11 +5532,11 @@ bool btf_ctx_access(int off, int size, enum bpf_access_type type,
 	if (!btf_type_is_struct(t)) {
 		bpf_log(log,
 			"func '%s' arg%d type %s is not a struct\n",
-			tname, arg, btf_kind_str[BTF_INFO_KIND(t->info)]);
+			tname, arg, btf_type_str(t));
 		return false;
 	}
 	bpf_log(log, "func '%s' arg%d has btf_id %d type %s '%s'\n",
-		tname, arg, info->btf_id, btf_kind_str[BTF_INFO_KIND(t->info)],
+		tname, arg, info->btf_id, btf_type_str(t),
 		__btf_name_by_offset(btf, t->name_off));
 	return true;
 }
@@ -5864,26 +5887,25 @@ again:
 }
 
 static int __get_type_size(struct btf *btf, u32 btf_id,
-			   const struct btf_type **bad_type)
+			   const struct btf_type **ret_type)
 {
 	const struct btf_type *t;
 
+	*ret_type = btf_type_by_id(btf, 0);
 	if (!btf_id)
 		/* void */
 		return 0;
 	t = btf_type_by_id(btf, btf_id);
 	while (t && btf_type_is_modifier(t))
 		t = btf_type_by_id(btf, t->type);
-	if (!t) {
-		*bad_type = btf_type_by_id(btf, 0);
+	if (!t)
 		return -EINVAL;
-	}
+	*ret_type = t;
 	if (btf_type_is_ptr(t))
 		/* kernel size of pointer. Not BPF's size of pointer*/
 		return sizeof(void *);
-	if (btf_type_is_int(t) || btf_is_any_enum(t))
+	if (btf_type_is_int(t) || btf_is_any_enum(t) || __btf_type_is_struct(t))
 		return t->size;
-	*bad_type = t;
 	return -EINVAL;
 }
 
@@ -5902,8 +5924,10 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
 		/* BTF function prototype doesn't match the verifier types.
 		 * Fall back to MAX_BPF_FUNC_REG_ARGS u64 args.
 		 */
-		for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++)
+		for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
 			m->arg_size[i] = 8;
+			m->arg_flags[i] = 0;
+		}
 		m->ret_size = 8;
 		m->nr_args = MAX_BPF_FUNC_REG_ARGS;
 		return 0;
@@ -5917,10 +5941,10 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
 		return -EINVAL;
 	}
 	ret = __get_type_size(btf, func->type, &t);
-	if (ret < 0) {
+	if (ret < 0 || __btf_type_is_struct(t)) {
 		bpf_log(log,
 			"The function %s return type %s is unsupported.\n",
-			tname, btf_kind_str[BTF_INFO_KIND(t->info)]);
+			tname, btf_type_str(t));
 		return -EINVAL;
 	}
 	m->ret_size = ret;
@@ -5933,10 +5957,12 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
 			return -EINVAL;
 		}
 		ret = __get_type_size(btf, args[i].type, &t);
-		if (ret < 0) {
+
+		/* No support of struct argument size greater than 16 bytes */
+		if (ret < 0 || ret > 16) {
 			bpf_log(log,
 				"The function %s arg%d type %s is unsupported.\n",
-				tname, i, btf_kind_str[BTF_INFO_KIND(t->info)]);
+				tname, i, btf_type_str(t));
 			return -EINVAL;
 		}
 		if (ret == 0) {
@@ -5946,6 +5972,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log,
 			return -EINVAL;
 		}
 		m->arg_size[i] = ret;
+		m->arg_flags[i] = __btf_type_is_struct(t) ? BTF_FMODEL_STRUCT_ARG : 0;
 	}
 	m->nr_args = nargs;
 	return 0;
@@ -6167,14 +6194,41 @@ static bool is_kfunc_arg_mem_size(const struct btf *btf,
 	return true;
 }
 
+static bool btf_is_kfunc_arg_mem_size(const struct btf *btf,
+				      const struct btf_param *arg,
+				      const struct bpf_reg_state *reg,
+				      const char *name)
+{
+	int len, target_len = strlen(name);
+	const struct btf_type *t;
+	const char *param_name;
+
+	t = btf_type_skip_modifiers(btf, arg->type, NULL);
+	if (!btf_type_is_scalar(t) || reg->type != SCALAR_VALUE)
+		return false;
+
+	param_name = btf_name_by_offset(btf, arg->name_off);
+	if (str_is_empty(param_name))
+		return false;
+	len = strlen(param_name);
+	if (len != target_len)
+		return false;
+	if (strcmp(param_name, name))
+		return false;
+
+	return true;
+}
+
 static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 				    const struct btf *btf, u32 func_id,
 				    struct bpf_reg_state *regs,
 				    bool ptr_to_mem_ok,
-				    u32 kfunc_flags)
+				    struct bpf_kfunc_arg_meta *kfunc_meta,
+				    bool processing_call)
 {
 	enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
-	bool rel = false, kptr_get = false, trusted_arg = false;
+	bool rel = false, kptr_get = false, trusted_args = false;
+	bool sleepable = false;
 	struct bpf_verifier_log *log = &env->log;
 	u32 i, nargs, ref_id, ref_obj_id = 0;
 	bool is_kfunc = btf_is_kernel(btf);
@@ -6207,11 +6261,12 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 		return -EINVAL;
 	}
 
-	if (is_kfunc) {
+	if (is_kfunc && kfunc_meta) {
 		/* Only kfunc can be release func */
-		rel = kfunc_flags & KF_RELEASE;
-		kptr_get = kfunc_flags & KF_KPTR_GET;
-		trusted_arg = kfunc_flags & KF_TRUSTED_ARGS;
+		rel = kfunc_meta->flags & KF_RELEASE;
+		kptr_get = kfunc_meta->flags & KF_KPTR_GET;
+		trusted_args = kfunc_meta->flags & KF_TRUSTED_ARGS;
+		sleepable = kfunc_meta->flags & KF_SLEEPABLE;
 	}
 
 	/* check that BTF function arguments match actual types that the
@@ -6221,9 +6276,42 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 		enum bpf_arg_type arg_type = ARG_DONTCARE;
 		u32 regno = i + 1;
 		struct bpf_reg_state *reg = &regs[regno];
+		bool obj_ptr = false;
 
 		t = btf_type_skip_modifiers(btf, args[i].type, NULL);
 		if (btf_type_is_scalar(t)) {
+			if (is_kfunc && kfunc_meta) {
+				bool is_buf_size = false;
+
+				/* check for any const scalar parameter of name "rdonly_buf_size"
+				 * or "rdwr_buf_size"
+				 */
+				if (btf_is_kfunc_arg_mem_size(btf, &args[i], reg,
+							      "rdonly_buf_size")) {
+					kfunc_meta->r0_rdonly = true;
+					is_buf_size = true;
+				} else if (btf_is_kfunc_arg_mem_size(btf, &args[i], reg,
+								     "rdwr_buf_size"))
+					is_buf_size = true;
+
+				if (is_buf_size) {
+					if (kfunc_meta->r0_size) {
+						bpf_log(log, "2 or more rdonly/rdwr_buf_size parameters for kfunc");
+						return -EINVAL;
+					}
+
+					if (!tnum_is_const(reg->var_off)) {
+						bpf_log(log, "R%d is not a const\n", regno);
+						return -EINVAL;
+					}
+
+					kfunc_meta->r0_size = reg->var_off.value;
+					ret = mark_chain_precision(env, regno);
+					if (ret)
+						return ret;
+				}
+			}
+
 			if (reg->type == SCALAR_VALUE)
 				continue;
 			bpf_log(log, "R%d is not a scalar\n", regno);
@@ -6236,10 +6324,17 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 			return -EINVAL;
 		}
 
+		/* These register types have special constraints wrt ref_obj_id
+		 * and offset checks. The rest of trusted args don't.
+		 */
+		obj_ptr = reg->type == PTR_TO_CTX || reg->type == PTR_TO_BTF_ID ||
+			  reg2btf_ids[base_type(reg->type)];
+
 		/* Check if argument must be a referenced pointer, args + i has
 		 * been verified to be a pointer (after skipping modifiers).
+		 * PTR_TO_CTX is ok without having non-zero ref_obj_id.
 		 */
-		if (is_kfunc && trusted_arg && !reg->ref_obj_id) {
+		if (is_kfunc && trusted_args && (obj_ptr && reg->type != PTR_TO_CTX) && !reg->ref_obj_id) {
 			bpf_log(log, "R%d must be referenced\n", regno);
 			return -EINVAL;
 		}
@@ -6248,12 +6343,23 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 		ref_tname = btf_name_by_offset(btf, ref_t->name_off);
 
 		/* Trusted args have the same offset checks as release arguments */
-		if (trusted_arg || (rel && reg->ref_obj_id))
+		if ((trusted_args && obj_ptr) || (rel && reg->ref_obj_id))
 			arg_type |= OBJ_RELEASE;
 		ret = check_func_arg_reg_off(env, reg, regno, arg_type);
 		if (ret < 0)
 			return ret;
 
+		if (is_kfunc && reg->ref_obj_id) {
+			/* Ensure only one argument is referenced PTR_TO_BTF_ID */
+			if (ref_obj_id) {
+				bpf_log(log, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
+					regno, reg->ref_obj_id, ref_obj_id);
+				return -EFAULT;
+			}
+			ref_regno = regno;
+			ref_obj_id = reg->ref_obj_id;
+		}
+
 		/* kptr_get is only true for kfunc */
 		if (i == 0 && kptr_get) {
 			struct bpf_map_value_off_desc *off_desc;
@@ -6326,16 +6432,6 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 			if (reg->type == PTR_TO_BTF_ID) {
 				reg_btf = reg->btf;
 				reg_ref_id = reg->btf_id;
-				/* Ensure only one argument is referenced PTR_TO_BTF_ID */
-				if (reg->ref_obj_id) {
-					if (ref_obj_id) {
-						bpf_log(log, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
-							regno, reg->ref_obj_id, ref_obj_id);
-						return -EFAULT;
-					}
-					ref_regno = regno;
-					ref_obj_id = reg->ref_obj_id;
-				}
 			} else {
 				reg_btf = btf_vmlinux;
 				reg_ref_id = *reg2btf_ids[base_type(reg->type)];
@@ -6347,7 +6443,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 							   reg_ref_t->name_off);
 			if (!btf_struct_ids_match(log, reg_btf, reg_ref_id,
 						  reg->off, btf, ref_id,
-						  trusted_arg || (rel && reg->ref_obj_id))) {
+						  trusted_args || (rel && reg->ref_obj_id))) {
 				bpf_log(log, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
 					func_name, i,
 					btf_type_str(ref_t), ref_tname,
@@ -6355,21 +6451,26 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 					reg_ref_tname);
 				return -EINVAL;
 			}
-		} else if (ptr_to_mem_ok) {
+		} else if (ptr_to_mem_ok && processing_call) {
 			const struct btf_type *resolve_ret;
 			u32 type_size;
 
 			if (is_kfunc) {
 				bool arg_mem_size = i + 1 < nargs && is_kfunc_arg_mem_size(btf, &args[i + 1], &regs[regno + 1]);
+				bool arg_dynptr = btf_type_is_struct(ref_t) &&
+						  !strcmp(ref_tname,
+							  stringify_struct(bpf_dynptr_kern));
 
 				/* Permit pointer to mem, but only when argument
 				 * type is pointer to scalar, or struct composed
 				 * (recursively) of scalars.
 				 * When arg_mem_size is true, the pointer can be
 				 * void *.
+				 * Also permit initialized local dynamic pointers.
 				 */
 				if (!btf_type_is_scalar(ref_t) &&
 				    !__btf_type_is_scalar_struct(log, btf, ref_t, 0) &&
+				    !arg_dynptr &&
 				    (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) {
 					bpf_log(log,
 						"arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n",
@@ -6377,6 +6478,34 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 					return -EINVAL;
 				}
 
+				if (arg_dynptr) {
+					if (reg->type != PTR_TO_STACK) {
+						bpf_log(log, "arg#%d pointer type %s %s not to stack\n",
+							i, btf_type_str(ref_t),
+							ref_tname);
+						return -EINVAL;
+					}
+
+					if (!is_dynptr_reg_valid_init(env, reg)) {
+						bpf_log(log,
+							"arg#%d pointer type %s %s must be valid and initialized\n",
+							i, btf_type_str(ref_t),
+							ref_tname);
+						return -EINVAL;
+					}
+
+					if (!is_dynptr_type_expected(env, reg,
+							ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL)) {
+						bpf_log(log,
+							"arg#%d pointer type %s %s points to unsupported dynamic pointer type\n",
+							i, btf_type_str(ref_t),
+							ref_tname);
+						return -EINVAL;
+					}
+
+					continue;
+				}
+
 				/* Check for mem, len pair */
 				if (arg_mem_size) {
 					if (check_kfunc_mem_size_reg(env, &regs[regno + 1], regno + 1)) {
@@ -6419,11 +6548,21 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env,
 			func_name);
 		return -EINVAL;
 	}
+
+	if (sleepable && !env->prog->aux->sleepable) {
+		bpf_log(log, "kernel function %s is sleepable but the program is not\n",
+			func_name);
+		return -EINVAL;
+	}
+
+	if (kfunc_meta && ref_obj_id)
+		kfunc_meta->ref_obj_id = ref_obj_id;
+
 	/* returns argument register number > 0 in case of reference release kfunc */
 	return rel ? ref_regno : 0;
 }
 
-/* Compare BTF of a function with given bpf_reg_state.
+/* Compare BTF of a function declaration with given bpf_reg_state.
  * Returns:
  * EFAULT - there is a verifier bug. Abort verification.
  * EINVAL - there is a type mismatch or BTF is not available.
@@ -6450,7 +6589,50 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
 		return -EINVAL;
 
 	is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
-	err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, 0);
+	err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, NULL, false);
+
+	/* Compiler optimizations can remove arguments from static functions
+	 * or mismatched type can be passed into a global function.
+	 * In such cases mark the function as unreliable from BTF point of view.
+	 */
+	if (err)
+		prog->aux->func_info_aux[subprog].unreliable = true;
+	return err;
+}
+
+/* Compare BTF of a function call with given bpf_reg_state.
+ * Returns:
+ * EFAULT - there is a verifier bug. Abort verification.
+ * EINVAL - there is a type mismatch or BTF is not available.
+ * 0 - BTF matches with what bpf_reg_state expects.
+ * Only PTR_TO_CTX and SCALAR_VALUE states are recognized.
+ *
+ * NOTE: the code is duplicated from btf_check_subprog_arg_match()
+ * because btf_check_func_arg_match() is still doing both. Once that
+ * function is split in 2, we can call from here btf_check_subprog_arg_match()
+ * first, and then treat the calling part in a new code path.
+ */
+int btf_check_subprog_call(struct bpf_verifier_env *env, int subprog,
+			   struct bpf_reg_state *regs)
+{
+	struct bpf_prog *prog = env->prog;
+	struct btf *btf = prog->aux->btf;
+	bool is_global;
+	u32 btf_id;
+	int err;
+
+	if (!prog->aux->func_info)
+		return -EINVAL;
+
+	btf_id = prog->aux->func_info[subprog].type_id;
+	if (!btf_id)
+		return -EFAULT;
+
+	if (prog->aux->func_info_aux[subprog].unreliable)
+		return -EINVAL;
+
+	is_global = prog->aux->func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
+	err = btf_check_func_arg_match(env, btf, btf_id, regs, is_global, NULL, true);
 
 	/* Compiler optimizations can remove arguments from static functions
 	 * or mismatched type can be passed into a global function.
@@ -6464,9 +6646,9 @@ int btf_check_subprog_arg_match(struct bpf_verifier_env *env, int subprog,
 int btf_check_kfunc_arg_match(struct bpf_verifier_env *env,
 			      const struct btf *btf, u32 func_id,
 			      struct bpf_reg_state *regs,
-			      u32 kfunc_flags)
+			      struct bpf_kfunc_arg_meta *meta)
 {
-	return btf_check_func_arg_match(env, btf, func_id, regs, true, kfunc_flags);
+	return btf_check_func_arg_match(env, btf, func_id, regs, true, meta, true);
 }
 
 /* Convert BTF of a function into bpf_reg_state if possible
@@ -6580,7 +6762,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog,
 			continue;
 		}
 		bpf_log(log, "Arg#%d type %s in %s() is not supported yet.\n",
-			i, btf_kind_str[BTF_INFO_KIND(t->info)], tname);
+			i, btf_type_str(t), tname);
 		return -EINVAL;
 	}
 	return 0;
@@ -7235,6 +7417,7 @@ static int bpf_prog_type_to_kfunc_hook(enum bpf_prog_type prog_type)
 	case BPF_PROG_TYPE_STRUCT_OPS:
 		return BTF_KFUNC_HOOK_STRUCT_OPS;
 	case BPF_PROG_TYPE_TRACING:
+	case BPF_PROG_TYPE_LSM:
 		return BTF_KFUNC_HOOK_TRACING;
 	case BPF_PROG_TYPE_SYSCALL:
 		return BTF_KFUNC_HOOK_SYSCALL;
diff --git a/kernel/bpf/cgroup.c b/kernel/bpf/cgroup.c
index 4a400cd63731..bf2fdb33fb31 100644
--- a/kernel/bpf/cgroup.c
+++ b/kernel/bpf/cgroup.c
@@ -1020,6 +1020,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
 			      union bpf_attr __user *uattr)
 {
 	__u32 __user *prog_attach_flags = u64_to_user_ptr(attr->query.prog_attach_flags);
+	bool effective_query = attr->query.query_flags & BPF_F_QUERY_EFFECTIVE;
 	__u32 __user *prog_ids = u64_to_user_ptr(attr->query.prog_ids);
 	enum bpf_attach_type type = attr->query.attach_type;
 	enum cgroup_bpf_attach_type from_atype, to_atype;
@@ -1029,8 +1030,12 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
 	int total_cnt = 0;
 	u32 flags;
 
+	if (effective_query && prog_attach_flags)
+		return -EINVAL;
+
 	if (type == BPF_LSM_CGROUP) {
-		if (attr->query.prog_cnt && prog_ids && !prog_attach_flags)
+		if (!effective_query && attr->query.prog_cnt &&
+		    prog_ids && !prog_attach_flags)
 			return -EINVAL;
 
 		from_atype = CGROUP_LSM_START;
@@ -1045,7 +1050,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
 	}
 
 	for (atype = from_atype; atype <= to_atype; atype++) {
-		if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) {
+		if (effective_query) {
 			effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
 							      lockdep_is_held(&cgroup_mutex));
 			total_cnt += bpf_prog_array_length(effective);
@@ -1054,6 +1059,8 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
 		}
 	}
 
+	/* always output uattr->query.attach_flags as 0 during effective query */
+	flags = effective_query ? 0 : flags;
 	if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags)))
 		return -EFAULT;
 	if (copy_to_user(&uattr->query.prog_cnt, &total_cnt, sizeof(total_cnt)))
@@ -1068,7 +1075,7 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
 	}
 
 	for (atype = from_atype; atype <= to_atype && total_cnt; atype++) {
-		if (attr->query.query_flags & BPF_F_QUERY_EFFECTIVE) {
+		if (effective_query) {
 			effective = rcu_dereference_protected(cgrp->bpf.effective[atype],
 							      lockdep_is_held(&cgroup_mutex));
 			cnt = min_t(int, bpf_prog_array_length(effective), total_cnt);
@@ -1090,15 +1097,16 @@ static int __cgroup_bpf_query(struct cgroup *cgrp, const union bpf_attr *attr,
 				if (++i == cnt)
 					break;
 			}
-		}
 
-		if (prog_attach_flags) {
-			flags = cgrp->bpf.flags[atype];
+			if (prog_attach_flags) {
+				flags = cgrp->bpf.flags[atype];
 
-			for (i = 0; i < cnt; i++)
-				if (copy_to_user(prog_attach_flags + i, &flags, sizeof(flags)))
-					return -EFAULT;
-			prog_attach_flags += cnt;
+				for (i = 0; i < cnt; i++)
+					if (copy_to_user(prog_attach_flags + i,
+							 &flags, sizeof(flags)))
+						return -EFAULT;
+				prog_attach_flags += cnt;
+			}
 		}
 
 		prog_ids += cnt;
@@ -1529,6 +1537,37 @@ int __cgroup_bpf_check_dev_permission(short dev_type, u32 major, u32 minor,
 	return ret;
 }
 
+BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
+{
+	/* flags argument is not used now,
+	 * but provides an ability to extend the API.
+	 * verifier checks that its value is correct.
+	 */
+	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
+	struct bpf_cgroup_storage *storage;
+	struct bpf_cg_run_ctx *ctx;
+	void *ptr;
+
+	/* get current cgroup storage from BPF run context */
+	ctx = container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);
+	storage = ctx->prog_item->cgroup_storage[stype];
+
+	if (stype == BPF_CGROUP_STORAGE_SHARED)
+		ptr = &READ_ONCE(storage->buf)->data[0];
+	else
+		ptr = this_cpu_ptr(storage->percpu_buf);
+
+	return (unsigned long)ptr;
+}
+
+const struct bpf_func_proto bpf_get_local_storage_proto = {
+	.func		= bpf_get_local_storage,
+	.gpl_only	= false,
+	.ret_type	= RET_PTR_TO_MAP_VALUE,
+	.arg1_type	= ARG_CONST_MAP_PTR,
+	.arg2_type	= ARG_ANYTHING,
+};
+
 BPF_CALL_0(bpf_get_retval)
 {
 	struct bpf_cg_run_ctx *ctx =
@@ -1560,32 +1599,26 @@ const struct bpf_func_proto bpf_set_retval_proto = {
 };
 
 static const struct bpf_func_proto *
-cgroup_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
+	const struct bpf_func_proto *func_proto;
+
+	func_proto = cgroup_common_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
+	func_proto = cgroup_current_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
 	switch (func_id) {
-	case BPF_FUNC_get_current_uid_gid:
-		return &bpf_get_current_uid_gid_proto;
-	case BPF_FUNC_get_local_storage:
-		return &bpf_get_local_storage_proto;
-	case BPF_FUNC_get_current_cgroup_id:
-		return &bpf_get_current_cgroup_id_proto;
 	case BPF_FUNC_perf_event_output:
 		return &bpf_event_output_data_proto;
-	case BPF_FUNC_get_retval:
-		return &bpf_get_retval_proto;
-	case BPF_FUNC_set_retval:
-		return &bpf_set_retval_proto;
 	default:
 		return bpf_base_func_proto(func_id);
 	}
 }
 
-static const struct bpf_func_proto *
-cgroup_dev_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
-{
-	return cgroup_base_func_proto(func_id, prog);
-}
-
 static bool cgroup_dev_is_valid_access(int off, int size,
 				       enum bpf_access_type type,
 				       const struct bpf_prog *prog,
@@ -2098,11 +2131,17 @@ static const struct bpf_func_proto bpf_sysctl_set_new_value_proto = {
 static const struct bpf_func_proto *
 sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
+	const struct bpf_func_proto *func_proto;
+
+	func_proto = cgroup_common_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
+	func_proto = cgroup_current_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
 	switch (func_id) {
-	case BPF_FUNC_strtol:
-		return &bpf_strtol_proto;
-	case BPF_FUNC_strtoul:
-		return &bpf_strtoul_proto;
 	case BPF_FUNC_sysctl_get_name:
 		return &bpf_sysctl_get_name_proto;
 	case BPF_FUNC_sysctl_get_current_value:
@@ -2113,8 +2152,10 @@ sysctl_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_sysctl_set_new_value_proto;
 	case BPF_FUNC_ktime_get_coarse_ns:
 		return &bpf_ktime_get_coarse_ns_proto;
+	case BPF_FUNC_perf_event_output:
+		return &bpf_event_output_data_proto;
 	default:
-		return cgroup_base_func_proto(func_id, prog);
+		return bpf_base_func_proto(func_id);
 	}
 }
 
@@ -2235,6 +2276,16 @@ static const struct bpf_func_proto bpf_get_netns_cookie_sockopt_proto = {
 static const struct bpf_func_proto *
 cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
+	const struct bpf_func_proto *func_proto;
+
+	func_proto = cgroup_common_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
+	func_proto = cgroup_current_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
 	switch (func_id) {
 #ifdef CONFIG_NET
 	case BPF_FUNC_get_netns_cookie:
@@ -2256,8 +2307,10 @@ cg_sockopt_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 	case BPF_FUNC_tcp_sock:
 		return &bpf_tcp_sock_proto;
 #endif
+	case BPF_FUNC_perf_event_output:
+		return &bpf_event_output_data_proto;
 	default:
-		return cgroup_base_func_proto(func_id, prog);
+		return bpf_base_func_proto(func_id);
 	}
 }
 
@@ -2422,3 +2475,69 @@ const struct bpf_verifier_ops cg_sockopt_verifier_ops = {
 
 const struct bpf_prog_ops cg_sockopt_prog_ops = {
 };
+
+/* Common helpers for cgroup hooks. */
+const struct bpf_func_proto *
+cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+{
+	switch (func_id) {
+	case BPF_FUNC_get_local_storage:
+		return &bpf_get_local_storage_proto;
+	case BPF_FUNC_get_retval:
+		switch (prog->expected_attach_type) {
+		case BPF_CGROUP_INET_INGRESS:
+		case BPF_CGROUP_INET_EGRESS:
+		case BPF_CGROUP_SOCK_OPS:
+		case BPF_CGROUP_UDP4_RECVMSG:
+		case BPF_CGROUP_UDP6_RECVMSG:
+		case BPF_CGROUP_INET4_GETPEERNAME:
+		case BPF_CGROUP_INET6_GETPEERNAME:
+		case BPF_CGROUP_INET4_GETSOCKNAME:
+		case BPF_CGROUP_INET6_GETSOCKNAME:
+			return NULL;
+		default:
+			return &bpf_get_retval_proto;
+		}
+	case BPF_FUNC_set_retval:
+		switch (prog->expected_attach_type) {
+		case BPF_CGROUP_INET_INGRESS:
+		case BPF_CGROUP_INET_EGRESS:
+		case BPF_CGROUP_SOCK_OPS:
+		case BPF_CGROUP_UDP4_RECVMSG:
+		case BPF_CGROUP_UDP6_RECVMSG:
+		case BPF_CGROUP_INET4_GETPEERNAME:
+		case BPF_CGROUP_INET6_GETPEERNAME:
+		case BPF_CGROUP_INET4_GETSOCKNAME:
+		case BPF_CGROUP_INET6_GETSOCKNAME:
+			return NULL;
+		default:
+			return &bpf_set_retval_proto;
+		}
+	default:
+		return NULL;
+	}
+}
+
+/* Common helpers for cgroup hooks with valid process context. */
+const struct bpf_func_proto *
+cgroup_current_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
+{
+	switch (func_id) {
+	case BPF_FUNC_get_current_uid_gid:
+		return &bpf_get_current_uid_gid_proto;
+	case BPF_FUNC_get_current_pid_tgid:
+		return &bpf_get_current_pid_tgid_proto;
+	case BPF_FUNC_get_current_comm:
+		return &bpf_get_current_comm_proto;
+	case BPF_FUNC_get_current_cgroup_id:
+		return &bpf_get_current_cgroup_id_proto;
+	case BPF_FUNC_get_current_ancestor_cgroup_id:
+		return &bpf_get_current_ancestor_cgroup_id_proto;
+#ifdef CONFIG_CGROUP_NET_CLASSID
+	case BPF_FUNC_get_cgroup_classid:
+		return &bpf_get_cgroup_classid_curr_proto;
+#endif
+	default:
+		return NULL;
+	}
+}
diff --git a/kernel/bpf/cgroup_iter.c b/kernel/bpf/cgroup_iter.c
new file mode 100644
index 000000000000..0d200a993489
--- /dev/null
+++ b/kernel/bpf/cgroup_iter.c
@@ -0,0 +1,282 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2022 Google */
+#include <linux/bpf.h>
+#include <linux/btf_ids.h>
+#include <linux/cgroup.h>
+#include <linux/kernel.h>
+#include <linux/seq_file.h>
+
+#include "../cgroup/cgroup-internal.h"  /* cgroup_mutex and cgroup_is_dead */
+
+/* cgroup_iter provides four modes of traversal to the cgroup hierarchy.
+ *
+ *  1. Walk the descendants of a cgroup in pre-order.
+ *  2. Walk the descendants of a cgroup in post-order.
+ *  3. Walk the ancestors of a cgroup.
+ *  4. Show the given cgroup only.
+ *
+ * For walking descendants, cgroup_iter can walk in either pre-order or
+ * post-order. For walking ancestors, the iter walks up from a cgroup to
+ * the root.
+ *
+ * The iter program can terminate the walk early by returning 1. Walk
+ * continues if prog returns 0.
+ *
+ * The prog can check (seq->num == 0) to determine whether this is
+ * the first element. The prog may also be passed a NULL cgroup,
+ * which means the walk has completed and the prog has a chance to
+ * do post-processing, such as outputting an epilogue.
+ *
+ * Note: the iter_prog is called with cgroup_mutex held.
+ *
+ * Currently only one session is supported, which means, depending on the
+ * volume of data bpf program intends to send to user space, the number
+ * of cgroups that can be walked is limited. For example, given the current
+ * buffer size is 8 * PAGE_SIZE, if the program sends 64B data for each
+ * cgroup, assuming PAGE_SIZE is 4kb, the total number of cgroups that can
+ * be walked is 512. This is a limitation of cgroup_iter. If the output data
+ * is larger than the kernel buffer size, after all data in the kernel buffer
+ * is consumed by user space, the subsequent read() syscall will signal
+ * EOPNOTSUPP. In order to work around, the user may have to update their
+ * program to reduce the volume of data sent to output. For example, skip
+ * some uninteresting cgroups.
+ */
+
+struct bpf_iter__cgroup {
+	__bpf_md_ptr(struct bpf_iter_meta *, meta);
+	__bpf_md_ptr(struct cgroup *, cgroup);
+};
+
+struct cgroup_iter_priv {
+	struct cgroup_subsys_state *start_css;
+	bool visited_all;
+	bool terminate;
+	int order;
+};
+
+static void *cgroup_iter_seq_start(struct seq_file *seq, loff_t *pos)
+{
+	struct cgroup_iter_priv *p = seq->private;
+
+	mutex_lock(&cgroup_mutex);
+
+	/* cgroup_iter doesn't support read across multiple sessions. */
+	if (*pos > 0) {
+		if (p->visited_all)
+			return NULL;
+
+		/* Haven't visited all, but because cgroup_mutex has dropped,
+		 * return -EOPNOTSUPP to indicate incomplete iteration.
+		 */
+		return ERR_PTR(-EOPNOTSUPP);
+	}
+
+	++*pos;
+	p->terminate = false;
+	p->visited_all = false;
+	if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE)
+		return css_next_descendant_pre(NULL, p->start_css);
+	else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST)
+		return css_next_descendant_post(NULL, p->start_css);
+	else /* BPF_CGROUP_ITER_SELF_ONLY and BPF_CGROUP_ITER_ANCESTORS_UP */
+		return p->start_css;
+}
+
+static int __cgroup_iter_seq_show(struct seq_file *seq,
+				  struct cgroup_subsys_state *css, int in_stop);
+
+static void cgroup_iter_seq_stop(struct seq_file *seq, void *v)
+{
+	struct cgroup_iter_priv *p = seq->private;
+
+	mutex_unlock(&cgroup_mutex);
+
+	/* pass NULL to the prog for post-processing */
+	if (!v) {
+		__cgroup_iter_seq_show(seq, NULL, true);
+		p->visited_all = true;
+	}
+}
+
+static void *cgroup_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+	struct cgroup_subsys_state *curr = (struct cgroup_subsys_state *)v;
+	struct cgroup_iter_priv *p = seq->private;
+
+	++*pos;
+	if (p->terminate)
+		return NULL;
+
+	if (p->order == BPF_CGROUP_ITER_DESCENDANTS_PRE)
+		return css_next_descendant_pre(curr, p->start_css);
+	else if (p->order == BPF_CGROUP_ITER_DESCENDANTS_POST)
+		return css_next_descendant_post(curr, p->start_css);
+	else if (p->order == BPF_CGROUP_ITER_ANCESTORS_UP)
+		return curr->parent;
+	else  /* BPF_CGROUP_ITER_SELF_ONLY */
+		return NULL;
+}
+
+static int __cgroup_iter_seq_show(struct seq_file *seq,
+				  struct cgroup_subsys_state *css, int in_stop)
+{
+	struct cgroup_iter_priv *p = seq->private;
+	struct bpf_iter__cgroup ctx;
+	struct bpf_iter_meta meta;
+	struct bpf_prog *prog;
+	int ret = 0;
+
+	/* cgroup is dead, skip this element */
+	if (css && cgroup_is_dead(css->cgroup))
+		return 0;
+
+	ctx.meta = &meta;
+	ctx.cgroup = css ? css->cgroup : NULL;
+	meta.seq = seq;
+	prog = bpf_iter_get_info(&meta, in_stop);
+	if (prog)
+		ret = bpf_iter_run_prog(prog, &ctx);
+
+	/* if prog returns > 0, terminate after this element. */
+	if (ret != 0)
+		p->terminate = true;
+
+	return 0;
+}
+
+static int cgroup_iter_seq_show(struct seq_file *seq, void *v)
+{
+	return __cgroup_iter_seq_show(seq, (struct cgroup_subsys_state *)v,
+				      false);
+}
+
+static const struct seq_operations cgroup_iter_seq_ops = {
+	.start  = cgroup_iter_seq_start,
+	.next   = cgroup_iter_seq_next,
+	.stop   = cgroup_iter_seq_stop,
+	.show   = cgroup_iter_seq_show,
+};
+
+BTF_ID_LIST_SINGLE(bpf_cgroup_btf_id, struct, cgroup)
+
+static int cgroup_iter_seq_init(void *priv, struct bpf_iter_aux_info *aux)
+{
+	struct cgroup_iter_priv *p = (struct cgroup_iter_priv *)priv;
+	struct cgroup *cgrp = aux->cgroup.start;
+
+	p->start_css = &cgrp->self;
+	p->terminate = false;
+	p->visited_all = false;
+	p->order = aux->cgroup.order;
+	return 0;
+}
+
+static const struct bpf_iter_seq_info cgroup_iter_seq_info = {
+	.seq_ops		= &cgroup_iter_seq_ops,
+	.init_seq_private	= cgroup_iter_seq_init,
+	.seq_priv_size		= sizeof(struct cgroup_iter_priv),
+};
+
+static int bpf_iter_attach_cgroup(struct bpf_prog *prog,
+				  union bpf_iter_link_info *linfo,
+				  struct bpf_iter_aux_info *aux)
+{
+	int fd = linfo->cgroup.cgroup_fd;
+	u64 id = linfo->cgroup.cgroup_id;
+	int order = linfo->cgroup.order;
+	struct cgroup *cgrp;
+
+	if (order != BPF_CGROUP_ITER_DESCENDANTS_PRE &&
+	    order != BPF_CGROUP_ITER_DESCENDANTS_POST &&
+	    order != BPF_CGROUP_ITER_ANCESTORS_UP &&
+	    order != BPF_CGROUP_ITER_SELF_ONLY)
+		return -EINVAL;
+
+	if (fd && id)
+		return -EINVAL;
+
+	if (fd)
+		cgrp = cgroup_get_from_fd(fd);
+	else if (id)
+		cgrp = cgroup_get_from_id(id);
+	else /* walk the entire hierarchy by default. */
+		cgrp = cgroup_get_from_path("/");
+
+	if (IS_ERR(cgrp))
+		return PTR_ERR(cgrp);
+
+	aux->cgroup.start = cgrp;
+	aux->cgroup.order = order;
+	return 0;
+}
+
+static void bpf_iter_detach_cgroup(struct bpf_iter_aux_info *aux)
+{
+	cgroup_put(aux->cgroup.start);
+}
+
+static void bpf_iter_cgroup_show_fdinfo(const struct bpf_iter_aux_info *aux,
+					struct seq_file *seq)
+{
+	char *buf;
+
+	buf = kzalloc(PATH_MAX, GFP_KERNEL);
+	if (!buf) {
+		seq_puts(seq, "cgroup_path:\t<unknown>\n");
+		goto show_order;
+	}
+
+	/* If cgroup_path_ns() fails, buf will be an empty string, cgroup_path
+	 * will print nothing.
+	 *
+	 * Path is in the calling process's cgroup namespace.
+	 */
+	cgroup_path_ns(aux->cgroup.start, buf, PATH_MAX,
+		       current->nsproxy->cgroup_ns);
+	seq_printf(seq, "cgroup_path:\t%s\n", buf);
+	kfree(buf);
+
+show_order:
+	if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_PRE)
+		seq_puts(seq, "order: descendants_pre\n");
+	else if (aux->cgroup.order == BPF_CGROUP_ITER_DESCENDANTS_POST)
+		seq_puts(seq, "order: descendants_post\n");
+	else if (aux->cgroup.order == BPF_CGROUP_ITER_ANCESTORS_UP)
+		seq_puts(seq, "order: ancestors_up\n");
+	else /* BPF_CGROUP_ITER_SELF_ONLY */
+		seq_puts(seq, "order: self_only\n");
+}
+
+static int bpf_iter_cgroup_fill_link_info(const struct bpf_iter_aux_info *aux,
+					  struct bpf_link_info *info)
+{
+	info->iter.cgroup.order = aux->cgroup.order;
+	info->iter.cgroup.cgroup_id = cgroup_id(aux->cgroup.start);
+	return 0;
+}
+
+DEFINE_BPF_ITER_FUNC(cgroup, struct bpf_iter_meta *meta,
+		     struct cgroup *cgroup)
+
+static struct bpf_iter_reg bpf_cgroup_reg_info = {
+	.target			= "cgroup",
+	.feature		= BPF_ITER_RESCHED,
+	.attach_target		= bpf_iter_attach_cgroup,
+	.detach_target		= bpf_iter_detach_cgroup,
+	.show_fdinfo		= bpf_iter_cgroup_show_fdinfo,
+	.fill_link_info		= bpf_iter_cgroup_fill_link_info,
+	.ctx_arg_info_size	= 1,
+	.ctx_arg_info		= {
+		{ offsetof(struct bpf_iter__cgroup, cgroup),
+		  PTR_TO_BTF_ID_OR_NULL },
+	},
+	.seq_info		= &cgroup_iter_seq_info,
+};
+
+static int __init bpf_cgroup_iter_init(void)
+{
+	bpf_cgroup_reg_info.ctx_arg_info[0].btf_id = bpf_cgroup_btf_id[0];
+	return bpf_iter_reg_target(&bpf_cgroup_reg_info);
+}
+
+late_initcall(bpf_cgroup_iter_init);
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 3d9eb3ae334c..711fd293b6de 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -825,6 +825,11 @@ struct bpf_prog_pack {
 	unsigned long bitmap[];
 };
 
+void bpf_jit_fill_hole_with_zero(void *area, unsigned int size)
+{
+	memset(area, 0, size);
+}
+
 #define BPF_PROG_SIZE_TO_NBITS(size)	(round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE)
 
 static DEFINE_MUTEX(pack_mutex);
@@ -864,7 +869,7 @@ static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_ins
 	return pack;
 }
 
-static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
+void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns)
 {
 	unsigned int nbits = BPF_PROG_SIZE_TO_NBITS(size);
 	struct bpf_prog_pack *pack;
@@ -905,7 +910,7 @@ out:
 	return ptr;
 }
 
-static void bpf_prog_pack_free(struct bpf_binary_header *hdr)
+void bpf_prog_pack_free(struct bpf_binary_header *hdr)
 {
 	struct bpf_prog_pack *pack = NULL, *tmp;
 	unsigned int nbits;
@@ -2623,6 +2628,7 @@ const struct bpf_func_proto bpf_get_numa_node_id_proto __weak;
 const struct bpf_func_proto bpf_ktime_get_ns_proto __weak;
 const struct bpf_func_proto bpf_ktime_get_boot_ns_proto __weak;
 const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto __weak;
+const struct bpf_func_proto bpf_ktime_get_tai_ns_proto __weak;
 
 const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak;
 const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak;
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index f4860ac756cd..b5ba34ddd4b6 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -97,7 +97,7 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
 	    attr->map_flags & ~BPF_F_NUMA_NODE)
 		return ERR_PTR(-EINVAL);
 
-	cmap = kzalloc(sizeof(*cmap), GFP_USER | __GFP_ACCOUNT);
+	cmap = bpf_map_area_alloc(sizeof(*cmap), NUMA_NO_NODE);
 	if (!cmap)
 		return ERR_PTR(-ENOMEM);
 
@@ -118,7 +118,7 @@ static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
 
 	return &cmap->map;
 free_cmap:
-	kfree(cmap);
+	bpf_map_area_free(cmap);
 	return ERR_PTR(err);
 }
 
@@ -623,7 +623,7 @@ static void cpu_map_free(struct bpf_map *map)
 		__cpu_map_entry_replace(cmap, i, NULL); /* call_rcu */
 	}
 	bpf_map_area_free(cmap->cpu_map);
-	kfree(cmap);
+	bpf_map_area_free(cmap);
 }
 
 /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index a0e02b009487..f9a87dcc5535 100644
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -163,13 +163,13 @@ static struct bpf_map *dev_map_alloc(union bpf_attr *attr)
 	if (!capable(CAP_NET_ADMIN))
 		return ERR_PTR(-EPERM);
 
-	dtab = kzalloc(sizeof(*dtab), GFP_USER | __GFP_ACCOUNT);
+	dtab = bpf_map_area_alloc(sizeof(*dtab), NUMA_NO_NODE);
 	if (!dtab)
 		return ERR_PTR(-ENOMEM);
 
 	err = dev_map_init_map(dtab, attr);
 	if (err) {
-		kfree(dtab);
+		bpf_map_area_free(dtab);
 		return ERR_PTR(err);
 	}
 
@@ -240,7 +240,7 @@ static void dev_map_free(struct bpf_map *map)
 		bpf_map_area_free(dtab->netdev_map);
 	}
 
-	kfree(dtab);
+	bpf_map_area_free(dtab);
 }
 
 static int dev_map_get_next_key(struct bpf_map *map, void *key, void *next_key)
diff --git a/kernel/bpf/dispatcher.c b/kernel/bpf/dispatcher.c
index 2444bd15cc2d..fa64b80b8bca 100644
--- a/kernel/bpf/dispatcher.c
+++ b/kernel/bpf/dispatcher.c
@@ -85,12 +85,12 @@ static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d,
 	return false;
 }
 
-int __weak arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs)
+int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs)
 {
 	return -ENOTSUPP;
 }
 
-static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image)
+static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf)
 {
 	s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0];
 	int i;
@@ -99,12 +99,12 @@ static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image)
 		if (d->progs[i].prog)
 			*ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func;
 	}
-	return arch_prepare_bpf_dispatcher(image, &ips[0], d->num_progs);
+	return arch_prepare_bpf_dispatcher(image, buf, &ips[0], d->num_progs);
 }
 
 static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
 {
-	void *old, *new;
+	void *old, *new, *tmp;
 	u32 noff;
 	int err;
 
@@ -117,8 +117,14 @@ static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs)
 	}
 
 	new = d->num_progs ? d->image + noff : NULL;
+	tmp = d->num_progs ? d->rw_image + noff : NULL;
 	if (new) {
-		if (bpf_dispatcher_prepare(d, new))
+		/* Prepare the dispatcher in d->rw_image. Then use
+		 * bpf_arch_text_copy to update d->image, which is RO+X.
+		 */
+		if (bpf_dispatcher_prepare(d, new, tmp))
+			return;
+		if (IS_ERR(bpf_arch_text_copy(new, tmp, PAGE_SIZE / 2)))
 			return;
 	}
 
@@ -140,9 +146,18 @@ void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from,
 
 	mutex_lock(&d->mutex);
 	if (!d->image) {
-		d->image = bpf_jit_alloc_exec_page();
+		d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero);
 		if (!d->image)
 			goto out;
+		d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE);
+		if (!d->rw_image) {
+			u32 size = PAGE_SIZE;
+
+			bpf_arch_text_copy(d->image, &size, sizeof(size));
+			bpf_prog_pack_free((struct bpf_binary_header *)d->image);
+			d->image = NULL;
+			goto out;
+		}
 		bpf_image_ksym_add(d->image, &d->ksym);
 	}
 
diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c
index 6c530a5e560a..ed3f8a53603b 100644
--- a/kernel/bpf/hashtab.c
+++ b/kernel/bpf/hashtab.c
@@ -14,6 +14,7 @@
 #include "percpu_freelist.h"
 #include "bpf_lru_list.h"
 #include "map_in_map.h"
+#include <linux/bpf_mem_alloc.h>
 
 #define HTAB_CREATE_FLAG_MASK						\
 	(BPF_F_NO_PREALLOC | BPF_F_NO_COMMON_LRU | BPF_F_NUMA_NODE |	\
@@ -67,24 +68,16 @@
  * In theory the BPF locks could be converted to regular spinlocks as well,
  * but the bucket locks and percpu_freelist locks can be taken from
  * arbitrary contexts (perf, kprobes, tracepoints) which are required to be
- * atomic contexts even on RT. These mechanisms require preallocated maps,
- * so there is no need to invoke memory allocations within the lock held
- * sections.
- *
- * BPF maps which need dynamic allocation are only used from (forced)
- * thread context on RT and can therefore use regular spinlocks which in
- * turn allows to invoke memory allocations from the lock held section.
- *
- * On a non RT kernel this distinction is neither possible nor required.
- * spinlock maps to raw_spinlock and the extra code is optimized out by the
- * compiler.
+ * atomic contexts even on RT. Before the introduction of bpf_mem_alloc,
+ * it is only safe to use raw spinlock for preallocated hash map on a RT kernel,
+ * because there is no memory allocation within the lock held sections. However
+ * after hash map was fully converted to use bpf_mem_alloc, there will be
+ * non-synchronous memory allocation for non-preallocated hash map, so it is
+ * safe to always use raw spinlock for bucket lock.
  */
 struct bucket {
 	struct hlist_nulls_head head;
-	union {
-		raw_spinlock_t raw_lock;
-		spinlock_t     lock;
-	};
+	raw_spinlock_t raw_lock;
 };
 
 #define HASHTAB_MAP_LOCK_COUNT 8
@@ -92,6 +85,8 @@ struct bucket {
 
 struct bpf_htab {
 	struct bpf_map map;
+	struct bpf_mem_alloc ma;
+	struct bpf_mem_alloc pcpu_ma;
 	struct bucket *buckets;
 	void *elems;
 	union {
@@ -99,7 +94,12 @@ struct bpf_htab {
 		struct bpf_lru lru;
 	};
 	struct htab_elem *__percpu *extra_elems;
-	atomic_t count;	/* number of elements in this hashtable */
+	/* number of elements in non-preallocated hashtable are kept
+	 * in either pcount or count
+	 */
+	struct percpu_counter pcount;
+	atomic_t count;
+	bool use_percpu_counter;
 	u32 n_buckets;	/* number of hash buckets */
 	u32 elem_size;	/* size of each element in bytes */
 	u32 hashrnd;
@@ -114,14 +114,14 @@ struct htab_elem {
 		struct {
 			void *padding;
 			union {
-				struct bpf_htab *htab;
 				struct pcpu_freelist_node fnode;
 				struct htab_elem *batch_flink;
 			};
 		};
 	};
 	union {
-		struct rcu_head rcu;
+		/* pointer to per-cpu pointer */
+		void *ptr_to_pptr;
 		struct bpf_lru_node lru_node;
 	};
 	u32 hash;
@@ -133,26 +133,15 @@ static inline bool htab_is_prealloc(const struct bpf_htab *htab)
 	return !(htab->map.map_flags & BPF_F_NO_PREALLOC);
 }
 
-static inline bool htab_use_raw_lock(const struct bpf_htab *htab)
-{
-	return (!IS_ENABLED(CONFIG_PREEMPT_RT) || htab_is_prealloc(htab));
-}
-
 static void htab_init_buckets(struct bpf_htab *htab)
 {
 	unsigned int i;
 
 	for (i = 0; i < htab->n_buckets; i++) {
 		INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i);
-		if (htab_use_raw_lock(htab)) {
-			raw_spin_lock_init(&htab->buckets[i].raw_lock);
-			lockdep_set_class(&htab->buckets[i].raw_lock,
-					  &htab->lockdep_key);
-		} else {
-			spin_lock_init(&htab->buckets[i].lock);
-			lockdep_set_class(&htab->buckets[i].lock,
+		raw_spin_lock_init(&htab->buckets[i].raw_lock);
+		lockdep_set_class(&htab->buckets[i].raw_lock,
 					  &htab->lockdep_key);
-		}
 		cond_resched();
 	}
 }
@@ -165,17 +154,14 @@ static inline int htab_lock_bucket(const struct bpf_htab *htab,
 
 	hash = hash & HASHTAB_MAP_LOCK_MASK;
 
-	migrate_disable();
+	preempt_disable();
 	if (unlikely(__this_cpu_inc_return(*(htab->map_locked[hash])) != 1)) {
 		__this_cpu_dec(*(htab->map_locked[hash]));
-		migrate_enable();
+		preempt_enable();
 		return -EBUSY;
 	}
 
-	if (htab_use_raw_lock(htab))
-		raw_spin_lock_irqsave(&b->raw_lock, flags);
-	else
-		spin_lock_irqsave(&b->lock, flags);
+	raw_spin_lock_irqsave(&b->raw_lock, flags);
 	*pflags = flags;
 
 	return 0;
@@ -186,12 +172,9 @@ static inline void htab_unlock_bucket(const struct bpf_htab *htab,
 				      unsigned long flags)
 {
 	hash = hash & HASHTAB_MAP_LOCK_MASK;
-	if (htab_use_raw_lock(htab))
-		raw_spin_unlock_irqrestore(&b->raw_lock, flags);
-	else
-		spin_unlock_irqrestore(&b->lock, flags);
+	raw_spin_unlock_irqrestore(&b->raw_lock, flags);
 	__this_cpu_dec(*(htab->map_locked[hash]));
-	migrate_enable();
+	preempt_enable();
 }
 
 static bool htab_lru_map_delete_node(void *arg, struct bpf_lru_node *node);
@@ -428,8 +411,6 @@ static int htab_map_alloc_check(union bpf_attr *attr)
 	bool zero_seed = (attr->map_flags & BPF_F_ZERO_SEED);
 	int numa_node = bpf_map_attr_numa_node(attr);
 
-	BUILD_BUG_ON(offsetof(struct htab_elem, htab) !=
-		     offsetof(struct htab_elem, hash_node.pprev));
 	BUILD_BUG_ON(offsetof(struct htab_elem, fnode.next) !=
 		     offsetof(struct htab_elem, hash_node.pprev));
 
@@ -491,7 +472,7 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
 	struct bpf_htab *htab;
 	int err, i;
 
-	htab = kzalloc(sizeof(*htab), GFP_USER | __GFP_ACCOUNT);
+	htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE);
 	if (!htab)
 		return ERR_PTR(-ENOMEM);
 
@@ -550,6 +531,29 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
 
 	htab_init_buckets(htab);
 
+/* compute_batch_value() computes batch value as num_online_cpus() * 2
+ * and __percpu_counter_compare() needs
+ * htab->max_entries - cur_number_of_elems to be more than batch * num_online_cpus()
+ * for percpu_counter to be faster than atomic_t. In practice the average bpf
+ * hash map size is 10k, which means that a system with 64 cpus will fill
+ * hashmap to 20% of 10k before percpu_counter becomes ineffective. Therefore
+ * define our own batch count as 32 then 10k hash map can be filled up to 80%:
+ * 10k - 8k > 32 _batch_ * 64 _cpus_
+ * and __percpu_counter_compare() will still be fast. At that point hash map
+ * collisions will dominate its performance anyway. Assume that hash map filled
+ * to 50+% isn't going to be O(1) and use the following formula to choose
+ * between percpu_counter and atomic_t.
+ */
+#define PERCPU_COUNTER_BATCH 32
+	if (attr->max_entries / 2 > num_online_cpus() * PERCPU_COUNTER_BATCH)
+		htab->use_percpu_counter = true;
+
+	if (htab->use_percpu_counter) {
+		err = percpu_counter_init(&htab->pcount, 0, GFP_KERNEL);
+		if (err)
+			goto free_map_locked;
+	}
+
 	if (prealloc) {
 		err = prealloc_init(htab);
 		if (err)
@@ -563,6 +567,16 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
 			if (err)
 				goto free_prealloc;
 		}
+	} else {
+		err = bpf_mem_alloc_init(&htab->ma, htab->elem_size, false);
+		if (err)
+			goto free_map_locked;
+		if (percpu) {
+			err = bpf_mem_alloc_init(&htab->pcpu_ma,
+						 round_up(htab->map.value_size, 8), true);
+			if (err)
+				goto free_map_locked;
+		}
 	}
 
 	return &htab->map;
@@ -570,12 +584,16 @@ static struct bpf_map *htab_map_alloc(union bpf_attr *attr)
 free_prealloc:
 	prealloc_destroy(htab);
 free_map_locked:
+	if (htab->use_percpu_counter)
+		percpu_counter_destroy(&htab->pcount);
 	for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++)
 		free_percpu(htab->map_locked[i]);
 	bpf_map_area_free(htab->buckets);
+	bpf_mem_alloc_destroy(&htab->pcpu_ma);
+	bpf_mem_alloc_destroy(&htab->ma);
 free_htab:
 	lockdep_unregister_key(&htab->lockdep_key);
-	kfree(htab);
+	bpf_map_area_free(htab);
 	return ERR_PTR(err);
 }
 
@@ -847,17 +865,9 @@ find_first_elem:
 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)
 {
 	if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)
-		free_percpu(htab_elem_get_ptr(l, htab->map.key_size));
+		bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);
 	check_and_free_fields(htab, l);
-	kfree(l);
-}
-
-static void htab_elem_free_rcu(struct rcu_head *head)
-{
-	struct htab_elem *l = container_of(head, struct htab_elem, rcu);
-	struct bpf_htab *htab = l->htab;
-
-	htab_elem_free(htab, l);
+	bpf_mem_cache_free(&htab->ma, l);
 }
 
 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)
@@ -871,6 +881,31 @@ static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)
 	}
 }
 
+static bool is_map_full(struct bpf_htab *htab)
+{
+	if (htab->use_percpu_counter)
+		return __percpu_counter_compare(&htab->pcount, htab->map.max_entries,
+						PERCPU_COUNTER_BATCH) >= 0;
+	return atomic_read(&htab->count) >= htab->map.max_entries;
+}
+
+static void inc_elem_count(struct bpf_htab *htab)
+{
+	if (htab->use_percpu_counter)
+		percpu_counter_add_batch(&htab->pcount, 1, PERCPU_COUNTER_BATCH);
+	else
+		atomic_inc(&htab->count);
+}
+
+static void dec_elem_count(struct bpf_htab *htab)
+{
+	if (htab->use_percpu_counter)
+		percpu_counter_add_batch(&htab->pcount, -1, PERCPU_COUNTER_BATCH);
+	else
+		atomic_dec(&htab->count);
+}
+
+
 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
 {
 	htab_put_fd_value(htab, l);
@@ -879,9 +914,8 @@ static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)
 		check_and_free_fields(htab, l);
 		__pcpu_freelist_push(&htab->freelist, &l->fnode);
 	} else {
-		atomic_dec(&htab->count);
-		l->htab = htab;
-		call_rcu(&l->rcu, htab_elem_free_rcu);
+		dec_elem_count(htab);
+		htab_elem_free(htab, l);
 	}
 }
 
@@ -906,13 +940,12 @@ static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,
 static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,
 			    void *value, bool onallcpus)
 {
-	/* When using prealloc and not setting the initial value on all cpus,
-	 * zero-fill element values for other cpus (just as what happens when
-	 * not using prealloc). Otherwise, bpf program has no way to ensure
+	/* When not setting the initial value on all cpus, zero-fill element
+	 * values for other cpus. Otherwise, bpf program has no way to ensure
 	 * known initial values for cpus other than current one
 	 * (onallcpus=false always when coming from bpf prog).
 	 */
-	if (htab_is_prealloc(htab) && !onallcpus) {
+	if (!onallcpus) {
 		u32 size = round_up(htab->map.value_size, 8);
 		int current_cpu = raw_smp_processor_id();
 		int cpu;
@@ -963,19 +996,16 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
 			l_new = container_of(l, struct htab_elem, fnode);
 		}
 	} else {
-		if (atomic_inc_return(&htab->count) > htab->map.max_entries)
-			if (!old_elem) {
+		if (is_map_full(htab))
+			if (!old_elem)
 				/* when map is full and update() is replacing
 				 * old element, it's ok to allocate, since
 				 * old element will be freed immediately.
 				 * Otherwise return an error
 				 */
-				l_new = ERR_PTR(-E2BIG);
-				goto dec_count;
-			}
-		l_new = bpf_map_kmalloc_node(&htab->map, htab->elem_size,
-					     GFP_NOWAIT | __GFP_NOWARN,
-					     htab->map.numa_node);
+				return ERR_PTR(-E2BIG);
+		inc_elem_count(htab);
+		l_new = bpf_mem_cache_alloc(&htab->ma);
 		if (!l_new) {
 			l_new = ERR_PTR(-ENOMEM);
 			goto dec_count;
@@ -986,18 +1016,18 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
 
 	memcpy(l_new->key, key, key_size);
 	if (percpu) {
-		size = round_up(size, 8);
 		if (prealloc) {
 			pptr = htab_elem_get_ptr(l_new, key_size);
 		} else {
 			/* alloc_percpu zero-fills */
-			pptr = bpf_map_alloc_percpu(&htab->map, size, 8,
-						    GFP_NOWAIT | __GFP_NOWARN);
+			pptr = bpf_mem_cache_alloc(&htab->pcpu_ma);
 			if (!pptr) {
-				kfree(l_new);
+				bpf_mem_cache_free(&htab->ma, l_new);
 				l_new = ERR_PTR(-ENOMEM);
 				goto dec_count;
 			}
+			l_new->ptr_to_pptr = pptr;
+			pptr = *(void **)pptr;
 		}
 
 		pcpu_init_value(htab, pptr, value, onallcpus);
@@ -1016,7 +1046,7 @@ static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,
 	l_new->hash = hash;
 	return l_new;
 dec_count:
-	atomic_dec(&htab->count);
+	dec_elem_count(htab);
 	return l_new;
 }
 
@@ -1416,6 +1446,10 @@ static void delete_all_elements(struct bpf_htab *htab)
 {
 	int i;
 
+	/* It's called from a worker thread, so disable migration here,
+	 * since bpf_mem_cache_free() relies on that.
+	 */
+	migrate_disable();
 	for (i = 0; i < htab->n_buckets; i++) {
 		struct hlist_nulls_head *head = select_bucket(htab, i);
 		struct hlist_nulls_node *n;
@@ -1426,6 +1460,7 @@ static void delete_all_elements(struct bpf_htab *htab)
 			htab_elem_free(htab, l);
 		}
 	}
+	migrate_enable();
 }
 
 static void htab_free_malloced_timers(struct bpf_htab *htab)
@@ -1475,10 +1510,10 @@ static void htab_map_free(struct bpf_map *map)
 	 * There is no need to synchronize_rcu() here to protect map elements.
 	 */
 
-	/* some of free_htab_elem() callbacks for elements of this map may
-	 * not have executed. Wait for them.
+	/* htab no longer uses call_rcu() directly. bpf_mem_alloc does it
+	 * underneath and is reponsible for waiting for callbacks to finish
+	 * during bpf_mem_alloc_destroy().
 	 */
-	rcu_barrier();
 	if (!htab_is_prealloc(htab)) {
 		delete_all_elements(htab);
 	} else {
@@ -1489,10 +1524,14 @@ static void htab_map_free(struct bpf_map *map)
 	bpf_map_free_kptr_off_tab(map);
 	free_percpu(htab->extra_elems);
 	bpf_map_area_free(htab->buckets);
+	bpf_mem_alloc_destroy(&htab->pcpu_ma);
+	bpf_mem_alloc_destroy(&htab->ma);
+	if (htab->use_percpu_counter)
+		percpu_counter_destroy(&htab->pcount);
 	for (i = 0; i < HASHTAB_MAP_LOCK_COUNT; i++)
 		free_percpu(htab->map_locked[i]);
 	lockdep_unregister_key(&htab->lockdep_key);
-	kfree(htab);
+	bpf_map_area_free(htab);
 }
 
 static void htab_map_seq_show_elem(struct bpf_map *map, void *key,
@@ -1691,8 +1730,11 @@ again_nocopy:
 	/* do not grab the lock unless need it (bucket_cnt > 0). */
 	if (locked) {
 		ret = htab_lock_bucket(htab, b, batch, &flags);
-		if (ret)
-			goto next_batch;
+		if (ret) {
+			rcu_read_unlock();
+			bpf_enable_instrumentation();
+			goto after_loop;
+		}
 	}
 
 	bucket_cnt = 0;
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 1f961f9982d2..a6b04faed282 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -15,6 +15,7 @@
 #include <linux/ctype.h>
 #include <linux/jiffies.h>
 #include <linux/pid_namespace.h>
+#include <linux/poison.h>
 #include <linux/proc_ns.h>
 #include <linux/security.h>
 #include <linux/btf_ids.h>
@@ -198,6 +199,18 @@ const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto = {
 	.ret_type	= RET_INTEGER,
 };
 
+BPF_CALL_0(bpf_ktime_get_tai_ns)
+{
+	/* NMI safe access to clock tai */
+	return ktime_get_tai_fast_ns();
+}
+
+const struct bpf_func_proto bpf_ktime_get_tai_ns_proto = {
+	.func		= bpf_ktime_get_tai_ns,
+	.gpl_only	= false,
+	.ret_type	= RET_INTEGER,
+};
+
 BPF_CALL_0(bpf_get_current_pid_tgid)
 {
 	struct task_struct *task = current;
@@ -415,40 +428,7 @@ const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto = {
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_ANYTHING,
 };
-
-#ifdef CONFIG_CGROUP_BPF
-
-BPF_CALL_2(bpf_get_local_storage, struct bpf_map *, map, u64, flags)
-{
-	/* flags argument is not used now,
-	 * but provides an ability to extend the API.
-	 * verifier checks that its value is correct.
-	 */
-	enum bpf_cgroup_storage_type stype = cgroup_storage_type(map);
-	struct bpf_cgroup_storage *storage;
-	struct bpf_cg_run_ctx *ctx;
-	void *ptr;
-
-	/* get current cgroup storage from BPF run context */
-	ctx = container_of(current->bpf_ctx, struct bpf_cg_run_ctx, run_ctx);
-	storage = ctx->prog_item->cgroup_storage[stype];
-
-	if (stype == BPF_CGROUP_STORAGE_SHARED)
-		ptr = &READ_ONCE(storage->buf)->data[0];
-	else
-		ptr = this_cpu_ptr(storage->percpu_buf);
-
-	return (unsigned long)ptr;
-}
-
-const struct bpf_func_proto bpf_get_local_storage_proto = {
-	.func		= bpf_get_local_storage,
-	.gpl_only	= false,
-	.ret_type	= RET_PTR_TO_MAP_VALUE,
-	.arg1_type	= ARG_CONST_MAP_PTR,
-	.arg2_type	= ARG_ANYTHING,
-};
-#endif
+#endif /* CONFIG_CGROUPS */
 
 #define BPF_STRTOX_BASE_MASK 0x1F
 
@@ -577,7 +557,6 @@ const struct bpf_func_proto bpf_strtoul_proto = {
 	.arg3_type	= ARG_ANYTHING,
 	.arg4_type	= ARG_PTR_TO_LONG,
 };
-#endif
 
 BPF_CALL_3(bpf_strncmp, const char *, s1, u32, s1_sz, const char *, s2)
 {
@@ -1398,10 +1377,9 @@ BPF_CALL_2(bpf_kptr_xchg, void *, map_value, void *, ptr)
 }
 
 /* Unlike other PTR_TO_BTF_ID helpers the btf_id in bpf_kptr_xchg()
- * helper is determined dynamically by the verifier.
+ * helper is determined dynamically by the verifier. Use BPF_PTR_POISON to
+ * denote type that verifier will determine.
  */
-#define BPF_PTR_POISON ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA))
-
 static const struct bpf_func_proto bpf_kptr_xchg_proto = {
 	.func         = bpf_kptr_xchg,
 	.gpl_only     = false,
@@ -1430,7 +1408,7 @@ static void bpf_dynptr_set_type(struct bpf_dynptr_kern *ptr, enum bpf_dynptr_typ
 	ptr->size |= type << DYNPTR_TYPE_SHIFT;
 }
 
-static u32 bpf_dynptr_get_size(struct bpf_dynptr_kern *ptr)
+u32 bpf_dynptr_get_size(struct bpf_dynptr_kern *ptr)
 {
 	return ptr->size & DYNPTR_SIZE_MASK;
 }
@@ -1468,6 +1446,8 @@ BPF_CALL_4(bpf_dynptr_from_mem, void *, data, u32, size, u64, flags, struct bpf_
 {
 	int err;
 
+	BTF_TYPE_EMIT(struct bpf_dynptr);
+
 	err = bpf_dynptr_check_size(size);
 	if (err)
 		goto error;
@@ -1617,6 +1597,8 @@ bpf_base_func_proto(enum bpf_func_id func_id)
 		return &bpf_ktime_get_ns_proto;
 	case BPF_FUNC_ktime_get_boot_ns:
 		return &bpf_ktime_get_boot_ns_proto;
+	case BPF_FUNC_ktime_get_tai_ns:
+		return &bpf_ktime_get_tai_ns_proto;
 	case BPF_FUNC_ringbuf_output:
 		return &bpf_ringbuf_output_proto;
 	case BPF_FUNC_ringbuf_reserve:
@@ -1627,26 +1609,12 @@ bpf_base_func_proto(enum bpf_func_id func_id)
 		return &bpf_ringbuf_discard_proto;
 	case BPF_FUNC_ringbuf_query:
 		return &bpf_ringbuf_query_proto;
-	case BPF_FUNC_ringbuf_reserve_dynptr:
-		return &bpf_ringbuf_reserve_dynptr_proto;
-	case BPF_FUNC_ringbuf_submit_dynptr:
-		return &bpf_ringbuf_submit_dynptr_proto;
-	case BPF_FUNC_ringbuf_discard_dynptr:
-		return &bpf_ringbuf_discard_dynptr_proto;
-	case BPF_FUNC_for_each_map_elem:
-		return &bpf_for_each_map_elem_proto;
-	case BPF_FUNC_loop:
-		return &bpf_loop_proto;
 	case BPF_FUNC_strncmp:
 		return &bpf_strncmp_proto;
-	case BPF_FUNC_dynptr_from_mem:
-		return &bpf_dynptr_from_mem_proto;
-	case BPF_FUNC_dynptr_read:
-		return &bpf_dynptr_read_proto;
-	case BPF_FUNC_dynptr_write:
-		return &bpf_dynptr_write_proto;
-	case BPF_FUNC_dynptr_data:
-		return &bpf_dynptr_data_proto;
+	case BPF_FUNC_strtol:
+		return &bpf_strtol_proto;
+	case BPF_FUNC_strtoul:
+		return &bpf_strtoul_proto;
 	default:
 		break;
 	}
@@ -1675,6 +1643,26 @@ bpf_base_func_proto(enum bpf_func_id func_id)
 		return &bpf_timer_cancel_proto;
 	case BPF_FUNC_kptr_xchg:
 		return &bpf_kptr_xchg_proto;
+	case BPF_FUNC_for_each_map_elem:
+		return &bpf_for_each_map_elem_proto;
+	case BPF_FUNC_loop:
+		return &bpf_loop_proto;
+	case BPF_FUNC_user_ringbuf_drain:
+		return &bpf_user_ringbuf_drain_proto;
+	case BPF_FUNC_ringbuf_reserve_dynptr:
+		return &bpf_ringbuf_reserve_dynptr_proto;
+	case BPF_FUNC_ringbuf_submit_dynptr:
+		return &bpf_ringbuf_submit_dynptr_proto;
+	case BPF_FUNC_ringbuf_discard_dynptr:
+		return &bpf_ringbuf_discard_dynptr_proto;
+	case BPF_FUNC_dynptr_from_mem:
+		return &bpf_dynptr_from_mem_proto;
+	case BPF_FUNC_dynptr_read:
+		return &bpf_dynptr_read_proto;
+	case BPF_FUNC_dynptr_write:
+		return &bpf_dynptr_write_proto;
+	case BPF_FUNC_dynptr_data:
+		return &bpf_dynptr_data_proto;
 	default:
 		break;
 	}
@@ -1711,3 +1699,21 @@ bpf_base_func_proto(enum bpf_func_id func_id)
 		return NULL;
 	}
 }
+
+BTF_SET8_START(tracing_btf_ids)
+#ifdef CONFIG_KEXEC_CORE
+BTF_ID_FLAGS(func, crash_kexec, KF_DESTRUCTIVE)
+#endif
+BTF_SET8_END(tracing_btf_ids)
+
+static const struct btf_kfunc_id_set tracing_kfunc_set = {
+	.owner = THIS_MODULE,
+	.set   = &tracing_btf_ids,
+};
+
+static int __init kfunc_init(void)
+{
+	return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &tracing_kfunc_set);
+}
+
+late_initcall(kfunc_init);
diff --git a/kernel/bpf/local_storage.c b/kernel/bpf/local_storage.c
index 49ef0ce040c7..098cf336fae6 100644
--- a/kernel/bpf/local_storage.c
+++ b/kernel/bpf/local_storage.c
@@ -313,8 +313,7 @@ static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr)
 		/* max_entries is not used and enforced to be 0 */
 		return ERR_PTR(-EINVAL);
 
-	map = kmalloc_node(sizeof(struct bpf_cgroup_storage_map),
-			   __GFP_ZERO | GFP_USER | __GFP_ACCOUNT, numa_node);
+	map = bpf_map_area_alloc(sizeof(struct bpf_cgroup_storage_map), numa_node);
 	if (!map)
 		return ERR_PTR(-ENOMEM);
 
@@ -346,7 +345,7 @@ static void cgroup_storage_map_free(struct bpf_map *_map)
 	WARN_ON(!RB_EMPTY_ROOT(&map->root));
 	WARN_ON(!list_empty(&map->list));
 
-	kfree(map);
+	bpf_map_area_free(map);
 }
 
 static int cgroup_storage_delete_elem(struct bpf_map *map, void *key)
diff --git a/kernel/bpf/lpm_trie.c b/kernel/bpf/lpm_trie.c
index d789e3b831ad..d833496e9e42 100644
--- a/kernel/bpf/lpm_trie.c
+++ b/kernel/bpf/lpm_trie.c
@@ -558,7 +558,7 @@ static struct bpf_map *trie_alloc(union bpf_attr *attr)
 	    attr->value_size > LPM_VAL_SIZE_MAX)
 		return ERR_PTR(-EINVAL);
 
-	trie = kzalloc(sizeof(*trie), GFP_USER | __GFP_NOWARN | __GFP_ACCOUNT);
+	trie = bpf_map_area_alloc(sizeof(*trie), NUMA_NO_NODE);
 	if (!trie)
 		return ERR_PTR(-ENOMEM);
 
@@ -609,7 +609,7 @@ static void trie_free(struct bpf_map *map)
 	}
 
 out:
-	kfree(trie);
+	bpf_map_area_free(trie);
 }
 
 static int trie_get_next_key(struct bpf_map *map, void *_key, void *_next_key)
diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c
new file mode 100644
index 000000000000..5f83be1d2018
--- /dev/null
+++ b/kernel/bpf/memalloc.c
@@ -0,0 +1,635 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
+#include <linux/mm.h>
+#include <linux/llist.h>
+#include <linux/bpf.h>
+#include <linux/irq_work.h>
+#include <linux/bpf_mem_alloc.h>
+#include <linux/memcontrol.h>
+#include <asm/local.h>
+
+/* Any context (including NMI) BPF specific memory allocator.
+ *
+ * Tracing BPF programs can attach to kprobe and fentry. Hence they
+ * run in unknown context where calling plain kmalloc() might not be safe.
+ *
+ * Front-end kmalloc() with per-cpu per-bucket cache of free elements.
+ * Refill this cache asynchronously from irq_work.
+ *
+ * CPU_0 buckets
+ * 16 32 64 96 128 196 256 512 1024 2048 4096
+ * ...
+ * CPU_N buckets
+ * 16 32 64 96 128 196 256 512 1024 2048 4096
+ *
+ * The buckets are prefilled at the start.
+ * BPF programs always run with migration disabled.
+ * It's safe to allocate from cache of the current cpu with irqs disabled.
+ * Free-ing is always done into bucket of the current cpu as well.
+ * irq_work trims extra free elements from buckets with kfree
+ * and refills them with kmalloc, so global kmalloc logic takes care
+ * of freeing objects allocated by one cpu and freed on another.
+ *
+ * Every allocated objected is padded with extra 8 bytes that contains
+ * struct llist_node.
+ */
+#define LLIST_NODE_SZ sizeof(struct llist_node)
+
+/* similar to kmalloc, but sizeof == 8 bucket is gone */
+static u8 size_index[24] __ro_after_init = {
+	3,	/* 8 */
+	3,	/* 16 */
+	4,	/* 24 */
+	4,	/* 32 */
+	5,	/* 40 */
+	5,	/* 48 */
+	5,	/* 56 */
+	5,	/* 64 */
+	1,	/* 72 */
+	1,	/* 80 */
+	1,	/* 88 */
+	1,	/* 96 */
+	6,	/* 104 */
+	6,	/* 112 */
+	6,	/* 120 */
+	6,	/* 128 */
+	2,	/* 136 */
+	2,	/* 144 */
+	2,	/* 152 */
+	2,	/* 160 */
+	2,	/* 168 */
+	2,	/* 176 */
+	2,	/* 184 */
+	2	/* 192 */
+};
+
+static int bpf_mem_cache_idx(size_t size)
+{
+	if (!size || size > 4096)
+		return -1;
+
+	if (size <= 192)
+		return size_index[(size - 1) / 8] - 1;
+
+	return fls(size - 1) - 1;
+}
+
+#define NUM_CACHES 11
+
+struct bpf_mem_cache {
+	/* per-cpu list of free objects of size 'unit_size'.
+	 * All accesses are done with interrupts disabled and 'active' counter
+	 * protection with __llist_add() and __llist_del_first().
+	 */
+	struct llist_head free_llist;
+	local_t active;
+
+	/* Operations on the free_list from unit_alloc/unit_free/bpf_mem_refill
+	 * are sequenced by per-cpu 'active' counter. But unit_free() cannot
+	 * fail. When 'active' is busy the unit_free() will add an object to
+	 * free_llist_extra.
+	 */
+	struct llist_head free_llist_extra;
+
+	struct irq_work refill_work;
+	struct obj_cgroup *objcg;
+	int unit_size;
+	/* count of objects in free_llist */
+	int free_cnt;
+	int low_watermark, high_watermark, batch;
+	int percpu_size;
+
+	struct rcu_head rcu;
+	struct llist_head free_by_rcu;
+	struct llist_head waiting_for_gp;
+	atomic_t call_rcu_in_progress;
+};
+
+struct bpf_mem_caches {
+	struct bpf_mem_cache cache[NUM_CACHES];
+};
+
+static struct llist_node notrace *__llist_del_first(struct llist_head *head)
+{
+	struct llist_node *entry, *next;
+
+	entry = head->first;
+	if (!entry)
+		return NULL;
+	next = entry->next;
+	head->first = next;
+	return entry;
+}
+
+static void *__alloc(struct bpf_mem_cache *c, int node)
+{
+	/* Allocate, but don't deplete atomic reserves that typical
+	 * GFP_ATOMIC would do. irq_work runs on this cpu and kmalloc
+	 * will allocate from the current numa node which is what we
+	 * want here.
+	 */
+	gfp_t flags = GFP_NOWAIT | __GFP_NOWARN | __GFP_ACCOUNT;
+
+	if (c->percpu_size) {
+		void **obj = kmalloc_node(c->percpu_size, flags, node);
+		void *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags);
+
+		if (!obj || !pptr) {
+			free_percpu(pptr);
+			kfree(obj);
+			return NULL;
+		}
+		obj[1] = pptr;
+		return obj;
+	}
+
+	return kmalloc_node(c->unit_size, flags, node);
+}
+
+static struct mem_cgroup *get_memcg(const struct bpf_mem_cache *c)
+{
+#ifdef CONFIG_MEMCG_KMEM
+	if (c->objcg)
+		return get_mem_cgroup_from_objcg(c->objcg);
+#endif
+
+#ifdef CONFIG_MEMCG
+	return root_mem_cgroup;
+#else
+	return NULL;
+#endif
+}
+
+/* Mostly runs from irq_work except __init phase. */
+static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node)
+{
+	struct mem_cgroup *memcg = NULL, *old_memcg;
+	unsigned long flags;
+	void *obj;
+	int i;
+
+	memcg = get_memcg(c);
+	old_memcg = set_active_memcg(memcg);
+	for (i = 0; i < cnt; i++) {
+		obj = __alloc(c, node);
+		if (!obj)
+			break;
+		if (IS_ENABLED(CONFIG_PREEMPT_RT))
+			/* In RT irq_work runs in per-cpu kthread, so disable
+			 * interrupts to avoid preemption and interrupts and
+			 * reduce the chance of bpf prog executing on this cpu
+			 * when active counter is busy.
+			 */
+			local_irq_save(flags);
+		/* alloc_bulk runs from irq_work which will not preempt a bpf
+		 * program that does unit_alloc/unit_free since IRQs are
+		 * disabled there. There is no race to increment 'active'
+		 * counter. It protects free_llist from corruption in case NMI
+		 * bpf prog preempted this loop.
+		 */
+		WARN_ON_ONCE(local_inc_return(&c->active) != 1);
+		__llist_add(obj, &c->free_llist);
+		c->free_cnt++;
+		local_dec(&c->active);
+		if (IS_ENABLED(CONFIG_PREEMPT_RT))
+			local_irq_restore(flags);
+	}
+	set_active_memcg(old_memcg);
+	mem_cgroup_put(memcg);
+}
+
+static void free_one(struct bpf_mem_cache *c, void *obj)
+{
+	if (c->percpu_size) {
+		free_percpu(((void **)obj)[1]);
+		kfree(obj);
+		return;
+	}
+
+	kfree(obj);
+}
+
+static void __free_rcu(struct rcu_head *head)
+{
+	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu);
+	struct llist_node *llnode = llist_del_all(&c->waiting_for_gp);
+	struct llist_node *pos, *t;
+
+	llist_for_each_safe(pos, t, llnode)
+		free_one(c, pos);
+	atomic_set(&c->call_rcu_in_progress, 0);
+}
+
+static void __free_rcu_tasks_trace(struct rcu_head *head)
+{
+	struct bpf_mem_cache *c = container_of(head, struct bpf_mem_cache, rcu);
+
+	call_rcu(&c->rcu, __free_rcu);
+}
+
+static void enque_to_free(struct bpf_mem_cache *c, void *obj)
+{
+	struct llist_node *llnode = obj;
+
+	/* bpf_mem_cache is a per-cpu object. Freeing happens in irq_work.
+	 * Nothing races to add to free_by_rcu list.
+	 */
+	__llist_add(llnode, &c->free_by_rcu);
+}
+
+static void do_call_rcu(struct bpf_mem_cache *c)
+{
+	struct llist_node *llnode, *t;
+
+	if (atomic_xchg(&c->call_rcu_in_progress, 1))
+		return;
+
+	WARN_ON_ONCE(!llist_empty(&c->waiting_for_gp));
+	llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu))
+		/* There is no concurrent __llist_add(waiting_for_gp) access.
+		 * It doesn't race with llist_del_all either.
+		 * But there could be two concurrent llist_del_all(waiting_for_gp):
+		 * from __free_rcu() and from drain_mem_cache().
+		 */
+		__llist_add(llnode, &c->waiting_for_gp);
+	/* Use call_rcu_tasks_trace() to wait for sleepable progs to finish.
+	 * Then use call_rcu() to wait for normal progs to finish
+	 * and finally do free_one() on each element.
+	 */
+	call_rcu_tasks_trace(&c->rcu, __free_rcu_tasks_trace);
+}
+
+static void free_bulk(struct bpf_mem_cache *c)
+{
+	struct llist_node *llnode, *t;
+	unsigned long flags;
+	int cnt;
+
+	do {
+		if (IS_ENABLED(CONFIG_PREEMPT_RT))
+			local_irq_save(flags);
+		WARN_ON_ONCE(local_inc_return(&c->active) != 1);
+		llnode = __llist_del_first(&c->free_llist);
+		if (llnode)
+			cnt = --c->free_cnt;
+		else
+			cnt = 0;
+		local_dec(&c->active);
+		if (IS_ENABLED(CONFIG_PREEMPT_RT))
+			local_irq_restore(flags);
+		if (llnode)
+			enque_to_free(c, llnode);
+	} while (cnt > (c->high_watermark + c->low_watermark) / 2);
+
+	/* and drain free_llist_extra */
+	llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra))
+		enque_to_free(c, llnode);
+	do_call_rcu(c);
+}
+
+static void bpf_mem_refill(struct irq_work *work)
+{
+	struct bpf_mem_cache *c = container_of(work, struct bpf_mem_cache, refill_work);
+	int cnt;
+
+	/* Racy access to free_cnt. It doesn't need to be 100% accurate */
+	cnt = c->free_cnt;
+	if (cnt < c->low_watermark)
+		/* irq_work runs on this cpu and kmalloc will allocate
+		 * from the current numa node which is what we want here.
+		 */
+		alloc_bulk(c, c->batch, NUMA_NO_NODE);
+	else if (cnt > c->high_watermark)
+		free_bulk(c);
+}
+
+static void notrace irq_work_raise(struct bpf_mem_cache *c)
+{
+	irq_work_queue(&c->refill_work);
+}
+
+/* For typical bpf map case that uses bpf_mem_cache_alloc and single bucket
+ * the freelist cache will be elem_size * 64 (or less) on each cpu.
+ *
+ * For bpf programs that don't have statically known allocation sizes and
+ * assuming (low_mark + high_mark) / 2 as an average number of elements per
+ * bucket and all buckets are used the total amount of memory in freelists
+ * on each cpu will be:
+ * 64*16 + 64*32 + 64*64 + 64*96 + 64*128 + 64*196 + 64*256 + 32*512 + 16*1024 + 8*2048 + 4*4096
+ * == ~ 116 Kbyte using below heuristic.
+ * Initialized, but unused bpf allocator (not bpf map specific one) will
+ * consume ~ 11 Kbyte per cpu.
+ * Typical case will be between 11K and 116K closer to 11K.
+ * bpf progs can and should share bpf_mem_cache when possible.
+ */
+
+static void prefill_mem_cache(struct bpf_mem_cache *c, int cpu)
+{
+	init_irq_work(&c->refill_work, bpf_mem_refill);
+	if (c->unit_size <= 256) {
+		c->low_watermark = 32;
+		c->high_watermark = 96;
+	} else {
+		/* When page_size == 4k, order-0 cache will have low_mark == 2
+		 * and high_mark == 6 with batch alloc of 3 individual pages at
+		 * a time.
+		 * 8k allocs and above low == 1, high == 3, batch == 1.
+		 */
+		c->low_watermark = max(32 * 256 / c->unit_size, 1);
+		c->high_watermark = max(96 * 256 / c->unit_size, 3);
+	}
+	c->batch = max((c->high_watermark - c->low_watermark) / 4 * 3, 1);
+
+	/* To avoid consuming memory assume that 1st run of bpf
+	 * prog won't be doing more than 4 map_update_elem from
+	 * irq disabled region
+	 */
+	alloc_bulk(c, c->unit_size <= 256 ? 4 : 1, cpu_to_node(cpu));
+}
+
+/* When size != 0 bpf_mem_cache for each cpu.
+ * This is typical bpf hash map use case when all elements have equal size.
+ *
+ * When size == 0 allocate 11 bpf_mem_cache-s for each cpu, then rely on
+ * kmalloc/kfree. Max allocation size is 4096 in this case.
+ * This is bpf_dynptr and bpf_kptr use case.
+ */
+int bpf_mem_alloc_init(struct bpf_mem_alloc *ma, int size, bool percpu)
+{
+	static u16 sizes[NUM_CACHES] = {96, 192, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096};
+	struct bpf_mem_caches *cc, __percpu *pcc;
+	struct bpf_mem_cache *c, __percpu *pc;
+	struct obj_cgroup *objcg = NULL;
+	int cpu, i, unit_size, percpu_size = 0;
+
+	if (size) {
+		pc = __alloc_percpu_gfp(sizeof(*pc), 8, GFP_KERNEL);
+		if (!pc)
+			return -ENOMEM;
+
+		if (percpu)
+			/* room for llist_node and per-cpu pointer */
+			percpu_size = LLIST_NODE_SZ + sizeof(void *);
+		else
+			size += LLIST_NODE_SZ; /* room for llist_node */
+		unit_size = size;
+
+#ifdef CONFIG_MEMCG_KMEM
+		objcg = get_obj_cgroup_from_current();
+#endif
+		for_each_possible_cpu(cpu) {
+			c = per_cpu_ptr(pc, cpu);
+			c->unit_size = unit_size;
+			c->objcg = objcg;
+			c->percpu_size = percpu_size;
+			prefill_mem_cache(c, cpu);
+		}
+		ma->cache = pc;
+		return 0;
+	}
+
+	/* size == 0 && percpu is an invalid combination */
+	if (WARN_ON_ONCE(percpu))
+		return -EINVAL;
+
+	pcc = __alloc_percpu_gfp(sizeof(*cc), 8, GFP_KERNEL);
+	if (!pcc)
+		return -ENOMEM;
+#ifdef CONFIG_MEMCG_KMEM
+	objcg = get_obj_cgroup_from_current();
+#endif
+	for_each_possible_cpu(cpu) {
+		cc = per_cpu_ptr(pcc, cpu);
+		for (i = 0; i < NUM_CACHES; i++) {
+			c = &cc->cache[i];
+			c->unit_size = sizes[i];
+			c->objcg = objcg;
+			prefill_mem_cache(c, cpu);
+		}
+	}
+	ma->caches = pcc;
+	return 0;
+}
+
+static void drain_mem_cache(struct bpf_mem_cache *c)
+{
+	struct llist_node *llnode, *t;
+
+	/* No progs are using this bpf_mem_cache, but htab_map_free() called
+	 * bpf_mem_cache_free() for all remaining elements and they can be in
+	 * free_by_rcu or in waiting_for_gp lists, so drain those lists now.
+	 */
+	llist_for_each_safe(llnode, t, __llist_del_all(&c->free_by_rcu))
+		free_one(c, llnode);
+	llist_for_each_safe(llnode, t, llist_del_all(&c->waiting_for_gp))
+		free_one(c, llnode);
+	llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist))
+		free_one(c, llnode);
+	llist_for_each_safe(llnode, t, llist_del_all(&c->free_llist_extra))
+		free_one(c, llnode);
+}
+
+static void free_mem_alloc_no_barrier(struct bpf_mem_alloc *ma)
+{
+	free_percpu(ma->cache);
+	free_percpu(ma->caches);
+	ma->cache = NULL;
+	ma->caches = NULL;
+}
+
+static void free_mem_alloc(struct bpf_mem_alloc *ma)
+{
+	/* waiting_for_gp lists was drained, but __free_rcu might
+	 * still execute. Wait for it now before we freeing percpu caches.
+	 */
+	rcu_barrier_tasks_trace();
+	rcu_barrier();
+	free_mem_alloc_no_barrier(ma);
+}
+
+static void free_mem_alloc_deferred(struct work_struct *work)
+{
+	struct bpf_mem_alloc *ma = container_of(work, struct bpf_mem_alloc, work);
+
+	free_mem_alloc(ma);
+	kfree(ma);
+}
+
+static void destroy_mem_alloc(struct bpf_mem_alloc *ma, int rcu_in_progress)
+{
+	struct bpf_mem_alloc *copy;
+
+	if (!rcu_in_progress) {
+		/* Fast path. No callbacks are pending, hence no need to do
+		 * rcu_barrier-s.
+		 */
+		free_mem_alloc_no_barrier(ma);
+		return;
+	}
+
+	copy = kmalloc(sizeof(*ma), GFP_KERNEL);
+	if (!copy) {
+		/* Slow path with inline barrier-s */
+		free_mem_alloc(ma);
+		return;
+	}
+
+	/* Defer barriers into worker to let the rest of map memory to be freed */
+	copy->cache = ma->cache;
+	ma->cache = NULL;
+	copy->caches = ma->caches;
+	ma->caches = NULL;
+	INIT_WORK(&copy->work, free_mem_alloc_deferred);
+	queue_work(system_unbound_wq, &copy->work);
+}
+
+void bpf_mem_alloc_destroy(struct bpf_mem_alloc *ma)
+{
+	struct bpf_mem_caches *cc;
+	struct bpf_mem_cache *c;
+	int cpu, i, rcu_in_progress;
+
+	if (ma->cache) {
+		rcu_in_progress = 0;
+		for_each_possible_cpu(cpu) {
+			c = per_cpu_ptr(ma->cache, cpu);
+			drain_mem_cache(c);
+			rcu_in_progress += atomic_read(&c->call_rcu_in_progress);
+		}
+		/* objcg is the same across cpus */
+		if (c->objcg)
+			obj_cgroup_put(c->objcg);
+		destroy_mem_alloc(ma, rcu_in_progress);
+	}
+	if (ma->caches) {
+		rcu_in_progress = 0;
+		for_each_possible_cpu(cpu) {
+			cc = per_cpu_ptr(ma->caches, cpu);
+			for (i = 0; i < NUM_CACHES; i++) {
+				c = &cc->cache[i];
+				drain_mem_cache(c);
+				rcu_in_progress += atomic_read(&c->call_rcu_in_progress);
+			}
+		}
+		if (c->objcg)
+			obj_cgroup_put(c->objcg);
+		destroy_mem_alloc(ma, rcu_in_progress);
+	}
+}
+
+/* notrace is necessary here and in other functions to make sure
+ * bpf programs cannot attach to them and cause llist corruptions.
+ */
+static void notrace *unit_alloc(struct bpf_mem_cache *c)
+{
+	struct llist_node *llnode = NULL;
+	unsigned long flags;
+	int cnt = 0;
+
+	/* Disable irqs to prevent the following race for majority of prog types:
+	 * prog_A
+	 *   bpf_mem_alloc
+	 *      preemption or irq -> prog_B
+	 *        bpf_mem_alloc
+	 *
+	 * but prog_B could be a perf_event NMI prog.
+	 * Use per-cpu 'active' counter to order free_list access between
+	 * unit_alloc/unit_free/bpf_mem_refill.
+	 */
+	local_irq_save(flags);
+	if (local_inc_return(&c->active) == 1) {
+		llnode = __llist_del_first(&c->free_llist);
+		if (llnode)
+			cnt = --c->free_cnt;
+	}
+	local_dec(&c->active);
+	local_irq_restore(flags);
+
+	WARN_ON(cnt < 0);
+
+	if (cnt < c->low_watermark)
+		irq_work_raise(c);
+	return llnode;
+}
+
+/* Though 'ptr' object could have been allocated on a different cpu
+ * add it to the free_llist of the current cpu.
+ * Let kfree() logic deal with it when it's later called from irq_work.
+ */
+static void notrace unit_free(struct bpf_mem_cache *c, void *ptr)
+{
+	struct llist_node *llnode = ptr - LLIST_NODE_SZ;
+	unsigned long flags;
+	int cnt = 0;
+
+	BUILD_BUG_ON(LLIST_NODE_SZ > 8);
+
+	local_irq_save(flags);
+	if (local_inc_return(&c->active) == 1) {
+		__llist_add(llnode, &c->free_llist);
+		cnt = ++c->free_cnt;
+	} else {
+		/* unit_free() cannot fail. Therefore add an object to atomic
+		 * llist. free_bulk() will drain it. Though free_llist_extra is
+		 * a per-cpu list we have to use atomic llist_add here, since
+		 * it also can be interrupted by bpf nmi prog that does another
+		 * unit_free() into the same free_llist_extra.
+		 */
+		llist_add(llnode, &c->free_llist_extra);
+	}
+	local_dec(&c->active);
+	local_irq_restore(flags);
+
+	if (cnt > c->high_watermark)
+		/* free few objects from current cpu into global kmalloc pool */
+		irq_work_raise(c);
+}
+
+/* Called from BPF program or from sys_bpf syscall.
+ * In both cases migration is disabled.
+ */
+void notrace *bpf_mem_alloc(struct bpf_mem_alloc *ma, size_t size)
+{
+	int idx;
+	void *ret;
+
+	if (!size)
+		return ZERO_SIZE_PTR;
+
+	idx = bpf_mem_cache_idx(size + LLIST_NODE_SZ);
+	if (idx < 0)
+		return NULL;
+
+	ret = unit_alloc(this_cpu_ptr(ma->caches)->cache + idx);
+	return !ret ? NULL : ret + LLIST_NODE_SZ;
+}
+
+void notrace bpf_mem_free(struct bpf_mem_alloc *ma, void *ptr)
+{
+	int idx;
+
+	if (!ptr)
+		return;
+
+	idx = bpf_mem_cache_idx(ksize(ptr - LLIST_NODE_SZ));
+	if (idx < 0)
+		return;
+
+	unit_free(this_cpu_ptr(ma->caches)->cache + idx, ptr);
+}
+
+void notrace *bpf_mem_cache_alloc(struct bpf_mem_alloc *ma)
+{
+	void *ret;
+
+	ret = unit_alloc(this_cpu_ptr(ma->cache));
+	return !ret ? NULL : ret + LLIST_NODE_SZ;
+}
+
+void notrace bpf_mem_cache_free(struct bpf_mem_alloc *ma, void *ptr)
+{
+	if (!ptr)
+		return;
+
+	unit_free(this_cpu_ptr(ma->cache), ptr);
+}
diff --git a/kernel/bpf/offload.c b/kernel/bpf/offload.c
index bd09290e3648..13e4efc971e6 100644
--- a/kernel/bpf/offload.c
+++ b/kernel/bpf/offload.c
@@ -372,7 +372,7 @@ struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
 	    attr->map_type != BPF_MAP_TYPE_HASH)
 		return ERR_PTR(-EINVAL);
 
-	offmap = kzalloc(sizeof(*offmap), GFP_USER);
+	offmap = bpf_map_area_alloc(sizeof(*offmap), NUMA_NO_NODE);
 	if (!offmap)
 		return ERR_PTR(-ENOMEM);
 
@@ -404,7 +404,7 @@ struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
 err_unlock:
 	up_write(&bpf_devs_lock);
 	rtnl_unlock();
-	kfree(offmap);
+	bpf_map_area_free(offmap);
 	return ERR_PTR(err);
 }
 
@@ -428,7 +428,7 @@ void bpf_map_offload_map_free(struct bpf_map *map)
 	up_write(&bpf_devs_lock);
 	rtnl_unlock();
 
-	kfree(offmap);
+	bpf_map_area_free(offmap);
 }
 
 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value)
diff --git a/kernel/bpf/percpu_freelist.c b/kernel/bpf/percpu_freelist.c
index 00b874c8e889..b6e7f5c5b9ab 100644
--- a/kernel/bpf/percpu_freelist.c
+++ b/kernel/bpf/percpu_freelist.c
@@ -58,23 +58,21 @@ static inline void ___pcpu_freelist_push_nmi(struct pcpu_freelist *s,
 {
 	int cpu, orig_cpu;
 
-	orig_cpu = cpu = raw_smp_processor_id();
+	orig_cpu = raw_smp_processor_id();
 	while (1) {
-		struct pcpu_freelist_head *head;
+		for_each_cpu_wrap(cpu, cpu_possible_mask, orig_cpu) {
+			struct pcpu_freelist_head *head;
 
-		head = per_cpu_ptr(s->freelist, cpu);
-		if (raw_spin_trylock(&head->lock)) {
-			pcpu_freelist_push_node(head, node);
-			raw_spin_unlock(&head->lock);
-			return;
+			head = per_cpu_ptr(s->freelist, cpu);
+			if (raw_spin_trylock(&head->lock)) {
+				pcpu_freelist_push_node(head, node);
+				raw_spin_unlock(&head->lock);
+				return;
+			}
 		}
-		cpu = cpumask_next(cpu, cpu_possible_mask);
-		if (cpu >= nr_cpu_ids)
-			cpu = 0;
 
 		/* cannot lock any per cpu lock, try extralist */
-		if (cpu == orig_cpu &&
-		    pcpu_freelist_try_push_extra(s, node))
+		if (pcpu_freelist_try_push_extra(s, node))
 			return;
 	}
 }
@@ -125,13 +123,12 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s)
 {
 	struct pcpu_freelist_head *head;
 	struct pcpu_freelist_node *node;
-	int orig_cpu, cpu;
+	int cpu;
 
-	orig_cpu = cpu = raw_smp_processor_id();
-	while (1) {
+	for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) {
 		head = per_cpu_ptr(s->freelist, cpu);
 		if (!READ_ONCE(head->first))
-			goto next_cpu;
+			continue;
 		raw_spin_lock(&head->lock);
 		node = head->first;
 		if (node) {
@@ -140,12 +137,6 @@ static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s)
 			return node;
 		}
 		raw_spin_unlock(&head->lock);
-next_cpu:
-		cpu = cpumask_next(cpu, cpu_possible_mask);
-		if (cpu >= nr_cpu_ids)
-			cpu = 0;
-		if (cpu == orig_cpu)
-			break;
 	}
 
 	/* per cpu lists are all empty, try extralist */
@@ -164,13 +155,12 @@ ___pcpu_freelist_pop_nmi(struct pcpu_freelist *s)
 {
 	struct pcpu_freelist_head *head;
 	struct pcpu_freelist_node *node;
-	int orig_cpu, cpu;
+	int cpu;
 
-	orig_cpu = cpu = raw_smp_processor_id();
-	while (1) {
+	for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) {
 		head = per_cpu_ptr(s->freelist, cpu);
 		if (!READ_ONCE(head->first))
-			goto next_cpu;
+			continue;
 		if (raw_spin_trylock(&head->lock)) {
 			node = head->first;
 			if (node) {
@@ -180,12 +170,6 @@ ___pcpu_freelist_pop_nmi(struct pcpu_freelist *s)
 			}
 			raw_spin_unlock(&head->lock);
 		}
-next_cpu:
-		cpu = cpumask_next(cpu, cpu_possible_mask);
-		if (cpu >= nr_cpu_ids)
-			cpu = 0;
-		if (cpu == orig_cpu)
-			break;
 	}
 
 	/* cannot pop from per cpu lists, try extralist */
diff --git a/kernel/bpf/queue_stack_maps.c b/kernel/bpf/queue_stack_maps.c
index a1c0794ae49d..8a5e060de63b 100644
--- a/kernel/bpf/queue_stack_maps.c
+++ b/kernel/bpf/queue_stack_maps.c
@@ -78,8 +78,6 @@ static struct bpf_map *queue_stack_map_alloc(union bpf_attr *attr)
 	if (!qs)
 		return ERR_PTR(-ENOMEM);
 
-	memset(qs, 0, sizeof(*qs));
-
 	bpf_map_init_from_attr(&qs->map, attr);
 
 	qs->size = size;
diff --git a/kernel/bpf/ringbuf.c b/kernel/bpf/ringbuf.c
index ded4faeca192..9e832acf4692 100644
--- a/kernel/bpf/ringbuf.c
+++ b/kernel/bpf/ringbuf.c
@@ -38,10 +38,43 @@ struct bpf_ringbuf {
 	struct page **pages;
 	int nr_pages;
 	spinlock_t spinlock ____cacheline_aligned_in_smp;
-	/* Consumer and producer counters are put into separate pages to allow
-	 * mapping consumer page as r/w, but restrict producer page to r/o.
-	 * This protects producer position from being modified by user-space
-	 * application and ruining in-kernel position tracking.
+	/* For user-space producer ring buffers, an atomic_t busy bit is used
+	 * to synchronize access to the ring buffers in the kernel, rather than
+	 * the spinlock that is used for kernel-producer ring buffers. This is
+	 * done because the ring buffer must hold a lock across a BPF program's
+	 * callback:
+	 *
+	 *    __bpf_user_ringbuf_peek() // lock acquired
+	 * -> program callback_fn()
+	 * -> __bpf_user_ringbuf_sample_release() // lock released
+	 *
+	 * It is unsafe and incorrect to hold an IRQ spinlock across what could
+	 * be a long execution window, so we instead simply disallow concurrent
+	 * access to the ring buffer by kernel consumers, and return -EBUSY from
+	 * __bpf_user_ringbuf_peek() if the busy bit is held by another task.
+	 */
+	atomic_t busy ____cacheline_aligned_in_smp;
+	/* Consumer and producer counters are put into separate pages to
+	 * allow each position to be mapped with different permissions.
+	 * This prevents a user-space application from modifying the
+	 * position and ruining in-kernel tracking. The permissions of the
+	 * pages depend on who is producing samples: user-space or the
+	 * kernel.
+	 *
+	 * Kernel-producer
+	 * ---------------
+	 * The producer position and data pages are mapped as r/o in
+	 * userspace. For this approach, bits in the header of samples are
+	 * used to signal to user-space, and to other producers, whether a
+	 * sample is currently being written.
+	 *
+	 * User-space producer
+	 * -------------------
+	 * Only the page containing the consumer position is mapped r/o in
+	 * user-space. User-space producers also use bits of the header to
+	 * communicate to the kernel, but the kernel must carefully check and
+	 * validate each sample to ensure that they're correctly formatted, and
+	 * fully contained within the ring buffer.
 	 */
 	unsigned long consumer_pos __aligned(PAGE_SIZE);
 	unsigned long producer_pos __aligned(PAGE_SIZE);
@@ -116,7 +149,7 @@ static struct bpf_ringbuf *bpf_ringbuf_area_alloc(size_t data_sz, int numa_node)
 err_free_pages:
 	for (i = 0; i < nr_pages; i++)
 		__free_page(pages[i]);
-	kvfree(pages);
+	bpf_map_area_free(pages);
 	return NULL;
 }
 
@@ -136,6 +169,7 @@ static struct bpf_ringbuf *bpf_ringbuf_alloc(size_t data_sz, int numa_node)
 		return NULL;
 
 	spin_lock_init(&rb->spinlock);
+	atomic_set(&rb->busy, 0);
 	init_waitqueue_head(&rb->waitq);
 	init_irq_work(&rb->work, bpf_ringbuf_notify);
 
@@ -164,7 +198,7 @@ static struct bpf_map *ringbuf_map_alloc(union bpf_attr *attr)
 		return ERR_PTR(-E2BIG);
 #endif
 
-	rb_map = kzalloc(sizeof(*rb_map), GFP_USER | __GFP_ACCOUNT);
+	rb_map = bpf_map_area_alloc(sizeof(*rb_map), NUMA_NO_NODE);
 	if (!rb_map)
 		return ERR_PTR(-ENOMEM);
 
@@ -172,7 +206,7 @@ static struct bpf_map *ringbuf_map_alloc(union bpf_attr *attr)
 
 	rb_map->rb = bpf_ringbuf_alloc(attr->max_entries, rb_map->map.numa_node);
 	if (!rb_map->rb) {
-		kfree(rb_map);
+		bpf_map_area_free(rb_map);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -190,7 +224,7 @@ static void bpf_ringbuf_free(struct bpf_ringbuf *rb)
 	vunmap(rb);
 	for (i = 0; i < nr_pages; i++)
 		__free_page(pages[i]);
-	kvfree(pages);
+	bpf_map_area_free(pages);
 }
 
 static void ringbuf_map_free(struct bpf_map *map)
@@ -199,7 +233,7 @@ static void ringbuf_map_free(struct bpf_map *map)
 
 	rb_map = container_of(map, struct bpf_ringbuf_map, map);
 	bpf_ringbuf_free(rb_map->rb);
-	kfree(rb_map);
+	bpf_map_area_free(rb_map);
 }
 
 static void *ringbuf_map_lookup_elem(struct bpf_map *map, void *key)
@@ -224,7 +258,7 @@ static int ringbuf_map_get_next_key(struct bpf_map *map, void *key,
 	return -ENOTSUPP;
 }
 
-static int ringbuf_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
+static int ringbuf_map_mmap_kern(struct bpf_map *map, struct vm_area_struct *vma)
 {
 	struct bpf_ringbuf_map *rb_map;
 
@@ -242,6 +276,26 @@ static int ringbuf_map_mmap(struct bpf_map *map, struct vm_area_struct *vma)
 				   vma->vm_pgoff + RINGBUF_PGOFF);
 }
 
+static int ringbuf_map_mmap_user(struct bpf_map *map, struct vm_area_struct *vma)
+{
+	struct bpf_ringbuf_map *rb_map;
+
+	rb_map = container_of(map, struct bpf_ringbuf_map, map);
+
+	if (vma->vm_flags & VM_WRITE) {
+		if (vma->vm_pgoff == 0)
+			/* Disallow writable mappings to the consumer pointer,
+			 * and allow writable mappings to both the producer
+			 * position, and the ring buffer data itself.
+			 */
+			return -EPERM;
+	} else {
+		vma->vm_flags &= ~VM_MAYWRITE;
+	}
+	/* remap_vmalloc_range() checks size and offset constraints */
+	return remap_vmalloc_range(vma, rb_map->rb, vma->vm_pgoff + RINGBUF_PGOFF);
+}
+
 static unsigned long ringbuf_avail_data_sz(struct bpf_ringbuf *rb)
 {
 	unsigned long cons_pos, prod_pos;
@@ -251,8 +305,13 @@ static unsigned long ringbuf_avail_data_sz(struct bpf_ringbuf *rb)
 	return prod_pos - cons_pos;
 }
 
-static __poll_t ringbuf_map_poll(struct bpf_map *map, struct file *filp,
-				 struct poll_table_struct *pts)
+static u32 ringbuf_total_data_sz(const struct bpf_ringbuf *rb)
+{
+	return rb->mask + 1;
+}
+
+static __poll_t ringbuf_map_poll_kern(struct bpf_map *map, struct file *filp,
+				      struct poll_table_struct *pts)
 {
 	struct bpf_ringbuf_map *rb_map;
 
@@ -264,13 +323,26 @@ static __poll_t ringbuf_map_poll(struct bpf_map *map, struct file *filp,
 	return 0;
 }
 
+static __poll_t ringbuf_map_poll_user(struct bpf_map *map, struct file *filp,
+				      struct poll_table_struct *pts)
+{
+	struct bpf_ringbuf_map *rb_map;
+
+	rb_map = container_of(map, struct bpf_ringbuf_map, map);
+	poll_wait(filp, &rb_map->rb->waitq, pts);
+
+	if (ringbuf_avail_data_sz(rb_map->rb) < ringbuf_total_data_sz(rb_map->rb))
+		return EPOLLOUT | EPOLLWRNORM;
+	return 0;
+}
+
 BTF_ID_LIST_SINGLE(ringbuf_map_btf_ids, struct, bpf_ringbuf_map)
 const struct bpf_map_ops ringbuf_map_ops = {
 	.map_meta_equal = bpf_map_meta_equal,
 	.map_alloc = ringbuf_map_alloc,
 	.map_free = ringbuf_map_free,
-	.map_mmap = ringbuf_map_mmap,
-	.map_poll = ringbuf_map_poll,
+	.map_mmap = ringbuf_map_mmap_kern,
+	.map_poll = ringbuf_map_poll_kern,
 	.map_lookup_elem = ringbuf_map_lookup_elem,
 	.map_update_elem = ringbuf_map_update_elem,
 	.map_delete_elem = ringbuf_map_delete_elem,
@@ -278,6 +350,20 @@ const struct bpf_map_ops ringbuf_map_ops = {
 	.map_btf_id = &ringbuf_map_btf_ids[0],
 };
 
+BTF_ID_LIST_SINGLE(user_ringbuf_map_btf_ids, struct, bpf_ringbuf_map)
+const struct bpf_map_ops user_ringbuf_map_ops = {
+	.map_meta_equal = bpf_map_meta_equal,
+	.map_alloc = ringbuf_map_alloc,
+	.map_free = ringbuf_map_free,
+	.map_mmap = ringbuf_map_mmap_user,
+	.map_poll = ringbuf_map_poll_user,
+	.map_lookup_elem = ringbuf_map_lookup_elem,
+	.map_update_elem = ringbuf_map_update_elem,
+	.map_delete_elem = ringbuf_map_delete_elem,
+	.map_get_next_key = ringbuf_map_get_next_key,
+	.map_btf_id = &user_ringbuf_map_btf_ids[0],
+};
+
 /* Given pointer to ring buffer record metadata and struct bpf_ringbuf itself,
  * calculate offset from record metadata to ring buffer in pages, rounded
  * down. This page offset is stored as part of record metadata and allows to
@@ -312,7 +398,7 @@ static void *__bpf_ringbuf_reserve(struct bpf_ringbuf *rb, u64 size)
 		return NULL;
 
 	len = round_up(size + BPF_RINGBUF_HDR_SZ, 8);
-	if (len > rb->mask + 1)
+	if (len > ringbuf_total_data_sz(rb))
 		return NULL;
 
 	cons_pos = smp_load_acquire(&rb->consumer_pos);
@@ -459,7 +545,7 @@ BPF_CALL_2(bpf_ringbuf_query, struct bpf_map *, map, u64, flags)
 	case BPF_RB_AVAIL_DATA:
 		return ringbuf_avail_data_sz(rb);
 	case BPF_RB_RING_SIZE:
-		return rb->mask + 1;
+		return ringbuf_total_data_sz(rb);
 	case BPF_RB_CONS_POS:
 		return smp_load_acquire(&rb->consumer_pos);
 	case BPF_RB_PROD_POS:
@@ -553,3 +639,138 @@ const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto = {
 	.arg1_type	= ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_RINGBUF | OBJ_RELEASE,
 	.arg2_type	= ARG_ANYTHING,
 };
+
+static int __bpf_user_ringbuf_peek(struct bpf_ringbuf *rb, void **sample, u32 *size)
+{
+	int err;
+	u32 hdr_len, sample_len, total_len, flags, *hdr;
+	u64 cons_pos, prod_pos;
+
+	/* Synchronizes with smp_store_release() in user-space producer. */
+	prod_pos = smp_load_acquire(&rb->producer_pos);
+	if (prod_pos % 8)
+		return -EINVAL;
+
+	/* Synchronizes with smp_store_release() in __bpf_user_ringbuf_sample_release() */
+	cons_pos = smp_load_acquire(&rb->consumer_pos);
+	if (cons_pos >= prod_pos)
+		return -ENODATA;
+
+	hdr = (u32 *)((uintptr_t)rb->data + (uintptr_t)(cons_pos & rb->mask));
+	/* Synchronizes with smp_store_release() in user-space producer. */
+	hdr_len = smp_load_acquire(hdr);
+	flags = hdr_len & (BPF_RINGBUF_BUSY_BIT | BPF_RINGBUF_DISCARD_BIT);
+	sample_len = hdr_len & ~flags;
+	total_len = round_up(sample_len + BPF_RINGBUF_HDR_SZ, 8);
+
+	/* The sample must fit within the region advertised by the producer position. */
+	if (total_len > prod_pos - cons_pos)
+		return -EINVAL;
+
+	/* The sample must fit within the data region of the ring buffer. */
+	if (total_len > ringbuf_total_data_sz(rb))
+		return -E2BIG;
+
+	/* The sample must fit into a struct bpf_dynptr. */
+	err = bpf_dynptr_check_size(sample_len);
+	if (err)
+		return -E2BIG;
+
+	if (flags & BPF_RINGBUF_DISCARD_BIT) {
+		/* If the discard bit is set, the sample should be skipped.
+		 *
+		 * Update the consumer pos, and return -EAGAIN so the caller
+		 * knows to skip this sample and try to read the next one.
+		 */
+		smp_store_release(&rb->consumer_pos, cons_pos + total_len);
+		return -EAGAIN;
+	}
+
+	if (flags & BPF_RINGBUF_BUSY_BIT)
+		return -ENODATA;
+
+	*sample = (void *)((uintptr_t)rb->data +
+			   (uintptr_t)((cons_pos + BPF_RINGBUF_HDR_SZ) & rb->mask));
+	*size = sample_len;
+	return 0;
+}
+
+static void __bpf_user_ringbuf_sample_release(struct bpf_ringbuf *rb, size_t size, u64 flags)
+{
+	u64 consumer_pos;
+	u32 rounded_size = round_up(size + BPF_RINGBUF_HDR_SZ, 8);
+
+	/* Using smp_load_acquire() is unnecessary here, as the busy-bit
+	 * prevents another task from writing to consumer_pos after it was read
+	 * by this task with smp_load_acquire() in __bpf_user_ringbuf_peek().
+	 */
+	consumer_pos = rb->consumer_pos;
+	 /* Synchronizes with smp_load_acquire() in user-space producer. */
+	smp_store_release(&rb->consumer_pos, consumer_pos + rounded_size);
+}
+
+BPF_CALL_4(bpf_user_ringbuf_drain, struct bpf_map *, map,
+	   void *, callback_fn, void *, callback_ctx, u64, flags)
+{
+	struct bpf_ringbuf *rb;
+	long samples, discarded_samples = 0, ret = 0;
+	bpf_callback_t callback = (bpf_callback_t)callback_fn;
+	u64 wakeup_flags = BPF_RB_NO_WAKEUP | BPF_RB_FORCE_WAKEUP;
+	int busy = 0;
+
+	if (unlikely(flags & ~wakeup_flags))
+		return -EINVAL;
+
+	rb = container_of(map, struct bpf_ringbuf_map, map)->rb;
+
+	/* If another consumer is already consuming a sample, wait for them to finish. */
+	if (!atomic_try_cmpxchg(&rb->busy, &busy, 1))
+		return -EBUSY;
+
+	for (samples = 0; samples < BPF_MAX_USER_RINGBUF_SAMPLES && ret == 0; samples++) {
+		int err;
+		u32 size;
+		void *sample;
+		struct bpf_dynptr_kern dynptr;
+
+		err = __bpf_user_ringbuf_peek(rb, &sample, &size);
+		if (err) {
+			if (err == -ENODATA) {
+				break;
+			} else if (err == -EAGAIN) {
+				discarded_samples++;
+				continue;
+			} else {
+				ret = err;
+				goto schedule_work_return;
+			}
+		}
+
+		bpf_dynptr_init(&dynptr, sample, BPF_DYNPTR_TYPE_LOCAL, 0, size);
+		ret = callback((uintptr_t)&dynptr, (uintptr_t)callback_ctx, 0, 0, 0);
+		__bpf_user_ringbuf_sample_release(rb, size, flags);
+	}
+	ret = samples - discarded_samples;
+
+schedule_work_return:
+	/* Prevent the clearing of the busy-bit from being reordered before the
+	 * storing of any rb consumer or producer positions.
+	 */
+	smp_mb__before_atomic();
+	atomic_set(&rb->busy, 0);
+
+	if (flags & BPF_RB_FORCE_WAKEUP)
+		irq_work_queue(&rb->work);
+	else if (!(flags & BPF_RB_NO_WAKEUP) && samples > 0)
+		irq_work_queue(&rb->work);
+	return ret;
+}
+
+const struct bpf_func_proto bpf_user_ringbuf_drain_proto = {
+	.func		= bpf_user_ringbuf_drain,
+	.ret_type	= RET_INTEGER,
+	.arg1_type	= ARG_CONST_MAP_PTR,
+	.arg2_type	= ARG_PTR_TO_FUNC,
+	.arg3_type	= ARG_PTR_TO_STACK_OR_NULL,
+	.arg4_type	= ARG_ANYTHING,
+};
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 27760627370d..7b373a5e861f 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -598,7 +598,7 @@ void bpf_map_free_kptrs(struct bpf_map *map, void *map_value)
 		if (off_desc->type == BPF_KPTR_UNREF) {
 			u64 *p = (u64 *)btf_id_ptr;
 
-			WRITE_ONCE(p, 0);
+			WRITE_ONCE(*p, 0);
 			continue;
 		}
 		old_ptr = xchg(btf_id_ptr, 0);
@@ -638,7 +638,10 @@ static void __bpf_map_put(struct bpf_map *map, bool do_idr_lock)
 		bpf_map_free_id(map, do_idr_lock);
 		btf_put(map->btf);
 		INIT_WORK(&map->work, bpf_map_free_deferred);
-		schedule_work(&map->work);
+		/* Avoid spawning kworkers, since they all might contend
+		 * for the same mutex like slab_mutex.
+		 */
+		queue_work(system_unbound_wq, &map->work);
 	}
 }
 
@@ -1046,7 +1049,8 @@ static int map_check_btf(struct bpf_map *map, const struct btf *btf,
 		}
 		if (map->map_type != BPF_MAP_TYPE_HASH &&
 		    map->map_type != BPF_MAP_TYPE_LRU_HASH &&
-		    map->map_type != BPF_MAP_TYPE_ARRAY) {
+		    map->map_type != BPF_MAP_TYPE_ARRAY &&
+		    map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) {
 			ret = -EOPNOTSUPP;
 			goto free_map_tab;
 		}
@@ -1413,19 +1417,14 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
 	}
 
 	value_size = bpf_map_value_size(map);
-
-	err = -ENOMEM;
-	value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
-	if (!value)
+	value = kvmemdup_bpfptr(uvalue, value_size);
+	if (IS_ERR(value)) {
+		err = PTR_ERR(value);
 		goto free_key;
-
-	err = -EFAULT;
-	if (copy_from_bpfptr(value, uvalue, value_size) != 0)
-		goto free_value;
+	}
 
 	err = bpf_map_update_value(map, f, key, value, attr->flags);
 
-free_value:
 	kvfree(value);
 free_key:
 	kvfree(key);
@@ -1437,9 +1436,9 @@ err_put:
 
 #define BPF_MAP_DELETE_ELEM_LAST_FIELD key
 
-static int map_delete_elem(union bpf_attr *attr)
+static int map_delete_elem(union bpf_attr *attr, bpfptr_t uattr)
 {
-	void __user *ukey = u64_to_user_ptr(attr->key);
+	bpfptr_t ukey = make_bpfptr(attr->key, uattr.is_kernel);
 	int ufd = attr->map_fd;
 	struct bpf_map *map;
 	struct fd f;
@@ -1459,7 +1458,7 @@ static int map_delete_elem(union bpf_attr *attr)
 		goto err_put;
 	}
 
-	key = __bpf_copy_key(ukey, map->key_size);
+	key = ___bpf_copy_key(ukey, map->key_size);
 	if (IS_ERR(key)) {
 		err = PTR_ERR(key);
 		goto err_put;
@@ -2094,6 +2093,17 @@ struct bpf_prog_kstats {
 	u64 misses;
 };
 
+void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog)
+{
+	struct bpf_prog_stats *stats;
+	unsigned int flags;
+
+	stats = this_cpu_ptr(prog->stats);
+	flags = u64_stats_update_begin_irqsave(&stats->syncp);
+	u64_stats_inc(&stats->misses);
+	u64_stats_update_end_irqrestore(&stats->syncp, flags);
+}
+
 static void bpf_prog_get_stats(const struct bpf_prog *prog,
 			       struct bpf_prog_kstats *stats)
 {
@@ -4395,7 +4405,9 @@ static int bpf_task_fd_query(const union bpf_attr *attr,
 	if (attr->task_fd_query.flags != 0)
 		return -EINVAL;
 
+	rcu_read_lock();
 	task = get_pid_task(find_vpid(pid), PIDTYPE_PID);
+	rcu_read_unlock();
 	if (!task)
 		return -ENOENT;
 
@@ -4941,7 +4953,7 @@ static int __sys_bpf(int cmd, bpfptr_t uattr, unsigned int size)
 		err = map_update_elem(&attr, uattr);
 		break;
 	case BPF_MAP_DELETE_ELEM:
-		err = map_delete_elem(&attr);
+		err = map_delete_elem(&attr, uattr);
 		break;
 	case BPF_MAP_GET_NEXT_KEY:
 		err = map_get_next_key(&attr);
@@ -5073,8 +5085,10 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
 {
 	switch (cmd) {
 	case BPF_MAP_CREATE:
+	case BPF_MAP_DELETE_ELEM:
 	case BPF_MAP_UPDATE_ELEM:
 	case BPF_MAP_FREEZE:
+	case BPF_MAP_GET_FD_BY_ID:
 	case BPF_PROG_LOAD:
 	case BPF_BTF_LOAD:
 	case BPF_LINK_CREATE:
diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c
index 8c921799def4..67e03e1833ba 100644
--- a/kernel/bpf/task_iter.c
+++ b/kernel/bpf/task_iter.c
@@ -10,8 +10,17 @@
 #include <linux/btf_ids.h>
 #include "mmap_unlock_work.h"
 
+static const char * const iter_task_type_names[] = {
+	"ALL",
+	"TID",
+	"PID",
+};
+
 struct bpf_iter_seq_task_common {
 	struct pid_namespace *ns;
+	enum bpf_iter_task_type	type;
+	u32 pid;
+	u32 pid_visiting;
 };
 
 struct bpf_iter_seq_task_info {
@@ -22,18 +31,115 @@ struct bpf_iter_seq_task_info {
 	u32 tid;
 };
 
-static struct task_struct *task_seq_get_next(struct pid_namespace *ns,
+static struct task_struct *task_group_seq_get_next(struct bpf_iter_seq_task_common *common,
+						   u32 *tid,
+						   bool skip_if_dup_files)
+{
+	struct task_struct *task, *next_task;
+	struct pid *pid;
+	u32 saved_tid;
+
+	if (!*tid) {
+		/* The first time, the iterator calls this function. */
+		pid = find_pid_ns(common->pid, common->ns);
+		if (!pid)
+			return NULL;
+
+		task = get_pid_task(pid, PIDTYPE_TGID);
+		if (!task)
+			return NULL;
+
+		*tid = common->pid;
+		common->pid_visiting = common->pid;
+
+		return task;
+	}
+
+	/* If the control returns to user space and comes back to the
+	 * kernel again, *tid and common->pid_visiting should be the
+	 * same for task_seq_start() to pick up the correct task.
+	 */
+	if (*tid == common->pid_visiting) {
+		pid = find_pid_ns(common->pid_visiting, common->ns);
+		task = get_pid_task(pid, PIDTYPE_PID);
+
+		return task;
+	}
+
+	pid = find_pid_ns(common->pid_visiting, common->ns);
+	if (!pid)
+		return NULL;
+
+	task = get_pid_task(pid, PIDTYPE_PID);
+	if (!task)
+		return NULL;
+
+retry:
+	if (!pid_alive(task)) {
+		put_task_struct(task);
+		return NULL;
+	}
+
+	next_task = next_thread(task);
+	put_task_struct(task);
+	if (!next_task)
+		return NULL;
+
+	saved_tid = *tid;
+	*tid = __task_pid_nr_ns(next_task, PIDTYPE_PID, common->ns);
+	if (!*tid || *tid == common->pid) {
+		/* Run out of tasks of a process.  The tasks of a
+		 * thread_group are linked as circular linked list.
+		 */
+		*tid = saved_tid;
+		return NULL;
+	}
+
+	get_task_struct(next_task);
+	common->pid_visiting = *tid;
+
+	if (skip_if_dup_files && task->files == task->group_leader->files) {
+		task = next_task;
+		goto retry;
+	}
+
+	return next_task;
+}
+
+static struct task_struct *task_seq_get_next(struct bpf_iter_seq_task_common *common,
 					     u32 *tid,
 					     bool skip_if_dup_files)
 {
 	struct task_struct *task = NULL;
 	struct pid *pid;
 
+	if (common->type == BPF_TASK_ITER_TID) {
+		if (*tid && *tid != common->pid)
+			return NULL;
+		rcu_read_lock();
+		pid = find_pid_ns(common->pid, common->ns);
+		if (pid) {
+			task = get_pid_task(pid, PIDTYPE_TGID);
+			*tid = common->pid;
+		}
+		rcu_read_unlock();
+
+		return task;
+	}
+
+	if (common->type == BPF_TASK_ITER_TGID) {
+		rcu_read_lock();
+		task = task_group_seq_get_next(common, tid, skip_if_dup_files);
+		rcu_read_unlock();
+
+		return task;
+	}
+
 	rcu_read_lock();
 retry:
-	pid = find_ge_pid(*tid, ns);
+	pid = find_ge_pid(*tid, common->ns);
 	if (pid) {
-		*tid = pid_nr_ns(pid, ns);
+		*tid = pid_nr_ns(pid, common->ns);
 		task = get_pid_task(pid, PIDTYPE_PID);
 		if (!task) {
 			++*tid;
@@ -56,7 +162,7 @@ static void *task_seq_start(struct seq_file *seq, loff_t *pos)
 	struct bpf_iter_seq_task_info *info = seq->private;
 	struct task_struct *task;
 
-	task = task_seq_get_next(info->common.ns, &info->tid, false);
+	task = task_seq_get_next(&info->common, &info->tid, false);
 	if (!task)
 		return NULL;
 
@@ -73,7 +179,7 @@ static void *task_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 	++*pos;
 	++info->tid;
 	put_task_struct((struct task_struct *)v);
-	task = task_seq_get_next(info->common.ns, &info->tid, false);
+	task = task_seq_get_next(&info->common, &info->tid, false);
 	if (!task)
 		return NULL;
 
@@ -117,6 +223,41 @@ static void task_seq_stop(struct seq_file *seq, void *v)
 		put_task_struct((struct task_struct *)v);
 }
 
+static int bpf_iter_attach_task(struct bpf_prog *prog,
+				union bpf_iter_link_info *linfo,
+				struct bpf_iter_aux_info *aux)
+{
+	unsigned int flags;
+	struct pid *pid;
+	pid_t tgid;
+
+	if ((!!linfo->task.tid + !!linfo->task.pid + !!linfo->task.pid_fd) > 1)
+		return -EINVAL;
+
+	aux->task.type = BPF_TASK_ITER_ALL;
+	if (linfo->task.tid != 0) {
+		aux->task.type = BPF_TASK_ITER_TID;
+		aux->task.pid = linfo->task.tid;
+	}
+	if (linfo->task.pid != 0) {
+		aux->task.type = BPF_TASK_ITER_TGID;
+		aux->task.pid = linfo->task.pid;
+	}
+	if (linfo->task.pid_fd != 0) {
+		aux->task.type = BPF_TASK_ITER_TGID;
+
+		pid = pidfd_get_pid(linfo->task.pid_fd, &flags);
+		if (IS_ERR(pid))
+			return PTR_ERR(pid);
+
+		tgid = pid_nr_ns(pid, task_active_pid_ns(current));
+		aux->task.pid = tgid;
+		put_pid(pid);
+	}
+
+	return 0;
+}
+
 static const struct seq_operations task_seq_ops = {
 	.start	= task_seq_start,
 	.next	= task_seq_next,
@@ -137,8 +278,7 @@ struct bpf_iter_seq_task_file_info {
 static struct file *
 task_file_seq_get_next(struct bpf_iter_seq_task_file_info *info)
 {
-	struct pid_namespace *ns = info->common.ns;
-	u32 curr_tid = info->tid;
+	u32 saved_tid = info->tid;
 	struct task_struct *curr_task;
 	unsigned int curr_fd = info->fd;
 
@@ -151,21 +291,18 @@ again:
 		curr_task = info->task;
 		curr_fd = info->fd;
 	} else {
-                curr_task = task_seq_get_next(ns, &curr_tid, true);
+		curr_task = task_seq_get_next(&info->common, &info->tid, true);
                 if (!curr_task) {
                         info->task = NULL;
-                        info->tid = curr_tid;
                         return NULL;
                 }
 
-                /* set info->task and info->tid */
+		/* set info->task */
 		info->task = curr_task;
-		if (curr_tid == info->tid) {
+		if (saved_tid == info->tid)
 			curr_fd = info->fd;
-		} else {
-			info->tid = curr_tid;
+		else
 			curr_fd = 0;
-		}
 	}
 
 	rcu_read_lock();
@@ -186,9 +323,15 @@ again:
 	/* the current task is done, go to the next task */
 	rcu_read_unlock();
 	put_task_struct(curr_task);
+
+	if (info->common.type == BPF_TASK_ITER_TID) {
+		info->task = NULL;
+		return NULL;
+	}
+
 	info->task = NULL;
 	info->fd = 0;
-	curr_tid = ++(info->tid);
+	saved_tid = ++(info->tid);
 	goto again;
 }
 
@@ -269,6 +412,9 @@ static int init_seq_pidns(void *priv_data, struct bpf_iter_aux_info *aux)
 	struct bpf_iter_seq_task_common *common = priv_data;
 
 	common->ns = get_pid_ns(task_active_pid_ns(current));
+	common->type = aux->task.type;
+	common->pid = aux->task.pid;
+
 	return 0;
 }
 
@@ -307,11 +453,10 @@ enum bpf_task_vma_iter_find_op {
 static struct vm_area_struct *
 task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info)
 {
-	struct pid_namespace *ns = info->common.ns;
 	enum bpf_task_vma_iter_find_op op;
 	struct vm_area_struct *curr_vma;
 	struct task_struct *curr_task;
-	u32 curr_tid = info->tid;
+	u32 saved_tid = info->tid;
 
 	/* If this function returns a non-NULL vma, it holds a reference to
 	 * the task_struct, and holds read lock on vma->mm->mmap_lock.
@@ -371,14 +516,13 @@ task_vma_seq_get_next(struct bpf_iter_seq_task_vma_info *info)
 		}
 	} else {
 again:
-		curr_task = task_seq_get_next(ns, &curr_tid, true);
+		curr_task = task_seq_get_next(&info->common, &info->tid, true);
 		if (!curr_task) {
-			info->tid = curr_tid + 1;
+			info->tid++;
 			goto finish;
 		}
 
-		if (curr_tid != info->tid) {
-			info->tid = curr_tid;
+		if (saved_tid != info->tid) {
 			/* new task, process the first vma */
 			op = task_vma_iter_first_vma;
 		} else {
@@ -430,9 +574,12 @@ again:
 	return curr_vma;
 
 next_task:
+	if (info->common.type == BPF_TASK_ITER_TID)
+		goto finish;
+
 	put_task_struct(curr_task);
 	info->task = NULL;
-	curr_tid++;
+	info->tid++;
 	goto again;
 
 finish:
@@ -531,8 +678,33 @@ static const struct bpf_iter_seq_info task_seq_info = {
 	.seq_priv_size		= sizeof(struct bpf_iter_seq_task_info),
 };
 
+static int bpf_iter_fill_link_info(const struct bpf_iter_aux_info *aux, struct bpf_link_info *info)
+{
+	switch (aux->task.type) {
+	case BPF_TASK_ITER_TID:
+		info->iter.task.tid = aux->task.pid;
+		break;
+	case BPF_TASK_ITER_TGID:
+		info->iter.task.pid = aux->task.pid;
+		break;
+	default:
+		break;
+	}
+	return 0;
+}
+
+static void bpf_iter_task_show_fdinfo(const struct bpf_iter_aux_info *aux, struct seq_file *seq)
+{
+	seq_printf(seq, "task_type:\t%s\n", iter_task_type_names[aux->task.type]);
+	if (aux->task.type == BPF_TASK_ITER_TID)
+		seq_printf(seq, "tid:\t%u\n", aux->task.pid);
+	else if (aux->task.type == BPF_TASK_ITER_TGID)
+		seq_printf(seq, "pid:\t%u\n", aux->task.pid);
+}
+
 static struct bpf_iter_reg task_reg_info = {
 	.target			= "task",
+	.attach_target		= bpf_iter_attach_task,
 	.feature		= BPF_ITER_RESCHED,
 	.ctx_arg_info_size	= 1,
 	.ctx_arg_info		= {
@@ -540,6 +712,8 @@ static struct bpf_iter_reg task_reg_info = {
 		  PTR_TO_BTF_ID_OR_NULL },
 	},
 	.seq_info		= &task_seq_info,
+	.fill_link_info		= bpf_iter_fill_link_info,
+	.show_fdinfo		= bpf_iter_task_show_fdinfo,
 };
 
 static const struct bpf_iter_seq_info task_file_seq_info = {
@@ -551,6 +725,7 @@ static const struct bpf_iter_seq_info task_file_seq_info = {
 
 static struct bpf_iter_reg task_file_reg_info = {
 	.target			= "task_file",
+	.attach_target		= bpf_iter_attach_task,
 	.feature		= BPF_ITER_RESCHED,
 	.ctx_arg_info_size	= 2,
 	.ctx_arg_info		= {
@@ -560,6 +735,8 @@ static struct bpf_iter_reg task_file_reg_info = {
 		  PTR_TO_BTF_ID_OR_NULL },
 	},
 	.seq_info		= &task_file_seq_info,
+	.fill_link_info		= bpf_iter_fill_link_info,
+	.show_fdinfo		= bpf_iter_task_show_fdinfo,
 };
 
 static const struct bpf_iter_seq_info task_vma_seq_info = {
@@ -571,6 +748,7 @@ static const struct bpf_iter_seq_info task_vma_seq_info = {
 
 static struct bpf_iter_reg task_vma_reg_info = {
 	.target			= "task_vma",
+	.attach_target		= bpf_iter_attach_task,
 	.feature		= BPF_ITER_RESCHED,
 	.ctx_arg_info_size	= 2,
 	.ctx_arg_info		= {
@@ -580,6 +758,8 @@ static struct bpf_iter_reg task_vma_reg_info = {
 		  PTR_TO_BTF_ID_OR_NULL },
 	},
 	.seq_info		= &task_vma_seq_info,
+	.fill_link_info		= bpf_iter_fill_link_info,
+	.show_fdinfo		= bpf_iter_task_show_fdinfo,
 };
 
 BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start,
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index ff87e38af8a7..bf0906e1e2b9 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -116,22 +116,6 @@ bool bpf_prog_has_trampoline(const struct bpf_prog *prog)
 		(ptype == BPF_PROG_TYPE_LSM && eatype == BPF_LSM_MAC);
 }
 
-void *bpf_jit_alloc_exec_page(void)
-{
-	void *image;
-
-	image = bpf_jit_alloc_exec(PAGE_SIZE);
-	if (!image)
-		return NULL;
-
-	set_vm_flush_reset_perms(image);
-	/* Keep image as writeable. The alternative is to keep flipping ro/rw
-	 * every time new program is attached or detached.
-	 */
-	set_memory_x((long)image, 1);
-	return image;
-}
-
 void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym)
 {
 	ksym->start = (unsigned long) data;
@@ -404,9 +388,10 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx)
 		goto out_free_im;
 
 	err = -ENOMEM;
-	im->image = image = bpf_jit_alloc_exec_page();
+	im->image = image = bpf_jit_alloc_exec(PAGE_SIZE);
 	if (!image)
 		goto out_uncharge;
+	set_vm_flush_reset_perms(image);
 
 	err = percpu_ref_init(&im->pcref, __bpf_tramp_image_release, 0, GFP_KERNEL);
 	if (err)
@@ -483,6 +468,9 @@ again:
 	if (err < 0)
 		goto out;
 
+	set_memory_ro((long)im->image, 1);
+	set_memory_x((long)im->image, 1);
+
 	WARN_ON(tr->cur_image && tr->selector == 0);
 	WARN_ON(!tr->cur_image && tr->selector);
 	if (tr->cur_image)
@@ -863,17 +851,6 @@ static __always_inline u64 notrace bpf_prog_start_time(void)
 	return start;
 }
 
-static void notrace inc_misses_counter(struct bpf_prog *prog)
-{
-	struct bpf_prog_stats *stats;
-	unsigned int flags;
-
-	stats = this_cpu_ptr(prog->stats);
-	flags = u64_stats_update_begin_irqsave(&stats->syncp);
-	u64_stats_inc(&stats->misses);
-	u64_stats_update_end_irqrestore(&stats->syncp, flags);
-}
-
 /* The logic is similar to bpf_prog_run(), but with an explicit
  * rcu_read_lock() and migrate_disable() which are required
  * for the trampoline. The macro is split into
@@ -895,8 +872,8 @@ u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *ru
 
 	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
 
-	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
-		inc_misses_counter(prog);
+	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
+		bpf_prog_inc_misses_counter(prog);
 		return 0;
 	}
 	return bpf_prog_start_time();
@@ -930,7 +907,7 @@ void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_
 	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
 
 	update_prog_stats(prog, start);
-	__this_cpu_dec(*(prog->active));
+	this_cpu_dec(*(prog->active));
 	migrate_enable();
 	rcu_read_unlock();
 }
@@ -966,8 +943,8 @@ u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_r
 	migrate_disable();
 	might_fault();
 
-	if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
-		inc_misses_counter(prog);
+	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
+		bpf_prog_inc_misses_counter(prog);
 		return 0;
 	}
 
@@ -982,11 +959,34 @@ void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
 	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
 
 	update_prog_stats(prog, start);
-	__this_cpu_dec(*(prog->active));
+	this_cpu_dec(*(prog->active));
 	migrate_enable();
 	rcu_read_unlock_trace();
 }
 
+u64 notrace __bpf_prog_enter_struct_ops(struct bpf_prog *prog,
+					struct bpf_tramp_run_ctx *run_ctx)
+	__acquires(RCU)
+{
+	rcu_read_lock();
+	migrate_disable();
+
+	run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
+
+	return bpf_prog_start_time();
+}
+
+void notrace __bpf_prog_exit_struct_ops(struct bpf_prog *prog, u64 start,
+					struct bpf_tramp_run_ctx *run_ctx)
+	__releases(RCU)
+{
+	bpf_reset_run_ctx(run_ctx->saved_run_ctx);
+
+	update_prog_stats(prog, start);
+	migrate_enable();
+	rcu_read_unlock();
+}
+
 void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr)
 {
 	percpu_ref_get(&tr->pcref);
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 3eadb14e090b..6f6d2d511c06 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -23,6 +23,7 @@
 #include <linux/error-injection.h>
 #include <linux/bpf_lsm.h>
 #include <linux/btf_ids.h>
+#include <linux/poison.h>
 
 #include "disasm.h"
 
@@ -370,6 +371,7 @@ __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
 	bpf_verifier_vlog(log, fmt, args);
 	va_end(args);
 }
+EXPORT_SYMBOL_GPL(bpf_log);
 
 static const char *ltrim(const char *s)
 {
@@ -427,6 +429,7 @@ static void verbose_invalid_scalar(struct bpf_verifier_env *env,
 
 static bool type_is_pkt_pointer(enum bpf_reg_type type)
 {
+	type = base_type(type);
 	return type == PTR_TO_PACKET ||
 	       type == PTR_TO_PACKET_META;
 }
@@ -456,10 +459,9 @@ static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
 
 static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
 {
-	return base_type(type) == PTR_TO_SOCKET ||
-		base_type(type) == PTR_TO_TCP_SOCK ||
-		base_type(type) == PTR_TO_MEM ||
-		base_type(type) == PTR_TO_BTF_ID;
+	type = base_type(type);
+	return type == PTR_TO_SOCKET || type == PTR_TO_TCP_SOCK ||
+		type == PTR_TO_MEM || type == PTR_TO_BTF_ID;
 }
 
 static bool type_is_rdonly_mem(u32 type)
@@ -467,25 +469,11 @@ static bool type_is_rdonly_mem(u32 type)
 	return type & MEM_RDONLY;
 }
 
-static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
-{
-	return type == ARG_PTR_TO_SOCK_COMMON;
-}
-
 static bool type_may_be_null(u32 type)
 {
 	return type & PTR_MAYBE_NULL;
 }
 
-static bool may_be_acquire_function(enum bpf_func_id func_id)
-{
-	return func_id == BPF_FUNC_sk_lookup_tcp ||
-		func_id == BPF_FUNC_sk_lookup_udp ||
-		func_id == BPF_FUNC_skc_lookup_tcp ||
-		func_id == BPF_FUNC_map_lookup_elem ||
-	        func_id == BPF_FUNC_ringbuf_reserve;
-}
-
 static bool is_acquire_function(enum bpf_func_id func_id,
 				const struct bpf_map *map)
 {
@@ -518,6 +506,26 @@ static bool is_ptr_cast_function(enum bpf_func_id func_id)
 		func_id == BPF_FUNC_skc_to_tcp_request_sock;
 }
 
+static bool is_dynptr_ref_function(enum bpf_func_id func_id)
+{
+	return func_id == BPF_FUNC_dynptr_data;
+}
+
+static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id,
+					const struct bpf_map *map)
+{
+	int ref_obj_uses = 0;
+
+	if (is_ptr_cast_function(func_id))
+		ref_obj_uses++;
+	if (is_acquire_function(func_id, map))
+		ref_obj_uses++;
+	if (is_dynptr_ref_function(func_id))
+		ref_obj_uses++;
+
+	return ref_obj_uses > 1;
+}
+
 static bool is_cmpxchg_insn(const struct bpf_insn *insn)
 {
 	return BPF_CLASS(insn->code) == BPF_STX &&
@@ -555,6 +563,7 @@ static const char *reg_type_str(struct bpf_verifier_env *env,
 		[PTR_TO_BUF]		= "buf",
 		[PTR_TO_FUNC]		= "func",
 		[PTR_TO_MAP_KEY]	= "map_key",
+		[PTR_TO_DYNPTR]		= "dynptr_ptr",
 	};
 
 	if (type & PTR_MAYBE_NULL) {
@@ -773,8 +782,8 @@ static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_
 	return true;
 }
 
-static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
-				     enum bpf_arg_type arg_type)
+bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env,
+			      struct bpf_reg_state *reg)
 {
 	struct bpf_func_state *state = func(env, reg);
 	int spi = get_spi(reg->off);
@@ -790,11 +799,24 @@ static bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env, struct bpf_re
 			return false;
 	}
 
+	return true;
+}
+
+bool is_dynptr_type_expected(struct bpf_verifier_env *env,
+			     struct bpf_reg_state *reg,
+			     enum bpf_arg_type arg_type)
+{
+	struct bpf_func_state *state = func(env, reg);
+	enum bpf_dynptr_type dynptr_type;
+	int spi = get_spi(reg->off);
+
 	/* ARG_PTR_TO_DYNPTR takes any type of dynptr */
 	if (arg_type == ARG_PTR_TO_DYNPTR)
 		return true;
 
-	return state->stack[spi].spilled_ptr.dynptr.type == arg_to_dynptr_type(arg_type);
+	dynptr_type = arg_to_dynptr_type(arg_type);
+
+	return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type;
 }
 
 /* The reg state of a pointer or a bounded scalar was saved when
@@ -1086,6 +1108,7 @@ static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
 	id = ++env->id_gen;
 	state->refs[new_ofs].id = id;
 	state->refs[new_ofs].insn_idx = insn_idx;
+	state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0;
 
 	return id;
 }
@@ -1098,6 +1121,9 @@ static int release_reference_state(struct bpf_func_state *state, int ptr_id)
 	last_idx = state->acquired_refs - 1;
 	for (i = 0; i < state->acquired_refs; i++) {
 		if (state->refs[i].id == ptr_id) {
+			/* Cannot release caller references in callbacks */
+			if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
+				return -EINVAL;
 			if (last_idx && i != last_idx)
 				memcpy(&state->refs[i], &state->refs[last_idx],
 				       sizeof(*state->refs));
@@ -1739,6 +1765,7 @@ static void init_func_state(struct bpf_verifier_env *env,
 	state->callsite = callsite;
 	state->frameno = frameno;
 	state->subprogno = subprogno;
+	state->callback_ret_range = tnum_range(0, 0);
 	init_reg_state(env, state);
 	mark_verifier_state_scratched(env);
 }
@@ -2898,7 +2925,7 @@ static int __mark_chain_precision(struct bpf_verifier_env *env, int regno,
 	return 0;
 }
 
-static int mark_chain_precision(struct bpf_verifier_env *env, int regno)
+int mark_chain_precision(struct bpf_verifier_env *env, int regno)
 {
 	return __mark_chain_precision(env, regno, -1);
 }
@@ -5223,6 +5250,25 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
 				env,
 				regno, reg->off, access_size,
 				zero_size_allowed, ACCESS_HELPER, meta);
+	case PTR_TO_CTX:
+		/* in case the function doesn't know how to access the context,
+		 * (because we are in a program of type SYSCALL for example), we
+		 * can not statically check its size.
+		 * Dynamically check it now.
+		 */
+		if (!env->ops->convert_ctx_access) {
+			enum bpf_access_type atype = meta && meta->raw_mode ? BPF_WRITE : BPF_READ;
+			int offset = access_size - 1;
+
+			/* Allow zero-byte read from PTR_TO_CTX */
+			if (access_size == 0)
+				return zero_size_allowed ? 0 : -EACCES;
+
+			return check_mem_access(env, env->insn_idx, regno, offset, BPF_B,
+						atype, -1, false);
+		}
+
+		fallthrough;
 	default: /* scalar_value or invalid ptr */
 		/* Allow zero-byte read from NULL, regardless of pointer type */
 		if (zero_size_allowed && access_size == 0 &&
@@ -5656,6 +5702,12 @@ static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK }
 static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };
 static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } };
 static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } };
+static const struct bpf_reg_types dynptr_types = {
+	.types = {
+		PTR_TO_STACK,
+		PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL,
+	}
+};
 
 static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
 	[ARG_PTR_TO_MAP_KEY]		= &map_key_value_types,
@@ -5682,7 +5734,7 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
 	[ARG_PTR_TO_CONST_STR]		= &const_str_ptr_types,
 	[ARG_PTR_TO_TIMER]		= &timer_types,
 	[ARG_PTR_TO_KPTR]		= &kptr_types,
-	[ARG_PTR_TO_DYNPTR]		= &stack_ptr_types,
+	[ARG_PTR_TO_DYNPTR]		= &dynptr_types,
 };
 
 static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
@@ -5751,13 +5803,22 @@ found:
 		if (meta->func_id == BPF_FUNC_kptr_xchg) {
 			if (map_kptr_match_type(env, meta->kptr_off_desc, reg, regno))
 				return -EACCES;
-		} else if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
-						 btf_vmlinux, *arg_btf_id,
-						 strict_type_match)) {
-			verbose(env, "R%d is of type %s but %s is expected\n",
-				regno, kernel_type_name(reg->btf, reg->btf_id),
-				kernel_type_name(btf_vmlinux, *arg_btf_id));
-			return -EACCES;
+		} else {
+			if (arg_btf_id == BPF_PTR_POISON) {
+				verbose(env, "verifier internal error:");
+				verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n",
+					regno);
+				return -EACCES;
+			}
+
+			if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
+						  btf_vmlinux, *arg_btf_id,
+						  strict_type_match)) {
+				verbose(env, "R%d is of type %s but %s is expected\n",
+					regno, kernel_type_name(reg->btf, reg->btf_id),
+					kernel_type_name(btf_vmlinux, *arg_btf_id));
+				return -EACCES;
+			}
 		}
 	}
 
@@ -6025,6 +6086,13 @@ skip_type_check:
 		err = check_mem_size_reg(env, reg, regno, true, meta);
 		break;
 	case ARG_PTR_TO_DYNPTR:
+		/* We only need to check for initialized / uninitialized helper
+		 * dynptr args if the dynptr is not PTR_TO_DYNPTR, as the
+		 * assumption is that if it is, that a helper function
+		 * initialized the dynptr on behalf of the BPF program.
+		 */
+		if (base_type(reg->type) == PTR_TO_DYNPTR)
+			break;
 		if (arg_type & MEM_UNINIT) {
 			if (!is_dynptr_reg_valid_uninit(env, reg)) {
 				verbose(env, "Dynptr has to be an uninitialized dynptr\n");
@@ -6040,21 +6108,27 @@ skip_type_check:
 			}
 
 			meta->uninit_dynptr_regno = regno;
-		} else if (!is_dynptr_reg_valid_init(env, reg, arg_type)) {
+		} else if (!is_dynptr_reg_valid_init(env, reg)) {
+			verbose(env,
+				"Expected an initialized dynptr as arg #%d\n",
+				arg + 1);
+			return -EINVAL;
+		} else if (!is_dynptr_type_expected(env, reg, arg_type)) {
 			const char *err_extra = "";
 
 			switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
 			case DYNPTR_TYPE_LOCAL:
-				err_extra = "local ";
+				err_extra = "local";
 				break;
 			case DYNPTR_TYPE_RINGBUF:
-				err_extra = "ringbuf ";
+				err_extra = "ringbuf";
 				break;
 			default:
+				err_extra = "<unknown>";
 				break;
 			}
-
-			verbose(env, "Expected an initialized %sdynptr as arg #%d\n",
+			verbose(env,
+				"Expected a dynptr of type %s as arg #%d\n",
 				err_extra, arg + 1);
 			return -EINVAL;
 		}
@@ -6199,6 +6273,10 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
 		    func_id != BPF_FUNC_ringbuf_discard_dynptr)
 			goto error;
 		break;
+	case BPF_MAP_TYPE_USER_RINGBUF:
+		if (func_id != BPF_FUNC_user_ringbuf_drain)
+			goto error;
+		break;
 	case BPF_MAP_TYPE_STACK_TRACE:
 		if (func_id != BPF_FUNC_get_stackid)
 			goto error;
@@ -6318,6 +6396,10 @@ static int check_map_func_compatibility(struct bpf_verifier_env *env,
 		if (map->map_type != BPF_MAP_TYPE_RINGBUF)
 			goto error;
 		break;
+	case BPF_FUNC_user_ringbuf_drain:
+		if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF)
+			goto error;
+		break;
 	case BPF_FUNC_get_stackid:
 		if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
 			goto error;
@@ -6456,33 +6538,6 @@ static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
 	return true;
 }
 
-static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id)
-{
-	int count = 0;
-
-	if (arg_type_may_be_refcounted(fn->arg1_type))
-		count++;
-	if (arg_type_may_be_refcounted(fn->arg2_type))
-		count++;
-	if (arg_type_may_be_refcounted(fn->arg3_type))
-		count++;
-	if (arg_type_may_be_refcounted(fn->arg4_type))
-		count++;
-	if (arg_type_may_be_refcounted(fn->arg5_type))
-		count++;
-
-	/* A reference acquiring function cannot acquire
-	 * another refcounted ptr.
-	 */
-	if (may_be_acquire_function(func_id) && count)
-		return false;
-
-	/* We only support one arg being unreferenced at the moment,
-	 * which is sufficient for the helper functions we have right now.
-	 */
-	return count <= 1;
-}
-
 static bool check_btf_id_ok(const struct bpf_func_proto *fn)
 {
 	int i;
@@ -6501,43 +6556,25 @@ static bool check_btf_id_ok(const struct bpf_func_proto *fn)
 	return true;
 }
 
-static int check_func_proto(const struct bpf_func_proto *fn, int func_id,
-			    struct bpf_call_arg_meta *meta)
+static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
 {
 	return check_raw_mode_ok(fn) &&
 	       check_arg_pair_ok(fn) &&
-	       check_btf_id_ok(fn) &&
-	       check_refcount_ok(fn, func_id) ? 0 : -EINVAL;
+	       check_btf_id_ok(fn) ? 0 : -EINVAL;
 }
 
 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
  * are now invalid, so turn them into unknown SCALAR_VALUE.
  */
-static void __clear_all_pkt_pointers(struct bpf_verifier_env *env,
-				     struct bpf_func_state *state)
+static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
 {
-	struct bpf_reg_state *regs = state->regs, *reg;
-	int i;
-
-	for (i = 0; i < MAX_BPF_REG; i++)
-		if (reg_is_pkt_pointer_any(&regs[i]))
-			mark_reg_unknown(env, regs, i);
+	struct bpf_func_state *state;
+	struct bpf_reg_state *reg;
 
-	bpf_for_each_spilled_reg(i, state, reg) {
-		if (!reg)
-			continue;
+	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
 		if (reg_is_pkt_pointer_any(reg))
 			__mark_reg_unknown(env, reg);
-	}
-}
-
-static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
-{
-	struct bpf_verifier_state *vstate = env->cur_state;
-	int i;
-
-	for (i = 0; i <= vstate->curframe; i++)
-		__clear_all_pkt_pointers(env, vstate->frame[i]);
+	}));
 }
 
 enum {
@@ -6566,41 +6603,24 @@ static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range
 		reg->range = AT_PKT_END;
 }
 
-static void release_reg_references(struct bpf_verifier_env *env,
-				   struct bpf_func_state *state,
-				   int ref_obj_id)
-{
-	struct bpf_reg_state *regs = state->regs, *reg;
-	int i;
-
-	for (i = 0; i < MAX_BPF_REG; i++)
-		if (regs[i].ref_obj_id == ref_obj_id)
-			mark_reg_unknown(env, regs, i);
-
-	bpf_for_each_spilled_reg(i, state, reg) {
-		if (!reg)
-			continue;
-		if (reg->ref_obj_id == ref_obj_id)
-			__mark_reg_unknown(env, reg);
-	}
-}
-
 /* The pointer with the specified id has released its reference to kernel
  * resources. Identify all copies of the same pointer and clear the reference.
  */
 static int release_reference(struct bpf_verifier_env *env,
 			     int ref_obj_id)
 {
-	struct bpf_verifier_state *vstate = env->cur_state;
+	struct bpf_func_state *state;
+	struct bpf_reg_state *reg;
 	int err;
-	int i;
 
 	err = release_reference_state(cur_func(env), ref_obj_id);
 	if (err)
 		return err;
 
-	for (i = 0; i <= vstate->curframe; i++)
-		release_reg_references(env, vstate->frame[i], ref_obj_id);
+	bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
+		if (reg->ref_obj_id == ref_obj_id)
+			__mark_reg_unknown(env, reg);
+	}));
 
 	return 0;
 }
@@ -6648,7 +6668,7 @@ static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 	func_info_aux = env->prog->aux->func_info_aux;
 	if (func_info_aux)
 		is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
-	err = btf_check_subprog_arg_match(env, subprog, caller->regs);
+	err = btf_check_subprog_call(env, subprog, caller->regs);
 	if (err == -EFAULT)
 		return err;
 	if (is_global) {
@@ -6822,6 +6842,7 @@ static int set_map_elem_callback_state(struct bpf_verifier_env *env,
 		return err;
 
 	callee->in_callback_fn = true;
+	callee->callback_ret_range = tnum_range(0, 1);
 	return 0;
 }
 
@@ -6843,6 +6864,7 @@ static int set_loop_callback_state(struct bpf_verifier_env *env,
 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
 
 	callee->in_callback_fn = true;
+	callee->callback_ret_range = tnum_range(0, 1);
 	return 0;
 }
 
@@ -6872,6 +6894,7 @@ static int set_timer_callback_state(struct bpf_verifier_env *env,
 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
 	callee->in_async_callback_fn = true;
+	callee->callback_ret_range = tnum_range(0, 1);
 	return 0;
 }
 
@@ -6899,6 +6922,30 @@ static int set_find_vma_callback_state(struct bpf_verifier_env *env,
 	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
 	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
 	callee->in_callback_fn = true;
+	callee->callback_ret_range = tnum_range(0, 1);
+	return 0;
+}
+
+static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,
+					   struct bpf_func_state *caller,
+					   struct bpf_func_state *callee,
+					   int insn_idx)
+{
+	/* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void
+	 *			  callback_ctx, u64 flags);
+	 * callback_fn(struct bpf_dynptr_t* dynptr, void *callback_ctx);
+	 */
+	__mark_reg_not_init(env, &callee->regs[BPF_REG_0]);
+	callee->regs[BPF_REG_1].type = PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL;
+	__mark_reg_known_zero(&callee->regs[BPF_REG_1]);
+	callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
+
+	/* unused */
+	__mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
+	__mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
+	__mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
+
+	callee->in_callback_fn = true;
 	return 0;
 }
 
@@ -6926,7 +6973,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 	caller = state->frame[state->curframe];
 	if (callee->in_callback_fn) {
 		/* enforce R0 return value range [0, 1]. */
-		struct tnum range = tnum_range(0, 1);
+		struct tnum range = callee->callback_ret_range;
 
 		if (r0->type != SCALAR_VALUE) {
 			verbose(env, "R0 not a scalar value\n");
@@ -6941,10 +6988,17 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 		caller->regs[BPF_REG_0] = *r0;
 	}
 
-	/* Transfer references to the caller */
-	err = copy_reference_state(caller, callee);
-	if (err)
-		return err;
+	/* callback_fn frame should have released its own additions to parent's
+	 * reference state at this point, or check_reference_leak would
+	 * complain, hence it must be the same as the caller. There is no need
+	 * to copy it back.
+	 */
+	if (!callee->in_callback_fn) {
+		/* Transfer references to the caller */
+		err = copy_reference_state(caller, callee);
+		if (err)
+			return err;
+	}
 
 	*insn_idx = callee->callsite + 1;
 	if (env->log.level & BPF_LOG_LEVEL) {
@@ -7066,13 +7120,20 @@ record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
 static int check_reference_leak(struct bpf_verifier_env *env)
 {
 	struct bpf_func_state *state = cur_func(env);
+	bool refs_lingering = false;
 	int i;
 
+	if (state->frameno && !state->in_callback_fn)
+		return 0;
+
 	for (i = 0; i < state->acquired_refs; i++) {
+		if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
+			continue;
 		verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
 			state->refs[i].id, state->refs[i].insn_idx);
+		refs_lingering = true;
 	}
-	return state->acquired_refs ? -EINVAL : 0;
+	return refs_lingering ? -EINVAL : 0;
 }
 
 static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
@@ -7219,7 +7280,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 	memset(&meta, 0, sizeof(meta));
 	meta.pkt_access = fn->pkt_access;
 
-	err = check_func_proto(fn, func_id, &meta);
+	err = check_func_proto(fn, func_id);
 	if (err) {
 		verbose(env, "kernel subsystem misconfigured func %s#%d\n",
 			func_id_name(func_id), func_id);
@@ -7344,6 +7405,33 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 			}
 		}
 		break;
+	case BPF_FUNC_dynptr_data:
+		for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
+			if (arg_type_is_dynptr(fn->arg_type[i])) {
+				struct bpf_reg_state *reg = &regs[BPF_REG_1 + i];
+
+				if (meta.ref_obj_id) {
+					verbose(env, "verifier internal error: meta.ref_obj_id already set\n");
+					return -EFAULT;
+				}
+
+				if (base_type(reg->type) != PTR_TO_DYNPTR)
+					/* Find the id of the dynptr we're
+					 * tracking the reference of
+					 */
+					meta.ref_obj_id = stack_slot_get_id(env, reg);
+				break;
+			}
+		}
+		if (i == MAX_BPF_FUNC_REG_ARGS) {
+			verbose(env, "verifier internal error: no dynptr in bpf_dynptr_data()\n");
+			return -EFAULT;
+		}
+		break;
+	case BPF_FUNC_user_ringbuf_drain:
+		err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
+					set_user_ringbuf_callback_state);
+		break;
 	}
 
 	if (err)
@@ -7360,13 +7448,17 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 
 	/* update return register (already marked as written above) */
 	ret_type = fn->ret_type;
-	ret_flag = type_flag(fn->ret_type);
-	if (ret_type == RET_INTEGER) {
+	ret_flag = type_flag(ret_type);
+
+	switch (base_type(ret_type)) {
+	case RET_INTEGER:
 		/* sets type to SCALAR_VALUE */
 		mark_reg_unknown(env, regs, BPF_REG_0);
-	} else if (ret_type == RET_VOID) {
+		break;
+	case RET_VOID:
 		regs[BPF_REG_0].type = NOT_INIT;
-	} else if (base_type(ret_type) == RET_PTR_TO_MAP_VALUE) {
+		break;
+	case RET_PTR_TO_MAP_VALUE:
 		/* There is no offset yet applied, variable or fixed */
 		mark_reg_known_zero(env, regs, BPF_REG_0);
 		/* remember map_ptr, so that check_map_access()
@@ -7385,20 +7477,26 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		    map_value_has_spin_lock(meta.map_ptr)) {
 			regs[BPF_REG_0].id = ++env->id_gen;
 		}
-	} else if (base_type(ret_type) == RET_PTR_TO_SOCKET) {
+		break;
+	case RET_PTR_TO_SOCKET:
 		mark_reg_known_zero(env, regs, BPF_REG_0);
 		regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag;
-	} else if (base_type(ret_type) == RET_PTR_TO_SOCK_COMMON) {
+		break;
+	case RET_PTR_TO_SOCK_COMMON:
 		mark_reg_known_zero(env, regs, BPF_REG_0);
 		regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag;
-	} else if (base_type(ret_type) == RET_PTR_TO_TCP_SOCK) {
+		break;
+	case RET_PTR_TO_TCP_SOCK:
 		mark_reg_known_zero(env, regs, BPF_REG_0);
 		regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag;
-	} else if (base_type(ret_type) == RET_PTR_TO_ALLOC_MEM) {
+		break;
+	case RET_PTR_TO_ALLOC_MEM:
 		mark_reg_known_zero(env, regs, BPF_REG_0);
 		regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
 		regs[BPF_REG_0].mem_size = meta.mem_size;
-	} else if (base_type(ret_type) == RET_PTR_TO_MEM_OR_BTF_ID) {
+		break;
+	case RET_PTR_TO_MEM_OR_BTF_ID:
+	{
 		const struct btf_type *t;
 
 		mark_reg_known_zero(env, regs, BPF_REG_0);
@@ -7430,7 +7528,10 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 			regs[BPF_REG_0].btf = meta.ret_btf;
 			regs[BPF_REG_0].btf_id = meta.ret_btf_id;
 		}
-	} else if (base_type(ret_type) == RET_PTR_TO_BTF_ID) {
+		break;
+	}
+	case RET_PTR_TO_BTF_ID:
+	{
 		struct btf *ret_btf;
 		int ret_btf_id;
 
@@ -7440,6 +7541,12 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 			ret_btf = meta.kptr_off_desc->kptr.btf;
 			ret_btf_id = meta.kptr_off_desc->kptr.btf_id;
 		} else {
+			if (fn->ret_btf_id == BPF_PTR_POISON) {
+				verbose(env, "verifier internal error:");
+				verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n",
+					func_id_name(func_id));
+				return -EINVAL;
+			}
 			ret_btf = btf_vmlinux;
 			ret_btf_id = *fn->ret_btf_id;
 		}
@@ -7451,7 +7558,9 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		}
 		regs[BPF_REG_0].btf = ret_btf;
 		regs[BPF_REG_0].btf_id = ret_btf_id;
-	} else {
+		break;
+	}
+	default:
 		verbose(env, "unknown return type %u of func %s#%d\n",
 			base_type(ret_type), func_id_name(func_id), func_id);
 		return -EINVAL;
@@ -7460,7 +7569,13 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 	if (type_may_be_null(regs[BPF_REG_0].type))
 		regs[BPF_REG_0].id = ++env->id_gen;
 
-	if (is_ptr_cast_function(func_id)) {
+	if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) {
+		verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n",
+			func_id_name(func_id), func_id);
+		return -EFAULT;
+	}
+
+	if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) {
 		/* For release_reference() */
 		regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
 	} else if (is_acquire_function(func_id, meta.map_ptr)) {
@@ -7472,21 +7587,6 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		regs[BPF_REG_0].id = id;
 		/* For release_reference() */
 		regs[BPF_REG_0].ref_obj_id = id;
-	} else if (func_id == BPF_FUNC_dynptr_data) {
-		int dynptr_id = 0, i;
-
-		/* Find the id of the dynptr we're acquiring a reference to */
-		for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
-			if (arg_type_is_dynptr(fn->arg_type[i])) {
-				if (dynptr_id) {
-					verbose(env, "verifier internal error: multiple dynptr args in func\n");
-					return -EFAULT;
-				}
-				dynptr_id = stack_slot_get_id(env, &regs[BPF_REG_1 + i]);
-			}
-		}
-		/* For release_reference() */
-		regs[BPF_REG_0].ref_obj_id = dynptr_id;
 	}
 
 	do_refine_retval_range(regs, fn->ret_type, func_id, &meta);
@@ -7558,6 +7658,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 {
 	const struct btf_type *t, *func, *func_proto, *ptr_type;
 	struct bpf_reg_state *regs = cur_regs(env);
+	struct bpf_kfunc_arg_meta meta = { 0 };
 	const char *func_name, *ptr_type_name;
 	u32 i, nargs, func_id, ptr_type_id;
 	int err, insn_idx = *insn_idx_p;
@@ -7585,10 +7686,17 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			func_name);
 		return -EACCES;
 	}
+	if (*kfunc_flags & KF_DESTRUCTIVE && !capable(CAP_SYS_BOOT)) {
+		verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capabilities\n");
+		return -EACCES;
+	}
+
 	acq = *kfunc_flags & KF_ACQUIRE;
 
+	meta.flags = *kfunc_flags;
+
 	/* Check the arguments */
-	err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs, *kfunc_flags);
+	err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs, &meta);
 	if (err < 0)
 		return err;
 	/* In case of release function, we get register number of refcounted
@@ -7609,7 +7717,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	/* Check return type */
 	t = btf_type_skip_modifiers(desc_btf, func_proto->type, NULL);
 
-	if (acq && !btf_type_is_ptr(t)) {
+	if (acq && !btf_type_is_struct_ptr(desc_btf, t)) {
 		verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n");
 		return -EINVAL;
 	}
@@ -7621,17 +7729,33 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		ptr_type = btf_type_skip_modifiers(desc_btf, t->type,
 						   &ptr_type_id);
 		if (!btf_type_is_struct(ptr_type)) {
-			ptr_type_name = btf_name_by_offset(desc_btf,
-							   ptr_type->name_off);
-			verbose(env, "kernel function %s returns pointer type %s %s is not supported\n",
-				func_name, btf_type_str(ptr_type),
-				ptr_type_name);
-			return -EINVAL;
+			if (!meta.r0_size) {
+				ptr_type_name = btf_name_by_offset(desc_btf,
+								   ptr_type->name_off);
+				verbose(env,
+					"kernel function %s returns pointer type %s %s is not supported\n",
+					func_name,
+					btf_type_str(ptr_type),
+					ptr_type_name);
+				return -EINVAL;
+			}
+
+			mark_reg_known_zero(env, regs, BPF_REG_0);
+			regs[BPF_REG_0].type = PTR_TO_MEM;
+			regs[BPF_REG_0].mem_size = meta.r0_size;
+
+			if (meta.r0_rdonly)
+				regs[BPF_REG_0].type |= MEM_RDONLY;
+
+			/* Ensures we don't access the memory after a release_reference() */
+			if (meta.ref_obj_id)
+				regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
+		} else {
+			mark_reg_known_zero(env, regs, BPF_REG_0);
+			regs[BPF_REG_0].btf = desc_btf;
+			regs[BPF_REG_0].type = PTR_TO_BTF_ID;
+			regs[BPF_REG_0].btf_id = ptr_type_id;
 		}
-		mark_reg_known_zero(env, regs, BPF_REG_0);
-		regs[BPF_REG_0].btf = desc_btf;
-		regs[BPF_REG_0].type = PTR_TO_BTF_ID;
-		regs[BPF_REG_0].btf_id = ptr_type_id;
 		if (*kfunc_flags & KF_RET_NULL) {
 			regs[BPF_REG_0].type |= PTR_MAYBE_NULL;
 			/* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */
@@ -9274,34 +9398,14 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
 	return 0;
 }
 
-static void __find_good_pkt_pointers(struct bpf_func_state *state,
-				     struct bpf_reg_state *dst_reg,
-				     enum bpf_reg_type type, int new_range)
-{
-	struct bpf_reg_state *reg;
-	int i;
-
-	for (i = 0; i < MAX_BPF_REG; i++) {
-		reg = &state->regs[i];
-		if (reg->type == type && reg->id == dst_reg->id)
-			/* keep the maximum range already checked */
-			reg->range = max(reg->range, new_range);
-	}
-
-	bpf_for_each_spilled_reg(i, state, reg) {
-		if (!reg)
-			continue;
-		if (reg->type == type && reg->id == dst_reg->id)
-			reg->range = max(reg->range, new_range);
-	}
-}
-
 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
 				   struct bpf_reg_state *dst_reg,
 				   enum bpf_reg_type type,
 				   bool range_right_open)
 {
-	int new_range, i;
+	struct bpf_func_state *state;
+	struct bpf_reg_state *reg;
+	int new_range;
 
 	if (dst_reg->off < 0 ||
 	    (dst_reg->off == 0 && range_right_open))
@@ -9366,9 +9470,11 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
 	 * the range won't allow anything.
 	 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
 	 */
-	for (i = 0; i <= vstate->curframe; i++)
-		__find_good_pkt_pointers(vstate->frame[i], dst_reg, type,
-					 new_range);
+	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
+		if (reg->type == type && reg->id == dst_reg->id)
+			/* keep the maximum range already checked */
+			reg->range = max(reg->range, new_range);
+	}));
 }
 
 static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode)
@@ -9857,7 +9963,7 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state,
 
 		if (!reg_may_point_to_spin_lock(reg)) {
 			/* For not-NULL ptr, reg->ref_obj_id will be reset
-			 * in release_reg_references().
+			 * in release_reference().
 			 *
 			 * reg->id is still used by spin_lock ptr. Other
 			 * than spin_lock ptr type, reg->id can be reset.
@@ -9867,22 +9973,6 @@ static void mark_ptr_or_null_reg(struct bpf_func_state *state,
 	}
 }
 
-static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id,
-				    bool is_null)
-{
-	struct bpf_reg_state *reg;
-	int i;
-
-	for (i = 0; i < MAX_BPF_REG; i++)
-		mark_ptr_or_null_reg(state, &state->regs[i], id, is_null);
-
-	bpf_for_each_spilled_reg(i, state, reg) {
-		if (!reg)
-			continue;
-		mark_ptr_or_null_reg(state, reg, id, is_null);
-	}
-}
-
 /* The logic is similar to find_good_pkt_pointers(), both could eventually
  * be folded together at some point.
  */
@@ -9890,10 +9980,9 @@ static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
 				  bool is_null)
 {
 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
-	struct bpf_reg_state *regs = state->regs;
+	struct bpf_reg_state *regs = state->regs, *reg;
 	u32 ref_obj_id = regs[regno].ref_obj_id;
 	u32 id = regs[regno].id;
-	int i;
 
 	if (ref_obj_id && ref_obj_id == id && is_null)
 		/* regs[regno] is in the " == NULL" branch.
@@ -9902,8 +9991,9 @@ static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
 		 */
 		WARN_ON_ONCE(release_reference_state(state, id));
 
-	for (i = 0; i <= vstate->curframe; i++)
-		__mark_ptr_or_null_regs(vstate->frame[i], id, is_null);
+	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
+		mark_ptr_or_null_reg(state, reg, id, is_null);
+	}));
 }
 
 static bool try_match_pkt_pointers(const struct bpf_insn *insn,
@@ -10016,23 +10106,11 @@ static void find_equal_scalars(struct bpf_verifier_state *vstate,
 {
 	struct bpf_func_state *state;
 	struct bpf_reg_state *reg;
-	int i, j;
-
-	for (i = 0; i <= vstate->curframe; i++) {
-		state = vstate->frame[i];
-		for (j = 0; j < MAX_BPF_REG; j++) {
-			reg = &state->regs[j];
-			if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
-				*reg = *known_reg;
-		}
 
-		bpf_for_each_spilled_reg(j, state, reg) {
-			if (!reg)
-				continue;
-			if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
-				*reg = *known_reg;
-		}
-	}
+	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
+		if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
+			*reg = *known_reg;
+	}));
 }
 
 static int check_cond_jmp_op(struct bpf_verifier_env *env,
@@ -12333,6 +12411,16 @@ static int do_check(struct bpf_verifier_env *env)
 					return -EINVAL;
 				}
 
+				/* We must do check_reference_leak here before
+				 * prepare_func_exit to handle the case when
+				 * state->curframe > 0, it may be a callback
+				 * function, for which reference_state must
+				 * match caller reference state when it exits.
+				 */
+				err = check_reference_leak(env);
+				if (err)
+					return err;
+
 				if (state->curframe) {
 					/* exit from nested function */
 					err = prepare_func_exit(env, &env->insn_idx);
@@ -12342,10 +12430,6 @@ static int do_check(struct bpf_verifier_env *env)
 					continue;
 				}
 
-				err = check_reference_leak(env);
-				if (err)
-					return err;
-
 				err = check_return_code(env);
 				if (err)
 					return err;
@@ -12558,14 +12642,6 @@ err_put:
 	return err;
 }
 
-static int check_map_prealloc(struct bpf_map *map)
-{
-	return (map->map_type != BPF_MAP_TYPE_HASH &&
-		map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
-		map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) ||
-		!(map->map_flags & BPF_F_NO_PREALLOC);
-}
-
 static bool is_tracing_prog_type(enum bpf_prog_type type)
 {
 	switch (type) {
@@ -12580,50 +12656,12 @@ static bool is_tracing_prog_type(enum bpf_prog_type type)
 	}
 }
 
-static bool is_preallocated_map(struct bpf_map *map)
-{
-	if (!check_map_prealloc(map))
-		return false;
-	if (map->inner_map_meta && !check_map_prealloc(map->inner_map_meta))
-		return false;
-	return true;
-}
-
 static int check_map_prog_compatibility(struct bpf_verifier_env *env,
 					struct bpf_map *map,
 					struct bpf_prog *prog)
 
 {
 	enum bpf_prog_type prog_type = resolve_prog_type(prog);
-	/*
-	 * Validate that trace type programs use preallocated hash maps.
-	 *
-	 * For programs attached to PERF events this is mandatory as the
-	 * perf NMI can hit any arbitrary code sequence.
-	 *
-	 * All other trace types using preallocated hash maps are unsafe as
-	 * well because tracepoint or kprobes can be inside locked regions
-	 * of the memory allocator or at a place where a recursion into the
-	 * memory allocator would see inconsistent state.
-	 *
-	 * On RT enabled kernels run-time allocation of all trace type
-	 * programs is strictly prohibited due to lock type constraints. On
-	 * !RT kernels it is allowed for backwards compatibility reasons for
-	 * now, but warnings are emitted so developers are made aware of
-	 * the unsafety and can fix their programs before this is enforced.
-	 */
-	if (is_tracing_prog_type(prog_type) && !is_preallocated_map(map)) {
-		if (prog_type == BPF_PROG_TYPE_PERF_EVENT) {
-			verbose(env, "perf_event programs can only use preallocated hash map\n");
-			return -EINVAL;
-		}
-		if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
-			verbose(env, "trace type programs can only use preallocated hash map\n");
-			return -EINVAL;
-		}
-		WARN_ONCE(1, "trace type BPF program uses run-time allocation\n");
-		verbose(env, "trace type programs with run-time allocated hash maps are unsafe. Switch to preallocated hash maps.\n");
-	}
 
 	if (map_value_has_spin_lock(map)) {
 		if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
@@ -12670,13 +12708,8 @@ static int check_map_prog_compatibility(struct bpf_verifier_env *env,
 		case BPF_MAP_TYPE_LRU_PERCPU_HASH:
 		case BPF_MAP_TYPE_ARRAY_OF_MAPS:
 		case BPF_MAP_TYPE_HASH_OF_MAPS:
-			if (!is_preallocated_map(map)) {
-				verbose(env,
-					"Sleepable programs can only use preallocated maps\n");
-				return -EINVAL;
-			}
-			break;
 		case BPF_MAP_TYPE_RINGBUF:
+		case BPF_MAP_TYPE_USER_RINGBUF:
 		case BPF_MAP_TYPE_INODE_STORAGE:
 		case BPF_MAP_TYPE_SK_STORAGE:
 		case BPF_MAP_TYPE_TASK_STORAGE:
@@ -13470,9 +13503,6 @@ static int convert_ctx_accesses(struct bpf_verifier_env *env)
 				insn->code = BPF_LDX | BPF_PROBE_MEM |
 					BPF_SIZE((insn)->code);
 				env->prog->aux->num_exentries++;
-			} else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) {
-				verbose(env, "Writes through BTF pointers are not allowed\n");
-				return -EINVAL;
 			}
 			continue;
 		default:
diff --git a/kernel/cfi.c b/kernel/cfi.c
index 2046276ee234..08caad776717 100644
--- a/kernel/cfi.c
+++ b/kernel/cfi.c
@@ -1,339 +1,101 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Clang Control Flow Integrity (CFI) error and slowpath handling.
+ * Clang Control Flow Integrity (CFI) error handling.
  *
- * Copyright (C) 2021 Google LLC
+ * Copyright (C) 2022 Google LLC
  */
 
-#include <linux/hardirq.h>
-#include <linux/kallsyms.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/printk.h>
-#include <linux/ratelimit.h>
-#include <linux/rcupdate.h>
-#include <linux/vmalloc.h>
-#include <asm/cacheflush.h>
-#include <asm/set_memory.h>
+#include <linux/cfi.h>
 
-/* Compiler-defined handler names */
-#ifdef CONFIG_CFI_PERMISSIVE
-#define cfi_failure_handler	__ubsan_handle_cfi_check_fail
-#else
-#define cfi_failure_handler	__ubsan_handle_cfi_check_fail_abort
-#endif
-
-static inline void handle_cfi_failure(void *ptr)
+enum bug_trap_type report_cfi_failure(struct pt_regs *regs, unsigned long addr,
+				      unsigned long *target, u32 type)
 {
-	if (IS_ENABLED(CONFIG_CFI_PERMISSIVE))
-		WARN_RATELIMIT(1, "CFI failure (target: %pS):\n", ptr);
+	if (target)
+		pr_err("CFI failure at %pS (target: %pS; expected type: 0x%08x)\n",
+		       (void *)addr, (void *)*target, type);
 	else
-		panic("CFI failure (target: %pS)\n", ptr);
-}
-
-#ifdef CONFIG_MODULES
-#ifdef CONFIG_CFI_CLANG_SHADOW
-/*
- * Index type. A 16-bit index can address at most (2^16)-2 pages (taking
- * into account SHADOW_INVALID), i.e. ~256M with 4k pages.
- */
-typedef u16 shadow_t;
-#define SHADOW_INVALID		((shadow_t)~0UL)
-
-struct cfi_shadow {
-	/* Page index for the beginning of the shadow */
-	unsigned long base;
-	/* An array of __cfi_check locations (as indices to the shadow) */
-	shadow_t shadow[1];
-} __packed;
-
-/*
- * The shadow covers ~128M from the beginning of the module region. If
- * the region is larger, we fall back to __module_address for the rest.
- */
-#define __SHADOW_RANGE		(_UL(SZ_128M) >> PAGE_SHIFT)
-
-/* The in-memory size of struct cfi_shadow, always at least one page */
-#define __SHADOW_PAGES		((__SHADOW_RANGE * sizeof(shadow_t)) >> PAGE_SHIFT)
-#define SHADOW_PAGES		max(1UL, __SHADOW_PAGES)
-#define SHADOW_SIZE		(SHADOW_PAGES << PAGE_SHIFT)
-
-/* The actual size of the shadow array, minus metadata */
-#define SHADOW_ARR_SIZE		(SHADOW_SIZE - offsetof(struct cfi_shadow, shadow))
-#define SHADOW_ARR_SLOTS	(SHADOW_ARR_SIZE / sizeof(shadow_t))
-
-static DEFINE_MUTEX(shadow_update_lock);
-static struct cfi_shadow __rcu *cfi_shadow __read_mostly;
+		pr_err("CFI failure at %pS (no target information)\n",
+		       (void *)addr);
 
-/* Returns the index in the shadow for the given address */
-static inline int ptr_to_shadow(const struct cfi_shadow *s, unsigned long ptr)
-{
-	unsigned long index;
-	unsigned long page = ptr >> PAGE_SHIFT;
-
-	if (unlikely(page < s->base))
-		return -1; /* Outside of module area */
-
-	index = page - s->base;
-
-	if (index >= SHADOW_ARR_SLOTS)
-		return -1; /* Cannot be addressed with shadow */
-
-	return (int)index;
-}
-
-/* Returns the page address for an index in the shadow */
-static inline unsigned long shadow_to_ptr(const struct cfi_shadow *s,
-	int index)
-{
-	if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS))
-		return 0;
-
-	return (s->base + index) << PAGE_SHIFT;
-}
-
-/* Returns the __cfi_check function address for the given shadow location */
-static inline unsigned long shadow_to_check_fn(const struct cfi_shadow *s,
-	int index)
-{
-	if (unlikely(index < 0 || index >= SHADOW_ARR_SLOTS))
-		return 0;
-
-	if (unlikely(s->shadow[index] == SHADOW_INVALID))
-		return 0;
-
-	/* __cfi_check is always page aligned */
-	return (s->base + s->shadow[index]) << PAGE_SHIFT;
-}
-
-static void prepare_next_shadow(const struct cfi_shadow __rcu *prev,
-		struct cfi_shadow *next)
-{
-	int i, index, check;
-
-	/* Mark everything invalid */
-	memset(next->shadow, 0xFF, SHADOW_ARR_SIZE);
-
-	if (!prev)
-		return; /* No previous shadow */
-
-	/* If the base address didn't change, an update is not needed */
-	if (prev->base == next->base) {
-		memcpy(next->shadow, prev->shadow, SHADOW_ARR_SIZE);
-		return;
-	}
-
-	/* Convert the previous shadow to the new address range */
-	for (i = 0; i < SHADOW_ARR_SLOTS; ++i) {
-		if (prev->shadow[i] == SHADOW_INVALID)
-			continue;
-
-		index = ptr_to_shadow(next, shadow_to_ptr(prev, i));
-		if (index < 0)
-			continue;
-
-		check = ptr_to_shadow(next,
-				shadow_to_check_fn(prev, prev->shadow[i]));
-		if (check < 0)
-			continue;
-
-		next->shadow[index] = (shadow_t)check;
-	}
-}
-
-static void add_module_to_shadow(struct cfi_shadow *s, struct module *mod,
-			unsigned long min_addr, unsigned long max_addr)
-{
-	int check_index;
-	unsigned long check = (unsigned long)mod->cfi_check;
-	unsigned long ptr;
-
-	if (unlikely(!PAGE_ALIGNED(check))) {
-		pr_warn("cfi: not using shadow for module %s\n", mod->name);
-		return;
+	if (IS_ENABLED(CONFIG_CFI_PERMISSIVE)) {
+		__warn(NULL, 0, (void *)addr, 0, regs, NULL);
+		return BUG_TRAP_TYPE_WARN;
 	}
 
-	check_index = ptr_to_shadow(s, check);
-	if (check_index < 0)
-		return; /* Module not addressable with shadow */
-
-	/* For each page, store the check function index in the shadow */
-	for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) {
-		int index = ptr_to_shadow(s, ptr);
-
-		if (index >= 0) {
-			/* Each page must only contain one module */
-			WARN_ON_ONCE(s->shadow[index] != SHADOW_INVALID);
-			s->shadow[index] = (shadow_t)check_index;
-		}
-	}
+	return BUG_TRAP_TYPE_BUG;
 }
 
-static void remove_module_from_shadow(struct cfi_shadow *s, struct module *mod,
-		unsigned long min_addr, unsigned long max_addr)
+#ifdef CONFIG_ARCH_USES_CFI_TRAPS
+static inline unsigned long trap_address(s32 *p)
 {
-	unsigned long ptr;
-
-	for (ptr = min_addr; ptr <= max_addr; ptr += PAGE_SIZE) {
-		int index = ptr_to_shadow(s, ptr);
-
-		if (index >= 0)
-			s->shadow[index] = SHADOW_INVALID;
-	}
+	return (unsigned long)((long)p + (long)*p);
 }
 
-typedef void (*update_shadow_fn)(struct cfi_shadow *, struct module *,
-			unsigned long min_addr, unsigned long max_addr);
-
-static void update_shadow(struct module *mod, unsigned long base_addr,
-		update_shadow_fn fn)
+static bool is_trap(unsigned long addr, s32 *start, s32 *end)
 {
-	struct cfi_shadow *prev;
-	struct cfi_shadow *next;
-	unsigned long min_addr, max_addr;
-
-	next = vmalloc(SHADOW_SIZE);
-
-	mutex_lock(&shadow_update_lock);
-	prev = rcu_dereference_protected(cfi_shadow,
-					 mutex_is_locked(&shadow_update_lock));
-
-	if (next) {
-		next->base = base_addr >> PAGE_SHIFT;
-		prepare_next_shadow(prev, next);
+	s32 *p;
 
-		min_addr = (unsigned long)mod->core_layout.base;
-		max_addr = min_addr + mod->core_layout.text_size;
-		fn(next, mod, min_addr & PAGE_MASK, max_addr & PAGE_MASK);
-
-		set_memory_ro((unsigned long)next, SHADOW_PAGES);
-	}
-
-	rcu_assign_pointer(cfi_shadow, next);
-	mutex_unlock(&shadow_update_lock);
-	synchronize_rcu();
-
-	if (prev) {
-		set_memory_rw((unsigned long)prev, SHADOW_PAGES);
-		vfree(prev);
+	for (p = start; p < end; ++p) {
+		if (trap_address(p) == addr)
+			return true;
 	}
-}
 
-void cfi_module_add(struct module *mod, unsigned long base_addr)
-{
-	update_shadow(mod, base_addr, add_module_to_shadow);
+	return false;
 }
 
-void cfi_module_remove(struct module *mod, unsigned long base_addr)
-{
-	update_shadow(mod, base_addr, remove_module_from_shadow);
-}
-
-static inline cfi_check_fn ptr_to_check_fn(const struct cfi_shadow __rcu *s,
-	unsigned long ptr)
+#ifdef CONFIG_MODULES
+/* Populates `kcfi_trap(_end)?` fields in `struct module`. */
+void module_cfi_finalize(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
+			 struct module *mod)
 {
-	int index;
-
-	if (unlikely(!s))
-		return NULL; /* No shadow available */
-
-	index = ptr_to_shadow(s, ptr);
-	if (index < 0)
-		return NULL; /* Cannot be addressed with shadow */
+	char *secstrings;
+	unsigned int i;
 
-	return (cfi_check_fn)shadow_to_check_fn(s, index);
-}
-
-static inline cfi_check_fn find_shadow_check_fn(unsigned long ptr)
-{
-	cfi_check_fn fn;
+	mod->kcfi_traps = NULL;
+	mod->kcfi_traps_end = NULL;
 
-	rcu_read_lock_sched_notrace();
-	fn = ptr_to_check_fn(rcu_dereference_sched(cfi_shadow), ptr);
-	rcu_read_unlock_sched_notrace();
+	secstrings = (char *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
 
-	return fn;
-}
-
-#else /* !CONFIG_CFI_CLANG_SHADOW */
+	for (i = 1; i < hdr->e_shnum; i++) {
+		if (strcmp(secstrings + sechdrs[i].sh_name, "__kcfi_traps"))
+			continue;
 
-static inline cfi_check_fn find_shadow_check_fn(unsigned long ptr)
-{
-	return NULL;
+		mod->kcfi_traps = (s32 *)sechdrs[i].sh_addr;
+		mod->kcfi_traps_end = (s32 *)(sechdrs[i].sh_addr + sechdrs[i].sh_size);
+		break;
+	}
 }
 
-#endif /* CONFIG_CFI_CLANG_SHADOW */
-
-static inline cfi_check_fn find_module_check_fn(unsigned long ptr)
+static bool is_module_cfi_trap(unsigned long addr)
 {
-	cfi_check_fn fn = NULL;
 	struct module *mod;
+	bool found = false;
 
 	rcu_read_lock_sched_notrace();
-	mod = __module_address(ptr);
-	if (mod)
-		fn = mod->cfi_check;
-	rcu_read_unlock_sched_notrace();
-
-	return fn;
-}
-
-static inline cfi_check_fn find_check_fn(unsigned long ptr)
-{
-	cfi_check_fn fn = NULL;
-	unsigned long flags;
-	bool rcu_idle;
-
-	if (is_kernel_text(ptr))
-		return __cfi_check;
 
-	/*
-	 * Indirect call checks can happen when RCU is not watching. Both
-	 * the shadow and __module_address use RCU, so we need to wake it
-	 * up if necessary.
-	 */
-	rcu_idle = !rcu_is_watching();
-	if (rcu_idle) {
-		local_irq_save(flags);
-		ct_irq_enter();
-	}
-
-	if (IS_ENABLED(CONFIG_CFI_CLANG_SHADOW))
-		fn = find_shadow_check_fn(ptr);
-	if (!fn)
-		fn = find_module_check_fn(ptr);
+	mod = __module_address(addr);
+	if (mod)
+		found = is_trap(addr, mod->kcfi_traps, mod->kcfi_traps_end);
 
-	if (rcu_idle) {
-		ct_irq_exit();
-		local_irq_restore(flags);
-	}
+	rcu_read_unlock_sched_notrace();
 
-	return fn;
+	return found;
 }
-
-void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag)
+#else /* CONFIG_MODULES */
+static inline bool is_module_cfi_trap(unsigned long addr)
 {
-	cfi_check_fn fn = find_check_fn((unsigned long)ptr);
-
-	if (likely(fn))
-		fn(id, ptr, diag);
-	else /* Don't allow unchecked modules */
-		handle_cfi_failure(ptr);
+	return false;
 }
-EXPORT_SYMBOL(__cfi_slowpath_diag);
+#endif /* CONFIG_MODULES */
 
-#else /* !CONFIG_MODULES */
+extern s32 __start___kcfi_traps[];
+extern s32 __stop___kcfi_traps[];
 
-void __cfi_slowpath_diag(uint64_t id, void *ptr, void *diag)
+bool is_cfi_trap(unsigned long addr)
 {
-	handle_cfi_failure(ptr); /* No modules */
-}
-EXPORT_SYMBOL(__cfi_slowpath_diag);
+	if (is_trap(addr, __start___kcfi_traps, __stop___kcfi_traps))
+		return true;
 
-#endif /* CONFIG_MODULES */
-
-void cfi_failure_handler(void *data, void *ptr, void *vtable)
-{
-	handle_cfi_failure(ptr);
+	return is_module_cfi_trap(addr);
 }
-EXPORT_SYMBOL(cfi_failure_handler);
+#endif /* CONFIG_ARCH_USES_CFI_TRAPS */
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 5f2090d051ac..8ad2c267ff47 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -6167,11 +6167,6 @@ static struct cgroup *cgroup_get_from_file(struct file *f)
 		return ERR_CAST(css);
 
 	cgrp = css->cgroup;
-	if (!cgroup_on_dfl(cgrp)) {
-		cgroup_put(cgrp);
-		return ERR_PTR(-EBADF);
-	}
-
 	return cgrp;
 }
 
diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
index feb59380c896..793ecff29038 100644
--- a/kernel/cgroup/rstat.c
+++ b/kernel/cgroup/rstat.c
@@ -3,6 +3,10 @@
 
 #include <linux/sched/cputime.h>
 
+#include <linux/bpf.h>
+#include <linux/btf.h>
+#include <linux/btf_ids.h>
+
 static DEFINE_SPINLOCK(cgroup_rstat_lock);
 static DEFINE_PER_CPU(raw_spinlock_t, cgroup_rstat_cpu_lock);
 
@@ -141,6 +145,31 @@ static struct cgroup *cgroup_rstat_cpu_pop_updated(struct cgroup *pos,
 	return pos;
 }
 
+/*
+ * A hook for bpf stat collectors to attach to and flush their stats.
+ * Together with providing bpf kfuncs for cgroup_rstat_updated() and
+ * cgroup_rstat_flush(), this enables a complete workflow where bpf progs that
+ * collect cgroup stats can integrate with rstat for efficient flushing.
+ *
+ * A static noinline declaration here could cause the compiler to optimize away
+ * the function. A global noinline declaration will keep the definition, but may
+ * optimize away the callsite. Therefore, __weak is needed to ensure that the
+ * call is still emitted, by telling the compiler that we don't know what the
+ * function might eventually be.
+ *
+ * __diag_* below are needed to dismiss the missing prototype warning.
+ */
+__diag_push();
+__diag_ignore_all("-Wmissing-prototypes",
+		  "kfuncs which will be used in BPF programs");
+
+__weak noinline void bpf_rstat_flush(struct cgroup *cgrp,
+				     struct cgroup *parent, int cpu)
+{
+}
+
+__diag_pop();
+
 /* see cgroup_rstat_flush() */
 static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep)
 	__releases(&cgroup_rstat_lock) __acquires(&cgroup_rstat_lock)
@@ -168,6 +197,7 @@ static void cgroup_rstat_flush_locked(struct cgroup *cgrp, bool may_sleep)
 			struct cgroup_subsys_state *css;
 
 			cgroup_base_stat_flush(pos, cpu);
+			bpf_rstat_flush(pos, cgroup_parent(pos), cpu);
 
 			rcu_read_lock();
 			list_for_each_entry_rcu(css, &pos->rstat_css_list,
@@ -501,3 +531,21 @@ void cgroup_base_stat_cputime_show(struct seq_file *seq)
 	seq_printf(seq, "core_sched.force_idle_usec %llu\n", forceidle_time);
 #endif
 }
+
+/* Add bpf kfuncs for cgroup_rstat_updated() and cgroup_rstat_flush() */
+BTF_SET8_START(bpf_rstat_kfunc_ids)
+BTF_ID_FLAGS(func, cgroup_rstat_updated)
+BTF_ID_FLAGS(func, cgroup_rstat_flush, KF_SLEEPABLE)
+BTF_SET8_END(bpf_rstat_kfunc_ids)
+
+static const struct btf_kfunc_id_set bpf_rstat_kfunc_set = {
+	.owner          = THIS_MODULE,
+	.set            = &bpf_rstat_kfunc_ids,
+};
+
+static int __init bpf_rstat_kfunc_init(void)
+{
+	return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING,
+					 &bpf_rstat_kfunc_set);
+}
+late_initcall(bpf_rstat_kfunc_init);
diff --git a/kernel/configs/rust.config b/kernel/configs/rust.config
new file mode 100644
index 000000000000..38a7c5362c9c
--- /dev/null
+++ b/kernel/configs/rust.config
@@ -0,0 +1 @@
+CONFIG_RUST=y
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 2621fd24ad26..ff4bffc502c6 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -6893,9 +6893,16 @@ static void perf_output_read_group(struct perf_output_handle *handle,
 {
 	struct perf_event *leader = event->group_leader, *sub;
 	u64 read_format = event->attr.read_format;
+	unsigned long flags;
 	u64 values[6];
 	int n = 0;
 
+	/*
+	 * Disabling interrupts avoids all counter scheduling
+	 * (context switches, timer based rotation and IPIs).
+	 */
+	local_irq_save(flags);
+
 	values[n++] = 1 + leader->nr_siblings;
 
 	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
@@ -6931,6 +6938,8 @@ static void perf_output_read_group(struct perf_output_handle *handle,
 
 		__output_copy(handle, values, n * sizeof(u64));
 	}
+
+	local_irq_restore(flags);
 }
 
 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c
index 3e7e2c2ad2f7..60c20f301a6b 100644
--- a/kernel/kallsyms.c
+++ b/kernel/kallsyms.c
@@ -50,12 +50,20 @@ static unsigned int kallsyms_expand_symbol(unsigned int off,
 	data = &kallsyms_names[off];
 	len = *data;
 	data++;
+	off++;
+
+	/* If MSB is 1, it is a "big" symbol, so needs an additional byte. */
+	if ((len & 0x80) != 0) {
+		len = (len & 0x7F) | (*data << 7);
+		data++;
+		off++;
+	}
 
 	/*
 	 * Update the offset to return the offset for the next symbol on
 	 * the compressed stream.
 	 */
-	off += len + 1;
+	off += len;
 
 	/*
 	 * For every byte on the compressed symbol data, copy the table
@@ -108,7 +116,7 @@ static char kallsyms_get_symbol_type(unsigned int off)
 static unsigned int get_symbol_offset(unsigned long pos)
 {
 	const u8 *name;
-	int i;
+	int i, len;
 
 	/*
 	 * Use the closest marker we have. We have markers every 256 positions,
@@ -122,8 +130,18 @@ static unsigned int get_symbol_offset(unsigned long pos)
 	 * so we just need to add the len to the current pointer for every
 	 * symbol we wish to skip.
 	 */
-	for (i = 0; i < (pos & 0xFF); i++)
-		name = name + (*name) + 1;
+	for (i = 0; i < (pos & 0xFF); i++) {
+		len = *name;
+
+		/*
+		 * If MSB is 1, it is a "big" symbol, so we need to look into
+		 * the next byte (and skip it, too).
+		 */
+		if ((len & 0x80) != 0)
+			len = ((len & 0x7F) | (name[1] << 7)) + 1;
+
+		name = name + len + 1;
+	}
 
 	return name - kallsyms_names;
 }
@@ -159,7 +177,6 @@ static bool cleanup_symbol_name(char *s)
 	 * character in an identifier in C. Suffixes observed:
 	 * - foo.llvm.[0-9a-f]+
 	 * - foo.[0-9a-f]+
-	 * - foo.[0-9a-f]+.cfi_jt
 	 */
 	res = strchr(s, '.');
 	if (res) {
@@ -167,22 +184,6 @@ static bool cleanup_symbol_name(char *s)
 		return true;
 	}
 
-	if (!IS_ENABLED(CONFIG_CFI_CLANG) ||
-	    !IS_ENABLED(CONFIG_LTO_CLANG_THIN) ||
-	    CONFIG_CLANG_VERSION >= 130000)
-		return false;
-
-	/*
-	 * Prior to LLVM 13, the following suffixes were observed when thinLTO
-	 * and CFI are both enabled:
-	 * - foo$[0-9]+
-	 */
-	res = strrchr(s, '$');
-	if (res) {
-		*res = '\0';
-		return true;
-	}
-
 	return false;
 }
 
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index ca9d834d0b84..3220b0a2fb4a 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -1607,9 +1607,10 @@ int register_kprobe(struct kprobe *p)
 	struct kprobe *old_p;
 	struct module *probed_mod;
 	kprobe_opcode_t *addr;
+	bool on_func_entry;
 
 	/* Adjust probe address from symbol */
-	addr = kprobe_addr(p);
+	addr = _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry);
 	if (IS_ERR(addr))
 		return PTR_ERR(addr);
 	p->addr = addr;
@@ -1629,6 +1630,9 @@ int register_kprobe(struct kprobe *p)
 
 	mutex_lock(&kprobe_mutex);
 
+	if (on_func_entry)
+		p->flags |= KPROBE_FLAG_ON_FUNC_ENTRY;
+
 	old_p = get_kprobe(p->addr);
 	if (old_p) {
 		/* Since this may unoptimize 'old_p', locking 'text_mutex'. */
diff --git a/kernel/kthread.c b/kernel/kthread.c
index 3c677918d8f2..28a6b7ab4a0f 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -1050,8 +1050,7 @@ static void __kthread_queue_delayed_work(struct kthread_worker *worker,
 	struct timer_list *timer = &dwork->timer;
 	struct kthread_work *work = &dwork->work;
 
-	WARN_ON_FUNCTION_MISMATCH(timer->function,
-				  kthread_delayed_work_timer_fn);
+	WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
 
 	/*
 	 * If @delay is 0, queue @dwork->work immediately.  This is for
diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c
index bc475e62279d..ec06ce59d728 100644
--- a/kernel/livepatch/core.c
+++ b/kernel/livepatch/core.c
@@ -213,7 +213,7 @@ static int klp_resolve_symbols(Elf_Shdr *sechdrs, const char *strtab,
 	 * we use the smallest/strictest upper bound possible (56, based on
 	 * the current definition of MODULE_NAME_LEN) to prevent overflows.
 	 */
-	BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 128);
+	BUILD_BUG_ON(MODULE_NAME_LEN < 56 || KSYM_NAME_LEN != 512);
 
 	relas = (Elf_Rela *) relasec->sh_addr;
 	/* For each rela in this klp relocation section */
@@ -227,7 +227,7 @@ static int klp_resolve_symbols(Elf_Shdr *sechdrs, const char *strtab,
 
 		/* Format: .klp.sym.sym_objname.sym_name,sympos */
 		cnt = sscanf(strtab + sym->st_name,
-			     ".klp.sym.%55[^.].%127[^,],%lu",
+			     ".klp.sym.%55[^.].%511[^,],%lu",
 			     sym_objname, sym_name, &sympos);
 		if (cnt != 3) {
 			pr_err("symbol %s has an incorrectly formatted name\n",
diff --git a/kernel/module/main.c b/kernel/module/main.c
index a4e4d84b6f4e..70c0b2c6fef8 100644
--- a/kernel/module/main.c
+++ b/kernel/module/main.c
@@ -53,6 +53,7 @@
 #include <linux/bsearch.h>
 #include <linux/dynamic_debug.h>
 #include <linux/audit.h>
+#include <linux/cfi.h>
 #include <uapi/linux/module.h>
 #include "internal.h"
 
@@ -1144,8 +1145,6 @@ void __weak module_arch_freeing_init(struct module *mod)
 {
 }
 
-static void cfi_cleanup(struct module *mod);
-
 /* Free a module, remove from lists, etc. */
 static void free_module(struct module *mod)
 {
@@ -1190,9 +1189,6 @@ static void free_module(struct module *mod)
 		       mod->name);
 	mutex_unlock(&module_mutex);
 
-	/* Clean up CFI for the module. */
-	cfi_cleanup(mod);
-
 	/* This may be empty, but that's OK */
 	module_arch_freeing_init(mod);
 	module_memfree(mod->init_layout.base);
@@ -2602,8 +2598,9 @@ static int complete_formation(struct module *mod, struct load_info *info)
 	if (err < 0)
 		goto out;
 
-	/* This relies on module_mutex for list integrity. */
+	/* These rely on module_mutex for list integrity. */
 	module_bug_finalize(info->hdr, info->sechdrs, mod);
+	module_cfi_finalize(info->hdr, info->sechdrs, mod);
 
 	if (module_check_misalignment(mod))
 		goto out_misaligned;
@@ -2665,8 +2662,6 @@ static int unknown_module_param_cb(char *param, char *val, const char *modname,
 	return 0;
 }
 
-static void cfi_init(struct module *mod);
-
 /*
  * Allocate and load the module: note that size of section 0 is always
  * zero, and we rely on this for optional sections.
@@ -2796,9 +2791,6 @@ static int load_module(struct load_info *info, const char __user *uargs,
 
 	flush_module_icache(mod);
 
-	/* Setup CFI for the module. */
-	cfi_init(mod);
-
 	/* Now copy in args */
 	mod->args = strndup_user(uargs, ~0UL >> 1);
 	if (IS_ERR(mod->args)) {
@@ -2875,7 +2867,6 @@ static int load_module(struct load_info *info, const char __user *uargs,
 	synchronize_rcu();
 	kfree(mod->args);
  free_arch_cleanup:
-	cfi_cleanup(mod);
 	module_arch_cleanup(mod);
  free_modinfo:
 	free_modinfo(mod);
@@ -2961,41 +2952,6 @@ static inline int within(unsigned long addr, void *start, unsigned long size)
 	return ((void *)addr >= start && (void *)addr < start + size);
 }
 
-static void cfi_init(struct module *mod)
-{
-#ifdef CONFIG_CFI_CLANG
-	initcall_t *init;
-#ifdef CONFIG_MODULE_UNLOAD
-	exitcall_t *exit;
-#endif
-
-	rcu_read_lock_sched();
-	mod->cfi_check = (cfi_check_fn)
-		find_kallsyms_symbol_value(mod, "__cfi_check");
-	init = (initcall_t *)
-		find_kallsyms_symbol_value(mod, "__cfi_jt_init_module");
-	/* Fix init/exit functions to point to the CFI jump table */
-	if (init)
-		mod->init = *init;
-#ifdef CONFIG_MODULE_UNLOAD
-	exit = (exitcall_t *)
-		find_kallsyms_symbol_value(mod, "__cfi_jt_cleanup_module");
-	if (exit)
-		mod->exit = *exit;
-#endif
-	rcu_read_unlock_sched();
-
-	cfi_module_add(mod, mod_tree.addr_min);
-#endif
-}
-
-static void cfi_cleanup(struct module *mod)
-{
-#ifdef CONFIG_CFI_CLANG
-	cfi_module_remove(mod, mod_tree.addr_min);
-#endif
-}
-
 /* Keep in sync with MODULE_FLAGS_BUF_SIZE !!! */
 char *module_flags(struct module *mod, char *buf, bool show_state)
 {
diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c
index 827075944d28..c6272d466e58 100644
--- a/kernel/power/suspend.c
+++ b/kernel/power/suspend.c
@@ -136,6 +136,9 @@ static void s2idle_loop(void)
 			break;
 		}
 
+		if (s2idle_ops && s2idle_ops->check)
+			s2idle_ops->check();
+
 		s2idle_enter();
 	}
 
diff --git a/kernel/rcu/rcutorture.c b/kernel/rcu/rcutorture.c
index d8e1b270a065..503c2aa845a4 100644
--- a/kernel/rcu/rcutorture.c
+++ b/kernel/rcu/rcutorture.c
@@ -84,10 +84,15 @@ torture_param(int, fwd_progress_holdoff, 60, "Time between forward-progress test
 torture_param(bool, fwd_progress_need_resched, 1, "Hide cond_resched() behind need_resched()");
 torture_param(bool, gp_cond, false, "Use conditional/async GP wait primitives");
 torture_param(bool, gp_cond_exp, false, "Use conditional/async expedited GP wait primitives");
+torture_param(bool, gp_cond_full, false, "Use conditional/async full-state GP wait primitives");
+torture_param(bool, gp_cond_exp_full, false,
+		    "Use conditional/async full-stateexpedited GP wait primitives");
 torture_param(bool, gp_exp, false, "Use expedited GP wait primitives");
 torture_param(bool, gp_normal, false, "Use normal (non-expedited) GP wait primitives");
 torture_param(bool, gp_poll, false, "Use polling GP wait primitives");
 torture_param(bool, gp_poll_exp, false, "Use polling expedited GP wait primitives");
+torture_param(bool, gp_poll_full, false, "Use polling full-state GP wait primitives");
+torture_param(bool, gp_poll_exp_full, false, "Use polling full-state expedited GP wait primitives");
 torture_param(bool, gp_sync, false, "Use synchronous GP wait primitives");
 torture_param(int, irqreader, 1, "Allow RCU readers from irq handlers");
 torture_param(int, leakpointer, 0, "Leak pointer dereferences from readers");
@@ -194,16 +199,24 @@ static int rcu_torture_writer_state;
 #define RTWS_DEF_FREE		3
 #define RTWS_EXP_SYNC		4
 #define RTWS_COND_GET		5
-#define RTWS_COND_GET_EXP	6
-#define RTWS_COND_SYNC		7
-#define RTWS_COND_SYNC_EXP	8
-#define RTWS_POLL_GET		9
-#define RTWS_POLL_GET_EXP	10
-#define RTWS_POLL_WAIT		11
-#define RTWS_POLL_WAIT_EXP	12
-#define RTWS_SYNC		13
-#define RTWS_STUTTER		14
-#define RTWS_STOPPING		15
+#define RTWS_COND_GET_FULL	6
+#define RTWS_COND_GET_EXP	7
+#define RTWS_COND_GET_EXP_FULL	8
+#define RTWS_COND_SYNC		9
+#define RTWS_COND_SYNC_FULL	10
+#define RTWS_COND_SYNC_EXP	11
+#define RTWS_COND_SYNC_EXP_FULL	12
+#define RTWS_POLL_GET		13
+#define RTWS_POLL_GET_FULL	14
+#define RTWS_POLL_GET_EXP	15
+#define RTWS_POLL_GET_EXP_FULL	16
+#define RTWS_POLL_WAIT		17
+#define RTWS_POLL_WAIT_FULL	18
+#define RTWS_POLL_WAIT_EXP	19
+#define RTWS_POLL_WAIT_EXP_FULL	20
+#define RTWS_SYNC		21
+#define RTWS_STUTTER		22
+#define RTWS_STOPPING		23
 static const char * const rcu_torture_writer_state_names[] = {
 	"RTWS_FIXED_DELAY",
 	"RTWS_DELAY",
@@ -211,13 +224,21 @@ static const char * const rcu_torture_writer_state_names[] = {
 	"RTWS_DEF_FREE",
 	"RTWS_EXP_SYNC",
 	"RTWS_COND_GET",
+	"RTWS_COND_GET_FULL",
 	"RTWS_COND_GET_EXP",
+	"RTWS_COND_GET_EXP_FULL",
 	"RTWS_COND_SYNC",
+	"RTWS_COND_SYNC_FULL",
 	"RTWS_COND_SYNC_EXP",
+	"RTWS_COND_SYNC_EXP_FULL",
 	"RTWS_POLL_GET",
+	"RTWS_POLL_GET_FULL",
 	"RTWS_POLL_GET_EXP",
+	"RTWS_POLL_GET_EXP_FULL",
 	"RTWS_POLL_WAIT",
+	"RTWS_POLL_WAIT_FULL",
 	"RTWS_POLL_WAIT_EXP",
+	"RTWS_POLL_WAIT_EXP_FULL",
 	"RTWS_SYNC",
 	"RTWS_STUTTER",
 	"RTWS_STOPPING",
@@ -332,13 +353,21 @@ struct rcu_torture_ops {
 	void (*exp_sync)(void);
 	unsigned long (*get_gp_state_exp)(void);
 	unsigned long (*start_gp_poll_exp)(void);
+	void (*start_gp_poll_exp_full)(struct rcu_gp_oldstate *rgosp);
 	bool (*poll_gp_state_exp)(unsigned long oldstate);
 	void (*cond_sync_exp)(unsigned long oldstate);
+	void (*cond_sync_exp_full)(struct rcu_gp_oldstate *rgosp);
 	unsigned long (*get_gp_state)(void);
+	void (*get_gp_state_full)(struct rcu_gp_oldstate *rgosp);
 	unsigned long (*get_gp_completed)(void);
+	void (*get_gp_completed_full)(struct rcu_gp_oldstate *rgosp);
 	unsigned long (*start_gp_poll)(void);
+	void (*start_gp_poll_full)(struct rcu_gp_oldstate *rgosp);
 	bool (*poll_gp_state)(unsigned long oldstate);
+	bool (*poll_gp_state_full)(struct rcu_gp_oldstate *rgosp);
+	bool (*poll_need_2gp)(bool poll, bool poll_full);
 	void (*cond_sync)(unsigned long oldstate);
+	void (*cond_sync_full)(struct rcu_gp_oldstate *rgosp);
 	call_rcu_func_t call;
 	void (*cb_barrier)(void);
 	void (*fqs)(void);
@@ -489,6 +518,11 @@ static void rcu_sync_torture_init(void)
 	INIT_LIST_HEAD(&rcu_torture_removed);
 }
 
+static bool rcu_poll_need_2gp(bool poll, bool poll_full)
+{
+	return poll;
+}
+
 static struct rcu_torture_ops rcu_ops = {
 	.ttype			= RCU_FLAVOR,
 	.init			= rcu_sync_torture_init,
@@ -502,12 +536,19 @@ static struct rcu_torture_ops rcu_ops = {
 	.sync			= synchronize_rcu,
 	.exp_sync		= synchronize_rcu_expedited,
 	.get_gp_state		= get_state_synchronize_rcu,
+	.get_gp_state_full	= get_state_synchronize_rcu_full,
 	.get_gp_completed	= get_completed_synchronize_rcu,
+	.get_gp_completed_full	= get_completed_synchronize_rcu_full,
 	.start_gp_poll		= start_poll_synchronize_rcu,
+	.start_gp_poll_full	= start_poll_synchronize_rcu_full,
 	.poll_gp_state		= poll_state_synchronize_rcu,
+	.poll_gp_state_full	= poll_state_synchronize_rcu_full,
+	.poll_need_2gp		= rcu_poll_need_2gp,
 	.cond_sync		= cond_synchronize_rcu,
+	.cond_sync_full		= cond_synchronize_rcu_full,
 	.get_gp_state_exp	= get_state_synchronize_rcu,
 	.start_gp_poll_exp	= start_poll_synchronize_rcu_expedited,
+	.start_gp_poll_exp_full	= start_poll_synchronize_rcu_expedited_full,
 	.poll_gp_state_exp	= poll_state_synchronize_rcu,
 	.cond_sync_exp		= cond_synchronize_rcu_expedited,
 	.call			= call_rcu,
@@ -709,6 +750,9 @@ static struct rcu_torture_ops srcud_ops = {
 	.deferred_free	= srcu_torture_deferred_free,
 	.sync		= srcu_torture_synchronize,
 	.exp_sync	= srcu_torture_synchronize_expedited,
+	.get_gp_state	= srcu_torture_get_gp_state,
+	.start_gp_poll	= srcu_torture_start_gp_poll,
+	.poll_gp_state	= srcu_torture_poll_gp_state,
 	.call		= srcu_torture_call,
 	.cb_barrier	= srcu_torture_barrier,
 	.stats		= srcu_torture_stats,
@@ -1148,15 +1192,35 @@ static int nsynctypes;
  */
 static void rcu_torture_write_types(void)
 {
-	bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_exp1 = gp_exp;
-	bool gp_poll_exp1 = gp_poll_exp, gp_normal1 = gp_normal, gp_poll1 = gp_poll;
-	bool gp_sync1 = gp_sync;
+	bool gp_cond1 = gp_cond, gp_cond_exp1 = gp_cond_exp, gp_cond_full1 = gp_cond_full;
+	bool gp_cond_exp_full1 = gp_cond_exp_full, gp_exp1 = gp_exp, gp_poll_exp1 = gp_poll_exp;
+	bool gp_poll_exp_full1 = gp_poll_exp_full, gp_normal1 = gp_normal, gp_poll1 = gp_poll;
+	bool gp_poll_full1 = gp_poll_full, gp_sync1 = gp_sync;
 
 	/* Initialize synctype[] array.  If none set, take default. */
-	if (!gp_cond1 && !gp_cond_exp1 && !gp_exp1 && !gp_poll_exp &&
-	    !gp_normal1 && !gp_poll1 && !gp_sync1)
-		gp_cond1 = gp_cond_exp1 = gp_exp1 = gp_poll_exp1 =
-			   gp_normal1 = gp_poll1 = gp_sync1 = true;
+	if (!gp_cond1 &&
+	    !gp_cond_exp1 &&
+	    !gp_cond_full1 &&
+	    !gp_cond_exp_full1 &&
+	    !gp_exp1 &&
+	    !gp_poll_exp1 &&
+	    !gp_poll_exp_full1 &&
+	    !gp_normal1 &&
+	    !gp_poll1 &&
+	    !gp_poll_full1 &&
+	    !gp_sync1) {
+		gp_cond1 = true;
+		gp_cond_exp1 = true;
+		gp_cond_full1 = true;
+		gp_cond_exp_full1 = true;
+		gp_exp1 = true;
+		gp_poll_exp1 = true;
+		gp_poll_exp_full1 = true;
+		gp_normal1 = true;
+		gp_poll1 = true;
+		gp_poll_full1 = true;
+		gp_sync1 = true;
+	}
 	if (gp_cond1 && cur_ops->get_gp_state && cur_ops->cond_sync) {
 		synctype[nsynctypes++] = RTWS_COND_GET;
 		pr_info("%s: Testing conditional GPs.\n", __func__);
@@ -1169,6 +1233,19 @@ static void rcu_torture_write_types(void)
 	} else if (gp_cond_exp && (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp)) {
 		pr_alert("%s: gp_cond_exp without primitives.\n", __func__);
 	}
+	if (gp_cond_full1 && cur_ops->get_gp_state && cur_ops->cond_sync_full) {
+		synctype[nsynctypes++] = RTWS_COND_GET_FULL;
+		pr_info("%s: Testing conditional full-state GPs.\n", __func__);
+	} else if (gp_cond_full && (!cur_ops->get_gp_state || !cur_ops->cond_sync_full)) {
+		pr_alert("%s: gp_cond_full without primitives.\n", __func__);
+	}
+	if (gp_cond_exp_full1 && cur_ops->get_gp_state_exp && cur_ops->cond_sync_exp_full) {
+		synctype[nsynctypes++] = RTWS_COND_GET_EXP_FULL;
+		pr_info("%s: Testing conditional full-state expedited GPs.\n", __func__);
+	} else if (gp_cond_exp_full &&
+		   (!cur_ops->get_gp_state_exp || !cur_ops->cond_sync_exp_full)) {
+		pr_alert("%s: gp_cond_exp_full without primitives.\n", __func__);
+	}
 	if (gp_exp1 && cur_ops->exp_sync) {
 		synctype[nsynctypes++] = RTWS_EXP_SYNC;
 		pr_info("%s: Testing expedited GPs.\n", __func__);
@@ -1187,12 +1264,25 @@ static void rcu_torture_write_types(void)
 	} else if (gp_poll && (!cur_ops->start_gp_poll || !cur_ops->poll_gp_state)) {
 		pr_alert("%s: gp_poll without primitives.\n", __func__);
 	}
+	if (gp_poll_full1 && cur_ops->start_gp_poll_full && cur_ops->poll_gp_state_full) {
+		synctype[nsynctypes++] = RTWS_POLL_GET_FULL;
+		pr_info("%s: Testing polling full-state GPs.\n", __func__);
+	} else if (gp_poll_full && (!cur_ops->start_gp_poll_full || !cur_ops->poll_gp_state_full)) {
+		pr_alert("%s: gp_poll_full without primitives.\n", __func__);
+	}
 	if (gp_poll_exp1 && cur_ops->start_gp_poll_exp && cur_ops->poll_gp_state_exp) {
 		synctype[nsynctypes++] = RTWS_POLL_GET_EXP;
 		pr_info("%s: Testing polling expedited GPs.\n", __func__);
 	} else if (gp_poll_exp && (!cur_ops->start_gp_poll_exp || !cur_ops->poll_gp_state_exp)) {
 		pr_alert("%s: gp_poll_exp without primitives.\n", __func__);
 	}
+	if (gp_poll_exp_full1 && cur_ops->start_gp_poll_exp_full && cur_ops->poll_gp_state_full) {
+		synctype[nsynctypes++] = RTWS_POLL_GET_EXP_FULL;
+		pr_info("%s: Testing polling full-state expedited GPs.\n", __func__);
+	} else if (gp_poll_exp_full &&
+		   (!cur_ops->start_gp_poll_exp_full || !cur_ops->poll_gp_state_full)) {
+		pr_alert("%s: gp_poll_exp_full without primitives.\n", __func__);
+	}
 	if (gp_sync1 && cur_ops->sync) {
 		synctype[nsynctypes++] = RTWS_SYNC;
 		pr_info("%s: Testing normal GPs.\n", __func__);
@@ -1202,6 +1292,40 @@ static void rcu_torture_write_types(void)
 }
 
 /*
+ * Do the specified rcu_torture_writer() synchronous grace period,
+ * while also testing out the polled APIs.  Note well that the single-CPU
+ * grace-period optimizations must be accounted for.
+ */
+static void do_rtws_sync(struct torture_random_state *trsp, void (*sync)(void))
+{
+	unsigned long cookie;
+	struct rcu_gp_oldstate cookie_full;
+	bool dopoll;
+	bool dopoll_full;
+	unsigned long r = torture_random(trsp);
+
+	dopoll = cur_ops->get_gp_state && cur_ops->poll_gp_state && !(r & 0x300);
+	dopoll_full = cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full && !(r & 0xc00);
+	if (dopoll || dopoll_full)
+		cpus_read_lock();
+	if (dopoll)
+		cookie = cur_ops->get_gp_state();
+	if (dopoll_full)
+		cur_ops->get_gp_state_full(&cookie_full);
+	if (cur_ops->poll_need_2gp && cur_ops->poll_need_2gp(dopoll, dopoll_full))
+		sync();
+	sync();
+	WARN_ONCE(dopoll && !cur_ops->poll_gp_state(cookie),
+		  "%s: Cookie check 3 failed %pS() online %*pbl.",
+		  __func__, sync, cpumask_pr_args(cpu_online_mask));
+	WARN_ONCE(dopoll_full && !cur_ops->poll_gp_state_full(&cookie_full),
+		  "%s: Cookie check 4 failed %pS() online %*pbl",
+		  __func__, sync, cpumask_pr_args(cpu_online_mask));
+	if (dopoll || dopoll_full)
+		cpus_read_unlock();
+}
+
+/*
  * RCU torture writer kthread.  Repeatedly substitutes a new structure
  * for that pointed to by rcu_torture_current, freeing the old structure
  * after a series of grace periods (the "pipeline").
@@ -1212,8 +1336,10 @@ rcu_torture_writer(void *arg)
 	bool boot_ended;
 	bool can_expedite = !rcu_gp_is_expedited() && !rcu_gp_is_normal();
 	unsigned long cookie;
+	struct rcu_gp_oldstate cookie_full;
 	int expediting = 0;
 	unsigned long gp_snap;
+	struct rcu_gp_oldstate gp_snap_full;
 	int i;
 	int idx;
 	int oldnice = task_nice(current);
@@ -1261,11 +1387,12 @@ rcu_torture_writer(void *arg)
 			atomic_inc(&rcu_torture_wcount[i]);
 			WRITE_ONCE(old_rp->rtort_pipe_count,
 				   old_rp->rtort_pipe_count + 1);
+
+			// Make sure readers block polled grace periods.
 			if (cur_ops->get_gp_state && cur_ops->poll_gp_state) {
 				idx = cur_ops->readlock();
 				cookie = cur_ops->get_gp_state();
-				WARN_ONCE(rcu_torture_writer_state != RTWS_DEF_FREE &&
-					  cur_ops->poll_gp_state(cookie),
+				WARN_ONCE(cur_ops->poll_gp_state(cookie),
 					  "%s: Cookie check 1 failed %s(%d) %lu->%lu\n",
 					  __func__,
 					  rcu_torture_writer_state_getname(),
@@ -1277,6 +1404,21 @@ rcu_torture_writer(void *arg)
 				}
 				cur_ops->readunlock(idx);
 			}
+			if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full) {
+				idx = cur_ops->readlock();
+				cur_ops->get_gp_state_full(&cookie_full);
+				WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),
+					  "%s: Cookie check 5 failed %s(%d) online %*pbl\n",
+					  __func__,
+					  rcu_torture_writer_state_getname(),
+					  rcu_torture_writer_state,
+					  cpumask_pr_args(cpu_online_mask));
+				if (cur_ops->get_gp_completed_full) {
+					cur_ops->get_gp_completed_full(&cookie_full);
+					WARN_ON_ONCE(!cur_ops->poll_gp_state_full(&cookie_full));
+				}
+				cur_ops->readunlock(idx);
+			}
 			switch (synctype[torture_random(&rand) % nsynctypes]) {
 			case RTWS_DEF_FREE:
 				rcu_torture_writer_state = RTWS_DEF_FREE;
@@ -1284,12 +1426,7 @@ rcu_torture_writer(void *arg)
 				break;
 			case RTWS_EXP_SYNC:
 				rcu_torture_writer_state = RTWS_EXP_SYNC;
-				if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
-					cookie = cur_ops->get_gp_state();
-				cur_ops->exp_sync();
-				cur_ops->exp_sync();
-				if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
-					WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie));
+				do_rtws_sync(&rand, cur_ops->exp_sync);
 				rcu_torture_pipe_update(old_rp);
 				break;
 			case RTWS_COND_GET:
@@ -1308,6 +1445,22 @@ rcu_torture_writer(void *arg)
 				cur_ops->cond_sync_exp(gp_snap);
 				rcu_torture_pipe_update(old_rp);
 				break;
+			case RTWS_COND_GET_FULL:
+				rcu_torture_writer_state = RTWS_COND_GET_FULL;
+				cur_ops->get_gp_state_full(&gp_snap_full);
+				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
+				rcu_torture_writer_state = RTWS_COND_SYNC_FULL;
+				cur_ops->cond_sync_full(&gp_snap_full);
+				rcu_torture_pipe_update(old_rp);
+				break;
+			case RTWS_COND_GET_EXP_FULL:
+				rcu_torture_writer_state = RTWS_COND_GET_EXP_FULL;
+				cur_ops->get_gp_state_full(&gp_snap_full);
+				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
+				rcu_torture_writer_state = RTWS_COND_SYNC_EXP_FULL;
+				cur_ops->cond_sync_exp_full(&gp_snap_full);
+				rcu_torture_pipe_update(old_rp);
+				break;
 			case RTWS_POLL_GET:
 				rcu_torture_writer_state = RTWS_POLL_GET;
 				gp_snap = cur_ops->start_gp_poll();
@@ -1317,6 +1470,15 @@ rcu_torture_writer(void *arg)
 								  &rand);
 				rcu_torture_pipe_update(old_rp);
 				break;
+			case RTWS_POLL_GET_FULL:
+				rcu_torture_writer_state = RTWS_POLL_GET_FULL;
+				cur_ops->start_gp_poll_full(&gp_snap_full);
+				rcu_torture_writer_state = RTWS_POLL_WAIT_FULL;
+				while (!cur_ops->poll_gp_state_full(&gp_snap_full))
+					torture_hrtimeout_jiffies(torture_random(&rand) % 16,
+								  &rand);
+				rcu_torture_pipe_update(old_rp);
+				break;
 			case RTWS_POLL_GET_EXP:
 				rcu_torture_writer_state = RTWS_POLL_GET_EXP;
 				gp_snap = cur_ops->start_gp_poll_exp();
@@ -1326,14 +1488,18 @@ rcu_torture_writer(void *arg)
 								  &rand);
 				rcu_torture_pipe_update(old_rp);
 				break;
+			case RTWS_POLL_GET_EXP_FULL:
+				rcu_torture_writer_state = RTWS_POLL_GET_EXP_FULL;
+				cur_ops->start_gp_poll_exp_full(&gp_snap_full);
+				rcu_torture_writer_state = RTWS_POLL_WAIT_EXP_FULL;
+				while (!cur_ops->poll_gp_state_full(&gp_snap_full))
+					torture_hrtimeout_jiffies(torture_random(&rand) % 16,
+								  &rand);
+				rcu_torture_pipe_update(old_rp);
+				break;
 			case RTWS_SYNC:
 				rcu_torture_writer_state = RTWS_SYNC;
-				if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
-					cookie = cur_ops->get_gp_state();
-				cur_ops->sync();
-				cur_ops->sync();
-				if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
-					WARN_ON_ONCE(!cur_ops->poll_gp_state(cookie));
+				do_rtws_sync(&rand, cur_ops->sync);
 				rcu_torture_pipe_update(old_rp);
 				break;
 			default:
@@ -1400,6 +1566,7 @@ static int
 rcu_torture_fakewriter(void *arg)
 {
 	unsigned long gp_snap;
+	struct rcu_gp_oldstate gp_snap_full;
 	DEFINE_TORTURE_RANDOM(rand);
 
 	VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
@@ -1438,6 +1605,16 @@ rcu_torture_fakewriter(void *arg)
 				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
 				cur_ops->cond_sync_exp(gp_snap);
 				break;
+			case RTWS_COND_GET_FULL:
+				cur_ops->get_gp_state_full(&gp_snap_full);
+				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
+				cur_ops->cond_sync_full(&gp_snap_full);
+				break;
+			case RTWS_COND_GET_EXP_FULL:
+				cur_ops->get_gp_state_full(&gp_snap_full);
+				torture_hrtimeout_jiffies(torture_random(&rand) % 16, &rand);
+				cur_ops->cond_sync_exp_full(&gp_snap_full);
+				break;
 			case RTWS_POLL_GET:
 				gp_snap = cur_ops->start_gp_poll();
 				while (!cur_ops->poll_gp_state(gp_snap)) {
@@ -1445,6 +1622,13 @@ rcu_torture_fakewriter(void *arg)
 								  &rand);
 				}
 				break;
+			case RTWS_POLL_GET_FULL:
+				cur_ops->start_gp_poll_full(&gp_snap_full);
+				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
+					torture_hrtimeout_jiffies(torture_random(&rand) % 16,
+								  &rand);
+				}
+				break;
 			case RTWS_POLL_GET_EXP:
 				gp_snap = cur_ops->start_gp_poll_exp();
 				while (!cur_ops->poll_gp_state_exp(gp_snap)) {
@@ -1452,6 +1636,13 @@ rcu_torture_fakewriter(void *arg)
 								  &rand);
 				}
 				break;
+			case RTWS_POLL_GET_EXP_FULL:
+				cur_ops->start_gp_poll_exp_full(&gp_snap_full);
+				while (!cur_ops->poll_gp_state_full(&gp_snap_full)) {
+					torture_hrtimeout_jiffies(torture_random(&rand) % 16,
+								  &rand);
+				}
+				break;
 			case RTWS_SYNC:
 				cur_ops->sync();
 				break;
@@ -1715,7 +1906,9 @@ rcutorture_loop_extend(int *readstate, struct torture_random_state *trsp,
  */
 static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
 {
+	bool checkpolling = !(torture_random(trsp) & 0xfff);
 	unsigned long cookie;
+	struct rcu_gp_oldstate cookie_full;
 	int i;
 	unsigned long started;
 	unsigned long completed;
@@ -1731,8 +1924,12 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
 	WARN_ON_ONCE(!rcu_is_watching());
 	newstate = rcutorture_extend_mask(readstate, trsp);
 	rcutorture_one_extend(&readstate, newstate, trsp, rtrsp++);
-	if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
-		cookie = cur_ops->get_gp_state();
+	if (checkpolling) {
+		if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
+			cookie = cur_ops->get_gp_state();
+		if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
+			cur_ops->get_gp_state_full(&cookie_full);
+	}
 	started = cur_ops->get_gp_seq();
 	ts = rcu_trace_clock_local();
 	p = rcu_dereference_check(rcu_torture_current,
@@ -1766,13 +1963,22 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp, long myid)
 	}
 	__this_cpu_inc(rcu_torture_batch[completed]);
 	preempt_enable();
-	if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
-		WARN_ONCE(cur_ops->poll_gp_state(cookie),
-			  "%s: Cookie check 2 failed %s(%d) %lu->%lu\n",
-			  __func__,
-			  rcu_torture_writer_state_getname(),
-			  rcu_torture_writer_state,
-			  cookie, cur_ops->get_gp_state());
+	if (checkpolling) {
+		if (cur_ops->get_gp_state && cur_ops->poll_gp_state)
+			WARN_ONCE(cur_ops->poll_gp_state(cookie),
+				  "%s: Cookie check 2 failed %s(%d) %lu->%lu\n",
+				  __func__,
+				  rcu_torture_writer_state_getname(),
+				  rcu_torture_writer_state,
+				  cookie, cur_ops->get_gp_state());
+		if (cur_ops->get_gp_state_full && cur_ops->poll_gp_state_full)
+			WARN_ONCE(cur_ops->poll_gp_state_full(&cookie_full),
+				  "%s: Cookie check 6 failed %s(%d) online %*pbl\n",
+				  __func__,
+				  rcu_torture_writer_state_getname(),
+				  rcu_torture_writer_state,
+				  cpumask_pr_args(cpu_online_mask));
+	}
 	rcutorture_one_extend(&readstate, 0, trsp, rtrsp);
 	WARN_ON_ONCE(readstate);
 	// This next splat is expected behavior if leakpointer, especially
@@ -2600,12 +2806,12 @@ static int rcutorture_oom_notify(struct notifier_block *self,
 	for (i = 0; i < fwd_progress; i++)
 		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
 	pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
-	rcu_barrier();
+	cur_ops->cb_barrier();
 	ncbs = 0;
 	for (i = 0; i < fwd_progress; i++)
 		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
 	pr_info("%s: Freed %lu RCU callbacks.\n", __func__, ncbs);
-	rcu_barrier();
+	cur_ops->cb_barrier();
 	ncbs = 0;
 	for (i = 0; i < fwd_progress; i++)
 		ncbs += rcu_torture_fwd_prog_cbfree(&rfp[i]);
diff --git a/kernel/rcu/srcutiny.c b/kernel/rcu/srcutiny.c
index 92c002d65482..33adafdad261 100644
--- a/kernel/rcu/srcutiny.c
+++ b/kernel/rcu/srcutiny.c
@@ -117,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp)
 	struct srcu_struct *ssp;
 
 	ssp = container_of(wp, struct srcu_struct, srcu_work);
-	if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
+	if (ssp->srcu_gp_running || ULONG_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
 		return; /* Already running or nothing to do. */
 
 	/* Remove recently arrived callbacks and wait for readers. */
@@ -150,17 +150,17 @@ void srcu_drive_gp(struct work_struct *wp)
 	 * straighten that out.
 	 */
 	WRITE_ONCE(ssp->srcu_gp_running, false);
-	if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
+	if (ULONG_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
 		schedule_work(&ssp->srcu_work);
 }
 EXPORT_SYMBOL_GPL(srcu_drive_gp);
 
 static void srcu_gp_start_if_needed(struct srcu_struct *ssp)
 {
-	unsigned short cookie;
+	unsigned long cookie;
 
 	cookie = get_state_synchronize_srcu(ssp);
-	if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))
+	if (ULONG_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))
 		return;
 	WRITE_ONCE(ssp->srcu_idx_max, cookie);
 	if (!READ_ONCE(ssp->srcu_gp_running)) {
@@ -215,7 +215,7 @@ unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)
 	barrier();
 	ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;
 	barrier();
-	return ret & USHRT_MAX;
+	return ret;
 }
 EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);
 
@@ -240,10 +240,10 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu);
  */
 bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)
 {
-	bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie);
+	unsigned long cur_s = READ_ONCE(ssp->srcu_idx);
 
 	barrier();
-	return ret;
+	return ULONG_CMP_GE(cur_s, cookie) || ULONG_CMP_LT(cur_s, cookie - 3);
 }
 EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);
 
diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h
index 83c7e6620d40..f5bf6fb430da 100644
--- a/kernel/rcu/tasks.h
+++ b/kernel/rcu/tasks.h
@@ -560,7 +560,7 @@ static int __noreturn rcu_tasks_kthread(void *arg)
 static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp)
 {
 	/* Complain if the scheduler has not started.  */
-	RCU_LOCKDEP_WARN(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE,
+	WARN_ONCE(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE,
 			 "synchronize_rcu_tasks called too soon");
 
 	// If the grace-period kthread is running, use it.
@@ -1500,6 +1500,7 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop)
 		if (rcu_tasks_trace_pertask_prep(t, true))
 			trc_add_holdout(t, hop);
 		rcu_read_unlock();
+		cond_resched_tasks_rcu_qs();
 	}
 
 	// Only after all running tasks have been accounted for is it
@@ -1520,6 +1521,7 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop)
 			raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
 		}
 		raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
+		cond_resched_tasks_rcu_qs();
 	}
 
 	// Re-enable CPU hotplug now that the holdout list is populated.
@@ -1619,6 +1621,7 @@ static void check_all_holdout_tasks_trace(struct list_head *hop,
 			trc_del_holdout(t);
 		else if (needreport)
 			show_stalled_task_trace(t, firstreport);
+		cond_resched_tasks_rcu_qs();
 	}
 
 	// Re-enable CPU hotplug now that the holdout list scan has completed.
diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c
index f0561ee16b9c..a33a8d4942c3 100644
--- a/kernel/rcu/tiny.c
+++ b/kernel/rcu/tiny.c
@@ -158,6 +158,10 @@ void synchronize_rcu(void)
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 
+static void tiny_rcu_leak_callback(struct rcu_head *rhp)
+{
+}
+
 /*
  * Post an RCU callback to be invoked after the end of an RCU grace
  * period.  But since we have but one CPU, that would be after any
@@ -165,9 +169,20 @@ EXPORT_SYMBOL_GPL(synchronize_rcu);
  */
 void call_rcu(struct rcu_head *head, rcu_callback_t func)
 {
+	static atomic_t doublefrees;
 	unsigned long flags;
 
-	debug_rcu_head_queue(head);
+	if (debug_rcu_head_queue(head)) {
+		if (atomic_inc_return(&doublefrees) < 4) {
+			pr_err("%s(): Double-freed CB %p->%pS()!!!  ", __func__, head, head->func);
+			mem_dump_obj(head);
+		}
+
+		if (!__is_kvfree_rcu_offset((unsigned long)head->func))
+			WRITE_ONCE(head->func, tiny_rcu_leak_callback);
+		return;
+	}
+
 	head->func = func;
 	head->next = NULL;
 
@@ -184,6 +199,16 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func)
 EXPORT_SYMBOL_GPL(call_rcu);
 
 /*
+ * Store a grace-period-counter "cookie".  For more information,
+ * see the Tree RCU header comment.
+ */
+void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+	rgosp->rgos_norm = RCU_GET_STATE_COMPLETED;
+}
+EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full);
+
+/*
  * Return a grace-period-counter "cookie".  For more information,
  * see the Tree RCU header comment.
  */
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 79aea7df4345..6bb8e72bc815 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -76,6 +76,7 @@
 /* Data structures. */
 
 static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, rcu_data) = {
+	.gpwrap = true,
 #ifdef CONFIG_RCU_NOCB_CPU
 	.cblist.flags = SEGCBLIST_RCU_CORE,
 #endif
@@ -1755,6 +1756,8 @@ static noinline void rcu_gp_cleanup(void)
 			dump_blkd_tasks(rnp, 10);
 		WARN_ON_ONCE(rnp->qsmask);
 		WRITE_ONCE(rnp->gp_seq, new_gp_seq);
+		if (!rnp->parent)
+			smp_mb(); // Order against failing poll_state_synchronize_rcu_full().
 		rdp = this_cpu_ptr(&rcu_data);
 		if (rnp == rdp->mynode)
 			needgp = __note_gp_changes(rnp, rdp) || needgp;
@@ -2341,8 +2344,8 @@ void rcu_sched_clock_irq(int user)
 	rcu_flavor_sched_clock_irq(user);
 	if (rcu_pending(user))
 		invoke_rcu_core();
-	if (user)
-		rcu_tasks_classic_qs(current, false);
+	if (user || rcu_is_cpu_rrupt_from_idle())
+		rcu_note_voluntary_context_switch(current);
 	lockdep_assert_irqs_disabled();
 
 	trace_rcu_utilization(TPS("End scheduler-tick"));
@@ -2832,7 +2835,7 @@ EXPORT_SYMBOL_GPL(call_rcu);
 
 
 /* Maximum number of jiffies to wait before draining a batch. */
-#define KFREE_DRAIN_JIFFIES (HZ / 50)
+#define KFREE_DRAIN_JIFFIES (5 * HZ)
 #define KFREE_N_BATCHES 2
 #define FREE_N_CHANNELS 2
 
@@ -3093,6 +3096,21 @@ need_offload_krc(struct kfree_rcu_cpu *krcp)
 	return !!krcp->head;
 }
 
+static void
+schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)
+{
+	long delay, delay_left;
+
+	delay = READ_ONCE(krcp->count) >= KVFREE_BULK_MAX_ENTR ? 1:KFREE_DRAIN_JIFFIES;
+	if (delayed_work_pending(&krcp->monitor_work)) {
+		delay_left = krcp->monitor_work.timer.expires - jiffies;
+		if (delay < delay_left)
+			mod_delayed_work(system_wq, &krcp->monitor_work, delay);
+		return;
+	}
+	queue_delayed_work(system_wq, &krcp->monitor_work, delay);
+}
+
 /*
  * This function is invoked after the KFREE_DRAIN_JIFFIES timeout.
  */
@@ -3150,7 +3168,7 @@ static void kfree_rcu_monitor(struct work_struct *work)
 	// work to repeat an attempt. Because previous batches are
 	// still in progress.
 	if (need_offload_krc(krcp))
-		schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
+		schedule_delayed_monitor_work(krcp);
 
 	raw_spin_unlock_irqrestore(&krcp->lock, flags);
 }
@@ -3183,15 +3201,16 @@ static void fill_page_cache_func(struct work_struct *work)
 		bnode = (struct kvfree_rcu_bulk_data *)
 			__get_free_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
 
-		if (bnode) {
-			raw_spin_lock_irqsave(&krcp->lock, flags);
-			pushed = put_cached_bnode(krcp, bnode);
-			raw_spin_unlock_irqrestore(&krcp->lock, flags);
+		if (!bnode)
+			break;
 
-			if (!pushed) {
-				free_page((unsigned long) bnode);
-				break;
-			}
+		raw_spin_lock_irqsave(&krcp->lock, flags);
+		pushed = put_cached_bnode(krcp, bnode);
+		raw_spin_unlock_irqrestore(&krcp->lock, flags);
+
+		if (!pushed) {
+			free_page((unsigned long) bnode);
+			break;
 		}
 	}
 
@@ -3338,7 +3357,7 @@ void kvfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
 
 	// Set timer to drain after KFREE_DRAIN_JIFFIES.
 	if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING)
-		schedule_delayed_work(&krcp->monitor_work, KFREE_DRAIN_JIFFIES);
+		schedule_delayed_monitor_work(krcp);
 
 unlock_return:
 	krc_this_cpu_unlock(krcp, flags);
@@ -3371,7 +3390,7 @@ kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
 		atomic_set(&krcp->backoff_page_cache_fill, 1);
 	}
 
-	return count;
+	return count == 0 ? SHRINK_EMPTY : count;
 }
 
 static unsigned long
@@ -3414,49 +3433,27 @@ void __init kfree_rcu_scheduler_running(void)
 
 		raw_spin_lock_irqsave(&krcp->lock, flags);
 		if (need_offload_krc(krcp))
-			schedule_delayed_work_on(cpu, &krcp->monitor_work, KFREE_DRAIN_JIFFIES);
+			schedule_delayed_monitor_work(krcp);
 		raw_spin_unlock_irqrestore(&krcp->lock, flags);
 	}
 }
 
 /*
  * During early boot, any blocking grace-period wait automatically
- * implies a grace period.  Later on, this is never the case for PREEMPTION.
+ * implies a grace period.
  *
- * However, because a context switch is a grace period for !PREEMPTION, any
- * blocking grace-period wait automatically implies a grace period if
- * there is only one CPU online at any point time during execution of
- * either synchronize_rcu() or synchronize_rcu_expedited().  It is OK to
- * occasionally incorrectly indicate that there are multiple CPUs online
- * when there was in fact only one the whole time, as this just adds some
- * overhead: RCU still operates correctly.
+ * Later on, this could in theory be the case for kernels built with
+ * CONFIG_SMP=y && CONFIG_PREEMPTION=y running on a single CPU, but this
+ * is not a common case.  Furthermore, this optimization would cause
+ * the rcu_gp_oldstate structure to expand by 50%, so this potential
+ * grace-period optimization is ignored once the scheduler is running.
  */
 static int rcu_blocking_is_gp(void)
 {
-	int ret;
-
-	// Invoking preempt_model_*() too early gets a splat.
-	if (rcu_scheduler_active == RCU_SCHEDULER_INACTIVE ||
-	    preempt_model_full() || preempt_model_rt())
-		return rcu_scheduler_active == RCU_SCHEDULER_INACTIVE;
+	if (rcu_scheduler_active != RCU_SCHEDULER_INACTIVE)
+		return false;
 	might_sleep();  /* Check for RCU read-side critical section. */
-	preempt_disable();
-	/*
-	 * If the rcu_state.n_online_cpus counter is equal to one,
-	 * there is only one CPU, and that CPU sees all prior accesses
-	 * made by any CPU that was online at the time of its access.
-	 * Furthermore, if this counter is equal to one, its value cannot
-	 * change until after the preempt_enable() below.
-	 *
-	 * Furthermore, if rcu_state.n_online_cpus is equal to one here,
-	 * all later CPUs (both this one and any that come online later
-	 * on) are guaranteed to see all accesses prior to this point
-	 * in the code, without the need for additional memory barriers.
-	 * Those memory barriers are provided by CPU-hotplug code.
-	 */
-	ret = READ_ONCE(rcu_state.n_online_cpus) <= 1;
-	preempt_enable();
-	return ret;
+	return true;
 }
 
 /**
@@ -3499,30 +3496,59 @@ static int rcu_blocking_is_gp(void)
  */
 void synchronize_rcu(void)
 {
+	unsigned long flags;
+	struct rcu_node *rnp;
+
 	RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map) ||
 			 lock_is_held(&rcu_lock_map) ||
 			 lock_is_held(&rcu_sched_lock_map),
 			 "Illegal synchronize_rcu() in RCU read-side critical section");
-	if (rcu_blocking_is_gp()) {
-		// Note well that this code runs with !PREEMPT && !SMP.
-		// In addition, all code that advances grace periods runs at
-		// process level.  Therefore, this normal GP overlaps with
-		// other normal GPs only by being fully nested within them,
-		// which allows reuse of ->gp_seq_polled_snap.
-		rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap);
-		rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap);
-		if (rcu_init_invoked())
-			cond_resched_tasks_rcu_qs();
-		return;  // Context allows vacuous grace periods.
+	if (!rcu_blocking_is_gp()) {
+		if (rcu_gp_is_expedited())
+			synchronize_rcu_expedited();
+		else
+			wait_rcu_gp(call_rcu);
+		return;
 	}
-	if (rcu_gp_is_expedited())
-		synchronize_rcu_expedited();
-	else
-		wait_rcu_gp(call_rcu);
+
+	// Context allows vacuous grace periods.
+	// Note well that this code runs with !PREEMPT && !SMP.
+	// In addition, all code that advances grace periods runs at
+	// process level.  Therefore, this normal GP overlaps with other
+	// normal GPs only by being fully nested within them, which allows
+	// reuse of ->gp_seq_polled_snap.
+	rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap);
+	rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap);
+
+	// Update the normal grace-period counters to record
+	// this grace period, but only those used by the boot CPU.
+	// The rcu_scheduler_starting() will take care of the rest of
+	// these counters.
+	local_irq_save(flags);
+	WARN_ON_ONCE(num_online_cpus() > 1);
+	rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT);
+	for (rnp = this_cpu_ptr(&rcu_data)->mynode; rnp; rnp = rnp->parent)
+		rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;
+	local_irq_restore(flags);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 
 /**
+ * get_completed_synchronize_rcu_full - Return a full pre-completed polled state cookie
+ * @rgosp: Place to put state cookie
+ *
+ * Stores into @rgosp a value that will always be treated by functions
+ * like poll_state_synchronize_rcu_full() as a cookie whose grace period
+ * has already completed.
+ */
+void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+	rgosp->rgos_norm = RCU_GET_STATE_COMPLETED;
+	rgosp->rgos_exp = RCU_GET_STATE_COMPLETED;
+}
+EXPORT_SYMBOL_GPL(get_completed_synchronize_rcu_full);
+
+/**
  * get_state_synchronize_rcu - Snapshot current RCU state
  *
  * Returns a cookie that is used by a later call to cond_synchronize_rcu()
@@ -3541,21 +3567,42 @@ unsigned long get_state_synchronize_rcu(void)
 EXPORT_SYMBOL_GPL(get_state_synchronize_rcu);
 
 /**
- * start_poll_synchronize_rcu - Snapshot and start RCU grace period
+ * get_state_synchronize_rcu_full - Snapshot RCU state, both normal and expedited
+ * @rgosp: location to place combined normal/expedited grace-period state
  *
- * Returns a cookie that is used by a later call to cond_synchronize_rcu()
- * or poll_state_synchronize_rcu() to determine whether or not a full
- * grace period has elapsed in the meantime.  If the needed grace period
- * is not already slated to start, notifies RCU core of the need for that
- * grace period.
+ * Places the normal and expedited grace-period states in @rgosp.  This
+ * state value can be passed to a later call to cond_synchronize_rcu_full()
+ * or poll_state_synchronize_rcu_full() to determine whether or not a
+ * grace period (whether normal or expedited) has elapsed in the meantime.
+ * The rcu_gp_oldstate structure takes up twice the memory of an unsigned
+ * long, but is guaranteed to see all grace periods.  In contrast, the
+ * combined state occupies less memory, but can sometimes fail to take
+ * grace periods into account.
  *
- * Interrupts must be enabled for the case where it is necessary to awaken
- * the grace-period kthread.
+ * This does not guarantee that the needed grace period will actually
+ * start.
  */
-unsigned long start_poll_synchronize_rcu(void)
+void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+	struct rcu_node *rnp = rcu_get_root();
+
+	/*
+	 * Any prior manipulation of RCU-protected data must happen
+	 * before the loads from ->gp_seq and ->expedited_sequence.
+	 */
+	smp_mb();  /* ^^^ */
+	rgosp->rgos_norm = rcu_seq_snap(&rnp->gp_seq);
+	rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence);
+}
+EXPORT_SYMBOL_GPL(get_state_synchronize_rcu_full);
+
+/*
+ * Helper function for start_poll_synchronize_rcu() and
+ * start_poll_synchronize_rcu_full().
+ */
+static void start_poll_synchronize_rcu_common(void)
 {
 	unsigned long flags;
-	unsigned long gp_seq = get_state_synchronize_rcu();
 	bool needwake;
 	struct rcu_data *rdp;
 	struct rcu_node *rnp;
@@ -3575,17 +3622,57 @@ unsigned long start_poll_synchronize_rcu(void)
 	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
 	if (needwake)
 		rcu_gp_kthread_wake();
+}
+
+/**
+ * start_poll_synchronize_rcu - Snapshot and start RCU grace period
+ *
+ * Returns a cookie that is used by a later call to cond_synchronize_rcu()
+ * or poll_state_synchronize_rcu() to determine whether or not a full
+ * grace period has elapsed in the meantime.  If the needed grace period
+ * is not already slated to start, notifies RCU core of the need for that
+ * grace period.
+ *
+ * Interrupts must be enabled for the case where it is necessary to awaken
+ * the grace-period kthread.
+ */
+unsigned long start_poll_synchronize_rcu(void)
+{
+	unsigned long gp_seq = get_state_synchronize_rcu();
+
+	start_poll_synchronize_rcu_common();
 	return gp_seq;
 }
 EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu);
 
 /**
- * poll_state_synchronize_rcu - Conditionally wait for an RCU grace period
+ * start_poll_synchronize_rcu_full - Take a full snapshot and start RCU grace period
+ * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full()
  *
+ * Places the normal and expedited grace-period states in *@rgos.  This
+ * state value can be passed to a later call to cond_synchronize_rcu_full()
+ * or poll_state_synchronize_rcu_full() to determine whether or not a
+ * grace period (whether normal or expedited) has elapsed in the meantime.
+ * If the needed grace period is not already slated to start, notifies
+ * RCU core of the need for that grace period.
+ *
+ * Interrupts must be enabled for the case where it is necessary to awaken
+ * the grace-period kthread.
+ */
+void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+	get_state_synchronize_rcu_full(rgosp);
+
+	start_poll_synchronize_rcu_common();
+}
+EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_full);
+
+/**
+ * poll_state_synchronize_rcu - Has the specified RCU grace period completed?
  * @oldstate: value from get_state_synchronize_rcu() or start_poll_synchronize_rcu()
  *
  * If a full RCU grace period has elapsed since the earlier call from
- * which oldstate was obtained, return @true, otherwise return @false.
+ * which @oldstate was obtained, return @true, otherwise return @false.
  * If @false is returned, it is the caller's responsibility to invoke this
  * function later on until it does return @true.  Alternatively, the caller
  * can explicitly wait for a grace period, for example, by passing @oldstate
@@ -3594,10 +3681,11 @@ EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu);
  * Yes, this function does not take counter wrap into account.
  * But counter wrap is harmless.  If the counter wraps, we have waited for
  * more than a billion grace periods (and way more on a 64-bit system!).
- * Those needing to keep oldstate values for very long time periods
- * (many hours even on 32-bit systems) should check them occasionally
- * and either refresh them or set a flag indicating that the grace period
- * has completed.
+ * Those needing to keep old state values for very long time periods
+ * (many hours even on 32-bit systems) should check them occasionally and
+ * either refresh them or set a flag indicating that the grace period has
+ * completed.  Alternatively, they can use get_completed_synchronize_rcu()
+ * to get a guaranteed-completed grace-period state.
  *
  * This function provides the same memory-ordering guarantees that
  * would be provided by a synchronize_rcu() that was invoked at the call
@@ -3616,8 +3704,56 @@ bool poll_state_synchronize_rcu(unsigned long oldstate)
 EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu);
 
 /**
- * cond_synchronize_rcu - Conditionally wait for an RCU grace period
+ * poll_state_synchronize_rcu_full - Has the specified RCU grace period completed?
+ * @rgosp: value from get_state_synchronize_rcu_full() or start_poll_synchronize_rcu_full()
  *
+ * If a full RCU grace period has elapsed since the earlier call from
+ * which *rgosp was obtained, return @true, otherwise return @false.
+ * If @false is returned, it is the caller's responsibility to invoke this
+ * function later on until it does return @true.  Alternatively, the caller
+ * can explicitly wait for a grace period, for example, by passing @rgosp
+ * to cond_synchronize_rcu() or by directly invoking synchronize_rcu().
+ *
+ * Yes, this function does not take counter wrap into account.
+ * But counter wrap is harmless.  If the counter wraps, we have waited
+ * for more than a billion grace periods (and way more on a 64-bit
+ * system!).  Those needing to keep rcu_gp_oldstate values for very
+ * long time periods (many hours even on 32-bit systems) should check
+ * them occasionally and either refresh them or set a flag indicating
+ * that the grace period has completed.  Alternatively, they can use
+ * get_completed_synchronize_rcu_full() to get a guaranteed-completed
+ * grace-period state.
+ *
+ * This function provides the same memory-ordering guarantees that would
+ * be provided by a synchronize_rcu() that was invoked at the call to
+ * the function that provided @rgosp, and that returned at the end of this
+ * function.  And this guarantee requires that the root rcu_node structure's
+ * ->gp_seq field be checked instead of that of the rcu_state structure.
+ * The problem is that the just-ending grace-period's callbacks can be
+ * invoked between the time that the root rcu_node structure's ->gp_seq
+ * field is updated and the time that the rcu_state structure's ->gp_seq
+ * field is updated.  Therefore, if a single synchronize_rcu() is to
+ * cause a subsequent poll_state_synchronize_rcu_full() to return @true,
+ * then the root rcu_node structure is the one that needs to be polled.
+ */
+bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+	struct rcu_node *rnp = rcu_get_root();
+
+	smp_mb(); // Order against root rcu_node structure grace-period cleanup.
+	if (rgosp->rgos_norm == RCU_GET_STATE_COMPLETED ||
+	    rcu_seq_done_exact(&rnp->gp_seq, rgosp->rgos_norm) ||
+	    rgosp->rgos_exp == RCU_GET_STATE_COMPLETED ||
+	    rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) {
+		smp_mb(); /* Ensure GP ends before subsequent accesses. */
+		return true;
+	}
+	return false;
+}
+EXPORT_SYMBOL_GPL(poll_state_synchronize_rcu_full);
+
+/**
+ * cond_synchronize_rcu - Conditionally wait for an RCU grace period
  * @oldstate: value from get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or start_poll_synchronize_rcu_expedited()
  *
  * If a full RCU grace period has elapsed since the earlier call to
@@ -3641,6 +3777,33 @@ void cond_synchronize_rcu(unsigned long oldstate)
 }
 EXPORT_SYMBOL_GPL(cond_synchronize_rcu);
 
+/**
+ * cond_synchronize_rcu_full - Conditionally wait for an RCU grace period
+ * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full()
+ *
+ * If a full RCU grace period has elapsed since the call to
+ * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(),
+ * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was
+ * obtained, just return.  Otherwise, invoke synchronize_rcu() to wait
+ * for a full grace period.
+ *
+ * Yes, this function does not take counter wrap into account.
+ * But counter wrap is harmless.  If the counter wraps, we have waited for
+ * more than 2 billion grace periods (and way more on a 64-bit system!),
+ * so waiting for a couple of additional grace periods should be just fine.
+ *
+ * This function provides the same memory-ordering guarantees that
+ * would be provided by a synchronize_rcu() that was invoked at the call
+ * to the function that provided @rgosp and that returned at the end of
+ * this function.
+ */
+void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp)
+{
+	if (!poll_state_synchronize_rcu_full(rgosp))
+		synchronize_rcu();
+}
+EXPORT_SYMBOL_GPL(cond_synchronize_rcu_full);
+
 /*
  * Check to see if there is any immediate RCU-related work to be done by
  * the current CPU, returning 1 if so and zero otherwise.  The checks are
@@ -4312,9 +4475,20 @@ early_initcall(rcu_spawn_gp_kthread);
  */
 void rcu_scheduler_starting(void)
 {
+	unsigned long flags;
+	struct rcu_node *rnp;
+
 	WARN_ON(num_online_cpus() != 1);
 	WARN_ON(nr_context_switches() > 0);
 	rcu_test_sync_prims();
+
+	// Fix up the ->gp_seq counters.
+	local_irq_save(flags);
+	rcu_for_each_node_breadth_first(rnp)
+		rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;
+	local_irq_restore(flags);
+
+	// Switch out of early boot mode.
 	rcu_scheduler_active = RCU_SCHEDULER_INIT;
 	rcu_test_sync_prims();
 }
diff --git a/kernel/rcu/tree_exp.h b/kernel/rcu/tree_exp.h
index be667583a554..18e9b4cd78ef 100644
--- a/kernel/rcu/tree_exp.h
+++ b/kernel/rcu/tree_exp.h
@@ -828,11 +828,13 @@ static void rcu_exp_handler(void *unused)
 {
 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
 	struct rcu_node *rnp = rdp->mynode;
+	bool preempt_bh_enabled = !(preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK));
 
 	if (!(READ_ONCE(rnp->expmask) & rdp->grpmask) ||
 	    __this_cpu_read(rcu_data.cpu_no_qs.b.exp))
 		return;
-	if (rcu_is_cpu_rrupt_from_idle()) {
+	if (rcu_is_cpu_rrupt_from_idle() ||
+	    (IS_ENABLED(CONFIG_PREEMPT_COUNT) && preempt_bh_enabled)) {
 		rcu_report_exp_rdp(this_cpu_ptr(&rcu_data));
 		return;
 	}
@@ -906,6 +908,7 @@ static int rcu_print_task_exp_stall(struct rcu_node *rnp)
 void synchronize_rcu_expedited(void)
 {
 	bool boottime = (rcu_scheduler_active == RCU_SCHEDULER_INIT);
+	unsigned long flags;
 	struct rcu_exp_work rew;
 	struct rcu_node *rnp;
 	unsigned long s;
@@ -924,8 +927,11 @@ void synchronize_rcu_expedited(void)
 		// them, which allows reuse of ->gp_seq_polled_exp_snap.
 		rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap);
 		rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap);
-		if (rcu_init_invoked())
-			cond_resched();
+
+		local_irq_save(flags);
+		WARN_ON_ONCE(num_online_cpus() > 1);
+		rcu_state.expedited_sequence += (1 << RCU_SEQ_CTR_SHIFT);
+		local_irq_restore(flags);
 		return;  // Context allows vacuous grace periods.
 	}
 
@@ -1028,6 +1034,24 @@ unsigned long start_poll_synchronize_rcu_expedited(void)
 EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited);
 
 /**
+ * start_poll_synchronize_rcu_expedited_full - Take a full snapshot and start expedited grace period
+ * @rgosp: Place to put snapshot of grace-period state
+ *
+ * Places the normal and expedited grace-period states in rgosp.  This
+ * state value can be passed to a later call to cond_synchronize_rcu_full()
+ * or poll_state_synchronize_rcu_full() to determine whether or not a
+ * grace period (whether normal or expedited) has elapsed in the meantime.
+ * If the needed expedited grace period is not already slated to start,
+ * initiates that grace period.
+ */
+void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)
+{
+	get_state_synchronize_rcu_full(rgosp);
+	(void)start_poll_synchronize_rcu_expedited();
+}
+EXPORT_SYMBOL_GPL(start_poll_synchronize_rcu_expedited_full);
+
+/**
  * cond_synchronize_rcu_expedited - Conditionally wait for an expedited RCU grace period
  *
  * @oldstate: value from get_state_synchronize_rcu(), start_poll_synchronize_rcu(), or start_poll_synchronize_rcu_expedited()
@@ -1053,3 +1077,30 @@ void cond_synchronize_rcu_expedited(unsigned long oldstate)
 		synchronize_rcu_expedited();
 }
 EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited);
+
+/**
+ * cond_synchronize_rcu_expedited_full - Conditionally wait for an expedited RCU grace period
+ * @rgosp: value from get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(), or start_poll_synchronize_rcu_expedited_full()
+ *
+ * If a full RCU grace period has elapsed since the call to
+ * get_state_synchronize_rcu_full(), start_poll_synchronize_rcu_full(),
+ * or start_poll_synchronize_rcu_expedited_full() from which @rgosp was
+ * obtained, just return.  Otherwise, invoke synchronize_rcu_expedited()
+ * to wait for a full grace period.
+ *
+ * Yes, this function does not take counter wrap into account.
+ * But counter wrap is harmless.  If the counter wraps, we have waited for
+ * more than 2 billion grace periods (and way more on a 64-bit system!),
+ * so waiting for a couple of additional grace periods should be just fine.
+ *
+ * This function provides the same memory-ordering guarantees that
+ * would be provided by a synchronize_rcu() that was invoked at the call
+ * to the function that provided @rgosp and that returned at the end of
+ * this function.
+ */
+void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp)
+{
+	if (!poll_state_synchronize_rcu_full(rgosp))
+		synchronize_rcu_expedited();
+}
+EXPORT_SYMBOL_GPL(cond_synchronize_rcu_expedited_full);
diff --git a/kernel/rcu/tree_nocb.h b/kernel/rcu/tree_nocb.h
index a8f574d8850d..0a5f0ef41484 100644
--- a/kernel/rcu/tree_nocb.h
+++ b/kernel/rcu/tree_nocb.h
@@ -1111,7 +1111,7 @@ int rcu_nocb_cpu_deoffload(int cpu)
 			if (!ret)
 				cpumask_clear_cpu(cpu, rcu_nocb_mask);
 		} else {
-			pr_info("NOCB: Can't CB-deoffload an offline CPU\n");
+			pr_info("NOCB: Cannot CB-deoffload offline CPU %d\n", rdp->cpu);
 			ret = -EINVAL;
 		}
 	}
@@ -1196,7 +1196,7 @@ int rcu_nocb_cpu_offload(int cpu)
 			if (!ret)
 				cpumask_set_cpu(cpu, rcu_nocb_mask);
 		} else {
-			pr_info("NOCB: Can't CB-offload an offline CPU\n");
+			pr_info("NOCB: Cannot CB-offload offline CPU %d\n", rdp->cpu);
 			ret = -EINVAL;
 		}
 	}
@@ -1452,8 +1452,8 @@ static void show_rcu_nocb_gp_state(struct rcu_data *rdp)
 		(long)rdp->nocb_gp_seq,
 		rnp->grplo, rnp->grphi, READ_ONCE(rdp->nocb_gp_loops),
 		rdp->nocb_gp_kthread ? task_state_to_char(rdp->nocb_gp_kthread) : '.',
-		rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1,
-		show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread));
+		rdp->nocb_gp_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1,
+		show_rcu_should_be_on_cpu(rdp->nocb_gp_kthread));
 }
 
 /* Dump out nocb kthread state for the specified rcu_data structure. */
@@ -1497,7 +1497,7 @@ static void show_rcu_nocb_state(struct rcu_data *rdp)
 		".B"[!!rcu_cblist_n_cbs(&rdp->nocb_bypass)],
 		rcu_segcblist_n_cbs(&rdp->cblist),
 		rdp->nocb_cb_kthread ? task_state_to_char(rdp->nocb_cb_kthread) : '.',
-		rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_gp_kthread) : -1,
+		rdp->nocb_cb_kthread ? (int)task_cpu(rdp->nocb_cb_kthread) : -1,
 		show_rcu_should_be_on_cpu(rdp->nocb_cb_kthread));
 
 	/* It is OK for GP kthreads to have GP state. */
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 438ecae6bd7e..e3142ee35fc6 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -641,7 +641,8 @@ static void rcu_read_unlock_special(struct task_struct *t)
 
 		expboost = (t->rcu_blocked_node && READ_ONCE(t->rcu_blocked_node->exp_tasks)) ||
 			   (rdp->grpmask & READ_ONCE(rnp->expmask)) ||
-			   IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) ||
+			   (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) &&
+			   ((rdp->grpmask & READ_ONCE(rnp->qsmask)) || t->rcu_blocked_node)) ||
 			   (IS_ENABLED(CONFIG_RCU_BOOST) && irqs_were_disabled &&
 			    t->rcu_blocked_node);
 		// Need to defer quiescent state until everything is enabled.
@@ -718,9 +719,6 @@ static void rcu_flavor_sched_clock_irq(int user)
 	struct task_struct *t = current;
 
 	lockdep_assert_irqs_disabled();
-	if (user || rcu_is_cpu_rrupt_from_idle()) {
-		rcu_note_voluntary_context_switch(current);
-	}
 	if (rcu_preempt_depth() > 0 ||
 	    (preempt_count() & (PREEMPT_MASK | SOFTIRQ_MASK))) {
 		/* No QS, force context switch if deferred. */
@@ -824,6 +822,7 @@ void rcu_read_unlock_strict(void)
 	if (irqs_disabled() || preempt_count() || !rcu_state.gp_kthread)
 		return;
 	rdp = this_cpu_ptr(&rcu_data);
+	rdp->cpu_no_qs.b.norm = false;
 	rcu_report_qs_rdp(rdp);
 	udelay(rcu_unlock_delay);
 }
@@ -869,7 +868,7 @@ void rcu_all_qs(void)
 
 	if (!raw_cpu_read(rcu_data.rcu_urgent_qs))
 		return;
-	preempt_disable();
+	preempt_disable();  // For CONFIG_PREEMPT_COUNT=y kernels
 	/* Load rcu_urgent_qs before other flags. */
 	if (!smp_load_acquire(this_cpu_ptr(&rcu_data.rcu_urgent_qs))) {
 		preempt_enable();
@@ -931,10 +930,13 @@ static notrace bool rcu_preempt_need_deferred_qs(struct task_struct *t)
 	return false;
 }
 
-// Except that we do need to respond to a request by an expedited grace
-// period for a quiescent state from this CPU.  Note that requests from
-// tasks are handled when removing the task from the blocked-tasks list
-// below.
+// Except that we do need to respond to a request by an expedited
+// grace period for a quiescent state from this CPU.  Note that in
+// non-preemptible kernels, there can be no context switches within RCU
+// read-side critical sections, which in turn means that the leaf rcu_node
+// structure's blocked-tasks list is always empty.  is therefore no need to
+// actually check it.  Instead, a quiescent state from this CPU suffices,
+// and this function is only called from such a quiescent state.
 notrace void rcu_preempt_deferred_qs(struct task_struct *t)
 {
 	struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
@@ -972,7 +974,6 @@ static void rcu_flavor_sched_clock_irq(int user)
 		 * neither access nor modify, at least not while the
 		 * corresponding CPU is online.
 		 */
-
 		rcu_qs();
 	}
 }
@@ -1238,8 +1239,11 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
 		    cpu != outgoingcpu)
 			cpumask_set_cpu(cpu, cm);
 	cpumask_and(cm, cm, housekeeping_cpumask(HK_TYPE_RCU));
-	if (cpumask_empty(cm))
+	if (cpumask_empty(cm)) {
 		cpumask_copy(cm, housekeeping_cpumask(HK_TYPE_RCU));
+		if (outgoingcpu >= 0)
+			cpumask_clear_cpu(outgoingcpu, cm);
+	}
 	set_cpus_allowed_ptr(t, cm);
 	mutex_unlock(&rnp->boost_kthread_mutex);
 	free_cpumask_var(cm);
diff --git a/kernel/rcu/tree_stall.h b/kernel/rcu/tree_stall.h
index c3fbbcc09327..5653560573e2 100644
--- a/kernel/rcu/tree_stall.h
+++ b/kernel/rcu/tree_stall.h
@@ -368,7 +368,7 @@ static void rcu_dump_cpu_stacks(void)
 			if (rnp->qsmask & leaf_node_cpu_bit(rnp, cpu)) {
 				if (cpu_is_offline(cpu))
 					pr_err("Offline CPU %d blocking current GP.\n", cpu);
-				else if (!trigger_single_cpu_backtrace(cpu))
+				else
 					dump_cpu_task(cpu);
 			}
 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
@@ -511,8 +511,7 @@ static void rcu_check_gp_kthread_starvation(void)
 					pr_err("RCU GP kthread last ran on offline CPU %d.\n", cpu);
 				} else  {
 					pr_err("Stack dump where RCU GP kthread last ran:\n");
-					if (!trigger_single_cpu_backtrace(cpu))
-						dump_cpu_task(cpu);
+					dump_cpu_task(cpu);
 				}
 			}
 			wake_up_process(gpk);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ee28253c9ac0..60fdc0faf1c9 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -73,6 +73,7 @@
 
 #include <uapi/linux/sched/types.h>
 
+#include <asm/irq_regs.h>
 #include <asm/switch_to.h>
 #include <asm/tlb.h>
 
@@ -11183,6 +11184,19 @@ struct cgroup_subsys cpu_cgrp_subsys = {
 
 void dump_cpu_task(int cpu)
 {
+	if (cpu == smp_processor_id() && in_hardirq()) {
+		struct pt_regs *regs;
+
+		regs = get_irq_regs();
+		if (regs) {
+			show_regs(regs);
+			return;
+		}
+	}
+
+	if (trigger_single_cpu_backtrace(cpu))
+		return;
+
 	pr_info("Task dump for CPU %d:\n", cpu);
 	sched_show_task(cpu_curr(cpu));
 }
diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
index 1207c78f85c1..9161d1136d01 100644
--- a/kernel/sched/cpufreq_schedutil.c
+++ b/kernel/sched/cpufreq_schedutil.c
@@ -25,6 +25,9 @@ struct sugov_policy {
 	unsigned int		next_freq;
 	unsigned int		cached_raw_freq;
 
+	/* max CPU capacity, which is equal for all CPUs in freq. domain */
+	unsigned long		max;
+
 	/* The next fields are only needed if fast switch cannot be used: */
 	struct			irq_work irq_work;
 	struct			kthread_work work;
@@ -48,7 +51,6 @@ struct sugov_cpu {
 
 	unsigned long		util;
 	unsigned long		bw_dl;
-	unsigned long		max;
 
 	/* The field below is for single-CPU policies only: */
 #ifdef CONFIG_NO_HZ_COMMON
@@ -158,7 +160,6 @@ static void sugov_get_util(struct sugov_cpu *sg_cpu)
 {
 	struct rq *rq = cpu_rq(sg_cpu->cpu);
 
-	sg_cpu->max = arch_scale_cpu_capacity(sg_cpu->cpu);
 	sg_cpu->bw_dl = cpu_bw_dl(rq);
 	sg_cpu->util = effective_cpu_util(sg_cpu->cpu, cpu_util_cfs(sg_cpu->cpu),
 					  FREQUENCY_UTIL, NULL);
@@ -253,6 +254,7 @@ static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,
  */
 static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time)
 {
+	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
 	unsigned long boost;
 
 	/* No boost currently required */
@@ -280,7 +282,8 @@ static void sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time)
 	 * sg_cpu->util is already in capacity scale; convert iowait_boost
 	 * into the same scale so we can compare.
 	 */
-	boost = (sg_cpu->iowait_boost * sg_cpu->max) >> SCHED_CAPACITY_SHIFT;
+	boost = sg_cpu->iowait_boost * sg_policy->max;
+	boost >>= SCHED_CAPACITY_SHIFT;
 	boost = uclamp_rq_util_with(cpu_rq(sg_cpu->cpu), boost, NULL);
 	if (sg_cpu->util < boost)
 		sg_cpu->util = boost;
@@ -337,7 +340,7 @@ static void sugov_update_single_freq(struct update_util_data *hook, u64 time,
 	if (!sugov_update_single_common(sg_cpu, time, flags))
 		return;
 
-	next_f = get_next_freq(sg_policy, sg_cpu->util, sg_cpu->max);
+	next_f = get_next_freq(sg_policy, sg_cpu->util, sg_policy->max);
 	/*
 	 * Do not reduce the frequency if the CPU has not been idle
 	 * recently, as the reduction is likely to be premature then.
@@ -373,6 +376,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time,
 				     unsigned int flags)
 {
 	struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
+	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
 	unsigned long prev_util = sg_cpu->util;
 
 	/*
@@ -399,7 +403,8 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time,
 		sg_cpu->util = prev_util;
 
 	cpufreq_driver_adjust_perf(sg_cpu->cpu, map_util_perf(sg_cpu->bw_dl),
-				   map_util_perf(sg_cpu->util), sg_cpu->max);
+				   map_util_perf(sg_cpu->util),
+				   sg_policy->max);
 
 	sg_cpu->sg_policy->last_freq_update_time = time;
 }
@@ -408,25 +413,19 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
 {
 	struct sugov_policy *sg_policy = sg_cpu->sg_policy;
 	struct cpufreq_policy *policy = sg_policy->policy;
-	unsigned long util = 0, max = 1;
+	unsigned long util = 0;
 	unsigned int j;
 
 	for_each_cpu(j, policy->cpus) {
 		struct sugov_cpu *j_sg_cpu = &per_cpu(sugov_cpu, j);
-		unsigned long j_util, j_max;
 
 		sugov_get_util(j_sg_cpu);
 		sugov_iowait_apply(j_sg_cpu, time);
-		j_util = j_sg_cpu->util;
-		j_max = j_sg_cpu->max;
 
-		if (j_util * max > j_max * util) {
-			util = j_util;
-			max = j_max;
-		}
+		util = max(j_sg_cpu->util, util);
 	}
 
-	return get_next_freq(sg_policy, util, max);
+	return get_next_freq(sg_policy, util, sg_policy->max);
 }
 
 static void
@@ -752,7 +751,7 @@ static int sugov_start(struct cpufreq_policy *policy)
 {
 	struct sugov_policy *sg_policy = policy->governor_data;
 	void (*uu)(struct update_util_data *data, u64 time, unsigned int flags);
-	unsigned int cpu;
+	unsigned int cpu = cpumask_first(policy->cpus);
 
 	sg_policy->freq_update_delay_ns	= sg_policy->tunables->rate_limit_us * NSEC_PER_USEC;
 	sg_policy->last_freq_update_time	= 0;
@@ -760,6 +759,7 @@ static int sugov_start(struct cpufreq_policy *policy)
 	sg_policy->work_in_progress		= false;
 	sg_policy->limits_changed		= false;
 	sg_policy->cached_raw_freq		= 0;
+	sg_policy->max				= arch_scale_cpu_capacity(cpu);
 
 	sg_policy->need_freq_update = cpufreq_driver_test_flags(CPUFREQ_NEED_UPDATE_LIMITS);
 
diff --git a/kernel/smp.c b/kernel/smp.c
index 650810a6f29b..e8cdc025a046 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -370,8 +370,7 @@ static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *
 	if (cpu >= 0) {
 		if (static_branch_unlikely(&csdlock_debug_extended))
 			csd_lock_print_extended(csd, cpu);
-		if (!trigger_single_cpu_backtrace(cpu))
-			dump_cpu_task(cpu);
+		dump_cpu_task(cpu);
 		if (!cpu_cur_csd) {
 			pr_alert("csd: Re-sending CSD lock (#%d) IPI from CPU#%02d to CPU#%02d\n", *bug_id, raw_smp_processor_id(), cpu);
 			arch_send_call_function_single_ipi(cpu);
diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index f7e246336218..8ce3fa0c19e2 100644
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -688,6 +688,7 @@ static struct genl_family family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.ops		= taskstats_ops,
 	.n_ops		= ARRAY_SIZE(taskstats_ops),
+	.resv_start_op	= CGROUPSTATS_CMD_GET + 1,
 	.netnsok	= true,
 };
 
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 1052126bdca2..e9e95c790b8e 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -51,6 +51,12 @@ config HAVE_DYNAMIC_FTRACE_WITH_ARGS
 	 This allows for use of regs_get_kernel_argument() and
 	 kernel_stack_pointer().
 
+config HAVE_DYNAMIC_FTRACE_NO_PATCHABLE
+	bool
+	help
+	  If the architecture generates __patchable_function_entries sections
+	  but does not want them included in the ftrace locations.
+
 config HAVE_FTRACE_MCOUNT_RECORD
 	bool
 	help
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index 68e5cdd24cef..688552df95ca 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -20,6 +20,8 @@
 #include <linux/fprobe.h>
 #include <linux/bsearch.h>
 #include <linux/sort.h>
+#include <linux/key.h>
+#include <linux/verification.h>
 
 #include <net/bpf_sk_storage.h>
 
@@ -1026,11 +1028,30 @@ static const struct bpf_func_proto bpf_get_func_ip_proto_tracing = {
 	.arg1_type	= ARG_PTR_TO_CTX,
 };
 
+#ifdef CONFIG_X86_KERNEL_IBT
+static unsigned long get_entry_ip(unsigned long fentry_ip)
+{
+	u32 instr;
+
+	/* Being extra safe in here in case entry ip is on the page-edge. */
+	if (get_kernel_nofault(instr, (u32 *) fentry_ip - 1))
+		return fentry_ip;
+	if (is_endbr(instr))
+		fentry_ip -= ENDBR_INSN_SIZE;
+	return fentry_ip;
+}
+#else
+#define get_entry_ip(fentry_ip) fentry_ip
+#endif
+
 BPF_CALL_1(bpf_get_func_ip_kprobe, struct pt_regs *, regs)
 {
 	struct kprobe *kp = kprobe_running();
 
-	return kp ? (uintptr_t)kp->addr : 0;
+	if (!kp || !(kp->flags & KPROBE_FLAG_ON_FUNC_ENTRY))
+		return 0;
+
+	return get_entry_ip((uintptr_t)kp->addr);
 }
 
 static const struct bpf_func_proto bpf_get_func_ip_proto_kprobe = {
@@ -1181,6 +1202,184 @@ static const struct bpf_func_proto bpf_get_func_arg_cnt_proto = {
 	.arg1_type	= ARG_PTR_TO_CTX,
 };
 
+#ifdef CONFIG_KEYS
+__diag_push();
+__diag_ignore_all("-Wmissing-prototypes",
+		  "kfuncs which will be used in BPF programs");
+
+/**
+ * bpf_lookup_user_key - lookup a key by its serial
+ * @serial: key handle serial number
+ * @flags: lookup-specific flags
+ *
+ * Search a key with a given *serial* and the provided *flags*.
+ * If found, increment the reference count of the key by one, and
+ * return it in the bpf_key structure.
+ *
+ * The bpf_key structure must be passed to bpf_key_put() when done
+ * with it, so that the key reference count is decremented and the
+ * bpf_key structure is freed.
+ *
+ * Permission checks are deferred to the time the key is used by
+ * one of the available key-specific kfuncs.
+ *
+ * Set *flags* with KEY_LOOKUP_CREATE, to attempt creating a requested
+ * special keyring (e.g. session keyring), if it doesn't yet exist.
+ * Set *flags* with KEY_LOOKUP_PARTIAL, to lookup a key without waiting
+ * for the key construction, and to retrieve uninstantiated keys (keys
+ * without data attached to them).
+ *
+ * Return: a bpf_key pointer with a valid key pointer if the key is found, a
+ *         NULL pointer otherwise.
+ */
+struct bpf_key *bpf_lookup_user_key(u32 serial, u64 flags)
+{
+	key_ref_t key_ref;
+	struct bpf_key *bkey;
+
+	if (flags & ~KEY_LOOKUP_ALL)
+		return NULL;
+
+	/*
+	 * Permission check is deferred until the key is used, as the
+	 * intent of the caller is unknown here.
+	 */
+	key_ref = lookup_user_key(serial, flags, KEY_DEFER_PERM_CHECK);
+	if (IS_ERR(key_ref))
+		return NULL;
+
+	bkey = kmalloc(sizeof(*bkey), GFP_KERNEL);
+	if (!bkey) {
+		key_put(key_ref_to_ptr(key_ref));
+		return NULL;
+	}
+
+	bkey->key = key_ref_to_ptr(key_ref);
+	bkey->has_ref = true;
+
+	return bkey;
+}
+
+/**
+ * bpf_lookup_system_key - lookup a key by a system-defined ID
+ * @id: key ID
+ *
+ * Obtain a bpf_key structure with a key pointer set to the passed key ID.
+ * The key pointer is marked as invalid, to prevent bpf_key_put() from
+ * attempting to decrement the key reference count on that pointer. The key
+ * pointer set in such way is currently understood only by
+ * verify_pkcs7_signature().
+ *
+ * Set *id* to one of the values defined in include/linux/verification.h:
+ * 0 for the primary keyring (immutable keyring of system keys);
+ * VERIFY_USE_SECONDARY_KEYRING for both the primary and secondary keyring
+ * (where keys can be added only if they are vouched for by existing keys
+ * in those keyrings); VERIFY_USE_PLATFORM_KEYRING for the platform
+ * keyring (primarily used by the integrity subsystem to verify a kexec'ed
+ * kerned image and, possibly, the initramfs signature).
+ *
+ * Return: a bpf_key pointer with an invalid key pointer set from the
+ *         pre-determined ID on success, a NULL pointer otherwise
+ */
+struct bpf_key *bpf_lookup_system_key(u64 id)
+{
+	struct bpf_key *bkey;
+
+	if (system_keyring_id_check(id) < 0)
+		return NULL;
+
+	bkey = kmalloc(sizeof(*bkey), GFP_ATOMIC);
+	if (!bkey)
+		return NULL;
+
+	bkey->key = (struct key *)(unsigned long)id;
+	bkey->has_ref = false;
+
+	return bkey;
+}
+
+/**
+ * bpf_key_put - decrement key reference count if key is valid and free bpf_key
+ * @bkey: bpf_key structure
+ *
+ * Decrement the reference count of the key inside *bkey*, if the pointer
+ * is valid, and free *bkey*.
+ */
+void bpf_key_put(struct bpf_key *bkey)
+{
+	if (bkey->has_ref)
+		key_put(bkey->key);
+
+	kfree(bkey);
+}
+
+#ifdef CONFIG_SYSTEM_DATA_VERIFICATION
+/**
+ * bpf_verify_pkcs7_signature - verify a PKCS#7 signature
+ * @data_ptr: data to verify
+ * @sig_ptr: signature of the data
+ * @trusted_keyring: keyring with keys trusted for signature verification
+ *
+ * Verify the PKCS#7 signature *sig_ptr* against the supplied *data_ptr*
+ * with keys in a keyring referenced by *trusted_keyring*.
+ *
+ * Return: 0 on success, a negative value on error.
+ */
+int bpf_verify_pkcs7_signature(struct bpf_dynptr_kern *data_ptr,
+			       struct bpf_dynptr_kern *sig_ptr,
+			       struct bpf_key *trusted_keyring)
+{
+	int ret;
+
+	if (trusted_keyring->has_ref) {
+		/*
+		 * Do the permission check deferred in bpf_lookup_user_key().
+		 * See bpf_lookup_user_key() for more details.
+		 *
+		 * A call to key_task_permission() here would be redundant, as
+		 * it is already done by keyring_search() called by
+		 * find_asymmetric_key().
+		 */
+		ret = key_validate(trusted_keyring->key);
+		if (ret < 0)
+			return ret;
+	}
+
+	return verify_pkcs7_signature(data_ptr->data,
+				      bpf_dynptr_get_size(data_ptr),
+				      sig_ptr->data,
+				      bpf_dynptr_get_size(sig_ptr),
+				      trusted_keyring->key,
+				      VERIFYING_UNSPECIFIED_SIGNATURE, NULL,
+				      NULL);
+}
+#endif /* CONFIG_SYSTEM_DATA_VERIFICATION */
+
+__diag_pop();
+
+BTF_SET8_START(key_sig_kfunc_set)
+BTF_ID_FLAGS(func, bpf_lookup_user_key, KF_ACQUIRE | KF_RET_NULL | KF_SLEEPABLE)
+BTF_ID_FLAGS(func, bpf_lookup_system_key, KF_ACQUIRE | KF_RET_NULL)
+BTF_ID_FLAGS(func, bpf_key_put, KF_RELEASE)
+#ifdef CONFIG_SYSTEM_DATA_VERIFICATION
+BTF_ID_FLAGS(func, bpf_verify_pkcs7_signature, KF_SLEEPABLE)
+#endif
+BTF_SET8_END(key_sig_kfunc_set)
+
+static const struct btf_kfunc_id_set bpf_key_sig_kfunc_set = {
+	.owner = THIS_MODULE,
+	.set = &key_sig_kfunc_set,
+};
+
+static int __init bpf_key_sig_kfuncs_init(void)
+{
+	return register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING,
+					 &bpf_key_sig_kfunc_set);
+}
+
+late_initcall(bpf_key_sig_kfuncs_init);
+#endif /* CONFIG_KEYS */
+
 static const struct bpf_func_proto *
 bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
@@ -2042,9 +2241,15 @@ static __always_inline
 void __bpf_trace_run(struct bpf_prog *prog, u64 *args)
 {
 	cant_sleep();
+	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {
+		bpf_prog_inc_misses_counter(prog);
+		goto out;
+	}
 	rcu_read_lock();
 	(void) bpf_prog_run(prog, args);
 	rcu_read_unlock();
+out:
+	this_cpu_dec(*(prog->active));
 }
 
 #define UNPACK(...)			__VA_ARGS__
@@ -2414,13 +2619,13 @@ kprobe_multi_link_prog_run(struct bpf_kprobe_multi_link *link,
 }
 
 static void
-kprobe_multi_link_handler(struct fprobe *fp, unsigned long entry_ip,
+kprobe_multi_link_handler(struct fprobe *fp, unsigned long fentry_ip,
 			  struct pt_regs *regs)
 {
 	struct bpf_kprobe_multi_link *link;
 
 	link = container_of(fp, struct bpf_kprobe_multi_link, fp);
-	kprobe_multi_link_prog_run(link, entry_ip, regs);
+	kprobe_multi_link_prog_run(link, get_entry_ip(fentry_ip), regs);
 }
 
 static int symbols_cmp_r(const void *a, const void *b, const void *priv)
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 439e2ab6905e..447d2e2a8549 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -8265,8 +8265,7 @@ static int kallsyms_callback(void *data, const char *name,
 	if (args->addrs[idx])
 		return 0;
 
-	addr = ftrace_location(addr);
-	if (!addr)
+	if (!ftrace_location(addr))
 		return 0;
 
 	args->addrs[idx] = addr;
diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c
index 5481ba44a8d6..3f464bbda0e9 100644
--- a/kernel/user_namespace.c
+++ b/kernel/user_namespace.c
@@ -9,6 +9,7 @@
 #include <linux/highuid.h>
 #include <linux/cred.h>
 #include <linux/securebits.h>
+#include <linux/security.h>
 #include <linux/keyctl.h>
 #include <linux/key-type.h>
 #include <keys/user-type.h>
@@ -113,6 +114,10 @@ int create_user_ns(struct cred *new)
 	    !kgid_has_mapping(parent_ns, group))
 		goto fail_dec;
 
+	ret = security_create_user_ns(new);
+	if (ret < 0)
+		goto fail_dec;
+
 	ret = -ENOMEM;
 	ns = kmem_cache_zalloc(user_ns_cachep, GFP_KERNEL);
 	if (!ns)
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 39060a5d0905..7cd5f5e7e0a1 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -1651,7 +1651,7 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
 	struct work_struct *work = &dwork->work;
 
 	WARN_ON_ONCE(!wq);
-	WARN_ON_FUNCTION_MISMATCH(timer->function, delayed_work_timer_fn);
+	WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
 	WARN_ON_ONCE(timer_pending(timer));
 	WARN_ON_ONCE(!list_empty(&work->entry));
 
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index d3e5f36bb01e..f473f7d8a0a2 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -2511,6 +2511,18 @@ config MEMCPY_KUNIT_TEST
 
 	  If unsure, say N.
 
+config IS_SIGNED_TYPE_KUNIT_TEST
+	tristate "Test is_signed_type() macro" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	default KUNIT_ALL_TESTS
+	help
+	  Builds unit tests for the is_signed_type() macro.
+
+	  For more information on KUnit and unit tests in general please refer
+	  to the KUnit documentation in Documentation/dev-tools/kunit/.
+
+	  If unsure, say N.
+
 config OVERFLOW_KUNIT_TEST
 	tristate "Test check_*_overflow() functions at runtime" if !KUNIT_ALL_TESTS
 	depends on KUNIT
@@ -2535,6 +2547,15 @@ config STACKINIT_KUNIT_TEST
 	  CONFIG_GCC_PLUGIN_STRUCTLEAK, CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF,
 	  or CONFIG_GCC_PLUGIN_STRUCTLEAK_BYREF_ALL.
 
+config FORTIFY_KUNIT_TEST
+	tristate "Test fortified str*() and mem*() function internals at runtime" if !KUNIT_ALL_TESTS
+	depends on KUNIT && FORTIFY_SOURCE
+	default KUNIT_ALL_TESTS
+	help
+	  Builds unit tests for checking internals of FORTIFY_SOURCE as used
+	  by the str*() and mem*() family of functions. For testing runtime
+	  traps of FORTIFY_SOURCE, see LKDTM's "FORTIFY_*" tests.
+
 config TEST_UDELAY
 	tristate "udelay test driver"
 	help
@@ -2710,6 +2731,40 @@ config HYPERV_TESTING
 
 endmenu # "Kernel Testing and Coverage"
 
+menu "Rust hacking"
+
+config RUST_DEBUG_ASSERTIONS
+	bool "Debug assertions"
+	depends on RUST
+	help
+	  Enables rustc's `-Cdebug-assertions` codegen option.
+
+	  This flag lets you turn `cfg(debug_assertions)` conditional
+	  compilation on or off. This can be used to enable extra debugging
+	  code in development but not in production. For example, it controls
+	  the behavior of the standard library's `debug_assert!` macro.
+
+	  Note that this will apply to all Rust code, including `core`.
+
+	  If unsure, say N.
+
+config RUST_OVERFLOW_CHECKS
+	bool "Overflow checks"
+	default y
+	depends on RUST
+	help
+	  Enables rustc's `-Coverflow-checks` codegen option.
+
+	  This flag allows you to control the behavior of runtime integer
+	  overflow. When overflow-checks are enabled, a Rust panic will occur
+	  on overflow.
+
+	  Note that this will apply to all Rust code, including `core`.
+
+	  If unsure, say Y.
+
+endmenu # "Rust"
+
 source "Documentation/Kconfig"
 
 endmenu # Kernel hacking
diff --git a/lib/Makefile b/lib/Makefile
index ffabc30a27d4..7d7c9f67eff6 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -377,9 +377,11 @@ obj-$(CONFIG_BITS_TEST) += test_bits.o
 obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o
 obj-$(CONFIG_SLUB_KUNIT_TEST) += slub_kunit.o
 obj-$(CONFIG_MEMCPY_KUNIT_TEST) += memcpy_kunit.o
+obj-$(CONFIG_IS_SIGNED_TYPE_KUNIT_TEST) += is_signed_type_kunit.o
 obj-$(CONFIG_OVERFLOW_KUNIT_TEST) += overflow_kunit.o
 CFLAGS_stackinit_kunit.o += $(call cc-disable-warning, switch-unreachable)
 obj-$(CONFIG_STACKINIT_KUNIT_TEST) += stackinit_kunit.o
+obj-$(CONFIG_FORTIFY_KUNIT_TEST) += fortify_kunit.o
 
 obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o
 
diff --git a/lib/fortify_kunit.c b/lib/fortify_kunit.c
new file mode 100644
index 000000000000..409af07f340a
--- /dev/null
+++ b/lib/fortify_kunit.c
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Runtime test cases for CONFIG_FORTIFY_SOURCE that aren't expected to
+ * Oops the kernel on success. (For those, see drivers/misc/lkdtm/fortify.c)
+ *
+ * For corner cases with UBSAN, try testing with:
+ *
+ * ./tools/testing/kunit/kunit.py run --arch=x86_64 \
+ *	--kconfig_add CONFIG_FORTIFY_SOURCE=y \
+ *	--kconfig_add CONFIG_UBSAN=y \
+ *	--kconfig_add CONFIG_UBSAN_TRAP=y \
+ *	--kconfig_add CONFIG_UBSAN_BOUNDS=y \
+ *	--kconfig_add CONFIG_UBSAN_LOCAL_BOUNDS=y \
+ *	--make_options LLVM=1 fortify
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <kunit/test.h>
+#include <linux/string.h>
+
+static const char array_of_10[] = "this is 10";
+static const char *ptr_of_11 = "this is 11!";
+static char array_unknown[] = "compiler thinks I might change";
+
+static void known_sizes_test(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, __compiletime_strlen("88888888"), 8);
+	KUNIT_EXPECT_EQ(test, __compiletime_strlen(array_of_10), 10);
+	KUNIT_EXPECT_EQ(test, __compiletime_strlen(ptr_of_11), 11);
+
+	KUNIT_EXPECT_EQ(test, __compiletime_strlen(array_unknown), SIZE_MAX);
+	/* Externally defined and dynamically sized string pointer: */
+	KUNIT_EXPECT_EQ(test, __compiletime_strlen(test->name), SIZE_MAX);
+}
+
+/* This is volatile so the optimizer can't perform DCE below. */
+static volatile int pick;
+
+/* Not inline to keep optimizer from figuring out which string we want. */
+static noinline size_t want_minus_one(int pick)
+{
+	const char *str;
+
+	switch (pick) {
+	case 1:
+		str = "4444";
+		break;
+	case 2:
+		str = "333";
+		break;
+	default:
+		str = "1";
+		break;
+	}
+	return __compiletime_strlen(str);
+}
+
+static void control_flow_split_test(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, want_minus_one(pick), SIZE_MAX);
+}
+
+static struct kunit_case fortify_test_cases[] = {
+	KUNIT_CASE(known_sizes_test),
+	KUNIT_CASE(control_flow_split_test),
+	{}
+};
+
+static struct kunit_suite fortify_test_suite = {
+	.name = "fortify",
+	.test_cases = fortify_test_cases,
+};
+
+kunit_test_suite(fortify_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/lib/is_signed_type_kunit.c b/lib/is_signed_type_kunit.c
new file mode 100644
index 000000000000..207207522925
--- /dev/null
+++ b/lib/is_signed_type_kunit.c
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ *	./tools/testing/kunit/kunit.py run is_signed_type [--raw_output]
+ */
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <kunit/test.h>
+#include <linux/compiler.h>
+
+enum unsigned_enum {
+	constant_a = 3,
+};
+
+enum signed_enum {
+	constant_b = -1,
+	constant_c = 2,
+};
+
+static void is_signed_type_test(struct kunit *test)
+{
+	KUNIT_EXPECT_EQ(test, is_signed_type(bool), false);
+	KUNIT_EXPECT_EQ(test, is_signed_type(signed char), true);
+	KUNIT_EXPECT_EQ(test, is_signed_type(unsigned char), false);
+#ifdef __CHAR_UNSIGNED__
+	KUNIT_EXPECT_EQ(test, is_signed_type(char), false);
+#else
+	KUNIT_EXPECT_EQ(test, is_signed_type(char), true);
+#endif
+	KUNIT_EXPECT_EQ(test, is_signed_type(int), true);
+	KUNIT_EXPECT_EQ(test, is_signed_type(unsigned int), false);
+	KUNIT_EXPECT_EQ(test, is_signed_type(long), true);
+	KUNIT_EXPECT_EQ(test, is_signed_type(unsigned long), false);
+	KUNIT_EXPECT_EQ(test, is_signed_type(long long), true);
+	KUNIT_EXPECT_EQ(test, is_signed_type(unsigned long long), false);
+	KUNIT_EXPECT_EQ(test, is_signed_type(enum unsigned_enum), false);
+	KUNIT_EXPECT_EQ(test, is_signed_type(enum signed_enum), true);
+	KUNIT_EXPECT_EQ(test, is_signed_type(void *), false);
+	KUNIT_EXPECT_EQ(test, is_signed_type(const char *), false);
+}
+
+static struct kunit_case is_signed_type_test_cases[] = {
+	KUNIT_CASE(is_signed_type_test),
+	{}
+};
+
+static struct kunit_suite is_signed_type_test_suite = {
+	.name = "is_signed_type",
+	.test_cases = is_signed_type_test_cases,
+};
+
+kunit_test_suite(is_signed_type_test_suite);
+
+MODULE_LICENSE("Dual MIT/GPL");
diff --git a/lib/kunit/Kconfig b/lib/kunit/Kconfig
index 0b5dfb001bac..626719b95bad 100644
--- a/lib/kunit/Kconfig
+++ b/lib/kunit/Kconfig
@@ -59,4 +59,15 @@ config KUNIT_ALL_TESTS
 
 	  If unsure, say N.
 
+config KUNIT_DEFAULT_ENABLED
+	bool "Default value of kunit.enable"
+	default y
+	help
+	  Sets the default value of kunit.enable. If set to N then KUnit
+	  tests will not execute unless kunit.enable=1 is passed to the
+	  kernel command line.
+
+	  In most cases this should be left as Y. Only if additional opt-in
+	  behavior is needed should this be set to N.
+
 endif # KUNIT
diff --git a/lib/kunit/executor.c b/lib/kunit/executor.c
index 5e223327196a..9bbc422c284b 100644
--- a/lib/kunit/executor.c
+++ b/lib/kunit/executor.c
@@ -190,6 +190,10 @@ int kunit_run_all_tests(void)
 {
 	struct suite_set suite_set = {__kunit_suites_start, __kunit_suites_end};
 	int err = 0;
+	if (!kunit_enabled()) {
+		pr_info("kunit: disabled\n");
+		goto out;
+	}
 
 	if (filter_glob_param) {
 		suite_set = kunit_filter_suites(&suite_set, filter_glob_param, &err);
diff --git a/lib/kunit/test.c b/lib/kunit/test.c
index b73d5bb5c473..1e54373309a4 100644
--- a/lib/kunit/test.c
+++ b/lib/kunit/test.c
@@ -55,6 +55,17 @@ EXPORT_SYMBOL_GPL(__kunit_fail_current_test);
 #endif
 
 /*
+ * Enable KUnit tests to run.
+ */
+#ifdef CONFIG_KUNIT_DEFAULT_ENABLED
+static bool enable_param = true;
+#else
+static bool enable_param;
+#endif
+module_param_named(enable, enable_param, bool, 0);
+MODULE_PARM_DESC(enable, "Enable KUnit tests");
+
+/*
  * KUnit statistic mode:
  * 0 - disabled
  * 1 - only when there is more than one subtest
@@ -586,10 +597,20 @@ static void kunit_init_suite(struct kunit_suite *suite)
 	suite->suite_init_err = 0;
 }
 
+bool kunit_enabled(void)
+{
+	return enable_param;
+}
+
 int __kunit_test_suites_init(struct kunit_suite * const * const suites, int num_suites)
 {
 	unsigned int i;
 
+	if (!kunit_enabled() && num_suites > 0) {
+		pr_info("kunit: disabled\n");
+		return 0;
+	}
+
 	for (i = 0; i < num_suites; i++) {
 		kunit_init_suite(suites[i]);
 		kunit_run_tests(suites[i]);
@@ -607,6 +628,9 @@ void __kunit_test_suites_exit(struct kunit_suite **suites, int num_suites)
 {
 	unsigned int i;
 
+	if (!kunit_enabled())
+		return;
+
 	for (i = 0; i < num_suites; i++)
 		kunit_exit_suite(suites[i]);
 
diff --git a/lib/memcpy_kunit.c b/lib/memcpy_kunit.c
index 62f8ffcbbaa3..2b5cc70ac53f 100644
--- a/lib/memcpy_kunit.c
+++ b/lib/memcpy_kunit.c
@@ -29,9 +29,8 @@ struct some_bytes {
 };
 
 #define check(instance, v) do {	\
-	int i;	\
 	BUILD_BUG_ON(sizeof(instance.data) != 32);	\
-	for (i = 0; i < sizeof(instance.data); i++) {	\
+	for (size_t i = 0; i < sizeof(instance.data); i++) {	\
 		KUNIT_ASSERT_EQ_MSG(test, instance.data[i], v, \
 			"line %d: '%s' not initialized to 0x%02x @ %d (saw 0x%02x)\n", \
 			__LINE__, #instance, v, i, instance.data[i]);	\
@@ -39,9 +38,8 @@ struct some_bytes {
 } while (0)
 
 #define compare(name, one, two) do { \
-	int i; \
 	BUILD_BUG_ON(sizeof(one) != sizeof(two)); \
-	for (i = 0; i < sizeof(one); i++) {	\
+	for (size_t i = 0; i < sizeof(one); i++) {	\
 		KUNIT_EXPECT_EQ_MSG(test, one.data[i], two.data[i], \
 			"line %d: %s.data[%d] (0x%02x) != %s.data[%d] (0x%02x)\n", \
 			__LINE__, #one, i, one.data[i], #two, i, two.data[i]); \
@@ -272,10 +270,63 @@ static void memset_test(struct kunit *test)
 #undef TEST_OP
 }
 
+static void strtomem_test(struct kunit *test)
+{
+	static const char input[sizeof(unsigned long)] = "hi";
+	static const char truncate[] = "this is too long";
+	struct {
+		unsigned long canary1;
+		unsigned char output[sizeof(unsigned long)] __nonstring;
+		unsigned long canary2;
+	} wrap;
+
+	memset(&wrap, 0xFF, sizeof(wrap));
+	KUNIT_EXPECT_EQ_MSG(test, wrap.canary1, ULONG_MAX,
+			    "bad initial canary value");
+	KUNIT_EXPECT_EQ_MSG(test, wrap.canary2, ULONG_MAX,
+			    "bad initial canary value");
+
+	/* Check unpadded copy leaves surroundings untouched. */
+	strtomem(wrap.output, input);
+	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
+	KUNIT_EXPECT_EQ(test, wrap.output[0], input[0]);
+	KUNIT_EXPECT_EQ(test, wrap.output[1], input[1]);
+	for (size_t i = 2; i < sizeof(wrap.output); i++)
+		KUNIT_EXPECT_EQ(test, wrap.output[i], 0xFF);
+	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);
+
+	/* Check truncated copy leaves surroundings untouched. */
+	memset(&wrap, 0xFF, sizeof(wrap));
+	strtomem(wrap.output, truncate);
+	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
+	for (size_t i = 0; i < sizeof(wrap.output); i++)
+		KUNIT_EXPECT_EQ(test, wrap.output[i], truncate[i]);
+	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);
+
+	/* Check padded copy leaves only string padded. */
+	memset(&wrap, 0xFF, sizeof(wrap));
+	strtomem_pad(wrap.output, input, 0xAA);
+	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
+	KUNIT_EXPECT_EQ(test, wrap.output[0], input[0]);
+	KUNIT_EXPECT_EQ(test, wrap.output[1], input[1]);
+	for (size_t i = 2; i < sizeof(wrap.output); i++)
+		KUNIT_EXPECT_EQ(test, wrap.output[i], 0xAA);
+	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);
+
+	/* Check truncated padded copy has no padding. */
+	memset(&wrap, 0xFF, sizeof(wrap));
+	strtomem(wrap.output, truncate);
+	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
+	for (size_t i = 0; i < sizeof(wrap.output); i++)
+		KUNIT_EXPECT_EQ(test, wrap.output[i], truncate[i]);
+	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);
+}
+
 static struct kunit_case memcpy_test_cases[] = {
 	KUNIT_CASE(memset_test),
 	KUNIT_CASE(memcpy_test),
 	KUNIT_CASE(memmove_test),
+	KUNIT_CASE(strtomem_test),
 	{}
 };
 
diff --git a/lib/nlattr.c b/lib/nlattr.c
index 86029ad5ead4..40f22b177d69 100644
--- a/lib/nlattr.c
+++ b/lib/nlattr.c
@@ -159,6 +159,31 @@ void nla_get_range_unsigned(const struct nla_policy *pt,
 	}
 }
 
+static u64 nla_get_attr_bo(const struct nla_policy *pt,
+			   const struct nlattr *nla)
+{
+	switch (pt->type) {
+	case NLA_U16:
+		if (pt->network_byte_order)
+			return ntohs(nla_get_be16(nla));
+
+		return nla_get_u16(nla);
+	case NLA_U32:
+		if (pt->network_byte_order)
+			return ntohl(nla_get_be32(nla));
+
+		return nla_get_u32(nla);
+	case NLA_U64:
+		if (pt->network_byte_order)
+			return be64_to_cpu(nla_get_be64(nla));
+
+		return nla_get_u64(nla);
+	}
+
+	WARN_ON_ONCE(1);
+	return 0;
+}
+
 static int nla_validate_range_unsigned(const struct nla_policy *pt,
 				       const struct nlattr *nla,
 				       struct netlink_ext_ack *extack,
@@ -172,12 +197,10 @@ static int nla_validate_range_unsigned(const struct nla_policy *pt,
 		value = nla_get_u8(nla);
 		break;
 	case NLA_U16:
-		value = nla_get_u16(nla);
-		break;
 	case NLA_U32:
-		value = nla_get_u32(nla);
-		break;
 	case NLA_U64:
+		value = nla_get_attr_bo(pt, nla);
+		break;
 	case NLA_MSECS:
 		value = nla_get_u64(nla);
 		break;
diff --git a/lib/once.c b/lib/once.c
index 59149bf3bfb4..2c306f0e891e 100644
--- a/lib/once.c
+++ b/lib/once.c
@@ -66,3 +66,33 @@ void __do_once_done(bool *done, struct static_key_true *once_key,
 	once_disable_jump(once_key, mod);
 }
 EXPORT_SYMBOL(__do_once_done);
+
+static DEFINE_MUTEX(once_mutex);
+
+bool __do_once_sleepable_start(bool *done)
+	__acquires(once_mutex)
+{
+	mutex_lock(&once_mutex);
+	if (*done) {
+		mutex_unlock(&once_mutex);
+		/* Keep sparse happy by restoring an even lock count on
+		 * this mutex. In case we return here, we don't call into
+		 * __do_once_done but return early in the DO_ONCE_SLEEPABLE() macro.
+		 */
+		__acquire(once_mutex);
+		return false;
+	}
+
+	return true;
+}
+EXPORT_SYMBOL(__do_once_sleepable_start);
+
+void __do_once_sleepable_done(bool *done, struct static_key_true *once_key,
+			 struct module *mod)
+	__releases(once_mutex)
+{
+	*done = true;
+	mutex_unlock(&once_mutex);
+	once_disable_jump(once_key, mod);
+}
+EXPORT_SYMBOL(__do_once_sleepable_done);
diff --git a/lib/overflow_kunit.c b/lib/overflow_kunit.c
index 7e3e43679b73..5369634701fa 100644
--- a/lib/overflow_kunit.c
+++ b/lib/overflow_kunit.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR MIT
 /*
  * Test cases for arithmetic overflow checks. See:
- * https://www.kernel.org/doc/html/latest/dev-tools/kunit/kunit-tool.html#configuring-building-and-running-tests
+ * "Running tests with kunit_tool" at Documentation/dev-tools/kunit/start.rst
  *	./tools/testing/kunit/kunit.py run overflow [--raw_output]
  */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
@@ -16,12 +16,15 @@
 #include <linux/types.h>
 #include <linux/vmalloc.h>
 
-#define DEFINE_TEST_ARRAY(t)			\
-	static const struct test_ ## t {	\
-		t a, b;				\
-		t sum, diff, prod;		\
-		bool s_of, d_of, p_of;		\
-	} t ## _tests[]
+#define DEFINE_TEST_ARRAY_TYPED(t1, t2, t)			\
+	static const struct test_ ## t1 ## _ ## t2 ## __ ## t {	\
+		t1 a;						\
+		t2 b;						\
+		t sum, diff, prod;				\
+		bool s_of, d_of, p_of;				\
+	} t1 ## _ ## t2 ## __ ## t ## _tests[]
+
+#define DEFINE_TEST_ARRAY(t)	DEFINE_TEST_ARRAY_TYPED(t, t, t)
 
 DEFINE_TEST_ARRAY(u8) = {
 	{0, 0, 0, 0, 0, false, false, false},
@@ -222,21 +225,27 @@ DEFINE_TEST_ARRAY(s64) = {
 };
 #endif
 
-#define check_one_op(t, fmt, op, sym, a, b, r, of) do {		\
-	t _r;							\
-	bool _of;						\
-								\
-	_of = check_ ## op ## _overflow(a, b, &_r);		\
-	KUNIT_EXPECT_EQ_MSG(test, _of, of,			\
+#define check_one_op(t, fmt, op, sym, a, b, r, of) do {			\
+	int _a_orig = a, _a_bump = a + 1;				\
+	int _b_orig = b, _b_bump = b + 1;				\
+	bool _of;							\
+	t _r;								\
+									\
+	_of = check_ ## op ## _overflow(a, b, &_r);			\
+	KUNIT_EXPECT_EQ_MSG(test, _of, of,				\
 		"expected "fmt" "sym" "fmt" to%s overflow (type %s)\n",	\
-		a, b, of ? "" : " not", #t);			\
-	KUNIT_EXPECT_EQ_MSG(test, _r, r,			\
+		a, b, of ? "" : " not", #t);				\
+	KUNIT_EXPECT_EQ_MSG(test, _r, r,				\
 		"expected "fmt" "sym" "fmt" == "fmt", got "fmt" (type %s)\n", \
-		a, b, r, _r, #t);				\
+		a, b, r, _r, #t);					\
+	/* Check for internal macro side-effects. */			\
+	_of = check_ ## op ## _overflow(_a_orig++, _b_orig++, &_r);	\
+	KUNIT_EXPECT_EQ_MSG(test, _a_orig, _a_bump, "Unexpected " #op " macro side-effect!\n"); \
+	KUNIT_EXPECT_EQ_MSG(test, _b_orig, _b_bump, "Unexpected " #op " macro side-effect!\n"); \
 } while (0)
 
-#define DEFINE_TEST_FUNC(t, fmt)					\
-static void do_test_ ## t(struct kunit *test, const struct test_ ## t *p) \
+#define DEFINE_TEST_FUNC_TYPED(n, t, fmt)				\
+static void do_test_ ## n(struct kunit *test, const struct test_ ## n *p) \
 {							   		\
 	check_one_op(t, fmt, add, "+", p->a, p->b, p->sum, p->s_of);	\
 	check_one_op(t, fmt, add, "+", p->b, p->a, p->sum, p->s_of);	\
@@ -245,15 +254,18 @@ static void do_test_ ## t(struct kunit *test, const struct test_ ## t *p) \
 	check_one_op(t, fmt, mul, "*", p->b, p->a, p->prod, p->p_of);	\
 }									\
 									\
-static void t ## _overflow_test(struct kunit *test) {			\
+static void n ## _overflow_test(struct kunit *test) {			\
 	unsigned i;							\
 									\
-	for (i = 0; i < ARRAY_SIZE(t ## _tests); ++i)			\
-		do_test_ ## t(test, &t ## _tests[i]);			\
+	for (i = 0; i < ARRAY_SIZE(n ## _tests); ++i)			\
+		do_test_ ## n(test, &n ## _tests[i]);			\
 	kunit_info(test, "%zu %s arithmetic tests finished\n",		\
-		ARRAY_SIZE(t ## _tests), #t);				\
+		ARRAY_SIZE(n ## _tests), #n);				\
 }
 
+#define DEFINE_TEST_FUNC(t, fmt)					\
+	DEFINE_TEST_FUNC_TYPED(t ## _ ## t ## __ ## t, t, fmt)
+
 DEFINE_TEST_FUNC(u8, "%d");
 DEFINE_TEST_FUNC(s8, "%d");
 DEFINE_TEST_FUNC(u16, "%d");
@@ -265,9 +277,32 @@ DEFINE_TEST_FUNC(u64, "%llu");
 DEFINE_TEST_FUNC(s64, "%lld");
 #endif
 
-static void overflow_shift_test(struct kunit *test)
-{
-	int count = 0;
+DEFINE_TEST_ARRAY_TYPED(u32, u32, u8) = {
+	{0, 0, 0, 0, 0, false, false, false},
+	{U8_MAX, 2, 1, U8_MAX - 2, U8_MAX - 1, true, false, true},
+	{U8_MAX + 1, 0, 0, 0, 0, true, true, false},
+};
+DEFINE_TEST_FUNC_TYPED(u32_u32__u8, u8, "%d");
+
+DEFINE_TEST_ARRAY_TYPED(u32, u32, int) = {
+	{0, 0, 0, 0, 0, false, false, false},
+	{U32_MAX, 0, -1, -1, 0, true, true, false},
+};
+DEFINE_TEST_FUNC_TYPED(u32_u32__int, int, "%d");
+
+DEFINE_TEST_ARRAY_TYPED(u8, u8, int) = {
+	{0, 0, 0, 0, 0, false, false, false},
+	{U8_MAX, U8_MAX, 2 * U8_MAX, 0, U8_MAX * U8_MAX, false, false, false},
+	{1, 2, 3, -1, 2, false, false, false},
+};
+DEFINE_TEST_FUNC_TYPED(u8_u8__int, int, "%d");
+
+DEFINE_TEST_ARRAY_TYPED(int, int, u8) = {
+	{0, 0, 0, 0, 0, false, false, false},
+	{1, 2, 3, U8_MAX, 2, false, true, false},
+	{-1, 0, U8_MAX, U8_MAX, 0, true, true, false},
+};
+DEFINE_TEST_FUNC_TYPED(int_int__u8, u8, "%d");
 
 /* Args are: value, shift, type, expected result, overflow expected */
 #define TEST_ONE_SHIFT(a, s, t, expect, of)	do {			\
@@ -292,6 +327,10 @@ static void overflow_shift_test(struct kunit *test)
 	count++;							\
 } while (0)
 
+static void shift_sane_test(struct kunit *test)
+{
+	int count = 0;
+
 	/* Sane shifts. */
 	TEST_ONE_SHIFT(1, 0, u8, 1 << 0, false);
 	TEST_ONE_SHIFT(1, 4, u8, 1 << 4, false);
@@ -334,6 +373,13 @@ static void overflow_shift_test(struct kunit *test)
 	TEST_ONE_SHIFT(0, 30, s32, 0, false);
 	TEST_ONE_SHIFT(0, 62, s64, 0, false);
 
+	kunit_info(test, "%d sane shift tests finished\n", count);
+}
+
+static void shift_overflow_test(struct kunit *test)
+{
+	int count = 0;
+
 	/* Overflow: shifted the bit off the end. */
 	TEST_ONE_SHIFT(1, 8, u8, 0, true);
 	TEST_ONE_SHIFT(1, 16, u16, 0, true);
@@ -381,6 +427,13 @@ static void overflow_shift_test(struct kunit *test)
 	/* 0100000100001000001000000010000001000010000001000100010001001011 */
 	TEST_ONE_SHIFT(4686030735197619275LL, 2, s64, 0, true);
 
+	kunit_info(test, "%d overflow shift tests finished\n", count);
+}
+
+static void shift_truncate_test(struct kunit *test)
+{
+	int count = 0;
+
 	/* Overflow: values larger than destination type. */
 	TEST_ONE_SHIFT(0x100, 0, u8, 0, true);
 	TEST_ONE_SHIFT(0xFF, 0, s8, 0, true);
@@ -392,6 +445,33 @@ static void overflow_shift_test(struct kunit *test)
 	TEST_ONE_SHIFT(0xFFFFFFFFUL, 0, int, 0, true);
 	TEST_ONE_SHIFT(0xFFFFFFFFFFFFFFFFULL, 0, s64, 0, true);
 
+	/* Overflow: shifted at or beyond entire type's bit width. */
+	TEST_ONE_SHIFT(0, 8, u8, 0, true);
+	TEST_ONE_SHIFT(0, 9, u8, 0, true);
+	TEST_ONE_SHIFT(0, 8, s8, 0, true);
+	TEST_ONE_SHIFT(0, 9, s8, 0, true);
+	TEST_ONE_SHIFT(0, 16, u16, 0, true);
+	TEST_ONE_SHIFT(0, 17, u16, 0, true);
+	TEST_ONE_SHIFT(0, 16, s16, 0, true);
+	TEST_ONE_SHIFT(0, 17, s16, 0, true);
+	TEST_ONE_SHIFT(0, 32, u32, 0, true);
+	TEST_ONE_SHIFT(0, 33, u32, 0, true);
+	TEST_ONE_SHIFT(0, 32, int, 0, true);
+	TEST_ONE_SHIFT(0, 33, int, 0, true);
+	TEST_ONE_SHIFT(0, 32, s32, 0, true);
+	TEST_ONE_SHIFT(0, 33, s32, 0, true);
+	TEST_ONE_SHIFT(0, 64, u64, 0, true);
+	TEST_ONE_SHIFT(0, 65, u64, 0, true);
+	TEST_ONE_SHIFT(0, 64, s64, 0, true);
+	TEST_ONE_SHIFT(0, 65, s64, 0, true);
+
+	kunit_info(test, "%d truncate shift tests finished\n", count);
+}
+
+static void shift_nonsense_test(struct kunit *test)
+{
+	int count = 0;
+
 	/* Nonsense: negative initial value. */
 	TEST_ONE_SHIFT(-1, 0, s8, 0, true);
 	TEST_ONE_SHIFT(-1, 0, u8, 0, true);
@@ -416,26 +496,6 @@ static void overflow_shift_test(struct kunit *test)
 	TEST_ONE_SHIFT(0, -30, s64, 0, true);
 	TEST_ONE_SHIFT(0, -30, u64, 0, true);
 
-	/* Overflow: shifted at or beyond entire type's bit width. */
-	TEST_ONE_SHIFT(0, 8, u8, 0, true);
-	TEST_ONE_SHIFT(0, 9, u8, 0, true);
-	TEST_ONE_SHIFT(0, 8, s8, 0, true);
-	TEST_ONE_SHIFT(0, 9, s8, 0, true);
-	TEST_ONE_SHIFT(0, 16, u16, 0, true);
-	TEST_ONE_SHIFT(0, 17, u16, 0, true);
-	TEST_ONE_SHIFT(0, 16, s16, 0, true);
-	TEST_ONE_SHIFT(0, 17, s16, 0, true);
-	TEST_ONE_SHIFT(0, 32, u32, 0, true);
-	TEST_ONE_SHIFT(0, 33, u32, 0, true);
-	TEST_ONE_SHIFT(0, 32, int, 0, true);
-	TEST_ONE_SHIFT(0, 33, int, 0, true);
-	TEST_ONE_SHIFT(0, 32, s32, 0, true);
-	TEST_ONE_SHIFT(0, 33, s32, 0, true);
-	TEST_ONE_SHIFT(0, 64, u64, 0, true);
-	TEST_ONE_SHIFT(0, 65, u64, 0, true);
-	TEST_ONE_SHIFT(0, 64, s64, 0, true);
-	TEST_ONE_SHIFT(0, 65, s64, 0, true);
-
 	/*
 	 * Corner case: for unsigned types, we fail when we've shifted
 	 * through the entire width of bits. For signed types, we might
@@ -451,9 +511,9 @@ static void overflow_shift_test(struct kunit *test)
 	TEST_ONE_SHIFT(0, 31, s32, 0, false);
 	TEST_ONE_SHIFT(0, 63, s64, 0, false);
 
-	kunit_info(test, "%d shift tests finished\n", count);
-#undef TEST_ONE_SHIFT
+	kunit_info(test, "%d nonsense shift tests finished\n", count);
 }
+#undef TEST_ONE_SHIFT
 
 /*
  * Deal with the various forms of allocator arguments. See comments above
@@ -649,18 +709,25 @@ static void overflow_size_helpers_test(struct kunit *test)
 }
 
 static struct kunit_case overflow_test_cases[] = {
-	KUNIT_CASE(u8_overflow_test),
-	KUNIT_CASE(s8_overflow_test),
-	KUNIT_CASE(u16_overflow_test),
-	KUNIT_CASE(s16_overflow_test),
-	KUNIT_CASE(u32_overflow_test),
-	KUNIT_CASE(s32_overflow_test),
+	KUNIT_CASE(u8_u8__u8_overflow_test),
+	KUNIT_CASE(s8_s8__s8_overflow_test),
+	KUNIT_CASE(u16_u16__u16_overflow_test),
+	KUNIT_CASE(s16_s16__s16_overflow_test),
+	KUNIT_CASE(u32_u32__u32_overflow_test),
+	KUNIT_CASE(s32_s32__s32_overflow_test),
 /* Clang 13 and earlier generate unwanted libcalls on 32-bit. */
 #if BITS_PER_LONG == 64
-	KUNIT_CASE(u64_overflow_test),
-	KUNIT_CASE(s64_overflow_test),
+	KUNIT_CASE(u64_u64__u64_overflow_test),
+	KUNIT_CASE(s64_s64__s64_overflow_test),
 #endif
-	KUNIT_CASE(overflow_shift_test),
+	KUNIT_CASE(u32_u32__u8_overflow_test),
+	KUNIT_CASE(u32_u32__int_overflow_test),
+	KUNIT_CASE(u8_u8__int_overflow_test),
+	KUNIT_CASE(int_int__u8_overflow_test),
+	KUNIT_CASE(shift_sane_test),
+	KUNIT_CASE(shift_overflow_test),
+	KUNIT_CASE(shift_truncate_test),
+	KUNIT_CASE(shift_nonsense_test),
 	KUNIT_CASE(overflow_allocation_test),
 	KUNIT_CASE(overflow_size_helpers_test),
 	{}
diff --git a/lib/stackinit_kunit.c b/lib/stackinit_kunit.c
index 35c69aa425b2..4591d6cf5e01 100644
--- a/lib/stackinit_kunit.c
+++ b/lib/stackinit_kunit.c
@@ -3,7 +3,7 @@
  * Test cases for compiler-based stack variable zeroing via
  * -ftrivial-auto-var-init={zero,pattern} or CONFIG_GCC_PLUGIN_STRUCTLEAK*.
  * For example, see:
- * https://www.kernel.org/doc/html/latest/dev-tools/kunit/kunit-tool.html#configuring-building-and-running-tests
+ * "Running tests with kunit_tool" at Documentation/dev-tools/kunit/start.rst
  *	./tools/testing/kunit/kunit.py run stackinit [--raw_output] \
  *		--make_option LLVM=1 \
  *		--kconfig_add CONFIG_INIT_STACK_ALL_ZERO=y
diff --git a/lib/string_helpers.c b/lib/string_helpers.c
index 5ed3beb066e6..230020a2e076 100644
--- a/lib/string_helpers.c
+++ b/lib/string_helpers.c
@@ -131,6 +131,50 @@ void string_get_size(u64 size, u64 blk_size, const enum string_size_units units,
 }
 EXPORT_SYMBOL(string_get_size);
 
+/**
+ * parse_int_array_user - Split string into a sequence of integers
+ * @from:	The user space buffer to read from
+ * @count:	The maximum number of bytes to read
+ * @array:	Returned pointer to sequence of integers
+ *
+ * On success @array is allocated and initialized with a sequence of
+ * integers extracted from the @from plus an additional element that
+ * begins the sequence and specifies the integers count.
+ *
+ * Caller takes responsibility for freeing @array when it is no longer
+ * needed.
+ */
+int parse_int_array_user(const char __user *from, size_t count, int **array)
+{
+	int *ints, nints;
+	char *buf;
+	int ret = 0;
+
+	buf = memdup_user_nul(from, count);
+	if (IS_ERR(buf))
+		return PTR_ERR(buf);
+
+	get_options(buf, 0, &nints);
+	if (!nints) {
+		ret = -ENOENT;
+		goto free_buf;
+	}
+
+	ints = kcalloc(nints + 1, sizeof(*ints), GFP_KERNEL);
+	if (!ints) {
+		ret = -ENOMEM;
+		goto free_buf;
+	}
+
+	get_options(buf, nints + 1, ints);
+	*array = ints;
+
+free_buf:
+	kfree(buf);
+	return ret;
+}
+EXPORT_SYMBOL(parse_int_array_user);
+
 static bool unescape_space(char **src, char **dst)
 {
 	char *p = *dst, *q = *src;
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index 3c1853a9d1c0..c414a8d9f1ea 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -2246,6 +2246,9 @@ int __init no_hash_pointers_enable(char *str)
 }
 early_param("no_hash_pointers", no_hash_pointers_enable);
 
+/* Used for Rust formatting ('%pA'). */
+char *rust_fmt_argument(char *buf, char *end, void *ptr);
+
 /*
  * Show a '%p' thing.  A kernel extension is that the '%p' is followed
  * by an extra set of alphanumeric characters that are extended format
@@ -2372,6 +2375,10 @@ early_param("no_hash_pointers", no_hash_pointers_enable);
  *
  * Note: The default behaviour (unadorned %p) is to hash the address,
  * rendering it useful as a unique identifier.
+ *
+ * There is also a '%pA' format specifier, but it is only intended to be used
+ * from Rust code to format core::fmt::Arguments. Do *not* use it from C.
+ * See rust/kernel/print.rs for details.
  */
 static noinline_for_stack
 char *pointer(const char *fmt, char *buf, char *end, void *ptr,
@@ -2444,6 +2451,12 @@ char *pointer(const char *fmt, char *buf, char *end, void *ptr,
 		return device_node_string(buf, end, ptr, spec, fmt + 1);
 	case 'f':
 		return fwnode_string(buf, end, ptr, spec, fmt + 1);
+	case 'A':
+		if (!IS_ENABLED(CONFIG_RUST)) {
+			WARN_ONCE(1, "Please remove %%pA from non-Rust code\n");
+			return error_string(buf, end, "(%pA?)", spec);
+		}
+		return rust_fmt_argument(buf, end, ptr);
 	case 'x':
 		return pointer_string(buf, end, ptr, spec);
 	case 'e':
diff --git a/mm/damon/dbgfs.c b/mm/damon/dbgfs.c
index cfdf63132d5a..4e51466c4e74 100644
--- a/mm/damon/dbgfs.c
+++ b/mm/damon/dbgfs.c
@@ -884,6 +884,7 @@ static int dbgfs_rm_context(char *name)
 	struct dentry *root, *dir, **new_dirs;
 	struct damon_ctx **new_ctxs;
 	int i, j;
+	int ret = 0;
 
 	if (damon_nr_running_ctxs())
 		return -EBUSY;
@@ -898,14 +899,16 @@ static int dbgfs_rm_context(char *name)
 
 	new_dirs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_dirs),
 			GFP_KERNEL);
-	if (!new_dirs)
-		return -ENOMEM;
+	if (!new_dirs) {
+		ret = -ENOMEM;
+		goto out_dput;
+	}
 
 	new_ctxs = kmalloc_array(dbgfs_nr_ctxs - 1, sizeof(*dbgfs_ctxs),
 			GFP_KERNEL);
 	if (!new_ctxs) {
-		kfree(new_dirs);
-		return -ENOMEM;
+		ret = -ENOMEM;
+		goto out_new_dirs;
 	}
 
 	for (i = 0, j = 0; i < dbgfs_nr_ctxs; i++) {
@@ -925,7 +928,13 @@ static int dbgfs_rm_context(char *name)
 	dbgfs_ctxs = new_ctxs;
 	dbgfs_nr_ctxs--;
 
-	return 0;
+	goto out_dput;
+
+out_new_dirs:
+	kfree(new_dirs);
+out_dput:
+	dput(dir);
+	return ret;
 }
 
 static ssize_t dbgfs_rm_context_write(struct file *file,
diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c
index 7488e27c87c3..bdef9682d0a0 100644
--- a/mm/damon/sysfs.c
+++ b/mm/damon/sysfs.c
@@ -2182,12 +2182,12 @@ static int damon_sysfs_add_target(struct damon_sysfs_target *sys_target,
 
 	if (!t)
 		return -ENOMEM;
+	damon_add_target(ctx, t);
 	if (damon_target_has_pid(ctx)) {
 		t->pid = find_get_pid(sys_target->pid);
 		if (!t->pid)
 			goto destroy_targets_out;
 	}
-	damon_add_target(ctx, t);
 	err = damon_sysfs_set_regions(t, sys_target->regions);
 	if (err)
 		goto destroy_targets_out;
diff --git a/mm/frontswap.c b/mm/frontswap.c
index 1a97610308cb..279e55b4ed87 100644
--- a/mm/frontswap.c
+++ b/mm/frontswap.c
@@ -125,6 +125,9 @@ void frontswap_init(unsigned type, unsigned long *map)
 	 * p->frontswap set to something valid to work properly.
 	 */
 	frontswap_map_set(sis, map);
+
+	if (!frontswap_enabled())
+		return;
 	frontswap_ops->init(type);
 }
 
diff --git a/mm/gup.c b/mm/gup.c
index 5abdaf487460..00926abb4426 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -2345,8 +2345,28 @@ static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
 }
 
 #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
-static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
-			 unsigned int flags, struct page **pages, int *nr)
+/*
+ * Fast-gup relies on pte change detection to avoid concurrent pgtable
+ * operations.
+ *
+ * To pin the page, fast-gup needs to do below in order:
+ * (1) pin the page (by prefetching pte), then (2) check pte not changed.
+ *
+ * For the rest of pgtable operations where pgtable updates can be racy
+ * with fast-gup, we need to do (1) clear pte, then (2) check whether page
+ * is pinned.
+ *
+ * Above will work for all pte-level operations, including THP split.
+ *
+ * For THP collapse, it's a bit more complicated because fast-gup may be
+ * walking a pgtable page that is being freed (pte is still valid but pmd
+ * can be cleared already).  To avoid race in such condition, we need to
+ * also check pmd here to make sure pmd doesn't change (corresponds to
+ * pmdp_collapse_flush() in the THP collapse code path).
+ */
+static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
+			 unsigned long end, unsigned int flags,
+			 struct page **pages, int *nr)
 {
 	struct dev_pagemap *pgmap = NULL;
 	int nr_start = *nr, ret = 0;
@@ -2392,7 +2412,8 @@ static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
 			goto pte_unmap;
 		}
 
-		if (unlikely(pte_val(pte) != pte_val(*ptep))) {
+		if (unlikely(pmd_val(pmd) != pmd_val(*pmdp)) ||
+		    unlikely(pte_val(pte) != pte_val(*ptep))) {
 			gup_put_folio(folio, 1, flags);
 			goto pte_unmap;
 		}
@@ -2439,8 +2460,9 @@ pte_unmap:
  * get_user_pages_fast_only implementation that can pin pages. Thus it's still
  * useful to have gup_huge_pmd even if we can't operate on ptes.
  */
-static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
-			 unsigned int flags, struct page **pages, int *nr)
+static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
+			 unsigned long end, unsigned int flags,
+			 struct page **pages, int *nr)
 {
 	return 0;
 }
@@ -2764,7 +2786,7 @@ static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned lo
 			if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
 					 PMD_SHIFT, next, flags, pages, nr))
 				return 0;
-		} else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
+		} else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr))
 			return 0;
 	} while (pmdp++, addr = next, addr != end);
 
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index e9414ee57c5b..f42bb51e023a 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2894,11 +2894,9 @@ static void split_huge_pages_all(void)
 		max_zone_pfn = zone_end_pfn(zone);
 		for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
 			int nr_pages;
-			if (!pfn_valid(pfn))
-				continue;
 
-			page = pfn_to_page(pfn);
-			if (!get_page_unless_zero(page))
+			page = pfn_to_online_page(pfn);
+			if (!page || !get_page_unless_zero(page))
 				continue;
 
 			if (zone != page_zone(page))
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index e070b8593b37..0bdfc7e1c933 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3420,6 +3420,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
 {
 	int i, nid = page_to_nid(page);
 	struct hstate *target_hstate;
+	struct page *subpage;
 	int rc = 0;
 
 	target_hstate = size_to_hstate(PAGE_SIZE << h->demote_order);
@@ -3453,15 +3454,16 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
 	mutex_lock(&target_hstate->resize_lock);
 	for (i = 0; i < pages_per_huge_page(h);
 				i += pages_per_huge_page(target_hstate)) {
+		subpage = nth_page(page, i);
 		if (hstate_is_gigantic(target_hstate))
-			prep_compound_gigantic_page_for_demote(page + i,
+			prep_compound_gigantic_page_for_demote(subpage,
 							target_hstate->order);
 		else
-			prep_compound_page(page + i, target_hstate->order);
-		set_page_private(page + i, 0);
-		set_page_refcounted(page + i);
-		prep_new_huge_page(target_hstate, page + i, nid);
-		put_page(page + i);
+			prep_compound_page(subpage, target_hstate->order);
+		set_page_private(subpage, 0);
+		set_page_refcounted(subpage);
+		prep_new_huge_page(target_hstate, subpage, nid);
+		put_page(subpage);
 	}
 	mutex_unlock(&target_hstate->resize_lock);
 
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 01f71786d530..70b7ac66411c 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1083,10 +1083,12 @@ static void collapse_huge_page(struct mm_struct *mm,
 
 	pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
 	/*
-	 * After this gup_fast can't run anymore. This also removes
-	 * any huge TLB entry from the CPU so we won't allow
-	 * huge and small TLB entries for the same virtual address
-	 * to avoid the risk of CPU bugs in that area.
+	 * This removes any huge TLB entry from the CPU so we won't allow
+	 * huge and small TLB entries for the same virtual address to
+	 * avoid the risk of CPU bugs in that area.
+	 *
+	 * Parallel fast GUP is fine since fast GUP will back off when
+	 * it detects PMD is changed.
 	 */
 	_pmd = pmdp_collapse_flush(vma, address, pmd);
 	spin_unlock(pmd_ptl);
diff --git a/mm/madvise.c b/mm/madvise.c
index 5f0f0948a50e..9ff51650f4f0 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -451,8 +451,11 @@ regular_page:
 			continue;
 		}
 
-		/* Do not interfere with other mappings of this page */
-		if (page_mapcount(page) != 1)
+		/*
+		 * Do not interfere with other mappings of this page and
+		 * non-LRU page.
+		 */
+		if (!PageLRU(page) || page_mapcount(page) != 1)
 			continue;
 
 		VM_BUG_ON_PAGE(PageTransCompound(page), page);
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index 14439806b5ef..e7ac570dda75 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -345,13 +345,17 @@ static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma,
  * not much we can do.	We just print a message and ignore otherwise.
  */
 
+#define FSDAX_INVALID_PGOFF ULONG_MAX
+
 /*
  * Schedule a process for later kill.
  * Uses GFP_ATOMIC allocations to avoid potential recursions in the VM.
  *
- * Notice: @fsdax_pgoff is used only when @p is a fsdax page.
- *   In other cases, such as anonymous and file-backend page, the address to be
- *   killed can be caculated by @p itself.
+ * Note: @fsdax_pgoff is used only when @p is a fsdax page and a
+ * filesystem with a memory failure handler has claimed the
+ * memory_failure event. In all other cases, page->index and
+ * page->mapping are sufficient for mapping the page back to its
+ * corresponding user virtual address.
  */
 static void add_to_kill(struct task_struct *tsk, struct page *p,
 			pgoff_t fsdax_pgoff, struct vm_area_struct *vma,
@@ -367,11 +371,7 @@ static void add_to_kill(struct task_struct *tsk, struct page *p,
 
 	tk->addr = page_address_in_vma(p, vma);
 	if (is_zone_device_page(p)) {
-		/*
-		 * Since page->mapping is not used for fsdax, we need
-		 * calculate the address based on the vma.
-		 */
-		if (p->pgmap->type == MEMORY_DEVICE_FS_DAX)
+		if (fsdax_pgoff != FSDAX_INVALID_PGOFF)
 			tk->addr = vma_pgoff_address(fsdax_pgoff, 1, vma);
 		tk->size_shift = dev_pagemap_mapping_shift(vma, tk->addr);
 	} else
@@ -523,7 +523,8 @@ static void collect_procs_anon(struct page *page, struct list_head *to_kill,
 			if (!page_mapped_in_vma(page, vma))
 				continue;
 			if (vma->vm_mm == t->mm)
-				add_to_kill(t, page, 0, vma, to_kill);
+				add_to_kill(t, page, FSDAX_INVALID_PGOFF, vma,
+					    to_kill);
 		}
 	}
 	read_unlock(&tasklist_lock);
@@ -559,7 +560,8 @@ static void collect_procs_file(struct page *page, struct list_head *to_kill,
 			 * to be informed of all such data corruptions.
 			 */
 			if (vma->vm_mm == t->mm)
-				add_to_kill(t, page, 0, vma, to_kill);
+				add_to_kill(t, page, FSDAX_INVALID_PGOFF, vma,
+					    to_kill);
 		}
 	}
 	read_unlock(&tasklist_lock);
@@ -743,6 +745,9 @@ static int kill_accessing_process(struct task_struct *p, unsigned long pfn,
 	};
 	priv.tk.tsk = p;
 
+	if (!p->mm)
+		return -EFAULT;
+
 	mmap_read_lock(p->mm);
 	ret = walk_page_range(p->mm, 0, TASK_SIZE, &hwp_walk_ops,
 			      (void *)&priv);
@@ -1928,7 +1933,7 @@ static int memory_failure_dev_pagemap(unsigned long pfn, int flags,
 	 * Call driver's implementation to handle the memory failure, otherwise
 	 * fall back to generic handler.
 	 */
-	if (pgmap->ops->memory_failure) {
+	if (pgmap_has_memory_failure(pgmap)) {
 		rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags);
 		/*
 		 * Fall back to generic handler too if operation is not
diff --git a/mm/memory.c b/mm/memory.c
index 4ba73f5aa8bb..a78814413ac0 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4386,14 +4386,20 @@ vm_fault_t finish_fault(struct vm_fault *vmf)
 
 	vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
 				      vmf->address, &vmf->ptl);
-	ret = 0;
+
 	/* Re-check under ptl */
-	if (likely(!vmf_pte_changed(vmf)))
+	if (likely(!vmf_pte_changed(vmf))) {
 		do_set_pte(vmf, page, vmf->address);
-	else
+
+		/* no need to invalidate: a not-present page won't be cached */
+		update_mmu_cache(vma, vmf->address, vmf->pte);
+
+		ret = 0;
+	} else {
+		update_mmu_tlb(vma, vmf->address, vmf->pte);
 		ret = VM_FAULT_NOPAGE;
+	}
 
-	update_mmu_tlb(vma, vmf->address, vmf->pte);
 	pte_unmap_unlock(vmf->pte, vmf->ptl);
 	return ret;
 }
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 27fb37d65476..dbf6c7a7a7c9 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -7,6 +7,7 @@
 #include <linux/export.h>
 #include <linux/memremap.h>
 #include <linux/migrate.h>
+#include <linux/mm.h>
 #include <linux/mm_inline.h>
 #include <linux/mmu_notifier.h>
 #include <linux/oom.h>
@@ -193,10 +194,10 @@ again:
 			bool anon_exclusive;
 			pte_t swp_pte;
 
+			flush_cache_page(vma, addr, pte_pfn(*ptep));
 			anon_exclusive = PageAnon(page) && PageAnonExclusive(page);
 			if (anon_exclusive) {
-				flush_cache_page(vma, addr, pte_pfn(*ptep));
-				ptep_clear_flush(vma, addr, ptep);
+				pte = ptep_clear_flush(vma, addr, ptep);
 
 				if (page_try_share_anon_rmap(page)) {
 					set_pte_at(mm, addr, ptep, pte);
@@ -206,11 +207,15 @@ again:
 					goto next;
 				}
 			} else {
-				ptep_get_and_clear(mm, addr, ptep);
+				pte = ptep_get_and_clear(mm, addr, ptep);
 			}
 
 			migrate->cpages++;
 
+			/* Set the dirty flag on the folio now the pte is gone. */
+			if (pte_dirty(pte))
+				folio_mark_dirty(page_folio(page));
+
 			/* Setup special migration page table entry */
 			if (mpfn & MIGRATE_PFN_WRITE)
 				entry = make_writable_migration_entry(
@@ -254,13 +259,14 @@ next:
 		migrate->dst[migrate->npages] = 0;
 		migrate->src[migrate->npages++] = mpfn;
 	}
-	arch_leave_lazy_mmu_mode();
-	pte_unmap_unlock(ptep - 1, ptl);
 
 	/* Only flush the TLB if we actually modified any entries */
 	if (unmapped)
 		flush_tlb_range(walk->vma, start, end);
 
+	arch_leave_lazy_mmu_mode();
+	pte_unmap_unlock(ptep - 1, ptl);
+
 	return 0;
 }
 
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index e5486d47406e..d04211f0ef0b 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -4708,6 +4708,30 @@ void fs_reclaim_release(gfp_t gfp_mask)
 EXPORT_SYMBOL_GPL(fs_reclaim_release);
 #endif
 
+/*
+ * Zonelists may change due to hotplug during allocation. Detect when zonelists
+ * have been rebuilt so allocation retries. Reader side does not lock and
+ * retries the allocation if zonelist changes. Writer side is protected by the
+ * embedded spin_lock.
+ */
+static DEFINE_SEQLOCK(zonelist_update_seq);
+
+static unsigned int zonelist_iter_begin(void)
+{
+	if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE))
+		return read_seqbegin(&zonelist_update_seq);
+
+	return 0;
+}
+
+static unsigned int check_retry_zonelist(unsigned int seq)
+{
+	if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE))
+		return read_seqretry(&zonelist_update_seq, seq);
+
+	return seq;
+}
+
 /* Perform direct synchronous page reclaim */
 static unsigned long
 __perform_reclaim(gfp_t gfp_mask, unsigned int order,
@@ -5001,6 +5025,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 	int compaction_retries;
 	int no_progress_loops;
 	unsigned int cpuset_mems_cookie;
+	unsigned int zonelist_iter_cookie;
 	int reserve_flags;
 
 	/*
@@ -5011,11 +5036,12 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 				(__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
 		gfp_mask &= ~__GFP_ATOMIC;
 
-retry_cpuset:
+restart:
 	compaction_retries = 0;
 	no_progress_loops = 0;
 	compact_priority = DEF_COMPACT_PRIORITY;
 	cpuset_mems_cookie = read_mems_allowed_begin();
+	zonelist_iter_cookie = zonelist_iter_begin();
 
 	/*
 	 * The fast path uses conservative alloc_flags to succeed only until
@@ -5187,9 +5213,13 @@ retry:
 		goto retry;
 
 
-	/* Deal with possible cpuset update races before we start OOM killing */
-	if (check_retry_cpuset(cpuset_mems_cookie, ac))
-		goto retry_cpuset;
+	/*
+	 * Deal with possible cpuset update races or zonelist updates to avoid
+	 * a unnecessary OOM kill.
+	 */
+	if (check_retry_cpuset(cpuset_mems_cookie, ac) ||
+	    check_retry_zonelist(zonelist_iter_cookie))
+		goto restart;
 
 	/* Reclaim has failed us, start killing things */
 	page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
@@ -5209,9 +5239,13 @@ retry:
 	}
 
 nopage:
-	/* Deal with possible cpuset update races before we fail */
-	if (check_retry_cpuset(cpuset_mems_cookie, ac))
-		goto retry_cpuset;
+	/*
+	 * Deal with possible cpuset update races or zonelist updates to avoid
+	 * a unnecessary OOM kill.
+	 */
+	if (check_retry_cpuset(cpuset_mems_cookie, ac) ||
+	    check_retry_zonelist(zonelist_iter_cookie))
+		goto restart;
 
 	/*
 	 * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
@@ -5706,6 +5740,18 @@ refill:
 		/* reset page count bias and offset to start of new frag */
 		nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
 		offset = size - fragsz;
+		if (unlikely(offset < 0)) {
+			/*
+			 * The caller is trying to allocate a fragment
+			 * with fragsz > PAGE_SIZE but the cache isn't big
+			 * enough to satisfy the request, this may
+			 * happen in low memory conditions.
+			 * We don't release the cache page because
+			 * it could make memory pressure worse
+			 * so we simply return NULL here.
+			 */
+			return NULL;
+		}
 	}
 
 	nc->pagecnt_bias--;
@@ -6514,9 +6560,8 @@ static void __build_all_zonelists(void *data)
 	int nid;
 	int __maybe_unused cpu;
 	pg_data_t *self = data;
-	static DEFINE_SPINLOCK(lock);
 
-	spin_lock(&lock);
+	write_seqlock(&zonelist_update_seq);
 
 #ifdef CONFIG_NUMA
 	memset(node_load, 0, sizeof(node_load));
@@ -6553,7 +6598,7 @@ static void __build_all_zonelists(void *data)
 #endif
 	}
 
-	spin_unlock(&lock);
+	write_sequnlock(&zonelist_update_seq);
 }
 
 static noinline void __init
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index 9d73dc38e3d7..eb3a68ca92ad 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -288,6 +288,7 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages)
  * @isolate_before:	isolate the pageblock before the boundary_pfn
  * @skip_isolation:	the flag to skip the pageblock isolation in second
  *			isolate_single_pageblock()
+ * @migratetype:	migrate type to set in error recovery.
  *
  * Free and in-use pages can be as big as MAX_ORDER-1 and contain more than one
  * pageblock. When not all pageblocks within a page are isolated at the same
@@ -302,9 +303,9 @@ __first_valid_page(unsigned long pfn, unsigned long nr_pages)
  * the in-use page then splitting the free page.
  */
 static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,
-			gfp_t gfp_flags, bool isolate_before, bool skip_isolation)
+			gfp_t gfp_flags, bool isolate_before, bool skip_isolation,
+			int migratetype)
 {
-	unsigned char saved_mt;
 	unsigned long start_pfn;
 	unsigned long isolate_pageblock;
 	unsigned long pfn;
@@ -328,13 +329,13 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,
 	start_pfn  = max(ALIGN_DOWN(isolate_pageblock, MAX_ORDER_NR_PAGES),
 				      zone->zone_start_pfn);
 
-	saved_mt = get_pageblock_migratetype(pfn_to_page(isolate_pageblock));
+	if (skip_isolation) {
+		int mt = get_pageblock_migratetype(pfn_to_page(isolate_pageblock));
 
-	if (skip_isolation)
-		VM_BUG_ON(!is_migrate_isolate(saved_mt));
-	else {
-		ret = set_migratetype_isolate(pfn_to_page(isolate_pageblock), saved_mt, flags,
-				isolate_pageblock, isolate_pageblock + pageblock_nr_pages);
+		VM_BUG_ON(!is_migrate_isolate(mt));
+	} else {
+		ret = set_migratetype_isolate(pfn_to_page(isolate_pageblock), migratetype,
+				flags, isolate_pageblock, isolate_pageblock + pageblock_nr_pages);
 
 		if (ret)
 			return ret;
@@ -475,7 +476,7 @@ static int isolate_single_pageblock(unsigned long boundary_pfn, int flags,
 failed:
 	/* restore the original migratetype */
 	if (!skip_isolation)
-		unset_migratetype_isolate(pfn_to_page(isolate_pageblock), saved_mt);
+		unset_migratetype_isolate(pfn_to_page(isolate_pageblock), migratetype);
 	return -EBUSY;
 }
 
@@ -537,7 +538,8 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
 	bool skip_isolation = false;
 
 	/* isolate [isolate_start, isolate_start + pageblock_nr_pages) pageblock */
-	ret = isolate_single_pageblock(isolate_start, flags, gfp_flags, false, skip_isolation);
+	ret = isolate_single_pageblock(isolate_start, flags, gfp_flags, false,
+			skip_isolation, migratetype);
 	if (ret)
 		return ret;
 
@@ -545,7 +547,8 @@ int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
 		skip_isolation = true;
 
 	/* isolate [isolate_end - pageblock_nr_pages, isolate_end) pageblock */
-	ret = isolate_single_pageblock(isolate_end, flags, gfp_flags, true, skip_isolation);
+	ret = isolate_single_pageblock(isolate_end, flags, gfp_flags, true,
+			skip_isolation, migratetype);
 	if (ret) {
 		unset_migratetype_isolate(pfn_to_page(isolate_start), migratetype);
 		return ret;
diff --git a/mm/secretmem.c b/mm/secretmem.c
index e3e9590c6fb3..3f7154099795 100644
--- a/mm/secretmem.c
+++ b/mm/secretmem.c
@@ -285,7 +285,7 @@ static int secretmem_init(void)
 
 	secretmem_mnt = kern_mount(&secretmem_fs);
 	if (IS_ERR(secretmem_mnt))
-		ret = PTR_ERR(secretmem_mnt);
+		return PTR_ERR(secretmem_mnt);
 
 	/* prevent secretmem mappings from ever getting PROT_EXEC */
 	secretmem_mnt->mnt_flags |= MNT_NOEXEC;
diff --git a/mm/swap_state.c b/mm/swap_state.c
index e166051566f4..41afa6d45b23 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -151,7 +151,7 @@ void __delete_from_swap_cache(struct folio *folio,
 
 	for (i = 0; i < nr; i++) {
 		void *entry = xas_store(&xas, shadow);
-		VM_BUG_ON_FOLIO(entry != folio, folio);
+		VM_BUG_ON_PAGE(entry != folio, entry);
 		set_page_private(folio_page(folio, i), 0);
 		xas_next(&xas);
 	}
diff --git a/mm/util.c b/mm/util.c
index c9439c66d8cf..346e40177bc6 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -619,6 +619,10 @@ void *kvmalloc_node(size_t size, gfp_t flags, int node)
 	if (ret || size <= PAGE_SIZE)
 		return ret;
 
+	/* non-sleeping allocations are not supported by vmalloc */
+	if (!gfpflags_allow_blocking(flags))
+		return NULL;
+
 	/* Don't even allow crazy sizes */
 	if (unlikely(size > INT_MAX)) {
 		WARN_ON_ONCE(!(flags & __GFP_NOWARN));
diff --git a/mm/vmscan.c b/mm/vmscan.c
index b2b1431352dc..382dbe97329f 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2550,8 +2550,8 @@ static void shrink_active_list(unsigned long nr_to_scan,
 		}
 
 		if (unlikely(buffer_heads_over_limit)) {
-			if (folio_get_private(folio) && folio_trylock(folio)) {
-				if (folio_get_private(folio))
+			if (folio_test_private(folio) && folio_trylock(folio)) {
+				if (folio_test_private(folio))
 					filemap_release_folio(folio, 0);
 				folio_unlock(folio);
 			}
diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c
index 5aa8144101dc..0beb44f2fe1f 100644
--- a/net/8021q/vlan_core.c
+++ b/net/8021q/vlan_core.c
@@ -467,12 +467,9 @@ static struct sk_buff *vlan_gro_receive(struct list_head *head,
 
 	off_vlan = skb_gro_offset(skb);
 	hlen = off_vlan + sizeof(*vhdr);
-	vhdr = skb_gro_header_fast(skb, off_vlan);
-	if (skb_gro_header_hard(skb, hlen)) {
-		vhdr = skb_gro_header_slow(skb, hlen, off_vlan);
-		if (unlikely(!vhdr))
-			goto out;
-	}
+	vhdr = skb_gro_header(skb, hlen, off_vlan);
+	if (unlikely(!vhdr))
+		goto out;
 
 	type = vhdr->h_vlan_encapsulated_proto;
 
diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c
index 035812b0461c..e1bb41a443c4 100644
--- a/net/8021q/vlan_dev.c
+++ b/net/8021q/vlan_dev.c
@@ -674,9 +674,9 @@ static int vlan_ethtool_get_link_ksettings(struct net_device *dev,
 static void vlan_ethtool_get_drvinfo(struct net_device *dev,
 				     struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, vlan_fullname, sizeof(info->driver));
-	strlcpy(info->version, vlan_version, sizeof(info->version));
-	strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
+	strscpy(info->driver, vlan_fullname, sizeof(info->driver));
+	strscpy(info->version, vlan_version, sizeof(info->version));
+	strscpy(info->fw_version, "N/A", sizeof(info->fw_version));
 }
 
 static int vlan_ethtool_get_ts_info(struct net_device *dev,
diff --git a/net/Kconfig b/net/Kconfig
index 6b78f695caa6..48c33c222199 100644
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -204,7 +204,6 @@ config BRIDGE_NETFILTER
 source "net/netfilter/Kconfig"
 source "net/ipv4/netfilter/Kconfig"
 source "net/ipv6/netfilter/Kconfig"
-source "net/decnet/netfilter/Kconfig"
 source "net/bridge/netfilter/Kconfig"
 
 endif
@@ -221,7 +220,6 @@ source "net/802/Kconfig"
 source "net/bridge/Kconfig"
 source "net/dsa/Kconfig"
 source "net/8021q/Kconfig"
-source "net/decnet/Kconfig"
 source "net/llc/Kconfig"
 source "drivers/net/appletalk/Kconfig"
 source "net/x25/Kconfig"
diff --git a/net/Kconfig.debug b/net/Kconfig.debug
index e6ae11cc2fb7..5e3fffe707dd 100644
--- a/net/Kconfig.debug
+++ b/net/Kconfig.debug
@@ -2,7 +2,7 @@
 
 config NET_DEV_REFCNT_TRACKER
 	bool "Enable net device refcount tracking"
-	depends on DEBUG_KERNEL && STACKTRACE_SUPPORT
+	depends on DEBUG_KERNEL && STACKTRACE_SUPPORT && NET
 	select REF_TRACKER
 	default n
 	help
@@ -11,7 +11,7 @@ config NET_DEV_REFCNT_TRACKER
 
 config NET_NS_REFCNT_TRACKER
 	bool "Enable networking namespace refcount tracking"
-	depends on DEBUG_KERNEL && STACKTRACE_SUPPORT
+	depends on DEBUG_KERNEL && STACKTRACE_SUPPORT && NET
 	select REF_TRACKER
 	default n
 	help
diff --git a/net/Makefile b/net/Makefile
index fbfeb8a0bb37..6a62e5b27378 100644
--- a/net/Makefile
+++ b/net/Makefile
@@ -38,7 +38,6 @@ obj-$(CONFIG_AF_KCM)		+= kcm/
 obj-$(CONFIG_STREAM_PARSER)	+= strparser/
 obj-$(CONFIG_ATM)		+= atm/
 obj-$(CONFIG_L2TP)		+= l2tp/
-obj-$(CONFIG_DECNET)		+= decnet/
 obj-$(CONFIG_PHONET)		+= phonet/
 ifneq ($(CONFIG_VLAN_8021Q),)
 obj-y				+= 8021q/
diff --git a/net/ax25/af_ax25.c b/net/ax25/af_ax25.c
index d82a51e69386..6b4c25a92377 100644
--- a/net/ax25/af_ax25.c
+++ b/net/ax25/af_ax25.c
@@ -778,7 +778,7 @@ static int ax25_getsockopt(struct socket *sock, int level, int optname,
 		ax25_dev = ax25->ax25_dev;
 
 		if (ax25_dev != NULL && ax25_dev->dev != NULL) {
-			strlcpy(devname, ax25_dev->dev->name, sizeof(devname));
+			strscpy(devname, ax25_dev->dev->name, sizeof(devname));
 			length = strlen(devname) + 1;
 		} else {
 			*devname = '\0';
diff --git a/net/batman-adv/bat_v_elp.c b/net/batman-adv/bat_v_elp.c
index b6db999abf75..f1741fbfb617 100644
--- a/net/batman-adv/bat_v_elp.c
+++ b/net/batman-adv/bat_v_elp.c
@@ -125,7 +125,6 @@ static u32 batadv_v_elp_get_throughput(struct batadv_hardif_neigh_node *neigh)
 	/* if not a wifi interface, check if this device provides data via
 	 * ethtool (e.g. an Ethernet adapter)
 	 */
-	memset(&link_settings, 0, sizeof(link_settings));
 	rtnl_lock();
 	ret = __ethtool_get_link_ksettings(hard_iface->net_dev, &link_settings);
 	rtnl_unlock();
diff --git a/net/batman-adv/main.h b/net/batman-adv/main.h
index 23f3d53f4b51..c48803b32bb0 100644
--- a/net/batman-adv/main.h
+++ b/net/batman-adv/main.h
@@ -13,7 +13,7 @@
 #define BATADV_DRIVER_DEVICE "batman-adv"
 
 #ifndef BATADV_SOURCE_VERSION
-#define BATADV_SOURCE_VERSION "2022.2"
+#define BATADV_SOURCE_VERSION "2022.3"
 #endif
 
 /* B.A.T.M.A.N. parameters */
diff --git a/net/batman-adv/netlink.c b/net/batman-adv/netlink.c
index 00875e1d8c44..a5e4a4e976cf 100644
--- a/net/batman-adv/netlink.c
+++ b/net/batman-adv/netlink.c
@@ -1493,6 +1493,7 @@ struct genl_family batadv_netlink_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.small_ops = batadv_netlink_ops,
 	.n_small_ops = ARRAY_SIZE(batadv_netlink_ops),
+	.resv_start_op = BATADV_CMD_SET_VLAN + 1,
 	.mcgrps = batadv_netlink_mcgrps,
 	.n_mcgrps = ARRAY_SIZE(batadv_netlink_mcgrps),
 };
diff --git a/net/batman-adv/trace.h b/net/batman-adv/trace.h
index 31c8f922651d..5dd52bc5cabb 100644
--- a/net/batman-adv/trace.h
+++ b/net/batman-adv/trace.h
@@ -9,8 +9,6 @@
 
 #include "main.h"
 
-#include <linux/bug.h>
-#include <linux/kernel.h>
 #include <linux/netdevice.h>
 #include <linux/percpu.h>
 #include <linux/printk.h>
diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h
index 2be5d4a712c5..758cd797a063 100644
--- a/net/batman-adv/types.h
+++ b/net/batman-adv/types.h
@@ -1740,45 +1740,6 @@ struct batadv_priv {
 #endif
 };
 
-/**
- * struct batadv_socket_client - layer2 icmp socket client data
- */
-struct batadv_socket_client {
-	/**
-	 * @queue_list: packet queue for packets destined for this socket client
-	 */
-	struct list_head queue_list;
-
-	/** @queue_len: number of packets in the packet queue (queue_list) */
-	unsigned int queue_len;
-
-	/** @index: socket client's index in the batadv_socket_client_hash */
-	unsigned char index;
-
-	/** @lock: lock protecting queue_list, queue_len & index */
-	spinlock_t lock;
-
-	/** @queue_wait: socket client's wait queue */
-	wait_queue_head_t queue_wait;
-
-	/** @bat_priv: pointer to soft_iface this client belongs to */
-	struct batadv_priv *bat_priv;
-};
-
-/**
- * struct batadv_socket_packet - layer2 icmp packet for socket client
- */
-struct batadv_socket_packet {
-	/** @list: list node for &batadv_socket_client.queue_list */
-	struct list_head list;
-
-	/** @icmp_len: size of the layer2 icmp packet */
-	size_t icmp_len;
-
-	/** @icmp_packet: layer2 icmp packet */
-	u8 icmp_packet[BATADV_ICMP_MAX_PACKET_SIZE];
-};
-
 #ifdef CONFIG_BATMAN_ADV_BLA
 
 /**
diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c
index 9777e7b109ee..7a59c4487050 100644
--- a/net/bluetooth/hci_conn.c
+++ b/net/bluetooth/hci_conn.c
@@ -44,6 +44,11 @@ struct sco_param {
 	u8  retrans_effort;
 };
 
+struct conn_handle_t {
+	struct hci_conn *conn;
+	__u16 handle;
+};
+
 static const struct sco_param esco_param_cvsd[] = {
 	{ EDR_ESCO_MASK & ~ESCO_2EV3, 0x000a,	0x01 }, /* S3 */
 	{ EDR_ESCO_MASK & ~ESCO_2EV3, 0x0007,	0x01 }, /* S2 */
@@ -316,17 +321,60 @@ static bool find_next_esco_param(struct hci_conn *conn,
 	return conn->attempt <= size;
 }
 
-static bool hci_enhanced_setup_sync_conn(struct hci_conn *conn, __u16 handle)
+static int configure_datapath_sync(struct hci_dev *hdev, struct bt_codec *codec)
 {
-	struct hci_dev *hdev = conn->hdev;
+	int err;
+	__u8 vnd_len, *vnd_data = NULL;
+	struct hci_op_configure_data_path *cmd = NULL;
+
+	err = hdev->get_codec_config_data(hdev, ESCO_LINK, codec, &vnd_len,
+					  &vnd_data);
+	if (err < 0)
+		goto error;
+
+	cmd = kzalloc(sizeof(*cmd) + vnd_len, GFP_KERNEL);
+	if (!cmd) {
+		err = -ENOMEM;
+		goto error;
+	}
+
+	err = hdev->get_data_path_id(hdev, &cmd->data_path_id);
+	if (err < 0)
+		goto error;
+
+	cmd->vnd_len = vnd_len;
+	memcpy(cmd->vnd_data, vnd_data, vnd_len);
+
+	cmd->direction = 0x00;
+	__hci_cmd_sync_status(hdev, HCI_CONFIGURE_DATA_PATH,
+			      sizeof(*cmd) + vnd_len, cmd, HCI_CMD_TIMEOUT);
+
+	cmd->direction = 0x01;
+	err = __hci_cmd_sync_status(hdev, HCI_CONFIGURE_DATA_PATH,
+				    sizeof(*cmd) + vnd_len, cmd,
+				    HCI_CMD_TIMEOUT);
+error:
+
+	kfree(cmd);
+	kfree(vnd_data);
+	return err;
+}
+
+static int hci_enhanced_setup_sync(struct hci_dev *hdev, void *data)
+{
+	struct conn_handle_t *conn_handle = data;
+	struct hci_conn *conn = conn_handle->conn;
+	__u16 handle = conn_handle->handle;
 	struct hci_cp_enhanced_setup_sync_conn cp;
 	const struct sco_param *param;
 
+	kfree(conn_handle);
+
 	bt_dev_dbg(hdev, "hcon %p", conn);
 
 	/* for offload use case, codec needs to configured before opening SCO */
 	if (conn->codec.data_path)
-		hci_req_configure_datapath(hdev, &conn->codec);
+		configure_datapath_sync(hdev, &conn->codec);
 
 	conn->state = BT_CONNECT;
 	conn->out = true;
@@ -344,7 +392,7 @@ static bool hci_enhanced_setup_sync_conn(struct hci_conn *conn, __u16 handle)
 	case BT_CODEC_MSBC:
 		if (!find_next_esco_param(conn, esco_param_msbc,
 					  ARRAY_SIZE(esco_param_msbc)))
-			return false;
+			return -EINVAL;
 
 		param = &esco_param_msbc[conn->attempt - 1];
 		cp.tx_coding_format.id = 0x05;
@@ -396,11 +444,11 @@ static bool hci_enhanced_setup_sync_conn(struct hci_conn *conn, __u16 handle)
 		if (lmp_esco_capable(conn->link)) {
 			if (!find_next_esco_param(conn, esco_param_cvsd,
 						  ARRAY_SIZE(esco_param_cvsd)))
-				return false;
+				return -EINVAL;
 			param = &esco_param_cvsd[conn->attempt - 1];
 		} else {
 			if (conn->attempt > ARRAY_SIZE(sco_param_cvsd))
-				return false;
+				return -EINVAL;
 			param = &sco_param_cvsd[conn->attempt - 1];
 		}
 		cp.tx_coding_format.id = 2;
@@ -423,7 +471,7 @@ static bool hci_enhanced_setup_sync_conn(struct hci_conn *conn, __u16 handle)
 		cp.out_transport_unit_size = 16;
 		break;
 	default:
-		return false;
+		return -EINVAL;
 	}
 
 	cp.retrans_effort = param->retrans_effort;
@@ -431,9 +479,9 @@ static bool hci_enhanced_setup_sync_conn(struct hci_conn *conn, __u16 handle)
 	cp.max_latency = __cpu_to_le16(param->max_latency);
 
 	if (hci_send_cmd(hdev, HCI_OP_ENHANCED_SETUP_SYNC_CONN, sizeof(cp), &cp) < 0)
-		return false;
+		return -EIO;
 
-	return true;
+	return 0;
 }
 
 static bool hci_setup_sync_conn(struct hci_conn *conn, __u16 handle)
@@ -490,8 +538,24 @@ static bool hci_setup_sync_conn(struct hci_conn *conn, __u16 handle)
 
 bool hci_setup_sync(struct hci_conn *conn, __u16 handle)
 {
-	if (enhanced_sync_conn_capable(conn->hdev))
-		return hci_enhanced_setup_sync_conn(conn, handle);
+	int result;
+	struct conn_handle_t *conn_handle;
+
+	if (enhanced_sync_conn_capable(conn->hdev)) {
+		conn_handle = kzalloc(sizeof(*conn_handle), GFP_KERNEL);
+
+		if (!conn_handle)
+			return false;
+
+		conn_handle->conn = conn;
+		conn_handle->handle = handle;
+		result = hci_cmd_sync_queue(conn->hdev, hci_enhanced_setup_sync,
+					    conn_handle, NULL);
+		if (result < 0)
+			kfree(conn_handle);
+
+		return result == 0;
+	}
 
 	return hci_setup_sync_conn(conn, handle);
 }
@@ -2696,3 +2760,79 @@ u32 hci_conn_get_phy(struct hci_conn *conn)
 
 	return phys;
 }
+
+int hci_abort_conn(struct hci_conn *conn, u8 reason)
+{
+	int r = 0;
+
+	switch (conn->state) {
+	case BT_CONNECTED:
+	case BT_CONFIG:
+		if (conn->type == AMP_LINK) {
+			struct hci_cp_disconn_phy_link cp;
+
+			cp.phy_handle = HCI_PHY_HANDLE(conn->handle);
+			cp.reason = reason;
+			r = hci_send_cmd(conn->hdev, HCI_OP_DISCONN_PHY_LINK,
+					 sizeof(cp), &cp);
+		} else {
+			struct hci_cp_disconnect dc;
+
+			dc.handle = cpu_to_le16(conn->handle);
+			dc.reason = reason;
+			r = hci_send_cmd(conn->hdev, HCI_OP_DISCONNECT,
+					 sizeof(dc), &dc);
+		}
+
+		conn->state = BT_DISCONN;
+
+		break;
+	case BT_CONNECT:
+		if (conn->type == LE_LINK) {
+			if (test_bit(HCI_CONN_SCANNING, &conn->flags))
+				break;
+			r = hci_send_cmd(conn->hdev,
+					 HCI_OP_LE_CREATE_CONN_CANCEL, 0, NULL);
+		} else if (conn->type == ACL_LINK) {
+			if (conn->hdev->hci_ver < BLUETOOTH_VER_1_2)
+				break;
+			r = hci_send_cmd(conn->hdev,
+					 HCI_OP_CREATE_CONN_CANCEL,
+					 6, &conn->dst);
+		}
+		break;
+	case BT_CONNECT2:
+		if (conn->type == ACL_LINK) {
+			struct hci_cp_reject_conn_req rej;
+
+			bacpy(&rej.bdaddr, &conn->dst);
+			rej.reason = reason;
+
+			r = hci_send_cmd(conn->hdev,
+					 HCI_OP_REJECT_CONN_REQ,
+					 sizeof(rej), &rej);
+		} else if (conn->type == SCO_LINK || conn->type == ESCO_LINK) {
+			struct hci_cp_reject_sync_conn_req rej;
+
+			bacpy(&rej.bdaddr, &conn->dst);
+
+			/* SCO rejection has its own limited set of
+			 * allowed error values (0x0D-0x0F) which isn't
+			 * compatible with most values passed to this
+			 * function. To be safe hard-code one of the
+			 * values that's suitable for SCO.
+			 */
+			rej.reason = HCI_ERROR_REJ_LIMITED_RESOURCES;
+
+			r = hci_send_cmd(conn->hdev,
+					 HCI_OP_REJECT_SYNC_CONN_REQ,
+					 sizeof(rej), &rej);
+		}
+		break;
+	default:
+		conn->state = BT_CLOSED;
+		break;
+	}
+
+	return r;
+}
diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c
index b3a5a3cc9372..0540555b3704 100644
--- a/net/bluetooth/hci_core.c
+++ b/net/bluetooth/hci_core.c
@@ -597,6 +597,15 @@ static int hci_dev_do_reset(struct hci_dev *hdev)
 
 	/* Cancel these to avoid queueing non-chained pending work */
 	hci_dev_set_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE);
+	/* Wait for
+	 *
+	 *    if (!hci_dev_test_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE))
+	 *        queue_delayed_work(&hdev->{cmd,ncmd}_timer)
+	 *
+	 * inside RCU section to see the flag or complete scheduling.
+	 */
+	synchronize_rcu();
+	/* Explicitly cancel works in case scheduled after setting the flag. */
 	cancel_delayed_work(&hdev->cmd_timer);
 	cancel_delayed_work(&hdev->ncmd_timer);
 
@@ -714,7 +723,7 @@ static void hci_update_passive_scan_state(struct hci_dev *hdev, u8 scan)
 		hci_dev_set_flag(hdev, HCI_BREDR_ENABLED);
 
 		if (hci_dev_test_flag(hdev, HCI_LE_ENABLED))
-			hci_req_update_adv_data(hdev, hdev->cur_adv_instance);
+			hci_update_adv_data(hdev, hdev->cur_adv_instance);
 
 		mgmt_new_settings(hdev);
 	}
@@ -1706,7 +1715,8 @@ struct adv_info *hci_add_adv_instance(struct hci_dev *hdev, u8 instance,
 				      u32 flags, u16 adv_data_len, u8 *adv_data,
 				      u16 scan_rsp_len, u8 *scan_rsp_data,
 				      u16 timeout, u16 duration, s8 tx_power,
-				      u32 min_interval, u32 max_interval)
+				      u32 min_interval, u32 max_interval,
+				      u8 mesh_handle)
 {
 	struct adv_info *adv;
 
@@ -1717,7 +1727,7 @@ struct adv_info *hci_add_adv_instance(struct hci_dev *hdev, u8 instance,
 		memset(adv->per_adv_data, 0, sizeof(adv->per_adv_data));
 	} else {
 		if (hdev->adv_instance_cnt >= hdev->le_num_of_adv_sets ||
-		    instance < 1 || instance > hdev->le_num_of_adv_sets)
+		    instance < 1 || instance > hdev->le_num_of_adv_sets + 1)
 			return ERR_PTR(-EOVERFLOW);
 
 		adv = kzalloc(sizeof(*adv), GFP_KERNEL);
@@ -1734,6 +1744,11 @@ struct adv_info *hci_add_adv_instance(struct hci_dev *hdev, u8 instance,
 	adv->min_interval = min_interval;
 	adv->max_interval = max_interval;
 	adv->tx_power = tx_power;
+	/* Defining a mesh_handle changes the timing units to ms,
+	 * rather than seconds, and ties the instance to the requested
+	 * mesh_tx queue.
+	 */
+	adv->mesh = mesh_handle;
 
 	hci_set_adv_instance_data(hdev, instance, adv_data_len, adv_data,
 				  scan_rsp_len, scan_rsp_data);
@@ -1762,7 +1777,7 @@ struct adv_info *hci_add_per_instance(struct hci_dev *hdev, u8 instance,
 
 	adv = hci_add_adv_instance(hdev, instance, flags, 0, NULL, 0, NULL,
 				   0, 0, HCI_ADV_TX_POWER_NO_PREFERENCE,
-				   min_interval, max_interval);
+				   min_interval, max_interval, 0);
 	if (IS_ERR(adv))
 		return adv;
 
@@ -2391,6 +2406,10 @@ static int hci_suspend_notifier(struct notifier_block *nb, unsigned long action,
 		container_of(nb, struct hci_dev, suspend_notifier);
 	int ret = 0;
 
+	/* Userspace has full control of this device. Do nothing. */
+	if (hci_dev_test_flag(hdev, HCI_USER_CHANNEL))
+		return NOTIFY_DONE;
+
 	if (action == PM_SUSPEND_PREPARE)
 		ret = hci_suspend_dev(hdev);
 	else if (action == PM_POST_SUSPEND)
@@ -2486,6 +2505,7 @@ struct hci_dev *hci_alloc_dev_priv(int sizeof_priv)
 	mutex_init(&hdev->lock);
 	mutex_init(&hdev->req_lock);
 
+	INIT_LIST_HEAD(&hdev->mesh_pending);
 	INIT_LIST_HEAD(&hdev->mgmt_pending);
 	INIT_LIST_HEAD(&hdev->reject_list);
 	INIT_LIST_HEAD(&hdev->accept_list);
@@ -3469,15 +3489,27 @@ static inline int __get_blocks(struct hci_dev *hdev, struct sk_buff *skb)
 	return DIV_ROUND_UP(skb->len - HCI_ACL_HDR_SIZE, hdev->block_len);
 }
 
-static void __check_timeout(struct hci_dev *hdev, unsigned int cnt)
+static void __check_timeout(struct hci_dev *hdev, unsigned int cnt, u8 type)
 {
-	if (!hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) {
-		/* ACL tx timeout must be longer than maximum
-		 * link supervision timeout (40.9 seconds) */
-		if (!cnt && time_after(jiffies, hdev->acl_last_tx +
-				       HCI_ACL_TX_TIMEOUT))
-			hci_link_tx_to(hdev, ACL_LINK);
+	unsigned long last_tx;
+
+	if (hci_dev_test_flag(hdev, HCI_UNCONFIGURED))
+		return;
+
+	switch (type) {
+	case LE_LINK:
+		last_tx = hdev->le_last_tx;
+		break;
+	default:
+		last_tx = hdev->acl_last_tx;
+		break;
 	}
+
+	/* tx timeout must be longer than maximum link supervision timeout
+	 * (40.9 seconds)
+	 */
+	if (!cnt && time_after(jiffies, last_tx + HCI_ACL_TX_TIMEOUT))
+		hci_link_tx_to(hdev, type);
 }
 
 /* Schedule SCO */
@@ -3535,7 +3567,7 @@ static void hci_sched_acl_pkt(struct hci_dev *hdev)
 	struct sk_buff *skb;
 	int quote;
 
-	__check_timeout(hdev, cnt);
+	__check_timeout(hdev, cnt, ACL_LINK);
 
 	while (hdev->acl_cnt &&
 	       (chan = hci_chan_sent(hdev, ACL_LINK, &quote))) {
@@ -3578,8 +3610,6 @@ static void hci_sched_acl_blk(struct hci_dev *hdev)
 	int quote;
 	u8 type;
 
-	__check_timeout(hdev, cnt);
-
 	BT_DBG("%s", hdev->name);
 
 	if (hdev->dev_type == HCI_AMP)
@@ -3587,6 +3617,8 @@ static void hci_sched_acl_blk(struct hci_dev *hdev)
 	else
 		type = ACL_LINK;
 
+	__check_timeout(hdev, cnt, type);
+
 	while (hdev->block_cnt > 0 &&
 	       (chan = hci_chan_sent(hdev, type, &quote))) {
 		u32 priority = (skb_peek(&chan->data_q))->priority;
@@ -3660,7 +3692,7 @@ static void hci_sched_le(struct hci_dev *hdev)
 
 	cnt = hdev->le_pkts ? hdev->le_cnt : hdev->acl_cnt;
 
-	__check_timeout(hdev, cnt);
+	__check_timeout(hdev, cnt, LE_LINK);
 
 	tmp = cnt;
 	while (cnt && (chan = hci_chan_sent(hdev, LE_LINK, &quote))) {
@@ -4056,12 +4088,14 @@ static void hci_cmd_work(struct work_struct *work)
 			if (res < 0)
 				__hci_cmd_sync_cancel(hdev, -res);
 
+			rcu_read_lock();
 			if (test_bit(HCI_RESET, &hdev->flags) ||
 			    hci_dev_test_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE))
 				cancel_delayed_work(&hdev->cmd_timer);
 			else
-				schedule_delayed_work(&hdev->cmd_timer,
-						      HCI_CMD_TIMEOUT);
+				queue_delayed_work(hdev->workqueue, &hdev->cmd_timer,
+						   HCI_CMD_TIMEOUT);
+			rcu_read_unlock();
 		} else {
 			skb_queue_head(&hdev->cmd_q, skb);
 			queue_work(hdev->workqueue, &hdev->cmd_work);
diff --git a/net/bluetooth/hci_debugfs.c b/net/bluetooth/hci_debugfs.c
index 902b40a90b91..3f401ec5bb0c 100644
--- a/net/bluetooth/hci_debugfs.c
+++ b/net/bluetooth/hci_debugfs.c
@@ -1245,7 +1245,7 @@ void hci_debugfs_create_conn(struct hci_conn *conn)
 	struct hci_dev *hdev = conn->hdev;
 	char name[6];
 
-	if (IS_ERR_OR_NULL(hdev->debugfs))
+	if (IS_ERR_OR_NULL(hdev->debugfs) || conn->debugfs)
 		return;
 
 	snprintf(name, sizeof(name), "%u", conn->handle);
diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c
index 6643c9c20fa4..faca701bce2a 100644
--- a/net/bluetooth/hci_event.c
+++ b/net/bluetooth/hci_event.c
@@ -712,6 +712,47 @@ static u8 hci_cc_read_local_version(struct hci_dev *hdev, void *data,
 	return rp->status;
 }
 
+static u8 hci_cc_read_enc_key_size(struct hci_dev *hdev, void *data,
+				   struct sk_buff *skb)
+{
+	struct hci_rp_read_enc_key_size *rp = data;
+	struct hci_conn *conn;
+	u16 handle;
+	u8 status = rp->status;
+
+	bt_dev_dbg(hdev, "status 0x%2.2x", status);
+
+	handle = le16_to_cpu(rp->handle);
+
+	hci_dev_lock(hdev);
+
+	conn = hci_conn_hash_lookup_handle(hdev, handle);
+	if (!conn) {
+		status = 0xFF;
+		goto done;
+	}
+
+	/* While unexpected, the read_enc_key_size command may fail. The most
+	 * secure approach is to then assume the key size is 0 to force a
+	 * disconnection.
+	 */
+	if (status) {
+		bt_dev_err(hdev, "failed to read key size for handle %u",
+			   handle);
+		conn->enc_key_size = 0;
+	} else {
+		conn->enc_key_size = rp->key_size;
+		status = 0;
+	}
+
+	hci_encrypt_cfm(conn, 0);
+
+done:
+	hci_dev_unlock(hdev);
+
+	return status;
+}
+
 static u8 hci_cc_read_local_commands(struct hci_dev *hdev, void *data,
 				     struct sk_buff *skb)
 {
@@ -1715,6 +1756,8 @@ static void le_set_scan_enable_complete(struct hci_dev *hdev, u8 enable)
 		hci_dev_set_flag(hdev, HCI_LE_SCAN);
 		if (hdev->le_scan_type == LE_SCAN_ACTIVE)
 			clear_pending_adv_report(hdev);
+		if (hci_dev_test_flag(hdev, HCI_MESH))
+			hci_discovery_set_state(hdev, DISCOVERY_FINDING);
 		break;
 
 	case LE_SCAN_DISABLE:
@@ -1729,7 +1772,7 @@ static void le_set_scan_enable_complete(struct hci_dev *hdev, u8 enable)
 					  d->last_adv_addr_type, NULL,
 					  d->last_adv_rssi, d->last_adv_flags,
 					  d->last_adv_data,
-					  d->last_adv_data_len, NULL, 0);
+					  d->last_adv_data_len, NULL, 0, 0);
 		}
 
 		/* Cancel this timer so that we don't try to disable scanning
@@ -1745,6 +1788,9 @@ static void le_set_scan_enable_complete(struct hci_dev *hdev, u8 enable)
 		 */
 		if (hci_dev_test_and_clear_flag(hdev, HCI_LE_SCAN_INTERRUPTED))
 			hci_discovery_set_state(hdev, DISCOVERY_STOPPED);
+		else if (!hci_dev_test_flag(hdev, HCI_LE_ADV) &&
+			 hdev->discovery.state == DISCOVERY_FINDING)
+			queue_work(hdev->workqueue, &hdev->reenable_adv_work);
 
 		break;
 
@@ -2152,7 +2198,7 @@ static u8 hci_cc_set_ext_adv_param(struct hci_dev *hdev, void *data,
 			adv_instance->tx_power = rp->tx_power;
 	}
 	/* Update adv data as tx power is known now */
-	hci_req_update_adv_data(hdev, cp->handle);
+	hci_update_adv_data(hdev, cp->handle);
 
 	hci_dev_unlock(hdev);
 
@@ -3071,7 +3117,7 @@ static void hci_inquiry_result_evt(struct hci_dev *hdev, void *edata,
 
 		mgmt_device_found(hdev, &info->bdaddr, ACL_LINK, 0x00,
 				  info->dev_class, HCI_RSSI_INVALID,
-				  flags, NULL, 0, NULL, 0);
+				  flags, NULL, 0, NULL, 0, 0);
 	}
 
 	hci_dev_unlock(hdev);
@@ -3534,47 +3580,6 @@ unlock:
 	hci_dev_unlock(hdev);
 }
 
-static void read_enc_key_size_complete(struct hci_dev *hdev, u8 status,
-				       u16 opcode, struct sk_buff *skb)
-{
-	const struct hci_rp_read_enc_key_size *rp;
-	struct hci_conn *conn;
-	u16 handle;
-
-	BT_DBG("%s status 0x%02x", hdev->name, status);
-
-	if (!skb || skb->len < sizeof(*rp)) {
-		bt_dev_err(hdev, "invalid read key size response");
-		return;
-	}
-
-	rp = (void *)skb->data;
-	handle = le16_to_cpu(rp->handle);
-
-	hci_dev_lock(hdev);
-
-	conn = hci_conn_hash_lookup_handle(hdev, handle);
-	if (!conn)
-		goto unlock;
-
-	/* While unexpected, the read_enc_key_size command may fail. The most
-	 * secure approach is to then assume the key size is 0 to force a
-	 * disconnection.
-	 */
-	if (rp->status) {
-		bt_dev_err(hdev, "failed to read key size for handle %u",
-			   handle);
-		conn->enc_key_size = 0;
-	} else {
-		conn->enc_key_size = rp->key_size;
-	}
-
-	hci_encrypt_cfm(conn, 0);
-
-unlock:
-	hci_dev_unlock(hdev);
-}
-
 static void hci_encrypt_change_evt(struct hci_dev *hdev, void *data,
 				   struct sk_buff *skb)
 {
@@ -3639,7 +3644,6 @@ static void hci_encrypt_change_evt(struct hci_dev *hdev, void *data,
 	/* Try reading the encryption key size for encrypted ACL links */
 	if (!ev->status && ev->encrypt && conn->type == ACL_LINK) {
 		struct hci_cp_read_enc_key_size cp;
-		struct hci_request req;
 
 		/* Only send HCI_Read_Encryption_Key_Size if the
 		 * controller really supports it. If it doesn't, assume
@@ -3650,12 +3654,9 @@ static void hci_encrypt_change_evt(struct hci_dev *hdev, void *data,
 			goto notify;
 		}
 
-		hci_req_init(&req, hdev);
-
 		cp.handle = cpu_to_le16(conn->handle);
-		hci_req_add(&req, HCI_OP_READ_ENC_KEY_SIZE, sizeof(cp), &cp);
-
-		if (hci_req_run_skb(&req, read_enc_key_size_complete)) {
+		if (hci_send_cmd(hdev, HCI_OP_READ_ENC_KEY_SIZE,
+				 sizeof(cp), &cp)) {
 			bt_dev_err(hdev, "sending read key size failed");
 			conn->enc_key_size = HCI_LINK_KEY_SIZE;
 			goto notify;
@@ -3766,16 +3767,18 @@ static inline void handle_cmd_cnt_and_timer(struct hci_dev *hdev, u8 ncmd)
 {
 	cancel_delayed_work(&hdev->cmd_timer);
 
+	rcu_read_lock();
 	if (!test_bit(HCI_RESET, &hdev->flags)) {
 		if (ncmd) {
 			cancel_delayed_work(&hdev->ncmd_timer);
 			atomic_set(&hdev->cmd_cnt, 1);
 		} else {
 			if (!hci_dev_test_flag(hdev, HCI_CMD_DRAIN_WORKQUEUE))
-				schedule_delayed_work(&hdev->ncmd_timer,
-						      HCI_NCMD_TIMEOUT);
+				queue_delayed_work(hdev->workqueue, &hdev->ncmd_timer,
+						   HCI_NCMD_TIMEOUT);
 		}
 	}
+	rcu_read_unlock();
 }
 
 static u8 hci_cc_le_read_buffer_size_v2(struct hci_dev *hdev, void *data,
@@ -4037,6 +4040,8 @@ static const struct hci_cc {
 	       sizeof(struct hci_rp_read_local_amp_info)),
 	HCI_CC(HCI_OP_READ_CLOCK, hci_cc_read_clock,
 	       sizeof(struct hci_rp_read_clock)),
+	HCI_CC(HCI_OP_READ_ENC_KEY_SIZE, hci_cc_read_enc_key_size,
+	       sizeof(struct hci_rp_read_enc_key_size)),
 	HCI_CC(HCI_OP_READ_INQ_RSP_TX_POWER, hci_cc_read_inq_rsp_tx_power,
 	       sizeof(struct hci_rp_read_inq_rsp_tx_power)),
 	HCI_CC(HCI_OP_READ_DEF_ERR_DATA_REPORTING,
@@ -4829,7 +4834,7 @@ static void hci_inquiry_result_with_rssi_evt(struct hci_dev *hdev, void *edata,
 
 			mgmt_device_found(hdev, &info->bdaddr, ACL_LINK, 0x00,
 					  info->dev_class, info->rssi,
-					  flags, NULL, 0, NULL, 0);
+					  flags, NULL, 0, NULL, 0, 0);
 		}
 	} else if (skb->len == array_size(ev->num,
 					  sizeof(struct inquiry_info_rssi))) {
@@ -4860,7 +4865,7 @@ static void hci_inquiry_result_with_rssi_evt(struct hci_dev *hdev, void *edata,
 
 			mgmt_device_found(hdev, &info->bdaddr, ACL_LINK, 0x00,
 					  info->dev_class, info->rssi,
-					  flags, NULL, 0, NULL, 0);
+					  flags, NULL, 0, NULL, 0, 0);
 		}
 	} else {
 		bt_dev_err(hdev, "Malformed HCI Event: 0x%2.2x",
@@ -5116,7 +5121,7 @@ static void hci_extended_inquiry_result_evt(struct hci_dev *hdev, void *edata,
 
 		mgmt_device_found(hdev, &info->bdaddr, ACL_LINK, 0x00,
 				  info->dev_class, info->rssi,
-				  flags, info->data, eir_len, NULL, 0);
+				  flags, info->data, eir_len, NULL, 0, 0);
 	}
 
 	hci_dev_unlock(hdev);
@@ -6172,7 +6177,7 @@ static struct hci_conn *check_pending_le_conn(struct hci_dev *hdev,
 static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr,
 			       u8 bdaddr_type, bdaddr_t *direct_addr,
 			       u8 direct_addr_type, s8 rssi, u8 *data, u8 len,
-			       bool ext_adv)
+			       bool ext_adv, bool ctl_time, u64 instant)
 {
 	struct discovery_state *d = &hdev->discovery;
 	struct smp_irk *irk;
@@ -6220,7 +6225,7 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr,
 	 * important to see if the address is matching the local
 	 * controller address.
 	 */
-	if (direct_addr) {
+	if (!hci_dev_test_flag(hdev, HCI_MESH) && direct_addr) {
 		direct_addr_type = ev_bdaddr_type(hdev, direct_addr_type,
 						  &bdaddr_resolved);
 
@@ -6268,6 +6273,18 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr,
 		conn->le_adv_data_len = len;
 	}
 
+	if (type == LE_ADV_NONCONN_IND || type == LE_ADV_SCAN_IND)
+		flags = MGMT_DEV_FOUND_NOT_CONNECTABLE;
+	else
+		flags = 0;
+
+	/* All scan results should be sent up for Mesh systems */
+	if (hci_dev_test_flag(hdev, HCI_MESH)) {
+		mgmt_device_found(hdev, bdaddr, LE_LINK, bdaddr_type, NULL,
+				  rssi, flags, data, len, NULL, 0, instant);
+		return;
+	}
+
 	/* Passive scanning shouldn't trigger any device found events,
 	 * except for devices marked as CONN_REPORT for which we do send
 	 * device found events, or advertisement monitoring requested.
@@ -6281,12 +6298,8 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr,
 		    idr_is_empty(&hdev->adv_monitors_idr))
 			return;
 
-		if (type == LE_ADV_NONCONN_IND || type == LE_ADV_SCAN_IND)
-			flags = MGMT_DEV_FOUND_NOT_CONNECTABLE;
-		else
-			flags = 0;
 		mgmt_device_found(hdev, bdaddr, LE_LINK, bdaddr_type, NULL,
-				  rssi, flags, data, len, NULL, 0);
+				  rssi, flags, data, len, NULL, 0, 0);
 		return;
 	}
 
@@ -6305,11 +6318,8 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr,
 	 * and just sends a scan response event, then it is marked as
 	 * not connectable as well.
 	 */
-	if (type == LE_ADV_NONCONN_IND || type == LE_ADV_SCAN_IND ||
-	    type == LE_ADV_SCAN_RSP)
+	if (type == LE_ADV_SCAN_RSP)
 		flags = MGMT_DEV_FOUND_NOT_CONNECTABLE;
-	else
-		flags = 0;
 
 	/* If there's nothing pending either store the data from this
 	 * event or send an immediate device found event if the data
@@ -6326,7 +6336,7 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr,
 		}
 
 		mgmt_device_found(hdev, bdaddr, LE_LINK, bdaddr_type, NULL,
-				  rssi, flags, data, len, NULL, 0);
+				  rssi, flags, data, len, NULL, 0, 0);
 		return;
 	}
 
@@ -6345,7 +6355,7 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr,
 					  d->last_adv_addr_type, NULL,
 					  d->last_adv_rssi, d->last_adv_flags,
 					  d->last_adv_data,
-					  d->last_adv_data_len, NULL, 0);
+					  d->last_adv_data_len, NULL, 0, 0);
 
 		/* If the new report will trigger a SCAN_REQ store it for
 		 * later merging.
@@ -6362,7 +6372,7 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr,
 		 */
 		clear_pending_adv_report(hdev);
 		mgmt_device_found(hdev, bdaddr, LE_LINK, bdaddr_type, NULL,
-				  rssi, flags, data, len, NULL, 0);
+				  rssi, flags, data, len, NULL, 0, 0);
 		return;
 	}
 
@@ -6372,7 +6382,7 @@ static void process_adv_report(struct hci_dev *hdev, u8 type, bdaddr_t *bdaddr,
 	 */
 	mgmt_device_found(hdev, &d->last_adv_addr, LE_LINK,
 			  d->last_adv_addr_type, NULL, rssi, d->last_adv_flags,
-			  d->last_adv_data, d->last_adv_data_len, data, len);
+			  d->last_adv_data, d->last_adv_data_len, data, len, 0);
 	clear_pending_adv_report(hdev);
 }
 
@@ -6380,6 +6390,7 @@ static void hci_le_adv_report_evt(struct hci_dev *hdev, void *data,
 				  struct sk_buff *skb)
 {
 	struct hci_ev_le_advertising_report *ev = data;
+	u64 instant = jiffies;
 
 	if (!ev->num)
 		return;
@@ -6404,7 +6415,8 @@ static void hci_le_adv_report_evt(struct hci_dev *hdev, void *data,
 			rssi = info->data[info->length];
 			process_adv_report(hdev, info->type, &info->bdaddr,
 					   info->bdaddr_type, NULL, 0, rssi,
-					   info->data, info->length, false);
+					   info->data, info->length, false,
+					   false, instant);
 		} else {
 			bt_dev_err(hdev, "Dropping invalid advertising data");
 		}
@@ -6461,6 +6473,7 @@ static void hci_le_ext_adv_report_evt(struct hci_dev *hdev, void *data,
 				      struct sk_buff *skb)
 {
 	struct hci_ev_le_ext_adv_report *ev = data;
+	u64 instant = jiffies;
 
 	if (!ev->num)
 		return;
@@ -6487,7 +6500,8 @@ static void hci_le_ext_adv_report_evt(struct hci_dev *hdev, void *data,
 			process_adv_report(hdev, legacy_evt_type, &info->bdaddr,
 					   info->bdaddr_type, NULL, 0,
 					   info->rssi, info->data, info->length,
-					   !(evt_type & LE_EXT_ADV_LEGACY_PDU));
+					   !(evt_type & LE_EXT_ADV_LEGACY_PDU),
+					   false, instant);
 		}
 	}
 
@@ -6710,6 +6724,7 @@ static void hci_le_direct_adv_report_evt(struct hci_dev *hdev, void *data,
 					 struct sk_buff *skb)
 {
 	struct hci_ev_le_direct_adv_report *ev = data;
+	u64 instant = jiffies;
 	int i;
 
 	if (!hci_le_ev_skb_pull(hdev, skb, HCI_EV_LE_DIRECT_ADV_REPORT,
@@ -6727,7 +6742,7 @@ static void hci_le_direct_adv_report_evt(struct hci_dev *hdev, void *data,
 		process_adv_report(hdev, info->type, &info->bdaddr,
 				   info->bdaddr_type, &info->direct_addr,
 				   info->direct_addr_type, info->rssi, NULL, 0,
-				   false);
+				   false, false, instant);
 	}
 
 	hci_dev_unlock(hdev);
@@ -6776,6 +6791,13 @@ static void hci_le_cis_estabilished_evt(struct hci_dev *hdev, void *data,
 		goto unlock;
 	}
 
+	if (conn->type != ISO_LINK) {
+		bt_dev_err(hdev,
+			   "Invalid connection link type handle 0x%4.4x",
+			   handle);
+		goto unlock;
+	}
+
 	if (conn->role == HCI_ROLE_SLAVE) {
 		__le32 interval;
 
@@ -6896,6 +6918,13 @@ static void hci_le_create_big_complete_evt(struct hci_dev *hdev, void *data,
 	if (!conn)
 		goto unlock;
 
+	if (conn->type != ISO_LINK) {
+		bt_dev_err(hdev,
+			   "Invalid connection link type handle 0x%2.2x",
+			   ev->handle);
+		goto unlock;
+	}
+
 	if (ev->num_bis)
 		conn->handle = __le16_to_cpu(ev->bis_handle[0]);
 
diff --git a/net/bluetooth/hci_request.c b/net/bluetooth/hci_request.c
index e64d558e5d69..5a0296a4352e 100644
--- a/net/bluetooth/hci_request.c
+++ b/net/bluetooth/hci_request.c
@@ -269,42 +269,10 @@ void hci_req_add_ev(struct hci_request *req, u16 opcode, u32 plen,
 void hci_req_add(struct hci_request *req, u16 opcode, u32 plen,
 		 const void *param)
 {
+	bt_dev_err(req->hdev, "HCI_REQ-0x%4.4x", opcode);
 	hci_req_add_ev(req, opcode, plen, param, 0);
 }
 
-void __hci_req_write_fast_connectable(struct hci_request *req, bool enable)
-{
-	struct hci_dev *hdev = req->hdev;
-	struct hci_cp_write_page_scan_activity acp;
-	u8 type;
-
-	if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED))
-		return;
-
-	if (hdev->hci_ver < BLUETOOTH_VER_1_2)
-		return;
-
-	if (enable) {
-		type = PAGE_SCAN_TYPE_INTERLACED;
-
-		/* 160 msec page scan interval */
-		acp.interval = cpu_to_le16(0x0100);
-	} else {
-		type = hdev->def_page_scan_type;
-		acp.interval = cpu_to_le16(hdev->def_page_scan_int);
-	}
-
-	acp.window = cpu_to_le16(hdev->def_page_scan_window);
-
-	if (__cpu_to_le16(hdev->page_scan_interval) != acp.interval ||
-	    __cpu_to_le16(hdev->page_scan_window) != acp.window)
-		hci_req_add(req, HCI_OP_WRITE_PAGE_SCAN_ACTIVITY,
-			    sizeof(acp), &acp);
-
-	if (hdev->page_scan_type != type)
-		hci_req_add(req, HCI_OP_WRITE_PAGE_SCAN_TYPE, 1, &type);
-}
-
 static void start_interleave_scan(struct hci_dev *hdev)
 {
 	hdev->interleave_scan_state = INTERLEAVE_SCAN_NO_FILTER;
@@ -357,45 +325,6 @@ static bool __hci_update_interleaved_scan(struct hci_dev *hdev)
 	return false;
 }
 
-void __hci_req_update_name(struct hci_request *req)
-{
-	struct hci_dev *hdev = req->hdev;
-	struct hci_cp_write_local_name cp;
-
-	memcpy(cp.name, hdev->dev_name, sizeof(cp.name));
-
-	hci_req_add(req, HCI_OP_WRITE_LOCAL_NAME, sizeof(cp), &cp);
-}
-
-void __hci_req_update_eir(struct hci_request *req)
-{
-	struct hci_dev *hdev = req->hdev;
-	struct hci_cp_write_eir cp;
-
-	if (!hdev_is_powered(hdev))
-		return;
-
-	if (!lmp_ext_inq_capable(hdev))
-		return;
-
-	if (!hci_dev_test_flag(hdev, HCI_SSP_ENABLED))
-		return;
-
-	if (hci_dev_test_flag(hdev, HCI_SERVICE_CACHE))
-		return;
-
-	memset(&cp, 0, sizeof(cp));
-
-	eir_create(hdev, cp.data);
-
-	if (memcmp(cp.data, hdev->eir, sizeof(cp.data)) == 0)
-		return;
-
-	memcpy(hdev->eir, cp.data, sizeof(cp.data));
-
-	hci_req_add(req, HCI_OP_WRITE_EIR, sizeof(cp), &cp);
-}
-
 void hci_req_add_le_scan_disable(struct hci_request *req, bool rpa_le_conn)
 {
 	struct hci_dev *hdev = req->hdev;
@@ -721,6 +650,96 @@ static inline bool hci_is_le_conn_scanning(struct hci_dev *hdev)
 	return false;
 }
 
+static void set_random_addr(struct hci_request *req, bdaddr_t *rpa);
+static int hci_update_random_address(struct hci_request *req,
+				     bool require_privacy, bool use_rpa,
+				     u8 *own_addr_type)
+{
+	struct hci_dev *hdev = req->hdev;
+	int err;
+
+	/* If privacy is enabled use a resolvable private address. If
+	 * current RPA has expired or there is something else than
+	 * the current RPA in use, then generate a new one.
+	 */
+	if (use_rpa) {
+		/* If Controller supports LL Privacy use own address type is
+		 * 0x03
+		 */
+		if (use_ll_privacy(hdev))
+			*own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED;
+		else
+			*own_addr_type = ADDR_LE_DEV_RANDOM;
+
+		if (rpa_valid(hdev))
+			return 0;
+
+		err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa);
+		if (err < 0) {
+			bt_dev_err(hdev, "failed to generate new RPA");
+			return err;
+		}
+
+		set_random_addr(req, &hdev->rpa);
+
+		return 0;
+	}
+
+	/* In case of required privacy without resolvable private address,
+	 * use an non-resolvable private address. This is useful for active
+	 * scanning and non-connectable advertising.
+	 */
+	if (require_privacy) {
+		bdaddr_t nrpa;
+
+		while (true) {
+			/* The non-resolvable private address is generated
+			 * from random six bytes with the two most significant
+			 * bits cleared.
+			 */
+			get_random_bytes(&nrpa, 6);
+			nrpa.b[5] &= 0x3f;
+
+			/* The non-resolvable private address shall not be
+			 * equal to the public address.
+			 */
+			if (bacmp(&hdev->bdaddr, &nrpa))
+				break;
+		}
+
+		*own_addr_type = ADDR_LE_DEV_RANDOM;
+		set_random_addr(req, &nrpa);
+		return 0;
+	}
+
+	/* If forcing static address is in use or there is no public
+	 * address use the static address as random address (but skip
+	 * the HCI command if the current random address is already the
+	 * static one.
+	 *
+	 * In case BR/EDR has been disabled on a dual-mode controller
+	 * and a static address has been configured, then use that
+	 * address instead of the public BR/EDR address.
+	 */
+	if (hci_dev_test_flag(hdev, HCI_FORCE_STATIC_ADDR) ||
+	    !bacmp(&hdev->bdaddr, BDADDR_ANY) ||
+	    (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED) &&
+	     bacmp(&hdev->static_addr, BDADDR_ANY))) {
+		*own_addr_type = ADDR_LE_DEV_RANDOM;
+		if (bacmp(&hdev->static_addr, &hdev->random_addr))
+			hci_req_add(req, HCI_OP_LE_SET_RANDOM_ADDR, 6,
+				    &hdev->static_addr);
+		return 0;
+	}
+
+	/* Neither privacy nor static address is being used so use a
+	 * public address.
+	 */
+	*own_addr_type = ADDR_LE_DEV_PUBLIC;
+
+	return 0;
+}
+
 /* Ensure to call hci_req_add_le_scan_disable() first to disable the
  * controller based address resolution to be able to reconfigure
  * resolving list.
@@ -810,366 +829,6 @@ void hci_req_add_le_passive_scan(struct hci_request *req)
 			   addr_resolv);
 }
 
-static void cancel_adv_timeout(struct hci_dev *hdev)
-{
-	if (hdev->adv_instance_timeout) {
-		hdev->adv_instance_timeout = 0;
-		cancel_delayed_work(&hdev->adv_instance_expire);
-	}
-}
-
-static bool adv_cur_instance_is_scannable(struct hci_dev *hdev)
-{
-	return hci_adv_instance_is_scannable(hdev, hdev->cur_adv_instance);
-}
-
-void __hci_req_disable_advertising(struct hci_request *req)
-{
-	if (ext_adv_capable(req->hdev)) {
-		__hci_req_disable_ext_adv_instance(req, 0x00);
-	} else {
-		u8 enable = 0x00;
-
-		hci_req_add(req, HCI_OP_LE_SET_ADV_ENABLE, sizeof(enable), &enable);
-	}
-}
-
-static bool adv_use_rpa(struct hci_dev *hdev, uint32_t flags)
-{
-	/* If privacy is not enabled don't use RPA */
-	if (!hci_dev_test_flag(hdev, HCI_PRIVACY))
-		return false;
-
-	/* If basic privacy mode is enabled use RPA */
-	if (!hci_dev_test_flag(hdev, HCI_LIMITED_PRIVACY))
-		return true;
-
-	/* If limited privacy mode is enabled don't use RPA if we're
-	 * both discoverable and bondable.
-	 */
-	if ((flags & MGMT_ADV_FLAG_DISCOV) &&
-	    hci_dev_test_flag(hdev, HCI_BONDABLE))
-		return false;
-
-	/* We're neither bondable nor discoverable in the limited
-	 * privacy mode, therefore use RPA.
-	 */
-	return true;
-}
-
-static bool is_advertising_allowed(struct hci_dev *hdev, bool connectable)
-{
-	/* If there is no connection we are OK to advertise. */
-	if (hci_conn_num(hdev, LE_LINK) == 0)
-		return true;
-
-	/* Check le_states if there is any connection in peripheral role. */
-	if (hdev->conn_hash.le_num_peripheral > 0) {
-		/* Peripheral connection state and non connectable mode bit 20.
-		 */
-		if (!connectable && !(hdev->le_states[2] & 0x10))
-			return false;
-
-		/* Peripheral connection state and connectable mode bit 38
-		 * and scannable bit 21.
-		 */
-		if (connectable && (!(hdev->le_states[4] & 0x40) ||
-				    !(hdev->le_states[2] & 0x20)))
-			return false;
-	}
-
-	/* Check le_states if there is any connection in central role. */
-	if (hci_conn_num(hdev, LE_LINK) != hdev->conn_hash.le_num_peripheral) {
-		/* Central connection state and non connectable mode bit 18. */
-		if (!connectable && !(hdev->le_states[2] & 0x02))
-			return false;
-
-		/* Central connection state and connectable mode bit 35 and
-		 * scannable 19.
-		 */
-		if (connectable && (!(hdev->le_states[4] & 0x08) ||
-				    !(hdev->le_states[2] & 0x08)))
-			return false;
-	}
-
-	return true;
-}
-
-void __hci_req_enable_advertising(struct hci_request *req)
-{
-	struct hci_dev *hdev = req->hdev;
-	struct adv_info *adv;
-	struct hci_cp_le_set_adv_param cp;
-	u8 own_addr_type, enable = 0x01;
-	bool connectable;
-	u16 adv_min_interval, adv_max_interval;
-	u32 flags;
-
-	flags = hci_adv_instance_flags(hdev, hdev->cur_adv_instance);
-	adv = hci_find_adv_instance(hdev, hdev->cur_adv_instance);
-
-	/* If the "connectable" instance flag was not set, then choose between
-	 * ADV_IND and ADV_NONCONN_IND based on the global connectable setting.
-	 */
-	connectable = (flags & MGMT_ADV_FLAG_CONNECTABLE) ||
-		      mgmt_get_connectable(hdev);
-
-	if (!is_advertising_allowed(hdev, connectable))
-		return;
-
-	if (hci_dev_test_flag(hdev, HCI_LE_ADV))
-		__hci_req_disable_advertising(req);
-
-	/* Clear the HCI_LE_ADV bit temporarily so that the
-	 * hci_update_random_address knows that it's safe to go ahead
-	 * and write a new random address. The flag will be set back on
-	 * as soon as the SET_ADV_ENABLE HCI command completes.
-	 */
-	hci_dev_clear_flag(hdev, HCI_LE_ADV);
-
-	/* Set require_privacy to true only when non-connectable
-	 * advertising is used. In that case it is fine to use a
-	 * non-resolvable private address.
-	 */
-	if (hci_update_random_address(req, !connectable,
-				      adv_use_rpa(hdev, flags),
-				      &own_addr_type) < 0)
-		return;
-
-	memset(&cp, 0, sizeof(cp));
-
-	if (adv) {
-		adv_min_interval = adv->min_interval;
-		adv_max_interval = adv->max_interval;
-	} else {
-		adv_min_interval = hdev->le_adv_min_interval;
-		adv_max_interval = hdev->le_adv_max_interval;
-	}
-
-	if (connectable) {
-		cp.type = LE_ADV_IND;
-	} else {
-		if (adv_cur_instance_is_scannable(hdev))
-			cp.type = LE_ADV_SCAN_IND;
-		else
-			cp.type = LE_ADV_NONCONN_IND;
-
-		if (!hci_dev_test_flag(hdev, HCI_DISCOVERABLE) ||
-		    hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) {
-			adv_min_interval = DISCOV_LE_FAST_ADV_INT_MIN;
-			adv_max_interval = DISCOV_LE_FAST_ADV_INT_MAX;
-		}
-	}
-
-	cp.min_interval = cpu_to_le16(adv_min_interval);
-	cp.max_interval = cpu_to_le16(adv_max_interval);
-	cp.own_address_type = own_addr_type;
-	cp.channel_map = hdev->le_adv_channel_map;
-
-	hci_req_add(req, HCI_OP_LE_SET_ADV_PARAM, sizeof(cp), &cp);
-
-	hci_req_add(req, HCI_OP_LE_SET_ADV_ENABLE, sizeof(enable), &enable);
-}
-
-void __hci_req_update_scan_rsp_data(struct hci_request *req, u8 instance)
-{
-	struct hci_dev *hdev = req->hdev;
-	u8 len;
-
-	if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED))
-		return;
-
-	if (ext_adv_capable(hdev)) {
-		struct {
-			struct hci_cp_le_set_ext_scan_rsp_data cp;
-			u8 data[HCI_MAX_EXT_AD_LENGTH];
-		} pdu;
-
-		memset(&pdu, 0, sizeof(pdu));
-
-		len = eir_create_scan_rsp(hdev, instance, pdu.data);
-
-		if (hdev->scan_rsp_data_len == len &&
-		    !memcmp(pdu.data, hdev->scan_rsp_data, len))
-			return;
-
-		memcpy(hdev->scan_rsp_data, pdu.data, len);
-		hdev->scan_rsp_data_len = len;
-
-		pdu.cp.handle = instance;
-		pdu.cp.length = len;
-		pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE;
-		pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG;
-
-		hci_req_add(req, HCI_OP_LE_SET_EXT_SCAN_RSP_DATA,
-			    sizeof(pdu.cp) + len, &pdu.cp);
-	} else {
-		struct hci_cp_le_set_scan_rsp_data cp;
-
-		memset(&cp, 0, sizeof(cp));
-
-		len = eir_create_scan_rsp(hdev, instance, cp.data);
-
-		if (hdev->scan_rsp_data_len == len &&
-		    !memcmp(cp.data, hdev->scan_rsp_data, len))
-			return;
-
-		memcpy(hdev->scan_rsp_data, cp.data, sizeof(cp.data));
-		hdev->scan_rsp_data_len = len;
-
-		cp.length = len;
-
-		hci_req_add(req, HCI_OP_LE_SET_SCAN_RSP_DATA, sizeof(cp), &cp);
-	}
-}
-
-void __hci_req_update_adv_data(struct hci_request *req, u8 instance)
-{
-	struct hci_dev *hdev = req->hdev;
-	u8 len;
-
-	if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED))
-		return;
-
-	if (ext_adv_capable(hdev)) {
-		struct {
-			struct hci_cp_le_set_ext_adv_data cp;
-			u8 data[HCI_MAX_EXT_AD_LENGTH];
-		} pdu;
-
-		memset(&pdu, 0, sizeof(pdu));
-
-		len = eir_create_adv_data(hdev, instance, pdu.data);
-
-		/* There's nothing to do if the data hasn't changed */
-		if (hdev->adv_data_len == len &&
-		    memcmp(pdu.data, hdev->adv_data, len) == 0)
-			return;
-
-		memcpy(hdev->adv_data, pdu.data, len);
-		hdev->adv_data_len = len;
-
-		pdu.cp.length = len;
-		pdu.cp.handle = instance;
-		pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE;
-		pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG;
-
-		hci_req_add(req, HCI_OP_LE_SET_EXT_ADV_DATA,
-			    sizeof(pdu.cp) + len, &pdu.cp);
-	} else {
-		struct hci_cp_le_set_adv_data cp;
-
-		memset(&cp, 0, sizeof(cp));
-
-		len = eir_create_adv_data(hdev, instance, cp.data);
-
-		/* There's nothing to do if the data hasn't changed */
-		if (hdev->adv_data_len == len &&
-		    memcmp(cp.data, hdev->adv_data, len) == 0)
-			return;
-
-		memcpy(hdev->adv_data, cp.data, sizeof(cp.data));
-		hdev->adv_data_len = len;
-
-		cp.length = len;
-
-		hci_req_add(req, HCI_OP_LE_SET_ADV_DATA, sizeof(cp), &cp);
-	}
-}
-
-int hci_req_update_adv_data(struct hci_dev *hdev, u8 instance)
-{
-	struct hci_request req;
-
-	hci_req_init(&req, hdev);
-	__hci_req_update_adv_data(&req, instance);
-
-	return hci_req_run(&req, NULL);
-}
-
-static void enable_addr_resolution_complete(struct hci_dev *hdev, u8 status,
-					    u16 opcode)
-{
-	BT_DBG("%s status %u", hdev->name, status);
-}
-
-void hci_req_disable_address_resolution(struct hci_dev *hdev)
-{
-	struct hci_request req;
-	__u8 enable = 0x00;
-
-	if (!hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION))
-		return;
-
-	hci_req_init(&req, hdev);
-
-	hci_req_add(&req, HCI_OP_LE_SET_ADDR_RESOLV_ENABLE, 1, &enable);
-
-	hci_req_run(&req, enable_addr_resolution_complete);
-}
-
-static void adv_enable_complete(struct hci_dev *hdev, u8 status, u16 opcode)
-{
-	bt_dev_dbg(hdev, "status %u", status);
-}
-
-void hci_req_reenable_advertising(struct hci_dev *hdev)
-{
-	struct hci_request req;
-
-	if (!hci_dev_test_flag(hdev, HCI_ADVERTISING) &&
-	    list_empty(&hdev->adv_instances))
-		return;
-
-	hci_req_init(&req, hdev);
-
-	if (hdev->cur_adv_instance) {
-		__hci_req_schedule_adv_instance(&req, hdev->cur_adv_instance,
-						true);
-	} else {
-		if (ext_adv_capable(hdev)) {
-			__hci_req_start_ext_adv(&req, 0x00);
-		} else {
-			__hci_req_update_adv_data(&req, 0x00);
-			__hci_req_update_scan_rsp_data(&req, 0x00);
-			__hci_req_enable_advertising(&req);
-		}
-	}
-
-	hci_req_run(&req, adv_enable_complete);
-}
-
-static void adv_timeout_expire(struct work_struct *work)
-{
-	struct hci_dev *hdev = container_of(work, struct hci_dev,
-					    adv_instance_expire.work);
-
-	struct hci_request req;
-	u8 instance;
-
-	bt_dev_dbg(hdev, "");
-
-	hci_dev_lock(hdev);
-
-	hdev->adv_instance_timeout = 0;
-
-	instance = hdev->cur_adv_instance;
-	if (instance == 0x00)
-		goto unlock;
-
-	hci_req_init(&req, hdev);
-
-	hci_req_clear_adv_instance(hdev, NULL, &req, instance, false);
-
-	if (list_empty(&hdev->adv_instances))
-		__hci_req_disable_advertising(&req);
-
-	hci_req_run(&req, NULL);
-
-unlock:
-	hci_dev_unlock(hdev);
-}
-
 static int hci_req_add_le_interleaved_scan(struct hci_request *req,
 					   unsigned long opt)
 {
@@ -1226,84 +885,6 @@ static void interleave_scan_work(struct work_struct *work)
 				   &hdev->interleave_scan, timeout);
 }
 
-int hci_get_random_address(struct hci_dev *hdev, bool require_privacy,
-			   bool use_rpa, struct adv_info *adv_instance,
-			   u8 *own_addr_type, bdaddr_t *rand_addr)
-{
-	int err;
-
-	bacpy(rand_addr, BDADDR_ANY);
-
-	/* If privacy is enabled use a resolvable private address. If
-	 * current RPA has expired then generate a new one.
-	 */
-	if (use_rpa) {
-		/* If Controller supports LL Privacy use own address type is
-		 * 0x03
-		 */
-		if (use_ll_privacy(hdev))
-			*own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED;
-		else
-			*own_addr_type = ADDR_LE_DEV_RANDOM;
-
-		if (adv_instance) {
-			if (adv_rpa_valid(adv_instance))
-				return 0;
-		} else {
-			if (rpa_valid(hdev))
-				return 0;
-		}
-
-		err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa);
-		if (err < 0) {
-			bt_dev_err(hdev, "failed to generate new RPA");
-			return err;
-		}
-
-		bacpy(rand_addr, &hdev->rpa);
-
-		return 0;
-	}
-
-	/* In case of required privacy without resolvable private address,
-	 * use an non-resolvable private address. This is useful for
-	 * non-connectable advertising.
-	 */
-	if (require_privacy) {
-		bdaddr_t nrpa;
-
-		while (true) {
-			/* The non-resolvable private address is generated
-			 * from random six bytes with the two most significant
-			 * bits cleared.
-			 */
-			get_random_bytes(&nrpa, 6);
-			nrpa.b[5] &= 0x3f;
-
-			/* The non-resolvable private address shall not be
-			 * equal to the public address.
-			 */
-			if (bacmp(&hdev->bdaddr, &nrpa))
-				break;
-		}
-
-		*own_addr_type = ADDR_LE_DEV_RANDOM;
-		bacpy(rand_addr, &nrpa);
-
-		return 0;
-	}
-
-	/* No privacy so use a public address. */
-	*own_addr_type = ADDR_LE_DEV_PUBLIC;
-
-	return 0;
-}
-
-void __hci_req_clear_ext_adv_sets(struct hci_request *req)
-{
-	hci_req_add(req, HCI_OP_LE_CLEAR_ADV_SETS, 0, NULL);
-}
-
 static void set_random_addr(struct hci_request *req, bdaddr_t *rpa)
 {
 	struct hci_dev *hdev = req->hdev;
@@ -1328,933 +909,8 @@ static void set_random_addr(struct hci_request *req, bdaddr_t *rpa)
 	hci_req_add(req, HCI_OP_LE_SET_RANDOM_ADDR, 6, rpa);
 }
 
-int __hci_req_setup_ext_adv_instance(struct hci_request *req, u8 instance)
-{
-	struct hci_cp_le_set_ext_adv_params cp;
-	struct hci_dev *hdev = req->hdev;
-	bool connectable;
-	u32 flags;
-	bdaddr_t random_addr;
-	u8 own_addr_type;
-	int err;
-	struct adv_info *adv;
-	bool secondary_adv, require_privacy;
-
-	if (instance > 0) {
-		adv = hci_find_adv_instance(hdev, instance);
-		if (!adv)
-			return -EINVAL;
-	} else {
-		adv = NULL;
-	}
-
-	flags = hci_adv_instance_flags(hdev, instance);
-
-	/* If the "connectable" instance flag was not set, then choose between
-	 * ADV_IND and ADV_NONCONN_IND based on the global connectable setting.
-	 */
-	connectable = (flags & MGMT_ADV_FLAG_CONNECTABLE) ||
-		      mgmt_get_connectable(hdev);
-
-	if (!is_advertising_allowed(hdev, connectable))
-		return -EPERM;
-
-	/* Set require_privacy to true only when non-connectable
-	 * advertising is used. In that case it is fine to use a
-	 * non-resolvable private address.
-	 */
-	require_privacy = !connectable;
-
-	/* Don't require privacy for periodic adv? */
-	if (adv && adv->periodic)
-		require_privacy = false;
-
-	err = hci_get_random_address(hdev, require_privacy,
-				     adv_use_rpa(hdev, flags), adv,
-				     &own_addr_type, &random_addr);
-	if (err < 0)
-		return err;
-
-	memset(&cp, 0, sizeof(cp));
-
-	if (adv) {
-		hci_cpu_to_le24(adv->min_interval, cp.min_interval);
-		hci_cpu_to_le24(adv->max_interval, cp.max_interval);
-		cp.tx_power = adv->tx_power;
-	} else {
-		hci_cpu_to_le24(hdev->le_adv_min_interval, cp.min_interval);
-		hci_cpu_to_le24(hdev->le_adv_max_interval, cp.max_interval);
-		cp.tx_power = HCI_ADV_TX_POWER_NO_PREFERENCE;
-	}
-
-	secondary_adv = (flags & MGMT_ADV_FLAG_SEC_MASK);
-
-	if (connectable) {
-		if (secondary_adv)
-			cp.evt_properties = cpu_to_le16(LE_EXT_ADV_CONN_IND);
-		else
-			cp.evt_properties = cpu_to_le16(LE_LEGACY_ADV_IND);
-	} else if (hci_adv_instance_is_scannable(hdev, instance) ||
-		   (flags & MGMT_ADV_PARAM_SCAN_RSP)) {
-		if (secondary_adv)
-			cp.evt_properties = cpu_to_le16(LE_EXT_ADV_SCAN_IND);
-		else
-			cp.evt_properties = cpu_to_le16(LE_LEGACY_ADV_SCAN_IND);
-	} else {
-		/* Secondary and periodic cannot use legacy PDUs */
-		if (secondary_adv || (adv && adv->periodic))
-			cp.evt_properties = cpu_to_le16(LE_EXT_ADV_NON_CONN_IND);
-		else
-			cp.evt_properties = cpu_to_le16(LE_LEGACY_NONCONN_IND);
-	}
-
-	cp.own_addr_type = own_addr_type;
-	cp.channel_map = hdev->le_adv_channel_map;
-	cp.handle = instance;
-
-	if (flags & MGMT_ADV_FLAG_SEC_2M) {
-		cp.primary_phy = HCI_ADV_PHY_1M;
-		cp.secondary_phy = HCI_ADV_PHY_2M;
-	} else if (flags & MGMT_ADV_FLAG_SEC_CODED) {
-		cp.primary_phy = HCI_ADV_PHY_CODED;
-		cp.secondary_phy = HCI_ADV_PHY_CODED;
-	} else {
-		/* In all other cases use 1M */
-		cp.primary_phy = HCI_ADV_PHY_1M;
-		cp.secondary_phy = HCI_ADV_PHY_1M;
-	}
-
-	hci_req_add(req, HCI_OP_LE_SET_EXT_ADV_PARAMS, sizeof(cp), &cp);
-
-	if ((own_addr_type == ADDR_LE_DEV_RANDOM ||
-	     own_addr_type == ADDR_LE_DEV_RANDOM_RESOLVED) &&
-	    bacmp(&random_addr, BDADDR_ANY)) {
-		struct hci_cp_le_set_adv_set_rand_addr cp;
-
-		/* Check if random address need to be updated */
-		if (adv) {
-			if (!bacmp(&random_addr, &adv->random_addr))
-				return 0;
-		} else {
-			if (!bacmp(&random_addr, &hdev->random_addr))
-				return 0;
-			/* Instance 0x00 doesn't have an adv_info, instead it
-			 * uses hdev->random_addr to track its address so
-			 * whenever it needs to be updated this also set the
-			 * random address since hdev->random_addr is shared with
-			 * scan state machine.
-			 */
-			set_random_addr(req, &random_addr);
-		}
-
-		memset(&cp, 0, sizeof(cp));
-
-		cp.handle = instance;
-		bacpy(&cp.bdaddr, &random_addr);
-
-		hci_req_add(req,
-			    HCI_OP_LE_SET_ADV_SET_RAND_ADDR,
-			    sizeof(cp), &cp);
-	}
-
-	return 0;
-}
-
-int __hci_req_enable_ext_advertising(struct hci_request *req, u8 instance)
-{
-	struct hci_dev *hdev = req->hdev;
-	struct hci_cp_le_set_ext_adv_enable *cp;
-	struct hci_cp_ext_adv_set *adv_set;
-	u8 data[sizeof(*cp) + sizeof(*adv_set) * 1];
-	struct adv_info *adv_instance;
-
-	if (instance > 0) {
-		adv_instance = hci_find_adv_instance(hdev, instance);
-		if (!adv_instance)
-			return -EINVAL;
-	} else {
-		adv_instance = NULL;
-	}
-
-	cp = (void *) data;
-	adv_set = (void *) cp->data;
-
-	memset(cp, 0, sizeof(*cp));
-
-	cp->enable = 0x01;
-	cp->num_of_sets = 0x01;
-
-	memset(adv_set, 0, sizeof(*adv_set));
-
-	adv_set->handle = instance;
-
-	/* Set duration per instance since controller is responsible for
-	 * scheduling it.
-	 */
-	if (adv_instance && adv_instance->duration) {
-		u16 duration = adv_instance->timeout * MSEC_PER_SEC;
-
-		/* Time = N * 10 ms */
-		adv_set->duration = cpu_to_le16(duration / 10);
-	}
-
-	hci_req_add(req, HCI_OP_LE_SET_EXT_ADV_ENABLE,
-		    sizeof(*cp) + sizeof(*adv_set) * cp->num_of_sets,
-		    data);
-
-	return 0;
-}
-
-int __hci_req_disable_ext_adv_instance(struct hci_request *req, u8 instance)
-{
-	struct hci_dev *hdev = req->hdev;
-	struct hci_cp_le_set_ext_adv_enable *cp;
-	struct hci_cp_ext_adv_set *adv_set;
-	u8 data[sizeof(*cp) + sizeof(*adv_set) * 1];
-	u8 req_size;
-
-	/* If request specifies an instance that doesn't exist, fail */
-	if (instance > 0 && !hci_find_adv_instance(hdev, instance))
-		return -EINVAL;
-
-	memset(data, 0, sizeof(data));
-
-	cp = (void *)data;
-	adv_set = (void *)cp->data;
-
-	/* Instance 0x00 indicates all advertising instances will be disabled */
-	cp->num_of_sets = !!instance;
-	cp->enable = 0x00;
-
-	adv_set->handle = instance;
-
-	req_size = sizeof(*cp) + sizeof(*adv_set) * cp->num_of_sets;
-	hci_req_add(req, HCI_OP_LE_SET_EXT_ADV_ENABLE, req_size, data);
-
-	return 0;
-}
-
-int __hci_req_remove_ext_adv_instance(struct hci_request *req, u8 instance)
-{
-	struct hci_dev *hdev = req->hdev;
-
-	/* If request specifies an instance that doesn't exist, fail */
-	if (instance > 0 && !hci_find_adv_instance(hdev, instance))
-		return -EINVAL;
-
-	hci_req_add(req, HCI_OP_LE_REMOVE_ADV_SET, sizeof(instance), &instance);
-
-	return 0;
-}
-
-int __hci_req_start_ext_adv(struct hci_request *req, u8 instance)
-{
-	struct hci_dev *hdev = req->hdev;
-	struct adv_info *adv_instance = hci_find_adv_instance(hdev, instance);
-	int err;
-
-	/* If instance isn't pending, the chip knows about it, and it's safe to
-	 * disable
-	 */
-	if (adv_instance && !adv_instance->pending)
-		__hci_req_disable_ext_adv_instance(req, instance);
-
-	err = __hci_req_setup_ext_adv_instance(req, instance);
-	if (err < 0)
-		return err;
-
-	__hci_req_update_scan_rsp_data(req, instance);
-	__hci_req_enable_ext_advertising(req, instance);
-
-	return 0;
-}
-
-int __hci_req_schedule_adv_instance(struct hci_request *req, u8 instance,
-				    bool force)
-{
-	struct hci_dev *hdev = req->hdev;
-	struct adv_info *adv_instance = NULL;
-	u16 timeout;
-
-	if (hci_dev_test_flag(hdev, HCI_ADVERTISING) ||
-	    list_empty(&hdev->adv_instances))
-		return -EPERM;
-
-	if (hdev->adv_instance_timeout)
-		return -EBUSY;
-
-	adv_instance = hci_find_adv_instance(hdev, instance);
-	if (!adv_instance)
-		return -ENOENT;
-
-	/* A zero timeout means unlimited advertising. As long as there is
-	 * only one instance, duration should be ignored. We still set a timeout
-	 * in case further instances are being added later on.
-	 *
-	 * If the remaining lifetime of the instance is more than the duration
-	 * then the timeout corresponds to the duration, otherwise it will be
-	 * reduced to the remaining instance lifetime.
-	 */
-	if (adv_instance->timeout == 0 ||
-	    adv_instance->duration <= adv_instance->remaining_time)
-		timeout = adv_instance->duration;
-	else
-		timeout = adv_instance->remaining_time;
-
-	/* The remaining time is being reduced unless the instance is being
-	 * advertised without time limit.
-	 */
-	if (adv_instance->timeout)
-		adv_instance->remaining_time =
-				adv_instance->remaining_time - timeout;
-
-	/* Only use work for scheduling instances with legacy advertising */
-	if (!ext_adv_capable(hdev)) {
-		hdev->adv_instance_timeout = timeout;
-		queue_delayed_work(hdev->req_workqueue,
-			   &hdev->adv_instance_expire,
-			   msecs_to_jiffies(timeout * 1000));
-	}
-
-	/* If we're just re-scheduling the same instance again then do not
-	 * execute any HCI commands. This happens when a single instance is
-	 * being advertised.
-	 */
-	if (!force && hdev->cur_adv_instance == instance &&
-	    hci_dev_test_flag(hdev, HCI_LE_ADV))
-		return 0;
-
-	hdev->cur_adv_instance = instance;
-	if (ext_adv_capable(hdev)) {
-		__hci_req_start_ext_adv(req, instance);
-	} else {
-		__hci_req_update_adv_data(req, instance);
-		__hci_req_update_scan_rsp_data(req, instance);
-		__hci_req_enable_advertising(req);
-	}
-
-	return 0;
-}
-
-/* For a single instance:
- * - force == true: The instance will be removed even when its remaining
- *   lifetime is not zero.
- * - force == false: the instance will be deactivated but kept stored unless
- *   the remaining lifetime is zero.
- *
- * For instance == 0x00:
- * - force == true: All instances will be removed regardless of their timeout
- *   setting.
- * - force == false: Only instances that have a timeout will be removed.
- */
-void hci_req_clear_adv_instance(struct hci_dev *hdev, struct sock *sk,
-				struct hci_request *req, u8 instance,
-				bool force)
-{
-	struct adv_info *adv_instance, *n, *next_instance = NULL;
-	int err;
-	u8 rem_inst;
-
-	/* Cancel any timeout concerning the removed instance(s). */
-	if (!instance || hdev->cur_adv_instance == instance)
-		cancel_adv_timeout(hdev);
-
-	/* Get the next instance to advertise BEFORE we remove
-	 * the current one. This can be the same instance again
-	 * if there is only one instance.
-	 */
-	if (instance && hdev->cur_adv_instance == instance)
-		next_instance = hci_get_next_instance(hdev, instance);
-
-	if (instance == 0x00) {
-		list_for_each_entry_safe(adv_instance, n, &hdev->adv_instances,
-					 list) {
-			if (!(force || adv_instance->timeout))
-				continue;
-
-			rem_inst = adv_instance->instance;
-			err = hci_remove_adv_instance(hdev, rem_inst);
-			if (!err)
-				mgmt_advertising_removed(sk, hdev, rem_inst);
-		}
-	} else {
-		adv_instance = hci_find_adv_instance(hdev, instance);
-
-		if (force || (adv_instance && adv_instance->timeout &&
-			      !adv_instance->remaining_time)) {
-			/* Don't advertise a removed instance. */
-			if (next_instance &&
-			    next_instance->instance == instance)
-				next_instance = NULL;
-
-			err = hci_remove_adv_instance(hdev, instance);
-			if (!err)
-				mgmt_advertising_removed(sk, hdev, instance);
-		}
-	}
-
-	if (!req || !hdev_is_powered(hdev) ||
-	    hci_dev_test_flag(hdev, HCI_ADVERTISING))
-		return;
-
-	if (next_instance && !ext_adv_capable(hdev))
-		__hci_req_schedule_adv_instance(req, next_instance->instance,
-						false);
-}
-
-int hci_update_random_address(struct hci_request *req, bool require_privacy,
-			      bool use_rpa, u8 *own_addr_type)
-{
-	struct hci_dev *hdev = req->hdev;
-	int err;
-
-	/* If privacy is enabled use a resolvable private address. If
-	 * current RPA has expired or there is something else than
-	 * the current RPA in use, then generate a new one.
-	 */
-	if (use_rpa) {
-		/* If Controller supports LL Privacy use own address type is
-		 * 0x03
-		 */
-		if (use_ll_privacy(hdev))
-			*own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED;
-		else
-			*own_addr_type = ADDR_LE_DEV_RANDOM;
-
-		if (rpa_valid(hdev))
-			return 0;
-
-		err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa);
-		if (err < 0) {
-			bt_dev_err(hdev, "failed to generate new RPA");
-			return err;
-		}
-
-		set_random_addr(req, &hdev->rpa);
-
-		return 0;
-	}
-
-	/* In case of required privacy without resolvable private address,
-	 * use an non-resolvable private address. This is useful for active
-	 * scanning and non-connectable advertising.
-	 */
-	if (require_privacy) {
-		bdaddr_t nrpa;
-
-		while (true) {
-			/* The non-resolvable private address is generated
-			 * from random six bytes with the two most significant
-			 * bits cleared.
-			 */
-			get_random_bytes(&nrpa, 6);
-			nrpa.b[5] &= 0x3f;
-
-			/* The non-resolvable private address shall not be
-			 * equal to the public address.
-			 */
-			if (bacmp(&hdev->bdaddr, &nrpa))
-				break;
-		}
-
-		*own_addr_type = ADDR_LE_DEV_RANDOM;
-		set_random_addr(req, &nrpa);
-		return 0;
-	}
-
-	/* If forcing static address is in use or there is no public
-	 * address use the static address as random address (but skip
-	 * the HCI command if the current random address is already the
-	 * static one.
-	 *
-	 * In case BR/EDR has been disabled on a dual-mode controller
-	 * and a static address has been configured, then use that
-	 * address instead of the public BR/EDR address.
-	 */
-	if (hci_dev_test_flag(hdev, HCI_FORCE_STATIC_ADDR) ||
-	    !bacmp(&hdev->bdaddr, BDADDR_ANY) ||
-	    (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED) &&
-	     bacmp(&hdev->static_addr, BDADDR_ANY))) {
-		*own_addr_type = ADDR_LE_DEV_RANDOM;
-		if (bacmp(&hdev->static_addr, &hdev->random_addr))
-			hci_req_add(req, HCI_OP_LE_SET_RANDOM_ADDR, 6,
-				    &hdev->static_addr);
-		return 0;
-	}
-
-	/* Neither privacy nor static address is being used so use a
-	 * public address.
-	 */
-	*own_addr_type = ADDR_LE_DEV_PUBLIC;
-
-	return 0;
-}
-
-static bool disconnected_accept_list_entries(struct hci_dev *hdev)
-{
-	struct bdaddr_list *b;
-
-	list_for_each_entry(b, &hdev->accept_list, list) {
-		struct hci_conn *conn;
-
-		conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &b->bdaddr);
-		if (!conn)
-			return true;
-
-		if (conn->state != BT_CONNECTED && conn->state != BT_CONFIG)
-			return true;
-	}
-
-	return false;
-}
-
-void __hci_req_update_scan(struct hci_request *req)
-{
-	struct hci_dev *hdev = req->hdev;
-	u8 scan;
-
-	if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED))
-		return;
-
-	if (!hdev_is_powered(hdev))
-		return;
-
-	if (mgmt_powering_down(hdev))
-		return;
-
-	if (hdev->scanning_paused)
-		return;
-
-	if (hci_dev_test_flag(hdev, HCI_CONNECTABLE) ||
-	    disconnected_accept_list_entries(hdev))
-		scan = SCAN_PAGE;
-	else
-		scan = SCAN_DISABLED;
-
-	if (hci_dev_test_flag(hdev, HCI_DISCOVERABLE))
-		scan |= SCAN_INQUIRY;
-
-	if (test_bit(HCI_PSCAN, &hdev->flags) == !!(scan & SCAN_PAGE) &&
-	    test_bit(HCI_ISCAN, &hdev->flags) == !!(scan & SCAN_INQUIRY))
-		return;
-
-	hci_req_add(req, HCI_OP_WRITE_SCAN_ENABLE, 1, &scan);
-}
-
-static u8 get_service_classes(struct hci_dev *hdev)
-{
-	struct bt_uuid *uuid;
-	u8 val = 0;
-
-	list_for_each_entry(uuid, &hdev->uuids, list)
-		val |= uuid->svc_hint;
-
-	return val;
-}
-
-void __hci_req_update_class(struct hci_request *req)
-{
-	struct hci_dev *hdev = req->hdev;
-	u8 cod[3];
-
-	bt_dev_dbg(hdev, "");
-
-	if (!hdev_is_powered(hdev))
-		return;
-
-	if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED))
-		return;
-
-	if (hci_dev_test_flag(hdev, HCI_SERVICE_CACHE))
-		return;
-
-	cod[0] = hdev->minor_class;
-	cod[1] = hdev->major_class;
-	cod[2] = get_service_classes(hdev);
-
-	if (hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE))
-		cod[1] |= 0x20;
-
-	if (memcmp(cod, hdev->dev_class, 3) == 0)
-		return;
-
-	hci_req_add(req, HCI_OP_WRITE_CLASS_OF_DEV, sizeof(cod), cod);
-}
-
-void __hci_abort_conn(struct hci_request *req, struct hci_conn *conn,
-		      u8 reason)
-{
-	switch (conn->state) {
-	case BT_CONNECTED:
-	case BT_CONFIG:
-		if (conn->type == AMP_LINK) {
-			struct hci_cp_disconn_phy_link cp;
-
-			cp.phy_handle = HCI_PHY_HANDLE(conn->handle);
-			cp.reason = reason;
-			hci_req_add(req, HCI_OP_DISCONN_PHY_LINK, sizeof(cp),
-				    &cp);
-		} else {
-			struct hci_cp_disconnect dc;
-
-			dc.handle = cpu_to_le16(conn->handle);
-			dc.reason = reason;
-			hci_req_add(req, HCI_OP_DISCONNECT, sizeof(dc), &dc);
-		}
-
-		conn->state = BT_DISCONN;
-
-		break;
-	case BT_CONNECT:
-		if (conn->type == LE_LINK) {
-			if (test_bit(HCI_CONN_SCANNING, &conn->flags))
-				break;
-			hci_req_add(req, HCI_OP_LE_CREATE_CONN_CANCEL,
-				    0, NULL);
-		} else if (conn->type == ACL_LINK) {
-			if (req->hdev->hci_ver < BLUETOOTH_VER_1_2)
-				break;
-			hci_req_add(req, HCI_OP_CREATE_CONN_CANCEL,
-				    6, &conn->dst);
-		}
-		break;
-	case BT_CONNECT2:
-		if (conn->type == ACL_LINK) {
-			struct hci_cp_reject_conn_req rej;
-
-			bacpy(&rej.bdaddr, &conn->dst);
-			rej.reason = reason;
-
-			hci_req_add(req, HCI_OP_REJECT_CONN_REQ,
-				    sizeof(rej), &rej);
-		} else if (conn->type == SCO_LINK || conn->type == ESCO_LINK) {
-			struct hci_cp_reject_sync_conn_req rej;
-
-			bacpy(&rej.bdaddr, &conn->dst);
-
-			/* SCO rejection has its own limited set of
-			 * allowed error values (0x0D-0x0F) which isn't
-			 * compatible with most values passed to this
-			 * function. To be safe hard-code one of the
-			 * values that's suitable for SCO.
-			 */
-			rej.reason = HCI_ERROR_REJ_LIMITED_RESOURCES;
-
-			hci_req_add(req, HCI_OP_REJECT_SYNC_CONN_REQ,
-				    sizeof(rej), &rej);
-		}
-		break;
-	default:
-		conn->state = BT_CLOSED;
-		break;
-	}
-}
-
-static void abort_conn_complete(struct hci_dev *hdev, u8 status, u16 opcode)
-{
-	if (status)
-		bt_dev_dbg(hdev, "Failed to abort connection: status 0x%2.2x", status);
-}
-
-int hci_abort_conn(struct hci_conn *conn, u8 reason)
-{
-	struct hci_request req;
-	int err;
-
-	hci_req_init(&req, conn->hdev);
-
-	__hci_abort_conn(&req, conn, reason);
-
-	err = hci_req_run(&req, abort_conn_complete);
-	if (err && err != -ENODATA) {
-		bt_dev_err(conn->hdev, "failed to run HCI request: err %d", err);
-		return err;
-	}
-
-	return 0;
-}
-
-static int le_scan_disable(struct hci_request *req, unsigned long opt)
-{
-	hci_req_add_le_scan_disable(req, false);
-	return 0;
-}
-
-static int bredr_inquiry(struct hci_request *req, unsigned long opt)
-{
-	u8 length = opt;
-	const u8 giac[3] = { 0x33, 0x8b, 0x9e };
-	const u8 liac[3] = { 0x00, 0x8b, 0x9e };
-	struct hci_cp_inquiry cp;
-
-	if (test_bit(HCI_INQUIRY, &req->hdev->flags))
-		return 0;
-
-	bt_dev_dbg(req->hdev, "");
-
-	hci_dev_lock(req->hdev);
-	hci_inquiry_cache_flush(req->hdev);
-	hci_dev_unlock(req->hdev);
-
-	memset(&cp, 0, sizeof(cp));
-
-	if (req->hdev->discovery.limited)
-		memcpy(&cp.lap, liac, sizeof(cp.lap));
-	else
-		memcpy(&cp.lap, giac, sizeof(cp.lap));
-
-	cp.length = length;
-
-	hci_req_add(req, HCI_OP_INQUIRY, sizeof(cp), &cp);
-
-	return 0;
-}
-
-static void le_scan_disable_work(struct work_struct *work)
-{
-	struct hci_dev *hdev = container_of(work, struct hci_dev,
-					    le_scan_disable.work);
-	u8 status;
-
-	bt_dev_dbg(hdev, "");
-
-	if (!hci_dev_test_flag(hdev, HCI_LE_SCAN))
-		return;
-
-	cancel_delayed_work(&hdev->le_scan_restart);
-
-	hci_req_sync(hdev, le_scan_disable, 0, HCI_CMD_TIMEOUT, &status);
-	if (status) {
-		bt_dev_err(hdev, "failed to disable LE scan: status 0x%02x",
-			   status);
-		return;
-	}
-
-	hdev->discovery.scan_start = 0;
-
-	/* If we were running LE only scan, change discovery state. If
-	 * we were running both LE and BR/EDR inquiry simultaneously,
-	 * and BR/EDR inquiry is already finished, stop discovery,
-	 * otherwise BR/EDR inquiry will stop discovery when finished.
-	 * If we will resolve remote device name, do not change
-	 * discovery state.
-	 */
-
-	if (hdev->discovery.type == DISCOV_TYPE_LE)
-		goto discov_stopped;
-
-	if (hdev->discovery.type != DISCOV_TYPE_INTERLEAVED)
-		return;
-
-	if (test_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks)) {
-		if (!test_bit(HCI_INQUIRY, &hdev->flags) &&
-		    hdev->discovery.state != DISCOVERY_RESOLVING)
-			goto discov_stopped;
-
-		return;
-	}
-
-	hci_req_sync(hdev, bredr_inquiry, DISCOV_INTERLEAVED_INQUIRY_LEN,
-		     HCI_CMD_TIMEOUT, &status);
-	if (status) {
-		bt_dev_err(hdev, "inquiry failed: status 0x%02x", status);
-		goto discov_stopped;
-	}
-
-	return;
-
-discov_stopped:
-	hci_dev_lock(hdev);
-	hci_discovery_set_state(hdev, DISCOVERY_STOPPED);
-	hci_dev_unlock(hdev);
-}
-
-static int le_scan_restart(struct hci_request *req, unsigned long opt)
-{
-	struct hci_dev *hdev = req->hdev;
-
-	/* If controller is not scanning we are done. */
-	if (!hci_dev_test_flag(hdev, HCI_LE_SCAN))
-		return 0;
-
-	if (hdev->scanning_paused) {
-		bt_dev_dbg(hdev, "Scanning is paused for suspend");
-		return 0;
-	}
-
-	hci_req_add_le_scan_disable(req, false);
-
-	if (use_ext_scan(hdev)) {
-		struct hci_cp_le_set_ext_scan_enable ext_enable_cp;
-
-		memset(&ext_enable_cp, 0, sizeof(ext_enable_cp));
-		ext_enable_cp.enable = LE_SCAN_ENABLE;
-		ext_enable_cp.filter_dup = LE_SCAN_FILTER_DUP_ENABLE;
-
-		hci_req_add(req, HCI_OP_LE_SET_EXT_SCAN_ENABLE,
-			    sizeof(ext_enable_cp), &ext_enable_cp);
-	} else {
-		struct hci_cp_le_set_scan_enable cp;
-
-		memset(&cp, 0, sizeof(cp));
-		cp.enable = LE_SCAN_ENABLE;
-		cp.filter_dup = LE_SCAN_FILTER_DUP_ENABLE;
-		hci_req_add(req, HCI_OP_LE_SET_SCAN_ENABLE, sizeof(cp), &cp);
-	}
-
-	return 0;
-}
-
-static void le_scan_restart_work(struct work_struct *work)
-{
-	struct hci_dev *hdev = container_of(work, struct hci_dev,
-					    le_scan_restart.work);
-	unsigned long timeout, duration, scan_start, now;
-	u8 status;
-
-	bt_dev_dbg(hdev, "");
-
-	hci_req_sync(hdev, le_scan_restart, 0, HCI_CMD_TIMEOUT, &status);
-	if (status) {
-		bt_dev_err(hdev, "failed to restart LE scan: status %d",
-			   status);
-		return;
-	}
-
-	hci_dev_lock(hdev);
-
-	if (!test_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks) ||
-	    !hdev->discovery.scan_start)
-		goto unlock;
-
-	/* When the scan was started, hdev->le_scan_disable has been queued
-	 * after duration from scan_start. During scan restart this job
-	 * has been canceled, and we need to queue it again after proper
-	 * timeout, to make sure that scan does not run indefinitely.
-	 */
-	duration = hdev->discovery.scan_duration;
-	scan_start = hdev->discovery.scan_start;
-	now = jiffies;
-	if (now - scan_start <= duration) {
-		int elapsed;
-
-		if (now >= scan_start)
-			elapsed = now - scan_start;
-		else
-			elapsed = ULONG_MAX - scan_start + now;
-
-		timeout = duration - elapsed;
-	} else {
-		timeout = 0;
-	}
-
-	queue_delayed_work(hdev->req_workqueue,
-			   &hdev->le_scan_disable, timeout);
-
-unlock:
-	hci_dev_unlock(hdev);
-}
-
-bool hci_req_stop_discovery(struct hci_request *req)
-{
-	struct hci_dev *hdev = req->hdev;
-	struct discovery_state *d = &hdev->discovery;
-	struct hci_cp_remote_name_req_cancel cp;
-	struct inquiry_entry *e;
-	bool ret = false;
-
-	bt_dev_dbg(hdev, "state %u", hdev->discovery.state);
-
-	if (d->state == DISCOVERY_FINDING || d->state == DISCOVERY_STOPPING) {
-		if (test_bit(HCI_INQUIRY, &hdev->flags))
-			hci_req_add(req, HCI_OP_INQUIRY_CANCEL, 0, NULL);
-
-		if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) {
-			cancel_delayed_work(&hdev->le_scan_disable);
-			cancel_delayed_work(&hdev->le_scan_restart);
-			hci_req_add_le_scan_disable(req, false);
-		}
-
-		ret = true;
-	} else {
-		/* Passive scanning */
-		if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) {
-			hci_req_add_le_scan_disable(req, false);
-			ret = true;
-		}
-	}
-
-	/* No further actions needed for LE-only discovery */
-	if (d->type == DISCOV_TYPE_LE)
-		return ret;
-
-	if (d->state == DISCOVERY_RESOLVING || d->state == DISCOVERY_STOPPING) {
-		e = hci_inquiry_cache_lookup_resolve(hdev, BDADDR_ANY,
-						     NAME_PENDING);
-		if (!e)
-			return ret;
-
-		bacpy(&cp.bdaddr, &e->data.bdaddr);
-		hci_req_add(req, HCI_OP_REMOTE_NAME_REQ_CANCEL, sizeof(cp),
-			    &cp);
-		ret = true;
-	}
-
-	return ret;
-}
-
-static void config_data_path_complete(struct hci_dev *hdev, u8 status,
-				      u16 opcode)
-{
-	bt_dev_dbg(hdev, "status %u", status);
-}
-
-int hci_req_configure_datapath(struct hci_dev *hdev, struct bt_codec *codec)
-{
-	struct hci_request req;
-	int err;
-	__u8 vnd_len, *vnd_data = NULL;
-	struct hci_op_configure_data_path *cmd = NULL;
-
-	hci_req_init(&req, hdev);
-
-	err = hdev->get_codec_config_data(hdev, ESCO_LINK, codec, &vnd_len,
-					  &vnd_data);
-	if (err < 0)
-		goto error;
-
-	cmd = kzalloc(sizeof(*cmd) + vnd_len, GFP_KERNEL);
-	if (!cmd) {
-		err = -ENOMEM;
-		goto error;
-	}
-
-	err = hdev->get_data_path_id(hdev, &cmd->data_path_id);
-	if (err < 0)
-		goto error;
-
-	cmd->vnd_len = vnd_len;
-	memcpy(cmd->vnd_data, vnd_data, vnd_len);
-
-	cmd->direction = 0x00;
-	hci_req_add(&req, HCI_CONFIGURE_DATA_PATH, sizeof(*cmd) + vnd_len, cmd);
-
-	cmd->direction = 0x01;
-	hci_req_add(&req, HCI_CONFIGURE_DATA_PATH, sizeof(*cmd) + vnd_len, cmd);
-
-	err = hci_req_run(&req, config_data_path_complete);
-error:
-
-	kfree(cmd);
-	kfree(vnd_data);
-	return err;
-}
-
 void hci_request_setup(struct hci_dev *hdev)
 {
-	INIT_DELAYED_WORK(&hdev->le_scan_disable, le_scan_disable_work);
-	INIT_DELAYED_WORK(&hdev->le_scan_restart, le_scan_restart_work);
-	INIT_DELAYED_WORK(&hdev->adv_instance_expire, adv_timeout_expire);
 	INIT_DELAYED_WORK(&hdev->interleave_scan, interleave_scan_work);
 }
 
@@ -2262,13 +918,5 @@ void hci_request_cancel_all(struct hci_dev *hdev)
 {
 	__hci_cmd_sync_cancel(hdev, ENODEV);
 
-	cancel_delayed_work_sync(&hdev->le_scan_disable);
-	cancel_delayed_work_sync(&hdev->le_scan_restart);
-
-	if (hdev->adv_instance_timeout) {
-		cancel_delayed_work_sync(&hdev->adv_instance_expire);
-		hdev->adv_instance_timeout = 0;
-	}
-
 	cancel_interleave_scan(hdev);
 }
diff --git a/net/bluetooth/hci_request.h b/net/bluetooth/hci_request.h
index 39d001fa3acf..b9c5a9823837 100644
--- a/net/bluetooth/hci_request.h
+++ b/net/bluetooth/hci_request.h
@@ -68,63 +68,10 @@ int __hci_req_sync(struct hci_dev *hdev, int (*func)(struct hci_request *req,
 struct sk_buff *hci_prepare_cmd(struct hci_dev *hdev, u16 opcode, u32 plen,
 				const void *param);
 
-void __hci_req_write_fast_connectable(struct hci_request *req, bool enable);
-void __hci_req_update_name(struct hci_request *req);
-void __hci_req_update_eir(struct hci_request *req);
-
 void hci_req_add_le_scan_disable(struct hci_request *req, bool rpa_le_conn);
 void hci_req_add_le_passive_scan(struct hci_request *req);
 
 void hci_req_prepare_suspend(struct hci_dev *hdev, enum suspended_state next);
 
-void hci_req_disable_address_resolution(struct hci_dev *hdev);
-void hci_req_reenable_advertising(struct hci_dev *hdev);
-void __hci_req_enable_advertising(struct hci_request *req);
-void __hci_req_disable_advertising(struct hci_request *req);
-void __hci_req_update_adv_data(struct hci_request *req, u8 instance);
-int hci_req_update_adv_data(struct hci_dev *hdev, u8 instance);
-int hci_req_start_per_adv(struct hci_dev *hdev, u8 instance, u32 flags,
-			  u16 min_interval, u16 max_interval,
-			  u16 sync_interval);
-void __hci_req_update_scan_rsp_data(struct hci_request *req, u8 instance);
-
-int __hci_req_schedule_adv_instance(struct hci_request *req, u8 instance,
-				    bool force);
-void hci_req_clear_adv_instance(struct hci_dev *hdev, struct sock *sk,
-				struct hci_request *req, u8 instance,
-				bool force);
-
-int __hci_req_setup_ext_adv_instance(struct hci_request *req, u8 instance);
-int __hci_req_setup_per_adv_instance(struct hci_request *req, u8 instance,
-				     u16 min_interval, u16 max_interval);
-int __hci_req_start_ext_adv(struct hci_request *req, u8 instance);
-int __hci_req_start_per_adv(struct hci_request *req, u8 instance, u32 flags,
-			    u16 min_interval, u16 max_interval,
-			    u16 sync_interval);
-int __hci_req_enable_ext_advertising(struct hci_request *req, u8 instance);
-int __hci_req_enable_per_advertising(struct hci_request *req, u8 instance);
-int __hci_req_disable_ext_adv_instance(struct hci_request *req, u8 instance);
-int __hci_req_remove_ext_adv_instance(struct hci_request *req, u8 instance);
-void __hci_req_clear_ext_adv_sets(struct hci_request *req);
-int hci_get_random_address(struct hci_dev *hdev, bool require_privacy,
-			   bool use_rpa, struct adv_info *adv_instance,
-			   u8 *own_addr_type, bdaddr_t *rand_addr);
-
-void __hci_req_update_class(struct hci_request *req);
-
-/* Returns true if HCI commands were queued */
-bool hci_req_stop_discovery(struct hci_request *req);
-
-int hci_req_configure_datapath(struct hci_dev *hdev, struct bt_codec *codec);
-
-void __hci_req_update_scan(struct hci_request *req);
-
-int hci_update_random_address(struct hci_request *req, bool require_privacy,
-			      bool use_rpa, u8 *own_addr_type);
-
-int hci_abort_conn(struct hci_conn *conn, u8 reason);
-void __hci_abort_conn(struct hci_request *req, struct hci_conn *conn,
-		      u8 reason);
-
 void hci_request_setup(struct hci_dev *hdev);
 void hci_request_cancel_all(struct hci_dev *hdev);
diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c
index 0d015d4a8e41..06581223238c 100644
--- a/net/bluetooth/hci_sock.c
+++ b/net/bluetooth/hci_sock.c
@@ -887,7 +887,6 @@ static int hci_sock_release(struct socket *sock)
 			 */
 			hci_dev_do_close(hdev);
 			hci_dev_clear_flag(hdev, HCI_USER_CHANNEL);
-			hci_register_suspend_notifier(hdev);
 			mgmt_index_added(hdev);
 		}
 
@@ -1216,7 +1215,6 @@ static int hci_sock_bind(struct socket *sock, struct sockaddr *addr,
 		}
 
 		mgmt_index_removed(hdev);
-		hci_unregister_suspend_notifier(hdev);
 
 		err = hci_dev_open(hdev->id);
 		if (err) {
@@ -1231,7 +1229,6 @@ static int hci_sock_bind(struct socket *sock, struct sockaddr *addr,
 				err = 0;
 			} else {
 				hci_dev_clear_flag(hdev, HCI_USER_CHANNEL);
-				hci_register_suspend_notifier(hdev);
 				mgmt_index_added(hdev);
 				hci_dev_put(hdev);
 				goto done;
@@ -2065,6 +2062,7 @@ static int hci_sock_getsockopt(struct socket *sock, int level, int optname,
 
 static void hci_sock_destruct(struct sock *sk)
 {
+	mgmt_cleanup(sk);
 	skb_queue_purge(&sk->sk_receive_queue);
 	skb_queue_purge(&sk->sk_write_queue);
 }
diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c
index fbd5613eebfc..76c3107c9f91 100644
--- a/net/bluetooth/hci_sync.c
+++ b/net/bluetooth/hci_sync.c
@@ -246,7 +246,7 @@ int __hci_cmd_sync_status_sk(struct hci_dev *hdev, u16 opcode, u32 plen,
 	skb = __hci_cmd_sync_sk(hdev, opcode, plen, param, event, timeout, sk);
 	if (IS_ERR(skb)) {
 		bt_dev_err(hdev, "Opcode 0x%4x failed: %ld", opcode,
-			   PTR_ERR(skb));
+				PTR_ERR(skb));
 		return PTR_ERR(skb);
 	}
 
@@ -321,6 +321,307 @@ static void hci_cmd_sync_cancel_work(struct work_struct *work)
 	wake_up_interruptible(&hdev->req_wait_q);
 }
 
+static int hci_scan_disable_sync(struct hci_dev *hdev);
+static int scan_disable_sync(struct hci_dev *hdev, void *data)
+{
+	return hci_scan_disable_sync(hdev);
+}
+
+static int hci_inquiry_sync(struct hci_dev *hdev, u8 length);
+static int interleaved_inquiry_sync(struct hci_dev *hdev, void *data)
+{
+	return hci_inquiry_sync(hdev, DISCOV_INTERLEAVED_INQUIRY_LEN);
+}
+
+static void le_scan_disable(struct work_struct *work)
+{
+	struct hci_dev *hdev = container_of(work, struct hci_dev,
+					    le_scan_disable.work);
+	int status;
+
+	bt_dev_dbg(hdev, "");
+	hci_dev_lock(hdev);
+
+	if (!hci_dev_test_flag(hdev, HCI_LE_SCAN))
+		goto _return;
+
+	cancel_delayed_work(&hdev->le_scan_restart);
+
+	status = hci_cmd_sync_queue(hdev, scan_disable_sync, NULL, NULL);
+	if (status) {
+		bt_dev_err(hdev, "failed to disable LE scan: %d", status);
+		goto _return;
+	}
+
+	hdev->discovery.scan_start = 0;
+
+	/* If we were running LE only scan, change discovery state. If
+	 * we were running both LE and BR/EDR inquiry simultaneously,
+	 * and BR/EDR inquiry is already finished, stop discovery,
+	 * otherwise BR/EDR inquiry will stop discovery when finished.
+	 * If we will resolve remote device name, do not change
+	 * discovery state.
+	 */
+
+	if (hdev->discovery.type == DISCOV_TYPE_LE)
+		goto discov_stopped;
+
+	if (hdev->discovery.type != DISCOV_TYPE_INTERLEAVED)
+		goto _return;
+
+	if (test_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks)) {
+		if (!test_bit(HCI_INQUIRY, &hdev->flags) &&
+		    hdev->discovery.state != DISCOVERY_RESOLVING)
+			goto discov_stopped;
+
+		goto _return;
+	}
+
+	status = hci_cmd_sync_queue(hdev, interleaved_inquiry_sync, NULL, NULL);
+	if (status) {
+		bt_dev_err(hdev, "inquiry failed: status %d", status);
+		goto discov_stopped;
+	}
+
+	goto _return;
+
+discov_stopped:
+	hci_discovery_set_state(hdev, DISCOVERY_STOPPED);
+
+_return:
+	hci_dev_unlock(hdev);
+}
+
+static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val,
+				       u8 filter_dup);
+static int hci_le_scan_restart_sync(struct hci_dev *hdev)
+{
+	/* If controller is not scanning we are done. */
+	if (!hci_dev_test_flag(hdev, HCI_LE_SCAN))
+		return 0;
+
+	if (hdev->scanning_paused) {
+		bt_dev_dbg(hdev, "Scanning is paused for suspend");
+		return 0;
+	}
+
+	hci_le_set_scan_enable_sync(hdev, LE_SCAN_DISABLE, 0x00);
+	return hci_le_set_scan_enable_sync(hdev, LE_SCAN_ENABLE,
+					   LE_SCAN_FILTER_DUP_ENABLE);
+}
+
+static int le_scan_restart_sync(struct hci_dev *hdev, void *data)
+{
+	return hci_le_scan_restart_sync(hdev);
+}
+
+static void le_scan_restart(struct work_struct *work)
+{
+	struct hci_dev *hdev = container_of(work, struct hci_dev,
+					    le_scan_restart.work);
+	unsigned long timeout, duration, scan_start, now;
+	int status;
+
+	bt_dev_dbg(hdev, "");
+
+	hci_dev_lock(hdev);
+
+	status = hci_cmd_sync_queue(hdev, le_scan_restart_sync, NULL, NULL);
+	if (status) {
+		bt_dev_err(hdev, "failed to restart LE scan: status %d",
+			   status);
+		goto unlock;
+	}
+
+	if (!test_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks) ||
+	    !hdev->discovery.scan_start)
+		goto unlock;
+
+	/* When the scan was started, hdev->le_scan_disable has been queued
+	 * after duration from scan_start. During scan restart this job
+	 * has been canceled, and we need to queue it again after proper
+	 * timeout, to make sure that scan does not run indefinitely.
+	 */
+	duration = hdev->discovery.scan_duration;
+	scan_start = hdev->discovery.scan_start;
+	now = jiffies;
+	if (now - scan_start <= duration) {
+		int elapsed;
+
+		if (now >= scan_start)
+			elapsed = now - scan_start;
+		else
+			elapsed = ULONG_MAX - scan_start + now;
+
+		timeout = duration - elapsed;
+	} else {
+		timeout = 0;
+	}
+
+	queue_delayed_work(hdev->req_workqueue,
+			   &hdev->le_scan_disable, timeout);
+
+unlock:
+	hci_dev_unlock(hdev);
+}
+
+static int reenable_adv_sync(struct hci_dev *hdev, void *data)
+{
+	bt_dev_dbg(hdev, "");
+
+	if (!hci_dev_test_flag(hdev, HCI_ADVERTISING) &&
+	    list_empty(&hdev->adv_instances))
+		return 0;
+
+	if (hdev->cur_adv_instance) {
+		return hci_schedule_adv_instance_sync(hdev,
+						      hdev->cur_adv_instance,
+						      true);
+	} else {
+		if (ext_adv_capable(hdev)) {
+			hci_start_ext_adv_sync(hdev, 0x00);
+		} else {
+			hci_update_adv_data_sync(hdev, 0x00);
+			hci_update_scan_rsp_data_sync(hdev, 0x00);
+			hci_enable_advertising_sync(hdev);
+		}
+	}
+
+	return 0;
+}
+
+static void reenable_adv(struct work_struct *work)
+{
+	struct hci_dev *hdev = container_of(work, struct hci_dev,
+					    reenable_adv_work);
+	int status;
+
+	bt_dev_dbg(hdev, "");
+
+	hci_dev_lock(hdev);
+
+	status = hci_cmd_sync_queue(hdev, reenable_adv_sync, NULL, NULL);
+	if (status)
+		bt_dev_err(hdev, "failed to reenable ADV: %d", status);
+
+	hci_dev_unlock(hdev);
+}
+
+static void cancel_adv_timeout(struct hci_dev *hdev)
+{
+	if (hdev->adv_instance_timeout) {
+		hdev->adv_instance_timeout = 0;
+		cancel_delayed_work(&hdev->adv_instance_expire);
+	}
+}
+
+/* For a single instance:
+ * - force == true: The instance will be removed even when its remaining
+ *   lifetime is not zero.
+ * - force == false: the instance will be deactivated but kept stored unless
+ *   the remaining lifetime is zero.
+ *
+ * For instance == 0x00:
+ * - force == true: All instances will be removed regardless of their timeout
+ *   setting.
+ * - force == false: Only instances that have a timeout will be removed.
+ */
+int hci_clear_adv_instance_sync(struct hci_dev *hdev, struct sock *sk,
+				u8 instance, bool force)
+{
+	struct adv_info *adv_instance, *n, *next_instance = NULL;
+	int err;
+	u8 rem_inst;
+
+	/* Cancel any timeout concerning the removed instance(s). */
+	if (!instance || hdev->cur_adv_instance == instance)
+		cancel_adv_timeout(hdev);
+
+	/* Get the next instance to advertise BEFORE we remove
+	 * the current one. This can be the same instance again
+	 * if there is only one instance.
+	 */
+	if (instance && hdev->cur_adv_instance == instance)
+		next_instance = hci_get_next_instance(hdev, instance);
+
+	if (instance == 0x00) {
+		list_for_each_entry_safe(adv_instance, n, &hdev->adv_instances,
+					 list) {
+			if (!(force || adv_instance->timeout))
+				continue;
+
+			rem_inst = adv_instance->instance;
+			err = hci_remove_adv_instance(hdev, rem_inst);
+			if (!err)
+				mgmt_advertising_removed(sk, hdev, rem_inst);
+		}
+	} else {
+		adv_instance = hci_find_adv_instance(hdev, instance);
+
+		if (force || (adv_instance && adv_instance->timeout &&
+			      !adv_instance->remaining_time)) {
+			/* Don't advertise a removed instance. */
+			if (next_instance &&
+			    next_instance->instance == instance)
+				next_instance = NULL;
+
+			err = hci_remove_adv_instance(hdev, instance);
+			if (!err)
+				mgmt_advertising_removed(sk, hdev, instance);
+		}
+	}
+
+	if (!hdev_is_powered(hdev) || hci_dev_test_flag(hdev, HCI_ADVERTISING))
+		return 0;
+
+	if (next_instance && !ext_adv_capable(hdev))
+		return hci_schedule_adv_instance_sync(hdev,
+						      next_instance->instance,
+						      false);
+
+	return 0;
+}
+
+static int adv_timeout_expire_sync(struct hci_dev *hdev, void *data)
+{
+	u8 instance = *(u8 *)data;
+
+	kfree(data);
+
+	hci_clear_adv_instance_sync(hdev, NULL, instance, false);
+
+	if (list_empty(&hdev->adv_instances))
+		return hci_disable_advertising_sync(hdev);
+
+	return 0;
+}
+
+static void adv_timeout_expire(struct work_struct *work)
+{
+	u8 *inst_ptr;
+	struct hci_dev *hdev = container_of(work, struct hci_dev,
+					    adv_instance_expire.work);
+
+	bt_dev_dbg(hdev, "");
+
+	hci_dev_lock(hdev);
+
+	hdev->adv_instance_timeout = 0;
+
+	if (hdev->cur_adv_instance == 0x00)
+		goto unlock;
+
+	inst_ptr = kmalloc(1, GFP_KERNEL);
+	if (!inst_ptr)
+		goto unlock;
+
+	*inst_ptr = hdev->cur_adv_instance;
+	hci_cmd_sync_queue(hdev, adv_timeout_expire_sync, inst_ptr, NULL);
+
+unlock:
+	hci_dev_unlock(hdev);
+}
+
 void hci_cmd_sync_init(struct hci_dev *hdev)
 {
 	INIT_WORK(&hdev->cmd_sync_work, hci_cmd_sync_work);
@@ -328,6 +629,10 @@ void hci_cmd_sync_init(struct hci_dev *hdev)
 	mutex_init(&hdev->cmd_sync_work_lock);
 
 	INIT_WORK(&hdev->cmd_sync_cancel_work, hci_cmd_sync_cancel_work);
+	INIT_WORK(&hdev->reenable_adv_work, reenable_adv);
+	INIT_DELAYED_WORK(&hdev->le_scan_disable, le_scan_disable);
+	INIT_DELAYED_WORK(&hdev->le_scan_restart, le_scan_restart);
+	INIT_DELAYED_WORK(&hdev->adv_instance_expire, adv_timeout_expire);
 }
 
 void hci_cmd_sync_clear(struct hci_dev *hdev)
@@ -335,6 +640,7 @@ void hci_cmd_sync_clear(struct hci_dev *hdev)
 	struct hci_cmd_sync_work_entry *entry, *tmp;
 
 	cancel_work_sync(&hdev->cmd_sync_work);
+	cancel_work_sync(&hdev->reenable_adv_work);
 
 	list_for_each_entry_safe(entry, tmp, &hdev->cmd_sync_work_list, list) {
 		if (entry->destroy)
@@ -1333,14 +1639,6 @@ int hci_le_terminate_big_sync(struct hci_dev *hdev, u8 handle, u8 reason)
 				     sizeof(cp), &cp, HCI_CMD_TIMEOUT);
 }
 
-static void cancel_adv_timeout(struct hci_dev *hdev)
-{
-	if (hdev->adv_instance_timeout) {
-		hdev->adv_instance_timeout = 0;
-		cancel_delayed_work(&hdev->adv_instance_expire);
-	}
-}
-
 static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance)
 {
 	struct {
@@ -1492,10 +1790,13 @@ static int hci_clear_adv_sets_sync(struct hci_dev *hdev, struct sock *sk)
 static int hci_clear_adv_sync(struct hci_dev *hdev, struct sock *sk, bool force)
 {
 	struct adv_info *adv, *n;
+	int err = 0;
 
 	if (ext_adv_capable(hdev))
 		/* Remove all existing sets */
-		return hci_clear_adv_sets_sync(hdev, sk);
+		err = hci_clear_adv_sets_sync(hdev, sk);
+	if (ext_adv_capable(hdev))
+		return err;
 
 	/* This is safe as long as there is no command send while the lock is
 	 * held.
@@ -1523,11 +1824,13 @@ static int hci_clear_adv_sync(struct hci_dev *hdev, struct sock *sk, bool force)
 static int hci_remove_adv_sync(struct hci_dev *hdev, u8 instance,
 			       struct sock *sk)
 {
-	int err;
+	int err = 0;
 
 	/* If we use extended advertising, instance has to be removed first. */
 	if (ext_adv_capable(hdev))
-		return hci_remove_ext_adv_instance_sync(hdev, instance, sk);
+		err = hci_remove_ext_adv_instance_sync(hdev, instance, sk);
+	if (ext_adv_capable(hdev))
+		return err;
 
 	/* This is safe as long as there is no command send while the lock is
 	 * held.
@@ -1626,13 +1929,16 @@ int hci_read_tx_power_sync(struct hci_dev *hdev, __le16 handle, u8 type)
 int hci_disable_advertising_sync(struct hci_dev *hdev)
 {
 	u8 enable = 0x00;
+	int err = 0;
 
 	/* If controller is not advertising we are done. */
 	if (!hci_dev_test_flag(hdev, HCI_LE_ADV))
 		return 0;
 
 	if (ext_adv_capable(hdev))
-		return hci_disable_ext_adv_instance_sync(hdev, 0x00);
+		err = hci_disable_ext_adv_instance_sync(hdev, 0x00);
+	if (ext_adv_capable(hdev))
+		return err;
 
 	return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_ENABLE,
 				     sizeof(enable), &enable, HCI_CMD_TIMEOUT);
@@ -1645,7 +1951,11 @@ static int hci_le_set_ext_scan_enable_sync(struct hci_dev *hdev, u8 val,
 
 	memset(&cp, 0, sizeof(cp));
 	cp.enable = val;
-	cp.filter_dup = filter_dup;
+
+	if (hci_dev_test_flag(hdev, HCI_MESH))
+		cp.filter_dup = LE_SCAN_FILTER_DUP_DISABLE;
+	else
+		cp.filter_dup = filter_dup;
 
 	return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_ENABLE,
 				     sizeof(cp), &cp, HCI_CMD_TIMEOUT);
@@ -1661,7 +1971,11 @@ static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val,
 
 	memset(&cp, 0, sizeof(cp));
 	cp.enable = val;
-	cp.filter_dup = filter_dup;
+
+	if (val && hci_dev_test_flag(hdev, HCI_MESH))
+		cp.filter_dup = LE_SCAN_FILTER_DUP_DISABLE;
+	else
+		cp.filter_dup = filter_dup;
 
 	return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_SCAN_ENABLE,
 				     sizeof(cp), &cp, HCI_CMD_TIMEOUT);
@@ -2300,6 +2614,7 @@ static int hci_passive_scan_sync(struct hci_dev *hdev)
 	u8 own_addr_type;
 	u8 filter_policy;
 	u16 window, interval;
+	u8 filter_dups = LE_SCAN_FILTER_DUP_ENABLE;
 	int err;
 
 	if (hdev->scanning_paused) {
@@ -2362,11 +2677,16 @@ static int hci_passive_scan_sync(struct hci_dev *hdev)
 		interval = hdev->le_scan_interval;
 	}
 
+	/* Disable all filtering for Mesh */
+	if (hci_dev_test_flag(hdev, HCI_MESH)) {
+		filter_policy = 0;
+		filter_dups = LE_SCAN_FILTER_DUP_DISABLE;
+	}
+
 	bt_dev_dbg(hdev, "LE passive scan with acceptlist = %d", filter_policy);
 
 	return hci_start_scan_sync(hdev, LE_SCAN_PASSIVE, interval, window,
-				   own_addr_type, filter_policy,
-				   LE_SCAN_FILTER_DUP_ENABLE);
+				   own_addr_type, filter_policy, filter_dups);
 }
 
 /* This function controls the passive scanning based on hdev->pend_le_conns
@@ -2416,7 +2736,8 @@ int hci_update_passive_scan_sync(struct hci_dev *hdev)
 	bt_dev_dbg(hdev, "ADV monitoring is %s",
 		   hci_is_adv_monitoring(hdev) ? "on" : "off");
 
-	if (list_empty(&hdev->pend_le_conns) &&
+	if (!hci_dev_test_flag(hdev, HCI_MESH) &&
+	    list_empty(&hdev->pend_le_conns) &&
 	    list_empty(&hdev->pend_le_reports) &&
 	    !hci_is_adv_monitoring(hdev) &&
 	    !hci_dev_test_flag(hdev, HCI_PA_SYNC)) {
@@ -4355,6 +4676,7 @@ int hci_dev_open_sync(struct hci_dev *hdev)
 		    hci_dev_test_flag(hdev, HCI_MGMT) &&
 		    hdev->dev_type == HCI_PRIMARY) {
 			ret = hci_powered_update_sync(hdev);
+			mgmt_power_on(hdev, ret);
 		}
 	} else {
 		/* Init failed, cleanup */
@@ -4406,6 +4728,31 @@ static void hci_pend_le_actions_clear(struct hci_dev *hdev)
 	BT_DBG("All LE pending actions cleared");
 }
 
+static int hci_dev_shutdown(struct hci_dev *hdev)
+{
+	int err = 0;
+	/* Similar to how we first do setup and then set the exclusive access
+	 * bit for userspace, we must first unset userchannel and then clean up.
+	 * Otherwise, the kernel can't properly use the hci channel to clean up
+	 * the controller (some shutdown routines require sending additional
+	 * commands to the controller for example).
+	 */
+	bool was_userchannel =
+		hci_dev_test_and_clear_flag(hdev, HCI_USER_CHANNEL);
+
+	if (!hci_dev_test_flag(hdev, HCI_UNREGISTER) &&
+	    test_bit(HCI_UP, &hdev->flags)) {
+		/* Execute vendor specific shutdown routine */
+		if (hdev->shutdown)
+			err = hdev->shutdown(hdev);
+	}
+
+	if (was_userchannel)
+		hci_dev_set_flag(hdev, HCI_USER_CHANNEL);
+
+	return err;
+}
+
 int hci_dev_close_sync(struct hci_dev *hdev)
 {
 	bool auto_off;
@@ -4415,17 +4762,18 @@ int hci_dev_close_sync(struct hci_dev *hdev)
 
 	cancel_delayed_work(&hdev->power_off);
 	cancel_delayed_work(&hdev->ncmd_timer);
+	cancel_delayed_work(&hdev->le_scan_disable);
+	cancel_delayed_work(&hdev->le_scan_restart);
 
 	hci_request_cancel_all(hdev);
 
-	if (!hci_dev_test_flag(hdev, HCI_UNREGISTER) &&
-	    !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) &&
-	    test_bit(HCI_UP, &hdev->flags)) {
-		/* Execute vendor specific shutdown routine */
-		if (hdev->shutdown)
-			err = hdev->shutdown(hdev);
+	if (hdev->adv_instance_timeout) {
+		cancel_delayed_work_sync(&hdev->adv_instance_expire);
+		hdev->adv_instance_timeout = 0;
 	}
 
+	err = hci_dev_shutdown(hdev);
+
 	if (!test_and_clear_bit(HCI_UP, &hdev->flags)) {
 		cancel_delayed_work_sync(&hdev->cmd_timer);
 		return err;
@@ -5023,7 +5371,7 @@ static int hci_active_scan_sync(struct hci_dev *hdev, uint16_t interval)
 	/* Pause advertising since active scanning disables address resolution
 	 * which advertising depend on in order to generate its RPAs.
 	 */
-	if (use_ll_privacy(hdev)) {
+	if (use_ll_privacy(hdev) && hci_dev_test_flag(hdev, HCI_PRIVACY)) {
 		err = hci_pause_advertising_sync(hdev);
 		if (err) {
 			bt_dev_err(hdev, "pause advertising failed: %d", err);
@@ -5737,3 +6085,96 @@ int hci_le_pa_terminate_sync(struct hci_dev *hdev, u16 handle)
 	return __hci_cmd_sync_status(hdev, HCI_OP_LE_PA_TERM_SYNC,
 				     sizeof(cp), &cp, HCI_CMD_TIMEOUT);
 }
+
+int hci_get_random_address(struct hci_dev *hdev, bool require_privacy,
+			   bool use_rpa, struct adv_info *adv_instance,
+			   u8 *own_addr_type, bdaddr_t *rand_addr)
+{
+	int err;
+
+	bacpy(rand_addr, BDADDR_ANY);
+
+	/* If privacy is enabled use a resolvable private address. If
+	 * current RPA has expired then generate a new one.
+	 */
+	if (use_rpa) {
+		/* If Controller supports LL Privacy use own address type is
+		 * 0x03
+		 */
+		if (use_ll_privacy(hdev))
+			*own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED;
+		else
+			*own_addr_type = ADDR_LE_DEV_RANDOM;
+
+		if (adv_instance) {
+			if (adv_rpa_valid(adv_instance))
+				return 0;
+		} else {
+			if (rpa_valid(hdev))
+				return 0;
+		}
+
+		err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa);
+		if (err < 0) {
+			bt_dev_err(hdev, "failed to generate new RPA");
+			return err;
+		}
+
+		bacpy(rand_addr, &hdev->rpa);
+
+		return 0;
+	}
+
+	/* In case of required privacy without resolvable private address,
+	 * use an non-resolvable private address. This is useful for
+	 * non-connectable advertising.
+	 */
+	if (require_privacy) {
+		bdaddr_t nrpa;
+
+		while (true) {
+			/* The non-resolvable private address is generated
+			 * from random six bytes with the two most significant
+			 * bits cleared.
+			 */
+			get_random_bytes(&nrpa, 6);
+			nrpa.b[5] &= 0x3f;
+
+			/* The non-resolvable private address shall not be
+			 * equal to the public address.
+			 */
+			if (bacmp(&hdev->bdaddr, &nrpa))
+				break;
+		}
+
+		*own_addr_type = ADDR_LE_DEV_RANDOM;
+		bacpy(rand_addr, &nrpa);
+
+		return 0;
+	}
+
+	/* No privacy so use a public address. */
+	*own_addr_type = ADDR_LE_DEV_PUBLIC;
+
+	return 0;
+}
+
+static int _update_adv_data_sync(struct hci_dev *hdev, void *data)
+{
+	u8 instance = *(u8 *)data;
+
+	kfree(data);
+
+	return hci_update_adv_data_sync(hdev, instance);
+}
+
+int hci_update_adv_data(struct hci_dev *hdev, u8 instance)
+{
+	u8 *inst_ptr = kmalloc(1, GFP_KERNEL);
+
+	if (!inst_ptr)
+		return -ENOMEM;
+
+	*inst_ptr = instance;
+	return hci_cmd_sync_queue(hdev, _update_adv_data_sync, inst_ptr, NULL);
+}
diff --git a/net/bluetooth/hci_sysfs.c b/net/bluetooth/hci_sysfs.c
index 4e3e0451b08c..08542dfc2dc5 100644
--- a/net/bluetooth/hci_sysfs.c
+++ b/net/bluetooth/hci_sysfs.c
@@ -48,6 +48,9 @@ void hci_conn_add_sysfs(struct hci_conn *conn)
 
 	BT_DBG("conn %p", conn);
 
+	if (device_is_registered(&conn->dev))
+		return;
+
 	dev_set_name(&conn->dev, "%s:%d", hdev->name, conn->handle);
 
 	if (device_add(&conn->dev) < 0) {
diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c
index 2c9de67daadc..1f34b82ca0ec 100644
--- a/net/bluetooth/l2cap_core.c
+++ b/net/bluetooth/l2cap_core.c
@@ -61,6 +61,9 @@ static void l2cap_send_disconn_req(struct l2cap_chan *chan, int err);
 
 static void l2cap_tx(struct l2cap_chan *chan, struct l2cap_ctrl *control,
 		     struct sk_buff_head *skbs, u8 event);
+static void l2cap_retrans_timeout(struct work_struct *work);
+static void l2cap_monitor_timeout(struct work_struct *work);
+static void l2cap_ack_timeout(struct work_struct *work);
 
 static inline u8 bdaddr_type(u8 link_type, u8 bdaddr_type)
 {
@@ -476,6 +479,9 @@ struct l2cap_chan *l2cap_chan_create(void)
 	write_unlock(&chan_list_lock);
 
 	INIT_DELAYED_WORK(&chan->chan_timer, l2cap_chan_timeout);
+	INIT_DELAYED_WORK(&chan->retrans_timer, l2cap_retrans_timeout);
+	INIT_DELAYED_WORK(&chan->monitor_timer, l2cap_monitor_timeout);
+	INIT_DELAYED_WORK(&chan->ack_timer, l2cap_ack_timeout);
 
 	chan->state = BT_OPEN;
 
@@ -3320,10 +3326,6 @@ int l2cap_ertm_init(struct l2cap_chan *chan)
 	chan->rx_state = L2CAP_RX_STATE_RECV;
 	chan->tx_state = L2CAP_TX_STATE_XMIT;
 
-	INIT_DELAYED_WORK(&chan->retrans_timer, l2cap_retrans_timeout);
-	INIT_DELAYED_WORK(&chan->monitor_timer, l2cap_monitor_timeout);
-	INIT_DELAYED_WORK(&chan->ack_timer, l2cap_ack_timeout);
-
 	skb_queue_head_init(&chan->srej_q);
 
 	err = l2cap_seq_list_init(&chan->srej_list, chan->tx_win);
@@ -4307,6 +4309,12 @@ static int l2cap_connect_create_rsp(struct l2cap_conn *conn,
 		}
 	}
 
+	chan = l2cap_chan_hold_unless_zero(chan);
+	if (!chan) {
+		err = -EBADSLT;
+		goto unlock;
+	}
+
 	err = 0;
 
 	l2cap_chan_lock(chan);
@@ -4336,6 +4344,7 @@ static int l2cap_connect_create_rsp(struct l2cap_conn *conn,
 	}
 
 	l2cap_chan_unlock(chan);
+	l2cap_chan_put(chan);
 
 unlock:
 	mutex_unlock(&conn->chan_lock);
diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c
index 72e6595a71cc..a92e7e485feb 100644
--- a/net/bluetooth/mgmt.c
+++ b/net/bluetooth/mgmt.c
@@ -129,6 +129,10 @@ static const u16 mgmt_commands[] = {
 	MGMT_OP_ADD_EXT_ADV_PARAMS,
 	MGMT_OP_ADD_EXT_ADV_DATA,
 	MGMT_OP_ADD_ADV_PATTERNS_MONITOR_RSSI,
+	MGMT_OP_SET_MESH_RECEIVER,
+	MGMT_OP_MESH_READ_FEATURES,
+	MGMT_OP_MESH_SEND,
+	MGMT_OP_MESH_SEND_CANCEL,
 };
 
 static const u16 mgmt_events[] = {
@@ -1048,9 +1052,66 @@ static void discov_off(struct work_struct *work)
 	hci_dev_unlock(hdev);
 }
 
+static int send_settings_rsp(struct sock *sk, u16 opcode, struct hci_dev *hdev);
+
+static void mesh_send_complete(struct hci_dev *hdev,
+			       struct mgmt_mesh_tx *mesh_tx, bool silent)
+{
+	u8 handle = mesh_tx->handle;
+
+	if (!silent)
+		mgmt_event(MGMT_EV_MESH_PACKET_CMPLT, hdev, &handle,
+			   sizeof(handle), NULL);
+
+	mgmt_mesh_remove(mesh_tx);
+}
+
+static int mesh_send_done_sync(struct hci_dev *hdev, void *data)
+{
+	struct mgmt_mesh_tx *mesh_tx;
+
+	hci_dev_clear_flag(hdev, HCI_MESH_SENDING);
+	hci_disable_advertising_sync(hdev);
+	mesh_tx = mgmt_mesh_next(hdev, NULL);
+
+	if (mesh_tx)
+		mesh_send_complete(hdev, mesh_tx, false);
+
+	return 0;
+}
+
+static int mesh_send_sync(struct hci_dev *hdev, void *data);
+static void mesh_send_start_complete(struct hci_dev *hdev, void *data, int err);
+static void mesh_next(struct hci_dev *hdev, void *data, int err)
+{
+	struct mgmt_mesh_tx *mesh_tx = mgmt_mesh_next(hdev, NULL);
+
+	if (!mesh_tx)
+		return;
+
+	err = hci_cmd_sync_queue(hdev, mesh_send_sync, mesh_tx,
+				 mesh_send_start_complete);
+
+	if (err < 0)
+		mesh_send_complete(hdev, mesh_tx, false);
+	else
+		hci_dev_set_flag(hdev, HCI_MESH_SENDING);
+}
+
+static void mesh_send_done(struct work_struct *work)
+{
+	struct hci_dev *hdev = container_of(work, struct hci_dev,
+					    mesh_send_done.work);
+
+	if (!hci_dev_test_flag(hdev, HCI_MESH_SENDING))
+		return;
+
+	hci_cmd_sync_queue(hdev, mesh_send_done_sync, NULL, mesh_next);
+}
+
 static void mgmt_init_hdev(struct sock *sk, struct hci_dev *hdev)
 {
-	if (hci_dev_test_and_set_flag(hdev, HCI_MGMT))
+	if (hci_dev_test_flag(hdev, HCI_MGMT))
 		return;
 
 	BT_INFO("MGMT ver %d.%d", MGMT_VERSION, MGMT_REVISION);
@@ -1058,6 +1119,7 @@ static void mgmt_init_hdev(struct sock *sk, struct hci_dev *hdev)
 	INIT_DELAYED_WORK(&hdev->discov_off, discov_off);
 	INIT_DELAYED_WORK(&hdev->service_cache, service_cache_off);
 	INIT_DELAYED_WORK(&hdev->rpa_expired, rpa_expired);
+	INIT_DELAYED_WORK(&hdev->mesh_send_done, mesh_send_done);
 
 	/* Non-mgmt controlled devices get this bit set
 	 * implicitly so that pairing works for them, however
@@ -1065,6 +1127,8 @@ static void mgmt_init_hdev(struct sock *sk, struct hci_dev *hdev)
 	 * it
 	 */
 	hci_dev_clear_flag(hdev, HCI_BONDABLE);
+
+	hci_dev_set_flag(hdev, HCI_MGMT);
 }
 
 static int read_controller_info(struct sock *sk, struct hci_dev *hdev,
@@ -2058,6 +2122,8 @@ static int set_le_sync(struct hci_dev *hdev, void *data)
 	int err;
 
 	if (!val) {
+		hci_clear_adv_instance_sync(hdev, NULL, 0x00, true);
+
 		if (hci_dev_test_flag(hdev, HCI_LE_ADV))
 			hci_disable_advertising_sync(hdev);
 
@@ -2092,6 +2158,317 @@ static int set_le_sync(struct hci_dev *hdev, void *data)
 	return err;
 }
 
+static void set_mesh_complete(struct hci_dev *hdev, void *data, int err)
+{
+	struct mgmt_pending_cmd *cmd = data;
+	u8 status = mgmt_status(err);
+	struct sock *sk = cmd->sk;
+
+	if (status) {
+		mgmt_pending_foreach(MGMT_OP_SET_MESH_RECEIVER, hdev,
+				     cmd_status_rsp, &status);
+		return;
+	}
+
+	mgmt_pending_remove(cmd);
+	mgmt_cmd_complete(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER, 0, NULL, 0);
+}
+
+static int set_mesh_sync(struct hci_dev *hdev, void *data)
+{
+	struct mgmt_pending_cmd *cmd = data;
+	struct mgmt_cp_set_mesh *cp = cmd->param;
+	size_t len = cmd->param_len;
+
+	memset(hdev->mesh_ad_types, 0, sizeof(hdev->mesh_ad_types));
+
+	if (cp->enable)
+		hci_dev_set_flag(hdev, HCI_MESH);
+	else
+		hci_dev_clear_flag(hdev, HCI_MESH);
+
+	len -= sizeof(*cp);
+
+	/* If filters don't fit, forward all adv pkts */
+	if (len <= sizeof(hdev->mesh_ad_types))
+		memcpy(hdev->mesh_ad_types, cp->ad_types, len);
+
+	hci_update_passive_scan_sync(hdev);
+	return 0;
+}
+
+static int set_mesh(struct sock *sk, struct hci_dev *hdev, void *data, u16 len)
+{
+	struct mgmt_cp_set_mesh *cp = data;
+	struct mgmt_pending_cmd *cmd;
+	int err = 0;
+
+	bt_dev_dbg(hdev, "sock %p", sk);
+
+	if (!lmp_le_capable(hdev) ||
+	    !hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL))
+		return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER,
+				       MGMT_STATUS_NOT_SUPPORTED);
+
+	if (cp->enable != 0x00 && cp->enable != 0x01)
+		return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER,
+				       MGMT_STATUS_INVALID_PARAMS);
+
+	hci_dev_lock(hdev);
+
+	cmd = mgmt_pending_add(sk, MGMT_OP_SET_MESH_RECEIVER, hdev, data, len);
+	if (!cmd)
+		err = -ENOMEM;
+	else
+		err = hci_cmd_sync_queue(hdev, set_mesh_sync, cmd,
+					 set_mesh_complete);
+
+	if (err < 0) {
+		err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_MESH_RECEIVER,
+				      MGMT_STATUS_FAILED);
+
+		if (cmd)
+			mgmt_pending_remove(cmd);
+	}
+
+	hci_dev_unlock(hdev);
+	return err;
+}
+
+static void mesh_send_start_complete(struct hci_dev *hdev, void *data, int err)
+{
+	struct mgmt_mesh_tx *mesh_tx = data;
+	struct mgmt_cp_mesh_send *send = (void *)mesh_tx->param;
+	unsigned long mesh_send_interval;
+	u8 mgmt_err = mgmt_status(err);
+
+	/* Report any errors here, but don't report completion */
+
+	if (mgmt_err) {
+		hci_dev_clear_flag(hdev, HCI_MESH_SENDING);
+		/* Send Complete Error Code for handle */
+		mesh_send_complete(hdev, mesh_tx, false);
+		return;
+	}
+
+	mesh_send_interval = msecs_to_jiffies((send->cnt) * 25);
+	queue_delayed_work(hdev->req_workqueue, &hdev->mesh_send_done,
+			   mesh_send_interval);
+}
+
+static int mesh_send_sync(struct hci_dev *hdev, void *data)
+{
+	struct mgmt_mesh_tx *mesh_tx = data;
+	struct mgmt_cp_mesh_send *send = (void *)mesh_tx->param;
+	struct adv_info *adv, *next_instance;
+	u8 instance = hdev->le_num_of_adv_sets + 1;
+	u16 timeout, duration;
+	int err = 0;
+
+	if (hdev->le_num_of_adv_sets <= hdev->adv_instance_cnt)
+		return MGMT_STATUS_BUSY;
+
+	timeout = 1000;
+	duration = send->cnt * INTERVAL_TO_MS(hdev->le_adv_max_interval);
+	adv = hci_add_adv_instance(hdev, instance, 0,
+				   send->adv_data_len, send->adv_data,
+				   0, NULL,
+				   timeout, duration,
+				   HCI_ADV_TX_POWER_NO_PREFERENCE,
+				   hdev->le_adv_min_interval,
+				   hdev->le_adv_max_interval,
+				   mesh_tx->handle);
+
+	if (!IS_ERR(adv))
+		mesh_tx->instance = instance;
+	else
+		err = PTR_ERR(adv);
+
+	if (hdev->cur_adv_instance == instance) {
+		/* If the currently advertised instance is being changed then
+		 * cancel the current advertising and schedule the next
+		 * instance. If there is only one instance then the overridden
+		 * advertising data will be visible right away.
+		 */
+		cancel_adv_timeout(hdev);
+
+		next_instance = hci_get_next_instance(hdev, instance);
+		if (next_instance)
+			instance = next_instance->instance;
+		else
+			instance = 0;
+	} else if (hdev->adv_instance_timeout) {
+		/* Immediately advertise the new instance if no other, or
+		 * let it go naturally from queue if ADV is already happening
+		 */
+		instance = 0;
+	}
+
+	if (instance)
+		return hci_schedule_adv_instance_sync(hdev, instance, true);
+
+	return err;
+}
+
+static void send_count(struct mgmt_mesh_tx *mesh_tx, void *data)
+{
+	struct mgmt_rp_mesh_read_features *rp = data;
+
+	if (rp->used_handles >= rp->max_handles)
+		return;
+
+	rp->handles[rp->used_handles++] = mesh_tx->handle;
+}
+
+static int mesh_features(struct sock *sk, struct hci_dev *hdev,
+			 void *data, u16 len)
+{
+	struct mgmt_rp_mesh_read_features rp;
+
+	if (!lmp_le_capable(hdev) ||
+	    !hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL))
+		return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_READ_FEATURES,
+				       MGMT_STATUS_NOT_SUPPORTED);
+
+	memset(&rp, 0, sizeof(rp));
+	rp.index = cpu_to_le16(hdev->id);
+	if (hci_dev_test_flag(hdev, HCI_LE_ENABLED))
+		rp.max_handles = MESH_HANDLES_MAX;
+
+	hci_dev_lock(hdev);
+
+	if (rp.max_handles)
+		mgmt_mesh_foreach(hdev, send_count, &rp, sk);
+
+	mgmt_cmd_complete(sk, hdev->id, MGMT_OP_MESH_READ_FEATURES, 0, &rp,
+			  rp.used_handles + sizeof(rp) - MESH_HANDLES_MAX);
+
+	hci_dev_unlock(hdev);
+	return 0;
+}
+
+static int send_cancel(struct hci_dev *hdev, void *data)
+{
+	struct mgmt_pending_cmd *cmd = data;
+	struct mgmt_cp_mesh_send_cancel *cancel = (void *)cmd->param;
+	struct mgmt_mesh_tx *mesh_tx;
+
+	if (!cancel->handle) {
+		do {
+			mesh_tx = mgmt_mesh_next(hdev, cmd->sk);
+
+			if (mesh_tx)
+				mesh_send_complete(hdev, mesh_tx, false);
+		} while (mesh_tx);
+	} else {
+		mesh_tx = mgmt_mesh_find(hdev, cancel->handle);
+
+		if (mesh_tx && mesh_tx->sk == cmd->sk)
+			mesh_send_complete(hdev, mesh_tx, false);
+	}
+
+	mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL,
+			  0, NULL, 0);
+	mgmt_pending_free(cmd);
+
+	return 0;
+}
+
+static int mesh_send_cancel(struct sock *sk, struct hci_dev *hdev,
+			    void *data, u16 len)
+{
+	struct mgmt_pending_cmd *cmd;
+	int err;
+
+	if (!lmp_le_capable(hdev) ||
+	    !hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL))
+		return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL,
+				       MGMT_STATUS_NOT_SUPPORTED);
+
+	if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED))
+		return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL,
+				       MGMT_STATUS_REJECTED);
+
+	hci_dev_lock(hdev);
+	cmd = mgmt_pending_new(sk, MGMT_OP_MESH_SEND_CANCEL, hdev, data, len);
+	if (!cmd)
+		err = -ENOMEM;
+	else
+		err = hci_cmd_sync_queue(hdev, send_cancel, cmd, NULL);
+
+	if (err < 0) {
+		err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND_CANCEL,
+				      MGMT_STATUS_FAILED);
+
+		if (cmd)
+			mgmt_pending_free(cmd);
+	}
+
+	hci_dev_unlock(hdev);
+	return err;
+}
+
+static int mesh_send(struct sock *sk, struct hci_dev *hdev, void *data, u16 len)
+{
+	struct mgmt_mesh_tx *mesh_tx;
+	struct mgmt_cp_mesh_send *send = data;
+	struct mgmt_rp_mesh_read_features rp;
+	bool sending;
+	int err = 0;
+
+	if (!lmp_le_capable(hdev) ||
+	    !hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL))
+		return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND,
+				       MGMT_STATUS_NOT_SUPPORTED);
+	if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED) ||
+	    len <= MGMT_MESH_SEND_SIZE ||
+	    len > (MGMT_MESH_SEND_SIZE + 31))
+		return mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND,
+				       MGMT_STATUS_REJECTED);
+
+	hci_dev_lock(hdev);
+
+	memset(&rp, 0, sizeof(rp));
+	rp.max_handles = MESH_HANDLES_MAX;
+
+	mgmt_mesh_foreach(hdev, send_count, &rp, sk);
+
+	if (rp.max_handles <= rp.used_handles) {
+		err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND,
+				      MGMT_STATUS_BUSY);
+		goto done;
+	}
+
+	sending = hci_dev_test_flag(hdev, HCI_MESH_SENDING);
+	mesh_tx = mgmt_mesh_add(sk, hdev, send, len);
+
+	if (!mesh_tx)
+		err = -ENOMEM;
+	else if (!sending)
+		err = hci_cmd_sync_queue(hdev, mesh_send_sync, mesh_tx,
+					 mesh_send_start_complete);
+
+	if (err < 0) {
+		bt_dev_err(hdev, "Send Mesh Failed %d", err);
+		err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_MESH_SEND,
+				      MGMT_STATUS_FAILED);
+
+		if (mesh_tx) {
+			if (sending)
+				mgmt_mesh_remove(mesh_tx);
+		}
+	} else {
+		hci_dev_set_flag(hdev, HCI_MESH_SENDING);
+
+		mgmt_cmd_complete(sk, hdev->id, MGMT_OP_MESH_SEND, 0,
+				  &mesh_tx->handle, 1);
+	}
+
+done:
+	hci_dev_unlock(hdev);
+	return err;
+}
+
 static int set_le(struct sock *sk, struct hci_dev *hdev, void *data, u16 len)
 {
 	struct mgmt_mode *cp = data;
@@ -2131,9 +2508,6 @@ static int set_le(struct sock *sk, struct hci_dev *hdev, void *data, u16 len)
 	val = !!cp->val;
 	enabled = lmp_host_le_capable(hdev);
 
-	if (!val)
-		hci_req_clear_adv_instance(hdev, NULL, NULL, 0x00, true);
-
 	if (!hdev_is_powered(hdev) || val == enabled) {
 		bool changed = false;
 
@@ -3186,6 +3560,18 @@ unlock:
 	return err;
 }
 
+static int abort_conn_sync(struct hci_dev *hdev, void *data)
+{
+	struct hci_conn *conn;
+	u16 handle = PTR_ERR(data);
+
+	conn = hci_conn_hash_lookup_handle(hdev, handle);
+	if (!conn)
+		return 0;
+
+	return hci_abort_conn_sync(hdev, conn, HCI_ERROR_REMOTE_USER_TERM);
+}
+
 static int cancel_pair_device(struct sock *sk, struct hci_dev *hdev, void *data,
 			      u16 len)
 {
@@ -3236,7 +3622,8 @@ static int cancel_pair_device(struct sock *sk, struct hci_dev *hdev, void *data,
 					      le_addr_type(addr->type));
 
 	if (conn->conn_reason == CONN_REASON_PAIR_DEVICE)
-		hci_abort_conn(conn, HCI_ERROR_REMOTE_USER_TERM);
+		hci_cmd_sync_queue(hdev, abort_conn_sync, ERR_PTR(conn->handle),
+				   NULL);
 
 unlock:
 	hci_dev_unlock(hdev);
@@ -3991,17 +4378,28 @@ static const u8 iso_socket_uuid[16] = {
 	0x6a, 0x49, 0xe0, 0x05, 0x88, 0xf1, 0xba, 0x6f,
 };
 
+/* 2ce463d7-7a03-4d8d-bf05-5f24e8f36e76 */
+static const u8 mgmt_mesh_uuid[16] = {
+	0x76, 0x6e, 0xf3, 0xe8, 0x24, 0x5f, 0x05, 0xbf,
+	0x8d, 0x4d, 0x03, 0x7a, 0xd7, 0x63, 0xe4, 0x2c,
+};
+
 static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev,
 				  void *data, u16 data_len)
 {
-	char buf[122];   /* Enough space for 6 features: 2 + 20 * 6 */
-	struct mgmt_rp_read_exp_features_info *rp = (void *)buf;
+	struct mgmt_rp_read_exp_features_info *rp;
+	size_t len;
 	u16 idx = 0;
 	u32 flags;
+	int status;
 
 	bt_dev_dbg(hdev, "sock %p", sk);
 
-	memset(&buf, 0, sizeof(buf));
+	/* Enough space for 7 features */
+	len = sizeof(*rp) + (sizeof(rp->features[0]) * 7);
+	rp = kzalloc(len, GFP_KERNEL);
+	if (!rp)
+		return -ENOMEM;
 
 #ifdef CONFIG_BT_FEATURE_DEBUG
 	if (!hdev) {
@@ -4065,6 +4463,17 @@ static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev,
 		idx++;
 	}
 
+	if (hdev && lmp_le_capable(hdev)) {
+		if (hci_dev_test_flag(hdev, HCI_MESH_EXPERIMENTAL))
+			flags = BIT(0);
+		else
+			flags = 0;
+
+		memcpy(rp->features[idx].uuid, mgmt_mesh_uuid, 16);
+		rp->features[idx].flags = cpu_to_le32(flags);
+		idx++;
+	}
+
 	rp->feature_count = cpu_to_le16(idx);
 
 	/* After reading the experimental features information, enable
@@ -4072,9 +4481,12 @@ static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev,
 	 */
 	hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS);
 
-	return mgmt_cmd_complete(sk, hdev ? hdev->id : MGMT_INDEX_NONE,
-				 MGMT_OP_READ_EXP_FEATURES_INFO,
-				 0, rp, sizeof(*rp) + (20 * idx));
+	status = mgmt_cmd_complete(sk, hdev ? hdev->id : MGMT_INDEX_NONE,
+				   MGMT_OP_READ_EXP_FEATURES_INFO,
+				   0, rp, sizeof(*rp) + (20 * idx));
+
+	kfree(rp);
+	return status;
 }
 
 static int exp_ll_privacy_feature_changed(bool enabled, struct hci_dev *hdev,
@@ -4202,6 +4614,63 @@ static int set_debug_func(struct sock *sk, struct hci_dev *hdev,
 }
 #endif
 
+static int set_mgmt_mesh_func(struct sock *sk, struct hci_dev *hdev,
+			      struct mgmt_cp_set_exp_feature *cp, u16 data_len)
+{
+	struct mgmt_rp_set_exp_feature rp;
+	bool val, changed;
+	int err;
+
+	/* Command requires to use the controller index */
+	if (!hdev)
+		return mgmt_cmd_status(sk, MGMT_INDEX_NONE,
+				       MGMT_OP_SET_EXP_FEATURE,
+				       MGMT_STATUS_INVALID_INDEX);
+
+	/* Changes can only be made when controller is powered down */
+	if (hdev_is_powered(hdev))
+		return mgmt_cmd_status(sk, hdev->id,
+				       MGMT_OP_SET_EXP_FEATURE,
+				       MGMT_STATUS_REJECTED);
+
+	/* Parameters are limited to a single octet */
+	if (data_len != MGMT_SET_EXP_FEATURE_SIZE + 1)
+		return mgmt_cmd_status(sk, hdev->id,
+				       MGMT_OP_SET_EXP_FEATURE,
+				       MGMT_STATUS_INVALID_PARAMS);
+
+	/* Only boolean on/off is supported */
+	if (cp->param[0] != 0x00 && cp->param[0] != 0x01)
+		return mgmt_cmd_status(sk, hdev->id,
+				       MGMT_OP_SET_EXP_FEATURE,
+				       MGMT_STATUS_INVALID_PARAMS);
+
+	val = !!cp->param[0];
+
+	if (val) {
+		changed = !hci_dev_test_and_set_flag(hdev,
+						     HCI_MESH_EXPERIMENTAL);
+	} else {
+		hci_dev_clear_flag(hdev, HCI_MESH);
+		changed = hci_dev_test_and_clear_flag(hdev,
+						      HCI_MESH_EXPERIMENTAL);
+	}
+
+	memcpy(rp.uuid, mgmt_mesh_uuid, 16);
+	rp.flags = cpu_to_le32(val ? BIT(0) : 0);
+
+	hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS);
+
+	err = mgmt_cmd_complete(sk, hdev->id,
+				MGMT_OP_SET_EXP_FEATURE, 0,
+				&rp, sizeof(rp));
+
+	if (changed)
+		exp_feature_changed(hdev, mgmt_mesh_uuid, val, sk);
+
+	return err;
+}
+
 static int set_rpa_resolution_func(struct sock *sk, struct hci_dev *hdev,
 				   struct mgmt_cp_set_exp_feature *cp,
 				   u16 data_len)
@@ -4517,6 +4986,7 @@ static const struct mgmt_exp_feature {
 #ifdef CONFIG_BT_FEATURE_DEBUG
 	EXP_FEAT(debug_uuid, set_debug_func),
 #endif
+	EXP_FEAT(mgmt_mesh_uuid, set_mgmt_mesh_func),
 	EXP_FEAT(rpa_resolution_uuid, set_rpa_resolution_func),
 	EXP_FEAT(quality_report_uuid, set_quality_report_func),
 	EXP_FEAT(offload_codecs_uuid, set_offload_codec_func),
@@ -5981,6 +6451,7 @@ static int set_advertising(struct sock *sk, struct hci_dev *hdev, void *data,
 	if (!hdev_is_powered(hdev) ||
 	    (val == hci_dev_test_flag(hdev, HCI_ADVERTISING) &&
 	     (cp->val == 0x02) == hci_dev_test_flag(hdev, HCI_ADVERTISING_CONNECTABLE)) ||
+	    hci_dev_test_flag(hdev, HCI_MESH) ||
 	    hci_conn_num(hdev, LE_LINK) > 0 ||
 	    (hci_dev_test_flag(hdev, HCI_LE_SCAN) &&
 	     hdev->le_scan_type == LE_SCAN_ACTIVE)) {
@@ -7909,8 +8380,7 @@ static u32 get_supported_adv_flags(struct hci_dev *hdev)
 	/* In extended adv TX_POWER returned from Set Adv Param
 	 * will be always valid.
 	 */
-	if ((hdev->adv_tx_power != HCI_TX_POWER_INVALID) ||
-	    ext_adv_capable(hdev))
+	if (hdev->adv_tx_power != HCI_TX_POWER_INVALID || ext_adv_capable(hdev))
 		flags |= MGMT_ADV_FLAG_TX_POWER;
 
 	if (ext_adv_capable(hdev)) {
@@ -7963,8 +8433,14 @@ static int read_adv_features(struct sock *sk, struct hci_dev *hdev,
 
 	instance = rp->instance;
 	list_for_each_entry(adv_instance, &hdev->adv_instances, list) {
-		*instance = adv_instance->instance;
-		instance++;
+		/* Only instances 1-le_num_of_adv_sets are externally visible */
+		if (adv_instance->instance <= hdev->adv_instance_cnt) {
+			*instance = adv_instance->instance;
+			instance++;
+		} else {
+			rp->num_instances--;
+			rp_len--;
+		}
 	}
 
 	hci_dev_unlock(hdev);
@@ -8226,7 +8702,7 @@ static int add_advertising(struct sock *sk, struct hci_dev *hdev,
 				   timeout, duration,
 				   HCI_ADV_TX_POWER_NO_PREFERENCE,
 				   hdev->le_adv_min_interval,
-				   hdev->le_adv_max_interval);
+				   hdev->le_adv_max_interval, 0);
 	if (IS_ERR(adv)) {
 		err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_ADVERTISING,
 				      MGMT_STATUS_FAILED);
@@ -8430,7 +8906,7 @@ static int add_ext_adv_params(struct sock *sk, struct hci_dev *hdev,
 	/* Create advertising instance with no advertising or response data */
 	adv = hci_add_adv_instance(hdev, cp->instance, flags, 0, NULL, 0, NULL,
 				   timeout, duration, tx_power, min_interval,
-				   max_interval);
+				   max_interval, 0);
 
 	if (IS_ERR(adv)) {
 		err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_EXT_ADV_PARAMS,
@@ -8876,8 +9352,13 @@ static const struct hci_mgmt_handler mgmt_handlers[] = {
 	{ add_ext_adv_data,        MGMT_ADD_EXT_ADV_DATA_SIZE,
 						HCI_MGMT_VAR_LEN },
 	{ add_adv_patterns_monitor_rssi,
-				   MGMT_ADD_ADV_PATTERNS_MONITOR_RSSI_SIZE,
+				   MGMT_ADD_ADV_PATTERNS_MONITOR_RSSI_SIZE },
+	{ set_mesh,                MGMT_SET_MESH_RECEIVER_SIZE,
+						HCI_MGMT_VAR_LEN },
+	{ mesh_features,           MGMT_MESH_READ_FEATURES_SIZE },
+	{ mesh_send,               MGMT_MESH_SEND_SIZE,
 						HCI_MGMT_VAR_LEN },
+	{ mesh_send_cancel,        MGMT_MESH_SEND_CANCEL_SIZE },
 };
 
 void mgmt_index_added(struct hci_dev *hdev)
@@ -9817,14 +10298,86 @@ static void mgmt_adv_monitor_device_found(struct hci_dev *hdev,
 		kfree_skb(skb);
 }
 
+static void mesh_device_found(struct hci_dev *hdev, bdaddr_t *bdaddr,
+			      u8 addr_type, s8 rssi, u32 flags, u8 *eir,
+			      u16 eir_len, u8 *scan_rsp, u8 scan_rsp_len,
+			      u64 instant)
+{
+	struct sk_buff *skb;
+	struct mgmt_ev_mesh_device_found *ev;
+	int i, j;
+
+	if (!hdev->mesh_ad_types[0])
+		goto accepted;
+
+	/* Scan for requested AD types */
+	if (eir_len > 0) {
+		for (i = 0; i + 1 < eir_len; i += eir[i] + 1) {
+			for (j = 0; j < sizeof(hdev->mesh_ad_types); j++) {
+				if (!hdev->mesh_ad_types[j])
+					break;
+
+				if (hdev->mesh_ad_types[j] == eir[i + 1])
+					goto accepted;
+			}
+		}
+	}
+
+	if (scan_rsp_len > 0) {
+		for (i = 0; i + 1 < scan_rsp_len; i += scan_rsp[i] + 1) {
+			for (j = 0; j < sizeof(hdev->mesh_ad_types); j++) {
+				if (!hdev->mesh_ad_types[j])
+					break;
+
+				if (hdev->mesh_ad_types[j] == scan_rsp[i + 1])
+					goto accepted;
+			}
+		}
+	}
+
+	return;
+
+accepted:
+	skb = mgmt_alloc_skb(hdev, MGMT_EV_MESH_DEVICE_FOUND,
+			     sizeof(*ev) + eir_len + scan_rsp_len);
+	if (!skb)
+		return;
+
+	ev = skb_put(skb, sizeof(*ev));
+
+	bacpy(&ev->addr.bdaddr, bdaddr);
+	ev->addr.type = link_to_bdaddr(LE_LINK, addr_type);
+	ev->rssi = rssi;
+	ev->flags = cpu_to_le32(flags);
+	ev->instant = cpu_to_le64(instant);
+
+	if (eir_len > 0)
+		/* Copy EIR or advertising data into event */
+		skb_put_data(skb, eir, eir_len);
+
+	if (scan_rsp_len > 0)
+		/* Append scan response data to event */
+		skb_put_data(skb, scan_rsp, scan_rsp_len);
+
+	ev->eir_len = cpu_to_le16(eir_len + scan_rsp_len);
+
+	mgmt_event_skb(skb, NULL);
+}
+
 void mgmt_device_found(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 link_type,
 		       u8 addr_type, u8 *dev_class, s8 rssi, u32 flags,
-		       u8 *eir, u16 eir_len, u8 *scan_rsp, u8 scan_rsp_len)
+		       u8 *eir, u16 eir_len, u8 *scan_rsp, u8 scan_rsp_len,
+		       u64 instant)
 {
 	struct sk_buff *skb;
 	struct mgmt_ev_device_found *ev;
 	bool report_device = hci_discovery_active(hdev);
 
+	if (hci_dev_test_flag(hdev, HCI_MESH) && link_type == LE_LINK)
+		mesh_device_found(hdev, bdaddr, addr_type, rssi, flags,
+				  eir, eir_len, scan_rsp, scan_rsp_len,
+				  instant);
+
 	/* Don't send events for a non-kernel initiated discovery. With
 	 * LE one exception is if we have pend_le_reports > 0 in which
 	 * case we're doing passive scanning and want these events.
@@ -9983,3 +10536,22 @@ void mgmt_exit(void)
 {
 	hci_mgmt_chan_unregister(&chan);
 }
+
+void mgmt_cleanup(struct sock *sk)
+{
+	struct mgmt_mesh_tx *mesh_tx;
+	struct hci_dev *hdev;
+
+	read_lock(&hci_dev_list_lock);
+
+	list_for_each_entry(hdev, &hci_dev_list, list) {
+		do {
+			mesh_tx = mgmt_mesh_next(hdev, sk);
+
+			if (mesh_tx)
+				mesh_send_complete(hdev, mesh_tx, true);
+		} while (mesh_tx);
+	}
+
+	read_unlock(&hci_dev_list_lock);
+}
diff --git a/net/bluetooth/mgmt_util.c b/net/bluetooth/mgmt_util.c
index b69cfed62088..0115f783bde8 100644
--- a/net/bluetooth/mgmt_util.c
+++ b/net/bluetooth/mgmt_util.c
@@ -314,3 +314,77 @@ void mgmt_pending_remove(struct mgmt_pending_cmd *cmd)
 	list_del(&cmd->list);
 	mgmt_pending_free(cmd);
 }
+
+void mgmt_mesh_foreach(struct hci_dev *hdev,
+		       void (*cb)(struct mgmt_mesh_tx *mesh_tx, void *data),
+		       void *data, struct sock *sk)
+{
+	struct mgmt_mesh_tx *mesh_tx, *tmp;
+
+	list_for_each_entry_safe(mesh_tx, tmp, &hdev->mgmt_pending, list) {
+		if (!sk || mesh_tx->sk == sk)
+			cb(mesh_tx, data);
+	}
+}
+
+struct mgmt_mesh_tx *mgmt_mesh_next(struct hci_dev *hdev, struct sock *sk)
+{
+	struct mgmt_mesh_tx *mesh_tx;
+
+	if (list_empty(&hdev->mesh_pending))
+		return NULL;
+
+	list_for_each_entry(mesh_tx, &hdev->mesh_pending, list) {
+		if (!sk || mesh_tx->sk == sk)
+			return mesh_tx;
+	}
+
+	return NULL;
+}
+
+struct mgmt_mesh_tx *mgmt_mesh_find(struct hci_dev *hdev, u8 handle)
+{
+	struct mgmt_mesh_tx *mesh_tx;
+
+	if (list_empty(&hdev->mesh_pending))
+		return NULL;
+
+	list_for_each_entry(mesh_tx, &hdev->mesh_pending, list) {
+		if (mesh_tx->handle == handle)
+			return mesh_tx;
+	}
+
+	return NULL;
+}
+
+struct mgmt_mesh_tx *mgmt_mesh_add(struct sock *sk, struct hci_dev *hdev,
+				   void *data, u16 len)
+{
+	struct mgmt_mesh_tx *mesh_tx;
+
+	mesh_tx = kzalloc(sizeof(*mesh_tx), GFP_KERNEL);
+	if (!mesh_tx)
+		return NULL;
+
+	hdev->mesh_send_ref++;
+	if (!hdev->mesh_send_ref)
+		hdev->mesh_send_ref++;
+
+	mesh_tx->handle = hdev->mesh_send_ref;
+	mesh_tx->index = hdev->id;
+	memcpy(mesh_tx->param, data, len);
+	mesh_tx->param_len = len;
+	mesh_tx->sk = sk;
+	sock_hold(sk);
+
+	list_add_tail(&mesh_tx->list, &hdev->mesh_pending);
+
+	return mesh_tx;
+}
+
+void mgmt_mesh_remove(struct mgmt_mesh_tx *mesh_tx)
+{
+	list_del(&mesh_tx->list);
+	sock_put(mesh_tx->sk);
+	kfree(mesh_tx);
+}
diff --git a/net/bluetooth/mgmt_util.h b/net/bluetooth/mgmt_util.h
index 98e40395a383..6a8b7e84293d 100644
--- a/net/bluetooth/mgmt_util.h
+++ b/net/bluetooth/mgmt_util.h
@@ -20,6 +20,16 @@
    SOFTWARE IS DISCLAIMED.
 */
 
+struct mgmt_mesh_tx {
+	struct list_head list;
+	int index;
+	size_t param_len;
+	struct sock *sk;
+	u8 handle;
+	u8 instance;
+	u8 param[sizeof(struct mgmt_cp_mesh_send) + 29];
+};
+
 struct mgmt_pending_cmd {
 	struct list_head list;
 	u16 opcode;
@@ -59,3 +69,11 @@ struct mgmt_pending_cmd *mgmt_pending_new(struct sock *sk, u16 opcode,
 					  void *data, u16 len);
 void mgmt_pending_free(struct mgmt_pending_cmd *cmd);
 void mgmt_pending_remove(struct mgmt_pending_cmd *cmd);
+void mgmt_mesh_foreach(struct hci_dev *hdev,
+		       void (*cb)(struct mgmt_mesh_tx *mesh_tx, void *data),
+		       void *data, struct sock *sk);
+struct mgmt_mesh_tx *mgmt_mesh_find(struct hci_dev *hdev, u8 handle);
+struct mgmt_mesh_tx *mgmt_mesh_next(struct hci_dev *hdev, struct sock *sk);
+struct mgmt_mesh_tx *mgmt_mesh_add(struct sock *sk, struct hci_dev *hdev,
+				   void *data, u16 len);
+void mgmt_mesh_remove(struct mgmt_mesh_tx *mesh_tx);
diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c
index 4bf4ea6cbb5e..21e24da4847f 100644
--- a/net/bluetooth/rfcomm/sock.c
+++ b/net/bluetooth/rfcomm/sock.c
@@ -902,7 +902,10 @@ static int rfcomm_sock_shutdown(struct socket *sock, int how)
 	lock_sock(sk);
 	if (!sk->sk_shutdown) {
 		sk->sk_shutdown = SHUTDOWN_MASK;
+
+		release_sock(sk);
 		__rfcomm_sock_close(sk);
+		lock_sock(sk);
 
 		if (sock_flag(sk, SOCK_LINGER) && sk->sk_lingertime &&
 		    !(current->flags & PF_EXITING))
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index d11209367dd0..13d578ce2a09 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -606,6 +606,38 @@ noinline void bpf_kfunc_call_memb1_release(struct prog_test_member1 *p)
 	WARN_ON_ONCE(1);
 }
 
+static int *__bpf_kfunc_call_test_get_mem(struct prog_test_ref_kfunc *p, const int size)
+{
+	if (size > 2 * sizeof(int))
+		return NULL;
+
+	return (int *)p;
+}
+
+noinline int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size)
+{
+	return __bpf_kfunc_call_test_get_mem(p, rdwr_buf_size);
+}
+
+noinline int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size)
+{
+	return __bpf_kfunc_call_test_get_mem(p, rdonly_buf_size);
+}
+
+/* the next 2 ones can't be really used for testing expect to ensure
+ * that the verifier rejects the call.
+ * Acquire functions must return struct pointers, so these ones are
+ * failing.
+ */
+noinline int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size)
+{
+	return __bpf_kfunc_call_test_get_mem(p, rdonly_buf_size);
+}
+
+noinline void bpf_kfunc_call_int_mem_release(int *p)
+{
+}
+
 noinline struct prog_test_ref_kfunc *
 bpf_kfunc_call_test_kptr_get(struct prog_test_ref_kfunc **pp, int a, int b)
 {
@@ -695,6 +727,10 @@ noinline void bpf_kfunc_call_test_ref(struct prog_test_ref_kfunc *p)
 {
 }
 
+noinline void bpf_kfunc_call_test_destructive(void)
+{
+}
+
 __diag_pop();
 
 ALLOW_ERROR_INJECTION(bpf_modify_return_test, ERRNO);
@@ -708,6 +744,10 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_memb_acquire, KF_ACQUIRE | KF_RET_NULL)
 BTF_ID_FLAGS(func, bpf_kfunc_call_test_release, KF_RELEASE)
 BTF_ID_FLAGS(func, bpf_kfunc_call_memb_release, KF_RELEASE)
 BTF_ID_FLAGS(func, bpf_kfunc_call_memb1_release, KF_RELEASE)
+BTF_ID_FLAGS(func, bpf_kfunc_call_test_get_rdwr_mem, KF_RET_NULL)
+BTF_ID_FLAGS(func, bpf_kfunc_call_test_get_rdonly_mem, KF_RET_NULL)
+BTF_ID_FLAGS(func, bpf_kfunc_call_test_acq_rdonly_mem, KF_ACQUIRE | KF_RET_NULL)
+BTF_ID_FLAGS(func, bpf_kfunc_call_int_mem_release, KF_RELEASE)
 BTF_ID_FLAGS(func, bpf_kfunc_call_test_kptr_get, KF_ACQUIRE | KF_RET_NULL | KF_KPTR_GET)
 BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass_ctx)
 BTF_ID_FLAGS(func, bpf_kfunc_call_test_pass1)
@@ -719,6 +759,7 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_pass1)
 BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail1)
 BTF_ID_FLAGS(func, bpf_kfunc_call_test_mem_len_fail2)
 BTF_ID_FLAGS(func, bpf_kfunc_call_test_ref, KF_TRUSTED_ARGS)
+BTF_ID_FLAGS(func, bpf_kfunc_call_test_destructive, KF_DESTRUCTIVE)
 BTF_SET8_END(test_sk_check_kfunc_ids)
 
 static void *bpf_test_init(const union bpf_attr *kattr, u32 user_size,
@@ -1629,6 +1670,7 @@ static int __init bpf_prog_test_run_init(void)
 
 	ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &bpf_prog_test_kfunc_set);
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &bpf_prog_test_kfunc_set);
+	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &bpf_prog_test_kfunc_set);
 	return ret ?: register_btf_id_dtor_kfuncs(bpf_prog_test_dtor_kfunc,
 						  ARRAY_SIZE(bpf_prog_test_dtor_kfunc),
 						  THIS_MODULE);
diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c
index 58a4f70e01e3..b82906fc999a 100644
--- a/net/bridge/br_device.c
+++ b/net/bridge/br_device.c
@@ -251,10 +251,10 @@ static int br_set_mac_address(struct net_device *dev, void *p)
 
 static void br_getinfo(struct net_device *dev, struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, "bridge", sizeof(info->driver));
-	strlcpy(info->version, BR_VERSION, sizeof(info->version));
-	strlcpy(info->fw_version, "N/A", sizeof(info->fw_version));
-	strlcpy(info->bus_info, "N/A", sizeof(info->bus_info));
+	strscpy(info->driver, "bridge", sizeof(info->driver));
+	strscpy(info->version, BR_VERSION, sizeof(info->version));
+	strscpy(info->fw_version, "N/A", sizeof(info->fw_version));
+	strscpy(info->bus_info, "N/A", sizeof(info->bus_info));
 }
 
 static int br_get_link_ksettings(struct net_device *dev,
diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c
index a84a7cfb9d6d..228fd5b20f10 100644
--- a/net/bridge/br_if.c
+++ b/net/bridge/br_if.c
@@ -40,12 +40,21 @@ static int port_cost(struct net_device *dev)
 		switch (ecmd.base.speed) {
 		case SPEED_10000:
 			return 2;
-		case SPEED_1000:
+		case SPEED_5000:
+			return 3;
+		case SPEED_2500:
 			return 4;
+		case SPEED_1000:
+			return 5;
 		case SPEED_100:
 			return 19;
 		case SPEED_10:
 			return 100;
+		case SPEED_UNKNOWN:
+			return 100;
+		default:
+			if (ecmd.base.speed > SPEED_10000)
+				return 1;
 		}
 	}
 
@@ -568,26 +577,6 @@ int br_add_if(struct net_bridge *br, struct net_device *dev,
 	    !is_valid_ether_addr(dev->dev_addr))
 		return -EINVAL;
 
-	/* Also don't allow bridging of net devices that are DSA masters, since
-	 * the bridge layer rx_handler prevents the DSA fake ethertype handler
-	 * to be invoked, so we don't get the chance to strip off and parse the
-	 * DSA switch tag protocol header (the bridge layer just returns
-	 * RX_HANDLER_CONSUMED, stopping RX processing for these frames).
-	 * The only case where that would not be an issue is when bridging can
-	 * already be offloaded, such as when the DSA master is itself a DSA
-	 * or plain switchdev port, and is bridged only with other ports from
-	 * the same hardware device.
-	 */
-	if (netdev_uses_dsa(dev)) {
-		list_for_each_entry(p, &br->port_list, list) {
-			if (!netdev_port_same_parent_id(dev, p->dev)) {
-				NL_SET_ERR_MSG(extack,
-					       "Cannot do software bridging with a DSA master");
-				return -EINVAL;
-			}
-		}
-	}
-
 	/* No bridging of bridges */
 	if (dev->netdev_ops->ndo_start_xmit == br_dev_xmit) {
 		NL_SET_ERR_MSG(extack,
diff --git a/net/bridge/br_sysfs_if.c b/net/bridge/br_sysfs_if.c
index 07fa76080512..74fdd8105dca 100644
--- a/net/bridge/br_sysfs_if.c
+++ b/net/bridge/br_sysfs_if.c
@@ -384,7 +384,7 @@ int br_sysfs_addif(struct net_bridge_port *p)
 			return err;
 	}
 
-	strlcpy(p->sysfs_name, p->dev->name, IFNAMSIZ);
+	strscpy(p->sysfs_name, p->dev->name, IFNAMSIZ);
 	return sysfs_create_link(br->ifobj, &p->kobj, p->sysfs_name);
 }
 
@@ -406,7 +406,7 @@ int br_sysfs_renameif(struct net_bridge_port *p)
 		netdev_notice(br->dev, "unable to rename link %s to %s",
 			      p->sysfs_name, p->dev->name);
 	else
-		strlcpy(p->sysfs_name, p->dev->name, IFNAMSIZ);
+		strscpy(p->sysfs_name, p->dev->name, IFNAMSIZ);
 
 	return err;
 }
diff --git a/net/bridge/netfilter/ebtables.c b/net/bridge/netfilter/ebtables.c
index 4f385d52a1c4..ce5dfa3babd2 100644
--- a/net/bridge/netfilter/ebtables.c
+++ b/net/bridge/netfilter/ebtables.c
@@ -1442,7 +1442,7 @@ static inline int ebt_obj_to_user(char __user *um, const char *_name,
 	/* ebtables expects 31 bytes long names but xt_match names are 29 bytes
 	 * long. Copy 29 bytes and fill remaining bytes with zeroes.
 	 */
-	strlcpy(name, _name, sizeof(name));
+	strscpy(name, _name, sizeof(name));
 	if (copy_to_user(um, name, EBT_EXTENSION_MAXNAMELEN) ||
 	    put_user(revision, (u8 __user *)(um + EBT_EXTENSION_MAXNAMELEN)) ||
 	    put_user(datasize, (int __user *)(um + EBT_EXTENSION_MAXNAMELEN + 1)) ||
diff --git a/net/caif/caif_dev.c b/net/caif/caif_dev.c
index 52dd0b6835bc..6a0cba4fc29f 100644
--- a/net/caif/caif_dev.c
+++ b/net/caif/caif_dev.c
@@ -342,7 +342,7 @@ int caif_enroll_dev(struct net_device *dev, struct caif_dev_common *caifdev,
 	mutex_lock(&caifdevs->lock);
 	list_add_rcu(&caifd->list, &caifdevs->list);
 
-	strlcpy(caifd->layer.name, dev->name,
+	strscpy(caifd->layer.name, dev->name,
 		sizeof(caifd->layer.name));
 	caifd->layer.transmit = transmit;
 	res = cfcnfg_add_phy_layer(cfg,
diff --git a/net/caif/caif_usb.c b/net/caif/caif_usb.c
index 4be6b04879a1..ebc202ffdd8d 100644
--- a/net/caif/caif_usb.c
+++ b/net/caif/caif_usb.c
@@ -184,7 +184,7 @@ static int cfusbl_device_notify(struct notifier_block *me, unsigned long what,
 		dev_add_pack(&caif_usb_type);
 	pack_added = true;
 
-	strlcpy(layer->name, dev->name, sizeof(layer->name));
+	strscpy(layer->name, dev->name, sizeof(layer->name));
 
 	return 0;
 err:
diff --git a/net/caif/cfcnfg.c b/net/caif/cfcnfg.c
index 23267c8db7c4..52509e185960 100644
--- a/net/caif/cfcnfg.c
+++ b/net/caif/cfcnfg.c
@@ -268,14 +268,14 @@ static int caif_connect_req_to_link_param(struct cfcnfg *cnfg,
 	case CAIFPROTO_RFM:
 		l->linktype = CFCTRL_SRV_RFM;
 		l->u.datagram.connid = s->sockaddr.u.rfm.connection_id;
-		strlcpy(l->u.rfm.volume, s->sockaddr.u.rfm.volume,
+		strscpy(l->u.rfm.volume, s->sockaddr.u.rfm.volume,
 			sizeof(l->u.rfm.volume));
 		break;
 	case CAIFPROTO_UTIL:
 		l->linktype = CFCTRL_SRV_UTIL;
 		l->endpoint = 0x00;
 		l->chtype = 0x00;
-		strlcpy(l->u.utility.name, s->sockaddr.u.util.service,
+		strscpy(l->u.utility.name, s->sockaddr.u.util.service,
 			sizeof(l->u.utility.name));
 		caif_assert(sizeof(l->u.utility.name) > 10);
 		l->u.utility.paramlen = s->param.size;
diff --git a/net/caif/cfctrl.c b/net/caif/cfctrl.c
index 2809cbd6b7f7..cc405d8c7c30 100644
--- a/net/caif/cfctrl.c
+++ b/net/caif/cfctrl.c
@@ -258,7 +258,7 @@ int cfctrl_linkup_request(struct cflayer *layer,
 		tmp16 = cpu_to_le16(param->u.utility.fifosize_bufs);
 		cfpkt_add_body(pkt, &tmp16, 2);
 		memset(utility_name, 0, sizeof(utility_name));
-		strlcpy(utility_name, param->u.utility.name,
+		strscpy(utility_name, param->u.utility.name,
 			UTILITY_NAME_LENGTH);
 		cfpkt_add_body(pkt, utility_name, UTILITY_NAME_LENGTH);
 		tmp8 = param->u.utility.paramlen;
diff --git a/net/can/af_can.c b/net/can/af_can.c
index 1fb49d51b25d..9503ab10f9b8 100644
--- a/net/can/af_can.c
+++ b/net/can/af_can.c
@@ -199,27 +199,26 @@ static int can_create(struct net *net, struct socket *sock, int protocol,
 int can_send(struct sk_buff *skb, int loop)
 {
 	struct sk_buff *newskb = NULL;
-	struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
 	struct can_pkg_stats *pkg_stats = dev_net(skb->dev)->can.pkg_stats;
 	int err = -EINVAL;
 
-	if (skb->len == CAN_MTU) {
+	if (can_is_canxl_skb(skb)) {
+		skb->protocol = htons(ETH_P_CANXL);
+	} else if (can_is_can_skb(skb)) {
 		skb->protocol = htons(ETH_P_CAN);
-		if (unlikely(cfd->len > CAN_MAX_DLEN))
-			goto inval_skb;
-	} else if (skb->len == CANFD_MTU) {
+	} else if (can_is_canfd_skb(skb)) {
+		struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
+
 		skb->protocol = htons(ETH_P_CANFD);
-		if (unlikely(cfd->len > CANFD_MAX_DLEN))
-			goto inval_skb;
+
+		/* set CAN FD flag for CAN FD frames by default */
+		cfd->flags |= CANFD_FDF;
 	} else {
 		goto inval_skb;
 	}
 
-	/* Make sure the CAN frame can pass the selected CAN netdevice.
-	 * As structs can_frame and canfd_frame are similar, we can provide
-	 * CAN FD frames to legacy CAN drivers as long as the length is <= 8
-	 */
-	if (unlikely(skb->len > skb->dev->mtu && cfd->len > CAN_MAX_DLEN)) {
+	/* Make sure the CAN frame can pass the selected CAN netdevice. */
+	if (unlikely(skb->len > skb->dev->mtu)) {
 		err = -EMSGSIZE;
 		goto inval_skb;
 	}
@@ -678,53 +677,46 @@ static void can_receive(struct sk_buff *skb, struct net_device *dev)
 static int can_rcv(struct sk_buff *skb, struct net_device *dev,
 		   struct packet_type *pt, struct net_device *orig_dev)
 {
-	struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
-
-	if (unlikely(dev->type != ARPHRD_CAN || skb->len != CAN_MTU)) {
+	if (unlikely(dev->type != ARPHRD_CAN || (!can_is_can_skb(skb)))) {
 		pr_warn_once("PF_CAN: dropped non conform CAN skbuff: dev type %d, len %d\n",
 			     dev->type, skb->len);
-		goto free_skb;
-	}
 
-	/* This check is made separately since cfd->len would be uninitialized if skb->len = 0. */
-	if (unlikely(cfd->len > CAN_MAX_DLEN)) {
-		pr_warn_once("PF_CAN: dropped non conform CAN skbuff: dev type %d, len %d, datalen %d\n",
-			     dev->type, skb->len, cfd->len);
-		goto free_skb;
+		kfree_skb(skb);
+		return NET_RX_DROP;
 	}
 
 	can_receive(skb, dev);
 	return NET_RX_SUCCESS;
-
-free_skb:
-	kfree_skb(skb);
-	return NET_RX_DROP;
 }
 
 static int canfd_rcv(struct sk_buff *skb, struct net_device *dev,
 		     struct packet_type *pt, struct net_device *orig_dev)
 {
-	struct canfd_frame *cfd = (struct canfd_frame *)skb->data;
-
-	if (unlikely(dev->type != ARPHRD_CAN || skb->len != CANFD_MTU)) {
+	if (unlikely(dev->type != ARPHRD_CAN || (!can_is_canfd_skb(skb)))) {
 		pr_warn_once("PF_CAN: dropped non conform CAN FD skbuff: dev type %d, len %d\n",
 			     dev->type, skb->len);
-		goto free_skb;
-	}
 
-	/* This check is made separately since cfd->len would be uninitialized if skb->len = 0. */
-	if (unlikely(cfd->len > CANFD_MAX_DLEN)) {
-		pr_warn_once("PF_CAN: dropped non conform CAN FD skbuff: dev type %d, len %d, datalen %d\n",
-			     dev->type, skb->len, cfd->len);
-		goto free_skb;
+		kfree_skb(skb);
+		return NET_RX_DROP;
 	}
 
 	can_receive(skb, dev);
 	return NET_RX_SUCCESS;
+}
 
-free_skb:
-	kfree_skb(skb);
-	return NET_RX_DROP;
+static int canxl_rcv(struct sk_buff *skb, struct net_device *dev,
+		     struct packet_type *pt, struct net_device *orig_dev)
+{
+	if (unlikely(dev->type != ARPHRD_CAN || (!can_is_canxl_skb(skb)))) {
+		pr_warn_once("PF_CAN: dropped non conform CAN XL skbuff: dev type %d, len %d\n",
+			     dev->type, skb->len);
+
+		kfree_skb(skb);
+		return NET_RX_DROP;
+	}
+
+	can_receive(skb, dev);
+	return NET_RX_SUCCESS;
 }
 
 /* af_can protocol functions */
@@ -851,6 +843,11 @@ static struct packet_type canfd_packet __read_mostly = {
 	.func = canfd_rcv,
 };
 
+static struct packet_type canxl_packet __read_mostly = {
+	.type = cpu_to_be16(ETH_P_CANXL),
+	.func = canxl_rcv,
+};
+
 static const struct net_proto_family can_family_ops = {
 	.family = PF_CAN,
 	.create = can_create,
@@ -890,6 +887,7 @@ static __init int can_init(void)
 
 	dev_add_pack(&can_packet);
 	dev_add_pack(&canfd_packet);
+	dev_add_pack(&canxl_packet);
 
 	return 0;
 
diff --git a/net/can/bcm.c b/net/can/bcm.c
index e60161bec850..27706f6ace34 100644
--- a/net/can/bcm.c
+++ b/net/can/bcm.c
@@ -274,6 +274,7 @@ static void bcm_can_tx(struct bcm_op *op)
 	struct sk_buff *skb;
 	struct net_device *dev;
 	struct canfd_frame *cf = op->frames + op->cfsiz * op->currframe;
+	int err;
 
 	/* no target device? => exit */
 	if (!op->ifindex)
@@ -298,11 +299,11 @@ static void bcm_can_tx(struct bcm_op *op)
 	/* send with loopback */
 	skb->dev = dev;
 	can_skb_set_owner(skb, op->sk);
-	can_send(skb, 1);
+	err = can_send(skb, 1);
+	if (!err)
+		op->frames_abs++;
 
-	/* update statistics */
 	op->currframe++;
-	op->frames_abs++;
 
 	/* reached last frame? */
 	if (op->currframe >= op->nframes)
@@ -648,8 +649,13 @@ static void bcm_rx_handler(struct sk_buff *skb, void *data)
 		return;
 
 	/* make sure to handle the correct frame type (CAN / CAN FD) */
-	if (skb->len != op->cfsiz)
-		return;
+	if (op->flags & CAN_FD_FRAME) {
+		if (!can_is_canfd_skb(skb))
+			return;
+	} else {
+		if (!can_is_can_skb(skb))
+			return;
+	}
 
 	/* disable timeout */
 	hrtimer_cancel(&op->timer);
@@ -1744,15 +1750,27 @@ static int __init bcm_module_init(void)
 
 	pr_info("can: broadcast manager protocol\n");
 
+	err = register_pernet_subsys(&canbcm_pernet_ops);
+	if (err)
+		return err;
+
+	err = register_netdevice_notifier(&canbcm_notifier);
+	if (err)
+		goto register_notifier_failed;
+
 	err = can_proto_register(&bcm_can_proto);
 	if (err < 0) {
 		printk(KERN_ERR "can: registration of bcm protocol failed\n");
-		return err;
+		goto register_proto_failed;
 	}
 
-	register_pernet_subsys(&canbcm_pernet_ops);
-	register_netdevice_notifier(&canbcm_notifier);
 	return 0;
+
+register_proto_failed:
+	unregister_netdevice_notifier(&canbcm_notifier);
+register_notifier_failed:
+	unregister_pernet_subsys(&canbcm_pernet_ops);
+	return err;
 }
 
 static void __exit bcm_module_exit(void)
diff --git a/net/can/gw.c b/net/can/gw.c
index 1ea4cc527db3..23a3d89cad81 100644
--- a/net/can/gw.c
+++ b/net/can/gw.c
@@ -463,10 +463,10 @@ static void can_can_gw_rcv(struct sk_buff *skb, void *data)
 
 	/* process strictly Classic CAN or CAN FD frames */
 	if (gwj->flags & CGW_FLAGS_CAN_FD) {
-		if (skb->len != CANFD_MTU)
+		if (!can_is_canfd_skb(skb))
 			return;
 	} else {
-		if (skb->len != CAN_MTU)
+		if (!can_is_can_skb(skb))
 			return;
 	}
 
diff --git a/net/can/isotp.c b/net/can/isotp.c
index 43a27d19cdac..a9d1357f8489 100644
--- a/net/can/isotp.c
+++ b/net/can/isotp.c
@@ -669,7 +669,7 @@ static void isotp_rcv(struct sk_buff *skb, void *data)
 		if (cf->len <= CAN_MAX_DLEN) {
 			isotp_rcv_sf(sk, cf, SF_PCI_SZ4 + ae, skb, sf_dl);
 		} else {
-			if (skb->len == CANFD_MTU) {
+			if (can_is_canfd_skb(skb)) {
 				/* We have a CAN FD frame and CAN_DL is greater than 8:
 				 * Only frames with the SF_DL == 0 ESC value are valid.
 				 *
diff --git a/net/can/j1939/main.c b/net/can/j1939/main.c
index 8452b0fbb78c..144c86b0e3ff 100644
--- a/net/can/j1939/main.c
+++ b/net/can/j1939/main.c
@@ -42,6 +42,10 @@ static void j1939_can_recv(struct sk_buff *iskb, void *data)
 	struct j1939_sk_buff_cb *skcb, *iskcb;
 	struct can_frame *cf;
 
+	/* make sure we only get Classical CAN frames */
+	if (!can_is_can_skb(iskb))
+		return;
+
 	/* create a copy of the skb
 	 * j1939 only delivers the real data bytes,
 	 * the header goes into sockaddr.
diff --git a/net/can/raw.c b/net/can/raw.c
index d1bd9cc51ebe..3eb7d3e2b541 100644
--- a/net/can/raw.c
+++ b/net/can/raw.c
@@ -50,6 +50,7 @@
 #include <linux/skbuff.h>
 #include <linux/can.h>
 #include <linux/can/core.h>
+#include <linux/can/dev.h> /* for can_is_canxl_dev_mtu() */
 #include <linux/can/skb.h>
 #include <linux/can/raw.h>
 #include <net/sock.h>
@@ -87,6 +88,7 @@ struct raw_sock {
 	int loopback;
 	int recv_own_msgs;
 	int fd_frames;
+	int xl_frames;
 	int join_filters;
 	int count;                 /* number of active filters */
 	struct can_filter dfilter; /* default/single filter */
@@ -129,21 +131,21 @@ static void raw_rcv(struct sk_buff *oskb, void *data)
 	if (!ro->recv_own_msgs && oskb->sk == sk)
 		return;
 
-	/* do not pass non-CAN2.0 frames to a legacy socket */
-	if (!ro->fd_frames && oskb->len != CAN_MTU)
+	/* make sure to not pass oversized frames to the socket */
+	if ((can_is_canfd_skb(oskb) && !ro->fd_frames && !ro->xl_frames) ||
+	    (can_is_canxl_skb(oskb) && !ro->xl_frames))
 		return;
 
 	/* eliminate multiple filter matches for the same skb */
 	if (this_cpu_ptr(ro->uniq)->skb == oskb &&
 	    this_cpu_ptr(ro->uniq)->skbcnt == can_skb_prv(oskb)->skbcnt) {
-		if (ro->join_filters) {
-			this_cpu_inc(ro->uniq->join_rx_count);
-			/* drop frame until all enabled filters matched */
-			if (this_cpu_ptr(ro->uniq)->join_rx_count < ro->count)
-				return;
-		} else {
+		if (!ro->join_filters)
+			return;
+
+		this_cpu_inc(ro->uniq->join_rx_count);
+		/* drop frame until all enabled filters matched */
+		if (this_cpu_ptr(ro->uniq)->join_rx_count < ro->count)
 			return;
-		}
 	} else {
 		this_cpu_ptr(ro->uniq)->skb = oskb;
 		this_cpu_ptr(ro->uniq)->skbcnt = can_skb_prv(oskb)->skbcnt;
@@ -346,6 +348,7 @@ static int raw_init(struct sock *sk)
 	ro->loopback         = 1;
 	ro->recv_own_msgs    = 0;
 	ro->fd_frames        = 0;
+	ro->xl_frames        = 0;
 	ro->join_filters     = 0;
 
 	/* alloc_percpu provides zero'ed memory */
@@ -669,6 +672,15 @@ static int raw_setsockopt(struct socket *sock, int level, int optname,
 
 		break;
 
+	case CAN_RAW_XL_FRAMES:
+		if (optlen != sizeof(ro->xl_frames))
+			return -EINVAL;
+
+		if (copy_from_sockptr(&ro->xl_frames, optval, optlen))
+			return -EFAULT;
+
+		break;
+
 	case CAN_RAW_JOIN_FILTERS:
 		if (optlen != sizeof(ro->join_filters))
 			return -EINVAL;
@@ -751,6 +763,12 @@ static int raw_getsockopt(struct socket *sock, int level, int optname,
 		val = &ro->fd_frames;
 		break;
 
+	case CAN_RAW_XL_FRAMES:
+		if (len > sizeof(int))
+			len = sizeof(int);
+		val = &ro->xl_frames;
+		break;
+
 	case CAN_RAW_JOIN_FILTERS:
 		if (len > sizeof(int))
 			len = sizeof(int);
@@ -776,7 +794,11 @@ static int raw_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 	struct sk_buff *skb;
 	struct net_device *dev;
 	int ifindex;
-	int err;
+	int err = -EINVAL;
+
+	/* check for valid CAN frame sizes */
+	if (size < CANXL_HDR_SIZE + CANXL_MIN_DLEN || size > CANXL_MTU)
+		return -EINVAL;
 
 	if (msg->msg_name) {
 		DECLARE_SOCKADDR(struct sockaddr_can *, addr, msg->msg_name);
@@ -796,15 +818,6 @@ static int raw_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 	if (!dev)
 		return -ENXIO;
 
-	err = -EINVAL;
-	if (ro->fd_frames && dev->mtu == CANFD_MTU) {
-		if (unlikely(size != CANFD_MTU && size != CAN_MTU))
-			goto put_dev;
-	} else {
-		if (unlikely(size != CAN_MTU))
-			goto put_dev;
-	}
-
 	skb = sock_alloc_send_skb(sk, size + sizeof(struct can_skb_priv),
 				  msg->msg_flags & MSG_DONTWAIT, &err);
 	if (!skb)
@@ -814,10 +827,27 @@ static int raw_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 	can_skb_prv(skb)->ifindex = dev->ifindex;
 	can_skb_prv(skb)->skbcnt = 0;
 
+	/* fill the skb before testing for valid CAN frames */
 	err = memcpy_from_msg(skb_put(skb, size), msg, size);
 	if (err < 0)
 		goto free_skb;
 
+	err = -EINVAL;
+	if (ro->xl_frames && can_is_canxl_dev_mtu(dev->mtu)) {
+		/* CAN XL, CAN FD and Classical CAN */
+		if (!can_is_canxl_skb(skb) && !can_is_canfd_skb(skb) &&
+		    !can_is_can_skb(skb))
+			goto free_skb;
+	} else if (ro->fd_frames && dev->mtu == CANFD_MTU) {
+		/* CAN FD and Classical CAN */
+		if (!can_is_canfd_skb(skb) && !can_is_can_skb(skb))
+			goto free_skb;
+	} else {
+		/* Classical CAN */
+		if (!can_is_can_skb(skb))
+			goto free_skb;
+	}
+
 	sockcm_init(&sockc, sk);
 	if (msg->msg_controllen) {
 		err = sock_cmsg_send(sk, msg, &sockc);
@@ -942,12 +972,20 @@ static __init int raw_module_init(void)
 
 	pr_info("can: raw protocol\n");
 
+	err = register_netdevice_notifier(&canraw_notifier);
+	if (err)
+		return err;
+
 	err = can_proto_register(&raw_can_proto);
-	if (err < 0)
+	if (err < 0) {
 		pr_err("can: registration of raw protocol failed\n");
-	else
-		register_netdevice_notifier(&canraw_notifier);
+		goto register_proto_failed;
+	}
+
+	return 0;
 
+register_proto_failed:
+	unregister_netdevice_notifier(&canraw_notifier);
 	return err;
 }
 
diff --git a/net/core/dev.c b/net/core/dev.c
index 56c8b0921c9f..fa53830d0683 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1100,7 +1100,7 @@ static int dev_alloc_name_ns(struct net *net,
 	BUG_ON(!net);
 	ret = __dev_alloc_name(net, name, buf);
 	if (ret >= 0)
-		strlcpy(dev->name, buf, IFNAMSIZ);
+		strscpy(dev->name, buf, IFNAMSIZ);
 	return ret;
 }
 
@@ -1137,7 +1137,7 @@ static int dev_get_valid_name(struct net *net, struct net_device *dev,
 	else if (netdev_name_in_use(net, name))
 		return -EEXIST;
 	else if (dev->name != name)
-		strlcpy(dev->name, name, IFNAMSIZ);
+		strscpy(dev->name, name, IFNAMSIZ);
 
 	return 0;
 }
@@ -6358,23 +6358,6 @@ int dev_set_threaded(struct net_device *dev, bool threaded)
 }
 EXPORT_SYMBOL(dev_set_threaded);
 
-/* Double check that napi_get_frags() allocates skbs with
- * skb->head being backed by slab, not a page fragment.
- * This is to make sure bug fixed in 3226b158e67c
- * ("net: avoid 32 x truesize under-estimation for tiny skbs")
- * does not accidentally come back.
- */
-static void napi_get_frags_check(struct napi_struct *napi)
-{
-	struct sk_buff *skb;
-
-	local_bh_disable();
-	skb = napi_get_frags(napi);
-	WARN_ON_ONCE(skb && skb->head_frag);
-	napi_free_frags(napi);
-	local_bh_enable();
-}
-
 void netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
 			   int (*poll)(struct napi_struct *, int), int weight)
 {
@@ -10370,9 +10353,7 @@ void netdev_run_todo(void)
 		BUG_ON(!list_empty(&dev->ptype_specific));
 		WARN_ON(rcu_access_pointer(dev->ip_ptr));
 		WARN_ON(rcu_access_pointer(dev->ip6_ptr));
-#if IS_ENABLED(CONFIG_DECNET)
-		WARN_ON(dev->dn_ptr);
-#endif
+
 		if (dev->priv_destructor)
 			dev->priv_destructor(dev);
 		if (dev->needs_free_netdev)
diff --git a/net/core/devlink.c b/net/core/devlink.c
index b50bcc18b8d9..89baa7c0938b 100644
--- a/net/core/devlink.c
+++ b/net/core/devlink.c
@@ -371,6 +371,13 @@ static struct devlink *devlink_get_from_attrs(struct net *net,
 	return ERR_PTR(-ENODEV);
 }
 
+#define ASSERT_DEVLINK_PORT_REGISTERED(devlink_port)				\
+	WARN_ON_ONCE(!(devlink_port)->registered)
+#define ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port)			\
+	WARN_ON_ONCE((devlink_port)->registered)
+#define ASSERT_DEVLINK_PORT_INITIALIZED(devlink_port)				\
+	WARN_ON_ONCE(!(devlink_port)->initialized)
+
 static struct devlink_port *devlink_port_get_by_index(struct devlink *devlink,
 						      unsigned int port_index)
 {
@@ -1710,7 +1717,7 @@ static int devlink_nl_cmd_port_split_doit(struct sk_buff *skb,
 	struct devlink *devlink = info->user_ptr[0];
 	u32 count;
 
-	if (!info->attrs[DEVLINK_ATTR_PORT_SPLIT_COUNT])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PORT_SPLIT_COUNT))
 		return -EINVAL;
 	if (!devlink->ops->port_split)
 		return -EOPNOTSUPP;
@@ -1838,7 +1845,7 @@ static int devlink_nl_cmd_port_del_doit(struct sk_buff *skb,
 	if (!devlink->ops->port_del)
 		return -EOPNOTSUPP;
 
-	if (!info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PORT_INDEX)) {
 		NL_SET_ERR_MSG_MOD(extack, "Port index is not specified");
 		return -EINVAL;
 	}
@@ -2690,7 +2697,7 @@ static int devlink_nl_cmd_sb_pool_set_doit(struct sk_buff *skb,
 	if (err)
 		return err;
 
-	if (!info->attrs[DEVLINK_ATTR_SB_POOL_SIZE])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_POOL_SIZE))
 		return -EINVAL;
 
 	size = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_POOL_SIZE]);
@@ -2900,7 +2907,7 @@ static int devlink_nl_cmd_sb_port_pool_set_doit(struct sk_buff *skb,
 	if (err)
 		return err;
 
-	if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_THRESHOLD))
 		return -EINVAL;
 
 	threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
@@ -3156,7 +3163,7 @@ static int devlink_nl_cmd_sb_tc_pool_bind_set_doit(struct sk_buff *skb,
 	if (err)
 		return err;
 
-	if (!info->attrs[DEVLINK_ATTR_SB_THRESHOLD])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SB_THRESHOLD))
 		return -EINVAL;
 
 	threshold = nla_get_u32(info->attrs[DEVLINK_ATTR_SB_THRESHOLD]);
@@ -3845,7 +3852,7 @@ static int devlink_nl_cmd_dpipe_entries_get(struct sk_buff *skb,
 	struct devlink_dpipe_table *table;
 	const char *table_name;
 
-	if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_DPIPE_TABLE_NAME))
 		return -EINVAL;
 
 	table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
@@ -4029,8 +4036,9 @@ static int devlink_nl_cmd_dpipe_table_counters_set(struct sk_buff *skb,
 	const char *table_name;
 	bool counters_enable;
 
-	if (!info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME] ||
-	    !info->attrs[DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_DPIPE_TABLE_NAME) ||
+	    GENL_REQ_ATTR_CHECK(info,
+				DEVLINK_ATTR_DPIPE_TABLE_COUNTERS_ENABLED))
 		return -EINVAL;
 
 	table_name = nla_data(info->attrs[DEVLINK_ATTR_DPIPE_TABLE_NAME]);
@@ -4119,8 +4127,8 @@ static int devlink_nl_cmd_resource_set(struct sk_buff *skb,
 	u64 size;
 	int err;
 
-	if (!info->attrs[DEVLINK_ATTR_RESOURCE_ID] ||
-	    !info->attrs[DEVLINK_ATTR_RESOURCE_SIZE])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_RESOURCE_ID) ||
+	    GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_RESOURCE_SIZE))
 		return -EINVAL;
 	resource_id = nla_get_u64(info->attrs[DEVLINK_ATTR_RESOURCE_ID]);
 
@@ -4742,10 +4750,76 @@ void devlink_flash_update_timeout_notify(struct devlink *devlink,
 }
 EXPORT_SYMBOL_GPL(devlink_flash_update_timeout_notify);
 
+struct devlink_info_req {
+	struct sk_buff *msg;
+	void (*version_cb)(const char *version_name,
+			   enum devlink_info_version_type version_type,
+			   void *version_cb_priv);
+	void *version_cb_priv;
+};
+
+struct devlink_flash_component_lookup_ctx {
+	const char *lookup_name;
+	bool lookup_name_found;
+};
+
+static void
+devlink_flash_component_lookup_cb(const char *version_name,
+				  enum devlink_info_version_type version_type,
+				  void *version_cb_priv)
+{
+	struct devlink_flash_component_lookup_ctx *lookup_ctx = version_cb_priv;
+
+	if (version_type != DEVLINK_INFO_VERSION_TYPE_COMPONENT ||
+	    lookup_ctx->lookup_name_found)
+		return;
+
+	lookup_ctx->lookup_name_found =
+		!strcmp(lookup_ctx->lookup_name, version_name);
+}
+
+static int devlink_flash_component_get(struct devlink *devlink,
+				       struct nlattr *nla_component,
+				       const char **p_component,
+				       struct netlink_ext_ack *extack)
+{
+	struct devlink_flash_component_lookup_ctx lookup_ctx = {};
+	struct devlink_info_req req = {};
+	const char *component;
+	int ret;
+
+	if (!nla_component)
+		return 0;
+
+	component = nla_data(nla_component);
+
+	if (!devlink->ops->info_get) {
+		NL_SET_ERR_MSG_ATTR(extack, nla_component,
+				    "component update is not supported by this device");
+		return -EOPNOTSUPP;
+	}
+
+	lookup_ctx.lookup_name = component;
+	req.version_cb = devlink_flash_component_lookup_cb;
+	req.version_cb_priv = &lookup_ctx;
+
+	ret = devlink->ops->info_get(devlink, &req, NULL);
+	if (ret)
+		return ret;
+
+	if (!lookup_ctx.lookup_name_found) {
+		NL_SET_ERR_MSG_ATTR(extack, nla_component,
+				    "selected component is not supported by this device");
+		return -EINVAL;
+	}
+	*p_component = component;
+	return 0;
+}
+
 static int devlink_nl_cmd_flash_update(struct sk_buff *skb,
 				       struct genl_info *info)
 {
-	struct nlattr *nla_component, *nla_overwrite_mask, *nla_file_name;
+	struct nlattr *nla_overwrite_mask, *nla_file_name;
 	struct devlink_flash_update_params params = {};
 	struct devlink *devlink = info->user_ptr[0];
 	const char *file_name;
@@ -4755,20 +4829,16 @@ static int devlink_nl_cmd_flash_update(struct sk_buff *skb,
 	if (!devlink->ops->flash_update)
 		return -EOPNOTSUPP;
 
-	if (!info->attrs[DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_FLASH_UPDATE_FILE_NAME))
 		return -EINVAL;
 
-	supported_params = devlink->ops->supported_flash_update_params;
+	ret = devlink_flash_component_get(devlink,
+					  info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT],
+					  &params.component, info->extack);
+	if (ret)
+		return ret;
 
-	nla_component = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_COMPONENT];
-	if (nla_component) {
-		if (!(supported_params & DEVLINK_SUPPORT_FLASH_UPDATE_COMPONENT)) {
-			NL_SET_ERR_MSG_ATTR(info->extack, nla_component,
-					    "component update is not supported by this device");
-			return -EOPNOTSUPP;
-		}
-		params.component = nla_data(nla_component);
-	}
+	supported_params = devlink->ops->supported_flash_update_params;
 
 	nla_overwrite_mask = info->attrs[DEVLINK_ATTR_FLASH_UPDATE_OVERWRITE_MASK];
 	if (nla_overwrite_mask) {
@@ -4936,10 +5006,8 @@ static int devlink_nl_cmd_selftests_run(struct sk_buff *skb,
 	if (!devlink->ops->selftest_run || !devlink->ops->selftest_check)
 		return -EOPNOTSUPP;
 
-	if (!info->attrs[DEVLINK_ATTR_SELFTESTS]) {
-		NL_SET_ERR_MSG_MOD(info->extack, "selftest required");
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_SELFTESTS))
 		return -EINVAL;
-	}
 
 	attrs = info->attrs[DEVLINK_ATTR_SELFTESTS];
 
@@ -5393,7 +5461,7 @@ static int
 devlink_param_type_get_from_info(struct genl_info *info,
 				 enum devlink_param_type *param_type)
 {
-	if (!info->attrs[DEVLINK_ATTR_PARAM_TYPE])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_TYPE))
 		return -EINVAL;
 
 	switch (nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_TYPE])) {
@@ -5470,7 +5538,7 @@ devlink_param_get_from_info(struct list_head *param_list,
 {
 	char *param_name;
 
-	if (!info->attrs[DEVLINK_ATTR_PARAM_NAME])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_NAME))
 		return NULL;
 
 	param_name = nla_data(info->attrs[DEVLINK_ATTR_PARAM_NAME]);
@@ -5536,7 +5604,7 @@ static int __devlink_nl_cmd_param_set_doit(struct devlink *devlink,
 			return err;
 	}
 
-	if (!info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_PARAM_VALUE_CMODE))
 		return -EINVAL;
 	cmode = nla_get_u8(info->attrs[DEVLINK_ATTR_PARAM_VALUE_CMODE]);
 	if (!devlink_param_cmode_is_supported(param, cmode))
@@ -5574,89 +5642,22 @@ static int devlink_nl_cmd_param_set_doit(struct sk_buff *skb,
 static int devlink_nl_cmd_port_param_get_dumpit(struct sk_buff *msg,
 						struct netlink_callback *cb)
 {
-	struct devlink_param_item *param_item;
-	struct devlink_port *devlink_port;
-	struct devlink *devlink;
-	int start = cb->args[0];
-	unsigned long index;
-	int idx = 0;
-	int err = 0;
-
-	devlinks_xa_for_each_registered_get(sock_net(msg->sk), index, devlink) {
-		devl_lock(devlink);
-		list_for_each_entry(devlink_port, &devlink->port_list, list) {
-			list_for_each_entry(param_item,
-					    &devlink_port->param_list, list) {
-				if (idx < start) {
-					idx++;
-					continue;
-				}
-				err = devlink_nl_param_fill(msg,
-						devlink_port->devlink,
-						devlink_port->index, param_item,
-						DEVLINK_CMD_PORT_PARAM_GET,
-						NETLINK_CB(cb->skb).portid,
-						cb->nlh->nlmsg_seq,
-						NLM_F_MULTI);
-				if (err == -EOPNOTSUPP) {
-					err = 0;
-				} else if (err) {
-					devl_unlock(devlink);
-					devlink_put(devlink);
-					goto out;
-				}
-				idx++;
-			}
-		}
-		devl_unlock(devlink);
-		devlink_put(devlink);
-	}
-out:
-	if (err != -EMSGSIZE)
-		return err;
-
-	cb->args[0] = idx;
+	NL_SET_ERR_MSG_MOD(cb->extack, "Port params are not supported");
 	return msg->len;
 }
 
 static int devlink_nl_cmd_port_param_get_doit(struct sk_buff *skb,
 					      struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
-	struct devlink_param_item *param_item;
-	struct sk_buff *msg;
-	int err;
-
-	param_item = devlink_param_get_from_info(&devlink_port->param_list,
-						 info);
-	if (!param_item)
-		return -EINVAL;
-
-	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
-	if (!msg)
-		return -ENOMEM;
-
-	err = devlink_nl_param_fill(msg, devlink_port->devlink,
-				    devlink_port->index, param_item,
-				    DEVLINK_CMD_PORT_PARAM_GET,
-				    info->snd_portid, info->snd_seq, 0);
-	if (err) {
-		nlmsg_free(msg);
-		return err;
-	}
-
-	return genlmsg_reply(msg, info);
+	NL_SET_ERR_MSG_MOD(info->extack, "Port params are not supported");
+	return -EINVAL;
 }
 
 static int devlink_nl_cmd_port_param_set_doit(struct sk_buff *skb,
 					      struct genl_info *info)
 {
-	struct devlink_port *devlink_port = info->user_ptr[1];
-
-	return __devlink_nl_cmd_param_set_doit(devlink_port->devlink,
-					       devlink_port->index,
-					       &devlink_port->param_list, info,
-					       DEVLINK_CMD_PORT_PARAM_NEW);
+	NL_SET_ERR_MSG_MOD(info->extack, "Port params are not supported");
+	return -EINVAL;
 }
 
 static int devlink_nl_region_snapshot_id_put(struct sk_buff *msg,
@@ -6056,7 +6057,7 @@ static int devlink_nl_cmd_region_get_doit(struct sk_buff *skb,
 	unsigned int index;
 	int err;
 
-	if (!info->attrs[DEVLINK_ATTR_REGION_NAME])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME))
 		return -EINVAL;
 
 	if (info->attrs[DEVLINK_ATTR_PORT_INDEX]) {
@@ -6189,8 +6190,8 @@ static int devlink_nl_cmd_region_del(struct sk_buff *skb,
 	unsigned int index;
 	u32 snapshot_id;
 
-	if (!info->attrs[DEVLINK_ATTR_REGION_NAME] ||
-	    !info->attrs[DEVLINK_ATTR_REGION_SNAPSHOT_ID])
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME) ||
+	    GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_SNAPSHOT_ID))
 		return -EINVAL;
 
 	region_name = nla_data(info->attrs[DEVLINK_ATTR_REGION_NAME]);
@@ -6238,7 +6239,7 @@ devlink_nl_cmd_region_new(struct sk_buff *skb, struct genl_info *info)
 	u8 *data;
 	int err;
 
-	if (!info->attrs[DEVLINK_ATTR_REGION_NAME]) {
+	if (GENL_REQ_ATTR_CHECK(info, DEVLINK_ATTR_REGION_NAME)) {
 		NL_SET_ERR_MSG_MOD(info->extack, "No region name provided");
 		return -EINVAL;
 	}
@@ -6553,18 +6554,18 @@ out_unlock:
 	return err;
 }
 
-struct devlink_info_req {
-	struct sk_buff *msg;
-};
-
 int devlink_info_driver_name_put(struct devlink_info_req *req, const char *name)
 {
+	if (!req->msg)
+		return 0;
 	return nla_put_string(req->msg, DEVLINK_ATTR_INFO_DRIVER_NAME, name);
 }
 EXPORT_SYMBOL_GPL(devlink_info_driver_name_put);
 
 int devlink_info_serial_number_put(struct devlink_info_req *req, const char *sn)
 {
+	if (!req->msg)
+		return 0;
 	return nla_put_string(req->msg, DEVLINK_ATTR_INFO_SERIAL_NUMBER, sn);
 }
 EXPORT_SYMBOL_GPL(devlink_info_serial_number_put);
@@ -6572,6 +6573,8 @@ EXPORT_SYMBOL_GPL(devlink_info_serial_number_put);
 int devlink_info_board_serial_number_put(struct devlink_info_req *req,
 					 const char *bsn)
 {
+	if (!req->msg)
+		return 0;
 	return nla_put_string(req->msg, DEVLINK_ATTR_INFO_BOARD_SERIAL_NUMBER,
 			      bsn);
 }
@@ -6579,11 +6582,19 @@ EXPORT_SYMBOL_GPL(devlink_info_board_serial_number_put);
 
 static int devlink_info_version_put(struct devlink_info_req *req, int attr,
 				    const char *version_name,
-				    const char *version_value)
+				    const char *version_value,
+				    enum devlink_info_version_type version_type)
 {
 	struct nlattr *nest;
 	int err;
 
+	if (req->version_cb)
+		req->version_cb(version_name, version_type,
+				req->version_cb_priv);
+
+	if (!req->msg)
+		return 0;
+
 	nest = nla_nest_start_noflag(req->msg, attr);
 	if (!nest)
 		return -EMSGSIZE;
@@ -6612,7 +6623,8 @@ int devlink_info_version_fixed_put(struct devlink_info_req *req,
 				   const char *version_value)
 {
 	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_FIXED,
-					version_name, version_value);
+					version_name, version_value,
+					DEVLINK_INFO_VERSION_TYPE_NONE);
 }
 EXPORT_SYMBOL_GPL(devlink_info_version_fixed_put);
 
@@ -6621,25 +6633,49 @@ int devlink_info_version_stored_put(struct devlink_info_req *req,
 				    const char *version_value)
 {
 	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
-					version_name, version_value);
+					version_name, version_value,
+					DEVLINK_INFO_VERSION_TYPE_NONE);
 }
 EXPORT_SYMBOL_GPL(devlink_info_version_stored_put);
 
+int devlink_info_version_stored_put_ext(struct devlink_info_req *req,
+					const char *version_name,
+					const char *version_value,
+					enum devlink_info_version_type version_type)
+{
+	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_STORED,
+					version_name, version_value,
+					version_type);
+}
+EXPORT_SYMBOL_GPL(devlink_info_version_stored_put_ext);
+
 int devlink_info_version_running_put(struct devlink_info_req *req,
 				     const char *version_name,
 				     const char *version_value)
 {
 	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
-					version_name, version_value);
+					version_name, version_value,
+					DEVLINK_INFO_VERSION_TYPE_NONE);
 }
 EXPORT_SYMBOL_GPL(devlink_info_version_running_put);
 
+int devlink_info_version_running_put_ext(struct devlink_info_req *req,
+					 const char *version_name,
+					 const char *version_value,
+					 enum devlink_info_version_type version_type)
+{
+	return devlink_info_version_put(req, DEVLINK_ATTR_INFO_VERSION_RUNNING,
+					version_name, version_value,
+					version_type);
+}
+EXPORT_SYMBOL_GPL(devlink_info_version_running_put_ext);
+
 static int
 devlink_nl_info_fill(struct sk_buff *msg, struct devlink *devlink,
 		     enum devlink_command cmd, u32 portid,
 		     u32 seq, int flags, struct netlink_ext_ack *extack)
 {
-	struct devlink_info_req req;
+	struct devlink_info_req req = {};
 	void *hdr;
 	int err;
 
@@ -9513,6 +9549,7 @@ static struct genl_family devlink_nl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= devlink_nl_ops,
 	.n_small_ops	= ARRAY_SIZE(devlink_nl_ops),
+	.resv_start_op	= DEVLINK_CMD_SELFTESTS_RUN + 1,
 	.mcgrps		= devlink_nl_mcgrps,
 	.n_mcgrps	= ARRAY_SIZE(devlink_nl_mcgrps),
 };
@@ -9818,6 +9855,44 @@ static void devlink_port_type_warn_cancel(struct devlink_port *devlink_port)
 }
 
 /**
+ * devlink_port_init() - Init devlink port
+ *
+ * @devlink: devlink
+ * @devlink_port: devlink port
+ *
+ * Initialize essencial stuff that is needed for functions
+ * that may be called before devlink port registration.
+ * Call to this function is optional and not needed
+ * in case the driver does not use such functions.
+ */
+void devlink_port_init(struct devlink *devlink,
+		       struct devlink_port *devlink_port)
+{
+	if (devlink_port->initialized)
+		return;
+	devlink_port->devlink = devlink;
+	INIT_LIST_HEAD(&devlink_port->region_list);
+	devlink_port->initialized = true;
+}
+EXPORT_SYMBOL_GPL(devlink_port_init);
+
+/**
+ * devlink_port_fini() - Deinitialize devlink port
+ *
+ * @devlink_port: devlink port
+ *
+ * Deinitialize essencial stuff that is in use for functions
+ * that may be called after devlink port unregistration.
+ * Call to this function is optional and not needed
+ * in case the driver does not use such functions.
+ */
+void devlink_port_fini(struct devlink_port *devlink_port)
+{
+	WARN_ON(!list_empty(&devlink_port->region_list));
+}
+EXPORT_SYMBOL_GPL(devlink_port_fini);
+
+/**
  * devl_port_register() - Register devlink port
  *
  * @devlink: devlink
@@ -9839,15 +9914,15 @@ int devl_port_register(struct devlink *devlink,
 	if (devlink_port_index_exists(devlink, port_index))
 		return -EEXIST;
 
-	WARN_ON(devlink_port->devlink);
-	devlink_port->devlink = devlink;
+	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+
+	devlink_port_init(devlink, devlink_port);
+	devlink_port->registered = true;
 	devlink_port->index = port_index;
 	spin_lock_init(&devlink_port->type_lock);
 	INIT_LIST_HEAD(&devlink_port->reporter_list);
 	mutex_init(&devlink_port->reporters_lock);
 	list_add_tail(&devlink_port->list, &devlink->port_list);
-	INIT_LIST_HEAD(&devlink_port->param_list);
-	INIT_LIST_HEAD(&devlink_port->region_list);
 
 	INIT_DELAYED_WORK(&devlink_port->type_warn_dw, &devlink_port_type_warn);
 	devlink_port_type_warn_schedule(devlink_port);
@@ -9897,8 +9972,8 @@ void devl_port_unregister(struct devlink_port *devlink_port)
 	devlink_port_notify(devlink_port, DEVLINK_CMD_PORT_DEL);
 	list_del(&devlink_port->list);
 	WARN_ON(!list_empty(&devlink_port->reporter_list));
-	WARN_ON(!list_empty(&devlink_port->region_list));
 	mutex_destroy(&devlink_port->reporters_lock);
+	devlink_port->registered = false;
 }
 EXPORT_SYMBOL_GPL(devl_port_unregister);
 
@@ -9923,8 +9998,8 @@ static void __devlink_port_type_set(struct devlink_port *devlink_port,
 				    enum devlink_port_type type,
 				    void *type_dev)
 {
-	if (WARN_ON(!devlink_port->devlink))
-		return;
+	ASSERT_DEVLINK_PORT_REGISTERED(devlink_port);
+
 	devlink_port_type_warn_cancel(devlink_port);
 	spin_lock_bh(&devlink_port->type_lock);
 	devlink_port->type = type;
@@ -10043,8 +10118,8 @@ void devlink_port_attrs_set(struct devlink_port *devlink_port,
 {
 	int ret;
 
-	if (WARN_ON(devlink_port->devlink))
-		return;
+	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+
 	devlink_port->attrs = *attrs;
 	ret = __devlink_port_attrs_set(devlink_port, attrs->flavour);
 	if (ret)
@@ -10067,8 +10142,8 @@ void devlink_port_attrs_pci_pf_set(struct devlink_port *devlink_port, u32 contro
 	struct devlink_port_attrs *attrs = &devlink_port->attrs;
 	int ret;
 
-	if (WARN_ON(devlink_port->devlink))
-		return;
+	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+
 	ret = __devlink_port_attrs_set(devlink_port,
 				       DEVLINK_PORT_FLAVOUR_PCI_PF);
 	if (ret)
@@ -10094,8 +10169,8 @@ void devlink_port_attrs_pci_vf_set(struct devlink_port *devlink_port, u32 contro
 	struct devlink_port_attrs *attrs = &devlink_port->attrs;
 	int ret;
 
-	if (WARN_ON(devlink_port->devlink))
-		return;
+	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+
 	ret = __devlink_port_attrs_set(devlink_port,
 				       DEVLINK_PORT_FLAVOUR_PCI_VF);
 	if (ret)
@@ -10122,8 +10197,8 @@ void devlink_port_attrs_pci_sf_set(struct devlink_port *devlink_port, u32 contro
 	struct devlink_port_attrs *attrs = &devlink_port->attrs;
 	int ret;
 
-	if (WARN_ON(devlink_port->devlink))
-		return;
+	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+
 	ret = __devlink_port_attrs_set(devlink_port,
 				       DEVLINK_PORT_FLAVOUR_PCI_SF);
 	if (ret)
@@ -10238,8 +10313,8 @@ EXPORT_SYMBOL_GPL(devl_rate_nodes_destroy);
 void devlink_port_linecard_set(struct devlink_port *devlink_port,
 			       struct devlink_linecard *linecard)
 {
-	if (WARN_ON(devlink_port->devlink))
-		return;
+	ASSERT_DEVLINK_PORT_NOT_REGISTERED(devlink_port);
+
 	devlink_port->linecard = linecard;
 }
 EXPORT_SYMBOL_GPL(devlink_port_linecard_set);
@@ -11310,6 +11385,8 @@ devlink_port_region_create(struct devlink_port *port,
 	struct devlink_region *region;
 	int err = 0;
 
+	ASSERT_DEVLINK_PORT_INITIALIZED(port);
+
 	if (WARN_ON(!ops) || WARN_ON(!ops->destructor))
 		return ERR_PTR(-EINVAL);
 
@@ -12306,8 +12383,8 @@ EXPORT_SYMBOL_GPL(devl_trap_policers_unregister);
 static void __devlink_compat_running_version(struct devlink *devlink,
 					     char *buf, size_t len)
 {
+	struct devlink_info_req req = {};
 	const struct nlattr *nlattr;
-	struct devlink_info_req req;
 	struct sk_buff *msg;
 	int rem, err;
 
diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c
index 75501e1bdd25..f084a4a6b7ab 100644
--- a/net/core/drop_monitor.c
+++ b/net/core/drop_monitor.c
@@ -464,7 +464,7 @@ net_dm_hw_trap_summary_probe(void *ignore, const struct devlink *devlink,
 		goto out;
 
 	hw_entry = &hw_entries->entries[hw_entries->num_entries];
-	strlcpy(hw_entry->trap_name, metadata->trap_name,
+	strscpy(hw_entry->trap_name, metadata->trap_name,
 		NET_DM_MAX_HW_TRAP_NAME_LEN - 1);
 	hw_entry->count = 1;
 	hw_entries->num_entries++;
@@ -1645,6 +1645,7 @@ static struct genl_family net_drop_monitor_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= dropmon_ops,
 	.n_small_ops	= ARRAY_SIZE(dropmon_ops),
+	.resv_start_op	= NET_DM_CMD_STATS_GET + 1,
 	.mcgrps		= dropmon_mcgrps,
 	.n_mcgrps	= ARRAY_SIZE(dropmon_mcgrps),
 };
diff --git a/net/core/filter.c b/net/core/filter.c
index c191db80ce93..bb0136e7a8e4 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -18,6 +18,7 @@
  */
 
 #include <linux/atomic.h>
+#include <linux/bpf_verifier.h>
 #include <linux/module.h>
 #include <linux/types.h>
 #include <linux/mm.h>
@@ -3010,7 +3011,7 @@ BPF_CALL_0(bpf_get_cgroup_classid_curr)
 	return __task_get_classid(current);
 }
 
-static const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = {
+const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto = {
 	.func		= bpf_get_cgroup_classid_curr,
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
@@ -4489,7 +4490,8 @@ BPF_CALL_4(bpf_skb_get_tunnel_key, struct sk_buff *, skb, struct bpf_tunnel_key
 	void *to_orig = to;
 	int err;
 
-	if (unlikely(!info || (flags & ~(BPF_F_TUNINFO_IPV6)))) {
+	if (unlikely(!info || (flags & ~(BPF_F_TUNINFO_IPV6 |
+					 BPF_F_TUNINFO_FLAGS)))) {
 		err = -EINVAL;
 		goto err_clear;
 	}
@@ -4521,7 +4523,10 @@ set_compat:
 	to->tunnel_id = be64_to_cpu(info->key.tun_id);
 	to->tunnel_tos = info->key.tos;
 	to->tunnel_ttl = info->key.ttl;
-	to->tunnel_ext = 0;
+	if (flags & BPF_F_TUNINFO_FLAGS)
+		to->tunnel_flags = info->key.tun_flags;
+	else
+		to->tunnel_ext = 0;
 
 	if (flags & BPF_F_TUNINFO_IPV6) {
 		memcpy(to->remote_ipv6, &info->key.u.ipv6.src,
@@ -5014,359 +5019,303 @@ static const struct bpf_func_proto bpf_get_socket_uid_proto = {
 	.arg1_type      = ARG_PTR_TO_CTX,
 };
 
-static int __bpf_setsockopt(struct sock *sk, int level, int optname,
-			    char *optval, int optlen)
+static int sol_socket_sockopt(struct sock *sk, int optname,
+			      char *optval, int *optlen,
+			      bool getopt)
+{
+	switch (optname) {
+	case SO_REUSEADDR:
+	case SO_SNDBUF:
+	case SO_RCVBUF:
+	case SO_KEEPALIVE:
+	case SO_PRIORITY:
+	case SO_REUSEPORT:
+	case SO_RCVLOWAT:
+	case SO_MARK:
+	case SO_MAX_PACING_RATE:
+	case SO_BINDTOIFINDEX:
+	case SO_TXREHASH:
+		if (*optlen != sizeof(int))
+			return -EINVAL;
+		break;
+	case SO_BINDTODEVICE:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (getopt) {
+		if (optname == SO_BINDTODEVICE)
+			return -EINVAL;
+		return sk_getsockopt(sk, SOL_SOCKET, optname,
+				     KERNEL_SOCKPTR(optval),
+				     KERNEL_SOCKPTR(optlen));
+	}
+
+	return sk_setsockopt(sk, SOL_SOCKET, optname,
+			     KERNEL_SOCKPTR(optval), *optlen);
+}
+
+static int bpf_sol_tcp_setsockopt(struct sock *sk, int optname,
+				  char *optval, int optlen)
 {
-	char devname[IFNAMSIZ];
-	int val, valbool;
-	struct net *net;
-	int ifindex;
-	int ret = 0;
+	struct tcp_sock *tp = tcp_sk(sk);
+	unsigned long timeout;
+	int val;
 
-	if (!sk_fullsock(sk))
+	if (optlen != sizeof(int))
 		return -EINVAL;
 
-	if (level == SOL_SOCKET) {
-		if (optlen != sizeof(int) && optname != SO_BINDTODEVICE)
+	val = *(int *)optval;
+
+	/* Only some options are supported */
+	switch (optname) {
+	case TCP_BPF_IW:
+		if (val <= 0 || tp->data_segs_out > tp->syn_data)
 			return -EINVAL;
-		val = *((int *)optval);
-		valbool = val ? 1 : 0;
-
-		/* Only some socketops are supported */
-		switch (optname) {
-		case SO_RCVBUF:
-			val = min_t(u32, val, READ_ONCE(sysctl_rmem_max));
-			val = min_t(int, val, INT_MAX / 2);
-			sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
-			WRITE_ONCE(sk->sk_rcvbuf,
-				   max_t(int, val * 2, SOCK_MIN_RCVBUF));
-			break;
-		case SO_SNDBUF:
-			val = min_t(u32, val, READ_ONCE(sysctl_wmem_max));
-			val = min_t(int, val, INT_MAX / 2);
-			sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
-			WRITE_ONCE(sk->sk_sndbuf,
-				   max_t(int, val * 2, SOCK_MIN_SNDBUF));
-			break;
-		case SO_MAX_PACING_RATE: /* 32bit version */
-			if (val != ~0U)
-				cmpxchg(&sk->sk_pacing_status,
-					SK_PACING_NONE,
-					SK_PACING_NEEDED);
-			sk->sk_max_pacing_rate = (val == ~0U) ?
-						 ~0UL : (unsigned int)val;
-			sk->sk_pacing_rate = min(sk->sk_pacing_rate,
-						 sk->sk_max_pacing_rate);
-			break;
-		case SO_PRIORITY:
-			sk->sk_priority = val;
-			break;
-		case SO_RCVLOWAT:
-			if (val < 0)
-				val = INT_MAX;
-			if (sk->sk_socket && sk->sk_socket->ops->set_rcvlowat)
-				ret = sk->sk_socket->ops->set_rcvlowat(sk, val);
-			else
-				WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
-			break;
-		case SO_MARK:
-			if (sk->sk_mark != val) {
-				sk->sk_mark = val;
-				sk_dst_reset(sk);
-			}
-			break;
-		case SO_BINDTODEVICE:
-			optlen = min_t(long, optlen, IFNAMSIZ - 1);
-			strncpy(devname, optval, optlen);
-			devname[optlen] = 0;
+		tcp_snd_cwnd_set(tp, val);
+		break;
+	case TCP_BPF_SNDCWND_CLAMP:
+		if (val <= 0)
+			return -EINVAL;
+		tp->snd_cwnd_clamp = val;
+		tp->snd_ssthresh = val;
+		break;
+	case TCP_BPF_DELACK_MAX:
+		timeout = usecs_to_jiffies(val);
+		if (timeout > TCP_DELACK_MAX ||
+		    timeout < TCP_TIMEOUT_MIN)
+			return -EINVAL;
+		inet_csk(sk)->icsk_delack_max = timeout;
+		break;
+	case TCP_BPF_RTO_MIN:
+		timeout = usecs_to_jiffies(val);
+		if (timeout > TCP_RTO_MIN ||
+		    timeout < TCP_TIMEOUT_MIN)
+			return -EINVAL;
+		inet_csk(sk)->icsk_rto_min = timeout;
+		break;
+	default:
+		return -EINVAL;
+	}
 
-			ifindex = 0;
-			if (devname[0] != '\0') {
-				struct net_device *dev;
+	return 0;
+}
 
-				ret = -ENODEV;
+static int sol_tcp_sockopt_congestion(struct sock *sk, char *optval,
+				      int *optlen, bool getopt)
+{
+	struct tcp_sock *tp;
+	int ret;
 
-				net = sock_net(sk);
-				dev = dev_get_by_name(net, devname);
-				if (!dev)
-					break;
-				ifindex = dev->ifindex;
-				dev_put(dev);
-			}
-			fallthrough;
-		case SO_BINDTOIFINDEX:
-			if (optname == SO_BINDTOIFINDEX)
-				ifindex = val;
-			ret = sock_bindtoindex(sk, ifindex, false);
-			break;
-		case SO_KEEPALIVE:
-			if (sk->sk_prot->keepalive)
-				sk->sk_prot->keepalive(sk, valbool);
-			sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
-			break;
-		case SO_REUSEPORT:
-			sk->sk_reuseport = valbool;
-			break;
-		case SO_TXREHASH:
-			if (val < -1 || val > 1) {
-				ret = -EINVAL;
-				break;
-			}
-			sk->sk_txrehash = (u8)val;
-			break;
-		default:
-			ret = -EINVAL;
-		}
-#ifdef CONFIG_INET
-	} else if (level == SOL_IP) {
-		if (optlen != sizeof(int) || sk->sk_family != AF_INET)
+	if (*optlen < 2)
+		return -EINVAL;
+
+	if (getopt) {
+		if (!inet_csk(sk)->icsk_ca_ops)
 			return -EINVAL;
+		/* BPF expects NULL-terminated tcp-cc string */
+		optval[--(*optlen)] = '\0';
+		return do_tcp_getsockopt(sk, SOL_TCP, TCP_CONGESTION,
+					 KERNEL_SOCKPTR(optval),
+					 KERNEL_SOCKPTR(optlen));
+	}
 
-		val = *((int *)optval);
-		/* Only some options are supported */
-		switch (optname) {
-		case IP_TOS:
-			if (val < -1 || val > 0xff) {
-				ret = -EINVAL;
-			} else {
-				struct inet_sock *inet = inet_sk(sk);
+	/* "cdg" is the only cc that alloc a ptr
+	 * in inet_csk_ca area.  The bpf-tcp-cc may
+	 * overwrite this ptr after switching to cdg.
+	 */
+	if (*optlen >= sizeof("cdg") - 1 && !strncmp("cdg", optval, *optlen))
+		return -ENOTSUPP;
 
-				if (val == -1)
-					val = 0;
-				inet->tos = val;
-			}
-			break;
-		default:
-			ret = -EINVAL;
-		}
-#if IS_ENABLED(CONFIG_IPV6)
-	} else if (level == SOL_IPV6) {
-		if (optlen != sizeof(int) || sk->sk_family != AF_INET6)
-			return -EINVAL;
+	/* It stops this looping
+	 *
+	 * .init => bpf_setsockopt(tcp_cc) => .init =>
+	 * bpf_setsockopt(tcp_cc)" => .init => ....
+	 *
+	 * The second bpf_setsockopt(tcp_cc) is not allowed
+	 * in order to break the loop when both .init
+	 * are the same bpf prog.
+	 *
+	 * This applies even the second bpf_setsockopt(tcp_cc)
+	 * does not cause a loop.  This limits only the first
+	 * '.init' can call bpf_setsockopt(TCP_CONGESTION) to
+	 * pick a fallback cc (eg. peer does not support ECN)
+	 * and the second '.init' cannot fallback to
+	 * another.
+	 */
+	tp = tcp_sk(sk);
+	if (tp->bpf_chg_cc_inprogress)
+		return -EBUSY;
+
+	tp->bpf_chg_cc_inprogress = 1;
+	ret = do_tcp_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
+				KERNEL_SOCKPTR(optval), *optlen);
+	tp->bpf_chg_cc_inprogress = 0;
+	return ret;
+}
 
-		val = *((int *)optval);
-		/* Only some options are supported */
-		switch (optname) {
-		case IPV6_TCLASS:
-			if (val < -1 || val > 0xff) {
-				ret = -EINVAL;
-			} else {
-				struct ipv6_pinfo *np = inet6_sk(sk);
+static int sol_tcp_sockopt(struct sock *sk, int optname,
+			   char *optval, int *optlen,
+			   bool getopt)
+{
+	if (sk->sk_prot->setsockopt != tcp_setsockopt)
+		return -EINVAL;
 
-				if (val == -1)
-					val = 0;
-				np->tclass = val;
-			}
-			break;
-		default:
-			ret = -EINVAL;
-		}
-#endif
-	} else if (level == SOL_TCP &&
-		   sk->sk_prot->setsockopt == tcp_setsockopt) {
-		if (optname == TCP_CONGESTION) {
-			char name[TCP_CA_NAME_MAX];
-
-			strncpy(name, optval, min_t(long, optlen,
-						    TCP_CA_NAME_MAX-1));
-			name[TCP_CA_NAME_MAX-1] = 0;
-			ret = tcp_set_congestion_control(sk, name, false, true);
-		} else {
-			struct inet_connection_sock *icsk = inet_csk(sk);
+	switch (optname) {
+	case TCP_NODELAY:
+	case TCP_MAXSEG:
+	case TCP_KEEPIDLE:
+	case TCP_KEEPINTVL:
+	case TCP_KEEPCNT:
+	case TCP_SYNCNT:
+	case TCP_WINDOW_CLAMP:
+	case TCP_THIN_LINEAR_TIMEOUTS:
+	case TCP_USER_TIMEOUT:
+	case TCP_NOTSENT_LOWAT:
+	case TCP_SAVE_SYN:
+		if (*optlen != sizeof(int))
+			return -EINVAL;
+		break;
+	case TCP_CONGESTION:
+		return sol_tcp_sockopt_congestion(sk, optval, optlen, getopt);
+	case TCP_SAVED_SYN:
+		if (*optlen < 1)
+			return -EINVAL;
+		break;
+	default:
+		if (getopt)
+			return -EINVAL;
+		return bpf_sol_tcp_setsockopt(sk, optname, optval, *optlen);
+	}
+
+	if (getopt) {
+		if (optname == TCP_SAVED_SYN) {
 			struct tcp_sock *tp = tcp_sk(sk);
-			unsigned long timeout;
 
-			if (optlen != sizeof(int))
+			if (!tp->saved_syn ||
+			    *optlen > tcp_saved_syn_len(tp->saved_syn))
 				return -EINVAL;
-
-			val = *((int *)optval);
-			/* Only some options are supported */
-			switch (optname) {
-			case TCP_BPF_IW:
-				if (val <= 0 || tp->data_segs_out > tp->syn_data)
-					ret = -EINVAL;
-				else
-					tcp_snd_cwnd_set(tp, val);
-				break;
-			case TCP_BPF_SNDCWND_CLAMP:
-				if (val <= 0) {
-					ret = -EINVAL;
-				} else {
-					tp->snd_cwnd_clamp = val;
-					tp->snd_ssthresh = val;
-				}
-				break;
-			case TCP_BPF_DELACK_MAX:
-				timeout = usecs_to_jiffies(val);
-				if (timeout > TCP_DELACK_MAX ||
-				    timeout < TCP_TIMEOUT_MIN)
-					return -EINVAL;
-				inet_csk(sk)->icsk_delack_max = timeout;
-				break;
-			case TCP_BPF_RTO_MIN:
-				timeout = usecs_to_jiffies(val);
-				if (timeout > TCP_RTO_MIN ||
-				    timeout < TCP_TIMEOUT_MIN)
-					return -EINVAL;
-				inet_csk(sk)->icsk_rto_min = timeout;
-				break;
-			case TCP_SAVE_SYN:
-				if (val < 0 || val > 1)
-					ret = -EINVAL;
-				else
-					tp->save_syn = val;
-				break;
-			case TCP_KEEPIDLE:
-				ret = tcp_sock_set_keepidle_locked(sk, val);
-				break;
-			case TCP_KEEPINTVL:
-				if (val < 1 || val > MAX_TCP_KEEPINTVL)
-					ret = -EINVAL;
-				else
-					tp->keepalive_intvl = val * HZ;
-				break;
-			case TCP_KEEPCNT:
-				if (val < 1 || val > MAX_TCP_KEEPCNT)
-					ret = -EINVAL;
-				else
-					tp->keepalive_probes = val;
-				break;
-			case TCP_SYNCNT:
-				if (val < 1 || val > MAX_TCP_SYNCNT)
-					ret = -EINVAL;
-				else
-					icsk->icsk_syn_retries = val;
-				break;
-			case TCP_USER_TIMEOUT:
-				if (val < 0)
-					ret = -EINVAL;
-				else
-					icsk->icsk_user_timeout = val;
-				break;
-			case TCP_NOTSENT_LOWAT:
-				tp->notsent_lowat = val;
-				sk->sk_write_space(sk);
-				break;
-			case TCP_WINDOW_CLAMP:
-				ret = tcp_set_window_clamp(sk, val);
-				break;
-			default:
-				ret = -EINVAL;
-			}
+			memcpy(optval, tp->saved_syn->data, *optlen);
+			/* It cannot free tp->saved_syn here because it
+			 * does not know if the user space still needs it.
+			 */
+			return 0;
 		}
-#endif
-	} else {
-		ret = -EINVAL;
+
+		return do_tcp_getsockopt(sk, SOL_TCP, optname,
+					 KERNEL_SOCKPTR(optval),
+					 KERNEL_SOCKPTR(optlen));
 	}
-	return ret;
+
+	return do_tcp_setsockopt(sk, SOL_TCP, optname,
+				 KERNEL_SOCKPTR(optval), *optlen);
 }
 
-static int _bpf_setsockopt(struct sock *sk, int level, int optname,
-			   char *optval, int optlen)
+static int sol_ip_sockopt(struct sock *sk, int optname,
+			  char *optval, int *optlen,
+			  bool getopt)
 {
-	if (sk_fullsock(sk))
-		sock_owned_by_me(sk);
-	return __bpf_setsockopt(sk, level, optname, optval, optlen);
+	if (sk->sk_family != AF_INET)
+		return -EINVAL;
+
+	switch (optname) {
+	case IP_TOS:
+		if (*optlen != sizeof(int))
+			return -EINVAL;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (getopt)
+		return do_ip_getsockopt(sk, SOL_IP, optname,
+					KERNEL_SOCKPTR(optval),
+					KERNEL_SOCKPTR(optlen));
+
+	return do_ip_setsockopt(sk, SOL_IP, optname,
+				KERNEL_SOCKPTR(optval), *optlen);
 }
 
-static int __bpf_getsockopt(struct sock *sk, int level, int optname,
-			    char *optval, int optlen)
+static int sol_ipv6_sockopt(struct sock *sk, int optname,
+			    char *optval, int *optlen,
+			    bool getopt)
 {
-	if (!sk_fullsock(sk))
-		goto err_clear;
+	if (sk->sk_family != AF_INET6)
+		return -EINVAL;
 
-	if (level == SOL_SOCKET) {
-		if (optlen != sizeof(int))
-			goto err_clear;
+	switch (optname) {
+	case IPV6_TCLASS:
+	case IPV6_AUTOFLOWLABEL:
+		if (*optlen != sizeof(int))
+			return -EINVAL;
+		break;
+	default:
+		return -EINVAL;
+	}
 
-		switch (optname) {
-		case SO_RCVBUF:
-			*((int *)optval) = sk->sk_rcvbuf;
-			break;
-		case SO_SNDBUF:
-			*((int *)optval) = sk->sk_sndbuf;
-			break;
-		case SO_MARK:
-			*((int *)optval) = sk->sk_mark;
-			break;
-		case SO_PRIORITY:
-			*((int *)optval) = sk->sk_priority;
-			break;
-		case SO_BINDTOIFINDEX:
-			*((int *)optval) = sk->sk_bound_dev_if;
-			break;
-		case SO_REUSEPORT:
-			*((int *)optval) = sk->sk_reuseport;
-			break;
-		case SO_TXREHASH:
-			*((int *)optval) = sk->sk_txrehash;
-			break;
-		default:
-			goto err_clear;
-		}
-#ifdef CONFIG_INET
-	} else if (level == SOL_TCP && sk->sk_prot->getsockopt == tcp_getsockopt) {
-		struct inet_connection_sock *icsk;
-		struct tcp_sock *tp;
+	if (getopt)
+		return ipv6_bpf_stub->ipv6_getsockopt(sk, SOL_IPV6, optname,
+						      KERNEL_SOCKPTR(optval),
+						      KERNEL_SOCKPTR(optlen));
 
-		switch (optname) {
-		case TCP_CONGESTION:
-			icsk = inet_csk(sk);
+	return ipv6_bpf_stub->ipv6_setsockopt(sk, SOL_IPV6, optname,
+					      KERNEL_SOCKPTR(optval), *optlen);
+}
 
-			if (!icsk->icsk_ca_ops || optlen <= 1)
-				goto err_clear;
-			strncpy(optval, icsk->icsk_ca_ops->name, optlen);
-			optval[optlen - 1] = 0;
-			break;
-		case TCP_SAVED_SYN:
-			tp = tcp_sk(sk);
+static int __bpf_setsockopt(struct sock *sk, int level, int optname,
+			    char *optval, int optlen)
+{
+	if (!sk_fullsock(sk))
+		return -EINVAL;
 
-			if (optlen <= 0 || !tp->saved_syn ||
-			    optlen > tcp_saved_syn_len(tp->saved_syn))
-				goto err_clear;
-			memcpy(optval, tp->saved_syn->data, optlen);
-			break;
-		default:
-			goto err_clear;
-		}
-	} else if (level == SOL_IP) {
-		struct inet_sock *inet = inet_sk(sk);
+	if (level == SOL_SOCKET)
+		return sol_socket_sockopt(sk, optname, optval, &optlen, false);
+	else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
+		return sol_ip_sockopt(sk, optname, optval, &optlen, false);
+	else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
+		return sol_ipv6_sockopt(sk, optname, optval, &optlen, false);
+	else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
+		return sol_tcp_sockopt(sk, optname, optval, &optlen, false);
 
-		if (optlen != sizeof(int) || sk->sk_family != AF_INET)
-			goto err_clear;
+	return -EINVAL;
+}
 
-		/* Only some options are supported */
-		switch (optname) {
-		case IP_TOS:
-			*((int *)optval) = (int)inet->tos;
-			break;
-		default:
-			goto err_clear;
-		}
-#if IS_ENABLED(CONFIG_IPV6)
-	} else if (level == SOL_IPV6) {
-		struct ipv6_pinfo *np = inet6_sk(sk);
+static int _bpf_setsockopt(struct sock *sk, int level, int optname,
+			   char *optval, int optlen)
+{
+	if (sk_fullsock(sk))
+		sock_owned_by_me(sk);
+	return __bpf_setsockopt(sk, level, optname, optval, optlen);
+}
 
-		if (optlen != sizeof(int) || sk->sk_family != AF_INET6)
-			goto err_clear;
+static int __bpf_getsockopt(struct sock *sk, int level, int optname,
+			    char *optval, int optlen)
+{
+	int err, saved_optlen = optlen;
 
-		/* Only some options are supported */
-		switch (optname) {
-		case IPV6_TCLASS:
-			*((int *)optval) = (int)np->tclass;
-			break;
-		default:
-			goto err_clear;
-		}
-#endif
-#endif
-	} else {
-		goto err_clear;
+	if (!sk_fullsock(sk)) {
+		err = -EINVAL;
+		goto done;
 	}
-	return 0;
-err_clear:
-	memset(optval, 0, optlen);
-	return -EINVAL;
+
+	if (level == SOL_SOCKET)
+		err = sol_socket_sockopt(sk, optname, optval, &optlen, true);
+	else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
+		err = sol_tcp_sockopt(sk, optname, optval, &optlen, true);
+	else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
+		err = sol_ip_sockopt(sk, optname, optval, &optlen, true);
+	else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
+		err = sol_ipv6_sockopt(sk, optname, optval, &optlen, true);
+	else
+		err = -EINVAL;
+
+done:
+	if (err)
+		optlen = 0;
+	if (optlen < saved_optlen)
+		memset(optval + optlen, 0, saved_optlen - optlen);
+	return err;
 }
 
 static int _bpf_getsockopt(struct sock *sk, int level, int optname,
@@ -5380,12 +5329,6 @@ static int _bpf_getsockopt(struct sock *sk, int level, int optname,
 BPF_CALL_5(bpf_sk_setsockopt, struct sock *, sk, int, level,
 	   int, optname, char *, optval, int, optlen)
 {
-	if (level == SOL_TCP && optname == TCP_CONGESTION) {
-		if (optlen >= sizeof("cdg") - 1 &&
-		    !strncmp("cdg", optval, optlen))
-			return -ENOTSUPP;
-	}
-
 	return _bpf_setsockopt(sk, level, optname, optval, optlen);
 }
 
@@ -6469,6 +6412,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_adjust_srh_proto = {
 static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
 			      int dif, int sdif, u8 family, u8 proto)
 {
+	struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo;
 	bool refcounted = false;
 	struct sock *sk = NULL;
 
@@ -6477,7 +6421,7 @@ static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
 		__be32 dst4 = tuple->ipv4.daddr;
 
 		if (proto == IPPROTO_TCP)
-			sk = __inet_lookup(net, &tcp_hashinfo, NULL, 0,
+			sk = __inet_lookup(net, hinfo, NULL, 0,
 					   src4, tuple->ipv4.sport,
 					   dst4, tuple->ipv4.dport,
 					   dif, sdif, &refcounted);
@@ -6491,7 +6435,7 @@ static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
 		struct in6_addr *dst6 = (struct in6_addr *)&tuple->ipv6.daddr;
 
 		if (proto == IPPROTO_TCP)
-			sk = __inet6_lookup(net, &tcp_hashinfo, NULL, 0,
+			sk = __inet6_lookup(net, hinfo, NULL, 0,
 					    src6, tuple->ipv6.sport,
 					    dst6, ntohs(tuple->ipv6.dport),
 					    dif, sdif, &refcounted);
@@ -7667,34 +7611,23 @@ const struct bpf_func_proto bpf_sk_storage_get_cg_sock_proto __weak;
 static const struct bpf_func_proto *
 sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
+	const struct bpf_func_proto *func_proto;
+
+	func_proto = cgroup_common_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
+	func_proto = cgroup_current_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
 	switch (func_id) {
-	/* inet and inet6 sockets are created in a process
-	 * context so there is always a valid uid/gid
-	 */
-	case BPF_FUNC_get_current_uid_gid:
-		return &bpf_get_current_uid_gid_proto;
-	case BPF_FUNC_get_local_storage:
-		return &bpf_get_local_storage_proto;
 	case BPF_FUNC_get_socket_cookie:
 		return &bpf_get_socket_cookie_sock_proto;
 	case BPF_FUNC_get_netns_cookie:
 		return &bpf_get_netns_cookie_sock_proto;
 	case BPF_FUNC_perf_event_output:
 		return &bpf_event_output_data_proto;
-	case BPF_FUNC_get_current_pid_tgid:
-		return &bpf_get_current_pid_tgid_proto;
-	case BPF_FUNC_get_current_comm:
-		return &bpf_get_current_comm_proto;
-#ifdef CONFIG_CGROUPS
-	case BPF_FUNC_get_current_cgroup_id:
-		return &bpf_get_current_cgroup_id_proto;
-	case BPF_FUNC_get_current_ancestor_cgroup_id:
-		return &bpf_get_current_ancestor_cgroup_id_proto;
-#endif
-#ifdef CONFIG_CGROUP_NET_CLASSID
-	case BPF_FUNC_get_cgroup_classid:
-		return &bpf_get_cgroup_classid_curr_proto;
-#endif
 	case BPF_FUNC_sk_storage_get:
 		return &bpf_sk_storage_get_cg_sock_proto;
 	case BPF_FUNC_ktime_get_coarse_ns:
@@ -7707,12 +7640,17 @@ sock_filter_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 static const struct bpf_func_proto *
 sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
+	const struct bpf_func_proto *func_proto;
+
+	func_proto = cgroup_common_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
+	func_proto = cgroup_current_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
 	switch (func_id) {
-	/* inet and inet6 sockets are created in a process
-	 * context so there is always a valid uid/gid
-	 */
-	case BPF_FUNC_get_current_uid_gid:
-		return &bpf_get_current_uid_gid_proto;
 	case BPF_FUNC_bind:
 		switch (prog->expected_attach_type) {
 		case BPF_CGROUP_INET4_CONNECT:
@@ -7725,24 +7663,8 @@ sock_addr_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_get_socket_cookie_sock_addr_proto;
 	case BPF_FUNC_get_netns_cookie:
 		return &bpf_get_netns_cookie_sock_addr_proto;
-	case BPF_FUNC_get_local_storage:
-		return &bpf_get_local_storage_proto;
 	case BPF_FUNC_perf_event_output:
 		return &bpf_event_output_data_proto;
-	case BPF_FUNC_get_current_pid_tgid:
-		return &bpf_get_current_pid_tgid_proto;
-	case BPF_FUNC_get_current_comm:
-		return &bpf_get_current_comm_proto;
-#ifdef CONFIG_CGROUPS
-	case BPF_FUNC_get_current_cgroup_id:
-		return &bpf_get_current_cgroup_id_proto;
-	case BPF_FUNC_get_current_ancestor_cgroup_id:
-		return &bpf_get_current_ancestor_cgroup_id_proto;
-#endif
-#ifdef CONFIG_CGROUP_NET_CLASSID
-	case BPF_FUNC_get_cgroup_classid:
-		return &bpf_get_cgroup_classid_curr_proto;
-#endif
 #ifdef CONFIG_INET
 	case BPF_FUNC_sk_lookup_tcp:
 		return &bpf_sock_addr_sk_lookup_tcp_proto;
@@ -7823,9 +7745,13 @@ const struct bpf_func_proto bpf_sk_storage_delete_proto __weak;
 static const struct bpf_func_proto *
 cg_skb_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
+	const struct bpf_func_proto *func_proto;
+
+	func_proto = cgroup_common_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
 	switch (func_id) {
-	case BPF_FUNC_get_local_storage:
-		return &bpf_get_local_storage_proto;
 	case BPF_FUNC_sk_fullsock:
 		return &bpf_sk_fullsock_proto;
 	case BPF_FUNC_sk_storage_get:
@@ -8065,6 +7991,12 @@ const struct bpf_func_proto bpf_sock_hash_update_proto __weak;
 static const struct bpf_func_proto *
 sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 {
+	const struct bpf_func_proto *func_proto;
+
+	func_proto = cgroup_common_func_proto(func_id, prog);
+	if (func_proto)
+		return func_proto;
+
 	switch (func_id) {
 	case BPF_FUNC_setsockopt:
 		return &bpf_sock_ops_setsockopt_proto;
@@ -8078,8 +8010,6 @@ sock_ops_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
 		return &bpf_sock_hash_update_proto;
 	case BPF_FUNC_get_socket_cookie:
 		return &bpf_get_socket_cookie_sock_ops_proto;
-	case BPF_FUNC_get_local_storage:
-		return &bpf_get_local_storage_proto;
 	case BPF_FUNC_perf_event_output:
 		return &bpf_event_output_data_proto;
 	case BPF_FUNC_sk_storage_get:
@@ -8714,6 +8644,36 @@ static bool tc_cls_act_is_valid_access(int off, int size,
 	return bpf_skb_is_valid_access(off, size, type, prog, info);
 }
 
+DEFINE_MUTEX(nf_conn_btf_access_lock);
+EXPORT_SYMBOL_GPL(nf_conn_btf_access_lock);
+
+int (*nfct_btf_struct_access)(struct bpf_verifier_log *log, const struct btf *btf,
+			      const struct btf_type *t, int off, int size,
+			      enum bpf_access_type atype, u32 *next_btf_id,
+			      enum bpf_type_flag *flag);
+EXPORT_SYMBOL_GPL(nfct_btf_struct_access);
+
+static int tc_cls_act_btf_struct_access(struct bpf_verifier_log *log,
+					const struct btf *btf,
+					const struct btf_type *t, int off,
+					int size, enum bpf_access_type atype,
+					u32 *next_btf_id,
+					enum bpf_type_flag *flag)
+{
+	int ret = -EACCES;
+
+	if (atype == BPF_READ)
+		return btf_struct_access(log, btf, t, off, size, atype, next_btf_id,
+					 flag);
+
+	mutex_lock(&nf_conn_btf_access_lock);
+	if (nfct_btf_struct_access)
+		ret = nfct_btf_struct_access(log, btf, t, off, size, atype, next_btf_id, flag);
+	mutex_unlock(&nf_conn_btf_access_lock);
+
+	return ret;
+}
+
 static bool __is_valid_xdp_access(int off, int size)
 {
 	if (off < 0 || off >= sizeof(struct xdp_md))
@@ -8773,6 +8733,27 @@ void bpf_warn_invalid_xdp_action(struct net_device *dev, struct bpf_prog *prog,
 }
 EXPORT_SYMBOL_GPL(bpf_warn_invalid_xdp_action);
 
+static int xdp_btf_struct_access(struct bpf_verifier_log *log,
+				 const struct btf *btf,
+				 const struct btf_type *t, int off,
+				 int size, enum bpf_access_type atype,
+				 u32 *next_btf_id,
+				 enum bpf_type_flag *flag)
+{
+	int ret = -EACCES;
+
+	if (atype == BPF_READ)
+		return btf_struct_access(log, btf, t, off, size, atype, next_btf_id,
+					 flag);
+
+	mutex_lock(&nf_conn_btf_access_lock);
+	if (nfct_btf_struct_access)
+		ret = nfct_btf_struct_access(log, btf, t, off, size, atype, next_btf_id, flag);
+	mutex_unlock(&nf_conn_btf_access_lock);
+
+	return ret;
+}
+
 static bool sock_addr_is_valid_access(int off, int size,
 				      enum bpf_access_type type,
 				      const struct bpf_prog *prog,
@@ -10667,6 +10648,7 @@ const struct bpf_verifier_ops tc_cls_act_verifier_ops = {
 	.convert_ctx_access	= tc_cls_act_convert_ctx_access,
 	.gen_prologue		= tc_cls_act_prologue,
 	.gen_ld_abs		= bpf_gen_ld_abs,
+	.btf_struct_access	= tc_cls_act_btf_struct_access,
 };
 
 const struct bpf_prog_ops tc_cls_act_prog_ops = {
@@ -10678,6 +10660,7 @@ const struct bpf_verifier_ops xdp_verifier_ops = {
 	.is_valid_access	= xdp_is_valid_access,
 	.convert_ctx_access	= xdp_convert_ctx_access,
 	.gen_prologue		= bpf_noop_prologue,
+	.btf_struct_access	= xdp_btf_struct_access,
 };
 
 const struct bpf_prog_ops xdp_prog_ops = {
@@ -10812,14 +10795,13 @@ int sk_detach_filter(struct sock *sk)
 }
 EXPORT_SYMBOL_GPL(sk_detach_filter);
 
-int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf,
-		  unsigned int len)
+int sk_get_filter(struct sock *sk, sockptr_t optval, unsigned int len)
 {
 	struct sock_fprog_kern *fprog;
 	struct sk_filter *filter;
 	int ret = 0;
 
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 	filter = rcu_dereference_protected(sk->sk_filter,
 					   lockdep_sock_is_held(sk));
 	if (!filter)
@@ -10844,7 +10826,7 @@ int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf,
 		goto out;
 
 	ret = -EFAULT;
-	if (copy_to_user(ubuf, fprog->filter, bpf_classic_proglen(fprog)))
+	if (copy_to_sockptr(optval, fprog->filter, bpf_classic_proglen(fprog)))
 		goto out;
 
 	/* Instead of bytes, the API requests to return the number
@@ -10852,7 +10834,7 @@ int sk_get_filter(struct sock *sk, struct sock_filter __user *ubuf,
 	 */
 	ret = fprog->len;
 out:
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	return ret;
 }
 
diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c
index 5dc3860e9fc7..25cd35f5922e 100644
--- a/net/core/flow_dissector.c
+++ b/net/core/flow_dissector.c
@@ -204,6 +204,30 @@ static void __skb_flow_dissect_icmp(const struct sk_buff *skb,
 	skb_flow_get_icmp_tci(skb, key_icmp, data, thoff, hlen);
 }
 
+static void __skb_flow_dissect_l2tpv3(const struct sk_buff *skb,
+				      struct flow_dissector *flow_dissector,
+				      void *target_container, const void *data,
+				      int nhoff, int hlen)
+{
+	struct flow_dissector_key_l2tpv3 *key_l2tpv3;
+	struct {
+		__be32 session_id;
+	} *hdr, _hdr;
+
+	if (!dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_L2TPV3))
+		return;
+
+	hdr = __skb_header_pointer(skb, nhoff, sizeof(_hdr), data, hlen, &_hdr);
+	if (!hdr)
+		return;
+
+	key_l2tpv3 = skb_flow_dissector_target(flow_dissector,
+					       FLOW_DISSECTOR_KEY_L2TPV3,
+					       target_container);
+
+	key_l2tpv3->session_id = hdr->session_id;
+}
+
 void skb_flow_dissect_meta(const struct sk_buff *skb,
 			   struct flow_dissector *flow_dissector,
 			   void *target_container)
@@ -866,8 +890,8 @@ static void __skb_flow_bpf_to_target(const struct bpf_flow_keys *flow_keys,
 	}
 }
 
-bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
-		      __be16 proto, int nhoff, int hlen, unsigned int flags)
+u32 bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
+		     __be16 proto, int nhoff, int hlen, unsigned int flags)
 {
 	struct bpf_flow_keys *flow_keys = ctx->flow_keys;
 	u32 result;
@@ -892,7 +916,7 @@ bool bpf_flow_dissect(struct bpf_prog *prog, struct bpf_flow_dissector *ctx,
 	flow_keys->thoff = clamp_t(u16, flow_keys->thoff,
 				   flow_keys->nhoff, hlen);
 
-	return result == BPF_OK;
+	return result;
 }
 
 static bool is_pppoe_ses_hdr_valid(const struct pppoe_hdr *hdr)
@@ -1008,6 +1032,7 @@ bool __skb_flow_dissect(const struct net *net,
 			};
 			__be16 n_proto = proto;
 			struct bpf_prog *prog;
+			u32 result;
 
 			if (skb) {
 				ctx.skb = skb;
@@ -1019,13 +1044,16 @@ bool __skb_flow_dissect(const struct net *net,
 			}
 
 			prog = READ_ONCE(run_array->items[0].prog);
-			ret = bpf_flow_dissect(prog, &ctx, n_proto, nhoff,
-					       hlen, flags);
+			result = bpf_flow_dissect(prog, &ctx, n_proto, nhoff,
+						  hlen, flags);
+			if (result == BPF_FLOW_DISSECTOR_CONTINUE)
+				goto dissect_continue;
 			__skb_flow_bpf_to_target(&flow_keys, flow_dissector,
 						 target_container);
 			rcu_read_unlock();
-			return ret;
+			return result == BPF_OK;
 		}
+dissect_continue:
 		rcu_read_unlock();
 	}
 
@@ -1173,8 +1201,8 @@ proto_again:
 			nhoff += sizeof(*vlan);
 		}
 
-		if (dissector_uses_key(flow_dissector,
-				       FLOW_DISSECTOR_KEY_NUM_OF_VLANS)) {
+		if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_NUM_OF_VLANS) &&
+		    !(key_control->flags & FLOW_DIS_ENCAPSULATION)) {
 			struct flow_dissector_key_num_of_vlans *key_nvs;
 
 			key_nvs = skb_flow_dissector_target(flow_dissector,
@@ -1497,6 +1525,10 @@ ip_proto_again:
 		__skb_flow_dissect_icmp(skb, flow_dissector, target_container,
 					data, nhoff, hlen);
 		break;
+	case IPPROTO_L2TP:
+		__skb_flow_dissect_l2tpv3(skb, flow_dissector, target_container,
+					  data, nhoff, hlen);
+		break;
 
 	default:
 		break;
diff --git a/net/core/flow_offload.c b/net/core/flow_offload.c
index 8cfb63528d18..abe423fd5736 100644
--- a/net/core/flow_offload.c
+++ b/net/core/flow_offload.c
@@ -237,6 +237,13 @@ void flow_rule_match_pppoe(const struct flow_rule *rule,
 }
 EXPORT_SYMBOL(flow_rule_match_pppoe);
 
+void flow_rule_match_l2tpv3(const struct flow_rule *rule,
+			    struct flow_match_l2tpv3 *out)
+{
+	FLOW_DISSECTOR_MATCH(rule, FLOW_DISSECTOR_KEY_L2TPV3, out);
+}
+EXPORT_SYMBOL(flow_rule_match_l2tpv3);
+
 struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb,
 					  void *cb_ident, void *cb_priv,
 					  void (*release)(void *cb_priv))
diff --git a/net/core/gro.c b/net/core/gro.c
index b4190eb08467..bc9451743307 100644
--- a/net/core/gro.c
+++ b/net/core/gro.c
@@ -160,6 +160,7 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
 	unsigned int gro_max_size;
 	unsigned int new_truesize;
 	struct sk_buff *lp;
+	int segs;
 
 	/* pairs with WRITE_ONCE() in netif_set_gro_max_size() */
 	gro_max_size = READ_ONCE(p->dev->gro_max_size);
@@ -175,6 +176,7 @@ int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb)
 			return -E2BIG;
 	}
 
+	segs = NAPI_GRO_CB(skb)->count;
 	lp = NAPI_GRO_CB(p)->last;
 	pinfo = skb_shinfo(lp);
 
@@ -265,7 +267,7 @@ merge:
 	lp = p;
 
 done:
-	NAPI_GRO_CB(p)->count++;
+	NAPI_GRO_CB(p)->count += segs;
 	p->data_len += len;
 	p->truesize += delta_truesize;
 	p->len += len;
@@ -496,8 +498,15 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
 		BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct napi_gro_cb, zeroed),
 					 sizeof(u32))); /* Avoid slow unaligned acc */
 		*(u32 *)&NAPI_GRO_CB(skb)->zeroed = 0;
-		NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb);
+		NAPI_GRO_CB(skb)->flush = skb_has_frag_list(skb);
 		NAPI_GRO_CB(skb)->is_atomic = 1;
+		NAPI_GRO_CB(skb)->count = 1;
+		if (unlikely(skb_is_gso(skb))) {
+			NAPI_GRO_CB(skb)->count = skb_shinfo(skb)->gso_segs;
+			/* Only support TCP at the moment. */
+			if (!skb_is_gso_tcp(skb))
+				NAPI_GRO_CB(skb)->flush = 1;
+		}
 
 		/* Setup for GRO checksum validation */
 		switch (skb->ip_summed) {
@@ -545,10 +554,10 @@ static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff
 	else
 		gro_list->count++;
 
-	NAPI_GRO_CB(skb)->count = 1;
 	NAPI_GRO_CB(skb)->age = jiffies;
 	NAPI_GRO_CB(skb)->last = skb;
-	skb_shinfo(skb)->gso_size = skb_gro_len(skb);
+	if (!skb_is_gso(skb))
+		skb_shinfo(skb)->gso_size = skb_gro_len(skb);
 	list_add(&skb->list, &gro_list->list);
 	ret = GRO_HELD;
 
@@ -660,6 +669,7 @@ static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb)
 
 	skb->encapsulation = 0;
 	skb_shinfo(skb)->gso_type = 0;
+	skb_shinfo(skb)->gso_size = 0;
 	if (unlikely(skb->slow_gro)) {
 		skb_orphan(skb);
 		skb_ext_reset(skb);
diff --git a/net/core/gro_cells.c b/net/core/gro_cells.c
index 21619c70a82b..ed5ec5de47f6 100644
--- a/net/core/gro_cells.c
+++ b/net/core/gro_cells.c
@@ -81,8 +81,7 @@ int gro_cells_init(struct gro_cells *gcells, struct net_device *dev)
 
 		set_bit(NAPI_STATE_NO_BUSY_POLL, &cell->napi.state);
 
-		netif_napi_add(dev, &cell->napi, gro_cell_poll,
-			       NAPI_POLL_WEIGHT);
+		netif_napi_add(dev, &cell->napi, gro_cell_poll);
 		napi_enable(&cell->napi);
 	}
 	return 0;
diff --git a/net/core/lwtunnel.c b/net/core/lwtunnel.c
index 9ccd64e8a666..6fac2f0ef074 100644
--- a/net/core/lwtunnel.c
+++ b/net/core/lwtunnel.c
@@ -50,6 +50,7 @@ static const char *lwtunnel_encap_str(enum lwtunnel_encap_types encap_type)
 		return "IOAM6";
 	case LWTUNNEL_ENCAP_IP6:
 	case LWTUNNEL_ENCAP_IP:
+	case LWTUNNEL_ENCAP_XFRM:
 	case LWTUNNEL_ENCAP_NONE:
 	case __LWTUNNEL_ENCAP_MAX:
 		/* should not have got here */
diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 78cc8fb68814..e93edb810103 100644
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -1853,9 +1853,6 @@ static struct neigh_table *neigh_find_table(int family)
 	case AF_INET6:
 		tbl = neigh_tables[NEIGH_ND_TABLE];
 		break;
-	case AF_DECnet:
-		tbl = neigh_tables[NEIGH_DN_TABLE];
-		break;
 	}
 
 	return tbl;
diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c
index d61afd21aab5..8409d41405df 100644
--- a/net/core/net-sysfs.c
+++ b/net/core/net-sysfs.c
@@ -59,7 +59,7 @@ static ssize_t netdev_show(const struct device *dev,
 #define NETDEVICE_SHOW(field, format_string)				\
 static ssize_t format_##field(const struct net_device *dev, char *buf)	\
 {									\
-	return sprintf(buf, format_string, dev->field);			\
+	return sysfs_emit(buf, format_string, dev->field);		\
 }									\
 static ssize_t field##_show(struct device *dev,				\
 			    struct device_attribute *attr, char *buf)	\
@@ -118,13 +118,13 @@ static ssize_t iflink_show(struct device *dev, struct device_attribute *attr,
 {
 	struct net_device *ndev = to_net_dev(dev);
 
-	return sprintf(buf, fmt_dec, dev_get_iflink(ndev));
+	return sysfs_emit(buf, fmt_dec, dev_get_iflink(ndev));
 }
 static DEVICE_ATTR_RO(iflink);
 
 static ssize_t format_name_assign_type(const struct net_device *dev, char *buf)
 {
-	return sprintf(buf, fmt_dec, dev->name_assign_type);
+	return sysfs_emit(buf, fmt_dec, dev->name_assign_type);
 }
 
 static ssize_t name_assign_type_show(struct device *dev,
@@ -194,7 +194,7 @@ static ssize_t carrier_show(struct device *dev,
 	struct net_device *netdev = to_net_dev(dev);
 
 	if (netif_running(netdev))
-		return sprintf(buf, fmt_dec, !!netif_carrier_ok(netdev));
+		return sysfs_emit(buf, fmt_dec, !!netif_carrier_ok(netdev));
 
 	return -EINVAL;
 }
@@ -219,7 +219,7 @@ static ssize_t speed_show(struct device *dev,
 		struct ethtool_link_ksettings cmd;
 
 		if (!__ethtool_get_link_ksettings(netdev, &cmd))
-			ret = sprintf(buf, fmt_dec, cmd.base.speed);
+			ret = sysfs_emit(buf, fmt_dec, cmd.base.speed);
 	}
 	rtnl_unlock();
 	return ret;
@@ -258,7 +258,7 @@ static ssize_t duplex_show(struct device *dev,
 				duplex = "unknown";
 				break;
 			}
-			ret = sprintf(buf, "%s\n", duplex);
+			ret = sysfs_emit(buf, "%s\n", duplex);
 		}
 	}
 	rtnl_unlock();
@@ -272,7 +272,7 @@ static ssize_t testing_show(struct device *dev,
 	struct net_device *netdev = to_net_dev(dev);
 
 	if (netif_running(netdev))
-		return sprintf(buf, fmt_dec, !!netif_testing(netdev));
+		return sysfs_emit(buf, fmt_dec, !!netif_testing(netdev));
 
 	return -EINVAL;
 }
@@ -284,7 +284,7 @@ static ssize_t dormant_show(struct device *dev,
 	struct net_device *netdev = to_net_dev(dev);
 
 	if (netif_running(netdev))
-		return sprintf(buf, fmt_dec, !!netif_dormant(netdev));
+		return sysfs_emit(buf, fmt_dec, !!netif_dormant(netdev));
 
 	return -EINVAL;
 }
@@ -315,7 +315,7 @@ static ssize_t operstate_show(struct device *dev,
 	if (operstate >= ARRAY_SIZE(operstates))
 		return -EINVAL; /* should not happen */
 
-	return sprintf(buf, "%s\n", operstates[operstate]);
+	return sysfs_emit(buf, "%s\n", operstates[operstate]);
 }
 static DEVICE_ATTR_RO(operstate);
 
@@ -325,9 +325,9 @@ static ssize_t carrier_changes_show(struct device *dev,
 {
 	struct net_device *netdev = to_net_dev(dev);
 
-	return sprintf(buf, fmt_dec,
-		       atomic_read(&netdev->carrier_up_count) +
-		       atomic_read(&netdev->carrier_down_count));
+	return sysfs_emit(buf, fmt_dec,
+			  atomic_read(&netdev->carrier_up_count) +
+			  atomic_read(&netdev->carrier_down_count));
 }
 static DEVICE_ATTR_RO(carrier_changes);
 
@@ -337,7 +337,7 @@ static ssize_t carrier_up_count_show(struct device *dev,
 {
 	struct net_device *netdev = to_net_dev(dev);
 
-	return sprintf(buf, fmt_dec, atomic_read(&netdev->carrier_up_count));
+	return sysfs_emit(buf, fmt_dec, atomic_read(&netdev->carrier_up_count));
 }
 static DEVICE_ATTR_RO(carrier_up_count);
 
@@ -347,7 +347,7 @@ static ssize_t carrier_down_count_show(struct device *dev,
 {
 	struct net_device *netdev = to_net_dev(dev);
 
-	return sprintf(buf, fmt_dec, atomic_read(&netdev->carrier_down_count));
+	return sysfs_emit(buf, fmt_dec, atomic_read(&netdev->carrier_down_count));
 }
 static DEVICE_ATTR_RO(carrier_down_count);
 
@@ -462,7 +462,7 @@ static ssize_t ifalias_show(struct device *dev,
 
 	ret = dev_get_alias(netdev, tmp, sizeof(tmp));
 	if (ret > 0)
-		ret = sprintf(buf, "%s\n", tmp);
+		ret = sysfs_emit(buf, "%s\n", tmp);
 	return ret;
 }
 static DEVICE_ATTR_RW(ifalias);
@@ -514,7 +514,7 @@ static ssize_t phys_port_id_show(struct device *dev,
 
 		ret = dev_get_phys_port_id(netdev, &ppid);
 		if (!ret)
-			ret = sprintf(buf, "%*phN\n", ppid.id_len, ppid.id);
+			ret = sysfs_emit(buf, "%*phN\n", ppid.id_len, ppid.id);
 	}
 	rtnl_unlock();
 
@@ -543,7 +543,7 @@ static ssize_t phys_port_name_show(struct device *dev,
 
 		ret = dev_get_phys_port_name(netdev, name, sizeof(name));
 		if (!ret)
-			ret = sprintf(buf, "%s\n", name);
+			ret = sysfs_emit(buf, "%s\n", name);
 	}
 	rtnl_unlock();
 
@@ -573,7 +573,7 @@ static ssize_t phys_switch_id_show(struct device *dev,
 
 		ret = dev_get_port_parent_id(netdev, &ppid, false);
 		if (!ret)
-			ret = sprintf(buf, "%*phN\n", ppid.id_len, ppid.id);
+			ret = sysfs_emit(buf, "%*phN\n", ppid.id_len, ppid.id);
 	}
 	rtnl_unlock();
 
@@ -591,7 +591,7 @@ static ssize_t threaded_show(struct device *dev,
 		return restart_syscall();
 
 	if (dev_isalive(netdev))
-		ret = sprintf(buf, fmt_dec, netdev->threaded);
+		ret = sysfs_emit(buf, fmt_dec, netdev->threaded);
 
 	rtnl_unlock();
 	return ret;
@@ -673,7 +673,7 @@ static ssize_t netstat_show(const struct device *d,
 		struct rtnl_link_stats64 temp;
 		const struct rtnl_link_stats64 *stats = dev_get_stats(dev, &temp);
 
-		ret = sprintf(buf, fmt_u64, *(u64 *)(((u8 *)stats) + offset));
+		ret = sysfs_emit(buf, fmt_u64, *(u64 *)(((u8 *)stats) + offset));
 	}
 	read_unlock(&dev_base_lock);
 	return ret;
@@ -824,7 +824,7 @@ static ssize_t show_rps_map(struct netdev_rx_queue *queue, char *buf)
 		for (i = 0; i < map->len; i++)
 			cpumask_set_cpu(map->cpus[i], mask);
 
-	len = snprintf(buf, PAGE_SIZE, "%*pb\n", cpumask_pr_args(mask));
+	len = sysfs_emit(buf, "%*pb\n", cpumask_pr_args(mask));
 	rcu_read_unlock();
 	free_cpumask_var(mask);
 
@@ -910,7 +910,7 @@ static ssize_t show_rps_dev_flow_table_cnt(struct netdev_rx_queue *queue,
 		val = (unsigned long)flow_table->mask + 1;
 	rcu_read_unlock();
 
-	return sprintf(buf, "%lu\n", val);
+	return sysfs_emit(buf, "%lu\n", val);
 }
 
 static void rps_dev_flow_table_release(struct rcu_head *rcu)
@@ -1208,7 +1208,7 @@ static ssize_t tx_timeout_show(struct netdev_queue *queue, char *buf)
 {
 	unsigned long trans_timeout = atomic_long_read(&queue->trans_timeout);
 
-	return sprintf(buf, fmt_ulong, trans_timeout);
+	return sysfs_emit(buf, fmt_ulong, trans_timeout);
 }
 
 static unsigned int get_netdev_queue_index(struct netdev_queue *queue)
@@ -1255,15 +1255,15 @@ static ssize_t traffic_class_show(struct netdev_queue *queue,
 	 * belongs to the root device it will be reported with just the
 	 * traffic class, so just "0" for TC 0 for example.
 	 */
-	return num_tc < 0 ? sprintf(buf, "%d%d\n", tc, num_tc) :
-			    sprintf(buf, "%d\n", tc);
+	return num_tc < 0 ? sysfs_emit(buf, "%d%d\n", tc, num_tc) :
+			    sysfs_emit(buf, "%d\n", tc);
 }
 
 #ifdef CONFIG_XPS
 static ssize_t tx_maxrate_show(struct netdev_queue *queue,
 			       char *buf)
 {
-	return sprintf(buf, "%lu\n", queue->tx_maxrate);
+	return sysfs_emit(buf, "%lu\n", queue->tx_maxrate);
 }
 
 static ssize_t tx_maxrate_store(struct netdev_queue *queue,
@@ -1317,7 +1317,7 @@ static struct netdev_queue_attribute queue_traffic_class __ro_after_init
  */
 static ssize_t bql_show(char *buf, unsigned int value)
 {
-	return sprintf(buf, "%u\n", value);
+	return sysfs_emit(buf, "%u\n", value);
 }
 
 static ssize_t bql_set(const char *buf, const size_t count,
@@ -1346,7 +1346,7 @@ static ssize_t bql_show_hold_time(struct netdev_queue *queue,
 {
 	struct dql *dql = &queue->dql;
 
-	return sprintf(buf, "%u\n", jiffies_to_msecs(dql->slack_hold_time));
+	return sysfs_emit(buf, "%u\n", jiffies_to_msecs(dql->slack_hold_time));
 }
 
 static ssize_t bql_set_hold_time(struct netdev_queue *queue,
@@ -1374,7 +1374,7 @@ static ssize_t bql_show_inflight(struct netdev_queue *queue,
 {
 	struct dql *dql = &queue->dql;
 
-	return sprintf(buf, "%u\n", dql->num_queued - dql->num_completed);
+	return sysfs_emit(buf, "%u\n", dql->num_queued - dql->num_completed);
 }
 
 static struct netdev_queue_attribute bql_inflight_attribute __ro_after_init =
diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c
index 6b9f19122ec1..0ec2f5906a27 100644
--- a/net/core/net_namespace.c
+++ b/net/core/net_namespace.c
@@ -18,7 +18,6 @@
 #include <linux/user_namespace.h>
 #include <linux/net_namespace.h>
 #include <linux/sched/task.h>
-#include <linux/sched/mm.h>
 #include <linux/uidgid.h>
 #include <linux/cookie.h>
 
@@ -1144,13 +1143,7 @@ static int __register_pernet_operations(struct list_head *list,
 		 * setup_net() and cleanup_net() are not possible.
 		 */
 		for_each_net(net) {
-			struct mem_cgroup *old, *memcg;
-
-			memcg = mem_cgroup_or_root(get_mem_cgroup_from_obj(net));
-			old = set_active_memcg(memcg);
 			error = ops_init(ops, net);
-			set_active_memcg(old);
-			mem_cgroup_put(memcg);
 			if (error)
 				goto out_undo;
 			list_add_tail(&net->exit_list, &net_exit_list);
diff --git a/net/core/netclassid_cgroup.c b/net/core/netclassid_cgroup.c
index 1a6a86693b74..d6a70aeaa503 100644
--- a/net/core/netclassid_cgroup.c
+++ b/net/core/netclassid_cgroup.c
@@ -66,7 +66,7 @@ struct update_classid_context {
 
 #define UPDATE_CLASSID_BATCH 1000
 
-static int update_classid_sock(const void *v, struct file *file, unsigned n)
+static int update_classid_sock(const void *v, struct file *file, unsigned int n)
 {
 	struct update_classid_context *ctx = (void *)v;
 	struct socket *sock = sock_from_file(file);
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index 5d27067b72d5..9be762e1d042 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -556,7 +556,7 @@ int netpoll_parse_options(struct netpoll *np, char *opt)
 		if ((delim = strchr(cur, ',')) == NULL)
 			goto parse_failed;
 		*delim = 0;
-		strlcpy(np->dev_name, cur, sizeof(np->dev_name));
+		strscpy(np->dev_name, cur, sizeof(np->dev_name));
 		cur = delim;
 	}
 	cur++;
@@ -610,7 +610,7 @@ int __netpoll_setup(struct netpoll *np, struct net_device *ndev)
 	int err;
 
 	np->dev = ndev;
-	strlcpy(np->dev_name, ndev->name, IFNAMSIZ);
+	strscpy(np->dev_name, ndev->name, IFNAMSIZ);
 
 	if (ndev->priv_flags & IFF_DISABLE_NETPOLL) {
 		np_err(np, "%s doesn't support polling, aborting\n",
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 4b5b15c684ed..74864dc46a7e 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -866,14 +866,12 @@ static void set_operstate(struct net_device *dev, unsigned char transition)
 		break;
 
 	case IF_OPER_TESTING:
-		if (operstate == IF_OPER_UP ||
-		    operstate == IF_OPER_UNKNOWN)
+		if (netif_oper_up(dev))
 			operstate = IF_OPER_TESTING;
 		break;
 
 	case IF_OPER_DORMANT:
-		if (operstate == IF_OPER_UP ||
-		    operstate == IF_OPER_UNKNOWN)
+		if (netif_oper_up(dev))
 			operstate = IF_OPER_DORMANT;
 		break;
 	}
@@ -1059,6 +1057,7 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev,
 	       + nla_total_size(4) /* IFLA_MASTER */
 	       + nla_total_size(1) /* IFLA_CARRIER */
 	       + nla_total_size(4) /* IFLA_PROMISCUITY */
+	       + nla_total_size(4) /* IFLA_ALLMULTI */
 	       + nla_total_size(4) /* IFLA_NUM_TX_QUEUES */
 	       + nla_total_size(4) /* IFLA_NUM_RX_QUEUES */
 	       + nla_total_size(4) /* IFLA_GSO_MAX_SEGS */
@@ -1767,6 +1766,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb,
 	    nla_put_u32(skb, IFLA_MAX_MTU, dev->max_mtu) ||
 	    nla_put_u32(skb, IFLA_GROUP, dev->group) ||
 	    nla_put_u32(skb, IFLA_PROMISCUITY, dev->promiscuity) ||
+	    nla_put_u32(skb, IFLA_ALLMULTI, dev->allmulti) ||
 	    nla_put_u32(skb, IFLA_NUM_TX_QUEUES, dev->num_tx_queues) ||
 	    nla_put_u32(skb, IFLA_GSO_MAX_SEGS, dev->gso_max_segs) ||
 	    nla_put_u32(skb, IFLA_GSO_MAX_SIZE, dev->gso_max_size) ||
@@ -1928,6 +1928,7 @@ static const struct nla_policy ifla_policy[IFLA_MAX+1] = {
 	[IFLA_GRO_MAX_SIZE]	= { .type = NLA_U32 },
 	[IFLA_TSO_MAX_SIZE]	= { .type = NLA_REJECT },
 	[IFLA_TSO_MAX_SEGS]	= { .type = NLA_REJECT },
+	[IFLA_ALLMULTI]		= { .type = NLA_REJECT },
 };
 
 static const struct nla_policy ifla_info_policy[IFLA_INFO_MAX+1] = {
@@ -2776,13 +2777,6 @@ static int do_setlink(const struct sk_buff *skb,
 		call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
 	}
 
-	if (ifm->ifi_flags || ifm->ifi_change) {
-		err = dev_change_flags(dev, rtnl_dev_combine_flags(dev, ifm),
-				       extack);
-		if (err < 0)
-			goto errout;
-	}
-
 	if (tb[IFLA_MASTER]) {
 		err = do_set_master(dev, nla_get_u32(tb[IFLA_MASTER]), extack);
 		if (err)
@@ -2790,6 +2784,13 @@ static int do_setlink(const struct sk_buff *skb,
 		status |= DO_SETLINK_MODIFIED;
 	}
 
+	if (ifm->ifi_flags || ifm->ifi_change) {
+		err = dev_change_flags(dev, rtnl_dev_combine_flags(dev, ifm),
+				       extack);
+		if (err < 0)
+			goto errout;
+	}
+
 	if (tb[IFLA_CARRIER]) {
 		err = dev_change_carrier(dev, nla_get_u8(tb[IFLA_CARRIER]));
 		if (err)
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 417463da4fac..1d9719e72f9d 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -134,8 +134,66 @@ static void skb_under_panic(struct sk_buff *skb, unsigned int sz, void *addr)
 #define NAPI_SKB_CACHE_BULK	16
 #define NAPI_SKB_CACHE_HALF	(NAPI_SKB_CACHE_SIZE / 2)
 
+#if PAGE_SIZE == SZ_4K
+
+#define NAPI_HAS_SMALL_PAGE_FRAG	1
+#define NAPI_SMALL_PAGE_PFMEMALLOC(nc)	((nc).pfmemalloc)
+
+/* specialized page frag allocator using a single order 0 page
+ * and slicing it into 1K sized fragment. Constrained to systems
+ * with a very limited amount of 1K fragments fitting a single
+ * page - to avoid excessive truesize underestimation
+ */
+
+struct page_frag_1k {
+	void *va;
+	u16 offset;
+	bool pfmemalloc;
+};
+
+static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp)
+{
+	struct page *page;
+	int offset;
+
+	offset = nc->offset - SZ_1K;
+	if (likely(offset >= 0))
+		goto use_frag;
+
+	page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
+	if (!page)
+		return NULL;
+
+	nc->va = page_address(page);
+	nc->pfmemalloc = page_is_pfmemalloc(page);
+	offset = PAGE_SIZE - SZ_1K;
+	page_ref_add(page, offset / SZ_1K);
+
+use_frag:
+	nc->offset = offset;
+	return nc->va + offset;
+}
+#else
+
+/* the small page is actually unused in this build; add dummy helpers
+ * to please the compiler and avoid later preprocessor's conditionals
+ */
+#define NAPI_HAS_SMALL_PAGE_FRAG	0
+#define NAPI_SMALL_PAGE_PFMEMALLOC(nc)	false
+
+struct page_frag_1k {
+};
+
+static void *page_frag_alloc_1k(struct page_frag_1k *nc, gfp_t gfp_mask)
+{
+	return NULL;
+}
+
+#endif
+
 struct napi_alloc_cache {
 	struct page_frag_cache page;
+	struct page_frag_1k page_small;
 	unsigned int skb_count;
 	void *skb_cache[NAPI_SKB_CACHE_SIZE];
 };
@@ -143,6 +201,23 @@ struct napi_alloc_cache {
 static DEFINE_PER_CPU(struct page_frag_cache, netdev_alloc_cache);
 static DEFINE_PER_CPU(struct napi_alloc_cache, napi_alloc_cache);
 
+/* Double check that napi_get_frags() allocates skbs with
+ * skb->head being backed by slab, not a page fragment.
+ * This is to make sure bug fixed in 3226b158e67c
+ * ("net: avoid 32 x truesize under-estimation for tiny skbs")
+ * does not accidentally come back.
+ */
+void napi_get_frags_check(struct napi_struct *napi)
+{
+	struct sk_buff *skb;
+
+	local_bh_disable();
+	skb = napi_get_frags(napi);
+	WARN_ON_ONCE(!NAPI_HAS_SMALL_PAGE_FRAG && skb && skb->head_frag);
+	napi_free_frags(napi);
+	local_bh_enable();
+}
+
 void *__napi_alloc_frag_align(unsigned int fragsz, unsigned int align_mask)
 {
 	struct napi_alloc_cache *nc = this_cpu_ptr(&napi_alloc_cache);
@@ -561,6 +636,7 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
 {
 	struct napi_alloc_cache *nc;
 	struct sk_buff *skb;
+	bool pfmemalloc;
 	void *data;
 
 	DEBUG_NET_WARN_ON_ONCE(!in_softirq());
@@ -568,8 +644,10 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
 
 	/* If requested length is either too small or too big,
 	 * we use kmalloc() for skb->head allocation.
+	 * When the small frag allocator is available, prefer it over kmalloc
+	 * for small fragments
 	 */
-	if (len <= SKB_WITH_OVERHEAD(1024) ||
+	if ((!NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) ||
 	    len > SKB_WITH_OVERHEAD(PAGE_SIZE) ||
 	    (gfp_mask & (__GFP_DIRECT_RECLAIM | GFP_DMA))) {
 		skb = __alloc_skb(len, gfp_mask, SKB_ALLOC_RX | SKB_ALLOC_NAPI,
@@ -580,13 +658,33 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
 	}
 
 	nc = this_cpu_ptr(&napi_alloc_cache);
-	len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
-	len = SKB_DATA_ALIGN(len);
 
 	if (sk_memalloc_socks())
 		gfp_mask |= __GFP_MEMALLOC;
 
-	data = page_frag_alloc(&nc->page, len, gfp_mask);
+	if (NAPI_HAS_SMALL_PAGE_FRAG && len <= SKB_WITH_OVERHEAD(1024)) {
+		/* we are artificially inflating the allocation size, but
+		 * that is not as bad as it may look like, as:
+		 * - 'len' less than GRO_MAX_HEAD makes little sense
+		 * - On most systems, larger 'len' values lead to fragment
+		 *   size above 512 bytes
+		 * - kmalloc would use the kmalloc-1k slab for such values
+		 * - Builds with smaller GRO_MAX_HEAD will very likely do
+		 *   little networking, as that implies no WiFi and no
+		 *   tunnels support, and 32 bits arches.
+		 */
+		len = SZ_1K;
+
+		data = page_frag_alloc_1k(&nc->page_small, gfp_mask);
+		pfmemalloc = NAPI_SMALL_PAGE_PFMEMALLOC(nc->page_small);
+	} else {
+		len += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+		len = SKB_DATA_ALIGN(len);
+
+		data = page_frag_alloc(&nc->page, len, gfp_mask);
+		pfmemalloc = nc->page.pfmemalloc;
+	}
+
 	if (unlikely(!data))
 		return NULL;
 
@@ -596,7 +694,7 @@ struct sk_buff *__napi_alloc_skb(struct napi_struct *napi, unsigned int len,
 		return NULL;
 	}
 
-	if (nc->page.pfmemalloc)
+	if (pfmemalloc)
 		skb->pfmemalloc = 1;
 	skb->head_frag = 1;
 
@@ -781,9 +879,10 @@ EXPORT_SYMBOL(__kfree_skb);
  *	hit zero. Meanwhile, pass the drop reason to 'kfree_skb'
  *	tracepoint.
  */
-void kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
+void __fix_address
+kfree_skb_reason(struct sk_buff *skb, enum skb_drop_reason reason)
 {
-	if (!skb_unref(skb))
+	if (unlikely(!skb_unref(skb)))
 		return;
 
 	DEBUG_NET_WARN_ON_ONCE(reason <= 0 || reason >= SKB_DROP_REASON_MAX);
@@ -1187,7 +1286,7 @@ EXPORT_SYMBOL_GPL(mm_unaccount_pinned_pages);
 
 static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
 {
-	struct ubuf_info *uarg;
+	struct ubuf_info_msgzc *uarg;
 	struct sk_buff *skb;
 
 	WARN_ON_ONCE(!in_task());
@@ -1205,19 +1304,19 @@ static struct ubuf_info *msg_zerocopy_alloc(struct sock *sk, size_t size)
 		return NULL;
 	}
 
-	uarg->callback = msg_zerocopy_callback;
+	uarg->ubuf.callback = msg_zerocopy_callback;
 	uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1;
 	uarg->len = 1;
 	uarg->bytelen = size;
 	uarg->zerocopy = 1;
-	uarg->flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
-	refcount_set(&uarg->refcnt, 1);
+	uarg->ubuf.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN;
+	refcount_set(&uarg->ubuf.refcnt, 1);
 	sock_hold(sk);
 
-	return uarg;
+	return &uarg->ubuf;
 }
 
-static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg)
+static inline struct sk_buff *skb_from_uarg(struct ubuf_info_msgzc *uarg)
 {
 	return container_of((void *)uarg, struct sk_buff, cb);
 }
@@ -1226,6 +1325,7 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
 				       struct ubuf_info *uarg)
 {
 	if (uarg) {
+		struct ubuf_info_msgzc *uarg_zc;
 		const u32 byte_limit = 1 << 19;		/* limit to a few TSO */
 		u32 bytelen, next;
 
@@ -1241,8 +1341,9 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
 			return NULL;
 		}
 
-		bytelen = uarg->bytelen + size;
-		if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) {
+		uarg_zc = uarg_to_msgzc(uarg);
+		bytelen = uarg_zc->bytelen + size;
+		if (uarg_zc->len == USHRT_MAX - 1 || bytelen > byte_limit) {
 			/* TCP can create new skb to attach new uarg */
 			if (sk->sk_type == SOCK_STREAM)
 				goto new_alloc;
@@ -1250,11 +1351,11 @@ struct ubuf_info *msg_zerocopy_realloc(struct sock *sk, size_t size,
 		}
 
 		next = (u32)atomic_read(&sk->sk_zckey);
-		if ((u32)(uarg->id + uarg->len) == next) {
-			if (mm_account_pinned_pages(&uarg->mmp, size))
+		if ((u32)(uarg_zc->id + uarg_zc->len) == next) {
+			if (mm_account_pinned_pages(&uarg_zc->mmp, size))
 				return NULL;
-			uarg->len++;
-			uarg->bytelen = bytelen;
+			uarg_zc->len++;
+			uarg_zc->bytelen = bytelen;
 			atomic_set(&sk->sk_zckey, ++next);
 
 			/* no extra ref when appending to datagram (MSG_MORE) */
@@ -1290,7 +1391,7 @@ static bool skb_zerocopy_notify_extend(struct sk_buff *skb, u32 lo, u16 len)
 	return true;
 }
 
-static void __msg_zerocopy_callback(struct ubuf_info *uarg)
+static void __msg_zerocopy_callback(struct ubuf_info_msgzc *uarg)
 {
 	struct sk_buff *tail, *skb = skb_from_uarg(uarg);
 	struct sock_exterr_skb *serr;
@@ -1343,19 +1444,21 @@ release:
 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg,
 			   bool success)
 {
-	uarg->zerocopy = uarg->zerocopy & success;
+	struct ubuf_info_msgzc *uarg_zc = uarg_to_msgzc(uarg);
+
+	uarg_zc->zerocopy = uarg_zc->zerocopy & success;
 
 	if (refcount_dec_and_test(&uarg->refcnt))
-		__msg_zerocopy_callback(uarg);
+		__msg_zerocopy_callback(uarg_zc);
 }
 EXPORT_SYMBOL_GPL(msg_zerocopy_callback);
 
 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)
 {
-	struct sock *sk = skb_from_uarg(uarg)->sk;
+	struct sock *sk = skb_from_uarg(uarg_to_msgzc(uarg))->sk;
 
 	atomic_dec(&sk->sk_zckey);
-	uarg->len--;
+	uarg_to_msgzc(uarg)->len--;
 
 	if (have_uref)
 		msg_zerocopy_callback(NULL, uarg, true);
diff --git a/net/core/skmsg.c b/net/core/skmsg.c
index 188f8558d27d..ca70525621c7 100644
--- a/net/core/skmsg.c
+++ b/net/core/skmsg.c
@@ -434,8 +434,10 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
 			if (copied + copy > len)
 				copy = len - copied;
 			copy = copy_page_to_iter(page, sge->offset, copy, iter);
-			if (!copy)
-				return copied ? copied : -EFAULT;
+			if (!copy) {
+				copied = copied ? copied : -EFAULT;
+				goto out;
+			}
 
 			copied += copy;
 			if (likely(!peek)) {
@@ -455,7 +457,7 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
 				 * didn't copy the entire length lets just break.
 				 */
 				if (copy != sge->length)
-					return copied;
+					goto out;
 				sk_msg_iter_var_next(i);
 			}
 
@@ -477,7 +479,9 @@ int sk_msg_recvmsg(struct sock *sk, struct sk_psock *psock, struct msghdr *msg,
 		}
 		msg_rx = sk_psock_peek_msg(psock);
 	}
-
+out:
+	if (psock->work_state.skb && copied > 0)
+		schedule_work(&psock->work);
 	return copied;
 }
 EXPORT_SYMBOL_GPL(sk_msg_recvmsg);
diff --git a/net/core/sock.c b/net/core/sock.c
index 788c1372663c..eeb6cbac6f49 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -703,15 +703,17 @@ static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen)
 			goto out;
 	}
 
-	return sock_bindtoindex(sk, index, true);
+	sockopt_lock_sock(sk);
+	ret = sock_bindtoindex_locked(sk, index);
+	sockopt_release_sock(sk);
 out:
 #endif
 
 	return ret;
 }
 
-static int sock_getbindtodevice(struct sock *sk, char __user *optval,
-				int __user *optlen, int len)
+static int sock_getbindtodevice(struct sock *sk, sockptr_t optval,
+				sockptr_t optlen, int len)
 {
 	int ret = -ENOPROTOOPT;
 #ifdef CONFIG_NETDEVICES
@@ -735,12 +737,12 @@ static int sock_getbindtodevice(struct sock *sk, char __user *optval,
 	len = strlen(devname) + 1;
 
 	ret = -EFAULT;
-	if (copy_to_user(optval, devname, len))
+	if (copy_to_sockptr(optval, devname, len))
 		goto out;
 
 zero:
 	ret = -EFAULT;
-	if (put_user(len, optlen))
+	if (copy_to_sockptr(optlen, &len, sizeof(int)))
 		goto out;
 
 	ret = 0;
@@ -1036,17 +1038,51 @@ static int sock_reserve_memory(struct sock *sk, int bytes)
 	return 0;
 }
 
+void sockopt_lock_sock(struct sock *sk)
+{
+	/* When current->bpf_ctx is set, the setsockopt is called from
+	 * a bpf prog.  bpf has ensured the sk lock has been
+	 * acquired before calling setsockopt().
+	 */
+	if (has_current_bpf_ctx())
+		return;
+
+	lock_sock(sk);
+}
+EXPORT_SYMBOL(sockopt_lock_sock);
+
+void sockopt_release_sock(struct sock *sk)
+{
+	if (has_current_bpf_ctx())
+		return;
+
+	release_sock(sk);
+}
+EXPORT_SYMBOL(sockopt_release_sock);
+
+bool sockopt_ns_capable(struct user_namespace *ns, int cap)
+{
+	return has_current_bpf_ctx() || ns_capable(ns, cap);
+}
+EXPORT_SYMBOL(sockopt_ns_capable);
+
+bool sockopt_capable(int cap)
+{
+	return has_current_bpf_ctx() || capable(cap);
+}
+EXPORT_SYMBOL(sockopt_capable);
+
 /*
  *	This is meant for all protocols to use and covers goings on
  *	at the socket level. Everything here is generic.
  */
 
-int sock_setsockopt(struct socket *sock, int level, int optname,
-		    sockptr_t optval, unsigned int optlen)
+int sk_setsockopt(struct sock *sk, int level, int optname,
+		  sockptr_t optval, unsigned int optlen)
 {
 	struct so_timestamping timestamping;
+	struct socket *sock = sk->sk_socket;
 	struct sock_txtime sk_txtime;
-	struct sock *sk = sock->sk;
 	int val;
 	int valbool;
 	struct linger ling;
@@ -1067,11 +1103,11 @@ int sock_setsockopt(struct socket *sock, int level, int optname,
 
 	valbool = val ? 1 : 0;
 
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 
 	switch (optname) {
 	case SO_DEBUG:
-		if (val && !capable(CAP_NET_ADMIN))
+		if (val && !sockopt_capable(CAP_NET_ADMIN))
 			ret = -EACCES;
 		else
 			sock_valbool_flag(sk, SOCK_DBG, valbool);
@@ -1115,7 +1151,7 @@ set_sndbuf:
 		break;
 
 	case SO_SNDBUFFORCE:
-		if (!capable(CAP_NET_ADMIN)) {
+		if (!sockopt_capable(CAP_NET_ADMIN)) {
 			ret = -EPERM;
 			break;
 		}
@@ -1137,7 +1173,7 @@ set_sndbuf:
 		break;
 
 	case SO_RCVBUFFORCE:
-		if (!capable(CAP_NET_ADMIN)) {
+		if (!sockopt_capable(CAP_NET_ADMIN)) {
 			ret = -EPERM;
 			break;
 		}
@@ -1164,8 +1200,8 @@ set_sndbuf:
 
 	case SO_PRIORITY:
 		if ((val >= 0 && val <= 6) ||
-		    ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) ||
-		    ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
+		    sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) ||
+		    sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
 			sk->sk_priority = val;
 		else
 			ret = -EPERM;
@@ -1228,7 +1264,7 @@ set_sndbuf:
 	case SO_RCVLOWAT:
 		if (val < 0)
 			val = INT_MAX;
-		if (sock->ops->set_rcvlowat)
+		if (sock && sock->ops->set_rcvlowat)
 			ret = sock->ops->set_rcvlowat(sk, val);
 		else
 			WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
@@ -1310,8 +1346,8 @@ set_sndbuf:
 			clear_bit(SOCK_PASSSEC, &sock->flags);
 		break;
 	case SO_MARK:
-		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
-		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
+		if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
+		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
 			ret = -EPERM;
 			break;
 		}
@@ -1319,8 +1355,8 @@ set_sndbuf:
 		__sock_set_mark(sk, val);
 		break;
 	case SO_RCVMARK:
-		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
-		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
+		if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
+		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
 			ret = -EPERM;
 			break;
 		}
@@ -1354,7 +1390,7 @@ set_sndbuf:
 #ifdef CONFIG_NET_RX_BUSY_POLL
 	case SO_BUSY_POLL:
 		/* allow unprivileged users to decrease the value */
-		if ((val > sk->sk_ll_usec) && !capable(CAP_NET_ADMIN))
+		if ((val > sk->sk_ll_usec) && !sockopt_capable(CAP_NET_ADMIN))
 			ret = -EPERM;
 		else {
 			if (val < 0)
@@ -1364,13 +1400,13 @@ set_sndbuf:
 		}
 		break;
 	case SO_PREFER_BUSY_POLL:
-		if (valbool && !capable(CAP_NET_ADMIN))
+		if (valbool && !sockopt_capable(CAP_NET_ADMIN))
 			ret = -EPERM;
 		else
 			WRITE_ONCE(sk->sk_prefer_busy_poll, valbool);
 		break;
 	case SO_BUSY_POLL_BUDGET:
-		if (val > READ_ONCE(sk->sk_busy_poll_budget) && !capable(CAP_NET_ADMIN)) {
+		if (val > READ_ONCE(sk->sk_busy_poll_budget) && !sockopt_capable(CAP_NET_ADMIN)) {
 			ret = -EPERM;
 		} else {
 			if (val < 0 || val > U16_MAX)
@@ -1441,7 +1477,7 @@ set_sndbuf:
 		 * scheduler has enough safe guards.
 		 */
 		if (sk_txtime.clockid != CLOCK_MONOTONIC &&
-		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
+		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
 			ret = -EPERM;
 			break;
 		}
@@ -1496,9 +1532,16 @@ set_sndbuf:
 		ret = -ENOPROTOOPT;
 		break;
 	}
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	return ret;
 }
+
+int sock_setsockopt(struct socket *sock, int level, int optname,
+		    sockptr_t optval, unsigned int optlen)
+{
+	return sk_setsockopt(sock->sk, level, optname,
+			     optval, optlen);
+}
 EXPORT_SYMBOL(sock_setsockopt);
 
 static const struct cred *sk_get_peer_cred(struct sock *sk)
@@ -1525,22 +1568,25 @@ static void cred_to_ucred(struct pid *pid, const struct cred *cred,
 	}
 }
 
-static int groups_to_user(gid_t __user *dst, const struct group_info *src)
+static int groups_to_user(sockptr_t dst, const struct group_info *src)
 {
 	struct user_namespace *user_ns = current_user_ns();
 	int i;
 
-	for (i = 0; i < src->ngroups; i++)
-		if (put_user(from_kgid_munged(user_ns, src->gid[i]), dst + i))
+	for (i = 0; i < src->ngroups; i++) {
+		gid_t gid = from_kgid_munged(user_ns, src->gid[i]);
+
+		if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid)))
 			return -EFAULT;
+	}
 
 	return 0;
 }
 
-int sock_getsockopt(struct socket *sock, int level, int optname,
-		    char __user *optval, int __user *optlen)
+int sk_getsockopt(struct sock *sk, int level, int optname,
+		  sockptr_t optval, sockptr_t optlen)
 {
-	struct sock *sk = sock->sk;
+	struct socket *sock = sk->sk_socket;
 
 	union {
 		int val;
@@ -1557,7 +1603,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
 	int lv = sizeof(int);
 	int len;
 
-	if (get_user(len, optlen))
+	if (copy_from_sockptr(&len, optlen, sizeof(int)))
 		return -EFAULT;
 	if (len < 0)
 		return -EINVAL;
@@ -1692,7 +1738,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
 		cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
 		spin_unlock(&sk->sk_peer_lock);
 
-		if (copy_to_user(optval, &peercred, len))
+		if (copy_to_sockptr(optval, &peercred, len))
 			return -EFAULT;
 		goto lenout;
 	}
@@ -1710,11 +1756,11 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
 		if (len < n * sizeof(gid_t)) {
 			len = n * sizeof(gid_t);
 			put_cred(cred);
-			return put_user(len, optlen) ? -EFAULT : -ERANGE;
+			return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE;
 		}
 		len = n * sizeof(gid_t);
 
-		ret = groups_to_user((gid_t __user *)optval, cred->group_info);
+		ret = groups_to_user(optval, cred->group_info);
 		put_cred(cred);
 		if (ret)
 			return ret;
@@ -1730,7 +1776,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
 			return -ENOTCONN;
 		if (lv < len)
 			return -EINVAL;
-		if (copy_to_user(optval, address, len))
+		if (copy_to_sockptr(optval, address, len))
 			return -EFAULT;
 		goto lenout;
 	}
@@ -1747,7 +1793,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
 		break;
 
 	case SO_PEERSEC:
-		return security_socket_getpeersec_stream(sock, optval, optlen, len);
+		return security_socket_getpeersec_stream(sock, optval.user, optlen.user, len);
 
 	case SO_MARK:
 		v.val = sk->sk_mark;
@@ -1779,7 +1825,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
 		return sock_getbindtodevice(sk, optval, optlen, len);
 
 	case SO_GET_FILTER:
-		len = sk_get_filter(sk, (struct sock_filter __user *)optval, len);
+		len = sk_get_filter(sk, optval, len);
 		if (len < 0)
 			return len;
 
@@ -1827,7 +1873,7 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
 		sk_get_meminfo(sk, meminfo);
 
 		len = min_t(unsigned int, len, sizeof(meminfo));
-		if (copy_to_user(optval, &meminfo, len))
+		if (copy_to_sockptr(optval, &meminfo, len))
 			return -EFAULT;
 
 		goto lenout;
@@ -1896,14 +1942,22 @@ int sock_getsockopt(struct socket *sock, int level, int optname,
 
 	if (len > lv)
 		len = lv;
-	if (copy_to_user(optval, &v, len))
+	if (copy_to_sockptr(optval, &v, len))
 		return -EFAULT;
 lenout:
-	if (put_user(len, optlen))
+	if (copy_to_sockptr(optlen, &len, sizeof(int)))
 		return -EFAULT;
 	return 0;
 }
 
+int sock_getsockopt(struct socket *sock, int level, int optname,
+		    char __user *optval, int __user *optlen)
+{
+	return sk_getsockopt(sock->sk, level, optname,
+			     USER_SOCKPTR(optval),
+			     USER_SOCKPTR(optlen));
+}
+
 /*
  * Initialize an sk_lock.
  *
diff --git a/net/core/sock_map.c b/net/core/sock_map.c
index 9a9fb9487d63..a660baedd9e7 100644
--- a/net/core/sock_map.c
+++ b/net/core/sock_map.c
@@ -41,7 +41,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
 	    attr->map_flags & ~SOCK_CREATE_FLAG_MASK)
 		return ERR_PTR(-EINVAL);
 
-	stab = kzalloc(sizeof(*stab), GFP_USER | __GFP_ACCOUNT);
+	stab = bpf_map_area_alloc(sizeof(*stab), NUMA_NO_NODE);
 	if (!stab)
 		return ERR_PTR(-ENOMEM);
 
@@ -52,7 +52,7 @@ static struct bpf_map *sock_map_alloc(union bpf_attr *attr)
 				       sizeof(struct sock *),
 				       stab->map.numa_node);
 	if (!stab->sks) {
-		kfree(stab);
+		bpf_map_area_free(stab);
 		return ERR_PTR(-ENOMEM);
 	}
 
@@ -361,7 +361,7 @@ static void sock_map_free(struct bpf_map *map)
 	synchronize_rcu();
 
 	bpf_map_area_free(stab->sks);
-	kfree(stab);
+	bpf_map_area_free(stab);
 }
 
 static void sock_map_release_progs(struct bpf_map *map)
@@ -1085,7 +1085,7 @@ static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)
 	if (attr->key_size > MAX_BPF_STACK)
 		return ERR_PTR(-E2BIG);
 
-	htab = kzalloc(sizeof(*htab), GFP_USER | __GFP_ACCOUNT);
+	htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE);
 	if (!htab)
 		return ERR_PTR(-ENOMEM);
 
@@ -1115,7 +1115,7 @@ static struct bpf_map *sock_hash_alloc(union bpf_attr *attr)
 
 	return &htab->map;
 free_htab:
-	kfree(htab);
+	bpf_map_area_free(htab);
 	return ERR_PTR(err);
 }
 
@@ -1168,7 +1168,7 @@ static void sock_hash_free(struct bpf_map *map)
 	synchronize_rcu();
 
 	bpf_map_area_free(htab->buckets);
-	kfree(htab);
+	bpf_map_area_free(htab);
 }
 
 static void *sock_hash_lookup_sys(struct bpf_map *map, void *key)
diff --git a/net/core/stream.c b/net/core/stream.c
index ccc083cdef23..1105057ce00a 100644
--- a/net/core/stream.c
+++ b/net/core/stream.c
@@ -159,7 +159,8 @@ int sk_stream_wait_memory(struct sock *sk, long *timeo_p)
 		*timeo_p = current_timeo;
 	}
 out:
-	remove_wait_queue(sk_sleep(sk), &wait);
+	if (!sock_flag(sk, SOCK_DEAD))
+		remove_wait_queue(sk_sleep(sk), &wait);
 	return err;
 
 do_error:
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index 725891527814..5b1ce656baa1 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -29,7 +29,6 @@ static int int_3600 = 3600;
 static int min_sndbuf = SOCK_MIN_SNDBUF;
 static int min_rcvbuf = SOCK_MIN_RCVBUF;
 static int max_skb_frags = MAX_SKB_FRAGS;
-static long long_max __maybe_unused = LONG_MAX;
 
 static int net_msg_warn;	/* Unused, but still a sysctl */
 
diff --git a/net/core/xdp.c b/net/core/xdp.c
index 24420209bf0e..844c9d99dc0e 100644
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -375,19 +375,17 @@ EXPORT_SYMBOL_GPL(xdp_rxq_info_reg_mem_model);
 void __xdp_return(void *data, struct xdp_mem_info *mem, bool napi_direct,
 		  struct xdp_buff *xdp)
 {
-	struct xdp_mem_allocator *xa;
 	struct page *page;
 
 	switch (mem->type) {
 	case MEM_TYPE_PAGE_POOL:
-		rcu_read_lock();
-		/* mem->id is valid, checked in xdp_rxq_info_reg_mem_model() */
-		xa = rhashtable_lookup(mem_id_ht, &mem->id, mem_id_rht_params);
 		page = virt_to_head_page(data);
 		if (napi_direct && xdp_return_frame_no_direct())
 			napi_direct = false;
-		page_pool_put_full_page(xa->page_pool, page, napi_direct);
-		rcu_read_unlock();
+		/* No need to check ((page->pp_magic & ~0x3UL) == PP_SIGNATURE)
+		 * as mem->type knows this a page_pool page
+		 */
+		page_pool_put_full_page(page->pp, page, napi_direct);
 		break;
 	case MEM_TYPE_PAGE_SHARED:
 		page_frag_free(data);
diff --git a/net/dccp/ipv4.c b/net/dccp/ipv4.c
index da6e3b20cd75..6a6e121dc00c 100644
--- a/net/dccp/ipv4.c
+++ b/net/dccp/ipv4.c
@@ -45,10 +45,11 @@ static unsigned int dccp_v4_pernet_id __read_mostly;
 int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
 {
 	const struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
+	struct inet_bind_hashbucket *prev_addr_hashbucket = NULL;
+	__be32 daddr, nexthop, prev_sk_rcv_saddr;
 	struct inet_sock *inet = inet_sk(sk);
 	struct dccp_sock *dp = dccp_sk(sk);
 	__be16 orig_sport, orig_dport;
-	__be32 daddr, nexthop;
 	struct flowi4 *fl4;
 	struct rtable *rt;
 	int err;
@@ -89,9 +90,29 @@ int dccp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
 	if (inet_opt == NULL || !inet_opt->opt.srr)
 		daddr = fl4->daddr;
 
-	if (inet->inet_saddr == 0)
+	if (inet->inet_saddr == 0) {
+		if (inet_csk(sk)->icsk_bind2_hash) {
+			prev_addr_hashbucket =
+				inet_bhashfn_portaddr(&dccp_hashinfo, sk,
+						      sock_net(sk),
+						      inet->inet_num);
+			prev_sk_rcv_saddr = sk->sk_rcv_saddr;
+		}
 		inet->inet_saddr = fl4->saddr;
+	}
+
 	sk_rcv_saddr_set(sk, inet->inet_saddr);
+
+	if (prev_addr_hashbucket) {
+		err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk);
+		if (err) {
+			inet->inet_saddr = 0;
+			sk_rcv_saddr_set(sk, prev_sk_rcv_saddr);
+			ip_rt_put(rt);
+			return err;
+		}
+	}
+
 	inet->inet_dport = usin->sin_port;
 	sk_daddr_set(sk, daddr);
 
diff --git a/net/dccp/ipv6.c b/net/dccp/ipv6.c
index fd44638ec16b..e57b43006074 100644
--- a/net/dccp/ipv6.c
+++ b/net/dccp/ipv6.c
@@ -934,8 +934,26 @@ static int dccp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 	}
 
 	if (saddr == NULL) {
+		struct inet_bind_hashbucket *prev_addr_hashbucket = NULL;
+		struct in6_addr prev_v6_rcv_saddr;
+
+		if (icsk->icsk_bind2_hash) {
+			prev_addr_hashbucket = inet_bhashfn_portaddr(&dccp_hashinfo,
+								     sk, sock_net(sk),
+								     inet->inet_num);
+			prev_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
+		}
+
 		saddr = &fl6.saddr;
 		sk->sk_v6_rcv_saddr = *saddr;
+
+		if (prev_addr_hashbucket) {
+			err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk);
+			if (err) {
+				sk->sk_v6_rcv_saddr = prev_v6_rcv_saddr;
+				goto failure;
+			}
+		}
 	}
 
 	/* set the source address */
diff --git a/net/dccp/proto.c b/net/dccp/proto.c
index e13641c65f88..c548ca3e9b0e 100644
--- a/net/dccp/proto.c
+++ b/net/dccp/proto.c
@@ -1120,6 +1120,12 @@ static int __init dccp_init(void)
 				  SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);
 	if (!dccp_hashinfo.bind_bucket_cachep)
 		goto out_free_hashinfo2;
+	dccp_hashinfo.bind2_bucket_cachep =
+		kmem_cache_create("dccp_bind2_bucket",
+				  sizeof(struct inet_bind2_bucket), 0,
+				  SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT, NULL);
+	if (!dccp_hashinfo.bind2_bucket_cachep)
+		goto out_free_bind_bucket_cachep;
 
 	/*
 	 * Size and allocate the main established and bind bucket
@@ -1150,7 +1156,7 @@ static int __init dccp_init(void)
 
 	if (!dccp_hashinfo.ehash) {
 		DCCP_CRIT("Failed to allocate DCCP established hash table");
-		goto out_free_bind_bucket_cachep;
+		goto out_free_bind2_bucket_cachep;
 	}
 
 	for (i = 0; i <= dccp_hashinfo.ehash_mask; i++)
@@ -1176,14 +1182,26 @@ static int __init dccp_init(void)
 		goto out_free_dccp_locks;
 	}
 
+	dccp_hashinfo.bhash2 = (struct inet_bind_hashbucket *)
+		__get_free_pages(GFP_ATOMIC | __GFP_NOWARN, bhash_order);
+
+	if (!dccp_hashinfo.bhash2) {
+		DCCP_CRIT("Failed to allocate DCCP bind2 hash table");
+		goto out_free_dccp_bhash;
+	}
+
 	for (i = 0; i < dccp_hashinfo.bhash_size; i++) {
 		spin_lock_init(&dccp_hashinfo.bhash[i].lock);
 		INIT_HLIST_HEAD(&dccp_hashinfo.bhash[i].chain);
+		spin_lock_init(&dccp_hashinfo.bhash2[i].lock);
+		INIT_HLIST_HEAD(&dccp_hashinfo.bhash2[i].chain);
 	}
 
+	dccp_hashinfo.pernet = false;
+
 	rc = dccp_mib_init();
 	if (rc)
-		goto out_free_dccp_bhash;
+		goto out_free_dccp_bhash2;
 
 	rc = dccp_ackvec_init();
 	if (rc)
@@ -1207,30 +1225,38 @@ out_ackvec_exit:
 	dccp_ackvec_exit();
 out_free_dccp_mib:
 	dccp_mib_exit();
+out_free_dccp_bhash2:
+	free_pages((unsigned long)dccp_hashinfo.bhash2, bhash_order);
 out_free_dccp_bhash:
 	free_pages((unsigned long)dccp_hashinfo.bhash, bhash_order);
 out_free_dccp_locks:
 	inet_ehash_locks_free(&dccp_hashinfo);
 out_free_dccp_ehash:
 	free_pages((unsigned long)dccp_hashinfo.ehash, ehash_order);
+out_free_bind2_bucket_cachep:
+	kmem_cache_destroy(dccp_hashinfo.bind2_bucket_cachep);
 out_free_bind_bucket_cachep:
 	kmem_cache_destroy(dccp_hashinfo.bind_bucket_cachep);
 out_free_hashinfo2:
 	inet_hashinfo2_free_mod(&dccp_hashinfo);
 out_fail:
 	dccp_hashinfo.bhash = NULL;
+	dccp_hashinfo.bhash2 = NULL;
 	dccp_hashinfo.ehash = NULL;
 	dccp_hashinfo.bind_bucket_cachep = NULL;
+	dccp_hashinfo.bind2_bucket_cachep = NULL;
 	return rc;
 }
 
 static void __exit dccp_fini(void)
 {
+	int bhash_order = get_order(dccp_hashinfo.bhash_size *
+				    sizeof(struct inet_bind_hashbucket));
+
 	ccid_cleanup_builtins();
 	dccp_mib_exit();
-	free_pages((unsigned long)dccp_hashinfo.bhash,
-		   get_order(dccp_hashinfo.bhash_size *
-			     sizeof(struct inet_bind_hashbucket)));
+	free_pages((unsigned long)dccp_hashinfo.bhash, bhash_order);
+	free_pages((unsigned long)dccp_hashinfo.bhash2, bhash_order);
 	free_pages((unsigned long)dccp_hashinfo.ehash,
 		   get_order((dccp_hashinfo.ehash_mask + 1) *
 			     sizeof(struct inet_ehash_bucket)));
diff --git a/net/decnet/Kconfig b/net/decnet/Kconfig
deleted file mode 100644
index 24336bdb1054..000000000000
--- a/net/decnet/Kconfig
+++ /dev/null
@@ -1,43 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# DECnet configuration
-#
-config DECNET
-	tristate "DECnet Support"
-	help
-	  The DECnet networking protocol was used in many products made by
-	  Digital (now Compaq).  It provides reliable stream and sequenced
-	  packet communications over which run a variety of services similar
-	  to those which run over TCP/IP.
-
-	  To find some tools to use with the kernel layer support, please
-	  look at Patrick Caulfield's web site:
-	  <http://linux-decnet.sourceforge.net/>.
-
-	  More detailed documentation is available in
-	  <file:Documentation/networking/decnet.rst>.
-
-	  Be sure to say Y to "/proc file system support" and "Sysctl support"
-	  below when using DECnet, since you will need sysctl support to aid
-	  in configuration at run time.
-
-	  The DECnet code is also available as a module ( = code which can be
-	  inserted in and removed from the running kernel whenever you want).
-	  The module is called decnet.
-
-config DECNET_ROUTER
-	bool "DECnet: router support"
-	depends on DECNET
-	select FIB_RULES
-	help
-	  Add support for turning your DECnet Endnode into a level 1 or 2
-	  router.  This is an experimental, but functional option.  If you
-	  do say Y here, then make sure that you also say Y to "Kernel/User
-	  network link driver", "Routing messages" and "Network packet
-	  filtering".  The first two are required to allow configuration via
-	  rtnetlink (you will need Alexey Kuznetsov's iproute2 package
-	  from <ftp://ftp.tux.org/pub/net/ip-routing/>). The "Network packet
-	  filtering" option will be required for the forthcoming routing daemon
-	  to work.
-
-	  See <file:Documentation/networking/decnet.rst> for more information.
diff --git a/net/decnet/Makefile b/net/decnet/Makefile
deleted file mode 100644
index 07b38e441b2d..000000000000
--- a/net/decnet/Makefile
+++ /dev/null
@@ -1,10 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-
-obj-$(CONFIG_DECNET) += decnet.o
-
-decnet-y := af_decnet.o dn_nsp_in.o dn_nsp_out.o \
-	    dn_route.o dn_dev.o dn_neigh.o dn_timer.o
-decnet-$(CONFIG_DECNET_ROUTER) += dn_fib.o dn_rules.o dn_table.o
-decnet-y += sysctl_net_decnet.o
-
-obj-$(CONFIG_NETFILTER) += netfilter/
diff --git a/net/decnet/README b/net/decnet/README
deleted file mode 100644
index 60e7ec88c81f..000000000000
--- a/net/decnet/README
+++ /dev/null
@@ -1,8 +0,0 @@
-                       Linux DECnet Project
-                      ======================
-
-The documentation for this kernel subsystem is available in the
-Documentation/networking subdirectory of this distribution and also
-on line at http://www.chygwyn.com/DECnet/
-
-Steve Whitehouse <SteveW@ACM.org>
diff --git a/net/decnet/af_decnet.c b/net/decnet/af_decnet.c
deleted file mode 100644
index 6582dfdfb932..000000000000
--- a/net/decnet/af_decnet.c
+++ /dev/null
@@ -1,2404 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Socket Layer Interface
- *
- * Authors:     Eduardo Marcelo Serrat <emserrat@geocities.com>
- *              Patrick Caulfield <patrick@pandh.demon.co.uk>
- *
- * Changes:
- *        Steve Whitehouse: Copied from Eduardo Serrat and Patrick Caulfield's
- *                          version of the code. Original copyright preserved
- *                          below.
- *        Steve Whitehouse: Some bug fixes, cleaning up some code to make it
- *                          compatible with my routing layer.
- *        Steve Whitehouse: Merging changes from Eduardo Serrat and Patrick
- *                          Caulfield.
- *        Steve Whitehouse: Further bug fixes, checking module code still works
- *                          with new routing layer.
- *        Steve Whitehouse: Additional set/get_sockopt() calls.
- *        Steve Whitehouse: Fixed TIOCINQ ioctl to be same as Eduardo's new
- *                          code.
- *        Steve Whitehouse: recvmsg() changed to try and behave in a POSIX like
- *                          way. Didn't manage it entirely, but its better.
- *        Steve Whitehouse: ditto for sendmsg().
- *        Steve Whitehouse: A selection of bug fixes to various things.
- *        Steve Whitehouse: Added TIOCOUTQ ioctl.
- *        Steve Whitehouse: Fixes to username2sockaddr & sockaddr2username.
- *        Steve Whitehouse: Fixes to connect() error returns.
- *       Patrick Caulfield: Fixes to delayed acceptance logic.
- *         David S. Miller: New socket locking
- *        Steve Whitehouse: Socket list hashing/locking
- *         Arnaldo C. Melo: use capable, not suser
- *        Steve Whitehouse: Removed unused code. Fix to use sk->allocation
- *                          when required.
- *       Patrick Caulfield: /proc/net/decnet now has object name/number
- *        Steve Whitehouse: Fixed local port allocation, hashed sk list
- *          Matthew Wilcox: Fixes for dn_ioctl()
- *        Steve Whitehouse: New connect/accept logic to allow timeouts and
- *                          prepare for sendpage etc.
- */
-
-
-/******************************************************************************
-    (c) 1995-1998 E.M. Serrat		emserrat@geocities.com
-
-
-HISTORY:
-
-Version           Kernel     Date       Author/Comments
--------           ------     ----       ---------------
-Version 0.0.1     2.0.30    01-dic-97	Eduardo Marcelo Serrat
-					(emserrat@geocities.com)
-
-					First Development of DECnet Socket La-
-					yer for Linux. Only supports outgoing
-					connections.
-
-Version 0.0.2	  2.1.105   20-jun-98   Patrick J. Caulfield
-					(patrick@pandh.demon.co.uk)
-
-					Port to new kernel development version.
-
-Version 0.0.3     2.1.106   25-jun-98   Eduardo Marcelo Serrat
-					(emserrat@geocities.com)
-					_
-					Added support for incoming connections
-					so we can start developing server apps
-					on Linux.
-					-
-					Module Support
-Version 0.0.4     2.1.109   21-jul-98   Eduardo Marcelo Serrat
-				       (emserrat@geocities.com)
-				       _
-					Added support for X11R6.4. Now we can
-					use DECnet transport for X on Linux!!!
-				       -
-Version 0.0.5    2.1.110   01-aug-98   Eduardo Marcelo Serrat
-				       (emserrat@geocities.com)
-				       Removed bugs on flow control
-				       Removed bugs on incoming accessdata
-				       order
-				       -
-Version 0.0.6    2.1.110   07-aug-98   Eduardo Marcelo Serrat
-				       dn_recvmsg fixes
-
-					Patrick J. Caulfield
-				       dn_bind fixes
-*******************************************************************************/
-
-#include <linux/module.h>
-#include <linux/errno.h>
-#include <linux/types.h>
-#include <linux/slab.h>
-#include <linux/socket.h>
-#include <linux/in.h>
-#include <linux/kernel.h>
-#include <linux/sched/signal.h>
-#include <linux/timer.h>
-#include <linux/string.h>
-#include <linux/sockios.h>
-#include <linux/net.h>
-#include <linux/netdevice.h>
-#include <linux/inet.h>
-#include <linux/route.h>
-#include <linux/netfilter.h>
-#include <linux/seq_file.h>
-#include <net/sock.h>
-#include <net/tcp_states.h>
-#include <net/flow.h>
-#include <asm/ioctls.h>
-#include <linux/capability.h>
-#include <linux/mm.h>
-#include <linux/interrupt.h>
-#include <linux/proc_fs.h>
-#include <linux/stat.h>
-#include <linux/init.h>
-#include <linux/poll.h>
-#include <linux/jiffies.h>
-#include <net/net_namespace.h>
-#include <net/neighbour.h>
-#include <net/dst.h>
-#include <net/fib_rules.h>
-#include <net/tcp.h>
-#include <net/dn.h>
-#include <net/dn_nsp.h>
-#include <net/dn_dev.h>
-#include <net/dn_route.h>
-#include <net/dn_fib.h>
-#include <net/dn_neigh.h>
-
-struct dn_sock {
-	struct sock sk;
-	struct dn_scp scp;
-};
-
-static void dn_keepalive(struct sock *sk);
-
-#define DN_SK_HASH_SHIFT 8
-#define DN_SK_HASH_SIZE (1 << DN_SK_HASH_SHIFT)
-#define DN_SK_HASH_MASK (DN_SK_HASH_SIZE - 1)
-
-
-static const struct proto_ops dn_proto_ops;
-static DEFINE_RWLOCK(dn_hash_lock);
-static struct hlist_head dn_sk_hash[DN_SK_HASH_SIZE];
-static struct hlist_head dn_wild_sk;
-static atomic_long_t decnet_memory_allocated;
-static DEFINE_PER_CPU(int, decnet_memory_per_cpu_fw_alloc);
-
-static int __dn_setsockopt(struct socket *sock, int level, int optname,
-		sockptr_t optval, unsigned int optlen, int flags);
-static int __dn_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen, int flags);
-
-static struct hlist_head *dn_find_list(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	if (scp->addr.sdn_flags & SDF_WILD)
-		return hlist_empty(&dn_wild_sk) ? &dn_wild_sk : NULL;
-
-	return &dn_sk_hash[le16_to_cpu(scp->addrloc) & DN_SK_HASH_MASK];
-}
-
-/*
- * Valid ports are those greater than zero and not already in use.
- */
-static int check_port(__le16 port)
-{
-	struct sock *sk;
-
-	if (port == 0)
-		return -1;
-
-	sk_for_each(sk, &dn_sk_hash[le16_to_cpu(port) & DN_SK_HASH_MASK]) {
-		struct dn_scp *scp = DN_SK(sk);
-		if (scp->addrloc == port)
-			return -1;
-	}
-	return 0;
-}
-
-static unsigned short port_alloc(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	static unsigned short port = 0x2000;
-	unsigned short i_port = port;
-
-	while(check_port(cpu_to_le16(++port)) != 0) {
-		if (port == i_port)
-			return 0;
-	}
-
-	scp->addrloc = cpu_to_le16(port);
-
-	return 1;
-}
-
-/*
- * Since this is only ever called from user
- * level, we don't need a write_lock() version
- * of this.
- */
-static int dn_hash_sock(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	struct hlist_head *list;
-	int rv = -EUSERS;
-
-	BUG_ON(sk_hashed(sk));
-
-	write_lock_bh(&dn_hash_lock);
-
-	if (!scp->addrloc && !port_alloc(sk))
-		goto out;
-
-	rv = -EADDRINUSE;
-	if ((list = dn_find_list(sk)) == NULL)
-		goto out;
-
-	sk_add_node(sk, list);
-	rv = 0;
-out:
-	write_unlock_bh(&dn_hash_lock);
-	return rv;
-}
-
-static void dn_unhash_sock(struct sock *sk)
-{
-	write_lock(&dn_hash_lock);
-	sk_del_node_init(sk);
-	write_unlock(&dn_hash_lock);
-}
-
-static void dn_unhash_sock_bh(struct sock *sk)
-{
-	write_lock_bh(&dn_hash_lock);
-	sk_del_node_init(sk);
-	write_unlock_bh(&dn_hash_lock);
-}
-
-static struct hlist_head *listen_hash(struct sockaddr_dn *addr)
-{
-	int i;
-	unsigned int hash = addr->sdn_objnum;
-
-	if (hash == 0) {
-		hash = addr->sdn_objnamel;
-		for(i = 0; i < le16_to_cpu(addr->sdn_objnamel); i++) {
-			hash ^= addr->sdn_objname[i];
-			hash ^= (hash << 3);
-		}
-	}
-
-	return &dn_sk_hash[hash & DN_SK_HASH_MASK];
-}
-
-/*
- * Called to transform a socket from bound (i.e. with a local address)
- * into a listening socket (doesn't need a local port number) and rehashes
- * based upon the object name/number.
- */
-static void dn_rehash_sock(struct sock *sk)
-{
-	struct hlist_head *list;
-	struct dn_scp *scp = DN_SK(sk);
-
-	if (scp->addr.sdn_flags & SDF_WILD)
-		return;
-
-	write_lock_bh(&dn_hash_lock);
-	sk_del_node_init(sk);
-	DN_SK(sk)->addrloc = 0;
-	list = listen_hash(&DN_SK(sk)->addr);
-	sk_add_node(sk, list);
-	write_unlock_bh(&dn_hash_lock);
-}
-
-int dn_sockaddr2username(struct sockaddr_dn *sdn, unsigned char *buf, unsigned char type)
-{
-	int len = 2;
-
-	*buf++ = type;
-
-	switch (type) {
-	case 0:
-		*buf++ = sdn->sdn_objnum;
-		break;
-	case 1:
-		*buf++ = 0;
-		*buf++ = le16_to_cpu(sdn->sdn_objnamel);
-		memcpy(buf, sdn->sdn_objname, le16_to_cpu(sdn->sdn_objnamel));
-		len = 3 + le16_to_cpu(sdn->sdn_objnamel);
-		break;
-	case 2:
-		memset(buf, 0, 5);
-		buf += 5;
-		*buf++ = le16_to_cpu(sdn->sdn_objnamel);
-		memcpy(buf, sdn->sdn_objname, le16_to_cpu(sdn->sdn_objnamel));
-		len = 7 + le16_to_cpu(sdn->sdn_objnamel);
-		break;
-	}
-
-	return len;
-}
-
-/*
- * On reception of usernames, we handle types 1 and 0 for destination
- * addresses only. Types 2 and 4 are used for source addresses, but the
- * UIC, GIC are ignored and they are both treated the same way. Type 3
- * is never used as I've no idea what its purpose might be or what its
- * format is.
- */
-int dn_username2sockaddr(unsigned char *data, int len, struct sockaddr_dn *sdn, unsigned char *fmt)
-{
-	unsigned char type;
-	int size = len;
-	int namel = 12;
-
-	sdn->sdn_objnum = 0;
-	sdn->sdn_objnamel = cpu_to_le16(0);
-	memset(sdn->sdn_objname, 0, DN_MAXOBJL);
-
-	if (len < 2)
-		return -1;
-
-	len -= 2;
-	*fmt = *data++;
-	type = *data++;
-
-	switch (*fmt) {
-	case 0:
-		sdn->sdn_objnum = type;
-		return 2;
-	case 1:
-		namel = 16;
-		break;
-	case 2:
-		len  -= 4;
-		data += 4;
-		break;
-	case 4:
-		len  -= 8;
-		data += 8;
-		break;
-	default:
-		return -1;
-	}
-
-	len -= 1;
-
-	if (len < 0)
-		return -1;
-
-	sdn->sdn_objnamel = cpu_to_le16(*data++);
-	len -= le16_to_cpu(sdn->sdn_objnamel);
-
-	if ((len < 0) || (le16_to_cpu(sdn->sdn_objnamel) > namel))
-		return -1;
-
-	memcpy(sdn->sdn_objname, data, le16_to_cpu(sdn->sdn_objnamel));
-
-	return size - len;
-}
-
-struct sock *dn_sklist_find_listener(struct sockaddr_dn *addr)
-{
-	struct hlist_head *list = listen_hash(addr);
-	struct sock *sk;
-
-	read_lock(&dn_hash_lock);
-	sk_for_each(sk, list) {
-		struct dn_scp *scp = DN_SK(sk);
-		if (sk->sk_state != TCP_LISTEN)
-			continue;
-		if (scp->addr.sdn_objnum) {
-			if (scp->addr.sdn_objnum != addr->sdn_objnum)
-				continue;
-		} else {
-			if (addr->sdn_objnum)
-				continue;
-			if (scp->addr.sdn_objnamel != addr->sdn_objnamel)
-				continue;
-			if (memcmp(scp->addr.sdn_objname, addr->sdn_objname, le16_to_cpu(addr->sdn_objnamel)) != 0)
-				continue;
-		}
-		sock_hold(sk);
-		read_unlock(&dn_hash_lock);
-		return sk;
-	}
-
-	sk = sk_head(&dn_wild_sk);
-	if (sk) {
-		if (sk->sk_state == TCP_LISTEN)
-			sock_hold(sk);
-		else
-			sk = NULL;
-	}
-
-	read_unlock(&dn_hash_lock);
-	return sk;
-}
-
-struct sock *dn_find_by_skb(struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	struct sock *sk;
-	struct dn_scp *scp;
-
-	read_lock(&dn_hash_lock);
-	sk_for_each(sk, &dn_sk_hash[le16_to_cpu(cb->dst_port) & DN_SK_HASH_MASK]) {
-		scp = DN_SK(sk);
-		if (cb->src != dn_saddr2dn(&scp->peer))
-			continue;
-		if (cb->dst_port != scp->addrloc)
-			continue;
-		if (scp->addrrem && (cb->src_port != scp->addrrem))
-			continue;
-		sock_hold(sk);
-		goto found;
-	}
-	sk = NULL;
-found:
-	read_unlock(&dn_hash_lock);
-	return sk;
-}
-
-
-
-static void dn_destruct(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	skb_queue_purge(&scp->data_xmit_queue);
-	skb_queue_purge(&scp->other_xmit_queue);
-	skb_queue_purge(&scp->other_receive_queue);
-
-	dst_release(rcu_dereference_protected(sk->sk_dst_cache, 1));
-}
-
-static unsigned long dn_memory_pressure;
-
-static void dn_enter_memory_pressure(struct sock *sk)
-{
-	if (!dn_memory_pressure) {
-		dn_memory_pressure = 1;
-	}
-}
-
-static struct proto dn_proto = {
-	.name			= "NSP",
-	.owner			= THIS_MODULE,
-	.enter_memory_pressure	= dn_enter_memory_pressure,
-	.memory_pressure	= &dn_memory_pressure,
-
-	.memory_allocated	= &decnet_memory_allocated,
-	.per_cpu_fw_alloc	= &decnet_memory_per_cpu_fw_alloc,
-
-	.sysctl_mem		= sysctl_decnet_mem,
-	.sysctl_wmem		= sysctl_decnet_wmem,
-	.sysctl_rmem		= sysctl_decnet_rmem,
-	.max_header		= DN_MAX_NSP_DATA_HEADER + 64,
-	.obj_size		= sizeof(struct dn_sock),
-};
-
-static struct sock *dn_alloc_sock(struct net *net, struct socket *sock, gfp_t gfp, int kern)
-{
-	struct dn_scp *scp;
-	struct sock *sk = sk_alloc(net, PF_DECnet, gfp, &dn_proto, kern);
-
-	if  (!sk)
-		goto out;
-
-	if (sock)
-		sock->ops = &dn_proto_ops;
-	sock_init_data(sock, sk);
-
-	sk->sk_backlog_rcv = dn_nsp_backlog_rcv;
-	sk->sk_destruct    = dn_destruct;
-	sk->sk_no_check_tx = 1;
-	sk->sk_family      = PF_DECnet;
-	sk->sk_protocol    = 0;
-	sk->sk_allocation  = gfp;
-	sk->sk_sndbuf	   = READ_ONCE(sysctl_decnet_wmem[1]);
-	sk->sk_rcvbuf	   = READ_ONCE(sysctl_decnet_rmem[1]);
-
-	/* Initialization of DECnet Session Control Port		*/
-	scp = DN_SK(sk);
-	scp->state	= DN_O;		/* Open			*/
-	scp->numdat	= 1;		/* Next data seg to tx	*/
-	scp->numoth	= 1;		/* Next oth data to tx  */
-	scp->ackxmt_dat = 0;		/* Last data seg ack'ed */
-	scp->ackxmt_oth = 0;		/* Last oth data ack'ed */
-	scp->ackrcv_dat = 0;		/* Highest data ack recv*/
-	scp->ackrcv_oth = 0;		/* Last oth data ack rec*/
-	scp->flowrem_sw = DN_SEND;
-	scp->flowloc_sw = DN_SEND;
-	scp->flowrem_dat = 0;
-	scp->flowrem_oth = 1;
-	scp->flowloc_dat = 0;
-	scp->flowloc_oth = 1;
-	scp->services_rem = 0;
-	scp->services_loc = 1 | NSP_FC_NONE;
-	scp->info_rem = 0;
-	scp->info_loc = 0x03; /* NSP version 4.1 */
-	scp->segsize_rem = 230 - DN_MAX_NSP_DATA_HEADER; /* Default: Updated by remote segsize */
-	scp->nonagle = 0;
-	scp->multi_ireq = 1;
-	scp->accept_mode = ACC_IMMED;
-	scp->addr.sdn_family    = AF_DECnet;
-	scp->peer.sdn_family    = AF_DECnet;
-	scp->accessdata.acc_accl = 5;
-	memcpy(scp->accessdata.acc_acc, "LINUX", 5);
-
-	scp->max_window   = NSP_MAX_WINDOW;
-	scp->snd_window   = NSP_MIN_WINDOW;
-	scp->nsp_srtt     = NSP_INITIAL_SRTT;
-	scp->nsp_rttvar   = NSP_INITIAL_RTTVAR;
-	scp->nsp_rxtshift = 0;
-
-	skb_queue_head_init(&scp->data_xmit_queue);
-	skb_queue_head_init(&scp->other_xmit_queue);
-	skb_queue_head_init(&scp->other_receive_queue);
-
-	scp->persist = 0;
-	scp->persist_fxn = NULL;
-	scp->keepalive = 10 * HZ;
-	scp->keepalive_fxn = dn_keepalive;
-
-	dn_start_slow_timer(sk);
-out:
-	return sk;
-}
-
-/*
- * Keepalive timer.
- * FIXME: Should respond to SO_KEEPALIVE etc.
- */
-static void dn_keepalive(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	/*
-	 * By checking the other_data transmit queue is empty
-	 * we are double checking that we are not sending too
-	 * many of these keepalive frames.
-	 */
-	if (skb_queue_empty(&scp->other_xmit_queue))
-		dn_nsp_send_link(sk, DN_NOCHANGE, 0);
-}
-
-
-/*
- * Timer for shutdown/destroyed sockets.
- * When socket is dead & no packets have been sent for a
- * certain amount of time, they are removed by this
- * routine. Also takes care of sending out DI & DC
- * frames at correct times.
- */
-int dn_destroy_timer(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	scp->persist = dn_nsp_persist(sk);
-
-	switch (scp->state) {
-	case DN_DI:
-		dn_nsp_send_disc(sk, NSP_DISCINIT, 0, GFP_ATOMIC);
-		if (scp->nsp_rxtshift >= decnet_di_count)
-			scp->state = DN_CN;
-		return 0;
-
-	case DN_DR:
-		dn_nsp_send_disc(sk, NSP_DISCINIT, 0, GFP_ATOMIC);
-		if (scp->nsp_rxtshift >= decnet_dr_count)
-			scp->state = DN_DRC;
-		return 0;
-
-	case DN_DN:
-		if (scp->nsp_rxtshift < decnet_dn_count) {
-			/* printk(KERN_DEBUG "dn_destroy_timer: DN\n"); */
-			dn_nsp_send_disc(sk, NSP_DISCCONF, NSP_REASON_DC,
-					 GFP_ATOMIC);
-			return 0;
-		}
-	}
-
-	scp->persist = (HZ * decnet_time_wait);
-
-	if (sk->sk_socket)
-		return 0;
-
-	if (time_after_eq(jiffies, scp->stamp + HZ * decnet_time_wait)) {
-		dn_unhash_sock(sk);
-		sock_put(sk);
-		return 1;
-	}
-
-	return 0;
-}
-
-static void dn_destroy_sock(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	scp->nsp_rxtshift = 0; /* reset back off */
-
-	if (sk->sk_socket) {
-		if (sk->sk_socket->state != SS_UNCONNECTED)
-			sk->sk_socket->state = SS_DISCONNECTING;
-	}
-
-	sk->sk_state = TCP_CLOSE;
-
-	switch (scp->state) {
-	case DN_DN:
-		dn_nsp_send_disc(sk, NSP_DISCCONF, NSP_REASON_DC,
-				 sk->sk_allocation);
-		scp->persist_fxn = dn_destroy_timer;
-		scp->persist = dn_nsp_persist(sk);
-		break;
-	case DN_CR:
-		scp->state = DN_DR;
-		goto disc_reject;
-	case DN_RUN:
-		scp->state = DN_DI;
-		fallthrough;
-	case DN_DI:
-	case DN_DR:
-disc_reject:
-		dn_nsp_send_disc(sk, NSP_DISCINIT, 0, sk->sk_allocation);
-		fallthrough;
-	case DN_NC:
-	case DN_NR:
-	case DN_RJ:
-	case DN_DIC:
-	case DN_CN:
-	case DN_DRC:
-	case DN_CI:
-	case DN_CD:
-		scp->persist_fxn = dn_destroy_timer;
-		scp->persist = dn_nsp_persist(sk);
-		break;
-	default:
-		printk(KERN_DEBUG "DECnet: dn_destroy_sock passed socket in invalid state\n");
-		fallthrough;
-	case DN_O:
-		dn_stop_slow_timer(sk);
-
-		dn_unhash_sock_bh(sk);
-		sock_put(sk);
-
-		break;
-	}
-}
-
-char *dn_addr2asc(__u16 addr, char *buf)
-{
-	unsigned short node, area;
-
-	node = addr & 0x03ff;
-	area = addr >> 10;
-	sprintf(buf, "%hd.%hd", area, node);
-
-	return buf;
-}
-
-
-
-static int dn_create(struct net *net, struct socket *sock, int protocol,
-		     int kern)
-{
-	struct sock *sk;
-
-	if (protocol < 0 || protocol > U8_MAX)
-		return -EINVAL;
-
-	if (!net_eq(net, &init_net))
-		return -EAFNOSUPPORT;
-
-	switch (sock->type) {
-	case SOCK_SEQPACKET:
-		if (protocol != DNPROTO_NSP)
-			return -EPROTONOSUPPORT;
-		break;
-	case SOCK_STREAM:
-		break;
-	default:
-		return -ESOCKTNOSUPPORT;
-	}
-
-
-	if ((sk = dn_alloc_sock(net, sock, GFP_KERNEL, kern)) == NULL)
-		return -ENOBUFS;
-
-	sk->sk_protocol = protocol;
-
-	return 0;
-}
-
-
-static int
-dn_release(struct socket *sock)
-{
-	struct sock *sk = sock->sk;
-
-	if (sk) {
-		sock_orphan(sk);
-		sock_hold(sk);
-		lock_sock(sk);
-		dn_destroy_sock(sk);
-		release_sock(sk);
-		sock_put(sk);
-	}
-
-	return 0;
-}
-
-static int dn_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
-{
-	struct sock *sk = sock->sk;
-	struct dn_scp *scp = DN_SK(sk);
-	struct sockaddr_dn *saddr = (struct sockaddr_dn *)uaddr;
-	struct net_device *dev, *ldev;
-	int rv;
-
-	if (addr_len != sizeof(struct sockaddr_dn))
-		return -EINVAL;
-
-	if (saddr->sdn_family != AF_DECnet)
-		return -EINVAL;
-
-	if (le16_to_cpu(saddr->sdn_nodeaddrl) && (le16_to_cpu(saddr->sdn_nodeaddrl) != 2))
-		return -EINVAL;
-
-	if (le16_to_cpu(saddr->sdn_objnamel) > DN_MAXOBJL)
-		return -EINVAL;
-
-	if (saddr->sdn_flags & ~SDF_WILD)
-		return -EINVAL;
-
-	if (!capable(CAP_NET_BIND_SERVICE) && (saddr->sdn_objnum ||
-	    (saddr->sdn_flags & SDF_WILD)))
-		return -EACCES;
-
-	if (!(saddr->sdn_flags & SDF_WILD)) {
-		if (le16_to_cpu(saddr->sdn_nodeaddrl)) {
-			rcu_read_lock();
-			ldev = NULL;
-			for_each_netdev_rcu(&init_net, dev) {
-				if (!dev->dn_ptr)
-					continue;
-				if (dn_dev_islocal(dev, dn_saddr2dn(saddr))) {
-					ldev = dev;
-					break;
-				}
-			}
-			rcu_read_unlock();
-			if (ldev == NULL)
-				return -EADDRNOTAVAIL;
-		}
-	}
-
-	rv = -EINVAL;
-	lock_sock(sk);
-	if (sock_flag(sk, SOCK_ZAPPED)) {
-		memcpy(&scp->addr, saddr, addr_len);
-		sock_reset_flag(sk, SOCK_ZAPPED);
-
-		rv = dn_hash_sock(sk);
-		if (rv)
-			sock_set_flag(sk, SOCK_ZAPPED);
-	}
-	release_sock(sk);
-
-	return rv;
-}
-
-
-static int dn_auto_bind(struct socket *sock)
-{
-	struct sock *sk = sock->sk;
-	struct dn_scp *scp = DN_SK(sk);
-	int rv;
-
-	sock_reset_flag(sk, SOCK_ZAPPED);
-
-	scp->addr.sdn_flags  = 0;
-	scp->addr.sdn_objnum = 0;
-
-	/*
-	 * This stuff is to keep compatibility with Eduardo's
-	 * patch. I hope I can dispense with it shortly...
-	 */
-	if ((scp->accessdata.acc_accl != 0) &&
-		(scp->accessdata.acc_accl <= 12)) {
-
-		scp->addr.sdn_objnamel = cpu_to_le16(scp->accessdata.acc_accl);
-		memcpy(scp->addr.sdn_objname, scp->accessdata.acc_acc, le16_to_cpu(scp->addr.sdn_objnamel));
-
-		scp->accessdata.acc_accl = 0;
-		memset(scp->accessdata.acc_acc, 0, 40);
-	}
-	/* End of compatibility stuff */
-
-	scp->addr.sdn_add.a_len = cpu_to_le16(2);
-	rv = dn_dev_bind_default((__le16 *)scp->addr.sdn_add.a_addr);
-	if (rv == 0) {
-		rv = dn_hash_sock(sk);
-		if (rv)
-			sock_set_flag(sk, SOCK_ZAPPED);
-	}
-
-	return rv;
-}
-
-static int dn_confirm_accept(struct sock *sk, long *timeo, gfp_t allocation)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	DEFINE_WAIT_FUNC(wait, woken_wake_function);
-	int err;
-
-	if (scp->state != DN_CR)
-		return -EINVAL;
-
-	scp->state = DN_CC;
-	scp->segsize_loc = dst_metric_advmss(__sk_dst_get(sk));
-	dn_send_conn_conf(sk, allocation);
-
-	add_wait_queue(sk_sleep(sk), &wait);
-	for(;;) {
-		release_sock(sk);
-		if (scp->state == DN_CC)
-			*timeo = wait_woken(&wait, TASK_INTERRUPTIBLE, *timeo);
-		lock_sock(sk);
-		err = 0;
-		if (scp->state == DN_RUN)
-			break;
-		err = sock_error(sk);
-		if (err)
-			break;
-		err = sock_intr_errno(*timeo);
-		if (signal_pending(current))
-			break;
-		err = -EAGAIN;
-		if (!*timeo)
-			break;
-	}
-	remove_wait_queue(sk_sleep(sk), &wait);
-	if (err == 0) {
-		sk->sk_socket->state = SS_CONNECTED;
-	} else if (scp->state != DN_CC) {
-		sk->sk_socket->state = SS_UNCONNECTED;
-	}
-	return err;
-}
-
-static int dn_wait_run(struct sock *sk, long *timeo)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	DEFINE_WAIT_FUNC(wait, woken_wake_function);
-	int err = 0;
-
-	if (scp->state == DN_RUN)
-		goto out;
-
-	if (!*timeo)
-		return -EALREADY;
-
-	add_wait_queue(sk_sleep(sk), &wait);
-	for(;;) {
-		release_sock(sk);
-		if (scp->state == DN_CI || scp->state == DN_CC)
-			*timeo = wait_woken(&wait, TASK_INTERRUPTIBLE, *timeo);
-		lock_sock(sk);
-		err = 0;
-		if (scp->state == DN_RUN)
-			break;
-		err = sock_error(sk);
-		if (err)
-			break;
-		err = sock_intr_errno(*timeo);
-		if (signal_pending(current))
-			break;
-		err = -ETIMEDOUT;
-		if (!*timeo)
-			break;
-	}
-	remove_wait_queue(sk_sleep(sk), &wait);
-out:
-	if (err == 0) {
-		sk->sk_socket->state = SS_CONNECTED;
-	} else if (scp->state != DN_CI && scp->state != DN_CC) {
-		sk->sk_socket->state = SS_UNCONNECTED;
-	}
-	return err;
-}
-
-static int __dn_connect(struct sock *sk, struct sockaddr_dn *addr, int addrlen, long *timeo, int flags)
-{
-	struct socket *sock = sk->sk_socket;
-	struct dn_scp *scp = DN_SK(sk);
-	int err = -EISCONN;
-	struct flowidn fld;
-	struct dst_entry *dst;
-
-	if (sock->state == SS_CONNECTED)
-		goto out;
-
-	if (sock->state == SS_CONNECTING) {
-		err = 0;
-		if (scp->state == DN_RUN) {
-			sock->state = SS_CONNECTED;
-			goto out;
-		}
-		err = -ECONNREFUSED;
-		if (scp->state != DN_CI && scp->state != DN_CC) {
-			sock->state = SS_UNCONNECTED;
-			goto out;
-		}
-		return dn_wait_run(sk, timeo);
-	}
-
-	err = -EINVAL;
-	if (scp->state != DN_O)
-		goto out;
-
-	if (addr == NULL || addrlen != sizeof(struct sockaddr_dn))
-		goto out;
-	if (addr->sdn_family != AF_DECnet)
-		goto out;
-	if (addr->sdn_flags & SDF_WILD)
-		goto out;
-
-	if (sock_flag(sk, SOCK_ZAPPED)) {
-		err = dn_auto_bind(sk->sk_socket);
-		if (err)
-			goto out;
-	}
-
-	memcpy(&scp->peer, addr, sizeof(struct sockaddr_dn));
-
-	err = -EHOSTUNREACH;
-	memset(&fld, 0, sizeof(fld));
-	fld.flowidn_oif = sk->sk_bound_dev_if;
-	fld.daddr = dn_saddr2dn(&scp->peer);
-	fld.saddr = dn_saddr2dn(&scp->addr);
-	dn_sk_ports_copy(&fld, scp);
-	fld.flowidn_proto = DNPROTO_NSP;
-	if (dn_route_output_sock(&sk->sk_dst_cache, &fld, sk, flags) < 0)
-		goto out;
-	dst = __sk_dst_get(sk);
-	sk->sk_route_caps = dst->dev->features;
-	sock->state = SS_CONNECTING;
-	scp->state = DN_CI;
-	scp->segsize_loc = dst_metric_advmss(dst);
-
-	dn_nsp_send_conninit(sk, NSP_CI);
-	err = -EINPROGRESS;
-	if (*timeo) {
-		err = dn_wait_run(sk, timeo);
-	}
-out:
-	return err;
-}
-
-static int dn_connect(struct socket *sock, struct sockaddr *uaddr, int addrlen, int flags)
-{
-	struct sockaddr_dn *addr = (struct sockaddr_dn *)uaddr;
-	struct sock *sk = sock->sk;
-	int err;
-	long timeo = sock_sndtimeo(sk, flags & O_NONBLOCK);
-
-	lock_sock(sk);
-	err = __dn_connect(sk, addr, addrlen, &timeo, 0);
-	release_sock(sk);
-
-	return err;
-}
-
-static inline int dn_check_state(struct sock *sk, struct sockaddr_dn *addr, int addrlen, long *timeo, int flags)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	switch (scp->state) {
-	case DN_RUN:
-		return 0;
-	case DN_CR:
-		return dn_confirm_accept(sk, timeo, sk->sk_allocation);
-	case DN_CI:
-	case DN_CC:
-		return dn_wait_run(sk, timeo);
-	case DN_O:
-		return __dn_connect(sk, addr, addrlen, timeo, flags);
-	}
-
-	return -EINVAL;
-}
-
-
-static void dn_access_copy(struct sk_buff *skb, struct accessdata_dn *acc)
-{
-	unsigned char *ptr = skb->data;
-
-	acc->acc_userl = *ptr++;
-	memcpy(&acc->acc_user, ptr, acc->acc_userl);
-	ptr += acc->acc_userl;
-
-	acc->acc_passl = *ptr++;
-	memcpy(&acc->acc_pass, ptr, acc->acc_passl);
-	ptr += acc->acc_passl;
-
-	acc->acc_accl = *ptr++;
-	memcpy(&acc->acc_acc, ptr, acc->acc_accl);
-
-	skb_pull(skb, acc->acc_accl + acc->acc_passl + acc->acc_userl + 3);
-
-}
-
-static void dn_user_copy(struct sk_buff *skb, struct optdata_dn *opt)
-{
-	unsigned char *ptr = skb->data;
-	u16 len = *ptr++; /* yes, it's 8bit on the wire */
-
-	BUG_ON(len > 16); /* we've checked the contents earlier */
-	opt->opt_optl   = cpu_to_le16(len);
-	opt->opt_status = 0;
-	memcpy(opt->opt_data, ptr, len);
-	skb_pull(skb, len + 1);
-}
-
-static struct sk_buff *dn_wait_for_connect(struct sock *sk, long *timeo)
-{
-	DEFINE_WAIT_FUNC(wait, woken_wake_function);
-	struct sk_buff *skb = NULL;
-	int err = 0;
-
-	add_wait_queue(sk_sleep(sk), &wait);
-	for(;;) {
-		release_sock(sk);
-		skb = skb_dequeue(&sk->sk_receive_queue);
-		if (skb == NULL) {
-			*timeo = wait_woken(&wait, TASK_INTERRUPTIBLE, *timeo);
-			skb = skb_dequeue(&sk->sk_receive_queue);
-		}
-		lock_sock(sk);
-		if (skb != NULL)
-			break;
-		err = -EINVAL;
-		if (sk->sk_state != TCP_LISTEN)
-			break;
-		err = sock_intr_errno(*timeo);
-		if (signal_pending(current))
-			break;
-		err = -EAGAIN;
-		if (!*timeo)
-			break;
-	}
-	remove_wait_queue(sk_sleep(sk), &wait);
-
-	return skb == NULL ? ERR_PTR(err) : skb;
-}
-
-static int dn_accept(struct socket *sock, struct socket *newsock, int flags,
-		     bool kern)
-{
-	struct sock *sk = sock->sk, *newsk;
-	struct sk_buff *skb = NULL;
-	struct dn_skb_cb *cb;
-	unsigned char menuver;
-	int err = 0;
-	unsigned char type;
-	long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
-	struct dst_entry *dst;
-
-	lock_sock(sk);
-
-	if (sk->sk_state != TCP_LISTEN || DN_SK(sk)->state != DN_O) {
-		release_sock(sk);
-		return -EINVAL;
-	}
-
-	skb = skb_dequeue(&sk->sk_receive_queue);
-	if (skb == NULL) {
-		skb = dn_wait_for_connect(sk, &timeo);
-		if (IS_ERR(skb)) {
-			release_sock(sk);
-			return PTR_ERR(skb);
-		}
-	}
-
-	cb = DN_SKB_CB(skb);
-	sk_acceptq_removed(sk);
-	newsk = dn_alloc_sock(sock_net(sk), newsock, sk->sk_allocation, kern);
-	if (newsk == NULL) {
-		release_sock(sk);
-		kfree_skb(skb);
-		return -ENOBUFS;
-	}
-	release_sock(sk);
-
-	dst = skb_dst(skb);
-	sk_dst_set(newsk, dst);
-	skb_dst_set(skb, NULL);
-
-	DN_SK(newsk)->state        = DN_CR;
-	DN_SK(newsk)->addrrem      = cb->src_port;
-	DN_SK(newsk)->services_rem = cb->services;
-	DN_SK(newsk)->info_rem     = cb->info;
-	DN_SK(newsk)->segsize_rem  = cb->segsize;
-	DN_SK(newsk)->accept_mode  = DN_SK(sk)->accept_mode;
-
-	if (DN_SK(newsk)->segsize_rem < 230)
-		DN_SK(newsk)->segsize_rem = 230;
-
-	if ((DN_SK(newsk)->services_rem & NSP_FC_MASK) == NSP_FC_NONE)
-		DN_SK(newsk)->max_window = decnet_no_fc_max_cwnd;
-
-	newsk->sk_state  = TCP_LISTEN;
-	memcpy(&(DN_SK(newsk)->addr), &(DN_SK(sk)->addr), sizeof(struct sockaddr_dn));
-
-	/*
-	 * If we are listening on a wild socket, we don't want
-	 * the newly created socket on the wrong hash queue.
-	 */
-	DN_SK(newsk)->addr.sdn_flags &= ~SDF_WILD;
-
-	skb_pull(skb, dn_username2sockaddr(skb->data, skb->len, &(DN_SK(newsk)->addr), &type));
-	skb_pull(skb, dn_username2sockaddr(skb->data, skb->len, &(DN_SK(newsk)->peer), &type));
-	*(__le16 *)(DN_SK(newsk)->peer.sdn_add.a_addr) = cb->src;
-	*(__le16 *)(DN_SK(newsk)->addr.sdn_add.a_addr) = cb->dst;
-
-	menuver = *skb->data;
-	skb_pull(skb, 1);
-
-	if (menuver & DN_MENUVER_ACC)
-		dn_access_copy(skb, &(DN_SK(newsk)->accessdata));
-
-	if (menuver & DN_MENUVER_USR)
-		dn_user_copy(skb, &(DN_SK(newsk)->conndata_in));
-
-	if (menuver & DN_MENUVER_PRX)
-		DN_SK(newsk)->peer.sdn_flags |= SDF_PROXY;
-
-	if (menuver & DN_MENUVER_UIC)
-		DN_SK(newsk)->peer.sdn_flags |= SDF_UICPROXY;
-
-	kfree_skb(skb);
-
-	memcpy(&(DN_SK(newsk)->conndata_out), &(DN_SK(sk)->conndata_out),
-		sizeof(struct optdata_dn));
-	memcpy(&(DN_SK(newsk)->discdata_out), &(DN_SK(sk)->discdata_out),
-		sizeof(struct optdata_dn));
-
-	lock_sock(newsk);
-	err = dn_hash_sock(newsk);
-	if (err == 0) {
-		sock_reset_flag(newsk, SOCK_ZAPPED);
-		dn_send_conn_ack(newsk);
-
-		/*
-		 * Here we use sk->sk_allocation since although the conn conf is
-		 * for the newsk, the context is the old socket.
-		 */
-		if (DN_SK(newsk)->accept_mode == ACC_IMMED)
-			err = dn_confirm_accept(newsk, &timeo,
-						sk->sk_allocation);
-	}
-	release_sock(newsk);
-	return err;
-}
-
-
-static int dn_getname(struct socket *sock, struct sockaddr *uaddr,int peer)
-{
-	struct sockaddr_dn *sa = (struct sockaddr_dn *)uaddr;
-	struct sock *sk = sock->sk;
-	struct dn_scp *scp = DN_SK(sk);
-
-	lock_sock(sk);
-
-	if (peer) {
-		if ((sock->state != SS_CONNECTED &&
-		     sock->state != SS_CONNECTING) &&
-		    scp->accept_mode == ACC_IMMED) {
-			release_sock(sk);
-			return -ENOTCONN;
-		}
-
-		memcpy(sa, &scp->peer, sizeof(struct sockaddr_dn));
-	} else {
-		memcpy(sa, &scp->addr, sizeof(struct sockaddr_dn));
-	}
-
-	release_sock(sk);
-
-	return sizeof(struct sockaddr_dn);
-}
-
-
-static __poll_t dn_poll(struct file *file, struct socket *sock, poll_table  *wait)
-{
-	struct sock *sk = sock->sk;
-	struct dn_scp *scp = DN_SK(sk);
-	__poll_t mask = datagram_poll(file, sock, wait);
-
-	if (!skb_queue_empty_lockless(&scp->other_receive_queue))
-		mask |= EPOLLRDBAND;
-
-	return mask;
-}
-
-static int dn_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
-{
-	struct sock *sk = sock->sk;
-	struct dn_scp *scp = DN_SK(sk);
-	int err = -EOPNOTSUPP;
-	long amount = 0;
-	struct sk_buff *skb;
-	int val;
-
-	switch(cmd)
-	{
-	case SIOCGIFADDR:
-	case SIOCSIFADDR:
-		return dn_dev_ioctl(cmd, (void __user *)arg);
-
-	case SIOCATMARK:
-		lock_sock(sk);
-		val = !skb_queue_empty(&scp->other_receive_queue);
-		if (scp->state != DN_RUN)
-			val = -ENOTCONN;
-		release_sock(sk);
-		return val;
-
-	case TIOCOUTQ:
-		amount = sk->sk_sndbuf - sk_wmem_alloc_get(sk);
-		if (amount < 0)
-			amount = 0;
-		err = put_user(amount, (int __user *)arg);
-		break;
-
-	case TIOCINQ:
-		lock_sock(sk);
-		skb = skb_peek(&scp->other_receive_queue);
-		if (skb) {
-			amount = skb->len;
-		} else {
-			skb_queue_walk(&sk->sk_receive_queue, skb)
-				amount += skb->len;
-		}
-		release_sock(sk);
-		err = put_user(amount, (int __user *)arg);
-		break;
-
-	default:
-		err = -ENOIOCTLCMD;
-		break;
-	}
-
-	return err;
-}
-
-static int dn_listen(struct socket *sock, int backlog)
-{
-	struct sock *sk = sock->sk;
-	int err = -EINVAL;
-
-	lock_sock(sk);
-
-	if (sock_flag(sk, SOCK_ZAPPED))
-		goto out;
-
-	if ((DN_SK(sk)->state != DN_O) || (sk->sk_state == TCP_LISTEN))
-		goto out;
-
-	sk->sk_max_ack_backlog = backlog;
-	sk->sk_ack_backlog     = 0;
-	sk->sk_state           = TCP_LISTEN;
-	err                 = 0;
-	dn_rehash_sock(sk);
-
-out:
-	release_sock(sk);
-
-	return err;
-}
-
-
-static int dn_shutdown(struct socket *sock, int how)
-{
-	struct sock *sk = sock->sk;
-	struct dn_scp *scp = DN_SK(sk);
-	int err = -ENOTCONN;
-
-	lock_sock(sk);
-
-	if (sock->state == SS_UNCONNECTED)
-		goto out;
-
-	err = 0;
-	if (sock->state == SS_DISCONNECTING)
-		goto out;
-
-	err = -EINVAL;
-	if (scp->state == DN_O)
-		goto out;
-
-	if (how != SHUT_RDWR)
-		goto out;
-
-	sk->sk_shutdown = SHUTDOWN_MASK;
-	dn_destroy_sock(sk);
-	err = 0;
-
-out:
-	release_sock(sk);
-
-	return err;
-}
-
-static int dn_setsockopt(struct socket *sock, int level, int optname,
-		sockptr_t optval, unsigned int optlen)
-{
-	struct sock *sk = sock->sk;
-	int err;
-
-	lock_sock(sk);
-	err = __dn_setsockopt(sock, level, optname, optval, optlen, 0);
-	release_sock(sk);
-#ifdef CONFIG_NETFILTER
-	/* we need to exclude all possible ENOPROTOOPTs except default case */
-	if (err == -ENOPROTOOPT && optname != DSO_LINKINFO &&
-	    optname != DSO_STREAM && optname != DSO_SEQPACKET)
-		err = nf_setsockopt(sk, PF_DECnet, optname, optval, optlen);
-#endif
-
-	return err;
-}
-
-static int __dn_setsockopt(struct socket *sock, int level, int optname,
-		sockptr_t optval, unsigned int optlen, int flags)
-{
-	struct	sock *sk = sock->sk;
-	struct dn_scp *scp = DN_SK(sk);
-	long timeo;
-	union {
-		struct optdata_dn opt;
-		struct accessdata_dn acc;
-		int mode;
-		unsigned long win;
-		int val;
-		unsigned char services;
-		unsigned char info;
-	} u;
-	int err;
-
-	if (optlen && sockptr_is_null(optval))
-		return -EINVAL;
-
-	if (optlen > sizeof(u))
-		return -EINVAL;
-
-	if (copy_from_sockptr(&u, optval, optlen))
-		return -EFAULT;
-
-	switch (optname) {
-	case DSO_CONDATA:
-		if (sock->state == SS_CONNECTED)
-			return -EISCONN;
-		if ((scp->state != DN_O) && (scp->state != DN_CR))
-			return -EINVAL;
-
-		if (optlen != sizeof(struct optdata_dn))
-			return -EINVAL;
-
-		if (le16_to_cpu(u.opt.opt_optl) > 16)
-			return -EINVAL;
-
-		memcpy(&scp->conndata_out, &u.opt, optlen);
-		break;
-
-	case DSO_DISDATA:
-		if (sock->state != SS_CONNECTED &&
-		    scp->accept_mode == ACC_IMMED)
-			return -ENOTCONN;
-
-		if (optlen != sizeof(struct optdata_dn))
-			return -EINVAL;
-
-		if (le16_to_cpu(u.opt.opt_optl) > 16)
-			return -EINVAL;
-
-		memcpy(&scp->discdata_out, &u.opt, optlen);
-		break;
-
-	case DSO_CONACCESS:
-		if (sock->state == SS_CONNECTED)
-			return -EISCONN;
-		if (scp->state != DN_O)
-			return -EINVAL;
-
-		if (optlen != sizeof(struct accessdata_dn))
-			return -EINVAL;
-
-		if ((u.acc.acc_accl > DN_MAXACCL) ||
-		    (u.acc.acc_passl > DN_MAXACCL) ||
-		    (u.acc.acc_userl > DN_MAXACCL))
-			return -EINVAL;
-
-		memcpy(&scp->accessdata, &u.acc, optlen);
-		break;
-
-	case DSO_ACCEPTMODE:
-		if (sock->state == SS_CONNECTED)
-			return -EISCONN;
-		if (scp->state != DN_O)
-			return -EINVAL;
-
-		if (optlen != sizeof(int))
-			return -EINVAL;
-
-		if ((u.mode != ACC_IMMED) && (u.mode != ACC_DEFER))
-			return -EINVAL;
-
-		scp->accept_mode = (unsigned char)u.mode;
-		break;
-
-	case DSO_CONACCEPT:
-		if (scp->state != DN_CR)
-			return -EINVAL;
-		timeo = sock_rcvtimeo(sk, 0);
-		err = dn_confirm_accept(sk, &timeo, sk->sk_allocation);
-		return err;
-
-	case DSO_CONREJECT:
-		if (scp->state != DN_CR)
-			return -EINVAL;
-
-		scp->state = DN_DR;
-		sk->sk_shutdown = SHUTDOWN_MASK;
-		dn_nsp_send_disc(sk, 0x38, 0, sk->sk_allocation);
-		break;
-
-	case DSO_MAXWINDOW:
-		if (optlen != sizeof(unsigned long))
-			return -EINVAL;
-		if (u.win > NSP_MAX_WINDOW)
-			u.win = NSP_MAX_WINDOW;
-		if (u.win == 0)
-			return -EINVAL;
-		scp->max_window = u.win;
-		if (scp->snd_window > u.win)
-			scp->snd_window = u.win;
-		break;
-
-	case DSO_NODELAY:
-		if (optlen != sizeof(int))
-			return -EINVAL;
-		if (scp->nonagle == TCP_NAGLE_CORK)
-			return -EINVAL;
-		scp->nonagle = (u.val == 0) ? 0 : TCP_NAGLE_OFF;
-		/* if (scp->nonagle == 1) { Push pending frames } */
-		break;
-
-	case DSO_CORK:
-		if (optlen != sizeof(int))
-			return -EINVAL;
-		if (scp->nonagle == TCP_NAGLE_OFF)
-			return -EINVAL;
-		scp->nonagle = (u.val == 0) ? 0 : TCP_NAGLE_CORK;
-		/* if (scp->nonagle == 0) { Push pending frames } */
-		break;
-
-	case DSO_SERVICES:
-		if (optlen != sizeof(unsigned char))
-			return -EINVAL;
-		if ((u.services & ~NSP_FC_MASK) != 0x01)
-			return -EINVAL;
-		if ((u.services & NSP_FC_MASK) == NSP_FC_MASK)
-			return -EINVAL;
-		scp->services_loc = u.services;
-		break;
-
-	case DSO_INFO:
-		if (optlen != sizeof(unsigned char))
-			return -EINVAL;
-		if (u.info & 0xfc)
-			return -EINVAL;
-		scp->info_loc = u.info;
-		break;
-
-	case DSO_LINKINFO:
-	case DSO_STREAM:
-	case DSO_SEQPACKET:
-	default:
-		return -ENOPROTOOPT;
-	}
-
-	return 0;
-}
-
-static int dn_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen)
-{
-	struct sock *sk = sock->sk;
-	int err;
-
-	lock_sock(sk);
-	err = __dn_getsockopt(sock, level, optname, optval, optlen, 0);
-	release_sock(sk);
-#ifdef CONFIG_NETFILTER
-	if (err == -ENOPROTOOPT && optname != DSO_STREAM &&
-	    optname != DSO_SEQPACKET && optname != DSO_CONACCEPT &&
-	    optname != DSO_CONREJECT) {
-		int len;
-
-		if (get_user(len, optlen))
-			return -EFAULT;
-
-		err = nf_getsockopt(sk, PF_DECnet, optname, optval, &len);
-		if (err >= 0)
-			err = put_user(len, optlen);
-	}
-#endif
-
-	return err;
-}
-
-static int __dn_getsockopt(struct socket *sock, int level,int optname, char __user *optval,int __user *optlen, int flags)
-{
-	struct	sock *sk = sock->sk;
-	struct dn_scp *scp = DN_SK(sk);
-	struct linkinfo_dn link;
-	unsigned int r_len;
-	void *r_data = NULL;
-	unsigned int val;
-
-	if(get_user(r_len , optlen))
-		return -EFAULT;
-
-	switch (optname) {
-	case DSO_CONDATA:
-		if (r_len > sizeof(struct optdata_dn))
-			r_len = sizeof(struct optdata_dn);
-		r_data = &scp->conndata_in;
-		break;
-
-	case DSO_DISDATA:
-		if (r_len > sizeof(struct optdata_dn))
-			r_len = sizeof(struct optdata_dn);
-		r_data = &scp->discdata_in;
-		break;
-
-	case DSO_CONACCESS:
-		if (r_len > sizeof(struct accessdata_dn))
-			r_len = sizeof(struct accessdata_dn);
-		r_data = &scp->accessdata;
-		break;
-
-	case DSO_ACCEPTMODE:
-		if (r_len > sizeof(unsigned char))
-			r_len = sizeof(unsigned char);
-		r_data = &scp->accept_mode;
-		break;
-
-	case DSO_LINKINFO:
-		if (r_len > sizeof(struct linkinfo_dn))
-			r_len = sizeof(struct linkinfo_dn);
-
-		memset(&link, 0, sizeof(link));
-
-		switch (sock->state) {
-		case SS_CONNECTING:
-			link.idn_linkstate = LL_CONNECTING;
-			break;
-		case SS_DISCONNECTING:
-			link.idn_linkstate = LL_DISCONNECTING;
-			break;
-		case SS_CONNECTED:
-			link.idn_linkstate = LL_RUNNING;
-			break;
-		default:
-			link.idn_linkstate = LL_INACTIVE;
-		}
-
-		link.idn_segsize = scp->segsize_rem;
-		r_data = &link;
-		break;
-
-	case DSO_MAXWINDOW:
-		if (r_len > sizeof(unsigned long))
-			r_len = sizeof(unsigned long);
-		r_data = &scp->max_window;
-		break;
-
-	case DSO_NODELAY:
-		if (r_len > sizeof(int))
-			r_len = sizeof(int);
-		val = (scp->nonagle == TCP_NAGLE_OFF);
-		r_data = &val;
-		break;
-
-	case DSO_CORK:
-		if (r_len > sizeof(int))
-			r_len = sizeof(int);
-		val = (scp->nonagle == TCP_NAGLE_CORK);
-		r_data = &val;
-		break;
-
-	case DSO_SERVICES:
-		if (r_len > sizeof(unsigned char))
-			r_len = sizeof(unsigned char);
-		r_data = &scp->services_rem;
-		break;
-
-	case DSO_INFO:
-		if (r_len > sizeof(unsigned char))
-			r_len = sizeof(unsigned char);
-		r_data = &scp->info_rem;
-		break;
-
-	case DSO_STREAM:
-	case DSO_SEQPACKET:
-	case DSO_CONACCEPT:
-	case DSO_CONREJECT:
-	default:
-		return -ENOPROTOOPT;
-	}
-
-	if (r_data) {
-		if (copy_to_user(optval, r_data, r_len))
-			return -EFAULT;
-		if (put_user(r_len, optlen))
-			return -EFAULT;
-	}
-
-	return 0;
-}
-
-
-static int dn_data_ready(struct sock *sk, struct sk_buff_head *q, int flags, int target)
-{
-	struct sk_buff *skb;
-	int len = 0;
-
-	if (flags & MSG_OOB)
-		return !skb_queue_empty(q) ? 1 : 0;
-
-	skb_queue_walk(q, skb) {
-		struct dn_skb_cb *cb = DN_SKB_CB(skb);
-		len += skb->len;
-
-		if (cb->nsp_flags & 0x40) {
-			/* SOCK_SEQPACKET reads to EOM */
-			if (sk->sk_type == SOCK_SEQPACKET)
-				return 1;
-			/* so does SOCK_STREAM unless WAITALL is specified */
-			if (!(flags & MSG_WAITALL))
-				return 1;
-		}
-
-		/* minimum data length for read exceeded */
-		if (len >= target)
-			return 1;
-	}
-
-	return 0;
-}
-
-
-static int dn_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
-		      int flags)
-{
-	struct sock *sk = sock->sk;
-	struct dn_scp *scp = DN_SK(sk);
-	struct sk_buff_head *queue = &sk->sk_receive_queue;
-	size_t target = size > 1 ? 1 : 0;
-	size_t copied = 0;
-	int rv = 0;
-	struct sk_buff *skb, *n;
-	struct dn_skb_cb *cb = NULL;
-	unsigned char eor = 0;
-	long timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
-
-	lock_sock(sk);
-
-	if (sock_flag(sk, SOCK_ZAPPED)) {
-		rv = -EADDRNOTAVAIL;
-		goto out;
-	}
-
-	if (sk->sk_shutdown & RCV_SHUTDOWN) {
-		rv = 0;
-		goto out;
-	}
-
-	rv = dn_check_state(sk, NULL, 0, &timeo, flags);
-	if (rv)
-		goto out;
-
-	if (flags & ~(MSG_CMSG_COMPAT|MSG_PEEK|MSG_OOB|MSG_WAITALL|MSG_DONTWAIT|MSG_NOSIGNAL)) {
-		rv = -EOPNOTSUPP;
-		goto out;
-	}
-
-	if (flags & MSG_OOB)
-		queue = &scp->other_receive_queue;
-
-	if (flags & MSG_WAITALL)
-		target = size;
-
-
-	/*
-	 * See if there is data ready to read, sleep if there isn't
-	 */
-	for(;;) {
-		DEFINE_WAIT_FUNC(wait, woken_wake_function);
-
-		if (sk->sk_err)
-			goto out;
-
-		if (!skb_queue_empty(&scp->other_receive_queue)) {
-			if (!(flags & MSG_OOB)) {
-				msg->msg_flags |= MSG_OOB;
-				if (!scp->other_report) {
-					scp->other_report = 1;
-					goto out;
-				}
-			}
-		}
-
-		if (scp->state != DN_RUN)
-			goto out;
-
-		if (signal_pending(current)) {
-			rv = sock_intr_errno(timeo);
-			goto out;
-		}
-
-		if (dn_data_ready(sk, queue, flags, target))
-			break;
-
-		if (flags & MSG_DONTWAIT) {
-			rv = -EWOULDBLOCK;
-			goto out;
-		}
-
-		add_wait_queue(sk_sleep(sk), &wait);
-		sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
-		sk_wait_event(sk, &timeo, dn_data_ready(sk, queue, flags, target), &wait);
-		sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
-		remove_wait_queue(sk_sleep(sk), &wait);
-	}
-
-	skb_queue_walk_safe(queue, skb, n) {
-		unsigned int chunk = skb->len;
-		cb = DN_SKB_CB(skb);
-
-		if ((chunk + copied) > size)
-			chunk = size - copied;
-
-		if (memcpy_to_msg(msg, skb->data, chunk)) {
-			rv = -EFAULT;
-			break;
-		}
-		copied += chunk;
-
-		if (!(flags & MSG_PEEK))
-			skb_pull(skb, chunk);
-
-		eor = cb->nsp_flags & 0x40;
-
-		if (skb->len == 0) {
-			skb_unlink(skb, queue);
-			kfree_skb(skb);
-			/*
-			 * N.B. Don't refer to skb or cb after this point
-			 * in loop.
-			 */
-			if ((scp->flowloc_sw == DN_DONTSEND) && !dn_congested(sk)) {
-				scp->flowloc_sw = DN_SEND;
-				dn_nsp_send_link(sk, DN_SEND, 0);
-			}
-		}
-
-		if (eor) {
-			if (sk->sk_type == SOCK_SEQPACKET)
-				break;
-			if (!(flags & MSG_WAITALL))
-				break;
-		}
-
-		if (flags & MSG_OOB)
-			break;
-
-		if (copied >= target)
-			break;
-	}
-
-	rv = copied;
-
-
-	if (eor && (sk->sk_type == SOCK_SEQPACKET))
-		msg->msg_flags |= MSG_EOR;
-
-out:
-	if (rv == 0)
-		rv = (flags & MSG_PEEK) ? -sk->sk_err : sock_error(sk);
-
-	if ((rv >= 0) && msg->msg_name) {
-		__sockaddr_check_size(sizeof(struct sockaddr_dn));
-		memcpy(msg->msg_name, &scp->peer, sizeof(struct sockaddr_dn));
-		msg->msg_namelen = sizeof(struct sockaddr_dn);
-	}
-
-	release_sock(sk);
-
-	return rv;
-}
-
-
-static inline int dn_queue_too_long(struct dn_scp *scp, struct sk_buff_head *queue, int flags)
-{
-	unsigned char fctype = scp->services_rem & NSP_FC_MASK;
-	if (skb_queue_len(queue) >= scp->snd_window)
-		return 1;
-	if (fctype != NSP_FC_NONE) {
-		if (flags & MSG_OOB) {
-			if (scp->flowrem_oth == 0)
-				return 1;
-		} else {
-			if (scp->flowrem_dat == 0)
-				return 1;
-		}
-	}
-	return 0;
-}
-
-/*
- * The DECnet spec requires that the "routing layer" accepts packets which
- * are at least 230 bytes in size. This excludes any headers which the NSP
- * layer might add, so we always assume that we'll be using the maximal
- * length header on data packets. The variation in length is due to the
- * inclusion (or not) of the two 16 bit acknowledgement fields so it doesn't
- * make much practical difference.
- */
-unsigned int dn_mss_from_pmtu(struct net_device *dev, int mtu)
-{
-	unsigned int mss = 230 - DN_MAX_NSP_DATA_HEADER;
-	if (dev) {
-		struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr);
-		mtu -= LL_RESERVED_SPACE(dev);
-		if (dn_db->use_long)
-			mtu -= 21;
-		else
-			mtu -= 6;
-		mtu -= DN_MAX_NSP_DATA_HEADER;
-	} else {
-		/*
-		 * 21 = long header, 16 = guess at MAC header length
-		 */
-		mtu -= (21 + DN_MAX_NSP_DATA_HEADER + 16);
-	}
-	if (mtu > mss)
-		mss = mtu;
-	return mss;
-}
-
-static inline unsigned int dn_current_mss(struct sock *sk, int flags)
-{
-	struct dst_entry *dst = __sk_dst_get(sk);
-	struct dn_scp *scp = DN_SK(sk);
-	int mss_now = min_t(int, scp->segsize_loc, scp->segsize_rem);
-
-	/* Other data messages are limited to 16 bytes per packet */
-	if (flags & MSG_OOB)
-		return 16;
-
-	/* This works out the maximum size of segment we can send out */
-	if (dst) {
-		u32 mtu = dst_mtu(dst);
-		mss_now = min_t(int, dn_mss_from_pmtu(dst->dev, mtu), mss_now);
-	}
-
-	return mss_now;
-}
-
-/*
- * N.B. We get the timeout wrong here, but then we always did get it
- * wrong before and this is another step along the road to correcting
- * it. It ought to get updated each time we pass through the routine,
- * but in practise it probably doesn't matter too much for now.
- */
-static inline struct sk_buff *dn_alloc_send_pskb(struct sock *sk,
-			      unsigned long datalen, int noblock,
-			      int *errcode)
-{
-	struct sk_buff *skb = sock_alloc_send_skb(sk, datalen,
-						   noblock, errcode);
-	if (skb) {
-		skb->protocol = htons(ETH_P_DNA_RT);
-		skb->pkt_type = PACKET_OUTGOING;
-	}
-	return skb;
-}
-
-static int dn_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
-{
-	struct sock *sk = sock->sk;
-	struct dn_scp *scp = DN_SK(sk);
-	size_t mss;
-	struct sk_buff_head *queue = &scp->data_xmit_queue;
-	int flags = msg->msg_flags;
-	int err = 0;
-	size_t sent = 0;
-	int addr_len = msg->msg_namelen;
-	DECLARE_SOCKADDR(struct sockaddr_dn *, addr, msg->msg_name);
-	struct sk_buff *skb = NULL;
-	struct dn_skb_cb *cb;
-	size_t len;
-	unsigned char fctype;
-	long timeo;
-
-	if (flags & ~(MSG_TRYHARD|MSG_OOB|MSG_DONTWAIT|MSG_EOR|MSG_NOSIGNAL|MSG_MORE|MSG_CMSG_COMPAT))
-		return -EOPNOTSUPP;
-
-	if (addr_len && (addr_len != sizeof(struct sockaddr_dn)))
-		return -EINVAL;
-
-	lock_sock(sk);
-	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
-	/*
-	 * The only difference between stream sockets and sequenced packet
-	 * sockets is that the stream sockets always behave as if MSG_EOR
-	 * has been set.
-	 */
-	if (sock->type == SOCK_STREAM) {
-		if (flags & MSG_EOR) {
-			err = -EINVAL;
-			goto out;
-		}
-		flags |= MSG_EOR;
-	}
-
-
-	err = dn_check_state(sk, addr, addr_len, &timeo, flags);
-	if (err)
-		goto out_err;
-
-	if (sk->sk_shutdown & SEND_SHUTDOWN) {
-		err = -EPIPE;
-		if (!(flags & MSG_NOSIGNAL))
-			send_sig(SIGPIPE, current, 0);
-		goto out_err;
-	}
-
-	if ((flags & MSG_TRYHARD) && sk->sk_dst_cache)
-		dst_negative_advice(sk);
-
-	mss = scp->segsize_rem;
-	fctype = scp->services_rem & NSP_FC_MASK;
-
-	mss = dn_current_mss(sk, flags);
-
-	if (flags & MSG_OOB) {
-		queue = &scp->other_xmit_queue;
-		if (size > mss) {
-			err = -EMSGSIZE;
-			goto out;
-		}
-	}
-
-	scp->persist_fxn = dn_nsp_xmit_timeout;
-
-	while(sent < size) {
-		err = sock_error(sk);
-		if (err)
-			goto out;
-
-		if (signal_pending(current)) {
-			err = sock_intr_errno(timeo);
-			goto out;
-		}
-
-		/*
-		 * Calculate size that we wish to send.
-		 */
-		len = size - sent;
-
-		if (len > mss)
-			len = mss;
-
-		/*
-		 * Wait for queue size to go down below the window
-		 * size.
-		 */
-		if (dn_queue_too_long(scp, queue, flags)) {
-			DEFINE_WAIT_FUNC(wait, woken_wake_function);
-
-			if (flags & MSG_DONTWAIT) {
-				err = -EWOULDBLOCK;
-				goto out;
-			}
-
-			add_wait_queue(sk_sleep(sk), &wait);
-			sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
-			sk_wait_event(sk, &timeo,
-				      !dn_queue_too_long(scp, queue, flags), &wait);
-			sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
-			remove_wait_queue(sk_sleep(sk), &wait);
-			continue;
-		}
-
-		/*
-		 * Get a suitably sized skb.
-		 * 64 is a bit of a hack really, but its larger than any
-		 * link-layer headers and has served us well as a good
-		 * guess as to their real length.
-		 */
-		skb = dn_alloc_send_pskb(sk, len + 64 + DN_MAX_NSP_DATA_HEADER,
-					 flags & MSG_DONTWAIT, &err);
-
-		if (err)
-			break;
-
-		if (!skb)
-			continue;
-
-		cb = DN_SKB_CB(skb);
-
-		skb_reserve(skb, 64 + DN_MAX_NSP_DATA_HEADER);
-
-		if (memcpy_from_msg(skb_put(skb, len), msg, len)) {
-			err = -EFAULT;
-			goto out;
-		}
-
-		if (flags & MSG_OOB) {
-			cb->nsp_flags = 0x30;
-			if (fctype != NSP_FC_NONE)
-				scp->flowrem_oth--;
-		} else {
-			cb->nsp_flags = 0x00;
-			if (scp->seg_total == 0)
-				cb->nsp_flags |= 0x20;
-
-			scp->seg_total += len;
-
-			if (((sent + len) == size) && (flags & MSG_EOR)) {
-				cb->nsp_flags |= 0x40;
-				scp->seg_total = 0;
-				if (fctype == NSP_FC_SCMC)
-					scp->flowrem_dat--;
-			}
-			if (fctype == NSP_FC_SRC)
-				scp->flowrem_dat--;
-		}
-
-		sent += len;
-		dn_nsp_queue_xmit(sk, skb, sk->sk_allocation, flags & MSG_OOB);
-		skb = NULL;
-
-		scp->persist = dn_nsp_persist(sk);
-
-	}
-out:
-
-	kfree_skb(skb);
-
-	release_sock(sk);
-
-	return sent ? sent : err;
-
-out_err:
-	err = sk_stream_error(sk, flags, err);
-	release_sock(sk);
-	return err;
-}
-
-static int dn_device_event(struct notifier_block *this, unsigned long event,
-			   void *ptr)
-{
-	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
-
-	if (!net_eq(dev_net(dev), &init_net))
-		return NOTIFY_DONE;
-
-	switch (event) {
-	case NETDEV_UP:
-		dn_dev_up(dev);
-		break;
-	case NETDEV_DOWN:
-		dn_dev_down(dev);
-		break;
-	default:
-		break;
-	}
-
-	return NOTIFY_DONE;
-}
-
-static struct notifier_block dn_dev_notifier = {
-	.notifier_call = dn_device_event,
-};
-
-static struct packet_type dn_dix_packet_type __read_mostly = {
-	.type =		cpu_to_be16(ETH_P_DNA_RT),
-	.func =		dn_route_rcv,
-};
-
-#ifdef CONFIG_PROC_FS
-struct dn_iter_state {
-	int bucket;
-};
-
-static struct sock *dn_socket_get_first(struct seq_file *seq)
-{
-	struct dn_iter_state *state = seq->private;
-	struct sock *n = NULL;
-
-	for(state->bucket = 0;
-	    state->bucket < DN_SK_HASH_SIZE;
-	    ++state->bucket) {
-		n = sk_head(&dn_sk_hash[state->bucket]);
-		if (n)
-			break;
-	}
-
-	return n;
-}
-
-static struct sock *dn_socket_get_next(struct seq_file *seq,
-				       struct sock *n)
-{
-	struct dn_iter_state *state = seq->private;
-
-	n = sk_next(n);
-	while (!n) {
-		if (++state->bucket >= DN_SK_HASH_SIZE)
-			break;
-		n = sk_head(&dn_sk_hash[state->bucket]);
-	}
-	return n;
-}
-
-static struct sock *socket_get_idx(struct seq_file *seq, loff_t *pos)
-{
-	struct sock *sk = dn_socket_get_first(seq);
-
-	if (sk) {
-		while(*pos && (sk = dn_socket_get_next(seq, sk)))
-			--*pos;
-	}
-	return *pos ? NULL : sk;
-}
-
-static void *dn_socket_get_idx(struct seq_file *seq, loff_t pos)
-{
-	void *rc;
-	read_lock_bh(&dn_hash_lock);
-	rc = socket_get_idx(seq, &pos);
-	if (!rc) {
-		read_unlock_bh(&dn_hash_lock);
-	}
-	return rc;
-}
-
-static void *dn_socket_seq_start(struct seq_file *seq, loff_t *pos)
-{
-	return *pos ? dn_socket_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
-}
-
-static void *dn_socket_seq_next(struct seq_file *seq, void *v, loff_t *pos)
-{
-	void *rc;
-
-	if (v == SEQ_START_TOKEN) {
-		rc = dn_socket_get_idx(seq, 0);
-		goto out;
-	}
-
-	rc = dn_socket_get_next(seq, v);
-	if (rc)
-		goto out;
-	read_unlock_bh(&dn_hash_lock);
-out:
-	++*pos;
-	return rc;
-}
-
-static void dn_socket_seq_stop(struct seq_file *seq, void *v)
-{
-	if (v && v != SEQ_START_TOKEN)
-		read_unlock_bh(&dn_hash_lock);
-}
-
-#define IS_NOT_PRINTABLE(x) ((x) < 32 || (x) > 126)
-
-static void dn_printable_object(struct sockaddr_dn *dn, unsigned char *buf)
-{
-	int i;
-
-	switch (le16_to_cpu(dn->sdn_objnamel)) {
-	case 0:
-		sprintf(buf, "%d", dn->sdn_objnum);
-		break;
-	default:
-		for (i = 0; i < le16_to_cpu(dn->sdn_objnamel); i++) {
-			buf[i] = dn->sdn_objname[i];
-			if (IS_NOT_PRINTABLE(buf[i]))
-				buf[i] = '.';
-		}
-		buf[i] = 0;
-	}
-}
-
-static char *dn_state2asc(unsigned char state)
-{
-	switch (state) {
-	case DN_O:
-		return "OPEN";
-	case DN_CR:
-		return "  CR";
-	case DN_DR:
-		return "  DR";
-	case DN_DRC:
-		return " DRC";
-	case DN_CC:
-		return "  CC";
-	case DN_CI:
-		return "  CI";
-	case DN_NR:
-		return "  NR";
-	case DN_NC:
-		return "  NC";
-	case DN_CD:
-		return "  CD";
-	case DN_RJ:
-		return "  RJ";
-	case DN_RUN:
-		return " RUN";
-	case DN_DI:
-		return "  DI";
-	case DN_DIC:
-		return " DIC";
-	case DN_DN:
-		return "  DN";
-	case DN_CL:
-		return "  CL";
-	case DN_CN:
-		return "  CN";
-	}
-
-	return "????";
-}
-
-static inline void dn_socket_format_entry(struct seq_file *seq, struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	char buf1[DN_ASCBUF_LEN];
-	char buf2[DN_ASCBUF_LEN];
-	char local_object[DN_MAXOBJL+3];
-	char remote_object[DN_MAXOBJL+3];
-
-	dn_printable_object(&scp->addr, local_object);
-	dn_printable_object(&scp->peer, remote_object);
-
-	seq_printf(seq,
-		   "%6s/%04X %04d:%04d %04d:%04d %01d %-16s "
-		   "%6s/%04X %04d:%04d %04d:%04d %01d %-16s %4s %s\n",
-		   dn_addr2asc(le16_to_cpu(dn_saddr2dn(&scp->addr)), buf1),
-		   scp->addrloc,
-		   scp->numdat,
-		   scp->numoth,
-		   scp->ackxmt_dat,
-		   scp->ackxmt_oth,
-		   scp->flowloc_sw,
-		   local_object,
-		   dn_addr2asc(le16_to_cpu(dn_saddr2dn(&scp->peer)), buf2),
-		   scp->addrrem,
-		   scp->numdat_rcv,
-		   scp->numoth_rcv,
-		   scp->ackrcv_dat,
-		   scp->ackrcv_oth,
-		   scp->flowrem_sw,
-		   remote_object,
-		   dn_state2asc(scp->state),
-		   ((scp->accept_mode == ACC_IMMED) ? "IMMED" : "DEFER"));
-}
-
-static int dn_socket_seq_show(struct seq_file *seq, void *v)
-{
-	if (v == SEQ_START_TOKEN) {
-		seq_puts(seq, "Local                                              Remote\n");
-	} else {
-		dn_socket_format_entry(seq, v);
-	}
-	return 0;
-}
-
-static const struct seq_operations dn_socket_seq_ops = {
-	.start	= dn_socket_seq_start,
-	.next	= dn_socket_seq_next,
-	.stop	= dn_socket_seq_stop,
-	.show	= dn_socket_seq_show,
-};
-#endif
-
-static const struct net_proto_family	dn_family_ops = {
-	.family =	AF_DECnet,
-	.create =	dn_create,
-	.owner	=	THIS_MODULE,
-};
-
-static const struct proto_ops dn_proto_ops = {
-	.family =	AF_DECnet,
-	.owner =	THIS_MODULE,
-	.release =	dn_release,
-	.bind =		dn_bind,
-	.connect =	dn_connect,
-	.socketpair =	sock_no_socketpair,
-	.accept =	dn_accept,
-	.getname =	dn_getname,
-	.poll =		dn_poll,
-	.ioctl =	dn_ioctl,
-	.listen =	dn_listen,
-	.shutdown =	dn_shutdown,
-	.setsockopt =	dn_setsockopt,
-	.getsockopt =	dn_getsockopt,
-	.sendmsg =	dn_sendmsg,
-	.recvmsg =	dn_recvmsg,
-	.mmap =		sock_no_mmap,
-	.sendpage =	sock_no_sendpage,
-};
-
-MODULE_DESCRIPTION("The Linux DECnet Network Protocol");
-MODULE_AUTHOR("Linux DECnet Project Team");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS_NETPROTO(PF_DECnet);
-
-static const char banner[] __initconst = KERN_INFO
-"NET4: DECnet for Linux: V.2.5.68s (C) 1995-2003 Linux DECnet Project Team\n";
-
-static int __init decnet_init(void)
-{
-	int rc;
-
-	printk(banner);
-
-	rc = proto_register(&dn_proto, 1);
-	if (rc != 0)
-		goto out;
-
-	dn_neigh_init();
-	dn_dev_init();
-	dn_route_init();
-	dn_fib_init();
-
-	sock_register(&dn_family_ops);
-	dev_add_pack(&dn_dix_packet_type);
-	register_netdevice_notifier(&dn_dev_notifier);
-
-	proc_create_seq_private("decnet", 0444, init_net.proc_net,
-			&dn_socket_seq_ops, sizeof(struct dn_iter_state),
-			NULL);
-	dn_register_sysctl();
-out:
-	return rc;
-
-}
-module_init(decnet_init);
-
-/*
- * Prevent DECnet module unloading until its fixed properly.
- * Requires an audit of the code to check for memory leaks and
- * initialisation problems etc.
- */
-#if 0
-static void __exit decnet_exit(void)
-{
-	sock_unregister(AF_DECnet);
-	rtnl_unregister_all(PF_DECnet);
-	dev_remove_pack(&dn_dix_packet_type);
-
-	dn_unregister_sysctl();
-
-	unregister_netdevice_notifier(&dn_dev_notifier);
-
-	dn_route_cleanup();
-	dn_dev_cleanup();
-	dn_neigh_cleanup();
-	dn_fib_cleanup();
-
-	remove_proc_entry("decnet", init_net.proc_net);
-
-	proto_unregister(&dn_proto);
-
-	rcu_barrier(); /* Wait for completion of call_rcu()'s */
-}
-module_exit(decnet_exit);
-#endif
diff --git a/net/decnet/dn_dev.c b/net/decnet/dn_dev.c
deleted file mode 100644
index a09ba642b5e7..000000000000
--- a/net/decnet/dn_dev.c
+++ /dev/null
@@ -1,1433 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Device Layer
- *
- * Authors:     Steve Whitehouse <SteveW@ACM.org>
- *              Eduardo Marcelo Serrat <emserrat@geocities.com>
- *
- * Changes:
- *          Steve Whitehouse : Devices now see incoming frames so they
- *                             can mark on who it came from.
- *          Steve Whitehouse : Fixed bug in creating neighbours. Each neighbour
- *                             can now have a device specific setup func.
- *          Steve Whitehouse : Added /proc/sys/net/decnet/conf/<dev>/
- *          Steve Whitehouse : Fixed bug which sometimes killed timer
- *          Steve Whitehouse : Multiple ifaddr support
- *          Steve Whitehouse : SIOCGIFCONF is now a compile time option
- *          Steve Whitehouse : /proc/sys/net/decnet/conf/<sys>/forwarding
- *          Steve Whitehouse : Removed timer1 - it's a user space issue now
- *         Patrick Caulfield : Fixed router hello message format
- *          Steve Whitehouse : Got rid of constant sizes for blksize for
- *                             devices. All mtu based now.
- */
-
-#include <linux/capability.h>
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/init.h>
-#include <linux/net.h>
-#include <linux/netdevice.h>
-#include <linux/proc_fs.h>
-#include <linux/seq_file.h>
-#include <linux/timer.h>
-#include <linux/string.h>
-#include <linux/if_addr.h>
-#include <linux/if_arp.h>
-#include <linux/if_ether.h>
-#include <linux/skbuff.h>
-#include <linux/sysctl.h>
-#include <linux/notifier.h>
-#include <linux/slab.h>
-#include <linux/jiffies.h>
-#include <linux/uaccess.h>
-#include <net/net_namespace.h>
-#include <net/neighbour.h>
-#include <net/dst.h>
-#include <net/flow.h>
-#include <net/fib_rules.h>
-#include <net/netlink.h>
-#include <net/dn.h>
-#include <net/dn_dev.h>
-#include <net/dn_route.h>
-#include <net/dn_neigh.h>
-#include <net/dn_fib.h>
-
-#define DN_IFREQ_SIZE (offsetof(struct ifreq, ifr_ifru) + sizeof(struct sockaddr_dn))
-
-static char dn_rt_all_end_mcast[ETH_ALEN] = {0xAB,0x00,0x00,0x04,0x00,0x00};
-static char dn_rt_all_rt_mcast[ETH_ALEN]  = {0xAB,0x00,0x00,0x03,0x00,0x00};
-static char dn_hiord[ETH_ALEN]            = {0xAA,0x00,0x04,0x00,0x00,0x00};
-static unsigned char dn_eco_version[3]    = {0x02,0x00,0x00};
-
-extern struct neigh_table dn_neigh_table;
-
-/*
- * decnet_address is kept in network order.
- */
-__le16 decnet_address = 0;
-
-static DEFINE_SPINLOCK(dndev_lock);
-static struct net_device *decnet_default_device;
-static BLOCKING_NOTIFIER_HEAD(dnaddr_chain);
-
-static struct dn_dev *dn_dev_create(struct net_device *dev, int *err);
-static void dn_dev_delete(struct net_device *dev);
-static void dn_ifaddr_notify(int event, struct dn_ifaddr *ifa);
-
-static int dn_eth_up(struct net_device *);
-static void dn_eth_down(struct net_device *);
-static void dn_send_brd_hello(struct net_device *dev, struct dn_ifaddr *ifa);
-static void dn_send_ptp_hello(struct net_device *dev, struct dn_ifaddr *ifa);
-
-static struct dn_dev_parms dn_dev_list[] =  {
-{
-	.type =		ARPHRD_ETHER, /* Ethernet */
-	.mode =		DN_DEV_BCAST,
-	.state =	DN_DEV_S_RU,
-	.t2 =		1,
-	.t3 =		10,
-	.name =		"ethernet",
-	.up =		dn_eth_up,
-	.down = 	dn_eth_down,
-	.timer3 =	dn_send_brd_hello,
-},
-{
-	.type =		ARPHRD_IPGRE, /* DECnet tunneled over GRE in IP */
-	.mode =		DN_DEV_BCAST,
-	.state =	DN_DEV_S_RU,
-	.t2 =		1,
-	.t3 =		10,
-	.name =		"ipgre",
-	.timer3 =	dn_send_brd_hello,
-},
-#if 0
-{
-	.type =		ARPHRD_X25, /* Bog standard X.25 */
-	.mode =		DN_DEV_UCAST,
-	.state =	DN_DEV_S_DS,
-	.t2 =		1,
-	.t3 =		120,
-	.name =		"x25",
-	.timer3 =	dn_send_ptp_hello,
-},
-#endif
-#if 0
-{
-	.type =		ARPHRD_PPP, /* DECnet over PPP */
-	.mode =		DN_DEV_BCAST,
-	.state =	DN_DEV_S_RU,
-	.t2 =		1,
-	.t3 =		10,
-	.name =		"ppp",
-	.timer3 =	dn_send_brd_hello,
-},
-#endif
-{
-	.type =		ARPHRD_DDCMP, /* DECnet over DDCMP */
-	.mode =		DN_DEV_UCAST,
-	.state =	DN_DEV_S_DS,
-	.t2 =		1,
-	.t3 =		120,
-	.name =		"ddcmp",
-	.timer3 =	dn_send_ptp_hello,
-},
-{
-	.type =		ARPHRD_LOOPBACK, /* Loopback interface - always last */
-	.mode =		DN_DEV_BCAST,
-	.state =	DN_DEV_S_RU,
-	.t2 =		1,
-	.t3 =		10,
-	.name =		"loopback",
-	.timer3 =	dn_send_brd_hello,
-}
-};
-
-#define DN_DEV_LIST_SIZE ARRAY_SIZE(dn_dev_list)
-
-#define DN_DEV_PARMS_OFFSET(x) offsetof(struct dn_dev_parms, x)
-
-#ifdef CONFIG_SYSCTL
-
-static int min_t2[] = { 1 };
-static int max_t2[] = { 60 }; /* No max specified, but this seems sensible */
-static int min_t3[] = { 1 };
-static int max_t3[] = { 8191 }; /* Must fit in 16 bits when multiplied by BCT3MULT or T3MULT */
-
-static int min_priority[1];
-static int max_priority[] = { 127 }; /* From DECnet spec */
-
-static int dn_forwarding_proc(struct ctl_table *, int, void *, size_t *,
-		loff_t *);
-static struct dn_dev_sysctl_table {
-	struct ctl_table_header *sysctl_header;
-	struct ctl_table dn_dev_vars[5];
-} dn_dev_sysctl = {
-	NULL,
-	{
-	{
-		.procname = "forwarding",
-		.data = (void *)DN_DEV_PARMS_OFFSET(forwarding),
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = dn_forwarding_proc,
-	},
-	{
-		.procname = "priority",
-		.data = (void *)DN_DEV_PARMS_OFFSET(priority),
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = proc_dointvec_minmax,
-		.extra1 = &min_priority,
-		.extra2 = &max_priority
-	},
-	{
-		.procname = "t2",
-		.data = (void *)DN_DEV_PARMS_OFFSET(t2),
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = proc_dointvec_minmax,
-		.extra1 = &min_t2,
-		.extra2 = &max_t2
-	},
-	{
-		.procname = "t3",
-		.data = (void *)DN_DEV_PARMS_OFFSET(t3),
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = proc_dointvec_minmax,
-		.extra1 = &min_t3,
-		.extra2 = &max_t3
-	},
-	{ }
-	},
-};
-
-static void dn_dev_sysctl_register(struct net_device *dev, struct dn_dev_parms *parms)
-{
-	struct dn_dev_sysctl_table *t;
-	int i;
-
-	char path[sizeof("net/decnet/conf/") + IFNAMSIZ];
-
-	t = kmemdup(&dn_dev_sysctl, sizeof(*t), GFP_KERNEL);
-	if (t == NULL)
-		return;
-
-	for(i = 0; i < ARRAY_SIZE(t->dn_dev_vars) - 1; i++) {
-		long offset = (long)t->dn_dev_vars[i].data;
-		t->dn_dev_vars[i].data = ((char *)parms) + offset;
-	}
-
-	snprintf(path, sizeof(path), "net/decnet/conf/%s",
-		dev? dev->name : parms->name);
-
-	t->dn_dev_vars[0].extra1 = (void *)dev;
-
-	t->sysctl_header = register_net_sysctl(&init_net, path, t->dn_dev_vars);
-	if (t->sysctl_header == NULL)
-		kfree(t);
-	else
-		parms->sysctl = t;
-}
-
-static void dn_dev_sysctl_unregister(struct dn_dev_parms *parms)
-{
-	if (parms->sysctl) {
-		struct dn_dev_sysctl_table *t = parms->sysctl;
-		parms->sysctl = NULL;
-		unregister_net_sysctl_table(t->sysctl_header);
-		kfree(t);
-	}
-}
-
-static int dn_forwarding_proc(struct ctl_table *table, int write,
-		void *buffer, size_t *lenp, loff_t *ppos)
-{
-#ifdef CONFIG_DECNET_ROUTER
-	struct net_device *dev = table->extra1;
-	struct dn_dev *dn_db;
-	int err;
-	int tmp, old;
-
-	if (table->extra1 == NULL)
-		return -EINVAL;
-
-	dn_db = rcu_dereference_raw(dev->dn_ptr);
-	old = dn_db->parms.forwarding;
-
-	err = proc_dointvec(table, write, buffer, lenp, ppos);
-
-	if ((err >= 0) && write) {
-		if (dn_db->parms.forwarding < 0)
-			dn_db->parms.forwarding = 0;
-		if (dn_db->parms.forwarding > 2)
-			dn_db->parms.forwarding = 2;
-		/*
-		 * What an ugly hack this is... its works, just. It
-		 * would be nice if sysctl/proc were just that little
-		 * bit more flexible so I don't have to write a special
-		 * routine, or suffer hacks like this - SJW
-		 */
-		tmp = dn_db->parms.forwarding;
-		dn_db->parms.forwarding = old;
-		if (dn_db->parms.down)
-			dn_db->parms.down(dev);
-		dn_db->parms.forwarding = tmp;
-		if (dn_db->parms.up)
-			dn_db->parms.up(dev);
-	}
-
-	return err;
-#else
-	return -EINVAL;
-#endif
-}
-
-#else /* CONFIG_SYSCTL */
-static void dn_dev_sysctl_unregister(struct dn_dev_parms *parms)
-{
-}
-static void dn_dev_sysctl_register(struct net_device *dev, struct dn_dev_parms *parms)
-{
-}
-
-#endif /* CONFIG_SYSCTL */
-
-static inline __u16 mtu2blksize(struct net_device *dev)
-{
-	u32 blksize = dev->mtu;
-	if (blksize > 0xffff)
-		blksize = 0xffff;
-
-	if (dev->type == ARPHRD_ETHER ||
-	    dev->type == ARPHRD_PPP ||
-	    dev->type == ARPHRD_IPGRE ||
-	    dev->type == ARPHRD_LOOPBACK)
-		blksize -= 2;
-
-	return (__u16)blksize;
-}
-
-static struct dn_ifaddr *dn_dev_alloc_ifa(void)
-{
-	struct dn_ifaddr *ifa;
-
-	ifa = kzalloc(sizeof(*ifa), GFP_KERNEL);
-
-	return ifa;
-}
-
-static void dn_dev_free_ifa(struct dn_ifaddr *ifa)
-{
-	kfree_rcu(ifa, rcu);
-}
-
-static void dn_dev_del_ifa(struct dn_dev *dn_db, struct dn_ifaddr __rcu **ifap, int destroy)
-{
-	struct dn_ifaddr *ifa1 = rtnl_dereference(*ifap);
-	unsigned char mac_addr[6];
-	struct net_device *dev = dn_db->dev;
-
-	ASSERT_RTNL();
-
-	*ifap = ifa1->ifa_next;
-
-	if (dn_db->dev->type == ARPHRD_ETHER) {
-		if (ifa1->ifa_local != dn_eth2dn(dev->dev_addr)) {
-			dn_dn2eth(mac_addr, ifa1->ifa_local);
-			dev_mc_del(dev, mac_addr);
-		}
-	}
-
-	dn_ifaddr_notify(RTM_DELADDR, ifa1);
-	blocking_notifier_call_chain(&dnaddr_chain, NETDEV_DOWN, ifa1);
-	if (destroy) {
-		dn_dev_free_ifa(ifa1);
-
-		if (dn_db->ifa_list == NULL)
-			dn_dev_delete(dn_db->dev);
-	}
-}
-
-static int dn_dev_insert_ifa(struct dn_dev *dn_db, struct dn_ifaddr *ifa)
-{
-	struct net_device *dev = dn_db->dev;
-	struct dn_ifaddr *ifa1;
-	unsigned char mac_addr[6];
-
-	ASSERT_RTNL();
-
-	/* Check for duplicates */
-	for (ifa1 = rtnl_dereference(dn_db->ifa_list);
-	     ifa1 != NULL;
-	     ifa1 = rtnl_dereference(ifa1->ifa_next)) {
-		if (ifa1->ifa_local == ifa->ifa_local)
-			return -EEXIST;
-	}
-
-	if (dev->type == ARPHRD_ETHER) {
-		if (ifa->ifa_local != dn_eth2dn(dev->dev_addr)) {
-			dn_dn2eth(mac_addr, ifa->ifa_local);
-			dev_mc_add(dev, mac_addr);
-		}
-	}
-
-	ifa->ifa_next = dn_db->ifa_list;
-	rcu_assign_pointer(dn_db->ifa_list, ifa);
-
-	dn_ifaddr_notify(RTM_NEWADDR, ifa);
-	blocking_notifier_call_chain(&dnaddr_chain, NETDEV_UP, ifa);
-
-	return 0;
-}
-
-static int dn_dev_set_ifa(struct net_device *dev, struct dn_ifaddr *ifa)
-{
-	struct dn_dev *dn_db = rtnl_dereference(dev->dn_ptr);
-	int rv;
-
-	if (dn_db == NULL) {
-		int err;
-		dn_db = dn_dev_create(dev, &err);
-		if (dn_db == NULL)
-			return err;
-	}
-
-	ifa->ifa_dev = dn_db;
-
-	if (dev->flags & IFF_LOOPBACK)
-		ifa->ifa_scope = RT_SCOPE_HOST;
-
-	rv = dn_dev_insert_ifa(dn_db, ifa);
-	if (rv)
-		dn_dev_free_ifa(ifa);
-	return rv;
-}
-
-
-int dn_dev_ioctl(unsigned int cmd, void __user *arg)
-{
-	char buffer[DN_IFREQ_SIZE];
-	struct ifreq *ifr = (struct ifreq *)buffer;
-	struct sockaddr_dn *sdn = (struct sockaddr_dn *)&ifr->ifr_addr;
-	struct dn_dev *dn_db;
-	struct net_device *dev;
-	struct dn_ifaddr *ifa = NULL;
-	struct dn_ifaddr __rcu **ifap = NULL;
-	int ret = 0;
-
-	if (copy_from_user(ifr, arg, DN_IFREQ_SIZE))
-		return -EFAULT;
-	ifr->ifr_name[IFNAMSIZ-1] = 0;
-
-	dev_load(&init_net, ifr->ifr_name);
-
-	switch (cmd) {
-	case SIOCGIFADDR:
-		break;
-	case SIOCSIFADDR:
-		if (!capable(CAP_NET_ADMIN))
-			return -EACCES;
-		if (sdn->sdn_family != AF_DECnet)
-			return -EINVAL;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	rtnl_lock();
-
-	if ((dev = __dev_get_by_name(&init_net, ifr->ifr_name)) == NULL) {
-		ret = -ENODEV;
-		goto done;
-	}
-
-	if ((dn_db = rtnl_dereference(dev->dn_ptr)) != NULL) {
-		for (ifap = &dn_db->ifa_list;
-		     (ifa = rtnl_dereference(*ifap)) != NULL;
-		     ifap = &ifa->ifa_next)
-			if (strcmp(ifr->ifr_name, ifa->ifa_label) == 0)
-				break;
-	}
-
-	if (ifa == NULL && cmd != SIOCSIFADDR) {
-		ret = -EADDRNOTAVAIL;
-		goto done;
-	}
-
-	switch (cmd) {
-	case SIOCGIFADDR:
-		*((__le16 *)sdn->sdn_nodeaddr) = ifa->ifa_local;
-		if (copy_to_user(arg, ifr, DN_IFREQ_SIZE))
-			ret = -EFAULT;
-		break;
-
-	case SIOCSIFADDR:
-		if (!ifa) {
-			if ((ifa = dn_dev_alloc_ifa()) == NULL) {
-				ret = -ENOBUFS;
-				break;
-			}
-			memcpy(ifa->ifa_label, dev->name, IFNAMSIZ);
-		} else {
-			if (ifa->ifa_local == dn_saddr2dn(sdn))
-				break;
-			dn_dev_del_ifa(dn_db, ifap, 0);
-		}
-
-		ifa->ifa_local = ifa->ifa_address = dn_saddr2dn(sdn);
-
-		ret = dn_dev_set_ifa(dev, ifa);
-	}
-done:
-	rtnl_unlock();
-
-	return ret;
-}
-
-struct net_device *dn_dev_get_default(void)
-{
-	struct net_device *dev;
-
-	spin_lock(&dndev_lock);
-	dev = decnet_default_device;
-	if (dev) {
-		if (dev->dn_ptr)
-			dev_hold(dev);
-		else
-			dev = NULL;
-	}
-	spin_unlock(&dndev_lock);
-
-	return dev;
-}
-
-int dn_dev_set_default(struct net_device *dev, int force)
-{
-	struct net_device *old = NULL;
-	int rv = -EBUSY;
-	if (!dev->dn_ptr)
-		return -ENODEV;
-
-	spin_lock(&dndev_lock);
-	if (force || decnet_default_device == NULL) {
-		old = decnet_default_device;
-		decnet_default_device = dev;
-		rv = 0;
-	}
-	spin_unlock(&dndev_lock);
-
-	dev_put(old);
-	return rv;
-}
-
-static void dn_dev_check_default(struct net_device *dev)
-{
-	spin_lock(&dndev_lock);
-	if (dev == decnet_default_device) {
-		decnet_default_device = NULL;
-	} else {
-		dev = NULL;
-	}
-	spin_unlock(&dndev_lock);
-
-	dev_put(dev);
-}
-
-/*
- * Called with RTNL
- */
-static struct dn_dev *dn_dev_by_index(int ifindex)
-{
-	struct net_device *dev;
-	struct dn_dev *dn_dev = NULL;
-
-	dev = __dev_get_by_index(&init_net, ifindex);
-	if (dev)
-		dn_dev = rtnl_dereference(dev->dn_ptr);
-
-	return dn_dev;
-}
-
-static const struct nla_policy dn_ifa_policy[IFA_MAX+1] = {
-	[IFA_ADDRESS]		= { .type = NLA_U16 },
-	[IFA_LOCAL]		= { .type = NLA_U16 },
-	[IFA_LABEL]		= { .type = NLA_STRING,
-				    .len = IFNAMSIZ - 1 },
-	[IFA_FLAGS]		= { .type = NLA_U32 },
-};
-
-static int dn_nl_deladdr(struct sk_buff *skb, struct nlmsghdr *nlh,
-			 struct netlink_ext_ack *extack)
-{
-	struct net *net = sock_net(skb->sk);
-	struct nlattr *tb[IFA_MAX+1];
-	struct dn_dev *dn_db;
-	struct ifaddrmsg *ifm;
-	struct dn_ifaddr *ifa;
-	struct dn_ifaddr __rcu **ifap;
-	int err = -EINVAL;
-
-	if (!netlink_capable(skb, CAP_NET_ADMIN))
-		return -EPERM;
-
-	if (!net_eq(net, &init_net))
-		goto errout;
-
-	err = nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFA_MAX,
-				     dn_ifa_policy, extack);
-	if (err < 0)
-		goto errout;
-
-	err = -ENODEV;
-	ifm = nlmsg_data(nlh);
-	if ((dn_db = dn_dev_by_index(ifm->ifa_index)) == NULL)
-		goto errout;
-
-	err = -EADDRNOTAVAIL;
-	for (ifap = &dn_db->ifa_list;
-	     (ifa = rtnl_dereference(*ifap)) != NULL;
-	     ifap = &ifa->ifa_next) {
-		if (tb[IFA_LOCAL] &&
-		    nla_memcmp(tb[IFA_LOCAL], &ifa->ifa_local, 2))
-			continue;
-
-		if (tb[IFA_LABEL] && nla_strcmp(tb[IFA_LABEL], ifa->ifa_label))
-			continue;
-
-		dn_dev_del_ifa(dn_db, ifap, 1);
-		return 0;
-	}
-
-errout:
-	return err;
-}
-
-static int dn_nl_newaddr(struct sk_buff *skb, struct nlmsghdr *nlh,
-			 struct netlink_ext_ack *extack)
-{
-	struct net *net = sock_net(skb->sk);
-	struct nlattr *tb[IFA_MAX+1];
-	struct net_device *dev;
-	struct dn_dev *dn_db;
-	struct ifaddrmsg *ifm;
-	struct dn_ifaddr *ifa;
-	int err;
-
-	if (!netlink_capable(skb, CAP_NET_ADMIN))
-		return -EPERM;
-
-	if (!net_eq(net, &init_net))
-		return -EINVAL;
-
-	err = nlmsg_parse_deprecated(nlh, sizeof(*ifm), tb, IFA_MAX,
-				     dn_ifa_policy, extack);
-	if (err < 0)
-		return err;
-
-	if (tb[IFA_LOCAL] == NULL)
-		return -EINVAL;
-
-	ifm = nlmsg_data(nlh);
-	if ((dev = __dev_get_by_index(&init_net, ifm->ifa_index)) == NULL)
-		return -ENODEV;
-
-	if ((dn_db = rtnl_dereference(dev->dn_ptr)) == NULL) {
-		dn_db = dn_dev_create(dev, &err);
-		if (!dn_db)
-			return err;
-	}
-
-	if ((ifa = dn_dev_alloc_ifa()) == NULL)
-		return -ENOBUFS;
-
-	if (tb[IFA_ADDRESS] == NULL)
-		tb[IFA_ADDRESS] = tb[IFA_LOCAL];
-
-	ifa->ifa_local = nla_get_le16(tb[IFA_LOCAL]);
-	ifa->ifa_address = nla_get_le16(tb[IFA_ADDRESS]);
-	ifa->ifa_flags = tb[IFA_FLAGS] ? nla_get_u32(tb[IFA_FLAGS]) :
-					 ifm->ifa_flags;
-	ifa->ifa_scope = ifm->ifa_scope;
-	ifa->ifa_dev = dn_db;
-
-	if (tb[IFA_LABEL])
-		nla_strscpy(ifa->ifa_label, tb[IFA_LABEL], IFNAMSIZ);
-	else
-		memcpy(ifa->ifa_label, dev->name, IFNAMSIZ);
-
-	err = dn_dev_insert_ifa(dn_db, ifa);
-	if (err)
-		dn_dev_free_ifa(ifa);
-
-	return err;
-}
-
-static inline size_t dn_ifaddr_nlmsg_size(void)
-{
-	return NLMSG_ALIGN(sizeof(struct ifaddrmsg))
-	       + nla_total_size(IFNAMSIZ) /* IFA_LABEL */
-	       + nla_total_size(2) /* IFA_ADDRESS */
-	       + nla_total_size(2) /* IFA_LOCAL */
-	       + nla_total_size(4); /* IFA_FLAGS */
-}
-
-static int dn_nl_fill_ifaddr(struct sk_buff *skb, struct dn_ifaddr *ifa,
-			     u32 portid, u32 seq, int event, unsigned int flags)
-{
-	struct ifaddrmsg *ifm;
-	struct nlmsghdr *nlh;
-	u32 ifa_flags = ifa->ifa_flags | IFA_F_PERMANENT;
-
-	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*ifm), flags);
-	if (nlh == NULL)
-		return -EMSGSIZE;
-
-	ifm = nlmsg_data(nlh);
-	ifm->ifa_family = AF_DECnet;
-	ifm->ifa_prefixlen = 16;
-	ifm->ifa_flags = ifa_flags;
-	ifm->ifa_scope = ifa->ifa_scope;
-	ifm->ifa_index = ifa->ifa_dev->dev->ifindex;
-
-	if ((ifa->ifa_address &&
-	     nla_put_le16(skb, IFA_ADDRESS, ifa->ifa_address)) ||
-	    (ifa->ifa_local &&
-	     nla_put_le16(skb, IFA_LOCAL, ifa->ifa_local)) ||
-	    (ifa->ifa_label[0] &&
-	     nla_put_string(skb, IFA_LABEL, ifa->ifa_label)) ||
-	     nla_put_u32(skb, IFA_FLAGS, ifa_flags))
-		goto nla_put_failure;
-	nlmsg_end(skb, nlh);
-	return 0;
-
-nla_put_failure:
-	nlmsg_cancel(skb, nlh);
-	return -EMSGSIZE;
-}
-
-static void dn_ifaddr_notify(int event, struct dn_ifaddr *ifa)
-{
-	struct sk_buff *skb;
-	int err = -ENOBUFS;
-
-	skb = alloc_skb(dn_ifaddr_nlmsg_size(), GFP_KERNEL);
-	if (skb == NULL)
-		goto errout;
-
-	err = dn_nl_fill_ifaddr(skb, ifa, 0, 0, event, 0);
-	if (err < 0) {
-		/* -EMSGSIZE implies BUG in dn_ifaddr_nlmsg_size() */
-		WARN_ON(err == -EMSGSIZE);
-		kfree_skb(skb);
-		goto errout;
-	}
-	rtnl_notify(skb, &init_net, 0, RTNLGRP_DECnet_IFADDR, NULL, GFP_KERNEL);
-	return;
-errout:
-	if (err < 0)
-		rtnl_set_sk_err(&init_net, RTNLGRP_DECnet_IFADDR, err);
-}
-
-static int dn_nl_dump_ifaddr(struct sk_buff *skb, struct netlink_callback *cb)
-{
-	struct net *net = sock_net(skb->sk);
-	int idx, dn_idx = 0, skip_ndevs, skip_naddr;
-	struct net_device *dev;
-	struct dn_dev *dn_db;
-	struct dn_ifaddr *ifa;
-
-	if (!net_eq(net, &init_net))
-		return 0;
-
-	skip_ndevs = cb->args[0];
-	skip_naddr = cb->args[1];
-
-	idx = 0;
-	rcu_read_lock();
-	for_each_netdev_rcu(&init_net, dev) {
-		if (idx < skip_ndevs)
-			goto cont;
-		else if (idx > skip_ndevs) {
-			/* Only skip over addresses for first dev dumped
-			 * in this iteration (idx == skip_ndevs) */
-			skip_naddr = 0;
-		}
-
-		if ((dn_db = rcu_dereference(dev->dn_ptr)) == NULL)
-			goto cont;
-
-		for (ifa = rcu_dereference(dn_db->ifa_list), dn_idx = 0; ifa;
-		     ifa = rcu_dereference(ifa->ifa_next), dn_idx++) {
-			if (dn_idx < skip_naddr)
-				continue;
-
-			if (dn_nl_fill_ifaddr(skb, ifa, NETLINK_CB(cb->skb).portid,
-					      cb->nlh->nlmsg_seq, RTM_NEWADDR,
-					      NLM_F_MULTI) < 0)
-				goto done;
-		}
-cont:
-		idx++;
-	}
-done:
-	rcu_read_unlock();
-	cb->args[0] = idx;
-	cb->args[1] = dn_idx;
-
-	return skb->len;
-}
-
-static int dn_dev_get_first(struct net_device *dev, __le16 *addr)
-{
-	struct dn_dev *dn_db;
-	struct dn_ifaddr *ifa;
-	int rv = -ENODEV;
-
-	rcu_read_lock();
-	dn_db = rcu_dereference(dev->dn_ptr);
-	if (dn_db == NULL)
-		goto out;
-
-	ifa = rcu_dereference(dn_db->ifa_list);
-	if (ifa != NULL) {
-		*addr = ifa->ifa_local;
-		rv = 0;
-	}
-out:
-	rcu_read_unlock();
-	return rv;
-}
-
-/*
- * Find a default address to bind to.
- *
- * This is one of those areas where the initial VMS concepts don't really
- * map onto the Linux concepts, and since we introduced multiple addresses
- * per interface we have to cope with slightly odd ways of finding out what
- * "our address" really is. Mostly it's not a problem; for this we just guess
- * a sensible default. Eventually the routing code will take care of all the
- * nasties for us I hope.
- */
-int dn_dev_bind_default(__le16 *addr)
-{
-	struct net_device *dev;
-	int rv;
-	dev = dn_dev_get_default();
-last_chance:
-	if (dev) {
-		rv = dn_dev_get_first(dev, addr);
-		dev_put(dev);
-		if (rv == 0 || dev == init_net.loopback_dev)
-			return rv;
-	}
-	dev = init_net.loopback_dev;
-	dev_hold(dev);
-	goto last_chance;
-}
-
-static void dn_send_endnode_hello(struct net_device *dev, struct dn_ifaddr *ifa)
-{
-	struct endnode_hello_message *msg;
-	struct sk_buff *skb = NULL;
-	__le16 *pktlen;
-	struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr);
-
-	if ((skb = dn_alloc_skb(NULL, sizeof(*msg), GFP_ATOMIC)) == NULL)
-		return;
-
-	skb->dev = dev;
-
-	msg = skb_put(skb, sizeof(*msg));
-
-	msg->msgflg  = 0x0D;
-	memcpy(msg->tiver, dn_eco_version, 3);
-	dn_dn2eth(msg->id, ifa->ifa_local);
-	msg->iinfo   = DN_RT_INFO_ENDN;
-	msg->blksize = cpu_to_le16(mtu2blksize(dev));
-	msg->area    = 0x00;
-	memset(msg->seed, 0, 8);
-	memcpy(msg->neighbor, dn_hiord, ETH_ALEN);
-
-	if (dn_db->router) {
-		struct dn_neigh *dn = container_of(dn_db->router, struct dn_neigh, n);
-		dn_dn2eth(msg->neighbor, dn->addr);
-	}
-
-	msg->timer   = cpu_to_le16((unsigned short)dn_db->parms.t3);
-	msg->mpd     = 0x00;
-	msg->datalen = 0x02;
-	memset(msg->data, 0xAA, 2);
-
-	pktlen = skb_push(skb, 2);
-	*pktlen = cpu_to_le16(skb->len - 2);
-
-	skb_reset_network_header(skb);
-
-	dn_rt_finish_output(skb, dn_rt_all_rt_mcast, msg->id);
-}
-
-
-#define DRDELAY (5 * HZ)
-
-static int dn_am_i_a_router(struct dn_neigh *dn, struct dn_dev *dn_db, struct dn_ifaddr *ifa)
-{
-	/* First check time since device went up */
-	if (time_before(jiffies, dn_db->uptime + DRDELAY))
-		return 0;
-
-	/* If there is no router, then yes... */
-	if (!dn_db->router)
-		return 1;
-
-	/* otherwise only if we have a higher priority or.. */
-	if (dn->priority < dn_db->parms.priority)
-		return 1;
-
-	/* if we have equal priority and a higher node number */
-	if (dn->priority != dn_db->parms.priority)
-		return 0;
-
-	if (le16_to_cpu(dn->addr) < le16_to_cpu(ifa->ifa_local))
-		return 1;
-
-	return 0;
-}
-
-static void dn_send_router_hello(struct net_device *dev, struct dn_ifaddr *ifa)
-{
-	int n;
-	struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr);
-	struct dn_neigh *dn = container_of(dn_db->router, struct dn_neigh, n);
-	struct sk_buff *skb;
-	size_t size;
-	unsigned char *ptr;
-	unsigned char *i1, *i2;
-	__le16 *pktlen;
-	char *src;
-
-	if (mtu2blksize(dev) < (26 + 7))
-		return;
-
-	n = mtu2blksize(dev) - 26;
-	n /= 7;
-
-	if (n > 32)
-		n = 32;
-
-	size = 2 + 26 + 7 * n;
-
-	if ((skb = dn_alloc_skb(NULL, size, GFP_ATOMIC)) == NULL)
-		return;
-
-	skb->dev = dev;
-	ptr = skb_put(skb, size);
-
-	*ptr++ = DN_RT_PKT_CNTL | DN_RT_PKT_ERTH;
-	*ptr++ = 2; /* ECO */
-	*ptr++ = 0;
-	*ptr++ = 0;
-	dn_dn2eth(ptr, ifa->ifa_local);
-	src = ptr;
-	ptr += ETH_ALEN;
-	*ptr++ = dn_db->parms.forwarding == 1 ?
-			DN_RT_INFO_L1RT : DN_RT_INFO_L2RT;
-	*((__le16 *)ptr) = cpu_to_le16(mtu2blksize(dev));
-	ptr += 2;
-	*ptr++ = dn_db->parms.priority; /* Priority */
-	*ptr++ = 0; /* Area: Reserved */
-	*((__le16 *)ptr) = cpu_to_le16((unsigned short)dn_db->parms.t3);
-	ptr += 2;
-	*ptr++ = 0; /* MPD: Reserved */
-	i1 = ptr++;
-	memset(ptr, 0, 7); /* Name: Reserved */
-	ptr += 7;
-	i2 = ptr++;
-
-	n = dn_neigh_elist(dev, ptr, n);
-
-	*i2 = 7 * n;
-	*i1 = 8 + *i2;
-
-	skb_trim(skb, (27 + *i2));
-
-	pktlen = skb_push(skb, 2);
-	*pktlen = cpu_to_le16(skb->len - 2);
-
-	skb_reset_network_header(skb);
-
-	if (dn_am_i_a_router(dn, dn_db, ifa)) {
-		struct sk_buff *skb2 = skb_copy(skb, GFP_ATOMIC);
-		if (skb2) {
-			dn_rt_finish_output(skb2, dn_rt_all_end_mcast, src);
-		}
-	}
-
-	dn_rt_finish_output(skb, dn_rt_all_rt_mcast, src);
-}
-
-static void dn_send_brd_hello(struct net_device *dev, struct dn_ifaddr *ifa)
-{
-	struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr);
-
-	if (dn_db->parms.forwarding == 0)
-		dn_send_endnode_hello(dev, ifa);
-	else
-		dn_send_router_hello(dev, ifa);
-}
-
-static void dn_send_ptp_hello(struct net_device *dev, struct dn_ifaddr *ifa)
-{
-	int tdlen = 16;
-	int size = dev->hard_header_len + 2 + 4 + tdlen;
-	struct sk_buff *skb = dn_alloc_skb(NULL, size, GFP_ATOMIC);
-	int i;
-	unsigned char *ptr;
-	char src[ETH_ALEN];
-
-	if (skb == NULL)
-		return ;
-
-	skb->dev = dev;
-	skb_push(skb, dev->hard_header_len);
-	ptr = skb_put(skb, 2 + 4 + tdlen);
-
-	*ptr++ = DN_RT_PKT_HELO;
-	*((__le16 *)ptr) = ifa->ifa_local;
-	ptr += 2;
-	*ptr++ = tdlen;
-
-	for(i = 0; i < tdlen; i++)
-		*ptr++ = 0252;
-
-	dn_dn2eth(src, ifa->ifa_local);
-	dn_rt_finish_output(skb, dn_rt_all_rt_mcast, src);
-}
-
-static int dn_eth_up(struct net_device *dev)
-{
-	struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr);
-
-	if (dn_db->parms.forwarding == 0)
-		dev_mc_add(dev, dn_rt_all_end_mcast);
-	else
-		dev_mc_add(dev, dn_rt_all_rt_mcast);
-
-	dn_db->use_long = 1;
-
-	return 0;
-}
-
-static void dn_eth_down(struct net_device *dev)
-{
-	struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr);
-
-	if (dn_db->parms.forwarding == 0)
-		dev_mc_del(dev, dn_rt_all_end_mcast);
-	else
-		dev_mc_del(dev, dn_rt_all_rt_mcast);
-}
-
-static void dn_dev_set_timer(struct net_device *dev);
-
-static void dn_dev_timer_func(struct timer_list *t)
-{
-	struct dn_dev *dn_db = from_timer(dn_db, t, timer);
-	struct net_device *dev;
-	struct dn_ifaddr *ifa;
-
-	rcu_read_lock();
-	dev = dn_db->dev;
-	if (dn_db->t3 <= dn_db->parms.t2) {
-		if (dn_db->parms.timer3) {
-			for (ifa = rcu_dereference(dn_db->ifa_list);
-			     ifa;
-			     ifa = rcu_dereference(ifa->ifa_next)) {
-				if (!(ifa->ifa_flags & IFA_F_SECONDARY))
-					dn_db->parms.timer3(dev, ifa);
-			}
-		}
-		dn_db->t3 = dn_db->parms.t3;
-	} else {
-		dn_db->t3 -= dn_db->parms.t2;
-	}
-	rcu_read_unlock();
-	dn_dev_set_timer(dev);
-}
-
-static void dn_dev_set_timer(struct net_device *dev)
-{
-	struct dn_dev *dn_db = rcu_dereference_raw(dev->dn_ptr);
-
-	if (dn_db->parms.t2 > dn_db->parms.t3)
-		dn_db->parms.t2 = dn_db->parms.t3;
-
-	dn_db->timer.expires = jiffies + (dn_db->parms.t2 * HZ);
-
-	add_timer(&dn_db->timer);
-}
-
-static struct dn_dev *dn_dev_create(struct net_device *dev, int *err)
-{
-	int i;
-	struct dn_dev_parms *p = dn_dev_list;
-	struct dn_dev *dn_db;
-
-	for(i = 0; i < DN_DEV_LIST_SIZE; i++, p++) {
-		if (p->type == dev->type)
-			break;
-	}
-
-	*err = -ENODEV;
-	if (i == DN_DEV_LIST_SIZE)
-		return NULL;
-
-	*err = -ENOBUFS;
-	if ((dn_db = kzalloc(sizeof(struct dn_dev), GFP_ATOMIC)) == NULL)
-		return NULL;
-
-	memcpy(&dn_db->parms, p, sizeof(struct dn_dev_parms));
-
-	rcu_assign_pointer(dev->dn_ptr, dn_db);
-	dn_db->dev = dev;
-	timer_setup(&dn_db->timer, dn_dev_timer_func, 0);
-
-	dn_db->uptime = jiffies;
-
-	dn_db->neigh_parms = neigh_parms_alloc(dev, &dn_neigh_table);
-	if (!dn_db->neigh_parms) {
-		RCU_INIT_POINTER(dev->dn_ptr, NULL);
-		kfree(dn_db);
-		return NULL;
-	}
-
-	if (dn_db->parms.up) {
-		if (dn_db->parms.up(dev) < 0) {
-			neigh_parms_release(&dn_neigh_table, dn_db->neigh_parms);
-			dev->dn_ptr = NULL;
-			kfree(dn_db);
-			return NULL;
-		}
-	}
-
-	dn_dev_sysctl_register(dev, &dn_db->parms);
-
-	dn_dev_set_timer(dev);
-
-	*err = 0;
-	return dn_db;
-}
-
-
-/*
- * This processes a device up event. We only start up
- * the loopback device & ethernet devices with correct
- * MAC addresses automatically. Others must be started
- * specifically.
- *
- * FIXME: How should we configure the loopback address ? If we could dispense
- * with using decnet_address here and for autobind, it will be one less thing
- * for users to worry about setting up.
- */
-
-void dn_dev_up(struct net_device *dev)
-{
-	struct dn_ifaddr *ifa;
-	__le16 addr = decnet_address;
-	int maybe_default = 0;
-	struct dn_dev *dn_db = rtnl_dereference(dev->dn_ptr);
-
-	if ((dev->type != ARPHRD_ETHER) && (dev->type != ARPHRD_LOOPBACK))
-		return;
-
-	/*
-	 * Need to ensure that loopback device has a dn_db attached to it
-	 * to allow creation of neighbours against it, even though it might
-	 * not have a local address of its own. Might as well do the same for
-	 * all autoconfigured interfaces.
-	 */
-	if (dn_db == NULL) {
-		int err;
-		dn_db = dn_dev_create(dev, &err);
-		if (dn_db == NULL)
-			return;
-	}
-
-	if (dev->type == ARPHRD_ETHER) {
-		if (memcmp(dev->dev_addr, dn_hiord, 4) != 0)
-			return;
-		addr = dn_eth2dn(dev->dev_addr);
-		maybe_default = 1;
-	}
-
-	if (addr == 0)
-		return;
-
-	if ((ifa = dn_dev_alloc_ifa()) == NULL)
-		return;
-
-	ifa->ifa_local = ifa->ifa_address = addr;
-	ifa->ifa_flags = 0;
-	ifa->ifa_scope = RT_SCOPE_UNIVERSE;
-	strcpy(ifa->ifa_label, dev->name);
-
-	dn_dev_set_ifa(dev, ifa);
-
-	/*
-	 * Automagically set the default device to the first automatically
-	 * configured ethernet card in the system.
-	 */
-	if (maybe_default) {
-		dev_hold(dev);
-		if (dn_dev_set_default(dev, 0))
-			dev_put(dev);
-	}
-}
-
-static void dn_dev_delete(struct net_device *dev)
-{
-	struct dn_dev *dn_db = rtnl_dereference(dev->dn_ptr);
-
-	if (dn_db == NULL)
-		return;
-
-	del_timer_sync(&dn_db->timer);
-	dn_dev_sysctl_unregister(&dn_db->parms);
-	dn_dev_check_default(dev);
-	neigh_ifdown(&dn_neigh_table, dev);
-
-	if (dn_db->parms.down)
-		dn_db->parms.down(dev);
-
-	dev->dn_ptr = NULL;
-
-	neigh_parms_release(&dn_neigh_table, dn_db->neigh_parms);
-	neigh_ifdown(&dn_neigh_table, dev);
-
-	if (dn_db->router)
-		neigh_release(dn_db->router);
-	if (dn_db->peer)
-		neigh_release(dn_db->peer);
-
-	kfree(dn_db);
-}
-
-void dn_dev_down(struct net_device *dev)
-{
-	struct dn_dev *dn_db = rtnl_dereference(dev->dn_ptr);
-	struct dn_ifaddr *ifa;
-
-	if (dn_db == NULL)
-		return;
-
-	while ((ifa = rtnl_dereference(dn_db->ifa_list)) != NULL) {
-		dn_dev_del_ifa(dn_db, &dn_db->ifa_list, 0);
-		dn_dev_free_ifa(ifa);
-	}
-
-	dn_dev_delete(dev);
-}
-
-void dn_dev_init_pkt(struct sk_buff *skb)
-{
-}
-
-void dn_dev_veri_pkt(struct sk_buff *skb)
-{
-}
-
-void dn_dev_hello(struct sk_buff *skb)
-{
-}
-
-void dn_dev_devices_off(void)
-{
-	struct net_device *dev;
-
-	rtnl_lock();
-	for_each_netdev(&init_net, dev)
-		dn_dev_down(dev);
-	rtnl_unlock();
-
-}
-
-void dn_dev_devices_on(void)
-{
-	struct net_device *dev;
-
-	rtnl_lock();
-	for_each_netdev(&init_net, dev) {
-		if (dev->flags & IFF_UP)
-			dn_dev_up(dev);
-	}
-	rtnl_unlock();
-}
-
-int register_dnaddr_notifier(struct notifier_block *nb)
-{
-	return blocking_notifier_chain_register(&dnaddr_chain, nb);
-}
-
-int unregister_dnaddr_notifier(struct notifier_block *nb)
-{
-	return blocking_notifier_chain_unregister(&dnaddr_chain, nb);
-}
-
-#ifdef CONFIG_PROC_FS
-static inline int is_dn_dev(struct net_device *dev)
-{
-	return dev->dn_ptr != NULL;
-}
-
-static void *dn_dev_seq_start(struct seq_file *seq, loff_t *pos)
-	__acquires(RCU)
-{
-	int i;
-	struct net_device *dev;
-
-	rcu_read_lock();
-
-	if (*pos == 0)
-		return SEQ_START_TOKEN;
-
-	i = 1;
-	for_each_netdev_rcu(&init_net, dev) {
-		if (!is_dn_dev(dev))
-			continue;
-
-		if (i++ == *pos)
-			return dev;
-	}
-
-	return NULL;
-}
-
-static void *dn_dev_seq_next(struct seq_file *seq, void *v, loff_t *pos)
-{
-	struct net_device *dev;
-
-	++*pos;
-
-	dev = v;
-	if (v == SEQ_START_TOKEN)
-		dev = net_device_entry(&init_net.dev_base_head);
-
-	for_each_netdev_continue_rcu(&init_net, dev) {
-		if (!is_dn_dev(dev))
-			continue;
-
-		return dev;
-	}
-
-	return NULL;
-}
-
-static void dn_dev_seq_stop(struct seq_file *seq, void *v)
-	__releases(RCU)
-{
-	rcu_read_unlock();
-}
-
-static char *dn_type2asc(char type)
-{
-	switch (type) {
-	case DN_DEV_BCAST:
-		return "B";
-	case DN_DEV_UCAST:
-		return "U";
-	case DN_DEV_MPOINT:
-		return "M";
-	}
-
-	return "?";
-}
-
-static int dn_dev_seq_show(struct seq_file *seq, void *v)
-{
-	if (v == SEQ_START_TOKEN)
-		seq_puts(seq, "Name     Flags T1   Timer1 T3   Timer3 BlkSize Pri State DevType    Router Peer\n");
-	else {
-		struct net_device *dev = v;
-		char peer_buf[DN_ASCBUF_LEN];
-		char router_buf[DN_ASCBUF_LEN];
-		struct dn_dev *dn_db = rcu_dereference(dev->dn_ptr);
-
-		seq_printf(seq, "%-8s %1s     %04u %04u   %04lu %04lu"
-				"   %04hu    %03d %02x    %-10s %-7s %-7s\n",
-				dev->name,
-				dn_type2asc(dn_db->parms.mode),
-				0, 0,
-				dn_db->t3, dn_db->parms.t3,
-				mtu2blksize(dev),
-				dn_db->parms.priority,
-				dn_db->parms.state, dn_db->parms.name,
-				dn_db->router ? dn_addr2asc(le16_to_cpu(*(__le16 *)dn_db->router->primary_key), router_buf) : "",
-				dn_db->peer ? dn_addr2asc(le16_to_cpu(*(__le16 *)dn_db->peer->primary_key), peer_buf) : "");
-	}
-	return 0;
-}
-
-static const struct seq_operations dn_dev_seq_ops = {
-	.start	= dn_dev_seq_start,
-	.next	= dn_dev_seq_next,
-	.stop	= dn_dev_seq_stop,
-	.show	= dn_dev_seq_show,
-};
-#endif /* CONFIG_PROC_FS */
-
-static int addr[2];
-module_param_array(addr, int, NULL, 0444);
-MODULE_PARM_DESC(addr, "The DECnet address of this machine: area,node");
-
-void __init dn_dev_init(void)
-{
-	if (addr[0] > 63 || addr[0] < 0) {
-		printk(KERN_ERR "DECnet: Area must be between 0 and 63");
-		return;
-	}
-
-	if (addr[1] > 1023 || addr[1] < 0) {
-		printk(KERN_ERR "DECnet: Node must be between 0 and 1023");
-		return;
-	}
-
-	decnet_address = cpu_to_le16((addr[0] << 10) | addr[1]);
-
-	dn_dev_devices_on();
-
-	rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_NEWADDR,
-			     dn_nl_newaddr, NULL, 0);
-	rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_DELADDR,
-			     dn_nl_deladdr, NULL, 0);
-	rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_GETADDR,
-			     NULL, dn_nl_dump_ifaddr, 0);
-
-	proc_create_seq("decnet_dev", 0444, init_net.proc_net, &dn_dev_seq_ops);
-
-#ifdef CONFIG_SYSCTL
-	{
-		int i;
-		for(i = 0; i < DN_DEV_LIST_SIZE; i++)
-			dn_dev_sysctl_register(NULL, &dn_dev_list[i]);
-	}
-#endif /* CONFIG_SYSCTL */
-}
-
-void __exit dn_dev_cleanup(void)
-{
-#ifdef CONFIG_SYSCTL
-	{
-		int i;
-		for(i = 0; i < DN_DEV_LIST_SIZE; i++)
-			dn_dev_sysctl_unregister(&dn_dev_list[i]);
-	}
-#endif /* CONFIG_SYSCTL */
-
-	remove_proc_entry("decnet_dev", init_net.proc_net);
-
-	dn_dev_devices_off();
-}
diff --git a/net/decnet/dn_fib.c b/net/decnet/dn_fib.c
deleted file mode 100644
index 269c029ad74f..000000000000
--- a/net/decnet/dn_fib.c
+++ /dev/null
@@ -1,798 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Routing Forwarding Information Base (Glue/Info List)
- *
- * Author:      Steve Whitehouse <SteveW@ACM.org>
- *
- *
- * Changes:
- *              Alexey Kuznetsov : SMP locking changes
- *              Steve Whitehouse : Rewrote it... Well to be more correct, I
- *                                 copied most of it from the ipv4 fib code.
- *              Steve Whitehouse : Updated it in style and fixed a few bugs
- *                                 which were fixed in the ipv4 code since
- *                                 this code was copied from it.
- *
- */
-#include <linux/string.h>
-#include <linux/net.h>
-#include <linux/socket.h>
-#include <linux/slab.h>
-#include <linux/sockios.h>
-#include <linux/init.h>
-#include <linux/skbuff.h>
-#include <linux/netlink.h>
-#include <linux/rtnetlink.h>
-#include <linux/proc_fs.h>
-#include <linux/netdevice.h>
-#include <linux/timer.h>
-#include <linux/spinlock.h>
-#include <linux/atomic.h>
-#include <linux/uaccess.h>
-#include <net/neighbour.h>
-#include <net/dst.h>
-#include <net/flow.h>
-#include <net/fib_rules.h>
-#include <net/dn.h>
-#include <net/dn_route.h>
-#include <net/dn_fib.h>
-#include <net/dn_neigh.h>
-#include <net/dn_dev.h>
-#include <net/rtnh.h>
-
-#define RT_MIN_TABLE 1
-
-#define for_fib_info() { struct dn_fib_info *fi;\
-	for(fi = dn_fib_info_list; fi; fi = fi->fib_next)
-#define endfor_fib_info() }
-
-#define for_nexthops(fi) { int nhsel; const struct dn_fib_nh *nh;\
-	for(nhsel = 0, nh = (fi)->fib_nh; nhsel < (fi)->fib_nhs; nh++, nhsel++)
-
-#define change_nexthops(fi) { int nhsel; struct dn_fib_nh *nh;\
-	for(nhsel = 0, nh = (struct dn_fib_nh *)((fi)->fib_nh); nhsel < (fi)->fib_nhs; nh++, nhsel++)
-
-#define endfor_nexthops(fi) }
-
-static DEFINE_SPINLOCK(dn_fib_multipath_lock);
-static struct dn_fib_info *dn_fib_info_list;
-static DEFINE_SPINLOCK(dn_fib_info_lock);
-
-static struct
-{
-	int error;
-	u8 scope;
-} dn_fib_props[RTN_MAX+1] = {
-	[RTN_UNSPEC] =      { .error = 0,       .scope = RT_SCOPE_NOWHERE },
-	[RTN_UNICAST] =     { .error = 0,       .scope = RT_SCOPE_UNIVERSE },
-	[RTN_LOCAL] =       { .error = 0,       .scope = RT_SCOPE_HOST },
-	[RTN_BROADCAST] =   { .error = -EINVAL, .scope = RT_SCOPE_NOWHERE },
-	[RTN_ANYCAST] =     { .error = -EINVAL, .scope = RT_SCOPE_NOWHERE },
-	[RTN_MULTICAST] =   { .error = -EINVAL, .scope = RT_SCOPE_NOWHERE },
-	[RTN_BLACKHOLE] =   { .error = -EINVAL, .scope = RT_SCOPE_UNIVERSE },
-	[RTN_UNREACHABLE] = { .error = -EHOSTUNREACH, .scope = RT_SCOPE_UNIVERSE },
-	[RTN_PROHIBIT] =    { .error = -EACCES, .scope = RT_SCOPE_UNIVERSE },
-	[RTN_THROW] =       { .error = -EAGAIN, .scope = RT_SCOPE_UNIVERSE },
-	[RTN_NAT] =         { .error = 0,       .scope = RT_SCOPE_NOWHERE },
-	[RTN_XRESOLVE] =    { .error = -EINVAL, .scope = RT_SCOPE_NOWHERE },
-};
-
-static int dn_fib_sync_down(__le16 local, struct net_device *dev, int force);
-static int dn_fib_sync_up(struct net_device *dev);
-
-void dn_fib_free_info(struct dn_fib_info *fi)
-{
-	if (fi->fib_dead == 0) {
-		printk(KERN_DEBUG "DECnet: BUG! Attempt to free alive dn_fib_info\n");
-		return;
-	}
-
-	change_nexthops(fi) {
-		dev_put(nh->nh_dev);
-		nh->nh_dev = NULL;
-	} endfor_nexthops(fi);
-	kfree(fi);
-}
-
-void dn_fib_release_info(struct dn_fib_info *fi)
-{
-	spin_lock(&dn_fib_info_lock);
-	if (fi && refcount_dec_and_test(&fi->fib_treeref)) {
-		if (fi->fib_next)
-			fi->fib_next->fib_prev = fi->fib_prev;
-		if (fi->fib_prev)
-			fi->fib_prev->fib_next = fi->fib_next;
-		if (fi == dn_fib_info_list)
-			dn_fib_info_list = fi->fib_next;
-		fi->fib_dead = 1;
-		dn_fib_info_put(fi);
-	}
-	spin_unlock(&dn_fib_info_lock);
-}
-
-static inline int dn_fib_nh_comp(const struct dn_fib_info *fi, const struct dn_fib_info *ofi)
-{
-	const struct dn_fib_nh *onh = ofi->fib_nh;
-
-	for_nexthops(fi) {
-		if (nh->nh_oif != onh->nh_oif ||
-			nh->nh_gw != onh->nh_gw ||
-			nh->nh_scope != onh->nh_scope ||
-			nh->nh_weight != onh->nh_weight ||
-			((nh->nh_flags^onh->nh_flags)&~RTNH_F_DEAD))
-				return -1;
-		onh++;
-	} endfor_nexthops(fi);
-	return 0;
-}
-
-static inline struct dn_fib_info *dn_fib_find_info(const struct dn_fib_info *nfi)
-{
-	for_fib_info() {
-		if (fi->fib_nhs != nfi->fib_nhs)
-			continue;
-		if (nfi->fib_protocol == fi->fib_protocol &&
-			nfi->fib_prefsrc == fi->fib_prefsrc &&
-			nfi->fib_priority == fi->fib_priority &&
-			memcmp(nfi->fib_metrics, fi->fib_metrics, sizeof(fi->fib_metrics)) == 0 &&
-			((nfi->fib_flags^fi->fib_flags)&~RTNH_F_DEAD) == 0 &&
-			(nfi->fib_nhs == 0 || dn_fib_nh_comp(fi, nfi) == 0))
-				return fi;
-	} endfor_fib_info();
-	return NULL;
-}
-
-static int dn_fib_count_nhs(const struct nlattr *attr)
-{
-	struct rtnexthop *nhp = nla_data(attr);
-	int nhs = 0, nhlen = nla_len(attr);
-
-	while (rtnh_ok(nhp, nhlen)) {
-		nhs++;
-		nhp = rtnh_next(nhp, &nhlen);
-	}
-
-	/* leftover implies invalid nexthop configuration, discard it */
-	return nhlen > 0 ? 0 : nhs;
-}
-
-static int dn_fib_get_nhs(struct dn_fib_info *fi, const struct nlattr *attr,
-			  const struct rtmsg *r)
-{
-	struct rtnexthop *nhp = nla_data(attr);
-	int nhlen = nla_len(attr);
-
-	change_nexthops(fi) {
-		int attrlen;
-
-		if (!rtnh_ok(nhp, nhlen))
-			return -EINVAL;
-
-		nh->nh_flags  = (r->rtm_flags&~0xFF) | nhp->rtnh_flags;
-		nh->nh_oif    = nhp->rtnh_ifindex;
-		nh->nh_weight = nhp->rtnh_hops + 1;
-
-		attrlen = rtnh_attrlen(nhp);
-		if (attrlen > 0) {
-			struct nlattr *gw_attr;
-
-			gw_attr = nla_find((struct nlattr *) (nhp + 1), attrlen, RTA_GATEWAY);
-			nh->nh_gw = gw_attr ? nla_get_le16(gw_attr) : 0;
-		}
-
-		nhp = rtnh_next(nhp, &nhlen);
-	} endfor_nexthops(fi);
-
-	return 0;
-}
-
-
-static int dn_fib_check_nh(const struct rtmsg *r, struct dn_fib_info *fi, struct dn_fib_nh *nh)
-{
-	int err;
-
-	if (nh->nh_gw) {
-		struct flowidn fld;
-		struct dn_fib_res res;
-
-		if (nh->nh_flags&RTNH_F_ONLINK) {
-			struct net_device *dev;
-
-			if (r->rtm_scope >= RT_SCOPE_LINK)
-				return -EINVAL;
-			if (dnet_addr_type(nh->nh_gw) != RTN_UNICAST)
-				return -EINVAL;
-			if ((dev = __dev_get_by_index(&init_net, nh->nh_oif)) == NULL)
-				return -ENODEV;
-			if (!(dev->flags&IFF_UP))
-				return -ENETDOWN;
-			nh->nh_dev = dev;
-			dev_hold(dev);
-			nh->nh_scope = RT_SCOPE_LINK;
-			return 0;
-		}
-
-		memset(&fld, 0, sizeof(fld));
-		fld.daddr = nh->nh_gw;
-		fld.flowidn_oif = nh->nh_oif;
-		fld.flowidn_scope = r->rtm_scope + 1;
-
-		if (fld.flowidn_scope < RT_SCOPE_LINK)
-			fld.flowidn_scope = RT_SCOPE_LINK;
-
-		if ((err = dn_fib_lookup(&fld, &res)) != 0)
-			return err;
-
-		err = -EINVAL;
-		if (res.type != RTN_UNICAST && res.type != RTN_LOCAL)
-			goto out;
-		nh->nh_scope = res.scope;
-		nh->nh_oif = DN_FIB_RES_OIF(res);
-		nh->nh_dev = DN_FIB_RES_DEV(res);
-		if (nh->nh_dev == NULL)
-			goto out;
-		dev_hold(nh->nh_dev);
-		err = -ENETDOWN;
-		if (!(nh->nh_dev->flags & IFF_UP))
-			goto out;
-		err = 0;
-out:
-		dn_fib_res_put(&res);
-		return err;
-	} else {
-		struct net_device *dev;
-
-		if (nh->nh_flags&(RTNH_F_PERVASIVE|RTNH_F_ONLINK))
-			return -EINVAL;
-
-		dev = __dev_get_by_index(&init_net, nh->nh_oif);
-		if (dev == NULL || dev->dn_ptr == NULL)
-			return -ENODEV;
-		if (!(dev->flags&IFF_UP))
-			return -ENETDOWN;
-		nh->nh_dev = dev;
-		dev_hold(nh->nh_dev);
-		nh->nh_scope = RT_SCOPE_HOST;
-	}
-
-	return 0;
-}
-
-
-struct dn_fib_info *dn_fib_create_info(const struct rtmsg *r, struct nlattr *attrs[],
-				       const struct nlmsghdr *nlh, int *errp)
-{
-	int err;
-	struct dn_fib_info *fi = NULL;
-	struct dn_fib_info *ofi;
-	int nhs = 1;
-
-	if (r->rtm_type > RTN_MAX)
-		goto err_inval;
-
-	if (dn_fib_props[r->rtm_type].scope > r->rtm_scope)
-		goto err_inval;
-
-	if (attrs[RTA_MULTIPATH] &&
-	    (nhs = dn_fib_count_nhs(attrs[RTA_MULTIPATH])) == 0)
-		goto err_inval;
-
-	fi = kzalloc(struct_size(fi, fib_nh, nhs), GFP_KERNEL);
-	err = -ENOBUFS;
-	if (fi == NULL)
-		goto failure;
-
-	fi->fib_protocol = r->rtm_protocol;
-	fi->fib_nhs = nhs;
-	fi->fib_flags = r->rtm_flags;
-
-	if (attrs[RTA_PRIORITY])
-		fi->fib_priority = nla_get_u32(attrs[RTA_PRIORITY]);
-
-	if (attrs[RTA_METRICS]) {
-		struct nlattr *attr;
-		int rem;
-
-		nla_for_each_nested(attr, attrs[RTA_METRICS], rem) {
-			int type = nla_type(attr);
-
-			if (type) {
-				if (type > RTAX_MAX || type == RTAX_CC_ALGO ||
-				    nla_len(attr) < 4)
-					goto err_inval;
-
-				fi->fib_metrics[type-1] = nla_get_u32(attr);
-			}
-		}
-	}
-
-	if (attrs[RTA_PREFSRC])
-		fi->fib_prefsrc = nla_get_le16(attrs[RTA_PREFSRC]);
-
-	if (attrs[RTA_MULTIPATH]) {
-		if ((err = dn_fib_get_nhs(fi, attrs[RTA_MULTIPATH], r)) != 0)
-			goto failure;
-
-		if (attrs[RTA_OIF] &&
-		    fi->fib_nh->nh_oif != nla_get_u32(attrs[RTA_OIF]))
-			goto err_inval;
-
-		if (attrs[RTA_GATEWAY] &&
-		    fi->fib_nh->nh_gw != nla_get_le16(attrs[RTA_GATEWAY]))
-			goto err_inval;
-	} else {
-		struct dn_fib_nh *nh = fi->fib_nh;
-
-		if (attrs[RTA_OIF])
-			nh->nh_oif = nla_get_u32(attrs[RTA_OIF]);
-
-		if (attrs[RTA_GATEWAY])
-			nh->nh_gw = nla_get_le16(attrs[RTA_GATEWAY]);
-
-		nh->nh_flags = r->rtm_flags;
-		nh->nh_weight = 1;
-	}
-
-	if (r->rtm_type == RTN_NAT) {
-		if (!attrs[RTA_GATEWAY] || nhs != 1 || attrs[RTA_OIF])
-			goto err_inval;
-
-		fi->fib_nh->nh_gw = nla_get_le16(attrs[RTA_GATEWAY]);
-		goto link_it;
-	}
-
-	if (dn_fib_props[r->rtm_type].error) {
-		if (attrs[RTA_GATEWAY] || attrs[RTA_OIF] || attrs[RTA_MULTIPATH])
-			goto err_inval;
-
-		goto link_it;
-	}
-
-	if (r->rtm_scope > RT_SCOPE_HOST)
-		goto err_inval;
-
-	if (r->rtm_scope == RT_SCOPE_HOST) {
-		struct dn_fib_nh *nh = fi->fib_nh;
-
-		/* Local address is added */
-		if (nhs != 1 || nh->nh_gw)
-			goto err_inval;
-		nh->nh_scope = RT_SCOPE_NOWHERE;
-		nh->nh_dev = dev_get_by_index(&init_net, fi->fib_nh->nh_oif);
-		err = -ENODEV;
-		if (nh->nh_dev == NULL)
-			goto failure;
-	} else {
-		change_nexthops(fi) {
-			if ((err = dn_fib_check_nh(r, fi, nh)) != 0)
-				goto failure;
-		} endfor_nexthops(fi)
-	}
-
-	if (fi->fib_prefsrc) {
-		if (r->rtm_type != RTN_LOCAL || !attrs[RTA_DST] ||
-		    fi->fib_prefsrc != nla_get_le16(attrs[RTA_DST]))
-			if (dnet_addr_type(fi->fib_prefsrc) != RTN_LOCAL)
-				goto err_inval;
-	}
-
-link_it:
-	if ((ofi = dn_fib_find_info(fi)) != NULL) {
-		fi->fib_dead = 1;
-		dn_fib_free_info(fi);
-		refcount_inc(&ofi->fib_treeref);
-		return ofi;
-	}
-
-	refcount_set(&fi->fib_treeref, 1);
-	refcount_set(&fi->fib_clntref, 1);
-	spin_lock(&dn_fib_info_lock);
-	fi->fib_next = dn_fib_info_list;
-	fi->fib_prev = NULL;
-	if (dn_fib_info_list)
-		dn_fib_info_list->fib_prev = fi;
-	dn_fib_info_list = fi;
-	spin_unlock(&dn_fib_info_lock);
-	return fi;
-
-err_inval:
-	err = -EINVAL;
-
-failure:
-	*errp = err;
-	if (fi) {
-		fi->fib_dead = 1;
-		dn_fib_free_info(fi);
-	}
-
-	return NULL;
-}
-
-int dn_fib_semantic_match(int type, struct dn_fib_info *fi, const struct flowidn *fld, struct dn_fib_res *res)
-{
-	int err = dn_fib_props[type].error;
-
-	if (err == 0) {
-		if (fi->fib_flags & RTNH_F_DEAD)
-			return 1;
-
-		res->fi = fi;
-
-		switch (type) {
-		case RTN_NAT:
-			DN_FIB_RES_RESET(*res);
-			refcount_inc(&fi->fib_clntref);
-			return 0;
-		case RTN_UNICAST:
-		case RTN_LOCAL:
-			for_nexthops(fi) {
-				if (nh->nh_flags & RTNH_F_DEAD)
-					continue;
-				if (!fld->flowidn_oif ||
-				    fld->flowidn_oif == nh->nh_oif)
-					break;
-			}
-			if (nhsel < fi->fib_nhs) {
-				res->nh_sel = nhsel;
-				refcount_inc(&fi->fib_clntref);
-				return 0;
-			}
-			endfor_nexthops(fi);
-			res->fi = NULL;
-			return 1;
-		default:
-			net_err_ratelimited("DECnet: impossible routing event : dn_fib_semantic_match type=%d\n",
-					    type);
-			res->fi = NULL;
-			return -EINVAL;
-		}
-	}
-	return err;
-}
-
-void dn_fib_select_multipath(const struct flowidn *fld, struct dn_fib_res *res)
-{
-	struct dn_fib_info *fi = res->fi;
-	int w;
-
-	spin_lock_bh(&dn_fib_multipath_lock);
-	if (fi->fib_power <= 0) {
-		int power = 0;
-		change_nexthops(fi) {
-			if (!(nh->nh_flags&RTNH_F_DEAD)) {
-				power += nh->nh_weight;
-				nh->nh_power = nh->nh_weight;
-			}
-		} endfor_nexthops(fi);
-		fi->fib_power = power;
-		if (power < 0) {
-			spin_unlock_bh(&dn_fib_multipath_lock);
-			res->nh_sel = 0;
-			return;
-		}
-	}
-
-	w = jiffies % fi->fib_power;
-
-	change_nexthops(fi) {
-		if (!(nh->nh_flags&RTNH_F_DEAD) && nh->nh_power) {
-			if ((w -= nh->nh_power) <= 0) {
-				nh->nh_power--;
-				fi->fib_power--;
-				res->nh_sel = nhsel;
-				spin_unlock_bh(&dn_fib_multipath_lock);
-				return;
-			}
-		}
-	} endfor_nexthops(fi);
-	res->nh_sel = 0;
-	spin_unlock_bh(&dn_fib_multipath_lock);
-}
-
-static inline u32 rtm_get_table(struct nlattr *attrs[], u8 table)
-{
-	if (attrs[RTA_TABLE])
-		table = nla_get_u32(attrs[RTA_TABLE]);
-
-	return table;
-}
-
-static int dn_fib_rtm_delroute(struct sk_buff *skb, struct nlmsghdr *nlh,
-			       struct netlink_ext_ack *extack)
-{
-	struct net *net = sock_net(skb->sk);
-	struct dn_fib_table *tb;
-	struct rtmsg *r = nlmsg_data(nlh);
-	struct nlattr *attrs[RTA_MAX+1];
-	int err;
-
-	if (!netlink_capable(skb, CAP_NET_ADMIN))
-		return -EPERM;
-
-	if (!net_eq(net, &init_net))
-		return -EINVAL;
-
-	err = nlmsg_parse_deprecated(nlh, sizeof(*r), attrs, RTA_MAX,
-				     rtm_dn_policy, extack);
-	if (err < 0)
-		return err;
-
-	tb = dn_fib_get_table(rtm_get_table(attrs, r->rtm_table), 0);
-	if (!tb)
-		return -ESRCH;
-
-	return tb->delete(tb, r, attrs, nlh, &NETLINK_CB(skb));
-}
-
-static int dn_fib_rtm_newroute(struct sk_buff *skb, struct nlmsghdr *nlh,
-			       struct netlink_ext_ack *extack)
-{
-	struct net *net = sock_net(skb->sk);
-	struct dn_fib_table *tb;
-	struct rtmsg *r = nlmsg_data(nlh);
-	struct nlattr *attrs[RTA_MAX+1];
-	int err;
-
-	if (!netlink_capable(skb, CAP_NET_ADMIN))
-		return -EPERM;
-
-	if (!net_eq(net, &init_net))
-		return -EINVAL;
-
-	err = nlmsg_parse_deprecated(nlh, sizeof(*r), attrs, RTA_MAX,
-				     rtm_dn_policy, extack);
-	if (err < 0)
-		return err;
-
-	tb = dn_fib_get_table(rtm_get_table(attrs, r->rtm_table), 1);
-	if (!tb)
-		return -ENOBUFS;
-
-	return tb->insert(tb, r, attrs, nlh, &NETLINK_CB(skb));
-}
-
-static void fib_magic(int cmd, int type, __le16 dst, int dst_len, struct dn_ifaddr *ifa)
-{
-	struct dn_fib_table *tb;
-	struct {
-		struct nlmsghdr nlh;
-		struct rtmsg rtm;
-	} req;
-	struct {
-		struct nlattr hdr;
-		__le16 dst;
-	} dst_attr = {
-		.dst = dst,
-	};
-	struct {
-		struct nlattr hdr;
-		__le16 prefsrc;
-	} prefsrc_attr = {
-		.prefsrc = ifa->ifa_local,
-	};
-	struct {
-		struct nlattr hdr;
-		u32 oif;
-	} oif_attr = {
-		.oif = ifa->ifa_dev->dev->ifindex,
-	};
-	struct nlattr *attrs[RTA_MAX+1] = {
-		[RTA_DST] = (struct nlattr *) &dst_attr,
-		[RTA_PREFSRC] = (struct nlattr * ) &prefsrc_attr,
-		[RTA_OIF] = (struct nlattr *) &oif_attr,
-	};
-
-	memset(&req.rtm, 0, sizeof(req.rtm));
-
-	if (type == RTN_UNICAST)
-		tb = dn_fib_get_table(RT_MIN_TABLE, 1);
-	else
-		tb = dn_fib_get_table(RT_TABLE_LOCAL, 1);
-
-	if (tb == NULL)
-		return;
-
-	req.nlh.nlmsg_len = sizeof(req);
-	req.nlh.nlmsg_type = cmd;
-	req.nlh.nlmsg_flags = NLM_F_REQUEST|NLM_F_CREATE|NLM_F_APPEND;
-	req.nlh.nlmsg_pid = 0;
-	req.nlh.nlmsg_seq = 0;
-
-	req.rtm.rtm_dst_len = dst_len;
-	req.rtm.rtm_table = tb->n;
-	req.rtm.rtm_protocol = RTPROT_KERNEL;
-	req.rtm.rtm_scope = (type != RTN_LOCAL ? RT_SCOPE_LINK : RT_SCOPE_HOST);
-	req.rtm.rtm_type = type;
-
-	if (cmd == RTM_NEWROUTE)
-		tb->insert(tb, &req.rtm, attrs, &req.nlh, NULL);
-	else
-		tb->delete(tb, &req.rtm, attrs, &req.nlh, NULL);
-}
-
-static void dn_fib_add_ifaddr(struct dn_ifaddr *ifa)
-{
-
-	fib_magic(RTM_NEWROUTE, RTN_LOCAL, ifa->ifa_local, 16, ifa);
-
-#if 0
-	if (!(dev->flags&IFF_UP))
-		return;
-	/* In the future, we will want to add default routes here */
-
-#endif
-}
-
-static void dn_fib_del_ifaddr(struct dn_ifaddr *ifa)
-{
-	int found_it = 0;
-	struct net_device *dev;
-	struct dn_dev *dn_db;
-	struct dn_ifaddr *ifa2;
-
-	ASSERT_RTNL();
-
-	/* Scan device list */
-	rcu_read_lock();
-	for_each_netdev_rcu(&init_net, dev) {
-		dn_db = rcu_dereference(dev->dn_ptr);
-		if (dn_db == NULL)
-			continue;
-		for (ifa2 = rcu_dereference(dn_db->ifa_list);
-		     ifa2 != NULL;
-		     ifa2 = rcu_dereference(ifa2->ifa_next)) {
-			if (ifa2->ifa_local == ifa->ifa_local) {
-				found_it = 1;
-				break;
-			}
-		}
-	}
-	rcu_read_unlock();
-
-	if (found_it == 0) {
-		fib_magic(RTM_DELROUTE, RTN_LOCAL, ifa->ifa_local, 16, ifa);
-
-		if (dnet_addr_type(ifa->ifa_local) != RTN_LOCAL) {
-			if (dn_fib_sync_down(ifa->ifa_local, NULL, 0))
-				dn_fib_flush();
-		}
-	}
-}
-
-static void dn_fib_disable_addr(struct net_device *dev, int force)
-{
-	if (dn_fib_sync_down(0, dev, force))
-		dn_fib_flush();
-	dn_rt_cache_flush(0);
-	neigh_ifdown(&dn_neigh_table, dev);
-}
-
-static int dn_fib_dnaddr_event(struct notifier_block *this, unsigned long event, void *ptr)
-{
-	struct dn_ifaddr *ifa = (struct dn_ifaddr *)ptr;
-
-	switch (event) {
-	case NETDEV_UP:
-		dn_fib_add_ifaddr(ifa);
-		dn_fib_sync_up(ifa->ifa_dev->dev);
-		dn_rt_cache_flush(-1);
-		break;
-	case NETDEV_DOWN:
-		dn_fib_del_ifaddr(ifa);
-		if (ifa->ifa_dev && ifa->ifa_dev->ifa_list == NULL) {
-			dn_fib_disable_addr(ifa->ifa_dev->dev, 1);
-		} else {
-			dn_rt_cache_flush(-1);
-		}
-		break;
-	}
-	return NOTIFY_DONE;
-}
-
-static int dn_fib_sync_down(__le16 local, struct net_device *dev, int force)
-{
-	int ret = 0;
-	int scope = RT_SCOPE_NOWHERE;
-
-	if (force)
-		scope = -1;
-
-	for_fib_info() {
-		/*
-		 * This makes no sense for DECnet.... we will almost
-		 * certainly have more than one local address the same
-		 * over all our interfaces. It needs thinking about
-		 * some more.
-		 */
-		if (local && fi->fib_prefsrc == local) {
-			fi->fib_flags |= RTNH_F_DEAD;
-			ret++;
-		} else if (dev && fi->fib_nhs) {
-			int dead = 0;
-
-			change_nexthops(fi) {
-				if (nh->nh_flags&RTNH_F_DEAD)
-					dead++;
-				else if (nh->nh_dev == dev &&
-						nh->nh_scope != scope) {
-					spin_lock_bh(&dn_fib_multipath_lock);
-					nh->nh_flags |= RTNH_F_DEAD;
-					fi->fib_power -= nh->nh_power;
-					nh->nh_power = 0;
-					spin_unlock_bh(&dn_fib_multipath_lock);
-					dead++;
-				}
-			} endfor_nexthops(fi)
-			if (dead == fi->fib_nhs) {
-				fi->fib_flags |= RTNH_F_DEAD;
-				ret++;
-			}
-		}
-	} endfor_fib_info();
-	return ret;
-}
-
-
-static int dn_fib_sync_up(struct net_device *dev)
-{
-	int ret = 0;
-
-	if (!(dev->flags&IFF_UP))
-		return 0;
-
-	for_fib_info() {
-		int alive = 0;
-
-		change_nexthops(fi) {
-			if (!(nh->nh_flags&RTNH_F_DEAD)) {
-				alive++;
-				continue;
-			}
-			if (nh->nh_dev == NULL || !(nh->nh_dev->flags&IFF_UP))
-				continue;
-			if (nh->nh_dev != dev || dev->dn_ptr == NULL)
-				continue;
-			alive++;
-			spin_lock_bh(&dn_fib_multipath_lock);
-			nh->nh_power = 0;
-			nh->nh_flags &= ~RTNH_F_DEAD;
-			spin_unlock_bh(&dn_fib_multipath_lock);
-		} endfor_nexthops(fi);
-
-		if (alive > 0) {
-			fi->fib_flags &= ~RTNH_F_DEAD;
-			ret++;
-		}
-	} endfor_fib_info();
-	return ret;
-}
-
-static struct notifier_block dn_fib_dnaddr_notifier = {
-	.notifier_call = dn_fib_dnaddr_event,
-};
-
-void __exit dn_fib_cleanup(void)
-{
-	dn_fib_table_cleanup();
-	dn_fib_rules_cleanup();
-
-	unregister_dnaddr_notifier(&dn_fib_dnaddr_notifier);
-}
-
-
-void __init dn_fib_init(void)
-{
-	dn_fib_table_init();
-	dn_fib_rules_init();
-
-	register_dnaddr_notifier(&dn_fib_dnaddr_notifier);
-
-	rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_NEWROUTE,
-			     dn_fib_rtm_newroute, NULL, 0);
-	rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_DELROUTE,
-			     dn_fib_rtm_delroute, NULL, 0);
-}
diff --git a/net/decnet/dn_neigh.c b/net/decnet/dn_neigh.c
deleted file mode 100644
index 7c569bcc0aca..000000000000
--- a/net/decnet/dn_neigh.c
+++ /dev/null
@@ -1,607 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Neighbour Functions (Adjacency Database and
- *                                                        On-Ethernet Cache)
- *
- * Author:      Steve Whitehouse <SteveW@ACM.org>
- *
- *
- * Changes:
- *     Steve Whitehouse     : Fixed router listing routine
- *     Steve Whitehouse     : Added error_report functions
- *     Steve Whitehouse     : Added default router detection
- *     Steve Whitehouse     : Hop counts in outgoing messages
- *     Steve Whitehouse     : Fixed src/dst in outgoing messages so
- *                            forwarding now stands a good chance of
- *                            working.
- *     Steve Whitehouse     : Fixed neighbour states (for now anyway).
- *     Steve Whitehouse     : Made error_report functions dummies. This
- *                            is not the right place to return skbs.
- *     Steve Whitehouse     : Convert to seq_file
- *
- */
-
-#include <linux/net.h>
-#include <linux/module.h>
-#include <linux/socket.h>
-#include <linux/if_arp.h>
-#include <linux/slab.h>
-#include <linux/if_ether.h>
-#include <linux/init.h>
-#include <linux/proc_fs.h>
-#include <linux/string.h>
-#include <linux/netfilter_decnet.h>
-#include <linux/spinlock.h>
-#include <linux/seq_file.h>
-#include <linux/rcupdate.h>
-#include <linux/jhash.h>
-#include <linux/atomic.h>
-#include <net/net_namespace.h>
-#include <net/neighbour.h>
-#include <net/dst.h>
-#include <net/flow.h>
-#include <net/dn.h>
-#include <net/dn_dev.h>
-#include <net/dn_neigh.h>
-#include <net/dn_route.h>
-
-static int dn_neigh_construct(struct neighbour *);
-static void dn_neigh_error_report(struct neighbour *, struct sk_buff *);
-static int dn_neigh_output(struct neighbour *neigh, struct sk_buff *skb);
-
-/*
- * Operations for adding the link layer header.
- */
-static const struct neigh_ops dn_neigh_ops = {
-	.family =		AF_DECnet,
-	.error_report =		dn_neigh_error_report,
-	.output =		dn_neigh_output,
-	.connected_output =	dn_neigh_output,
-};
-
-static u32 dn_neigh_hash(const void *pkey,
-			 const struct net_device *dev,
-			 __u32 *hash_rnd)
-{
-	return jhash_2words(*(__u16 *)pkey, 0, hash_rnd[0]);
-}
-
-static bool dn_key_eq(const struct neighbour *neigh, const void *pkey)
-{
-	return neigh_key_eq16(neigh, pkey);
-}
-
-struct neigh_table dn_neigh_table = {
-	.family =			PF_DECnet,
-	.entry_size =			NEIGH_ENTRY_SIZE(sizeof(struct dn_neigh)),
-	.key_len =			sizeof(__le16),
-	.protocol =			cpu_to_be16(ETH_P_DNA_RT),
-	.hash =				dn_neigh_hash,
-	.key_eq =			dn_key_eq,
-	.constructor =			dn_neigh_construct,
-	.id =				"dn_neigh_cache",
-	.parms ={
-		.tbl =			&dn_neigh_table,
-		.reachable_time =	30 * HZ,
-		.data = {
-			[NEIGH_VAR_MCAST_PROBES] = 0,
-			[NEIGH_VAR_UCAST_PROBES] = 0,
-			[NEIGH_VAR_APP_PROBES] = 0,
-			[NEIGH_VAR_RETRANS_TIME] = 1 * HZ,
-			[NEIGH_VAR_BASE_REACHABLE_TIME] = 30 * HZ,
-			[NEIGH_VAR_DELAY_PROBE_TIME] = 5 * HZ,
-			[NEIGH_VAR_INTERVAL_PROBE_TIME_MS] = 5 * HZ,
-			[NEIGH_VAR_GC_STALETIME] = 60 * HZ,
-			[NEIGH_VAR_QUEUE_LEN_BYTES] = SK_WMEM_MAX,
-			[NEIGH_VAR_PROXY_QLEN] = 0,
-			[NEIGH_VAR_ANYCAST_DELAY] = 0,
-			[NEIGH_VAR_PROXY_DELAY] = 0,
-			[NEIGH_VAR_LOCKTIME] = 1 * HZ,
-		},
-	},
-	.gc_interval =			30 * HZ,
-	.gc_thresh1 =			128,
-	.gc_thresh2 =			512,
-	.gc_thresh3 =			1024,
-};
-
-static int dn_neigh_construct(struct neighbour *neigh)
-{
-	struct net_device *dev = neigh->dev;
-	struct dn_neigh *dn = container_of(neigh, struct dn_neigh, n);
-	struct dn_dev *dn_db;
-	struct neigh_parms *parms;
-
-	rcu_read_lock();
-	dn_db = rcu_dereference(dev->dn_ptr);
-	if (dn_db == NULL) {
-		rcu_read_unlock();
-		return -EINVAL;
-	}
-
-	parms = dn_db->neigh_parms;
-	if (!parms) {
-		rcu_read_unlock();
-		return -EINVAL;
-	}
-
-	__neigh_parms_put(neigh->parms);
-	neigh->parms = neigh_parms_clone(parms);
-	rcu_read_unlock();
-
-	neigh->ops = &dn_neigh_ops;
-	neigh->nud_state = NUD_NOARP;
-	neigh->output = neigh->ops->connected_output;
-
-	if ((dev->type == ARPHRD_IPGRE) || (dev->flags & IFF_POINTOPOINT))
-		memcpy(neigh->ha, dev->broadcast, dev->addr_len);
-	else if ((dev->type == ARPHRD_ETHER) || (dev->type == ARPHRD_LOOPBACK))
-		dn_dn2eth(neigh->ha, dn->addr);
-	else {
-		net_dbg_ratelimited("Trying to create neigh for hw %d\n",
-				    dev->type);
-		return -EINVAL;
-	}
-
-	/*
-	 * Make an estimate of the remote block size by assuming that its
-	 * two less then the device mtu, which it true for ethernet (and
-	 * other things which support long format headers) since there is
-	 * an extra length field (of 16 bits) which isn't part of the
-	 * ethernet headers and which the DECnet specs won't admit is part
-	 * of the DECnet routing headers either.
-	 *
-	 * If we over estimate here its no big deal, the NSP negotiations
-	 * will prevent us from sending packets which are too large for the
-	 * remote node to handle. In any case this figure is normally updated
-	 * by a hello message in most cases.
-	 */
-	dn->blksize = dev->mtu - 2;
-
-	return 0;
-}
-
-static void dn_neigh_error_report(struct neighbour *neigh, struct sk_buff *skb)
-{
-	printk(KERN_DEBUG "dn_neigh_error_report: called\n");
-	kfree_skb(skb);
-}
-
-static int dn_neigh_output(struct neighbour *neigh, struct sk_buff *skb)
-{
-	struct dst_entry *dst = skb_dst(skb);
-	struct dn_route *rt = (struct dn_route *)dst;
-	struct net_device *dev = neigh->dev;
-	char mac_addr[ETH_ALEN];
-	unsigned int seq;
-	int err;
-
-	dn_dn2eth(mac_addr, rt->rt_local_src);
-	do {
-		seq = read_seqbegin(&neigh->ha_lock);
-		err = dev_hard_header(skb, dev, ntohs(skb->protocol),
-				      neigh->ha, mac_addr, skb->len);
-	} while (read_seqretry(&neigh->ha_lock, seq));
-
-	if (err >= 0)
-		err = dev_queue_xmit(skb);
-	else {
-		kfree_skb(skb);
-		err = -EINVAL;
-	}
-	return err;
-}
-
-static int dn_neigh_output_packet(struct net *net, struct sock *sk, struct sk_buff *skb)
-{
-	struct dst_entry *dst = skb_dst(skb);
-	struct dn_route *rt = (struct dn_route *)dst;
-	struct neighbour *neigh = rt->n;
-
-	return neigh->output(neigh, skb);
-}
-
-/*
- * For talking to broadcast devices: Ethernet & PPP
- */
-static int dn_long_output(struct neighbour *neigh, struct sock *sk,
-			  struct sk_buff *skb)
-{
-	struct net_device *dev = neigh->dev;
-	int headroom = dev->hard_header_len + sizeof(struct dn_long_packet) + 3;
-	unsigned char *data;
-	struct dn_long_packet *lp;
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-
-
-	if (skb_headroom(skb) < headroom) {
-		struct sk_buff *skb2 = skb_realloc_headroom(skb, headroom);
-		if (skb2 == NULL) {
-			net_crit_ratelimited("dn_long_output: no memory\n");
-			kfree_skb(skb);
-			return -ENOBUFS;
-		}
-		consume_skb(skb);
-		skb = skb2;
-		net_info_ratelimited("dn_long_output: Increasing headroom\n");
-	}
-
-	data = skb_push(skb, sizeof(struct dn_long_packet) + 3);
-	lp = (struct dn_long_packet *)(data+3);
-
-	*((__le16 *)data) = cpu_to_le16(skb->len - 2);
-	*(data + 2) = 1 | DN_RT_F_PF; /* Padding */
-
-	lp->msgflg   = DN_RT_PKT_LONG|(cb->rt_flags&(DN_RT_F_IE|DN_RT_F_RQR|DN_RT_F_RTS));
-	lp->d_area   = lp->d_subarea = 0;
-	dn_dn2eth(lp->d_id, cb->dst);
-	lp->s_area   = lp->s_subarea = 0;
-	dn_dn2eth(lp->s_id, cb->src);
-	lp->nl2      = 0;
-	lp->visit_ct = cb->hops & 0x3f;
-	lp->s_class  = 0;
-	lp->pt       = 0;
-
-	skb_reset_network_header(skb);
-
-	return NF_HOOK(NFPROTO_DECNET, NF_DN_POST_ROUTING,
-		       &init_net, sk, skb, NULL, neigh->dev,
-		       dn_neigh_output_packet);
-}
-
-/*
- * For talking to pointopoint and multidrop devices: DDCMP and X.25
- */
-static int dn_short_output(struct neighbour *neigh, struct sock *sk,
-			   struct sk_buff *skb)
-{
-	struct net_device *dev = neigh->dev;
-	int headroom = dev->hard_header_len + sizeof(struct dn_short_packet) + 2;
-	struct dn_short_packet *sp;
-	unsigned char *data;
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-
-
-	if (skb_headroom(skb) < headroom) {
-		struct sk_buff *skb2 = skb_realloc_headroom(skb, headroom);
-		if (skb2 == NULL) {
-			net_crit_ratelimited("dn_short_output: no memory\n");
-			kfree_skb(skb);
-			return -ENOBUFS;
-		}
-		consume_skb(skb);
-		skb = skb2;
-		net_info_ratelimited("dn_short_output: Increasing headroom\n");
-	}
-
-	data = skb_push(skb, sizeof(struct dn_short_packet) + 2);
-	*((__le16 *)data) = cpu_to_le16(skb->len - 2);
-	sp = (struct dn_short_packet *)(data+2);
-
-	sp->msgflg     = DN_RT_PKT_SHORT|(cb->rt_flags&(DN_RT_F_RQR|DN_RT_F_RTS));
-	sp->dstnode    = cb->dst;
-	sp->srcnode    = cb->src;
-	sp->forward    = cb->hops & 0x3f;
-
-	skb_reset_network_header(skb);
-
-	return NF_HOOK(NFPROTO_DECNET, NF_DN_POST_ROUTING,
-		       &init_net, sk, skb, NULL, neigh->dev,
-		       dn_neigh_output_packet);
-}
-
-/*
- * For talking to DECnet phase III nodes
- * Phase 3 output is the same as short output, execpt that
- * it clears the area bits before transmission.
- */
-static int dn_phase3_output(struct neighbour *neigh, struct sock *sk,
-			    struct sk_buff *skb)
-{
-	struct net_device *dev = neigh->dev;
-	int headroom = dev->hard_header_len + sizeof(struct dn_short_packet) + 2;
-	struct dn_short_packet *sp;
-	unsigned char *data;
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-
-	if (skb_headroom(skb) < headroom) {
-		struct sk_buff *skb2 = skb_realloc_headroom(skb, headroom);
-		if (skb2 == NULL) {
-			net_crit_ratelimited("dn_phase3_output: no memory\n");
-			kfree_skb(skb);
-			return -ENOBUFS;
-		}
-		consume_skb(skb);
-		skb = skb2;
-		net_info_ratelimited("dn_phase3_output: Increasing headroom\n");
-	}
-
-	data = skb_push(skb, sizeof(struct dn_short_packet) + 2);
-	*((__le16 *)data) = cpu_to_le16(skb->len - 2);
-	sp = (struct dn_short_packet *)(data + 2);
-
-	sp->msgflg   = DN_RT_PKT_SHORT|(cb->rt_flags&(DN_RT_F_RQR|DN_RT_F_RTS));
-	sp->dstnode  = cb->dst & cpu_to_le16(0x03ff);
-	sp->srcnode  = cb->src & cpu_to_le16(0x03ff);
-	sp->forward  = cb->hops & 0x3f;
-
-	skb_reset_network_header(skb);
-
-	return NF_HOOK(NFPROTO_DECNET, NF_DN_POST_ROUTING,
-		       &init_net, sk, skb, NULL, neigh->dev,
-		       dn_neigh_output_packet);
-}
-
-int dn_to_neigh_output(struct net *net, struct sock *sk, struct sk_buff *skb)
-{
-	struct dst_entry *dst = skb_dst(skb);
-	struct dn_route *rt = (struct dn_route *) dst;
-	struct neighbour *neigh = rt->n;
-	struct dn_neigh *dn = container_of(neigh, struct dn_neigh, n);
-	struct dn_dev *dn_db;
-	bool use_long;
-
-	rcu_read_lock();
-	dn_db = rcu_dereference(neigh->dev->dn_ptr);
-	if (dn_db == NULL) {
-		rcu_read_unlock();
-		return -EINVAL;
-	}
-	use_long = dn_db->use_long;
-	rcu_read_unlock();
-
-	if (dn->flags & DN_NDFLAG_P3)
-		return dn_phase3_output(neigh, sk, skb);
-	if (use_long)
-		return dn_long_output(neigh, sk, skb);
-	else
-		return dn_short_output(neigh, sk, skb);
-}
-
-/*
- * Unfortunately, the neighbour code uses the device in its hash
- * function, so we don't get any advantage from it. This function
- * basically does a neigh_lookup(), but without comparing the device
- * field. This is required for the On-Ethernet cache
- */
-
-/*
- * Pointopoint link receives a hello message
- */
-void dn_neigh_pointopoint_hello(struct sk_buff *skb)
-{
-	kfree_skb(skb);
-}
-
-/*
- * Ethernet router hello message received
- */
-int dn_neigh_router_hello(struct net *net, struct sock *sk, struct sk_buff *skb)
-{
-	struct rtnode_hello_message *msg = (struct rtnode_hello_message *)skb->data;
-
-	struct neighbour *neigh;
-	struct dn_neigh *dn;
-	struct dn_dev *dn_db;
-	__le16 src;
-
-	src = dn_eth2dn(msg->id);
-
-	neigh = __neigh_lookup(&dn_neigh_table, &src, skb->dev, 1);
-
-	dn = container_of(neigh, struct dn_neigh, n);
-
-	if (neigh) {
-		write_lock(&neigh->lock);
-
-		neigh->used = jiffies;
-		dn_db = rcu_dereference(neigh->dev->dn_ptr);
-
-		if (!(neigh->nud_state & NUD_PERMANENT)) {
-			neigh->updated = jiffies;
-
-			if (neigh->dev->type == ARPHRD_ETHER)
-				memcpy(neigh->ha, &eth_hdr(skb)->h_source, ETH_ALEN);
-
-			dn->blksize  = le16_to_cpu(msg->blksize);
-			dn->priority = msg->priority;
-
-			dn->flags &= ~DN_NDFLAG_P3;
-
-			switch (msg->iinfo & DN_RT_INFO_TYPE) {
-			case DN_RT_INFO_L1RT:
-				dn->flags &=~DN_NDFLAG_R2;
-				dn->flags |= DN_NDFLAG_R1;
-				break;
-			case DN_RT_INFO_L2RT:
-				dn->flags |= DN_NDFLAG_R2;
-			}
-		}
-
-		/* Only use routers in our area */
-		if ((le16_to_cpu(src)>>10) == (le16_to_cpu((decnet_address))>>10)) {
-			if (!dn_db->router) {
-				dn_db->router = neigh_clone(neigh);
-			} else {
-				if (msg->priority > container_of(dn_db->router,
-								 struct dn_neigh, n)->priority)
-					neigh_release(xchg(&dn_db->router, neigh_clone(neigh)));
-			}
-		}
-		write_unlock(&neigh->lock);
-		neigh_release(neigh);
-	}
-
-	kfree_skb(skb);
-	return 0;
-}
-
-/*
- * Endnode hello message received
- */
-int dn_neigh_endnode_hello(struct net *net, struct sock *sk, struct sk_buff *skb)
-{
-	struct endnode_hello_message *msg = (struct endnode_hello_message *)skb->data;
-	struct neighbour *neigh;
-	struct dn_neigh *dn;
-	__le16 src;
-
-	src = dn_eth2dn(msg->id);
-
-	neigh = __neigh_lookup(&dn_neigh_table, &src, skb->dev, 1);
-
-	dn = container_of(neigh, struct dn_neigh, n);
-
-	if (neigh) {
-		write_lock(&neigh->lock);
-
-		neigh->used = jiffies;
-
-		if (!(neigh->nud_state & NUD_PERMANENT)) {
-			neigh->updated = jiffies;
-
-			if (neigh->dev->type == ARPHRD_ETHER)
-				memcpy(neigh->ha, &eth_hdr(skb)->h_source, ETH_ALEN);
-			dn->flags   &= ~(DN_NDFLAG_R1 | DN_NDFLAG_R2);
-			dn->blksize  = le16_to_cpu(msg->blksize);
-			dn->priority = 0;
-		}
-
-		write_unlock(&neigh->lock);
-		neigh_release(neigh);
-	}
-
-	kfree_skb(skb);
-	return 0;
-}
-
-static char *dn_find_slot(char *base, int max, int priority)
-{
-	int i;
-	unsigned char *min = NULL;
-
-	base += 6; /* skip first id */
-
-	for(i = 0; i < max; i++) {
-		if (!min || (*base < *min))
-			min = base;
-		base += 7; /* find next priority */
-	}
-
-	if (!min)
-		return NULL;
-
-	return (*min < priority) ? (min - 6) : NULL;
-}
-
-struct elist_cb_state {
-	struct net_device *dev;
-	unsigned char *ptr;
-	unsigned char *rs;
-	int t, n;
-};
-
-static void neigh_elist_cb(struct neighbour *neigh, void *_info)
-{
-	struct elist_cb_state *s = _info;
-	struct dn_neigh *dn;
-
-	if (neigh->dev != s->dev)
-		return;
-
-	dn = container_of(neigh, struct dn_neigh, n);
-	if (!(dn->flags & (DN_NDFLAG_R1|DN_NDFLAG_R2)))
-		return;
-
-	if (s->t == s->n)
-		s->rs = dn_find_slot(s->ptr, s->n, dn->priority);
-	else
-		s->t++;
-	if (s->rs == NULL)
-		return;
-
-	dn_dn2eth(s->rs, dn->addr);
-	s->rs += 6;
-	*(s->rs) = neigh->nud_state & NUD_CONNECTED ? 0x80 : 0x0;
-	*(s->rs) |= dn->priority;
-	s->rs++;
-}
-
-int dn_neigh_elist(struct net_device *dev, unsigned char *ptr, int n)
-{
-	struct elist_cb_state state;
-
-	state.dev = dev;
-	state.t = 0;
-	state.n = n;
-	state.ptr = ptr;
-	state.rs = ptr;
-
-	neigh_for_each(&dn_neigh_table, neigh_elist_cb, &state);
-
-	return state.t;
-}
-
-
-#ifdef CONFIG_PROC_FS
-
-static inline void dn_neigh_format_entry(struct seq_file *seq,
-					 struct neighbour *n)
-{
-	struct dn_neigh *dn = container_of(n, struct dn_neigh, n);
-	char buf[DN_ASCBUF_LEN];
-
-	read_lock(&n->lock);
-	seq_printf(seq, "%-7s %s%s%s   %02x    %02d  %07ld %-8s\n",
-		   dn_addr2asc(le16_to_cpu(dn->addr), buf),
-		   (dn->flags&DN_NDFLAG_R1) ? "1" : "-",
-		   (dn->flags&DN_NDFLAG_R2) ? "2" : "-",
-		   (dn->flags&DN_NDFLAG_P3) ? "3" : "-",
-		   dn->n.nud_state,
-		   refcount_read(&dn->n.refcnt),
-		   dn->blksize,
-		   (dn->n.dev) ? dn->n.dev->name : "?");
-	read_unlock(&n->lock);
-}
-
-static int dn_neigh_seq_show(struct seq_file *seq, void *v)
-{
-	if (v == SEQ_START_TOKEN) {
-		seq_puts(seq, "Addr    Flags State Use Blksize Dev\n");
-	} else {
-		dn_neigh_format_entry(seq, v);
-	}
-
-	return 0;
-}
-
-static void *dn_neigh_seq_start(struct seq_file *seq, loff_t *pos)
-{
-	return neigh_seq_start(seq, pos, &dn_neigh_table,
-			       NEIGH_SEQ_NEIGH_ONLY);
-}
-
-static const struct seq_operations dn_neigh_seq_ops = {
-	.start = dn_neigh_seq_start,
-	.next  = neigh_seq_next,
-	.stop  = neigh_seq_stop,
-	.show  = dn_neigh_seq_show,
-};
-#endif
-
-void __init dn_neigh_init(void)
-{
-	neigh_table_init(NEIGH_DN_TABLE, &dn_neigh_table);
-	proc_create_net("decnet_neigh", 0444, init_net.proc_net,
-			&dn_neigh_seq_ops, sizeof(struct neigh_seq_state));
-}
-
-void __exit dn_neigh_cleanup(void)
-{
-	remove_proc_entry("decnet_neigh", init_net.proc_net);
-	neigh_table_clear(NEIGH_DN_TABLE, &dn_neigh_table);
-}
diff --git a/net/decnet/dn_nsp_in.c b/net/decnet/dn_nsp_in.c
deleted file mode 100644
index c59be5b04479..000000000000
--- a/net/decnet/dn_nsp_in.c
+++ /dev/null
@@ -1,907 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Network Services Protocol (Input)
- *
- * Author:      Eduardo Marcelo Serrat <emserrat@geocities.com>
- *
- * Changes:
- *
- *    Steve Whitehouse:  Split into dn_nsp_in.c and dn_nsp_out.c from
- *                       original dn_nsp.c.
- *    Steve Whitehouse:  Updated to work with my new routing architecture.
- *    Steve Whitehouse:  Add changes from Eduardo Serrat's patches.
- *    Steve Whitehouse:  Put all ack handling code in a common routine.
- *    Steve Whitehouse:  Put other common bits into dn_nsp_rx()
- *    Steve Whitehouse:  More checks on skb->len to catch bogus packets
- *                       Fixed various race conditions and possible nasties.
- *    Steve Whitehouse:  Now handles returned conninit frames.
- *     David S. Miller:  New socket locking
- *    Steve Whitehouse:  Fixed lockup when socket filtering was enabled.
- *         Paul Koning:  Fix to push CC sockets into RUN when acks are
- *                       received.
- *    Steve Whitehouse:
- *   Patrick Caulfield:  Checking conninits for correctness & sending of error
- *                       responses.
- *    Steve Whitehouse:  Added backlog congestion level return codes.
- *   Patrick Caulfield:
- *    Steve Whitehouse:  Added flow control support (outbound)
- *    Steve Whitehouse:  Prepare for nonlinear skbs
- */
-
-/******************************************************************************
-    (c) 1995-1998 E.M. Serrat		emserrat@geocities.com
-
-*******************************************************************************/
-
-#include <linux/errno.h>
-#include <linux/filter.h>
-#include <linux/types.h>
-#include <linux/socket.h>
-#include <linux/in.h>
-#include <linux/kernel.h>
-#include <linux/timer.h>
-#include <linux/string.h>
-#include <linux/sockios.h>
-#include <linux/net.h>
-#include <linux/netdevice.h>
-#include <linux/inet.h>
-#include <linux/route.h>
-#include <linux/slab.h>
-#include <net/sock.h>
-#include <net/tcp_states.h>
-#include <linux/fcntl.h>
-#include <linux/mm.h>
-#include <linux/termios.h>
-#include <linux/interrupt.h>
-#include <linux/proc_fs.h>
-#include <linux/stat.h>
-#include <linux/init.h>
-#include <linux/poll.h>
-#include <linux/netfilter_decnet.h>
-#include <net/neighbour.h>
-#include <net/dst.h>
-#include <net/dn.h>
-#include <net/dn_nsp.h>
-#include <net/dn_dev.h>
-#include <net/dn_route.h>
-
-extern int decnet_log_martians;
-
-static void dn_log_martian(struct sk_buff *skb, const char *msg)
-{
-	if (decnet_log_martians) {
-		char *devname = skb->dev ? skb->dev->name : "???";
-		struct dn_skb_cb *cb = DN_SKB_CB(skb);
-		net_info_ratelimited("DECnet: Martian packet (%s) dev=%s src=0x%04hx dst=0x%04hx srcport=0x%04hx dstport=0x%04hx\n",
-				     msg, devname,
-				     le16_to_cpu(cb->src),
-				     le16_to_cpu(cb->dst),
-				     le16_to_cpu(cb->src_port),
-				     le16_to_cpu(cb->dst_port));
-	}
-}
-
-/*
- * For this function we've flipped the cross-subchannel bit
- * if the message is an otherdata or linkservice message. Thus
- * we can use it to work out what to update.
- */
-static void dn_ack(struct sock *sk, struct sk_buff *skb, unsigned short ack)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	unsigned short type = ((ack >> 12) & 0x0003);
-	int wakeup = 0;
-
-	switch (type) {
-	case 0: /* ACK - Data */
-		if (dn_after(ack, scp->ackrcv_dat)) {
-			scp->ackrcv_dat = ack & 0x0fff;
-			wakeup |= dn_nsp_check_xmit_queue(sk, skb,
-							  &scp->data_xmit_queue,
-							  ack);
-		}
-		break;
-	case 1: /* NAK - Data */
-		break;
-	case 2: /* ACK - OtherData */
-		if (dn_after(ack, scp->ackrcv_oth)) {
-			scp->ackrcv_oth = ack & 0x0fff;
-			wakeup |= dn_nsp_check_xmit_queue(sk, skb,
-							  &scp->other_xmit_queue,
-							  ack);
-		}
-		break;
-	case 3: /* NAK - OtherData */
-		break;
-	}
-
-	if (wakeup && !sock_flag(sk, SOCK_DEAD))
-		sk->sk_state_change(sk);
-}
-
-/*
- * This function is a universal ack processor.
- */
-static int dn_process_ack(struct sock *sk, struct sk_buff *skb, int oth)
-{
-	__le16 *ptr = (__le16 *)skb->data;
-	int len = 0;
-	unsigned short ack;
-
-	if (skb->len < 2)
-		return len;
-
-	if ((ack = le16_to_cpu(*ptr)) & 0x8000) {
-		skb_pull(skb, 2);
-		ptr++;
-		len += 2;
-		if ((ack & 0x4000) == 0) {
-			if (oth)
-				ack ^= 0x2000;
-			dn_ack(sk, skb, ack);
-		}
-	}
-
-	if (skb->len < 2)
-		return len;
-
-	if ((ack = le16_to_cpu(*ptr)) & 0x8000) {
-		skb_pull(skb, 2);
-		len += 2;
-		if ((ack & 0x4000) == 0) {
-			if (oth)
-				ack ^= 0x2000;
-			dn_ack(sk, skb, ack);
-		}
-	}
-
-	return len;
-}
-
-
-/**
- * dn_check_idf - Check an image data field format is correct.
- * @pptr: Pointer to pointer to image data
- * @len: Pointer to length of image data
- * @max: The maximum allowed length of the data in the image data field
- * @follow_on: Check that this many bytes exist beyond the end of the image data
- *
- * Returns: 0 if ok, -1 on error
- */
-static inline int dn_check_idf(unsigned char **pptr, int *len, unsigned char max, unsigned char follow_on)
-{
-	unsigned char *ptr = *pptr;
-	unsigned char flen = *ptr++;
-
-	(*len)--;
-	if (flen > max)
-		return -1;
-	if ((flen + follow_on) > *len)
-		return -1;
-
-	*len -= flen;
-	*pptr = ptr + flen;
-	return 0;
-}
-
-/*
- * Table of reason codes to pass back to node which sent us a badly
- * formed message, plus text messages for the log. A zero entry in
- * the reason field means "don't reply" otherwise a disc init is sent with
- * the specified reason code.
- */
-static struct {
-	unsigned short reason;
-	const char *text;
-} ci_err_table[] = {
- { 0,             "CI: Truncated message" },
- { NSP_REASON_ID, "CI: Destination username error" },
- { NSP_REASON_ID, "CI: Destination username type" },
- { NSP_REASON_US, "CI: Source username error" },
- { 0,             "CI: Truncated at menuver" },
- { 0,             "CI: Truncated before access or user data" },
- { NSP_REASON_IO, "CI: Access data format error" },
- { NSP_REASON_IO, "CI: User data format error" }
-};
-
-/*
- * This function uses a slightly different lookup method
- * to find its sockets, since it searches on object name/number
- * rather than port numbers. Various tests are done to ensure that
- * the incoming data is in the correct format before it is queued to
- * a socket.
- */
-static struct sock *dn_find_listener(struct sk_buff *skb, unsigned short *reason)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	struct nsp_conn_init_msg *msg = (struct nsp_conn_init_msg *)skb->data;
-	struct sockaddr_dn dstaddr;
-	struct sockaddr_dn srcaddr;
-	unsigned char type = 0;
-	int dstlen;
-	int srclen;
-	unsigned char *ptr;
-	int len;
-	int err = 0;
-	unsigned char menuver;
-
-	memset(&dstaddr, 0, sizeof(struct sockaddr_dn));
-	memset(&srcaddr, 0, sizeof(struct sockaddr_dn));
-
-	/*
-	 * 1. Decode & remove message header
-	 */
-	cb->src_port = msg->srcaddr;
-	cb->dst_port = msg->dstaddr;
-	cb->services = msg->services;
-	cb->info     = msg->info;
-	cb->segsize  = le16_to_cpu(msg->segsize);
-
-	if (!pskb_may_pull(skb, sizeof(*msg)))
-		goto err_out;
-
-	skb_pull(skb, sizeof(*msg));
-
-	len = skb->len;
-	ptr = skb->data;
-
-	/*
-	 * 2. Check destination end username format
-	 */
-	dstlen = dn_username2sockaddr(ptr, len, &dstaddr, &type);
-	err++;
-	if (dstlen < 0)
-		goto err_out;
-
-	err++;
-	if (type > 1)
-		goto err_out;
-
-	len -= dstlen;
-	ptr += dstlen;
-
-	/*
-	 * 3. Check source end username format
-	 */
-	srclen = dn_username2sockaddr(ptr, len, &srcaddr, &type);
-	err++;
-	if (srclen < 0)
-		goto err_out;
-
-	len -= srclen;
-	ptr += srclen;
-	err++;
-	if (len < 1)
-		goto err_out;
-
-	menuver = *ptr;
-	ptr++;
-	len--;
-
-	/*
-	 * 4. Check that optional data actually exists if menuver says it does
-	 */
-	err++;
-	if ((menuver & (DN_MENUVER_ACC | DN_MENUVER_USR)) && (len < 1))
-		goto err_out;
-
-	/*
-	 * 5. Check optional access data format
-	 */
-	err++;
-	if (menuver & DN_MENUVER_ACC) {
-		if (dn_check_idf(&ptr, &len, 39, 1))
-			goto err_out;
-		if (dn_check_idf(&ptr, &len, 39, 1))
-			goto err_out;
-		if (dn_check_idf(&ptr, &len, 39, (menuver & DN_MENUVER_USR) ? 1 : 0))
-			goto err_out;
-	}
-
-	/*
-	 * 6. Check optional user data format
-	 */
-	err++;
-	if (menuver & DN_MENUVER_USR) {
-		if (dn_check_idf(&ptr, &len, 16, 0))
-			goto err_out;
-	}
-
-	/*
-	 * 7. Look up socket based on destination end username
-	 */
-	return dn_sklist_find_listener(&dstaddr);
-err_out:
-	dn_log_martian(skb, ci_err_table[err].text);
-	*reason = ci_err_table[err].reason;
-	return NULL;
-}
-
-
-static void dn_nsp_conn_init(struct sock *sk, struct sk_buff *skb)
-{
-	if (sk_acceptq_is_full(sk)) {
-		kfree_skb(skb);
-		return;
-	}
-
-	sk_acceptq_added(sk);
-	skb_queue_tail(&sk->sk_receive_queue, skb);
-	sk->sk_state_change(sk);
-}
-
-static void dn_nsp_conn_conf(struct sock *sk, struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	struct dn_scp *scp = DN_SK(sk);
-	unsigned char *ptr;
-
-	if (skb->len < 4)
-		goto out;
-
-	ptr = skb->data;
-	cb->services = *ptr++;
-	cb->info = *ptr++;
-	cb->segsize = le16_to_cpu(*(__le16 *)ptr);
-
-	if ((scp->state == DN_CI) || (scp->state == DN_CD)) {
-		scp->persist = 0;
-		scp->addrrem = cb->src_port;
-		sk->sk_state = TCP_ESTABLISHED;
-		scp->state = DN_RUN;
-		scp->services_rem = cb->services;
-		scp->info_rem = cb->info;
-		scp->segsize_rem = cb->segsize;
-
-		if ((scp->services_rem & NSP_FC_MASK) == NSP_FC_NONE)
-			scp->max_window = decnet_no_fc_max_cwnd;
-
-		if (skb->len > 0) {
-			u16 dlen = *skb->data;
-			if ((dlen <= 16) && (dlen <= skb->len)) {
-				scp->conndata_in.opt_optl = cpu_to_le16(dlen);
-				skb_copy_from_linear_data_offset(skb, 1,
-					      scp->conndata_in.opt_data, dlen);
-			}
-		}
-		dn_nsp_send_link(sk, DN_NOCHANGE, 0);
-		if (!sock_flag(sk, SOCK_DEAD))
-			sk->sk_state_change(sk);
-	}
-
-out:
-	kfree_skb(skb);
-}
-
-static void dn_nsp_conn_ack(struct sock *sk, struct sk_buff *skb)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	if (scp->state == DN_CI) {
-		scp->state = DN_CD;
-		scp->persist = 0;
-	}
-
-	kfree_skb(skb);
-}
-
-static void dn_nsp_disc_init(struct sock *sk, struct sk_buff *skb)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	unsigned short reason;
-
-	if (skb->len < 2)
-		goto out;
-
-	reason = le16_to_cpu(*(__le16 *)skb->data);
-	skb_pull(skb, 2);
-
-	scp->discdata_in.opt_status = cpu_to_le16(reason);
-	scp->discdata_in.opt_optl   = 0;
-	memset(scp->discdata_in.opt_data, 0, 16);
-
-	if (skb->len > 0) {
-		u16 dlen = *skb->data;
-		if ((dlen <= 16) && (dlen <= skb->len)) {
-			scp->discdata_in.opt_optl = cpu_to_le16(dlen);
-			skb_copy_from_linear_data_offset(skb, 1, scp->discdata_in.opt_data, dlen);
-		}
-	}
-
-	scp->addrrem = cb->src_port;
-	sk->sk_state = TCP_CLOSE;
-
-	switch (scp->state) {
-	case DN_CI:
-	case DN_CD:
-		scp->state = DN_RJ;
-		sk->sk_err = ECONNREFUSED;
-		break;
-	case DN_RUN:
-		sk->sk_shutdown |= SHUTDOWN_MASK;
-		scp->state = DN_DN;
-		break;
-	case DN_DI:
-		scp->state = DN_DIC;
-		break;
-	}
-
-	if (!sock_flag(sk, SOCK_DEAD)) {
-		if (sk->sk_socket->state != SS_UNCONNECTED)
-			sk->sk_socket->state = SS_DISCONNECTING;
-		sk->sk_state_change(sk);
-	}
-
-	/*
-	 * It appears that its possible for remote machines to send disc
-	 * init messages with no port identifier if we are in the CI and
-	 * possibly also the CD state. Obviously we shouldn't reply with
-	 * a message if we don't know what the end point is.
-	 */
-	if (scp->addrrem) {
-		dn_nsp_send_disc(sk, NSP_DISCCONF, NSP_REASON_DC, GFP_ATOMIC);
-	}
-	scp->persist_fxn = dn_destroy_timer;
-	scp->persist = dn_nsp_persist(sk);
-
-out:
-	kfree_skb(skb);
-}
-
-/*
- * disc_conf messages are also called no_resources or no_link
- * messages depending upon the "reason" field.
- */
-static void dn_nsp_disc_conf(struct sock *sk, struct sk_buff *skb)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	unsigned short reason;
-
-	if (skb->len != 2)
-		goto out;
-
-	reason = le16_to_cpu(*(__le16 *)skb->data);
-
-	sk->sk_state = TCP_CLOSE;
-
-	switch (scp->state) {
-	case DN_CI:
-		scp->state = DN_NR;
-		break;
-	case DN_DR:
-		if (reason == NSP_REASON_DC)
-			scp->state = DN_DRC;
-		if (reason == NSP_REASON_NL)
-			scp->state = DN_CN;
-		break;
-	case DN_DI:
-		scp->state = DN_DIC;
-		break;
-	case DN_RUN:
-		sk->sk_shutdown |= SHUTDOWN_MASK;
-		fallthrough;
-	case DN_CC:
-		scp->state = DN_CN;
-	}
-
-	if (!sock_flag(sk, SOCK_DEAD)) {
-		if (sk->sk_socket->state != SS_UNCONNECTED)
-			sk->sk_socket->state = SS_DISCONNECTING;
-		sk->sk_state_change(sk);
-	}
-
-	scp->persist_fxn = dn_destroy_timer;
-	scp->persist = dn_nsp_persist(sk);
-
-out:
-	kfree_skb(skb);
-}
-
-static void dn_nsp_linkservice(struct sock *sk, struct sk_buff *skb)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	unsigned short segnum;
-	unsigned char lsflags;
-	signed char fcval;
-	int wake_up = 0;
-	char *ptr = skb->data;
-	unsigned char fctype = scp->services_rem & NSP_FC_MASK;
-
-	if (skb->len != 4)
-		goto out;
-
-	segnum = le16_to_cpu(*(__le16 *)ptr);
-	ptr += 2;
-	lsflags = *(unsigned char *)ptr++;
-	fcval = *ptr;
-
-	/*
-	 * Here we ignore erroneous packets which should really
-	 * should cause a connection abort. It is not critical
-	 * for now though.
-	 */
-	if (lsflags & 0xf8)
-		goto out;
-
-	if (seq_next(scp->numoth_rcv, segnum)) {
-		seq_add(&scp->numoth_rcv, 1);
-		switch(lsflags & 0x04) { /* FCVAL INT */
-		case 0x00: /* Normal Request */
-			switch(lsflags & 0x03) { /* FCVAL MOD */
-			case 0x00: /* Request count */
-				if (fcval < 0) {
-					unsigned char p_fcval = -fcval;
-					if ((scp->flowrem_dat > p_fcval) &&
-					    (fctype == NSP_FC_SCMC)) {
-						scp->flowrem_dat -= p_fcval;
-					}
-				} else if (fcval > 0) {
-					scp->flowrem_dat += fcval;
-					wake_up = 1;
-				}
-				break;
-			case 0x01: /* Stop outgoing data */
-				scp->flowrem_sw = DN_DONTSEND;
-				break;
-			case 0x02: /* Ok to start again */
-				scp->flowrem_sw = DN_SEND;
-				dn_nsp_output(sk);
-				wake_up = 1;
-			}
-			break;
-		case 0x04: /* Interrupt Request */
-			if (fcval > 0) {
-				scp->flowrem_oth += fcval;
-				wake_up = 1;
-			}
-			break;
-		}
-		if (wake_up && !sock_flag(sk, SOCK_DEAD))
-			sk->sk_state_change(sk);
-	}
-
-	dn_nsp_send_oth_ack(sk);
-
-out:
-	kfree_skb(skb);
-}
-
-/*
- * Copy of sock_queue_rcv_skb (from sock.h) without
- * bh_lock_sock() (its already held when this is called) which
- * also allows data and other data to be queued to a socket.
- */
-static __inline__ int dn_queue_skb(struct sock *sk, struct sk_buff *skb, int sig, struct sk_buff_head *queue)
-{
-	int err;
-
-	/* Cast skb->rcvbuf to unsigned... It's pointless, but reduces
-	   number of warnings when compiling with -W --ANK
-	 */
-	if (atomic_read(&sk->sk_rmem_alloc) + skb->truesize >=
-	    (unsigned int)sk->sk_rcvbuf) {
-		err = -ENOMEM;
-		goto out;
-	}
-
-	err = sk_filter(sk, skb);
-	if (err)
-		goto out;
-
-	skb_set_owner_r(skb, sk);
-	skb_queue_tail(queue, skb);
-
-	if (!sock_flag(sk, SOCK_DEAD))
-		sk->sk_data_ready(sk);
-out:
-	return err;
-}
-
-static void dn_nsp_otherdata(struct sock *sk, struct sk_buff *skb)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	unsigned short segnum;
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	int queued = 0;
-
-	if (skb->len < 2)
-		goto out;
-
-	cb->segnum = segnum = le16_to_cpu(*(__le16 *)skb->data);
-	skb_pull(skb, 2);
-
-	if (seq_next(scp->numoth_rcv, segnum)) {
-
-		if (dn_queue_skb(sk, skb, SIGURG, &scp->other_receive_queue) == 0) {
-			seq_add(&scp->numoth_rcv, 1);
-			scp->other_report = 0;
-			queued = 1;
-		}
-	}
-
-	dn_nsp_send_oth_ack(sk);
-out:
-	if (!queued)
-		kfree_skb(skb);
-}
-
-static void dn_nsp_data(struct sock *sk, struct sk_buff *skb)
-{
-	int queued = 0;
-	unsigned short segnum;
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	struct dn_scp *scp = DN_SK(sk);
-
-	if (skb->len < 2)
-		goto out;
-
-	cb->segnum = segnum = le16_to_cpu(*(__le16 *)skb->data);
-	skb_pull(skb, 2);
-
-	if (seq_next(scp->numdat_rcv, segnum)) {
-		if (dn_queue_skb(sk, skb, SIGIO, &sk->sk_receive_queue) == 0) {
-			seq_add(&scp->numdat_rcv, 1);
-			queued = 1;
-		}
-
-		if ((scp->flowloc_sw == DN_SEND) && dn_congested(sk)) {
-			scp->flowloc_sw = DN_DONTSEND;
-			dn_nsp_send_link(sk, DN_DONTSEND, 0);
-		}
-	}
-
-	dn_nsp_send_data_ack(sk);
-out:
-	if (!queued)
-		kfree_skb(skb);
-}
-
-/*
- * If one of our conninit messages is returned, this function
- * deals with it. It puts the socket into the NO_COMMUNICATION
- * state.
- */
-static void dn_returned_conn_init(struct sock *sk, struct sk_buff *skb)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	if (scp->state == DN_CI) {
-		scp->state = DN_NC;
-		sk->sk_state = TCP_CLOSE;
-		if (!sock_flag(sk, SOCK_DEAD))
-			sk->sk_state_change(sk);
-	}
-
-	kfree_skb(skb);
-}
-
-static int dn_nsp_no_socket(struct sk_buff *skb, unsigned short reason)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	int ret = NET_RX_DROP;
-
-	/* Must not reply to returned packets */
-	if (cb->rt_flags & DN_RT_F_RTS)
-		goto out;
-
-	if ((reason != NSP_REASON_OK) && ((cb->nsp_flags & 0x0c) == 0x08)) {
-		switch (cb->nsp_flags & 0x70) {
-		case 0x10:
-		case 0x60: /* (Retransmitted) Connect Init */
-			dn_nsp_return_disc(skb, NSP_DISCINIT, reason);
-			ret = NET_RX_SUCCESS;
-			break;
-		case 0x20: /* Connect Confirm */
-			dn_nsp_return_disc(skb, NSP_DISCCONF, reason);
-			ret = NET_RX_SUCCESS;
-			break;
-		}
-	}
-
-out:
-	kfree_skb(skb);
-	return ret;
-}
-
-static int dn_nsp_rx_packet(struct net *net, struct sock *sk2,
-			    struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	struct sock *sk = NULL;
-	unsigned char *ptr = (unsigned char *)skb->data;
-	unsigned short reason = NSP_REASON_NL;
-
-	if (!pskb_may_pull(skb, 2))
-		goto free_out;
-
-	skb_reset_transport_header(skb);
-	cb->nsp_flags = *ptr++;
-
-	if (decnet_debug_level & 2)
-		printk(KERN_DEBUG "dn_nsp_rx: Message type 0x%02x\n", (int)cb->nsp_flags);
-
-	if (cb->nsp_flags & 0x83)
-		goto free_out;
-
-	/*
-	 * Filter out conninits and useless packet types
-	 */
-	if ((cb->nsp_flags & 0x0c) == 0x08) {
-		switch (cb->nsp_flags & 0x70) {
-		case 0x00: /* NOP */
-		case 0x70: /* Reserved */
-		case 0x50: /* Reserved, Phase II node init */
-			goto free_out;
-		case 0x10:
-		case 0x60:
-			if (unlikely(cb->rt_flags & DN_RT_F_RTS))
-				goto free_out;
-			sk = dn_find_listener(skb, &reason);
-			goto got_it;
-		}
-	}
-
-	if (!pskb_may_pull(skb, 3))
-		goto free_out;
-
-	/*
-	 * Grab the destination address.
-	 */
-	cb->dst_port = *(__le16 *)ptr;
-	cb->src_port = 0;
-	ptr += 2;
-
-	/*
-	 * If not a connack, grab the source address too.
-	 */
-	if (pskb_may_pull(skb, 5)) {
-		cb->src_port = *(__le16 *)ptr;
-		ptr += 2;
-		skb_pull(skb, 5);
-	}
-
-	/*
-	 * Returned packets...
-	 * Swap src & dst and look up in the normal way.
-	 */
-	if (unlikely(cb->rt_flags & DN_RT_F_RTS)) {
-		swap(cb->dst_port, cb->src_port);
-		swap(cb->dst, cb->src);
-	}
-
-	/*
-	 * Find the socket to which this skb is destined.
-	 */
-	sk = dn_find_by_skb(skb);
-got_it:
-	if (sk != NULL) {
-		struct dn_scp *scp = DN_SK(sk);
-
-		/* Reset backoff */
-		scp->nsp_rxtshift = 0;
-
-		/*
-		 * We linearize everything except data segments here.
-		 */
-		if (cb->nsp_flags & ~0x60) {
-			if (unlikely(skb_linearize(skb)))
-				goto free_out;
-		}
-
-		return sk_receive_skb(sk, skb, 0);
-	}
-
-	return dn_nsp_no_socket(skb, reason);
-
-free_out:
-	kfree_skb(skb);
-	return NET_RX_DROP;
-}
-
-int dn_nsp_rx(struct sk_buff *skb)
-{
-	return NF_HOOK(NFPROTO_DECNET, NF_DN_LOCAL_IN,
-		       &init_net, NULL, skb, skb->dev, NULL,
-		       dn_nsp_rx_packet);
-}
-
-/*
- * This is the main receive routine for sockets. It is called
- * from the above when the socket is not busy, and also from
- * sock_release() when there is a backlog queued up.
- */
-int dn_nsp_backlog_rcv(struct sock *sk, struct sk_buff *skb)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-
-	if (cb->rt_flags & DN_RT_F_RTS) {
-		if (cb->nsp_flags == 0x18 || cb->nsp_flags == 0x68)
-			dn_returned_conn_init(sk, skb);
-		else
-			kfree_skb(skb);
-		return NET_RX_SUCCESS;
-	}
-
-	/*
-	 * Control packet.
-	 */
-	if ((cb->nsp_flags & 0x0c) == 0x08) {
-		switch (cb->nsp_flags & 0x70) {
-		case 0x10:
-		case 0x60:
-			dn_nsp_conn_init(sk, skb);
-			break;
-		case 0x20:
-			dn_nsp_conn_conf(sk, skb);
-			break;
-		case 0x30:
-			dn_nsp_disc_init(sk, skb);
-			break;
-		case 0x40:
-			dn_nsp_disc_conf(sk, skb);
-			break;
-		}
-
-	} else if (cb->nsp_flags == 0x24) {
-		/*
-		 * Special for connacks, 'cos they don't have
-		 * ack data or ack otherdata info.
-		 */
-		dn_nsp_conn_ack(sk, skb);
-	} else {
-		int other = 1;
-
-		/* both data and ack frames can kick a CC socket into RUN */
-		if ((scp->state == DN_CC) && !sock_flag(sk, SOCK_DEAD)) {
-			scp->state = DN_RUN;
-			sk->sk_state = TCP_ESTABLISHED;
-			sk->sk_state_change(sk);
-		}
-
-		if ((cb->nsp_flags & 0x1c) == 0)
-			other = 0;
-		if (cb->nsp_flags == 0x04)
-			other = 0;
-
-		/*
-		 * Read out ack data here, this applies equally
-		 * to data, other data, link service and both
-		 * ack data and ack otherdata.
-		 */
-		dn_process_ack(sk, skb, other);
-
-		/*
-		 * If we've some sort of data here then call a
-		 * suitable routine for dealing with it, otherwise
-		 * the packet is an ack and can be discarded.
-		 */
-		if ((cb->nsp_flags & 0x0c) == 0) {
-
-			if (scp->state != DN_RUN)
-				goto free_out;
-
-			switch (cb->nsp_flags) {
-			case 0x10: /* LS */
-				dn_nsp_linkservice(sk, skb);
-				break;
-			case 0x30: /* OD */
-				dn_nsp_otherdata(sk, skb);
-				break;
-			default:
-				dn_nsp_data(sk, skb);
-			}
-
-		} else { /* Ack, chuck it out here */
-free_out:
-			kfree_skb(skb);
-		}
-	}
-
-	return NET_RX_SUCCESS;
-}
diff --git a/net/decnet/dn_nsp_out.c b/net/decnet/dn_nsp_out.c
deleted file mode 100644
index b05639bdfc8f..000000000000
--- a/net/decnet/dn_nsp_out.c
+++ /dev/null
@@ -1,696 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Network Services Protocol (Output)
- *
- * Author:      Eduardo Marcelo Serrat <emserrat@geocities.com>
- *
- * Changes:
- *
- *    Steve Whitehouse:  Split into dn_nsp_in.c and dn_nsp_out.c from
- *                       original dn_nsp.c.
- *    Steve Whitehouse:  Updated to work with my new routing architecture.
- *    Steve Whitehouse:  Added changes from Eduardo Serrat's patches.
- *    Steve Whitehouse:  Now conninits have the "return" bit set.
- *    Steve Whitehouse:  Fixes to check alloc'd skbs are non NULL!
- *                       Moved output state machine into one function
- *    Steve Whitehouse:  New output state machine
- *         Paul Koning:  Connect Confirm message fix.
- *      Eduardo Serrat:  Fix to stop dn_nsp_do_disc() sending malformed packets.
- *    Steve Whitehouse:  dn_nsp_output() and friends needed a spring clean
- *    Steve Whitehouse:  Moved dn_nsp_send() in here from route.h
- */
-
-/******************************************************************************
-    (c) 1995-1998 E.M. Serrat		emserrat@geocities.com
-
-*******************************************************************************/
-
-#include <linux/errno.h>
-#include <linux/types.h>
-#include <linux/socket.h>
-#include <linux/in.h>
-#include <linux/kernel.h>
-#include <linux/timer.h>
-#include <linux/string.h>
-#include <linux/sockios.h>
-#include <linux/net.h>
-#include <linux/netdevice.h>
-#include <linux/inet.h>
-#include <linux/route.h>
-#include <linux/slab.h>
-#include <net/sock.h>
-#include <linux/fcntl.h>
-#include <linux/mm.h>
-#include <linux/termios.h>
-#include <linux/interrupt.h>
-#include <linux/proc_fs.h>
-#include <linux/stat.h>
-#include <linux/init.h>
-#include <linux/poll.h>
-#include <linux/if_packet.h>
-#include <linux/jiffies.h>
-#include <net/neighbour.h>
-#include <net/dst.h>
-#include <net/flow.h>
-#include <net/dn.h>
-#include <net/dn_nsp.h>
-#include <net/dn_dev.h>
-#include <net/dn_route.h>
-
-
-static int nsp_backoff[NSP_MAXRXTSHIFT + 1] = { 1, 2, 4, 8, 16, 32, 64, 64, 64, 64, 64, 64, 64 };
-
-static void dn_nsp_send(struct sk_buff *skb)
-{
-	struct sock *sk = skb->sk;
-	struct dn_scp *scp = DN_SK(sk);
-	struct dst_entry *dst;
-	struct flowidn fld;
-
-	skb_reset_transport_header(skb);
-	scp->stamp = jiffies;
-
-	dst = sk_dst_check(sk, 0);
-	if (dst) {
-try_again:
-		skb_dst_set(skb, dst);
-		dst_output(&init_net, skb->sk, skb);
-		return;
-	}
-
-	memset(&fld, 0, sizeof(fld));
-	fld.flowidn_oif = sk->sk_bound_dev_if;
-	fld.saddr = dn_saddr2dn(&scp->addr);
-	fld.daddr = dn_saddr2dn(&scp->peer);
-	dn_sk_ports_copy(&fld, scp);
-	fld.flowidn_proto = DNPROTO_NSP;
-	if (dn_route_output_sock(&sk->sk_dst_cache, &fld, sk, 0) == 0) {
-		dst = sk_dst_get(sk);
-		sk->sk_route_caps = dst->dev->features;
-		goto try_again;
-	}
-
-	sk->sk_err = EHOSTUNREACH;
-	if (!sock_flag(sk, SOCK_DEAD))
-		sk->sk_state_change(sk);
-}
-
-
-/*
- * If sk == NULL, then we assume that we are supposed to be making
- * a routing layer skb. If sk != NULL, then we are supposed to be
- * creating an skb for the NSP layer.
- *
- * The eventual aim is for each socket to have a cached header size
- * for its outgoing packets, and to set hdr from this when sk != NULL.
- */
-struct sk_buff *dn_alloc_skb(struct sock *sk, int size, gfp_t pri)
-{
-	struct sk_buff *skb;
-	int hdr = 64;
-
-	if ((skb = alloc_skb(size + hdr, pri)) == NULL)
-		return NULL;
-
-	skb->protocol = htons(ETH_P_DNA_RT);
-	skb->pkt_type = PACKET_OUTGOING;
-
-	if (sk)
-		skb_set_owner_w(skb, sk);
-
-	skb_reserve(skb, hdr);
-
-	return skb;
-}
-
-/*
- * Calculate persist timer based upon the smoothed round
- * trip time and the variance. Backoff according to the
- * nsp_backoff[] array.
- */
-unsigned long dn_nsp_persist(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	unsigned long t = ((scp->nsp_srtt >> 2) + scp->nsp_rttvar) >> 1;
-
-	t *= nsp_backoff[scp->nsp_rxtshift];
-
-	if (t < HZ) t = HZ;
-	if (t > (600*HZ)) t = (600*HZ);
-
-	if (scp->nsp_rxtshift < NSP_MAXRXTSHIFT)
-		scp->nsp_rxtshift++;
-
-	/* printk(KERN_DEBUG "rxtshift %lu, t=%lu\n", scp->nsp_rxtshift, t); */
-
-	return t;
-}
-
-/*
- * This is called each time we get an estimate for the rtt
- * on the link.
- */
-static void dn_nsp_rtt(struct sock *sk, long rtt)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	long srtt = (long)scp->nsp_srtt;
-	long rttvar = (long)scp->nsp_rttvar;
-	long delta;
-
-	/*
-	 * If the jiffies clock flips over in the middle of timestamp
-	 * gathering this value might turn out negative, so we make sure
-	 * that is it always positive here.
-	 */
-	if (rtt < 0)
-		rtt = -rtt;
-	/*
-	 * Add new rtt to smoothed average
-	 */
-	delta = ((rtt << 3) - srtt);
-	srtt += (delta >> 3);
-	if (srtt >= 1)
-		scp->nsp_srtt = (unsigned long)srtt;
-	else
-		scp->nsp_srtt = 1;
-
-	/*
-	 * Add new rtt variance to smoothed varience
-	 */
-	delta >>= 1;
-	rttvar += ((((delta>0)?(delta):(-delta)) - rttvar) >> 2);
-	if (rttvar >= 1)
-		scp->nsp_rttvar = (unsigned long)rttvar;
-	else
-		scp->nsp_rttvar = 1;
-
-	/* printk(KERN_DEBUG "srtt=%lu rttvar=%lu\n", scp->nsp_srtt, scp->nsp_rttvar); */
-}
-
-/**
- * dn_nsp_clone_and_send - Send a data packet by cloning it
- * @skb: The packet to clone and transmit
- * @gfp: memory allocation flag
- *
- * Clone a queued data or other data packet and transmit it.
- *
- * Returns: The number of times the packet has been sent previously
- */
-static inline unsigned int dn_nsp_clone_and_send(struct sk_buff *skb,
-					     gfp_t gfp)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	struct sk_buff *skb2;
-	int ret = 0;
-
-	if ((skb2 = skb_clone(skb, gfp)) != NULL) {
-		ret = cb->xmit_count;
-		cb->xmit_count++;
-		cb->stamp = jiffies;
-		skb2->sk = skb->sk;
-		dn_nsp_send(skb2);
-	}
-
-	return ret;
-}
-
-/**
- * dn_nsp_output - Try and send something from socket queues
- * @sk: The socket whose queues are to be investigated
- *
- * Try and send the packet on the end of the data and other data queues.
- * Other data gets priority over data, and if we retransmit a packet we
- * reduce the window by dividing it in two.
- *
- */
-void dn_nsp_output(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	struct sk_buff *skb;
-	unsigned int reduce_win = 0;
-
-	/*
-	 * First we check for otherdata/linkservice messages
-	 */
-	if ((skb = skb_peek(&scp->other_xmit_queue)) != NULL)
-		reduce_win = dn_nsp_clone_and_send(skb, GFP_ATOMIC);
-
-	/*
-	 * If we may not send any data, we don't.
-	 * If we are still trying to get some other data down the
-	 * channel, we don't try and send any data.
-	 */
-	if (reduce_win || (scp->flowrem_sw != DN_SEND))
-		goto recalc_window;
-
-	if ((skb = skb_peek(&scp->data_xmit_queue)) != NULL)
-		reduce_win = dn_nsp_clone_and_send(skb, GFP_ATOMIC);
-
-	/*
-	 * If we've sent any frame more than once, we cut the
-	 * send window size in half. There is always a minimum
-	 * window size of one available.
-	 */
-recalc_window:
-	if (reduce_win) {
-		scp->snd_window >>= 1;
-		if (scp->snd_window < NSP_MIN_WINDOW)
-			scp->snd_window = NSP_MIN_WINDOW;
-	}
-}
-
-int dn_nsp_xmit_timeout(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	dn_nsp_output(sk);
-
-	if (!skb_queue_empty(&scp->data_xmit_queue) ||
-	    !skb_queue_empty(&scp->other_xmit_queue))
-		scp->persist = dn_nsp_persist(sk);
-
-	return 0;
-}
-
-static inline __le16 *dn_mk_common_header(struct dn_scp *scp, struct sk_buff *skb, unsigned char msgflag, int len)
-{
-	unsigned char *ptr = skb_push(skb, len);
-
-	BUG_ON(len < 5);
-
-	*ptr++ = msgflag;
-	*((__le16 *)ptr) = scp->addrrem;
-	ptr += 2;
-	*((__le16 *)ptr) = scp->addrloc;
-	ptr += 2;
-	return (__le16 __force *)ptr;
-}
-
-static __le16 *dn_mk_ack_header(struct sock *sk, struct sk_buff *skb, unsigned char msgflag, int hlen, int other)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	unsigned short acknum = scp->numdat_rcv & 0x0FFF;
-	unsigned short ackcrs = scp->numoth_rcv & 0x0FFF;
-	__le16 *ptr;
-
-	BUG_ON(hlen < 9);
-
-	scp->ackxmt_dat = acknum;
-	scp->ackxmt_oth = ackcrs;
-	acknum |= 0x8000;
-	ackcrs |= 0x8000;
-
-	/* If this is an "other data/ack" message, swap acknum and ackcrs */
-	if (other)
-		swap(acknum, ackcrs);
-
-	/* Set "cross subchannel" bit in ackcrs */
-	ackcrs |= 0x2000;
-
-	ptr = dn_mk_common_header(scp, skb, msgflag, hlen);
-
-	*ptr++ = cpu_to_le16(acknum);
-	*ptr++ = cpu_to_le16(ackcrs);
-
-	return ptr;
-}
-
-static __le16 *dn_nsp_mk_data_header(struct sock *sk, struct sk_buff *skb, int oth)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	__le16 *ptr = dn_mk_ack_header(sk, skb, cb->nsp_flags, 11, oth);
-
-	if (unlikely(oth)) {
-		cb->segnum = scp->numoth;
-		seq_add(&scp->numoth, 1);
-	} else {
-		cb->segnum = scp->numdat;
-		seq_add(&scp->numdat, 1);
-	}
-	*(ptr++) = cpu_to_le16(cb->segnum);
-
-	return ptr;
-}
-
-void dn_nsp_queue_xmit(struct sock *sk, struct sk_buff *skb,
-			gfp_t gfp, int oth)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	unsigned long t = ((scp->nsp_srtt >> 2) + scp->nsp_rttvar) >> 1;
-
-	cb->xmit_count = 0;
-	dn_nsp_mk_data_header(sk, skb, oth);
-
-	/*
-	 * Slow start: If we have been idle for more than
-	 * one RTT, then reset window to min size.
-	 */
-	if (time_is_before_jiffies(scp->stamp + t))
-		scp->snd_window = NSP_MIN_WINDOW;
-
-	if (oth)
-		skb_queue_tail(&scp->other_xmit_queue, skb);
-	else
-		skb_queue_tail(&scp->data_xmit_queue, skb);
-
-	if (scp->flowrem_sw != DN_SEND)
-		return;
-
-	dn_nsp_clone_and_send(skb, gfp);
-}
-
-
-int dn_nsp_check_xmit_queue(struct sock *sk, struct sk_buff *skb, struct sk_buff_head *q, unsigned short acknum)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	struct dn_scp *scp = DN_SK(sk);
-	struct sk_buff *skb2, *n, *ack = NULL;
-	int wakeup = 0;
-	int try_retrans = 0;
-	unsigned long reftime = cb->stamp;
-	unsigned long pkttime;
-	unsigned short xmit_count;
-	unsigned short segnum;
-
-	skb_queue_walk_safe(q, skb2, n) {
-		struct dn_skb_cb *cb2 = DN_SKB_CB(skb2);
-
-		if (dn_before_or_equal(cb2->segnum, acknum))
-			ack = skb2;
-
-		/* printk(KERN_DEBUG "ack: %s %04x %04x\n", ack ? "ACK" : "SKIP", (int)cb2->segnum, (int)acknum); */
-
-		if (ack == NULL)
-			continue;
-
-		/* printk(KERN_DEBUG "check_xmit_queue: %04x, %d\n", acknum, cb2->xmit_count); */
-
-		/* Does _last_ packet acked have xmit_count > 1 */
-		try_retrans = 0;
-		/* Remember to wake up the sending process */
-		wakeup = 1;
-		/* Keep various statistics */
-		pkttime = cb2->stamp;
-		xmit_count = cb2->xmit_count;
-		segnum = cb2->segnum;
-		/* Remove and drop ack'ed packet */
-		skb_unlink(ack, q);
-		kfree_skb(ack);
-		ack = NULL;
-
-		/*
-		 * We don't expect to see acknowledgements for packets we
-		 * haven't sent yet.
-		 */
-		WARN_ON(xmit_count == 0);
-
-		/*
-		 * If the packet has only been sent once, we can use it
-		 * to calculate the RTT and also open the window a little
-		 * further.
-		 */
-		if (xmit_count == 1) {
-			if (dn_equal(segnum, acknum))
-				dn_nsp_rtt(sk, (long)(pkttime - reftime));
-
-			if (scp->snd_window < scp->max_window)
-				scp->snd_window++;
-		}
-
-		/*
-		 * Packet has been sent more than once. If this is the last
-		 * packet to be acknowledged then we want to send the next
-		 * packet in the send queue again (assumes the remote host does
-		 * go-back-N error control).
-		 */
-		if (xmit_count > 1)
-			try_retrans = 1;
-	}
-
-	if (try_retrans)
-		dn_nsp_output(sk);
-
-	return wakeup;
-}
-
-void dn_nsp_send_data_ack(struct sock *sk)
-{
-	struct sk_buff *skb = NULL;
-
-	if ((skb = dn_alloc_skb(sk, 9, GFP_ATOMIC)) == NULL)
-		return;
-
-	skb_reserve(skb, 9);
-	dn_mk_ack_header(sk, skb, 0x04, 9, 0);
-	dn_nsp_send(skb);
-}
-
-void dn_nsp_send_oth_ack(struct sock *sk)
-{
-	struct sk_buff *skb = NULL;
-
-	if ((skb = dn_alloc_skb(sk, 9, GFP_ATOMIC)) == NULL)
-		return;
-
-	skb_reserve(skb, 9);
-	dn_mk_ack_header(sk, skb, 0x14, 9, 1);
-	dn_nsp_send(skb);
-}
-
-
-void dn_send_conn_ack (struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	struct sk_buff *skb = NULL;
-	struct nsp_conn_ack_msg *msg;
-
-	if ((skb = dn_alloc_skb(sk, 3, sk->sk_allocation)) == NULL)
-		return;
-
-	msg = skb_put(skb, 3);
-	msg->msgflg = 0x24;
-	msg->dstaddr = scp->addrrem;
-
-	dn_nsp_send(skb);
-}
-
-static int dn_nsp_retrans_conn_conf(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	if (scp->state == DN_CC)
-		dn_send_conn_conf(sk, GFP_ATOMIC);
-
-	return 0;
-}
-
-void dn_send_conn_conf(struct sock *sk, gfp_t gfp)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	struct sk_buff *skb = NULL;
-	struct nsp_conn_init_msg *msg;
-	__u8 len = (__u8)le16_to_cpu(scp->conndata_out.opt_optl);
-
-	if ((skb = dn_alloc_skb(sk, 50 + len, gfp)) == NULL)
-		return;
-
-	msg = skb_put(skb, sizeof(*msg));
-	msg->msgflg = 0x28;
-	msg->dstaddr = scp->addrrem;
-	msg->srcaddr = scp->addrloc;
-	msg->services = scp->services_loc;
-	msg->info = scp->info_loc;
-	msg->segsize = cpu_to_le16(scp->segsize_loc);
-
-	skb_put_u8(skb, len);
-
-	if (len > 0)
-		skb_put_data(skb, scp->conndata_out.opt_data, len);
-
-
-	dn_nsp_send(skb);
-
-	scp->persist = dn_nsp_persist(sk);
-	scp->persist_fxn = dn_nsp_retrans_conn_conf;
-}
-
-
-static __inline__ void dn_nsp_do_disc(struct sock *sk, unsigned char msgflg,
-			unsigned short reason, gfp_t gfp,
-			struct dst_entry *dst,
-			int ddl, unsigned char *dd, __le16 rem, __le16 loc)
-{
-	struct sk_buff *skb = NULL;
-	int size = 7 + ddl + ((msgflg == NSP_DISCINIT) ? 1 : 0);
-	unsigned char *msg;
-
-	if ((dst == NULL) || (rem == 0)) {
-		net_dbg_ratelimited("DECnet: dn_nsp_do_disc: BUG! Please report this to SteveW@ACM.org rem=%u dst=%p\n",
-				    le16_to_cpu(rem), dst);
-		return;
-	}
-
-	if ((skb = dn_alloc_skb(sk, size, gfp)) == NULL)
-		return;
-
-	msg = skb_put(skb, size);
-	*msg++ = msgflg;
-	*(__le16 *)msg = rem;
-	msg += 2;
-	*(__le16 *)msg = loc;
-	msg += 2;
-	*(__le16 *)msg = cpu_to_le16(reason);
-	msg += 2;
-	if (msgflg == NSP_DISCINIT)
-		*msg++ = ddl;
-
-	if (ddl) {
-		memcpy(msg, dd, ddl);
-	}
-
-	/*
-	 * This doesn't go via the dn_nsp_send() function since we need
-	 * to be able to send disc packets out which have no socket
-	 * associations.
-	 */
-	skb_dst_set(skb, dst_clone(dst));
-	dst_output(&init_net, skb->sk, skb);
-}
-
-
-void dn_nsp_send_disc(struct sock *sk, unsigned char msgflg,
-			unsigned short reason, gfp_t gfp)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	int ddl = 0;
-
-	if (msgflg == NSP_DISCINIT)
-		ddl = le16_to_cpu(scp->discdata_out.opt_optl);
-
-	if (reason == 0)
-		reason = le16_to_cpu(scp->discdata_out.opt_status);
-
-	dn_nsp_do_disc(sk, msgflg, reason, gfp, __sk_dst_get(sk), ddl,
-		scp->discdata_out.opt_data, scp->addrrem, scp->addrloc);
-}
-
-
-void dn_nsp_return_disc(struct sk_buff *skb, unsigned char msgflg,
-			unsigned short reason)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	int ddl = 0;
-	gfp_t gfp = GFP_ATOMIC;
-
-	dn_nsp_do_disc(NULL, msgflg, reason, gfp, skb_dst(skb), ddl,
-			NULL, cb->src_port, cb->dst_port);
-}
-
-
-void dn_nsp_send_link(struct sock *sk, unsigned char lsflags, char fcval)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	struct sk_buff *skb;
-	unsigned char *ptr;
-	gfp_t gfp = GFP_ATOMIC;
-
-	if ((skb = dn_alloc_skb(sk, DN_MAX_NSP_DATA_HEADER + 2, gfp)) == NULL)
-		return;
-
-	skb_reserve(skb, DN_MAX_NSP_DATA_HEADER);
-	ptr = skb_put(skb, 2);
-	DN_SKB_CB(skb)->nsp_flags = 0x10;
-	*ptr++ = lsflags;
-	*ptr = fcval;
-
-	dn_nsp_queue_xmit(sk, skb, gfp, 1);
-
-	scp->persist = dn_nsp_persist(sk);
-	scp->persist_fxn = dn_nsp_xmit_timeout;
-}
-
-static int dn_nsp_retrans_conninit(struct sock *sk)
-{
-	struct dn_scp *scp = DN_SK(sk);
-
-	if (scp->state == DN_CI)
-		dn_nsp_send_conninit(sk, NSP_RCI);
-
-	return 0;
-}
-
-void dn_nsp_send_conninit(struct sock *sk, unsigned char msgflg)
-{
-	struct dn_scp *scp = DN_SK(sk);
-	struct nsp_conn_init_msg *msg;
-	unsigned char aux;
-	unsigned char menuver;
-	struct dn_skb_cb *cb;
-	unsigned char type = 1;
-	gfp_t allocation = (msgflg == NSP_CI) ? sk->sk_allocation : GFP_ATOMIC;
-	struct sk_buff *skb = dn_alloc_skb(sk, 200, allocation);
-
-	if (!skb)
-		return;
-
-	cb  = DN_SKB_CB(skb);
-	msg = skb_put(skb, sizeof(*msg));
-
-	msg->msgflg	= msgflg;
-	msg->dstaddr	= 0x0000;		/* Remote Node will assign it*/
-
-	msg->srcaddr	= scp->addrloc;
-	msg->services	= scp->services_loc;	/* Requested flow control    */
-	msg->info	= scp->info_loc;	/* Version Number            */
-	msg->segsize	= cpu_to_le16(scp->segsize_loc);	/* Max segment size  */
-
-	if (scp->peer.sdn_objnum)
-		type = 0;
-
-	skb_put(skb, dn_sockaddr2username(&scp->peer,
-					  skb_tail_pointer(skb), type));
-	skb_put(skb, dn_sockaddr2username(&scp->addr,
-					  skb_tail_pointer(skb), 2));
-
-	menuver = DN_MENUVER_ACC | DN_MENUVER_USR;
-	if (scp->peer.sdn_flags & SDF_PROXY)
-		menuver |= DN_MENUVER_PRX;
-	if (scp->peer.sdn_flags & SDF_UICPROXY)
-		menuver |= DN_MENUVER_UIC;
-
-	skb_put_u8(skb, menuver);	/* Menu Version		*/
-
-	aux = scp->accessdata.acc_userl;
-	skb_put_u8(skb, aux);
-	if (aux > 0)
-		skb_put_data(skb, scp->accessdata.acc_user, aux);
-
-	aux = scp->accessdata.acc_passl;
-	skb_put_u8(skb, aux);
-	if (aux > 0)
-		skb_put_data(skb, scp->accessdata.acc_pass, aux);
-
-	aux = scp->accessdata.acc_accl;
-	skb_put_u8(skb, aux);
-	if (aux > 0)
-		skb_put_data(skb, scp->accessdata.acc_acc, aux);
-
-	aux = (__u8)le16_to_cpu(scp->conndata_out.opt_optl);
-	skb_put_u8(skb, aux);
-	if (aux > 0)
-		skb_put_data(skb, scp->conndata_out.opt_data, aux);
-
-	scp->persist = dn_nsp_persist(sk);
-	scp->persist_fxn = dn_nsp_retrans_conninit;
-
-	cb->rt_flags = DN_RT_F_RQR;
-
-	dn_nsp_send(skb);
-}
diff --git a/net/decnet/dn_route.c b/net/decnet/dn_route.c
deleted file mode 100644
index ac2ee1689111..000000000000
--- a/net/decnet/dn_route.c
+++ /dev/null
@@ -1,1922 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Routing Functions (Endnode and Router)
- *
- * Authors:     Steve Whitehouse <SteveW@ACM.org>
- *              Eduardo Marcelo Serrat <emserrat@geocities.com>
- *
- * Changes:
- *              Steve Whitehouse : Fixes to allow "intra-ethernet" and
- *                                 "return-to-sender" bits on outgoing
- *                                 packets.
- *		Steve Whitehouse : Timeouts for cached routes.
- *              Steve Whitehouse : Use dst cache for input routes too.
- *              Steve Whitehouse : Fixed error values in dn_send_skb.
- *              Steve Whitehouse : Rework routing functions to better fit
- *                                 DECnet routing design
- *              Alexey Kuznetsov : New SMP locking
- *              Steve Whitehouse : More SMP locking changes & dn_cache_dump()
- *              Steve Whitehouse : Prerouting NF hook, now really is prerouting.
- *				   Fixed possible skb leak in rtnetlink funcs.
- *              Steve Whitehouse : Dave Miller's dynamic hash table sizing and
- *                                 Alexey Kuznetsov's finer grained locking
- *                                 from ipv4/route.c.
- *              Steve Whitehouse : Routing is now starting to look like a
- *                                 sensible set of code now, mainly due to
- *                                 my copying the IPv4 routing code. The
- *                                 hooks here are modified and will continue
- *                                 to evolve for a while.
- *              Steve Whitehouse : Real SMP at last :-) Also new netfilter
- *                                 stuff. Look out raw sockets your days
- *                                 are numbered!
- *              Steve Whitehouse : Added return-to-sender functions. Added
- *                                 backlog congestion level return codes.
- *		Steve Whitehouse : Fixed bug where routes were set up with
- *                                 no ref count on net devices.
- *              Steve Whitehouse : RCU for the route cache
- *              Steve Whitehouse : Preparations for the flow cache
- *              Steve Whitehouse : Prepare for nonlinear skbs
- */
-
-/******************************************************************************
-    (c) 1995-1998 E.M. Serrat		emserrat@geocities.com
-
-*******************************************************************************/
-
-#include <linux/errno.h>
-#include <linux/types.h>
-#include <linux/socket.h>
-#include <linux/in.h>
-#include <linux/kernel.h>
-#include <linux/sockios.h>
-#include <linux/net.h>
-#include <linux/netdevice.h>
-#include <linux/inet.h>
-#include <linux/route.h>
-#include <linux/in_route.h>
-#include <linux/slab.h>
-#include <net/sock.h>
-#include <linux/mm.h>
-#include <linux/proc_fs.h>
-#include <linux/seq_file.h>
-#include <linux/init.h>
-#include <linux/rtnetlink.h>
-#include <linux/string.h>
-#include <linux/netfilter_decnet.h>
-#include <linux/rcupdate.h>
-#include <linux/times.h>
-#include <linux/export.h>
-#include <asm/errno.h>
-#include <net/net_namespace.h>
-#include <net/netlink.h>
-#include <net/neighbour.h>
-#include <net/dst.h>
-#include <net/flow.h>
-#include <net/fib_rules.h>
-#include <net/dn.h>
-#include <net/dn_dev.h>
-#include <net/dn_nsp.h>
-#include <net/dn_route.h>
-#include <net/dn_neigh.h>
-#include <net/dn_fib.h>
-
-struct dn_rt_hash_bucket {
-	struct dn_route __rcu *chain;
-	spinlock_t lock;
-};
-
-extern struct neigh_table dn_neigh_table;
-
-
-static unsigned char dn_hiord_addr[6] = {0xAA, 0x00, 0x04, 0x00, 0x00, 0x00};
-
-static const int dn_rt_min_delay = 2 * HZ;
-static const int dn_rt_max_delay = 10 * HZ;
-static const int dn_rt_mtu_expires = 10 * 60 * HZ;
-
-static unsigned long dn_rt_deadline;
-
-static int dn_dst_gc(struct dst_ops *ops);
-static struct dst_entry *dn_dst_check(struct dst_entry *, __u32);
-static unsigned int dn_dst_default_advmss(const struct dst_entry *dst);
-static unsigned int dn_dst_mtu(const struct dst_entry *dst);
-static void dn_dst_destroy(struct dst_entry *);
-static void dn_dst_ifdown(struct dst_entry *, struct net_device *dev, int how);
-static struct dst_entry *dn_dst_negative_advice(struct dst_entry *);
-static void dn_dst_link_failure(struct sk_buff *);
-static void dn_dst_update_pmtu(struct dst_entry *dst, struct sock *sk,
-			       struct sk_buff *skb , u32 mtu,
-			       bool confirm_neigh);
-static void dn_dst_redirect(struct dst_entry *dst, struct sock *sk,
-			    struct sk_buff *skb);
-static struct neighbour *dn_dst_neigh_lookup(const struct dst_entry *dst,
-					     struct sk_buff *skb,
-					     const void *daddr);
-static int dn_route_input(struct sk_buff *);
-static void dn_run_flush(struct timer_list *unused);
-
-static struct dn_rt_hash_bucket *dn_rt_hash_table;
-static unsigned int dn_rt_hash_mask;
-
-static struct timer_list dn_route_timer;
-static DEFINE_TIMER(dn_rt_flush_timer, dn_run_flush);
-int decnet_dst_gc_interval = 2;
-
-static struct dst_ops dn_dst_ops = {
-	.family =		PF_DECnet,
-	.gc_thresh =		128,
-	.gc =			dn_dst_gc,
-	.check =		dn_dst_check,
-	.default_advmss =	dn_dst_default_advmss,
-	.mtu =			dn_dst_mtu,
-	.cow_metrics =		dst_cow_metrics_generic,
-	.destroy =		dn_dst_destroy,
-	.ifdown =		dn_dst_ifdown,
-	.negative_advice =	dn_dst_negative_advice,
-	.link_failure =		dn_dst_link_failure,
-	.update_pmtu =		dn_dst_update_pmtu,
-	.redirect =		dn_dst_redirect,
-	.neigh_lookup =		dn_dst_neigh_lookup,
-};
-
-static void dn_dst_destroy(struct dst_entry *dst)
-{
-	struct dn_route *rt = (struct dn_route *) dst;
-
-	if (rt->n)
-		neigh_release(rt->n);
-	dst_destroy_metrics_generic(dst);
-}
-
-static void dn_dst_ifdown(struct dst_entry *dst, struct net_device *dev, int how)
-{
-	if (how) {
-		struct dn_route *rt = (struct dn_route *) dst;
-		struct neighbour *n = rt->n;
-
-		if (n && n->dev == dev) {
-			n->dev = blackhole_netdev;
-			dev_hold(n->dev);
-			dev_put(dev);
-		}
-	}
-}
-
-static __inline__ unsigned int dn_hash(__le16 src, __le16 dst)
-{
-	__u16 tmp = (__u16 __force)(src ^ dst);
-	tmp ^= (tmp >> 3);
-	tmp ^= (tmp >> 5);
-	tmp ^= (tmp >> 10);
-	return dn_rt_hash_mask & (unsigned int)tmp;
-}
-
-static void dn_dst_check_expire(struct timer_list *unused)
-{
-	int i;
-	struct dn_route *rt;
-	struct dn_route __rcu **rtp;
-	unsigned long now = jiffies;
-	unsigned long expire = 120 * HZ;
-
-	for (i = 0; i <= dn_rt_hash_mask; i++) {
-		rtp = &dn_rt_hash_table[i].chain;
-
-		spin_lock(&dn_rt_hash_table[i].lock);
-		while ((rt = rcu_dereference_protected(*rtp,
-						lockdep_is_held(&dn_rt_hash_table[i].lock))) != NULL) {
-			if (atomic_read(&rt->dst.__refcnt) > 1 ||
-			    (now - rt->dst.lastuse) < expire) {
-				rtp = &rt->dn_next;
-				continue;
-			}
-			*rtp = rt->dn_next;
-			rt->dn_next = NULL;
-			dst_dev_put(&rt->dst);
-			dst_release(&rt->dst);
-		}
-		spin_unlock(&dn_rt_hash_table[i].lock);
-
-		if (jiffies != now)
-			break;
-	}
-
-	mod_timer(&dn_route_timer, now + decnet_dst_gc_interval * HZ);
-}
-
-static int dn_dst_gc(struct dst_ops *ops)
-{
-	struct dn_route *rt;
-	struct dn_route __rcu **rtp;
-	int i;
-	unsigned long now = jiffies;
-	unsigned long expire = 10 * HZ;
-
-	for (i = 0; i <= dn_rt_hash_mask; i++) {
-
-		spin_lock_bh(&dn_rt_hash_table[i].lock);
-		rtp = &dn_rt_hash_table[i].chain;
-
-		while ((rt = rcu_dereference_protected(*rtp,
-						lockdep_is_held(&dn_rt_hash_table[i].lock))) != NULL) {
-			if (atomic_read(&rt->dst.__refcnt) > 1 ||
-			    (now - rt->dst.lastuse) < expire) {
-				rtp = &rt->dn_next;
-				continue;
-			}
-			*rtp = rt->dn_next;
-			rt->dn_next = NULL;
-			dst_dev_put(&rt->dst);
-			dst_release(&rt->dst);
-			break;
-		}
-		spin_unlock_bh(&dn_rt_hash_table[i].lock);
-	}
-
-	return 0;
-}
-
-/*
- * The decnet standards don't impose a particular minimum mtu, what they
- * do insist on is that the routing layer accepts a datagram of at least
- * 230 bytes long. Here we have to subtract the routing header length from
- * 230 to get the minimum acceptable mtu. If there is no neighbour, then we
- * assume the worst and use a long header size.
- *
- * We update both the mtu and the advertised mss (i.e. the segment size we
- * advertise to the other end).
- */
-static void dn_dst_update_pmtu(struct dst_entry *dst, struct sock *sk,
-			       struct sk_buff *skb, u32 mtu,
-			       bool confirm_neigh)
-{
-	struct dn_route *rt = (struct dn_route *) dst;
-	struct neighbour *n = rt->n;
-	u32 min_mtu = 230;
-	struct dn_dev *dn;
-
-	dn = n ? rcu_dereference_raw(n->dev->dn_ptr) : NULL;
-
-	if (dn && dn->use_long == 0)
-		min_mtu -= 6;
-	else
-		min_mtu -= 21;
-
-	if (dst_metric(dst, RTAX_MTU) > mtu && mtu >= min_mtu) {
-		if (!(dst_metric_locked(dst, RTAX_MTU))) {
-			dst_metric_set(dst, RTAX_MTU, mtu);
-			dst_set_expires(dst, dn_rt_mtu_expires);
-		}
-		if (!(dst_metric_locked(dst, RTAX_ADVMSS))) {
-			u32 mss = mtu - DN_MAX_NSP_DATA_HEADER;
-			u32 existing_mss = dst_metric_raw(dst, RTAX_ADVMSS);
-			if (!existing_mss || existing_mss > mss)
-				dst_metric_set(dst, RTAX_ADVMSS, mss);
-		}
-	}
-}
-
-static void dn_dst_redirect(struct dst_entry *dst, struct sock *sk,
-			    struct sk_buff *skb)
-{
-}
-
-/*
- * When a route has been marked obsolete. (e.g. routing cache flush)
- */
-static struct dst_entry *dn_dst_check(struct dst_entry *dst, __u32 cookie)
-{
-	return NULL;
-}
-
-static struct dst_entry *dn_dst_negative_advice(struct dst_entry *dst)
-{
-	dst_release(dst);
-	return NULL;
-}
-
-static void dn_dst_link_failure(struct sk_buff *skb)
-{
-}
-
-static inline int compare_keys(struct flowidn *fl1, struct flowidn *fl2)
-{
-	return ((fl1->daddr ^ fl2->daddr) |
-		(fl1->saddr ^ fl2->saddr) |
-		(fl1->flowidn_mark ^ fl2->flowidn_mark) |
-		(fl1->flowidn_scope ^ fl2->flowidn_scope) |
-		(fl1->flowidn_oif ^ fl2->flowidn_oif) |
-		(fl1->flowidn_iif ^ fl2->flowidn_iif)) == 0;
-}
-
-static int dn_insert_route(struct dn_route *rt, unsigned int hash, struct dn_route **rp)
-{
-	struct dn_route *rth;
-	struct dn_route __rcu **rthp;
-	unsigned long now = jiffies;
-
-	rthp = &dn_rt_hash_table[hash].chain;
-
-	spin_lock_bh(&dn_rt_hash_table[hash].lock);
-	while ((rth = rcu_dereference_protected(*rthp,
-						lockdep_is_held(&dn_rt_hash_table[hash].lock))) != NULL) {
-		if (compare_keys(&rth->fld, &rt->fld)) {
-			/* Put it first */
-			*rthp = rth->dn_next;
-			rcu_assign_pointer(rth->dn_next,
-					   dn_rt_hash_table[hash].chain);
-			rcu_assign_pointer(dn_rt_hash_table[hash].chain, rth);
-
-			dst_hold_and_use(&rth->dst, now);
-			spin_unlock_bh(&dn_rt_hash_table[hash].lock);
-
-			dst_release_immediate(&rt->dst);
-			*rp = rth;
-			return 0;
-		}
-		rthp = &rth->dn_next;
-	}
-
-	rcu_assign_pointer(rt->dn_next, dn_rt_hash_table[hash].chain);
-	rcu_assign_pointer(dn_rt_hash_table[hash].chain, rt);
-
-	dst_hold_and_use(&rt->dst, now);
-	spin_unlock_bh(&dn_rt_hash_table[hash].lock);
-	*rp = rt;
-	return 0;
-}
-
-static void dn_run_flush(struct timer_list *unused)
-{
-	int i;
-	struct dn_route *rt, *next;
-
-	for (i = 0; i < dn_rt_hash_mask; i++) {
-		spin_lock_bh(&dn_rt_hash_table[i].lock);
-
-		rt = xchg((struct dn_route **)&dn_rt_hash_table[i].chain, NULL);
-		if (!rt)
-			goto nothing_to_declare;
-
-		for (; rt; rt = next) {
-			next = rcu_dereference_raw(rt->dn_next);
-			RCU_INIT_POINTER(rt->dn_next, NULL);
-			dst_dev_put(&rt->dst);
-			dst_release(&rt->dst);
-		}
-
-nothing_to_declare:
-		spin_unlock_bh(&dn_rt_hash_table[i].lock);
-	}
-}
-
-static DEFINE_SPINLOCK(dn_rt_flush_lock);
-
-void dn_rt_cache_flush(int delay)
-{
-	unsigned long now = jiffies;
-	int user_mode = !in_interrupt();
-
-	if (delay < 0)
-		delay = dn_rt_min_delay;
-
-	spin_lock_bh(&dn_rt_flush_lock);
-
-	if (del_timer(&dn_rt_flush_timer) && delay > 0 && dn_rt_deadline) {
-		long tmo = (long)(dn_rt_deadline - now);
-
-		if (user_mode && tmo < dn_rt_max_delay - dn_rt_min_delay)
-			tmo = 0;
-
-		if (delay > tmo)
-			delay = tmo;
-	}
-
-	if (delay <= 0) {
-		spin_unlock_bh(&dn_rt_flush_lock);
-		dn_run_flush(NULL);
-		return;
-	}
-
-	if (dn_rt_deadline == 0)
-		dn_rt_deadline = now + dn_rt_max_delay;
-
-	dn_rt_flush_timer.expires = now + delay;
-	add_timer(&dn_rt_flush_timer);
-	spin_unlock_bh(&dn_rt_flush_lock);
-}
-
-/**
- * dn_return_short - Return a short packet to its sender
- * @skb: The packet to return
- *
- */
-static int dn_return_short(struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb;
-	unsigned char *ptr;
-	__le16 *src;
-	__le16 *dst;
-
-	/* Add back headers */
-	skb_push(skb, skb->data - skb_network_header(skb));
-
-	skb = skb_unshare(skb, GFP_ATOMIC);
-	if (!skb)
-		return NET_RX_DROP;
-
-	cb = DN_SKB_CB(skb);
-	/* Skip packet length and point to flags */
-	ptr = skb->data + 2;
-	*ptr++ = (cb->rt_flags & ~DN_RT_F_RQR) | DN_RT_F_RTS;
-
-	dst = (__le16 *)ptr;
-	ptr += 2;
-	src = (__le16 *)ptr;
-	ptr += 2;
-	*ptr = 0; /* Zero hop count */
-
-	swap(*src, *dst);
-
-	skb->pkt_type = PACKET_OUTGOING;
-	dn_rt_finish_output(skb, NULL, NULL);
-	return NET_RX_SUCCESS;
-}
-
-/**
- * dn_return_long - Return a long packet to its sender
- * @skb: The long format packet to return
- *
- */
-static int dn_return_long(struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb;
-	unsigned char *ptr;
-	unsigned char *src_addr, *dst_addr;
-	unsigned char tmp[ETH_ALEN];
-
-	/* Add back all headers */
-	skb_push(skb, skb->data - skb_network_header(skb));
-
-	skb = skb_unshare(skb, GFP_ATOMIC);
-	if (!skb)
-		return NET_RX_DROP;
-
-	cb = DN_SKB_CB(skb);
-	/* Ignore packet length and point to flags */
-	ptr = skb->data + 2;
-
-	/* Skip padding */
-	if (*ptr & DN_RT_F_PF) {
-		char padlen = (*ptr & ~DN_RT_F_PF);
-		ptr += padlen;
-	}
-
-	*ptr++ = (cb->rt_flags & ~DN_RT_F_RQR) | DN_RT_F_RTS;
-	ptr += 2;
-	dst_addr = ptr;
-	ptr += 8;
-	src_addr = ptr;
-	ptr += 6;
-	*ptr = 0; /* Zero hop count */
-
-	/* Swap source and destination */
-	memcpy(tmp, src_addr, ETH_ALEN);
-	memcpy(src_addr, dst_addr, ETH_ALEN);
-	memcpy(dst_addr, tmp, ETH_ALEN);
-
-	skb->pkt_type = PACKET_OUTGOING;
-	dn_rt_finish_output(skb, dst_addr, src_addr);
-	return NET_RX_SUCCESS;
-}
-
-/**
- * dn_route_rx_packet - Try and find a route for an incoming packet
- * @net: The applicable net namespace
- * @sk: Socket packet transmitted on
- * @skb: The packet to find a route for
- *
- * Returns: result of input function if route is found, error code otherwise
- */
-static int dn_route_rx_packet(struct net *net, struct sock *sk, struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb;
-	int err;
-
-	err = dn_route_input(skb);
-	if (err == 0)
-		return dst_input(skb);
-
-	cb = DN_SKB_CB(skb);
-	if (decnet_debug_level & 4) {
-		char *devname = skb->dev ? skb->dev->name : "???";
-
-		printk(KERN_DEBUG
-			"DECnet: dn_route_rx_packet: rt_flags=0x%02x dev=%s len=%d src=0x%04hx dst=0x%04hx err=%d type=%d\n",
-			(int)cb->rt_flags, devname, skb->len,
-			le16_to_cpu(cb->src), le16_to_cpu(cb->dst),
-			err, skb->pkt_type);
-	}
-
-	if ((skb->pkt_type == PACKET_HOST) && (cb->rt_flags & DN_RT_F_RQR)) {
-		switch (cb->rt_flags & DN_RT_PKT_MSK) {
-		case DN_RT_PKT_SHORT:
-			return dn_return_short(skb);
-		case DN_RT_PKT_LONG:
-			return dn_return_long(skb);
-		}
-	}
-
-	kfree_skb(skb);
-	return NET_RX_DROP;
-}
-
-static int dn_route_rx_long(struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	unsigned char *ptr = skb->data;
-
-	if (!pskb_may_pull(skb, 21)) /* 20 for long header, 1 for shortest nsp */
-		goto drop_it;
-
-	skb_pull(skb, 20);
-	skb_reset_transport_header(skb);
-
-	/* Destination info */
-	ptr += 2;
-	cb->dst = dn_eth2dn(ptr);
-	if (memcmp(ptr, dn_hiord_addr, 4) != 0)
-		goto drop_it;
-	ptr += 6;
-
-
-	/* Source info */
-	ptr += 2;
-	cb->src = dn_eth2dn(ptr);
-	if (memcmp(ptr, dn_hiord_addr, 4) != 0)
-		goto drop_it;
-	ptr += 6;
-	/* Other junk */
-	ptr++;
-	cb->hops = *ptr++; /* Visit Count */
-
-	return NF_HOOK(NFPROTO_DECNET, NF_DN_PRE_ROUTING,
-		       &init_net, NULL, skb, skb->dev, NULL,
-		       dn_route_rx_packet);
-
-drop_it:
-	kfree_skb(skb);
-	return NET_RX_DROP;
-}
-
-
-
-static int dn_route_rx_short(struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	unsigned char *ptr = skb->data;
-
-	if (!pskb_may_pull(skb, 6)) /* 5 for short header + 1 for shortest nsp */
-		goto drop_it;
-
-	skb_pull(skb, 5);
-	skb_reset_transport_header(skb);
-
-	cb->dst = *(__le16 *)ptr;
-	ptr += 2;
-	cb->src = *(__le16 *)ptr;
-	ptr += 2;
-	cb->hops = *ptr & 0x3f;
-
-	return NF_HOOK(NFPROTO_DECNET, NF_DN_PRE_ROUTING,
-		       &init_net, NULL, skb, skb->dev, NULL,
-		       dn_route_rx_packet);
-
-drop_it:
-	kfree_skb(skb);
-	return NET_RX_DROP;
-}
-
-static int dn_route_discard(struct net *net, struct sock *sk, struct sk_buff *skb)
-{
-	/*
-	 * I know we drop the packet here, but that's considered success in
-	 * this case
-	 */
-	kfree_skb(skb);
-	return NET_RX_SUCCESS;
-}
-
-static int dn_route_ptp_hello(struct net *net, struct sock *sk, struct sk_buff *skb)
-{
-	dn_dev_hello(skb);
-	dn_neigh_pointopoint_hello(skb);
-	return NET_RX_SUCCESS;
-}
-
-int dn_route_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev)
-{
-	struct dn_skb_cb *cb;
-	unsigned char flags = 0;
-	__u16 len = le16_to_cpu(*(__le16 *)skb->data);
-	struct dn_dev *dn = rcu_dereference(dev->dn_ptr);
-	unsigned char padlen = 0;
-
-	if (!net_eq(dev_net(dev), &init_net))
-		goto dump_it;
-
-	if (dn == NULL)
-		goto dump_it;
-
-	skb = skb_share_check(skb, GFP_ATOMIC);
-	if (!skb)
-		goto out;
-
-	if (!pskb_may_pull(skb, 3))
-		goto dump_it;
-
-	skb_pull(skb, 2);
-
-	if (len > skb->len)
-		goto dump_it;
-
-	skb_trim(skb, len);
-
-	flags = *skb->data;
-
-	cb = DN_SKB_CB(skb);
-	cb->stamp = jiffies;
-	cb->iif = dev->ifindex;
-
-	/*
-	 * If we have padding, remove it.
-	 */
-	if (flags & DN_RT_F_PF) {
-		padlen = flags & ~DN_RT_F_PF;
-		if (!pskb_may_pull(skb, padlen + 1))
-			goto dump_it;
-		skb_pull(skb, padlen);
-		flags = *skb->data;
-	}
-
-	skb_reset_network_header(skb);
-
-	/*
-	 * Weed out future version DECnet
-	 */
-	if (flags & DN_RT_F_VER)
-		goto dump_it;
-
-	cb->rt_flags = flags;
-
-	if (decnet_debug_level & 1)
-		printk(KERN_DEBUG
-			"dn_route_rcv: got 0x%02x from %s [%d %d %d]\n",
-			(int)flags, dev->name, len, skb->len,
-			padlen);
-
-	if (flags & DN_RT_PKT_CNTL) {
-		if (unlikely(skb_linearize(skb)))
-			goto dump_it;
-
-		switch (flags & DN_RT_CNTL_MSK) {
-		case DN_RT_PKT_INIT:
-			dn_dev_init_pkt(skb);
-			break;
-		case DN_RT_PKT_VERI:
-			dn_dev_veri_pkt(skb);
-			break;
-		}
-
-		if (dn->parms.state != DN_DEV_S_RU)
-			goto dump_it;
-
-		switch (flags & DN_RT_CNTL_MSK) {
-		case DN_RT_PKT_HELO:
-			return NF_HOOK(NFPROTO_DECNET, NF_DN_HELLO,
-				       &init_net, NULL, skb, skb->dev, NULL,
-				       dn_route_ptp_hello);
-
-		case DN_RT_PKT_L1RT:
-		case DN_RT_PKT_L2RT:
-			return NF_HOOK(NFPROTO_DECNET, NF_DN_ROUTE,
-				       &init_net, NULL, skb, skb->dev, NULL,
-				       dn_route_discard);
-		case DN_RT_PKT_ERTH:
-			return NF_HOOK(NFPROTO_DECNET, NF_DN_HELLO,
-				       &init_net, NULL, skb, skb->dev, NULL,
-				       dn_neigh_router_hello);
-
-		case DN_RT_PKT_EEDH:
-			return NF_HOOK(NFPROTO_DECNET, NF_DN_HELLO,
-				       &init_net, NULL, skb, skb->dev, NULL,
-				       dn_neigh_endnode_hello);
-		}
-	} else {
-		if (dn->parms.state != DN_DEV_S_RU)
-			goto dump_it;
-
-		skb_pull(skb, 1); /* Pull flags */
-
-		switch (flags & DN_RT_PKT_MSK) {
-		case DN_RT_PKT_LONG:
-			return dn_route_rx_long(skb);
-		case DN_RT_PKT_SHORT:
-			return dn_route_rx_short(skb);
-		}
-	}
-
-dump_it:
-	kfree_skb(skb);
-out:
-	return NET_RX_DROP;
-}
-
-static int dn_output(struct net *net, struct sock *sk, struct sk_buff *skb)
-{
-	struct dst_entry *dst = skb_dst(skb);
-	struct dn_route *rt = (struct dn_route *)dst;
-	struct net_device *dev = dst->dev;
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-
-	int err = -EINVAL;
-
-	if (rt->n == NULL)
-		goto error;
-
-	skb->dev = dev;
-
-	cb->src = rt->rt_saddr;
-	cb->dst = rt->rt_daddr;
-
-	/*
-	 * Always set the Intra-Ethernet bit on all outgoing packets
-	 * originated on this node. Only valid flag from upper layers
-	 * is return-to-sender-requested. Set hop count to 0 too.
-	 */
-	cb->rt_flags &= ~DN_RT_F_RQR;
-	cb->rt_flags |= DN_RT_F_IE;
-	cb->hops = 0;
-
-	return NF_HOOK(NFPROTO_DECNET, NF_DN_LOCAL_OUT,
-		       &init_net, sk, skb, NULL, dev,
-		       dn_to_neigh_output);
-
-error:
-	net_dbg_ratelimited("dn_output: This should not happen\n");
-
-	kfree_skb(skb);
-
-	return err;
-}
-
-static int dn_forward(struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	struct dst_entry *dst = skb_dst(skb);
-	struct dn_dev *dn_db = rcu_dereference(dst->dev->dn_ptr);
-	struct dn_route *rt;
-	int header_len;
-	struct net_device *dev = skb->dev;
-
-	if (skb->pkt_type != PACKET_HOST)
-		goto drop;
-
-	/* Ensure that we have enough space for headers */
-	rt = (struct dn_route *)skb_dst(skb);
-	header_len = dn_db->use_long ? 21 : 6;
-	if (skb_cow(skb, LL_RESERVED_SPACE(rt->dst.dev)+header_len))
-		goto drop;
-
-	/*
-	 * Hop count exceeded.
-	 */
-	if (++cb->hops > 30)
-		goto drop;
-
-	skb->dev = rt->dst.dev;
-
-	/*
-	 * If packet goes out same interface it came in on, then set
-	 * the Intra-Ethernet bit. This has no effect for short
-	 * packets, so we don't need to test for them here.
-	 */
-	cb->rt_flags &= ~DN_RT_F_IE;
-	if (rt->rt_flags & RTCF_DOREDIRECT)
-		cb->rt_flags |= DN_RT_F_IE;
-
-	return NF_HOOK(NFPROTO_DECNET, NF_DN_FORWARD,
-		       &init_net, NULL, skb, dev, skb->dev,
-		       dn_to_neigh_output);
-
-drop:
-	kfree_skb(skb);
-	return NET_RX_DROP;
-}
-
-/*
- * Used to catch bugs. This should never normally get
- * called.
- */
-static int dn_rt_bug_out(struct net *net, struct sock *sk, struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-
-	net_dbg_ratelimited("dn_rt_bug: skb from:%04x to:%04x\n",
-			    le16_to_cpu(cb->src), le16_to_cpu(cb->dst));
-
-	kfree_skb(skb);
-
-	return NET_RX_DROP;
-}
-
-static int dn_rt_bug(struct sk_buff *skb)
-{
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-
-	net_dbg_ratelimited("dn_rt_bug: skb from:%04x to:%04x\n",
-			    le16_to_cpu(cb->src), le16_to_cpu(cb->dst));
-
-	kfree_skb(skb);
-
-	return NET_RX_DROP;
-}
-
-static unsigned int dn_dst_default_advmss(const struct dst_entry *dst)
-{
-	return dn_mss_from_pmtu(dst->dev, dst_mtu(dst));
-}
-
-static unsigned int dn_dst_mtu(const struct dst_entry *dst)
-{
-	unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
-
-	return mtu ? : dst->dev->mtu;
-}
-
-static struct neighbour *dn_dst_neigh_lookup(const struct dst_entry *dst,
-					     struct sk_buff *skb,
-					     const void *daddr)
-{
-	return __neigh_lookup_errno(&dn_neigh_table, daddr, dst->dev);
-}
-
-static int dn_rt_set_next_hop(struct dn_route *rt, struct dn_fib_res *res)
-{
-	struct dn_fib_info *fi = res->fi;
-	struct net_device *dev = rt->dst.dev;
-	unsigned int mss_metric;
-	struct neighbour *n;
-
-	if (fi) {
-		if (DN_FIB_RES_GW(*res) &&
-		    DN_FIB_RES_NH(*res).nh_scope == RT_SCOPE_LINK)
-			rt->rt_gateway = DN_FIB_RES_GW(*res);
-		dst_init_metrics(&rt->dst, fi->fib_metrics, true);
-	}
-	rt->rt_type = res->type;
-
-	if (dev != NULL && rt->n == NULL) {
-		n = __neigh_lookup_errno(&dn_neigh_table, &rt->rt_gateway, dev);
-		if (IS_ERR(n))
-			return PTR_ERR(n);
-		rt->n = n;
-	}
-
-	if (dst_metric(&rt->dst, RTAX_MTU) > rt->dst.dev->mtu)
-		dst_metric_set(&rt->dst, RTAX_MTU, rt->dst.dev->mtu);
-	mss_metric = dst_metric_raw(&rt->dst, RTAX_ADVMSS);
-	if (mss_metric) {
-		unsigned int mss = dn_mss_from_pmtu(dev, dst_mtu(&rt->dst));
-		if (mss_metric > mss)
-			dst_metric_set(&rt->dst, RTAX_ADVMSS, mss);
-	}
-	return 0;
-}
-
-static inline int dn_match_addr(__le16 addr1, __le16 addr2)
-{
-	__u16 tmp = le16_to_cpu(addr1) ^ le16_to_cpu(addr2);
-	int match = 16;
-	while (tmp) {
-		tmp >>= 1;
-		match--;
-	}
-	return match;
-}
-
-static __le16 dnet_select_source(const struct net_device *dev, __le16 daddr, int scope)
-{
-	__le16 saddr = 0;
-	struct dn_dev *dn_db;
-	struct dn_ifaddr *ifa;
-	int best_match = 0;
-	int ret;
-
-	rcu_read_lock();
-	dn_db = rcu_dereference(dev->dn_ptr);
-	for (ifa = rcu_dereference(dn_db->ifa_list);
-	     ifa != NULL;
-	     ifa = rcu_dereference(ifa->ifa_next)) {
-		if (ifa->ifa_scope > scope)
-			continue;
-		if (!daddr) {
-			saddr = ifa->ifa_local;
-			break;
-		}
-		ret = dn_match_addr(daddr, ifa->ifa_local);
-		if (ret > best_match)
-			saddr = ifa->ifa_local;
-		if (best_match == 0)
-			saddr = ifa->ifa_local;
-	}
-	rcu_read_unlock();
-
-	return saddr;
-}
-
-static inline __le16 __dn_fib_res_prefsrc(struct dn_fib_res *res)
-{
-	return dnet_select_source(DN_FIB_RES_DEV(*res), DN_FIB_RES_GW(*res), res->scope);
-}
-
-static inline __le16 dn_fib_rules_map_destination(__le16 daddr, struct dn_fib_res *res)
-{
-	__le16 mask = dnet_make_mask(res->prefixlen);
-	return (daddr&~mask)|res->fi->fib_nh->nh_gw;
-}
-
-static int dn_route_output_slow(struct dst_entry **pprt, const struct flowidn *oldflp, int try_hard)
-{
-	struct flowidn fld = {
-		.daddr = oldflp->daddr,
-		.saddr = oldflp->saddr,
-		.flowidn_scope = RT_SCOPE_UNIVERSE,
-		.flowidn_mark = oldflp->flowidn_mark,
-		.flowidn_iif = LOOPBACK_IFINDEX,
-		.flowidn_oif = oldflp->flowidn_oif,
-	};
-	struct dn_route *rt = NULL;
-	struct net_device *dev_out = NULL, *dev;
-	struct neighbour *neigh = NULL;
-	unsigned int hash;
-	unsigned int flags = 0;
-	struct dn_fib_res res = { .fi = NULL, .type = RTN_UNICAST };
-	int err;
-	int free_res = 0;
-	__le16 gateway = 0;
-
-	if (decnet_debug_level & 16)
-		printk(KERN_DEBUG
-		       "dn_route_output_slow: dst=%04x src=%04x mark=%d"
-		       " iif=%d oif=%d\n", le16_to_cpu(oldflp->daddr),
-		       le16_to_cpu(oldflp->saddr),
-		       oldflp->flowidn_mark, LOOPBACK_IFINDEX,
-		       oldflp->flowidn_oif);
-
-	/* If we have an output interface, verify its a DECnet device */
-	if (oldflp->flowidn_oif) {
-		dev_out = dev_get_by_index(&init_net, oldflp->flowidn_oif);
-		err = -ENODEV;
-		if (dev_out && dev_out->dn_ptr == NULL) {
-			dev_put(dev_out);
-			dev_out = NULL;
-		}
-		if (dev_out == NULL)
-			goto out;
-	}
-
-	/* If we have a source address, verify that its a local address */
-	if (oldflp->saddr) {
-		err = -EADDRNOTAVAIL;
-
-		if (dev_out) {
-			if (dn_dev_islocal(dev_out, oldflp->saddr))
-				goto source_ok;
-			dev_put(dev_out);
-			goto out;
-		}
-		rcu_read_lock();
-		for_each_netdev_rcu(&init_net, dev) {
-			if (!dev->dn_ptr)
-				continue;
-			if (!dn_dev_islocal(dev, oldflp->saddr))
-				continue;
-			if ((dev->flags & IFF_LOOPBACK) &&
-			    oldflp->daddr &&
-			    !dn_dev_islocal(dev, oldflp->daddr))
-				continue;
-
-			dev_out = dev;
-			break;
-		}
-		rcu_read_unlock();
-		if (dev_out == NULL)
-			goto out;
-		dev_hold(dev_out);
-source_ok:
-		;
-	}
-
-	/* No destination? Assume its local */
-	if (!fld.daddr) {
-		fld.daddr = fld.saddr;
-
-		dev_put(dev_out);
-		err = -EINVAL;
-		dev_out = init_net.loopback_dev;
-		if (!dev_out->dn_ptr)
-			goto out;
-		err = -EADDRNOTAVAIL;
-		dev_hold(dev_out);
-		if (!fld.daddr) {
-			fld.daddr =
-			fld.saddr = dnet_select_source(dev_out, 0,
-						       RT_SCOPE_HOST);
-			if (!fld.daddr)
-				goto done;
-		}
-		fld.flowidn_oif = LOOPBACK_IFINDEX;
-		res.type = RTN_LOCAL;
-		goto make_route;
-	}
-
-	if (decnet_debug_level & 16)
-		printk(KERN_DEBUG
-		       "dn_route_output_slow: initial checks complete."
-		       " dst=%04x src=%04x oif=%d try_hard=%d\n",
-		       le16_to_cpu(fld.daddr), le16_to_cpu(fld.saddr),
-		       fld.flowidn_oif, try_hard);
-
-	/*
-	 * N.B. If the kernel is compiled without router support then
-	 * dn_fib_lookup() will evaluate to non-zero so this if () block
-	 * will always be executed.
-	 */
-	err = -ESRCH;
-	if (try_hard || (err = dn_fib_lookup(&fld, &res)) != 0) {
-		struct dn_dev *dn_db;
-		if (err != -ESRCH)
-			goto out;
-		/*
-		 * Here the fallback is basically the standard algorithm for
-		 * routing in endnodes which is described in the DECnet routing
-		 * docs
-		 *
-		 * If we are not trying hard, look in neighbour cache.
-		 * The result is tested to ensure that if a specific output
-		 * device/source address was requested, then we honour that
-		 * here
-		 */
-		if (!try_hard) {
-			neigh = neigh_lookup_nodev(&dn_neigh_table, &init_net, &fld.daddr);
-			if (neigh) {
-				if ((oldflp->flowidn_oif &&
-				    (neigh->dev->ifindex != oldflp->flowidn_oif)) ||
-				    (oldflp->saddr &&
-				    (!dn_dev_islocal(neigh->dev,
-						     oldflp->saddr)))) {
-					neigh_release(neigh);
-					neigh = NULL;
-				} else {
-					dev_put(dev_out);
-					if (dn_dev_islocal(neigh->dev, fld.daddr)) {
-						dev_out = init_net.loopback_dev;
-						res.type = RTN_LOCAL;
-					} else {
-						dev_out = neigh->dev;
-					}
-					dev_hold(dev_out);
-					goto select_source;
-				}
-			}
-		}
-
-		/* Not there? Perhaps its a local address */
-		if (dev_out == NULL)
-			dev_out = dn_dev_get_default();
-		err = -ENODEV;
-		if (dev_out == NULL)
-			goto out;
-		dn_db = rcu_dereference_raw(dev_out->dn_ptr);
-		if (!dn_db)
-			goto e_inval;
-		/* Possible improvement - check all devices for local addr */
-		if (dn_dev_islocal(dev_out, fld.daddr)) {
-			dev_put(dev_out);
-			dev_out = init_net.loopback_dev;
-			dev_hold(dev_out);
-			res.type = RTN_LOCAL;
-			goto select_source;
-		}
-		/* Not local either.... try sending it to the default router */
-		neigh = neigh_clone(dn_db->router);
-		BUG_ON(neigh && neigh->dev != dev_out);
-
-		/* Ok then, we assume its directly connected and move on */
-select_source:
-		if (neigh)
-			gateway = container_of(neigh, struct dn_neigh, n)->addr;
-		if (gateway == 0)
-			gateway = fld.daddr;
-		if (fld.saddr == 0) {
-			fld.saddr = dnet_select_source(dev_out, gateway,
-						       res.type == RTN_LOCAL ?
-						       RT_SCOPE_HOST :
-						       RT_SCOPE_LINK);
-			if (fld.saddr == 0 && res.type != RTN_LOCAL)
-				goto e_addr;
-		}
-		fld.flowidn_oif = dev_out->ifindex;
-		goto make_route;
-	}
-	free_res = 1;
-
-	if (res.type == RTN_NAT)
-		goto e_inval;
-
-	if (res.type == RTN_LOCAL) {
-		if (!fld.saddr)
-			fld.saddr = fld.daddr;
-		dev_put(dev_out);
-		dev_out = init_net.loopback_dev;
-		dev_hold(dev_out);
-		if (!dev_out->dn_ptr)
-			goto e_inval;
-		fld.flowidn_oif = dev_out->ifindex;
-		if (res.fi)
-			dn_fib_info_put(res.fi);
-		res.fi = NULL;
-		goto make_route;
-	}
-
-	if (res.fi->fib_nhs > 1 && fld.flowidn_oif == 0)
-		dn_fib_select_multipath(&fld, &res);
-
-	/*
-	 * We could add some logic to deal with default routes here and
-	 * get rid of some of the special casing above.
-	 */
-
-	if (!fld.saddr)
-		fld.saddr = DN_FIB_RES_PREFSRC(res);
-
-	dev_put(dev_out);
-	dev_out = DN_FIB_RES_DEV(res);
-	dev_hold(dev_out);
-	fld.flowidn_oif = dev_out->ifindex;
-	gateway = DN_FIB_RES_GW(res);
-
-make_route:
-	if (dev_out->flags & IFF_LOOPBACK)
-		flags |= RTCF_LOCAL;
-
-	rt = dst_alloc(&dn_dst_ops, dev_out, 0, DST_OBSOLETE_NONE, 0);
-	if (rt == NULL)
-		goto e_nobufs;
-
-	rt->dn_next = NULL;
-	memset(&rt->fld, 0, sizeof(rt->fld));
-	rt->fld.saddr        = oldflp->saddr;
-	rt->fld.daddr        = oldflp->daddr;
-	rt->fld.flowidn_oif  = oldflp->flowidn_oif;
-	rt->fld.flowidn_iif  = 0;
-	rt->fld.flowidn_mark = oldflp->flowidn_mark;
-
-	rt->rt_saddr      = fld.saddr;
-	rt->rt_daddr      = fld.daddr;
-	rt->rt_gateway    = gateway ? gateway : fld.daddr;
-	rt->rt_local_src  = fld.saddr;
-
-	rt->rt_dst_map    = fld.daddr;
-	rt->rt_src_map    = fld.saddr;
-
-	rt->n = neigh;
-	neigh = NULL;
-
-	rt->dst.lastuse = jiffies;
-	rt->dst.output  = dn_output;
-	rt->dst.input   = dn_rt_bug;
-	rt->rt_flags      = flags;
-	if (flags & RTCF_LOCAL)
-		rt->dst.input = dn_nsp_rx;
-
-	err = dn_rt_set_next_hop(rt, &res);
-	if (err)
-		goto e_neighbour;
-
-	hash = dn_hash(rt->fld.saddr, rt->fld.daddr);
-	/* dn_insert_route() increments dst->__refcnt */
-	dn_insert_route(rt, hash, (struct dn_route **)pprt);
-
-done:
-	if (neigh)
-		neigh_release(neigh);
-	if (free_res)
-		dn_fib_res_put(&res);
-	dev_put(dev_out);
-out:
-	return err;
-
-e_addr:
-	err = -EADDRNOTAVAIL;
-	goto done;
-e_inval:
-	err = -EINVAL;
-	goto done;
-e_nobufs:
-	err = -ENOBUFS;
-	goto done;
-e_neighbour:
-	dst_release_immediate(&rt->dst);
-	goto e_nobufs;
-}
-
-
-/*
- * N.B. The flags may be moved into the flowi at some future stage.
- */
-static int __dn_route_output_key(struct dst_entry **pprt, const struct flowidn *flp, int flags)
-{
-	unsigned int hash = dn_hash(flp->saddr, flp->daddr);
-	struct dn_route *rt = NULL;
-
-	if (!(flags & MSG_TRYHARD)) {
-		rcu_read_lock_bh();
-		for (rt = rcu_dereference_bh(dn_rt_hash_table[hash].chain); rt;
-			rt = rcu_dereference_bh(rt->dn_next)) {
-			if ((flp->daddr == rt->fld.daddr) &&
-			    (flp->saddr == rt->fld.saddr) &&
-			    (flp->flowidn_mark == rt->fld.flowidn_mark) &&
-			    dn_is_output_route(rt) &&
-			    (rt->fld.flowidn_oif == flp->flowidn_oif)) {
-				dst_hold_and_use(&rt->dst, jiffies);
-				rcu_read_unlock_bh();
-				*pprt = &rt->dst;
-				return 0;
-			}
-		}
-		rcu_read_unlock_bh();
-	}
-
-	return dn_route_output_slow(pprt, flp, flags);
-}
-
-static int dn_route_output_key(struct dst_entry **pprt, struct flowidn *flp, int flags)
-{
-	int err;
-
-	err = __dn_route_output_key(pprt, flp, flags);
-	if (err == 0 && flp->flowidn_proto) {
-		*pprt = xfrm_lookup(&init_net, *pprt,
-				    flowidn_to_flowi(flp), NULL, 0);
-		if (IS_ERR(*pprt)) {
-			err = PTR_ERR(*pprt);
-			*pprt = NULL;
-		}
-	}
-	return err;
-}
-
-int dn_route_output_sock(struct dst_entry __rcu **pprt, struct flowidn *fl, struct sock *sk, int flags)
-{
-	int err;
-
-	err = __dn_route_output_key(pprt, fl, flags & MSG_TRYHARD);
-	if (err == 0 && fl->flowidn_proto) {
-		*pprt = xfrm_lookup(&init_net, *pprt,
-				    flowidn_to_flowi(fl), sk, 0);
-		if (IS_ERR(*pprt)) {
-			err = PTR_ERR(*pprt);
-			*pprt = NULL;
-		}
-	}
-	return err;
-}
-
-static int dn_route_input_slow(struct sk_buff *skb)
-{
-	struct dn_route *rt = NULL;
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	struct net_device *in_dev = skb->dev;
-	struct net_device *out_dev = NULL;
-	struct dn_dev *dn_db;
-	struct neighbour *neigh = NULL;
-	unsigned int hash;
-	int flags = 0;
-	__le16 gateway = 0;
-	__le16 local_src = 0;
-	struct flowidn fld = {
-		.daddr = cb->dst,
-		.saddr = cb->src,
-		.flowidn_scope = RT_SCOPE_UNIVERSE,
-		.flowidn_mark = skb->mark,
-		.flowidn_iif = skb->dev->ifindex,
-	};
-	struct dn_fib_res res = { .fi = NULL, .type = RTN_UNREACHABLE };
-	int err = -EINVAL;
-	int free_res = 0;
-
-	dev_hold(in_dev);
-
-	dn_db = rcu_dereference(in_dev->dn_ptr);
-	if (!dn_db)
-		goto out;
-
-	/* Zero source addresses are not allowed */
-	if (fld.saddr == 0)
-		goto out;
-
-	/*
-	 * In this case we've just received a packet from a source
-	 * outside ourselves pretending to come from us. We don't
-	 * allow it any further to prevent routing loops, spoofing and
-	 * other nasties. Loopback packets already have the dst attached
-	 * so this only affects packets which have originated elsewhere.
-	 */
-	err  = -ENOTUNIQ;
-	if (dn_dev_islocal(in_dev, cb->src))
-		goto out;
-
-	err = dn_fib_lookup(&fld, &res);
-	if (err) {
-		if (err != -ESRCH)
-			goto out;
-		/*
-		 * Is the destination us ?
-		 */
-		if (!dn_dev_islocal(in_dev, cb->dst))
-			goto e_inval;
-
-		res.type = RTN_LOCAL;
-	} else {
-		__le16 src_map = fld.saddr;
-		free_res = 1;
-
-		out_dev = DN_FIB_RES_DEV(res);
-		if (out_dev == NULL) {
-			net_crit_ratelimited("Bug in dn_route_input_slow() No output device\n");
-			goto e_inval;
-		}
-		dev_hold(out_dev);
-
-		if (res.r)
-			src_map = fld.saddr; /* no NAT support for now */
-
-		gateway = DN_FIB_RES_GW(res);
-		if (res.type == RTN_NAT) {
-			fld.daddr = dn_fib_rules_map_destination(fld.daddr, &res);
-			dn_fib_res_put(&res);
-			free_res = 0;
-			if (dn_fib_lookup(&fld, &res))
-				goto e_inval;
-			free_res = 1;
-			if (res.type != RTN_UNICAST)
-				goto e_inval;
-			flags |= RTCF_DNAT;
-			gateway = fld.daddr;
-		}
-		fld.saddr = src_map;
-	}
-
-	switch (res.type) {
-	case RTN_UNICAST:
-		/*
-		 * Forwarding check here, we only check for forwarding
-		 * being turned off, if you want to only forward intra
-		 * area, its up to you to set the routing tables up
-		 * correctly.
-		 */
-		if (dn_db->parms.forwarding == 0)
-			goto e_inval;
-
-		if (res.fi->fib_nhs > 1 && fld.flowidn_oif == 0)
-			dn_fib_select_multipath(&fld, &res);
-
-		/*
-		 * Check for out_dev == in_dev. We use the RTCF_DOREDIRECT
-		 * flag as a hint to set the intra-ethernet bit when
-		 * forwarding. If we've got NAT in operation, we don't do
-		 * this optimisation.
-		 */
-		if (out_dev == in_dev && !(flags & RTCF_NAT))
-			flags |= RTCF_DOREDIRECT;
-
-		local_src = DN_FIB_RES_PREFSRC(res);
-		break;
-	case RTN_BLACKHOLE:
-	case RTN_UNREACHABLE:
-		break;
-	case RTN_LOCAL:
-		flags |= RTCF_LOCAL;
-		fld.saddr = cb->dst;
-		fld.daddr = cb->src;
-
-		/* Routing tables gave us a gateway */
-		if (gateway)
-			goto make_route;
-
-		/* Packet was intra-ethernet, so we know its on-link */
-		if (cb->rt_flags & DN_RT_F_IE) {
-			gateway = cb->src;
-			goto make_route;
-		}
-
-		/* Use the default router if there is one */
-		neigh = neigh_clone(dn_db->router);
-		if (neigh) {
-			gateway = container_of(neigh, struct dn_neigh, n)->addr;
-			goto make_route;
-		}
-
-		/* Close eyes and pray */
-		gateway = cb->src;
-		goto make_route;
-	default:
-		goto e_inval;
-	}
-
-make_route:
-	rt = dst_alloc(&dn_dst_ops, out_dev, 1, DST_OBSOLETE_NONE, 0);
-	if (rt == NULL)
-		goto e_nobufs;
-
-	rt->dn_next = NULL;
-	memset(&rt->fld, 0, sizeof(rt->fld));
-	rt->rt_saddr      = fld.saddr;
-	rt->rt_daddr      = fld.daddr;
-	rt->rt_gateway    = fld.daddr;
-	if (gateway)
-		rt->rt_gateway = gateway;
-	rt->rt_local_src  = local_src ? local_src : rt->rt_saddr;
-
-	rt->rt_dst_map    = fld.daddr;
-	rt->rt_src_map    = fld.saddr;
-
-	rt->fld.saddr        = cb->src;
-	rt->fld.daddr        = cb->dst;
-	rt->fld.flowidn_oif  = 0;
-	rt->fld.flowidn_iif  = in_dev->ifindex;
-	rt->fld.flowidn_mark = fld.flowidn_mark;
-
-	rt->n = neigh;
-	rt->dst.lastuse = jiffies;
-	rt->dst.output = dn_rt_bug_out;
-	switch (res.type) {
-	case RTN_UNICAST:
-		rt->dst.input = dn_forward;
-		break;
-	case RTN_LOCAL:
-		rt->dst.output = dn_output;
-		rt->dst.input = dn_nsp_rx;
-		rt->dst.dev = in_dev;
-		flags |= RTCF_LOCAL;
-		break;
-	default:
-	case RTN_UNREACHABLE:
-	case RTN_BLACKHOLE:
-		rt->dst.input = dst_discard;
-	}
-	rt->rt_flags = flags;
-
-	err = dn_rt_set_next_hop(rt, &res);
-	if (err)
-		goto e_neighbour;
-
-	hash = dn_hash(rt->fld.saddr, rt->fld.daddr);
-	/* dn_insert_route() increments dst->__refcnt */
-	dn_insert_route(rt, hash, &rt);
-	skb_dst_set(skb, &rt->dst);
-
-done:
-	if (neigh)
-		neigh_release(neigh);
-	if (free_res)
-		dn_fib_res_put(&res);
-	dev_put(in_dev);
-	dev_put(out_dev);
-out:
-	return err;
-
-e_inval:
-	err = -EINVAL;
-	goto done;
-
-e_nobufs:
-	err = -ENOBUFS;
-	goto done;
-
-e_neighbour:
-	dst_release_immediate(&rt->dst);
-	goto done;
-}
-
-static int dn_route_input(struct sk_buff *skb)
-{
-	struct dn_route *rt;
-	struct dn_skb_cb *cb = DN_SKB_CB(skb);
-	unsigned int hash = dn_hash(cb->src, cb->dst);
-
-	if (skb_dst(skb))
-		return 0;
-
-	rcu_read_lock();
-	for (rt = rcu_dereference(dn_rt_hash_table[hash].chain); rt != NULL;
-	    rt = rcu_dereference(rt->dn_next)) {
-		if ((rt->fld.saddr == cb->src) &&
-		    (rt->fld.daddr == cb->dst) &&
-		    (rt->fld.flowidn_oif == 0) &&
-		    (rt->fld.flowidn_mark == skb->mark) &&
-		    (rt->fld.flowidn_iif == cb->iif)) {
-			dst_hold_and_use(&rt->dst, jiffies);
-			rcu_read_unlock();
-			skb_dst_set(skb, (struct dst_entry *)rt);
-			return 0;
-		}
-	}
-	rcu_read_unlock();
-
-	return dn_route_input_slow(skb);
-}
-
-static int dn_rt_fill_info(struct sk_buff *skb, u32 portid, u32 seq,
-			   int event, int nowait, unsigned int flags)
-{
-	struct dn_route *rt = (struct dn_route *)skb_dst(skb);
-	struct rtmsg *r;
-	struct nlmsghdr *nlh;
-	long expires;
-
-	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*r), flags);
-	if (!nlh)
-		return -EMSGSIZE;
-
-	r = nlmsg_data(nlh);
-	r->rtm_family = AF_DECnet;
-	r->rtm_dst_len = 16;
-	r->rtm_src_len = 0;
-	r->rtm_tos = 0;
-	r->rtm_table = RT_TABLE_MAIN;
-	r->rtm_type = rt->rt_type;
-	r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
-	r->rtm_scope = RT_SCOPE_UNIVERSE;
-	r->rtm_protocol = RTPROT_UNSPEC;
-
-	if (rt->rt_flags & RTCF_NOTIFY)
-		r->rtm_flags |= RTM_F_NOTIFY;
-
-	if (nla_put_u32(skb, RTA_TABLE, RT_TABLE_MAIN) < 0 ||
-	    nla_put_le16(skb, RTA_DST, rt->rt_daddr) < 0)
-		goto errout;
-
-	if (rt->fld.saddr) {
-		r->rtm_src_len = 16;
-		if (nla_put_le16(skb, RTA_SRC, rt->fld.saddr) < 0)
-			goto errout;
-	}
-	if (rt->dst.dev &&
-	    nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex) < 0)
-		goto errout;
-
-	/*
-	 * Note to self - change this if input routes reverse direction when
-	 * they deal only with inputs and not with replies like they do
-	 * currently.
-	 */
-	if (nla_put_le16(skb, RTA_PREFSRC, rt->rt_local_src) < 0)
-		goto errout;
-
-	if (rt->rt_daddr != rt->rt_gateway &&
-	    nla_put_le16(skb, RTA_GATEWAY, rt->rt_gateway) < 0)
-		goto errout;
-
-	if (rtnetlink_put_metrics(skb, dst_metrics_ptr(&rt->dst)) < 0)
-		goto errout;
-
-	expires = rt->dst.expires ? rt->dst.expires - jiffies : 0;
-	if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires,
-			       rt->dst.error) < 0)
-		goto errout;
-
-	if (dn_is_input_route(rt) &&
-	    nla_put_u32(skb, RTA_IIF, rt->fld.flowidn_iif) < 0)
-		goto errout;
-
-	nlmsg_end(skb, nlh);
-	return 0;
-
-errout:
-	nlmsg_cancel(skb, nlh);
-	return -EMSGSIZE;
-}
-
-const struct nla_policy rtm_dn_policy[RTA_MAX + 1] = {
-	[RTA_DST]		= { .type = NLA_U16 },
-	[RTA_SRC]		= { .type = NLA_U16 },
-	[RTA_IIF]		= { .type = NLA_U32 },
-	[RTA_OIF]		= { .type = NLA_U32 },
-	[RTA_GATEWAY]		= { .type = NLA_U16 },
-	[RTA_PRIORITY]		= { .type = NLA_U32 },
-	[RTA_PREFSRC]		= { .type = NLA_U16 },
-	[RTA_METRICS]		= { .type = NLA_NESTED },
-	[RTA_MULTIPATH]		= { .type = NLA_NESTED },
-	[RTA_TABLE]		= { .type = NLA_U32 },
-	[RTA_MARK]		= { .type = NLA_U32 },
-};
-
-/*
- * This is called by both endnodes and routers now.
- */
-static int dn_cache_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
-			     struct netlink_ext_ack *extack)
-{
-	struct net *net = sock_net(in_skb->sk);
-	struct rtmsg *rtm = nlmsg_data(nlh);
-	struct dn_route *rt = NULL;
-	struct dn_skb_cb *cb;
-	int err;
-	struct sk_buff *skb;
-	struct flowidn fld;
-	struct nlattr *tb[RTA_MAX+1];
-
-	if (!net_eq(net, &init_net))
-		return -EINVAL;
-
-	err = nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
-				     rtm_dn_policy, extack);
-	if (err < 0)
-		return err;
-
-	memset(&fld, 0, sizeof(fld));
-	fld.flowidn_proto = DNPROTO_NSP;
-
-	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
-	if (skb == NULL)
-		return -ENOBUFS;
-	skb_reset_mac_header(skb);
-	cb = DN_SKB_CB(skb);
-
-	if (tb[RTA_SRC])
-		fld.saddr = nla_get_le16(tb[RTA_SRC]);
-
-	if (tb[RTA_DST])
-		fld.daddr = nla_get_le16(tb[RTA_DST]);
-
-	if (tb[RTA_IIF])
-		fld.flowidn_iif = nla_get_u32(tb[RTA_IIF]);
-
-	if (fld.flowidn_iif) {
-		struct net_device *dev;
-		dev = __dev_get_by_index(&init_net, fld.flowidn_iif);
-		if (!dev || !dev->dn_ptr) {
-			kfree_skb(skb);
-			return -ENODEV;
-		}
-		skb->protocol = htons(ETH_P_DNA_RT);
-		skb->dev = dev;
-		cb->src = fld.saddr;
-		cb->dst = fld.daddr;
-		local_bh_disable();
-		err = dn_route_input(skb);
-		local_bh_enable();
-		memset(cb, 0, sizeof(struct dn_skb_cb));
-		rt = (struct dn_route *)skb_dst(skb);
-		if (!err && -rt->dst.error)
-			err = rt->dst.error;
-	} else {
-		if (tb[RTA_OIF])
-			fld.flowidn_oif = nla_get_u32(tb[RTA_OIF]);
-
-		err = dn_route_output_key((struct dst_entry **)&rt, &fld, 0);
-	}
-
-	skb->dev = NULL;
-	if (err)
-		goto out_free;
-	skb_dst_set(skb, &rt->dst);
-	if (rtm->rtm_flags & RTM_F_NOTIFY)
-		rt->rt_flags |= RTCF_NOTIFY;
-
-	err = dn_rt_fill_info(skb, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq, RTM_NEWROUTE, 0, 0);
-	if (err < 0) {
-		err = -EMSGSIZE;
-		goto out_free;
-	}
-
-	return rtnl_unicast(skb, &init_net, NETLINK_CB(in_skb).portid);
-
-out_free:
-	kfree_skb(skb);
-	return err;
-}
-
-/*
- * For routers, this is called from dn_fib_dump, but for endnodes its
- * called directly from the rtnetlink dispatch table.
- */
-int dn_cache_dump(struct sk_buff *skb, struct netlink_callback *cb)
-{
-	struct net *net = sock_net(skb->sk);
-	struct dn_route *rt;
-	int h, s_h;
-	int idx, s_idx;
-	struct rtmsg *rtm;
-
-	if (!net_eq(net, &init_net))
-		return 0;
-
-	if (nlmsg_len(cb->nlh) < sizeof(struct rtmsg))
-		return -EINVAL;
-
-	rtm = nlmsg_data(cb->nlh);
-	if (!(rtm->rtm_flags & RTM_F_CLONED))
-		return 0;
-
-	s_h = cb->args[0];
-	s_idx = idx = cb->args[1];
-	for (h = 0; h <= dn_rt_hash_mask; h++) {
-		if (h < s_h)
-			continue;
-		if (h > s_h)
-			s_idx = 0;
-		rcu_read_lock_bh();
-		for (rt = rcu_dereference_bh(dn_rt_hash_table[h].chain), idx = 0;
-			rt;
-			rt = rcu_dereference_bh(rt->dn_next), idx++) {
-			if (idx < s_idx)
-				continue;
-			skb_dst_set(skb, dst_clone(&rt->dst));
-			if (dn_rt_fill_info(skb, NETLINK_CB(cb->skb).portid,
-					cb->nlh->nlmsg_seq, RTM_NEWROUTE,
-					1, NLM_F_MULTI) < 0) {
-				skb_dst_drop(skb);
-				rcu_read_unlock_bh();
-				goto done;
-			}
-			skb_dst_drop(skb);
-		}
-		rcu_read_unlock_bh();
-	}
-
-done:
-	cb->args[0] = h;
-	cb->args[1] = idx;
-	return skb->len;
-}
-
-#ifdef CONFIG_PROC_FS
-struct dn_rt_cache_iter_state {
-	int bucket;
-};
-
-static struct dn_route *dn_rt_cache_get_first(struct seq_file *seq)
-{
-	struct dn_route *rt = NULL;
-	struct dn_rt_cache_iter_state *s = seq->private;
-
-	for (s->bucket = dn_rt_hash_mask; s->bucket >= 0; --s->bucket) {
-		rcu_read_lock_bh();
-		rt = rcu_dereference_bh(dn_rt_hash_table[s->bucket].chain);
-		if (rt)
-			break;
-		rcu_read_unlock_bh();
-	}
-	return rt;
-}
-
-static struct dn_route *dn_rt_cache_get_next(struct seq_file *seq, struct dn_route *rt)
-{
-	struct dn_rt_cache_iter_state *s = seq->private;
-
-	rt = rcu_dereference_bh(rt->dn_next);
-	while (!rt) {
-		rcu_read_unlock_bh();
-		if (--s->bucket < 0)
-			break;
-		rcu_read_lock_bh();
-		rt = rcu_dereference_bh(dn_rt_hash_table[s->bucket].chain);
-	}
-	return rt;
-}
-
-static void *dn_rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
-{
-	struct dn_route *rt = dn_rt_cache_get_first(seq);
-
-	if (rt) {
-		while (*pos && (rt = dn_rt_cache_get_next(seq, rt)))
-			--*pos;
-	}
-	return *pos ? NULL : rt;
-}
-
-static void *dn_rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
-{
-	struct dn_route *rt = dn_rt_cache_get_next(seq, v);
-	++*pos;
-	return rt;
-}
-
-static void dn_rt_cache_seq_stop(struct seq_file *seq, void *v)
-{
-	if (v)
-		rcu_read_unlock_bh();
-}
-
-static int dn_rt_cache_seq_show(struct seq_file *seq, void *v)
-{
-	struct dn_route *rt = v;
-	char buf1[DN_ASCBUF_LEN], buf2[DN_ASCBUF_LEN];
-
-	seq_printf(seq, "%-8s %-7s %-7s %04d %04d %04d\n",
-		   rt->dst.dev ? rt->dst.dev->name : "*",
-		   dn_addr2asc(le16_to_cpu(rt->rt_daddr), buf1),
-		   dn_addr2asc(le16_to_cpu(rt->rt_saddr), buf2),
-		   atomic_read(&rt->dst.__refcnt),
-		   rt->dst.__use, 0);
-	return 0;
-}
-
-static const struct seq_operations dn_rt_cache_seq_ops = {
-	.start	= dn_rt_cache_seq_start,
-	.next	= dn_rt_cache_seq_next,
-	.stop	= dn_rt_cache_seq_stop,
-	.show	= dn_rt_cache_seq_show,
-};
-#endif /* CONFIG_PROC_FS */
-
-void __init dn_route_init(void)
-{
-	int i, goal, order;
-
-	dn_dst_ops.kmem_cachep =
-		kmem_cache_create("dn_dst_cache", sizeof(struct dn_route), 0,
-				  SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
-	dst_entries_init(&dn_dst_ops);
-	timer_setup(&dn_route_timer, dn_dst_check_expire, 0);
-	dn_route_timer.expires = jiffies + decnet_dst_gc_interval * HZ;
-	add_timer(&dn_route_timer);
-
-	goal = totalram_pages() >> (26 - PAGE_SHIFT);
-
-	for (order = 0; (1UL << order) < goal; order++)
-		/* NOTHING */;
-
-	/*
-	 * Only want 1024 entries max, since the table is very, very unlikely
-	 * to be larger than that.
-	 */
-	while (order && ((((1UL << order) * PAGE_SIZE) /
-				sizeof(struct dn_rt_hash_bucket)) >= 2048))
-		order--;
-
-	do {
-		dn_rt_hash_mask = (1UL << order) * PAGE_SIZE /
-			sizeof(struct dn_rt_hash_bucket);
-		while (dn_rt_hash_mask & (dn_rt_hash_mask - 1))
-			dn_rt_hash_mask--;
-		dn_rt_hash_table = (struct dn_rt_hash_bucket *)
-			__get_free_pages(GFP_ATOMIC, order);
-	} while (dn_rt_hash_table == NULL && --order > 0);
-
-	if (!dn_rt_hash_table)
-		panic("Failed to allocate DECnet route cache hash table\n");
-
-	printk(KERN_INFO
-		"DECnet: Routing cache hash table of %u buckets, %ldKbytes\n",
-		dn_rt_hash_mask,
-		(long)(dn_rt_hash_mask*sizeof(struct dn_rt_hash_bucket))/1024);
-
-	dn_rt_hash_mask--;
-	for (i = 0; i <= dn_rt_hash_mask; i++) {
-		spin_lock_init(&dn_rt_hash_table[i].lock);
-		dn_rt_hash_table[i].chain = NULL;
-	}
-
-	dn_dst_ops.gc_thresh = (dn_rt_hash_mask + 1);
-
-	proc_create_seq_private("decnet_cache", 0444, init_net.proc_net,
-			&dn_rt_cache_seq_ops,
-			sizeof(struct dn_rt_cache_iter_state), NULL);
-
-#ifdef CONFIG_DECNET_ROUTER
-	rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_GETROUTE,
-			     dn_cache_getroute, dn_fib_dump, 0);
-#else
-	rtnl_register_module(THIS_MODULE, PF_DECnet, RTM_GETROUTE,
-			     dn_cache_getroute, dn_cache_dump, 0);
-#endif
-}
-
-void __exit dn_route_cleanup(void)
-{
-	del_timer(&dn_route_timer);
-	dn_run_flush(NULL);
-
-	remove_proc_entry("decnet_cache", init_net.proc_net);
-	dst_entries_destroy(&dn_dst_ops);
-}
diff --git a/net/decnet/dn_rules.c b/net/decnet/dn_rules.c
deleted file mode 100644
index ee73057529cf..000000000000
--- a/net/decnet/dn_rules.c
+++ /dev/null
@@ -1,253 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Routing Forwarding Information Base (Rules)
- *
- * Author:      Steve Whitehouse <SteveW@ACM.org>
- *              Mostly copied from Alexey Kuznetsov's ipv4/fib_rules.c
- *
- *
- * Changes:
- *              Steve Whitehouse <steve@chygwyn.com>
- *              Updated for Thomas Graf's generic rules
- *
- */
-#include <linux/net.h>
-#include <linux/init.h>
-#include <linux/netlink.h>
-#include <linux/rtnetlink.h>
-#include <linux/netdevice.h>
-#include <linux/spinlock.h>
-#include <linux/list.h>
-#include <linux/rcupdate.h>
-#include <linux/export.h>
-#include <net/neighbour.h>
-#include <net/dst.h>
-#include <net/flow.h>
-#include <net/fib_rules.h>
-#include <net/dn.h>
-#include <net/dn_fib.h>
-#include <net/dn_neigh.h>
-#include <net/dn_dev.h>
-#include <net/dn_route.h>
-
-static struct fib_rules_ops *dn_fib_rules_ops;
-
-struct dn_fib_rule
-{
-	struct fib_rule		common;
-	unsigned char		dst_len;
-	unsigned char		src_len;
-	__le16			src;
-	__le16			srcmask;
-	__le16			dst;
-	__le16			dstmask;
-	__le16			srcmap;
-	u8			flags;
-};
-
-
-int dn_fib_lookup(struct flowidn *flp, struct dn_fib_res *res)
-{
-	struct fib_lookup_arg arg = {
-		.result = res,
-	};
-	int err;
-
-	err = fib_rules_lookup(dn_fib_rules_ops,
-			       flowidn_to_flowi(flp), 0, &arg);
-	res->r = arg.rule;
-
-	return err;
-}
-
-static int dn_fib_rule_action(struct fib_rule *rule, struct flowi *flp,
-			      int flags, struct fib_lookup_arg *arg)
-{
-	struct flowidn *fld = &flp->u.dn;
-	int err = -EAGAIN;
-	struct dn_fib_table *tbl;
-
-	switch(rule->action) {
-	case FR_ACT_TO_TBL:
-		break;
-
-	case FR_ACT_UNREACHABLE:
-		err = -ENETUNREACH;
-		goto errout;
-
-	case FR_ACT_PROHIBIT:
-		err = -EACCES;
-		goto errout;
-
-	case FR_ACT_BLACKHOLE:
-	default:
-		err = -EINVAL;
-		goto errout;
-	}
-
-	tbl = dn_fib_get_table(rule->table, 0);
-	if (tbl == NULL)
-		goto errout;
-
-	err = tbl->lookup(tbl, fld, (struct dn_fib_res *)arg->result);
-	if (err > 0)
-		err = -EAGAIN;
-errout:
-	return err;
-}
-
-static int dn_fib_rule_match(struct fib_rule *rule, struct flowi *fl, int flags)
-{
-	struct dn_fib_rule *r = (struct dn_fib_rule *)rule;
-	struct flowidn *fld = &fl->u.dn;
-	__le16 daddr = fld->daddr;
-	__le16 saddr = fld->saddr;
-
-	if (((saddr ^ r->src) & r->srcmask) ||
-	    ((daddr ^ r->dst) & r->dstmask))
-		return 0;
-
-	return 1;
-}
-
-static int dn_fib_rule_configure(struct fib_rule *rule, struct sk_buff *skb,
-				 struct fib_rule_hdr *frh,
-				 struct nlattr **tb,
-				 struct netlink_ext_ack *extack)
-{
-	int err = -EINVAL;
-	struct dn_fib_rule *r = (struct dn_fib_rule *)rule;
-
-	if (frh->tos) {
-		NL_SET_ERR_MSG(extack, "Invalid tos value");
-		goto  errout;
-	}
-
-	if (rule->table == RT_TABLE_UNSPEC) {
-		if (rule->action == FR_ACT_TO_TBL) {
-			struct dn_fib_table *table;
-
-			table = dn_fib_empty_table();
-			if (table == NULL) {
-				err = -ENOBUFS;
-				goto errout;
-			}
-
-			rule->table = table->n;
-		}
-	}
-
-	if (frh->src_len)
-		r->src = nla_get_le16(tb[FRA_SRC]);
-
-	if (frh->dst_len)
-		r->dst = nla_get_le16(tb[FRA_DST]);
-
-	r->src_len = frh->src_len;
-	r->srcmask = dnet_make_mask(r->src_len);
-	r->dst_len = frh->dst_len;
-	r->dstmask = dnet_make_mask(r->dst_len);
-	err = 0;
-errout:
-	return err;
-}
-
-static int dn_fib_rule_compare(struct fib_rule *rule, struct fib_rule_hdr *frh,
-			       struct nlattr **tb)
-{
-	struct dn_fib_rule *r = (struct dn_fib_rule *)rule;
-
-	if (frh->src_len && (r->src_len != frh->src_len))
-		return 0;
-
-	if (frh->dst_len && (r->dst_len != frh->dst_len))
-		return 0;
-
-	if (frh->src_len && (r->src != nla_get_le16(tb[FRA_SRC])))
-		return 0;
-
-	if (frh->dst_len && (r->dst != nla_get_le16(tb[FRA_DST])))
-		return 0;
-
-	return 1;
-}
-
-unsigned int dnet_addr_type(__le16 addr)
-{
-	struct flowidn fld = { .daddr = addr };
-	struct dn_fib_res res;
-	unsigned int ret = RTN_UNICAST;
-	struct dn_fib_table *tb = dn_fib_get_table(RT_TABLE_LOCAL, 0);
-
-	res.r = NULL;
-
-	if (tb) {
-		if (!tb->lookup(tb, &fld, &res)) {
-			ret = res.type;
-			dn_fib_res_put(&res);
-		}
-	}
-	return ret;
-}
-
-static int dn_fib_rule_fill(struct fib_rule *rule, struct sk_buff *skb,
-			    struct fib_rule_hdr *frh)
-{
-	struct dn_fib_rule *r = (struct dn_fib_rule *)rule;
-
-	frh->dst_len = r->dst_len;
-	frh->src_len = r->src_len;
-	frh->tos = 0;
-
-	if ((r->dst_len &&
-	     nla_put_le16(skb, FRA_DST, r->dst)) ||
-	    (r->src_len &&
-	     nla_put_le16(skb, FRA_SRC, r->src)))
-		goto nla_put_failure;
-	return 0;
-
-nla_put_failure:
-	return -ENOBUFS;
-}
-
-static void dn_fib_rule_flush_cache(struct fib_rules_ops *ops)
-{
-	dn_rt_cache_flush(-1);
-}
-
-static const struct fib_rules_ops __net_initconst dn_fib_rules_ops_template = {
-	.family		= AF_DECnet,
-	.rule_size	= sizeof(struct dn_fib_rule),
-	.addr_size	= sizeof(u16),
-	.action		= dn_fib_rule_action,
-	.match		= dn_fib_rule_match,
-	.configure	= dn_fib_rule_configure,
-	.compare	= dn_fib_rule_compare,
-	.fill		= dn_fib_rule_fill,
-	.flush_cache	= dn_fib_rule_flush_cache,
-	.nlgroup	= RTNLGRP_DECnet_RULE,
-	.owner		= THIS_MODULE,
-	.fro_net	= &init_net,
-};
-
-void __init dn_fib_rules_init(void)
-{
-	dn_fib_rules_ops =
-		fib_rules_register(&dn_fib_rules_ops_template, &init_net);
-	BUG_ON(IS_ERR(dn_fib_rules_ops));
-	BUG_ON(fib_default_rule_add(dn_fib_rules_ops, 0x7fff,
-			            RT_TABLE_MAIN, 0));
-}
-
-void __exit dn_fib_rules_cleanup(void)
-{
-	rtnl_lock();
-	fib_rules_unregister(dn_fib_rules_ops);
-	rtnl_unlock();
-	rcu_barrier();
-}
diff --git a/net/decnet/dn_table.c b/net/decnet/dn_table.c
deleted file mode 100644
index 4086f9c746af..000000000000
--- a/net/decnet/dn_table.c
+++ /dev/null
@@ -1,929 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Routing Forwarding Information Base (Routing Tables)
- *
- * Author:      Steve Whitehouse <SteveW@ACM.org>
- *              Mostly copied from the IPv4 routing code
- *
- *
- * Changes:
- *
- */
-#include <linux/string.h>
-#include <linux/net.h>
-#include <linux/socket.h>
-#include <linux/slab.h>
-#include <linux/sockios.h>
-#include <linux/init.h>
-#include <linux/skbuff.h>
-#include <linux/rtnetlink.h>
-#include <linux/proc_fs.h>
-#include <linux/netdevice.h>
-#include <linux/timer.h>
-#include <linux/spinlock.h>
-#include <linux/atomic.h>
-#include <linux/uaccess.h>
-#include <linux/route.h> /* RTF_xxx */
-#include <net/neighbour.h>
-#include <net/netlink.h>
-#include <net/tcp.h>
-#include <net/dst.h>
-#include <net/flow.h>
-#include <net/fib_rules.h>
-#include <net/dn.h>
-#include <net/dn_route.h>
-#include <net/dn_fib.h>
-#include <net/dn_neigh.h>
-#include <net/dn_dev.h>
-
-struct dn_zone
-{
-	struct dn_zone		*dz_next;
-	struct dn_fib_node 	**dz_hash;
-	int			dz_nent;
-	int			dz_divisor;
-	u32			dz_hashmask;
-#define DZ_HASHMASK(dz)	((dz)->dz_hashmask)
-	int			dz_order;
-	__le16			dz_mask;
-#define DZ_MASK(dz)	((dz)->dz_mask)
-};
-
-struct dn_hash
-{
-	struct dn_zone	*dh_zones[17];
-	struct dn_zone	*dh_zone_list;
-};
-
-#define dz_key_0(key)		((key).datum = 0)
-
-#define for_nexthops(fi) { int nhsel; const struct dn_fib_nh *nh;\
-	for(nhsel = 0, nh = (fi)->fib_nh; nhsel < (fi)->fib_nhs; nh++, nhsel++)
-
-#define endfor_nexthops(fi) }
-
-#define DN_MAX_DIVISOR 1024
-#define DN_S_ZOMBIE 1
-#define DN_S_ACCESSED 2
-
-#define DN_FIB_SCAN(f, fp) \
-for( ; ((f) = *(fp)) != NULL; (fp) = &(f)->fn_next)
-
-#define DN_FIB_SCAN_KEY(f, fp, key) \
-for( ; ((f) = *(fp)) != NULL && dn_key_eq((f)->fn_key, (key)); (fp) = &(f)->fn_next)
-
-#define RT_TABLE_MIN 1
-#define DN_FIB_TABLE_HASHSZ 256
-static struct hlist_head dn_fib_table_hash[DN_FIB_TABLE_HASHSZ];
-static DEFINE_RWLOCK(dn_fib_tables_lock);
-
-static struct kmem_cache *dn_hash_kmem __read_mostly;
-static int dn_fib_hash_zombies;
-
-static inline dn_fib_idx_t dn_hash(dn_fib_key_t key, struct dn_zone *dz)
-{
-	u16 h = le16_to_cpu(key.datum)>>(16 - dz->dz_order);
-	h ^= (h >> 10);
-	h ^= (h >> 6);
-	h &= DZ_HASHMASK(dz);
-	return *(dn_fib_idx_t *)&h;
-}
-
-static inline dn_fib_key_t dz_key(__le16 dst, struct dn_zone *dz)
-{
-	dn_fib_key_t k;
-	k.datum = dst & DZ_MASK(dz);
-	return k;
-}
-
-static inline struct dn_fib_node **dn_chain_p(dn_fib_key_t key, struct dn_zone *dz)
-{
-	return &dz->dz_hash[dn_hash(key, dz).datum];
-}
-
-static inline struct dn_fib_node *dz_chain(dn_fib_key_t key, struct dn_zone *dz)
-{
-	return dz->dz_hash[dn_hash(key, dz).datum];
-}
-
-static inline int dn_key_eq(dn_fib_key_t a, dn_fib_key_t b)
-{
-	return a.datum == b.datum;
-}
-
-static inline int dn_key_leq(dn_fib_key_t a, dn_fib_key_t b)
-{
-	return a.datum <= b.datum;
-}
-
-static inline void dn_rebuild_zone(struct dn_zone *dz,
-				   struct dn_fib_node **old_ht,
-				   int old_divisor)
-{
-	struct dn_fib_node *f, **fp, *next;
-	int i;
-
-	for(i = 0; i < old_divisor; i++) {
-		for(f = old_ht[i]; f; f = next) {
-			next = f->fn_next;
-			for(fp = dn_chain_p(f->fn_key, dz);
-				*fp && dn_key_leq((*fp)->fn_key, f->fn_key);
-				fp = &(*fp)->fn_next)
-				/* NOTHING */;
-			f->fn_next = *fp;
-			*fp = f;
-		}
-	}
-}
-
-static void dn_rehash_zone(struct dn_zone *dz)
-{
-	struct dn_fib_node **ht, **old_ht;
-	int old_divisor, new_divisor;
-	u32 new_hashmask;
-
-	old_divisor = dz->dz_divisor;
-
-	switch (old_divisor) {
-	case 16:
-		new_divisor = 256;
-		new_hashmask = 0xFF;
-		break;
-	default:
-		printk(KERN_DEBUG "DECnet: dn_rehash_zone: BUG! %d\n",
-		       old_divisor);
-		fallthrough;
-	case 256:
-		new_divisor = 1024;
-		new_hashmask = 0x3FF;
-		break;
-	}
-
-	ht = kcalloc(new_divisor, sizeof(struct dn_fib_node*), GFP_KERNEL);
-	if (ht == NULL)
-		return;
-
-	write_lock_bh(&dn_fib_tables_lock);
-	old_ht = dz->dz_hash;
-	dz->dz_hash = ht;
-	dz->dz_hashmask = new_hashmask;
-	dz->dz_divisor = new_divisor;
-	dn_rebuild_zone(dz, old_ht, old_divisor);
-	write_unlock_bh(&dn_fib_tables_lock);
-	kfree(old_ht);
-}
-
-static void dn_free_node(struct dn_fib_node *f)
-{
-	dn_fib_release_info(DN_FIB_INFO(f));
-	kmem_cache_free(dn_hash_kmem, f);
-}
-
-
-static struct dn_zone *dn_new_zone(struct dn_hash *table, int z)
-{
-	int i;
-	struct dn_zone *dz = kzalloc(sizeof(struct dn_zone), GFP_KERNEL);
-	if (!dz)
-		return NULL;
-
-	if (z) {
-		dz->dz_divisor = 16;
-		dz->dz_hashmask = 0x0F;
-	} else {
-		dz->dz_divisor = 1;
-		dz->dz_hashmask = 0;
-	}
-
-	dz->dz_hash = kcalloc(dz->dz_divisor, sizeof(struct dn_fib_node *), GFP_KERNEL);
-	if (!dz->dz_hash) {
-		kfree(dz);
-		return NULL;
-	}
-
-	dz->dz_order = z;
-	dz->dz_mask = dnet_make_mask(z);
-
-	for(i = z + 1; i <= 16; i++)
-		if (table->dh_zones[i])
-			break;
-
-	write_lock_bh(&dn_fib_tables_lock);
-	if (i>16) {
-		dz->dz_next = table->dh_zone_list;
-		table->dh_zone_list = dz;
-	} else {
-		dz->dz_next = table->dh_zones[i]->dz_next;
-		table->dh_zones[i]->dz_next = dz;
-	}
-	table->dh_zones[z] = dz;
-	write_unlock_bh(&dn_fib_tables_lock);
-	return dz;
-}
-
-
-static int dn_fib_nh_match(struct rtmsg *r, struct nlmsghdr *nlh, struct nlattr *attrs[], struct dn_fib_info *fi)
-{
-	struct rtnexthop *nhp;
-	int nhlen;
-
-	if (attrs[RTA_PRIORITY] &&
-	    nla_get_u32(attrs[RTA_PRIORITY]) != fi->fib_priority)
-		return 1;
-
-	if (attrs[RTA_OIF] || attrs[RTA_GATEWAY]) {
-		if ((!attrs[RTA_OIF] || nla_get_u32(attrs[RTA_OIF]) == fi->fib_nh->nh_oif) &&
-		    (!attrs[RTA_GATEWAY]  || nla_get_le16(attrs[RTA_GATEWAY]) != fi->fib_nh->nh_gw))
-			return 0;
-		return 1;
-	}
-
-	if (!attrs[RTA_MULTIPATH])
-		return 0;
-
-	nhp = nla_data(attrs[RTA_MULTIPATH]);
-	nhlen = nla_len(attrs[RTA_MULTIPATH]);
-
-	for_nexthops(fi) {
-		int attrlen = nhlen - sizeof(struct rtnexthop);
-		__le16 gw;
-
-		if (attrlen < 0 || (nhlen -= nhp->rtnh_len) < 0)
-			return -EINVAL;
-		if (nhp->rtnh_ifindex && nhp->rtnh_ifindex != nh->nh_oif)
-			return 1;
-		if (attrlen) {
-			struct nlattr *gw_attr;
-
-			gw_attr = nla_find((struct nlattr *) (nhp + 1), attrlen, RTA_GATEWAY);
-			gw = gw_attr ? nla_get_le16(gw_attr) : 0;
-
-			if (gw && gw != nh->nh_gw)
-				return 1;
-		}
-		nhp = RTNH_NEXT(nhp);
-	} endfor_nexthops(fi);
-
-	return 0;
-}
-
-static inline size_t dn_fib_nlmsg_size(struct dn_fib_info *fi)
-{
-	size_t payload = NLMSG_ALIGN(sizeof(struct rtmsg))
-			 + nla_total_size(4) /* RTA_TABLE */
-			 + nla_total_size(2) /* RTA_DST */
-			 + nla_total_size(4) /* RTA_PRIORITY */
-			 + nla_total_size(TCP_CA_NAME_MAX); /* RTAX_CC_ALGO */
-
-	/* space for nested metrics */
-	payload += nla_total_size((RTAX_MAX * nla_total_size(4)));
-
-	if (fi->fib_nhs) {
-		/* Also handles the special case fib_nhs == 1 */
-
-		/* each nexthop is packed in an attribute */
-		size_t nhsize = nla_total_size(sizeof(struct rtnexthop));
-
-		/* may contain a gateway attribute */
-		nhsize += nla_total_size(4);
-
-		/* all nexthops are packed in a nested attribute */
-		payload += nla_total_size(fi->fib_nhs * nhsize);
-	}
-
-	return payload;
-}
-
-static int dn_fib_dump_info(struct sk_buff *skb, u32 portid, u32 seq, int event,
-			u32 tb_id, u8 type, u8 scope, void *dst, int dst_len,
-			struct dn_fib_info *fi, unsigned int flags)
-{
-	struct rtmsg *rtm;
-	struct nlmsghdr *nlh;
-
-	nlh = nlmsg_put(skb, portid, seq, event, sizeof(*rtm), flags);
-	if (!nlh)
-		return -EMSGSIZE;
-
-	rtm = nlmsg_data(nlh);
-	rtm->rtm_family = AF_DECnet;
-	rtm->rtm_dst_len = dst_len;
-	rtm->rtm_src_len = 0;
-	rtm->rtm_tos = 0;
-	rtm->rtm_table = tb_id;
-	rtm->rtm_flags = fi->fib_flags;
-	rtm->rtm_scope = scope;
-	rtm->rtm_type  = type;
-	rtm->rtm_protocol = fi->fib_protocol;
-
-	if (nla_put_u32(skb, RTA_TABLE, tb_id) < 0)
-		goto errout;
-
-	if (rtm->rtm_dst_len &&
-	    nla_put(skb, RTA_DST, 2, dst) < 0)
-		goto errout;
-
-	if (fi->fib_priority &&
-	    nla_put_u32(skb, RTA_PRIORITY, fi->fib_priority) < 0)
-		goto errout;
-
-	if (rtnetlink_put_metrics(skb, fi->fib_metrics) < 0)
-		goto errout;
-
-	if (fi->fib_nhs == 1) {
-		if (fi->fib_nh->nh_gw &&
-		    nla_put_le16(skb, RTA_GATEWAY, fi->fib_nh->nh_gw) < 0)
-			goto errout;
-
-		if (fi->fib_nh->nh_oif &&
-		    nla_put_u32(skb, RTA_OIF, fi->fib_nh->nh_oif) < 0)
-			goto errout;
-	}
-
-	if (fi->fib_nhs > 1) {
-		struct rtnexthop *nhp;
-		struct nlattr *mp_head;
-
-		mp_head = nla_nest_start_noflag(skb, RTA_MULTIPATH);
-		if (!mp_head)
-			goto errout;
-
-		for_nexthops(fi) {
-			if (!(nhp = nla_reserve_nohdr(skb, sizeof(*nhp))))
-				goto errout;
-
-			nhp->rtnh_flags = nh->nh_flags & 0xFF;
-			nhp->rtnh_hops = nh->nh_weight - 1;
-			nhp->rtnh_ifindex = nh->nh_oif;
-
-			if (nh->nh_gw &&
-			    nla_put_le16(skb, RTA_GATEWAY, nh->nh_gw) < 0)
-				goto errout;
-
-			nhp->rtnh_len = skb_tail_pointer(skb) - (unsigned char *)nhp;
-		} endfor_nexthops(fi);
-
-		nla_nest_end(skb, mp_head);
-	}
-
-	nlmsg_end(skb, nlh);
-	return 0;
-
-errout:
-	nlmsg_cancel(skb, nlh);
-	return -EMSGSIZE;
-}
-
-
-static void dn_rtmsg_fib(int event, struct dn_fib_node *f, int z, u32 tb_id,
-			struct nlmsghdr *nlh, struct netlink_skb_parms *req)
-{
-	struct sk_buff *skb;
-	u32 portid = req ? req->portid : 0;
-	int err = -ENOBUFS;
-
-	skb = nlmsg_new(dn_fib_nlmsg_size(DN_FIB_INFO(f)), GFP_KERNEL);
-	if (skb == NULL)
-		goto errout;
-
-	err = dn_fib_dump_info(skb, portid, nlh->nlmsg_seq, event, tb_id,
-			       f->fn_type, f->fn_scope, &f->fn_key, z,
-			       DN_FIB_INFO(f), 0);
-	if (err < 0) {
-		/* -EMSGSIZE implies BUG in dn_fib_nlmsg_size() */
-		WARN_ON(err == -EMSGSIZE);
-		kfree_skb(skb);
-		goto errout;
-	}
-	rtnl_notify(skb, &init_net, portid, RTNLGRP_DECnet_ROUTE, nlh, GFP_KERNEL);
-	return;
-errout:
-	if (err < 0)
-		rtnl_set_sk_err(&init_net, RTNLGRP_DECnet_ROUTE, err);
-}
-
-static __inline__ int dn_hash_dump_bucket(struct sk_buff *skb,
-				struct netlink_callback *cb,
-				struct dn_fib_table *tb,
-				struct dn_zone *dz,
-				struct dn_fib_node *f)
-{
-	int i, s_i;
-
-	s_i = cb->args[4];
-	for(i = 0; f; i++, f = f->fn_next) {
-		if (i < s_i)
-			continue;
-		if (f->fn_state & DN_S_ZOMBIE)
-			continue;
-		if (dn_fib_dump_info(skb, NETLINK_CB(cb->skb).portid,
-				cb->nlh->nlmsg_seq,
-				RTM_NEWROUTE,
-				tb->n,
-				(f->fn_state & DN_S_ZOMBIE) ? 0 : f->fn_type,
-				f->fn_scope, &f->fn_key, dz->dz_order,
-				f->fn_info, NLM_F_MULTI) < 0) {
-			cb->args[4] = i;
-			return -1;
-		}
-	}
-	cb->args[4] = i;
-	return skb->len;
-}
-
-static __inline__ int dn_hash_dump_zone(struct sk_buff *skb,
-				struct netlink_callback *cb,
-				struct dn_fib_table *tb,
-				struct dn_zone *dz)
-{
-	int h, s_h;
-
-	s_h = cb->args[3];
-	for(h = 0; h < dz->dz_divisor; h++) {
-		if (h < s_h)
-			continue;
-		if (h > s_h)
-			memset(&cb->args[4], 0, sizeof(cb->args) - 4*sizeof(cb->args[0]));
-		if (dz->dz_hash == NULL || dz->dz_hash[h] == NULL)
-			continue;
-		if (dn_hash_dump_bucket(skb, cb, tb, dz, dz->dz_hash[h]) < 0) {
-			cb->args[3] = h;
-			return -1;
-		}
-	}
-	cb->args[3] = h;
-	return skb->len;
-}
-
-static int dn_fib_table_dump(struct dn_fib_table *tb, struct sk_buff *skb,
-				struct netlink_callback *cb)
-{
-	int m, s_m;
-	struct dn_zone *dz;
-	struct dn_hash *table = (struct dn_hash *)tb->data;
-
-	s_m = cb->args[2];
-	read_lock(&dn_fib_tables_lock);
-	for(dz = table->dh_zone_list, m = 0; dz; dz = dz->dz_next, m++) {
-		if (m < s_m)
-			continue;
-		if (m > s_m)
-			memset(&cb->args[3], 0, sizeof(cb->args) - 3*sizeof(cb->args[0]));
-
-		if (dn_hash_dump_zone(skb, cb, tb, dz) < 0) {
-			cb->args[2] = m;
-			read_unlock(&dn_fib_tables_lock);
-			return -1;
-		}
-	}
-	read_unlock(&dn_fib_tables_lock);
-	cb->args[2] = m;
-
-	return skb->len;
-}
-
-int dn_fib_dump(struct sk_buff *skb, struct netlink_callback *cb)
-{
-	struct net *net = sock_net(skb->sk);
-	unsigned int h, s_h;
-	unsigned int e = 0, s_e;
-	struct dn_fib_table *tb;
-	int dumped = 0;
-
-	if (!net_eq(net, &init_net))
-		return 0;
-
-	if (nlmsg_len(cb->nlh) >= sizeof(struct rtmsg) &&
-		((struct rtmsg *)nlmsg_data(cb->nlh))->rtm_flags&RTM_F_CLONED)
-			return dn_cache_dump(skb, cb);
-
-	s_h = cb->args[0];
-	s_e = cb->args[1];
-
-	for (h = s_h; h < DN_FIB_TABLE_HASHSZ; h++, s_h = 0) {
-		e = 0;
-		hlist_for_each_entry(tb, &dn_fib_table_hash[h], hlist) {
-			if (e < s_e)
-				goto next;
-			if (dumped)
-				memset(&cb->args[2], 0, sizeof(cb->args) -
-						 2 * sizeof(cb->args[0]));
-			if (tb->dump(tb, skb, cb) < 0)
-				goto out;
-			dumped = 1;
-next:
-			e++;
-		}
-	}
-out:
-	cb->args[1] = e;
-	cb->args[0] = h;
-
-	return skb->len;
-}
-
-static int dn_fib_table_insert(struct dn_fib_table *tb, struct rtmsg *r, struct nlattr *attrs[],
-			       struct nlmsghdr *n, struct netlink_skb_parms *req)
-{
-	struct dn_hash *table = (struct dn_hash *)tb->data;
-	struct dn_fib_node *new_f, *f, **fp, **del_fp;
-	struct dn_zone *dz;
-	struct dn_fib_info *fi;
-	int z = r->rtm_dst_len;
-	int type = r->rtm_type;
-	dn_fib_key_t key;
-	int err;
-
-	if (z > 16)
-		return -EINVAL;
-
-	dz = table->dh_zones[z];
-	if (!dz && !(dz = dn_new_zone(table, z)))
-		return -ENOBUFS;
-
-	dz_key_0(key);
-	if (attrs[RTA_DST]) {
-		__le16 dst = nla_get_le16(attrs[RTA_DST]);
-		if (dst & ~DZ_MASK(dz))
-			return -EINVAL;
-		key = dz_key(dst, dz);
-	}
-
-	if ((fi = dn_fib_create_info(r, attrs, n, &err)) == NULL)
-		return err;
-
-	if (dz->dz_nent > (dz->dz_divisor << 2) &&
-			dz->dz_divisor > DN_MAX_DIVISOR &&
-			(z==16 || (1<<z) > dz->dz_divisor))
-		dn_rehash_zone(dz);
-
-	fp = dn_chain_p(key, dz);
-
-	DN_FIB_SCAN(f, fp) {
-		if (dn_key_leq(key, f->fn_key))
-			break;
-	}
-
-	del_fp = NULL;
-
-	if (f && (f->fn_state & DN_S_ZOMBIE) &&
-			dn_key_eq(f->fn_key, key)) {
-		del_fp = fp;
-		fp = &f->fn_next;
-		f = *fp;
-		goto create;
-	}
-
-	DN_FIB_SCAN_KEY(f, fp, key) {
-		if (fi->fib_priority <= DN_FIB_INFO(f)->fib_priority)
-			break;
-	}
-
-	if (f && dn_key_eq(f->fn_key, key) &&
-			fi->fib_priority == DN_FIB_INFO(f)->fib_priority) {
-		struct dn_fib_node **ins_fp;
-
-		err = -EEXIST;
-		if (n->nlmsg_flags & NLM_F_EXCL)
-			goto out;
-
-		if (n->nlmsg_flags & NLM_F_REPLACE) {
-			del_fp = fp;
-			fp = &f->fn_next;
-			f = *fp;
-			goto replace;
-		}
-
-		ins_fp = fp;
-		err = -EEXIST;
-
-		DN_FIB_SCAN_KEY(f, fp, key) {
-			if (fi->fib_priority != DN_FIB_INFO(f)->fib_priority)
-				break;
-			if (f->fn_type == type &&
-			    f->fn_scope == r->rtm_scope &&
-			    DN_FIB_INFO(f) == fi)
-				goto out;
-		}
-
-		if (!(n->nlmsg_flags & NLM_F_APPEND)) {
-			fp = ins_fp;
-			f = *fp;
-		}
-	}
-
-create:
-	err = -ENOENT;
-	if (!(n->nlmsg_flags & NLM_F_CREATE))
-		goto out;
-
-replace:
-	err = -ENOBUFS;
-	new_f = kmem_cache_zalloc(dn_hash_kmem, GFP_KERNEL);
-	if (new_f == NULL)
-		goto out;
-
-	new_f->fn_key = key;
-	new_f->fn_type = type;
-	new_f->fn_scope = r->rtm_scope;
-	DN_FIB_INFO(new_f) = fi;
-
-	new_f->fn_next = f;
-	write_lock_bh(&dn_fib_tables_lock);
-	*fp = new_f;
-	write_unlock_bh(&dn_fib_tables_lock);
-	dz->dz_nent++;
-
-	if (del_fp) {
-		f = *del_fp;
-		write_lock_bh(&dn_fib_tables_lock);
-		*del_fp = f->fn_next;
-		write_unlock_bh(&dn_fib_tables_lock);
-
-		if (!(f->fn_state & DN_S_ZOMBIE))
-			dn_rtmsg_fib(RTM_DELROUTE, f, z, tb->n, n, req);
-		if (f->fn_state & DN_S_ACCESSED)
-			dn_rt_cache_flush(-1);
-		dn_free_node(f);
-		dz->dz_nent--;
-	} else {
-		dn_rt_cache_flush(-1);
-	}
-
-	dn_rtmsg_fib(RTM_NEWROUTE, new_f, z, tb->n, n, req);
-
-	return 0;
-out:
-	dn_fib_release_info(fi);
-	return err;
-}
-
-
-static int dn_fib_table_delete(struct dn_fib_table *tb, struct rtmsg *r, struct nlattr *attrs[],
-			       struct nlmsghdr *n, struct netlink_skb_parms *req)
-{
-	struct dn_hash *table = (struct dn_hash*)tb->data;
-	struct dn_fib_node **fp, **del_fp, *f;
-	int z = r->rtm_dst_len;
-	struct dn_zone *dz;
-	dn_fib_key_t key;
-	int matched;
-
-
-	if (z > 16)
-		return -EINVAL;
-
-	if ((dz = table->dh_zones[z]) == NULL)
-		return -ESRCH;
-
-	dz_key_0(key);
-	if (attrs[RTA_DST]) {
-		__le16 dst = nla_get_le16(attrs[RTA_DST]);
-		if (dst & ~DZ_MASK(dz))
-			return -EINVAL;
-		key = dz_key(dst, dz);
-	}
-
-	fp = dn_chain_p(key, dz);
-
-	DN_FIB_SCAN(f, fp) {
-		if (dn_key_eq(f->fn_key, key))
-			break;
-		if (dn_key_leq(key, f->fn_key))
-			return -ESRCH;
-	}
-
-	matched = 0;
-	del_fp = NULL;
-	DN_FIB_SCAN_KEY(f, fp, key) {
-		struct dn_fib_info *fi = DN_FIB_INFO(f);
-
-		if (f->fn_state & DN_S_ZOMBIE)
-			return -ESRCH;
-
-		matched++;
-
-		if (del_fp == NULL &&
-				(!r->rtm_type || f->fn_type == r->rtm_type) &&
-				(r->rtm_scope == RT_SCOPE_NOWHERE || f->fn_scope == r->rtm_scope) &&
-				(!r->rtm_protocol ||
-					fi->fib_protocol == r->rtm_protocol) &&
-				dn_fib_nh_match(r, n, attrs, fi) == 0)
-			del_fp = fp;
-	}
-
-	if (del_fp) {
-		f = *del_fp;
-		dn_rtmsg_fib(RTM_DELROUTE, f, z, tb->n, n, req);
-
-		if (matched != 1) {
-			write_lock_bh(&dn_fib_tables_lock);
-			*del_fp = f->fn_next;
-			write_unlock_bh(&dn_fib_tables_lock);
-
-			if (f->fn_state & DN_S_ACCESSED)
-				dn_rt_cache_flush(-1);
-			dn_free_node(f);
-			dz->dz_nent--;
-		} else {
-			f->fn_state |= DN_S_ZOMBIE;
-			if (f->fn_state & DN_S_ACCESSED) {
-				f->fn_state &= ~DN_S_ACCESSED;
-				dn_rt_cache_flush(-1);
-			}
-			if (++dn_fib_hash_zombies > 128)
-				dn_fib_flush();
-		}
-
-		return 0;
-	}
-
-	return -ESRCH;
-}
-
-static inline int dn_flush_list(struct dn_fib_node **fp, int z, struct dn_hash *table)
-{
-	int found = 0;
-	struct dn_fib_node *f;
-
-	while((f = *fp) != NULL) {
-		struct dn_fib_info *fi = DN_FIB_INFO(f);
-
-		if (fi && ((f->fn_state & DN_S_ZOMBIE) || (fi->fib_flags & RTNH_F_DEAD))) {
-			write_lock_bh(&dn_fib_tables_lock);
-			*fp = f->fn_next;
-			write_unlock_bh(&dn_fib_tables_lock);
-
-			dn_free_node(f);
-			found++;
-			continue;
-		}
-		fp = &f->fn_next;
-	}
-
-	return found;
-}
-
-static int dn_fib_table_flush(struct dn_fib_table *tb)
-{
-	struct dn_hash *table = (struct dn_hash *)tb->data;
-	struct dn_zone *dz;
-	int found = 0;
-
-	dn_fib_hash_zombies = 0;
-	for(dz = table->dh_zone_list; dz; dz = dz->dz_next) {
-		int i;
-		int tmp = 0;
-		for(i = dz->dz_divisor-1; i >= 0; i--)
-			tmp += dn_flush_list(&dz->dz_hash[i], dz->dz_order, table);
-		dz->dz_nent -= tmp;
-		found += tmp;
-	}
-
-	return found;
-}
-
-static int dn_fib_table_lookup(struct dn_fib_table *tb, const struct flowidn *flp, struct dn_fib_res *res)
-{
-	int err;
-	struct dn_zone *dz;
-	struct dn_hash *t = (struct dn_hash *)tb->data;
-
-	read_lock(&dn_fib_tables_lock);
-	for(dz = t->dh_zone_list; dz; dz = dz->dz_next) {
-		struct dn_fib_node *f;
-		dn_fib_key_t k = dz_key(flp->daddr, dz);
-
-		for(f = dz_chain(k, dz); f; f = f->fn_next) {
-			if (!dn_key_eq(k, f->fn_key)) {
-				if (dn_key_leq(k, f->fn_key))
-					break;
-				else
-					continue;
-			}
-
-			f->fn_state |= DN_S_ACCESSED;
-
-			if (f->fn_state&DN_S_ZOMBIE)
-				continue;
-
-			if (f->fn_scope < flp->flowidn_scope)
-				continue;
-
-			err = dn_fib_semantic_match(f->fn_type, DN_FIB_INFO(f), flp, res);
-
-			if (err == 0) {
-				res->type = f->fn_type;
-				res->scope = f->fn_scope;
-				res->prefixlen = dz->dz_order;
-				goto out;
-			}
-			if (err < 0)
-				goto out;
-		}
-	}
-	err = 1;
-out:
-	read_unlock(&dn_fib_tables_lock);
-	return err;
-}
-
-
-struct dn_fib_table *dn_fib_get_table(u32 n, int create)
-{
-	struct dn_fib_table *t;
-	unsigned int h;
-
-	if (n < RT_TABLE_MIN)
-		return NULL;
-
-	if (n > RT_TABLE_MAX)
-		return NULL;
-
-	h = n & (DN_FIB_TABLE_HASHSZ - 1);
-	rcu_read_lock();
-	hlist_for_each_entry_rcu(t, &dn_fib_table_hash[h], hlist) {
-		if (t->n == n) {
-			rcu_read_unlock();
-			return t;
-		}
-	}
-	rcu_read_unlock();
-
-	if (!create)
-		return NULL;
-
-	if (in_interrupt()) {
-		net_dbg_ratelimited("DECnet: BUG! Attempt to create routing table from interrupt\n");
-		return NULL;
-	}
-
-	t = kzalloc(sizeof(struct dn_fib_table) + sizeof(struct dn_hash),
-		    GFP_KERNEL);
-	if (t == NULL)
-		return NULL;
-
-	t->n = n;
-	t->insert = dn_fib_table_insert;
-	t->delete = dn_fib_table_delete;
-	t->lookup = dn_fib_table_lookup;
-	t->flush  = dn_fib_table_flush;
-	t->dump = dn_fib_table_dump;
-	hlist_add_head_rcu(&t->hlist, &dn_fib_table_hash[h]);
-
-	return t;
-}
-
-struct dn_fib_table *dn_fib_empty_table(void)
-{
-	u32 id;
-
-	for(id = RT_TABLE_MIN; id <= RT_TABLE_MAX; id++)
-		if (dn_fib_get_table(id, 0) == NULL)
-			return dn_fib_get_table(id, 1);
-	return NULL;
-}
-
-void dn_fib_flush(void)
-{
-	int flushed = 0;
-	struct dn_fib_table *tb;
-	unsigned int h;
-
-	for (h = 0; h < DN_FIB_TABLE_HASHSZ; h++) {
-		hlist_for_each_entry(tb, &dn_fib_table_hash[h], hlist)
-			flushed += tb->flush(tb);
-	}
-
-	if (flushed)
-		dn_rt_cache_flush(-1);
-}
-
-void __init dn_fib_table_init(void)
-{
-	dn_hash_kmem = kmem_cache_create("dn_fib_info_cache",
-					sizeof(struct dn_fib_info),
-					0, SLAB_HWCACHE_ALIGN,
-					NULL);
-}
-
-void __exit dn_fib_table_cleanup(void)
-{
-	struct dn_fib_table *t;
-	struct hlist_node *next;
-	unsigned int h;
-
-	write_lock(&dn_fib_tables_lock);
-	for (h = 0; h < DN_FIB_TABLE_HASHSZ; h++) {
-		hlist_for_each_entry_safe(t, next, &dn_fib_table_hash[h],
-					  hlist) {
-			hlist_del(&t->hlist);
-			kfree(t);
-		}
-	}
-	write_unlock(&dn_fib_tables_lock);
-}
diff --git a/net/decnet/dn_timer.c b/net/decnet/dn_timer.c
deleted file mode 100644
index aa4155875ca8..000000000000
--- a/net/decnet/dn_timer.c
+++ /dev/null
@@ -1,104 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Socket Timer Functions
- *
- * Author:      Steve Whitehouse <SteveW@ACM.org>
- *
- *
- * Changes:
- *       Steve Whitehouse      : Made keepalive timer part of the same
- *                               timer idea.
- *       Steve Whitehouse      : Added checks for sk->sock_readers
- *       David S. Miller       : New socket locking
- *       Steve Whitehouse      : Timer grabs socket ref.
- */
-#include <linux/net.h>
-#include <linux/socket.h>
-#include <linux/skbuff.h>
-#include <linux/netdevice.h>
-#include <linux/timer.h>
-#include <linux/spinlock.h>
-#include <net/sock.h>
-#include <linux/atomic.h>
-#include <linux/jiffies.h>
-#include <net/flow.h>
-#include <net/dn.h>
-
-/*
- * Slow timer is for everything else (n * 500mS)
- */
-
-#define SLOW_INTERVAL (HZ/2)
-
-static void dn_slow_timer(struct timer_list *t);
-
-void dn_start_slow_timer(struct sock *sk)
-{
-	timer_setup(&sk->sk_timer, dn_slow_timer, 0);
-	sk_reset_timer(sk, &sk->sk_timer, jiffies + SLOW_INTERVAL);
-}
-
-void dn_stop_slow_timer(struct sock *sk)
-{
-	sk_stop_timer(sk, &sk->sk_timer);
-}
-
-static void dn_slow_timer(struct timer_list *t)
-{
-	struct sock *sk = from_timer(sk, t, sk_timer);
-	struct dn_scp *scp = DN_SK(sk);
-
-	bh_lock_sock(sk);
-
-	if (sock_owned_by_user(sk)) {
-		sk_reset_timer(sk, &sk->sk_timer, jiffies + HZ / 10);
-		goto out;
-	}
-
-	/*
-	 * The persist timer is the standard slow timer used for retransmits
-	 * in both connection establishment and disconnection as well as
-	 * in the RUN state. The different states are catered for by changing
-	 * the function pointer in the socket. Setting the timer to a value
-	 * of zero turns it off. We allow the persist_fxn to turn the
-	 * timer off in a permant way by returning non-zero, so that
-	 * timer based routines may remove sockets. This is why we have a
-	 * sock_hold()/sock_put() around the timer to prevent the socket
-	 * going away in the middle.
-	 */
-	if (scp->persist && scp->persist_fxn) {
-		if (scp->persist <= SLOW_INTERVAL) {
-			scp->persist = 0;
-
-			if (scp->persist_fxn(sk))
-				goto out;
-		} else {
-			scp->persist -= SLOW_INTERVAL;
-		}
-	}
-
-	/*
-	 * Check for keepalive timeout. After the other timer 'cos if
-	 * the previous timer caused a retransmit, we don't need to
-	 * do this. scp->stamp is the last time that we sent a packet.
-	 * The keepalive function sends a link service packet to the
-	 * other end. If it remains unacknowledged, the standard
-	 * socket timers will eventually shut the socket down. Each
-	 * time we do this, scp->stamp will be updated, thus
-	 * we won't try and send another until scp->keepalive has passed
-	 * since the last successful transmission.
-	 */
-	if (scp->keepalive && scp->keepalive_fxn && (scp->state == DN_RUN)) {
-		if (time_after_eq(jiffies, scp->stamp + scp->keepalive))
-			scp->keepalive_fxn(sk);
-	}
-
-	sk_reset_timer(sk, &sk->sk_timer, jiffies + SLOW_INTERVAL);
-out:
-	bh_unlock_sock(sk);
-	sock_put(sk);
-}
diff --git a/net/decnet/netfilter/Kconfig b/net/decnet/netfilter/Kconfig
deleted file mode 100644
index 14ec4ef95fab..000000000000
--- a/net/decnet/netfilter/Kconfig
+++ /dev/null
@@ -1,17 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# DECnet netfilter configuration
-#
-
-menu "DECnet: Netfilter Configuration"
-	depends on DECNET && NETFILTER
-	depends on NETFILTER_ADVANCED
-
-config DECNET_NF_GRABULATOR
-	tristate "Routing message grabulator (for userland routing daemon)"
-	help
-	  Enable this module if you want to use the userland DECnet routing
-	  daemon. You will also need to enable routing support for DECnet
-	  unless you just want to monitor routing messages from other nodes.
-
-endmenu
diff --git a/net/decnet/netfilter/Makefile b/net/decnet/netfilter/Makefile
deleted file mode 100644
index 429c84289d0f..000000000000
--- a/net/decnet/netfilter/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-#
-# Makefile for DECnet netfilter modules
-#
-
-obj-$(CONFIG_DECNET_NF_GRABULATOR) += dn_rtmsg.o
diff --git a/net/decnet/netfilter/dn_rtmsg.c b/net/decnet/netfilter/dn_rtmsg.c
deleted file mode 100644
index 26a9193df783..000000000000
--- a/net/decnet/netfilter/dn_rtmsg.c
+++ /dev/null
@@ -1,158 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet Routing Message Grabulator
- *
- *              (C) 2000 ChyGwyn Limited  -  https://www.chygwyn.com/
- *
- * Author:      Steven Whitehouse <steve@chygwyn.com>
- */
-#include <linux/module.h>
-#include <linux/skbuff.h>
-#include <linux/slab.h>
-#include <linux/init.h>
-#include <linux/netdevice.h>
-#include <linux/netfilter.h>
-#include <linux/spinlock.h>
-#include <net/netlink.h>
-#include <linux/netfilter_decnet.h>
-
-#include <net/sock.h>
-#include <net/flow.h>
-#include <net/dn.h>
-#include <net/dn_route.h>
-
-static struct sock *dnrmg = NULL;
-
-
-static struct sk_buff *dnrmg_build_message(struct sk_buff *rt_skb, int *errp)
-{
-	struct sk_buff *skb = NULL;
-	size_t size;
-	sk_buff_data_t old_tail;
-	struct nlmsghdr *nlh;
-	unsigned char *ptr;
-	struct nf_dn_rtmsg *rtm;
-
-	size = NLMSG_ALIGN(rt_skb->len) +
-	       NLMSG_ALIGN(sizeof(struct nf_dn_rtmsg));
-	skb = nlmsg_new(size, GFP_ATOMIC);
-	if (!skb) {
-		*errp = -ENOMEM;
-		return NULL;
-	}
-	old_tail = skb->tail;
-	nlh = nlmsg_put(skb, 0, 0, 0, size, 0);
-	if (!nlh) {
-		kfree_skb(skb);
-		*errp = -ENOMEM;
-		return NULL;
-	}
-	rtm = (struct nf_dn_rtmsg *)nlmsg_data(nlh);
-	rtm->nfdn_ifindex = rt_skb->dev->ifindex;
-	ptr = NFDN_RTMSG(rtm);
-	skb_copy_from_linear_data(rt_skb, ptr, rt_skb->len);
-	nlh->nlmsg_len = skb->tail - old_tail;
-	return skb;
-}
-
-static void dnrmg_send_peer(struct sk_buff *skb)
-{
-	struct sk_buff *skb2;
-	int status = 0;
-	int group = 0;
-	unsigned char flags = *skb->data;
-
-	switch (flags & DN_RT_CNTL_MSK) {
-	case DN_RT_PKT_L1RT:
-		group = DNRNG_NLGRP_L1;
-		break;
-	case DN_RT_PKT_L2RT:
-		group = DNRNG_NLGRP_L2;
-		break;
-	default:
-		return;
-	}
-
-	skb2 = dnrmg_build_message(skb, &status);
-	if (skb2 == NULL)
-		return;
-	NETLINK_CB(skb2).dst_group = group;
-	netlink_broadcast(dnrmg, skb2, 0, group, GFP_ATOMIC);
-}
-
-
-static unsigned int dnrmg_hook(void *priv,
-			struct sk_buff *skb,
-			const struct nf_hook_state *state)
-{
-	dnrmg_send_peer(skb);
-	return NF_ACCEPT;
-}
-
-
-#define RCV_SKB_FAIL(err) do { netlink_ack(skb, nlh, (err), NULL); return; } while (0)
-
-static inline void dnrmg_receive_user_skb(struct sk_buff *skb)
-{
-	struct nlmsghdr *nlh = nlmsg_hdr(skb);
-
-	if (skb->len < sizeof(*nlh) ||
-	    nlh->nlmsg_len < sizeof(*nlh) ||
-	    skb->len < nlh->nlmsg_len)
-		return;
-
-	if (!netlink_capable(skb, CAP_NET_ADMIN))
-		RCV_SKB_FAIL(-EPERM);
-
-	/* Eventually we might send routing messages too */
-
-	RCV_SKB_FAIL(-EINVAL);
-}
-
-static const struct nf_hook_ops dnrmg_ops = {
-	.hook		= dnrmg_hook,
-	.pf		= NFPROTO_DECNET,
-	.hooknum	= NF_DN_ROUTE,
-	.priority	= NF_DN_PRI_DNRTMSG,
-};
-
-static int __init dn_rtmsg_init(void)
-{
-	int rv = 0;
-	struct netlink_kernel_cfg cfg = {
-		.groups	= DNRNG_NLGRP_MAX,
-		.input	= dnrmg_receive_user_skb,
-	};
-
-	dnrmg = netlink_kernel_create(&init_net, NETLINK_DNRTMSG, &cfg);
-	if (dnrmg == NULL) {
-		printk(KERN_ERR "dn_rtmsg: Cannot create netlink socket");
-		return -ENOMEM;
-	}
-
-	rv = nf_register_net_hook(&init_net, &dnrmg_ops);
-	if (rv) {
-		netlink_kernel_release(dnrmg);
-	}
-
-	return rv;
-}
-
-static void __exit dn_rtmsg_fini(void)
-{
-	nf_unregister_net_hook(&init_net, &dnrmg_ops);
-	netlink_kernel_release(dnrmg);
-}
-
-
-MODULE_DESCRIPTION("DECnet Routing Message Grabulator");
-MODULE_AUTHOR("Steven Whitehouse <steve@chygwyn.com>");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS_NET_PF_PROTO(PF_NETLINK, NETLINK_DNRTMSG);
-
-module_init(dn_rtmsg_init);
-module_exit(dn_rtmsg_fini);
diff --git a/net/decnet/sysctl_net_decnet.c b/net/decnet/sysctl_net_decnet.c
deleted file mode 100644
index 67b5ab2657b7..000000000000
--- a/net/decnet/sysctl_net_decnet.c
+++ /dev/null
@@ -1,362 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * DECnet       An implementation of the DECnet protocol suite for the LINUX
- *              operating system.  DECnet is implemented using the  BSD Socket
- *              interface as the means of communication with the user level.
- *
- *              DECnet sysctl support functions
- *
- * Author:      Steve Whitehouse <SteveW@ACM.org>
- *
- *
- * Changes:
- * Steve Whitehouse - C99 changes and default device handling
- * Steve Whitehouse - Memory buffer settings, like the tcp ones
- *
- */
-#include <linux/mm.h>
-#include <linux/sysctl.h>
-#include <linux/fs.h>
-#include <linux/netdevice.h>
-#include <linux/string.h>
-#include <net/neighbour.h>
-#include <net/dst.h>
-#include <net/flow.h>
-
-#include <linux/uaccess.h>
-
-#include <net/dn.h>
-#include <net/dn_dev.h>
-#include <net/dn_route.h>
-
-
-int decnet_debug_level;
-int decnet_time_wait = 30;
-int decnet_dn_count = 1;
-int decnet_di_count = 3;
-int decnet_dr_count = 3;
-int decnet_log_martians = 1;
-int decnet_no_fc_max_cwnd = NSP_MIN_WINDOW;
-
-/* Reasonable defaults, I hope, based on tcp's defaults */
-long sysctl_decnet_mem[3] = { 768 << 3, 1024 << 3, 1536 << 3 };
-int sysctl_decnet_wmem[3] = { 4 * 1024, 16 * 1024, 128 * 1024 };
-int sysctl_decnet_rmem[3] = { 4 * 1024, 87380, 87380 * 2 };
-
-#ifdef CONFIG_SYSCTL
-extern int decnet_dst_gc_interval;
-static int min_decnet_time_wait[] = { 5 };
-static int max_decnet_time_wait[] = { 600 };
-static int min_state_count[] = { 1 };
-static int max_state_count[] = { NSP_MAXRXTSHIFT };
-static int min_decnet_dst_gc_interval[] = { 1 };
-static int max_decnet_dst_gc_interval[] = { 60 };
-static int min_decnet_no_fc_max_cwnd[] = { NSP_MIN_WINDOW };
-static int max_decnet_no_fc_max_cwnd[] = { NSP_MAX_WINDOW };
-static char node_name[7] = "???";
-
-static struct ctl_table_header *dn_table_header = NULL;
-
-/*
- * ctype.h :-)
- */
-#define ISNUM(x) (((x) >= '0') && ((x) <= '9'))
-#define ISLOWER(x) (((x) >= 'a') && ((x) <= 'z'))
-#define ISUPPER(x) (((x) >= 'A') && ((x) <= 'Z'))
-#define ISALPHA(x) (ISLOWER(x) || ISUPPER(x))
-#define INVALID_END_CHAR(x) (ISNUM(x) || ISALPHA(x))
-
-static void strip_it(char *str)
-{
-	for(;;) {
-		switch (*str) {
-		case ' ':
-		case '\n':
-		case '\r':
-		case ':':
-			*str = 0;
-			fallthrough;
-		case 0:
-			return;
-		}
-		str++;
-	}
-}
-
-/*
- * Simple routine to parse an ascii DECnet address
- * into a network order address.
- */
-static int parse_addr(__le16 *addr, char *str)
-{
-	__u16 area, node;
-
-	while(*str && !ISNUM(*str)) str++;
-
-	if (*str == 0)
-		return -1;
-
-	area = (*str++ - '0');
-	if (ISNUM(*str)) {
-		area *= 10;
-		area += (*str++ - '0');
-	}
-
-	if (*str++ != '.')
-		return -1;
-
-	if (!ISNUM(*str))
-		return -1;
-
-	node = *str++ - '0';
-	if (ISNUM(*str)) {
-		node *= 10;
-		node += (*str++ - '0');
-	}
-	if (ISNUM(*str)) {
-		node *= 10;
-		node += (*str++ - '0');
-	}
-	if (ISNUM(*str)) {
-		node *= 10;
-		node += (*str++ - '0');
-	}
-
-	if ((node > 1023) || (area > 63))
-		return -1;
-
-	if (INVALID_END_CHAR(*str))
-		return -1;
-
-	*addr = cpu_to_le16((area << 10) | node);
-
-	return 0;
-}
-
-static int dn_node_address_handler(struct ctl_table *table, int write,
-		void *buffer, size_t *lenp, loff_t *ppos)
-{
-	char addr[DN_ASCBUF_LEN];
-	size_t len;
-	__le16 dnaddr;
-
-	if (!*lenp || (*ppos && !write)) {
-		*lenp = 0;
-		return 0;
-	}
-
-	if (write) {
-		len = (*lenp < DN_ASCBUF_LEN) ? *lenp : (DN_ASCBUF_LEN-1);
-		memcpy(addr, buffer, len);
-		addr[len] = 0;
-		strip_it(addr);
-
-		if (parse_addr(&dnaddr, addr))
-			return -EINVAL;
-
-		dn_dev_devices_off();
-
-		decnet_address = dnaddr;
-
-		dn_dev_devices_on();
-
-		*ppos += len;
-
-		return 0;
-	}
-
-	dn_addr2asc(le16_to_cpu(decnet_address), addr);
-	len = strlen(addr);
-	addr[len++] = '\n';
-
-	if (len > *lenp)
-		len = *lenp;
-	memcpy(buffer, addr, len);
-	*lenp = len;
-	*ppos += len;
-
-	return 0;
-}
-
-static int dn_def_dev_handler(struct ctl_table *table, int write,
-		void *buffer, size_t *lenp, loff_t *ppos)
-{
-	size_t len;
-	struct net_device *dev;
-	char devname[17];
-
-	if (!*lenp || (*ppos && !write)) {
-		*lenp = 0;
-		return 0;
-	}
-
-	if (write) {
-		if (*lenp > 16)
-			return -E2BIG;
-
-		memcpy(devname, buffer, *lenp);
-		devname[*lenp] = 0;
-		strip_it(devname);
-
-		dev = dev_get_by_name(&init_net, devname);
-		if (dev == NULL)
-			return -ENODEV;
-
-		if (dev->dn_ptr == NULL) {
-			dev_put(dev);
-			return -ENODEV;
-		}
-
-		if (dn_dev_set_default(dev, 1)) {
-			dev_put(dev);
-			return -ENODEV;
-		}
-		*ppos += *lenp;
-
-		return 0;
-	}
-
-	dev = dn_dev_get_default();
-	if (dev == NULL) {
-		*lenp = 0;
-		return 0;
-	}
-
-	strcpy(devname, dev->name);
-	dev_put(dev);
-	len = strlen(devname);
-	devname[len++] = '\n';
-
-	if (len > *lenp) len = *lenp;
-
-	memcpy(buffer, devname, len);
-	*lenp = len;
-	*ppos += len;
-
-	return 0;
-}
-
-static struct ctl_table dn_table[] = {
-	{
-		.procname = "node_address",
-		.maxlen = 7,
-		.mode = 0644,
-		.proc_handler = dn_node_address_handler,
-	},
-	{
-		.procname = "node_name",
-		.data = node_name,
-		.maxlen = 7,
-		.mode = 0644,
-		.proc_handler = proc_dostring,
-	},
-	{
-		.procname = "default_device",
-		.maxlen = 16,
-		.mode = 0644,
-		.proc_handler = dn_def_dev_handler,
-	},
-	{
-		.procname = "time_wait",
-		.data = &decnet_time_wait,
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = proc_dointvec_minmax,
-		.extra1 = &min_decnet_time_wait,
-		.extra2 = &max_decnet_time_wait
-	},
-	{
-		.procname = "dn_count",
-		.data = &decnet_dn_count,
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = proc_dointvec_minmax,
-		.extra1 = &min_state_count,
-		.extra2 = &max_state_count
-	},
-	{
-		.procname = "di_count",
-		.data = &decnet_di_count,
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = proc_dointvec_minmax,
-		.extra1 = &min_state_count,
-		.extra2 = &max_state_count
-	},
-	{
-		.procname = "dr_count",
-		.data = &decnet_dr_count,
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = proc_dointvec_minmax,
-		.extra1 = &min_state_count,
-		.extra2 = &max_state_count
-	},
-	{
-		.procname = "dst_gc_interval",
-		.data = &decnet_dst_gc_interval,
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = proc_dointvec_minmax,
-		.extra1 = &min_decnet_dst_gc_interval,
-		.extra2 = &max_decnet_dst_gc_interval
-	},
-	{
-		.procname = "no_fc_max_cwnd",
-		.data = &decnet_no_fc_max_cwnd,
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = proc_dointvec_minmax,
-		.extra1 = &min_decnet_no_fc_max_cwnd,
-		.extra2 = &max_decnet_no_fc_max_cwnd
-	},
-       {
-		.procname = "decnet_mem",
-		.data = &sysctl_decnet_mem,
-		.maxlen = sizeof(sysctl_decnet_mem),
-		.mode = 0644,
-		.proc_handler = proc_doulongvec_minmax
-	},
-	{
-		.procname = "decnet_rmem",
-		.data = &sysctl_decnet_rmem,
-		.maxlen = sizeof(sysctl_decnet_rmem),
-		.mode = 0644,
-		.proc_handler = proc_dointvec,
-	},
-	{
-		.procname = "decnet_wmem",
-		.data = &sysctl_decnet_wmem,
-		.maxlen = sizeof(sysctl_decnet_wmem),
-		.mode = 0644,
-		.proc_handler = proc_dointvec,
-	},
-	{
-		.procname = "debug",
-		.data = &decnet_debug_level,
-		.maxlen = sizeof(int),
-		.mode = 0644,
-		.proc_handler = proc_dointvec,
-	},
-	{ }
-};
-
-void dn_register_sysctl(void)
-{
-	dn_table_header = register_net_sysctl(&init_net, "net/decnet", dn_table);
-}
-
-void dn_unregister_sysctl(void)
-{
-	unregister_net_sysctl_table(dn_table_header);
-}
-
-#else  /* CONFIG_SYSCTL */
-void dn_unregister_sysctl(void)
-{
-}
-void dn_register_sysctl(void)
-{
-}
-
-#endif
diff --git a/net/dsa/Makefile b/net/dsa/Makefile
index af28c24ead18..bf57ef3bce2a 100644
--- a/net/dsa/Makefile
+++ b/net/dsa/Makefile
@@ -1,7 +1,15 @@
 # SPDX-License-Identifier: GPL-2.0
 # the core
 obj-$(CONFIG_NET_DSA) += dsa_core.o
-dsa_core-y += dsa.o dsa2.o master.o port.o slave.o switch.o tag_8021q.o
+dsa_core-y += \
+	dsa.o \
+	dsa2.o \
+	master.o \
+	netlink.o \
+	port.o \
+	slave.o \
+	switch.o \
+	tag_8021q.o
 
 # tagging formats
 obj-$(CONFIG_NET_DSA_TAG_AR9331) += tag_ar9331.o
diff --git a/net/dsa/dsa.c b/net/dsa/dsa.c
index be7b320cda76..64b14f655b23 100644
--- a/net/dsa/dsa.c
+++ b/net/dsa/dsa.c
@@ -536,8 +536,16 @@ static int __init dsa_init_module(void)
 	dsa_tag_driver_register(&DSA_TAG_DRIVER_NAME(none_ops),
 				THIS_MODULE);
 
+	rc = rtnl_link_register(&dsa_link_ops);
+	if (rc)
+		goto netlink_register_fail;
+
 	return 0;
 
+netlink_register_fail:
+	dsa_tag_driver_unregister(&DSA_TAG_DRIVER_NAME(none_ops));
+	dsa_slave_unregister_notifier();
+	dev_remove_pack(&dsa_pack_type);
 register_notifier_fail:
 	destroy_workqueue(dsa_owq);
 
@@ -547,6 +555,7 @@ module_init(dsa_init_module);
 
 static void __exit dsa_cleanup_module(void)
 {
+	rtnl_link_unregister(&dsa_link_ops);
 	dsa_tag_driver_unregister(&DSA_TAG_DRIVER_NAME(none_ops));
 
 	dsa_slave_unregister_notifier();
diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c
index cac48a741f27..af0e2c0394ac 100644
--- a/net/dsa/dsa2.c
+++ b/net/dsa/dsa2.c
@@ -387,6 +387,20 @@ static struct dsa_port *dsa_tree_find_first_cpu(struct dsa_switch_tree *dst)
 	return NULL;
 }
 
+struct net_device *dsa_tree_find_first_master(struct dsa_switch_tree *dst)
+{
+	struct device_node *ethernet;
+	struct net_device *master;
+	struct dsa_port *cpu_dp;
+
+	cpu_dp = dsa_tree_find_first_cpu(dst);
+	ethernet = of_parse_phandle(cpu_dp->dn, "ethernet", 0);
+	master = of_find_net_device_by_node(ethernet);
+	of_node_put(ethernet);
+
+	return master;
+}
+
 /* Assign the default CPU port (the first one in the tree) to all ports of the
  * fabric which don't already have one as part of their own switch.
  */
@@ -447,6 +461,72 @@ static void dsa_tree_teardown_cpu_ports(struct dsa_switch_tree *dst)
 			dp->cpu_dp = NULL;
 }
 
+static int dsa_port_devlink_setup(struct dsa_port *dp)
+{
+	struct devlink_port *dlp = &dp->devlink_port;
+	struct dsa_switch_tree *dst = dp->ds->dst;
+	struct devlink_port_attrs attrs = {};
+	struct devlink *dl = dp->ds->devlink;
+	struct dsa_switch *ds = dp->ds;
+	const unsigned char *id;
+	unsigned char len;
+	int err;
+
+	memset(dlp, 0, sizeof(*dlp));
+	devlink_port_init(dl, dlp);
+
+	if (ds->ops->port_setup) {
+		err = ds->ops->port_setup(ds, dp->index);
+		if (err)
+			return err;
+	}
+
+	id = (const unsigned char *)&dst->index;
+	len = sizeof(dst->index);
+
+	attrs.phys.port_number = dp->index;
+	memcpy(attrs.switch_id.id, id, len);
+	attrs.switch_id.id_len = len;
+
+	switch (dp->type) {
+	case DSA_PORT_TYPE_UNUSED:
+		attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED;
+		break;
+	case DSA_PORT_TYPE_CPU:
+		attrs.flavour = DEVLINK_PORT_FLAVOUR_CPU;
+		break;
+	case DSA_PORT_TYPE_DSA:
+		attrs.flavour = DEVLINK_PORT_FLAVOUR_DSA;
+		break;
+	case DSA_PORT_TYPE_USER:
+		attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
+		break;
+	}
+
+	devlink_port_attrs_set(dlp, &attrs);
+	err = devlink_port_register(dl, dlp, dp->index);
+	if (err) {
+		if (ds->ops->port_teardown)
+			ds->ops->port_teardown(ds, dp->index);
+		return err;
+	}
+
+	return 0;
+}
+
+static void dsa_port_devlink_teardown(struct dsa_port *dp)
+{
+	struct devlink_port *dlp = &dp->devlink_port;
+	struct dsa_switch *ds = dp->ds;
+
+	devlink_port_unregister(dlp);
+
+	if (ds->ops->port_teardown)
+		ds->ops->port_teardown(ds, dp->index);
+
+	devlink_port_fini(dlp);
+}
+
 static int dsa_port_setup(struct dsa_port *dp)
 {
 	struct devlink_port *dlp = &dp->devlink_port;
@@ -458,21 +538,25 @@ static int dsa_port_setup(struct dsa_port *dp)
 	if (dp->setup)
 		return 0;
 
-	if (ds->ops->port_setup) {
-		err = ds->ops->port_setup(ds, dp->index);
-		if (err)
-			return err;
-	}
+	err = dsa_port_devlink_setup(dp);
+	if (err)
+		return err;
 
 	switch (dp->type) {
 	case DSA_PORT_TYPE_UNUSED:
 		dsa_port_disable(dp);
 		break;
 	case DSA_PORT_TYPE_CPU:
-		err = dsa_port_link_register_of(dp);
-		if (err)
-			break;
-		dsa_port_link_registered = true;
+		if (dp->dn) {
+			err = dsa_shared_port_link_register_of(dp);
+			if (err)
+				break;
+			dsa_port_link_registered = true;
+		} else {
+			dev_warn(ds->dev,
+				 "skipping link registration for CPU port %d\n",
+				 dp->index);
+		}
 
 		err = dsa_port_enable(dp, NULL);
 		if (err)
@@ -481,10 +565,16 @@ static int dsa_port_setup(struct dsa_port *dp)
 
 		break;
 	case DSA_PORT_TYPE_DSA:
-		err = dsa_port_link_register_of(dp);
-		if (err)
-			break;
-		dsa_port_link_registered = true;
+		if (dp->dn) {
+			err = dsa_shared_port_link_register_of(dp);
+			if (err)
+				break;
+			dsa_port_link_registered = true;
+		} else {
+			dev_warn(ds->dev,
+				 "skipping link registration for DSA port %d\n",
+				 dp->index);
+		}
 
 		err = dsa_port_enable(dp, NULL);
 		if (err)
@@ -505,10 +595,9 @@ static int dsa_port_setup(struct dsa_port *dp)
 	if (err && dsa_port_enabled)
 		dsa_port_disable(dp);
 	if (err && dsa_port_link_registered)
-		dsa_port_link_unregister_of(dp);
+		dsa_shared_port_link_unregister_of(dp);
 	if (err) {
-		if (ds->ops->port_teardown)
-			ds->ops->port_teardown(ds, dp->index);
+		dsa_port_devlink_teardown(dp);
 		return err;
 	}
 
@@ -517,59 +606,13 @@ static int dsa_port_setup(struct dsa_port *dp)
 	return 0;
 }
 
-static int dsa_port_devlink_setup(struct dsa_port *dp)
-{
-	struct devlink_port *dlp = &dp->devlink_port;
-	struct dsa_switch_tree *dst = dp->ds->dst;
-	struct devlink_port_attrs attrs = {};
-	struct devlink *dl = dp->ds->devlink;
-	const unsigned char *id;
-	unsigned char len;
-	int err;
-
-	id = (const unsigned char *)&dst->index;
-	len = sizeof(dst->index);
-
-	attrs.phys.port_number = dp->index;
-	memcpy(attrs.switch_id.id, id, len);
-	attrs.switch_id.id_len = len;
-	memset(dlp, 0, sizeof(*dlp));
-
-	switch (dp->type) {
-	case DSA_PORT_TYPE_UNUSED:
-		attrs.flavour = DEVLINK_PORT_FLAVOUR_UNUSED;
-		break;
-	case DSA_PORT_TYPE_CPU:
-		attrs.flavour = DEVLINK_PORT_FLAVOUR_CPU;
-		break;
-	case DSA_PORT_TYPE_DSA:
-		attrs.flavour = DEVLINK_PORT_FLAVOUR_DSA;
-		break;
-	case DSA_PORT_TYPE_USER:
-		attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
-		break;
-	}
-
-	devlink_port_attrs_set(dlp, &attrs);
-	err = devlink_port_register(dl, dlp, dp->index);
-
-	if (!err)
-		dp->devlink_port_setup = true;
-
-	return err;
-}
-
 static void dsa_port_teardown(struct dsa_port *dp)
 {
 	struct devlink_port *dlp = &dp->devlink_port;
-	struct dsa_switch *ds = dp->ds;
 
 	if (!dp->setup)
 		return;
 
-	if (ds->ops->port_teardown)
-		ds->ops->port_teardown(ds, dp->index);
-
 	devlink_port_type_clear(dlp);
 
 	switch (dp->type) {
@@ -577,11 +620,13 @@ static void dsa_port_teardown(struct dsa_port *dp)
 		break;
 	case DSA_PORT_TYPE_CPU:
 		dsa_port_disable(dp);
-		dsa_port_link_unregister_of(dp);
+		if (dp->dn)
+			dsa_shared_port_link_unregister_of(dp);
 		break;
 	case DSA_PORT_TYPE_DSA:
 		dsa_port_disable(dp);
-		dsa_port_link_unregister_of(dp);
+		if (dp->dn)
+			dsa_shared_port_link_unregister_of(dp);
 		break;
 	case DSA_PORT_TYPE_USER:
 		if (dp->slave) {
@@ -591,46 +636,15 @@ static void dsa_port_teardown(struct dsa_port *dp)
 		break;
 	}
 
-	dp->setup = false;
-}
-
-static void dsa_port_devlink_teardown(struct dsa_port *dp)
-{
-	struct devlink_port *dlp = &dp->devlink_port;
+	dsa_port_devlink_teardown(dp);
 
-	if (dp->devlink_port_setup)
-		devlink_port_unregister(dlp);
-	dp->devlink_port_setup = false;
+	dp->setup = false;
 }
 
-/* Destroy the current devlink port, and create a new one which has the UNUSED
- * flavour. At this point, any call to ds->ops->port_setup has been already
- * balanced out by a call to ds->ops->port_teardown, so we know that any
- * devlink port regions the driver had are now unregistered. We then call its
- * ds->ops->port_setup again, in order for the driver to re-create them on the
- * new devlink port.
- */
-static int dsa_port_reinit_as_unused(struct dsa_port *dp)
+static int dsa_port_setup_as_unused(struct dsa_port *dp)
 {
-	struct dsa_switch *ds = dp->ds;
-	int err;
-
-	dsa_port_devlink_teardown(dp);
 	dp->type = DSA_PORT_TYPE_UNUSED;
-	err = dsa_port_devlink_setup(dp);
-	if (err)
-		return err;
-
-	if (ds->ops->port_setup) {
-		/* On error, leave the devlink port registered,
-		 * dsa_switch_teardown will clean it up later.
-		 */
-		err = ds->ops->port_setup(ds, dp->index);
-		if (err)
-			return err;
-	}
-
-	return 0;
+	return dsa_port_setup(dp);
 }
 
 static int dsa_devlink_info_get(struct devlink *dl,
@@ -854,7 +868,6 @@ static int dsa_switch_setup(struct dsa_switch *ds)
 {
 	struct dsa_devlink_priv *dl_priv;
 	struct device_node *dn;
-	struct dsa_port *dp;
 	int err;
 
 	if (ds->setup)
@@ -877,18 +890,9 @@ static int dsa_switch_setup(struct dsa_switch *ds)
 	dl_priv = devlink_priv(ds->devlink);
 	dl_priv->ds = ds;
 
-	/* Setup devlink port instances now, so that the switch
-	 * setup() can register regions etc, against the ports
-	 */
-	dsa_switch_for_each_port(dp, ds) {
-		err = dsa_port_devlink_setup(dp);
-		if (err)
-			goto unregister_devlink_ports;
-	}
-
 	err = dsa_switch_register_notifier(ds);
 	if (err)
-		goto unregister_devlink_ports;
+		goto devlink_free;
 
 	ds->configure_vlan_while_not_filtering = true;
 
@@ -929,9 +933,7 @@ teardown:
 		ds->ops->teardown(ds);
 unregister_notifier:
 	dsa_switch_unregister_notifier(ds);
-unregister_devlink_ports:
-	dsa_switch_for_each_port(dp, ds)
-		dsa_port_devlink_teardown(dp);
+devlink_free:
 	devlink_free(ds->devlink);
 	ds->devlink = NULL;
 	return err;
@@ -939,8 +941,6 @@ unregister_devlink_ports:
 
 static void dsa_switch_teardown(struct dsa_switch *ds)
 {
-	struct dsa_port *dp;
-
 	if (!ds->setup)
 		return;
 
@@ -959,8 +959,6 @@ static void dsa_switch_teardown(struct dsa_switch *ds)
 	dsa_switch_unregister_notifier(ds);
 
 	if (ds->devlink) {
-		dsa_switch_for_each_port(dp, ds)
-			dsa_port_devlink_teardown(dp);
 		devlink_free(ds->devlink);
 		ds->devlink = NULL;
 	}
@@ -1013,7 +1011,7 @@ static int dsa_tree_setup_ports(struct dsa_switch_tree *dst)
 		if (dsa_port_is_user(dp) || dsa_port_is_unused(dp)) {
 			err = dsa_port_setup(dp);
 			if (err) {
-				err = dsa_port_reinit_as_unused(dp);
+				err = dsa_port_setup_as_unused(dp);
 				if (err)
 					goto teardown;
 			}
@@ -1046,26 +1044,24 @@ static int dsa_tree_setup_switches(struct dsa_switch_tree *dst)
 
 static int dsa_tree_setup_master(struct dsa_switch_tree *dst)
 {
-	struct dsa_port *dp;
+	struct dsa_port *cpu_dp;
 	int err = 0;
 
 	rtnl_lock();
 
-	list_for_each_entry(dp, &dst->ports, list) {
-		if (dsa_port_is_cpu(dp)) {
-			struct net_device *master = dp->master;
-			bool admin_up = (master->flags & IFF_UP) &&
-					!qdisc_tx_is_noop(master);
+	dsa_tree_for_each_cpu_port(cpu_dp, dst) {
+		struct net_device *master = cpu_dp->master;
+		bool admin_up = (master->flags & IFF_UP) &&
+				!qdisc_tx_is_noop(master);
 
-			err = dsa_master_setup(master, dp);
-			if (err)
-				break;
+		err = dsa_master_setup(master, cpu_dp);
+		if (err)
+			break;
 
-			/* Replay master state event */
-			dsa_tree_master_admin_state_change(dst, master, admin_up);
-			dsa_tree_master_oper_state_change(dst, master,
-							  netif_oper_up(master));
-		}
+		/* Replay master state event */
+		dsa_tree_master_admin_state_change(dst, master, admin_up);
+		dsa_tree_master_oper_state_change(dst, master,
+						  netif_oper_up(master));
 	}
 
 	rtnl_unlock();
@@ -1075,22 +1071,20 @@ static int dsa_tree_setup_master(struct dsa_switch_tree *dst)
 
 static void dsa_tree_teardown_master(struct dsa_switch_tree *dst)
 {
-	struct dsa_port *dp;
+	struct dsa_port *cpu_dp;
 
 	rtnl_lock();
 
-	list_for_each_entry(dp, &dst->ports, list) {
-		if (dsa_port_is_cpu(dp)) {
-			struct net_device *master = dp->master;
+	dsa_tree_for_each_cpu_port(cpu_dp, dst) {
+		struct net_device *master = cpu_dp->master;
 
-			/* Synthesizing an "admin down" state is sufficient for
-			 * the switches to get a notification if the master is
-			 * currently up and running.
-			 */
-			dsa_tree_master_admin_state_change(dst, master, false);
+		/* Synthesizing an "admin down" state is sufficient for
+		 * the switches to get a notification if the master is
+		 * currently up and running.
+		 */
+		dsa_tree_master_admin_state_change(dst, master, false);
 
-			dsa_master_teardown(master);
-		}
+		dsa_master_teardown(master);
 	}
 
 	rtnl_unlock();
@@ -1238,7 +1232,6 @@ out_disconnect:
  * they would have formed disjoint trees (different "dsa,member" values).
  */
 int dsa_tree_change_tag_proto(struct dsa_switch_tree *dst,
-			      struct net_device *master,
 			      const struct dsa_device_ops *tag_ops,
 			      const struct dsa_device_ops *old_tag_ops)
 {
@@ -1254,12 +1247,9 @@ int dsa_tree_change_tag_proto(struct dsa_switch_tree *dst,
 	 * attempts to change the tagging protocol. If we ever lift the IFF_UP
 	 * restriction, there needs to be another mutex which serializes this.
 	 */
-	if (master->flags & IFF_UP)
-		goto out_unlock;
-
-	list_for_each_entry(dp, &dst->ports, list) {
-		if (!dsa_port_is_user(dp))
-			continue;
+	dsa_tree_for_each_user_port(dp, dst) {
+		if (dsa_port_to_master(dp)->flags & IFF_UP)
+			goto out_unlock;
 
 		if (dp->slave->flags & IFF_UP)
 			goto out_unlock;
@@ -1306,6 +1296,12 @@ void dsa_tree_master_admin_state_change(struct dsa_switch_tree *dst,
 	struct dsa_port *cpu_dp = master->dsa_ptr;
 	bool notify = false;
 
+	/* Don't keep track of admin state on LAG DSA masters,
+	 * but rather just of physical DSA masters
+	 */
+	if (netif_is_lag_master(master))
+		return;
+
 	if ((dsa_port_master_is_operational(cpu_dp)) !=
 	    (up && cpu_dp->master_oper_up))
 		notify = true;
@@ -1323,6 +1319,12 @@ void dsa_tree_master_oper_state_change(struct dsa_switch_tree *dst,
 	struct dsa_port *cpu_dp = master->dsa_ptr;
 	bool notify = false;
 
+	/* Don't keep track of oper state on LAG DSA masters,
+	 * but rather just of physical DSA masters
+	 */
+	if (netif_is_lag_master(master))
+		return;
+
 	if ((dsa_port_master_is_operational(cpu_dp)) !=
 	    (cpu_dp->master_admin_up && up))
 		notify = true;
@@ -1791,7 +1793,7 @@ void dsa_switch_shutdown(struct dsa_switch *ds)
 	rtnl_lock();
 
 	dsa_switch_for_each_user_port(dp, ds) {
-		master = dp->cpu_dp->master;
+		master = dsa_port_to_master(dp);
 		slave_dev = dp->slave;
 
 		netdev_upper_dev_unlink(master, slave_dev);
diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h
index d9722e49864b..6e65c7ffd6f3 100644
--- a/net/dsa/dsa_priv.h
+++ b/net/dsa/dsa_priv.h
@@ -88,6 +88,7 @@ struct dsa_notifier_lag_info {
 	const struct dsa_port *dp;
 	struct dsa_lag lag;
 	struct netdev_lag_upper_info *info;
+	struct netlink_ext_ack *extack;
 };
 
 /* DSA_NOTIFIER_VLAN_* */
@@ -184,6 +185,11 @@ static inline int dsa_tag_protocol_overhead(const struct dsa_device_ops *ops)
 /* master.c */
 int dsa_master_setup(struct net_device *dev, struct dsa_port *cpu_dp);
 void dsa_master_teardown(struct net_device *dev);
+int dsa_master_lag_setup(struct net_device *lag_dev, struct dsa_port *cpu_dp,
+			 struct netdev_lag_upper_info *uinfo,
+			 struct netlink_ext_ack *extack);
+void dsa_master_lag_teardown(struct net_device *lag_dev,
+			     struct dsa_port *cpu_dp);
 
 static inline struct net_device *dsa_master_find_slave(struct net_device *dev,
 						       int device, int port)
@@ -200,6 +206,9 @@ static inline struct net_device *dsa_master_find_slave(struct net_device *dev,
 	return NULL;
 }
 
+/* netlink.c */
+extern struct rtnl_link_ops dsa_link_ops __read_mostly;
+
 /* port.c */
 void dsa_port_set_tag_protocol(struct dsa_port *cpu_dp,
 			       const struct dsa_device_ops *tag_ops);
@@ -285,13 +294,16 @@ int dsa_port_mrp_add_ring_role(const struct dsa_port *dp,
 int dsa_port_mrp_del_ring_role(const struct dsa_port *dp,
 			       const struct switchdev_obj_ring_role_mrp *mrp);
 int dsa_port_phylink_create(struct dsa_port *dp);
-int dsa_port_link_register_of(struct dsa_port *dp);
-void dsa_port_link_unregister_of(struct dsa_port *dp);
+void dsa_port_phylink_destroy(struct dsa_port *dp);
+int dsa_shared_port_link_register_of(struct dsa_port *dp);
+void dsa_shared_port_link_unregister_of(struct dsa_port *dp);
 int dsa_port_hsr_join(struct dsa_port *dp, struct net_device *hsr);
 void dsa_port_hsr_leave(struct dsa_port *dp, struct net_device *hsr);
 int dsa_port_tag_8021q_vlan_add(struct dsa_port *dp, u16 vid, bool broadcast);
 void dsa_port_tag_8021q_vlan_del(struct dsa_port *dp, u16 vid, bool broadcast);
 void dsa_port_set_host_flood(struct dsa_port *dp, bool uc, bool mc);
+int dsa_port_change_master(struct dsa_port *dp, struct net_device *master,
+			   struct netlink_ext_ack *extack);
 
 /* slave.c */
 extern const struct dsa_device_ops notag_netdev_ops;
@@ -305,8 +317,12 @@ int dsa_slave_suspend(struct net_device *slave_dev);
 int dsa_slave_resume(struct net_device *slave_dev);
 int dsa_slave_register_notifier(void);
 void dsa_slave_unregister_notifier(void);
+void dsa_slave_sync_ha(struct net_device *dev);
+void dsa_slave_unsync_ha(struct net_device *dev);
 void dsa_slave_setup_tagger(struct net_device *slave);
 int dsa_slave_change_mtu(struct net_device *dev, int new_mtu);
+int dsa_slave_change_master(struct net_device *dev, struct net_device *master,
+			    struct netlink_ext_ack *extack);
 int dsa_slave_manage_vlan_filtering(struct net_device *dev,
 				    bool vlan_filtering);
 
@@ -322,7 +338,7 @@ dsa_slave_to_master(const struct net_device *dev)
 {
 	struct dsa_port *dp = dsa_slave_to_port(dev);
 
-	return dp->cpu_dp->master;
+	return dsa_port_to_master(dp);
 }
 
 /* If under a bridge with vlan_filtering=0, make sure to send pvid-tagged
@@ -542,10 +558,10 @@ void dsa_lag_map(struct dsa_switch_tree *dst, struct dsa_lag *lag);
 void dsa_lag_unmap(struct dsa_switch_tree *dst, struct dsa_lag *lag);
 struct dsa_lag *dsa_tree_lag_find(struct dsa_switch_tree *dst,
 				  const struct net_device *lag_dev);
+struct net_device *dsa_tree_find_first_master(struct dsa_switch_tree *dst);
 int dsa_tree_notify(struct dsa_switch_tree *dst, unsigned long e, void *v);
 int dsa_broadcast(unsigned long e, void *v);
 int dsa_tree_change_tag_proto(struct dsa_switch_tree *dst,
-			      struct net_device *master,
 			      const struct dsa_device_ops *tag_ops,
 			      const struct dsa_device_ops *old_tag_ops);
 void dsa_tree_master_admin_state_change(struct dsa_switch_tree *dst,
diff --git a/net/dsa/master.c b/net/dsa/master.c
index 2851e44c4cf0..40367ab41cf8 100644
--- a/net/dsa/master.c
+++ b/net/dsa/master.c
@@ -58,7 +58,7 @@ static void dsa_master_get_regs(struct net_device *dev,
 	}
 
 	cpu_info = (struct ethtool_drvinfo *)data;
-	strlcpy(cpu_info->driver, "dsa", sizeof(cpu_info->driver));
+	strscpy(cpu_info->driver, "dsa", sizeof(cpu_info->driver));
 	data += sizeof(*cpu_info);
 	cpu_regs = (struct ethtool_regs *)data;
 	data += sizeof(*cpu_regs);
@@ -226,6 +226,9 @@ static int dsa_master_ethtool_setup(struct net_device *dev)
 	struct dsa_switch *ds = cpu_dp->ds;
 	struct ethtool_ops *ops;
 
+	if (netif_is_lag_master(dev))
+		return 0;
+
 	ops = devm_kzalloc(ds->dev, sizeof(*ops), GFP_KERNEL);
 	if (!ops)
 		return -ENOMEM;
@@ -250,6 +253,9 @@ static void dsa_master_ethtool_teardown(struct net_device *dev)
 {
 	struct dsa_port *cpu_dp = dev->dsa_ptr;
 
+	if (netif_is_lag_master(dev))
+		return;
+
 	dev->ethtool_ops = cpu_dp->orig_ethtool_ops;
 	cpu_dp->orig_ethtool_ops = NULL;
 }
@@ -257,6 +263,9 @@ static void dsa_master_ethtool_teardown(struct net_device *dev)
 static void dsa_netdev_ops_set(struct net_device *dev,
 			       const struct dsa_netdevice_ops *ops)
 {
+	if (netif_is_lag_master(dev))
+		return;
+
 	dev->dsa_ptr->netdev_ops = ops;
 }
 
@@ -307,7 +316,7 @@ static ssize_t tagging_store(struct device *d, struct device_attribute *attr,
 		 */
 		goto out;
 
-	err = dsa_tree_change_tag_proto(cpu_dp->ds->dst, dev, new_tag_ops,
+	err = dsa_tree_change_tag_proto(cpu_dp->ds->dst, new_tag_ops,
 					old_tag_ops);
 	if (err) {
 		/* On failure the old tagger is restored, so we don't need the
@@ -355,12 +364,14 @@ int dsa_master_setup(struct net_device *dev, struct dsa_port *cpu_dp)
 	mtu = ETH_DATA_LEN + dsa_tag_protocol_overhead(tag_ops);
 
 	/* The DSA master must use SET_NETDEV_DEV for this to work. */
-	consumer_link = device_link_add(ds->dev, dev->dev.parent,
-					DL_FLAG_AUTOREMOVE_CONSUMER);
-	if (!consumer_link)
-		netdev_err(dev,
-			   "Failed to create a device link to DSA switch %s\n",
-			   dev_name(ds->dev));
+	if (!netif_is_lag_master(dev)) {
+		consumer_link = device_link_add(ds->dev, dev->dev.parent,
+						DL_FLAG_AUTOREMOVE_CONSUMER);
+		if (!consumer_link)
+			netdev_err(dev,
+				   "Failed to create a device link to DSA switch %s\n",
+				   dev_name(ds->dev));
+	}
 
 	/* The switch driver may not implement ->port_change_mtu(), case in
 	 * which dsa_slave_change_mtu() will not update the master MTU either,
@@ -417,3 +428,52 @@ void dsa_master_teardown(struct net_device *dev)
 	 */
 	wmb();
 }
+
+int dsa_master_lag_setup(struct net_device *lag_dev, struct dsa_port *cpu_dp,
+			 struct netdev_lag_upper_info *uinfo,
+			 struct netlink_ext_ack *extack)
+{
+	bool master_setup = false;
+	int err;
+
+	if (!netdev_uses_dsa(lag_dev)) {
+		err = dsa_master_setup(lag_dev, cpu_dp);
+		if (err)
+			return err;
+
+		master_setup = true;
+	}
+
+	err = dsa_port_lag_join(cpu_dp, lag_dev, uinfo, extack);
+	if (err) {
+		if (extack && !extack->_msg)
+			NL_SET_ERR_MSG_MOD(extack,
+					   "CPU port failed to join LAG");
+		goto out_master_teardown;
+	}
+
+	return 0;
+
+out_master_teardown:
+	if (master_setup)
+		dsa_master_teardown(lag_dev);
+	return err;
+}
+
+/* Tear down a master if there isn't any other user port on it,
+ * optionally also destroying LAG information.
+ */
+void dsa_master_lag_teardown(struct net_device *lag_dev,
+			     struct dsa_port *cpu_dp)
+{
+	struct net_device *upper;
+	struct list_head *iter;
+
+	dsa_port_lag_leave(cpu_dp, lag_dev);
+
+	netdev_for_each_upper_dev_rcu(lag_dev, upper, iter)
+		if (dsa_slave_dev_check(upper))
+			return;
+
+	dsa_master_teardown(lag_dev);
+}
diff --git a/net/dsa/netlink.c b/net/dsa/netlink.c
new file mode 100644
index 000000000000..ecf9ed1de185
--- /dev/null
+++ b/net/dsa/netlink.c
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2022 NXP
+ */
+#include <linux/netdevice.h>
+#include <net/rtnetlink.h>
+
+#include "dsa_priv.h"
+
+static const struct nla_policy dsa_policy[IFLA_DSA_MAX + 1] = {
+	[IFLA_DSA_MASTER]	= { .type = NLA_U32 },
+};
+
+static int dsa_changelink(struct net_device *dev, struct nlattr *tb[],
+			  struct nlattr *data[],
+			  struct netlink_ext_ack *extack)
+{
+	int err;
+
+	if (!data)
+		return 0;
+
+	if (data[IFLA_DSA_MASTER]) {
+		u32 ifindex = nla_get_u32(data[IFLA_DSA_MASTER]);
+		struct net_device *master;
+
+		master = __dev_get_by_index(dev_net(dev), ifindex);
+		if (!master)
+			return -EINVAL;
+
+		err = dsa_slave_change_master(dev, master, extack);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static size_t dsa_get_size(const struct net_device *dev)
+{
+	return nla_total_size(sizeof(u32)) +	/* IFLA_DSA_MASTER  */
+	       0;
+}
+
+static int dsa_fill_info(struct sk_buff *skb, const struct net_device *dev)
+{
+	struct net_device *master = dsa_slave_to_master(dev);
+
+	if (nla_put_u32(skb, IFLA_DSA_MASTER, master->ifindex))
+		return -EMSGSIZE;
+
+	return 0;
+}
+
+struct rtnl_link_ops dsa_link_ops __read_mostly = {
+	.kind			= "dsa",
+	.priv_size		= sizeof(struct dsa_port),
+	.maxtype		= IFLA_DSA_MAX,
+	.policy			= dsa_policy,
+	.changelink		= dsa_changelink,
+	.get_size		= dsa_get_size,
+	.fill_info		= dsa_fill_info,
+	.netns_refund		= true,
+};
diff --git a/net/dsa/port.c b/net/dsa/port.c
index a8895ee3cd60..e4a0513816bb 100644
--- a/net/dsa/port.c
+++ b/net/dsa/port.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/if_bridge.h>
+#include <linux/netdevice.h>
 #include <linux/notifier.h>
 #include <linux/of_mdio.h>
 #include <linux/of_net.h>
@@ -634,6 +635,7 @@ int dsa_port_lag_join(struct dsa_port *dp, struct net_device *lag_dev,
 	struct dsa_notifier_lag_info info = {
 		.dp = dp,
 		.info = uinfo,
+		.extack = extack,
 	};
 	struct net_device *bridge_dev;
 	int err;
@@ -1026,7 +1028,7 @@ int dsa_port_standalone_host_fdb_add(struct dsa_port *dp,
 int dsa_port_bridge_host_fdb_add(struct dsa_port *dp,
 				 const unsigned char *addr, u16 vid)
 {
-	struct dsa_port *cpu_dp = dp->cpu_dp;
+	struct net_device *master = dsa_port_to_master(dp);
 	struct dsa_db db = {
 		.type = DSA_DB_BRIDGE,
 		.bridge = *dp->bridge,
@@ -1037,8 +1039,8 @@ int dsa_port_bridge_host_fdb_add(struct dsa_port *dp,
 	 * requires rtnl_lock(), since we can't guarantee that is held here,
 	 * and we can't take it either.
 	 */
-	if (cpu_dp->master->priv_flags & IFF_UNICAST_FLT) {
-		err = dev_uc_add(cpu_dp->master, addr);
+	if (master->priv_flags & IFF_UNICAST_FLT) {
+		err = dev_uc_add(master, addr);
 		if (err)
 			return err;
 	}
@@ -1077,15 +1079,15 @@ int dsa_port_standalone_host_fdb_del(struct dsa_port *dp,
 int dsa_port_bridge_host_fdb_del(struct dsa_port *dp,
 				 const unsigned char *addr, u16 vid)
 {
-	struct dsa_port *cpu_dp = dp->cpu_dp;
+	struct net_device *master = dsa_port_to_master(dp);
 	struct dsa_db db = {
 		.type = DSA_DB_BRIDGE,
 		.bridge = *dp->bridge,
 	};
 	int err;
 
-	if (cpu_dp->master->priv_flags & IFF_UNICAST_FLT) {
-		err = dev_uc_del(cpu_dp->master, addr);
+	if (master->priv_flags & IFF_UNICAST_FLT) {
+		err = dev_uc_del(master, addr);
 		if (err)
 			return err;
 	}
@@ -1208,14 +1210,14 @@ int dsa_port_standalone_host_mdb_add(const struct dsa_port *dp,
 int dsa_port_bridge_host_mdb_add(const struct dsa_port *dp,
 				 const struct switchdev_obj_port_mdb *mdb)
 {
-	struct dsa_port *cpu_dp = dp->cpu_dp;
+	struct net_device *master = dsa_port_to_master(dp);
 	struct dsa_db db = {
 		.type = DSA_DB_BRIDGE,
 		.bridge = *dp->bridge,
 	};
 	int err;
 
-	err = dev_mc_add(cpu_dp->master, mdb->addr);
+	err = dev_mc_add(master, mdb->addr);
 	if (err)
 		return err;
 
@@ -1252,14 +1254,14 @@ int dsa_port_standalone_host_mdb_del(const struct dsa_port *dp,
 int dsa_port_bridge_host_mdb_del(const struct dsa_port *dp,
 				 const struct switchdev_obj_port_mdb *mdb)
 {
-	struct dsa_port *cpu_dp = dp->cpu_dp;
+	struct net_device *master = dsa_port_to_master(dp);
 	struct dsa_db db = {
 		.type = DSA_DB_BRIDGE,
 		.bridge = *dp->bridge,
 	};
 	int err;
 
-	err = dev_mc_del(cpu_dp->master, mdb->addr);
+	err = dev_mc_del(master, mdb->addr);
 	if (err)
 		return err;
 
@@ -1294,19 +1296,19 @@ int dsa_port_host_vlan_add(struct dsa_port *dp,
 			   const struct switchdev_obj_port_vlan *vlan,
 			   struct netlink_ext_ack *extack)
 {
+	struct net_device *master = dsa_port_to_master(dp);
 	struct dsa_notifier_vlan_info info = {
 		.dp = dp,
 		.vlan = vlan,
 		.extack = extack,
 	};
-	struct dsa_port *cpu_dp = dp->cpu_dp;
 	int err;
 
 	err = dsa_port_notify(dp, DSA_NOTIFIER_HOST_VLAN_ADD, &info);
 	if (err && err != -EOPNOTSUPP)
 		return err;
 
-	vlan_vid_add(cpu_dp->master, htons(ETH_P_8021Q), vlan->vid);
+	vlan_vid_add(master, htons(ETH_P_8021Q), vlan->vid);
 
 	return err;
 }
@@ -1314,18 +1316,18 @@ int dsa_port_host_vlan_add(struct dsa_port *dp,
 int dsa_port_host_vlan_del(struct dsa_port *dp,
 			   const struct switchdev_obj_port_vlan *vlan)
 {
+	struct net_device *master = dsa_port_to_master(dp);
 	struct dsa_notifier_vlan_info info = {
 		.dp = dp,
 		.vlan = vlan,
 	};
-	struct dsa_port *cpu_dp = dp->cpu_dp;
 	int err;
 
 	err = dsa_port_notify(dp, DSA_NOTIFIER_HOST_VLAN_DEL, &info);
 	if (err && err != -EOPNOTSUPP)
 		return err;
 
-	vlan_vid_del(cpu_dp->master, htons(ETH_P_8021Q), vlan->vid);
+	vlan_vid_del(master, htons(ETH_P_8021Q), vlan->vid);
 
 	return err;
 }
@@ -1374,6 +1376,136 @@ int dsa_port_mrp_del_ring_role(const struct dsa_port *dp,
 	return ds->ops->port_mrp_del_ring_role(ds, dp->index, mrp);
 }
 
+static int dsa_port_assign_master(struct dsa_port *dp,
+				  struct net_device *master,
+				  struct netlink_ext_ack *extack,
+				  bool fail_on_err)
+{
+	struct dsa_switch *ds = dp->ds;
+	int port = dp->index, err;
+
+	err = ds->ops->port_change_master(ds, port, master, extack);
+	if (err && !fail_on_err)
+		dev_err(ds->dev, "port %d failed to assign master %s: %pe\n",
+			port, master->name, ERR_PTR(err));
+
+	if (err && fail_on_err)
+		return err;
+
+	dp->cpu_dp = master->dsa_ptr;
+	dp->cpu_port_in_lag = netif_is_lag_master(master);
+
+	return 0;
+}
+
+/* Change the dp->cpu_dp affinity for a user port. Note that both cross-chip
+ * notifiers and drivers have implicit assumptions about user-to-CPU-port
+ * mappings, so we unfortunately cannot delay the deletion of the objects
+ * (switchdev, standalone addresses, standalone VLANs) on the old CPU port
+ * until the new CPU port has been set up. So we need to completely tear down
+ * the old CPU port before changing it, and restore it on errors during the
+ * bringup of the new one.
+ */
+int dsa_port_change_master(struct dsa_port *dp, struct net_device *master,
+			   struct netlink_ext_ack *extack)
+{
+	struct net_device *bridge_dev = dsa_port_bridge_dev_get(dp);
+	struct net_device *old_master = dsa_port_to_master(dp);
+	struct net_device *dev = dp->slave;
+	struct dsa_switch *ds = dp->ds;
+	bool vlan_filtering;
+	int err, tmp;
+
+	/* Bridges may hold host FDB, MDB and VLAN objects. These need to be
+	 * migrated, so dynamically unoffload and later reoffload the bridge
+	 * port.
+	 */
+	if (bridge_dev) {
+		dsa_port_pre_bridge_leave(dp, bridge_dev);
+		dsa_port_bridge_leave(dp, bridge_dev);
+	}
+
+	/* The port might still be VLAN filtering even if it's no longer
+	 * under a bridge, either due to ds->vlan_filtering_is_global or
+	 * ds->needs_standalone_vlan_filtering. In turn this means VLANs
+	 * on the CPU port.
+	 */
+	vlan_filtering = dsa_port_is_vlan_filtering(dp);
+	if (vlan_filtering) {
+		err = dsa_slave_manage_vlan_filtering(dev, false);
+		if (err) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Failed to remove standalone VLANs");
+			goto rewind_old_bridge;
+		}
+	}
+
+	/* Standalone addresses, and addresses of upper interfaces like
+	 * VLAN, LAG, HSR need to be migrated.
+	 */
+	dsa_slave_unsync_ha(dev);
+
+	err = dsa_port_assign_master(dp, master, extack, true);
+	if (err)
+		goto rewind_old_addrs;
+
+	dsa_slave_sync_ha(dev);
+
+	if (vlan_filtering) {
+		err = dsa_slave_manage_vlan_filtering(dev, true);
+		if (err) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Failed to restore standalone VLANs");
+			goto rewind_new_addrs;
+		}
+	}
+
+	if (bridge_dev) {
+		err = dsa_port_bridge_join(dp, bridge_dev, extack);
+		if (err && err == -EOPNOTSUPP) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Failed to reoffload bridge");
+			goto rewind_new_vlan;
+		}
+	}
+
+	return 0;
+
+rewind_new_vlan:
+	if (vlan_filtering)
+		dsa_slave_manage_vlan_filtering(dev, false);
+
+rewind_new_addrs:
+	dsa_slave_unsync_ha(dev);
+
+	dsa_port_assign_master(dp, old_master, NULL, false);
+
+/* Restore the objects on the old CPU port */
+rewind_old_addrs:
+	dsa_slave_sync_ha(dev);
+
+	if (vlan_filtering) {
+		tmp = dsa_slave_manage_vlan_filtering(dev, true);
+		if (tmp) {
+			dev_err(ds->dev,
+				"port %d failed to restore standalone VLANs: %pe\n",
+				dp->index, ERR_PTR(tmp));
+		}
+	}
+
+rewind_old_bridge:
+	if (bridge_dev) {
+		tmp = dsa_port_bridge_join(dp, bridge_dev, extack);
+		if (tmp) {
+			dev_err(ds->dev,
+				"port %d failed to rejoin bridge %s: %pe\n",
+				dp->index, bridge_dev->name, ERR_PTR(tmp));
+		}
+	}
+
+	return err;
+}
+
 void dsa_port_set_tag_protocol(struct dsa_port *cpu_dp,
 			       const struct dsa_device_ops *tag_ops)
 {
@@ -1529,6 +1661,7 @@ int dsa_port_phylink_create(struct dsa_port *dp)
 {
 	struct dsa_switch *ds = dp->ds;
 	phy_interface_t mode;
+	struct phylink *pl;
 	int err;
 
 	err = of_get_phy_mode(dp->dn, &mode);
@@ -1545,17 +1678,25 @@ int dsa_port_phylink_create(struct dsa_port *dp)
 	if (ds->ops->phylink_get_caps)
 		ds->ops->phylink_get_caps(ds, dp->index, &dp->pl_config);
 
-	dp->pl = phylink_create(&dp->pl_config, of_fwnode_handle(dp->dn),
-				mode, &dsa_port_phylink_mac_ops);
-	if (IS_ERR(dp->pl)) {
+	pl = phylink_create(&dp->pl_config, of_fwnode_handle(dp->dn),
+			    mode, &dsa_port_phylink_mac_ops);
+	if (IS_ERR(pl)) {
 		pr_err("error creating PHYLINK: %ld\n", PTR_ERR(dp->pl));
-		return PTR_ERR(dp->pl);
+		return PTR_ERR(pl);
 	}
 
+	dp->pl = pl;
+
 	return 0;
 }
 
-static int dsa_port_setup_phy_of(struct dsa_port *dp, bool enable)
+void dsa_port_phylink_destroy(struct dsa_port *dp)
+{
+	phylink_destroy(dp->pl);
+	dp->pl = NULL;
+}
+
+static int dsa_shared_port_setup_phy_of(struct dsa_port *dp, bool enable)
 {
 	struct dsa_switch *ds = dp->ds;
 	struct phy_device *phydev;
@@ -1593,7 +1734,7 @@ err_put_dev:
 	return err;
 }
 
-static int dsa_port_fixed_link_register_of(struct dsa_port *dp)
+static int dsa_shared_port_fixed_link_register_of(struct dsa_port *dp)
 {
 	struct device_node *dn = dp->dn;
 	struct dsa_switch *ds = dp->ds;
@@ -1627,7 +1768,7 @@ static int dsa_port_fixed_link_register_of(struct dsa_port *dp)
 	return 0;
 }
 
-static int dsa_port_phylink_register(struct dsa_port *dp)
+static int dsa_shared_port_phylink_register(struct dsa_port *dp)
 {
 	struct dsa_switch *ds = dp->ds;
 	struct device_node *port_dn = dp->dn;
@@ -1649,26 +1790,188 @@ static int dsa_port_phylink_register(struct dsa_port *dp)
 	return 0;
 
 err_phy_connect:
-	phylink_destroy(dp->pl);
+	dsa_port_phylink_destroy(dp);
 	return err;
 }
 
-int dsa_port_link_register_of(struct dsa_port *dp)
+/* During the initial DSA driver migration to OF, port nodes were sometimes
+ * added to device trees with no indication of how they should operate from a
+ * link management perspective (phy-handle, fixed-link, etc). Additionally, the
+ * phy-mode may be absent. The interpretation of these port OF nodes depends on
+ * their type.
+ *
+ * User ports with no phy-handle or fixed-link are expected to connect to an
+ * internal PHY located on the ds->slave_mii_bus at an MDIO address equal to
+ * the port number. This description is still actively supported.
+ *
+ * Shared (CPU and DSA) ports with no phy-handle or fixed-link are expected to
+ * operate at the maximum speed that their phy-mode is capable of. If the
+ * phy-mode is absent, they are expected to operate using the phy-mode
+ * supported by the port that gives the highest link speed. It is unspecified
+ * if the port should use flow control or not, half duplex or full duplex, or
+ * if the phy-mode is a SERDES link, whether in-band autoneg is expected to be
+ * enabled or not.
+ *
+ * In the latter case of shared ports, omitting the link management description
+ * from the firmware node is deprecated and strongly discouraged. DSA uses
+ * phylink, which rejects the firmware nodes of these ports for lacking
+ * required properties.
+ *
+ * For switches in this table, DSA will skip enforcing validation and will
+ * later omit registering a phylink instance for the shared ports, if they lack
+ * a fixed-link, a phy-handle, or a managed = "in-band-status" property.
+ * It becomes the responsibility of the driver to ensure that these ports
+ * operate at the maximum speed (whatever this means) and will interoperate
+ * with the DSA master or other cascade port, since phylink methods will not be
+ * invoked for them.
+ *
+ * If you are considering expanding this table for newly introduced switches,
+ * think again. It is OK to remove switches from this table if there aren't DT
+ * blobs in circulation which rely on defaulting the shared ports.
+ */
+static const char * const dsa_switches_apply_workarounds[] = {
+#if IS_ENABLED(CONFIG_NET_DSA_XRS700X)
+	"arrow,xrs7003e",
+	"arrow,xrs7003f",
+	"arrow,xrs7004e",
+	"arrow,xrs7004f",
+#endif
+#if IS_ENABLED(CONFIG_B53)
+	"brcm,bcm5325",
+	"brcm,bcm53115",
+	"brcm,bcm53125",
+	"brcm,bcm53128",
+	"brcm,bcm5365",
+	"brcm,bcm5389",
+	"brcm,bcm5395",
+	"brcm,bcm5397",
+	"brcm,bcm5398",
+	"brcm,bcm53010-srab",
+	"brcm,bcm53011-srab",
+	"brcm,bcm53012-srab",
+	"brcm,bcm53018-srab",
+	"brcm,bcm53019-srab",
+	"brcm,bcm5301x-srab",
+	"brcm,bcm11360-srab",
+	"brcm,bcm58522-srab",
+	"brcm,bcm58525-srab",
+	"brcm,bcm58535-srab",
+	"brcm,bcm58622-srab",
+	"brcm,bcm58623-srab",
+	"brcm,bcm58625-srab",
+	"brcm,bcm88312-srab",
+	"brcm,cygnus-srab",
+	"brcm,nsp-srab",
+	"brcm,omega-srab",
+	"brcm,bcm3384-switch",
+	"brcm,bcm6328-switch",
+	"brcm,bcm6368-switch",
+	"brcm,bcm63xx-switch",
+#endif
+#if IS_ENABLED(CONFIG_NET_DSA_BCM_SF2)
+	"brcm,bcm7445-switch-v4.0",
+	"brcm,bcm7278-switch-v4.0",
+	"brcm,bcm7278-switch-v4.8",
+#endif
+#if IS_ENABLED(CONFIG_NET_DSA_LANTIQ_GSWIP)
+	"lantiq,xrx200-gswip",
+	"lantiq,xrx300-gswip",
+	"lantiq,xrx330-gswip",
+#endif
+#if IS_ENABLED(CONFIG_NET_DSA_MV88E6060)
+	"marvell,mv88e6060",
+#endif
+#if IS_ENABLED(CONFIG_NET_DSA_MV88E6XXX)
+	"marvell,mv88e6085",
+	"marvell,mv88e6190",
+	"marvell,mv88e6250",
+#endif
+#if IS_ENABLED(CONFIG_NET_DSA_MICROCHIP_KSZ_COMMON)
+	"microchip,ksz8765",
+	"microchip,ksz8794",
+	"microchip,ksz8795",
+	"microchip,ksz8863",
+	"microchip,ksz8873",
+	"microchip,ksz9477",
+	"microchip,ksz9897",
+	"microchip,ksz9893",
+	"microchip,ksz9563",
+	"microchip,ksz8563",
+	"microchip,ksz9567",
+#endif
+#if IS_ENABLED(CONFIG_NET_DSA_SMSC_LAN9303_MDIO)
+	"smsc,lan9303-mdio",
+#endif
+#if IS_ENABLED(CONFIG_NET_DSA_SMSC_LAN9303_I2C)
+	"smsc,lan9303-i2c",
+#endif
+	NULL,
+};
+
+static void dsa_shared_port_validate_of(struct dsa_port *dp,
+					bool *missing_phy_mode,
+					bool *missing_link_description)
 {
+	struct device_node *dn = dp->dn, *phy_np;
 	struct dsa_switch *ds = dp->ds;
-	struct device_node *phy_np;
+	phy_interface_t mode;
+
+	*missing_phy_mode = false;
+	*missing_link_description = false;
+
+	if (of_get_phy_mode(dn, &mode)) {
+		*missing_phy_mode = true;
+		dev_err(ds->dev,
+			"OF node %pOF of %s port %d lacks the required \"phy-mode\" property\n",
+			dn, dsa_port_is_cpu(dp) ? "CPU" : "DSA", dp->index);
+	}
+
+	/* Note: of_phy_is_fixed_link() also returns true for
+	 * managed = "in-band-status"
+	 */
+	if (of_phy_is_fixed_link(dn))
+		return;
+
+	phy_np = of_parse_phandle(dn, "phy-handle", 0);
+	if (phy_np) {
+		of_node_put(phy_np);
+		return;
+	}
+
+	*missing_link_description = true;
+
+	dev_err(ds->dev,
+		"OF node %pOF of %s port %d lacks the required \"phy-handle\", \"fixed-link\" or \"managed\" properties\n",
+		dn, dsa_port_is_cpu(dp) ? "CPU" : "DSA", dp->index);
+}
+
+int dsa_shared_port_link_register_of(struct dsa_port *dp)
+{
+	struct dsa_switch *ds = dp->ds;
+	bool missing_link_description;
+	bool missing_phy_mode;
 	int port = dp->index;
 
+	dsa_shared_port_validate_of(dp, &missing_phy_mode,
+				    &missing_link_description);
+
+	if ((missing_phy_mode || missing_link_description) &&
+	    !of_device_compatible_match(ds->dev->of_node,
+					dsa_switches_apply_workarounds))
+		return -EINVAL;
+
 	if (!ds->ops->adjust_link) {
-		phy_np = of_parse_phandle(dp->dn, "phy-handle", 0);
-		if (of_phy_is_fixed_link(dp->dn) || phy_np) {
+		if (missing_link_description) {
+			dev_warn(ds->dev,
+				 "Skipping phylink registration for %s port %d\n",
+				 dsa_port_is_cpu(dp) ? "CPU" : "DSA", dp->index);
+		} else {
 			if (ds->ops->phylink_mac_link_down)
 				ds->ops->phylink_mac_link_down(ds, port,
 					MLO_AN_FIXED, PHY_INTERFACE_MODE_NA);
-			of_node_put(phy_np);
-			return dsa_port_phylink_register(dp);
+
+			return dsa_shared_port_phylink_register(dp);
 		}
-		of_node_put(phy_np);
 		return 0;
 	}
 
@@ -1676,12 +1979,12 @@ int dsa_port_link_register_of(struct dsa_port *dp)
 		 "Using legacy PHYLIB callbacks. Please migrate to PHYLINK!\n");
 
 	if (of_phy_is_fixed_link(dp->dn))
-		return dsa_port_fixed_link_register_of(dp);
+		return dsa_shared_port_fixed_link_register_of(dp);
 	else
-		return dsa_port_setup_phy_of(dp, true);
+		return dsa_shared_port_setup_phy_of(dp, true);
 }
 
-void dsa_port_link_unregister_of(struct dsa_port *dp)
+void dsa_shared_port_link_unregister_of(struct dsa_port *dp)
 {
 	struct dsa_switch *ds = dp->ds;
 
@@ -1689,15 +1992,14 @@ void dsa_port_link_unregister_of(struct dsa_port *dp)
 		rtnl_lock();
 		phylink_disconnect_phy(dp->pl);
 		rtnl_unlock();
-		phylink_destroy(dp->pl);
-		dp->pl = NULL;
+		dsa_port_phylink_destroy(dp);
 		return;
 	}
 
 	if (of_phy_is_fixed_link(dp->dn))
 		of_phy_deregister_fixed_link(dp->dn);
 	else
-		dsa_port_setup_phy_of(dp, false);
+		dsa_shared_port_setup_phy_of(dp, false);
 }
 
 int dsa_port_hsr_join(struct dsa_port *dp, struct net_device *hsr)
diff --git a/net/dsa/slave.c b/net/dsa/slave.c
index 1291c2431d44..1a59918d3b30 100644
--- a/net/dsa/slave.c
+++ b/net/dsa/slave.c
@@ -164,6 +164,48 @@ static int dsa_slave_unsync_mc(struct net_device *dev,
 	return dsa_slave_schedule_standalone_work(dev, DSA_MC_DEL, addr, 0);
 }
 
+void dsa_slave_sync_ha(struct net_device *dev)
+{
+	struct dsa_port *dp = dsa_slave_to_port(dev);
+	struct dsa_switch *ds = dp->ds;
+	struct netdev_hw_addr *ha;
+
+	netif_addr_lock_bh(dev);
+
+	netdev_for_each_synced_mc_addr(ha, dev)
+		dsa_slave_sync_mc(dev, ha->addr);
+
+	netdev_for_each_synced_uc_addr(ha, dev)
+		dsa_slave_sync_uc(dev, ha->addr);
+
+	netif_addr_unlock_bh(dev);
+
+	if (dsa_switch_supports_uc_filtering(ds) ||
+	    dsa_switch_supports_mc_filtering(ds))
+		dsa_flush_workqueue();
+}
+
+void dsa_slave_unsync_ha(struct net_device *dev)
+{
+	struct dsa_port *dp = dsa_slave_to_port(dev);
+	struct dsa_switch *ds = dp->ds;
+	struct netdev_hw_addr *ha;
+
+	netif_addr_lock_bh(dev);
+
+	netdev_for_each_synced_uc_addr(ha, dev)
+		dsa_slave_unsync_uc(dev, ha->addr);
+
+	netdev_for_each_synced_mc_addr(ha, dev)
+		dsa_slave_unsync_mc(dev, ha->addr);
+
+	netif_addr_unlock_bh(dev);
+
+	if (dsa_switch_supports_uc_filtering(ds) ||
+	    dsa_switch_supports_mc_filtering(ds))
+		dsa_flush_workqueue();
+}
+
 /* slave mii_bus handling ***************************************************/
 static int dsa_slave_phy_read(struct mii_bus *bus, int addr, int reg)
 {
@@ -826,9 +868,9 @@ static netdev_tx_t dsa_slave_xmit(struct sk_buff *skb, struct net_device *dev)
 static void dsa_slave_get_drvinfo(struct net_device *dev,
 				  struct ethtool_drvinfo *drvinfo)
 {
-	strlcpy(drvinfo->driver, "dsa", sizeof(drvinfo->driver));
-	strlcpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version));
-	strlcpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info));
+	strscpy(drvinfo->driver, "dsa", sizeof(drvinfo->driver));
+	strscpy(drvinfo->fw_version, "N/A", sizeof(drvinfo->fw_version));
+	strscpy(drvinfo->bus_info, "platform", sizeof(drvinfo->bus_info));
 }
 
 static int dsa_slave_get_regs_len(struct net_device *dev)
@@ -1503,8 +1545,7 @@ static int dsa_slave_setup_tc_block(struct net_device *dev,
 static int dsa_slave_setup_ft_block(struct dsa_switch *ds, int port,
 				    void *type_data)
 {
-	struct dsa_port *cpu_dp = dsa_to_port(ds, port)->cpu_dp;
-	struct net_device *master = cpu_dp->master;
+	struct net_device *master = dsa_port_to_master(dsa_to_port(ds, port));
 
 	if (!master->netdev_ops->ndo_setup_tc)
 		return -EOPNOTSUPP;
@@ -2147,13 +2188,14 @@ static int dsa_slave_fill_forward_path(struct net_device_path_ctx *ctx,
 				       struct net_device_path *path)
 {
 	struct dsa_port *dp = dsa_slave_to_port(ctx->dev);
+	struct net_device *master = dsa_port_to_master(dp);
 	struct dsa_port *cpu_dp = dp->cpu_dp;
 
 	path->dev = ctx->dev;
 	path->type = DEV_PATH_DSA;
 	path->dsa.proto = cpu_dp->tag_ops->proto;
 	path->dsa.port = dp->index;
-	ctx->dev = cpu_dp->master;
+	ctx->dev = master;
 
 	return 0;
 }
@@ -2262,7 +2304,7 @@ static int dsa_slave_phy_setup(struct net_device *slave_dev)
 	if (ret) {
 		netdev_err(slave_dev, "failed to connect to PHY: %pe\n",
 			   ERR_PTR(ret));
-		phylink_destroy(dp->pl);
+		dsa_port_phylink_destroy(dp);
 	}
 
 	return ret;
@@ -2271,9 +2313,9 @@ static int dsa_slave_phy_setup(struct net_device *slave_dev)
 void dsa_slave_setup_tagger(struct net_device *slave)
 {
 	struct dsa_port *dp = dsa_slave_to_port(slave);
+	struct net_device *master = dsa_port_to_master(dp);
 	struct dsa_slave_priv *p = netdev_priv(slave);
 	const struct dsa_port *cpu_dp = dp->cpu_dp;
-	struct net_device *master = cpu_dp->master;
 	const struct dsa_switch *ds = dp->ds;
 
 	slave->needed_headroom = cpu_dp->tag_ops->needed_headroom;
@@ -2330,8 +2372,7 @@ int dsa_slave_resume(struct net_device *slave_dev)
 
 int dsa_slave_create(struct dsa_port *port)
 {
-	const struct dsa_port *cpu_dp = port->cpu_dp;
-	struct net_device *master = cpu_dp->master;
+	struct net_device *master = dsa_port_to_master(port);
 	struct dsa_switch *ds = port->ds;
 	const char *name = port->name;
 	struct net_device *slave_dev;
@@ -2347,6 +2388,7 @@ int dsa_slave_create(struct dsa_port *port)
 	if (slave_dev == NULL)
 		return -ENOMEM;
 
+	slave_dev->rtnl_link_ops = &dsa_link_ops;
 	slave_dev->ethtool_ops = &dsa_slave_ethtool_ops;
 #if IS_ENABLED(CONFIG_DCB)
 	slave_dev->dcbnl_ops = &dsa_slave_dcbnl_ops;
@@ -2434,7 +2476,7 @@ out_phy:
 	rtnl_lock();
 	phylink_disconnect_phy(p->dp->pl);
 	rtnl_unlock();
-	phylink_destroy(p->dp->pl);
+	dsa_port_phylink_destroy(p->dp);
 out_gcells:
 	gro_cells_destroy(&p->gcells);
 out_free:
@@ -2457,12 +2499,89 @@ void dsa_slave_destroy(struct net_device *slave_dev)
 	phylink_disconnect_phy(dp->pl);
 	rtnl_unlock();
 
-	phylink_destroy(dp->pl);
+	dsa_port_phylink_destroy(dp);
 	gro_cells_destroy(&p->gcells);
 	free_percpu(slave_dev->tstats);
 	free_netdev(slave_dev);
 }
 
+int dsa_slave_change_master(struct net_device *dev, struct net_device *master,
+			    struct netlink_ext_ack *extack)
+{
+	struct net_device *old_master = dsa_slave_to_master(dev);
+	struct dsa_port *dp = dsa_slave_to_port(dev);
+	struct dsa_switch *ds = dp->ds;
+	struct net_device *upper;
+	struct list_head *iter;
+	int err;
+
+	if (master == old_master)
+		return 0;
+
+	if (!ds->ops->port_change_master) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Driver does not support changing DSA master");
+		return -EOPNOTSUPP;
+	}
+
+	if (!netdev_uses_dsa(master)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Interface not eligible as DSA master");
+		return -EOPNOTSUPP;
+	}
+
+	netdev_for_each_upper_dev_rcu(master, upper, iter) {
+		if (dsa_slave_dev_check(upper))
+			continue;
+		if (netif_is_bridge_master(upper))
+			continue;
+		NL_SET_ERR_MSG_MOD(extack, "Cannot join master with unknown uppers");
+		return -EOPNOTSUPP;
+	}
+
+	/* Since we allow live-changing the DSA master, plus we auto-open the
+	 * DSA master when the user port opens => we need to ensure that the
+	 * new DSA master is open too.
+	 */
+	if (dev->flags & IFF_UP) {
+		err = dev_open(master, extack);
+		if (err)
+			return err;
+	}
+
+	netdev_upper_dev_unlink(old_master, dev);
+
+	err = netdev_upper_dev_link(master, dev, extack);
+	if (err)
+		goto out_revert_old_master_unlink;
+
+	err = dsa_port_change_master(dp, master, extack);
+	if (err)
+		goto out_revert_master_link;
+
+	/* Update the MTU of the new CPU port through cross-chip notifiers */
+	err = dsa_slave_change_mtu(dev, dev->mtu);
+	if (err && err != -EOPNOTSUPP) {
+		netdev_warn(dev,
+			    "nonfatal error updating MTU with new master: %pe\n",
+			    ERR_PTR(err));
+	}
+
+	/* If the port doesn't have its own MAC address and relies on the DSA
+	 * master's one, inherit it again from the new DSA master.
+	 */
+	if (is_zero_ether_addr(dp->mac))
+		eth_hw_addr_inherit(dev, master);
+
+	return 0;
+
+out_revert_master_link:
+	netdev_upper_dev_unlink(master, dev);
+out_revert_old_master_unlink:
+	netdev_upper_dev_link(old_master, dev, NULL);
+	return err;
+}
+
 bool dsa_slave_dev_check(const struct net_device *dev)
 {
 	return dev->netdev_ops == &dsa_slave_netdev_ops;
@@ -2476,6 +2595,9 @@ static int dsa_slave_changeupper(struct net_device *dev,
 	struct netlink_ext_ack *extack;
 	int err = NOTIFY_DONE;
 
+	if (!dsa_slave_dev_check(dev))
+		return err;
+
 	extack = netdev_notifier_info_to_extack(&info->info);
 
 	if (netif_is_bridge_master(info->upper_dev)) {
@@ -2531,6 +2653,9 @@ static int dsa_slave_prechangeupper(struct net_device *dev,
 {
 	struct dsa_port *dp = dsa_slave_to_port(dev);
 
+	if (!dsa_slave_dev_check(dev))
+		return NOTIFY_DONE;
+
 	if (netif_is_bridge_master(info->upper_dev) && !info->linking)
 		dsa_port_pre_bridge_leave(dp, info->upper_dev);
 	else if (netif_is_lag_master(info->upper_dev) && !info->linking)
@@ -2551,6 +2676,9 @@ dsa_slave_lag_changeupper(struct net_device *dev,
 	int err = NOTIFY_DONE;
 	struct dsa_port *dp;
 
+	if (!netif_is_lag_master(dev))
+		return err;
+
 	netdev_for_each_lower_dev(dev, lower, iter) {
 		if (!dsa_slave_dev_check(lower))
 			continue;
@@ -2580,6 +2708,9 @@ dsa_slave_lag_prechangeupper(struct net_device *dev,
 	int err = NOTIFY_DONE;
 	struct dsa_port *dp;
 
+	if (!netif_is_lag_master(dev))
+		return err;
+
 	netdev_for_each_lower_dev(dev, lower, iter) {
 		if (!dsa_slave_dev_check(lower))
 			continue;
@@ -2687,6 +2818,277 @@ dsa_slave_prechangeupper_sanity_check(struct net_device *dev,
 	return NOTIFY_DONE;
 }
 
+/* To be eligible as a DSA master, a LAG must have all lower interfaces be
+ * eligible DSA masters. Additionally, all LAG slaves must be DSA masters of
+ * switches in the same switch tree.
+ */
+static int dsa_lag_master_validate(struct net_device *lag_dev,
+				   struct netlink_ext_ack *extack)
+{
+	struct net_device *lower1, *lower2;
+	struct list_head *iter1, *iter2;
+
+	netdev_for_each_lower_dev(lag_dev, lower1, iter1) {
+		netdev_for_each_lower_dev(lag_dev, lower2, iter2) {
+			if (!netdev_uses_dsa(lower1) ||
+			    !netdev_uses_dsa(lower2)) {
+				NL_SET_ERR_MSG_MOD(extack,
+						   "All LAG ports must be eligible as DSA masters");
+				return notifier_from_errno(-EINVAL);
+			}
+
+			if (lower1 == lower2)
+				continue;
+
+			if (!dsa_port_tree_same(lower1->dsa_ptr,
+						lower2->dsa_ptr)) {
+				NL_SET_ERR_MSG_MOD(extack,
+						   "LAG contains DSA masters of disjoint switch trees");
+				return notifier_from_errno(-EINVAL);
+			}
+		}
+	}
+
+	return NOTIFY_DONE;
+}
+
+static int
+dsa_master_prechangeupper_sanity_check(struct net_device *master,
+				       struct netdev_notifier_changeupper_info *info)
+{
+	struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(&info->info);
+
+	if (!netdev_uses_dsa(master))
+		return NOTIFY_DONE;
+
+	if (!info->linking)
+		return NOTIFY_DONE;
+
+	/* Allow DSA switch uppers */
+	if (dsa_slave_dev_check(info->upper_dev))
+		return NOTIFY_DONE;
+
+	/* Allow bridge uppers of DSA masters, subject to further
+	 * restrictions in dsa_bridge_prechangelower_sanity_check()
+	 */
+	if (netif_is_bridge_master(info->upper_dev))
+		return NOTIFY_DONE;
+
+	/* Allow LAG uppers, subject to further restrictions in
+	 * dsa_lag_master_prechangelower_sanity_check()
+	 */
+	if (netif_is_lag_master(info->upper_dev))
+		return dsa_lag_master_validate(info->upper_dev, extack);
+
+	NL_SET_ERR_MSG_MOD(extack,
+			   "DSA master cannot join unknown upper interfaces");
+	return notifier_from_errno(-EBUSY);
+}
+
+static int
+dsa_lag_master_prechangelower_sanity_check(struct net_device *dev,
+					   struct netdev_notifier_changeupper_info *info)
+{
+	struct netlink_ext_ack *extack = netdev_notifier_info_to_extack(&info->info);
+	struct net_device *lag_dev = info->upper_dev;
+	struct net_device *lower;
+	struct list_head *iter;
+
+	if (!netdev_uses_dsa(lag_dev) || !netif_is_lag_master(lag_dev))
+		return NOTIFY_DONE;
+
+	if (!info->linking)
+		return NOTIFY_DONE;
+
+	if (!netdev_uses_dsa(dev)) {
+		NL_SET_ERR_MSG(extack,
+			       "Only DSA masters can join a LAG DSA master");
+		return notifier_from_errno(-EINVAL);
+	}
+
+	netdev_for_each_lower_dev(lag_dev, lower, iter) {
+		if (!dsa_port_tree_same(dev->dsa_ptr, lower->dsa_ptr)) {
+			NL_SET_ERR_MSG(extack,
+				       "Interface is DSA master for a different switch tree than this LAG");
+			return notifier_from_errno(-EINVAL);
+		}
+
+		break;
+	}
+
+	return NOTIFY_DONE;
+}
+
+/* Don't allow bridging of DSA masters, since the bridge layer rx_handler
+ * prevents the DSA fake ethertype handler to be invoked, so we don't get the
+ * chance to strip off and parse the DSA switch tag protocol header (the bridge
+ * layer just returns RX_HANDLER_CONSUMED, stopping RX processing for these
+ * frames).
+ * The only case where that would not be an issue is when bridging can already
+ * be offloaded, such as when the DSA master is itself a DSA or plain switchdev
+ * port, and is bridged only with other ports from the same hardware device.
+ */
+static int
+dsa_bridge_prechangelower_sanity_check(struct net_device *new_lower,
+				       struct netdev_notifier_changeupper_info *info)
+{
+	struct net_device *br = info->upper_dev;
+	struct netlink_ext_ack *extack;
+	struct net_device *lower;
+	struct list_head *iter;
+
+	if (!netif_is_bridge_master(br))
+		return NOTIFY_DONE;
+
+	if (!info->linking)
+		return NOTIFY_DONE;
+
+	extack = netdev_notifier_info_to_extack(&info->info);
+
+	netdev_for_each_lower_dev(br, lower, iter) {
+		if (!netdev_uses_dsa(new_lower) && !netdev_uses_dsa(lower))
+			continue;
+
+		if (!netdev_port_same_parent_id(lower, new_lower)) {
+			NL_SET_ERR_MSG(extack,
+				       "Cannot do software bridging with a DSA master");
+			return notifier_from_errno(-EINVAL);
+		}
+	}
+
+	return NOTIFY_DONE;
+}
+
+static void dsa_tree_migrate_ports_from_lag_master(struct dsa_switch_tree *dst,
+						   struct net_device *lag_dev)
+{
+	struct net_device *new_master = dsa_tree_find_first_master(dst);
+	struct dsa_port *dp;
+	int err;
+
+	dsa_tree_for_each_user_port(dp, dst) {
+		if (dsa_port_to_master(dp) != lag_dev)
+			continue;
+
+		err = dsa_slave_change_master(dp->slave, new_master, NULL);
+		if (err) {
+			netdev_err(dp->slave,
+				   "failed to restore master to %s: %pe\n",
+				   new_master->name, ERR_PTR(err));
+		}
+	}
+}
+
+static int dsa_master_lag_join(struct net_device *master,
+			       struct net_device *lag_dev,
+			       struct netdev_lag_upper_info *uinfo,
+			       struct netlink_ext_ack *extack)
+{
+	struct dsa_port *cpu_dp = master->dsa_ptr;
+	struct dsa_switch_tree *dst = cpu_dp->dst;
+	struct dsa_port *dp;
+	int err;
+
+	err = dsa_master_lag_setup(lag_dev, cpu_dp, uinfo, extack);
+	if (err)
+		return err;
+
+	dsa_tree_for_each_user_port(dp, dst) {
+		if (dsa_port_to_master(dp) != master)
+			continue;
+
+		err = dsa_slave_change_master(dp->slave, lag_dev, extack);
+		if (err)
+			goto restore;
+	}
+
+	return 0;
+
+restore:
+	dsa_tree_for_each_user_port_continue_reverse(dp, dst) {
+		if (dsa_port_to_master(dp) != lag_dev)
+			continue;
+
+		err = dsa_slave_change_master(dp->slave, master, NULL);
+		if (err) {
+			netdev_err(dp->slave,
+				   "failed to restore master to %s: %pe\n",
+				   master->name, ERR_PTR(err));
+		}
+	}
+
+	dsa_master_lag_teardown(lag_dev, master->dsa_ptr);
+
+	return err;
+}
+
+static void dsa_master_lag_leave(struct net_device *master,
+				 struct net_device *lag_dev)
+{
+	struct dsa_port *dp, *cpu_dp = lag_dev->dsa_ptr;
+	struct dsa_switch_tree *dst = cpu_dp->dst;
+	struct dsa_port *new_cpu_dp = NULL;
+	struct net_device *lower;
+	struct list_head *iter;
+
+	netdev_for_each_lower_dev(lag_dev, lower, iter) {
+		if (netdev_uses_dsa(lower)) {
+			new_cpu_dp = lower->dsa_ptr;
+			break;
+		}
+	}
+
+	if (new_cpu_dp) {
+		/* Update the CPU port of the user ports still under the LAG
+		 * so that dsa_port_to_master() continues to work properly
+		 */
+		dsa_tree_for_each_user_port(dp, dst)
+			if (dsa_port_to_master(dp) == lag_dev)
+				dp->cpu_dp = new_cpu_dp;
+
+		/* Update the index of the virtual CPU port to match the lowest
+		 * physical CPU port
+		 */
+		lag_dev->dsa_ptr = new_cpu_dp;
+		wmb();
+	} else {
+		/* If the LAG DSA master has no ports left, migrate back all
+		 * user ports to the first physical CPU port
+		 */
+		dsa_tree_migrate_ports_from_lag_master(dst, lag_dev);
+	}
+
+	/* This DSA master has left its LAG in any case, so let
+	 * the CPU port leave the hardware LAG as well
+	 */
+	dsa_master_lag_teardown(lag_dev, master->dsa_ptr);
+}
+
+static int dsa_master_changeupper(struct net_device *dev,
+				  struct netdev_notifier_changeupper_info *info)
+{
+	struct netlink_ext_ack *extack;
+	int err = NOTIFY_DONE;
+
+	if (!netdev_uses_dsa(dev))
+		return err;
+
+	extack = netdev_notifier_info_to_extack(&info->info);
+
+	if (netif_is_lag_master(info->upper_dev)) {
+		if (info->linking) {
+			err = dsa_master_lag_join(dev, info->upper_dev,
+						  info->upper_info, extack);
+			err = notifier_from_errno(err);
+		} else {
+			dsa_master_lag_leave(dev, info->upper_dev);
+			err = NOTIFY_OK;
+		}
+	}
+
+	return err;
+}
+
 static int dsa_slave_netdevice_event(struct notifier_block *nb,
 				     unsigned long event, void *ptr)
 {
@@ -2698,36 +3100,68 @@ static int dsa_slave_netdevice_event(struct notifier_block *nb,
 		int err;
 
 		err = dsa_slave_prechangeupper_sanity_check(dev, info);
-		if (err != NOTIFY_DONE)
+		if (notifier_to_errno(err))
+			return err;
+
+		err = dsa_master_prechangeupper_sanity_check(dev, info);
+		if (notifier_to_errno(err))
+			return err;
+
+		err = dsa_lag_master_prechangelower_sanity_check(dev, info);
+		if (notifier_to_errno(err))
+			return err;
+
+		err = dsa_bridge_prechangelower_sanity_check(dev, info);
+		if (notifier_to_errno(err))
 			return err;
 
-		if (dsa_slave_dev_check(dev))
-			return dsa_slave_prechangeupper(dev, ptr);
+		err = dsa_slave_prechangeupper(dev, ptr);
+		if (notifier_to_errno(err))
+			return err;
 
-		if (netif_is_lag_master(dev))
-			return dsa_slave_lag_prechangeupper(dev, ptr);
+		err = dsa_slave_lag_prechangeupper(dev, ptr);
+		if (notifier_to_errno(err))
+			return err;
 
 		break;
 	}
-	case NETDEV_CHANGEUPPER:
-		if (dsa_slave_dev_check(dev))
-			return dsa_slave_changeupper(dev, ptr);
+	case NETDEV_CHANGEUPPER: {
+		int err;
 
-		if (netif_is_lag_master(dev))
-			return dsa_slave_lag_changeupper(dev, ptr);
+		err = dsa_slave_changeupper(dev, ptr);
+		if (notifier_to_errno(err))
+			return err;
+
+		err = dsa_slave_lag_changeupper(dev, ptr);
+		if (notifier_to_errno(err))
+			return err;
+
+		err = dsa_master_changeupper(dev, ptr);
+		if (notifier_to_errno(err))
+			return err;
 
 		break;
+	}
 	case NETDEV_CHANGELOWERSTATE: {
 		struct netdev_notifier_changelowerstate_info *info = ptr;
 		struct dsa_port *dp;
 		int err;
 
-		if (!dsa_slave_dev_check(dev))
-			break;
+		if (dsa_slave_dev_check(dev)) {
+			dp = dsa_slave_to_port(dev);
 
-		dp = dsa_slave_to_port(dev);
+			err = dsa_port_lag_change(dp, info->lower_state_info);
+		}
+
+		/* Mirror LAG port events on DSA masters that are in
+		 * a LAG towards their respective switch CPU ports
+		 */
+		if (netdev_uses_dsa(dev)) {
+			dp = dev->dsa_ptr;
+
+			err = dsa_port_lag_change(dp, info->lower_state_info);
+		}
 
-		err = dsa_port_lag_change(dp, info->lower_state_info);
 		return notifier_from_errno(err);
 	}
 	case NETDEV_CHANGE:
@@ -2777,6 +3211,9 @@ static int dsa_slave_netdevice_event(struct notifier_block *nb,
 			if (!dsa_port_is_user(dp))
 				continue;
 
+			if (dp->cpu_dp != cpu_dp)
+				continue;
+
 			list_add(&dp->slave->close_list, &close_list);
 		}
 
diff --git a/net/dsa/switch.c b/net/dsa/switch.c
index 4dfd68cf61c5..ce56acdba203 100644
--- a/net/dsa/switch.c
+++ b/net/dsa/switch.c
@@ -398,8 +398,15 @@ static int dsa_switch_host_fdb_add(struct dsa_switch *ds,
 
 	dsa_switch_for_each_port(dp, ds) {
 		if (dsa_port_host_address_match(dp, info->dp)) {
-			err = dsa_port_do_fdb_add(dp, info->addr, info->vid,
-						  info->db);
+			if (dsa_port_is_cpu(dp) && info->dp->cpu_port_in_lag) {
+				err = dsa_switch_do_lag_fdb_add(ds, dp->lag,
+								info->addr,
+								info->vid,
+								info->db);
+			} else {
+				err = dsa_port_do_fdb_add(dp, info->addr,
+							  info->vid, info->db);
+			}
 			if (err)
 				break;
 		}
@@ -419,8 +426,15 @@ static int dsa_switch_host_fdb_del(struct dsa_switch *ds,
 
 	dsa_switch_for_each_port(dp, ds) {
 		if (dsa_port_host_address_match(dp, info->dp)) {
-			err = dsa_port_do_fdb_del(dp, info->addr, info->vid,
-						  info->db);
+			if (dsa_port_is_cpu(dp) && info->dp->cpu_port_in_lag) {
+				err = dsa_switch_do_lag_fdb_del(ds, dp->lag,
+								info->addr,
+								info->vid,
+								info->db);
+			} else {
+				err = dsa_port_do_fdb_del(dp, info->addr,
+							  info->vid, info->db);
+			}
 			if (err)
 				break;
 		}
@@ -507,12 +521,12 @@ static int dsa_switch_lag_join(struct dsa_switch *ds,
 {
 	if (info->dp->ds == ds && ds->ops->port_lag_join)
 		return ds->ops->port_lag_join(ds, info->dp->index, info->lag,
-					      info->info);
+					      info->info, info->extack);
 
 	if (info->dp->ds != ds && ds->ops->crosschip_lag_join)
 		return ds->ops->crosschip_lag_join(ds, info->dp->ds->index,
 						   info->dp->index, info->lag,
-						   info->info);
+						   info->info, info->extack);
 
 	return -EOPNOTSUPP;
 }
diff --git a/net/dsa/tag_8021q.c b/net/dsa/tag_8021q.c
index 01a427800797..34e5ec5d3e23 100644
--- a/net/dsa/tag_8021q.c
+++ b/net/dsa/tag_8021q.c
@@ -2,9 +2,7 @@
 /* Copyright (c) 2019, Vladimir Oltean <olteanv@gmail.com>
  *
  * This module is not a complete tagger implementation. It only provides
- * primitives for taggers that rely on 802.1Q VLAN tags to use. The
- * dsa_8021q_netdev_ops is registered for API compliance and not used
- * directly by callers.
+ * primitives for taggers that rely on 802.1Q VLAN tags to use.
  */
 #include <linux/if_vlan.h>
 #include <linux/dsa/8021q.h>
@@ -332,7 +330,7 @@ static int dsa_tag_8021q_port_setup(struct dsa_switch *ds, int port)
 	if (!dsa_port_is_user(dp))
 		return 0;
 
-	master = dp->cpu_dp->master;
+	master = dsa_port_to_master(dp);
 
 	err = dsa_port_tag_8021q_vlan_add(dp, vid, false);
 	if (err) {
@@ -361,7 +359,7 @@ static void dsa_tag_8021q_port_teardown(struct dsa_switch *ds, int port)
 	if (!dsa_port_is_user(dp))
 		return;
 
-	master = dp->cpu_dp->master;
+	master = dsa_port_to_master(dp);
 
 	dsa_port_tag_8021q_vlan_del(dp, vid, false);
 
diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c
index 62b89d6f54fd..e02daa74e833 100644
--- a/net/ethernet/eth.c
+++ b/net/ethernet/eth.c
@@ -414,12 +414,9 @@ struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb)
 
 	off_eth = skb_gro_offset(skb);
 	hlen = off_eth + sizeof(*eh);
-	eh = skb_gro_header_fast(skb, off_eth);
-	if (skb_gro_header_hard(skb, hlen)) {
-		eh = skb_gro_header_slow(skb, hlen, off_eth);
-		if (unlikely(!eh))
-			goto out;
-	}
+	eh = skb_gro_header(skb, hlen, off_eth);
+	if (unlikely(!eh))
+		goto out;
 
 	flush = 0;
 
diff --git a/net/ethtool/Makefile b/net/ethtool/Makefile
index b76432e70e6b..72ab0944262a 100644
--- a/net/ethtool/Makefile
+++ b/net/ethtool/Makefile
@@ -7,4 +7,5 @@ obj-$(CONFIG_ETHTOOL_NETLINK)	+= ethtool_nl.o
 ethtool_nl-y	:= netlink.o bitset.o strset.o linkinfo.o linkmodes.o \
 		   linkstate.o debug.o wol.o features.o privflags.o rings.o \
 		   channels.o coalesce.o pause.o eee.o tsinfo.o cabletest.o \
-		   tunnels.o fec.o eeprom.o stats.o phc_vclocks.o module.o
+		   tunnels.o fec.o eeprom.o stats.o phc_vclocks.o module.o \
+		   pse-pd.o
diff --git a/net/ethtool/common.h b/net/ethtool/common.h
index 2dc2b80aea5f..c1779657e074 100644
--- a/net/ethtool/common.h
+++ b/net/ethtool/common.h
@@ -46,6 +46,7 @@ int ethtool_get_max_rxfh_channel(struct net_device *dev, u32 *max);
 int __ethtool_get_ts_info(struct net_device *dev, struct ethtool_ts_info *info);
 
 extern const struct ethtool_phy_ops *ethtool_phy_ops;
+extern const struct ethtool_pse_ops *ethtool_pse_ops;
 
 int ethtool_get_module_info_call(struct net_device *dev,
 				 struct ethtool_modinfo *modinfo);
diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c
index 6a7308de192d..57e7238a4136 100644
--- a/net/ethtool/ioctl.c
+++ b/net/ethtool/ioctl.c
@@ -571,6 +571,7 @@ static int ethtool_get_link_ksettings(struct net_device *dev,
 		= __ETHTOOL_LINK_MODE_MASK_NU32;
 	link_ksettings.base.master_slave_cfg = MASTER_SLAVE_CFG_UNSUPPORTED;
 	link_ksettings.base.master_slave_state = MASTER_SLAVE_STATE_UNSUPPORTED;
+	link_ksettings.base.rate_matching = RATE_MATCH_NONE;
 
 	return store_link_ksettings_for_user(useraddr, &link_ksettings);
 }
@@ -714,16 +715,16 @@ ethtool_get_drvinfo(struct net_device *dev, struct ethtool_devlink_compat *rsp)
 	const struct ethtool_ops *ops = dev->ethtool_ops;
 
 	rsp->info.cmd = ETHTOOL_GDRVINFO;
-	strlcpy(rsp->info.version, UTS_RELEASE, sizeof(rsp->info.version));
+	strscpy(rsp->info.version, UTS_RELEASE, sizeof(rsp->info.version));
 	if (ops->get_drvinfo) {
 		ops->get_drvinfo(dev, &rsp->info);
 	} else if (dev->dev.parent && dev->dev.parent->driver) {
-		strlcpy(rsp->info.bus_info, dev_name(dev->dev.parent),
+		strscpy(rsp->info.bus_info, dev_name(dev->dev.parent),
 			sizeof(rsp->info.bus_info));
-		strlcpy(rsp->info.driver, dev->dev.parent->driver->name,
+		strscpy(rsp->info.driver, dev->dev.parent->driver->name,
 			sizeof(rsp->info.driver));
 	} else if (dev->rtnl_link_ops) {
-		strlcpy(rsp->info.driver, dev->rtnl_link_ops->kind,
+		strscpy(rsp->info.driver, dev->rtnl_link_ops->kind,
 			sizeof(rsp->info.driver));
 	} else {
 		return -EOPNOTSUPP;
diff --git a/net/ethtool/linkmodes.c b/net/ethtool/linkmodes.c
index 99b29b4fe947..126e06c713a3 100644
--- a/net/ethtool/linkmodes.c
+++ b/net/ethtool/linkmodes.c
@@ -70,6 +70,7 @@ static int linkmodes_reply_size(const struct ethnl_req_info *req_base,
 		+ nla_total_size(sizeof(u32)) /* LINKMODES_SPEED */
 		+ nla_total_size(sizeof(u32)) /* LINKMODES_LANES */
 		+ nla_total_size(sizeof(u8)) /* LINKMODES_DUPLEX */
+		+ nla_total_size(sizeof(u8)) /* LINKMODES_RATE_MATCHING */
 		+ 0;
 	ret = ethnl_bitset_size(ksettings->link_modes.advertising,
 				ksettings->link_modes.supported,
@@ -143,6 +144,10 @@ static int linkmodes_fill_reply(struct sk_buff *skb,
 		       lsettings->master_slave_state))
 		return -EMSGSIZE;
 
+	if (nla_put_u8(skb, ETHTOOL_A_LINKMODES_RATE_MATCHING,
+		       lsettings->rate_matching))
+		return -EMSGSIZE;
+
 	return 0;
 }
 
diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c
index e26079e11835..1a4c11356c96 100644
--- a/net/ethtool/netlink.c
+++ b/net/ethtool/netlink.c
@@ -286,6 +286,7 @@ ethnl_default_requests[__ETHTOOL_MSG_USER_CNT] = {
 	[ETHTOOL_MSG_STATS_GET]		= &ethnl_stats_request_ops,
 	[ETHTOOL_MSG_PHC_VCLOCKS_GET]	= &ethnl_phc_vclocks_request_ops,
 	[ETHTOOL_MSG_MODULE_GET]	= &ethnl_module_request_ops,
+	[ETHTOOL_MSG_PSE_GET]		= &ethnl_pse_request_ops,
 };
 
 static struct ethnl_dump_ctx *ethnl_dump_context(struct netlink_callback *cb)
@@ -361,6 +362,9 @@ static int ethnl_default_doit(struct sk_buff *skb, struct genl_info *info)
 	ops = ethnl_default_requests[cmd];
 	if (WARN_ONCE(!ops, "cmd %u has no ethnl_request_ops\n", cmd))
 		return -EOPNOTSUPP;
+	if (GENL_REQ_ATTR_CHECK(info, ops->hdr_attr))
+		return -EINVAL;
+
 	req_info = kzalloc(ops->req_info_size, GFP_KERNEL);
 	if (!req_info)
 		return -ENOMEM;
@@ -1020,6 +1024,22 @@ static const struct genl_ops ethtool_genl_ops[] = {
 		.policy = ethnl_module_set_policy,
 		.maxattr = ARRAY_SIZE(ethnl_module_set_policy) - 1,
 	},
+	{
+		.cmd	= ETHTOOL_MSG_PSE_GET,
+		.doit	= ethnl_default_doit,
+		.start	= ethnl_default_start,
+		.dumpit	= ethnl_default_dumpit,
+		.done	= ethnl_default_done,
+		.policy = ethnl_pse_get_policy,
+		.maxattr = ARRAY_SIZE(ethnl_pse_get_policy) - 1,
+	},
+	{
+		.cmd	= ETHTOOL_MSG_PSE_SET,
+		.flags	= GENL_UNS_ADMIN_PERM,
+		.doit	= ethnl_set_pse,
+		.policy = ethnl_pse_set_policy,
+		.maxattr = ARRAY_SIZE(ethnl_pse_set_policy) - 1,
+	},
 };
 
 static const struct genl_multicast_group ethtool_nl_mcgrps[] = {
@@ -1033,6 +1053,7 @@ static struct genl_family ethtool_genl_family __ro_after_init = {
 	.parallel_ops	= true,
 	.ops		= ethtool_genl_ops,
 	.n_ops		= ARRAY_SIZE(ethtool_genl_ops),
+	.resv_start_op	= ETHTOOL_MSG_MODULE_GET + 1,
 	.mcgrps		= ethtool_nl_mcgrps,
 	.n_mcgrps	= ARRAY_SIZE(ethtool_nl_mcgrps),
 };
diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h
index c0d587611854..1bfd374f9718 100644
--- a/net/ethtool/netlink.h
+++ b/net/ethtool/netlink.h
@@ -345,6 +345,7 @@ extern const struct ethnl_request_ops ethnl_module_eeprom_request_ops;
 extern const struct ethnl_request_ops ethnl_stats_request_ops;
 extern const struct ethnl_request_ops ethnl_phc_vclocks_request_ops;
 extern const struct ethnl_request_ops ethnl_module_request_ops;
+extern const struct ethnl_request_ops ethnl_pse_request_ops;
 
 extern const struct nla_policy ethnl_header_policy[ETHTOOL_A_HEADER_FLAGS + 1];
 extern const struct nla_policy ethnl_header_policy_stats[ETHTOOL_A_HEADER_FLAGS + 1];
@@ -383,6 +384,8 @@ extern const struct nla_policy ethnl_stats_get_policy[ETHTOOL_A_STATS_GROUPS + 1
 extern const struct nla_policy ethnl_phc_vclocks_get_policy[ETHTOOL_A_PHC_VCLOCKS_HEADER + 1];
 extern const struct nla_policy ethnl_module_get_policy[ETHTOOL_A_MODULE_HEADER + 1];
 extern const struct nla_policy ethnl_module_set_policy[ETHTOOL_A_MODULE_POWER_MODE_POLICY + 1];
+extern const struct nla_policy ethnl_pse_get_policy[ETHTOOL_A_PSE_HEADER + 1];
+extern const struct nla_policy ethnl_pse_set_policy[ETHTOOL_A_PSE_MAX + 1];
 
 int ethnl_set_linkinfo(struct sk_buff *skb, struct genl_info *info);
 int ethnl_set_linkmodes(struct sk_buff *skb, struct genl_info *info);
@@ -402,6 +405,7 @@ int ethnl_tunnel_info_start(struct netlink_callback *cb);
 int ethnl_tunnel_info_dumpit(struct sk_buff *skb, struct netlink_callback *cb);
 int ethnl_set_fec(struct sk_buff *skb, struct genl_info *info);
 int ethnl_set_module(struct sk_buff *skb, struct genl_info *info);
+int ethnl_set_pse(struct sk_buff *skb, struct genl_info *info);
 
 extern const char stats_std_names[__ETHTOOL_STATS_CNT][ETH_GSTRING_LEN];
 extern const char stats_eth_phy_names[__ETHTOOL_A_STATS_ETH_PHY_CNT][ETH_GSTRING_LEN];
diff --git a/net/ethtool/pse-pd.c b/net/ethtool/pse-pd.c
new file mode 100644
index 000000000000..5a471e115b66
--- /dev/null
+++ b/net/ethtool/pse-pd.c
@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// ethtool interface for for Ethernet PSE (Power Sourcing Equipment)
+// and PD (Powered Device)
+//
+// Copyright (c) 2022 Pengutronix, Oleksij Rempel <kernel@pengutronix.de>
+//
+
+#include "common.h"
+#include "linux/pse-pd/pse.h"
+#include "netlink.h"
+#include <linux/ethtool_netlink.h>
+#include <linux/ethtool.h>
+#include <linux/phy.h>
+
+struct pse_req_info {
+	struct ethnl_req_info base;
+};
+
+struct pse_reply_data {
+	struct ethnl_reply_data	base;
+	struct pse_control_status status;
+};
+
+#define PSE_REPDATA(__reply_base) \
+	container_of(__reply_base, struct pse_reply_data, base)
+
+/* PSE_GET */
+
+const struct nla_policy ethnl_pse_get_policy[ETHTOOL_A_PSE_HEADER + 1] = {
+	[ETHTOOL_A_PSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy),
+};
+
+static int pse_get_pse_attributes(struct net_device *dev,
+				  struct netlink_ext_ack *extack,
+				  struct pse_reply_data *data)
+{
+	struct phy_device *phydev = dev->phydev;
+
+	if (!phydev) {
+		NL_SET_ERR_MSG(extack, "No PHY is attached");
+		return -EOPNOTSUPP;
+	}
+
+	if (!phydev->psec) {
+		NL_SET_ERR_MSG(extack, "No PSE is attached");
+		return -EOPNOTSUPP;
+	}
+
+	memset(&data->status, 0, sizeof(data->status));
+
+	return pse_ethtool_get_status(phydev->psec, extack, &data->status);
+}
+
+static int pse_prepare_data(const struct ethnl_req_info *req_base,
+			       struct ethnl_reply_data *reply_base,
+			       struct genl_info *info)
+{
+	struct pse_reply_data *data = PSE_REPDATA(reply_base);
+	struct net_device *dev = reply_base->dev;
+	int ret;
+
+	ret = ethnl_ops_begin(dev);
+	if (ret < 0)
+		return ret;
+
+	ret = pse_get_pse_attributes(dev, info->extack, data);
+
+	ethnl_ops_complete(dev);
+
+	return ret;
+}
+
+static int pse_reply_size(const struct ethnl_req_info *req_base,
+			  const struct ethnl_reply_data *reply_base)
+{
+	const struct pse_reply_data *data = PSE_REPDATA(reply_base);
+	const struct pse_control_status *st = &data->status;
+	int len = 0;
+
+	if (st->podl_admin_state > 0)
+		len += nla_total_size(sizeof(u32)); /* _PODL_PSE_ADMIN_STATE */
+	if (st->podl_pw_status > 0)
+		len += nla_total_size(sizeof(u32)); /* _PODL_PSE_PW_D_STATUS */
+
+	return len;
+}
+
+static int pse_fill_reply(struct sk_buff *skb,
+			  const struct ethnl_req_info *req_base,
+			  const struct ethnl_reply_data *reply_base)
+{
+	const struct pse_reply_data *data = PSE_REPDATA(reply_base);
+	const struct pse_control_status *st = &data->status;
+
+	if (st->podl_admin_state > 0 &&
+	    nla_put_u32(skb, ETHTOOL_A_PODL_PSE_ADMIN_STATE,
+			st->podl_admin_state))
+		return -EMSGSIZE;
+
+	if (st->podl_pw_status > 0 &&
+	    nla_put_u32(skb, ETHTOOL_A_PODL_PSE_PW_D_STATUS,
+			st->podl_pw_status))
+		return -EMSGSIZE;
+
+	return 0;
+}
+
+const struct ethnl_request_ops ethnl_pse_request_ops = {
+	.request_cmd		= ETHTOOL_MSG_PSE_GET,
+	.reply_cmd		= ETHTOOL_MSG_PSE_GET_REPLY,
+	.hdr_attr		= ETHTOOL_A_PSE_HEADER,
+	.req_info_size		= sizeof(struct pse_req_info),
+	.reply_data_size	= sizeof(struct pse_reply_data),
+
+	.prepare_data		= pse_prepare_data,
+	.reply_size		= pse_reply_size,
+	.fill_reply		= pse_fill_reply,
+};
+
+/* PSE_SET */
+
+const struct nla_policy ethnl_pse_set_policy[ETHTOOL_A_PSE_MAX + 1] = {
+	[ETHTOOL_A_PSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy),
+	[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL] =
+		NLA_POLICY_RANGE(NLA_U32, ETHTOOL_PODL_PSE_ADMIN_STATE_DISABLED,
+				 ETHTOOL_PODL_PSE_ADMIN_STATE_ENABLED),
+};
+
+static int pse_set_pse_config(struct net_device *dev,
+			      struct netlink_ext_ack *extack,
+			      struct nlattr **tb)
+{
+	struct phy_device *phydev = dev->phydev;
+	struct pse_control_config config = {};
+
+	/* Optional attribute. Do not return error if not set. */
+	if (!tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL])
+		return 0;
+
+	/* this values are already validated by the ethnl_pse_set_policy */
+	config.admin_cotrol = nla_get_u32(tb[ETHTOOL_A_PODL_PSE_ADMIN_CONTROL]);
+
+	if (!phydev) {
+		NL_SET_ERR_MSG(extack, "No PHY is attached");
+		return -EOPNOTSUPP;
+	}
+
+	if (!phydev->psec) {
+		NL_SET_ERR_MSG(extack, "No PSE is attached");
+		return -EOPNOTSUPP;
+	}
+
+	return pse_ethtool_set_config(phydev->psec, extack, &config);
+}
+
+int ethnl_set_pse(struct sk_buff *skb, struct genl_info *info)
+{
+	struct ethnl_req_info req_info = {};
+	struct nlattr **tb = info->attrs;
+	struct net_device *dev;
+	int ret;
+
+	ret = ethnl_parse_header_dev_get(&req_info, tb[ETHTOOL_A_PSE_HEADER],
+					 genl_info_net(info), info->extack,
+					 true);
+	if (ret < 0)
+		return ret;
+
+	dev = req_info.dev;
+
+	rtnl_lock();
+	ret = ethnl_ops_begin(dev);
+	if (ret < 0)
+		goto out_rtnl;
+
+	ret = pse_set_pse_config(dev, info->extack, tb);
+	ethnl_ops_complete(dev);
+out_rtnl:
+	rtnl_unlock();
+
+	ethnl_parse_header_dev_put(&req_info);
+
+	return ret;
+}
diff --git a/net/ethtool/strset.c b/net/ethtool/strset.c
index 2d51b7ab4dc5..3f7de54d85fb 100644
--- a/net/ethtool/strset.c
+++ b/net/ethtool/strset.c
@@ -167,7 +167,7 @@ static int strset_get_id(const struct nlattr *nest, u32 *val,
 			       get_stringset_policy, extack);
 	if (ret < 0)
 		return ret;
-	if (!tb[ETHTOOL_A_STRINGSET_ID])
+	if (NL_REQ_ATTR_CHECK(extack, nest, tb, ETHTOOL_A_STRINGSET_ID))
 		return -EINVAL;
 
 	*val = nla_get_u32(tb[ETHTOOL_A_STRINGSET_ID]);
diff --git a/net/ethtool/tunnels.c b/net/ethtool/tunnels.c
index efde33536687..67fb414ca859 100644
--- a/net/ethtool/tunnels.c
+++ b/net/ethtool/tunnels.c
@@ -136,6 +136,8 @@ ethnl_tunnel_info_fill_reply(const struct ethnl_req_info *req_base,
 			goto err_cancel_table;
 
 		entry = nla_nest_start(skb, ETHTOOL_A_TUNNEL_UDP_TABLE_ENTRY);
+		if (!entry)
+			goto err_cancel_entry;
 
 		if (nla_put_be16(skb, ETHTOOL_A_TUNNEL_UDP_ENTRY_PORT,
 				 htons(IANA_VXLAN_UDP_PORT)) ||
diff --git a/net/hsr/hsr_netlink.c b/net/hsr/hsr_netlink.c
index 1405c037cf7a..7174a9092900 100644
--- a/net/hsr/hsr_netlink.c
+++ b/net/hsr/hsr_netlink.c
@@ -522,6 +522,7 @@ static struct genl_family hsr_genl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.small_ops = hsr_ops,
 	.n_small_ops = ARRAY_SIZE(hsr_ops),
+	.resv_start_op = HSR_C_SET_NODE_LIST + 1,
 	.mcgrps = hsr_mcgrps,
 	.n_mcgrps = ARRAY_SIZE(hsr_mcgrps),
 };
diff --git a/net/ieee802154/netlink.c b/net/ieee802154/netlink.c
index b07abc38b4b3..7d2de4ee6992 100644
--- a/net/ieee802154/netlink.c
+++ b/net/ieee802154/netlink.c
@@ -132,6 +132,7 @@ struct genl_family nl802154_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= ieee802154_ops,
 	.n_small_ops	= ARRAY_SIZE(ieee802154_ops),
+	.resv_start_op	= IEEE802154_LLSEC_DEL_SECLEVEL + 1,
 	.mcgrps		= ieee802154_mcgrps,
 	.n_mcgrps	= ARRAY_SIZE(ieee802154_mcgrps),
 };
diff --git a/net/ieee802154/nl802154.c b/net/ieee802154/nl802154.c
index e0b072aecf0f..38c4f3cb010e 100644
--- a/net/ieee802154/nl802154.c
+++ b/net/ieee802154/nl802154.c
@@ -2500,6 +2500,7 @@ static struct genl_family nl802154_fam __ro_after_init = {
 	.module = THIS_MODULE,
 	.ops = nl802154_ops,
 	.n_ops = ARRAY_SIZE(nl802154_ops),
+	.resv_start_op = NL802154_CMD_DEL_SEC_LEVEL + 1,
 	.mcgrps = nl802154_mcgrps,
 	.n_mcgrps = ARRAY_SIZE(nl802154_mcgrps),
 };
diff --git a/net/ieee802154/socket.c b/net/ieee802154/socket.c
index 7889e1ef7fad..cbd0e2ac4ffe 100644
--- a/net/ieee802154/socket.c
+++ b/net/ieee802154/socket.c
@@ -251,6 +251,9 @@ static int raw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
 		return -EOPNOTSUPP;
 	}
 
+	if (!size)
+		return -EINVAL;
+
 	lock_sock(sk);
 	if (!sk->sk_bound_dev_if)
 		dev = dev_getfirstbyhwtype(sock_net(sk), ARPHRD_IEEE802154);
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 3ca0cc467886..e2c219382345 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1219,6 +1219,7 @@ EXPORT_SYMBOL(inet_unregister_protosw);
 
 static int inet_sk_reselect_saddr(struct sock *sk)
 {
+	struct inet_bind_hashbucket *prev_addr_hashbucket;
 	struct inet_sock *inet = inet_sk(sk);
 	__be32 old_saddr = inet->inet_saddr;
 	__be32 daddr = inet->inet_daddr;
@@ -1226,6 +1227,7 @@ static int inet_sk_reselect_saddr(struct sock *sk)
 	struct rtable *rt;
 	__be32 new_saddr;
 	struct ip_options_rcu *inet_opt;
+	int err;
 
 	inet_opt = rcu_dereference_protected(inet->inet_opt,
 					     lockdep_sock_is_held(sk));
@@ -1240,20 +1242,34 @@ static int inet_sk_reselect_saddr(struct sock *sk)
 	if (IS_ERR(rt))
 		return PTR_ERR(rt);
 
-	sk_setup_caps(sk, &rt->dst);
-
 	new_saddr = fl4->saddr;
 
-	if (new_saddr == old_saddr)
+	if (new_saddr == old_saddr) {
+		sk_setup_caps(sk, &rt->dst);
 		return 0;
+	}
+
+	prev_addr_hashbucket =
+		inet_bhashfn_portaddr(tcp_or_dccp_get_hashinfo(sk), sk,
+				      sock_net(sk), inet->inet_num);
+
+	inet->inet_saddr = inet->inet_rcv_saddr = new_saddr;
+
+	err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk);
+	if (err) {
+		inet->inet_saddr = old_saddr;
+		inet->inet_rcv_saddr = old_saddr;
+		ip_rt_put(rt);
+		return err;
+	}
+
+	sk_setup_caps(sk, &rt->dst);
 
 	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_ip_dynaddr) > 1) {
 		pr_info("%s(): shifting inet->saddr from %pI4 to %pI4\n",
 			__func__, &old_saddr, &new_saddr);
 	}
 
-	inet->inet_saddr = inet->inet_rcv_saddr = new_saddr;
-
 	/*
 	 * XXX The only one ugly spot where we need to
 	 * XXX really change the sockets identity after
@@ -1448,12 +1464,9 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
 
 	off = skb_gro_offset(skb);
 	hlen = off + sizeof(*iph);
-	iph = skb_gro_header_fast(skb, off);
-	if (skb_gro_header_hard(skb, hlen)) {
-		iph = skb_gro_header_slow(skb, hlen, off);
-		if (unlikely(!iph))
-			goto out;
-	}
+	iph = skb_gro_header(skb, hlen, off);
+	if (unlikely(!iph))
+		goto out;
 
 	proto = iph->protocol;
 
diff --git a/net/ipv4/ah4.c b/net/ipv4/ah4.c
index f8ad04470d3a..ee4e578c7f20 100644
--- a/net/ipv4/ah4.c
+++ b/net/ipv4/ah4.c
@@ -471,30 +471,38 @@ static int ah4_err(struct sk_buff *skb, u32 info)
 	return 0;
 }
 
-static int ah_init_state(struct xfrm_state *x)
+static int ah_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
 	struct ah_data *ahp = NULL;
 	struct xfrm_algo_desc *aalg_desc;
 	struct crypto_ahash *ahash;
 
-	if (!x->aalg)
+	if (!x->aalg) {
+		NL_SET_ERR_MSG(extack, "AH requires a state with an AUTH algorithm");
 		goto error;
+	}
 
-	if (x->encap)
+	if (x->encap) {
+		NL_SET_ERR_MSG(extack, "AH is not compatible with encapsulation");
 		goto error;
+	}
 
 	ahp = kzalloc(sizeof(*ahp), GFP_KERNEL);
 	if (!ahp)
 		return -ENOMEM;
 
 	ahash = crypto_alloc_ahash(x->aalg->alg_name, 0, 0);
-	if (IS_ERR(ahash))
+	if (IS_ERR(ahash)) {
+		NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 		goto error;
+	}
 
 	ahp->ahash = ahash;
 	if (crypto_ahash_setkey(ahash, x->aalg->alg_key,
-				(x->aalg->alg_key_len + 7) / 8))
+				(x->aalg->alg_key_len + 7) / 8)) {
+		NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 		goto error;
+	}
 
 	/*
 	 * Lookup the algorithm description maintained by xfrm_algo,
@@ -507,10 +515,7 @@ static int ah_init_state(struct xfrm_state *x)
 
 	if (aalg_desc->uinfo.auth.icv_fullbits/8 !=
 	    crypto_ahash_digestsize(ahash)) {
-		pr_info("%s: %s digestsize %u != %u\n",
-			__func__, x->aalg->alg_name,
-			crypto_ahash_digestsize(ahash),
-			aalg_desc->uinfo.auth.icv_fullbits / 8);
+		NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 		goto error;
 	}
 
diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c
index 87c7e3fc5197..4f7237661afb 100644
--- a/net/ipv4/arp.c
+++ b/net/ipv4/arp.c
@@ -1129,7 +1129,7 @@ static int arp_req_get(struct arpreq *r, struct net_device *dev)
 			r->arp_flags = arp_state_to_flags(neigh);
 			read_unlock_bh(&neigh->lock);
 			r->arp_ha.sa_family = dev->type;
-			strlcpy(r->arp_dev, dev->name, sizeof(r->arp_dev));
+			strscpy(r->arp_dev, dev->name, sizeof(r->arp_dev));
 			err = 0;
 		}
 		neigh_release(neigh);
diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c
index 85a9e500c42d..6da16ae6a962 100644
--- a/net/ipv4/bpf_tcp_ca.c
+++ b/net/ipv4/bpf_tcp_ca.c
@@ -124,7 +124,7 @@ static int bpf_tcp_ca_btf_struct_access(struct bpf_verifier_log *log,
 		return -EACCES;
 	}
 
-	return NOT_INIT;
+	return 0;
 }
 
 BPF_CALL_2(bpf_tcp_send_ack, struct tcp_sock *, tp, u32, rcv_nxt)
diff --git a/net/ipv4/datagram.c b/net/ipv4/datagram.c
index ffd57523331f..405a8c2aea64 100644
--- a/net/ipv4/datagram.c
+++ b/net/ipv4/datagram.c
@@ -42,6 +42,8 @@ int __ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len
 			oif = inet->mc_index;
 		if (!saddr)
 			saddr = inet->mc_addr;
+	} else if (!oif) {
+		oif = inet->uc_index;
 	}
 	fl4 = &inet->cork.fl.u.ip4;
 	rt = ip_route_connect(fl4, usin->sin_addr.s_addr, saddr, oif,
diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c
index 5c03eba787e5..52c8047efedb 100644
--- a/net/ipv4/esp4.c
+++ b/net/ipv4/esp4.c
@@ -134,6 +134,7 @@ static void esp_free_tcp_sk(struct rcu_head *head)
 static struct sock *esp_find_tcp_sk(struct xfrm_state *x)
 {
 	struct xfrm_encap_tmpl *encap = x->encap;
+	struct net *net = xs_net(x);
 	struct esp_tcp_sk *esk;
 	__be16 sport, dport;
 	struct sock *nsk;
@@ -160,7 +161,7 @@ static struct sock *esp_find_tcp_sk(struct xfrm_state *x)
 	}
 	spin_unlock_bh(&x->lock);
 
-	sk = inet_lookup_established(xs_net(x), &tcp_hashinfo, x->id.daddr.a4,
+	sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, x->id.daddr.a4,
 				     dport, x->props.saddr.a4, sport, 0);
 	if (!sk)
 		return ERR_PTR(-ENOENT);
@@ -1007,16 +1008,17 @@ static void esp_destroy(struct xfrm_state *x)
 	crypto_free_aead(aead);
 }
 
-static int esp_init_aead(struct xfrm_state *x)
+static int esp_init_aead(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
 	char aead_name[CRYPTO_MAX_ALG_NAME];
 	struct crypto_aead *aead;
 	int err;
 
-	err = -ENAMETOOLONG;
 	if (snprintf(aead_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
-		     x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME)
-		goto error;
+		     x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME) {
+		NL_SET_ERR_MSG(extack, "Algorithm name is too long");
+		return -ENAMETOOLONG;
+	}
 
 	aead = crypto_alloc_aead(aead_name, 0, 0);
 	err = PTR_ERR(aead);
@@ -1034,11 +1036,15 @@ static int esp_init_aead(struct xfrm_state *x)
 	if (err)
 		goto error;
 
+	return 0;
+
 error:
+	NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 	return err;
 }
 
-static int esp_init_authenc(struct xfrm_state *x)
+static int esp_init_authenc(struct xfrm_state *x,
+			    struct netlink_ext_ack *extack)
 {
 	struct crypto_aead *aead;
 	struct crypto_authenc_key_param *param;
@@ -1049,10 +1055,6 @@ static int esp_init_authenc(struct xfrm_state *x)
 	unsigned int keylen;
 	int err;
 
-	err = -EINVAL;
-	if (!x->ealg)
-		goto error;
-
 	err = -ENAMETOOLONG;
 
 	if ((x->props.flags & XFRM_STATE_ESN)) {
@@ -1061,22 +1063,28 @@ static int esp_init_authenc(struct xfrm_state *x)
 			     x->geniv ?: "", x->geniv ? "(" : "",
 			     x->aalg ? x->aalg->alg_name : "digest_null",
 			     x->ealg->alg_name,
-			     x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME)
+			     x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) {
+			NL_SET_ERR_MSG(extack, "Algorithm name is too long");
 			goto error;
+		}
 	} else {
 		if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME,
 			     "%s%sauthenc(%s,%s)%s",
 			     x->geniv ?: "", x->geniv ? "(" : "",
 			     x->aalg ? x->aalg->alg_name : "digest_null",
 			     x->ealg->alg_name,
-			     x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME)
+			     x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) {
+			NL_SET_ERR_MSG(extack, "Algorithm name is too long");
 			goto error;
+		}
 	}
 
 	aead = crypto_alloc_aead(authenc_name, 0, 0);
 	err = PTR_ERR(aead);
-	if (IS_ERR(aead))
+	if (IS_ERR(aead)) {
+		NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 		goto error;
+	}
 
 	x->data = aead;
 
@@ -1106,17 +1114,16 @@ static int esp_init_authenc(struct xfrm_state *x)
 		err = -EINVAL;
 		if (aalg_desc->uinfo.auth.icv_fullbits / 8 !=
 		    crypto_aead_authsize(aead)) {
-			pr_info("ESP: %s digestsize %u != %u\n",
-				x->aalg->alg_name,
-				crypto_aead_authsize(aead),
-				aalg_desc->uinfo.auth.icv_fullbits / 8);
+			NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 			goto free_key;
 		}
 
 		err = crypto_aead_setauthsize(
 			aead, x->aalg->alg_trunc_len / 8);
-		if (err)
+		if (err) {
+			NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 			goto free_key;
+		}
 	}
 
 	param->enckeylen = cpu_to_be32((x->ealg->alg_key_len + 7) / 8);
@@ -1131,7 +1138,7 @@ error:
 	return err;
 }
 
-static int esp_init_state(struct xfrm_state *x)
+static int esp_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
 	struct crypto_aead *aead;
 	u32 align;
@@ -1139,10 +1146,14 @@ static int esp_init_state(struct xfrm_state *x)
 
 	x->data = NULL;
 
-	if (x->aead)
-		err = esp_init_aead(x);
-	else
-		err = esp_init_authenc(x);
+	if (x->aead) {
+		err = esp_init_aead(x, extack);
+	} else if (x->ealg) {
+		err = esp_init_authenc(x, extack);
+	} else {
+		NL_SET_ERR_MSG(extack, "ESP: AEAD or CRYPT must be provided");
+		err = -EINVAL;
+	}
 
 	if (err)
 		goto error;
@@ -1160,6 +1171,7 @@ static int esp_init_state(struct xfrm_state *x)
 
 		switch (encap->encap_type) {
 		default:
+			NL_SET_ERR_MSG(extack, "Unsupported encapsulation type for ESP");
 			err = -EINVAL;
 			goto error;
 		case UDP_ENCAP_ESPINUDP:
diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c
index 935026f4c807..170152772d33 100644
--- a/net/ipv4/esp4_offload.c
+++ b/net/ipv4/esp4_offload.c
@@ -110,7 +110,10 @@ static struct sk_buff *xfrm4_tunnel_gso_segment(struct xfrm_state *x,
 						struct sk_buff *skb,
 						netdev_features_t features)
 {
-	return skb_eth_gso_segment(skb, features, htons(ETH_P_IP));
+	__be16 type = x->inner_mode.family == AF_INET6 ? htons(ETH_P_IPV6)
+						       : htons(ETH_P_IP);
+
+	return skb_eth_gso_segment(skb, features, type);
 }
 
 static struct sk_buff *xfrm4_transport_gso_segment(struct xfrm_state *x,
diff --git a/net/ipv4/fou.c b/net/ipv4/fou.c
index 025a33c1b04d..0c3c6d0cee29 100644
--- a/net/ipv4/fou.c
+++ b/net/ipv4/fou.c
@@ -323,12 +323,9 @@ static struct sk_buff *gue_gro_receive(struct sock *sk,
 	off = skb_gro_offset(skb);
 	len = off + sizeof(*guehdr);
 
-	guehdr = skb_gro_header_fast(skb, off);
-	if (skb_gro_header_hard(skb, len)) {
-		guehdr = skb_gro_header_slow(skb, len, off);
-		if (unlikely(!guehdr))
-			goto out;
-	}
+	guehdr = skb_gro_header(skb, len, off);
+	if (unlikely(!guehdr))
+		goto out;
 
 	switch (guehdr->version) {
 	case 0:
@@ -931,6 +928,7 @@ static struct genl_family fou_nl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= fou_nl_ops,
 	.n_small_ops	= ARRAY_SIZE(fou_nl_ops),
+	.resv_start_op	= FOU_CMD_GET + 1,
 };
 
 size_t fou_encap_hlen(struct ip_tunnel_encap *e)
diff --git a/net/ipv4/gre_offload.c b/net/ipv4/gre_offload.c
index 07073fa35205..2b9cb5398335 100644
--- a/net/ipv4/gre_offload.c
+++ b/net/ipv4/gre_offload.c
@@ -137,12 +137,9 @@ static struct sk_buff *gre_gro_receive(struct list_head *head,
 
 	off = skb_gro_offset(skb);
 	hlen = off + sizeof(*greh);
-	greh = skb_gro_header_fast(skb, off);
-	if (skb_gro_header_hard(skb, hlen)) {
-		greh = skb_gro_header_slow(skb, hlen, off);
-		if (unlikely(!greh))
-			goto out;
-	}
+	greh = skb_gro_header(skb, hlen, off);
+	if (unlikely(!greh))
+		goto out;
 
 	/* Only support version 0 and K (key), C (csum) flags. Note that
 	 * although the support for the S (seq#) flag can be added easily
diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c
index e3ab0cb61624..df0660d818ac 100644
--- a/net/ipv4/igmp.c
+++ b/net/ipv4/igmp.c
@@ -2529,11 +2529,10 @@ done:
 		err = ip_mc_leave_group(sk, &imr);
 	return err;
 }
-
 int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf,
-	struct ip_msfilter __user *optval, int __user *optlen)
+		 sockptr_t optval, sockptr_t optlen)
 {
-	int err, len, count, copycount;
+	int err, len, count, copycount, msf_size;
 	struct ip_mreqn	imr;
 	__be32 addr = msf->imsf_multiaddr;
 	struct ip_mc_socklist *pmc;
@@ -2575,12 +2574,15 @@ int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf,
 	copycount = count < msf->imsf_numsrc ? count : msf->imsf_numsrc;
 	len = flex_array_size(psl, sl_addr, copycount);
 	msf->imsf_numsrc = count;
-	if (put_user(IP_MSFILTER_SIZE(copycount), optlen) ||
-	    copy_to_user(optval, msf, IP_MSFILTER_SIZE(0))) {
+	msf_size = IP_MSFILTER_SIZE(copycount);
+	if (copy_to_sockptr(optlen, &msf_size, sizeof(int)) ||
+	    copy_to_sockptr(optval, msf, IP_MSFILTER_SIZE(0))) {
 		return -EFAULT;
 	}
 	if (len &&
-	    copy_to_user(&optval->imsf_slist_flex[0], psl->sl_addr, len))
+	    copy_to_sockptr_offset(optval,
+				   offsetof(struct ip_msfilter, imsf_slist_flex),
+				   psl->sl_addr, len))
 		return -EFAULT;
 	return 0;
 done:
@@ -2588,7 +2590,7 @@ done:
 }
 
 int ip_mc_gsfget(struct sock *sk, struct group_filter *gsf,
-	struct sockaddr_storage __user *p)
+		 sockptr_t optval, size_t ss_offset)
 {
 	int i, count, copycount;
 	struct sockaddr_in *psin;
@@ -2618,15 +2620,17 @@ int ip_mc_gsfget(struct sock *sk, struct group_filter *gsf,
 	count = psl ? psl->sl_count : 0;
 	copycount = count < gsf->gf_numsrc ? count : gsf->gf_numsrc;
 	gsf->gf_numsrc = count;
-	for (i = 0; i < copycount; i++, p++) {
+	for (i = 0; i < copycount; i++) {
 		struct sockaddr_storage ss;
 
 		psin = (struct sockaddr_in *)&ss;
 		memset(&ss, 0, sizeof(ss));
 		psin->sin_family = AF_INET;
 		psin->sin_addr.s_addr = psl->sl_addr[i];
-		if (copy_to_user(p, &ss, sizeof(ss)))
+		if (copy_to_sockptr_offset(optval, ss_offset,
+					   &ss, sizeof(ss)))
 			return -EFAULT;
+		ss_offset += sizeof(ss);
 	}
 	return 0;
 }
diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index eb31c7158b39..ebca860e113f 100644
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -130,14 +130,75 @@ void inet_get_local_port_range(struct net *net, int *low, int *high)
 }
 EXPORT_SYMBOL(inet_get_local_port_range);
 
+static bool inet_use_bhash2_on_bind(const struct sock *sk)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+	if (sk->sk_family == AF_INET6) {
+		int addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
+
+		return addr_type != IPV6_ADDR_ANY &&
+			addr_type != IPV6_ADDR_MAPPED;
+	}
+#endif
+	return sk->sk_rcv_saddr != htonl(INADDR_ANY);
+}
+
+static bool inet_bind_conflict(const struct sock *sk, struct sock *sk2,
+			       kuid_t sk_uid, bool relax,
+			       bool reuseport_cb_ok, bool reuseport_ok)
+{
+	int bound_dev_if2;
+
+	if (sk == sk2)
+		return false;
+
+	bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if);
+
+	if (!sk->sk_bound_dev_if || !bound_dev_if2 ||
+	    sk->sk_bound_dev_if == bound_dev_if2) {
+		if (sk->sk_reuse && sk2->sk_reuse &&
+		    sk2->sk_state != TCP_LISTEN) {
+			if (!relax || (!reuseport_ok && sk->sk_reuseport &&
+				       sk2->sk_reuseport && reuseport_cb_ok &&
+				       (sk2->sk_state == TCP_TIME_WAIT ||
+					uid_eq(sk_uid, sock_i_uid(sk2)))))
+				return true;
+		} else if (!reuseport_ok || !sk->sk_reuseport ||
+			   !sk2->sk_reuseport || !reuseport_cb_ok ||
+			   (sk2->sk_state != TCP_TIME_WAIT &&
+			    !uid_eq(sk_uid, sock_i_uid(sk2)))) {
+			return true;
+		}
+	}
+	return false;
+}
+
+static bool inet_bhash2_conflict(const struct sock *sk,
+				 const struct inet_bind2_bucket *tb2,
+				 kuid_t sk_uid,
+				 bool relax, bool reuseport_cb_ok,
+				 bool reuseport_ok)
+{
+	struct sock *sk2;
+
+	sk_for_each_bound_bhash2(sk2, &tb2->owners) {
+		if (sk->sk_family == AF_INET && ipv6_only_sock(sk2))
+			continue;
+
+		if (inet_bind_conflict(sk, sk2, sk_uid, relax,
+				       reuseport_cb_ok, reuseport_ok))
+			return true;
+	}
+	return false;
+}
+
+/* This should be called only when the tb and tb2 hashbuckets' locks are held */
 static int inet_csk_bind_conflict(const struct sock *sk,
 				  const struct inet_bind_bucket *tb,
+				  const struct inet_bind2_bucket *tb2, /* may be null */
 				  bool relax, bool reuseport_ok)
 {
-	struct sock *sk2;
 	bool reuseport_cb_ok;
-	bool reuse = sk->sk_reuse;
-	bool reuseport = !!sk->sk_reuseport;
 	struct sock_reuseport *reuseport_cb;
 	kuid_t uid = sock_i_uid((struct sock *)sk);
 
@@ -150,58 +211,88 @@ static int inet_csk_bind_conflict(const struct sock *sk,
 	/*
 	 * Unlike other sk lookup places we do not check
 	 * for sk_net here, since _all_ the socks listed
-	 * in tb->owners list belong to the same net - the
-	 * one this bucket belongs to.
+	 * in tb->owners and tb2->owners list belong
+	 * to the same net - the one this bucket belongs to.
 	 */
 
-	sk_for_each_bound(sk2, &tb->owners) {
-		int bound_dev_if2;
+	if (!inet_use_bhash2_on_bind(sk)) {
+		struct sock *sk2;
 
-		if (sk == sk2)
-			continue;
-		bound_dev_if2 = READ_ONCE(sk2->sk_bound_dev_if);
-		if ((!sk->sk_bound_dev_if ||
-		     !bound_dev_if2 ||
-		     sk->sk_bound_dev_if == bound_dev_if2)) {
-			if (reuse && sk2->sk_reuse &&
-			    sk2->sk_state != TCP_LISTEN) {
-				if ((!relax ||
-				     (!reuseport_ok &&
-				      reuseport && sk2->sk_reuseport &&
-				      reuseport_cb_ok &&
-				      (sk2->sk_state == TCP_TIME_WAIT ||
-				       uid_eq(uid, sock_i_uid(sk2))))) &&
-				    inet_rcv_saddr_equal(sk, sk2, true))
-					break;
-			} else if (!reuseport_ok ||
-				   !reuseport || !sk2->sk_reuseport ||
-				   !reuseport_cb_ok ||
-				   (sk2->sk_state != TCP_TIME_WAIT &&
-				    !uid_eq(uid, sock_i_uid(sk2)))) {
-				if (inet_rcv_saddr_equal(sk, sk2, true))
-					break;
-			}
-		}
+		sk_for_each_bound(sk2, &tb->owners)
+			if (inet_bind_conflict(sk, sk2, uid, relax,
+					       reuseport_cb_ok, reuseport_ok) &&
+			    inet_rcv_saddr_equal(sk, sk2, true))
+				return true;
+
+		return false;
+	}
+
+	/* Conflicts with an existing IPV6_ADDR_ANY (if ipv6) or INADDR_ANY (if
+	 * ipv4) should have been checked already. We need to do these two
+	 * checks separately because their spinlocks have to be acquired/released
+	 * independently of each other, to prevent possible deadlocks
+	 */
+	return tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok,
+					   reuseport_ok);
+}
+
+/* Determine if there is a bind conflict with an existing IPV6_ADDR_ANY (if ipv6) or
+ * INADDR_ANY (if ipv4) socket.
+ *
+ * Caller must hold bhash hashbucket lock with local bh disabled, to protect
+ * against concurrent binds on the port for addr any
+ */
+static bool inet_bhash2_addr_any_conflict(const struct sock *sk, int port, int l3mdev,
+					  bool relax, bool reuseport_ok)
+{
+	kuid_t uid = sock_i_uid((struct sock *)sk);
+	const struct net *net = sock_net(sk);
+	struct sock_reuseport *reuseport_cb;
+	struct inet_bind_hashbucket *head2;
+	struct inet_bind2_bucket *tb2;
+	bool reuseport_cb_ok;
+
+	rcu_read_lock();
+	reuseport_cb = rcu_dereference(sk->sk_reuseport_cb);
+	/* paired with WRITE_ONCE() in __reuseport_(add|detach)_closed_sock */
+	reuseport_cb_ok = !reuseport_cb || READ_ONCE(reuseport_cb->num_closed_socks);
+	rcu_read_unlock();
+
+	head2 = inet_bhash2_addr_any_hashbucket(sk, net, port);
+
+	spin_lock(&head2->lock);
+
+	inet_bind_bucket_for_each(tb2, &head2->chain)
+		if (inet_bind2_bucket_match_addr_any(tb2, net, port, l3mdev, sk))
+			break;
+
+	if (tb2 && inet_bhash2_conflict(sk, tb2, uid, relax, reuseport_cb_ok,
+					reuseport_ok)) {
+		spin_unlock(&head2->lock);
+		return true;
 	}
-	return sk2 != NULL;
+
+	spin_unlock(&head2->lock);
+	return false;
 }
 
 /*
  * Find an open port number for the socket.  Returns with the
- * inet_bind_hashbucket lock held.
+ * inet_bind_hashbucket locks held if successful.
  */
 static struct inet_bind_hashbucket *
-inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret, int *port_ret)
+inet_csk_find_open_port(const struct sock *sk, struct inet_bind_bucket **tb_ret,
+			struct inet_bind2_bucket **tb2_ret,
+			struct inet_bind_hashbucket **head2_ret, int *port_ret)
 {
-	struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
-	int port = 0;
-	struct inet_bind_hashbucket *head;
+	struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk);
+	int i, low, high, attempt_half, port, l3mdev;
+	struct inet_bind_hashbucket *head, *head2;
 	struct net *net = sock_net(sk);
-	bool relax = false;
-	int i, low, high, attempt_half;
+	struct inet_bind2_bucket *tb2;
 	struct inet_bind_bucket *tb;
 	u32 remaining, offset;
-	int l3mdev;
+	bool relax = false;
 
 	l3mdev = inet_sk_bound_l3mdev(sk);
 ports_exhausted:
@@ -239,11 +330,20 @@ other_parity_scan:
 		head = &hinfo->bhash[inet_bhashfn(net, port,
 						  hinfo->bhash_size)];
 		spin_lock_bh(&head->lock);
+		if (inet_use_bhash2_on_bind(sk)) {
+			if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, relax, false))
+				goto next_port;
+		}
+
+		head2 = inet_bhashfn_portaddr(hinfo, sk, net, port);
+		spin_lock(&head2->lock);
+		tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk);
 		inet_bind_bucket_for_each(tb, &head->chain)
-			if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
-			    tb->port == port) {
-				if (!inet_csk_bind_conflict(sk, tb, relax, false))
+			if (inet_bind_bucket_match(tb, net, port, l3mdev)) {
+				if (!inet_csk_bind_conflict(sk, tb, tb2,
+							    relax, false))
 					goto success;
+				spin_unlock(&head2->lock);
 				goto next_port;
 			}
 		tb = NULL;
@@ -272,6 +372,8 @@ next_port:
 success:
 	*port_ret = port;
 	*tb_ret = tb;
+	*tb2_ret = tb2;
+	*head2_ret = head2;
 	return head;
 }
 
@@ -365,56 +467,97 @@ void inet_csk_update_fastreuse(struct inet_bind_bucket *tb,
  */
 int inet_csk_get_port(struct sock *sk, unsigned short snum)
 {
+	struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk);
 	bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
-	struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
-	int ret = 1, port = snum;
-	struct inet_bind_hashbucket *head;
-	struct net *net = sock_net(sk);
+	bool found_port = false, check_bind_conflict = true;
+	bool bhash_created = false, bhash2_created = false;
+	struct inet_bind_hashbucket *head, *head2;
+	struct inet_bind2_bucket *tb2 = NULL;
 	struct inet_bind_bucket *tb = NULL;
-	int l3mdev;
+	bool head2_lock_acquired = false;
+	int ret = 1, port = snum, l3mdev;
+	struct net *net = sock_net(sk);
 
 	l3mdev = inet_sk_bound_l3mdev(sk);
 
 	if (!port) {
-		head = inet_csk_find_open_port(sk, &tb, &port);
+		head = inet_csk_find_open_port(sk, &tb, &tb2, &head2, &port);
 		if (!head)
 			return ret;
+
+		head2_lock_acquired = true;
+
+		if (tb && tb2)
+			goto success;
+		found_port = true;
+	} else {
+		head = &hinfo->bhash[inet_bhashfn(net, port,
+						  hinfo->bhash_size)];
+		spin_lock_bh(&head->lock);
+		inet_bind_bucket_for_each(tb, &head->chain)
+			if (inet_bind_bucket_match(tb, net, port, l3mdev))
+				break;
+	}
+
+	if (!tb) {
+		tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep, net,
+					     head, port, l3mdev);
 		if (!tb)
-			goto tb_not_found;
-		goto success;
+			goto fail_unlock;
+		bhash_created = true;
 	}
-	head = &hinfo->bhash[inet_bhashfn(net, port,
-					  hinfo->bhash_size)];
-	spin_lock_bh(&head->lock);
-	inet_bind_bucket_for_each(tb, &head->chain)
-		if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
-		    tb->port == port)
-			goto tb_found;
-tb_not_found:
-	tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
-				     net, head, port, l3mdev);
-	if (!tb)
-		goto fail_unlock;
-tb_found:
-	if (!hlist_empty(&tb->owners)) {
-		if (sk->sk_reuse == SK_FORCE_REUSE)
-			goto success;
 
-		if ((tb->fastreuse > 0 && reuse) ||
-		    sk_reuseport_match(tb, sk))
-			goto success;
-		if (inet_csk_bind_conflict(sk, tb, true, true))
+	if (!found_port) {
+		if (!hlist_empty(&tb->owners)) {
+			if (sk->sk_reuse == SK_FORCE_REUSE ||
+			    (tb->fastreuse > 0 && reuse) ||
+			    sk_reuseport_match(tb, sk))
+				check_bind_conflict = false;
+		}
+
+		if (check_bind_conflict && inet_use_bhash2_on_bind(sk)) {
+			if (inet_bhash2_addr_any_conflict(sk, port, l3mdev, true, true))
+				goto fail_unlock;
+		}
+
+		head2 = inet_bhashfn_portaddr(hinfo, sk, net, port);
+		spin_lock(&head2->lock);
+		head2_lock_acquired = true;
+		tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk);
+	}
+
+	if (!tb2) {
+		tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep,
+					       net, head2, port, l3mdev, sk);
+		if (!tb2)
 			goto fail_unlock;
+		bhash2_created = true;
 	}
+
+	if (!found_port && check_bind_conflict) {
+		if (inet_csk_bind_conflict(sk, tb, tb2, true, true))
+			goto fail_unlock;
+	}
+
 success:
 	inet_csk_update_fastreuse(tb, sk);
 
 	if (!inet_csk(sk)->icsk_bind_hash)
-		inet_bind_hash(sk, tb, port);
+		inet_bind_hash(sk, tb, tb2, port);
 	WARN_ON(inet_csk(sk)->icsk_bind_hash != tb);
+	WARN_ON(inet_csk(sk)->icsk_bind2_hash != tb2);
 	ret = 0;
 
 fail_unlock:
+	if (ret) {
+		if (bhash_created)
+			inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb);
+		if (bhash2_created)
+			inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep,
+						  tb2);
+	}
+	if (head2_lock_acquired)
+		spin_unlock(&head2->lock);
 	spin_unlock_bh(&head->lock);
 	return ret;
 }
@@ -763,14 +906,15 @@ static void reqsk_migrate_reset(struct request_sock *req)
 /* return true if req was found in the ehash table */
 static bool reqsk_queue_unlink(struct request_sock *req)
 {
-	struct inet_hashinfo *hashinfo = req_to_sk(req)->sk_prot->h.hashinfo;
+	struct sock *sk = req_to_sk(req);
 	bool found = false;
 
-	if (sk_hashed(req_to_sk(req))) {
+	if (sk_hashed(sk)) {
+		struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk);
 		spinlock_t *lock = inet_ehash_lockp(hashinfo, req->rsk_hash);
 
 		spin_lock(lock);
-		found = __sk_nulls_del_node_init_rcu(req_to_sk(req));
+		found = __sk_nulls_del_node_init_rcu(sk);
 		spin_unlock(lock);
 	}
 	if (timer_pending(&req->rsk_timer) && del_timer_sync(&req->rsk_timer))
@@ -962,6 +1106,7 @@ struct sock *inet_csk_clone_lock(const struct sock *sk,
 
 		inet_sk_set_state(newsk, TCP_SYN_RECV);
 		newicsk->icsk_bind_hash = NULL;
+		newicsk->icsk_bind2_hash = NULL;
 
 		inet_sk(newsk)->inet_dport = inet_rsk(req)->ir_rmt_port;
 		inet_sk(newsk)->inet_num = inet_rsk(req)->ir_num;
diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index b9d995b5ce24..a0ad34e4f044 100644
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -92,12 +92,79 @@ void inet_bind_bucket_destroy(struct kmem_cache *cachep, struct inet_bind_bucket
 	}
 }
 
+bool inet_bind_bucket_match(const struct inet_bind_bucket *tb, const struct net *net,
+			    unsigned short port, int l3mdev)
+{
+	return net_eq(ib_net(tb), net) && tb->port == port &&
+		tb->l3mdev == l3mdev;
+}
+
+static void inet_bind2_bucket_init(struct inet_bind2_bucket *tb,
+				   struct net *net,
+				   struct inet_bind_hashbucket *head,
+				   unsigned short port, int l3mdev,
+				   const struct sock *sk)
+{
+	write_pnet(&tb->ib_net, net);
+	tb->l3mdev    = l3mdev;
+	tb->port      = port;
+#if IS_ENABLED(CONFIG_IPV6)
+	tb->family    = sk->sk_family;
+	if (sk->sk_family == AF_INET6)
+		tb->v6_rcv_saddr = sk->sk_v6_rcv_saddr;
+	else
+#endif
+		tb->rcv_saddr = sk->sk_rcv_saddr;
+	INIT_HLIST_HEAD(&tb->owners);
+	hlist_add_head(&tb->node, &head->chain);
+}
+
+struct inet_bind2_bucket *inet_bind2_bucket_create(struct kmem_cache *cachep,
+						   struct net *net,
+						   struct inet_bind_hashbucket *head,
+						   unsigned short port,
+						   int l3mdev,
+						   const struct sock *sk)
+{
+	struct inet_bind2_bucket *tb = kmem_cache_alloc(cachep, GFP_ATOMIC);
+
+	if (tb)
+		inet_bind2_bucket_init(tb, net, head, port, l3mdev, sk);
+
+	return tb;
+}
+
+/* Caller must hold hashbucket lock for this tb with local BH disabled */
+void inet_bind2_bucket_destroy(struct kmem_cache *cachep, struct inet_bind2_bucket *tb)
+{
+	if (hlist_empty(&tb->owners)) {
+		__hlist_del(&tb->node);
+		kmem_cache_free(cachep, tb);
+	}
+}
+
+static bool inet_bind2_bucket_addr_match(const struct inet_bind2_bucket *tb2,
+					 const struct sock *sk)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+	if (sk->sk_family != tb2->family)
+		return false;
+
+	if (sk->sk_family == AF_INET6)
+		return ipv6_addr_equal(&tb2->v6_rcv_saddr,
+				       &sk->sk_v6_rcv_saddr);
+#endif
+	return tb2->rcv_saddr == sk->sk_rcv_saddr;
+}
+
 void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
-		    const unsigned short snum)
+		    struct inet_bind2_bucket *tb2, unsigned short port)
 {
-	inet_sk(sk)->inet_num = snum;
+	inet_sk(sk)->inet_num = port;
 	sk_add_bind_node(sk, &tb->owners);
 	inet_csk(sk)->icsk_bind_hash = tb;
+	sk_add_bind2_node(sk, &tb2->owners);
+	inet_csk(sk)->icsk_bind2_hash = tb2;
 }
 
 /*
@@ -105,11 +172,15 @@ void inet_bind_hash(struct sock *sk, struct inet_bind_bucket *tb,
  */
 static void __inet_put_port(struct sock *sk)
 {
-	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
-	const int bhash = inet_bhashfn(sock_net(sk), inet_sk(sk)->inet_num,
-			hashinfo->bhash_size);
-	struct inet_bind_hashbucket *head = &hashinfo->bhash[bhash];
+	struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk);
+	struct inet_bind_hashbucket *head, *head2;
+	struct net *net = sock_net(sk);
 	struct inet_bind_bucket *tb;
+	int bhash;
+
+	bhash = inet_bhashfn(net, inet_sk(sk)->inet_num, hashinfo->bhash_size);
+	head = &hashinfo->bhash[bhash];
+	head2 = inet_bhashfn_portaddr(hashinfo, sk, net, inet_sk(sk)->inet_num);
 
 	spin_lock(&head->lock);
 	tb = inet_csk(sk)->icsk_bind_hash;
@@ -117,6 +188,17 @@ static void __inet_put_port(struct sock *sk)
 	inet_csk(sk)->icsk_bind_hash = NULL;
 	inet_sk(sk)->inet_num = 0;
 	inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
+
+	spin_lock(&head2->lock);
+	if (inet_csk(sk)->icsk_bind2_hash) {
+		struct inet_bind2_bucket *tb2 = inet_csk(sk)->icsk_bind2_hash;
+
+		__sk_del_bind2_node(sk);
+		inet_csk(sk)->icsk_bind2_hash = NULL;
+		inet_bind2_bucket_destroy(hashinfo->bind2_bucket_cachep, tb2);
+	}
+	spin_unlock(&head2->lock);
+
 	spin_unlock(&head->lock);
 }
 
@@ -130,17 +212,26 @@ EXPORT_SYMBOL(inet_put_port);
 
 int __inet_inherit_port(const struct sock *sk, struct sock *child)
 {
-	struct inet_hashinfo *table = sk->sk_prot->h.hashinfo;
+	struct inet_hashinfo *table = tcp_or_dccp_get_hashinfo(sk);
 	unsigned short port = inet_sk(child)->inet_num;
-	const int bhash = inet_bhashfn(sock_net(sk), port,
-			table->bhash_size);
-	struct inet_bind_hashbucket *head = &table->bhash[bhash];
+	struct inet_bind_hashbucket *head, *head2;
+	bool created_inet_bind_bucket = false;
+	struct net *net = sock_net(sk);
+	bool update_fastreuse = false;
+	struct inet_bind2_bucket *tb2;
 	struct inet_bind_bucket *tb;
-	int l3mdev;
+	int bhash, l3mdev;
+
+	bhash = inet_bhashfn(net, port, table->bhash_size);
+	head = &table->bhash[bhash];
+	head2 = inet_bhashfn_portaddr(table, child, net, port);
 
 	spin_lock(&head->lock);
+	spin_lock(&head2->lock);
 	tb = inet_csk(sk)->icsk_bind_hash;
-	if (unlikely(!tb)) {
+	tb2 = inet_csk(sk)->icsk_bind2_hash;
+	if (unlikely(!tb || !tb2)) {
+		spin_unlock(&head2->lock);
 		spin_unlock(&head->lock);
 		return -ENOENT;
 	}
@@ -153,25 +244,49 @@ int __inet_inherit_port(const struct sock *sk, struct sock *child)
 		 * as that of the child socket. We have to look up or
 		 * create a new bind bucket for the child here. */
 		inet_bind_bucket_for_each(tb, &head->chain) {
-			if (net_eq(ib_net(tb), sock_net(sk)) &&
-			    tb->l3mdev == l3mdev && tb->port == port)
+			if (inet_bind_bucket_match(tb, net, port, l3mdev))
 				break;
 		}
 		if (!tb) {
 			tb = inet_bind_bucket_create(table->bind_bucket_cachep,
-						     sock_net(sk), head, port,
-						     l3mdev);
+						     net, head, port, l3mdev);
 			if (!tb) {
+				spin_unlock(&head2->lock);
 				spin_unlock(&head->lock);
 				return -ENOMEM;
 			}
+			created_inet_bind_bucket = true;
+		}
+		update_fastreuse = true;
+
+		goto bhash2_find;
+	} else if (!inet_bind2_bucket_addr_match(tb2, child)) {
+		l3mdev = inet_sk_bound_l3mdev(sk);
+
+bhash2_find:
+		tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, child);
+		if (!tb2) {
+			tb2 = inet_bind2_bucket_create(table->bind2_bucket_cachep,
+						       net, head2, port,
+						       l3mdev, child);
+			if (!tb2)
+				goto error;
 		}
-		inet_csk_update_fastreuse(tb, child);
 	}
-	inet_bind_hash(child, tb, port);
+	if (update_fastreuse)
+		inet_csk_update_fastreuse(tb, child);
+	inet_bind_hash(child, tb, tb2, port);
+	spin_unlock(&head2->lock);
 	spin_unlock(&head->lock);
 
 	return 0;
+
+error:
+	if (created_inet_bind_bucket)
+		inet_bind_bucket_destroy(table->bind_bucket_cachep, tb);
+	spin_unlock(&head2->lock);
+	spin_unlock(&head->lock);
+	return -ENOMEM;
 }
 EXPORT_SYMBOL_GPL(__inet_inherit_port);
 
@@ -275,7 +390,7 @@ static inline struct sock *inet_lookup_run_bpf(struct net *net,
 	struct sock *sk, *reuse_sk;
 	bool no_reuseport;
 
-	if (hashinfo != &tcp_hashinfo)
+	if (hashinfo != net->ipv4.tcp_death_row.hashinfo)
 		return NULL; /* only TCP is supported */
 
 	no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_TCP, saddr, sport,
@@ -518,9 +633,9 @@ static bool inet_ehash_lookup_by_sk(struct sock *sk,
  */
 bool inet_ehash_insert(struct sock *sk, struct sock *osk, bool *found_dup_sk)
 {
-	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
-	struct hlist_nulls_head *list;
+	struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk);
 	struct inet_ehash_bucket *head;
+	struct hlist_nulls_head *list;
 	spinlock_t *lock;
 	bool ret = true;
 
@@ -590,7 +705,7 @@ static int inet_reuseport_add_sock(struct sock *sk,
 
 int __inet_hash(struct sock *sk, struct sock *osk)
 {
-	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
+	struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk);
 	struct inet_listen_hashbucket *ilb2;
 	int err = 0;
 
@@ -636,7 +751,7 @@ EXPORT_SYMBOL_GPL(inet_hash);
 
 void inet_unhash(struct sock *sk)
 {
-	struct inet_hashinfo *hashinfo = sk->sk_prot->h.hashinfo;
+	struct inet_hashinfo *hashinfo = tcp_or_dccp_get_hashinfo(sk);
 
 	if (sk_unhashed(sk))
 		return;
@@ -675,6 +790,118 @@ void inet_unhash(struct sock *sk)
 }
 EXPORT_SYMBOL_GPL(inet_unhash);
 
+static bool inet_bind2_bucket_match(const struct inet_bind2_bucket *tb,
+				    const struct net *net, unsigned short port,
+				    int l3mdev, const struct sock *sk)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+	if (sk->sk_family != tb->family)
+		return false;
+
+	if (sk->sk_family == AF_INET6)
+		return net_eq(ib2_net(tb), net) && tb->port == port &&
+			tb->l3mdev == l3mdev &&
+			ipv6_addr_equal(&tb->v6_rcv_saddr, &sk->sk_v6_rcv_saddr);
+	else
+#endif
+		return net_eq(ib2_net(tb), net) && tb->port == port &&
+			tb->l3mdev == l3mdev && tb->rcv_saddr == sk->sk_rcv_saddr;
+}
+
+bool inet_bind2_bucket_match_addr_any(const struct inet_bind2_bucket *tb, const struct net *net,
+				      unsigned short port, int l3mdev, const struct sock *sk)
+{
+#if IS_ENABLED(CONFIG_IPV6)
+	struct in6_addr addr_any = {};
+
+	if (sk->sk_family != tb->family)
+		return false;
+
+	if (sk->sk_family == AF_INET6)
+		return net_eq(ib2_net(tb), net) && tb->port == port &&
+			tb->l3mdev == l3mdev &&
+			ipv6_addr_equal(&tb->v6_rcv_saddr, &addr_any);
+	else
+#endif
+		return net_eq(ib2_net(tb), net) && tb->port == port &&
+			tb->l3mdev == l3mdev && tb->rcv_saddr == 0;
+}
+
+/* The socket's bhash2 hashbucket spinlock must be held when this is called */
+struct inet_bind2_bucket *
+inet_bind2_bucket_find(const struct inet_bind_hashbucket *head, const struct net *net,
+		       unsigned short port, int l3mdev, const struct sock *sk)
+{
+	struct inet_bind2_bucket *bhash2 = NULL;
+
+	inet_bind_bucket_for_each(bhash2, &head->chain)
+		if (inet_bind2_bucket_match(bhash2, net, port, l3mdev, sk))
+			break;
+
+	return bhash2;
+}
+
+struct inet_bind_hashbucket *
+inet_bhash2_addr_any_hashbucket(const struct sock *sk, const struct net *net, int port)
+{
+	struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk);
+	u32 hash;
+#if IS_ENABLED(CONFIG_IPV6)
+	struct in6_addr addr_any = {};
+
+	if (sk->sk_family == AF_INET6)
+		hash = ipv6_portaddr_hash(net, &addr_any, port);
+	else
+#endif
+		hash = ipv4_portaddr_hash(net, 0, port);
+
+	return &hinfo->bhash2[hash & (hinfo->bhash_size - 1)];
+}
+
+int inet_bhash2_update_saddr(struct inet_bind_hashbucket *prev_saddr, struct sock *sk)
+{
+	struct inet_hashinfo *hinfo = tcp_or_dccp_get_hashinfo(sk);
+	struct inet_bind2_bucket *tb2, *new_tb2;
+	int l3mdev = inet_sk_bound_l3mdev(sk);
+	struct inet_bind_hashbucket *head2;
+	int port = inet_sk(sk)->inet_num;
+	struct net *net = sock_net(sk);
+
+	/* Allocate a bind2 bucket ahead of time to avoid permanently putting
+	 * the bhash2 table in an inconsistent state if a new tb2 bucket
+	 * allocation fails.
+	 */
+	new_tb2 = kmem_cache_alloc(hinfo->bind2_bucket_cachep, GFP_ATOMIC);
+	if (!new_tb2)
+		return -ENOMEM;
+
+	head2 = inet_bhashfn_portaddr(hinfo, sk, net, port);
+
+	if (prev_saddr) {
+		spin_lock_bh(&prev_saddr->lock);
+		__sk_del_bind2_node(sk);
+		inet_bind2_bucket_destroy(hinfo->bind2_bucket_cachep,
+					  inet_csk(sk)->icsk_bind2_hash);
+		spin_unlock_bh(&prev_saddr->lock);
+	}
+
+	spin_lock_bh(&head2->lock);
+	tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk);
+	if (!tb2) {
+		tb2 = new_tb2;
+		inet_bind2_bucket_init(tb2, net, head2, port, l3mdev, sk);
+	}
+	sk_add_bind2_node(sk, &tb2->owners);
+	inet_csk(sk)->icsk_bind2_hash = tb2;
+	spin_unlock_bh(&head2->lock);
+
+	if (tb2 != new_tb2)
+		kmem_cache_free(hinfo->bind2_bucket_cachep, new_tb2);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(inet_bhash2_update_saddr);
+
 /* RFC 6056 3.3.4.  Algorithm 4: Double-Hash Port Selection Algorithm
  * Note that we use 32bit integers (vs RFC 'short integers')
  * because 2^16 is not a multiple of num_ephemeral and this
@@ -694,11 +921,13 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
 			struct sock *, __u16, struct inet_timewait_sock **))
 {
 	struct inet_hashinfo *hinfo = death_row->hashinfo;
+	struct inet_bind_hashbucket *head, *head2;
 	struct inet_timewait_sock *tw = NULL;
-	struct inet_bind_hashbucket *head;
 	int port = inet_sk(sk)->inet_num;
 	struct net *net = sock_net(sk);
+	struct inet_bind2_bucket *tb2;
 	struct inet_bind_bucket *tb;
+	bool tb_created = false;
 	u32 remaining, offset;
 	int ret, i, low, high;
 	int l3mdev;
@@ -729,8 +958,8 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
 	if (likely(remaining > 1))
 		remaining &= ~1U;
 
-	net_get_random_once(table_perturb,
-			    INET_TABLE_PERTURB_SIZE * sizeof(*table_perturb));
+	get_random_sleepable_once(table_perturb,
+				  INET_TABLE_PERTURB_SIZE * sizeof(*table_perturb));
 	index = port_offset & (INET_TABLE_PERTURB_SIZE - 1);
 
 	offset = READ_ONCE(table_perturb[index]) + (port_offset >> 32);
@@ -755,8 +984,7 @@ other_parity_scan:
 		 * the established check is already unique enough.
 		 */
 		inet_bind_bucket_for_each(tb, &head->chain) {
-			if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
-			    tb->port == port) {
+			if (inet_bind_bucket_match(tb, net, port, l3mdev)) {
 				if (tb->fastreuse >= 0 ||
 				    tb->fastreuseport >= 0)
 					goto next_port;
@@ -774,6 +1002,7 @@ other_parity_scan:
 			spin_unlock_bh(&head->lock);
 			return -ENOMEM;
 		}
+		tb_created = true;
 		tb->fastreuse = -1;
 		tb->fastreuseport = -1;
 		goto ok;
@@ -789,6 +1018,20 @@ next_port:
 	return -EADDRNOTAVAIL;
 
 ok:
+	/* Find the corresponding tb2 bucket since we need to
+	 * add the socket to the bhash2 table as well
+	 */
+	head2 = inet_bhashfn_portaddr(hinfo, sk, net, port);
+	spin_lock(&head2->lock);
+
+	tb2 = inet_bind2_bucket_find(head2, net, port, l3mdev, sk);
+	if (!tb2) {
+		tb2 = inet_bind2_bucket_create(hinfo->bind2_bucket_cachep, net,
+					       head2, port, l3mdev, sk);
+		if (!tb2)
+			goto error;
+	}
+
 	/* Here we want to add a little bit of randomness to the next source
 	 * port that will be chosen. We use a max() with a random here so that
 	 * on low contention the randomness is maximal and on high contention
@@ -798,7 +1041,10 @@ ok:
 	WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);
 
 	/* Head lock still held and bh's disabled */
-	inet_bind_hash(sk, tb, port);
+	inet_bind_hash(sk, tb, tb2, port);
+
+	spin_unlock(&head2->lock);
+
 	if (sk_unhashed(sk)) {
 		inet_sk(sk)->inet_sport = htons(port);
 		inet_ehash_nolisten(sk, (struct sock *)tw, NULL);
@@ -810,6 +1056,13 @@ ok:
 		inet_twsk_deschedule_put(tw);
 	local_bh_enable();
 	return 0;
+
+error:
+	spin_unlock(&head2->lock);
+	if (tb_created)
+		inet_bind_bucket_destroy(hinfo->bind_bucket_cachep, tb);
+	spin_unlock_bh(&head->lock);
+	return -ENOMEM;
 }
 
 /*
@@ -902,3 +1155,50 @@ int inet_ehash_locks_alloc(struct inet_hashinfo *hashinfo)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(inet_ehash_locks_alloc);
+
+struct inet_hashinfo *inet_pernet_hashinfo_alloc(struct inet_hashinfo *hashinfo,
+						 unsigned int ehash_entries)
+{
+	struct inet_hashinfo *new_hashinfo;
+	int i;
+
+	new_hashinfo = kmemdup(hashinfo, sizeof(*hashinfo), GFP_KERNEL);
+	if (!new_hashinfo)
+		goto err;
+
+	new_hashinfo->ehash = vmalloc_huge(ehash_entries * sizeof(struct inet_ehash_bucket),
+					   GFP_KERNEL_ACCOUNT);
+	if (!new_hashinfo->ehash)
+		goto free_hashinfo;
+
+	new_hashinfo->ehash_mask = ehash_entries - 1;
+
+	if (inet_ehash_locks_alloc(new_hashinfo))
+		goto free_ehash;
+
+	for (i = 0; i < ehash_entries; i++)
+		INIT_HLIST_NULLS_HEAD(&new_hashinfo->ehash[i].chain, i);
+
+	new_hashinfo->pernet = true;
+
+	return new_hashinfo;
+
+free_ehash:
+	vfree(new_hashinfo->ehash);
+free_hashinfo:
+	kfree(new_hashinfo);
+err:
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(inet_pernet_hashinfo_alloc);
+
+void inet_pernet_hashinfo_free(struct inet_hashinfo *hashinfo)
+{
+	if (!hashinfo->pernet)
+		return;
+
+	inet_ehash_locks_free(hashinfo);
+	vfree(hashinfo->ehash);
+	kfree(hashinfo);
+}
+EXPORT_SYMBOL_GPL(inet_pernet_hashinfo_free);
diff --git a/net/ipv4/inet_timewait_sock.c b/net/ipv4/inet_timewait_sock.c
index 47ccc343c9fb..71d3bb0abf6c 100644
--- a/net/ipv4/inet_timewait_sock.c
+++ b/net/ipv4/inet_timewait_sock.c
@@ -59,9 +59,7 @@ static void inet_twsk_kill(struct inet_timewait_sock *tw)
 	inet_twsk_bind_unhash(tw, hashinfo);
 	spin_unlock(&bhead->lock);
 
-	if (refcount_dec_and_test(&tw->tw_dr->tw_refcount))
-		kfree(tw->tw_dr);
-
+	refcount_dec(&tw->tw_dr->tw_refcount);
 	inet_twsk_put(tw);
 }
 
diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index 04e2034f2f8e..1ae83ad629b2 100644
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -1043,7 +1043,7 @@ static int __ip_append_data(struct sock *sk,
 				paged = true;
 				zc = true;
 			} else {
-				uarg->zerocopy = 0;
+				uarg_to_msgzc(uarg)->zerocopy = 0;
 				skb_zcopy_set(skb, uarg, &extra_uref);
 			}
 		}
@@ -1109,10 +1109,7 @@ alloc_new_skb:
 				 (fraglen + alloc_extra < SKB_MAX_ALLOC ||
 				  !(rt->dst.dev->features & NETIF_F_SG)))
 				alloclen = fraglen;
-			else if (!zc) {
-				alloclen = min_t(int, fraglen, MAX_HEADER);
-				pagedlen = fraglen - alloclen;
-			} else {
+			else {
 				alloclen = fragheaderlen + transhdrlen;
 				pagedlen = datalen - transhdrlen;
 			}
diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c
index e49a61a053a6..6e19cad154f5 100644
--- a/net/ipv4/ip_sockglue.c
+++ b/net/ipv4/ip_sockglue.c
@@ -888,8 +888,8 @@ static int compat_ip_mcast_join_leave(struct sock *sk, int optname,
 
 DEFINE_STATIC_KEY_FALSE(ip4_min_ttl);
 
-static int do_ip_setsockopt(struct sock *sk, int level, int optname,
-		sockptr_t optval, unsigned int optlen)
+int do_ip_setsockopt(struct sock *sk, int level, int optname,
+		     sockptr_t optval, unsigned int optlen)
 {
 	struct inet_sock *inet = inet_sk(sk);
 	struct net *net = sock_net(sk);
@@ -944,7 +944,7 @@ static int do_ip_setsockopt(struct sock *sk, int level, int optname,
 	err = 0;
 	if (needs_rtnl)
 		rtnl_lock();
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 
 	switch (optname) {
 	case IP_OPTIONS:
@@ -1333,14 +1333,14 @@ static int do_ip_setsockopt(struct sock *sk, int level, int optname,
 	case IP_IPSEC_POLICY:
 	case IP_XFRM_POLICY:
 		err = -EPERM;
-		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
+		if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
 			break;
 		err = xfrm_user_policy(sk, optname, optval, optlen);
 		break;
 
 	case IP_TRANSPARENT:
-		if (!!val && !ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
-		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
+		if (!!val && !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
+		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
 			err = -EPERM;
 			break;
 		}
@@ -1368,13 +1368,13 @@ static int do_ip_setsockopt(struct sock *sk, int level, int optname,
 		err = -ENOPROTOOPT;
 		break;
 	}
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (needs_rtnl)
 		rtnl_unlock();
 	return err;
 
 e_inval:
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (needs_rtnl)
 		rtnl_unlock();
 	return -EINVAL;
@@ -1462,37 +1462,37 @@ static bool getsockopt_needs_rtnl(int optname)
 	return false;
 }
 
-static int ip_get_mcast_msfilter(struct sock *sk, void __user *optval,
-		int __user *optlen, int len)
+static int ip_get_mcast_msfilter(struct sock *sk, sockptr_t optval,
+				 sockptr_t optlen, int len)
 {
 	const int size0 = offsetof(struct group_filter, gf_slist_flex);
-	struct group_filter __user *p = optval;
 	struct group_filter gsf;
-	int num;
+	int num, gsf_size;
 	int err;
 
 	if (len < size0)
 		return -EINVAL;
-	if (copy_from_user(&gsf, p, size0))
+	if (copy_from_sockptr(&gsf, optval, size0))
 		return -EFAULT;
 
 	num = gsf.gf_numsrc;
-	err = ip_mc_gsfget(sk, &gsf, p->gf_slist_flex);
+	err = ip_mc_gsfget(sk, &gsf, optval,
+			   offsetof(struct group_filter, gf_slist_flex));
 	if (err)
 		return err;
 	if (gsf.gf_numsrc < num)
 		num = gsf.gf_numsrc;
-	if (put_user(GROUP_FILTER_SIZE(num), optlen) ||
-	    copy_to_user(p, &gsf, size0))
+	gsf_size = GROUP_FILTER_SIZE(num);
+	if (copy_to_sockptr(optlen, &gsf_size, sizeof(int)) ||
+	    copy_to_sockptr(optval, &gsf, size0))
 		return -EFAULT;
 	return 0;
 }
 
-static int compat_ip_get_mcast_msfilter(struct sock *sk, void __user *optval,
-		int __user *optlen, int len)
+static int compat_ip_get_mcast_msfilter(struct sock *sk, sockptr_t optval,
+					sockptr_t optlen, int len)
 {
 	const int size0 = offsetof(struct compat_group_filter, gf_slist_flex);
-	struct compat_group_filter __user *p = optval;
 	struct compat_group_filter gf32;
 	struct group_filter gf;
 	int num;
@@ -1500,7 +1500,7 @@ static int compat_ip_get_mcast_msfilter(struct sock *sk, void __user *optval,
 
 	if (len < size0)
 		return -EINVAL;
-	if (copy_from_user(&gf32, p, size0))
+	if (copy_from_sockptr(&gf32, optval, size0))
 		return -EFAULT;
 
 	gf.gf_interface = gf32.gf_interface;
@@ -1508,21 +1508,24 @@ static int compat_ip_get_mcast_msfilter(struct sock *sk, void __user *optval,
 	num = gf.gf_numsrc = gf32.gf_numsrc;
 	gf.gf_group = gf32.gf_group;
 
-	err = ip_mc_gsfget(sk, &gf, p->gf_slist_flex);
+	err = ip_mc_gsfget(sk, &gf, optval,
+			   offsetof(struct compat_group_filter, gf_slist_flex));
 	if (err)
 		return err;
 	if (gf.gf_numsrc < num)
 		num = gf.gf_numsrc;
 	len = GROUP_FILTER_SIZE(num) - (sizeof(gf) - sizeof(gf32));
-	if (put_user(len, optlen) ||
-	    put_user(gf.gf_fmode, &p->gf_fmode) ||
-	    put_user(gf.gf_numsrc, &p->gf_numsrc))
+	if (copy_to_sockptr(optlen, &len, sizeof(int)) ||
+	    copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_fmode),
+				   &gf.gf_fmode, sizeof(gf.gf_fmode)) ||
+	    copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_numsrc),
+				   &gf.gf_numsrc, sizeof(gf.gf_numsrc)))
 		return -EFAULT;
 	return 0;
 }
 
-static int do_ip_getsockopt(struct sock *sk, int level, int optname,
-			    char __user *optval, int __user *optlen)
+int do_ip_getsockopt(struct sock *sk, int level, int optname,
+		     sockptr_t optval, sockptr_t optlen)
 {
 	struct inet_sock *inet = inet_sk(sk);
 	bool needs_rtnl = getsockopt_needs_rtnl(optname);
@@ -1535,14 +1538,14 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname,
 	if (ip_mroute_opt(optname))
 		return ip_mroute_getsockopt(sk, optname, optval, optlen);
 
-	if (get_user(len, optlen))
+	if (copy_from_sockptr(&len, optlen, sizeof(int)))
 		return -EFAULT;
 	if (len < 0)
 		return -EINVAL;
 
 	if (needs_rtnl)
 		rtnl_lock();
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 
 	switch (optname) {
 	case IP_OPTIONS:
@@ -1558,17 +1561,19 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname,
 			memcpy(optbuf, &inet_opt->opt,
 			       sizeof(struct ip_options) +
 			       inet_opt->opt.optlen);
-		release_sock(sk);
+		sockopt_release_sock(sk);
 
-		if (opt->optlen == 0)
-			return put_user(0, optlen);
+		if (opt->optlen == 0) {
+			len = 0;
+			return copy_to_sockptr(optlen, &len, sizeof(int));
+		}
 
 		ip_options_undo(opt);
 
 		len = min_t(unsigned int, len, opt->optlen);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, opt->__data, len))
+		if (copy_to_sockptr(optval, opt->__data, len))
 			return -EFAULT;
 		return 0;
 	}
@@ -1632,7 +1637,7 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname,
 			dst_release(dst);
 		}
 		if (!val) {
-			release_sock(sk);
+			sockopt_release_sock(sk);
 			return -ENOTCONN;
 		}
 		break;
@@ -1657,11 +1662,11 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname,
 		struct in_addr addr;
 		len = min_t(unsigned int, len, sizeof(struct in_addr));
 		addr.s_addr = inet->mc_addr;
-		release_sock(sk);
+		sockopt_release_sock(sk);
 
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &addr, len))
+		if (copy_to_sockptr(optval, &addr, len))
 			return -EFAULT;
 		return 0;
 	}
@@ -1673,12 +1678,11 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname,
 			err = -EINVAL;
 			goto out;
 		}
-		if (copy_from_user(&msf, optval, IP_MSFILTER_SIZE(0))) {
+		if (copy_from_sockptr(&msf, optval, IP_MSFILTER_SIZE(0))) {
 			err = -EFAULT;
 			goto out;
 		}
-		err = ip_mc_msfget(sk, &msf,
-				   (struct ip_msfilter __user *)optval, optlen);
+		err = ip_mc_msfget(sk, &msf, optval, optlen);
 		goto out;
 	}
 	case MCAST_MSFILTER:
@@ -1695,13 +1699,18 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname,
 	{
 		struct msghdr msg;
 
-		release_sock(sk);
+		sockopt_release_sock(sk);
 
 		if (sk->sk_type != SOCK_STREAM)
 			return -ENOPROTOOPT;
 
-		msg.msg_control_is_user = true;
-		msg.msg_control_user = optval;
+		if (optval.is_kernel) {
+			msg.msg_control_is_user = false;
+			msg.msg_control = optval.kernel;
+		} else {
+			msg.msg_control_is_user = true;
+			msg.msg_control_user = optval.user;
+		}
 		msg.msg_controllen = len;
 		msg.msg_flags = in_compat_syscall() ? MSG_CMSG_COMPAT : 0;
 
@@ -1722,7 +1731,7 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname,
 			put_cmsg(&msg, SOL_IP, IP_TOS, sizeof(tos), &tos);
 		}
 		len -= msg.msg_controllen;
-		return put_user(len, optlen);
+		return copy_to_sockptr(optlen, &len, sizeof(int));
 	}
 	case IP_FREEBIND:
 		val = inet->freebind;
@@ -1734,29 +1743,29 @@ static int do_ip_getsockopt(struct sock *sk, int level, int optname,
 		val = inet->min_ttl;
 		break;
 	default:
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		return -ENOPROTOOPT;
 	}
-	release_sock(sk);
+	sockopt_release_sock(sk);
 
 	if (len < sizeof(int) && len > 0 && val >= 0 && val <= 255) {
 		unsigned char ucval = (unsigned char)val;
 		len = 1;
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &ucval, 1))
+		if (copy_to_sockptr(optval, &ucval, 1))
 			return -EFAULT;
 	} else {
 		len = min_t(unsigned int, sizeof(int), len);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &val, len))
+		if (copy_to_sockptr(optval, &val, len))
 			return -EFAULT;
 	}
 	return 0;
 
 out:
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (needs_rtnl)
 		rtnl_unlock();
 	return err;
@@ -1767,7 +1776,8 @@ int ip_getsockopt(struct sock *sk, int level,
 {
 	int err;
 
-	err = do_ip_getsockopt(sk, level, optname, optval, optlen);
+	err = do_ip_getsockopt(sk, level, optname,
+			       USER_SOCKPTR(optval), USER_SOCKPTR(optlen));
 
 #if IS_ENABLED(CONFIG_BPFILTER_UMH)
 	if (optname >= BPFILTER_IPT_SO_GET_INFO &&
diff --git a/net/ipv4/ip_tunnel_core.c b/net/ipv4/ip_tunnel_core.c
index cc1caab4a654..92c02c886fe7 100644
--- a/net/ipv4/ip_tunnel_core.c
+++ b/net/ipv4/ip_tunnel_core.c
@@ -1079,3 +1079,70 @@ EXPORT_SYMBOL(ip_tunnel_parse_protocol);
 
 const struct header_ops ip_tunnel_header_ops = { .parse_protocol = ip_tunnel_parse_protocol };
 EXPORT_SYMBOL(ip_tunnel_header_ops);
+
+/* This function returns true when ENCAP attributes are present in the nl msg */
+bool ip_tunnel_netlink_encap_parms(struct nlattr *data[],
+				   struct ip_tunnel_encap *encap)
+{
+	bool ret = false;
+
+	memset(encap, 0, sizeof(*encap));
+
+	if (!data)
+		return ret;
+
+	if (data[IFLA_IPTUN_ENCAP_TYPE]) {
+		ret = true;
+		encap->type = nla_get_u16(data[IFLA_IPTUN_ENCAP_TYPE]);
+	}
+
+	if (data[IFLA_IPTUN_ENCAP_FLAGS]) {
+		ret = true;
+		encap->flags = nla_get_u16(data[IFLA_IPTUN_ENCAP_FLAGS]);
+	}
+
+	if (data[IFLA_IPTUN_ENCAP_SPORT]) {
+		ret = true;
+		encap->sport = nla_get_be16(data[IFLA_IPTUN_ENCAP_SPORT]);
+	}
+
+	if (data[IFLA_IPTUN_ENCAP_DPORT]) {
+		ret = true;
+		encap->dport = nla_get_be16(data[IFLA_IPTUN_ENCAP_DPORT]);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ip_tunnel_netlink_encap_parms);
+
+void ip_tunnel_netlink_parms(struct nlattr *data[],
+			     struct ip_tunnel_parm *parms)
+{
+	if (data[IFLA_IPTUN_LINK])
+		parms->link = nla_get_u32(data[IFLA_IPTUN_LINK]);
+
+	if (data[IFLA_IPTUN_LOCAL])
+		parms->iph.saddr = nla_get_be32(data[IFLA_IPTUN_LOCAL]);
+
+	if (data[IFLA_IPTUN_REMOTE])
+		parms->iph.daddr = nla_get_be32(data[IFLA_IPTUN_REMOTE]);
+
+	if (data[IFLA_IPTUN_TTL]) {
+		parms->iph.ttl = nla_get_u8(data[IFLA_IPTUN_TTL]);
+		if (parms->iph.ttl)
+			parms->iph.frag_off = htons(IP_DF);
+	}
+
+	if (data[IFLA_IPTUN_TOS])
+		parms->iph.tos = nla_get_u8(data[IFLA_IPTUN_TOS]);
+
+	if (!data[IFLA_IPTUN_PMTUDISC] || nla_get_u8(data[IFLA_IPTUN_PMTUDISC]))
+		parms->iph.frag_off = htons(IP_DF);
+
+	if (data[IFLA_IPTUN_FLAGS])
+		parms->i_flags = nla_get_be16(data[IFLA_IPTUN_FLAGS]);
+
+	if (data[IFLA_IPTUN_PROTO])
+		parms->iph.protocol = nla_get_u8(data[IFLA_IPTUN_PROTO]);
+}
+EXPORT_SYMBOL_GPL(ip_tunnel_netlink_parms);
diff --git a/net/ipv4/ipcomp.c b/net/ipv4/ipcomp.c
index 366094c1ce6c..5a4fb2539b08 100644
--- a/net/ipv4/ipcomp.c
+++ b/net/ipv4/ipcomp.c
@@ -117,7 +117,8 @@ out:
 	return err;
 }
 
-static int ipcomp4_init_state(struct xfrm_state *x)
+static int ipcomp4_init_state(struct xfrm_state *x,
+			      struct netlink_ext_ack *extack)
 {
 	int err = -EINVAL;
 
@@ -129,17 +130,20 @@ static int ipcomp4_init_state(struct xfrm_state *x)
 		x->props.header_len += sizeof(struct iphdr);
 		break;
 	default:
+		NL_SET_ERR_MSG(extack, "Unsupported XFRM mode for IPcomp");
 		goto out;
 	}
 
-	err = ipcomp_init_state(x);
+	err = ipcomp_init_state(x, extack);
 	if (err)
 		goto out;
 
 	if (x->props.mode == XFRM_MODE_TUNNEL) {
 		err = ipcomp_tunnel_attach(x);
-		if (err)
+		if (err) {
+			NL_SET_ERR_MSG(extack, "Kernel error: failed to initialize the associated state");
 			goto out;
+		}
 	}
 
 	err = 0;
diff --git a/net/ipv4/ipip.c b/net/ipv4/ipip.c
index 123ea63a04cb..180f9daf5bec 100644
--- a/net/ipv4/ipip.c
+++ b/net/ipv4/ipip.c
@@ -417,29 +417,7 @@ static void ipip_netlink_parms(struct nlattr *data[],
 	if (!data)
 		return;
 
-	if (data[IFLA_IPTUN_LINK])
-		parms->link = nla_get_u32(data[IFLA_IPTUN_LINK]);
-
-	if (data[IFLA_IPTUN_LOCAL])
-		parms->iph.saddr = nla_get_in_addr(data[IFLA_IPTUN_LOCAL]);
-
-	if (data[IFLA_IPTUN_REMOTE])
-		parms->iph.daddr = nla_get_in_addr(data[IFLA_IPTUN_REMOTE]);
-
-	if (data[IFLA_IPTUN_TTL]) {
-		parms->iph.ttl = nla_get_u8(data[IFLA_IPTUN_TTL]);
-		if (parms->iph.ttl)
-			parms->iph.frag_off = htons(IP_DF);
-	}
-
-	if (data[IFLA_IPTUN_TOS])
-		parms->iph.tos = nla_get_u8(data[IFLA_IPTUN_TOS]);
-
-	if (data[IFLA_IPTUN_PROTO])
-		parms->iph.protocol = nla_get_u8(data[IFLA_IPTUN_PROTO]);
-
-	if (!data[IFLA_IPTUN_PMTUDISC] || nla_get_u8(data[IFLA_IPTUN_PMTUDISC]))
-		parms->iph.frag_off = htons(IP_DF);
+	ip_tunnel_netlink_parms(data, parms);
 
 	if (data[IFLA_IPTUN_COLLECT_METADATA])
 		*collect_md = true;
@@ -448,40 +426,6 @@ static void ipip_netlink_parms(struct nlattr *data[],
 		*fwmark = nla_get_u32(data[IFLA_IPTUN_FWMARK]);
 }
 
-/* This function returns true when ENCAP attributes are present in the nl msg */
-static bool ipip_netlink_encap_parms(struct nlattr *data[],
-				     struct ip_tunnel_encap *ipencap)
-{
-	bool ret = false;
-
-	memset(ipencap, 0, sizeof(*ipencap));
-
-	if (!data)
-		return ret;
-
-	if (data[IFLA_IPTUN_ENCAP_TYPE]) {
-		ret = true;
-		ipencap->type = nla_get_u16(data[IFLA_IPTUN_ENCAP_TYPE]);
-	}
-
-	if (data[IFLA_IPTUN_ENCAP_FLAGS]) {
-		ret = true;
-		ipencap->flags = nla_get_u16(data[IFLA_IPTUN_ENCAP_FLAGS]);
-	}
-
-	if (data[IFLA_IPTUN_ENCAP_SPORT]) {
-		ret = true;
-		ipencap->sport = nla_get_be16(data[IFLA_IPTUN_ENCAP_SPORT]);
-	}
-
-	if (data[IFLA_IPTUN_ENCAP_DPORT]) {
-		ret = true;
-		ipencap->dport = nla_get_be16(data[IFLA_IPTUN_ENCAP_DPORT]);
-	}
-
-	return ret;
-}
-
 static int ipip_newlink(struct net *src_net, struct net_device *dev,
 			struct nlattr *tb[], struct nlattr *data[],
 			struct netlink_ext_ack *extack)
@@ -491,7 +435,7 @@ static int ipip_newlink(struct net *src_net, struct net_device *dev,
 	struct ip_tunnel_encap ipencap;
 	__u32 fwmark = 0;
 
-	if (ipip_netlink_encap_parms(data, &ipencap)) {
+	if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
 		int err = ip_tunnel_encap_setup(t, &ipencap);
 
 		if (err < 0)
@@ -512,7 +456,7 @@ static int ipip_changelink(struct net_device *dev, struct nlattr *tb[],
 	bool collect_md;
 	__u32 fwmark = t->fwmark;
 
-	if (ipip_netlink_encap_parms(data, &ipencap)) {
+	if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
 		int err = ip_tunnel_encap_setup(t, &ipencap);
 
 		if (err < 0)
diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c
index e11d6b0b62b7..e04544ac4b45 100644
--- a/net/ipv4/ipmr.c
+++ b/net/ipv4/ipmr.c
@@ -1548,7 +1548,8 @@ out:
 }
 
 /* Getsock opt support for the multicast routing system. */
-int ip_mroute_getsockopt(struct sock *sk, int optname, char __user *optval, int __user *optlen)
+int ip_mroute_getsockopt(struct sock *sk, int optname, sockptr_t optval,
+			 sockptr_t optlen)
 {
 	int olr;
 	int val;
@@ -1579,14 +1580,14 @@ int ip_mroute_getsockopt(struct sock *sk, int optname, char __user *optval, int
 		return -ENOPROTOOPT;
 	}
 
-	if (get_user(olr, optlen))
+	if (copy_from_sockptr(&olr, optlen, sizeof(int)))
 		return -EFAULT;
 	olr = min_t(unsigned int, olr, sizeof(int));
 	if (olr < 0)
 		return -EINVAL;
-	if (put_user(olr, optlen))
+	if (copy_to_sockptr(optlen, &olr, sizeof(int)))
 		return -EFAULT;
-	if (copy_to_user(optval, &val, olr))
+	if (copy_to_sockptr(optval, &val, olr))
 		return -EFAULT;
 	return 0;
 }
diff --git a/net/ipv4/netfilter/ipt_rpfilter.c b/net/ipv4/netfilter/ipt_rpfilter.c
index 8cd3224d913e..8183bbcabb4a 100644
--- a/net/ipv4/netfilter/ipt_rpfilter.c
+++ b/net/ipv4/netfilter/ipt_rpfilter.c
@@ -33,7 +33,6 @@ static bool rpfilter_lookup_reverse(struct net *net, struct flowi4 *fl4,
 				const struct net_device *dev, u8 flags)
 {
 	struct fib_result res;
-	int ret __maybe_unused;
 
 	if (fib_lookup(net, fl4, &res, FIB_LOOKUP_IGNORE_LINKSTATE))
 		return false;
diff --git a/net/ipv4/netfilter/nf_nat_h323.c b/net/ipv4/netfilter/nf_nat_h323.c
index a334f0dcc2d0..faee20af4856 100644
--- a/net/ipv4/netfilter/nf_nat_h323.c
+++ b/net/ipv4/netfilter/nf_nat_h323.c
@@ -291,20 +291,7 @@ static int nat_t120(struct sk_buff *skb, struct nf_conn *ct,
 	exp->expectfn = nf_nat_follow_master;
 	exp->dir = !dir;
 
-	/* Try to get same port: if not, try to change it. */
-	for (; nated_port != 0; nated_port++) {
-		int ret;
-
-		exp->tuple.dst.u.tcp.port = htons(nated_port);
-		ret = nf_ct_expect_related(exp, 0);
-		if (ret == 0)
-			break;
-		else if (ret != -EBUSY) {
-			nated_port = 0;
-			break;
-		}
-	}
-
+	nated_port = nf_nat_exp_find_port(exp, nated_port);
 	if (nated_port == 0) {	/* No port available */
 		net_notice_ratelimited("nf_nat_h323: out of TCP ports\n");
 		return 0;
@@ -347,20 +334,7 @@ static int nat_h245(struct sk_buff *skb, struct nf_conn *ct,
 	if (info->sig_port[dir] == port)
 		nated_port = ntohs(info->sig_port[!dir]);
 
-	/* Try to get same port: if not, try to change it. */
-	for (; nated_port != 0; nated_port++) {
-		int ret;
-
-		exp->tuple.dst.u.tcp.port = htons(nated_port);
-		ret = nf_ct_expect_related(exp, 0);
-		if (ret == 0)
-			break;
-		else if (ret != -EBUSY) {
-			nated_port = 0;
-			break;
-		}
-	}
-
+	nated_port = nf_nat_exp_find_port(exp, nated_port);
 	if (nated_port == 0) {	/* No port available */
 		net_notice_ratelimited("nf_nat_q931: out of TCP ports\n");
 		return 0;
@@ -439,20 +413,7 @@ static int nat_q931(struct sk_buff *skb, struct nf_conn *ct,
 	if (info->sig_port[dir] == port)
 		nated_port = ntohs(info->sig_port[!dir]);
 
-	/* Try to get same port: if not, try to change it. */
-	for (; nated_port != 0; nated_port++) {
-		int ret;
-
-		exp->tuple.dst.u.tcp.port = htons(nated_port);
-		ret = nf_ct_expect_related(exp, 0);
-		if (ret == 0)
-			break;
-		else if (ret != -EBUSY) {
-			nated_port = 0;
-			break;
-		}
-	}
-
+	nated_port = nf_nat_exp_find_port(exp, nated_port);
 	if (nated_port == 0) {	/* No port available */
 		net_notice_ratelimited("nf_nat_ras: out of TCP ports\n");
 		return 0;
@@ -532,20 +493,7 @@ static int nat_callforwarding(struct sk_buff *skb, struct nf_conn *ct,
 	exp->expectfn = ip_nat_callforwarding_expect;
 	exp->dir = !dir;
 
-	/* Try to get same port: if not, try to change it. */
-	for (nated_port = ntohs(port); nated_port != 0; nated_port++) {
-		int ret;
-
-		exp->tuple.dst.u.tcp.port = htons(nated_port);
-		ret = nf_ct_expect_related(exp, 0);
-		if (ret == 0)
-			break;
-		else if (ret != -EBUSY) {
-			nated_port = 0;
-			break;
-		}
-	}
-
+	nated_port = nf_nat_exp_find_port(exp, ntohs(port));
 	if (nated_port == 0) {	/* No port available */
 		net_notice_ratelimited("nf_nat_q931: out of TCP ports\n");
 		return 0;
diff --git a/net/ipv4/netfilter/nf_socket_ipv4.c b/net/ipv4/netfilter/nf_socket_ipv4.c
index 2d42e4c35a20..a1350fc25838 100644
--- a/net/ipv4/netfilter/nf_socket_ipv4.c
+++ b/net/ipv4/netfilter/nf_socket_ipv4.c
@@ -71,8 +71,8 @@ nf_socket_get_sock_v4(struct net *net, struct sk_buff *skb, const int doff,
 {
 	switch (protocol) {
 	case IPPROTO_TCP:
-		return inet_lookup(net, &tcp_hashinfo, skb, doff,
-				   saddr, sport, daddr, dport,
+		return inet_lookup(net, net->ipv4.tcp_death_row.hashinfo,
+				   skb, doff, saddr, sport, daddr, dport,
 				   in->ifindex);
 	case IPPROTO_UDP:
 		return udp4_lib_lookup(net, saddr, sport, daddr, dport,
diff --git a/net/ipv4/netfilter/nf_tproxy_ipv4.c b/net/ipv4/netfilter/nf_tproxy_ipv4.c
index b2bae0b0e42a..b22b2c745c76 100644
--- a/net/ipv4/netfilter/nf_tproxy_ipv4.c
+++ b/net/ipv4/netfilter/nf_tproxy_ipv4.c
@@ -79,6 +79,7 @@ nf_tproxy_get_sock_v4(struct net *net, struct sk_buff *skb,
 		      const struct net_device *in,
 		      const enum nf_tproxy_lookup_t lookup_type)
 {
+	struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo;
 	struct sock *sk;
 
 	switch (protocol) {
@@ -92,12 +93,10 @@ nf_tproxy_get_sock_v4(struct net *net, struct sk_buff *skb,
 
 		switch (lookup_type) {
 		case NF_TPROXY_LOOKUP_LISTENER:
-			sk = inet_lookup_listener(net, &tcp_hashinfo, skb,
-						    ip_hdrlen(skb) +
-						      __tcp_hdrlen(hp),
-						    saddr, sport,
-						    daddr, dport,
-						    in->ifindex, 0);
+			sk = inet_lookup_listener(net, hinfo, skb,
+						  ip_hdrlen(skb) + __tcp_hdrlen(hp),
+						  saddr, sport, daddr, dport,
+						  in->ifindex, 0);
 
 			if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
 				sk = NULL;
@@ -108,9 +107,8 @@ nf_tproxy_get_sock_v4(struct net *net, struct sk_buff *skb,
 			 */
 			break;
 		case NF_TPROXY_LOOKUP_ESTABLISHED:
-			sk = inet_lookup_established(net, &tcp_hashinfo,
-						    saddr, sport, daddr, dport,
-						    in->ifindex);
+			sk = inet_lookup_established(net, hinfo, saddr, sport,
+						     daddr, dport, in->ifindex);
 			break;
 		default:
 			BUG();
diff --git a/net/ipv4/netfilter/nft_fib_ipv4.c b/net/ipv4/netfilter/nft_fib_ipv4.c
index b75cac69bd7e..7ade04ff972d 100644
--- a/net/ipv4/netfilter/nft_fib_ipv4.c
+++ b/net/ipv4/netfilter/nft_fib_ipv4.c
@@ -83,6 +83,9 @@ void nft_fib4_eval(const struct nft_expr *expr, struct nft_regs *regs,
 	else
 		oif = NULL;
 
+	if (priv->flags & NFTA_FIB_F_IIF)
+		fl4.flowi4_oif = l3mdev_master_ifindex_rcu(oif);
+
 	if (nft_hook(pkt) == NF_INET_PRE_ROUTING &&
 	    nft_fib_is_loopback(pkt->skb, nft_in(pkt))) {
 		nft_fib_store_result(dest, priv, nft_in(pkt));
diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c
index b83c2bd9d722..517042caf6dc 100644
--- a/net/ipv4/ping.c
+++ b/net/ipv4/ping.c
@@ -33,6 +33,7 @@
 #include <linux/skbuff.h>
 #include <linux/proc_fs.h>
 #include <linux/export.h>
+#include <linux/bpf-cgroup.h>
 #include <net/sock.h>
 #include <net/ping.h>
 #include <net/udp.h>
@@ -295,6 +296,19 @@ void ping_close(struct sock *sk, long timeout)
 }
 EXPORT_SYMBOL_GPL(ping_close);
 
+static int ping_pre_connect(struct sock *sk, struct sockaddr *uaddr,
+			    int addr_len)
+{
+	/* This check is replicated from __ip4_datagram_connect() and
+	 * intended to prevent BPF program called below from accessing bytes
+	 * that are out of the bound specified by user in addr_len.
+	 */
+	if (addr_len < sizeof(struct sockaddr_in))
+		return -EINVAL;
+
+	return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr);
+}
+
 /* Checks the bind address and possibly modifies sk->sk_bound_dev_if. */
 static int ping_check_bind_addr(struct sock *sk, struct inet_sock *isk,
 				struct sockaddr *uaddr, int addr_len)
@@ -1009,6 +1023,7 @@ struct proto ping_prot = {
 	.owner =	THIS_MODULE,
 	.init =		ping_init_sock,
 	.close =	ping_close,
+	.pre_connect =	ping_pre_connect,
 	.connect =	ip4_datagram_connect,
 	.disconnect =	__udp_disconnect,
 	.setsockopt =	ip_setsockopt,
diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c
index 0088a4c64d77..5386f460bd20 100644
--- a/net/ipv4/proc.c
+++ b/net/ipv4/proc.c
@@ -59,7 +59,7 @@ static int sockstat_seq_show(struct seq_file *seq, void *v)
 	socket_seq_show(seq);
 	seq_printf(seq, "TCP: inuse %d orphan %d tw %d alloc %d mem %ld\n",
 		   sock_prot_inuse_get(net, &tcp_prot), orphans,
-		   refcount_read(&net->ipv4.tcp_death_row->tw_refcount) - 1,
+		   refcount_read(&net->ipv4.tcp_death_row.tw_refcount) - 1,
 		   sockets, proto_memory_allocated(&tcp_prot));
 	seq_printf(seq, "UDP: inuse %d mem %ld\n",
 		   sock_prot_inuse_get(net, &udp_prot),
diff --git a/net/ipv4/sysctl_net_ipv4.c b/net/ipv4/sysctl_net_ipv4.c
index 5490c285668b..9b8a6db7a66b 100644
--- a/net/ipv4/sysctl_net_ipv4.c
+++ b/net/ipv4/sysctl_net_ipv4.c
@@ -39,6 +39,7 @@ static u32 u32_max_div_HZ = UINT_MAX / HZ;
 static int one_day_secs = 24 * 3600;
 static u32 fib_multipath_hash_fields_all_mask __maybe_unused =
 	FIB_MULTIPATH_HASH_FIELD_ALL_MASK;
+static unsigned int tcp_child_ehash_entries_max = 16 * 1024 * 1024;
 
 /* obsolete */
 static int sysctl_tcp_low_latency __read_mostly;
@@ -382,6 +383,29 @@ static int proc_tcp_available_ulp(struct ctl_table *ctl,
 	return ret;
 }
 
+static int proc_tcp_ehash_entries(struct ctl_table *table, int write,
+				  void *buffer, size_t *lenp, loff_t *ppos)
+{
+	struct net *net = container_of(table->data, struct net,
+				       ipv4.sysctl_tcp_child_ehash_entries);
+	struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo;
+	int tcp_ehash_entries;
+	struct ctl_table tbl;
+
+	tcp_ehash_entries = hinfo->ehash_mask + 1;
+
+	/* A negative number indicates that the child netns
+	 * shares the global ehash.
+	 */
+	if (!net_eq(net, &init_net) && !hinfo->pernet)
+		tcp_ehash_entries *= -1;
+
+	tbl.data = &tcp_ehash_entries;
+	tbl.maxlen = sizeof(int);
+
+	return proc_dointvec(&tbl, write, buffer, lenp, ppos);
+}
+
 #ifdef CONFIG_IP_ROUTE_MULTIPATH
 static int proc_fib_multipath_hash_policy(struct ctl_table *table, int write,
 					  void *buffer, size_t *lenp,
@@ -530,10 +554,9 @@ static struct ctl_table ipv4_table[] = {
 };
 
 static struct ctl_table ipv4_net_table[] = {
-	/* tcp_max_tw_buckets must be first in this table. */
 	{
 		.procname	= "tcp_max_tw_buckets",
-/*		.data		= &init_net.ipv4.tcp_death_row.sysctl_max_tw_buckets, */
+		.data		= &init_net.ipv4.tcp_death_row.sysctl_max_tw_buckets,
 		.maxlen		= sizeof(int),
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec
@@ -1322,6 +1345,21 @@ static struct ctl_table ipv4_net_table[] = {
 		.extra2         = SYSCTL_ONE,
 	},
 	{
+		.procname	= "tcp_ehash_entries",
+		.data		= &init_net.ipv4.sysctl_tcp_child_ehash_entries,
+		.mode		= 0444,
+		.proc_handler	= proc_tcp_ehash_entries,
+	},
+	{
+		.procname	= "tcp_child_ehash_entries",
+		.data		= &init_net.ipv4.sysctl_tcp_child_ehash_entries,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= proc_douintvec_minmax,
+		.extra1		= SYSCTL_ZERO,
+		.extra2		= &tcp_child_ehash_entries_max,
+	},
+	{
 		.procname	= "udp_rmem_min",
 		.data		= &init_net.ipv4.sysctl_udp_rmem_min,
 		.maxlen		= sizeof(init_net.ipv4.sysctl_udp_rmem_min),
@@ -1361,8 +1399,7 @@ static __net_init int ipv4_sysctl_init_net(struct net *net)
 		if (!table)
 			goto err_alloc;
 
-		/* skip first entry (sysctl_max_tw_buckets) */
-		for (i = 1; i < ARRAY_SIZE(ipv4_net_table) - 1; i++) {
+		for (i = 0; i < ARRAY_SIZE(ipv4_net_table) - 1; i++) {
 			if (table[i].data) {
 				/* Update the variables to point into
 				 * the current struct net
@@ -1377,8 +1414,6 @@ static __net_init int ipv4_sysctl_init_net(struct net *net)
 		}
 	}
 
-	table[0].data = &net->ipv4.tcp_death_row->sysctl_max_tw_buckets;
-
 	net->ipv4.ipv4_hdr = register_net_sysctl(net, "net/ipv4", table);
 	if (!net->ipv4.ipv4_hdr)
 		goto err_reg;
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index e373dde1f46f..0c51abeee172 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -1162,9 +1162,8 @@ void tcp_free_fastopen_req(struct tcp_sock *tp)
 	}
 }
 
-static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
-				int *copied, size_t size,
-				struct ubuf_info *uarg)
+int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
+			 size_t size, struct ubuf_info *uarg)
 {
 	struct tcp_sock *tp = tcp_sk(sk);
 	struct inet_sock *inet = inet_sk(sk);
@@ -1239,7 +1238,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
 			}
 			zc = sk->sk_route_caps & NETIF_F_SG;
 			if (!zc)
-				uarg->zerocopy = 0;
+				uarg_to_msgzc(uarg)->zerocopy = 0;
 		}
 	}
 
@@ -3137,6 +3136,8 @@ int tcp_disconnect(struct sock *sk, int flags)
 	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
 	tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
 	tp->snd_cwnd_cnt = 0;
+	tp->is_cwnd_limited = 0;
+	tp->max_packets_out = 0;
 	tp->window_clamp = 0;
 	tp->delivered = 0;
 	tp->delivered_ce = 0;
@@ -3208,7 +3209,7 @@ EXPORT_SYMBOL(tcp_disconnect);
 
 static inline bool tcp_can_repair_sock(const struct sock *sk)
 {
-	return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
+	return sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
 		(sk->sk_state != TCP_LISTEN);
 }
 
@@ -3485,8 +3486,8 @@ int tcp_set_window_clamp(struct sock *sk, int val)
 /*
  *	Socket option code for TCP.
  */
-static int do_tcp_setsockopt(struct sock *sk, int level, int optname,
-		sockptr_t optval, unsigned int optlen)
+int do_tcp_setsockopt(struct sock *sk, int level, int optname,
+		      sockptr_t optval, unsigned int optlen)
 {
 	struct tcp_sock *tp = tcp_sk(sk);
 	struct inet_connection_sock *icsk = inet_csk(sk);
@@ -3508,11 +3509,11 @@ static int do_tcp_setsockopt(struct sock *sk, int level, int optname,
 			return -EFAULT;
 		name[val] = 0;
 
-		lock_sock(sk);
-		err = tcp_set_congestion_control(sk, name, true,
-						 ns_capable(sock_net(sk)->user_ns,
-							    CAP_NET_ADMIN));
-		release_sock(sk);
+		sockopt_lock_sock(sk);
+		err = tcp_set_congestion_control(sk, name, !has_current_bpf_ctx(),
+						 sockopt_ns_capable(sock_net(sk)->user_ns,
+								    CAP_NET_ADMIN));
+		sockopt_release_sock(sk);
 		return err;
 	}
 	case TCP_ULP: {
@@ -3528,9 +3529,9 @@ static int do_tcp_setsockopt(struct sock *sk, int level, int optname,
 			return -EFAULT;
 		name[val] = 0;
 
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
 		err = tcp_set_ulp(sk, name);
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		return err;
 	}
 	case TCP_FASTOPEN_KEY: {
@@ -3563,7 +3564,7 @@ static int do_tcp_setsockopt(struct sock *sk, int level, int optname,
 	if (copy_from_sockptr(&val, optval, sizeof(val)))
 		return -EFAULT;
 
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 
 	switch (optname) {
 	case TCP_MAXSEG:
@@ -3785,7 +3786,7 @@ static int do_tcp_setsockopt(struct sock *sk, int level, int optname,
 		break;
 	}
 
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	return err;
 }
 
@@ -4049,15 +4050,15 @@ struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk,
 	return stats;
 }
 
-static int do_tcp_getsockopt(struct sock *sk, int level,
-		int optname, char __user *optval, int __user *optlen)
+int do_tcp_getsockopt(struct sock *sk, int level,
+		      int optname, sockptr_t optval, sockptr_t optlen)
 {
 	struct inet_connection_sock *icsk = inet_csk(sk);
 	struct tcp_sock *tp = tcp_sk(sk);
 	struct net *net = sock_net(sk);
 	int val, len;
 
-	if (get_user(len, optlen))
+	if (copy_from_sockptr(&len, optlen, sizeof(int)))
 		return -EFAULT;
 
 	len = min_t(unsigned int, len, sizeof(int));
@@ -4107,15 +4108,15 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
 	case TCP_INFO: {
 		struct tcp_info info;
 
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 
 		tcp_get_info(sk, &info);
 
 		len = min_t(unsigned int, len, sizeof(info));
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &info, len))
+		if (copy_to_sockptr(optval, &info, len))
 			return -EFAULT;
 		return 0;
 	}
@@ -4125,7 +4126,7 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
 		size_t sz = 0;
 		int attr;
 
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 
 		ca_ops = icsk->icsk_ca_ops;
@@ -4133,9 +4134,9 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
 			sz = ca_ops->get_info(sk, ~0U, &attr, &info);
 
 		len = min_t(unsigned int, len, sz);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &info, len))
+		if (copy_to_sockptr(optval, &info, len))
 			return -EFAULT;
 		return 0;
 	}
@@ -4144,27 +4145,28 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
 		break;
 
 	case TCP_CONGESTION:
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 		len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
+		if (copy_to_sockptr(optval, icsk->icsk_ca_ops->name, len))
 			return -EFAULT;
 		return 0;
 
 	case TCP_ULP:
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 		len = min_t(unsigned int, len, TCP_ULP_NAME_MAX);
 		if (!icsk->icsk_ulp_ops) {
-			if (put_user(0, optlen))
+			len = 0;
+			if (copy_to_sockptr(optlen, &len, sizeof(int)))
 				return -EFAULT;
 			return 0;
 		}
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, icsk->icsk_ulp_ops->name, len))
+		if (copy_to_sockptr(optval, icsk->icsk_ulp_ops->name, len))
 			return -EFAULT;
 		return 0;
 
@@ -4172,15 +4174,15 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
 		u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)];
 		unsigned int key_len;
 
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 
 		key_len = tcp_fastopen_get_cipher(net, icsk, key) *
 				TCP_FASTOPEN_KEY_LENGTH;
 		len = min_t(unsigned int, len, key_len);
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, key, len))
+		if (copy_to_sockptr(optval, key, len))
 			return -EFAULT;
 		return 0;
 	}
@@ -4206,7 +4208,7 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
 	case TCP_REPAIR_WINDOW: {
 		struct tcp_repair_window opt;
 
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 
 		if (len != sizeof(opt))
@@ -4221,7 +4223,7 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
 		opt.rcv_wnd	= tp->rcv_wnd;
 		opt.rcv_wup	= tp->rcv_wup;
 
-		if (copy_to_user(optval, &opt, len))
+		if (copy_to_sockptr(optval, &opt, len))
 			return -EFAULT;
 		return 0;
 	}
@@ -4267,35 +4269,35 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
 		val = tp->save_syn;
 		break;
 	case TCP_SAVED_SYN: {
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
 		if (tp->saved_syn) {
 			if (len < tcp_saved_syn_len(tp->saved_syn)) {
-				if (put_user(tcp_saved_syn_len(tp->saved_syn),
-					     optlen)) {
-					release_sock(sk);
+				len = tcp_saved_syn_len(tp->saved_syn);
+				if (copy_to_sockptr(optlen, &len, sizeof(int))) {
+					sockopt_release_sock(sk);
 					return -EFAULT;
 				}
-				release_sock(sk);
+				sockopt_release_sock(sk);
 				return -EINVAL;
 			}
 			len = tcp_saved_syn_len(tp->saved_syn);
-			if (put_user(len, optlen)) {
-				release_sock(sk);
+			if (copy_to_sockptr(optlen, &len, sizeof(int))) {
+				sockopt_release_sock(sk);
 				return -EFAULT;
 			}
-			if (copy_to_user(optval, tp->saved_syn->data, len)) {
-				release_sock(sk);
+			if (copy_to_sockptr(optval, tp->saved_syn->data, len)) {
+				sockopt_release_sock(sk);
 				return -EFAULT;
 			}
 			tcp_saved_syn_free(tp);
-			release_sock(sk);
+			sockopt_release_sock(sk);
 		} else {
-			release_sock(sk);
+			sockopt_release_sock(sk);
 			len = 0;
-			if (put_user(len, optlen))
+			if (copy_to_sockptr(optlen, &len, sizeof(int)))
 				return -EFAULT;
 		}
 		return 0;
@@ -4306,31 +4308,31 @@ static int do_tcp_getsockopt(struct sock *sk, int level,
 		struct tcp_zerocopy_receive zc = {};
 		int err;
 
-		if (get_user(len, optlen))
+		if (copy_from_sockptr(&len, optlen, sizeof(int)))
 			return -EFAULT;
 		if (len < 0 ||
 		    len < offsetofend(struct tcp_zerocopy_receive, length))
 			return -EINVAL;
 		if (unlikely(len > sizeof(zc))) {
-			err = check_zeroed_user(optval + sizeof(zc),
-						len - sizeof(zc));
+			err = check_zeroed_sockptr(optval, sizeof(zc),
+						   len - sizeof(zc));
 			if (err < 1)
 				return err == 0 ? -EINVAL : err;
 			len = sizeof(zc);
-			if (put_user(len, optlen))
+			if (copy_to_sockptr(optlen, &len, sizeof(int)))
 				return -EFAULT;
 		}
-		if (copy_from_user(&zc, optval, len))
+		if (copy_from_sockptr(&zc, optval, len))
 			return -EFAULT;
 		if (zc.reserved)
 			return -EINVAL;
 		if (zc.msg_flags &  ~(TCP_VALID_ZC_MSG_FLAGS))
 			return -EINVAL;
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
 		err = tcp_zerocopy_receive(sk, &zc, &tss);
 		err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname,
 							  &zc, &len, err);
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags))
 			goto zerocopy_rcv_cmsg;
 		switch (len) {
@@ -4360,7 +4362,7 @@ zerocopy_rcv_sk_err:
 zerocopy_rcv_inq:
 		zc.inq = tcp_inq_hint(sk);
 zerocopy_rcv_out:
-		if (!err && copy_to_user(optval, &zc, len))
+		if (!err && copy_to_sockptr(optval, &zc, len))
 			err = -EFAULT;
 		return err;
 	}
@@ -4369,9 +4371,9 @@ zerocopy_rcv_out:
 		return -ENOPROTOOPT;
 	}
 
-	if (put_user(len, optlen))
+	if (copy_to_sockptr(optlen, &len, sizeof(int)))
 		return -EFAULT;
-	if (copy_to_user(optval, &val, len))
+	if (copy_to_sockptr(optval, &val, len))
 		return -EFAULT;
 	return 0;
 }
@@ -4396,7 +4398,8 @@ int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
 	if (level != SOL_TCP)
 		return icsk->icsk_af_ops->getsockopt(sk, level, optname,
 						     optval, optlen);
-	return do_tcp_getsockopt(sk, level, optname, optval, optlen);
+	return do_tcp_getsockopt(sk, level, optname, USER_SOCKPTR(optval),
+				 USER_SOCKPTR(optlen));
 }
 EXPORT_SYMBOL(tcp_getsockopt);
 
@@ -4442,12 +4445,16 @@ static void __tcp_alloc_md5sig_pool(void)
 	 * to memory. See smp_rmb() in tcp_get_md5sig_pool()
 	 */
 	smp_wmb();
-	tcp_md5sig_pool_populated = true;
+	/* Paired with READ_ONCE() from tcp_alloc_md5sig_pool()
+	 * and tcp_get_md5sig_pool().
+	*/
+	WRITE_ONCE(tcp_md5sig_pool_populated, true);
 }
 
 bool tcp_alloc_md5sig_pool(void)
 {
-	if (unlikely(!tcp_md5sig_pool_populated)) {
+	/* Paired with WRITE_ONCE() from __tcp_alloc_md5sig_pool() */
+	if (unlikely(!READ_ONCE(tcp_md5sig_pool_populated))) {
 		mutex_lock(&tcp_md5sig_mutex);
 
 		if (!tcp_md5sig_pool_populated) {
@@ -4458,7 +4465,8 @@ bool tcp_alloc_md5sig_pool(void)
 
 		mutex_unlock(&tcp_md5sig_mutex);
 	}
-	return tcp_md5sig_pool_populated;
+	/* Paired with WRITE_ONCE() from __tcp_alloc_md5sig_pool() */
+	return READ_ONCE(tcp_md5sig_pool_populated);
 }
 EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
 
@@ -4474,7 +4482,8 @@ struct tcp_md5sig_pool *tcp_get_md5sig_pool(void)
 {
 	local_bh_disable();
 
-	if (tcp_md5sig_pool_populated) {
+	/* Paired with WRITE_ONCE() from __tcp_alloc_md5sig_pool() */
+	if (READ_ONCE(tcp_md5sig_pool_populated)) {
 		/* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */
 		smp_rmb();
 		return this_cpu_ptr(&tcp_md5sig_pool);
@@ -4745,6 +4754,12 @@ void __init tcp_init(void)
 				  SLAB_HWCACHE_ALIGN | SLAB_PANIC |
 				  SLAB_ACCOUNT,
 				  NULL);
+	tcp_hashinfo.bind2_bucket_cachep =
+		kmem_cache_create("tcp_bind2_bucket",
+				  sizeof(struct inet_bind2_bucket), 0,
+				  SLAB_HWCACHE_ALIGN | SLAB_PANIC |
+				  SLAB_ACCOUNT,
+				  NULL);
 
 	/* Size and allocate the main established and bind bucket
 	 * hash tables.
@@ -4768,7 +4783,7 @@ void __init tcp_init(void)
 		panic("TCP: failed to alloc ehash_locks");
 	tcp_hashinfo.bhash =
 		alloc_large_system_hash("TCP bind",
-					sizeof(struct inet_bind_hashbucket),
+					2 * sizeof(struct inet_bind_hashbucket),
 					tcp_hashinfo.ehash_mask + 1,
 					17, /* one slot per 128 KB of memory */
 					0,
@@ -4777,11 +4792,15 @@ void __init tcp_init(void)
 					0,
 					64 * 1024);
 	tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
+	tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size;
 	for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
 		spin_lock_init(&tcp_hashinfo.bhash[i].lock);
 		INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
+		spin_lock_init(&tcp_hashinfo.bhash2[i].lock);
+		INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain);
 	}
 
+	tcp_hashinfo.pernet = false;
 
 	cnt = tcp_hashinfo.ehash_mask + 1;
 	sysctl_tcp_max_orphans = cnt / 2;
diff --git a/net/ipv4/tcp_diag.c b/net/ipv4/tcp_diag.c
index 75a1c985f49a..01b50fa79189 100644
--- a/net/ipv4/tcp_diag.c
+++ b/net/ipv4/tcp_diag.c
@@ -181,13 +181,21 @@ static size_t tcp_diag_get_aux_size(struct sock *sk, bool net_admin)
 static void tcp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb,
 			  const struct inet_diag_req_v2 *r)
 {
-	inet_diag_dump_icsk(&tcp_hashinfo, skb, cb, r);
+	struct inet_hashinfo *hinfo;
+
+	hinfo = sock_net(cb->skb->sk)->ipv4.tcp_death_row.hashinfo;
+
+	inet_diag_dump_icsk(hinfo, skb, cb, r);
 }
 
 static int tcp_diag_dump_one(struct netlink_callback *cb,
 			     const struct inet_diag_req_v2 *req)
 {
-	return inet_diag_dump_one_icsk(&tcp_hashinfo, cb, req);
+	struct inet_hashinfo *hinfo;
+
+	hinfo = sock_net(cb->skb->sk)->ipv4.tcp_death_row.hashinfo;
+
+	return inet_diag_dump_one_icsk(hinfo, cb, req);
 }
 
 #ifdef CONFIG_INET_DIAG_DESTROY
@@ -195,9 +203,13 @@ static int tcp_diag_destroy(struct sk_buff *in_skb,
 			    const struct inet_diag_req_v2 *req)
 {
 	struct net *net = sock_net(in_skb->sk);
-	struct sock *sk = inet_diag_find_one_icsk(net, &tcp_hashinfo, req);
+	struct inet_hashinfo *hinfo;
+	struct sock *sk;
 	int err;
 
+	hinfo = net->ipv4.tcp_death_row.hashinfo;
+	sk = inet_diag_find_one_icsk(net, hinfo, req);
+
 	if (IS_ERR(sk))
 		return PTR_ERR(sk);
 
diff --git a/net/ipv4/tcp_fastopen.c b/net/ipv4/tcp_fastopen.c
index 825b216d11f5..45cc7f1ca296 100644
--- a/net/ipv4/tcp_fastopen.c
+++ b/net/ipv4/tcp_fastopen.c
@@ -272,8 +272,9 @@ static struct sock *tcp_fastopen_create_child(struct sock *sk,
 	 * The request socket is not added to the ehash
 	 * because it's been added to the accept queue directly.
 	 */
+	req->timeout = tcp_timeout_init(child);
 	inet_csk_reset_xmit_timer(child, ICSK_TIME_RETRANS,
-				  TCP_TIMEOUT_INIT, TCP_RTO_MAX);
+				  req->timeout, TCP_RTO_MAX);
 
 	refcount_set(&req->rsk_refcnt, 2);
 
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 5b019ba2b9d2..6376ad915765 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -199,16 +199,18 @@ static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr,
 /* This will initiate an outgoing connection. */
 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
 {
+	struct inet_bind_hashbucket *prev_addr_hashbucket = NULL;
 	struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
+	struct inet_timewait_death_row *tcp_death_row;
+	__be32 daddr, nexthop, prev_sk_rcv_saddr;
 	struct inet_sock *inet = inet_sk(sk);
 	struct tcp_sock *tp = tcp_sk(sk);
+	struct ip_options_rcu *inet_opt;
+	struct net *net = sock_net(sk);
 	__be16 orig_sport, orig_dport;
-	__be32 daddr, nexthop;
 	struct flowi4 *fl4;
 	struct rtable *rt;
 	int err;
-	struct ip_options_rcu *inet_opt;
-	struct inet_timewait_death_row *tcp_death_row = sock_net(sk)->ipv4.tcp_death_row;
 
 	if (addr_len < sizeof(struct sockaddr_in))
 		return -EINVAL;
@@ -234,7 +236,7 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
 	if (IS_ERR(rt)) {
 		err = PTR_ERR(rt);
 		if (err == -ENETUNREACH)
-			IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES);
+			IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
 		return err;
 	}
 
@@ -246,10 +248,29 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
 	if (!inet_opt || !inet_opt->opt.srr)
 		daddr = fl4->daddr;
 
-	if (!inet->inet_saddr)
+	tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
+
+	if (!inet->inet_saddr) {
+		if (inet_csk(sk)->icsk_bind2_hash) {
+			prev_addr_hashbucket = inet_bhashfn_portaddr(tcp_death_row->hashinfo,
+								     sk, net, inet->inet_num);
+			prev_sk_rcv_saddr = sk->sk_rcv_saddr;
+		}
 		inet->inet_saddr = fl4->saddr;
+	}
+
 	sk_rcv_saddr_set(sk, inet->inet_saddr);
 
+	if (prev_addr_hashbucket) {
+		err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk);
+		if (err) {
+			inet->inet_saddr = 0;
+			sk_rcv_saddr_set(sk, prev_sk_rcv_saddr);
+			ip_rt_put(rt);
+			return err;
+		}
+	}
+
 	if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
 		/* Reset inherited state */
 		tp->rx_opt.ts_recent	   = 0;
@@ -298,8 +319,7 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
 						  inet->inet_daddr,
 						  inet->inet_sport,
 						  usin->sin_port));
-		tp->tsoffset = secure_tcp_ts_off(sock_net(sk),
-						 inet->inet_saddr,
+		tp->tsoffset = secure_tcp_ts_off(net, inet->inet_saddr,
 						 inet->inet_daddr);
 	}
 
@@ -475,9 +495,9 @@ int tcp_v4_err(struct sk_buff *skb, u32 info)
 	int err;
 	struct net *net = dev_net(skb->dev);
 
-	sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr,
-				       th->dest, iph->saddr, ntohs(th->source),
-				       inet_iif(skb), 0);
+	sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
+				       iph->daddr, th->dest, iph->saddr,
+				       ntohs(th->source), inet_iif(skb), 0);
 	if (!sk) {
 		__ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
 		return -ENOENT;
@@ -740,8 +760,8 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
 		 * Incoming packet is checked with md5 hash with finding key,
 		 * no RST generated if md5 hash doesn't match.
 		 */
-		sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0,
-					     ip_hdr(skb)->saddr,
+		sk1 = __inet_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
+					     NULL, 0, ip_hdr(skb)->saddr,
 					     th->source, ip_hdr(skb)->daddr,
 					     ntohs(th->source), dif, sdif);
 		/* don't send rst if it can't find key */
@@ -1709,6 +1729,7 @@ EXPORT_SYMBOL(tcp_v4_do_rcv);
 
 int tcp_v4_early_demux(struct sk_buff *skb)
 {
+	struct net *net = dev_net(skb->dev);
 	const struct iphdr *iph;
 	const struct tcphdr *th;
 	struct sock *sk;
@@ -1725,7 +1746,7 @@ int tcp_v4_early_demux(struct sk_buff *skb)
 	if (th->doff < sizeof(struct tcphdr) / 4)
 		return 0;
 
-	sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
+	sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 				       iph->saddr, th->source,
 				       iph->daddr, ntohs(th->dest),
 				       skb->skb_iif, inet_sdif(skb));
@@ -1951,7 +1972,8 @@ int tcp_v4_rcv(struct sk_buff *skb)
 	th = (const struct tcphdr *)skb->data;
 	iph = ip_hdr(skb);
 lookup:
-	sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source,
+	sk = __inet_lookup_skb(net->ipv4.tcp_death_row.hashinfo,
+			       skb, __tcp_hdrlen(th), th->source,
 			       th->dest, sdif, &refcounted);
 	if (!sk)
 		goto no_tcp_socket;
@@ -2133,9 +2155,9 @@ do_time_wait:
 	}
 	switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
 	case TCP_TW_SYN: {
-		struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev),
-							&tcp_hashinfo, skb,
-							__tcp_hdrlen(th),
+		struct sock *sk2 = inet_lookup_listener(net,
+							net->ipv4.tcp_death_row.hashinfo,
+							skb, __tcp_hdrlen(th),
 							iph->saddr, th->source,
 							iph->daddr, th->dest,
 							inet_iif(skb),
@@ -2285,15 +2307,16 @@ static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
  */
 static void *listening_get_first(struct seq_file *seq)
 {
+	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
 	struct tcp_iter_state *st = seq->private;
 
 	st->offset = 0;
-	for (; st->bucket <= tcp_hashinfo.lhash2_mask; st->bucket++) {
+	for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) {
 		struct inet_listen_hashbucket *ilb2;
 		struct hlist_nulls_node *node;
 		struct sock *sk;
 
-		ilb2 = &tcp_hashinfo.lhash2[st->bucket];
+		ilb2 = &hinfo->lhash2[st->bucket];
 		if (hlist_nulls_empty(&ilb2->nulls_head))
 			continue;
 
@@ -2318,6 +2341,7 @@ static void *listening_get_next(struct seq_file *seq, void *cur)
 	struct tcp_iter_state *st = seq->private;
 	struct inet_listen_hashbucket *ilb2;
 	struct hlist_nulls_node *node;
+	struct inet_hashinfo *hinfo;
 	struct sock *sk = cur;
 
 	++st->num;
@@ -2329,7 +2353,8 @@ static void *listening_get_next(struct seq_file *seq, void *cur)
 			return sk;
 	}
 
-	ilb2 = &tcp_hashinfo.lhash2[st->bucket];
+	hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
+	ilb2 = &hinfo->lhash2[st->bucket];
 	spin_unlock(&ilb2->lock);
 	++st->bucket;
 	return listening_get_first(seq);
@@ -2351,9 +2376,10 @@ static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
 	return rc;
 }
 
-static inline bool empty_bucket(const struct tcp_iter_state *st)
+static inline bool empty_bucket(struct inet_hashinfo *hinfo,
+				const struct tcp_iter_state *st)
 {
-	return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain);
+	return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain);
 }
 
 /*
@@ -2362,20 +2388,21 @@ static inline bool empty_bucket(const struct tcp_iter_state *st)
  */
 static void *established_get_first(struct seq_file *seq)
 {
+	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
 	struct tcp_iter_state *st = seq->private;
 
 	st->offset = 0;
-	for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) {
+	for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) {
 		struct sock *sk;
 		struct hlist_nulls_node *node;
-		spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket);
+		spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket);
 
 		/* Lockless fast path for the common case of empty buckets */
-		if (empty_bucket(st))
+		if (empty_bucket(hinfo, st))
 			continue;
 
 		spin_lock_bh(lock);
-		sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) {
+		sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) {
 			if (seq_sk_match(seq, sk))
 				return sk;
 		}
@@ -2387,9 +2414,10 @@ static void *established_get_first(struct seq_file *seq)
 
 static void *established_get_next(struct seq_file *seq, void *cur)
 {
-	struct sock *sk = cur;
-	struct hlist_nulls_node *node;
+	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
 	struct tcp_iter_state *st = seq->private;
+	struct hlist_nulls_node *node;
+	struct sock *sk = cur;
 
 	++st->num;
 	++st->offset;
@@ -2401,7 +2429,7 @@ static void *established_get_next(struct seq_file *seq, void *cur)
 			return sk;
 	}
 
-	spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
+	spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
 	++st->bucket;
 	return established_get_first(seq);
 }
@@ -2439,6 +2467,7 @@ static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
 
 static void *tcp_seek_last_pos(struct seq_file *seq)
 {
+	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
 	struct tcp_iter_state *st = seq->private;
 	int bucket = st->bucket;
 	int offset = st->offset;
@@ -2447,7 +2476,7 @@ static void *tcp_seek_last_pos(struct seq_file *seq)
 
 	switch (st->state) {
 	case TCP_SEQ_STATE_LISTENING:
-		if (st->bucket > tcp_hashinfo.lhash2_mask)
+		if (st->bucket > hinfo->lhash2_mask)
 			break;
 		st->state = TCP_SEQ_STATE_LISTENING;
 		rc = listening_get_first(seq);
@@ -2459,7 +2488,7 @@ static void *tcp_seek_last_pos(struct seq_file *seq)
 		st->state = TCP_SEQ_STATE_ESTABLISHED;
 		fallthrough;
 	case TCP_SEQ_STATE_ESTABLISHED:
-		if (st->bucket > tcp_hashinfo.ehash_mask)
+		if (st->bucket > hinfo->ehash_mask)
 			break;
 		rc = established_get_first(seq);
 		while (offset-- && rc && bucket == st->bucket)
@@ -2527,16 +2556,17 @@ EXPORT_SYMBOL(tcp_seq_next);
 
 void tcp_seq_stop(struct seq_file *seq, void *v)
 {
+	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
 	struct tcp_iter_state *st = seq->private;
 
 	switch (st->state) {
 	case TCP_SEQ_STATE_LISTENING:
 		if (v != SEQ_START_TOKEN)
-			spin_unlock(&tcp_hashinfo.lhash2[st->bucket].lock);
+			spin_unlock(&hinfo->lhash2[st->bucket].lock);
 		break;
 	case TCP_SEQ_STATE_ESTABLISHED:
 		if (v)
-			spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
+			spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
 		break;
 	}
 }
@@ -2731,6 +2761,7 @@ static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
 static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
 						 struct sock *start_sk)
 {
+	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
 	struct bpf_tcp_iter_state *iter = seq->private;
 	struct tcp_iter_state *st = &iter->state;
 	struct hlist_nulls_node *node;
@@ -2750,7 +2781,7 @@ static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
 			expected++;
 		}
 	}
-	spin_unlock(&tcp_hashinfo.lhash2[st->bucket].lock);
+	spin_unlock(&hinfo->lhash2[st->bucket].lock);
 
 	return expected;
 }
@@ -2758,6 +2789,7 @@ static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
 static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
 						   struct sock *start_sk)
 {
+	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
 	struct bpf_tcp_iter_state *iter = seq->private;
 	struct tcp_iter_state *st = &iter->state;
 	struct hlist_nulls_node *node;
@@ -2777,13 +2809,14 @@ static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
 			expected++;
 		}
 	}
-	spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket));
+	spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
 
 	return expected;
 }
 
 static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
 {
+	struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
 	struct bpf_tcp_iter_state *iter = seq->private;
 	struct tcp_iter_state *st = &iter->state;
 	unsigned int expected;
@@ -2799,7 +2832,7 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
 		st->offset = 0;
 		st->bucket++;
 		if (st->state == TCP_SEQ_STATE_LISTENING &&
-		    st->bucket > tcp_hashinfo.lhash2_mask) {
+		    st->bucket > hinfo->lhash2_mask) {
 			st->state = TCP_SEQ_STATE_ESTABLISHED;
 			st->bucket = 0;
 		}
@@ -3064,7 +3097,7 @@ struct proto tcp_prot = {
 	.slab_flags		= SLAB_TYPESAFE_BY_RCU,
 	.twsk_prot		= &tcp_timewait_sock_ops,
 	.rsk_prot		= &tcp_request_sock_ops,
-	.h.hashinfo		= &tcp_hashinfo,
+	.h.hashinfo		= NULL,
 	.no_autobind		= true,
 	.diag_destroy		= tcp_abort,
 };
@@ -3072,19 +3105,43 @@ EXPORT_SYMBOL(tcp_prot);
 
 static void __net_exit tcp_sk_exit(struct net *net)
 {
-	struct inet_timewait_death_row *tcp_death_row = net->ipv4.tcp_death_row;
-
 	if (net->ipv4.tcp_congestion_control)
 		bpf_module_put(net->ipv4.tcp_congestion_control,
 			       net->ipv4.tcp_congestion_control->owner);
-	if (refcount_dec_and_test(&tcp_death_row->tw_refcount))
-		kfree(tcp_death_row);
 }
 
-static int __net_init tcp_sk_init(struct net *net)
+static void __net_init tcp_set_hashinfo(struct net *net)
 {
-	int cnt;
+	struct inet_hashinfo *hinfo;
+	unsigned int ehash_entries;
+	struct net *old_net;
+
+	if (net_eq(net, &init_net))
+		goto fallback;
+
+	old_net = current->nsproxy->net_ns;
+	ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries);
+	if (!ehash_entries)
+		goto fallback;
+
+	ehash_entries = roundup_pow_of_two(ehash_entries);
+	hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries);
+	if (!hinfo) {
+		pr_warn("Failed to allocate TCP ehash (entries: %u) "
+			"for a netns, fallback to the global one\n",
+			ehash_entries);
+fallback:
+		hinfo = &tcp_hashinfo;
+		ehash_entries = tcp_hashinfo.ehash_mask + 1;
+	}
 
+	net->ipv4.tcp_death_row.hashinfo = hinfo;
+	net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2;
+	net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128);
+}
+
+static int __net_init tcp_sk_init(struct net *net)
+{
 	net->ipv4.sysctl_tcp_ecn = 2;
 	net->ipv4.sysctl_tcp_ecn_fallback = 1;
 
@@ -3110,15 +3167,9 @@ static int __net_init tcp_sk_init(struct net *net)
 	net->ipv4.sysctl_tcp_tw_reuse = 2;
 	net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
 
-	net->ipv4.tcp_death_row = kzalloc(sizeof(struct inet_timewait_death_row), GFP_KERNEL);
-	if (!net->ipv4.tcp_death_row)
-		return -ENOMEM;
-	refcount_set(&net->ipv4.tcp_death_row->tw_refcount, 1);
-	cnt = tcp_hashinfo.ehash_mask + 1;
-	net->ipv4.tcp_death_row->sysctl_max_tw_buckets = cnt / 2;
-	net->ipv4.tcp_death_row->hashinfo = &tcp_hashinfo;
+	refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1);
+	tcp_set_hashinfo(net);
 
-	net->ipv4.sysctl_max_syn_backlog = max(128, cnt / 128);
 	net->ipv4.sysctl_tcp_sack = 1;
 	net->ipv4.sysctl_tcp_window_scaling = 1;
 	net->ipv4.sysctl_tcp_timestamps = 1;
@@ -3180,10 +3231,13 @@ static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
 {
 	struct net *net;
 
-	inet_twsk_purge(&tcp_hashinfo, AF_INET);
+	tcp_twsk_purge(net_exit_list, AF_INET);
 
-	list_for_each_entry(net, net_exit_list, exit_list)
+	list_for_each_entry(net, net_exit_list, exit_list) {
+		inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo);
+		WARN_ON_ONCE(!refcount_dec_and_test(&net->ipv4.tcp_death_row.tw_refcount));
 		tcp_fastopen_ctx_destroy(net);
+	}
 }
 
 static struct pernet_operations __net_initdata tcp_sk_ops = {
diff --git a/net/ipv4/tcp_metrics.c b/net/ipv4/tcp_metrics.c
index d58e672be31c..82f4575f9cd9 100644
--- a/net/ipv4/tcp_metrics.c
+++ b/net/ipv4/tcp_metrics.c
@@ -969,6 +969,7 @@ static struct genl_family tcp_metrics_nl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= tcp_metrics_nl_ops,
 	.n_small_ops	= ARRAY_SIZE(tcp_metrics_nl_ops),
+	.resv_start_op	= TCP_METRICS_CMD_DEL + 1,
 };
 
 static unsigned int tcpmhash_entries;
diff --git a/net/ipv4/tcp_minisocks.c b/net/ipv4/tcp_minisocks.c
index cb95d88497ae..79f30f026d89 100644
--- a/net/ipv4/tcp_minisocks.c
+++ b/net/ipv4/tcp_minisocks.c
@@ -247,10 +247,10 @@ void tcp_time_wait(struct sock *sk, int state, int timeo)
 {
 	const struct inet_connection_sock *icsk = inet_csk(sk);
 	const struct tcp_sock *tp = tcp_sk(sk);
+	struct net *net = sock_net(sk);
 	struct inet_timewait_sock *tw;
-	struct inet_timewait_death_row *tcp_death_row = sock_net(sk)->ipv4.tcp_death_row;
 
-	tw = inet_twsk_alloc(sk, tcp_death_row, state);
+	tw = inet_twsk_alloc(sk, &net->ipv4.tcp_death_row, state);
 
 	if (tw) {
 		struct tcp_timewait_sock *tcptw = tcp_twsk((struct sock *)tw);
@@ -319,14 +319,14 @@ void tcp_time_wait(struct sock *sk, int state, int timeo)
 		/* Linkage updates.
 		 * Note that access to tw after this point is illegal.
 		 */
-		inet_twsk_hashdance(tw, sk, &tcp_hashinfo);
+		inet_twsk_hashdance(tw, sk, net->ipv4.tcp_death_row.hashinfo);
 		local_bh_enable();
 	} else {
 		/* Sorry, if we're out of memory, just CLOSE this
 		 * socket up.  We've got bigger problems than
 		 * non-graceful socket closings.
 		 */
-		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPTIMEWAITOVERFLOW);
+		NET_INC_STATS(net, LINUX_MIB_TCPTIMEWAITOVERFLOW);
 	}
 
 	tcp_update_metrics(sk);
@@ -347,6 +347,26 @@ void tcp_twsk_destructor(struct sock *sk)
 }
 EXPORT_SYMBOL_GPL(tcp_twsk_destructor);
 
+void tcp_twsk_purge(struct list_head *net_exit_list, int family)
+{
+	bool purged_once = false;
+	struct net *net;
+
+	list_for_each_entry(net, net_exit_list, exit_list) {
+		/* The last refcount is decremented in tcp_sk_exit_batch() */
+		if (refcount_read(&net->ipv4.tcp_death_row.tw_refcount) == 1)
+			continue;
+
+		if (net->ipv4.tcp_death_row.hashinfo->pernet) {
+			inet_twsk_purge(net->ipv4.tcp_death_row.hashinfo, family);
+		} else if (!purged_once) {
+			inet_twsk_purge(&tcp_hashinfo, family);
+			purged_once = true;
+		}
+	}
+}
+EXPORT_SYMBOL_GPL(tcp_twsk_purge);
+
 /* Warning : This function is called without sk_listener being locked.
  * Be sure to read socket fields once, as their value could change under us.
  */
@@ -541,6 +561,7 @@ struct sock *tcp_create_openreq_child(const struct sock *sk,
 	newtp->fastopen_req = NULL;
 	RCU_INIT_POINTER(newtp->fastopen_rsk, NULL);
 
+	newtp->bpf_chg_cc_inprogress = 0;
 	tcp_bpf_clone(sk, newsk);
 
 	__TCP_INC_STATS(sock_net(sk), TCP_MIB_PASSIVEOPENS);
diff --git a/net/ipv4/tcp_offload.c b/net/ipv4/tcp_offload.c
index 30abde86db45..45dda7889387 100644
--- a/net/ipv4/tcp_offload.c
+++ b/net/ipv4/tcp_offload.c
@@ -195,12 +195,9 @@ struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb)
 
 	off = skb_gro_offset(skb);
 	hlen = off + sizeof(*th);
-	th = skb_gro_header_fast(skb, off);
-	if (skb_gro_header_hard(skb, hlen)) {
-		th = skb_gro_header_slow(skb, hlen, off);
-		if (unlikely(!th))
-			goto out;
-	}
+	th = skb_gro_header(skb, hlen, off);
+	if (unlikely(!th))
+		goto out;
 
 	thlen = th->doff * 4;
 	if (thlen < sizeof(*th))
@@ -258,7 +255,15 @@ found:
 
 	mss = skb_shinfo(p)->gso_size;
 
-	flush |= (len - 1) >= mss;
+	/* If skb is a GRO packet, make sure its gso_size matches prior packet mss.
+	 * If it is a single frame, do not aggregate it if its length
+	 * is bigger than our mss.
+	 */
+	if (unlikely(skb_is_gso(skb)))
+		flush |= (mss != skb_shinfo(skb)->gso_size);
+	else
+		flush |= (len - 1) >= mss;
+
 	flush |= (ntohl(th2->seq) + skb_gro_len(p)) ^ ntohl(th->seq);
 #ifdef CONFIG_TLS_DEVICE
 	flush |= p->decrypted ^ skb->decrypted;
@@ -272,7 +277,12 @@ found:
 	tcp_flag_word(th2) |= flags & (TCP_FLAG_FIN | TCP_FLAG_PSH);
 
 out_check_final:
-	flush = len < mss;
+	/* Force a flush if last segment is smaller than mss. */
+	if (unlikely(skb_is_gso(skb)))
+		flush = len != NAPI_GRO_CB(skb)->count * skb_shinfo(skb)->gso_size;
+	else
+		flush = len < mss;
+
 	flush |= (__force int)(flags & (TCP_FLAG_URG | TCP_FLAG_PSH |
 					TCP_FLAG_RST | TCP_FLAG_SYN |
 					TCP_FLAG_FIN));
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index 290019de766d..c69f4d966024 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -1875,15 +1875,20 @@ static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited)
 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
 	struct tcp_sock *tp = tcp_sk(sk);
 
-	/* Track the maximum number of outstanding packets in each
-	 * window, and remember whether we were cwnd-limited then.
+	/* Track the strongest available signal of the degree to which the cwnd
+	 * is fully utilized. If cwnd-limited then remember that fact for the
+	 * current window. If not cwnd-limited then track the maximum number of
+	 * outstanding packets in the current window. (If cwnd-limited then we
+	 * chose to not update tp->max_packets_out to avoid an extra else
+	 * clause with no functional impact.)
 	 */
-	if (!before(tp->snd_una, tp->max_packets_seq) ||
-	    tp->packets_out > tp->max_packets_out ||
-	    is_cwnd_limited) {
-		tp->max_packets_out = tp->packets_out;
-		tp->max_packets_seq = tp->snd_nxt;
+	if (!before(tp->snd_una, tp->cwnd_usage_seq) ||
+	    is_cwnd_limited ||
+	    (!tp->is_cwnd_limited &&
+	     tp->packets_out > tp->max_packets_out)) {
 		tp->is_cwnd_limited = is_cwnd_limited;
+		tp->max_packets_out = tp->packets_out;
+		tp->cwnd_usage_seq = tp->snd_nxt;
 	}
 
 	if (tcp_is_cwnd_limited(sk)) {
diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c
index b4dfb82d6ecb..cb79127f45c3 100644
--- a/net/ipv4/tcp_timer.c
+++ b/net/ipv4/tcp_timer.c
@@ -428,7 +428,7 @@ static void tcp_fastopen_synack_timer(struct sock *sk, struct request_sock *req)
 	if (!tp->retrans_stamp)
 		tp->retrans_stamp = tcp_time_stamp(tp);
 	inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
-			  TCP_TIMEOUT_INIT << req->num_timeout, TCP_RTO_MAX);
+			  req->timeout << req->num_timeout, TCP_RTO_MAX);
 }
 
 
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 560d9eadeaa5..d63118ce5900 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1801,41 +1801,29 @@ EXPORT_SYMBOL(__skb_recv_udp);
 
 int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
 {
-	int copied = 0;
-
-	while (1) {
-		struct sk_buff *skb;
-		int err, used;
-
-		skb = skb_recv_udp(sk, MSG_DONTWAIT, &err);
-		if (!skb)
-			return err;
+	struct sk_buff *skb;
+	int err, copied;
 
-		if (udp_lib_checksum_complete(skb)) {
-			__UDP_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS,
-					IS_UDPLITE(sk));
-			__UDP_INC_STATS(sock_net(sk), UDP_MIB_INERRORS,
-					IS_UDPLITE(sk));
-			atomic_inc(&sk->sk_drops);
-			kfree_skb(skb);
-			continue;
-		}
+try_again:
+	skb = skb_recv_udp(sk, MSG_DONTWAIT, &err);
+	if (!skb)
+		return err;
 
-		WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
-		used = recv_actor(sk, skb);
-		if (used <= 0) {
-			if (!copied)
-				copied = used;
-			kfree_skb(skb);
-			break;
-		} else if (used <= skb->len) {
-			copied += used;
-		}
+	if (udp_lib_checksum_complete(skb)) {
+		int is_udplite = IS_UDPLITE(sk);
+		struct net *net = sock_net(sk);
 
+		__UDP_INC_STATS(net, UDP_MIB_CSUMERRORS, is_udplite);
+		__UDP_INC_STATS(net, UDP_MIB_INERRORS, is_udplite);
+		atomic_inc(&sk->sk_drops);
 		kfree_skb(skb);
-		break;
+		goto try_again;
 	}
 
+	WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
+	copied = recv_actor(sk, skb);
+	kfree_skb(skb);
+
 	return copied;
 }
 EXPORT_SYMBOL(udp_read_skb);
diff --git a/net/ipv4/xfrm4_tunnel.c b/net/ipv4/xfrm4_tunnel.c
index 9d4f418f1bf8..8489fa106583 100644
--- a/net/ipv4/xfrm4_tunnel.c
+++ b/net/ipv4/xfrm4_tunnel.c
@@ -22,13 +22,17 @@ static int ipip_xfrm_rcv(struct xfrm_state *x, struct sk_buff *skb)
 	return ip_hdr(skb)->protocol;
 }
 
-static int ipip_init_state(struct xfrm_state *x)
+static int ipip_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
-	if (x->props.mode != XFRM_MODE_TUNNEL)
+	if (x->props.mode != XFRM_MODE_TUNNEL) {
+		NL_SET_ERR_MSG(extack, "IPv4 tunnel can only be used with tunnel mode");
 		return -EINVAL;
+	}
 
-	if (x->encap)
+	if (x->encap) {
+		NL_SET_ERR_MSG(extack, "IPv4 tunnel is not compatible with encapsulation");
 		return -EINVAL;
+	}
 
 	x->props.header_len = sizeof(struct iphdr);
 
diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c
index dbb1430d6cc2..d40b7d60e00e 100644
--- a/net/ipv6/af_inet6.c
+++ b/net/ipv6/af_inet6.c
@@ -1057,6 +1057,8 @@ static const struct ipv6_stub ipv6_stub_impl = {
 static const struct ipv6_bpf_stub ipv6_bpf_stub_impl = {
 	.inet6_bind = __inet6_bind,
 	.udp6_lib_lookup = __udp6_lib_lookup,
+	.ipv6_setsockopt = do_ipv6_setsockopt,
+	.ipv6_getsockopt = do_ipv6_getsockopt,
 };
 
 static int __init inet6_init(void)
diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c
index b5995c1f4d7a..5228d2716289 100644
--- a/net/ipv6/ah6.c
+++ b/net/ipv6/ah6.c
@@ -666,30 +666,38 @@ static int ah6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 	return 0;
 }
 
-static int ah6_init_state(struct xfrm_state *x)
+static int ah6_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
 	struct ah_data *ahp = NULL;
 	struct xfrm_algo_desc *aalg_desc;
 	struct crypto_ahash *ahash;
 
-	if (!x->aalg)
+	if (!x->aalg) {
+		NL_SET_ERR_MSG(extack, "AH requires a state with an AUTH algorithm");
 		goto error;
+	}
 
-	if (x->encap)
+	if (x->encap) {
+		NL_SET_ERR_MSG(extack, "AH is not compatible with encapsulation");
 		goto error;
+	}
 
 	ahp = kzalloc(sizeof(*ahp), GFP_KERNEL);
 	if (!ahp)
 		return -ENOMEM;
 
 	ahash = crypto_alloc_ahash(x->aalg->alg_name, 0, 0);
-	if (IS_ERR(ahash))
+	if (IS_ERR(ahash)) {
+		NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 		goto error;
+	}
 
 	ahp->ahash = ahash;
 	if (crypto_ahash_setkey(ahash, x->aalg->alg_key,
-			       (x->aalg->alg_key_len + 7) / 8))
+			       (x->aalg->alg_key_len + 7) / 8)) {
+		NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 		goto error;
+	}
 
 	/*
 	 * Lookup the algorithm description maintained by xfrm_algo,
@@ -702,9 +710,7 @@ static int ah6_init_state(struct xfrm_state *x)
 
 	if (aalg_desc->uinfo.auth.icv_fullbits/8 !=
 	    crypto_ahash_digestsize(ahash)) {
-		pr_info("AH: %s digestsize %u != %u\n",
-			x->aalg->alg_name, crypto_ahash_digestsize(ahash),
-			aalg_desc->uinfo.auth.icv_fullbits/8);
+		NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 		goto error;
 	}
 
@@ -721,6 +727,7 @@ static int ah6_init_state(struct xfrm_state *x)
 		x->props.header_len += sizeof(struct ipv6hdr);
 		break;
 	default:
+		NL_SET_ERR_MSG(extack, "Invalid mode requested for AH, must be one of TRANSPORT, TUNNEL, BEET");
 		goto error;
 	}
 	x->data = ahp;
diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c
index 8220923a12f7..14ed868680c6 100644
--- a/net/ipv6/esp6.c
+++ b/net/ipv6/esp6.c
@@ -151,6 +151,7 @@ static void esp_free_tcp_sk(struct rcu_head *head)
 static struct sock *esp6_find_tcp_sk(struct xfrm_state *x)
 {
 	struct xfrm_encap_tmpl *encap = x->encap;
+	struct net *net = xs_net(x);
 	struct esp_tcp_sk *esk;
 	__be16 sport, dport;
 	struct sock *nsk;
@@ -177,7 +178,7 @@ static struct sock *esp6_find_tcp_sk(struct xfrm_state *x)
 	}
 	spin_unlock_bh(&x->lock);
 
-	sk = __inet6_lookup_established(xs_net(x), &tcp_hashinfo, &x->id.daddr.in6,
+	sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, &x->id.daddr.in6,
 					dport, &x->props.saddr.in6, ntohs(sport), 0, 0);
 	if (!sk)
 		return ERR_PTR(-ENOENT);
@@ -1050,16 +1051,17 @@ static void esp6_destroy(struct xfrm_state *x)
 	crypto_free_aead(aead);
 }
 
-static int esp_init_aead(struct xfrm_state *x)
+static int esp_init_aead(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
 	char aead_name[CRYPTO_MAX_ALG_NAME];
 	struct crypto_aead *aead;
 	int err;
 
-	err = -ENAMETOOLONG;
 	if (snprintf(aead_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
-		     x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME)
-		goto error;
+		     x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME) {
+		NL_SET_ERR_MSG(extack, "Algorithm name is too long");
+		return -ENAMETOOLONG;
+	}
 
 	aead = crypto_alloc_aead(aead_name, 0, 0);
 	err = PTR_ERR(aead);
@@ -1077,11 +1079,15 @@ static int esp_init_aead(struct xfrm_state *x)
 	if (err)
 		goto error;
 
+	return 0;
+
 error:
+	NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 	return err;
 }
 
-static int esp_init_authenc(struct xfrm_state *x)
+static int esp_init_authenc(struct xfrm_state *x,
+			    struct netlink_ext_ack *extack)
 {
 	struct crypto_aead *aead;
 	struct crypto_authenc_key_param *param;
@@ -1092,10 +1098,6 @@ static int esp_init_authenc(struct xfrm_state *x)
 	unsigned int keylen;
 	int err;
 
-	err = -EINVAL;
-	if (!x->ealg)
-		goto error;
-
 	err = -ENAMETOOLONG;
 
 	if ((x->props.flags & XFRM_STATE_ESN)) {
@@ -1104,22 +1106,28 @@ static int esp_init_authenc(struct xfrm_state *x)
 			     x->geniv ?: "", x->geniv ? "(" : "",
 			     x->aalg ? x->aalg->alg_name : "digest_null",
 			     x->ealg->alg_name,
-			     x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME)
+			     x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) {
+			NL_SET_ERR_MSG(extack, "Algorithm name is too long");
 			goto error;
+		}
 	} else {
 		if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME,
 			     "%s%sauthenc(%s,%s)%s",
 			     x->geniv ?: "", x->geniv ? "(" : "",
 			     x->aalg ? x->aalg->alg_name : "digest_null",
 			     x->ealg->alg_name,
-			     x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME)
+			     x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME) {
+			NL_SET_ERR_MSG(extack, "Algorithm name is too long");
 			goto error;
+		}
 	}
 
 	aead = crypto_alloc_aead(authenc_name, 0, 0);
 	err = PTR_ERR(aead);
-	if (IS_ERR(aead))
+	if (IS_ERR(aead)) {
+		NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 		goto error;
+	}
 
 	x->data = aead;
 
@@ -1149,17 +1157,16 @@ static int esp_init_authenc(struct xfrm_state *x)
 		err = -EINVAL;
 		if (aalg_desc->uinfo.auth.icv_fullbits / 8 !=
 		    crypto_aead_authsize(aead)) {
-			pr_info("ESP: %s digestsize %u != %u\n",
-				x->aalg->alg_name,
-				crypto_aead_authsize(aead),
-				aalg_desc->uinfo.auth.icv_fullbits / 8);
+			NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 			goto free_key;
 		}
 
 		err = crypto_aead_setauthsize(
 			aead, x->aalg->alg_trunc_len / 8);
-		if (err)
+		if (err) {
+			NL_SET_ERR_MSG(extack, "Kernel was unable to initialize cryptographic operations");
 			goto free_key;
+		}
 	}
 
 	param->enckeylen = cpu_to_be32((x->ealg->alg_key_len + 7) / 8);
@@ -1174,7 +1181,7 @@ error:
 	return err;
 }
 
-static int esp6_init_state(struct xfrm_state *x)
+static int esp6_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
 	struct crypto_aead *aead;
 	u32 align;
@@ -1182,10 +1189,14 @@ static int esp6_init_state(struct xfrm_state *x)
 
 	x->data = NULL;
 
-	if (x->aead)
-		err = esp_init_aead(x);
-	else
-		err = esp_init_authenc(x);
+	if (x->aead) {
+		err = esp_init_aead(x, extack);
+	} else if (x->ealg) {
+		err = esp_init_authenc(x, extack);
+	} else {
+		NL_SET_ERR_MSG(extack, "ESP: AEAD or CRYPT must be provided");
+		err = -EINVAL;
+	}
 
 	if (err)
 		goto error;
@@ -1213,6 +1224,7 @@ static int esp6_init_state(struct xfrm_state *x)
 
 		switch (encap->encap_type) {
 		default:
+			NL_SET_ERR_MSG(extack, "Unsupported encapsulation type for ESP");
 			err = -EINVAL;
 			goto error;
 		case UDP_ENCAP_ESPINUDP:
diff --git a/net/ipv6/esp6_offload.c b/net/ipv6/esp6_offload.c
index 3a293838a91d..79d43548279c 100644
--- a/net/ipv6/esp6_offload.c
+++ b/net/ipv6/esp6_offload.c
@@ -145,7 +145,10 @@ static struct sk_buff *xfrm6_tunnel_gso_segment(struct xfrm_state *x,
 						struct sk_buff *skb,
 						netdev_features_t features)
 {
-	return skb_eth_gso_segment(skb, features, htons(ETH_P_IPV6));
+	__be16 type = x->inner_mode.family == AF_INET ? htons(ETH_P_IP)
+						      : htons(ETH_P_IPV6);
+
+	return skb_eth_gso_segment(skb, features, type);
 }
 
 static struct sk_buff *xfrm6_transport_gso_segment(struct xfrm_state *x,
diff --git a/net/ipv6/ila/ila_main.c b/net/ipv6/ila/ila_main.c
index 36c58aa257e8..3faf62530d6a 100644
--- a/net/ipv6/ila/ila_main.c
+++ b/net/ipv6/ila/ila_main.c
@@ -55,6 +55,7 @@ struct genl_family ila_nl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.ops		= ila_nl_ops,
 	.n_ops		= ARRAY_SIZE(ila_nl_ops),
+	.resv_start_op	= ILA_CMD_FLUSH + 1,
 };
 
 static __net_init int ila_init_net(struct net *net)
diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c
index 7d53d62783b1..b64b49012655 100644
--- a/net/ipv6/inet6_hashtables.c
+++ b/net/ipv6/inet6_hashtables.c
@@ -21,8 +21,6 @@
 #include <net/ip.h>
 #include <net/sock_reuseport.h>
 
-extern struct inet_hashinfo tcp_hashinfo;
-
 u32 inet6_ehashfn(const struct net *net,
 		  const struct in6_addr *laddr, const u16 lport,
 		  const struct in6_addr *faddr, const __be16 fport)
@@ -169,7 +167,7 @@ static inline struct sock *inet6_lookup_run_bpf(struct net *net,
 	struct sock *sk, *reuse_sk;
 	bool no_reuseport;
 
-	if (hashinfo != &tcp_hashinfo)
+	if (hashinfo != net->ipv4.tcp_death_row.hashinfo)
 		return NULL; /* only TCP is supported */
 
 	no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP, saddr, sport,
diff --git a/net/ipv6/ioam6.c b/net/ipv6/ioam6.c
index 1098131ed90c..571f0e4d9cf3 100644
--- a/net/ipv6/ioam6.c
+++ b/net/ipv6/ioam6.c
@@ -619,6 +619,7 @@ static struct genl_family ioam6_genl_family __ro_after_init = {
 	.parallel_ops	= true,
 	.ops		= ioam6_genl_ops,
 	.n_ops		= ARRAY_SIZE(ioam6_genl_ops),
+	.resv_start_op	= IOAM6_CMD_NS_SET_SCHEMA + 1,
 	.module		= THIS_MODULE,
 };
 
diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c
index 80cb50d459e4..48b4ff0294f6 100644
--- a/net/ipv6/ip6_gre.c
+++ b/net/ipv6/ip6_gre.c
@@ -360,7 +360,7 @@ static struct ip6_tnl *ip6gre_tunnel_locate(struct net *net,
 	if (parms->name[0]) {
 		if (!dev_valid_name(parms->name))
 			return NULL;
-		strlcpy(name, parms->name, IFNAMSIZ);
+		strscpy(name, parms->name, IFNAMSIZ);
 	} else {
 		strcpy(name, "ip6gre%d");
 	}
diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c
index d12dba2dd535..3ee345672849 100644
--- a/net/ipv6/ip6_offload.c
+++ b/net/ipv6/ip6_offload.c
@@ -219,12 +219,9 @@ INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
 
 	off = skb_gro_offset(skb);
 	hlen = off + sizeof(*iph);
-	iph = skb_gro_header_fast(skb, off);
-	if (skb_gro_header_hard(skb, hlen)) {
-		iph = skb_gro_header_slow(skb, hlen, off);
-		if (unlikely(!iph))
-			goto out;
-	}
+	iph = skb_gro_header(skb, hlen, off);
+	if (unlikely(!iph))
+		goto out;
 
 	skb_set_network_header(skb, off);
 	skb_gro_pull(skb, sizeof(*iph));
@@ -235,7 +232,7 @@ INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
 	proto = iph->nexthdr;
 	ops = rcu_dereference(inet6_offloads[proto]);
 	if (!ops || !ops->callbacks.gro_receive) {
-		__pskb_pull(skb, skb_gro_offset(skb));
+		pskb_pull(skb, skb_gro_offset(skb));
 		skb_gro_frag0_invalidate(skb);
 		proto = ipv6_gso_pull_exthdrs(skb, proto);
 		skb_gro_pull(skb, -skb_transport_offset(skb));
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index f152e51242cb..e19507614f64 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -1567,7 +1567,7 @@ emsgsize:
 				paged = true;
 				zc = true;
 			} else {
-				uarg->zerocopy = 0;
+				uarg_to_msgzc(uarg)->zerocopy = 0;
 				skb_zcopy_set(skb, uarg, &extra_uref);
 			}
 		}
@@ -1648,10 +1648,7 @@ alloc_new_skb:
 				 (fraglen + alloc_extra < SKB_MAX_ALLOC ||
 				  !(rt->dst.dev->features & NETIF_F_SG)))
 				alloclen = fraglen;
-			else if (!zc) {
-				alloclen = min_t(int, fraglen, MAX_HEADER);
-				pagedlen = fraglen - alloclen;
-			} else {
+			else {
 				alloclen = fragheaderlen + transhdrlen;
 				pagedlen = datalen - transhdrlen;
 			}
diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c
index 79c6a827dea9..cc5d5e75b658 100644
--- a/net/ipv6/ip6_tunnel.c
+++ b/net/ipv6/ip6_tunnel.c
@@ -293,7 +293,7 @@ static struct ip6_tnl *ip6_tnl_create(struct net *net, struct __ip6_tnl_parm *p)
 	if (p->name[0]) {
 		if (!dev_valid_name(p->name))
 			goto failed;
-		strlcpy(name, p->name, IFNAMSIZ);
+		strscpy(name, p->name, IFNAMSIZ);
 	} else {
 		sprintf(name, "ip6tnl%%d");
 	}
@@ -1988,39 +1988,6 @@ static void ip6_tnl_netlink_parms(struct nlattr *data[],
 		parms->fwmark = nla_get_u32(data[IFLA_IPTUN_FWMARK]);
 }
 
-static bool ip6_tnl_netlink_encap_parms(struct nlattr *data[],
-					struct ip_tunnel_encap *ipencap)
-{
-	bool ret = false;
-
-	memset(ipencap, 0, sizeof(*ipencap));
-
-	if (!data)
-		return ret;
-
-	if (data[IFLA_IPTUN_ENCAP_TYPE]) {
-		ret = true;
-		ipencap->type = nla_get_u16(data[IFLA_IPTUN_ENCAP_TYPE]);
-	}
-
-	if (data[IFLA_IPTUN_ENCAP_FLAGS]) {
-		ret = true;
-		ipencap->flags = nla_get_u16(data[IFLA_IPTUN_ENCAP_FLAGS]);
-	}
-
-	if (data[IFLA_IPTUN_ENCAP_SPORT]) {
-		ret = true;
-		ipencap->sport = nla_get_be16(data[IFLA_IPTUN_ENCAP_SPORT]);
-	}
-
-	if (data[IFLA_IPTUN_ENCAP_DPORT]) {
-		ret = true;
-		ipencap->dport = nla_get_be16(data[IFLA_IPTUN_ENCAP_DPORT]);
-	}
-
-	return ret;
-}
-
 static int ip6_tnl_newlink(struct net *src_net, struct net_device *dev,
 			   struct nlattr *tb[], struct nlattr *data[],
 			   struct netlink_ext_ack *extack)
@@ -2033,7 +2000,7 @@ static int ip6_tnl_newlink(struct net *src_net, struct net_device *dev,
 
 	nt = netdev_priv(dev);
 
-	if (ip6_tnl_netlink_encap_parms(data, &ipencap)) {
+	if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
 		err = ip6_tnl_encap_setup(nt, &ipencap);
 		if (err < 0)
 			return err;
@@ -2070,7 +2037,7 @@ static int ip6_tnl_changelink(struct net_device *dev, struct nlattr *tb[],
 	if (dev == ip6n->fb_tnl_dev)
 		return -EINVAL;
 
-	if (ip6_tnl_netlink_encap_parms(data, &ipencap)) {
+	if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
 		int err = ip6_tnl_encap_setup(t, &ipencap);
 
 		if (err < 0)
diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c
index 8fe59a79e800..151337d7f67b 100644
--- a/net/ipv6/ip6_vti.c
+++ b/net/ipv6/ip6_vti.c
@@ -154,7 +154,7 @@ vti6_tnl_link(struct vti6_net *ip6n, struct ip6_tnl *t)
 {
 	struct ip6_tnl __rcu **tp = vti6_tnl_bucket(ip6n, &t->parms);
 
-	rcu_assign_pointer(t->next , rtnl_dereference(*tp));
+	rcu_assign_pointer(t->next, rtnl_dereference(*tp));
 	rcu_assign_pointer(*tp, t);
 }
 
@@ -211,7 +211,7 @@ static struct ip6_tnl *vti6_tnl_create(struct net *net, struct __ip6_tnl_parm *p
 	if (p->name[0]) {
 		if (!dev_valid_name(p->name))
 			goto failed;
-		strlcpy(name, p->name, IFNAMSIZ);
+		strscpy(name, p->name, IFNAMSIZ);
 	} else {
 		sprintf(name, "ip6_vti%%d");
 	}
diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c
index 858fd8a28b5b..facdc78a43e5 100644
--- a/net/ipv6/ip6mr.c
+++ b/net/ipv6/ip6mr.c
@@ -1830,8 +1830,8 @@ int ip6_mroute_setsockopt(struct sock *sk, int optname, sockptr_t optval,
  *	Getsock opt support for the multicast routing system.
  */
 
-int ip6_mroute_getsockopt(struct sock *sk, int optname, char __user *optval,
-			  int __user *optlen)
+int ip6_mroute_getsockopt(struct sock *sk, int optname, sockptr_t optval,
+			  sockptr_t optlen)
 {
 	int olr;
 	int val;
@@ -1862,16 +1862,16 @@ int ip6_mroute_getsockopt(struct sock *sk, int optname, char __user *optval,
 		return -ENOPROTOOPT;
 	}
 
-	if (get_user(olr, optlen))
+	if (copy_from_sockptr(&olr, optlen, sizeof(int)))
 		return -EFAULT;
 
 	olr = min_t(int, olr, sizeof(int));
 	if (olr < 0)
 		return -EINVAL;
 
-	if (put_user(olr, optlen))
+	if (copy_to_sockptr(optlen, &olr, sizeof(int)))
 		return -EFAULT;
-	if (copy_to_user(optval, &val, olr))
+	if (copy_to_sockptr(optval, &val, olr))
 		return -EFAULT;
 	return 0;
 }
diff --git a/net/ipv6/ipcomp6.c b/net/ipv6/ipcomp6.c
index 15f984be3570..72d4858dec18 100644
--- a/net/ipv6/ipcomp6.c
+++ b/net/ipv6/ipcomp6.c
@@ -136,7 +136,8 @@ out:
 	return err;
 }
 
-static int ipcomp6_init_state(struct xfrm_state *x)
+static int ipcomp6_init_state(struct xfrm_state *x,
+			      struct netlink_ext_ack *extack)
 {
 	int err = -EINVAL;
 
@@ -148,17 +149,20 @@ static int ipcomp6_init_state(struct xfrm_state *x)
 		x->props.header_len += sizeof(struct ipv6hdr);
 		break;
 	default:
+		NL_SET_ERR_MSG(extack, "Unsupported XFRM mode for IPcomp");
 		goto out;
 	}
 
-	err = ipcomp_init_state(x);
+	err = ipcomp_init_state(x, extack);
 	if (err)
 		goto out;
 
 	if (x->props.mode == XFRM_MODE_TUNNEL) {
 		err = ipcomp6_tunnel_attach(x);
-		if (err)
+		if (err) {
+			NL_SET_ERR_MSG(extack, "Kernel error: failed to initialize the associated state");
 			goto out;
+		}
 	}
 
 	err = 0;
diff --git a/net/ipv6/ipv6_sockglue.c b/net/ipv6/ipv6_sockglue.c
index e0dcc7a193df..2d2f4dd9e5df 100644
--- a/net/ipv6/ipv6_sockglue.c
+++ b/net/ipv6/ipv6_sockglue.c
@@ -327,7 +327,7 @@ static int ipv6_set_opt_hdr(struct sock *sk, int optname, sockptr_t optval,
 	int err;
 
 	/* hop-by-hop / destination options are privileged option */
-	if (optname != IPV6_RTHDR && !ns_capable(net->user_ns, CAP_NET_RAW))
+	if (optname != IPV6_RTHDR && !sockopt_ns_capable(net->user_ns, CAP_NET_RAW))
 		return -EPERM;
 
 	/* remove any sticky options header with a zero option
@@ -391,8 +391,8 @@ sticky_done:
 	return err;
 }
 
-static int do_ipv6_setsockopt(struct sock *sk, int level, int optname,
-		   sockptr_t optval, unsigned int optlen)
+int do_ipv6_setsockopt(struct sock *sk, int level, int optname,
+		       sockptr_t optval, unsigned int optlen)
 {
 	struct ipv6_pinfo *np = inet6_sk(sk);
 	struct net *net = sock_net(sk);
@@ -417,7 +417,7 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname,
 
 	if (needs_rtnl)
 		rtnl_lock();
-	lock_sock(sk);
+	sockopt_lock_sock(sk);
 
 	switch (optname) {
 
@@ -634,8 +634,8 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname,
 		break;
 
 	case IPV6_TRANSPARENT:
-		if (valbool && !ns_capable(net->user_ns, CAP_NET_RAW) &&
-		    !ns_capable(net->user_ns, CAP_NET_ADMIN)) {
+		if (valbool && !sockopt_ns_capable(net->user_ns, CAP_NET_RAW) &&
+		    !sockopt_ns_capable(net->user_ns, CAP_NET_ADMIN)) {
 			retv = -EPERM;
 			break;
 		}
@@ -946,7 +946,7 @@ done:
 	case IPV6_IPSEC_POLICY:
 	case IPV6_XFRM_POLICY:
 		retv = -EPERM;
-		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
+		if (!sockopt_ns_capable(net->user_ns, CAP_NET_ADMIN))
 			break;
 		retv = xfrm_user_policy(sk, optname, optval, optlen);
 		break;
@@ -994,14 +994,14 @@ done:
 		break;
 	}
 
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (needs_rtnl)
 		rtnl_unlock();
 
 	return retv;
 
 e_inval:
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	if (needs_rtnl)
 		rtnl_unlock();
 	return -EINVAL;
@@ -1030,7 +1030,7 @@ int ipv6_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
 EXPORT_SYMBOL(ipv6_setsockopt);
 
 static int ipv6_getsockopt_sticky(struct sock *sk, struct ipv6_txoptions *opt,
-				  int optname, char __user *optval, int len)
+				  int optname, sockptr_t optval, int len)
 {
 	struct ipv6_opt_hdr *hdr;
 
@@ -1058,56 +1058,53 @@ static int ipv6_getsockopt_sticky(struct sock *sk, struct ipv6_txoptions *opt,
 		return 0;
 
 	len = min_t(unsigned int, len, ipv6_optlen(hdr));
-	if (copy_to_user(optval, hdr, len))
+	if (copy_to_sockptr(optval, hdr, len))
 		return -EFAULT;
 	return len;
 }
 
-static int ipv6_get_msfilter(struct sock *sk, void __user *optval,
-		int __user *optlen, int len)
+static int ipv6_get_msfilter(struct sock *sk, sockptr_t optval,
+			     sockptr_t optlen, int len)
 {
 	const int size0 = offsetof(struct group_filter, gf_slist_flex);
-	struct group_filter __user *p = optval;
 	struct group_filter gsf;
 	int num;
 	int err;
 
 	if (len < size0)
 		return -EINVAL;
-	if (copy_from_user(&gsf, p, size0))
+	if (copy_from_sockptr(&gsf, optval, size0))
 		return -EFAULT;
 	if (gsf.gf_group.ss_family != AF_INET6)
 		return -EADDRNOTAVAIL;
 	num = gsf.gf_numsrc;
-	lock_sock(sk);
-	err = ip6_mc_msfget(sk, &gsf, p->gf_slist_flex);
+	sockopt_lock_sock(sk);
+	err = ip6_mc_msfget(sk, &gsf, optval, size0);
 	if (!err) {
 		if (num > gsf.gf_numsrc)
 			num = gsf.gf_numsrc;
-		if (put_user(GROUP_FILTER_SIZE(num), optlen) ||
-		    copy_to_user(p, &gsf, size0))
+		len = GROUP_FILTER_SIZE(num);
+		if (copy_to_sockptr(optlen, &len, sizeof(int)) ||
+		    copy_to_sockptr(optval, &gsf, size0))
 			err = -EFAULT;
 	}
-	release_sock(sk);
+	sockopt_release_sock(sk);
 	return err;
 }
 
-static int compat_ipv6_get_msfilter(struct sock *sk, void __user *optval,
-		int __user *optlen)
+static int compat_ipv6_get_msfilter(struct sock *sk, sockptr_t optval,
+				    sockptr_t optlen, int len)
 {
 	const int size0 = offsetof(struct compat_group_filter, gf_slist_flex);
-	struct compat_group_filter __user *p = optval;
 	struct compat_group_filter gf32;
 	struct group_filter gf;
-	int len, err;
+	int err;
 	int num;
 
-	if (get_user(len, optlen))
-		return -EFAULT;
 	if (len < size0)
 		return -EINVAL;
 
-	if (copy_from_user(&gf32, p, size0))
+	if (copy_from_sockptr(&gf32, optval, size0))
 		return -EFAULT;
 	gf.gf_interface = gf32.gf_interface;
 	gf.gf_fmode = gf32.gf_fmode;
@@ -1117,23 +1114,25 @@ static int compat_ipv6_get_msfilter(struct sock *sk, void __user *optval,
 	if (gf.gf_group.ss_family != AF_INET6)
 		return -EADDRNOTAVAIL;
 
-	lock_sock(sk);
-	err = ip6_mc_msfget(sk, &gf, p->gf_slist_flex);
-	release_sock(sk);
+	sockopt_lock_sock(sk);
+	err = ip6_mc_msfget(sk, &gf, optval, size0);
+	sockopt_release_sock(sk);
 	if (err)
 		return err;
 	if (num > gf.gf_numsrc)
 		num = gf.gf_numsrc;
 	len = GROUP_FILTER_SIZE(num) - (sizeof(gf)-sizeof(gf32));
-	if (put_user(len, optlen) ||
-	    put_user(gf.gf_fmode, &p->gf_fmode) ||
-	    put_user(gf.gf_numsrc, &p->gf_numsrc))
+	if (copy_to_sockptr(optlen, &len, sizeof(int)) ||
+	    copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_fmode),
+				   &gf.gf_fmode, sizeof(gf32.gf_fmode)) ||
+	    copy_to_sockptr_offset(optval, offsetof(struct compat_group_filter, gf_numsrc),
+				   &gf.gf_numsrc, sizeof(gf32.gf_numsrc)))
 		return -EFAULT;
 	return 0;
 }
 
-static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
-		    char __user *optval, int __user *optlen, unsigned int flags)
+int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
+		       sockptr_t optval, sockptr_t optlen)
 {
 	struct ipv6_pinfo *np = inet6_sk(sk);
 	int len;
@@ -1142,7 +1141,7 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
 	if (ip6_mroute_opt(optname))
 		return ip6_mroute_getsockopt(sk, optname, optval, optlen);
 
-	if (get_user(len, optlen))
+	if (copy_from_sockptr(&len, optlen, sizeof(int)))
 		return -EFAULT;
 	switch (optname) {
 	case IPV6_ADDRFORM:
@@ -1156,7 +1155,7 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
 		break;
 	case MCAST_MSFILTER:
 		if (in_compat_syscall())
-			return compat_ipv6_get_msfilter(sk, optval, optlen);
+			return compat_ipv6_get_msfilter(sk, optval, optlen, len);
 		return ipv6_get_msfilter(sk, optval, optlen, len);
 	case IPV6_2292PKTOPTIONS:
 	{
@@ -1166,16 +1165,21 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
 		if (sk->sk_type != SOCK_STREAM)
 			return -ENOPROTOOPT;
 
-		msg.msg_control_user = optval;
+		if (optval.is_kernel) {
+			msg.msg_control_is_user = false;
+			msg.msg_control = optval.kernel;
+		} else {
+			msg.msg_control_is_user = true;
+			msg.msg_control_user = optval.user;
+		}
 		msg.msg_controllen = len;
-		msg.msg_flags = flags;
-		msg.msg_control_is_user = true;
+		msg.msg_flags = 0;
 
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
 		skb = np->pktoptions;
 		if (skb)
 			ip6_datagram_recv_ctl(sk, &msg, skb);
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		if (!skb) {
 			if (np->rxopt.bits.rxinfo) {
 				struct in6_pktinfo src_info;
@@ -1212,7 +1216,7 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
 			}
 		}
 		len -= msg.msg_controllen;
-		return put_user(len, optlen);
+		return copy_to_sockptr(optlen, &len, sizeof(int));
 	}
 	case IPV6_MTU:
 	{
@@ -1264,15 +1268,15 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
 	{
 		struct ipv6_txoptions *opt;
 
-		lock_sock(sk);
+		sockopt_lock_sock(sk);
 		opt = rcu_dereference_protected(np->opt,
 						lockdep_sock_is_held(sk));
 		len = ipv6_getsockopt_sticky(sk, opt, optname, optval, len);
-		release_sock(sk);
+		sockopt_release_sock(sk);
 		/* check if ipv6_getsockopt_sticky() returns err code */
 		if (len < 0)
 			return len;
-		return put_user(len, optlen);
+		return copy_to_sockptr(optlen, &len, sizeof(int));
 	}
 
 	case IPV6_RECVHOPOPTS:
@@ -1326,9 +1330,9 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
 		if (!mtuinfo.ip6m_mtu)
 			return -ENOTCONN;
 
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &mtuinfo, len))
+		if (copy_to_sockptr(optval, &mtuinfo, len))
 			return -EFAULT;
 
 		return 0;
@@ -1405,7 +1409,7 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
 		if (len < sizeof(freq))
 			return -EINVAL;
 
-		if (copy_from_user(&freq, optval, sizeof(freq)))
+		if (copy_from_sockptr(&freq, optval, sizeof(freq)))
 			return -EFAULT;
 
 		if (freq.flr_action != IPV6_FL_A_GET)
@@ -1420,9 +1424,9 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
 		if (val < 0)
 			return val;
 
-		if (put_user(len, optlen))
+		if (copy_to_sockptr(optlen, &len, sizeof(int)))
 			return -EFAULT;
-		if (copy_to_user(optval, &freq, len))
+		if (copy_to_sockptr(optval, &freq, len))
 			return -EFAULT;
 
 		return 0;
@@ -1474,9 +1478,9 @@ static int do_ipv6_getsockopt(struct sock *sk, int level, int optname,
 		return -ENOPROTOOPT;
 	}
 	len = min_t(unsigned int, sizeof(int), len);
-	if (put_user(len, optlen))
+	if (copy_to_sockptr(optlen, &len, sizeof(int)))
 		return -EFAULT;
-	if (copy_to_user(optval, &val, len))
+	if (copy_to_sockptr(optval, &val, len))
 		return -EFAULT;
 	return 0;
 }
@@ -1492,7 +1496,8 @@ int ipv6_getsockopt(struct sock *sk, int level, int optname,
 	if (level != SOL_IPV6)
 		return -ENOPROTOOPT;
 
-	err = do_ipv6_getsockopt(sk, level, optname, optval, optlen, 0);
+	err = do_ipv6_getsockopt(sk, level, optname,
+				 USER_SOCKPTR(optval), USER_SOCKPTR(optlen));
 #ifdef CONFIG_NETFILTER
 	/* we need to exclude all possible ENOPROTOOPTs except default case */
 	if (err == -ENOPROTOOPT && optname != IPV6_2292PKTOPTIONS) {
diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c
index 87c699d57b36..0566ab03ddbe 100644
--- a/net/ipv6/mcast.c
+++ b/net/ipv6/mcast.c
@@ -580,7 +580,7 @@ done:
 }
 
 int ip6_mc_msfget(struct sock *sk, struct group_filter *gsf,
-		  struct sockaddr_storage __user *p)
+		  sockptr_t optval, size_t ss_offset)
 {
 	struct ipv6_pinfo *inet6 = inet6_sk(sk);
 	const struct in6_addr *group;
@@ -612,8 +612,7 @@ int ip6_mc_msfget(struct sock *sk, struct group_filter *gsf,
 
 	copycount = count < gsf->gf_numsrc ? count : gsf->gf_numsrc;
 	gsf->gf_numsrc = count;
-
-	for (i = 0; i < copycount; i++, p++) {
+	for (i = 0; i < copycount; i++) {
 		struct sockaddr_in6 *psin6;
 		struct sockaddr_storage ss;
 
@@ -621,8 +620,9 @@ int ip6_mc_msfget(struct sock *sk, struct group_filter *gsf,
 		memset(&ss, 0, sizeof(ss));
 		psin6->sin6_family = AF_INET6;
 		psin6->sin6_addr = psl->sl_addr[i];
-		if (copy_to_user(p, &ss, sizeof(ss)))
+		if (copy_to_sockptr_offset(optval, ss_offset, &ss, sizeof(ss)))
 			return -EFAULT;
+		ss_offset += sizeof(ss);
 	}
 	return 0;
 }
diff --git a/net/ipv6/mip6.c b/net/ipv6/mip6.c
index aeb35d26e474..83d2a8be263f 100644
--- a/net/ipv6/mip6.c
+++ b/net/ipv6/mip6.c
@@ -247,15 +247,14 @@ static int mip6_destopt_reject(struct xfrm_state *x, struct sk_buff *skb,
 	return err;
 }
 
-static int mip6_destopt_init_state(struct xfrm_state *x)
+static int mip6_destopt_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
 	if (x->id.spi) {
-		pr_info("%s: spi is not 0: %u\n", __func__, x->id.spi);
+		NL_SET_ERR_MSG(extack, "SPI must be 0");
 		return -EINVAL;
 	}
 	if (x->props.mode != XFRM_MODE_ROUTEOPTIMIZATION) {
-		pr_info("%s: state's mode is not %u: %u\n",
-			__func__, XFRM_MODE_ROUTEOPTIMIZATION, x->props.mode);
+		NL_SET_ERR_MSG(extack, "XFRM mode must be XFRM_MODE_ROUTEOPTIMIZATION");
 		return -EINVAL;
 	}
 
@@ -333,15 +332,14 @@ static int mip6_rthdr_output(struct xfrm_state *x, struct sk_buff *skb)
 	return 0;
 }
 
-static int mip6_rthdr_init_state(struct xfrm_state *x)
+static int mip6_rthdr_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
 	if (x->id.spi) {
-		pr_info("%s: spi is not 0: %u\n", __func__, x->id.spi);
+		NL_SET_ERR_MSG(extack, "SPI must be 0");
 		return -EINVAL;
 	}
 	if (x->props.mode != XFRM_MODE_ROUTEOPTIMIZATION) {
-		pr_info("%s: state's mode is not %u: %u\n",
-			__func__, XFRM_MODE_ROUTEOPTIMIZATION, x->props.mode);
+		NL_SET_ERR_MSG(extack, "XFRM mode must be XFRM_MODE_ROUTEOPTIMIZATION");
 		return -EINVAL;
 	}
 
diff --git a/net/ipv6/netfilter/nf_socket_ipv6.c b/net/ipv6/netfilter/nf_socket_ipv6.c
index aa5bb8789ba0..a7690ec62325 100644
--- a/net/ipv6/netfilter/nf_socket_ipv6.c
+++ b/net/ipv6/netfilter/nf_socket_ipv6.c
@@ -83,8 +83,8 @@ nf_socket_get_sock_v6(struct net *net, struct sk_buff *skb, int doff,
 {
 	switch (protocol) {
 	case IPPROTO_TCP:
-		return inet6_lookup(net, &tcp_hashinfo, skb, doff,
-				    saddr, sport, daddr, dport,
+		return inet6_lookup(net, net->ipv4.tcp_death_row.hashinfo,
+				    skb, doff, saddr, sport, daddr, dport,
 				    in->ifindex);
 	case IPPROTO_UDP:
 		return udp6_lib_lookup(net, saddr, sport, daddr, dport,
diff --git a/net/ipv6/netfilter/nf_tproxy_ipv6.c b/net/ipv6/netfilter/nf_tproxy_ipv6.c
index 6bac68fb27a3..929502e51203 100644
--- a/net/ipv6/netfilter/nf_tproxy_ipv6.c
+++ b/net/ipv6/netfilter/nf_tproxy_ipv6.c
@@ -80,6 +80,7 @@ nf_tproxy_get_sock_v6(struct net *net, struct sk_buff *skb, int thoff,
 		      const struct net_device *in,
 		      const enum nf_tproxy_lookup_t lookup_type)
 {
+	struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo;
 	struct sock *sk;
 
 	switch (protocol) {
@@ -93,7 +94,7 @@ nf_tproxy_get_sock_v6(struct net *net, struct sk_buff *skb, int thoff,
 
 		switch (lookup_type) {
 		case NF_TPROXY_LOOKUP_LISTENER:
-			sk = inet6_lookup_listener(net, &tcp_hashinfo, skb,
+			sk = inet6_lookup_listener(net, hinfo, skb,
 						   thoff + __tcp_hdrlen(hp),
 						   saddr, sport,
 						   daddr, ntohs(dport),
@@ -108,9 +109,8 @@ nf_tproxy_get_sock_v6(struct net *net, struct sk_buff *skb, int thoff,
 			 */
 			break;
 		case NF_TPROXY_LOOKUP_ESTABLISHED:
-			sk = __inet6_lookup_established(net, &tcp_hashinfo,
-							saddr, sport, daddr, ntohs(dport),
-							in->ifindex, 0);
+			sk = __inet6_lookup_established(net, hinfo, saddr, sport, daddr,
+							ntohs(dport), in->ifindex, 0);
 			break;
 		default:
 			BUG();
diff --git a/net/ipv6/netfilter/nft_fib_ipv6.c b/net/ipv6/netfilter/nft_fib_ipv6.c
index 8970d0b4faeb..1d7e520d9966 100644
--- a/net/ipv6/netfilter/nft_fib_ipv6.c
+++ b/net/ipv6/netfilter/nft_fib_ipv6.c
@@ -41,6 +41,9 @@ static int nft_fib6_flowi_init(struct flowi6 *fl6, const struct nft_fib *priv,
 	if (ipv6_addr_type(&fl6->daddr) & IPV6_ADDR_LINKLOCAL) {
 		lookup_flags |= RT6_LOOKUP_F_IFACE;
 		fl6->flowi6_oif = get_ifindex(dev ? dev : pkt->skb->dev);
+	} else if ((priv->flags & NFTA_FIB_F_IIF) &&
+		   (netif_is_l3_master(dev) || netif_is_l3_slave(dev))) {
+		fl6->flowi6_oif = dev->ifindex;
 	}
 
 	if (ipv6_addr_type(&fl6->saddr) & IPV6_ADDR_UNICAST)
@@ -197,7 +200,8 @@ void nft_fib6_eval(const struct nft_expr *expr, struct nft_regs *regs,
 	if (rt->rt6i_flags & (RTF_REJECT | RTF_ANYCAST | RTF_LOCAL))
 		goto put_rt_err;
 
-	if (oif && oif != rt->rt6i_idev->dev)
+	if (oif && oif != rt->rt6i_idev->dev &&
+	    l3mdev_master_ifindex_rcu(rt->rt6i_idev->dev) != oif->ifindex)
 		goto put_rt_err;
 
 	nft_fib_store_result(dest, priv, rt->rt6i_idev->dev);
diff --git a/net/ipv6/ping.c b/net/ipv6/ping.c
index 91b840514656..5f2ef8493714 100644
--- a/net/ipv6/ping.c
+++ b/net/ipv6/ping.c
@@ -20,6 +20,7 @@
 #include <net/udp.h>
 #include <net/transp_v6.h>
 #include <linux/proc_fs.h>
+#include <linux/bpf-cgroup.h>
 #include <net/ping.h>
 
 static void ping_v6_destroy(struct sock *sk)
@@ -49,6 +50,20 @@ static int dummy_ipv6_chk_addr(struct net *net, const struct in6_addr *addr,
 	return 0;
 }
 
+static int ping_v6_pre_connect(struct sock *sk, struct sockaddr *uaddr,
+			       int addr_len)
+{
+	/* This check is replicated from __ip6_datagram_connect() and
+	 * intended to prevent BPF program called below from accessing
+	 * bytes that are out of the bound specified by user in addr_len.
+	 */
+
+	if (addr_len < SIN6_LEN_RFC2133)
+		return -EINVAL;
+
+	return BPF_CGROUP_RUN_PROG_INET6_CONNECT_LOCK(sk, uaddr);
+}
+
 static int ping_v6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
 {
 	struct inet_sock *inet = inet_sk(sk);
@@ -191,6 +206,7 @@ struct proto pingv6_prot = {
 	.init =		ping_init_sock,
 	.close =	ping_close,
 	.destroy =	ping_v6_destroy,
+	.pre_connect =	ping_v6_pre_connect,
 	.connect =	ip6_datagram_connect_v6_only,
 	.disconnect =	__udp_disconnect,
 	.setsockopt =	ipv6_setsockopt,
diff --git a/net/ipv6/seg6.c b/net/ipv6/seg6.c
index 0b0e34ddc64e..29346a6eec9f 100644
--- a/net/ipv6/seg6.c
+++ b/net/ipv6/seg6.c
@@ -504,6 +504,7 @@ static struct genl_family seg6_genl_family __ro_after_init = {
 	.parallel_ops	= true,
 	.ops		= seg6_genl_ops,
 	.n_ops		= ARRAY_SIZE(seg6_genl_ops),
+	.resv_start_op	= SEG6_CMD_GET_TUNSRC + 1,
 	.module		= THIS_MODULE,
 };
 
diff --git a/net/ipv6/seg6_local.c b/net/ipv6/seg6_local.c
index b7de5e46fdd8..8370726ae7bf 100644
--- a/net/ipv6/seg6_local.c
+++ b/net/ipv6/seg6_local.c
@@ -73,6 +73,55 @@ struct bpf_lwt_prog {
 	char *name;
 };
 
+/* default length values (expressed in bits) for both Locator-Block and
+ * Locator-Node Function.
+ *
+ * Both SEG6_LOCAL_LCBLOCK_DBITS and SEG6_LOCAL_LCNODE_FN_DBITS *must* be:
+ *    i) greater than 0;
+ *   ii) evenly divisible by 8. In other terms, the lengths of the
+ *	 Locator-Block and Locator-Node Function must be byte-aligned (we can
+ *	 relax this constraint in the future if really needed).
+ *
+ * Moreover, a third condition must hold:
+ *  iii) SEG6_LOCAL_LCBLOCK_DBITS + SEG6_LOCAL_LCNODE_FN_DBITS <= 128.
+ *
+ * The correctness of SEG6_LOCAL_LCBLOCK_DBITS and SEG6_LOCAL_LCNODE_FN_DBITS
+ * values are checked during the kernel compilation. If the compilation stops,
+ * check the value of these parameters to see if they meet conditions (i), (ii)
+ * and (iii).
+ */
+#define SEG6_LOCAL_LCBLOCK_DBITS	32
+#define SEG6_LOCAL_LCNODE_FN_DBITS	16
+
+/* The following next_csid_chk_{cntr,lcblock,lcblock_fn}_bits macros can be
+ * used directly to check whether the lengths (in bits) of Locator-Block and
+ * Locator-Node Function are valid according to (i), (ii), (iii).
+ */
+#define next_csid_chk_cntr_bits(blen, flen)		\
+	((blen) + (flen) > 128)
+
+#define next_csid_chk_lcblock_bits(blen)		\
+({							\
+	typeof(blen) __tmp = blen;			\
+	(!__tmp || __tmp > 120 || (__tmp & 0x07));	\
+})
+
+#define next_csid_chk_lcnode_fn_bits(flen)		\
+	next_csid_chk_lcblock_bits(flen)
+
+/* Supported Flavor operations are reported in this bitmask */
+#define SEG6_LOCAL_FLV_SUPP_OPS	(BIT(SEG6_LOCAL_FLV_OP_NEXT_CSID))
+
+struct seg6_flavors_info {
+	/* Flavor operations */
+	__u32 flv_ops;
+
+	/* Locator-Block length, expressed in bits */
+	__u8 lcblock_bits;
+	/* Locator-Node Function length, expressed in bits*/
+	__u8 lcnode_func_bits;
+};
+
 enum seg6_end_dt_mode {
 	DT_INVALID_MODE	= -EINVAL,
 	DT_LEGACY_MODE	= 0,
@@ -136,6 +185,8 @@ struct seg6_local_lwt {
 #ifdef CONFIG_NET_L3_MASTER_DEV
 	struct seg6_end_dt_info dt_info;
 #endif
+	struct seg6_flavors_info flv_info;
+
 	struct pcpu_seg6_local_counters __percpu *pcpu_counters;
 
 	int headroom;
@@ -271,8 +322,50 @@ int seg6_lookup_nexthop(struct sk_buff *skb,
 	return seg6_lookup_any_nexthop(skb, nhaddr, tbl_id, false);
 }
 
-/* regular endpoint function */
-static int input_action_end(struct sk_buff *skb, struct seg6_local_lwt *slwt)
+static __u8 seg6_flv_lcblock_octects(const struct seg6_flavors_info *finfo)
+{
+	return finfo->lcblock_bits >> 3;
+}
+
+static __u8 seg6_flv_lcnode_func_octects(const struct seg6_flavors_info *finfo)
+{
+	return finfo->lcnode_func_bits >> 3;
+}
+
+static bool seg6_next_csid_is_arg_zero(const struct in6_addr *addr,
+				       const struct seg6_flavors_info *finfo)
+{
+	__u8 fnc_octects = seg6_flv_lcnode_func_octects(finfo);
+	__u8 blk_octects = seg6_flv_lcblock_octects(finfo);
+	__u8 arg_octects;
+	int i;
+
+	arg_octects = 16 - blk_octects - fnc_octects;
+	for (i = 0; i < arg_octects; ++i) {
+		if (addr->s6_addr[blk_octects + fnc_octects + i] != 0x00)
+			return false;
+	}
+
+	return true;
+}
+
+/* assume that DA.Argument length > 0 */
+static void seg6_next_csid_advance_arg(struct in6_addr *addr,
+				       const struct seg6_flavors_info *finfo)
+{
+	__u8 fnc_octects = seg6_flv_lcnode_func_octects(finfo);
+	__u8 blk_octects = seg6_flv_lcblock_octects(finfo);
+
+	/* advance DA.Argument */
+	memmove(&addr->s6_addr[blk_octects],
+		&addr->s6_addr[blk_octects + fnc_octects],
+		16 - blk_octects - fnc_octects);
+
+	memset(&addr->s6_addr[16 - fnc_octects], 0x00, fnc_octects);
+}
+
+static int input_action_end_core(struct sk_buff *skb,
+				 struct seg6_local_lwt *slwt)
 {
 	struct ipv6_sr_hdr *srh;
 
@@ -291,6 +384,38 @@ drop:
 	return -EINVAL;
 }
 
+static int end_next_csid_core(struct sk_buff *skb, struct seg6_local_lwt *slwt)
+{
+	const struct seg6_flavors_info *finfo = &slwt->flv_info;
+	struct in6_addr *daddr = &ipv6_hdr(skb)->daddr;
+
+	if (seg6_next_csid_is_arg_zero(daddr, finfo))
+		return input_action_end_core(skb, slwt);
+
+	/* update DA */
+	seg6_next_csid_advance_arg(daddr, finfo);
+
+	seg6_lookup_nexthop(skb, NULL, 0);
+
+	return dst_input(skb);
+}
+
+static bool seg6_next_csid_enabled(__u32 fops)
+{
+	return fops & BIT(SEG6_LOCAL_FLV_OP_NEXT_CSID);
+}
+
+/* regular endpoint function */
+static int input_action_end(struct sk_buff *skb, struct seg6_local_lwt *slwt)
+{
+	const struct seg6_flavors_info *finfo = &slwt->flv_info;
+
+	if (seg6_next_csid_enabled(finfo->flv_ops))
+		return end_next_csid_core(skb, slwt);
+
+	return input_action_end_core(skb, slwt);
+}
+
 /* regular endpoint, and forward to specified nexthop */
 static int input_action_end_x(struct sk_buff *skb, struct seg6_local_lwt *slwt)
 {
@@ -951,7 +1076,8 @@ static struct seg6_action_desc seg6_action_table[] = {
 	{
 		.action		= SEG6_LOCAL_ACTION_END,
 		.attrs		= 0,
-		.optattrs	= SEG6_F_LOCAL_COUNTERS,
+		.optattrs	= SEG6_F_LOCAL_COUNTERS |
+				  SEG6_F_ATTR(SEG6_LOCAL_FLAVORS),
 		.input		= input_action_end,
 	},
 	{
@@ -1132,9 +1258,11 @@ static const struct nla_policy seg6_local_policy[SEG6_LOCAL_MAX + 1] = {
 	[SEG6_LOCAL_OIF]	= { .type = NLA_U32 },
 	[SEG6_LOCAL_BPF]	= { .type = NLA_NESTED },
 	[SEG6_LOCAL_COUNTERS]	= { .type = NLA_NESTED },
+	[SEG6_LOCAL_FLAVORS]	= { .type = NLA_NESTED },
 };
 
-static int parse_nla_srh(struct nlattr **attrs, struct seg6_local_lwt *slwt)
+static int parse_nla_srh(struct nlattr **attrs, struct seg6_local_lwt *slwt,
+			 struct netlink_ext_ack *extack)
 {
 	struct ipv6_sr_hdr *srh;
 	int len;
@@ -1191,7 +1319,8 @@ static void destroy_attr_srh(struct seg6_local_lwt *slwt)
 	kfree(slwt->srh);
 }
 
-static int parse_nla_table(struct nlattr **attrs, struct seg6_local_lwt *slwt)
+static int parse_nla_table(struct nlattr **attrs, struct seg6_local_lwt *slwt,
+			   struct netlink_ext_ack *extack)
 {
 	slwt->table = nla_get_u32(attrs[SEG6_LOCAL_TABLE]);
 
@@ -1225,7 +1354,8 @@ seg6_end_dt_info *seg6_possible_end_dt_info(struct seg6_local_lwt *slwt)
 }
 
 static int parse_nla_vrftable(struct nlattr **attrs,
-			      struct seg6_local_lwt *slwt)
+			      struct seg6_local_lwt *slwt,
+			      struct netlink_ext_ack *extack)
 {
 	struct seg6_end_dt_info *info = seg6_possible_end_dt_info(slwt);
 
@@ -1261,7 +1391,8 @@ static int cmp_nla_vrftable(struct seg6_local_lwt *a, struct seg6_local_lwt *b)
 	return 0;
 }
 
-static int parse_nla_nh4(struct nlattr **attrs, struct seg6_local_lwt *slwt)
+static int parse_nla_nh4(struct nlattr **attrs, struct seg6_local_lwt *slwt,
+			 struct netlink_ext_ack *extack)
 {
 	memcpy(&slwt->nh4, nla_data(attrs[SEG6_LOCAL_NH4]),
 	       sizeof(struct in_addr));
@@ -1287,7 +1418,8 @@ static int cmp_nla_nh4(struct seg6_local_lwt *a, struct seg6_local_lwt *b)
 	return memcmp(&a->nh4, &b->nh4, sizeof(struct in_addr));
 }
 
-static int parse_nla_nh6(struct nlattr **attrs, struct seg6_local_lwt *slwt)
+static int parse_nla_nh6(struct nlattr **attrs, struct seg6_local_lwt *slwt,
+			 struct netlink_ext_ack *extack)
 {
 	memcpy(&slwt->nh6, nla_data(attrs[SEG6_LOCAL_NH6]),
 	       sizeof(struct in6_addr));
@@ -1313,7 +1445,8 @@ static int cmp_nla_nh6(struct seg6_local_lwt *a, struct seg6_local_lwt *b)
 	return memcmp(&a->nh6, &b->nh6, sizeof(struct in6_addr));
 }
 
-static int parse_nla_iif(struct nlattr **attrs, struct seg6_local_lwt *slwt)
+static int parse_nla_iif(struct nlattr **attrs, struct seg6_local_lwt *slwt,
+			 struct netlink_ext_ack *extack)
 {
 	slwt->iif = nla_get_u32(attrs[SEG6_LOCAL_IIF]);
 
@@ -1336,7 +1469,8 @@ static int cmp_nla_iif(struct seg6_local_lwt *a, struct seg6_local_lwt *b)
 	return 0;
 }
 
-static int parse_nla_oif(struct nlattr **attrs, struct seg6_local_lwt *slwt)
+static int parse_nla_oif(struct nlattr **attrs, struct seg6_local_lwt *slwt,
+			 struct netlink_ext_ack *extack)
 {
 	slwt->oif = nla_get_u32(attrs[SEG6_LOCAL_OIF]);
 
@@ -1366,7 +1500,8 @@ static const struct nla_policy bpf_prog_policy[SEG6_LOCAL_BPF_PROG_MAX + 1] = {
 				       .len = MAX_PROG_NAME },
 };
 
-static int parse_nla_bpf(struct nlattr **attrs, struct seg6_local_lwt *slwt)
+static int parse_nla_bpf(struct nlattr **attrs, struct seg6_local_lwt *slwt,
+			 struct netlink_ext_ack *extack)
 {
 	struct nlattr *tb[SEG6_LOCAL_BPF_PROG_MAX + 1];
 	struct bpf_prog *p;
@@ -1444,7 +1579,8 @@ nla_policy seg6_local_counters_policy[SEG6_LOCAL_CNT_MAX + 1] = {
 };
 
 static int parse_nla_counters(struct nlattr **attrs,
-			      struct seg6_local_lwt *slwt)
+			      struct seg6_local_lwt *slwt,
+			      struct netlink_ext_ack *extack)
 {
 	struct pcpu_seg6_local_counters __percpu *pcounters;
 	struct nlattr *tb[SEG6_LOCAL_CNT_MAX + 1];
@@ -1542,8 +1678,195 @@ static void destroy_attr_counters(struct seg6_local_lwt *slwt)
 	free_percpu(slwt->pcpu_counters);
 }
 
+static const
+struct nla_policy seg6_local_flavors_policy[SEG6_LOCAL_FLV_MAX + 1] = {
+	[SEG6_LOCAL_FLV_OPERATION]	= { .type = NLA_U32 },
+	[SEG6_LOCAL_FLV_LCBLOCK_BITS]	= { .type = NLA_U8 },
+	[SEG6_LOCAL_FLV_LCNODE_FN_BITS]	= { .type = NLA_U8 },
+};
+
+/* check whether the lengths of the Locator-Block and Locator-Node Function
+ * are compatible with the dimension of a C-SID container.
+ */
+static int seg6_chk_next_csid_cfg(__u8 block_len, __u8 func_len)
+{
+	/* Locator-Block and Locator-Node Function cannot exceed 128 bits
+	 * (i.e. C-SID container lenghts).
+	 */
+	if (next_csid_chk_cntr_bits(block_len, func_len))
+		return -EINVAL;
+
+	/* Locator-Block length must be greater than zero and evenly divisible
+	 * by 8. There must be room for a Locator-Node Function, at least.
+	 */
+	if (next_csid_chk_lcblock_bits(block_len))
+		return -EINVAL;
+
+	/* Locator-Node Function length must be greater than zero and evenly
+	 * divisible by 8. There must be room for the Locator-Block.
+	 */
+	if (next_csid_chk_lcnode_fn_bits(func_len))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int seg6_parse_nla_next_csid_cfg(struct nlattr **tb,
+					struct seg6_flavors_info *finfo,
+					struct netlink_ext_ack *extack)
+{
+	__u8 func_len = SEG6_LOCAL_LCNODE_FN_DBITS;
+	__u8 block_len = SEG6_LOCAL_LCBLOCK_DBITS;
+	int rc;
+
+	if (tb[SEG6_LOCAL_FLV_LCBLOCK_BITS])
+		block_len = nla_get_u8(tb[SEG6_LOCAL_FLV_LCBLOCK_BITS]);
+
+	if (tb[SEG6_LOCAL_FLV_LCNODE_FN_BITS])
+		func_len = nla_get_u8(tb[SEG6_LOCAL_FLV_LCNODE_FN_BITS]);
+
+	rc = seg6_chk_next_csid_cfg(block_len, func_len);
+	if (rc < 0) {
+		NL_SET_ERR_MSG(extack,
+			       "Invalid Locator Block/Node Function lengths");
+		return rc;
+	}
+
+	finfo->lcblock_bits = block_len;
+	finfo->lcnode_func_bits = func_len;
+
+	return 0;
+}
+
+static int parse_nla_flavors(struct nlattr **attrs, struct seg6_local_lwt *slwt,
+			     struct netlink_ext_ack *extack)
+{
+	struct seg6_flavors_info *finfo = &slwt->flv_info;
+	struct nlattr *tb[SEG6_LOCAL_FLV_MAX + 1];
+	unsigned long fops;
+	int rc;
+
+	rc = nla_parse_nested_deprecated(tb, SEG6_LOCAL_FLV_MAX,
+					 attrs[SEG6_LOCAL_FLAVORS],
+					 seg6_local_flavors_policy, NULL);
+	if (rc < 0)
+		return rc;
+
+	/* this attribute MUST always be present since it represents the Flavor
+	 * operation(s) to be carried out.
+	 */
+	if (!tb[SEG6_LOCAL_FLV_OPERATION])
+		return -EINVAL;
+
+	fops = nla_get_u32(tb[SEG6_LOCAL_FLV_OPERATION]);
+	if (fops & ~SEG6_LOCAL_FLV_SUPP_OPS) {
+		NL_SET_ERR_MSG(extack, "Unsupported Flavor operation(s)");
+		return -EOPNOTSUPP;
+	}
+
+	finfo->flv_ops = fops;
+
+	if (seg6_next_csid_enabled(fops)) {
+		/* Locator-Block and Locator-Node Function lengths can be
+		 * provided by the user space. Otherwise, default values are
+		 * applied.
+		 */
+		rc = seg6_parse_nla_next_csid_cfg(tb, finfo, extack);
+		if (rc < 0)
+			return rc;
+	}
+
+	return 0;
+}
+
+static int seg6_fill_nla_next_csid_cfg(struct sk_buff *skb,
+				       struct seg6_flavors_info *finfo)
+{
+	if (nla_put_u8(skb, SEG6_LOCAL_FLV_LCBLOCK_BITS, finfo->lcblock_bits))
+		return -EMSGSIZE;
+
+	if (nla_put_u8(skb, SEG6_LOCAL_FLV_LCNODE_FN_BITS,
+		       finfo->lcnode_func_bits))
+		return -EMSGSIZE;
+
+	return 0;
+}
+
+static int put_nla_flavors(struct sk_buff *skb, struct seg6_local_lwt *slwt)
+{
+	struct seg6_flavors_info *finfo = &slwt->flv_info;
+	__u32 fops = finfo->flv_ops;
+	struct nlattr *nest;
+	int rc;
+
+	nest = nla_nest_start(skb, SEG6_LOCAL_FLAVORS);
+	if (!nest)
+		return -EMSGSIZE;
+
+	if (nla_put_u32(skb, SEG6_LOCAL_FLV_OPERATION, fops)) {
+		rc = -EMSGSIZE;
+		goto err;
+	}
+
+	if (seg6_next_csid_enabled(fops)) {
+		rc = seg6_fill_nla_next_csid_cfg(skb, finfo);
+		if (rc < 0)
+			goto err;
+	}
+
+	return nla_nest_end(skb, nest);
+
+err:
+	nla_nest_cancel(skb, nest);
+	return rc;
+}
+
+static int seg6_cmp_nla_next_csid_cfg(struct seg6_flavors_info *finfo_a,
+				      struct seg6_flavors_info *finfo_b)
+{
+	if (finfo_a->lcblock_bits != finfo_b->lcblock_bits)
+		return 1;
+
+	if (finfo_a->lcnode_func_bits != finfo_b->lcnode_func_bits)
+		return 1;
+
+	return 0;
+}
+
+static int cmp_nla_flavors(struct seg6_local_lwt *a, struct seg6_local_lwt *b)
+{
+	struct seg6_flavors_info *finfo_a = &a->flv_info;
+	struct seg6_flavors_info *finfo_b = &b->flv_info;
+
+	if (finfo_a->flv_ops != finfo_b->flv_ops)
+		return 1;
+
+	if (seg6_next_csid_enabled(finfo_a->flv_ops)) {
+		if (seg6_cmp_nla_next_csid_cfg(finfo_a, finfo_b))
+			return 1;
+	}
+
+	return 0;
+}
+
+static int encap_size_flavors(struct seg6_local_lwt *slwt)
+{
+	struct seg6_flavors_info *finfo = &slwt->flv_info;
+	int nlsize;
+
+	nlsize = nla_total_size(0) +	/* nest SEG6_LOCAL_FLAVORS */
+		 nla_total_size(4);	/* SEG6_LOCAL_FLV_OPERATION */
+
+	if (seg6_next_csid_enabled(finfo->flv_ops))
+		nlsize += nla_total_size(1) + /* SEG6_LOCAL_FLV_LCBLOCK_BITS */
+			  nla_total_size(1); /* SEG6_LOCAL_FLV_LCNODE_FN_BITS */
+
+	return nlsize;
+}
+
 struct seg6_action_param {
-	int (*parse)(struct nlattr **attrs, struct seg6_local_lwt *slwt);
+	int (*parse)(struct nlattr **attrs, struct seg6_local_lwt *slwt,
+		     struct netlink_ext_ack *extack);
 	int (*put)(struct sk_buff *skb, struct seg6_local_lwt *slwt);
 	int (*cmp)(struct seg6_local_lwt *a, struct seg6_local_lwt *b);
 
@@ -1593,6 +1916,10 @@ static struct seg6_action_param seg6_action_params[SEG6_LOCAL_MAX + 1] = {
 				    .put = put_nla_counters,
 				    .cmp = cmp_nla_counters,
 				    .destroy = destroy_attr_counters },
+
+	[SEG6_LOCAL_FLAVORS]	= { .parse = parse_nla_flavors,
+				    .put = put_nla_flavors,
+				    .cmp = cmp_nla_flavors },
 };
 
 /* call the destroy() callback (if available) for each set attribute in
@@ -1636,7 +1963,8 @@ static void destroy_attrs(struct seg6_local_lwt *slwt)
 }
 
 static int parse_nla_optional_attrs(struct nlattr **attrs,
-				    struct seg6_local_lwt *slwt)
+				    struct seg6_local_lwt *slwt,
+				    struct netlink_ext_ack *extack)
 {
 	struct seg6_action_desc *desc = slwt->desc;
 	unsigned long parsed_optattrs = 0;
@@ -1652,7 +1980,7 @@ static int parse_nla_optional_attrs(struct nlattr **attrs,
 		 */
 		param = &seg6_action_params[i];
 
-		err = param->parse(attrs, slwt);
+		err = param->parse(attrs, slwt, extack);
 		if (err < 0)
 			goto parse_optattrs_err;
 
@@ -1705,7 +2033,8 @@ static void seg6_local_lwtunnel_destroy_state(struct seg6_local_lwt *slwt)
 	ops->destroy_state(slwt);
 }
 
-static int parse_nla_action(struct nlattr **attrs, struct seg6_local_lwt *slwt)
+static int parse_nla_action(struct nlattr **attrs, struct seg6_local_lwt *slwt,
+			    struct netlink_ext_ack *extack)
 {
 	struct seg6_action_param *param;
 	struct seg6_action_desc *desc;
@@ -1749,14 +2078,14 @@ static int parse_nla_action(struct nlattr **attrs, struct seg6_local_lwt *slwt)
 
 			param = &seg6_action_params[i];
 
-			err = param->parse(attrs, slwt);
+			err = param->parse(attrs, slwt, extack);
 			if (err < 0)
 				goto parse_attrs_err;
 		}
 	}
 
 	/* parse the optional attributes, if any */
-	err = parse_nla_optional_attrs(attrs, slwt);
+	err = parse_nla_optional_attrs(attrs, slwt, extack);
 	if (err < 0)
 		goto parse_attrs_err;
 
@@ -1800,7 +2129,7 @@ static int seg6_local_build_state(struct net *net, struct nlattr *nla,
 	slwt = seg6_local_lwtunnel(newts);
 	slwt->action = nla_get_u32(tb[SEG6_LOCAL_ACTION]);
 
-	err = parse_nla_action(tb, slwt);
+	err = parse_nla_action(tb, slwt, extack);
 	if (err < 0)
 		goto out_free;
 
@@ -1904,6 +2233,9 @@ static int seg6_local_get_encap_size(struct lwtunnel_state *lwt)
 			  /* SEG6_LOCAL_CNT_ERRORS */
 			  nla_total_size_64bit(sizeof(__u64));
 
+	if (attrs & SEG6_F_ATTR(SEG6_LOCAL_FLAVORS))
+		nlsize += encap_size_flavors(slwt);
+
 	return nlsize;
 }
 
@@ -1959,6 +2291,15 @@ int __init seg6_local_init(void)
 	 */
 	BUILD_BUG_ON(SEG6_LOCAL_MAX + 1 > BITS_PER_TYPE(unsigned long));
 
+	/* If the default NEXT-C-SID Locator-Block/Node Function lengths (in
+	 * bits) have been changed with invalid values, kernel build stops
+	 * here.
+	 */
+	BUILD_BUG_ON(next_csid_chk_cntr_bits(SEG6_LOCAL_LCBLOCK_DBITS,
+					     SEG6_LOCAL_LCNODE_FN_DBITS));
+	BUILD_BUG_ON(next_csid_chk_lcblock_bits(SEG6_LOCAL_LCBLOCK_DBITS));
+	BUILD_BUG_ON(next_csid_chk_lcnode_fn_bits(SEG6_LOCAL_LCNODE_FN_DBITS));
+
 	return lwtunnel_encap_add_ops(&seg6_local_ops,
 				      LWTUNNEL_ENCAP_SEG6_LOCAL);
 }
diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c
index 6b73b7a5f175..d27683e3fc97 100644
--- a/net/ipv6/sit.c
+++ b/net/ipv6/sit.c
@@ -254,7 +254,7 @@ static struct ip_tunnel *ipip6_tunnel_locate(struct net *net,
 	if (parms->name[0]) {
 		if (!dev_valid_name(parms->name))
 			goto failed;
-		strlcpy(name, parms->name, IFNAMSIZ);
+		strscpy(name, parms->name, IFNAMSIZ);
 	} else {
 		strcpy(name, "sit%d");
 	}
@@ -1503,71 +1503,12 @@ static void ipip6_netlink_parms(struct nlattr *data[],
 	if (!data)
 		return;
 
-	if (data[IFLA_IPTUN_LINK])
-		parms->link = nla_get_u32(data[IFLA_IPTUN_LINK]);
-
-	if (data[IFLA_IPTUN_LOCAL])
-		parms->iph.saddr = nla_get_be32(data[IFLA_IPTUN_LOCAL]);
-
-	if (data[IFLA_IPTUN_REMOTE])
-		parms->iph.daddr = nla_get_be32(data[IFLA_IPTUN_REMOTE]);
-
-	if (data[IFLA_IPTUN_TTL]) {
-		parms->iph.ttl = nla_get_u8(data[IFLA_IPTUN_TTL]);
-		if (parms->iph.ttl)
-			parms->iph.frag_off = htons(IP_DF);
-	}
-
-	if (data[IFLA_IPTUN_TOS])
-		parms->iph.tos = nla_get_u8(data[IFLA_IPTUN_TOS]);
-
-	if (!data[IFLA_IPTUN_PMTUDISC] || nla_get_u8(data[IFLA_IPTUN_PMTUDISC]))
-		parms->iph.frag_off = htons(IP_DF);
-
-	if (data[IFLA_IPTUN_FLAGS])
-		parms->i_flags = nla_get_be16(data[IFLA_IPTUN_FLAGS]);
-
-	if (data[IFLA_IPTUN_PROTO])
-		parms->iph.protocol = nla_get_u8(data[IFLA_IPTUN_PROTO]);
+	ip_tunnel_netlink_parms(data, parms);
 
 	if (data[IFLA_IPTUN_FWMARK])
 		*fwmark = nla_get_u32(data[IFLA_IPTUN_FWMARK]);
 }
 
-/* This function returns true when ENCAP attributes are present in the nl msg */
-static bool ipip6_netlink_encap_parms(struct nlattr *data[],
-				      struct ip_tunnel_encap *ipencap)
-{
-	bool ret = false;
-
-	memset(ipencap, 0, sizeof(*ipencap));
-
-	if (!data)
-		return ret;
-
-	if (data[IFLA_IPTUN_ENCAP_TYPE]) {
-		ret = true;
-		ipencap->type = nla_get_u16(data[IFLA_IPTUN_ENCAP_TYPE]);
-	}
-
-	if (data[IFLA_IPTUN_ENCAP_FLAGS]) {
-		ret = true;
-		ipencap->flags = nla_get_u16(data[IFLA_IPTUN_ENCAP_FLAGS]);
-	}
-
-	if (data[IFLA_IPTUN_ENCAP_SPORT]) {
-		ret = true;
-		ipencap->sport = nla_get_be16(data[IFLA_IPTUN_ENCAP_SPORT]);
-	}
-
-	if (data[IFLA_IPTUN_ENCAP_DPORT]) {
-		ret = true;
-		ipencap->dport = nla_get_be16(data[IFLA_IPTUN_ENCAP_DPORT]);
-	}
-
-	return ret;
-}
-
 #ifdef CONFIG_IPV6_SIT_6RD
 /* This function returns true when 6RD attributes are present in the nl msg */
 static bool ipip6_netlink_6rd_parms(struct nlattr *data[],
@@ -1619,7 +1560,7 @@ static int ipip6_newlink(struct net *src_net, struct net_device *dev,
 
 	nt = netdev_priv(dev);
 
-	if (ipip6_netlink_encap_parms(data, &ipencap)) {
+	if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
 		err = ip_tunnel_encap_setup(nt, &ipencap);
 		if (err < 0)
 			return err;
@@ -1671,7 +1612,7 @@ static int ipip6_changelink(struct net_device *dev, struct nlattr *tb[],
 	if (dev == sitn->fb_tunnel_dev)
 		return -EINVAL;
 
-	if (ipip6_netlink_encap_parms(data, &ipencap)) {
+	if (ip_tunnel_netlink_encap_parms(data, &ipencap)) {
 		err = ip_tunnel_encap_setup(t, &ipencap);
 		if (err < 0)
 			return err;
diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index e54eee80ce5f..a8adda623da1 100644
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -146,15 +146,16 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 			  int addr_len)
 {
 	struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
-	struct inet_sock *inet = inet_sk(sk);
 	struct inet_connection_sock *icsk = inet_csk(sk);
+	struct in6_addr *saddr = NULL, *final_p, final;
 	struct inet_timewait_death_row *tcp_death_row;
 	struct ipv6_pinfo *np = tcp_inet6_sk(sk);
+	struct inet_sock *inet = inet_sk(sk);
 	struct tcp_sock *tp = tcp_sk(sk);
-	struct in6_addr *saddr = NULL, *final_p, final;
+	struct net *net = sock_net(sk);
 	struct ipv6_txoptions *opt;
-	struct flowi6 fl6;
 	struct dst_entry *dst;
+	struct flowi6 fl6;
 	int addr_type;
 	int err;
 
@@ -280,15 +281,33 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 
 	security_sk_classify_flow(sk, flowi6_to_flowi_common(&fl6));
 
-	dst = ip6_dst_lookup_flow(sock_net(sk), sk, &fl6, final_p);
+	dst = ip6_dst_lookup_flow(net, sk, &fl6, final_p);
 	if (IS_ERR(dst)) {
 		err = PTR_ERR(dst);
 		goto failure;
 	}
 
+	tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
+
 	if (!saddr) {
+		struct inet_bind_hashbucket *prev_addr_hashbucket = NULL;
+		struct in6_addr prev_v6_rcv_saddr;
+
+		if (icsk->icsk_bind2_hash) {
+			prev_addr_hashbucket = inet_bhashfn_portaddr(tcp_death_row->hashinfo,
+								     sk, net, inet->inet_num);
+			prev_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
+		}
 		saddr = &fl6.saddr;
 		sk->sk_v6_rcv_saddr = *saddr;
+
+		if (prev_addr_hashbucket) {
+			err = inet_bhash2_update_saddr(prev_addr_hashbucket, sk);
+			if (err) {
+				sk->sk_v6_rcv_saddr = prev_v6_rcv_saddr;
+				goto failure;
+			}
+		}
 	}
 
 	/* set the source address */
@@ -308,7 +327,6 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 	inet->inet_dport = usin->sin6_port;
 
 	tcp_set_state(sk, TCP_SYN_SENT);
-	tcp_death_row = sock_net(sk)->ipv4.tcp_death_row;
 	err = inet6_hash_connect(tcp_death_row, sk);
 	if (err)
 		goto late_failure;
@@ -322,8 +340,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
 						    sk->sk_v6_daddr.s6_addr32,
 						    inet->inet_sport,
 						    inet->inet_dport));
-		tp->tsoffset = secure_tcpv6_ts_off(sock_net(sk),
-						   np->saddr.s6_addr32,
+		tp->tsoffset = secure_tcpv6_ts_off(net, np->saddr.s6_addr32,
 						   sk->sk_v6_daddr.s6_addr32);
 	}
 
@@ -386,7 +403,7 @@ static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 	bool fatal;
 	int err;
 
-	sk = __inet6_lookup_established(net, &tcp_hashinfo,
+	sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 					&hdr->daddr, th->dest,
 					&hdr->saddr, ntohs(th->source),
 					skb->dev->ifindex, inet6_sdif(skb));
@@ -841,7 +858,7 @@ const struct tcp_request_sock_ops tcp_request_sock_ipv6_ops = {
 static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32 seq,
 				 u32 ack, u32 win, u32 tsval, u32 tsecr,
 				 int oif, struct tcp_md5sig_key *key, int rst,
-				 u8 tclass, __be32 label, u32 priority)
+				 u8 tclass, __be32 label, u32 priority, u32 txhash)
 {
 	const struct tcphdr *th = tcp_hdr(skb);
 	struct tcphdr *t1;
@@ -932,16 +949,16 @@ static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32
 	}
 
 	if (sk) {
-		if (sk->sk_state == TCP_TIME_WAIT) {
+		if (sk->sk_state == TCP_TIME_WAIT)
 			mark = inet_twsk(sk)->tw_mark;
-			/* autoflowlabel relies on buff->hash */
-			skb_set_hash(buff, inet_twsk(sk)->tw_txhash,
-				     PKT_HASH_TYPE_L4);
-		} else {
+		else
 			mark = sk->sk_mark;
-		}
 		skb_set_delivery_time(buff, tcp_transmit_time(sk), true);
 	}
+	if (txhash) {
+		/* autoflowlabel/skb_get_hash_flowi6 rely on buff->hash */
+		skb_set_hash(buff, txhash, PKT_HASH_TYPE_L4);
+	}
 	fl6.flowi6_mark = IP6_REPLY_MARK(net, skb->mark) ?: mark;
 	fl6.fl6_dport = t1->dest;
 	fl6.fl6_sport = t1->source;
@@ -984,6 +1001,7 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
 	__be32 label = 0;
 	u32 priority = 0;
 	struct net *net;
+	u32 txhash = 0;
 	int oif = 0;
 
 	if (th->rst)
@@ -1019,11 +1037,10 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
 		 * Incoming packet is checked with md5 hash with finding key,
 		 * no RST generated if md5 hash doesn't match.
 		 */
-		sk1 = inet6_lookup_listener(net,
-					   &tcp_hashinfo, NULL, 0,
-					   &ipv6h->saddr,
-					   th->source, &ipv6h->daddr,
-					   ntohs(th->source), dif, sdif);
+		sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
+					    NULL, 0, &ipv6h->saddr, th->source,
+					    &ipv6h->daddr, ntohs(th->source),
+					    dif, sdif);
 		if (!sk1)
 			goto out;
 
@@ -1057,10 +1074,12 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
 			if (np->repflow)
 				label = ip6_flowlabel(ipv6h);
 			priority = sk->sk_priority;
+			txhash = sk->sk_hash;
 		}
 		if (sk->sk_state == TCP_TIME_WAIT) {
 			label = cpu_to_be32(inet_twsk(sk)->tw_flowlabel);
 			priority = inet_twsk(sk)->tw_priority;
+			txhash = inet_twsk(sk)->tw_txhash;
 		}
 	} else {
 		if (net->ipv6.sysctl.flowlabel_reflect & FLOWLABEL_REFLECT_TCP_RESET)
@@ -1068,7 +1087,7 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
 	}
 
 	tcp_v6_send_response(sk, skb, seq, ack_seq, 0, 0, 0, oif, key, 1,
-			     ipv6_get_dsfield(ipv6h), label, priority);
+			     ipv6_get_dsfield(ipv6h), label, priority, txhash);
 
 #ifdef CONFIG_TCP_MD5SIG
 out:
@@ -1079,10 +1098,10 @@ out:
 static void tcp_v6_send_ack(const struct sock *sk, struct sk_buff *skb, u32 seq,
 			    u32 ack, u32 win, u32 tsval, u32 tsecr, int oif,
 			    struct tcp_md5sig_key *key, u8 tclass,
-			    __be32 label, u32 priority)
+			    __be32 label, u32 priority, u32 txhash)
 {
 	tcp_v6_send_response(sk, skb, seq, ack, win, tsval, tsecr, oif, key, 0,
-			     tclass, label, priority);
+			     tclass, label, priority, txhash);
 }
 
 static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb)
@@ -1094,7 +1113,8 @@ static void tcp_v6_timewait_ack(struct sock *sk, struct sk_buff *skb)
 			tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
 			tcp_time_stamp_raw() + tcptw->tw_ts_offset,
 			tcptw->tw_ts_recent, tw->tw_bound_dev_if, tcp_twsk_md5_key(tcptw),
-			tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel), tw->tw_priority);
+			tw->tw_tclass, cpu_to_be32(tw->tw_flowlabel), tw->tw_priority,
+			tw->tw_txhash);
 
 	inet_twsk_put(tw);
 }
@@ -1121,7 +1141,8 @@ static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
 			tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
 			req->ts_recent, sk->sk_bound_dev_if,
 			tcp_v6_md5_do_lookup(sk, &ipv6_hdr(skb)->saddr, l3index),
-			ipv6_get_dsfield(ipv6_hdr(skb)), 0, sk->sk_priority);
+			ipv6_get_dsfield(ipv6_hdr(skb)), 0, sk->sk_priority,
+			tcp_rsk(req)->txhash);
 }
 
 
@@ -1619,7 +1640,7 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
 	hdr = ipv6_hdr(skb);
 
 lookup:
-	sk = __inet6_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th),
+	sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th),
 				th->source, th->dest, inet6_iif(skb), sdif,
 				&refcounted);
 	if (!sk)
@@ -1794,7 +1815,7 @@ do_time_wait:
 	{
 		struct sock *sk2;
 
-		sk2 = inet6_lookup_listener(dev_net(skb->dev), &tcp_hashinfo,
+		sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
 					    skb, __tcp_hdrlen(th),
 					    &ipv6_hdr(skb)->saddr, th->source,
 					    &ipv6_hdr(skb)->daddr,
@@ -1827,6 +1848,7 @@ do_time_wait:
 
 void tcp_v6_early_demux(struct sk_buff *skb)
 {
+	struct net *net = dev_net(skb->dev);
 	const struct ipv6hdr *hdr;
 	const struct tcphdr *th;
 	struct sock *sk;
@@ -1844,7 +1866,7 @@ void tcp_v6_early_demux(struct sk_buff *skb)
 		return;
 
 	/* Note : We use inet6_iif() here, not tcp_v6_iif() */
-	sk = __inet6_lookup_established(dev_net(skb->dev), &tcp_hashinfo,
+	sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
 					&hdr->saddr, th->source,
 					&hdr->daddr, ntohs(th->dest),
 					inet6_iif(skb), inet6_sdif(skb));
@@ -2176,7 +2198,7 @@ struct proto tcpv6_prot = {
 	.slab_flags		= SLAB_TYPESAFE_BY_RCU,
 	.twsk_prot		= &tcp6_timewait_sock_ops,
 	.rsk_prot		= &tcp6_request_sock_ops,
-	.h.hashinfo		= &tcp_hashinfo,
+	.h.hashinfo		= NULL,
 	.no_autobind		= true,
 	.diag_destroy		= tcp_abort,
 };
@@ -2210,7 +2232,7 @@ static void __net_exit tcpv6_net_exit(struct net *net)
 
 static void __net_exit tcpv6_net_exit_batch(struct list_head *net_exit_list)
 {
-	inet_twsk_purge(&tcp_hashinfo, AF_INET6);
+	tcp_twsk_purge(net_exit_list, AF_INET6);
 }
 
 static struct pernet_operations tcpv6_net_ops = {
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 3366d6a77ff2..91e795bb9ade 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -650,16 +650,20 @@ static int __udpv6_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
 	rc = __udp_enqueue_schedule_skb(sk, skb);
 	if (rc < 0) {
 		int is_udplite = IS_UDPLITE(sk);
+		enum skb_drop_reason drop_reason;
 
 		/* Note that an ENOMEM error is charged twice */
-		if (rc == -ENOMEM)
+		if (rc == -ENOMEM) {
 			UDP6_INC_STATS(sock_net(sk),
 					 UDP_MIB_RCVBUFERRORS, is_udplite);
-		else
+			drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF;
+		} else {
 			UDP6_INC_STATS(sock_net(sk),
 				       UDP_MIB_MEMERRORS, is_udplite);
+			drop_reason = SKB_DROP_REASON_PROTO_MEM;
+		}
 		UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
-		kfree_skb(skb);
+		kfree_skb_reason(skb, drop_reason);
 		return -1;
 	}
 
@@ -675,11 +679,14 @@ static __inline__ int udpv6_err(struct sk_buff *skb,
 
 static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
 {
+	enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
 	struct udp_sock *up = udp_sk(sk);
 	int is_udplite = IS_UDPLITE(sk);
 
-	if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb))
+	if (!xfrm6_policy_check(sk, XFRM_POLICY_IN, skb)) {
+		drop_reason = SKB_DROP_REASON_XFRM_POLICY;
 		goto drop;
+	}
 
 	if (static_branch_unlikely(&udpv6_encap_needed_key) && up->encap_type) {
 		int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
@@ -738,8 +745,10 @@ static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
 	    udp_lib_checksum_complete(skb))
 		goto csum_error;
 
-	if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr)))
+	if (sk_filter_trim_cap(sk, skb, sizeof(struct udphdr))) {
+		drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
 		goto drop;
+	}
 
 	udp_csum_pull_header(skb);
 
@@ -748,11 +757,12 @@ static int udpv6_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
 	return __udpv6_queue_rcv_skb(sk, skb);
 
 csum_error:
+	drop_reason = SKB_DROP_REASON_UDP_CSUM;
 	__UDP6_INC_STATS(sock_net(sk), UDP_MIB_CSUMERRORS, is_udplite);
 drop:
 	__UDP6_INC_STATS(sock_net(sk), UDP_MIB_INERRORS, is_udplite);
 	atomic_inc(&sk->sk_drops);
-	kfree_skb(skb);
+	kfree_skb_reason(skb, drop_reason);
 	return -1;
 }
 
diff --git a/net/ipv6/xfrm6_tunnel.c b/net/ipv6/xfrm6_tunnel.c
index 2b31112c0856..1323f2f6928e 100644
--- a/net/ipv6/xfrm6_tunnel.c
+++ b/net/ipv6/xfrm6_tunnel.c
@@ -270,13 +270,17 @@ static int xfrm6_tunnel_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
 	return 0;
 }
 
-static int xfrm6_tunnel_init_state(struct xfrm_state *x)
+static int xfrm6_tunnel_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
-	if (x->props.mode != XFRM_MODE_TUNNEL)
+	if (x->props.mode != XFRM_MODE_TUNNEL) {
+		NL_SET_ERR_MSG(extack, "IPv6 tunnel can only be used with tunnel mode");
 		return -EINVAL;
+	}
 
-	if (x->encap)
+	if (x->encap) {
+		NL_SET_ERR_MSG(extack, "IPv6 tunnel is not compatible with encapsulation");
 		return -EINVAL;
+	}
 
 	x->props.header_len = sizeof(struct ipv6hdr);
 
diff --git a/net/l2tp/l2tp_eth.c b/net/l2tp/l2tp_eth.c
index 6cd97c75445c..f2ae03c40473 100644
--- a/net/l2tp/l2tp_eth.c
+++ b/net/l2tp/l2tp_eth.c
@@ -254,7 +254,7 @@ static int l2tp_eth_create(struct net *net, struct l2tp_tunnel *tunnel,
 	int rc;
 
 	if (cfg->ifname) {
-		strlcpy(name, cfg->ifname, IFNAMSIZ);
+		strscpy(name, cfg->ifname, IFNAMSIZ);
 		name_assign_type = NET_NAME_USER;
 	} else {
 		strcpy(name, L2TP_ETH_DEV_NAME);
@@ -314,7 +314,7 @@ static int l2tp_eth_create(struct net *net, struct l2tp_tunnel *tunnel,
 		return rc;
 	}
 
-	strlcpy(session->ifname, dev->name, IFNAMSIZ);
+	strscpy(session->ifname, dev->name, IFNAMSIZ);
 	rcu_assign_pointer(spriv->dev, dev);
 
 	rtnl_unlock();
diff --git a/net/l2tp/l2tp_netlink.c b/net/l2tp/l2tp_netlink.c
index 96eb91be9238..a901fd14fe3b 100644
--- a/net/l2tp/l2tp_netlink.c
+++ b/net/l2tp/l2tp_netlink.c
@@ -989,6 +989,7 @@ static struct genl_family l2tp_nl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= l2tp_nl_ops,
 	.n_small_ops	= ARRAY_SIZE(l2tp_nl_ops),
+	.resv_start_op	= L2TP_CMD_SESSION_GET + 1,
 	.mcgrps		= l2tp_multicast_group,
 	.n_mcgrps	= ARRAY_SIZE(l2tp_multicast_group),
 };
diff --git a/net/mac80211/Makefile b/net/mac80211/Makefile
index af1df3a6bd55..b8de44da1fb8 100644
--- a/net/mac80211/Makefile
+++ b/net/mac80211/Makefile
@@ -16,6 +16,7 @@ mac80211-y := \
 	s1g.o \
 	ibss.o \
 	iface.o \
+	link.o \
 	rate.o \
 	michael.o \
 	tkip.o \
diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c
index a4f6971b7a19..687b4c878d4a 100644
--- a/net/mac80211/cfg.c
+++ b/net/mac80211/cfg.c
@@ -23,6 +23,30 @@
 #include "mesh.h"
 #include "wme.h"
 
+static struct ieee80211_link_data *
+ieee80211_link_or_deflink(struct ieee80211_sub_if_data *sdata, int link_id,
+			  bool require_valid)
+{
+	struct ieee80211_link_data *link;
+
+	if (link_id < 0) {
+		/*
+		 * For keys, if sdata is not an MLD, we might not use
+		 * the return value at all (if it's not a pairwise key),
+		 * so in that case (require_valid==false) don't error.
+		 */
+		if (require_valid && sdata->vif.valid_links)
+			return ERR_PTR(-EINVAL);
+
+		return &sdata->deflink;
+	}
+
+	link = sdata_dereference(sdata->link[link_id], sdata);
+	if (!link)
+		return ERR_PTR(-ENOLINK);
+	return link;
+}
+
 static void ieee80211_set_mu_mimo_follow(struct ieee80211_sub_if_data *sdata,
 					 struct vif_params *params)
 {
@@ -202,6 +226,10 @@ static int ieee80211_change_iface(struct wiphy *wiphy,
 		if (params->use_4addr == ifmgd->use_4addr)
 			return 0;
 
+		/* FIXME: no support for 4-addr MLO yet */
+		if (sdata->vif.valid_links)
+			return -EOPNOTSUPP;
+
 		sdata->u.mgd.use_4addr = params->use_4addr;
 		if (!ifmgd->associated)
 			return 0;
@@ -434,10 +462,12 @@ static int ieee80211_set_tx(struct ieee80211_sub_if_data *sdata,
 }
 
 static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev,
-			     u8 key_idx, bool pairwise, const u8 *mac_addr,
-			     struct key_params *params)
+			     int link_id, u8 key_idx, bool pairwise,
+			     const u8 *mac_addr, struct key_params *params)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
+	struct ieee80211_link_data *link =
+		ieee80211_link_or_deflink(sdata, link_id, false);
 	struct ieee80211_local *local = sdata->local;
 	struct sta_info *sta = NULL;
 	struct ieee80211_key *key;
@@ -446,6 +476,9 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev,
 	if (!ieee80211_sdata_running(sdata))
 		return -ENETDOWN;
 
+	if (IS_ERR(link))
+		return PTR_ERR(link);
+
 	if (pairwise && params->mode == NL80211_KEY_SET_TX)
 		return ieee80211_set_tx(sdata, mac_addr, key_idx);
 
@@ -454,6 +487,8 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev,
 	case WLAN_CIPHER_SUITE_WEP40:
 	case WLAN_CIPHER_SUITE_TKIP:
 	case WLAN_CIPHER_SUITE_WEP104:
+		if (link_id >= 0)
+			return -EINVAL;
 		if (WARN_ON_ONCE(fips_enabled))
 			return -EINVAL;
 		break;
@@ -466,6 +501,8 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev,
 	if (IS_ERR(key))
 		return PTR_ERR(key);
 
+	key->conf.link_id = link_id;
+
 	if (pairwise)
 		key->conf.flags |= IEEE80211_KEY_FLAG_PAIRWISE;
 
@@ -527,7 +564,7 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev,
 		break;
 	}
 
-	err = ieee80211_key_link(key, sdata, sta);
+	err = ieee80211_key_link(key, link, sta);
 
  out_unlock:
 	mutex_unlock(&local->sta_mtx);
@@ -536,18 +573,37 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static struct ieee80211_key *
-ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata,
+ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata, int link_id,
 		     u8 key_idx, bool pairwise, const u8 *mac_addr)
 {
 	struct ieee80211_local *local = sdata->local;
+	struct ieee80211_link_data *link = &sdata->deflink;
 	struct ieee80211_key *key;
-	struct sta_info *sta;
+
+	if (link_id >= 0) {
+		link = rcu_dereference_check(sdata->link[link_id],
+					     lockdep_is_held(&sdata->wdev.mtx));
+		if (!link)
+			return NULL;
+	}
 
 	if (mac_addr) {
+		struct sta_info *sta;
+		struct link_sta_info *link_sta;
+
 		sta = sta_info_get_bss(sdata, mac_addr);
 		if (!sta)
 			return NULL;
 
+		if (link_id >= 0) {
+			link_sta = rcu_dereference_check(sta->link[link_id],
+							 lockdep_is_held(&local->sta_mtx));
+			if (!link_sta)
+				return NULL;
+		} else {
+			link_sta = &sta->deflink;
+		}
+
 		if (pairwise && key_idx < NUM_DEFAULT_KEYS)
 			return rcu_dereference_check_key_mtx(local,
 							     sta->ptk[key_idx]);
@@ -557,7 +613,7 @@ ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata,
 			      NUM_DEFAULT_MGMT_KEYS +
 			      NUM_DEFAULT_BEACON_KEYS)
 			return rcu_dereference_check_key_mtx(local,
-							     sta->deflink.gtk[key_idx]);
+							     link_sta->gtk[key_idx]);
 
 		return NULL;
 	}
@@ -566,7 +622,7 @@ ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata,
 		return rcu_dereference_check_key_mtx(local,
 						     sdata->keys[key_idx]);
 
-	key = rcu_dereference_check_key_mtx(local, sdata->deflink.gtk[key_idx]);
+	key = rcu_dereference_check_key_mtx(local, link->gtk[key_idx]);
 	if (key)
 		return key;
 
@@ -578,7 +634,8 @@ ieee80211_lookup_key(struct ieee80211_sub_if_data *sdata,
 }
 
 static int ieee80211_del_key(struct wiphy *wiphy, struct net_device *dev,
-			     u8 key_idx, bool pairwise, const u8 *mac_addr)
+			     int link_id, u8 key_idx, bool pairwise,
+			     const u8 *mac_addr)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
 	struct ieee80211_local *local = sdata->local;
@@ -588,7 +645,7 @@ static int ieee80211_del_key(struct wiphy *wiphy, struct net_device *dev,
 	mutex_lock(&local->sta_mtx);
 	mutex_lock(&local->key_mtx);
 
-	key = ieee80211_lookup_key(sdata, key_idx, pairwise, mac_addr);
+	key = ieee80211_lookup_key(sdata, link_id, key_idx, pairwise, mac_addr);
 	if (!key) {
 		ret = -ENOENT;
 		goto out_unlock;
@@ -605,8 +662,8 @@ static int ieee80211_del_key(struct wiphy *wiphy, struct net_device *dev,
 }
 
 static int ieee80211_get_key(struct wiphy *wiphy, struct net_device *dev,
-			     u8 key_idx, bool pairwise, const u8 *mac_addr,
-			     void *cookie,
+			     int link_id, u8 key_idx, bool pairwise,
+			     const u8 *mac_addr, void *cookie,
 			     void (*callback)(void *cookie,
 					      struct key_params *params))
 {
@@ -624,7 +681,7 @@ static int ieee80211_get_key(struct wiphy *wiphy, struct net_device *dev,
 
 	rcu_read_lock();
 
-	key = ieee80211_lookup_key(sdata, key_idx, pairwise, mac_addr);
+	key = ieee80211_lookup_key(sdata, link_id, key_idx, pairwise, mac_addr);
 	if (!key)
 		goto out;
 
@@ -711,34 +768,49 @@ static int ieee80211_get_key(struct wiphy *wiphy, struct net_device *dev,
 
 static int ieee80211_config_default_key(struct wiphy *wiphy,
 					struct net_device *dev,
-					u8 key_idx, bool uni,
+					int link_id, u8 key_idx, bool uni,
 					bool multi)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
+	struct ieee80211_link_data *link =
+		ieee80211_link_or_deflink(sdata, link_id, false);
 
-	ieee80211_set_default_key(sdata, key_idx, uni, multi);
+	if (IS_ERR(link))
+		return PTR_ERR(link);
+
+	ieee80211_set_default_key(link, key_idx, uni, multi);
 
 	return 0;
 }
 
 static int ieee80211_config_default_mgmt_key(struct wiphy *wiphy,
 					     struct net_device *dev,
-					     u8 key_idx)
+					     int link_id, u8 key_idx)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
+	struct ieee80211_link_data *link =
+		ieee80211_link_or_deflink(sdata, link_id, true);
+
+	if (IS_ERR(link))
+		return PTR_ERR(link);
 
-	ieee80211_set_default_mgmt_key(sdata, key_idx);
+	ieee80211_set_default_mgmt_key(link, key_idx);
 
 	return 0;
 }
 
 static int ieee80211_config_default_beacon_key(struct wiphy *wiphy,
 					       struct net_device *dev,
-					       u8 key_idx)
+					       int link_id, u8 key_idx)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
+	struct ieee80211_link_data *link =
+		ieee80211_link_or_deflink(sdata, link_id, true);
+
+	if (IS_ERR(link))
+		return PTR_ERR(link);
 
-	ieee80211_set_default_beacon_key(sdata, key_idx);
+	ieee80211_set_default_beacon_key(link, key_idx);
 
 	return 0;
 }
@@ -1610,6 +1682,18 @@ static int sta_link_apply_parameters(struct ieee80211_local *local,
 		rcu_dereference_protected(sta->link[link_id],
 					  lockdep_is_held(&local->sta_mtx));
 
+	/*
+	 * If there are no changes, then accept a link that doesn't exist,
+	 * unless it's a new link.
+	 */
+	if (params->link_id < 0 && !new_link &&
+	    !params->link_mac && !params->txpwr_set &&
+	    !params->supported_rates_len &&
+	    !params->ht_capa && !params->vht_capa &&
+	    !params->he_capa && !params->eht_capa &&
+	    !params->opmode_notif_used)
+		return 0;
+
 	if (!link || !link_sta)
 		return -EINVAL;
 
@@ -1625,6 +1709,8 @@ static int sta_link_apply_parameters(struct ieee80211_local *local,
 					     params->link_mac)) {
 			return -EINVAL;
 		}
+	} else if (new_link) {
+		return -EINVAL;
 	}
 
 	if (params->txpwr_set) {
@@ -2554,7 +2640,8 @@ static int ieee80211_set_txq_params(struct wiphy *wiphy,
 {
 	struct ieee80211_local *local = wiphy_priv(wiphy);
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
-	struct ieee80211_link_data *link = &sdata->deflink;
+	struct ieee80211_link_data *link =
+		ieee80211_link_or_deflink(sdata, params->link_id, true);
 	struct ieee80211_tx_queue_params p;
 
 	if (!local->ops->conf_tx)
@@ -2563,6 +2650,9 @@ static int ieee80211_set_txq_params(struct wiphy *wiphy,
 	if (local->hw.queues < IEEE80211_NUM_ACS)
 		return -EOPNOTSUPP;
 
+	if (IS_ERR(link))
+		return PTR_ERR(link);
+
 	memset(&p, 0, sizeof(p));
 	p.aifs = params->aifs;
 	p.cw_max = params->cwmax;
@@ -3597,9 +3687,6 @@ static int ieee80211_set_csa_beacon(struct ieee80211_sub_if_data *sdata,
 	case NL80211_IFTYPE_MESH_POINT: {
 		struct ieee80211_if_mesh *ifmsh = &sdata->u.mesh;
 
-		if (params->chandef.width != sdata->vif.bss_conf.chandef.width)
-			return -EINVAL;
-
 		/* changes into another band are not supported */
 		if (sdata->vif.bss_conf.chandef.chan->band !=
 		    params->chandef.chan->band)
@@ -3732,7 +3819,7 @@ __ieee80211_channel_switch(struct wiphy *wiphy, struct net_device *dev,
 					  IEEE80211_QUEUE_STOP_REASON_CSA);
 
 	cfg80211_ch_switch_started_notify(sdata->dev,
-					  &sdata->deflink.csa_chandef,
+					  &sdata->deflink.csa_chandef, 0,
 					  params->count, params->block_tx);
 
 	if (changed) {
@@ -4614,6 +4701,9 @@ static int ieee80211_add_intf_link(struct wiphy *wiphy,
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_WDEV_TO_SUB_IF(wdev);
 
+	if (wdev->use_4addr)
+		return -EOPNOTSUPP;
+
 	return ieee80211_vif_set_links(sdata, wdev->valid_links);
 }
 
diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c
index f247daa41563..e72cf0749d49 100644
--- a/net/mac80211/chan.c
+++ b/net/mac80211/chan.c
@@ -1799,6 +1799,12 @@ int ieee80211_link_use_channel(struct ieee80211_link_data *link,
 
 	lockdep_assert_held(&local->mtx);
 
+	if (sdata->vif.active_links &&
+	    !(sdata->vif.active_links & BIT(link->link_id))) {
+		ieee80211_link_update_chandef(link, chandef);
+		return 0;
+	}
+
 	mutex_lock(&local->chanctx_mtx);
 
 	ret = cfg80211_chandef_dfs_required(local->hw.wiphy,
diff --git a/net/mac80211/debugfs_netdev.c b/net/mac80211/debugfs_netdev.c
index 1e5b041a5cea..5b014786fd2d 100644
--- a/net/mac80211/debugfs_netdev.c
+++ b/net/mac80211/debugfs_netdev.c
@@ -570,6 +570,30 @@ static ssize_t ieee80211_if_parse_tsf(
 }
 IEEE80211_IF_FILE_RW(tsf);
 
+static ssize_t ieee80211_if_fmt_valid_links(const struct ieee80211_sub_if_data *sdata,
+					    char *buf, int buflen)
+{
+	return snprintf(buf, buflen, "0x%x\n", sdata->vif.valid_links);
+}
+IEEE80211_IF_FILE_R(valid_links);
+
+static ssize_t ieee80211_if_fmt_active_links(const struct ieee80211_sub_if_data *sdata,
+					     char *buf, int buflen)
+{
+	return snprintf(buf, buflen, "0x%x\n", sdata->vif.active_links);
+}
+
+static ssize_t ieee80211_if_parse_active_links(struct ieee80211_sub_if_data *sdata,
+					       const char *buf, int buflen)
+{
+	u16 active_links;
+
+	if (kstrtou16(buf, 0, &active_links))
+		return -EINVAL;
+
+	return ieee80211_set_active_links(&sdata->vif, active_links) ?: buflen;
+}
+IEEE80211_IF_FILE_RW(active_links);
 
 #ifdef CONFIG_MAC80211_MESH
 IEEE80211_IF_FILE(estab_plinks, u.mesh.estab_plinks, ATOMIC);
@@ -670,6 +694,8 @@ static void add_sta_files(struct ieee80211_sub_if_data *sdata)
 	DEBUGFS_ADD_MODE(uapsd_queues, 0600);
 	DEBUGFS_ADD_MODE(uapsd_max_sp_len, 0600);
 	DEBUGFS_ADD_MODE(tdls_wider_bw, 0600);
+	DEBUGFS_ADD_MODE(valid_links, 0200);
+	DEBUGFS_ADD_MODE(active_links, 0600);
 }
 
 static void add_ap_files(struct ieee80211_sub_if_data *sdata)
diff --git a/net/mac80211/driver-ops.c b/net/mac80211/driver-ops.c
index 9b61dc7889c2..5392ffa18270 100644
--- a/net/mac80211/driver-ops.c
+++ b/net/mac80211/driver-ops.c
@@ -192,6 +192,10 @@ int drv_conf_tx(struct ieee80211_local *local,
 	if (!check_sdata_in_driver(sdata))
 		return -EIO;
 
+	if (sdata->vif.active_links &&
+	    !(sdata->vif.active_links & BIT(link->link_id)))
+		return 0;
+
 	if (params->cw_min == 0 || params->cw_min > params->cw_max) {
 		/*
 		 * If we can't configure hardware anyway, don't warn. We may
@@ -272,6 +276,60 @@ void drv_reset_tsf(struct ieee80211_local *local,
 	trace_drv_return_void(local);
 }
 
+int drv_assign_vif_chanctx(struct ieee80211_local *local,
+			   struct ieee80211_sub_if_data *sdata,
+			   struct ieee80211_bss_conf *link_conf,
+			   struct ieee80211_chanctx *ctx)
+{
+	int ret = 0;
+
+	drv_verify_link_exists(sdata, link_conf);
+	if (!check_sdata_in_driver(sdata))
+		return -EIO;
+
+	if (sdata->vif.active_links &&
+	    !(sdata->vif.active_links & BIT(link_conf->link_id)))
+		return 0;
+
+	trace_drv_assign_vif_chanctx(local, sdata, link_conf, ctx);
+	if (local->ops->assign_vif_chanctx) {
+		WARN_ON_ONCE(!ctx->driver_present);
+		ret = local->ops->assign_vif_chanctx(&local->hw,
+						     &sdata->vif,
+						     link_conf,
+						     &ctx->conf);
+	}
+	trace_drv_return_int(local, ret);
+
+	return ret;
+}
+
+void drv_unassign_vif_chanctx(struct ieee80211_local *local,
+			      struct ieee80211_sub_if_data *sdata,
+			      struct ieee80211_bss_conf *link_conf,
+			      struct ieee80211_chanctx *ctx)
+{
+	might_sleep();
+
+	drv_verify_link_exists(sdata, link_conf);
+	if (!check_sdata_in_driver(sdata))
+		return;
+
+	if (sdata->vif.active_links &&
+	    !(sdata->vif.active_links & BIT(link_conf->link_id)))
+		return;
+
+	trace_drv_unassign_vif_chanctx(local, sdata, link_conf, ctx);
+	if (local->ops->unassign_vif_chanctx) {
+		WARN_ON_ONCE(!ctx->driver_present);
+		local->ops->unassign_vif_chanctx(&local->hw,
+						 &sdata->vif,
+						 link_conf,
+						 &ctx->conf);
+	}
+	trace_drv_return_void(local);
+}
+
 int drv_switch_vif_chanctx(struct ieee80211_local *local,
 			   struct ieee80211_vif_chanctx_switch *vifs,
 			   int n_vifs, enum ieee80211_chanctx_switch_mode mode)
@@ -346,3 +404,117 @@ int drv_ampdu_action(struct ieee80211_local *local,
 
 	return ret;
 }
+
+void drv_link_info_changed(struct ieee80211_local *local,
+			   struct ieee80211_sub_if_data *sdata,
+			   struct ieee80211_bss_conf *info,
+			   int link_id, u64 changed)
+{
+	might_sleep();
+
+	if (WARN_ON_ONCE(changed & (BSS_CHANGED_BEACON |
+				    BSS_CHANGED_BEACON_ENABLED) &&
+			 sdata->vif.type != NL80211_IFTYPE_AP &&
+			 sdata->vif.type != NL80211_IFTYPE_ADHOC &&
+			 sdata->vif.type != NL80211_IFTYPE_MESH_POINT &&
+			 sdata->vif.type != NL80211_IFTYPE_OCB))
+		return;
+
+	if (WARN_ON_ONCE(sdata->vif.type == NL80211_IFTYPE_P2P_DEVICE ||
+			 sdata->vif.type == NL80211_IFTYPE_NAN ||
+			 (sdata->vif.type == NL80211_IFTYPE_MONITOR &&
+			  !sdata->vif.bss_conf.mu_mimo_owner &&
+			  !(changed & BSS_CHANGED_TXPOWER))))
+		return;
+
+	if (!check_sdata_in_driver(sdata))
+		return;
+
+	if (sdata->vif.active_links &&
+	    !(sdata->vif.active_links & BIT(link_id)))
+		return;
+
+	trace_drv_link_info_changed(local, sdata, info, changed);
+	if (local->ops->link_info_changed)
+		local->ops->link_info_changed(&local->hw, &sdata->vif,
+					      info, changed);
+	else if (local->ops->bss_info_changed)
+		local->ops->bss_info_changed(&local->hw, &sdata->vif,
+					     info, changed);
+	trace_drv_return_void(local);
+}
+
+int drv_set_key(struct ieee80211_local *local,
+		enum set_key_cmd cmd,
+		struct ieee80211_sub_if_data *sdata,
+		struct ieee80211_sta *sta,
+		struct ieee80211_key_conf *key)
+{
+	int ret;
+
+	might_sleep();
+
+	sdata = get_bss_sdata(sdata);
+	if (!check_sdata_in_driver(sdata))
+		return -EIO;
+
+	if (WARN_ON(key->link_id >= 0 && sdata->vif.active_links &&
+		    !(sdata->vif.active_links & BIT(key->link_id))))
+		return -ENOLINK;
+
+	trace_drv_set_key(local, cmd, sdata, sta, key);
+	ret = local->ops->set_key(&local->hw, cmd, &sdata->vif, sta, key);
+	trace_drv_return_int(local, ret);
+	return ret;
+}
+
+int drv_change_vif_links(struct ieee80211_local *local,
+			 struct ieee80211_sub_if_data *sdata,
+			 u16 old_links, u16 new_links,
+			 struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS])
+{
+	int ret = -EOPNOTSUPP;
+
+	might_sleep();
+
+	if (!check_sdata_in_driver(sdata))
+		return -EIO;
+
+	if (old_links == new_links)
+		return 0;
+
+	trace_drv_change_vif_links(local, sdata, old_links, new_links);
+	if (local->ops->change_vif_links)
+		ret = local->ops->change_vif_links(&local->hw, &sdata->vif,
+						   old_links, new_links, old);
+	trace_drv_return_int(local, ret);
+
+	return ret;
+}
+
+int drv_change_sta_links(struct ieee80211_local *local,
+			 struct ieee80211_sub_if_data *sdata,
+			 struct ieee80211_sta *sta,
+			 u16 old_links, u16 new_links)
+{
+	int ret = -EOPNOTSUPP;
+
+	might_sleep();
+
+	if (!check_sdata_in_driver(sdata))
+		return -EIO;
+
+	old_links &= sdata->vif.active_links;
+	new_links &= sdata->vif.active_links;
+
+	if (old_links == new_links)
+		return 0;
+
+	trace_drv_change_sta_links(local, sdata, sta, old_links, new_links);
+	if (local->ops->change_sta_links)
+		ret = local->ops->change_sta_links(&local->hw, &sdata->vif, sta,
+						   old_links, new_links);
+	trace_drv_return_int(local, ret);
+
+	return ret;
+}
diff --git a/net/mac80211/driver-ops.h b/net/mac80211/driver-ops.h
index 482f5c97a72b..81e40b0a3b16 100644
--- a/net/mac80211/driver-ops.h
+++ b/net/mac80211/driver-ops.h
@@ -165,40 +165,10 @@ static inline void drv_vif_cfg_changed(struct ieee80211_local *local,
 	trace_drv_return_void(local);
 }
 
-static inline void drv_link_info_changed(struct ieee80211_local *local,
-					 struct ieee80211_sub_if_data *sdata,
-					 struct ieee80211_bss_conf *info,
-					 int link_id, u64 changed)
-{
-	might_sleep();
-
-	if (WARN_ON_ONCE(changed & (BSS_CHANGED_BEACON |
-				    BSS_CHANGED_BEACON_ENABLED) &&
-			 sdata->vif.type != NL80211_IFTYPE_AP &&
-			 sdata->vif.type != NL80211_IFTYPE_ADHOC &&
-			 sdata->vif.type != NL80211_IFTYPE_MESH_POINT &&
-			 sdata->vif.type != NL80211_IFTYPE_OCB))
-		return;
-
-	if (WARN_ON_ONCE(sdata->vif.type == NL80211_IFTYPE_P2P_DEVICE ||
-			 sdata->vif.type == NL80211_IFTYPE_NAN ||
-			 (sdata->vif.type == NL80211_IFTYPE_MONITOR &&
-			  !sdata->vif.bss_conf.mu_mimo_owner &&
-			  !(changed & BSS_CHANGED_TXPOWER))))
-		return;
-
-	if (!check_sdata_in_driver(sdata))
-		return;
-
-	trace_drv_link_info_changed(local, sdata, info, changed);
-	if (local->ops->link_info_changed)
-		local->ops->link_info_changed(&local->hw, &sdata->vif,
-					      info, changed);
-	else if (local->ops->bss_info_changed)
-		local->ops->bss_info_changed(&local->hw, &sdata->vif,
-					     info, changed);
-	trace_drv_return_void(local);
-}
+void drv_link_info_changed(struct ieee80211_local *local,
+			   struct ieee80211_sub_if_data *sdata,
+			   struct ieee80211_bss_conf *info,
+			   int link_id, u64 changed);
 
 static inline u64 drv_prepare_multicast(struct ieee80211_local *local,
 					struct netdev_hw_addr_list *mc_list)
@@ -256,25 +226,11 @@ static inline int drv_set_tim(struct ieee80211_local *local,
 	return ret;
 }
 
-static inline int drv_set_key(struct ieee80211_local *local,
-			      enum set_key_cmd cmd,
-			      struct ieee80211_sub_if_data *sdata,
-			      struct ieee80211_sta *sta,
-			      struct ieee80211_key_conf *key)
-{
-	int ret;
-
-	might_sleep();
-
-	sdata = get_bss_sdata(sdata);
-	if (!check_sdata_in_driver(sdata))
-		return -EIO;
-
-	trace_drv_set_key(local, cmd, sdata, sta, key);
-	ret = local->ops->set_key(&local->hw, cmd, &sdata->vif, sta, key);
-	trace_drv_return_int(local, ret);
-	return ret;
-}
+int drv_set_key(struct ieee80211_local *local,
+		enum set_key_cmd cmd,
+		struct ieee80211_sub_if_data *sdata,
+		struct ieee80211_sta *sta,
+		struct ieee80211_key_conf *key);
 
 static inline void drv_update_tkip_key(struct ieee80211_local *local,
 				       struct ieee80211_sub_if_data *sdata,
@@ -945,52 +901,14 @@ static inline void drv_verify_link_exists(struct ieee80211_sub_if_data *sdata,
 		sdata_assert_lock(sdata);
 }
 
-static inline int drv_assign_vif_chanctx(struct ieee80211_local *local,
-					 struct ieee80211_sub_if_data *sdata,
-					 struct ieee80211_bss_conf *link_conf,
-					 struct ieee80211_chanctx *ctx)
-{
-	int ret = 0;
-
-	drv_verify_link_exists(sdata, link_conf);
-	if (!check_sdata_in_driver(sdata))
-		return -EIO;
-
-	trace_drv_assign_vif_chanctx(local, sdata, link_conf, ctx);
-	if (local->ops->assign_vif_chanctx) {
-		WARN_ON_ONCE(!ctx->driver_present);
-		ret = local->ops->assign_vif_chanctx(&local->hw,
-						     &sdata->vif,
-						     link_conf,
-						     &ctx->conf);
-	}
-	trace_drv_return_int(local, ret);
-
-	return ret;
-}
-
-static inline void drv_unassign_vif_chanctx(struct ieee80211_local *local,
-					    struct ieee80211_sub_if_data *sdata,
-					    struct ieee80211_bss_conf *link_conf,
-					    struct ieee80211_chanctx *ctx)
-{
-	might_sleep();
-
-	drv_verify_link_exists(sdata, link_conf);
-	if (!check_sdata_in_driver(sdata))
-		return;
-
-	trace_drv_unassign_vif_chanctx(local, sdata, link_conf, ctx);
-	if (local->ops->unassign_vif_chanctx) {
-		WARN_ON_ONCE(!ctx->driver_present);
-		local->ops->unassign_vif_chanctx(&local->hw,
-						 &sdata->vif,
-						 link_conf,
-						 &ctx->conf);
-	}
-	trace_drv_return_void(local);
-}
-
+int drv_assign_vif_chanctx(struct ieee80211_local *local,
+			   struct ieee80211_sub_if_data *sdata,
+			   struct ieee80211_bss_conf *link_conf,
+			   struct ieee80211_chanctx *ctx);
+void drv_unassign_vif_chanctx(struct ieee80211_local *local,
+			      struct ieee80211_sub_if_data *sdata,
+			      struct ieee80211_bss_conf *link_conf,
+			      struct ieee80211_chanctx *ctx);
 int drv_switch_vif_chanctx(struct ieee80211_local *local,
 			   struct ieee80211_vif_chanctx_switch *vifs,
 			   int n_vifs, enum ieee80211_chanctx_switch_mode mode);
@@ -1552,46 +1470,13 @@ static inline int drv_net_fill_forward_path(struct ieee80211_local *local,
 	return ret;
 }
 
-static inline int drv_change_vif_links(struct ieee80211_local *local,
-				       struct ieee80211_sub_if_data *sdata,
-				       u16 old_links, u16 new_links,
-				       struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS])
-{
-	int ret = -EOPNOTSUPP;
-
-	might_sleep();
-
-	if (!check_sdata_in_driver(sdata))
-		return -EIO;
-
-	trace_drv_change_vif_links(local, sdata, old_links, new_links);
-	if (local->ops->change_vif_links)
-		ret = local->ops->change_vif_links(&local->hw, &sdata->vif,
-						   old_links, new_links, old);
-	trace_drv_return_int(local, ret);
-
-	return ret;
-}
-
-static inline int drv_change_sta_links(struct ieee80211_local *local,
-				       struct ieee80211_sub_if_data *sdata,
-				       struct ieee80211_sta *sta,
-				       u16 old_links, u16 new_links)
-{
-	int ret = -EOPNOTSUPP;
-
-	might_sleep();
-
-	if (!check_sdata_in_driver(sdata))
-		return -EIO;
-
-	trace_drv_change_sta_links(local, sdata, sta, old_links, new_links);
-	if (local->ops->change_sta_links)
-		ret = local->ops->change_sta_links(&local->hw, &sdata->vif, sta,
-						   old_links, new_links);
-	trace_drv_return_int(local, ret);
-
-	return ret;
-}
+int drv_change_vif_links(struct ieee80211_local *local,
+			 struct ieee80211_sub_if_data *sdata,
+			 u16 old_links, u16 new_links,
+			 struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]);
+int drv_change_sta_links(struct ieee80211_local *local,
+			 struct ieee80211_sub_if_data *sdata,
+			 struct ieee80211_sta *sta,
+			 u16 old_links, u16 new_links);
 
 #endif /* __MAC80211_DRIVER_OPS */
diff --git a/net/mac80211/eht.c b/net/mac80211/eht.c
index 31e20a342f21..18bc6b78b267 100644
--- a/net/mac80211/eht.c
+++ b/net/mac80211/eht.c
@@ -30,7 +30,9 @@ ieee80211_eht_cap_ie_to_sta_eht_cap(struct ieee80211_sub_if_data *sdata,
 		return;
 
 	mcs_nss_size = ieee80211_eht_mcs_nss_size(he_cap_ie_elem,
-						  &eht_cap_ie_elem->fixed);
+						  &eht_cap_ie_elem->fixed,
+						  sdata->vif.type ==
+							NL80211_IFTYPE_STATION);
 
 	eht_total_size += mcs_nss_size;
 
diff --git a/net/mac80211/ethtool.c b/net/mac80211/ethtool.c
index c2b38370bfb1..a3830d925cc2 100644
--- a/net/mac80211/ethtool.c
+++ b/net/mac80211/ethtool.c
@@ -83,17 +83,17 @@ static void ieee80211_get_stats(struct net_device *dev,
 
 #define ADD_STA_STATS(sta)					\
 	do {							\
-		data[i++] += (sta)->rx_stats.packets;		\
-		data[i++] += (sta)->rx_stats.bytes;		\
+		data[i++] += sinfo.rx_packets;			\
+		data[i++] += sinfo.rx_bytes;			\
 		data[i++] += (sta)->rx_stats.num_duplicates;	\
 		data[i++] += (sta)->rx_stats.fragments;		\
-		data[i++] += (sta)->rx_stats.dropped;		\
+		data[i++] += sinfo.rx_dropped_misc;		\
 								\
 		data[i++] += sinfo.tx_packets;			\
 		data[i++] += sinfo.tx_bytes;			\
 		data[i++] += (sta)->status_stats.filtered;	\
-		data[i++] += (sta)->status_stats.retry_failed;	\
-		data[i++] += (sta)->status_stats.retry_count;	\
+		data[i++] += sinfo.tx_failed;			\
+		data[i++] += sinfo.tx_retries;			\
 	} while (0)
 
 	/* For Managed stations, find the single station based on BSSID
diff --git a/net/mac80211/he.c b/net/mac80211/he.c
index d9228fd3f77a..729f261520c7 100644
--- a/net/mac80211/he.c
+++ b/net/mac80211/he.c
@@ -31,25 +31,27 @@ ieee80211_update_from_he_6ghz_capa(const struct ieee80211_he_6ghz_capa *he_6ghz_
 			break;
 		}
 
-		sta->sta.smps_mode = smps_mode;
+		link_sta->pub->smps_mode = smps_mode;
 	} else {
-		sta->sta.smps_mode = IEEE80211_SMPS_OFF;
+		link_sta->pub->smps_mode = IEEE80211_SMPS_OFF;
 	}
 
 	switch (le16_get_bits(he_6ghz_capa->capa,
 			      IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN)) {
 	case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454:
-		sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_11454;
+		link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_11454;
 		break;
 	case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991:
-		sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_7991;
+		link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_7991;
 		break;
 	case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895:
 	default:
-		sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_3895;
+		link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_3895;
 		break;
 	}
 
+	ieee80211_sta_recalc_aggregates(&sta->sta);
+
 	link_sta->pub->he_6ghz_capa = *he_6ghz_capa;
 }
 
diff --git a/net/mac80211/ht.c b/net/mac80211/ht.c
index 8c24817cd497..83bc41346ae7 100644
--- a/net/mac80211/ht.c
+++ b/net/mac80211/ht.c
@@ -241,9 +241,11 @@ bool ieee80211_ht_cap_ie_to_sta_ht_cap(struct ieee80211_sub_if_data *sdata,
 	ht_cap.mcs.rx_highest = ht_cap_ie->mcs.rx_highest;
 
 	if (ht_cap.cap & IEEE80211_HT_CAP_MAX_AMSDU)
-		sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_7935;
+		link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_7935;
 	else
-		sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_3839;
+		link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_3839;
+
+	ieee80211_sta_recalc_aggregates(&sta->sta);
 
  apply:
 	changed = memcmp(&link_sta->pub->ht_cap, &ht_cap, sizeof(ht_cap));
@@ -299,12 +301,13 @@ bool ieee80211_ht_cap_ie_to_sta_ht_cap(struct ieee80211_sub_if_data *sdata,
 			break;
 		}
 
-		if (smps_mode != sta->sta.smps_mode)
+		if (smps_mode != link_sta->pub->smps_mode)
 			changed = true;
-		sta->sta.smps_mode = smps_mode;
+		link_sta->pub->smps_mode = smps_mode;
 	} else {
-		sta->sta.smps_mode = IEEE80211_SMPS_OFF;
+		link_sta->pub->smps_mode = IEEE80211_SMPS_OFF;
 	}
+
 	return changed;
 }
 
diff --git a/net/mac80211/ibss.c b/net/mac80211/ibss.c
index 9b283bbc7bb4..9dffc3079588 100644
--- a/net/mac80211/ibss.c
+++ b/net/mac80211/ibss.c
@@ -1350,10 +1350,10 @@ static void ieee80211_sta_create_ibss(struct ieee80211_sub_if_data *sdata)
 				  capability, 0, true);
 }
 
-static unsigned ibss_setup_channels(struct wiphy *wiphy,
-				    struct ieee80211_channel **channels,
-				    unsigned int channels_max,
-				    u32 center_freq, u32 width)
+static unsigned int ibss_setup_channels(struct wiphy *wiphy,
+					struct ieee80211_channel **channels,
+					unsigned int channels_max,
+					u32 center_freq, u32 width)
 {
 	struct ieee80211_channel *chan = NULL;
 	unsigned int n_chan = 0;
diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h
index e192e1ec0261..4e1d4c339f2d 100644
--- a/net/mac80211/ieee80211_i.h
+++ b/net/mac80211/ieee80211_i.h
@@ -213,6 +213,7 @@ struct ieee80211_rx_data {
 	struct ieee80211_sub_if_data *sdata;
 	struct ieee80211_link_data *link;
 	struct sta_info *sta;
+	struct link_sta_info *link_sta;
 	struct ieee80211_key *key;
 
 	unsigned int flags;
@@ -1080,6 +1081,10 @@ struct ieee80211_sub_if_data {
 	struct ieee80211_link_data deflink;
 	struct ieee80211_link_data __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS];
 
+	/* for ieee80211_set_active_links_async() */
+	struct work_struct activate_links_work;
+	u16 desired_active_links;
+
 #ifdef CONFIG_MAC80211_DEBUGFS
 	struct {
 		struct dentry *subdir_stations;
@@ -1810,6 +1815,7 @@ void ieee80211_sta_connection_lost(struct ieee80211_sub_if_data *sdata,
 				   u8 reason, bool tx);
 void ieee80211_mgd_setup_link(struct ieee80211_link_data *link);
 void ieee80211_mgd_stop_link(struct ieee80211_link_data *link);
+void ieee80211_mgd_set_link_qos_params(struct ieee80211_link_data *link);
 
 /* IBSS code */
 void ieee80211_ibss_notify_scan_completed(struct ieee80211_local *local);
@@ -1929,9 +1935,6 @@ void ieee80211_sdata_stop(struct ieee80211_sub_if_data *sdata);
 int ieee80211_add_virtual_monitor(struct ieee80211_local *local);
 void ieee80211_del_virtual_monitor(struct ieee80211_local *local);
 
-int ieee80211_vif_set_links(struct ieee80211_sub_if_data *sdata,
-			    u16 new_links);
-
 bool __ieee80211_recalc_txpower(struct ieee80211_sub_if_data *sdata);
 void ieee80211_recalc_txpower(struct ieee80211_sub_if_data *sdata,
 			      bool update_bss);
@@ -1942,6 +1945,17 @@ static inline bool ieee80211_sdata_running(struct ieee80211_sub_if_data *sdata)
 	return test_bit(SDATA_STATE_RUNNING, &sdata->state);
 }
 
+/* link handling */
+void ieee80211_link_setup(struct ieee80211_link_data *link);
+void ieee80211_link_init(struct ieee80211_sub_if_data *sdata,
+			 int link_id,
+			 struct ieee80211_link_data *link,
+			 struct ieee80211_bss_conf *link_conf);
+void ieee80211_link_stop(struct ieee80211_link_data *link);
+int ieee80211_vif_set_links(struct ieee80211_sub_if_data *sdata,
+			    u16 new_links);
+void ieee80211_vif_clear_links(struct ieee80211_sub_if_data *sdata);
+
 /* tx handling */
 void ieee80211_clear_tx_pending(struct ieee80211_local *local);
 void ieee80211_tx_pending(struct tasklet_struct *t);
@@ -2184,6 +2198,8 @@ static inline void ieee80211_tx_skb(struct ieee80211_sub_if_data *sdata,
  *	for that non-transmitting BSS is returned
  * @link_id: the link ID to parse elements for, if a STA profile
  *	is present in the multi-link element, or -1 to ignore
+ * @from_ap: frame is received from an AP (currently used only
+ *	for EHT capabilities parsing)
  */
 struct ieee80211_elems_parse_params {
 	const u8 *start;
@@ -2193,6 +2209,7 @@ struct ieee80211_elems_parse_params {
 	u32 crc;
 	struct cfg80211_bss *bss;
 	int link_id;
+	bool from_ap;
 };
 
 struct ieee802_11_elems *
@@ -2382,6 +2399,7 @@ u8 *ieee80211_ie_build_he_cap(ieee80211_conn_flags_t disable_flags, u8 *pos,
 			      const struct ieee80211_sta_he_cap *he_cap,
 			      u8 *end);
 void ieee80211_ie_build_he_6ghz_cap(struct ieee80211_sub_if_data *sdata,
+				    enum ieee80211_smps_mode smps_mode,
 				    struct sk_buff *skb);
 u8 *ieee80211_ie_build_he_oper(u8 *pos, struct cfg80211_chan_def *chandef);
 int ieee80211_parse_bitrates(enum nl80211_chan_width width,
@@ -2407,8 +2425,7 @@ bool ieee80211_chandef_vht_oper(struct ieee80211_hw *hw, u32 vht_cap_info,
 				const struct ieee80211_vht_operation *oper,
 				const struct ieee80211_ht_operation *htop,
 				struct cfg80211_chan_def *chandef);
-void ieee80211_chandef_eht_oper(struct ieee80211_sub_if_data *sdata,
-				const struct ieee80211_eht_operation *eht_oper,
+void ieee80211_chandef_eht_oper(const struct ieee80211_eht_operation *eht_oper,
 				bool support_160, bool support_320,
 				struct cfg80211_chan_def *chandef);
 bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_sub_if_data *sdata,
@@ -2513,7 +2530,8 @@ u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata, u8 iftype);
 u8 *ieee80211_ie_build_eht_cap(u8 *pos,
 			       const struct ieee80211_sta_he_cap *he_cap,
 			       const struct ieee80211_sta_eht_cap *eht_cap,
-			       u8 *end);
+			       u8 *end,
+			       bool for_ap);
 
 void
 ieee80211_eht_cap_ie_to_sta_eht_cap(struct ieee80211_sub_if_data *sdata,
diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c
index 95b58c5cac07..572254366a0f 100644
--- a/net/mac80211/iface.c
+++ b/net/mac80211/iface.c
@@ -200,15 +200,73 @@ static int ieee80211_verify_mac(struct ieee80211_sub_if_data *sdata, u8 *addr,
 	return ret;
 }
 
+static int ieee80211_can_powered_addr_change(struct ieee80211_sub_if_data *sdata)
+{
+	struct ieee80211_roc_work *roc;
+	struct ieee80211_local *local = sdata->local;
+	struct ieee80211_sub_if_data *scan_sdata;
+	int ret = 0;
+
+	/* To be the most flexible here we want to only limit changing the
+	 * address if the specific interface is doing offchannel work or
+	 * scanning.
+	 */
+	if (netif_carrier_ok(sdata->dev))
+		return -EBUSY;
+
+	mutex_lock(&local->mtx);
+
+	/* First check no ROC work is happening on this iface */
+	list_for_each_entry(roc, &local->roc_list, list) {
+		if (roc->sdata != sdata)
+			continue;
+
+		if (roc->started) {
+			ret = -EBUSY;
+			goto unlock;
+		}
+	}
+
+	/* And if this iface is scanning */
+	if (local->scanning) {
+		scan_sdata = rcu_dereference_protected(local->scan_sdata,
+						       lockdep_is_held(&local->mtx));
+		if (sdata == scan_sdata)
+			ret = -EBUSY;
+	}
+
+	switch (sdata->vif.type) {
+	case NL80211_IFTYPE_STATION:
+	case NL80211_IFTYPE_P2P_CLIENT:
+		/* More interface types could be added here but changing the
+		 * address while powered makes the most sense in client modes.
+		 */
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+unlock:
+	mutex_unlock(&local->mtx);
+	return ret;
+}
+
 static int ieee80211_change_mac(struct net_device *dev, void *addr)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
+	struct ieee80211_local *local = sdata->local;
 	struct sockaddr *sa = addr;
 	bool check_dup = true;
+	bool live = false;
 	int ret;
 
-	if (ieee80211_sdata_running(sdata))
-		return -EBUSY;
+	if (ieee80211_sdata_running(sdata)) {
+		ret = ieee80211_can_powered_addr_change(sdata);
+		if (ret)
+			return ret;
+
+		live = true;
+	}
 
 	if (sdata->vif.type == NL80211_IFTYPE_MONITOR &&
 	    !(sdata->u.mntr.flags & MONITOR_FLAG_ACTIVE))
@@ -218,6 +276,8 @@ static int ieee80211_change_mac(struct net_device *dev, void *addr)
 	if (ret)
 		return ret;
 
+	if (live)
+		drv_remove_interface(local, sdata);
 	ret = eth_mac_addr(dev, sa);
 
 	if (ret == 0) {
@@ -225,6 +285,12 @@ static int ieee80211_change_mac(struct net_device *dev, void *addr)
 		ether_addr_copy(sdata->vif.bss_conf.addr, sdata->vif.addr);
 	}
 
+	/* Regardless of eth_mac_addr() return we still want to add the
+	 * interface back. This should not fail...
+	 */
+	if (live)
+		WARN_ON(drv_add_interface(local, sdata));
+
 	return ret;
 }
 
@@ -296,6 +362,11 @@ static int ieee80211_check_concurrent_iface(struct ieee80211_sub_if_data *sdata,
 							nsdata->vif.type))
 				return -ENOTUNIQ;
 
+			/* No support for VLAN with MLO yet */
+			if (iftype == NL80211_IFTYPE_AP_VLAN &&
+			    nsdata->wdev.use_4addr)
+				return -EOPNOTSUPP;
+
 			/*
 			 * can only add VLANs to enabled APs
 			 */
@@ -368,246 +439,6 @@ static int ieee80211_open(struct net_device *dev)
 	return err;
 }
 
-static void ieee80211_link_setup(struct ieee80211_link_data *link)
-{
-	if (link->sdata->vif.type == NL80211_IFTYPE_STATION)
-		ieee80211_mgd_setup_link(link);
-}
-
-static void ieee80211_link_init(struct ieee80211_sub_if_data *sdata,
-				int link_id,
-				struct ieee80211_link_data *link,
-				struct ieee80211_bss_conf *link_conf)
-{
-	bool deflink = link_id < 0;
-
-	if (link_id < 0)
-		link_id = 0;
-
-	rcu_assign_pointer(sdata->vif.link_conf[link_id], link_conf);
-	rcu_assign_pointer(sdata->link[link_id], link);
-
-	link->sdata = sdata;
-	link->link_id = link_id;
-	link->conf = link_conf;
-	link_conf->link_id = link_id;
-
-	INIT_WORK(&link->csa_finalize_work,
-		  ieee80211_csa_finalize_work);
-	INIT_WORK(&link->color_change_finalize_work,
-		  ieee80211_color_change_finalize_work);
-	INIT_LIST_HEAD(&link->assigned_chanctx_list);
-	INIT_LIST_HEAD(&link->reserved_chanctx_list);
-	INIT_DELAYED_WORK(&link->dfs_cac_timer_work,
-			  ieee80211_dfs_cac_timer_work);
-
-	if (!deflink) {
-		switch (sdata->vif.type) {
-		case NL80211_IFTYPE_AP:
-			ether_addr_copy(link_conf->addr,
-					sdata->wdev.links[link_id].addr);
-			WARN_ON(!(sdata->wdev.valid_links & BIT(link_id)));
-			break;
-		case NL80211_IFTYPE_STATION:
-			break;
-		default:
-			WARN_ON(1);
-		}
-	}
-}
-
-static void ieee80211_link_stop(struct ieee80211_link_data *link)
-{
-	if (link->sdata->vif.type == NL80211_IFTYPE_STATION)
-		ieee80211_mgd_stop_link(link);
-
-	ieee80211_link_release_channel(link);
-}
-
-struct link_container {
-	struct ieee80211_link_data data;
-	struct ieee80211_bss_conf conf;
-};
-
-static void ieee80211_free_links(struct ieee80211_sub_if_data *sdata,
-				 struct link_container **links)
-{
-	unsigned int link_id;
-
-	synchronize_rcu();
-
-	for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) {
-		if (!links[link_id])
-			continue;
-		ieee80211_link_stop(&links[link_id]->data);
-		kfree(links[link_id]);
-	}
-}
-
-static int ieee80211_check_dup_link_addrs(struct ieee80211_sub_if_data *sdata)
-{
-	unsigned int i, j;
-
-	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
-		struct ieee80211_link_data *link1;
-
-		link1 = sdata_dereference(sdata->link[i], sdata);
-		if (!link1)
-			continue;
-		for (j = i + 1; j < IEEE80211_MLD_MAX_NUM_LINKS; j++) {
-			struct ieee80211_link_data *link2;
-
-			link2 = sdata_dereference(sdata->link[j], sdata);
-			if (!link2)
-				continue;
-
-			if (ether_addr_equal(link1->conf->addr,
-					     link2->conf->addr))
-				return -EALREADY;
-		}
-	}
-
-	return 0;
-}
-
-static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata,
-				      struct link_container **to_free,
-				      u16 new_links)
-{
-	u16 old_links = sdata->vif.valid_links;
-	unsigned long add = new_links & ~old_links;
-	unsigned long rem = old_links & ~new_links;
-	unsigned int link_id;
-	int ret;
-	struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS] = {}, *link;
-	struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS];
-	struct ieee80211_link_data *old_data[IEEE80211_MLD_MAX_NUM_LINKS];
-	bool use_deflink = old_links == 0; /* set for error case */
-
-	sdata_assert_lock(sdata);
-
-	memset(to_free, 0, sizeof(links));
-
-	if (old_links == new_links)
-		return 0;
-
-	/* if there were no old links, need to clear the pointers to deflink */
-	if (!old_links)
-		rem |= BIT(0);
-
-	/* allocate new link structures first */
-	for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) {
-		link = kzalloc(sizeof(*link), GFP_KERNEL);
-		if (!link) {
-			ret = -ENOMEM;
-			goto free;
-		}
-		links[link_id] = link;
-	}
-
-	/* keep track of the old pointers for the driver */
-	BUILD_BUG_ON(sizeof(old) != sizeof(sdata->vif.link_conf));
-	memcpy(old, sdata->vif.link_conf, sizeof(old));
-	/* and for us in error cases */
-	BUILD_BUG_ON(sizeof(old_data) != sizeof(sdata->link));
-	memcpy(old_data, sdata->link, sizeof(old_data));
-
-	/* grab old links to free later */
-	for_each_set_bit(link_id, &rem, IEEE80211_MLD_MAX_NUM_LINKS) {
-		if (rcu_access_pointer(sdata->link[link_id]) != &sdata->deflink) {
-			/*
-			 * we must have allocated the data through this path so
-			 * we know we can free both at the same time
-			 */
-			to_free[link_id] = container_of(rcu_access_pointer(sdata->link[link_id]),
-							typeof(*links[link_id]),
-							data);
-		}
-
-		RCU_INIT_POINTER(sdata->link[link_id], NULL);
-		RCU_INIT_POINTER(sdata->vif.link_conf[link_id], NULL);
-	}
-
-	/* link them into data structures */
-	for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) {
-		WARN_ON(!use_deflink &&
-			rcu_access_pointer(sdata->link[link_id]) == &sdata->deflink);
-
-		link = links[link_id];
-		ieee80211_link_init(sdata, link_id, &link->data, &link->conf);
-		ieee80211_link_setup(&link->data);
-	}
-
-	if (new_links == 0)
-		ieee80211_link_init(sdata, -1, &sdata->deflink,
-				    &sdata->vif.bss_conf);
-
-	sdata->vif.valid_links = new_links;
-
-	ret = ieee80211_check_dup_link_addrs(sdata);
-	if (!ret) {
-		/* tell the driver */
-		ret = drv_change_vif_links(sdata->local, sdata,
-					   old_links, new_links,
-					   old);
-	}
-
-	if (ret) {
-		/* restore config */
-		memcpy(sdata->link, old_data, sizeof(old_data));
-		memcpy(sdata->vif.link_conf, old, sizeof(old));
-		sdata->vif.valid_links = old_links;
-		/* and free (only) the newly allocated links */
-		memset(to_free, 0, sizeof(links));
-		goto free;
-	}
-
-	/* use deflink/bss_conf again if and only if there are no more links */
-	use_deflink = new_links == 0;
-
-	goto deinit;
-free:
-	/* if we failed during allocation, only free all */
-	for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) {
-		kfree(links[link_id]);
-		links[link_id] = NULL;
-	}
-deinit:
-	if (use_deflink)
-		ieee80211_link_init(sdata, -1, &sdata->deflink,
-				    &sdata->vif.bss_conf);
-	return ret;
-}
-
-int ieee80211_vif_set_links(struct ieee80211_sub_if_data *sdata,
-			    u16 new_links)
-{
-	struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS];
-	int ret;
-
-	ret = ieee80211_vif_update_links(sdata, links, new_links);
-	ieee80211_free_links(sdata, links);
-
-	return ret;
-}
-
-static void ieee80211_vif_clear_links(struct ieee80211_sub_if_data *sdata)
-{
-	struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS];
-
-	/*
-	 * The locking here is different because when we free links
-	 * in the station case we need to be able to cancel_work_sync()
-	 * something that also takes the lock.
-	 */
-
-	sdata_lock(sdata);
-	ieee80211_vif_update_links(sdata, links, 0);
-	sdata_unlock(sdata);
-
-	ieee80211_free_links(sdata, links);
-}
-
 static void ieee80211_do_stop(struct ieee80211_sub_if_data *sdata, bool going_down)
 {
 	struct ieee80211_local *local = sdata->local;
@@ -923,6 +754,8 @@ static int ieee80211_stop(struct net_device *dev)
 		ieee80211_stop_mbssid(sdata);
 	}
 
+	cancel_work_sync(&sdata->activate_links_work);
+
 	wiphy_lock(sdata->local->hw.wiphy);
 	ieee80211_do_stop(sdata, true);
 	wiphy_unlock(sdata->local->hw.wiphy);
@@ -1893,6 +1726,15 @@ static void ieee80211_recalc_smps_work(struct work_struct *work)
 	ieee80211_recalc_smps(sdata, &sdata->deflink);
 }
 
+static void ieee80211_activate_links_work(struct work_struct *work)
+{
+	struct ieee80211_sub_if_data *sdata =
+		container_of(work, struct ieee80211_sub_if_data,
+			     activate_links_work);
+
+	ieee80211_set_active_links(&sdata->vif, sdata->desired_active_links);
+}
+
 /*
  * Helper function to initialise an interface to a specific type.
  */
@@ -1930,6 +1772,7 @@ static void ieee80211_setup_sdata(struct ieee80211_sub_if_data *sdata,
 	skb_queue_head_init(&sdata->status_queue);
 	INIT_WORK(&sdata->work, ieee80211_iface_work);
 	INIT_WORK(&sdata->recalc_smps, ieee80211_recalc_smps_work);
+	INIT_WORK(&sdata->activate_links_work, ieee80211_activate_links_work);
 
 	switch (type) {
 	case NL80211_IFTYPE_P2P_GO:
@@ -2267,7 +2110,7 @@ int ieee80211_if_add(struct ieee80211_local *local, const char *name,
 		wdev = &sdata->wdev;
 
 		sdata->dev = NULL;
-		strlcpy(sdata->name, name, IFNAMSIZ);
+		strscpy(sdata->name, name, IFNAMSIZ);
 		ieee80211_assign_perm_addr(local, wdev->address, type);
 		memcpy(sdata->vif.addr, wdev->address, ETH_ALEN);
 		ether_addr_copy(sdata->vif.bss_conf.addr, sdata->vif.addr);
@@ -2396,6 +2239,7 @@ int ieee80211_if_add(struct ieee80211_local *local, const char *name,
 			sdata->u.mgd.use_4addr = params->use_4addr;
 
 		ndev->features |= local->hw.netdev_features;
+		ndev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
 		ndev->hw_features |= ndev->features &
 					MAC80211_SUPPORTED_FEATURES_TX;
 
diff --git a/net/mac80211/key.c b/net/mac80211/key.c
index 6befb578ed9e..e8f6c1e5eabf 100644
--- a/net/mac80211/key.c
+++ b/net/mac80211/key.c
@@ -177,6 +177,10 @@ static int ieee80211_key_enable_hw_accel(struct ieee80211_key *key)
 		}
 	}
 
+	if (key->conf.link_id >= 0 && sdata->vif.active_links &&
+	    !(sdata->vif.active_links & BIT(key->conf.link_id)))
+		return 0;
+
 	ret = drv_set_key(key->local, SET_KEY, sdata,
 			  sta ? &sta->sta : NULL, &key->conf);
 
@@ -246,6 +250,10 @@ static void ieee80211_key_disable_hw_accel(struct ieee80211_key *key)
 	sta = key->sta;
 	sdata = key->sdata;
 
+	if (key->conf.link_id >= 0 && sdata->vif.active_links &&
+	    !(sdata->vif.active_links & BIT(key->conf.link_id)))
+		return;
+
 	if (!(key->conf.flags & (IEEE80211_KEY_FLAG_GENERATE_MMIC |
 				 IEEE80211_KEY_FLAG_PUT_MIC_SPACE |
 				 IEEE80211_KEY_FLAG_RESERVE_TAILROOM)))
@@ -344,9 +352,10 @@ static void ieee80211_pairwise_rekey(struct ieee80211_key *old,
 	}
 }
 
-static void __ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata,
+static void __ieee80211_set_default_key(struct ieee80211_link_data *link,
 					int idx, bool uni, bool multi)
 {
+	struct ieee80211_sub_if_data *sdata = link->sdata;
 	struct ieee80211_key *key = NULL;
 
 	assert_key_lock(sdata->local);
@@ -354,7 +363,7 @@ static void __ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata,
 	if (idx >= 0 && idx < NUM_DEFAULT_KEYS) {
 		key = key_mtx_dereference(sdata->local, sdata->keys[idx]);
 		if (!key)
-			key = key_mtx_dereference(sdata->local, sdata->deflink.gtk[idx]);
+			key = key_mtx_dereference(sdata->local, link->gtk[idx]);
 	}
 
 	if (uni) {
@@ -365,47 +374,48 @@ static void __ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata,
 	}
 
 	if (multi)
-		rcu_assign_pointer(sdata->deflink.default_multicast_key, key);
+		rcu_assign_pointer(link->default_multicast_key, key);
 
 	ieee80211_debugfs_key_update_default(sdata);
 }
 
-void ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata, int idx,
+void ieee80211_set_default_key(struct ieee80211_link_data *link, int idx,
 			       bool uni, bool multi)
 {
-	mutex_lock(&sdata->local->key_mtx);
-	__ieee80211_set_default_key(sdata, idx, uni, multi);
-	mutex_unlock(&sdata->local->key_mtx);
+	mutex_lock(&link->sdata->local->key_mtx);
+	__ieee80211_set_default_key(link, idx, uni, multi);
+	mutex_unlock(&link->sdata->local->key_mtx);
 }
 
 static void
-__ieee80211_set_default_mgmt_key(struct ieee80211_sub_if_data *sdata, int idx)
+__ieee80211_set_default_mgmt_key(struct ieee80211_link_data *link, int idx)
 {
+	struct ieee80211_sub_if_data *sdata = link->sdata;
 	struct ieee80211_key *key = NULL;
 
 	assert_key_lock(sdata->local);
 
 	if (idx >= NUM_DEFAULT_KEYS &&
 	    idx < NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS)
-		key = key_mtx_dereference(sdata->local,
-					  sdata->deflink.gtk[idx]);
+		key = key_mtx_dereference(sdata->local, link->gtk[idx]);
 
-	rcu_assign_pointer(sdata->deflink.default_mgmt_key, key);
+	rcu_assign_pointer(link->default_mgmt_key, key);
 
 	ieee80211_debugfs_key_update_default(sdata);
 }
 
-void ieee80211_set_default_mgmt_key(struct ieee80211_sub_if_data *sdata,
+void ieee80211_set_default_mgmt_key(struct ieee80211_link_data *link,
 				    int idx)
 {
-	mutex_lock(&sdata->local->key_mtx);
-	__ieee80211_set_default_mgmt_key(sdata, idx);
-	mutex_unlock(&sdata->local->key_mtx);
+	mutex_lock(&link->sdata->local->key_mtx);
+	__ieee80211_set_default_mgmt_key(link, idx);
+	mutex_unlock(&link->sdata->local->key_mtx);
 }
 
 static void
-__ieee80211_set_default_beacon_key(struct ieee80211_sub_if_data *sdata, int idx)
+__ieee80211_set_default_beacon_key(struct ieee80211_link_data *link, int idx)
 {
+	struct ieee80211_sub_if_data *sdata = link->sdata;
 	struct ieee80211_key *key = NULL;
 
 	assert_key_lock(sdata->local);
@@ -413,28 +423,30 @@ __ieee80211_set_default_beacon_key(struct ieee80211_sub_if_data *sdata, int idx)
 	if (idx >= NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS &&
 	    idx < NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS +
 	    NUM_DEFAULT_BEACON_KEYS)
-		key = key_mtx_dereference(sdata->local,
-					  sdata->deflink.gtk[idx]);
+		key = key_mtx_dereference(sdata->local, link->gtk[idx]);
 
-	rcu_assign_pointer(sdata->deflink.default_beacon_key, key);
+	rcu_assign_pointer(link->default_beacon_key, key);
 
 	ieee80211_debugfs_key_update_default(sdata);
 }
 
-void ieee80211_set_default_beacon_key(struct ieee80211_sub_if_data *sdata,
+void ieee80211_set_default_beacon_key(struct ieee80211_link_data *link,
 				      int idx)
 {
-	mutex_lock(&sdata->local->key_mtx);
-	__ieee80211_set_default_beacon_key(sdata, idx);
-	mutex_unlock(&sdata->local->key_mtx);
+	mutex_lock(&link->sdata->local->key_mtx);
+	__ieee80211_set_default_beacon_key(link, idx);
+	mutex_unlock(&link->sdata->local->key_mtx);
 }
 
 static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata,
-				  struct sta_info *sta,
-				  bool pairwise,
-				  struct ieee80211_key *old,
-				  struct ieee80211_key *new)
+				 struct ieee80211_link_data *link,
+				 struct sta_info *sta,
+				 bool pairwise,
+				 struct ieee80211_key *old,
+				 struct ieee80211_key *new)
 {
+	struct link_sta_info *link_sta = sta ? &sta->deflink : NULL;
+	int link_id;
 	int idx;
 	int ret = 0;
 	bool defunikey, defmultikey, defmgmtkey, defbeaconkey;
@@ -446,13 +458,36 @@ static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata,
 
 	if (new) {
 		idx = new->conf.keyidx;
-		list_add_tail_rcu(&new->list, &sdata->key_list);
 		is_wep = new->conf.cipher == WLAN_CIPHER_SUITE_WEP40 ||
 			 new->conf.cipher == WLAN_CIPHER_SUITE_WEP104;
+		link_id = new->conf.link_id;
 	} else {
 		idx = old->conf.keyidx;
 		is_wep = old->conf.cipher == WLAN_CIPHER_SUITE_WEP40 ||
 			 old->conf.cipher == WLAN_CIPHER_SUITE_WEP104;
+		link_id = old->conf.link_id;
+	}
+
+	if (WARN(old && old->conf.link_id != link_id,
+		 "old link ID %d doesn't match new link ID %d\n",
+		 old->conf.link_id, link_id))
+		return -EINVAL;
+
+	if (link_id >= 0) {
+		if (!link) {
+			link = sdata_dereference(sdata->link[link_id], sdata);
+			if (!link)
+				return -ENOLINK;
+		}
+
+		if (sta) {
+			link_sta = rcu_dereference_protected(sta->link[link_id],
+							     lockdep_is_held(&sta->local->sta_mtx));
+			if (!link_sta)
+				return -ENOLINK;
+		}
+	} else {
+		link = &sdata->deflink;
 	}
 
 	if ((is_wep || pairwise) && idx >= NUM_DEFAULT_KEYS)
@@ -482,6 +517,9 @@ static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata,
 	if (ret)
 		return ret;
 
+	if (new)
+		list_add_tail_rcu(&new->list, &sdata->key_list);
+
 	if (sta) {
 		if (pairwise) {
 			rcu_assign_pointer(sta->ptk[idx], new);
@@ -489,7 +527,7 @@ static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata,
 			    !(new->conf.flags & IEEE80211_KEY_FLAG_NO_AUTO_TX))
 				_ieee80211_set_tx_key(new, true);
 		} else {
-			rcu_assign_pointer(sta->deflink.gtk[idx], new);
+			rcu_assign_pointer(link_sta->gtk[idx], new);
 		}
 		/* Only needed for transition from no key -> key.
 		 * Still triggers unnecessary when using Extended Key ID
@@ -503,39 +541,39 @@ static int ieee80211_key_replace(struct ieee80211_sub_if_data *sdata,
 						sdata->default_unicast_key);
 		defmultikey = old &&
 			old == key_mtx_dereference(sdata->local,
-						sdata->deflink.default_multicast_key);
+						   link->default_multicast_key);
 		defmgmtkey = old &&
 			old == key_mtx_dereference(sdata->local,
-						sdata->deflink.default_mgmt_key);
+						   link->default_mgmt_key);
 		defbeaconkey = old &&
 			old == key_mtx_dereference(sdata->local,
-						   sdata->deflink.default_beacon_key);
+						   link->default_beacon_key);
 
 		if (defunikey && !new)
-			__ieee80211_set_default_key(sdata, -1, true, false);
+			__ieee80211_set_default_key(link, -1, true, false);
 		if (defmultikey && !new)
-			__ieee80211_set_default_key(sdata, -1, false, true);
+			__ieee80211_set_default_key(link, -1, false, true);
 		if (defmgmtkey && !new)
-			__ieee80211_set_default_mgmt_key(sdata, -1);
+			__ieee80211_set_default_mgmt_key(link, -1);
 		if (defbeaconkey && !new)
-			__ieee80211_set_default_beacon_key(sdata, -1);
+			__ieee80211_set_default_beacon_key(link, -1);
 
 		if (is_wep || pairwise)
 			rcu_assign_pointer(sdata->keys[idx], new);
 		else
-			rcu_assign_pointer(sdata->deflink.gtk[idx], new);
+			rcu_assign_pointer(link->gtk[idx], new);
 
 		if (defunikey && new)
-			__ieee80211_set_default_key(sdata, new->conf.keyidx,
+			__ieee80211_set_default_key(link, new->conf.keyidx,
 						    true, false);
 		if (defmultikey && new)
-			__ieee80211_set_default_key(sdata, new->conf.keyidx,
+			__ieee80211_set_default_key(link, new->conf.keyidx,
 						    false, true);
 		if (defmgmtkey && new)
-			__ieee80211_set_default_mgmt_key(sdata,
+			__ieee80211_set_default_mgmt_key(link,
 							 new->conf.keyidx);
 		if (defbeaconkey && new)
-			__ieee80211_set_default_beacon_key(sdata,
+			__ieee80211_set_default_beacon_key(link,
 							   new->conf.keyidx);
 	}
 
@@ -569,6 +607,7 @@ ieee80211_key_alloc(u32 cipher, int idx, size_t key_len,
 	key->conf.flags = 0;
 	key->flags = 0;
 
+	key->conf.link_id = -1;
 	key->conf.cipher = cipher;
 	key->conf.keyidx = idx;
 	key->conf.keylen = key_len;
@@ -797,9 +836,10 @@ static bool ieee80211_key_identical(struct ieee80211_sub_if_data *sdata,
 }
 
 int ieee80211_key_link(struct ieee80211_key *key,
-		       struct ieee80211_sub_if_data *sdata,
+		       struct ieee80211_link_data *link,
 		       struct sta_info *sta)
 {
+	struct ieee80211_sub_if_data *sdata = link->sdata;
 	static atomic_t key_color = ATOMIC_INIT(0);
 	struct ieee80211_key *old_key = NULL;
 	int idx = key->conf.keyidx;
@@ -827,15 +867,26 @@ int ieee80211_key_link(struct ieee80211_key *key,
 		    (old_key && old_key->conf.cipher != key->conf.cipher))
 			goto out;
 	} else if (sta) {
-		old_key = key_mtx_dereference(sdata->local,
-					      sta->deflink.gtk[idx]);
+		struct link_sta_info *link_sta = &sta->deflink;
+		int link_id = key->conf.link_id;
+
+		if (link_id >= 0) {
+			link_sta = rcu_dereference_protected(sta->link[link_id],
+							     lockdep_is_held(&sta->local->sta_mtx));
+			if (!link_sta) {
+				ret = -ENOLINK;
+				goto out;
+			}
+		}
+
+		old_key = key_mtx_dereference(sdata->local, link_sta->gtk[idx]);
 	} else {
 		if (idx < NUM_DEFAULT_KEYS)
 			old_key = key_mtx_dereference(sdata->local,
 						      sdata->keys[idx]);
 		if (!old_key)
 			old_key = key_mtx_dereference(sdata->local,
-						      sdata->deflink.gtk[idx]);
+						      link->gtk[idx]);
 	}
 
 	/* Non-pairwise keys must also not switch the cipher on rekey */
@@ -866,7 +917,7 @@ int ieee80211_key_link(struct ieee80211_key *key,
 
 	increment_tailroom_need_count(sdata);
 
-	ret = ieee80211_key_replace(sdata, sta, pairwise, old_key, key);
+	ret = ieee80211_key_replace(sdata, link, sta, pairwise, old_key, key);
 
 	if (!ret) {
 		ieee80211_debugfs_key_add(key);
@@ -890,9 +941,9 @@ void ieee80211_key_free(struct ieee80211_key *key, bool delay_tailroom)
 	 * Replace key with nothingness if it was ever used.
 	 */
 	if (key->sdata)
-		ieee80211_key_replace(key->sdata, key->sta,
-				key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
-				key, NULL);
+		ieee80211_key_replace(key->sdata, NULL, key->sta,
+				      key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
+				      key, NULL);
 	ieee80211_key_destroy(key, delay_tailroom);
 }
 
@@ -1019,15 +1070,45 @@ static void ieee80211_free_keys_iface(struct ieee80211_sub_if_data *sdata,
 	ieee80211_debugfs_key_remove_beacon_default(sdata);
 
 	list_for_each_entry_safe(key, tmp, &sdata->key_list, list) {
-		ieee80211_key_replace(key->sdata, key->sta,
-				key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
-				key, NULL);
+		ieee80211_key_replace(key->sdata, NULL, key->sta,
+				      key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
+				      key, NULL);
 		list_add_tail(&key->list, keys);
 	}
 
 	ieee80211_debugfs_key_update_default(sdata);
 }
 
+void ieee80211_remove_link_keys(struct ieee80211_link_data *link,
+				struct list_head *keys)
+{
+	struct ieee80211_sub_if_data *sdata = link->sdata;
+	struct ieee80211_local *local = sdata->local;
+	struct ieee80211_key *key, *tmp;
+
+	mutex_lock(&local->key_mtx);
+	list_for_each_entry_safe(key, tmp, &sdata->key_list, list) {
+		if (key->conf.link_id != link->link_id)
+			continue;
+		ieee80211_key_replace(key->sdata, link, key->sta,
+				      key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
+				      key, NULL);
+		list_add_tail(&key->list, keys);
+	}
+	mutex_unlock(&local->key_mtx);
+}
+
+void ieee80211_free_key_list(struct ieee80211_local *local,
+			     struct list_head *keys)
+{
+	struct ieee80211_key *key, *tmp;
+
+	mutex_lock(&local->key_mtx);
+	list_for_each_entry_safe(key, tmp, keys, list)
+		__ieee80211_key_destroy(key, false);
+	mutex_unlock(&local->key_mtx);
+}
+
 void ieee80211_free_keys(struct ieee80211_sub_if_data *sdata,
 			 bool force_synchronize)
 {
@@ -1087,9 +1168,9 @@ void ieee80211_free_sta_keys(struct ieee80211_local *local,
 		key = key_mtx_dereference(local, sta->deflink.gtk[i]);
 		if (!key)
 			continue;
-		ieee80211_key_replace(key->sdata, key->sta,
-				key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
-				key, NULL);
+		ieee80211_key_replace(key->sdata, NULL, key->sta,
+				      key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
+				      key, NULL);
 		__ieee80211_key_destroy(key, key->sdata->vif.type ==
 					NL80211_IFTYPE_STATION);
 	}
@@ -1098,9 +1179,9 @@ void ieee80211_free_sta_keys(struct ieee80211_local *local,
 		key = key_mtx_dereference(local, sta->ptk[i]);
 		if (!key)
 			continue;
-		ieee80211_key_replace(key->sdata, key->sta,
-				key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
-				key, NULL);
+		ieee80211_key_replace(key->sdata, NULL, key->sta,
+				      key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE,
+				      key, NULL);
 		__ieee80211_key_destroy(key, key->sdata->vif.type ==
 					NL80211_IFTYPE_STATION);
 	}
@@ -1307,7 +1388,8 @@ ieee80211_gtk_rekey_add(struct ieee80211_vif *vif,
 	if (sdata->u.mgd.mfp != IEEE80211_MFP_DISABLED)
 		key->conf.flags |= IEEE80211_KEY_FLAG_RX_MGMT;
 
-	err = ieee80211_key_link(key, sdata, NULL);
+	/* FIXME: this function needs to get a link ID */
+	err = ieee80211_key_link(key, &sdata->deflink, NULL);
 	if (err)
 		return ERR_PTR(err);
 
@@ -1363,3 +1445,37 @@ void ieee80211_key_replay(struct ieee80211_key_conf *keyconf)
 	}
 }
 EXPORT_SYMBOL_GPL(ieee80211_key_replay);
+
+int ieee80211_key_switch_links(struct ieee80211_sub_if_data *sdata,
+			       unsigned long del_links_mask,
+			       unsigned long add_links_mask)
+{
+	struct ieee80211_key *key;
+	int ret;
+
+	list_for_each_entry(key, &sdata->key_list, list) {
+		if (key->conf.link_id < 0 ||
+		    !(del_links_mask & BIT(key->conf.link_id)))
+			continue;
+
+		/* shouldn't happen for per-link keys */
+		WARN_ON(key->sta);
+
+		ieee80211_key_disable_hw_accel(key);
+	}
+
+	list_for_each_entry(key, &sdata->key_list, list) {
+		if (key->conf.link_id < 0 ||
+		    !(add_links_mask & BIT(key->conf.link_id)))
+			continue;
+
+		/* shouldn't happen for per-link keys */
+		WARN_ON(key->sta);
+
+		ret = ieee80211_key_enable_hw_accel(key);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
diff --git a/net/mac80211/key.h b/net/mac80211/key.h
index e994dcea1ce3..f3df97df4b72 100644
--- a/net/mac80211/key.h
+++ b/net/mac80211/key.h
@@ -22,6 +22,7 @@
 
 struct ieee80211_local;
 struct ieee80211_sub_if_data;
+struct ieee80211_link_data;
 struct sta_info;
 
 /**
@@ -144,22 +145,29 @@ ieee80211_key_alloc(u32 cipher, int idx, size_t key_len,
  * to make it used, free old key. On failure, also free the new key.
  */
 int ieee80211_key_link(struct ieee80211_key *key,
-		       struct ieee80211_sub_if_data *sdata,
+		       struct ieee80211_link_data *link,
 		       struct sta_info *sta);
 int ieee80211_set_tx_key(struct ieee80211_key *key);
 void ieee80211_key_free(struct ieee80211_key *key, bool delay_tailroom);
 void ieee80211_key_free_unused(struct ieee80211_key *key);
-void ieee80211_set_default_key(struct ieee80211_sub_if_data *sdata, int idx,
+void ieee80211_set_default_key(struct ieee80211_link_data *link, int idx,
 			       bool uni, bool multi);
-void ieee80211_set_default_mgmt_key(struct ieee80211_sub_if_data *sdata,
+void ieee80211_set_default_mgmt_key(struct ieee80211_link_data *link,
 				    int idx);
-void ieee80211_set_default_beacon_key(struct ieee80211_sub_if_data *sdata,
+void ieee80211_set_default_beacon_key(struct ieee80211_link_data *link,
 				      int idx);
+void ieee80211_remove_link_keys(struct ieee80211_link_data *link,
+				struct list_head *keys);
+void ieee80211_free_key_list(struct ieee80211_local *local,
+			     struct list_head *keys);
 void ieee80211_free_keys(struct ieee80211_sub_if_data *sdata,
 			 bool force_synchronize);
 void ieee80211_free_sta_keys(struct ieee80211_local *local,
 			     struct sta_info *sta);
 void ieee80211_reenable_keys(struct ieee80211_sub_if_data *sdata);
+int ieee80211_key_switch_links(struct ieee80211_sub_if_data *sdata,
+			       unsigned long del_links_mask,
+			       unsigned long add_links_mask);
 
 #define key_mtx_dereference(local, ref) \
 	rcu_dereference_protected(ref, lockdep_is_held(&((local)->key_mtx)))
diff --git a/net/mac80211/link.c b/net/mac80211/link.c
new file mode 100644
index 000000000000..e309708abae8
--- /dev/null
+++ b/net/mac80211/link.c
@@ -0,0 +1,473 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * MLO link handling
+ *
+ * Copyright (C) 2022 Intel Corporation
+ */
+#include <linux/slab.h>
+#include <linux/kernel.h>
+#include <net/mac80211.h>
+#include "ieee80211_i.h"
+#include "driver-ops.h"
+#include "key.h"
+
+void ieee80211_link_setup(struct ieee80211_link_data *link)
+{
+	if (link->sdata->vif.type == NL80211_IFTYPE_STATION)
+		ieee80211_mgd_setup_link(link);
+}
+
+void ieee80211_link_init(struct ieee80211_sub_if_data *sdata,
+			 int link_id,
+			 struct ieee80211_link_data *link,
+			 struct ieee80211_bss_conf *link_conf)
+{
+	bool deflink = link_id < 0;
+
+	if (link_id < 0)
+		link_id = 0;
+
+	rcu_assign_pointer(sdata->vif.link_conf[link_id], link_conf);
+	rcu_assign_pointer(sdata->link[link_id], link);
+
+	link->sdata = sdata;
+	link->link_id = link_id;
+	link->conf = link_conf;
+	link_conf->link_id = link_id;
+
+	INIT_WORK(&link->csa_finalize_work,
+		  ieee80211_csa_finalize_work);
+	INIT_WORK(&link->color_change_finalize_work,
+		  ieee80211_color_change_finalize_work);
+	INIT_LIST_HEAD(&link->assigned_chanctx_list);
+	INIT_LIST_HEAD(&link->reserved_chanctx_list);
+	INIT_DELAYED_WORK(&link->dfs_cac_timer_work,
+			  ieee80211_dfs_cac_timer_work);
+
+	if (!deflink) {
+		switch (sdata->vif.type) {
+		case NL80211_IFTYPE_AP:
+			ether_addr_copy(link_conf->addr,
+					sdata->wdev.links[link_id].addr);
+			link_conf->bssid = link_conf->addr;
+			WARN_ON(!(sdata->wdev.valid_links & BIT(link_id)));
+			break;
+		case NL80211_IFTYPE_STATION:
+			/* station sets the bssid in ieee80211_mgd_setup_link */
+			break;
+		default:
+			WARN_ON(1);
+		}
+	}
+}
+
+void ieee80211_link_stop(struct ieee80211_link_data *link)
+{
+	if (link->sdata->vif.type == NL80211_IFTYPE_STATION)
+		ieee80211_mgd_stop_link(link);
+
+	ieee80211_link_release_channel(link);
+}
+
+struct link_container {
+	struct ieee80211_link_data data;
+	struct ieee80211_bss_conf conf;
+};
+
+static void ieee80211_tear_down_links(struct ieee80211_sub_if_data *sdata,
+				      struct link_container **links, u16 mask)
+{
+	struct ieee80211_link_data *link;
+	LIST_HEAD(keys);
+	unsigned int link_id;
+
+	for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) {
+		if (!(mask & BIT(link_id)))
+			continue;
+		link = &links[link_id]->data;
+		if (link_id == 0 && !link)
+			link = &sdata->deflink;
+		if (WARN_ON(!link))
+			continue;
+		ieee80211_remove_link_keys(link, &keys);
+		ieee80211_link_stop(link);
+	}
+
+	synchronize_rcu();
+
+	ieee80211_free_key_list(sdata->local, &keys);
+}
+
+static void ieee80211_free_links(struct ieee80211_sub_if_data *sdata,
+				 struct link_container **links)
+{
+	unsigned int link_id;
+
+	for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++)
+		kfree(links[link_id]);
+}
+
+static int ieee80211_check_dup_link_addrs(struct ieee80211_sub_if_data *sdata)
+{
+	unsigned int i, j;
+
+	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
+		struct ieee80211_link_data *link1;
+
+		link1 = sdata_dereference(sdata->link[i], sdata);
+		if (!link1)
+			continue;
+		for (j = i + 1; j < IEEE80211_MLD_MAX_NUM_LINKS; j++) {
+			struct ieee80211_link_data *link2;
+
+			link2 = sdata_dereference(sdata->link[j], sdata);
+			if (!link2)
+				continue;
+
+			if (ether_addr_equal(link1->conf->addr,
+					     link2->conf->addr))
+				return -EALREADY;
+		}
+	}
+
+	return 0;
+}
+
+static void ieee80211_set_vif_links_bitmaps(struct ieee80211_sub_if_data *sdata,
+					    u16 links)
+{
+	sdata->vif.valid_links = links;
+
+	if (!links) {
+		sdata->vif.active_links = 0;
+		return;
+	}
+
+	switch (sdata->vif.type) {
+	case NL80211_IFTYPE_AP:
+		/* in an AP all links are always active */
+		sdata->vif.active_links = links;
+		break;
+	case NL80211_IFTYPE_STATION:
+		if (sdata->vif.active_links)
+			break;
+		WARN_ON(hweight16(links) > 1);
+		sdata->vif.active_links = links;
+		break;
+	default:
+		WARN_ON(1);
+	}
+}
+
+static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata,
+				      struct link_container **to_free,
+				      u16 new_links)
+{
+	u16 old_links = sdata->vif.valid_links;
+	u16 old_active = sdata->vif.active_links;
+	unsigned long add = new_links & ~old_links;
+	unsigned long rem = old_links & ~new_links;
+	unsigned int link_id;
+	int ret;
+	struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS] = {}, *link;
+	struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS];
+	struct ieee80211_link_data *old_data[IEEE80211_MLD_MAX_NUM_LINKS];
+	bool use_deflink = old_links == 0; /* set for error case */
+
+	sdata_assert_lock(sdata);
+
+	memset(to_free, 0, sizeof(links));
+
+	if (old_links == new_links)
+		return 0;
+
+	/* if there were no old links, need to clear the pointers to deflink */
+	if (!old_links)
+		rem |= BIT(0);
+
+	/* allocate new link structures first */
+	for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) {
+		link = kzalloc(sizeof(*link), GFP_KERNEL);
+		if (!link) {
+			ret = -ENOMEM;
+			goto free;
+		}
+		links[link_id] = link;
+	}
+
+	/* keep track of the old pointers for the driver */
+	BUILD_BUG_ON(sizeof(old) != sizeof(sdata->vif.link_conf));
+	memcpy(old, sdata->vif.link_conf, sizeof(old));
+	/* and for us in error cases */
+	BUILD_BUG_ON(sizeof(old_data) != sizeof(sdata->link));
+	memcpy(old_data, sdata->link, sizeof(old_data));
+
+	/* grab old links to free later */
+	for_each_set_bit(link_id, &rem, IEEE80211_MLD_MAX_NUM_LINKS) {
+		if (rcu_access_pointer(sdata->link[link_id]) != &sdata->deflink) {
+			/*
+			 * we must have allocated the data through this path so
+			 * we know we can free both at the same time
+			 */
+			to_free[link_id] = container_of(rcu_access_pointer(sdata->link[link_id]),
+							typeof(*links[link_id]),
+							data);
+		}
+
+		RCU_INIT_POINTER(sdata->link[link_id], NULL);
+		RCU_INIT_POINTER(sdata->vif.link_conf[link_id], NULL);
+	}
+
+	/* link them into data structures */
+	for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) {
+		WARN_ON(!use_deflink &&
+			rcu_access_pointer(sdata->link[link_id]) == &sdata->deflink);
+
+		link = links[link_id];
+		ieee80211_link_init(sdata, link_id, &link->data, &link->conf);
+		ieee80211_link_setup(&link->data);
+	}
+
+	if (new_links == 0)
+		ieee80211_link_init(sdata, -1, &sdata->deflink,
+				    &sdata->vif.bss_conf);
+
+	ret = ieee80211_check_dup_link_addrs(sdata);
+	if (!ret) {
+		/* for keys we will not be able to undo this */
+		ieee80211_tear_down_links(sdata, to_free, rem);
+
+		ieee80211_set_vif_links_bitmaps(sdata, new_links);
+
+		/* tell the driver */
+		ret = drv_change_vif_links(sdata->local, sdata,
+					   old_links & old_active,
+					   new_links & sdata->vif.active_links,
+					   old);
+	}
+
+	if (ret) {
+		/* restore config */
+		memcpy(sdata->link, old_data, sizeof(old_data));
+		memcpy(sdata->vif.link_conf, old, sizeof(old));
+		ieee80211_set_vif_links_bitmaps(sdata, old_links);
+		/* and free (only) the newly allocated links */
+		memset(to_free, 0, sizeof(links));
+		goto free;
+	}
+
+	/* use deflink/bss_conf again if and only if there are no more links */
+	use_deflink = new_links == 0;
+
+	goto deinit;
+free:
+	/* if we failed during allocation, only free all */
+	for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) {
+		kfree(links[link_id]);
+		links[link_id] = NULL;
+	}
+deinit:
+	if (use_deflink)
+		ieee80211_link_init(sdata, -1, &sdata->deflink,
+				    &sdata->vif.bss_conf);
+	return ret;
+}
+
+int ieee80211_vif_set_links(struct ieee80211_sub_if_data *sdata,
+			    u16 new_links)
+{
+	struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS];
+	int ret;
+
+	ret = ieee80211_vif_update_links(sdata, links, new_links);
+	ieee80211_free_links(sdata, links);
+
+	return ret;
+}
+
+void ieee80211_vif_clear_links(struct ieee80211_sub_if_data *sdata)
+{
+	struct link_container *links[IEEE80211_MLD_MAX_NUM_LINKS];
+
+	/*
+	 * The locking here is different because when we free links
+	 * in the station case we need to be able to cancel_work_sync()
+	 * something that also takes the lock.
+	 */
+
+	sdata_lock(sdata);
+	ieee80211_vif_update_links(sdata, links, 0);
+	sdata_unlock(sdata);
+
+	ieee80211_free_links(sdata, links);
+}
+
+static int _ieee80211_set_active_links(struct ieee80211_sub_if_data *sdata,
+				       u16 active_links)
+{
+	struct ieee80211_bss_conf *link_confs[IEEE80211_MLD_MAX_NUM_LINKS];
+	struct ieee80211_local *local = sdata->local;
+	u16 old_active = sdata->vif.active_links;
+	unsigned long rem = old_active & ~active_links;
+	unsigned long add = active_links & ~old_active;
+	struct sta_info *sta;
+	unsigned int link_id;
+	int ret, i;
+
+	if (!ieee80211_sdata_running(sdata))
+		return -ENETDOWN;
+
+	if (sdata->vif.type != NL80211_IFTYPE_STATION)
+		return -EINVAL;
+
+	/* cannot activate links that don't exist */
+	if (active_links & ~sdata->vif.valid_links)
+		return -EINVAL;
+
+	/* nothing to do */
+	if (old_active == active_links)
+		return 0;
+
+	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++)
+		link_confs[i] = sdata_dereference(sdata->vif.link_conf[i],
+						  sdata);
+
+	if (add) {
+		sdata->vif.active_links |= active_links;
+		ret = drv_change_vif_links(local, sdata,
+					   old_active,
+					   sdata->vif.active_links,
+					   link_confs);
+		if (ret) {
+			sdata->vif.active_links = old_active;
+			return ret;
+		}
+	}
+
+	for_each_set_bit(link_id, &rem, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_link_data *link;
+
+		link = sdata_dereference(sdata->link[link_id], sdata);
+
+		/* FIXME: kill TDLS connections on the link */
+
+		ieee80211_link_release_channel(link);
+	}
+
+	list_for_each_entry(sta, &local->sta_list, list) {
+		if (sdata != sta->sdata)
+			continue;
+		ret = drv_change_sta_links(local, sdata, &sta->sta,
+					   old_active,
+					   old_active | active_links);
+		WARN_ON_ONCE(ret);
+	}
+
+	ret = ieee80211_key_switch_links(sdata, rem, add);
+	WARN_ON_ONCE(ret);
+
+	list_for_each_entry(sta, &local->sta_list, list) {
+		if (sdata != sta->sdata)
+			continue;
+		ret = drv_change_sta_links(local, sdata, &sta->sta,
+					   old_active | active_links,
+					   active_links);
+		WARN_ON_ONCE(ret);
+	}
+
+	for_each_set_bit(link_id, &add, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_link_data *link;
+
+		link = sdata_dereference(sdata->link[link_id], sdata);
+
+		ret = ieee80211_link_use_channel(link, &link->conf->chandef,
+						 IEEE80211_CHANCTX_SHARED);
+		WARN_ON_ONCE(ret);
+
+		ieee80211_link_info_change_notify(sdata, link,
+						  BSS_CHANGED_ERP_CTS_PROT |
+						  BSS_CHANGED_ERP_PREAMBLE |
+						  BSS_CHANGED_ERP_SLOT |
+						  BSS_CHANGED_HT |
+						  BSS_CHANGED_BASIC_RATES |
+						  BSS_CHANGED_BSSID |
+						  BSS_CHANGED_CQM |
+						  BSS_CHANGED_QOS |
+						  BSS_CHANGED_TXPOWER |
+						  BSS_CHANGED_BANDWIDTH |
+						  BSS_CHANGED_TWT |
+						  BSS_CHANGED_HE_OBSS_PD |
+						  BSS_CHANGED_HE_BSS_COLOR);
+		ieee80211_mgd_set_link_qos_params(link);
+	}
+
+	old_active = sdata->vif.active_links;
+	sdata->vif.active_links = active_links;
+
+	if (rem) {
+		ret = drv_change_vif_links(local, sdata, old_active,
+					   active_links, link_confs);
+		WARN_ON_ONCE(ret);
+	}
+
+	return 0;
+}
+
+int ieee80211_set_active_links(struct ieee80211_vif *vif, u16 active_links)
+{
+	struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
+	struct ieee80211_local *local = sdata->local;
+	u16 old_active;
+	int ret;
+
+	sdata_lock(sdata);
+	mutex_lock(&local->sta_mtx);
+	mutex_lock(&local->mtx);
+	mutex_lock(&local->key_mtx);
+	old_active = sdata->vif.active_links;
+	if (old_active & active_links) {
+		/*
+		 * if there's at least one link that stays active across
+		 * the change then switch to it (to those) first, and
+		 * then enable the additional links
+		 */
+		ret = _ieee80211_set_active_links(sdata,
+						  old_active & active_links);
+		if (!ret)
+			ret = _ieee80211_set_active_links(sdata, active_links);
+	} else {
+		/* otherwise switch directly */
+		ret = _ieee80211_set_active_links(sdata, active_links);
+	}
+	mutex_unlock(&local->key_mtx);
+	mutex_unlock(&local->mtx);
+	mutex_unlock(&local->sta_mtx);
+	sdata_unlock(sdata);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(ieee80211_set_active_links);
+
+void ieee80211_set_active_links_async(struct ieee80211_vif *vif,
+				      u16 active_links)
+{
+	struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
+
+	if (!ieee80211_sdata_running(sdata))
+		return;
+
+	if (sdata->vif.type != NL80211_IFTYPE_STATION)
+		return;
+
+	/* cannot activate links that don't exist */
+	if (active_links & ~sdata->vif.valid_links)
+		return;
+
+	/* nothing to do */
+	if (sdata->vif.active_links == active_links)
+		return;
+
+	sdata->desired_active_links = active_links;
+	schedule_work(&sdata->activate_links_work);
+}
+EXPORT_SYMBOL_GPL(ieee80211_set_active_links_async);
diff --git a/net/mac80211/main.c b/net/mac80211/main.c
index 5b1c47ed0cc0..46f3eddc2388 100644
--- a/net/mac80211/main.c
+++ b/net/mac80211/main.c
@@ -699,6 +699,8 @@ struct ieee80211_hw *ieee80211_alloc_hw_nm(size_t priv_data_len,
 			      NL80211_EXT_FEATURE_CONTROL_PORT_OVER_NL80211_TX_STATUS);
 	wiphy_ext_feature_set(wiphy,
 			      NL80211_EXT_FEATURE_SCAN_FREQ_KHZ);
+	wiphy_ext_feature_set(wiphy,
+			      NL80211_EXT_FEATURE_POWERED_ADDR_CHANGE);
 
 	if (!ops->hw_scan) {
 		wiphy->features |= NL80211_FEATURE_LOW_PRIORITY_SCAN |
diff --git a/net/mac80211/mesh.c b/net/mac80211/mesh.c
index 6991c4c479da..5a99b8f6e465 100644
--- a/net/mac80211/mesh.c
+++ b/net/mac80211/mesh.c
@@ -634,7 +634,7 @@ int mesh_add_he_6ghz_cap_ie(struct ieee80211_sub_if_data *sdata,
 	if (!iftd)
 		return 0;
 
-	ieee80211_ie_build_he_6ghz_cap(sdata, skb);
+	ieee80211_ie_build_he_6ghz_cap(sdata, sdata->deflink.smps_mode, skb);
 	return 0;
 }
 
diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c
index 5265d2b6db12..54b8d5065bbd 100644
--- a/net/mac80211/mlme.c
+++ b/net/mac80211/mlme.c
@@ -314,7 +314,7 @@ ieee80211_determine_chantype(struct ieee80211_sub_if_data *sdata,
 	if (eht_oper && (eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT)) {
 		struct cfg80211_chan_def eht_chandef = *chandef;
 
-		ieee80211_chandef_eht_oper(sdata, eht_oper,
+		ieee80211_chandef_eht_oper(eht_oper,
 					   eht_chandef.width ==
 					   NL80211_CHAN_WIDTH_160,
 					   false, &eht_chandef);
@@ -695,6 +695,7 @@ static bool ieee80211_add_vht_ie(struct ieee80211_sub_if_data *sdata,
 static void ieee80211_add_he_ie(struct ieee80211_sub_if_data *sdata,
 				struct sk_buff *skb,
 				struct ieee80211_supported_band *sband,
+				enum ieee80211_smps_mode smps_mode,
 				ieee80211_conn_flags_t conn_flags)
 {
 	u8 *pos, *pre_he_pos;
@@ -719,7 +720,7 @@ static void ieee80211_add_he_ie(struct ieee80211_sub_if_data *sdata,
 	/* trim excess if any */
 	skb_trim(skb, skb->len - (pre_he_pos + he_cap_size - pos));
 
-	ieee80211_ie_build_he_6ghz_cap(sdata, skb);
+	ieee80211_ie_build_he_6ghz_cap(sdata, smps_mode, skb);
 }
 
 static void ieee80211_add_eht_ie(struct ieee80211_sub_if_data *sdata,
@@ -746,11 +747,13 @@ static void ieee80211_add_eht_ie(struct ieee80211_sub_if_data *sdata,
 	eht_cap_size =
 		2 + 1 + sizeof(eht_cap->eht_cap_elem) +
 		ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem,
-					   &eht_cap->eht_cap_elem) +
+					   &eht_cap->eht_cap_elem,
+					   false) +
 		ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0],
 				       eht_cap->eht_cap_elem.phy_cap_info);
 	pos = skb_put(skb, eht_cap_size);
-	ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, pos + eht_cap_size);
+	ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, pos + eht_cap_size,
+				   false);
 }
 
 static void ieee80211_assoc_add_rates(struct sk_buff *skb,
@@ -1098,7 +1101,7 @@ static size_t ieee80211_assoc_link_elems(struct ieee80211_sub_if_data *sdata,
 					       offset);
 
 	if (!(assoc_data->link[link_id].conn_flags & IEEE80211_CONN_DISABLE_HE)) {
-		ieee80211_add_he_ie(sdata, skb, sband,
+		ieee80211_add_he_ie(sdata, skb, sband, smps_mode,
 				    assoc_data->link[link_id].conn_flags);
 		ADD_PRESENT_EXT_ELEM(WLAN_EID_EXT_HE_CAPABILITY);
 	}
@@ -1220,14 +1223,21 @@ static void ieee80211_assoc_add_ml_elem(struct ieee80211_sub_if_data *sdata,
 	ml_elem = skb_put(skb, sizeof(*ml_elem));
 	ml_elem->control =
 		cpu_to_le16(IEEE80211_ML_CONTROL_TYPE_BASIC |
-			    IEEE80211_MLC_BASIC_PRES_EML_CAPA |
 			    IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP);
 	common = skb_put(skb, sizeof(*common));
 	common->len = sizeof(*common) +
-		      2 + /* EML capabilities */
 		      2;  /* MLD capa/ops */
 	memcpy(common->mld_mac_addr, sdata->vif.addr, ETH_ALEN);
-	skb_put_data(skb, &eml_capa, sizeof(eml_capa));
+
+	/* add EML_CAPA only if needed, see Draft P802.11be_D2.1, 35.3.17 */
+	if (eml_capa &
+	    cpu_to_le16((IEEE80211_EML_CAP_EMLSR_SUPP |
+			 IEEE80211_EML_CAP_EMLMR_SUPPORT))) {
+		common->len += 2; /* EML capabilities */
+		ml_elem->control |=
+			cpu_to_le16(IEEE80211_MLC_BASIC_PRES_EML_CAPA);
+		skb_put_data(skb, &eml_capa, sizeof(eml_capa));
+	}
 	/* need indication from userspace to support this */
 	mld_capa_ops &= ~cpu_to_le16(IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP);
 	skb_put_data(skb, &mld_capa_ops, sizeof(mld_capa_ops));
@@ -1536,8 +1546,9 @@ void ieee80211_send_nullfunc(struct ieee80211_local *local,
 	struct ieee80211_hdr_3addr *nullfunc;
 	struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
 
-	skb = ieee80211_nullfunc_get(&local->hw, &sdata->vif,
-		!ieee80211_hw_check(&local->hw, DOESNT_SUPPORT_QOS_NDP));
+	skb = ieee80211_nullfunc_get(&local->hw, &sdata->vif, -1,
+				     !ieee80211_hw_check(&local->hw,
+							 DOESNT_SUPPORT_QOS_NDP));
 	if (!skb)
 		return;
 
@@ -1902,7 +1913,7 @@ ieee80211_sta_process_chanswitch(struct ieee80211_link_data *link,
 					  IEEE80211_QUEUE_STOP_REASON_CSA);
 	mutex_unlock(&local->mtx);
 
-	cfg80211_ch_switch_started_notify(sdata->dev, &csa_ie.chandef,
+	cfg80211_ch_switch_started_notify(sdata->dev, &csa_ie.chandef, 0,
 					  csa_ie.count, csa_ie.mode);
 
 	if (local->ops->channel_switch) {
@@ -2435,6 +2446,29 @@ static void ieee80211_sta_handle_tspec_ac_params_wk(struct work_struct *work)
 	ieee80211_sta_handle_tspec_ac_params(sdata);
 }
 
+void ieee80211_mgd_set_link_qos_params(struct ieee80211_link_data *link)
+{
+	struct ieee80211_sub_if_data *sdata = link->sdata;
+	struct ieee80211_local *local = sdata->local;
+	struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
+	struct ieee80211_tx_queue_params *params = link->tx_conf;
+	u8 ac;
+
+	for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
+		mlme_dbg(sdata,
+			 "WMM AC=%d acm=%d aifs=%d cWmin=%d cWmax=%d txop=%d uapsd=%d, downgraded=%d\n",
+			 ac, params[ac].acm,
+			 params[ac].aifs, params[ac].cw_min, params[ac].cw_max,
+			 params[ac].txop, params[ac].uapsd,
+			 ifmgd->tx_tspec[ac].downgraded);
+		if (!ifmgd->tx_tspec[ac].downgraded &&
+		    drv_conf_tx(local, link, ac, &params[ac]))
+			link_err(link,
+				 "failed to set TX queue parameters for AC %d\n",
+				 ac);
+	}
+}
+
 /* MLME */
 static bool
 ieee80211_sta_wmm_params(struct ieee80211_local *local,
@@ -2566,20 +2600,10 @@ ieee80211_sta_wmm_params(struct ieee80211_local *local,
 		}
 	}
 
-	for (ac = 0; ac < IEEE80211_NUM_ACS; ac++) {
-		mlme_dbg(sdata,
-			 "WMM AC=%d acm=%d aifs=%d cWmin=%d cWmax=%d txop=%d uapsd=%d, downgraded=%d\n",
-			 ac, params[ac].acm,
-			 params[ac].aifs, params[ac].cw_min, params[ac].cw_max,
-			 params[ac].txop, params[ac].uapsd,
-			 ifmgd->tx_tspec[ac].downgraded);
+	for (ac = 0; ac < IEEE80211_NUM_ACS; ac++)
 		link->tx_conf[ac] = params[ac];
-		if (!ifmgd->tx_tspec[ac].downgraded &&
-		    drv_conf_tx(local, link, ac, &params[ac]))
-			link_err(link,
-				 "failed to set TX queue parameters for AC %d\n",
-				 ac);
-	}
+
+	ieee80211_mgd_set_link_qos_params(link);
 
 	/* enable WMM or activate new settings */
 	link->conf->qos = true;
@@ -3904,6 +3928,7 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link,
 		.len = elem_len,
 		.bss = cbss,
 		.link_id = link == &sdata->deflink ? -1 : link->link_id,
+		.from_ap = true,
 	};
 	bool is_6ghz = cbss->channel->band == NL80211_BAND_6GHZ;
 	bool is_s1g = cbss->channel->band == NL80211_BAND_S1GHZ;
@@ -4032,15 +4057,14 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link,
 		goto out;
 	}
 
-	sband = ieee80211_get_link_sband(link);
-	if (!sband) {
+	if (WARN_ON(!link->conf->chandef.chan)) {
 		ret = false;
 		goto out;
 	}
+	sband = local->hw.wiphy->bands[link->conf->chandef.chan->band];
 
 	if (!(link->u.mgd.conn_flags & IEEE80211_CONN_DISABLE_HE) &&
 	    (!elems->he_cap || !elems->he_operation)) {
-		mutex_unlock(&sdata->local->sta_mtx);
 		sdata_info(sdata,
 			   "HE AP is missing HE capability/operation\n");
 		ret = false;
@@ -4572,6 +4596,11 @@ static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata,
 	bool is_6ghz = cbss->channel->band == NL80211_BAND_6GHZ;
 	bool is_5ghz = cbss->channel->band == NL80211_BAND_5GHZ;
 	struct ieee80211_bss *bss = (void *)cbss->priv;
+	struct ieee80211_elems_parse_params parse_params = {
+		.bss = cbss,
+		.link_id = -1,
+		.from_ap = true,
+	};
 	struct ieee802_11_elems *elems;
 	const struct cfg80211_bss_ies *ies;
 	int ret;
@@ -4581,7 +4610,9 @@ static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata,
 	rcu_read_lock();
 
 	ies = rcu_dereference(cbss->ies);
-	elems = ieee802_11_parse_elems(ies->data, ies->len, false, cbss);
+	parse_params.start = ies->data;
+	parse_params.len = ies->len;
+	elems = ieee802_11_parse_elems_full(&parse_params);
 	if (!elems) {
 		rcu_read_unlock();
 		return -ENOMEM;
@@ -4787,6 +4818,40 @@ static int ieee80211_prep_channel(struct ieee80211_sub_if_data *sdata,
 	return ret;
 }
 
+static bool ieee80211_get_dtim(const struct cfg80211_bss_ies *ies,
+			       u8 *dtim_count, u8 *dtim_period)
+{
+	const u8 *tim_ie = cfg80211_find_ie(WLAN_EID_TIM, ies->data, ies->len);
+	const u8 *idx_ie = cfg80211_find_ie(WLAN_EID_MULTI_BSSID_IDX, ies->data,
+					 ies->len);
+	const struct ieee80211_tim_ie *tim = NULL;
+	const struct ieee80211_bssid_index *idx;
+	bool valid = tim_ie && tim_ie[1] >= 2;
+
+	if (valid)
+		tim = (void *)(tim_ie + 2);
+
+	if (dtim_count)
+		*dtim_count = valid ? tim->dtim_count : 0;
+
+	if (dtim_period)
+		*dtim_period = valid ? tim->dtim_period : 0;
+
+	/* Check if value is overridden by non-transmitted profile */
+	if (!idx_ie || idx_ie[1] < 3)
+		return valid;
+
+	idx = (void *)(idx_ie + 2);
+
+	if (dtim_count)
+		*dtim_count = idx->dtim_count;
+
+	if (dtim_period)
+		*dtim_period = idx->dtim_period;
+
+	return true;
+}
+
 static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata,
 				    struct ieee80211_mgmt *mgmt,
 				    struct ieee802_11_elems *elems,
@@ -4850,11 +4915,24 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata,
 			goto out_err;
 
 		if (link_id != assoc_data->assoc_link_id) {
-			err = ieee80211_prep_channel(sdata, link,
-						     assoc_data->link[link_id].bss,
+			struct cfg80211_bss *cbss = assoc_data->link[link_id].bss;
+			const struct cfg80211_bss_ies *ies;
+
+			rcu_read_lock();
+			ies = rcu_dereference(cbss->ies);
+			ieee80211_get_dtim(ies,
+					   &link->conf->sync_dtim_count,
+					   &link->u.mgd.dtim_period);
+			link->conf->dtim_period = link->u.mgd.dtim_period ?: 1;
+			link->conf->beacon_int = cbss->beacon_interval;
+			rcu_read_unlock();
+
+			err = ieee80211_prep_channel(sdata, link, cbss,
 						     &link->u.mgd.conn_flags);
-			if (err)
+			if (err) {
+				link_info(link, "prep_channel failed\n");
 				goto out_err;
+			}
 		}
 
 		err = ieee80211_mgd_setup_link_sta(link, sta, link_sta,
@@ -4936,6 +5014,11 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata,
 	struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
 	struct ieee80211_mgd_assoc_data *assoc_data = ifmgd->assoc_data;
 	u16 capab_info, status_code, aid;
+	struct ieee80211_elems_parse_params parse_params = {
+		.bss = NULL,
+		.link_id = -1,
+		.from_ap = true,
+	};
 	struct ieee802_11_elems *elems;
 	int ac;
 	const u8 *elem_start;
@@ -4990,7 +5073,9 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata,
 		return;
 
 	elem_len = len - (elem_start - (u8 *)mgmt);
-	elems = ieee802_11_parse_elems(elem_start, elem_len, false, NULL);
+	parse_params.start = elem_start;
+	parse_params.len = elem_len;
+	elems = ieee802_11_parse_elems_full(&parse_params);
 	if (!elems)
 		goto notify_driver;
 
@@ -5123,7 +5208,7 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata,
 	resp.req_ies = ifmgd->assoc_req_ies;
 	resp.req_ies_len = ifmgd->assoc_req_ies_len;
 	if (sdata->vif.valid_links)
-		resp.ap_mld_addr = assoc_data->ap_addr;
+		resp.ap_mld_addr = sdata->vif.cfg.ap_addr;
 	cfg80211_rx_assoc_resp(sdata->dev, &resp);
 notify_driver:
 	drv_mgd_complete_tx(sdata->local, sdata, &info);
@@ -5355,6 +5440,10 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link,
 	u32 ncrc = 0;
 	u8 *bssid, *variable = mgmt->u.beacon.variable;
 	u8 deauth_buf[IEEE80211_DEAUTH_FRAME_LEN];
+	struct ieee80211_elems_parse_params parse_params = {
+		.link_id = -1,
+		.from_ap = true,
+	};
 
 	sdata_assert_lock(sdata);
 
@@ -5373,6 +5462,9 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link,
 	if (baselen > len)
 		return;
 
+	parse_params.start = variable;
+	parse_params.len = len - baselen;
+
 	rcu_read_lock();
 	chanctx_conf = rcu_dereference(link->conf->chanctx_conf);
 	if (!chanctx_conf) {
@@ -5391,8 +5483,8 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link,
 	if (ifmgd->assoc_data && ifmgd->assoc_data->need_beacon &&
 	    !WARN_ON(sdata->vif.valid_links) &&
 	    ieee80211_rx_our_beacon(bssid, ifmgd->assoc_data->link[0].bss)) {
-		elems = ieee802_11_parse_elems(variable, len - baselen, false,
-					       ifmgd->assoc_data->link[0].bss);
+		parse_params.bss = ifmgd->assoc_data->link[0].bss;
+		elems = ieee802_11_parse_elems_full(&parse_params);
 		if (!elems)
 			return;
 
@@ -5458,9 +5550,10 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link,
 	 */
 	if (!ieee80211_is_s1g_beacon(hdr->frame_control))
 		ncrc = crc32_be(0, (void *)&mgmt->u.beacon.beacon_int, 4);
-	elems = ieee802_11_parse_elems_crc(variable, len - baselen,
-					   false, care_about_ies, ncrc,
-					   link->u.mgd.bss);
+	parse_params.bss = link->u.mgd.bss;
+	parse_params.filter = care_about_ies;
+	parse_params.crc = ncrc;
+	elems = ieee802_11_parse_elems_full(&parse_params);
 	if (!elems)
 		return;
 	ncrc = elems->crc;
@@ -5589,12 +5682,16 @@ static void ieee80211_rx_mgmt_beacon(struct ieee80211_link_data *link,
 
 	mutex_lock(&local->sta_mtx);
 	sta = sta_info_get(sdata, sdata->vif.cfg.ap_addr);
-	if (WARN_ON(!sta))
+	if (WARN_ON(!sta)) {
+		mutex_unlock(&local->sta_mtx);
 		goto free;
+	}
 	link_sta = rcu_dereference_protected(sta->link[link->link_id],
 					     lockdep_is_held(&local->sta_mtx));
-	if (WARN_ON(!link_sta))
+	if (WARN_ON(!link_sta)) {
+		mutex_unlock(&local->sta_mtx);
 		goto free;
+	}
 
 	changed |= ieee80211_recalc_twt_req(link, link_sta, elems);
 
@@ -5670,6 +5767,13 @@ void ieee80211_sta_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata,
 
 	sdata_lock(sdata);
 
+	if (rx_status->link_valid) {
+		link = sdata_dereference(sdata->link[rx_status->link_id],
+					 sdata);
+		if (!link)
+			goto out;
+	}
+
 	switch (fc & IEEE80211_FCTL_STYPE) {
 	case IEEE80211_STYPE_BEACON:
 		ieee80211_rx_mgmt_beacon(link, (void *)mgmt,
@@ -5746,6 +5850,7 @@ void ieee80211_sta_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata,
 		}
 		break;
 	}
+out:
 	sdata_unlock(sdata);
 }
 
@@ -6281,6 +6386,8 @@ void ieee80211_mgd_setup_link(struct ieee80211_link_data *link)
 	if (sdata->u.mgd.assoc_data)
 		ether_addr_copy(link->conf->addr,
 				sdata->u.mgd.assoc_data->link[link_id].addr);
+	else if (!is_valid_ether_addr(link->conf->addr))
+		eth_random_addr(link->conf->addr);
 }
 
 /* scan finished notification */
@@ -6297,40 +6404,6 @@ void ieee80211_mlme_notify_scan_completed(struct ieee80211_local *local)
 	rcu_read_unlock();
 }
 
-static bool ieee80211_get_dtim(const struct cfg80211_bss_ies *ies,
-			       u8 *dtim_count, u8 *dtim_period)
-{
-	const u8 *tim_ie = cfg80211_find_ie(WLAN_EID_TIM, ies->data, ies->len);
-	const u8 *idx_ie = cfg80211_find_ie(WLAN_EID_MULTI_BSSID_IDX, ies->data,
-					 ies->len);
-	const struct ieee80211_tim_ie *tim = NULL;
-	const struct ieee80211_bssid_index *idx;
-	bool valid = tim_ie && tim_ie[1] >= 2;
-
-	if (valid)
-		tim = (void *)(tim_ie + 2);
-
-	if (dtim_count)
-		*dtim_count = valid ? tim->dtim_count : 0;
-
-	if (dtim_period)
-		*dtim_period = valid ? tim->dtim_period : 0;
-
-	/* Check if value is overridden by non-transmitted profile */
-	if (!idx_ie || idx_ie[1] < 3)
-		return valid;
-
-	idx = (void *)(idx_ie + 2);
-
-	if (dtim_count)
-		*dtim_count = idx->dtim_count;
-
-	if (dtim_period)
-		*dtim_period = idx->dtim_period;
-
-	return true;
-}
-
 static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata,
 				     struct cfg80211_bss *cbss, s8 link_id,
 				     const u8 *ap_mld_addr, bool assoc,
@@ -6368,9 +6441,6 @@ static int ieee80211_prep_connection(struct ieee80211_sub_if_data *sdata,
 		goto out_err;
 	}
 
-	if (mlo && !is_valid_ether_addr(link->conf->addr))
-		eth_random_addr(link->conf->addr);
-
 	if (WARN_ON(!ifmgd->auth_data && !ifmgd->assoc_data)) {
 		err = -EINVAL;
 		goto out_err;
@@ -6836,22 +6906,9 @@ int ieee80211_mgd_assoc(struct ieee80211_sub_if_data *sdata,
 	for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++)
 		size += req->links[i].elems_len;
 
-	if (req->ap_mld_addr) {
-		for (i = 0; i < IEEE80211_MLD_MAX_NUM_LINKS; i++) {
-			if (!req->links[i].bss)
-				continue;
-			if (i == assoc_link_id)
-				continue;
-			/*
-			 * For now, support only a single link in MLO, we
-			 * don't have the necessary parsing of the multi-
-			 * link element in the association response, etc.
-			 */
-			sdata_info(sdata,
-				   "refusing MLO association with >1 links\n");
-			return -EINVAL;
-		}
-	}
+	/* FIXME: no support for 4-addr MLO yet */
+	if (sdata->u.mgd.use_4addr && req->link_id >= 0)
+		return -EOPNOTSUPP;
 
 	assoc_data = kzalloc(size, GFP_KERNEL);
 	if (!assoc_data)
diff --git a/net/mac80211/rc80211_minstrel_ht.c b/net/mac80211/rc80211_minstrel_ht.c
index 5f27e6746762..7f3f5f51081d 100644
--- a/net/mac80211/rc80211_minstrel_ht.c
+++ b/net/mac80211/rc80211_minstrel_ht.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * Copyright (C) 2010-2013 Felix Fietkau <nbd@openwrt.org>
- * Copyright (C) 2019-2021 Intel Corporation
+ * Copyright (C) 2019-2022 Intel Corporation
  */
 #include <linux/netdevice.h>
 #include <linux/types.h>
@@ -10,6 +10,7 @@
 #include <linux/random.h>
 #include <linux/moduleparam.h>
 #include <linux/ieee80211.h>
+#include <linux/minmax.h>
 #include <net/mac80211.h>
 #include "rate.h"
 #include "sta_info.h"
@@ -1478,7 +1479,7 @@ minstrel_ht_set_rate(struct minstrel_priv *mp, struct minstrel_ht_sta *mi,
 	 *  - for fallback rates, to increase chances of getting through
 	 */
 	if (offset > 0 ||
-	    (mi->sta->smps_mode == IEEE80211_SMPS_DYNAMIC &&
+	    (mi->sta->deflink.smps_mode == IEEE80211_SMPS_DYNAMIC &&
 	     group->streams > 1)) {
 		ratetbl->rate[offset].count = ratetbl->rate[offset].count_rts;
 		flags |= IEEE80211_TX_RC_USE_RTS_CTS;
@@ -1550,6 +1551,7 @@ minstrel_ht_update_rates(struct minstrel_priv *mp, struct minstrel_ht_sta *mi)
 {
 	struct ieee80211_sta_rates *rates;
 	int i = 0;
+	int max_rates = min_t(int, mp->hw->max_rates, IEEE80211_TX_RATE_TABLE_SIZE);
 
 	rates = kzalloc(sizeof(*rates), GFP_ATOMIC);
 	if (!rates)
@@ -1559,16 +1561,17 @@ minstrel_ht_update_rates(struct minstrel_priv *mp, struct minstrel_ht_sta *mi)
 	minstrel_ht_set_rate(mp, mi, rates, i++, mi->max_tp_rate[0]);
 
 	/* Fill up remaining, keep one entry for max_probe_rate */
-	for (; i < (mp->hw->max_rates - 1); i++)
+	for (; i < (max_rates - 1); i++)
 		minstrel_ht_set_rate(mp, mi, rates, i, mi->max_tp_rate[i]);
 
-	if (i < mp->hw->max_rates)
+	if (i < max_rates)
 		minstrel_ht_set_rate(mp, mi, rates, i++, mi->max_prob_rate);
 
 	if (i < IEEE80211_TX_RATE_TABLE_SIZE)
 		rates->rate[i].idx = -1;
 
-	mi->sta->max_rc_amsdu_len = minstrel_ht_get_max_amsdu_len(mi);
+	mi->sta->deflink.agg.max_rc_amsdu_len = minstrel_ht_get_max_amsdu_len(mi);
+	ieee80211_sta_recalc_aggregates(mi->sta);
 	rate_control_set_rates(mp->hw, mi->sta, rates);
 }
 
@@ -1779,7 +1782,7 @@ minstrel_ht_update_caps(void *priv, struct ieee80211_supported_band *sband,
 		nss = minstrel_mcs_groups[i].streams;
 
 		/* Mark MCS > 7 as unsupported if STA is in static SMPS mode */
-		if (sta->smps_mode == IEEE80211_SMPS_STATIC && nss > 1)
+		if (sta->deflink.smps_mode == IEEE80211_SMPS_STATIC && nss > 1)
 			continue;
 
 		/* HT rate */
diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c
index 45d7e71661e3..bd215fe3c796 100644
--- a/net/mac80211/rx.c
+++ b/net/mac80211/rx.c
@@ -49,7 +49,7 @@ static struct sk_buff *ieee80211_clean_skb(struct sk_buff *skb,
 
 	if (present_fcs_len)
 		__pskb_trim(skb, skb->len - present_fcs_len);
-	__pskb_pull(skb, rtap_space);
+	pskb_pull(skb, rtap_space);
 
 	hdr = (void *)skb->data;
 	fc = hdr->frame_control;
@@ -74,7 +74,7 @@ static struct sk_buff *ieee80211_clean_skb(struct sk_buff *skb,
 
 	memmove(skb->data + IEEE80211_HT_CTL_LEN, skb->data,
 		hdrlen - IEEE80211_HT_CTL_LEN);
-	__pskb_pull(skb, IEEE80211_HT_CTL_LEN);
+	pskb_pull(skb, IEEE80211_HT_CTL_LEN);
 
 	return skb;
 }
@@ -215,9 +215,19 @@ ieee80211_rx_radiotap_hdrlen(struct ieee80211_local *local,
 }
 
 static void __ieee80211_queue_skb_to_iface(struct ieee80211_sub_if_data *sdata,
+					   int link_id,
 					   struct sta_info *sta,
 					   struct sk_buff *skb)
 {
+	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
+
+	if (link_id >= 0) {
+		status->link_valid = 1;
+		status->link_id = link_id;
+	} else {
+		status->link_valid = 0;
+	}
+
 	skb_queue_tail(&sdata->skb_queue, skb);
 	ieee80211_queue_work(&sdata->local->hw, &sdata->work);
 	if (sta)
@@ -225,11 +235,12 @@ static void __ieee80211_queue_skb_to_iface(struct ieee80211_sub_if_data *sdata,
 }
 
 static void ieee80211_queue_skb_to_iface(struct ieee80211_sub_if_data *sdata,
+					 int link_id,
 					 struct sta_info *sta,
 					 struct sk_buff *skb)
 {
 	skb->protocol = 0;
-	__ieee80211_queue_skb_to_iface(sdata, sta, skb);
+	__ieee80211_queue_skb_to_iface(sdata, link_id, sta, skb);
 }
 
 static void ieee80211_handle_mu_mimo_mon(struct ieee80211_sub_if_data *sdata,
@@ -272,7 +283,7 @@ static void ieee80211_handle_mu_mimo_mon(struct ieee80211_sub_if_data *sdata,
 	if (!skb)
 		return;
 
-	ieee80211_queue_skb_to_iface(sdata, NULL, skb);
+	ieee80211_queue_skb_to_iface(sdata, -1, NULL, skb);
 }
 
 /*
@@ -1394,7 +1405,7 @@ static void ieee80211_rx_reorder_ampdu(struct ieee80211_rx_data *rx,
 	/* if this mpdu is fragmented - terminate rx aggregation session */
 	sc = le16_to_cpu(hdr->seq_ctrl);
 	if (sc & IEEE80211_SCTL_FRAG) {
-		ieee80211_queue_skb_to_iface(rx->sdata, NULL, skb);
+		ieee80211_queue_skb_to_iface(rx->sdata, rx->link_id, NULL, skb);
 		return;
 	}
 
@@ -1441,7 +1452,7 @@ ieee80211_rx_h_check_dup(struct ieee80211_rx_data *rx)
 	if (unlikely(ieee80211_has_retry(hdr->frame_control) &&
 		     rx->sta->last_seq_ctrl[rx->seqno_idx] == hdr->seq_ctrl)) {
 		I802_DEBUG_INC(rx->local->dot11FrameDuplicateCount);
-		rx->sta->deflink.rx_stats.num_duplicates++;
+		rx->link_sta->rx_stats.num_duplicates++;
 		return RX_DROP_UNUSABLE;
 	} else if (!(status->flag & RX_FLAG_AMSDU_MORE)) {
 		rx->sta->last_seq_ctrl[rx->seqno_idx] = hdr->seq_ctrl;
@@ -1720,12 +1731,13 @@ static ieee80211_rx_result debug_noinline
 ieee80211_rx_h_sta_process(struct ieee80211_rx_data *rx)
 {
 	struct sta_info *sta = rx->sta;
+	struct link_sta_info *link_sta = rx->link_sta;
 	struct sk_buff *skb = rx->skb;
 	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
 	int i;
 
-	if (!sta)
+	if (!sta || !link_sta)
 		return RX_CONTINUE;
 
 	/*
@@ -1741,47 +1753,47 @@ ieee80211_rx_h_sta_process(struct ieee80211_rx_data *rx)
 						NL80211_IFTYPE_ADHOC);
 		if (ether_addr_equal(bssid, rx->sdata->u.ibss.bssid) &&
 		    test_sta_flag(sta, WLAN_STA_AUTHORIZED)) {
-			sta->deflink.rx_stats.last_rx = jiffies;
+			link_sta->rx_stats.last_rx = jiffies;
 			if (ieee80211_is_data(hdr->frame_control) &&
 			    !is_multicast_ether_addr(hdr->addr1))
-				sta->deflink.rx_stats.last_rate =
+				link_sta->rx_stats.last_rate =
 					sta_stats_encode_rate(status);
 		}
 	} else if (rx->sdata->vif.type == NL80211_IFTYPE_OCB) {
-		sta->deflink.rx_stats.last_rx = jiffies;
+		link_sta->rx_stats.last_rx = jiffies;
 	} else if (!ieee80211_is_s1g_beacon(hdr->frame_control) &&
 		   !is_multicast_ether_addr(hdr->addr1)) {
 		/*
 		 * Mesh beacons will update last_rx when if they are found to
 		 * match the current local configuration when processed.
 		 */
-		sta->deflink.rx_stats.last_rx = jiffies;
+		link_sta->rx_stats.last_rx = jiffies;
 		if (ieee80211_is_data(hdr->frame_control))
-			sta->deflink.rx_stats.last_rate = sta_stats_encode_rate(status);
+			link_sta->rx_stats.last_rate = sta_stats_encode_rate(status);
 	}
 
-	sta->deflink.rx_stats.fragments++;
+	link_sta->rx_stats.fragments++;
 
-	u64_stats_update_begin(&rx->sta->deflink.rx_stats.syncp);
-	sta->deflink.rx_stats.bytes += rx->skb->len;
-	u64_stats_update_end(&rx->sta->deflink.rx_stats.syncp);
+	u64_stats_update_begin(&link_sta->rx_stats.syncp);
+	link_sta->rx_stats.bytes += rx->skb->len;
+	u64_stats_update_end(&link_sta->rx_stats.syncp);
 
 	if (!(status->flag & RX_FLAG_NO_SIGNAL_VAL)) {
-		sta->deflink.rx_stats.last_signal = status->signal;
-		ewma_signal_add(&sta->deflink.rx_stats_avg.signal,
+		link_sta->rx_stats.last_signal = status->signal;
+		ewma_signal_add(&link_sta->rx_stats_avg.signal,
 				-status->signal);
 	}
 
 	if (status->chains) {
-		sta->deflink.rx_stats.chains = status->chains;
+		link_sta->rx_stats.chains = status->chains;
 		for (i = 0; i < ARRAY_SIZE(status->chain_signal); i++) {
 			int signal = status->chain_signal[i];
 
 			if (!(status->chains & BIT(i)))
 				continue;
 
-			sta->deflink.rx_stats.chain_signal_last[i] = signal;
-			ewma_signal_add(&sta->deflink.rx_stats_avg.chain_signal[i],
+			link_sta->rx_stats.chain_signal_last[i] = signal;
+			ewma_signal_add(&link_sta->rx_stats_avg.chain_signal[i],
 					-signal);
 		}
 	}
@@ -1842,7 +1854,7 @@ ieee80211_rx_h_sta_process(struct ieee80211_rx_data *rx)
 		 * Update counter and free packet here to avoid
 		 * counting this as a dropped packed.
 		 */
-		sta->deflink.rx_stats.packets++;
+		link_sta->rx_stats.packets++;
 		dev_kfree_skb(rx->skb);
 		return RX_QUEUED;
 	}
@@ -1854,7 +1866,6 @@ static struct ieee80211_key *
 ieee80211_rx_get_bigtk(struct ieee80211_rx_data *rx, int idx)
 {
 	struct ieee80211_key *key = NULL;
-	struct ieee80211_sub_if_data *sdata = rx->sdata;
 	int idx2;
 
 	/* Make sure key gets set if either BIGTK key index is set so that
@@ -1873,14 +1884,14 @@ ieee80211_rx_get_bigtk(struct ieee80211_rx_data *rx, int idx)
 			idx2 = idx - 1;
 	}
 
-	if (rx->sta)
-		key = rcu_dereference(rx->sta->deflink.gtk[idx]);
+	if (rx->link_sta)
+		key = rcu_dereference(rx->link_sta->gtk[idx]);
 	if (!key)
-		key = rcu_dereference(sdata->deflink.gtk[idx]);
-	if (!key && rx->sta)
-		key = rcu_dereference(rx->sta->deflink.gtk[idx2]);
+		key = rcu_dereference(rx->link->gtk[idx]);
+	if (!key && rx->link_sta)
+		key = rcu_dereference(rx->link_sta->gtk[idx2]);
 	if (!key)
-		key = rcu_dereference(sdata->deflink.gtk[idx2]);
+		key = rcu_dereference(rx->link->gtk[idx2]);
 
 	return key;
 }
@@ -1986,15 +1997,15 @@ ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx)
 		if (mmie_keyidx < NUM_DEFAULT_KEYS ||
 		    mmie_keyidx >= NUM_DEFAULT_KEYS + NUM_DEFAULT_MGMT_KEYS)
 			return RX_DROP_MONITOR; /* unexpected BIP keyidx */
-		if (rx->sta) {
+		if (rx->link_sta) {
 			if (ieee80211_is_group_privacy_action(skb) &&
 			    test_sta_flag(rx->sta, WLAN_STA_MFP))
 				return RX_DROP_MONITOR;
 
-			rx->key = rcu_dereference(rx->sta->deflink.gtk[mmie_keyidx]);
+			rx->key = rcu_dereference(rx->link_sta->gtk[mmie_keyidx]);
 		}
 		if (!rx->key)
-			rx->key = rcu_dereference(rx->sdata->deflink.gtk[mmie_keyidx]);
+			rx->key = rcu_dereference(rx->link->gtk[mmie_keyidx]);
 	} else if (!ieee80211_has_protected(fc)) {
 		/*
 		 * The frame was not protected, so skip decryption. However, we
@@ -2003,25 +2014,24 @@ ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx)
 		 * have been expected.
 		 */
 		struct ieee80211_key *key = NULL;
-		struct ieee80211_sub_if_data *sdata = rx->sdata;
 		int i;
 
 		if (ieee80211_is_beacon(fc)) {
 			key = ieee80211_rx_get_bigtk(rx, -1);
 		} else if (ieee80211_is_mgmt(fc) &&
 			   is_multicast_ether_addr(hdr->addr1)) {
-			key = rcu_dereference(rx->sdata->deflink.default_mgmt_key);
+			key = rcu_dereference(rx->link->default_mgmt_key);
 		} else {
-			if (rx->sta) {
+			if (rx->link_sta) {
 				for (i = 0; i < NUM_DEFAULT_KEYS; i++) {
-					key = rcu_dereference(rx->sta->deflink.gtk[i]);
+					key = rcu_dereference(rx->link_sta->gtk[i]);
 					if (key)
 						break;
 				}
 			}
 			if (!key) {
 				for (i = 0; i < NUM_DEFAULT_KEYS; i++) {
-					key = rcu_dereference(sdata->deflink.gtk[i]);
+					key = rcu_dereference(rx->link->gtk[i]);
 					if (key)
 						break;
 				}
@@ -2050,13 +2060,13 @@ ieee80211_rx_h_decrypt(struct ieee80211_rx_data *rx)
 			return RX_DROP_UNUSABLE;
 
 		/* check per-station GTK first, if multicast packet */
-		if (is_multicast_ether_addr(hdr->addr1) && rx->sta)
-			rx->key = rcu_dereference(rx->sta->deflink.gtk[keyidx]);
+		if (is_multicast_ether_addr(hdr->addr1) && rx->link_sta)
+			rx->key = rcu_dereference(rx->link_sta->gtk[keyidx]);
 
 		/* if not found, try default key */
 		if (!rx->key) {
 			if (is_multicast_ether_addr(hdr->addr1))
-				rx->key = rcu_dereference(rx->sdata->deflink.gtk[keyidx]);
+				rx->key = rcu_dereference(rx->link->gtk[keyidx]);
 			if (!rx->key)
 				rx->key = rcu_dereference(rx->sdata->keys[keyidx]);
 
@@ -2380,7 +2390,7 @@ ieee80211_rx_h_defragment(struct ieee80211_rx_data *rx)
  out:
 	ieee80211_led_rx(rx->local);
 	if (rx->sta)
-		rx->sta->deflink.rx_stats.packets++;
+		rx->link_sta->rx_stats.packets++;
 	return RX_CONTINUE;
 }
 
@@ -2656,9 +2666,9 @@ ieee80211_deliver_skb(struct ieee80211_rx_data *rx)
 		 * for non-QoS-data frames. Here we know it's a data
 		 * frame, so count MSDUs.
 		 */
-		u64_stats_update_begin(&rx->sta->deflink.rx_stats.syncp);
-		rx->sta->deflink.rx_stats.msdu[rx->seqno_idx]++;
-		u64_stats_update_end(&rx->sta->deflink.rx_stats.syncp);
+		u64_stats_update_begin(&rx->link_sta->rx_stats.syncp);
+		rx->link_sta->rx_stats.msdu[rx->seqno_idx]++;
+		u64_stats_update_end(&rx->link_sta->rx_stats.syncp);
 	}
 
 	if ((sdata->vif.type == NL80211_IFTYPE_AP ||
@@ -3046,7 +3056,8 @@ ieee80211_rx_h_data(struct ieee80211_rx_data *rx)
 		    (tf->action_code == WLAN_TDLS_CHANNEL_SWITCH_REQUEST ||
 		     tf->action_code == WLAN_TDLS_CHANNEL_SWITCH_RESPONSE)) {
 			rx->skb->protocol = cpu_to_be16(ETH_P_TDLS);
-			__ieee80211_queue_skb_to_iface(sdata, rx->sta, rx->skb);
+			__ieee80211_queue_skb_to_iface(sdata, rx->link_id,
+						       rx->sta, rx->skb);
 			return RX_QUEUED;
 		}
 	}
@@ -3354,7 +3365,7 @@ ieee80211_rx_h_action(struct ieee80211_rx_data *rx)
 	switch (mgmt->u.action.category) {
 	case WLAN_CATEGORY_HT:
 		/* reject HT action frames from stations not supporting HT */
-		if (!rx->sta->sta.deflink.ht_cap.ht_supported)
+		if (!rx->link_sta->pub->ht_cap.ht_supported)
 			goto invalid;
 
 		if (sdata->vif.type != NL80211_IFTYPE_STATION &&
@@ -3394,9 +3405,9 @@ ieee80211_rx_h_action(struct ieee80211_rx_data *rx)
 			}
 
 			/* if no change do nothing */
-			if (rx->sta->sta.smps_mode == smps_mode)
+			if (rx->link_sta->pub->smps_mode == smps_mode)
 				goto handled;
-			rx->sta->sta.smps_mode = smps_mode;
+			rx->link_sta->pub->smps_mode = smps_mode;
 			sta_opmode.smps_mode =
 				ieee80211_smps_mode_to_smps_mode(smps_mode);
 			sta_opmode.changed = STA_OPMODE_SMPS_MODE_CHANGED;
@@ -3418,26 +3429,26 @@ ieee80211_rx_h_action(struct ieee80211_rx_data *rx)
 			struct sta_opmode_info sta_opmode = {};
 
 			/* If it doesn't support 40 MHz it can't change ... */
-			if (!(rx->sta->sta.deflink.ht_cap.cap &
+			if (!(rx->link_sta->pub->ht_cap.cap &
 					IEEE80211_HT_CAP_SUP_WIDTH_20_40))
 				goto handled;
 
 			if (chanwidth == IEEE80211_HT_CHANWIDTH_20MHZ)
 				max_bw = IEEE80211_STA_RX_BW_20;
 			else
-				max_bw = ieee80211_sta_cap_rx_bw(&rx->sta->deflink);
+				max_bw = ieee80211_sta_cap_rx_bw(rx->link_sta);
 
 			/* set cur_max_bandwidth and recalc sta bw */
-			rx->sta->deflink.cur_max_bandwidth = max_bw;
-			new_bw = ieee80211_sta_cur_vht_bw(&rx->sta->deflink);
+			rx->link_sta->cur_max_bandwidth = max_bw;
+			new_bw = ieee80211_sta_cur_vht_bw(rx->link_sta);
 
-			if (rx->sta->sta.deflink.bandwidth == new_bw)
+			if (rx->link_sta->pub->bandwidth == new_bw)
 				goto handled;
 
-			rx->sta->sta.deflink.bandwidth = new_bw;
+			rx->link_sta->pub->bandwidth = new_bw;
 			sband = rx->local->hw.wiphy->bands[status->band];
 			sta_opmode.bw =
-				ieee80211_sta_rx_bw_to_chan_width(&rx->sta->deflink);
+				ieee80211_sta_rx_bw_to_chan_width(rx->link_sta);
 			sta_opmode.changed = STA_OPMODE_MAX_BW_CHANGED;
 
 			rate_control_rate_update(local, sband, rx->sta, 0,
@@ -3631,12 +3642,12 @@ ieee80211_rx_h_action(struct ieee80211_rx_data *rx)
 
  handled:
 	if (rx->sta)
-		rx->sta->deflink.rx_stats.packets++;
+		rx->link_sta->rx_stats.packets++;
 	dev_kfree_skb(rx->skb);
 	return RX_QUEUED;
 
  queue:
-	ieee80211_queue_skb_to_iface(sdata, rx->sta, rx->skb);
+	ieee80211_queue_skb_to_iface(sdata, rx->link_id, rx->sta, rx->skb);
 	return RX_QUEUED;
 }
 
@@ -3675,7 +3686,7 @@ ieee80211_rx_h_userspace_mgmt(struct ieee80211_rx_data *rx)
 
 	if (cfg80211_rx_mgmt_ext(&rx->sdata->wdev, &info)) {
 		if (rx->sta)
-			rx->sta->deflink.rx_stats.packets++;
+			rx->link_sta->rx_stats.packets++;
 		dev_kfree_skb(rx->skb);
 		return RX_QUEUED;
 	}
@@ -3713,7 +3724,7 @@ ieee80211_rx_h_action_post_userspace(struct ieee80211_rx_data *rx)
 
  handled:
 	if (rx->sta)
-		rx->sta->deflink.rx_stats.packets++;
+		rx->link_sta->rx_stats.packets++;
 	dev_kfree_skb(rx->skb);
 	return RX_QUEUED;
 }
@@ -3794,7 +3805,7 @@ ieee80211_rx_h_ext(struct ieee80211_rx_data *rx)
 		return RX_DROP_MONITOR;
 
 	/* for now only beacons are ext, so queue them */
-	ieee80211_queue_skb_to_iface(sdata, rx->sta, rx->skb);
+	ieee80211_queue_skb_to_iface(sdata, rx->link_id, rx->sta, rx->skb);
 
 	return RX_QUEUED;
 }
@@ -3851,7 +3862,7 @@ ieee80211_rx_h_mgmt(struct ieee80211_rx_data *rx)
 		return RX_DROP_MONITOR;
 	}
 
-	ieee80211_queue_skb_to_iface(sdata, rx->sta, rx->skb);
+	ieee80211_queue_skb_to_iface(sdata, rx->link_id, rx->sta, rx->skb);
 
 	return RX_QUEUED;
 }
@@ -3933,7 +3944,7 @@ static void ieee80211_rx_handlers_result(struct ieee80211_rx_data *rx,
 	case RX_DROP_MONITOR:
 		I802_DEBUG_INC(rx->sdata->local->rx_handlers_drop);
 		if (rx->sta)
-			rx->sta->deflink.rx_stats.dropped++;
+			rx->link_sta->rx_stats.dropped++;
 		fallthrough;
 	case RX_CONTINUE: {
 		struct ieee80211_rate *rate = NULL;
@@ -3952,7 +3963,7 @@ static void ieee80211_rx_handlers_result(struct ieee80211_rx_data *rx,
 	case RX_DROP_UNUSABLE:
 		I802_DEBUG_INC(rx->sdata->local->rx_handlers_drop);
 		if (rx->sta)
-			rx->sta->deflink.rx_stats.dropped++;
+			rx->link_sta->rx_stats.dropped++;
 		dev_kfree_skb(rx->skb);
 		break;
 	case RX_QUEUED:
@@ -4097,6 +4108,7 @@ void ieee80211_release_reorder_timeout(struct sta_info *sta, int tid)
 	/* FIXME: statistics won't be right with this */
 	link_id = sta->sta.valid_links ? ffs(sta->sta.valid_links) - 1 : 0;
 	rx.link = rcu_dereference(sta->sdata->link[link_id]);
+	rx.link_sta = rcu_dereference(sta->link[link_id]);
 
 	ieee80211_rx_handlers(&rx, &frames);
 }
@@ -4512,6 +4524,15 @@ void ieee80211_check_fast_rx_iface(struct ieee80211_sub_if_data *sdata)
 	mutex_unlock(&local->sta_mtx);
 }
 
+static bool
+ieee80211_rx_is_valid_sta_link_id(struct ieee80211_sta *sta, u8 link_id)
+{
+	if (!sta->mlo)
+		return false;
+
+	return !!(sta->valid_links & BIT(link_id));
+}
+
 static void ieee80211_rx_8023(struct ieee80211_rx_data *rx,
 			      struct ieee80211_fast_rx *fast_rx,
 			      int orig_len)
@@ -4519,19 +4540,30 @@ static void ieee80211_rx_8023(struct ieee80211_rx_data *rx,
 	struct ieee80211_sta_rx_stats *stats;
 	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(rx->skb);
 	struct sta_info *sta = rx->sta;
+	struct link_sta_info *link_sta;
 	struct sk_buff *skb = rx->skb;
 	void *sa = skb->data + ETH_ALEN;
 	void *da = skb->data;
 
-	stats = &sta->deflink.rx_stats;
+	if (rx->link_id >= 0) {
+		link_sta = rcu_dereference(sta->link[rx->link_id]);
+		if (WARN_ON_ONCE(!link_sta)) {
+			dev_kfree_skb(rx->skb);
+			return;
+		}
+	} else {
+		link_sta = &sta->deflink;
+	}
+
+	stats = &link_sta->rx_stats;
 	if (fast_rx->uses_rss)
-		stats = this_cpu_ptr(sta->deflink.pcpu_rx_stats);
+		stats = this_cpu_ptr(link_sta->pcpu_rx_stats);
 
 	/* statistics part of ieee80211_rx_h_sta_process() */
 	if (!(status->flag & RX_FLAG_NO_SIGNAL_VAL)) {
 		stats->last_signal = status->signal;
 		if (!fast_rx->uses_rss)
-			ewma_signal_add(&sta->deflink.rx_stats_avg.signal,
+			ewma_signal_add(&link_sta->rx_stats_avg.signal,
 					-status->signal);
 	}
 
@@ -4547,7 +4579,7 @@ static void ieee80211_rx_8023(struct ieee80211_rx_data *rx,
 
 			stats->chain_signal_last[i] = signal;
 			if (!fast_rx->uses_rss)
-				ewma_signal_add(&sta->deflink.rx_stats_avg.chain_signal[i],
+				ewma_signal_add(&link_sta->rx_stats_avg.chain_signal[i],
 						-signal);
 		}
 	}
@@ -4623,7 +4655,8 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx,
 		u8 da[ETH_ALEN];
 		u8 sa[ETH_ALEN];
 	} addrs __aligned(2);
-	struct ieee80211_sta_rx_stats *stats = &sta->deflink.rx_stats;
+	struct link_sta_info *link_sta;
+	struct ieee80211_sta_rx_stats *stats;
 
 	/* for parallel-rx, we need to have DUP_VALIDATED, otherwise we write
 	 * to a common data structure; drivers can implement that per queue
@@ -4724,8 +4757,19 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx,
 	return true;
  drop:
 	dev_kfree_skb(skb);
+
+	if (rx->link_id >= 0) {
+		link_sta = rcu_dereference(sta->link[rx->link_id]);
+		if (!link_sta)
+			return true;
+	} else {
+		link_sta = &sta->deflink;
+	}
+
 	if (fast_rx->uses_rss)
-		stats = this_cpu_ptr(sta->deflink.pcpu_rx_stats);
+		stats = this_cpu_ptr(link_sta->pcpu_rx_stats);
+	else
+		stats = &link_sta->rx_stats;
 
 	stats->dropped++;
 	return true;
@@ -4773,7 +4817,17 @@ static bool ieee80211_prepare_and_rx_handle(struct ieee80211_rx_data *rx,
 		if (!link)
 			return true;
 		rx->link = link;
+
+		if (rx->sta) {
+			rx->link_sta =
+				rcu_dereference(rx->sta->link[rx->link_id]);
+			if (!rx->link_sta)
+				return true;
+		}
 	} else {
+		if (rx->sta)
+			rx->link_sta = &rx->sta->deflink;
+
 		rx->link = &sdata->deflink;
 	}
 
@@ -4831,6 +4885,7 @@ static void __ieee80211_rx_handle_8023(struct ieee80211_hw *hw,
 				       struct list_head *list)
 {
 	struct ieee80211_local *local = hw_to_local(hw);
+	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_fast_rx *fast_rx;
 	struct ieee80211_rx_data rx;
 
@@ -4851,7 +4906,31 @@ static void __ieee80211_rx_handle_8023(struct ieee80211_hw *hw,
 
 	rx.sta = container_of(pubsta, struct sta_info, sta);
 	rx.sdata = rx.sta->sdata;
-	rx.link = &rx.sdata->deflink;
+
+	if (status->link_valid &&
+	    !ieee80211_rx_is_valid_sta_link_id(pubsta, status->link_id))
+		goto drop;
+
+	/*
+	 * TODO: Should the frame be dropped if the right link_id is not
+	 * available? Or may be it is fine in the current form to proceed with
+	 * the frame processing because with frame being in 802.3 format,
+	 * link_id is used only for stats purpose and updating the stats on
+	 * the deflink is fine?
+	 */
+	if (status->link_valid)
+		rx.link_id = status->link_id;
+
+	if (rx.link_id >= 0) {
+		struct ieee80211_link_data *link;
+
+		link =  rcu_dereference(rx.sdata->link[rx.link_id]);
+		if (!link)
+			goto drop;
+		rx.link = link;
+	} else {
+		rx.link = &rx.sdata->deflink;
+	}
 
 	fast_rx = rcu_dereference(rx.sta->fast_rx);
 	if (!fast_rx)
@@ -4881,7 +4960,19 @@ static bool ieee80211_rx_for_interface(struct ieee80211_rx_data *rx,
 		rx->sta = link_sta->sta;
 		rx->link_id = link_sta->link_id;
 	} else {
+		struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
+
 		rx->sta = sta_info_get_bss(rx->sdata, hdr->addr2);
+		if (rx->sta) {
+			if (status->link_valid &&
+			    !ieee80211_rx_is_valid_sta_link_id(&rx->sta->sta,
+							       status->link_id))
+				return false;
+
+			rx->link_id = status->link_valid ? status->link_id : -1;
+		} else {
+			rx->link_id = -1;
+		}
 	}
 
 	return ieee80211_prepare_and_rx_handle(rx, skb, consume);
@@ -4897,6 +4988,7 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
 					 struct list_head *list)
 {
 	struct ieee80211_local *local = hw_to_local(hw);
+	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
 	struct ieee80211_sub_if_data *sdata;
 	struct ieee80211_hdr *hdr;
 	__le16 fc;
@@ -4941,10 +5033,39 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
 
 	if (ieee80211_is_data(fc)) {
 		struct sta_info *sta, *prev_sta;
+		u8 link_id = status->link_id;
 
 		if (pubsta) {
 			rx.sta = container_of(pubsta, struct sta_info, sta);
 			rx.sdata = rx.sta->sdata;
+
+			if (status->link_valid &&
+			    !ieee80211_rx_is_valid_sta_link_id(pubsta, link_id))
+				goto out;
+
+			if (status->link_valid)
+				rx.link_id = status->link_id;
+
+			/*
+			 * In MLO connection, fetch the link_id using addr2
+			 * when the driver does not pass link_id in status.
+			 * When the address translation is already performed by
+			 * driver/hw, the valid link_id must be passed in
+			 * status.
+			 */
+
+			if (!status->link_valid && pubsta->mlo) {
+				struct ieee80211_hdr *hdr = (void *)skb->data;
+				struct link_sta_info *link_sta;
+
+				link_sta = link_sta_info_get_bss(rx.sdata,
+								 hdr->addr2);
+				if (!link_sta)
+					goto out;
+
+				rx.link_id = link_sta->link_id;
+			}
+
 			if (ieee80211_prepare_and_rx_handle(&rx, skb, true))
 				return;
 			goto out;
@@ -4958,6 +5079,13 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
 				continue;
 			}
 
+			if ((status->link_valid &&
+			     !ieee80211_rx_is_valid_sta_link_id(&prev_sta->sta,
+								link_id)) ||
+			    (!status->link_valid && prev_sta->sta.mlo))
+				continue;
+
+			rx.link_id = status->link_valid ? link_id : -1;
 			rx.sta = prev_sta;
 			rx.sdata = prev_sta->sdata;
 			ieee80211_prepare_and_rx_handle(&rx, skb, false);
@@ -4966,6 +5094,13 @@ static void __ieee80211_rx_handle_packet(struct ieee80211_hw *hw,
 		}
 
 		if (prev_sta) {
+			if ((status->link_valid &&
+			     !ieee80211_rx_is_valid_sta_link_id(&prev_sta->sta,
+								link_id)) ||
+			    (!status->link_valid && prev_sta->sta.mlo))
+				goto out;
+
+			rx.link_id = status->link_valid ? link_id : -1;
 			rx.sta = prev_sta;
 			rx.sdata = prev_sta->sdata;
 
@@ -5108,6 +5243,9 @@ void ieee80211_rx_list(struct ieee80211_hw *hw, struct ieee80211_sta *pubsta,
 		}
 	}
 
+	if (WARN_ON_ONCE(status->link_id >= IEEE80211_LINK_UNSPECIFIED))
+		goto drop;
+
 	status->rx_flags = 0;
 
 	kcov_remote_start_common(skb_get_kcov_handle(skb));
diff --git a/net/mac80211/scan.c b/net/mac80211/scan.c
index c4f2aeb31da3..0e8c4f48c36d 100644
--- a/net/mac80211/scan.c
+++ b/net/mac80211/scan.c
@@ -485,7 +485,7 @@ static void __ieee80211_scan_completed(struct ieee80211_hw *hw, bool aborted)
 	/* Set power back to normal operating levels. */
 	ieee80211_hw_config(local, 0);
 
-	if (!hw_scan) {
+	if (!hw_scan && was_scanning) {
 		ieee80211_configure_filter(local);
 		drv_sw_scan_complete(local, scan_sdata);
 		ieee80211_offchannel_return(local);
diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c
index 58998d821778..cebfd148bb40 100644
--- a/net/mac80211/sta_info.c
+++ b/net/mac80211/sta_info.c
@@ -274,6 +274,43 @@ link_sta_info_get_bss(struct ieee80211_sub_if_data *sdata, const u8 *addr)
 	return NULL;
 }
 
+struct ieee80211_sta *
+ieee80211_find_sta_by_link_addrs(struct ieee80211_hw *hw,
+				 const u8 *addr,
+				 const u8 *localaddr,
+				 unsigned int *link_id)
+{
+	struct ieee80211_local *local = hw_to_local(hw);
+	struct link_sta_info *link_sta;
+	struct rhlist_head *tmp;
+
+	for_each_link_sta_info(local, addr, link_sta, tmp) {
+		struct sta_info *sta = link_sta->sta;
+		struct ieee80211_link_data *link;
+		u8 _link_id = link_sta->link_id;
+
+		if (!localaddr) {
+			if (link_id)
+				*link_id = _link_id;
+			return &sta->sta;
+		}
+
+		link = rcu_dereference(sta->sdata->link[_link_id]);
+		if (!link)
+			continue;
+
+		if (memcmp(link->conf->addr, localaddr, ETH_ALEN))
+			continue;
+
+		if (link_id)
+			*link_id = _link_id;
+		return &sta->sta;
+	}
+
+	return NULL;
+}
+EXPORT_SYMBOL_GPL(ieee80211_find_sta_by_link_addrs);
+
 struct sta_info *sta_info_get_by_addrs(struct ieee80211_local *local,
 				       const u8 *sta_addr, const u8 *vif_addr)
 {
@@ -339,6 +376,8 @@ static void sta_remove_link(struct sta_info *sta, unsigned int link_id,
 		sta_info_free_link(&alloc->info);
 		kfree_rcu(alloc, rcu_head);
 	}
+
+	ieee80211_sta_recalc_aggregates(&sta->sta);
 }
 
 /**
@@ -472,8 +511,12 @@ static void sta_info_add_link(struct sta_info *sta,
 	link_info->sta = sta;
 	link_info->link_id = link_id;
 	link_info->pub = link_sta;
+	link_sta->link_id = link_id;
 	rcu_assign_pointer(sta->link[link_id], link_info);
 	rcu_assign_pointer(sta->sta.link[link_id], link_sta);
+
+	link_sta->smps_mode = IEEE80211_SMPS_OFF;
+	link_sta->agg.max_rc_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_BA;
 }
 
 static struct sta_info *
@@ -504,6 +547,8 @@ __sta_info_alloc(struct ieee80211_sub_if_data *sdata,
 		sta_info_add_link(sta, 0, &sta->deflink, &sta->sta.deflink);
 	}
 
+	sta->sta.cur = &sta->sta.deflink.agg;
+
 	spin_lock_init(&sta->lock);
 	spin_lock_init(&sta->ps_lock);
 	INIT_WORK(&sta->drv_deliver_wk, sta_deliver_ps_frames);
@@ -627,9 +672,6 @@ __sta_info_alloc(struct ieee80211_sub_if_data *sdata,
 		}
 	}
 
-	sta->sta.smps_mode = IEEE80211_SMPS_OFF;
-	sta->sta.max_rc_amsdu_len = IEEE80211_MAX_MPDU_LEN_HT_BA;
-
 	sta->cparams.ce_threshold = CODEL_DISABLED_THRESHOLD;
 	sta->cparams.target = MS2TIME(20);
 	sta->cparams.interval = MS2TIME(100);
@@ -2085,6 +2127,44 @@ void ieee80211_sta_register_airtime(struct ieee80211_sta *pubsta, u8 tid,
 }
 EXPORT_SYMBOL(ieee80211_sta_register_airtime);
 
+void ieee80211_sta_recalc_aggregates(struct ieee80211_sta *pubsta)
+{
+	struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
+	struct ieee80211_link_sta *link_sta;
+	int link_id, i;
+	bool first = true;
+
+	if (!pubsta->valid_links || !pubsta->mlo) {
+		pubsta->cur = &pubsta->deflink.agg;
+		return;
+	}
+
+	rcu_read_lock();
+	for_each_sta_active_link(&sta->sdata->vif, pubsta, link_sta, link_id) {
+		if (first) {
+			sta->cur = pubsta->deflink.agg;
+			first = false;
+			continue;
+		}
+
+		sta->cur.max_amsdu_len =
+			min(sta->cur.max_amsdu_len,
+			    link_sta->agg.max_amsdu_len);
+		sta->cur.max_rc_amsdu_len =
+			min(sta->cur.max_rc_amsdu_len,
+			    link_sta->agg.max_rc_amsdu_len);
+
+		for (i = 0; i < ARRAY_SIZE(sta->cur.max_tid_amsdu_len); i++)
+			sta->cur.max_tid_amsdu_len[i] =
+				min(sta->cur.max_tid_amsdu_len[i],
+				    link_sta->agg.max_tid_amsdu_len[i]);
+	}
+	rcu_read_unlock();
+
+	pubsta->cur = &sta->cur;
+}
+EXPORT_SYMBOL(ieee80211_sta_recalc_aggregates);
+
 void ieee80211_sta_update_pending_airtime(struct ieee80211_local *local,
 					  struct sta_info *sta, u8 ac,
 					  u16 tx_airtime, bool tx_completed)
@@ -2777,10 +2857,13 @@ int ieee80211_sta_activate_link(struct sta_info *sta, unsigned int link_id)
 
 	sta->sta.valid_links = new_links;
 
-	if (!test_sta_flag(sta, WLAN_STA_INSERTED)) {
-		ret = 0;
+	if (!test_sta_flag(sta, WLAN_STA_INSERTED))
 		goto hash;
-	}
+
+	/* Ensure the values are updated for the driver,
+	 * redone by sta_remove_link on failure.
+	 */
+	ieee80211_sta_recalc_aggregates(&sta->sta);
 
 	ret = drv_change_sta_links(sdata->local, sdata, &sta->sta,
 				   old_links, new_links);
@@ -2799,6 +2882,7 @@ hash:
 void ieee80211_sta_remove_link(struct sta_info *sta, unsigned int link_id)
 {
 	struct ieee80211_sub_if_data *sdata = sta->sdata;
+	u16 old_links = sta->sta.valid_links;
 
 	lockdep_assert_held(&sdata->local->sta_mtx);
 
@@ -2806,8 +2890,7 @@ void ieee80211_sta_remove_link(struct sta_info *sta, unsigned int link_id)
 
 	if (test_sta_flag(sta, WLAN_STA_INSERTED))
 		drv_change_sta_links(sdata->local, sdata, &sta->sta,
-				     sta->sta.valid_links,
-				     sta->sta.valid_links & ~BIT(link_id));
+				     old_links, sta->sta.valid_links);
 
 	sta_remove_link(sta, link_id, true);
 }
@@ -2834,3 +2917,13 @@ void ieee80211_sta_set_max_amsdu_subframes(struct sta_info *sta,
 	if (val)
 		sta->sta.max_amsdu_subframes = 4 << val;
 }
+
+#ifdef CONFIG_LOCKDEP
+bool lockdep_sta_mutex_held(struct ieee80211_sta *pubsta)
+{
+	struct sta_info *sta = container_of(pubsta, struct sta_info, sta);
+
+	return lockdep_is_held(&sta->local->sta_mtx);
+}
+EXPORT_SYMBOL(lockdep_sta_mutex_held);
+#endif
diff --git a/net/mac80211/sta_info.h b/net/mac80211/sta_info.h
index 2eb3a9452e07..2517ea714dc4 100644
--- a/net/mac80211/sta_info.h
+++ b/net/mac80211/sta_info.h
@@ -622,6 +622,8 @@ struct link_sta_info {
  * @tdls_chandef: a TDLS peer can have a wider chandef that is compatible to
  *	the BSS one.
  * @frags: fragment cache
+ * @cur: storage for aggregation data
+ *	&struct ieee80211_sta points either here or to deflink.agg.
  * @deflink: This is the default link STA information, for non MLO STA all link
  *	specific STA information is accessed through @deflink or through
  *	link[0] which points to address of @deflink. For MLO Link STA
@@ -705,6 +707,7 @@ struct sta_info {
 
 	struct ieee80211_fragment_cache frags;
 
+	struct ieee80211_sta_aggregates cur;
 	struct link_sta_info deflink;
 	struct link_sta_info __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS];
 
diff --git a/net/mac80211/status.c b/net/mac80211/status.c
index 8e77fd2e9fdf..3f9ddd7f04b6 100644
--- a/net/mac80211/status.c
+++ b/net/mac80211/status.c
@@ -729,7 +729,7 @@ static void ieee80211_report_used_skb(struct ieee80211_local *local,
 
 		if (!sdata) {
 			skb->dev = NULL;
-		} else {
+		} else if (!dropped) {
 			unsigned int hdr_size =
 				ieee80211_hdrlen(hdr->frame_control);
 
diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c
index bf7fe6cd9dfc..27c964be102e 100644
--- a/net/mac80211/tx.c
+++ b/net/mac80211/tx.c
@@ -576,6 +576,51 @@ ieee80211_tx_h_check_control_port_protocol(struct ieee80211_tx_data *tx)
 	return TX_CONTINUE;
 }
 
+static struct ieee80211_key *
+ieee80211_select_link_key(struct ieee80211_tx_data *tx)
+{
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)tx->skb->data;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx->skb);
+	enum {
+		USE_NONE,
+		USE_MGMT_KEY,
+		USE_MCAST_KEY,
+	} which_key = USE_NONE;
+	struct ieee80211_link_data *link;
+	unsigned int link_id;
+
+	if (ieee80211_is_group_privacy_action(tx->skb))
+		which_key = USE_MCAST_KEY;
+	else if (ieee80211_is_mgmt(hdr->frame_control) &&
+		 is_multicast_ether_addr(hdr->addr1) &&
+		 ieee80211_is_robust_mgmt_frame(tx->skb))
+		which_key = USE_MGMT_KEY;
+	else if (is_multicast_ether_addr(hdr->addr1))
+		which_key = USE_MCAST_KEY;
+	else
+		return NULL;
+
+	link_id = u32_get_bits(info->control.flags, IEEE80211_TX_CTRL_MLO_LINK);
+	if (link_id == IEEE80211_LINK_UNSPECIFIED) {
+		link = &tx->sdata->deflink;
+	} else {
+		link = rcu_dereference(tx->sdata->link[link_id]);
+		if (!link)
+			return NULL;
+	}
+
+	switch (which_key) {
+	case USE_NONE:
+		break;
+	case USE_MGMT_KEY:
+		return rcu_dereference(link->default_mgmt_key);
+	case USE_MCAST_KEY:
+		return rcu_dereference(link->default_multicast_key);
+	}
+
+	return NULL;
+}
+
 static ieee80211_tx_result debug_noinline
 ieee80211_tx_h_select_key(struct ieee80211_tx_data *tx)
 {
@@ -591,16 +636,7 @@ ieee80211_tx_h_select_key(struct ieee80211_tx_data *tx)
 	if (tx->sta &&
 	    (key = rcu_dereference(tx->sta->ptk[tx->sta->ptk_idx])))
 		tx->key = key;
-	else if (ieee80211_is_group_privacy_action(tx->skb) &&
-		(key = rcu_dereference(tx->sdata->deflink.default_multicast_key)))
-		tx->key = key;
-	else if (ieee80211_is_mgmt(hdr->frame_control) &&
-		 is_multicast_ether_addr(hdr->addr1) &&
-		 ieee80211_is_robust_mgmt_frame(tx->skb) &&
-		 (key = rcu_dereference(tx->sdata->deflink.default_mgmt_key)))
-		tx->key = key;
-	else if (is_multicast_ether_addr(hdr->addr1) &&
-		 (key = rcu_dereference(tx->sdata->deflink.default_multicast_key)))
+	else if ((key = ieee80211_select_link_key(tx)))
 		tx->key = key;
 	else if (!is_multicast_ether_addr(hdr->addr1) &&
 		 (key = rcu_dereference(tx->sdata->default_unicast_key)))
@@ -2640,7 +2676,8 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata,
 				goto free;
 			}
 			memcpy(hdr.addr2, link->conf->addr, ETH_ALEN);
-		} else if (link_id == IEEE80211_LINK_UNSPECIFIED) {
+		} else if (link_id == IEEE80211_LINK_UNSPECIFIED ||
+			   (sta && sta->sta.mlo)) {
 			memcpy(hdr.addr2, sdata->vif.addr, ETH_ALEN);
 		} else {
 			struct ieee80211_bss_conf *conf;
@@ -3350,7 +3387,7 @@ static bool ieee80211_amsdu_aggregate(struct ieee80211_sub_if_data *sdata,
 	int subframe_len = skb->len - ETH_ALEN;
 	u8 max_subframes = sta->sta.max_amsdu_subframes;
 	int max_frags = local->hw.max_tx_fragments;
-	int max_amsdu_len = sta->sta.max_amsdu_len;
+	int max_amsdu_len = sta->sta.cur->max_amsdu_len;
 	int orig_truesize;
 	u32 flow_idx;
 	__be16 len;
@@ -3376,13 +3413,13 @@ static bool ieee80211_amsdu_aggregate(struct ieee80211_sub_if_data *sdata,
 	if (test_bit(IEEE80211_TXQ_NO_AMSDU, &txqi->flags))
 		return false;
 
-	if (sta->sta.max_rc_amsdu_len)
+	if (sta->sta.cur->max_rc_amsdu_len)
 		max_amsdu_len = min_t(int, max_amsdu_len,
-				      sta->sta.max_rc_amsdu_len);
+				      sta->sta.cur->max_rc_amsdu_len);
 
-	if (sta->sta.max_tid_amsdu_len[tid])
+	if (sta->sta.cur->max_tid_amsdu_len[tid])
 		max_amsdu_len = min_t(int, max_amsdu_len,
-				      sta->sta.max_tid_amsdu_len[tid]);
+				      sta->sta.cur->max_tid_amsdu_len[tid]);
 
 	flow_idx = fq_flow_idx(fq, skb);
 
@@ -3735,8 +3772,8 @@ begin:
 			     !test_sta_flag(tx.sta, WLAN_STA_AUTHORIZED) &&
 			     (!(info->control.flags &
 				IEEE80211_TX_CTRL_PORT_CTRL_PROTO) ||
-			      !ether_addr_equal(tx.sdata->vif.addr,
-						hdr->addr2)))) {
+			      !ieee80211_is_our_addr(tx.sdata, hdr->addr2,
+						     NULL)))) {
 			I802_DEBUG_INC(local->tx_handlers_drop_unauth_port);
 			ieee80211_free_txskb(&local->hw, skb);
 			goto begin;
@@ -5061,6 +5098,8 @@ ieee80211_beacon_get_finish(struct ieee80211_hw *hw,
 	rate_control_get_rate(sdata, NULL, &txrc);
 
 	info->control.vif = vif;
+	info->control.flags |= u32_encode_bits(link->link_id,
+					       IEEE80211_TX_CTRL_MLO_LINK);
 	info->flags |= IEEE80211_TX_CTL_CLEAR_PS_FILT |
 		       IEEE80211_TX_CTL_ASSIGN_SEQ |
 		       IEEE80211_TX_CTL_FIRST_FRAGMENT;
@@ -5430,33 +5469,39 @@ EXPORT_SYMBOL(ieee80211_pspoll_get);
 
 struct sk_buff *ieee80211_nullfunc_get(struct ieee80211_hw *hw,
 				       struct ieee80211_vif *vif,
-				       bool qos_ok)
+				       int link_id, bool qos_ok)
 {
+	struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
+	struct ieee80211_local *local = sdata->local;
+	struct ieee80211_link_data *link = NULL;
 	struct ieee80211_hdr_3addr *nullfunc;
-	struct ieee80211_sub_if_data *sdata;
-	struct ieee80211_local *local;
 	struct sk_buff *skb;
 	bool qos = false;
 
 	if (WARN_ON(vif->type != NL80211_IFTYPE_STATION))
 		return NULL;
 
-	sdata = vif_to_sdata(vif);
-	local = sdata->local;
+	skb = dev_alloc_skb(local->hw.extra_tx_headroom +
+			    sizeof(*nullfunc) + 2);
+	if (!skb)
+		return NULL;
 
+	rcu_read_lock();
 	if (qos_ok) {
 		struct sta_info *sta;
 
-		rcu_read_lock();
-		sta = sta_info_get(sdata, sdata->deflink.u.mgd.bssid);
+		sta = sta_info_get(sdata, vif->cfg.ap_addr);
 		qos = sta && sta->sta.wme;
-		rcu_read_unlock();
 	}
 
-	skb = dev_alloc_skb(local->hw.extra_tx_headroom +
-			    sizeof(*nullfunc) + 2);
-	if (!skb)
-		return NULL;
+	if (link_id >= 0) {
+		link = rcu_dereference(sdata->link[link_id]);
+		if (WARN_ON_ONCE(!link)) {
+			rcu_read_unlock();
+			kfree_skb(skb);
+			return NULL;
+		}
+	}
 
 	skb_reserve(skb, local->hw.extra_tx_headroom);
 
@@ -5477,9 +5522,16 @@ struct sk_buff *ieee80211_nullfunc_get(struct ieee80211_hw *hw,
 		skb_put_data(skb, &qoshdr, sizeof(qoshdr));
 	}
 
-	memcpy(nullfunc->addr1, sdata->deflink.u.mgd.bssid, ETH_ALEN);
-	memcpy(nullfunc->addr2, vif->addr, ETH_ALEN);
-	memcpy(nullfunc->addr3, sdata->deflink.u.mgd.bssid, ETH_ALEN);
+	if (link) {
+		memcpy(nullfunc->addr1, link->conf->bssid, ETH_ALEN);
+		memcpy(nullfunc->addr2, link->conf->addr, ETH_ALEN);
+		memcpy(nullfunc->addr3, link->conf->bssid, ETH_ALEN);
+	} else {
+		memcpy(nullfunc->addr1, vif->cfg.ap_addr, ETH_ALEN);
+		memcpy(nullfunc->addr2, vif->addr, ETH_ALEN);
+		memcpy(nullfunc->addr3, vif->cfg.ap_addr, ETH_ALEN);
+	}
+	rcu_read_unlock();
 
 	return skb;
 }
@@ -5878,6 +5930,9 @@ int ieee80211_tx_control_port(struct wiphy *wiphy, struct net_device *dev,
 	skb_reset_network_header(skb);
 	skb_reset_mac_header(skb);
 
+	if (local->hw.queues < IEEE80211_NUM_ACS)
+		goto start_xmit;
+
 	/* update QoS header to prioritize control port frames if possible,
 	 * priorization also happens for control port frames send over
 	 * AF_PACKET
@@ -5905,6 +5960,7 @@ int ieee80211_tx_control_port(struct wiphy *wiphy, struct net_device *dev,
 	}
 	rcu_read_unlock();
 
+start_xmit:
 	/* mutex lock is only needed for incrementing the cookie counter */
 	mutex_lock(&local->mtx);
 
diff --git a/net/mac80211/util.c b/net/mac80211/util.c
index 53826c663723..bf7461c41bef 100644
--- a/net/mac80211/util.c
+++ b/net/mac80211/util.c
@@ -301,14 +301,14 @@ static void __ieee80211_wake_txqs(struct ieee80211_sub_if_data *sdata, int ac)
 	local_bh_disable();
 	spin_lock(&fq->lock);
 
+	sdata->vif.txqs_stopped[ac] = false;
+
 	if (!test_bit(SDATA_STATE_RUNNING, &sdata->state))
 		goto out;
 
 	if (sdata->vif.type == NL80211_IFTYPE_AP)
 		ps = &sdata->bss->ps;
 
-	sdata->vif.txqs_stopped[ac] = false;
-
 	list_for_each_entry_rcu(sta, &local->sta_list, list) {
 		if (sdata != sta->sdata)
 			continue;
@@ -954,9 +954,11 @@ void ieee80211_queue_delayed_work(struct ieee80211_hw *hw,
 }
 EXPORT_SYMBOL(ieee80211_queue_delayed_work);
 
-static void ieee80211_parse_extension_element(u32 *crc,
-					      const struct element *elem,
-					      struct ieee802_11_elems *elems)
+static void
+ieee80211_parse_extension_element(u32 *crc,
+				  const struct element *elem,
+				  struct ieee802_11_elems *elems,
+				  struct ieee80211_elems_parse_params *params)
 {
 	const void *data = elem->data + 1;
 	u8 len;
@@ -1013,7 +1015,8 @@ static void ieee80211_parse_extension_element(u32 *crc,
 		break;
 	case WLAN_EID_EXT_EHT_CAPABILITY:
 		if (ieee80211_eht_capa_size_ok(elems->he_cap,
-					       data, len)) {
+					       data, len,
+					       params->from_ap)) {
 			elems->eht_cap = data;
 			elems->eht_cap_len = len;
 		}
@@ -1385,7 +1388,7 @@ _ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params,
 		case WLAN_EID_EXTENSION:
 			ieee80211_parse_extension_element(calc_crc ?
 								&crc : NULL,
-							  elem, elems);
+							  elem, elems, params);
 			break;
 		case WLAN_EID_S1G_CAPABILITIES:
 			if (elen >= sizeof(*elems->s1g_capab))
@@ -2025,7 +2028,8 @@ static int ieee80211_build_preq_ies_band(struct ieee80211_sub_if_data *sdata,
 	    cfg80211_any_usable_channels(local->hw.wiphy, BIT(sband->band),
 					 IEEE80211_CHAN_NO_HE |
 					 IEEE80211_CHAN_NO_EHT)) {
-		pos = ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, end);
+		pos = ieee80211_ie_build_eht_cap(pos, he_cap, eht_cap, end,
+						 sdata->vif.type == NL80211_IFTYPE_AP);
 		if (!pos)
 			goto out_err;
 	}
@@ -2526,7 +2530,6 @@ int ieee80211_reconfig(struct ieee80211_local *local)
 			if (link)
 				ieee80211_assign_chanctx(local, sdata, link);
 		}
-		sdata_unlock(sdata);
 
 		switch (sdata->vif.type) {
 		case NL80211_IFTYPE_AP_VLAN:
@@ -2545,6 +2548,7 @@ int ieee80211_reconfig(struct ieee80211_local *local)
 					    &sdata->deflink.tx_conf[i]);
 			break;
 		}
+		sdata_unlock(sdata);
 
 		/* common change flags for all interface types */
 		changed = BSS_CHANGED_ERP_CTS_PROT |
@@ -2653,23 +2657,21 @@ int ieee80211_reconfig(struct ieee80211_local *local)
 	}
 
 	/* APs are now beaconing, add back stations */
-	mutex_lock(&local->sta_mtx);
-	list_for_each_entry(sta, &local->sta_list, list) {
-		enum ieee80211_sta_state state;
-
-		if (!sta->uploaded)
-			continue;
-
-		if (sta->sdata->vif.type != NL80211_IFTYPE_AP &&
-		    sta->sdata->vif.type != NL80211_IFTYPE_AP_VLAN)
+	list_for_each_entry(sdata, &local->interfaces, list) {
+		if (!ieee80211_sdata_running(sdata))
 			continue;
 
-		for (state = IEEE80211_STA_NOTEXIST;
-		     state < sta->sta_state; state++)
-			WARN_ON(drv_sta_state(local, sta->sdata, sta, state,
-					      state + 1));
+		sdata_lock(sdata);
+		switch (sdata->vif.type) {
+		case NL80211_IFTYPE_AP_VLAN:
+		case NL80211_IFTYPE_AP:
+			ieee80211_reconfig_stations(sdata);
+			break;
+		default:
+			break;
+		}
+		sdata_unlock(sdata);
 	}
-	mutex_unlock(&local->sta_mtx);
 
 	/* add back keys */
 	list_for_each_entry(sdata, &local->interfaces, list)
@@ -2898,7 +2900,7 @@ void ieee80211_recalc_min_chandef(struct ieee80211_sub_if_data *sdata,
 		 */
 		rcu_read_unlock();
 
-		if (WARN_ON_ONCE(!chanctx_conf))
+		if (!chanctx_conf)
 			goto unlock;
 
 		chanctx = container_of(chanctx_conf, struct ieee80211_chanctx,
@@ -3080,6 +3082,7 @@ end:
 }
 
 void ieee80211_ie_build_he_6ghz_cap(struct ieee80211_sub_if_data *sdata,
+				    enum ieee80211_smps_mode smps_mode,
 				    struct sk_buff *skb)
 {
 	struct ieee80211_supported_band *sband;
@@ -3106,7 +3109,7 @@ void ieee80211_ie_build_he_6ghz_cap(struct ieee80211_sub_if_data *sdata,
 	cap = le16_to_cpu(iftd->he_6ghz_capa.capa);
 	cap &= ~IEEE80211_HE_6GHZ_CAP_SM_PS;
 
-	switch (sdata->deflink.smps_mode) {
+	switch (smps_mode) {
 	case IEEE80211_SMPS_AUTOMATIC:
 	case IEEE80211_SMPS_NUM_MODES:
 		WARN_ON(1);
@@ -3507,8 +3510,7 @@ bool ieee80211_chandef_vht_oper(struct ieee80211_hw *hw, u32 vht_cap_info,
 	return true;
 }
 
-void ieee80211_chandef_eht_oper(struct ieee80211_sub_if_data *sdata,
-				const struct ieee80211_eht_operation *eht_oper,
+void ieee80211_chandef_eht_oper(const struct ieee80211_eht_operation *eht_oper,
 				bool support_160, bool support_320,
 				struct cfg80211_chan_def *chandef)
 {
@@ -3684,7 +3686,7 @@ bool ieee80211_chandef_he_6ghz_oper(struct ieee80211_sub_if_data *sdata,
 		support_320 =
 			eht_phy_cap & IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ;
 
-		ieee80211_chandef_eht_oper(sdata, eht_oper, support_160,
+		ieee80211_chandef_eht_oper(eht_oper, support_160,
 					   support_320, &he_chandef);
 	}
 
@@ -4770,6 +4772,7 @@ u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata, u8 iftype)
 	const struct ieee80211_sta_he_cap *he_cap;
 	const struct ieee80211_sta_eht_cap *eht_cap;
 	struct ieee80211_supported_band *sband;
+	bool is_ap;
 	u8 n;
 
 	sband = ieee80211_get_sband(sdata);
@@ -4781,8 +4784,12 @@ u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata, u8 iftype)
 	if (!he_cap || !eht_cap)
 		return 0;
 
+	is_ap = iftype == NL80211_IFTYPE_AP ||
+		iftype == NL80211_IFTYPE_P2P_GO;
+
 	n = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem,
-				       &eht_cap->eht_cap_elem);
+				       &eht_cap->eht_cap_elem,
+				       is_ap);
 	return 2 + 1 +
 	       sizeof(he_cap->he_cap_elem) + n +
 	       ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0],
@@ -4793,7 +4800,8 @@ u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata, u8 iftype)
 u8 *ieee80211_ie_build_eht_cap(u8 *pos,
 			       const struct ieee80211_sta_he_cap *he_cap,
 			       const struct ieee80211_sta_eht_cap *eht_cap,
-			       u8 *end)
+			       u8 *end,
+			       bool for_ap)
 {
 	u8 mcs_nss_len, ppet_len;
 	u8 ie_len;
@@ -4804,7 +4812,8 @@ u8 *ieee80211_ie_build_eht_cap(u8 *pos,
 		return orig_pos;
 
 	mcs_nss_len = ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem,
-						 &eht_cap->eht_cap_elem);
+						 &eht_cap->eht_cap_elem,
+						 for_ap);
 	ppet_len = ieee80211_eht_ppe_size(eht_cap->eht_ppe_thres[0],
 					  eht_cap->eht_cap_elem.phy_cap_info);
 
diff --git a/net/mac80211/vht.c b/net/mac80211/vht.c
index b2b09d421e8b..803de5881485 100644
--- a/net/mac80211/vht.c
+++ b/net/mac80211/vht.c
@@ -323,16 +323,18 @@ ieee80211_vht_cap_ie_to_sta_vht_cap(struct ieee80211_sub_if_data *sdata,
 	 */
 	switch (vht_cap->cap & IEEE80211_VHT_CAP_MAX_MPDU_MASK) {
 	case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454:
-		link_sta->sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_11454;
+		link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_11454;
 		break;
 	case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_7991:
-		link_sta->sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_7991;
+		link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_7991;
 		break;
 	case IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_3895:
 	default:
-		link_sta->sta->sta.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_3895;
+		link_sta->pub->agg.max_amsdu_len = IEEE80211_MAX_MPDU_LEN_VHT_3895;
 		break;
 	}
+
+	ieee80211_sta_recalc_aggregates(&link_sta->sta->sta);
 }
 
 /* FIXME: move this to some better location - parses HE/EHT now */
diff --git a/net/mptcp/mptcp_diag.c b/net/mptcp/mptcp_diag.c
index 7f9a71780437..8df1bdb647e2 100644
--- a/net/mptcp/mptcp_diag.c
+++ b/net/mptcp/mptcp_diag.c
@@ -81,15 +81,18 @@ static void mptcp_diag_dump_listeners(struct sk_buff *skb, struct netlink_callba
 	struct mptcp_diag_ctx *diag_ctx = (void *)cb->ctx;
 	struct nlattr *bc = cb_data->inet_diag_nla_bc;
 	struct net *net = sock_net(skb->sk);
+	struct inet_hashinfo *hinfo;
 	int i;
 
-	for (i = diag_ctx->l_slot; i <= tcp_hashinfo.lhash2_mask; i++) {
+	hinfo = net->ipv4.tcp_death_row.hashinfo;
+
+	for (i = diag_ctx->l_slot; i <= hinfo->lhash2_mask; i++) {
 		struct inet_listen_hashbucket *ilb;
 		struct hlist_nulls_node *node;
 		struct sock *sk;
 		int num = 0;
 
-		ilb = &tcp_hashinfo.lhash2[i];
+		ilb = &hinfo->lhash2[i];
 
 		rcu_read_lock();
 		spin_lock(&ilb->lock);
diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c
index 291b5da42fdb..9813ed0fde9b 100644
--- a/net/mptcp/pm_netlink.c
+++ b/net/mptcp/pm_netlink.c
@@ -796,7 +796,7 @@ static void mptcp_pm_nl_rm_addr_or_subflow(struct mptcp_sock *msk,
 		u8 rm_id = rm_list->ids[i];
 		bool removed = false;
 
-		list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) {
+		mptcp_for_each_subflow_safe(msk, subflow, tmp) {
 			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
 			int how = RCV_SHUTDOWN | SEND_SHUTDOWN;
 			u8 id = subflow->local_id;
@@ -1327,7 +1327,7 @@ static int mptcp_nl_cmd_add_addr(struct sk_buff *skb, struct genl_info *info)
 		return -EINVAL;
 	}
 
-	entry = kmalloc(sizeof(*entry), GFP_KERNEL);
+	entry = kmalloc(sizeof(*entry), GFP_KERNEL_ACCOUNT);
 	if (!entry) {
 		GENL_SET_ERR_MSG(info, "can't allocate addr");
 		return -ENOMEM;
@@ -2218,17 +2218,17 @@ static const struct genl_small_ops mptcp_pm_ops[] = {
 	{
 		.cmd    = MPTCP_PM_CMD_ADD_ADDR,
 		.doit   = mptcp_nl_cmd_add_addr,
-		.flags  = GENL_ADMIN_PERM,
+		.flags  = GENL_UNS_ADMIN_PERM,
 	},
 	{
 		.cmd    = MPTCP_PM_CMD_DEL_ADDR,
 		.doit   = mptcp_nl_cmd_del_addr,
-		.flags  = GENL_ADMIN_PERM,
+		.flags  = GENL_UNS_ADMIN_PERM,
 	},
 	{
 		.cmd    = MPTCP_PM_CMD_FLUSH_ADDRS,
 		.doit   = mptcp_nl_cmd_flush_addrs,
-		.flags  = GENL_ADMIN_PERM,
+		.flags  = GENL_UNS_ADMIN_PERM,
 	},
 	{
 		.cmd    = MPTCP_PM_CMD_GET_ADDR,
@@ -2238,7 +2238,7 @@ static const struct genl_small_ops mptcp_pm_ops[] = {
 	{
 		.cmd    = MPTCP_PM_CMD_SET_LIMITS,
 		.doit   = mptcp_nl_cmd_set_limits,
-		.flags  = GENL_ADMIN_PERM,
+		.flags  = GENL_UNS_ADMIN_PERM,
 	},
 	{
 		.cmd    = MPTCP_PM_CMD_GET_LIMITS,
@@ -2247,27 +2247,27 @@ static const struct genl_small_ops mptcp_pm_ops[] = {
 	{
 		.cmd    = MPTCP_PM_CMD_SET_FLAGS,
 		.doit   = mptcp_nl_cmd_set_flags,
-		.flags  = GENL_ADMIN_PERM,
+		.flags  = GENL_UNS_ADMIN_PERM,
 	},
 	{
 		.cmd    = MPTCP_PM_CMD_ANNOUNCE,
 		.doit   = mptcp_nl_cmd_announce,
-		.flags  = GENL_ADMIN_PERM,
+		.flags  = GENL_UNS_ADMIN_PERM,
 	},
 	{
 		.cmd    = MPTCP_PM_CMD_REMOVE,
 		.doit   = mptcp_nl_cmd_remove,
-		.flags  = GENL_ADMIN_PERM,
+		.flags  = GENL_UNS_ADMIN_PERM,
 	},
 	{
 		.cmd    = MPTCP_PM_CMD_SUBFLOW_CREATE,
 		.doit   = mptcp_nl_cmd_sf_create,
-		.flags  = GENL_ADMIN_PERM,
+		.flags  = GENL_UNS_ADMIN_PERM,
 	},
 	{
 		.cmd    = MPTCP_PM_CMD_SUBFLOW_DESTROY,
 		.doit   = mptcp_nl_cmd_sf_destroy,
-		.flags  = GENL_ADMIN_PERM,
+		.flags  = GENL_UNS_ADMIN_PERM,
 	},
 };
 
@@ -2280,6 +2280,7 @@ static struct genl_family mptcp_genl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= mptcp_pm_ops,
 	.n_small_ops	= ARRAY_SIZE(mptcp_pm_ops),
+	.resv_start_op	= MPTCP_PM_CMD_SUBFLOW_DESTROY + 1,
 	.mcgrps		= mptcp_pm_mcgrps,
 	.n_mcgrps	= ARRAY_SIZE(mptcp_pm_mcgrps),
 };
diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
index 969b33a9dd64..f599ad44ed24 100644
--- a/net/mptcp/protocol.c
+++ b/net/mptcp/protocol.c
@@ -662,9 +662,9 @@ static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,
 
 		skb = skb_peek(&ssk->sk_receive_queue);
 		if (!skb) {
-			/* if no data is found, a racing workqueue/recvmsg
-			 * already processed the new data, stop here or we
-			 * can enter an infinite loop
+			/* With racing move_skbs_to_msk() and __mptcp_move_skbs(),
+			 * a different CPU can have already processed the pending
+			 * data, stop here or we can enter an infinite loop
 			 */
 			if (!moved)
 				done = true;
@@ -672,9 +672,9 @@ static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,
 		}
 
 		if (__mptcp_check_fallback(msk)) {
-			/* if we are running under the workqueue, TCP could have
-			 * collapsed skbs between dummy map creation and now
-			 * be sure to adjust the size
+			/* Under fallback skbs have no MPTCP extension and TCP could
+			 * collapse them between the dummy map creation and the
+			 * current dequeue. Be sure to adjust the map size.
 			 */
 			map_remaining = skb->len;
 			subflow->map_data_len = skb->len;
@@ -1544,8 +1544,9 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags)
 	struct mptcp_sendmsg_info info = {
 				.flags = flags,
 	};
+	bool do_check_data_fin = false;
 	struct mptcp_data_frag *dfrag;
-	int len, copied = 0;
+	int len;
 
 	while ((dfrag = mptcp_send_head(sk))) {
 		info.sent = dfrag->already_sent;
@@ -1580,8 +1581,8 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags)
 				goto out;
 			}
 
+			do_check_data_fin = true;
 			info.sent += ret;
-			copied += ret;
 			len -= ret;
 
 			mptcp_update_post_push(msk, dfrag, ret);
@@ -1597,7 +1598,7 @@ out:
 	/* ensure the rtx timer is running */
 	if (!mptcp_timer_pending(sk))
 		mptcp_reset_timer(sk);
-	if (copied)
+	if (do_check_data_fin)
 		__mptcp_check_send_data_fin(sk);
 }
 
@@ -1676,6 +1677,7 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
 {
 	struct mptcp_sock *msk = mptcp_sk(sk);
 	struct page_frag *pfrag;
+	struct socket *ssock;
 	size_t copied = 0;
 	int ret = 0;
 	long timeo;
@@ -1689,14 +1691,39 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
 
 	lock_sock(sk);
 
+	ssock = __mptcp_nmpc_socket(msk);
+	if (unlikely(ssock && inet_sk(ssock->sk)->defer_connect)) {
+		struct sock *ssk = ssock->sk;
+		int copied_syn = 0;
+
+		lock_sock(ssk);
+
+		ret = tcp_sendmsg_fastopen(ssk, msg, &copied_syn, len, NULL);
+		copied += copied_syn;
+		if (ret == -EINPROGRESS && copied_syn > 0) {
+			/* reflect the new state on the MPTCP socket */
+			inet_sk_state_store(sk, inet_sk_state_load(ssk));
+			release_sock(ssk);
+			goto out;
+		} else if (ret) {
+			release_sock(ssk);
+			goto do_error;
+		}
+		release_sock(ssk);
+	}
+
 	timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
 
 	if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
 		ret = sk_stream_wait_connect(sk, &timeo);
 		if (ret)
-			goto out;
+			goto do_error;
 	}
 
+	ret = -EPIPE;
+	if (unlikely(sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)))
+		goto do_error;
+
 	pfrag = sk_page_frag(sk);
 
 	while (msg_data_left(msg)) {
@@ -1705,11 +1732,6 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
 		bool dfrag_collapsed;
 		size_t psize, offset;
 
-		if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) {
-			ret = -EPIPE;
-			goto out;
-		}
-
 		/* reuse tail pfrag, if possible, or carve a new one from the
 		 * page allocator
 		 */
@@ -1741,7 +1763,7 @@ static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
 		if (copy_page_from_iter(dfrag->page, offset, psize,
 					&msg->msg_iter) != psize) {
 			ret = -EFAULT;
-			goto out;
+			goto do_error;
 		}
 
 		/* data successfully copied into the write queue */
@@ -1773,7 +1795,7 @@ wait_for_memory:
 		__mptcp_push_pending(sk, msg->msg_flags);
 		ret = sk_stream_wait_memory(sk, &timeo);
 		if (ret)
-			goto out;
+			goto do_error;
 	}
 
 	if (copied)
@@ -1781,7 +1803,14 @@ wait_for_memory:
 
 out:
 	release_sock(sk);
-	return copied ? : ret;
+	return copied;
+
+do_error:
+	if (copied)
+		goto out;
+
+	copied = sk_stream_error(sk, msg->msg_flags, ret);
+	goto out;
 }
 
 static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk,
@@ -2284,8 +2313,14 @@ static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,
 
 	lock_sock_nested(ssk, SINGLE_DEPTH_NESTING);
 
-	if (flags & MPTCP_CF_FASTCLOSE)
+	if (flags & MPTCP_CF_FASTCLOSE) {
+		/* be sure to force the tcp_disconnect() path,
+		 * to generate the egress reset
+		 */
+		ssk->sk_lingertime = 0;
+		sock_set_flag(ssk, SOCK_LINGER);
 		subflow->send_fastclose = 1;
+	}
 
 	need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk);
 	if (!dispose_it) {
@@ -2363,7 +2398,7 @@ static void __mptcp_close_subflow(struct mptcp_sock *msk)
 
 	might_sleep();
 
-	list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) {
+	mptcp_for_each_subflow_safe(msk, subflow, tmp) {
 		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
 
 		if (inet_sk_state_load(ssk) != TCP_CLOSE)
@@ -2406,7 +2441,7 @@ static void mptcp_check_fastclose(struct mptcp_sock *msk)
 
 	mptcp_token_destroy(msk);
 
-	list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) {
+	mptcp_for_each_subflow_safe(msk, subflow, tmp) {
 		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
 		bool slow;
 
@@ -2418,12 +2453,31 @@ static void mptcp_check_fastclose(struct mptcp_sock *msk)
 		unlock_sock_fast(tcp_sk, slow);
 	}
 
+	/* Mirror the tcp_reset() error propagation */
+	switch (sk->sk_state) {
+	case TCP_SYN_SENT:
+		sk->sk_err = ECONNREFUSED;
+		break;
+	case TCP_CLOSE_WAIT:
+		sk->sk_err = EPIPE;
+		break;
+	case TCP_CLOSE:
+		return;
+	default:
+		sk->sk_err = ECONNRESET;
+	}
+
 	inet_sk_state_store(sk, TCP_CLOSE);
 	sk->sk_shutdown = SHUTDOWN_MASK;
 	smp_mb__before_atomic(); /* SHUTDOWN must be visible first */
 	set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags);
 
-	mptcp_close_wake_up(sk);
+	/* the calling mptcp_worker will properly destroy the socket */
+	if (sock_flag(sk, SOCK_DEAD))
+		return;
+
+	sk->sk_state_change(sk);
+	sk_error_report(sk);
 }
 
 static void __mptcp_retrans(struct sock *sk)
@@ -2529,6 +2583,16 @@ static void mptcp_mp_fail_no_response(struct mptcp_sock *msk)
 	mptcp_reset_timeout(msk, 0);
 }
 
+static void mptcp_do_fastclose(struct sock *sk)
+{
+	struct mptcp_subflow_context *subflow, *tmp;
+	struct mptcp_sock *msk = mptcp_sk(sk);
+
+	mptcp_for_each_subflow_safe(msk, subflow, tmp)
+		__mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow),
+				  subflow, MPTCP_CF_FASTCLOSE);
+}
+
 static void mptcp_worker(struct work_struct *work)
 {
 	struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work);
@@ -2557,11 +2621,15 @@ static void mptcp_worker(struct work_struct *work)
 	 * closed, but we need the msk around to reply to incoming DATA_FIN,
 	 * even if it is orphaned and in FIN_WAIT2 state
 	 */
-	if (sock_flag(sk, SOCK_DEAD) &&
-	    (mptcp_check_close_timeout(sk) || sk->sk_state == TCP_CLOSE)) {
-		inet_sk_state_store(sk, TCP_CLOSE);
-		__mptcp_destroy_sock(sk);
-		goto unlock;
+	if (sock_flag(sk, SOCK_DEAD)) {
+		if (mptcp_check_close_timeout(sk)) {
+			inet_sk_state_store(sk, TCP_CLOSE);
+			mptcp_do_fastclose(sk);
+		}
+		if (sk->sk_state == TCP_CLOSE) {
+			__mptcp_destroy_sock(sk);
+			goto unlock;
+		}
 	}
 
 	if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags))
@@ -2662,7 +2730,7 @@ static void __mptcp_clear_xmit(struct sock *sk)
 		dfrag_clear(sk, dfrag);
 }
 
-static void mptcp_cancel_work(struct sock *sk)
+void mptcp_cancel_work(struct sock *sk)
 {
 	struct mptcp_sock *msk = mptcp_sk(sk);
 
@@ -2802,13 +2870,24 @@ static void __mptcp_destroy_sock(struct sock *sk)
 	sock_put(sk);
 }
 
-static void mptcp_close(struct sock *sk, long timeout)
+static __poll_t mptcp_check_readable(struct mptcp_sock *msk)
+{
+	/* Concurrent splices from sk_receive_queue into receive_queue will
+	 * always show at least one non-empty queue when checked in this order.
+	 */
+	if (skb_queue_empty_lockless(&((struct sock *)msk)->sk_receive_queue) &&
+	    skb_queue_empty_lockless(&msk->receive_queue))
+		return 0;
+
+	return EPOLLIN | EPOLLRDNORM;
+}
+
+bool __mptcp_close(struct sock *sk, long timeout)
 {
 	struct mptcp_subflow_context *subflow;
 	struct mptcp_sock *msk = mptcp_sk(sk);
 	bool do_cancel_work = false;
 
-	lock_sock(sk);
 	sk->sk_shutdown = SHUTDOWN_MASK;
 
 	if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) {
@@ -2816,8 +2895,13 @@ static void mptcp_close(struct sock *sk, long timeout)
 		goto cleanup;
 	}
 
-	if (mptcp_close_state(sk))
+	if (mptcp_check_readable(msk)) {
+		/* the msk has read data, do the MPTCP equivalent of TCP reset */
+		inet_sk_state_store(sk, TCP_CLOSE);
+		mptcp_do_fastclose(sk);
+	} else if (mptcp_close_state(sk)) {
 		__mptcp_wr_shutdown(sk);
+	}
 
 	sk_stream_wait_close(sk, timeout);
 
@@ -2850,6 +2934,17 @@ cleanup:
 	} else {
 		mptcp_reset_timeout(msk, 0);
 	}
+
+	return do_cancel_work;
+}
+
+static void mptcp_close(struct sock *sk, long timeout)
+{
+	bool do_cancel_work;
+
+	lock_sock(sk);
+
+	do_cancel_work = __mptcp_close(sk, timeout);
 	release_sock(sk);
 	if (do_cancel_work)
 		mptcp_cancel_work(sk);
@@ -3053,7 +3148,7 @@ void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags)
 	__mptcp_clear_xmit(sk);
 
 	/* join list will be eventually flushed (with rst) at sock lock release time */
-	list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node)
+	mptcp_for_each_subflow_safe(msk, subflow, tmp)
 		__mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags);
 
 	/* move to sk_receive_queue, sk_stream_kill_queues will purge it */
@@ -3525,6 +3620,7 @@ static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr,
 
 do_connect:
 	err = ssock->ops->connect(ssock, uaddr, addr_len, flags);
+	inet_sk(sock->sk)->defer_connect = inet_sk(ssock->sk)->defer_connect;
 	sock->state = ssock->state;
 
 	/* on successful connect, the msk state will be moved to established by
@@ -3622,18 +3718,6 @@ static int mptcp_stream_accept(struct socket *sock, struct socket *newsock,
 	return err;
 }
 
-static __poll_t mptcp_check_readable(struct mptcp_sock *msk)
-{
-	/* Concurrent splices from sk_receive_queue into receive_queue will
-	 * always show at least one non-empty queue when checked in this order.
-	 */
-	if (skb_queue_empty_lockless(&((struct sock *)msk)->sk_receive_queue) &&
-	    skb_queue_empty_lockless(&msk->receive_queue))
-		return 0;
-
-	return EPOLLIN | EPOLLRDNORM;
-}
-
 static __poll_t mptcp_check_writeable(struct mptcp_sock *msk)
 {
 	struct sock *sk = (struct sock *)msk;
@@ -3675,13 +3759,16 @@ static __poll_t mptcp_poll(struct file *file, struct socket *sock,
 	if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) {
 		mask |= mptcp_check_readable(msk);
 		mask |= mptcp_check_writeable(msk);
+	} else if (state == TCP_SYN_SENT && inet_sk(sk)->defer_connect) {
+		/* cf tcp_poll() note about TFO */
+		mask |= EPOLLOUT | EPOLLWRNORM;
 	}
 	if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
 		mask |= EPOLLHUP;
 	if (sk->sk_shutdown & RCV_SHUTDOWN)
 		mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
 
-	/* This barrier is coupled with smp_wmb() in tcp_reset() */
+	/* This barrier is coupled with smp_wmb() in __mptcp_error_report() */
 	smp_rmb();
 	if (sk->sk_err)
 		mask |= EPOLLERR;
diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h
index 132d50833df1..c0b5b4628f65 100644
--- a/net/mptcp/protocol.h
+++ b/net/mptcp/protocol.h
@@ -314,6 +314,8 @@ struct mptcp_sock {
 
 #define mptcp_for_each_subflow(__msk, __subflow)			\
 	list_for_each_entry(__subflow, &((__msk)->conn_list), node)
+#define mptcp_for_each_subflow_safe(__msk, __subflow, __tmp)			\
+	list_for_each_entry_safe(__subflow, __tmp, &((__msk)->conn_list), node)
 
 static inline void msk_owned_by_me(const struct mptcp_sock *msk)
 {
@@ -612,6 +614,8 @@ void mptcp_subflow_reset(struct sock *ssk);
 void mptcp_subflow_queue_clean(struct sock *ssk);
 void mptcp_sock_graft(struct sock *sk, struct socket *parent);
 struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk);
+bool __mptcp_close(struct sock *sk, long timeout);
+void mptcp_cancel_work(struct sock *sk);
 
 bool mptcp_addresses_equal(const struct mptcp_addr_info *a,
 			   const struct mptcp_addr_info *b, bool use_port);
diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c
index 423d3826ca1e..c7cb68c725b2 100644
--- a/net/mptcp/sockopt.c
+++ b/net/mptcp/sockopt.c
@@ -559,6 +559,7 @@ static bool mptcp_supported_sockopt(int level, int optname)
 		case TCP_NOTSENT_LOWAT:
 		case TCP_TX_DELAY:
 		case TCP_INQ:
+		case TCP_FASTOPEN_CONNECT:
 			return true;
 		}
 
@@ -567,7 +568,7 @@ static bool mptcp_supported_sockopt(int level, int optname)
 		/* TCP_REPAIR, TCP_REPAIR_QUEUE, TCP_QUEUE_SEQ, TCP_REPAIR_OPTIONS,
 		 * TCP_REPAIR_WINDOW are not supported, better avoid this mess
 		 */
-		/* TCP_FASTOPEN_KEY, TCP_FASTOPEN TCP_FASTOPEN_CONNECT, TCP_FASTOPEN_NO_COOKIE,
+		/* TCP_FASTOPEN_KEY, TCP_FASTOPEN, TCP_FASTOPEN_NO_COOKIE,
 		 * are not supported fastopen is currently unsupported
 		 */
 	}
@@ -768,6 +769,19 @@ static int mptcp_setsockopt_sol_tcp_defer(struct mptcp_sock *msk, sockptr_t optv
 	return tcp_setsockopt(listener->sk, SOL_TCP, TCP_DEFER_ACCEPT, optval, optlen);
 }
 
+static int mptcp_setsockopt_sol_tcp_fastopen_connect(struct mptcp_sock *msk, sockptr_t optval,
+						     unsigned int optlen)
+{
+	struct socket *sock;
+
+	/* Limit to first subflow */
+	sock = __mptcp_nmpc_socket(msk);
+	if (!sock)
+		return -EINVAL;
+
+	return tcp_setsockopt(sock->sk, SOL_TCP, TCP_FASTOPEN_CONNECT, optval, optlen);
+}
+
 static int mptcp_setsockopt_sol_tcp(struct mptcp_sock *msk, int optname,
 				    sockptr_t optval, unsigned int optlen)
 {
@@ -796,6 +810,8 @@ static int mptcp_setsockopt_sol_tcp(struct mptcp_sock *msk, int optname,
 		return mptcp_setsockopt_sol_tcp_nodelay(msk, optval, optlen);
 	case TCP_DEFER_ACCEPT:
 		return mptcp_setsockopt_sol_tcp_defer(msk, optval, optlen);
+	case TCP_FASTOPEN_CONNECT:
+		return mptcp_setsockopt_sol_tcp_fastopen_connect(msk, optval, optlen);
 	}
 
 	return -EOPNOTSUPP;
@@ -1157,6 +1173,7 @@ static int mptcp_getsockopt_sol_tcp(struct mptcp_sock *msk, int optname,
 	case TCP_INFO:
 	case TCP_CC_INFO:
 	case TCP_DEFER_ACCEPT:
+	case TCP_FASTOPEN_CONNECT:
 		return mptcp_getsockopt_first_sf_only(msk, SOL_TCP, optname,
 						      optval, optlen);
 	case TCP_INQ:
diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c
index c7d49fb6e7bd..07dd23d0fe04 100644
--- a/net/mptcp/subflow.c
+++ b/net/mptcp/subflow.c
@@ -602,30 +602,6 @@ static bool subflow_hmac_valid(const struct request_sock *req,
 	return !crypto_memneq(hmac, mp_opt->hmac, MPTCPOPT_HMAC_LEN);
 }
 
-static void mptcp_sock_destruct(struct sock *sk)
-{
-	/* if new mptcp socket isn't accepted, it is free'd
-	 * from the tcp listener sockets request queue, linked
-	 * from req->sk.  The tcp socket is released.
-	 * This calls the ULP release function which will
-	 * also remove the mptcp socket, via
-	 * sock_put(ctx->conn).
-	 *
-	 * Problem is that the mptcp socket will be in
-	 * ESTABLISHED state and will not have the SOCK_DEAD flag.
-	 * Both result in warnings from inet_sock_destruct.
-	 */
-	if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
-		sk->sk_state = TCP_CLOSE;
-		WARN_ON_ONCE(sk->sk_socket);
-		sock_orphan(sk);
-	}
-
-	/* We don't need to clear msk->subflow, as it's still NULL at this point */
-	mptcp_destroy_common(mptcp_sk(sk), 0);
-	inet_sock_destruct(sk);
-}
-
 static void mptcp_force_close(struct sock *sk)
 {
 	/* the msk is not yet exposed to user-space */
@@ -768,7 +744,6 @@ create_child:
 			/* new mpc subflow takes ownership of the newly
 			 * created mptcp socket
 			 */
-			new_msk->sk_destruct = mptcp_sock_destruct;
 			mptcp_sk(new_msk)->setsockopt_seq = ctx->setsockopt_seq;
 			mptcp_pm_new_connection(mptcp_sk(new_msk), child, 1);
 			mptcp_token_accept(subflow_req, mptcp_sk(new_msk));
@@ -1763,13 +1738,19 @@ void mptcp_subflow_queue_clean(struct sock *listener_ssk)
 
 	for (msk = head; msk; msk = next) {
 		struct sock *sk = (struct sock *)msk;
-		bool slow;
+		bool slow, do_cancel_work;
 
+		sock_hold(sk);
 		slow = lock_sock_fast_nested(sk);
 		next = msk->dl_next;
 		msk->first = NULL;
 		msk->dl_next = NULL;
+
+		do_cancel_work = __mptcp_close(sk, 0);
 		unlock_sock_fast(sk, slow);
+		if (do_cancel_work)
+			mptcp_cancel_work(sk);
+		sock_put(sk);
 	}
 
 	/* we are still under the listener msk socket lock */
diff --git a/net/ncsi/ncsi-netlink.c b/net/ncsi/ncsi-netlink.c
index c189b4c8a182..d27f4eccce6d 100644
--- a/net/ncsi/ncsi-netlink.c
+++ b/net/ncsi/ncsi-netlink.c
@@ -768,6 +768,7 @@ static struct genl_family ncsi_genl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.small_ops = ncsi_ops,
 	.n_small_ops = ARRAY_SIZE(ncsi_ops),
+	.resv_start_op = NCSI_CMD_SET_CHANNEL_MASK + 1,
 };
 
 static int __init ncsi_init_netlink(void)
diff --git a/net/netfilter/Makefile b/net/netfilter/Makefile
index 06df49ea6329..0f060d100880 100644
--- a/net/netfilter/Makefile
+++ b/net/netfilter/Makefile
@@ -60,6 +60,12 @@ obj-$(CONFIG_NF_NAT) += nf_nat.o
 nf_nat-$(CONFIG_NF_NAT_REDIRECT) += nf_nat_redirect.o
 nf_nat-$(CONFIG_NF_NAT_MASQUERADE) += nf_nat_masquerade.o
 
+ifeq ($(CONFIG_NF_NAT),m)
+nf_nat-$(CONFIG_DEBUG_INFO_BTF_MODULES) += nf_nat_bpf.o
+else ifeq ($(CONFIG_NF_NAT),y)
+nf_nat-$(CONFIG_DEBUG_INFO_BTF) += nf_nat_bpf.o
+endif
+
 # NAT helpers
 obj-$(CONFIG_NF_NAT_AMANDA) += nf_nat_amanda.o
 obj-$(CONFIG_NF_NAT_FTP) += nf_nat_ftp.o
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
index dcf752b55a52..5a6705a0e4ec 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
@@ -300,12 +300,6 @@ nf_hook_entry_head(struct net *net, int pf, unsigned int hooknum,
 		if (WARN_ON_ONCE(ARRAY_SIZE(net->nf.hooks_ipv6) <= hooknum))
 			return NULL;
 		return net->nf.hooks_ipv6 + hooknum;
-#if IS_ENABLED(CONFIG_DECNET)
-	case NFPROTO_DECNET:
-		if (WARN_ON_ONCE(ARRAY_SIZE(net->nf.hooks_decnet) <= hooknum))
-			return NULL;
-		return net->nf.hooks_decnet + hooknum;
-#endif
 	default:
 		WARN_ON_ONCE(1);
 		return NULL;
@@ -750,10 +744,6 @@ static int __net_init netfilter_net_init(struct net *net)
 #ifdef CONFIG_NETFILTER_FAMILY_BRIDGE
 	__netfilter_net_init(net->nf.hooks_bridge, ARRAY_SIZE(net->nf.hooks_bridge));
 #endif
-#if IS_ENABLED(CONFIG_DECNET)
-	__netfilter_net_init(net->nf.hooks_decnet, ARRAY_SIZE(net->nf.hooks_decnet));
-#endif
-
 #ifdef CONFIG_PROC_FS
 	net->nf.proc_netfilter = proc_net_mkdir(net, "netfilter",
 						net->proc_net);
diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c
index 16ae92054baa..e7ba5b6dd2b7 100644
--- a/net/netfilter/ipset/ip_set_core.c
+++ b/net/netfilter/ipset/ip_set_core.c
@@ -353,7 +353,7 @@ ip_set_init_comment(struct ip_set *set, struct ip_set_comment *comment,
 	c = kmalloc(sizeof(*c) + len + 1, GFP_ATOMIC);
 	if (unlikely(!c))
 		return;
-	strlcpy(c->str, ext->comment, len + 1);
+	strscpy(c->str, ext->comment, len + 1);
 	set->ext_size += sizeof(*c) + strlen(c->str) + 1;
 	rcu_assign_pointer(comment->c, c);
 }
@@ -1072,7 +1072,7 @@ static int ip_set_create(struct sk_buff *skb, const struct nfnl_info *info,
 	if (!set)
 		return -ENOMEM;
 	spin_lock_init(&set->lock);
-	strlcpy(set->name, name, IPSET_MAXNAMELEN);
+	strscpy(set->name, name, IPSET_MAXNAMELEN);
 	set->family = family;
 	set->revision = revision;
 
@@ -1719,11 +1719,13 @@ call_ad(struct net *net, struct sock *ctnl, struct sk_buff *skb,
 		skb2 = nlmsg_new(payload, GFP_KERNEL);
 		if (!skb2)
 			return -ENOMEM;
-		rep = __nlmsg_put(skb2, NETLINK_CB(skb).portid,
-				  nlh->nlmsg_seq, NLMSG_ERROR, payload, 0);
+		rep = nlmsg_put(skb2, NETLINK_CB(skb).portid,
+				nlh->nlmsg_seq, NLMSG_ERROR, payload, 0);
 		errmsg = nlmsg_data(rep);
 		errmsg->error = ret;
-		memcpy(&errmsg->msg, nlh, nlh->nlmsg_len);
+		unsafe_memcpy(&errmsg->msg, nlh, nlh->nlmsg_len,
+			      /* Bounds checked by the skb layer. */);
+
 		cmdattr = (void *)&errmsg->msg + min_len;
 
 		ret = nla_parse(cda, IPSET_ATTR_CMD_MAX, cmdattr,
diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c
index efab2b06d373..988222fff9f0 100644
--- a/net/netfilter/ipvs/ip_vs_ctl.c
+++ b/net/netfilter/ipvs/ip_vs_ctl.c
@@ -2611,7 +2611,7 @@ ip_vs_copy_service(struct ip_vs_service_entry *dst, struct ip_vs_service *src)
 	dst->addr = src->addr.ip;
 	dst->port = src->port;
 	dst->fwmark = src->fwmark;
-	strlcpy(dst->sched_name, sched_name, sizeof(dst->sched_name));
+	strscpy(dst->sched_name, sched_name, sizeof(dst->sched_name));
 	dst->flags = src->flags;
 	dst->timeout = src->timeout / HZ;
 	dst->netmask = src->netmask;
@@ -2805,13 +2805,13 @@ do_ip_vs_get_ctl(struct sock *sk, int cmd, void __user *user, int *len)
 		mutex_lock(&ipvs->sync_mutex);
 		if (ipvs->sync_state & IP_VS_STATE_MASTER) {
 			d[0].state = IP_VS_STATE_MASTER;
-			strlcpy(d[0].mcast_ifn, ipvs->mcfg.mcast_ifn,
+			strscpy(d[0].mcast_ifn, ipvs->mcfg.mcast_ifn,
 				sizeof(d[0].mcast_ifn));
 			d[0].syncid = ipvs->mcfg.syncid;
 		}
 		if (ipvs->sync_state & IP_VS_STATE_BACKUP) {
 			d[1].state = IP_VS_STATE_BACKUP;
-			strlcpy(d[1].mcast_ifn, ipvs->bcfg.mcast_ifn,
+			strscpy(d[1].mcast_ifn, ipvs->bcfg.mcast_ifn,
 				sizeof(d[1].mcast_ifn));
 			d[1].syncid = ipvs->bcfg.syncid;
 		}
@@ -3561,7 +3561,7 @@ static int ip_vs_genl_new_daemon(struct netns_ipvs *ipvs, struct nlattr **attrs)
 	      attrs[IPVS_DAEMON_ATTR_MCAST_IFN] &&
 	      attrs[IPVS_DAEMON_ATTR_SYNC_ID]))
 		return -EINVAL;
-	strlcpy(c.mcast_ifn, nla_data(attrs[IPVS_DAEMON_ATTR_MCAST_IFN]),
+	strscpy(c.mcast_ifn, nla_data(attrs[IPVS_DAEMON_ATTR_MCAST_IFN]),
 		sizeof(c.mcast_ifn));
 	c.syncid = nla_get_u32(attrs[IPVS_DAEMON_ATTR_SYNC_ID]);
 
@@ -4005,6 +4005,7 @@ static struct genl_family ip_vs_genl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= ip_vs_genl_ops,
 	.n_small_ops	= ARRAY_SIZE(ip_vs_genl_ops),
+	.resv_start_op	= IPVS_CMD_FLUSH + 1,
 };
 
 static int __init ip_vs_genl_register(void)
diff --git a/net/netfilter/nf_conntrack_bpf.c b/net/netfilter/nf_conntrack_bpf.c
index 1cd87b28c9b0..8639e7efd0e2 100644
--- a/net/netfilter/nf_conntrack_bpf.c
+++ b/net/netfilter/nf_conntrack_bpf.c
@@ -6,12 +6,14 @@
  * are exposed through to BPF programs is explicitly unstable.
  */
 
+#include <linux/bpf_verifier.h>
 #include <linux/bpf.h>
 #include <linux/btf.h>
+#include <linux/filter.h>
+#include <linux/mutex.h>
 #include <linux/types.h>
 #include <linux/btf_ids.h>
 #include <linux/net_namespace.h>
-#include <net/netfilter/nf_conntrack.h>
 #include <net/netfilter/nf_conntrack_bpf.h>
 #include <net/netfilter/nf_conntrack_core.h>
 
@@ -134,7 +136,6 @@ __bpf_nf_ct_alloc_entry(struct net *net, struct bpf_sock_tuple *bpf_tuple,
 
 	memset(&ct->proto, 0, sizeof(ct->proto));
 	__nf_ct_set_timeout(ct, timeout * HZ);
-	ct->status |= IPS_CONFIRMED;
 
 out:
 	if (opts->netns_id >= 0)
@@ -184,14 +185,58 @@ static struct nf_conn *__bpf_nf_ct_lookup(struct net *net,
 	return ct;
 }
 
+BTF_ID_LIST(btf_nf_conn_ids)
+BTF_ID(struct, nf_conn)
+BTF_ID(struct, nf_conn___init)
+
+/* Check writes into `struct nf_conn` */
+static int _nf_conntrack_btf_struct_access(struct bpf_verifier_log *log,
+					   const struct btf *btf,
+					   const struct btf_type *t, int off,
+					   int size, enum bpf_access_type atype,
+					   u32 *next_btf_id,
+					   enum bpf_type_flag *flag)
+{
+	const struct btf_type *ncit;
+	const struct btf_type *nct;
+	size_t end;
+
+	ncit = btf_type_by_id(btf, btf_nf_conn_ids[1]);
+	nct = btf_type_by_id(btf, btf_nf_conn_ids[0]);
+
+	if (t != nct && t != ncit) {
+		bpf_log(log, "only read is supported\n");
+		return -EACCES;
+	}
+
+	/* `struct nf_conn` and `struct nf_conn___init` have the same layout
+	 * so we are safe to simply merge offset checks here
+	 */
+	switch (off) {
+#if defined(CONFIG_NF_CONNTRACK_MARK)
+	case offsetof(struct nf_conn, mark):
+		end = offsetofend(struct nf_conn, mark);
+		break;
+#endif
+	default:
+		bpf_log(log, "no write support to nf_conn at off %d\n", off);
+		return -EACCES;
+	}
+
+	if (off + size > end) {
+		bpf_log(log,
+			"write access at off %d with size %d beyond the member of nf_conn ended at %zu\n",
+			off, size, end);
+		return -EACCES;
+	}
+
+	return 0;
+}
+
 __diag_push();
 __diag_ignore_all("-Wmissing-prototypes",
 		  "Global functions as their definitions will be in nf_conntrack BTF");
 
-struct nf_conn___init {
-	struct nf_conn ct;
-};
-
 /* bpf_xdp_ct_alloc - Allocate a new CT entry
  *
  * Parameters:
@@ -339,6 +384,7 @@ struct nf_conn *bpf_ct_insert_entry(struct nf_conn___init *nfct_i)
 	struct nf_conn *nfct = (struct nf_conn *)nfct_i;
 	int err;
 
+	nfct->status |= IPS_CONFIRMED;
 	err = nf_conntrack_hash_check_insert(nfct);
 	if (err < 0) {
 		nf_conntrack_free(nfct);
@@ -449,5 +495,19 @@ int register_nf_conntrack_bpf(void)
 	int ret;
 
 	ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP, &nf_conntrack_kfunc_set);
-	return ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &nf_conntrack_kfunc_set);
+	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS, &nf_conntrack_kfunc_set);
+	if (!ret) {
+		mutex_lock(&nf_conn_btf_access_lock);
+		nfct_btf_struct_access = _nf_conntrack_btf_struct_access;
+		mutex_unlock(&nf_conn_btf_access_lock);
+	}
+
+	return ret;
+}
+
+void cleanup_nf_conntrack_bpf(void)
+{
+	mutex_lock(&nf_conn_btf_access_lock);
+	nfct_btf_struct_access = NULL;
+	mutex_unlock(&nf_conn_btf_access_lock);
 }
diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c
index 1357a2729a4b..f97bda06d2a9 100644
--- a/net/netfilter/nf_conntrack_core.c
+++ b/net/netfilter/nf_conntrack_core.c
@@ -67,6 +67,7 @@ struct conntrack_gc_work {
 	struct delayed_work	dwork;
 	u32			next_bucket;
 	u32			avg_timeout;
+	u32			count;
 	u32			start_time;
 	bool			exiting;
 	bool			early_drop;
@@ -85,10 +86,12 @@ static DEFINE_MUTEX(nf_conntrack_mutex);
 /* clamp timeouts to this value (TCP unacked) */
 #define GC_SCAN_INTERVAL_CLAMP	(300ul * HZ)
 
-/* large initial bias so that we don't scan often just because we have
- * three entries with a 1s timeout.
+/* Initial bias pretending we have 100 entries at the upper bound so we don't
+ * wakeup often just because we have three entries with a 1s timeout while still
+ * allowing non-idle machines to wakeup more often when needed.
  */
-#define GC_SCAN_INTERVAL_INIT	INT_MAX
+#define GC_SCAN_INITIAL_COUNT	100
+#define GC_SCAN_INTERVAL_INIT	GC_SCAN_INTERVAL_MAX
 
 #define GC_SCAN_MAX_DURATION	msecs_to_jiffies(10)
 #define GC_SCAN_EXPIRED_MAX	(64000u / HZ)
@@ -1466,6 +1469,7 @@ static void gc_worker(struct work_struct *work)
 	unsigned int expired_count = 0;
 	unsigned long next_run;
 	s32 delta_time;
+	long count;
 
 	gc_work = container_of(work, struct conntrack_gc_work, dwork.work);
 
@@ -1475,10 +1479,12 @@ static void gc_worker(struct work_struct *work)
 
 	if (i == 0) {
 		gc_work->avg_timeout = GC_SCAN_INTERVAL_INIT;
+		gc_work->count = GC_SCAN_INITIAL_COUNT;
 		gc_work->start_time = start_time;
 	}
 
 	next_run = gc_work->avg_timeout;
+	count = gc_work->count;
 
 	end_time = start_time + GC_SCAN_MAX_DURATION;
 
@@ -1498,8 +1504,8 @@ static void gc_worker(struct work_struct *work)
 
 		hlist_nulls_for_each_entry_rcu(h, n, &ct_hash[i], hnnode) {
 			struct nf_conntrack_net *cnet;
-			unsigned long expires;
 			struct net *net;
+			long expires;
 
 			tmp = nf_ct_tuplehash_to_ctrack(h);
 
@@ -1513,6 +1519,7 @@ static void gc_worker(struct work_struct *work)
 
 				gc_work->next_bucket = i;
 				gc_work->avg_timeout = next_run;
+				gc_work->count = count;
 
 				delta_time = nfct_time_stamp - gc_work->start_time;
 
@@ -1528,8 +1535,8 @@ static void gc_worker(struct work_struct *work)
 			}
 
 			expires = clamp(nf_ct_expires(tmp), GC_SCAN_INTERVAL_MIN, GC_SCAN_INTERVAL_CLAMP);
+			expires = (expires - (long)next_run) / ++count;
 			next_run += expires;
-			next_run /= 2u;
 
 			if (nf_conntrack_max95 == 0 || gc_worker_skip_ct(tmp))
 				continue;
@@ -1570,6 +1577,7 @@ static void gc_worker(struct work_struct *work)
 		delta_time = nfct_time_stamp - end_time;
 		if (delta_time > 0 && i < hashsz) {
 			gc_work->avg_timeout = next_run;
+			gc_work->count = count;
 			gc_work->next_bucket = i;
 			next_run = 0;
 			goto early_exit;
@@ -2508,6 +2516,7 @@ static int kill_all(struct nf_conn *i, void *data)
 
 void nf_conntrack_cleanup_start(void)
 {
+	cleanup_nf_conntrack_bpf();
 	conntrack_gc_work.exiting = true;
 }
 
@@ -2802,10 +2811,6 @@ err_expect:
 	return ret;
 }
 
-#if (IS_BUILTIN(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) || \
-    (IS_MODULE(CONFIG_NF_CONNTRACK) && IS_ENABLED(CONFIG_DEBUG_INFO_BTF_MODULES) || \
-    IS_ENABLED(CONFIG_NF_CT_NETLINK))
-
 /* ctnetlink code shared by both ctnetlink and nf_conntrack_bpf */
 
 int __nf_ct_change_timeout(struct nf_conn *ct, u64 timeout)
@@ -2861,5 +2866,3 @@ int nf_ct_change_status_common(struct nf_conn *ct, unsigned int status)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(nf_ct_change_status_common);
-
-#endif
diff --git a/net/netfilter/nf_conntrack_proto_tcp.c b/net/netfilter/nf_conntrack_proto_tcp.c
index a634c72b1ffc..656631083177 100644
--- a/net/netfilter/nf_conntrack_proto_tcp.c
+++ b/net/netfilter/nf_conntrack_proto_tcp.c
@@ -47,6 +47,12 @@ static const char *const tcp_conntrack_names[] = {
 	"SYN_SENT2",
 };
 
+enum nf_ct_tcp_action {
+	NFCT_TCP_IGNORE,
+	NFCT_TCP_INVALID,
+	NFCT_TCP_ACCEPT,
+};
+
 #define SECS * HZ
 #define MINS * 60 SECS
 #define HOURS * 60 MINS
@@ -472,23 +478,45 @@ static void tcp_init_sender(struct ip_ct_tcp_state *sender,
 	}
 }
 
-static bool tcp_in_window(struct nf_conn *ct,
-			  enum ip_conntrack_dir dir,
-			  unsigned int index,
-			  const struct sk_buff *skb,
-			  unsigned int dataoff,
-			  const struct tcphdr *tcph,
-			  const struct nf_hook_state *hook_state)
+__printf(6, 7)
+static enum nf_ct_tcp_action nf_tcp_log_invalid(const struct sk_buff *skb,
+						const struct nf_conn *ct,
+						const struct nf_hook_state *state,
+						const struct ip_ct_tcp_state *sender,
+						enum nf_ct_tcp_action ret,
+						const char *fmt, ...)
+{
+	const struct nf_tcp_net *tn = nf_tcp_pernet(nf_ct_net(ct));
+	struct va_format vaf;
+	va_list args;
+	bool be_liberal;
+
+	be_liberal = sender->flags & IP_CT_TCP_FLAG_BE_LIBERAL || tn->tcp_be_liberal;
+	if (be_liberal)
+		return NFCT_TCP_ACCEPT;
+
+	va_start(args, fmt);
+	vaf.fmt = fmt;
+	vaf.va = &args;
+	nf_ct_l4proto_log_invalid(skb, ct, state, "%pV", &vaf);
+	va_end(args);
+
+	return ret;
+}
+
+static enum nf_ct_tcp_action
+tcp_in_window(struct nf_conn *ct, enum ip_conntrack_dir dir,
+	      unsigned int index, const struct sk_buff *skb,
+	      unsigned int dataoff, const struct tcphdr *tcph,
+	      const struct nf_hook_state *hook_state)
 {
 	struct ip_ct_tcp *state = &ct->proto.tcp;
-	struct net *net = nf_ct_net(ct);
-	struct nf_tcp_net *tn = nf_tcp_pernet(net);
 	struct ip_ct_tcp_state *sender = &state->seen[dir];
 	struct ip_ct_tcp_state *receiver = &state->seen[!dir];
 	__u32 seq, ack, sack, end, win, swin;
-	u16 win_raw;
+	bool in_recv_win, seq_ok;
 	s32 receiver_offset;
-	bool res, in_recv_win;
+	u16 win_raw;
 
 	/*
 	 * Get the required data from the packet.
@@ -517,7 +545,7 @@ static bool tcp_in_window(struct nf_conn *ct,
 					end, win);
 			if (!tcph->ack)
 				/* Simultaneous open */
-				return true;
+				return NFCT_TCP_ACCEPT;
 		} else {
 			/*
 			 * We are in the middle of a connection,
@@ -560,7 +588,7 @@ static bool tcp_in_window(struct nf_conn *ct,
 				end, win);
 
 		if (dir == IP_CT_DIR_REPLY && !tcph->ack)
-			return true;
+			return NFCT_TCP_ACCEPT;
 	}
 
 	if (!(tcph->ack)) {
@@ -584,122 +612,166 @@ static bool tcp_in_window(struct nf_conn *ct,
 		 */
 		seq = end = sender->td_end;
 
-	/* Is the ending sequence in the receive window (if available)? */
-	in_recv_win = !receiver->td_maxwin ||
-		      after(end, sender->td_end - receiver->td_maxwin - 1);
-
-	if (before(seq, sender->td_maxend + 1) &&
-	    in_recv_win &&
-	    before(sack, receiver->td_end + 1) &&
-	    after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1)) {
-		/*
-		 * Take into account window scaling (RFC 1323).
-		 */
-		if (!tcph->syn)
-			win <<= sender->td_scale;
-
-		/*
-		 * Update sender data.
-		 */
-		swin = win + (sack - ack);
-		if (sender->td_maxwin < swin)
-			sender->td_maxwin = swin;
-		if (after(end, sender->td_end)) {
+	seq_ok = before(seq, sender->td_maxend + 1);
+	if (!seq_ok) {
+		u32 overshot = end - sender->td_maxend + 1;
+		bool ack_ok;
+
+		ack_ok = after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1);
+		in_recv_win = receiver->td_maxwin &&
+			      after(end, sender->td_end - receiver->td_maxwin - 1);
+
+		if (in_recv_win &&
+		    ack_ok &&
+		    overshot <= receiver->td_maxwin &&
+		    before(sack, receiver->td_end + 1)) {
+			/* Work around TCPs that send more bytes than allowed by
+			 * the receive window.
+			 *
+			 * If the (marked as invalid) packet is allowed to pass by
+			 * the ruleset and the peer acks this data, then its possible
+			 * all future packets will trigger 'ACK is over upper bound' check.
+			 *
+			 * Thus if only the sequence check fails then do update td_end so
+			 * possible ACK for this data can update internal state.
+			 */
 			sender->td_end = end;
 			sender->flags |= IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED;
-		}
-		if (tcph->ack) {
-			if (!(sender->flags & IP_CT_TCP_FLAG_MAXACK_SET)) {
-				sender->td_maxack = ack;
-				sender->flags |= IP_CT_TCP_FLAG_MAXACK_SET;
-			} else if (after(ack, sender->td_maxack))
-				sender->td_maxack = ack;
+
+			return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE,
+						  "%u bytes more than expected", overshot);
 		}
 
-		/*
-		 * Update receiver data.
-		 */
-		if (receiver->td_maxwin != 0 && after(end, sender->td_maxend))
-			receiver->td_maxwin += end - sender->td_maxend;
-		if (after(sack + win, receiver->td_maxend - 1)) {
-			receiver->td_maxend = sack + win;
-			if (win == 0)
-				receiver->td_maxend++;
+		return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_INVALID,
+					  "SEQ is over upper bound %u (over the window of the receiver)",
+					  sender->td_maxend + 1);
+	}
+
+	if (!before(sack, receiver->td_end + 1))
+		return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_INVALID,
+					  "ACK is over upper bound %u (ACKed data not seen yet)",
+					  receiver->td_end + 1);
+
+	/* Is the ending sequence in the receive window (if available)? */
+	in_recv_win = !receiver->td_maxwin ||
+		      after(end, sender->td_end - receiver->td_maxwin - 1);
+	if (!in_recv_win)
+		return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE,
+					  "SEQ is under lower bound %u (already ACKed data retransmitted)",
+					  sender->td_end - receiver->td_maxwin - 1);
+	if (!after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1))
+		return nf_tcp_log_invalid(skb, ct, hook_state, sender, NFCT_TCP_IGNORE,
+					  "ignored ACK under lower bound %u (possible overly delayed)",
+					  receiver->td_end - MAXACKWINDOW(sender) - 1);
+
+	/* Take into account window scaling (RFC 1323). */
+	if (!tcph->syn)
+		win <<= sender->td_scale;
+
+	/* Update sender data. */
+	swin = win + (sack - ack);
+	if (sender->td_maxwin < swin)
+		sender->td_maxwin = swin;
+	if (after(end, sender->td_end)) {
+		sender->td_end = end;
+		sender->flags |= IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED;
+	}
+	if (tcph->ack) {
+		if (!(sender->flags & IP_CT_TCP_FLAG_MAXACK_SET)) {
+			sender->td_maxack = ack;
+			sender->flags |= IP_CT_TCP_FLAG_MAXACK_SET;
+		} else if (after(ack, sender->td_maxack)) {
+			sender->td_maxack = ack;
 		}
-		if (ack == receiver->td_end)
-			receiver->flags &= ~IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED;
+	}
 
-		/*
-		 * Check retransmissions.
-		 */
-		if (index == TCP_ACK_SET) {
-			if (state->last_dir == dir
-			    && state->last_seq == seq
-			    && state->last_ack == ack
-			    && state->last_end == end
-			    && state->last_win == win_raw)
-				state->retrans++;
-			else {
-				state->last_dir = dir;
-				state->last_seq = seq;
-				state->last_ack = ack;
-				state->last_end = end;
-				state->last_win = win_raw;
-				state->retrans = 0;
-			}
+	/* Update receiver data. */
+	if (receiver->td_maxwin != 0 && after(end, sender->td_maxend))
+		receiver->td_maxwin += end - sender->td_maxend;
+	if (after(sack + win, receiver->td_maxend - 1)) {
+		receiver->td_maxend = sack + win;
+		if (win == 0)
+			receiver->td_maxend++;
+	}
+	if (ack == receiver->td_end)
+		receiver->flags &= ~IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED;
+
+	/* Check retransmissions. */
+	if (index == TCP_ACK_SET) {
+		if (state->last_dir == dir &&
+		    state->last_seq == seq &&
+		    state->last_ack == ack &&
+		    state->last_end == end &&
+		    state->last_win == win_raw) {
+			state->retrans++;
+		} else {
+			state->last_dir = dir;
+			state->last_seq = seq;
+			state->last_ack = ack;
+			state->last_end = end;
+			state->last_win = win_raw;
+			state->retrans = 0;
 		}
-		res = true;
-	} else {
-		res = false;
-		if (sender->flags & IP_CT_TCP_FLAG_BE_LIBERAL ||
-		    tn->tcp_be_liberal)
-			res = true;
-		if (!res) {
-			bool seq_ok = before(seq, sender->td_maxend + 1);
-
-			if (!seq_ok) {
-				u32 overshot = end - sender->td_maxend + 1;
-				bool ack_ok;
-
-				ack_ok = after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1);
-
-				if (in_recv_win &&
-				    ack_ok &&
-				    overshot <= receiver->td_maxwin &&
-				    before(sack, receiver->td_end + 1)) {
-					/* Work around TCPs that send more bytes than allowed by
-					 * the receive window.
-					 *
-					 * If the (marked as invalid) packet is allowed to pass by
-					 * the ruleset and the peer acks this data, then its possible
-					 * all future packets will trigger 'ACK is over upper bound' check.
-					 *
-					 * Thus if only the sequence check fails then do update td_end so
-					 * possible ACK for this data can update internal state.
-					 */
-					sender->td_end = end;
-					sender->flags |= IP_CT_TCP_FLAG_DATA_UNACKNOWLEDGED;
-
-					nf_ct_l4proto_log_invalid(skb, ct, hook_state,
-								  "%u bytes more than expected", overshot);
-					return res;
-				}
-			}
+	}
+
+	return NFCT_TCP_ACCEPT;
+}
+
+static void __cold nf_tcp_handle_invalid(struct nf_conn *ct,
+					 enum ip_conntrack_dir dir,
+					 int index,
+					 const struct sk_buff *skb,
+					 const struct nf_hook_state *hook_state)
+{
+	const unsigned int *timeouts;
+	const struct nf_tcp_net *tn;
+	unsigned int timeout;
+	u32 expires;
+
+	if (!test_bit(IPS_ASSURED_BIT, &ct->status) ||
+	    test_bit(IPS_FIXED_TIMEOUT_BIT, &ct->status))
+		return;
+
+	/* We don't want to have connections hanging around in ESTABLISHED
+	 * state for long time 'just because' conntrack deemed a FIN/RST
+	 * out-of-window.
+	 *
+	 * Shrink the timeout just like when there is unacked data.
+	 * This speeds up eviction of 'dead' connections where the
+	 * connection and conntracks internal state are out of sync.
+	 */
+	switch (index) {
+	case TCP_RST_SET:
+	case TCP_FIN_SET:
+		break;
+	default:
+		return;
+	}
+
+	if (ct->proto.tcp.last_dir != dir &&
+	    (ct->proto.tcp.last_index == TCP_FIN_SET ||
+	     ct->proto.tcp.last_index == TCP_RST_SET)) {
+		expires = nf_ct_expires(ct);
+		if (expires < 120 * HZ)
+			return;
+
+		tn = nf_tcp_pernet(nf_ct_net(ct));
+		timeouts = nf_ct_timeout_lookup(ct);
+		if (!timeouts)
+			timeouts = tn->timeouts;
 
+		timeout = READ_ONCE(timeouts[TCP_CONNTRACK_UNACK]);
+		if (expires > timeout) {
 			nf_ct_l4proto_log_invalid(skb, ct, hook_state,
-			"%s",
-			before(seq, sender->td_maxend + 1) ?
-			in_recv_win ?
-			before(sack, receiver->td_end + 1) ?
-			after(sack, receiver->td_end - MAXACKWINDOW(sender) - 1) ? "BUG"
-			: "ACK is under the lower bound (possible overly delayed ACK)"
-			: "ACK is over the upper bound (ACKed data not seen yet)"
-			: "SEQ is under the lower bound (already ACKed data retransmitted)"
-			: "SEQ is over the upper bound (over the window of the receiver)");
+					  "packet (index %d, dir %d) response for index %d lower timeout to %u",
+					  index, dir, ct->proto.tcp.last_index, timeout);
+
+			WRITE_ONCE(ct->timeout, timeout + nfct_time_stamp);
 		}
+	} else {
+		ct->proto.tcp.last_index = index;
+		ct->proto.tcp.last_dir = dir;
 	}
-
-	return res;
 }
 
 /* table of valid flag combinations - PUSH, ECE and CWR are always valid */
@@ -861,6 +933,7 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct,
 	struct nf_conntrack_tuple *tuple;
 	enum tcp_conntrack new_state, old_state;
 	unsigned int index, *timeouts;
+	enum nf_ct_tcp_action res;
 	enum ip_conntrack_dir dir;
 	const struct tcphdr *th;
 	struct tcphdr _tcph;
@@ -1126,10 +1199,18 @@ int nf_conntrack_tcp_packet(struct nf_conn *ct,
 		break;
 	}
 
-	if (!tcp_in_window(ct, dir, index,
-			   skb, dataoff, th, state)) {
+	res = tcp_in_window(ct, dir, index,
+			    skb, dataoff, th, state);
+	switch (res) {
+	case NFCT_TCP_IGNORE:
+		spin_unlock_bh(&ct->lock);
+		return NF_ACCEPT;
+	case NFCT_TCP_INVALID:
+		nf_tcp_handle_invalid(ct, dir, index, skb, state);
 		spin_unlock_bh(&ct->lock);
 		return -NF_ACCEPT;
+	case NFCT_TCP_ACCEPT:
+		break;
 	}
      in_window:
 	/* From now on we have got in-window packets */
diff --git a/net/netfilter/nf_log.c b/net/netfilter/nf_log.c
index edee7fa944c1..8a29290149bd 100644
--- a/net/netfilter/nf_log.c
+++ b/net/netfilter/nf_log.c
@@ -443,9 +443,9 @@ static int nf_log_proc_dostring(struct ctl_table *table, int write,
 		mutex_lock(&nf_log_mutex);
 		logger = nft_log_dereference(net->nf.nf_loggers[tindex]);
 		if (!logger)
-			strlcpy(buf, "NONE", sizeof(buf));
+			strscpy(buf, "NONE", sizeof(buf));
 		else
-			strlcpy(buf, logger->name, sizeof(buf));
+			strscpy(buf, logger->name, sizeof(buf));
 		mutex_unlock(&nf_log_mutex);
 		r = proc_dostring(&tmp, write, buffer, lenp, ppos);
 	}
diff --git a/net/netfilter/nf_nat_amanda.c b/net/netfilter/nf_nat_amanda.c
index 3bc7e0854efe..98deef6cde69 100644
--- a/net/netfilter/nf_nat_amanda.c
+++ b/net/netfilter/nf_nat_amanda.c
@@ -44,19 +44,7 @@ static unsigned int help(struct sk_buff *skb,
 	exp->expectfn = nf_nat_follow_master;
 
 	/* Try to get same port: if not, try to change it. */
-	for (port = ntohs(exp->saved_proto.tcp.port); port != 0; port++) {
-		int res;
-
-		exp->tuple.dst.u.tcp.port = htons(port);
-		res = nf_ct_expect_related(exp, 0);
-		if (res == 0)
-			break;
-		else if (res != -EBUSY) {
-			port = 0;
-			break;
-		}
-	}
-
+	port = nf_nat_exp_find_port(exp, ntohs(exp->saved_proto.tcp.port));
 	if (port == 0) {
 		nf_ct_helper_log(skb, exp->master, "all ports in use");
 		return NF_DROP;
diff --git a/net/netfilter/nf_nat_bpf.c b/net/netfilter/nf_nat_bpf.c
new file mode 100644
index 000000000000..0fa5a0bbb0ff
--- /dev/null
+++ b/net/netfilter/nf_nat_bpf.c
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Unstable NAT Helpers for XDP and TC-BPF hook
+ *
+ * These are called from the XDP and SCHED_CLS BPF programs. Note that it is
+ * allowed to break compatibility for these functions since the interface they
+ * are exposed through to BPF programs is explicitly unstable.
+ */
+
+#include <linux/bpf.h>
+#include <linux/btf_ids.h>
+#include <net/netfilter/nf_conntrack_bpf.h>
+#include <net/netfilter/nf_conntrack_core.h>
+#include <net/netfilter/nf_nat.h>
+
+__diag_push();
+__diag_ignore_all("-Wmissing-prototypes",
+		  "Global functions as their definitions will be in nf_nat BTF");
+
+/* bpf_ct_set_nat_info - Set source or destination nat address
+ *
+ * Set source or destination nat address of the newly allocated
+ * nf_conn before insertion. This must be invoked for referenced
+ * PTR_TO_BTF_ID to nf_conn___init.
+ *
+ * Parameters:
+ * @nfct	- Pointer to referenced nf_conn object, obtained using
+ *		  bpf_xdp_ct_alloc or bpf_skb_ct_alloc.
+ * @addr	- Nat source/destination address
+ * @port	- Nat source/destination port. Non-positive values are
+ *		  interpreted as select a random port.
+ * @manip	- NF_NAT_MANIP_SRC or NF_NAT_MANIP_DST
+ */
+int bpf_ct_set_nat_info(struct nf_conn___init *nfct,
+			union nf_inet_addr *addr, int port,
+			enum nf_nat_manip_type manip)
+{
+	struct nf_conn *ct = (struct nf_conn *)nfct;
+	u16 proto = nf_ct_l3num(ct);
+	struct nf_nat_range2 range;
+
+	if (proto != NFPROTO_IPV4 && proto != NFPROTO_IPV6)
+		return -EINVAL;
+
+	memset(&range, 0, sizeof(struct nf_nat_range2));
+	range.flags = NF_NAT_RANGE_MAP_IPS;
+	range.min_addr = *addr;
+	range.max_addr = range.min_addr;
+	if (port > 0) {
+		range.flags |= NF_NAT_RANGE_PROTO_SPECIFIED;
+		range.min_proto.all = cpu_to_be16(port);
+		range.max_proto.all = range.min_proto.all;
+	}
+
+	return nf_nat_setup_info(ct, &range, manip) == NF_DROP ? -ENOMEM : 0;
+}
+
+__diag_pop()
+
+BTF_SET8_START(nf_nat_kfunc_set)
+BTF_ID_FLAGS(func, bpf_ct_set_nat_info, KF_TRUSTED_ARGS)
+BTF_SET8_END(nf_nat_kfunc_set)
+
+static const struct btf_kfunc_id_set nf_bpf_nat_kfunc_set = {
+	.owner = THIS_MODULE,
+	.set   = &nf_nat_kfunc_set,
+};
+
+int register_nf_nat_bpf(void)
+{
+	int ret;
+
+	ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_XDP,
+					&nf_bpf_nat_kfunc_set);
+	if (ret)
+		return ret;
+
+	return register_btf_kfunc_id_set(BPF_PROG_TYPE_SCHED_CLS,
+					 &nf_bpf_nat_kfunc_set);
+}
diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c
index 7981be526f26..d8e6380f6337 100644
--- a/net/netfilter/nf_nat_core.c
+++ b/net/netfilter/nf_nat_core.c
@@ -16,7 +16,7 @@
 #include <linux/siphash.h>
 #include <linux/rtnetlink.h>
 
-#include <net/netfilter/nf_conntrack.h>
+#include <net/netfilter/nf_conntrack_bpf.h>
 #include <net/netfilter/nf_conntrack_core.h>
 #include <net/netfilter/nf_conntrack_helper.h>
 #include <net/netfilter/nf_conntrack_seqadj.h>
@@ -1152,7 +1152,7 @@ static int __init nf_nat_init(void)
 	WARN_ON(nf_nat_hook != NULL);
 	RCU_INIT_POINTER(nf_nat_hook, &nat_hook);
 
-	return 0;
+	return register_nf_nat_bpf();
 }
 
 static void __exit nf_nat_cleanup(void)
diff --git a/net/netfilter/nf_nat_ftp.c b/net/netfilter/nf_nat_ftp.c
index aace6768a64e..c92a436d9c48 100644
--- a/net/netfilter/nf_nat_ftp.c
+++ b/net/netfilter/nf_nat_ftp.c
@@ -86,22 +86,9 @@ static unsigned int nf_nat_ftp(struct sk_buff *skb,
 	 * this one. */
 	exp->expectfn = nf_nat_follow_master;
 
-	/* Try to get same port: if not, try to change it. */
-	for (port = ntohs(exp->saved_proto.tcp.port); port != 0; port++) {
-		int ret;
-
-		exp->tuple.dst.u.tcp.port = htons(port);
-		ret = nf_ct_expect_related(exp, 0);
-		if (ret == 0)
-			break;
-		else if (ret != -EBUSY) {
-			port = 0;
-			break;
-		}
-	}
-
+	port = nf_nat_exp_find_port(exp, ntohs(exp->saved_proto.tcp.port));
 	if (port == 0) {
-		nf_ct_helper_log(skb, ct, "all ports in use");
+		nf_ct_helper_log(skb, exp->master, "all ports in use");
 		return NF_DROP;
 	}
 
diff --git a/net/netfilter/nf_nat_helper.c b/net/netfilter/nf_nat_helper.c
index a263505455fc..a95a25196943 100644
--- a/net/netfilter/nf_nat_helper.c
+++ b/net/netfilter/nf_nat_helper.c
@@ -198,3 +198,34 @@ void nf_nat_follow_master(struct nf_conn *ct,
 	nf_nat_setup_info(ct, &range, NF_NAT_MANIP_DST);
 }
 EXPORT_SYMBOL(nf_nat_follow_master);
+
+u16 nf_nat_exp_find_port(struct nf_conntrack_expect *exp, u16 port)
+{
+	static const unsigned int max_attempts = 128;
+	int range, attempts_left;
+	u16 min = port;
+
+	range = USHRT_MAX - port;
+	attempts_left = range;
+
+	if (attempts_left > max_attempts)
+		attempts_left = max_attempts;
+
+	/* Try to get same port: if not, try to change it. */
+	for (;;) {
+		int res;
+
+		exp->tuple.dst.u.tcp.port = htons(port);
+		res = nf_ct_expect_related(exp, 0);
+		if (res == 0)
+			return port;
+
+		if (res != -EBUSY || (--attempts_left < 0))
+			break;
+
+		port = min + prandom_u32_max(range);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nf_nat_exp_find_port);
diff --git a/net/netfilter/nf_nat_irc.c b/net/netfilter/nf_nat_irc.c
index c691ab8d234c..19c4fcc60c50 100644
--- a/net/netfilter/nf_nat_irc.c
+++ b/net/netfilter/nf_nat_irc.c
@@ -48,20 +48,8 @@ static unsigned int help(struct sk_buff *skb,
 	exp->dir = IP_CT_DIR_REPLY;
 	exp->expectfn = nf_nat_follow_master;
 
-	/* Try to get same port: if not, try to change it. */
-	for (port = ntohs(exp->saved_proto.tcp.port); port != 0; port++) {
-		int ret;
-
-		exp->tuple.dst.u.tcp.port = htons(port);
-		ret = nf_ct_expect_related(exp, 0);
-		if (ret == 0)
-			break;
-		else if (ret != -EBUSY) {
-			port = 0;
-			break;
-		}
-	}
-
+	port = nf_nat_exp_find_port(exp,
+				    ntohs(exp->saved_proto.tcp.port));
 	if (port == 0) {
 		nf_ct_helper_log(skb, ct, "all ports in use");
 		return NF_DROP;
diff --git a/net/netfilter/nf_nat_sip.c b/net/netfilter/nf_nat_sip.c
index f0a735e86851..cf4aeb299bde 100644
--- a/net/netfilter/nf_nat_sip.c
+++ b/net/netfilter/nf_nat_sip.c
@@ -410,19 +410,7 @@ static unsigned int nf_nat_sip_expect(struct sk_buff *skb, unsigned int protoff,
 	exp->dir = !dir;
 	exp->expectfn = nf_nat_sip_expected;
 
-	for (; port != 0; port++) {
-		int ret;
-
-		exp->tuple.dst.u.udp.port = htons(port);
-		ret = nf_ct_expect_related(exp, NF_CT_EXP_F_SKIP_MASTER);
-		if (ret == 0)
-			break;
-		else if (ret != -EBUSY) {
-			port = 0;
-			break;
-		}
-	}
-
+	port = nf_nat_exp_find_port(exp, port);
 	if (port == 0) {
 		nf_ct_helper_log(skb, ct, "all ports in use for SIP");
 		return NF_DROP;
diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c
index 63c70141b3e5..a0653a8dfa82 100644
--- a/net/netfilter/nf_tables_api.c
+++ b/net/netfilter/nf_tables_api.c
@@ -742,7 +742,7 @@ __printf(2, 3) int nft_request_module(struct net *net, const char *fmt,
 		return -ENOMEM;
 
 	req->done = false;
-	strlcpy(req->module, module_name, MODULE_NAME_LEN);
+	strscpy(req->module, module_name, MODULE_NAME_LEN);
 	list_add_tail(&req->list, &nft_net->module_list);
 
 	return -EAGAIN;
diff --git a/net/netfilter/nfnetlink_hook.c b/net/netfilter/nfnetlink_hook.c
index 71e29adac48b..8120aadf6a0f 100644
--- a/net/netfilter/nfnetlink_hook.c
+++ b/net/netfilter/nfnetlink_hook.c
@@ -215,13 +215,6 @@ nfnl_hook_entries_head(u8 pf, unsigned int hook, struct net *net, const char *de
 		hook_head = rcu_dereference(net->nf.hooks_bridge[hook]);
 #endif
 		break;
-#if IS_ENABLED(CONFIG_DECNET)
-	case NFPROTO_DECNET:
-		if (hook >= ARRAY_SIZE(net->nf.hooks_decnet))
-			return ERR_PTR(-EINVAL);
-		hook_head = rcu_dereference(net->nf.hooks_decnet[hook]);
-		break;
-#endif
 #if defined(CONFIG_NETFILTER_INGRESS) || defined(CONFIG_NETFILTER_EGRESS)
 	case NFPROTO_NETDEV:
 		if (hook >= NF_NETDEV_NUMHOOKS)
diff --git a/net/netfilter/nft_osf.c b/net/netfilter/nft_osf.c
index 89342ccccdcc..adacf95b6e2b 100644
--- a/net/netfilter/nft_osf.c
+++ b/net/netfilter/nft_osf.c
@@ -51,7 +51,7 @@ static void nft_osf_eval(const struct nft_expr *expr, struct nft_regs *regs,
 			snprintf(os_match, NFT_OSF_MAXGENRELEN, "%s:%s",
 				 data.genre, data.version);
 		else
-			strlcpy(os_match, data.genre, NFT_OSF_MAXGENRELEN);
+			strscpy(os_match, data.genre, NFT_OSF_MAXGENRELEN);
 
 		strncpy((char *)dest, os_match, NFT_OSF_MAXGENRELEN);
 	}
diff --git a/net/netfilter/nft_payload.c b/net/netfilter/nft_payload.c
index eb0e40c29712..088244f9d838 100644
--- a/net/netfilter/nft_payload.c
+++ b/net/netfilter/nft_payload.c
@@ -173,10 +173,10 @@ static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = {
 	[NFTA_PAYLOAD_SREG]		= { .type = NLA_U32 },
 	[NFTA_PAYLOAD_DREG]		= { .type = NLA_U32 },
 	[NFTA_PAYLOAD_BASE]		= { .type = NLA_U32 },
-	[NFTA_PAYLOAD_OFFSET]		= { .type = NLA_U32 },
-	[NFTA_PAYLOAD_LEN]		= { .type = NLA_U32 },
+	[NFTA_PAYLOAD_OFFSET]		= NLA_POLICY_MAX_BE(NLA_U32, 255),
+	[NFTA_PAYLOAD_LEN]		= NLA_POLICY_MAX_BE(NLA_U32, 255),
 	[NFTA_PAYLOAD_CSUM_TYPE]	= { .type = NLA_U32 },
-	[NFTA_PAYLOAD_CSUM_OFFSET]	= { .type = NLA_U32 },
+	[NFTA_PAYLOAD_CSUM_OFFSET]	= NLA_POLICY_MAX_BE(NLA_U32, 255),
 	[NFTA_PAYLOAD_CSUM_FLAGS]	= { .type = NLA_U32 },
 };
 
diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c
index 54a489f16b17..470282cf3fae 100644
--- a/net/netfilter/x_tables.c
+++ b/net/netfilter/x_tables.c
@@ -766,7 +766,7 @@ void xt_compat_match_from_user(struct xt_entry_match *m, void **dstptr,
 
 	msize += off;
 	m->u.user.match_size = msize;
-	strlcpy(name, match->name, sizeof(name));
+	strscpy(name, match->name, sizeof(name));
 	module_put(match->me);
 	strncpy(m->u.user.name, name, sizeof(m->u.user.name));
 
@@ -1146,7 +1146,7 @@ void xt_compat_target_from_user(struct xt_entry_target *t, void **dstptr,
 
 	tsize += off;
 	t->u.user.target_size = tsize;
-	strlcpy(name, target->name, sizeof(name));
+	strscpy(name, target->name, sizeof(name));
 	module_put(target->me);
 	strncpy(t->u.user.name, name, sizeof(t->u.user.name));
 
@@ -1827,7 +1827,7 @@ int xt_proto_init(struct net *net, u_int8_t af)
 	root_uid = make_kuid(net->user_ns, 0);
 	root_gid = make_kgid(net->user_ns, 0);
 
-	strlcpy(buf, xt_prefix[af], sizeof(buf));
+	strscpy(buf, xt_prefix[af], sizeof(buf));
 	strlcat(buf, FORMAT_TABLES, sizeof(buf));
 	proc = proc_create_net_data(buf, 0440, net->proc_net, &xt_table_seq_ops,
 			sizeof(struct seq_net_private),
@@ -1837,7 +1837,7 @@ int xt_proto_init(struct net *net, u_int8_t af)
 	if (uid_valid(root_uid) && gid_valid(root_gid))
 		proc_set_user(proc, root_uid, root_gid);
 
-	strlcpy(buf, xt_prefix[af], sizeof(buf));
+	strscpy(buf, xt_prefix[af], sizeof(buf));
 	strlcat(buf, FORMAT_MATCHES, sizeof(buf));
 	proc = proc_create_seq_private(buf, 0440, net->proc_net,
 			&xt_match_seq_ops, sizeof(struct nf_mttg_trav),
@@ -1847,7 +1847,7 @@ int xt_proto_init(struct net *net, u_int8_t af)
 	if (uid_valid(root_uid) && gid_valid(root_gid))
 		proc_set_user(proc, root_uid, root_gid);
 
-	strlcpy(buf, xt_prefix[af], sizeof(buf));
+	strscpy(buf, xt_prefix[af], sizeof(buf));
 	strlcat(buf, FORMAT_TARGETS, sizeof(buf));
 	proc = proc_create_seq_private(buf, 0440, net->proc_net,
 			 &xt_target_seq_ops, sizeof(struct nf_mttg_trav),
@@ -1862,12 +1862,12 @@ int xt_proto_init(struct net *net, u_int8_t af)
 
 #ifdef CONFIG_PROC_FS
 out_remove_matches:
-	strlcpy(buf, xt_prefix[af], sizeof(buf));
+	strscpy(buf, xt_prefix[af], sizeof(buf));
 	strlcat(buf, FORMAT_MATCHES, sizeof(buf));
 	remove_proc_entry(buf, net->proc_net);
 
 out_remove_tables:
-	strlcpy(buf, xt_prefix[af], sizeof(buf));
+	strscpy(buf, xt_prefix[af], sizeof(buf));
 	strlcat(buf, FORMAT_TABLES, sizeof(buf));
 	remove_proc_entry(buf, net->proc_net);
 out:
@@ -1881,15 +1881,15 @@ void xt_proto_fini(struct net *net, u_int8_t af)
 #ifdef CONFIG_PROC_FS
 	char buf[XT_FUNCTION_MAXNAMELEN];
 
-	strlcpy(buf, xt_prefix[af], sizeof(buf));
+	strscpy(buf, xt_prefix[af], sizeof(buf));
 	strlcat(buf, FORMAT_TABLES, sizeof(buf));
 	remove_proc_entry(buf, net->proc_net);
 
-	strlcpy(buf, xt_prefix[af], sizeof(buf));
+	strscpy(buf, xt_prefix[af], sizeof(buf));
 	strlcat(buf, FORMAT_TARGETS, sizeof(buf));
 	remove_proc_entry(buf, net->proc_net);
 
-	strlcpy(buf, xt_prefix[af], sizeof(buf));
+	strscpy(buf, xt_prefix[af], sizeof(buf));
 	strlcat(buf, FORMAT_MATCHES, sizeof(buf));
 	remove_proc_entry(buf, net->proc_net);
 #endif /*CONFIG_PROC_FS*/
diff --git a/net/netfilter/xt_RATEEST.c b/net/netfilter/xt_RATEEST.c
index 8aec1b529364..80f6624e2355 100644
--- a/net/netfilter/xt_RATEEST.c
+++ b/net/netfilter/xt_RATEEST.c
@@ -144,7 +144,7 @@ static int xt_rateest_tg_checkentry(const struct xt_tgchk_param *par)
 		goto err1;
 
 	gnet_stats_basic_sync_init(&est->bstats);
-	strlcpy(est->name, info->name, sizeof(est->name));
+	strscpy(est->name, info->name, sizeof(est->name));
 	spin_lock_init(&est->lock);
 	est->refcnt		= 1;
 	est->params.interval	= info->interval;
diff --git a/net/netlabel/netlabel_calipso.c b/net/netlabel/netlabel_calipso.c
index 91a19c3ea1a3..f1d5b8465217 100644
--- a/net/netlabel/netlabel_calipso.c
+++ b/net/netlabel/netlabel_calipso.c
@@ -344,6 +344,7 @@ static struct genl_family netlbl_calipso_gnl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.small_ops = netlbl_calipso_ops,
 	.n_small_ops = ARRAY_SIZE(netlbl_calipso_ops),
+	.resv_start_op = NLBL_CALIPSO_C_LISTALL + 1,
 };
 
 /* NetLabel Generic NETLINK Protocol Functions
diff --git a/net/netlabel/netlabel_cipso_v4.c b/net/netlabel/netlabel_cipso_v4.c
index 894e6b8f1a86..fa08ee75ac06 100644
--- a/net/netlabel/netlabel_cipso_v4.c
+++ b/net/netlabel/netlabel_cipso_v4.c
@@ -767,6 +767,7 @@ static struct genl_family netlbl_cipsov4_gnl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.small_ops = netlbl_cipsov4_ops,
 	.n_small_ops = ARRAY_SIZE(netlbl_cipsov4_ops),
+	.resv_start_op = NLBL_CIPSOV4_C_LISTALL + 1,
 };
 
 /*
diff --git a/net/netlabel/netlabel_mgmt.c b/net/netlabel/netlabel_mgmt.c
index 032b7d7b32c7..689eaa2afbec 100644
--- a/net/netlabel/netlabel_mgmt.c
+++ b/net/netlabel/netlabel_mgmt.c
@@ -826,6 +826,7 @@ static struct genl_family netlbl_mgmt_gnl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.small_ops = netlbl_mgmt_genl_ops,
 	.n_small_ops = ARRAY_SIZE(netlbl_mgmt_genl_ops),
+	.resv_start_op = NLBL_MGMT_C_VERSION + 1,
 };
 
 /*
diff --git a/net/netlabel/netlabel_unlabeled.c b/net/netlabel/netlabel_unlabeled.c
index 0555dffd80e0..9996883bf2b7 100644
--- a/net/netlabel/netlabel_unlabeled.c
+++ b/net/netlabel/netlabel_unlabeled.c
@@ -1374,6 +1374,7 @@ static struct genl_family netlbl_unlabel_gnl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.small_ops = netlbl_unlabel_genl_ops,
 	.n_small_ops = ARRAY_SIZE(netlbl_unlabel_genl_ops),
+	.resv_start_op = NLBL_UNLABEL_C_STATICLISTDEF + 1,
 };
 
 /*
diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
index 0cd91f813a3b..a662e8a5ff84 100644
--- a/net/netlink/af_netlink.c
+++ b/net/netlink/af_netlink.c
@@ -2400,6 +2400,69 @@ error_free:
 }
 EXPORT_SYMBOL(__netlink_dump_start);
 
+static size_t
+netlink_ack_tlv_len(struct netlink_sock *nlk, int err,
+		    const struct netlink_ext_ack *extack)
+{
+	size_t tlvlen;
+
+	if (!extack || !(nlk->flags & NETLINK_F_EXT_ACK))
+		return 0;
+
+	tlvlen = 0;
+	if (extack->_msg)
+		tlvlen += nla_total_size(strlen(extack->_msg) + 1);
+	if (extack->cookie_len)
+		tlvlen += nla_total_size(extack->cookie_len);
+
+	/* Following attributes are only reported as error (not warning) */
+	if (!err)
+		return tlvlen;
+
+	if (extack->bad_attr)
+		tlvlen += nla_total_size(sizeof(u32));
+	if (extack->policy)
+		tlvlen += netlink_policy_dump_attr_size_estimate(extack->policy);
+	if (extack->miss_type)
+		tlvlen += nla_total_size(sizeof(u32));
+	if (extack->miss_nest)
+		tlvlen += nla_total_size(sizeof(u32));
+
+	return tlvlen;
+}
+
+static void
+netlink_ack_tlv_fill(struct sk_buff *in_skb, struct sk_buff *skb,
+		     struct nlmsghdr *nlh, int err,
+		     const struct netlink_ext_ack *extack)
+{
+	if (extack->_msg)
+		WARN_ON(nla_put_string(skb, NLMSGERR_ATTR_MSG, extack->_msg));
+	if (extack->cookie_len)
+		WARN_ON(nla_put(skb, NLMSGERR_ATTR_COOKIE,
+				extack->cookie_len, extack->cookie));
+
+	if (!err)
+		return;
+
+	if (extack->bad_attr &&
+	    !WARN_ON((u8 *)extack->bad_attr < in_skb->data ||
+		     (u8 *)extack->bad_attr >= in_skb->data + in_skb->len))
+		WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_OFFS,
+				    (u8 *)extack->bad_attr - (u8 *)nlh));
+	if (extack->policy)
+		netlink_policy_dump_write_attr(skb, extack->policy,
+					       NLMSGERR_ATTR_POLICY);
+	if (extack->miss_type)
+		WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_MISS_TYPE,
+				    extack->miss_type));
+	if (extack->miss_nest &&
+	    !WARN_ON((u8 *)extack->miss_nest < in_skb->data ||
+		     (u8 *)extack->miss_nest > in_skb->data + in_skb->len))
+		WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_MISS_NEST,
+				    (u8 *)extack->miss_nest - (u8 *)nlh));
+}
+
 void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err,
 		 const struct netlink_ext_ack *extack)
 {
@@ -2407,29 +2470,20 @@ void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err,
 	struct nlmsghdr *rep;
 	struct nlmsgerr *errmsg;
 	size_t payload = sizeof(*errmsg);
-	size_t tlvlen = 0;
 	struct netlink_sock *nlk = nlk_sk(NETLINK_CB(in_skb).sk);
 	unsigned int flags = 0;
-	bool nlk_has_extack = nlk->flags & NETLINK_F_EXT_ACK;
+	size_t tlvlen;
 
 	/* Error messages get the original request appened, unless the user
 	 * requests to cap the error message, and get extra error data if
 	 * requested.
 	 */
-	if (nlk_has_extack && extack && extack->_msg)
-		tlvlen += nla_total_size(strlen(extack->_msg) + 1);
-
 	if (err && !(nlk->flags & NETLINK_F_CAP_ACK))
 		payload += nlmsg_len(nlh);
 	else
 		flags |= NLM_F_CAPPED;
-	if (err && nlk_has_extack && extack && extack->bad_attr)
-		tlvlen += nla_total_size(sizeof(u32));
-	if (nlk_has_extack && extack && extack->cookie_len)
-		tlvlen += nla_total_size(extack->cookie_len);
-	if (err && nlk_has_extack && extack && extack->policy)
-		tlvlen += netlink_policy_dump_attr_size_estimate(extack->policy);
 
+	tlvlen = netlink_ack_tlv_len(nlk, err, extack);
 	if (tlvlen)
 		flags |= NLM_F_ACK_TLVS;
 
@@ -2440,31 +2494,16 @@ void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err,
 		return;
 	}
 
-	rep = __nlmsg_put(skb, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
-			  NLMSG_ERROR, payload, flags);
+	rep = nlmsg_put(skb, NETLINK_CB(in_skb).portid, nlh->nlmsg_seq,
+			NLMSG_ERROR, payload, flags);
 	errmsg = nlmsg_data(rep);
 	errmsg->error = err;
-	memcpy(&errmsg->msg, nlh, payload > sizeof(*errmsg) ? nlh->nlmsg_len : sizeof(*nlh));
+	unsafe_memcpy(&errmsg->msg, nlh, payload > sizeof(*errmsg)
+					 ? nlh->nlmsg_len : sizeof(*nlh),
+		      /* Bounds checked by the skb layer. */);
 
-	if (nlk_has_extack && extack) {
-		if (extack->_msg) {
-			WARN_ON(nla_put_string(skb, NLMSGERR_ATTR_MSG,
-					       extack->_msg));
-		}
-		if (err && extack->bad_attr &&
-		    !WARN_ON((u8 *)extack->bad_attr < in_skb->data ||
-			     (u8 *)extack->bad_attr >= in_skb->data +
-						       in_skb->len))
-			WARN_ON(nla_put_u32(skb, NLMSGERR_ATTR_OFFS,
-					    (u8 *)extack->bad_attr -
-					    (u8 *)nlh));
-		if (extack->cookie_len)
-			WARN_ON(nla_put(skb, NLMSGERR_ATTR_COOKIE,
-					extack->cookie_len, extack->cookie));
-		if (extack->policy)
-			netlink_policy_dump_write_attr(skb, extack->policy,
-						       NLMSGERR_ATTR_POLICY);
-	}
+	if (tlvlen)
+		netlink_ack_tlv_fill(in_skb, skb, nlh, err, extack);
 
 	nlmsg_end(skb, rep);
 
diff --git a/net/netlink/genetlink.c b/net/netlink/genetlink.c
index 57010927e20a..39b7c00e4cef 100644
--- a/net/netlink/genetlink.c
+++ b/net/netlink/genetlink.c
@@ -739,6 +739,36 @@ out:
 	return err;
 }
 
+static int genl_header_check(const struct genl_family *family,
+			     struct nlmsghdr *nlh, struct genlmsghdr *hdr,
+			     struct netlink_ext_ack *extack)
+{
+	u16 flags;
+
+	/* Only for commands added after we started validating */
+	if (hdr->cmd < family->resv_start_op)
+		return 0;
+
+	if (hdr->reserved) {
+		NL_SET_ERR_MSG(extack, "genlmsghdr.reserved field is not 0");
+		return -EINVAL;
+	}
+
+	/* Old netlink flags have pretty loose semantics, allow only the flags
+	 * consumed by the core where we can enforce the meaning.
+	 */
+	flags = nlh->nlmsg_flags;
+	if ((flags & NLM_F_DUMP) == NLM_F_DUMP) /* DUMP is 2 bits */
+		flags &= ~NLM_F_DUMP;
+	if (flags & ~(NLM_F_REQUEST | NLM_F_ACK | NLM_F_ECHO)) {
+		NL_SET_ERR_MSG(extack,
+			       "ambiguous or reserved bits set in nlmsg_flags");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int genl_family_rcv_msg(const struct genl_family *family,
 			       struct sk_buff *skb,
 			       struct nlmsghdr *nlh,
@@ -757,6 +787,9 @@ static int genl_family_rcv_msg(const struct genl_family *family,
 	if (nlh->nlmsg_len < nlmsg_msg_size(hdrlen))
 		return -EINVAL;
 
+	if (genl_header_check(family, nlh, hdr, extack))
+		return -EINVAL;
+
 	if (genl_get_cmd(hdr->cmd, family, &op))
 		return -EOPNOTSUPP;
 
@@ -1348,6 +1381,7 @@ static struct genl_family genl_ctrl __ro_after_init = {
 	.module = THIS_MODULE,
 	.ops = genl_ctrl_ops,
 	.n_ops = ARRAY_SIZE(genl_ctrl_ops),
+	.resv_start_op = CTRL_CMD_GETPOLICY + 1,
 	.mcgrps = genl_ctrl_groups,
 	.n_mcgrps = ARRAY_SIZE(genl_ctrl_groups),
 	.id = GENL_ID_CTRL,
@@ -1362,7 +1396,7 @@ static int genl_bind(struct net *net, int group)
 	unsigned int id;
 	int ret = 0;
 
-	genl_lock_all();
+	down_read(&cb_lock);
 
 	idr_for_each_entry(&genl_fam_idr, family, id) {
 		const struct genl_multicast_group *grp;
@@ -1383,7 +1417,7 @@ static int genl_bind(struct net *net, int group)
 		break;
 	}
 
-	genl_unlock_all();
+	up_read(&cb_lock);
 	return ret;
 }
 
diff --git a/net/nfc/hci/hcp.c b/net/nfc/hci/hcp.c
index 05c60988f59a..4902f5064098 100644
--- a/net/nfc/hci/hcp.c
+++ b/net/nfc/hci/hcp.c
@@ -73,14 +73,12 @@ int nfc_hci_hcp_message_tx(struct nfc_hci_dev *hdev, u8 pipe,
 		if (firstfrag) {
 			firstfrag = false;
 			packet->message.header = HCP_HEADER(type, instruction);
-			if (ptr) {
-				memcpy(packet->message.data, ptr,
-				       data_link_len - 1);
-				ptr += data_link_len - 1;
-			}
 		} else {
-			memcpy(&packet->message, ptr, data_link_len);
-			ptr += data_link_len;
+			packet->message.header = *ptr++;
+		}
+		if (ptr) {
+			memcpy(packet->message.data, ptr, data_link_len - 1);
+			ptr += data_link_len - 1;
 		}
 
 		/* This is the last fragment, set the cb bit */
diff --git a/net/nfc/netlink.c b/net/nfc/netlink.c
index 7c62417ccfd7..9d91087b9399 100644
--- a/net/nfc/netlink.c
+++ b/net/nfc/netlink.c
@@ -1783,6 +1783,7 @@ static struct genl_family nfc_genl_family __ro_after_init = {
 	.module = THIS_MODULE,
 	.ops = nfc_genl_ops,
 	.n_ops = ARRAY_SIZE(nfc_genl_ops),
+	.resv_start_op = NFC_CMD_DEACTIVATE_TARGET + 1,
 	.mcgrps = nfc_genl_mcgrps,
 	.n_mcgrps = ARRAY_SIZE(nfc_genl_mcgrps),
 };
diff --git a/net/openvswitch/conntrack.c b/net/openvswitch/conntrack.c
index 4e70df91d0f2..cb255d8ed99a 100644
--- a/net/openvswitch/conntrack.c
+++ b/net/openvswitch/conntrack.c
@@ -1982,7 +1982,8 @@ static int ovs_ct_limit_set_zone_limit(struct nlattr *nla_zone_limit,
 		} else {
 			struct ovs_ct_limit *ct_limit;
 
-			ct_limit = kmalloc(sizeof(*ct_limit), GFP_KERNEL);
+			ct_limit = kmalloc(sizeof(*ct_limit),
+					   GFP_KERNEL_ACCOUNT);
 			if (!ct_limit)
 				return -ENOMEM;
 
@@ -2252,14 +2253,16 @@ exit_err:
 static const struct genl_small_ops ct_limit_genl_ops[] = {
 	{ .cmd = OVS_CT_LIMIT_CMD_SET,
 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN
-					   * privilege. */
+		.flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN
+					       * privilege.
+					       */
 		.doit = ovs_ct_limit_cmd_set,
 	},
 	{ .cmd = OVS_CT_LIMIT_CMD_DEL,
 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN
-					   * privilege. */
+		.flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN
+					       * privilege.
+					       */
 		.doit = ovs_ct_limit_cmd_del,
 	},
 	{ .cmd = OVS_CT_LIMIT_CMD_GET,
@@ -2283,6 +2286,7 @@ struct genl_family dp_ct_limit_genl_family __ro_after_init = {
 	.parallel_ops = true,
 	.small_ops = ct_limit_genl_ops,
 	.n_small_ops = ARRAY_SIZE(ct_limit_genl_ops),
+	.resv_start_op = OVS_CT_LIMIT_CMD_GET + 1,
 	.mcgrps = &ovs_ct_limit_multicast_group,
 	.n_mcgrps = 1,
 	.module = THIS_MODULE,
diff --git a/net/openvswitch/datapath.c b/net/openvswitch/datapath.c
index 6c9d153afbee..c8a9075ddd0a 100644
--- a/net/openvswitch/datapath.c
+++ b/net/openvswitch/datapath.c
@@ -252,10 +252,17 @@ void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
 
 		upcall.mru = OVS_CB(skb)->mru;
 		error = ovs_dp_upcall(dp, skb, key, &upcall, 0);
-		if (unlikely(error))
-			kfree_skb(skb);
-		else
+		switch (error) {
+		case 0:
+		case -EAGAIN:
+		case -ERESTARTSYS:
+		case -EINTR:
 			consume_skb(skb);
+			break;
+		default:
+			kfree_skb(skb);
+			break;
+		}
 		stats_counter = &stats->n_missed;
 		goto out;
 	}
@@ -551,8 +558,9 @@ static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
 out:
 	if (err)
 		skb_tx_error(skb);
-	kfree_skb(user_skb);
-	kfree_skb(nskb);
+	consume_skb(user_skb);
+	consume_skb(nskb);
+
 	return err;
 }
 
@@ -684,6 +692,7 @@ static struct genl_family dp_packet_genl_family __ro_after_init = {
 	.parallel_ops = true,
 	.small_ops = dp_packet_genl_ops,
 	.n_small_ops = ARRAY_SIZE(dp_packet_genl_ops),
+	.resv_start_op = OVS_PACKET_CMD_EXECUTE + 1,
 	.module = THIS_MODULE,
 };
 
@@ -1501,6 +1510,7 @@ static struct genl_family dp_flow_genl_family __ro_after_init = {
 	.parallel_ops = true,
 	.small_ops = dp_flow_genl_ops,
 	.n_small_ops = ARRAY_SIZE(dp_flow_genl_ops),
+	.resv_start_op = OVS_FLOW_CMD_SET + 1,
 	.mcgrps = &ovs_dp_flow_multicast_group,
 	.n_mcgrps = 1,
 	.module = THIS_MODULE,
@@ -1515,6 +1525,7 @@ static size_t ovs_dp_cmd_msg_size(void)
 	msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats));
 	msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
 	msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_MASKS_CACHE_SIZE */
+	msgsize += nla_total_size(sizeof(u32) * nr_cpu_ids); /* OVS_DP_ATTR_PER_CPU_PIDS */
 
 	return msgsize;
 }
@@ -1526,7 +1537,8 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
 	struct ovs_header *ovs_header;
 	struct ovs_dp_stats dp_stats;
 	struct ovs_dp_megaflow_stats dp_megaflow_stats;
-	int err;
+	struct dp_nlsk_pids *pids = ovsl_dereference(dp->upcall_portids);
+	int err, pids_len;
 
 	ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
 				 flags, cmd);
@@ -1556,6 +1568,12 @@ static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
 			ovs_flow_tbl_masks_cache_size(&dp->table)))
 		goto nla_put_failure;
 
+	if (dp->user_features & OVS_DP_F_DISPATCH_UPCALL_PER_CPU && pids) {
+		pids_len = min(pids->n_pids, nr_cpu_ids) * sizeof(u32);
+		if (nla_put(skb, OVS_DP_ATTR_PER_CPU_PIDS, pids_len, &pids->pids))
+			goto nla_put_failure;
+	}
+
 	genlmsg_end(skb, ovs_header);
 	return 0;
 
@@ -1779,6 +1797,8 @@ static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
 	parms.dp = dp;
 	parms.port_no = OVSP_LOCAL;
 	parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
+	parms.desired_ifindex = a[OVS_DP_ATTR_IFINDEX]
+		? nla_get_u32(a[OVS_DP_ATTR_IFINDEX]) : 0;
 
 	/* So far only local changes have been made, now need the lock. */
 	ovs_lock();
@@ -1998,6 +2018,7 @@ static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
 	[OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
 	[OVS_DP_ATTR_MASKS_CACHE_SIZE] =  NLA_POLICY_RANGE(NLA_U32, 0,
 		PCPU_MIN_UNIT_SIZE / sizeof(struct mask_cache_entry)),
+	[OVS_DP_ATTR_IFINDEX] = {.type = NLA_U32 },
 };
 
 static const struct genl_small_ops dp_datapath_genl_ops[] = {
@@ -2034,6 +2055,7 @@ static struct genl_family dp_datapath_genl_family __ro_after_init = {
 	.parallel_ops = true,
 	.small_ops = dp_datapath_genl_ops,
 	.n_small_ops = ARRAY_SIZE(dp_datapath_genl_ops),
+	.resv_start_op = OVS_DP_CMD_SET + 1,
 	.mcgrps = &ovs_dp_datapath_multicast_group,
 	.n_mcgrps = 1,
 	.module = THIS_MODULE,
@@ -2201,7 +2223,10 @@ static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
 	if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
 	    !a[OVS_VPORT_ATTR_UPCALL_PID])
 		return -EINVAL;
-	if (a[OVS_VPORT_ATTR_IFINDEX])
+
+	parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
+
+	if (a[OVS_VPORT_ATTR_IFINDEX] && parms.type != OVS_VPORT_TYPE_INTERNAL)
 		return -EOPNOTSUPP;
 
 	port_no = a[OVS_VPORT_ATTR_PORT_NO]
@@ -2238,11 +2263,12 @@ restart:
 	}
 
 	parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
-	parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
 	parms.options = a[OVS_VPORT_ATTR_OPTIONS];
 	parms.dp = dp;
 	parms.port_no = port_no;
 	parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
+	parms.desired_ifindex = a[OVS_VPORT_ATTR_IFINDEX]
+		? nla_get_u32(a[OVS_VPORT_ATTR_IFINDEX]) : 0;
 
 	vport = new_vport(&parms);
 	err = PTR_ERR(vport);
diff --git a/net/openvswitch/flow_netlink.c b/net/openvswitch/flow_netlink.c
index 4c09cf8a0ab2..4a07ab094a84 100644
--- a/net/openvswitch/flow_netlink.c
+++ b/net/openvswitch/flow_netlink.c
@@ -3304,7 +3304,7 @@ static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
 
 			/* Disallow subsequent L2.5+ set actions and mpls_pop
 			 * actions once the last MPLS label in the packet is
-			 * is popped as there is no check here to ensure that
+			 * popped as there is no check here to ensure that
 			 * the new eth type is valid and thus set actions could
 			 * write off the end of the packet or otherwise corrupt
 			 * it.
diff --git a/net/openvswitch/meter.c b/net/openvswitch/meter.c
index 04a060ac7fdf..6e38f68f88c2 100644
--- a/net/openvswitch/meter.c
+++ b/net/openvswitch/meter.c
@@ -343,7 +343,7 @@ static struct dp_meter *dp_meter_create(struct nlattr **a)
 			return ERR_PTR(-EINVAL);
 
 	/* Allocate and set up the meter before locking anything. */
-	meter = kzalloc(struct_size(meter, bands, n_bands), GFP_KERNEL);
+	meter = kzalloc(struct_size(meter, bands, n_bands), GFP_KERNEL_ACCOUNT);
 	if (!meter)
 		return ERR_PTR(-ENOMEM);
 
@@ -687,9 +687,9 @@ static const struct genl_small_ops dp_meter_genl_ops[] = {
 	},
 	{ .cmd = OVS_METER_CMD_SET,
 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN
-					   *  privilege.
-					   */
+		.flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN
+					       *  privilege.
+					       */
 		.doit = ovs_meter_cmd_set,
 	},
 	{ .cmd = OVS_METER_CMD_GET,
@@ -699,9 +699,9 @@ static const struct genl_small_ops dp_meter_genl_ops[] = {
 	},
 	{ .cmd = OVS_METER_CMD_DEL,
 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN
-					   *  privilege.
-					   */
+		.flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN
+					       *  privilege.
+					       */
 		.doit = ovs_meter_cmd_del
 	},
 };
@@ -720,6 +720,7 @@ struct genl_family dp_meter_genl_family __ro_after_init = {
 	.parallel_ops = true,
 	.small_ops = dp_meter_genl_ops,
 	.n_small_ops = ARRAY_SIZE(dp_meter_genl_ops),
+	.resv_start_op = OVS_METER_CMD_GET + 1,
 	.mcgrps = &ovs_meter_multicast_group,
 	.n_mcgrps = 1,
 	.module = THIS_MODULE,
diff --git a/net/openvswitch/vport-internal_dev.c b/net/openvswitch/vport-internal_dev.c
index 5b2ee9c1c00b..74c88a6baa43 100644
--- a/net/openvswitch/vport-internal_dev.c
+++ b/net/openvswitch/vport-internal_dev.c
@@ -65,7 +65,7 @@ static int internal_dev_stop(struct net_device *netdev)
 static void internal_dev_getinfo(struct net_device *netdev,
 				 struct ethtool_drvinfo *info)
 {
-	strlcpy(info->driver, "openvswitch", sizeof(info->driver));
+	strscpy(info->driver, "openvswitch", sizeof(info->driver));
 }
 
 static const struct ethtool_ops internal_dev_ethtool_ops = {
@@ -147,6 +147,7 @@ static struct vport *internal_dev_create(const struct vport_parms *parms)
 	}
 
 	dev_net_set(vport->dev, ovs_dp_get_net(vport->dp));
+	dev->ifindex = parms->desired_ifindex;
 	internal_dev = internal_dev_priv(vport->dev);
 	internal_dev->vport = vport;
 
@@ -189,7 +190,7 @@ static void internal_dev_destroy(struct vport *vport)
 	rtnl_unlock();
 }
 
-static netdev_tx_t internal_dev_recv(struct sk_buff *skb)
+static int internal_dev_recv(struct sk_buff *skb)
 {
 	struct net_device *netdev = skb->dev;
 
diff --git a/net/openvswitch/vport.h b/net/openvswitch/vport.h
index 9de5030d9801..6ff45e8a0868 100644
--- a/net/openvswitch/vport.h
+++ b/net/openvswitch/vport.h
@@ -90,12 +90,14 @@ struct vport {
  * @type: New vport's type.
  * @options: %OVS_VPORT_ATTR_OPTIONS attribute from Netlink message, %NULL if
  * none was supplied.
+ * @desired_ifindex: New vport's ifindex.
  * @dp: New vport's datapath.
  * @port_no: New vport's port number.
  */
 struct vport_parms {
 	const char *name;
 	enum ovs_vport_type type;
+	int desired_ifindex;
 	struct nlattr *options;
 
 	/* For ovs_vport_alloc(). */
@@ -130,7 +132,7 @@ struct vport_ops {
 	int (*set_options)(struct vport *, struct nlattr *);
 	int (*get_options)(const struct vport *, struct sk_buff *);
 
-	netdev_tx_t (*send) (struct sk_buff *skb);
+	int (*send)(struct sk_buff *skb);
 	struct module *owner;
 	struct list_head list;
 };
diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index 5cbe07116e04..d3f6db350de7 100644
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -1905,7 +1905,7 @@ static int packet_rcv_spkt(struct sk_buff *skb, struct net_device *dev,
 	 */
 
 	spkt->spkt_family = dev->type;
-	strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
+	strscpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device));
 	spkt->spkt_protocol = skb->protocol;
 
 	/*
@@ -3565,7 +3565,7 @@ static int packet_getname_spkt(struct socket *sock, struct sockaddr *uaddr,
 	rcu_read_lock();
 	dev = dev_get_by_index_rcu(sock_net(sk), READ_ONCE(pkt_sk(sk)->ifindex));
 	if (dev)
-		strlcpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
+		strscpy(uaddr->sa_data, dev->name, sizeof(uaddr->sa_data));
 	rcu_read_unlock();
 
 	return sizeof(*uaddr);
@@ -4725,37 +4725,37 @@ static struct pernet_operations packet_net_ops = {
 
 static void __exit packet_exit(void)
 {
-	unregister_netdevice_notifier(&packet_netdev_notifier);
-	unregister_pernet_subsys(&packet_net_ops);
 	sock_unregister(PF_PACKET);
 	proto_unregister(&packet_proto);
+	unregister_netdevice_notifier(&packet_netdev_notifier);
+	unregister_pernet_subsys(&packet_net_ops);
 }
 
 static int __init packet_init(void)
 {
 	int rc;
 
-	rc = proto_register(&packet_proto, 0);
-	if (rc)
-		goto out;
-	rc = sock_register(&packet_family_ops);
-	if (rc)
-		goto out_proto;
 	rc = register_pernet_subsys(&packet_net_ops);
 	if (rc)
-		goto out_sock;
+		goto out;
 	rc = register_netdevice_notifier(&packet_netdev_notifier);
 	if (rc)
 		goto out_pernet;
+	rc = proto_register(&packet_proto, 0);
+	if (rc)
+		goto out_notifier;
+	rc = sock_register(&packet_family_ops);
+	if (rc)
+		goto out_proto;
 
 	return 0;
 
-out_pernet:
-	unregister_pernet_subsys(&packet_net_ops);
-out_sock:
-	sock_unregister(PF_PACKET);
 out_proto:
 	proto_unregister(&packet_proto);
+out_notifier:
+	unregister_netdevice_notifier(&packet_netdev_notifier);
+out_pernet:
+	unregister_pernet_subsys(&packet_net_ops);
 out:
 	return rc;
 }
diff --git a/net/psample/psample.c b/net/psample/psample.c
index 118d5d2a81a0..81a794e36f53 100644
--- a/net/psample/psample.c
+++ b/net/psample/psample.c
@@ -115,6 +115,7 @@ static struct genl_family psample_nl_family __ro_after_init = {
 	.mcgrps		= psample_nl_mcgrps,
 	.small_ops	= psample_nl_ops,
 	.n_small_ops	= ARRAY_SIZE(psample_nl_ops),
+	.resv_start_op	= PSAMPLE_CMD_GET_GROUP + 1,
 	.n_mcgrps	= ARRAY_SIZE(psample_nl_mcgrps),
 };
 
diff --git a/net/rds/af_rds.c b/net/rds/af_rds.c
index b239120dd9ca..3ff6995244e5 100644
--- a/net/rds/af_rds.c
+++ b/net/rds/af_rds.c
@@ -894,7 +894,7 @@ module_exit(rds_exit);
 
 u32 rds_gen_num;
 
-static int rds_init(void)
+static int __init rds_init(void)
 {
 	int ret;
 
diff --git a/net/rds/message.c b/net/rds/message.c
index d74be4e3f3fa..44dbc612ef54 100644
--- a/net/rds/message.c
+++ b/net/rds/message.c
@@ -354,7 +354,7 @@ struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned in
 
 	for (i = 0; i < rm->data.op_nents; ++i) {
 		sg_set_page(&rm->data.op_sg[i],
-				virt_to_page(page_addrs[i]),
+				virt_to_page((void *)page_addrs[i]),
 				PAGE_SIZE, 0);
 	}
 
diff --git a/net/rds/rdma_transport.c b/net/rds/rdma_transport.c
index a9e4ff948a7d..d36f3f6b4351 100644
--- a/net/rds/rdma_transport.c
+++ b/net/rds/rdma_transport.c
@@ -291,7 +291,7 @@ static void rds_rdma_listen_stop(void)
 #endif
 }
 
-static int rds_rdma_init(void)
+static int __init rds_rdma_init(void)
 {
 	int ret;
 
@@ -307,7 +307,7 @@ out:
 }
 module_init(rds_rdma_init);
 
-static void rds_rdma_exit(void)
+static void __exit rds_rdma_exit(void)
 {
 	/* stop listening first to ensure no new connections are attempted */
 	rds_rdma_listen_stop();
diff --git a/net/rds/tcp.c b/net/rds/tcp.c
index 73ee2771093d..4444fd82b66d 100644
--- a/net/rds/tcp.c
+++ b/net/rds/tcp.c
@@ -166,10 +166,10 @@ void rds_tcp_reset_callbacks(struct socket *sock,
 	 */
 	atomic_set(&cp->cp_state, RDS_CONN_RESETTING);
 	wait_event(cp->cp_waitq, !test_bit(RDS_IN_XMIT, &cp->cp_flags));
-	lock_sock(osock->sk);
 	/* reset receive side state for rds_tcp_data_recv() for osock  */
 	cancel_delayed_work_sync(&cp->cp_send_w);
 	cancel_delayed_work_sync(&cp->cp_recv_w);
+	lock_sock(osock->sk);
 	if (tc->t_tinc) {
 		rds_inc_put(&tc->t_tinc->ti_inc);
 		tc->t_tinc = NULL;
@@ -712,7 +712,7 @@ static void rds_tcp_exit(void)
 }
 module_exit(rds_tcp_exit);
 
-static int rds_tcp_init(void)
+static int __init rds_tcp_init(void)
 {
 	int ret;
 
diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h
index 62c70709d798..1ad0ec5afb50 100644
--- a/net/rxrpc/ar-internal.h
+++ b/net/rxrpc/ar-internal.h
@@ -782,7 +782,6 @@ void rxrpc_delete_call_timer(struct rxrpc_call *call);
  */
 extern const char *const rxrpc_call_states[];
 extern const char *const rxrpc_call_completions[];
-extern unsigned int rxrpc_max_call_lifetime;
 extern struct kmem_cache *rxrpc_call_jar;
 
 struct rxrpc_call *rxrpc_find_call_by_user_ID(struct rxrpc_sock *, unsigned long);
diff --git a/net/sched/act_api.c b/net/sched/act_api.c
index 817065aa2833..9b31a10cc639 100644
--- a/net/sched/act_api.c
+++ b/net/sched/act_api.c
@@ -676,6 +676,31 @@ int tcf_idr_search(struct tc_action_net *tn, struct tc_action **a, u32 index)
 }
 EXPORT_SYMBOL(tcf_idr_search);
 
+static int __tcf_generic_walker(struct net *net, struct sk_buff *skb,
+				struct netlink_callback *cb, int type,
+				const struct tc_action_ops *ops,
+				struct netlink_ext_ack *extack)
+{
+	struct tc_action_net *tn = net_generic(net, ops->net_id);
+
+	if (unlikely(ops->walk))
+		return ops->walk(net, skb, cb, type, ops, extack);
+
+	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
+}
+
+static int __tcf_idr_search(struct net *net,
+			    const struct tc_action_ops *ops,
+			    struct tc_action **a, u32 index)
+{
+	struct tc_action_net *tn = net_generic(net, ops->net_id);
+
+	if (unlikely(ops->lookup))
+		return ops->lookup(net, a, index);
+
+	return tcf_idr_search(tn, a, index);
+}
+
 static int tcf_idr_delete_index(struct tcf_idrinfo *idrinfo, u32 index)
 {
 	struct tc_action *p;
@@ -926,7 +951,7 @@ int tcf_register_action(struct tc_action_ops *act,
 	struct tc_action_ops *a;
 	int ret;
 
-	if (!act->act || !act->dump || !act->init || !act->walk || !act->lookup)
+	if (!act->act || !act->dump || !act->init)
 		return -EINVAL;
 
 	/* We have to register pernet ops before making the action ops visible,
@@ -1638,7 +1663,7 @@ static struct tc_action *tcf_action_get_1(struct net *net, struct nlattr *nla,
 		goto err_out;
 	}
 	err = -ENOENT;
-	if (ops->lookup(net, &a, index) == 0) {
+	if (__tcf_idr_search(net, ops, &a, index) == 0) {
 		NL_SET_ERR_MSG(extack, "TC action with specified index not found");
 		goto err_mod;
 	}
@@ -1703,7 +1728,7 @@ static int tca_action_flush(struct net *net, struct nlattr *nla,
 		goto out_module_put;
 	}
 
-	err = ops->walk(net, skb, &dcb, RTM_DELACTION, ops, extack);
+	err = __tcf_generic_walker(net, skb, &dcb, RTM_DELACTION, ops, extack);
 	if (err <= 0) {
 		nla_nest_cancel(skb, nest);
 		goto out_module_put;
@@ -2121,7 +2146,7 @@ static int tc_dump_action(struct sk_buff *skb, struct netlink_callback *cb)
 	if (nest == NULL)
 		goto out_module_put;
 
-	ret = a_o->walk(net, skb, cb, RTM_GETACTION, a_o, NULL);
+	ret = __tcf_generic_walker(net, skb, cb, RTM_GETACTION, a_o, NULL);
 	if (ret < 0)
 		goto out_module_put;
 
diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c
index fea2d78b9ddc..b79eee44e24e 100644
--- a/net/sched/act_bpf.c
+++ b/net/sched/act_bpf.c
@@ -29,7 +29,6 @@ struct tcf_bpf_cfg {
 	bool is_ebpf;
 };
 
-static unsigned int bpf_net_id;
 static struct tc_action_ops act_bpf_ops;
 
 static int tcf_bpf_act(struct sk_buff *skb, const struct tc_action *act,
@@ -280,7 +279,7 @@ static int tcf_bpf_init(struct net *net, struct nlattr *nla,
 			struct tcf_proto *tp, u32 flags,
 			struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, bpf_net_id);
+	struct tc_action_net *tn = net_generic(net, act_bpf_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_ACT_BPF_MAX + 1];
 	struct tcf_chain *goto_ch = NULL;
@@ -334,7 +333,7 @@ static int tcf_bpf_init(struct net *net, struct nlattr *nla,
 	is_bpf = tb[TCA_ACT_BPF_OPS_LEN] && tb[TCA_ACT_BPF_OPS];
 	is_ebpf = tb[TCA_ACT_BPF_FD];
 
-	if ((!is_bpf && !is_ebpf) || (is_bpf && is_ebpf)) {
+	if (is_bpf == is_ebpf) {
 		ret = -EINVAL;
 		goto put_chain;
 	}
@@ -390,23 +389,6 @@ static void tcf_bpf_cleanup(struct tc_action *act)
 	tcf_bpf_cfg_cleanup(&tmp);
 }
 
-static int tcf_bpf_walker(struct net *net, struct sk_buff *skb,
-			  struct netlink_callback *cb, int type,
-			  const struct tc_action_ops *ops,
-			  struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, bpf_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_bpf_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, bpf_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static struct tc_action_ops act_bpf_ops __read_mostly = {
 	.kind		=	"bpf",
 	.id		=	TCA_ID_BPF,
@@ -415,27 +397,25 @@ static struct tc_action_ops act_bpf_ops __read_mostly = {
 	.dump		=	tcf_bpf_dump,
 	.cleanup	=	tcf_bpf_cleanup,
 	.init		=	tcf_bpf_init,
-	.walk		=	tcf_bpf_walker,
-	.lookup		=	tcf_bpf_search,
 	.size		=	sizeof(struct tcf_bpf),
 };
 
 static __net_init int bpf_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, bpf_net_id);
+	struct tc_action_net *tn = net_generic(net, act_bpf_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_bpf_ops);
 }
 
 static void __net_exit bpf_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, bpf_net_id);
+	tc_action_net_exit(net_list, act_bpf_ops.net_id);
 }
 
 static struct pernet_operations bpf_net_ops = {
 	.init = bpf_init_net,
 	.exit_batch = bpf_exit_net,
-	.id   = &bpf_net_id,
+	.id   = &act_bpf_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_connmark.c b/net/sched/act_connmark.c
index 09e2aafc8943..66b143bb04ac 100644
--- a/net/sched/act_connmark.c
+++ b/net/sched/act_connmark.c
@@ -25,7 +25,6 @@
 #include <net/netfilter/nf_conntrack_core.h>
 #include <net/netfilter/nf_conntrack_zones.h>
 
-static unsigned int connmark_net_id;
 static struct tc_action_ops act_connmark_ops;
 
 static int tcf_connmark_act(struct sk_buff *skb, const struct tc_action *a,
@@ -99,7 +98,7 @@ static int tcf_connmark_init(struct net *net, struct nlattr *nla,
 			     struct tcf_proto *tp, u32 flags,
 			     struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, connmark_net_id);
+	struct tc_action_net *tn = net_generic(net, act_connmark_ops.net_id);
 	struct nlattr *tb[TCA_CONNMARK_MAX + 1];
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct tcf_chain *goto_ch = NULL;
@@ -200,23 +199,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_connmark_walker(struct net *net, struct sk_buff *skb,
-			       struct netlink_callback *cb, int type,
-			       const struct tc_action_ops *ops,
-			       struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, connmark_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_connmark_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, connmark_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static struct tc_action_ops act_connmark_ops = {
 	.kind		=	"connmark",
 	.id		=	TCA_ID_CONNMARK,
@@ -224,27 +206,25 @@ static struct tc_action_ops act_connmark_ops = {
 	.act		=	tcf_connmark_act,
 	.dump		=	tcf_connmark_dump,
 	.init		=	tcf_connmark_init,
-	.walk		=	tcf_connmark_walker,
-	.lookup		=	tcf_connmark_search,
 	.size		=	sizeof(struct tcf_connmark_info),
 };
 
 static __net_init int connmark_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, connmark_net_id);
+	struct tc_action_net *tn = net_generic(net, act_connmark_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_connmark_ops);
 }
 
 static void __net_exit connmark_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, connmark_net_id);
+	tc_action_net_exit(net_list, act_connmark_ops.net_id);
 }
 
 static struct pernet_operations connmark_net_ops = {
 	.init = connmark_init_net,
 	.exit_batch = connmark_exit_net,
-	.id   = &connmark_net_id,
+	.id   = &act_connmark_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_csum.c b/net/sched/act_csum.c
index 22847ee009ef..1366adf9b909 100644
--- a/net/sched/act_csum.c
+++ b/net/sched/act_csum.c
@@ -37,7 +37,6 @@ static const struct nla_policy csum_policy[TCA_CSUM_MAX + 1] = {
 	[TCA_CSUM_PARMS] = { .len = sizeof(struct tc_csum), },
 };
 
-static unsigned int csum_net_id;
 static struct tc_action_ops act_csum_ops;
 
 static int tcf_csum_init(struct net *net, struct nlattr *nla,
@@ -45,7 +44,7 @@ static int tcf_csum_init(struct net *net, struct nlattr *nla,
 			 struct tcf_proto *tp,
 			 u32 flags, struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, csum_net_id);
+	struct tc_action_net *tn = net_generic(net, act_csum_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct tcf_csum_params *params_new;
 	struct nlattr *tb[TCA_CSUM_MAX + 1];
@@ -673,23 +672,6 @@ static void tcf_csum_cleanup(struct tc_action *a)
 		kfree_rcu(params, rcu);
 }
 
-static int tcf_csum_walker(struct net *net, struct sk_buff *skb,
-			   struct netlink_callback *cb, int type,
-			   const struct tc_action_ops *ops,
-			   struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, csum_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_csum_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, csum_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static size_t tcf_csum_get_fill_size(const struct tc_action *act)
 {
 	return nla_total_size(sizeof(struct tc_csum));
@@ -722,8 +704,6 @@ static struct tc_action_ops act_csum_ops = {
 	.dump		= tcf_csum_dump,
 	.init		= tcf_csum_init,
 	.cleanup	= tcf_csum_cleanup,
-	.walk		= tcf_csum_walker,
-	.lookup		= tcf_csum_search,
 	.get_fill_size  = tcf_csum_get_fill_size,
 	.offload_act_setup = tcf_csum_offload_act_setup,
 	.size		= sizeof(struct tcf_csum),
@@ -731,20 +711,20 @@ static struct tc_action_ops act_csum_ops = {
 
 static __net_init int csum_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, csum_net_id);
+	struct tc_action_net *tn = net_generic(net, act_csum_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_csum_ops);
 }
 
 static void __net_exit csum_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, csum_net_id);
+	tc_action_net_exit(net_list, act_csum_ops.net_id);
 }
 
 static struct pernet_operations csum_net_ops = {
 	.init = csum_init_net,
 	.exit_batch = csum_exit_net,
-	.id   = &csum_net_id,
+	.id   = &act_csum_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c
index d55afb8d14be..b38d91d6b249 100644
--- a/net/sched/act_ct.c
+++ b/net/sched/act_ct.c
@@ -649,7 +649,6 @@ static void tcf_ct_flow_tables_uninit(void)
 }
 
 static struct tc_action_ops act_ct_ops;
-static unsigned int ct_net_id;
 
 struct tc_ct_action_net {
 	struct tc_action_net tn; /* Must be first */
@@ -697,7 +696,6 @@ drop_ct:
 static int tcf_ct_skb_network_trim(struct sk_buff *skb, int family)
 {
 	unsigned int len;
-	int err;
 
 	switch (family) {
 	case NFPROTO_IPV4:
@@ -711,9 +709,7 @@ static int tcf_ct_skb_network_trim(struct sk_buff *skb, int family)
 		len = skb->len;
 	}
 
-	err = pskb_trim_rcsum(skb, len);
-
-	return err;
+	return pskb_trim_rcsum(skb, len);
 }
 
 static u8 tcf_ct_skb_nf_family(struct sk_buff *skb)
@@ -1255,7 +1251,7 @@ static int tcf_ct_fill_params(struct net *net,
 			      struct nlattr **tb,
 			      struct netlink_ext_ack *extack)
 {
-	struct tc_ct_action_net *tn = net_generic(net, ct_net_id);
+	struct tc_ct_action_net *tn = net_generic(net, act_ct_ops.net_id);
 	struct nf_conntrack_zone zone;
 	struct nf_conn *tmpl;
 	int err;
@@ -1330,7 +1326,7 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla,
 		       struct tcf_proto *tp, u32 flags,
 		       struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, ct_net_id);
+	struct tc_action_net *tn = net_generic(net, act_ct_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct tcf_ct_params *params = NULL;
 	struct nlattr *tb[TCA_CT_MAX + 1];
@@ -1394,7 +1390,7 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla,
 
 	err = tcf_ct_flow_table_get(net, params);
 	if (err)
-		goto cleanup;
+		goto cleanup_params;
 
 	spin_lock_bh(&c->tcf_lock);
 	goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
@@ -1409,6 +1405,9 @@ static int tcf_ct_init(struct net *net, struct nlattr *nla,
 
 	return res;
 
+cleanup_params:
+	if (params->tmpl)
+		nf_ct_put(params->tmpl);
 cleanup:
 	if (goto_ch)
 		tcf_chain_put_by_act(goto_ch);
@@ -1558,23 +1557,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_ct_walker(struct net *net, struct sk_buff *skb,
-			 struct netlink_callback *cb, int type,
-			 const struct tc_action_ops *ops,
-			 struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, ct_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_ct_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, ct_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static void tcf_stats_update(struct tc_action *a, u64 bytes, u64 packets,
 			     u64 drops, u64 lastuse, bool hw)
 {
@@ -1613,8 +1595,6 @@ static struct tc_action_ops act_ct_ops = {
 	.dump		=	tcf_ct_dump,
 	.init		=	tcf_ct_init,
 	.cleanup	=	tcf_ct_cleanup,
-	.walk		=	tcf_ct_walker,
-	.lookup		=	tcf_ct_search,
 	.stats_update	=	tcf_stats_update,
 	.offload_act_setup =	tcf_ct_offload_act_setup,
 	.size		=	sizeof(struct tcf_ct),
@@ -1623,7 +1603,7 @@ static struct tc_action_ops act_ct_ops = {
 static __net_init int ct_init_net(struct net *net)
 {
 	unsigned int n_bits = sizeof_field(struct tcf_ct_params, labels) * 8;
-	struct tc_ct_action_net *tn = net_generic(net, ct_net_id);
+	struct tc_ct_action_net *tn = net_generic(net, act_ct_ops.net_id);
 
 	if (nf_connlabels_get(net, n_bits - 1)) {
 		tn->labels = false;
@@ -1641,20 +1621,20 @@ static void __net_exit ct_exit_net(struct list_head *net_list)
 
 	rtnl_lock();
 	list_for_each_entry(net, net_list, exit_list) {
-		struct tc_ct_action_net *tn = net_generic(net, ct_net_id);
+		struct tc_ct_action_net *tn = net_generic(net, act_ct_ops.net_id);
 
 		if (tn->labels)
 			nf_connlabels_put(net);
 	}
 	rtnl_unlock();
 
-	tc_action_net_exit(net_list, ct_net_id);
+	tc_action_net_exit(net_list, act_ct_ops.net_id);
 }
 
 static struct pernet_operations ct_net_ops = {
 	.init = ct_init_net,
 	.exit_batch = ct_exit_net,
-	.id   = &ct_net_id,
+	.id   = &act_ct_ops.net_id,
 	.size = sizeof(struct tc_ct_action_net),
 };
 
diff --git a/net/sched/act_ctinfo.c b/net/sched/act_ctinfo.c
index 0281e45987a4..d4102f0a9abd 100644
--- a/net/sched/act_ctinfo.c
+++ b/net/sched/act_ctinfo.c
@@ -25,7 +25,6 @@
 #include <net/netfilter/nf_conntrack_zones.h>
 
 static struct tc_action_ops act_ctinfo_ops;
-static unsigned int ctinfo_net_id;
 
 static void tcf_ctinfo_dscp_set(struct nf_conn *ct, struct tcf_ctinfo *ca,
 				struct tcf_ctinfo_params *cp,
@@ -157,7 +156,7 @@ static int tcf_ctinfo_init(struct net *net, struct nlattr *nla,
 			   struct tcf_proto *tp, u32 flags,
 			   struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, ctinfo_net_id);
+	struct tc_action_net *tn = net_generic(net, act_ctinfo_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	u32 dscpmask = 0, dscpstatemask, index;
 	struct nlattr *tb[TCA_CTINFO_MAX + 1];
@@ -342,23 +341,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_ctinfo_walker(struct net *net, struct sk_buff *skb,
-			     struct netlink_callback *cb, int type,
-			     const struct tc_action_ops *ops,
-			     struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, ctinfo_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_ctinfo_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, ctinfo_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static void tcf_ctinfo_cleanup(struct tc_action *a)
 {
 	struct tcf_ctinfo *ci = to_ctinfo(a);
@@ -377,27 +359,25 @@ static struct tc_action_ops act_ctinfo_ops = {
 	.dump	= tcf_ctinfo_dump,
 	.init	= tcf_ctinfo_init,
 	.cleanup= tcf_ctinfo_cleanup,
-	.walk	= tcf_ctinfo_walker,
-	.lookup	= tcf_ctinfo_search,
 	.size	= sizeof(struct tcf_ctinfo),
 };
 
 static __net_init int ctinfo_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, ctinfo_net_id);
+	struct tc_action_net *tn = net_generic(net, act_ctinfo_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_ctinfo_ops);
 }
 
 static void __net_exit ctinfo_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, ctinfo_net_id);
+	tc_action_net_exit(net_list, act_ctinfo_ops.net_id);
 }
 
 static struct pernet_operations ctinfo_net_ops = {
 	.init		= ctinfo_init_net,
 	.exit_batch	= ctinfo_exit_net,
-	.id		= &ctinfo_net_id,
+	.id		= &act_ctinfo_ops.net_id,
 	.size		= sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_gact.c b/net/sched/act_gact.c
index ac29d1065232..abe1bcc5c797 100644
--- a/net/sched/act_gact.c
+++ b/net/sched/act_gact.c
@@ -19,7 +19,6 @@
 #include <linux/tc_act/tc_gact.h>
 #include <net/tc_act/tc_gact.h>
 
-static unsigned int gact_net_id;
 static struct tc_action_ops act_gact_ops;
 
 #ifdef CONFIG_GACT_PROB
@@ -55,7 +54,7 @@ static int tcf_gact_init(struct net *net, struct nlattr *nla,
 			 struct tcf_proto *tp, u32 flags,
 			 struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, gact_net_id);
+	struct tc_action_net *tn = net_generic(net, act_gact_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_GACT_MAX + 1];
 	struct tcf_chain *goto_ch = NULL;
@@ -222,23 +221,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_gact_walker(struct net *net, struct sk_buff *skb,
-			   struct netlink_callback *cb, int type,
-			   const struct tc_action_ops *ops,
-			   struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, gact_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_gact_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, gact_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static size_t tcf_gact_get_fill_size(const struct tc_action *act)
 {
 	size_t sz = nla_total_size(sizeof(struct tc_gact)); /* TCA_GACT_PARMS */
@@ -308,8 +290,6 @@ static struct tc_action_ops act_gact_ops = {
 	.stats_update	=	tcf_gact_stats_update,
 	.dump		=	tcf_gact_dump,
 	.init		=	tcf_gact_init,
-	.walk		=	tcf_gact_walker,
-	.lookup		=	tcf_gact_search,
 	.get_fill_size	=	tcf_gact_get_fill_size,
 	.offload_act_setup =	tcf_gact_offload_act_setup,
 	.size		=	sizeof(struct tcf_gact),
@@ -317,20 +297,20 @@ static struct tc_action_ops act_gact_ops = {
 
 static __net_init int gact_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, gact_net_id);
+	struct tc_action_net *tn = net_generic(net, act_gact_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_gact_ops);
 }
 
 static void __net_exit gact_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, gact_net_id);
+	tc_action_net_exit(net_list, act_gact_ops.net_id);
 }
 
 static struct pernet_operations gact_net_ops = {
 	.init = gact_init_net,
 	.exit_batch = gact_exit_net,
-	.id   = &gact_net_id,
+	.id   = &act_gact_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_gate.c b/net/sched/act_gate.c
index fd5155274733..3049878e7315 100644
--- a/net/sched/act_gate.c
+++ b/net/sched/act_gate.c
@@ -15,7 +15,6 @@
 #include <net/pkt_cls.h>
 #include <net/tc_act/tc_gate.h>
 
-static unsigned int gate_net_id;
 static struct tc_action_ops act_gate_ops;
 
 static ktime_t gate_get_time(struct tcf_gate *gact)
@@ -298,7 +297,7 @@ static int tcf_gate_init(struct net *net, struct nlattr *nla,
 			 struct tcf_proto *tp, u32 flags,
 			 struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, gate_net_id);
+	struct tc_action_net *tn = net_generic(net, act_gate_ops.net_id);
 	enum tk_offsets tk_offset = TK_OFFS_TAI;
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_GATE_MAX + 1];
@@ -565,16 +564,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_gate_walker(struct net *net, struct sk_buff *skb,
-			   struct netlink_callback *cb, int type,
-			   const struct tc_action_ops *ops,
-			   struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, gate_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
 static void tcf_gate_stats_update(struct tc_action *a, u64 bytes, u64 packets,
 				  u64 drops, u64 lastuse, bool hw)
 {
@@ -585,13 +574,6 @@ static void tcf_gate_stats_update(struct tc_action *a, u64 bytes, u64 packets,
 	tm->lastuse = max_t(u64, tm->lastuse, lastuse);
 }
 
-static int tcf_gate_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, gate_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static size_t tcf_gate_get_fill_size(const struct tc_action *act)
 {
 	return nla_total_size(sizeof(struct tc_gate));
@@ -654,30 +636,28 @@ static struct tc_action_ops act_gate_ops = {
 	.dump		=	tcf_gate_dump,
 	.init		=	tcf_gate_init,
 	.cleanup	=	tcf_gate_cleanup,
-	.walk		=	tcf_gate_walker,
 	.stats_update	=	tcf_gate_stats_update,
 	.get_fill_size	=	tcf_gate_get_fill_size,
-	.lookup		=	tcf_gate_search,
 	.offload_act_setup =	tcf_gate_offload_act_setup,
 	.size		=	sizeof(struct tcf_gate),
 };
 
 static __net_init int gate_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, gate_net_id);
+	struct tc_action_net *tn = net_generic(net, act_gate_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_gate_ops);
 }
 
 static void __net_exit gate_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, gate_net_id);
+	tc_action_net_exit(net_list, act_gate_ops.net_id);
 }
 
 static struct pernet_operations gate_net_ops = {
 	.init = gate_init_net,
 	.exit_batch = gate_exit_net,
-	.id   = &gate_net_id,
+	.id   = &act_gate_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c
index 41ba55e60b1b..41d63b33461d 100644
--- a/net/sched/act_ife.c
+++ b/net/sched/act_ife.c
@@ -30,7 +30,6 @@
 #include <linux/etherdevice.h>
 #include <net/ife.h>
 
-static unsigned int ife_net_id;
 static int max_metacnt = IFE_META_MAX + 1;
 static struct tc_action_ops act_ife_ops;
 
@@ -482,7 +481,7 @@ static int tcf_ife_init(struct net *net, struct nlattr *nla,
 			struct tcf_proto *tp, u32 flags,
 			struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, ife_net_id);
+	struct tc_action_net *tn = net_generic(net, act_ife_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_IFE_MAX + 1];
 	struct nlattr *tb2[IFE_META_MAX + 1];
@@ -878,23 +877,6 @@ static int tcf_ife_act(struct sk_buff *skb, const struct tc_action *a,
 	return tcf_ife_decode(skb, a, res);
 }
 
-static int tcf_ife_walker(struct net *net, struct sk_buff *skb,
-			  struct netlink_callback *cb, int type,
-			  const struct tc_action_ops *ops,
-			  struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, ife_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_ife_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, ife_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static struct tc_action_ops act_ife_ops = {
 	.kind = "ife",
 	.id = TCA_ID_IFE,
@@ -903,27 +885,25 @@ static struct tc_action_ops act_ife_ops = {
 	.dump = tcf_ife_dump,
 	.cleanup = tcf_ife_cleanup,
 	.init = tcf_ife_init,
-	.walk = tcf_ife_walker,
-	.lookup = tcf_ife_search,
 	.size =	sizeof(struct tcf_ife_info),
 };
 
 static __net_init int ife_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, ife_net_id);
+	struct tc_action_net *tn = net_generic(net, act_ife_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_ife_ops);
 }
 
 static void __net_exit ife_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, ife_net_id);
+	tc_action_net_exit(net_list, act_ife_ops.net_id);
 }
 
 static struct pernet_operations ife_net_ops = {
 	.init = ife_init_net,
 	.exit_batch = ife_exit_net,
-	.id   = &ife_net_id,
+	.id   = &act_ife_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_ipt.c b/net/sched/act_ipt.c
index 2f3d507c24a1..1625e1037416 100644
--- a/net/sched/act_ipt.c
+++ b/net/sched/act_ipt.c
@@ -24,10 +24,7 @@
 #include <linux/netfilter_ipv4/ip_tables.h>
 
 
-static unsigned int ipt_net_id;
 static struct tc_action_ops act_ipt_ops;
-
-static unsigned int xt_net_id;
 static struct tc_action_ops act_xt_ops;
 
 static int ipt_init_target(struct net *net, struct xt_entry_target *t,
@@ -206,8 +203,8 @@ static int tcf_ipt_init(struct net *net, struct nlattr *nla,
 			struct tcf_proto *tp,
 			u32 flags, struct netlink_ext_ack *extack)
 {
-	return __tcf_ipt_init(net, ipt_net_id, nla, est, a, &act_ipt_ops,
-			      tp, flags);
+	return __tcf_ipt_init(net, act_ipt_ops.net_id, nla, est,
+			      a, &act_ipt_ops, tp, flags);
 }
 
 static int tcf_xt_init(struct net *net, struct nlattr *nla,
@@ -215,8 +212,8 @@ static int tcf_xt_init(struct net *net, struct nlattr *nla,
 		       struct tcf_proto *tp,
 		       u32 flags, struct netlink_ext_ack *extack)
 {
-	return __tcf_ipt_init(net, xt_net_id, nla, est, a, &act_xt_ops,
-			      tp, flags);
+	return __tcf_ipt_init(net, act_xt_ops.net_id, nla, est,
+			      a, &act_xt_ops, tp, flags);
 }
 
 static int tcf_ipt_act(struct sk_buff *skb, const struct tc_action *a,
@@ -316,23 +313,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_ipt_walker(struct net *net, struct sk_buff *skb,
-			  struct netlink_callback *cb, int type,
-			  const struct tc_action_ops *ops,
-			  struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, ipt_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_ipt_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, ipt_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static struct tc_action_ops act_ipt_ops = {
 	.kind		=	"ipt",
 	.id		=	TCA_ID_IPT,
@@ -341,47 +321,28 @@ static struct tc_action_ops act_ipt_ops = {
 	.dump		=	tcf_ipt_dump,
 	.cleanup	=	tcf_ipt_release,
 	.init		=	tcf_ipt_init,
-	.walk		=	tcf_ipt_walker,
-	.lookup		=	tcf_ipt_search,
 	.size		=	sizeof(struct tcf_ipt),
 };
 
 static __net_init int ipt_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, ipt_net_id);
+	struct tc_action_net *tn = net_generic(net, act_ipt_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_ipt_ops);
 }
 
 static void __net_exit ipt_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, ipt_net_id);
+	tc_action_net_exit(net_list, act_ipt_ops.net_id);
 }
 
 static struct pernet_operations ipt_net_ops = {
 	.init = ipt_init_net,
 	.exit_batch = ipt_exit_net,
-	.id   = &ipt_net_id,
+	.id   = &act_ipt_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
-static int tcf_xt_walker(struct net *net, struct sk_buff *skb,
-			 struct netlink_callback *cb, int type,
-			 const struct tc_action_ops *ops,
-			 struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, xt_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_xt_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, xt_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static struct tc_action_ops act_xt_ops = {
 	.kind		=	"xt",
 	.id		=	TCA_ID_XT,
@@ -390,27 +351,25 @@ static struct tc_action_ops act_xt_ops = {
 	.dump		=	tcf_ipt_dump,
 	.cleanup	=	tcf_ipt_release,
 	.init		=	tcf_xt_init,
-	.walk		=	tcf_xt_walker,
-	.lookup		=	tcf_xt_search,
 	.size		=	sizeof(struct tcf_ipt),
 };
 
 static __net_init int xt_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, xt_net_id);
+	struct tc_action_net *tn = net_generic(net, act_xt_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_xt_ops);
 }
 
 static void __net_exit xt_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, xt_net_id);
+	tc_action_net_exit(net_list, act_xt_ops.net_id);
 }
 
 static struct pernet_operations xt_net_ops = {
 	.init = xt_init_net,
 	.exit_batch = xt_exit_net,
-	.id   = &xt_net_id,
+	.id   = &act_xt_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c
index a1d70cf86843..b8ad6ae282c0 100644
--- a/net/sched/act_mirred.c
+++ b/net/sched/act_mirred.c
@@ -86,7 +86,6 @@ static const struct nla_policy mirred_policy[TCA_MIRRED_MAX + 1] = {
 	[TCA_MIRRED_PARMS]	= { .len = sizeof(struct tc_mirred) },
 };
 
-static unsigned int mirred_net_id;
 static struct tc_action_ops act_mirred_ops;
 
 static int tcf_mirred_init(struct net *net, struct nlattr *nla,
@@ -94,7 +93,7 @@ static int tcf_mirred_init(struct net *net, struct nlattr *nla,
 			   struct tcf_proto *tp,
 			   u32 flags, struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, mirred_net_id);
+	struct tc_action_net *tn = net_generic(net, act_mirred_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_MIRRED_MAX + 1];
 	struct tcf_chain *goto_ch = NULL;
@@ -306,8 +305,7 @@ static int tcf_mirred_act(struct sk_buff *skb, const struct tc_action *a,
 
 		/* let's the caller reinsert the packet, if possible */
 		if (use_reinsert) {
-			res->ingress = want_ingress;
-			err = tcf_mirred_forward(res->ingress, skb);
+			err = tcf_mirred_forward(want_ingress, skb);
 			if (err)
 				tcf_action_inc_overlimit_qstats(&m->common);
 			__this_cpu_dec(mirred_rec_level);
@@ -373,23 +371,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_mirred_walker(struct net *net, struct sk_buff *skb,
-			     struct netlink_callback *cb, int type,
-			     const struct tc_action_ops *ops,
-			     struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, mirred_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_mirred_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, mirred_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static int mirred_device_event(struct notifier_block *unused,
 			       unsigned long event, void *ptr)
 {
@@ -510,8 +491,6 @@ static struct tc_action_ops act_mirred_ops = {
 	.dump		=	tcf_mirred_dump,
 	.cleanup	=	tcf_mirred_release,
 	.init		=	tcf_mirred_init,
-	.walk		=	tcf_mirred_walker,
-	.lookup		=	tcf_mirred_search,
 	.get_fill_size	=	tcf_mirred_get_fill_size,
 	.offload_act_setup =	tcf_mirred_offload_act_setup,
 	.size		=	sizeof(struct tcf_mirred),
@@ -520,20 +499,20 @@ static struct tc_action_ops act_mirred_ops = {
 
 static __net_init int mirred_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, mirred_net_id);
+	struct tc_action_net *tn = net_generic(net, act_mirred_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_mirred_ops);
 }
 
 static void __net_exit mirred_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, mirred_net_id);
+	tc_action_net_exit(net_list, act_mirred_ops.net_id);
 }
 
 static struct pernet_operations mirred_net_ops = {
 	.init = mirred_init_net,
 	.exit_batch = mirred_exit_net,
-	.id   = &mirred_net_id,
+	.id   = &act_mirred_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_mpls.c b/net/sched/act_mpls.c
index adabeccb63e1..8ad25cc8ccd5 100644
--- a/net/sched/act_mpls.c
+++ b/net/sched/act_mpls.c
@@ -15,7 +15,6 @@
 #include <net/pkt_cls.h>
 #include <net/tc_act/tc_mpls.h>
 
-static unsigned int mpls_net_id;
 static struct tc_action_ops act_mpls_ops;
 
 #define ACT_MPLS_TTL_DEFAULT	255
@@ -155,7 +154,7 @@ static int tcf_mpls_init(struct net *net, struct nlattr *nla,
 			 struct tcf_proto *tp, u32 flags,
 			 struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, mpls_net_id);
+	struct tc_action_net *tn = net_generic(net, act_mpls_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_MPLS_MAX + 1];
 	struct tcf_chain *goto_ch = NULL;
@@ -367,23 +366,6 @@ nla_put_failure:
 	return -EMSGSIZE;
 }
 
-static int tcf_mpls_walker(struct net *net, struct sk_buff *skb,
-			   struct netlink_callback *cb, int type,
-			   const struct tc_action_ops *ops,
-			   struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, mpls_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_mpls_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, mpls_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static int tcf_mpls_offload_act_setup(struct tc_action *act, void *entry_data,
 				      u32 *index_inc, bool bind,
 				      struct netlink_ext_ack *extack)
@@ -451,28 +433,26 @@ static struct tc_action_ops act_mpls_ops = {
 	.dump		=	tcf_mpls_dump,
 	.init		=	tcf_mpls_init,
 	.cleanup	=	tcf_mpls_cleanup,
-	.walk		=	tcf_mpls_walker,
-	.lookup		=	tcf_mpls_search,
 	.offload_act_setup =	tcf_mpls_offload_act_setup,
 	.size		=	sizeof(struct tcf_mpls),
 };
 
 static __net_init int mpls_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, mpls_net_id);
+	struct tc_action_net *tn = net_generic(net, act_mpls_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_mpls_ops);
 }
 
 static void __net_exit mpls_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, mpls_net_id);
+	tc_action_net_exit(net_list, act_mpls_ops.net_id);
 }
 
 static struct pernet_operations mpls_net_ops = {
 	.init = mpls_init_net,
 	.exit_batch = mpls_exit_net,
-	.id   = &mpls_net_id,
+	.id   = &act_mpls_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_nat.c b/net/sched/act_nat.c
index 2a39b3729e84..9265145f1040 100644
--- a/net/sched/act_nat.c
+++ b/net/sched/act_nat.c
@@ -26,7 +26,6 @@
 #include <net/udp.h>
 
 
-static unsigned int nat_net_id;
 static struct tc_action_ops act_nat_ops;
 
 static const struct nla_policy nat_policy[TCA_NAT_MAX + 1] = {
@@ -37,7 +36,7 @@ static int tcf_nat_init(struct net *net, struct nlattr *nla, struct nlattr *est,
 			struct tc_action **a, struct tcf_proto *tp,
 			u32 flags, struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, nat_net_id);
+	struct tc_action_net *tn = net_generic(net, act_nat_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_NAT_MAX + 1];
 	struct tcf_chain *goto_ch = NULL;
@@ -289,23 +288,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_nat_walker(struct net *net, struct sk_buff *skb,
-			  struct netlink_callback *cb, int type,
-			  const struct tc_action_ops *ops,
-			  struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, nat_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_nat_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, nat_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static struct tc_action_ops act_nat_ops = {
 	.kind		=	"nat",
 	.id		=	TCA_ID_NAT,
@@ -313,27 +295,25 @@ static struct tc_action_ops act_nat_ops = {
 	.act		=	tcf_nat_act,
 	.dump		=	tcf_nat_dump,
 	.init		=	tcf_nat_init,
-	.walk		=	tcf_nat_walker,
-	.lookup		=	tcf_nat_search,
 	.size		=	sizeof(struct tcf_nat),
 };
 
 static __net_init int nat_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, nat_net_id);
+	struct tc_action_net *tn = net_generic(net, act_nat_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_nat_ops);
 }
 
 static void __net_exit nat_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, nat_net_id);
+	tc_action_net_exit(net_list, act_nat_ops.net_id);
 }
 
 static struct pernet_operations nat_net_ops = {
 	.init = nat_init_net,
 	.exit_batch = nat_exit_net,
-	.id   = &nat_net_id,
+	.id   = &act_nat_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_pedit.c b/net/sched/act_pedit.c
index 823ee643371c..94ed5857ce67 100644
--- a/net/sched/act_pedit.c
+++ b/net/sched/act_pedit.c
@@ -21,7 +21,6 @@
 #include <uapi/linux/tc_act/tc_pedit.h>
 #include <net/pkt_cls.h>
 
-static unsigned int pedit_net_id;
 static struct tc_action_ops act_pedit_ops;
 
 static const struct nla_policy pedit_policy[TCA_PEDIT_MAX + 1] = {
@@ -139,7 +138,7 @@ static int tcf_pedit_init(struct net *net, struct nlattr *nla,
 			  struct tcf_proto *tp, u32 flags,
 			  struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, pedit_net_id);
+	struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_PEDIT_MAX + 1];
 	struct tcf_chain *goto_ch = NULL;
@@ -492,23 +491,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_pedit_walker(struct net *net, struct sk_buff *skb,
-			    struct netlink_callback *cb, int type,
-			    const struct tc_action_ops *ops,
-			    struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, pedit_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_pedit_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, pedit_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data,
 				       u32 *index_inc, bool bind,
 				       struct netlink_ext_ack *extack)
@@ -553,28 +535,26 @@ static struct tc_action_ops act_pedit_ops = {
 	.dump		=	tcf_pedit_dump,
 	.cleanup	=	tcf_pedit_cleanup,
 	.init		=	tcf_pedit_init,
-	.walk		=	tcf_pedit_walker,
-	.lookup		=	tcf_pedit_search,
 	.offload_act_setup =	tcf_pedit_offload_act_setup,
 	.size		=	sizeof(struct tcf_pedit),
 };
 
 static __net_init int pedit_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, pedit_net_id);
+	struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_pedit_ops);
 }
 
 static void __net_exit pedit_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, pedit_net_id);
+	tc_action_net_exit(net_list, act_pedit_ops.net_id);
 }
 
 static struct pernet_operations pedit_net_ops = {
 	.init = pedit_init_net,
 	.exit_batch = pedit_exit_net,
-	.id   = &pedit_net_id,
+	.id   = &act_pedit_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_police.c b/net/sched/act_police.c
index b759628a47c2..0adb26e366a7 100644
--- a/net/sched/act_police.c
+++ b/net/sched/act_police.c
@@ -22,19 +22,8 @@
 
 /* Each policer is serialized by its individual spinlock */
 
-static unsigned int police_net_id;
 static struct tc_action_ops act_police_ops;
 
-static int tcf_police_walker(struct net *net, struct sk_buff *skb,
-				 struct netlink_callback *cb, int type,
-				 const struct tc_action_ops *ops,
-				 struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, police_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
 static const struct nla_policy police_policy[TCA_POLICE_MAX + 1] = {
 	[TCA_POLICE_RATE]	= { .len = TC_RTAB_SIZE },
 	[TCA_POLICE_PEAKRATE]	= { .len = TC_RTAB_SIZE },
@@ -58,7 +47,7 @@ static int tcf_police_init(struct net *net, struct nlattr *nla,
 	struct tc_police *parm;
 	struct tcf_police *police;
 	struct qdisc_rate_table *R_tab = NULL, *P_tab = NULL;
-	struct tc_action_net *tn = net_generic(net, police_net_id);
+	struct tc_action_net *tn = net_generic(net, act_police_ops.net_id);
 	struct tcf_police_params *new;
 	bool exists = false;
 	u32 index;
@@ -412,13 +401,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_police_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, police_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static int tcf_police_act_to_flow_act(int tc_act, u32 *extval,
 				      struct netlink_ext_ack *extack)
 {
@@ -513,8 +495,6 @@ static struct tc_action_ops act_police_ops = {
 	.act		=	tcf_police_act,
 	.dump		=	tcf_police_dump,
 	.init		=	tcf_police_init,
-	.walk		=	tcf_police_walker,
-	.lookup		=	tcf_police_search,
 	.cleanup	=	tcf_police_cleanup,
 	.offload_act_setup =	tcf_police_offload_act_setup,
 	.size		=	sizeof(struct tcf_police),
@@ -522,20 +502,20 @@ static struct tc_action_ops act_police_ops = {
 
 static __net_init int police_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, police_net_id);
+	struct tc_action_net *tn = net_generic(net, act_police_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_police_ops);
 }
 
 static void __net_exit police_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, police_net_id);
+	tc_action_net_exit(net_list, act_police_ops.net_id);
 }
 
 static struct pernet_operations police_net_ops = {
 	.init = police_init_net,
 	.exit_batch = police_exit_net,
-	.id   = &police_net_id,
+	.id   = &act_police_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_sample.c b/net/sched/act_sample.c
index 2f7f5e44d28c..5ba36f70e3a1 100644
--- a/net/sched/act_sample.c
+++ b/net/sched/act_sample.c
@@ -23,7 +23,6 @@
 
 #include <linux/if_arp.h>
 
-static unsigned int sample_net_id;
 static struct tc_action_ops act_sample_ops;
 
 static const struct nla_policy sample_policy[TCA_SAMPLE_MAX + 1] = {
@@ -38,7 +37,7 @@ static int tcf_sample_init(struct net *net, struct nlattr *nla,
 			   struct tcf_proto *tp,
 			   u32 flags, struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, sample_net_id);
+	struct tc_action_net *tn = net_generic(net, act_sample_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_SAMPLE_MAX + 1];
 	struct psample_group *psample_group;
@@ -241,23 +240,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_sample_walker(struct net *net, struct sk_buff *skb,
-			     struct netlink_callback *cb, int type,
-			     const struct tc_action_ops *ops,
-			     struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, sample_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_sample_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, sample_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static void tcf_psample_group_put(void *priv)
 {
 	struct psample_group *group = priv;
@@ -321,8 +303,6 @@ static struct tc_action_ops act_sample_ops = {
 	.dump	  = tcf_sample_dump,
 	.init	  = tcf_sample_init,
 	.cleanup  = tcf_sample_cleanup,
-	.walk	  = tcf_sample_walker,
-	.lookup	  = tcf_sample_search,
 	.get_psample_group = tcf_sample_get_group,
 	.offload_act_setup    = tcf_sample_offload_act_setup,
 	.size	  = sizeof(struct tcf_sample),
@@ -330,20 +310,20 @@ static struct tc_action_ops act_sample_ops = {
 
 static __net_init int sample_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, sample_net_id);
+	struct tc_action_net *tn = net_generic(net, act_sample_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_sample_ops);
 }
 
 static void __net_exit sample_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, sample_net_id);
+	tc_action_net_exit(net_list, act_sample_ops.net_id);
 }
 
 static struct pernet_operations sample_net_ops = {
 	.init = sample_init_net,
 	.exit_batch = sample_exit_net,
-	.id   = &sample_net_id,
+	.id   = &act_sample_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_simple.c b/net/sched/act_simple.c
index 8c1d60bde93e..18d376135461 100644
--- a/net/sched/act_simple.c
+++ b/net/sched/act_simple.c
@@ -18,7 +18,6 @@
 #include <linux/tc_act/tc_defact.h>
 #include <net/tc_act/tc_defact.h>
 
-static unsigned int simp_net_id;
 static struct tc_action_ops act_simp_ops;
 
 #define SIMP_MAX_DATA	32
@@ -89,7 +88,7 @@ static int tcf_simp_init(struct net *net, struct nlattr *nla,
 			 struct tcf_proto *tp, u32 flags,
 			 struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, simp_net_id);
+	struct tc_action_net *tn = net_generic(net, act_simp_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_DEF_MAX + 1];
 	struct tcf_chain *goto_ch = NULL;
@@ -198,23 +197,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_simp_walker(struct net *net, struct sk_buff *skb,
-			   struct netlink_callback *cb, int type,
-			   const struct tc_action_ops *ops,
-			   struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, simp_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_simp_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, simp_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static struct tc_action_ops act_simp_ops = {
 	.kind		=	"simple",
 	.id		=	TCA_ID_SIMP,
@@ -223,27 +205,25 @@ static struct tc_action_ops act_simp_ops = {
 	.dump		=	tcf_simp_dump,
 	.cleanup	=	tcf_simp_release,
 	.init		=	tcf_simp_init,
-	.walk		=	tcf_simp_walker,
-	.lookup		=	tcf_simp_search,
 	.size		=	sizeof(struct tcf_defact),
 };
 
 static __net_init int simp_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, simp_net_id);
+	struct tc_action_net *tn = net_generic(net, act_simp_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_simp_ops);
 }
 
 static void __net_exit simp_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, simp_net_id);
+	tc_action_net_exit(net_list, act_simp_ops.net_id);
 }
 
 static struct pernet_operations simp_net_ops = {
 	.init = simp_init_net,
 	.exit_batch = simp_exit_net,
-	.id   = &simp_net_id,
+	.id   = &act_simp_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c
index e3bd11dfe1ca..7f598784fd30 100644
--- a/net/sched/act_skbedit.c
+++ b/net/sched/act_skbedit.c
@@ -20,7 +20,6 @@
 #include <linux/tc_act/tc_skbedit.h>
 #include <net/tc_act/tc_skbedit.h>
 
-static unsigned int skbedit_net_id;
 static struct tc_action_ops act_skbedit_ops;
 
 static u16 tcf_skbedit_hash(struct tcf_skbedit_params *params,
@@ -118,7 +117,7 @@ static int tcf_skbedit_init(struct net *net, struct nlattr *nla,
 			    struct tcf_proto *tp, u32 act_flags,
 			    struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, skbedit_net_id);
+	struct tc_action_net *tn = net_generic(net, act_skbedit_ops.net_id);
 	bool bind = act_flags & TCA_ACT_FLAGS_BIND;
 	struct tcf_skbedit_params *params_new;
 	struct nlattr *tb[TCA_SKBEDIT_MAX + 1];
@@ -347,23 +346,6 @@ static void tcf_skbedit_cleanup(struct tc_action *a)
 		kfree_rcu(params, rcu);
 }
 
-static int tcf_skbedit_walker(struct net *net, struct sk_buff *skb,
-			      struct netlink_callback *cb, int type,
-			      const struct tc_action_ops *ops,
-			      struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, skbedit_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_skbedit_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, skbedit_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static size_t tcf_skbedit_get_fill_size(const struct tc_action *act)
 {
 	return nla_total_size(sizeof(struct tc_skbedit))
@@ -428,29 +410,27 @@ static struct tc_action_ops act_skbedit_ops = {
 	.dump		=	tcf_skbedit_dump,
 	.init		=	tcf_skbedit_init,
 	.cleanup	=	tcf_skbedit_cleanup,
-	.walk		=	tcf_skbedit_walker,
 	.get_fill_size	=	tcf_skbedit_get_fill_size,
-	.lookup		=	tcf_skbedit_search,
 	.offload_act_setup =	tcf_skbedit_offload_act_setup,
 	.size		=	sizeof(struct tcf_skbedit),
 };
 
 static __net_init int skbedit_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, skbedit_net_id);
+	struct tc_action_net *tn = net_generic(net, act_skbedit_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_skbedit_ops);
 }
 
 static void __net_exit skbedit_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, skbedit_net_id);
+	tc_action_net_exit(net_list, act_skbedit_ops.net_id);
 }
 
 static struct pernet_operations skbedit_net_ops = {
 	.init = skbedit_init_net,
 	.exit_batch = skbedit_exit_net,
-	.id   = &skbedit_net_id,
+	.id   = &act_skbedit_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c
index 2083612d8780..d98758a63934 100644
--- a/net/sched/act_skbmod.c
+++ b/net/sched/act_skbmod.c
@@ -19,7 +19,6 @@
 #include <linux/tc_act/tc_skbmod.h>
 #include <net/tc_act/tc_skbmod.h>
 
-static unsigned int skbmod_net_id;
 static struct tc_action_ops act_skbmod_ops;
 
 static int tcf_skbmod_act(struct sk_buff *skb, const struct tc_action *a,
@@ -103,7 +102,7 @@ static int tcf_skbmod_init(struct net *net, struct nlattr *nla,
 			   struct tcf_proto *tp, u32 flags,
 			   struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, skbmod_net_id);
+	struct tc_action_net *tn = net_generic(net, act_skbmod_ops.net_id);
 	bool ovr = flags & TCA_ACT_FLAGS_REPLACE;
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_SKBMOD_MAX + 1];
@@ -276,23 +275,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_skbmod_walker(struct net *net, struct sk_buff *skb,
-			     struct netlink_callback *cb, int type,
-			     const struct tc_action_ops *ops,
-			     struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, skbmod_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tcf_skbmod_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, skbmod_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static struct tc_action_ops act_skbmod_ops = {
 	.kind		=	"skbmod",
 	.id		=	TCA_ACT_SKBMOD,
@@ -301,27 +283,25 @@ static struct tc_action_ops act_skbmod_ops = {
 	.dump		=	tcf_skbmod_dump,
 	.init		=	tcf_skbmod_init,
 	.cleanup	=	tcf_skbmod_cleanup,
-	.walk		=	tcf_skbmod_walker,
-	.lookup		=	tcf_skbmod_search,
 	.size		=	sizeof(struct tcf_skbmod),
 };
 
 static __net_init int skbmod_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, skbmod_net_id);
+	struct tc_action_net *tn = net_generic(net, act_skbmod_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_skbmod_ops);
 }
 
 static void __net_exit skbmod_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, skbmod_net_id);
+	tc_action_net_exit(net_list, act_skbmod_ops.net_id);
 }
 
 static struct pernet_operations skbmod_net_ops = {
 	.init = skbmod_init_net,
 	.exit_batch = skbmod_exit_net,
-	.id   = &skbmod_net_id,
+	.id   = &act_skbmod_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_tunnel_key.c b/net/sched/act_tunnel_key.c
index 856dc23cef8c..2691a3d8e451 100644
--- a/net/sched/act_tunnel_key.c
+++ b/net/sched/act_tunnel_key.c
@@ -20,7 +20,6 @@
 #include <linux/tc_act/tc_tunnel_key.h>
 #include <net/tc_act/tc_tunnel_key.h>
 
-static unsigned int tunnel_key_net_id;
 static struct tc_action_ops act_tunnel_key_ops;
 
 static int tunnel_key_act(struct sk_buff *skb, const struct tc_action *a,
@@ -358,7 +357,7 @@ static int tunnel_key_init(struct net *net, struct nlattr *nla,
 			   struct tcf_proto *tp, u32 act_flags,
 			   struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, tunnel_key_net_id);
+	struct tc_action_net *tn = net_generic(net, act_tunnel_key_ops.net_id);
 	bool bind = act_flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_TUNNEL_KEY_MAX + 1];
 	struct tcf_tunnel_key_params *params_new;
@@ -770,23 +769,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tunnel_key_walker(struct net *net, struct sk_buff *skb,
-			     struct netlink_callback *cb, int type,
-			     const struct tc_action_ops *ops,
-			     struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, tunnel_key_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
-static int tunnel_key_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, tunnel_key_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static void tcf_tunnel_encap_put_tunnel(void *priv)
 {
 	struct ip_tunnel_info *tunnel = priv;
@@ -850,28 +832,26 @@ static struct tc_action_ops act_tunnel_key_ops = {
 	.dump		=	tunnel_key_dump,
 	.init		=	tunnel_key_init,
 	.cleanup	=	tunnel_key_release,
-	.walk		=	tunnel_key_walker,
-	.lookup		=	tunnel_key_search,
 	.offload_act_setup =	tcf_tunnel_key_offload_act_setup,
 	.size		=	sizeof(struct tcf_tunnel_key),
 };
 
 static __net_init int tunnel_key_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, tunnel_key_net_id);
+	struct tc_action_net *tn = net_generic(net, act_tunnel_key_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_tunnel_key_ops);
 }
 
 static void __net_exit tunnel_key_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, tunnel_key_net_id);
+	tc_action_net_exit(net_list, act_tunnel_key_ops.net_id);
 }
 
 static struct pernet_operations tunnel_key_net_ops = {
 	.init = tunnel_key_init_net,
 	.exit_batch = tunnel_key_exit_net,
-	.id   = &tunnel_key_net_id,
+	.id   = &act_tunnel_key_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/act_vlan.c b/net/sched/act_vlan.c
index 68b5e772386a..7b24e898a3e6 100644
--- a/net/sched/act_vlan.c
+++ b/net/sched/act_vlan.c
@@ -16,7 +16,6 @@
 #include <linux/tc_act/tc_vlan.h>
 #include <net/tc_act/tc_vlan.h>
 
-static unsigned int vlan_net_id;
 static struct tc_action_ops act_vlan_ops;
 
 static int tcf_vlan_act(struct sk_buff *skb, const struct tc_action *a,
@@ -117,7 +116,7 @@ static int tcf_vlan_init(struct net *net, struct nlattr *nla,
 			 struct tcf_proto *tp, u32 flags,
 			 struct netlink_ext_ack *extack)
 {
-	struct tc_action_net *tn = net_generic(net, vlan_net_id);
+	struct tc_action_net *tn = net_generic(net, act_vlan_ops.net_id);
 	bool bind = flags & TCA_ACT_FLAGS_BIND;
 	struct nlattr *tb[TCA_VLAN_MAX + 1];
 	struct tcf_chain *goto_ch = NULL;
@@ -333,16 +332,6 @@ nla_put_failure:
 	return -1;
 }
 
-static int tcf_vlan_walker(struct net *net, struct sk_buff *skb,
-			   struct netlink_callback *cb, int type,
-			   const struct tc_action_ops *ops,
-			   struct netlink_ext_ack *extack)
-{
-	struct tc_action_net *tn = net_generic(net, vlan_net_id);
-
-	return tcf_generic_walker(tn, skb, cb, type, ops, extack);
-}
-
 static void tcf_vlan_stats_update(struct tc_action *a, u64 bytes, u64 packets,
 				  u64 drops, u64 lastuse, bool hw)
 {
@@ -353,13 +342,6 @@ static void tcf_vlan_stats_update(struct tc_action *a, u64 bytes, u64 packets,
 	tm->lastuse = max_t(u64, tm->lastuse, lastuse);
 }
 
-static int tcf_vlan_search(struct net *net, struct tc_action **a, u32 index)
-{
-	struct tc_action_net *tn = net_generic(net, vlan_net_id);
-
-	return tcf_idr_search(tn, a, index);
-}
-
 static size_t tcf_vlan_get_fill_size(const struct tc_action *act)
 {
 	return nla_total_size(sizeof(struct tc_vlan))
@@ -438,30 +420,28 @@ static struct tc_action_ops act_vlan_ops = {
 	.dump		=	tcf_vlan_dump,
 	.init		=	tcf_vlan_init,
 	.cleanup	=	tcf_vlan_cleanup,
-	.walk		=	tcf_vlan_walker,
 	.stats_update	=	tcf_vlan_stats_update,
 	.get_fill_size	=	tcf_vlan_get_fill_size,
-	.lookup		=	tcf_vlan_search,
 	.offload_act_setup =	tcf_vlan_offload_act_setup,
 	.size		=	sizeof(struct tcf_vlan),
 };
 
 static __net_init int vlan_init_net(struct net *net)
 {
-	struct tc_action_net *tn = net_generic(net, vlan_net_id);
+	struct tc_action_net *tn = net_generic(net, act_vlan_ops.net_id);
 
 	return tc_action_net_init(net, tn, &act_vlan_ops);
 }
 
 static void __net_exit vlan_exit_net(struct list_head *net_list)
 {
-	tc_action_net_exit(net_list, vlan_net_id);
+	tc_action_net_exit(net_list, act_vlan_ops.net_id);
 }
 
 static struct pernet_operations vlan_net_ops = {
 	.init = vlan_init_net,
 	.exit_batch = vlan_exit_net,
-	.id   = &vlan_net_id,
+	.id   = &act_vlan_ops.net_id,
 	.size = sizeof(struct tc_action_net),
 };
 
diff --git a/net/sched/cls_api.c b/net/sched/cls_api.c
index 51d175f3fbcb..50566db45949 100644
--- a/net/sched/cls_api.c
+++ b/net/sched/cls_api.c
@@ -1977,9 +1977,6 @@ static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
 	bool rtnl_held = false;
 	u32 flags;
 
-	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
-		return -EPERM;
-
 replay:
 	tp_created = 0;
 
@@ -2209,9 +2206,6 @@ static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n,
 	int err;
 	bool rtnl_held = false;
 
-	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
-		return -EPERM;
-
 	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
 				     rtm_tca_policy, extack);
 	if (err < 0)
@@ -2827,10 +2821,6 @@ static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n,
 	unsigned long cl;
 	int err;
 
-	if (n->nlmsg_type != RTM_GETCHAIN &&
-	    !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
-		return -EPERM;
-
 replay:
 	q = NULL;
 	err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX,
@@ -3640,9 +3630,6 @@ int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch,
 	if (err)
 		return err;
 
-	if (!block_index)
-		return 0;
-
 	qe->info.binder_type = binder_type;
 	qe->info.chain_head_change = tcf_chain_head_change_dflt;
 	qe->info.chain_head_change_priv = &qe->filter_chain;
diff --git a/net/sched/cls_basic.c b/net/sched/cls_basic.c
index 8158fc9ee1ab..d229ce99e554 100644
--- a/net/sched/cls_basic.c
+++ b/net/sched/cls_basic.c
@@ -251,15 +251,8 @@ static void basic_walk(struct tcf_proto *tp, struct tcf_walker *arg,
 	struct basic_filter *f;
 
 	list_for_each_entry(f, &head->flist, link) {
-		if (arg->count < arg->skip)
-			goto skip;
-
-		if (arg->fn(tp, f, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_cls_stats_dump(tp, arg, f))
 			break;
-		}
-skip:
-		arg->count++;
 	}
 }
 
@@ -268,12 +261,7 @@ static void basic_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
 {
 	struct basic_filter *f = fh;
 
-	if (f && f->res.classid == classid) {
-		if (cl)
-			__tcf_bind_filter(q, &f->res, base);
-		else
-			__tcf_unbind_filter(q, &f->res);
-	}
+	tc_cls_bind_class(classid, cl, q, &f->res, base);
 }
 
 static int basic_dump(struct net *net, struct tcf_proto *tp, void *fh,
diff --git a/net/sched/cls_bpf.c b/net/sched/cls_bpf.c
index c85b85a192bf..bc317b3eac12 100644
--- a/net/sched/cls_bpf.c
+++ b/net/sched/cls_bpf.c
@@ -635,12 +635,7 @@ static void cls_bpf_bind_class(void *fh, u32 classid, unsigned long cl,
 {
 	struct cls_bpf_prog *prog = fh;
 
-	if (prog && prog->res.classid == classid) {
-		if (cl)
-			__tcf_bind_filter(q, &prog->res, base);
-		else
-			__tcf_unbind_filter(q, &prog->res);
-	}
+	tc_cls_bind_class(classid, cl, q, &prog->res, base);
 }
 
 static void cls_bpf_walk(struct tcf_proto *tp, struct tcf_walker *arg,
@@ -650,14 +645,8 @@ static void cls_bpf_walk(struct tcf_proto *tp, struct tcf_walker *arg,
 	struct cls_bpf_prog *prog;
 
 	list_for_each_entry(prog, &head->plist, link) {
-		if (arg->count < arg->skip)
-			goto skip;
-		if (arg->fn(tp, prog, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_cls_stats_dump(tp, arg, prog))
 			break;
-		}
-skip:
-		arg->count++;
 	}
 }
 
diff --git a/net/sched/cls_flow.c b/net/sched/cls_flow.c
index 972303aa8edd..014cd3de7b5d 100644
--- a/net/sched/cls_flow.c
+++ b/net/sched/cls_flow.c
@@ -683,14 +683,8 @@ static void flow_walk(struct tcf_proto *tp, struct tcf_walker *arg,
 	struct flow_filter *f;
 
 	list_for_each_entry(f, &head->filters, list) {
-		if (arg->count < arg->skip)
-			goto skip;
-		if (arg->fn(tp, f, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_cls_stats_dump(tp, arg, f))
 			break;
-		}
-skip:
-		arg->count++;
 	}
 }
 
diff --git a/net/sched/cls_flower.c b/net/sched/cls_flower.c
index 041d63ff809a..25bc57ee6ea1 100644
--- a/net/sched/cls_flower.c
+++ b/net/sched/cls_flower.c
@@ -69,6 +69,7 @@ struct fl_flow_key {
 	struct flow_dissector_key_hash hash;
 	struct flow_dissector_key_num_of_vlans num_of_vlans;
 	struct flow_dissector_key_pppoe pppoe;
+	struct flow_dissector_key_l2tpv3 l2tpv3;
 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
 
 struct fl_flow_mask_range {
@@ -712,6 +713,7 @@ static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = {
 	[TCA_FLOWER_KEY_NUM_OF_VLANS]	= { .type = NLA_U8 },
 	[TCA_FLOWER_KEY_PPPOE_SID]	= { .type = NLA_U16 },
 	[TCA_FLOWER_KEY_PPP_PROTO]	= { .type = NLA_U16 },
+	[TCA_FLOWER_KEY_L2TPV3_SID]	= { .type = NLA_U32 },
 
 };
 
@@ -1790,6 +1792,11 @@ static int fl_set_key(struct net *net, struct nlattr **tb,
 		fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA,
 			       mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK,
 			       sizeof(key->arp.tha));
+	} else if (key->basic.ip_proto == IPPROTO_L2TP) {
+		fl_set_key_val(tb, &key->l2tpv3.session_id,
+			       TCA_FLOWER_KEY_L2TPV3_SID,
+			       &mask->l2tpv3.session_id, TCA_FLOWER_UNSPEC,
+			       sizeof(key->l2tpv3.session_id));
 	}
 
 	if (key->basic.ip_proto == IPPROTO_TCP ||
@@ -1970,6 +1977,8 @@ static void fl_init_dissector(struct flow_dissector *dissector,
 			     FLOW_DISSECTOR_KEY_NUM_OF_VLANS, num_of_vlans);
 	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
 			     FLOW_DISSECTOR_KEY_PPPOE, pppoe);
+	FL_KEY_SET_IF_MASKED(mask, keys, cnt,
+			     FLOW_DISSECTOR_KEY_L2TPV3, l2tpv3);
 
 	skb_flow_dissector_init(dissector, keys, cnt);
 }
@@ -3196,6 +3205,13 @@ static int fl_dump_key(struct sk_buff *skb, struct net *net,
 				  mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK,
 				  sizeof(key->arp.tha))))
 		goto nla_put_failure;
+	else if (key->basic.ip_proto == IPPROTO_L2TP &&
+		 fl_dump_key_val(skb, &key->l2tpv3.session_id,
+				 TCA_FLOWER_KEY_L2TPV3_SID,
+				 &mask->l2tpv3.session_id,
+				 TCA_FLOWER_UNSPEC,
+				 sizeof(key->l2tpv3.session_id)))
+		goto nla_put_failure;
 
 	if ((key->basic.ip_proto == IPPROTO_TCP ||
 	     key->basic.ip_proto == IPPROTO_UDP ||
@@ -3389,12 +3405,7 @@ static void fl_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
 {
 	struct cls_fl_filter *f = fh;
 
-	if (f && f->res.classid == classid) {
-		if (cl)
-			__tcf_bind_filter(q, &f->res, base);
-		else
-			__tcf_unbind_filter(q, &f->res);
-	}
+	tc_cls_bind_class(classid, cl, q, &f->res, base);
 }
 
 static bool fl_delete_empty(struct tcf_proto *tp)
diff --git a/net/sched/cls_fw.c b/net/sched/cls_fw.c
index 8654b0ce997c..a32351da968c 100644
--- a/net/sched/cls_fw.c
+++ b/net/sched/cls_fw.c
@@ -358,15 +358,8 @@ static void fw_walk(struct tcf_proto *tp, struct tcf_walker *arg,
 
 		for (f = rtnl_dereference(head->ht[h]); f;
 		     f = rtnl_dereference(f->next)) {
-			if (arg->count < arg->skip) {
-				arg->count++;
-				continue;
-			}
-			if (arg->fn(tp, f, arg) < 0) {
-				arg->stop = 1;
+			if (!tc_cls_stats_dump(tp, arg, f))
 				return;
-			}
-			arg->count++;
 		}
 	}
 }
@@ -423,12 +416,7 @@ static void fw_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
 {
 	struct fw_filter *f = fh;
 
-	if (f && f->res.classid == classid) {
-		if (cl)
-			__tcf_bind_filter(q, &f->res, base);
-		else
-			__tcf_unbind_filter(q, &f->res);
-	}
+	tc_cls_bind_class(classid, cl, q, &f->res, base);
 }
 
 static struct tcf_proto_ops cls_fw_ops __read_mostly = {
diff --git a/net/sched/cls_matchall.c b/net/sched/cls_matchall.c
index 06cf22adbab7..39a5d9c170de 100644
--- a/net/sched/cls_matchall.c
+++ b/net/sched/cls_matchall.c
@@ -313,10 +313,7 @@ static int mall_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb,
 	tc_cleanup_offload_action(&cls_mall.rule->action);
 	kfree(cls_mall.rule);
 
-	if (err)
-		return err;
-
-	return 0;
+	return err;
 }
 
 static void mall_stats_hw_filter(struct tcf_proto *tp,
@@ -397,12 +394,7 @@ static void mall_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
 {
 	struct cls_mall_head *head = fh;
 
-	if (head && head->res.classid == classid) {
-		if (cl)
-			__tcf_bind_filter(q, &head->res, base);
-		else
-			__tcf_unbind_filter(q, &head->res);
-	}
+	tc_cls_bind_class(classid, cl, q, &head->res, base);
 }
 
 static struct tcf_proto_ops cls_mall_ops __read_mostly = {
diff --git a/net/sched/cls_route.c b/net/sched/cls_route.c
index 48712bc51bda..9e43b929d4ca 100644
--- a/net/sched/cls_route.c
+++ b/net/sched/cls_route.c
@@ -488,7 +488,7 @@ static int route4_change(struct net *net, struct sk_buff *in_skb,
 	}
 
 	if (opt == NULL)
-		return handle ? -EINVAL : 0;
+		return -EINVAL;
 
 	err = nla_parse_nested_deprecated(tb, TCA_ROUTE4_MAX, opt,
 					  route4_policy, NULL);
@@ -496,7 +496,7 @@ static int route4_change(struct net *net, struct sk_buff *in_skb,
 		return err;
 
 	fold = *arg;
-	if (fold && handle && fold->handle != handle)
+	if (fold && fold->handle != handle)
 			return -EINVAL;
 
 	err = -ENOBUFS;
@@ -587,15 +587,8 @@ static void route4_walk(struct tcf_proto *tp, struct tcf_walker *arg,
 				for (f = rtnl_dereference(b->ht[h1]);
 				     f;
 				     f = rtnl_dereference(f->next)) {
-					if (arg->count < arg->skip) {
-						arg->count++;
-						continue;
-					}
-					if (arg->fn(tp, f, arg) < 0) {
-						arg->stop = 1;
+					if (!tc_cls_stats_dump(tp, arg, f))
 						return;
-					}
-					arg->count++;
 				}
 			}
 		}
@@ -656,12 +649,7 @@ static void route4_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
 {
 	struct route4_filter *f = fh;
 
-	if (f && f->res.classid == classid) {
-		if (cl)
-			__tcf_bind_filter(q, &f->res, base);
-		else
-			__tcf_unbind_filter(q, &f->res);
-	}
+	tc_cls_bind_class(classid, cl, q, &f->res, base);
 }
 
 static struct tcf_proto_ops cls_route4_ops __read_mostly = {
diff --git a/net/sched/cls_rsvp.h b/net/sched/cls_rsvp.h
index 5cd9d6b143c4..b00a7dbd0587 100644
--- a/net/sched/cls_rsvp.h
+++ b/net/sched/cls_rsvp.h
@@ -671,15 +671,8 @@ static void rsvp_walk(struct tcf_proto *tp, struct tcf_walker *arg,
 
 				for (f = rtnl_dereference(s->ht[h1]); f;
 				     f = rtnl_dereference(f->next)) {
-					if (arg->count < arg->skip) {
-						arg->count++;
-						continue;
-					}
-					if (arg->fn(tp, f, arg) < 0) {
-						arg->stop = 1;
+					if (!tc_cls_stats_dump(tp, arg, f))
 						return;
-					}
-					arg->count++;
 				}
 			}
 		}
@@ -740,12 +733,7 @@ static void rsvp_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
 {
 	struct rsvp_filter *f = fh;
 
-	if (f && f->res.classid == classid) {
-		if (cl)
-			__tcf_bind_filter(q, &f->res, base);
-		else
-			__tcf_unbind_filter(q, &f->res);
-	}
+	tc_cls_bind_class(classid, cl, q, &f->res, base);
 }
 
 static struct tcf_proto_ops RSVP_OPS __read_mostly = {
diff --git a/net/sched/cls_tcindex.c b/net/sched/cls_tcindex.c
index 742c7d49a958..1c9eeb98d826 100644
--- a/net/sched/cls_tcindex.c
+++ b/net/sched/cls_tcindex.c
@@ -566,13 +566,8 @@ static void tcindex_walk(struct tcf_proto *tp, struct tcf_walker *walker,
 		for (i = 0; i < p->hash; i++) {
 			if (!p->perfect[i].res.class)
 				continue;
-			if (walker->count >= walker->skip) {
-				if (walker->fn(tp, p->perfect + i, walker) < 0) {
-					walker->stop = 1;
-					return;
-				}
-			}
-			walker->count++;
+			if (!tc_cls_stats_dump(tp, walker, p->perfect + i))
+				return;
 		}
 	}
 	if (!p->h)
@@ -580,13 +575,8 @@ static void tcindex_walk(struct tcf_proto *tp, struct tcf_walker *walker,
 	for (i = 0; i < p->hash; i++) {
 		for (f = rtnl_dereference(p->h[i]); f; f = next) {
 			next = rtnl_dereference(f->next);
-			if (walker->count >= walker->skip) {
-				if (walker->fn(tp, &f->result, walker) < 0) {
-					walker->stop = 1;
-					return;
-				}
-			}
-			walker->count++;
+			if (!tc_cls_stats_dump(tp, walker, &f->result))
+				return;
 		}
 	}
 }
@@ -701,12 +691,7 @@ static void tcindex_bind_class(void *fh, u32 classid, unsigned long cl,
 {
 	struct tcindex_filter_result *r = fh;
 
-	if (r && r->res.classid == classid) {
-		if (cl)
-			__tcf_bind_filter(q, &r->res, base);
-		else
-			__tcf_unbind_filter(q, &r->res);
-	}
+	tc_cls_bind_class(classid, cl, q, &r->res, base);
 }
 
 static struct tcf_proto_ops cls_tcindex_ops __read_mostly = {
diff --git a/net/sched/cls_u32.c b/net/sched/cls_u32.c
index 4d27300c287c..34d25f7a0687 100644
--- a/net/sched/cls_u32.c
+++ b/net/sched/cls_u32.c
@@ -1040,7 +1040,11 @@ static int u32_change(struct net *net, struct sk_buff *in_skb,
 	}
 #endif
 
-	memcpy(&n->sel, s, sel_size);
+	unsafe_memcpy(&n->sel, s, sel_size,
+		      /* A composite flex-array structure destination,
+		       * which was correctly sized with struct_size(),
+		       * bounds-checked against nla_len(), and allocated
+		       * above. */);
 	RCU_INIT_POINTER(n->ht_up, ht);
 	n->handle = handle;
 	n->fshift = s->hmask ? ffs(ntohl(s->hmask)) - 1 : 0;
@@ -1125,26 +1129,16 @@ static void u32_walk(struct tcf_proto *tp, struct tcf_walker *arg,
 	     ht = rtnl_dereference(ht->next)) {
 		if (ht->prio != tp->prio)
 			continue;
-		if (arg->count >= arg->skip) {
-			if (arg->fn(tp, ht, arg) < 0) {
-				arg->stop = 1;
-				return;
-			}
-		}
-		arg->count++;
+
+		if (!tc_cls_stats_dump(tp, arg, ht))
+			return;
+
 		for (h = 0; h <= ht->divisor; h++) {
 			for (n = rtnl_dereference(ht->ht[h]);
 			     n;
 			     n = rtnl_dereference(n->next)) {
-				if (arg->count < arg->skip) {
-					arg->count++;
-					continue;
-				}
-				if (arg->fn(tp, n, arg) < 0) {
-					arg->stop = 1;
+				if (!tc_cls_stats_dump(tp, arg, n))
 					return;
-				}
-				arg->count++;
 			}
 		}
 	}
@@ -1256,12 +1250,7 @@ static void u32_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
 {
 	struct tc_u_knode *n = fh;
 
-	if (n && n->res.classid == classid) {
-		if (cl)
-			__tcf_bind_filter(q, &n->res, base);
-		else
-			__tcf_unbind_filter(q, &n->res);
-	}
+	tc_cls_bind_class(classid, cl, q, &n->res, base);
 }
 
 static int u32_dump(struct net *net, struct tcf_proto *tp, void *fh,
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index bf87b50837a8..c98af0ada706 100644
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -171,7 +171,7 @@ out_einval:
 }
 EXPORT_SYMBOL(register_qdisc);
 
-int unregister_qdisc(struct Qdisc_ops *qops)
+void unregister_qdisc(struct Qdisc_ops *qops)
 {
 	struct Qdisc_ops *q, **qp;
 	int err = -ENOENT;
@@ -186,7 +186,8 @@ int unregister_qdisc(struct Qdisc_ops *qops)
 		err = 0;
 	}
 	write_unlock(&qdisc_mod_lock);
-	return err;
+
+	WARN(err, "unregister qdisc(%s) failed\n", qops->id);
 }
 EXPORT_SYMBOL(unregister_qdisc);
 
@@ -194,7 +195,7 @@ EXPORT_SYMBOL(unregister_qdisc);
 void qdisc_get_default(char *name, size_t len)
 {
 	read_lock(&qdisc_mod_lock);
-	strlcpy(name, default_qdisc_ops->id, len);
+	strscpy(name, default_qdisc_ops->id, len);
 	read_unlock(&qdisc_mod_lock);
 }
 
@@ -867,6 +868,23 @@ void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
 }
 EXPORT_SYMBOL(qdisc_offload_graft_helper);
 
+void qdisc_offload_query_caps(struct net_device *dev,
+			      enum tc_setup_type type,
+			      void *caps, size_t caps_len)
+{
+	const struct net_device_ops *ops = dev->netdev_ops;
+	struct tc_query_caps_base base = {
+		.type = type,
+		.caps = caps,
+	};
+
+	memset(caps, 0, caps_len);
+
+	if (ops->ndo_setup_tc)
+		ops->ndo_setup_tc(dev, TC_QUERY_CAPS, &base);
+}
+EXPORT_SYMBOL(qdisc_offload_query_caps);
+
 static void qdisc_offload_graft_root(struct net_device *dev,
 				     struct Qdisc *new, struct Qdisc *old,
 				     struct netlink_ext_ack *extack)
@@ -1163,7 +1181,7 @@ static int qdisc_block_indexes_set(struct Qdisc *sch, struct nlattr **tca,
 
 static struct Qdisc *qdisc_create(struct net_device *dev,
 				  struct netdev_queue *dev_queue,
-				  struct Qdisc *p, u32 parent, u32 handle,
+				  u32 parent, u32 handle,
 				  struct nlattr **tca, int *errp,
 				  struct netlink_ext_ack *extack)
 {
@@ -1424,10 +1442,6 @@ static int tc_get_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
 	struct Qdisc *p = NULL;
 	int err;
 
-	if ((n->nlmsg_type != RTM_GETQDISC) &&
-	    !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
-		return -EPERM;
-
 	err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
 				     rtm_tca_policy, extack);
 	if (err < 0)
@@ -1508,9 +1522,6 @@ static int tc_modify_qdisc(struct sk_buff *skb, struct nlmsghdr *n,
 	struct Qdisc *q, *p;
 	int err;
 
-	if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
-		return -EPERM;
-
 replay:
 	/* Reinit, just in case something touches this. */
 	err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
@@ -1640,7 +1651,7 @@ create_n_graft:
 	}
 	if (clid == TC_H_INGRESS) {
 		if (dev_ingress_queue(dev)) {
-			q = qdisc_create(dev, dev_ingress_queue(dev), p,
+			q = qdisc_create(dev, dev_ingress_queue(dev),
 					 tcm->tcm_parent, tcm->tcm_parent,
 					 tca, &err, extack);
 		} else {
@@ -1657,7 +1668,7 @@ create_n_graft:
 		else
 			dev_queue = netdev_get_tx_queue(dev, 0);
 
-		q = qdisc_create(dev, dev_queue, p,
+		q = qdisc_create(dev, dev_queue,
 				 tcm->tcm_parent, tcm->tcm_handle,
 				 tca, &err, extack);
 	}
@@ -1904,7 +1915,7 @@ static int tcf_node_bind(struct tcf_proto *tp, void *n, struct tcf_walker *arg)
 {
 	struct tcf_bind_args *a = (void *)arg;
 
-	if (tp->ops->bind_class) {
+	if (n && tp->ops->bind_class) {
 		struct Qdisc *q = tcf_block_q(tp->chain->block);
 
 		sch_tree_lock(q);
@@ -1992,10 +2003,6 @@ static int tc_ctl_tclass(struct sk_buff *skb, struct nlmsghdr *n,
 	u32 qid;
 	int err;
 
-	if ((n->nlmsg_type != RTM_GETTCLASS) &&
-	    !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN))
-		return -EPERM;
-
 	err = nlmsg_parse_deprecated(n, sizeof(*tcm), tca, TCA_MAX,
 				     rtm_tca_policy, extack);
 	if (err < 0)
diff --git a/net/sched/sch_atm.c b/net/sched/sch_atm.c
index 4c8e994cf0a5..f52255fea652 100644
--- a/net/sched/sch_atm.c
+++ b/net/sched/sch_atm.c
@@ -354,12 +354,8 @@ static void atm_tc_walk(struct Qdisc *sch, struct qdisc_walker *walker)
 	if (walker->stop)
 		return;
 	list_for_each_entry(flow, &p->flows, list) {
-		if (walker->count >= walker->skip &&
-		    walker->fn(sch, (unsigned long)flow, walker) < 0) {
-			walker->stop = 1;
+		if (!tc_qdisc_stats_dump(sch, (unsigned long)flow, walker))
 			break;
-		}
-		walker->count++;
 	}
 }
 
@@ -577,7 +573,6 @@ static void atm_tc_reset(struct Qdisc *sch)
 	pr_debug("atm_tc_reset(sch %p,[qdisc %p])\n", sch, p);
 	list_for_each_entry(flow, &p->flows, list)
 		qdisc_reset(flow->q);
-	sch->q.qlen = 0;
 }
 
 static void atm_tc_destroy(struct Qdisc *sch)
diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c
index a43a58a73d09..55c6879d2c7e 100644
--- a/net/sched/sch_cake.c
+++ b/net/sched/sch_cake.c
@@ -2569,9 +2569,6 @@ static int cake_change(struct Qdisc *sch, struct nlattr *opt,
 	struct nlattr *tb[TCA_CAKE_MAX + 1];
 	int err;
 
-	if (!opt)
-		return -EINVAL;
-
 	err = nla_parse_nested_deprecated(tb, TCA_CAKE_MAX, opt, cake_policy,
 					  extack);
 	if (err < 0)
@@ -3064,16 +3061,13 @@ static void cake_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 		struct cake_tin_data *b = &q->tins[q->tin_order[i]];
 
 		for (j = 0; j < CAKE_QUEUES; j++) {
-			if (list_empty(&b->flows[j].flowchain) ||
-			    arg->count < arg->skip) {
+			if (list_empty(&b->flows[j].flowchain)) {
 				arg->count++;
 				continue;
 			}
-			if (arg->fn(sch, i * CAKE_QUEUES + j + 1, arg) < 0) {
-				arg->stop = 1;
+			if (!tc_qdisc_stats_dump(sch, i * CAKE_QUEUES + j + 1,
+						 arg))
 				break;
-			}
-			arg->count++;
 		}
 	}
 }
diff --git a/net/sched/sch_cbq.c b/net/sched/sch_cbq.c
index 91a0dc463c48..6568e17c4c63 100644
--- a/net/sched/sch_cbq.c
+++ b/net/sched/sch_cbq.c
@@ -975,7 +975,6 @@ cbq_reset(struct Qdisc *sch)
 			cl->cpriority = cl->priority;
 		}
 	}
-	sch->q.qlen = 0;
 }
 
 
@@ -1677,15 +1676,8 @@ static void cbq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 
 	for (h = 0; h < q->clhash.hashsize; h++) {
 		hlist_for_each_entry(cl, &q->clhash.hash[h], common.hnode) {
-			if (arg->count < arg->skip) {
-				arg->count++;
-				continue;
-			}
-			if (arg->fn(sch, (unsigned long)cl, arg) < 0) {
-				arg->stop = 1;
+			if (!tc_qdisc_stats_dump(sch, (unsigned long)cl, arg))
 				return;
-			}
-			arg->count++;
 		}
 	}
 }
diff --git a/net/sched/sch_cbs.c b/net/sched/sch_cbs.c
index 459cc240eda9..cac870eb7897 100644
--- a/net/sched/sch_cbs.c
+++ b/net/sched/sch_cbs.c
@@ -520,13 +520,7 @@ static unsigned long cbs_find(struct Qdisc *sch, u32 classid)
 static void cbs_walk(struct Qdisc *sch, struct qdisc_walker *walker)
 {
 	if (!walker->stop) {
-		if (walker->count >= walker->skip) {
-			if (walker->fn(sch, 1, walker) < 0) {
-				walker->stop = 1;
-				return;
-			}
-		}
-		walker->count++;
+		tc_qdisc_stats_dump(sch, 1, walker);
 	}
 }
 
diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c
index 2adbd945bf15..3ac3e5c80b6f 100644
--- a/net/sched/sch_choke.c
+++ b/net/sched/sch_choke.c
@@ -60,7 +60,6 @@ struct choke_sched_data {
 		u32	forced_drop;	/* Forced drops, qavg > max_thresh */
 		u32	forced_mark;	/* Forced marks, qavg > max_thresh */
 		u32	pdrop;          /* Drops due to queue limits */
-		u32	other;          /* Drops due to drop() calls */
 		u32	matched;	/* Drops to flow match */
 	} stats;
 
@@ -315,8 +314,6 @@ static void choke_reset(struct Qdisc *sch)
 		rtnl_qdisc_drop(skb, sch);
 	}
 
-	sch->q.qlen = 0;
-	sch->qstats.backlog = 0;
 	if (q->tab)
 		memset(q->tab, 0, (q->tab_mask + 1) * sizeof(struct sk_buff *));
 	q->head = q->tail = 0;
@@ -466,7 +463,6 @@ static int choke_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
 		.early	= q->stats.prob_drop + q->stats.forced_drop,
 		.marked	= q->stats.prob_mark + q->stats.forced_mark,
 		.pdrop	= q->stats.pdrop,
-		.other	= q->stats.other,
 		.matched = q->stats.matched,
 	};
 
diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c
index 30169b3adbbb..d7a4874543de 100644
--- a/net/sched/sch_codel.c
+++ b/net/sched/sch_codel.c
@@ -138,9 +138,6 @@ static int codel_change(struct Qdisc *sch, struct nlattr *opt,
 	unsigned int qlen, dropped = 0;
 	int err;
 
-	if (!opt)
-		return -EINVAL;
-
 	err = nla_parse_nested_deprecated(tb, TCA_CODEL_MAX, opt,
 					  codel_policy, NULL);
 	if (err < 0)
diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c
index 18e4f7a0b291..e35a4e90f4e6 100644
--- a/net/sched/sch_drr.c
+++ b/net/sched/sch_drr.c
@@ -284,15 +284,8 @@ static void drr_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 
 	for (i = 0; i < q->clhash.hashsize; i++) {
 		hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) {
-			if (arg->count < arg->skip) {
-				arg->count++;
-				continue;
-			}
-			if (arg->fn(sch, (unsigned long)cl, arg) < 0) {
-				arg->stop = 1;
+			if (!tc_qdisc_stats_dump(sch, (unsigned long)cl, arg))
 				return;
-			}
-			arg->count++;
 		}
 	}
 }
@@ -441,8 +434,6 @@ static void drr_reset_qdisc(struct Qdisc *sch)
 			qdisc_reset(cl->qdisc);
 		}
 	}
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
 }
 
 static void drr_destroy_qdisc(struct Qdisc *sch)
diff --git a/net/sched/sch_dsmark.c b/net/sched/sch_dsmark.c
index 4c100d105269..401ffaf87d62 100644
--- a/net/sched/sch_dsmark.c
+++ b/net/sched/sch_dsmark.c
@@ -176,16 +176,12 @@ static void dsmark_walk(struct Qdisc *sch, struct qdisc_walker *walker)
 		return;
 
 	for (i = 0; i < p->indices; i++) {
-		if (p->mv[i].mask == 0xff && !p->mv[i].value)
-			goto ignore;
-		if (walker->count >= walker->skip) {
-			if (walker->fn(sch, i + 1, walker) < 0) {
-				walker->stop = 1;
-				break;
-			}
+		if (p->mv[i].mask == 0xff && !p->mv[i].value) {
+			walker->count++;
+			continue;
 		}
-ignore:
-		walker->count++;
+		if (!tc_qdisc_stats_dump(sch, i + 1, walker))
+			break;
 	}
 }
 
@@ -409,8 +405,6 @@ static void dsmark_reset(struct Qdisc *sch)
 	pr_debug("%s(sch %p,[qdisc %p])\n", __func__, sch, p);
 	if (p->q)
 		qdisc_reset(p->q);
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
 }
 
 static void dsmark_destroy(struct Qdisc *sch)
diff --git a/net/sched/sch_etf.c b/net/sched/sch_etf.c
index c48f91075b5c..61d1f0e32cf3 100644
--- a/net/sched/sch_etf.c
+++ b/net/sched/sch_etf.c
@@ -323,9 +323,6 @@ static int etf_enable_offload(struct net_device *dev, struct etf_sched_data *q,
 	struct tc_etf_qopt_offload etf = { };
 	int err;
 
-	if (q->offload)
-		return 0;
-
 	if (!ops->ndo_setup_tc) {
 		NL_SET_ERR_MSG(extack, "Specified device does not support ETF offload");
 		return -EOPNOTSUPP;
@@ -445,9 +442,6 @@ static void etf_reset(struct Qdisc *sch)
 	timesortedlist_clear(sch);
 	__qdisc_reset_queue(&sch->q);
 
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
-
 	q->last = 0;
 }
 
diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c
index d73393493553..b10efeaf0629 100644
--- a/net/sched/sch_ets.c
+++ b/net/sched/sch_ets.c
@@ -341,15 +341,8 @@ static void ets_qdisc_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 		return;
 
 	for (i = 0; i < q->nbands; i++) {
-		if (arg->count < arg->skip) {
-			arg->count++;
-			continue;
-		}
-		if (arg->fn(sch, i + 1, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_qdisc_stats_dump(sch, i + 1, arg))
 			break;
-		}
-		arg->count++;
 	}
 }
 
@@ -594,11 +587,6 @@ static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt,
 	unsigned int i;
 	int err;
 
-	if (!opt) {
-		NL_SET_ERR_MSG(extack, "ETS options are required for this operation");
-		return -EINVAL;
-	}
-
 	err = nla_parse_nested(tb, TCA_ETS_MAX, opt, ets_policy, extack);
 	if (err < 0)
 		return err;
@@ -727,8 +715,6 @@ static void ets_qdisc_reset(struct Qdisc *sch)
 	}
 	for (band = 0; band < q->nbands; band++)
 		qdisc_reset(q->classes[band].qdisc);
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
 }
 
 static void ets_qdisc_destroy(struct Qdisc *sch)
diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c
index 2fb76fc0cc31..48d14fb90ba0 100644
--- a/net/sched/sch_fq.c
+++ b/net/sched/sch_fq.c
@@ -808,9 +808,6 @@ static int fq_change(struct Qdisc *sch, struct nlattr *opt,
 	unsigned drop_len = 0;
 	u32 fq_log;
 
-	if (!opt)
-		return -EINVAL;
-
 	err = nla_parse_nested_deprecated(tb, TCA_FQ_MAX, opt, fq_policy,
 					  NULL);
 	if (err < 0)
diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c
index 839e1235db05..99d318b60568 100644
--- a/net/sched/sch_fq_codel.c
+++ b/net/sched/sch_fq_codel.c
@@ -347,8 +347,6 @@ static void fq_codel_reset(struct Qdisc *sch)
 		codel_vars_init(&flow->cvars);
 	}
 	memset(q->backlogs, 0, q->flows_cnt * sizeof(u32));
-	sch->q.qlen = 0;
-	sch->qstats.backlog = 0;
 	q->memory_usage = 0;
 }
 
@@ -374,9 +372,6 @@ static int fq_codel_change(struct Qdisc *sch, struct nlattr *opt,
 	u32 quantum = 0;
 	int err;
 
-	if (!opt)
-		return -EINVAL;
-
 	err = nla_parse_nested_deprecated(tb, TCA_FQ_CODEL_MAX, opt,
 					  fq_codel_policy, NULL);
 	if (err < 0)
@@ -483,26 +478,24 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt,
 	if (opt) {
 		err = fq_codel_change(sch, opt, extack);
 		if (err)
-			goto init_failure;
+			return err;
 	}
 
 	err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
 	if (err)
-		goto init_failure;
+		return err;
 
 	if (!q->flows) {
 		q->flows = kvcalloc(q->flows_cnt,
 				    sizeof(struct fq_codel_flow),
 				    GFP_KERNEL);
-		if (!q->flows) {
-			err = -ENOMEM;
-			goto init_failure;
-		}
+		if (!q->flows)
+			return -ENOMEM;
+
 		q->backlogs = kvcalloc(q->flows_cnt, sizeof(u32), GFP_KERNEL);
-		if (!q->backlogs) {
-			err = -ENOMEM;
-			goto alloc_failure;
-		}
+		if (!q->backlogs)
+			return -ENOMEM;
+
 		for (i = 0; i < q->flows_cnt; i++) {
 			struct fq_codel_flow *flow = q->flows + i;
 
@@ -515,13 +508,6 @@ static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt,
 	else
 		sch->flags &= ~TCQ_F_CAN_BYPASS;
 	return 0;
-
-alloc_failure:
-	kvfree(q->flows);
-	q->flows = NULL;
-init_failure:
-	q->flows_cnt = 0;
-	return err;
 }
 
 static int fq_codel_dump(struct Qdisc *sch, struct sk_buff *skb)
@@ -687,16 +673,12 @@ static void fq_codel_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 		return;
 
 	for (i = 0; i < q->flows_cnt; i++) {
-		if (list_empty(&q->flows[i].flowchain) ||
-		    arg->count < arg->skip) {
+		if (list_empty(&q->flows[i].flowchain)) {
 			arg->count++;
 			continue;
 		}
-		if (arg->fn(sch, i + 1, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_qdisc_stats_dump(sch, i + 1, arg))
 			break;
-		}
-		arg->count++;
 	}
 }
 
diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c
index d6aba6edd16e..6980796d435d 100644
--- a/net/sched/sch_fq_pie.c
+++ b/net/sched/sch_fq_pie.c
@@ -283,9 +283,6 @@ static int fq_pie_change(struct Qdisc *sch, struct nlattr *opt,
 	unsigned int num_dropped = 0;
 	int err;
 
-	if (!opt)
-		return -EINVAL;
-
 	err = nla_parse_nested(tb, TCA_FQ_PIE_MAX, opt, fq_pie_policy, extack);
 	if (err < 0)
 		return err;
@@ -521,9 +518,6 @@ static void fq_pie_reset(struct Qdisc *sch)
 		INIT_LIST_HEAD(&flow->flowchain);
 		pie_vars_init(&flow->vars);
 	}
-
-	sch->q.qlen = 0;
-	sch->qstats.backlog = 0;
 }
 
 static void fq_pie_destroy(struct Qdisc *sch)
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index 7a8ea03f673d..a9aadc4e6858 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -941,7 +941,6 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
 		goto errout;
 	__skb_queue_head_init(&sch->gso_skb);
 	__skb_queue_head_init(&sch->skb_bad_txq);
-	qdisc_skb_head_init(&sch->q);
 	gnet_stats_basic_sync_init(&sch->bstats);
 	spin_lock_init(&sch->q.lock);
 
diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c
index 1073c76d05c4..a661b062cca8 100644
--- a/net/sched/sch_gred.c
+++ b/net/sched/sch_gred.c
@@ -648,9 +648,6 @@ static int gred_change(struct Qdisc *sch, struct nlattr *opt,
 	u32 max_P;
 	struct gred_sched_data *prealloc;
 
-	if (opt == NULL)
-		return -EINVAL;
-
 	err = nla_parse_nested_deprecated(tb, TCA_GRED_MAX, opt, gred_policy,
 					  extack);
 	if (err < 0)
@@ -829,7 +826,6 @@ static int gred_dump(struct Qdisc *sch, struct sk_buff *skb)
 		opt.Wlog	= q->parms.Wlog;
 		opt.Plog	= q->parms.Plog;
 		opt.Scell_log	= q->parms.Scell_log;
-		opt.other	= q->stats.other;
 		opt.early	= q->stats.prob_drop;
 		opt.forced	= q->stats.forced_drop;
 		opt.pdrop	= q->stats.pdrop;
@@ -895,8 +891,6 @@ append_opt:
 			goto nla_put_failure;
 		if (nla_put_u32(skb, TCA_GRED_VQ_STAT_PDROP, q->stats.pdrop))
 			goto nla_put_failure;
-		if (nla_put_u32(skb, TCA_GRED_VQ_STAT_OTHER, q->stats.other))
-			goto nla_put_failure;
 
 		nla_nest_end(skb, vq);
 	}
@@ -914,10 +908,9 @@ static void gred_destroy(struct Qdisc *sch)
 	struct gred_sched *table = qdisc_priv(sch);
 	int i;
 
-	for (i = 0; i < table->DPs; i++) {
-		if (table->tab[i])
-			gred_destroy_vq(table->tab[i]);
-	}
+	for (i = 0; i < table->DPs; i++)
+		gred_destroy_vq(table->tab[i]);
+
 	gred_offload(sch, TC_GRED_DESTROY);
 	kfree(table->opt);
 }
diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c
index d3979a6000e7..70b0c5873d32 100644
--- a/net/sched/sch_hfsc.c
+++ b/net/sched/sch_hfsc.c
@@ -1349,15 +1349,8 @@ hfsc_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 	for (i = 0; i < q->clhash.hashsize; i++) {
 		hlist_for_each_entry(cl, &q->clhash.hash[i],
 				     cl_common.hnode) {
-			if (arg->count < arg->skip) {
-				arg->count++;
-				continue;
-			}
-			if (arg->fn(sch, (unsigned long)cl, arg) < 0) {
-				arg->stop = 1;
+			if (!tc_qdisc_stats_dump(sch, (unsigned long)cl, arg))
 				return;
-			}
-			arg->count++;
 		}
 	}
 }
@@ -1430,7 +1423,7 @@ hfsc_change_qdisc(struct Qdisc *sch, struct nlattr *opt,
 	struct hfsc_sched *q = qdisc_priv(sch);
 	struct tc_hfsc_qopt *qopt;
 
-	if (opt == NULL || nla_len(opt) < sizeof(*qopt))
+	if (nla_len(opt) < sizeof(*qopt))
 		return -EINVAL;
 	qopt = nla_data(opt);
 
@@ -1484,8 +1477,6 @@ hfsc_reset_qdisc(struct Qdisc *sch)
 	}
 	q->eligible = RB_ROOT;
 	qdisc_watchdog_cancel(&q->watchdog);
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
 }
 
 static void
diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c
index 420ede875322..d26cd436cbe3 100644
--- a/net/sched/sch_hhf.c
+++ b/net/sched/sch_hhf.c
@@ -516,9 +516,6 @@ static int hhf_change(struct Qdisc *sch, struct nlattr *opt,
 	u32 new_quantum = q->quantum;
 	u32 new_hhf_non_hh_weight = q->hhf_non_hh_weight;
 
-	if (!opt)
-		return -EINVAL;
-
 	err = nla_parse_nested_deprecated(tb, TCA_HHF_MAX, opt, hhf_policy,
 					  NULL);
 	if (err < 0)
diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c
index 23a9d6242429..e5b4bbf3ce3d 100644
--- a/net/sched/sch_htb.c
+++ b/net/sched/sch_htb.c
@@ -1008,8 +1008,6 @@ static void htb_reset(struct Qdisc *sch)
 	}
 	qdisc_watchdog_cancel(&q->watchdog);
 	__qdisc_reset_queue(&q->direct_queue);
-	sch->q.qlen = 0;
-	sch->qstats.backlog = 0;
 	memset(q->hlevel, 0, sizeof(q->hlevel));
 	memset(q->row_mask, 0, sizeof(q->row_mask));
 }
@@ -1104,9 +1102,7 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt,
 
 	err = qdisc_class_hash_init(&q->clhash);
 	if (err < 0)
-		goto err_free_direct_qdiscs;
-
-	qdisc_skb_head_init(&q->direct_queue);
+		return err;
 
 	if (tb[TCA_HTB_DIRECT_QLEN])
 		q->direct_qlen = nla_get_u32(tb[TCA_HTB_DIRECT_QLEN]);
@@ -1127,8 +1123,7 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt,
 		qdisc = qdisc_create_dflt(dev_queue, &pfifo_qdisc_ops,
 					  TC_H_MAKE(sch->handle, 0), extack);
 		if (!qdisc) {
-			err = -ENOMEM;
-			goto err_free_qdiscs;
+			return -ENOMEM;
 		}
 
 		htb_set_lockdep_class_child(qdisc);
@@ -1146,7 +1141,7 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt,
 	};
 	err = htb_offload(dev, &offload_opt);
 	if (err)
-		goto err_free_qdiscs;
+		return err;
 
 	/* Defer this assignment, so that htb_destroy skips offload-related
 	 * parts (especially calling ndo_setup_tc) on errors.
@@ -1154,22 +1149,6 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt,
 	q->offload = true;
 
 	return 0;
-
-err_free_qdiscs:
-	for (ntx = 0; ntx < q->num_direct_qdiscs && q->direct_qdiscs[ntx];
-	     ntx++)
-		qdisc_put(q->direct_qdiscs[ntx]);
-
-	qdisc_class_hash_destroy(&q->clhash);
-	/* Prevent use-after-free and double-free when htb_destroy gets called.
-	 */
-	q->clhash.hash = NULL;
-	q->clhash.hashsize = 0;
-
-err_free_direct_qdiscs:
-	kfree(q->direct_qdiscs);
-	q->direct_qdiscs = NULL;
-	return err;
 }
 
 static void htb_attach_offload(struct Qdisc *sch)
@@ -1692,13 +1671,12 @@ static void htb_destroy(struct Qdisc *sch)
 	qdisc_class_hash_destroy(&q->clhash);
 	__qdisc_reset_queue(&q->direct_queue);
 
-	if (!q->offload)
-		return;
-
-	offload_opt = (struct tc_htb_qopt_offload) {
-		.command = TC_HTB_DESTROY,
-	};
-	htb_offload(dev, &offload_opt);
+	if (q->offload) {
+		offload_opt = (struct tc_htb_qopt_offload) {
+			.command = TC_HTB_DESTROY,
+		};
+		htb_offload(dev, &offload_opt);
+	}
 
 	if (!q->direct_qdiscs)
 		return;
@@ -2141,15 +2119,8 @@ static void htb_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 
 	for (i = 0; i < q->clhash.hashsize; i++) {
 		hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) {
-			if (arg->count < arg->skip) {
-				arg->count++;
-				continue;
-			}
-			if (arg->fn(sch, (unsigned long)cl, arg) < 0) {
-				arg->stop = 1;
+			if (!tc_qdisc_stats_dump(sch, (unsigned long)cl, arg))
 				return;
-			}
-			arg->count++;
 		}
 	}
 }
diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c
index 83d2e54bf303..d0bc660d7401 100644
--- a/net/sched/sch_mq.c
+++ b/net/sched/sch_mq.c
@@ -247,11 +247,8 @@ static void mq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 
 	arg->count = arg->skip;
 	for (ntx = arg->skip; ntx < dev->num_tx_queues; ntx++) {
-		if (arg->fn(sch, ntx + 1, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_qdisc_stats_dump(sch, ntx + 1, arg))
 			break;
-		}
-		arg->count++;
 	}
 }
 
diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c
index b29f3453c6ea..4c68abaa289b 100644
--- a/net/sched/sch_mqprio.c
+++ b/net/sched/sch_mqprio.c
@@ -558,11 +558,8 @@ static void mqprio_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 	/* Walk hierarchy with a virtual class per tc */
 	arg->count = arg->skip;
 	for (ntx = arg->skip; ntx < netdev_get_num_tc(dev); ntx++) {
-		if (arg->fn(sch, ntx + TC_H_MIN_PRIORITY, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_qdisc_stats_dump(sch, ntx + TC_H_MIN_PRIORITY, arg))
 			return;
-		}
-		arg->count++;
 	}
 
 	/* Pad the values and skip over unused traffic classes */
diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c
index cd8ab90c4765..75c9c860182b 100644
--- a/net/sched/sch_multiq.c
+++ b/net/sched/sch_multiq.c
@@ -152,7 +152,6 @@ multiq_reset(struct Qdisc *sch)
 
 	for (band = 0; band < q->bands; band++)
 		qdisc_reset(q->queues[band]);
-	sch->q.qlen = 0;
 	q->curband = 0;
 }
 
@@ -354,15 +353,8 @@ static void multiq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 		return;
 
 	for (band = 0; band < q->bands; band++) {
-		if (arg->count < arg->skip) {
-			arg->count++;
-			continue;
-		}
-		if (arg->fn(sch, band + 1, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_qdisc_stats_dump(sch, band + 1, arg))
 			break;
-		}
-		arg->count++;
 	}
 }
 
diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c
index 5449ed114e40..18f4273a835b 100644
--- a/net/sched/sch_netem.c
+++ b/net/sched/sch_netem.c
@@ -961,9 +961,6 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt,
 	int old_loss_model = CLG_RANDOM;
 	int ret;
 
-	if (opt == NULL)
-		return -EINVAL;
-
 	qopt = nla_data(opt);
 	ret = parse_attr(tb, TCA_NETEM_MAX, opt, netem_policy, sizeof(*qopt));
 	if (ret < 0)
@@ -1254,12 +1251,8 @@ static unsigned long netem_find(struct Qdisc *sch, u32 classid)
 static void netem_walk(struct Qdisc *sch, struct qdisc_walker *walker)
 {
 	if (!walker->stop) {
-		if (walker->count >= walker->skip)
-			if (walker->fn(sch, 1, walker) < 0) {
-				walker->stop = 1;
-				return;
-			}
-		walker->count++;
+		if (!tc_qdisc_stats_dump(sch, 1, walker))
+			return;
 	}
 }
 
diff --git a/net/sched/sch_pie.c b/net/sched/sch_pie.c
index 5a457ff61acd..974038ba6c7b 100644
--- a/net/sched/sch_pie.c
+++ b/net/sched/sch_pie.c
@@ -143,9 +143,6 @@ static int pie_change(struct Qdisc *sch, struct nlattr *opt,
 	unsigned int qlen, dropped = 0;
 	int err;
 
-	if (!opt)
-		return -EINVAL;
-
 	err = nla_parse_nested_deprecated(tb, TCA_PIE_MAX, opt, pie_policy,
 					  NULL);
 	if (err < 0)
diff --git a/net/sched/sch_plug.c b/net/sched/sch_plug.c
index cbc2ebca4548..ea8c4a7174bb 100644
--- a/net/sched/sch_plug.c
+++ b/net/sched/sch_plug.c
@@ -161,9 +161,6 @@ static int plug_change(struct Qdisc *sch, struct nlattr *opt,
 	struct plug_sched_data *q = qdisc_priv(sch);
 	struct tc_plug_qopt *msg;
 
-	if (opt == NULL)
-		return -EINVAL;
-
 	msg = nla_data(opt);
 	if (nla_len(opt) < sizeof(*msg))
 		return -EINVAL;
diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c
index 3b8d7197c06b..fdc5ef52c3ee 100644
--- a/net/sched/sch_prio.c
+++ b/net/sched/sch_prio.c
@@ -135,8 +135,6 @@ prio_reset(struct Qdisc *sch)
 
 	for (prio = 0; prio < q->bands; prio++)
 		qdisc_reset(q->queues[prio]);
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
 }
 
 static int prio_offload(struct Qdisc *sch, struct tc_prio_qopt *qopt)
@@ -187,7 +185,7 @@ static int prio_tune(struct Qdisc *sch, struct nlattr *opt,
 		return -EINVAL;
 	qopt = nla_data(opt);
 
-	if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < 2)
+	if (qopt->bands > TCQ_PRIO_BANDS || qopt->bands < TCQ_MIN_PRIO_BANDS)
 		return -EINVAL;
 
 	for (i = 0; i <= TC_PRIO_MAX; i++) {
@@ -378,15 +376,8 @@ static void prio_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 		return;
 
 	for (prio = 0; prio < q->bands; prio++) {
-		if (arg->count < arg->skip) {
-			arg->count++;
-			continue;
-		}
-		if (arg->fn(sch, prio + 1, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_qdisc_stats_dump(sch, prio + 1, arg))
 			break;
-		}
-		arg->count++;
 	}
 }
 
diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c
index d4ce58c90f9f..cf5ebe43b3b4 100644
--- a/net/sched/sch_qfq.c
+++ b/net/sched/sch_qfq.c
@@ -659,15 +659,8 @@ static void qfq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 
 	for (i = 0; i < q->clhash.hashsize; i++) {
 		hlist_for_each_entry(cl, &q->clhash.hash[i], common.hnode) {
-			if (arg->count < arg->skip) {
-				arg->count++;
-				continue;
-			}
-			if (arg->fn(sch, (unsigned long)cl, arg) < 0) {
-				arg->stop = 1;
+			if (!tc_qdisc_stats_dump(sch, (unsigned long)cl, arg))
 				return;
-			}
-			arg->count++;
 		}
 	}
 }
@@ -1458,8 +1451,6 @@ static void qfq_reset_qdisc(struct Qdisc *sch)
 			qdisc_reset(cl->qdisc);
 		}
 	}
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
 }
 
 static void qfq_destroy_qdisc(struct Qdisc *sch)
diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c
index 40adf1f07a82..a5a401f93c1a 100644
--- a/net/sched/sch_red.c
+++ b/net/sched/sch_red.c
@@ -176,8 +176,6 @@ static void red_reset(struct Qdisc *sch)
 	struct red_sched_data *q = qdisc_priv(sch);
 
 	qdisc_reset(q->qdisc);
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
 	red_restart(&q->vars);
 }
 
@@ -370,9 +368,6 @@ static int red_change(struct Qdisc *sch, struct nlattr *opt,
 	struct nlattr *tb[TCA_RED_MAX + 1];
 	int err;
 
-	if (!opt)
-		return -EINVAL;
-
 	err = nla_parse_nested_deprecated(tb, TCA_RED_MAX, opt, red_policy,
 					  extack);
 	if (err < 0)
@@ -463,7 +458,6 @@ static int red_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
 	}
 	st.early = q->stats.prob_drop + q->stats.forced_drop;
 	st.pdrop = q->stats.pdrop;
-	st.other = q->stats.other;
 	st.marked = q->stats.prob_mark + q->stats.forced_mark;
 
 	return gnet_stats_copy_app(d, &st, sizeof(st));
@@ -522,12 +516,7 @@ static unsigned long red_find(struct Qdisc *sch, u32 classid)
 static void red_walk(struct Qdisc *sch, struct qdisc_walker *walker)
 {
 	if (!walker->stop) {
-		if (walker->count >= walker->skip)
-			if (walker->fn(sch, 1, walker) < 0) {
-				walker->stop = 1;
-				return;
-			}
-		walker->count++;
+		tc_qdisc_stats_dump(sch, 1, walker);
 	}
 }
 
diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c
index 2829455211f8..e2389fa3cff8 100644
--- a/net/sched/sch_sfb.c
+++ b/net/sched/sch_sfb.c
@@ -456,8 +456,6 @@ static void sfb_reset(struct Qdisc *sch)
 	struct sfb_sched_data *q = qdisc_priv(sch);
 
 	qdisc_reset(q->qdisc);
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
 	q->slot = 0;
 	q->double_buffering = false;
 	sfb_zero_all_buckets(q);
@@ -661,12 +659,7 @@ static int sfb_delete(struct Qdisc *sch, unsigned long cl,
 static void sfb_walk(struct Qdisc *sch, struct qdisc_walker *walker)
 {
 	if (!walker->stop) {
-		if (walker->count >= walker->skip)
-			if (walker->fn(sch, 1, walker) < 0) {
-				walker->stop = 1;
-				return;
-			}
-		walker->count++;
+		tc_qdisc_stats_dump(sch, 1, walker);
 	}
 }
 
diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c
index f8e569f79f13..abd436307d6a 100644
--- a/net/sched/sch_sfq.c
+++ b/net/sched/sch_sfq.c
@@ -888,16 +888,12 @@ static void sfq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 		return;
 
 	for (i = 0; i < q->divisor; i++) {
-		if (q->ht[i] == SFQ_EMPTY_SLOT ||
-		    arg->count < arg->skip) {
+		if (q->ht[i] == SFQ_EMPTY_SLOT) {
 			arg->count++;
 			continue;
 		}
-		if (arg->fn(sch, i + 1, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_qdisc_stats_dump(sch, i + 1, arg))
 			break;
-		}
-		arg->count++;
 	}
 }
 
diff --git a/net/sched/sch_skbprio.c b/net/sched/sch_skbprio.c
index 7a5e4c454715..5df2dacb7b1a 100644
--- a/net/sched/sch_skbprio.c
+++ b/net/sched/sch_skbprio.c
@@ -213,9 +213,6 @@ static void skbprio_reset(struct Qdisc *sch)
 	struct skbprio_sched_data *q = qdisc_priv(sch);
 	int prio;
 
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
-
 	for (prio = 0; prio < SKBPRIO_MAX_PRIORITY; prio++)
 		__skb_queue_purge(&q->qdiscs[prio]);
 
@@ -268,15 +265,8 @@ static void skbprio_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 		return;
 
 	for (i = 0; i < SKBPRIO_MAX_PRIORITY; i++) {
-		if (arg->count < arg->skip) {
-			arg->count++;
-			continue;
-		}
-		if (arg->fn(sch, i + 1, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_qdisc_stats_dump(sch, i + 1, arg))
 			break;
-		}
-		arg->count++;
 	}
 }
 
diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c
index 86675a79da1e..435d866fcfa0 100644
--- a/net/sched/sch_taprio.c
+++ b/net/sched/sch_taprio.c
@@ -27,7 +27,6 @@
 #include <net/tcp.h>
 
 static LIST_HEAD(taprio_list);
-static DEFINE_SPINLOCK(taprio_list_lock);
 
 #define TAPRIO_ALL_GATES_OPEN -1
 
@@ -79,8 +78,8 @@ struct taprio_sched {
 	struct sched_gate_list __rcu *admin_sched;
 	struct hrtimer advance_timer;
 	struct list_head taprio_list;
-	struct sk_buff *(*dequeue)(struct Qdisc *sch);
-	struct sk_buff *(*peek)(struct Qdisc *sch);
+	u32 max_frm_len[TC_MAX_QUEUE]; /* for the fast path */
+	u32 max_sdu[TC_MAX_QUEUE]; /* for dump and offloading */
 	u32 txtime_delay;
 };
 
@@ -418,6 +417,9 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch,
 			      struct Qdisc *child, struct sk_buff **to_free)
 {
 	struct taprio_sched *q = qdisc_priv(sch);
+	struct net_device *dev = qdisc_dev(sch);
+	int prio = skb->priority;
+	u8 tc;
 
 	/* sk_flags are only safe to use on full sockets. */
 	if (skb->sk && sk_fullsock(skb->sk) && sock_flag(skb->sk, SOCK_TXTIME)) {
@@ -429,12 +431,20 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch,
 			return qdisc_drop(skb, sch, to_free);
 	}
 
+	/* Devices with full offload are expected to honor this in hardware */
+	tc = netdev_get_prio_tc_map(dev, prio);
+	if (skb->len > q->max_frm_len[tc])
+		return qdisc_drop(skb, sch, to_free);
+
 	qdisc_qstats_backlog_inc(sch, skb);
 	sch->q.qlen++;
 
 	return qdisc_enqueue(skb, child, to_free);
 }
 
+/* Will not be called in the full offload case, since the TX queues are
+ * attached to the Qdisc created using qdisc_create_dflt()
+ */
 static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 			  struct sk_buff **to_free)
 {
@@ -442,11 +452,6 @@ static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	struct Qdisc *child;
 	int queue;
 
-	if (unlikely(FULL_OFFLOAD_IS_ENABLED(q->flags))) {
-		WARN_ONCE(1, "Trying to enqueue skb into the root of a taprio qdisc configured with full offload\n");
-		return qdisc_drop(skb, sch, to_free);
-	}
-
 	queue = skb_get_queue_mapping(skb);
 
 	child = q->qdiscs[queue];
@@ -455,10 +460,10 @@ static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 
 	/* Large packets might not be transmitted when the transmission duration
 	 * exceeds any configured interval. Therefore, segment the skb into
-	 * smaller chunks. Skip it for the full offload case, as the driver
-	 * and/or the hardware is expected to handle this.
+	 * smaller chunks. Drivers with full offload are expected to handle
+	 * this in hardware.
 	 */
-	if (skb_is_gso(skb) && !FULL_OFFLOAD_IS_ENABLED(q->flags)) {
+	if (skb_is_gso(skb)) {
 		unsigned int slen = 0, numsegs = 0, len = qdisc_pkt_len(skb);
 		netdev_features_t features = netif_skb_features(skb);
 		struct sk_buff *segs, *nskb;
@@ -492,7 +497,10 @@ static int taprio_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	return taprio_enqueue_one(skb, sch, child, to_free);
 }
 
-static struct sk_buff *taprio_peek_soft(struct Qdisc *sch)
+/* Will not be called in the full offload case, since the TX queues are
+ * attached to the Qdisc created using qdisc_create_dflt()
+ */
+static struct sk_buff *taprio_peek(struct Qdisc *sch)
 {
 	struct taprio_sched *q = qdisc_priv(sch);
 	struct net_device *dev = qdisc_dev(sch);
@@ -536,20 +544,6 @@ static struct sk_buff *taprio_peek_soft(struct Qdisc *sch)
 	return NULL;
 }
 
-static struct sk_buff *taprio_peek_offload(struct Qdisc *sch)
-{
-	WARN_ONCE(1, "Trying to peek into the root of a taprio qdisc configured with full offload\n");
-
-	return NULL;
-}
-
-static struct sk_buff *taprio_peek(struct Qdisc *sch)
-{
-	struct taprio_sched *q = qdisc_priv(sch);
-
-	return q->peek(sch);
-}
-
 static void taprio_set_budget(struct taprio_sched *q, struct sched_entry *entry)
 {
 	atomic_set(&entry->budget,
@@ -557,7 +551,10 @@ static void taprio_set_budget(struct taprio_sched *q, struct sched_entry *entry)
 			     atomic64_read(&q->picos_per_byte)));
 }
 
-static struct sk_buff *taprio_dequeue_soft(struct Qdisc *sch)
+/* Will not be called in the full offload case, since the TX queues are
+ * attached to the Qdisc created using qdisc_create_dflt()
+ */
+static struct sk_buff *taprio_dequeue(struct Qdisc *sch)
 {
 	struct taprio_sched *q = qdisc_priv(sch);
 	struct net_device *dev = qdisc_dev(sch);
@@ -645,20 +642,6 @@ done:
 	return skb;
 }
 
-static struct sk_buff *taprio_dequeue_offload(struct Qdisc *sch)
-{
-	WARN_ONCE(1, "Trying to dequeue from the root of a taprio qdisc configured with full offload\n");
-
-	return NULL;
-}
-
-static struct sk_buff *taprio_dequeue(struct Qdisc *sch)
-{
-	struct taprio_sched *q = qdisc_priv(sch);
-
-	return q->dequeue(sch);
-}
-
 static bool should_restart_cycle(const struct sched_gate_list *oper,
 				 const struct sched_entry *entry)
 {
@@ -781,6 +764,11 @@ static const struct nla_policy entry_policy[TCA_TAPRIO_SCHED_ENTRY_MAX + 1] = {
 	[TCA_TAPRIO_SCHED_ENTRY_INTERVAL]  = { .type = NLA_U32 },
 };
 
+static const struct nla_policy taprio_tc_policy[TCA_TAPRIO_TC_ENTRY_MAX + 1] = {
+	[TCA_TAPRIO_TC_ENTRY_INDEX]	   = { .type = NLA_U32 },
+	[TCA_TAPRIO_TC_ENTRY_MAX_SDU]	   = { .type = NLA_U32 },
+};
+
 static const struct nla_policy taprio_policy[TCA_TAPRIO_ATTR_MAX + 1] = {
 	[TCA_TAPRIO_ATTR_PRIOMAP]	       = {
 		.len = sizeof(struct tc_mqprio_qopt)
@@ -793,6 +781,7 @@ static const struct nla_policy taprio_policy[TCA_TAPRIO_ATTR_MAX + 1] = {
 	[TCA_TAPRIO_ATTR_SCHED_CYCLE_TIME_EXTENSION] = { .type = NLA_S64 },
 	[TCA_TAPRIO_ATTR_FLAGS]                      = { .type = NLA_U32 },
 	[TCA_TAPRIO_ATTR_TXTIME_DELAY]		     = { .type = NLA_U32 },
+	[TCA_TAPRIO_ATTR_TC_ENTRY]		     = { .type = NLA_NESTED },
 };
 
 static int fill_sched_entry(struct taprio_sched *q, struct nlattr **tb,
@@ -1099,27 +1088,20 @@ static int taprio_dev_notifier(struct notifier_block *nb, unsigned long event,
 			       void *ptr)
 {
 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
-	struct net_device *qdev;
 	struct taprio_sched *q;
-	bool found = false;
 
 	ASSERT_RTNL();
 
 	if (event != NETDEV_UP && event != NETDEV_CHANGE)
 		return NOTIFY_DONE;
 
-	spin_lock(&taprio_list_lock);
 	list_for_each_entry(q, &taprio_list, taprio_list) {
-		qdev = qdisc_dev(q->root);
-		if (qdev == dev) {
-			found = true;
-			break;
-		}
-	}
-	spin_unlock(&taprio_list_lock);
+		if (dev != qdisc_dev(q->root))
+			continue;
 
-	if (found)
 		taprio_set_picos_per_byte(dev, q);
+		break;
+	}
 
 	return NOTIFY_DONE;
 }
@@ -1194,16 +1176,10 @@ static void taprio_offload_config_changed(struct taprio_sched *q)
 {
 	struct sched_gate_list *oper, *admin;
 
-	spin_lock(&q->current_entry_lock);
-
-	oper = rcu_dereference_protected(q->oper_sched,
-					 lockdep_is_held(&q->current_entry_lock));
-	admin = rcu_dereference_protected(q->admin_sched,
-					  lockdep_is_held(&q->current_entry_lock));
+	oper = rtnl_dereference(q->oper_sched);
+	admin = rtnl_dereference(q->admin_sched);
 
 	switch_schedules(q, &admin, &oper);
-
-	spin_unlock(&q->current_entry_lock);
 }
 
 static u32 tc_map_to_queue_mask(struct net_device *dev, u32 tc_mask)
@@ -1256,7 +1232,8 @@ static int taprio_enable_offload(struct net_device *dev,
 {
 	const struct net_device_ops *ops = dev->netdev_ops;
 	struct tc_taprio_qopt_offload *offload;
-	int err = 0;
+	struct tc_taprio_caps caps;
+	int tc, err = 0;
 
 	if (!ops->ndo_setup_tc) {
 		NL_SET_ERR_MSG(extack,
@@ -1264,6 +1241,19 @@ static int taprio_enable_offload(struct net_device *dev,
 		return -EOPNOTSUPP;
 	}
 
+	qdisc_offload_query_caps(dev, TC_SETUP_QDISC_TAPRIO,
+				 &caps, sizeof(caps));
+
+	if (!caps.supports_queue_max_sdu) {
+		for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
+			if (q->max_sdu[tc]) {
+				NL_SET_ERR_MSG_MOD(extack,
+						   "Device does not handle queueMaxSDU");
+				return -EOPNOTSUPP;
+			}
+		}
+	}
+
 	offload = taprio_offload_alloc(sched->num_entries);
 	if (!offload) {
 		NL_SET_ERR_MSG(extack,
@@ -1273,6 +1263,9 @@ static int taprio_enable_offload(struct net_device *dev,
 	offload->enable = 1;
 	taprio_sched_to_offload(dev, sched, offload);
 
+	for (tc = 0; tc < TC_MAX_QUEUE; tc++)
+		offload->max_sdu[tc] = q->max_sdu[tc];
+
 	err = ops->ndo_setup_tc(dev, TC_SETUP_QDISC_TAPRIO, offload);
 	if (err < 0) {
 		NL_SET_ERR_MSG(extack,
@@ -1407,6 +1400,89 @@ out:
 	return err;
 }
 
+static int taprio_parse_tc_entry(struct Qdisc *sch,
+				 struct nlattr *opt,
+				 u32 max_sdu[TC_QOPT_MAX_QUEUE],
+				 unsigned long *seen_tcs,
+				 struct netlink_ext_ack *extack)
+{
+	struct nlattr *tb[TCA_TAPRIO_TC_ENTRY_MAX + 1] = { };
+	struct net_device *dev = qdisc_dev(sch);
+	u32 val = 0;
+	int err, tc;
+
+	err = nla_parse_nested(tb, TCA_TAPRIO_TC_ENTRY_MAX, opt,
+			       taprio_tc_policy, extack);
+	if (err < 0)
+		return err;
+
+	if (!tb[TCA_TAPRIO_TC_ENTRY_INDEX]) {
+		NL_SET_ERR_MSG_MOD(extack, "TC entry index missing");
+		return -EINVAL;
+	}
+
+	tc = nla_get_u32(tb[TCA_TAPRIO_TC_ENTRY_INDEX]);
+	if (tc >= TC_QOPT_MAX_QUEUE) {
+		NL_SET_ERR_MSG_MOD(extack, "TC entry index out of range");
+		return -ERANGE;
+	}
+
+	if (*seen_tcs & BIT(tc)) {
+		NL_SET_ERR_MSG_MOD(extack, "Duplicate TC entry");
+		return -EINVAL;
+	}
+
+	*seen_tcs |= BIT(tc);
+
+	if (tb[TCA_TAPRIO_TC_ENTRY_MAX_SDU])
+		val = nla_get_u32(tb[TCA_TAPRIO_TC_ENTRY_MAX_SDU]);
+
+	if (val > dev->max_mtu) {
+		NL_SET_ERR_MSG_MOD(extack, "TC max SDU exceeds device max MTU");
+		return -ERANGE;
+	}
+
+	max_sdu[tc] = val;
+
+	return 0;
+}
+
+static int taprio_parse_tc_entries(struct Qdisc *sch,
+				   struct nlattr *opt,
+				   struct netlink_ext_ack *extack)
+{
+	struct taprio_sched *q = qdisc_priv(sch);
+	struct net_device *dev = qdisc_dev(sch);
+	u32 max_sdu[TC_QOPT_MAX_QUEUE];
+	unsigned long seen_tcs = 0;
+	struct nlattr *n;
+	int tc, rem;
+	int err = 0;
+
+	for (tc = 0; tc < TC_QOPT_MAX_QUEUE; tc++)
+		max_sdu[tc] = q->max_sdu[tc];
+
+	nla_for_each_nested(n, opt, rem) {
+		if (nla_type(n) != TCA_TAPRIO_ATTR_TC_ENTRY)
+			continue;
+
+		err = taprio_parse_tc_entry(sch, n, max_sdu, &seen_tcs, extack);
+		if (err)
+			goto out;
+	}
+
+	for (tc = 0; tc < TC_QOPT_MAX_QUEUE; tc++) {
+		q->max_sdu[tc] = max_sdu[tc];
+		if (max_sdu[tc])
+			q->max_frm_len[tc] = max_sdu[tc] + dev->hard_header_len;
+		else
+			q->max_frm_len[tc] = U32_MAX; /* never oversized */
+	}
+
+out:
+	return err;
+}
+
 static int taprio_mqprio_cmp(const struct net_device *dev,
 			     const struct tc_mqprio_qopt *mqprio)
 {
@@ -1485,6 +1561,10 @@ static int taprio_change(struct Qdisc *sch, struct nlattr *opt,
 	if (err < 0)
 		return err;
 
+	err = taprio_parse_tc_entries(sch, opt, extack);
+	if (err)
+		return err;
+
 	new_admin = kzalloc(sizeof(*new_admin), GFP_KERNEL);
 	if (!new_admin) {
 		NL_SET_ERR_MSG(extack, "Not enough memory for a new schedule");
@@ -1492,10 +1572,8 @@ static int taprio_change(struct Qdisc *sch, struct nlattr *opt,
 	}
 	INIT_LIST_HEAD(&new_admin->entries);
 
-	rcu_read_lock();
-	oper = rcu_dereference(q->oper_sched);
-	admin = rcu_dereference(q->admin_sched);
-	rcu_read_unlock();
+	oper = rtnl_dereference(q->oper_sched);
+	admin = rtnl_dereference(q->admin_sched);
 
 	/* no changes - no new mqprio settings */
 	if (!taprio_mqprio_cmp(dev, mqprio))
@@ -1565,17 +1643,6 @@ static int taprio_change(struct Qdisc *sch, struct nlattr *opt,
 		q->advance_timer.function = advance_sched;
 	}
 
-	if (FULL_OFFLOAD_IS_ENABLED(q->flags)) {
-		q->dequeue = taprio_dequeue_offload;
-		q->peek = taprio_peek_offload;
-	} else {
-		/* Be sure to always keep the function pointers
-		 * in a consistent state.
-		 */
-		q->dequeue = taprio_dequeue_soft;
-		q->peek = taprio_peek_soft;
-	}
-
 	err = taprio_get_start_time(sch, new_admin, &start);
 	if (err < 0) {
 		NL_SET_ERR_MSG(extack, "Internal error: failed get start time");
@@ -1638,19 +1705,16 @@ static void taprio_reset(struct Qdisc *sch)
 			if (q->qdiscs[i])
 				qdisc_reset(q->qdiscs[i]);
 	}
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
 }
 
 static void taprio_destroy(struct Qdisc *sch)
 {
 	struct taprio_sched *q = qdisc_priv(sch);
 	struct net_device *dev = qdisc_dev(sch);
+	struct sched_gate_list *oper, *admin;
 	unsigned int i;
 
-	spin_lock(&taprio_list_lock);
 	list_del(&q->taprio_list);
-	spin_unlock(&taprio_list_lock);
 
 	/* Note that taprio_reset() might not be called if an error
 	 * happens in qdisc_create(), after taprio_init() has been called.
@@ -1669,11 +1733,14 @@ static void taprio_destroy(struct Qdisc *sch)
 
 	netdev_reset_tc(dev);
 
-	if (q->oper_sched)
-		call_rcu(&q->oper_sched->rcu, taprio_free_sched_cb);
+	oper = rtnl_dereference(q->oper_sched);
+	admin = rtnl_dereference(q->admin_sched);
+
+	if (oper)
+		call_rcu(&oper->rcu, taprio_free_sched_cb);
 
-	if (q->admin_sched)
-		call_rcu(&q->admin_sched->rcu, taprio_free_sched_cb);
+	if (admin)
+		call_rcu(&admin->rcu, taprio_free_sched_cb);
 }
 
 static int taprio_init(struct Qdisc *sch, struct nlattr *opt,
@@ -1688,9 +1755,6 @@ static int taprio_init(struct Qdisc *sch, struct nlattr *opt,
 	hrtimer_init(&q->advance_timer, CLOCK_TAI, HRTIMER_MODE_ABS);
 	q->advance_timer.function = advance_sched;
 
-	q->dequeue = taprio_dequeue_soft;
-	q->peek = taprio_peek_soft;
-
 	q->root = sch;
 
 	/* We only support static clockids. Use an invalid value as default
@@ -1699,15 +1763,17 @@ static int taprio_init(struct Qdisc *sch, struct nlattr *opt,
 	q->clockid = -1;
 	q->flags = TAPRIO_FLAGS_INVALID;
 
-	spin_lock(&taprio_list_lock);
 	list_add(&q->taprio_list, &taprio_list);
-	spin_unlock(&taprio_list_lock);
 
-	if (sch->parent != TC_H_ROOT)
+	if (sch->parent != TC_H_ROOT) {
+		NL_SET_ERR_MSG_MOD(extack, "Can only be attached as root qdisc");
 		return -EOPNOTSUPP;
+	}
 
-	if (!netif_is_multiqueue(dev))
+	if (!netif_is_multiqueue(dev)) {
+		NL_SET_ERR_MSG_MOD(extack, "Multi-queue device is required");
 		return -EOPNOTSUPP;
+	}
 
 	/* pre-allocate qdisc, attachment can't fail */
 	q->qdiscs = kcalloc(dev->num_tx_queues,
@@ -1879,6 +1945,33 @@ error_nest:
 	return -1;
 }
 
+static int taprio_dump_tc_entries(struct taprio_sched *q, struct sk_buff *skb)
+{
+	struct nlattr *n;
+	int tc;
+
+	for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
+		n = nla_nest_start(skb, TCA_TAPRIO_ATTR_TC_ENTRY);
+		if (!n)
+			return -EMSGSIZE;
+
+		if (nla_put_u32(skb, TCA_TAPRIO_TC_ENTRY_INDEX, tc))
+			goto nla_put_failure;
+
+		if (nla_put_u32(skb, TCA_TAPRIO_TC_ENTRY_MAX_SDU,
+				q->max_sdu[tc]))
+			goto nla_put_failure;
+
+		nla_nest_end(skb, n);
+	}
+
+	return 0;
+
+nla_put_failure:
+	nla_nest_cancel(skb, n);
+	return -EMSGSIZE;
+}
+
 static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb)
 {
 	struct taprio_sched *q = qdisc_priv(sch);
@@ -1888,9 +1981,8 @@ static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb)
 	struct nlattr *nest, *sched_nest;
 	unsigned int i;
 
-	rcu_read_lock();
-	oper = rcu_dereference(q->oper_sched);
-	admin = rcu_dereference(q->admin_sched);
+	oper = rtnl_dereference(q->oper_sched);
+	admin = rtnl_dereference(q->admin_sched);
 
 	opt.num_tc = netdev_get_num_tc(dev);
 	memcpy(opt.prio_tc_map, dev->prio_tc_map, sizeof(opt.prio_tc_map));
@@ -1918,6 +2010,9 @@ static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb)
 	    nla_put_u32(skb, TCA_TAPRIO_ATTR_TXTIME_DELAY, q->txtime_delay))
 		goto options_error;
 
+	if (taprio_dump_tc_entries(q, skb))
+		goto options_error;
+
 	if (oper && dump_schedule(skb, oper))
 		goto options_error;
 
@@ -1934,8 +2029,6 @@ static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb)
 	nla_nest_end(skb, sched_nest);
 
 done:
-	rcu_read_unlock();
-
 	return nla_nest_end(skb, nest);
 
 admin_error:
@@ -1945,7 +2038,6 @@ options_error:
 	nla_nest_cancel(skb, nest);
 
 start_error:
-	rcu_read_unlock();
 	return -ENOSPC;
 }
 
@@ -2006,11 +2098,8 @@ static void taprio_walk(struct Qdisc *sch, struct qdisc_walker *arg)
 
 	arg->count = arg->skip;
 	for (ntx = arg->skip; ntx < dev->num_tx_queues; ntx++) {
-		if (arg->fn(sch, ntx + 1, arg) < 0) {
-			arg->stop = 1;
+		if (!tc_qdisc_stats_dump(sch, ntx + 1, arg))
 			break;
-		}
-		arg->count++;
 	}
 }
 
diff --git a/net/sched/sch_tbf.c b/net/sched/sch_tbf.c
index 36079fdde2cb..277ad11f4d61 100644
--- a/net/sched/sch_tbf.c
+++ b/net/sched/sch_tbf.c
@@ -330,8 +330,6 @@ static void tbf_reset(struct Qdisc *sch)
 	struct tbf_sched_data *q = qdisc_priv(sch);
 
 	qdisc_reset(q->qdisc);
-	sch->qstats.backlog = 0;
-	sch->q.qlen = 0;
 	q->t_c = ktime_get_ns();
 	q->tokens = q->buffer;
 	q->ptokens = q->mtu;
@@ -582,12 +580,7 @@ static unsigned long tbf_find(struct Qdisc *sch, u32 classid)
 static void tbf_walk(struct Qdisc *sch, struct qdisc_walker *walker)
 {
 	if (!walker->stop) {
-		if (walker->count >= walker->skip)
-			if (walker->fn(sch, 1, walker) < 0) {
-				walker->stop = 1;
-				return;
-			}
-		walker->count++;
+		tc_qdisc_stats_dump(sch, 1, walker);
 	}
 }
 
diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c
index 6af6b95bdb67..16f9238aa51d 100644
--- a/net/sched/sch_teql.c
+++ b/net/sched/sch_teql.c
@@ -124,7 +124,6 @@ teql_reset(struct Qdisc *sch)
 	struct teql_sched_data *dat = qdisc_priv(sch);
 
 	skb_queue_purge(&dat->q);
-	sch->q.qlen = 0;
 }
 
 static void
@@ -492,7 +491,7 @@ static int __init teql_init(void)
 
 		master = netdev_priv(dev);
 
-		strlcpy(master->qops.id, dev->name, IFNAMSIZ);
+		strscpy(master->qops.id, dev->name, IFNAMSIZ);
 		err = register_qdisc(&master->qops);
 
 		if (err) {
diff --git a/net/sctp/auth.c b/net/sctp/auth.c
index db6b7373d16c..34964145514e 100644
--- a/net/sctp/auth.c
+++ b/net/sctp/auth.c
@@ -863,12 +863,17 @@ int sctp_auth_set_key(struct sctp_endpoint *ep,
 	}
 
 	list_del_init(&shkey->key_list);
-	sctp_auth_shkey_release(shkey);
 	list_add(&cur_key->key_list, sh_keys);
 
-	if (asoc && asoc->active_key_id == auth_key->sca_keynumber)
-		sctp_auth_asoc_init_active_key(asoc, GFP_KERNEL);
+	if (asoc && asoc->active_key_id == auth_key->sca_keynumber &&
+	    sctp_auth_asoc_init_active_key(asoc, GFP_KERNEL)) {
+		list_del_init(&cur_key->key_list);
+		sctp_auth_shkey_release(cur_key);
+		list_add(&shkey->key_list, sh_keys);
+		return -ENOMEM;
+	}
 
+	sctp_auth_shkey_release(shkey);
 	return 0;
 }
 
@@ -902,8 +907,13 @@ int sctp_auth_set_active_key(struct sctp_endpoint *ep,
 		return -EINVAL;
 
 	if (asoc) {
+		__u16  active_key_id = asoc->active_key_id;
+
 		asoc->active_key_id = key_id;
-		sctp_auth_asoc_init_active_key(asoc, GFP_KERNEL);
+		if (sctp_auth_asoc_init_active_key(asoc, GFP_KERNEL)) {
+			asoc->active_key_id = active_key_id;
+			return -ENOMEM;
+		}
 	} else
 		ep->active_key_id = key_id;
 
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index 0939cc3b915a..3ccbf3c201cd 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -379,6 +379,8 @@ static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
 	sk->sk_state = SMC_INIT;
 	sk->sk_destruct = smc_destruct;
 	sk->sk_protocol = protocol;
+	WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(net->smc.sysctl_wmem));
+	WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(net->smc.sysctl_rmem));
 	smc = smc_sk(sk);
 	INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
 	INIT_WORK(&smc->connect_work, smc_connect_work);
@@ -427,6 +429,7 @@ static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
 		goto out_rel;
 
 	smc->clcsock->sk->sk_reuse = sk->sk_reuse;
+	smc->clcsock->sk->sk_reuseport = sk->sk_reuseport;
 	rc = kernel_bind(smc->clcsock, uaddr, addr_len);
 
 out_rel:
@@ -3253,9 +3256,6 @@ static int __smc_create(struct net *net, struct socket *sock, int protocol,
 		smc->clcsock = clcsock;
 	}
 
-	smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
-	smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
-
 out:
 	return rc;
 }
diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index df89c2e08cbf..e6ee797640b4 100644
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -2310,10 +2310,10 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb)
 
 	if (is_rmb)
 		/* use socket recv buffer size (w/o overhead) as start value */
-		sk_buf_size = smc->sk.sk_rcvbuf / 2;
+		sk_buf_size = smc->sk.sk_rcvbuf;
 	else
 		/* use socket send buffer size (w/o overhead) as start value */
-		sk_buf_size = smc->sk.sk_sndbuf / 2;
+		sk_buf_size = smc->sk.sk_sndbuf;
 
 	for (bufsize_short = smc_compress_bufsize(sk_buf_size, is_smcd, is_rmb);
 	     bufsize_short >= 0; bufsize_short--) {
@@ -2372,7 +2372,7 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb)
 	if (is_rmb) {
 		conn->rmb_desc = buf_desc;
 		conn->rmbe_size_short = bufsize_short;
-		smc->sk.sk_rcvbuf = bufsize * 2;
+		smc->sk.sk_rcvbuf = bufsize;
 		atomic_set(&conn->bytes_to_rcv, 0);
 		conn->rmbe_update_limit =
 			smc_rmb_wnd_update_limit(buf_desc->len);
@@ -2380,7 +2380,7 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb)
 			smc_ism_set_conn(conn); /* map RMB/smcd_dev to conn */
 	} else {
 		conn->sndbuf_desc = buf_desc;
-		smc->sk.sk_sndbuf = bufsize * 2;
+		smc->sk.sk_sndbuf = bufsize;
 		atomic_set(&conn->sndbuf_space, bufsize);
 	}
 	return 0;
diff --git a/net/smc/smc_llc.c b/net/smc/smc_llc.c
index 175026ae33ae..524649d0ab65 100644
--- a/net/smc/smc_llc.c
+++ b/net/smc/smc_llc.c
@@ -2127,7 +2127,7 @@ void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc)
 	init_waitqueue_head(&lgr->llc_flow_waiter);
 	init_waitqueue_head(&lgr->llc_msg_waiter);
 	mutex_init(&lgr->llc_conf_mutex);
-	lgr->llc_testlink_time = READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);
+	lgr->llc_testlink_time = READ_ONCE(net->smc.sysctl_smcr_testlink_time);
 }
 
 /* called after lgr was removed from lgr_list */
diff --git a/net/smc/smc_llc.h b/net/smc/smc_llc.h
index 4404e52b3346..7e7a3162c68b 100644
--- a/net/smc/smc_llc.h
+++ b/net/smc/smc_llc.h
@@ -19,6 +19,7 @@
 
 #define SMC_LLC_WAIT_FIRST_TIME		(5 * HZ)
 #define SMC_LLC_WAIT_TIME		(2 * HZ)
+#define SMC_LLC_TESTLINK_DEFAULT_TIME	(30 * HZ)
 
 enum smc_llc_reqresp {
 	SMC_LLC_REQ,
diff --git a/net/smc/smc_netlink.c b/net/smc/smc_netlink.c
index c5a62f6f52ba..621c46c70073 100644
--- a/net/smc/smc_netlink.c
+++ b/net/smc/smc_netlink.c
@@ -142,7 +142,8 @@ struct genl_family smc_gen_nl_family __ro_after_init = {
 	.netnsok =	true,
 	.module =	THIS_MODULE,
 	.ops =		smc_gen_nl_ops,
-	.n_ops =	ARRAY_SIZE(smc_gen_nl_ops)
+	.n_ops =	ARRAY_SIZE(smc_gen_nl_ops),
+	.resv_start_op = SMC_NETLINK_DISABLE_HS_LIMITATION + 1,
 };
 
 int __init smc_nl_init(void)
diff --git a/net/smc/smc_pnet.c b/net/smc/smc_pnet.c
index 4c3bf6db7038..25fb2fd186e2 100644
--- a/net/smc/smc_pnet.c
+++ b/net/smc/smc_pnet.c
@@ -715,7 +715,8 @@ static struct genl_family smc_pnet_nl_family __ro_after_init = {
 	.netnsok = true,
 	.module = THIS_MODULE,
 	.ops = smc_pnet_ops,
-	.n_ops =  ARRAY_SIZE(smc_pnet_ops)
+	.n_ops =  ARRAY_SIZE(smc_pnet_ops),
+	.resv_start_op = SMC_PNETID_FLUSH + 1,
 };
 
 bool smc_pnet_is_ndev_pnetid(struct net *net, u8 *pnetid)
diff --git a/net/smc/smc_sysctl.c b/net/smc/smc_sysctl.c
index 0613868fdb97..b6f79fabb9d3 100644
--- a/net/smc/smc_sysctl.c
+++ b/net/smc/smc_sysctl.c
@@ -16,8 +16,12 @@
 
 #include "smc.h"
 #include "smc_core.h"
+#include "smc_llc.h"
 #include "smc_sysctl.h"
 
+static int min_sndbuf = SMC_BUF_MIN_SIZE;
+static int min_rcvbuf = SMC_BUF_MIN_SIZE;
+
 static struct ctl_table smc_table[] = {
 	{
 		.procname       = "autocorking_size",
@@ -35,6 +39,29 @@ static struct ctl_table smc_table[] = {
 		.extra1		= SYSCTL_ZERO,
 		.extra2		= SYSCTL_TWO,
 	},
+	{
+		.procname	= "smcr_testlink_time",
+		.data		= &init_net.smc.sysctl_smcr_testlink_time,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_jiffies,
+	},
+	{
+		.procname	= "wmem",
+		.data		= &init_net.smc.sysctl_wmem,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_sndbuf,
+	},
+	{
+		.procname	= "rmem",
+		.data		= &init_net.smc.sysctl_rmem,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &min_rcvbuf,
+	},
 	{  }
 };
 
@@ -60,6 +87,9 @@ int __net_init smc_sysctl_net_init(struct net *net)
 
 	net->smc.sysctl_autocorking_size = SMC_AUTOCORKING_DEFAULT_SIZE;
 	net->smc.sysctl_smcr_buf_type = SMCR_PHYS_CONT_BUFS;
+	net->smc.sysctl_smcr_testlink_time = SMC_LLC_TESTLINK_DEFAULT_TIME;
+	WRITE_ONCE(net->smc.sysctl_wmem, READ_ONCE(net->ipv4.sysctl_tcp_wmem[1]));
+	WRITE_ONCE(net->smc.sysctl_rmem, READ_ONCE(net->ipv4.sysctl_tcp_rmem[1]));
 
 	return 0;
 
diff --git a/net/socket.c b/net/socket.c
index 7378375d3a5b..00da9ce3dba0 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -355,7 +355,7 @@ static const struct super_operations sockfs_ops = {
  */
 static char *sockfs_dname(struct dentry *dentry, char *buffer, int buflen)
 {
-	return dynamic_dname(dentry, buffer, buflen, "socket:[%lu]",
+	return dynamic_dname(buffer, buflen, "socket:[%lu]",
 				d_inode(dentry)->i_ino);
 }
 
diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c
index 7c9a0d0b1230..149171774bc6 100644
--- a/net/sunrpc/svc.c
+++ b/net/sunrpc/svc.c
@@ -1205,7 +1205,7 @@ svc_generic_init_request(struct svc_rqst *rqstp,
 		goto err_bad_proc;
 
 	/* Initialize storage for argp and resp */
-	memset(rqstp->rq_argp, 0, procp->pc_argsize);
+	memset(rqstp->rq_argp, 0, procp->pc_argzero);
 	memset(rqstp->rq_resp, 0, procp->pc_ressize);
 
 	/* Bump per-procedure stats counter */
@@ -1434,8 +1434,7 @@ svc_process(struct svc_rqst *rqstp)
 {
 	struct kvec		*argv = &rqstp->rq_arg.head[0];
 	struct kvec		*resv = &rqstp->rq_res.head[0];
-	struct svc_serv		*serv = rqstp->rq_server;
-	u32			dir;
+	__be32			dir;
 
 #if IS_ENABLED(CONFIG_FAIL_SUNRPC)
 	if (!fail_sunrpc.ignore_server_disconnect &&
@@ -1450,7 +1449,7 @@ svc_process(struct svc_rqst *rqstp)
 	rqstp->rq_next_page = &rqstp->rq_respages[1];
 	resv->iov_base = page_address(rqstp->rq_respages[0]);
 	resv->iov_len = 0;
-	rqstp->rq_res.pages = rqstp->rq_respages + 1;
+	rqstp->rq_res.pages = rqstp->rq_next_page;
 	rqstp->rq_res.len = 0;
 	rqstp->rq_res.page_base = 0;
 	rqstp->rq_res.page_len = 0;
@@ -1458,18 +1457,17 @@ svc_process(struct svc_rqst *rqstp)
 	rqstp->rq_res.tail[0].iov_base = NULL;
 	rqstp->rq_res.tail[0].iov_len = 0;
 
-	dir  = svc_getnl(argv);
-	if (dir != 0) {
-		/* direction != CALL */
-		svc_printk(rqstp, "bad direction %d, dropping request\n", dir);
-		serv->sv_stats->rpcbadfmt++;
+	dir = svc_getu32(argv);
+	if (dir != rpc_call)
+		goto out_baddir;
+	if (!svc_process_common(rqstp, argv, resv))
 		goto out_drop;
-	}
-
-	/* Returns 1 for send, 0 for drop */
-	if (likely(svc_process_common(rqstp, argv, resv)))
-		return svc_send(rqstp);
+	return svc_send(rqstp);
 
+out_baddir:
+	svc_printk(rqstp, "bad direction 0x%08x, dropping request\n",
+		   be32_to_cpu(dir));
+	rqstp->rq_server->sv_stats->rpcbadfmt++;
 out_drop:
 	svc_drop(rqstp);
 	return 0;
@@ -1556,8 +1554,12 @@ out:
 EXPORT_SYMBOL_GPL(bc_svc_process);
 #endif /* CONFIG_SUNRPC_BACKCHANNEL */
 
-/*
- * Return (transport-specific) limit on the rpc payload.
+/**
+ * svc_max_payload - Return transport-specific limit on the RPC payload
+ * @rqstp: RPC transaction context
+ *
+ * Returns the maximum number of payload bytes the current transport
+ * allows.
  */
 u32 svc_max_payload(const struct svc_rqst *rqstp)
 {
diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c
index 482586c23fdd..336a7c7833e4 100644
--- a/net/sunrpc/xdr.c
+++ b/net/sunrpc/xdr.c
@@ -947,6 +947,28 @@ void xdr_init_encode(struct xdr_stream *xdr, struct xdr_buf *buf, __be32 *p,
 EXPORT_SYMBOL_GPL(xdr_init_encode);
 
 /**
+ * xdr_init_encode_pages - Initialize an xdr_stream for encoding into pages
+ * @xdr: pointer to xdr_stream struct
+ * @buf: pointer to XDR buffer into which to encode data
+ * @pages: list of pages to decode into
+ * @rqst: pointer to controlling rpc_rqst, for debugging
+ *
+ */
+void xdr_init_encode_pages(struct xdr_stream *xdr, struct xdr_buf *buf,
+			   struct page **pages, struct rpc_rqst *rqst)
+{
+	xdr_reset_scratch_buffer(xdr);
+
+	xdr->buf = buf;
+	xdr->page_ptr = pages;
+	xdr->iov = NULL;
+	xdr->p = page_address(*pages);
+	xdr->end = (void *)xdr->p + min_t(u32, buf->buflen, PAGE_SIZE);
+	xdr->rqst = rqst;
+}
+EXPORT_SYMBOL_GPL(xdr_init_encode_pages);
+
+/**
  * __xdr_commit_encode - Ensure all data is written to buffer
  * @xdr: pointer to xdr_stream
  *
@@ -1575,7 +1597,7 @@ EXPORT_SYMBOL_GPL(xdr_buf_from_iov);
  *
  * @buf and @subbuf may be pointers to the same struct xdr_buf.
  *
- * Returns -1 if base of length are out of bounds.
+ * Returns -1 if base or length are out of bounds.
  */
 int xdr_buf_subsegment(const struct xdr_buf *buf, struct xdr_buf *subbuf,
 		       unsigned int base, unsigned int len)
diff --git a/net/tipc/name_distr.c b/net/tipc/name_distr.c
index 8267b751a526..190b49c5cbc3 100644
--- a/net/tipc/name_distr.c
+++ b/net/tipc/name_distr.c
@@ -41,14 +41,6 @@
 
 int sysctl_tipc_named_timeout __read_mostly = 2000;
 
-struct distr_queue_item {
-	struct distr_item i;
-	u32 dtype;
-	u32 node;
-	unsigned long expires;
-	struct list_head next;
-};
-
 /**
  * publ_to_item - add publication info to a publication message
  * @p: publication info
diff --git a/net/tipc/netlink.c b/net/tipc/netlink.c
index c447cb5f879e..e8fd257c0e68 100644
--- a/net/tipc/netlink.c
+++ b/net/tipc/netlink.c
@@ -294,6 +294,7 @@ struct genl_family tipc_genl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.ops		= tipc_genl_v2_ops,
 	.n_ops		= ARRAY_SIZE(tipc_genl_v2_ops),
+	.resv_start_op	= TIPC_NL_ADDR_LEGACY_GET + 1,
 };
 
 int __init tipc_netlink_start(void)
diff --git a/net/tipc/netlink_compat.c b/net/tipc/netlink_compat.c
index 0749df80454d..fc68733673ba 100644
--- a/net/tipc/netlink_compat.c
+++ b/net/tipc/netlink_compat.c
@@ -1357,6 +1357,7 @@ static struct genl_family tipc_genl_compat_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.small_ops	= tipc_genl_compat_ops,
 	.n_small_ops	= ARRAY_SIZE(tipc_genl_compat_ops),
+	.resv_start_op	= TIPC_GENL_CMD + 1,
 };
 
 int __init tipc_netlink_compat_start(void)
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 0f983e5f7dde..a03d66046ca3 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -902,17 +902,28 @@ static void tls_device_core_ctrl_rx_resync(struct tls_context *tls_ctx,
 }
 
 static int
-tls_device_reencrypt(struct sock *sk, struct tls_sw_context_rx *sw_ctx)
+tls_device_reencrypt(struct sock *sk, struct tls_context *tls_ctx)
 {
+	struct tls_sw_context_rx *sw_ctx = tls_sw_ctx_rx(tls_ctx);
+	const struct tls_cipher_size_desc *cipher_sz;
 	int err, offset, copy, data_len, pos;
 	struct sk_buff *skb, *skb_iter;
 	struct scatterlist sg[1];
 	struct strp_msg *rxm;
 	char *orig_buf, *buf;
 
+	switch (tls_ctx->crypto_recv.info.cipher_type) {
+	case TLS_CIPHER_AES_GCM_128:
+	case TLS_CIPHER_AES_GCM_256:
+		break;
+	default:
+		return -EINVAL;
+	}
+	cipher_sz = &tls_cipher_size_desc[tls_ctx->crypto_recv.info.cipher_type];
+
 	rxm = strp_msg(tls_strp_msg(sw_ctx));
-	orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE +
-			   TLS_CIPHER_AES_GCM_128_IV_SIZE, sk->sk_allocation);
+	orig_buf = kmalloc(rxm->full_len + TLS_HEADER_SIZE + cipher_sz->iv,
+			   sk->sk_allocation);
 	if (!orig_buf)
 		return -ENOMEM;
 	buf = orig_buf;
@@ -927,10 +938,8 @@ tls_device_reencrypt(struct sock *sk, struct tls_sw_context_rx *sw_ctx)
 
 	sg_init_table(sg, 1);
 	sg_set_buf(&sg[0], buf,
-		   rxm->full_len + TLS_HEADER_SIZE +
-		   TLS_CIPHER_AES_GCM_128_IV_SIZE);
-	err = skb_copy_bits(skb, offset, buf,
-			    TLS_HEADER_SIZE + TLS_CIPHER_AES_GCM_128_IV_SIZE);
+		   rxm->full_len + TLS_HEADER_SIZE + cipher_sz->iv);
+	err = skb_copy_bits(skb, offset, buf, TLS_HEADER_SIZE + cipher_sz->iv);
 	if (err)
 		goto free_buf;
 
@@ -941,7 +950,7 @@ tls_device_reencrypt(struct sock *sk, struct tls_sw_context_rx *sw_ctx)
 	else
 		err = 0;
 
-	data_len = rxm->full_len - TLS_CIPHER_AES_GCM_128_TAG_SIZE;
+	data_len = rxm->full_len - cipher_sz->tag;
 
 	if (skb_pagelen(skb) > offset) {
 		copy = min_t(int, skb_pagelen(skb) - offset, data_len);
@@ -1024,7 +1033,7 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
 		 * likely have initial fragments decrypted, and final ones not
 		 * decrypted. We need to reencrypt that single SKB.
 		 */
-		return tls_device_reencrypt(sk, sw_ctx);
+		return tls_device_reencrypt(sk, tls_ctx);
 	}
 
 	/* Return immediately if the record is either entirely plaintext or
@@ -1041,7 +1050,7 @@ int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx)
 	}
 
 	ctx->resync_nh_reset = 1;
-	return tls_device_reencrypt(sk, sw_ctx);
+	return tls_device_reencrypt(sk, tls_ctx);
 }
 
 static void tls_device_attach(struct tls_context *ctx, struct sock *sk,
@@ -1062,9 +1071,9 @@ static void tls_device_attach(struct tls_context *ctx, struct sock *sk,
 
 int tls_set_device_offload(struct sock *sk, struct tls_context *ctx)
 {
-	u16 nonce_size, tag_size, iv_size, rec_seq_size, salt_size;
 	struct tls_context *tls_ctx = tls_get_ctx(sk);
 	struct tls_prot_info *prot = &tls_ctx->prot_info;
+	const struct tls_cipher_size_desc *cipher_sz;
 	struct tls_record_info *start_marker_record;
 	struct tls_offload_context_tx *offload_ctx;
 	struct tls_crypto_info *crypto_info;
@@ -1099,44 +1108,44 @@ int tls_set_device_offload(struct sock *sk, struct tls_context *ctx)
 
 	switch (crypto_info->cipher_type) {
 	case TLS_CIPHER_AES_GCM_128:
-		nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
-		tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
-		iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
 		iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
-		rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
-		salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
 		rec_seq =
 		 ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
 		break;
+	case TLS_CIPHER_AES_GCM_256:
+		iv = ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->iv;
+		rec_seq =
+		 ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->rec_seq;
+		break;
 	default:
 		rc = -EINVAL;
 		goto release_netdev;
 	}
+	cipher_sz = &tls_cipher_size_desc[crypto_info->cipher_type];
 
 	/* Sanity-check the rec_seq_size for stack allocations */
-	if (rec_seq_size > TLS_MAX_REC_SEQ_SIZE) {
+	if (cipher_sz->rec_seq > TLS_MAX_REC_SEQ_SIZE) {
 		rc = -EINVAL;
 		goto release_netdev;
 	}
 
 	prot->version = crypto_info->version;
 	prot->cipher_type = crypto_info->cipher_type;
-	prot->prepend_size = TLS_HEADER_SIZE + nonce_size;
-	prot->tag_size = tag_size;
+	prot->prepend_size = TLS_HEADER_SIZE + cipher_sz->iv;
+	prot->tag_size = cipher_sz->tag;
 	prot->overhead_size = prot->prepend_size + prot->tag_size;
-	prot->iv_size = iv_size;
-	prot->salt_size = salt_size;
-	ctx->tx.iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
-			     GFP_KERNEL);
+	prot->iv_size = cipher_sz->iv;
+	prot->salt_size = cipher_sz->salt;
+	ctx->tx.iv = kmalloc(cipher_sz->iv + cipher_sz->salt, GFP_KERNEL);
 	if (!ctx->tx.iv) {
 		rc = -ENOMEM;
 		goto release_netdev;
 	}
 
-	memcpy(ctx->tx.iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
+	memcpy(ctx->tx.iv + cipher_sz->salt, iv, cipher_sz->iv);
 
-	prot->rec_seq_size = rec_seq_size;
-	ctx->tx.rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL);
+	prot->rec_seq_size = cipher_sz->rec_seq;
+	ctx->tx.rec_seq = kmemdup(rec_seq, cipher_sz->rec_seq, GFP_KERNEL);
 	if (!ctx->tx.rec_seq) {
 		rc = -ENOMEM;
 		goto free_iv;
diff --git a/net/tls/tls_device_fallback.c b/net/tls/tls_device_fallback.c
index 7dfc8023e0f1..cdb391a8754b 100644
--- a/net/tls/tls_device_fallback.c
+++ b/net/tls/tls_device_fallback.c
@@ -54,13 +54,25 @@ static int tls_enc_record(struct aead_request *aead_req,
 			  struct scatter_walk *out, int *in_len,
 			  struct tls_prot_info *prot)
 {
-	unsigned char buf[TLS_HEADER_SIZE + TLS_CIPHER_AES_GCM_128_IV_SIZE];
+	unsigned char buf[TLS_HEADER_SIZE + MAX_IV_SIZE];
+	const struct tls_cipher_size_desc *cipher_sz;
 	struct scatterlist sg_in[3];
 	struct scatterlist sg_out[3];
+	unsigned int buf_size;
 	u16 len;
 	int rc;
 
-	len = min_t(int, *in_len, ARRAY_SIZE(buf));
+	switch (prot->cipher_type) {
+	case TLS_CIPHER_AES_GCM_128:
+	case TLS_CIPHER_AES_GCM_256:
+		break;
+	default:
+		return -EINVAL;
+	}
+	cipher_sz = &tls_cipher_size_desc[prot->cipher_type];
+
+	buf_size = TLS_HEADER_SIZE + cipher_sz->iv;
+	len = min_t(int, *in_len, buf_size);
 
 	scatterwalk_copychunks(buf, in, len, 0);
 	scatterwalk_copychunks(buf, out, len, 1);
@@ -73,13 +85,11 @@ static int tls_enc_record(struct aead_request *aead_req,
 	scatterwalk_pagedone(out, 1, 1);
 
 	len = buf[4] | (buf[3] << 8);
-	len -= TLS_CIPHER_AES_GCM_128_IV_SIZE;
+	len -= cipher_sz->iv;
 
-	tls_make_aad(aad, len - TLS_CIPHER_AES_GCM_128_TAG_SIZE,
-		(char *)&rcd_sn, buf[0], prot);
+	tls_make_aad(aad, len - cipher_sz->tag, (char *)&rcd_sn, buf[0], prot);
 
-	memcpy(iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, buf + TLS_HEADER_SIZE,
-	       TLS_CIPHER_AES_GCM_128_IV_SIZE);
+	memcpy(iv + cipher_sz->salt, buf + TLS_HEADER_SIZE, cipher_sz->iv);
 
 	sg_init_table(sg_in, ARRAY_SIZE(sg_in));
 	sg_init_table(sg_out, ARRAY_SIZE(sg_out));
@@ -90,7 +100,7 @@ static int tls_enc_record(struct aead_request *aead_req,
 
 	*in_len -= len;
 	if (*in_len < 0) {
-		*in_len += TLS_CIPHER_AES_GCM_128_TAG_SIZE;
+		*in_len += cipher_sz->tag;
 		/* the input buffer doesn't contain the entire record.
 		 * trim len accordingly. The resulting authentication tag
 		 * will contain garbage, but we don't care, so we won't
@@ -111,7 +121,7 @@ static int tls_enc_record(struct aead_request *aead_req,
 		scatterwalk_pagedone(out, 1, 1);
 	}
 
-	len -= TLS_CIPHER_AES_GCM_128_TAG_SIZE;
+	len -= cipher_sz->tag;
 	aead_request_set_crypt(aead_req, sg_in, sg_out, len, iv);
 
 	rc = crypto_aead_encrypt(aead_req);
@@ -299,11 +309,14 @@ static void fill_sg_out(struct scatterlist sg_out[3], void *buf,
 			int sync_size,
 			void *dummy_buf)
 {
+	const struct tls_cipher_size_desc *cipher_sz =
+		&tls_cipher_size_desc[tls_ctx->crypto_send.info.cipher_type];
+
 	sg_set_buf(&sg_out[0], dummy_buf, sync_size);
 	sg_set_buf(&sg_out[1], nskb->data + tcp_payload_offset, payload_len);
 	/* Add room for authentication tag produced by crypto */
 	dummy_buf += sync_size;
-	sg_set_buf(&sg_out[2], dummy_buf, TLS_CIPHER_AES_GCM_128_TAG_SIZE);
+	sg_set_buf(&sg_out[2], dummy_buf, cipher_sz->tag);
 }
 
 static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
@@ -315,7 +328,8 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
 	struct tls_offload_context_tx *ctx = tls_offload_ctx_tx(tls_ctx);
 	int tcp_payload_offset = skb_tcp_all_headers(skb);
 	int payload_len = skb->len - tcp_payload_offset;
-	void *buf, *iv, *aad, *dummy_buf;
+	const struct tls_cipher_size_desc *cipher_sz;
+	void *buf, *iv, *aad, *dummy_buf, *salt;
 	struct aead_request *aead_req;
 	struct sk_buff *nskb = NULL;
 	int buf_len;
@@ -324,20 +338,26 @@ static struct sk_buff *tls_enc_skb(struct tls_context *tls_ctx,
 	if (!aead_req)
 		return NULL;
 
-	buf_len = TLS_CIPHER_AES_GCM_128_SALT_SIZE +
-		  TLS_CIPHER_AES_GCM_128_IV_SIZE +
-		  TLS_AAD_SPACE_SIZE +
-		  sync_size +
-		  TLS_CIPHER_AES_GCM_128_TAG_SIZE;
+	switch (tls_ctx->crypto_send.info.cipher_type) {
+	case TLS_CIPHER_AES_GCM_128:
+		salt = tls_ctx->crypto_send.aes_gcm_128.salt;
+		break;
+	case TLS_CIPHER_AES_GCM_256:
+		salt = tls_ctx->crypto_send.aes_gcm_256.salt;
+		break;
+	default:
+		return NULL;
+	}
+	cipher_sz = &tls_cipher_size_desc[tls_ctx->crypto_send.info.cipher_type];
+	buf_len = cipher_sz->salt + cipher_sz->iv + TLS_AAD_SPACE_SIZE +
+		  sync_size + cipher_sz->tag;
 	buf = kmalloc(buf_len, GFP_ATOMIC);
 	if (!buf)
 		goto free_req;
 
 	iv = buf;
-	memcpy(iv, tls_ctx->crypto_send.aes_gcm_128.salt,
-	       TLS_CIPHER_AES_GCM_128_SALT_SIZE);
-	aad = buf + TLS_CIPHER_AES_GCM_128_SALT_SIZE +
-	      TLS_CIPHER_AES_GCM_128_IV_SIZE;
+	memcpy(iv, salt, cipher_sz->salt);
+	aad = buf + cipher_sz->salt + cipher_sz->iv;
 	dummy_buf = aad + TLS_AAD_SPACE_SIZE;
 
 	nskb = alloc_skb(skb_headroom(skb) + skb->len, GFP_ATOMIC);
@@ -451,6 +471,7 @@ int tls_sw_fallback_init(struct sock *sk,
 			 struct tls_offload_context_tx *offload_ctx,
 			 struct tls_crypto_info *crypto_info)
 {
+	const struct tls_cipher_size_desc *cipher_sz;
 	const u8 *key;
 	int rc;
 
@@ -463,15 +484,23 @@ int tls_sw_fallback_init(struct sock *sk,
 		goto err_out;
 	}
 
-	key = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->key;
+	switch (crypto_info->cipher_type) {
+	case TLS_CIPHER_AES_GCM_128:
+		key = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->key;
+		break;
+	case TLS_CIPHER_AES_GCM_256:
+		key = ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->key;
+		break;
+	default:
+		return -EINVAL;
+	}
+	cipher_sz = &tls_cipher_size_desc[crypto_info->cipher_type];
 
-	rc = crypto_aead_setkey(offload_ctx->aead_send, key,
-				TLS_CIPHER_AES_GCM_128_KEY_SIZE);
+	rc = crypto_aead_setkey(offload_ctx->aead_send, key, cipher_sz->key);
 	if (rc)
 		goto free_aead;
 
-	rc = crypto_aead_setauthsize(offload_ctx->aead_send,
-				     TLS_CIPHER_AES_GCM_128_TAG_SIZE);
+	rc = crypto_aead_setauthsize(offload_ctx->aead_send, cipher_sz->tag);
 	if (rc)
 		goto free_aead;
 
diff --git a/net/tls/tls_main.c b/net/tls/tls_main.c
index 08ddf9d837ae..3735cb00905d 100644
--- a/net/tls/tls_main.c
+++ b/net/tls/tls_main.c
@@ -58,6 +58,23 @@ enum {
 	TLS_NUM_PROTS,
 };
 
+#define CIPHER_SIZE_DESC(cipher) [cipher] = { \
+	.iv = cipher ## _IV_SIZE, \
+	.key = cipher ## _KEY_SIZE, \
+	.salt = cipher ## _SALT_SIZE, \
+	.tag = cipher ## _TAG_SIZE, \
+	.rec_seq = cipher ## _REC_SEQ_SIZE, \
+}
+
+const struct tls_cipher_size_desc tls_cipher_size_desc[] = {
+	CIPHER_SIZE_DESC(TLS_CIPHER_AES_GCM_128),
+	CIPHER_SIZE_DESC(TLS_CIPHER_AES_GCM_256),
+	CIPHER_SIZE_DESC(TLS_CIPHER_AES_CCM_128),
+	CIPHER_SIZE_DESC(TLS_CIPHER_CHACHA20_POLY1305),
+	CIPHER_SIZE_DESC(TLS_CIPHER_SM4_GCM),
+	CIPHER_SIZE_DESC(TLS_CIPHER_SM4_CCM),
+};
+
 static const struct proto *saved_tcpv6_prot;
 static DEFINE_MUTEX(tcpv6_prot_mutex);
 static const struct proto *saved_tcpv4_prot;
@@ -507,6 +524,54 @@ static int do_tls_getsockopt_conf(struct sock *sk, char __user *optval,
 			rc = -EFAULT;
 		break;
 	}
+	case TLS_CIPHER_ARIA_GCM_128: {
+		struct tls12_crypto_info_aria_gcm_128 *
+		  crypto_info_aria_gcm_128 =
+		  container_of(crypto_info,
+			       struct tls12_crypto_info_aria_gcm_128,
+			       info);
+
+		if (len != sizeof(*crypto_info_aria_gcm_128)) {
+			rc = -EINVAL;
+			goto out;
+		}
+		lock_sock(sk);
+		memcpy(crypto_info_aria_gcm_128->iv,
+		       cctx->iv + TLS_CIPHER_ARIA_GCM_128_SALT_SIZE,
+		       TLS_CIPHER_ARIA_GCM_128_IV_SIZE);
+		memcpy(crypto_info_aria_gcm_128->rec_seq, cctx->rec_seq,
+		       TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE);
+		release_sock(sk);
+		if (copy_to_user(optval,
+				 crypto_info_aria_gcm_128,
+				 sizeof(*crypto_info_aria_gcm_128)))
+			rc = -EFAULT;
+		break;
+	}
+	case TLS_CIPHER_ARIA_GCM_256: {
+		struct tls12_crypto_info_aria_gcm_256 *
+		  crypto_info_aria_gcm_256 =
+		  container_of(crypto_info,
+			       struct tls12_crypto_info_aria_gcm_256,
+			       info);
+
+		if (len != sizeof(*crypto_info_aria_gcm_256)) {
+			rc = -EINVAL;
+			goto out;
+		}
+		lock_sock(sk);
+		memcpy(crypto_info_aria_gcm_256->iv,
+		       cctx->iv + TLS_CIPHER_ARIA_GCM_256_SALT_SIZE,
+		       TLS_CIPHER_ARIA_GCM_256_IV_SIZE);
+		memcpy(crypto_info_aria_gcm_256->rec_seq, cctx->rec_seq,
+		       TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE);
+		release_sock(sk);
+		if (copy_to_user(optval,
+				 crypto_info_aria_gcm_256,
+				 sizeof(*crypto_info_aria_gcm_256)))
+			rc = -EFAULT;
+		break;
+	}
 	default:
 		rc = -EINVAL;
 	}
@@ -668,6 +733,20 @@ static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
 	case TLS_CIPHER_SM4_CCM:
 		optsize = sizeof(struct tls12_crypto_info_sm4_ccm);
 		break;
+	case TLS_CIPHER_ARIA_GCM_128:
+		if (crypto_info->version != TLS_1_2_VERSION) {
+			rc = -EINVAL;
+			goto err_crypto_info;
+		}
+		optsize = sizeof(struct tls12_crypto_info_aria_gcm_128);
+		break;
+	case TLS_CIPHER_ARIA_GCM_256:
+		if (crypto_info->version != TLS_1_2_VERSION) {
+			rc = -EINVAL;
+			goto err_crypto_info;
+		}
+		optsize = sizeof(struct tls12_crypto_info_aria_gcm_256);
+		break;
 	default:
 		rc = -EINVAL;
 		goto err_crypto_info;
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index fe27241cd13f..264cf367e265 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -2629,6 +2629,40 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
 		cipher_name = "ccm(sm4)";
 		break;
 	}
+	case TLS_CIPHER_ARIA_GCM_128: {
+		struct tls12_crypto_info_aria_gcm_128 *aria_gcm_128_info;
+
+		aria_gcm_128_info = (void *)crypto_info;
+		nonce_size = TLS_CIPHER_ARIA_GCM_128_IV_SIZE;
+		tag_size = TLS_CIPHER_ARIA_GCM_128_TAG_SIZE;
+		iv_size = TLS_CIPHER_ARIA_GCM_128_IV_SIZE;
+		iv = aria_gcm_128_info->iv;
+		rec_seq_size = TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE;
+		rec_seq = aria_gcm_128_info->rec_seq;
+		keysize = TLS_CIPHER_ARIA_GCM_128_KEY_SIZE;
+		key = aria_gcm_128_info->key;
+		salt = aria_gcm_128_info->salt;
+		salt_size = TLS_CIPHER_ARIA_GCM_128_SALT_SIZE;
+		cipher_name = "gcm(aria)";
+		break;
+	}
+	case TLS_CIPHER_ARIA_GCM_256: {
+		struct tls12_crypto_info_aria_gcm_256 *gcm_256_info;
+
+		gcm_256_info = (void *)crypto_info;
+		nonce_size = TLS_CIPHER_ARIA_GCM_256_IV_SIZE;
+		tag_size = TLS_CIPHER_ARIA_GCM_256_TAG_SIZE;
+		iv_size = TLS_CIPHER_ARIA_GCM_256_IV_SIZE;
+		iv = gcm_256_info->iv;
+		rec_seq_size = TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE;
+		rec_seq = gcm_256_info->rec_seq;
+		keysize = TLS_CIPHER_ARIA_GCM_256_KEY_SIZE;
+		key = gcm_256_info->key;
+		salt = gcm_256_info->salt;
+		salt_size = TLS_CIPHER_ARIA_GCM_256_SALT_SIZE;
+		cipher_name = "gcm(aria)";
+		break;
+	}
 	default:
 		rc = -EINVAL;
 		goto free_priv;
diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index bf338b782fc4..0f08c3177872 100644
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -569,12 +569,6 @@ static void unix_sock_destructor(struct sock *sk)
 
 	skb_queue_purge(&sk->sk_receive_queue);
 
-#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
-	if (u->oob_skb) {
-		kfree_skb(u->oob_skb);
-		u->oob_skb = NULL;
-	}
-#endif
 	DEBUG_NET_WARN_ON_ONCE(refcount_read(&sk->sk_wmem_alloc));
 	DEBUG_NET_WARN_ON_ONCE(!sk_unhashed(sk));
 	DEBUG_NET_WARN_ON_ONCE(sk->sk_socket);
@@ -620,6 +614,13 @@ static void unix_release_sock(struct sock *sk, int embrion)
 
 	unix_state_unlock(sk);
 
+#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
+	if (u->oob_skb) {
+		kfree_skb(u->oob_skb);
+		u->oob_skb = NULL;
+	}
+#endif
+
 	wake_up_interruptible_all(&u->peer_wait);
 
 	if (skpair != NULL) {
@@ -785,15 +786,45 @@ static int unix_set_peek_off(struct sock *sk, int val)
 }
 
 #ifdef CONFIG_PROC_FS
+static int unix_count_nr_fds(struct sock *sk)
+{
+	struct sk_buff *skb;
+	struct unix_sock *u;
+	int nr_fds = 0;
+
+	spin_lock(&sk->sk_receive_queue.lock);
+	skb = skb_peek(&sk->sk_receive_queue);
+	while (skb) {
+		u = unix_sk(skb->sk);
+		nr_fds += atomic_read(&u->scm_stat.nr_fds);
+		skb = skb_peek_next(skb, &sk->sk_receive_queue);
+	}
+	spin_unlock(&sk->sk_receive_queue.lock);
+
+	return nr_fds;
+}
+
 static void unix_show_fdinfo(struct seq_file *m, struct socket *sock)
 {
 	struct sock *sk = sock->sk;
 	struct unix_sock *u;
+	int nr_fds;
 
 	if (sk) {
-		u = unix_sk(sock->sk);
-		seq_printf(m, "scm_fds: %u\n",
-			   atomic_read(&u->scm_stat.nr_fds));
+		u = unix_sk(sk);
+		if (sock->type == SOCK_DGRAM) {
+			nr_fds = atomic_read(&u->scm_stat.nr_fds);
+			goto out_print;
+		}
+
+		unix_state_lock(sk);
+		if (sk->sk_state != TCP_LISTEN)
+			nr_fds = atomic_read(&u->scm_stat.nr_fds);
+		else
+			nr_fds = unix_count_nr_fds(sk);
+		unix_state_unlock(sk);
+out_print:
+		seq_printf(m, "scm_fds: %u\n", nr_fds);
 	}
 }
 #else
@@ -2506,32 +2537,18 @@ static int unix_dgram_recvmsg(struct socket *sock, struct msghdr *msg, size_t si
 
 static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
 {
-	int copied = 0;
-
-	while (1) {
-		struct unix_sock *u = unix_sk(sk);
-		struct sk_buff *skb;
-		int used, err;
-
-		mutex_lock(&u->iolock);
-		skb = skb_recv_datagram(sk, MSG_DONTWAIT, &err);
-		mutex_unlock(&u->iolock);
-		if (!skb)
-			return err;
+	struct unix_sock *u = unix_sk(sk);
+	struct sk_buff *skb;
+	int err, copied;
 
-		used = recv_actor(sk, skb);
-		if (used <= 0) {
-			if (!copied)
-				copied = used;
-			kfree_skb(skb);
-			break;
-		} else if (used <= skb->len) {
-			copied += used;
-		}
+	mutex_lock(&u->iolock);
+	skb = skb_recv_datagram(sk, MSG_DONTWAIT, &err);
+	mutex_unlock(&u->iolock);
+	if (!skb)
+		return err;
 
-		kfree_skb(skb);
-		break;
-	}
+	copied = recv_actor(sk, skb);
+	kfree_skb(skb);
 
 	return copied;
 }
diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c
index b4ee163154a6..ee418701cdee 100644
--- a/net/vmw_vsock/af_vsock.c
+++ b/net/vmw_vsock/af_vsock.c
@@ -882,6 +882,16 @@ s64 vsock_stream_has_space(struct vsock_sock *vsk)
 }
 EXPORT_SYMBOL_GPL(vsock_stream_has_space);
 
+void vsock_data_ready(struct sock *sk)
+{
+	struct vsock_sock *vsk = vsock_sk(sk);
+
+	if (vsock_stream_has_data(vsk) >= sk->sk_rcvlowat ||
+	    sock_flag(sk, SOCK_DONE))
+		sk->sk_data_ready(sk);
+}
+EXPORT_SYMBOL_GPL(vsock_data_ready);
+
 static int vsock_release(struct socket *sock)
 {
 	__vsock_release(sock->sk, 0);
@@ -1066,8 +1076,9 @@ static __poll_t vsock_poll(struct file *file, struct socket *sock,
 		if (transport && transport->stream_is_active(vsk) &&
 		    !(sk->sk_shutdown & RCV_SHUTDOWN)) {
 			bool data_ready_now = false;
+			int target = sock_rcvlowat(sk, 0, INT_MAX);
 			int ret = transport->notify_poll_in(
-					vsk, 1, &data_ready_now);
+					vsk, target, &data_ready_now);
 			if (ret < 0) {
 				mask |= EPOLLERR;
 			} else {
@@ -2137,6 +2148,25 @@ out:
 	return err;
 }
 
+static int vsock_set_rcvlowat(struct sock *sk, int val)
+{
+	const struct vsock_transport *transport;
+	struct vsock_sock *vsk;
+
+	vsk = vsock_sk(sk);
+
+	if (val > vsk->buffer_size)
+		return -EINVAL;
+
+	transport = vsk->transport;
+
+	if (transport && transport->set_rcvlowat)
+		return transport->set_rcvlowat(vsk, val);
+
+	WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
+	return 0;
+}
+
 static const struct proto_ops vsock_stream_ops = {
 	.family = PF_VSOCK,
 	.owner = THIS_MODULE,
@@ -2156,6 +2186,7 @@ static const struct proto_ops vsock_stream_ops = {
 	.recvmsg = vsock_connectible_recvmsg,
 	.mmap = sock_no_mmap,
 	.sendpage = sock_no_sendpage,
+	.set_rcvlowat = vsock_set_rcvlowat,
 };
 
 static const struct proto_ops vsock_seqpacket_ops = {
diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c
index fd98229e3db3..59c3e2697069 100644
--- a/net/vmw_vsock/hyperv_transport.c
+++ b/net/vmw_vsock/hyperv_transport.c
@@ -815,6 +815,12 @@ int hvs_notify_send_post_enqueue(struct vsock_sock *vsk, ssize_t written,
 	return 0;
 }
 
+static
+int hvs_set_rcvlowat(struct vsock_sock *vsk, int val)
+{
+	return -EOPNOTSUPP;
+}
+
 static struct vsock_transport hvs_transport = {
 	.module                   = THIS_MODULE,
 
@@ -850,6 +856,7 @@ static struct vsock_transport hvs_transport = {
 	.notify_send_pre_enqueue  = hvs_notify_send_pre_enqueue,
 	.notify_send_post_enqueue = hvs_notify_send_post_enqueue,
 
+	.set_rcvlowat             = hvs_set_rcvlowat
 };
 
 static bool hvs_check_transport(struct vsock_sock *vsk)
diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
index ec2c2afbf0d0..a9980e9b9304 100644
--- a/net/vmw_vsock/virtio_transport_common.c
+++ b/net/vmw_vsock/virtio_transport_common.c
@@ -634,10 +634,7 @@ virtio_transport_notify_poll_in(struct vsock_sock *vsk,
 				size_t target,
 				bool *data_ready_now)
 {
-	if (vsock_stream_has_data(vsk))
-		*data_ready_now = true;
-	else
-		*data_ready_now = false;
+	*data_ready_now = vsock_stream_has_data(vsk) >= target;
 
 	return 0;
 }
@@ -1084,7 +1081,7 @@ virtio_transport_recv_connected(struct sock *sk,
 	switch (le16_to_cpu(pkt->hdr.op)) {
 	case VIRTIO_VSOCK_OP_RW:
 		virtio_transport_recv_enqueue(vsk, pkt);
-		sk->sk_data_ready(sk);
+		vsock_data_ready(sk);
 		return err;
 	case VIRTIO_VSOCK_OP_CREDIT_REQUEST:
 		virtio_transport_send_credit_update(vsk);
@@ -1342,7 +1339,7 @@ EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt);
 
 void virtio_transport_free_pkt(struct virtio_vsock_pkt *pkt)
 {
-	kfree(pkt->buf);
+	kvfree(pkt->buf);
 	kfree(pkt);
 }
 EXPORT_SYMBOL_GPL(virtio_transport_free_pkt);
diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c
index b14f0ed7427b..842c94286d31 100644
--- a/net/vmw_vsock/vmci_transport.c
+++ b/net/vmw_vsock/vmci_transport.c
@@ -951,7 +951,7 @@ static int vmci_transport_recv_listen(struct sock *sk,
 	 * for ourself or any previous connection requests that we received.
 	 * If it's the latter, we try to find a socket in our list of pending
 	 * connections and, if we do, call the appropriate handler for the
-	 * state that that socket is in.  Otherwise we try to service the
+	 * state that socket is in.  Otherwise we try to service the
 	 * connection request.
 	 */
 	pending = vmci_transport_get_pending(sk, pkt);
diff --git a/net/vmw_vsock/vmci_transport_notify.c b/net/vmw_vsock/vmci_transport_notify.c
index d69fc4b595ad..7c3a7db134b2 100644
--- a/net/vmw_vsock/vmci_transport_notify.c
+++ b/net/vmw_vsock/vmci_transport_notify.c
@@ -307,7 +307,7 @@ vmci_transport_handle_wrote(struct sock *sk,
 	struct vsock_sock *vsk = vsock_sk(sk);
 	PKT_FIELD(vsk, sent_waiting_read) = false;
 #endif
-	sk->sk_data_ready(sk);
+	vsock_data_ready(sk);
 }
 
 static void vmci_transport_notify_pkt_socket_init(struct sock *sk)
@@ -340,12 +340,12 @@ vmci_transport_notify_pkt_poll_in(struct sock *sk,
 {
 	struct vsock_sock *vsk = vsock_sk(sk);
 
-	if (vsock_stream_has_data(vsk)) {
+	if (vsock_stream_has_data(vsk) >= target) {
 		*data_ready_now = true;
 	} else {
-		/* We can't read right now because there is nothing in the
-		 * queue. Ask for notifications when there is something to
-		 * read.
+		/* We can't read right now because there is not enough data
+		 * in the queue. Ask for notifications when there is something
+		 * to read.
 		 */
 		if (sk->sk_state == TCP_ESTABLISHED) {
 			if (!send_waiting_read(sk, 1))
diff --git a/net/vmw_vsock/vmci_transport_notify_qstate.c b/net/vmw_vsock/vmci_transport_notify_qstate.c
index 0f36d7c45db3..e96a88d850a8 100644
--- a/net/vmw_vsock/vmci_transport_notify_qstate.c
+++ b/net/vmw_vsock/vmci_transport_notify_qstate.c
@@ -84,7 +84,7 @@ vmci_transport_handle_wrote(struct sock *sk,
 			    bool bottom_half,
 			    struct sockaddr_vm *dst, struct sockaddr_vm *src)
 {
-	sk->sk_data_ready(sk);
+	vsock_data_ready(sk);
 }
 
 static void vsock_block_update_write_window(struct sock *sk)
@@ -161,12 +161,12 @@ vmci_transport_notify_pkt_poll_in(struct sock *sk,
 {
 	struct vsock_sock *vsk = vsock_sk(sk);
 
-	if (vsock_stream_has_data(vsk)) {
+	if (vsock_stream_has_data(vsk) >= target) {
 		*data_ready_now = true;
 	} else {
-		/* We can't read right now because there is nothing in the
-		 * queue. Ask for notifications when there is something to
-		 * read.
+		/* We can't read right now because there is not enough data
+		 * in the queue. Ask for notifications when there is something
+		 * to read.
 		 */
 		if (sk->sk_state == TCP_ESTABLISHED)
 			vsock_block_update_write_window(sk);
@@ -282,7 +282,7 @@ vmci_transport_notify_pkt_recv_post_dequeue(
 		/* See the comment in
 		 * vmci_transport_notify_pkt_send_post_enqueue().
 		 */
-		sk->sk_data_ready(sk);
+		vsock_data_ready(sk);
 	}
 
 	return err;
diff --git a/net/wireless/core.c b/net/wireless/core.c
index eefd6d8ff465..5b0c4d5b80cf 100644
--- a/net/wireless/core.c
+++ b/net/wireless/core.c
@@ -860,6 +860,9 @@ int wiphy_register(struct wiphy *wiphy)
 
 		for (i = 0; i < sband->n_iftype_data; i++) {
 			const struct ieee80211_sband_iftype_data *iftd;
+			bool has_ap, has_non_ap;
+			u32 ap_bits = BIT(NL80211_IFTYPE_AP) |
+				      BIT(NL80211_IFTYPE_P2P_GO);
 
 			iftd = &sband->iftype_data[i];
 
@@ -879,6 +882,19 @@ int wiphy_register(struct wiphy *wiphy)
 			else
 				have_he = have_he &&
 					  iftd->he_cap.has_he;
+
+			has_ap = iftd->types_mask & ap_bits;
+			has_non_ap = iftd->types_mask & ~ap_bits;
+
+			/*
+			 * For EHT 20 MHz STA, the capabilities format differs
+			 * but to simplify, don't check 20 MHz but rather check
+			 * only if AP and non-AP were mentioned at the same time,
+			 * reject if so.
+			 */
+			if (WARN_ON(iftd->eht_cap.has_eht &&
+				    has_ap && has_non_ap))
+				return -EINVAL;
 		}
 
 		if (WARN_ON(!have_he && band == NL80211_BAND_6GHZ))
diff --git a/net/wireless/ibss.c b/net/wireless/ibss.c
index 4935f94d1acc..edd062f104f4 100644
--- a/net/wireless/ibss.c
+++ b/net/wireless/ibss.c
@@ -171,7 +171,7 @@ static void __cfg80211_clear_ibss(struct net_device *dev, bool nowext)
 	 */
 	if (rdev->ops->del_key)
 		for (i = 0; i < 6; i++)
-			rdev_del_key(rdev, dev, i, false, NULL);
+			rdev_del_key(rdev, dev, -1, i, false, NULL);
 
 	if (wdev->u.ibss.current_bss) {
 		cfg80211_unhold_bss(wdev->u.ibss.current_bss);
diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c
index 2705e3ee8fc4..8ff8b1c040f0 100644
--- a/net/wireless/nl80211.c
+++ b/net/wireless/nl80211.c
@@ -1545,7 +1545,6 @@ static int nl80211_key_allowed(struct wireless_dev *wdev)
 		return -ENOLINK;
 	case NL80211_IFTYPE_STATION:
 	case NL80211_IFTYPE_P2P_CLIENT:
-		/* for MLO, require driver validation of the link ID */
 		if (wdev->connected)
 			return 0;
 		return -ENOLINK;
@@ -1821,10 +1820,15 @@ nl80211_send_iftype_data(struct sk_buff *msg,
 	if (eht_cap->has_eht && he_cap->has_he) {
 		u8 mcs_nss_size, ppe_thresh_size;
 		u16 ppe_thres_hdr;
+		bool is_ap;
+
+		is_ap = iftdata->types_mask & BIT(NL80211_IFTYPE_AP) ||
+			iftdata->types_mask & BIT(NL80211_IFTYPE_P2P_GO);
 
 		mcs_nss_size =
 			ieee80211_eht_mcs_nss_size(&he_cap->he_cap_elem,
-						   &eht_cap->eht_cap_elem);
+						   &eht_cap->eht_cap_elem,
+						   is_ap);
 
 		ppe_thres_hdr = get_unaligned_le16(&eht_cap->eht_ppe_thres[0]);
 		ppe_thresh_size =
@@ -3476,8 +3480,21 @@ static int nl80211_set_wiphy(struct sk_buff *skb, struct genl_info *info)
 			if (result)
 				goto out;
 
-			result = rdev_set_txq_params(rdev, netdev,
-						     &txq_params);
+			txq_params.link_id =
+				nl80211_link_id_or_invalid(info->attrs);
+
+			wdev_lock(netdev->ieee80211_ptr);
+			if (txq_params.link_id >= 0 &&
+			    !(netdev->ieee80211_ptr->valid_links &
+			      BIT(txq_params.link_id)))
+				result = -ENOLINK;
+			else if (txq_params.link_id >= 0 &&
+				 !netdev->ieee80211_ptr->valid_links)
+				result = -EINVAL;
+			else
+				result = rdev_set_txq_params(rdev, netdev,
+							     &txq_params);
+			wdev_unlock(netdev->ieee80211_ptr);
 			if (result)
 				goto out;
 		}
@@ -3848,12 +3865,19 @@ static int nl80211_send_iface(struct sk_buff *msg, u32 portid, u32 seq, int flag
 
 		for_each_valid_link(wdev, link_id) {
 			struct nlattr *link = nla_nest_start(msg, link_id + 1);
+			struct cfg80211_chan_def chandef = {};
+			int ret;
 
 			if (nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_id))
 				goto nla_put_failure;
 			if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN,
 				    wdev->links[link_id].addr))
 				goto nla_put_failure;
+
+			ret = rdev_get_channel(rdev, wdev, link_id, &chandef);
+			if (ret == 0 && nl80211_send_chandef(msg, &chandef))
+				goto nla_put_failure;
+
 			nla_nest_end(msg, link);
 		}
 
@@ -4320,6 +4344,38 @@ static int nl80211_set_noack_map(struct sk_buff *skb, struct genl_info *info)
 	return rdev_set_noack_map(rdev, dev, noack_map);
 }
 
+static int nl80211_validate_key_link_id(struct genl_info *info,
+					struct wireless_dev *wdev,
+					int link_id, bool pairwise)
+{
+	if (pairwise) {
+		if (link_id != -1) {
+			GENL_SET_ERR_MSG(info,
+					 "link ID not allowed for pairwise key");
+			return -EINVAL;
+		}
+
+		return 0;
+	}
+
+	if (wdev->valid_links) {
+		if (link_id == -1) {
+			GENL_SET_ERR_MSG(info,
+					 "link ID must for MLO group key");
+			return -EINVAL;
+		}
+		if (!(wdev->valid_links & BIT(link_id))) {
+			GENL_SET_ERR_MSG(info, "invalid link ID for MLO group key");
+			return -EINVAL;
+		}
+	} else if (link_id != -1) {
+		GENL_SET_ERR_MSG(info, "link ID not allowed for non-MLO group key");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 struct get_key_cookie {
 	struct sk_buff *msg;
 	int error;
@@ -4381,13 +4437,15 @@ static int nl80211_get_key(struct sk_buff *skb, struct genl_info *info)
 	void *hdr;
 	struct sk_buff *msg;
 	bool bigtk_support = false;
+	int link_id = nl80211_link_id_or_invalid(info->attrs);
+	struct wireless_dev *wdev = dev->ieee80211_ptr;
 
 	if (wiphy_ext_feature_isset(&rdev->wiphy,
 				    NL80211_EXT_FEATURE_BEACON_PROTECTION))
 		bigtk_support = true;
 
-	if ((dev->ieee80211_ptr->iftype == NL80211_IFTYPE_STATION ||
-	     dev->ieee80211_ptr->iftype == NL80211_IFTYPE_P2P_CLIENT) &&
+	if ((wdev->iftype == NL80211_IFTYPE_STATION ||
+	     wdev->iftype == NL80211_IFTYPE_P2P_CLIENT) &&
 	    wiphy_ext_feature_isset(&rdev->wiphy,
 				    NL80211_EXT_FEATURE_BEACON_PROTECTION_CLIENT))
 		bigtk_support = true;
@@ -4439,8 +4497,12 @@ static int nl80211_get_key(struct sk_buff *skb, struct genl_info *info)
 	    nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, mac_addr))
 		goto nla_put_failure;
 
-	err = rdev_get_key(rdev, dev, key_idx, pairwise, mac_addr, &cookie,
-			   get_key_callback);
+	err = nl80211_validate_key_link_id(info, wdev, link_id, pairwise);
+	if (err)
+		goto free_msg;
+
+	err = rdev_get_key(rdev, dev, link_id, key_idx, pairwise, mac_addr,
+			   &cookie, get_key_callback);
 
 	if (err)
 		goto free_msg;
@@ -4464,6 +4526,8 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info)
 	struct key_parse key;
 	int err;
 	struct net_device *dev = info->user_ptr[1];
+	int link_id = nl80211_link_id_or_invalid(info->attrs);
+	struct wireless_dev *wdev = dev->ieee80211_ptr;
 
 	err = nl80211_parse_key(info, &key);
 	if (err)
@@ -4479,7 +4543,7 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info)
 	    !(key.p.mode == NL80211_KEY_SET_TX))
 		return -EINVAL;
 
-	wdev_lock(dev->ieee80211_ptr);
+	wdev_lock(wdev);
 
 	if (key.def) {
 		if (!rdev->ops->set_default_key) {
@@ -4487,18 +4551,22 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info)
 			goto out;
 		}
 
-		err = nl80211_key_allowed(dev->ieee80211_ptr);
+		err = nl80211_key_allowed(wdev);
 		if (err)
 			goto out;
 
-		err = rdev_set_default_key(rdev, dev, key.idx,
-						 key.def_uni, key.def_multi);
+		err = nl80211_validate_key_link_id(info, wdev, link_id, false);
+		if (err)
+			goto out;
+
+		err = rdev_set_default_key(rdev, dev, link_id, key.idx,
+					   key.def_uni, key.def_multi);
 
 		if (err)
 			goto out;
 
 #ifdef CONFIG_CFG80211_WEXT
-		dev->ieee80211_ptr->wext.default_key = key.idx;
+		wdev->wext.default_key = key.idx;
 #endif
 	} else if (key.defmgmt) {
 		if (key.def_uni || !key.def_multi) {
@@ -4511,16 +4579,20 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info)
 			goto out;
 		}
 
-		err = nl80211_key_allowed(dev->ieee80211_ptr);
+		err = nl80211_key_allowed(wdev);
 		if (err)
 			goto out;
 
-		err = rdev_set_default_mgmt_key(rdev, dev, key.idx);
+		err = nl80211_validate_key_link_id(info, wdev, link_id, false);
+		if (err)
+			goto out;
+
+		err = rdev_set_default_mgmt_key(rdev, dev, link_id, key.idx);
 		if (err)
 			goto out;
 
 #ifdef CONFIG_CFG80211_WEXT
-		dev->ieee80211_ptr->wext.default_mgmt_key = key.idx;
+		wdev->wext.default_mgmt_key = key.idx;
 #endif
 	} else if (key.defbeacon) {
 		if (key.def_uni || !key.def_multi) {
@@ -4533,11 +4605,15 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info)
 			goto out;
 		}
 
-		err = nl80211_key_allowed(dev->ieee80211_ptr);
+		err = nl80211_key_allowed(wdev);
+		if (err)
+			goto out;
+
+		err = nl80211_validate_key_link_id(info, wdev, link_id, false);
 		if (err)
 			goto out;
 
-		err = rdev_set_default_beacon_key(rdev, dev, key.idx);
+		err = rdev_set_default_beacon_key(rdev, dev, link_id, key.idx);
 		if (err)
 			goto out;
 	} else if (key.p.mode == NL80211_KEY_SET_TX &&
@@ -4553,14 +4629,18 @@ static int nl80211_set_key(struct sk_buff *skb, struct genl_info *info)
 			goto out;
 		}
 
-		err = rdev_add_key(rdev, dev, key.idx,
+		err = nl80211_validate_key_link_id(info, wdev, link_id, true);
+		if (err)
+			goto out;
+
+		err = rdev_add_key(rdev, dev, link_id, key.idx,
 				   NL80211_KEYTYPE_PAIRWISE,
 				   mac_addr, &key.p);
 	} else {
 		err = -EINVAL;
 	}
  out:
-	wdev_unlock(dev->ieee80211_ptr);
+	wdev_unlock(wdev);
 
 	return err;
 }
@@ -4572,6 +4652,8 @@ static int nl80211_new_key(struct sk_buff *skb, struct genl_info *info)
 	struct net_device *dev = info->user_ptr[1];
 	struct key_parse key;
 	const u8 *mac_addr = NULL;
+	int link_id = nl80211_link_id_or_invalid(info->attrs);
+	struct wireless_dev *wdev = dev->ieee80211_ptr;
 
 	err = nl80211_parse_key(info, &key);
 	if (err)
@@ -4613,18 +4695,23 @@ static int nl80211_new_key(struct sk_buff *skb, struct genl_info *info)
 		return -EINVAL;
 	}
 
-	wdev_lock(dev->ieee80211_ptr);
-	err = nl80211_key_allowed(dev->ieee80211_ptr);
+	wdev_lock(wdev);
+	err = nl80211_key_allowed(wdev);
 	if (err)
 		GENL_SET_ERR_MSG(info, "key not allowed");
+
+	if (!err)
+		err = nl80211_validate_key_link_id(info, wdev, link_id,
+				key.type == NL80211_KEYTYPE_PAIRWISE);
+
 	if (!err) {
-		err = rdev_add_key(rdev, dev, key.idx,
+		err = rdev_add_key(rdev, dev, link_id, key.idx,
 				   key.type == NL80211_KEYTYPE_PAIRWISE,
 				    mac_addr, &key.p);
 		if (err)
 			GENL_SET_ERR_MSG(info, "key addition failed");
 	}
-	wdev_unlock(dev->ieee80211_ptr);
+	wdev_unlock(wdev);
 
 	return err;
 }
@@ -4636,6 +4723,8 @@ static int nl80211_del_key(struct sk_buff *skb, struct genl_info *info)
 	struct net_device *dev = info->user_ptr[1];
 	u8 *mac_addr = NULL;
 	struct key_parse key;
+	int link_id = nl80211_link_id_or_invalid(info->attrs);
+	struct wireless_dev *wdev = dev->ieee80211_ptr;
 
 	err = nl80211_parse_key(info, &key);
 	if (err)
@@ -4663,27 +4752,31 @@ static int nl80211_del_key(struct sk_buff *skb, struct genl_info *info)
 	if (!rdev->ops->del_key)
 		return -EOPNOTSUPP;
 
-	wdev_lock(dev->ieee80211_ptr);
-	err = nl80211_key_allowed(dev->ieee80211_ptr);
+	wdev_lock(wdev);
+	err = nl80211_key_allowed(wdev);
 
 	if (key.type == NL80211_KEYTYPE_GROUP && mac_addr &&
 	    !(rdev->wiphy.flags & WIPHY_FLAG_IBSS_RSN))
 		err = -ENOENT;
 
 	if (!err)
-		err = rdev_del_key(rdev, dev, key.idx,
+		err = nl80211_validate_key_link_id(info, wdev, link_id,
+				key.type == NL80211_KEYTYPE_PAIRWISE);
+
+	if (!err)
+		err = rdev_del_key(rdev, dev, link_id, key.idx,
 				   key.type == NL80211_KEYTYPE_PAIRWISE,
 				   mac_addr);
 
 #ifdef CONFIG_CFG80211_WEXT
 	if (!err) {
-		if (key.idx == dev->ieee80211_ptr->wext.default_key)
-			dev->ieee80211_ptr->wext.default_key = -1;
-		else if (key.idx == dev->ieee80211_ptr->wext.default_mgmt_key)
-			dev->ieee80211_ptr->wext.default_mgmt_key = -1;
+		if (key.idx == wdev->wext.default_key)
+			wdev->wext.default_key = -1;
+		else if (key.idx == wdev->wext.default_mgmt_key)
+			wdev->wext.default_mgmt_key = -1;
 	}
 #endif
-	wdev_unlock(dev->ieee80211_ptr);
+	wdev_unlock(wdev);
 
 	return err;
 }
@@ -5587,7 +5680,7 @@ static int nl80211_calculate_ap_params(struct cfg80211_ap_settings *params)
 		params->eht_cap = (void *)(cap->data + 1);
 		if (!ieee80211_eht_capa_size_ok((const u8 *)params->he_cap,
 						(const u8 *)params->eht_cap,
-						cap->datalen - 1))
+						cap->datalen - 1, true))
 			return -EINVAL;
 	}
 	cap = cfg80211_find_ext_elem(WLAN_EID_EXT_EHT_OPERATION, ies, ies_len);
@@ -6816,7 +6909,8 @@ static int nl80211_set_station_tdls(struct genl_info *info,
 
 			if (!ieee80211_eht_capa_size_ok((const u8 *)params->link_sta_params.he_capa,
 							(const u8 *)params->link_sta_params.eht_capa,
-							params->link_sta_params.eht_capa_len))
+							params->link_sta_params.eht_capa_len,
+							false))
 				return -EINVAL;
 		}
 	}
@@ -7127,7 +7221,8 @@ static int nl80211_new_station(struct sk_buff *skb, struct genl_info *info)
 
 			if (!ieee80211_eht_capa_size_ok((const u8 *)params.link_sta_params.he_capa,
 							(const u8 *)params.link_sta_params.eht_capa,
-							params.link_sta_params.eht_capa_len))
+							params.link_sta_params.eht_capa_len,
+							false))
 				return -EINVAL;
 		}
 	}
@@ -10087,8 +10182,10 @@ static int nl80211_send_bss(struct sk_buff *msg, struct netlink_callback *cb,
 			    (nla_put_u32(msg, NL80211_BSS_STATUS,
 					 NL80211_BSS_STATUS_ASSOCIATED) ||
 			     (wdev->valid_links &&
-			      nla_put_u8(msg, NL80211_BSS_MLO_LINK_ID,
-					 link_id))))
+			      (nla_put_u8(msg, NL80211_BSS_MLO_LINK_ID,
+					  link_id) ||
+			       nla_put(msg, NL80211_BSS_MLD_ADDR, ETH_ALEN,
+				       wdev->u.client.connected_addr)))))
 				goto nla_put_failure;
 		}
 		break;
@@ -11184,7 +11281,6 @@ static int nl80211_set_mcast_rate(struct sk_buff *skb, struct genl_info *info)
 	struct net_device *dev = info->user_ptr[1];
 	int mcast_rate[NUM_NL80211_BANDS];
 	u32 nla_rate;
-	int err;
 
 	if (dev->ieee80211_ptr->iftype != NL80211_IFTYPE_ADHOC &&
 	    dev->ieee80211_ptr->iftype != NL80211_IFTYPE_MESH_POINT &&
@@ -11203,9 +11299,7 @@ static int nl80211_set_mcast_rate(struct sk_buff *skb, struct genl_info *info)
 	if (!nl80211_parse_mcast_rate(rdev, mcast_rate, nla_rate))
 		return -EINVAL;
 
-	err = rdev_set_mcast_rate(rdev, dev, mcast_rate);
-
-	return err;
+	return rdev_set_mcast_rate(rdev, dev, mcast_rate);
 }
 
 static struct sk_buff *
@@ -15887,7 +15981,8 @@ nl80211_add_mod_link_station(struct sk_buff *skb, struct genl_info *info,
 
 			if (!ieee80211_eht_capa_size_ok((const u8 *)params.he_capa,
 							(const u8 *)params.eht_capa,
-							params.eht_capa_len))
+							params.eht_capa_len,
+							false))
 				return -EINVAL;
 		}
 	}
@@ -17141,6 +17236,7 @@ static struct genl_family nl80211_fam __ro_after_init = {
 	.n_ops = ARRAY_SIZE(nl80211_ops),
 	.small_ops = nl80211_small_ops,
 	.n_small_ops = ARRAY_SIZE(nl80211_small_ops),
+	.resv_start_op = NL80211_CMD_REMOVE_LINK_STA + 1,
 	.mcgrps = nl80211_mcgrps,
 	.n_mcgrps = ARRAY_SIZE(nl80211_mcgrps),
 	.parallel_ops = true,
@@ -18836,11 +18932,13 @@ EXPORT_SYMBOL(cfg80211_pmksa_candidate_notify);
 
 static void nl80211_ch_switch_notify(struct cfg80211_registered_device *rdev,
 				     struct net_device *netdev,
+				     unsigned int link_id,
 				     struct cfg80211_chan_def *chandef,
 				     gfp_t gfp,
 				     enum nl80211_commands notif,
 				     u8 count, bool quiet)
 {
+	struct wireless_dev *wdev = netdev->ieee80211_ptr;
 	struct sk_buff *msg;
 	void *hdr;
 
@@ -18857,6 +18955,10 @@ static void nl80211_ch_switch_notify(struct cfg80211_registered_device *rdev,
 	if (nla_put_u32(msg, NL80211_ATTR_IFINDEX, netdev->ifindex))
 		goto nla_put_failure;
 
+	if (wdev->valid_links &&
+	    nla_put_u8(msg, NL80211_ATTR_MLO_LINK_ID, link_id))
+		goto nla_put_failure;
+
 	if (nl80211_send_chandef(msg, chandef))
 		goto nla_put_failure;
 
@@ -18916,22 +19018,26 @@ void cfg80211_ch_switch_notify(struct net_device *dev,
 
 	cfg80211_sched_dfs_chan_update(rdev);
 
-	nl80211_ch_switch_notify(rdev, dev, chandef, GFP_KERNEL,
+	nl80211_ch_switch_notify(rdev, dev, link_id, chandef, GFP_KERNEL,
 				 NL80211_CMD_CH_SWITCH_NOTIFY, 0, false);
 }
 EXPORT_SYMBOL(cfg80211_ch_switch_notify);
 
 void cfg80211_ch_switch_started_notify(struct net_device *dev,
 				       struct cfg80211_chan_def *chandef,
-				       u8 count, bool quiet)
+				       unsigned int link_id, u8 count,
+				       bool quiet)
 {
 	struct wireless_dev *wdev = dev->ieee80211_ptr;
 	struct wiphy *wiphy = wdev->wiphy;
 	struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
 
-	trace_cfg80211_ch_switch_started_notify(dev, chandef);
+	ASSERT_WDEV_LOCK(wdev);
+	WARN_INVALID_LINK_ID(wdev, link_id);
+
+	trace_cfg80211_ch_switch_started_notify(dev, chandef, link_id);
 
-	nl80211_ch_switch_notify(rdev, dev, chandef, GFP_KERNEL,
+	nl80211_ch_switch_notify(rdev, dev, link_id, chandef, GFP_KERNEL,
 				 NL80211_CMD_CH_SWITCH_STARTED_NOTIFY,
 				 count, quiet);
 }
diff --git a/net/wireless/rdev-ops.h b/net/wireless/rdev-ops.h
index 40915a82da73..13b209a8db28 100644
--- a/net/wireless/rdev-ops.h
+++ b/net/wireless/rdev-ops.h
@@ -77,65 +77,69 @@ rdev_change_virtual_intf(struct cfg80211_registered_device *rdev,
 }
 
 static inline int rdev_add_key(struct cfg80211_registered_device *rdev,
-			       struct net_device *netdev, u8 key_index,
-			       bool pairwise, const u8 *mac_addr,
+			       struct net_device *netdev, int link_id,
+			       u8 key_index, bool pairwise, const u8 *mac_addr,
 			       struct key_params *params)
 {
 	int ret;
-	trace_rdev_add_key(&rdev->wiphy, netdev, key_index, pairwise,
+	trace_rdev_add_key(&rdev->wiphy, netdev, link_id, key_index, pairwise,
 			   mac_addr, params->mode);
-	ret = rdev->ops->add_key(&rdev->wiphy, netdev, key_index, pairwise,
-				  mac_addr, params);
+	ret = rdev->ops->add_key(&rdev->wiphy, netdev, link_id, key_index,
+				  pairwise, mac_addr, params);
 	trace_rdev_return_int(&rdev->wiphy, ret);
 	return ret;
 }
 
 static inline int
 rdev_get_key(struct cfg80211_registered_device *rdev, struct net_device *netdev,
-	     u8 key_index, bool pairwise, const u8 *mac_addr, void *cookie,
+	     int link_id, u8 key_index, bool pairwise, const u8 *mac_addr,
+	     void *cookie,
 	     void (*callback)(void *cookie, struct key_params*))
 {
 	int ret;
-	trace_rdev_get_key(&rdev->wiphy, netdev, key_index, pairwise, mac_addr);
-	ret = rdev->ops->get_key(&rdev->wiphy, netdev, key_index, pairwise,
-				  mac_addr, cookie, callback);
+	trace_rdev_get_key(&rdev->wiphy, netdev, link_id, key_index, pairwise,
+			   mac_addr);
+	ret = rdev->ops->get_key(&rdev->wiphy, netdev, link_id, key_index,
+				  pairwise, mac_addr, cookie, callback);
 	trace_rdev_return_int(&rdev->wiphy, ret);
 	return ret;
 }
 
 static inline int rdev_del_key(struct cfg80211_registered_device *rdev,
-			       struct net_device *netdev, u8 key_index,
-			       bool pairwise, const u8 *mac_addr)
+			       struct net_device *netdev, int link_id,
+			       u8 key_index, bool pairwise, const u8 *mac_addr)
 {
 	int ret;
-	trace_rdev_del_key(&rdev->wiphy, netdev, key_index, pairwise, mac_addr);
-	ret = rdev->ops->del_key(&rdev->wiphy, netdev, key_index, pairwise,
-				  mac_addr);
+	trace_rdev_del_key(&rdev->wiphy, netdev, link_id, key_index, pairwise,
+			   mac_addr);
+	ret = rdev->ops->del_key(&rdev->wiphy, netdev, link_id, key_index,
+				  pairwise, mac_addr);
 	trace_rdev_return_int(&rdev->wiphy, ret);
 	return ret;
 }
 
 static inline int
 rdev_set_default_key(struct cfg80211_registered_device *rdev,
-		     struct net_device *netdev, u8 key_index, bool unicast,
-		     bool multicast)
+		     struct net_device *netdev, int link_id, u8 key_index,
+		     bool unicast, bool multicast)
 {
 	int ret;
-	trace_rdev_set_default_key(&rdev->wiphy, netdev, key_index,
+	trace_rdev_set_default_key(&rdev->wiphy, netdev, link_id, key_index,
 				   unicast, multicast);
-	ret = rdev->ops->set_default_key(&rdev->wiphy, netdev, key_index,
-					  unicast, multicast);
+	ret = rdev->ops->set_default_key(&rdev->wiphy, netdev, link_id,
+					  key_index, unicast, multicast);
 	trace_rdev_return_int(&rdev->wiphy, ret);
 	return ret;
 }
 
 static inline int
 rdev_set_default_mgmt_key(struct cfg80211_registered_device *rdev,
-			  struct net_device *netdev, u8 key_index)
+			  struct net_device *netdev, int link_id, u8 key_index)
 {
 	int ret;
-	trace_rdev_set_default_mgmt_key(&rdev->wiphy, netdev, key_index);
-	ret = rdev->ops->set_default_mgmt_key(&rdev->wiphy, netdev,
+	trace_rdev_set_default_mgmt_key(&rdev->wiphy, netdev, link_id,
+					key_index);
+	ret = rdev->ops->set_default_mgmt_key(&rdev->wiphy, netdev, link_id,
 					       key_index);
 	trace_rdev_return_int(&rdev->wiphy, ret);
 	return ret;
@@ -143,13 +147,15 @@ rdev_set_default_mgmt_key(struct cfg80211_registered_device *rdev,
 
 static inline int
 rdev_set_default_beacon_key(struct cfg80211_registered_device *rdev,
-			    struct net_device *netdev, u8 key_index)
+			    struct net_device *netdev, int link_id,
+			    u8 key_index)
 {
 	int ret;
 
-	trace_rdev_set_default_beacon_key(&rdev->wiphy, netdev, key_index);
-	ret = rdev->ops->set_default_beacon_key(&rdev->wiphy, netdev,
-						key_index);
+	trace_rdev_set_default_beacon_key(&rdev->wiphy, netdev, link_id,
+					  key_index);
+	ret = rdev->ops->set_default_beacon_key(&rdev->wiphy, netdev, link_id,
+						 key_index);
 	trace_rdev_return_int(&rdev->wiphy, ret);
 	return ret;
 }
diff --git a/net/wireless/reg.c b/net/wireless/reg.c
index c7383ede794f..d5c7a5aa6853 100644
--- a/net/wireless/reg.c
+++ b/net/wireless/reg.c
@@ -2389,6 +2389,10 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev)
 		switch (iftype) {
 		case NL80211_IFTYPE_AP:
 		case NL80211_IFTYPE_P2P_GO:
+			if (!wdev->links[link].ap.beacon_interval)
+				continue;
+			chandef = wdev->links[link].ap.chandef;
+			break;
 		case NL80211_IFTYPE_MESH_POINT:
 			if (!wdev->u.mesh.beacon_interval)
 				continue;
diff --git a/net/wireless/scan.c b/net/wireless/scan.c
index 0134e5d5c81a..5382fc2003db 100644
--- a/net/wireless/scan.c
+++ b/net/wireless/scan.c
@@ -540,7 +540,7 @@ static int cfg80211_parse_ap_info(struct cfg80211_colocated_ap *entry,
 	memcpy(entry->bssid, pos, ETH_ALEN);
 	pos += ETH_ALEN;
 
-	if (length == IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM) {
+	if (length >= IEEE80211_TBTT_INFO_OFFSET_BSSID_SSSID_BSS_PARAM) {
 		memcpy(&entry->short_ssid, pos,
 		       sizeof(entry->short_ssid));
 		entry->short_ssid_valid = true;
diff --git a/net/wireless/sme.c b/net/wireless/sme.c
index 27fb2a0c4052..d513536617bd 100644
--- a/net/wireless/sme.c
+++ b/net/wireless/sme.c
@@ -747,6 +747,9 @@ void __cfg80211_connect_result(struct net_device *dev,
 			if (WARN_ON(!cr->links[link].addr))
 				goto out;
 		}
+
+		if (WARN_ON(wdev->connect_keys))
+			goto out;
 	}
 
 	wdev->unprot_beacon_reported = 0;
@@ -1325,7 +1328,7 @@ void __cfg80211_disconnected(struct net_device *dev, const u8 *ie,
 			    NL80211_EXT_FEATURE_BEACON_PROTECTION_CLIENT))
 			max_key_idx = 7;
 		for (i = 0; i <= max_key_idx; i++)
-			rdev_del_key(rdev, dev, i, false, NULL);
+			rdev_del_key(rdev, dev, -1, i, false, NULL);
 	}
 
 	rdev_set_qos_map(rdev, dev, NULL);
diff --git a/net/wireless/trace.h b/net/wireless/trace.h
index 10b2fd9bacb5..a405c3edbc47 100644
--- a/net/wireless/trace.h
+++ b/net/wireless/trace.h
@@ -434,13 +434,14 @@ TRACE_EVENT(rdev_change_virtual_intf,
 );
 
 DECLARE_EVENT_CLASS(key_handle,
-	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index,
-		 bool pairwise, const u8 *mac_addr),
-	TP_ARGS(wiphy, netdev, key_index, pairwise, mac_addr),
+	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+		 u8 key_index, bool pairwise, const u8 *mac_addr),
+	TP_ARGS(wiphy, netdev, link_id, key_index, pairwise, mac_addr),
 	TP_STRUCT__entry(
 		WIPHY_ENTRY
 		NETDEV_ENTRY
 		MAC_ENTRY(mac_addr)
+		__field(int, link_id)
 		__field(u8, key_index)
 		__field(bool, pairwise)
 	),
@@ -448,34 +449,38 @@ DECLARE_EVENT_CLASS(key_handle,
 		WIPHY_ASSIGN;
 		NETDEV_ASSIGN;
 		MAC_ASSIGN(mac_addr, mac_addr);
+		__entry->link_id = link_id;
 		__entry->key_index = key_index;
 		__entry->pairwise = pairwise;
 	),
-	TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", key_index: %u, pairwise: %s, mac addr: " MAC_PR_FMT,
-		  WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->key_index,
-		  BOOL_TO_STR(__entry->pairwise), MAC_PR_ARG(mac_addr))
+	TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, "
+		  "key_index: %u, pairwise: %s, mac addr: " MAC_PR_FMT,
+		  WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->link_id,
+		  __entry->key_index, BOOL_TO_STR(__entry->pairwise),
+		  MAC_PR_ARG(mac_addr))
 );
 
 DEFINE_EVENT(key_handle, rdev_get_key,
-	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index,
-		 bool pairwise, const u8 *mac_addr),
-	TP_ARGS(wiphy, netdev, key_index, pairwise, mac_addr)
+	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+		 u8 key_index, bool pairwise, const u8 *mac_addr),
+	TP_ARGS(wiphy, netdev, link_id, key_index, pairwise, mac_addr)
 );
 
 DEFINE_EVENT(key_handle, rdev_del_key,
-	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index,
-		 bool pairwise, const u8 *mac_addr),
-	TP_ARGS(wiphy, netdev, key_index, pairwise, mac_addr)
+	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+		 u8 key_index, bool pairwise, const u8 *mac_addr),
+	TP_ARGS(wiphy, netdev, link_id, key_index, pairwise, mac_addr)
 );
 
 TRACE_EVENT(rdev_add_key,
-	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index,
-		 bool pairwise, const u8 *mac_addr, u8 mode),
-	TP_ARGS(wiphy, netdev, key_index, pairwise, mac_addr, mode),
+	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+		 u8 key_index, bool pairwise, const u8 *mac_addr, u8 mode),
+	TP_ARGS(wiphy, netdev, link_id, key_index, pairwise, mac_addr, mode),
 	TP_STRUCT__entry(
 		WIPHY_ENTRY
 		NETDEV_ENTRY
 		MAC_ENTRY(mac_addr)
+		__field(int, link_id)
 		__field(u8, key_index)
 		__field(bool, pairwise)
 		__field(u8, mode)
@@ -484,24 +489,27 @@ TRACE_EVENT(rdev_add_key,
 		WIPHY_ASSIGN;
 		NETDEV_ASSIGN;
 		MAC_ASSIGN(mac_addr, mac_addr);
+		__entry->link_id = link_id;
 		__entry->key_index = key_index;
 		__entry->pairwise = pairwise;
 		__entry->mode = mode;
 	),
-	TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", key_index: %u, "
-		  "mode: %u, pairwise: %s, mac addr: " MAC_PR_FMT,
-		  WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->key_index,
-		  __entry->mode, BOOL_TO_STR(__entry->pairwise),
-		  MAC_PR_ARG(mac_addr))
+	TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, "
+		  "key_index: %u, mode: %u, pairwise: %s, "
+		  "mac addr: " MAC_PR_FMT,
+		  WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->link_id,
+		  __entry->key_index, __entry->mode,
+		  BOOL_TO_STR(__entry->pairwise), MAC_PR_ARG(mac_addr))
 );
 
 TRACE_EVENT(rdev_set_default_key,
-	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index,
-		 bool unicast, bool multicast),
-	TP_ARGS(wiphy, netdev, key_index, unicast, multicast),
+	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+		 u8 key_index, bool unicast, bool multicast),
+	TP_ARGS(wiphy, netdev, link_id, key_index, unicast, multicast),
 	TP_STRUCT__entry(
 		WIPHY_ENTRY
 		NETDEV_ENTRY
+		__field(int, link_id)
 		__field(u8, key_index)
 		__field(bool, unicast)
 		__field(bool, multicast)
@@ -509,48 +517,58 @@ TRACE_EVENT(rdev_set_default_key,
 	TP_fast_assign(
 		WIPHY_ASSIGN;
 		NETDEV_ASSIGN;
+		__entry->link_id = link_id;
 		__entry->key_index = key_index;
 		__entry->unicast = unicast;
 		__entry->multicast = multicast;
 	),
-	TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", key index: %u, unicast: %s, multicast: %s",
-		  WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->key_index,
-		  BOOL_TO_STR(__entry->unicast),
+	TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, "
+		  "key index: %u, unicast: %s, multicast: %s",
+		  WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->link_id,
+		  __entry->key_index, BOOL_TO_STR(__entry->unicast),
 		  BOOL_TO_STR(__entry->multicast))
 );
 
 TRACE_EVENT(rdev_set_default_mgmt_key,
-	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index),
-	TP_ARGS(wiphy, netdev, key_index),
+	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+		 u8 key_index),
+	TP_ARGS(wiphy, netdev, link_id, key_index),
 	TP_STRUCT__entry(
 		WIPHY_ENTRY
 		NETDEV_ENTRY
+		__field(int, link_id)
 		__field(u8, key_index)
 	),
 	TP_fast_assign(
 		WIPHY_ASSIGN;
 		NETDEV_ASSIGN;
+		__entry->link_id = link_id;
 		__entry->key_index = key_index;
 	),
-	TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", key index: %u",
-		  WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->key_index)
+	TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, "
+		  "key index: %u", WIPHY_PR_ARG, NETDEV_PR_ARG,
+		  __entry->link_id, __entry->key_index)
 );
 
 TRACE_EVENT(rdev_set_default_beacon_key,
-	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, u8 key_index),
-	TP_ARGS(wiphy, netdev, key_index),
+	TP_PROTO(struct wiphy *wiphy, struct net_device *netdev, int link_id,
+		 u8 key_index),
+	TP_ARGS(wiphy, netdev, link_id, key_index),
 	TP_STRUCT__entry(
 		WIPHY_ENTRY
 		NETDEV_ENTRY
+		__field(int, link_id)
 		__field(u8, key_index)
 	),
 	TP_fast_assign(
 		WIPHY_ASSIGN;
 		NETDEV_ASSIGN;
+		__entry->link_id = link_id;
 		__entry->key_index = key_index;
 	),
-	TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", key index: %u",
-		  WIPHY_PR_ARG, NETDEV_PR_ARG, __entry->key_index)
+	TP_printk(WIPHY_PR_FMT ", " NETDEV_PR_FMT ", link_id: %d, "
+		  "key index: %u", WIPHY_PR_ARG, NETDEV_PR_ARG,
+		  __entry->link_id, __entry->key_index)
 );
 
 TRACE_EVENT(rdev_start_ap,
@@ -3245,18 +3263,21 @@ TRACE_EVENT(cfg80211_ch_switch_notify,
 
 TRACE_EVENT(cfg80211_ch_switch_started_notify,
 	TP_PROTO(struct net_device *netdev,
-		 struct cfg80211_chan_def *chandef),
-	TP_ARGS(netdev, chandef),
+		 struct cfg80211_chan_def *chandef,
+		 unsigned int link_id),
+	TP_ARGS(netdev, chandef, link_id),
 	TP_STRUCT__entry(
 		NETDEV_ENTRY
 		CHAN_DEF_ENTRY
+		__field(unsigned int, link_id)
 	),
 	TP_fast_assign(
 		NETDEV_ASSIGN;
 		CHAN_DEF_ASSIGN(chandef);
+		__entry->link_id = link_id;
 	),
-	TP_printk(NETDEV_PR_FMT ", " CHAN_DEF_PR_FMT,
-		  NETDEV_PR_ARG, CHAN_DEF_PR_ARG)
+	TP_printk(NETDEV_PR_FMT ", " CHAN_DEF_PR_FMT ", link:%d",
+		  NETDEV_PR_ARG, CHAN_DEF_PR_ARG, __entry->link_id)
 );
 
 TRACE_EVENT(cfg80211_radar_event,
diff --git a/net/wireless/util.c b/net/wireless/util.c
index 2c127951764a..01493568a21d 100644
--- a/net/wireless/util.c
+++ b/net/wireless/util.c
@@ -935,13 +935,13 @@ void cfg80211_upload_connect_keys(struct wireless_dev *wdev)
 	for (i = 0; i < CFG80211_MAX_WEP_KEYS; i++) {
 		if (!wdev->connect_keys->params[i].cipher)
 			continue;
-		if (rdev_add_key(rdev, dev, i, false, NULL,
+		if (rdev_add_key(rdev, dev, -1, i, false, NULL,
 				 &wdev->connect_keys->params[i])) {
 			netdev_err(dev, "failed to set key %d\n", i);
 			continue;
 		}
 		if (wdev->connect_keys->def == i &&
-		    rdev_set_default_key(rdev, dev, i, true, true)) {
+		    rdev_set_default_key(rdev, dev, -1, i, true, true)) {
 			netdev_err(dev, "failed to set defkey %d\n", i);
 			continue;
 		}
@@ -1361,7 +1361,7 @@ static u32 cfg80211_calculate_bitrate_he(struct rate_info *rate)
 		 25599, /*  4.166666... */
 		 17067, /*  2.777777... */
 		 12801, /*  2.083333... */
-		 11769, /*  1.851851... */
+		 11377, /*  1.851725... */
 		 10239, /*  1.666666... */
 		  8532, /*  1.388888... */
 		  7680, /*  1.250000... */
@@ -1444,7 +1444,7 @@ static u32 cfg80211_calculate_bitrate_eht(struct rate_info *rate)
 		 25599, /*  4.166666... */
 		 17067, /*  2.777777... */
 		 12801, /*  2.083333... */
-		 11769, /*  1.851851... */
+		 11377, /*  1.851725... */
 		 10239, /*  1.666666... */
 		  8532, /*  1.388888... */
 		  7680, /*  1.250000... */
diff --git a/net/wireless/wext-compat.c b/net/wireless/wext-compat.c
index a9767bfe7330..ddf340bfa07a 100644
--- a/net/wireless/wext-compat.c
+++ b/net/wireless/wext-compat.c
@@ -470,7 +470,7 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev,
 			    !(rdev->wiphy.flags & WIPHY_FLAG_IBSS_RSN))
 				err = -ENOENT;
 			else
-				err = rdev_del_key(rdev, dev, idx, pairwise,
+				err = rdev_del_key(rdev, dev, -1, idx, pairwise,
 						   addr);
 		}
 		wdev->wext.connect.privacy = false;
@@ -509,7 +509,7 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev,
 	if (wdev->connected ||
 	    (wdev->iftype == NL80211_IFTYPE_ADHOC &&
 	     wdev->u.ibss.current_bss))
-		err = rdev_add_key(rdev, dev, idx, pairwise, addr, params);
+		err = rdev_add_key(rdev, dev, -1, idx, pairwise, addr, params);
 	else if (params->cipher != WLAN_CIPHER_SUITE_WEP40 &&
 		 params->cipher != WLAN_CIPHER_SUITE_WEP104)
 		return -EINVAL;
@@ -546,7 +546,8 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev,
 				__cfg80211_leave_ibss(rdev, wdev->netdev, true);
 				rejoin = true;
 			}
-			err = rdev_set_default_key(rdev, dev, idx, true, true);
+			err = rdev_set_default_key(rdev, dev, -1, idx, true,
+						   true);
 		}
 		if (!err) {
 			wdev->wext.default_key = idx;
@@ -561,7 +562,7 @@ static int __cfg80211_set_encryption(struct cfg80211_registered_device *rdev,
 		if (wdev->connected ||
 		    (wdev->iftype == NL80211_IFTYPE_ADHOC &&
 		     wdev->u.ibss.current_bss))
-			err = rdev_set_default_mgmt_key(rdev, dev, idx);
+			err = rdev_set_default_mgmt_key(rdev, dev, -1, idx);
 		if (!err)
 			wdev->wext.default_mgmt_key = idx;
 		return err;
@@ -632,7 +633,7 @@ static int cfg80211_wext_siwencode(struct net_device *dev,
 		if (wdev->connected ||
 		    (wdev->iftype == NL80211_IFTYPE_ADHOC &&
 		     wdev->u.ibss.current_bss))
-			err = rdev_set_default_key(rdev, dev, idx, true,
+			err = rdev_set_default_key(rdev, dev, -1, idx, true,
 						   true);
 		if (!err)
 			wdev->wext.default_key = idx;
@@ -685,6 +686,13 @@ static int cfg80211_wext_siwencodeext(struct net_device *dev,
 	    !rdev->ops->set_default_key)
 		return -EOPNOTSUPP;
 
+	wdev_lock(wdev);
+	if (wdev->valid_links) {
+		wdev_unlock(wdev);
+		return -EOPNOTSUPP;
+	}
+	wdev_unlock(wdev);
+
 	switch (ext->alg) {
 	case IW_ENCODE_ALG_NONE:
 		remove = true;
diff --git a/net/xdp/xdp_umem.c b/net/xdp/xdp_umem.c
index 869b9b9b9fad..4681e8e8ad94 100644
--- a/net/xdp/xdp_umem.c
+++ b/net/xdp/xdp_umem.c
@@ -19,8 +19,6 @@
 #include "xdp_umem.h"
 #include "xsk_queue.h"
 
-#define XDP_UMEM_MIN_CHUNK_SIZE 2048
-
 static DEFINE_IDA(umem_ida);
 
 static void xdp_umem_unpin_pages(struct xdp_umem *umem)
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 5b4ce6ba1bc7..9f0561b67c12 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -355,16 +355,15 @@ static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entr
 	return nb_pkts;
 }
 
-u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max_entries)
+u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 nb_pkts)
 {
 	struct xdp_sock *xs;
-	u32 nb_pkts;
 
 	rcu_read_lock();
 	if (!list_is_singular(&pool->xsk_tx_list)) {
 		/* Fallback to the non-batched version */
 		rcu_read_unlock();
-		return xsk_tx_peek_release_fallback(pool, max_entries);
+		return xsk_tx_peek_release_fallback(pool, nb_pkts);
 	}
 
 	xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list);
@@ -373,12 +372,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max_entries)
 		goto out;
 	}
 
-	max_entries = xskq_cons_nb_entries(xs->tx, max_entries);
-	nb_pkts = xskq_cons_read_desc_batch(xs->tx, pool, max_entries);
-	if (!nb_pkts) {
-		xs->tx->queue_empty_descs++;
-		goto out;
-	}
+	nb_pkts = xskq_cons_nb_entries(xs->tx, nb_pkts);
 
 	/* This is the backpressure mechanism for the Tx path. Try to
 	 * reserve space in the completion queue for all packets, but
@@ -386,12 +380,18 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max_entries)
 	 * packets. This avoids having to implement any buffering in
 	 * the Tx path.
 	 */
-	nb_pkts = xskq_prod_reserve_addr_batch(pool->cq, pool->tx_descs, nb_pkts);
+	nb_pkts = xskq_prod_nb_free(pool->cq, nb_pkts);
 	if (!nb_pkts)
 		goto out;
 
-	xskq_cons_release_n(xs->tx, max_entries);
+	nb_pkts = xskq_cons_read_desc_batch(xs->tx, pool, nb_pkts);
+	if (!nb_pkts) {
+		xs->tx->queue_empty_descs++;
+		goto out;
+	}
+
 	__xskq_cons_release(xs->tx);
+	xskq_prod_write_addr_batch(pool->cq, pool->tx_descs, nb_pkts);
 	xs->sk.sk_write_space(&xs->sk);
 
 out:
@@ -954,8 +954,8 @@ static int xsk_bind(struct socket *sock, struct sockaddr *addr, int addr_len)
 				goto out_unlock;
 			}
 
-			err = xp_assign_dev_shared(xs->pool, umem_xs->umem,
-						   dev, qid);
+			err = xp_assign_dev_shared(xs->pool, umem_xs, dev,
+						   qid);
 			if (err) {
 				xp_destroy(xs->pool);
 				xs->pool = NULL;
diff --git a/net/xdp/xsk_buff_pool.c b/net/xdp/xsk_buff_pool.c
index a71a8c6edf55..ed6c71826d31 100644
--- a/net/xdp/xsk_buff_pool.c
+++ b/net/xdp/xsk_buff_pool.c
@@ -212,17 +212,18 @@ err_unreg_pool:
 	return err;
 }
 
-int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_umem *umem,
+int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs,
 			 struct net_device *dev, u16 queue_id)
 {
 	u16 flags;
+	struct xdp_umem *umem = umem_xs->umem;
 
 	/* One fill and completion ring required for each queue id. */
 	if (!pool->fq || !pool->cq)
 		return -EINVAL;
 
 	flags = umem->zc ? XDP_ZEROCOPY : XDP_COPY;
-	if (pool->uses_need_wakeup)
+	if (umem_xs->pool->uses_need_wakeup)
 		flags |= XDP_USE_NEED_WAKEUP;
 
 	return xp_assign_dev(pool, dev, queue_id, flags);
diff --git a/net/xdp/xsk_queue.h b/net/xdp/xsk_queue.h
index fb20bf7207cf..c6fb6b763658 100644
--- a/net/xdp/xsk_queue.h
+++ b/net/xdp/xsk_queue.h
@@ -205,6 +205,11 @@ static inline bool xskq_cons_read_desc(struct xsk_queue *q,
 	return false;
 }
 
+static inline void xskq_cons_release_n(struct xsk_queue *q, u32 cnt)
+{
+	q->cached_cons += cnt;
+}
+
 static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool,
 					    u32 max)
 {
@@ -226,6 +231,8 @@ static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff
 		cached_cons++;
 	}
 
+	/* Release valid plus any invalid entries */
+	xskq_cons_release_n(q, cached_cons - q->cached_cons);
 	return nb_entries;
 }
 
@@ -291,11 +298,6 @@ static inline void xskq_cons_release(struct xsk_queue *q)
 	q->cached_cons++;
 }
 
-static inline void xskq_cons_release_n(struct xsk_queue *q, u32 cnt)
-{
-	q->cached_cons += cnt;
-}
-
 static inline u32 xskq_cons_present_entries(struct xsk_queue *q)
 {
 	/* No barriers needed since data is not accessed */
@@ -350,21 +352,17 @@ static inline int xskq_prod_reserve_addr(struct xsk_queue *q, u64 addr)
 	return 0;
 }
 
-static inline u32 xskq_prod_reserve_addr_batch(struct xsk_queue *q, struct xdp_desc *descs,
-					       u32 max)
+static inline void xskq_prod_write_addr_batch(struct xsk_queue *q, struct xdp_desc *descs,
+					      u32 nb_entries)
 {
 	struct xdp_umem_ring *ring = (struct xdp_umem_ring *)q->ring;
-	u32 nb_entries, i, cached_prod;
-
-	nb_entries = xskq_prod_nb_free(q, max);
+	u32 i, cached_prod;
 
 	/* A, matches D */
 	cached_prod = q->cached_prod;
 	for (i = 0; i < nb_entries; i++)
 		ring->desc[cached_prod++ & q->ring_mask] = descs[i].addr;
 	q->cached_prod = cached_prod;
-
-	return nb_entries;
 }
 
 static inline int xskq_prod_reserve_desc(struct xsk_queue *q,
diff --git a/net/xfrm/espintcp.c b/net/xfrm/espintcp.c
index 974eb97b77d2..29a540dcb5a7 100644
--- a/net/xfrm/espintcp.c
+++ b/net/xfrm/espintcp.c
@@ -91,7 +91,7 @@ static void espintcp_rcv(struct strparser *strp, struct sk_buff *skb)
 	}
 
 	/* remove header, leave non-ESP marker/SPI */
-	if (!__pskb_pull(skb, rxm->offset + 2)) {
+	if (!pskb_pull(skb, rxm->offset + 2)) {
 		XFRM_INC_STATS(sock_net(strp->sk), LINUX_MIB_XFRMINERROR);
 		kfree_skb(skb);
 		return;
diff --git a/net/xfrm/xfrm_device.c b/net/xfrm/xfrm_device.c
index 637ca8838436..5f5aafd418af 100644
--- a/net/xfrm/xfrm_device.c
+++ b/net/xfrm/xfrm_device.c
@@ -207,7 +207,8 @@ struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t featur
 EXPORT_SYMBOL_GPL(validate_xmit_xfrm);
 
 int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
-		       struct xfrm_user_offload *xuo)
+		       struct xfrm_user_offload *xuo,
+		       struct netlink_ext_ack *extack)
 {
 	int err;
 	struct dst_entry *dst;
@@ -216,15 +217,21 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
 	xfrm_address_t *saddr;
 	xfrm_address_t *daddr;
 
-	if (!x->type_offload)
+	if (!x->type_offload) {
+		NL_SET_ERR_MSG(extack, "Type doesn't support offload");
 		return -EINVAL;
+	}
 
 	/* We don't yet support UDP encapsulation and TFC padding. */
-	if (x->encap || x->tfcpad)
+	if (x->encap || x->tfcpad) {
+		NL_SET_ERR_MSG(extack, "Encapsulation and TFC padding can't be offloaded");
 		return -EINVAL;
+	}
 
-	if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND))
+	if (xuo->flags & ~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND)) {
+		NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
 		return -EINVAL;
+	}
 
 	dev = dev_get_by_index(net, xuo->ifindex);
 	if (!dev) {
@@ -256,6 +263,7 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
 
 	if (x->props.flags & XFRM_STATE_ESN &&
 	    !dev->xfrmdev_ops->xdo_dev_state_advance_esn) {
+		NL_SET_ERR_MSG(extack, "Device doesn't support offload with ESN");
 		xso->dev = NULL;
 		dev_put(dev);
 		return -EINVAL;
@@ -277,8 +285,10 @@ int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
 		xso->real_dev = NULL;
 		netdev_put(dev, &xso->dev_tracker);
 
-		if (err != -EOPNOTSUPP)
+		if (err != -EOPNOTSUPP) {
+			NL_SET_ERR_MSG(extack, "Device failed to offload this state");
 			return err;
+		}
 	}
 
 	return 0;
diff --git a/net/xfrm/xfrm_input.c b/net/xfrm/xfrm_input.c
index b2f4ec9c537f..97074f6f2bde 100644
--- a/net/xfrm/xfrm_input.c
+++ b/net/xfrm/xfrm_input.c
@@ -20,11 +20,13 @@
 #include <net/xfrm.h>
 #include <net/ip_tunnels.h>
 #include <net/ip6_tunnel.h>
+#include <net/dst_metadata.h>
 
 #include "xfrm_inout.h"
 
 struct xfrm_trans_tasklet {
-	struct tasklet_struct tasklet;
+	struct work_struct work;
+	spinlock_t queue_lock;
 	struct sk_buff_head queue;
 };
 
@@ -719,7 +721,8 @@ resume:
 		sp = skb_sec_path(skb);
 		if (sp)
 			sp->olen = 0;
-		skb_dst_drop(skb);
+		if (skb_valid_dst(skb))
+			skb_dst_drop(skb);
 		gro_cells_receive(&gro_cells, skb);
 		return 0;
 	} else {
@@ -737,7 +740,8 @@ resume:
 			sp = skb_sec_path(skb);
 			if (sp)
 				sp->olen = 0;
-			skb_dst_drop(skb);
+			if (skb_valid_dst(skb))
+				skb_dst_drop(skb);
 			gro_cells_receive(&gro_cells, skb);
 			return err;
 		}
@@ -760,18 +764,22 @@ int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
 }
 EXPORT_SYMBOL(xfrm_input_resume);
 
-static void xfrm_trans_reinject(struct tasklet_struct *t)
+static void xfrm_trans_reinject(struct work_struct *work)
 {
-	struct xfrm_trans_tasklet *trans = from_tasklet(trans, t, tasklet);
+	struct xfrm_trans_tasklet *trans = container_of(work, struct xfrm_trans_tasklet, work);
 	struct sk_buff_head queue;
 	struct sk_buff *skb;
 
 	__skb_queue_head_init(&queue);
+	spin_lock_bh(&trans->queue_lock);
 	skb_queue_splice_init(&trans->queue, &queue);
+	spin_unlock_bh(&trans->queue_lock);
 
+	local_bh_disable();
 	while ((skb = __skb_dequeue(&queue)))
 		XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net,
 					       NULL, skb);
+	local_bh_enable();
 }
 
 int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
@@ -789,8 +797,10 @@ int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
 
 	XFRM_TRANS_SKB_CB(skb)->finish = finish;
 	XFRM_TRANS_SKB_CB(skb)->net = net;
+	spin_lock_bh(&trans->queue_lock);
 	__skb_queue_tail(&trans->queue, skb);
-	tasklet_schedule(&trans->tasklet);
+	spin_unlock_bh(&trans->queue_lock);
+	schedule_work(&trans->work);
 	return 0;
 }
 EXPORT_SYMBOL(xfrm_trans_queue_net);
@@ -817,7 +827,8 @@ void __init xfrm_input_init(void)
 		struct xfrm_trans_tasklet *trans;
 
 		trans = &per_cpu(xfrm_trans_tasklet, i);
+		spin_lock_init(&trans->queue_lock);
 		__skb_queue_head_init(&trans->queue);
-		tasklet_setup(&trans->tasklet, xfrm_trans_reinject);
+		INIT_WORK(&trans->work, xfrm_trans_reinject);
 	}
 }
diff --git a/net/xfrm/xfrm_interface.c b/net/xfrm/xfrm_interface.c
index 5113fa0fbcee..5a67b120c4db 100644
--- a/net/xfrm/xfrm_interface.c
+++ b/net/xfrm/xfrm_interface.c
@@ -41,6 +41,7 @@
 #include <net/addrconf.h>
 #include <net/xfrm.h>
 #include <net/net_namespace.h>
+#include <net/dst_metadata.h>
 #include <net/netns/generic.h>
 #include <linux/etherdevice.h>
 
@@ -56,6 +57,89 @@ static const struct net_device_ops xfrmi_netdev_ops;
 struct xfrmi_net {
 	/* lists for storing interfaces in use */
 	struct xfrm_if __rcu *xfrmi[XFRMI_HASH_SIZE];
+	struct xfrm_if __rcu *collect_md_xfrmi;
+};
+
+static const struct nla_policy xfrm_lwt_policy[LWT_XFRM_MAX + 1] = {
+	[LWT_XFRM_IF_ID]	= NLA_POLICY_MIN(NLA_U32, 1),
+	[LWT_XFRM_LINK]		= NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+static void xfrmi_destroy_state(struct lwtunnel_state *lwt)
+{
+}
+
+static int xfrmi_build_state(struct net *net, struct nlattr *nla,
+			     unsigned int family, const void *cfg,
+			     struct lwtunnel_state **ts,
+			     struct netlink_ext_ack *extack)
+{
+	struct nlattr *tb[LWT_XFRM_MAX + 1];
+	struct lwtunnel_state *new_state;
+	struct xfrm_md_info *info;
+	int ret;
+
+	ret = nla_parse_nested(tb, LWT_XFRM_MAX, nla, xfrm_lwt_policy, extack);
+	if (ret < 0)
+		return ret;
+
+	if (!tb[LWT_XFRM_IF_ID]) {
+		NL_SET_ERR_MSG(extack, "if_id must be set");
+		return -EINVAL;
+	}
+
+	new_state = lwtunnel_state_alloc(sizeof(*info));
+	if (!new_state) {
+		NL_SET_ERR_MSG(extack, "failed to create encap info");
+		return -ENOMEM;
+	}
+
+	new_state->type = LWTUNNEL_ENCAP_XFRM;
+
+	info = lwt_xfrm_info(new_state);
+
+	info->if_id = nla_get_u32(tb[LWT_XFRM_IF_ID]);
+
+	if (tb[LWT_XFRM_LINK])
+		info->link = nla_get_u32(tb[LWT_XFRM_LINK]);
+
+	*ts = new_state;
+	return 0;
+}
+
+static int xfrmi_fill_encap_info(struct sk_buff *skb,
+				 struct lwtunnel_state *lwt)
+{
+	struct xfrm_md_info *info = lwt_xfrm_info(lwt);
+
+	if (nla_put_u32(skb, LWT_XFRM_IF_ID, info->if_id) ||
+	    (info->link && nla_put_u32(skb, LWT_XFRM_LINK, info->link)))
+		return -EMSGSIZE;
+
+	return 0;
+}
+
+static int xfrmi_encap_nlsize(struct lwtunnel_state *lwtstate)
+{
+	return nla_total_size(sizeof(u32)) + /* LWT_XFRM_IF_ID */
+		nla_total_size(sizeof(u32)); /* LWT_XFRM_LINK */
+}
+
+static int xfrmi_encap_cmp(struct lwtunnel_state *a, struct lwtunnel_state *b)
+{
+	struct xfrm_md_info *a_info = lwt_xfrm_info(a);
+	struct xfrm_md_info *b_info = lwt_xfrm_info(b);
+
+	return memcmp(a_info, b_info, sizeof(*a_info));
+}
+
+static const struct lwtunnel_encap_ops xfrmi_encap_ops = {
+	.build_state	= xfrmi_build_state,
+	.destroy_state	= xfrmi_destroy_state,
+	.fill_encap	= xfrmi_fill_encap_info,
+	.get_encap_size = xfrmi_encap_nlsize,
+	.cmp_encap	= xfrmi_encap_cmp,
+	.owner		= THIS_MODULE,
 };
 
 #define for_each_xfrmi_rcu(start, xi) \
@@ -77,17 +161,23 @@ static struct xfrm_if *xfrmi_lookup(struct net *net, struct xfrm_state *x)
 			return xi;
 	}
 
+	xi = rcu_dereference(xfrmn->collect_md_xfrmi);
+	if (xi && (xi->dev->flags & IFF_UP))
+		return xi;
+
 	return NULL;
 }
 
-static struct xfrm_if *xfrmi_decode_session(struct sk_buff *skb,
-					    unsigned short family)
+static bool xfrmi_decode_session(struct sk_buff *skb,
+				 unsigned short family,
+				 struct xfrm_if_decode_session_result *res)
 {
 	struct net_device *dev;
+	struct xfrm_if *xi;
 	int ifindex = 0;
 
 	if (!secpath_exists(skb) || !skb->dev)
-		return NULL;
+		return false;
 
 	switch (family) {
 	case AF_INET6:
@@ -107,11 +197,18 @@ static struct xfrm_if *xfrmi_decode_session(struct sk_buff *skb,
 	}
 
 	if (!dev || !(dev->flags & IFF_UP))
-		return NULL;
+		return false;
 	if (dev->netdev_ops != &xfrmi_netdev_ops)
-		return NULL;
+		return false;
 
-	return netdev_priv(dev);
+	xi = netdev_priv(dev);
+	res->net = xi->net;
+
+	if (xi->p.collect_md)
+		res->if_id = xfrm_input_state(skb)->if_id;
+	else
+		res->if_id = xi->p.if_id;
+	return true;
 }
 
 static void xfrmi_link(struct xfrmi_net *xfrmn, struct xfrm_if *xi)
@@ -157,7 +254,10 @@ static int xfrmi_create(struct net_device *dev)
 	if (err < 0)
 		goto out;
 
-	xfrmi_link(xfrmn, xi);
+	if (xi->p.collect_md)
+		rcu_assign_pointer(xfrmn->collect_md_xfrmi, xi);
+	else
+		xfrmi_link(xfrmn, xi);
 
 	return 0;
 
@@ -185,7 +285,10 @@ static void xfrmi_dev_uninit(struct net_device *dev)
 	struct xfrm_if *xi = netdev_priv(dev);
 	struct xfrmi_net *xfrmn = net_generic(xi->net, xfrmi_net_id);
 
-	xfrmi_unlink(xfrmn, xi);
+	if (xi->p.collect_md)
+		RCU_INIT_POINTER(xfrmn->collect_md_xfrmi, NULL);
+	else
+		xfrmi_unlink(xfrmn, xi);
 }
 
 static void xfrmi_scrub_packet(struct sk_buff *skb, bool xnet)
@@ -214,6 +317,7 @@ static int xfrmi_rcv_cb(struct sk_buff *skb, int err)
 	struct xfrm_state *x;
 	struct xfrm_if *xi;
 	bool xnet;
+	int link;
 
 	if (err && !secpath_exists(skb))
 		return 0;
@@ -224,6 +328,7 @@ static int xfrmi_rcv_cb(struct sk_buff *skb, int err)
 	if (!xi)
 		return 1;
 
+	link = skb->dev->ifindex;
 	dev = xi->dev;
 	skb->dev = dev;
 
@@ -254,6 +359,17 @@ static int xfrmi_rcv_cb(struct sk_buff *skb, int err)
 	}
 
 	xfrmi_scrub_packet(skb, xnet);
+	if (xi->p.collect_md) {
+		struct metadata_dst *md_dst;
+
+		md_dst = metadata_dst_alloc(0, METADATA_XFRM, GFP_ATOMIC);
+		if (!md_dst)
+			return -ENOMEM;
+
+		md_dst->u.xfrm_info.if_id = x->if_id;
+		md_dst->u.xfrm_info.link = link;
+		skb_dst_set(skb, (struct dst_entry *)md_dst);
+	}
 	dev_sw_netstats_rx_add(dev, skb->len);
 
 	return 0;
@@ -269,10 +385,23 @@ xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl)
 	struct net_device *tdev;
 	struct xfrm_state *x;
 	int err = -1;
+	u32 if_id;
 	int mtu;
 
+	if (xi->p.collect_md) {
+		struct xfrm_md_info *md_info = skb_xfrm_md_info(skb);
+
+		if (unlikely(!md_info))
+			return -EINVAL;
+
+		if_id = md_info->if_id;
+		fl->flowi_oif = md_info->link;
+	} else {
+		if_id = xi->p.if_id;
+	}
+
 	dst_hold(dst);
-	dst = xfrm_lookup_with_ifid(xi->net, dst, fl, NULL, 0, xi->p.if_id);
+	dst = xfrm_lookup_with_ifid(xi->net, dst, fl, NULL, 0, if_id);
 	if (IS_ERR(dst)) {
 		err = PTR_ERR(dst);
 		dst = NULL;
@@ -283,7 +412,7 @@ xfrmi_xmit2(struct sk_buff *skb, struct net_device *dev, struct flowi *fl)
 	if (!x)
 		goto tx_err_link_failure;
 
-	if (x->if_id != xi->p.if_id)
+	if (x->if_id != if_id)
 		goto tx_err_link_failure;
 
 	tdev = dst->dev;
@@ -633,6 +762,9 @@ static void xfrmi_netlink_parms(struct nlattr *data[],
 
 	if (data[IFLA_XFRM_IF_ID])
 		parms->if_id = nla_get_u32(data[IFLA_XFRM_IF_ID]);
+
+	if (data[IFLA_XFRM_COLLECT_METADATA])
+		parms->collect_md = true;
 }
 
 static int xfrmi_newlink(struct net *src_net, struct net_device *dev,
@@ -645,14 +777,27 @@ static int xfrmi_newlink(struct net *src_net, struct net_device *dev,
 	int err;
 
 	xfrmi_netlink_parms(data, &p);
-	if (!p.if_id) {
-		NL_SET_ERR_MSG(extack, "if_id must be non zero");
-		return -EINVAL;
-	}
+	if (p.collect_md) {
+		struct xfrmi_net *xfrmn = net_generic(net, xfrmi_net_id);
 
-	xi = xfrmi_locate(net, &p);
-	if (xi)
-		return -EEXIST;
+		if (p.link || p.if_id) {
+			NL_SET_ERR_MSG(extack, "link and if_id must be zero");
+			return -EINVAL;
+		}
+
+		if (rtnl_dereference(xfrmn->collect_md_xfrmi))
+			return -EEXIST;
+
+	} else {
+		if (!p.if_id) {
+			NL_SET_ERR_MSG(extack, "if_id must be non zero");
+			return -EINVAL;
+		}
+
+		xi = xfrmi_locate(net, &p);
+		if (xi)
+			return -EEXIST;
+	}
 
 	xi = netdev_priv(dev);
 	xi->p = p;
@@ -682,12 +827,22 @@ static int xfrmi_changelink(struct net_device *dev, struct nlattr *tb[],
 		return -EINVAL;
 	}
 
+	if (p.collect_md) {
+		NL_SET_ERR_MSG(extack, "collect_md can't be changed");
+		return -EINVAL;
+	}
+
 	xi = xfrmi_locate(net, &p);
 	if (!xi) {
 		xi = netdev_priv(dev);
 	} else {
 		if (xi->dev != dev)
 			return -EEXIST;
+		if (xi->p.collect_md) {
+			NL_SET_ERR_MSG(extack,
+				       "device can't be changed to collect_md");
+			return -EINVAL;
+		}
 	}
 
 	return xfrmi_update(xi, &p);
@@ -700,6 +855,8 @@ static size_t xfrmi_get_size(const struct net_device *dev)
 		nla_total_size(4) +
 		/* IFLA_XFRM_IF_ID */
 		nla_total_size(4) +
+		/* IFLA_XFRM_COLLECT_METADATA */
+		nla_total_size(0) +
 		0;
 }
 
@@ -709,7 +866,8 @@ static int xfrmi_fill_info(struct sk_buff *skb, const struct net_device *dev)
 	struct xfrm_if_parms *parm = &xi->p;
 
 	if (nla_put_u32(skb, IFLA_XFRM_LINK, parm->link) ||
-	    nla_put_u32(skb, IFLA_XFRM_IF_ID, parm->if_id))
+	    nla_put_u32(skb, IFLA_XFRM_IF_ID, parm->if_id) ||
+	    (xi->p.collect_md && nla_put_flag(skb, IFLA_XFRM_COLLECT_METADATA)))
 		goto nla_put_failure;
 	return 0;
 
@@ -725,8 +883,10 @@ static struct net *xfrmi_get_link_net(const struct net_device *dev)
 }
 
 static const struct nla_policy xfrmi_policy[IFLA_XFRM_MAX + 1] = {
-	[IFLA_XFRM_LINK]	= { .type = NLA_U32 },
-	[IFLA_XFRM_IF_ID]	= { .type = NLA_U32 },
+	[IFLA_XFRM_UNSPEC]		= { .strict_start_type = IFLA_XFRM_COLLECT_METADATA },
+	[IFLA_XFRM_LINK]		= { .type = NLA_U32 },
+	[IFLA_XFRM_IF_ID]		= { .type = NLA_U32 },
+	[IFLA_XFRM_COLLECT_METADATA]	= { .type = NLA_FLAG },
 };
 
 static struct rtnl_link_ops xfrmi_link_ops __read_mostly = {
@@ -762,6 +922,9 @@ static void __net_exit xfrmi_exit_batch_net(struct list_head *net_exit_list)
 			     xip = &xi->next)
 				unregister_netdevice_queue(xi->dev, &list);
 		}
+		xi = rtnl_dereference(xfrmn->collect_md_xfrmi);
+		if (xi)
+			unregister_netdevice_queue(xi->dev, &list);
 	}
 	unregister_netdevice_many(&list);
 	rtnl_unlock();
@@ -999,6 +1162,8 @@ static int __init xfrmi_init(void)
 	if (err < 0)
 		goto rtnl_link_failed;
 
+	lwtunnel_encap_add_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM);
+
 	xfrm_if_register_cb(&xfrm_if_cb);
 
 	return err;
@@ -1017,6 +1182,7 @@ pernet_dev_failed:
 static void __exit xfrmi_fini(void)
 {
 	xfrm_if_unregister_cb();
+	lwtunnel_encap_del_ops(&xfrmi_encap_ops, LWTUNNEL_ENCAP_XFRM);
 	rtnl_link_unregister(&xfrmi_link_ops);
 	xfrmi4_fini();
 	xfrmi6_fini();
diff --git a/net/xfrm/xfrm_ipcomp.c b/net/xfrm/xfrm_ipcomp.c
index cb40ff0ff28d..80143360bf09 100644
--- a/net/xfrm/xfrm_ipcomp.c
+++ b/net/xfrm/xfrm_ipcomp.c
@@ -203,6 +203,7 @@ static void ipcomp_free_scratches(void)
 		vfree(*per_cpu_ptr(scratches, i));
 
 	free_percpu(scratches);
+	ipcomp_scratches = NULL;
 }
 
 static void * __percpu *ipcomp_alloc_scratches(void)
@@ -325,18 +326,22 @@ void ipcomp_destroy(struct xfrm_state *x)
 }
 EXPORT_SYMBOL_GPL(ipcomp_destroy);
 
-int ipcomp_init_state(struct xfrm_state *x)
+int ipcomp_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
 	int err;
 	struct ipcomp_data *ipcd;
 	struct xfrm_algo_desc *calg_desc;
 
 	err = -EINVAL;
-	if (!x->calg)
+	if (!x->calg) {
+		NL_SET_ERR_MSG(extack, "Missing required compression algorithm");
 		goto out;
+	}
 
-	if (x->encap)
+	if (x->encap) {
+		NL_SET_ERR_MSG(extack, "IPComp is not compatible with encapsulation");
 		goto out;
+	}
 
 	err = -ENOMEM;
 	ipcd = kzalloc(sizeof(*ipcd), GFP_KERNEL);
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index cc6ab79609e2..e392d8d05e0c 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -1889,7 +1889,7 @@ EXPORT_SYMBOL(xfrm_policy_walk_done);
  */
 static int xfrm_policy_match(const struct xfrm_policy *pol,
 			     const struct flowi *fl,
-			     u8 type, u16 family, int dir, u32 if_id)
+			     u8 type, u16 family, u32 if_id)
 {
 	const struct xfrm_selector *sel = &pol->selector;
 	int ret = -ESRCH;
@@ -2014,7 +2014,7 @@ static struct xfrm_policy *
 __xfrm_policy_eval_candidates(struct hlist_head *chain,
 			      struct xfrm_policy *prefer,
 			      const struct flowi *fl,
-			      u8 type, u16 family, int dir, u32 if_id)
+			      u8 type, u16 family, u32 if_id)
 {
 	u32 priority = prefer ? prefer->priority : ~0u;
 	struct xfrm_policy *pol;
@@ -2028,7 +2028,7 @@ __xfrm_policy_eval_candidates(struct hlist_head *chain,
 		if (pol->priority > priority)
 			break;
 
-		err = xfrm_policy_match(pol, fl, type, family, dir, if_id);
+		err = xfrm_policy_match(pol, fl, type, family, if_id);
 		if (err) {
 			if (err != -ESRCH)
 				return ERR_PTR(err);
@@ -2053,7 +2053,7 @@ static struct xfrm_policy *
 xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand,
 			    struct xfrm_policy *prefer,
 			    const struct flowi *fl,
-			    u8 type, u16 family, int dir, u32 if_id)
+			    u8 type, u16 family, u32 if_id)
 {
 	struct xfrm_policy *tmp;
 	int i;
@@ -2061,8 +2061,7 @@ xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand,
 	for (i = 0; i < ARRAY_SIZE(cand->res); i++) {
 		tmp = __xfrm_policy_eval_candidates(cand->res[i],
 						    prefer,
-						    fl, type, family, dir,
-						    if_id);
+						    fl, type, family, if_id);
 		if (!tmp)
 			continue;
 
@@ -2101,7 +2100,7 @@ static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
 
 	ret = NULL;
 	hlist_for_each_entry_rcu(pol, chain, bydst) {
-		err = xfrm_policy_match(pol, fl, type, family, dir, if_id);
+		err = xfrm_policy_match(pol, fl, type, family, if_id);
 		if (err) {
 			if (err == -ESRCH)
 				continue;
@@ -2120,7 +2119,7 @@ static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
 		goto skip_inexact;
 
 	pol = xfrm_policy_eval_candidates(&cand, ret, fl, type,
-					  family, dir, if_id);
+					  family, if_id);
 	if (pol) {
 		ret = pol;
 		if (IS_ERR(pol))
@@ -3516,17 +3515,17 @@ int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
 	int xerr_idx = -1;
 	const struct xfrm_if_cb *ifcb;
 	struct sec_path *sp;
-	struct xfrm_if *xi;
 	u32 if_id = 0;
 
 	rcu_read_lock();
 	ifcb = xfrm_if_get_cb();
 
 	if (ifcb) {
-		xi = ifcb->decode_session(skb, family);
-		if (xi) {
-			if_id = xi->p.if_id;
-			net = xi->net;
+		struct xfrm_if_decode_session_result r;
+
+		if (ifcb->decode_session(skb, family, &r)) {
+			if_id = r.if_id;
+			net = r.net;
 		}
 	}
 	rcu_read_unlock();
diff --git a/net/xfrm/xfrm_replay.c b/net/xfrm/xfrm_replay.c
index 9277d81b344c..9f4d42eb090f 100644
--- a/net/xfrm/xfrm_replay.c
+++ b/net/xfrm/xfrm_replay.c
@@ -766,18 +766,22 @@ int xfrm_replay_overflow(struct xfrm_state *x, struct sk_buff *skb)
 }
 #endif
 
-int xfrm_init_replay(struct xfrm_state *x)
+int xfrm_init_replay(struct xfrm_state *x, struct netlink_ext_ack *extack)
 {
 	struct xfrm_replay_state_esn *replay_esn = x->replay_esn;
 
 	if (replay_esn) {
 		if (replay_esn->replay_window >
-		    replay_esn->bmp_len * sizeof(__u32) * 8)
+		    replay_esn->bmp_len * sizeof(__u32) * 8) {
+			NL_SET_ERR_MSG(extack, "ESN replay window is too large for the chosen bitmap size");
 			return -EINVAL;
+		}
 
 		if (x->props.flags & XFRM_STATE_ESN) {
-			if (replay_esn->replay_window == 0)
+			if (replay_esn->replay_window == 0) {
+				NL_SET_ERR_MSG(extack, "ESN replay window must be > 0");
 				return -EINVAL;
+			}
 			x->repl_mode = XFRM_REPLAY_MODE_ESN;
 		} else {
 			x->repl_mode = XFRM_REPLAY_MODE_BMP;
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 91c32a3b6924..81df34b3da6e 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -2611,7 +2611,8 @@ u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
 }
 EXPORT_SYMBOL_GPL(xfrm_state_mtu);
 
-int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload)
+int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload,
+		      struct netlink_ext_ack *extack)
 {
 	const struct xfrm_mode *inner_mode;
 	const struct xfrm_mode *outer_mode;
@@ -2626,12 +2627,16 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload)
 
 	if (x->sel.family != AF_UNSPEC) {
 		inner_mode = xfrm_get_mode(x->props.mode, x->sel.family);
-		if (inner_mode == NULL)
+		if (inner_mode == NULL) {
+			NL_SET_ERR_MSG(extack, "Requested mode not found");
 			goto error;
+		}
 
 		if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&
-		    family != x->sel.family)
+		    family != x->sel.family) {
+			NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family");
 			goto error;
+		}
 
 		x->inner_mode = *inner_mode;
 	} else {
@@ -2639,11 +2644,15 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload)
 		int iafamily = AF_INET;
 
 		inner_mode = xfrm_get_mode(x->props.mode, x->props.family);
-		if (inner_mode == NULL)
+		if (inner_mode == NULL) {
+			NL_SET_ERR_MSG(extack, "Requested mode not found");
 			goto error;
+		}
 
-		if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL))
+		if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL)) {
+			NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate an AF_UNSPEC selector");
 			goto error;
+		}
 
 		x->inner_mode = *inner_mode;
 
@@ -2658,24 +2667,27 @@ int __xfrm_init_state(struct xfrm_state *x, bool init_replay, bool offload)
 	}
 
 	x->type = xfrm_get_type(x->id.proto, family);
-	if (x->type == NULL)
+	if (x->type == NULL) {
+		NL_SET_ERR_MSG(extack, "Requested type not found");
 		goto error;
+	}
 
 	x->type_offload = xfrm_get_type_offload(x->id.proto, family, offload);
 
-	err = x->type->init_state(x);
+	err = x->type->init_state(x, extack);
 	if (err)
 		goto error;
 
 	outer_mode = xfrm_get_mode(x->props.mode, family);
 	if (!outer_mode) {
+		NL_SET_ERR_MSG(extack, "Requested mode not found");
 		err = -EPROTONOSUPPORT;
 		goto error;
 	}
 
 	x->outer_mode = *outer_mode;
 	if (init_replay) {
-		err = xfrm_init_replay(x);
+		err = xfrm_init_replay(x, extack);
 		if (err)
 			goto error;
 	}
@@ -2690,7 +2702,7 @@ int xfrm_init_state(struct xfrm_state *x)
 {
 	int err;
 
-	err = __xfrm_init_state(x, true, false);
+	err = __xfrm_init_state(x, true, false, NULL);
 	if (!err)
 		x->km.state = XFRM_STATE_VALID;
 
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index 2ff017117730..e73f9efc54c1 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -35,7 +35,8 @@
 #endif
 #include <asm/unaligned.h>
 
-static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type)
+static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type,
+			  struct netlink_ext_ack *extack)
 {
 	struct nlattr *rt = attrs[type];
 	struct xfrm_algo *algp;
@@ -44,8 +45,10 @@ static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type)
 		return 0;
 
 	algp = nla_data(rt);
-	if (nla_len(rt) < (int)xfrm_alg_len(algp))
+	if (nla_len(rt) < (int)xfrm_alg_len(algp)) {
+		NL_SET_ERR_MSG(extack, "Invalid AUTH/CRYPT/COMP attribute length");
 		return -EINVAL;
+	}
 
 	switch (type) {
 	case XFRMA_ALG_AUTH:
@@ -54,6 +57,7 @@ static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type)
 		break;
 
 	default:
+		NL_SET_ERR_MSG(extack, "Invalid algorithm attribute type");
 		return -EINVAL;
 	}
 
@@ -61,7 +65,8 @@ static int verify_one_alg(struct nlattr **attrs, enum xfrm_attr_type_t type)
 	return 0;
 }
 
-static int verify_auth_trunc(struct nlattr **attrs)
+static int verify_auth_trunc(struct nlattr **attrs,
+			     struct netlink_ext_ack *extack)
 {
 	struct nlattr *rt = attrs[XFRMA_ALG_AUTH_TRUNC];
 	struct xfrm_algo_auth *algp;
@@ -70,14 +75,16 @@ static int verify_auth_trunc(struct nlattr **attrs)
 		return 0;
 
 	algp = nla_data(rt);
-	if (nla_len(rt) < (int)xfrm_alg_auth_len(algp))
+	if (nla_len(rt) < (int)xfrm_alg_auth_len(algp)) {
+		NL_SET_ERR_MSG(extack, "Invalid AUTH_TRUNC attribute length");
 		return -EINVAL;
+	}
 
 	algp->alg_name[sizeof(algp->alg_name) - 1] = '\0';
 	return 0;
 }
 
-static int verify_aead(struct nlattr **attrs)
+static int verify_aead(struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	struct nlattr *rt = attrs[XFRMA_ALG_AEAD];
 	struct xfrm_algo_aead *algp;
@@ -86,8 +93,10 @@ static int verify_aead(struct nlattr **attrs)
 		return 0;
 
 	algp = nla_data(rt);
-	if (nla_len(rt) < (int)aead_len(algp))
+	if (nla_len(rt) < (int)aead_len(algp)) {
+		NL_SET_ERR_MSG(extack, "Invalid AEAD attribute length");
 		return -EINVAL;
+	}
 
 	algp->alg_name[sizeof(algp->alg_name) - 1] = '\0';
 	return 0;
@@ -102,7 +111,7 @@ static void verify_one_addr(struct nlattr **attrs, enum xfrm_attr_type_t type,
 		*addrp = nla_data(rt);
 }
 
-static inline int verify_sec_ctx_len(struct nlattr **attrs)
+static inline int verify_sec_ctx_len(struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	struct nlattr *rt = attrs[XFRMA_SEC_CTX];
 	struct xfrm_user_sec_ctx *uctx;
@@ -112,42 +121,59 @@ static inline int verify_sec_ctx_len(struct nlattr **attrs)
 
 	uctx = nla_data(rt);
 	if (uctx->len > nla_len(rt) ||
-	    uctx->len != (sizeof(struct xfrm_user_sec_ctx) + uctx->ctx_len))
+	    uctx->len != (sizeof(struct xfrm_user_sec_ctx) + uctx->ctx_len)) {
+		NL_SET_ERR_MSG(extack, "Invalid security context length");
 		return -EINVAL;
+	}
 
 	return 0;
 }
 
 static inline int verify_replay(struct xfrm_usersa_info *p,
-				struct nlattr **attrs)
+				struct nlattr **attrs,
+				struct netlink_ext_ack *extack)
 {
 	struct nlattr *rt = attrs[XFRMA_REPLAY_ESN_VAL];
 	struct xfrm_replay_state_esn *rs;
 
-	if (!rt)
-		return (p->flags & XFRM_STATE_ESN) ? -EINVAL : 0;
+	if (!rt) {
+		if (p->flags & XFRM_STATE_ESN) {
+			NL_SET_ERR_MSG(extack, "Missing required attribute for ESN");
+			return -EINVAL;
+		}
+		return 0;
+	}
 
 	rs = nla_data(rt);
 
-	if (rs->bmp_len > XFRMA_REPLAY_ESN_MAX / sizeof(rs->bmp[0]) / 8)
+	if (rs->bmp_len > XFRMA_REPLAY_ESN_MAX / sizeof(rs->bmp[0]) / 8) {
+		NL_SET_ERR_MSG(extack, "ESN bitmap length must be <= 128");
 		return -EINVAL;
+	}
 
 	if (nla_len(rt) < (int)xfrm_replay_state_esn_len(rs) &&
-	    nla_len(rt) != sizeof(*rs))
+	    nla_len(rt) != sizeof(*rs)) {
+		NL_SET_ERR_MSG(extack, "ESN attribute is too short to fit the full bitmap length");
 		return -EINVAL;
+	}
 
 	/* As only ESP and AH support ESN feature. */
-	if ((p->id.proto != IPPROTO_ESP) && (p->id.proto != IPPROTO_AH))
+	if ((p->id.proto != IPPROTO_ESP) && (p->id.proto != IPPROTO_AH)) {
+		NL_SET_ERR_MSG(extack, "ESN only supported for ESP and AH");
 		return -EINVAL;
+	}
 
-	if (p->replay_window != 0)
+	if (p->replay_window != 0) {
+		NL_SET_ERR_MSG(extack, "ESN not compatible with legacy replay_window");
 		return -EINVAL;
+	}
 
 	return 0;
 }
 
 static int verify_newsa_info(struct xfrm_usersa_info *p,
-			     struct nlattr **attrs)
+			     struct nlattr **attrs,
+			     struct netlink_ext_ack *extack)
 {
 	int err;
 
@@ -161,10 +187,12 @@ static int verify_newsa_info(struct xfrm_usersa_info *p,
 		break;
 #else
 		err = -EAFNOSUPPORT;
+		NL_SET_ERR_MSG(extack, "IPv6 support disabled");
 		goto out;
 #endif
 
 	default:
+		NL_SET_ERR_MSG(extack, "Invalid address family");
 		goto out;
 	}
 
@@ -173,65 +201,98 @@ static int verify_newsa_info(struct xfrm_usersa_info *p,
 		break;
 
 	case AF_INET:
-		if (p->sel.prefixlen_d > 32 || p->sel.prefixlen_s > 32)
+		if (p->sel.prefixlen_d > 32 || p->sel.prefixlen_s > 32) {
+			NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 32 for IPv4)");
 			goto out;
+		}
 
 		break;
 
 	case AF_INET6:
 #if IS_ENABLED(CONFIG_IPV6)
-		if (p->sel.prefixlen_d > 128 || p->sel.prefixlen_s > 128)
+		if (p->sel.prefixlen_d > 128 || p->sel.prefixlen_s > 128) {
+			NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 128 for IPv6)");
 			goto out;
+		}
 
 		break;
 #else
+		NL_SET_ERR_MSG(extack, "IPv6 support disabled");
 		err = -EAFNOSUPPORT;
 		goto out;
 #endif
 
 	default:
+		NL_SET_ERR_MSG(extack, "Invalid address family in selector");
 		goto out;
 	}
 
 	err = -EINVAL;
 	switch (p->id.proto) {
 	case IPPROTO_AH:
-		if ((!attrs[XFRMA_ALG_AUTH]	&&
-		     !attrs[XFRMA_ALG_AUTH_TRUNC]) ||
-		    attrs[XFRMA_ALG_AEAD]	||
+		if (!attrs[XFRMA_ALG_AUTH]	&&
+		    !attrs[XFRMA_ALG_AUTH_TRUNC]) {
+			NL_SET_ERR_MSG(extack, "Missing required attribute for AH: AUTH_TRUNC or AUTH");
+			goto out;
+		}
+
+		if (attrs[XFRMA_ALG_AEAD]	||
 		    attrs[XFRMA_ALG_CRYPT]	||
 		    attrs[XFRMA_ALG_COMP]	||
-		    attrs[XFRMA_TFCPAD])
+		    attrs[XFRMA_TFCPAD]) {
+			NL_SET_ERR_MSG(extack, "Invalid attributes for AH: AEAD, CRYPT, COMP, TFCPAD");
 			goto out;
+		}
 		break;
 
 	case IPPROTO_ESP:
-		if (attrs[XFRMA_ALG_COMP])
+		if (attrs[XFRMA_ALG_COMP]) {
+			NL_SET_ERR_MSG(extack, "Invalid attribute for ESP: COMP");
 			goto out;
+		}
+
 		if (!attrs[XFRMA_ALG_AUTH] &&
 		    !attrs[XFRMA_ALG_AUTH_TRUNC] &&
 		    !attrs[XFRMA_ALG_CRYPT] &&
-		    !attrs[XFRMA_ALG_AEAD])
+		    !attrs[XFRMA_ALG_AEAD]) {
+			NL_SET_ERR_MSG(extack, "Missing required attribute for ESP: at least one of AUTH, AUTH_TRUNC, CRYPT, AEAD");
 			goto out;
+		}
+
 		if ((attrs[XFRMA_ALG_AUTH] ||
 		     attrs[XFRMA_ALG_AUTH_TRUNC] ||
 		     attrs[XFRMA_ALG_CRYPT]) &&
-		    attrs[XFRMA_ALG_AEAD])
+		    attrs[XFRMA_ALG_AEAD]) {
+			NL_SET_ERR_MSG(extack, "Invalid attribute combination for ESP: AEAD can't be used with AUTH, AUTH_TRUNC, CRYPT");
 			goto out;
+		}
+
 		if (attrs[XFRMA_TFCPAD] &&
-		    p->mode != XFRM_MODE_TUNNEL)
+		    p->mode != XFRM_MODE_TUNNEL) {
+			NL_SET_ERR_MSG(extack, "TFC padding can only be used in tunnel mode");
 			goto out;
+		}
 		break;
 
 	case IPPROTO_COMP:
-		if (!attrs[XFRMA_ALG_COMP]	||
-		    attrs[XFRMA_ALG_AEAD]	||
+		if (!attrs[XFRMA_ALG_COMP]) {
+			NL_SET_ERR_MSG(extack, "Missing required attribute for COMP: COMP");
+			goto out;
+		}
+
+		if (attrs[XFRMA_ALG_AEAD]	||
 		    attrs[XFRMA_ALG_AUTH]	||
 		    attrs[XFRMA_ALG_AUTH_TRUNC]	||
 		    attrs[XFRMA_ALG_CRYPT]	||
-		    attrs[XFRMA_TFCPAD]		||
-		    (ntohl(p->id.spi) >= 0x10000))
+		    attrs[XFRMA_TFCPAD]) {
+			NL_SET_ERR_MSG(extack, "Invalid attributes for COMP: AEAD, AUTH, AUTH_TRUNC, CRYPT, TFCPAD");
 			goto out;
+		}
+
+		if (ntohl(p->id.spi) >= 0x10000) {
+			NL_SET_ERR_MSG(extack, "SPI is too large for COMP (must be < 0x10000)");
+			goto out;
+		}
 		break;
 
 #if IS_ENABLED(CONFIG_IPV6)
@@ -244,29 +305,36 @@ static int verify_newsa_info(struct xfrm_usersa_info *p,
 		    attrs[XFRMA_ALG_CRYPT]	||
 		    attrs[XFRMA_ENCAP]		||
 		    attrs[XFRMA_SEC_CTX]	||
-		    attrs[XFRMA_TFCPAD]		||
-		    !attrs[XFRMA_COADDR])
+		    attrs[XFRMA_TFCPAD]) {
+			NL_SET_ERR_MSG(extack, "Invalid attributes for DSTOPTS/ROUTING");
 			goto out;
+		}
+
+		if (!attrs[XFRMA_COADDR]) {
+			NL_SET_ERR_MSG(extack, "Missing required COADDR attribute for DSTOPTS/ROUTING");
+			goto out;
+		}
 		break;
 #endif
 
 	default:
+		NL_SET_ERR_MSG(extack, "Unsupported protocol");
 		goto out;
 	}
 
-	if ((err = verify_aead(attrs)))
+	if ((err = verify_aead(attrs, extack)))
 		goto out;
-	if ((err = verify_auth_trunc(attrs)))
+	if ((err = verify_auth_trunc(attrs, extack)))
 		goto out;
-	if ((err = verify_one_alg(attrs, XFRMA_ALG_AUTH)))
+	if ((err = verify_one_alg(attrs, XFRMA_ALG_AUTH, extack)))
 		goto out;
-	if ((err = verify_one_alg(attrs, XFRMA_ALG_CRYPT)))
+	if ((err = verify_one_alg(attrs, XFRMA_ALG_CRYPT, extack)))
 		goto out;
-	if ((err = verify_one_alg(attrs, XFRMA_ALG_COMP)))
+	if ((err = verify_one_alg(attrs, XFRMA_ALG_COMP, extack)))
 		goto out;
-	if ((err = verify_sec_ctx_len(attrs)))
+	if ((err = verify_sec_ctx_len(attrs, extack)))
 		goto out;
-	if ((err = verify_replay(p, attrs)))
+	if ((err = verify_replay(p, attrs, extack)))
 		goto out;
 
 	err = -EINVAL;
@@ -278,14 +346,19 @@ static int verify_newsa_info(struct xfrm_usersa_info *p,
 		break;
 
 	default:
+		NL_SET_ERR_MSG(extack, "Unsupported mode");
 		goto out;
 	}
 
 	err = 0;
 
-	if (attrs[XFRMA_MTIMER_THRESH])
-		if (!attrs[XFRMA_ENCAP])
+	if (attrs[XFRMA_MTIMER_THRESH]) {
+		if (!attrs[XFRMA_ENCAP]) {
+			NL_SET_ERR_MSG(extack, "MTIMER_THRESH attribute can only be set on ENCAP states");
 			err = -EINVAL;
+			goto out;
+		}
+	}
 
 out:
 	return err;
@@ -293,7 +366,7 @@ out:
 
 static int attach_one_algo(struct xfrm_algo **algpp, u8 *props,
 			   struct xfrm_algo_desc *(*get_byname)(const char *, int),
-			   struct nlattr *rta)
+			   struct nlattr *rta, struct netlink_ext_ack *extack)
 {
 	struct xfrm_algo *p, *ualg;
 	struct xfrm_algo_desc *algo;
@@ -304,8 +377,10 @@ static int attach_one_algo(struct xfrm_algo **algpp, u8 *props,
 	ualg = nla_data(rta);
 
 	algo = get_byname(ualg->alg_name, 1);
-	if (!algo)
+	if (!algo) {
+		NL_SET_ERR_MSG(extack, "Requested COMP algorithm not found");
 		return -ENOSYS;
+	}
 	*props = algo->desc.sadb_alg_id;
 
 	p = kmemdup(ualg, xfrm_alg_len(ualg), GFP_KERNEL);
@@ -317,7 +392,8 @@ static int attach_one_algo(struct xfrm_algo **algpp, u8 *props,
 	return 0;
 }
 
-static int attach_crypt(struct xfrm_state *x, struct nlattr *rta)
+static int attach_crypt(struct xfrm_state *x, struct nlattr *rta,
+			struct netlink_ext_ack *extack)
 {
 	struct xfrm_algo *p, *ualg;
 	struct xfrm_algo_desc *algo;
@@ -328,8 +404,10 @@ static int attach_crypt(struct xfrm_state *x, struct nlattr *rta)
 	ualg = nla_data(rta);
 
 	algo = xfrm_ealg_get_byname(ualg->alg_name, 1);
-	if (!algo)
+	if (!algo) {
+		NL_SET_ERR_MSG(extack, "Requested CRYPT algorithm not found");
 		return -ENOSYS;
+	}
 	x->props.ealgo = algo->desc.sadb_alg_id;
 
 	p = kmemdup(ualg, xfrm_alg_len(ualg), GFP_KERNEL);
@@ -343,7 +421,7 @@ static int attach_crypt(struct xfrm_state *x, struct nlattr *rta)
 }
 
 static int attach_auth(struct xfrm_algo_auth **algpp, u8 *props,
-		       struct nlattr *rta)
+		       struct nlattr *rta, struct netlink_ext_ack *extack)
 {
 	struct xfrm_algo *ualg;
 	struct xfrm_algo_auth *p;
@@ -355,8 +433,10 @@ static int attach_auth(struct xfrm_algo_auth **algpp, u8 *props,
 	ualg = nla_data(rta);
 
 	algo = xfrm_aalg_get_byname(ualg->alg_name, 1);
-	if (!algo)
+	if (!algo) {
+		NL_SET_ERR_MSG(extack, "Requested AUTH algorithm not found");
 		return -ENOSYS;
+	}
 	*props = algo->desc.sadb_alg_id;
 
 	p = kmalloc(sizeof(*p) + (ualg->alg_key_len + 7) / 8, GFP_KERNEL);
@@ -373,7 +453,7 @@ static int attach_auth(struct xfrm_algo_auth **algpp, u8 *props,
 }
 
 static int attach_auth_trunc(struct xfrm_algo_auth **algpp, u8 *props,
-			     struct nlattr *rta)
+			     struct nlattr *rta, struct netlink_ext_ack *extack)
 {
 	struct xfrm_algo_auth *p, *ualg;
 	struct xfrm_algo_desc *algo;
@@ -384,10 +464,14 @@ static int attach_auth_trunc(struct xfrm_algo_auth **algpp, u8 *props,
 	ualg = nla_data(rta);
 
 	algo = xfrm_aalg_get_byname(ualg->alg_name, 1);
-	if (!algo)
+	if (!algo) {
+		NL_SET_ERR_MSG(extack, "Requested AUTH_TRUNC algorithm not found");
 		return -ENOSYS;
-	if (ualg->alg_trunc_len > algo->uinfo.auth.icv_fullbits)
+	}
+	if (ualg->alg_trunc_len > algo->uinfo.auth.icv_fullbits) {
+		NL_SET_ERR_MSG(extack, "Invalid length requested for truncated ICV");
 		return -EINVAL;
+	}
 	*props = algo->desc.sadb_alg_id;
 
 	p = kmemdup(ualg, xfrm_alg_auth_len(ualg), GFP_KERNEL);
@@ -402,7 +486,8 @@ static int attach_auth_trunc(struct xfrm_algo_auth **algpp, u8 *props,
 	return 0;
 }
 
-static int attach_aead(struct xfrm_state *x, struct nlattr *rta)
+static int attach_aead(struct xfrm_state *x, struct nlattr *rta,
+		       struct netlink_ext_ack *extack)
 {
 	struct xfrm_algo_aead *p, *ualg;
 	struct xfrm_algo_desc *algo;
@@ -413,8 +498,10 @@ static int attach_aead(struct xfrm_state *x, struct nlattr *rta)
 	ualg = nla_data(rta);
 
 	algo = xfrm_aead_get_byname(ualg->alg_name, ualg->alg_icv_len, 1);
-	if (!algo)
+	if (!algo) {
+		NL_SET_ERR_MSG(extack, "Requested AEAD algorithm not found");
 		return -ENOSYS;
+	}
 	x->props.ealgo = algo->desc.sadb_alg_id;
 
 	p = kmemdup(ualg, aead_len(ualg), GFP_KERNEL);
@@ -579,7 +666,8 @@ static void xfrm_smark_init(struct nlattr **attrs, struct xfrm_mark *m)
 static struct xfrm_state *xfrm_state_construct(struct net *net,
 					       struct xfrm_usersa_info *p,
 					       struct nlattr **attrs,
-					       int *errp)
+					       int *errp,
+					       struct netlink_ext_ack *extack)
 {
 	struct xfrm_state *x = xfrm_state_alloc(net);
 	int err = -ENOMEM;
@@ -606,21 +694,21 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
 	if (attrs[XFRMA_SA_EXTRA_FLAGS])
 		x->props.extra_flags = nla_get_u32(attrs[XFRMA_SA_EXTRA_FLAGS]);
 
-	if ((err = attach_aead(x, attrs[XFRMA_ALG_AEAD])))
+	if ((err = attach_aead(x, attrs[XFRMA_ALG_AEAD], extack)))
 		goto error;
 	if ((err = attach_auth_trunc(&x->aalg, &x->props.aalgo,
-				     attrs[XFRMA_ALG_AUTH_TRUNC])))
+				     attrs[XFRMA_ALG_AUTH_TRUNC], extack)))
 		goto error;
 	if (!x->props.aalgo) {
 		if ((err = attach_auth(&x->aalg, &x->props.aalgo,
-				       attrs[XFRMA_ALG_AUTH])))
+				       attrs[XFRMA_ALG_AUTH], extack)))
 			goto error;
 	}
-	if ((err = attach_crypt(x, attrs[XFRMA_ALG_CRYPT])))
+	if ((err = attach_crypt(x, attrs[XFRMA_ALG_CRYPT], extack)))
 		goto error;
 	if ((err = attach_one_algo(&x->calg, &x->props.calgo,
 				   xfrm_calg_get_byname,
-				   attrs[XFRMA_ALG_COMP])))
+				   attrs[XFRMA_ALG_COMP], extack)))
 		goto error;
 
 	if (attrs[XFRMA_TFCPAD])
@@ -633,7 +721,7 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
 	if (attrs[XFRMA_IF_ID])
 		x->if_id = nla_get_u32(attrs[XFRMA_IF_ID]);
 
-	err = __xfrm_init_state(x, false, attrs[XFRMA_OFFLOAD_DEV]);
+	err = __xfrm_init_state(x, false, attrs[XFRMA_OFFLOAD_DEV], extack);
 	if (err)
 		goto error;
 
@@ -653,7 +741,7 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
 	/* sysctl_xfrm_aevent_etime is in 100ms units */
 	x->replay_maxage = (net->xfrm.sysctl_aevent_etime*HZ)/XFRM_AE_ETH_M;
 
-	if ((err = xfrm_init_replay(x)))
+	if ((err = xfrm_init_replay(x, extack)))
 		goto error;
 
 	/* override default values from above */
@@ -662,7 +750,8 @@ static struct xfrm_state *xfrm_state_construct(struct net *net,
 	/* configure the hardware if offload is requested */
 	if (attrs[XFRMA_OFFLOAD_DEV]) {
 		err = xfrm_dev_state_add(net, x,
-					 nla_data(attrs[XFRMA_OFFLOAD_DEV]));
+					 nla_data(attrs[XFRMA_OFFLOAD_DEV]),
+					 extack);
 		if (err)
 			goto error;
 	}
@@ -678,7 +767,7 @@ error_no_put:
 }
 
 static int xfrm_add_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+		       struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_usersa_info *p = nlmsg_data(nlh);
@@ -686,11 +775,11 @@ static int xfrm_add_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
 	int err;
 	struct km_event c;
 
-	err = verify_newsa_info(p, attrs);
+	err = verify_newsa_info(p, attrs, extack);
 	if (err)
 		return err;
 
-	x = xfrm_state_construct(net, p, attrs, &err);
+	x = xfrm_state_construct(net, p, attrs, &err, extack);
 	if (!x)
 		return err;
 
@@ -757,7 +846,7 @@ static struct xfrm_state *xfrm_user_state_lookup(struct net *net,
 }
 
 static int xfrm_del_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+		       struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_state *x;
@@ -1254,7 +1343,8 @@ static int build_spdinfo(struct sk_buff *skb, struct net *net,
 }
 
 static int xfrm_set_spdinfo(struct sk_buff *skb, struct nlmsghdr *nlh,
-			    struct nlattr **attrs)
+			    struct nlattr **attrs,
+			    struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrmu_spdhthresh *thresh4 = NULL;
@@ -1299,7 +1389,8 @@ static int xfrm_set_spdinfo(struct sk_buff *skb, struct nlmsghdr *nlh,
 }
 
 static int xfrm_get_spdinfo(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+			    struct nlattr **attrs,
+			    struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct sk_buff *r_skb;
@@ -1358,7 +1449,8 @@ static int build_sadinfo(struct sk_buff *skb, struct net *net,
 }
 
 static int xfrm_get_sadinfo(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+			    struct nlattr **attrs,
+			    struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct sk_buff *r_skb;
@@ -1378,7 +1470,7 @@ static int xfrm_get_sadinfo(struct sk_buff *skb, struct nlmsghdr *nlh,
 }
 
 static int xfrm_get_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+		       struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_usersa_id *p = nlmsg_data(nlh);
@@ -1402,7 +1494,8 @@ out_noput:
 }
 
 static int xfrm_alloc_userspi(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+			      struct nlattr **attrs,
+			      struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_state *x;
@@ -1477,7 +1570,7 @@ out_noput:
 	return err;
 }
 
-static int verify_policy_dir(u8 dir)
+static int verify_policy_dir(u8 dir, struct netlink_ext_ack *extack)
 {
 	switch (dir) {
 	case XFRM_POLICY_IN:
@@ -1486,13 +1579,14 @@ static int verify_policy_dir(u8 dir)
 		break;
 
 	default:
+		NL_SET_ERR_MSG(extack, "Invalid policy direction");
 		return -EINVAL;
 	}
 
 	return 0;
 }
 
-static int verify_policy_type(u8 type)
+static int verify_policy_type(u8 type, struct netlink_ext_ack *extack)
 {
 	switch (type) {
 	case XFRM_POLICY_TYPE_MAIN:
@@ -1502,13 +1596,15 @@ static int verify_policy_type(u8 type)
 		break;
 
 	default:
+		NL_SET_ERR_MSG(extack, "Invalid policy type");
 		return -EINVAL;
 	}
 
 	return 0;
 }
 
-static int verify_newpolicy_info(struct xfrm_userpolicy_info *p)
+static int verify_newpolicy_info(struct xfrm_userpolicy_info *p,
+				 struct netlink_ext_ack *extack)
 {
 	int ret;
 
@@ -1520,6 +1616,7 @@ static int verify_newpolicy_info(struct xfrm_userpolicy_info *p)
 		break;
 
 	default:
+		NL_SET_ERR_MSG(extack, "Invalid policy share");
 		return -EINVAL;
 	}
 
@@ -1529,35 +1626,44 @@ static int verify_newpolicy_info(struct xfrm_userpolicy_info *p)
 		break;
 
 	default:
+		NL_SET_ERR_MSG(extack, "Invalid policy action");
 		return -EINVAL;
 	}
 
 	switch (p->sel.family) {
 	case AF_INET:
-		if (p->sel.prefixlen_d > 32 || p->sel.prefixlen_s > 32)
+		if (p->sel.prefixlen_d > 32 || p->sel.prefixlen_s > 32) {
+			NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 32 for IPv4)");
 			return -EINVAL;
+		}
 
 		break;
 
 	case AF_INET6:
 #if IS_ENABLED(CONFIG_IPV6)
-		if (p->sel.prefixlen_d > 128 || p->sel.prefixlen_s > 128)
+		if (p->sel.prefixlen_d > 128 || p->sel.prefixlen_s > 128) {
+			NL_SET_ERR_MSG(extack, "Invalid prefix length in selector (must be <= 128 for IPv6)");
 			return -EINVAL;
+		}
 
 		break;
 #else
+		NL_SET_ERR_MSG(extack, "IPv6 support disabled");
 		return  -EAFNOSUPPORT;
 #endif
 
 	default:
+		NL_SET_ERR_MSG(extack, "Invalid selector family");
 		return -EINVAL;
 	}
 
-	ret = verify_policy_dir(p->dir);
+	ret = verify_policy_dir(p->dir, extack);
 	if (ret)
 		return ret;
-	if (p->index && (xfrm_policy_id2dir(p->index) != p->dir))
+	if (p->index && (xfrm_policy_id2dir(p->index) != p->dir)) {
+		NL_SET_ERR_MSG(extack, "Policy index doesn't match direction");
 		return -EINVAL;
+	}
 
 	return 0;
 }
@@ -1599,13 +1705,16 @@ static void copy_templates(struct xfrm_policy *xp, struct xfrm_user_tmpl *ut,
 	}
 }
 
-static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family)
+static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family,
+			 struct netlink_ext_ack *extack)
 {
 	u16 prev_family;
 	int i;
 
-	if (nr > XFRM_MAX_DEPTH)
+	if (nr > XFRM_MAX_DEPTH) {
+		NL_SET_ERR_MSG(extack, "Template count must be <= XFRM_MAX_DEPTH (" __stringify(XFRM_MAX_DEPTH) ")");
 		return -EINVAL;
+	}
 
 	prev_family = family;
 
@@ -1625,12 +1734,16 @@ static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family)
 		case XFRM_MODE_BEET:
 			break;
 		default:
-			if (ut[i].family != prev_family)
+			if (ut[i].family != prev_family) {
+				NL_SET_ERR_MSG(extack, "Mode in template doesn't support a family change");
 				return -EINVAL;
+			}
 			break;
 		}
-		if (ut[i].mode >= XFRM_MODE_MAX)
+		if (ut[i].mode >= XFRM_MODE_MAX) {
+			NL_SET_ERR_MSG(extack, "Mode in template must be < XFRM_MODE_MAX (" __stringify(XFRM_MODE_MAX) ")");
 			return -EINVAL;
+		}
 
 		prev_family = ut[i].family;
 
@@ -1642,17 +1755,21 @@ static int validate_tmpl(int nr, struct xfrm_user_tmpl *ut, u16 family)
 			break;
 #endif
 		default:
+			NL_SET_ERR_MSG(extack, "Invalid family in template");
 			return -EINVAL;
 		}
 
-		if (!xfrm_id_proto_valid(ut[i].id.proto))
+		if (!xfrm_id_proto_valid(ut[i].id.proto)) {
+			NL_SET_ERR_MSG(extack, "Invalid XFRM protocol in template");
 			return -EINVAL;
+		}
 	}
 
 	return 0;
 }
 
-static int copy_from_user_tmpl(struct xfrm_policy *pol, struct nlattr **attrs)
+static int copy_from_user_tmpl(struct xfrm_policy *pol, struct nlattr **attrs,
+			       struct netlink_ext_ack *extack)
 {
 	struct nlattr *rt = attrs[XFRMA_TMPL];
 
@@ -1663,7 +1780,7 @@ static int copy_from_user_tmpl(struct xfrm_policy *pol, struct nlattr **attrs)
 		int nr = nla_len(rt) / sizeof(*utmpl);
 		int err;
 
-		err = validate_tmpl(nr, utmpl, pol->family);
+		err = validate_tmpl(nr, utmpl, pol->family, extack);
 		if (err)
 			return err;
 
@@ -1672,7 +1789,8 @@ static int copy_from_user_tmpl(struct xfrm_policy *pol, struct nlattr **attrs)
 	return 0;
 }
 
-static int copy_from_user_policy_type(u8 *tp, struct nlattr **attrs)
+static int copy_from_user_policy_type(u8 *tp, struct nlattr **attrs,
+				      struct netlink_ext_ack *extack)
 {
 	struct nlattr *rt = attrs[XFRMA_POLICY_TYPE];
 	struct xfrm_userpolicy_type *upt;
@@ -1684,7 +1802,7 @@ static int copy_from_user_policy_type(u8 *tp, struct nlattr **attrs)
 		type = upt->type;
 	}
 
-	err = verify_policy_type(type);
+	err = verify_policy_type(type, extack);
 	if (err)
 		return err;
 
@@ -1719,7 +1837,11 @@ static void copy_to_user_policy(struct xfrm_policy *xp, struct xfrm_userpolicy_i
 	p->share = XFRM_SHARE_ANY; /* XXX xp->share */
 }
 
-static struct xfrm_policy *xfrm_policy_construct(struct net *net, struct xfrm_userpolicy_info *p, struct nlattr **attrs, int *errp)
+static struct xfrm_policy *xfrm_policy_construct(struct net *net,
+						 struct xfrm_userpolicy_info *p,
+						 struct nlattr **attrs,
+						 int *errp,
+						 struct netlink_ext_ack *extack)
 {
 	struct xfrm_policy *xp = xfrm_policy_alloc(net, GFP_KERNEL);
 	int err;
@@ -1731,11 +1853,11 @@ static struct xfrm_policy *xfrm_policy_construct(struct net *net, struct xfrm_us
 
 	copy_from_user_policy(xp, p);
 
-	err = copy_from_user_policy_type(&xp->type, attrs);
+	err = copy_from_user_policy_type(&xp->type, attrs, extack);
 	if (err)
 		goto error;
 
-	if (!(err = copy_from_user_tmpl(xp, attrs)))
+	if (!(err = copy_from_user_tmpl(xp, attrs, extack)))
 		err = copy_from_user_sec_ctx(xp, attrs);
 	if (err)
 		goto error;
@@ -1754,7 +1876,8 @@ static struct xfrm_policy *xfrm_policy_construct(struct net *net, struct xfrm_us
 }
 
 static int xfrm_add_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+			   struct nlattr **attrs,
+			   struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_userpolicy_info *p = nlmsg_data(nlh);
@@ -1763,14 +1886,14 @@ static int xfrm_add_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
 	int err;
 	int excl;
 
-	err = verify_newpolicy_info(p);
+	err = verify_newpolicy_info(p, extack);
 	if (err)
 		return err;
-	err = verify_sec_ctx_len(attrs);
+	err = verify_sec_ctx_len(attrs, extack);
 	if (err)
 		return err;
 
-	xp = xfrm_policy_construct(net, p, attrs, &err);
+	xp = xfrm_policy_construct(net, p, attrs, &err, extack);
 	if (!xp)
 		return err;
 
@@ -2015,7 +2138,7 @@ static bool xfrm_userpolicy_is_valid(__u8 policy)
 }
 
 static int xfrm_set_default(struct sk_buff *skb, struct nlmsghdr *nlh,
-			    struct nlattr **attrs)
+			    struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_userpolicy_default *up = nlmsg_data(nlh);
@@ -2036,7 +2159,7 @@ static int xfrm_set_default(struct sk_buff *skb, struct nlmsghdr *nlh,
 }
 
 static int xfrm_get_default(struct sk_buff *skb, struct nlmsghdr *nlh,
-			    struct nlattr **attrs)
+			    struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	struct sk_buff *r_skb;
 	struct nlmsghdr *r_nlh;
@@ -2066,7 +2189,8 @@ static int xfrm_get_default(struct sk_buff *skb, struct nlmsghdr *nlh,
 }
 
 static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+			   struct nlattr **attrs,
+			   struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_policy *xp;
@@ -2081,11 +2205,11 @@ static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
 	p = nlmsg_data(nlh);
 	delete = nlh->nlmsg_type == XFRM_MSG_DELPOLICY;
 
-	err = copy_from_user_policy_type(&type, attrs);
+	err = copy_from_user_policy_type(&type, attrs, extack);
 	if (err)
 		return err;
 
-	err = verify_policy_dir(p->dir);
+	err = verify_policy_dir(p->dir, extack);
 	if (err)
 		return err;
 
@@ -2101,7 +2225,7 @@ static int xfrm_get_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
 		struct nlattr *rt = attrs[XFRMA_SEC_CTX];
 		struct xfrm_sec_ctx *ctx;
 
-		err = verify_sec_ctx_len(attrs);
+		err = verify_sec_ctx_len(attrs, extack);
 		if (err)
 			return err;
 
@@ -2149,7 +2273,8 @@ out:
 }
 
 static int xfrm_flush_sa(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+			 struct nlattr **attrs,
+			 struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct km_event c;
@@ -2249,7 +2374,7 @@ out_cancel:
 }
 
 static int xfrm_get_ae(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+		       struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_state *x;
@@ -2293,7 +2418,7 @@ static int xfrm_get_ae(struct sk_buff *skb, struct nlmsghdr *nlh,
 }
 
 static int xfrm_new_ae(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+		       struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_state *x;
@@ -2344,14 +2469,15 @@ out:
 }
 
 static int xfrm_flush_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+			     struct nlattr **attrs,
+			     struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct km_event c;
 	u8 type = XFRM_POLICY_TYPE_MAIN;
 	int err;
 
-	err = copy_from_user_policy_type(&type, attrs);
+	err = copy_from_user_policy_type(&type, attrs, extack);
 	if (err)
 		return err;
 
@@ -2372,7 +2498,8 @@ static int xfrm_flush_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
 }
 
 static int xfrm_add_pol_expire(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+			       struct nlattr **attrs,
+			       struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_policy *xp;
@@ -2383,11 +2510,11 @@ static int xfrm_add_pol_expire(struct sk_buff *skb, struct nlmsghdr *nlh,
 	struct xfrm_mark m;
 	u32 if_id = 0;
 
-	err = copy_from_user_policy_type(&type, attrs);
+	err = copy_from_user_policy_type(&type, attrs, extack);
 	if (err)
 		return err;
 
-	err = verify_policy_dir(p->dir);
+	err = verify_policy_dir(p->dir, extack);
 	if (err)
 		return err;
 
@@ -2403,7 +2530,7 @@ static int xfrm_add_pol_expire(struct sk_buff *skb, struct nlmsghdr *nlh,
 		struct nlattr *rt = attrs[XFRMA_SEC_CTX];
 		struct xfrm_sec_ctx *ctx;
 
-		err = verify_sec_ctx_len(attrs);
+		err = verify_sec_ctx_len(attrs, extack);
 		if (err)
 			return err;
 
@@ -2438,7 +2565,8 @@ out:
 }
 
 static int xfrm_add_sa_expire(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+			      struct nlattr **attrs,
+			      struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_state *x;
@@ -2472,7 +2600,8 @@ out:
 }
 
 static int xfrm_add_acquire(struct sk_buff *skb, struct nlmsghdr *nlh,
-		struct nlattr **attrs)
+			    struct nlattr **attrs,
+			    struct netlink_ext_ack *extack)
 {
 	struct net *net = sock_net(skb->sk);
 	struct xfrm_policy *xp;
@@ -2490,15 +2619,15 @@ static int xfrm_add_acquire(struct sk_buff *skb, struct nlmsghdr *nlh,
 
 	xfrm_mark_get(attrs, &mark);
 
-	err = verify_newpolicy_info(&ua->policy);
+	err = verify_newpolicy_info(&ua->policy, extack);
 	if (err)
 		goto free_state;
-	err = verify_sec_ctx_len(attrs);
+	err = verify_sec_ctx_len(attrs, extack);
 	if (err)
 		goto free_state;
 
 	/*   build an XP */
-	xp = xfrm_policy_construct(net, &ua->policy, attrs, &err);
+	xp = xfrm_policy_construct(net, &ua->policy, attrs, &err, extack);
 	if (!xp)
 		goto free_state;
 
@@ -2577,7 +2706,7 @@ static int copy_from_user_migrate(struct xfrm_migrate *ma,
 }
 
 static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
-			   struct nlattr **attrs)
+			   struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	struct xfrm_userpolicy_id *pi = nlmsg_data(nlh);
 	struct xfrm_migrate m[XFRM_MAX_DEPTH];
@@ -2594,7 +2723,7 @@ static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
 
 	kmp = attrs[XFRMA_KMADDRESS] ? &km : NULL;
 
-	err = copy_from_user_policy_type(&type, attrs);
+	err = copy_from_user_policy_type(&type, attrs, extack);
 	if (err)
 		return err;
 
@@ -2623,7 +2752,7 @@ static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
 }
 #else
 static int xfrm_do_migrate(struct sk_buff *skb, struct nlmsghdr *nlh,
-			   struct nlattr **attrs)
+			   struct nlattr **attrs, struct netlink_ext_ack *extack)
 {
 	return -ENOPROTOOPT;
 }
@@ -2819,7 +2948,8 @@ static const struct nla_policy xfrma_spd_policy[XFRMA_SPD_MAX+1] = {
 };
 
 static const struct xfrm_link {
-	int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **);
+	int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **,
+		    struct netlink_ext_ack *);
 	int (*start)(struct netlink_callback *);
 	int (*dump)(struct sk_buff *, struct netlink_callback *);
 	int (*done)(struct netlink_callback *);
@@ -2921,7 +3051,7 @@ static int xfrm_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh,
 		goto err;
 	}
 
-	err = link->doit(skb, nlh, attrs);
+	err = link->doit(skb, nlh, attrs, extack);
 
 	/* We need to free skb allocated in xfrm_alloc_compat() before
 	 * returning from this function, because consume_skb() won't take
@@ -3272,11 +3402,11 @@ static struct xfrm_policy *xfrm_compile_policy(struct sock *sk, int opt,
 	*dir = -EINVAL;
 
 	if (len < sizeof(*p) ||
-	    verify_newpolicy_info(p))
+	    verify_newpolicy_info(p, NULL))
 		return NULL;
 
 	nr = ((len - sizeof(*p)) / sizeof(*ut));
-	if (validate_tmpl(nr, ut, p->sel.family))
+	if (validate_tmpl(nr, ut, p->sel.family, NULL))
 		return NULL;
 
 	if (p->dir > XFRM_POLICY_OUT)
diff --git a/rust/.gitignore b/rust/.gitignore
new file mode 100644
index 000000000000..9bd1af8e05a1
--- /dev/null
+++ b/rust/.gitignore
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+
+target.json
+bindings_generated.rs
+bindings_helpers_generated.rs
+exports_*_generated.h
+doc/
+test/
diff --git a/rust/Makefile b/rust/Makefile
new file mode 100644
index 000000000000..7700d3853404
--- /dev/null
+++ b/rust/Makefile
@@ -0,0 +1,381 @@
+# SPDX-License-Identifier: GPL-2.0
+
+always-$(CONFIG_RUST) += target.json
+no-clean-files += target.json
+
+obj-$(CONFIG_RUST) += core.o compiler_builtins.o
+always-$(CONFIG_RUST) += exports_core_generated.h
+
+# Missing prototypes are expected in the helpers since these are exported
+# for Rust only, thus there is no header nor prototypes.
+obj-$(CONFIG_RUST) += helpers.o
+CFLAGS_REMOVE_helpers.o = -Wmissing-prototypes -Wmissing-declarations
+
+always-$(CONFIG_RUST) += libmacros.so
+no-clean-files += libmacros.so
+
+always-$(CONFIG_RUST) += bindings/bindings_generated.rs bindings/bindings_helpers_generated.rs
+obj-$(CONFIG_RUST) += alloc.o bindings.o kernel.o
+always-$(CONFIG_RUST) += exports_alloc_generated.h exports_bindings_generated.h \
+    exports_kernel_generated.h
+
+obj-$(CONFIG_RUST) += exports.o
+
+# Avoids running `$(RUSTC)` for the sysroot when it may not be available.
+ifdef CONFIG_RUST
+
+# `$(rust_flags)` is passed in case the user added `--sysroot`.
+rustc_sysroot := $(shell $(RUSTC) $(rust_flags) --print sysroot)
+rustc_host_target := $(shell $(RUSTC) --version --verbose | grep -F 'host: ' | cut -d' ' -f2)
+RUST_LIB_SRC ?= $(rustc_sysroot)/lib/rustlib/src/rust/library
+
+ifeq ($(quiet),silent_)
+cargo_quiet=-q
+rust_test_quiet=-q
+rustdoc_test_quiet=--test-args -q
+else ifeq ($(quiet),quiet_)
+rust_test_quiet=-q
+rustdoc_test_quiet=--test-args -q
+else
+cargo_quiet=--verbose
+endif
+
+core-cfgs = \
+    --cfg no_fp_fmt_parse
+
+alloc-cfgs = \
+    --cfg no_fmt \
+    --cfg no_global_oom_handling \
+    --cfg no_macros \
+    --cfg no_rc \
+    --cfg no_str \
+    --cfg no_string \
+    --cfg no_sync \
+    --cfg no_thin
+
+quiet_cmd_rustdoc = RUSTDOC $(if $(rustdoc_host),H, ) $<
+      cmd_rustdoc = \
+	OBJTREE=$(abspath $(objtree)) \
+	$(RUSTDOC) $(if $(rustdoc_host),$(rust_common_flags),$(rust_flags)) \
+		$(rustc_target_flags) -L$(objtree)/$(obj) \
+		--output $(objtree)/$(obj)/doc \
+		--crate-name $(subst rustdoc-,,$@) \
+		@$(objtree)/include/generated/rustc_cfg $<
+
+# The `html_logo_url` and `html_favicon_url` forms of the `doc` attribute
+# can be used to specify a custom logo. However:
+#   - The given value is used as-is, thus it cannot be relative or a local file
+#     (unlike the non-custom case) since the generated docs have subfolders.
+#   - It requires adding it to every crate.
+#   - It requires changing `core` which comes from the sysroot.
+#
+# Using `-Zcrate-attr` would solve the last two points, but not the first.
+# The https://github.com/rust-lang/rfcs/pull/3226 RFC suggests two new
+# command-like flags to solve the issue. Meanwhile, we use the non-custom case
+# and then retouch the generated files.
+rustdoc: rustdoc-core rustdoc-macros rustdoc-compiler_builtins \
+    rustdoc-alloc rustdoc-kernel
+	$(Q)cp $(srctree)/Documentation/images/logo.svg $(objtree)/$(obj)/doc
+	$(Q)cp $(srctree)/Documentation/images/COPYING-logo $(objtree)/$(obj)/doc
+	$(Q)find $(objtree)/$(obj)/doc -name '*.html' -type f -print0 | xargs -0 sed -Ei \
+		-e 's:rust-logo\.svg:logo.svg:g' \
+		-e 's:rust-logo\.png:logo.svg:g' \
+		-e 's:favicon\.svg:logo.svg:g' \
+		-e 's:<link rel="alternate icon" type="image/png" href="[./]*favicon-(16x16|32x32)\.png">::g'
+	$(Q)echo '.logo-container > img { object-fit: contain; }' \
+		>> $(objtree)/$(obj)/doc/rustdoc.css
+
+rustdoc-macros: private rustdoc_host = yes
+rustdoc-macros: private rustc_target_flags = --crate-type proc-macro \
+    --extern proc_macro
+rustdoc-macros: $(src)/macros/lib.rs FORCE
+	$(call if_changed,rustdoc)
+
+rustdoc-core: private rustc_target_flags = $(core-cfgs)
+rustdoc-core: $(RUST_LIB_SRC)/core/src/lib.rs FORCE
+	$(call if_changed,rustdoc)
+
+rustdoc-compiler_builtins: $(src)/compiler_builtins.rs rustdoc-core FORCE
+	$(call if_changed,rustdoc)
+
+# We need to allow `rustdoc::broken_intra_doc_links` because some
+# `no_global_oom_handling` functions refer to non-`no_global_oom_handling`
+# functions. Ideally `rustdoc` would have a way to distinguish broken links
+# due to things that are "configured out" vs. entirely non-existing ones.
+rustdoc-alloc: private rustc_target_flags = $(alloc-cfgs) \
+    -Arustdoc::broken_intra_doc_links
+rustdoc-alloc: $(src)/alloc/lib.rs rustdoc-core rustdoc-compiler_builtins FORCE
+	$(call if_changed,rustdoc)
+
+rustdoc-kernel: private rustc_target_flags = --extern alloc \
+    --extern macros=$(objtree)/$(obj)/libmacros.so \
+    --extern bindings
+rustdoc-kernel: $(src)/kernel/lib.rs rustdoc-core rustdoc-macros \
+    rustdoc-compiler_builtins rustdoc-alloc $(obj)/libmacros.so \
+    $(obj)/bindings.o FORCE
+	$(call if_changed,rustdoc)
+
+quiet_cmd_rustc_test_library = RUSTC TL $<
+      cmd_rustc_test_library = \
+	OBJTREE=$(abspath $(objtree)) \
+	$(RUSTC) $(rust_common_flags) \
+		@$(objtree)/include/generated/rustc_cfg $(rustc_target_flags) \
+		--crate-type $(if $(rustc_test_library_proc),proc-macro,rlib) \
+		--out-dir $(objtree)/$(obj)/test --cfg testlib \
+		--sysroot $(objtree)/$(obj)/test/sysroot \
+		-L$(objtree)/$(obj)/test \
+		--crate-name $(subst rusttest-,,$(subst rusttestlib-,,$@)) $<
+
+rusttestlib-macros: private rustc_target_flags = --extern proc_macro
+rusttestlib-macros: private rustc_test_library_proc = yes
+rusttestlib-macros: $(src)/macros/lib.rs rusttest-prepare FORCE
+	$(call if_changed,rustc_test_library)
+
+rusttestlib-bindings: $(src)/bindings/lib.rs rusttest-prepare FORCE
+	$(call if_changed,rustc_test_library)
+
+quiet_cmd_rustdoc_test = RUSTDOC T $<
+      cmd_rustdoc_test = \
+	OBJTREE=$(abspath $(objtree)) \
+	$(RUSTDOC) --test $(rust_common_flags) \
+		@$(objtree)/include/generated/rustc_cfg \
+		$(rustc_target_flags) $(rustdoc_test_target_flags) \
+		--sysroot $(objtree)/$(obj)/test/sysroot $(rustdoc_test_quiet) \
+		-L$(objtree)/$(obj)/test --output $(objtree)/$(obj)/doc \
+		--crate-name $(subst rusttest-,,$@) $<
+
+# We cannot use `-Zpanic-abort-tests` because some tests are dynamic,
+# so for the moment we skip `-Cpanic=abort`.
+quiet_cmd_rustc_test = RUSTC T  $<
+      cmd_rustc_test = \
+	OBJTREE=$(abspath $(objtree)) \
+	$(RUSTC) --test $(rust_common_flags) \
+		@$(objtree)/include/generated/rustc_cfg \
+		$(rustc_target_flags) --out-dir $(objtree)/$(obj)/test \
+		--sysroot $(objtree)/$(obj)/test/sysroot \
+		-L$(objtree)/$(obj)/test \
+		--crate-name $(subst rusttest-,,$@) $<; \
+	$(objtree)/$(obj)/test/$(subst rusttest-,,$@) $(rust_test_quiet) \
+		$(rustc_test_run_flags)
+
+rusttest: rusttest-macros rusttest-kernel
+
+# This prepares a custom sysroot with our custom `alloc` instead of
+# the standard one.
+#
+# This requires several hacks:
+#   - Unlike `core` and `alloc`, `std` depends on more than a dozen crates,
+#     including third-party crates that need to be downloaded, plus custom
+#     `build.rs` steps. Thus hardcoding things here is not maintainable.
+#   - `cargo` knows how to build the standard library, but it is an unstable
+#     feature so far (`-Zbuild-std`).
+#   - `cargo` only considers the use case of building the standard library
+#     to use it in a given package. Thus we need to create a dummy package
+#     and pick the generated libraries from there.
+#   - Since we only keep a subset of upstream `alloc` in-tree, we need
+#     to recreate it on the fly by putting our sources on top.
+#   - The usual ways of modifying the dependency graph in `cargo` do not seem
+#     to apply for the `-Zbuild-std` steps, thus we have to mislead it
+#     by modifying the sources in the sysroot.
+#   - To avoid messing with the user's Rust installation, we create a clone
+#     of the sysroot. However, `cargo` ignores `RUSTFLAGS` in the `-Zbuild-std`
+#     steps, thus we use a wrapper binary passed via `RUSTC` to pass the flag.
+#
+# In the future, we hope to avoid the whole ordeal by either:
+#   - Making the `test` crate not depend on `std` (either improving upstream
+#     or having our own custom crate).
+#   - Making the tests run in kernel space (requires the previous point).
+#   - Making `std` and friends be more like a "normal" crate, so that
+#     `-Zbuild-std` and related hacks are not needed.
+quiet_cmd_rustsysroot = RUSTSYSROOT
+      cmd_rustsysroot = \
+	rm -rf $(objtree)/$(obj)/test; \
+	mkdir -p $(objtree)/$(obj)/test; \
+	cp -a $(rustc_sysroot) $(objtree)/$(obj)/test/sysroot; \
+	cp -r $(srctree)/$(src)/alloc/* \
+		$(objtree)/$(obj)/test/sysroot/lib/rustlib/src/rust/library/alloc/src; \
+	echo '\#!/bin/sh' > $(objtree)/$(obj)/test/rustc_sysroot; \
+	echo "$(RUSTC) --sysroot=$(abspath $(objtree)/$(obj)/test/sysroot) \"\$$@\"" \
+		>> $(objtree)/$(obj)/test/rustc_sysroot; \
+	chmod u+x $(objtree)/$(obj)/test/rustc_sysroot; \
+	$(CARGO) -q new $(objtree)/$(obj)/test/dummy; \
+	RUSTC=$(objtree)/$(obj)/test/rustc_sysroot $(CARGO) $(cargo_quiet) \
+		test -Zbuild-std --target $(rustc_host_target) \
+		--manifest-path $(objtree)/$(obj)/test/dummy/Cargo.toml; \
+	rm $(objtree)/$(obj)/test/sysroot/lib/rustlib/$(rustc_host_target)/lib/*; \
+	cp $(objtree)/$(obj)/test/dummy/target/$(rustc_host_target)/debug/deps/* \
+		$(objtree)/$(obj)/test/sysroot/lib/rustlib/$(rustc_host_target)/lib
+
+rusttest-prepare: FORCE
+	$(call if_changed,rustsysroot)
+
+rusttest-macros: private rustc_target_flags = --extern proc_macro
+rusttest-macros: private rustdoc_test_target_flags = --crate-type proc-macro
+rusttest-macros: $(src)/macros/lib.rs rusttest-prepare FORCE
+	$(call if_changed,rustc_test)
+	$(call if_changed,rustdoc_test)
+
+rusttest-kernel: private rustc_target_flags = --extern alloc \
+    --extern macros --extern bindings
+rusttest-kernel: $(src)/kernel/lib.rs rusttest-prepare \
+    rusttestlib-macros rusttestlib-bindings FORCE
+	$(call if_changed,rustc_test)
+	$(call if_changed,rustc_test_library)
+
+filechk_rust_target = $(objtree)/scripts/generate_rust_target < $<
+
+$(obj)/target.json: $(objtree)/include/config/auto.conf FORCE
+	$(call filechk,rust_target)
+
+ifdef CONFIG_CC_IS_CLANG
+bindgen_c_flags = $(c_flags)
+else
+# bindgen relies on libclang to parse C. Ideally, bindgen would support a GCC
+# plugin backend and/or the Clang driver would be perfectly compatible with GCC.
+#
+# For the moment, here we are tweaking the flags on the fly. This is a hack,
+# and some kernel configurations may not work (e.g. `GCC_PLUGIN_RANDSTRUCT`
+# if we end up using one of those structs).
+bindgen_skip_c_flags := -mno-fp-ret-in-387 -mpreferred-stack-boundary=% \
+	-mskip-rax-setup -mgeneral-regs-only -msign-return-address=% \
+	-mindirect-branch=thunk-extern -mindirect-branch-register \
+	-mfunction-return=thunk-extern -mrecord-mcount -mabi=lp64 \
+	-mindirect-branch-cs-prefix -mstack-protector-guard% -mtraceback=no \
+	-mno-pointers-to-nested-functions -mno-string \
+	-mno-strict-align -mstrict-align \
+	-fconserve-stack -falign-jumps=% -falign-loops=% \
+	-femit-struct-debug-baseonly -fno-ipa-cp-clone -fno-ipa-sra \
+	-fno-partial-inlining -fplugin-arg-arm_ssp_per_task_plugin-% \
+	-fno-reorder-blocks -fno-allow-store-data-races -fasan-shadow-offset=% \
+	-fzero-call-used-regs=% -fno-stack-clash-protection \
+	-fno-inline-functions-called-once \
+	--param=% --param asan-%
+
+# Derived from `scripts/Makefile.clang`.
+BINDGEN_TARGET_x86	:= x86_64-linux-gnu
+BINDGEN_TARGET		:= $(BINDGEN_TARGET_$(SRCARCH))
+
+# All warnings are inhibited since GCC builds are very experimental,
+# many GCC warnings are not supported by Clang, they may only appear in
+# some configurations, with new GCC versions, etc.
+bindgen_extra_c_flags = -w --target=$(BINDGEN_TARGET)
+
+bindgen_c_flags = $(filter-out $(bindgen_skip_c_flags), $(c_flags)) \
+	$(bindgen_extra_c_flags)
+endif
+
+ifdef CONFIG_LTO
+bindgen_c_flags_lto = $(filter-out $(CC_FLAGS_LTO), $(bindgen_c_flags))
+else
+bindgen_c_flags_lto = $(bindgen_c_flags)
+endif
+
+bindgen_c_flags_final = $(bindgen_c_flags_lto) -D__BINDGEN__
+
+quiet_cmd_bindgen = BINDGEN $@
+      cmd_bindgen = \
+	$(BINDGEN) $< $(bindgen_target_flags) \
+		--use-core --with-derive-default --ctypes-prefix core::ffi --no-layout-tests \
+		--no-debug '.*' \
+		--size_t-is-usize -o $@ -- $(bindgen_c_flags_final) -DMODULE \
+		$(bindgen_target_cflags) $(bindgen_target_extra)
+
+$(obj)/bindings/bindings_generated.rs: private bindgen_target_flags = \
+    $(shell grep -v '^\#\|^$$' $(srctree)/$(src)/bindgen_parameters)
+$(obj)/bindings/bindings_generated.rs: $(src)/bindings/bindings_helper.h \
+    $(src)/bindgen_parameters FORCE
+	$(call if_changed_dep,bindgen)
+
+# See `CFLAGS_REMOVE_helpers.o` above. In addition, Clang on C does not warn
+# with `-Wmissing-declarations` (unlike GCC), so it is not strictly needed here
+# given it is `libclang`; but for consistency, future Clang changes and/or
+# a potential future GCC backend for `bindgen`, we disable it too.
+$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_flags = \
+    --blacklist-type '.*' --whitelist-var '' \
+    --whitelist-function 'rust_helper_.*'
+$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_cflags = \
+    -I$(objtree)/$(obj) -Wno-missing-prototypes -Wno-missing-declarations
+$(obj)/bindings/bindings_helpers_generated.rs: private bindgen_target_extra = ; \
+    sed -Ei 's/pub fn rust_helper_([a-zA-Z0-9_]*)/#[link_name="rust_helper_\1"]\n    pub fn \1/g' $@
+$(obj)/bindings/bindings_helpers_generated.rs: $(src)/helpers.c FORCE
+	$(call if_changed_dep,bindgen)
+
+quiet_cmd_exports = EXPORTS $@
+      cmd_exports = \
+	$(NM) -p --defined-only $< \
+		| grep -E ' (T|R|D) ' | cut -d ' ' -f 3 \
+		| xargs -Isymbol \
+		echo 'EXPORT_SYMBOL_RUST_GPL(symbol);' > $@
+
+$(obj)/exports_core_generated.h: $(obj)/core.o FORCE
+	$(call if_changed,exports)
+
+$(obj)/exports_alloc_generated.h: $(obj)/alloc.o FORCE
+	$(call if_changed,exports)
+
+$(obj)/exports_bindings_generated.h: $(obj)/bindings.o FORCE
+	$(call if_changed,exports)
+
+$(obj)/exports_kernel_generated.h: $(obj)/kernel.o FORCE
+	$(call if_changed,exports)
+
+quiet_cmd_rustc_procmacro = $(RUSTC_OR_CLIPPY_QUIET) P $@
+      cmd_rustc_procmacro = \
+	$(RUSTC_OR_CLIPPY) $(rust_common_flags) \
+		--emit=dep-info,link --extern proc_macro \
+		--crate-type proc-macro --out-dir $(objtree)/$(obj) \
+		--crate-name $(patsubst lib%.so,%,$(notdir $@)) $<; \
+	mv $(objtree)/$(obj)/$(patsubst lib%.so,%,$(notdir $@)).d $(depfile); \
+	sed -i '/^\#/d' $(depfile)
+
+# Procedural macros can only be used with the `rustc` that compiled it.
+# Therefore, to get `libmacros.so` automatically recompiled when the compiler
+# version changes, we add `core.o` as a dependency (even if it is not needed).
+$(obj)/libmacros.so: $(src)/macros/lib.rs $(obj)/core.o FORCE
+	$(call if_changed_dep,rustc_procmacro)
+
+quiet_cmd_rustc_library = $(if $(skip_clippy),RUSTC,$(RUSTC_OR_CLIPPY_QUIET)) L $@
+      cmd_rustc_library = \
+	OBJTREE=$(abspath $(objtree)) \
+	$(if $(skip_clippy),$(RUSTC),$(RUSTC_OR_CLIPPY)) \
+		$(filter-out $(skip_flags),$(rust_flags) $(rustc_target_flags)) \
+		--emit=dep-info,obj,metadata --crate-type rlib \
+		--out-dir $(objtree)/$(obj) -L$(objtree)/$(obj) \
+		--crate-name $(patsubst %.o,%,$(notdir $@)) $<; \
+	mv $(objtree)/$(obj)/$(patsubst %.o,%,$(notdir $@)).d $(depfile); \
+	sed -i '/^\#/d' $(depfile) \
+	$(if $(rustc_objcopy),;$(OBJCOPY) $(rustc_objcopy) $@)
+
+rust-analyzer:
+	$(Q)$(srctree)/scripts/generate_rust_analyzer.py $(srctree) $(objtree) \
+		$(RUST_LIB_SRC) > $(objtree)/rust-project.json
+
+$(obj)/core.o: private skip_clippy = 1
+$(obj)/core.o: private skip_flags = -Dunreachable_pub
+$(obj)/core.o: private rustc_target_flags = $(core-cfgs)
+$(obj)/core.o: $(RUST_LIB_SRC)/core/src/lib.rs $(obj)/target.json FORCE
+	$(call if_changed_dep,rustc_library)
+
+$(obj)/compiler_builtins.o: private rustc_objcopy = -w -W '__*'
+$(obj)/compiler_builtins.o: $(src)/compiler_builtins.rs $(obj)/core.o FORCE
+	$(call if_changed_dep,rustc_library)
+
+$(obj)/alloc.o: private skip_clippy = 1
+$(obj)/alloc.o: private skip_flags = -Dunreachable_pub
+$(obj)/alloc.o: private rustc_target_flags = $(alloc-cfgs)
+$(obj)/alloc.o: $(src)/alloc/lib.rs $(obj)/compiler_builtins.o FORCE
+	$(call if_changed_dep,rustc_library)
+
+$(obj)/bindings.o: $(src)/bindings/lib.rs \
+    $(obj)/compiler_builtins.o \
+    $(obj)/bindings/bindings_generated.rs \
+    $(obj)/bindings/bindings_helpers_generated.rs FORCE
+	$(call if_changed_dep,rustc_library)
+
+$(obj)/kernel.o: private rustc_target_flags = --extern alloc \
+    --extern macros --extern bindings
+$(obj)/kernel.o: $(src)/kernel/lib.rs $(obj)/alloc.o \
+    $(obj)/libmacros.so $(obj)/bindings.o FORCE
+	$(call if_changed_dep,rustc_library)
+
+endif # CONFIG_RUST
diff --git a/rust/alloc/README.md b/rust/alloc/README.md
new file mode 100644
index 000000000000..c89c753720b5
--- /dev/null
+++ b/rust/alloc/README.md
@@ -0,0 +1,33 @@
+# `alloc`
+
+These source files come from the Rust standard library, hosted in
+the <https://github.com/rust-lang/rust> repository, licensed under
+"Apache-2.0 OR MIT" and adapted for kernel use. For copyright details,
+see <https://github.com/rust-lang/rust/blob/master/COPYRIGHT>.
+
+Please note that these files should be kept as close as possible to
+upstream. In general, only additions should be performed (e.g. new
+methods). Eventually, changes should make it into upstream so that,
+at some point, this fork can be dropped from the kernel tree.
+
+
+## Rationale
+
+On one hand, kernel folks wanted to keep `alloc` in-tree to have more
+freedom in both workflow and actual features if actually needed
+(e.g. receiver types if we ended up using them), which is reasonable.
+
+On the other hand, Rust folks wanted to keep `alloc` as close as
+upstream as possible and avoid as much divergence as possible, which
+is also reasonable.
+
+We agreed on a middle-ground: we would keep a subset of `alloc`
+in-tree that would be as small and as close as possible to upstream.
+Then, upstream can start adding the functions that we add to `alloc`
+etc., until we reach a point where the kernel already knows exactly
+what it needs in `alloc` and all the new methods are merged into
+upstream, so that we can drop `alloc` from the kernel tree and go back
+to using the upstream one.
+
+By doing this, the kernel can go a bit faster now, and Rust can
+slowly incorporate and discuss the changes as needed.
diff --git a/rust/alloc/alloc.rs b/rust/alloc/alloc.rs
new file mode 100644
index 000000000000..ca224a541770
--- /dev/null
+++ b/rust/alloc/alloc.rs
@@ -0,0 +1,440 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+//! Memory allocation APIs
+
+#![stable(feature = "alloc_module", since = "1.28.0")]
+
+#[cfg(not(test))]
+use core::intrinsics;
+use core::intrinsics::{min_align_of_val, size_of_val};
+
+use core::ptr::Unique;
+#[cfg(not(test))]
+use core::ptr::{self, NonNull};
+
+#[stable(feature = "alloc_module", since = "1.28.0")]
+#[doc(inline)]
+pub use core::alloc::*;
+
+use core::marker::Destruct;
+
+#[cfg(test)]
+mod tests;
+
+extern "Rust" {
+    // These are the magic symbols to call the global allocator.  rustc generates
+    // them to call `__rg_alloc` etc. if there is a `#[global_allocator]` attribute
+    // (the code expanding that attribute macro generates those functions), or to call
+    // the default implementations in libstd (`__rdl_alloc` etc. in `library/std/src/alloc.rs`)
+    // otherwise.
+    // The rustc fork of LLVM also special-cases these function names to be able to optimize them
+    // like `malloc`, `realloc`, and `free`, respectively.
+    #[rustc_allocator]
+    #[rustc_allocator_nounwind]
+    fn __rust_alloc(size: usize, align: usize) -> *mut u8;
+    #[rustc_allocator_nounwind]
+    fn __rust_dealloc(ptr: *mut u8, size: usize, align: usize);
+    #[rustc_allocator_nounwind]
+    fn __rust_realloc(ptr: *mut u8, old_size: usize, align: usize, new_size: usize) -> *mut u8;
+    #[rustc_allocator_nounwind]
+    fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8;
+}
+
+/// The global memory allocator.
+///
+/// This type implements the [`Allocator`] trait by forwarding calls
+/// to the allocator registered with the `#[global_allocator]` attribute
+/// if there is one, or the `std` crate’s default.
+///
+/// Note: while this type is unstable, the functionality it provides can be
+/// accessed through the [free functions in `alloc`](self#functions).
+#[unstable(feature = "allocator_api", issue = "32838")]
+#[derive(Copy, Clone, Default, Debug)]
+#[cfg(not(test))]
+pub struct Global;
+
+#[cfg(test)]
+pub use std::alloc::Global;
+
+/// Allocate memory with the global allocator.
+///
+/// This function forwards calls to the [`GlobalAlloc::alloc`] method
+/// of the allocator registered with the `#[global_allocator]` attribute
+/// if there is one, or the `std` crate’s default.
+///
+/// This function is expected to be deprecated in favor of the `alloc` method
+/// of the [`Global`] type when it and the [`Allocator`] trait become stable.
+///
+/// # Safety
+///
+/// See [`GlobalAlloc::alloc`].
+///
+/// # Examples
+///
+/// ```
+/// use std::alloc::{alloc, dealloc, Layout};
+///
+/// unsafe {
+///     let layout = Layout::new::<u16>();
+///     let ptr = alloc(layout);
+///
+///     *(ptr as *mut u16) = 42;
+///     assert_eq!(*(ptr as *mut u16), 42);
+///
+///     dealloc(ptr, layout);
+/// }
+/// ```
+#[stable(feature = "global_alloc", since = "1.28.0")]
+#[must_use = "losing the pointer will leak memory"]
+#[inline]
+pub unsafe fn alloc(layout: Layout) -> *mut u8 {
+    unsafe { __rust_alloc(layout.size(), layout.align()) }
+}
+
+/// Deallocate memory with the global allocator.
+///
+/// This function forwards calls to the [`GlobalAlloc::dealloc`] method
+/// of the allocator registered with the `#[global_allocator]` attribute
+/// if there is one, or the `std` crate’s default.
+///
+/// This function is expected to be deprecated in favor of the `dealloc` method
+/// of the [`Global`] type when it and the [`Allocator`] trait become stable.
+///
+/// # Safety
+///
+/// See [`GlobalAlloc::dealloc`].
+#[stable(feature = "global_alloc", since = "1.28.0")]
+#[inline]
+pub unsafe fn dealloc(ptr: *mut u8, layout: Layout) {
+    unsafe { __rust_dealloc(ptr, layout.size(), layout.align()) }
+}
+
+/// Reallocate memory with the global allocator.
+///
+/// This function forwards calls to the [`GlobalAlloc::realloc`] method
+/// of the allocator registered with the `#[global_allocator]` attribute
+/// if there is one, or the `std` crate’s default.
+///
+/// This function is expected to be deprecated in favor of the `realloc` method
+/// of the [`Global`] type when it and the [`Allocator`] trait become stable.
+///
+/// # Safety
+///
+/// See [`GlobalAlloc::realloc`].
+#[stable(feature = "global_alloc", since = "1.28.0")]
+#[must_use = "losing the pointer will leak memory"]
+#[inline]
+pub unsafe fn realloc(ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
+    unsafe { __rust_realloc(ptr, layout.size(), layout.align(), new_size) }
+}
+
+/// Allocate zero-initialized memory with the global allocator.
+///
+/// This function forwards calls to the [`GlobalAlloc::alloc_zeroed`] method
+/// of the allocator registered with the `#[global_allocator]` attribute
+/// if there is one, or the `std` crate’s default.
+///
+/// This function is expected to be deprecated in favor of the `alloc_zeroed` method
+/// of the [`Global`] type when it and the [`Allocator`] trait become stable.
+///
+/// # Safety
+///
+/// See [`GlobalAlloc::alloc_zeroed`].
+///
+/// # Examples
+///
+/// ```
+/// use std::alloc::{alloc_zeroed, dealloc, Layout};
+///
+/// unsafe {
+///     let layout = Layout::new::<u16>();
+///     let ptr = alloc_zeroed(layout);
+///
+///     assert_eq!(*(ptr as *mut u16), 0);
+///
+///     dealloc(ptr, layout);
+/// }
+/// ```
+#[stable(feature = "global_alloc", since = "1.28.0")]
+#[must_use = "losing the pointer will leak memory"]
+#[inline]
+pub unsafe fn alloc_zeroed(layout: Layout) -> *mut u8 {
+    unsafe { __rust_alloc_zeroed(layout.size(), layout.align()) }
+}
+
+#[cfg(not(test))]
+impl Global {
+    #[inline]
+    fn alloc_impl(&self, layout: Layout, zeroed: bool) -> Result<NonNull<[u8]>, AllocError> {
+        match layout.size() {
+            0 => Ok(NonNull::slice_from_raw_parts(layout.dangling(), 0)),
+            // SAFETY: `layout` is non-zero in size,
+            size => unsafe {
+                let raw_ptr = if zeroed { alloc_zeroed(layout) } else { alloc(layout) };
+                let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
+                Ok(NonNull::slice_from_raw_parts(ptr, size))
+            },
+        }
+    }
+
+    // SAFETY: Same as `Allocator::grow`
+    #[inline]
+    unsafe fn grow_impl(
+        &self,
+        ptr: NonNull<u8>,
+        old_layout: Layout,
+        new_layout: Layout,
+        zeroed: bool,
+    ) -> Result<NonNull<[u8]>, AllocError> {
+        debug_assert!(
+            new_layout.size() >= old_layout.size(),
+            "`new_layout.size()` must be greater than or equal to `old_layout.size()`"
+        );
+
+        match old_layout.size() {
+            0 => self.alloc_impl(new_layout, zeroed),
+
+            // SAFETY: `new_size` is non-zero as `old_size` is greater than or equal to `new_size`
+            // as required by safety conditions. Other conditions must be upheld by the caller
+            old_size if old_layout.align() == new_layout.align() => unsafe {
+                let new_size = new_layout.size();
+
+                // `realloc` probably checks for `new_size >= old_layout.size()` or something similar.
+                intrinsics::assume(new_size >= old_layout.size());
+
+                let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size);
+                let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
+                if zeroed {
+                    raw_ptr.add(old_size).write_bytes(0, new_size - old_size);
+                }
+                Ok(NonNull::slice_from_raw_parts(ptr, new_size))
+            },
+
+            // SAFETY: because `new_layout.size()` must be greater than or equal to `old_size`,
+            // both the old and new memory allocation are valid for reads and writes for `old_size`
+            // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap
+            // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract
+            // for `dealloc` must be upheld by the caller.
+            old_size => unsafe {
+                let new_ptr = self.alloc_impl(new_layout, zeroed)?;
+                ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), old_size);
+                self.deallocate(ptr, old_layout);
+                Ok(new_ptr)
+            },
+        }
+    }
+}
+
+#[unstable(feature = "allocator_api", issue = "32838")]
+#[cfg(not(test))]
+unsafe impl Allocator for Global {
+    #[inline]
+    fn allocate(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
+        self.alloc_impl(layout, false)
+    }
+
+    #[inline]
+    fn allocate_zeroed(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
+        self.alloc_impl(layout, true)
+    }
+
+    #[inline]
+    unsafe fn deallocate(&self, ptr: NonNull<u8>, layout: Layout) {
+        if layout.size() != 0 {
+            // SAFETY: `layout` is non-zero in size,
+            // other conditions must be upheld by the caller
+            unsafe { dealloc(ptr.as_ptr(), layout) }
+        }
+    }
+
+    #[inline]
+    unsafe fn grow(
+        &self,
+        ptr: NonNull<u8>,
+        old_layout: Layout,
+        new_layout: Layout,
+    ) -> Result<NonNull<[u8]>, AllocError> {
+        // SAFETY: all conditions must be upheld by the caller
+        unsafe { self.grow_impl(ptr, old_layout, new_layout, false) }
+    }
+
+    #[inline]
+    unsafe fn grow_zeroed(
+        &self,
+        ptr: NonNull<u8>,
+        old_layout: Layout,
+        new_layout: Layout,
+    ) -> Result<NonNull<[u8]>, AllocError> {
+        // SAFETY: all conditions must be upheld by the caller
+        unsafe { self.grow_impl(ptr, old_layout, new_layout, true) }
+    }
+
+    #[inline]
+    unsafe fn shrink(
+        &self,
+        ptr: NonNull<u8>,
+        old_layout: Layout,
+        new_layout: Layout,
+    ) -> Result<NonNull<[u8]>, AllocError> {
+        debug_assert!(
+            new_layout.size() <= old_layout.size(),
+            "`new_layout.size()` must be smaller than or equal to `old_layout.size()`"
+        );
+
+        match new_layout.size() {
+            // SAFETY: conditions must be upheld by the caller
+            0 => unsafe {
+                self.deallocate(ptr, old_layout);
+                Ok(NonNull::slice_from_raw_parts(new_layout.dangling(), 0))
+            },
+
+            // SAFETY: `new_size` is non-zero. Other conditions must be upheld by the caller
+            new_size if old_layout.align() == new_layout.align() => unsafe {
+                // `realloc` probably checks for `new_size <= old_layout.size()` or something similar.
+                intrinsics::assume(new_size <= old_layout.size());
+
+                let raw_ptr = realloc(ptr.as_ptr(), old_layout, new_size);
+                let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
+                Ok(NonNull::slice_from_raw_parts(ptr, new_size))
+            },
+
+            // SAFETY: because `new_size` must be smaller than or equal to `old_layout.size()`,
+            // both the old and new memory allocation are valid for reads and writes for `new_size`
+            // bytes. Also, because the old allocation wasn't yet deallocated, it cannot overlap
+            // `new_ptr`. Thus, the call to `copy_nonoverlapping` is safe. The safety contract
+            // for `dealloc` must be upheld by the caller.
+            new_size => unsafe {
+                let new_ptr = self.allocate(new_layout)?;
+                ptr::copy_nonoverlapping(ptr.as_ptr(), new_ptr.as_mut_ptr(), new_size);
+                self.deallocate(ptr, old_layout);
+                Ok(new_ptr)
+            },
+        }
+    }
+}
+
+/// The allocator for unique pointers.
+#[cfg(all(not(no_global_oom_handling), not(test)))]
+#[lang = "exchange_malloc"]
+#[inline]
+unsafe fn exchange_malloc(size: usize, align: usize) -> *mut u8 {
+    let layout = unsafe { Layout::from_size_align_unchecked(size, align) };
+    match Global.allocate(layout) {
+        Ok(ptr) => ptr.as_mut_ptr(),
+        Err(_) => handle_alloc_error(layout),
+    }
+}
+
+#[cfg_attr(not(test), lang = "box_free")]
+#[inline]
+#[rustc_const_unstable(feature = "const_box", issue = "92521")]
+// This signature has to be the same as `Box`, otherwise an ICE will happen.
+// When an additional parameter to `Box` is added (like `A: Allocator`), this has to be added here as
+// well.
+// For example if `Box` is changed to  `struct Box<T: ?Sized, A: Allocator>(Unique<T>, A)`,
+// this function has to be changed to `fn box_free<T: ?Sized, A: Allocator>(Unique<T>, A)` as well.
+pub(crate) const unsafe fn box_free<T: ?Sized, A: ~const Allocator + ~const Destruct>(
+    ptr: Unique<T>,
+    alloc: A,
+) {
+    unsafe {
+        let size = size_of_val(ptr.as_ref());
+        let align = min_align_of_val(ptr.as_ref());
+        let layout = Layout::from_size_align_unchecked(size, align);
+        alloc.deallocate(From::from(ptr.cast()), layout)
+    }
+}
+
+// # Allocation error handler
+
+#[cfg(not(no_global_oom_handling))]
+extern "Rust" {
+    // This is the magic symbol to call the global alloc error handler.  rustc generates
+    // it to call `__rg_oom` if there is a `#[alloc_error_handler]`, or to call the
+    // default implementations below (`__rdl_oom`) otherwise.
+    fn __rust_alloc_error_handler(size: usize, align: usize) -> !;
+}
+
+/// Abort on memory allocation error or failure.
+///
+/// Callers of memory allocation APIs wishing to abort computation
+/// in response to an allocation error are encouraged to call this function,
+/// rather than directly invoking `panic!` or similar.
+///
+/// The default behavior of this function is to print a message to standard error
+/// and abort the process.
+/// It can be replaced with [`set_alloc_error_hook`] and [`take_alloc_error_hook`].
+///
+/// [`set_alloc_error_hook`]: ../../std/alloc/fn.set_alloc_error_hook.html
+/// [`take_alloc_error_hook`]: ../../std/alloc/fn.take_alloc_error_hook.html
+#[stable(feature = "global_alloc", since = "1.28.0")]
+#[rustc_const_unstable(feature = "const_alloc_error", issue = "92523")]
+#[cfg(all(not(no_global_oom_handling), not(test)))]
+#[cold]
+pub const fn handle_alloc_error(layout: Layout) -> ! {
+    const fn ct_error(_: Layout) -> ! {
+        panic!("allocation failed");
+    }
+
+    fn rt_error(layout: Layout) -> ! {
+        unsafe {
+            __rust_alloc_error_handler(layout.size(), layout.align());
+        }
+    }
+
+    unsafe { core::intrinsics::const_eval_select((layout,), ct_error, rt_error) }
+}
+
+// For alloc test `std::alloc::handle_alloc_error` can be used directly.
+#[cfg(all(not(no_global_oom_handling), test))]
+pub use std::alloc::handle_alloc_error;
+
+#[cfg(all(not(no_global_oom_handling), not(test)))]
+#[doc(hidden)]
+#[allow(unused_attributes)]
+#[unstable(feature = "alloc_internals", issue = "none")]
+pub mod __alloc_error_handler {
+    use crate::alloc::Layout;
+
+    // called via generated `__rust_alloc_error_handler`
+
+    // if there is no `#[alloc_error_handler]`
+    #[rustc_std_internal_symbol]
+    pub unsafe extern "C-unwind" fn __rdl_oom(size: usize, _align: usize) -> ! {
+        panic!("memory allocation of {size} bytes failed")
+    }
+
+    // if there is an `#[alloc_error_handler]`
+    #[rustc_std_internal_symbol]
+    pub unsafe extern "C-unwind" fn __rg_oom(size: usize, align: usize) -> ! {
+        let layout = unsafe { Layout::from_size_align_unchecked(size, align) };
+        extern "Rust" {
+            #[lang = "oom"]
+            fn oom_impl(layout: Layout) -> !;
+        }
+        unsafe { oom_impl(layout) }
+    }
+}
+
+/// Specialize clones into pre-allocated, uninitialized memory.
+/// Used by `Box::clone` and `Rc`/`Arc::make_mut`.
+pub(crate) trait WriteCloneIntoRaw: Sized {
+    unsafe fn write_clone_into_raw(&self, target: *mut Self);
+}
+
+impl<T: Clone> WriteCloneIntoRaw for T {
+    #[inline]
+    default unsafe fn write_clone_into_raw(&self, target: *mut Self) {
+        // Having allocated *first* may allow the optimizer to create
+        // the cloned value in-place, skipping the local and move.
+        unsafe { target.write(self.clone()) };
+    }
+}
+
+impl<T: Copy> WriteCloneIntoRaw for T {
+    #[inline]
+    unsafe fn write_clone_into_raw(&self, target: *mut Self) {
+        // We can always copy in-place, without ever involving a local value.
+        unsafe { target.copy_from_nonoverlapping(self, 1) };
+    }
+}
diff --git a/rust/alloc/borrow.rs b/rust/alloc/borrow.rs
new file mode 100644
index 000000000000..dde4957200d4
--- /dev/null
+++ b/rust/alloc/borrow.rs
@@ -0,0 +1,498 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+//! A module for working with borrowed data.
+
+#![stable(feature = "rust1", since = "1.0.0")]
+
+use core::cmp::Ordering;
+use core::hash::{Hash, Hasher};
+use core::ops::Deref;
+#[cfg(not(no_global_oom_handling))]
+use core::ops::{Add, AddAssign};
+
+#[stable(feature = "rust1", since = "1.0.0")]
+pub use core::borrow::{Borrow, BorrowMut};
+
+use core::fmt;
+#[cfg(not(no_global_oom_handling))]
+use crate::string::String;
+
+use Cow::*;
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<'a, B: ?Sized> Borrow<B> for Cow<'a, B>
+where
+    B: ToOwned,
+    <B as ToOwned>::Owned: 'a,
+{
+    fn borrow(&self) -> &B {
+        &**self
+    }
+}
+
+/// A generalization of `Clone` to borrowed data.
+///
+/// Some types make it possible to go from borrowed to owned, usually by
+/// implementing the `Clone` trait. But `Clone` works only for going from `&T`
+/// to `T`. The `ToOwned` trait generalizes `Clone` to construct owned data
+/// from any borrow of a given type.
+#[cfg_attr(not(test), rustc_diagnostic_item = "ToOwned")]
+#[stable(feature = "rust1", since = "1.0.0")]
+pub trait ToOwned {
+    /// The resulting type after obtaining ownership.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    type Owned: Borrow<Self>;
+
+    /// Creates owned data from borrowed data, usually by cloning.
+    ///
+    /// # Examples
+    ///
+    /// Basic usage:
+    ///
+    /// ```
+    /// let s: &str = "a";
+    /// let ss: String = s.to_owned();
+    ///
+    /// let v: &[i32] = &[1, 2];
+    /// let vv: Vec<i32> = v.to_owned();
+    /// ```
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[must_use = "cloning is often expensive and is not expected to have side effects"]
+    fn to_owned(&self) -> Self::Owned;
+
+    /// Uses borrowed data to replace owned data, usually by cloning.
+    ///
+    /// This is borrow-generalized version of `Clone::clone_from`.
+    ///
+    /// # Examples
+    ///
+    /// Basic usage:
+    ///
+    /// ```
+    /// # #![feature(toowned_clone_into)]
+    /// let mut s: String = String::new();
+    /// "hello".clone_into(&mut s);
+    ///
+    /// let mut v: Vec<i32> = Vec::new();
+    /// [1, 2][..].clone_into(&mut v);
+    /// ```
+    #[unstable(feature = "toowned_clone_into", reason = "recently added", issue = "41263")]
+    fn clone_into(&self, target: &mut Self::Owned) {
+        *target = self.to_owned();
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T> ToOwned for T
+where
+    T: Clone,
+{
+    type Owned = T;
+    fn to_owned(&self) -> T {
+        self.clone()
+    }
+
+    fn clone_into(&self, target: &mut T) {
+        target.clone_from(self);
+    }
+}
+
+/// A clone-on-write smart pointer.
+///
+/// The type `Cow` is a smart pointer providing clone-on-write functionality: it
+/// can enclose and provide immutable access to borrowed data, and clone the
+/// data lazily when mutation or ownership is required. The type is designed to
+/// work with general borrowed data via the `Borrow` trait.
+///
+/// `Cow` implements `Deref`, which means that you can call
+/// non-mutating methods directly on the data it encloses. If mutation
+/// is desired, `to_mut` will obtain a mutable reference to an owned
+/// value, cloning if necessary.
+///
+/// If you need reference-counting pointers, note that
+/// [`Rc::make_mut`][crate::rc::Rc::make_mut] and
+/// [`Arc::make_mut`][crate::sync::Arc::make_mut] can provide clone-on-write
+/// functionality as well.
+///
+/// # Examples
+///
+/// ```
+/// use std::borrow::Cow;
+///
+/// fn abs_all(input: &mut Cow<[i32]>) {
+///     for i in 0..input.len() {
+///         let v = input[i];
+///         if v < 0 {
+///             // Clones into a vector if not already owned.
+///             input.to_mut()[i] = -v;
+///         }
+///     }
+/// }
+///
+/// // No clone occurs because `input` doesn't need to be mutated.
+/// let slice = [0, 1, 2];
+/// let mut input = Cow::from(&slice[..]);
+/// abs_all(&mut input);
+///
+/// // Clone occurs because `input` needs to be mutated.
+/// let slice = [-1, 0, 1];
+/// let mut input = Cow::from(&slice[..]);
+/// abs_all(&mut input);
+///
+/// // No clone occurs because `input` is already owned.
+/// let mut input = Cow::from(vec![-1, 0, 1]);
+/// abs_all(&mut input);
+/// ```
+///
+/// Another example showing how to keep `Cow` in a struct:
+///
+/// ```
+/// use std::borrow::Cow;
+///
+/// struct Items<'a, X: 'a> where [X]: ToOwned<Owned = Vec<X>> {
+///     values: Cow<'a, [X]>,
+/// }
+///
+/// impl<'a, X: Clone + 'a> Items<'a, X> where [X]: ToOwned<Owned = Vec<X>> {
+///     fn new(v: Cow<'a, [X]>) -> Self {
+///         Items { values: v }
+///     }
+/// }
+///
+/// // Creates a container from borrowed values of a slice
+/// let readonly = [1, 2];
+/// let borrowed = Items::new((&readonly[..]).into());
+/// match borrowed {
+///     Items { values: Cow::Borrowed(b) } => println!("borrowed {b:?}"),
+///     _ => panic!("expect borrowed value"),
+/// }
+///
+/// let mut clone_on_write = borrowed;
+/// // Mutates the data from slice into owned vec and pushes a new value on top
+/// clone_on_write.values.to_mut().push(3);
+/// println!("clone_on_write = {:?}", clone_on_write.values);
+///
+/// // The data was mutated. Let's check it out.
+/// match clone_on_write {
+///     Items { values: Cow::Owned(_) } => println!("clone_on_write contains owned data"),
+///     _ => panic!("expect owned data"),
+/// }
+/// ```
+#[stable(feature = "rust1", since = "1.0.0")]
+#[cfg_attr(not(test), rustc_diagnostic_item = "Cow")]
+pub enum Cow<'a, B: ?Sized + 'a>
+where
+    B: ToOwned,
+{
+    /// Borrowed data.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    Borrowed(#[stable(feature = "rust1", since = "1.0.0")] &'a B),
+
+    /// Owned data.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    Owned(#[stable(feature = "rust1", since = "1.0.0")] <B as ToOwned>::Owned),
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<B: ?Sized + ToOwned> Clone for Cow<'_, B> {
+    fn clone(&self) -> Self {
+        match *self {
+            Borrowed(b) => Borrowed(b),
+            Owned(ref o) => {
+                let b: &B = o.borrow();
+                Owned(b.to_owned())
+            }
+        }
+    }
+
+    fn clone_from(&mut self, source: &Self) {
+        match (self, source) {
+            (&mut Owned(ref mut dest), &Owned(ref o)) => o.borrow().clone_into(dest),
+            (t, s) => *t = s.clone(),
+        }
+    }
+}
+
+impl<B: ?Sized + ToOwned> Cow<'_, B> {
+    /// Returns true if the data is borrowed, i.e. if `to_mut` would require additional work.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(cow_is_borrowed)]
+    /// use std::borrow::Cow;
+    ///
+    /// let cow = Cow::Borrowed("moo");
+    /// assert!(cow.is_borrowed());
+    ///
+    /// let bull: Cow<'_, str> = Cow::Owned("...moo?".to_string());
+    /// assert!(!bull.is_borrowed());
+    /// ```
+    #[unstable(feature = "cow_is_borrowed", issue = "65143")]
+    #[rustc_const_unstable(feature = "const_cow_is_borrowed", issue = "65143")]
+    pub const fn is_borrowed(&self) -> bool {
+        match *self {
+            Borrowed(_) => true,
+            Owned(_) => false,
+        }
+    }
+
+    /// Returns true if the data is owned, i.e. if `to_mut` would be a no-op.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(cow_is_borrowed)]
+    /// use std::borrow::Cow;
+    ///
+    /// let cow: Cow<'_, str> = Cow::Owned("moo".to_string());
+    /// assert!(cow.is_owned());
+    ///
+    /// let bull = Cow::Borrowed("...moo?");
+    /// assert!(!bull.is_owned());
+    /// ```
+    #[unstable(feature = "cow_is_borrowed", issue = "65143")]
+    #[rustc_const_unstable(feature = "const_cow_is_borrowed", issue = "65143")]
+    pub const fn is_owned(&self) -> bool {
+        !self.is_borrowed()
+    }
+
+    /// Acquires a mutable reference to the owned form of the data.
+    ///
+    /// Clones the data if it is not already owned.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::borrow::Cow;
+    ///
+    /// let mut cow = Cow::Borrowed("foo");
+    /// cow.to_mut().make_ascii_uppercase();
+    ///
+    /// assert_eq!(
+    ///   cow,
+    ///   Cow::Owned(String::from("FOO")) as Cow<str>
+    /// );
+    /// ```
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn to_mut(&mut self) -> &mut <B as ToOwned>::Owned {
+        match *self {
+            Borrowed(borrowed) => {
+                *self = Owned(borrowed.to_owned());
+                match *self {
+                    Borrowed(..) => unreachable!(),
+                    Owned(ref mut owned) => owned,
+                }
+            }
+            Owned(ref mut owned) => owned,
+        }
+    }
+
+    /// Extracts the owned data.
+    ///
+    /// Clones the data if it is not already owned.
+    ///
+    /// # Examples
+    ///
+    /// Calling `into_owned` on a `Cow::Borrowed` returns a clone of the borrowed data:
+    ///
+    /// ```
+    /// use std::borrow::Cow;
+    ///
+    /// let s = "Hello world!";
+    /// let cow = Cow::Borrowed(s);
+    ///
+    /// assert_eq!(
+    ///   cow.into_owned(),
+    ///   String::from(s)
+    /// );
+    /// ```
+    ///
+    /// Calling `into_owned` on a `Cow::Owned` returns the owned data. The data is moved out of the
+    /// `Cow` without being cloned.
+    ///
+    /// ```
+    /// use std::borrow::Cow;
+    ///
+    /// let s = "Hello world!";
+    /// let cow: Cow<str> = Cow::Owned(String::from(s));
+    ///
+    /// assert_eq!(
+    ///   cow.into_owned(),
+    ///   String::from(s)
+    /// );
+    /// ```
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn into_owned(self) -> <B as ToOwned>::Owned {
+        match self {
+            Borrowed(borrowed) => borrowed.to_owned(),
+            Owned(owned) => owned,
+        }
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+#[rustc_const_unstable(feature = "const_deref", issue = "88955")]
+impl<B: ?Sized + ToOwned> const Deref for Cow<'_, B>
+where
+    B::Owned: ~const Borrow<B>,
+{
+    type Target = B;
+
+    fn deref(&self) -> &B {
+        match *self {
+            Borrowed(borrowed) => borrowed,
+            Owned(ref owned) => owned.borrow(),
+        }
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<B: ?Sized> Eq for Cow<'_, B> where B: Eq + ToOwned {}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<B: ?Sized> Ord for Cow<'_, B>
+where
+    B: Ord + ToOwned,
+{
+    #[inline]
+    fn cmp(&self, other: &Self) -> Ordering {
+        Ord::cmp(&**self, &**other)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<'a, 'b, B: ?Sized, C: ?Sized> PartialEq<Cow<'b, C>> for Cow<'a, B>
+where
+    B: PartialEq<C> + ToOwned,
+    C: ToOwned,
+{
+    #[inline]
+    fn eq(&self, other: &Cow<'b, C>) -> bool {
+        PartialEq::eq(&**self, &**other)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<'a, B: ?Sized> PartialOrd for Cow<'a, B>
+where
+    B: PartialOrd + ToOwned,
+{
+    #[inline]
+    fn partial_cmp(&self, other: &Cow<'a, B>) -> Option<Ordering> {
+        PartialOrd::partial_cmp(&**self, &**other)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<B: ?Sized> fmt::Debug for Cow<'_, B>
+where
+    B: fmt::Debug + ToOwned<Owned: fmt::Debug>,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match *self {
+            Borrowed(ref b) => fmt::Debug::fmt(b, f),
+            Owned(ref o) => fmt::Debug::fmt(o, f),
+        }
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<B: ?Sized> fmt::Display for Cow<'_, B>
+where
+    B: fmt::Display + ToOwned<Owned: fmt::Display>,
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match *self {
+            Borrowed(ref b) => fmt::Display::fmt(b, f),
+            Owned(ref o) => fmt::Display::fmt(o, f),
+        }
+    }
+}
+
+#[stable(feature = "default", since = "1.11.0")]
+impl<B: ?Sized> Default for Cow<'_, B>
+where
+    B: ToOwned<Owned: Default>,
+{
+    /// Creates an owned Cow<'a, B> with the default value for the contained owned value.
+    fn default() -> Self {
+        Owned(<B as ToOwned>::Owned::default())
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<B: ?Sized> Hash for Cow<'_, B>
+where
+    B: Hash + ToOwned,
+{
+    #[inline]
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        Hash::hash(&**self, state)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: ?Sized + ToOwned> AsRef<T> for Cow<'_, T> {
+    fn as_ref(&self) -> &T {
+        self
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "cow_add", since = "1.14.0")]
+impl<'a> Add<&'a str> for Cow<'a, str> {
+    type Output = Cow<'a, str>;
+
+    #[inline]
+    fn add(mut self, rhs: &'a str) -> Self::Output {
+        self += rhs;
+        self
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "cow_add", since = "1.14.0")]
+impl<'a> Add<Cow<'a, str>> for Cow<'a, str> {
+    type Output = Cow<'a, str>;
+
+    #[inline]
+    fn add(mut self, rhs: Cow<'a, str>) -> Self::Output {
+        self += rhs;
+        self
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "cow_add", since = "1.14.0")]
+impl<'a> AddAssign<&'a str> for Cow<'a, str> {
+    fn add_assign(&mut self, rhs: &'a str) {
+        if self.is_empty() {
+            *self = Cow::Borrowed(rhs)
+        } else if !rhs.is_empty() {
+            if let Cow::Borrowed(lhs) = *self {
+                let mut s = String::with_capacity(lhs.len() + rhs.len());
+                s.push_str(lhs);
+                *self = Cow::Owned(s);
+            }
+            self.to_mut().push_str(rhs);
+        }
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "cow_add", since = "1.14.0")]
+impl<'a> AddAssign<Cow<'a, str>> for Cow<'a, str> {
+    fn add_assign(&mut self, rhs: Cow<'a, str>) {
+        if self.is_empty() {
+            *self = rhs
+        } else if !rhs.is_empty() {
+            if let Cow::Borrowed(lhs) = *self {
+                let mut s = String::with_capacity(lhs.len() + rhs.len());
+                s.push_str(lhs);
+                *self = Cow::Owned(s);
+            }
+            self.to_mut().push_str(&rhs);
+        }
+    }
+}
diff --git a/rust/alloc/boxed.rs b/rust/alloc/boxed.rs
new file mode 100644
index 000000000000..dcfe87b14f3a
--- /dev/null
+++ b/rust/alloc/boxed.rs
@@ -0,0 +1,2028 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+//! A pointer type for heap allocation.
+//!
+//! [`Box<T>`], casually referred to as a 'box', provides the simplest form of
+//! heap allocation in Rust. Boxes provide ownership for this allocation, and
+//! drop their contents when they go out of scope. Boxes also ensure that they
+//! never allocate more than `isize::MAX` bytes.
+//!
+//! # Examples
+//!
+//! Move a value from the stack to the heap by creating a [`Box`]:
+//!
+//! ```
+//! let val: u8 = 5;
+//! let boxed: Box<u8> = Box::new(val);
+//! ```
+//!
+//! Move a value from a [`Box`] back to the stack by [dereferencing]:
+//!
+//! ```
+//! let boxed: Box<u8> = Box::new(5);
+//! let val: u8 = *boxed;
+//! ```
+//!
+//! Creating a recursive data structure:
+//!
+//! ```
+//! #[derive(Debug)]
+//! enum List<T> {
+//!     Cons(T, Box<List<T>>),
+//!     Nil,
+//! }
+//!
+//! let list: List<i32> = List::Cons(1, Box::new(List::Cons(2, Box::new(List::Nil))));
+//! println!("{list:?}");
+//! ```
+//!
+//! This will print `Cons(1, Cons(2, Nil))`.
+//!
+//! Recursive structures must be boxed, because if the definition of `Cons`
+//! looked like this:
+//!
+//! ```compile_fail,E0072
+//! # enum List<T> {
+//! Cons(T, List<T>),
+//! # }
+//! ```
+//!
+//! It wouldn't work. This is because the size of a `List` depends on how many
+//! elements are in the list, and so we don't know how much memory to allocate
+//! for a `Cons`. By introducing a [`Box<T>`], which has a defined size, we know how
+//! big `Cons` needs to be.
+//!
+//! # Memory layout
+//!
+//! For non-zero-sized values, a [`Box`] will use the [`Global`] allocator for
+//! its allocation. It is valid to convert both ways between a [`Box`] and a
+//! raw pointer allocated with the [`Global`] allocator, given that the
+//! [`Layout`] used with the allocator is correct for the type. More precisely,
+//! a `value: *mut T` that has been allocated with the [`Global`] allocator
+//! with `Layout::for_value(&*value)` may be converted into a box using
+//! [`Box::<T>::from_raw(value)`]. Conversely, the memory backing a `value: *mut
+//! T` obtained from [`Box::<T>::into_raw`] may be deallocated using the
+//! [`Global`] allocator with [`Layout::for_value(&*value)`].
+//!
+//! For zero-sized values, the `Box` pointer still has to be [valid] for reads
+//! and writes and sufficiently aligned. In particular, casting any aligned
+//! non-zero integer literal to a raw pointer produces a valid pointer, but a
+//! pointer pointing into previously allocated memory that since got freed is
+//! not valid. The recommended way to build a Box to a ZST if `Box::new` cannot
+//! be used is to use [`ptr::NonNull::dangling`].
+//!
+//! So long as `T: Sized`, a `Box<T>` is guaranteed to be represented
+//! as a single pointer and is also ABI-compatible with C pointers
+//! (i.e. the C type `T*`). This means that if you have extern "C"
+//! Rust functions that will be called from C, you can define those
+//! Rust functions using `Box<T>` types, and use `T*` as corresponding
+//! type on the C side. As an example, consider this C header which
+//! declares functions that create and destroy some kind of `Foo`
+//! value:
+//!
+//! ```c
+//! /* C header */
+//!
+//! /* Returns ownership to the caller */
+//! struct Foo* foo_new(void);
+//!
+//! /* Takes ownership from the caller; no-op when invoked with null */
+//! void foo_delete(struct Foo*);
+//! ```
+//!
+//! These two functions might be implemented in Rust as follows. Here, the
+//! `struct Foo*` type from C is translated to `Box<Foo>`, which captures
+//! the ownership constraints. Note also that the nullable argument to
+//! `foo_delete` is represented in Rust as `Option<Box<Foo>>`, since `Box<Foo>`
+//! cannot be null.
+//!
+//! ```
+//! #[repr(C)]
+//! pub struct Foo;
+//!
+//! #[no_mangle]
+//! pub extern "C" fn foo_new() -> Box<Foo> {
+//!     Box::new(Foo)
+//! }
+//!
+//! #[no_mangle]
+//! pub extern "C" fn foo_delete(_: Option<Box<Foo>>) {}
+//! ```
+//!
+//! Even though `Box<T>` has the same representation and C ABI as a C pointer,
+//! this does not mean that you can convert an arbitrary `T*` into a `Box<T>`
+//! and expect things to work. `Box<T>` values will always be fully aligned,
+//! non-null pointers. Moreover, the destructor for `Box<T>` will attempt to
+//! free the value with the global allocator. In general, the best practice
+//! is to only use `Box<T>` for pointers that originated from the global
+//! allocator.
+//!
+//! **Important.** At least at present, you should avoid using
+//! `Box<T>` types for functions that are defined in C but invoked
+//! from Rust. In those cases, you should directly mirror the C types
+//! as closely as possible. Using types like `Box<T>` where the C
+//! definition is just using `T*` can lead to undefined behavior, as
+//! described in [rust-lang/unsafe-code-guidelines#198][ucg#198].
+//!
+//! [ucg#198]: https://github.com/rust-lang/unsafe-code-guidelines/issues/198
+//! [dereferencing]: core::ops::Deref
+//! [`Box::<T>::from_raw(value)`]: Box::from_raw
+//! [`Global`]: crate::alloc::Global
+//! [`Layout`]: crate::alloc::Layout
+//! [`Layout::for_value(&*value)`]: crate::alloc::Layout::for_value
+//! [valid]: ptr#safety
+
+#![stable(feature = "rust1", since = "1.0.0")]
+
+use core::any::Any;
+use core::async_iter::AsyncIterator;
+use core::borrow;
+use core::cmp::Ordering;
+use core::convert::{From, TryFrom};
+use core::fmt;
+use core::future::Future;
+use core::hash::{Hash, Hasher};
+#[cfg(not(no_global_oom_handling))]
+use core::iter::FromIterator;
+use core::iter::{FusedIterator, Iterator};
+use core::marker::{Destruct, Unpin, Unsize};
+use core::mem;
+use core::ops::{
+    CoerceUnsized, Deref, DerefMut, DispatchFromDyn, Generator, GeneratorState, Receiver,
+};
+use core::pin::Pin;
+use core::ptr::{self, Unique};
+use core::task::{Context, Poll};
+
+#[cfg(not(no_global_oom_handling))]
+use crate::alloc::{handle_alloc_error, WriteCloneIntoRaw};
+use crate::alloc::{AllocError, Allocator, Global, Layout};
+#[cfg(not(no_global_oom_handling))]
+use crate::borrow::Cow;
+use crate::raw_vec::RawVec;
+#[cfg(not(no_global_oom_handling))]
+use crate::str::from_boxed_utf8_unchecked;
+#[cfg(not(no_global_oom_handling))]
+use crate::vec::Vec;
+
+#[cfg(not(no_thin))]
+#[unstable(feature = "thin_box", issue = "92791")]
+pub use thin::ThinBox;
+
+#[cfg(not(no_thin))]
+mod thin;
+
+/// A pointer type for heap allocation.
+///
+/// See the [module-level documentation](../../std/boxed/index.html) for more.
+#[lang = "owned_box"]
+#[fundamental]
+#[stable(feature = "rust1", since = "1.0.0")]
+// The declaration of the `Box` struct must be kept in sync with the
+// `alloc::alloc::box_free` function or ICEs will happen. See the comment
+// on `box_free` for more details.
+pub struct Box<
+    T: ?Sized,
+    #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global,
+>(Unique<T>, A);
+
+impl<T> Box<T> {
+    /// Allocates memory on the heap and then places `x` into it.
+    ///
+    /// This doesn't actually allocate if `T` is zero-sized.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let five = Box::new(5);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[inline(always)]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[must_use]
+    pub fn new(x: T) -> Self {
+        box x
+    }
+
+    /// Constructs a new box with uninitialized contents.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(new_uninit)]
+    ///
+    /// let mut five = Box::<u32>::new_uninit();
+    ///
+    /// let five = unsafe {
+    ///     // Deferred initialization:
+    ///     five.as_mut_ptr().write(5);
+    ///
+    ///     five.assume_init()
+    /// };
+    ///
+    /// assert_eq!(*five, 5)
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[unstable(feature = "new_uninit", issue = "63291")]
+    #[must_use]
+    #[inline]
+    pub fn new_uninit() -> Box<mem::MaybeUninit<T>> {
+        Self::new_uninit_in(Global)
+    }
+
+    /// Constructs a new `Box` with uninitialized contents, with the memory
+    /// being filled with `0` bytes.
+    ///
+    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
+    /// of this method.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(new_uninit)]
+    ///
+    /// let zero = Box::<u32>::new_zeroed();
+    /// let zero = unsafe { zero.assume_init() };
+    ///
+    /// assert_eq!(*zero, 0)
+    /// ```
+    ///
+    /// [zeroed]: mem::MaybeUninit::zeroed
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    #[unstable(feature = "new_uninit", issue = "63291")]
+    #[must_use]
+    pub fn new_zeroed() -> Box<mem::MaybeUninit<T>> {
+        Self::new_zeroed_in(Global)
+    }
+
+    /// Constructs a new `Pin<Box<T>>`. If `T` does not implement `Unpin`, then
+    /// `x` will be pinned in memory and unable to be moved.
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "pin", since = "1.33.0")]
+    #[must_use]
+    #[inline(always)]
+    pub fn pin(x: T) -> Pin<Box<T>> {
+        (box x).into()
+    }
+
+    /// Allocates memory on the heap then places `x` into it,
+    /// returning an error if the allocation fails
+    ///
+    /// This doesn't actually allocate if `T` is zero-sized.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api)]
+    ///
+    /// let five = Box::try_new(5)?;
+    /// # Ok::<(), std::alloc::AllocError>(())
+    /// ```
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[inline]
+    pub fn try_new(x: T) -> Result<Self, AllocError> {
+        Self::try_new_in(x, Global)
+    }
+
+    /// Constructs a new box with uninitialized contents on the heap,
+    /// returning an error if the allocation fails
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, new_uninit)]
+    ///
+    /// let mut five = Box::<u32>::try_new_uninit()?;
+    ///
+    /// let five = unsafe {
+    ///     // Deferred initialization:
+    ///     five.as_mut_ptr().write(5);
+    ///
+    ///     five.assume_init()
+    /// };
+    ///
+    /// assert_eq!(*five, 5);
+    /// # Ok::<(), std::alloc::AllocError>(())
+    /// ```
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    // #[unstable(feature = "new_uninit", issue = "63291")]
+    #[inline]
+    pub fn try_new_uninit() -> Result<Box<mem::MaybeUninit<T>>, AllocError> {
+        Box::try_new_uninit_in(Global)
+    }
+
+    /// Constructs a new `Box` with uninitialized contents, with the memory
+    /// being filled with `0` bytes on the heap
+    ///
+    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
+    /// of this method.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, new_uninit)]
+    ///
+    /// let zero = Box::<u32>::try_new_zeroed()?;
+    /// let zero = unsafe { zero.assume_init() };
+    ///
+    /// assert_eq!(*zero, 0);
+    /// # Ok::<(), std::alloc::AllocError>(())
+    /// ```
+    ///
+    /// [zeroed]: mem::MaybeUninit::zeroed
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    // #[unstable(feature = "new_uninit", issue = "63291")]
+    #[inline]
+    pub fn try_new_zeroed() -> Result<Box<mem::MaybeUninit<T>>, AllocError> {
+        Box::try_new_zeroed_in(Global)
+    }
+}
+
+impl<T, A: Allocator> Box<T, A> {
+    /// Allocates memory in the given allocator then places `x` into it.
+    ///
+    /// This doesn't actually allocate if `T` is zero-sized.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let five = Box::new_in(5, System);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[must_use]
+    #[inline]
+    pub const fn new_in(x: T, alloc: A) -> Self
+    where
+        A: ~const Allocator + ~const Destruct,
+    {
+        let mut boxed = Self::new_uninit_in(alloc);
+        unsafe {
+            boxed.as_mut_ptr().write(x);
+            boxed.assume_init()
+        }
+    }
+
+    /// Allocates memory in the given allocator then places `x` into it,
+    /// returning an error if the allocation fails
+    ///
+    /// This doesn't actually allocate if `T` is zero-sized.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let five = Box::try_new_in(5, System)?;
+    /// # Ok::<(), std::alloc::AllocError>(())
+    /// ```
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[inline]
+    pub const fn try_new_in(x: T, alloc: A) -> Result<Self, AllocError>
+    where
+        T: ~const Destruct,
+        A: ~const Allocator + ~const Destruct,
+    {
+        let mut boxed = Self::try_new_uninit_in(alloc)?;
+        unsafe {
+            boxed.as_mut_ptr().write(x);
+            Ok(boxed.assume_init())
+        }
+    }
+
+    /// Constructs a new box with uninitialized contents in the provided allocator.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, new_uninit)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let mut five = Box::<u32, _>::new_uninit_in(System);
+    ///
+    /// let five = unsafe {
+    ///     // Deferred initialization:
+    ///     five.as_mut_ptr().write(5);
+    ///
+    ///     five.assume_init()
+    /// };
+    ///
+    /// assert_eq!(*five, 5)
+    /// ```
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[cfg(not(no_global_oom_handling))]
+    #[must_use]
+    // #[unstable(feature = "new_uninit", issue = "63291")]
+    pub const fn new_uninit_in(alloc: A) -> Box<mem::MaybeUninit<T>, A>
+    where
+        A: ~const Allocator + ~const Destruct,
+    {
+        let layout = Layout::new::<mem::MaybeUninit<T>>();
+        // NOTE: Prefer match over unwrap_or_else since closure sometimes not inlineable.
+        // That would make code size bigger.
+        match Box::try_new_uninit_in(alloc) {
+            Ok(m) => m,
+            Err(_) => handle_alloc_error(layout),
+        }
+    }
+
+    /// Constructs a new box with uninitialized contents in the provided allocator,
+    /// returning an error if the allocation fails
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, new_uninit)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let mut five = Box::<u32, _>::try_new_uninit_in(System)?;
+    ///
+    /// let five = unsafe {
+    ///     // Deferred initialization:
+    ///     five.as_mut_ptr().write(5);
+    ///
+    ///     five.assume_init()
+    /// };
+    ///
+    /// assert_eq!(*five, 5);
+    /// # Ok::<(), std::alloc::AllocError>(())
+    /// ```
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    // #[unstable(feature = "new_uninit", issue = "63291")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    pub const fn try_new_uninit_in(alloc: A) -> Result<Box<mem::MaybeUninit<T>, A>, AllocError>
+    where
+        A: ~const Allocator + ~const Destruct,
+    {
+        let layout = Layout::new::<mem::MaybeUninit<T>>();
+        let ptr = alloc.allocate(layout)?.cast();
+        unsafe { Ok(Box::from_raw_in(ptr.as_ptr(), alloc)) }
+    }
+
+    /// Constructs a new `Box` with uninitialized contents, with the memory
+    /// being filled with `0` bytes in the provided allocator.
+    ///
+    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
+    /// of this method.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, new_uninit)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let zero = Box::<u32, _>::new_zeroed_in(System);
+    /// let zero = unsafe { zero.assume_init() };
+    ///
+    /// assert_eq!(*zero, 0)
+    /// ```
+    ///
+    /// [zeroed]: mem::MaybeUninit::zeroed
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[cfg(not(no_global_oom_handling))]
+    // #[unstable(feature = "new_uninit", issue = "63291")]
+    #[must_use]
+    pub const fn new_zeroed_in(alloc: A) -> Box<mem::MaybeUninit<T>, A>
+    where
+        A: ~const Allocator + ~const Destruct,
+    {
+        let layout = Layout::new::<mem::MaybeUninit<T>>();
+        // NOTE: Prefer match over unwrap_or_else since closure sometimes not inlineable.
+        // That would make code size bigger.
+        match Box::try_new_zeroed_in(alloc) {
+            Ok(m) => m,
+            Err(_) => handle_alloc_error(layout),
+        }
+    }
+
+    /// Constructs a new `Box` with uninitialized contents, with the memory
+    /// being filled with `0` bytes in the provided allocator,
+    /// returning an error if the allocation fails,
+    ///
+    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
+    /// of this method.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, new_uninit)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let zero = Box::<u32, _>::try_new_zeroed_in(System)?;
+    /// let zero = unsafe { zero.assume_init() };
+    ///
+    /// assert_eq!(*zero, 0);
+    /// # Ok::<(), std::alloc::AllocError>(())
+    /// ```
+    ///
+    /// [zeroed]: mem::MaybeUninit::zeroed
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    // #[unstable(feature = "new_uninit", issue = "63291")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    pub const fn try_new_zeroed_in(alloc: A) -> Result<Box<mem::MaybeUninit<T>, A>, AllocError>
+    where
+        A: ~const Allocator + ~const Destruct,
+    {
+        let layout = Layout::new::<mem::MaybeUninit<T>>();
+        let ptr = alloc.allocate_zeroed(layout)?.cast();
+        unsafe { Ok(Box::from_raw_in(ptr.as_ptr(), alloc)) }
+    }
+
+    /// Constructs a new `Pin<Box<T, A>>`. If `T` does not implement `Unpin`, then
+    /// `x` will be pinned in memory and unable to be moved.
+    #[cfg(not(no_global_oom_handling))]
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[must_use]
+    #[inline(always)]
+    pub const fn pin_in(x: T, alloc: A) -> Pin<Self>
+    where
+        A: 'static + ~const Allocator + ~const Destruct,
+    {
+        Self::into_pin(Self::new_in(x, alloc))
+    }
+
+    /// Converts a `Box<T>` into a `Box<[T]>`
+    ///
+    /// This conversion does not allocate on the heap and happens in place.
+    #[unstable(feature = "box_into_boxed_slice", issue = "71582")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    pub const fn into_boxed_slice(boxed: Self) -> Box<[T], A> {
+        let (raw, alloc) = Box::into_raw_with_allocator(boxed);
+        unsafe { Box::from_raw_in(raw as *mut [T; 1], alloc) }
+    }
+
+    /// Consumes the `Box`, returning the wrapped value.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(box_into_inner)]
+    ///
+    /// let c = Box::new(5);
+    ///
+    /// assert_eq!(Box::into_inner(c), 5);
+    /// ```
+    #[unstable(feature = "box_into_inner", issue = "80437")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[inline]
+    pub const fn into_inner(boxed: Self) -> T
+    where
+        Self: ~const Destruct,
+    {
+        *boxed
+    }
+}
+
+impl<T> Box<[T]> {
+    /// Constructs a new boxed slice with uninitialized contents.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(new_uninit)]
+    ///
+    /// let mut values = Box::<[u32]>::new_uninit_slice(3);
+    ///
+    /// let values = unsafe {
+    ///     // Deferred initialization:
+    ///     values[0].as_mut_ptr().write(1);
+    ///     values[1].as_mut_ptr().write(2);
+    ///     values[2].as_mut_ptr().write(3);
+    ///
+    ///     values.assume_init()
+    /// };
+    ///
+    /// assert_eq!(*values, [1, 2, 3])
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[unstable(feature = "new_uninit", issue = "63291")]
+    #[must_use]
+    pub fn new_uninit_slice(len: usize) -> Box<[mem::MaybeUninit<T>]> {
+        unsafe { RawVec::with_capacity(len).into_box(len) }
+    }
+
+    /// Constructs a new boxed slice with uninitialized contents, with the memory
+    /// being filled with `0` bytes.
+    ///
+    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
+    /// of this method.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(new_uninit)]
+    ///
+    /// let values = Box::<[u32]>::new_zeroed_slice(3);
+    /// let values = unsafe { values.assume_init() };
+    ///
+    /// assert_eq!(*values, [0, 0, 0])
+    /// ```
+    ///
+    /// [zeroed]: mem::MaybeUninit::zeroed
+    #[cfg(not(no_global_oom_handling))]
+    #[unstable(feature = "new_uninit", issue = "63291")]
+    #[must_use]
+    pub fn new_zeroed_slice(len: usize) -> Box<[mem::MaybeUninit<T>]> {
+        unsafe { RawVec::with_capacity_zeroed(len).into_box(len) }
+    }
+
+    /// Constructs a new boxed slice with uninitialized contents. Returns an error if
+    /// the allocation fails
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, new_uninit)]
+    ///
+    /// let mut values = Box::<[u32]>::try_new_uninit_slice(3)?;
+    /// let values = unsafe {
+    ///     // Deferred initialization:
+    ///     values[0].as_mut_ptr().write(1);
+    ///     values[1].as_mut_ptr().write(2);
+    ///     values[2].as_mut_ptr().write(3);
+    ///     values.assume_init()
+    /// };
+    ///
+    /// assert_eq!(*values, [1, 2, 3]);
+    /// # Ok::<(), std::alloc::AllocError>(())
+    /// ```
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[inline]
+    pub fn try_new_uninit_slice(len: usize) -> Result<Box<[mem::MaybeUninit<T>]>, AllocError> {
+        unsafe {
+            let layout = match Layout::array::<mem::MaybeUninit<T>>(len) {
+                Ok(l) => l,
+                Err(_) => return Err(AllocError),
+            };
+            let ptr = Global.allocate(layout)?;
+            Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len))
+        }
+    }
+
+    /// Constructs a new boxed slice with uninitialized contents, with the memory
+    /// being filled with `0` bytes. Returns an error if the allocation fails
+    ///
+    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
+    /// of this method.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, new_uninit)]
+    ///
+    /// let values = Box::<[u32]>::try_new_zeroed_slice(3)?;
+    /// let values = unsafe { values.assume_init() };
+    ///
+    /// assert_eq!(*values, [0, 0, 0]);
+    /// # Ok::<(), std::alloc::AllocError>(())
+    /// ```
+    ///
+    /// [zeroed]: mem::MaybeUninit::zeroed
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[inline]
+    pub fn try_new_zeroed_slice(len: usize) -> Result<Box<[mem::MaybeUninit<T>]>, AllocError> {
+        unsafe {
+            let layout = match Layout::array::<mem::MaybeUninit<T>>(len) {
+                Ok(l) => l,
+                Err(_) => return Err(AllocError),
+            };
+            let ptr = Global.allocate_zeroed(layout)?;
+            Ok(RawVec::from_raw_parts_in(ptr.as_mut_ptr() as *mut _, len, Global).into_box(len))
+        }
+    }
+}
+
+impl<T, A: Allocator> Box<[T], A> {
+    /// Constructs a new boxed slice with uninitialized contents in the provided allocator.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, new_uninit)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let mut values = Box::<[u32], _>::new_uninit_slice_in(3, System);
+    ///
+    /// let values = unsafe {
+    ///     // Deferred initialization:
+    ///     values[0].as_mut_ptr().write(1);
+    ///     values[1].as_mut_ptr().write(2);
+    ///     values[2].as_mut_ptr().write(3);
+    ///
+    ///     values.assume_init()
+    /// };
+    ///
+    /// assert_eq!(*values, [1, 2, 3])
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    // #[unstable(feature = "new_uninit", issue = "63291")]
+    #[must_use]
+    pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> {
+        unsafe { RawVec::with_capacity_in(len, alloc).into_box(len) }
+    }
+
+    /// Constructs a new boxed slice with uninitialized contents in the provided allocator,
+    /// with the memory being filled with `0` bytes.
+    ///
+    /// See [`MaybeUninit::zeroed`][zeroed] for examples of correct and incorrect usage
+    /// of this method.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, new_uninit)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let values = Box::<[u32], _>::new_zeroed_slice_in(3, System);
+    /// let values = unsafe { values.assume_init() };
+    ///
+    /// assert_eq!(*values, [0, 0, 0])
+    /// ```
+    ///
+    /// [zeroed]: mem::MaybeUninit::zeroed
+    #[cfg(not(no_global_oom_handling))]
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    // #[unstable(feature = "new_uninit", issue = "63291")]
+    #[must_use]
+    pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> {
+        unsafe { RawVec::with_capacity_zeroed_in(len, alloc).into_box(len) }
+    }
+}
+
+impl<T, A: Allocator> Box<mem::MaybeUninit<T>, A> {
+    /// Converts to `Box<T, A>`.
+    ///
+    /// # Safety
+    ///
+    /// As with [`MaybeUninit::assume_init`],
+    /// it is up to the caller to guarantee that the value
+    /// really is in an initialized state.
+    /// Calling this when the content is not yet fully initialized
+    /// causes immediate undefined behavior.
+    ///
+    /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(new_uninit)]
+    ///
+    /// let mut five = Box::<u32>::new_uninit();
+    ///
+    /// let five: Box<u32> = unsafe {
+    ///     // Deferred initialization:
+    ///     five.as_mut_ptr().write(5);
+    ///
+    ///     five.assume_init()
+    /// };
+    ///
+    /// assert_eq!(*five, 5)
+    /// ```
+    #[unstable(feature = "new_uninit", issue = "63291")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[inline]
+    pub const unsafe fn assume_init(self) -> Box<T, A> {
+        let (raw, alloc) = Box::into_raw_with_allocator(self);
+        unsafe { Box::from_raw_in(raw as *mut T, alloc) }
+    }
+
+    /// Writes the value and converts to `Box<T, A>`.
+    ///
+    /// This method converts the box similarly to [`Box::assume_init`] but
+    /// writes `value` into it before conversion thus guaranteeing safety.
+    /// In some scenarios use of this method may improve performance because
+    /// the compiler may be able to optimize copying from stack.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(new_uninit)]
+    ///
+    /// let big_box = Box::<[usize; 1024]>::new_uninit();
+    ///
+    /// let mut array = [0; 1024];
+    /// for (i, place) in array.iter_mut().enumerate() {
+    ///     *place = i;
+    /// }
+    ///
+    /// // The optimizer may be able to elide this copy, so previous code writes
+    /// // to heap directly.
+    /// let big_box = Box::write(big_box, array);
+    ///
+    /// for (i, x) in big_box.iter().enumerate() {
+    ///     assert_eq!(*x, i);
+    /// }
+    /// ```
+    #[unstable(feature = "new_uninit", issue = "63291")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[inline]
+    pub const fn write(mut boxed: Self, value: T) -> Box<T, A> {
+        unsafe {
+            (*boxed).write(value);
+            boxed.assume_init()
+        }
+    }
+}
+
+impl<T, A: Allocator> Box<[mem::MaybeUninit<T>], A> {
+    /// Converts to `Box<[T], A>`.
+    ///
+    /// # Safety
+    ///
+    /// As with [`MaybeUninit::assume_init`],
+    /// it is up to the caller to guarantee that the values
+    /// really are in an initialized state.
+    /// Calling this when the content is not yet fully initialized
+    /// causes immediate undefined behavior.
+    ///
+    /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(new_uninit)]
+    ///
+    /// let mut values = Box::<[u32]>::new_uninit_slice(3);
+    ///
+    /// let values = unsafe {
+    ///     // Deferred initialization:
+    ///     values[0].as_mut_ptr().write(1);
+    ///     values[1].as_mut_ptr().write(2);
+    ///     values[2].as_mut_ptr().write(3);
+    ///
+    ///     values.assume_init()
+    /// };
+    ///
+    /// assert_eq!(*values, [1, 2, 3])
+    /// ```
+    #[unstable(feature = "new_uninit", issue = "63291")]
+    #[inline]
+    pub unsafe fn assume_init(self) -> Box<[T], A> {
+        let (raw, alloc) = Box::into_raw_with_allocator(self);
+        unsafe { Box::from_raw_in(raw as *mut [T], alloc) }
+    }
+}
+
+impl<T: ?Sized> Box<T> {
+    /// Constructs a box from a raw pointer.
+    ///
+    /// After calling this function, the raw pointer is owned by the
+    /// resulting `Box`. Specifically, the `Box` destructor will call
+    /// the destructor of `T` and free the allocated memory. For this
+    /// to be safe, the memory must have been allocated in accordance
+    /// with the [memory layout] used by `Box` .
+    ///
+    /// # Safety
+    ///
+    /// This function is unsafe because improper use may lead to
+    /// memory problems. For example, a double-free may occur if the
+    /// function is called twice on the same raw pointer.
+    ///
+    /// The safety conditions are described in the [memory layout] section.
+    ///
+    /// # Examples
+    ///
+    /// Recreate a `Box` which was previously converted to a raw pointer
+    /// using [`Box::into_raw`]:
+    /// ```
+    /// let x = Box::new(5);
+    /// let ptr = Box::into_raw(x);
+    /// let x = unsafe { Box::from_raw(ptr) };
+    /// ```
+    /// Manually create a `Box` from scratch by using the global allocator:
+    /// ```
+    /// use std::alloc::{alloc, Layout};
+    ///
+    /// unsafe {
+    ///     let ptr = alloc(Layout::new::<i32>()) as *mut i32;
+    ///     // In general .write is required to avoid attempting to destruct
+    ///     // the (uninitialized) previous contents of `ptr`, though for this
+    ///     // simple example `*ptr = 5` would have worked as well.
+    ///     ptr.write(5);
+    ///     let x = Box::from_raw(ptr);
+    /// }
+    /// ```
+    ///
+    /// [memory layout]: self#memory-layout
+    /// [`Layout`]: crate::Layout
+    #[stable(feature = "box_raw", since = "1.4.0")]
+    #[inline]
+    pub unsafe fn from_raw(raw: *mut T) -> Self {
+        unsafe { Self::from_raw_in(raw, Global) }
+    }
+}
+
+impl<T: ?Sized, A: Allocator> Box<T, A> {
+    /// Constructs a box from a raw pointer in the given allocator.
+    ///
+    /// After calling this function, the raw pointer is owned by the
+    /// resulting `Box`. Specifically, the `Box` destructor will call
+    /// the destructor of `T` and free the allocated memory. For this
+    /// to be safe, the memory must have been allocated in accordance
+    /// with the [memory layout] used by `Box` .
+    ///
+    /// # Safety
+    ///
+    /// This function is unsafe because improper use may lead to
+    /// memory problems. For example, a double-free may occur if the
+    /// function is called twice on the same raw pointer.
+    ///
+    ///
+    /// # Examples
+    ///
+    /// Recreate a `Box` which was previously converted to a raw pointer
+    /// using [`Box::into_raw_with_allocator`]:
+    /// ```
+    /// #![feature(allocator_api)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let x = Box::new_in(5, System);
+    /// let (ptr, alloc) = Box::into_raw_with_allocator(x);
+    /// let x = unsafe { Box::from_raw_in(ptr, alloc) };
+    /// ```
+    /// Manually create a `Box` from scratch by using the system allocator:
+    /// ```
+    /// #![feature(allocator_api, slice_ptr_get)]
+    ///
+    /// use std::alloc::{Allocator, Layout, System};
+    ///
+    /// unsafe {
+    ///     let ptr = System.allocate(Layout::new::<i32>())?.as_mut_ptr() as *mut i32;
+    ///     // In general .write is required to avoid attempting to destruct
+    ///     // the (uninitialized) previous contents of `ptr`, though for this
+    ///     // simple example `*ptr = 5` would have worked as well.
+    ///     ptr.write(5);
+    ///     let x = Box::from_raw_in(ptr, System);
+    /// }
+    /// # Ok::<(), std::alloc::AllocError>(())
+    /// ```
+    ///
+    /// [memory layout]: self#memory-layout
+    /// [`Layout`]: crate::Layout
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[inline]
+    pub const unsafe fn from_raw_in(raw: *mut T, alloc: A) -> Self {
+        Box(unsafe { Unique::new_unchecked(raw) }, alloc)
+    }
+
+    /// Consumes the `Box`, returning a wrapped raw pointer.
+    ///
+    /// The pointer will be properly aligned and non-null.
+    ///
+    /// After calling this function, the caller is responsible for the
+    /// memory previously managed by the `Box`. In particular, the
+    /// caller should properly destroy `T` and release the memory, taking
+    /// into account the [memory layout] used by `Box`. The easiest way to
+    /// do this is to convert the raw pointer back into a `Box` with the
+    /// [`Box::from_raw`] function, allowing the `Box` destructor to perform
+    /// the cleanup.
+    ///
+    /// Note: this is an associated function, which means that you have
+    /// to call it as `Box::into_raw(b)` instead of `b.into_raw()`. This
+    /// is so that there is no conflict with a method on the inner type.
+    ///
+    /// # Examples
+    /// Converting the raw pointer back into a `Box` with [`Box::from_raw`]
+    /// for automatic cleanup:
+    /// ```
+    /// let x = Box::new(String::from("Hello"));
+    /// let ptr = Box::into_raw(x);
+    /// let x = unsafe { Box::from_raw(ptr) };
+    /// ```
+    /// Manual cleanup by explicitly running the destructor and deallocating
+    /// the memory:
+    /// ```
+    /// use std::alloc::{dealloc, Layout};
+    /// use std::ptr;
+    ///
+    /// let x = Box::new(String::from("Hello"));
+    /// let p = Box::into_raw(x);
+    /// unsafe {
+    ///     ptr::drop_in_place(p);
+    ///     dealloc(p as *mut u8, Layout::new::<String>());
+    /// }
+    /// ```
+    ///
+    /// [memory layout]: self#memory-layout
+    #[stable(feature = "box_raw", since = "1.4.0")]
+    #[inline]
+    pub fn into_raw(b: Self) -> *mut T {
+        Self::into_raw_with_allocator(b).0
+    }
+
+    /// Consumes the `Box`, returning a wrapped raw pointer and the allocator.
+    ///
+    /// The pointer will be properly aligned and non-null.
+    ///
+    /// After calling this function, the caller is responsible for the
+    /// memory previously managed by the `Box`. In particular, the
+    /// caller should properly destroy `T` and release the memory, taking
+    /// into account the [memory layout] used by `Box`. The easiest way to
+    /// do this is to convert the raw pointer back into a `Box` with the
+    /// [`Box::from_raw_in`] function, allowing the `Box` destructor to perform
+    /// the cleanup.
+    ///
+    /// Note: this is an associated function, which means that you have
+    /// to call it as `Box::into_raw_with_allocator(b)` instead of `b.into_raw_with_allocator()`. This
+    /// is so that there is no conflict with a method on the inner type.
+    ///
+    /// # Examples
+    /// Converting the raw pointer back into a `Box` with [`Box::from_raw_in`]
+    /// for automatic cleanup:
+    /// ```
+    /// #![feature(allocator_api)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let x = Box::new_in(String::from("Hello"), System);
+    /// let (ptr, alloc) = Box::into_raw_with_allocator(x);
+    /// let x = unsafe { Box::from_raw_in(ptr, alloc) };
+    /// ```
+    /// Manual cleanup by explicitly running the destructor and deallocating
+    /// the memory:
+    /// ```
+    /// #![feature(allocator_api)]
+    ///
+    /// use std::alloc::{Allocator, Layout, System};
+    /// use std::ptr::{self, NonNull};
+    ///
+    /// let x = Box::new_in(String::from("Hello"), System);
+    /// let (ptr, alloc) = Box::into_raw_with_allocator(x);
+    /// unsafe {
+    ///     ptr::drop_in_place(ptr);
+    ///     let non_null = NonNull::new_unchecked(ptr);
+    ///     alloc.deallocate(non_null.cast(), Layout::new::<String>());
+    /// }
+    /// ```
+    ///
+    /// [memory layout]: self#memory-layout
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[inline]
+    pub const fn into_raw_with_allocator(b: Self) -> (*mut T, A) {
+        let (leaked, alloc) = Box::into_unique(b);
+        (leaked.as_ptr(), alloc)
+    }
+
+    #[unstable(
+        feature = "ptr_internals",
+        issue = "none",
+        reason = "use `Box::leak(b).into()` or `Unique::from(Box::leak(b))` instead"
+    )]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[inline]
+    #[doc(hidden)]
+    pub const fn into_unique(b: Self) -> (Unique<T>, A) {
+        // Box is recognized as a "unique pointer" by Stacked Borrows, but internally it is a
+        // raw pointer for the type system. Turning it directly into a raw pointer would not be
+        // recognized as "releasing" the unique pointer to permit aliased raw accesses,
+        // so all raw pointer methods have to go through `Box::leak`. Turning *that* to a raw pointer
+        // behaves correctly.
+        let alloc = unsafe { ptr::read(&b.1) };
+        (Unique::from(Box::leak(b)), alloc)
+    }
+
+    /// Returns a reference to the underlying allocator.
+    ///
+    /// Note: this is an associated function, which means that you have
+    /// to call it as `Box::allocator(&b)` instead of `b.allocator()`. This
+    /// is so that there is no conflict with a method on the inner type.
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[inline]
+    pub const fn allocator(b: &Self) -> &A {
+        &b.1
+    }
+
+    /// Consumes and leaks the `Box`, returning a mutable reference,
+    /// `&'a mut T`. Note that the type `T` must outlive the chosen lifetime
+    /// `'a`. If the type has only static references, or none at all, then this
+    /// may be chosen to be `'static`.
+    ///
+    /// This function is mainly useful for data that lives for the remainder of
+    /// the program's life. Dropping the returned reference will cause a memory
+    /// leak. If this is not acceptable, the reference should first be wrapped
+    /// with the [`Box::from_raw`] function producing a `Box`. This `Box` can
+    /// then be dropped which will properly destroy `T` and release the
+    /// allocated memory.
+    ///
+    /// Note: this is an associated function, which means that you have
+    /// to call it as `Box::leak(b)` instead of `b.leak()`. This
+    /// is so that there is no conflict with a method on the inner type.
+    ///
+    /// # Examples
+    ///
+    /// Simple usage:
+    ///
+    /// ```
+    /// let x = Box::new(41);
+    /// let static_ref: &'static mut usize = Box::leak(x);
+    /// *static_ref += 1;
+    /// assert_eq!(*static_ref, 42);
+    /// ```
+    ///
+    /// Unsized data:
+    ///
+    /// ```
+    /// let x = vec![1, 2, 3].into_boxed_slice();
+    /// let static_ref = Box::leak(x);
+    /// static_ref[0] = 4;
+    /// assert_eq!(*static_ref, [4, 2, 3]);
+    /// ```
+    #[stable(feature = "box_leak", since = "1.26.0")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    #[inline]
+    pub const fn leak<'a>(b: Self) -> &'a mut T
+    where
+        A: 'a,
+    {
+        unsafe { &mut *mem::ManuallyDrop::new(b).0.as_ptr() }
+    }
+
+    /// Converts a `Box<T>` into a `Pin<Box<T>>`
+    ///
+    /// This conversion does not allocate on the heap and happens in place.
+    ///
+    /// This is also available via [`From`].
+    #[unstable(feature = "box_into_pin", issue = "62370")]
+    #[rustc_const_unstable(feature = "const_box", issue = "92521")]
+    pub const fn into_pin(boxed: Self) -> Pin<Self>
+    where
+        A: 'static,
+    {
+        // It's not possible to move or replace the insides of a `Pin<Box<T>>`
+        // when `T: !Unpin`,  so it's safe to pin it directly without any
+        // additional requirements.
+        unsafe { Pin::new_unchecked(boxed) }
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+unsafe impl<#[may_dangle] T: ?Sized, A: Allocator> Drop for Box<T, A> {
+    fn drop(&mut self) {
+        // FIXME: Do nothing, drop is currently performed by compiler.
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: Default> Default for Box<T> {
+    /// Creates a `Box<T>`, with the `Default` value for T.
+    fn default() -> Self {
+        box T::default()
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")]
+impl<T> const Default for Box<[T]> {
+    fn default() -> Self {
+        let ptr: Unique<[T]> = Unique::<[T; 0]>::dangling();
+        Box(ptr, Global)
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "default_box_extra", since = "1.17.0")]
+#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")]
+impl const Default for Box<str> {
+    fn default() -> Self {
+        // SAFETY: This is the same as `Unique::cast<U>` but with an unsized `U = str`.
+        let ptr: Unique<str> = unsafe {
+            let bytes: Unique<[u8]> = Unique::<[u8; 0]>::dangling();
+            Unique::new_unchecked(bytes.as_ptr() as *mut str)
+        };
+        Box(ptr, Global)
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: Clone, A: Allocator + Clone> Clone for Box<T, A> {
+    /// Returns a new box with a `clone()` of this box's contents.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let x = Box::new(5);
+    /// let y = x.clone();
+    ///
+    /// // The value is the same
+    /// assert_eq!(x, y);
+    ///
+    /// // But they are unique objects
+    /// assert_ne!(&*x as *const i32, &*y as *const i32);
+    /// ```
+    #[inline]
+    fn clone(&self) -> Self {
+        // Pre-allocate memory to allow writing the cloned value directly.
+        let mut boxed = Self::new_uninit_in(self.1.clone());
+        unsafe {
+            (**self).write_clone_into_raw(boxed.as_mut_ptr());
+            boxed.assume_init()
+        }
+    }
+
+    /// Copies `source`'s contents into `self` without creating a new allocation.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let x = Box::new(5);
+    /// let mut y = Box::new(10);
+    /// let yp: *const i32 = &*y;
+    ///
+    /// y.clone_from(&x);
+    ///
+    /// // The value is the same
+    /// assert_eq!(x, y);
+    ///
+    /// // And no allocation occurred
+    /// assert_eq!(yp, &*y);
+    /// ```
+    #[inline]
+    fn clone_from(&mut self, source: &Self) {
+        (**self).clone_from(&(**source));
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "box_slice_clone", since = "1.3.0")]
+impl Clone for Box<str> {
+    fn clone(&self) -> Self {
+        // this makes a copy of the data
+        let buf: Box<[u8]> = self.as_bytes().into();
+        unsafe { from_boxed_utf8_unchecked(buf) }
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: ?Sized + PartialEq, A: Allocator> PartialEq for Box<T, A> {
+    #[inline]
+    fn eq(&self, other: &Self) -> bool {
+        PartialEq::eq(&**self, &**other)
+    }
+    #[inline]
+    fn ne(&self, other: &Self) -> bool {
+        PartialEq::ne(&**self, &**other)
+    }
+}
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: ?Sized + PartialOrd, A: Allocator> PartialOrd for Box<T, A> {
+    #[inline]
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        PartialOrd::partial_cmp(&**self, &**other)
+    }
+    #[inline]
+    fn lt(&self, other: &Self) -> bool {
+        PartialOrd::lt(&**self, &**other)
+    }
+    #[inline]
+    fn le(&self, other: &Self) -> bool {
+        PartialOrd::le(&**self, &**other)
+    }
+    #[inline]
+    fn ge(&self, other: &Self) -> bool {
+        PartialOrd::ge(&**self, &**other)
+    }
+    #[inline]
+    fn gt(&self, other: &Self) -> bool {
+        PartialOrd::gt(&**self, &**other)
+    }
+}
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: ?Sized + Ord, A: Allocator> Ord for Box<T, A> {
+    #[inline]
+    fn cmp(&self, other: &Self) -> Ordering {
+        Ord::cmp(&**self, &**other)
+    }
+}
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: ?Sized + Eq, A: Allocator> Eq for Box<T, A> {}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: ?Sized + Hash, A: Allocator> Hash for Box<T, A> {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        (**self).hash(state);
+    }
+}
+
+#[stable(feature = "indirect_hasher_impl", since = "1.22.0")]
+impl<T: ?Sized + Hasher, A: Allocator> Hasher for Box<T, A> {
+    fn finish(&self) -> u64 {
+        (**self).finish()
+    }
+    fn write(&mut self, bytes: &[u8]) {
+        (**self).write(bytes)
+    }
+    fn write_u8(&mut self, i: u8) {
+        (**self).write_u8(i)
+    }
+    fn write_u16(&mut self, i: u16) {
+        (**self).write_u16(i)
+    }
+    fn write_u32(&mut self, i: u32) {
+        (**self).write_u32(i)
+    }
+    fn write_u64(&mut self, i: u64) {
+        (**self).write_u64(i)
+    }
+    fn write_u128(&mut self, i: u128) {
+        (**self).write_u128(i)
+    }
+    fn write_usize(&mut self, i: usize) {
+        (**self).write_usize(i)
+    }
+    fn write_i8(&mut self, i: i8) {
+        (**self).write_i8(i)
+    }
+    fn write_i16(&mut self, i: i16) {
+        (**self).write_i16(i)
+    }
+    fn write_i32(&mut self, i: i32) {
+        (**self).write_i32(i)
+    }
+    fn write_i64(&mut self, i: i64) {
+        (**self).write_i64(i)
+    }
+    fn write_i128(&mut self, i: i128) {
+        (**self).write_i128(i)
+    }
+    fn write_isize(&mut self, i: isize) {
+        (**self).write_isize(i)
+    }
+    fn write_length_prefix(&mut self, len: usize) {
+        (**self).write_length_prefix(len)
+    }
+    fn write_str(&mut self, s: &str) {
+        (**self).write_str(s)
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "from_for_ptrs", since = "1.6.0")]
+impl<T> From<T> for Box<T> {
+    /// Converts a `T` into a `Box<T>`
+    ///
+    /// The conversion allocates on the heap and moves `t`
+    /// from the stack into it.
+    ///
+    /// # Examples
+    ///
+    /// ```rust
+    /// let x = 5;
+    /// let boxed = Box::new(5);
+    ///
+    /// assert_eq!(Box::from(x), boxed);
+    /// ```
+    fn from(t: T) -> Self {
+        Box::new(t)
+    }
+}
+
+#[stable(feature = "pin", since = "1.33.0")]
+#[rustc_const_unstable(feature = "const_box", issue = "92521")]
+impl<T: ?Sized, A: Allocator> const From<Box<T, A>> for Pin<Box<T, A>>
+where
+    A: 'static,
+{
+    /// Converts a `Box<T>` into a `Pin<Box<T>>`
+    ///
+    /// This conversion does not allocate on the heap and happens in place.
+    fn from(boxed: Box<T, A>) -> Self {
+        Box::into_pin(boxed)
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "box_from_slice", since = "1.17.0")]
+impl<T: Copy> From<&[T]> for Box<[T]> {
+    /// Converts a `&[T]` into a `Box<[T]>`
+    ///
+    /// This conversion allocates on the heap
+    /// and performs a copy of `slice`.
+    ///
+    /// # Examples
+    /// ```rust
+    /// // create a &[u8] which will be used to create a Box<[u8]>
+    /// let slice: &[u8] = &[104, 101, 108, 108, 111];
+    /// let boxed_slice: Box<[u8]> = Box::from(slice);
+    ///
+    /// println!("{boxed_slice:?}");
+    /// ```
+    fn from(slice: &[T]) -> Box<[T]> {
+        let len = slice.len();
+        let buf = RawVec::with_capacity(len);
+        unsafe {
+            ptr::copy_nonoverlapping(slice.as_ptr(), buf.ptr(), len);
+            buf.into_box(slice.len()).assume_init()
+        }
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "box_from_cow", since = "1.45.0")]
+impl<T: Copy> From<Cow<'_, [T]>> for Box<[T]> {
+    /// Converts a `Cow<'_, [T]>` into a `Box<[T]>`
+    ///
+    /// When `cow` is the `Cow::Borrowed` variant, this
+    /// conversion allocates on the heap and copies the
+    /// underlying slice. Otherwise, it will try to reuse the owned
+    /// `Vec`'s allocation.
+    #[inline]
+    fn from(cow: Cow<'_, [T]>) -> Box<[T]> {
+        match cow {
+            Cow::Borrowed(slice) => Box::from(slice),
+            Cow::Owned(slice) => Box::from(slice),
+        }
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "box_from_slice", since = "1.17.0")]
+impl From<&str> for Box<str> {
+    /// Converts a `&str` into a `Box<str>`
+    ///
+    /// This conversion allocates on the heap
+    /// and performs a copy of `s`.
+    ///
+    /// # Examples
+    ///
+    /// ```rust
+    /// let boxed: Box<str> = Box::from("hello");
+    /// println!("{boxed}");
+    /// ```
+    #[inline]
+    fn from(s: &str) -> Box<str> {
+        unsafe { from_boxed_utf8_unchecked(Box::from(s.as_bytes())) }
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "box_from_cow", since = "1.45.0")]
+impl From<Cow<'_, str>> for Box<str> {
+    /// Converts a `Cow<'_, str>` into a `Box<str>`
+    ///
+    /// When `cow` is the `Cow::Borrowed` variant, this
+    /// conversion allocates on the heap and copies the
+    /// underlying `str`. Otherwise, it will try to reuse the owned
+    /// `String`'s allocation.
+    ///
+    /// # Examples
+    ///
+    /// ```rust
+    /// use std::borrow::Cow;
+    ///
+    /// let unboxed = Cow::Borrowed("hello");
+    /// let boxed: Box<str> = Box::from(unboxed);
+    /// println!("{boxed}");
+    /// ```
+    ///
+    /// ```rust
+    /// # use std::borrow::Cow;
+    /// let unboxed = Cow::Owned("hello".to_string());
+    /// let boxed: Box<str> = Box::from(unboxed);
+    /// println!("{boxed}");
+    /// ```
+    #[inline]
+    fn from(cow: Cow<'_, str>) -> Box<str> {
+        match cow {
+            Cow::Borrowed(s) => Box::from(s),
+            Cow::Owned(s) => Box::from(s),
+        }
+    }
+}
+
+#[stable(feature = "boxed_str_conv", since = "1.19.0")]
+impl<A: Allocator> From<Box<str, A>> for Box<[u8], A> {
+    /// Converts a `Box<str>` into a `Box<[u8]>`
+    ///
+    /// This conversion does not allocate on the heap and happens in place.
+    ///
+    /// # Examples
+    /// ```rust
+    /// // create a Box<str> which will be used to create a Box<[u8]>
+    /// let boxed: Box<str> = Box::from("hello");
+    /// let boxed_str: Box<[u8]> = Box::from(boxed);
+    ///
+    /// // create a &[u8] which will be used to create a Box<[u8]>
+    /// let slice: &[u8] = &[104, 101, 108, 108, 111];
+    /// let boxed_slice = Box::from(slice);
+    ///
+    /// assert_eq!(boxed_slice, boxed_str);
+    /// ```
+    #[inline]
+    fn from(s: Box<str, A>) -> Self {
+        let (raw, alloc) = Box::into_raw_with_allocator(s);
+        unsafe { Box::from_raw_in(raw as *mut [u8], alloc) }
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "box_from_array", since = "1.45.0")]
+impl<T, const N: usize> From<[T; N]> for Box<[T]> {
+    /// Converts a `[T; N]` into a `Box<[T]>`
+    ///
+    /// This conversion moves the array to newly heap-allocated memory.
+    ///
+    /// # Examples
+    ///
+    /// ```rust
+    /// let boxed: Box<[u8]> = Box::from([4, 2]);
+    /// println!("{boxed:?}");
+    /// ```
+    fn from(array: [T; N]) -> Box<[T]> {
+        box array
+    }
+}
+
+#[stable(feature = "boxed_slice_try_from", since = "1.43.0")]
+impl<T, const N: usize> TryFrom<Box<[T]>> for Box<[T; N]> {
+    type Error = Box<[T]>;
+
+    /// Attempts to convert a `Box<[T]>` into a `Box<[T; N]>`.
+    ///
+    /// The conversion occurs in-place and does not require a
+    /// new memory allocation.
+    ///
+    /// # Errors
+    ///
+    /// Returns the old `Box<[T]>` in the `Err` variant if
+    /// `boxed_slice.len()` does not equal `N`.
+    fn try_from(boxed_slice: Box<[T]>) -> Result<Self, Self::Error> {
+        if boxed_slice.len() == N {
+            Ok(unsafe { Box::from_raw(Box::into_raw(boxed_slice) as *mut [T; N]) })
+        } else {
+            Err(boxed_slice)
+        }
+    }
+}
+
+impl<A: Allocator> Box<dyn Any, A> {
+    /// Attempt to downcast the box to a concrete type.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::any::Any;
+    ///
+    /// fn print_if_string(value: Box<dyn Any>) {
+    ///     if let Ok(string) = value.downcast::<String>() {
+    ///         println!("String ({}): {}", string.len(), string);
+    ///     }
+    /// }
+    ///
+    /// let my_string = "Hello World".to_string();
+    /// print_if_string(Box::new(my_string));
+    /// print_if_string(Box::new(0i8));
+    /// ```
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn downcast<T: Any>(self) -> Result<Box<T, A>, Self> {
+        if self.is::<T>() { unsafe { Ok(self.downcast_unchecked::<T>()) } } else { Err(self) }
+    }
+
+    /// Downcasts the box to a concrete type.
+    ///
+    /// For a safe alternative see [`downcast`].
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(downcast_unchecked)]
+    ///
+    /// use std::any::Any;
+    ///
+    /// let x: Box<dyn Any> = Box::new(1_usize);
+    ///
+    /// unsafe {
+    ///     assert_eq!(*x.downcast_unchecked::<usize>(), 1);
+    /// }
+    /// ```
+    ///
+    /// # Safety
+    ///
+    /// The contained value must be of type `T`. Calling this method
+    /// with the incorrect type is *undefined behavior*.
+    ///
+    /// [`downcast`]: Self::downcast
+    #[inline]
+    #[unstable(feature = "downcast_unchecked", issue = "90850")]
+    pub unsafe fn downcast_unchecked<T: Any>(self) -> Box<T, A> {
+        debug_assert!(self.is::<T>());
+        unsafe {
+            let (raw, alloc): (*mut dyn Any, _) = Box::into_raw_with_allocator(self);
+            Box::from_raw_in(raw as *mut T, alloc)
+        }
+    }
+}
+
+impl<A: Allocator> Box<dyn Any + Send, A> {
+    /// Attempt to downcast the box to a concrete type.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::any::Any;
+    ///
+    /// fn print_if_string(value: Box<dyn Any + Send>) {
+    ///     if let Ok(string) = value.downcast::<String>() {
+    ///         println!("String ({}): {}", string.len(), string);
+    ///     }
+    /// }
+    ///
+    /// let my_string = "Hello World".to_string();
+    /// print_if_string(Box::new(my_string));
+    /// print_if_string(Box::new(0i8));
+    /// ```
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn downcast<T: Any>(self) -> Result<Box<T, A>, Self> {
+        if self.is::<T>() { unsafe { Ok(self.downcast_unchecked::<T>()) } } else { Err(self) }
+    }
+
+    /// Downcasts the box to a concrete type.
+    ///
+    /// For a safe alternative see [`downcast`].
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(downcast_unchecked)]
+    ///
+    /// use std::any::Any;
+    ///
+    /// let x: Box<dyn Any + Send> = Box::new(1_usize);
+    ///
+    /// unsafe {
+    ///     assert_eq!(*x.downcast_unchecked::<usize>(), 1);
+    /// }
+    /// ```
+    ///
+    /// # Safety
+    ///
+    /// The contained value must be of type `T`. Calling this method
+    /// with the incorrect type is *undefined behavior*.
+    ///
+    /// [`downcast`]: Self::downcast
+    #[inline]
+    #[unstable(feature = "downcast_unchecked", issue = "90850")]
+    pub unsafe fn downcast_unchecked<T: Any>(self) -> Box<T, A> {
+        debug_assert!(self.is::<T>());
+        unsafe {
+            let (raw, alloc): (*mut (dyn Any + Send), _) = Box::into_raw_with_allocator(self);
+            Box::from_raw_in(raw as *mut T, alloc)
+        }
+    }
+}
+
+impl<A: Allocator> Box<dyn Any + Send + Sync, A> {
+    /// Attempt to downcast the box to a concrete type.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::any::Any;
+    ///
+    /// fn print_if_string(value: Box<dyn Any + Send + Sync>) {
+    ///     if let Ok(string) = value.downcast::<String>() {
+    ///         println!("String ({}): {}", string.len(), string);
+    ///     }
+    /// }
+    ///
+    /// let my_string = "Hello World".to_string();
+    /// print_if_string(Box::new(my_string));
+    /// print_if_string(Box::new(0i8));
+    /// ```
+    #[inline]
+    #[stable(feature = "box_send_sync_any_downcast", since = "1.51.0")]
+    pub fn downcast<T: Any>(self) -> Result<Box<T, A>, Self> {
+        if self.is::<T>() { unsafe { Ok(self.downcast_unchecked::<T>()) } } else { Err(self) }
+    }
+
+    /// Downcasts the box to a concrete type.
+    ///
+    /// For a safe alternative see [`downcast`].
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(downcast_unchecked)]
+    ///
+    /// use std::any::Any;
+    ///
+    /// let x: Box<dyn Any + Send + Sync> = Box::new(1_usize);
+    ///
+    /// unsafe {
+    ///     assert_eq!(*x.downcast_unchecked::<usize>(), 1);
+    /// }
+    /// ```
+    ///
+    /// # Safety
+    ///
+    /// The contained value must be of type `T`. Calling this method
+    /// with the incorrect type is *undefined behavior*.
+    ///
+    /// [`downcast`]: Self::downcast
+    #[inline]
+    #[unstable(feature = "downcast_unchecked", issue = "90850")]
+    pub unsafe fn downcast_unchecked<T: Any>(self) -> Box<T, A> {
+        debug_assert!(self.is::<T>());
+        unsafe {
+            let (raw, alloc): (*mut (dyn Any + Send + Sync), _) =
+                Box::into_raw_with_allocator(self);
+            Box::from_raw_in(raw as *mut T, alloc)
+        }
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: fmt::Display + ?Sized, A: Allocator> fmt::Display for Box<T, A> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(&**self, f)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: fmt::Debug + ?Sized, A: Allocator> fmt::Debug for Box<T, A> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(&**self, f)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: ?Sized, A: Allocator> fmt::Pointer for Box<T, A> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // It's not possible to extract the inner Uniq directly from the Box,
+        // instead we cast it to a *const which aliases the Unique
+        let ptr: *const T = &**self;
+        fmt::Pointer::fmt(&ptr, f)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+#[rustc_const_unstable(feature = "const_box", issue = "92521")]
+impl<T: ?Sized, A: Allocator> const Deref for Box<T, A> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &**self
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+#[rustc_const_unstable(feature = "const_box", issue = "92521")]
+impl<T: ?Sized, A: Allocator> const DerefMut for Box<T, A> {
+    fn deref_mut(&mut self) -> &mut T {
+        &mut **self
+    }
+}
+
+#[unstable(feature = "receiver_trait", issue = "none")]
+impl<T: ?Sized, A: Allocator> Receiver for Box<T, A> {}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<I: Iterator + ?Sized, A: Allocator> Iterator for Box<I, A> {
+    type Item = I::Item;
+    fn next(&mut self) -> Option<I::Item> {
+        (**self).next()
+    }
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (**self).size_hint()
+    }
+    fn nth(&mut self, n: usize) -> Option<I::Item> {
+        (**self).nth(n)
+    }
+    fn last(self) -> Option<I::Item> {
+        BoxIter::last(self)
+    }
+}
+
+trait BoxIter {
+    type Item;
+    fn last(self) -> Option<Self::Item>;
+}
+
+impl<I: Iterator + ?Sized, A: Allocator> BoxIter for Box<I, A> {
+    type Item = I::Item;
+    default fn last(self) -> Option<I::Item> {
+        #[inline]
+        fn some<T>(_: Option<T>, x: T) -> Option<T> {
+            Some(x)
+        }
+
+        self.fold(None, some)
+    }
+}
+
+/// Specialization for sized `I`s that uses `I`s implementation of `last()`
+/// instead of the default.
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<I: Iterator, A: Allocator> BoxIter for Box<I, A> {
+    fn last(self) -> Option<I::Item> {
+        (*self).last()
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<I: DoubleEndedIterator + ?Sized, A: Allocator> DoubleEndedIterator for Box<I, A> {
+    fn next_back(&mut self) -> Option<I::Item> {
+        (**self).next_back()
+    }
+    fn nth_back(&mut self, n: usize) -> Option<I::Item> {
+        (**self).nth_back(n)
+    }
+}
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<I: ExactSizeIterator + ?Sized, A: Allocator> ExactSizeIterator for Box<I, A> {
+    fn len(&self) -> usize {
+        (**self).len()
+    }
+    fn is_empty(&self) -> bool {
+        (**self).is_empty()
+    }
+}
+
+#[stable(feature = "fused", since = "1.26.0")]
+impl<I: FusedIterator + ?Sized, A: Allocator> FusedIterator for Box<I, A> {}
+
+#[stable(feature = "boxed_closure_impls", since = "1.35.0")]
+impl<Args, F: FnOnce<Args> + ?Sized, A: Allocator> FnOnce<Args> for Box<F, A> {
+    type Output = <F as FnOnce<Args>>::Output;
+
+    extern "rust-call" fn call_once(self, args: Args) -> Self::Output {
+        <F as FnOnce<Args>>::call_once(*self, args)
+    }
+}
+
+#[stable(feature = "boxed_closure_impls", since = "1.35.0")]
+impl<Args, F: FnMut<Args> + ?Sized, A: Allocator> FnMut<Args> for Box<F, A> {
+    extern "rust-call" fn call_mut(&mut self, args: Args) -> Self::Output {
+        <F as FnMut<Args>>::call_mut(self, args)
+    }
+}
+
+#[stable(feature = "boxed_closure_impls", since = "1.35.0")]
+impl<Args, F: Fn<Args> + ?Sized, A: Allocator> Fn<Args> for Box<F, A> {
+    extern "rust-call" fn call(&self, args: Args) -> Self::Output {
+        <F as Fn<Args>>::call(self, args)
+    }
+}
+
+#[unstable(feature = "coerce_unsized", issue = "27732")]
+impl<T: ?Sized + Unsize<U>, U: ?Sized, A: Allocator> CoerceUnsized<Box<U, A>> for Box<T, A> {}
+
+#[unstable(feature = "dispatch_from_dyn", issue = "none")]
+impl<T: ?Sized + Unsize<U>, U: ?Sized> DispatchFromDyn<Box<U>> for Box<T, Global> {}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "boxed_slice_from_iter", since = "1.32.0")]
+impl<I> FromIterator<I> for Box<[I]> {
+    fn from_iter<T: IntoIterator<Item = I>>(iter: T) -> Self {
+        iter.into_iter().collect::<Vec<_>>().into_boxed_slice()
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "box_slice_clone", since = "1.3.0")]
+impl<T: Clone, A: Allocator + Clone> Clone for Box<[T], A> {
+    fn clone(&self) -> Self {
+        let alloc = Box::allocator(self).clone();
+        self.to_vec_in(alloc).into_boxed_slice()
+    }
+
+    fn clone_from(&mut self, other: &Self) {
+        if self.len() == other.len() {
+            self.clone_from_slice(&other);
+        } else {
+            *self = other.clone();
+        }
+    }
+}
+
+#[stable(feature = "box_borrow", since = "1.1.0")]
+impl<T: ?Sized, A: Allocator> borrow::Borrow<T> for Box<T, A> {
+    fn borrow(&self) -> &T {
+        &**self
+    }
+}
+
+#[stable(feature = "box_borrow", since = "1.1.0")]
+impl<T: ?Sized, A: Allocator> borrow::BorrowMut<T> for Box<T, A> {
+    fn borrow_mut(&mut self) -> &mut T {
+        &mut **self
+    }
+}
+
+#[stable(since = "1.5.0", feature = "smart_ptr_as_ref")]
+impl<T: ?Sized, A: Allocator> AsRef<T> for Box<T, A> {
+    fn as_ref(&self) -> &T {
+        &**self
+    }
+}
+
+#[stable(since = "1.5.0", feature = "smart_ptr_as_ref")]
+impl<T: ?Sized, A: Allocator> AsMut<T> for Box<T, A> {
+    fn as_mut(&mut self) -> &mut T {
+        &mut **self
+    }
+}
+
+/* Nota bene
+ *
+ *  We could have chosen not to add this impl, and instead have written a
+ *  function of Pin<Box<T>> to Pin<T>. Such a function would not be sound,
+ *  because Box<T> implements Unpin even when T does not, as a result of
+ *  this impl.
+ *
+ *  We chose this API instead of the alternative for a few reasons:
+ *      - Logically, it is helpful to understand pinning in regard to the
+ *        memory region being pointed to. For this reason none of the
+ *        standard library pointer types support projecting through a pin
+ *        (Box<T> is the only pointer type in std for which this would be
+ *        safe.)
+ *      - It is in practice very useful to have Box<T> be unconditionally
+ *        Unpin because of trait objects, for which the structural auto
+ *        trait functionality does not apply (e.g., Box<dyn Foo> would
+ *        otherwise not be Unpin).
+ *
+ *  Another type with the same semantics as Box but only a conditional
+ *  implementation of `Unpin` (where `T: Unpin`) would be valid/safe, and
+ *  could have a method to project a Pin<T> from it.
+ */
+#[stable(feature = "pin", since = "1.33.0")]
+#[rustc_const_unstable(feature = "const_box", issue = "92521")]
+impl<T: ?Sized, A: Allocator> const Unpin for Box<T, A> where A: 'static {}
+
+#[unstable(feature = "generator_trait", issue = "43122")]
+impl<G: ?Sized + Generator<R> + Unpin, R, A: Allocator> Generator<R> for Box<G, A>
+where
+    A: 'static,
+{
+    type Yield = G::Yield;
+    type Return = G::Return;
+
+    fn resume(mut self: Pin<&mut Self>, arg: R) -> GeneratorState<Self::Yield, Self::Return> {
+        G::resume(Pin::new(&mut *self), arg)
+    }
+}
+
+#[unstable(feature = "generator_trait", issue = "43122")]
+impl<G: ?Sized + Generator<R>, R, A: Allocator> Generator<R> for Pin<Box<G, A>>
+where
+    A: 'static,
+{
+    type Yield = G::Yield;
+    type Return = G::Return;
+
+    fn resume(mut self: Pin<&mut Self>, arg: R) -> GeneratorState<Self::Yield, Self::Return> {
+        G::resume((*self).as_mut(), arg)
+    }
+}
+
+#[stable(feature = "futures_api", since = "1.36.0")]
+impl<F: ?Sized + Future + Unpin, A: Allocator> Future for Box<F, A>
+where
+    A: 'static,
+{
+    type Output = F::Output;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        F::poll(Pin::new(&mut *self), cx)
+    }
+}
+
+#[unstable(feature = "async_iterator", issue = "79024")]
+impl<S: ?Sized + AsyncIterator + Unpin> AsyncIterator for Box<S> {
+    type Item = S::Item;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        Pin::new(&mut **self).poll_next(cx)
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (**self).size_hint()
+    }
+}
diff --git a/rust/alloc/collections/mod.rs b/rust/alloc/collections/mod.rs
new file mode 100644
index 000000000000..1eec265b28f8
--- /dev/null
+++ b/rust/alloc/collections/mod.rs
@@ -0,0 +1,156 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+//! Collection types.
+
+#![stable(feature = "rust1", since = "1.0.0")]
+
+#[cfg(not(no_global_oom_handling))]
+pub mod binary_heap;
+#[cfg(not(no_global_oom_handling))]
+mod btree;
+#[cfg(not(no_global_oom_handling))]
+pub mod linked_list;
+#[cfg(not(no_global_oom_handling))]
+pub mod vec_deque;
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+pub mod btree_map {
+    //! An ordered map based on a B-Tree.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub use super::btree::map::*;
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+pub mod btree_set {
+    //! An ordered set based on a B-Tree.
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub use super::btree::set::*;
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+#[doc(no_inline)]
+pub use binary_heap::BinaryHeap;
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+#[doc(no_inline)]
+pub use btree_map::BTreeMap;
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+#[doc(no_inline)]
+pub use btree_set::BTreeSet;
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+#[doc(no_inline)]
+pub use linked_list::LinkedList;
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+#[doc(no_inline)]
+pub use vec_deque::VecDeque;
+
+use crate::alloc::{Layout, LayoutError};
+use core::fmt::Display;
+
+/// The error type for `try_reserve` methods.
+#[derive(Clone, PartialEq, Eq, Debug)]
+#[stable(feature = "try_reserve", since = "1.57.0")]
+pub struct TryReserveError {
+    kind: TryReserveErrorKind,
+}
+
+impl TryReserveError {
+    /// Details about the allocation that caused the error
+    #[inline]
+    #[must_use]
+    #[unstable(
+        feature = "try_reserve_kind",
+        reason = "Uncertain how much info should be exposed",
+        issue = "48043"
+    )]
+    pub fn kind(&self) -> TryReserveErrorKind {
+        self.kind.clone()
+    }
+}
+
+/// Details of the allocation that caused a `TryReserveError`
+#[derive(Clone, PartialEq, Eq, Debug)]
+#[unstable(
+    feature = "try_reserve_kind",
+    reason = "Uncertain how much info should be exposed",
+    issue = "48043"
+)]
+pub enum TryReserveErrorKind {
+    /// Error due to the computed capacity exceeding the collection's maximum
+    /// (usually `isize::MAX` bytes).
+    CapacityOverflow,
+
+    /// The memory allocator returned an error
+    AllocError {
+        /// The layout of allocation request that failed
+        layout: Layout,
+
+        #[doc(hidden)]
+        #[unstable(
+            feature = "container_error_extra",
+            issue = "none",
+            reason = "\
+            Enable exposing the allocator’s custom error value \
+            if an associated type is added in the future: \
+            https://github.com/rust-lang/wg-allocators/issues/23"
+        )]
+        non_exhaustive: (),
+    },
+}
+
+#[unstable(
+    feature = "try_reserve_kind",
+    reason = "Uncertain how much info should be exposed",
+    issue = "48043"
+)]
+impl From<TryReserveErrorKind> for TryReserveError {
+    #[inline]
+    fn from(kind: TryReserveErrorKind) -> Self {
+        Self { kind }
+    }
+}
+
+#[unstable(feature = "try_reserve_kind", reason = "new API", issue = "48043")]
+impl From<LayoutError> for TryReserveErrorKind {
+    /// Always evaluates to [`TryReserveErrorKind::CapacityOverflow`].
+    #[inline]
+    fn from(_: LayoutError) -> Self {
+        TryReserveErrorKind::CapacityOverflow
+    }
+}
+
+#[stable(feature = "try_reserve", since = "1.57.0")]
+impl Display for TryReserveError {
+    fn fmt(
+        &self,
+        fmt: &mut core::fmt::Formatter<'_>,
+    ) -> core::result::Result<(), core::fmt::Error> {
+        fmt.write_str("memory allocation failed")?;
+        let reason = match self.kind {
+            TryReserveErrorKind::CapacityOverflow => {
+                " because the computed capacity exceeded the collection's maximum"
+            }
+            TryReserveErrorKind::AllocError { .. } => {
+                " because the memory allocator returned a error"
+            }
+        };
+        fmt.write_str(reason)
+    }
+}
+
+/// An intermediate trait for specialization of `Extend`.
+#[doc(hidden)]
+trait SpecExtend<I: IntoIterator> {
+    /// Extends `self` with the contents of the given iterator.
+    fn spec_extend(&mut self, iter: I);
+}
diff --git a/rust/alloc/lib.rs b/rust/alloc/lib.rs
new file mode 100644
index 000000000000..233bcd5e4654
--- /dev/null
+++ b/rust/alloc/lib.rs
@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+//! # The Rust core allocation and collections library
+//!
+//! This library provides smart pointers and collections for managing
+//! heap-allocated values.
+//!
+//! This library, like libcore, normally doesn’t need to be used directly
+//! since its contents are re-exported in the [`std` crate](../std/index.html).
+//! Crates that use the `#![no_std]` attribute however will typically
+//! not depend on `std`, so they’d use this crate instead.
+//!
+//! ## Boxed values
+//!
+//! The [`Box`] type is a smart pointer type. There can only be one owner of a
+//! [`Box`], and the owner can decide to mutate the contents, which live on the
+//! heap.
+//!
+//! This type can be sent among threads efficiently as the size of a `Box` value
+//! is the same as that of a pointer. Tree-like data structures are often built
+//! with boxes because each node often has only one owner, the parent.
+//!
+//! ## Reference counted pointers
+//!
+//! The [`Rc`] type is a non-threadsafe reference-counted pointer type intended
+//! for sharing memory within a thread. An [`Rc`] pointer wraps a type, `T`, and
+//! only allows access to `&T`, a shared reference.
+//!
+//! This type is useful when inherited mutability (such as using [`Box`]) is too
+//! constraining for an application, and is often paired with the [`Cell`] or
+//! [`RefCell`] types in order to allow mutation.
+//!
+//! ## Atomically reference counted pointers
+//!
+//! The [`Arc`] type is the threadsafe equivalent of the [`Rc`] type. It
+//! provides all the same functionality of [`Rc`], except it requires that the
+//! contained type `T` is shareable. Additionally, [`Arc<T>`][`Arc`] is itself
+//! sendable while [`Rc<T>`][`Rc`] is not.
+//!
+//! This type allows for shared access to the contained data, and is often
+//! paired with synchronization primitives such as mutexes to allow mutation of
+//! shared resources.
+//!
+//! ## Collections
+//!
+//! Implementations of the most common general purpose data structures are
+//! defined in this library. They are re-exported through the
+//! [standard collections library](../std/collections/index.html).
+//!
+//! ## Heap interfaces
+//!
+//! The [`alloc`](alloc/index.html) module defines the low-level interface to the
+//! default global allocator. It is not compatible with the libc allocator API.
+//!
+//! [`Arc`]: sync
+//! [`Box`]: boxed
+//! [`Cell`]: core::cell
+//! [`Rc`]: rc
+//! [`RefCell`]: core::cell
+
+// To run liballoc tests without x.py without ending up with two copies of liballoc, Miri needs to be
+// able to "empty" this crate. See <https://github.com/rust-lang/miri-test-libstd/issues/4>.
+// rustc itself never sets the feature, so this line has no affect there.
+#![cfg(any(not(feature = "miri-test-libstd"), test, doctest))]
+#![allow(unused_attributes)]
+#![stable(feature = "alloc", since = "1.36.0")]
+#![doc(
+    html_playground_url = "https://play.rust-lang.org/",
+    issue_tracker_base_url = "https://github.com/rust-lang/rust/issues/",
+    test(no_crate_inject, attr(allow(unused_variables), deny(warnings)))
+)]
+#![doc(cfg_hide(
+    not(test),
+    not(any(test, bootstrap)),
+    any(not(feature = "miri-test-libstd"), test, doctest),
+    no_global_oom_handling,
+    not(no_global_oom_handling),
+    target_has_atomic = "ptr"
+))]
+#![no_std]
+#![needs_allocator]
+//
+// Lints:
+#![deny(unsafe_op_in_unsafe_fn)]
+#![warn(deprecated_in_future)]
+#![warn(missing_debug_implementations)]
+#![warn(missing_docs)]
+#![allow(explicit_outlives_requirements)]
+//
+// Library features:
+#![cfg_attr(not(no_global_oom_handling), feature(alloc_c_string))]
+#![feature(alloc_layout_extra)]
+#![feature(allocator_api)]
+#![feature(array_chunks)]
+#![feature(array_methods)]
+#![feature(array_windows)]
+#![feature(assert_matches)]
+#![feature(async_iterator)]
+#![feature(coerce_unsized)]
+#![cfg_attr(not(no_global_oom_handling), feature(const_alloc_error))]
+#![feature(const_box)]
+#![cfg_attr(not(no_global_oom_handling), feature(const_btree_new))]
+#![feature(const_cow_is_borrowed)]
+#![feature(const_convert)]
+#![feature(const_size_of_val)]
+#![feature(const_align_of_val)]
+#![feature(const_ptr_read)]
+#![feature(const_maybe_uninit_write)]
+#![feature(const_maybe_uninit_as_mut_ptr)]
+#![feature(const_refs_to_cell)]
+#![feature(core_c_str)]
+#![feature(core_intrinsics)]
+#![feature(core_ffi_c)]
+#![feature(const_eval_select)]
+#![feature(const_pin)]
+#![feature(cstr_from_bytes_until_nul)]
+#![feature(dispatch_from_dyn)]
+#![feature(exact_size_is_empty)]
+#![feature(extend_one)]
+#![feature(fmt_internals)]
+#![feature(fn_traits)]
+#![feature(hasher_prefixfree_extras)]
+#![feature(inplace_iteration)]
+#![feature(iter_advance_by)]
+#![feature(layout_for_ptr)]
+#![feature(maybe_uninit_slice)]
+#![cfg_attr(test, feature(new_uninit))]
+#![feature(nonnull_slice_from_raw_parts)]
+#![feature(pattern)]
+#![feature(ptr_internals)]
+#![feature(ptr_metadata)]
+#![feature(ptr_sub_ptr)]
+#![feature(receiver_trait)]
+#![feature(set_ptr_value)]
+#![feature(slice_group_by)]
+#![feature(slice_ptr_get)]
+#![feature(slice_ptr_len)]
+#![feature(slice_range)]
+#![feature(str_internals)]
+#![feature(strict_provenance)]
+#![feature(trusted_len)]
+#![feature(trusted_random_access)]
+#![feature(try_trait_v2)]
+#![feature(unchecked_math)]
+#![feature(unicode_internals)]
+#![feature(unsize)]
+//
+// Language features:
+#![feature(allocator_internals)]
+#![feature(allow_internal_unstable)]
+#![feature(associated_type_bounds)]
+#![feature(box_syntax)]
+#![feature(cfg_sanitize)]
+#![feature(const_deref)]
+#![feature(const_mut_refs)]
+#![feature(const_ptr_write)]
+#![feature(const_precise_live_drops)]
+#![feature(const_trait_impl)]
+#![feature(const_try)]
+#![feature(dropck_eyepatch)]
+#![feature(exclusive_range_pattern)]
+#![feature(fundamental)]
+#![cfg_attr(not(test), feature(generator_trait))]
+#![feature(hashmap_internals)]
+#![feature(lang_items)]
+#![feature(let_else)]
+#![feature(min_specialization)]
+#![feature(negative_impls)]
+#![feature(never_type)]
+#![feature(nll)] // Not necessary, but here to test the `nll` feature.
+#![feature(rustc_allow_const_fn_unstable)]
+#![feature(rustc_attrs)]
+#![feature(slice_internals)]
+#![feature(staged_api)]
+#![cfg_attr(test, feature(test))]
+#![feature(unboxed_closures)]
+#![feature(unsized_fn_params)]
+#![feature(c_unwind)]
+//
+// Rustdoc features:
+#![feature(doc_cfg)]
+#![feature(doc_cfg_hide)]
+// Technically, this is a bug in rustdoc: rustdoc sees the documentation on `#[lang = slice_alloc]`
+// blocks is for `&[T]`, which also has documentation using this feature in `core`, and gets mad
+// that the feature-gate isn't enabled. Ideally, it wouldn't check for the feature gate for docs
+// from other crates, but since this can only appear for lang items, it doesn't seem worth fixing.
+#![feature(intra_doc_pointers)]
+
+// Allow testing this library
+#[cfg(test)]
+#[macro_use]
+extern crate std;
+#[cfg(test)]
+extern crate test;
+
+// Module with internal macros used by other modules (needs to be included before other modules).
+#[cfg(not(no_macros))]
+#[macro_use]
+mod macros;
+
+mod raw_vec;
+
+// Heaps provided for low-level allocation strategies
+
+pub mod alloc;
+
+// Primitive types using the heaps above
+
+// Need to conditionally define the mod from `boxed.rs` to avoid
+// duplicating the lang-items when building in test cfg; but also need
+// to allow code to have `use boxed::Box;` declarations.
+#[cfg(not(test))]
+pub mod boxed;
+#[cfg(test)]
+mod boxed {
+    pub use std::boxed::Box;
+}
+pub mod borrow;
+pub mod collections;
+#[cfg(not(no_global_oom_handling))]
+pub mod ffi;
+#[cfg(not(no_fmt))]
+pub mod fmt;
+#[cfg(not(no_rc))]
+pub mod rc;
+pub mod slice;
+#[cfg(not(no_str))]
+pub mod str;
+#[cfg(not(no_string))]
+pub mod string;
+#[cfg(not(no_sync))]
+#[cfg(target_has_atomic = "ptr")]
+pub mod sync;
+#[cfg(all(not(no_global_oom_handling), target_has_atomic = "ptr"))]
+pub mod task;
+#[cfg(test)]
+mod tests;
+pub mod vec;
+
+#[doc(hidden)]
+#[unstable(feature = "liballoc_internals", issue = "none", reason = "implementation detail")]
+pub mod __export {
+    pub use core::format_args;
+}
diff --git a/rust/alloc/raw_vec.rs b/rust/alloc/raw_vec.rs
new file mode 100644
index 000000000000..daf5f2da7168
--- /dev/null
+++ b/rust/alloc/raw_vec.rs
@@ -0,0 +1,527 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+#![unstable(feature = "raw_vec_internals", reason = "unstable const warnings", issue = "none")]
+
+use core::alloc::LayoutError;
+use core::cmp;
+use core::intrinsics;
+use core::mem::{self, ManuallyDrop, MaybeUninit};
+use core::ops::Drop;
+use core::ptr::{self, NonNull, Unique};
+use core::slice;
+
+#[cfg(not(no_global_oom_handling))]
+use crate::alloc::handle_alloc_error;
+use crate::alloc::{Allocator, Global, Layout};
+use crate::boxed::Box;
+use crate::collections::TryReserveError;
+use crate::collections::TryReserveErrorKind::*;
+
+#[cfg(test)]
+mod tests;
+
+#[cfg(not(no_global_oom_handling))]
+enum AllocInit {
+    /// The contents of the new memory are uninitialized.
+    Uninitialized,
+    /// The new memory is guaranteed to be zeroed.
+    Zeroed,
+}
+
+/// A low-level utility for more ergonomically allocating, reallocating, and deallocating
+/// a buffer of memory on the heap without having to worry about all the corner cases
+/// involved. This type is excellent for building your own data structures like Vec and VecDeque.
+/// In particular:
+///
+/// * Produces `Unique::dangling()` on zero-sized types.
+/// * Produces `Unique::dangling()` on zero-length allocations.
+/// * Avoids freeing `Unique::dangling()`.
+/// * Catches all overflows in capacity computations (promotes them to "capacity overflow" panics).
+/// * Guards against 32-bit systems allocating more than isize::MAX bytes.
+/// * Guards against overflowing your length.
+/// * Calls `handle_alloc_error` for fallible allocations.
+/// * Contains a `ptr::Unique` and thus endows the user with all related benefits.
+/// * Uses the excess returned from the allocator to use the largest available capacity.
+///
+/// This type does not in anyway inspect the memory that it manages. When dropped it *will*
+/// free its memory, but it *won't* try to drop its contents. It is up to the user of `RawVec`
+/// to handle the actual things *stored* inside of a `RawVec`.
+///
+/// Note that the excess of a zero-sized types is always infinite, so `capacity()` always returns
+/// `usize::MAX`. This means that you need to be careful when round-tripping this type with a
+/// `Box<[T]>`, since `capacity()` won't yield the length.
+#[allow(missing_debug_implementations)]
+pub(crate) struct RawVec<T, A: Allocator = Global> {
+    ptr: Unique<T>,
+    cap: usize,
+    alloc: A,
+}
+
+impl<T> RawVec<T, Global> {
+    /// HACK(Centril): This exists because stable `const fn` can only call stable `const fn`, so
+    /// they cannot call `Self::new()`.
+    ///
+    /// If you change `RawVec<T>::new` or dependencies, please take care to not introduce anything
+    /// that would truly const-call something unstable.
+    pub const NEW: Self = Self::new();
+
+    /// Creates the biggest possible `RawVec` (on the system heap)
+    /// without allocating. If `T` has positive size, then this makes a
+    /// `RawVec` with capacity `0`. If `T` is zero-sized, then it makes a
+    /// `RawVec` with capacity `usize::MAX`. Useful for implementing
+    /// delayed allocation.
+    #[must_use]
+    pub const fn new() -> Self {
+        Self::new_in(Global)
+    }
+
+    /// Creates a `RawVec` (on the system heap) with exactly the
+    /// capacity and alignment requirements for a `[T; capacity]`. This is
+    /// equivalent to calling `RawVec::new` when `capacity` is `0` or `T` is
+    /// zero-sized. Note that if `T` is zero-sized this means you will
+    /// *not* get a `RawVec` with the requested capacity.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the requested capacity exceeds `isize::MAX` bytes.
+    ///
+    /// # Aborts
+    ///
+    /// Aborts on OOM.
+    #[cfg(not(any(no_global_oom_handling, test)))]
+    #[must_use]
+    #[inline]
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self::with_capacity_in(capacity, Global)
+    }
+
+    /// Like `with_capacity`, but guarantees the buffer is zeroed.
+    #[cfg(not(any(no_global_oom_handling, test)))]
+    #[must_use]
+    #[inline]
+    pub fn with_capacity_zeroed(capacity: usize) -> Self {
+        Self::with_capacity_zeroed_in(capacity, Global)
+    }
+}
+
+impl<T, A: Allocator> RawVec<T, A> {
+    // Tiny Vecs are dumb. Skip to:
+    // - 8 if the element size is 1, because any heap allocators is likely
+    //   to round up a request of less than 8 bytes to at least 8 bytes.
+    // - 4 if elements are moderate-sized (<= 1 KiB).
+    // - 1 otherwise, to avoid wasting too much space for very short Vecs.
+    pub(crate) const MIN_NON_ZERO_CAP: usize = if mem::size_of::<T>() == 1 {
+        8
+    } else if mem::size_of::<T>() <= 1024 {
+        4
+    } else {
+        1
+    };
+
+    /// Like `new`, but parameterized over the choice of allocator for
+    /// the returned `RawVec`.
+    pub const fn new_in(alloc: A) -> Self {
+        // `cap: 0` means "unallocated". zero-sized types are ignored.
+        Self { ptr: Unique::dangling(), cap: 0, alloc }
+    }
+
+    /// Like `with_capacity`, but parameterized over the choice of
+    /// allocator for the returned `RawVec`.
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    pub fn with_capacity_in(capacity: usize, alloc: A) -> Self {
+        Self::allocate_in(capacity, AllocInit::Uninitialized, alloc)
+    }
+
+    /// Like `with_capacity_zeroed`, but parameterized over the choice
+    /// of allocator for the returned `RawVec`.
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    pub fn with_capacity_zeroed_in(capacity: usize, alloc: A) -> Self {
+        Self::allocate_in(capacity, AllocInit::Zeroed, alloc)
+    }
+
+    /// Converts the entire buffer into `Box<[MaybeUninit<T>]>` with the specified `len`.
+    ///
+    /// Note that this will correctly reconstitute any `cap` changes
+    /// that may have been performed. (See description of type for details.)
+    ///
+    /// # Safety
+    ///
+    /// * `len` must be greater than or equal to the most recently requested capacity, and
+    /// * `len` must be less than or equal to `self.capacity()`.
+    ///
+    /// Note, that the requested capacity and `self.capacity()` could differ, as
+    /// an allocator could overallocate and return a greater memory block than requested.
+    pub unsafe fn into_box(self, len: usize) -> Box<[MaybeUninit<T>], A> {
+        // Sanity-check one half of the safety requirement (we cannot check the other half).
+        debug_assert!(
+            len <= self.capacity(),
+            "`len` must be smaller than or equal to `self.capacity()`"
+        );
+
+        let me = ManuallyDrop::new(self);
+        unsafe {
+            let slice = slice::from_raw_parts_mut(me.ptr() as *mut MaybeUninit<T>, len);
+            Box::from_raw_in(slice, ptr::read(&me.alloc))
+        }
+    }
+
+    #[cfg(not(no_global_oom_handling))]
+    fn allocate_in(capacity: usize, init: AllocInit, alloc: A) -> Self {
+        // Don't allocate here because `Drop` will not deallocate when `capacity` is 0.
+        if mem::size_of::<T>() == 0 || capacity == 0 {
+            Self::new_in(alloc)
+        } else {
+            // We avoid `unwrap_or_else` here because it bloats the amount of
+            // LLVM IR generated.
+            let layout = match Layout::array::<T>(capacity) {
+                Ok(layout) => layout,
+                Err(_) => capacity_overflow(),
+            };
+            match alloc_guard(layout.size()) {
+                Ok(_) => {}
+                Err(_) => capacity_overflow(),
+            }
+            let result = match init {
+                AllocInit::Uninitialized => alloc.allocate(layout),
+                AllocInit::Zeroed => alloc.allocate_zeroed(layout),
+            };
+            let ptr = match result {
+                Ok(ptr) => ptr,
+                Err(_) => handle_alloc_error(layout),
+            };
+
+            // Allocators currently return a `NonNull<[u8]>` whose length
+            // matches the size requested. If that ever changes, the capacity
+            // here should change to `ptr.len() / mem::size_of::<T>()`.
+            Self {
+                ptr: unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) },
+                cap: capacity,
+                alloc,
+            }
+        }
+    }
+
+    /// Reconstitutes a `RawVec` from a pointer, capacity, and allocator.
+    ///
+    /// # Safety
+    ///
+    /// The `ptr` must be allocated (via the given allocator `alloc`), and with the given
+    /// `capacity`.
+    /// The `capacity` cannot exceed `isize::MAX` for sized types. (only a concern on 32-bit
+    /// systems). ZST vectors may have a capacity up to `usize::MAX`.
+    /// If the `ptr` and `capacity` come from a `RawVec` created via `alloc`, then this is
+    /// guaranteed.
+    #[inline]
+    pub unsafe fn from_raw_parts_in(ptr: *mut T, capacity: usize, alloc: A) -> Self {
+        Self { ptr: unsafe { Unique::new_unchecked(ptr) }, cap: capacity, alloc }
+    }
+
+    /// Gets a raw pointer to the start of the allocation. Note that this is
+    /// `Unique::dangling()` if `capacity == 0` or `T` is zero-sized. In the former case, you must
+    /// be careful.
+    #[inline]
+    pub fn ptr(&self) -> *mut T {
+        self.ptr.as_ptr()
+    }
+
+    /// Gets the capacity of the allocation.
+    ///
+    /// This will always be `usize::MAX` if `T` is zero-sized.
+    #[inline(always)]
+    pub fn capacity(&self) -> usize {
+        if mem::size_of::<T>() == 0 { usize::MAX } else { self.cap }
+    }
+
+    /// Returns a shared reference to the allocator backing this `RawVec`.
+    pub fn allocator(&self) -> &A {
+        &self.alloc
+    }
+
+    fn current_memory(&self) -> Option<(NonNull<u8>, Layout)> {
+        if mem::size_of::<T>() == 0 || self.cap == 0 {
+            None
+        } else {
+            // We have an allocated chunk of memory, so we can bypass runtime
+            // checks to get our current layout.
+            unsafe {
+                let layout = Layout::array::<T>(self.cap).unwrap_unchecked();
+                Some((self.ptr.cast().into(), layout))
+            }
+        }
+    }
+
+    /// Ensures that the buffer contains at least enough space to hold `len +
+    /// additional` elements. If it doesn't already have enough capacity, will
+    /// reallocate enough space plus comfortable slack space to get amortized
+    /// *O*(1) behavior. Will limit this behavior if it would needlessly cause
+    /// itself to panic.
+    ///
+    /// If `len` exceeds `self.capacity()`, this may fail to actually allocate
+    /// the requested space. This is not really unsafe, but the unsafe
+    /// code *you* write that relies on the behavior of this function may break.
+    ///
+    /// This is ideal for implementing a bulk-push operation like `extend`.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the new capacity exceeds `isize::MAX` bytes.
+    ///
+    /// # Aborts
+    ///
+    /// Aborts on OOM.
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    pub fn reserve(&mut self, len: usize, additional: usize) {
+        // Callers expect this function to be very cheap when there is already sufficient capacity.
+        // Therefore, we move all the resizing and error-handling logic from grow_amortized and
+        // handle_reserve behind a call, while making sure that this function is likely to be
+        // inlined as just a comparison and a call if the comparison fails.
+        #[cold]
+        fn do_reserve_and_handle<T, A: Allocator>(
+            slf: &mut RawVec<T, A>,
+            len: usize,
+            additional: usize,
+        ) {
+            handle_reserve(slf.grow_amortized(len, additional));
+        }
+
+        if self.needs_to_grow(len, additional) {
+            do_reserve_and_handle(self, len, additional);
+        }
+    }
+
+    /// A specialized version of `reserve()` used only by the hot and
+    /// oft-instantiated `Vec::push()`, which does its own capacity check.
+    #[cfg(not(no_global_oom_handling))]
+    #[inline(never)]
+    pub fn reserve_for_push(&mut self, len: usize) {
+        handle_reserve(self.grow_amortized(len, 1));
+    }
+
+    /// The same as `reserve`, but returns on errors instead of panicking or aborting.
+    pub fn try_reserve(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> {
+        if self.needs_to_grow(len, additional) {
+            self.grow_amortized(len, additional)
+        } else {
+            Ok(())
+        }
+    }
+
+    /// The same as `reserve_for_push`, but returns on errors instead of panicking or aborting.
+    #[inline(never)]
+    pub fn try_reserve_for_push(&mut self, len: usize) -> Result<(), TryReserveError> {
+        self.grow_amortized(len, 1)
+    }
+
+    /// Ensures that the buffer contains at least enough space to hold `len +
+    /// additional` elements. If it doesn't already, will reallocate the
+    /// minimum possible amount of memory necessary. Generally this will be
+    /// exactly the amount of memory necessary, but in principle the allocator
+    /// is free to give back more than we asked for.
+    ///
+    /// If `len` exceeds `self.capacity()`, this may fail to actually allocate
+    /// the requested space. This is not really unsafe, but the unsafe code
+    /// *you* write that relies on the behavior of this function may break.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the new capacity exceeds `isize::MAX` bytes.
+    ///
+    /// # Aborts
+    ///
+    /// Aborts on OOM.
+    #[cfg(not(no_global_oom_handling))]
+    pub fn reserve_exact(&mut self, len: usize, additional: usize) {
+        handle_reserve(self.try_reserve_exact(len, additional));
+    }
+
+    /// The same as `reserve_exact`, but returns on errors instead of panicking or aborting.
+    pub fn try_reserve_exact(
+        &mut self,
+        len: usize,
+        additional: usize,
+    ) -> Result<(), TryReserveError> {
+        if self.needs_to_grow(len, additional) { self.grow_exact(len, additional) } else { Ok(()) }
+    }
+
+    /// Shrinks the buffer down to the specified capacity. If the given amount
+    /// is 0, actually completely deallocates.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the given amount is *larger* than the current capacity.
+    ///
+    /// # Aborts
+    ///
+    /// Aborts on OOM.
+    #[cfg(not(no_global_oom_handling))]
+    pub fn shrink_to_fit(&mut self, cap: usize) {
+        handle_reserve(self.shrink(cap));
+    }
+}
+
+impl<T, A: Allocator> RawVec<T, A> {
+    /// Returns if the buffer needs to grow to fulfill the needed extra capacity.
+    /// Mainly used to make inlining reserve-calls possible without inlining `grow`.
+    fn needs_to_grow(&self, len: usize, additional: usize) -> bool {
+        additional > self.capacity().wrapping_sub(len)
+    }
+
+    fn set_ptr_and_cap(&mut self, ptr: NonNull<[u8]>, cap: usize) {
+        // Allocators currently return a `NonNull<[u8]>` whose length matches
+        // the size requested. If that ever changes, the capacity here should
+        // change to `ptr.len() / mem::size_of::<T>()`.
+        self.ptr = unsafe { Unique::new_unchecked(ptr.cast().as_ptr()) };
+        self.cap = cap;
+    }
+
+    // This method is usually instantiated many times. So we want it to be as
+    // small as possible, to improve compile times. But we also want as much of
+    // its contents to be statically computable as possible, to make the
+    // generated code run faster. Therefore, this method is carefully written
+    // so that all of the code that depends on `T` is within it, while as much
+    // of the code that doesn't depend on `T` as possible is in functions that
+    // are non-generic over `T`.
+    fn grow_amortized(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> {
+        // This is ensured by the calling contexts.
+        debug_assert!(additional > 0);
+
+        if mem::size_of::<T>() == 0 {
+            // Since we return a capacity of `usize::MAX` when `elem_size` is
+            // 0, getting to here necessarily means the `RawVec` is overfull.
+            return Err(CapacityOverflow.into());
+        }
+
+        // Nothing we can really do about these checks, sadly.
+        let required_cap = len.checked_add(additional).ok_or(CapacityOverflow)?;
+
+        // This guarantees exponential growth. The doubling cannot overflow
+        // because `cap <= isize::MAX` and the type of `cap` is `usize`.
+        let cap = cmp::max(self.cap * 2, required_cap);
+        let cap = cmp::max(Self::MIN_NON_ZERO_CAP, cap);
+
+        let new_layout = Layout::array::<T>(cap);
+
+        // `finish_grow` is non-generic over `T`.
+        let ptr = finish_grow(new_layout, self.current_memory(), &mut self.alloc)?;
+        self.set_ptr_and_cap(ptr, cap);
+        Ok(())
+    }
+
+    // The constraints on this method are much the same as those on
+    // `grow_amortized`, but this method is usually instantiated less often so
+    // it's less critical.
+    fn grow_exact(&mut self, len: usize, additional: usize) -> Result<(), TryReserveError> {
+        if mem::size_of::<T>() == 0 {
+            // Since we return a capacity of `usize::MAX` when the type size is
+            // 0, getting to here necessarily means the `RawVec` is overfull.
+            return Err(CapacityOverflow.into());
+        }
+
+        let cap = len.checked_add(additional).ok_or(CapacityOverflow)?;
+        let new_layout = Layout::array::<T>(cap);
+
+        // `finish_grow` is non-generic over `T`.
+        let ptr = finish_grow(new_layout, self.current_memory(), &mut self.alloc)?;
+        self.set_ptr_and_cap(ptr, cap);
+        Ok(())
+    }
+
+    #[allow(dead_code)]
+    fn shrink(&mut self, cap: usize) -> Result<(), TryReserveError> {
+        assert!(cap <= self.capacity(), "Tried to shrink to a larger capacity");
+
+        let (ptr, layout) = if let Some(mem) = self.current_memory() { mem } else { return Ok(()) };
+
+        let ptr = unsafe {
+            // `Layout::array` cannot overflow here because it would have
+            // overflowed earlier when capacity was larger.
+            let new_layout = Layout::array::<T>(cap).unwrap_unchecked();
+            self.alloc
+                .shrink(ptr, layout, new_layout)
+                .map_err(|_| AllocError { layout: new_layout, non_exhaustive: () })?
+        };
+        self.set_ptr_and_cap(ptr, cap);
+        Ok(())
+    }
+}
+
+// This function is outside `RawVec` to minimize compile times. See the comment
+// above `RawVec::grow_amortized` for details. (The `A` parameter isn't
+// significant, because the number of different `A` types seen in practice is
+// much smaller than the number of `T` types.)
+#[inline(never)]
+fn finish_grow<A>(
+    new_layout: Result<Layout, LayoutError>,
+    current_memory: Option<(NonNull<u8>, Layout)>,
+    alloc: &mut A,
+) -> Result<NonNull<[u8]>, TryReserveError>
+where
+    A: Allocator,
+{
+    // Check for the error here to minimize the size of `RawVec::grow_*`.
+    let new_layout = new_layout.map_err(|_| CapacityOverflow)?;
+
+    alloc_guard(new_layout.size())?;
+
+    let memory = if let Some((ptr, old_layout)) = current_memory {
+        debug_assert_eq!(old_layout.align(), new_layout.align());
+        unsafe {
+            // The allocator checks for alignment equality
+            intrinsics::assume(old_layout.align() == new_layout.align());
+            alloc.grow(ptr, old_layout, new_layout)
+        }
+    } else {
+        alloc.allocate(new_layout)
+    };
+
+    memory.map_err(|_| AllocError { layout: new_layout, non_exhaustive: () }.into())
+}
+
+unsafe impl<#[may_dangle] T, A: Allocator> Drop for RawVec<T, A> {
+    /// Frees the memory owned by the `RawVec` *without* trying to drop its contents.
+    fn drop(&mut self) {
+        if let Some((ptr, layout)) = self.current_memory() {
+            unsafe { self.alloc.deallocate(ptr, layout) }
+        }
+    }
+}
+
+// Central function for reserve error handling.
+#[cfg(not(no_global_oom_handling))]
+#[inline]
+fn handle_reserve(result: Result<(), TryReserveError>) {
+    match result.map_err(|e| e.kind()) {
+        Err(CapacityOverflow) => capacity_overflow(),
+        Err(AllocError { layout, .. }) => handle_alloc_error(layout),
+        Ok(()) => { /* yay */ }
+    }
+}
+
+// We need to guarantee the following:
+// * We don't ever allocate `> isize::MAX` byte-size objects.
+// * We don't overflow `usize::MAX` and actually allocate too little.
+//
+// On 64-bit we just need to check for overflow since trying to allocate
+// `> isize::MAX` bytes will surely fail. On 32-bit and 16-bit we need to add
+// an extra guard for this in case we're running on a platform which can use
+// all 4GB in user-space, e.g., PAE or x32.
+
+#[inline]
+fn alloc_guard(alloc_size: usize) -> Result<(), TryReserveError> {
+    if usize::BITS < 64 && alloc_size > isize::MAX as usize {
+        Err(CapacityOverflow.into())
+    } else {
+        Ok(())
+    }
+}
+
+// One central function responsible for reporting capacity overflows. This'll
+// ensure that the code generation related to these panics is minimal as there's
+// only one location which panics rather than a bunch throughout the module.
+#[cfg(not(no_global_oom_handling))]
+fn capacity_overflow() -> ! {
+    panic!("capacity overflow");
+}
diff --git a/rust/alloc/slice.rs b/rust/alloc/slice.rs
new file mode 100644
index 000000000000..e444e97fa145
--- /dev/null
+++ b/rust/alloc/slice.rs
@@ -0,0 +1,1204 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+//! A dynamically-sized view into a contiguous sequence, `[T]`.
+//!
+//! *[See also the slice primitive type](slice).*
+//!
+//! Slices are a view into a block of memory represented as a pointer and a
+//! length.
+//!
+//! ```
+//! // slicing a Vec
+//! let vec = vec![1, 2, 3];
+//! let int_slice = &vec[..];
+//! // coercing an array to a slice
+//! let str_slice: &[&str] = &["one", "two", "three"];
+//! ```
+//!
+//! Slices are either mutable or shared. The shared slice type is `&[T]`,
+//! while the mutable slice type is `&mut [T]`, where `T` represents the element
+//! type. For example, you can mutate the block of memory that a mutable slice
+//! points to:
+//!
+//! ```
+//! let x = &mut [1, 2, 3];
+//! x[1] = 7;
+//! assert_eq!(x, &[1, 7, 3]);
+//! ```
+//!
+//! Here are some of the things this module contains:
+//!
+//! ## Structs
+//!
+//! There are several structs that are useful for slices, such as [`Iter`], which
+//! represents iteration over a slice.
+//!
+//! ## Trait Implementations
+//!
+//! There are several implementations of common traits for slices. Some examples
+//! include:
+//!
+//! * [`Clone`]
+//! * [`Eq`], [`Ord`] - for slices whose element type are [`Eq`] or [`Ord`].
+//! * [`Hash`] - for slices whose element type is [`Hash`].
+//!
+//! ## Iteration
+//!
+//! The slices implement `IntoIterator`. The iterator yields references to the
+//! slice elements.
+//!
+//! ```
+//! let numbers = &[0, 1, 2];
+//! for n in numbers {
+//!     println!("{n} is a number!");
+//! }
+//! ```
+//!
+//! The mutable slice yields mutable references to the elements:
+//!
+//! ```
+//! let mut scores = [7, 8, 9];
+//! for score in &mut scores[..] {
+//!     *score += 1;
+//! }
+//! ```
+//!
+//! This iterator yields mutable references to the slice's elements, so while
+//! the element type of the slice is `i32`, the element type of the iterator is
+//! `&mut i32`.
+//!
+//! * [`.iter`] and [`.iter_mut`] are the explicit methods to return the default
+//!   iterators.
+//! * Further methods that return iterators are [`.split`], [`.splitn`],
+//!   [`.chunks`], [`.windows`] and more.
+//!
+//! [`Hash`]: core::hash::Hash
+//! [`.iter`]: slice::iter
+//! [`.iter_mut`]: slice::iter_mut
+//! [`.split`]: slice::split
+//! [`.splitn`]: slice::splitn
+//! [`.chunks`]: slice::chunks
+//! [`.windows`]: slice::windows
+#![stable(feature = "rust1", since = "1.0.0")]
+// Many of the usings in this module are only used in the test configuration.
+// It's cleaner to just turn off the unused_imports warning than to fix them.
+#![cfg_attr(test, allow(unused_imports, dead_code))]
+
+use core::borrow::{Borrow, BorrowMut};
+#[cfg(not(no_global_oom_handling))]
+use core::cmp::Ordering::{self, Less};
+#[cfg(not(no_global_oom_handling))]
+use core::mem;
+#[cfg(not(no_global_oom_handling))]
+use core::mem::size_of;
+#[cfg(not(no_global_oom_handling))]
+use core::ptr;
+
+use crate::alloc::Allocator;
+#[cfg(not(no_global_oom_handling))]
+use crate::alloc::Global;
+#[cfg(not(no_global_oom_handling))]
+use crate::borrow::ToOwned;
+use crate::boxed::Box;
+use crate::vec::Vec;
+
+#[unstable(feature = "slice_range", issue = "76393")]
+pub use core::slice::range;
+#[unstable(feature = "array_chunks", issue = "74985")]
+pub use core::slice::ArrayChunks;
+#[unstable(feature = "array_chunks", issue = "74985")]
+pub use core::slice::ArrayChunksMut;
+#[unstable(feature = "array_windows", issue = "75027")]
+pub use core::slice::ArrayWindows;
+#[stable(feature = "inherent_ascii_escape", since = "1.60.0")]
+pub use core::slice::EscapeAscii;
+#[stable(feature = "slice_get_slice", since = "1.28.0")]
+pub use core::slice::SliceIndex;
+#[stable(feature = "from_ref", since = "1.28.0")]
+pub use core::slice::{from_mut, from_ref};
+#[stable(feature = "rust1", since = "1.0.0")]
+pub use core::slice::{from_raw_parts, from_raw_parts_mut};
+#[stable(feature = "rust1", since = "1.0.0")]
+pub use core::slice::{Chunks, Windows};
+#[stable(feature = "chunks_exact", since = "1.31.0")]
+pub use core::slice::{ChunksExact, ChunksExactMut};
+#[stable(feature = "rust1", since = "1.0.0")]
+pub use core::slice::{ChunksMut, Split, SplitMut};
+#[unstable(feature = "slice_group_by", issue = "80552")]
+pub use core::slice::{GroupBy, GroupByMut};
+#[stable(feature = "rust1", since = "1.0.0")]
+pub use core::slice::{Iter, IterMut};
+#[stable(feature = "rchunks", since = "1.31.0")]
+pub use core::slice::{RChunks, RChunksExact, RChunksExactMut, RChunksMut};
+#[stable(feature = "slice_rsplit", since = "1.27.0")]
+pub use core::slice::{RSplit, RSplitMut};
+#[stable(feature = "rust1", since = "1.0.0")]
+pub use core::slice::{RSplitN, RSplitNMut, SplitN, SplitNMut};
+#[stable(feature = "split_inclusive", since = "1.51.0")]
+pub use core::slice::{SplitInclusive, SplitInclusiveMut};
+
+////////////////////////////////////////////////////////////////////////////////
+// Basic slice extension methods
+////////////////////////////////////////////////////////////////////////////////
+
+// HACK(japaric) needed for the implementation of `vec!` macro during testing
+// N.B., see the `hack` module in this file for more details.
+#[cfg(test)]
+pub use hack::into_vec;
+
+// HACK(japaric) needed for the implementation of `Vec::clone` during testing
+// N.B., see the `hack` module in this file for more details.
+#[cfg(test)]
+pub use hack::to_vec;
+
+// HACK(japaric): With cfg(test) `impl [T]` is not available, these three
+// functions are actually methods that are in `impl [T]` but not in
+// `core::slice::SliceExt` - we need to supply these functions for the
+// `test_permutations` test
+pub(crate) mod hack {
+    use core::alloc::Allocator;
+
+    use crate::boxed::Box;
+    use crate::vec::Vec;
+
+    // We shouldn't add inline attribute to this since this is used in
+    // `vec!` macro mostly and causes perf regression. See #71204 for
+    // discussion and perf results.
+    pub fn into_vec<T, A: Allocator>(b: Box<[T], A>) -> Vec<T, A> {
+        unsafe {
+            let len = b.len();
+            let (b, alloc) = Box::into_raw_with_allocator(b);
+            Vec::from_raw_parts_in(b as *mut T, len, len, alloc)
+        }
+    }
+
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    pub fn to_vec<T: ConvertVec, A: Allocator>(s: &[T], alloc: A) -> Vec<T, A> {
+        T::to_vec(s, alloc)
+    }
+
+    #[cfg(not(no_global_oom_handling))]
+    pub trait ConvertVec {
+        fn to_vec<A: Allocator>(s: &[Self], alloc: A) -> Vec<Self, A>
+        where
+            Self: Sized;
+    }
+
+    #[cfg(not(no_global_oom_handling))]
+    impl<T: Clone> ConvertVec for T {
+        #[inline]
+        default fn to_vec<A: Allocator>(s: &[Self], alloc: A) -> Vec<Self, A> {
+            struct DropGuard<'a, T, A: Allocator> {
+                vec: &'a mut Vec<T, A>,
+                num_init: usize,
+            }
+            impl<'a, T, A: Allocator> Drop for DropGuard<'a, T, A> {
+                #[inline]
+                fn drop(&mut self) {
+                    // SAFETY:
+                    // items were marked initialized in the loop below
+                    unsafe {
+                        self.vec.set_len(self.num_init);
+                    }
+                }
+            }
+            let mut vec = Vec::with_capacity_in(s.len(), alloc);
+            let mut guard = DropGuard { vec: &mut vec, num_init: 0 };
+            let slots = guard.vec.spare_capacity_mut();
+            // .take(slots.len()) is necessary for LLVM to remove bounds checks
+            // and has better codegen than zip.
+            for (i, b) in s.iter().enumerate().take(slots.len()) {
+                guard.num_init = i;
+                slots[i].write(b.clone());
+            }
+            core::mem::forget(guard);
+            // SAFETY:
+            // the vec was allocated and initialized above to at least this length.
+            unsafe {
+                vec.set_len(s.len());
+            }
+            vec
+        }
+    }
+
+    #[cfg(not(no_global_oom_handling))]
+    impl<T: Copy> ConvertVec for T {
+        #[inline]
+        fn to_vec<A: Allocator>(s: &[Self], alloc: A) -> Vec<Self, A> {
+            let mut v = Vec::with_capacity_in(s.len(), alloc);
+            // SAFETY:
+            // allocated above with the capacity of `s`, and initialize to `s.len()` in
+            // ptr::copy_to_non_overlapping below.
+            unsafe {
+                s.as_ptr().copy_to_nonoverlapping(v.as_mut_ptr(), s.len());
+                v.set_len(s.len());
+            }
+            v
+        }
+    }
+}
+
+#[cfg(not(test))]
+impl<T> [T] {
+    /// Sorts the slice.
+    ///
+    /// This sort is stable (i.e., does not reorder equal elements) and *O*(*n* \* log(*n*)) worst-case.
+    ///
+    /// When applicable, unstable sorting is preferred because it is generally faster than stable
+    /// sorting and it doesn't allocate auxiliary memory.
+    /// See [`sort_unstable`](slice::sort_unstable).
+    ///
+    /// # Current implementation
+    ///
+    /// The current algorithm is an adaptive, iterative merge sort inspired by
+    /// [timsort](https://en.wikipedia.org/wiki/Timsort).
+    /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of
+    /// two or more sorted sequences concatenated one after another.
+    ///
+    /// Also, it allocates temporary storage half the size of `self`, but for short slices a
+    /// non-allocating insertion sort is used instead.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = [-5, 4, 1, -3, 2];
+    ///
+    /// v.sort();
+    /// assert!(v == [-5, -3, 1, 2, 4]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[rustc_allow_incoherent_impl]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[inline]
+    pub fn sort(&mut self)
+    where
+        T: Ord,
+    {
+        merge_sort(self, |a, b| a.lt(b));
+    }
+
+    /// Sorts the slice with a comparator function.
+    ///
+    /// This sort is stable (i.e., does not reorder equal elements) and *O*(*n* \* log(*n*)) worst-case.
+    ///
+    /// The comparator function must define a total ordering for the elements in the slice. If
+    /// the ordering is not total, the order of the elements is unspecified. An order is a
+    /// total order if it is (for all `a`, `b` and `c`):
+    ///
+    /// * total and antisymmetric: exactly one of `a < b`, `a == b` or `a > b` is true, and
+    /// * transitive, `a < b` and `b < c` implies `a < c`. The same must hold for both `==` and `>`.
+    ///
+    /// For example, while [`f64`] doesn't implement [`Ord`] because `NaN != NaN`, we can use
+    /// `partial_cmp` as our sort function when we know the slice doesn't contain a `NaN`.
+    ///
+    /// ```
+    /// let mut floats = [5f64, 4.0, 1.0, 3.0, 2.0];
+    /// floats.sort_by(|a, b| a.partial_cmp(b).unwrap());
+    /// assert_eq!(floats, [1.0, 2.0, 3.0, 4.0, 5.0]);
+    /// ```
+    ///
+    /// When applicable, unstable sorting is preferred because it is generally faster than stable
+    /// sorting and it doesn't allocate auxiliary memory.
+    /// See [`sort_unstable_by`](slice::sort_unstable_by).
+    ///
+    /// # Current implementation
+    ///
+    /// The current algorithm is an adaptive, iterative merge sort inspired by
+    /// [timsort](https://en.wikipedia.org/wiki/Timsort).
+    /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of
+    /// two or more sorted sequences concatenated one after another.
+    ///
+    /// Also, it allocates temporary storage half the size of `self`, but for short slices a
+    /// non-allocating insertion sort is used instead.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = [5, 4, 1, 3, 2];
+    /// v.sort_by(|a, b| a.cmp(b));
+    /// assert!(v == [1, 2, 3, 4, 5]);
+    ///
+    /// // reverse sorting
+    /// v.sort_by(|a, b| b.cmp(a));
+    /// assert!(v == [5, 4, 3, 2, 1]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[rustc_allow_incoherent_impl]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[inline]
+    pub fn sort_by<F>(&mut self, mut compare: F)
+    where
+        F: FnMut(&T, &T) -> Ordering,
+    {
+        merge_sort(self, |a, b| compare(a, b) == Less);
+    }
+
+    /// Sorts the slice with a key extraction function.
+    ///
+    /// This sort is stable (i.e., does not reorder equal elements) and *O*(*m* \* *n* \* log(*n*))
+    /// worst-case, where the key function is *O*(*m*).
+    ///
+    /// For expensive key functions (e.g. functions that are not simple property accesses or
+    /// basic operations), [`sort_by_cached_key`](slice::sort_by_cached_key) is likely to be
+    /// significantly faster, as it does not recompute element keys.
+    ///
+    /// When applicable, unstable sorting is preferred because it is generally faster than stable
+    /// sorting and it doesn't allocate auxiliary memory.
+    /// See [`sort_unstable_by_key`](slice::sort_unstable_by_key).
+    ///
+    /// # Current implementation
+    ///
+    /// The current algorithm is an adaptive, iterative merge sort inspired by
+    /// [timsort](https://en.wikipedia.org/wiki/Timsort).
+    /// It is designed to be very fast in cases where the slice is nearly sorted, or consists of
+    /// two or more sorted sequences concatenated one after another.
+    ///
+    /// Also, it allocates temporary storage half the size of `self`, but for short slices a
+    /// non-allocating insertion sort is used instead.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = [-5i32, 4, 1, -3, 2];
+    ///
+    /// v.sort_by_key(|k| k.abs());
+    /// assert!(v == [1, 2, -3, 4, -5]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[rustc_allow_incoherent_impl]
+    #[stable(feature = "slice_sort_by_key", since = "1.7.0")]
+    #[inline]
+    pub fn sort_by_key<K, F>(&mut self, mut f: F)
+    where
+        F: FnMut(&T) -> K,
+        K: Ord,
+    {
+        merge_sort(self, |a, b| f(a).lt(&f(b)));
+    }
+
+    /// Sorts the slice with a key extraction function.
+    ///
+    /// During sorting, the key function is called at most once per element, by using
+    /// temporary storage to remember the results of key evaluation.
+    /// The order of calls to the key function is unspecified and may change in future versions
+    /// of the standard library.
+    ///
+    /// This sort is stable (i.e., does not reorder equal elements) and *O*(*m* \* *n* + *n* \* log(*n*))
+    /// worst-case, where the key function is *O*(*m*).
+    ///
+    /// For simple key functions (e.g., functions that are property accesses or
+    /// basic operations), [`sort_by_key`](slice::sort_by_key) is likely to be
+    /// faster.
+    ///
+    /// # Current implementation
+    ///
+    /// The current algorithm is based on [pattern-defeating quicksort][pdqsort] by Orson Peters,
+    /// which combines the fast average case of randomized quicksort with the fast worst case of
+    /// heapsort, while achieving linear time on slices with certain patterns. It uses some
+    /// randomization to avoid degenerate cases, but with a fixed seed to always provide
+    /// deterministic behavior.
+    ///
+    /// In the worst case, the algorithm allocates temporary storage in a `Vec<(K, usize)>` the
+    /// length of the slice.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = [-5i32, 4, 32, -3, 2];
+    ///
+    /// v.sort_by_cached_key(|k| k.to_string());
+    /// assert!(v == [-3, -5, 2, 32, 4]);
+    /// ```
+    ///
+    /// [pdqsort]: https://github.com/orlp/pdqsort
+    #[cfg(not(no_global_oom_handling))]
+    #[rustc_allow_incoherent_impl]
+    #[stable(feature = "slice_sort_by_cached_key", since = "1.34.0")]
+    #[inline]
+    pub fn sort_by_cached_key<K, F>(&mut self, f: F)
+    where
+        F: FnMut(&T) -> K,
+        K: Ord,
+    {
+        // Helper macro for indexing our vector by the smallest possible type, to reduce allocation.
+        macro_rules! sort_by_key {
+            ($t:ty, $slice:ident, $f:ident) => {{
+                let mut indices: Vec<_> =
+                    $slice.iter().map($f).enumerate().map(|(i, k)| (k, i as $t)).collect();
+                // The elements of `indices` are unique, as they are indexed, so any sort will be
+                // stable with respect to the original slice. We use `sort_unstable` here because
+                // it requires less memory allocation.
+                indices.sort_unstable();
+                for i in 0..$slice.len() {
+                    let mut index = indices[i].1;
+                    while (index as usize) < i {
+                        index = indices[index as usize].1;
+                    }
+                    indices[i].1 = index;
+                    $slice.swap(i, index as usize);
+                }
+            }};
+        }
+
+        let sz_u8 = mem::size_of::<(K, u8)>();
+        let sz_u16 = mem::size_of::<(K, u16)>();
+        let sz_u32 = mem::size_of::<(K, u32)>();
+        let sz_usize = mem::size_of::<(K, usize)>();
+
+        let len = self.len();
+        if len < 2 {
+            return;
+        }
+        if sz_u8 < sz_u16 && len <= (u8::MAX as usize) {
+            return sort_by_key!(u8, self, f);
+        }
+        if sz_u16 < sz_u32 && len <= (u16::MAX as usize) {
+            return sort_by_key!(u16, self, f);
+        }
+        if sz_u32 < sz_usize && len <= (u32::MAX as usize) {
+            return sort_by_key!(u32, self, f);
+        }
+        sort_by_key!(usize, self, f)
+    }
+
+    /// Copies `self` into a new `Vec`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let s = [10, 40, 30];
+    /// let x = s.to_vec();
+    /// // Here, `s` and `x` can be modified independently.
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[rustc_allow_incoherent_impl]
+    #[rustc_conversion_suggestion]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[inline]
+    pub fn to_vec(&self) -> Vec<T>
+    where
+        T: Clone,
+    {
+        self.to_vec_in(Global)
+    }
+
+    /// Copies `self` into a new `Vec` with an allocator.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let s = [10, 40, 30];
+    /// let x = s.to_vec_in(System);
+    /// // Here, `s` and `x` can be modified independently.
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[rustc_allow_incoherent_impl]
+    #[inline]
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    pub fn to_vec_in<A: Allocator>(&self, alloc: A) -> Vec<T, A>
+    where
+        T: Clone,
+    {
+        // N.B., see the `hack` module in this file for more details.
+        hack::to_vec(self, alloc)
+    }
+
+    /// Converts `self` into a vector without clones or allocation.
+    ///
+    /// The resulting vector can be converted back into a box via
+    /// `Vec<T>`'s `into_boxed_slice` method.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let s: Box<[i32]> = Box::new([10, 40, 30]);
+    /// let x = s.into_vec();
+    /// // `s` cannot be used anymore because it has been converted into `x`.
+    ///
+    /// assert_eq!(x, vec![10, 40, 30]);
+    /// ```
+    #[rustc_allow_incoherent_impl]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[inline]
+    pub fn into_vec<A: Allocator>(self: Box<Self, A>) -> Vec<T, A> {
+        // N.B., see the `hack` module in this file for more details.
+        hack::into_vec(self)
+    }
+
+    /// Creates a vector by repeating a slice `n` times.
+    ///
+    /// # Panics
+    ///
+    /// This function will panic if the capacity would overflow.
+    ///
+    /// # Examples
+    ///
+    /// Basic usage:
+    ///
+    /// ```
+    /// assert_eq!([1, 2].repeat(3), vec![1, 2, 1, 2, 1, 2]);
+    /// ```
+    ///
+    /// A panic upon overflow:
+    ///
+    /// ```should_panic
+    /// // this will panic at runtime
+    /// b"0123456789abcdef".repeat(usize::MAX);
+    /// ```
+    #[rustc_allow_incoherent_impl]
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "repeat_generic_slice", since = "1.40.0")]
+    pub fn repeat(&self, n: usize) -> Vec<T>
+    where
+        T: Copy,
+    {
+        if n == 0 {
+            return Vec::new();
+        }
+
+        // If `n` is larger than zero, it can be split as
+        // `n = 2^expn + rem (2^expn > rem, expn >= 0, rem >= 0)`.
+        // `2^expn` is the number represented by the leftmost '1' bit of `n`,
+        // and `rem` is the remaining part of `n`.
+
+        // Using `Vec` to access `set_len()`.
+        let capacity = self.len().checked_mul(n).expect("capacity overflow");
+        let mut buf = Vec::with_capacity(capacity);
+
+        // `2^expn` repetition is done by doubling `buf` `expn`-times.
+        buf.extend(self);
+        {
+            let mut m = n >> 1;
+            // If `m > 0`, there are remaining bits up to the leftmost '1'.
+            while m > 0 {
+                // `buf.extend(buf)`:
+                unsafe {
+                    ptr::copy_nonoverlapping(
+                        buf.as_ptr(),
+                        (buf.as_mut_ptr() as *mut T).add(buf.len()),
+                        buf.len(),
+                    );
+                    // `buf` has capacity of `self.len() * n`.
+                    let buf_len = buf.len();
+                    buf.set_len(buf_len * 2);
+                }
+
+                m >>= 1;
+            }
+        }
+
+        // `rem` (`= n - 2^expn`) repetition is done by copying
+        // first `rem` repetitions from `buf` itself.
+        let rem_len = capacity - buf.len(); // `self.len() * rem`
+        if rem_len > 0 {
+            // `buf.extend(buf[0 .. rem_len])`:
+            unsafe {
+                // This is non-overlapping since `2^expn > rem`.
+                ptr::copy_nonoverlapping(
+                    buf.as_ptr(),
+                    (buf.as_mut_ptr() as *mut T).add(buf.len()),
+                    rem_len,
+                );
+                // `buf.len() + rem_len` equals to `buf.capacity()` (`= self.len() * n`).
+                buf.set_len(capacity);
+            }
+        }
+        buf
+    }
+
+    /// Flattens a slice of `T` into a single value `Self::Output`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// assert_eq!(["hello", "world"].concat(), "helloworld");
+    /// assert_eq!([[1, 2], [3, 4]].concat(), [1, 2, 3, 4]);
+    /// ```
+    #[rustc_allow_incoherent_impl]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn concat<Item: ?Sized>(&self) -> <Self as Concat<Item>>::Output
+    where
+        Self: Concat<Item>,
+    {
+        Concat::concat(self)
+    }
+
+    /// Flattens a slice of `T` into a single value `Self::Output`, placing a
+    /// given separator between each.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// assert_eq!(["hello", "world"].join(" "), "hello world");
+    /// assert_eq!([[1, 2], [3, 4]].join(&0), [1, 2, 0, 3, 4]);
+    /// assert_eq!([[1, 2], [3, 4]].join(&[0, 0][..]), [1, 2, 0, 0, 3, 4]);
+    /// ```
+    #[rustc_allow_incoherent_impl]
+    #[stable(feature = "rename_connect_to_join", since = "1.3.0")]
+    pub fn join<Separator>(&self, sep: Separator) -> <Self as Join<Separator>>::Output
+    where
+        Self: Join<Separator>,
+    {
+        Join::join(self, sep)
+    }
+
+    /// Flattens a slice of `T` into a single value `Self::Output`, placing a
+    /// given separator between each.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #![allow(deprecated)]
+    /// assert_eq!(["hello", "world"].connect(" "), "hello world");
+    /// assert_eq!([[1, 2], [3, 4]].connect(&0), [1, 2, 0, 3, 4]);
+    /// ```
+    #[rustc_allow_incoherent_impl]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[deprecated(since = "1.3.0", note = "renamed to join")]
+    pub fn connect<Separator>(&self, sep: Separator) -> <Self as Join<Separator>>::Output
+    where
+        Self: Join<Separator>,
+    {
+        Join::join(self, sep)
+    }
+}
+
+#[cfg(not(test))]
+impl [u8] {
+    /// Returns a vector containing a copy of this slice where each byte
+    /// is mapped to its ASCII upper case equivalent.
+    ///
+    /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z',
+    /// but non-ASCII letters are unchanged.
+    ///
+    /// To uppercase the value in-place, use [`make_ascii_uppercase`].
+    ///
+    /// [`make_ascii_uppercase`]: slice::make_ascii_uppercase
+    #[cfg(not(no_global_oom_handling))]
+    #[rustc_allow_incoherent_impl]
+    #[must_use = "this returns the uppercase bytes as a new Vec, \
+                  without modifying the original"]
+    #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")]
+    #[inline]
+    pub fn to_ascii_uppercase(&self) -> Vec<u8> {
+        let mut me = self.to_vec();
+        me.make_ascii_uppercase();
+        me
+    }
+
+    /// Returns a vector containing a copy of this slice where each byte
+    /// is mapped to its ASCII lower case equivalent.
+    ///
+    /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z',
+    /// but non-ASCII letters are unchanged.
+    ///
+    /// To lowercase the value in-place, use [`make_ascii_lowercase`].
+    ///
+    /// [`make_ascii_lowercase`]: slice::make_ascii_lowercase
+    #[cfg(not(no_global_oom_handling))]
+    #[rustc_allow_incoherent_impl]
+    #[must_use = "this returns the lowercase bytes as a new Vec, \
+                  without modifying the original"]
+    #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")]
+    #[inline]
+    pub fn to_ascii_lowercase(&self) -> Vec<u8> {
+        let mut me = self.to_vec();
+        me.make_ascii_lowercase();
+        me
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Extension traits for slices over specific kinds of data
+////////////////////////////////////////////////////////////////////////////////
+
+/// Helper trait for [`[T]::concat`](slice::concat).
+///
+/// Note: the `Item` type parameter is not used in this trait,
+/// but it allows impls to be more generic.
+/// Without it, we get this error:
+///
+/// ```error
+/// error[E0207]: the type parameter `T` is not constrained by the impl trait, self type, or predica
+///    --> src/liballoc/slice.rs:608:6
+///     |
+/// 608 | impl<T: Clone, V: Borrow<[T]>> Concat for [V] {
+///     |      ^ unconstrained type parameter
+/// ```
+///
+/// This is because there could exist `V` types with multiple `Borrow<[_]>` impls,
+/// such that multiple `T` types would apply:
+///
+/// ```
+/// # #[allow(dead_code)]
+/// pub struct Foo(Vec<u32>, Vec<String>);
+///
+/// impl std::borrow::Borrow<[u32]> for Foo {
+///     fn borrow(&self) -> &[u32] { &self.0 }
+/// }
+///
+/// impl std::borrow::Borrow<[String]> for Foo {
+///     fn borrow(&self) -> &[String] { &self.1 }
+/// }
+/// ```
+#[unstable(feature = "slice_concat_trait", issue = "27747")]
+pub trait Concat<Item: ?Sized> {
+    #[unstable(feature = "slice_concat_trait", issue = "27747")]
+    /// The resulting type after concatenation
+    type Output;
+
+    /// Implementation of [`[T]::concat`](slice::concat)
+    #[unstable(feature = "slice_concat_trait", issue = "27747")]
+    fn concat(slice: &Self) -> Self::Output;
+}
+
+/// Helper trait for [`[T]::join`](slice::join)
+#[unstable(feature = "slice_concat_trait", issue = "27747")]
+pub trait Join<Separator> {
+    #[unstable(feature = "slice_concat_trait", issue = "27747")]
+    /// The resulting type after concatenation
+    type Output;
+
+    /// Implementation of [`[T]::join`](slice::join)
+    #[unstable(feature = "slice_concat_trait", issue = "27747")]
+    fn join(slice: &Self, sep: Separator) -> Self::Output;
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[unstable(feature = "slice_concat_ext", issue = "27747")]
+impl<T: Clone, V: Borrow<[T]>> Concat<T> for [V] {
+    type Output = Vec<T>;
+
+    fn concat(slice: &Self) -> Vec<T> {
+        let size = slice.iter().map(|slice| slice.borrow().len()).sum();
+        let mut result = Vec::with_capacity(size);
+        for v in slice {
+            result.extend_from_slice(v.borrow())
+        }
+        result
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[unstable(feature = "slice_concat_ext", issue = "27747")]
+impl<T: Clone, V: Borrow<[T]>> Join<&T> for [V] {
+    type Output = Vec<T>;
+
+    fn join(slice: &Self, sep: &T) -> Vec<T> {
+        let mut iter = slice.iter();
+        let first = match iter.next() {
+            Some(first) => first,
+            None => return vec![],
+        };
+        let size = slice.iter().map(|v| v.borrow().len()).sum::<usize>() + slice.len() - 1;
+        let mut result = Vec::with_capacity(size);
+        result.extend_from_slice(first.borrow());
+
+        for v in iter {
+            result.push(sep.clone());
+            result.extend_from_slice(v.borrow())
+        }
+        result
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[unstable(feature = "slice_concat_ext", issue = "27747")]
+impl<T: Clone, V: Borrow<[T]>> Join<&[T]> for [V] {
+    type Output = Vec<T>;
+
+    fn join(slice: &Self, sep: &[T]) -> Vec<T> {
+        let mut iter = slice.iter();
+        let first = match iter.next() {
+            Some(first) => first,
+            None => return vec![],
+        };
+        let size =
+            slice.iter().map(|v| v.borrow().len()).sum::<usize>() + sep.len() * (slice.len() - 1);
+        let mut result = Vec::with_capacity(size);
+        result.extend_from_slice(first.borrow());
+
+        for v in iter {
+            result.extend_from_slice(sep);
+            result.extend_from_slice(v.borrow())
+        }
+        result
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Standard trait implementations for slices
+////////////////////////////////////////////////////////////////////////////////
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T> Borrow<[T]> for Vec<T> {
+    fn borrow(&self) -> &[T] {
+        &self[..]
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T> BorrowMut<[T]> for Vec<T> {
+    fn borrow_mut(&mut self) -> &mut [T] {
+        &mut self[..]
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: Clone> ToOwned for [T] {
+    type Owned = Vec<T>;
+    #[cfg(not(test))]
+    fn to_owned(&self) -> Vec<T> {
+        self.to_vec()
+    }
+
+    #[cfg(test)]
+    fn to_owned(&self) -> Vec<T> {
+        hack::to_vec(self, Global)
+    }
+
+    fn clone_into(&self, target: &mut Vec<T>) {
+        // drop anything in target that will not be overwritten
+        target.truncate(self.len());
+
+        // target.len <= self.len due to the truncate above, so the
+        // slices here are always in-bounds.
+        let (init, tail) = self.split_at(target.len());
+
+        // reuse the contained values' allocations/resources.
+        target.clone_from_slice(init);
+        target.extend_from_slice(tail);
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Sorting
+////////////////////////////////////////////////////////////////////////////////
+
+/// Inserts `v[0]` into pre-sorted sequence `v[1..]` so that whole `v[..]` becomes sorted.
+///
+/// This is the integral subroutine of insertion sort.
+#[cfg(not(no_global_oom_handling))]
+fn insert_head<T, F>(v: &mut [T], is_less: &mut F)
+where
+    F: FnMut(&T, &T) -> bool,
+{
+    if v.len() >= 2 && is_less(&v[1], &v[0]) {
+        unsafe {
+            // There are three ways to implement insertion here:
+            //
+            // 1. Swap adjacent elements until the first one gets to its final destination.
+            //    However, this way we copy data around more than is necessary. If elements are big
+            //    structures (costly to copy), this method will be slow.
+            //
+            // 2. Iterate until the right place for the first element is found. Then shift the
+            //    elements succeeding it to make room for it and finally place it into the
+            //    remaining hole. This is a good method.
+            //
+            // 3. Copy the first element into a temporary variable. Iterate until the right place
+            //    for it is found. As we go along, copy every traversed element into the slot
+            //    preceding it. Finally, copy data from the temporary variable into the remaining
+            //    hole. This method is very good. Benchmarks demonstrated slightly better
+            //    performance than with the 2nd method.
+            //
+            // All methods were benchmarked, and the 3rd showed best results. So we chose that one.
+            let tmp = mem::ManuallyDrop::new(ptr::read(&v[0]));
+
+            // Intermediate state of the insertion process is always tracked by `hole`, which
+            // serves two purposes:
+            // 1. Protects integrity of `v` from panics in `is_less`.
+            // 2. Fills the remaining hole in `v` in the end.
+            //
+            // Panic safety:
+            //
+            // If `is_less` panics at any point during the process, `hole` will get dropped and
+            // fill the hole in `v` with `tmp`, thus ensuring that `v` still holds every object it
+            // initially held exactly once.
+            let mut hole = InsertionHole { src: &*tmp, dest: &mut v[1] };
+            ptr::copy_nonoverlapping(&v[1], &mut v[0], 1);
+
+            for i in 2..v.len() {
+                if !is_less(&v[i], &*tmp) {
+                    break;
+                }
+                ptr::copy_nonoverlapping(&v[i], &mut v[i - 1], 1);
+                hole.dest = &mut v[i];
+            }
+            // `hole` gets dropped and thus copies `tmp` into the remaining hole in `v`.
+        }
+    }
+
+    // When dropped, copies from `src` into `dest`.
+    struct InsertionHole<T> {
+        src: *const T,
+        dest: *mut T,
+    }
+
+    impl<T> Drop for InsertionHole<T> {
+        fn drop(&mut self) {
+            unsafe {
+                ptr::copy_nonoverlapping(self.src, self.dest, 1);
+            }
+        }
+    }
+}
+
+/// Merges non-decreasing runs `v[..mid]` and `v[mid..]` using `buf` as temporary storage, and
+/// stores the result into `v[..]`.
+///
+/// # Safety
+///
+/// The two slices must be non-empty and `mid` must be in bounds. Buffer `buf` must be long enough
+/// to hold a copy of the shorter slice. Also, `T` must not be a zero-sized type.
+#[cfg(not(no_global_oom_handling))]
+unsafe fn merge<T, F>(v: &mut [T], mid: usize, buf: *mut T, is_less: &mut F)
+where
+    F: FnMut(&T, &T) -> bool,
+{
+    let len = v.len();
+    let v = v.as_mut_ptr();
+    let (v_mid, v_end) = unsafe { (v.add(mid), v.add(len)) };
+
+    // The merge process first copies the shorter run into `buf`. Then it traces the newly copied
+    // run and the longer run forwards (or backwards), comparing their next unconsumed elements and
+    // copying the lesser (or greater) one into `v`.
+    //
+    // As soon as the shorter run is fully consumed, the process is done. If the longer run gets
+    // consumed first, then we must copy whatever is left of the shorter run into the remaining
+    // hole in `v`.
+    //
+    // Intermediate state of the process is always tracked by `hole`, which serves two purposes:
+    // 1. Protects integrity of `v` from panics in `is_less`.
+    // 2. Fills the remaining hole in `v` if the longer run gets consumed first.
+    //
+    // Panic safety:
+    //
+    // If `is_less` panics at any point during the process, `hole` will get dropped and fill the
+    // hole in `v` with the unconsumed range in `buf`, thus ensuring that `v` still holds every
+    // object it initially held exactly once.
+    let mut hole;
+
+    if mid <= len - mid {
+        // The left run is shorter.
+        unsafe {
+            ptr::copy_nonoverlapping(v, buf, mid);
+            hole = MergeHole { start: buf, end: buf.add(mid), dest: v };
+        }
+
+        // Initially, these pointers point to the beginnings of their arrays.
+        let left = &mut hole.start;
+        let mut right = v_mid;
+        let out = &mut hole.dest;
+
+        while *left < hole.end && right < v_end {
+            // Consume the lesser side.
+            // If equal, prefer the left run to maintain stability.
+            unsafe {
+                let to_copy = if is_less(&*right, &**left) {
+                    get_and_increment(&mut right)
+                } else {
+                    get_and_increment(left)
+                };
+                ptr::copy_nonoverlapping(to_copy, get_and_increment(out), 1);
+            }
+        }
+    } else {
+        // The right run is shorter.
+        unsafe {
+            ptr::copy_nonoverlapping(v_mid, buf, len - mid);
+            hole = MergeHole { start: buf, end: buf.add(len - mid), dest: v_mid };
+        }
+
+        // Initially, these pointers point past the ends of their arrays.
+        let left = &mut hole.dest;
+        let right = &mut hole.end;
+        let mut out = v_end;
+
+        while v < *left && buf < *right {
+            // Consume the greater side.
+            // If equal, prefer the right run to maintain stability.
+            unsafe {
+                let to_copy = if is_less(&*right.offset(-1), &*left.offset(-1)) {
+                    decrement_and_get(left)
+                } else {
+                    decrement_and_get(right)
+                };
+                ptr::copy_nonoverlapping(to_copy, decrement_and_get(&mut out), 1);
+            }
+        }
+    }
+    // Finally, `hole` gets dropped. If the shorter run was not fully consumed, whatever remains of
+    // it will now be copied into the hole in `v`.
+
+    unsafe fn get_and_increment<T>(ptr: &mut *mut T) -> *mut T {
+        let old = *ptr;
+        *ptr = unsafe { ptr.offset(1) };
+        old
+    }
+
+    unsafe fn decrement_and_get<T>(ptr: &mut *mut T) -> *mut T {
+        *ptr = unsafe { ptr.offset(-1) };
+        *ptr
+    }
+
+    // When dropped, copies the range `start..end` into `dest..`.
+    struct MergeHole<T> {
+        start: *mut T,
+        end: *mut T,
+        dest: *mut T,
+    }
+
+    impl<T> Drop for MergeHole<T> {
+        fn drop(&mut self) {
+            // `T` is not a zero-sized type, and these are pointers into a slice's elements.
+            unsafe {
+                let len = self.end.sub_ptr(self.start);
+                ptr::copy_nonoverlapping(self.start, self.dest, len);
+            }
+        }
+    }
+}
+
+/// This merge sort borrows some (but not all) ideas from TimSort, which is described in detail
+/// [here](https://github.com/python/cpython/blob/main/Objects/listsort.txt).
+///
+/// The algorithm identifies strictly descending and non-descending subsequences, which are called
+/// natural runs. There is a stack of pending runs yet to be merged. Each newly found run is pushed
+/// onto the stack, and then some pairs of adjacent runs are merged until these two invariants are
+/// satisfied:
+///
+/// 1. for every `i` in `1..runs.len()`: `runs[i - 1].len > runs[i].len`
+/// 2. for every `i` in `2..runs.len()`: `runs[i - 2].len > runs[i - 1].len + runs[i].len`
+///
+/// The invariants ensure that the total running time is *O*(*n* \* log(*n*)) worst-case.
+#[cfg(not(no_global_oom_handling))]
+fn merge_sort<T, F>(v: &mut [T], mut is_less: F)
+where
+    F: FnMut(&T, &T) -> bool,
+{
+    // Slices of up to this length get sorted using insertion sort.
+    const MAX_INSERTION: usize = 20;
+    // Very short runs are extended using insertion sort to span at least this many elements.
+    const MIN_RUN: usize = 10;
+
+    // Sorting has no meaningful behavior on zero-sized types.
+    if size_of::<T>() == 0 {
+        return;
+    }
+
+    let len = v.len();
+
+    // Short arrays get sorted in-place via insertion sort to avoid allocations.
+    if len <= MAX_INSERTION {
+        if len >= 2 {
+            for i in (0..len - 1).rev() {
+                insert_head(&mut v[i..], &mut is_less);
+            }
+        }
+        return;
+    }
+
+    // Allocate a buffer to use as scratch memory. We keep the length 0 so we can keep in it
+    // shallow copies of the contents of `v` without risking the dtors running on copies if
+    // `is_less` panics. When merging two sorted runs, this buffer holds a copy of the shorter run,
+    // which will always have length at most `len / 2`.
+    let mut buf = Vec::with_capacity(len / 2);
+
+    // In order to identify natural runs in `v`, we traverse it backwards. That might seem like a
+    // strange decision, but consider the fact that merges more often go in the opposite direction
+    // (forwards). According to benchmarks, merging forwards is slightly faster than merging
+    // backwards. To conclude, identifying runs by traversing backwards improves performance.
+    let mut runs = vec![];
+    let mut end = len;
+    while end > 0 {
+        // Find the next natural run, and reverse it if it's strictly descending.
+        let mut start = end - 1;
+        if start > 0 {
+            start -= 1;
+            unsafe {
+                if is_less(v.get_unchecked(start + 1), v.get_unchecked(start)) {
+                    while start > 0 && is_less(v.get_unchecked(start), v.get_unchecked(start - 1)) {
+                        start -= 1;
+                    }
+                    v[start..end].reverse();
+                } else {
+                    while start > 0 && !is_less(v.get_unchecked(start), v.get_unchecked(start - 1))
+                    {
+                        start -= 1;
+                    }
+                }
+            }
+        }
+
+        // Insert some more elements into the run if it's too short. Insertion sort is faster than
+        // merge sort on short sequences, so this significantly improves performance.
+        while start > 0 && end - start < MIN_RUN {
+            start -= 1;
+            insert_head(&mut v[start..end], &mut is_less);
+        }
+
+        // Push this run onto the stack.
+        runs.push(Run { start, len: end - start });
+        end = start;
+
+        // Merge some pairs of adjacent runs to satisfy the invariants.
+        while let Some(r) = collapse(&runs) {
+            let left = runs[r + 1];
+            let right = runs[r];
+            unsafe {
+                merge(
+                    &mut v[left.start..right.start + right.len],
+                    left.len,
+                    buf.as_mut_ptr(),
+                    &mut is_less,
+                );
+            }
+            runs[r] = Run { start: left.start, len: left.len + right.len };
+            runs.remove(r + 1);
+        }
+    }
+
+    // Finally, exactly one run must remain in the stack.
+    debug_assert!(runs.len() == 1 && runs[0].start == 0 && runs[0].len == len);
+
+    // Examines the stack of runs and identifies the next pair of runs to merge. More specifically,
+    // if `Some(r)` is returned, that means `runs[r]` and `runs[r + 1]` must be merged next. If the
+    // algorithm should continue building a new run instead, `None` is returned.
+    //
+    // TimSort is infamous for its buggy implementations, as described here:
+    // http://envisage-project.eu/timsort-specification-and-verification/
+    //
+    // The gist of the story is: we must enforce the invariants on the top four runs on the stack.
+    // Enforcing them on just top three is not sufficient to ensure that the invariants will still
+    // hold for *all* runs in the stack.
+    //
+    // This function correctly checks invariants for the top four runs. Additionally, if the top
+    // run starts at index 0, it will always demand a merge operation until the stack is fully
+    // collapsed, in order to complete the sort.
+    #[inline]
+    fn collapse(runs: &[Run]) -> Option<usize> {
+        let n = runs.len();
+        if n >= 2
+            && (runs[n - 1].start == 0
+                || runs[n - 2].len <= runs[n - 1].len
+                || (n >= 3 && runs[n - 3].len <= runs[n - 2].len + runs[n - 1].len)
+                || (n >= 4 && runs[n - 4].len <= runs[n - 3].len + runs[n - 2].len))
+        {
+            if n >= 3 && runs[n - 3].len < runs[n - 1].len { Some(n - 3) } else { Some(n - 2) }
+        } else {
+            None
+        }
+    }
+
+    #[derive(Clone, Copy)]
+    struct Run {
+        start: usize,
+        len: usize,
+    }
+}
diff --git a/rust/alloc/vec/drain.rs b/rust/alloc/vec/drain.rs
new file mode 100644
index 000000000000..b6a5f98e4fcd
--- /dev/null
+++ b/rust/alloc/vec/drain.rs
@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use crate::alloc::{Allocator, Global};
+use core::fmt;
+use core::iter::{FusedIterator, TrustedLen};
+use core::mem;
+use core::ptr::{self, NonNull};
+use core::slice::{self};
+
+use super::Vec;
+
+/// A draining iterator for `Vec<T>`.
+///
+/// This `struct` is created by [`Vec::drain`].
+/// See its documentation for more.
+///
+/// # Example
+///
+/// ```
+/// let mut v = vec![0, 1, 2];
+/// let iter: std::vec::Drain<_> = v.drain(..);
+/// ```
+#[stable(feature = "drain", since = "1.6.0")]
+pub struct Drain<
+    'a,
+    T: 'a,
+    #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator + 'a = Global,
+> {
+    /// Index of tail to preserve
+    pub(super) tail_start: usize,
+    /// Length of tail
+    pub(super) tail_len: usize,
+    /// Current remaining range to remove
+    pub(super) iter: slice::Iter<'a, T>,
+    pub(super) vec: NonNull<Vec<T, A>>,
+}
+
+#[stable(feature = "collection_debug", since = "1.17.0")]
+impl<T: fmt::Debug, A: Allocator> fmt::Debug for Drain<'_, T, A> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_tuple("Drain").field(&self.iter.as_slice()).finish()
+    }
+}
+
+impl<'a, T, A: Allocator> Drain<'a, T, A> {
+    /// Returns the remaining items of this iterator as a slice.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec!['a', 'b', 'c'];
+    /// let mut drain = vec.drain(..);
+    /// assert_eq!(drain.as_slice(), &['a', 'b', 'c']);
+    /// let _ = drain.next().unwrap();
+    /// assert_eq!(drain.as_slice(), &['b', 'c']);
+    /// ```
+    #[must_use]
+    #[stable(feature = "vec_drain_as_slice", since = "1.46.0")]
+    pub fn as_slice(&self) -> &[T] {
+        self.iter.as_slice()
+    }
+
+    /// Returns a reference to the underlying allocator.
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[must_use]
+    #[inline]
+    pub fn allocator(&self) -> &A {
+        unsafe { self.vec.as_ref().allocator() }
+    }
+}
+
+#[stable(feature = "vec_drain_as_slice", since = "1.46.0")]
+impl<'a, T, A: Allocator> AsRef<[T]> for Drain<'a, T, A> {
+    fn as_ref(&self) -> &[T] {
+        self.as_slice()
+    }
+}
+
+#[stable(feature = "drain", since = "1.6.0")]
+unsafe impl<T: Sync, A: Sync + Allocator> Sync for Drain<'_, T, A> {}
+#[stable(feature = "drain", since = "1.6.0")]
+unsafe impl<T: Send, A: Send + Allocator> Send for Drain<'_, T, A> {}
+
+#[stable(feature = "drain", since = "1.6.0")]
+impl<T, A: Allocator> Iterator for Drain<'_, T, A> {
+    type Item = T;
+
+    #[inline]
+    fn next(&mut self) -> Option<T> {
+        self.iter.next().map(|elt| unsafe { ptr::read(elt as *const _) })
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.iter.size_hint()
+    }
+}
+
+#[stable(feature = "drain", since = "1.6.0")]
+impl<T, A: Allocator> DoubleEndedIterator for Drain<'_, T, A> {
+    #[inline]
+    fn next_back(&mut self) -> Option<T> {
+        self.iter.next_back().map(|elt| unsafe { ptr::read(elt as *const _) })
+    }
+}
+
+#[stable(feature = "drain", since = "1.6.0")]
+impl<T, A: Allocator> Drop for Drain<'_, T, A> {
+    fn drop(&mut self) {
+        /// Moves back the un-`Drain`ed elements to restore the original `Vec`.
+        struct DropGuard<'r, 'a, T, A: Allocator>(&'r mut Drain<'a, T, A>);
+
+        impl<'r, 'a, T, A: Allocator> Drop for DropGuard<'r, 'a, T, A> {
+            fn drop(&mut self) {
+                if self.0.tail_len > 0 {
+                    unsafe {
+                        let source_vec = self.0.vec.as_mut();
+                        // memmove back untouched tail, update to new length
+                        let start = source_vec.len();
+                        let tail = self.0.tail_start;
+                        if tail != start {
+                            let src = source_vec.as_ptr().add(tail);
+                            let dst = source_vec.as_mut_ptr().add(start);
+                            ptr::copy(src, dst, self.0.tail_len);
+                        }
+                        source_vec.set_len(start + self.0.tail_len);
+                    }
+                }
+            }
+        }
+
+        let iter = mem::replace(&mut self.iter, (&mut []).iter());
+        let drop_len = iter.len();
+
+        let mut vec = self.vec;
+
+        if mem::size_of::<T>() == 0 {
+            // ZSTs have no identity, so we don't need to move them around, we only need to drop the correct amount.
+            // this can be achieved by manipulating the Vec length instead of moving values out from `iter`.
+            unsafe {
+                let vec = vec.as_mut();
+                let old_len = vec.len();
+                vec.set_len(old_len + drop_len + self.tail_len);
+                vec.truncate(old_len + self.tail_len);
+            }
+
+            return;
+        }
+
+        // ensure elements are moved back into their appropriate places, even when drop_in_place panics
+        let _guard = DropGuard(self);
+
+        if drop_len == 0 {
+            return;
+        }
+
+        // as_slice() must only be called when iter.len() is > 0 because
+        // vec::Splice modifies vec::Drain fields and may grow the vec which would invalidate
+        // the iterator's internal pointers. Creating a reference to deallocated memory
+        // is invalid even when it is zero-length
+        let drop_ptr = iter.as_slice().as_ptr();
+
+        unsafe {
+            // drop_ptr comes from a slice::Iter which only gives us a &[T] but for drop_in_place
+            // a pointer with mutable provenance is necessary. Therefore we must reconstruct
+            // it from the original vec but also avoid creating a &mut to the front since that could
+            // invalidate raw pointers to it which some unsafe code might rely on.
+            let vec_ptr = vec.as_mut().as_mut_ptr();
+            let drop_offset = drop_ptr.sub_ptr(vec_ptr);
+            let to_drop = ptr::slice_from_raw_parts_mut(vec_ptr.add(drop_offset), drop_len);
+            ptr::drop_in_place(to_drop);
+        }
+    }
+}
+
+#[stable(feature = "drain", since = "1.6.0")]
+impl<T, A: Allocator> ExactSizeIterator for Drain<'_, T, A> {
+    fn is_empty(&self) -> bool {
+        self.iter.is_empty()
+    }
+}
+
+#[unstable(feature = "trusted_len", issue = "37572")]
+unsafe impl<T, A: Allocator> TrustedLen for Drain<'_, T, A> {}
+
+#[stable(feature = "fused", since = "1.26.0")]
+impl<T, A: Allocator> FusedIterator for Drain<'_, T, A> {}
diff --git a/rust/alloc/vec/drain_filter.rs b/rust/alloc/vec/drain_filter.rs
new file mode 100644
index 000000000000..b04fce041622
--- /dev/null
+++ b/rust/alloc/vec/drain_filter.rs
@@ -0,0 +1,145 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use crate::alloc::{Allocator, Global};
+use core::ptr::{self};
+use core::slice::{self};
+
+use super::Vec;
+
+/// An iterator which uses a closure to determine if an element should be removed.
+///
+/// This struct is created by [`Vec::drain_filter`].
+/// See its documentation for more.
+///
+/// # Example
+///
+/// ```
+/// #![feature(drain_filter)]
+///
+/// let mut v = vec![0, 1, 2];
+/// let iter: std::vec::DrainFilter<_, _> = v.drain_filter(|x| *x % 2 == 0);
+/// ```
+#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")]
+#[derive(Debug)]
+pub struct DrainFilter<
+    'a,
+    T,
+    F,
+    #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global,
+> where
+    F: FnMut(&mut T) -> bool,
+{
+    pub(super) vec: &'a mut Vec<T, A>,
+    /// The index of the item that will be inspected by the next call to `next`.
+    pub(super) idx: usize,
+    /// The number of items that have been drained (removed) thus far.
+    pub(super) del: usize,
+    /// The original length of `vec` prior to draining.
+    pub(super) old_len: usize,
+    /// The filter test predicate.
+    pub(super) pred: F,
+    /// A flag that indicates a panic has occurred in the filter test predicate.
+    /// This is used as a hint in the drop implementation to prevent consumption
+    /// of the remainder of the `DrainFilter`. Any unprocessed items will be
+    /// backshifted in the `vec`, but no further items will be dropped or
+    /// tested by the filter predicate.
+    pub(super) panic_flag: bool,
+}
+
+impl<T, F, A: Allocator> DrainFilter<'_, T, F, A>
+where
+    F: FnMut(&mut T) -> bool,
+{
+    /// Returns a reference to the underlying allocator.
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[inline]
+    pub fn allocator(&self) -> &A {
+        self.vec.allocator()
+    }
+}
+
+#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")]
+impl<T, F, A: Allocator> Iterator for DrainFilter<'_, T, F, A>
+where
+    F: FnMut(&mut T) -> bool,
+{
+    type Item = T;
+
+    fn next(&mut self) -> Option<T> {
+        unsafe {
+            while self.idx < self.old_len {
+                let i = self.idx;
+                let v = slice::from_raw_parts_mut(self.vec.as_mut_ptr(), self.old_len);
+                self.panic_flag = true;
+                let drained = (self.pred)(&mut v[i]);
+                self.panic_flag = false;
+                // Update the index *after* the predicate is called. If the index
+                // is updated prior and the predicate panics, the element at this
+                // index would be leaked.
+                self.idx += 1;
+                if drained {
+                    self.del += 1;
+                    return Some(ptr::read(&v[i]));
+                } else if self.del > 0 {
+                    let del = self.del;
+                    let src: *const T = &v[i];
+                    let dst: *mut T = &mut v[i - del];
+                    ptr::copy_nonoverlapping(src, dst, 1);
+                }
+            }
+            None
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (0, Some(self.old_len - self.idx))
+    }
+}
+
+#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")]
+impl<T, F, A: Allocator> Drop for DrainFilter<'_, T, F, A>
+where
+    F: FnMut(&mut T) -> bool,
+{
+    fn drop(&mut self) {
+        struct BackshiftOnDrop<'a, 'b, T, F, A: Allocator>
+        where
+            F: FnMut(&mut T) -> bool,
+        {
+            drain: &'b mut DrainFilter<'a, T, F, A>,
+        }
+
+        impl<'a, 'b, T, F, A: Allocator> Drop for BackshiftOnDrop<'a, 'b, T, F, A>
+        where
+            F: FnMut(&mut T) -> bool,
+        {
+            fn drop(&mut self) {
+                unsafe {
+                    if self.drain.idx < self.drain.old_len && self.drain.del > 0 {
+                        // This is a pretty messed up state, and there isn't really an
+                        // obviously right thing to do. We don't want to keep trying
+                        // to execute `pred`, so we just backshift all the unprocessed
+                        // elements and tell the vec that they still exist. The backshift
+                        // is required to prevent a double-drop of the last successfully
+                        // drained item prior to a panic in the predicate.
+                        let ptr = self.drain.vec.as_mut_ptr();
+                        let src = ptr.add(self.drain.idx);
+                        let dst = src.sub(self.drain.del);
+                        let tail_len = self.drain.old_len - self.drain.idx;
+                        src.copy_to(dst, tail_len);
+                    }
+                    self.drain.vec.set_len(self.drain.old_len - self.drain.del);
+                }
+            }
+        }
+
+        let backshift = BackshiftOnDrop { drain: self };
+
+        // Attempt to consume any remaining elements if the filter predicate
+        // has not yet panicked. We'll backshift any remaining elements
+        // whether we've already panicked or if the consumption here panics.
+        if !backshift.drain.panic_flag {
+            backshift.drain.for_each(drop);
+        }
+    }
+}
diff --git a/rust/alloc/vec/into_iter.rs b/rust/alloc/vec/into_iter.rs
new file mode 100644
index 000000000000..f7a50e76691e
--- /dev/null
+++ b/rust/alloc/vec/into_iter.rs
@@ -0,0 +1,366 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+#[cfg(not(no_global_oom_handling))]
+use super::AsVecIntoIter;
+use crate::alloc::{Allocator, Global};
+use crate::raw_vec::RawVec;
+use core::fmt;
+use core::intrinsics::arith_offset;
+use core::iter::{
+    FusedIterator, InPlaceIterable, SourceIter, TrustedLen, TrustedRandomAccessNoCoerce,
+};
+use core::marker::PhantomData;
+use core::mem::{self, ManuallyDrop};
+#[cfg(not(no_global_oom_handling))]
+use core::ops::Deref;
+use core::ptr::{self, NonNull};
+use core::slice::{self};
+
+/// An iterator that moves out of a vector.
+///
+/// This `struct` is created by the `into_iter` method on [`Vec`](super::Vec)
+/// (provided by the [`IntoIterator`] trait).
+///
+/// # Example
+///
+/// ```
+/// let v = vec![0, 1, 2];
+/// let iter: std::vec::IntoIter<_> = v.into_iter();
+/// ```
+#[stable(feature = "rust1", since = "1.0.0")]
+#[rustc_insignificant_dtor]
+pub struct IntoIter<
+    T,
+    #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global,
+> {
+    pub(super) buf: NonNull<T>,
+    pub(super) phantom: PhantomData<T>,
+    pub(super) cap: usize,
+    // the drop impl reconstructs a RawVec from buf, cap and alloc
+    // to avoid dropping the allocator twice we need to wrap it into ManuallyDrop
+    pub(super) alloc: ManuallyDrop<A>,
+    pub(super) ptr: *const T,
+    pub(super) end: *const T,
+}
+
+#[stable(feature = "vec_intoiter_debug", since = "1.13.0")]
+impl<T: fmt::Debug, A: Allocator> fmt::Debug for IntoIter<T, A> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_tuple("IntoIter").field(&self.as_slice()).finish()
+    }
+}
+
+impl<T, A: Allocator> IntoIter<T, A> {
+    /// Returns the remaining items of this iterator as a slice.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let vec = vec!['a', 'b', 'c'];
+    /// let mut into_iter = vec.into_iter();
+    /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']);
+    /// let _ = into_iter.next().unwrap();
+    /// assert_eq!(into_iter.as_slice(), &['b', 'c']);
+    /// ```
+    #[stable(feature = "vec_into_iter_as_slice", since = "1.15.0")]
+    pub fn as_slice(&self) -> &[T] {
+        unsafe { slice::from_raw_parts(self.ptr, self.len()) }
+    }
+
+    /// Returns the remaining items of this iterator as a mutable slice.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let vec = vec!['a', 'b', 'c'];
+    /// let mut into_iter = vec.into_iter();
+    /// assert_eq!(into_iter.as_slice(), &['a', 'b', 'c']);
+    /// into_iter.as_mut_slice()[2] = 'z';
+    /// assert_eq!(into_iter.next().unwrap(), 'a');
+    /// assert_eq!(into_iter.next().unwrap(), 'b');
+    /// assert_eq!(into_iter.next().unwrap(), 'z');
+    /// ```
+    #[stable(feature = "vec_into_iter_as_slice", since = "1.15.0")]
+    pub fn as_mut_slice(&mut self) -> &mut [T] {
+        unsafe { &mut *self.as_raw_mut_slice() }
+    }
+
+    /// Returns a reference to the underlying allocator.
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[inline]
+    pub fn allocator(&self) -> &A {
+        &self.alloc
+    }
+
+    fn as_raw_mut_slice(&mut self) -> *mut [T] {
+        ptr::slice_from_raw_parts_mut(self.ptr as *mut T, self.len())
+    }
+
+    /// Drops remaining elements and relinquishes the backing allocation.
+    ///
+    /// This is roughly equivalent to the following, but more efficient
+    ///
+    /// ```
+    /// # let mut into_iter = Vec::<u8>::with_capacity(10).into_iter();
+    /// (&mut into_iter).for_each(core::mem::drop);
+    /// unsafe { core::ptr::write(&mut into_iter, Vec::new().into_iter()); }
+    /// ```
+    ///
+    /// This method is used by in-place iteration, refer to the vec::in_place_collect
+    /// documentation for an overview.
+    #[cfg(not(no_global_oom_handling))]
+    pub(super) fn forget_allocation_drop_remaining(&mut self) {
+        let remaining = self.as_raw_mut_slice();
+
+        // overwrite the individual fields instead of creating a new
+        // struct and then overwriting &mut self.
+        // this creates less assembly
+        self.cap = 0;
+        self.buf = unsafe { NonNull::new_unchecked(RawVec::NEW.ptr()) };
+        self.ptr = self.buf.as_ptr();
+        self.end = self.buf.as_ptr();
+
+        unsafe {
+            ptr::drop_in_place(remaining);
+        }
+    }
+
+    /// Forgets to Drop the remaining elements while still allowing the backing allocation to be freed.
+    #[allow(dead_code)]
+    pub(crate) fn forget_remaining_elements(&mut self) {
+        self.ptr = self.end;
+    }
+}
+
+#[stable(feature = "vec_intoiter_as_ref", since = "1.46.0")]
+impl<T, A: Allocator> AsRef<[T]> for IntoIter<T, A> {
+    fn as_ref(&self) -> &[T] {
+        self.as_slice()
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+unsafe impl<T: Send, A: Allocator + Send> Send for IntoIter<T, A> {}
+#[stable(feature = "rust1", since = "1.0.0")]
+unsafe impl<T: Sync, A: Allocator + Sync> Sync for IntoIter<T, A> {}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T, A: Allocator> Iterator for IntoIter<T, A> {
+    type Item = T;
+
+    #[inline]
+    fn next(&mut self) -> Option<T> {
+        if self.ptr as *const _ == self.end {
+            None
+        } else if mem::size_of::<T>() == 0 {
+            // purposefully don't use 'ptr.offset' because for
+            // vectors with 0-size elements this would return the
+            // same pointer.
+            self.ptr = unsafe { arith_offset(self.ptr as *const i8, 1) as *mut T };
+
+            // Make up a value of this ZST.
+            Some(unsafe { mem::zeroed() })
+        } else {
+            let old = self.ptr;
+            self.ptr = unsafe { self.ptr.offset(1) };
+
+            Some(unsafe { ptr::read(old) })
+        }
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let exact = if mem::size_of::<T>() == 0 {
+            self.end.addr().wrapping_sub(self.ptr.addr())
+        } else {
+            unsafe { self.end.sub_ptr(self.ptr) }
+        };
+        (exact, Some(exact))
+    }
+
+    #[inline]
+    fn advance_by(&mut self, n: usize) -> Result<(), usize> {
+        let step_size = self.len().min(n);
+        let to_drop = ptr::slice_from_raw_parts_mut(self.ptr as *mut T, step_size);
+        if mem::size_of::<T>() == 0 {
+            // SAFETY: due to unchecked casts of unsigned amounts to signed offsets the wraparound
+            // effectively results in unsigned pointers representing positions 0..usize::MAX,
+            // which is valid for ZSTs.
+            self.ptr = unsafe { arith_offset(self.ptr as *const i8, step_size as isize) as *mut T }
+        } else {
+            // SAFETY: the min() above ensures that step_size is in bounds
+            self.ptr = unsafe { self.ptr.add(step_size) };
+        }
+        // SAFETY: the min() above ensures that step_size is in bounds
+        unsafe {
+            ptr::drop_in_place(to_drop);
+        }
+        if step_size < n {
+            return Err(step_size);
+        }
+        Ok(())
+    }
+
+    #[inline]
+    fn count(self) -> usize {
+        self.len()
+    }
+
+    unsafe fn __iterator_get_unchecked(&mut self, i: usize) -> Self::Item
+    where
+        Self: TrustedRandomAccessNoCoerce,
+    {
+        // SAFETY: the caller must guarantee that `i` is in bounds of the
+        // `Vec<T>`, so `i` cannot overflow an `isize`, and the `self.ptr.add(i)`
+        // is guaranteed to pointer to an element of the `Vec<T>` and
+        // thus guaranteed to be valid to dereference.
+        //
+        // Also note the implementation of `Self: TrustedRandomAccess` requires
+        // that `T: Copy` so reading elements from the buffer doesn't invalidate
+        // them for `Drop`.
+        unsafe {
+            if mem::size_of::<T>() == 0 { mem::zeroed() } else { ptr::read(self.ptr.add(i)) }
+        }
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T, A: Allocator> DoubleEndedIterator for IntoIter<T, A> {
+    #[inline]
+    fn next_back(&mut self) -> Option<T> {
+        if self.end == self.ptr {
+            None
+        } else if mem::size_of::<T>() == 0 {
+            // See above for why 'ptr.offset' isn't used
+            self.end = unsafe { arith_offset(self.end as *const i8, -1) as *mut T };
+
+            // Make up a value of this ZST.
+            Some(unsafe { mem::zeroed() })
+        } else {
+            self.end = unsafe { self.end.offset(-1) };
+
+            Some(unsafe { ptr::read(self.end) })
+        }
+    }
+
+    #[inline]
+    fn advance_back_by(&mut self, n: usize) -> Result<(), usize> {
+        let step_size = self.len().min(n);
+        if mem::size_of::<T>() == 0 {
+            // SAFETY: same as for advance_by()
+            self.end = unsafe {
+                arith_offset(self.end as *const i8, step_size.wrapping_neg() as isize) as *mut T
+            }
+        } else {
+            // SAFETY: same as for advance_by()
+            self.end = unsafe { self.end.offset(step_size.wrapping_neg() as isize) };
+        }
+        let to_drop = ptr::slice_from_raw_parts_mut(self.end as *mut T, step_size);
+        // SAFETY: same as for advance_by()
+        unsafe {
+            ptr::drop_in_place(to_drop);
+        }
+        if step_size < n {
+            return Err(step_size);
+        }
+        Ok(())
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T, A: Allocator> ExactSizeIterator for IntoIter<T, A> {
+    fn is_empty(&self) -> bool {
+        self.ptr == self.end
+    }
+}
+
+#[stable(feature = "fused", since = "1.26.0")]
+impl<T, A: Allocator> FusedIterator for IntoIter<T, A> {}
+
+#[unstable(feature = "trusted_len", issue = "37572")]
+unsafe impl<T, A: Allocator> TrustedLen for IntoIter<T, A> {}
+
+#[doc(hidden)]
+#[unstable(issue = "none", feature = "std_internals")]
+#[rustc_unsafe_specialization_marker]
+pub trait NonDrop {}
+
+// T: Copy as approximation for !Drop since get_unchecked does not advance self.ptr
+// and thus we can't implement drop-handling
+#[unstable(issue = "none", feature = "std_internals")]
+impl<T: Copy> NonDrop for T {}
+
+#[doc(hidden)]
+#[unstable(issue = "none", feature = "std_internals")]
+// TrustedRandomAccess (without NoCoerce) must not be implemented because
+// subtypes/supertypes of `T` might not be `NonDrop`
+unsafe impl<T, A: Allocator> TrustedRandomAccessNoCoerce for IntoIter<T, A>
+where
+    T: NonDrop,
+{
+    const MAY_HAVE_SIDE_EFFECT: bool = false;
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "vec_into_iter_clone", since = "1.8.0")]
+impl<T: Clone, A: Allocator + Clone> Clone for IntoIter<T, A> {
+    #[cfg(not(test))]
+    fn clone(&self) -> Self {
+        self.as_slice().to_vec_in(self.alloc.deref().clone()).into_iter()
+    }
+    #[cfg(test)]
+    fn clone(&self) -> Self {
+        crate::slice::to_vec(self.as_slice(), self.alloc.deref().clone()).into_iter()
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+unsafe impl<#[may_dangle] T, A: Allocator> Drop for IntoIter<T, A> {
+    fn drop(&mut self) {
+        struct DropGuard<'a, T, A: Allocator>(&'a mut IntoIter<T, A>);
+
+        impl<T, A: Allocator> Drop for DropGuard<'_, T, A> {
+            fn drop(&mut self) {
+                unsafe {
+                    // `IntoIter::alloc` is not used anymore after this and will be dropped by RawVec
+                    let alloc = ManuallyDrop::take(&mut self.0.alloc);
+                    // RawVec handles deallocation
+                    let _ = RawVec::from_raw_parts_in(self.0.buf.as_ptr(), self.0.cap, alloc);
+                }
+            }
+        }
+
+        let guard = DropGuard(self);
+        // destroy the remaining elements
+        unsafe {
+            ptr::drop_in_place(guard.0.as_raw_mut_slice());
+        }
+        // now `guard` will be dropped and do the rest
+    }
+}
+
+// In addition to the SAFETY invariants of the following three unsafe traits
+// also refer to the vec::in_place_collect module documentation to get an overview
+#[unstable(issue = "none", feature = "inplace_iteration")]
+#[doc(hidden)]
+unsafe impl<T, A: Allocator> InPlaceIterable for IntoIter<T, A> {}
+
+#[unstable(issue = "none", feature = "inplace_iteration")]
+#[doc(hidden)]
+unsafe impl<T, A: Allocator> SourceIter for IntoIter<T, A> {
+    type Source = Self;
+
+    #[inline]
+    unsafe fn as_inner(&mut self) -> &mut Self::Source {
+        self
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+unsafe impl<T> AsVecIntoIter for IntoIter<T> {
+    type Item = T;
+
+    fn as_into_iter(&mut self) -> &mut IntoIter<Self::Item> {
+        self
+    }
+}
diff --git a/rust/alloc/vec/is_zero.rs b/rust/alloc/vec/is_zero.rs
new file mode 100644
index 000000000000..377f3d172777
--- /dev/null
+++ b/rust/alloc/vec/is_zero.rs
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use crate::boxed::Box;
+
+#[rustc_specialization_trait]
+pub(super) unsafe trait IsZero {
+    /// Whether this value's representation is all zeros
+    fn is_zero(&self) -> bool;
+}
+
+macro_rules! impl_is_zero {
+    ($t:ty, $is_zero:expr) => {
+        unsafe impl IsZero for $t {
+            #[inline]
+            fn is_zero(&self) -> bool {
+                $is_zero(*self)
+            }
+        }
+    };
+}
+
+impl_is_zero!(i16, |x| x == 0);
+impl_is_zero!(i32, |x| x == 0);
+impl_is_zero!(i64, |x| x == 0);
+impl_is_zero!(i128, |x| x == 0);
+impl_is_zero!(isize, |x| x == 0);
+
+impl_is_zero!(u16, |x| x == 0);
+impl_is_zero!(u32, |x| x == 0);
+impl_is_zero!(u64, |x| x == 0);
+impl_is_zero!(u128, |x| x == 0);
+impl_is_zero!(usize, |x| x == 0);
+
+impl_is_zero!(bool, |x| x == false);
+impl_is_zero!(char, |x| x == '\0');
+
+impl_is_zero!(f32, |x: f32| x.to_bits() == 0);
+impl_is_zero!(f64, |x: f64| x.to_bits() == 0);
+
+unsafe impl<T> IsZero for *const T {
+    #[inline]
+    fn is_zero(&self) -> bool {
+        (*self).is_null()
+    }
+}
+
+unsafe impl<T> IsZero for *mut T {
+    #[inline]
+    fn is_zero(&self) -> bool {
+        (*self).is_null()
+    }
+}
+
+unsafe impl<T: IsZero, const N: usize> IsZero for [T; N] {
+    #[inline]
+    fn is_zero(&self) -> bool {
+        // Because this is generated as a runtime check, it's not obvious that
+        // it's worth doing if the array is really long.  The threshold here
+        // is largely arbitrary, but was picked because as of 2022-05-01 LLVM
+        // can const-fold the check in `vec![[0; 32]; n]` but not in
+        // `vec![[0; 64]; n]`: https://godbolt.org/z/WTzjzfs5b
+        // Feel free to tweak if you have better evidence.
+
+        N <= 32 && self.iter().all(IsZero::is_zero)
+    }
+}
+
+// `Option<&T>` and `Option<Box<T>>` are guaranteed to represent `None` as null.
+// For fat pointers, the bytes that would be the pointer metadata in the `Some`
+// variant are padding in the `None` variant, so ignoring them and
+// zero-initializing instead is ok.
+// `Option<&mut T>` never implements `Clone`, so there's no need for an impl of
+// `SpecFromElem`.
+
+unsafe impl<T: ?Sized> IsZero for Option<&T> {
+    #[inline]
+    fn is_zero(&self) -> bool {
+        self.is_none()
+    }
+}
+
+unsafe impl<T: ?Sized> IsZero for Option<Box<T>> {
+    #[inline]
+    fn is_zero(&self) -> bool {
+        self.is_none()
+    }
+}
+
+// `Option<num::NonZeroU32>` and similar have a representation guarantee that
+// they're the same size as the corresponding `u32` type, as well as a guarantee
+// that transmuting between `NonZeroU32` and `Option<num::NonZeroU32>` works.
+// While the documentation officially makes it UB to transmute from `None`,
+// we're the standard library so we can make extra inferences, and we know that
+// the only niche available to represent `None` is the one that's all zeros.
+
+macro_rules! impl_is_zero_option_of_nonzero {
+    ($($t:ident,)+) => {$(
+        unsafe impl IsZero for Option<core::num::$t> {
+            #[inline]
+            fn is_zero(&self) -> bool {
+                self.is_none()
+            }
+        }
+    )+};
+}
+
+impl_is_zero_option_of_nonzero!(
+    NonZeroU8,
+    NonZeroU16,
+    NonZeroU32,
+    NonZeroU64,
+    NonZeroU128,
+    NonZeroI8,
+    NonZeroI16,
+    NonZeroI32,
+    NonZeroI64,
+    NonZeroI128,
+    NonZeroUsize,
+    NonZeroIsize,
+);
diff --git a/rust/alloc/vec/mod.rs b/rust/alloc/vec/mod.rs
new file mode 100644
index 000000000000..540787804cc2
--- /dev/null
+++ b/rust/alloc/vec/mod.rs
@@ -0,0 +1,3140 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+//! A contiguous growable array type with heap-allocated contents, written
+//! `Vec<T>`.
+//!
+//! Vectors have *O*(1) indexing, amortized *O*(1) push (to the end) and
+//! *O*(1) pop (from the end).
+//!
+//! Vectors ensure they never allocate more than `isize::MAX` bytes.
+//!
+//! # Examples
+//!
+//! You can explicitly create a [`Vec`] with [`Vec::new`]:
+//!
+//! ```
+//! let v: Vec<i32> = Vec::new();
+//! ```
+//!
+//! ...or by using the [`vec!`] macro:
+//!
+//! ```
+//! let v: Vec<i32> = vec![];
+//!
+//! let v = vec![1, 2, 3, 4, 5];
+//!
+//! let v = vec![0; 10]; // ten zeroes
+//! ```
+//!
+//! You can [`push`] values onto the end of a vector (which will grow the vector
+//! as needed):
+//!
+//! ```
+//! let mut v = vec![1, 2];
+//!
+//! v.push(3);
+//! ```
+//!
+//! Popping values works in much the same way:
+//!
+//! ```
+//! let mut v = vec![1, 2];
+//!
+//! let two = v.pop();
+//! ```
+//!
+//! Vectors also support indexing (through the [`Index`] and [`IndexMut`] traits):
+//!
+//! ```
+//! let mut v = vec![1, 2, 3];
+//! let three = v[2];
+//! v[1] = v[1] + 5;
+//! ```
+//!
+//! [`push`]: Vec::push
+
+#![stable(feature = "rust1", since = "1.0.0")]
+
+#[cfg(not(no_global_oom_handling))]
+use core::cmp;
+use core::cmp::Ordering;
+use core::convert::TryFrom;
+use core::fmt;
+use core::hash::{Hash, Hasher};
+use core::intrinsics::{arith_offset, assume};
+use core::iter;
+#[cfg(not(no_global_oom_handling))]
+use core::iter::FromIterator;
+use core::marker::PhantomData;
+use core::mem::{self, ManuallyDrop, MaybeUninit};
+use core::ops::{self, Index, IndexMut, Range, RangeBounds};
+use core::ptr::{self, NonNull};
+use core::slice::{self, SliceIndex};
+
+use crate::alloc::{Allocator, Global};
+use crate::borrow::{Cow, ToOwned};
+use crate::boxed::Box;
+use crate::collections::TryReserveError;
+use crate::raw_vec::RawVec;
+
+#[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")]
+pub use self::drain_filter::DrainFilter;
+
+mod drain_filter;
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "vec_splice", since = "1.21.0")]
+pub use self::splice::Splice;
+
+#[cfg(not(no_global_oom_handling))]
+mod splice;
+
+#[stable(feature = "drain", since = "1.6.0")]
+pub use self::drain::Drain;
+
+mod drain;
+
+#[cfg(not(no_global_oom_handling))]
+mod cow;
+
+#[cfg(not(no_global_oom_handling))]
+pub(crate) use self::in_place_collect::AsVecIntoIter;
+#[stable(feature = "rust1", since = "1.0.0")]
+pub use self::into_iter::IntoIter;
+
+mod into_iter;
+
+#[cfg(not(no_global_oom_handling))]
+use self::is_zero::IsZero;
+
+mod is_zero;
+
+#[cfg(not(no_global_oom_handling))]
+mod in_place_collect;
+
+mod partial_eq;
+
+#[cfg(not(no_global_oom_handling))]
+use self::spec_from_elem::SpecFromElem;
+
+#[cfg(not(no_global_oom_handling))]
+mod spec_from_elem;
+
+#[cfg(not(no_global_oom_handling))]
+use self::set_len_on_drop::SetLenOnDrop;
+
+#[cfg(not(no_global_oom_handling))]
+mod set_len_on_drop;
+
+#[cfg(not(no_global_oom_handling))]
+use self::in_place_drop::InPlaceDrop;
+
+#[cfg(not(no_global_oom_handling))]
+mod in_place_drop;
+
+#[cfg(not(no_global_oom_handling))]
+use self::spec_from_iter_nested::SpecFromIterNested;
+
+#[cfg(not(no_global_oom_handling))]
+mod spec_from_iter_nested;
+
+#[cfg(not(no_global_oom_handling))]
+use self::spec_from_iter::SpecFromIter;
+
+#[cfg(not(no_global_oom_handling))]
+mod spec_from_iter;
+
+#[cfg(not(no_global_oom_handling))]
+use self::spec_extend::SpecExtend;
+
+#[cfg(not(no_global_oom_handling))]
+mod spec_extend;
+
+/// A contiguous growable array type, written as `Vec<T>`, short for 'vector'.
+///
+/// # Examples
+///
+/// ```
+/// let mut vec = Vec::new();
+/// vec.push(1);
+/// vec.push(2);
+///
+/// assert_eq!(vec.len(), 2);
+/// assert_eq!(vec[0], 1);
+///
+/// assert_eq!(vec.pop(), Some(2));
+/// assert_eq!(vec.len(), 1);
+///
+/// vec[0] = 7;
+/// assert_eq!(vec[0], 7);
+///
+/// vec.extend([1, 2, 3].iter().copied());
+///
+/// for x in &vec {
+///     println!("{x}");
+/// }
+/// assert_eq!(vec, [7, 1, 2, 3]);
+/// ```
+///
+/// The [`vec!`] macro is provided for convenient initialization:
+///
+/// ```
+/// let mut vec1 = vec![1, 2, 3];
+/// vec1.push(4);
+/// let vec2 = Vec::from([1, 2, 3, 4]);
+/// assert_eq!(vec1, vec2);
+/// ```
+///
+/// It can also initialize each element of a `Vec<T>` with a given value.
+/// This may be more efficient than performing allocation and initialization
+/// in separate steps, especially when initializing a vector of zeros:
+///
+/// ```
+/// let vec = vec![0; 5];
+/// assert_eq!(vec, [0, 0, 0, 0, 0]);
+///
+/// // The following is equivalent, but potentially slower:
+/// let mut vec = Vec::with_capacity(5);
+/// vec.resize(5, 0);
+/// assert_eq!(vec, [0, 0, 0, 0, 0]);
+/// ```
+///
+/// For more information, see
+/// [Capacity and Reallocation](#capacity-and-reallocation).
+///
+/// Use a `Vec<T>` as an efficient stack:
+///
+/// ```
+/// let mut stack = Vec::new();
+///
+/// stack.push(1);
+/// stack.push(2);
+/// stack.push(3);
+///
+/// while let Some(top) = stack.pop() {
+///     // Prints 3, 2, 1
+///     println!("{top}");
+/// }
+/// ```
+///
+/// # Indexing
+///
+/// The `Vec` type allows to access values by index, because it implements the
+/// [`Index`] trait. An example will be more explicit:
+///
+/// ```
+/// let v = vec![0, 2, 4, 6];
+/// println!("{}", v[1]); // it will display '2'
+/// ```
+///
+/// However be careful: if you try to access an index which isn't in the `Vec`,
+/// your software will panic! You cannot do this:
+///
+/// ```should_panic
+/// let v = vec![0, 2, 4, 6];
+/// println!("{}", v[6]); // it will panic!
+/// ```
+///
+/// Use [`get`] and [`get_mut`] if you want to check whether the index is in
+/// the `Vec`.
+///
+/// # Slicing
+///
+/// A `Vec` can be mutable. On the other hand, slices are read-only objects.
+/// To get a [slice][prim@slice], use [`&`]. Example:
+///
+/// ```
+/// fn read_slice(slice: &[usize]) {
+///     // ...
+/// }
+///
+/// let v = vec![0, 1];
+/// read_slice(&v);
+///
+/// // ... and that's all!
+/// // you can also do it like this:
+/// let u: &[usize] = &v;
+/// // or like this:
+/// let u: &[_] = &v;
+/// ```
+///
+/// In Rust, it's more common to pass slices as arguments rather than vectors
+/// when you just want to provide read access. The same goes for [`String`] and
+/// [`&str`].
+///
+/// # Capacity and reallocation
+///
+/// The capacity of a vector is the amount of space allocated for any future
+/// elements that will be added onto the vector. This is not to be confused with
+/// the *length* of a vector, which specifies the number of actual elements
+/// within the vector. If a vector's length exceeds its capacity, its capacity
+/// will automatically be increased, but its elements will have to be
+/// reallocated.
+///
+/// For example, a vector with capacity 10 and length 0 would be an empty vector
+/// with space for 10 more elements. Pushing 10 or fewer elements onto the
+/// vector will not change its capacity or cause reallocation to occur. However,
+/// if the vector's length is increased to 11, it will have to reallocate, which
+/// can be slow. For this reason, it is recommended to use [`Vec::with_capacity`]
+/// whenever possible to specify how big the vector is expected to get.
+///
+/// # Guarantees
+///
+/// Due to its incredibly fundamental nature, `Vec` makes a lot of guarantees
+/// about its design. This ensures that it's as low-overhead as possible in
+/// the general case, and can be correctly manipulated in primitive ways
+/// by unsafe code. Note that these guarantees refer to an unqualified `Vec<T>`.
+/// If additional type parameters are added (e.g., to support custom allocators),
+/// overriding their defaults may change the behavior.
+///
+/// Most fundamentally, `Vec` is and always will be a (pointer, capacity, length)
+/// triplet. No more, no less. The order of these fields is completely
+/// unspecified, and you should use the appropriate methods to modify these.
+/// The pointer will never be null, so this type is null-pointer-optimized.
+///
+/// However, the pointer might not actually point to allocated memory. In particular,
+/// if you construct a `Vec` with capacity 0 via [`Vec::new`], [`vec![]`][`vec!`],
+/// [`Vec::with_capacity(0)`][`Vec::with_capacity`], or by calling [`shrink_to_fit`]
+/// on an empty Vec, it will not allocate memory. Similarly, if you store zero-sized
+/// types inside a `Vec`, it will not allocate space for them. *Note that in this case
+/// the `Vec` might not report a [`capacity`] of 0*. `Vec` will allocate if and only
+/// if <code>[mem::size_of::\<T>]\() * [capacity]\() > 0</code>. In general, `Vec`'s allocation
+/// details are very subtle --- if you intend to allocate memory using a `Vec`
+/// and use it for something else (either to pass to unsafe code, or to build your
+/// own memory-backed collection), be sure to deallocate this memory by using
+/// `from_raw_parts` to recover the `Vec` and then dropping it.
+///
+/// If a `Vec` *has* allocated memory, then the memory it points to is on the heap
+/// (as defined by the allocator Rust is configured to use by default), and its
+/// pointer points to [`len`] initialized, contiguous elements in order (what
+/// you would see if you coerced it to a slice), followed by <code>[capacity] - [len]</code>
+/// logically uninitialized, contiguous elements.
+///
+/// A vector containing the elements `'a'` and `'b'` with capacity 4 can be
+/// visualized as below. The top part is the `Vec` struct, it contains a
+/// pointer to the head of the allocation in the heap, length and capacity.
+/// The bottom part is the allocation on the heap, a contiguous memory block.
+///
+/// ```text
+///             ptr      len  capacity
+///        +--------+--------+--------+
+///        | 0x0123 |      2 |      4 |
+///        +--------+--------+--------+
+///             |
+///             v
+/// Heap   +--------+--------+--------+--------+
+///        |    'a' |    'b' | uninit | uninit |
+///        +--------+--------+--------+--------+
+/// ```
+///
+/// - **uninit** represents memory that is not initialized, see [`MaybeUninit`].
+/// - Note: the ABI is not stable and `Vec` makes no guarantees about its memory
+///   layout (including the order of fields).
+///
+/// `Vec` will never perform a "small optimization" where elements are actually
+/// stored on the stack for two reasons:
+///
+/// * It would make it more difficult for unsafe code to correctly manipulate
+///   a `Vec`. The contents of a `Vec` wouldn't have a stable address if it were
+///   only moved, and it would be more difficult to determine if a `Vec` had
+///   actually allocated memory.
+///
+/// * It would penalize the general case, incurring an additional branch
+///   on every access.
+///
+/// `Vec` will never automatically shrink itself, even if completely empty. This
+/// ensures no unnecessary allocations or deallocations occur. Emptying a `Vec`
+/// and then filling it back up to the same [`len`] should incur no calls to
+/// the allocator. If you wish to free up unused memory, use
+/// [`shrink_to_fit`] or [`shrink_to`].
+///
+/// [`push`] and [`insert`] will never (re)allocate if the reported capacity is
+/// sufficient. [`push`] and [`insert`] *will* (re)allocate if
+/// <code>[len] == [capacity]</code>. That is, the reported capacity is completely
+/// accurate, and can be relied on. It can even be used to manually free the memory
+/// allocated by a `Vec` if desired. Bulk insertion methods *may* reallocate, even
+/// when not necessary.
+///
+/// `Vec` does not guarantee any particular growth strategy when reallocating
+/// when full, nor when [`reserve`] is called. The current strategy is basic
+/// and it may prove desirable to use a non-constant growth factor. Whatever
+/// strategy is used will of course guarantee *O*(1) amortized [`push`].
+///
+/// `vec![x; n]`, `vec![a, b, c, d]`, and
+/// [`Vec::with_capacity(n)`][`Vec::with_capacity`], will all produce a `Vec`
+/// with exactly the requested capacity. If <code>[len] == [capacity]</code>,
+/// (as is the case for the [`vec!`] macro), then a `Vec<T>` can be converted to
+/// and from a [`Box<[T]>`][owned slice] without reallocating or moving the elements.
+///
+/// `Vec` will not specifically overwrite any data that is removed from it,
+/// but also won't specifically preserve it. Its uninitialized memory is
+/// scratch space that it may use however it wants. It will generally just do
+/// whatever is most efficient or otherwise easy to implement. Do not rely on
+/// removed data to be erased for security purposes. Even if you drop a `Vec`, its
+/// buffer may simply be reused by another allocation. Even if you zero a `Vec`'s memory
+/// first, that might not actually happen because the optimizer does not consider
+/// this a side-effect that must be preserved. There is one case which we will
+/// not break, however: using `unsafe` code to write to the excess capacity,
+/// and then increasing the length to match, is always valid.
+///
+/// Currently, `Vec` does not guarantee the order in which elements are dropped.
+/// The order has changed in the past and may change again.
+///
+/// [`get`]: ../../std/vec/struct.Vec.html#method.get
+/// [`get_mut`]: ../../std/vec/struct.Vec.html#method.get_mut
+/// [`String`]: crate::string::String
+/// [`&str`]: type@str
+/// [`shrink_to_fit`]: Vec::shrink_to_fit
+/// [`shrink_to`]: Vec::shrink_to
+/// [capacity]: Vec::capacity
+/// [`capacity`]: Vec::capacity
+/// [mem::size_of::\<T>]: core::mem::size_of
+/// [len]: Vec::len
+/// [`len`]: Vec::len
+/// [`push`]: Vec::push
+/// [`insert`]: Vec::insert
+/// [`reserve`]: Vec::reserve
+/// [`MaybeUninit`]: core::mem::MaybeUninit
+/// [owned slice]: Box
+#[stable(feature = "rust1", since = "1.0.0")]
+#[cfg_attr(not(test), rustc_diagnostic_item = "Vec")]
+#[rustc_insignificant_dtor]
+pub struct Vec<T, #[unstable(feature = "allocator_api", issue = "32838")] A: Allocator = Global> {
+    buf: RawVec<T, A>,
+    len: usize,
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Inherent methods
+////////////////////////////////////////////////////////////////////////////////
+
+impl<T> Vec<T> {
+    /// Constructs a new, empty `Vec<T>`.
+    ///
+    /// The vector will not allocate until elements are pushed onto it.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # #![allow(unused_mut)]
+    /// let mut vec: Vec<i32> = Vec::new();
+    /// ```
+    #[inline]
+    #[rustc_const_stable(feature = "const_vec_new", since = "1.39.0")]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[must_use]
+    pub const fn new() -> Self {
+        Vec { buf: RawVec::NEW, len: 0 }
+    }
+
+    /// Constructs a new, empty `Vec<T>` with the specified capacity.
+    ///
+    /// The vector will be able to hold exactly `capacity` elements without
+    /// reallocating. If `capacity` is 0, the vector will not allocate.
+    ///
+    /// It is important to note that although the returned vector has the
+    /// *capacity* specified, the vector will have a zero *length*. For an
+    /// explanation of the difference between length and capacity, see
+    /// *[Capacity and reallocation]*.
+    ///
+    /// [Capacity and reallocation]: #capacity-and-reallocation
+    ///
+    /// # Panics
+    ///
+    /// Panics if the new capacity exceeds `isize::MAX` bytes.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = Vec::with_capacity(10);
+    ///
+    /// // The vector contains no items, even though it has capacity for more
+    /// assert_eq!(vec.len(), 0);
+    /// assert_eq!(vec.capacity(), 10);
+    ///
+    /// // These are all done without reallocating...
+    /// for i in 0..10 {
+    ///     vec.push(i);
+    /// }
+    /// assert_eq!(vec.len(), 10);
+    /// assert_eq!(vec.capacity(), 10);
+    ///
+    /// // ...but this may make the vector reallocate
+    /// vec.push(11);
+    /// assert_eq!(vec.len(), 11);
+    /// assert!(vec.capacity() >= 11);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[must_use]
+    pub fn with_capacity(capacity: usize) -> Self {
+        Self::with_capacity_in(capacity, Global)
+    }
+
+    /// Creates a `Vec<T>` directly from the raw components of another vector.
+    ///
+    /// # Safety
+    ///
+    /// This is highly unsafe, due to the number of invariants that aren't
+    /// checked:
+    ///
+    /// * `ptr` needs to have been previously allocated via [`String`]/`Vec<T>`
+    ///   (at least, it's highly likely to be incorrect if it wasn't).
+    /// * `T` needs to have the same alignment as what `ptr` was allocated with.
+    ///   (`T` having a less strict alignment is not sufficient, the alignment really
+    ///   needs to be equal to satisfy the [`dealloc`] requirement that memory must be
+    ///   allocated and deallocated with the same layout.)
+    /// * The size of `T` times the `capacity` (ie. the allocated size in bytes) needs
+    ///   to be the same size as the pointer was allocated with. (Because similar to
+    ///   alignment, [`dealloc`] must be called with the same layout `size`.)
+    /// * `length` needs to be less than or equal to `capacity`.
+    ///
+    /// Violating these may cause problems like corrupting the allocator's
+    /// internal data structures. For example it is normally **not** safe
+    /// to build a `Vec<u8>` from a pointer to a C `char` array with length
+    /// `size_t`, doing so is only safe if the array was initially allocated by
+    /// a `Vec` or `String`.
+    /// It's also not safe to build one from a `Vec<u16>` and its length, because
+    /// the allocator cares about the alignment, and these two types have different
+    /// alignments. The buffer was allocated with alignment 2 (for `u16`), but after
+    /// turning it into a `Vec<u8>` it'll be deallocated with alignment 1. To avoid
+    /// these issues, it is often preferable to do casting/transmuting using
+    /// [`slice::from_raw_parts`] instead.
+    ///
+    /// The ownership of `ptr` is effectively transferred to the
+    /// `Vec<T>` which may then deallocate, reallocate or change the
+    /// contents of memory pointed to by the pointer at will. Ensure
+    /// that nothing else uses the pointer after calling this
+    /// function.
+    ///
+    /// [`String`]: crate::string::String
+    /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::ptr;
+    /// use std::mem;
+    ///
+    /// let v = vec![1, 2, 3];
+    ///
+    // FIXME Update this when vec_into_raw_parts is stabilized
+    /// // Prevent running `v`'s destructor so we are in complete control
+    /// // of the allocation.
+    /// let mut v = mem::ManuallyDrop::new(v);
+    ///
+    /// // Pull out the various important pieces of information about `v`
+    /// let p = v.as_mut_ptr();
+    /// let len = v.len();
+    /// let cap = v.capacity();
+    ///
+    /// unsafe {
+    ///     // Overwrite memory with 4, 5, 6
+    ///     for i in 0..len as isize {
+    ///         ptr::write(p.offset(i), 4 + i);
+    ///     }
+    ///
+    ///     // Put everything back together into a Vec
+    ///     let rebuilt = Vec::from_raw_parts(p, len, cap);
+    ///     assert_eq!(rebuilt, [4, 5, 6]);
+    /// }
+    /// ```
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub unsafe fn from_raw_parts(ptr: *mut T, length: usize, capacity: usize) -> Self {
+        unsafe { Self::from_raw_parts_in(ptr, length, capacity, Global) }
+    }
+}
+
+impl<T, A: Allocator> Vec<T, A> {
+    /// Constructs a new, empty `Vec<T, A>`.
+    ///
+    /// The vector will not allocate until elements are pushed onto it.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// # #[allow(unused_mut)]
+    /// let mut vec: Vec<i32, _> = Vec::new_in(System);
+    /// ```
+    #[inline]
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    pub const fn new_in(alloc: A) -> Self {
+        Vec { buf: RawVec::new_in(alloc), len: 0 }
+    }
+
+    /// Constructs a new, empty `Vec<T, A>` with the specified capacity with the provided
+    /// allocator.
+    ///
+    /// The vector will be able to hold exactly `capacity` elements without
+    /// reallocating. If `capacity` is 0, the vector will not allocate.
+    ///
+    /// It is important to note that although the returned vector has the
+    /// *capacity* specified, the vector will have a zero *length*. For an
+    /// explanation of the difference between length and capacity, see
+    /// *[Capacity and reallocation]*.
+    ///
+    /// [Capacity and reallocation]: #capacity-and-reallocation
+    ///
+    /// # Panics
+    ///
+    /// Panics if the new capacity exceeds `isize::MAX` bytes.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let mut vec = Vec::with_capacity_in(10, System);
+    ///
+    /// // The vector contains no items, even though it has capacity for more
+    /// assert_eq!(vec.len(), 0);
+    /// assert_eq!(vec.capacity(), 10);
+    ///
+    /// // These are all done without reallocating...
+    /// for i in 0..10 {
+    ///     vec.push(i);
+    /// }
+    /// assert_eq!(vec.len(), 10);
+    /// assert_eq!(vec.capacity(), 10);
+    ///
+    /// // ...but this may make the vector reallocate
+    /// vec.push(11);
+    /// assert_eq!(vec.len(), 11);
+    /// assert!(vec.capacity() >= 11);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    pub fn with_capacity_in(capacity: usize, alloc: A) -> Self {
+        Vec { buf: RawVec::with_capacity_in(capacity, alloc), len: 0 }
+    }
+
+    /// Creates a `Vec<T, A>` directly from the raw components of another vector.
+    ///
+    /// # Safety
+    ///
+    /// This is highly unsafe, due to the number of invariants that aren't
+    /// checked:
+    ///
+    /// * `ptr` needs to have been previously allocated via [`String`]/`Vec<T>`
+    ///   (at least, it's highly likely to be incorrect if it wasn't).
+    /// * `T` needs to have the same size and alignment as what `ptr` was allocated with.
+    ///   (`T` having a less strict alignment is not sufficient, the alignment really
+    ///   needs to be equal to satisfy the [`dealloc`] requirement that memory must be
+    ///   allocated and deallocated with the same layout.)
+    /// * `length` needs to be less than or equal to `capacity`.
+    /// * `capacity` needs to be the capacity that the pointer was allocated with.
+    ///
+    /// Violating these may cause problems like corrupting the allocator's
+    /// internal data structures. For example it is **not** safe
+    /// to build a `Vec<u8>` from a pointer to a C `char` array with length `size_t`.
+    /// It's also not safe to build one from a `Vec<u16>` and its length, because
+    /// the allocator cares about the alignment, and these two types have different
+    /// alignments. The buffer was allocated with alignment 2 (for `u16`), but after
+    /// turning it into a `Vec<u8>` it'll be deallocated with alignment 1.
+    ///
+    /// The ownership of `ptr` is effectively transferred to the
+    /// `Vec<T>` which may then deallocate, reallocate or change the
+    /// contents of memory pointed to by the pointer at will. Ensure
+    /// that nothing else uses the pointer after calling this
+    /// function.
+    ///
+    /// [`String`]: crate::string::String
+    /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// use std::ptr;
+    /// use std::mem;
+    ///
+    /// let mut v = Vec::with_capacity_in(3, System);
+    /// v.push(1);
+    /// v.push(2);
+    /// v.push(3);
+    ///
+    // FIXME Update this when vec_into_raw_parts is stabilized
+    /// // Prevent running `v`'s destructor so we are in complete control
+    /// // of the allocation.
+    /// let mut v = mem::ManuallyDrop::new(v);
+    ///
+    /// // Pull out the various important pieces of information about `v`
+    /// let p = v.as_mut_ptr();
+    /// let len = v.len();
+    /// let cap = v.capacity();
+    /// let alloc = v.allocator();
+    ///
+    /// unsafe {
+    ///     // Overwrite memory with 4, 5, 6
+    ///     for i in 0..len as isize {
+    ///         ptr::write(p.offset(i), 4 + i);
+    ///     }
+    ///
+    ///     // Put everything back together into a Vec
+    ///     let rebuilt = Vec::from_raw_parts_in(p, len, cap, alloc.clone());
+    ///     assert_eq!(rebuilt, [4, 5, 6]);
+    /// }
+    /// ```
+    #[inline]
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    pub unsafe fn from_raw_parts_in(ptr: *mut T, length: usize, capacity: usize, alloc: A) -> Self {
+        unsafe { Vec { buf: RawVec::from_raw_parts_in(ptr, capacity, alloc), len: length } }
+    }
+
+    /// Decomposes a `Vec<T>` into its raw components.
+    ///
+    /// Returns the raw pointer to the underlying data, the length of
+    /// the vector (in elements), and the allocated capacity of the
+    /// data (in elements). These are the same arguments in the same
+    /// order as the arguments to [`from_raw_parts`].
+    ///
+    /// After calling this function, the caller is responsible for the
+    /// memory previously managed by the `Vec`. The only way to do
+    /// this is to convert the raw pointer, length, and capacity back
+    /// into a `Vec` with the [`from_raw_parts`] function, allowing
+    /// the destructor to perform the cleanup.
+    ///
+    /// [`from_raw_parts`]: Vec::from_raw_parts
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(vec_into_raw_parts)]
+    /// let v: Vec<i32> = vec![-1, 0, 1];
+    ///
+    /// let (ptr, len, cap) = v.into_raw_parts();
+    ///
+    /// let rebuilt = unsafe {
+    ///     // We can now make changes to the components, such as
+    ///     // transmuting the raw pointer to a compatible type.
+    ///     let ptr = ptr as *mut u32;
+    ///
+    ///     Vec::from_raw_parts(ptr, len, cap)
+    /// };
+    /// assert_eq!(rebuilt, [4294967295, 0, 1]);
+    /// ```
+    #[unstable(feature = "vec_into_raw_parts", reason = "new API", issue = "65816")]
+    pub fn into_raw_parts(self) -> (*mut T, usize, usize) {
+        let mut me = ManuallyDrop::new(self);
+        (me.as_mut_ptr(), me.len(), me.capacity())
+    }
+
+    /// Decomposes a `Vec<T>` into its raw components.
+    ///
+    /// Returns the raw pointer to the underlying data, the length of the vector (in elements),
+    /// the allocated capacity of the data (in elements), and the allocator. These are the same
+    /// arguments in the same order as the arguments to [`from_raw_parts_in`].
+    ///
+    /// After calling this function, the caller is responsible for the
+    /// memory previously managed by the `Vec`. The only way to do
+    /// this is to convert the raw pointer, length, and capacity back
+    /// into a `Vec` with the [`from_raw_parts_in`] function, allowing
+    /// the destructor to perform the cleanup.
+    ///
+    /// [`from_raw_parts_in`]: Vec::from_raw_parts_in
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(allocator_api, vec_into_raw_parts)]
+    ///
+    /// use std::alloc::System;
+    ///
+    /// let mut v: Vec<i32, System> = Vec::new_in(System);
+    /// v.push(-1);
+    /// v.push(0);
+    /// v.push(1);
+    ///
+    /// let (ptr, len, cap, alloc) = v.into_raw_parts_with_alloc();
+    ///
+    /// let rebuilt = unsafe {
+    ///     // We can now make changes to the components, such as
+    ///     // transmuting the raw pointer to a compatible type.
+    ///     let ptr = ptr as *mut u32;
+    ///
+    ///     Vec::from_raw_parts_in(ptr, len, cap, alloc)
+    /// };
+    /// assert_eq!(rebuilt, [4294967295, 0, 1]);
+    /// ```
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    // #[unstable(feature = "vec_into_raw_parts", reason = "new API", issue = "65816")]
+    pub fn into_raw_parts_with_alloc(self) -> (*mut T, usize, usize, A) {
+        let mut me = ManuallyDrop::new(self);
+        let len = me.len();
+        let capacity = me.capacity();
+        let ptr = me.as_mut_ptr();
+        let alloc = unsafe { ptr::read(me.allocator()) };
+        (ptr, len, capacity, alloc)
+    }
+
+    /// Returns the number of elements the vector can hold without
+    /// reallocating.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let vec: Vec<i32> = Vec::with_capacity(10);
+    /// assert_eq!(vec.capacity(), 10);
+    /// ```
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn capacity(&self) -> usize {
+        self.buf.capacity()
+    }
+
+    /// Reserves capacity for at least `additional` more elements to be inserted
+    /// in the given `Vec<T>`. The collection may reserve more space to avoid
+    /// frequent reallocations. After calling `reserve`, capacity will be
+    /// greater than or equal to `self.len() + additional`. Does nothing if
+    /// capacity is already sufficient.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the new capacity exceeds `isize::MAX` bytes.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1];
+    /// vec.reserve(10);
+    /// assert!(vec.capacity() >= 11);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn reserve(&mut self, additional: usize) {
+        self.buf.reserve(self.len, additional);
+    }
+
+    /// Reserves the minimum capacity for exactly `additional` more elements to
+    /// be inserted in the given `Vec<T>`. After calling `reserve_exact`,
+    /// capacity will be greater than or equal to `self.len() + additional`.
+    /// Does nothing if the capacity is already sufficient.
+    ///
+    /// Note that the allocator may give the collection more space than it
+    /// requests. Therefore, capacity can not be relied upon to be precisely
+    /// minimal. Prefer [`reserve`] if future insertions are expected.
+    ///
+    /// [`reserve`]: Vec::reserve
+    ///
+    /// # Panics
+    ///
+    /// Panics if the new capacity exceeds `isize::MAX` bytes.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1];
+    /// vec.reserve_exact(10);
+    /// assert!(vec.capacity() >= 11);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn reserve_exact(&mut self, additional: usize) {
+        self.buf.reserve_exact(self.len, additional);
+    }
+
+    /// Tries to reserve capacity for at least `additional` more elements to be inserted
+    /// in the given `Vec<T>`. The collection may reserve more space to avoid
+    /// frequent reallocations. After calling `try_reserve`, capacity will be
+    /// greater than or equal to `self.len() + additional`. Does nothing if
+    /// capacity is already sufficient.
+    ///
+    /// # Errors
+    ///
+    /// If the capacity overflows, or the allocator reports a failure, then an error
+    /// is returned.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::collections::TryReserveError;
+    ///
+    /// fn process_data(data: &[u32]) -> Result<Vec<u32>, TryReserveError> {
+    ///     let mut output = Vec::new();
+    ///
+    ///     // Pre-reserve the memory, exiting if we can't
+    ///     output.try_reserve(data.len())?;
+    ///
+    ///     // Now we know this can't OOM in the middle of our complex work
+    ///     output.extend(data.iter().map(|&val| {
+    ///         val * 2 + 5 // very complicated
+    ///     }));
+    ///
+    ///     Ok(output)
+    /// }
+    /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?");
+    /// ```
+    #[stable(feature = "try_reserve", since = "1.57.0")]
+    pub fn try_reserve(&mut self, additional: usize) -> Result<(), TryReserveError> {
+        self.buf.try_reserve(self.len, additional)
+    }
+
+    /// Tries to reserve the minimum capacity for exactly `additional`
+    /// elements to be inserted in the given `Vec<T>`. After calling
+    /// `try_reserve_exact`, capacity will be greater than or equal to
+    /// `self.len() + additional` if it returns `Ok(())`.
+    /// Does nothing if the capacity is already sufficient.
+    ///
+    /// Note that the allocator may give the collection more space than it
+    /// requests. Therefore, capacity can not be relied upon to be precisely
+    /// minimal. Prefer [`try_reserve`] if future insertions are expected.
+    ///
+    /// [`try_reserve`]: Vec::try_reserve
+    ///
+    /// # Errors
+    ///
+    /// If the capacity overflows, or the allocator reports a failure, then an error
+    /// is returned.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::collections::TryReserveError;
+    ///
+    /// fn process_data(data: &[u32]) -> Result<Vec<u32>, TryReserveError> {
+    ///     let mut output = Vec::new();
+    ///
+    ///     // Pre-reserve the memory, exiting if we can't
+    ///     output.try_reserve_exact(data.len())?;
+    ///
+    ///     // Now we know this can't OOM in the middle of our complex work
+    ///     output.extend(data.iter().map(|&val| {
+    ///         val * 2 + 5 // very complicated
+    ///     }));
+    ///
+    ///     Ok(output)
+    /// }
+    /// # process_data(&[1, 2, 3]).expect("why is the test harness OOMing on 12 bytes?");
+    /// ```
+    #[stable(feature = "try_reserve", since = "1.57.0")]
+    pub fn try_reserve_exact(&mut self, additional: usize) -> Result<(), TryReserveError> {
+        self.buf.try_reserve_exact(self.len, additional)
+    }
+
+    /// Shrinks the capacity of the vector as much as possible.
+    ///
+    /// It will drop down as close as possible to the length but the allocator
+    /// may still inform the vector that there is space for a few more elements.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = Vec::with_capacity(10);
+    /// vec.extend([1, 2, 3]);
+    /// assert_eq!(vec.capacity(), 10);
+    /// vec.shrink_to_fit();
+    /// assert!(vec.capacity() >= 3);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn shrink_to_fit(&mut self) {
+        // The capacity is never less than the length, and there's nothing to do when
+        // they are equal, so we can avoid the panic case in `RawVec::shrink_to_fit`
+        // by only calling it with a greater capacity.
+        if self.capacity() > self.len {
+            self.buf.shrink_to_fit(self.len);
+        }
+    }
+
+    /// Shrinks the capacity of the vector with a lower bound.
+    ///
+    /// The capacity will remain at least as large as both the length
+    /// and the supplied value.
+    ///
+    /// If the current capacity is less than the lower limit, this is a no-op.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = Vec::with_capacity(10);
+    /// vec.extend([1, 2, 3]);
+    /// assert_eq!(vec.capacity(), 10);
+    /// vec.shrink_to(4);
+    /// assert!(vec.capacity() >= 4);
+    /// vec.shrink_to(0);
+    /// assert!(vec.capacity() >= 3);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "shrink_to", since = "1.56.0")]
+    pub fn shrink_to(&mut self, min_capacity: usize) {
+        if self.capacity() > min_capacity {
+            self.buf.shrink_to_fit(cmp::max(self.len, min_capacity));
+        }
+    }
+
+    /// Converts the vector into [`Box<[T]>`][owned slice].
+    ///
+    /// Note that this will drop any excess capacity.
+    ///
+    /// [owned slice]: Box
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let v = vec![1, 2, 3];
+    ///
+    /// let slice = v.into_boxed_slice();
+    /// ```
+    ///
+    /// Any excess capacity is removed:
+    ///
+    /// ```
+    /// let mut vec = Vec::with_capacity(10);
+    /// vec.extend([1, 2, 3]);
+    ///
+    /// assert_eq!(vec.capacity(), 10);
+    /// let slice = vec.into_boxed_slice();
+    /// assert_eq!(slice.into_vec().capacity(), 3);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn into_boxed_slice(mut self) -> Box<[T], A> {
+        unsafe {
+            self.shrink_to_fit();
+            let me = ManuallyDrop::new(self);
+            let buf = ptr::read(&me.buf);
+            let len = me.len();
+            buf.into_box(len).assume_init()
+        }
+    }
+
+    /// Shortens the vector, keeping the first `len` elements and dropping
+    /// the rest.
+    ///
+    /// If `len` is greater than the vector's current length, this has no
+    /// effect.
+    ///
+    /// The [`drain`] method can emulate `truncate`, but causes the excess
+    /// elements to be returned instead of dropped.
+    ///
+    /// Note that this method has no effect on the allocated capacity
+    /// of the vector.
+    ///
+    /// # Examples
+    ///
+    /// Truncating a five element vector to two elements:
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3, 4, 5];
+    /// vec.truncate(2);
+    /// assert_eq!(vec, [1, 2]);
+    /// ```
+    ///
+    /// No truncation occurs when `len` is greater than the vector's current
+    /// length:
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3];
+    /// vec.truncate(8);
+    /// assert_eq!(vec, [1, 2, 3]);
+    /// ```
+    ///
+    /// Truncating when `len == 0` is equivalent to calling the [`clear`]
+    /// method.
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3];
+    /// vec.truncate(0);
+    /// assert_eq!(vec, []);
+    /// ```
+    ///
+    /// [`clear`]: Vec::clear
+    /// [`drain`]: Vec::drain
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn truncate(&mut self, len: usize) {
+        // This is safe because:
+        //
+        // * the slice passed to `drop_in_place` is valid; the `len > self.len`
+        //   case avoids creating an invalid slice, and
+        // * the `len` of the vector is shrunk before calling `drop_in_place`,
+        //   such that no value will be dropped twice in case `drop_in_place`
+        //   were to panic once (if it panics twice, the program aborts).
+        unsafe {
+            // Note: It's intentional that this is `>` and not `>=`.
+            //       Changing it to `>=` has negative performance
+            //       implications in some cases. See #78884 for more.
+            if len > self.len {
+                return;
+            }
+            let remaining_len = self.len - len;
+            let s = ptr::slice_from_raw_parts_mut(self.as_mut_ptr().add(len), remaining_len);
+            self.len = len;
+            ptr::drop_in_place(s);
+        }
+    }
+
+    /// Extracts a slice containing the entire vector.
+    ///
+    /// Equivalent to `&s[..]`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::io::{self, Write};
+    /// let buffer = vec![1, 2, 3, 5, 8];
+    /// io::sink().write(buffer.as_slice()).unwrap();
+    /// ```
+    #[inline]
+    #[stable(feature = "vec_as_slice", since = "1.7.0")]
+    pub fn as_slice(&self) -> &[T] {
+        self
+    }
+
+    /// Extracts a mutable slice of the entire vector.
+    ///
+    /// Equivalent to `&mut s[..]`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// use std::io::{self, Read};
+    /// let mut buffer = vec![0; 3];
+    /// io::repeat(0b101).read_exact(buffer.as_mut_slice()).unwrap();
+    /// ```
+    #[inline]
+    #[stable(feature = "vec_as_slice", since = "1.7.0")]
+    pub fn as_mut_slice(&mut self) -> &mut [T] {
+        self
+    }
+
+    /// Returns a raw pointer to the vector's buffer.
+    ///
+    /// The caller must ensure that the vector outlives the pointer this
+    /// function returns, or else it will end up pointing to garbage.
+    /// Modifying the vector may cause its buffer to be reallocated,
+    /// which would also make any pointers to it invalid.
+    ///
+    /// The caller must also ensure that the memory the pointer (non-transitively) points to
+    /// is never written to (except inside an `UnsafeCell`) using this pointer or any pointer
+    /// derived from it. If you need to mutate the contents of the slice, use [`as_mut_ptr`].
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let x = vec![1, 2, 4];
+    /// let x_ptr = x.as_ptr();
+    ///
+    /// unsafe {
+    ///     for i in 0..x.len() {
+    ///         assert_eq!(*x_ptr.add(i), 1 << i);
+    ///     }
+    /// }
+    /// ```
+    ///
+    /// [`as_mut_ptr`]: Vec::as_mut_ptr
+    #[stable(feature = "vec_as_ptr", since = "1.37.0")]
+    #[inline]
+    pub fn as_ptr(&self) -> *const T {
+        // We shadow the slice method of the same name to avoid going through
+        // `deref`, which creates an intermediate reference.
+        let ptr = self.buf.ptr();
+        unsafe {
+            assume(!ptr.is_null());
+        }
+        ptr
+    }
+
+    /// Returns an unsafe mutable pointer to the vector's buffer.
+    ///
+    /// The caller must ensure that the vector outlives the pointer this
+    /// function returns, or else it will end up pointing to garbage.
+    /// Modifying the vector may cause its buffer to be reallocated,
+    /// which would also make any pointers to it invalid.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// // Allocate vector big enough for 4 elements.
+    /// let size = 4;
+    /// let mut x: Vec<i32> = Vec::with_capacity(size);
+    /// let x_ptr = x.as_mut_ptr();
+    ///
+    /// // Initialize elements via raw pointer writes, then set length.
+    /// unsafe {
+    ///     for i in 0..size {
+    ///         *x_ptr.add(i) = i as i32;
+    ///     }
+    ///     x.set_len(size);
+    /// }
+    /// assert_eq!(&*x, &[0, 1, 2, 3]);
+    /// ```
+    #[stable(feature = "vec_as_ptr", since = "1.37.0")]
+    #[inline]
+    pub fn as_mut_ptr(&mut self) -> *mut T {
+        // We shadow the slice method of the same name to avoid going through
+        // `deref_mut`, which creates an intermediate reference.
+        let ptr = self.buf.ptr();
+        unsafe {
+            assume(!ptr.is_null());
+        }
+        ptr
+    }
+
+    /// Returns a reference to the underlying allocator.
+    #[unstable(feature = "allocator_api", issue = "32838")]
+    #[inline]
+    pub fn allocator(&self) -> &A {
+        self.buf.allocator()
+    }
+
+    /// Forces the length of the vector to `new_len`.
+    ///
+    /// This is a low-level operation that maintains none of the normal
+    /// invariants of the type. Normally changing the length of a vector
+    /// is done using one of the safe operations instead, such as
+    /// [`truncate`], [`resize`], [`extend`], or [`clear`].
+    ///
+    /// [`truncate`]: Vec::truncate
+    /// [`resize`]: Vec::resize
+    /// [`extend`]: Extend::extend
+    /// [`clear`]: Vec::clear
+    ///
+    /// # Safety
+    ///
+    /// - `new_len` must be less than or equal to [`capacity()`].
+    /// - The elements at `old_len..new_len` must be initialized.
+    ///
+    /// [`capacity()`]: Vec::capacity
+    ///
+    /// # Examples
+    ///
+    /// This method can be useful for situations in which the vector
+    /// is serving as a buffer for other code, particularly over FFI:
+    ///
+    /// ```no_run
+    /// # #![allow(dead_code)]
+    /// # // This is just a minimal skeleton for the doc example;
+    /// # // don't use this as a starting point for a real library.
+    /// # pub struct StreamWrapper { strm: *mut std::ffi::c_void }
+    /// # const Z_OK: i32 = 0;
+    /// # extern "C" {
+    /// #     fn deflateGetDictionary(
+    /// #         strm: *mut std::ffi::c_void,
+    /// #         dictionary: *mut u8,
+    /// #         dictLength: *mut usize,
+    /// #     ) -> i32;
+    /// # }
+    /// # impl StreamWrapper {
+    /// pub fn get_dictionary(&self) -> Option<Vec<u8>> {
+    ///     // Per the FFI method's docs, "32768 bytes is always enough".
+    ///     let mut dict = Vec::with_capacity(32_768);
+    ///     let mut dict_length = 0;
+    ///     // SAFETY: When `deflateGetDictionary` returns `Z_OK`, it holds that:
+    ///     // 1. `dict_length` elements were initialized.
+    ///     // 2. `dict_length` <= the capacity (32_768)
+    ///     // which makes `set_len` safe to call.
+    ///     unsafe {
+    ///         // Make the FFI call...
+    ///         let r = deflateGetDictionary(self.strm, dict.as_mut_ptr(), &mut dict_length);
+    ///         if r == Z_OK {
+    ///             // ...and update the length to what was initialized.
+    ///             dict.set_len(dict_length);
+    ///             Some(dict)
+    ///         } else {
+    ///             None
+    ///         }
+    ///     }
+    /// }
+    /// # }
+    /// ```
+    ///
+    /// While the following example is sound, there is a memory leak since
+    /// the inner vectors were not freed prior to the `set_len` call:
+    ///
+    /// ```
+    /// let mut vec = vec![vec![1, 0, 0],
+    ///                    vec![0, 1, 0],
+    ///                    vec![0, 0, 1]];
+    /// // SAFETY:
+    /// // 1. `old_len..0` is empty so no elements need to be initialized.
+    /// // 2. `0 <= capacity` always holds whatever `capacity` is.
+    /// unsafe {
+    ///     vec.set_len(0);
+    /// }
+    /// ```
+    ///
+    /// Normally, here, one would use [`clear`] instead to correctly drop
+    /// the contents and thus not leak memory.
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub unsafe fn set_len(&mut self, new_len: usize) {
+        debug_assert!(new_len <= self.capacity());
+
+        self.len = new_len;
+    }
+
+    /// Removes an element from the vector and returns it.
+    ///
+    /// The removed element is replaced by the last element of the vector.
+    ///
+    /// This does not preserve ordering, but is *O*(1).
+    /// If you need to preserve the element order, use [`remove`] instead.
+    ///
+    /// [`remove`]: Vec::remove
+    ///
+    /// # Panics
+    ///
+    /// Panics if `index` is out of bounds.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = vec!["foo", "bar", "baz", "qux"];
+    ///
+    /// assert_eq!(v.swap_remove(1), "bar");
+    /// assert_eq!(v, ["foo", "qux", "baz"]);
+    ///
+    /// assert_eq!(v.swap_remove(0), "foo");
+    /// assert_eq!(v, ["baz", "qux"]);
+    /// ```
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn swap_remove(&mut self, index: usize) -> T {
+        #[cold]
+        #[inline(never)]
+        fn assert_failed(index: usize, len: usize) -> ! {
+            panic!("swap_remove index (is {index}) should be < len (is {len})");
+        }
+
+        let len = self.len();
+        if index >= len {
+            assert_failed(index, len);
+        }
+        unsafe {
+            // We replace self[index] with the last element. Note that if the
+            // bounds check above succeeds there must be a last element (which
+            // can be self[index] itself).
+            let value = ptr::read(self.as_ptr().add(index));
+            let base_ptr = self.as_mut_ptr();
+            ptr::copy(base_ptr.add(len - 1), base_ptr.add(index), 1);
+            self.set_len(len - 1);
+            value
+        }
+    }
+
+    /// Inserts an element at position `index` within the vector, shifting all
+    /// elements after it to the right.
+    ///
+    /// # Panics
+    ///
+    /// Panics if `index > len`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3];
+    /// vec.insert(1, 4);
+    /// assert_eq!(vec, [1, 4, 2, 3]);
+    /// vec.insert(4, 5);
+    /// assert_eq!(vec, [1, 4, 2, 3, 5]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn insert(&mut self, index: usize, element: T) {
+        #[cold]
+        #[inline(never)]
+        fn assert_failed(index: usize, len: usize) -> ! {
+            panic!("insertion index (is {index}) should be <= len (is {len})");
+        }
+
+        let len = self.len();
+        if index > len {
+            assert_failed(index, len);
+        }
+
+        // space for the new element
+        if len == self.buf.capacity() {
+            self.reserve(1);
+        }
+
+        unsafe {
+            // infallible
+            // The spot to put the new value
+            {
+                let p = self.as_mut_ptr().add(index);
+                // Shift everything over to make space. (Duplicating the
+                // `index`th element into two consecutive places.)
+                ptr::copy(p, p.offset(1), len - index);
+                // Write it in, overwriting the first copy of the `index`th
+                // element.
+                ptr::write(p, element);
+            }
+            self.set_len(len + 1);
+        }
+    }
+
+    /// Removes and returns the element at position `index` within the vector,
+    /// shifting all elements after it to the left.
+    ///
+    /// Note: Because this shifts over the remaining elements, it has a
+    /// worst-case performance of *O*(*n*). If you don't need the order of elements
+    /// to be preserved, use [`swap_remove`] instead. If you'd like to remove
+    /// elements from the beginning of the `Vec`, consider using
+    /// [`VecDeque::pop_front`] instead.
+    ///
+    /// [`swap_remove`]: Vec::swap_remove
+    /// [`VecDeque::pop_front`]: crate::collections::VecDeque::pop_front
+    ///
+    /// # Panics
+    ///
+    /// Panics if `index` is out of bounds.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = vec![1, 2, 3];
+    /// assert_eq!(v.remove(1), 2);
+    /// assert_eq!(v, [1, 3]);
+    /// ```
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[track_caller]
+    pub fn remove(&mut self, index: usize) -> T {
+        #[cold]
+        #[inline(never)]
+        #[track_caller]
+        fn assert_failed(index: usize, len: usize) -> ! {
+            panic!("removal index (is {index}) should be < len (is {len})");
+        }
+
+        let len = self.len();
+        if index >= len {
+            assert_failed(index, len);
+        }
+        unsafe {
+            // infallible
+            let ret;
+            {
+                // the place we are taking from.
+                let ptr = self.as_mut_ptr().add(index);
+                // copy it out, unsafely having a copy of the value on
+                // the stack and in the vector at the same time.
+                ret = ptr::read(ptr);
+
+                // Shift everything down to fill in that spot.
+                ptr::copy(ptr.offset(1), ptr, len - index - 1);
+            }
+            self.set_len(len - 1);
+            ret
+        }
+    }
+
+    /// Retains only the elements specified by the predicate.
+    ///
+    /// In other words, remove all elements `e` for which `f(&e)` returns `false`.
+    /// This method operates in place, visiting each element exactly once in the
+    /// original order, and preserves the order of the retained elements.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3, 4];
+    /// vec.retain(|&x| x % 2 == 0);
+    /// assert_eq!(vec, [2, 4]);
+    /// ```
+    ///
+    /// Because the elements are visited exactly once in the original order,
+    /// external state may be used to decide which elements to keep.
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3, 4, 5];
+    /// let keep = [false, true, true, false, true];
+    /// let mut iter = keep.iter();
+    /// vec.retain(|_| *iter.next().unwrap());
+    /// assert_eq!(vec, [2, 3, 5]);
+    /// ```
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn retain<F>(&mut self, mut f: F)
+    where
+        F: FnMut(&T) -> bool,
+    {
+        self.retain_mut(|elem| f(elem));
+    }
+
+    /// Retains only the elements specified by the predicate, passing a mutable reference to it.
+    ///
+    /// In other words, remove all elements `e` such that `f(&mut e)` returns `false`.
+    /// This method operates in place, visiting each element exactly once in the
+    /// original order, and preserves the order of the retained elements.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3, 4];
+    /// vec.retain_mut(|x| if *x > 3 {
+    ///     false
+    /// } else {
+    ///     *x += 1;
+    ///     true
+    /// });
+    /// assert_eq!(vec, [2, 3, 4]);
+    /// ```
+    #[stable(feature = "vec_retain_mut", since = "1.61.0")]
+    pub fn retain_mut<F>(&mut self, mut f: F)
+    where
+        F: FnMut(&mut T) -> bool,
+    {
+        let original_len = self.len();
+        // Avoid double drop if the drop guard is not executed,
+        // since we may make some holes during the process.
+        unsafe { self.set_len(0) };
+
+        // Vec: [Kept, Kept, Hole, Hole, Hole, Hole, Unchecked, Unchecked]
+        //      |<-              processed len   ->| ^- next to check
+        //                  |<-  deleted cnt     ->|
+        //      |<-              original_len                          ->|
+        // Kept: Elements which predicate returns true on.
+        // Hole: Moved or dropped element slot.
+        // Unchecked: Unchecked valid elements.
+        //
+        // This drop guard will be invoked when predicate or `drop` of element panicked.
+        // It shifts unchecked elements to cover holes and `set_len` to the correct length.
+        // In cases when predicate and `drop` never panick, it will be optimized out.
+        struct BackshiftOnDrop<'a, T, A: Allocator> {
+            v: &'a mut Vec<T, A>,
+            processed_len: usize,
+            deleted_cnt: usize,
+            original_len: usize,
+        }
+
+        impl<T, A: Allocator> Drop for BackshiftOnDrop<'_, T, A> {
+            fn drop(&mut self) {
+                if self.deleted_cnt > 0 {
+                    // SAFETY: Trailing unchecked items must be valid since we never touch them.
+                    unsafe {
+                        ptr::copy(
+                            self.v.as_ptr().add(self.processed_len),
+                            self.v.as_mut_ptr().add(self.processed_len - self.deleted_cnt),
+                            self.original_len - self.processed_len,
+                        );
+                    }
+                }
+                // SAFETY: After filling holes, all items are in contiguous memory.
+                unsafe {
+                    self.v.set_len(self.original_len - self.deleted_cnt);
+                }
+            }
+        }
+
+        let mut g = BackshiftOnDrop { v: self, processed_len: 0, deleted_cnt: 0, original_len };
+
+        fn process_loop<F, T, A: Allocator, const DELETED: bool>(
+            original_len: usize,
+            f: &mut F,
+            g: &mut BackshiftOnDrop<'_, T, A>,
+        ) where
+            F: FnMut(&mut T) -> bool,
+        {
+            while g.processed_len != original_len {
+                // SAFETY: Unchecked element must be valid.
+                let cur = unsafe { &mut *g.v.as_mut_ptr().add(g.processed_len) };
+                if !f(cur) {
+                    // Advance early to avoid double drop if `drop_in_place` panicked.
+                    g.processed_len += 1;
+                    g.deleted_cnt += 1;
+                    // SAFETY: We never touch this element again after dropped.
+                    unsafe { ptr::drop_in_place(cur) };
+                    // We already advanced the counter.
+                    if DELETED {
+                        continue;
+                    } else {
+                        break;
+                    }
+                }
+                if DELETED {
+                    // SAFETY: `deleted_cnt` > 0, so the hole slot must not overlap with current element.
+                    // We use copy for move, and never touch this element again.
+                    unsafe {
+                        let hole_slot = g.v.as_mut_ptr().add(g.processed_len - g.deleted_cnt);
+                        ptr::copy_nonoverlapping(cur, hole_slot, 1);
+                    }
+                }
+                g.processed_len += 1;
+            }
+        }
+
+        // Stage 1: Nothing was deleted.
+        process_loop::<F, T, A, false>(original_len, &mut f, &mut g);
+
+        // Stage 2: Some elements were deleted.
+        process_loop::<F, T, A, true>(original_len, &mut f, &mut g);
+
+        // All item are processed. This can be optimized to `set_len` by LLVM.
+        drop(g);
+    }
+
+    /// Removes all but the first of consecutive elements in the vector that resolve to the same
+    /// key.
+    ///
+    /// If the vector is sorted, this removes all duplicates.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![10, 20, 21, 30, 20];
+    ///
+    /// vec.dedup_by_key(|i| *i / 10);
+    ///
+    /// assert_eq!(vec, [10, 20, 30, 20]);
+    /// ```
+    #[stable(feature = "dedup_by", since = "1.16.0")]
+    #[inline]
+    pub fn dedup_by_key<F, K>(&mut self, mut key: F)
+    where
+        F: FnMut(&mut T) -> K,
+        K: PartialEq,
+    {
+        self.dedup_by(|a, b| key(a) == key(b))
+    }
+
+    /// Removes all but the first of consecutive elements in the vector satisfying a given equality
+    /// relation.
+    ///
+    /// The `same_bucket` function is passed references to two elements from the vector and
+    /// must determine if the elements compare equal. The elements are passed in opposite order
+    /// from their order in the slice, so if `same_bucket(a, b)` returns `true`, `a` is removed.
+    ///
+    /// If the vector is sorted, this removes all duplicates.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec!["foo", "bar", "Bar", "baz", "bar"];
+    ///
+    /// vec.dedup_by(|a, b| a.eq_ignore_ascii_case(b));
+    ///
+    /// assert_eq!(vec, ["foo", "bar", "baz", "bar"]);
+    /// ```
+    #[stable(feature = "dedup_by", since = "1.16.0")]
+    pub fn dedup_by<F>(&mut self, mut same_bucket: F)
+    where
+        F: FnMut(&mut T, &mut T) -> bool,
+    {
+        let len = self.len();
+        if len <= 1 {
+            return;
+        }
+
+        /* INVARIANT: vec.len() > read >= write > write-1 >= 0 */
+        struct FillGapOnDrop<'a, T, A: core::alloc::Allocator> {
+            /* Offset of the element we want to check if it is duplicate */
+            read: usize,
+
+            /* Offset of the place where we want to place the non-duplicate
+             * when we find it. */
+            write: usize,
+
+            /* The Vec that would need correction if `same_bucket` panicked */
+            vec: &'a mut Vec<T, A>,
+        }
+
+        impl<'a, T, A: core::alloc::Allocator> Drop for FillGapOnDrop<'a, T, A> {
+            fn drop(&mut self) {
+                /* This code gets executed when `same_bucket` panics */
+
+                /* SAFETY: invariant guarantees that `read - write`
+                 * and `len - read` never overflow and that the copy is always
+                 * in-bounds. */
+                unsafe {
+                    let ptr = self.vec.as_mut_ptr();
+                    let len = self.vec.len();
+
+                    /* How many items were left when `same_bucket` panicked.
+                     * Basically vec[read..].len() */
+                    let items_left = len.wrapping_sub(self.read);
+
+                    /* Pointer to first item in vec[write..write+items_left] slice */
+                    let dropped_ptr = ptr.add(self.write);
+                    /* Pointer to first item in vec[read..] slice */
+                    let valid_ptr = ptr.add(self.read);
+
+                    /* Copy `vec[read..]` to `vec[write..write+items_left]`.
+                     * The slices can overlap, so `copy_nonoverlapping` cannot be used */
+                    ptr::copy(valid_ptr, dropped_ptr, items_left);
+
+                    /* How many items have been already dropped
+                     * Basically vec[read..write].len() */
+                    let dropped = self.read.wrapping_sub(self.write);
+
+                    self.vec.set_len(len - dropped);
+                }
+            }
+        }
+
+        let mut gap = FillGapOnDrop { read: 1, write: 1, vec: self };
+        let ptr = gap.vec.as_mut_ptr();
+
+        /* Drop items while going through Vec, it should be more efficient than
+         * doing slice partition_dedup + truncate */
+
+        /* SAFETY: Because of the invariant, read_ptr, prev_ptr and write_ptr
+         * are always in-bounds and read_ptr never aliases prev_ptr */
+        unsafe {
+            while gap.read < len {
+                let read_ptr = ptr.add(gap.read);
+                let prev_ptr = ptr.add(gap.write.wrapping_sub(1));
+
+                if same_bucket(&mut *read_ptr, &mut *prev_ptr) {
+                    // Increase `gap.read` now since the drop may panic.
+                    gap.read += 1;
+                    /* We have found duplicate, drop it in-place */
+                    ptr::drop_in_place(read_ptr);
+                } else {
+                    let write_ptr = ptr.add(gap.write);
+
+                    /* Because `read_ptr` can be equal to `write_ptr`, we either
+                     * have to use `copy` or conditional `copy_nonoverlapping`.
+                     * Looks like the first option is faster. */
+                    ptr::copy(read_ptr, write_ptr, 1);
+
+                    /* We have filled that place, so go further */
+                    gap.write += 1;
+                    gap.read += 1;
+                }
+            }
+
+            /* Technically we could let `gap` clean up with its Drop, but
+             * when `same_bucket` is guaranteed to not panic, this bloats a little
+             * the codegen, so we just do it manually */
+            gap.vec.set_len(gap.write);
+            mem::forget(gap);
+        }
+    }
+
+    /// Appends an element to the back of a collection.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the new capacity exceeds `isize::MAX` bytes.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2];
+    /// vec.push(3);
+    /// assert_eq!(vec, [1, 2, 3]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn push(&mut self, value: T) {
+        // This will panic or abort if we would allocate > isize::MAX bytes
+        // or if the length increment would overflow for zero-sized types.
+        if self.len == self.buf.capacity() {
+            self.buf.reserve_for_push(self.len);
+        }
+        unsafe {
+            let end = self.as_mut_ptr().add(self.len);
+            ptr::write(end, value);
+            self.len += 1;
+        }
+    }
+
+    /// Tries to append an element to the back of a collection.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2];
+    /// vec.try_push(3).unwrap();
+    /// assert_eq!(vec, [1, 2, 3]);
+    /// ```
+    #[inline]
+    #[stable(feature = "kernel", since = "1.0.0")]
+    pub fn try_push(&mut self, value: T) -> Result<(), TryReserveError> {
+        if self.len == self.buf.capacity() {
+            self.buf.try_reserve_for_push(self.len)?;
+        }
+        unsafe {
+            let end = self.as_mut_ptr().add(self.len);
+            ptr::write(end, value);
+            self.len += 1;
+        }
+        Ok(())
+    }
+
+    /// Removes the last element from a vector and returns it, or [`None`] if it
+    /// is empty.
+    ///
+    /// If you'd like to pop the first element, consider using
+    /// [`VecDeque::pop_front`] instead.
+    ///
+    /// [`VecDeque::pop_front`]: crate::collections::VecDeque::pop_front
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3];
+    /// assert_eq!(vec.pop(), Some(3));
+    /// assert_eq!(vec, [1, 2]);
+    /// ```
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn pop(&mut self) -> Option<T> {
+        if self.len == 0 {
+            None
+        } else {
+            unsafe {
+                self.len -= 1;
+                Some(ptr::read(self.as_ptr().add(self.len())))
+            }
+        }
+    }
+
+    /// Moves all the elements of `other` into `self`, leaving `other` empty.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the number of elements in the vector overflows a `usize`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3];
+    /// let mut vec2 = vec![4, 5, 6];
+    /// vec.append(&mut vec2);
+    /// assert_eq!(vec, [1, 2, 3, 4, 5, 6]);
+    /// assert_eq!(vec2, []);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    #[stable(feature = "append", since = "1.4.0")]
+    pub fn append(&mut self, other: &mut Self) {
+        unsafe {
+            self.append_elements(other.as_slice() as _);
+            other.set_len(0);
+        }
+    }
+
+    /// Appends elements to `self` from other buffer.
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    unsafe fn append_elements(&mut self, other: *const [T]) {
+        let count = unsafe { (*other).len() };
+        self.reserve(count);
+        let len = self.len();
+        unsafe { ptr::copy_nonoverlapping(other as *const T, self.as_mut_ptr().add(len), count) };
+        self.len += count;
+    }
+
+    /// Removes the specified range from the vector in bulk, returning all
+    /// removed elements as an iterator. If the iterator is dropped before
+    /// being fully consumed, it drops the remaining removed elements.
+    ///
+    /// The returned iterator keeps a mutable borrow on the vector to optimize
+    /// its implementation.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the starting point is greater than the end point or if
+    /// the end point is greater than the length of the vector.
+    ///
+    /// # Leaking
+    ///
+    /// If the returned iterator goes out of scope without being dropped (due to
+    /// [`mem::forget`], for example), the vector may have lost and leaked
+    /// elements arbitrarily, including elements outside the range.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = vec![1, 2, 3];
+    /// let u: Vec<_> = v.drain(1..).collect();
+    /// assert_eq!(v, &[1]);
+    /// assert_eq!(u, &[2, 3]);
+    ///
+    /// // A full range clears the vector, like `clear()` does
+    /// v.drain(..);
+    /// assert_eq!(v, &[]);
+    /// ```
+    #[stable(feature = "drain", since = "1.6.0")]
+    pub fn drain<R>(&mut self, range: R) -> Drain<'_, T, A>
+    where
+        R: RangeBounds<usize>,
+    {
+        // Memory safety
+        //
+        // When the Drain is first created, it shortens the length of
+        // the source vector to make sure no uninitialized or moved-from elements
+        // are accessible at all if the Drain's destructor never gets to run.
+        //
+        // Drain will ptr::read out the values to remove.
+        // When finished, remaining tail of the vec is copied back to cover
+        // the hole, and the vector length is restored to the new length.
+        //
+        let len = self.len();
+        let Range { start, end } = slice::range(range, ..len);
+
+        unsafe {
+            // set self.vec length's to start, to be safe in case Drain is leaked
+            self.set_len(start);
+            // Use the borrow in the IterMut to indicate borrowing behavior of the
+            // whole Drain iterator (like &mut T).
+            let range_slice = slice::from_raw_parts_mut(self.as_mut_ptr().add(start), end - start);
+            Drain {
+                tail_start: end,
+                tail_len: len - end,
+                iter: range_slice.iter(),
+                vec: NonNull::from(self),
+            }
+        }
+    }
+
+    /// Clears the vector, removing all values.
+    ///
+    /// Note that this method has no effect on the allocated capacity
+    /// of the vector.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = vec![1, 2, 3];
+    ///
+    /// v.clear();
+    ///
+    /// assert!(v.is_empty());
+    /// ```
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn clear(&mut self) {
+        let elems: *mut [T] = self.as_mut_slice();
+
+        // SAFETY:
+        // - `elems` comes directly from `as_mut_slice` and is therefore valid.
+        // - Setting `self.len` before calling `drop_in_place` means that,
+        //   if an element's `Drop` impl panics, the vector's `Drop` impl will
+        //   do nothing (leaking the rest of the elements) instead of dropping
+        //   some twice.
+        unsafe {
+            self.len = 0;
+            ptr::drop_in_place(elems);
+        }
+    }
+
+    /// Returns the number of elements in the vector, also referred to
+    /// as its 'length'.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let a = vec![1, 2, 3];
+    /// assert_eq!(a.len(), 3);
+    /// ```
+    #[inline]
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    /// Returns `true` if the vector contains no elements.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = Vec::new();
+    /// assert!(v.is_empty());
+    ///
+    /// v.push(1);
+    /// assert!(!v.is_empty());
+    /// ```
+    #[stable(feature = "rust1", since = "1.0.0")]
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Splits the collection into two at the given index.
+    ///
+    /// Returns a newly allocated vector containing the elements in the range
+    /// `[at, len)`. After the call, the original vector will be left containing
+    /// the elements `[0, at)` with its previous capacity unchanged.
+    ///
+    /// # Panics
+    ///
+    /// Panics if `at > len`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3];
+    /// let vec2 = vec.split_off(1);
+    /// assert_eq!(vec, [1]);
+    /// assert_eq!(vec2, [2, 3]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    #[must_use = "use `.truncate()` if you don't need the other half"]
+    #[stable(feature = "split_off", since = "1.4.0")]
+    pub fn split_off(&mut self, at: usize) -> Self
+    where
+        A: Clone,
+    {
+        #[cold]
+        #[inline(never)]
+        fn assert_failed(at: usize, len: usize) -> ! {
+            panic!("`at` split index (is {at}) should be <= len (is {len})");
+        }
+
+        if at > self.len() {
+            assert_failed(at, self.len());
+        }
+
+        if at == 0 {
+            // the new vector can take over the original buffer and avoid the copy
+            return mem::replace(
+                self,
+                Vec::with_capacity_in(self.capacity(), self.allocator().clone()),
+            );
+        }
+
+        let other_len = self.len - at;
+        let mut other = Vec::with_capacity_in(other_len, self.allocator().clone());
+
+        // Unsafely `set_len` and copy items to `other`.
+        unsafe {
+            self.set_len(at);
+            other.set_len(other_len);
+
+            ptr::copy_nonoverlapping(self.as_ptr().add(at), other.as_mut_ptr(), other.len());
+        }
+        other
+    }
+
+    /// Resizes the `Vec` in-place so that `len` is equal to `new_len`.
+    ///
+    /// If `new_len` is greater than `len`, the `Vec` is extended by the
+    /// difference, with each additional slot filled with the result of
+    /// calling the closure `f`. The return values from `f` will end up
+    /// in the `Vec` in the order they have been generated.
+    ///
+    /// If `new_len` is less than `len`, the `Vec` is simply truncated.
+    ///
+    /// This method uses a closure to create new values on every push. If
+    /// you'd rather [`Clone`] a given value, use [`Vec::resize`]. If you
+    /// want to use the [`Default`] trait to generate values, you can
+    /// pass [`Default::default`] as the second argument.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 3];
+    /// vec.resize_with(5, Default::default);
+    /// assert_eq!(vec, [1, 2, 3, 0, 0]);
+    ///
+    /// let mut vec = vec![];
+    /// let mut p = 1;
+    /// vec.resize_with(4, || { p *= 2; p });
+    /// assert_eq!(vec, [2, 4, 8, 16]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "vec_resize_with", since = "1.33.0")]
+    pub fn resize_with<F>(&mut self, new_len: usize, f: F)
+    where
+        F: FnMut() -> T,
+    {
+        let len = self.len();
+        if new_len > len {
+            self.extend_with(new_len - len, ExtendFunc(f));
+        } else {
+            self.truncate(new_len);
+        }
+    }
+
+    /// Consumes and leaks the `Vec`, returning a mutable reference to the contents,
+    /// `&'a mut [T]`. Note that the type `T` must outlive the chosen lifetime
+    /// `'a`. If the type has only static references, or none at all, then this
+    /// may be chosen to be `'static`.
+    ///
+    /// As of Rust 1.57, this method does not reallocate or shrink the `Vec`,
+    /// so the leaked allocation may include unused capacity that is not part
+    /// of the returned slice.
+    ///
+    /// This function is mainly useful for data that lives for the remainder of
+    /// the program's life. Dropping the returned reference will cause a memory
+    /// leak.
+    ///
+    /// # Examples
+    ///
+    /// Simple usage:
+    ///
+    /// ```
+    /// let x = vec![1, 2, 3];
+    /// let static_ref: &'static mut [usize] = x.leak();
+    /// static_ref[0] += 1;
+    /// assert_eq!(static_ref, &[2, 2, 3]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "vec_leak", since = "1.47.0")]
+    #[inline]
+    pub fn leak<'a>(self) -> &'a mut [T]
+    where
+        A: 'a,
+    {
+        let mut me = ManuallyDrop::new(self);
+        unsafe { slice::from_raw_parts_mut(me.as_mut_ptr(), me.len) }
+    }
+
+    /// Returns the remaining spare capacity of the vector as a slice of
+    /// `MaybeUninit<T>`.
+    ///
+    /// The returned slice can be used to fill the vector with data (e.g. by
+    /// reading from a file) before marking the data as initialized using the
+    /// [`set_len`] method.
+    ///
+    /// [`set_len`]: Vec::set_len
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// // Allocate vector big enough for 10 elements.
+    /// let mut v = Vec::with_capacity(10);
+    ///
+    /// // Fill in the first 3 elements.
+    /// let uninit = v.spare_capacity_mut();
+    /// uninit[0].write(0);
+    /// uninit[1].write(1);
+    /// uninit[2].write(2);
+    ///
+    /// // Mark the first 3 elements of the vector as being initialized.
+    /// unsafe {
+    ///     v.set_len(3);
+    /// }
+    ///
+    /// assert_eq!(&v, &[0, 1, 2]);
+    /// ```
+    #[stable(feature = "vec_spare_capacity", since = "1.60.0")]
+    #[inline]
+    pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit<T>] {
+        // Note:
+        // This method is not implemented in terms of `split_at_spare_mut`,
+        // to prevent invalidation of pointers to the buffer.
+        unsafe {
+            slice::from_raw_parts_mut(
+                self.as_mut_ptr().add(self.len) as *mut MaybeUninit<T>,
+                self.buf.capacity() - self.len,
+            )
+        }
+    }
+
+    /// Returns vector content as a slice of `T`, along with the remaining spare
+    /// capacity of the vector as a slice of `MaybeUninit<T>`.
+    ///
+    /// The returned spare capacity slice can be used to fill the vector with data
+    /// (e.g. by reading from a file) before marking the data as initialized using
+    /// the [`set_len`] method.
+    ///
+    /// [`set_len`]: Vec::set_len
+    ///
+    /// Note that this is a low-level API, which should be used with care for
+    /// optimization purposes. If you need to append data to a `Vec`
+    /// you can use [`push`], [`extend`], [`extend_from_slice`],
+    /// [`extend_from_within`], [`insert`], [`append`], [`resize`] or
+    /// [`resize_with`], depending on your exact needs.
+    ///
+    /// [`push`]: Vec::push
+    /// [`extend`]: Vec::extend
+    /// [`extend_from_slice`]: Vec::extend_from_slice
+    /// [`extend_from_within`]: Vec::extend_from_within
+    /// [`insert`]: Vec::insert
+    /// [`append`]: Vec::append
+    /// [`resize`]: Vec::resize
+    /// [`resize_with`]: Vec::resize_with
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(vec_split_at_spare)]
+    ///
+    /// let mut v = vec![1, 1, 2];
+    ///
+    /// // Reserve additional space big enough for 10 elements.
+    /// v.reserve(10);
+    ///
+    /// let (init, uninit) = v.split_at_spare_mut();
+    /// let sum = init.iter().copied().sum::<u32>();
+    ///
+    /// // Fill in the next 4 elements.
+    /// uninit[0].write(sum);
+    /// uninit[1].write(sum * 2);
+    /// uninit[2].write(sum * 3);
+    /// uninit[3].write(sum * 4);
+    ///
+    /// // Mark the 4 elements of the vector as being initialized.
+    /// unsafe {
+    ///     let len = v.len();
+    ///     v.set_len(len + 4);
+    /// }
+    ///
+    /// assert_eq!(&v, &[1, 1, 2, 4, 8, 12, 16]);
+    /// ```
+    #[unstable(feature = "vec_split_at_spare", issue = "81944")]
+    #[inline]
+    pub fn split_at_spare_mut(&mut self) -> (&mut [T], &mut [MaybeUninit<T>]) {
+        // SAFETY:
+        // - len is ignored and so never changed
+        let (init, spare, _) = unsafe { self.split_at_spare_mut_with_len() };
+        (init, spare)
+    }
+
+    /// Safety: changing returned .2 (&mut usize) is considered the same as calling `.set_len(_)`.
+    ///
+    /// This method provides unique access to all vec parts at once in `extend_from_within`.
+    unsafe fn split_at_spare_mut_with_len(
+        &mut self,
+    ) -> (&mut [T], &mut [MaybeUninit<T>], &mut usize) {
+        let ptr = self.as_mut_ptr();
+        // SAFETY:
+        // - `ptr` is guaranteed to be valid for `self.len` elements
+        // - but the allocation extends out to `self.buf.capacity()` elements, possibly
+        // uninitialized
+        let spare_ptr = unsafe { ptr.add(self.len) };
+        let spare_ptr = spare_ptr.cast::<MaybeUninit<T>>();
+        let spare_len = self.buf.capacity() - self.len;
+
+        // SAFETY:
+        // - `ptr` is guaranteed to be valid for `self.len` elements
+        // - `spare_ptr` is pointing one element past the buffer, so it doesn't overlap with `initialized`
+        unsafe {
+            let initialized = slice::from_raw_parts_mut(ptr, self.len);
+            let spare = slice::from_raw_parts_mut(spare_ptr, spare_len);
+
+            (initialized, spare, &mut self.len)
+        }
+    }
+}
+
+impl<T: Clone, A: Allocator> Vec<T, A> {
+    /// Resizes the `Vec` in-place so that `len` is equal to `new_len`.
+    ///
+    /// If `new_len` is greater than `len`, the `Vec` is extended by the
+    /// difference, with each additional slot filled with `value`.
+    /// If `new_len` is less than `len`, the `Vec` is simply truncated.
+    ///
+    /// This method requires `T` to implement [`Clone`],
+    /// in order to be able to clone the passed value.
+    /// If you need more flexibility (or want to rely on [`Default`] instead of
+    /// [`Clone`]), use [`Vec::resize_with`].
+    /// If you only need to resize to a smaller size, use [`Vec::truncate`].
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec!["hello"];
+    /// vec.resize(3, "world");
+    /// assert_eq!(vec, ["hello", "world", "world"]);
+    ///
+    /// let mut vec = vec![1, 2, 3, 4];
+    /// vec.resize(2, 0);
+    /// assert_eq!(vec, [1, 2]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "vec_resize", since = "1.5.0")]
+    pub fn resize(&mut self, new_len: usize, value: T) {
+        let len = self.len();
+
+        if new_len > len {
+            self.extend_with(new_len - len, ExtendElement(value))
+        } else {
+            self.truncate(new_len);
+        }
+    }
+
+    /// Clones and appends all elements in a slice to the `Vec`.
+    ///
+    /// Iterates over the slice `other`, clones each element, and then appends
+    /// it to this `Vec`. The `other` slice is traversed in-order.
+    ///
+    /// Note that this function is same as [`extend`] except that it is
+    /// specialized to work with slices instead. If and when Rust gets
+    /// specialization this function will likely be deprecated (but still
+    /// available).
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1];
+    /// vec.extend_from_slice(&[2, 3, 4]);
+    /// assert_eq!(vec, [1, 2, 3, 4]);
+    /// ```
+    ///
+    /// [`extend`]: Vec::extend
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "vec_extend_from_slice", since = "1.6.0")]
+    pub fn extend_from_slice(&mut self, other: &[T]) {
+        self.spec_extend(other.iter())
+    }
+
+    /// Copies elements from `src` range to the end of the vector.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the starting point is greater than the end point or if
+    /// the end point is greater than the length of the vector.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![0, 1, 2, 3, 4];
+    ///
+    /// vec.extend_from_within(2..);
+    /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4]);
+    ///
+    /// vec.extend_from_within(..2);
+    /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4, 0, 1]);
+    ///
+    /// vec.extend_from_within(4..8);
+    /// assert_eq!(vec, [0, 1, 2, 3, 4, 2, 3, 4, 0, 1, 4, 2, 3, 4]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[stable(feature = "vec_extend_from_within", since = "1.53.0")]
+    pub fn extend_from_within<R>(&mut self, src: R)
+    where
+        R: RangeBounds<usize>,
+    {
+        let range = slice::range(src, ..self.len());
+        self.reserve(range.len());
+
+        // SAFETY:
+        // - `slice::range` guarantees  that the given range is valid for indexing self
+        unsafe {
+            self.spec_extend_from_within(range);
+        }
+    }
+}
+
+impl<T, A: Allocator, const N: usize> Vec<[T; N], A> {
+    /// Takes a `Vec<[T; N]>` and flattens it into a `Vec<T>`.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the length of the resulting vector would overflow a `usize`.
+    ///
+    /// This is only possible when flattening a vector of arrays of zero-sized
+    /// types, and thus tends to be irrelevant in practice. If
+    /// `size_of::<T>() > 0`, this will never panic.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// #![feature(slice_flatten)]
+    ///
+    /// let mut vec = vec![[1, 2, 3], [4, 5, 6], [7, 8, 9]];
+    /// assert_eq!(vec.pop(), Some([7, 8, 9]));
+    ///
+    /// let mut flattened = vec.into_flattened();
+    /// assert_eq!(flattened.pop(), Some(6));
+    /// ```
+    #[unstable(feature = "slice_flatten", issue = "95629")]
+    pub fn into_flattened(self) -> Vec<T, A> {
+        let (ptr, len, cap, alloc) = self.into_raw_parts_with_alloc();
+        let (new_len, new_cap) = if mem::size_of::<T>() == 0 {
+            (len.checked_mul(N).expect("vec len overflow"), usize::MAX)
+        } else {
+            // SAFETY:
+            // - `cap * N` cannot overflow because the allocation is already in
+            // the address space.
+            // - Each `[T; N]` has `N` valid elements, so there are `len * N`
+            // valid elements in the allocation.
+            unsafe { (len.unchecked_mul(N), cap.unchecked_mul(N)) }
+        };
+        // SAFETY:
+        // - `ptr` was allocated by `self`
+        // - `ptr` is well-aligned because `[T; N]` has the same alignment as `T`.
+        // - `new_cap` refers to the same sized allocation as `cap` because
+        // `new_cap * size_of::<T>()` == `cap * size_of::<[T; N]>()`
+        // - `len` <= `cap`, so `len * N` <= `cap * N`.
+        unsafe { Vec::<T, A>::from_raw_parts_in(ptr.cast(), new_len, new_cap, alloc) }
+    }
+}
+
+// This code generalizes `extend_with_{element,default}`.
+trait ExtendWith<T> {
+    fn next(&mut self) -> T;
+    fn last(self) -> T;
+}
+
+struct ExtendElement<T>(T);
+impl<T: Clone> ExtendWith<T> for ExtendElement<T> {
+    fn next(&mut self) -> T {
+        self.0.clone()
+    }
+    fn last(self) -> T {
+        self.0
+    }
+}
+
+struct ExtendFunc<F>(F);
+impl<T, F: FnMut() -> T> ExtendWith<T> for ExtendFunc<F> {
+    fn next(&mut self) -> T {
+        (self.0)()
+    }
+    fn last(mut self) -> T {
+        (self.0)()
+    }
+}
+
+impl<T, A: Allocator> Vec<T, A> {
+    #[cfg(not(no_global_oom_handling))]
+    /// Extend the vector by `n` values, using the given generator.
+    fn extend_with<E: ExtendWith<T>>(&mut self, n: usize, mut value: E) {
+        self.reserve(n);
+
+        unsafe {
+            let mut ptr = self.as_mut_ptr().add(self.len());
+            // Use SetLenOnDrop to work around bug where compiler
+            // might not realize the store through `ptr` through self.set_len()
+            // don't alias.
+            let mut local_len = SetLenOnDrop::new(&mut self.len);
+
+            // Write all elements except the last one
+            for _ in 1..n {
+                ptr::write(ptr, value.next());
+                ptr = ptr.offset(1);
+                // Increment the length in every step in case next() panics
+                local_len.increment_len(1);
+            }
+
+            if n > 0 {
+                // We can write the last element directly without cloning needlessly
+                ptr::write(ptr, value.last());
+                local_len.increment_len(1);
+            }
+
+            // len set by scope guard
+        }
+    }
+}
+
+impl<T: PartialEq, A: Allocator> Vec<T, A> {
+    /// Removes consecutive repeated elements in the vector according to the
+    /// [`PartialEq`] trait implementation.
+    ///
+    /// If the vector is sorted, this removes all duplicates.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut vec = vec![1, 2, 2, 3, 2];
+    ///
+    /// vec.dedup();
+    ///
+    /// assert_eq!(vec, [1, 2, 3, 2]);
+    /// ```
+    #[stable(feature = "rust1", since = "1.0.0")]
+    #[inline]
+    pub fn dedup(&mut self) {
+        self.dedup_by(|a, b| a == b)
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Internal methods and functions
+////////////////////////////////////////////////////////////////////////////////
+
+#[doc(hidden)]
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+pub fn from_elem<T: Clone>(elem: T, n: usize) -> Vec<T> {
+    <T as SpecFromElem>::from_elem(elem, n, Global)
+}
+
+#[doc(hidden)]
+#[cfg(not(no_global_oom_handling))]
+#[unstable(feature = "allocator_api", issue = "32838")]
+pub fn from_elem_in<T: Clone, A: Allocator>(elem: T, n: usize, alloc: A) -> Vec<T, A> {
+    <T as SpecFromElem>::from_elem(elem, n, alloc)
+}
+
+trait ExtendFromWithinSpec {
+    /// # Safety
+    ///
+    /// - `src` needs to be valid index
+    /// - `self.capacity() - self.len()` must be `>= src.len()`
+    unsafe fn spec_extend_from_within(&mut self, src: Range<usize>);
+}
+
+impl<T: Clone, A: Allocator> ExtendFromWithinSpec for Vec<T, A> {
+    default unsafe fn spec_extend_from_within(&mut self, src: Range<usize>) {
+        // SAFETY:
+        // - len is increased only after initializing elements
+        let (this, spare, len) = unsafe { self.split_at_spare_mut_with_len() };
+
+        // SAFETY:
+        // - caller guaratees that src is a valid index
+        let to_clone = unsafe { this.get_unchecked(src) };
+
+        iter::zip(to_clone, spare)
+            .map(|(src, dst)| dst.write(src.clone()))
+            // Note:
+            // - Element was just initialized with `MaybeUninit::write`, so it's ok to increase len
+            // - len is increased after each element to prevent leaks (see issue #82533)
+            .for_each(|_| *len += 1);
+    }
+}
+
+impl<T: Copy, A: Allocator> ExtendFromWithinSpec for Vec<T, A> {
+    unsafe fn spec_extend_from_within(&mut self, src: Range<usize>) {
+        let count = src.len();
+        {
+            let (init, spare) = self.split_at_spare_mut();
+
+            // SAFETY:
+            // - caller guaratees that `src` is a valid index
+            let source = unsafe { init.get_unchecked(src) };
+
+            // SAFETY:
+            // - Both pointers are created from unique slice references (`&mut [_]`)
+            //   so they are valid and do not overlap.
+            // - Elements are :Copy so it's OK to to copy them, without doing
+            //   anything with the original values
+            // - `count` is equal to the len of `source`, so source is valid for
+            //   `count` reads
+            // - `.reserve(count)` guarantees that `spare.len() >= count` so spare
+            //   is valid for `count` writes
+            unsafe { ptr::copy_nonoverlapping(source.as_ptr(), spare.as_mut_ptr() as _, count) };
+        }
+
+        // SAFETY:
+        // - The elements were just initialized by `copy_nonoverlapping`
+        self.len += count;
+    }
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Common trait implementations for Vec
+////////////////////////////////////////////////////////////////////////////////
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T, A: Allocator> ops::Deref for Vec<T, A> {
+    type Target = [T];
+
+    fn deref(&self) -> &[T] {
+        unsafe { slice::from_raw_parts(self.as_ptr(), self.len) }
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T, A: Allocator> ops::DerefMut for Vec<T, A> {
+    fn deref_mut(&mut self) -> &mut [T] {
+        unsafe { slice::from_raw_parts_mut(self.as_mut_ptr(), self.len) }
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+trait SpecCloneFrom {
+    fn clone_from(this: &mut Self, other: &Self);
+}
+
+#[cfg(not(no_global_oom_handling))]
+impl<T: Clone, A: Allocator> SpecCloneFrom for Vec<T, A> {
+    default fn clone_from(this: &mut Self, other: &Self) {
+        // drop anything that will not be overwritten
+        this.truncate(other.len());
+
+        // self.len <= other.len due to the truncate above, so the
+        // slices here are always in-bounds.
+        let (init, tail) = other.split_at(this.len());
+
+        // reuse the contained values' allocations/resources.
+        this.clone_from_slice(init);
+        this.extend_from_slice(tail);
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+impl<T: Copy, A: Allocator> SpecCloneFrom for Vec<T, A> {
+    fn clone_from(this: &mut Self, other: &Self) {
+        this.clear();
+        this.extend_from_slice(other);
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: Clone, A: Allocator + Clone> Clone for Vec<T, A> {
+    #[cfg(not(test))]
+    fn clone(&self) -> Self {
+        let alloc = self.allocator().clone();
+        <[T]>::to_vec_in(&**self, alloc)
+    }
+
+    // HACK(japaric): with cfg(test) the inherent `[T]::to_vec` method, which is
+    // required for this method definition, is not available. Instead use the
+    // `slice::to_vec`  function which is only available with cfg(test)
+    // NB see the slice::hack module in slice.rs for more information
+    #[cfg(test)]
+    fn clone(&self) -> Self {
+        let alloc = self.allocator().clone();
+        crate::slice::to_vec(&**self, alloc)
+    }
+
+    fn clone_from(&mut self, other: &Self) {
+        SpecCloneFrom::clone_from(self, other)
+    }
+}
+
+/// The hash of a vector is the same as that of the corresponding slice,
+/// as required by the `core::borrow::Borrow` implementation.
+///
+/// ```
+/// #![feature(build_hasher_simple_hash_one)]
+/// use std::hash::BuildHasher;
+///
+/// let b = std::collections::hash_map::RandomState::new();
+/// let v: Vec<u8> = vec![0xa8, 0x3c, 0x09];
+/// let s: &[u8] = &[0xa8, 0x3c, 0x09];
+/// assert_eq!(b.hash_one(v), b.hash_one(s));
+/// ```
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: Hash, A: Allocator> Hash for Vec<T, A> {
+    #[inline]
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        Hash::hash(&**self, state)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+#[rustc_on_unimplemented(
+    message = "vector indices are of type `usize` or ranges of `usize`",
+    label = "vector indices are of type `usize` or ranges of `usize`"
+)]
+impl<T, I: SliceIndex<[T]>, A: Allocator> Index<I> for Vec<T, A> {
+    type Output = I::Output;
+
+    #[inline]
+    fn index(&self, index: I) -> &Self::Output {
+        Index::index(&**self, index)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+#[rustc_on_unimplemented(
+    message = "vector indices are of type `usize` or ranges of `usize`",
+    label = "vector indices are of type `usize` or ranges of `usize`"
+)]
+impl<T, I: SliceIndex<[T]>, A: Allocator> IndexMut<I> for Vec<T, A> {
+    #[inline]
+    fn index_mut(&mut self, index: I) -> &mut Self::Output {
+        IndexMut::index_mut(&mut **self, index)
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T> FromIterator<T> for Vec<T> {
+    #[inline]
+    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Vec<T> {
+        <Self as SpecFromIter<T, I::IntoIter>>::from_iter(iter.into_iter())
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T, A: Allocator> IntoIterator for Vec<T, A> {
+    type Item = T;
+    type IntoIter = IntoIter<T, A>;
+
+    /// Creates a consuming iterator, that is, one that moves each value out of
+    /// the vector (from start to end). The vector cannot be used after calling
+    /// this.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let v = vec!["a".to_string(), "b".to_string()];
+    /// for s in v.into_iter() {
+    ///     // s has type String, not &String
+    ///     println!("{s}");
+    /// }
+    /// ```
+    #[inline]
+    fn into_iter(self) -> IntoIter<T, A> {
+        unsafe {
+            let mut me = ManuallyDrop::new(self);
+            let alloc = ManuallyDrop::new(ptr::read(me.allocator()));
+            let begin = me.as_mut_ptr();
+            let end = if mem::size_of::<T>() == 0 {
+                arith_offset(begin as *const i8, me.len() as isize) as *const T
+            } else {
+                begin.add(me.len()) as *const T
+            };
+            let cap = me.buf.capacity();
+            IntoIter {
+                buf: NonNull::new_unchecked(begin),
+                phantom: PhantomData,
+                cap,
+                alloc,
+                ptr: begin,
+                end,
+            }
+        }
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<'a, T, A: Allocator> IntoIterator for &'a Vec<T, A> {
+    type Item = &'a T;
+    type IntoIter = slice::Iter<'a, T>;
+
+    fn into_iter(self) -> slice::Iter<'a, T> {
+        self.iter()
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<'a, T, A: Allocator> IntoIterator for &'a mut Vec<T, A> {
+    type Item = &'a mut T;
+    type IntoIter = slice::IterMut<'a, T>;
+
+    fn into_iter(self) -> slice::IterMut<'a, T> {
+        self.iter_mut()
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T, A: Allocator> Extend<T> for Vec<T, A> {
+    #[inline]
+    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
+        <Self as SpecExtend<T, I::IntoIter>>::spec_extend(self, iter.into_iter())
+    }
+
+    #[inline]
+    fn extend_one(&mut self, item: T) {
+        self.push(item);
+    }
+
+    #[inline]
+    fn extend_reserve(&mut self, additional: usize) {
+        self.reserve(additional);
+    }
+}
+
+impl<T, A: Allocator> Vec<T, A> {
+    // leaf method to which various SpecFrom/SpecExtend implementations delegate when
+    // they have no further optimizations to apply
+    #[cfg(not(no_global_oom_handling))]
+    fn extend_desugared<I: Iterator<Item = T>>(&mut self, mut iterator: I) {
+        // This is the case for a general iterator.
+        //
+        // This function should be the moral equivalent of:
+        //
+        //      for item in iterator {
+        //          self.push(item);
+        //      }
+        while let Some(element) = iterator.next() {
+            let len = self.len();
+            if len == self.capacity() {
+                let (lower, _) = iterator.size_hint();
+                self.reserve(lower.saturating_add(1));
+            }
+            unsafe {
+                ptr::write(self.as_mut_ptr().add(len), element);
+                // Since next() executes user code which can panic we have to bump the length
+                // after each step.
+                // NB can't overflow since we would have had to alloc the address space
+                self.set_len(len + 1);
+            }
+        }
+    }
+
+    /// Creates a splicing iterator that replaces the specified range in the vector
+    /// with the given `replace_with` iterator and yields the removed items.
+    /// `replace_with` does not need to be the same length as `range`.
+    ///
+    /// `range` is removed even if the iterator is not consumed until the end.
+    ///
+    /// It is unspecified how many elements are removed from the vector
+    /// if the `Splice` value is leaked.
+    ///
+    /// The input iterator `replace_with` is only consumed when the `Splice` value is dropped.
+    ///
+    /// This is optimal if:
+    ///
+    /// * The tail (elements in the vector after `range`) is empty,
+    /// * or `replace_with` yields fewer or equal elements than `range`’s length
+    /// * or the lower bound of its `size_hint()` is exact.
+    ///
+    /// Otherwise, a temporary vector is allocated and the tail is moved twice.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the starting point is greater than the end point or if
+    /// the end point is greater than the length of the vector.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = vec![1, 2, 3, 4];
+    /// let new = [7, 8, 9];
+    /// let u: Vec<_> = v.splice(1..3, new).collect();
+    /// assert_eq!(v, &[1, 7, 8, 9, 4]);
+    /// assert_eq!(u, &[2, 3]);
+    /// ```
+    #[cfg(not(no_global_oom_handling))]
+    #[inline]
+    #[stable(feature = "vec_splice", since = "1.21.0")]
+    pub fn splice<R, I>(&mut self, range: R, replace_with: I) -> Splice<'_, I::IntoIter, A>
+    where
+        R: RangeBounds<usize>,
+        I: IntoIterator<Item = T>,
+    {
+        Splice { drain: self.drain(range), replace_with: replace_with.into_iter() }
+    }
+
+    /// Creates an iterator which uses a closure to determine if an element should be removed.
+    ///
+    /// If the closure returns true, then the element is removed and yielded.
+    /// If the closure returns false, the element will remain in the vector and will not be yielded
+    /// by the iterator.
+    ///
+    /// Using this method is equivalent to the following code:
+    ///
+    /// ```
+    /// # let some_predicate = |x: &mut i32| { *x == 2 || *x == 3 || *x == 6 };
+    /// # let mut vec = vec![1, 2, 3, 4, 5, 6];
+    /// let mut i = 0;
+    /// while i < vec.len() {
+    ///     if some_predicate(&mut vec[i]) {
+    ///         let val = vec.remove(i);
+    ///         // your code here
+    ///     } else {
+    ///         i += 1;
+    ///     }
+    /// }
+    ///
+    /// # assert_eq!(vec, vec![1, 4, 5]);
+    /// ```
+    ///
+    /// But `drain_filter` is easier to use. `drain_filter` is also more efficient,
+    /// because it can backshift the elements of the array in bulk.
+    ///
+    /// Note that `drain_filter` also lets you mutate every element in the filter closure,
+    /// regardless of whether you choose to keep or remove it.
+    ///
+    /// # Examples
+    ///
+    /// Splitting an array into evens and odds, reusing the original allocation:
+    ///
+    /// ```
+    /// #![feature(drain_filter)]
+    /// let mut numbers = vec![1, 2, 3, 4, 5, 6, 8, 9, 11, 13, 14, 15];
+    ///
+    /// let evens = numbers.drain_filter(|x| *x % 2 == 0).collect::<Vec<_>>();
+    /// let odds = numbers;
+    ///
+    /// assert_eq!(evens, vec![2, 4, 6, 8, 14]);
+    /// assert_eq!(odds, vec![1, 3, 5, 9, 11, 13, 15]);
+    /// ```
+    #[unstable(feature = "drain_filter", reason = "recently added", issue = "43244")]
+    pub fn drain_filter<F>(&mut self, filter: F) -> DrainFilter<'_, T, F, A>
+    where
+        F: FnMut(&mut T) -> bool,
+    {
+        let old_len = self.len();
+
+        // Guard against us getting leaked (leak amplification)
+        unsafe {
+            self.set_len(0);
+        }
+
+        DrainFilter { vec: self, idx: 0, del: 0, old_len, pred: filter, panic_flag: false }
+    }
+}
+
+/// Extend implementation that copies elements out of references before pushing them onto the Vec.
+///
+/// This implementation is specialized for slice iterators, where it uses [`copy_from_slice`] to
+/// append the entire slice at once.
+///
+/// [`copy_from_slice`]: slice::copy_from_slice
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "extend_ref", since = "1.2.0")]
+impl<'a, T: Copy + 'a, A: Allocator + 'a> Extend<&'a T> for Vec<T, A> {
+    fn extend<I: IntoIterator<Item = &'a T>>(&mut self, iter: I) {
+        self.spec_extend(iter.into_iter())
+    }
+
+    #[inline]
+    fn extend_one(&mut self, &item: &'a T) {
+        self.push(item);
+    }
+
+    #[inline]
+    fn extend_reserve(&mut self, additional: usize) {
+        self.reserve(additional);
+    }
+}
+
+/// Implements comparison of vectors, [lexicographically](core::cmp::Ord#lexicographical-comparison).
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: PartialOrd, A: Allocator> PartialOrd for Vec<T, A> {
+    #[inline]
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        PartialOrd::partial_cmp(&**self, &**other)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: Eq, A: Allocator> Eq for Vec<T, A> {}
+
+/// Implements ordering of vectors, [lexicographically](core::cmp::Ord#lexicographical-comparison).
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: Ord, A: Allocator> Ord for Vec<T, A> {
+    #[inline]
+    fn cmp(&self, other: &Self) -> Ordering {
+        Ord::cmp(&**self, &**other)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+unsafe impl<#[may_dangle] T, A: Allocator> Drop for Vec<T, A> {
+    fn drop(&mut self) {
+        unsafe {
+            // use drop for [T]
+            // use a raw slice to refer to the elements of the vector as weakest necessary type;
+            // could avoid questions of validity in certain cases
+            ptr::drop_in_place(ptr::slice_from_raw_parts_mut(self.as_mut_ptr(), self.len))
+        }
+        // RawVec handles deallocation
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+#[rustc_const_unstable(feature = "const_default_impls", issue = "87864")]
+impl<T> const Default for Vec<T> {
+    /// Creates an empty `Vec<T>`.
+    fn default() -> Vec<T> {
+        Vec::new()
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: fmt::Debug, A: Allocator> fmt::Debug for Vec<T, A> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(&**self, f)
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T, A: Allocator> AsRef<Vec<T, A>> for Vec<T, A> {
+    fn as_ref(&self) -> &Vec<T, A> {
+        self
+    }
+}
+
+#[stable(feature = "vec_as_mut", since = "1.5.0")]
+impl<T, A: Allocator> AsMut<Vec<T, A>> for Vec<T, A> {
+    fn as_mut(&mut self) -> &mut Vec<T, A> {
+        self
+    }
+}
+
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T, A: Allocator> AsRef<[T]> for Vec<T, A> {
+    fn as_ref(&self) -> &[T] {
+        self
+    }
+}
+
+#[stable(feature = "vec_as_mut", since = "1.5.0")]
+impl<T, A: Allocator> AsMut<[T]> for Vec<T, A> {
+    fn as_mut(&mut self) -> &mut [T] {
+        self
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+impl<T: Clone> From<&[T]> for Vec<T> {
+    /// Allocate a `Vec<T>` and fill it by cloning `s`'s items.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// assert_eq!(Vec::from(&[1, 2, 3][..]), vec![1, 2, 3]);
+    /// ```
+    #[cfg(not(test))]
+    fn from(s: &[T]) -> Vec<T> {
+        s.to_vec()
+    }
+    #[cfg(test)]
+    fn from(s: &[T]) -> Vec<T> {
+        crate::slice::to_vec(s, Global)
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "vec_from_mut", since = "1.19.0")]
+impl<T: Clone> From<&mut [T]> for Vec<T> {
+    /// Allocate a `Vec<T>` and fill it by cloning `s`'s items.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// assert_eq!(Vec::from(&mut [1, 2, 3][..]), vec![1, 2, 3]);
+    /// ```
+    #[cfg(not(test))]
+    fn from(s: &mut [T]) -> Vec<T> {
+        s.to_vec()
+    }
+    #[cfg(test)]
+    fn from(s: &mut [T]) -> Vec<T> {
+        crate::slice::to_vec(s, Global)
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "vec_from_array", since = "1.44.0")]
+impl<T, const N: usize> From<[T; N]> for Vec<T> {
+    /// Allocate a `Vec<T>` and move `s`'s items into it.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// assert_eq!(Vec::from([1, 2, 3]), vec![1, 2, 3]);
+    /// ```
+    #[cfg(not(test))]
+    fn from(s: [T; N]) -> Vec<T> {
+        <[T]>::into_vec(box s)
+    }
+
+    #[cfg(test)]
+    fn from(s: [T; N]) -> Vec<T> {
+        crate::slice::into_vec(box s)
+    }
+}
+
+#[stable(feature = "vec_from_cow_slice", since = "1.14.0")]
+impl<'a, T> From<Cow<'a, [T]>> for Vec<T>
+where
+    [T]: ToOwned<Owned = Vec<T>>,
+{
+    /// Convert a clone-on-write slice into a vector.
+    ///
+    /// If `s` already owns a `Vec<T>`, it will be returned directly.
+    /// If `s` is borrowing a slice, a new `Vec<T>` will be allocated and
+    /// filled by cloning `s`'s items into it.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// # use std::borrow::Cow;
+    /// let o: Cow<[i32]> = Cow::Owned(vec![1, 2, 3]);
+    /// let b: Cow<[i32]> = Cow::Borrowed(&[1, 2, 3]);
+    /// assert_eq!(Vec::from(o), Vec::from(b));
+    /// ```
+    fn from(s: Cow<'a, [T]>) -> Vec<T> {
+        s.into_owned()
+    }
+}
+
+// note: test pulls in libstd, which causes errors here
+#[cfg(not(test))]
+#[stable(feature = "vec_from_box", since = "1.18.0")]
+impl<T, A: Allocator> From<Box<[T], A>> for Vec<T, A> {
+    /// Convert a boxed slice into a vector by transferring ownership of
+    /// the existing heap allocation.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let b: Box<[i32]> = vec![1, 2, 3].into_boxed_slice();
+    /// assert_eq!(Vec::from(b), vec![1, 2, 3]);
+    /// ```
+    fn from(s: Box<[T], A>) -> Self {
+        s.into_vec()
+    }
+}
+
+// note: test pulls in libstd, which causes errors here
+#[cfg(not(no_global_oom_handling))]
+#[cfg(not(test))]
+#[stable(feature = "box_from_vec", since = "1.20.0")]
+impl<T, A: Allocator> From<Vec<T, A>> for Box<[T], A> {
+    /// Convert a vector into a boxed slice.
+    ///
+    /// If `v` has excess capacity, its items will be moved into a
+    /// newly-allocated buffer with exactly the right capacity.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// assert_eq!(Box::from(vec![1, 2, 3]), vec![1, 2, 3].into_boxed_slice());
+    /// ```
+    fn from(v: Vec<T, A>) -> Self {
+        v.into_boxed_slice()
+    }
+}
+
+#[cfg(not(no_global_oom_handling))]
+#[stable(feature = "rust1", since = "1.0.0")]
+impl From<&str> for Vec<u8> {
+    /// Allocate a `Vec<u8>` and fill it with a UTF-8 string.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// assert_eq!(Vec::from("123"), vec![b'1', b'2', b'3']);
+    /// ```
+    fn from(s: &str) -> Vec<u8> {
+        From::from(s.as_bytes())
+    }
+}
+
+#[stable(feature = "array_try_from_vec", since = "1.48.0")]
+impl<T, A: Allocator, const N: usize> TryFrom<Vec<T, A>> for [T; N] {
+    type Error = Vec<T, A>;
+
+    /// Gets the entire contents of the `Vec<T>` as an array,
+    /// if its size exactly matches that of the requested array.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// assert_eq!(vec![1, 2, 3].try_into(), Ok([1, 2, 3]));
+    /// assert_eq!(<Vec<i32>>::new().try_into(), Ok([]));
+    /// ```
+    ///
+    /// If the length doesn't match, the input comes back in `Err`:
+    /// ```
+    /// let r: Result<[i32; 4], _> = (0..10).collect::<Vec<_>>().try_into();
+    /// assert_eq!(r, Err(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]));
+    /// ```
+    ///
+    /// If you're fine with just getting a prefix of the `Vec<T>`,
+    /// you can call [`.truncate(N)`](Vec::truncate) first.
+    /// ```
+    /// let mut v = String::from("hello world").into_bytes();
+    /// v.sort();
+    /// v.truncate(2);
+    /// let [a, b]: [_; 2] = v.try_into().unwrap();
+    /// assert_eq!(a, b' ');
+    /// assert_eq!(b, b'd');
+    /// ```
+    fn try_from(mut vec: Vec<T, A>) -> Result<[T; N], Vec<T, A>> {
+        if vec.len() != N {
+            return Err(vec);
+        }
+
+        // SAFETY: `.set_len(0)` is always sound.
+        unsafe { vec.set_len(0) };
+
+        // SAFETY: A `Vec`'s pointer is always aligned properly, and
+        // the alignment the array needs is the same as the items.
+        // We checked earlier that we have sufficient items.
+        // The items will not double-drop as the `set_len`
+        // tells the `Vec` not to also drop them.
+        let array = unsafe { ptr::read(vec.as_ptr() as *const [T; N]) };
+        Ok(array)
+    }
+}
diff --git a/rust/alloc/vec/partial_eq.rs b/rust/alloc/vec/partial_eq.rs
new file mode 100644
index 000000000000..10ad4e492287
--- /dev/null
+++ b/rust/alloc/vec/partial_eq.rs
@@ -0,0 +1,49 @@
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use crate::alloc::Allocator;
+#[cfg(not(no_global_oom_handling))]
+use crate::borrow::Cow;
+
+use super::Vec;
+
+macro_rules! __impl_slice_eq1 {
+    ([$($vars:tt)*] $lhs:ty, $rhs:ty $(where $ty:ty: $bound:ident)?, #[$stability:meta]) => {
+        #[$stability]
+        impl<T, U, $($vars)*> PartialEq<$rhs> for $lhs
+        where
+            T: PartialEq<U>,
+            $($ty: $bound)?
+        {
+            #[inline]
+            fn eq(&self, other: &$rhs) -> bool { self[..] == other[..] }
+            #[inline]
+            fn ne(&self, other: &$rhs) -> bool { self[..] != other[..] }
+        }
+    }
+}
+
+__impl_slice_eq1! { [A1: Allocator, A2: Allocator] Vec<T, A1>, Vec<U, A2>, #[stable(feature = "rust1", since = "1.0.0")] }
+__impl_slice_eq1! { [A: Allocator] Vec<T, A>, &[U], #[stable(feature = "rust1", since = "1.0.0")] }
+__impl_slice_eq1! { [A: Allocator] Vec<T, A>, &mut [U], #[stable(feature = "rust1", since = "1.0.0")] }
+__impl_slice_eq1! { [A: Allocator] &[T], Vec<U, A>, #[stable(feature = "partialeq_vec_for_ref_slice", since = "1.46.0")] }
+__impl_slice_eq1! { [A: Allocator] &mut [T], Vec<U, A>, #[stable(feature = "partialeq_vec_for_ref_slice", since = "1.46.0")] }
+__impl_slice_eq1! { [A: Allocator] Vec<T, A>, [U], #[stable(feature = "partialeq_vec_for_slice", since = "1.48.0")]  }
+__impl_slice_eq1! { [A: Allocator] [T], Vec<U, A>, #[stable(feature = "partialeq_vec_for_slice", since = "1.48.0")]  }
+#[cfg(not(no_global_oom_handling))]
+__impl_slice_eq1! { [A: Allocator] Cow<'_, [T]>, Vec<U, A> where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] }
+#[cfg(not(no_global_oom_handling))]
+__impl_slice_eq1! { [] Cow<'_, [T]>, &[U] where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] }
+#[cfg(not(no_global_oom_handling))]
+__impl_slice_eq1! { [] Cow<'_, [T]>, &mut [U] where T: Clone, #[stable(feature = "rust1", since = "1.0.0")] }
+__impl_slice_eq1! { [A: Allocator, const N: usize] Vec<T, A>, [U; N], #[stable(feature = "rust1", since = "1.0.0")] }
+__impl_slice_eq1! { [A: Allocator, const N: usize] Vec<T, A>, &[U; N], #[stable(feature = "rust1", since = "1.0.0")] }
+
+// NOTE: some less important impls are omitted to reduce code bloat
+// FIXME(Centril): Reconsider this?
+//__impl_slice_eq1! { [const N: usize] Vec<A>, &mut [B; N], }
+//__impl_slice_eq1! { [const N: usize] [A; N], Vec<B>, }
+//__impl_slice_eq1! { [const N: usize] &[A; N], Vec<B>, }
+//__impl_slice_eq1! { [const N: usize] &mut [A; N], Vec<B>, }
+//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, [B; N], }
+//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, &[B; N], }
+//__impl_slice_eq1! { [const N: usize] Cow<'a, [A]>, &mut [B; N], }
diff --git a/rust/bindgen_parameters b/rust/bindgen_parameters
new file mode 100644
index 000000000000..be4963bf7203
--- /dev/null
+++ b/rust/bindgen_parameters
@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: GPL-2.0
+
+--opaque-type xregs_state
+--opaque-type desc_struct
+--opaque-type arch_lbr_state
+--opaque-type local_apic
+
+# Packed type cannot transitively contain a `#[repr(align)]` type.
+--opaque-type x86_msi_data
+--opaque-type x86_msi_addr_lo
+
+# `try` is a reserved keyword since Rust 2018; solved in `bindgen` v0.59.2,
+# commit 2aed6b021680 ("context: Escape the try keyword properly").
+--opaque-type kunit_try_catch
+
+# If SMP is disabled, `arch_spinlock_t` is defined as a ZST which triggers a Rust
+# warning. We don't need to peek into it anyway.
+--opaque-type spinlock
+
+# `seccomp`'s comment gets understood as a doctest
+--no-doc-comments
diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h
new file mode 100644
index 000000000000..c48bc284214a
--- /dev/null
+++ b/rust/bindings/bindings_helper.h
@@ -0,0 +1,13 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Header that contains the code (mostly headers) for which Rust bindings
+ * will be automatically generated by `bindgen`.
+ *
+ * Sorted alphabetically.
+ */
+
+#include <linux/slab.h>
+
+/* `bindgen` gets confused at certain things. */
+const gfp_t BINDINGS_GFP_KERNEL = GFP_KERNEL;
+const gfp_t BINDINGS___GFP_ZERO = __GFP_ZERO;
diff --git a/rust/bindings/lib.rs b/rust/bindings/lib.rs
new file mode 100644
index 000000000000..6c50ee62c56b
--- /dev/null
+++ b/rust/bindings/lib.rs
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Bindings.
+//!
+//! Imports the generated bindings by `bindgen`.
+//!
+//! This crate may not be directly used. If you need a kernel C API that is
+//! not ported or wrapped in the `kernel` crate, then do so first instead of
+//! using this crate.
+
+#![no_std]
+#![feature(core_ffi_c)]
+// See <https://github.com/rust-lang/rust-bindgen/issues/1651>.
+#![cfg_attr(test, allow(deref_nullptr))]
+#![cfg_attr(test, allow(unaligned_references))]
+#![cfg_attr(test, allow(unsafe_op_in_unsafe_fn))]
+#![allow(
+    clippy::all,
+    missing_docs,
+    non_camel_case_types,
+    non_upper_case_globals,
+    non_snake_case,
+    improper_ctypes,
+    unreachable_pub,
+    unsafe_op_in_unsafe_fn
+)]
+
+mod bindings_raw {
+    // Use glob import here to expose all helpers.
+    // Symbols defined within the module will take precedence to the glob import.
+    pub use super::bindings_helper::*;
+    include!(concat!(
+        env!("OBJTREE"),
+        "/rust/bindings/bindings_generated.rs"
+    ));
+}
+
+// When both a directly exposed symbol and a helper exists for the same function,
+// the directly exposed symbol is preferred and the helper becomes dead code, so
+// ignore the warning here.
+#[allow(dead_code)]
+mod bindings_helper {
+    // Import the generated bindings for types.
+    include!(concat!(
+        env!("OBJTREE"),
+        "/rust/bindings/bindings_helpers_generated.rs"
+    ));
+}
+
+pub use bindings_raw::*;
+
+pub const GFP_KERNEL: gfp_t = BINDINGS_GFP_KERNEL;
+pub const __GFP_ZERO: gfp_t = BINDINGS___GFP_ZERO;
diff --git a/rust/compiler_builtins.rs b/rust/compiler_builtins.rs
new file mode 100644
index 000000000000..f8f39a3e6855
--- /dev/null
+++ b/rust/compiler_builtins.rs
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Our own `compiler_builtins`.
+//!
+//! Rust provides [`compiler_builtins`] as a port of LLVM's [`compiler-rt`].
+//! Since we do not need the vast majority of them, we avoid the dependency
+//! by providing this file.
+//!
+//! At the moment, some builtins are required that should not be. For instance,
+//! [`core`] has 128-bit integers functionality which we should not be compiling
+//! in. We will work with upstream [`core`] to provide feature flags to disable
+//! the parts we do not need. For the moment, we define them to [`panic!`] at
+//! runtime for simplicity to catch mistakes, instead of performing surgery
+//! on `core.o`.
+//!
+//! In any case, all these symbols are weakened to ensure we do not override
+//! those that may be provided by the rest of the kernel.
+//!
+//! [`compiler_builtins`]: https://github.com/rust-lang/compiler-builtins
+//! [`compiler-rt`]: https://compiler-rt.llvm.org/
+
+#![feature(compiler_builtins)]
+#![compiler_builtins]
+#![no_builtins]
+#![no_std]
+
+macro_rules! define_panicking_intrinsics(
+    ($reason: tt, { $($ident: ident, )* }) => {
+        $(
+            #[doc(hidden)]
+            #[no_mangle]
+            pub extern "C" fn $ident() {
+                panic!($reason);
+            }
+        )*
+    }
+);
+
+define_panicking_intrinsics!("`f32` should not be used", {
+    __eqsf2,
+    __gesf2,
+    __lesf2,
+    __nesf2,
+    __unordsf2,
+});
+
+define_panicking_intrinsics!("`f64` should not be used", {
+    __unorddf2,
+});
+
+define_panicking_intrinsics!("`i128` should not be used", {
+    __ashrti3,
+    __muloti4,
+    __multi3,
+});
+
+define_panicking_intrinsics!("`u128` should not be used", {
+    __ashlti3,
+    __lshrti3,
+    __udivmodti4,
+    __udivti3,
+    __umodti3,
+});
diff --git a/rust/exports.c b/rust/exports.c
new file mode 100644
index 000000000000..bb7cc64cecd0
--- /dev/null
+++ b/rust/exports.c
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * A hack to export Rust symbols for loadable modules without having to redo
+ * the entire `include/linux/export.h` logic in Rust.
+ *
+ * This requires the Rust's new/future `v0` mangling scheme because the default
+ * one ("legacy") uses invalid characters for C identifiers (thus we cannot use
+ * the `EXPORT_SYMBOL_*` macros).
+ *
+ * All symbols are exported as GPL-only to guarantee no GPL-only feature is
+ * accidentally exposed.
+ */
+
+#include <linux/module.h>
+
+#define EXPORT_SYMBOL_RUST_GPL(sym) extern int sym; EXPORT_SYMBOL_GPL(sym)
+
+#include "exports_core_generated.h"
+#include "exports_alloc_generated.h"
+#include "exports_bindings_generated.h"
+#include "exports_kernel_generated.h"
diff --git a/rust/helpers.c b/rust/helpers.c
new file mode 100644
index 000000000000..b4f15eee2ffd
--- /dev/null
+++ b/rust/helpers.c
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Non-trivial C macros cannot be used in Rust. Similarly, inlined C functions
+ * cannot be called either. This file explicitly creates functions ("helpers")
+ * that wrap those so that they can be called from Rust.
+ *
+ * Even though Rust kernel modules should never use directly the bindings, some
+ * of these helpers need to be exported because Rust generics and inlined
+ * functions may not get their code generated in the crate where they are
+ * defined. Other helpers, called from non-inline functions, may not be
+ * exported, in principle. However, in general, the Rust compiler does not
+ * guarantee codegen will be performed for a non-inline function either.
+ * Therefore, this file exports all the helpers. In the future, this may be
+ * revisited to reduce the number of exports after the compiler is informed
+ * about the places codegen is required.
+ *
+ * All symbols are exported as GPL-only to guarantee no GPL-only feature is
+ * accidentally exposed.
+ */
+
+#include <linux/bug.h>
+#include <linux/build_bug.h>
+
+__noreturn void rust_helper_BUG(void)
+{
+	BUG();
+}
+EXPORT_SYMBOL_GPL(rust_helper_BUG);
+
+/*
+ * We use `bindgen`'s `--size_t-is-usize` option to bind the C `size_t` type
+ * as the Rust `usize` type, so we can use it in contexts where Rust
+ * expects a `usize` like slice (array) indices. `usize` is defined to be
+ * the same as C's `uintptr_t` type (can hold any pointer) but not
+ * necessarily the same as `size_t` (can hold the size of any single
+ * object). Most modern platforms use the same concrete integer type for
+ * both of them, but in case we find ourselves on a platform where
+ * that's not true, fail early instead of risking ABI or
+ * integer-overflow issues.
+ *
+ * If your platform fails this assertion, it means that you are in
+ * danger of integer-overflow bugs (even if you attempt to remove
+ * `--size_t-is-usize`). It may be easiest to change the kernel ABI on
+ * your platform such that `size_t` matches `uintptr_t` (i.e., to increase
+ * `size_t`, because `uintptr_t` has to be at least as big as `size_t`).
+ */
+static_assert(
+	sizeof(size_t) == sizeof(uintptr_t) &&
+	__alignof__(size_t) == __alignof__(uintptr_t),
+	"Rust code expects C `size_t` to match Rust `usize`"
+);
diff --git a/rust/kernel/allocator.rs b/rust/kernel/allocator.rs
new file mode 100644
index 000000000000..397a3dd57a9b
--- /dev/null
+++ b/rust/kernel/allocator.rs
@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Allocator support.
+
+use core::alloc::{GlobalAlloc, Layout};
+use core::ptr;
+
+use crate::bindings;
+
+struct KernelAllocator;
+
+unsafe impl GlobalAlloc for KernelAllocator {
+    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        // `krealloc()` is used instead of `kmalloc()` because the latter is
+        // an inline function and cannot be bound to as a result.
+        unsafe { bindings::krealloc(ptr::null(), layout.size(), bindings::GFP_KERNEL) as *mut u8 }
+    }
+
+    unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
+        unsafe {
+            bindings::kfree(ptr as *const core::ffi::c_void);
+        }
+    }
+}
+
+#[global_allocator]
+static ALLOCATOR: KernelAllocator = KernelAllocator;
+
+// `rustc` only generates these for some crate types. Even then, we would need
+// to extract the object file that has them from the archive. For the moment,
+// let's generate them ourselves instead.
+//
+// Note that `#[no_mangle]` implies exported too, nowadays.
+#[no_mangle]
+fn __rust_alloc(size: usize, _align: usize) -> *mut u8 {
+    unsafe { bindings::krealloc(core::ptr::null(), size, bindings::GFP_KERNEL) as *mut u8 }
+}
+
+#[no_mangle]
+fn __rust_dealloc(ptr: *mut u8, _size: usize, _align: usize) {
+    unsafe { bindings::kfree(ptr as *const core::ffi::c_void) };
+}
+
+#[no_mangle]
+fn __rust_realloc(ptr: *mut u8, _old_size: usize, _align: usize, new_size: usize) -> *mut u8 {
+    unsafe {
+        bindings::krealloc(
+            ptr as *const core::ffi::c_void,
+            new_size,
+            bindings::GFP_KERNEL,
+        ) as *mut u8
+    }
+}
+
+#[no_mangle]
+fn __rust_alloc_zeroed(size: usize, _align: usize) -> *mut u8 {
+    unsafe {
+        bindings::krealloc(
+            core::ptr::null(),
+            size,
+            bindings::GFP_KERNEL | bindings::__GFP_ZERO,
+        ) as *mut u8
+    }
+}
diff --git a/rust/kernel/error.rs b/rust/kernel/error.rs
new file mode 100644
index 000000000000..466b2a8fe569
--- /dev/null
+++ b/rust/kernel/error.rs
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Kernel errors.
+//!
+//! C header: [`include/uapi/asm-generic/errno-base.h`](../../../include/uapi/asm-generic/errno-base.h)
+
+use alloc::collections::TryReserveError;
+
+/// Contains the C-compatible error codes.
+pub mod code {
+    /// Out of memory.
+    pub const ENOMEM: super::Error = super::Error(-(crate::bindings::ENOMEM as i32));
+}
+
+/// Generic integer kernel error.
+///
+/// The kernel defines a set of integer generic error codes based on C and
+/// POSIX ones. These codes may have a more specific meaning in some contexts.
+///
+/// # Invariants
+///
+/// The value is a valid `errno` (i.e. `>= -MAX_ERRNO && < 0`).
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub struct Error(core::ffi::c_int);
+
+impl Error {
+    /// Returns the kernel error code.
+    pub fn to_kernel_errno(self) -> core::ffi::c_int {
+        self.0
+    }
+}
+
+impl From<TryReserveError> for Error {
+    fn from(_: TryReserveError) -> Error {
+        code::ENOMEM
+    }
+}
+
+/// A [`Result`] with an [`Error`] error type.
+///
+/// To be used as the return type for functions that may fail.
+///
+/// # Error codes in C and Rust
+///
+/// In C, it is common that functions indicate success or failure through
+/// their return value; modifying or returning extra data through non-`const`
+/// pointer parameters. In particular, in the kernel, functions that may fail
+/// typically return an `int` that represents a generic error code. We model
+/// those as [`Error`].
+///
+/// In Rust, it is idiomatic to model functions that may fail as returning
+/// a [`Result`]. Since in the kernel many functions return an error code,
+/// [`Result`] is a type alias for a [`core::result::Result`] that uses
+/// [`Error`] as its error type.
+///
+/// Note that even if a function does not return anything when it succeeds,
+/// it should still be modeled as returning a `Result` rather than
+/// just an [`Error`].
+pub type Result<T = ()> = core::result::Result<T, Error>;
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
new file mode 100644
index 000000000000..abd46261d385
--- /dev/null
+++ b/rust/kernel/lib.rs
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! The `kernel` crate.
+//!
+//! This crate contains the kernel APIs that have been ported or wrapped for
+//! usage by Rust code in the kernel and is shared by all of them.
+//!
+//! In other words, all the rest of the Rust code in the kernel (e.g. kernel
+//! modules written in Rust) depends on [`core`], [`alloc`] and this crate.
+//!
+//! If you need a kernel C API that is not ported or wrapped yet here, then
+//! do so first instead of bypassing this crate.
+
+#![no_std]
+#![feature(core_ffi_c)]
+
+// Ensure conditional compilation based on the kernel configuration works;
+// otherwise we may silently break things like initcall handling.
+#[cfg(not(CONFIG_RUST))]
+compile_error!("Missing kernel configuration for conditional compilation");
+
+#[cfg(not(test))]
+#[cfg(not(testlib))]
+mod allocator;
+pub mod error;
+pub mod prelude;
+pub mod print;
+pub mod str;
+
+#[doc(hidden)]
+pub use bindings;
+pub use macros;
+
+/// Prefix to appear before log messages printed from within the `kernel` crate.
+const __LOG_PREFIX: &[u8] = b"rust_kernel\0";
+
+/// The top level entrypoint to implementing a kernel module.
+///
+/// For any teardown or cleanup operations, your type may implement [`Drop`].
+pub trait Module: Sized + Sync {
+    /// Called at module initialization time.
+    ///
+    /// Use this method to perform whatever setup or registration your module
+    /// should do.
+    ///
+    /// Equivalent to the `module_init` macro in the C API.
+    fn init(module: &'static ThisModule) -> error::Result<Self>;
+}
+
+/// Equivalent to `THIS_MODULE` in the C API.
+///
+/// C header: `include/linux/export.h`
+pub struct ThisModule(*mut bindings::module);
+
+// SAFETY: `THIS_MODULE` may be used from all threads within a module.
+unsafe impl Sync for ThisModule {}
+
+impl ThisModule {
+    /// Creates a [`ThisModule`] given the `THIS_MODULE` pointer.
+    ///
+    /// # Safety
+    ///
+    /// The pointer must be equal to the right `THIS_MODULE`.
+    pub const unsafe fn from_ptr(ptr: *mut bindings::module) -> ThisModule {
+        ThisModule(ptr)
+    }
+}
+
+#[cfg(not(any(testlib, test)))]
+#[panic_handler]
+fn panic(info: &core::panic::PanicInfo<'_>) -> ! {
+    pr_emerg!("{}\n", info);
+    // SAFETY: FFI call.
+    unsafe { bindings::BUG() };
+    // Bindgen currently does not recognize `__noreturn` so `BUG` returns `()`
+    // instead of `!`. See <https://github.com/rust-lang/rust-bindgen/issues/2094>.
+    loop {}
+}
diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs
new file mode 100644
index 000000000000..495e22250726
--- /dev/null
+++ b/rust/kernel/prelude.rs
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! The `kernel` prelude.
+//!
+//! These are the most common items used by Rust code in the kernel,
+//! intended to be imported by all Rust code, for convenience.
+//!
+//! # Examples
+//!
+//! ```
+//! use kernel::prelude::*;
+//! ```
+
+pub use super::{
+    error::{Error, Result},
+    pr_emerg, pr_info, ThisModule,
+};
+pub use alloc::{boxed::Box, vec::Vec};
+pub use core::pin::Pin;
+pub use macros::module;
diff --git a/rust/kernel/print.rs b/rust/kernel/print.rs
new file mode 100644
index 000000000000..55db5a1ba752
--- /dev/null
+++ b/rust/kernel/print.rs
@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Printing facilities.
+//!
+//! C header: [`include/linux/printk.h`](../../../../include/linux/printk.h)
+//!
+//! Reference: <https://www.kernel.org/doc/html/latest/core-api/printk-basics.html>
+
+use core::{
+    ffi::{c_char, c_void},
+    fmt,
+};
+
+use crate::str::RawFormatter;
+
+#[cfg(CONFIG_PRINTK)]
+use crate::bindings;
+
+// Called from `vsprintf` with format specifier `%pA`.
+#[no_mangle]
+unsafe fn rust_fmt_argument(buf: *mut c_char, end: *mut c_char, ptr: *const c_void) -> *mut c_char {
+    use fmt::Write;
+    // SAFETY: The C contract guarantees that `buf` is valid if it's less than `end`.
+    let mut w = unsafe { RawFormatter::from_ptrs(buf.cast(), end.cast()) };
+    let _ = w.write_fmt(unsafe { *(ptr as *const fmt::Arguments<'_>) });
+    w.pos().cast()
+}
+
+/// Format strings.
+///
+/// Public but hidden since it should only be used from public macros.
+#[doc(hidden)]
+pub mod format_strings {
+    use crate::bindings;
+
+    /// The length we copy from the `KERN_*` kernel prefixes.
+    const LENGTH_PREFIX: usize = 2;
+
+    /// The length of the fixed format strings.
+    pub const LENGTH: usize = 10;
+
+    /// Generates a fixed format string for the kernel's [`_printk`].
+    ///
+    /// The format string is always the same for a given level, i.e. for a
+    /// given `prefix`, which are the kernel's `KERN_*` constants.
+    ///
+    /// [`_printk`]: ../../../../include/linux/printk.h
+    const fn generate(is_cont: bool, prefix: &[u8; 3]) -> [u8; LENGTH] {
+        // Ensure the `KERN_*` macros are what we expect.
+        assert!(prefix[0] == b'\x01');
+        if is_cont {
+            assert!(prefix[1] == b'c');
+        } else {
+            assert!(prefix[1] >= b'0' && prefix[1] <= b'7');
+        }
+        assert!(prefix[2] == b'\x00');
+
+        let suffix: &[u8; LENGTH - LENGTH_PREFIX] = if is_cont {
+            b"%pA\0\0\0\0\0"
+        } else {
+            b"%s: %pA\0"
+        };
+
+        [
+            prefix[0], prefix[1], suffix[0], suffix[1], suffix[2], suffix[3], suffix[4], suffix[5],
+            suffix[6], suffix[7],
+        ]
+    }
+
+    // Generate the format strings at compile-time.
+    //
+    // This avoids the compiler generating the contents on the fly in the stack.
+    //
+    // Furthermore, `static` instead of `const` is used to share the strings
+    // for all the kernel.
+    pub static EMERG: [u8; LENGTH] = generate(false, bindings::KERN_EMERG);
+    pub static INFO: [u8; LENGTH] = generate(false, bindings::KERN_INFO);
+}
+
+/// Prints a message via the kernel's [`_printk`].
+///
+/// Public but hidden since it should only be used from public macros.
+///
+/// # Safety
+///
+/// The format string must be one of the ones in [`format_strings`], and
+/// the module name must be null-terminated.
+///
+/// [`_printk`]: ../../../../include/linux/_printk.h
+#[doc(hidden)]
+#[cfg_attr(not(CONFIG_PRINTK), allow(unused_variables))]
+pub unsafe fn call_printk(
+    format_string: &[u8; format_strings::LENGTH],
+    module_name: &[u8],
+    args: fmt::Arguments<'_>,
+) {
+    // `_printk` does not seem to fail in any path.
+    #[cfg(CONFIG_PRINTK)]
+    unsafe {
+        bindings::_printk(
+            format_string.as_ptr() as _,
+            module_name.as_ptr(),
+            &args as *const _ as *const c_void,
+        );
+    }
+}
+
+/// Performs formatting and forwards the string to [`call_printk`].
+///
+/// Public but hidden since it should only be used from public macros.
+#[doc(hidden)]
+#[cfg(not(testlib))]
+#[macro_export]
+#[allow(clippy::crate_in_macro_def)]
+macro_rules! print_macro (
+    // The non-continuation cases (most of them, e.g. `INFO`).
+    ($format_string:path, $($arg:tt)+) => (
+        // SAFETY: This hidden macro should only be called by the documented
+        // printing macros which ensure the format string is one of the fixed
+        // ones. All `__LOG_PREFIX`s are null-terminated as they are generated
+        // by the `module!` proc macro or fixed values defined in a kernel
+        // crate.
+        unsafe {
+            $crate::print::call_printk(
+                &$format_string,
+                crate::__LOG_PREFIX,
+                format_args!($($arg)+),
+            );
+        }
+    );
+);
+
+/// Stub for doctests
+#[cfg(testlib)]
+#[macro_export]
+macro_rules! print_macro (
+    ($format_string:path, $e:expr, $($arg:tt)+) => (
+        ()
+    );
+);
+
+// We could use a macro to generate these macros. However, doing so ends
+// up being a bit ugly: it requires the dollar token trick to escape `$` as
+// well as playing with the `doc` attribute. Furthermore, they cannot be easily
+// imported in the prelude due to [1]. So, for the moment, we just write them
+// manually, like in the C side; while keeping most of the logic in another
+// macro, i.e. [`print_macro`].
+//
+// [1]: https://github.com/rust-lang/rust/issues/52234
+
+/// Prints an emergency-level message (level 0).
+///
+/// Use this level if the system is unusable.
+///
+/// Equivalent to the kernel's [`pr_emerg`] macro.
+///
+/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
+/// `alloc::format!` for information about the formatting syntax.
+///
+/// [`pr_emerg`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_emerg
+/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+///
+/// # Examples
+///
+/// ```
+/// pr_emerg!("hello {}\n", "there");
+/// ```
+#[macro_export]
+macro_rules! pr_emerg (
+    ($($arg:tt)*) => (
+        $crate::print_macro!($crate::print::format_strings::EMERG, $($arg)*)
+    )
+);
+
+/// Prints an info-level message (level 6).
+///
+/// Use this level for informational messages.
+///
+/// Equivalent to the kernel's [`pr_info`] macro.
+///
+/// Mimics the interface of [`std::print!`]. See [`core::fmt`] and
+/// `alloc::format!` for information about the formatting syntax.
+///
+/// [`pr_info`]: https://www.kernel.org/doc/html/latest/core-api/printk-basics.html#c.pr_info
+/// [`std::print!`]: https://doc.rust-lang.org/std/macro.print.html
+///
+/// # Examples
+///
+/// ```
+/// pr_info!("hello {}\n", "there");
+/// ```
+#[macro_export]
+#[doc(alias = "print")]
+macro_rules! pr_info (
+    ($($arg:tt)*) => (
+        $crate::print_macro!($crate::print::format_strings::INFO, $($arg)*)
+    )
+);
diff --git a/rust/kernel/str.rs b/rust/kernel/str.rs
new file mode 100644
index 000000000000..e45ff220ae50
--- /dev/null
+++ b/rust/kernel/str.rs
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! String representations.
+
+use core::fmt;
+
+/// Allows formatting of [`fmt::Arguments`] into a raw buffer.
+///
+/// It does not fail if callers write past the end of the buffer so that they can calculate the
+/// size required to fit everything.
+///
+/// # Invariants
+///
+/// The memory region between `pos` (inclusive) and `end` (exclusive) is valid for writes if `pos`
+/// is less than `end`.
+pub(crate) struct RawFormatter {
+    // Use `usize` to use `saturating_*` functions.
+    #[allow(dead_code)]
+    beg: usize,
+    pos: usize,
+    end: usize,
+}
+
+impl RawFormatter {
+    /// Creates a new instance of [`RawFormatter`] with the given buffer pointers.
+    ///
+    /// # Safety
+    ///
+    /// If `pos` is less than `end`, then the region between `pos` (inclusive) and `end`
+    /// (exclusive) must be valid for writes for the lifetime of the returned [`RawFormatter`].
+    pub(crate) unsafe fn from_ptrs(pos: *mut u8, end: *mut u8) -> Self {
+        // INVARIANT: The safety requierments guarantee the type invariants.
+        Self {
+            beg: pos as _,
+            pos: pos as _,
+            end: end as _,
+        }
+    }
+
+    /// Returns the current insert position.
+    ///
+    /// N.B. It may point to invalid memory.
+    pub(crate) fn pos(&self) -> *mut u8 {
+        self.pos as _
+    }
+}
+
+impl fmt::Write for RawFormatter {
+    fn write_str(&mut self, s: &str) -> fmt::Result {
+        // `pos` value after writing `len` bytes. This does not have to be bounded by `end`, but we
+        // don't want it to wrap around to 0.
+        let pos_new = self.pos.saturating_add(s.len());
+
+        // Amount that we can copy. `saturating_sub` ensures we get 0 if `pos` goes past `end`.
+        let len_to_copy = core::cmp::min(pos_new, self.end).saturating_sub(self.pos);
+
+        if len_to_copy > 0 {
+            // SAFETY: If `len_to_copy` is non-zero, then we know `pos` has not gone past `end`
+            // yet, so it is valid for write per the type invariants.
+            unsafe {
+                core::ptr::copy_nonoverlapping(
+                    s.as_bytes().as_ptr(),
+                    self.pos as *mut u8,
+                    len_to_copy,
+                )
+            };
+        }
+
+        self.pos = pos_new;
+        Ok(())
+    }
+}
diff --git a/rust/macros/helpers.rs b/rust/macros/helpers.rs
new file mode 100644
index 000000000000..cdc7dc6135d2
--- /dev/null
+++ b/rust/macros/helpers.rs
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use proc_macro::{token_stream, TokenTree};
+
+pub(crate) fn try_ident(it: &mut token_stream::IntoIter) -> Option<String> {
+    if let Some(TokenTree::Ident(ident)) = it.next() {
+        Some(ident.to_string())
+    } else {
+        None
+    }
+}
+
+pub(crate) fn try_literal(it: &mut token_stream::IntoIter) -> Option<String> {
+    if let Some(TokenTree::Literal(literal)) = it.next() {
+        Some(literal.to_string())
+    } else {
+        None
+    }
+}
+
+pub(crate) fn try_byte_string(it: &mut token_stream::IntoIter) -> Option<String> {
+    try_literal(it).and_then(|byte_string| {
+        if byte_string.starts_with("b\"") && byte_string.ends_with('\"') {
+            Some(byte_string[2..byte_string.len() - 1].to_string())
+        } else {
+            None
+        }
+    })
+}
+
+pub(crate) fn expect_ident(it: &mut token_stream::IntoIter) -> String {
+    try_ident(it).expect("Expected Ident")
+}
+
+pub(crate) fn expect_punct(it: &mut token_stream::IntoIter) -> char {
+    if let TokenTree::Punct(punct) = it.next().expect("Reached end of token stream for Punct") {
+        punct.as_char()
+    } else {
+        panic!("Expected Punct");
+    }
+}
+
+pub(crate) fn expect_byte_string(it: &mut token_stream::IntoIter) -> String {
+    try_byte_string(it).expect("Expected byte string")
+}
+
+pub(crate) fn expect_end(it: &mut token_stream::IntoIter) {
+    if it.next().is_some() {
+        panic!("Expected end");
+    }
+}
diff --git a/rust/macros/lib.rs b/rust/macros/lib.rs
new file mode 100644
index 000000000000..91764bfb1f89
--- /dev/null
+++ b/rust/macros/lib.rs
@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Crate for all kernel procedural macros.
+
+mod helpers;
+mod module;
+
+use proc_macro::TokenStream;
+
+/// Declares a kernel module.
+///
+/// The `type` argument should be a type which implements the [`Module`]
+/// trait. Also accepts various forms of kernel metadata.
+///
+/// C header: [`include/linux/moduleparam.h`](../../../include/linux/moduleparam.h)
+///
+/// [`Module`]: ../kernel/trait.Module.html
+///
+/// # Examples
+///
+/// ```ignore
+/// use kernel::prelude::*;
+///
+/// module!{
+///     type: MyModule,
+///     name: b"my_kernel_module",
+///     author: b"Rust for Linux Contributors",
+///     description: b"My very own kernel module!",
+///     license: b"GPL",
+///     params: {
+///        my_i32: i32 {
+///            default: 42,
+///            permissions: 0o000,
+///            description: b"Example of i32",
+///        },
+///        writeable_i32: i32 {
+///            default: 42,
+///            permissions: 0o644,
+///            description: b"Example of i32",
+///        },
+///    },
+/// }
+///
+/// struct MyModule;
+///
+/// impl kernel::Module for MyModule {
+///     fn init() -> Result<Self> {
+///         // If the parameter is writeable, then the kparam lock must be
+///         // taken to read the parameter:
+///         {
+///             let lock = THIS_MODULE.kernel_param_lock();
+///             pr_info!("i32 param is:  {}\n", writeable_i32.read(&lock));
+///         }
+///         // If the parameter is read only, it can be read without locking
+///         // the kernel parameters:
+///         pr_info!("i32 param is:  {}\n", my_i32.read());
+///         Ok(Self)
+///     }
+/// }
+/// ```
+///
+/// # Supported argument types
+///   - `type`: type which implements the [`Module`] trait (required).
+///   - `name`: byte array of the name of the kernel module (required).
+///   - `author`: byte array of the author of the kernel module.
+///   - `description`: byte array of the description of the kernel module.
+///   - `license`: byte array of the license of the kernel module (required).
+///   - `alias`: byte array of alias name of the kernel module.
+#[proc_macro]
+pub fn module(ts: TokenStream) -> TokenStream {
+    module::module(ts)
+}
diff --git a/rust/macros/module.rs b/rust/macros/module.rs
new file mode 100644
index 000000000000..186a5b8be23c
--- /dev/null
+++ b/rust/macros/module.rs
@@ -0,0 +1,282 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use crate::helpers::*;
+use proc_macro::{token_stream, Literal, TokenStream, TokenTree};
+use std::fmt::Write;
+
+struct ModInfoBuilder<'a> {
+    module: &'a str,
+    counter: usize,
+    buffer: String,
+}
+
+impl<'a> ModInfoBuilder<'a> {
+    fn new(module: &'a str) -> Self {
+        ModInfoBuilder {
+            module,
+            counter: 0,
+            buffer: String::new(),
+        }
+    }
+
+    fn emit_base(&mut self, field: &str, content: &str, builtin: bool) {
+        let string = if builtin {
+            // Built-in modules prefix their modinfo strings by `module.`.
+            format!(
+                "{module}.{field}={content}\0",
+                module = self.module,
+                field = field,
+                content = content
+            )
+        } else {
+            // Loadable modules' modinfo strings go as-is.
+            format!("{field}={content}\0", field = field, content = content)
+        };
+
+        write!(
+            &mut self.buffer,
+            "
+                {cfg}
+                #[doc(hidden)]
+                #[link_section = \".modinfo\"]
+                #[used]
+                pub static __{module}_{counter}: [u8; {length}] = *{string};
+            ",
+            cfg = if builtin {
+                "#[cfg(not(MODULE))]"
+            } else {
+                "#[cfg(MODULE)]"
+            },
+            module = self.module.to_uppercase(),
+            counter = self.counter,
+            length = string.len(),
+            string = Literal::byte_string(string.as_bytes()),
+        )
+        .unwrap();
+
+        self.counter += 1;
+    }
+
+    fn emit_only_builtin(&mut self, field: &str, content: &str) {
+        self.emit_base(field, content, true)
+    }
+
+    fn emit_only_loadable(&mut self, field: &str, content: &str) {
+        self.emit_base(field, content, false)
+    }
+
+    fn emit(&mut self, field: &str, content: &str) {
+        self.emit_only_builtin(field, content);
+        self.emit_only_loadable(field, content);
+    }
+}
+
+#[derive(Debug, Default)]
+struct ModuleInfo {
+    type_: String,
+    license: String,
+    name: String,
+    author: Option<String>,
+    description: Option<String>,
+    alias: Option<String>,
+}
+
+impl ModuleInfo {
+    fn parse(it: &mut token_stream::IntoIter) -> Self {
+        let mut info = ModuleInfo::default();
+
+        const EXPECTED_KEYS: &[&str] =
+            &["type", "name", "author", "description", "license", "alias"];
+        const REQUIRED_KEYS: &[&str] = &["type", "name", "license"];
+        let mut seen_keys = Vec::new();
+
+        loop {
+            let key = match it.next() {
+                Some(TokenTree::Ident(ident)) => ident.to_string(),
+                Some(_) => panic!("Expected Ident or end"),
+                None => break,
+            };
+
+            if seen_keys.contains(&key) {
+                panic!(
+                    "Duplicated key \"{}\". Keys can only be specified once.",
+                    key
+                );
+            }
+
+            assert_eq!(expect_punct(it), ':');
+
+            match key.as_str() {
+                "type" => info.type_ = expect_ident(it),
+                "name" => info.name = expect_byte_string(it),
+                "author" => info.author = Some(expect_byte_string(it)),
+                "description" => info.description = Some(expect_byte_string(it)),
+                "license" => info.license = expect_byte_string(it),
+                "alias" => info.alias = Some(expect_byte_string(it)),
+                _ => panic!(
+                    "Unknown key \"{}\". Valid keys are: {:?}.",
+                    key, EXPECTED_KEYS
+                ),
+            }
+
+            assert_eq!(expect_punct(it), ',');
+
+            seen_keys.push(key);
+        }
+
+        expect_end(it);
+
+        for key in REQUIRED_KEYS {
+            if !seen_keys.iter().any(|e| e == key) {
+                panic!("Missing required key \"{}\".", key);
+            }
+        }
+
+        let mut ordered_keys: Vec<&str> = Vec::new();
+        for key in EXPECTED_KEYS {
+            if seen_keys.iter().any(|e| e == key) {
+                ordered_keys.push(key);
+            }
+        }
+
+        if seen_keys != ordered_keys {
+            panic!(
+                "Keys are not ordered as expected. Order them like: {:?}.",
+                ordered_keys
+            );
+        }
+
+        info
+    }
+}
+
+pub(crate) fn module(ts: TokenStream) -> TokenStream {
+    let mut it = ts.into_iter();
+
+    let info = ModuleInfo::parse(&mut it);
+
+    let mut modinfo = ModInfoBuilder::new(info.name.as_ref());
+    if let Some(author) = info.author {
+        modinfo.emit("author", &author);
+    }
+    if let Some(description) = info.description {
+        modinfo.emit("description", &description);
+    }
+    modinfo.emit("license", &info.license);
+    if let Some(alias) = info.alias {
+        modinfo.emit("alias", &alias);
+    }
+
+    // Built-in modules also export the `file` modinfo string.
+    let file =
+        std::env::var("RUST_MODFILE").expect("Unable to fetch RUST_MODFILE environmental variable");
+    modinfo.emit_only_builtin("file", &file);
+
+    format!(
+        "
+            /// The module name.
+            ///
+            /// Used by the printing macros, e.g. [`info!`].
+            const __LOG_PREFIX: &[u8] = b\"{name}\\0\";
+
+            /// The \"Rust loadable module\" mark, for `scripts/is_rust_module.sh`.
+            //
+            // This may be best done another way later on, e.g. as a new modinfo
+            // key or a new section. For the moment, keep it simple.
+            #[cfg(MODULE)]
+            #[doc(hidden)]
+            #[used]
+            static __IS_RUST_MODULE: () = ();
+
+            static mut __MOD: Option<{type_}> = None;
+
+            // SAFETY: `__this_module` is constructed by the kernel at load time and will not be
+            // freed until the module is unloaded.
+            #[cfg(MODULE)]
+            static THIS_MODULE: kernel::ThisModule = unsafe {{
+                kernel::ThisModule::from_ptr(&kernel::bindings::__this_module as *const _ as *mut _)
+            }};
+            #[cfg(not(MODULE))]
+            static THIS_MODULE: kernel::ThisModule = unsafe {{
+                kernel::ThisModule::from_ptr(core::ptr::null_mut())
+            }};
+
+            // Loadable modules need to export the `{{init,cleanup}}_module` identifiers.
+            #[cfg(MODULE)]
+            #[doc(hidden)]
+            #[no_mangle]
+            pub extern \"C\" fn init_module() -> core::ffi::c_int {{
+                __init()
+            }}
+
+            #[cfg(MODULE)]
+            #[doc(hidden)]
+            #[no_mangle]
+            pub extern \"C\" fn cleanup_module() {{
+                __exit()
+            }}
+
+            // Built-in modules are initialized through an initcall pointer
+            // and the identifiers need to be unique.
+            #[cfg(not(MODULE))]
+            #[cfg(not(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS))]
+            #[doc(hidden)]
+            #[link_section = \"{initcall_section}\"]
+            #[used]
+            pub static __{name}_initcall: extern \"C\" fn() -> core::ffi::c_int = __{name}_init;
+
+            #[cfg(not(MODULE))]
+            #[cfg(CONFIG_HAVE_ARCH_PREL32_RELOCATIONS)]
+            core::arch::global_asm!(
+                r#\".section \"{initcall_section}\", \"a\"
+                __{name}_initcall:
+                    .long   __{name}_init - .
+                    .previous
+                \"#
+            );
+
+            #[cfg(not(MODULE))]
+            #[doc(hidden)]
+            #[no_mangle]
+            pub extern \"C\" fn __{name}_init() -> core::ffi::c_int {{
+                __init()
+            }}
+
+            #[cfg(not(MODULE))]
+            #[doc(hidden)]
+            #[no_mangle]
+            pub extern \"C\" fn __{name}_exit() {{
+                __exit()
+            }}
+
+            fn __init() -> core::ffi::c_int {{
+                match <{type_} as kernel::Module>::init(&THIS_MODULE) {{
+                    Ok(m) => {{
+                        unsafe {{
+                            __MOD = Some(m);
+                        }}
+                        return 0;
+                    }}
+                    Err(e) => {{
+                        return e.to_kernel_errno();
+                    }}
+                }}
+            }}
+
+            fn __exit() {{
+                unsafe {{
+                    // Invokes `drop()` on `__MOD`, which should be used for cleanup.
+                    __MOD = None;
+                }}
+            }}
+
+            {modinfo}
+        ",
+        type_ = info.type_,
+        name = info.name,
+        modinfo = modinfo.buffer,
+        initcall_section = ".initcall6.init"
+    )
+    .parse()
+    .expect("Error parsing formatted string into token stream.")
+}
diff --git a/samples/Kconfig b/samples/Kconfig
index 470ee3baf2e1..0d81c00289ee 100644
--- a/samples/Kconfig
+++ b/samples/Kconfig
@@ -263,6 +263,8 @@ config SAMPLE_CORESIGHT_SYSCFG
 	  This demonstrates how a user may create their own CoreSight
 	  configurations and easily load them into the system at runtime.
 
+source "samples/rust/Kconfig"
+
 endif # SAMPLES
 
 config HAVE_SAMPLE_FTRACE_DIRECT
diff --git a/samples/Makefile b/samples/Makefile
index 701e912ab5af..9832ef3f8fcb 100644
--- a/samples/Makefile
+++ b/samples/Makefile
@@ -35,3 +35,4 @@ subdir-$(CONFIG_SAMPLE_WATCH_QUEUE)	+= watch_queue
 obj-$(CONFIG_DEBUG_KMEMLEAK_TEST)	+= kmemleak/
 obj-$(CONFIG_SAMPLE_CORESIGHT_SYSCFG)	+= coresight/
 obj-$(CONFIG_SAMPLE_FPROBE)		+= fprobe/
+obj-$(CONFIG_SAMPLES_RUST)		+= rust/
diff --git a/samples/bpf/map_perf_test_kern.c b/samples/bpf/map_perf_test_kern.c
index 8773f22b6a98..7342c5b2f278 100644
--- a/samples/bpf/map_perf_test_kern.c
+++ b/samples/bpf/map_perf_test_kern.c
@@ -108,11 +108,14 @@ int stress_hmap(struct pt_regs *ctx)
 	u32 key = bpf_get_current_pid_tgid();
 	long init_val = 1;
 	long *value;
+	int i;
 
-	bpf_map_update_elem(&hash_map, &key, &init_val, BPF_ANY);
-	value = bpf_map_lookup_elem(&hash_map, &key);
-	if (value)
-		bpf_map_delete_elem(&hash_map, &key);
+	for (i = 0; i < 10; i++) {
+		bpf_map_update_elem(&hash_map, &key, &init_val, BPF_ANY);
+		value = bpf_map_lookup_elem(&hash_map, &key);
+		if (value)
+			bpf_map_delete_elem(&hash_map, &key);
+	}
 
 	return 0;
 }
@@ -123,11 +126,14 @@ int stress_percpu_hmap(struct pt_regs *ctx)
 	u32 key = bpf_get_current_pid_tgid();
 	long init_val = 1;
 	long *value;
+	int i;
 
-	bpf_map_update_elem(&percpu_hash_map, &key, &init_val, BPF_ANY);
-	value = bpf_map_lookup_elem(&percpu_hash_map, &key);
-	if (value)
-		bpf_map_delete_elem(&percpu_hash_map, &key);
+	for (i = 0; i < 10; i++) {
+		bpf_map_update_elem(&percpu_hash_map, &key, &init_val, BPF_ANY);
+		value = bpf_map_lookup_elem(&percpu_hash_map, &key);
+		if (value)
+			bpf_map_delete_elem(&percpu_hash_map, &key);
+	}
 	return 0;
 }
 
@@ -137,11 +143,14 @@ int stress_hmap_alloc(struct pt_regs *ctx)
 	u32 key = bpf_get_current_pid_tgid();
 	long init_val = 1;
 	long *value;
+	int i;
 
-	bpf_map_update_elem(&hash_map_alloc, &key, &init_val, BPF_ANY);
-	value = bpf_map_lookup_elem(&hash_map_alloc, &key);
-	if (value)
-		bpf_map_delete_elem(&hash_map_alloc, &key);
+	for (i = 0; i < 10; i++) {
+		bpf_map_update_elem(&hash_map_alloc, &key, &init_val, BPF_ANY);
+		value = bpf_map_lookup_elem(&hash_map_alloc, &key);
+		if (value)
+			bpf_map_delete_elem(&hash_map_alloc, &key);
+	}
 	return 0;
 }
 
@@ -151,11 +160,14 @@ int stress_percpu_hmap_alloc(struct pt_regs *ctx)
 	u32 key = bpf_get_current_pid_tgid();
 	long init_val = 1;
 	long *value;
+	int i;
 
-	bpf_map_update_elem(&percpu_hash_map_alloc, &key, &init_val, BPF_ANY);
-	value = bpf_map_lookup_elem(&percpu_hash_map_alloc, &key);
-	if (value)
-		bpf_map_delete_elem(&percpu_hash_map_alloc, &key);
+	for (i = 0; i < 10; i++) {
+		bpf_map_update_elem(&percpu_hash_map_alloc, &key, &init_val, BPF_ANY);
+		value = bpf_map_lookup_elem(&percpu_hash_map_alloc, &key);
+		if (value)
+			bpf_map_delete_elem(&percpu_hash_map_alloc, &key);
+	}
 	return 0;
 }
 
diff --git a/samples/bpf/map_perf_test_user.c b/samples/bpf/map_perf_test_user.c
index b6fc174ab1f2..1bb53f4b29e1 100644
--- a/samples/bpf/map_perf_test_user.c
+++ b/samples/bpf/map_perf_test_user.c
@@ -72,7 +72,7 @@ static int test_flags = ~0;
 static uint32_t num_map_entries;
 static uint32_t inner_lru_hash_size;
 static int lru_hash_lookup_test_entries = 32;
-static uint32_t max_cnt = 1000000;
+static uint32_t max_cnt = 10000;
 
 static int check_test_flags(enum test_type t)
 {
diff --git a/samples/bpf/task_fd_query_kern.c b/samples/bpf/task_fd_query_kern.c
index c821294e1774..186ac0a79c0a 100644
--- a/samples/bpf/task_fd_query_kern.c
+++ b/samples/bpf/task_fd_query_kern.c
@@ -10,7 +10,7 @@ int bpf_prog1(struct pt_regs *ctx)
 	return 0;
 }
 
-SEC("kretprobe/blk_account_io_done")
+SEC("kretprobe/__blk_account_io_done")
 int bpf_prog2(struct pt_regs *ctx)
 {
 	return 0;
diff --git a/samples/bpf/task_fd_query_user.c b/samples/bpf/task_fd_query_user.c
index 424718c0872c..a33d74bd3a4b 100644
--- a/samples/bpf/task_fd_query_user.c
+++ b/samples/bpf/task_fd_query_user.c
@@ -348,7 +348,7 @@ int main(int argc, char **argv)
 	/* test two functions in the corresponding *_kern.c file */
 	CHECK_AND_RET(test_debug_fs_kprobe(0, "blk_mq_start_request",
 					   BPF_FD_TYPE_KPROBE));
-	CHECK_AND_RET(test_debug_fs_kprobe(1, "blk_account_io_done",
+	CHECK_AND_RET(test_debug_fs_kprobe(1, "__blk_account_io_done",
 					   BPF_FD_TYPE_KRETPROBE));
 
 	/* test nondebug fs kprobe */
diff --git a/samples/bpf/tracex3_kern.c b/samples/bpf/tracex3_kern.c
index 710a4410b2fb..bde6591cb20c 100644
--- a/samples/bpf/tracex3_kern.c
+++ b/samples/bpf/tracex3_kern.c
@@ -49,7 +49,7 @@ struct {
 	__uint(max_entries, SLOTS);
 } lat_map SEC(".maps");
 
-SEC("kprobe/blk_account_io_done")
+SEC("kprobe/__blk_account_io_done")
 int bpf_prog2(struct pt_regs *ctx)
 {
 	long rq = PT_REGS_PARM1(ctx);
diff --git a/samples/bpf/xdp_router_ipv4_user.c b/samples/bpf/xdp_router_ipv4_user.c
index 294fc15ad1cb..683913bbf279 100644
--- a/samples/bpf/xdp_router_ipv4_user.c
+++ b/samples/bpf/xdp_router_ipv4_user.c
@@ -209,7 +209,7 @@ static void read_route(struct nlmsghdr *nh, int nll)
 					/* Rereading the route table to check if
 					 * there is an entry with the same
 					 * prefix but a different metric as the
-					 * deleted enty.
+					 * deleted entry.
 					 */
 					get_route_table(AF_INET);
 				} else if (prefix_key->data[0] ==
diff --git a/samples/landlock/sandboxer.c b/samples/landlock/sandboxer.c
index 3e404e51ec64..f29bb3c72230 100644
--- a/samples/landlock/sandboxer.c
+++ b/samples/landlock/sandboxer.c
@@ -162,11 +162,10 @@ out_free_name:
 	LANDLOCK_ACCESS_FS_MAKE_SYM | \
 	LANDLOCK_ACCESS_FS_REFER)
 
-#define ACCESS_ABI_2 ( \
-	LANDLOCK_ACCESS_FS_REFER)
-
 /* clang-format on */
 
+#define LANDLOCK_ABI_LAST 2
+
 int main(const int argc, char *const argv[], char *const *const envp)
 {
 	const char *cmd_path;
@@ -196,8 +195,12 @@ int main(const int argc, char *const argv[], char *const *const envp)
 			"\nexample:\n"
 			"%s=\"/bin:/lib:/usr:/proc:/etc:/dev/urandom\" "
 			"%s=\"/dev/null:/dev/full:/dev/zero:/dev/pts:/tmp\" "
-			"%s bash -i\n",
+			"%s bash -i\n\n",
 			ENV_FS_RO_NAME, ENV_FS_RW_NAME, argv[0]);
+		fprintf(stderr,
+			"This sandboxer can use Landlock features "
+			"up to ABI version %d.\n",
+			LANDLOCK_ABI_LAST);
 		return 1;
 	}
 
@@ -225,12 +228,30 @@ int main(const int argc, char *const argv[], char *const *const envp)
 		}
 		return 1;
 	}
+
 	/* Best-effort security. */
-	if (abi < 2) {
-		ruleset_attr.handled_access_fs &= ~ACCESS_ABI_2;
-		access_fs_ro &= ~ACCESS_ABI_2;
-		access_fs_rw &= ~ACCESS_ABI_2;
+	switch (abi) {
+	case 1:
+		/* Removes LANDLOCK_ACCESS_FS_REFER for ABI < 2 */
+		ruleset_attr.handled_access_fs &= ~LANDLOCK_ACCESS_FS_REFER;
+
+		fprintf(stderr,
+			"Hint: You should update the running kernel "
+			"to leverage Landlock features "
+			"provided by ABI version %d (instead of %d).\n",
+			LANDLOCK_ABI_LAST, abi);
+		__attribute__((fallthrough));
+	case LANDLOCK_ABI_LAST:
+		break;
+	default:
+		fprintf(stderr,
+			"Hint: You should update this sandboxer "
+			"to leverage Landlock features "
+			"provided by ABI version %d (instead of %d).\n",
+			abi, LANDLOCK_ABI_LAST);
 	}
+	access_fs_ro &= ruleset_attr.handled_access_fs;
+	access_fs_rw &= ruleset_attr.handled_access_fs;
 
 	ruleset_fd =
 		landlock_create_ruleset(&ruleset_attr, sizeof(ruleset_attr), 0);
diff --git a/samples/qmi/qmi_sample_client.c b/samples/qmi/qmi_sample_client.c
index 78fcedbd25e2..c045e3d24326 100644
--- a/samples/qmi/qmi_sample_client.c
+++ b/samples/qmi/qmi_sample_client.c
@@ -42,7 +42,7 @@ struct test_name_type_v01 {
 	char name[TEST_MAX_NAME_SIZE_V01];
 };
 
-static struct qmi_elem_info test_name_type_v01_ei[] = {
+static const struct qmi_elem_info test_name_type_v01_ei[] = {
 	{
 		.data_type	= QMI_DATA_LEN,
 		.elem_len	= 1,
@@ -71,7 +71,7 @@ struct test_ping_req_msg_v01 {
 	struct test_name_type_v01 client_name;
 };
 
-static struct qmi_elem_info test_ping_req_msg_v01_ei[] = {
+static const struct qmi_elem_info test_ping_req_msg_v01_ei[] = {
 	{
 		.data_type	= QMI_UNSIGNED_1_BYTE,
 		.elem_len	= 4,
@@ -113,7 +113,7 @@ struct test_ping_resp_msg_v01 {
 	struct test_name_type_v01 service_name;
 };
 
-static struct qmi_elem_info test_ping_resp_msg_v01_ei[] = {
+static const struct qmi_elem_info test_ping_resp_msg_v01_ei[] = {
 	{
 		.data_type	= QMI_STRUCT,
 		.elem_len	= 1,
@@ -172,7 +172,7 @@ struct test_data_req_msg_v01 {
 	struct test_name_type_v01 client_name;
 };
 
-static struct qmi_elem_info test_data_req_msg_v01_ei[] = {
+static const struct qmi_elem_info test_data_req_msg_v01_ei[] = {
 	{
 		.data_type	= QMI_DATA_LEN,
 		.elem_len	= 1,
@@ -224,7 +224,7 @@ struct test_data_resp_msg_v01 {
 	struct test_name_type_v01 service_name;
 };
 
-static struct qmi_elem_info test_data_resp_msg_v01_ei[] = {
+static const struct qmi_elem_info test_data_resp_msg_v01_ei[] = {
 	{
 		.data_type	= QMI_STRUCT,
 		.elem_len	= 1,
diff --git a/samples/rust/Kconfig b/samples/rust/Kconfig
new file mode 100644
index 000000000000..841e0906e943
--- /dev/null
+++ b/samples/rust/Kconfig
@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: GPL-2.0
+
+menuconfig SAMPLES_RUST
+	bool "Rust samples"
+	depends on RUST
+	help
+	  You can build sample Rust kernel code here.
+
+	  If unsure, say N.
+
+if SAMPLES_RUST
+
+config SAMPLE_RUST_MINIMAL
+	tristate "Minimal"
+	help
+	  This option builds the Rust minimal module sample.
+
+	  To compile this as a module, choose M here:
+	  the module will be called rust_minimal.
+
+	  If unsure, say N.
+
+config SAMPLE_RUST_HOSTPROGS
+	bool "Host programs"
+	help
+	  This option builds the Rust host program samples.
+
+	  If unsure, say N.
+
+endif # SAMPLES_RUST
diff --git a/samples/rust/Makefile b/samples/rust/Makefile
new file mode 100644
index 000000000000..1daba5f8658a
--- /dev/null
+++ b/samples/rust/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_SAMPLE_RUST_MINIMAL)		+= rust_minimal.o
+
+subdir-$(CONFIG_SAMPLE_RUST_HOSTPROGS)		+= hostprogs
diff --git a/samples/rust/hostprogs/.gitignore b/samples/rust/hostprogs/.gitignore
new file mode 100644
index 000000000000..a6c173da5048
--- /dev/null
+++ b/samples/rust/hostprogs/.gitignore
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+
+single
diff --git a/samples/rust/hostprogs/Makefile b/samples/rust/hostprogs/Makefile
new file mode 100644
index 000000000000..8ddcbd7416db
--- /dev/null
+++ b/samples/rust/hostprogs/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+
+hostprogs-always-y := single
+
+single-rust := y
diff --git a/samples/rust/hostprogs/a.rs b/samples/rust/hostprogs/a.rs
new file mode 100644
index 000000000000..f7a4a3d0f4e0
--- /dev/null
+++ b/samples/rust/hostprogs/a.rs
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust single host program sample: module `a`.
+
+pub(crate) fn f(x: i32) {
+    println!("The number is {}.", x);
+}
diff --git a/samples/rust/hostprogs/b.rs b/samples/rust/hostprogs/b.rs
new file mode 100644
index 000000000000..c1675890648f
--- /dev/null
+++ b/samples/rust/hostprogs/b.rs
@@ -0,0 +1,5 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust single host program sample: module `b`.
+
+pub(crate) const CONSTANT: i32 = 42;
diff --git a/samples/rust/hostprogs/single.rs b/samples/rust/hostprogs/single.rs
new file mode 100644
index 000000000000..8c48a119339a
--- /dev/null
+++ b/samples/rust/hostprogs/single.rs
@@ -0,0 +1,12 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust single host program sample.
+
+mod a;
+mod b;
+
+fn main() {
+    println!("Hello world!");
+
+    a::f(b::CONSTANT);
+}
diff --git a/samples/rust/rust_minimal.rs b/samples/rust/rust_minimal.rs
new file mode 100644
index 000000000000..54ad17685742
--- /dev/null
+++ b/samples/rust/rust_minimal.rs
@@ -0,0 +1,38 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Rust minimal sample.
+
+use kernel::prelude::*;
+
+module! {
+    type: RustMinimal,
+    name: b"rust_minimal",
+    author: b"Rust for Linux Contributors",
+    description: b"Rust minimal sample",
+    license: b"GPL",
+}
+
+struct RustMinimal {
+    numbers: Vec<i32>,
+}
+
+impl kernel::Module for RustMinimal {
+    fn init(_module: &'static ThisModule) -> Result<Self> {
+        pr_info!("Rust minimal sample (init)\n");
+        pr_info!("Am I built-in? {}\n", !cfg!(MODULE));
+
+        let mut numbers = Vec::new();
+        numbers.try_push(72)?;
+        numbers.try_push(108)?;
+        numbers.try_push(200)?;
+
+        Ok(RustMinimal { numbers })
+    }
+}
+
+impl Drop for RustMinimal {
+    fn drop(&mut self) {
+        pr_info!("My numbers are {:?}\n", self.numbers);
+        pr_info!("Rust minimal sample (exit)\n");
+    }
+}
diff --git a/scripts/.gitignore b/scripts/.gitignore
index eed308bef604..b7aec8eb1bd4 100644
--- a/scripts/.gitignore
+++ b/scripts/.gitignore
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 /asn1_compiler
 /bin2c
+/generate_rust_target
 /insert-sys-cert
 /kallsyms
 /module.lds
diff --git a/scripts/Kconfig.include b/scripts/Kconfig.include
index a0ccceb22cf8..274125307ebd 100644
--- a/scripts/Kconfig.include
+++ b/scripts/Kconfig.include
@@ -36,12 +36,12 @@ ld-option = $(success,$(LD) -v $(1))
 as-instr = $(success,printf "%b\n" "$(1)" | $(CC) $(CLANG_FLAGS) -c -x assembler -o /dev/null -)
 
 # check if $(CC) and $(LD) exist
-$(error-if,$(failure,command -v $(CC)),compiler '$(CC)' not found)
+$(error-if,$(failure,command -v $(CC)),C compiler '$(CC)' not found)
 $(error-if,$(failure,command -v $(LD)),linker '$(LD)' not found)
 
-# Get the compiler name, version, and error out if it is not supported.
+# Get the C compiler name, version, and error out if it is not supported.
 cc-info := $(shell,$(srctree)/scripts/cc-version.sh $(CC))
-$(error-if,$(success,test -z "$(cc-info)"),Sorry$(comma) this compiler is not supported.)
+$(error-if,$(success,test -z "$(cc-info)"),Sorry$(comma) this C compiler is not supported.)
 cc-name := $(shell,set -- $(cc-info) && echo $1)
 cc-version := $(shell,set -- $(cc-info) && echo $2)
 
diff --git a/scripts/Makefile b/scripts/Makefile
index f084f08ed176..1575af84d557 100644
--- a/scripts/Makefile
+++ b/scripts/Makefile
@@ -10,6 +10,9 @@ hostprogs-always-$(CONFIG_BUILDTIME_TABLE_SORT)		+= sorttable
 hostprogs-always-$(CONFIG_ASN1)				+= asn1_compiler
 hostprogs-always-$(CONFIG_MODULE_SIG_FORMAT)		+= sign-file
 hostprogs-always-$(CONFIG_SYSTEM_EXTRA_CERTIFICATE)	+= insert-sys-cert
+hostprogs-always-$(CONFIG_RUST)				+= generate_rust_target
+
+generate_rust_target-rust := y
 
 HOSTCFLAGS_sorttable.o = -I$(srctree)/tools/include
 HOSTLDLIBS_sorttable = -lpthread
diff --git a/scripts/Makefile.build b/scripts/Makefile.build
index 784f46d41959..27be77c0d6d8 100644
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -26,6 +26,7 @@ EXTRA_CPPFLAGS :=
 EXTRA_LDFLAGS  :=
 asflags-y  :=
 ccflags-y  :=
+rustflags-y :=
 cppflags-y :=
 ldflags-y  :=
 
@@ -271,6 +272,65 @@ quiet_cmd_cc_lst_c = MKLST   $@
 $(obj)/%.lst: $(src)/%.c FORCE
 	$(call if_changed_dep,cc_lst_c)
 
+# Compile Rust sources (.rs)
+# ---------------------------------------------------------------------------
+
+rust_allowed_features := core_ffi_c
+
+rust_common_cmd = \
+	RUST_MODFILE=$(modfile) $(RUSTC_OR_CLIPPY) $(rust_flags) \
+	-Zallow-features=$(rust_allowed_features) \
+	-Zcrate-attr=no_std \
+	-Zcrate-attr='feature($(rust_allowed_features))' \
+	--extern alloc --extern kernel \
+	--crate-type rlib --out-dir $(obj) -L $(objtree)/rust/ \
+	--crate-name $(basename $(notdir $@))
+
+rust_handle_depfile = \
+	mv $(obj)/$(basename $(notdir $@)).d $(depfile); \
+	sed -i '/^\#/d' $(depfile)
+
+# `--emit=obj`, `--emit=asm` and `--emit=llvm-ir` imply a single codegen unit
+# will be used. We explicitly request `-Ccodegen-units=1` in any case, and
+# the compiler shows a warning if it is not 1. However, if we ever stop
+# requesting it explicitly and we start using some other `--emit` that does not
+# imply it (and for which codegen is performed), then we would be out of sync,
+# i.e. the outputs we would get for the different single targets (e.g. `.ll`)
+# would not match each other.
+
+quiet_cmd_rustc_o_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@
+      cmd_rustc_o_rs = \
+	$(rust_common_cmd) --emit=dep-info,obj $<; \
+	$(rust_handle_depfile)
+
+$(obj)/%.o: $(src)/%.rs FORCE
+	$(call if_changed_dep,rustc_o_rs)
+
+quiet_cmd_rustc_rsi_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@
+      cmd_rustc_rsi_rs = \
+	$(rust_common_cmd) --emit=dep-info -Zunpretty=expanded $< >$@; \
+	command -v $(RUSTFMT) >/dev/null && $(RUSTFMT) $@; \
+	$(rust_handle_depfile)
+
+$(obj)/%.rsi: $(src)/%.rs FORCE
+	$(call if_changed_dep,rustc_rsi_rs)
+
+quiet_cmd_rustc_s_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@
+      cmd_rustc_s_rs = \
+	$(rust_common_cmd) --emit=dep-info,asm $<; \
+	$(rust_handle_depfile)
+
+$(obj)/%.s: $(src)/%.rs FORCE
+	$(call if_changed_dep,rustc_s_rs)
+
+quiet_cmd_rustc_ll_rs = $(RUSTC_OR_CLIPPY_QUIET) $(quiet_modtag) $@
+      cmd_rustc_ll_rs = \
+	$(rust_common_cmd) --emit=dep-info,llvm-ir $<; \
+	$(rust_handle_depfile)
+
+$(obj)/%.ll: $(src)/%.rs FORCE
+	$(call if_changed_dep,rustc_ll_rs)
+
 # Compile assembler sources (.S)
 # ---------------------------------------------------------------------------
 
diff --git a/scripts/Makefile.debug b/scripts/Makefile.debug
index 8cf1cb22dd93..332c486f705f 100644
--- a/scripts/Makefile.debug
+++ b/scripts/Makefile.debug
@@ -1,4 +1,6 @@
 DEBUG_CFLAGS	:=
+DEBUG_RUSTFLAGS	:=
+
 debug-flags-y	:= -g
 
 ifdef CONFIG_DEBUG_INFO_SPLIT
@@ -17,9 +19,12 @@ KBUILD_AFLAGS	+= $(debug-flags-y)
 
 ifdef CONFIG_DEBUG_INFO_REDUCED
 DEBUG_CFLAGS	+= -fno-var-tracking
+DEBUG_RUSTFLAGS	+= -Cdebuginfo=1
 ifdef CONFIG_CC_IS_GCC
 DEBUG_CFLAGS	+= -femit-struct-debug-baseonly
 endif
+else
+DEBUG_RUSTFLAGS	+= -Cdebuginfo=2
 endif
 
 ifdef CONFIG_DEBUG_INFO_COMPRESSED
@@ -30,3 +35,6 @@ endif
 
 KBUILD_CFLAGS	+= $(DEBUG_CFLAGS)
 export DEBUG_CFLAGS
+
+KBUILD_RUSTFLAGS += $(DEBUG_RUSTFLAGS)
+export DEBUG_RUSTFLAGS
diff --git a/scripts/Makefile.extrawarn b/scripts/Makefile.extrawarn
index 6ae482158bc4..52bd7df84fd6 100644
--- a/scripts/Makefile.extrawarn
+++ b/scripts/Makefile.extrawarn
@@ -64,6 +64,7 @@ KBUILD_CFLAGS += -Wno-sign-compare
 KBUILD_CFLAGS += $(call cc-disable-warning, pointer-to-enum-cast)
 KBUILD_CFLAGS += -Wno-tautological-constant-out-of-range-compare
 KBUILD_CFLAGS += $(call cc-disable-warning, unaligned-access)
+KBUILD_CFLAGS += $(call cc-disable-warning, cast-function-type-strict)
 endif
 
 endif
diff --git a/scripts/Makefile.host b/scripts/Makefile.host
index 278b4d6ac945..da133780b751 100644
--- a/scripts/Makefile.host
+++ b/scripts/Makefile.host
@@ -22,6 +22,8 @@ $(obj)/%.tab.c $(obj)/%.tab.h: $(src)/%.y FORCE
 # to preprocess a data file.
 #
 # Both C and C++ are supported, but preferred language is C for such utilities.
+# Rust is also supported, but it may only be used in scenarios where a Rust
+# toolchain is required to be available (e.g. when  `CONFIG_RUST` is enabled).
 #
 # Sample syntax (see Documentation/kbuild/makefiles.rst for reference)
 # hostprogs := bin2hex
@@ -37,15 +39,20 @@ $(obj)/%.tab.c $(obj)/%.tab.h: $(src)/%.y FORCE
 # qconf-objs      := menu.o
 # Will compile qconf as a C++ program, and menu as a C program.
 # They are linked as C++ code to the executable qconf
+#
+# hostprogs   := target
+# target-rust := y
+# Will compile `target` as a Rust program, using `target.rs` as the crate root.
+# The crate may consist of several source files.
 
 # C code
 # Executables compiled from a single .c file
 host-csingle	:= $(foreach m,$(hostprogs), \
-			$(if $($(m)-objs)$($(m)-cxxobjs),,$(m)))
+			$(if $($(m)-objs)$($(m)-cxxobjs)$($(m)-rust),,$(m)))
 
 # C executables linked based on several .o files
 host-cmulti	:= $(foreach m,$(hostprogs),\
-		   $(if $($(m)-cxxobjs),,$(if $($(m)-objs),$(m))))
+		   $(if $($(m)-cxxobjs)$($(m)-rust),,$(if $($(m)-objs),$(m))))
 
 # Object (.o) files compiled from .c files
 host-cobjs	:= $(sort $(foreach m,$(hostprogs),$($(m)-objs)))
@@ -58,11 +65,17 @@ host-cxxmulti	:= $(foreach m,$(hostprogs),$(if $($(m)-cxxobjs),$(m)))
 # C++ Object (.o) files compiled from .cc files
 host-cxxobjs	:= $(sort $(foreach m,$(host-cxxmulti),$($(m)-cxxobjs)))
 
+# Rust code
+# Executables compiled from a single Rust crate (which may consist of
+# one or more .rs files)
+host-rust	:= $(foreach m,$(hostprogs),$(if $($(m)-rust),$(m)))
+
 host-csingle	:= $(addprefix $(obj)/,$(host-csingle))
 host-cmulti	:= $(addprefix $(obj)/,$(host-cmulti))
 host-cobjs	:= $(addprefix $(obj)/,$(host-cobjs))
 host-cxxmulti	:= $(addprefix $(obj)/,$(host-cxxmulti))
 host-cxxobjs	:= $(addprefix $(obj)/,$(host-cxxobjs))
+host-rust	:= $(addprefix $(obj)/,$(host-rust))
 
 #####
 # Handle options to gcc. Support building with separate output directory
@@ -71,6 +84,8 @@ _hostc_flags   = $(KBUILD_HOSTCFLAGS)   $(HOST_EXTRACFLAGS)   \
                  $(HOSTCFLAGS_$(target-stem).o)
 _hostcxx_flags = $(KBUILD_HOSTCXXFLAGS) $(HOST_EXTRACXXFLAGS) \
                  $(HOSTCXXFLAGS_$(target-stem).o)
+_hostrust_flags = $(KBUILD_HOSTRUSTFLAGS) $(HOST_EXTRARUSTFLAGS) \
+                  $(HOSTRUSTFLAGS_$(target-stem))
 
 # $(objtree)/$(obj) for including generated headers from checkin source files
 ifeq ($(KBUILD_EXTMOD),)
@@ -82,6 +97,7 @@ endif
 
 hostc_flags    = -Wp,-MMD,$(depfile) $(_hostc_flags)
 hostcxx_flags  = -Wp,-MMD,$(depfile) $(_hostcxx_flags)
+hostrust_flags = $(_hostrust_flags)
 
 #####
 # Compile programs on the host
@@ -128,5 +144,17 @@ quiet_cmd_host-cxxobjs	= HOSTCXX $@
 $(host-cxxobjs): $(obj)/%.o: $(src)/%.cc FORCE
 	$(call if_changed_dep,host-cxxobjs)
 
+# Create executable from a single Rust crate (which may consist of
+# one or more `.rs` files)
+# host-rust -> Executable
+quiet_cmd_host-rust	= HOSTRUSTC $@
+      cmd_host-rust	= \
+	$(HOSTRUSTC) $(hostrust_flags) --emit=dep-info,link \
+		--out-dir=$(obj)/ $<; \
+	mv $(obj)/$(target-stem).d $(depfile); \
+	sed -i '/^\#/d' $(depfile)
+$(host-rust): $(obj)/%: $(src)/%.rs FORCE
+	$(call if_changed_dep,host-rust)
+
 targets += $(host-csingle) $(host-cmulti) $(host-cobjs) \
-	   $(host-cxxmulti) $(host-cxxobjs)
+	   $(host-cxxmulti) $(host-cxxobjs) $(host-rust)
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 3fb6a99e78c4..c88b98b5dc44 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -8,6 +8,7 @@ ldflags-y  += $(EXTRA_LDFLAGS)
 # flags that take effect in current and sub directories
 KBUILD_AFLAGS += $(subdir-asflags-y)
 KBUILD_CFLAGS += $(subdir-ccflags-y)
+KBUILD_RUSTFLAGS += $(subdir-rustflags-y)
 
 # Figure out what we need to build from the various variables
 # ===========================================================================
@@ -128,6 +129,10 @@ _c_flags       = $(filter-out $(CFLAGS_REMOVE_$(target-stem).o), \
                      $(filter-out $(ccflags-remove-y), \
                          $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) $(ccflags-y)) \
                      $(CFLAGS_$(target-stem).o))
+_rust_flags    = $(filter-out $(RUSTFLAGS_REMOVE_$(target-stem).o), \
+                     $(filter-out $(rustflags-remove-y), \
+                         $(KBUILD_RUSTFLAGS) $(rustflags-y)) \
+                     $(RUSTFLAGS_$(target-stem).o))
 _a_flags       = $(filter-out $(AFLAGS_REMOVE_$(target-stem).o), \
                      $(filter-out $(asflags-remove-y), \
                          $(KBUILD_CPPFLAGS) $(KBUILD_AFLAGS) $(asflags-y)) \
@@ -202,6 +207,11 @@ modkern_cflags =                                          \
 		$(KBUILD_CFLAGS_MODULE) $(CFLAGS_MODULE), \
 		$(KBUILD_CFLAGS_KERNEL) $(CFLAGS_KERNEL) $(modfile_flags))
 
+modkern_rustflags =                                              \
+	$(if $(part-of-module),                                   \
+		$(KBUILD_RUSTFLAGS_MODULE) $(RUSTFLAGS_MODULE), \
+		$(KBUILD_RUSTFLAGS_KERNEL) $(RUSTFLAGS_KERNEL))
+
 modkern_aflags = $(if $(part-of-module),				\
 			$(KBUILD_AFLAGS_MODULE) $(AFLAGS_MODULE),	\
 			$(KBUILD_AFLAGS_KERNEL) $(AFLAGS_KERNEL))
@@ -211,6 +221,8 @@ c_flags        = -Wp,-MMD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE)     \
 		 $(_c_flags) $(modkern_cflags)                           \
 		 $(basename_flags) $(modname_flags)
 
+rust_flags     = $(_rust_flags) $(modkern_rustflags) @$(objtree)/include/generated/rustc_cfg
+
 a_flags        = -Wp,-MMD,$(depfile) $(NOSTDINC_FLAGS) $(LINUXINCLUDE)     \
 		 $(_a_flags) $(modkern_aflags)
 
diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal
index 35100e981f4a..9a1fa6aa30fe 100644
--- a/scripts/Makefile.modfinal
+++ b/scripts/Makefile.modfinal
@@ -39,11 +39,13 @@ quiet_cmd_ld_ko_o = LD [M]  $@
 
 quiet_cmd_btf_ko = BTF [M] $@
       cmd_btf_ko = 							\
-	if [ -f vmlinux ]; then						\
+	if [ ! -f vmlinux ]; then					\
+		printf "Skipping BTF generation for %s due to unavailability of vmlinux\n" $@ 1>&2; \
+	elif [ -n "$(CONFIG_RUST)" ] && $(srctree)/scripts/is_rust_module.sh $@; then 		\
+		printf "Skipping BTF generation for %s because it's a Rust module\n" $@ 1>&2; \
+	else								\
 		LLVM_OBJCOPY="$(OBJCOPY)" $(PAHOLE) -J $(PAHOLE_FLAGS) --btf_base vmlinux $@; \
 		$(RESOLVE_BTFIDS) -b vmlinux $@; 			\
-	else								\
-		printf "Skipping BTF generation for %s due to unavailability of vmlinux\n" $@ 1>&2; \
 	fi;
 
 # Same as newer-prereqs, but allows to exclude specified extra dependencies
diff --git a/scripts/bpf_doc.py b/scripts/bpf_doc.py
index dfb260de17a8..d5c389df6045 100755
--- a/scripts/bpf_doc.py
+++ b/scripts/bpf_doc.py
@@ -10,6 +10,9 @@ from __future__ import print_function
 import argparse
 import re
 import sys, os
+import subprocess
+
+helpersDocStart = 'Start of BPF helper function descriptions:'
 
 class NoHelperFound(BaseException):
     pass
@@ -47,6 +50,10 @@ class Helper(APIElement):
     @desc: textual description of the helper function
     @ret: description of the return value of the helper function
     """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.enum_val = None
+
     def proto_break_down(self):
         """
         Break down helper function protocol into smaller chunks: return type,
@@ -89,6 +96,7 @@ class HeaderParser(object):
         self.commands = []
         self.desc_unique_helpers = set()
         self.define_unique_helpers = []
+        self.helper_enum_vals = {}
         self.desc_syscalls = []
         self.enum_syscalls = []
 
@@ -233,7 +241,7 @@ class HeaderParser(object):
         self.enum_syscalls = re.findall('(BPF\w+)+', bpf_cmd_str)
 
     def parse_desc_helpers(self):
-        self.seek_to('* Start of BPF helper function descriptions:',
+        self.seek_to(helpersDocStart,
                      'Could not find start of eBPF helper descriptions list')
         while True:
             try:
@@ -245,30 +253,54 @@ class HeaderParser(object):
                 break
 
     def parse_define_helpers(self):
-        # Parse the number of FN(...) in #define __BPF_FUNC_MAPPER to compare
-        # later with the number of unique function names present in description.
+        # Parse FN(...) in #define __BPF_FUNC_MAPPER to compare later with the
+        # number of unique function names present in description and use the
+        # correct enumeration value.
         # Note: seek_to(..) discards the first line below the target search text,
         # resulting in FN(unspec) being skipped and not added to self.define_unique_helpers.
         self.seek_to('#define __BPF_FUNC_MAPPER(FN)',
                      'Could not find start of eBPF helper definition list')
-        # Searches for either one or more FN(\w+) defines or a backslash for newline
-        p = re.compile('\s*(FN\(\w+\))+|\\\\')
+        # Searches for one FN(\w+) define or a backslash for newline
+        p = re.compile('\s*FN\((\w+)\)|\\\\')
         fn_defines_str = ''
+        i = 1  # 'unspec' is skipped as mentioned above
         while True:
             capture = p.match(self.line)
             if capture:
                 fn_defines_str += self.line
+                self.helper_enum_vals[capture.expand(r'bpf_\1')] = i
+                i += 1
             else:
                 break
             self.line = self.reader.readline()
         # Find the number of occurences of FN(\w+)
         self.define_unique_helpers = re.findall('FN\(\w+\)', fn_defines_str)
 
+    def assign_helper_values(self):
+        seen_helpers = set()
+        for helper in self.helpers:
+            proto = helper.proto_break_down()
+            name = proto['name']
+            try:
+                enum_val = self.helper_enum_vals[name]
+            except KeyError:
+                raise Exception("Helper %s is missing from enum bpf_func_id" % name)
+
+            # Enforce current practice of having the descriptions ordered
+            # by enum value.
+            seen_helpers.add(name)
+            desc_val = len(seen_helpers)
+            if desc_val != enum_val:
+                raise Exception("Helper %s comment order (#%d) must be aligned with its position (#%d) in enum bpf_func_id" % (name, desc_val, enum_val))
+
+            helper.enum_val = enum_val
+
     def run(self):
         self.parse_desc_syscall()
         self.parse_enum_syscall()
         self.parse_desc_helpers()
         self.parse_define_helpers()
+        self.assign_helper_values()
         self.reader.close()
 
 ###############################################################################
@@ -357,6 +389,31 @@ class PrinterRST(Printer):
 
         print('')
 
+    def get_kernel_version(self):
+        try:
+            version = subprocess.run(['git', 'describe'], cwd=linuxRoot,
+                                     capture_output=True, check=True)
+            version = version.stdout.decode().rstrip()
+        except:
+            try:
+                version = subprocess.run(['make', 'kernelversion'], cwd=linuxRoot,
+                                         capture_output=True, check=True)
+                version = version.stdout.decode().rstrip()
+            except:
+                return 'Linux'
+        return 'Linux {version}'.format(version=version)
+
+    def get_last_doc_update(self, delimiter):
+        try:
+            cmd = ['git', 'log', '-1', '--pretty=format:%cs', '--no-patch',
+                   '-L',
+                   '/{}/,/\*\//:include/uapi/linux/bpf.h'.format(delimiter)]
+            date = subprocess.run(cmd, cwd=linuxRoot,
+                                  capture_output=True, check=True)
+            return date.stdout.decode().rstrip()
+        except:
+            return ''
+
 class PrinterHelpersRST(PrinterRST):
     """
     A printer for dumping collected information about helpers as a ReStructured
@@ -378,6 +435,8 @@ list of eBPF helper functions
 -------------------------------------------------------------------------------
 
 :Manual section: 7
+:Version: {version}
+{date_field}{date}
 
 DESCRIPTION
 ===========
@@ -410,8 +469,13 @@ kernel at the top).
 HELPERS
 =======
 '''
+        kernelVersion = self.get_kernel_version()
+        lastUpdate = self.get_last_doc_update(helpersDocStart)
+
         PrinterRST.print_license(self)
-        print(header)
+        print(header.format(version=kernelVersion,
+                            date_field = ':Date: ' if lastUpdate else '',
+                            date=lastUpdate))
 
     def print_footer(self):
         footer = '''
@@ -761,7 +825,7 @@ class PrinterHelpers(Printer):
             comma = ', '
             print(one_arg, end='')
 
-        print(') = (void *) %d;' % len(self.seen_helpers))
+        print(') = (void *) %d;' % helper.enum_val)
         print('')
 
 ###############################################################################
diff --git a/scripts/cc-version.sh b/scripts/cc-version.sh
index f1952c522466..2401c86fcf53 100755
--- a/scripts/cc-version.sh
+++ b/scripts/cc-version.sh
@@ -1,13 +1,13 @@
 #!/bin/sh
 # SPDX-License-Identifier: GPL-2.0
 #
-# Print the compiler name and its version in a 5 or 6-digit form.
+# Print the C compiler name and its version in a 5 or 6-digit form.
 # Also, perform the minimum version check.
 
 set -e
 
-# Print the compiler name and some version components.
-get_compiler_info()
+# Print the C compiler name and some version components.
+get_c_compiler_info()
 {
 	cat <<- EOF | "$@" -E -P -x c - 2>/dev/null
 	#if defined(__clang__)
@@ -32,7 +32,7 @@ get_canonical_version()
 
 # $@ instead of $1 because multiple words might be given, e.g. CC="ccache gcc".
 orig_args="$@"
-set -- $(get_compiler_info "$@")
+set -- $(get_c_compiler_info "$@")
 
 name=$1
 
@@ -52,7 +52,7 @@ ICC)
 	min_version=$($min_tool_version icc)
 	;;
 *)
-	echo "$orig_args: unknown compiler" >&2
+	echo "$orig_args: unknown C compiler" >&2
 	exit 1
 	;;
 esac
@@ -62,7 +62,7 @@ min_cversion=$(get_canonical_version $min_version)
 
 if [ "$cversion" -lt "$min_cversion" ]; then
 	echo >&2 "***"
-	echo >&2 "*** Compiler is too old."
+	echo >&2 "*** C compiler is too old."
 	echo >&2 "***   Your $name version:    $version"
 	echo >&2 "***   Minimum $name version: $min_version"
 	echo >&2 "***"
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 79e759aac543..c8a616a9d034 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -3616,7 +3616,7 @@ sub process {
 				my $comment = "";
 				if ($realfile =~ /\.(h|s|S)$/) {
 					$comment = '/*';
-				} elsif ($realfile =~ /\.(c|dts|dtsi)$/) {
+				} elsif ($realfile =~ /\.(c|rs|dts|dtsi)$/) {
 					$comment = '//';
 				} elsif (($checklicenseline == 2) || $realfile =~ /\.(sh|pl|py|awk|tc|yaml)$/) {
 					$comment = '#';
@@ -3664,7 +3664,7 @@ sub process {
 		}
 
 # check we are in a valid source file if not then ignore this hunk
-		next if ($realfile !~ /\.(h|c|s|S|sh|dtsi|dts)$/);
+		next if ($realfile !~ /\.(h|c|rs|s|S|sh|dtsi|dts)$/);
 
 # check for using SPDX-License-Identifier on the wrong line number
 		if ($realline != $checklicenseline &&
@@ -3751,7 +3751,7 @@ sub process {
 		if ($realfile =~ /\.S$/ &&
 		    $line =~ /^\+\s*(?:[A-Z]+_)?SYM_[A-Z]+_(?:START|END)(?:_[A-Z_]+)?\s*\(\s*\.L/) {
 			WARN("AVOID_L_PREFIX",
-			     "Avoid using '.L' prefixed local symbol names for denoting a range of code via 'SYM_*_START/END' annotations; see Documentation/asm-annotations.rst\n" . $herecurr);
+			     "Avoid using '.L' prefixed local symbol names for denoting a range of code via 'SYM_*_START/END' annotations; see Documentation/core-api/asm-annotations.rst\n" . $herecurr);
 		}
 
 # check we are in a valid source file C or perl if not then ignore this hunk
@@ -4695,12 +4695,12 @@ sub process {
 			}
 		}
 
-# avoid BUG() or BUG_ON()
-		if ($line =~ /\b(?:BUG|BUG_ON)\b/) {
+# do not use BUG() or variants
+		if ($line =~ /\b(?!AA_|BUILD_|DCCP_|IDA_|KVM_|RWLOCK_|snd_|SPIN_)(?:[a-zA-Z_]*_)?BUG(?:_ON)?(?:_[A-Z_]+)?\s*\(/) {
 			my $msg_level = \&WARN;
 			$msg_level = \&CHK if ($file);
 			&{$msg_level}("AVOID_BUG",
-				      "Avoid crashing the kernel - try using WARN_ON & recovery code rather than BUG() or BUG_ON()\n" . $herecurr);
+				      "Do not crash the kernel unless it is absolutely unavoidable--use WARN_ON_ONCE() plus recovery code (if feasible) instead of BUG() or variants\n" . $herecurr);
 		}
 
 # avoid LINUX_VERSION_CODE
@@ -6783,15 +6783,19 @@ sub process {
 				}
 				if ($bad_specifier ne "") {
 					my $stat_real = get_stat_real($linenr, $lc);
+					my $msg_level = \&WARN;
 					my $ext_type = "Invalid";
 					my $use = "";
 					if ($bad_specifier =~ /p[Ff]/) {
 						$use = " - use %pS instead";
 						$use =~ s/pS/ps/ if ($bad_specifier =~ /pf/);
+					} elsif ($bad_specifier =~ /pA/) {
+						$use =  " - '%pA' is only intended to be used from Rust code";
+						$msg_level = \&ERROR;
 					}
 
-					WARN("VSPRINTF_POINTER_EXTENSION",
-					     "$ext_type vsprintf pointer extension '$bad_specifier'$use\n" . "$here\n$stat_real\n");
+					&{$msg_level}("VSPRINTF_POINTER_EXTENSION",
+						      "$ext_type vsprintf pointer extension '$bad_specifier'$use\n" . "$here\n$stat_real\n");
 				}
 			}
 		}
diff --git a/scripts/decode_stacktrace.sh b/scripts/decode_stacktrace.sh
index 7075e26ab2c4..564c5632e1a2 100755
--- a/scripts/decode_stacktrace.sh
+++ b/scripts/decode_stacktrace.sh
@@ -8,6 +8,14 @@ usage() {
 	echo "	$0 -r <release> | <vmlinux> [<base path>|auto] [<modules path>]"
 }
 
+# Try to find a Rust demangler
+if type llvm-cxxfilt >/dev/null 2>&1 ; then
+	cppfilt=llvm-cxxfilt
+elif type c++filt >/dev/null 2>&1 ; then
+	cppfilt=c++filt
+	cppfilt_opts=-i
+fi
+
 if [[ $1 == "-r" ]] ; then
 	vmlinux=""
 	basepath="auto"
@@ -180,6 +188,12 @@ parse_symbol() {
 	# In the case of inlines, move everything to same line
 	code=${code//$'\n'/' '}
 
+	# Demangle if the name looks like a Rust symbol and if
+	# we got a Rust demangler
+	if [[ $name =~ ^_R && $cppfilt != "" ]] ; then
+		name=$("$cppfilt" "$cppfilt_opts" "$name")
+	fi
+
 	# Replace old address with pretty line numbers
 	symbol="$segment$name ($code)"
 }
diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
new file mode 100755
index 000000000000..75bb611bd751
--- /dev/null
+++ b/scripts/generate_rust_analyzer.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+"""generate_rust_analyzer - Generates the `rust-project.json` file for `rust-analyzer`.
+"""
+
+import argparse
+import json
+import logging
+import pathlib
+import sys
+
+def generate_crates(srctree, objtree, sysroot_src):
+    # Generate the configuration list.
+    cfg = []
+    with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
+        for line in fd:
+            line = line.replace("--cfg=", "")
+            line = line.replace("\n", "")
+            cfg.append(line)
+
+    # Now fill the crates list -- dependencies need to come first.
+    #
+    # Avoid O(n^2) iterations by keeping a map of indexes.
+    crates = []
+    crates_indexes = {}
+
+    def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=True, is_proc_macro=False):
+        crates_indexes[display_name] = len(crates)
+        crates.append({
+            "display_name": display_name,
+            "root_module": str(root_module),
+            "is_workspace_member": is_workspace_member,
+            "is_proc_macro": is_proc_macro,
+            "deps": [{"crate": crates_indexes[dep], "name": dep} for dep in deps],
+            "cfg": cfg,
+            "edition": "2021",
+            "env": {
+                "RUST_MODFILE": "This is only for rust-analyzer"
+            }
+        })
+
+    # First, the ones in `rust/` since they are a bit special.
+    append_crate(
+        "core",
+        sysroot_src / "core" / "src" / "lib.rs",
+        [],
+        is_workspace_member=False,
+    )
+
+    append_crate(
+        "compiler_builtins",
+        srctree / "rust" / "compiler_builtins.rs",
+        [],
+    )
+
+    append_crate(
+        "alloc",
+        srctree / "rust" / "alloc" / "lib.rs",
+        ["core", "compiler_builtins"],
+    )
+
+    append_crate(
+        "macros",
+        srctree / "rust" / "macros" / "lib.rs",
+        [],
+        is_proc_macro=True,
+    )
+    crates[-1]["proc_macro_dylib_path"] = "rust/libmacros.so"
+
+    append_crate(
+        "bindings",
+        srctree / "rust"/ "bindings" / "lib.rs",
+        ["core"],
+        cfg=cfg,
+    )
+    crates[-1]["env"]["OBJTREE"] = str(objtree.resolve(True))
+
+    append_crate(
+        "kernel",
+        srctree / "rust" / "kernel" / "lib.rs",
+        ["core", "alloc", "macros", "bindings"],
+        cfg=cfg,
+    )
+    crates[-1]["source"] = {
+        "include_dirs": [
+            str(srctree / "rust" / "kernel"),
+            str(objtree / "rust")
+        ],
+        "exclude_dirs": [],
+    }
+
+    # Then, the rest outside of `rust/`.
+    #
+    # We explicitly mention the top-level folders we want to cover.
+    for folder in ("samples", "drivers"):
+        for path in (srctree / folder).rglob("*.rs"):
+            logging.info("Checking %s", path)
+            name = path.name.replace(".rs", "")
+
+            # Skip those that are not crate roots.
+            if f"{name}.o" not in open(path.parent / "Makefile").read():
+                continue
+
+            logging.info("Adding %s", name)
+            append_crate(
+                name,
+                path,
+                ["core", "alloc", "kernel"],
+                cfg=cfg,
+            )
+
+    return crates
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--verbose', '-v', action='store_true')
+    parser.add_argument("srctree", type=pathlib.Path)
+    parser.add_argument("objtree", type=pathlib.Path)
+    parser.add_argument("sysroot_src", type=pathlib.Path)
+    args = parser.parse_args()
+
+    logging.basicConfig(
+        format="[%(asctime)s] [%(levelname)s] %(message)s",
+        level=logging.INFO if args.verbose else logging.WARNING
+    )
+
+    rust_project = {
+        "crates": generate_crates(args.srctree, args.objtree, args.sysroot_src),
+        "sysroot_src": str(args.sysroot_src),
+    }
+
+    json.dump(rust_project, sys.stdout, sort_keys=True, indent=4)
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/generate_rust_target.rs b/scripts/generate_rust_target.rs
new file mode 100644
index 000000000000..3c6cbe2b278d
--- /dev/null
+++ b/scripts/generate_rust_target.rs
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! The custom target specification file generator for `rustc`.
+//!
+//! To configure a target from scratch, a JSON-encoded file has to be passed
+//! to `rustc` (introduced in [RFC 131]). These options and the file itself are
+//! unstable. Eventually, `rustc` should provide a way to do this in a stable
+//! manner. For instance, via command-line arguments. Therefore, this file
+//! should avoid using keys which can be set via `-C` or `-Z` options.
+//!
+//! [RFC 131]: https://rust-lang.github.io/rfcs/0131-target-specification.html
+
+use std::{
+    collections::HashMap,
+    fmt::{Display, Formatter, Result},
+    io::BufRead,
+};
+
+enum Value {
+    Boolean(bool),
+    Number(i32),
+    String(String),
+    Object(Object),
+}
+
+type Object = Vec<(String, Value)>;
+
+/// Minimal "almost JSON" generator (e.g. no `null`s, no arrays, no escaping),
+/// enough for this purpose.
+impl Display for Value {
+    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
+        match self {
+            Value::Boolean(boolean) => write!(formatter, "{}", boolean),
+            Value::Number(number) => write!(formatter, "{}", number),
+            Value::String(string) => write!(formatter, "\"{}\"", string),
+            Value::Object(object) => {
+                formatter.write_str("{")?;
+                if let [ref rest @ .., ref last] = object[..] {
+                    for (key, value) in rest {
+                        write!(formatter, "\"{}\": {},", key, value)?;
+                    }
+                    write!(formatter, "\"{}\": {}", last.0, last.1)?;
+                }
+                formatter.write_str("}")
+            }
+        }
+    }
+}
+
+struct TargetSpec(Object);
+
+impl TargetSpec {
+    fn new() -> TargetSpec {
+        TargetSpec(Vec::new())
+    }
+}
+
+trait Push<T> {
+    fn push(&mut self, key: &str, value: T);
+}
+
+impl Push<bool> for TargetSpec {
+    fn push(&mut self, key: &str, value: bool) {
+        self.0.push((key.to_string(), Value::Boolean(value)));
+    }
+}
+
+impl Push<i32> for TargetSpec {
+    fn push(&mut self, key: &str, value: i32) {
+        self.0.push((key.to_string(), Value::Number(value)));
+    }
+}
+
+impl Push<String> for TargetSpec {
+    fn push(&mut self, key: &str, value: String) {
+        self.0.push((key.to_string(), Value::String(value)));
+    }
+}
+
+impl Push<&str> for TargetSpec {
+    fn push(&mut self, key: &str, value: &str) {
+        self.push(key, value.to_string());
+    }
+}
+
+impl Push<Object> for TargetSpec {
+    fn push(&mut self, key: &str, value: Object) {
+        self.0.push((key.to_string(), Value::Object(value)));
+    }
+}
+
+impl Display for TargetSpec {
+    fn fmt(&self, formatter: &mut Formatter<'_>) -> Result {
+        // We add some newlines for clarity.
+        formatter.write_str("{\n")?;
+        if let [ref rest @ .., ref last] = self.0[..] {
+            for (key, value) in rest {
+                write!(formatter, "    \"{}\": {},\n", key, value)?;
+            }
+            write!(formatter, "    \"{}\": {}\n", last.0, last.1)?;
+        }
+        formatter.write_str("}")
+    }
+}
+
+struct KernelConfig(HashMap<String, String>);
+
+impl KernelConfig {
+    /// Parses `include/config/auto.conf` from `stdin`.
+    fn from_stdin() -> KernelConfig {
+        let mut result = HashMap::new();
+
+        let stdin = std::io::stdin();
+        let mut handle = stdin.lock();
+        let mut line = String::new();
+
+        loop {
+            line.clear();
+
+            if handle.read_line(&mut line).unwrap() == 0 {
+                break;
+            }
+
+            if line.starts_with('#') {
+                continue;
+            }
+
+            let (key, value) = line.split_once('=').expect("Missing `=` in line.");
+            result.insert(key.to_string(), value.trim_end_matches('\n').to_string());
+        }
+
+        KernelConfig(result)
+    }
+
+    /// Does the option exist in the configuration (any value)?
+    ///
+    /// The argument must be passed without the `CONFIG_` prefix.
+    /// This avoids repetition and it also avoids `fixdep` making us
+    /// depend on it.
+    fn has(&self, option: &str) -> bool {
+        let option = "CONFIG_".to_owned() + option;
+        self.0.contains_key(&option)
+    }
+}
+
+fn main() {
+    let cfg = KernelConfig::from_stdin();
+    let mut ts = TargetSpec::new();
+
+    // `llvm-target`s are taken from `scripts/Makefile.clang`.
+    if cfg.has("X86_64") {
+        ts.push("arch", "x86_64");
+        ts.push(
+            "data-layout",
+            "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128",
+        );
+        let mut features = "-3dnow,-3dnowa,-mmx,+soft-float".to_string();
+        if cfg.has("RETPOLINE") {
+            features += ",+retpoline-external-thunk";
+        }
+        ts.push("features", features);
+        ts.push("llvm-target", "x86_64-linux-gnu");
+        ts.push("target-pointer-width", "64");
+    } else {
+        panic!("Unsupported architecture");
+    }
+
+    ts.push("emit-debug-gdb-scripts", false);
+    ts.push("frame-pointer", "may-omit");
+    ts.push(
+        "stack-probes",
+        vec![("kind".to_string(), Value::String("none".to_string()))],
+    );
+
+    // Everything else is LE, whether `CPU_LITTLE_ENDIAN` is declared or not
+    // (e.g. x86). It is also `rustc`'s default.
+    if cfg.has("CPU_BIG_ENDIAN") {
+        ts.push("target-endian", "big");
+    }
+
+    println!("{}", ts);
+}
diff --git a/scripts/is_rust_module.sh b/scripts/is_rust_module.sh
new file mode 100755
index 000000000000..28b3831a7593
--- /dev/null
+++ b/scripts/is_rust_module.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+#
+# is_rust_module.sh module.ko
+#
+# Returns `0` if `module.ko` is a Rust module, `1` otherwise.
+
+set -e
+
+# Using the `16_` prefix ensures other symbols with the same substring
+# are not picked up (even if it would be unlikely). The last part is
+# used just in case LLVM decides to use the `.` suffix.
+#
+# In the future, checking for the `.comment` section may be another
+# option, see https://github.com/rust-lang/rust/pull/97550.
+${NM} "$*" | grep -qE '^[0-9a-fA-F]+ r _R[^[:space:]]+16___IS_RUST_MODULE[^[:space:]]*$'
diff --git a/scripts/kallsyms.c b/scripts/kallsyms.c
index f18e6dfc68c5..ff5e7810e437 100644
--- a/scripts/kallsyms.c
+++ b/scripts/kallsyms.c
@@ -27,7 +27,23 @@
 
 #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof(arr[0]))
 
-#define KSYM_NAME_LEN		128
+#define _stringify_1(x)	#x
+#define _stringify(x)	_stringify_1(x)
+
+#define KSYM_NAME_LEN		512
+
+/*
+ * A substantially bigger size than the current maximum.
+ *
+ * It cannot be defined as an expression because it gets stringified
+ * for the fscanf() format string. Therefore, a _Static_assert() is
+ * used instead to maintain the relationship with KSYM_NAME_LEN.
+ */
+#define KSYM_NAME_LEN_BUFFER	2048
+_Static_assert(
+	KSYM_NAME_LEN_BUFFER == KSYM_NAME_LEN * 4,
+	"Please keep KSYM_NAME_LEN_BUFFER in sync with KSYM_NAME_LEN"
+);
 
 struct sym_entry {
 	unsigned long long addr;
@@ -119,6 +135,7 @@ static bool is_ignored_symbol(const char *name, char type)
 		"__ThumbV7PILongThunk_",
 		"__LA25Thunk_",		/* mips lld */
 		"__microLA25Thunk_",
+		"__kcfi_typeid_",	/* CFI type identifiers */
 		NULL
 	};
 
@@ -198,15 +215,15 @@ static void check_symbol_range(const char *sym, unsigned long long addr,
 
 static struct sym_entry *read_symbol(FILE *in)
 {
-	char name[500], type;
+	char name[KSYM_NAME_LEN_BUFFER+1], type;
 	unsigned long long addr;
 	unsigned int len;
 	struct sym_entry *sym;
 	int rc;
 
-	rc = fscanf(in, "%llx %c %499s\n", &addr, &type, name);
+	rc = fscanf(in, "%llx %c %" _stringify(KSYM_NAME_LEN_BUFFER) "s\n", &addr, &type, name);
 	if (rc != 3) {
-		if (rc != EOF && fgets(name, 500, in) == NULL)
+		if (rc != EOF && fgets(name, ARRAY_SIZE(name), in) == NULL)
 			fprintf(stderr, "Read error or end of file.\n");
 		return NULL;
 	}
@@ -471,12 +488,35 @@ static void write_src(void)
 		if ((i & 0xFF) == 0)
 			markers[i >> 8] = off;
 
-		printf("\t.byte 0x%02x", table[i]->len);
+		/* There cannot be any symbol of length zero. */
+		if (table[i]->len == 0) {
+			fprintf(stderr, "kallsyms failure: "
+				"unexpected zero symbol length\n");
+			exit(EXIT_FAILURE);
+		}
+
+		/* Only lengths that fit in up-to-two-byte ULEB128 are supported. */
+		if (table[i]->len > 0x3FFF) {
+			fprintf(stderr, "kallsyms failure: "
+				"unexpected huge symbol length\n");
+			exit(EXIT_FAILURE);
+		}
+
+		/* Encode length with ULEB128. */
+		if (table[i]->len <= 0x7F) {
+			/* Most symbols use a single byte for the length. */
+			printf("\t.byte 0x%02x", table[i]->len);
+			off += table[i]->len + 1;
+		} else {
+			/* "Big" symbols use two bytes. */
+			printf("\t.byte 0x%02x, 0x%02x",
+				(table[i]->len & 0x7F) | 0x80,
+				(table[i]->len >> 7) & 0x7F);
+			off += table[i]->len + 2;
+		}
 		for (k = 0; k < table[i]->len; k++)
 			printf(", 0x%02x", table[i]->sym[k]);
 		printf("\n");
-
-		off += table[i]->len + 1;
 	}
 	printf("\n");
 
diff --git a/scripts/kconfig/confdata.c b/scripts/kconfig/confdata.c
index c4340c90e172..b7c9f1dd5e42 100644
--- a/scripts/kconfig/confdata.c
+++ b/scripts/kconfig/confdata.c
@@ -216,6 +216,13 @@ static const char *conf_get_autoheader_name(void)
 	return name ? name : "include/generated/autoconf.h";
 }
 
+static const char *conf_get_rustccfg_name(void)
+{
+	char *name = getenv("KCONFIG_RUSTCCFG");
+
+	return name ? name : "include/generated/rustc_cfg";
+}
+
 static int conf_set_sym_val(struct symbol *sym, int def, int def_flags, char *p)
 {
 	char *p2;
@@ -605,6 +612,9 @@ static const struct comment_style comment_style_c = {
 
 static void conf_write_heading(FILE *fp, const struct comment_style *cs)
 {
+	if (!cs)
+		return;
+
 	fprintf(fp, "%s\n", cs->prefix);
 
 	fprintf(fp, "%s Automatically generated file; DO NOT EDIT.\n",
@@ -745,6 +755,65 @@ static void print_symbol_for_c(FILE *fp, struct symbol *sym)
 	free(escaped);
 }
 
+static void print_symbol_for_rustccfg(FILE *fp, struct symbol *sym)
+{
+	const char *val;
+	const char *val_prefix = "";
+	char *val_prefixed = NULL;
+	size_t val_prefixed_len;
+	char *escaped = NULL;
+
+	if (sym->type == S_UNKNOWN)
+		return;
+
+	val = sym_get_string_value(sym);
+
+	switch (sym->type) {
+	case S_BOOLEAN:
+	case S_TRISTATE:
+		/*
+		 * We do not care about disabled ones, i.e. no need for
+		 * what otherwise are "comments" in other printers.
+		 */
+		if (*val == 'n')
+			return;
+
+		/*
+		 * To have similar functionality to the C macro `IS_ENABLED()`
+		 * we provide an empty `--cfg CONFIG_X` here in both `y`
+		 * and `m` cases.
+		 *
+		 * Then, the common `fprintf()` below will also give us
+		 * a `--cfg CONFIG_X="y"` or `--cfg CONFIG_X="m"`, which can
+		 * be used as the equivalent of `IS_BUILTIN()`/`IS_MODULE()`.
+		 */
+		fprintf(fp, "--cfg=%s%s\n", CONFIG_, sym->name);
+		break;
+	case S_HEX:
+		if (val[0] != '0' || (val[1] != 'x' && val[1] != 'X'))
+			val_prefix = "0x";
+		break;
+	default:
+		break;
+	}
+
+	if (strlen(val_prefix) > 0) {
+		val_prefixed_len = strlen(val) + strlen(val_prefix) + 1;
+		val_prefixed = xmalloc(val_prefixed_len);
+		snprintf(val_prefixed, val_prefixed_len, "%s%s", val_prefix, val);
+		val = val_prefixed;
+	}
+
+	/* All values get escaped: the `--cfg` option only takes strings */
+	escaped = escape_string_value(val);
+	val = escaped;
+
+	fprintf(fp, "--cfg=%s%s=%s\n", CONFIG_, sym->name, val);
+
+	free(escaped);
+	free(val_prefixed);
+}
+
 /*
  * Write out a minimal config.
  * All values that has default values are skipped as this is redundant.
@@ -1132,6 +1201,12 @@ int conf_write_autoconf(int overwrite)
 	if (ret)
 		return ret;
 
+	ret = __conf_write_autoconf(conf_get_rustccfg_name(),
+				    print_symbol_for_rustccfg,
+				    NULL);
+	if (ret)
+		return ret;
+
 	/*
 	 * Create include/config/auto.conf. This must be the last step because
 	 * Kbuild has a dependency on auto.conf and this marks the successful
diff --git a/scripts/min-tool-version.sh b/scripts/min-tool-version.sh
index 250925aab101..b6593eac5003 100755
--- a/scripts/min-tool-version.sh
+++ b/scripts/min-tool-version.sh
@@ -30,6 +30,12 @@ llvm)
 		echo 11.0.0
 	fi
 	;;
+rustc)
+	echo 1.62.0
+	;;
+bindgen)
+	echo 0.56.0
+	;;
 *)
 	echo "$1: unknown tool" >&2
 	exit 1
diff --git a/scripts/module.lds.S b/scripts/module.lds.S
index 3a3aa2354ed8..da4bddd26171 100644
--- a/scripts/module.lds.S
+++ b/scripts/module.lds.S
@@ -3,20 +3,10 @@
  * Archs are free to supply their own linker scripts.  ld will
  * combine them automatically.
  */
-#ifdef CONFIG_CFI_CLANG
-# include <asm/page.h>
-# define ALIGN_CFI 		ALIGN(PAGE_SIZE)
-# define SANITIZER_DISCARDS	*(.eh_frame)
-#else
-# define ALIGN_CFI
-# define SANITIZER_DISCARDS
-#endif
-
 SECTIONS {
 	/DISCARD/ : {
 		*(.discard)
 		*(.discard.*)
-		SANITIZER_DISCARDS
 	}
 
 	__ksymtab		0 : { *(SORT(___ksymtab+*)) }
@@ -33,6 +23,10 @@ SECTIONS {
 
 	__patchable_function_entries : { *(__patchable_function_entries) }
 
+#ifdef CONFIG_ARCH_USES_CFI_TRAPS
+	__kcfi_traps 		: { KEEP(*(.kcfi_traps)) }
+#endif
+
 #ifdef CONFIG_LTO_CLANG
 	/*
 	 * With CONFIG_LTO_CLANG, LLD always enables -fdata-sections and
@@ -53,15 +47,6 @@ SECTIONS {
 		*(.rodata .rodata.[0-9a-zA-Z_]*)
 		*(.rodata..L*)
 	}
-
-	/*
-	 * With CONFIG_CFI_CLANG, we assume __cfi_check is at the beginning
-	 * of the .text section, and is aligned to PAGE_SIZE.
-	 */
-	.text : ALIGN_CFI {
-		*(.text.__cfi_check)
-		*(.text .text.[0-9a-zA-Z_]* .text..L.cfi*)
-	}
 #endif
 }
 
diff --git a/scripts/rust_is_available.sh b/scripts/rust_is_available.sh
new file mode 100755
index 000000000000..aebbf1913970
--- /dev/null
+++ b/scripts/rust_is_available.sh
@@ -0,0 +1,160 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+#
+# Tests whether a suitable Rust toolchain is available.
+#
+# Pass `-v` for human output and more checks (as warnings).
+
+set -e
+
+min_tool_version=$(dirname $0)/min-tool-version.sh
+
+# Convert the version string x.y.z to a canonical up-to-7-digits form.
+#
+# Note that this function uses one more digit (compared to other
+# instances in other version scripts) to give a bit more space to
+# `rustc` since it will reach 1.100.0 in late 2026.
+get_canonical_version()
+{
+	IFS=.
+	set -- $1
+	echo $((100000 * $1 + 100 * $2 + $3))
+}
+
+# Check that the Rust compiler exists.
+if ! command -v "$RUSTC" >/dev/null; then
+	if [ "$1" = -v ]; then
+		echo >&2 "***"
+		echo >&2 "*** Rust compiler '$RUSTC' could not be found."
+		echo >&2 "***"
+	fi
+	exit 1
+fi
+
+# Check that the Rust bindings generator exists.
+if ! command -v "$BINDGEN" >/dev/null; then
+	if [ "$1" = -v ]; then
+		echo >&2 "***"
+		echo >&2 "*** Rust bindings generator '$BINDGEN' could not be found."
+		echo >&2 "***"
+	fi
+	exit 1
+fi
+
+# Check that the Rust compiler version is suitable.
+#
+# Non-stable and distributions' versions may have a version suffix, e.g. `-dev`.
+rust_compiler_version=$( \
+	LC_ALL=C "$RUSTC" --version 2>/dev/null \
+		| head -n 1 \
+		| grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \
+)
+rust_compiler_min_version=$($min_tool_version rustc)
+rust_compiler_cversion=$(get_canonical_version $rust_compiler_version)
+rust_compiler_min_cversion=$(get_canonical_version $rust_compiler_min_version)
+if [ "$rust_compiler_cversion" -lt "$rust_compiler_min_cversion" ]; then
+	if [ "$1" = -v ]; then
+		echo >&2 "***"
+		echo >&2 "*** Rust compiler '$RUSTC' is too old."
+		echo >&2 "***   Your version:    $rust_compiler_version"
+		echo >&2 "***   Minimum version: $rust_compiler_min_version"
+		echo >&2 "***"
+	fi
+	exit 1
+fi
+if [ "$1" = -v ] && [ "$rust_compiler_cversion" -gt "$rust_compiler_min_cversion" ]; then
+	echo >&2 "***"
+	echo >&2 "*** Rust compiler '$RUSTC' is too new. This may or may not work."
+	echo >&2 "***   Your version:     $rust_compiler_version"
+	echo >&2 "***   Expected version: $rust_compiler_min_version"
+	echo >&2 "***"
+fi
+
+# Check that the Rust bindings generator is suitable.
+#
+# Non-stable and distributions' versions may have a version suffix, e.g. `-dev`.
+rust_bindings_generator_version=$( \
+	LC_ALL=C "$BINDGEN" --version 2>/dev/null \
+		| head -n 1 \
+		| grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \
+)
+rust_bindings_generator_min_version=$($min_tool_version bindgen)
+rust_bindings_generator_cversion=$(get_canonical_version $rust_bindings_generator_version)
+rust_bindings_generator_min_cversion=$(get_canonical_version $rust_bindings_generator_min_version)
+if [ "$rust_bindings_generator_cversion" -lt "$rust_bindings_generator_min_cversion" ]; then
+	if [ "$1" = -v ]; then
+		echo >&2 "***"
+		echo >&2 "*** Rust bindings generator '$BINDGEN' is too old."
+		echo >&2 "***   Your version:    $rust_bindings_generator_version"
+		echo >&2 "***   Minimum version: $rust_bindings_generator_min_version"
+		echo >&2 "***"
+	fi
+	exit 1
+fi
+if [ "$1" = -v ] && [ "$rust_bindings_generator_cversion" -gt "$rust_bindings_generator_min_cversion" ]; then
+	echo >&2 "***"
+	echo >&2 "*** Rust bindings generator '$BINDGEN' is too new. This may or may not work."
+	echo >&2 "***   Your version:     $rust_bindings_generator_version"
+	echo >&2 "***   Expected version: $rust_bindings_generator_min_version"
+	echo >&2 "***"
+fi
+
+# Check that the `libclang` used by the Rust bindings generator is suitable.
+bindgen_libclang_version=$( \
+	LC_ALL=C "$BINDGEN" $(dirname $0)/rust_is_available_bindgen_libclang.h 2>&1 >/dev/null \
+		| grep -F 'clang version ' \
+		| grep -oE '[0-9]+\.[0-9]+\.[0-9]+' \
+		| head -n 1 \
+)
+bindgen_libclang_min_version=$($min_tool_version llvm)
+bindgen_libclang_cversion=$(get_canonical_version $bindgen_libclang_version)
+bindgen_libclang_min_cversion=$(get_canonical_version $bindgen_libclang_min_version)
+if [ "$bindgen_libclang_cversion" -lt "$bindgen_libclang_min_cversion" ]; then
+	if [ "$1" = -v ]; then
+		echo >&2 "***"
+		echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN') is too old."
+		echo >&2 "***   Your version:    $bindgen_libclang_version"
+		echo >&2 "***   Minimum version: $bindgen_libclang_min_version"
+		echo >&2 "***"
+	fi
+	exit 1
+fi
+
+# If the C compiler is Clang, then we can also check whether its version
+# matches the `libclang` version used by the Rust bindings generator.
+#
+# In the future, we might be able to perform a full version check, see
+# https://github.com/rust-lang/rust-bindgen/issues/2138.
+if [ "$1" = -v ]; then
+	cc_name=$($(dirname $0)/cc-version.sh "$CC" | cut -f1 -d' ')
+	if [ "$cc_name" = Clang ]; then
+		clang_version=$( \
+			LC_ALL=C "$CC" --version 2>/dev/null \
+				| sed -nE '1s:.*version ([0-9]+\.[0-9]+\.[0-9]+).*:\1:p'
+		)
+		if [ "$clang_version" != "$bindgen_libclang_version" ]; then
+			echo >&2 "***"
+			echo >&2 "*** libclang (used by the Rust bindings generator '$BINDGEN')"
+			echo >&2 "*** version does not match Clang's. This may be a problem."
+			echo >&2 "***   libclang version: $bindgen_libclang_version"
+			echo >&2 "***   Clang version:    $clang_version"
+			echo >&2 "***"
+		fi
+	fi
+fi
+
+# Check that the source code for the `core` standard library exists.
+#
+# `$KRUSTFLAGS` is passed in case the user added `--sysroot`.
+rustc_sysroot=$("$RUSTC" $KRUSTFLAGS --print sysroot)
+rustc_src=${RUST_LIB_SRC:-"$rustc_sysroot/lib/rustlib/src/rust/library"}
+rustc_src_core="$rustc_src/core/src/lib.rs"
+if [ ! -e "$rustc_src_core" ]; then
+	if [ "$1" = -v ]; then
+		echo >&2 "***"
+		echo >&2 "*** Source code for the 'core' standard library could not be found"
+		echo >&2 "*** at '$rustc_src_core'."
+		echo >&2 "***"
+	fi
+	exit 1
+fi
diff --git a/scripts/rust_is_available_bindgen_libclang.h b/scripts/rust_is_available_bindgen_libclang.h
new file mode 100644
index 000000000000..0ef6db10d674
--- /dev/null
+++ b/scripts/rust_is_available_bindgen_libclang.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#pragma message("clang version " __clang_version__)
diff --git a/scripts/selinux/install_policy.sh b/scripts/selinux/install_policy.sh
index 2dccf141241d..24086793b0d8 100755
--- a/scripts/selinux/install_policy.sh
+++ b/scripts/selinux/install_policy.sh
@@ -31,8 +31,7 @@ fi
 if selinuxenabled; then
     echo "SELinux is already enabled"
     echo "This prevents safely relabeling all files."
-    echo "Boot with selinux=0 on the kernel command-line or"
-    echo "SELINUX=disabled in /etc/selinux/config."
+    echo "Boot with selinux=0 on the kernel command-line."
     exit 1
 fi
 
@@ -78,7 +77,7 @@ cd /etc/selinux/dummy/contexts/files
 $SF -F file_contexts /
 
 mounts=`cat /proc/$$/mounts | \
-	egrep "ext[234]|jfs|xfs|reiserfs|jffs2|gfs2|btrfs|f2fs|ocfs2" | \
+	grep -E "ext[234]|jfs|xfs|reiserfs|jffs2|gfs2|btrfs|f2fs|ocfs2" | \
 	awk '{ print $2 '}`
 $SF -F file_contexts $mounts
 
diff --git a/security/Kconfig.hardening b/security/Kconfig.hardening
index bd2aabb2c60f..995bc42003e6 100644
--- a/security/Kconfig.hardening
+++ b/security/Kconfig.hardening
@@ -22,11 +22,17 @@ menu "Memory initialization"
 config CC_HAS_AUTO_VAR_INIT_PATTERN
 	def_bool $(cc-option,-ftrivial-auto-var-init=pattern)
 
-config CC_HAS_AUTO_VAR_INIT_ZERO
-	# GCC ignores the -enable flag, so we can test for the feature with
-	# a single invocation using the flag, but drop it as appropriate in
-	# the Makefile, depending on the presence of Clang.
+config CC_HAS_AUTO_VAR_INIT_ZERO_BARE
+	def_bool $(cc-option,-ftrivial-auto-var-init=zero)
+
+config CC_HAS_AUTO_VAR_INIT_ZERO_ENABLER
+	# Clang 16 and later warn about using the -enable flag, but it
+	# is required before then.
 	def_bool $(cc-option,-ftrivial-auto-var-init=zero -enable-trivial-auto-var-init-zero-knowing-it-will-be-removed-from-clang)
+	depends on !CC_HAS_AUTO_VAR_INIT_ZERO_BARE
+
+config CC_HAS_AUTO_VAR_INIT_ZERO
+	def_bool CC_HAS_AUTO_VAR_INIT_ZERO_BARE || CC_HAS_AUTO_VAR_INIT_ZERO_ENABLER
 
 choice
 	prompt "Initialize kernel stack variables at function entry"
diff --git a/security/apparmor/lsm.c b/security/apparmor/lsm.c
index e29cade7b662..f56070270c69 100644
--- a/security/apparmor/lsm.c
+++ b/security/apparmor/lsm.c
@@ -614,7 +614,7 @@ static int apparmor_sb_pivotroot(const struct path *old_path,
 	return error;
 }
 
-static int apparmor_getprocattr(struct task_struct *task, char *name,
+static int apparmor_getprocattr(struct task_struct *task, const char *name,
 				char **value)
 {
 	int error = -ENOENT;
diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c
index 2e6fb6e2ffd2..23d484e05e6f 100644
--- a/security/integrity/evm/evm_main.c
+++ b/security/integrity/evm/evm_main.c
@@ -457,10 +457,21 @@ static int evm_xattr_acl_change(struct user_namespace *mnt_userns,
 	int rc;
 
 	/*
-	 * user_ns is not relevant here, ACL_USER/ACL_GROUP don't have impact
-	 * on the inode mode (see posix_acl_equiv_mode()).
+	 * An earlier comment here mentioned that the idmappings for
+	 * ACL_{GROUP,USER} don't matter since EVM is only interested in the
+	 * mode stored as part of POSIX ACLs. Nonetheless, if it must translate
+	 * from the uapi POSIX ACL representation to the VFS internal POSIX ACL
+	 * representation it should do so correctly. There's no guarantee that
+	 * we won't change POSIX ACLs in a way that ACL_{GROUP,USER} matters
+	 * for the mode at some point and it's difficult to keep track of all
+	 * the LSM and integrity modules and what they do to POSIX ACLs.
+	 *
+	 * Frankly, EVM shouldn't try to interpret the uapi struct for POSIX
+	 * ACLs it received. It requires knowledge that only the VFS is
+	 * guaranteed to have.
 	 */
-	acl = posix_acl_from_xattr(&init_user_ns, xattr_value, xattr_value_len);
+	acl = vfs_set_acl_prepare(mnt_userns, i_user_ns(inode),
+				  xattr_value, xattr_value_len);
 	if (IS_ERR_OR_NULL(acl))
 		return 1;
 
diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c
index bde74fcecee3..3e0fbbd99534 100644
--- a/security/integrity/ima/ima_appraise.c
+++ b/security/integrity/ima/ima_appraise.c
@@ -750,22 +750,26 @@ int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name,
 	const struct evm_ima_xattr_data *xvalue = xattr_value;
 	int digsig = 0;
 	int result;
+	int err;
 
 	result = ima_protect_xattr(dentry, xattr_name, xattr_value,
 				   xattr_value_len);
 	if (result == 1) {
 		if (!xattr_value_len || (xvalue->type >= IMA_XATTR_LAST))
 			return -EINVAL;
+
+		err = validate_hash_algo(dentry, xvalue, xattr_value_len);
+		if (err)
+			return err;
+
 		digsig = (xvalue->type == EVM_IMA_XATTR_DIGSIG);
 	} else if (!strcmp(xattr_name, XATTR_NAME_EVM) && xattr_value_len > 0) {
 		digsig = (xvalue->type == EVM_XATTR_PORTABLE_DIGSIG);
 	}
 	if (result == 1 || evm_revalidate_status(xattr_name)) {
-		result = validate_hash_algo(dentry, xvalue, xattr_value_len);
-		if (result)
-			return result;
-
 		ima_reset_appraise_flags(d_backing_inode(dentry), digsig);
+		if (result == 1)
+			result = 0;
 	}
 	return result;
 }
diff --git a/security/integrity/platform_certs/load_uefi.c b/security/integrity/platform_certs/load_uefi.c
index 093894a640dc..b78753d27d8e 100644
--- a/security/integrity/platform_certs/load_uefi.c
+++ b/security/integrity/platform_certs/load_uefi.c
@@ -31,7 +31,7 @@ static const struct dmi_system_id uefi_skip_cert[] = {
 	{ UEFI_QUIRK_SKIP_CERT("Apple Inc.", "MacBookAir8,1") },
 	{ UEFI_QUIRK_SKIP_CERT("Apple Inc.", "MacBookAir8,2") },
 	{ UEFI_QUIRK_SKIP_CERT("Apple Inc.", "MacBookAir9,1") },
-	{ UEFI_QUIRK_SKIP_CERT("Apple Inc.", "MacMini8,1") },
+	{ UEFI_QUIRK_SKIP_CERT("Apple Inc.", "Macmini8,1") },
 	{ UEFI_QUIRK_SKIP_CERT("Apple Inc.", "MacPro7,1") },
 	{ UEFI_QUIRK_SKIP_CERT("Apple Inc.", "iMac20,1") },
 	{ UEFI_QUIRK_SKIP_CERT("Apple Inc.", "iMac20,2") },
diff --git a/security/keys/internal.h b/security/keys/internal.h
index 9b9cf3b6fcbb..3c1e7122076b 100644
--- a/security/keys/internal.h
+++ b/security/keys/internal.h
@@ -165,8 +165,6 @@ extern struct key *request_key_and_link(struct key_type *type,
 
 extern bool lookup_user_key_possessed(const struct key *key,
 				      const struct key_match_data *match_data);
-#define KEY_LOOKUP_CREATE	0x01
-#define KEY_LOOKUP_PARTIAL	0x02
 
 extern long join_session_keyring(const char *name);
 extern void key_change_session_keyring(struct callback_head *twork);
diff --git a/security/landlock/fs.c b/security/landlock/fs.c
index a9dbd99d9ee7..64ed7665455f 100644
--- a/security/landlock/fs.c
+++ b/security/landlock/fs.c
@@ -712,7 +712,7 @@ static inline access_mask_t maybe_remove(const struct dentry *const dentry)
  * allowed accesses in @layer_masks_dom.
  *
  * This is similar to check_access_path_dual() but much simpler because it only
- * handles walking on the same mount point and only check one set of accesses.
+ * handles walking on the same mount point and only checks one set of accesses.
  *
  * Returns:
  * - true if all the domain access rights are allowed for @dir;
diff --git a/security/landlock/syscalls.c b/security/landlock/syscalls.c
index 735a0865ea11..2ca0ccbd905a 100644
--- a/security/landlock/syscalls.c
+++ b/security/landlock/syscalls.c
@@ -149,10 +149,10 @@ static const struct file_operations ruleset_fops = {
  *
  * Possible returned errors are:
  *
- * - EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time;
- * - EINVAL: unknown @flags, or unknown access, or too small @size;
- * - E2BIG or EFAULT: @attr or @size inconsistencies;
- * - ENOMSG: empty &landlock_ruleset_attr.handled_access_fs.
+ * - %EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time;
+ * - %EINVAL: unknown @flags, or unknown access, or too small @size;
+ * - %E2BIG or %EFAULT: @attr or @size inconsistencies;
+ * - %ENOMSG: empty &landlock_ruleset_attr.handled_access_fs.
  */
 SYSCALL_DEFINE3(landlock_create_ruleset,
 		const struct landlock_ruleset_attr __user *const, attr,
@@ -280,7 +280,7 @@ out_fdput:
  * @ruleset_fd: File descriptor tied to the ruleset that should be extended
  *		with the new rule.
  * @rule_type: Identify the structure type pointed to by @rule_attr (only
- *             LANDLOCK_RULE_PATH_BENEATH for now).
+ *             %LANDLOCK_RULE_PATH_BENEATH for now).
  * @rule_attr: Pointer to a rule (only of type &struct
  *             landlock_path_beneath_attr for now).
  * @flags: Must be 0.
@@ -290,17 +290,17 @@ out_fdput:
  *
  * Possible returned errors are:
  *
- * - EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time;
- * - EINVAL: @flags is not 0, or inconsistent access in the rule (i.e.
+ * - %EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time;
+ * - %EINVAL: @flags is not 0, or inconsistent access in the rule (i.e.
  *   &landlock_path_beneath_attr.allowed_access is not a subset of the
  *   ruleset handled accesses);
- * - ENOMSG: Empty accesses (e.g. &landlock_path_beneath_attr.allowed_access);
- * - EBADF: @ruleset_fd is not a file descriptor for the current thread, or a
+ * - %ENOMSG: Empty accesses (e.g. &landlock_path_beneath_attr.allowed_access);
+ * - %EBADF: @ruleset_fd is not a file descriptor for the current thread, or a
  *   member of @rule_attr is not a file descriptor as expected;
- * - EBADFD: @ruleset_fd is not a ruleset file descriptor, or a member of
+ * - %EBADFD: @ruleset_fd is not a ruleset file descriptor, or a member of
  *   @rule_attr is not the expected file descriptor type;
- * - EPERM: @ruleset_fd has no write access to the underlying ruleset;
- * - EFAULT: @rule_attr inconsistency.
+ * - %EPERM: @ruleset_fd has no write access to the underlying ruleset;
+ * - %EFAULT: @rule_attr inconsistency.
  */
 SYSCALL_DEFINE4(landlock_add_rule, const int, ruleset_fd,
 		const enum landlock_rule_type, rule_type,
@@ -378,20 +378,20 @@ out_put_ruleset:
  * @flags: Must be 0.
  *
  * This system call enables to enforce a Landlock ruleset on the current
- * thread.  Enforcing a ruleset requires that the task has CAP_SYS_ADMIN in its
+ * thread.  Enforcing a ruleset requires that the task has %CAP_SYS_ADMIN in its
  * namespace or is running with no_new_privs.  This avoids scenarios where
  * unprivileged tasks can affect the behavior of privileged children.
  *
  * Possible returned errors are:
  *
- * - EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time;
- * - EINVAL: @flags is not 0.
- * - EBADF: @ruleset_fd is not a file descriptor for the current thread;
- * - EBADFD: @ruleset_fd is not a ruleset file descriptor;
- * - EPERM: @ruleset_fd has no read access to the underlying ruleset, or the
+ * - %EOPNOTSUPP: Landlock is supported by the kernel but disabled at boot time;
+ * - %EINVAL: @flags is not 0.
+ * - %EBADF: @ruleset_fd is not a file descriptor for the current thread;
+ * - %EBADFD: @ruleset_fd is not a ruleset file descriptor;
+ * - %EPERM: @ruleset_fd has no read access to the underlying ruleset, or the
  *   current thread is not running with no_new_privs, or it doesn't have
- *   CAP_SYS_ADMIN in its namespace.
- * - E2BIG: The maximum number of stacked rulesets is reached for the current
+ *   %CAP_SYS_ADMIN in its namespace.
+ * - %E2BIG: The maximum number of stacked rulesets is reached for the current
  *   thread.
  */
 SYSCALL_DEFINE2(landlock_restrict_self, const int, ruleset_fd, const __u32,
diff --git a/security/loadpin/Kconfig b/security/loadpin/Kconfig
index 70e7985b2561..6724eaba3d36 100644
--- a/security/loadpin/Kconfig
+++ b/security/loadpin/Kconfig
@@ -33,4 +33,9 @@ config SECURITY_LOADPIN_VERITY
 	  on the LoadPin securityfs entry 'dm-verity'. The ioctl
 	  expects a file descriptor of a file with verity digests as
 	  parameter. The file must be located on the pinned root and
-	  contain a comma separated list of digests.
+	  start with the line:
+
+	  # LOADPIN_TRUSTED_VERITY_ROOT_DIGESTS
+
+	  This is followed by the verity digests, with one digest per
+	  line.
diff --git a/security/loadpin/loadpin.c b/security/loadpin/loadpin.c
index 44521582dcba..de41621f4998 100644
--- a/security/loadpin/loadpin.c
+++ b/security/loadpin/loadpin.c
@@ -21,6 +21,8 @@
 #include <linux/dm-verity-loadpin.h>
 #include <uapi/linux/loadpin.h>
 
+#define VERITY_DIGEST_FILE_HEADER "# LOADPIN_TRUSTED_VERITY_ROOT_DIGESTS"
+
 static void report_load(const char *origin, struct file *file, char *operation)
 {
 	char *cmdline, *pathname;
@@ -292,9 +294,21 @@ static int read_trusted_verity_root_digests(unsigned int fd)
 
 	p = strim(data);
 	while ((d = strsep(&p, "\n")) != NULL) {
-		int len = strlen(d);
+		int len;
 		struct dm_verity_loadpin_trusted_root_digest *trd;
 
+		if (d == data) {
+			/* first line, validate header */
+			if (strcmp(d, VERITY_DIGEST_FILE_HEADER)) {
+				rc = -EPROTO;
+				goto err;
+			}
+
+			continue;
+		}
+
+		len = strlen(d);
+
 		if (len % 2) {
 			rc = -EPROTO;
 			goto err;
diff --git a/security/lockdown/lockdown.c b/security/lockdown/lockdown.c
index 87cbdc64d272..a79b985e917e 100644
--- a/security/lockdown/lockdown.c
+++ b/security/lockdown/lockdown.c
@@ -63,7 +63,7 @@ static int lockdown_is_locked_down(enum lockdown_reason what)
 
 	if (kernel_locked_down >= what) {
 		if (lockdown_reasons[what])
-			pr_notice("Lockdown: %s: %s is restricted; see man kernel_lockdown.7\n",
+			pr_notice_ratelimited("Lockdown: %s: %s is restricted; see man kernel_lockdown.7\n",
 				  current->comm, lockdown_reasons[what]);
 		return -EPERM;
 	}
diff --git a/security/lsm_audit.c b/security/lsm_audit.c
index 78a278f28e49..75cc3f8d2a42 100644
--- a/security/lsm_audit.c
+++ b/security/lsm_audit.c
@@ -44,9 +44,6 @@ int ipv4_skb_to_auditdata(struct sk_buff *skb,
 	struct iphdr *ih;
 
 	ih = ip_hdr(skb);
-	if (ih == NULL)
-		return -EINVAL;
-
 	ad->u.net->v4info.saddr = ih->saddr;
 	ad->u.net->v4info.daddr = ih->daddr;
 
@@ -59,8 +56,6 @@ int ipv4_skb_to_auditdata(struct sk_buff *skb,
 	switch (ih->protocol) {
 	case IPPROTO_TCP: {
 		struct tcphdr *th = tcp_hdr(skb);
-		if (th == NULL)
-			break;
 
 		ad->u.net->sport = th->source;
 		ad->u.net->dport = th->dest;
@@ -68,8 +63,6 @@ int ipv4_skb_to_auditdata(struct sk_buff *skb,
 	}
 	case IPPROTO_UDP: {
 		struct udphdr *uh = udp_hdr(skb);
-		if (uh == NULL)
-			break;
 
 		ad->u.net->sport = uh->source;
 		ad->u.net->dport = uh->dest;
@@ -77,8 +70,6 @@ int ipv4_skb_to_auditdata(struct sk_buff *skb,
 	}
 	case IPPROTO_DCCP: {
 		struct dccp_hdr *dh = dccp_hdr(skb);
-		if (dh == NULL)
-			break;
 
 		ad->u.net->sport = dh->dccph_sport;
 		ad->u.net->dport = dh->dccph_dport;
@@ -86,8 +77,7 @@ int ipv4_skb_to_auditdata(struct sk_buff *skb,
 	}
 	case IPPROTO_SCTP: {
 		struct sctphdr *sh = sctp_hdr(skb);
-		if (sh == NULL)
-			break;
+
 		ad->u.net->sport = sh->source;
 		ad->u.net->dport = sh->dest;
 		break;
@@ -115,8 +105,6 @@ int ipv6_skb_to_auditdata(struct sk_buff *skb,
 	__be16 frag_off;
 
 	ip6 = ipv6_hdr(skb);
-	if (ip6 == NULL)
-		return -EINVAL;
 	ad->u.net->v6info.saddr = ip6->saddr;
 	ad->u.net->v6info.daddr = ip6->daddr;
 	/* IPv6 can have several extension header before the Transport header
diff --git a/security/security.c b/security/security.c
index 4b95de24bc8d..8312b3bf1169 100644
--- a/security/security.c
+++ b/security/security.c
@@ -1909,6 +1909,11 @@ void security_task_to_inode(struct task_struct *p, struct inode *inode)
 	call_void_hook(task_to_inode, p, inode);
 }
 
+int security_create_user_ns(const struct cred *cred)
+{
+	return call_int_hook(userns_create, 0, cred);
+}
+
 int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag)
 {
 	return call_int_hook(ipc_permission, 0, ipcp, flag);
@@ -2057,8 +2062,8 @@ void security_d_instantiate(struct dentry *dentry, struct inode *inode)
 }
 EXPORT_SYMBOL(security_d_instantiate);
 
-int security_getprocattr(struct task_struct *p, const char *lsm, char *name,
-				char **value)
+int security_getprocattr(struct task_struct *p, const char *lsm,
+			 const char *name, char **value)
 {
 	struct security_hook_list *hp;
 
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index 03bca97c8b29..f553c370397e 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -4222,6 +4222,14 @@ static void selinux_task_to_inode(struct task_struct *p,
 	spin_unlock(&isec->lock);
 }
 
+static int selinux_userns_create(const struct cred *cred)
+{
+	u32 sid = current_sid();
+
+	return avc_has_perm(&selinux_state, sid, sid, SECCLASS_USER_NAMESPACE,
+						USER_NAMESPACE__CREATE, NULL);
+}
+
 /* Returns error only if unable to parse addresses */
 static int selinux_parse_skb_ipv4(struct sk_buff *skb,
 			struct common_audit_data *ad, u8 *proto)
@@ -5987,7 +5995,6 @@ static int selinux_msg_queue_alloc_security(struct kern_ipc_perm *msq)
 	struct ipc_security_struct *isec;
 	struct common_audit_data ad;
 	u32 sid = current_sid();
-	int rc;
 
 	isec = selinux_ipc(msq);
 	ipc_init_security(isec, SECCLASS_MSGQ);
@@ -5995,10 +6002,9 @@ static int selinux_msg_queue_alloc_security(struct kern_ipc_perm *msq)
 	ad.type = LSM_AUDIT_DATA_IPC;
 	ad.u.ipc_id = msq->key;
 
-	rc = avc_has_perm(&selinux_state,
-			  sid, isec->sid, SECCLASS_MSGQ,
-			  MSGQ__CREATE, &ad);
-	return rc;
+	return avc_has_perm(&selinux_state,
+			    sid, isec->sid, SECCLASS_MSGQ,
+			    MSGQ__CREATE, &ad);
 }
 
 static int selinux_msg_queue_associate(struct kern_ipc_perm *msq, int msqflg)
@@ -6126,7 +6132,6 @@ static int selinux_shm_alloc_security(struct kern_ipc_perm *shp)
 	struct ipc_security_struct *isec;
 	struct common_audit_data ad;
 	u32 sid = current_sid();
-	int rc;
 
 	isec = selinux_ipc(shp);
 	ipc_init_security(isec, SECCLASS_SHM);
@@ -6134,10 +6139,9 @@ static int selinux_shm_alloc_security(struct kern_ipc_perm *shp)
 	ad.type = LSM_AUDIT_DATA_IPC;
 	ad.u.ipc_id = shp->key;
 
-	rc = avc_has_perm(&selinux_state,
-			  sid, isec->sid, SECCLASS_SHM,
-			  SHM__CREATE, &ad);
-	return rc;
+	return avc_has_perm(&selinux_state,
+			    sid, isec->sid, SECCLASS_SHM,
+			    SHM__CREATE, &ad);
 }
 
 static int selinux_shm_associate(struct kern_ipc_perm *shp, int shmflg)
@@ -6211,7 +6215,6 @@ static int selinux_sem_alloc_security(struct kern_ipc_perm *sma)
 	struct ipc_security_struct *isec;
 	struct common_audit_data ad;
 	u32 sid = current_sid();
-	int rc;
 
 	isec = selinux_ipc(sma);
 	ipc_init_security(isec, SECCLASS_SEM);
@@ -6219,10 +6222,9 @@ static int selinux_sem_alloc_security(struct kern_ipc_perm *sma)
 	ad.type = LSM_AUDIT_DATA_IPC;
 	ad.u.ipc_id = sma->key;
 
-	rc = avc_has_perm(&selinux_state,
-			  sid, isec->sid, SECCLASS_SEM,
-			  SEM__CREATE, &ad);
-	return rc;
+	return avc_has_perm(&selinux_state,
+			    sid, isec->sid, SECCLASS_SEM,
+			    SEM__CREATE, &ad);
 }
 
 static int selinux_sem_associate(struct kern_ipc_perm *sma, int semflg)
@@ -6328,7 +6330,7 @@ static void selinux_d_instantiate(struct dentry *dentry, struct inode *inode)
 }
 
 static int selinux_getprocattr(struct task_struct *p,
-			       char *name, char **value)
+			       const char *name, char **value)
 {
 	const struct task_security_struct *__tsec;
 	u32 sid;
@@ -7134,6 +7136,7 @@ static struct security_hook_list selinux_hooks[] __lsm_ro_after_init = {
 	LSM_HOOK_INIT(task_movememory, selinux_task_movememory),
 	LSM_HOOK_INIT(task_kill, selinux_task_kill),
 	LSM_HOOK_INIT(task_to_inode, selinux_task_to_inode),
+	LSM_HOOK_INIT(userns_create, selinux_userns_create),
 
 	LSM_HOOK_INIT(ipc_permission, selinux_ipc_permission),
 	LSM_HOOK_INIT(ipc_getsecid, selinux_ipc_getsecid),
diff --git a/security/selinux/include/classmap.h b/security/selinux/include/classmap.h
index 1c2f41ff4e55..a3c380775d41 100644
--- a/security/selinux/include/classmap.h
+++ b/security/selinux/include/classmap.h
@@ -254,6 +254,8 @@ const struct security_class_mapping secclass_map[] = {
 	  { COMMON_FILE_PERMS, NULL } },
 	{ "io_uring",
 	  { "override_creds", "sqpoll", "cmd", NULL } },
+	{ "user_namespace",
+	  { "create", NULL } },
 	{ NULL }
   };
 
diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c
index 8fcdd494af27..a00d19139436 100644
--- a/security/selinux/selinuxfs.c
+++ b/security/selinux/selinuxfs.c
@@ -71,7 +71,7 @@ struct selinux_fs_info {
 	struct dentry *bool_dir;
 	unsigned int bool_num;
 	char **bool_pending_names;
-	unsigned int *bool_pending_values;
+	int *bool_pending_values;
 	struct dentry *class_dir;
 	unsigned long last_class_ino;
 	bool policy_opened;
@@ -356,7 +356,7 @@ static const struct file_operations sel_policyvers_ops = {
 /* declaration for sel_write_load */
 static int sel_make_bools(struct selinux_policy *newpolicy, struct dentry *bool_dir,
 			  unsigned int *bool_num, char ***bool_pending_names,
-			  unsigned int **bool_pending_values);
+			  int **bool_pending_values);
 static int sel_make_classes(struct selinux_policy *newpolicy,
 			    struct dentry *class_dir,
 			    unsigned long *last_class_ino);
@@ -527,7 +527,7 @@ static const struct file_operations sel_policy_ops = {
 };
 
 static void sel_remove_old_bool_data(unsigned int bool_num, char **bool_names,
-				unsigned int *bool_values)
+				     int *bool_values)
 {
 	u32 i;
 
@@ -545,7 +545,7 @@ static int sel_make_policy_nodes(struct selinux_fs_info *fsi,
 	struct dentry *tmp_parent, *tmp_bool_dir, *tmp_class_dir, *old_dentry;
 	unsigned int tmp_bool_num, old_bool_num;
 	char **tmp_bool_names, **old_bool_names;
-	unsigned int *tmp_bool_values, *old_bool_values;
+	int *tmp_bool_values, *old_bool_values;
 	unsigned long tmp_ino = fsi->last_ino; /* Don't increment last_ino in this function */
 
 	tmp_parent = sel_make_disconnected_dir(fsi->sb, &tmp_ino);
@@ -1423,7 +1423,7 @@ static void sel_remove_entries(struct dentry *de)
 
 static int sel_make_bools(struct selinux_policy *newpolicy, struct dentry *bool_dir,
 			  unsigned int *bool_num, char ***bool_pending_names,
-			  unsigned int **bool_pending_values)
+			  int **bool_pending_values)
 {
 	int ret;
 	ssize_t len;
@@ -1917,7 +1917,6 @@ static int sel_make_class_dir_entries(struct selinux_policy *newpolicy,
 	struct selinux_fs_info *fsi = sb->s_fs_info;
 	struct dentry *dentry = NULL;
 	struct inode *inode = NULL;
-	int rc;
 
 	dentry = d_alloc_name(dir, "index");
 	if (!dentry)
@@ -1937,9 +1936,7 @@ static int sel_make_class_dir_entries(struct selinux_policy *newpolicy,
 	if (IS_ERR(dentry))
 		return PTR_ERR(dentry);
 
-	rc = sel_make_perm_files(newpolicy, classname, index, dentry);
-
-	return rc;
+	return sel_make_perm_files(newpolicy, classname, index, dentry);
 }
 
 static int sel_make_classes(struct selinux_policy *newpolicy,
diff --git a/security/selinux/ss/context.h b/security/selinux/ss/context.h
index 62990aa1ec9e..eda32c3d4c0a 100644
--- a/security/selinux/ss/context.h
+++ b/security/selinux/ss/context.h
@@ -38,7 +38,7 @@ static inline void mls_context_init(struct context *c)
 	memset(&c->range, 0, sizeof(c->range));
 }
 
-static inline int mls_context_cpy(struct context *dst, struct context *src)
+static inline int mls_context_cpy(struct context *dst, const struct context *src)
 {
 	int rc;
 
@@ -58,7 +58,7 @@ out:
 /*
  * Sets both levels in the MLS range of 'dst' to the low level of 'src'.
  */
-static inline int mls_context_cpy_low(struct context *dst, struct context *src)
+static inline int mls_context_cpy_low(struct context *dst, const struct context *src)
 {
 	int rc;
 
@@ -78,7 +78,7 @@ out:
 /*
  * Sets both levels in the MLS range of 'dst' to the high level of 'src'.
  */
-static inline int mls_context_cpy_high(struct context *dst, struct context *src)
+static inline int mls_context_cpy_high(struct context *dst, const struct context *src)
 {
 	int rc;
 
@@ -97,9 +97,10 @@ out:
 
 
 static inline int mls_context_glblub(struct context *dst,
-				     struct context *c1, struct context *c2)
+				     const struct context *c1, const struct context *c2)
 {
-	struct mls_range *dr = &dst->range, *r1 = &c1->range, *r2 = &c2->range;
+	struct mls_range *dr = &dst->range;
+	const struct mls_range *r1 = &c1->range, *r2 = &c2->range;
 	int rc = 0;
 
 	if (r1->level[1].sens < r2->level[0].sens ||
@@ -127,7 +128,7 @@ out:
 	return rc;
 }
 
-static inline int mls_context_cmp(struct context *c1, struct context *c2)
+static inline int mls_context_cmp(const struct context *c1, const struct context *c2)
 {
 	return ((c1->range.level[0].sens == c2->range.level[0].sens) &&
 		ebitmap_cmp(&c1->range.level[0].cat, &c2->range.level[0].cat) &&
@@ -147,7 +148,7 @@ static inline void context_init(struct context *c)
 	memset(c, 0, sizeof(*c));
 }
 
-static inline int context_cpy(struct context *dst, struct context *src)
+static inline int context_cpy(struct context *dst, const struct context *src)
 {
 	int rc;
 
@@ -180,7 +181,7 @@ static inline void context_destroy(struct context *c)
 	mls_context_destroy(c);
 }
 
-static inline int context_cmp(struct context *c1, struct context *c2)
+static inline int context_cmp(const struct context *c1, const struct context *c2)
 {
 	if (c1->len && c2->len)
 		return (c1->len == c2->len && !strcmp(c1->str, c2->str));
diff --git a/security/selinux/ss/ebitmap.c b/security/selinux/ss/ebitmap.c
index abde349c8321..d31b87be9a1e 100644
--- a/security/selinux/ss/ebitmap.c
+++ b/security/selinux/ss/ebitmap.c
@@ -28,9 +28,9 @@
 
 static struct kmem_cache *ebitmap_node_cachep __ro_after_init;
 
-int ebitmap_cmp(struct ebitmap *e1, struct ebitmap *e2)
+int ebitmap_cmp(const struct ebitmap *e1, const struct ebitmap *e2)
 {
-	struct ebitmap_node *n1, *n2;
+	const struct ebitmap_node *n1, *n2;
 
 	if (e1->highbit != e2->highbit)
 		return 0;
@@ -50,9 +50,10 @@ int ebitmap_cmp(struct ebitmap *e1, struct ebitmap *e2)
 	return 1;
 }
 
-int ebitmap_cpy(struct ebitmap *dst, struct ebitmap *src)
+int ebitmap_cpy(struct ebitmap *dst, const struct ebitmap *src)
 {
-	struct ebitmap_node *n, *new, *prev;
+	struct ebitmap_node *new, *prev;
+	const struct ebitmap_node *n;
 
 	ebitmap_init(dst);
 	n = src->node;
@@ -78,7 +79,7 @@ int ebitmap_cpy(struct ebitmap *dst, struct ebitmap *src)
 	return 0;
 }
 
-int ebitmap_and(struct ebitmap *dst, struct ebitmap *e1, struct ebitmap *e2)
+int ebitmap_and(struct ebitmap *dst, const struct ebitmap *e1, const struct ebitmap *e2)
 {
 	struct ebitmap_node *n;
 	int bit, rc;
@@ -217,9 +218,9 @@ netlbl_import_failure:
  * if last_e2bit is non-zero, the highest set bit in e2 cannot exceed
  * last_e2bit.
  */
-int ebitmap_contains(struct ebitmap *e1, struct ebitmap *e2, u32 last_e2bit)
+int ebitmap_contains(const struct ebitmap *e1, const struct ebitmap *e2, u32 last_e2bit)
 {
-	struct ebitmap_node *n1, *n2;
+	const struct ebitmap_node *n1, *n2;
 	int i;
 
 	if (e1->highbit < e2->highbit)
@@ -258,9 +259,9 @@ int ebitmap_contains(struct ebitmap *e1, struct ebitmap *e2, u32 last_e2bit)
 	return 1;
 }
 
-int ebitmap_get_bit(struct ebitmap *e, unsigned long bit)
+int ebitmap_get_bit(const struct ebitmap *e, unsigned long bit)
 {
-	struct ebitmap_node *n;
+	const struct ebitmap_node *n;
 
 	if (e->highbit < bit)
 		return 0;
@@ -467,7 +468,7 @@ bad:
 	goto out;
 }
 
-int ebitmap_write(struct ebitmap *e, void *fp)
+int ebitmap_write(const struct ebitmap *e, void *fp)
 {
 	struct ebitmap_node *n;
 	u32 count;
diff --git a/security/selinux/ss/ebitmap.h b/security/selinux/ss/ebitmap.h
index 58eb822f11ee..e5b57dc3fc53 100644
--- a/security/selinux/ss/ebitmap.h
+++ b/security/selinux/ss/ebitmap.h
@@ -44,7 +44,7 @@ struct ebitmap {
 
 #define ebitmap_length(e) ((e)->highbit)
 
-static inline unsigned int ebitmap_start_positive(struct ebitmap *e,
+static inline unsigned int ebitmap_start_positive(const struct ebitmap *e,
 						  struct ebitmap_node **n)
 {
 	unsigned int ofs;
@@ -62,7 +62,7 @@ static inline void ebitmap_init(struct ebitmap *e)
 	memset(e, 0, sizeof(*e));
 }
 
-static inline unsigned int ebitmap_next_positive(struct ebitmap *e,
+static inline unsigned int ebitmap_next_positive(const struct ebitmap *e,
 						 struct ebitmap_node **n,
 						 unsigned int bit)
 {
@@ -85,7 +85,7 @@ static inline unsigned int ebitmap_next_positive(struct ebitmap *e,
 #define EBITMAP_NODE_OFFSET(node, bit)	\
 	(((bit) - (node)->startbit) % EBITMAP_UNIT_SIZE)
 
-static inline int ebitmap_node_get_bit(struct ebitmap_node *n,
+static inline int ebitmap_node_get_bit(const struct ebitmap_node *n,
 				       unsigned int bit)
 {
 	unsigned int index = EBITMAP_NODE_INDEX(n, bit);
@@ -122,15 +122,15 @@ static inline void ebitmap_node_clr_bit(struct ebitmap_node *n,
 	     (bit) < ebitmap_length(e);			\
 	     (bit) = ebitmap_next_positive(e, &(n), bit))	\
 
-int ebitmap_cmp(struct ebitmap *e1, struct ebitmap *e2);
-int ebitmap_cpy(struct ebitmap *dst, struct ebitmap *src);
-int ebitmap_and(struct ebitmap *dst, struct ebitmap *e1, struct ebitmap *e2);
-int ebitmap_contains(struct ebitmap *e1, struct ebitmap *e2, u32 last_e2bit);
-int ebitmap_get_bit(struct ebitmap *e, unsigned long bit);
+int ebitmap_cmp(const struct ebitmap *e1, const struct ebitmap *e2);
+int ebitmap_cpy(struct ebitmap *dst, const struct ebitmap *src);
+int ebitmap_and(struct ebitmap *dst, const struct ebitmap *e1, const struct ebitmap *e2);
+int ebitmap_contains(const struct ebitmap *e1, const struct ebitmap *e2, u32 last_e2bit);
+int ebitmap_get_bit(const struct ebitmap *e, unsigned long bit);
 int ebitmap_set_bit(struct ebitmap *e, unsigned long bit, int value);
 void ebitmap_destroy(struct ebitmap *e);
 int ebitmap_read(struct ebitmap *e, void *fp);
-int ebitmap_write(struct ebitmap *e, void *fp);
+int ebitmap_write(const struct ebitmap *e, void *fp);
 u32 ebitmap_hash(const struct ebitmap *e, u32 hash);
 
 #ifdef CONFIG_NETLABEL
diff --git a/security/selinux/ss/mls_types.h b/security/selinux/ss/mls_types.h
index 068e0d7809db..7d48d5e52233 100644
--- a/security/selinux/ss/mls_types.h
+++ b/security/selinux/ss/mls_types.h
@@ -27,13 +27,13 @@ struct mls_range {
 	struct mls_level level[2]; /* low == level[0], high == level[1] */
 };
 
-static inline int mls_level_eq(struct mls_level *l1, struct mls_level *l2)
+static inline int mls_level_eq(const struct mls_level *l1, const struct mls_level *l2)
 {
 	return ((l1->sens == l2->sens) &&
 		ebitmap_cmp(&l1->cat, &l2->cat));
 }
 
-static inline int mls_level_dom(struct mls_level *l1, struct mls_level *l2)
+static inline int mls_level_dom(const struct mls_level *l1, const struct mls_level *l2)
 {
 	return ((l1->sens >= l2->sens) &&
 		ebitmap_contains(&l1->cat, &l2->cat, 0));
diff --git a/security/smack/smack.h b/security/smack/smack.h
index fc837dcebf96..e2239be7bd60 100644
--- a/security/smack/smack.h
+++ b/security/smack/smack.h
@@ -180,15 +180,6 @@ struct smack_known_list_elem {
 	struct smack_known	*smk_label;
 };
 
-/* Super block security struct flags for mount options */
-#define FSDEFAULT_MNT	0x01
-#define FSFLOOR_MNT	0x02
-#define FSHAT_MNT	0x04
-#define FSROOT_MNT	0x08
-#define FSTRANS_MNT	0x10
-
-#define NUM_SMK_MNT_OPTS	5
-
 enum {
 	Opt_error = -1,
 	Opt_fsdefault = 0,
diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c
index bffccdc494cb..b6306d71c908 100644
--- a/security/smack/smack_lsm.c
+++ b/security/smack/smack_lsm.c
@@ -497,13 +497,11 @@ static int smack_ptrace_access_check(struct task_struct *ctp, unsigned int mode)
  */
 static int smack_ptrace_traceme(struct task_struct *ptp)
 {
-	int rc;
 	struct smack_known *skp;
 
 	skp = smk_of_task(smack_cred(current_cred()));
 
-	rc = smk_ptrace_rule_check(ptp, skp, PTRACE_MODE_ATTACH, __func__);
-	return rc;
+	return smk_ptrace_rule_check(ptp, skp, PTRACE_MODE_ATTACH, __func__);
 }
 
 /**
@@ -2280,6 +2278,21 @@ static void smack_sk_free_security(struct sock *sk)
 }
 
 /**
+ * smack_sk_clone_security - Copy security context
+ * @sk: the old socket
+ * @newsk: the new socket
+ *
+ * Copy the security context of the old socket pointer to the cloned
+ */
+static void smack_sk_clone_security(const struct sock *sk, struct sock *newsk)
+{
+	struct socket_smack *ssp_old = sk->sk_security;
+	struct socket_smack *ssp_new = newsk->sk_security;
+
+	*ssp_new = *ssp_old;
+}
+
+/**
 * smack_ipv4host_label - check host based restrictions
 * @sip: the object end
 *
@@ -3480,7 +3493,7 @@ static void smack_d_instantiate(struct dentry *opt_dentry, struct inode *inode)
  *
  * Returns the length of the smack label or an error code
  */
-static int smack_getprocattr(struct task_struct *p, char *name, char **value)
+static int smack_getprocattr(struct task_struct *p, const char *name, char **value)
 {
 	struct smack_known *skp = smk_of_task_struct_obj(p);
 	char *cp;
@@ -4882,6 +4895,7 @@ static struct security_hook_list smack_hooks[] __lsm_ro_after_init = {
 	LSM_HOOK_INIT(socket_getpeersec_dgram, smack_socket_getpeersec_dgram),
 	LSM_HOOK_INIT(sk_alloc_security, smack_sk_alloc_security),
 	LSM_HOOK_INIT(sk_free_security, smack_sk_free_security),
+	LSM_HOOK_INIT(sk_clone_security, smack_sk_clone_security),
 	LSM_HOOK_INIT(sock_graft, smack_sock_graft),
 	LSM_HOOK_INIT(inet_conn_request, smack_inet_conn_request),
 	LSM_HOOK_INIT(inet_csk_clone, smack_inet_csk_clone),
diff --git a/security/tomoyo/audit.c b/security/tomoyo/audit.c
index 023bedd9dfa3..7cf8fdbb29bf 100644
--- a/security/tomoyo/audit.c
+++ b/security/tomoyo/audit.c
@@ -423,7 +423,7 @@ void tomoyo_write_log(struct tomoyo_request_info *r, const char *fmt, ...)
 	int len;
 
 	va_start(args, fmt);
-	len = vsnprintf((char *) &len, 1, fmt, args) + 1;
+	len = vsnprintf(NULL, 0, fmt, args) + 1;
 	va_end(args);
 	va_start(args, fmt);
 	tomoyo_write_log2(r, len, fmt, args);
diff --git a/security/tomoyo/common.c b/security/tomoyo/common.c
index ff17abc96e5c..f4cd9b58b205 100644
--- a/security/tomoyo/common.c
+++ b/security/tomoyo/common.c
@@ -2057,7 +2057,7 @@ int tomoyo_supervisor(struct tomoyo_request_info *r, const char *fmt, ...)
 	bool quota_exceeded = false;
 
 	va_start(args, fmt);
-	len = vsnprintf((char *) &len, 1, fmt, args) + 1;
+	len = vsnprintf(NULL, 0, fmt, args) + 1;
 	va_end(args);
 	/* Write /sys/kernel/security/tomoyo/audit. */
 	va_start(args, fmt);
diff --git a/security/tomoyo/file.c b/security/tomoyo/file.c
index 1e6077568fde..8f3b90b6e03d 100644
--- a/security/tomoyo/file.c
+++ b/security/tomoyo/file.c
@@ -717,7 +717,7 @@ int tomoyo_path_number_perm(const u8 type, const struct path *path,
 	int idx;
 
 	if (tomoyo_init_request_info(&r, NULL, tomoyo_pn2mac[type])
-	    == TOMOYO_CONFIG_DISABLED || !path->dentry)
+	    == TOMOYO_CONFIG_DISABLED)
 		return 0;
 	idx = tomoyo_read_lock();
 	if (!tomoyo_get_realpath(&buf, path))
diff --git a/security/tomoyo/realpath.c b/security/tomoyo/realpath.c
index df4798980416..1c483ee7f93d 100644
--- a/security/tomoyo/realpath.c
+++ b/security/tomoyo/realpath.c
@@ -240,11 +240,8 @@ char *tomoyo_realpath_from_path(const struct path *path)
 	char *name = NULL;
 	unsigned int buf_len = PAGE_SIZE / 2;
 	struct dentry *dentry = path->dentry;
-	struct super_block *sb;
+	struct super_block *sb = dentry->d_sb;
 
-	if (!dentry)
-		return NULL;
-	sb = dentry->d_sb;
 	while (1) {
 		char *pos;
 		struct inode *inode;
@@ -264,10 +261,8 @@ char *tomoyo_realpath_from_path(const struct path *path)
 		inode = d_backing_inode(sb->s_root);
 		/*
 		 * Get local name for filesystems without rename() operation
-		 * or dentry without vfsmount.
 		 */
-		if (!path->mnt ||
-		    (!inode->i_op->rename &&
+		if ((!inode->i_op->rename &&
 		     !(sb->s_type->fs_flags & FS_REQUIRES_DEV)))
 			pos = tomoyo_get_local_path(path->dentry, buf,
 						    buf_len - 1);
diff --git a/sound/aoa/codecs/onyx.c b/sound/aoa/codecs/onyx.c
index 1abee841cc45..2d0f904aba00 100644
--- a/sound/aoa/codecs/onyx.c
+++ b/sound/aoa/codecs/onyx.c
@@ -1029,7 +1029,7 @@ static int onyx_i2c_probe(struct i2c_client *client,
 	return -ENODEV;
 }
 
-static int onyx_i2c_remove(struct i2c_client *client)
+static void onyx_i2c_remove(struct i2c_client *client)
 {
 	struct onyx *onyx = i2c_get_clientdata(client);
 
@@ -1037,7 +1037,6 @@ static int onyx_i2c_remove(struct i2c_client *client)
 	of_node_put(onyx->codec.node);
 	kfree(onyx->codec_info);
 	kfree(onyx);
-	return 0;
 }
 
 static const struct i2c_device_id onyx_i2c_id[] = {
diff --git a/sound/aoa/codecs/tas.c b/sound/aoa/codecs/tas.c
index ab19a37e2a68..ab89475b7715 100644
--- a/sound/aoa/codecs/tas.c
+++ b/sound/aoa/codecs/tas.c
@@ -912,7 +912,7 @@ static int tas_i2c_probe(struct i2c_client *client,
 	return -EINVAL;
 }
 
-static int tas_i2c_remove(struct i2c_client *client)
+static void tas_i2c_remove(struct i2c_client *client)
 {
 	struct tas *tas = i2c_get_clientdata(client);
 	u8 tmp = TAS_ACR_ANALOG_PDOWN;
@@ -925,7 +925,6 @@ static int tas_i2c_remove(struct i2c_client *client)
 
 	mutex_destroy(&tas->mtx);
 	kfree(tas);
-	return 0;
 }
 
 static const struct i2c_device_id tas_i2c_id[] = {
diff --git a/sound/core/memalloc.c b/sound/core/memalloc.c
index cfcd8eff4139..03cffe771366 100644
--- a/sound/core/memalloc.c
+++ b/sound/core/memalloc.c
@@ -18,25 +18,18 @@
 #include <sound/memalloc.h>
 #include "memalloc_local.h"
 
+#define DEFAULT_GFP \
+	(GFP_KERNEL | \
+	 __GFP_COMP |    /* compound page lets parts be mapped */ \
+	 __GFP_RETRY_MAYFAIL | /* don't trigger OOM-killer */ \
+	 __GFP_NOWARN)   /* no stack trace print - this call is non-critical */
+
 static const struct snd_malloc_ops *snd_dma_get_ops(struct snd_dma_buffer *dmab);
 
 #ifdef CONFIG_SND_DMA_SGBUF
-static void *do_alloc_fallback_pages(struct device *dev, size_t size,
-				     dma_addr_t *addr, bool wc);
-static void do_free_fallback_pages(void *p, size_t size, bool wc);
 static void *snd_dma_sg_fallback_alloc(struct snd_dma_buffer *dmab, size_t size);
 #endif
 
-/* a cast to gfp flag from the dev pointer; for CONTINUOUS and VMALLOC types */
-static inline gfp_t snd_mem_get_gfp_flags(const struct snd_dma_buffer *dmab,
-					  gfp_t default_gfp)
-{
-	if (!dmab->dev.dev)
-		return default_gfp;
-	else
-		return (__force gfp_t)(unsigned long)dmab->dev.dev;
-}
-
 static void *__snd_dma_alloc_pages(struct snd_dma_buffer *dmab, size_t size)
 {
 	const struct snd_malloc_ops *ops = snd_dma_get_ops(dmab);
@@ -284,24 +277,54 @@ EXPORT_SYMBOL(snd_sgbuf_get_chunk_size);
 /*
  * Continuous pages allocator
  */
-static void *do_alloc_pages(size_t size, dma_addr_t *addr, gfp_t gfp)
+static void *do_alloc_pages(struct device *dev, size_t size, dma_addr_t *addr,
+			    bool wc)
 {
-	void *p = alloc_pages_exact(size, gfp);
+	void *p;
+	gfp_t gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN;
 
-	if (p)
-		*addr = page_to_phys(virt_to_page(p));
+ again:
+	p = alloc_pages_exact(size, gfp);
+	if (!p)
+		return NULL;
+	*addr = page_to_phys(virt_to_page(p));
+	if (!dev)
+		return p;
+	if ((*addr + size - 1) & ~dev->coherent_dma_mask) {
+		if (IS_ENABLED(CONFIG_ZONE_DMA32) && !(gfp & GFP_DMA32)) {
+			gfp |= GFP_DMA32;
+			goto again;
+		}
+		if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) {
+			gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
+			goto again;
+		}
+	}
+#ifdef CONFIG_X86
+	if (wc)
+		set_memory_wc((unsigned long)(p), size >> PAGE_SHIFT);
+#endif
 	return p;
 }
 
+static void do_free_pages(void *p, size_t size, bool wc)
+{
+#ifdef CONFIG_X86
+	if (wc)
+		set_memory_wb((unsigned long)(p), size >> PAGE_SHIFT);
+#endif
+	free_pages_exact(p, size);
+}
+
+
 static void *snd_dma_continuous_alloc(struct snd_dma_buffer *dmab, size_t size)
 {
-	return do_alloc_pages(size, &dmab->addr,
-			      snd_mem_get_gfp_flags(dmab, GFP_KERNEL));
+	return do_alloc_pages(dmab->dev.dev, size, &dmab->addr, false);
 }
 
 static void snd_dma_continuous_free(struct snd_dma_buffer *dmab)
 {
-	free_pages_exact(dmab->area, dmab->bytes);
+	do_free_pages(dmab->area, dmab->bytes, false);
 }
 
 static int snd_dma_continuous_mmap(struct snd_dma_buffer *dmab,
@@ -324,9 +347,7 @@ static const struct snd_malloc_ops snd_dma_continuous_ops = {
  */
 static void *snd_dma_vmalloc_alloc(struct snd_dma_buffer *dmab, size_t size)
 {
-	gfp_t gfp = snd_mem_get_gfp_flags(dmab, GFP_KERNEL | __GFP_HIGHMEM);
-
-	return __vmalloc(size, gfp);
+	return vmalloc(size);
 }
 
 static void snd_dma_vmalloc_free(struct snd_dma_buffer *dmab)
@@ -440,12 +461,6 @@ static const struct snd_malloc_ops snd_dma_iram_ops = {
 };
 #endif /* CONFIG_GENERIC_ALLOCATOR */
 
-#define DEFAULT_GFP \
-	(GFP_KERNEL | \
-	 __GFP_COMP |    /* compound page lets parts be mapped */ \
-	 __GFP_NORETRY | /* don't trigger OOM-killer */ \
-	 __GFP_NOWARN)   /* no stack trace print - this call is non-critical */
-
 /*
  * Coherent device pages allocator
  */
@@ -479,12 +494,12 @@ static const struct snd_malloc_ops snd_dma_dev_ops = {
 #ifdef CONFIG_SND_DMA_SGBUF
 static void *snd_dma_wc_alloc(struct snd_dma_buffer *dmab, size_t size)
 {
-	return do_alloc_fallback_pages(dmab->dev.dev, size, &dmab->addr, true);
+	return do_alloc_pages(dmab->dev.dev, size, &dmab->addr, true);
 }
 
 static void snd_dma_wc_free(struct snd_dma_buffer *dmab)
 {
-	do_free_fallback_pages(dmab->area, dmab->bytes, true);
+	do_free_pages(dmab->area, dmab->bytes, true);
 }
 
 static int snd_dma_wc_mmap(struct snd_dma_buffer *dmab,
@@ -700,37 +715,6 @@ static const struct snd_malloc_ops snd_dma_sg_wc_ops = {
 	.get_chunk_size = snd_dma_noncontig_get_chunk_size,
 };
 
-/* manual page allocations with wc setup */
-static void *do_alloc_fallback_pages(struct device *dev, size_t size,
-				     dma_addr_t *addr, bool wc)
-{
-	gfp_t gfp = GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN;
-	void *p;
-
- again:
-	p = do_alloc_pages(size, addr, gfp);
-	if (!p || (*addr + size - 1) & ~dev->coherent_dma_mask) {
-		if (IS_ENABLED(CONFIG_ZONE_DMA32) && !(gfp & GFP_DMA32)) {
-			gfp |= GFP_DMA32;
-			goto again;
-		}
-		if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) {
-			gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
-			goto again;
-		}
-	}
-	if (p && wc)
-		set_memory_wc((unsigned long)(p), size >> PAGE_SHIFT);
-	return p;
-}
-
-static void do_free_fallback_pages(void *p, size_t size, bool wc)
-{
-	if (wc)
-		set_memory_wb((unsigned long)(p), size >> PAGE_SHIFT);
-	free_pages_exact(p, size);
-}
-
 /* Fallback SG-buffer allocations for x86 */
 struct snd_dma_sg_fallback {
 	size_t count;
@@ -745,7 +729,7 @@ static void __snd_dma_sg_fallback_free(struct snd_dma_buffer *dmab,
 	size_t i;
 
 	for (i = 0; i < sgbuf->count && sgbuf->pages[i]; i++)
-		do_free_fallback_pages(page_address(sgbuf->pages[i]), PAGE_SIZE, wc);
+		do_free_pages(page_address(sgbuf->pages[i]), PAGE_SIZE, wc);
 	kvfree(sgbuf->pages);
 	kvfree(sgbuf->addrs);
 	kfree(sgbuf);
@@ -772,8 +756,7 @@ static void *snd_dma_sg_fallback_alloc(struct snd_dma_buffer *dmab, size_t size)
 		goto error;
 
 	for (i = 0; i < count; sgbuf->count++, i++) {
-		p = do_alloc_fallback_pages(dmab->dev.dev, PAGE_SIZE,
-					    &sgbuf->addrs[i], wc);
+		p = do_alloc_pages(dmab->dev.dev, PAGE_SIZE, &sgbuf->addrs[i], wc);
 		if (!p)
 			goto error;
 		sgbuf->pages[i] = virt_to_page(p);
diff --git a/sound/core/memalloc_local.h b/sound/core/memalloc_local.h
index a6f3a87194da..8b19f3a68a4b 100644
--- a/sound/core/memalloc_local.h
+++ b/sound/core/memalloc_local.h
@@ -13,8 +13,4 @@ struct snd_malloc_ops {
 	void (*sync)(struct snd_dma_buffer *dmab, enum snd_dma_sync_mode mode);
 };
 
-#ifdef CONFIG_SND_DMA_SGBUF
-extern const struct snd_malloc_ops snd_dma_sg_ops;
-#endif
-
 #endif /* __MEMALLOC_LOCAL_H */
diff --git a/sound/core/oss/pcm_oss.c b/sound/core/oss/pcm_oss.c
index 02df915eb3c6..ac2efeb63a39 100644
--- a/sound/core/oss/pcm_oss.c
+++ b/sound/core/oss/pcm_oss.c
@@ -1237,12 +1237,12 @@ snd_pcm_sframes_t snd_pcm_oss_write3(struct snd_pcm_substream *substream, const
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	int ret;
 	while (1) {
-		if (runtime->status->state == SNDRV_PCM_STATE_XRUN ||
-		    runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) {
+		if (runtime->state == SNDRV_PCM_STATE_XRUN ||
+		    runtime->state == SNDRV_PCM_STATE_SUSPENDED) {
 #ifdef OSS_DEBUG
 			pcm_dbg(substream->pcm,
 				"pcm_oss: write: recovering from %s\n",
-				runtime->status->state == SNDRV_PCM_STATE_XRUN ?
+				runtime->state == SNDRV_PCM_STATE_XRUN ?
 				"XRUN" : "SUSPEND");
 #endif
 			ret = snd_pcm_oss_prepare(substream);
@@ -1257,7 +1257,7 @@ snd_pcm_sframes_t snd_pcm_oss_write3(struct snd_pcm_substream *substream, const
 			break;
 		/* test, if we can't store new data, because the stream */
 		/* has not been started */
-		if (runtime->status->state == SNDRV_PCM_STATE_PREPARED)
+		if (runtime->state == SNDRV_PCM_STATE_PREPARED)
 			return -EAGAIN;
 	}
 	return ret;
@@ -1269,18 +1269,18 @@ snd_pcm_sframes_t snd_pcm_oss_read3(struct snd_pcm_substream *substream, char *p
 	snd_pcm_sframes_t delay;
 	int ret;
 	while (1) {
-		if (runtime->status->state == SNDRV_PCM_STATE_XRUN ||
-		    runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) {
+		if (runtime->state == SNDRV_PCM_STATE_XRUN ||
+		    runtime->state == SNDRV_PCM_STATE_SUSPENDED) {
 #ifdef OSS_DEBUG
 			pcm_dbg(substream->pcm,
 				"pcm_oss: read: recovering from %s\n",
-				runtime->status->state == SNDRV_PCM_STATE_XRUN ?
+				runtime->state == SNDRV_PCM_STATE_XRUN ?
 				"XRUN" : "SUSPEND");
 #endif
 			ret = snd_pcm_kernel_ioctl(substream, SNDRV_PCM_IOCTL_DRAIN, NULL);
 			if (ret < 0)
 				break;
-		} else if (runtime->status->state == SNDRV_PCM_STATE_SETUP) {
+		} else if (runtime->state == SNDRV_PCM_STATE_SETUP) {
 			ret = snd_pcm_oss_prepare(substream);
 			if (ret < 0)
 				break;
@@ -1293,7 +1293,7 @@ snd_pcm_sframes_t snd_pcm_oss_read3(struct snd_pcm_substream *substream, char *p
 					 frames, in_kernel);
 		mutex_lock(&runtime->oss.params_lock);
 		if (ret == -EPIPE) {
-			if (runtime->status->state == SNDRV_PCM_STATE_DRAINING) {
+			if (runtime->state == SNDRV_PCM_STATE_DRAINING) {
 				ret = snd_pcm_kernel_ioctl(substream, SNDRV_PCM_IOCTL_DROP, NULL);
 				if (ret < 0)
 					break;
@@ -1312,12 +1312,12 @@ snd_pcm_sframes_t snd_pcm_oss_writev3(struct snd_pcm_substream *substream, void
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	int ret;
 	while (1) {
-		if (runtime->status->state == SNDRV_PCM_STATE_XRUN ||
-		    runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) {
+		if (runtime->state == SNDRV_PCM_STATE_XRUN ||
+		    runtime->state == SNDRV_PCM_STATE_SUSPENDED) {
 #ifdef OSS_DEBUG
 			pcm_dbg(substream->pcm,
 				"pcm_oss: writev: recovering from %s\n",
-				runtime->status->state == SNDRV_PCM_STATE_XRUN ?
+				runtime->state == SNDRV_PCM_STATE_XRUN ?
 				"XRUN" : "SUSPEND");
 #endif
 			ret = snd_pcm_oss_prepare(substream);
@@ -1330,7 +1330,7 @@ snd_pcm_sframes_t snd_pcm_oss_writev3(struct snd_pcm_substream *substream, void
 
 		/* test, if we can't store new data, because the stream */
 		/* has not been started */
-		if (runtime->status->state == SNDRV_PCM_STATE_PREPARED)
+		if (runtime->state == SNDRV_PCM_STATE_PREPARED)
 			return -EAGAIN;
 	}
 	return ret;
@@ -1341,18 +1341,18 @@ snd_pcm_sframes_t snd_pcm_oss_readv3(struct snd_pcm_substream *substream, void *
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	int ret;
 	while (1) {
-		if (runtime->status->state == SNDRV_PCM_STATE_XRUN ||
-		    runtime->status->state == SNDRV_PCM_STATE_SUSPENDED) {
+		if (runtime->state == SNDRV_PCM_STATE_XRUN ||
+		    runtime->state == SNDRV_PCM_STATE_SUSPENDED) {
 #ifdef OSS_DEBUG
 			pcm_dbg(substream->pcm,
 				"pcm_oss: readv: recovering from %s\n",
-				runtime->status->state == SNDRV_PCM_STATE_XRUN ?
+				runtime->state == SNDRV_PCM_STATE_XRUN ?
 				"XRUN" : "SUSPEND");
 #endif
 			ret = snd_pcm_kernel_ioctl(substream, SNDRV_PCM_IOCTL_DRAIN, NULL);
 			if (ret < 0)
 				break;
-		} else if (runtime->status->state == SNDRV_PCM_STATE_SETUP) {
+		} else if (runtime->state == SNDRV_PCM_STATE_SETUP) {
 			ret = snd_pcm_oss_prepare(substream);
 			if (ret < 0)
 				break;
@@ -1635,7 +1635,7 @@ static int snd_pcm_oss_sync1(struct snd_pcm_substream *substream, size_t size)
 		result = 0;
 		set_current_state(TASK_INTERRUPTIBLE);
 		snd_pcm_stream_lock_irq(substream);
-		state = runtime->status->state;
+		state = runtime->state;
 		snd_pcm_stream_unlock_irq(substream);
 		if (state != SNDRV_PCM_STATE_RUNNING) {
 			set_current_state(TASK_RUNNING);
@@ -2854,8 +2854,8 @@ static __poll_t snd_pcm_oss_poll(struct file *file, poll_table * wait)
 		struct snd_pcm_runtime *runtime = psubstream->runtime;
 		poll_wait(file, &runtime->sleep, wait);
 		snd_pcm_stream_lock_irq(psubstream);
-		if (runtime->status->state != SNDRV_PCM_STATE_DRAINING &&
-		    (runtime->status->state != SNDRV_PCM_STATE_RUNNING ||
+		if (runtime->state != SNDRV_PCM_STATE_DRAINING &&
+		    (runtime->state != SNDRV_PCM_STATE_RUNNING ||
 		     snd_pcm_oss_playback_ready(psubstream)))
 			mask |= EPOLLOUT | EPOLLWRNORM;
 		snd_pcm_stream_unlock_irq(psubstream);
@@ -2865,7 +2865,7 @@ static __poll_t snd_pcm_oss_poll(struct file *file, poll_table * wait)
 		snd_pcm_state_t ostate;
 		poll_wait(file, &runtime->sleep, wait);
 		snd_pcm_stream_lock_irq(csubstream);
-		ostate = runtime->status->state;
+		ostate = runtime->state;
 		if (ostate != SNDRV_PCM_STATE_RUNNING ||
 		    snd_pcm_oss_capture_ready(csubstream))
 			mask |= EPOLLIN | EPOLLRDNORM;
diff --git a/sound/core/pcm.c b/sound/core/pcm.c
index 82925709fa12..9d95e3731123 100644
--- a/sound/core/pcm.c
+++ b/sound/core/pcm.c
@@ -387,7 +387,7 @@ static void snd_pcm_substream_proc_hw_params_read(struct snd_info_entry *entry,
 		snd_iprintf(buffer, "closed\n");
 		goto unlock;
 	}
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (runtime->state == SNDRV_PCM_STATE_OPEN) {
 		snd_iprintf(buffer, "no setup\n");
 		goto unlock;
 	}
@@ -424,7 +424,7 @@ static void snd_pcm_substream_proc_sw_params_read(struct snd_info_entry *entry,
 		snd_iprintf(buffer, "closed\n");
 		goto unlock;
 	}
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (runtime->state == SNDRV_PCM_STATE_OPEN) {
 		snd_iprintf(buffer, "no setup\n");
 		goto unlock;
 	}
@@ -970,7 +970,7 @@ int snd_pcm_attach_substream(struct snd_pcm *pcm, int stream,
 	init_waitqueue_head(&runtime->sleep);
 	init_waitqueue_head(&runtime->tsleep);
 
-	runtime->status->state = SNDRV_PCM_STATE_OPEN;
+	__snd_pcm_set_state(runtime, SNDRV_PCM_STATE_OPEN);
 	mutex_init(&runtime->buffer_mutex);
 	atomic_set(&runtime->buffer_accessing, 0);
 
@@ -1112,7 +1112,8 @@ static int snd_pcm_dev_disconnect(struct snd_device *device)
 			if (snd_pcm_running(substream))
 				snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED);
 			/* to be sure, set the state unconditionally */
-			substream->runtime->status->state = SNDRV_PCM_STATE_DISCONNECTED;
+			__snd_pcm_set_state(substream->runtime,
+					    SNDRV_PCM_STATE_DISCONNECTED);
 			wake_up(&substream->runtime->sleep);
 			wake_up(&substream->runtime->tsleep);
 		}
diff --git a/sound/core/pcm_compat.c b/sound/core/pcm_compat.c
index 917c5b4f19d7..42c2ada8e888 100644
--- a/sound/core/pcm_compat.c
+++ b/sound/core/pcm_compat.c
@@ -295,7 +295,7 @@ static int snd_pcm_ioctl_xferi_compat(struct snd_pcm_substream *substream,
 		return -ENOTTY;
 	if (substream->stream != dir)
 		return -EINVAL;
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN)
 		return -EBADFD;
 
 	if (get_user(buf, &data32->buf) ||
@@ -341,7 +341,7 @@ static int snd_pcm_ioctl_xfern_compat(struct snd_pcm_substream *substream,
 		return -ENOTTY;
 	if (substream->stream != dir)
 		return -EINVAL;
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN)
 		return -EBADFD;
 
 	ch = substream->runtime->channels;
diff --git a/sound/core/pcm_dmaengine.c b/sound/core/pcm_dmaengine.c
index 5b2ca028f5aa..494ec0c207fa 100644
--- a/sound/core/pcm_dmaengine.c
+++ b/sound/core/pcm_dmaengine.c
@@ -133,12 +133,14 @@ EXPORT_SYMBOL_GPL(snd_dmaengine_pcm_set_config_from_dai_data);
 
 static void dmaengine_pcm_dma_complete(void *arg)
 {
+	unsigned int new_pos;
 	struct snd_pcm_substream *substream = arg;
 	struct dmaengine_pcm_runtime_data *prtd = substream_to_prtd(substream);
 
-	prtd->pos += snd_pcm_lib_period_bytes(substream);
-	if (prtd->pos >= snd_pcm_lib_buffer_bytes(substream))
-		prtd->pos = 0;
+	new_pos = prtd->pos + snd_pcm_lib_period_bytes(substream);
+	if (new_pos >= snd_pcm_lib_buffer_bytes(substream))
+		new_pos = 0;
+	prtd->pos = new_pos;
 
 	snd_pcm_period_elapsed(substream);
 }
diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c
index 40751e5aff09..8b6aeb8a78f7 100644
--- a/sound/core/pcm_lib.c
+++ b/sound/core/pcm_lib.c
@@ -186,7 +186,7 @@ int snd_pcm_update_state(struct snd_pcm_substream *substream,
 	avail = snd_pcm_avail(substream);
 	if (avail > runtime->avail_max)
 		runtime->avail_max = avail;
-	if (runtime->status->state == SNDRV_PCM_STATE_DRAINING) {
+	if (runtime->state == SNDRV_PCM_STATE_DRAINING) {
 		if (avail >= runtime->buffer_size) {
 			snd_pcm_drain_done(substream);
 			return -EPIPE;
@@ -1911,7 +1911,7 @@ static int wait_for_avail(struct snd_pcm_substream *substream,
 
 		snd_pcm_stream_lock_irq(substream);
 		set_current_state(TASK_INTERRUPTIBLE);
-		switch (runtime->status->state) {
+		switch (runtime->state) {
 		case SNDRV_PCM_STATE_SUSPENDED:
 			err = -ESTRPIPE;
 			goto _endloop;
@@ -2099,14 +2099,14 @@ static int pcm_sanity_check(struct snd_pcm_substream *substream)
 	runtime = substream->runtime;
 	if (snd_BUG_ON(!substream->ops->copy_user && !runtime->dma_area))
 		return -EINVAL;
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN)
 		return -EBADFD;
 	return 0;
 }
 
 static int pcm_accessible_state(struct snd_pcm_runtime *runtime)
 {
-	switch (runtime->status->state) {
+	switch (runtime->state) {
 	case SNDRV_PCM_STATE_PREPARED:
 	case SNDRV_PCM_STATE_RUNNING:
 	case SNDRV_PCM_STATE_PAUSED:
@@ -2225,7 +2225,7 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream,
 		goto _end_unlock;
 
 	runtime->twake = runtime->control->avail_min ? : 1;
-	if (runtime->status->state == SNDRV_PCM_STATE_RUNNING)
+	if (runtime->state == SNDRV_PCM_STATE_RUNNING)
 		snd_pcm_update_hw_ptr(substream);
 
 	/*
@@ -2233,7 +2233,7 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream,
 	 * thread may start capture
 	 */
 	if (!is_playback &&
-	    runtime->status->state == SNDRV_PCM_STATE_PREPARED &&
+	    runtime->state == SNDRV_PCM_STATE_PREPARED &&
 	    size >= runtime->start_threshold) {
 		err = snd_pcm_start(substream);
 		if (err < 0)
@@ -2247,7 +2247,7 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream,
 		snd_pcm_uframes_t cont;
 		if (!avail) {
 			if (!is_playback &&
-			    runtime->status->state == SNDRV_PCM_STATE_DRAINING) {
+			    runtime->state == SNDRV_PCM_STATE_DRAINING) {
 				snd_pcm_stop(substream, SNDRV_PCM_STATE_SETUP);
 				goto _end_unlock;
 			}
@@ -2303,7 +2303,7 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream,
 		xfer += frames;
 		avail -= frames;
 		if (is_playback &&
-		    runtime->status->state == SNDRV_PCM_STATE_PREPARED &&
+		    runtime->state == SNDRV_PCM_STATE_PREPARED &&
 		    snd_pcm_playback_hw_avail(runtime) >= (snd_pcm_sframes_t)runtime->start_threshold) {
 			err = snd_pcm_start(substream);
 			if (err < 0)
diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c
index ad0541e9e888..33769ca78cc8 100644
--- a/sound/core/pcm_native.c
+++ b/sound/core/pcm_native.c
@@ -595,8 +595,8 @@ static void snd_pcm_set_state(struct snd_pcm_substream *substream,
 			      snd_pcm_state_t state)
 {
 	snd_pcm_stream_lock_irq(substream);
-	if (substream->runtime->status->state != SNDRV_PCM_STATE_DISCONNECTED)
-		substream->runtime->status->state = state;
+	if (substream->runtime->state != SNDRV_PCM_STATE_DISCONNECTED)
+		__snd_pcm_set_state(substream->runtime, state);
 	snd_pcm_stream_unlock_irq(substream);
 }
 
@@ -724,7 +724,7 @@ static int snd_pcm_hw_params(struct snd_pcm_substream *substream,
 	if (err < 0)
 		return err;
 	snd_pcm_stream_lock_irq(substream);
-	switch (runtime->status->state) {
+	switch (runtime->state) {
 	case SNDRV_PCM_STATE_OPEN:
 	case SNDRV_PCM_STATE_SETUP:
 	case SNDRV_PCM_STATE_PREPARED:
@@ -889,7 +889,7 @@ static int snd_pcm_hw_free(struct snd_pcm_substream *substream)
 	if (result < 0)
 		return result;
 	snd_pcm_stream_lock_irq(substream);
-	switch (runtime->status->state) {
+	switch (runtime->state) {
 	case SNDRV_PCM_STATE_SETUP:
 	case SNDRV_PCM_STATE_PREPARED:
 		if (atomic_read(&substream->mmap_count))
@@ -920,7 +920,7 @@ static int snd_pcm_sw_params(struct snd_pcm_substream *substream,
 		return -ENXIO;
 	runtime = substream->runtime;
 	snd_pcm_stream_lock_irq(substream);
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (runtime->state == SNDRV_PCM_STATE_OPEN) {
 		snd_pcm_stream_unlock_irq(substream);
 		return -EBADFD;
 	}
@@ -1013,8 +1013,8 @@ int snd_pcm_status64(struct snd_pcm_substream *substream,
 	} else
 		runtime->audio_tstamp_report.valid = 1;
 
-	status->state = runtime->status->state;
-	status->suspended_state = runtime->status->suspended_state;
+	status->state = runtime->state;
+	status->suspended_state = runtime->suspended_state;
 	if (status->state == SNDRV_PCM_STATE_OPEN)
 		goto _end;
 	status->trigger_tstamp_sec = runtime->trigger_tstamp.tv_sec;
@@ -1148,7 +1148,7 @@ static int snd_pcm_channel_info(struct snd_pcm_substream *substream,
 	channel = info->channel;
 	runtime = substream->runtime;
 	snd_pcm_stream_lock_irq(substream);
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (runtime->state == SNDRV_PCM_STATE_OPEN) {
 		snd_pcm_stream_unlock_irq(substream);
 		return -EBADFD;
 	}
@@ -1411,7 +1411,7 @@ static int snd_pcm_pre_start(struct snd_pcm_substream *substream,
 			     snd_pcm_state_t state)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
-	if (runtime->status->state != SNDRV_PCM_STATE_PREPARED)
+	if (runtime->state != SNDRV_PCM_STATE_PREPARED)
 		return -EBADFD;
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK &&
 	    !snd_pcm_playback_data(substream))
@@ -1444,7 +1444,7 @@ static void snd_pcm_post_start(struct snd_pcm_substream *substream,
 	runtime->hw_ptr_jiffies = jiffies;
 	runtime->hw_ptr_buffer_jiffies = (runtime->buffer_size * HZ) / 
 							    runtime->rate;
-	runtime->status->state = state;
+	__snd_pcm_set_state(runtime, state);
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK &&
 	    runtime->silence_size > 0)
 		snd_pcm_playback_silence(substream, ULONG_MAX);
@@ -1485,7 +1485,7 @@ static int snd_pcm_pre_stop(struct snd_pcm_substream *substream,
 			    snd_pcm_state_t state)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN)
 		return -EBADFD;
 	runtime->trigger_master = substream;
 	return 0;
@@ -1506,9 +1506,9 @@ static void snd_pcm_post_stop(struct snd_pcm_substream *substream,
 			      snd_pcm_state_t state)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
-	if (runtime->status->state != state) {
+	if (runtime->state != state) {
 		snd_pcm_trigger_tstamp(substream);
-		runtime->status->state = state;
+		__snd_pcm_set_state(runtime, state);
 		snd_pcm_timer_notify(substream, SNDRV_TIMER_EVENT_MSTOP);
 	}
 	wake_up(&runtime->sleep);
@@ -1584,9 +1584,9 @@ static int snd_pcm_pre_pause(struct snd_pcm_substream *substream,
 	if (!(runtime->info & SNDRV_PCM_INFO_PAUSE))
 		return -ENOSYS;
 	if (pause_pushed(state)) {
-		if (runtime->status->state != SNDRV_PCM_STATE_RUNNING)
+		if (runtime->state != SNDRV_PCM_STATE_RUNNING)
 			return -EBADFD;
-	} else if (runtime->status->state != SNDRV_PCM_STATE_PAUSED)
+	} else if (runtime->state != SNDRV_PCM_STATE_PAUSED)
 		return -EBADFD;
 	runtime->trigger_master = substream;
 	return 0;
@@ -1628,12 +1628,12 @@ static void snd_pcm_post_pause(struct snd_pcm_substream *substream,
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	snd_pcm_trigger_tstamp(substream);
 	if (pause_pushed(state)) {
-		runtime->status->state = SNDRV_PCM_STATE_PAUSED;
+		__snd_pcm_set_state(runtime, SNDRV_PCM_STATE_PAUSED);
 		snd_pcm_timer_notify(substream, SNDRV_TIMER_EVENT_MPAUSE);
 		wake_up(&runtime->sleep);
 		wake_up(&runtime->tsleep);
 	} else {
-		runtime->status->state = SNDRV_PCM_STATE_RUNNING;
+		__snd_pcm_set_state(runtime, SNDRV_PCM_STATE_RUNNING);
 		snd_pcm_timer_notify(substream, SNDRV_TIMER_EVENT_MCONTINUE);
 	}
 }
@@ -1668,7 +1668,7 @@ static int snd_pcm_pre_suspend(struct snd_pcm_substream *substream,
 			       snd_pcm_state_t state)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
-	switch (runtime->status->state) {
+	switch (runtime->state) {
 	case SNDRV_PCM_STATE_SUSPENDED:
 		return -EBUSY;
 	/* unresumable PCM state; return -EBUSY for skipping suspend */
@@ -1699,8 +1699,9 @@ static void snd_pcm_post_suspend(struct snd_pcm_substream *substream,
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	snd_pcm_trigger_tstamp(substream);
-	runtime->status->suspended_state = runtime->status->state;
-	runtime->status->state = SNDRV_PCM_STATE_SUSPENDED;
+	runtime->suspended_state = runtime->state;
+	runtime->status->suspended_state = runtime->suspended_state;
+	__snd_pcm_set_state(runtime, SNDRV_PCM_STATE_SUSPENDED);
 	snd_pcm_timer_notify(substream, SNDRV_TIMER_EVENT_MSUSPEND);
 	wake_up(&runtime->sleep);
 	wake_up(&runtime->tsleep);
@@ -1791,8 +1792,8 @@ static int snd_pcm_do_resume(struct snd_pcm_substream *substream,
 	if (runtime->trigger_master != substream)
 		return 0;
 	/* DMA not running previously? */
-	if (runtime->status->suspended_state != SNDRV_PCM_STATE_RUNNING &&
-	    (runtime->status->suspended_state != SNDRV_PCM_STATE_DRAINING ||
+	if (runtime->suspended_state != SNDRV_PCM_STATE_RUNNING &&
+	    (runtime->suspended_state != SNDRV_PCM_STATE_DRAINING ||
 	     substream->stream != SNDRV_PCM_STREAM_PLAYBACK))
 		return 0;
 	return substream->ops->trigger(substream, SNDRV_PCM_TRIGGER_RESUME);
@@ -1811,7 +1812,7 @@ static void snd_pcm_post_resume(struct snd_pcm_substream *substream,
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	snd_pcm_trigger_tstamp(substream);
-	runtime->status->state = runtime->status->suspended_state;
+	__snd_pcm_set_state(runtime, runtime->suspended_state);
 	snd_pcm_timer_notify(substream, SNDRV_TIMER_EVENT_MRESUME);
 }
 
@@ -1848,7 +1849,7 @@ static int snd_pcm_xrun(struct snd_pcm_substream *substream)
 	int result;
 
 	snd_pcm_stream_lock_irq(substream);
-	switch (runtime->status->state) {
+	switch (runtime->state) {
 	case SNDRV_PCM_STATE_XRUN:
 		result = 0;	/* already there */
 		break;
@@ -1871,7 +1872,7 @@ static int snd_pcm_pre_reset(struct snd_pcm_substream *substream,
 			     snd_pcm_state_t state)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
-	switch (runtime->status->state) {
+	switch (runtime->state) {
 	case SNDRV_PCM_STATE_RUNNING:
 	case SNDRV_PCM_STATE_PREPARED:
 	case SNDRV_PCM_STATE_PAUSED:
@@ -1933,8 +1934,8 @@ static int snd_pcm_pre_prepare(struct snd_pcm_substream *substream,
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	int f_flags = (__force int)state;
 
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN ||
-	    runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN ||
+	    runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 	if (snd_pcm_running(substream))
 		return -EBUSY;
@@ -1985,7 +1986,7 @@ static int snd_pcm_prepare(struct snd_pcm_substream *substream,
 		f_flags = substream->f_flags;
 
 	snd_pcm_stream_lock_irq(substream);
-	switch (substream->runtime->status->state) {
+	switch (substream->runtime->state) {
 	case SNDRV_PCM_STATE_PAUSED:
 		snd_pcm_pause(substream, false);
 		fallthrough;
@@ -2009,7 +2010,7 @@ static int snd_pcm_pre_drain_init(struct snd_pcm_substream *substream,
 				  snd_pcm_state_t state)
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
-	switch (runtime->status->state) {
+	switch (runtime->state) {
 	case SNDRV_PCM_STATE_OPEN:
 	case SNDRV_PCM_STATE_DISCONNECTED:
 	case SNDRV_PCM_STATE_SUSPENDED:
@@ -2024,28 +2025,28 @@ static int snd_pcm_do_drain_init(struct snd_pcm_substream *substream,
 {
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
-		switch (runtime->status->state) {
+		switch (runtime->state) {
 		case SNDRV_PCM_STATE_PREPARED:
 			/* start playback stream if possible */
 			if (! snd_pcm_playback_empty(substream)) {
 				snd_pcm_do_start(substream, SNDRV_PCM_STATE_DRAINING);
 				snd_pcm_post_start(substream, SNDRV_PCM_STATE_DRAINING);
 			} else {
-				runtime->status->state = SNDRV_PCM_STATE_SETUP;
+				__snd_pcm_set_state(runtime, SNDRV_PCM_STATE_SETUP);
 			}
 			break;
 		case SNDRV_PCM_STATE_RUNNING:
-			runtime->status->state = SNDRV_PCM_STATE_DRAINING;
+			__snd_pcm_set_state(runtime, SNDRV_PCM_STATE_DRAINING);
 			break;
 		case SNDRV_PCM_STATE_XRUN:
-			runtime->status->state = SNDRV_PCM_STATE_SETUP;
+			__snd_pcm_set_state(runtime, SNDRV_PCM_STATE_SETUP);
 			break;
 		default:
 			break;
 		}
 	} else {
 		/* stop running stream */
-		if (runtime->status->state == SNDRV_PCM_STATE_RUNNING) {
+		if (runtime->state == SNDRV_PCM_STATE_RUNNING) {
 			snd_pcm_state_t new_state;
 
 			new_state = snd_pcm_capture_avail(runtime) > 0 ?
@@ -2055,7 +2056,7 @@ static int snd_pcm_do_drain_init(struct snd_pcm_substream *substream,
 		}
 	}
 
-	if (runtime->status->state == SNDRV_PCM_STATE_DRAINING &&
+	if (runtime->state == SNDRV_PCM_STATE_DRAINING &&
 	    runtime->trigger_master == substream &&
 	    (runtime->hw.info & SNDRV_PCM_INFO_DRAIN_TRIGGER))
 		return substream->ops->trigger(substream,
@@ -2096,7 +2097,7 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream,
 	card = substream->pcm->card;
 	runtime = substream->runtime;
 
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN)
 		return -EBADFD;
 
 	if (file) {
@@ -2107,7 +2108,7 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream,
 
 	snd_pcm_stream_lock_irq(substream);
 	/* resume pause */
-	if (runtime->status->state == SNDRV_PCM_STATE_PAUSED)
+	if (runtime->state == SNDRV_PCM_STATE_PAUSED)
 		snd_pcm_pause(substream, false);
 
 	/* pre-start/stop - all running streams are changed to DRAINING state */
@@ -2135,7 +2136,7 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream,
 			if (s->stream != SNDRV_PCM_STREAM_PLAYBACK)
 				continue;
 			runtime = s->runtime;
-			if (runtime->status->state == SNDRV_PCM_STATE_DRAINING) {
+			if (runtime->state == SNDRV_PCM_STATE_DRAINING) {
 				to_check = runtime;
 				break;
 			}
@@ -2174,7 +2175,7 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream,
 			break;
 		}
 		if (tout == 0) {
-			if (substream->runtime->status->state == SNDRV_PCM_STATE_SUSPENDED)
+			if (substream->runtime->state == SNDRV_PCM_STATE_SUSPENDED)
 				result = -ESTRPIPE;
 			else {
 				dev_dbg(substream->pcm->card->dev,
@@ -2206,13 +2207,13 @@ static int snd_pcm_drop(struct snd_pcm_substream *substream)
 		return -ENXIO;
 	runtime = substream->runtime;
 
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN ||
-	    runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN ||
+	    runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 
 	snd_pcm_stream_lock_irq(substream);
 	/* resume pause */
-	if (runtime->status->state == SNDRV_PCM_STATE_PAUSED)
+	if (runtime->state == SNDRV_PCM_STATE_PAUSED)
 		snd_pcm_pause(substream, false);
 
 	snd_pcm_stop(substream, SNDRV_PCM_STATE_SETUP);
@@ -2275,8 +2276,8 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd)
 	snd_pcm_group_init(group);
 
 	down_write(&snd_pcm_link_rwsem);
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN ||
-	    substream->runtime->status->state != substream1->runtime->status->state ||
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN ||
+	    substream->runtime->state != substream1->runtime->state ||
 	    substream->pcm->nonatomic != substream1->pcm->nonatomic) {
 		res = -EBADFD;
 		goto _end;
@@ -2700,7 +2701,7 @@ void snd_pcm_release_substream(struct snd_pcm_substream *substream)
 
 	snd_pcm_drop(substream);
 	if (substream->hw_opened) {
-		if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN)
+		if (substream->runtime->state != SNDRV_PCM_STATE_OPEN)
 			do_hw_free(substream);
 		substream->ops->close(substream);
 		substream->hw_opened = 0;
@@ -2904,7 +2905,7 @@ static int snd_pcm_release(struct inode *inode, struct file *file)
  */
 static int do_pcm_hwsync(struct snd_pcm_substream *substream)
 {
-	switch (substream->runtime->status->state) {
+	switch (substream->runtime->state) {
 	case SNDRV_PCM_STATE_DRAINING:
 		if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
 			return -EBADFD;
@@ -3203,7 +3204,7 @@ static int snd_pcm_xferi_frames_ioctl(struct snd_pcm_substream *substream,
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	snd_pcm_sframes_t result;
 
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN)
 		return -EBADFD;
 	if (put_user(0, &_xferi->result))
 		return -EFAULT;
@@ -3226,7 +3227,7 @@ static int snd_pcm_xfern_frames_ioctl(struct snd_pcm_substream *substream,
 	void *bufs;
 	snd_pcm_sframes_t result;
 
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN)
 		return -EBADFD;
 	if (runtime->channels > 128)
 		return -EINVAL;
@@ -3290,7 +3291,7 @@ static int snd_pcm_common_ioctl(struct file *file,
 	if (PCM_RUNTIME_CHECK(substream))
 		return -ENXIO;
 
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (substream->runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 
 	res = snd_power_wait(substream->pcm->card);
@@ -3421,7 +3422,7 @@ int snd_pcm_kernel_ioctl(struct snd_pcm_substream *substream,
 	snd_pcm_uframes_t *frames = arg;
 	snd_pcm_sframes_t result;
 	
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (substream->runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 
 	switch (cmd) {
@@ -3466,8 +3467,8 @@ static ssize_t snd_pcm_read(struct file *file, char __user *buf, size_t count,
 	if (PCM_RUNTIME_CHECK(substream))
 		return -ENXIO;
 	runtime = substream->runtime;
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN ||
-	    runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN ||
+	    runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 	if (!frame_aligned(runtime, count))
 		return -EINVAL;
@@ -3491,8 +3492,8 @@ static ssize_t snd_pcm_write(struct file *file, const char __user *buf,
 	if (PCM_RUNTIME_CHECK(substream))
 		return -ENXIO;
 	runtime = substream->runtime;
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN ||
-	    runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN ||
+	    runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 	if (!frame_aligned(runtime, count))
 		return -EINVAL;
@@ -3518,8 +3519,8 @@ static ssize_t snd_pcm_readv(struct kiocb *iocb, struct iov_iter *to)
 	if (PCM_RUNTIME_CHECK(substream))
 		return -ENXIO;
 	runtime = substream->runtime;
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN ||
-	    runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN ||
+	    runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 	if (!iter_is_iovec(to))
 		return -EINVAL;
@@ -3555,8 +3556,8 @@ static ssize_t snd_pcm_writev(struct kiocb *iocb, struct iov_iter *from)
 	if (PCM_RUNTIME_CHECK(substream))
 		return -ENXIO;
 	runtime = substream->runtime;
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN ||
-	    runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN ||
+	    runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 	if (!iter_is_iovec(from))
 		return -EINVAL;
@@ -3595,7 +3596,7 @@ static __poll_t snd_pcm_poll(struct file *file, poll_table *wait)
 		return ok | EPOLLERR;
 
 	runtime = substream->runtime;
-	if (runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return ok | EPOLLERR;
 
 	poll_wait(file, &runtime->sleep, wait);
@@ -3603,7 +3604,7 @@ static __poll_t snd_pcm_poll(struct file *file, poll_table *wait)
 	mask = 0;
 	snd_pcm_stream_lock_irq(substream);
 	avail = snd_pcm_avail(substream);
-	switch (runtime->status->state) {
+	switch (runtime->state) {
 	case SNDRV_PCM_STATE_RUNNING:
 	case SNDRV_PCM_STATE_PREPARED:
 	case SNDRV_PCM_STATE_PAUSED:
@@ -3667,6 +3668,7 @@ static int snd_pcm_mmap_status(struct snd_pcm_substream *substream, struct file
 	area->vm_ops = &snd_pcm_vm_ops_status;
 	area->vm_private_data = substream;
 	area->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
+	area->vm_flags &= ~(VM_WRITE | VM_MAYWRITE);
 	return 0;
 }
 
@@ -3874,7 +3876,7 @@ int snd_pcm_mmap_data(struct snd_pcm_substream *substream, struct file *file,
 			return -EINVAL;
 	}
 	runtime = substream->runtime;
-	if (runtime->status->state == SNDRV_PCM_STATE_OPEN)
+	if (runtime->state == SNDRV_PCM_STATE_OPEN)
 		return -EBADFD;
 	if (!(runtime->info & SNDRV_PCM_INFO_MMAP))
 		return -ENXIO;
@@ -3911,7 +3913,7 @@ static int snd_pcm_mmap(struct file *file, struct vm_area_struct *area)
 	substream = pcm_file->substream;
 	if (PCM_RUNTIME_CHECK(substream))
 		return -ENXIO;
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (substream->runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 
 	offset = area->vm_pgoff << PAGE_SHIFT;
@@ -3949,7 +3951,7 @@ static int snd_pcm_fasync(int fd, struct file * file, int on)
 	if (PCM_RUNTIME_CHECK(substream))
 		return -ENXIO;
 	runtime = substream->runtime;
-	if (runtime->status->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 	return snd_fasync_helper(fd, file, on, &runtime->fasync);
 }
diff --git a/sound/drivers/aloop.c b/sound/drivers/aloop.c
index 12f12a294df5..a38e602b4fc6 100644
--- a/sound/drivers/aloop.c
+++ b/sound/drivers/aloop.c
@@ -535,7 +535,7 @@ static void copy_play_buf(struct loopback_pcm *play,
 
 	/* check if playback is draining, trim the capture copy size
 	 * when our pointer is at the end of playback ring buffer */
-	if (runtime->status->state == SNDRV_PCM_STATE_DRAINING &&
+	if (runtime->state == SNDRV_PCM_STATE_DRAINING &&
 	    snd_pcm_playback_hw_avail(runtime) < runtime->buffer_size) { 
 	    	snd_pcm_uframes_t appl_ptr, appl_ptr1, diff;
 		appl_ptr = appl_ptr1 = runtime->control->appl_ptr;
@@ -730,7 +730,7 @@ static void loopback_snd_timer_period_elapsed(struct loopback_cable *cable,
 
 	if (event == SNDRV_TIMER_EVENT_MSTOP) {
 		if (!dpcm_play ||
-		    dpcm_play->substream->runtime->status->state !=
+		    dpcm_play->substream->runtime->state !=
 				SNDRV_PCM_STATE_DRAINING) {
 			spin_unlock_irqrestore(&cable->lock, flags);
 			return;
diff --git a/sound/drivers/dummy.c b/sound/drivers/dummy.c
index 2a7fc49c1a7c..9c17b49a2ae1 100644
--- a/sound/drivers/dummy.c
+++ b/sound/drivers/dummy.c
@@ -42,6 +42,8 @@ MODULE_LICENSE("GPL");
 #define USE_CHANNELS_MAX 	2
 #define USE_PERIODS_MIN 	1
 #define USE_PERIODS_MAX 	1024
+#define USE_MIXER_VOLUME_LEVEL_MIN	-50
+#define USE_MIXER_VOLUME_LEVEL_MAX	100
 
 static int index[SNDRV_CARDS] = SNDRV_DEFAULT_IDX;	/* Index 0-MAX */
 static char *id[SNDRV_CARDS] = SNDRV_DEFAULT_STR;	/* ID for this card */
@@ -50,6 +52,8 @@ static char *model[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS - 1)] = NULL};
 static int pcm_devs[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS - 1)] = 1};
 static int pcm_substreams[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS - 1)] = 8};
 //static int midi_devs[SNDRV_CARDS] = {[0 ... (SNDRV_CARDS - 1)] = 2};
+static int mixer_volume_level_min = USE_MIXER_VOLUME_LEVEL_MIN;
+static int mixer_volume_level_max = USE_MIXER_VOLUME_LEVEL_MAX;
 #ifdef CONFIG_HIGH_RES_TIMERS
 static bool hrtimer = 1;
 #endif
@@ -69,6 +73,10 @@ module_param_array(pcm_substreams, int, NULL, 0444);
 MODULE_PARM_DESC(pcm_substreams, "PCM substreams # (1-128) for dummy driver.");
 //module_param_array(midi_devs, int, NULL, 0444);
 //MODULE_PARM_DESC(midi_devs, "MIDI devices # (0-2) for dummy driver.");
+module_param(mixer_volume_level_min, int, 0444);
+MODULE_PARM_DESC(mixer_volume_level_min, "Minimum mixer volume level for dummy driver. Default: -50");
+module_param(mixer_volume_level_max, int, 0444);
+MODULE_PARM_DESC(mixer_volume_level_max, "Maximum mixer volume level for dummy driver. Default: 100");
 module_param(fake_buffer, bool, 0444);
 MODULE_PARM_DESC(fake_buffer, "Fake buffer allocations.");
 #ifdef CONFIG_HIGH_RES_TIMERS
@@ -296,7 +304,7 @@ static void dummy_systimer_callback(struct timer_list *t)
 	struct dummy_systimer_pcm *dpcm = from_timer(dpcm, t, timer);
 	unsigned long flags;
 	int elapsed = 0;
-	
+
 	spin_lock_irqsave(&dpcm->lock, flags);
 	dummy_systimer_update(dpcm);
 	dummy_systimer_rearm(dpcm);
@@ -713,11 +721,11 @@ static int snd_dummy_volume_info(struct snd_kcontrol *kcontrol,
 {
 	uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
 	uinfo->count = 2;
-	uinfo->value.integer.min = -50;
-	uinfo->value.integer.max = 100;
+	uinfo->value.integer.min = mixer_volume_level_min;
+	uinfo->value.integer.max = mixer_volume_level_max;
 	return 0;
 }
- 
+
 static int snd_dummy_volume_get(struct snd_kcontrol *kcontrol,
 				struct snd_ctl_elem_value *ucontrol)
 {
@@ -739,15 +747,15 @@ static int snd_dummy_volume_put(struct snd_kcontrol *kcontrol,
 	int left, right;
 
 	left = ucontrol->value.integer.value[0];
-	if (left < -50)
-		left = -50;
-	if (left > 100)
-		left = 100;
+	if (left < mixer_volume_level_min)
+		left = mixer_volume_level_min;
+	if (left > mixer_volume_level_max)
+		left = mixer_volume_level_max;
 	right = ucontrol->value.integer.value[1];
-	if (right < -50)
-		right = -50;
-	if (right > 100)
-		right = 100;
+	if (right < mixer_volume_level_min)
+		right = mixer_volume_level_min;
+	if (right > mixer_volume_level_max)
+		right = mixer_volume_level_max;
 	spin_lock_irq(&dummy->mixer_lock);
 	change = dummy->mixer_volume[addr][0] != left ||
 	         dummy->mixer_volume[addr][1] != right;
@@ -766,7 +774,7 @@ static const DECLARE_TLV_DB_SCALE(db_scale_dummy, -4500, 30, 0);
   .private_value = addr }
 
 #define snd_dummy_capsrc_info	snd_ctl_boolean_stereo_info
- 
+
 static int snd_dummy_capsrc_get(struct snd_kcontrol *kcontrol,
 				struct snd_ctl_elem_value *ucontrol)
 {
@@ -1076,6 +1084,12 @@ static int snd_dummy_probe(struct platform_device *devptr)
 			dummy->pcm_hw.channels_max = m->channels_max;
 	}
 
+	if (mixer_volume_level_min > mixer_volume_level_max) {
+		pr_warn("snd-dummy: Invalid mixer volume level: min=%d, max=%d. Fall back to default value.\n",
+		mixer_volume_level_min, mixer_volume_level_max);
+		mixer_volume_level_min = USE_MIXER_VOLUME_LEVEL_MIN;
+		mixer_volume_level_max = USE_MIXER_VOLUME_LEVEL_MAX;
+	}
 	err = snd_card_dummy_new_mixer(dummy);
 	if (err < 0)
 		return err;
@@ -1100,7 +1114,7 @@ static int snd_dummy_suspend(struct device *pdev)
 	snd_power_change_state(card, SNDRV_CTL_POWER_D3hot);
 	return 0;
 }
-	
+
 static int snd_dummy_resume(struct device *pdev)
 {
 	struct snd_card *card = dev_get_drvdata(pdev);
diff --git a/sound/drivers/vx/vx_pcm.c b/sound/drivers/vx/vx_pcm.c
index 3924f5283745..ceaeb257003b 100644
--- a/sound/drivers/vx/vx_pcm.c
+++ b/sound/drivers/vx/vx_pcm.c
@@ -1215,8 +1215,7 @@ int snd_vx_pcm_new(struct vx_core *chip)
 		if (ins)
 			snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &vx_pcm_capture_ops);
 		snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC,
-					       snd_dma_continuous_data(GFP_KERNEL | GFP_DMA32),
-					       0, 0);
+					       NULL, 0, 0);
 
 		pcm->private_data = chip;
 		pcm->private_free = snd_vx_pcm_free;
diff --git a/sound/firewire/bebob/bebob_pcm.c b/sound/firewire/bebob/bebob_pcm.c
index f8d9a2041264..ce49eef0fcba 100644
--- a/sound/firewire/bebob/bebob_pcm.c
+++ b/sound/firewire/bebob/bebob_pcm.c
@@ -214,7 +214,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream,
 	struct snd_bebob *bebob = substream->private_data;
 	int err = 0;
 
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) {
 		unsigned int rate = params_rate(hw_params);
 		unsigned int frames_per_period = params_period_size(hw_params);
 		unsigned int frames_per_buffer = params_buffer_size(hw_params);
@@ -236,7 +236,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream)
 
 	mutex_lock(&bebob->mutex);
 
-	if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state != SNDRV_PCM_STATE_OPEN)
 		bebob->substreams_counter--;
 
 	snd_bebob_stream_stop_duplex(bebob);
diff --git a/sound/firewire/dice/dice-harman.c b/sound/firewire/dice/dice-harman.c
index a8ca00c397e8..212ae77dfca2 100644
--- a/sound/firewire/dice/dice-harman.c
+++ b/sound/firewire/dice/dice-harman.c
@@ -2,8 +2,6 @@
 // dice-harman.c - a part of driver for DICE based devices
 //
 // Copyright (c) 2021 Takashi Sakamoto
-//
-// Licensed under the terms of the GNU General Public License, version 2.
 
 #include "dice.h"
 
diff --git a/sound/firewire/dice/dice-pcm.c b/sound/firewire/dice/dice-pcm.c
index a69ca1111b03..d64366217d57 100644
--- a/sound/firewire/dice/dice-pcm.c
+++ b/sound/firewire/dice/dice-pcm.c
@@ -266,7 +266,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream,
 	struct snd_dice *dice = substream->private_data;
 	int err = 0;
 
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) {
 		unsigned int rate = params_rate(hw_params);
 		unsigned int events_per_period = params_period_size(hw_params);
 		unsigned int events_per_buffer = params_buffer_size(hw_params);
@@ -293,7 +293,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream)
 
 	mutex_lock(&dice->mutex);
 
-	if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state != SNDRV_PCM_STATE_OPEN)
 		--dice->substreams_counter;
 
 	snd_dice_stream_stop_duplex(dice);
diff --git a/sound/firewire/dice/dice-presonus.c b/sound/firewire/dice/dice-presonus.c
index 503f462a83f4..967cc3119a64 100644
--- a/sound/firewire/dice/dice-presonus.c
+++ b/sound/firewire/dice/dice-presonus.c
@@ -2,8 +2,6 @@
 // dice-presonus.c - a part of driver for DICE based devices
 //
 // Copyright (c) 2019 Takashi Sakamoto
-//
-// Licensed under the terms of the GNU General Public License, version 2.
 
 #include "dice.h"
 
diff --git a/sound/firewire/digi00x/digi00x-pcm.c b/sound/firewire/digi00x/digi00x-pcm.c
index b7f6eda09f9f..3bd1575c9d9c 100644
--- a/sound/firewire/digi00x/digi00x-pcm.c
+++ b/sound/firewire/digi00x/digi00x-pcm.c
@@ -190,7 +190,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream,
 	struct snd_dg00x *dg00x = substream->private_data;
 	int err = 0;
 
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) {
 		unsigned int rate = params_rate(hw_params);
 		unsigned int frames_per_period = params_period_size(hw_params);
 		unsigned int frames_per_buffer = params_buffer_size(hw_params);
@@ -212,7 +212,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream)
 
 	mutex_lock(&dg00x->mutex);
 
-	if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state != SNDRV_PCM_STATE_OPEN)
 		--dg00x->substreams_counter;
 
 	snd_dg00x_stream_stop_duplex(dg00x);
diff --git a/sound/firewire/fireface/ff-pcm.c b/sound/firewire/fireface/ff-pcm.c
index f978cc2fed7d..ec915671a79b 100644
--- a/sound/firewire/fireface/ff-pcm.c
+++ b/sound/firewire/fireface/ff-pcm.c
@@ -230,7 +230,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream,
 	struct snd_ff *ff = substream->private_data;
 	int err = 0;
 
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) {
 		unsigned int rate = params_rate(hw_params);
 		unsigned int frames_per_period = params_period_size(hw_params);
 		unsigned int frames_per_buffer = params_buffer_size(hw_params);
@@ -252,7 +252,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream)
 
 	mutex_lock(&ff->mutex);
 
-	if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state != SNDRV_PCM_STATE_OPEN)
 		--ff->substreams_counter;
 
 	snd_ff_stream_stop_duplex(ff);
diff --git a/sound/firewire/fireface/ff-protocol-former.c b/sound/firewire/fireface/ff-protocol-former.c
index bf44cad7985e..8900ffe517ed 100644
--- a/sound/firewire/fireface/ff-protocol-former.c
+++ b/sound/firewire/fireface/ff-protocol-former.c
@@ -2,8 +2,6 @@
 // ff-protocol-former.c - a part of driver for RME Fireface series
 //
 // Copyright (c) 2019 Takashi Sakamoto
-//
-// Licensed under the terms of the GNU General Public License, version 2.
 
 #include <linux/delay.h>
 
diff --git a/sound/firewire/fireface/ff-protocol-latter.c b/sound/firewire/fireface/ff-protocol-latter.c
index 7ddb7b97f02d..76c3eab36d4e 100644
--- a/sound/firewire/fireface/ff-protocol-latter.c
+++ b/sound/firewire/fireface/ff-protocol-latter.c
@@ -1,9 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
-// ff-protocol-latter - a part of driver for RME Fireface series
+// ff-protocol-latter.c - a part of driver for RME Fireface series
 //
 // Copyright (c) 2019 Takashi Sakamoto
-//
-// Licensed under the terms of the GNU General Public License, version 2.
 
 #include <linux/delay.h>
 
diff --git a/sound/firewire/fireworks/fireworks_pcm.c b/sound/firewire/fireworks/fireworks_pcm.c
index a0d5db1d8eb2..c3c21860b245 100644
--- a/sound/firewire/fireworks/fireworks_pcm.c
+++ b/sound/firewire/fireworks/fireworks_pcm.c
@@ -250,7 +250,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream,
 	struct snd_efw *efw = substream->private_data;
 	int err = 0;
 
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) {
 		unsigned int rate = params_rate(hw_params);
 		unsigned int frames_per_period = params_period_size(hw_params);
 		unsigned int frames_per_buffer = params_buffer_size(hw_params);
@@ -272,7 +272,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream)
 
 	mutex_lock(&efw->mutex);
 
-	if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state != SNDRV_PCM_STATE_OPEN)
 		--efw->substreams_counter;
 
 	snd_efw_stream_stop_duplex(efw);
diff --git a/sound/firewire/motu/motu-pcm.c b/sound/firewire/motu/motu-pcm.c
index 8e1437371263..d410c2efbde5 100644
--- a/sound/firewire/motu/motu-pcm.c
+++ b/sound/firewire/motu/motu-pcm.c
@@ -210,7 +210,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream,
 	struct snd_motu *motu = substream->private_data;
 	int err = 0;
 
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) {
 		unsigned int rate = params_rate(hw_params);
 		unsigned int frames_per_period = params_period_size(hw_params);
 		unsigned int frames_per_buffer = params_buffer_size(hw_params);
@@ -232,7 +232,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream)
 
 	mutex_lock(&motu->mutex);
 
-	if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state != SNDRV_PCM_STATE_OPEN)
 		--motu->substreams_counter;
 
 	snd_motu_stream_stop_duplex(motu);
diff --git a/sound/firewire/motu/motu-protocol-v1.c b/sound/firewire/motu/motu-protocol-v1.c
index f1d6a326dc07..e811629f167b 100644
--- a/sound/firewire/motu/motu-protocol-v1.c
+++ b/sound/firewire/motu/motu-protocol-v1.c
@@ -1,10 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
-
 // motu-protocol-v1.c - a part of driver for MOTU FireWire series
 //
 // Copyright (c) 2021 Takashi Sakamoto <o-takashi@sakamocchi.jp>
-//
-// Licensed under the terms of the GNU General Public License, version 2.
 
 #include "motu.h"
 
diff --git a/sound/firewire/oxfw/oxfw-pcm.c b/sound/firewire/oxfw/oxfw-pcm.c
index 2dfa7e179cb6..5f43a0b826d2 100644
--- a/sound/firewire/oxfw/oxfw-pcm.c
+++ b/sound/firewire/oxfw/oxfw-pcm.c
@@ -239,7 +239,7 @@ static int pcm_capture_hw_params(struct snd_pcm_substream *substream,
 	struct snd_oxfw *oxfw = substream->private_data;
 	int err = 0;
 
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) {
 		unsigned int rate = params_rate(hw_params);
 		unsigned int channels = params_channels(hw_params);
 		unsigned int frames_per_period = params_period_size(hw_params);
@@ -262,7 +262,7 @@ static int pcm_playback_hw_params(struct snd_pcm_substream *substream,
 	struct snd_oxfw *oxfw = substream->private_data;
 	int err = 0;
 
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) {
 		unsigned int rate = params_rate(hw_params);
 		unsigned int channels = params_channels(hw_params);
 		unsigned int frames_per_period = params_period_size(hw_params);
@@ -286,7 +286,7 @@ static int pcm_capture_hw_free(struct snd_pcm_substream *substream)
 
 	mutex_lock(&oxfw->mutex);
 
-	if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state != SNDRV_PCM_STATE_OPEN)
 		--oxfw->substreams_count;
 
 	snd_oxfw_stream_stop_duplex(oxfw);
@@ -301,7 +301,7 @@ static int pcm_playback_hw_free(struct snd_pcm_substream *substream)
 
 	mutex_lock(&oxfw->mutex);
 
-	if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state != SNDRV_PCM_STATE_OPEN)
 		--oxfw->substreams_count;
 
 	snd_oxfw_stream_stop_duplex(oxfw);
diff --git a/sound/firewire/tascam/tascam-pcm.c b/sound/firewire/tascam/tascam-pcm.c
index 36c1353f2494..f6da571707ac 100644
--- a/sound/firewire/tascam/tascam-pcm.c
+++ b/sound/firewire/tascam/tascam-pcm.c
@@ -119,7 +119,7 @@ static int pcm_hw_params(struct snd_pcm_substream *substream,
 	struct snd_tscm *tscm = substream->private_data;
 	int err = 0;
 
-	if (substream->runtime->status->state == SNDRV_PCM_STATE_OPEN) {
+	if (substream->runtime->state == SNDRV_PCM_STATE_OPEN) {
 		unsigned int rate = params_rate(hw_params);
 		unsigned int frames_per_period = params_period_size(hw_params);
 		unsigned int frames_per_buffer = params_buffer_size(hw_params);
@@ -141,7 +141,7 @@ static int pcm_hw_free(struct snd_pcm_substream *substream)
 
 	mutex_lock(&tscm->mutex);
 
-	if (substream->runtime->status->state != SNDRV_PCM_STATE_OPEN)
+	if (substream->runtime->state != SNDRV_PCM_STATE_OPEN)
 		--tscm->substreams_counter;
 
 	snd_tscm_stream_stop_duplex(tscm);
diff --git a/sound/hda/ext/hdac_ext_bus.c b/sound/hda/ext/hdac_ext_bus.c
index 765c40a6ccba..6004ea1c373e 100644
--- a/sound/hda/ext/hdac_ext_bus.c
+++ b/sound/hda/ext/hdac_ext_bus.c
@@ -60,59 +60,6 @@ void snd_hdac_ext_bus_exit(struct hdac_bus *bus)
 }
 EXPORT_SYMBOL_GPL(snd_hdac_ext_bus_exit);
 
-static void default_release(struct device *dev)
-{
-	snd_hdac_ext_bus_device_exit(dev_to_hdac_dev(dev));
-}
-
-/**
- * snd_hdac_ext_bus_device_init - initialize the HDA extended codec base device
- * @bus: hdac bus to attach to
- * @addr: codec address
- * @hdev: hdac device to init
- * @type: codec type (HDAC_DEV_*) to use for this device
- *
- * Returns zero for success or a negative error code.
- */
-int snd_hdac_ext_bus_device_init(struct hdac_bus *bus, int addr,
-				 struct hdac_device *hdev, int type)
-{
-	char name[15];
-	int ret;
-
-	hdev->bus = bus;
-
-	snprintf(name, sizeof(name), "ehdaudio%dD%d", bus->idx, addr);
-
-	ret  = snd_hdac_device_init(hdev, bus, name, addr);
-	if (ret < 0) {
-		dev_err(bus->dev, "device init failed for hdac device\n");
-		return ret;
-	}
-	hdev->type = type;
-	hdev->dev.release = default_release;
-
-	ret = snd_hdac_device_register(hdev);
-	if (ret) {
-		dev_err(bus->dev, "failed to register hdac device\n");
-		snd_hdac_ext_bus_device_exit(hdev);
-		return ret;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(snd_hdac_ext_bus_device_init);
-
-/**
- * snd_hdac_ext_bus_device_exit - clean up a HD-audio extended codec base device
- * @hdev: hdac device to clean up
- */
-void snd_hdac_ext_bus_device_exit(struct hdac_device *hdev)
-{
-	snd_hdac_device_exit(hdev);
-}
-EXPORT_SYMBOL_GPL(snd_hdac_ext_bus_device_exit);
-
 /**
  * snd_hdac_ext_bus_device_remove - remove HD-audio extended codec base devices
  *
diff --git a/sound/hda/ext/hdac_ext_controller.c b/sound/hda/ext/hdac_ext_controller.c
index a42f66f561f5..80876b9a87f4 100644
--- a/sound/hda/ext/hdac_ext_controller.c
+++ b/sound/hda/ext/hdac_ext_controller.c
@@ -170,7 +170,7 @@ static int check_hdac_link_power_active(struct hdac_ext_link *link, bool enable)
 {
 	int timeout;
 	u32 val;
-	int mask = (1 << AZX_MLCTL_CPA_SHIFT);
+	int mask = (1 << AZX_ML_LCTL_CPA_SHIFT);
 
 	udelay(3);
 	timeout = 150;
@@ -178,10 +178,10 @@ static int check_hdac_link_power_active(struct hdac_ext_link *link, bool enable)
 	do {
 		val = readl(link->ml_addr + AZX_REG_ML_LCTL);
 		if (enable) {
-			if (((val & mask) >> AZX_MLCTL_CPA_SHIFT))
+			if (((val & mask) >> AZX_ML_LCTL_CPA_SHIFT))
 				return 0;
 		} else {
-			if (!((val & mask) >> AZX_MLCTL_CPA_SHIFT))
+			if (!((val & mask) >> AZX_ML_LCTL_CPA_SHIFT))
 				return 0;
 		}
 		udelay(3);
@@ -197,7 +197,7 @@ static int check_hdac_link_power_active(struct hdac_ext_link *link, bool enable)
 int snd_hdac_ext_bus_link_power_up(struct hdac_ext_link *link)
 {
 	snd_hdac_updatel(link->ml_addr, AZX_REG_ML_LCTL,
-			 AZX_MLCTL_SPA, AZX_MLCTL_SPA);
+			 AZX_ML_LCTL_SPA, AZX_ML_LCTL_SPA);
 
 	return check_hdac_link_power_active(link, true);
 }
@@ -209,7 +209,7 @@ EXPORT_SYMBOL_GPL(snd_hdac_ext_bus_link_power_up);
  */
 int snd_hdac_ext_bus_link_power_down(struct hdac_ext_link *link)
 {
-	snd_hdac_updatel(link->ml_addr, AZX_REG_ML_LCTL, AZX_MLCTL_SPA, 0);
+	snd_hdac_updatel(link->ml_addr, AZX_REG_ML_LCTL, AZX_ML_LCTL_SPA, 0);
 
 	return check_hdac_link_power_active(link, false);
 }
@@ -226,7 +226,7 @@ int snd_hdac_ext_bus_link_power_up_all(struct hdac_bus *bus)
 
 	list_for_each_entry(hlink, &bus->hlink_list, list) {
 		snd_hdac_updatel(hlink->ml_addr, AZX_REG_ML_LCTL,
-				 AZX_MLCTL_SPA, AZX_MLCTL_SPA);
+				 AZX_ML_LCTL_SPA, AZX_ML_LCTL_SPA);
 		ret = check_hdac_link_power_active(hlink, true);
 		if (ret < 0)
 			return ret;
@@ -247,7 +247,7 @@ int snd_hdac_ext_bus_link_power_down_all(struct hdac_bus *bus)
 
 	list_for_each_entry(hlink, &bus->hlink_list, list) {
 		snd_hdac_updatel(hlink->ml_addr, AZX_REG_ML_LCTL,
-				 AZX_MLCTL_SPA, 0);
+				 AZX_ML_LCTL_SPA, 0);
 		ret = check_hdac_link_power_active(hlink, false);
 		if (ret < 0)
 			return ret;
@@ -281,7 +281,7 @@ int snd_hdac_ext_bus_link_get(struct hdac_bus *bus,
 		 * clear the register to invalidate all the output streams
 		 */
 		snd_hdac_updatew(link->ml_addr, AZX_REG_ML_LOSIDV,
-				 ML_LOSIDV_STREAM_MASK, 0);
+				 AZX_ML_LOSIDV_STREAM_MASK, 0);
 		/*
 		 *  wait for 521usec for codec to report status
 		 *  HDA spec section 4.3 - Codec Discovery
diff --git a/sound/hda/ext/hdac_ext_stream.c b/sound/hda/ext/hdac_ext_stream.c
index d2b5724b463f..70f3ad71aaf0 100644
--- a/sound/hda/ext/hdac_ext_stream.c
+++ b/sound/hda/ext/hdac_ext_stream.c
@@ -26,9 +26,9 @@
  * initialize the stream, if ppcap is enabled then init those and then
  * invoke hdac stream initialization routine
  */
-void snd_hdac_ext_stream_init(struct hdac_bus *bus,
-			      struct hdac_ext_stream *hext_stream,
-			      int idx, int direction, int tag)
+static void snd_hdac_ext_stream_init(struct hdac_bus *bus,
+				     struct hdac_ext_stream *hext_stream,
+				     int idx, int direction, int tag)
 {
 	if (bus->ppcap) {
 		hext_stream->pphc_addr = bus->ppcap + AZX_PPHC_BASE +
@@ -56,7 +56,6 @@ void snd_hdac_ext_stream_init(struct hdac_bus *bus,
 	hext_stream->decoupled = false;
 	snd_hdac_stream_init(bus, &hext_stream->hstream, idx, direction, tag);
 }
-EXPORT_SYMBOL_GPL(snd_hdac_ext_stream_init);
 
 /**
  * snd_hdac_ext_stream_init_all - create and initialize the stream objects
@@ -88,11 +87,11 @@ int snd_hdac_ext_stream_init_all(struct hdac_bus *bus, int start_idx,
 EXPORT_SYMBOL_GPL(snd_hdac_ext_stream_init_all);
 
 /**
- * snd_hdac_stream_free_all - free hdac extended stream objects
+ * snd_hdac_ext_stream_free_all - free hdac extended stream objects
  *
  * @bus: HD-audio core bus
  */
-void snd_hdac_stream_free_all(struct hdac_bus *bus)
+void snd_hdac_ext_stream_free_all(struct hdac_bus *bus)
 {
 	struct hdac_stream *s, *_s;
 	struct hdac_ext_stream *hext_stream;
@@ -104,7 +103,7 @@ void snd_hdac_stream_free_all(struct hdac_bus *bus)
 		kfree(hext_stream);
 	}
 }
-EXPORT_SYMBOL_GPL(snd_hdac_stream_free_all);
+EXPORT_SYMBOL_GPL(snd_hdac_ext_stream_free_all);
 
 void snd_hdac_ext_stream_decouple_locked(struct hdac_bus *bus,
 					 struct hdac_ext_stream *hext_stream,
@@ -268,19 +267,15 @@ hdac_ext_link_stream_assign(struct hdac_bus *bus,
 		if (hstream->direction != substream->stream)
 			continue;
 
-		/* check if decoupled stream and not in use is available */
-		if (hext_stream->decoupled && !hext_stream->link_locked) {
-			res = hext_stream;
-			break;
-		}
-
+		/* check if link stream is available */
 		if (!hext_stream->link_locked) {
-			snd_hdac_ext_stream_decouple_locked(bus, hext_stream, true);
 			res = hext_stream;
 			break;
 		}
+
 	}
 	if (res) {
+		snd_hdac_ext_stream_decouple_locked(bus, res, true);
 		res->link_locked = 1;
 		res->link_substream = substream;
 	}
@@ -309,13 +304,12 @@ hdac_ext_host_stream_assign(struct hdac_bus *bus,
 			continue;
 
 		if (!hstream->opened) {
-			if (!hext_stream->decoupled)
-				snd_hdac_ext_stream_decouple_locked(bus, hext_stream, true);
 			res = hext_stream;
 			break;
 		}
 	}
 	if (res) {
+		snd_hdac_ext_stream_decouple_locked(bus, res, true);
 		res->hstream.opened = 1;
 		res->hstream.running = 0;
 		res->hstream.substream = substream;
@@ -388,15 +382,17 @@ void snd_hdac_ext_stream_release(struct hdac_ext_stream *hext_stream, int type)
 
 	case HDAC_EXT_STREAM_TYPE_HOST:
 		spin_lock_irq(&bus->reg_lock);
-		if (hext_stream->decoupled && !hext_stream->link_locked)
+		/* couple link only if not in use */
+		if (!hext_stream->link_locked)
 			snd_hdac_ext_stream_decouple_locked(bus, hext_stream, false);
+		snd_hdac_stream_release_locked(&hext_stream->hstream);
 		spin_unlock_irq(&bus->reg_lock);
-		snd_hdac_stream_release(&hext_stream->hstream);
 		break;
 
 	case HDAC_EXT_STREAM_TYPE_LINK:
 		spin_lock_irq(&bus->reg_lock);
-		if (hext_stream->decoupled && !hext_stream->hstream.opened)
+		/* couple host only if not in use */
+		if (!hext_stream->hstream.opened)
 			snd_hdac_ext_stream_decouple_locked(bus, hext_stream, false);
 		hext_stream->link_locked = 0;
 		hext_stream->link_substream = NULL;
diff --git a/sound/hda/hdac_stream.c b/sound/hda/hdac_stream.c
index f3582012d22f..1b8be39c38a9 100644
--- a/sound/hda/hdac_stream.c
+++ b/sound/hda/hdac_stream.c
@@ -13,6 +13,39 @@
 #include <sound/hda_register.h>
 #include "trace.h"
 
+/*
+ * the hdac_stream library is intended to be used with the following
+ * transitions. The states are not formally defined in the code but loosely
+ * inspired by boolean variables. Note that the 'prepared' field is not used
+ * in this library but by the callers during the hw_params/prepare transitions
+ *
+ *			   |
+ *	stream_init()	   |
+ *			   v
+ *			+--+-------+
+ *			|  unused  |
+ *			+--+----+--+
+ *			   |    ^
+ *	stream_assign()	   | 	|    stream_release()
+ *			   v	|
+ *			+--+----+--+
+ *			|  opened  |
+ *			+--+----+--+
+ *			   |    ^
+ *	stream_reset()	   |    |
+ *	stream_setup()	   |	|    stream_cleanup()
+ *			   v	|
+ *			+--+----+--+
+ *			| prepared |
+ *			+--+----+--+
+ *			   |    ^
+ *	stream_start()	   | 	|    stream_stop()
+ *			   v	|
+ *			+--+----+--+
+ *			|  running |
+ *			+----------+
+ */
+
 /**
  * snd_hdac_get_stream_stripe_ctl - get stripe control value
  * @bus: HD-audio core bus
@@ -112,10 +145,10 @@ void snd_hdac_stream_start(struct hdac_stream *azx_dev, bool fresh_start)
 EXPORT_SYMBOL_GPL(snd_hdac_stream_start);
 
 /**
- * snd_hdac_stream_clear - stop a stream DMA
+ * snd_hdac_stream_clear - helper to clear stream registers and stop DMA transfers
  * @azx_dev: HD-audio core stream to stop
  */
-void snd_hdac_stream_clear(struct hdac_stream *azx_dev)
+static void snd_hdac_stream_clear(struct hdac_stream *azx_dev)
 {
 	snd_hdac_stream_updateb(azx_dev, SD_CTL,
 				SD_CTL_DMA_START | SD_INT_MASK, 0);
@@ -124,7 +157,6 @@ void snd_hdac_stream_clear(struct hdac_stream *azx_dev)
 		snd_hdac_stream_updateb(azx_dev, SD_CTL_3B, SD_CTL_STRIPE_MASK, 0);
 	azx_dev->running = false;
 }
-EXPORT_SYMBOL_GPL(snd_hdac_stream_clear);
 
 /**
  * snd_hdac_stream_stop - stop a stream
@@ -143,16 +175,27 @@ void snd_hdac_stream_stop(struct hdac_stream *azx_dev)
 EXPORT_SYMBOL_GPL(snd_hdac_stream_stop);
 
 /**
+ * snd_hdac_stop_streams - stop all streams
+ * @bus: HD-audio core bus
+ */
+void snd_hdac_stop_streams(struct hdac_bus *bus)
+{
+	struct hdac_stream *stream;
+
+	list_for_each_entry(stream, &bus->stream_list, list)
+		snd_hdac_stream_stop(stream);
+}
+EXPORT_SYMBOL_GPL(snd_hdac_stop_streams);
+
+/**
  * snd_hdac_stop_streams_and_chip - stop all streams and chip if running
  * @bus: HD-audio core bus
  */
 void snd_hdac_stop_streams_and_chip(struct hdac_bus *bus)
 {
-	struct hdac_stream *stream;
 
 	if (bus->chip_init) {
-		list_for_each_entry(stream, &bus->stream_list, list)
-			snd_hdac_stream_stop(stream);
+		snd_hdac_stop_streams(bus);
 		snd_hdac_bus_stop_chip(bus);
 	}
 }
@@ -165,7 +208,6 @@ EXPORT_SYMBOL_GPL(snd_hdac_stop_streams_and_chip);
 void snd_hdac_stream_reset(struct hdac_stream *azx_dev)
 {
 	unsigned char val;
-	int timeout;
 	int dma_run_state;
 
 	snd_hdac_stream_clear(azx_dev);
@@ -173,30 +215,17 @@ void snd_hdac_stream_reset(struct hdac_stream *azx_dev)
 	dma_run_state = snd_hdac_stream_readb(azx_dev, SD_CTL) & SD_CTL_DMA_START;
 
 	snd_hdac_stream_updateb(azx_dev, SD_CTL, 0, SD_CTL_STREAM_RESET);
-	udelay(3);
-	timeout = 300;
-	do {
-		val = snd_hdac_stream_readb(azx_dev, SD_CTL) &
-			SD_CTL_STREAM_RESET;
-		if (val)
-			break;
-	} while (--timeout);
+
+	/* wait for hardware to report that the stream entered reset */
+	snd_hdac_stream_readb_poll(azx_dev, SD_CTL, val, (val & SD_CTL_STREAM_RESET), 3, 300);
 
 	if (azx_dev->bus->dma_stop_delay && dma_run_state)
 		udelay(azx_dev->bus->dma_stop_delay);
 
-	val &= ~SD_CTL_STREAM_RESET;
-	snd_hdac_stream_writeb(azx_dev, SD_CTL, val);
-	udelay(3);
+	snd_hdac_stream_updateb(azx_dev, SD_CTL, SD_CTL_STREAM_RESET, 0);
 
-	timeout = 300;
-	/* waiting for hardware to report that the stream is out of reset */
-	do {
-		val = snd_hdac_stream_readb(azx_dev, SD_CTL) &
-			SD_CTL_STREAM_RESET;
-		if (!val)
-			break;
-	} while (--timeout);
+	/* wait for hardware to report that the stream is out of reset */
+	snd_hdac_stream_readb_poll(azx_dev, SD_CTL, val, !(val & SD_CTL_STREAM_RESET), 3, 300);
 
 	/* reset first position - may not be synced with hw at this time */
 	if (azx_dev->posbuf)
@@ -337,6 +366,21 @@ struct hdac_stream *snd_hdac_stream_assign(struct hdac_bus *bus,
 EXPORT_SYMBOL_GPL(snd_hdac_stream_assign);
 
 /**
+ * snd_hdac_stream_release_locked - release the assigned stream
+ * @azx_dev: HD-audio core stream to release
+ *
+ * Release the stream that has been assigned by snd_hdac_stream_assign().
+ * The bus->reg_lock needs to be taken at a higher level
+ */
+void snd_hdac_stream_release_locked(struct hdac_stream *azx_dev)
+{
+	azx_dev->opened = 0;
+	azx_dev->running = 0;
+	azx_dev->substream = NULL;
+}
+EXPORT_SYMBOL_GPL(snd_hdac_stream_release_locked);
+
+/**
  * snd_hdac_stream_release - release the assigned stream
  * @azx_dev: HD-audio core stream to release
  *
@@ -347,9 +391,7 @@ void snd_hdac_stream_release(struct hdac_stream *azx_dev)
 	struct hdac_bus *bus = azx_dev->bus;
 
 	spin_lock_irq(&bus->reg_lock);
-	azx_dev->opened = 0;
-	azx_dev->running = 0;
-	azx_dev->substream = NULL;
+	snd_hdac_stream_release_locked(azx_dev);
 	spin_unlock_irq(&bus->reg_lock);
 }
 EXPORT_SYMBOL_GPL(snd_hdac_stream_release);
diff --git a/sound/hda/hdmi_chmap.c b/sound/hda/hdmi_chmap.c
index aad5c4bf4d34..5d8e1d944b0a 100644
--- a/sound/hda/hdmi_chmap.c
+++ b/sound/hda/hdmi_chmap.c
@@ -774,7 +774,7 @@ static int hdmi_chmap_ctl_put(struct snd_kcontrol *kcontrol,
 	substream = snd_pcm_chmap_substream(info, ctl_idx);
 	if (!substream || !substream->runtime)
 		return 0; /* just for avoiding error from alsactl restore */
-	switch (substream->runtime->status->state) {
+	switch (substream->runtime->state) {
 	case SNDRV_PCM_STATE_OPEN:
 	case SNDRV_PCM_STATE_SETUP:
 		break;
diff --git a/sound/hda/intel-dsp-config.c b/sound/hda/intel-dsp-config.c
index d84ffdf47210..b9eb3208f288 100644
--- a/sound/hda/intel-dsp-config.c
+++ b/sound/hda/intel-dsp-config.c
@@ -428,6 +428,11 @@ static const struct config_entry config_table[] = {
 	},
 	/* Alderlake-PS */
 	{
+		.flags = FLAG_SOF,
+		.device = 0x51c9,
+		.codec_hid =  &essx_83x6,
+	},
+	{
 		.flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE,
 		.device = 0x51c9,
 	},
@@ -450,6 +455,16 @@ static const struct config_entry config_table[] = {
 		.flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE,
 		.device = 0x51cb,
 	},
+	/* RaptorLake-M */
+	{
+		.flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE,
+		.device = 0x51ce,
+	},
+	/* RaptorLake-PX */
+	{
+		.flags = FLAG_SOF | FLAG_SOF_ONLY_IF_DMIC_OR_SOUNDWIRE,
+		.device = 0x51cf,
+	},
 #endif
 
 };
diff --git a/sound/hda/intel-nhlt.c b/sound/hda/intel-nhlt.c
index 13bb0ccfb36c..2c4dfc0b7e34 100644
--- a/sound/hda/intel-nhlt.c
+++ b/sound/hda/intel-nhlt.c
@@ -157,6 +157,85 @@ int intel_nhlt_ssp_endpoint_mask(struct nhlt_acpi_table *nhlt, u8 device_type)
 }
 EXPORT_SYMBOL(intel_nhlt_ssp_endpoint_mask);
 
+#define SSP_BLOB_V1_0_SIZE		84
+#define SSP_BLOB_V1_0_MDIVC_OFFSET	19 /* offset in u32 */
+
+#define SSP_BLOB_V1_5_SIZE		96
+#define SSP_BLOB_V1_5_MDIVC_OFFSET	21 /* offset in u32 */
+#define SSP_BLOB_VER_1_5		0xEE000105
+
+#define SSP_BLOB_V2_0_SIZE		88
+#define SSP_BLOB_V2_0_MDIVC_OFFSET	20 /* offset in u32 */
+#define SSP_BLOB_VER_2_0		0xEE000200
+
+int intel_nhlt_ssp_mclk_mask(struct nhlt_acpi_table *nhlt, int ssp_num)
+{
+	struct nhlt_endpoint *epnt;
+	struct nhlt_fmt *fmt;
+	struct nhlt_fmt_cfg *cfg;
+	int mclk_mask = 0;
+	int i, j;
+
+	if (!nhlt)
+		return 0;
+
+	epnt = (struct nhlt_endpoint *)nhlt->desc;
+	for (i = 0; i < nhlt->endpoint_count; i++) {
+
+		/* we only care about endpoints connected to an audio codec over SSP */
+		if (epnt->linktype == NHLT_LINK_SSP &&
+		    epnt->device_type == NHLT_DEVICE_I2S &&
+		    epnt->virtual_bus_id == ssp_num) {
+
+			fmt = (struct nhlt_fmt *)(epnt->config.caps + epnt->config.size);
+			cfg = fmt->fmt_config;
+
+			/*
+			 * In theory all formats should use the same MCLK but it doesn't hurt to
+			 * double-check that the configuration is consistent
+			 */
+			for (j = 0; j < fmt->fmt_count; j++) {
+				u32 *blob;
+				int mdivc_offset;
+				int size;
+
+				/* first check we have enough data to read the blob type */
+				if (cfg->config.size < 8)
+					return -EINVAL;
+
+				blob = (u32 *)cfg->config.caps;
+
+				if (blob[1] == SSP_BLOB_VER_2_0) {
+					mdivc_offset = SSP_BLOB_V2_0_MDIVC_OFFSET;
+					size = SSP_BLOB_V2_0_SIZE;
+				} else if (blob[1] == SSP_BLOB_VER_1_5) {
+					mdivc_offset = SSP_BLOB_V1_5_MDIVC_OFFSET;
+					size = SSP_BLOB_V1_5_SIZE;
+				} else {
+					mdivc_offset = SSP_BLOB_V1_0_MDIVC_OFFSET;
+					size = SSP_BLOB_V1_0_SIZE;
+				}
+
+				/* make sure we have enough data for the fixed part of the blob */
+				if (cfg->config.size < size)
+					return -EINVAL;
+
+				mclk_mask |=  blob[mdivc_offset] & GENMASK(1, 0);
+
+				cfg = (struct nhlt_fmt_cfg *)(cfg->config.caps + cfg->config.size);
+			}
+		}
+		epnt = (struct nhlt_endpoint *)((u8 *)epnt + epnt->length);
+	}
+
+	/* make sure only one MCLK is used */
+	if (hweight_long(mclk_mask) != 1)
+		return -EINVAL;
+
+	return mclk_mask;
+}
+EXPORT_SYMBOL(intel_nhlt_ssp_mclk_mask);
+
 static struct nhlt_specific_cfg *
 nhlt_get_specific_cfg(struct device *dev, struct nhlt_fmt *fmt, u8 num_ch,
 		      u32 rate, u8 vbps, u8 bps)
diff --git a/sound/isa/es18xx.c b/sound/isa/es18xx.c
index 3fcd168480b6..0a32845b1017 100644
--- a/sound/isa/es18xx.c
+++ b/sound/isa/es18xx.c
@@ -1344,11 +1344,8 @@ ES18XX_SINGLE("GPO1 Switch", 0, ES18XX_PM, 1, 1, ES18XX_FL_PMPORT),
 
 static int snd_es18xx_config_read(struct snd_es18xx *chip, unsigned char reg)
 {
-	int data;
-
 	outb(reg, chip->ctrl_port);
-	data = inb(chip->ctrl_port + 1);
-	return data;
+	return inb(chip->ctrl_port + 1);
 }
 
 static void snd_es18xx_config_write(struct snd_es18xx *chip,
diff --git a/sound/isa/sb/emu8000_pcm.c b/sound/isa/sb/emu8000_pcm.c
index f8d90a1e989b..c8afc4347c54 100644
--- a/sound/isa/sb/emu8000_pcm.c
+++ b/sound/isa/sb/emu8000_pcm.c
@@ -236,7 +236,7 @@ static int emu8k_pcm_open(struct snd_pcm_substream *subs)
 
 	/* use timer to update periods.. (specified in msec) */
 	snd_pcm_hw_constraint_minmax(runtime, SNDRV_PCM_HW_PARAM_PERIOD_TIME,
-				     (1000000 + HZ - 1) / HZ, UINT_MAX);
+				     DIV_ROUND_UP(1000000, HZ), UINT_MAX);
 
 	return 0;
 }
diff --git a/sound/oss/dmasound/dmasound.h b/sound/oss/dmasound/dmasound.h
index ad8ce6a1c25c..f065840c0efb 100644
--- a/sound/oss/dmasound/dmasound.h
+++ b/sound/oss/dmasound/dmasound.h
@@ -250,7 +250,4 @@ extern int dmasound_catchRadius;
 #define SW_INPUT_VOLUME_SCALE	4
 #define SW_INPUT_VOLUME_DEFAULT	(128 / SW_INPUT_VOLUME_SCALE)
 
-extern int expand_read_bal;	/* Balance factor for reading */
-extern uint software_input_volume; /* software implemented recording volume! */
-
 #endif /* _dmasound_h_ */
diff --git a/sound/pci/asihpi/asihpi.c b/sound/pci/asihpi/asihpi.c
index 5e1f9f10051b..8de43aaa10aa 100644
--- a/sound/pci/asihpi/asihpi.c
+++ b/sound/pci/asihpi/asihpi.c
@@ -632,7 +632,7 @@ static int snd_card_asihpi_trigger(struct snd_pcm_substream *substream,
 
 			/*? workaround linked streams don't
 			transition to SETUP 20070706*/
-			s->runtime->status->state = SNDRV_PCM_STATE_SETUP;
+			__snd_pcm_set_state(s->runtime, SNDRV_PCM_STATE_SETUP);
 
 			if (card->support_grouping) {
 				snd_printdd("%d group\n", s->number);
diff --git a/sound/pci/asihpi/hpifunc.c b/sound/pci/asihpi/hpifunc.c
index 1de05383126a..24047fafef51 100644
--- a/sound/pci/asihpi/hpifunc.c
+++ b/sound/pci/asihpi/hpifunc.c
@@ -2020,7 +2020,6 @@ u16 hpi_meter_get_peak(u32 h_control, short an_peakdB[HPI_MAX_CHANNELS]
 		HPI_CONTROL_GET_STATE);
 	if (hpi_handle_indexes(h_control, &hm.adapter_index, &hm.obj_index))
 		return HPI_ERROR_INVALID_HANDLE;
-	hm.obj_index = hm.obj_index;
 	hm.u.c.attribute = HPI_METER_PEAK;
 
 	hpi_send_recv(&hm, &hr);
diff --git a/sound/pci/asihpi/hpimsgx.c b/sound/pci/asihpi/hpimsgx.c
index f7427f8eb630..d0caef299481 100644
--- a/sound/pci/asihpi/hpimsgx.c
+++ b/sound/pci/asihpi/hpimsgx.c
@@ -93,11 +93,6 @@ static void HPIMSGX__cleanup(u16 adapter_index, void *h_owner);
 #pragma pack(push, 1)
 #endif
 
-struct hpi_subsys_response {
-	struct hpi_response_header h;
-	struct hpi_subsys_res s;
-};
-
 struct hpi_adapter_response {
 	struct hpi_response_header h;
 	struct hpi_adapter_res a;
diff --git a/sound/pci/hda/cs35l41_hda.c b/sound/pci/hda/cs35l41_hda.c
index 15e2a0009080..3952f2853703 100644
--- a/sound/pci/hda/cs35l41_hda.c
+++ b/sound/pci/hda/cs35l41_hda.c
@@ -842,8 +842,8 @@ static int cs35l41_hda_bind(struct device *dev, struct device *master, void *mas
 
 	comps->dev = dev;
 	if (!cs35l41->acpi_subsystem_id)
-		cs35l41->acpi_subsystem_id = devm_kasprintf(dev, GFP_KERNEL, "%.8x",
-							    comps->codec->core.subsystem_id);
+		cs35l41->acpi_subsystem_id = kasprintf(GFP_KERNEL, "%.8x",
+						       comps->codec->core.subsystem_id);
 	cs35l41->codec = comps->codec;
 	strscpy(comps->name, dev_name(dev), sizeof(comps->name));
 
@@ -1048,36 +1048,6 @@ static int cs35l41_hda_apply_properties(struct cs35l41_hda *cs35l41)
 	return cs35l41_hda_channel_map(cs35l41->dev, 0, NULL, 1, &hw_cfg->spk_pos);
 }
 
-static int cs35l41_get_acpi_sub_string(struct device *dev, struct acpi_device *adev,
-				       const char **subsysid)
-{
-	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
-	union acpi_object *obj;
-	acpi_status status;
-	int ret = 0;
-
-	status = acpi_evaluate_object(adev->handle, "_SUB", NULL, &buffer);
-	if (ACPI_SUCCESS(status)) {
-		obj = buffer.pointer;
-		if (obj->type == ACPI_TYPE_STRING) {
-			*subsysid = devm_kstrdup(dev, obj->string.pointer, GFP_KERNEL);
-			if (*subsysid == NULL) {
-				dev_err(dev, "Cannot allocate Subsystem ID");
-				ret = -ENOMEM;
-			}
-		} else {
-			dev_warn(dev, "Warning ACPI _SUB did not return a string\n");
-			ret = -ENODEV;
-		}
-		acpi_os_free(buffer.pointer);
-	} else {
-		dev_dbg(dev, "Warning ACPI _SUB failed: %#x\n", status);
-		ret = -ENODEV;
-	}
-
-	return ret;
-}
-
 static int cs35l41_get_speaker_id(struct device *dev, int amp_index,
 				  int num_amps, int fixed_gpio_id)
 {
@@ -1154,7 +1124,6 @@ static int cs35l41_no_acpi_dsd(struct cs35l41_hda *cs35l41, struct device *physd
 	hw_cfg->gpio2.func = CS35L41_INTERRUPT;
 	hw_cfg->gpio2.valid = true;
 	hw_cfg->valid = true;
-	put_device(physdev);
 
 	if (strncmp(hid, "CLSA0100", 8) == 0) {
 		hw_cfg->bst_type = CS35L41_EXT_BOOST_NO_VSPK_SWITCH;
@@ -1183,6 +1152,7 @@ static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, i
 	u32 values[HDA_MAX_COMPONENTS];
 	struct acpi_device *adev;
 	struct device *physdev;
+	const char *sub;
 	char *property;
 	size_t nval;
 	int i, ret;
@@ -1196,17 +1166,17 @@ static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, i
 	physdev = get_device(acpi_get_first_physical_node(adev));
 	acpi_dev_put(adev);
 
-	ret = cs35l41_get_acpi_sub_string(cs35l41->dev, adev, &cs35l41->acpi_subsystem_id);
-	if (ret)
-		dev_info(cs35l41->dev, "No Subsystem ID found in ACPI: %d", ret);
-	else
-		dev_dbg(cs35l41->dev, "Subsystem ID %s found", cs35l41->acpi_subsystem_id);
+	sub = acpi_get_subsystem_id(ACPI_HANDLE(physdev));
+	if (IS_ERR(sub))
+		sub = NULL;
+	cs35l41->acpi_subsystem_id = sub;
 
 	property = "cirrus,dev-index";
 	ret = device_property_count_u32(physdev, property);
-	if (ret <= 0)
-		return cs35l41_no_acpi_dsd(cs35l41, physdev, id, hid);
-
+	if (ret <= 0) {
+		ret = cs35l41_no_acpi_dsd(cs35l41, physdev, id, hid);
+		goto err_put_physdev;
+	}
 	if (ret > ARRAY_SIZE(values)) {
 		ret = -EINVAL;
 		goto err;
@@ -1295,8 +1265,9 @@ static int cs35l41_hda_read_acpi(struct cs35l41_hda *cs35l41, const char *hid, i
 	return 0;
 
 err:
-	put_device(physdev);
 	dev_err(cs35l41->dev, "Failed property %s: %d\n", property, ret);
+err_put_physdev:
+	put_device(physdev);
 
 	return ret;
 }
@@ -1433,6 +1404,7 @@ err:
 	if (cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type))
 		gpiod_set_value_cansleep(cs35l41->reset_gpio, 0);
 	gpiod_put(cs35l41->reset_gpio);
+	kfree(cs35l41->acpi_subsystem_id);
 
 	return ret;
 }
@@ -1455,6 +1427,7 @@ void cs35l41_hda_remove(struct device *dev)
 	if (cs35l41_safe_reset(cs35l41->regmap, cs35l41->hw_cfg.bst_type))
 		gpiod_set_value_cansleep(cs35l41->reset_gpio, 0);
 	gpiod_put(cs35l41->reset_gpio);
+	kfree(cs35l41->acpi_subsystem_id);
 }
 EXPORT_SYMBOL_NS_GPL(cs35l41_hda_remove, SND_HDA_SCODEC_CS35L41);
 
diff --git a/sound/pci/hda/cs35l41_hda_i2c.c b/sound/pci/hda/cs35l41_hda_i2c.c
index 5baacfde4f16..5a6252d9b9e1 100644
--- a/sound/pci/hda/cs35l41_hda_i2c.c
+++ b/sound/pci/hda/cs35l41_hda_i2c.c
@@ -33,11 +33,9 @@ static int cs35l41_hda_i2c_probe(struct i2c_client *clt, const struct i2c_device
 				 devm_regmap_init_i2c(clt, &cs35l41_regmap_i2c));
 }
 
-static int cs35l41_hda_i2c_remove(struct i2c_client *clt)
+static void cs35l41_hda_i2c_remove(struct i2c_client *clt)
 {
 	cs35l41_hda_remove(&clt->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id cs35l41_hda_i2c_id[] = {
diff --git a/sound/pci/hda/hda_beep.c b/sound/pci/hda/hda_beep.c
index 53a2b89f8983..e63621bcb214 100644
--- a/sound/pci/hda/hda_beep.c
+++ b/sound/pci/hda/hda_beep.c
@@ -118,6 +118,12 @@ static int snd_hda_beep_event(struct input_dev *dev, unsigned int type,
 	return 0;
 }
 
+static void turn_on_beep(struct hda_beep *beep)
+{
+	if (beep->keep_power_at_enable)
+		snd_hda_power_up_pm(beep->codec);
+}
+
 static void turn_off_beep(struct hda_beep *beep)
 {
 	cancel_work_sync(&beep->beep_work);
@@ -125,6 +131,8 @@ static void turn_off_beep(struct hda_beep *beep)
 		/* turn off beep */
 		generate_tone(beep, 0);
 	}
+	if (beep->keep_power_at_enable)
+		snd_hda_power_down_pm(beep->codec);
 }
 
 /**
@@ -140,7 +148,9 @@ int snd_hda_enable_beep_device(struct hda_codec *codec, int enable)
 	enable = !!enable;
 	if (beep->enabled != enable) {
 		beep->enabled = enable;
-		if (!enable)
+		if (enable)
+			turn_on_beep(beep);
+		else
 			turn_off_beep(beep);
 		return 1;
 	}
@@ -167,7 +177,8 @@ static int beep_dev_disconnect(struct snd_device *device)
 		input_unregister_device(beep->dev);
 	else
 		input_free_device(beep->dev);
-	turn_off_beep(beep);
+	if (beep->enabled)
+		turn_off_beep(beep);
 	return 0;
 }
 
diff --git a/sound/pci/hda/hda_beep.h b/sound/pci/hda/hda_beep.h
index a25358a4807a..db76e3ddba65 100644
--- a/sound/pci/hda/hda_beep.h
+++ b/sound/pci/hda/hda_beep.h
@@ -25,6 +25,7 @@ struct hda_beep {
 	unsigned int enabled:1;
 	unsigned int linear_tone:1;	/* linear tone for IDT/STAC codec */
 	unsigned int playing:1;
+	unsigned int keep_power_at_enable:1;	/* set by driver */
 	struct work_struct beep_work; /* scheduled task for beep event */
 	struct mutex mutex;
 	void (*power_hook)(struct hda_beep *beep, bool on);
diff --git a/sound/pci/hda/hda_codec.c b/sound/pci/hda/hda_codec.c
index 384426d7e9dd..b4d1e658c556 100644
--- a/sound/pci/hda/hda_codec.c
+++ b/sound/pci/hda/hda_codec.c
@@ -883,13 +883,7 @@ static void snd_hda_codec_dev_release(struct device *dev)
 	snd_hda_sysfs_clear(codec);
 	kfree(codec->modelname);
 	kfree(codec->wcaps);
-
-	/*
-	 * In the case of ASoC HD-audio, hda_codec is device managed.
-	 * It will be freed when the ASoC device is removed.
-	 */
-	if (codec->core.type == HDA_DEV_LEGACY)
-		kfree(codec);
+	kfree(codec);
 }
 
 #define DEV_NAME_LEN 31
@@ -931,8 +925,28 @@ snd_hda_codec_device_init(struct hda_bus *bus, unsigned int codec_addr,
 	}
 
 	codec->bus = bus;
+	codec->depop_delay = -1;
+	codec->fixup_id = HDA_FIXUP_ID_NOT_SET;
+	codec->core.dev.release = snd_hda_codec_dev_release;
+	codec->core.exec_verb = codec_exec_verb;
 	codec->core.type = HDA_DEV_LEGACY;
 
+	mutex_init(&codec->spdif_mutex);
+	mutex_init(&codec->control_mutex);
+	snd_array_init(&codec->mixers, sizeof(struct hda_nid_item), 32);
+	snd_array_init(&codec->nids, sizeof(struct hda_nid_item), 32);
+	snd_array_init(&codec->init_pins, sizeof(struct hda_pincfg), 16);
+	snd_array_init(&codec->driver_pins, sizeof(struct hda_pincfg), 16);
+	snd_array_init(&codec->cvt_setups, sizeof(struct hda_cvt_setup), 8);
+	snd_array_init(&codec->spdif_out, sizeof(struct hda_spdif_out), 16);
+	snd_array_init(&codec->jacktbl, sizeof(struct hda_jack_tbl), 16);
+	snd_array_init(&codec->verbs, sizeof(struct hda_verb *), 8);
+	INIT_LIST_HEAD(&codec->conn_list);
+	INIT_LIST_HEAD(&codec->pcm_list_head);
+	INIT_DELAYED_WORK(&codec->jackpoll_work, hda_jackpoll_work);
+	refcount_set(&codec->pcm_ref, 1);
+	init_waitqueue_head(&codec->remove_sleep);
+
 	return codec;
 }
 EXPORT_SYMBOL_GPL(snd_hda_codec_device_init);
@@ -985,29 +999,8 @@ int snd_hda_codec_device_new(struct hda_bus *bus, struct snd_card *card,
 	if (snd_BUG_ON(codec_addr > HDA_MAX_CODEC_ADDRESS))
 		return -EINVAL;
 
-	codec->core.dev.release = snd_hda_codec_dev_release;
-	codec->core.exec_verb = codec_exec_verb;
-
 	codec->card = card;
 	codec->addr = codec_addr;
-	mutex_init(&codec->spdif_mutex);
-	mutex_init(&codec->control_mutex);
-	snd_array_init(&codec->mixers, sizeof(struct hda_nid_item), 32);
-	snd_array_init(&codec->nids, sizeof(struct hda_nid_item), 32);
-	snd_array_init(&codec->init_pins, sizeof(struct hda_pincfg), 16);
-	snd_array_init(&codec->driver_pins, sizeof(struct hda_pincfg), 16);
-	snd_array_init(&codec->cvt_setups, sizeof(struct hda_cvt_setup), 8);
-	snd_array_init(&codec->spdif_out, sizeof(struct hda_spdif_out), 16);
-	snd_array_init(&codec->jacktbl, sizeof(struct hda_jack_tbl), 16);
-	snd_array_init(&codec->verbs, sizeof(struct hda_verb *), 8);
-	INIT_LIST_HEAD(&codec->conn_list);
-	INIT_LIST_HEAD(&codec->pcm_list_head);
-	refcount_set(&codec->pcm_ref, 1);
-	init_waitqueue_head(&codec->remove_sleep);
-
-	INIT_DELAYED_WORK(&codec->jackpoll_work, hda_jackpoll_work);
-	codec->depop_delay = -1;
-	codec->fixup_id = HDA_FIXUP_ID_NOT_SET;
 
 #ifdef CONFIG_PM
 	codec->power_jiffies = jiffies;
diff --git a/sound/pci/hda/hda_controller.c b/sound/pci/hda/hda_controller.c
index 75dcb14ff20a..0ff286b7b66b 100644
--- a/sound/pci/hda/hda_controller.c
+++ b/sound/pci/hda/hda_controller.c
@@ -1033,10 +1033,8 @@ EXPORT_SYMBOL_GPL(azx_init_chip);
 void azx_stop_all_streams(struct azx *chip)
 {
 	struct hdac_bus *bus = azx_bus(chip);
-	struct hdac_stream *s;
 
-	list_for_each_entry(s, &bus->stream_list, list)
-		snd_hdac_stream_stop(s);
+	snd_hdac_stop_streams(bus);
 }
 EXPORT_SYMBOL_GPL(azx_stop_all_streams);
 
diff --git a/sound/pci/hda/hda_eld.c b/sound/pci/hda/hda_eld.c
index 9e97443795f8..1d108ed5c6f2 100644
--- a/sound/pci/hda/hda_eld.c
+++ b/sound/pci/hda/hda_eld.c
@@ -440,7 +440,8 @@ static void hdmi_print_sad_info(int i, struct cea_sad *a,
 }
 
 void snd_hdmi_print_eld_info(struct hdmi_eld *eld,
-			     struct snd_info_buffer *buffer)
+			     struct snd_info_buffer *buffer,
+			     hda_nid_t pin_nid, int dev_id, hda_nid_t cvt_nid)
 {
 	struct parsed_hdmi_eld *e = &eld->info;
 	char buf[SND_PRINT_CHANNEL_ALLOCATION_ADVISED_BUFSIZE];
@@ -462,6 +463,9 @@ void snd_hdmi_print_eld_info(struct hdmi_eld *eld,
 
 	snd_iprintf(buffer, "monitor_present\t\t%d\n", eld->monitor_present);
 	snd_iprintf(buffer, "eld_valid\t\t%d\n", eld->eld_valid);
+	snd_iprintf(buffer, "codec_pin_nid\t\t0x%x\n", pin_nid);
+	snd_iprintf(buffer, "codec_dev_id\t\t0x%x\n", dev_id);
+	snd_iprintf(buffer, "codec_cvt_nid\t\t0x%x\n", cvt_nid);
 	if (!eld->eld_valid)
 		return;
 	snd_iprintf(buffer, "monitor_name\t\t%s\n", e->monitor_name);
diff --git a/sound/pci/hda/hda_intel.c b/sound/pci/hda/hda_intel.c
index 6f30c374f896..6ff19dd0d10c 100644
--- a/sound/pci/hda/hda_intel.c
+++ b/sound/pci/hda/hda_intel.c
@@ -86,9 +86,6 @@ enum {
 #define INTEL_SCH_HDA_DEVC      0x78
 #define INTEL_SCH_HDA_DEVC_NOSNOOP       (0x1<<11)
 
-/* Define VIA HD Audio Device ID*/
-#define VIA_HDAC_DEVICE_ID		0x3288
-
 /* max number of SDs */
 /* ICH, ATI and VIA have 4 playback and 4 capture */
 #define ICH6_NUM_CAPTURE	4
@@ -102,10 +99,6 @@ enum {
 #define ATIHDMI_NUM_CAPTURE	0
 #define ATIHDMI_NUM_PLAYBACK	8
 
-/* TERA has 4 playback and 3 capture */
-#define TERA_NUM_CAPTURE	3
-#define TERA_NUM_PLAYBACK	4
-
 
 static int index[SNDRV_CARDS] = SNDRV_DEFAULT_IDX;
 static char *id[SNDRV_CARDS] = SNDRV_DEFAULT_STR;
@@ -496,14 +489,14 @@ static int intel_ml_lctl_set_power(struct azx *chip, int state)
 	 * If other links are enabled for stream, they need similar fix
 	 */
 	val = readl(bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL);
-	val &= ~AZX_MLCTL_SPA;
-	val |= state << AZX_MLCTL_SPA_SHIFT;
+	val &= ~AZX_ML_LCTL_SPA;
+	val |= state << AZX_ML_LCTL_SPA_SHIFT;
 	writel(val, bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL);
 	/* wait for CPA */
 	timeout = 50;
 	while (timeout) {
 		if (((readl(bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL)) &
-		    AZX_MLCTL_CPA) == (state << AZX_MLCTL_CPA_SHIFT))
+		    AZX_ML_LCTL_CPA) == (state << AZX_ML_LCTL_CPA_SHIFT))
 			return 0;
 		timeout--;
 		udelay(10);
@@ -521,15 +514,15 @@ static void intel_init_lctl(struct azx *chip)
 	/* 0. check lctl register value is correct or not */
 	val = readl(bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL);
 	/* if SCF is already set, let's use it */
-	if ((val & ML_LCTL_SCF_MASK) != 0)
+	if ((val & AZX_ML_LCTL_SCF) != 0)
 		return;
 
 	/*
 	 * Before operating on SPA, CPA must match SPA.
 	 * Any deviation may result in undefined behavior.
 	 */
-	if (((val & AZX_MLCTL_SPA) >> AZX_MLCTL_SPA_SHIFT) !=
-		((val & AZX_MLCTL_CPA) >> AZX_MLCTL_CPA_SHIFT))
+	if (((val & AZX_ML_LCTL_SPA) >> AZX_ML_LCTL_SPA_SHIFT) !=
+		((val & AZX_ML_LCTL_CPA) >> AZX_ML_LCTL_CPA_SHIFT))
 		return;
 
 	/* 1. turn link down: set SPA to 0 and wait CPA to 0 */
@@ -539,7 +532,7 @@ static void intel_init_lctl(struct azx *chip)
 		goto set_spa;
 
 	/* 2. update SCF to select a properly audio clock*/
-	val &= ~ML_LCTL_SCF_MASK;
+	val &= ~AZX_ML_LCTL_SCF;
 	val |= intel_get_lctl_scf(chip);
 	writel(val, bus->mlcap + AZX_ML_BASE + AZX_REG_ML_LCTL);
 
@@ -2554,7 +2547,8 @@ static const struct pci_device_id azx_ids[] = {
 	  .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH_NOPM },
 	/* Poulsbo */
 	{ PCI_DEVICE(0x8086, 0x811b),
-	  .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH_BASE },
+	  .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH_BASE |
+	  AZX_DCAPS_POSFIX_LPIB },
 	/* Oaktrail */
 	{ PCI_DEVICE(0x8086, 0x080a),
 	  .driver_data = AZX_DRIVER_SCH | AZX_DCAPS_INTEL_PCH_BASE },
diff --git a/sound/pci/hda/hda_local.h b/sound/pci/hda/hda_local.h
index 682dca2057db..53a5a62b78fa 100644
--- a/sound/pci/hda/hda_local.h
+++ b/sound/pci/hda/hda_local.h
@@ -712,7 +712,8 @@ int snd_hdmi_get_eld_ati(struct hda_codec *codec, hda_nid_t nid,
 
 #ifdef CONFIG_SND_PROC_FS
 void snd_hdmi_print_eld_info(struct hdmi_eld *eld,
-			     struct snd_info_buffer *buffer);
+			     struct snd_info_buffer *buffer,
+			     hda_nid_t pin_nid, int dev_id, hda_nid_t cvt_nid);
 void snd_hdmi_write_eld_info(struct hdmi_eld *eld,
 			     struct snd_info_buffer *buffer);
 #endif
diff --git a/sound/pci/hda/hda_sysfs.c b/sound/pci/hda/hda_sysfs.c
index bf951c10ae61..69ebc37a4d6f 100644
--- a/sound/pci/hda/hda_sysfs.c
+++ b/sound/pci/hda/hda_sysfs.c
@@ -375,8 +375,6 @@ static ssize_t user_pin_configs_show(struct device *dev,
 	return pin_configs_show(codec, &codec->user_pins, buf);
 }
 
-#define MAX_PIN_CONFIGS		32
-
 static int parse_user_pin_configs(struct hda_codec *codec, const char *buf)
 {
 	int nid, cfg, err;
diff --git a/sound/pci/hda/patch_ca0132.c b/sound/pci/hda/patch_ca0132.c
index 208933792787..9580fe00cbd9 100644
--- a/sound/pci/hda/patch_ca0132.c
+++ b/sound/pci/hda/patch_ca0132.c
@@ -2962,7 +2962,6 @@ static int dsp_allocate_ports_format(struct hda_codec *codec,
 			const unsigned short fmt,
 			unsigned int *port_map)
 {
-	int status;
 	unsigned int num_chans;
 
 	unsigned int sample_rate_div = ((get_hdafmt_rate(fmt) >> 0) & 3) + 1;
@@ -2976,9 +2975,7 @@ static int dsp_allocate_ports_format(struct hda_codec *codec,
 
 	num_chans = get_hdafmt_chs(fmt) + 1;
 
-	status = dsp_allocate_ports(codec, num_chans, rate_multi, port_map);
-
-	return status;
+	return dsp_allocate_ports(codec, num_chans, rate_multi, port_map);
 }
 
 /*
diff --git a/sound/pci/hda/patch_hdmi.c b/sound/pci/hda/patch_hdmi.c
index c239d9dbbaef..21edf7a619f0 100644
--- a/sound/pci/hda/patch_hdmi.c
+++ b/sound/pci/hda/patch_hdmi.c
@@ -53,7 +53,8 @@ MODULE_PARM_DESC(enable_all_pins, "Forcibly enable all pins");
 
 struct hdmi_spec_per_cvt {
 	hda_nid_t cvt_nid;
-	int assigned;
+	bool assigned;		/* the stream has been assigned */
+	bool silent_stream;	/* silent stream activated */
 	unsigned int channels_min;
 	unsigned int channels_max;
 	u32 rates;
@@ -150,7 +151,7 @@ struct hdmi_spec {
 	 */
 	int dev_num;
 	struct snd_array pins; /* struct hdmi_spec_per_pin */
-	struct hdmi_pcm pcm_rec[16];
+	struct hdmi_pcm pcm_rec[8];
 	struct mutex pcm_lock;
 	struct mutex bind_lock; /* for audio component binding */
 	/* pcm_bitmap means which pcms have been assigned to pins*/
@@ -166,8 +167,6 @@ struct hdmi_spec {
 	struct hdmi_ops ops;
 
 	bool dyn_pin_out;
-	bool dyn_pcm_assign;
-	bool dyn_pcm_no_legacy;
 	/* hdmi interrupt trigger control flag for Nvidia codec */
 	bool hdmi_intr_trig_ctrl;
 	bool nv_dp_workaround; /* workaround DP audio infoframe for Nvidia */
@@ -231,7 +230,7 @@ struct dp_audio_infoframe {
 union audio_infoframe {
 	struct hdmi_audio_infoframe hdmi;
 	struct dp_audio_infoframe dp;
-	u8 bytes[0];
+	DECLARE_FLEX_ARRAY(u8, bytes);
 };
 
 /*
@@ -495,7 +494,8 @@ static void print_eld_info(struct snd_info_entry *entry,
 	struct hdmi_spec_per_pin *per_pin = entry->private_data;
 
 	mutex_lock(&per_pin->lock);
-	snd_hdmi_print_eld_info(&per_pin->sink_eld, buffer);
+	snd_hdmi_print_eld_info(&per_pin->sink_eld, buffer, per_pin->pin_nid,
+				per_pin->dev_id, per_pin->cvt_nid);
 	mutex_unlock(&per_pin->lock);
 }
 
@@ -988,7 +988,8 @@ static int hdmi_setup_stream(struct hda_codec *codec, hda_nid_t cvt_nid,
  * of the pin.
  */
 static int hdmi_choose_cvt(struct hda_codec *codec,
-			   int pin_idx, int *cvt_id)
+			   int pin_idx, int *cvt_id,
+			   bool silent)
 {
 	struct hdmi_spec *spec = codec->spec;
 	struct hdmi_spec_per_pin *per_pin;
@@ -1003,6 +1004,9 @@ static int hdmi_choose_cvt(struct hda_codec *codec,
 
 	if (per_pin && per_pin->silent_stream) {
 		cvt_idx = cvt_nid_to_cvt_index(codec, per_pin->cvt_nid);
+		per_cvt = get_cvt(spec, cvt_idx);
+		if (per_cvt->assigned && !silent)
+			return -EBUSY;
 		if (cvt_id)
 			*cvt_id = cvt_idx;
 		return 0;
@@ -1013,7 +1017,7 @@ static int hdmi_choose_cvt(struct hda_codec *codec,
 		per_cvt = get_cvt(spec, cvt_idx);
 
 		/* Must not already be assigned */
-		if (per_cvt->assigned)
+		if (per_cvt->assigned || per_cvt->silent_stream)
 			continue;
 		if (per_pin == NULL)
 			break;
@@ -1182,9 +1186,7 @@ static void pin_cvt_fixup(struct hda_codec *codec,
 		spec->ops.pin_cvt_fixup(codec, per_pin, cvt_nid);
 }
 
-/* called in hdmi_pcm_open when no pin is assigned to the PCM
- * in dyn_pcm_assign mode.
- */
+/* called in hdmi_pcm_open when no pin is assigned to the PCM */
 static int hdmi_pcm_open_no_pin(struct hda_pcm_stream *hinfo,
 			 struct hda_codec *codec,
 			 struct snd_pcm_substream *substream)
@@ -1199,12 +1201,12 @@ static int hdmi_pcm_open_no_pin(struct hda_pcm_stream *hinfo,
 	if (pcm_idx < 0)
 		return -EINVAL;
 
-	err = hdmi_choose_cvt(codec, -1, &cvt_idx);
+	err = hdmi_choose_cvt(codec, -1, &cvt_idx, false);
 	if (err)
 		return err;
 
 	per_cvt = get_cvt(spec, cvt_idx);
-	per_cvt->assigned = 1;
+	per_cvt->assigned = true;
 	hinfo->nid = per_cvt->cvt_nid;
 
 	pin_cvt_fixup(codec, NULL, per_cvt->cvt_nid);
@@ -1252,33 +1254,25 @@ static int hdmi_pcm_open(struct hda_pcm_stream *hinfo,
 
 	mutex_lock(&spec->pcm_lock);
 	pin_idx = hinfo_to_pin_index(codec, hinfo);
-	if (!spec->dyn_pcm_assign) {
-		if (snd_BUG_ON(pin_idx < 0)) {
-			err = -EINVAL;
-			goto unlock;
-		}
-	} else {
-		/* no pin is assigned to the PCM
-		 * PA need pcm open successfully when probe
-		 */
-		if (pin_idx < 0) {
-			err = hdmi_pcm_open_no_pin(hinfo, codec, substream);
-			goto unlock;
-		}
+	/* no pin is assigned to the PCM
+	 * PA need pcm open successfully when probe
+	 */
+	if (pin_idx < 0) {
+		err = hdmi_pcm_open_no_pin(hinfo, codec, substream);
+		goto unlock;
 	}
 
-	err = hdmi_choose_cvt(codec, pin_idx, &cvt_idx);
+	err = hdmi_choose_cvt(codec, pin_idx, &cvt_idx, false);
 	if (err < 0)
 		goto unlock;
 
 	per_cvt = get_cvt(spec, cvt_idx);
 	/* Claim converter */
-	per_cvt->assigned = 1;
+	per_cvt->assigned = true;
 
 	set_bit(pcm_idx, &spec->pcm_in_use);
 	per_pin = get_pin(spec, pin_idx);
 	per_pin->cvt_nid = per_cvt->cvt_nid;
-	per_pin->silent_stream = false;
 	hinfo->nid = per_cvt->cvt_nid;
 
 	/* flip stripe flag for the assigned stream if supported */
@@ -1308,7 +1302,7 @@ static int hdmi_pcm_open(struct hda_pcm_stream *hinfo,
 		snd_hdmi_eld_update_pcm_info(&eld->info, hinfo);
 		if (hinfo->channels_min > hinfo->channels_max ||
 		    !hinfo->rates || !hinfo->formats) {
-			per_cvt->assigned = 0;
+			per_cvt->assigned = false;
 			hinfo->nid = 0;
 			snd_hda_spdif_ctls_unassign(codec, pcm_idx);
 			err = -ENODEV;
@@ -1370,43 +1364,6 @@ static int hdmi_find_pcm_slot(struct hdmi_spec *spec,
 {
 	int i;
 
-	/* on the new machines, try to assign the pcm slot dynamically,
-	 * not use the preferred fixed map (legacy way) anymore.
-	 */
-	if (spec->dyn_pcm_no_legacy)
-		goto last_try;
-
-	/*
-	 * generic_hdmi_build_pcms() may allocate extra PCMs on some
-	 * platforms (with maximum of 'num_nids + dev_num - 1')
-	 *
-	 * The per_pin of pin_nid_idx=n and dev_id=m prefers to get pcm-n
-	 * if m==0. This guarantees that dynamic pcm assignments are compatible
-	 * with the legacy static per_pin-pcm assignment that existed in the
-	 * days before DP-MST.
-	 *
-	 * Intel DP-MST prefers this legacy behavior for compatibility, too.
-	 *
-	 * per_pin of m!=0 prefers to get pcm=(num_nids + (m - 1)).
-	 */
-
-	if (per_pin->dev_id == 0 || spec->intel_hsw_fixup) {
-		if (!test_bit(per_pin->pin_nid_idx, &spec->pcm_bitmap))
-			return per_pin->pin_nid_idx;
-	} else {
-		i = spec->num_nids + (per_pin->dev_id - 1);
-		if (i < spec->pcm_used && !(test_bit(i, &spec->pcm_bitmap)))
-			return i;
-	}
-
-	/* have a second try; check the area over num_nids */
-	for (i = spec->num_nids; i < spec->pcm_used; i++) {
-		if (!test_bit(i, &spec->pcm_bitmap))
-			return i;
-	}
-
- last_try:
-	/* the last try; check the empty slots in pins */
 	for (i = 0; i < spec->pcm_used; i++) {
 		if (!test_bit(i, &spec->pcm_bitmap))
 			return i;
@@ -1468,10 +1425,9 @@ static void hdmi_pcm_setup_pin(struct hdmi_spec *spec,
 	int mux_idx;
 	bool non_pcm;
 
-	if (per_pin->pcm_idx >= 0 && per_pin->pcm_idx < spec->pcm_used)
-		pcm = get_pcm_rec(spec, per_pin->pcm_idx);
-	else
+	if (per_pin->pcm_idx < 0 || per_pin->pcm_idx >= spec->pcm_used)
 		return;
+	pcm = get_pcm_rec(spec, per_pin->pcm_idx);
 	if (!pcm->pcm)
 		return;
 	if (!test_bit(per_pin->pcm_idx, &spec->pcm_in_use))
@@ -1569,14 +1525,12 @@ static void update_eld(struct hda_codec *codec,
 	 */
 	pcm_jack = pin_idx_to_pcm_jack(codec, per_pin);
 
-	if (spec->dyn_pcm_assign) {
-		if (eld->eld_valid) {
-			hdmi_attach_hda_pcm(spec, per_pin);
-			hdmi_pcm_setup_pin(spec, per_pin);
-		} else {
-			hdmi_pcm_reset_pin(spec, per_pin);
-			hdmi_detach_hda_pcm(spec, per_pin);
-		}
+	if (eld->eld_valid) {
+		hdmi_attach_hda_pcm(spec, per_pin);
+		hdmi_pcm_setup_pin(spec, per_pin);
+	} else {
+		hdmi_pcm_reset_pin(spec, per_pin);
+		hdmi_detach_hda_pcm(spec, per_pin);
 	}
 	/* if pcm_idx == -1, it means this is in monitor connection event
 	 * we can get the correct pcm_idx now.
@@ -1760,14 +1714,14 @@ static void silent_stream_enable(struct hda_codec *codec,
 	}
 
 	pin_idx = pin_id_to_pin_index(codec, per_pin->pin_nid, per_pin->dev_id);
-	err = hdmi_choose_cvt(codec, pin_idx, &cvt_idx);
+	err = hdmi_choose_cvt(codec, pin_idx, &cvt_idx, true);
 	if (err) {
 		codec_err(codec, "hdmi: no free converter to enable silent mode\n");
 		goto unlock_out;
 	}
 
 	per_cvt = get_cvt(spec, cvt_idx);
-	per_cvt->assigned = 1;
+	per_cvt->silent_stream = true;
 	per_pin->cvt_nid = per_cvt->cvt_nid;
 	per_pin->silent_stream = true;
 
@@ -1827,7 +1781,7 @@ static void silent_stream_disable(struct hda_codec *codec,
 	cvt_idx = cvt_nid_to_cvt_index(codec, per_pin->cvt_nid);
 	if (cvt_idx >= 0 && cvt_idx < spec->num_cvts) {
 		per_cvt = get_cvt(spec, cvt_idx);
-		per_cvt->assigned = 0;
+		per_cvt->silent_stream = false;
 	}
 
 	if (spec->silent_stream_type == SILENT_STREAM_I915) {
@@ -1938,7 +1892,7 @@ static int hdmi_add_pin(struct hda_codec *codec, hda_nid_t pin_nid)
 		 * structures based on worst case.
 		 */
 		dev_num = spec->dev_num;
-	} else if (spec->dyn_pcm_assign && codec->dp_mst) {
+	} else if (codec->dp_mst) {
 		dev_num = snd_hda_get_num_devices(codec, pin_nid) + 1;
 		/*
 		 * spec->dev_num is the maxinum number of device entries
@@ -1963,13 +1917,8 @@ static int hdmi_add_pin(struct hda_codec *codec, hda_nid_t pin_nid)
 		if (!per_pin)
 			return -ENOMEM;
 
-		if (spec->dyn_pcm_assign) {
-			per_pin->pcm = NULL;
-			per_pin->pcm_idx = -1;
-		} else {
-			per_pin->pcm = get_hdmi_pcm(spec, pin_idx);
-			per_pin->pcm_idx = pin_idx;
-		}
+		per_pin->pcm = NULL;
+		per_pin->pcm_idx = -1;
 		per_pin->pin_nid = pin_nid;
 		per_pin->pin_nid_idx = spec->num_nids;
 		per_pin->dev_id = i;
@@ -1978,6 +1927,8 @@ static int hdmi_add_pin(struct hda_codec *codec, hda_nid_t pin_nid)
 		err = hdmi_read_pin_conn(codec, pin_idx);
 		if (err < 0)
 			return err;
+		if (!is_jack_detectable(codec, pin_nid))
+			codec_warn(codec, "HDMI: pin NID 0x%x - jack not detectable\n", pin_nid);
 		spec->num_pins++;
 	}
 	spec->num_nids++;
@@ -2125,10 +2076,9 @@ static int generic_hdmi_playback_pcm_prepare(struct hda_pcm_stream *hinfo,
 
 	mutex_lock(&spec->pcm_lock);
 	pin_idx = hinfo_to_pin_index(codec, hinfo);
-	if (spec->dyn_pcm_assign && pin_idx < 0) {
-		/* when dyn_pcm_assign and pcm is not bound to a pin
-		 * skip pin setup and return 0 to make audio playback
-		 * be ongoing
+	if (pin_idx < 0) {
+		/* when pcm is not bound to a pin skip pin setup and return 0
+		 * to make audio playback be ongoing
 		 */
 		pin_cvt_fixup(codec, NULL, cvt_nid);
 		snd_hda_codec_setup_stream(codec, cvt_nid,
@@ -2223,7 +2173,7 @@ static int hdmi_pcm_close(struct hda_pcm_stream *hinfo,
 			goto unlock;
 		}
 		per_cvt = get_cvt(spec, cvt_idx);
-		per_cvt->assigned = 0;
+		per_cvt->assigned = false;
 		hinfo->nid = 0;
 
 		azx_stream(get_azx_dev(substream))->stripe = 0;
@@ -2231,7 +2181,7 @@ static int hdmi_pcm_close(struct hda_pcm_stream *hinfo,
 		snd_hda_spdif_ctls_unassign(codec, pcm_idx);
 		clear_bit(pcm_idx, &spec->pcm_in_use);
 		pin_idx = hinfo_to_pin_index(codec, hinfo);
-		if (spec->dyn_pcm_assign && pin_idx < 0)
+		if (pin_idx < 0)
 			goto unlock;
 
 		if (snd_BUG_ON(pin_idx < 0)) {
@@ -2329,21 +2279,8 @@ static int generic_hdmi_build_pcms(struct hda_codec *codec)
 	struct hdmi_spec *spec = codec->spec;
 	int idx, pcm_num;
 
-	/*
-	 * for non-mst mode, pcm number is the same as before
-	 * for DP MST mode without extra PCM, pcm number is same
-	 * for DP MST mode with extra PCMs, pcm number is
-	 *  (nid number + dev_num - 1)
-	 * dev_num is the device entry number in a pin
-	 */
-
-	if (spec->dyn_pcm_no_legacy && codec->mst_no_extra_pcms)
-		pcm_num = spec->num_cvts;
-	else if (codec->mst_no_extra_pcms)
-		pcm_num = spec->num_nids;
-	else
-		pcm_num = spec->num_nids + spec->dev_num - 1;
-
+	/* limit the PCM devices to the codec converters */
+	pcm_num = spec->num_cvts;
 	codec_dbg(codec, "hdmi: pcm_num set to %d\n", pcm_num);
 
 	for (idx = 0; idx < pcm_num; idx++) {
@@ -2362,8 +2299,8 @@ static int generic_hdmi_build_pcms(struct hda_codec *codec)
 		pstr = &info->stream[SNDRV_PCM_STREAM_PLAYBACK];
 		pstr->substreams = 1;
 		pstr->ops = generic_ops;
-		/* pcm number is less than 16 */
-		if (spec->pcm_used >= 16)
+		/* pcm number is less than pcm_rec array size */
+		if (spec->pcm_used >= ARRAY_SIZE(spec->pcm_rec))
 			break;
 		/* other pstr fields are set in open */
 	}
@@ -2382,17 +2319,12 @@ static int generic_hdmi_build_jack(struct hda_codec *codec, int pcm_idx)
 {
 	char hdmi_str[32] = "HDMI/DP";
 	struct hdmi_spec *spec = codec->spec;
-	struct hdmi_spec_per_pin *per_pin = get_pin(spec, pcm_idx);
 	struct snd_jack *jack;
 	int pcmdev = get_pcm_rec(spec, pcm_idx)->device;
 	int err;
 
 	if (pcmdev > 0)
 		sprintf(hdmi_str + strlen(hdmi_str), ",pcm=%d", pcmdev);
-	if (!spec->dyn_pcm_assign &&
-	    !is_jack_detectable(codec, per_pin->pin_nid))
-		strncat(hdmi_str, " Phantom",
-			sizeof(hdmi_str) - strlen(hdmi_str) - 1);
 
 	err = snd_jack_new(codec->card, hdmi_str, SND_JACK_AVOUT, &jack,
 			   true, false);
@@ -2425,18 +2357,9 @@ static int generic_hdmi_build_controls(struct hda_codec *codec)
 		/* create the spdif for each pcm
 		 * pin will be bound when monitor is connected
 		 */
-		if (spec->dyn_pcm_assign)
-			err = snd_hda_create_dig_out_ctls(codec,
+		err = snd_hda_create_dig_out_ctls(codec,
 					  0, spec->cvt_nids[0],
 					  HDA_PCM_TYPE_HDMI);
-		else {
-			struct hdmi_spec_per_pin *per_pin =
-				get_pin(spec, pcm_idx);
-			err = snd_hda_create_dig_out_ctls(codec,
-						  per_pin->pin_nid,
-						  per_pin->mux_nids[0],
-						  HDA_PCM_TYPE_HDMI);
-		}
 		if (err < 0)
 			return err;
 		snd_hda_spdif_ctls_unassign(codec, pcm_idx);
@@ -2556,11 +2479,7 @@ static void generic_hdmi_free(struct hda_codec *codec)
 	for (pcm_idx = 0; pcm_idx < spec->pcm_used; pcm_idx++) {
 		if (spec->pcm_rec[pcm_idx].jack == NULL)
 			continue;
-		if (spec->dyn_pcm_assign)
-			snd_device_free(codec->card,
-					spec->pcm_rec[pcm_idx].jack);
-		else
-			spec->pcm_rec[pcm_idx].jack = NULL;
+		snd_device_free(codec->card, spec->pcm_rec[pcm_idx].jack);
 	}
 
 	generic_spec_free(codec);
@@ -2747,9 +2666,6 @@ static void generic_acomp_pin_eld_notify(void *audio_ptr, int port, int dev_id)
 	 */
 	if (codec->core.dev.power.power_state.event == PM_EVENT_SUSPEND)
 		return;
-	/* ditto during suspend/resume process itself */
-	if (snd_hdac_is_in_pm(&codec->core))
-		return;
 
 	check_presence_and_report(codec, pin_nid, dev_id);
 }
@@ -2933,9 +2849,6 @@ static void intel_pin_eld_notify(void *audio_ptr, int port, int pipe)
 	 */
 	if (codec->core.dev.power.power_state.event == PM_EVENT_SUSPEND)
 		return;
-	/* ditto during suspend/resume process itself */
-	if (snd_hdac_is_in_pm(&codec->core))
-		return;
 
 	snd_hdac_i915_set_bclk(&codec->bus->core);
 	check_presence_and_report(codec, pin_nid, dev_id);
@@ -3040,7 +2953,6 @@ static int intel_hsw_common_init(struct hda_codec *codec, hda_nid_t vendor_nid,
 		return err;
 	spec = codec->spec;
 	codec->dp_mst = true;
-	spec->dyn_pcm_assign = true;
 	spec->vendor_nid = vendor_nid;
 	spec->port_map = port_map;
 	spec->port_num = port_num;
@@ -3104,17 +3016,9 @@ static int patch_i915_tgl_hdmi(struct hda_codec *codec)
 	 * the index indicate the port number.
 	 */
 	static const int map[] = {0x4, 0x6, 0x8, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf};
-	int ret;
-
-	ret = intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map), 4,
-				    enable_silent_stream);
-	if (!ret) {
-		struct hdmi_spec *spec = codec->spec;
 
-		spec->dyn_pcm_no_legacy = true;
-	}
-
-	return ret;
+	return intel_hsw_common_init(codec, 0x02, map, ARRAY_SIZE(map), 4,
+				     enable_silent_stream);
 }
 
 static int patch_i915_adlp_hdmi(struct hda_codec *codec)
@@ -3754,7 +3658,6 @@ static int patch_nvhdmi(struct hda_codec *codec)
 	codec->dp_mst = true;
 
 	spec = codec->spec;
-	spec->dyn_pcm_assign = true;
 
 	err = hdmi_parse_codec(codec);
 	if (err < 0) {
@@ -4034,10 +3937,8 @@ static int patch_tegra234_hdmi(struct hda_codec *codec)
 		return err;
 
 	codec->dp_mst = true;
-	codec->mst_no_extra_pcms = true;
 	spec = codec->spec;
 	spec->dyn_pin_out = true;
-	spec->dyn_pcm_assign = true;
 	spec->hdmi_intr_trig_ctrl = true;
 
 	return tegra_hdmi_init(codec);
diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c
index f9d46ae4c7b7..bce82b834cec 100644
--- a/sound/pci/hda/patch_realtek.c
+++ b/sound/pci/hda/patch_realtek.c
@@ -18,6 +18,7 @@
 #include <linux/module.h>
 #include <linux/input.h>
 #include <linux/leds.h>
+#include <linux/ctype.h>
 #include <sound/core.h>
 #include <sound/jack.h>
 #include <sound/hda_codec.h>
@@ -6704,23 +6705,51 @@ static void comp_generic_playback_hook(struct hda_pcm_stream *hinfo, struct hda_
 	}
 }
 
+struct cs35l41_dev_name {
+	const char *bus;
+	const char *hid;
+	int index;
+};
+
+/* match the device name in a slightly relaxed manner */
+static int comp_match_cs35l41_dev_name(struct device *dev, void *data)
+{
+	struct cs35l41_dev_name *p = data;
+	const char *d = dev_name(dev);
+	int n = strlen(p->bus);
+	char tmp[32];
+
+	/* check the bus name */
+	if (strncmp(d, p->bus, n))
+		return 0;
+	/* skip the bus number */
+	if (isdigit(d[n]))
+		n++;
+	/* the rest must be exact matching */
+	snprintf(tmp, sizeof(tmp), "-%s:00-cs35l41-hda.%d", p->hid, p->index);
+	return !strcmp(d + n, tmp);
+}
+
 static void cs35l41_generic_fixup(struct hda_codec *cdc, int action, const char *bus,
 				  const char *hid, int count)
 {
 	struct device *dev = hda_codec_dev(cdc);
 	struct alc_spec *spec = cdc->spec;
-	char *name;
+	struct cs35l41_dev_name *rec;
 	int ret, i;
 
 	switch (action) {
 	case HDA_FIXUP_ACT_PRE_PROBE:
 		for (i = 0; i < count; i++) {
-			name = devm_kasprintf(dev, GFP_KERNEL,
-					      "%s-%s:00-cs35l41-hda.%d", bus, hid, i);
-			if (!name)
+			rec = devm_kmalloc(dev, sizeof(*rec), GFP_KERNEL);
+			if (!rec)
 				return;
+			rec->bus = bus;
+			rec->hid = hid;
+			rec->index = i;
 			spec->comps[i].codec = cdc;
-			component_match_add(dev, &spec->match, component_compare_dev_name, name);
+			component_match_add(dev, &spec->match,
+					    comp_match_cs35l41_dev_name, rec);
 		}
 		ret = component_master_add_with_match(dev, &comp_master_ops, spec->match);
 		if (ret)
@@ -6738,12 +6767,12 @@ static void cs35l41_fixup_i2c_two(struct hda_codec *cdc, const struct hda_fixup
 
 static void cs35l41_fixup_spi_two(struct hda_codec *codec, const struct hda_fixup *fix, int action)
 {
-	cs35l41_generic_fixup(codec, action, "spi0", "CSC3551", 2);
+	cs35l41_generic_fixup(codec, action, "spi", "CSC3551", 2);
 }
 
 static void cs35l41_fixup_spi_four(struct hda_codec *codec, const struct hda_fixup *fix, int action)
 {
-	cs35l41_generic_fixup(codec, action, "spi0", "CSC3551", 4);
+	cs35l41_generic_fixup(codec, action, "spi", "CSC3551", 4);
 }
 
 static void alc287_fixup_legion_16achg6_speakers(struct hda_codec *cdc, const struct hda_fixup *fix,
@@ -9341,6 +9370,7 @@ static const struct snd_pci_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8aa3, "HP ProBook 450 G9 (MB 8AA1)", ALC236_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8aa8, "HP EliteBook 640 G9 (MB 8AA6)", ALC236_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8aab, "HP EliteBook 650 G9 (MB 8AA9)", ALC236_FIXUP_HP_GPIO_LED),
+	 SND_PCI_QUIRK(0x103c, 0x8abb, "HP ZBook Firefly 14 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8ad1, "HP EliteBook 840 14 inch G9 Notebook PC", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8ad2, "HP EliteBook 860 16 inch G9 Notebook PC", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x1043, 0x103e, "ASUS X540SA", ALC256_FIXUP_ASUS_MIC),
diff --git a/sound/pci/hda/patch_sigmatel.c b/sound/pci/hda/patch_sigmatel.c
index 7f340f18599c..a794a01a68ca 100644
--- a/sound/pci/hda/patch_sigmatel.c
+++ b/sound/pci/hda/patch_sigmatel.c
@@ -4311,6 +4311,8 @@ static int stac_parse_auto_config(struct hda_codec *codec)
 		if (codec->beep) {
 			/* IDT/STAC codecs have linear beep tone parameter */
 			codec->beep->linear_tone = spec->linear_tone_beep;
+			/* keep power up while beep is enabled */
+			codec->beep->keep_power_at_enable = 1;
 			/* if no beep switch is available, make its own one */
 			caps = query_amp_caps(codec, nid, HDA_OUTPUT);
 			if (!(caps & AC_AMPCAP_MUTE)) {
@@ -4444,28 +4446,6 @@ static int stac_suspend(struct hda_codec *codec)
 
 	return 0;
 }
-
-static int stac_check_power_status(struct hda_codec *codec, hda_nid_t nid)
-{
-#ifdef CONFIG_SND_HDA_INPUT_BEEP
-	struct sigmatel_spec *spec = codec->spec;
-#endif
-	int ret = snd_hda_gen_check_power_status(codec, nid);
-
-#ifdef CONFIG_SND_HDA_INPUT_BEEP
-	if (nid == spec->gen.beep_nid && codec->beep) {
-		if (codec->beep->enabled != spec->beep_power_on) {
-			spec->beep_power_on = codec->beep->enabled;
-			if (spec->beep_power_on)
-				snd_hda_power_up_pm(codec);
-			else
-				snd_hda_power_down_pm(codec);
-		}
-		ret |= spec->beep_power_on;
-	}
-#endif
-	return ret;
-}
 #else
 #define stac_suspend		NULL
 #endif /* CONFIG_PM */
@@ -4478,7 +4458,6 @@ static const struct hda_codec_ops stac_patch_ops = {
 	.unsol_event = snd_hda_jack_unsol_event,
 #ifdef CONFIG_PM
 	.suspend = stac_suspend,
-	.check_power_status = stac_check_power_status,
 #endif
 };
 
diff --git a/sound/pcmcia/pdaudiocf/pdaudiocf_pcm.c b/sound/pcmcia/pdaudiocf/pdaudiocf_pcm.c
index dfc4295b69c4..aaa82ec36540 100644
--- a/sound/pcmcia/pdaudiocf/pdaudiocf_pcm.c
+++ b/sound/pcmcia/pdaudiocf/pdaudiocf_pcm.c
@@ -257,8 +257,7 @@ int snd_pdacf_pcm_new(struct snd_pdacf *chip)
 		return err;
 		
 	snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_CAPTURE, &pdacf_pcm_capture_ops);
-	snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC,
-				       snd_dma_continuous_data(GFP_KERNEL | GFP_DMA32),
+	snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_VMALLOC, NULL,
 				       0, 0);
 
 	pcm->private_data = chip;
diff --git a/sound/ppc/keywest.c b/sound/ppc/keywest.c
index 6e5daae18f9d..80e5108157ef 100644
--- a/sound/ppc/keywest.c
+++ b/sound/ppc/keywest.c
@@ -71,14 +71,12 @@ static int keywest_attach_adapter(struct i2c_adapter *adapter)
 	return 0;
 }
 
-static int keywest_remove(struct i2c_client *client)
+static void keywest_remove(struct i2c_client *client)
 {
 	if (! keywest_ctx)
-		return 0;
+		return;
 	if (client == keywest_ctx->client)
 		keywest_ctx->client = NULL;
-
-	return 0;
 }
 
 
diff --git a/sound/ppc/tumbler.c b/sound/ppc/tumbler.c
index c65e74d7cd0a..f3f8ad7c3df8 100644
--- a/sound/ppc/tumbler.c
+++ b/sound/ppc/tumbler.c
@@ -1060,8 +1060,7 @@ static struct device_node *find_audio_device(const char *name)
 	if (! gpiop)
 		return NULL;
   
-	for (np = of_get_next_child(gpiop, NULL); np;
-			np = of_get_next_child(gpiop, np)) {
+	for_each_child_of_node(gpiop, np) {
 		const char *property = of_get_property(np, "audio-gpio", NULL);
 		if (property && strcmp(property, name) == 0)
 			break;
@@ -1080,8 +1079,7 @@ static struct device_node *find_compatible_audio_device(const char *name)
 	if (!gpiop)
 		return NULL;
   
-	for (np = of_get_next_child(gpiop, NULL); np;
-			np = of_get_next_child(gpiop, np)) {
+	for_each_child_of_node(gpiop, np) {
 		if (of_device_is_compatible(np, name))
 			break;
 	}  
diff --git a/sound/soc/Kconfig b/sound/soc/Kconfig
index 7d4747b6bab2..848fbae26c3b 100644
--- a/sound/soc/Kconfig
+++ b/sound/soc/Kconfig
@@ -68,6 +68,7 @@ config SND_SOC_ACPI
 # All the supported SoCs
 source "sound/soc/adi/Kconfig"
 source "sound/soc/amd/Kconfig"
+source "sound/soc/apple/Kconfig"
 source "sound/soc/atmel/Kconfig"
 source "sound/soc/au1x/Kconfig"
 source "sound/soc/bcm/Kconfig"
diff --git a/sound/soc/Makefile b/sound/soc/Makefile
index 453181ef6c94..507eaed1d6a1 100644
--- a/sound/soc/Makefile
+++ b/sound/soc/Makefile
@@ -34,6 +34,7 @@ obj-$(CONFIG_SND_SOC_ACPI) += snd-soc-acpi.o
 obj-$(CONFIG_SND_SOC)	+= snd-soc-core.o
 obj-$(CONFIG_SND_SOC)	+= codecs/
 obj-$(CONFIG_SND_SOC)	+= generic/
+obj-$(CONFIG_SND_SOC)	+= apple/
 obj-$(CONFIG_SND_SOC)	+= adi/
 obj-$(CONFIG_SND_SOC)	+= amd/
 obj-$(CONFIG_SND_SOC)	+= atmel/
diff --git a/sound/soc/amd/Kconfig b/sound/soc/amd/Kconfig
index 08f5289dac54..150786279257 100644
--- a/sound/soc/amd/Kconfig
+++ b/sound/soc/amd/Kconfig
@@ -122,8 +122,29 @@ config SND_SOC_AMD_RPL_ACP6x
         tristate "AMD Audio Coprocessor-v6.2 RPL support"
         depends on X86 && PCI
         help
-          This option enables Audio Coprocessor i.e ACP v6.2 support on
+          This option enables Audio Coprocessor i.e. ACP v6.2 support on
           AMD RPL platform. By enabling this flag build will be
           triggered for ACP PCI driver.
           Say m if you have such a device.
           If unsure select "N".
+
+config SND_SOC_AMD_PS
+        tristate "AMD Audio Coprocessor-v6.2 Pink Sardine support"
+        depends on X86 && PCI && ACPI
+        help
+          This option enables Audio Coprocessor i.e ACP v6.2 support on
+          AMD Pink sardine platform. By enabling this flag build will be
+          triggered for ACP PCI driver, ACP PDM DMA driver.
+          Say m if you have such a device.
+          If unsure select "N".
+
+config SND_SOC_AMD_PS_MACH
+        tristate "AMD PINK SARDINE support for DMIC"
+        select SND_SOC_DMIC
+        depends on SND_SOC_AMD_PS
+        help
+          This option enables machine driver for Pink Sardine platform
+          using dmic. ACP IP has PDM Decoder block with DMA controller.
+          DMIC can be connected directly to ACP IP.
+          Say m if you have such a device.
+          If unsure select "N".
diff --git a/sound/soc/amd/Makefile b/sound/soc/amd/Makefile
index 0592e7c5c407..82e1cf864a40 100644
--- a/sound/soc/amd/Makefile
+++ b/sound/soc/amd/Makefile
@@ -18,3 +18,4 @@ obj-$(CONFIG_SND_SOC_AMD_ACP6x) += yc/
 obj-$(CONFIG_SND_SOC_AMD_ACP_COMMON) += acp/
 obj-$(CONFIG_SND_AMD_ACP_CONFIG) += snd-acp-config.o
 obj-$(CONFIG_SND_SOC_AMD_RPL_ACP6x) += rpl/
+obj-$(CONFIG_SND_SOC_AMD_PS) += ps/
diff --git a/sound/soc/amd/acp/acp-i2s.c b/sound/soc/amd/acp/acp-i2s.c
index 393f729ef561..ac416572db0d 100644
--- a/sound/soc/amd/acp/acp-i2s.c
+++ b/sound/soc/amd/acp/acp-i2s.c
@@ -25,6 +25,65 @@
 
 #define DRV_NAME "acp_i2s_playcap"
 
+static int acp_i2s_set_fmt(struct snd_soc_dai *cpu_dai,
+			   unsigned int fmt)
+{
+	struct acp_dev_data *adata = snd_soc_dai_get_drvdata(cpu_dai);
+	int mode;
+
+	mode = fmt & SND_SOC_DAIFMT_FORMAT_MASK;
+	switch (mode) {
+	case SND_SOC_DAIFMT_I2S:
+		adata->tdm_mode = TDM_DISABLE;
+		break;
+	case SND_SOC_DAIFMT_DSP_A:
+		adata->tdm_mode = TDM_ENABLE;
+		break;
+	default:
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int acp_i2s_set_tdm_slot(struct snd_soc_dai *dai, u32 tx_mask, u32 rx_mask,
+				int slots, int slot_width)
+{
+	struct device *dev = dai->component->dev;
+	struct acp_dev_data *adata = snd_soc_dai_get_drvdata(dai);
+	struct acp_stream *stream;
+	int slot_len;
+
+	switch (slot_width) {
+	case SLOT_WIDTH_8:
+		slot_len = 8;
+		break;
+	case SLOT_WIDTH_16:
+		slot_len = 16;
+		break;
+	case SLOT_WIDTH_24:
+		slot_len = 24;
+		break;
+	case SLOT_WIDTH_32:
+		slot_len = 0;
+		break;
+	default:
+		dev_err(dev, "Unsupported bitdepth %d\n", slot_width);
+		return -EINVAL;
+	}
+
+	spin_lock_irq(&adata->acp_lock);
+	list_for_each_entry(stream, &adata->stream_list, list) {
+		if (tx_mask && stream->dir == SNDRV_PCM_STREAM_PLAYBACK)
+			adata->tdm_tx_fmt[stream->dai_id - 1] =
+					FRM_LEN | (slots << 15) | (slot_len << 18);
+		else if (rx_mask && stream->dir == SNDRV_PCM_STREAM_CAPTURE)
+			adata->tdm_rx_fmt[stream->dai_id - 1] =
+					FRM_LEN | (slots << 15) | (slot_len << 18);
+	}
+	spin_unlock_irq(&adata->acp_lock);
+	return 0;
+}
+
 static int acp_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *params,
 			    struct snd_soc_dai *dai)
 {
@@ -33,7 +92,7 @@ static int acp_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_pcm_
 	struct acp_resource *rsrc;
 	u32 val;
 	u32 xfer_resolution;
-	u32 reg_val;
+	u32 reg_val, fmt_reg, tdm_fmt;
 	u32 lrclk_div_val, bclk_div_val;
 
 	adata = snd_soc_dai_get_drvdata(dai);
@@ -62,12 +121,15 @@ static int acp_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_pcm_
 		switch (dai->driver->id) {
 		case I2S_BT_INSTANCE:
 			reg_val = ACP_BTTDM_ITER;
+			fmt_reg = ACP_BTTDM_TXFRMT;
 			break;
 		case I2S_SP_INSTANCE:
 			reg_val = ACP_I2STDM_ITER;
+			fmt_reg = ACP_I2STDM_TXFRMT;
 			break;
 		case I2S_HS_INSTANCE:
 			reg_val = ACP_HSTDM_ITER;
+			fmt_reg = ACP_HSTDM_TXFRMT;
 			break;
 		default:
 			dev_err(dev, "Invalid dai id %x\n", dai->driver->id);
@@ -77,12 +139,15 @@ static int acp_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_pcm_
 		switch (dai->driver->id) {
 		case I2S_BT_INSTANCE:
 			reg_val = ACP_BTTDM_IRER;
+			fmt_reg = ACP_BTTDM_RXFRMT;
 			break;
 		case I2S_SP_INSTANCE:
 			reg_val = ACP_I2STDM_IRER;
+			fmt_reg = ACP_I2STDM_RXFRMT;
 			break;
 		case I2S_HS_INSTANCE:
 			reg_val = ACP_HSTDM_IRER;
+			fmt_reg = ACP_HSTDM_RXFRMT;
 			break;
 		default:
 			dev_err(dev, "Invalid dai id %x\n", dai->driver->id);
@@ -95,6 +160,16 @@ static int acp_i2s_hwparams(struct snd_pcm_substream *substream, struct snd_pcm_
 	val = val | (xfer_resolution  << 3);
 	writel(val, adata->acp_base + reg_val);
 
+	if (adata->tdm_mode) {
+		val = readl(adata->acp_base + reg_val);
+		writel(val | BIT(1), adata->acp_base + reg_val);
+		if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+			tdm_fmt = adata->tdm_tx_fmt[dai->driver->id - 1];
+		else
+			tdm_fmt = adata->tdm_rx_fmt[dai->driver->id - 1];
+		writel(tdm_fmt, adata->acp_base + fmt_reg);
+	}
+
 	if (rsrc->soc_mclk) {
 		switch (params_format(params)) {
 		case SNDRV_PCM_FORMAT_S16_LE:
@@ -443,6 +518,7 @@ static int acp_i2s_startup(struct snd_pcm_substream *substream, struct snd_soc_d
 	stream->id = dai->driver->id + dir;
 	stream->dai_id = dai->driver->id;
 	stream->irq_bit = irq_bit;
+	stream->dir = substream->stream;
 
 	return 0;
 }
@@ -452,6 +528,8 @@ const struct snd_soc_dai_ops asoc_acp_cpu_dai_ops = {
 	.hw_params = acp_i2s_hwparams,
 	.prepare = acp_i2s_prepare,
 	.trigger = acp_i2s_trigger,
+	.set_fmt = acp_i2s_set_fmt,
+	.set_tdm_slot = acp_i2s_set_tdm_slot,
 };
 EXPORT_SYMBOL_NS_GPL(asoc_acp_cpu_dai_ops, SND_SOC_ACP_COMMON);
 
diff --git a/sound/soc/amd/acp/acp-mach-common.c b/sound/soc/amd/acp/acp-mach-common.c
index f0c49127aad1..4c69cb6e3400 100644
--- a/sound/soc/amd/acp/acp-mach-common.c
+++ b/sound/soc/amd/acp/acp-mach-common.c
@@ -584,7 +584,7 @@ int acp_sofdsp_dai_links_create(struct snd_soc_card *card)
 	if (drv_data->dmic_cpu_id)
 		num_links++;
 
-	links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) * num_links, GFP_KERNEL);
+	links = devm_kcalloc(dev, num_links, sizeof(struct snd_soc_dai_link), GFP_KERNEL);
 	if (!links)
 		return -ENOMEM;
 
@@ -749,7 +749,7 @@ int acp_legacy_dai_links_create(struct snd_soc_card *card)
 	if (drv_data->dmic_cpu_id)
 		num_links++;
 
-	links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) * num_links, GFP_KERNEL);
+	links = devm_kcalloc(dev, num_links, sizeof(struct snd_soc_dai_link), GFP_KERNEL);
 	if (!links)
 		return -ENOMEM;
 
diff --git a/sound/soc/amd/acp/acp-pci.c b/sound/soc/amd/acp/acp-pci.c
index 2c8e960cc9a6..a0c84cd07fde 100644
--- a/sound/soc/amd/acp/acp-pci.c
+++ b/sound/soc/amd/acp/acp-pci.c
@@ -62,10 +62,9 @@ static int acp_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id
 	if (!chip)
 		return -ENOMEM;
 
-	if (pci_enable_device(pci)) {
-		dev_err(&pci->dev, "pci_enable_device failed\n");
-		return -ENODEV;
-	}
+	if (pci_enable_device(pci))
+		return dev_err_probe(&pci->dev, -ENODEV,
+				     "pci_enable_device failed\n");
 
 	ret = pci_request_regions(pci, "AMD ACP3x audio");
 	if (ret < 0) {
@@ -105,14 +104,13 @@ static int acp_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id
 	chip->base = devm_ioremap(&pci->dev, addr, pci_resource_len(pci, 0));
 	if (!chip->base) {
 		ret = -ENOMEM;
-		goto release_regions;
+		goto unregister_dmic_dev;
 	}
 
-	res = devm_kzalloc(&pci->dev, sizeof(struct resource) * num_res, GFP_KERNEL);
+	res = devm_kcalloc(&pci->dev, num_res, sizeof(struct resource), GFP_KERNEL);
 	if (!res) {
-		platform_device_unregister(dmic_dev);
 		ret = -ENOMEM;
-		goto release_regions;
+		goto unregister_dmic_dev;
 	}
 
 	for (i = 0; i < num_res; i++, res_acp++) {
@@ -139,13 +137,14 @@ static int acp_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id
 	pdev = platform_device_register_full(&pdevinfo);
 	if (IS_ERR(pdev)) {
 		dev_err(&pci->dev, "cannot register %s device\n", pdevinfo.name);
-		platform_device_unregister(dmic_dev);
 		ret = PTR_ERR(pdev);
-		goto release_regions;
+		goto unregister_dmic_dev;
 	}
 
 	return ret;
 
+unregister_dmic_dev:
+	platform_device_unregister(dmic_dev);
 release_regions:
 	pci_release_regions(pci);
 disable_pci:
diff --git a/sound/soc/amd/acp/acp-platform.c b/sound/soc/amd/acp/acp-platform.c
index f561d39b33e2..85a81add4ef9 100644
--- a/sound/soc/amd/acp/acp-platform.c
+++ b/sound/soc/amd/acp/acp-platform.c
@@ -94,7 +94,7 @@ static irqreturn_t i2s_irq_handler(int irq, void *data)
 	struct acp_resource *rsrc = adata->rsrc;
 	struct acp_stream *stream;
 	u16 i2s_flag = 0;
-	u32 ext_intr_stat, ext_intr_stat1, i;
+	u32 ext_intr_stat, ext_intr_stat1;
 
 	if (!adata)
 		return IRQ_NONE;
@@ -104,25 +104,24 @@ static irqreturn_t i2s_irq_handler(int irq, void *data)
 
 	ext_intr_stat = readl(ACP_EXTERNAL_INTR_STAT(adata, rsrc->irqp_used));
 
-	for (i = 0; i < ACP_MAX_STREAM; i++) {
-		stream = adata->stream[i];
-		if (stream && (ext_intr_stat & stream->irq_bit)) {
+	spin_lock(&adata->acp_lock);
+	list_for_each_entry(stream, &adata->stream_list, list) {
+		if (ext_intr_stat & stream->irq_bit) {
 			writel(stream->irq_bit,
 			       ACP_EXTERNAL_INTR_STAT(adata, rsrc->irqp_used));
 			snd_pcm_period_elapsed(stream->substream);
 			i2s_flag = 1;
-			break;
 		}
 		if (adata->rsrc->no_of_ctrls == 2) {
-			if (stream && (ext_intr_stat1 & stream->irq_bit)) {
+			if (ext_intr_stat1 & stream->irq_bit) {
 				writel(stream->irq_bit, ACP_EXTERNAL_INTR_STAT(adata,
 				       (rsrc->irqp_used - 1)));
 				snd_pcm_period_elapsed(stream->substream);
 				i2s_flag = 1;
-				break;
 			}
 		}
 	}
+	spin_unlock(&adata->acp_lock);
 	if (i2s_flag)
 		return IRQ_HANDLED;
 
@@ -146,9 +145,8 @@ static void config_pte_for_stream(struct acp_dev_data *adata, struct acp_stream
 	writel(0x01, adata->acp_base + ACPAXI2AXI_ATU_CTRL);
 }
 
-static void config_acp_dma(struct acp_dev_data *adata, int cpu_id, int size)
+static void config_acp_dma(struct acp_dev_data *adata, struct acp_stream *stream, int size)
 {
-	struct acp_stream *stream = adata->stream[cpu_id];
 	struct snd_pcm_substream *substream = stream->substream;
 	struct acp_resource *rsrc = adata->rsrc;
 	dma_addr_t addr = substream->dma_buffer.addr;
@@ -174,13 +172,10 @@ static void config_acp_dma(struct acp_dev_data *adata, int cpu_id, int size)
 
 static int acp_dma_open(struct snd_soc_component *component, struct snd_pcm_substream *substream)
 {
-	struct snd_soc_pcm_runtime *soc_runtime = asoc_substream_to_rtd(substream);
-	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(soc_runtime, 0);
 	struct snd_pcm_runtime *runtime = substream->runtime;
 	struct device *dev = component->dev;
 	struct acp_dev_data *adata = dev_get_drvdata(dev);
 	struct acp_stream *stream;
-	int stream_id = cpu_dai->driver->id * 2 + substream->stream;
 	int ret;
 
 	stream = kzalloc(sizeof(*stream), GFP_KERNEL);
@@ -188,7 +183,10 @@ static int acp_dma_open(struct snd_soc_component *component, struct snd_pcm_subs
 		return -ENOMEM;
 
 	stream->substream = substream;
-	adata->stream[stream_id] = stream;
+
+	spin_lock_irq(&adata->acp_lock);
+	list_add_tail(&stream->list, &adata->stream_list);
+	spin_unlock_irq(&adata->acp_lock);
 
 	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
 		runtime->hw = acp_pcm_hardware_playback;
@@ -212,16 +210,13 @@ static int acp_dma_hw_params(struct snd_soc_component *component,
 			     struct snd_pcm_substream *substream,
 			     struct snd_pcm_hw_params *params)
 {
-	struct snd_soc_pcm_runtime *soc_runtime = asoc_substream_to_rtd(substream);
 	struct acp_dev_data *adata = snd_soc_component_get_drvdata(component);
-	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(soc_runtime, 0);
 	struct acp_stream *stream = substream->runtime->private_data;
-	int stream_id = cpu_dai->driver->id * 2 + substream->stream;
 	u64 size = params_buffer_bytes(params);
 
 	/* Configure ACP DMA block with params */
 	config_pte_for_stream(adata, stream);
-	config_acp_dma(adata, stream_id, size);
+	config_acp_dma(adata, stream, size);
 
 	return 0;
 }
@@ -261,16 +256,15 @@ static int acp_dma_new(struct snd_soc_component *component,
 static int acp_dma_close(struct snd_soc_component *component,
 			 struct snd_pcm_substream *substream)
 {
-	struct snd_soc_pcm_runtime *soc_runtime = asoc_substream_to_rtd(substream);
-	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(soc_runtime, 0);
 	struct device *dev = component->dev;
 	struct acp_dev_data *adata = dev_get_drvdata(dev);
-	struct acp_stream *stream;
-	int stream_id = cpu_dai->driver->id * 2 + substream->stream;
+	struct acp_stream *stream = substream->runtime->private_data;
 
-	stream = adata->stream[stream_id];
+	/* Remove entry from list */
+	spin_lock_irq(&adata->acp_lock);
+	list_del(&stream->list);
+	spin_unlock_irq(&adata->acp_lock);
 	kfree(stream);
-	adata->stream[stream_id] = NULL;
 
 	return 0;
 }
@@ -305,6 +299,10 @@ int acp_platform_register(struct device *dev)
 		dev_err(dev, "Fail to register acp i2s component\n");
 		return status;
 	}
+
+	INIT_LIST_HEAD(&adata->stream_list);
+	spin_lock_init(&adata->acp_lock);
+
 	return 0;
 }
 EXPORT_SYMBOL_NS_GPL(acp_platform_register, SND_SOC_ACP_COMMON);
diff --git a/sound/soc/amd/acp/amd.h b/sound/soc/amd/acp/amd.h
index af9603724a68..5f2119f42271 100644
--- a/sound/soc/amd/acp/amd.h
+++ b/sound/soc/amd/acp/amd.h
@@ -21,9 +21,9 @@
 #define ACP3X_DEV			3
 #define ACP6X_DEV			6
 
-#define I2S_SP_INSTANCE			0x00
-#define I2S_BT_INSTANCE			0x01
-#define DMIC_INSTANCE			0x02
+#define DMIC_INSTANCE			0x00
+#define I2S_SP_INSTANCE			0x01
+#define I2S_BT_INSTANCE			0x02
 #define I2S_HS_INSTANCE			0x03
 
 #define MEM_WINDOW_START		0x4080000
@@ -84,6 +84,14 @@
 
 #define ACP_MAX_STREAM			8
 
+#define TDM_ENABLE	1
+#define TDM_DISABLE	0
+
+#define SLOT_WIDTH_8	0x8
+#define SLOT_WIDTH_16	0x10
+#define SLOT_WIDTH_24	0x18
+#define SLOT_WIDTH_32	0x20
+
 struct acp_chip_info {
 	char *name;		/* Platform name */
 	unsigned int acp_rev;	/* ACP Revision id */
@@ -91,10 +99,12 @@ struct acp_chip_info {
 };
 
 struct acp_stream {
+	struct list_head list;
 	struct snd_pcm_substream *substream;
 	int irq_bit;
 	int dai_id;
 	int id;
+	int dir;
 	u64 bytescount;
 	u32 reg_offset;
 	u32 pte_offset;
@@ -119,11 +129,13 @@ struct acp_dev_data {
 	void __iomem *acp_base;
 	unsigned int i2s_irq;
 
+	bool tdm_mode;
 	/* SOC specific dais */
 	struct snd_soc_dai_driver *dai_driver;
 	int num_dai;
 
-	struct acp_stream *stream[ACP_MAX_STREAM];
+	struct list_head stream_list;
+	spinlock_t acp_lock;
 
 	struct snd_soc_acpi_mach *machines;
 	struct platform_device *mach_dev;
@@ -132,6 +144,8 @@ struct acp_dev_data {
 	u32 lrclk_div;
 
 	struct acp_resource *rsrc;
+	u32 tdm_tx_fmt[3];
+	u32 tdm_rx_fmt[3];
 };
 
 union acp_i2stdm_mstrclkgen {
diff --git a/sound/soc/amd/ps/Makefile b/sound/soc/amd/ps/Makefile
new file mode 100644
index 000000000000..383973a12f6a
--- /dev/null
+++ b/sound/soc/amd/ps/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0+
+# Pink Sardine platform Support
+snd-pci-ps-objs := pci-ps.o
+snd-ps-pdm-dma-objs := ps-pdm-dma.o
+snd-soc-ps-mach-objs := ps-mach.o
+
+obj-$(CONFIG_SND_SOC_AMD_PS) += snd-pci-ps.o
+obj-$(CONFIG_SND_SOC_AMD_PS) += snd-ps-pdm-dma.o
+obj-$(CONFIG_SND_SOC_AMD_PS_MACH)   += snd-soc-ps-mach.o
diff --git a/sound/soc/amd/ps/acp62.h b/sound/soc/amd/ps/acp62.h
new file mode 100644
index 000000000000..8b30aefa4cd0
--- /dev/null
+++ b/sound/soc/amd/ps/acp62.h
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * AMD ALSA SoC PDM Driver
+ *
+ * Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
+ */
+
+#include <sound/acp62_chip_offset_byte.h>
+
+#define ACP_DEVICE_ID 0x15E2
+#define ACP6x_REG_START		0x1240000
+#define ACP6x_REG_END		0x1250200
+#define ACP6x_DEVS		3
+#define ACP6x_PDM_MODE		1
+
+#define ACP_SOFT_RESET_SOFTRESET_AUDDONE_MASK	0x00010001
+#define ACP_PGFSM_CNTL_POWER_ON_MASK	1
+#define ACP_PGFSM_CNTL_POWER_OFF_MASK	0
+#define ACP_PGFSM_STATUS_MASK		3
+#define ACP_POWERED_ON			0
+#define ACP_POWER_ON_IN_PROGRESS	1
+#define ACP_POWERED_OFF		2
+#define ACP_POWER_OFF_IN_PROGRESS	3
+
+#define ACP_ERROR_MASK 0x20000000
+#define ACP_EXT_INTR_STAT_CLEAR_MASK 0xFFFFFFFF
+#define PDM_DMA_STAT 0x10
+
+#define PDM_DMA_INTR_MASK	0x10000
+#define ACP_ERROR_STAT	29
+#define PDM_DECIMATION_FACTOR	2
+#define ACP_PDM_CLK_FREQ_MASK	7
+#define ACP_WOV_MISC_CTRL_MASK	0x10
+#define ACP_PDM_ENABLE		1
+#define ACP_PDM_DISABLE		0
+#define ACP_PDM_DMA_EN_STATUS	2
+#define TWO_CH		2
+#define DELAY_US	5
+#define ACP_COUNTER	20000
+
+#define ACP_SRAM_PTE_OFFSET	0x03800000
+#define PAGE_SIZE_4K_ENABLE	2
+#define PDM_PTE_OFFSET		0
+#define PDM_MEM_WINDOW_START	0x4000000
+
+#define CAPTURE_MIN_NUM_PERIODS     4
+#define CAPTURE_MAX_NUM_PERIODS     4
+#define CAPTURE_MAX_PERIOD_SIZE     8192
+#define CAPTURE_MIN_PERIOD_SIZE     4096
+
+#define MAX_BUFFER (CAPTURE_MAX_PERIOD_SIZE * CAPTURE_MAX_NUM_PERIODS)
+#define MIN_BUFFER MAX_BUFFER
+
+/* time in ms for runtime suspend delay */
+#define ACP_SUSPEND_DELAY_MS	2000
+
+enum acp_config {
+	ACP_CONFIG_0 = 0,
+	ACP_CONFIG_1,
+	ACP_CONFIG_2,
+	ACP_CONFIG_3,
+	ACP_CONFIG_4,
+	ACP_CONFIG_5,
+	ACP_CONFIG_6,
+	ACP_CONFIG_7,
+	ACP_CONFIG_8,
+	ACP_CONFIG_9,
+	ACP_CONFIG_10,
+	ACP_CONFIG_11,
+	ACP_CONFIG_12,
+	ACP_CONFIG_13,
+	ACP_CONFIG_14,
+	ACP_CONFIG_15,
+};
+
+struct pdm_stream_instance {
+	u16 num_pages;
+	u16 channels;
+	dma_addr_t dma_addr;
+	u64 bytescount;
+	void __iomem *acp62_base;
+};
+
+struct pdm_dev_data {
+	u32 pdm_irq;
+	void __iomem *acp62_base;
+	struct snd_pcm_substream *capture_stream;
+};
+
+static inline u32 acp62_readl(void __iomem *base_addr)
+{
+	return readl(base_addr);
+}
+
+static inline void acp62_writel(u32 val, void __iomem *base_addr)
+{
+	writel(val, base_addr);
+}
diff --git a/sound/soc/amd/ps/pci-ps.c b/sound/soc/amd/ps/pci-ps.c
new file mode 100644
index 000000000000..dff2e2376bbf
--- /dev/null
+++ b/sound/soc/amd/ps/pci-ps.c
@@ -0,0 +1,351 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * AMD Pink Sardine ACP PCI Driver
+ *
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ */
+
+#include <linux/pci.h>
+#include <linux/module.h>
+#include <linux/io.h>
+#include <linux/delay.h>
+#include <linux/platform_device.h>
+#include <linux/acpi.h>
+#include <linux/interrupt.h>
+#include <sound/pcm_params.h>
+#include <linux/pm_runtime.h>
+
+#include "acp62.h"
+
+struct acp62_dev_data {
+	void __iomem *acp62_base;
+	struct resource *res;
+	bool acp62_audio_mode;
+	struct platform_device *pdev[ACP6x_DEVS];
+};
+
+static int acp62_power_on(void __iomem *acp_base)
+{
+	u32 val;
+	int timeout;
+
+	val = acp62_readl(acp_base + ACP_PGFSM_STATUS);
+
+	if (!val)
+		return val;
+
+	if ((val & ACP_PGFSM_STATUS_MASK) != ACP_POWER_ON_IN_PROGRESS)
+		acp62_writel(ACP_PGFSM_CNTL_POWER_ON_MASK, acp_base + ACP_PGFSM_CONTROL);
+	timeout = 0;
+	while (++timeout < 500) {
+		val = acp62_readl(acp_base + ACP_PGFSM_STATUS);
+		if (!val)
+			return 0;
+		udelay(1);
+	}
+	return -ETIMEDOUT;
+}
+
+static int acp62_reset(void __iomem *acp_base)
+{
+	u32 val;
+	int timeout;
+
+	acp62_writel(1, acp_base + ACP_SOFT_RESET);
+	timeout = 0;
+	while (++timeout < 500) {
+		val = acp62_readl(acp_base + ACP_SOFT_RESET);
+		if (val & ACP_SOFT_RESET_SOFTRESET_AUDDONE_MASK)
+			break;
+		cpu_relax();
+	}
+	acp62_writel(0, acp_base + ACP_SOFT_RESET);
+	timeout = 0;
+	while (++timeout < 500) {
+		val = acp62_readl(acp_base + ACP_SOFT_RESET);
+		if (!val)
+			return 0;
+		cpu_relax();
+	}
+	return -ETIMEDOUT;
+}
+
+static void acp62_enable_interrupts(void __iomem *acp_base)
+{
+	acp62_writel(1, acp_base + ACP_EXTERNAL_INTR_ENB);
+}
+
+static void acp62_disable_interrupts(void __iomem *acp_base)
+{
+	acp62_writel(ACP_EXT_INTR_STAT_CLEAR_MASK, acp_base +
+		     ACP_EXTERNAL_INTR_STAT);
+	acp62_writel(0, acp_base + ACP_EXTERNAL_INTR_CNTL);
+	acp62_writel(0, acp_base + ACP_EXTERNAL_INTR_ENB);
+}
+
+static int acp62_init(void __iomem *acp_base, struct device *dev)
+{
+	int ret;
+
+	ret = acp62_power_on(acp_base);
+	if (ret) {
+		dev_err(dev, "ACP power on failed\n");
+		return ret;
+	}
+	acp62_writel(0x01, acp_base + ACP_CONTROL);
+	ret = acp62_reset(acp_base);
+	if (ret) {
+		dev_err(dev, "ACP reset failed\n");
+		return ret;
+	}
+	acp62_writel(0x03, acp_base + ACP_CLKMUX_SEL);
+	acp62_enable_interrupts(acp_base);
+	return 0;
+}
+
+static int acp62_deinit(void __iomem *acp_base, struct device *dev)
+{
+	int ret;
+
+	acp62_disable_interrupts(acp_base);
+	ret = acp62_reset(acp_base);
+	if (ret) {
+		dev_err(dev, "ACP reset failed\n");
+		return ret;
+	}
+	acp62_writel(0, acp_base + ACP_CLKMUX_SEL);
+	acp62_writel(0, acp_base + ACP_CONTROL);
+	return 0;
+}
+
+static irqreturn_t acp62_irq_handler(int irq, void *dev_id)
+{
+	struct acp62_dev_data *adata;
+	struct pdm_dev_data *ps_pdm_data;
+	u32 val;
+
+	adata = dev_id;
+	if (!adata)
+		return IRQ_NONE;
+
+	val = acp62_readl(adata->acp62_base + ACP_EXTERNAL_INTR_STAT);
+	if (val & BIT(PDM_DMA_STAT)) {
+		ps_pdm_data = dev_get_drvdata(&adata->pdev[0]->dev);
+		acp62_writel(BIT(PDM_DMA_STAT), adata->acp62_base + ACP_EXTERNAL_INTR_STAT);
+		if (ps_pdm_data->capture_stream)
+			snd_pcm_period_elapsed(ps_pdm_data->capture_stream);
+		return IRQ_HANDLED;
+	}
+	return IRQ_NONE;
+}
+
+static int snd_acp62_probe(struct pci_dev *pci,
+			   const struct pci_device_id *pci_id)
+{
+	struct acp62_dev_data *adata;
+	struct platform_device_info pdevinfo[ACP6x_DEVS];
+	int index, ret;
+	int val = 0x00;
+	struct acpi_device *adev;
+	const union acpi_object *obj;
+	u32 addr;
+	unsigned int irqflags;
+
+	irqflags = IRQF_SHARED;
+	/* Pink Sardine device check */
+	switch (pci->revision) {
+	case 0x63:
+		break;
+	default:
+		dev_dbg(&pci->dev, "acp62 pci device not found\n");
+		return -ENODEV;
+	}
+	if (pci_enable_device(pci)) {
+		dev_err(&pci->dev, "pci_enable_device failed\n");
+		return -ENODEV;
+	}
+
+	ret = pci_request_regions(pci, "AMD ACP6.2 audio");
+	if (ret < 0) {
+		dev_err(&pci->dev, "pci_request_regions failed\n");
+		goto disable_pci;
+	}
+	adata = devm_kzalloc(&pci->dev, sizeof(struct acp62_dev_data),
+			     GFP_KERNEL);
+	if (!adata) {
+		ret = -ENOMEM;
+		goto release_regions;
+	}
+
+	addr = pci_resource_start(pci, 0);
+	adata->acp62_base = devm_ioremap(&pci->dev, addr,
+					 pci_resource_len(pci, 0));
+	if (!adata->acp62_base) {
+		ret = -ENOMEM;
+		goto release_regions;
+	}
+	pci_set_master(pci);
+	pci_set_drvdata(pci, adata);
+	ret = acp62_init(adata->acp62_base, &pci->dev);
+	if (ret)
+		goto release_regions;
+	val = acp62_readl(adata->acp62_base + ACP_PIN_CONFIG);
+	switch (val) {
+	case ACP_CONFIG_0:
+	case ACP_CONFIG_1:
+	case ACP_CONFIG_2:
+	case ACP_CONFIG_3:
+	case ACP_CONFIG_9:
+	case ACP_CONFIG_15:
+		dev_info(&pci->dev, "Audio Mode %d\n", val);
+		break;
+	default:
+
+		/* Checking DMIC hardware*/
+		adev = acpi_find_child_device(ACPI_COMPANION(&pci->dev), 0x02, 0);
+
+		if (!adev)
+			break;
+
+		if (!acpi_dev_get_property(adev, "acp-audio-device-type",
+					   ACPI_TYPE_INTEGER, &obj) &&
+					   obj->integer.value == 2) {
+			adata->res = devm_kzalloc(&pci->dev, sizeof(struct resource), GFP_KERNEL);
+			if (!adata->res) {
+				ret = -ENOMEM;
+				goto de_init;
+			}
+
+			adata->res->name = "acp_iomem";
+			adata->res->flags = IORESOURCE_MEM;
+			adata->res->start = addr;
+			adata->res->end = addr + (ACP6x_REG_END - ACP6x_REG_START);
+			adata->acp62_audio_mode = ACP6x_PDM_MODE;
+
+			memset(&pdevinfo, 0, sizeof(pdevinfo));
+			pdevinfo[0].name = "acp_ps_pdm_dma";
+			pdevinfo[0].id = 0;
+			pdevinfo[0].parent = &pci->dev;
+			pdevinfo[0].num_res = 1;
+			pdevinfo[0].res = adata->res;
+
+			pdevinfo[1].name = "dmic-codec";
+			pdevinfo[1].id = 0;
+			pdevinfo[1].parent = &pci->dev;
+
+			pdevinfo[2].name = "acp_ps_mach";
+			pdevinfo[2].id = 0;
+			pdevinfo[2].parent = &pci->dev;
+
+			for (index = 0; index < ACP6x_DEVS; index++) {
+				adata->pdev[index] =
+					platform_device_register_full(&pdevinfo[index]);
+
+				if (IS_ERR(adata->pdev[index])) {
+					dev_err(&pci->dev,
+						"cannot register %s device\n",
+						 pdevinfo[index].name);
+					ret = PTR_ERR(adata->pdev[index]);
+					goto unregister_devs;
+				}
+				ret = devm_request_irq(&pci->dev, pci->irq, acp62_irq_handler,
+						       irqflags, "ACP_PCI_IRQ", adata);
+				if (ret) {
+					dev_err(&pci->dev, "ACP PCI IRQ request failed\n");
+					goto unregister_devs;
+				}
+			}
+		}
+		break;
+	}
+	pm_runtime_set_autosuspend_delay(&pci->dev, ACP_SUSPEND_DELAY_MS);
+	pm_runtime_use_autosuspend(&pci->dev);
+	pm_runtime_put_noidle(&pci->dev);
+	pm_runtime_allow(&pci->dev);
+	return 0;
+unregister_devs:
+	for (--index; index >= 0; index--)
+		platform_device_unregister(adata->pdev[index]);
+de_init:
+	if (acp62_deinit(adata->acp62_base, &pci->dev))
+		dev_err(&pci->dev, "ACP de-init failed\n");
+release_regions:
+	pci_release_regions(pci);
+disable_pci:
+	pci_disable_device(pci);
+
+	return ret;
+}
+
+static int __maybe_unused snd_acp62_suspend(struct device *dev)
+{
+	struct acp62_dev_data *adata;
+	int ret;
+
+	adata = dev_get_drvdata(dev);
+	ret = acp62_deinit(adata->acp62_base, dev);
+	if (ret)
+		dev_err(dev, "ACP de-init failed\n");
+	return ret;
+}
+
+static int __maybe_unused snd_acp62_resume(struct device *dev)
+{
+	struct acp62_dev_data *adata;
+	int ret;
+
+	adata = dev_get_drvdata(dev);
+	ret = acp62_init(adata->acp62_base, dev);
+	if (ret)
+		dev_err(dev, "ACP init failed\n");
+	return ret;
+}
+
+static const struct dev_pm_ops acp62_pm_ops = {
+	SET_RUNTIME_PM_OPS(snd_acp62_suspend, snd_acp62_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(snd_acp62_suspend, snd_acp62_resume)
+};
+
+static void snd_acp62_remove(struct pci_dev *pci)
+{
+	struct acp62_dev_data *adata;
+	int ret, index;
+
+	adata = pci_get_drvdata(pci);
+	if (adata->acp62_audio_mode == ACP6x_PDM_MODE) {
+		for (index = 0; index < ACP6x_DEVS; index++)
+			platform_device_unregister(adata->pdev[index]);
+	}
+	ret = acp62_deinit(adata->acp62_base, &pci->dev);
+	if (ret)
+		dev_err(&pci->dev, "ACP de-init failed\n");
+	pm_runtime_forbid(&pci->dev);
+	pm_runtime_get_noresume(&pci->dev);
+	pci_release_regions(pci);
+	pci_disable_device(pci);
+}
+
+static const struct pci_device_id snd_acp62_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, ACP_DEVICE_ID),
+	.class = PCI_CLASS_MULTIMEDIA_OTHER << 8,
+	.class_mask = 0xffffff },
+	{ 0, },
+};
+MODULE_DEVICE_TABLE(pci, snd_acp62_ids);
+
+static struct pci_driver ps_acp62_driver  = {
+	.name = KBUILD_MODNAME,
+	.id_table = snd_acp62_ids,
+	.probe = snd_acp62_probe,
+	.remove = snd_acp62_remove,
+	.driver = {
+		.pm = &acp62_pm_ops,
+	}
+};
+
+module_pci_driver(ps_acp62_driver);
+
+MODULE_AUTHOR("Vijendar.Mukunda@amd.com");
+MODULE_AUTHOR("Syed.SabaKareem@amd.com");
+MODULE_DESCRIPTION("AMD ACP Pink Sardine PCI driver");
+MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/amd/ps/ps-mach.c b/sound/soc/amd/ps/ps-mach.c
new file mode 100644
index 000000000000..b3e97093481d
--- /dev/null
+++ b/sound/soc/amd/ps/ps-mach.c
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Machine driver for AMD Pink Sardine platform using DMIC
+ *
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ */
+
+#include <sound/soc.h>
+#include <sound/soc-dapm.h>
+#include <linux/module.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <linux/io.h>
+#include <linux/dmi.h>
+
+#include "acp62.h"
+
+#define DRV_NAME "acp_ps_mach"
+
+SND_SOC_DAILINK_DEF(acp62_pdm,
+		    DAILINK_COMP_ARRAY(COMP_CPU("acp_ps_pdm_dma.0")));
+
+SND_SOC_DAILINK_DEF(dmic_codec,
+		    DAILINK_COMP_ARRAY(COMP_CODEC("dmic-codec.0",
+						  "dmic-hifi")));
+
+SND_SOC_DAILINK_DEF(pdm_platform,
+		    DAILINK_COMP_ARRAY(COMP_PLATFORM("acp_ps_pdm_dma.0")));
+
+static struct snd_soc_dai_link acp62_dai_pdm[] = {
+	{
+		.name = "acp62-dmic-capture",
+		.stream_name = "DMIC capture",
+		.capture_only = 1,
+		SND_SOC_DAILINK_REG(acp62_pdm, dmic_codec, pdm_platform),
+	},
+};
+
+static struct snd_soc_card acp62_card = {
+	.name = "acp62",
+	.owner = THIS_MODULE,
+	.dai_link = acp62_dai_pdm,
+	.num_links = 1,
+};
+
+static int acp62_probe(struct platform_device *pdev)
+{
+	struct acp62_pdm *machine = NULL;
+	struct snd_soc_card *card;
+	int ret;
+
+	platform_set_drvdata(pdev, &acp62_card);
+	card = platform_get_drvdata(pdev);
+	acp62_card.dev = &pdev->dev;
+
+	snd_soc_card_set_drvdata(card, machine);
+	ret = devm_snd_soc_register_card(&pdev->dev, card);
+	if (ret) {
+		return dev_err_probe(&pdev->dev, ret,
+				"snd_soc_register_card(%s) failed\n",
+				card->name);
+	}
+
+	return 0;
+}
+
+static struct platform_driver acp62_mach_driver = {
+	.driver = {
+		.name = "acp_ps_mach",
+		.pm = &snd_soc_pm_ops,
+	},
+	.probe = acp62_probe,
+};
+
+module_platform_driver(acp62_mach_driver);
+
+MODULE_AUTHOR("Syed.SabaKareem@amd.com");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:" DRV_NAME);
diff --git a/sound/soc/amd/ps/ps-pdm-dma.c b/sound/soc/amd/ps/ps-pdm-dma.c
new file mode 100644
index 000000000000..b207b726cd82
--- /dev/null
+++ b/sound/soc/amd/ps/ps-pdm-dma.c
@@ -0,0 +1,452 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * AMD ALSA SoC Pink Sardine PDM Driver
+ *
+ * Copyright 2022 Advanced Micro Devices, Inc.
+ */
+
+#include <linux/platform_device.h>
+#include <linux/module.h>
+#include <linux/err.h>
+#include <linux/io.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/soc-dai.h>
+#include <linux/pm_runtime.h>
+
+#include "acp62.h"
+
+#define DRV_NAME "acp_ps_pdm_dma"
+
+static const struct snd_pcm_hardware acp62_pdm_hardware_capture = {
+	.info = SNDRV_PCM_INFO_INTERLEAVED |
+		SNDRV_PCM_INFO_BLOCK_TRANSFER |
+		SNDRV_PCM_INFO_MMAP |
+		SNDRV_PCM_INFO_MMAP_VALID |
+		SNDRV_PCM_INFO_PAUSE | SNDRV_PCM_INFO_RESUME,
+	.formats = SNDRV_PCM_FMTBIT_S32_LE,
+	.channels_min = 2,
+	.channels_max = 2,
+	.rates = SNDRV_PCM_RATE_48000,
+	.rate_min = 48000,
+	.rate_max = 48000,
+	.buffer_bytes_max = CAPTURE_MAX_NUM_PERIODS * CAPTURE_MAX_PERIOD_SIZE,
+	.period_bytes_min = CAPTURE_MIN_PERIOD_SIZE,
+	.period_bytes_max = CAPTURE_MAX_PERIOD_SIZE,
+	.periods_min = CAPTURE_MIN_NUM_PERIODS,
+	.periods_max = CAPTURE_MAX_NUM_PERIODS,
+};
+
+static void acp62_init_pdm_ring_buffer(u32 physical_addr, u32 buffer_size,
+				       u32 watermark_size, void __iomem *acp_base)
+{
+	acp62_writel(physical_addr, acp_base + ACP_WOV_RX_RINGBUFADDR);
+	acp62_writel(buffer_size, acp_base + ACP_WOV_RX_RINGBUFSIZE);
+	acp62_writel(watermark_size, acp_base + ACP_WOV_RX_INTR_WATERMARK_SIZE);
+	acp62_writel(0x01, acp_base + ACPAXI2AXI_ATU_CTRL);
+}
+
+static void acp62_enable_pdm_clock(void __iomem *acp_base)
+{
+	u32 pdm_clk_enable, pdm_ctrl;
+
+	pdm_clk_enable = ACP_PDM_CLK_FREQ_MASK;
+	pdm_ctrl = 0x00;
+
+	acp62_writel(pdm_clk_enable, acp_base + ACP_WOV_CLK_CTRL);
+	pdm_ctrl = acp62_readl(acp_base + ACP_WOV_MISC_CTRL);
+	pdm_ctrl |= ACP_WOV_MISC_CTRL_MASK;
+	acp62_writel(pdm_ctrl, acp_base + ACP_WOV_MISC_CTRL);
+}
+
+static void acp62_enable_pdm_interrupts(void __iomem *acp_base)
+{
+	u32 ext_int_ctrl;
+
+	ext_int_ctrl = acp62_readl(acp_base + ACP_EXTERNAL_INTR_CNTL);
+	ext_int_ctrl |= PDM_DMA_INTR_MASK;
+	acp62_writel(ext_int_ctrl, acp_base + ACP_EXTERNAL_INTR_CNTL);
+}
+
+static void acp62_disable_pdm_interrupts(void __iomem *acp_base)
+{
+	u32 ext_int_ctrl;
+
+	ext_int_ctrl = acp62_readl(acp_base + ACP_EXTERNAL_INTR_CNTL);
+	ext_int_ctrl &= ~PDM_DMA_INTR_MASK;
+	acp62_writel(ext_int_ctrl, acp_base + ACP_EXTERNAL_INTR_CNTL);
+}
+
+static bool acp62_check_pdm_dma_status(void __iomem *acp_base)
+{
+	bool pdm_dma_status;
+	u32 pdm_enable, pdm_dma_enable;
+
+	pdm_dma_status = false;
+	pdm_enable = acp62_readl(acp_base + ACP_WOV_PDM_ENABLE);
+	pdm_dma_enable = acp62_readl(acp_base + ACP_WOV_PDM_DMA_ENABLE);
+	if ((pdm_enable & ACP_PDM_ENABLE) && (pdm_dma_enable & ACP_PDM_DMA_EN_STATUS))
+		pdm_dma_status = true;
+
+	return pdm_dma_status;
+}
+
+static int acp62_start_pdm_dma(void __iomem *acp_base)
+{
+	u32 pdm_enable;
+	u32 pdm_dma_enable;
+	int timeout;
+
+	pdm_enable = 0x01;
+	pdm_dma_enable  = 0x01;
+
+	acp62_enable_pdm_clock(acp_base);
+	acp62_writel(pdm_enable, acp_base + ACP_WOV_PDM_ENABLE);
+	acp62_writel(pdm_dma_enable, acp_base + ACP_WOV_PDM_DMA_ENABLE);
+	timeout = 0;
+	while (++timeout < ACP_COUNTER) {
+		pdm_dma_enable = acp62_readl(acp_base + ACP_WOV_PDM_DMA_ENABLE);
+		if ((pdm_dma_enable & 0x02) == ACP_PDM_DMA_EN_STATUS)
+			return 0;
+		udelay(DELAY_US);
+	}
+	return -ETIMEDOUT;
+}
+
+static int acp62_stop_pdm_dma(void __iomem *acp_base)
+{
+	u32 pdm_enable, pdm_dma_enable;
+	int timeout;
+
+	pdm_enable = 0x00;
+	pdm_dma_enable  = 0x00;
+
+	pdm_enable = acp62_readl(acp_base + ACP_WOV_PDM_ENABLE);
+	pdm_dma_enable = acp62_readl(acp_base + ACP_WOV_PDM_DMA_ENABLE);
+	if (pdm_dma_enable & 0x01) {
+		pdm_dma_enable = 0x02;
+		acp62_writel(pdm_dma_enable, acp_base + ACP_WOV_PDM_DMA_ENABLE);
+		timeout = 0;
+		while (++timeout < ACP_COUNTER) {
+			pdm_dma_enable = acp62_readl(acp_base + ACP_WOV_PDM_DMA_ENABLE);
+			if ((pdm_dma_enable & 0x02) == 0x00)
+				break;
+			udelay(DELAY_US);
+		}
+		if (timeout == ACP_COUNTER)
+			return -ETIMEDOUT;
+	}
+	if (pdm_enable == ACP_PDM_ENABLE) {
+		pdm_enable = ACP_PDM_DISABLE;
+		acp62_writel(pdm_enable, acp_base + ACP_WOV_PDM_ENABLE);
+	}
+	acp62_writel(0x01, acp_base + ACP_WOV_PDM_FIFO_FLUSH);
+	return 0;
+}
+
+static void acp62_config_dma(struct pdm_stream_instance *rtd, int direction)
+{
+	u16 page_idx;
+	u32 low, high, val;
+	dma_addr_t addr;
+
+	addr = rtd->dma_addr;
+	val = PDM_PTE_OFFSET;
+
+	/* Group Enable */
+	acp62_writel(ACP_SRAM_PTE_OFFSET | BIT(31), rtd->acp62_base +
+		     ACPAXI2AXI_ATU_BASE_ADDR_GRP_1);
+	acp62_writel(PAGE_SIZE_4K_ENABLE, rtd->acp62_base +
+		     ACPAXI2AXI_ATU_PAGE_SIZE_GRP_1);
+	for (page_idx = 0; page_idx < rtd->num_pages; page_idx++) {
+		/* Load the low address of page int ACP SRAM through SRBM */
+		low = lower_32_bits(addr);
+		high = upper_32_bits(addr);
+
+		acp62_writel(low, rtd->acp62_base + ACP_SCRATCH_REG_0 + val);
+		high |= BIT(31);
+		acp62_writel(high, rtd->acp62_base + ACP_SCRATCH_REG_0 + val + 4);
+		val += 8;
+		addr += PAGE_SIZE;
+	}
+}
+
+static int acp62_pdm_dma_open(struct snd_soc_component *component,
+			      struct snd_pcm_substream *substream)
+{
+	struct snd_pcm_runtime *runtime;
+	struct pdm_dev_data *adata;
+	struct pdm_stream_instance *pdm_data;
+	int ret;
+
+	runtime = substream->runtime;
+	adata = dev_get_drvdata(component->dev);
+	pdm_data = kzalloc(sizeof(*pdm_data), GFP_KERNEL);
+	if (!pdm_data)
+		return -EINVAL;
+
+	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
+		runtime->hw = acp62_pdm_hardware_capture;
+
+	ret = snd_pcm_hw_constraint_integer(runtime,
+					    SNDRV_PCM_HW_PARAM_PERIODS);
+	if (ret < 0) {
+		dev_err(component->dev, "set integer constraint failed\n");
+		kfree(pdm_data);
+		return ret;
+	}
+
+	acp62_enable_pdm_interrupts(adata->acp62_base);
+
+	if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
+		adata->capture_stream = substream;
+
+	pdm_data->acp62_base = adata->acp62_base;
+	runtime->private_data = pdm_data;
+	return ret;
+}
+
+static int acp62_pdm_dma_hw_params(struct snd_soc_component *component,
+				   struct snd_pcm_substream *substream,
+				   struct snd_pcm_hw_params *params)
+{
+	struct pdm_stream_instance *rtd;
+	size_t size, period_bytes;
+
+	rtd = substream->runtime->private_data;
+	if (!rtd)
+		return -EINVAL;
+	size = params_buffer_bytes(params);
+	period_bytes = params_period_bytes(params);
+	rtd->dma_addr = substream->runtime->dma_addr;
+	rtd->num_pages = (PAGE_ALIGN(size) >> PAGE_SHIFT);
+	acp62_config_dma(rtd, substream->stream);
+	acp62_init_pdm_ring_buffer(PDM_MEM_WINDOW_START, size,
+				   period_bytes, rtd->acp62_base);
+	return 0;
+}
+
+static u64 acp62_pdm_get_byte_count(struct pdm_stream_instance *rtd,
+				    int direction)
+{
+	u32 high, low;
+	u64 byte_count;
+
+	high = acp62_readl(rtd->acp62_base + ACP_WOV_RX_LINEARPOSITIONCNTR_HIGH);
+	byte_count = high;
+	low = acp62_readl(rtd->acp62_base + ACP_WOV_RX_LINEARPOSITIONCNTR_LOW);
+	byte_count = (byte_count << 32) | low;
+	return byte_count;
+}
+
+static snd_pcm_uframes_t acp62_pdm_dma_pointer(struct snd_soc_component *comp,
+					       struct snd_pcm_substream *stream)
+{
+	struct pdm_stream_instance *rtd;
+	u32 pos, buffersize;
+	u64 bytescount;
+
+	rtd = stream->runtime->private_data;
+	buffersize = frames_to_bytes(stream->runtime,
+				     stream->runtime->buffer_size);
+	bytescount = acp62_pdm_get_byte_count(rtd, stream->stream);
+	if (bytescount > rtd->bytescount)
+		bytescount -= rtd->bytescount;
+	pos = do_div(bytescount, buffersize);
+	return bytes_to_frames(stream->runtime, pos);
+}
+
+static int acp62_pdm_dma_new(struct snd_soc_component *component,
+			     struct snd_soc_pcm_runtime *rtd)
+{
+	struct device *parent = component->dev->parent;
+
+	snd_pcm_set_managed_buffer_all(rtd->pcm, SNDRV_DMA_TYPE_DEV,
+				       parent, MIN_BUFFER, MAX_BUFFER);
+	return 0;
+}
+
+static int acp62_pdm_dma_close(struct snd_soc_component *component,
+			       struct snd_pcm_substream *substream)
+{
+	struct pdm_dev_data *adata = dev_get_drvdata(component->dev);
+	struct snd_pcm_runtime *runtime = substream->runtime;
+
+	acp62_disable_pdm_interrupts(adata->acp62_base);
+	adata->capture_stream = NULL;
+	kfree(runtime->private_data);
+	return 0;
+}
+
+static int acp62_pdm_dai_trigger(struct snd_pcm_substream *substream,
+				 int cmd, struct snd_soc_dai *dai)
+{
+	struct pdm_stream_instance *rtd;
+	int ret;
+	bool pdm_status;
+	unsigned int ch_mask;
+
+	rtd = substream->runtime->private_data;
+	ret = 0;
+	switch (substream->runtime->channels) {
+	case TWO_CH:
+		ch_mask = 0x00;
+		break;
+	default:
+		return -EINVAL;
+	}
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		acp62_writel(ch_mask, rtd->acp62_base + ACP_WOV_PDM_NO_OF_CHANNELS);
+		acp62_writel(PDM_DECIMATION_FACTOR, rtd->acp62_base +
+			     ACP_WOV_PDM_DECIMATION_FACTOR);
+		rtd->bytescount = acp62_pdm_get_byte_count(rtd, substream->stream);
+		pdm_status = acp62_check_pdm_dma_status(rtd->acp62_base);
+		if (!pdm_status)
+			ret = acp62_start_pdm_dma(rtd->acp62_base);
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		pdm_status = acp62_check_pdm_dma_status(rtd->acp62_base);
+		if (pdm_status)
+			ret = acp62_stop_pdm_dma(rtd->acp62_base);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static const struct snd_soc_dai_ops acp62_pdm_dai_ops = {
+	.trigger   = acp62_pdm_dai_trigger,
+};
+
+static struct snd_soc_dai_driver acp62_pdm_dai_driver = {
+	.name = "acp_ps_pdm_dma.0",
+	.capture = {
+		.rates = SNDRV_PCM_RATE_48000,
+		.formats = SNDRV_PCM_FMTBIT_S32_LE,
+		.channels_min = 2,
+		.channels_max = 2,
+		.rate_min = 48000,
+		.rate_max = 48000,
+	},
+	.ops = &acp62_pdm_dai_ops,
+};
+
+static const struct snd_soc_component_driver acp62_pdm_component = {
+	.name		= DRV_NAME,
+	.open		= acp62_pdm_dma_open,
+	.close		= acp62_pdm_dma_close,
+	.hw_params	= acp62_pdm_dma_hw_params,
+	.pointer	= acp62_pdm_dma_pointer,
+	.pcm_construct	= acp62_pdm_dma_new,
+};
+
+static int acp62_pdm_audio_probe(struct platform_device *pdev)
+{
+	struct resource *res;
+	struct pdm_dev_data *adata;
+	int status;
+
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res) {
+		dev_err(&pdev->dev, "IORESOURCE_MEM FAILED\n");
+		return -ENODEV;
+	}
+
+	adata = devm_kzalloc(&pdev->dev, sizeof(*adata), GFP_KERNEL);
+	if (!adata)
+		return -ENOMEM;
+
+	adata->acp62_base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
+	if (!adata->acp62_base)
+		return -ENOMEM;
+
+	adata->capture_stream = NULL;
+
+	dev_set_drvdata(&pdev->dev, adata);
+	status = devm_snd_soc_register_component(&pdev->dev,
+						 &acp62_pdm_component,
+						 &acp62_pdm_dai_driver, 1);
+	if (status) {
+		dev_err(&pdev->dev, "Fail to register acp pdm dai\n");
+
+		return -ENODEV;
+	}
+	pm_runtime_set_autosuspend_delay(&pdev->dev, ACP_SUSPEND_DELAY_MS);
+	pm_runtime_use_autosuspend(&pdev->dev);
+	pm_runtime_enable(&pdev->dev);
+	pm_runtime_allow(&pdev->dev);
+	return 0;
+}
+
+static int acp62_pdm_audio_remove(struct platform_device *pdev)
+{
+	pm_runtime_disable(&pdev->dev);
+	return 0;
+}
+
+static int __maybe_unused acp62_pdm_resume(struct device *dev)
+{
+	struct pdm_dev_data *adata;
+	struct snd_pcm_runtime *runtime;
+	struct pdm_stream_instance *rtd;
+	u32 period_bytes, buffer_len;
+
+	adata = dev_get_drvdata(dev);
+	if (adata->capture_stream && adata->capture_stream->runtime) {
+		runtime = adata->capture_stream->runtime;
+		rtd = runtime->private_data;
+		period_bytes = frames_to_bytes(runtime, runtime->period_size);
+		buffer_len = frames_to_bytes(runtime, runtime->buffer_size);
+		acp62_config_dma(rtd, SNDRV_PCM_STREAM_CAPTURE);
+		acp62_init_pdm_ring_buffer(PDM_MEM_WINDOW_START, buffer_len,
+					   period_bytes, adata->acp62_base);
+	}
+	acp62_enable_pdm_interrupts(adata->acp62_base);
+	return 0;
+}
+
+static int __maybe_unused acp62_pdm_suspend(struct device *dev)
+{
+	struct pdm_dev_data *adata;
+
+	adata = dev_get_drvdata(dev);
+	acp62_disable_pdm_interrupts(adata->acp62_base);
+	return 0;
+}
+
+static int __maybe_unused acp62_pdm_runtime_resume(struct device *dev)
+{
+	struct pdm_dev_data *adata;
+
+	adata = dev_get_drvdata(dev);
+	acp62_enable_pdm_interrupts(adata->acp62_base);
+	return 0;
+}
+
+static const struct dev_pm_ops acp62_pdm_pm_ops = {
+	SET_RUNTIME_PM_OPS(acp62_pdm_suspend, acp62_pdm_runtime_resume, NULL)
+	SET_SYSTEM_SLEEP_PM_OPS(acp62_pdm_suspend, acp62_pdm_resume)
+};
+
+static struct platform_driver acp62_pdm_dma_driver = {
+	.probe = acp62_pdm_audio_probe,
+	.remove = acp62_pdm_audio_remove,
+	.driver = {
+		.name = "acp_ps_pdm_dma",
+		.pm = &acp62_pdm_pm_ops,
+	},
+};
+
+module_platform_driver(acp62_pdm_dma_driver);
+
+MODULE_AUTHOR("Syed.SabaKareem@amd.com");
+MODULE_DESCRIPTION("AMD PINK SARDINE PDM Driver");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS("platform:" DRV_NAME);
diff --git a/sound/soc/amd/vangogh/acp5x-mach.c b/sound/soc/amd/vangogh/acp5x-mach.c
index af3737ef9707..eebf2650ad27 100644
--- a/sound/soc/amd/vangogh/acp5x-mach.c
+++ b/sound/soc/amd/vangogh/acp5x-mach.c
@@ -172,7 +172,7 @@ static int acp5x_cs35l41_hw_params(struct snd_pcm_substream *substream,
 	struct snd_soc_card *card = rtd->card;
 	struct snd_soc_dai *codec_dai;
 	int ret, i;
-	unsigned int num_codecs = rtd->num_codecs;
+	unsigned int num_codecs = rtd->dai_link->num_codecs;
 	unsigned int bclk_val;
 
 	ret = 0;
diff --git a/sound/soc/amd/yc/acp6x-mach.c b/sound/soc/amd/yc/acp6x-mach.c
index e0b24e1daef3..2cb50d5cf1a9 100644
--- a/sound/soc/amd/yc/acp6x-mach.c
+++ b/sound/soc/amd/yc/acp6x-mach.c
@@ -171,6 +171,20 @@ static const struct dmi_system_id yc_acp_quirk_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "21J6"),
 		}
 	},
+	{
+		.driver_data = &acp6x_card,
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "82"),
+		}
+	},
+	{
+		.driver_data = &acp6x_card,
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "UM5302TA"),
+		}
+	},
 	{}
 };
 
diff --git a/sound/soc/apple/Kconfig b/sound/soc/apple/Kconfig
new file mode 100644
index 000000000000..793f7782e0d7
--- /dev/null
+++ b/sound/soc/apple/Kconfig
@@ -0,0 +1,8 @@
+config SND_SOC_APPLE_MCA
+	tristate "Apple Silicon MCA driver"
+	depends on ARCH_APPLE || COMPILE_TEST
+	select SND_DMAENGINE_PCM
+	default ARCH_APPLE
+	help
+	  This option enables an ASoC platform driver for MCA peripherals found
+	  on Apple Silicon SoCs.
diff --git a/sound/soc/apple/Makefile b/sound/soc/apple/Makefile
new file mode 100644
index 000000000000..7a30bf452817
--- /dev/null
+++ b/sound/soc/apple/Makefile
@@ -0,0 +1,3 @@
+snd-soc-apple-mca-objs	:= mca.o
+
+obj-$(CONFIG_SND_SOC_APPLE_MCA)	+= snd-soc-apple-mca.o
diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c
new file mode 100644
index 000000000000..24381c42eb54
--- /dev/null
+++ b/sound/soc/apple/mca.c
@@ -0,0 +1,1174 @@
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// Apple SoCs MCA driver
+//
+// Copyright (C) The Asahi Linux Contributors
+//
+// The MCA peripheral is made up of a number of identical units called clusters.
+// Each cluster has its separate clock parent, SYNC signal generator, carries
+// four SERDES units and has a dedicated I2S port on the SoC's periphery.
+//
+// The clusters can operate independently, or can be combined together in a
+// configurable manner. We mostly treat them as self-contained independent
+// units and don't configure any cross-cluster connections except for the I2S
+// ports. The I2S ports can be routed to any of the clusters (irrespective
+// of their native cluster). We map this onto ASoC's (DPCM) notion of backend
+// and frontend DAIs. The 'cluster guts' are frontends which are dynamically
+// routed to backend I2S ports.
+//
+// DAI references in devicetree are resolved to backends. The routing between
+// frontends and backends is determined by the machine driver in the DAPM paths
+// it supplies.
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/dma-mapping.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_clk.h>
+#include <linux/of_dma.h>
+#include <linux/platform_device.h>
+#include <linux/pm_domain.h>
+#include <linux/regmap.h>
+#include <linux/reset.h>
+#include <linux/slab.h>
+
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/dmaengine_pcm.h>
+
+#define USE_RXB_FOR_CAPTURE
+
+/* Relative to cluster base */
+#define REG_STATUS		0x0
+#define STATUS_MCLK_EN		BIT(0)
+#define REG_MCLK_CONF		0x4
+#define MCLK_CONF_DIV		GENMASK(11, 8)
+
+#define REG_SYNCGEN_STATUS	0x100
+#define SYNCGEN_STATUS_EN	BIT(0)
+#define REG_SYNCGEN_MCLK_SEL	0x104
+#define SYNCGEN_MCLK_SEL	GENMASK(3, 0)
+#define REG_SYNCGEN_HI_PERIOD	0x108
+#define REG_SYNCGEN_LO_PERIOD	0x10c
+
+#define REG_PORT_ENABLES	0x600
+#define PORT_ENABLES_CLOCKS	GENMASK(2, 1)
+#define PORT_ENABLES_TX_DATA	BIT(3)
+#define REG_PORT_CLOCK_SEL	0x604
+#define PORT_CLOCK_SEL		GENMASK(11, 8)
+#define REG_PORT_DATA_SEL	0x608
+#define PORT_DATA_SEL_TXA(cl)	(1 << ((cl)*2))
+#define PORT_DATA_SEL_TXB(cl)	(2 << ((cl)*2))
+
+#define REG_INTSTATE		0x700
+#define REG_INTMASK		0x704
+
+/* Bases of serdes units (relative to cluster) */
+#define CLUSTER_RXA_OFF	0x200
+#define CLUSTER_TXA_OFF	0x300
+#define CLUSTER_RXB_OFF	0x400
+#define CLUSTER_TXB_OFF	0x500
+
+#define CLUSTER_TX_OFF	CLUSTER_TXA_OFF
+
+#ifndef USE_RXB_FOR_CAPTURE
+#define CLUSTER_RX_OFF	CLUSTER_RXA_OFF
+#else
+#define CLUSTER_RX_OFF	CLUSTER_RXB_OFF
+#endif
+
+/* Relative to serdes unit base */
+#define REG_SERDES_STATUS	0x00
+#define SERDES_STATUS_EN	BIT(0)
+#define SERDES_STATUS_RST	BIT(1)
+#define REG_TX_SERDES_CONF	0x04
+#define REG_RX_SERDES_CONF	0x08
+#define SERDES_CONF_NCHANS	GENMASK(3, 0)
+#define SERDES_CONF_WIDTH_MASK	GENMASK(8, 4)
+#define SERDES_CONF_WIDTH_16BIT 0x40
+#define SERDES_CONF_WIDTH_20BIT 0x80
+#define SERDES_CONF_WIDTH_24BIT 0xc0
+#define SERDES_CONF_WIDTH_32BIT 0x100
+#define SERDES_CONF_BCLK_POL	0x400
+#define SERDES_CONF_LSB_FIRST	0x800
+#define SERDES_CONF_UNK1	BIT(12)
+#define SERDES_CONF_UNK2	BIT(13)
+#define SERDES_CONF_UNK3	BIT(14)
+#define SERDES_CONF_NO_DATA_FEEDBACK	BIT(15)
+#define SERDES_CONF_SYNC_SEL	GENMASK(18, 16)
+#define SERDES_CONF_SOME_RST	BIT(19)
+#define REG_TX_SERDES_BITSTART	0x08
+#define REG_RX_SERDES_BITSTART	0x0c
+#define REG_TX_SERDES_SLOTMASK	0x0c
+#define REG_RX_SERDES_SLOTMASK	0x10
+#define REG_RX_SERDES_PORT	0x04
+
+/* Relative to switch base */
+#define REG_DMA_ADAPTER_A(cl)	(0x8000 * (cl))
+#define REG_DMA_ADAPTER_B(cl)	(0x8000 * (cl) + 0x4000)
+#define DMA_ADAPTER_TX_LSB_PAD	GENMASK(4, 0)
+#define DMA_ADAPTER_TX_NCHANS	GENMASK(6, 5)
+#define DMA_ADAPTER_RX_MSB_PAD	GENMASK(12, 8)
+#define DMA_ADAPTER_RX_NCHANS	GENMASK(14, 13)
+#define DMA_ADAPTER_NCHANS	GENMASK(22, 20)
+
+#define SWITCH_STRIDE	0x8000
+#define CLUSTER_STRIDE	0x4000
+
+#define MAX_NCLUSTERS	6
+
+#define APPLE_MCA_FMTBITS (SNDRV_PCM_FMTBIT_S16_LE | \
+			   SNDRV_PCM_FMTBIT_S24_LE | \
+			   SNDRV_PCM_FMTBIT_S32_LE)
+
+struct mca_cluster {
+	int no;
+	__iomem void *base;
+	struct mca_data *host;
+	struct device *pd_dev;
+	struct clk *clk_parent;
+	struct dma_chan *dma_chans[SNDRV_PCM_STREAM_LAST + 1];
+
+	bool port_started[SNDRV_PCM_STREAM_LAST + 1];
+	int port_driver; /* The cluster driving this cluster's port */
+
+	bool clocks_in_use[SNDRV_PCM_STREAM_LAST + 1];
+	struct device_link *pd_link;
+
+	unsigned int bclk_ratio;
+
+	/* Masks etc. picked up via the set_tdm_slot method */
+	int tdm_slots;
+	int tdm_slot_width;
+	unsigned int tdm_tx_mask;
+	unsigned int tdm_rx_mask;
+};
+
+struct mca_data {
+	struct device *dev;
+
+	__iomem void *switch_base;
+
+	struct device *pd_dev;
+	struct reset_control *rstc;
+	struct device_link *pd_link;
+
+	/* Mutex for accessing port_driver of foreign clusters */
+	struct mutex port_mutex;
+
+	int nclusters;
+	struct mca_cluster clusters[];
+};
+
+static void mca_modify(struct mca_cluster *cl, int regoffset, u32 mask, u32 val)
+{
+	__iomem void *ptr = cl->base + regoffset;
+	u32 newval;
+
+	newval = (val & mask) | (readl_relaxed(ptr) & ~mask);
+	writel_relaxed(newval, ptr);
+}
+
+/*
+ * Get the cluster of FE or BE DAI
+ */
+static struct mca_cluster *mca_dai_to_cluster(struct snd_soc_dai *dai)
+{
+	struct mca_data *mca = snd_soc_dai_get_drvdata(dai);
+	/*
+	 * FE DAIs are         0 ... nclusters - 1
+	 * BE DAIs are nclusters ... 2*nclusters - 1
+	 */
+	int cluster_no = dai->id % mca->nclusters;
+
+	return &mca->clusters[cluster_no];
+}
+
+/* called before PCM trigger */
+static void mca_fe_early_trigger(struct snd_pcm_substream *substream, int cmd,
+				 struct snd_soc_dai *dai)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+	bool is_tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
+	int serdes_unit = is_tx ? CLUSTER_TX_OFF : CLUSTER_RX_OFF;
+	int serdes_conf =
+		serdes_unit + (is_tx ? REG_TX_SERDES_CONF : REG_RX_SERDES_CONF);
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		mca_modify(cl, serdes_unit + REG_SERDES_STATUS,
+			   SERDES_STATUS_EN | SERDES_STATUS_RST,
+			   SERDES_STATUS_RST);
+		mca_modify(cl, serdes_conf, SERDES_CONF_SOME_RST,
+			   SERDES_CONF_SOME_RST);
+		readl_relaxed(cl->base + serdes_conf);
+		mca_modify(cl, serdes_conf, SERDES_STATUS_RST, 0);
+		WARN_ON(readl_relaxed(cl->base + REG_SERDES_STATUS) &
+			SERDES_STATUS_RST);
+		break;
+	default:
+		break;
+	}
+}
+
+static int mca_fe_trigger(struct snd_pcm_substream *substream, int cmd,
+			  struct snd_soc_dai *dai)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+	bool is_tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
+	int serdes_unit = is_tx ? CLUSTER_TX_OFF : CLUSTER_RX_OFF;
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		mca_modify(cl, serdes_unit + REG_SERDES_STATUS,
+			   SERDES_STATUS_EN | SERDES_STATUS_RST,
+			   SERDES_STATUS_EN);
+		break;
+
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		mca_modify(cl, serdes_unit + REG_SERDES_STATUS,
+			   SERDES_STATUS_EN, 0);
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int mca_fe_enable_clocks(struct mca_cluster *cl)
+{
+	struct mca_data *mca = cl->host;
+	int ret;
+
+	ret = clk_prepare_enable(cl->clk_parent);
+	if (ret) {
+		dev_err(mca->dev,
+			"cluster %d: unable to enable clock parent: %d\n",
+			cl->no, ret);
+		return ret;
+	}
+
+	/*
+	 * We can't power up the device earlier than this because
+	 * the power state driver would error out on seeing the device
+	 * as clock-gated.
+	 */
+	cl->pd_link = device_link_add(mca->dev, cl->pd_dev,
+				      DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME |
+					      DL_FLAG_RPM_ACTIVE);
+	if (!cl->pd_link) {
+		dev_err(mca->dev,
+			"cluster %d: unable to prop-up power domain\n", cl->no);
+		clk_disable_unprepare(cl->clk_parent);
+		return -EINVAL;
+	}
+
+	writel_relaxed(cl->no + 1, cl->base + REG_SYNCGEN_MCLK_SEL);
+	mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN,
+		   SYNCGEN_STATUS_EN);
+	mca_modify(cl, REG_STATUS, STATUS_MCLK_EN, STATUS_MCLK_EN);
+
+	return 0;
+}
+
+static void mca_fe_disable_clocks(struct mca_cluster *cl)
+{
+	mca_modify(cl, REG_SYNCGEN_STATUS, SYNCGEN_STATUS_EN, 0);
+	mca_modify(cl, REG_STATUS, STATUS_MCLK_EN, 0);
+
+	device_link_del(cl->pd_link);
+	clk_disable_unprepare(cl->clk_parent);
+}
+
+static bool mca_fe_clocks_in_use(struct mca_cluster *cl)
+{
+	struct mca_data *mca = cl->host;
+	struct mca_cluster *be_cl;
+	int stream, i;
+
+	mutex_lock(&mca->port_mutex);
+	for (i = 0; i < mca->nclusters; i++) {
+		be_cl = &mca->clusters[i];
+
+		if (be_cl->port_driver != cl->no)
+			continue;
+
+		for_each_pcm_streams(stream) {
+			if (be_cl->clocks_in_use[stream]) {
+				mutex_unlock(&mca->port_mutex);
+				return true;
+			}
+		}
+	}
+	mutex_unlock(&mca->port_mutex);
+	return false;
+}
+
+static int mca_be_prepare(struct snd_pcm_substream *substream,
+			  struct snd_soc_dai *dai)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+	struct mca_data *mca = cl->host;
+	struct mca_cluster *fe_cl;
+	int ret;
+
+	if (cl->port_driver < 0)
+		return -EINVAL;
+
+	fe_cl = &mca->clusters[cl->port_driver];
+
+	/*
+	 * Typically the CODECs we are paired with will require clocks
+	 * to be present at time of unmute with the 'mute_stream' op
+	 * or at time of DAPM widget power-up. We need to enable clocks
+	 * here at the latest (frontend prepare would be too late).
+	 */
+	if (!mca_fe_clocks_in_use(fe_cl)) {
+		ret = mca_fe_enable_clocks(fe_cl);
+		if (ret < 0)
+			return ret;
+	}
+
+	cl->clocks_in_use[substream->stream] = true;
+
+	return 0;
+}
+
+static int mca_be_hw_free(struct snd_pcm_substream *substream,
+			  struct snd_soc_dai *dai)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+	struct mca_data *mca = cl->host;
+	struct mca_cluster *fe_cl;
+
+	if (cl->port_driver < 0)
+		return -EINVAL;
+
+	/*
+	 * We are operating on a foreign cluster here, but since we
+	 * belong to the same PCM, accesses should have been
+	 * synchronized at ASoC level.
+	 */
+	fe_cl = &mca->clusters[cl->port_driver];
+	if (!mca_fe_clocks_in_use(fe_cl))
+		return 0; /* Nothing to do */
+
+	cl->clocks_in_use[substream->stream] = false;
+
+	if (!mca_fe_clocks_in_use(fe_cl))
+		mca_fe_disable_clocks(fe_cl);
+
+	return 0;
+}
+
+static unsigned int mca_crop_mask(unsigned int mask, int nchans)
+{
+	while (hweight32(mask) > nchans)
+		mask &= ~(1 << __fls(mask));
+
+	return mask;
+}
+
+static int mca_configure_serdes(struct mca_cluster *cl, int serdes_unit,
+				unsigned int mask, int slots, int nchans,
+				int slot_width, bool is_tx, int port)
+{
+	__iomem void *serdes_base = cl->base + serdes_unit;
+	u32 serdes_conf, serdes_conf_mask;
+
+	serdes_conf_mask = SERDES_CONF_WIDTH_MASK | SERDES_CONF_NCHANS;
+	serdes_conf = FIELD_PREP(SERDES_CONF_NCHANS, max(slots, 1) - 1);
+	switch (slot_width) {
+	case 16:
+		serdes_conf |= SERDES_CONF_WIDTH_16BIT;
+		break;
+	case 20:
+		serdes_conf |= SERDES_CONF_WIDTH_20BIT;
+		break;
+	case 24:
+		serdes_conf |= SERDES_CONF_WIDTH_24BIT;
+		break;
+	case 32:
+		serdes_conf |= SERDES_CONF_WIDTH_32BIT;
+		break;
+	default:
+		goto err;
+	}
+
+	serdes_conf_mask |= SERDES_CONF_SYNC_SEL;
+	serdes_conf |= FIELD_PREP(SERDES_CONF_SYNC_SEL, cl->no + 1);
+
+	if (is_tx) {
+		serdes_conf_mask |= SERDES_CONF_UNK1 | SERDES_CONF_UNK2 |
+				    SERDES_CONF_UNK3;
+		serdes_conf |= SERDES_CONF_UNK1 | SERDES_CONF_UNK2 |
+			       SERDES_CONF_UNK3;
+	} else {
+		serdes_conf_mask |= SERDES_CONF_UNK1 | SERDES_CONF_UNK2 |
+				    SERDES_CONF_UNK3 |
+				    SERDES_CONF_NO_DATA_FEEDBACK;
+		serdes_conf |= SERDES_CONF_UNK1 | SERDES_CONF_UNK2 |
+			       SERDES_CONF_NO_DATA_FEEDBACK;
+	}
+
+	mca_modify(cl,
+		   serdes_unit +
+			   (is_tx ? REG_TX_SERDES_CONF : REG_RX_SERDES_CONF),
+		   serdes_conf_mask, serdes_conf);
+
+	if (is_tx) {
+		writel_relaxed(0xffffffff,
+			       serdes_base + REG_TX_SERDES_SLOTMASK);
+		writel_relaxed(~((u32)mca_crop_mask(mask, nchans)),
+			       serdes_base + REG_TX_SERDES_SLOTMASK + 0x4);
+		writel_relaxed(0xffffffff,
+			       serdes_base + REG_TX_SERDES_SLOTMASK + 0x8);
+		writel_relaxed(~((u32)mask),
+			       serdes_base + REG_TX_SERDES_SLOTMASK + 0xc);
+	} else {
+		writel_relaxed(0xffffffff,
+			       serdes_base + REG_RX_SERDES_SLOTMASK);
+		writel_relaxed(~((u32)mca_crop_mask(mask, nchans)),
+			       serdes_base + REG_RX_SERDES_SLOTMASK + 0x4);
+		writel_relaxed(1 << port,
+			       serdes_base + REG_RX_SERDES_PORT);
+	}
+
+	return 0;
+
+err:
+	dev_err(cl->host->dev,
+		"unsupported SERDES configuration requested (mask=0x%x slots=%d slot_width=%d)\n",
+		mask, slots, slot_width);
+	return -EINVAL;
+}
+
+static int mca_fe_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
+			       unsigned int rx_mask, int slots, int slot_width)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+
+	cl->tdm_slots = slots;
+	cl->tdm_slot_width = slot_width;
+	cl->tdm_tx_mask = tx_mask;
+	cl->tdm_rx_mask = rx_mask;
+
+	return 0;
+}
+
+static int mca_fe_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+	struct mca_data *mca = cl->host;
+	bool fpol_inv = false;
+	u32 serdes_conf = 0;
+	u32 bitstart;
+
+	if ((fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) !=
+	    SND_SOC_DAIFMT_BP_FP)
+		goto err;
+
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		fpol_inv = 0;
+		bitstart = 1;
+		break;
+	case SND_SOC_DAIFMT_LEFT_J:
+		fpol_inv = 1;
+		bitstart = 0;
+		break;
+	default:
+		goto err;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
+	case SND_SOC_DAIFMT_NB_IF:
+	case SND_SOC_DAIFMT_IB_IF:
+		fpol_inv ^= 1;
+		break;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
+	case SND_SOC_DAIFMT_NB_NF:
+	case SND_SOC_DAIFMT_NB_IF:
+		serdes_conf |= SERDES_CONF_BCLK_POL;
+		break;
+	}
+
+	if (!fpol_inv)
+		goto err;
+
+	mca_modify(cl, CLUSTER_TX_OFF + REG_TX_SERDES_CONF,
+		   SERDES_CONF_BCLK_POL, serdes_conf);
+	mca_modify(cl, CLUSTER_RX_OFF + REG_RX_SERDES_CONF,
+		   SERDES_CONF_BCLK_POL, serdes_conf);
+	writel_relaxed(bitstart,
+		       cl->base + CLUSTER_TX_OFF + REG_TX_SERDES_BITSTART);
+	writel_relaxed(bitstart,
+		       cl->base + CLUSTER_RX_OFF + REG_RX_SERDES_BITSTART);
+
+	return 0;
+
+err:
+	dev_err(mca->dev, "unsupported DAI format (0x%x) requested\n", fmt);
+	return -EINVAL;
+}
+
+static int mca_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+
+	cl->bclk_ratio = ratio;
+
+	return 0;
+}
+
+static int mca_fe_get_port(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *fe = asoc_substream_to_rtd(substream);
+	struct snd_soc_pcm_runtime *be;
+	struct snd_soc_dpcm *dpcm;
+
+	be = NULL;
+	for_each_dpcm_be(fe, substream->stream, dpcm) {
+		be = dpcm->be;
+		break;
+	}
+
+	if (!be)
+		return -EINVAL;
+
+	return mca_dai_to_cluster(asoc_rtd_to_cpu(be, 0))->no;
+}
+
+static int mca_fe_hw_params(struct snd_pcm_substream *substream,
+			    struct snd_pcm_hw_params *params,
+			    struct snd_soc_dai *dai)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+	struct mca_data *mca = cl->host;
+	struct device *dev = mca->dev;
+	unsigned int samp_rate = params_rate(params);
+	bool is_tx = substream->stream == SNDRV_PCM_STREAM_PLAYBACK;
+	bool refine_tdm = false;
+	unsigned long bclk_ratio;
+	unsigned int tdm_slots, tdm_slot_width, tdm_mask;
+	u32 regval, pad;
+	int ret, port, nchans_ceiled;
+
+	if (!cl->tdm_slot_width) {
+		/*
+		 * We were not given TDM settings from above, set initial
+		 * guesses which will later be refined.
+		 */
+		tdm_slot_width = params_width(params);
+		tdm_slots = params_channels(params);
+		refine_tdm = true;
+	} else {
+		tdm_slot_width = cl->tdm_slot_width;
+		tdm_slots = cl->tdm_slots;
+		tdm_mask = is_tx ? cl->tdm_tx_mask : cl->tdm_rx_mask;
+	}
+
+	if (cl->bclk_ratio)
+		bclk_ratio = cl->bclk_ratio;
+	else
+		bclk_ratio = tdm_slot_width * tdm_slots;
+
+	if (refine_tdm) {
+		int nchannels = params_channels(params);
+
+		if (nchannels > 2) {
+			dev_err(dev, "missing TDM for stream with two or more channels\n");
+			return -EINVAL;
+		}
+
+		if ((bclk_ratio % nchannels) != 0) {
+			dev_err(dev, "BCLK ratio (%ld) not divisible by no. of channels (%d)\n",
+				bclk_ratio, nchannels);
+			return -EINVAL;
+		}
+
+		tdm_slot_width = bclk_ratio / nchannels;
+
+		if (tdm_slot_width > 32 && nchannels == 1)
+			tdm_slot_width = 32;
+
+		if (tdm_slot_width < params_width(params)) {
+			dev_err(dev, "TDM slots too narrow (tdm=%d params=%d)\n",
+				tdm_slot_width, params_width(params));
+			return -EINVAL;
+		}
+
+		tdm_mask = (1 << tdm_slots) - 1;
+	}
+
+	port = mca_fe_get_port(substream);
+	if (port < 0)
+		return port;
+
+	ret = mca_configure_serdes(cl, is_tx ? CLUSTER_TX_OFF : CLUSTER_RX_OFF,
+				   tdm_mask, tdm_slots, params_channels(params),
+				   tdm_slot_width, is_tx, port);
+	if (ret)
+		return ret;
+
+	pad = 32 - params_width(params);
+
+	/*
+	 * TODO: Here the register semantics aren't clear.
+	 */
+	nchans_ceiled = min_t(int, params_channels(params), 4);
+	regval = FIELD_PREP(DMA_ADAPTER_NCHANS, nchans_ceiled) |
+		 FIELD_PREP(DMA_ADAPTER_TX_NCHANS, 0x2) |
+		 FIELD_PREP(DMA_ADAPTER_RX_NCHANS, 0x2) |
+		 FIELD_PREP(DMA_ADAPTER_TX_LSB_PAD, pad) |
+		 FIELD_PREP(DMA_ADAPTER_RX_MSB_PAD, pad);
+
+#ifndef USE_RXB_FOR_CAPTURE
+	writel_relaxed(regval, mca->switch_base + REG_DMA_ADAPTER_A(cl->no));
+#else
+	if (is_tx)
+		writel_relaxed(regval,
+			       mca->switch_base + REG_DMA_ADAPTER_A(cl->no));
+	else
+		writel_relaxed(regval,
+			       mca->switch_base + REG_DMA_ADAPTER_B(cl->no));
+#endif
+
+	if (!mca_fe_clocks_in_use(cl)) {
+		/*
+		 * Set up FSYNC duty cycle as even as possible.
+		 */
+		writel_relaxed((bclk_ratio / 2) - 1,
+			       cl->base + REG_SYNCGEN_HI_PERIOD);
+		writel_relaxed(((bclk_ratio + 1) / 2) - 1,
+			       cl->base + REG_SYNCGEN_LO_PERIOD);
+		writel_relaxed(FIELD_PREP(MCLK_CONF_DIV, 0x1),
+			       cl->base + REG_MCLK_CONF);
+
+		ret = clk_set_rate(cl->clk_parent, bclk_ratio * samp_rate);
+		if (ret) {
+			dev_err(mca->dev, "cluster %d: unable to set clock parent: %d\n",
+				cl->no, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dai_ops mca_fe_ops = {
+	.set_fmt = mca_fe_set_fmt,
+	.set_bclk_ratio = mca_set_bclk_ratio,
+	.set_tdm_slot = mca_fe_set_tdm_slot,
+	.hw_params = mca_fe_hw_params,
+	.trigger = mca_fe_trigger,
+};
+
+static bool mca_be_started(struct mca_cluster *cl)
+{
+	int stream;
+
+	for_each_pcm_streams(stream)
+		if (cl->port_started[stream])
+			return true;
+	return false;
+}
+
+static int mca_be_startup(struct snd_pcm_substream *substream,
+			  struct snd_soc_dai *dai)
+{
+	struct snd_soc_pcm_runtime *be = asoc_substream_to_rtd(substream);
+	struct snd_soc_pcm_runtime *fe;
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+	struct mca_cluster *fe_cl;
+	struct mca_data *mca = cl->host;
+	struct snd_soc_dpcm *dpcm;
+
+	fe = NULL;
+
+	for_each_dpcm_fe(be, substream->stream, dpcm) {
+		if (fe && dpcm->fe != fe) {
+			dev_err(mca->dev, "many FE per one BE unsupported\n");
+			return -EINVAL;
+		}
+
+		fe = dpcm->fe;
+	}
+
+	if (!fe)
+		return -EINVAL;
+
+	fe_cl = mca_dai_to_cluster(asoc_rtd_to_cpu(fe, 0));
+
+	if (mca_be_started(cl)) {
+		/*
+		 * Port is already started in the other direction.
+		 * Make sure there isn't a conflict with another cluster
+		 * driving the port.
+		 */
+		if (cl->port_driver != fe_cl->no)
+			return -EINVAL;
+
+		cl->port_started[substream->stream] = true;
+		return 0;
+	}
+
+	writel_relaxed(PORT_ENABLES_CLOCKS | PORT_ENABLES_TX_DATA,
+		       cl->base + REG_PORT_ENABLES);
+	writel_relaxed(FIELD_PREP(PORT_CLOCK_SEL, fe_cl->no + 1),
+		       cl->base + REG_PORT_CLOCK_SEL);
+	writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no),
+		       cl->base + REG_PORT_DATA_SEL);
+	mutex_lock(&mca->port_mutex);
+	cl->port_driver = fe_cl->no;
+	mutex_unlock(&mca->port_mutex);
+	cl->port_started[substream->stream] = true;
+
+	return 0;
+}
+
+static void mca_be_shutdown(struct snd_pcm_substream *substream,
+			    struct snd_soc_dai *dai)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(dai);
+	struct mca_data *mca = cl->host;
+
+	cl->port_started[substream->stream] = false;
+
+	if (!mca_be_started(cl)) {
+		/*
+		 * Were we the last direction to shutdown?
+		 * Turn off the lights.
+		 */
+		writel_relaxed(0, cl->base + REG_PORT_ENABLES);
+		writel_relaxed(0, cl->base + REG_PORT_DATA_SEL);
+		mutex_lock(&mca->port_mutex);
+		cl->port_driver = -1;
+		mutex_unlock(&mca->port_mutex);
+	}
+}
+
+static const struct snd_soc_dai_ops mca_be_ops = {
+	.prepare = mca_be_prepare,
+	.hw_free = mca_be_hw_free,
+	.startup = mca_be_startup,
+	.shutdown = mca_be_shutdown,
+};
+
+static int mca_set_runtime_hwparams(struct snd_soc_component *component,
+				    struct snd_pcm_substream *substream,
+				    struct dma_chan *chan)
+{
+	struct device *dma_dev = chan->device->dev;
+	struct snd_dmaengine_dai_dma_data dma_data = {};
+	int ret;
+
+	struct snd_pcm_hardware hw;
+
+	memset(&hw, 0, sizeof(hw));
+
+	hw.info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID |
+		  SNDRV_PCM_INFO_INTERLEAVED;
+	hw.periods_min = 2;
+	hw.periods_max = UINT_MAX;
+	hw.period_bytes_min = 256;
+	hw.period_bytes_max = dma_get_max_seg_size(dma_dev);
+	hw.buffer_bytes_max = SIZE_MAX;
+	hw.fifo_size = 16;
+
+	ret = snd_dmaengine_pcm_refine_runtime_hwparams(substream, &dma_data,
+							&hw, chan);
+
+	if (ret)
+		return ret;
+
+	return snd_soc_set_runtime_hwparams(substream, &hw);
+}
+
+static int mca_pcm_open(struct snd_soc_component *component,
+			struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
+	struct mca_cluster *cl = mca_dai_to_cluster(asoc_rtd_to_cpu(rtd, 0));
+	struct dma_chan *chan = cl->dma_chans[substream->stream];
+	int ret;
+
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	ret = mca_set_runtime_hwparams(component, substream, chan);
+	if (ret)
+		return ret;
+
+	return snd_dmaengine_pcm_open(substream, chan);
+}
+
+static int mca_hw_params(struct snd_soc_component *component,
+			 struct snd_pcm_substream *substream,
+			 struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
+	struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream);
+	struct dma_slave_config slave_config;
+	int ret;
+
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	memset(&slave_config, 0, sizeof(slave_config));
+	ret = snd_hwparams_to_dma_slave_config(substream, params,
+					       &slave_config);
+	if (ret < 0)
+		return ret;
+
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		slave_config.dst_port_window_size =
+			min_t(u32, params_channels(params), 4);
+	else
+		slave_config.src_port_window_size =
+			min_t(u32, params_channels(params), 4);
+
+	return dmaengine_slave_config(chan, &slave_config);
+}
+
+static int mca_close(struct snd_soc_component *component,
+		     struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
+
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	return snd_dmaengine_pcm_close(substream);
+}
+
+static int mca_trigger(struct snd_soc_component *component,
+		       struct snd_pcm_substream *substream, int cmd)
+{
+	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
+
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	/*
+	 * Before we do the PCM trigger proper, insert an opportunity
+	 * to reset the frontend's SERDES.
+	 */
+	mca_fe_early_trigger(substream, cmd, asoc_rtd_to_cpu(rtd, 0));
+
+	return snd_dmaengine_pcm_trigger(substream, cmd);
+}
+
+static snd_pcm_uframes_t mca_pointer(struct snd_soc_component *component,
+				     struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
+
+	if (rtd->dai_link->no_pcm)
+		return -ENOTSUPP;
+
+	return snd_dmaengine_pcm_pointer(substream);
+}
+
+static struct dma_chan *mca_request_dma_channel(struct mca_cluster *cl, unsigned int stream)
+{
+	bool is_tx = (stream == SNDRV_PCM_STREAM_PLAYBACK);
+#ifndef USE_RXB_FOR_CAPTURE
+	char *name = devm_kasprintf(cl->host->dev, GFP_KERNEL,
+				    is_tx ? "tx%da" : "rx%da", cl->no);
+#else
+	char *name = devm_kasprintf(cl->host->dev, GFP_KERNEL,
+				    is_tx ? "tx%da" : "rx%db", cl->no);
+#endif
+	return of_dma_request_slave_channel(cl->host->dev->of_node, name);
+
+}
+
+static void mca_pcm_free(struct snd_soc_component *component,
+			 struct snd_pcm *pcm)
+{
+	struct snd_soc_pcm_runtime *rtd = snd_pcm_chip(pcm);
+	struct mca_cluster *cl = mca_dai_to_cluster(asoc_rtd_to_cpu(rtd, 0));
+	unsigned int i;
+
+	if (rtd->dai_link->no_pcm)
+		return;
+
+	for_each_pcm_streams(i) {
+		struct snd_pcm_substream *substream =
+			rtd->pcm->streams[i].substream;
+
+		if (!substream || !cl->dma_chans[i])
+			continue;
+
+		dma_release_channel(cl->dma_chans[i]);
+		cl->dma_chans[i] = NULL;
+	}
+}
+
+
+static int mca_pcm_new(struct snd_soc_component *component,
+		       struct snd_soc_pcm_runtime *rtd)
+{
+	struct mca_cluster *cl = mca_dai_to_cluster(asoc_rtd_to_cpu(rtd, 0));
+	unsigned int i;
+
+	if (rtd->dai_link->no_pcm)
+		return 0;
+
+	for_each_pcm_streams(i) {
+		struct snd_pcm_substream *substream =
+			rtd->pcm->streams[i].substream;
+		struct dma_chan *chan;
+
+		if (!substream)
+			continue;
+
+		chan = mca_request_dma_channel(cl, i);
+
+		if (IS_ERR_OR_NULL(chan)) {
+			dev_err(component->dev, "unable to obtain DMA channel (stream %d cluster %d): %pe\n",
+				i, cl->no, chan);
+			mca_pcm_free(component, rtd->pcm);
+			return -EINVAL;
+		}
+
+		cl->dma_chans[i] = chan;
+		snd_pcm_set_managed_buffer(substream, SNDRV_DMA_TYPE_DEV_IRAM,
+					   chan->device->dev, 512 * 1024 * 6,
+					   SIZE_MAX);
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_component_driver mca_component = {
+	.name = "apple-mca",
+	.open = mca_pcm_open,
+	.close = mca_close,
+	.hw_params = mca_hw_params,
+	.trigger = mca_trigger,
+	.pointer = mca_pointer,
+	.pcm_construct = mca_pcm_new,
+	.pcm_destruct = mca_pcm_free,
+};
+
+static void apple_mca_release(struct mca_data *mca)
+{
+	int i;
+
+	for (i = 0; i < mca->nclusters; i++) {
+		struct mca_cluster *cl = &mca->clusters[i];
+
+		if (!IS_ERR_OR_NULL(cl->clk_parent))
+			clk_put(cl->clk_parent);
+
+		if (!IS_ERR_OR_NULL(cl->pd_dev))
+			dev_pm_domain_detach(cl->pd_dev, true);
+	}
+
+	if (mca->pd_link)
+		device_link_del(mca->pd_link);
+
+	if (!IS_ERR_OR_NULL(mca->pd_dev))
+		dev_pm_domain_detach(mca->pd_dev, true);
+
+	reset_control_rearm(mca->rstc);
+}
+
+static int apple_mca_probe(struct platform_device *pdev)
+{
+	struct mca_data *mca;
+	struct mca_cluster *clusters;
+	struct snd_soc_dai_driver *dai_drivers;
+	struct resource *res;
+	void __iomem *base;
+	int nclusters;
+	int ret, i;
+
+	base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	if (resource_size(res) < CLUSTER_STRIDE)
+		return -EINVAL;
+	nclusters = (resource_size(res) - CLUSTER_STRIDE) / CLUSTER_STRIDE + 1;
+
+	mca = devm_kzalloc(&pdev->dev, struct_size(mca, clusters, nclusters),
+			   GFP_KERNEL);
+	if (!mca)
+		return -ENOMEM;
+	mca->dev = &pdev->dev;
+	mca->nclusters = nclusters;
+	mutex_init(&mca->port_mutex);
+	platform_set_drvdata(pdev, mca);
+	clusters = mca->clusters;
+
+	mca->switch_base =
+		devm_platform_ioremap_resource(pdev, 1);
+	if (IS_ERR(mca->switch_base))
+		return PTR_ERR(mca->switch_base);
+
+	mca->rstc = devm_reset_control_get_optional_shared(&pdev->dev, NULL);
+	if (IS_ERR(mca->rstc))
+		return PTR_ERR(mca->rstc);
+
+	dai_drivers = devm_kzalloc(
+		&pdev->dev, sizeof(*dai_drivers) * 2 * nclusters, GFP_KERNEL);
+	if (!dai_drivers)
+		return -ENOMEM;
+
+	mca->pd_dev = dev_pm_domain_attach_by_id(&pdev->dev, 0);
+	if (IS_ERR(mca->pd_dev))
+		return -EINVAL;
+
+	mca->pd_link = device_link_add(&pdev->dev, mca->pd_dev,
+				       DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME |
+					       DL_FLAG_RPM_ACTIVE);
+	if (!mca->pd_link) {
+		ret = -EINVAL;
+		/* Prevent an unbalanced reset rearm */
+		mca->rstc = NULL;
+		goto err_release;
+	}
+
+	reset_control_reset(mca->rstc);
+
+	for (i = 0; i < nclusters; i++) {
+		struct mca_cluster *cl = &clusters[i];
+		struct snd_soc_dai_driver *fe =
+			&dai_drivers[mca->nclusters + i];
+		struct snd_soc_dai_driver *be = &dai_drivers[i];
+
+		cl->host = mca;
+		cl->no = i;
+		cl->base = base + CLUSTER_STRIDE * i;
+		cl->port_driver = -1;
+		cl->clk_parent = of_clk_get(pdev->dev.of_node, i);
+		if (IS_ERR(cl->clk_parent)) {
+			dev_err(&pdev->dev, "unable to obtain clock %d: %ld\n",
+				i, PTR_ERR(cl->clk_parent));
+			ret = PTR_ERR(cl->clk_parent);
+			goto err_release;
+		}
+		cl->pd_dev = dev_pm_domain_attach_by_id(&pdev->dev, i + 1);
+		if (IS_ERR(cl->pd_dev)) {
+			dev_err(&pdev->dev,
+				"unable to obtain cluster %d PD: %ld\n", i,
+				PTR_ERR(cl->pd_dev));
+			ret = PTR_ERR(cl->pd_dev);
+			goto err_release;
+		}
+
+		fe->id = i;
+		fe->name =
+			devm_kasprintf(&pdev->dev, GFP_KERNEL, "mca-pcm-%d", i);
+		if (!fe->name) {
+			ret = -ENOMEM;
+			goto err_release;
+		}
+		fe->ops = &mca_fe_ops;
+		fe->playback.channels_min = 1;
+		fe->playback.channels_max = 32;
+		fe->playback.rates = SNDRV_PCM_RATE_8000_192000;
+		fe->playback.formats = APPLE_MCA_FMTBITS;
+		fe->capture.channels_min = 1;
+		fe->capture.channels_max = 32;
+		fe->capture.rates = SNDRV_PCM_RATE_8000_192000;
+		fe->capture.formats = APPLE_MCA_FMTBITS;
+		fe->symmetric_rate = 1;
+
+		fe->playback.stream_name =
+			devm_kasprintf(&pdev->dev, GFP_KERNEL, "PCM%d TX", i);
+		fe->capture.stream_name =
+			devm_kasprintf(&pdev->dev, GFP_KERNEL, "PCM%d RX", i);
+
+		if (!fe->playback.stream_name || !fe->capture.stream_name) {
+			ret = -ENOMEM;
+			goto err_release;
+		}
+
+		be->id = i + nclusters;
+		be->name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "mca-i2s-%d", i);
+		if (!be->name) {
+			ret = -ENOMEM;
+			goto err_release;
+		}
+		be->ops = &mca_be_ops;
+		be->playback.channels_min = 1;
+		be->playback.channels_max = 32;
+		be->playback.rates = SNDRV_PCM_RATE_8000_192000;
+		be->playback.formats = APPLE_MCA_FMTBITS;
+		be->capture.channels_min = 1;
+		be->capture.channels_max = 32;
+		be->capture.rates = SNDRV_PCM_RATE_8000_192000;
+		be->capture.formats = APPLE_MCA_FMTBITS;
+
+		be->playback.stream_name =
+			devm_kasprintf(&pdev->dev, GFP_KERNEL, "I2S%d TX", i);
+		be->capture.stream_name =
+			devm_kasprintf(&pdev->dev, GFP_KERNEL, "I2S%d RX", i);
+		if (!be->playback.stream_name || !be->capture.stream_name) {
+			ret = -ENOMEM;
+			goto err_release;
+		}
+	}
+
+	ret = snd_soc_register_component(&pdev->dev, &mca_component,
+					 dai_drivers, nclusters * 2);
+	if (ret) {
+		dev_err(&pdev->dev, "unable to register ASoC component: %d\n",
+			ret);
+		goto err_release;
+	}
+
+	return 0;
+
+err_release:
+	apple_mca_release(mca);
+	return ret;
+}
+
+static int apple_mca_remove(struct platform_device *pdev)
+{
+	struct mca_data *mca = platform_get_drvdata(pdev);
+
+	snd_soc_unregister_component(&pdev->dev);
+	apple_mca_release(mca);
+	return 0;
+}
+
+static const struct of_device_id apple_mca_of_match[] = {
+	{ .compatible = "apple,mca", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, apple_mca_of_match);
+
+static struct platform_driver apple_mca_driver = {
+	.driver = {
+		.name = "apple-mca",
+		.of_match_table = apple_mca_of_match,
+	},
+	.probe = apple_mca_probe,
+	.remove = apple_mca_remove,
+};
+module_platform_driver(apple_mca_driver);
+
+MODULE_AUTHOR("Martin Povišer <povik+lin@cutebit.org>");
+MODULE_DESCRIPTION("ASoC Apple MCA driver");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/atmel/atmel_ssc_dai.c b/sound/soc/atmel/atmel_ssc_dai.c
index e868b7e028d6..3763454436c1 100644
--- a/sound/soc/atmel/atmel_ssc_dai.c
+++ b/sound/soc/atmel/atmel_ssc_dai.c
@@ -891,7 +891,6 @@ static int asoc_ssc_init(struct device *dev)
 int atmel_ssc_set_audio(int ssc_id)
 {
 	struct ssc_device *ssc;
-	int ret;
 
 	/* If we can grab the SSC briefly to parent the DAI device off it */
 	ssc = ssc_request(ssc_id);
@@ -903,9 +902,7 @@ int atmel_ssc_set_audio(int ssc_id)
 		ssc_info[ssc_id].ssc = ssc;
 	}
 
-	ret = asoc_ssc_init(&ssc->pdev->dev);
-
-	return ret;
+	return asoc_ssc_init(&ssc->pdev->dev);
 }
 EXPORT_SYMBOL_GPL(atmel_ssc_set_audio);
 
diff --git a/sound/soc/atmel/sam9g20_wm8731.c b/sound/soc/atmel/sam9g20_wm8731.c
index 4d25fb61c652..1430642c8433 100644
--- a/sound/soc/atmel/sam9g20_wm8731.c
+++ b/sound/soc/atmel/sam9g20_wm8731.c
@@ -172,7 +172,7 @@ static int at91sam9g20ek_audio_probe(struct platform_device *pdev)
 	ret = snd_soc_register_card(card);
 	if (ret) {
 		dev_err_probe(&pdev->dev, ret,
-			      "snd_soc_register_card() failed: %d\n", ret);
+			      "snd_soc_register_card() failed\n");
 		goto err;
 	}
 
diff --git a/sound/soc/bcm/bcm2835-i2s.c b/sound/soc/bcm/bcm2835-i2s.c
index f4d84774dac7..85f705afcdbb 100644
--- a/sound/soc/bcm/bcm2835-i2s.c
+++ b/sound/soc/bcm/bcm2835-i2s.c
@@ -841,14 +841,9 @@ static int bcm2835_i2s_probe(struct platform_device *pdev)
 	/* get the clock */
 	dev->clk_prepared = false;
 	dev->clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(dev->clk)) {
-		ret = PTR_ERR(dev->clk);
-		if (ret == -EPROBE_DEFER)
-			dev_dbg(&pdev->dev, "could not get clk: %d\n", ret);
-		else
-			dev_err(&pdev->dev, "could not get clk: %d\n", ret);
-		return ret;
-	}
+	if (IS_ERR(dev->clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk),
+				     "could not get clk\n");
 
 	/* Request ioarea */
 	base = devm_platform_ioremap_resource(pdev, 0);
diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig
index d16b4efb88a7..e3b90c425faf 100644
--- a/sound/soc/codecs/Kconfig
+++ b/sound/soc/codecs/Kconfig
@@ -98,6 +98,7 @@ config SND_SOC_ALL_CODECS
 	imply SND_SOC_DA9055
 	imply SND_SOC_DMIC
 	imply SND_SOC_ES8316
+	imply SND_SOC_ES8326
 	imply SND_SOC_ES8328_SPI
 	imply SND_SOC_ES8328_I2C
 	imply SND_SOC_ES7134
@@ -205,6 +206,7 @@ config SND_SOC_ALL_CODECS
 	imply SND_SOC_SIMPLE_AMPLIFIER
 	imply SND_SOC_SIMPLE_MUX
 	imply SND_SOC_SPDIF
+	imply SND_SOC_SRC4XXX_I2C
 	imply SND_SOC_SSM2305
 	imply SND_SOC_SSM2518
 	imply SND_SOC_SSM2602_SPI
@@ -608,7 +610,7 @@ config SND_SOC_BT_SCO
 
 config SND_SOC_CPCAP
 	tristate "Motorola CPCAP codec"
-	depends on MFD_CPCAP
+	depends on MFD_CPCAP || COMPILE_TEST
 
 config SND_SOC_CQ0093VC
 	tristate
@@ -690,9 +692,15 @@ config SND_SOC_CS35L45_I2C
 	  Enable support for Cirrus Logic CS35L45 smart speaker amplifier
 	  with I2C control.
 
+config SND_SOC_CS42L42_CORE
+	tristate
+
 config SND_SOC_CS42L42
-	tristate "Cirrus Logic CS42L42 CODEC"
+	tristate "Cirrus Logic CS42L42 CODEC (I2C)"
 	depends on I2C
+	select REGMAP
+	select REGMAP_I2C
+	select SND_SOC_CS42L42_CORE
 
 config SND_SOC_CS42L51
 	tristate
@@ -714,6 +722,13 @@ config SND_SOC_CS42L73
 	tristate "Cirrus Logic CS42L73 CODEC"
 	depends on I2C
 
+config SND_SOC_CS42L83
+	tristate "Cirrus Logic CS42L83 CODEC"
+	depends on I2C
+	select REGMAP
+	select REGMAP_I2C
+	select SND_SOC_CS42L42_CORE
+
 config SND_SOC_CS4234
 	tristate "Cirrus Logic CS4234 CODEC"
 	depends on I2C
@@ -913,6 +928,10 @@ config SND_SOC_ES8316
 	tristate "Everest Semi ES8316 CODEC"
 	depends on I2C
 
+config SND_SOC_ES8326
+	tristate "Everest Semi ES8326 CODEC"
+	depends on I2C
+
 config SND_SOC_ES8328
 	tristate
 
@@ -966,7 +985,7 @@ config SND_SOC_LM49453
 
 config SND_SOC_LOCHNAGAR_SC
 	tristate "Lochnagar Sound Card"
-	depends on MFD_LOCHNAGAR
+	depends on MFD_LOCHNAGAR || COMPILE_TEST
 	help
 	  This driver support the sound card functionality of the Cirrus
 	  Logic Lochnagar audio development board.
@@ -1191,8 +1210,7 @@ config SND_SOC_RK3328
 
 config SND_SOC_RK817
 	tristate "Rockchip RK817 audio CODEC"
-	depends on MFD_RK808
-	select REGMAP_I2C
+	depends on MFD_RK808 || COMPILE_TEST
 
 config SND_SOC_RL6231
 	tristate
@@ -1471,6 +1489,18 @@ config SND_SOC_SIMPLE_MUX
 config SND_SOC_SPDIF
 	tristate "S/PDIF CODEC"
 
+config SND_SOC_SRC4XXX_I2C
+	tristate "Texas Instruments SRC4XXX DIR/DIT and SRC codecs"
+	depends on I2C
+	select SND_SOC_SRC4XXX
+	help
+	  Enable support for the TI SRC4XXX family of codecs. These include the
+	  scr4392 which has digital receivers, transmitters, and
+	  a sample rate converter, including numerous ports.
+
+config SND_SOC_SRC4XXX
+	tristate
+
 config SND_SOC_SSM2305
 	tristate "Analog Devices SSM2305 Class-D Amplifier"
 	help
@@ -1726,8 +1756,10 @@ config SND_SOC_WCD_MBHC
 config SND_SOC_WCD934X
 	tristate "WCD9340/WCD9341 Codec"
 	depends on COMMON_CLK
+	depends on SLIMBUS
+	select REGMAP_SLIMBUS
 	select SND_SOC_WCD_MBHC
-	depends on MFD_WCD934X
+	depends on MFD_WCD934X || COMPILE_TEST
 	help
 	  The WCD9340/9341 is a audio codec IC Integrated in
 	  Qualcomm SoCs like SDM845.
diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile
index 92fd441d426a..9170ee1447dd 100644
--- a/sound/soc/codecs/Makefile
+++ b/sound/soc/codecs/Makefile
@@ -65,11 +65,13 @@ snd-soc-cs35l45-objs := cs35l45.o
 snd-soc-cs35l45-spi-objs := cs35l45-spi.o
 snd-soc-cs35l45-i2c-objs := cs35l45-i2c.o
 snd-soc-cs42l42-objs := cs42l42.o
+snd-soc-cs42l42-i2c-objs := cs42l42-i2c.o
 snd-soc-cs42l51-objs := cs42l51.o
 snd-soc-cs42l51-i2c-objs := cs42l51-i2c.o
 snd-soc-cs42l52-objs := cs42l52.o
 snd-soc-cs42l56-objs := cs42l56.o
 snd-soc-cs42l73-objs := cs42l73.o
+snd-soc-cs42l83-i2c-objs := cs42l83-i2c.o
 snd-soc-cs4234-objs := cs4234.o
 snd-soc-cs4265-objs := cs4265.o
 snd-soc-cs4270-objs := cs4270.o
@@ -100,6 +102,7 @@ snd-soc-dmic-objs := dmic.o
 snd-soc-es7134-objs := es7134.o
 snd-soc-es7241-objs := es7241.o
 snd-soc-es8316-objs := es8316.o
+snd-soc-es8326-objs := es8326.o
 snd-soc-es8328-objs := es8328.o
 snd-soc-es8328-i2c-objs := es8328-i2c.o
 snd-soc-es8328-spi-objs := es8328-spi.o
@@ -231,6 +234,8 @@ snd-soc-sigmadsp-regmap-objs := sigmadsp-regmap.o
 snd-soc-si476x-objs := si476x.o
 snd-soc-spdif-tx-objs := spdif_transmitter.o
 snd-soc-spdif-rx-objs := spdif_receiver.o
+snd-soc-src4xxx-objs := src4xxx.o
+snd-soc-src4xxx-i2c-objs := src4xxx-i2c.o
 snd-soc-ssm2305-objs := ssm2305.o
 snd-soc-ssm2518-objs := ssm2518.o
 snd-soc-ssm2602-objs := ssm2602.o
@@ -419,12 +424,14 @@ obj-$(CONFIG_SND_SOC_CS35L45_TABLES)	+= snd-soc-cs35l45-tables.o
 obj-$(CONFIG_SND_SOC_CS35L45)	+= snd-soc-cs35l45.o
 obj-$(CONFIG_SND_SOC_CS35L45_SPI)	+= snd-soc-cs35l45-spi.o
 obj-$(CONFIG_SND_SOC_CS35L45_I2C)	+= snd-soc-cs35l45-i2c.o
-obj-$(CONFIG_SND_SOC_CS42L42)	+= snd-soc-cs42l42.o
+obj-$(CONFIG_SND_SOC_CS42L42_CORE)	+= snd-soc-cs42l42.o
+obj-$(CONFIG_SND_SOC_CS42L42)	+= snd-soc-cs42l42-i2c.o
 obj-$(CONFIG_SND_SOC_CS42L51)	+= snd-soc-cs42l51.o
 obj-$(CONFIG_SND_SOC_CS42L51_I2C)	+= snd-soc-cs42l51-i2c.o
 obj-$(CONFIG_SND_SOC_CS42L52)	+= snd-soc-cs42l52.o
 obj-$(CONFIG_SND_SOC_CS42L56)	+= snd-soc-cs42l56.o
 obj-$(CONFIG_SND_SOC_CS42L73)	+= snd-soc-cs42l73.o
+obj-$(CONFIG_SND_SOC_CS42L83)	+= snd-soc-cs42l83-i2c.o
 obj-$(CONFIG_SND_SOC_CS4234)	+= snd-soc-cs4234.o
 obj-$(CONFIG_SND_SOC_CS4265)	+= snd-soc-cs4265.o
 obj-$(CONFIG_SND_SOC_CS4270)	+= snd-soc-cs4270.o
@@ -455,6 +462,7 @@ obj-$(CONFIG_SND_SOC_DMIC)	+= snd-soc-dmic.o
 obj-$(CONFIG_SND_SOC_ES7134)	+= snd-soc-es7134.o
 obj-$(CONFIG_SND_SOC_ES7241)	+= snd-soc-es7241.o
 obj-$(CONFIG_SND_SOC_ES8316)    += snd-soc-es8316.o
+obj-$(CONFIG_SND_SOC_ES8326)    += snd-soc-es8326.o
 obj-$(CONFIG_SND_SOC_ES8328)	+= snd-soc-es8328.o
 obj-$(CONFIG_SND_SOC_ES8328_I2C)+= snd-soc-es8328-i2c.o
 obj-$(CONFIG_SND_SOC_ES8328_SPI)+= snd-soc-es8328-spi.o
@@ -579,6 +587,8 @@ obj-$(CONFIG_SND_SOC_SIGMADSP_I2C)	+= snd-soc-sigmadsp-i2c.o
 obj-$(CONFIG_SND_SOC_SIGMADSP_REGMAP)	+= snd-soc-sigmadsp-regmap.o
 obj-$(CONFIG_SND_SOC_SI476X)	+= snd-soc-si476x.o
 obj-$(CONFIG_SND_SOC_SPDIF)	+= snd-soc-spdif-rx.o snd-soc-spdif-tx.o
+obj-$(CONFIG_SND_SOC_SRC4XXX)	+= snd-soc-src4xxx.o
+obj-$(CONFIG_SND_SOC_SRC4XXX_I2C)	+= snd-soc-src4xxx-i2c.o
 obj-$(CONFIG_SND_SOC_SSM2305)	+= snd-soc-ssm2305.o
 obj-$(CONFIG_SND_SOC_SSM2518)	+= snd-soc-ssm2518.o
 obj-$(CONFIG_SND_SOC_SSM2602)	+= snd-soc-ssm2602.o
diff --git a/sound/soc/codecs/adau1761-i2c.c b/sound/soc/codecs/adau1761-i2c.c
index 0683caf86aea..0cefff49569c 100644
--- a/sound/soc/codecs/adau1761-i2c.c
+++ b/sound/soc/codecs/adau1761-i2c.c
@@ -30,10 +30,9 @@ static int adau1761_i2c_probe(struct i2c_client *client)
 		id->driver_data, NULL);
 }
 
-static int adau1761_i2c_remove(struct i2c_client *client)
+static void adau1761_i2c_remove(struct i2c_client *client)
 {
 	adau17x1_remove(&client->dev);
-	return 0;
 }
 
 static const struct i2c_device_id adau1761_i2c_ids[] = {
diff --git a/sound/soc/codecs/adau1781-i2c.c b/sound/soc/codecs/adau1781-i2c.c
index e046de0ebcc7..39021b8cfb62 100644
--- a/sound/soc/codecs/adau1781-i2c.c
+++ b/sound/soc/codecs/adau1781-i2c.c
@@ -30,10 +30,9 @@ static int adau1781_i2c_probe(struct i2c_client *client)
 		id->driver_data, NULL);
 }
 
-static int adau1781_i2c_remove(struct i2c_client *client)
+static void adau1781_i2c_remove(struct i2c_client *client)
 {
 	adau17x1_remove(&client->dev);
-	return 0;
 }
 
 static const struct i2c_device_id adau1781_i2c_ids[] = {
diff --git a/sound/soc/codecs/ak4375.c b/sound/soc/codecs/ak4375.c
index 1ed004ba7cd2..573389e402f8 100644
--- a/sound/soc/codecs/ak4375.c
+++ b/sound/soc/codecs/ak4375.c
@@ -580,11 +580,9 @@ static int ak4375_i2c_probe(struct i2c_client *i2c)
 	return 0;
 }
 
-static int ak4375_i2c_remove(struct i2c_client *i2c)
+static void ak4375_i2c_remove(struct i2c_client *i2c)
 {
 	pm_runtime_disable(&i2c->dev);
-
-	return 0;
 }
 
 static const struct of_device_id ak4375_of_match[] = {
diff --git a/sound/soc/codecs/ak4458.c b/sound/soc/codecs/ak4458.c
index ea33cc83c86c..1db73552c746 100644
--- a/sound/soc/codecs/ak4458.c
+++ b/sound/soc/codecs/ak4458.c
@@ -447,6 +447,13 @@ static int ak4458_hw_params(struct snd_pcm_substream *substream,
 	snd_soc_component_update_bits(component, AK4458_0B_CONTROL7,
 				      AK4458_DCHAIN_MASK, dchn);
 
+	if (ak4458->drvdata->type == AK4497) {
+		ret = snd_soc_component_update_bits(component, AK4458_09_DSD2,
+						    0x4, (ak4458->dsd_path << 2));
+		if (ret < 0)
+			return ret;
+	}
+
 	ret = ak4458_rstn_control(component, 0);
 	if (ret)
 		return ret;
@@ -629,48 +636,6 @@ static void ak4458_reset(struct ak4458_priv *ak4458, bool active)
 	}
 }
 
-static int ak4458_init(struct snd_soc_component *component)
-{
-	struct ak4458_priv *ak4458 = snd_soc_component_get_drvdata(component);
-	int ret;
-
-	/* External Mute ON */
-	if (ak4458->mute_gpiod)
-		gpiod_set_value_cansleep(ak4458->mute_gpiod, 1);
-
-	ak4458_reset(ak4458, false);
-
-	ret = snd_soc_component_update_bits(component, AK4458_00_CONTROL1,
-			    0x80, 0x80);   /* ACKS bit = 1; 10000000 */
-	if (ret < 0)
-		return ret;
-
-	if (ak4458->drvdata->type == AK4497) {
-		ret = snd_soc_component_update_bits(component, AK4458_09_DSD2,
-						    0x4, (ak4458->dsd_path << 2));
-		if (ret < 0)
-			return ret;
-	}
-
-	return ak4458_rstn_control(component, 1);
-}
-
-static int ak4458_probe(struct snd_soc_component *component)
-{
-	struct ak4458_priv *ak4458 = snd_soc_component_get_drvdata(component);
-
-	ak4458->fs = 48000;
-
-	return ak4458_init(component);
-}
-
-static void ak4458_remove(struct snd_soc_component *component)
-{
-	struct ak4458_priv *ak4458 = snd_soc_component_get_drvdata(component);
-
-	ak4458_reset(ak4458, true);
-}
-
 #ifdef CONFIG_PM
 static int __maybe_unused ak4458_runtime_suspend(struct device *dev)
 {
@@ -714,8 +679,6 @@ static int __maybe_unused ak4458_runtime_resume(struct device *dev)
 #endif /* CONFIG_PM */
 
 static const struct snd_soc_component_driver soc_codec_dev_ak4458 = {
-	.probe			= ak4458_probe,
-	.remove			= ak4458_remove,
 	.controls		= ak4458_snd_controls,
 	.num_controls		= ARRAY_SIZE(ak4458_snd_controls),
 	.dapm_widgets		= ak4458_dapm_widgets,
@@ -728,8 +691,6 @@ static const struct snd_soc_component_driver soc_codec_dev_ak4458 = {
 };
 
 static const struct snd_soc_component_driver soc_codec_dev_ak4497 = {
-	.probe			= ak4458_probe,
-	.remove			= ak4458_remove,
 	.controls		= ak4497_snd_controls,
 	.num_controls		= ARRAY_SIZE(ak4497_snd_controls),
 	.dapm_widgets		= ak4497_dapm_widgets,
@@ -820,15 +781,17 @@ static int ak4458_i2c_probe(struct i2c_client *i2c)
 
 	pm_runtime_enable(&i2c->dev);
 	regcache_cache_only(ak4458->regmap, true);
+	ak4458_reset(ak4458, false);
 
 	return 0;
 }
 
-static int ak4458_i2c_remove(struct i2c_client *i2c)
+static void ak4458_i2c_remove(struct i2c_client *i2c)
 {
-	pm_runtime_disable(&i2c->dev);
+	struct ak4458_priv *ak4458 = i2c_get_clientdata(i2c);
 
-	return 0;
+	ak4458_reset(ak4458, true);
+	pm_runtime_disable(&i2c->dev);
 }
 
 static const struct of_device_id ak4458_of_match[] = {
diff --git a/sound/soc/codecs/ak4641.c b/sound/soc/codecs/ak4641.c
index 88851e94b045..0d3ee195b3cc 100644
--- a/sound/soc/codecs/ak4641.c
+++ b/sound/soc/codecs/ak4641.c
@@ -604,7 +604,7 @@ err_out:
 	return ret;
 }
 
-static int ak4641_i2c_remove(struct i2c_client *i2c)
+static void ak4641_i2c_remove(struct i2c_client *i2c)
 {
 	struct ak4641_platform_data *pdata = i2c->dev.platform_data;
 
@@ -616,8 +616,6 @@ static int ak4641_i2c_remove(struct i2c_client *i2c)
 		if (gpio_is_valid(pdata->gpio_npdn))
 			gpio_free(pdata->gpio_npdn);
 	}
-
-	return 0;
 }
 
 static const struct i2c_device_id ak4641_i2c_id[] = {
diff --git a/sound/soc/codecs/ak5558.c b/sound/soc/codecs/ak5558.c
index 887d2c04d647..60abcffe6a0c 100644
--- a/sound/soc/codecs/ak5558.c
+++ b/sound/soc/codecs/ak5558.c
@@ -479,11 +479,9 @@ static int ak5558_i2c_probe(struct i2c_client *i2c)
 	return 0;
 }
 
-static int ak5558_i2c_remove(struct i2c_client *i2c)
+static void ak5558_i2c_remove(struct i2c_client *i2c)
 {
 	pm_runtime_disable(&i2c->dev);
-
-	return 0;
 }
 
 static const struct of_device_id ak5558_i2c_dt_ids[] __maybe_unused = {
diff --git a/sound/soc/codecs/cs35l32.c b/sound/soc/codecs/cs35l32.c
index 8ff6f66be86f..dc7a58d68076 100644
--- a/sound/soc/codecs/cs35l32.c
+++ b/sound/soc/codecs/cs35l32.c
@@ -497,14 +497,12 @@ err_supplies:
 	return ret;
 }
 
-static int cs35l32_i2c_remove(struct i2c_client *i2c_client)
+static void cs35l32_i2c_remove(struct i2c_client *i2c_client)
 {
 	struct cs35l32_private *cs35l32 = i2c_get_clientdata(i2c_client);
 
 	/* Hold down reset */
 	gpiod_set_value_cansleep(cs35l32->reset_gpio, 0);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/sound/soc/codecs/cs35l33.c b/sound/soc/codecs/cs35l33.c
index 082025fa0370..15e79168d256 100644
--- a/sound/soc/codecs/cs35l33.c
+++ b/sound/soc/codecs/cs35l33.c
@@ -1250,7 +1250,7 @@ err_enable:
 	return ret;
 }
 
-static int cs35l33_i2c_remove(struct i2c_client *client)
+static void cs35l33_i2c_remove(struct i2c_client *client)
 {
 	struct cs35l33_private *cs35l33 = i2c_get_clientdata(client);
 
@@ -1259,8 +1259,6 @@ static int cs35l33_i2c_remove(struct i2c_client *client)
 	pm_runtime_disable(&client->dev);
 	regulator_bulk_disable(cs35l33->num_core_supplies,
 		cs35l33->core_supplies);
-
-	return 0;
 }
 
 static const struct of_device_id cs35l33_of_match[] = {
diff --git a/sound/soc/codecs/cs35l34.c b/sound/soc/codecs/cs35l34.c
index 472ac982779b..b3f98023e6a7 100644
--- a/sound/soc/codecs/cs35l34.c
+++ b/sound/soc/codecs/cs35l34.c
@@ -1128,7 +1128,7 @@ err_regulator:
 	return ret;
 }
 
-static int cs35l34_i2c_remove(struct i2c_client *client)
+static void cs35l34_i2c_remove(struct i2c_client *client)
 {
 	struct cs35l34_private *cs35l34 = i2c_get_clientdata(client);
 
@@ -1137,8 +1137,6 @@ static int cs35l34_i2c_remove(struct i2c_client *client)
 	pm_runtime_disable(&client->dev);
 	regulator_bulk_disable(cs35l34->num_core_supplies,
 		cs35l34->core_supplies);
-
-	return 0;
 }
 
 static int __maybe_unused cs35l34_runtime_resume(struct device *dev)
diff --git a/sound/soc/codecs/cs35l35.c b/sound/soc/codecs/cs35l35.c
index 714a759dca21..947a440a3a47 100644
--- a/sound/soc/codecs/cs35l35.c
+++ b/sound/soc/codecs/cs35l35.c
@@ -1627,14 +1627,12 @@ err:
 	return ret;
 }
 
-static int cs35l35_i2c_remove(struct i2c_client *i2c_client)
+static void cs35l35_i2c_remove(struct i2c_client *i2c_client)
 {
 	struct cs35l35_private *cs35l35 = i2c_get_clientdata(i2c_client);
 
 	regulator_bulk_disable(cs35l35->num_supplies, cs35l35->supplies);
 	gpiod_set_value_cansleep(cs35l35->reset_gpio, 0);
-
-	return 0;
 }
 
 static const struct of_device_id cs35l35_of_match[] = {
diff --git a/sound/soc/codecs/cs35l36.c b/sound/soc/codecs/cs35l36.c
index 4dc13e6f4874..31ae752e242f 100644
--- a/sound/soc/codecs/cs35l36.c
+++ b/sound/soc/codecs/cs35l36.c
@@ -1910,7 +1910,7 @@ err_disable_regs:
 	return ret;
 }
 
-static int cs35l36_i2c_remove(struct i2c_client *client)
+static void cs35l36_i2c_remove(struct i2c_client *client)
 {
 	struct cs35l36_private *cs35l36 = i2c_get_clientdata(client);
 
@@ -1924,8 +1924,6 @@ static int cs35l36_i2c_remove(struct i2c_client *client)
 		gpiod_set_value_cansleep(cs35l36->reset_gpio, 0);
 
 	regulator_bulk_disable(cs35l36->num_supplies, cs35l36->supplies);
-
-	return 0;
 }
 static const struct of_device_id cs35l36_of_match[] = {
 	{.compatible = "cirrus,cs35l36"},
diff --git a/sound/soc/codecs/cs35l41-i2c.c b/sound/soc/codecs/cs35l41-i2c.c
index 37c703c08fd5..3676b596f60b 100644
--- a/sound/soc/codecs/cs35l41-i2c.c
+++ b/sound/soc/codecs/cs35l41-i2c.c
@@ -56,13 +56,11 @@ static int cs35l41_i2c_probe(struct i2c_client *client)
 	return cs35l41_probe(cs35l41, hw_cfg);
 }
 
-static int cs35l41_i2c_remove(struct i2c_client *client)
+static void cs35l41_i2c_remove(struct i2c_client *client)
 {
 	struct cs35l41_private *cs35l41 = i2c_get_clientdata(client);
 
 	cs35l41_remove(cs35l41);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
diff --git a/sound/soc/codecs/cs35l45-i2c.c b/sound/soc/codecs/cs35l45-i2c.c
index 06c2ddffb9c5..39d28641429e 100644
--- a/sound/soc/codecs/cs35l45-i2c.c
+++ b/sound/soc/codecs/cs35l45-i2c.c
@@ -36,13 +36,11 @@ static int cs35l45_i2c_probe(struct i2c_client *client)
 	return cs35l45_probe(cs35l45);
 }
 
-static int cs35l45_i2c_remove(struct i2c_client *client)
+static void cs35l45_i2c_remove(struct i2c_client *client)
 {
 	struct cs35l45_private *cs35l45 = i2c_get_clientdata(client);
 
 	cs35l45_remove(cs35l45);
-
-	return 0;
 }
 
 static const struct of_device_id cs35l45_of_match[] = {
diff --git a/sound/soc/codecs/cs4234.c b/sound/soc/codecs/cs4234.c
index b49a3cf21ebe..dee1a6662c2e 100644
--- a/sound/soc/codecs/cs4234.c
+++ b/sound/soc/codecs/cs4234.c
@@ -850,7 +850,7 @@ fail_shutdown:
 	return ret;
 }
 
-static int cs4234_i2c_remove(struct i2c_client *i2c_client)
+static void cs4234_i2c_remove(struct i2c_client *i2c_client)
 {
 	struct cs4234 *cs4234 = i2c_get_clientdata(i2c_client);
 	struct device *dev = &i2c_client->dev;
@@ -858,8 +858,6 @@ static int cs4234_i2c_remove(struct i2c_client *i2c_client)
 	snd_soc_unregister_component(dev);
 	pm_runtime_disable(dev);
 	cs4234_shutdown(cs4234);
-
-	return 0;
 }
 
 static int __maybe_unused cs4234_runtime_resume(struct device *dev)
diff --git a/sound/soc/codecs/cs4265.c b/sound/soc/codecs/cs4265.c
index 76c19802d5fe..3573363b7e31 100644
--- a/sound/soc/codecs/cs4265.c
+++ b/sound/soc/codecs/cs4265.c
@@ -623,14 +623,12 @@ static int cs4265_i2c_probe(struct i2c_client *i2c_client)
 			ARRAY_SIZE(cs4265_dai));
 }
 
-static int cs4265_i2c_remove(struct i2c_client *i2c)
+static void cs4265_i2c_remove(struct i2c_client *i2c)
 {
 	struct cs4265_private *cs4265 = i2c_get_clientdata(i2c);
 
 	if (cs4265->reset_gpio)
 		gpiod_set_value_cansleep(cs4265->reset_gpio, 0);
-
-	return 0;
 }
 
 static const struct of_device_id cs4265_of_match[] = {
diff --git a/sound/soc/codecs/cs4270.c b/sound/soc/codecs/cs4270.c
index ba67e43edf35..1b640d8232ba 100644
--- a/sound/soc/codecs/cs4270.c
+++ b/sound/soc/codecs/cs4270.c
@@ -650,13 +650,11 @@ static const struct regmap_config cs4270_regmap = {
  * This function puts the chip into low power mode when the i2c device
  * is removed.
  */
-static int cs4270_i2c_remove(struct i2c_client *i2c_client)
+static void cs4270_i2c_remove(struct i2c_client *i2c_client)
 {
 	struct cs4270_private *cs4270 = i2c_get_clientdata(i2c_client);
 
 	gpiod_set_value_cansleep(cs4270->reset_gpio, 0);
-
-	return 0;
 }
 
 /**
diff --git a/sound/soc/codecs/cs42l42-i2c.c b/sound/soc/codecs/cs42l42-i2c.c
new file mode 100644
index 000000000000..67b253287daf
--- /dev/null
+++ b/sound/soc/codecs/cs42l42-i2c.c
@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * cs42l42-i2c.c -- CS42L42 ALSA SoC audio driver for I2C
+ *
+ * Copyright 2016, 2022 Cirrus Logic, Inc.
+ */
+
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include "cs42l42.h"
+
+static int cs42l42_i2c_probe(struct i2c_client *i2c_client)
+{
+	struct device *dev = &i2c_client->dev;
+	struct cs42l42_private *cs42l42;
+	struct regmap *regmap;
+	int ret;
+
+	cs42l42 = devm_kzalloc(dev, sizeof(*cs42l42), GFP_KERNEL);
+	if (!cs42l42)
+		return -ENOMEM;
+
+	regmap = devm_regmap_init_i2c(i2c_client, &cs42l42_regmap);
+	if (IS_ERR(regmap))
+		return dev_err_probe(&i2c_client->dev, PTR_ERR(regmap),
+				     "regmap_init() failed\n");
+
+	cs42l42->devid = CS42L42_CHIP_ID;
+	cs42l42->dev = dev;
+	cs42l42->regmap = regmap;
+	cs42l42->irq = i2c_client->irq;
+
+	ret = cs42l42_common_probe(cs42l42, &cs42l42_soc_component, &cs42l42_dai);
+	if (ret)
+		return ret;
+
+	return cs42l42_init(cs42l42);
+}
+
+static void cs42l42_i2c_remove(struct i2c_client *i2c_client)
+{
+	struct cs42l42_private *cs42l42 = dev_get_drvdata(&i2c_client->dev);
+
+	cs42l42_common_remove(cs42l42);
+}
+
+static int __maybe_unused cs42l42_i2c_resume(struct device *dev)
+{
+	int ret;
+
+	ret = cs42l42_resume(dev);
+	if (ret)
+		return ret;
+
+	cs42l42_resume_restore(dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops cs42l42_i2c_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(cs42l42_suspend, cs42l42_i2c_resume)
+};
+
+static const struct of_device_id __maybe_unused cs42l42_of_match[] = {
+	{ .compatible = "cirrus,cs42l42", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, cs42l42_of_match);
+
+static const struct acpi_device_id __maybe_unused cs42l42_acpi_match[] = {
+	{"10134242", 0,},
+	{}
+};
+MODULE_DEVICE_TABLE(acpi, cs42l42_acpi_match);
+
+static const struct i2c_device_id cs42l42_id[] = {
+	{"cs42l42", 0},
+	{}
+};
+
+MODULE_DEVICE_TABLE(i2c, cs42l42_id);
+
+static struct i2c_driver cs42l42_i2c_driver = {
+	.driver = {
+		.name = "cs42l42",
+		.pm = &cs42l42_i2c_pm_ops,
+		.of_match_table = of_match_ptr(cs42l42_of_match),
+		.acpi_match_table = ACPI_PTR(cs42l42_acpi_match),
+		},
+	.id_table = cs42l42_id,
+	.probe_new = cs42l42_i2c_probe,
+	.remove = cs42l42_i2c_remove,
+};
+
+module_i2c_driver(cs42l42_i2c_driver);
+
+MODULE_DESCRIPTION("ASoC CS42L42 I2C driver");
+MODULE_AUTHOR("Richard Fitzgerald <rf@opensource.cirrus.com>");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS(SND_SOC_CS42L42_CORE);
diff --git a/sound/soc/codecs/cs42l42.c b/sound/soc/codecs/cs42l42.c
index daafd4251ce6..2fefbcf7bd13 100644
--- a/sound/soc/codecs/cs42l42.c
+++ b/sound/soc/codecs/cs42l42.c
@@ -12,10 +12,9 @@
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/version.h>
-#include <linux/kernel.h>
+#include <linux/types.h>
 #include <linux/init.h>
 #include <linux/delay.h>
-#include <linux/i2c.h>
 #include <linux/gpio.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
@@ -37,6 +36,14 @@
 #include "cs42l42.h"
 #include "cirrus_legacy.h"
 
+static const char * const cs42l42_supply_names[] = {
+	"VA",
+	"VP",
+	"VCP",
+	"VD_FILT",
+	"VL",
+};
+
 static const struct reg_default cs42l42_reg_defaults[] = {
 	{ CS42L42_FRZ_CTL,			0x00 },
 	{ CS42L42_SRC_CTL,			0x10 },
@@ -164,7 +171,7 @@ static const struct reg_default cs42l42_reg_defaults[] = {
 	{ CS42L42_ASP_RX_DAI1_CH2_BIT_LSB,	0x00 },
 };
 
-static bool cs42l42_readable_register(struct device *dev, unsigned int reg)
+bool cs42l42_readable_register(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
 	case CS42L42_PAGE_REGISTER:
@@ -323,8 +330,9 @@ static bool cs42l42_readable_register(struct device *dev, unsigned int reg)
 		return false;
 	}
 }
+EXPORT_SYMBOL_NS_GPL(cs42l42_readable_register, SND_SOC_CS42L42_CORE);
 
-static bool cs42l42_volatile_register(struct device *dev, unsigned int reg)
+bool cs42l42_volatile_register(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
 	case CS42L42_DEVID_AB:
@@ -355,8 +363,9 @@ static bool cs42l42_volatile_register(struct device *dev, unsigned int reg)
 		return false;
 	}
 }
+EXPORT_SYMBOL_NS_GPL(cs42l42_volatile_register, SND_SOC_CS42L42_CORE);
 
-static const struct regmap_range_cfg cs42l42_page_range = {
+const struct regmap_range_cfg cs42l42_page_range = {
 	.name = "Pages",
 	.range_min = 0,
 	.range_max = CS42L42_MAX_REGISTER,
@@ -366,8 +375,9 @@ static const struct regmap_range_cfg cs42l42_page_range = {
 	.window_start = 0,
 	.window_len = 256,
 };
+EXPORT_SYMBOL_NS_GPL(cs42l42_page_range, SND_SOC_CS42L42_CORE);
 
-static const struct regmap_config cs42l42_regmap = {
+const struct regmap_config cs42l42_regmap = {
 	.reg_bits = 8,
 	.val_bits = 8,
 
@@ -385,6 +395,7 @@ static const struct regmap_config cs42l42_regmap = {
 	.use_single_read = true,
 	.use_single_write = true,
 };
+EXPORT_SYMBOL_NS_GPL(cs42l42_regmap, SND_SOC_CS42L42_CORE);
 
 static DECLARE_TLV_DB_SCALE(adc_tlv, -9700, 100, true);
 static DECLARE_TLV_DB_SCALE(mixer_tlv, -6300, 100, true);
@@ -395,7 +406,7 @@ static int cs42l42_slow_start_put(struct snd_kcontrol *kcontrol,
 	struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol);
 	u8 val;
 
-	/* all bits of SLOW_START_EN much change together */
+	/* all bits of SLOW_START_EN must change together */
 	switch (ucontrol->value.integer.value[0]) {
 	case 0:
 		val = 0;
@@ -571,7 +582,7 @@ static int cs42l42_set_jack(struct snd_soc_component *component, struct snd_soc_
 	return 0;
 }
 
-static const struct snd_soc_component_driver soc_component_dev_cs42l42 = {
+const struct snd_soc_component_driver cs42l42_soc_component = {
 	.set_jack		= cs42l42_set_jack,
 	.dapm_widgets		= cs42l42_dapm_widgets,
 	.num_dapm_widgets	= ARRAY_SIZE(cs42l42_dapm_widgets),
@@ -582,6 +593,7 @@ static const struct snd_soc_component_driver soc_component_dev_cs42l42 = {
 	.idle_bias_on		= 1,
 	.endianness		= 1,
 };
+EXPORT_SYMBOL_NS_GPL(cs42l42_soc_component, SND_SOC_CS42L42_CORE);
 
 /* Switch to SCLK. Atomic delay after the write to allow the switch to complete. */
 static const struct reg_sequence cs42l42_to_sclk_seq[] = {
@@ -639,18 +651,12 @@ static const struct cs42l42_pll_params pll_ratio_table[] = {
 	{ 24576000, 1, 0x03, 0x40, 0x000000, 0x03, 0x10, 12288000, 128, 1}
 };
 
-static int cs42l42_pll_config(struct snd_soc_component *component)
+static int cs42l42_pll_config(struct snd_soc_component *component, unsigned int clk)
 {
 	struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(component);
 	int i;
-	u32 clk;
 	u32 fsync;
 
-	if (!cs42l42->sclk)
-		clk = cs42l42->bclk;
-	else
-		clk = cs42l42->sclk;
-
 	/* Don't reconfigure if there is an audio stream running */
 	if (cs42l42->stream_use) {
 		if (pll_ratio_table[cs42l42->pll_config].sclk == clk)
@@ -885,22 +891,30 @@ static int cs42l42_pcm_hw_params(struct snd_pcm_substream *substream,
 	struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(component);
 	unsigned int channels = params_channels(params);
 	unsigned int width = (params_width(params) / 8) - 1;
+	unsigned int slot_width = 0;
 	unsigned int val = 0;
+	unsigned int bclk;
 	int ret;
 
 	cs42l42->srate = params_rate(params);
-	cs42l42->bclk = snd_soc_params_to_bclk(params);
 
-	/* I2S frame always has 2 channels even for mono audio */
-	if (channels == 1)
-		cs42l42->bclk *= 2;
+	if (cs42l42->bclk_ratio) {
+		/* machine driver has set the BCLK/samp-rate ratio */
+		bclk = cs42l42->bclk_ratio * params_rate(params);
+	} else if (cs42l42->sclk) {
+		/* machine driver has set the SCLK */
+		bclk = cs42l42->sclk;
+	} else {
+		/*
+		 * Assume 24-bit samples are in 32-bit slots, to prevent SCLK being
+		 * more than assumed (which would result in overclocking).
+		 */
+		if (params_width(params) == 24)
+			slot_width = 32;
 
-	/*
-	 * Assume 24-bit samples are in 32-bit slots, to prevent SCLK being
-	 * more than assumed (which would result in overclocking).
-	 */
-	if (params_width(params) == 24)
-		cs42l42->bclk = (cs42l42->bclk / 3) * 4;
+		/* I2S frame always has multiple of 2 channels */
+		bclk = snd_soc_tdm_params_to_bclk(params, slot_width, 0, 2);
+	}
 
 	switch (substream->stream) {
 	case SNDRV_PCM_STREAM_CAPTURE:
@@ -940,7 +954,7 @@ static int cs42l42_pcm_hw_params(struct snd_pcm_substream *substream,
 		break;
 	}
 
-	ret = cs42l42_pll_config(component);
+	ret = cs42l42_pll_config(component, bclk);
 	if (ret)
 		return ret;
 
@@ -973,6 +987,17 @@ static int cs42l42_set_sysclk(struct snd_soc_dai *dai,
 	return -EINVAL;
 }
 
+static int cs42l42_set_bclk_ratio(struct snd_soc_dai *dai,
+				unsigned int bclk_ratio)
+{
+	struct snd_soc_component *component = dai->component;
+	struct cs42l42_private *cs42l42 = snd_soc_component_get_drvdata(component);
+
+	cs42l42->bclk_ratio = bclk_ratio;
+
+	return 0;
+}
+
 static int cs42l42_mute_stream(struct snd_soc_dai *dai, int mute, int stream)
 {
 	struct snd_soc_component *component = dai->component;
@@ -1076,10 +1101,11 @@ static const struct snd_soc_dai_ops cs42l42_ops = {
 	.hw_params	= cs42l42_pcm_hw_params,
 	.set_fmt	= cs42l42_set_dai_fmt,
 	.set_sysclk	= cs42l42_set_sysclk,
+	.set_bclk_ratio	= cs42l42_set_bclk_ratio,
 	.mute_stream	= cs42l42_mute_stream,
 };
 
-static struct snd_soc_dai_driver cs42l42_dai = {
+struct snd_soc_dai_driver cs42l42_dai = {
 		.name = "cs42l42",
 		.playback = {
 			.stream_name = "Playback",
@@ -1099,6 +1125,7 @@ static struct snd_soc_dai_driver cs42l42_dai = {
 		.symmetric_sample_bits = 1,
 		.ops = &cs42l42_ops,
 };
+EXPORT_SYMBOL_NS_GPL(cs42l42_dai, SND_SOC_CS42L42_CORE);
 
 static void cs42l42_manual_hs_type_detect(struct cs42l42_private *cs42l42)
 {
@@ -1172,14 +1199,11 @@ static void cs42l42_manual_hs_type_detect(struct cs42l42_private *cs42l42)
 			cs42l42->hs_type = CS42L42_PLUG_OMTP;
 			hs_det_sw = CS42L42_HSDET_SW_TYPE2;
 			break;
-		case CS42L42_HSDET_COMP_TYPE3:
+		/* Detect Type 3 and Type 4 Headsets as Headphones */
+		default:
 			cs42l42->hs_type = CS42L42_PLUG_HEADPHONE;
 			hs_det_sw = CS42L42_HSDET_SW_TYPE3;
 			break;
-		default:
-			cs42l42->hs_type = CS42L42_PLUG_INVALID;
-			hs_det_sw = CS42L42_HSDET_SW_TYPE4;
-			break;
 		}
 	}
 
@@ -1619,7 +1643,7 @@ static irqreturn_t cs42l42_irq_thread(int irq, void *data)
 	unsigned int i;
 
 	mutex_lock(&cs42l42->irq_lock);
-	if (cs42l42->suspended) {
+	if (cs42l42->suspended || !cs42l42->init_done) {
 		mutex_unlock(&cs42l42->irq_lock);
 		return IRQ_NONE;
 	}
@@ -2094,7 +2118,7 @@ static const struct reg_sequence __maybe_unused cs42l42_shutdown_seq[] = {
 	REG_SEQ0(CS42L42_PWR_CTL1,		0xFF)
 };
 
-static int __maybe_unused cs42l42_suspend(struct device *dev)
+int cs42l42_suspend(struct device *dev)
 {
 	struct cs42l42_private *cs42l42 = dev_get_drvdata(dev);
 	unsigned int reg;
@@ -2154,8 +2178,9 @@ static int __maybe_unused cs42l42_suspend(struct device *dev)
 	return 0;
 
 }
+EXPORT_SYMBOL_NS_GPL(cs42l42_suspend, SND_SOC_CS42L42_CORE);
 
-static int __maybe_unused cs42l42_resume(struct device *dev)
+int cs42l42_resume(struct device *dev)
 {
 	struct cs42l42_private *cs42l42 = dev_get_drvdata(dev);
 	int ret;
@@ -2177,6 +2202,16 @@ static int __maybe_unused cs42l42_resume(struct device *dev)
 	gpiod_set_value_cansleep(cs42l42->reset_gpio, 1);
 	usleep_range(CS42L42_BOOT_TIME_US, CS42L42_BOOT_TIME_US * 2);
 
+	dev_dbg(dev, "System resume powered up\n");
+
+	return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cs42l42_resume, SND_SOC_CS42L42_CORE);
+
+void cs42l42_resume_restore(struct device *dev)
+{
+	struct cs42l42_private *cs42l42 = dev_get_drvdata(dev);
+
 	regcache_cache_only(cs42l42->regmap, false);
 	regcache_mark_dirty(cs42l42->regmap);
 
@@ -2189,40 +2224,40 @@ static int __maybe_unused cs42l42_resume(struct device *dev)
 	mutex_unlock(&cs42l42->irq_lock);
 
 	dev_dbg(dev, "System resumed\n");
+}
+EXPORT_SYMBOL_NS_GPL(cs42l42_resume_restore, SND_SOC_CS42L42_CORE);
+
+static int __maybe_unused cs42l42_i2c_resume(struct device *dev)
+{
+	int ret;
+
+	ret = cs42l42_resume(dev);
+	if (ret)
+		return ret;
+
+	cs42l42_resume_restore(dev);
 
 	return 0;
 }
 
-static int cs42l42_i2c_probe(struct i2c_client *i2c_client)
+int cs42l42_common_probe(struct cs42l42_private *cs42l42,
+			 const struct snd_soc_component_driver *component_drv,
+			 struct snd_soc_dai_driver *dai)
 {
-	struct cs42l42_private *cs42l42;
-	int ret, i, devid;
-	unsigned int reg;
-
-	cs42l42 = devm_kzalloc(&i2c_client->dev, sizeof(struct cs42l42_private),
-			       GFP_KERNEL);
-	if (!cs42l42)
-		return -ENOMEM;
+	int ret, i;
 
-	cs42l42->dev = &i2c_client->dev;
-	i2c_set_clientdata(i2c_client, cs42l42);
+	dev_set_drvdata(cs42l42->dev, cs42l42);
 	mutex_init(&cs42l42->irq_lock);
 
-	cs42l42->regmap = devm_regmap_init_i2c(i2c_client, &cs42l42_regmap);
-	if (IS_ERR(cs42l42->regmap)) {
-		ret = PTR_ERR(cs42l42->regmap);
-		dev_err(&i2c_client->dev, "regmap_init() failed: %d\n", ret);
-		return ret;
-	}
-
+	BUILD_BUG_ON(ARRAY_SIZE(cs42l42_supply_names) != ARRAY_SIZE(cs42l42->supplies));
 	for (i = 0; i < ARRAY_SIZE(cs42l42->supplies); i++)
 		cs42l42->supplies[i].supply = cs42l42_supply_names[i];
 
-	ret = devm_regulator_bulk_get(&i2c_client->dev,
+	ret = devm_regulator_bulk_get(cs42l42->dev,
 				      ARRAY_SIZE(cs42l42->supplies),
 				      cs42l42->supplies);
 	if (ret != 0) {
-		dev_err(&i2c_client->dev,
+		dev_err(cs42l42->dev,
 			"Failed to request supplies: %d\n", ret);
 		return ret;
 	}
@@ -2230,13 +2265,13 @@ static int cs42l42_i2c_probe(struct i2c_client *i2c_client)
 	ret = regulator_bulk_enable(ARRAY_SIZE(cs42l42->supplies),
 				    cs42l42->supplies);
 	if (ret != 0) {
-		dev_err(&i2c_client->dev,
+		dev_err(cs42l42->dev,
 			"Failed to enable supplies: %d\n", ret);
 		return ret;
 	}
 
 	/* Reset the Device */
-	cs42l42->reset_gpio = devm_gpiod_get_optional(&i2c_client->dev,
+	cs42l42->reset_gpio = devm_gpiod_get_optional(cs42l42->dev,
 		"reset", GPIOD_OUT_LOW);
 	if (IS_ERR(cs42l42->reset_gpio)) {
 		ret = PTR_ERR(cs42l42->reset_gpio);
@@ -2244,50 +2279,74 @@ static int cs42l42_i2c_probe(struct i2c_client *i2c_client)
 	}
 
 	if (cs42l42->reset_gpio) {
-		dev_dbg(&i2c_client->dev, "Found reset GPIO\n");
+		dev_dbg(cs42l42->dev, "Found reset GPIO\n");
 		gpiod_set_value_cansleep(cs42l42->reset_gpio, 1);
 	}
 	usleep_range(CS42L42_BOOT_TIME_US, CS42L42_BOOT_TIME_US * 2);
 
 	/* Request IRQ if one was specified */
-	if (i2c_client->irq) {
-		ret = request_threaded_irq(i2c_client->irq,
+	if (cs42l42->irq) {
+		ret = request_threaded_irq(cs42l42->irq,
 					   NULL, cs42l42_irq_thread,
 					   IRQF_ONESHOT | IRQF_TRIGGER_LOW,
 					   "cs42l42", cs42l42);
-		if (ret == -EPROBE_DEFER) {
-			goto err_disable_noirq;
-		} else if (ret != 0) {
-			dev_err(&i2c_client->dev,
-				"Failed to request IRQ: %d\n", ret);
+		if (ret) {
+			dev_err_probe(cs42l42->dev, ret,
+				"Failed to request IRQ\n");
 			goto err_disable_noirq;
 		}
 	}
 
+	/* Register codec now so it can EPROBE_DEFER */
+	ret = devm_snd_soc_register_component(cs42l42->dev, component_drv, dai, 1);
+	if (ret < 0)
+		goto err;
+
+	return 0;
+
+err:
+	if (cs42l42->irq)
+		free_irq(cs42l42->irq, cs42l42);
+
+err_disable_noirq:
+	gpiod_set_value_cansleep(cs42l42->reset_gpio, 0);
+err_disable_noreset:
+	regulator_bulk_disable(ARRAY_SIZE(cs42l42->supplies), cs42l42->supplies);
+
+	return ret;
+}
+EXPORT_SYMBOL_NS_GPL(cs42l42_common_probe, SND_SOC_CS42L42_CORE);
+
+int cs42l42_init(struct cs42l42_private *cs42l42)
+{
+	unsigned int reg;
+	int devid, ret;
+
 	/* initialize codec */
 	devid = cirrus_read_device_id(cs42l42->regmap, CS42L42_DEVID_AB);
 	if (devid < 0) {
 		ret = devid;
-		dev_err(&i2c_client->dev, "Failed to read device ID: %d\n", ret);
+		dev_err(cs42l42->dev, "Failed to read device ID: %d\n", ret);
 		goto err_disable;
 	}
 
-	if (devid != CS42L42_CHIP_ID) {
+	if (devid != cs42l42->devid) {
 		ret = -ENODEV;
-		dev_err(&i2c_client->dev,
-			"CS42L42 Device ID (%X). Expected %X\n",
-			devid, CS42L42_CHIP_ID);
+		dev_err(cs42l42->dev,
+			"CS42L%x Device ID (%X). Expected %X\n",
+			cs42l42->devid & 0xff, devid, cs42l42->devid);
 		goto err_disable;
 	}
 
 	ret = regmap_read(cs42l42->regmap, CS42L42_REVID, &reg);
 	if (ret < 0) {
-		dev_err(&i2c_client->dev, "Get Revision ID failed\n");
+		dev_err(cs42l42->dev, "Get Revision ID failed\n");
 		goto err_shutdown;
 	}
 
-	dev_info(&i2c_client->dev,
-		 "Cirrus Logic CS42L42, Revision: %02X\n", reg & 0xFF);
+	dev_info(cs42l42->dev,
+		 "Cirrus Logic CS42L%x, Revision: %02X\n",
+		 cs42l42->devid & 0xff, reg & 0xFF);
 
 	/* Power up the codec */
 	regmap_update_bits(cs42l42->regmap, CS42L42_PWR_CTL1,
@@ -2306,22 +2365,22 @@ static int cs42l42_i2c_probe(struct i2c_client *i2c_client)
 			(1 << CS42L42_ADC_PDN_SHIFT) |
 			(0 << CS42L42_PDN_ALL_SHIFT));
 
-	ret = cs42l42_handle_device_data(&i2c_client->dev, cs42l42);
+	ret = cs42l42_handle_device_data(cs42l42->dev, cs42l42);
 	if (ret != 0)
 		goto err_shutdown;
 
 	/* Setup headset detection */
 	cs42l42_setup_hs_type_detect(cs42l42);
 
+	/*
+	 * Set init_done before unmasking interrupts so any triggered
+	 * immediately will be handled.
+	 */
+	cs42l42->init_done = true;
+
 	/* Mask/Unmask Interrupts */
 	cs42l42_set_interrupt_masks(cs42l42);
 
-	/* Register codec for machine driver */
-	ret = devm_snd_soc_register_component(&i2c_client->dev,
-			&soc_component_dev_cs42l42, &cs42l42_dai, 1);
-	if (ret < 0)
-		goto err_shutdown;
-
 	return 0;
 
 err_shutdown:
@@ -2330,78 +2389,35 @@ err_shutdown:
 	regmap_write(cs42l42->regmap, CS42L42_PWR_CTL1, 0xff);
 
 err_disable:
-	if (i2c_client->irq)
-		free_irq(i2c_client->irq, cs42l42);
+	if (cs42l42->irq)
+		free_irq(cs42l42->irq, cs42l42);
 
-err_disable_noirq:
 	gpiod_set_value_cansleep(cs42l42->reset_gpio, 0);
-err_disable_noreset:
 	regulator_bulk_disable(ARRAY_SIZE(cs42l42->supplies),
 				cs42l42->supplies);
 	return ret;
 }
+EXPORT_SYMBOL_NS_GPL(cs42l42_init, SND_SOC_CS42L42_CORE);
 
-static int cs42l42_i2c_remove(struct i2c_client *i2c_client)
+void cs42l42_common_remove(struct cs42l42_private *cs42l42)
 {
-	struct cs42l42_private *cs42l42 = i2c_get_clientdata(i2c_client);
-
-	if (i2c_client->irq)
-		free_irq(i2c_client->irq, cs42l42);
+	if (cs42l42->irq)
+		free_irq(cs42l42->irq, cs42l42);
 
 	/*
 	 * The driver might not have control of reset and power supplies,
 	 * so ensure that the chip internals are powered down.
 	 */
-	regmap_write(cs42l42->regmap, CS42L42_CODEC_INT_MASK, 0xff);
-	regmap_write(cs42l42->regmap, CS42L42_TSRS_PLUG_INT_MASK, 0xff);
-	regmap_write(cs42l42->regmap, CS42L42_PWR_CTL1, 0xff);
+	if (cs42l42->init_done) {
+		regmap_write(cs42l42->regmap, CS42L42_CODEC_INT_MASK, 0xff);
+		regmap_write(cs42l42->regmap, CS42L42_TSRS_PLUG_INT_MASK, 0xff);
+		regmap_write(cs42l42->regmap, CS42L42_PWR_CTL1, 0xff);
+	}
 
 	gpiod_set_value_cansleep(cs42l42->reset_gpio, 0);
 	regulator_bulk_disable(ARRAY_SIZE(cs42l42->supplies), cs42l42->supplies);
-
-	return 0;
 }
-
-static const struct dev_pm_ops cs42l42_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(cs42l42_suspend, cs42l42_resume)
-};
-
-#ifdef CONFIG_OF
-static const struct of_device_id cs42l42_of_match[] = {
-	{ .compatible = "cirrus,cs42l42", },
-	{}
-};
-MODULE_DEVICE_TABLE(of, cs42l42_of_match);
-#endif
-
-#ifdef CONFIG_ACPI
-static const struct acpi_device_id cs42l42_acpi_match[] = {
-	{"10134242", 0,},
-	{}
-};
-MODULE_DEVICE_TABLE(acpi, cs42l42_acpi_match);
-#endif
-
-static const struct i2c_device_id cs42l42_id[] = {
-	{"cs42l42", 0},
-	{}
-};
-
-MODULE_DEVICE_TABLE(i2c, cs42l42_id);
-
-static struct i2c_driver cs42l42_i2c_driver = {
-	.driver = {
-		.name = "cs42l42",
-		.pm = &cs42l42_pm_ops,
-		.of_match_table = of_match_ptr(cs42l42_of_match),
-		.acpi_match_table = ACPI_PTR(cs42l42_acpi_match),
-		},
-	.id_table = cs42l42_id,
-	.probe_new = cs42l42_i2c_probe,
-	.remove = cs42l42_i2c_remove,
-};
-
-module_i2c_driver(cs42l42_i2c_driver);
+EXPORT_SYMBOL_NS_GPL(cs42l42_common_remove, SND_SOC_CS42L42_CORE);
 
 MODULE_DESCRIPTION("ASoC CS42L42 driver");
 MODULE_AUTHOR("James Schulman, Cirrus Logic Inc, <james.schulman@cirrus.com>");
diff --git a/sound/soc/codecs/cs42l42.h b/sound/soc/codecs/cs42l42.h
index 5f50970375d4..a72136664112 100644
--- a/sound/soc/codecs/cs42l42.h
+++ b/sound/soc/codecs/cs42l42.h
@@ -12,17 +12,16 @@
 #ifndef __CS42L42_H__
 #define __CS42L42_H__
 
+#include <dt-bindings/sound/cs42l42.h>
+#include <linux/device.h>
+#include <linux/gpio.h>
 #include <linux/mutex.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
 #include <sound/jack.h>
 #include <sound/cs42l42.h>
-
-static const char *const cs42l42_supply_names[CS42L42_NUM_SUPPLIES] = {
-	"VA",
-	"VP",
-	"VCP",
-	"VD_FILT",
-	"VL",
-};
+#include <sound/soc-component.h>
+#include <sound/soc-dai.h>
 
 struct  cs42l42_private {
 	struct regmap *regmap;
@@ -32,9 +31,11 @@ struct  cs42l42_private {
 	struct completion pdn_done;
 	struct snd_soc_jack *jack;
 	struct mutex irq_lock;
+	int devid;
+	int irq;
 	int pll_config;
-	int bclk;
 	u32 sclk;
+	u32 bclk_ratio;
 	u32 srate;
 	u8 plug_state;
 	u8 hs_type;
@@ -50,6 +51,24 @@ struct  cs42l42_private {
 	u8 stream_use;
 	bool hp_adc_up_pending;
 	bool suspended;
+	bool init_done;
 };
 
+extern const struct regmap_range_cfg cs42l42_page_range;
+extern const struct regmap_config cs42l42_regmap;
+extern const struct snd_soc_component_driver cs42l42_soc_component;
+extern struct snd_soc_dai_driver cs42l42_dai;
+
+bool cs42l42_readable_register(struct device *dev, unsigned int reg);
+bool cs42l42_volatile_register(struct device *dev, unsigned int reg);
+
+int cs42l42_suspend(struct device *dev);
+int cs42l42_resume(struct device *dev);
+void cs42l42_resume_restore(struct device *dev);
+int cs42l42_common_probe(struct cs42l42_private *cs42l42,
+			 const struct snd_soc_component_driver *component_drv,
+			 struct snd_soc_dai_driver *dai);
+int cs42l42_init(struct cs42l42_private *cs42l42);
+void cs42l42_common_remove(struct cs42l42_private *cs42l42);
+
 #endif /* __CS42L42_H__ */
diff --git a/sound/soc/codecs/cs42l51-i2c.c b/sound/soc/codecs/cs42l51-i2c.c
index 3613fb12d623..85238339fbca 100644
--- a/sound/soc/codecs/cs42l51-i2c.c
+++ b/sound/soc/codecs/cs42l51-i2c.c
@@ -28,11 +28,9 @@ static int cs42l51_i2c_probe(struct i2c_client *i2c)
 	return cs42l51_probe(&i2c->dev, devm_regmap_init_i2c(i2c, &config));
 }
 
-static int cs42l51_i2c_remove(struct i2c_client *i2c)
+static void cs42l51_i2c_remove(struct i2c_client *i2c)
 {
 	cs42l51_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct dev_pm_ops cs42l51_pm_ops = {
diff --git a/sound/soc/codecs/cs42l56.c b/sound/soc/codecs/cs42l56.c
index 03e2540a0ba1..26066682c983 100644
--- a/sound/soc/codecs/cs42l56.c
+++ b/sound/soc/codecs/cs42l56.c
@@ -1320,13 +1320,12 @@ err_enable:
 	return ret;
 }
 
-static int cs42l56_i2c_remove(struct i2c_client *client)
+static void cs42l56_i2c_remove(struct i2c_client *client)
 {
 	struct cs42l56_private *cs42l56 = i2c_get_clientdata(client);
 
 	regulator_bulk_disable(ARRAY_SIZE(cs42l56->supplies),
 			       cs42l56->supplies);
-	return 0;
 }
 
 static const struct of_device_id cs42l56_of_match[] = {
diff --git a/sound/soc/codecs/cs42l83-i2c.c b/sound/soc/codecs/cs42l83-i2c.c
new file mode 100644
index 000000000000..f90d43996a51
--- /dev/null
+++ b/sound/soc/codecs/cs42l83-i2c.c
@@ -0,0 +1,240 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * cs42l83-i2c.c -- CS42L83 ALSA SoC audio driver for I2C
+ *
+ * Based on cs42l42-i2c.c:
+ *   Copyright 2016, 2022 Cirrus Logic, Inc.
+ */
+
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include "cs42l42.h"
+
+static const struct reg_default cs42l83_reg_defaults[] = {
+	{ CS42L42_FRZ_CTL,			0x00 },
+	{ CS42L42_SRC_CTL,			0x10 },
+	{ CS42L42_MCLK_CTL,			0x00 }, /* <- only deviation from CS42L42 */
+	{ CS42L42_SFTRAMP_RATE,			0xA4 },
+	{ CS42L42_SLOW_START_ENABLE,		0x70 },
+	{ CS42L42_I2C_DEBOUNCE,			0x88 },
+	{ CS42L42_I2C_STRETCH,			0x03 },
+	{ CS42L42_I2C_TIMEOUT,			0xB7 },
+	{ CS42L42_PWR_CTL1,			0xFF },
+	{ CS42L42_PWR_CTL2,			0x84 },
+	{ CS42L42_PWR_CTL3,			0x20 },
+	{ CS42L42_RSENSE_CTL1,			0x40 },
+	{ CS42L42_RSENSE_CTL2,			0x00 },
+	{ CS42L42_OSC_SWITCH,			0x00 },
+	{ CS42L42_RSENSE_CTL3,			0x1B },
+	{ CS42L42_TSENSE_CTL,			0x1B },
+	{ CS42L42_TSRS_INT_DISABLE,		0x00 },
+	{ CS42L42_HSDET_CTL1,			0x77 },
+	{ CS42L42_HSDET_CTL2,			0x00 },
+	{ CS42L42_HS_SWITCH_CTL,		0xF3 },
+	{ CS42L42_HS_CLAMP_DISABLE,		0x00 },
+	{ CS42L42_MCLK_SRC_SEL,			0x00 },
+	{ CS42L42_SPDIF_CLK_CFG,		0x00 },
+	{ CS42L42_FSYNC_PW_LOWER,		0x00 },
+	{ CS42L42_FSYNC_PW_UPPER,		0x00 },
+	{ CS42L42_FSYNC_P_LOWER,		0xF9 },
+	{ CS42L42_FSYNC_P_UPPER,		0x00 },
+	{ CS42L42_ASP_CLK_CFG,			0x00 },
+	{ CS42L42_ASP_FRM_CFG,			0x10 },
+	{ CS42L42_FS_RATE_EN,			0x00 },
+	{ CS42L42_IN_ASRC_CLK,			0x00 },
+	{ CS42L42_OUT_ASRC_CLK,			0x00 },
+	{ CS42L42_PLL_DIV_CFG1,			0x00 },
+	{ CS42L42_ADC_OVFL_INT_MASK,		0x01 },
+	{ CS42L42_MIXER_INT_MASK,		0x0F },
+	{ CS42L42_SRC_INT_MASK,			0x0F },
+	{ CS42L42_ASP_RX_INT_MASK,		0x1F },
+	{ CS42L42_ASP_TX_INT_MASK,		0x0F },
+	{ CS42L42_CODEC_INT_MASK,		0x03 },
+	{ CS42L42_SRCPL_INT_MASK,		0x7F },
+	{ CS42L42_VPMON_INT_MASK,		0x01 },
+	{ CS42L42_PLL_LOCK_INT_MASK,		0x01 },
+	{ CS42L42_TSRS_PLUG_INT_MASK,		0x0F },
+	{ CS42L42_PLL_CTL1,			0x00 },
+	{ CS42L42_PLL_DIV_FRAC0,		0x00 },
+	{ CS42L42_PLL_DIV_FRAC1,		0x00 },
+	{ CS42L42_PLL_DIV_FRAC2,		0x00 },
+	{ CS42L42_PLL_DIV_INT,			0x40 },
+	{ CS42L42_PLL_CTL3,			0x10 },
+	{ CS42L42_PLL_CAL_RATIO,		0x80 },
+	{ CS42L42_PLL_CTL4,			0x03 },
+	{ CS42L42_LOAD_DET_EN,			0x00 },
+	{ CS42L42_HSBIAS_SC_AUTOCTL,		0x03 },
+	{ CS42L42_WAKE_CTL,			0xC0 },
+	{ CS42L42_ADC_DISABLE_MUTE,		0x00 },
+	{ CS42L42_TIPSENSE_CTL,			0x02 },
+	{ CS42L42_MISC_DET_CTL,			0x03 },
+	{ CS42L42_MIC_DET_CTL1,			0x1F },
+	{ CS42L42_MIC_DET_CTL2,			0x2F },
+	{ CS42L42_DET_INT1_MASK,		0xE0 },
+	{ CS42L42_DET_INT2_MASK,		0xFF },
+	{ CS42L42_HS_BIAS_CTL,			0xC2 },
+	{ CS42L42_ADC_CTL,			0x00 },
+	{ CS42L42_ADC_VOLUME,			0x00 },
+	{ CS42L42_ADC_WNF_HPF_CTL,		0x71 },
+	{ CS42L42_DAC_CTL1,			0x00 },
+	{ CS42L42_DAC_CTL2,			0x02 },
+	{ CS42L42_HP_CTL,			0x0D },
+	{ CS42L42_CLASSH_CTL,			0x07 },
+	{ CS42L42_MIXER_CHA_VOL,		0x3F },
+	{ CS42L42_MIXER_ADC_VOL,		0x3F },
+	{ CS42L42_MIXER_CHB_VOL,		0x3F },
+	{ CS42L42_EQ_COEF_IN0,			0x00 },
+	{ CS42L42_EQ_COEF_IN1,			0x00 },
+	{ CS42L42_EQ_COEF_IN2,			0x00 },
+	{ CS42L42_EQ_COEF_IN3,			0x00 },
+	{ CS42L42_EQ_COEF_RW,			0x00 },
+	{ CS42L42_EQ_COEF_OUT0,			0x00 },
+	{ CS42L42_EQ_COEF_OUT1,			0x00 },
+	{ CS42L42_EQ_COEF_OUT2,			0x00 },
+	{ CS42L42_EQ_COEF_OUT3,			0x00 },
+	{ CS42L42_EQ_INIT_STAT,			0x00 },
+	{ CS42L42_EQ_START_FILT,		0x00 },
+	{ CS42L42_EQ_MUTE_CTL,			0x00 },
+	{ CS42L42_SP_RX_CH_SEL,			0x04 },
+	{ CS42L42_SP_RX_ISOC_CTL,		0x04 },
+	{ CS42L42_SP_RX_FS,			0x8C },
+	{ CS42l42_SPDIF_CH_SEL,			0x0E },
+	{ CS42L42_SP_TX_ISOC_CTL,		0x04 },
+	{ CS42L42_SP_TX_FS,			0xCC },
+	{ CS42L42_SPDIF_SW_CTL1,		0x3F },
+	{ CS42L42_SRC_SDIN_FS,			0x40 },
+	{ CS42L42_SRC_SDOUT_FS,			0x40 },
+	{ CS42L42_SPDIF_CTL1,			0x01 },
+	{ CS42L42_SPDIF_CTL2,			0x00 },
+	{ CS42L42_SPDIF_CTL3,			0x00 },
+	{ CS42L42_SPDIF_CTL4,			0x42 },
+	{ CS42L42_ASP_TX_SZ_EN,			0x00 },
+	{ CS42L42_ASP_TX_CH_EN,			0x00 },
+	{ CS42L42_ASP_TX_CH_AP_RES,		0x0F },
+	{ CS42L42_ASP_TX_CH1_BIT_MSB,		0x00 },
+	{ CS42L42_ASP_TX_CH1_BIT_LSB,		0x00 },
+	{ CS42L42_ASP_TX_HIZ_DLY_CFG,		0x00 },
+	{ CS42L42_ASP_TX_CH2_BIT_MSB,		0x00 },
+	{ CS42L42_ASP_TX_CH2_BIT_LSB,		0x00 },
+	{ CS42L42_ASP_RX_DAI0_EN,		0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH1_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI0_CH1_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH1_BIT_LSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH2_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI0_CH2_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH2_BIT_LSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH3_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI0_CH3_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH3_BIT_LSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH4_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI0_CH4_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI0_CH4_BIT_LSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI1_CH1_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI1_CH1_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI1_CH1_BIT_LSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI1_CH2_AP_RES,	0x03 },
+	{ CS42L42_ASP_RX_DAI1_CH2_BIT_MSB,	0x00 },
+	{ CS42L42_ASP_RX_DAI1_CH2_BIT_LSB,	0x00 },
+};
+
+/*
+ * This is all the same as for CS42L42 but we
+ * replace the on-reset register defaults.
+ */
+const struct regmap_config cs42l83_regmap = {
+	.reg_bits = 8,
+	.val_bits = 8,
+
+	.readable_reg = cs42l42_readable_register,
+	.volatile_reg = cs42l42_volatile_register,
+
+	.ranges = &cs42l42_page_range,
+	.num_ranges = 1,
+
+	.max_register = CS42L42_MAX_REGISTER,
+	.reg_defaults = cs42l83_reg_defaults,
+	.num_reg_defaults = ARRAY_SIZE(cs42l83_reg_defaults),
+	.cache_type = REGCACHE_RBTREE,
+
+	.use_single_read = true,
+	.use_single_write = true,
+};
+
+static int cs42l83_i2c_probe(struct i2c_client *i2c_client)
+{
+	struct device *dev = &i2c_client->dev;
+	struct cs42l42_private *cs42l83;
+	struct regmap *regmap;
+	int ret;
+
+	cs42l83 = devm_kzalloc(dev, sizeof(*cs42l83), GFP_KERNEL);
+	if (!cs42l83)
+		return -ENOMEM;
+
+	regmap = devm_regmap_init_i2c(i2c_client, &cs42l83_regmap);
+	if (IS_ERR(regmap))
+		return dev_err_probe(&i2c_client->dev, PTR_ERR(regmap),
+				     "regmap_init() failed\n");
+
+	cs42l83->devid = CS42L83_CHIP_ID;
+	cs42l83->dev = dev;
+	cs42l83->regmap = regmap;
+	cs42l83->irq = i2c_client->irq;
+
+	ret = cs42l42_common_probe(cs42l83, &cs42l42_soc_component, &cs42l42_dai);
+	if (ret)
+		return ret;
+
+	return cs42l42_init(cs42l83);
+}
+
+static void cs42l83_i2c_remove(struct i2c_client *i2c_client)
+{
+	struct cs42l42_private *cs42l83 = dev_get_drvdata(&i2c_client->dev);
+
+	cs42l42_common_remove(cs42l83);
+}
+
+static int __maybe_unused cs42l83_i2c_resume(struct device *dev)
+{
+	int ret;
+
+	ret = cs42l42_resume(dev);
+	if (ret)
+		return ret;
+
+	cs42l42_resume_restore(dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops cs42l83_i2c_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(cs42l42_suspend, cs42l83_i2c_resume)
+};
+
+static const struct of_device_id __maybe_unused cs42l83_of_match[] = {
+	{ .compatible = "cirrus,cs42l83", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, cs42l83_of_match);
+
+static struct i2c_driver cs42l83_i2c_driver = {
+	.driver = {
+		.name = "cs42l83",
+		.pm = &cs42l83_i2c_pm_ops,
+		.of_match_table = of_match_ptr(cs42l83_of_match),
+		},
+	.probe_new = cs42l83_i2c_probe,
+	.remove = cs42l83_i2c_remove,
+};
+
+module_i2c_driver(cs42l83_i2c_driver);
+
+MODULE_DESCRIPTION("ASoC CS42L83 I2C driver");
+MODULE_AUTHOR("Martin Povišer <povik+lin@cutebit.org>");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS(SND_SOC_CS42L42_CORE);
diff --git a/sound/soc/codecs/cs42xx8-i2c.c b/sound/soc/codecs/cs42xx8-i2c.c
index cb06a06d48b0..bd80e9fc907f 100644
--- a/sound/soc/codecs/cs42xx8-i2c.c
+++ b/sound/soc/codecs/cs42xx8-i2c.c
@@ -30,11 +30,9 @@ static int cs42xx8_i2c_probe(struct i2c_client *i2c)
 	return 0;
 }
 
-static int cs42xx8_i2c_remove(struct i2c_client *i2c)
+static void cs42xx8_i2c_remove(struct i2c_client *i2c)
 {
 	pm_runtime_disable(&i2c->dev);
-
-	return 0;
 }
 
 static struct i2c_device_id cs42xx8_i2c_id[] = {
diff --git a/sound/soc/codecs/cs43130.c b/sound/soc/codecs/cs43130.c
index ca4d47cc9c91..db39abb2a31b 100644
--- a/sound/soc/codecs/cs43130.c
+++ b/sound/soc/codecs/cs43130.c
@@ -1666,10 +1666,9 @@ static int cs43130_show_dc(struct device *dev, char *buf, u8 ch)
 	struct cs43130_private *cs43130 = i2c_get_clientdata(client);
 
 	if (!cs43130->hpload_done)
-		return scnprintf(buf, PAGE_SIZE, "NO_HPLOAD\n");
+		return sysfs_emit(buf, "NO_HPLOAD\n");
 	else
-		return scnprintf(buf, PAGE_SIZE, "%u\n",
-				 cs43130->hpload_dc[ch]);
+		return sysfs_emit(buf, "%u\n", cs43130->hpload_dc[ch]);
 }
 
 static ssize_t hpload_dc_l_show(struct device *dev,
@@ -1705,8 +1704,8 @@ static int cs43130_show_ac(struct device *dev, char *buf, u8 ch)
 
 	if (cs43130->hpload_done && cs43130->ac_meas) {
 		for (i = 0; i < ARRAY_SIZE(cs43130_ac_freq); i++) {
-			tmp = scnprintf(buf + j, PAGE_SIZE - j, "%u\n",
-					cs43130->hpload_ac[i][ch]);
+			tmp = sysfs_emit_at(buf, j, "%u\n",
+					    cs43130->hpload_ac[i][ch]);
 			if (!tmp)
 				break;
 
@@ -1715,7 +1714,7 @@ static int cs43130_show_ac(struct device *dev, char *buf, u8 ch)
 
 		return j;
 	} else {
-		return scnprintf(buf, PAGE_SIZE, "NO_HPLOAD\n");
+		return sysfs_emit(buf, "NO_HPLOAD\n");
 	}
 }
 
@@ -2583,7 +2582,7 @@ err_supplies:
 	return ret;
 }
 
-static int cs43130_i2c_remove(struct i2c_client *client)
+static void cs43130_i2c_remove(struct i2c_client *client)
 {
 	struct cs43130_private *cs43130 = i2c_get_clientdata(client);
 
@@ -2610,8 +2609,6 @@ static int cs43130_i2c_remove(struct i2c_client *client)
 
 	pm_runtime_disable(&client->dev);
 	regulator_bulk_disable(CS43130_NUM_SUPPLIES, cs43130->supplies);
-
-	return 0;
 }
 
 static int __maybe_unused cs43130_runtime_suspend(struct device *dev)
diff --git a/sound/soc/codecs/cs4349.c b/sound/soc/codecs/cs4349.c
index f7c5c2fd4304..ba94ffd0a7e4 100644
--- a/sound/soc/codecs/cs4349.c
+++ b/sound/soc/codecs/cs4349.c
@@ -305,14 +305,12 @@ static int cs4349_i2c_probe(struct i2c_client *client)
 		&cs4349_dai, 1);
 }
 
-static int cs4349_i2c_remove(struct i2c_client *client)
+static void cs4349_i2c_remove(struct i2c_client *client)
 {
 	struct cs4349_private *cs4349 = i2c_get_clientdata(client);
 
 	/* Hold down reset */
 	gpiod_set_value_cansleep(cs4349->reset_gpio, 0);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/sound/soc/codecs/cs53l30.c b/sound/soc/codecs/cs53l30.c
index 8796d8e84b7a..69db0013d243 100644
--- a/sound/soc/codecs/cs53l30.c
+++ b/sound/soc/codecs/cs53l30.c
@@ -1043,7 +1043,7 @@ error_supplies:
 	return ret;
 }
 
-static int cs53l30_i2c_remove(struct i2c_client *client)
+static void cs53l30_i2c_remove(struct i2c_client *client)
 {
 	struct cs53l30_private *cs53l30 = i2c_get_clientdata(client);
 
@@ -1052,8 +1052,6 @@ static int cs53l30_i2c_remove(struct i2c_client *client)
 
 	regulator_bulk_disable(ARRAY_SIZE(cs53l30->supplies),
 			       cs53l30->supplies);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/sound/soc/codecs/cx2072x.c b/sound/soc/codecs/cx2072x.c
index b6667e8a6099..5deceaa89282 100644
--- a/sound/soc/codecs/cx2072x.c
+++ b/sound/soc/codecs/cx2072x.c
@@ -1673,10 +1673,9 @@ static int cx2072x_i2c_probe(struct i2c_client *i2c)
 	return 0;
 }
 
-static int cx2072x_i2c_remove(struct i2c_client *i2c)
+static void cx2072x_i2c_remove(struct i2c_client *i2c)
 {
 	pm_runtime_disable(&i2c->dev);
-	return 0;
 }
 
 static const struct i2c_device_id cx2072x_i2c_id[] = {
diff --git a/sound/soc/codecs/da7219.c b/sound/soc/codecs/da7219.c
index 50ecf30e6136..4746c8700451 100644
--- a/sound/soc/codecs/da7219.c
+++ b/sound/soc/codecs/da7219.c
@@ -2196,6 +2196,7 @@ static int da7219_register_dai_clks(struct snd_soc_component *component)
 			dai_clk_lookup = clkdev_hw_create(dai_clk_hw, init.name,
 							  "%s", dev_name(dev));
 			if (!dai_clk_lookup) {
+				clk_hw_unregister(dai_clk_hw);
 				ret = -ENOMEM;
 				goto err;
 			} else {
@@ -2217,12 +2218,12 @@ static int da7219_register_dai_clks(struct snd_soc_component *component)
 	return 0;
 
 err:
-	do {
+	while (--i >= 0) {
 		if (da7219->dai_clks_lookup[i])
 			clkdev_drop(da7219->dai_clks_lookup[i]);
 
 		clk_hw_unregister(&da7219->dai_clks_hw[i]);
-	} while (i-- > 0);
+	}
 
 	if (np)
 		kfree(da7219->clk_hw_data);
diff --git a/sound/soc/codecs/es8316.c b/sound/soc/codecs/es8316.c
index de7185f73e1e..056c3082fe02 100644
--- a/sound/soc/codecs/es8316.c
+++ b/sound/soc/codecs/es8316.c
@@ -767,9 +767,31 @@ static void es8316_remove(struct snd_soc_component *component)
 	clk_disable_unprepare(es8316->mclk);
 }
 
+static int es8316_resume(struct snd_soc_component *component)
+{
+	struct es8316_priv *es8316 = snd_soc_component_get_drvdata(component);
+
+	regcache_cache_only(es8316->regmap, false);
+	regcache_sync(es8316->regmap);
+
+	return 0;
+}
+
+static int es8316_suspend(struct snd_soc_component *component)
+{
+	struct es8316_priv *es8316 = snd_soc_component_get_drvdata(component);
+
+	regcache_cache_only(es8316->regmap, true);
+	regcache_mark_dirty(es8316->regmap);
+
+	return 0;
+}
+
 static const struct snd_soc_component_driver soc_component_dev_es8316 = {
 	.probe			= es8316_probe,
 	.remove			= es8316_remove,
+	.resume			= es8316_resume,
+	.suspend		= es8316_suspend,
 	.set_jack		= es8316_set_jack,
 	.controls		= es8316_snd_controls,
 	.num_controls		= ARRAY_SIZE(es8316_snd_controls),
@@ -793,6 +815,8 @@ static const struct regmap_access_table es8316_volatile_table = {
 static const struct regmap_config es8316_regmap = {
 	.reg_bits = 8,
 	.val_bits = 8,
+	.use_single_read = true,
+	.use_single_write = true,
 	.max_register = 0x53,
 	.volatile_table	= &es8316_volatile_table,
 	.cache_type = REGCACHE_RBTREE,
diff --git a/sound/soc/codecs/es8326.c b/sound/soc/codecs/es8326.c
new file mode 100755
index 000000000000..87c1cc16592b
--- /dev/null
+++ b/sound/soc/codecs/es8326.c
@@ -0,0 +1,905 @@
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// es8326.c -- es8326 ALSA SoC audio driver
+// Copyright Everest Semiconductor Co., Ltd
+//
+// Authors: David Yang <yangxiaohua@everest-semi.com>
+//
+
+#include <linux/clk.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <sound/jack.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+#include <sound/soc-dapm.h>
+#include <sound/tlv.h>
+#include "es8326.h"
+
+struct es8326_priv {
+	struct clk *mclk;
+	struct i2c_client *i2c;
+	struct regmap *regmap;
+	struct snd_soc_component *component;
+	struct delayed_work jack_detect_work;
+	struct delayed_work button_press_work;
+	struct snd_soc_jack *jack;
+	int irq;
+	/* The lock protects the situation that an irq is generated
+	 * while enabling or disabling or during an irq.
+	 */
+	struct mutex lock;
+	u8 mic1_src;
+	u8 mic2_src;
+	u8 jack_pol;
+	u8 interrupt_src;
+	u8 interrupt_clk;
+	bool jd_inverted;
+	unsigned int sysclk;
+};
+
+static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(dac_vol_tlv, -9550, 50, 0);
+static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(adc_vol_tlv, -9550, 50, 0);
+static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(adc_analog_pga_tlv, 0, 300, 0);
+static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(adc_pga_tlv, 0, 600, 0);
+static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(softramp_rate, 0, 100, 0);
+static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(drc_target_tlv, -3200, 200, 0);
+static const SNDRV_CTL_TLVD_DECLARE_DB_SCALE(drc_recovery_tlv, -125, 250, 0);
+
+static const char *const winsize[] = {
+	"0.25db/2  LRCK",
+	"0.25db/4  LRCK",
+	"0.25db/8  LRCK",
+	"0.25db/16  LRCK",
+	"0.25db/32  LRCK",
+	"0.25db/64  LRCK",
+	"0.25db/128  LRCK",
+	"0.25db/256  LRCK",
+	"0.25db/512  LRCK",
+	"0.25db/1024  LRCK",
+	"0.25db/2048  LRCK",
+	"0.25db/4096  LRCK",
+	"0.25db/8192  LRCK",
+	"0.25db/16384  LRCK",
+	"0.25db/32768  LRCK",
+	"0.25db/65536  LRCK",
+};
+
+static const char *const dacpol_txt[] =	{
+	"Normal", "R Invert", "L Invert", "L + R Invert" };
+
+static const struct soc_enum dacpol =
+	SOC_ENUM_SINGLE(ES8326_DAC_DSM, 4, 4, dacpol_txt);
+static const struct soc_enum alc_winsize =
+	SOC_ENUM_SINGLE(ES8326_ADC_RAMPRATE, 4, 16, winsize);
+static const struct soc_enum drc_winsize =
+	SOC_ENUM_SINGLE(ES8326_DRC_WINSIZE, 4, 16, winsize);
+
+static const struct snd_kcontrol_new es8326_snd_controls[] = {
+	SOC_SINGLE_TLV("DAC Playback Volume", ES8326_DAC_VOL, 0, 0xbf, 0, dac_vol_tlv),
+	SOC_ENUM("Playback Polarity", dacpol),
+	SOC_SINGLE_TLV("DAC Ramp Rate", ES8326_DAC_RAMPRATE, 0, 0x0f, 0, softramp_rate),
+	SOC_SINGLE_TLV("DRC Recovery Level", ES8326_DRC_RECOVERY, 0, 4, 0, drc_recovery_tlv),
+	SOC_ENUM("DRC Winsize", drc_winsize),
+	SOC_SINGLE_TLV("DRC Target Level", ES8326_DRC_WINSIZE, 0, 0x0f, 0, drc_target_tlv),
+
+	SOC_DOUBLE_R_TLV("ADC Capture Volume", ES8326_ADC1_VOL, ES8326_ADC2_VOL, 0, 0xff, 0,
+			 adc_vol_tlv),
+	SOC_DOUBLE_TLV("ADC PGA Volume", ES8326_ADC_SCALE, 4, 0, 5, 0, adc_pga_tlv),
+	SOC_SINGLE_TLV("ADC PGA Gain Volume", ES8326_PGAGAIN, 0, 10, 0, adc_analog_pga_tlv),
+	SOC_SINGLE_TLV("ADC Ramp Rate", ES8326_ADC_RAMPRATE, 0, 0x0f, 0, softramp_rate),
+	SOC_SINGLE("ALC Capture Switch", ES8326_ALC_RECOVERY, 3, 1, 0),
+	SOC_SINGLE_TLV("ALC Capture Recovery Level", ES8326_ALC_LEVEL,
+			0, 4, 0, drc_recovery_tlv),
+	SOC_ENUM("ALC Capture Winsize", alc_winsize),
+	SOC_SINGLE_TLV("ALC Capture Target Level", ES8326_ALC_LEVEL,
+			0, 0x0f, 0, drc_target_tlv),
+
+};
+
+static const struct snd_soc_dapm_widget es8326_dapm_widgets[] = {
+	SND_SOC_DAPM_INPUT("MIC1"),
+	SND_SOC_DAPM_INPUT("MIC2"),
+	SND_SOC_DAPM_INPUT("MIC3"),
+	SND_SOC_DAPM_INPUT("MIC4"),
+
+	SND_SOC_DAPM_ADC("ADC L", NULL, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_ADC("ADC R", NULL, SND_SOC_NOPM, 0, 0),
+
+	/* Digital Interface */
+	SND_SOC_DAPM_AIF_OUT("I2S OUT", "I2S1 Capture", 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("I2S IN", "I2S1 Playback", 0, SND_SOC_NOPM, 0, 0),
+
+	/* ADC Digital Mute */
+	SND_SOC_DAPM_PGA("ADC L1", ES8326_ADC_MUTE, 0, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("ADC R1", ES8326_ADC_MUTE, 1, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("ADC L2", ES8326_ADC_MUTE, 2, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("ADC R2", ES8326_ADC_MUTE, 3, 1, NULL, 0),
+
+	/* Analog Power Supply*/
+	SND_SOC_DAPM_DAC("Right DAC", NULL, ES8326_ANA_PDN, 0, 1),
+	SND_SOC_DAPM_DAC("Left DAC", NULL, ES8326_ANA_PDN, 1, 1),
+	SND_SOC_DAPM_SUPPLY("Analog Power", ES8326_ANA_PDN, 7, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("IBias Power", ES8326_ANA_PDN, 6, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("ADC Vref", ES8326_ANA_PDN, 5, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("DAC Vref", ES8326_ANA_PDN, 4, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("Vref Power", ES8326_ANA_PDN, 3, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("MICBIAS1", ES8326_ANA_MICBIAS, 2, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("MICBIAS2", ES8326_ANA_MICBIAS, 3, 0, NULL, 0),
+
+	SND_SOC_DAPM_PGA("LHPMIX", ES8326_DAC2HPMIX, 7, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("RHPMIX", ES8326_DAC2HPMIX, 3, 0, NULL, 0),
+
+	/* Headphone Charge Pump and Output */
+	SND_SOC_DAPM_SUPPLY("HPOR Cal", ES8326_HP_CAL, 7, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("HPOL Cal", ES8326_HP_CAL, 3, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("Headphone Charge Pump", ES8326_HP_DRIVER,
+			    3, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("Headphone Driver Bias", ES8326_HP_DRIVER,
+			    2, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("Headphone LDO", ES8326_HP_DRIVER,
+			    1, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("Headphone Reference", ES8326_HP_DRIVER,
+			    0, 1, NULL, 0),
+	SND_SOC_DAPM_REG(snd_soc_dapm_supply, "HPOR Supply", ES8326_HP_CAL,
+			 ES8326_HPOR_SHIFT, 7, 7, 0),
+	SND_SOC_DAPM_REG(snd_soc_dapm_supply, "HPOL Supply", ES8326_HP_CAL,
+			 0, 7, 7, 0),
+
+	SND_SOC_DAPM_OUTPUT("HPOL"),
+	SND_SOC_DAPM_OUTPUT("HPOR"),
+};
+
+static const struct snd_soc_dapm_route es8326_dapm_routes[] = {
+	{"ADC L1", NULL, "MIC1"},
+	{"ADC R1", NULL, "MIC2"},
+	{"ADC L2", NULL, "MIC3"},
+	{"ADC R2", NULL, "MIC4"},
+
+	{"ADC L", NULL, "ADC L1"},
+	{"ADC R", NULL, "ADC R1"},
+	{"ADC L", NULL, "ADC L2"},
+	{"ADC R", NULL, "ADC R2"},
+
+	{"I2S OUT", NULL, "ADC L"},
+	{"I2S OUT", NULL, "ADC R"},
+
+	{"I2S OUT", NULL, "Analog Power"},
+	{"I2S OUT", NULL, "ADC Vref"},
+	{"I2S OUT", NULL, "Vref Power"},
+	{"I2S OUT", NULL, "IBias Power"},
+	{"I2S IN", NULL, "Analog Power"},
+	{"I2S IN", NULL, "DAC Vref"},
+	{"I2S IN", NULL, "Vref Power"},
+	{"I2S IN", NULL, "IBias Power"},
+
+	{"Right DAC", NULL, "I2S IN"},
+	{"Left DAC", NULL, "I2S IN"},
+
+	{"LHPMIX", NULL, "Left DAC"},
+	{"RHPMIX", NULL, "Right DAC"},
+
+	{"HPOR", NULL, "HPOR Cal"},
+	{"HPOL", NULL, "HPOL Cal"},
+	{"HPOR", NULL, "HPOR Supply"},
+	{"HPOL", NULL, "HPOL Supply"},
+	{"HPOL", NULL, "Headphone Charge Pump"},
+	{"HPOR", NULL, "Headphone Charge Pump"},
+	{"HPOL", NULL, "Headphone Driver Bias"},
+	{"HPOR", NULL, "Headphone Driver Bias"},
+	{"HPOL", NULL, "Headphone LDO"},
+	{"HPOR", NULL, "Headphone LDO"},
+	{"HPOL", NULL, "Headphone Reference"},
+	{"HPOR", NULL, "Headphone Reference"},
+
+	{"HPOL", NULL, "LHPMIX"},
+	{"HPOR", NULL, "RHPMIX"},
+};
+
+static const struct regmap_range es8326_volatile_ranges[] = {
+	regmap_reg_range(ES8326_HP_DETECT, ES8326_HP_DETECT),
+};
+
+static const struct regmap_access_table es8326_volatile_table = {
+	.yes_ranges = es8326_volatile_ranges,
+	.n_yes_ranges = ARRAY_SIZE(es8326_volatile_ranges),
+};
+
+static const struct regmap_config es8326_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = 0xff,
+	.volatile_table = &es8326_volatile_table,
+	.cache_type = REGCACHE_RBTREE,
+};
+
+struct _coeff_div {
+	u16 fs;
+	u32 rate;
+	u32 mclk;
+	u8 reg4;
+	u8 reg5;
+	u8 reg6;
+	u8 reg7;
+	u8 reg8;
+	u8 reg9;
+	u8 rega;
+	u8 regb;
+};
+
+/* codec hifi mclk clock divider coefficients */
+/* {ratio, LRCK, MCLK, REG04, REG05, REG06, REG07, REG08, REG09, REG10, REG11} */
+static const struct _coeff_div coeff_div[] = {
+	{32, 8000, 256000, 0x60, 0x00, 0x0F, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
+	{32, 16000, 512000, 0x20, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x3F},
+	{32, 44100, 1411200, 0x00, 0x00, 0x13, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{32, 48000, 1536000, 0x00, 0x00, 0x13, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{36, 8000, 288000, 0x20, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x23, 0x47},
+	{36, 16000, 576000, 0x20, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x23, 0x47},
+	{48, 8000, 384000, 0x60, 0x02, 0x1F, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
+	{48, 16000, 768000, 0x20, 0x02, 0x0F, 0x75, 0x0A, 0x1B, 0x1F, 0x3F},
+	{48, 48000, 2304000, 0x00, 0x02, 0x0D, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{64, 8000, 512000, 0x60, 0x00, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
+	{64, 16000, 1024000, 0x20, 0x00, 0x05, 0x75, 0x0A, 0x1B, 0x1F, 0x3F},
+
+	{64, 44100, 2822400, 0x00, 0x00, 0x11, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{64, 48000, 3072000, 0x00, 0x00, 0x11, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{72, 8000, 576000, 0x20, 0x00, 0x13, 0x35, 0x0A, 0x1B, 0x23, 0x47},
+	{72, 16000, 1152000, 0x20, 0x00, 0x05, 0x75, 0x0A, 0x1B, 0x23, 0x47},
+	{96, 8000, 768000, 0x60, 0x02, 0x1D, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
+	{96, 16000, 1536000, 0x20, 0x02, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x3F},
+	{100, 48000, 4800000, 0x04, 0x04, 0x3F, 0x6D, 0x38, 0x08, 0x4f, 0x1f},
+	{125, 48000, 6000000, 0x04, 0x04, 0x1F, 0x2D, 0x0A, 0x0A, 0x27, 0x27},
+	{128, 8000, 1024000, 0x60, 0x00, 0x13, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+	{128, 16000, 2048000, 0x20, 0x00, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+
+	{128, 44100, 5644800, 0x00, 0x00, 0x01, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{128, 48000, 6144000, 0x00, 0x00, 0x01, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{144, 8000, 1152000, 0x20, 0x00, 0x03, 0x35, 0x0A, 0x1B, 0x23, 0x47},
+	{144, 16000, 2304000, 0x20, 0x00, 0x11, 0x35, 0x0A, 0x1B, 0x23, 0x47},
+	{192, 8000, 1536000, 0x60, 0x02, 0x0D, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
+	{192, 16000, 3072000, 0x20, 0x02, 0x05, 0x75, 0x0A, 0x1B, 0x1F, 0x3F},
+	{200, 48000, 9600000, 0x04, 0x04, 0x0F, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{250, 48000, 12000000, 0x04, 0x04, 0x0F, 0x2D, 0x0A, 0x0A, 0x27, 0x27},
+	{256, 8000, 2048000, 0x60, 0x00, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+	{256, 16000, 4096000, 0x20, 0x00, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+
+	{256, 44100, 11289600, 0x00, 0x00, 0x10, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{256, 48000, 12288000, 0x00, 0x00, 0x30, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{288, 8000, 2304000, 0x20, 0x00, 0x01, 0x35, 0x0A, 0x1B, 0x23, 0x47},
+	{384, 8000, 3072000, 0x60, 0x02, 0x05, 0x75, 0x0A, 0x1B, 0x1F, 0x7F},
+	{384, 16000, 6144000, 0x20, 0x02, 0x03, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+	{384, 48000, 18432000, 0x00, 0x02, 0x01, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{400, 48000, 19200000, 0x09, 0x04, 0x0f, 0x6d, 0x3a, 0x0A, 0x4F, 0x1F},
+	{500, 48000, 24000000, 0x18, 0x04, 0x1F, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{512, 8000, 4096000, 0x60, 0x00, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+	{512, 16000, 8192000, 0x20, 0x00, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+
+	{512, 44100, 22579200, 0x00, 0x00, 0x00, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{512, 48000, 24576000, 0x00, 0x00, 0x00, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{768, 8000, 6144000, 0x60, 0x02, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+	{768, 16000, 12288000, 0x20, 0x02, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+	{800, 48000, 38400000, 0x00, 0x18, 0x13, 0x2D, 0x0A, 0x0A, 0x1F, 0x1F},
+	{1024, 8000, 8192000, 0x60, 0x00, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+	{1024, 16000, 16384000, 0x20, 0x00, 0x00, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+	{1152, 16000, 18432000, 0x20, 0x08, 0x11, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+	{1536, 8000, 12288000, 0x60, 0x02, 0x01, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+
+	{1536, 16000, 24576000, 0x20, 0x02, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x3F},
+	{1625, 8000, 13000000, 0x0C, 0x18, 0x1F, 0x2D, 0x0A, 0x0A, 0x27, 0x27},
+	{1625, 16000, 26000000, 0x0C, 0x18, 0x1F, 0x2D, 0x0A, 0x0A, 0x27, 0x27},
+	{2048, 8000, 16384000, 0x60, 0x00, 0x00, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+	{2304, 8000, 18432000, 0x40, 0x02, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x5F},
+	{3072, 8000, 24576000, 0x60, 0x02, 0x10, 0x35, 0x0A, 0x1B, 0x1F, 0x7F},
+	{3250, 8000, 26000000, 0x0C, 0x18, 0x0F, 0x2D, 0x0A, 0x0A, 0x27, 0x27},
+
+};
+
+static inline int get_coeff(int mclk, int rate)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(coeff_div); i++) {
+		if (coeff_div[i].rate == rate && coeff_div[i].mclk == mclk)
+			return i;
+	}
+
+	return -EINVAL;
+}
+
+static int es8326_set_dai_sysclk(struct snd_soc_dai *codec_dai,
+				 int clk_id, unsigned int freq, int dir)
+{
+	struct snd_soc_component *codec = codec_dai->component;
+	struct es8326_priv *es8326 = snd_soc_component_get_drvdata(codec);
+
+	es8326->sysclk = freq;
+
+	return 0;
+}
+
+static int es8326_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
+{
+	struct snd_soc_component *component = codec_dai->component;
+	u8 iface = 0;
+
+	switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) {
+	case SND_SOC_DAIFMT_CBC_CFP:
+		snd_soc_component_update_bits(component, ES8326_RESET,
+					      ES8326_MASTER_MODE_EN, ES8326_MASTER_MODE_EN);
+		break;
+	case SND_SOC_DAIFMT_CBC_CFC:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* interface format */
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		break;
+	case SND_SOC_DAIFMT_RIGHT_J:
+		dev_err(component->dev, "Codec driver does not support right justified\n");
+		return -EINVAL;
+	case SND_SOC_DAIFMT_LEFT_J:
+		iface |= ES8326_DAIFMT_LEFT_J;
+		break;
+	case SND_SOC_DAIFMT_DSP_A:
+		iface |= ES8326_DAIFMT_DSP_A;
+		break;
+	case SND_SOC_DAIFMT_DSP_B:
+		iface |= ES8326_DAIFMT_DSP_B;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	snd_soc_component_update_bits(component, ES8326_FMT, ES8326_DAIFMT_MASK, iface);
+
+	return 0;
+}
+
+static int es8326_pcm_hw_params(struct snd_pcm_substream *substream,
+				struct snd_pcm_hw_params *params,
+				struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component);
+	u8 srate = 0;
+	int coeff;
+
+	coeff = get_coeff(es8326->sysclk, params_rate(params));
+	/* bit size */
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16_LE:
+		srate |= ES8326_S16_LE;
+		break;
+	case SNDRV_PCM_FORMAT_S20_3LE:
+		srate |= ES8326_S20_3_LE;
+		break;
+	case SNDRV_PCM_FORMAT_S18_3LE:
+		srate |= ES8326_S18_LE;
+		break;
+	case SNDRV_PCM_FORMAT_S24_LE:
+		srate |= ES8326_S24_LE;
+		break;
+	case SNDRV_PCM_FORMAT_S32_LE:
+		srate |= ES8326_S32_LE;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* set iface & srate */
+	snd_soc_component_update_bits(component, ES8326_FMT, ES8326_DATA_LEN_MASK, srate);
+
+	if (coeff >= 0) {
+		regmap_write(es8326->regmap,  ES8326_CLK_DIV1,
+			     coeff_div[coeff].reg4);
+		regmap_write(es8326->regmap,  ES8326_CLK_DIV2,
+			     coeff_div[coeff].reg5);
+		regmap_write(es8326->regmap,  ES8326_CLK_DLL,
+			     coeff_div[coeff].reg6);
+		regmap_write(es8326->regmap,  ES8326_CLK_MUX,
+			     coeff_div[coeff].reg7);
+		regmap_write(es8326->regmap,  ES8326_CLK_ADC_SEL,
+			     coeff_div[coeff].reg8);
+		regmap_write(es8326->regmap,  ES8326_CLK_DAC_SEL,
+			     coeff_div[coeff].reg9);
+		regmap_write(es8326->regmap,  ES8326_CLK_ADC_OSR,
+			     coeff_div[coeff].rega);
+		regmap_write(es8326->regmap,  ES8326_CLK_DAC_OSR,
+			     coeff_div[coeff].regb);
+	} else {
+		dev_warn(component->dev, "Clock coefficients do not match");
+	}
+
+	return 0;
+}
+
+static int es8326_set_bias_level(struct snd_soc_component *codec,
+				 enum snd_soc_bias_level level)
+{
+	struct es8326_priv *es8326 = snd_soc_component_get_drvdata(codec);
+	int ret;
+
+	switch (level) {
+	case SND_SOC_BIAS_ON:
+		ret = clk_prepare_enable(es8326->mclk);
+		if (ret)
+			return ret;
+		regmap_write(es8326->regmap, ES8326_RESET, ES8326_PWRUP_SEQ_EN);
+		regmap_write(es8326->regmap, ES8326_INTOUT_IO, 0x45);
+		regmap_write(es8326->regmap, ES8326_SDINOUT1_IO,
+			    (ES8326_IO_DMIC_CLK << ES8326_SDINOUT1_SHIFT));
+		regmap_write(es8326->regmap, ES8326_SDINOUT23_IO, ES8326_IO_INPUT);
+		regmap_write(es8326->regmap, ES8326_CLK_RESAMPLE, 0x05);
+		regmap_write(es8326->regmap, ES8326_VMIDSEL, 0x02);
+		regmap_write(es8326->regmap, ES8326_PGA_PDN, 0x40);
+		regmap_write(es8326->regmap, ES8326_DAC2HPMIX, 0xAA);
+		regmap_write(es8326->regmap, ES8326_RESET, ES8326_CSM_ON);
+		break;
+	case SND_SOC_BIAS_PREPARE:
+		break;
+	case SND_SOC_BIAS_STANDBY:
+		break;
+	case SND_SOC_BIAS_OFF:
+		clk_disable_unprepare(es8326->mclk);
+		regmap_write(es8326->regmap, ES8326_DAC2HPMIX, 0x11);
+		regmap_write(es8326->regmap, ES8326_RESET, ES8326_CSM_OFF);
+		regmap_write(es8326->regmap, ES8326_PGA_PDN, 0xF8);
+		regmap_write(es8326->regmap, ES8326_VMIDSEL, 0x00);
+		regmap_write(es8326->regmap, ES8326_INT_SOURCE, 0x08);
+		regmap_write(es8326->regmap, ES8326_SDINOUT1_IO, ES8326_IO_INPUT);
+		regmap_write(es8326->regmap, ES8326_SDINOUT23_IO, ES8326_IO_INPUT);
+		regmap_write(es8326->regmap, ES8326_RESET,
+			     ES8326_CODEC_RESET | ES8326_PWRUP_SEQ_EN);
+		break;
+	}
+
+	return 0;
+}
+
+#define es8326_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_3LE |\
+	SNDRV_PCM_FMTBIT_S24_LE)
+
+static const struct snd_soc_dai_ops es8326_ops = {
+	.hw_params = es8326_pcm_hw_params,
+	.set_fmt = es8326_set_dai_fmt,
+	.set_sysclk = es8326_set_dai_sysclk,
+};
+
+static struct snd_soc_dai_driver es8326_dai = {
+	.name = "ES8326 HiFi",
+	.playback = {
+		.stream_name = "Playback",
+		.channels_min = 1,
+		.channels_max = 2,
+		.rates = SNDRV_PCM_RATE_8000_48000,
+		.formats = es8326_FORMATS,
+		},
+	.capture = {
+		.stream_name = "Capture",
+		.channels_min = 1,
+		.channels_max = 2,
+		.rates = SNDRV_PCM_RATE_8000_48000,
+		.formats = es8326_FORMATS,
+		},
+	.ops = &es8326_ops,
+	.symmetric_rate = 1,
+};
+
+static void es8326_enable_micbias(struct snd_soc_component *component)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component);
+
+	snd_soc_dapm_mutex_lock(dapm);
+	snd_soc_dapm_force_enable_pin_unlocked(dapm, "MICBIAS1");
+	snd_soc_dapm_force_enable_pin_unlocked(dapm, "MICBIAS2");
+	snd_soc_dapm_sync_unlocked(dapm);
+	snd_soc_dapm_mutex_unlock(dapm);
+}
+
+static void es8326_disable_micbias(struct snd_soc_component *component)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component);
+
+	snd_soc_dapm_mutex_lock(dapm);
+	snd_soc_dapm_disable_pin_unlocked(dapm, "MICBIAS1");
+	snd_soc_dapm_disable_pin_unlocked(dapm, "MICBIAS2");
+	snd_soc_dapm_sync_unlocked(dapm);
+	snd_soc_dapm_mutex_unlock(dapm);
+}
+
+/*
+ *	For button detection, set the following in soundcard
+ *	snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_PLAYPAUSE);
+ *	snd_jack_set_key(jack->jack, SND_JACK_BTN_1, KEY_VOLUMEUP);
+ *	snd_jack_set_key(jack->jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN);
+ */
+static void es8326_jack_button_handler(struct work_struct *work)
+{
+	struct es8326_priv *es8326 =
+		container_of(work, struct es8326_priv, button_press_work.work);
+	struct snd_soc_component *comp = es8326->component;
+	unsigned int iface;
+	static int button_to_report, press_count;
+	static int prev_button, cur_button;
+
+	if (!(es8326->jack->status & SND_JACK_HEADSET)) /* Jack unplugged */
+		return;
+
+	mutex_lock(&es8326->lock);
+	iface = snd_soc_component_read(comp, ES8326_HP_DETECT);
+	switch (iface) {
+	case 0x93:
+		/* pause button detected */
+		cur_button = SND_JACK_BTN_0;
+		break;
+	case 0x6f:
+		/* button volume up */
+		cur_button = SND_JACK_BTN_1;
+		break;
+	case 0x27:
+		/* button volume down */
+		cur_button = SND_JACK_BTN_2;
+		break;
+	case 0x1e:
+		/* button released or not pressed */
+		cur_button = 0;
+		break;
+	default:
+		break;
+	}
+
+	if ((prev_button == cur_button) && (cur_button != 0)) {
+		press_count++;
+		if (press_count > 10) {
+			/* report a press every 500ms */
+			snd_soc_jack_report(es8326->jack, cur_button,
+					SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2);
+			press_count = 0;
+		}
+		button_to_report = cur_button;
+		queue_delayed_work(system_wq, &es8326->button_press_work,
+				   msecs_to_jiffies(50));
+	} else if (prev_button != cur_button) {
+		/* mismatch, detect again */
+		prev_button = cur_button;
+		queue_delayed_work(system_wq, &es8326->button_press_work,
+				   msecs_to_jiffies(50));
+	} else {
+		/* released or no pressed */
+		if (button_to_report != 0) {
+			snd_soc_jack_report(es8326->jack, button_to_report,
+				    SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2);
+			snd_soc_jack_report(es8326->jack, 0,
+				    SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2);
+			button_to_report = 0;
+		}
+	}
+	mutex_unlock(&es8326->lock);
+}
+
+static void es8326_jack_detect_handler(struct work_struct *work)
+{
+	struct es8326_priv *es8326 =
+		container_of(work, struct es8326_priv, jack_detect_work.work);
+	struct snd_soc_component *comp = es8326->component;
+	unsigned int iface;
+
+	mutex_lock(&es8326->lock);
+	iface = snd_soc_component_read(comp, ES8326_HP_DETECT);
+	dev_dbg(comp->dev, "gpio flag %#04x", iface);
+	if ((iface & ES8326_HPINSERT_FLAG) == 0) {
+		/* Jack unplugged or spurious IRQ */
+		dev_dbg(comp->dev, "No headset detected");
+		if (es8326->jack->status & SND_JACK_HEADPHONE) {
+			snd_soc_jack_report(es8326->jack, 0, SND_JACK_HEADSET);
+			snd_soc_component_write(comp, ES8326_ADC1_SRC, es8326->mic2_src);
+			es8326_disable_micbias(comp);
+		}
+	} else if ((iface & ES8326_HPINSERT_FLAG) == ES8326_HPINSERT_FLAG) {
+		if (es8326->jack->status & SND_JACK_HEADSET) {
+			/* detect button */
+			queue_delayed_work(system_wq, &es8326->button_press_work, 10);
+		} else {
+			if ((iface & ES8326_HPBUTTON_FLAG) == 0x00) {
+				dev_dbg(comp->dev, "Headset detected");
+				snd_soc_jack_report(es8326->jack,
+						    SND_JACK_HEADSET, SND_JACK_HEADSET);
+				snd_soc_component_write(comp,
+							ES8326_ADC1_SRC, es8326->mic1_src);
+			} else {
+				dev_dbg(comp->dev, "Headphone detected");
+				snd_soc_jack_report(es8326->jack,
+						    SND_JACK_HEADPHONE, SND_JACK_HEADSET);
+			}
+		}
+	}
+	mutex_unlock(&es8326->lock);
+}
+
+static irqreturn_t es8326_irq(int irq, void *dev_id)
+{
+	struct es8326_priv *es8326 = dev_id;
+	struct snd_soc_component *comp = es8326->component;
+
+	if (!es8326->jack)
+		goto out;
+
+	es8326_enable_micbias(comp);
+
+	if (es8326->jack->status & SND_JACK_HEADSET)
+		queue_delayed_work(system_wq, &es8326->jack_detect_work,
+				   msecs_to_jiffies(10));
+	else
+		queue_delayed_work(system_wq, &es8326->jack_detect_work,
+				   msecs_to_jiffies(300));
+
+out:
+	return IRQ_HANDLED;
+}
+
+static int es8326_resume(struct snd_soc_component *component)
+{
+	struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component);
+	unsigned int reg;
+
+	regcache_cache_only(es8326->regmap, false);
+	regcache_sync(es8326->regmap);
+
+	regmap_write(es8326->regmap, ES8326_CLK_CTL, ES8326_CLK_ON);
+	/* Two channel ADC */
+	regmap_write(es8326->regmap, ES8326_PULLUP_CTL, 0x02);
+	regmap_write(es8326->regmap, ES8326_CLK_INV, 0x00);
+	regmap_write(es8326->regmap, ES8326_CLK_DIV_CPC, 0x1F);
+	regmap_write(es8326->regmap, ES8326_CLK_VMIDS1, 0xC8);
+	regmap_write(es8326->regmap, ES8326_CLK_VMIDS2, 0x88);
+	regmap_write(es8326->regmap, ES8326_CLK_CAL_TIME, 0x20);
+	regmap_write(es8326->regmap, ES8326_SYS_BIAS, 0x08);
+	regmap_write(es8326->regmap, ES8326_DAC2HPMIX, 0x22);
+	regmap_write(es8326->regmap, ES8326_ADC1_SRC, es8326->mic1_src);
+	regmap_write(es8326->regmap, ES8326_ADC2_SRC, es8326->mic2_src);
+	regmap_write(es8326->regmap, ES8326_HPJACK_TIMER, 0x88);
+	regmap_write(es8326->regmap, ES8326_HP_DET,
+		     ES8326_HP_DET_SRC_PIN9 | es8326->jack_pol);
+	regmap_write(es8326->regmap, ES8326_INT_SOURCE, es8326->interrupt_src);
+	regmap_write(es8326->regmap, ES8326_INTOUT_IO, es8326->interrupt_clk);
+	regmap_write(es8326->regmap, ES8326_RESET, ES8326_CSM_ON);
+	snd_soc_component_update_bits(component, ES8326_PGAGAIN,
+				      ES8326_MIC_SEL_MASK, ES8326_MIC1_SEL);
+
+	regmap_read(es8326->regmap, ES8326_CHIP_VERSION, &reg);
+	if ((reg & ES8326_VERSION_B) == 1) {
+		regmap_write(es8326->regmap, ES8326_ANA_MICBIAS, 0xDD);
+		regmap_write(es8326->regmap, ES8326_ANA_VSEL, 0x7F);
+		regmap_write(es8326->regmap, ES8326_VMIDLOW, 0x0F);
+		/* enable button detect */
+		regmap_write(es8326->regmap, ES8326_HP_DRIVER, 0xA0);
+	}
+
+	es8326_irq(es8326->irq, es8326);
+	return 0;
+}
+
+static int es8326_suspend(struct snd_soc_component *component)
+{
+	struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component);
+
+	cancel_delayed_work_sync(&es8326->jack_detect_work);
+	es8326_disable_micbias(component);
+
+	regmap_write(es8326->regmap, ES8326_CLK_CTL, ES8326_CLK_OFF);
+	regcache_cache_only(es8326->regmap, true);
+	regcache_mark_dirty(es8326->regmap);
+
+	return 0;
+}
+
+static int es8326_probe(struct snd_soc_component *component)
+{
+	struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component);
+	int ret;
+
+	es8326->component = component;
+	es8326->jd_inverted = device_property_read_bool(component->dev,
+							"everest,jack-detect-inverted");
+
+	ret = device_property_read_u8(component->dev, "everest,mic1-src", &es8326->mic1_src);
+	if (ret != 0) {
+		dev_dbg(component->dev, "mic1-src return %d", ret);
+		es8326->mic1_src = ES8326_ADC_AMIC;
+	}
+	dev_dbg(component->dev, "mic1-src %x", es8326->mic1_src);
+
+	ret = device_property_read_u8(component->dev, "everest,mic2-src", &es8326->mic2_src);
+	if (ret != 0) {
+		dev_dbg(component->dev, "mic2-src return %d", ret);
+		es8326->mic2_src = ES8326_ADC_DMIC;
+	}
+	dev_dbg(component->dev, "mic2-src %x", es8326->mic2_src);
+
+	ret = device_property_read_u8(component->dev, "everest,jack-pol", &es8326->jack_pol);
+	if (ret != 0) {
+		dev_dbg(component->dev, "jack-pol return %d", ret);
+		es8326->jack_pol = ES8326_HP_DET_BUTTON_POL | ES8326_HP_TYPE_OMTP;
+	}
+	dev_dbg(component->dev, "jack-pol %x", es8326->jack_pol);
+
+	ret = device_property_read_u8(component->dev, "everest,interrupt-src", &es8326->jack_pol);
+	if (ret != 0) {
+		dev_dbg(component->dev, "interrupt-src return %d", ret);
+		es8326->interrupt_src = ES8326_HP_DET_SRC_PIN9;
+	}
+	dev_dbg(component->dev, "interrupt-src %x", es8326->interrupt_src);
+
+	ret = device_property_read_u8(component->dev, "everest,interrupt-clk", &es8326->jack_pol);
+	if (ret != 0) {
+		dev_dbg(component->dev, "interrupt-clk return %d", ret);
+		es8326->interrupt_clk = 0x45;
+	}
+	dev_dbg(component->dev, "interrupt-clk %x", es8326->interrupt_clk);
+
+	es8326_resume(component);
+	return 0;
+}
+
+static void es8326_enable_jack_detect(struct snd_soc_component *component,
+				struct snd_soc_jack *jack)
+{
+	struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component);
+
+	mutex_lock(&es8326->lock);
+	if (es8326->jd_inverted)
+		snd_soc_component_update_bits(component, ES8326_HP_DET,
+					      ES8326_HP_DET_JACK_POL, ~es8326->jack_pol);
+	es8326->jack = jack;
+
+	mutex_unlock(&es8326->lock);
+	es8326_irq(es8326->irq, es8326);
+}
+
+static void es8326_disable_jack_detect(struct snd_soc_component *component)
+{
+	struct es8326_priv *es8326 = snd_soc_component_get_drvdata(component);
+
+	dev_dbg(component->dev, "Enter into %s\n", __func__);
+	if (!es8326->jack)
+		return; /* Already disabled (or never enabled) */
+	cancel_delayed_work_sync(&es8326->jack_detect_work);
+
+	mutex_lock(&es8326->lock);
+	if (es8326->jack->status & SND_JACK_MICROPHONE) {
+		es8326_disable_micbias(component);
+		snd_soc_jack_report(es8326->jack, 0, SND_JACK_HEADSET);
+	}
+	es8326->jack = NULL;
+	mutex_unlock(&es8326->lock);
+}
+
+static int es8326_set_jack(struct snd_soc_component *component,
+			struct snd_soc_jack *jack, void *data)
+{
+	if (jack)
+		es8326_enable_jack_detect(component, jack);
+	else
+		es8326_disable_jack_detect(component);
+
+	return 0;
+}
+
+static void es8326_remove(struct snd_soc_component *component)
+{
+	es8326_disable_jack_detect(component);
+	es8326_set_bias_level(component, SND_SOC_BIAS_OFF);
+}
+
+static const struct snd_soc_component_driver soc_component_dev_es8326 = {
+	.probe		= es8326_probe,
+	.remove		= es8326_remove,
+	.resume		= es8326_resume,
+	.suspend	= es8326_suspend,
+	.set_bias_level = es8326_set_bias_level,
+	.set_jack	= es8326_set_jack,
+	.dapm_widgets	= es8326_dapm_widgets,
+	.num_dapm_widgets	= ARRAY_SIZE(es8326_dapm_widgets),
+	.dapm_routes		= es8326_dapm_routes,
+	.num_dapm_routes	= ARRAY_SIZE(es8326_dapm_routes),
+	.controls		= es8326_snd_controls,
+	.num_controls		= ARRAY_SIZE(es8326_snd_controls),
+	.use_pmdown_time	= 1,
+	.endianness		= 1,
+};
+
+static int es8326_i2c_probe(struct i2c_client *i2c,
+			    const struct i2c_device_id *id)
+{
+	struct es8326_priv *es8326;
+	int ret;
+
+	es8326 = devm_kzalloc(&i2c->dev, sizeof(struct es8326_priv), GFP_KERNEL);
+	if (!es8326)
+		return -ENOMEM;
+
+	i2c_set_clientdata(i2c, es8326);
+	es8326->i2c = i2c;
+	mutex_init(&es8326->lock);
+	es8326->regmap = devm_regmap_init_i2c(i2c, &es8326_regmap_config);
+	if (IS_ERR(es8326->regmap)) {
+		ret = PTR_ERR(es8326->regmap);
+		dev_err(&i2c->dev, "Failed to init regmap: %d\n", ret);
+		return ret;
+	}
+
+	es8326->irq = i2c->irq;
+	INIT_DELAYED_WORK(&es8326->jack_detect_work,
+			  es8326_jack_detect_handler);
+	INIT_DELAYED_WORK(&es8326->button_press_work,
+			  es8326_jack_button_handler);
+	/* ES8316 is level-based while ES8326 is edge-based */
+	ret = devm_request_threaded_irq(&i2c->dev, es8326->irq, NULL, es8326_irq,
+					IRQF_TRIGGER_RISING | IRQF_ONESHOT,
+					"es8326", es8326);
+	if (ret) {
+		dev_warn(&i2c->dev, "Failed to request IRQ: %d: %d\n",
+		es8326->irq, ret);
+		es8326->irq = -ENXIO;
+	}
+
+	es8326->mclk = devm_clk_get_optional(&i2c->dev, "mclk");
+	if (IS_ERR(es8326->mclk)) {
+		dev_err(&i2c->dev, "unable to get mclk\n");
+		return PTR_ERR(es8326->mclk);
+	}
+	if (!es8326->mclk)
+		dev_warn(&i2c->dev, "assuming static mclk\n");
+
+	ret = clk_prepare_enable(es8326->mclk);
+	if (ret) {
+		dev_err(&i2c->dev, "unable to enable mclk\n");
+		return ret;
+	}
+	return devm_snd_soc_register_component(&i2c->dev,
+					&soc_component_dev_es8326,
+					&es8326_dai, 1);
+}
+
+static const struct i2c_device_id es8326_i2c_id[] = {
+	{"es8326", 0 },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, es8326_i2c_id);
+
+#ifdef CONFIG_OF
+static const struct of_device_id es8326_of_match[] = {
+	{ .compatible = "everest,es8326", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, es8326_of_match);
+#endif
+
+#ifdef CONFIG_ACPI
+static const struct acpi_device_id es8326_acpi_match[] = {
+	{"ESSX8326", 0},
+	{},
+};
+MODULE_DEVICE_TABLE(acpi, es8326_acpi_match);
+#endif
+
+static struct i2c_driver es8326_i2c_driver = {
+	.driver = {
+		.name = "es8326",
+		.acpi_match_table = ACPI_PTR(es8326_acpi_match),
+		.of_match_table = of_match_ptr(es8326_of_match),
+	},
+	.probe = es8326_i2c_probe,
+	.id_table = es8326_i2c_id,
+};
+module_i2c_driver(es8326_i2c_driver);
+
+MODULE_DESCRIPTION("ASoC es8326 driver");
+MODULE_AUTHOR("David Yang <yangxiaohua@everest-semi.com>");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/es8326.h b/sound/soc/codecs/es8326.h
new file mode 100755
index 000000000000..8e5ffe5ee10d
--- /dev/null
+++ b/sound/soc/codecs/es8326.h
@@ -0,0 +1,182 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * es8326.h -- es8326 ALSA SoC audio driver
+ * Copyright Everest Semiconductor Co.,Ltd
+ *
+ * Authors: David Yang <yangxiaohua@everest-semi.com>
+ */
+
+#ifndef _ES8326_H
+#define _ES8326_H
+
+#define CONFIG_HHTECH_MINIPMP	1
+
+/* ES8326 register space */
+#define ES8326_RESET		0x00
+#define ES8326_CLK_CTL		0x01
+#define ES8326_CLK_INV		0x02
+#define ES8326_CLK_RESAMPLE	0x03
+#define ES8326_CLK_DIV1		0x04
+#define ES8326_CLK_DIV2		0x05
+#define ES8326_CLK_DLL		0x06
+#define ES8326_CLK_MUX		0x07
+#define ES8326_CLK_ADC_SEL	0x08
+#define ES8326_CLK_DAC_SEL	0x09
+#define ES8326_CLK_ADC_OSR	0x0a
+#define ES8326_CLK_DAC_OSR	0x0b
+#define ES8326_CLK_DIV_CPC	0x0c
+#define ES8326_CLK_DIV_BCLK	0x0d
+#define ES8326_CLK_TRI		0x0e
+#define ES8326_CLK_DIV_LRCK	0x0f
+#define ES8326_CLK_VMIDS1	0x10
+#define ES8326_CLK_VMIDS2	0x11
+#define ES8326_CLK_CAL_TIME	0x12
+#define ES8326_FMT		0x13
+
+#define ES8326_DAC_MUTE		0x14
+#define ES8326_ADC_MUTE		0x15
+#define ES8326_ANA_PDN		0x16
+#define ES8326_PGA_PDN		0x17
+#define ES8326_VMIDSEL		0x18
+#define ES8326_ANA_LP		0x19
+#define ES8326_ANA_DMS		0x1a
+#define ES8326_ANA_MICBIAS	0x1b
+#define ES8326_ANA_VSEL		0x1c
+#define ES8326_SYS_BIAS		0x1d
+#define ES8326_BIAS_SW1		0x1e
+#define ES8326_BIAS_SW2		0x1f
+#define ES8326_BIAS_SW3		0x20
+#define ES8326_BIAS_SW4		0x21
+#define ES8326_VMIDLOW		0x22
+#define ES8326_PGAGAIN		0x23
+#define ES8326_HP_DRIVER	0x24
+#define ES8326_DAC2HPMIX	0x25
+#define ES8326_HP_VOL		0x26
+#define ES8326_HP_CAL		0x27
+#define ES8326_HP_DRIVER_REF	0x28
+#define ES8326_ADC_SCALE	0x29
+#define ES8326_ADC1_SRC		0x2a
+#define ES8326_ADC2_SRC		0x2b
+#define ES8326_ADC1_VOL		0x2c
+#define ES8326_ADC2_VOL		0x2d
+#define ES8326_ADC_RAMPRATE	0x2e
+#define ES8326_ALC_RECOVERY	0x32
+#define ES8326_ALC_LEVEL	0x33
+#define ES8326_ADC_HPFS1	0x34
+#define ES8326_ADC_HPFS2	0x35
+#define ES8326_ADC_EQ		0x36
+#define ES8326_HP_OFFSET_CAL	0x4A
+#define ES8326_HPL_OFFSET_INI	0x4B
+#define ES8326_HPR_OFFSET_INI	0x4C
+#define ES8326_DAC_DSM		0x4D
+#define ES8326_DAC_RAMPRATE	0x4E
+#define ES8326_DAC_VPPSCALE	0x4F
+#define ES8326_DAC_VOL		0x50
+#define ES8326_DRC_RECOVERY	0x53
+#define ES8326_DRC_WINSIZE	0x54
+#define ES8326_HPJACK_TIMER	0x56
+#define ES8326_HP_DET		0x57
+#define ES8326_INT_SOURCE	0x58
+#define ES8326_INTOUT_IO	0x59
+#define ES8326_SDINOUT1_IO	0x5A
+#define ES8326_SDINOUT23_IO	0x5B
+#define ES8326_JACK_PULSE	0x5C
+
+#define ES8326_PULLUP_CTL	0xF9
+#define ES8326_HP_DETECT	0xFB
+#define ES8326_CHIP_ID1		0xFD
+#define ES8326_CHIP_ID2		0xFE
+#define ES8326_CHIP_VERSION	0xFF
+
+/* ES8326_RESET */
+#define ES8326_CSM_ON (1 << 7)
+#define ES8326_MASTER_MODE_EN	(1 << 6)
+#define	ES8326_PWRUP_SEQ_EN	(1 << 5)
+#define ES8326_CODEC_RESET (0x0f << 0)
+#define ES8326_CSM_OFF (0 << 7)
+
+/* ES8326_CLK_CTL */
+#define ES8326_CLK_ON (0x7f << 0)
+#define ES8326_CLK_OFF (0 << 0)
+
+/* ES8326_CLK_INV */
+#define ES8326_BCLK_AS_MCLK (1 << 3)
+
+/* ES8326_FMT */
+#define ES8326_S24_LE	(0 << 2)
+#define ES8326_S20_3_LE	(1 << 2)
+#define ES8326_S18_LE	(2 << 2)
+#define ES8326_S16_LE	(3 << 2)
+#define ES8326_S32_LE	(4 << 2)
+#define ES8326_DATA_LEN_MASK	(7 << 2)
+
+#define ES8326_DAIFMT_MASK	((1 << 5) | (3 << 0))
+#define ES8326_DAIFMT_I2S	0
+#define ES8326_DAIFMT_LEFT_J	(1 << 0)
+#define ES8326_DAIFMT_DSP_A	(3 << 0)
+#define ES8326_DAIFMT_DSP_B	((1 << 5) | (3 << 0))
+
+/* ES8326_PGAGAIN */
+#define ES8326_MIC_SEL_MASK (3 << 4)
+#define ES8326_MIC1_SEL	(1 << 4)
+#define ES8326_MIC2_SEL (1 << 5)
+
+/* ES8326_HP_CAL */
+#define ES8326_HPOR_SHIFT 4
+
+/* ES8326_ADC1_SRC */
+#define ES8326_ADC1_SHIFT 0
+#define ES8326_ADC2_SHIFT 4
+#define ES8326_ADC_SRC_ANA 0
+#define ES8326_ADC_SRC_ANA_INV_SW0 1
+#define ES8326_ADC_SRC_ANA_INV_SW1 2
+#define ES8326_ADC_SRC_DMIC_MCLK 3
+#define ES8326_ADC_SRC_DMIC_SDIN2 4
+#define ES8326_ADC_SRC_DMIC_SDIN2_INV 5
+#define ES8326_ADC_SRC_DMIC_SDIN3 6
+#define ES8326_ADC_SRC_DMIC_SDIN3_INV 7
+
+#define ES8326_ADC_AMIC	((ES8326_ADC_SRC_ANA_INV_SW1 << ES8326_ADC2_SHIFT) \
+		| (ES8326_ADC_SRC_ANA_INV_SW1 << ES8326_ADC1_SHIFT))
+#define ES8326_ADC_DMIC	((ES8326_ADC_SRC_DMIC_SDIN2 << ES8326_ADC2_SHIFT) \
+		| (ES8326_ADC_SRC_DMIC_SDIN2 << ES8326_ADC1_SHIFT))
+/* ES8326_ADC2_SRC */
+#define ES8326_ADC3_SHIFT 0
+#define ES8326_ADC4_SHIFT 3
+
+/* ES8326_HP_DET */
+#define ES8326_HP_DET_SRC_PIN27 (1 << 5)
+#define ES8326_HP_DET_SRC_PIN9 (1 << 4)
+#define ES8326_HP_DET_JACK_POL (1 << 3)
+#define ES8326_HP_DET_BUTTON_POL (1 << 2)
+#define ES8326_HP_TYPE_OMTP	(3 << 0)
+#define ES8326_HP_TYPE_CTIA	(2 << 0)
+#define ES8326_HP_TYPE_AUTO	(1 << 0)
+#define ES8326_HP_TYPE_AUTO_INV	(0 << 0)
+
+/* ES8326_SDINOUT1_IO */
+#define ES8326_IO_INPUT	(0 << 0)
+#define ES8326_IO_SDIN_SLOT0 (1 << 0)
+#define ES8326_IO_SDIN_SLOT1 (2 << 0)
+#define ES8326_IO_SDIN_SLOT2 (3 << 0)
+#define ES8326_IO_SDIN_SLOT7 (8 << 0)
+#define ES8326_IO_DMIC_CLK (9 << 0)
+#define ES8326_IO_DMIC_CLK_INV (0x0a << 0)
+#define ES8326_IO_SDOUT2 (0x0b << 0)
+#define ES8326_IO_LOW (0x0e << 0)
+#define ES8326_IO_HIGH (0x0f << 0)
+#define ES8326_ADC2DAC (1 << 3)
+#define ES8326_SDINOUT1_SHIFT 4
+
+/* ES8326_SDINOUT23_IO */
+#define ES8326_SDINOUT2_SHIFT 4
+#define ES8326_SDINOUT3_SHIFT 0
+
+/* ES8326_HP_DETECT */
+#define ES8326_HPINSERT_FLAG (1 << 1)
+#define ES8326_HPBUTTON_FLAG (1 << 0)
+
+/* ES8326_CHIP_VERSION 0xFF */
+#define ES8326_VERSION_B (1 << 0)
+
+#endif
diff --git a/sound/soc/codecs/hda.c b/sound/soc/codecs/hda.c
index ad20a3dff9b7..61e8e9be6b8d 100644
--- a/sound/soc/codecs/hda.c
+++ b/sound/soc/codecs/hda.c
@@ -224,9 +224,6 @@ static int hda_codec_probe(struct snd_soc_component *component)
 		goto err;
 	}
 
-	/* configure codec for 1:1 PCM:DAI mapping */
-	codec->mst_no_extra_pcms = 1;
-
 	ret = snd_hda_codec_parse_pcms(codec);
 	if (ret < 0) {
 		dev_err(&hdev->dev, "unable to map pcms to dai %d\n", ret);
diff --git a/sound/soc/codecs/hdac_hda.c b/sound/soc/codecs/hdac_hda.c
index 8debcee59224..8af434e14bfb 100644
--- a/sound/soc/codecs/hdac_hda.c
+++ b/sound/soc/codecs/hdac_hda.c
@@ -246,7 +246,7 @@ static int hdac_hda_dai_hw_free(struct snd_pcm_substream *substream,
 		return -EINVAL;
 
 	hda_stream = &pcm->stream[substream->stream];
-	snd_hda_codec_cleanup(&hda_pvt->codec, hda_stream, substream);
+	snd_hda_codec_cleanup(hda_pvt->codec, hda_stream, substream);
 
 	return 0;
 }
@@ -264,7 +264,7 @@ static int hdac_hda_dai_prepare(struct snd_pcm_substream *substream,
 	int ret = 0;
 
 	hda_pvt = snd_soc_component_get_drvdata(component);
-	hdev = &hda_pvt->codec.core;
+	hdev = &hda_pvt->codec->core;
 	pcm = snd_soc_find_pcm_from_dai(hda_pvt, dai);
 	if (!pcm)
 		return -EINVAL;
@@ -274,7 +274,7 @@ static int hdac_hda_dai_prepare(struct snd_pcm_substream *substream,
 	stream = hda_pvt->pcm[dai->id].stream_tag[substream->stream];
 	format_val = hda_pvt->pcm[dai->id].format_val[substream->stream];
 
-	ret = snd_hda_codec_prepare(&hda_pvt->codec, hda_stream,
+	ret = snd_hda_codec_prepare(hda_pvt->codec, hda_stream,
 				    stream, format_val, substream);
 	if (ret < 0)
 		dev_err(&hdev->dev, "codec prepare failed %d\n", ret);
@@ -299,7 +299,7 @@ static int hdac_hda_dai_open(struct snd_pcm_substream *substream,
 
 	hda_stream = &pcm->stream[substream->stream];
 
-	return hda_stream->ops.open(hda_stream, &hda_pvt->codec, substream);
+	return hda_stream->ops.open(hda_stream, hda_pvt->codec, substream);
 }
 
 static void hdac_hda_dai_close(struct snd_pcm_substream *substream,
@@ -317,7 +317,7 @@ static void hdac_hda_dai_close(struct snd_pcm_substream *substream,
 
 	hda_stream = &pcm->stream[substream->stream];
 
-	hda_stream->ops.close(hda_stream, &hda_pvt->codec, substream);
+	hda_stream->ops.close(hda_stream, hda_pvt->codec, substream);
 
 	snd_hda_codec_pcm_put(pcm);
 }
@@ -325,7 +325,7 @@ static void hdac_hda_dai_close(struct snd_pcm_substream *substream,
 static struct hda_pcm *snd_soc_find_pcm_from_dai(struct hdac_hda_priv *hda_pvt,
 						 struct snd_soc_dai *dai)
 {
-	struct hda_codec *hcodec = &hda_pvt->codec;
+	struct hda_codec *hcodec = hda_pvt->codec;
 	struct hda_pcm *cpcm;
 	const char *pcm_name;
 
@@ -394,8 +394,8 @@ static int hdac_hda_codec_probe(struct snd_soc_component *component)
 			snd_soc_component_get_drvdata(component);
 	struct snd_soc_dapm_context *dapm =
 			snd_soc_component_get_dapm(component);
-	struct hdac_device *hdev = &hda_pvt->codec.core;
-	struct hda_codec *hcodec = &hda_pvt->codec;
+	struct hdac_device *hdev = &hda_pvt->codec->core;
+	struct hda_codec *hcodec = hda_pvt->codec;
 	struct hdac_ext_link *hlink;
 	hda_codec_patch_t patch;
 	int ret;
@@ -461,9 +461,6 @@ static int hdac_hda_codec_probe(struct snd_soc_component *component)
 		dev_dbg(&hdev->dev, "no patch file found\n");
 	}
 
-	/* configure codec for 1:1 PCM:DAI mapping */
-	hcodec->mst_no_extra_pcms = 1;
-
 	ret = snd_hda_codec_parse_pcms(hcodec);
 	if (ret < 0) {
 		dev_err(&hdev->dev, "unable to map pcms to dai %d\n", ret);
@@ -515,8 +512,8 @@ static void hdac_hda_codec_remove(struct snd_soc_component *component)
 {
 	struct hdac_hda_priv *hda_pvt =
 		      snd_soc_component_get_drvdata(component);
-	struct hdac_device *hdev = &hda_pvt->codec.core;
-	struct hda_codec *codec = &hda_pvt->codec;
+	struct hdac_device *hdev = &hda_pvt->codec->core;
+	struct hda_codec *codec = hda_pvt->codec;
 	struct hdac_ext_link *hlink = NULL;
 
 	hlink = snd_hdac_ext_bus_get_link(hdev->bus, dev_name(&hdev->dev));
@@ -584,7 +581,6 @@ static const struct snd_soc_component_driver hdac_hda_codec = {
 static int hdac_hda_dev_probe(struct hdac_device *hdev)
 {
 	struct hdac_ext_link *hlink;
-	struct hdac_hda_priv *hda_pvt;
 	int ret;
 
 	/* hold the ref while we probe */
@@ -595,10 +591,6 @@ static int hdac_hda_dev_probe(struct hdac_device *hdev)
 	}
 	snd_hdac_ext_bus_link_get(hdev->bus, hlink);
 
-	hda_pvt = hdac_to_hda_priv(hdev);
-	if (!hda_pvt)
-		return -ENOMEM;
-
 	/* ASoC specific initialization */
 	ret = devm_snd_soc_register_component(&hdev->dev,
 					 &hdac_hda_codec, hdac_hda_dais,
@@ -608,7 +600,6 @@ static int hdac_hda_dev_probe(struct hdac_device *hdev)
 		return ret;
 	}
 
-	dev_set_drvdata(&hdev->dev, hda_pvt);
 	snd_hdac_ext_bus_link_put(hdev->bus, hlink);
 
 	return ret;
diff --git a/sound/soc/codecs/hdac_hda.h b/sound/soc/codecs/hdac_hda.h
index d0efc5e254ae..fc19c34ca00e 100644
--- a/sound/soc/codecs/hdac_hda.h
+++ b/sound/soc/codecs/hdac_hda.h
@@ -23,7 +23,7 @@ struct hdac_hda_pcm {
 };
 
 struct hdac_hda_priv {
-	struct hda_codec codec;
+	struct hda_codec *codec;
 	struct hdac_hda_pcm pcm[HDAC_LAST_DAI_ID];
 	bool need_display_power;
 };
diff --git a/sound/soc/codecs/hdmi-codec.c b/sound/soc/codecs/hdmi-codec.c
index 5679102de91f..0b1cdb2d6049 100644
--- a/sound/soc/codecs/hdmi-codec.c
+++ b/sound/soc/codecs/hdmi-codec.c
@@ -20,10 +20,6 @@
 
 #define HDMI_CODEC_CHMAP_IDX_UNKNOWN  -1
 
-struct hdmi_codec_channel_map_table {
-	unsigned char map;	/* ALSA API channel map position */
-};
-
 /*
  * CEA speaker placement for HDMI 1.4:
  *
@@ -827,7 +823,7 @@ static int hdmi_dai_probe(struct snd_soc_dai *dai)
 	if (ret)
 		return ret;
 
-	daifmt = kzalloc(sizeof(*daifmt), GFP_KERNEL);
+	daifmt = devm_kzalloc(dai->dev, sizeof(*daifmt), GFP_KERNEL);
 	if (!daifmt)
 		return -ENOMEM;
 
@@ -894,17 +890,10 @@ static int hdmi_dai_spdif_probe(struct snd_soc_dai *dai)
 	return 0;
 }
 
-static int hdmi_codec_dai_remove(struct snd_soc_dai *dai)
-{
-	kfree(dai->playback_dma_data);
-	return 0;
-}
-
 static const struct snd_soc_dai_driver hdmi_i2s_dai = {
 	.name = "i2s-hifi",
 	.id = DAI_ID_I2S,
 	.probe = hdmi_dai_probe,
-	.remove = hdmi_codec_dai_remove,
 	.playback = {
 		.stream_name = "I2S Playback",
 		.channels_min = 2,
@@ -929,7 +918,6 @@ static const struct snd_soc_dai_driver hdmi_spdif_dai = {
 	.name = "spdif-hifi",
 	.id = DAI_ID_SPDIF,
 	.probe = hdmi_dai_spdif_probe,
-	.remove = hdmi_codec_dai_remove,
 	.playback = {
 		.stream_name = "SPDIF Playback",
 		.channels_min = 2,
diff --git a/sound/soc/codecs/lpass-rx-macro.c b/sound/soc/codecs/lpass-rx-macro.c
index 3143f9cd7277..a9ef9d5ffcc5 100644
--- a/sound/soc/codecs/lpass-rx-macro.c
+++ b/sound/soc/codecs/lpass-rx-macro.c
@@ -596,7 +596,6 @@ struct rx_macro {
 	int rx_port_value[RX_MACRO_PORTS_MAX];
 	u16 prim_int_users[INTERP_MAX];
 	int rx_mclk_users;
-	bool reset_swr;
 	int clsh_users;
 	int rx_mclk_cnt;
 	bool is_ear_mode_on;
@@ -3442,18 +3441,15 @@ static int swclk_gate_enable(struct clk_hw *hw)
 	}
 
 	rx_macro_mclk_enable(rx, true);
-	if (rx->reset_swr)
-		regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL,
-				   CDC_RX_SWR_RESET_MASK,
-				   CDC_RX_SWR_RESET);
+	regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL,
+			   CDC_RX_SWR_RESET_MASK,
+			   CDC_RX_SWR_RESET);
 
 	regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL,
 			   CDC_RX_SWR_CLK_EN_MASK, 1);
 
-	if (rx->reset_swr)
-		regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL,
-				   CDC_RX_SWR_RESET_MASK, 0);
-	rx->reset_swr = false;
+	regmap_update_bits(rx->regmap, CDC_RX_CLK_RST_CTRL_SWR_CONTROL,
+			   CDC_RX_SWR_RESET_MASK, 0);
 
 	return 0;
 }
@@ -3579,7 +3575,6 @@ static int rx_macro_probe(struct platform_device *pdev)
 
 	dev_set_drvdata(dev, rx);
 
-	rx->reset_swr = true;
 	rx->dev = dev;
 
 	/* set MCLK and NPL rates */
@@ -3659,6 +3654,8 @@ static int rx_macro_remove(struct platform_device *pdev)
 static const struct of_device_id rx_macro_dt_match[] = {
 	{ .compatible = "qcom,sc7280-lpass-rx-macro" },
 	{ .compatible = "qcom,sm8250-lpass-rx-macro" },
+	{ .compatible = "qcom,sm8450-lpass-rx-macro" },
+	{ .compatible = "qcom,sc8280xp-lpass-rx-macro" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, rx_macro_dt_match);
@@ -3701,7 +3698,6 @@ static int __maybe_unused rx_macro_runtime_resume(struct device *dev)
 	}
 	regcache_cache_only(rx->regmap, false);
 	regcache_sync(rx->regmap);
-	rx->reset_swr = true;
 
 	return 0;
 err_fsgen:
diff --git a/sound/soc/codecs/lpass-tx-macro.c b/sound/soc/codecs/lpass-tx-macro.c
index 55503ba480bb..ee15cf6b98bb 100644
--- a/sound/soc/codecs/lpass-tx-macro.c
+++ b/sound/soc/codecs/lpass-tx-macro.c
@@ -259,7 +259,7 @@ struct tx_macro {
 	struct tx_mute_work tx_mute_dwork[NUM_DECIMATORS];
 	unsigned long active_ch_mask[TX_MACRO_MAX_DAIS];
 	unsigned long active_ch_cnt[TX_MACRO_MAX_DAIS];
-	unsigned long active_decimator[TX_MACRO_MAX_DAIS];
+	int active_decimator[TX_MACRO_MAX_DAIS];
 	struct regmap *regmap;
 	struct clk *mclk;
 	struct clk *npl;
@@ -268,7 +268,6 @@ struct tx_macro {
 	struct clk *fsgen;
 	struct clk_hw hw;
 	bool dec_active[NUM_DECIMATORS];
-	bool reset_swr;
 	int tx_mclk_users;
 	u16 dmic_clk_div;
 	bool bcs_enable;
@@ -823,17 +822,23 @@ static int tx_macro_tx_mixer_put(struct snd_kcontrol *kcontrol,
 	struct tx_macro *tx = snd_soc_component_get_drvdata(component);
 
 	if (enable) {
+		if (tx->active_decimator[dai_id] == dec_id)
+			return 0;
+
 		set_bit(dec_id, &tx->active_ch_mask[dai_id]);
 		tx->active_ch_cnt[dai_id]++;
 		tx->active_decimator[dai_id] = dec_id;
 	} else {
+		if (tx->active_decimator[dai_id] == -1)
+			return 0;
+
 		tx->active_ch_cnt[dai_id]--;
 		clear_bit(dec_id, &tx->active_ch_mask[dai_id]);
 		tx->active_decimator[dai_id] = -1;
 	}
 	snd_soc_dapm_mixer_update_power(widget->dapm, kcontrol, enable, update);
 
-	return 0;
+	return 1;
 }
 
 static int tx_macro_enable_dec(struct snd_soc_dapm_widget *w,
@@ -1019,9 +1024,12 @@ static int tx_macro_dec_mode_put(struct snd_kcontrol *kcontrol,
 	int path = e->shift_l;
 	struct tx_macro *tx = snd_soc_component_get_drvdata(component);
 
+	if (tx->dec_mode[path] == value)
+		return 0;
+
 	tx->dec_mode[path] = value;
 
-	return 0;
+	return 1;
 }
 
 static int tx_macro_get_bcs(struct snd_kcontrol *kcontrol,
@@ -1118,6 +1126,10 @@ static int tx_macro_digital_mute(struct snd_soc_dai *dai, int mute, int stream)
 	struct tx_macro *tx = snd_soc_component_get_drvdata(component);
 	u16 decimator;
 
+	/* active decimator not set yet */
+	if (tx->active_decimator[dai->id] == -1)
+		return 0;
+
 	decimator = tx->active_decimator[dai->id];
 
 	if (mute)
@@ -1702,18 +1714,14 @@ static int swclk_gate_enable(struct clk_hw *hw)
 	}
 
 	tx_macro_mclk_enable(tx, true);
-	if (tx->reset_swr)
-		regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL,
-				   CDC_TX_SWR_RESET_MASK,
-				   CDC_TX_SWR_RESET_ENABLE);
+	regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL,
+			   CDC_TX_SWR_RESET_MASK, CDC_TX_SWR_RESET_ENABLE);
 
 	regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL,
 			   CDC_TX_SWR_CLK_EN_MASK,
 			   CDC_TX_SWR_CLK_ENABLE);
-	if (tx->reset_swr)
-		regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL,
-				   CDC_TX_SWR_RESET_MASK, 0x0);
-	tx->reset_swr = false;
+	regmap_update_bits(regmap, CDC_TX_CLK_RST_CTRL_SWR_CONTROL,
+			   CDC_TX_SWR_RESET_MASK, 0x0);
 
 	return 0;
 }
@@ -1855,7 +1863,6 @@ static int tx_macro_probe(struct platform_device *pdev)
 
 	dev_set_drvdata(dev, tx);
 
-	tx->reset_swr = true;
 	tx->dev = dev;
 
 	/* set MCLK and NPL rates */
@@ -1970,7 +1977,6 @@ static int __maybe_unused tx_macro_runtime_resume(struct device *dev)
 
 	regcache_cache_only(tx->regmap, false);
 	regcache_sync(tx->regmap);
-	tx->reset_swr = true;
 
 	return 0;
 err_fsgen:
@@ -1988,6 +1994,8 @@ static const struct dev_pm_ops tx_macro_pm_ops = {
 static const struct of_device_id tx_macro_dt_match[] = {
 	{ .compatible = "qcom,sc7280-lpass-tx-macro" },
 	{ .compatible = "qcom,sm8250-lpass-tx-macro" },
+	{ .compatible = "qcom,sm8450-lpass-tx-macro" },
+	{ .compatible = "qcom,sc8280xp-lpass-tx-macro" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, tx_macro_dt_match);
diff --git a/sound/soc/codecs/lpass-va-macro.c b/sound/soc/codecs/lpass-va-macro.c
index 1ea10dc70748..b0b6cf29cba3 100644
--- a/sound/soc/codecs/lpass-va-macro.c
+++ b/sound/soc/codecs/lpass-va-macro.c
@@ -23,7 +23,12 @@
 #define CDC_VA_MCLK_CONTROL_EN			BIT(0)
 #define CDC_VA_CLK_RST_CTRL_FS_CNT_CONTROL	(0x0004)
 #define CDC_VA_FS_CONTROL_EN			BIT(0)
+#define CDC_VA_FS_COUNTER_CLR			BIT(1)
 #define CDC_VA_CLK_RST_CTRL_SWR_CONTROL		(0x0008)
+#define CDC_VA_SWR_RESET_MASK		BIT(1)
+#define CDC_VA_SWR_RESET_ENABLE		BIT(1)
+#define CDC_VA_SWR_CLK_EN_MASK		BIT(0)
+#define CDC_VA_SWR_CLK_ENABLE		BIT(0)
 #define CDC_VA_TOP_CSR_TOP_CFG0			(0x0080)
 #define CDC_VA_FS_BROADCAST_EN			BIT(1)
 #define CDC_VA_TOP_CSR_DMIC0_CTL		(0x0084)
@@ -65,6 +70,8 @@
 #define CDC_VA_TOP_CSR_SWR_MIC_CTL0		(0x00D0)
 #define CDC_VA_TOP_CSR_SWR_MIC_CTL1		(0x00D4)
 #define CDC_VA_TOP_CSR_SWR_MIC_CTL2		(0x00D8)
+#define CDC_VA_SWR_MIC_CLK_SEL_0_1_MASK		(0xEE)
+#define CDC_VA_SWR_MIC_CLK_SEL_0_1_DIV1		(0xCC)
 #define CDC_VA_TOP_CSR_SWR_CTRL			(0x00DC)
 #define CDC_VA_INP_MUX_ADC_MUX0_CFG0		(0x0100)
 #define CDC_VA_INP_MUX_ADC_MUX0_CFG1		(0x0104)
@@ -193,6 +200,7 @@ struct va_macro {
 	unsigned long active_ch_mask[VA_MACRO_MAX_DAIS];
 	unsigned long active_ch_cnt[VA_MACRO_MAX_DAIS];
 	u16 dmic_clk_div;
+	bool has_swr_master;
 
 	int dec_mode[VA_MACRO_NUM_DECIMATORS];
 	struct regmap *regmap;
@@ -215,6 +223,18 @@ struct va_macro {
 
 #define to_va_macro(_hw) container_of(_hw, struct va_macro, hw)
 
+struct va_macro_data {
+	bool has_swr_master;
+};
+
+static const struct va_macro_data sm8250_va_data = {
+	.has_swr_master = false,
+};
+
+static const struct va_macro_data sm8450_va_data = {
+	.has_swr_master = true,
+};
+
 static bool va_is_volatile_register(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
@@ -324,6 +344,9 @@ static bool va_is_rw_register(struct device *dev, unsigned int reg)
 	case CDC_VA_TOP_CSR_DMIC2_CTL:
 	case CDC_VA_TOP_CSR_DMIC3_CTL:
 	case CDC_VA_TOP_CSR_DMIC_CFG:
+	case CDC_VA_TOP_CSR_SWR_MIC_CTL0:
+	case CDC_VA_TOP_CSR_SWR_MIC_CTL1:
+	case CDC_VA_TOP_CSR_SWR_MIC_CTL2:
 	case CDC_VA_TOP_CSR_DEBUG_BUS:
 	case CDC_VA_TOP_CSR_DEBUG_EN:
 	case CDC_VA_TOP_CSR_TX_I2S_CTL:
@@ -423,9 +446,12 @@ static int va_clk_rsc_fs_gen_request(struct va_macro *va, bool enable)
 		regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_MCLK_CONTROL,
 				   CDC_VA_MCLK_CONTROL_EN,
 				   CDC_VA_MCLK_CONTROL_EN);
-
+		/* clear the fs counter */
 		regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_FS_CNT_CONTROL,
-				   CDC_VA_FS_CONTROL_EN,
+				   CDC_VA_FS_CONTROL_EN | CDC_VA_FS_COUNTER_CLR,
+				   CDC_VA_FS_CONTROL_EN | CDC_VA_FS_COUNTER_CLR);
+		regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_FS_CNT_CONTROL,
+				   CDC_VA_FS_CONTROL_EN | CDC_VA_FS_COUNTER_CLR,
 				   CDC_VA_FS_CONTROL_EN);
 
 		regmap_update_bits(regmap, CDC_VA_TOP_CSR_TOP_CFG0,
@@ -1302,12 +1328,36 @@ static const struct snd_soc_component_driver va_macro_component_drv = {
 
 static int fsgen_gate_enable(struct clk_hw *hw)
 {
-	return va_macro_mclk_enable(to_va_macro(hw), true);
+	struct va_macro *va = to_va_macro(hw);
+	struct regmap *regmap = va->regmap;
+	int ret;
+
+	ret = va_macro_mclk_enable(va, true);
+	if (!va->has_swr_master)
+		return ret;
+
+	regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL,
+			   CDC_VA_SWR_RESET_MASK,  CDC_VA_SWR_RESET_ENABLE);
+
+	regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL,
+			   CDC_VA_SWR_CLK_EN_MASK,
+			   CDC_VA_SWR_CLK_ENABLE);
+	regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL,
+			   CDC_VA_SWR_RESET_MASK, 0x0);
+
+	return ret;
 }
 
 static void fsgen_gate_disable(struct clk_hw *hw)
 {
-	va_macro_mclk_enable(to_va_macro(hw), false);
+	struct va_macro *va = to_va_macro(hw);
+	struct regmap *regmap = va->regmap;
+
+	if (va->has_swr_master)
+		regmap_update_bits(regmap, CDC_VA_CLK_RST_CTRL_SWR_CONTROL,
+			   CDC_VA_SWR_CLK_EN_MASK, 0x0);
+
+	va_macro_mclk_enable(va, false);
 }
 
 static int fsgen_gate_is_enabled(struct clk_hw *hw)
@@ -1401,6 +1451,7 @@ undefined_rate:
 static int va_macro_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
+	const struct va_macro_data *data;
 	struct va_macro *va;
 	void __iomem *base;
 	u32 sample_rate = 0;
@@ -1455,6 +1506,9 @@ static int va_macro_probe(struct platform_device *pdev)
 
 	dev_set_drvdata(dev, va);
 
+	data = of_device_get_match_data(dev);
+	va->has_swr_master = data->has_swr_master;
+
 	/* mclk rate */
 	clk_set_rate(va->mclk, 2 * VA_MACRO_MCLK_FREQ);
 
@@ -1480,6 +1534,20 @@ static int va_macro_probe(struct platform_device *pdev)
 		goto err_clkout;
 	}
 
+	if (va->has_swr_master) {
+		/* Set default CLK div to 1 */
+		regmap_update_bits(va->regmap, CDC_VA_TOP_CSR_SWR_MIC_CTL0,
+				  CDC_VA_SWR_MIC_CLK_SEL_0_1_MASK,
+				  CDC_VA_SWR_MIC_CLK_SEL_0_1_DIV1);
+		regmap_update_bits(va->regmap, CDC_VA_TOP_CSR_SWR_MIC_CTL1,
+				  CDC_VA_SWR_MIC_CLK_SEL_0_1_MASK,
+				  CDC_VA_SWR_MIC_CLK_SEL_0_1_DIV1);
+		regmap_update_bits(va->regmap, CDC_VA_TOP_CSR_SWR_MIC_CTL2,
+				  CDC_VA_SWR_MIC_CLK_SEL_0_1_MASK,
+				  CDC_VA_SWR_MIC_CLK_SEL_0_1_DIV1);
+
+	}
+
 	ret = devm_snd_soc_register_component(dev, &va_macro_component_drv,
 					      va_macro_dais,
 					      ARRAY_SIZE(va_macro_dais));
@@ -1554,8 +1622,10 @@ static const struct dev_pm_ops va_macro_pm_ops = {
 };
 
 static const struct of_device_id va_macro_dt_match[] = {
-	{ .compatible = "qcom,sc7280-lpass-va-macro" },
-	{ .compatible = "qcom,sm8250-lpass-va-macro" },
+	{ .compatible = "qcom,sc7280-lpass-va-macro", .data = &sm8250_va_data },
+	{ .compatible = "qcom,sm8250-lpass-va-macro", .data = &sm8250_va_data },
+	{ .compatible = "qcom,sm8450-lpass-va-macro", .data = &sm8450_va_data },
+	{ .compatible = "qcom,sc8280xp-lpass-va-macro", .data = &sm8450_va_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, va_macro_dt_match);
diff --git a/sound/soc/codecs/lpass-wsa-macro.c b/sound/soc/codecs/lpass-wsa-macro.c
index 27da6c6c3c5a..5e0abefe7cce 100644
--- a/sound/soc/codecs/lpass-wsa-macro.c
+++ b/sound/soc/codecs/lpass-wsa-macro.c
@@ -338,7 +338,6 @@ struct wsa_macro {
 	int ec_hq[WSA_MACRO_RX1 + 1];
 	u16 prim_int_users[WSA_MACRO_RX1 + 1];
 	u16 wsa_mclk_users;
-	bool reset_swr;
 	unsigned long active_ch_mask[WSA_MACRO_MAX_DAIS];
 	unsigned long active_ch_cnt[WSA_MACRO_MAX_DAIS];
 	int rx_port_value[WSA_MACRO_RX_MAX];
@@ -2271,23 +2270,16 @@ static int wsa_swrm_clock(struct wsa_macro *wsa, bool enable)
 		wsa_macro_mclk_enable(wsa, true);
 
 		/* reset swr ip */
-		if (wsa->reset_swr)
-			regmap_update_bits(regmap,
-					   CDC_WSA_CLK_RST_CTRL_SWR_CONTROL,
-					   CDC_WSA_SWR_RST_EN_MASK,
-					   CDC_WSA_SWR_RST_ENABLE);
+		regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL,
+				   CDC_WSA_SWR_RST_EN_MASK, CDC_WSA_SWR_RST_ENABLE);
 
 		regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL,
 				   CDC_WSA_SWR_CLK_EN_MASK,
 				   CDC_WSA_SWR_CLK_ENABLE);
 
 		/* Bring out of reset */
-		if (wsa->reset_swr)
-			regmap_update_bits(regmap,
-					   CDC_WSA_CLK_RST_CTRL_SWR_CONTROL,
-					   CDC_WSA_SWR_RST_EN_MASK,
-					   CDC_WSA_SWR_RST_DISABLE);
-		wsa->reset_swr = false;
+		regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL,
+				   CDC_WSA_SWR_RST_EN_MASK, CDC_WSA_SWR_RST_DISABLE);
 	} else {
 		regmap_update_bits(regmap, CDC_WSA_CLK_RST_CTRL_SWR_CONTROL,
 				   CDC_WSA_SWR_CLK_EN_MASK, 0);
@@ -2431,7 +2423,6 @@ static int wsa_macro_probe(struct platform_device *pdev)
 
 	dev_set_drvdata(dev, wsa);
 
-	wsa->reset_swr = true;
 	wsa->dev = dev;
 
 	/* set MCLK and NPL rates */
@@ -2561,6 +2552,8 @@ static const struct dev_pm_ops wsa_macro_pm_ops = {
 static const struct of_device_id wsa_macro_dt_match[] = {
 	{.compatible = "qcom,sc7280-lpass-wsa-macro"},
 	{.compatible = "qcom,sm8250-lpass-wsa-macro"},
+	{.compatible = "qcom,sm8450-lpass-wsa-macro"},
+	{.compatible = "qcom,sc8280xp-lpass-wsa-macro" },
 	{}
 };
 MODULE_DEVICE_TABLE(of, wsa_macro_dt_match);
diff --git a/sound/soc/codecs/max98088.c b/sound/soc/codecs/max98088.c
index 5435a49604cf..405ec16be2b6 100644
--- a/sound/soc/codecs/max98088.c
+++ b/sound/soc/codecs/max98088.c
@@ -474,6 +474,9 @@ static const struct snd_kcontrol_new max98088_snd_controls[] = {
                        max98088_mic2pre_get, max98088_mic2pre_set,
                        max98088_micboost_tlv),
 
+        SOC_SINGLE("Noise Gate Threshold", M98088_REG_40_MICAGC_THRESH,
+               4, 15, 0),
+
        SOC_SINGLE("INA Volume", M98088_REG_37_LVL_INA, 0, 7, 1),
        SOC_SINGLE("INB Volume", M98088_REG_38_LVL_INB, 0, 7, 1),
 
@@ -1746,7 +1749,6 @@ MODULE_DEVICE_TABLE(i2c, max98088_i2c_id);
 static int max98088_i2c_probe(struct i2c_client *i2c)
 {
 	struct max98088_priv *max98088;
-	int ret;
 	const struct i2c_device_id *id;
 
 	max98088 = devm_kzalloc(&i2c->dev, sizeof(struct max98088_priv),
@@ -1769,9 +1771,8 @@ static int max98088_i2c_probe(struct i2c_client *i2c)
 	i2c_set_clientdata(i2c, max98088);
 	max98088->pdata = i2c->dev.platform_data;
 
-	ret = devm_snd_soc_register_component(&i2c->dev, &soc_component_dev_max98088,
+	return devm_snd_soc_register_component(&i2c->dev, &soc_component_dev_max98088,
 					      &max98088_dai[0], 2);
-	return ret;
 }
 
 #if defined(CONFIG_OF)
diff --git a/sound/soc/codecs/max98090.c b/sound/soc/codecs/max98090.c
index 142083b13ac3..06ed2a938108 100644
--- a/sound/soc/codecs/max98090.c
+++ b/sound/soc/codecs/max98090.c
@@ -2615,11 +2615,9 @@ static void max98090_i2c_shutdown(struct i2c_client *i2c)
 	msleep(40);
 }
 
-static int max98090_i2c_remove(struct i2c_client *client)
+static void max98090_i2c_remove(struct i2c_client *client)
 {
 	max98090_i2c_shutdown(client);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/sound/soc/codecs/max98373-sdw.c b/sound/soc/codecs/max98373-sdw.c
index 97b64477dde6..899965b19d12 100644
--- a/sound/soc/codecs/max98373-sdw.c
+++ b/sound/soc/codecs/max98373-sdw.c
@@ -281,6 +281,8 @@ static __maybe_unused int max98373_resume(struct device *dev)
 					   msecs_to_jiffies(MAX98373_PROBE_TIMEOUT));
 	if (!time) {
 		dev_err(dev, "Initialization not complete, timed out\n");
+		sdw_show_ping_status(slave->bus, true);
+
 		return -ETIMEDOUT;
 	}
 
diff --git a/sound/soc/codecs/max98390.c b/sound/soc/codecs/max98390.c
index 5c08166a8dc6..7a5260ff8d6b 100644
--- a/sound/soc/codecs/max98390.c
+++ b/sound/soc/codecs/max98390.c
@@ -161,8 +161,6 @@ static struct reg_default max98390_reg_defaults[] = {
 	{MAX98390_R23FF_GLOBAL_EN, 0x00},
 };
 
-static int max98390_dsm_calibrate(struct snd_soc_component *component);
-
 static int max98390_dai_set_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
 {
 	struct snd_soc_component *component = codec_dai->component;
@@ -635,10 +633,48 @@ static int max98390_dsm_calib_get(struct snd_kcontrol *kcontrol,
 static int max98390_dsm_calib_put(struct snd_kcontrol *kcontrol,
 		struct snd_ctl_elem_value *ucontrol)
 {
-	struct snd_soc_component *component =
-		snd_soc_kcontrol_component(kcontrol);
+	struct snd_soc_component *component = snd_soc_kcontrol_component(kcontrol);
+	struct max98390_priv *max98390 = snd_soc_component_get_drvdata(component);
+	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component);
+	unsigned int rdc, rdc_cal_result, rdc_integer, rdc_factor, temp, val;
+
+	snd_soc_dapm_mutex_lock(dapm);
+
+	regmap_read(max98390->regmap, MAX98390_R23FF_GLOBAL_EN, &val);
+	if (!val) {
+		/* Enable the codec for the duration of calibration readout */
+		regmap_update_bits(max98390->regmap, MAX98390_R203A_AMP_EN,
+				   MAX98390_AMP_EN_MASK, 1);
+		regmap_update_bits(max98390->regmap, MAX98390_R23FF_GLOBAL_EN,
+				   MAX98390_GLOBAL_EN_MASK, 1);
+	}
+
+	regmap_read(max98390->regmap, THERMAL_RDC_RD_BACK_BYTE1, &rdc);
+	regmap_read(max98390->regmap, THERMAL_RDC_RD_BACK_BYTE0, &rdc_cal_result);
+	regmap_read(max98390->regmap, MAX98390_MEAS_ADC_CH2_READ, &temp);
+
+	if (!val) {
+		/* Disable the codec if it was disabled */
+		regmap_update_bits(max98390->regmap, MAX98390_R23FF_GLOBAL_EN,
+				   MAX98390_GLOBAL_EN_MASK, 0);
+		regmap_update_bits(max98390->regmap, MAX98390_R203A_AMP_EN,
+				   MAX98390_AMP_EN_MASK, 0);
+	}
+
+	snd_soc_dapm_mutex_unlock(dapm);
 
-	max98390_dsm_calibrate(component);
+	rdc_cal_result |= (rdc << 8) & 0x0000FFFF;
+	if (rdc_cal_result)
+		max98390->ref_rdc_value = 268435456U / rdc_cal_result;
+
+	max98390->ambient_temp_value = temp * 52 - 1188;
+
+	rdc_integer =  rdc_cal_result * 937  / 65536;
+	rdc_factor = ((rdc_cal_result * 937 * 100) / 65536) - (rdc_integer * 100);
+
+	dev_info(component->dev,
+		 "rdc resistance about %d.%02d ohm, reg=0x%X temp reg=0x%X\n",
+		 rdc_integer, rdc_factor, rdc_cal_result, temp);
 
 	return 0;
 }
@@ -819,40 +855,6 @@ err:
 	return ret;
 }
 
-static int max98390_dsm_calibrate(struct snd_soc_component *component)
-{
-	unsigned int rdc, rdc_cal_result, temp;
-	unsigned int rdc_integer, rdc_factor;
-	struct max98390_priv *max98390 =
-		snd_soc_component_get_drvdata(component);
-
-	regmap_write(max98390->regmap, MAX98390_R203A_AMP_EN, 0x81);
-	regmap_write(max98390->regmap, MAX98390_R23FF_GLOBAL_EN, 0x01);
-
-	regmap_read(max98390->regmap,
-		THERMAL_RDC_RD_BACK_BYTE1, &rdc);
-	regmap_read(max98390->regmap,
-		THERMAL_RDC_RD_BACK_BYTE0, &rdc_cal_result);
-	rdc_cal_result |= (rdc << 8) & 0x0000FFFF;
-	if (rdc_cal_result)
-		max98390->ref_rdc_value = 268435456U / rdc_cal_result;
-
-	regmap_read(max98390->regmap, MAX98390_MEAS_ADC_CH2_READ, &temp);
-	max98390->ambient_temp_value = temp * 52 - 1188;
-
-	rdc_integer =  rdc_cal_result * 937  / 65536;
-	rdc_factor = ((rdc_cal_result * 937 * 100) / 65536)
-					- (rdc_integer * 100);
-
-	dev_info(component->dev, "rdc resistance about %d.%02d ohm, reg=0x%X temp reg=0x%X\n",
-		 rdc_integer, rdc_factor, rdc_cal_result, temp);
-
-	regmap_write(max98390->regmap, MAX98390_R23FF_GLOBAL_EN, 0x00);
-	regmap_write(max98390->regmap, MAX98390_R203A_AMP_EN, 0x80);
-
-	return 0;
-}
-
 static void max98390_init_regs(struct snd_soc_component *component)
 {
 	struct max98390_priv *max98390 =
diff --git a/sound/soc/codecs/max98396.c b/sound/soc/codecs/max98396.c
index 364b4b7ee033..a7b6a23f2cd8 100644
--- a/sound/soc/codecs/max98396.c
+++ b/sound/soc/codecs/max98396.c
@@ -1483,6 +1483,87 @@ static int max98396_probe(struct snd_soc_component *component)
 			   MAX98396_CLK_MON_AUTO_RESTART_MASK,
 			   MAX98396_CLK_MON_AUTO_RESTART_MASK);
 
+	regmap_update_bits(max98396->regmap,
+			   MAX98396_R203F_ENABLE_CTRLS,
+			   MAX98396_CTRL_DMON_STUCK_EN_MASK,
+			   max98396->dmon_stuck_enable ?
+				MAX98396_CTRL_DMON_STUCK_EN_MASK : 0);
+
+	regmap_update_bits(max98396->regmap,
+			   MAX98396_R203F_ENABLE_CTRLS,
+			   MAX98396_CTRL_DMON_MAG_EN_MASK,
+			   max98396->dmon_mag_enable ?
+				MAX98396_CTRL_DMON_MAG_EN_MASK : 0);
+
+	switch (max98396->dmon_duration) {
+	case 64:
+		regmap_update_bits(max98396->regmap,
+				   MAX98396_R2039_DATA_MON_CTRL,
+				   MAX98396_DMON_DURATION_MASK, 0);
+		break;
+	case 256:
+		regmap_update_bits(max98396->regmap,
+				   MAX98396_R2039_DATA_MON_CTRL,
+				   MAX98396_DMON_DURATION_MASK, 1);
+		break;
+	case 1024:
+		regmap_update_bits(max98396->regmap,
+				   MAX98396_R2039_DATA_MON_CTRL,
+				   MAX98396_DMON_DURATION_MASK, 2);
+		break;
+	case 4096:
+		regmap_update_bits(max98396->regmap,
+				   MAX98396_R2039_DATA_MON_CTRL,
+				   MAX98396_DMON_DURATION_MASK, 3);
+		break;
+	default:
+		dev_err(component->dev, "Invalid DMON duration %d\n",
+			max98396->dmon_duration);
+	}
+
+	switch (max98396->dmon_stuck_threshold) {
+	case 15:
+		regmap_update_bits(max98396->regmap,
+				   MAX98396_R2039_DATA_MON_CTRL,
+				   MAX98396_DMON_STUCK_THRESH_MASK,
+				   0 << MAX98396_DMON_STUCK_THRESH_SHIFT);
+		break;
+	case 13:
+		regmap_update_bits(max98396->regmap,
+				   MAX98396_R2039_DATA_MON_CTRL,
+				   MAX98396_DMON_STUCK_THRESH_MASK,
+				   1 << MAX98396_DMON_STUCK_THRESH_SHIFT);
+		break;
+	case 22:
+		regmap_update_bits(max98396->regmap,
+				   MAX98396_R2039_DATA_MON_CTRL,
+				   MAX98396_DMON_STUCK_THRESH_MASK,
+				   2 << MAX98396_DMON_STUCK_THRESH_SHIFT);
+		break;
+	case 9:
+		regmap_update_bits(max98396->regmap,
+				   MAX98396_R2039_DATA_MON_CTRL,
+				   MAX98396_DMON_STUCK_THRESH_MASK,
+				   3 << MAX98396_DMON_STUCK_THRESH_SHIFT);
+		break;
+	default:
+		dev_err(component->dev, "Invalid DMON stuck threshold %d\n",
+			max98396->dmon_stuck_threshold);
+	}
+
+	switch (max98396->dmon_mag_threshold) {
+	case 2 ... 5:
+		regmap_update_bits(max98396->regmap,
+				   MAX98396_R2039_DATA_MON_CTRL,
+				   MAX98396_DMON_STUCK_THRESH_MASK,
+				   (5 - max98396->dmon_mag_threshold)
+					<< MAX98396_DMON_MAG_THRESH_SHIFT);
+		break;
+	default:
+		dev_err(component->dev, "Invalid DMON magnitude threshold %d\n",
+			max98396->dmon_mag_threshold);
+	}
+
 	/* Speaker Amplifier PCM RX Enable by default */
 	regmap_update_bits(max98396->regmap,
 			   MAX98396_R205E_PCM_RX_EN,
@@ -1614,6 +1695,27 @@ static void max98396_read_device_property(struct device *dev,
 		max98396->bypass_slot = value & 0xF;
 	else
 		max98396->bypass_slot = 0;
+
+	max98396->dmon_stuck_enable =
+		device_property_read_bool(dev, "adi,dmon-stuck-enable");
+
+	if (!device_property_read_u32(dev, "adi,dmon-stuck-threshold-bits", &value))
+		max98396->dmon_stuck_threshold = value;
+	else
+		max98396->dmon_stuck_threshold = 15;
+
+	max98396->dmon_mag_enable =
+		device_property_read_bool(dev, "adi,dmon-magnitude-enable");
+
+	if (!device_property_read_u32(dev, "adi,dmon-magnitude-threshold-bits", &value))
+		max98396->dmon_mag_threshold = value;
+	else
+		max98396->dmon_mag_threshold = 5;
+
+	if (!device_property_read_u32(dev, "adi,dmon-duration-ms", &value))
+		max98396->dmon_duration = value;
+	else
+		max98396->dmon_duration = 64;
 }
 
 static void max98396_core_supplies_disable(void *priv)
diff --git a/sound/soc/codecs/max98396.h b/sound/soc/codecs/max98396.h
index 7278c779989a..d396aa3e698b 100644
--- a/sound/soc/codecs/max98396.h
+++ b/sound/soc/codecs/max98396.h
@@ -212,8 +212,17 @@
 #define MAX98396_CLK_MON_AUTO_RESTART_MASK	(0x1 << 0)
 #define MAX98396_CLK_MON_AUTO_RESTART_SHIFT	(0)
 
+/* MAX98396_R2039_DATA_MON_CTRL */
+#define MAX98396_DMON_MAG_THRESH_SHIFT		(4)
+#define MAX98396_DMON_MAG_THRESH_MASK		(0x3 << MAX98396_DMON_MAG_THRESH_SHIFT)
+#define MAX98396_DMON_STUCK_THRESH_SHIFT	(2)
+#define MAX98396_DMON_STUCK_THRESH_MASK		(0x3 << MAX98396_DMON_STUCK_THRESH_SHIFT)
+#define MAX98396_DMON_DURATION_MASK		(0x3)
+
 /* MAX98396_R203F_ENABLE_CTRLS */
 #define MAX98396_CTRL_CMON_EN_SHIFT		(0)
+#define MAX98396_CTRL_DMON_STUCK_EN_MASK	(0x1 << 1)
+#define MAX98396_CTRL_DMON_MAG_EN_MASK		(0x1 << 2)
 
 /* MAX98396_R2041_PCM_MODE_CFG */
 #define MAX98396_PCM_MODE_CFG_FORMAT_MASK	(0x7 << 3)
@@ -305,6 +314,11 @@ struct max98396_priv {
 	unsigned int i_slot;
 	unsigned int spkfb_slot;
 	unsigned int bypass_slot;
+	bool dmon_stuck_enable;
+	unsigned int dmon_stuck_threshold;
+	bool dmon_mag_enable;
+	unsigned int dmon_mag_threshold;
+	unsigned int dmon_duration;
 	bool interleave_mode;
 	bool tdm_mode;
 	int tdm_max_samplerate;
diff --git a/sound/soc/codecs/max9860.c b/sound/soc/codecs/max9860.c
index 771b3dcd6cc3..9611ab1e79e5 100644
--- a/sound/soc/codecs/max9860.c
+++ b/sound/soc/codecs/max9860.c
@@ -701,14 +701,13 @@ err_regulator:
 	return ret;
 }
 
-static int max9860_remove(struct i2c_client *i2c)
+static void max9860_remove(struct i2c_client *i2c)
 {
 	struct device *dev = &i2c->dev;
 	struct max9860_priv *max9860 = dev_get_drvdata(dev);
 
 	pm_runtime_disable(dev);
 	regulator_disable(max9860->dvddio);
-	return 0;
 }
 
 static const struct i2c_device_id max9860_i2c_id[] = {
diff --git a/sound/soc/codecs/max98927.c b/sound/soc/codecs/max98927.c
index 9cce7c0f0142..331d3e1d735c 100644
--- a/sound/soc/codecs/max98927.c
+++ b/sound/soc/codecs/max98927.c
@@ -934,15 +934,13 @@ static int max98927_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static int max98927_i2c_remove(struct i2c_client *i2c)
+static void max98927_i2c_remove(struct i2c_client *i2c)
 {
 	struct max98927_priv *max98927 = i2c_get_clientdata(i2c);
 
 	if (max98927->reset_gpio) {
 		gpiod_set_value_cansleep(max98927->reset_gpio, 1);
 	}
-
-	return 0;
 }
 
 static const struct i2c_device_id max98927_i2c_id[] = {
diff --git a/sound/soc/codecs/mt6359-accdet.c b/sound/soc/codecs/mt6359-accdet.c
index c190628e2905..7f624854948c 100644
--- a/sound/soc/codecs/mt6359-accdet.c
+++ b/sound/soc/codecs/mt6359-accdet.c
@@ -965,7 +965,7 @@ static int mt6359_accdet_probe(struct platform_device *pdev)
 	mutex_init(&priv->res_lock);
 
 	priv->accdet_irq = platform_get_irq(pdev, 0);
-	if (priv->accdet_irq) {
+	if (priv->accdet_irq >= 0) {
 		ret = devm_request_threaded_irq(&pdev->dev, priv->accdet_irq,
 						NULL, mt6359_accdet_irq,
 						IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
@@ -979,7 +979,7 @@ static int mt6359_accdet_probe(struct platform_device *pdev)
 
 	if (priv->caps & ACCDET_PMIC_EINT0) {
 		priv->accdet_eint0 = platform_get_irq(pdev, 1);
-		if (priv->accdet_eint0) {
+		if (priv->accdet_eint0 >= 0) {
 			ret = devm_request_threaded_irq(&pdev->dev,
 							priv->accdet_eint0,
 							NULL, mt6359_accdet_irq,
@@ -994,7 +994,7 @@ static int mt6359_accdet_probe(struct platform_device *pdev)
 		}
 	} else if (priv->caps & ACCDET_PMIC_EINT1) {
 		priv->accdet_eint1 = platform_get_irq(pdev, 2);
-		if (priv->accdet_eint1) {
+		if (priv->accdet_eint1 >= 0) {
 			ret = devm_request_threaded_irq(&pdev->dev,
 							priv->accdet_eint1,
 							NULL, mt6359_accdet_irq,
diff --git a/sound/soc/codecs/mt6660.c b/sound/soc/codecs/mt6660.c
index ba11555796ad..554c33e8b62f 100644
--- a/sound/soc/codecs/mt6660.c
+++ b/sound/soc/codecs/mt6660.c
@@ -503,27 +503,30 @@ static int mt6660_i2c_probe(struct i2c_client *client)
 		dev_err(chip->dev, "read chip revision fail\n");
 		goto probe_fail;
 	}
-	pm_runtime_set_active(chip->dev);
-	pm_runtime_enable(chip->dev);
 
 	ret = devm_snd_soc_register_component(chip->dev,
 					       &mt6660_component_driver,
 					       &mt6660_codec_dai, 1);
+	if (!ret) {
+		pm_runtime_set_active(chip->dev);
+		pm_runtime_enable(chip->dev);
+	}
+
 	return ret;
+
 probe_fail:
 	_mt6660_chip_power_on(chip, 0);
 	mutex_destroy(&chip->io_lock);
 	return ret;
 }
 
-static int mt6660_i2c_remove(struct i2c_client *client)
+static void mt6660_i2c_remove(struct i2c_client *client)
 {
 	struct mt6660_chip *chip = i2c_get_clientdata(client);
 
 	pm_runtime_disable(chip->dev);
 	pm_runtime_set_suspended(chip->dev);
 	mutex_destroy(&chip->io_lock);
-	return 0;
 }
 
 static int __maybe_unused mt6660_i2c_runtime_suspend(struct device *dev)
diff --git a/sound/soc/codecs/nau8824.c b/sound/soc/codecs/nau8824.c
index 15596452ca37..4f19fd9b65d1 100644
--- a/sound/soc/codecs/nau8824.c
+++ b/sound/soc/codecs/nau8824.c
@@ -901,7 +901,10 @@ static void nau8824_jdet_work(struct work_struct *work)
 		NAU8824_IRQ_KEY_RELEASE_DIS |
 		NAU8824_IRQ_KEY_SHORT_PRESS_DIS, 0);
 
-	nau8824_sema_release(nau8824);
+	if (nau8824->resume_lock) {
+		nau8824_sema_release(nau8824);
+		nau8824->resume_lock = false;
+	}
 }
 
 static void nau8824_setup_auto_irq(struct nau8824 *nau8824)
@@ -966,7 +969,10 @@ static irqreturn_t nau8824_interrupt(int irq, void *data)
 		/* release semaphore held after resume,
 		 * and cancel jack detection
 		 */
-		nau8824_sema_release(nau8824);
+		if (nau8824->resume_lock) {
+			nau8824_sema_release(nau8824);
+			nau8824->resume_lock = false;
+		}
 		cancel_work_sync(&nau8824->jdet_work);
 	} else if (active_irq & NAU8824_KEY_SHORT_PRESS_IRQ) {
 		int key_status, button_pressed;
@@ -1524,6 +1530,7 @@ static int __maybe_unused nau8824_suspend(struct snd_soc_component *component)
 static int __maybe_unused nau8824_resume(struct snd_soc_component *component)
 {
 	struct nau8824 *nau8824 = snd_soc_component_get_drvdata(component);
+	int ret;
 
 	regcache_cache_only(nau8824->regmap, false);
 	regcache_sync(nau8824->regmap);
@@ -1531,7 +1538,10 @@ static int __maybe_unused nau8824_resume(struct snd_soc_component *component)
 		/* Hold semaphore to postpone playback happening
 		 * until jack detection done.
 		 */
-		nau8824_sema_acquire(nau8824, 0);
+		nau8824->resume_lock = true;
+		ret = nau8824_sema_acquire(nau8824, 0);
+		if (ret)
+			nau8824->resume_lock = false;
 		enable_irq(nau8824->irq);
 	}
 
@@ -1940,6 +1950,7 @@ static int nau8824_i2c_probe(struct i2c_client *i2c)
 	nau8824->regmap = devm_regmap_init_i2c(i2c, &nau8824_regmap_config);
 	if (IS_ERR(nau8824->regmap))
 		return PTR_ERR(nau8824->regmap);
+	nau8824->resume_lock = false;
 	nau8824->dev = dev;
 	nau8824->irq = i2c->irq;
 	sema_init(&nau8824->jd_sem, 1);
diff --git a/sound/soc/codecs/nau8824.h b/sound/soc/codecs/nau8824.h
index de4bae8281d0..5fcfc43dfc85 100644
--- a/sound/soc/codecs/nau8824.h
+++ b/sound/soc/codecs/nau8824.h
@@ -436,6 +436,7 @@ struct nau8824 {
 	struct semaphore jd_sem;
 	int fs;
 	int irq;
+	int resume_lock;
 	int micbias_voltage;
 	int vref_impedance;
 	int jkdet_polarity;
diff --git a/sound/soc/codecs/nau8825.c b/sound/soc/codecs/nau8825.c
index 8213273f501e..3eac7c92df88 100644
--- a/sound/soc/codecs/nau8825.c
+++ b/sound/soc/codecs/nau8825.c
@@ -1425,10 +1425,107 @@ static int nau8825_set_dai_fmt(struct snd_soc_dai *codec_dai, unsigned int fmt)
 	return 0;
 }
 
+/**
+ * nau8825_set_tdm_slot - configure DAI TDM.
+ * @dai: DAI
+ * @tx_mask: bitmask representing active TX slots.
+ * @rx_mask: bitmask representing active RX slots.
+ * @slots: Number of slots in use.
+ * @slot_width: Width in bits for each slot.
+ *
+ * Configures a DAI for TDM operation. Support TDM 4/8 slots.
+ * The limitation is DAC and ADC need shift 4 slots at 8 slots mode.
+ */
+static int nau8825_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
+				unsigned int rx_mask, int slots, int slot_width)
+{
+	struct snd_soc_component *component = dai->component;
+	struct nau8825 *nau8825 = snd_soc_component_get_drvdata(component);
+	unsigned int ctrl_val = 0, ctrl_offset = 0, value = 0, dac_s, adc_s;
+
+	if (slots != 4 && slots != 8) {
+		dev_err(nau8825->dev, "Only support 4 or 8 slots!\n");
+		return -EINVAL;
+	}
+
+	/* The driver is limited to 1-channel for ADC, and 2-channel for DAC on TDM mode */
+	if (hweight_long((unsigned long) tx_mask) != 1 ||
+	    hweight_long((unsigned long) rx_mask) != 2) {
+		dev_err(nau8825->dev,
+			"The limitation is 1-channel for ADC, and 2-channel for DAC on TDM mode.\n");
+		return -EINVAL;
+	}
+
+	if (((tx_mask & 0xf) && (tx_mask & 0xf0)) ||
+	    ((rx_mask & 0xf) && (rx_mask & 0xf0)) ||
+	    ((tx_mask & 0xf) && (rx_mask & 0xf0)) ||
+	    ((rx_mask & 0xf) && (tx_mask & 0xf0))) {
+		dev_err(nau8825->dev,
+			"Slot assignment of DAC and ADC need to set same interval.\n");
+		return -EINVAL;
+	}
+
+	/* The offset of fixed 4 slots for 8 slots support */
+	if (rx_mask & 0xf0) {
+		regmap_update_bits(nau8825->regmap, NAU8825_REG_I2S_PCM_CTRL2,
+				   NAU8825_I2S_PCM_TS_EN_MASK, NAU8825_I2S_PCM_TS_EN);
+		regmap_read(nau8825->regmap, NAU8825_REG_I2S_PCM_CTRL1, &value);
+		ctrl_val |= NAU8825_TDM_OFFSET_EN;
+		ctrl_offset = 4 * slot_width;
+		if (!(value & NAU8825_I2S_PCMB_MASK))
+			ctrl_offset += 1;
+		dac_s = (rx_mask & 0xf0) >> 4;
+		adc_s = fls((tx_mask & 0xf0) >> 4);
+	} else {
+		dac_s = rx_mask & 0xf;
+		adc_s = fls(tx_mask & 0xf);
+	}
+
+	ctrl_val |= NAU8825_TDM_MODE;
+
+	switch (dac_s) {
+	case 0x3:
+		ctrl_val |= 1 << NAU8825_TDM_DACR_RX_SFT;
+		break;
+	case 0x5:
+		ctrl_val |= 2 << NAU8825_TDM_DACR_RX_SFT;
+		break;
+	case 0x6:
+		ctrl_val |= 1 << NAU8825_TDM_DACL_RX_SFT;
+		ctrl_val |= 2 << NAU8825_TDM_DACR_RX_SFT;
+		break;
+	case 0x9:
+		ctrl_val |= 3 << NAU8825_TDM_DACR_RX_SFT;
+		break;
+	case 0xa:
+		ctrl_val |= 1 << NAU8825_TDM_DACL_RX_SFT;
+		ctrl_val |= 3 << NAU8825_TDM_DACR_RX_SFT;
+		break;
+	case 0xc:
+		ctrl_val |= 2 << NAU8825_TDM_DACL_RX_SFT;
+		ctrl_val |= 3 << NAU8825_TDM_DACR_RX_SFT;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	ctrl_val |= adc_s - 1;
+
+	regmap_update_bits(nau8825->regmap, NAU8825_REG_TDM_CTRL,
+			   NAU8825_TDM_MODE | NAU8825_TDM_OFFSET_EN |
+			   NAU8825_TDM_DACL_RX_MASK | NAU8825_TDM_DACR_RX_MASK |
+			   NAU8825_TDM_TX_MASK, ctrl_val);
+	regmap_update_bits(nau8825->regmap, NAU8825_REG_LEFT_TIME_SLOT,
+			   NAU8825_TSLOT_L0_MASK, ctrl_offset);
+
+	return 0;
+}
+
 static const struct snd_soc_dai_ops nau8825_dai_ops = {
 	.startup	= nau8825_dai_startup,
 	.hw_params	= nau8825_hw_params,
 	.set_fmt	= nau8825_set_dai_fmt,
+	.set_tdm_slot	= nau8825_set_tdm_slot,
 };
 
 #define NAU8825_RATES	SNDRV_PCM_RATE_8000_192000
@@ -1983,6 +2080,10 @@ static void nau8825_init_regs(struct nau8825 *nau8825)
 	/* Disable short Frame Sync detection logic */
 	regmap_update_bits(regmap, NAU8825_REG_LEFT_TIME_SLOT,
 		NAU8825_DIS_FS_SHORT_DET, NAU8825_DIS_FS_SHORT_DET);
+	/* ADCDAT IO drive strength control */
+	regmap_update_bits(regmap, NAU8825_REG_CHARGE_PUMP,
+			   NAU8825_ADCOUT_DS_MASK,
+			   nau8825->adcout_ds << NAU8825_ADCOUT_DS_SFT);
 }
 
 static const struct regmap_config nau8825_regmap_config = {
@@ -2521,6 +2622,7 @@ static void nau8825_print_device_properties(struct nau8825 *nau8825)
 			nau8825->jack_eject_debounce);
 	dev_dbg(dev, "crosstalk-enable:     %d\n",
 			nau8825->xtalk_enable);
+	dev_dbg(dev, "adcout-drive-strong:  %d\n", nau8825->adcout_ds);
 }
 
 static int nau8825_read_device_properties(struct device *dev,
@@ -2587,6 +2689,7 @@ static int nau8825_read_device_properties(struct device *dev,
 		nau8825->jack_eject_debounce = 0;
 	nau8825->xtalk_enable = device_property_read_bool(dev,
 		"nuvoton,crosstalk-enable");
+	nau8825->adcout_ds = device_property_read_bool(dev, "nuvoton,adcout-drive-strong");
 
 	nau8825->mclk = devm_clk_get(dev, "mclk");
 	if (PTR_ERR(nau8825->mclk) == -EPROBE_DEFER) {
@@ -2675,10 +2778,8 @@ static int nau8825_i2c_probe(struct i2c_client *i2c)
 		&nau8825_dai, 1);
 }
 
-static int nau8825_i2c_remove(struct i2c_client *client)
-{
-	return 0;
-}
+static void nau8825_i2c_remove(struct i2c_client *client)
+{}
 
 static const struct i2c_device_id nau8825_i2c_ids[] = {
 	{ "nau8825", 0 },
diff --git a/sound/soc/codecs/nau8825.h b/sound/soc/codecs/nau8825.h
index 887bbff03ec6..d84191a7beb2 100644
--- a/sound/soc/codecs/nau8825.h
+++ b/sound/soc/codecs/nau8825.h
@@ -225,6 +225,15 @@
 #define NAU8825_JKDET_PULL_EN	(1 << 9) /* 0 - enable pull, 1 - disable */
 #define NAU8825_JKDET_OUTPUT_EN	(1 << 8) /* 0 - enable input, 1 - enable output */
 
+/* TDM_CTRL (0x1b) */
+#define NAU8825_TDM_MODE		(0x1 << 15)
+#define NAU8825_TDM_OFFSET_EN		(0x1 << 14)
+#define NAU8825_TDM_DACL_RX_SFT		6
+#define NAU8825_TDM_DACL_RX_MASK	(0x3 << NAU8825_TDM_DACL_RX_SFT)
+#define NAU8825_TDM_DACR_RX_SFT		4
+#define NAU8825_TDM_DACR_RX_MASK	(0x3 << NAU8825_TDM_DACR_RX_SFT)
+#define NAU8825_TDM_TX_MASK		0x3
+
 /* I2S_PCM_CTRL1 (0x1c) */
 #define NAU8825_I2S_BP_SFT	7
 #define NAU8825_I2S_BP_MASK	(1 << NAU8825_I2S_BP_SFT)
@@ -249,6 +258,9 @@
 #define NAU8825_I2S_TRISTATE	(1 << 15) /* 0 - normal mode, 1 - Hi-Z output */
 #define NAU8825_I2S_LRC_DIV_SFT	12
 #define NAU8825_I2S_LRC_DIV_MASK	(0x3 << NAU8825_I2S_LRC_DIV_SFT)
+#define NAU8825_I2S_PCM_TS_EN_SFT	10
+#define NAU8825_I2S_PCM_TS_EN_MASK	(1 << NAU8825_I2S_PCM_TS_EN_SFT)
+#define NAU8825_I2S_PCM_TS_EN		(1 << NAU8825_I2S_PCM_TS_EN_SFT)
 #define NAU8825_I2S_MS_SFT	3
 #define NAU8825_I2S_MS_MASK	(1 << NAU8825_I2S_MS_SFT)
 #define NAU8825_I2S_MS_MASTER	(1 << NAU8825_I2S_MS_SFT)
@@ -259,6 +271,8 @@
 #define NAU8825_FS_ERR_CMP_SEL_SFT	14
 #define NAU8825_FS_ERR_CMP_SEL_MASK	(0x3 << NAU8825_FS_ERR_CMP_SEL_SFT)
 #define NAU8825_DIS_FS_SHORT_DET	(1 << 13)
+#define NAU8825_TSLOT_L0_MASK		0x3ff
+#define NAU8825_TSLOT_R0_MASK		0x3ff
 
 /* BIQ_CTRL (0x20) */
 #define NAU8825_BIQ_WRT_SFT   4
@@ -418,6 +432,8 @@
 #define NAU8825_POWERUP_HP_DRV_L	(1 << 0)
 
 /* CHARGE_PUMP (0x80) */
+#define NAU8825_ADCOUT_DS_SFT	12
+#define NAU8825_ADCOUT_DS_MASK	(1 << NAU8825_ADCOUT_DS_SFT)
 #define NAU8825_JAMNODCLOW	(1 << 10)
 #define NAU8825_POWER_DOWN_DACR	(1 << 9)
 #define NAU8825_POWER_DOWN_DACL	(1 << 8)
@@ -477,6 +493,7 @@ struct nau8825 {
 	int imp_rms[NAU8825_XTALK_IMM];
 	int xtalk_enable;
 	bool xtalk_baktab_initialized; /* True if initialized. */
+	bool adcout_ds;
 };
 
 int nau8825_enable_jack_detect(struct snd_soc_component *component,
diff --git a/sound/soc/codecs/pcm1789-i2c.c b/sound/soc/codecs/pcm1789-i2c.c
index 1d2f7480a6e4..fafe0dcbe4ea 100644
--- a/sound/soc/codecs/pcm1789-i2c.c
+++ b/sound/soc/codecs/pcm1789-i2c.c
@@ -27,11 +27,9 @@ static int pcm1789_i2c_probe(struct i2c_client *client)
 	return pcm1789_common_init(&client->dev, regmap);
 }
 
-static int pcm1789_i2c_remove(struct i2c_client *client)
+static void pcm1789_i2c_remove(struct i2c_client *client)
 {
 	pcm1789_common_exit(&client->dev);
-
-	return 0;
 }
 
 #ifdef CONFIG_OF
diff --git a/sound/soc/codecs/pcm3168a-i2c.c b/sound/soc/codecs/pcm3168a-i2c.c
index c0fa0dc80e8f..a0eec82e9872 100644
--- a/sound/soc/codecs/pcm3168a-i2c.c
+++ b/sound/soc/codecs/pcm3168a-i2c.c
@@ -26,11 +26,9 @@ static int pcm3168a_i2c_probe(struct i2c_client *i2c)
 	return pcm3168a_probe(&i2c->dev, regmap);
 }
 
-static int pcm3168a_i2c_remove(struct i2c_client *i2c)
+static void pcm3168a_i2c_remove(struct i2c_client *i2c)
 {
 	pcm3168a_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id pcm3168a_i2c_id[] = {
diff --git a/sound/soc/codecs/pcm512x-i2c.c b/sound/soc/codecs/pcm512x-i2c.c
index 81754e141a55..9dfbbe8f4a0b 100644
--- a/sound/soc/codecs/pcm512x-i2c.c
+++ b/sound/soc/codecs/pcm512x-i2c.c
@@ -29,10 +29,9 @@ static int pcm512x_i2c_probe(struct i2c_client *i2c)
 	return pcm512x_probe(&i2c->dev, regmap);
 }
 
-static int pcm512x_i2c_remove(struct i2c_client *i2c)
+static void pcm512x_i2c_remove(struct i2c_client *i2c)
 {
 	pcm512x_remove(&i2c->dev);
-	return 0;
 }
 
 static const struct i2c_device_id pcm512x_i2c_id[] = {
diff --git a/sound/soc/codecs/rt1308-sdw.c b/sound/soc/codecs/rt1308-sdw.c
index 0be6e72ff5a9..5c29416aa781 100644
--- a/sound/soc/codecs/rt1308-sdw.c
+++ b/sound/soc/codecs/rt1308-sdw.c
@@ -749,6 +749,8 @@ static int __maybe_unused rt1308_dev_resume(struct device *dev)
 				msecs_to_jiffies(RT1308_PROBE_TIMEOUT));
 	if (!time) {
 		dev_err(&slave->dev, "Initialization not complete, timed out\n");
+		sdw_show_ping_status(slave->bus, true);
+
 		return -ETIMEDOUT;
 	}
 
diff --git a/sound/soc/codecs/rt1316-sdw.c b/sound/soc/codecs/rt1316-sdw.c
index e53396606a1c..ed0a11436362 100644
--- a/sound/soc/codecs/rt1316-sdw.c
+++ b/sound/soc/codecs/rt1316-sdw.c
@@ -734,6 +734,8 @@ static int __maybe_unused rt1316_dev_resume(struct device *dev)
 				msecs_to_jiffies(RT1316_PROBE_TIMEOUT));
 	if (!time) {
 		dev_err(&slave->dev, "Initialization not complete, timed out\n");
+		sdw_show_ping_status(slave->bus, true);
+
 		return -ETIMEDOUT;
 	}
 
diff --git a/sound/soc/codecs/rt274.c b/sound/soc/codecs/rt274.c
index f2c50b11e4d0..4667bf7561b1 100644
--- a/sound/soc/codecs/rt274.c
+++ b/sound/soc/codecs/rt274.c
@@ -1204,14 +1204,12 @@ static int rt274_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static int rt274_i2c_remove(struct i2c_client *i2c)
+static void rt274_i2c_remove(struct i2c_client *i2c)
 {
 	struct rt274_priv *rt274 = i2c_get_clientdata(i2c);
 
 	if (i2c->irq)
 		free_irq(i2c->irq, rt274);
-
-	return 0;
 }
 
 
diff --git a/sound/soc/codecs/rt286.c b/sound/soc/codecs/rt286.c
index c4f7c4c2d793..ceb56647e369 100644
--- a/sound/soc/codecs/rt286.c
+++ b/sound/soc/codecs/rt286.c
@@ -1249,14 +1249,12 @@ static int rt286_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static int rt286_i2c_remove(struct i2c_client *i2c)
+static void rt286_i2c_remove(struct i2c_client *i2c)
 {
 	struct rt286_priv *rt286 = i2c_get_clientdata(i2c);
 
 	if (i2c->irq)
 		free_irq(i2c->irq, rt286);
-
-	return 0;
 }
 
 
diff --git a/sound/soc/codecs/rt298.c b/sound/soc/codecs/rt298.c
index b0b53d4f07df..a2ce52dafea8 100644
--- a/sound/soc/codecs/rt298.c
+++ b/sound/soc/codecs/rt298.c
@@ -1290,14 +1290,12 @@ static int rt298_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static int rt298_i2c_remove(struct i2c_client *i2c)
+static void rt298_i2c_remove(struct i2c_client *i2c)
 {
 	struct rt298_priv *rt298 = i2c_get_clientdata(i2c);
 
 	if (i2c->irq)
 		free_irq(i2c->irq, rt298);
-
-	return 0;
 }
 
 
diff --git a/sound/soc/codecs/rt5616.c b/sound/soc/codecs/rt5616.c
index 970d6c4a358e..948abde10463 100644
--- a/sound/soc/codecs/rt5616.c
+++ b/sound/soc/codecs/rt5616.c
@@ -1388,10 +1388,8 @@ static int rt5616_i2c_probe(struct i2c_client *i2c)
 				      rt5616_dai, ARRAY_SIZE(rt5616_dai));
 }
 
-static int rt5616_i2c_remove(struct i2c_client *i2c)
-{
-	return 0;
-}
+static void rt5616_i2c_remove(struct i2c_client *i2c)
+{}
 
 static void rt5616_i2c_shutdown(struct i2c_client *client)
 {
diff --git a/sound/soc/codecs/rt5631.c b/sound/soc/codecs/rt5631.c
index 957f6b19beec..55c232413e2b 100644
--- a/sound/soc/codecs/rt5631.c
+++ b/sound/soc/codecs/rt5631.c
@@ -1720,10 +1720,8 @@ static int rt5631_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static int rt5631_i2c_remove(struct i2c_client *client)
-{
-	return 0;
-}
+static void rt5631_i2c_remove(struct i2c_client *client)
+{}
 
 static struct i2c_driver rt5631_i2c_driver = {
 	.driver = {
diff --git a/sound/soc/codecs/rt5640.c b/sound/soc/codecs/rt5640.c
index 5a844329800f..0f8e6dd214b0 100644
--- a/sound/soc/codecs/rt5640.c
+++ b/sound/soc/codecs/rt5640.c
@@ -2494,7 +2494,7 @@ static void rt5640_enable_jack_detect(struct snd_soc_component *component,
 
 	/* Select JD-source */
 	snd_soc_component_update_bits(component, RT5640_JD_CTRL,
-		RT5640_JD_MASK, rt5640->jd_src);
+		RT5640_JD_MASK, rt5640->jd_src << RT5640_JD_SFT);
 
 	/* Selecting GPIO01 as an interrupt */
 	snd_soc_component_update_bits(component, RT5640_GPIO_CTRL1,
@@ -2504,12 +2504,8 @@ static void rt5640_enable_jack_detect(struct snd_soc_component *component,
 	snd_soc_component_update_bits(component, RT5640_GPIO_CTRL3,
 		RT5640_GP1_PF_MASK, RT5640_GP1_PF_OUT);
 
-	/* Enabling jd2 in general control 1 */
 	snd_soc_component_write(component, RT5640_DUMMY1, 0x3f41);
 
-	/* Enabling jd2 in general control 2 */
-	snd_soc_component_write(component, RT5640_DUMMY2, 0x4001);
-
 	rt5640_set_ovcd_params(component);
 
 	/*
@@ -2518,12 +2514,25 @@ static void rt5640_enable_jack_detect(struct snd_soc_component *component,
 	 * pin 0/1 instead of it being stuck to 1. So we invert the JD polarity
 	 * on systems where the hardware does not already do this.
 	 */
-	if (rt5640->jd_inverted)
-		snd_soc_component_write(component, RT5640_IRQ_CTRL1,
-					RT5640_IRQ_JD_NOR);
-	else
-		snd_soc_component_write(component, RT5640_IRQ_CTRL1,
-					RT5640_IRQ_JD_NOR | RT5640_JD_P_INV);
+	if (rt5640->jd_inverted) {
+		if (rt5640->jd_src == RT5640_JD_SRC_JD1_IN4P)
+			snd_soc_component_write(component, RT5640_IRQ_CTRL1,
+				RT5640_IRQ_JD_NOR);
+		else if (rt5640->jd_src == RT5640_JD_SRC_JD2_IN4N)
+			snd_soc_component_update_bits(component, RT5640_DUMMY2,
+				RT5640_IRQ_JD2_MASK | RT5640_JD2_MASK,
+				RT5640_IRQ_JD2_NOR | RT5640_JD2_EN);
+	} else {
+		if (rt5640->jd_src == RT5640_JD_SRC_JD1_IN4P)
+			snd_soc_component_write(component, RT5640_IRQ_CTRL1,
+				RT5640_IRQ_JD_NOR | RT5640_JD_P_INV);
+		else if (rt5640->jd_src == RT5640_JD_SRC_JD2_IN4N)
+			snd_soc_component_update_bits(component, RT5640_DUMMY2,
+				RT5640_IRQ_JD2_MASK | RT5640_JD2_P_MASK |
+				RT5640_JD2_MASK,
+				RT5640_IRQ_JD2_NOR | RT5640_JD2_P_INV |
+				RT5640_JD2_EN);
+	}
 
 	rt5640->jack = jack;
 	if (rt5640->jack->status & SND_JACK_MICROPHONE) {
@@ -2725,10 +2734,8 @@ static int rt5640_probe(struct snd_soc_component *component)
 
 	if (device_property_read_u32(component->dev,
 				     "realtek,jack-detect-source", &val) == 0) {
-		if (val <= RT5640_JD_SRC_GPIO4)
-			rt5640->jd_src = val << RT5640_JD_SFT;
-		else if (val == RT5640_JD_SRC_HDA_HEADER)
-			rt5640->jd_src = RT5640_JD_SRC_HDA_HEADER;
+		if (val <= RT5640_JD_SRC_HDA_HEADER)
+			rt5640->jd_src = val;
 		else
 			dev_warn(component->dev, "Warning: Invalid jack-detect-source value: %d, leaving jack-detect disabled\n",
 				 val);
@@ -2809,12 +2816,31 @@ static int rt5640_resume(struct snd_soc_component *component)
 	regcache_sync(rt5640->regmap);
 
 	if (rt5640->jack) {
-		if (rt5640->jd_src == RT5640_JD_SRC_HDA_HEADER)
+		if (rt5640->jd_src == RT5640_JD_SRC_HDA_HEADER) {
 			snd_soc_component_update_bits(component,
 				RT5640_DUMMY2, 0x1100, 0x1100);
-		else
-			snd_soc_component_write(component, RT5640_DUMMY2,
-				0x4001);
+		} else {
+			if (rt5640->jd_inverted) {
+				if (rt5640->jd_src == RT5640_JD_SRC_JD2_IN4N)
+					snd_soc_component_update_bits(
+						component, RT5640_DUMMY2,
+						RT5640_IRQ_JD2_MASK |
+						RT5640_JD2_MASK,
+						RT5640_IRQ_JD2_NOR |
+						RT5640_JD2_EN);
+
+			} else {
+				if (rt5640->jd_src == RT5640_JD_SRC_JD2_IN4N)
+					snd_soc_component_update_bits(
+						component, RT5640_DUMMY2,
+						RT5640_IRQ_JD2_MASK |
+						RT5640_JD2_P_MASK |
+						RT5640_JD2_MASK,
+						RT5640_IRQ_JD2_NOR |
+						RT5640_JD2_P_INV |
+						RT5640_JD2_EN);
+			}
+		}
 
 		queue_delayed_work(system_long_wq, &rt5640->jack_work, 0);
 	}
diff --git a/sound/soc/codecs/rt5640.h b/sound/soc/codecs/rt5640.h
index 505c93514051..f58b88e3325b 100644
--- a/sound/soc/codecs/rt5640.h
+++ b/sound/soc/codecs/rt5640.h
@@ -1984,6 +1984,20 @@
 #define RT5640_M_MONO_ADC_R_SFT			12
 #define RT5640_MCLK_DET				(0x1 << 11)
 
+/* General Control 1 (0xfb) */
+#define RT5640_IRQ_JD2_MASK			(0x1 << 12)
+#define RT5640_IRQ_JD2_SFT			12
+#define RT5640_IRQ_JD2_BP			(0x0 << 12)
+#define RT5640_IRQ_JD2_NOR			(0x1 << 12)
+#define RT5640_JD2_P_MASK			(0x1 << 10)
+#define RT5640_JD2_P_SFT			10
+#define RT5640_JD2_P_NOR			(0x0 << 10)
+#define RT5640_JD2_P_INV			(0x1 << 10)
+#define RT5640_JD2_MASK				(0x1 << 8)
+#define RT5640_JD2_SFT				8
+#define RT5640_JD2_DIS				(0x0 << 8)
+#define RT5640_JD2_EN				(0x1 << 8)
+
 /* Codec Private Register definition */
 
 /* MIC Over current threshold scale factor (0x15) */
diff --git a/sound/soc/codecs/rt5645.c b/sound/soc/codecs/rt5645.c
index 8635bc6567dc..620ecbfa4a7a 100644
--- a/sound/soc/codecs/rt5645.c
+++ b/sound/soc/codecs/rt5645.c
@@ -4145,7 +4145,7 @@ err_enable:
 	return ret;
 }
 
-static int rt5645_i2c_remove(struct i2c_client *i2c)
+static void rt5645_i2c_remove(struct i2c_client *i2c)
 {
 	struct rt5645_priv *rt5645 = i2c_get_clientdata(i2c);
 
@@ -4162,8 +4162,6 @@ static int rt5645_i2c_remove(struct i2c_client *i2c)
 	cancel_delayed_work_sync(&rt5645->rcclock_work);
 
 	regulator_bulk_disable(ARRAY_SIZE(rt5645->supplies), rt5645->supplies);
-
-	return 0;
 }
 
 static void rt5645_i2c_shutdown(struct i2c_client *i2c)
diff --git a/sound/soc/codecs/rt5663.c b/sound/soc/codecs/rt5663.c
index ca981b374b0c..f73751dbde30 100644
--- a/sound/soc/codecs/rt5663.c
+++ b/sound/soc/codecs/rt5663.c
@@ -3710,7 +3710,7 @@ err_enable:
 	return ret;
 }
 
-static int rt5663_i2c_remove(struct i2c_client *i2c)
+static void rt5663_i2c_remove(struct i2c_client *i2c)
 {
 	struct rt5663_priv *rt5663 = i2c_get_clientdata(i2c);
 
@@ -3718,8 +3718,6 @@ static int rt5663_i2c_remove(struct i2c_client *i2c)
 		free_irq(i2c->irq, rt5663);
 
 	regulator_bulk_disable(ARRAY_SIZE(rt5663->supplies), rt5663->supplies);
-
-	return 0;
 }
 
 static void rt5663_i2c_shutdown(struct i2c_client *client)
diff --git a/sound/soc/codecs/rt5670.c b/sound/soc/codecs/rt5670.c
index 60dbfa2a54f1..ebac6caeb40a 100644
--- a/sound/soc/codecs/rt5670.c
+++ b/sound/soc/codecs/rt5670.c
@@ -3320,11 +3320,9 @@ err:
 	return ret;
 }
 
-static int rt5670_i2c_remove(struct i2c_client *i2c)
+static void rt5670_i2c_remove(struct i2c_client *i2c)
 {
 	pm_runtime_disable(&i2c->dev);
-
-	return 0;
 }
 
 static struct i2c_driver rt5670_i2c_driver = {
diff --git a/sound/soc/codecs/rt5677.c b/sound/soc/codecs/rt5677.c
index 31a2dd0aafb6..c26395f42d8e 100644
--- a/sound/soc/codecs/rt5677.c
+++ b/sound/soc/codecs/rt5677.c
@@ -5693,11 +5693,9 @@ static int rt5677_i2c_probe(struct i2c_client *i2c)
 				      rt5677_dai, ARRAY_SIZE(rt5677_dai));
 }
 
-static int rt5677_i2c_remove(struct i2c_client *i2c)
+static void rt5677_i2c_remove(struct i2c_client *i2c)
 {
 	rt5677_free_gpio(i2c);
-
-	return 0;
 }
 
 static struct i2c_driver rt5677_i2c_driver = {
diff --git a/sound/soc/codecs/rt5682-i2c.c b/sound/soc/codecs/rt5682-i2c.c
index 3f72f6093436..2935c1bb81f3 100644
--- a/sound/soc/codecs/rt5682-i2c.c
+++ b/sound/soc/codecs/rt5682-i2c.c
@@ -302,11 +302,9 @@ static void rt5682_i2c_shutdown(struct i2c_client *client)
 	rt5682_reset(rt5682);
 }
 
-static int rt5682_i2c_remove(struct i2c_client *client)
+static void rt5682_i2c_remove(struct i2c_client *client)
 {
 	rt5682_i2c_shutdown(client);
-
-	return 0;
 }
 
 static const struct of_device_id rt5682_of_match[] = {
diff --git a/sound/soc/codecs/rt5682-sdw.c b/sound/soc/codecs/rt5682-sdw.c
index f04e18c32489..c1a94229dc7e 100644
--- a/sound/soc/codecs/rt5682-sdw.c
+++ b/sound/soc/codecs/rt5682-sdw.c
@@ -793,6 +793,8 @@ static int __maybe_unused rt5682_dev_resume(struct device *dev)
 				msecs_to_jiffies(RT5682_PROBE_TIMEOUT));
 	if (!time) {
 		dev_err(&slave->dev, "Initialization not complete, timed out\n");
+		sdw_show_ping_status(slave->bus, true);
+
 		return -ETIMEDOUT;
 	}
 
diff --git a/sound/soc/codecs/rt5682s.c b/sound/soc/codecs/rt5682s.c
index eb47e7cd485a..466a37f3500c 100644
--- a/sound/soc/codecs/rt5682s.c
+++ b/sound/soc/codecs/rt5682s.c
@@ -739,6 +739,7 @@ static void rt5682s_disable_push_button_irq(struct snd_soc_component *component)
  */
 static int rt5682s_headset_detect(struct snd_soc_component *component, int jack_insert)
 {
+	struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component);
 	unsigned int val, count;
 	int jack_type = 0;
 
@@ -805,12 +806,10 @@ static int rt5682s_headset_detect(struct snd_soc_component *component, int jack_
 		snd_soc_component_update_bits(component, RT5682S_CBJ_CTRL_1,
 			RT5682S_TRIG_JD_MASK, RT5682S_TRIG_JD_LOW);
 
-		if (!snd_soc_dapm_get_pin_status(&component->dapm, "MICBIAS"))
-			snd_soc_component_update_bits(component,
-				RT5682S_PWR_ANLG_1, RT5682S_PWR_MB, 0);
-		if (!snd_soc_dapm_get_pin_status(&component->dapm, "Vref2"))
+		if (!rt5682s->wclk_enabled) {
 			snd_soc_component_update_bits(component,
-				RT5682S_PWR_ANLG_1, RT5682S_PWR_VREF2, 0);
+				RT5682S_PWR_ANLG_1, RT5682S_PWR_VREF2 | RT5682S_PWR_MB, 0);
+		}
 
 		snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_3,
 			RT5682S_PWR_CBJ, 0);
@@ -845,6 +844,7 @@ static void rt5682s_jack_detect_handler(struct work_struct *work)
 
 	snd_soc_dapm_mutex_lock(dapm);
 	mutex_lock(&rt5682s->calibrate_mutex);
+	mutex_lock(&rt5682s->wclk_mutex);
 
 	val = snd_soc_component_read(rt5682s->component, RT5682S_AJD1_CTRL)
 		& RT5682S_JDH_RS_MASK;
@@ -900,6 +900,7 @@ static void rt5682s_jack_detect_handler(struct work_struct *work)
 		rt5682s->irq_work_delay_time = 50;
 	}
 
+	mutex_unlock(&rt5682s->wclk_mutex);
 	mutex_unlock(&rt5682s->calibrate_mutex);
 	snd_soc_dapm_mutex_unlock(dapm);
 
@@ -1154,29 +1155,52 @@ static int set_dmic_clk(struct snd_soc_dapm_widget *w,
 	return 0;
 }
 
-static int set_filter_clk(struct snd_soc_dapm_widget *w,
+
+static int rt5682s_set_pllb_power(struct rt5682s_priv *rt5682s, int on)
+{
+	struct snd_soc_component *component = rt5682s->component;
+
+	if (on) {
+		snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_3,
+			RT5682S_PWR_LDO_PLLB | RT5682S_PWR_BIAS_PLLB | RT5682S_PWR_PLLB,
+			RT5682S_PWR_LDO_PLLB | RT5682S_PWR_BIAS_PLLB | RT5682S_PWR_PLLB);
+		snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_3,
+			RT5682S_RSTB_PLLB, RT5682S_RSTB_PLLB);
+	} else {
+		snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_3,
+			RT5682S_PWR_LDO_PLLB | RT5682S_PWR_BIAS_PLLB |
+			RT5682S_RSTB_PLLB | RT5682S_PWR_PLLB, 0);
+	}
+
+	return 0;
+}
+
+static int set_pllb_event(struct snd_soc_dapm_widget *w,
 		struct snd_kcontrol *kcontrol, int event)
 {
 	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
 	struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component);
-	int ref, val, reg, idx;
-	static const int div_f[] = {1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48};
-	static const int div_o[] = {1, 2, 4, 6, 8, 12, 16, 24, 32, 48};
+	int on = 0;
 
-	val = snd_soc_component_read(component, RT5682S_GPIO_CTRL_1)
-			& RT5682S_GP4_PIN_MASK;
+	if (rt5682s->wclk_enabled)
+		return 0;
 
-	if (w->shift == RT5682S_PWR_ADC_S1F_BIT && val == RT5682S_GP4_PIN_ADCDAT2)
-		ref = 256 * rt5682s->lrck[RT5682S_AIF2];
-	else
-		ref = 256 * rt5682s->lrck[RT5682S_AIF1];
+	if (SND_SOC_DAPM_EVENT_ON(event))
+		on = 1;
 
-	idx = rt5682s_div_sel(rt5682s, ref, div_f, ARRAY_SIZE(div_f));
+	rt5682s_set_pllb_power(rt5682s, on);
 
-	if (w->shift == RT5682S_PWR_ADC_S1F_BIT)
-		reg = RT5682S_PLL_TRACK_3;
-	else
-		reg = RT5682S_PLL_TRACK_2;
+	return 0;
+}
+
+static void rt5682s_set_filter_clk(struct rt5682s_priv *rt5682s, int reg, int ref)
+{
+	struct snd_soc_component *component = rt5682s->component;
+	int idx;
+	static const int div_f[] = {1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48};
+	static const int div_o[] = {1, 2, 4, 6, 8, 12, 16, 24, 32, 48};
+
+	idx = rt5682s_div_sel(rt5682s, ref, div_f, ARRAY_SIZE(div_f));
 
 	snd_soc_component_update_bits(component, reg,
 		RT5682S_FILTER_CLK_DIV_MASK, idx << RT5682S_FILTER_CLK_DIV_SFT);
@@ -1190,6 +1214,29 @@ static int set_filter_clk(struct snd_soc_dapm_widget *w,
 	snd_soc_component_update_bits(component, RT5682S_ADDA_CLK_1,
 		RT5682S_ADC_OSR_MASK | RT5682S_DAC_OSR_MASK,
 		(idx << RT5682S_ADC_OSR_SFT) | (idx << RT5682S_DAC_OSR_SFT));
+}
+
+static int set_filter_clk(struct snd_soc_dapm_widget *w,
+		struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
+	struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component);
+	int ref, reg, val;
+
+	val = snd_soc_component_read(component, RT5682S_GPIO_CTRL_1)
+			& RT5682S_GP4_PIN_MASK;
+
+	if (w->shift == RT5682S_PWR_ADC_S1F_BIT && val == RT5682S_GP4_PIN_ADCDAT2)
+		ref = 256 * rt5682s->lrck[RT5682S_AIF2];
+	else
+		ref = 256 * rt5682s->lrck[RT5682S_AIF1];
+
+	if (w->shift == RT5682S_PWR_ADC_S1F_BIT)
+		reg = RT5682S_PLL_TRACK_3;
+	else
+		reg = RT5682S_PLL_TRACK_2;
+
+	rt5682s_set_filter_clk(rt5682s, reg, ref);
 
 	return 0;
 }
@@ -1218,13 +1265,9 @@ static int set_dmic_power(struct snd_soc_dapm_widget *w,
 		break;
 
 	case SND_SOC_DAPM_POST_PMD:
-		if (!rt5682s->jack_type) {
-			if (!snd_soc_dapm_get_pin_status(w->dapm, "MICBIAS"))
-				snd_soc_component_update_bits(component,
-					RT5682S_PWR_ANLG_1, RT5682S_PWR_MB, 0);
-			if (!snd_soc_dapm_get_pin_status(w->dapm, "Vref2"))
-				snd_soc_component_update_bits(component,
-					RT5682S_PWR_ANLG_1, RT5682S_PWR_VREF2, 0);
+		if (!rt5682s->jack_type && !rt5682s->wclk_enabled) {
+			snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_1,
+				RT5682S_PWR_VREF2 | RT5682S_PWR_MB, 0);
 		}
 		break;
 	}
@@ -1232,41 +1275,58 @@ static int set_dmic_power(struct snd_soc_dapm_widget *w,
 	return 0;
 }
 
-static int set_i2s_clk(struct snd_soc_dapm_widget *w,
-		struct snd_kcontrol *kcontrol, int event)
+static void rt5682s_set_i2s(struct rt5682s_priv *rt5682s, int id, int on)
 {
-	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
-	struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component);
-	int pre_div, id;
-	unsigned int reg, mask, sft;
-
-	if (event != SND_SOC_DAPM_PRE_PMU)
-		return 0;
-
-	if (w->shift == RT5682S_PWR_I2S2_BIT) {
-		id = RT5682S_AIF2;
-		reg = RT5682S_I2S2_M_CLK_CTRL_1;
-		mask = RT5682S_I2S2_M_D_MASK;
-		sft = RT5682S_I2S2_M_D_SFT;
+	struct snd_soc_component *component = rt5682s->component;
+	int pre_div;
+	unsigned int p_reg, p_mask, p_sft;
+	unsigned int c_reg, c_mask, c_sft;
+
+	if (id == RT5682S_AIF1) {
+		c_reg = RT5682S_ADDA_CLK_1;
+		c_mask = RT5682S_I2S_M_D_MASK;
+		c_sft = RT5682S_I2S_M_D_SFT;
+		p_reg = RT5682S_PWR_DIG_1;
+		p_mask = RT5682S_PWR_I2S1;
+		p_sft = RT5682S_PWR_I2S1_BIT;
 	} else {
-		id = RT5682S_AIF1;
-		reg = RT5682S_ADDA_CLK_1;
-		mask = RT5682S_I2S_M_D_MASK;
-		sft = RT5682S_I2S_M_D_SFT;
+		c_reg = RT5682S_I2S2_M_CLK_CTRL_1;
+		c_mask = RT5682S_I2S2_M_D_MASK;
+		c_sft = RT5682S_I2S2_M_D_SFT;
+		p_reg = RT5682S_PWR_DIG_1;
+		p_mask = RT5682S_PWR_I2S2;
+		p_sft = RT5682S_PWR_I2S2_BIT;
 	}
 
-	if (!rt5682s->master[id])
-		return 0;
+	if (on && rt5682s->master[id]) {
+		pre_div = get_clk_info(rt5682s->sysclk, rt5682s->lrck[id]);
+		if (pre_div < 0) {
+			dev_err(component->dev, "get pre_div failed\n");
+			return;
+		}
 
-	pre_div = get_clk_info(rt5682s->sysclk, rt5682s->lrck[id]);
-	if (pre_div < 0) {
-		dev_err(component->dev, "get pre_div failed\n");
-		return -EINVAL;
+		dev_dbg(component->dev, "lrck is %dHz and pre_div is %d for iis %d master\n",
+			rt5682s->lrck[id], pre_div, id);
+		snd_soc_component_update_bits(component, c_reg, c_mask, pre_div << c_sft);
 	}
 
-	dev_dbg(component->dev, "lrck is %dHz and pre_div is %d for iis %d master\n",
-		rt5682s->lrck[id], pre_div, id);
-	snd_soc_component_update_bits(component, reg, mask, pre_div << sft);
+	snd_soc_component_update_bits(component, p_reg, p_mask, on << p_sft);
+}
+
+static int set_i2s_event(struct snd_soc_dapm_widget *w,
+		struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_component *component = snd_soc_dapm_to_component(w->dapm);
+	struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component);
+	int on = 0;
+
+	if (SND_SOC_DAPM_EVENT_ON(event))
+		on = 1;
+
+	if (!strcmp(w->name, "I2S1") && !rt5682s->wclk_enabled)
+		rt5682s_set_i2s(rt5682s, RT5682S_AIF1, on);
+	else if (!strcmp(w->name, "I2S2"))
+		rt5682s_set_i2s(rt5682s, RT5682S_AIF2, on);
 
 	return 0;
 }
@@ -1615,26 +1675,18 @@ static const struct snd_soc_dapm_widget rt5682s_dapm_widgets[] = {
 		RT5682S_PWR_LDO_MB2_BIT, 0, NULL, 0),
 	SND_SOC_DAPM_SUPPLY("LDO", RT5682S_PWR_ANLG_3,
 		RT5682S_PWR_LDO_BIT, 0, NULL, 0),
-	SND_SOC_DAPM_SUPPLY("Vref2", SND_SOC_NOPM, 0, 0, NULL, 0),
-	SND_SOC_DAPM_SUPPLY("MICBIAS", SND_SOC_NOPM, 0, 0, NULL, 0),
 
 	/* PLL Powers */
 	SND_SOC_DAPM_SUPPLY_S("PLLA_LDO", 0, RT5682S_PWR_ANLG_3,
 		RT5682S_PWR_LDO_PLLA_BIT, 0, NULL, 0),
-	SND_SOC_DAPM_SUPPLY_S("PLLB_LDO", 0, RT5682S_PWR_ANLG_3,
-		RT5682S_PWR_LDO_PLLB_BIT, 0, NULL, 0),
 	SND_SOC_DAPM_SUPPLY_S("PLLA_BIAS", 0, RT5682S_PWR_ANLG_3,
 		RT5682S_PWR_BIAS_PLLA_BIT, 0, NULL, 0),
-	SND_SOC_DAPM_SUPPLY_S("PLLB_BIAS", 0, RT5682S_PWR_ANLG_3,
-		RT5682S_PWR_BIAS_PLLB_BIT, 0, NULL, 0),
 	SND_SOC_DAPM_SUPPLY_S("PLLA", 0, RT5682S_PWR_ANLG_3,
 		RT5682S_PWR_PLLA_BIT, 0, NULL, 0),
-	SND_SOC_DAPM_SUPPLY_S("PLLB", 0, RT5682S_PWR_ANLG_3,
-		RT5682S_PWR_PLLB_BIT, 0, set_filter_clk, SND_SOC_DAPM_PRE_PMU),
 	SND_SOC_DAPM_SUPPLY_S("PLLA_RST", 1, RT5682S_PWR_ANLG_3,
 		RT5682S_RSTB_PLLA_BIT, 0, NULL, 0),
-	SND_SOC_DAPM_SUPPLY_S("PLLB_RST", 1, RT5682S_PWR_ANLG_3,
-		RT5682S_RSTB_PLLB_BIT, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("PLLB", SND_SOC_NOPM, 0, 0,
+		set_pllb_event, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 
 	/* ASRC */
 	SND_SOC_DAPM_SUPPLY_S("DAC STO1 ASRC", 1, RT5682S_PLL_TRACK_1,
@@ -1720,10 +1772,10 @@ static const struct snd_soc_dapm_widget rt5682s_dapm_widgets[] = {
 	SND_SOC_DAPM_PGA("Stereo1 ADC MIX", SND_SOC_NOPM, 0, 0, NULL, 0),
 
 	/* Digital Interface */
-	SND_SOC_DAPM_SUPPLY("I2S1", RT5682S_PWR_DIG_1, RT5682S_PWR_I2S1_BIT,
-		0, set_i2s_clk, SND_SOC_DAPM_PRE_PMU),
-	SND_SOC_DAPM_SUPPLY("I2S2", RT5682S_PWR_DIG_1, RT5682S_PWR_I2S2_BIT,
-		0, set_i2s_clk, SND_SOC_DAPM_PRE_PMU),
+	SND_SOC_DAPM_SUPPLY("I2S1", SND_SOC_NOPM, 0, 0,
+		set_i2s_event, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY("I2S2", SND_SOC_NOPM, 0, 0,
+		set_i2s_event, SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
 	SND_SOC_DAPM_PGA("IF1 DAC1 L", SND_SOC_NOPM, 0, 0, NULL, 0),
 	SND_SOC_DAPM_PGA("IF1 DAC1 R", SND_SOC_NOPM, 0, 0, NULL, 0),
 
@@ -1801,9 +1853,6 @@ static const struct snd_soc_dapm_route rt5682s_dapm_routes[] = {
 	{"PLLA", NULL, "PLLA_LDO"},
 	{"PLLA", NULL, "PLLA_BIAS"},
 	{"PLLA", NULL, "PLLA_RST"},
-	{"PLLB", NULL, "PLLB_LDO"},
-	{"PLLB", NULL, "PLLB_BIAS"},
-	{"PLLB", NULL, "PLLB_RST"},
 
 	/*ASRC*/
 	{"ADC Stereo1 Filter", NULL, "ADC STO1 ASRC", is_using_asrc},
@@ -2431,12 +2480,15 @@ static int rt5682s_set_bias_level(struct snd_soc_component *component,
 			RT5682S_PWR_LDO, RT5682S_PWR_LDO);
 		break;
 	case SND_SOC_BIAS_STANDBY:
-		regmap_update_bits(rt5682s->regmap, RT5682S_PWR_DIG_1,
-			RT5682S_DIG_GATE_CTRL, RT5682S_DIG_GATE_CTRL);
+		if (snd_soc_component_get_bias_level(component) == SND_SOC_BIAS_OFF)
+			regmap_update_bits(rt5682s->regmap, RT5682S_PWR_DIG_1,
+				RT5682S_DIG_GATE_CTRL, RT5682S_DIG_GATE_CTRL);
 		break;
 	case SND_SOC_BIAS_OFF:
-		regmap_update_bits(rt5682s->regmap, RT5682S_PWR_DIG_1,
-			RT5682S_DIG_GATE_CTRL | RT5682S_PWR_LDO, 0);
+		regmap_update_bits(rt5682s->regmap, RT5682S_PWR_DIG_1, RT5682S_PWR_LDO, 0);
+		if (!rt5682s->wclk_enabled)
+			regmap_update_bits(rt5682s->regmap, RT5682S_PWR_DIG_1,
+				RT5682S_DIG_GATE_CTRL, 0);
 		break;
 	case SND_SOC_BIAS_ON:
 		break;
@@ -2464,30 +2516,34 @@ static int rt5682s_wclk_prepare(struct clk_hw *hw)
 	struct rt5682s_priv *rt5682s =
 		container_of(hw, struct rt5682s_priv, dai_clks_hw[RT5682S_DAI_WCLK_IDX]);
 	struct snd_soc_component *component = rt5682s->component;
-	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component);
+	int ref, reg;
 
 	if (!rt5682s_clk_check(rt5682s))
 		return -EINVAL;
 
-	snd_soc_dapm_mutex_lock(dapm);
-
-	snd_soc_dapm_force_enable_pin_unlocked(dapm, "MICBIAS");
-	snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_1,
-		RT5682S_PWR_MB, RT5682S_PWR_MB);
+	mutex_lock(&rt5682s->wclk_mutex);
 
-	snd_soc_dapm_force_enable_pin_unlocked(dapm, "Vref2");
 	snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_1,
-		RT5682S_PWR_VREF2 | RT5682S_PWR_FV2, RT5682S_PWR_VREF2);
+		RT5682S_PWR_VREF2 | RT5682S_PWR_FV2 | RT5682S_PWR_MB,
+		RT5682S_PWR_VREF2 | RT5682S_PWR_MB);
 	usleep_range(15000, 20000);
 	snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_1,
 		RT5682S_PWR_FV2, RT5682S_PWR_FV2);
 
-	snd_soc_dapm_force_enable_pin_unlocked(dapm, "I2S1");
-	/* Only need to power PLLB due to the rate set restriction */
-	snd_soc_dapm_force_enable_pin_unlocked(dapm, "PLLB");
-	snd_soc_dapm_sync_unlocked(dapm);
+	/* Set and power on I2S1 */
+	snd_soc_component_update_bits(component, RT5682S_PWR_DIG_1,
+		RT5682S_DIG_GATE_CTRL, RT5682S_DIG_GATE_CTRL);
+	rt5682s_set_i2s(rt5682s, RT5682S_AIF1, 1);
 
-	snd_soc_dapm_mutex_unlock(dapm);
+	/* Only need to power on PLLB due to the rate set restriction */
+	reg = RT5682S_PLL_TRACK_2;
+	ref = 256 * rt5682s->lrck[RT5682S_AIF1];
+	rt5682s_set_filter_clk(rt5682s, reg, ref);
+	rt5682s_set_pllb_power(rt5682s, 1);
+
+	rt5682s->wclk_enabled = 1;
+
+	mutex_unlock(&rt5682s->wclk_mutex);
 
 	return 0;
 }
@@ -2497,24 +2553,27 @@ static void rt5682s_wclk_unprepare(struct clk_hw *hw)
 	struct rt5682s_priv *rt5682s =
 		container_of(hw, struct rt5682s_priv, dai_clks_hw[RT5682S_DAI_WCLK_IDX]);
 	struct snd_soc_component *component = rt5682s->component;
-	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(component);
 
 	if (!rt5682s_clk_check(rt5682s))
 		return;
 
-	snd_soc_dapm_mutex_lock(dapm);
+	mutex_lock(&rt5682s->wclk_mutex);
 
-	snd_soc_dapm_disable_pin_unlocked(dapm, "MICBIAS");
-	snd_soc_dapm_disable_pin_unlocked(dapm, "Vref2");
 	if (!rt5682s->jack_type)
 		snd_soc_component_update_bits(component, RT5682S_PWR_ANLG_1,
 			RT5682S_PWR_VREF2 | RT5682S_PWR_FV2 | RT5682S_PWR_MB, 0);
 
-	snd_soc_dapm_disable_pin_unlocked(dapm, "I2S1");
-	snd_soc_dapm_disable_pin_unlocked(dapm, "PLLB");
-	snd_soc_dapm_sync_unlocked(dapm);
+	/* Power down I2S1 */
+	rt5682s_set_i2s(rt5682s, RT5682S_AIF1, 0);
+	snd_soc_component_update_bits(component, RT5682S_PWR_DIG_1,
+		RT5682S_DIG_GATE_CTRL, 0);
 
-	snd_soc_dapm_mutex_unlock(dapm);
+	/* Power down PLLB */
+	rt5682s_set_pllb_power(rt5682s, 0);
+
+	rt5682s->wclk_enabled = 0;
+
+	mutex_unlock(&rt5682s->wclk_mutex);
 }
 
 static unsigned long rt5682s_wclk_recalc_rate(struct clk_hw *hw,
@@ -2805,19 +2864,10 @@ static inline int rt5682s_dai_probe_clks(struct snd_soc_component *component)
 static int rt5682s_probe(struct snd_soc_component *component)
 {
 	struct rt5682s_priv *rt5682s = snd_soc_component_get_drvdata(component);
-	struct snd_soc_dapm_context *dapm = &component->dapm;
-	int ret;
 
 	rt5682s->component = component;
 
-	ret = rt5682s_dai_probe_clks(component);
-	if (ret)
-		return ret;
-
-	snd_soc_dapm_disable_pin(dapm, "MICBIAS");
-	snd_soc_dapm_disable_pin(dapm, "Vref2");
-	snd_soc_dapm_sync(dapm);
-	return 0;
+	return rt5682s_dai_probe_clks(component);
 }
 
 static void rt5682s_remove(struct snd_soc_component *component)
@@ -3113,6 +3163,7 @@ static int rt5682s_i2c_probe(struct i2c_client *i2c)
 
 	mutex_init(&rt5682s->calibrate_mutex);
 	mutex_init(&rt5682s->sar_mutex);
+	mutex_init(&rt5682s->wclk_mutex);
 	rt5682s_calibrate(rt5682s);
 
 	regmap_update_bits(rt5682s->regmap, RT5682S_MICBIAS_2,
@@ -3194,11 +3245,9 @@ static void rt5682s_i2c_shutdown(struct i2c_client *client)
 	rt5682s_reset(rt5682s);
 }
 
-static int rt5682s_i2c_remove(struct i2c_client *client)
+static void rt5682s_i2c_remove(struct i2c_client *client)
 {
 	rt5682s_i2c_shutdown(client);
-
-	return 0;
 }
 
 static const struct of_device_id rt5682s_of_match[] = {
diff --git a/sound/soc/codecs/rt5682s.h b/sound/soc/codecs/rt5682s.h
index 7353831c73dd..824dc6543c18 100644
--- a/sound/soc/codecs/rt5682s.h
+++ b/sound/soc/codecs/rt5682s.h
@@ -1450,6 +1450,7 @@ struct rt5682s_priv {
 	struct delayed_work jd_check_work;
 	struct mutex calibrate_mutex;
 	struct mutex sar_mutex;
+	struct mutex wclk_mutex;
 
 #ifdef CONFIG_COMMON_CLK
 	struct clk_hw dai_clks_hw[RT5682S_DAI_NUM_CLKS];
@@ -1469,6 +1470,7 @@ struct rt5682s_priv {
 
 	int jack_type;
 	int irq_work_delay_time;
+	int wclk_enabled;
 };
 
 int rt5682s_sel_asrc_clk_src(struct snd_soc_component *component,
diff --git a/sound/soc/codecs/rt700-sdw.c b/sound/soc/codecs/rt700-sdw.c
index f7439e40ca8b..96fc5f36d0d0 100644
--- a/sound/soc/codecs/rt700-sdw.c
+++ b/sound/soc/codecs/rt700-sdw.c
@@ -542,6 +542,8 @@ static int __maybe_unused rt700_dev_resume(struct device *dev)
 				msecs_to_jiffies(RT700_PROBE_TIMEOUT));
 	if (!time) {
 		dev_err(&slave->dev, "Initialization not complete, timed out\n");
+		sdw_show_ping_status(slave->bus, true);
+
 		return -ETIMEDOUT;
 	}
 
diff --git a/sound/soc/codecs/rt711-sdca-sdw.c b/sound/soc/codecs/rt711-sdca-sdw.c
index a085b2f530aa..4120842fe699 100644
--- a/sound/soc/codecs/rt711-sdca-sdw.c
+++ b/sound/soc/codecs/rt711-sdca-sdw.c
@@ -449,6 +449,8 @@ static int __maybe_unused rt711_sdca_dev_resume(struct device *dev)
 				msecs_to_jiffies(RT711_PROBE_TIMEOUT));
 	if (!time) {
 		dev_err(&slave->dev, "Initialization not complete, timed out\n");
+		sdw_show_ping_status(slave->bus, true);
+
 		return -ETIMEDOUT;
 	}
 
diff --git a/sound/soc/codecs/rt715-sdca-sdw.c b/sound/soc/codecs/rt715-sdca-sdw.c
index 13e731d16675..3f981a9e7fb6 100644
--- a/sound/soc/codecs/rt715-sdca-sdw.c
+++ b/sound/soc/codecs/rt715-sdca-sdw.c
@@ -244,6 +244,8 @@ static int __maybe_unused rt715_dev_resume(struct device *dev)
 					   msecs_to_jiffies(RT715_PROBE_TIMEOUT));
 	if (!time) {
 		dev_err(&slave->dev, "Enumeration not complete, timed out\n");
+		sdw_show_ping_status(slave->bus, true);
+
 		return -ETIMEDOUT;
 	}
 
diff --git a/sound/soc/codecs/rt715-sdw.c b/sound/soc/codecs/rt715-sdw.c
index b047bf87a100..4e61e16470ed 100644
--- a/sound/soc/codecs/rt715-sdw.c
+++ b/sound/soc/codecs/rt715-sdw.c
@@ -562,6 +562,8 @@ static int __maybe_unused rt715_dev_resume(struct device *dev)
 					   msecs_to_jiffies(RT715_PROBE_TIMEOUT));
 	if (!time) {
 		dev_err(&slave->dev, "Initialization not complete, timed out\n");
+		sdw_show_ping_status(slave->bus, true);
+
 		return -ETIMEDOUT;
 	}
 
diff --git a/sound/soc/codecs/rt9120.c b/sound/soc/codecs/rt9120.c
index da495bdc8415..644300e88b4c 100644
--- a/sound/soc/codecs/rt9120.c
+++ b/sound/soc/codecs/rt9120.c
@@ -572,11 +572,10 @@ static int rt9120_probe(struct i2c_client *i2c)
 					       &rt9120_dai, 1);
 }
 
-static int rt9120_remove(struct i2c_client *i2c)
+static void rt9120_remove(struct i2c_client *i2c)
 {
 	pm_runtime_disable(&i2c->dev);
 	pm_runtime_set_suspended(&i2c->dev);
-	return 0;
 }
 
 static int __maybe_unused rt9120_runtime_suspend(struct device *dev)
diff --git a/sound/soc/codecs/sgtl5000.c b/sound/soc/codecs/sgtl5000.c
index 3fafd9fc5cfd..4b2135eba74d 100644
--- a/sound/soc/codecs/sgtl5000.c
+++ b/sound/soc/codecs/sgtl5000.c
@@ -1790,7 +1790,7 @@ disable_regs:
 	return ret;
 }
 
-static int sgtl5000_i2c_remove(struct i2c_client *client)
+static void sgtl5000_i2c_remove(struct i2c_client *client)
 {
 	struct sgtl5000_priv *sgtl5000 = i2c_get_clientdata(client);
 
@@ -1800,8 +1800,6 @@ static int sgtl5000_i2c_remove(struct i2c_client *client)
 	clk_disable_unprepare(sgtl5000->mclk);
 	regulator_bulk_disable(sgtl5000->num_supplies, sgtl5000->supplies);
 	regulator_bulk_free(sgtl5000->num_supplies, sgtl5000->supplies);
-
-	return 0;
 }
 
 static void sgtl5000_i2c_shutdown(struct i2c_client *client)
diff --git a/sound/soc/codecs/sigmadsp.c b/sound/soc/codecs/sigmadsp.c
index b992216aee55..3047a6fbb380 100644
--- a/sound/soc/codecs/sigmadsp.c
+++ b/sound/soc/codecs/sigmadsp.c
@@ -227,13 +227,11 @@ static int sigma_fw_load_control(struct sigmadsp *sigmadsp,
 	if (!ctrl)
 		return -ENOMEM;
 
-	name = kzalloc(name_len + 1, GFP_KERNEL);
+	name = kmemdup_nul(ctrl_chunk->name, name_len, GFP_KERNEL);
 	if (!name) {
 		ret = -ENOMEM;
 		goto err_free_ctrl;
 	}
-	memcpy(name, ctrl_chunk->name, name_len);
-	name[name_len] = '\0';
 	ctrl->name = name;
 
 	/*
diff --git a/sound/soc/codecs/src4xxx-i2c.c b/sound/soc/codecs/src4xxx-i2c.c
new file mode 100644
index 000000000000..43daa9dc8ab5
--- /dev/null
+++ b/sound/soc/codecs/src4xxx-i2c.c
@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// Driver for SRC4XXX codecs
+//
+// Copyright 2021-2022 Deqx Pty Ltd
+// Author: Matt Flax <flatmax@flatmax.com>
+
+#include <linux/i2c.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+
+#include "src4xxx.h"
+
+static int src4xxx_i2c_probe(struct i2c_client *i2c,
+			const struct i2c_device_id *id)
+{
+	return src4xxx_probe(&i2c->dev,
+		devm_regmap_init_i2c(i2c, &src4xxx_regmap_config), NULL);
+}
+
+static const struct i2c_device_id src4xxx_i2c_ids[] = {
+	{ "src4392", 0 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, src4xxx_i2c_ids);
+
+static const struct of_device_id src4xxx_of_match[] = {
+	{ .compatible = "ti,src4392", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, src4xxx_of_match);
+
+
+static struct i2c_driver src4xxx_i2c_driver = {
+	.driver = {
+		.name = "src4xxx",
+		.of_match_table = of_match_ptr(src4xxx_of_match),
+	},
+	.probe = src4xxx_i2c_probe,
+	.id_table = src4xxx_i2c_ids,
+};
+module_i2c_driver(src4xxx_i2c_driver);
+
+MODULE_DESCRIPTION("ASoC SRC4392 CODEC I2C driver");
+MODULE_AUTHOR("Matt Flax <flatmax@flatmax.com>");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/src4xxx.c b/sound/soc/codecs/src4xxx.c
new file mode 100644
index 000000000000..db4e280dd055
--- /dev/null
+++ b/sound/soc/codecs/src4xxx.c
@@ -0,0 +1,518 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// TI SRC4xxx Audio Codec driver
+//
+// Copyright 2021-2022 Deqx Pty Ltd
+// Author: Matt Flax <flatmax@flatmax.com>
+
+#include <linux/module.h>
+
+#include <sound/soc.h>
+#include <sound/tlv.h>
+
+#include "src4xxx.h"
+
+struct src4xxx {
+	struct regmap *regmap;
+	bool master[2];
+	int mclk_hz;
+	struct device *dev;
+};
+
+enum {SRC4XXX_PORTA, SRC4XXX_PORTB};
+
+/* SRC attenuation */
+static const DECLARE_TLV_DB_SCALE(src_tlv, -12750, 50, 0);
+
+static const struct snd_kcontrol_new src4xxx_controls[] = {
+	SOC_DOUBLE_R_TLV("SRC Volume",
+		SRC4XXX_SCR_CTL_30, SRC4XXX_SCR_CTL_31, 0, 255, 1, src_tlv),
+};
+
+/* I2S port control */
+static const char * const port_out_src_text[] = {
+	"loopback", "other_port", "DIR", "SRC"
+};
+static SOC_ENUM_SINGLE_DECL(porta_out_src_enum, SRC4XXX_PORTA_CTL_03, 4,
+	port_out_src_text);
+static SOC_ENUM_SINGLE_DECL(portb_out_src_enum, SRC4XXX_PORTB_CTL_05, 4,
+	port_out_src_text);
+static const struct snd_kcontrol_new porta_out_control =
+	SOC_DAPM_ENUM("Port A source select", porta_out_src_enum);
+static const struct snd_kcontrol_new portb_out_control =
+	SOC_DAPM_ENUM("Port B source select", portb_out_src_enum);
+
+/* Digital audio transmitter control */
+static const char * const dit_mux_text[] = {"Port A", "Port B", "DIR", "SRC"};
+static SOC_ENUM_SINGLE_DECL(dit_mux_enum, SRC4XXX_TX_CTL_07, 3, dit_mux_text);
+static const struct snd_kcontrol_new dit_mux_control =
+	SOC_DAPM_ENUM("DIT source", dit_mux_enum);
+
+/* SRC control */
+static const char * const src_in_text[] = {"Port A", "Port B", "DIR"};
+static SOC_ENUM_SINGLE_DECL(src_in_enum, SRC4XXX_SCR_CTL_2D, 0, src_in_text);
+static const struct snd_kcontrol_new src_in_control =
+	SOC_DAPM_ENUM("SRC source select", src_in_enum);
+
+/* DIR control */
+static const char * const dir_in_text[] = {"Ch 1", "Ch 2", "Ch 3", "Ch 4"};
+static SOC_ENUM_SINGLE_DECL(dir_in_enum, SRC4XXX_RCV_CTL_0D, 0, dir_in_text);
+static const struct snd_kcontrol_new dir_in_control =
+	SOC_DAPM_ENUM("Digital Input", dir_in_enum);
+
+static const struct snd_soc_dapm_widget src4xxx_dapm_widgets[] = {
+	SND_SOC_DAPM_INPUT("loopback_A"),
+	SND_SOC_DAPM_INPUT("other_port_A"),
+	SND_SOC_DAPM_INPUT("DIR_A"),
+	SND_SOC_DAPM_INPUT("SRC_A"),
+	SND_SOC_DAPM_MUX("Port A source",
+		SND_SOC_NOPM, 0, 0, &porta_out_control),
+
+	SND_SOC_DAPM_INPUT("loopback_B"),
+	SND_SOC_DAPM_INPUT("other_port_B"),
+	SND_SOC_DAPM_INPUT("DIR_B"),
+	SND_SOC_DAPM_INPUT("SRC_B"),
+	SND_SOC_DAPM_MUX("Port B source",
+		SND_SOC_NOPM, 0, 0, &portb_out_control),
+
+	SND_SOC_DAPM_INPUT("Port_A"),
+	SND_SOC_DAPM_INPUT("Port_B"),
+	SND_SOC_DAPM_INPUT("DIR_"),
+
+	/* Digital audio receivers and transmitters */
+	SND_SOC_DAPM_OUTPUT("DIR_OUT"),
+	SND_SOC_DAPM_OUTPUT("SRC_OUT"),
+	SND_SOC_DAPM_MUX("DIT Out Src", SRC4XXX_PWR_RST_01,
+		SRC4XXX_ENABLE_DIT_SHIFT, 1, &dit_mux_control),
+
+	/* Audio Interface */
+	SND_SOC_DAPM_AIF_IN("AIF_A_RX", "Playback A", 0,
+		SRC4XXX_PWR_RST_01, SRC4XXX_ENABLE_PORT_A_SHIFT, 1),
+	SND_SOC_DAPM_AIF_OUT("AIF_A_TX", "Capture A", 0,
+		SRC4XXX_PWR_RST_01, SRC4XXX_ENABLE_PORT_A_SHIFT, 1),
+	SND_SOC_DAPM_AIF_IN("AIF_B_RX", "Playback B", 0,
+		SRC4XXX_PWR_RST_01, SRC4XXX_ENABLE_PORT_B_SHIFT, 1),
+	SND_SOC_DAPM_AIF_OUT("AIF_B_TX", "Capture B", 0,
+		SRC4XXX_PWR_RST_01, SRC4XXX_ENABLE_PORT_B_SHIFT, 1),
+
+	SND_SOC_DAPM_MUX("SRC source", SND_SOC_NOPM, 0, 0, &src_in_control),
+
+	SND_SOC_DAPM_INPUT("MCLK"),
+	SND_SOC_DAPM_INPUT("RXMCLKI"),
+	SND_SOC_DAPM_INPUT("RXMCLKO"),
+
+	SND_SOC_DAPM_INPUT("RX1"),
+	SND_SOC_DAPM_INPUT("RX2"),
+	SND_SOC_DAPM_INPUT("RX3"),
+	SND_SOC_DAPM_INPUT("RX4"),
+	SND_SOC_DAPM_MUX("Digital Input", SRC4XXX_PWR_RST_01,
+		SRC4XXX_ENABLE_DIR_SHIFT, 1, &dir_in_control),
+};
+
+static const struct snd_soc_dapm_route src4xxx_audio_routes[] = {
+	/* I2S Input to Output Routing */
+	{"Port A source", "loopback", "loopback_A"},
+	{"Port A source", "other_port", "other_port_A"},
+	{"Port A source", "DIR", "DIR_A"},
+	{"Port A source", "SRC", "SRC_A"},
+	{"Port B source", "loopback", "loopback_B"},
+	{"Port B source", "other_port", "other_port_B"},
+	{"Port B source", "DIR", "DIR_B"},
+	{"Port B source", "SRC", "SRC_B"},
+	/* DIT muxing */
+	{"DIT Out Src", "Port A", "Capture A"},
+	{"DIT Out Src", "Port B", "Capture B"},
+	{"DIT Out Src", "DIR", "DIR_OUT"},
+	{"DIT Out Src", "SRC", "SRC_OUT"},
+
+	/* SRC input selection */
+	{"SRC source", "Port A", "Port_A"},
+	{"SRC source", "Port B", "Port_B"},
+	{"SRC source", "DIR", "DIR_"},
+	/* SRC mclk selection */
+	{"SRC mclk source", "Master (MCLK)", "MCLK"},
+	{"SRC mclk source", "Master (RXCLKI)", "RXMCLKI"},
+	{"SRC mclk source", "Recovered receiver clk", "RXMCLKO"},
+	/* DIR input selection */
+	{"Digital Input", "Ch 1", "RX1"},
+	{"Digital Input", "Ch 2", "RX2"},
+	{"Digital Input", "Ch 3", "RX3"},
+	{"Digital Input", "Ch 4", "RX4"},
+};
+
+
+static const struct snd_soc_component_driver src4xxx_driver = {
+	.controls = src4xxx_controls,
+	.num_controls = ARRAY_SIZE(src4xxx_controls),
+
+	.dapm_widgets = src4xxx_dapm_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(src4xxx_dapm_widgets),
+	.dapm_routes = src4xxx_audio_routes,
+	.num_dapm_routes = ARRAY_SIZE(src4xxx_audio_routes),
+};
+
+static int src4xxx_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+{
+	struct snd_soc_component *component = dai->component;
+	struct src4xxx *src4xxx = snd_soc_component_get_drvdata(component);
+	unsigned int ctrl;
+
+	switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) {
+	case SND_SOC_DAIFMT_CBM_CFM:
+		ctrl = SRC4XXX_BUS_MASTER;
+		src4xxx->master[dai->id] = true;
+		break;
+	case SND_SOC_DAIFMT_CBS_CFS:
+		ctrl = 0;
+		src4xxx->master[dai->id] = false;
+		break;
+	default:
+		return -EINVAL;
+		break;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		ctrl |= SRC4XXX_BUS_I2S;
+		break;
+	case SND_SOC_DAIFMT_LEFT_J:
+		ctrl |= SRC4XXX_BUS_LEFT_J;
+		break;
+	case SND_SOC_DAIFMT_RIGHT_J:
+		ctrl |= SRC4XXX_BUS_RIGHT_J_24;
+		break;
+	default:
+		return -EINVAL;
+		break;
+	}
+
+	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
+	case SND_SOC_DAIFMT_NB_NF:
+		break;
+	default:
+		return -EINVAL;
+		break;
+	}
+
+	regmap_update_bits(src4xxx->regmap, SRC4XXX_BUS_FMT(dai->id),
+		SRC4XXX_BUS_FMT_MS_MASK, ctrl);
+
+	return 0;
+}
+
+static int src4xxx_set_mclk_hz(struct snd_soc_dai *codec_dai,
+		int clk_id, unsigned int freq, int dir)
+{
+	struct snd_soc_component *component = codec_dai->component;
+	struct src4xxx *src4xxx = snd_soc_component_get_drvdata(component);
+
+	dev_info(component->dev, "changing mclk rate from %d to %d Hz\n",
+		src4xxx->mclk_hz, freq);
+	src4xxx->mclk_hz = freq;
+
+	return 0;
+}
+
+static int src4xxx_hw_params(struct snd_pcm_substream *substream,
+			struct snd_pcm_hw_params *params,
+			struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct src4xxx *src4xxx = snd_soc_component_get_drvdata(component);
+	unsigned int mclk_div;
+	int val, pj, jd, d;
+	int reg;
+	int ret;
+
+	switch (dai->id) {
+	case SRC4XXX_PORTB:
+		reg = SRC4XXX_PORTB_CTL_06;
+		break;
+	default:
+		reg = SRC4XXX_PORTA_CTL_04;
+		break;
+	}
+
+	if (src4xxx->master[dai->id]) {
+		mclk_div = src4xxx->mclk_hz/params_rate(params);
+		if (src4xxx->mclk_hz != mclk_div*params_rate(params)) {
+			dev_err(component->dev,
+				"mclk %d / rate %d has a remainder.\n",
+				src4xxx->mclk_hz, params_rate(params));
+			return -EINVAL;
+		}
+
+		val = ((int)mclk_div - 128) / 128;
+		if ((val < 0) | (val > 3)) {
+			dev_err(component->dev,
+				"div register setting %d is out of range\n",
+				val);
+			dev_err(component->dev,
+				"unsupported sample rate %d Hz for the master clock of %d Hz\n",
+				params_rate(params), src4xxx->mclk_hz);
+			return -EINVAL;
+		}
+
+		/* set the TX DIV */
+		ret = regmap_update_bits(src4xxx->regmap,
+			SRC4XXX_TX_CTL_07, SRC4XXX_TX_MCLK_DIV_MASK,
+			val<<SRC4XXX_TX_MCLK_DIV_SHIFT);
+		if (ret) {
+			dev_err(component->dev,
+				"Couldn't set the TX's div register to %d << %d = 0x%x\n",
+				val, SRC4XXX_TX_MCLK_DIV_SHIFT,
+				val<<SRC4XXX_TX_MCLK_DIV_SHIFT);
+			return ret;
+		}
+
+		/* set the PLL for the digital receiver */
+		switch (src4xxx->mclk_hz) {
+		case 24576000:
+			pj = 0x22;
+			jd = 0x00;
+			d = 0x00;
+			break;
+		case 22579200:
+			pj = 0x22;
+			jd = 0x1b;
+			d = 0xa3;
+			break;
+		default:
+			/* don't error out here,
+			 * other parts of the chip are still functional
+			 * Dummy initialize variables to avoid
+			 * -Wsometimes-uninitialized from clang.
+			 */
+			dev_info(component->dev,
+				"Couldn't set the RCV PLL as this master clock rate is unknown. Chosen regmap values may not match real world values.\n");
+			pj = 0x0;
+			jd = 0xff;
+			d = 0xff;
+			break;
+		}
+		ret = regmap_write(src4xxx->regmap, SRC4XXX_RCV_PLL_0F, pj);
+		if (ret < 0)
+			dev_err(component->dev,
+				"Failed to update PLL register 0x%x\n",
+				SRC4XXX_RCV_PLL_0F);
+		ret = regmap_write(src4xxx->regmap, SRC4XXX_RCV_PLL_10, jd);
+		if (ret < 0)
+			dev_err(component->dev,
+				"Failed to update PLL register 0x%x\n",
+				SRC4XXX_RCV_PLL_10);
+		ret = regmap_write(src4xxx->regmap, SRC4XXX_RCV_PLL_11, d);
+		if (ret < 0)
+			dev_err(component->dev,
+				"Failed to update PLL register 0x%x\n",
+				SRC4XXX_RCV_PLL_11);
+
+		ret = regmap_update_bits(src4xxx->regmap,
+			SRC4XXX_TX_CTL_07, SRC4XXX_TX_MCLK_DIV_MASK,
+			val<<SRC4XXX_TX_MCLK_DIV_SHIFT);
+		if (ret < 0) {
+			dev_err(component->dev,
+				"Couldn't set the TX's div register to %d << %d = 0x%x\n",
+				val, SRC4XXX_TX_MCLK_DIV_SHIFT,
+				val<<SRC4XXX_TX_MCLK_DIV_SHIFT);
+			return ret;
+		}
+
+		return regmap_update_bits(src4xxx->regmap, reg,
+					SRC4XXX_MCLK_DIV_MASK, val);
+	} else {
+		dev_info(dai->dev, "not setting up MCLK as not master\n");
+	}
+
+	return 0;
+};
+
+static const struct snd_soc_dai_ops src4xxx_dai_ops = {
+	.hw_params	= src4xxx_hw_params,
+	.set_sysclk	= src4xxx_set_mclk_hz,
+	.set_fmt	= src4xxx_set_dai_fmt,
+};
+
+#define SRC4XXX_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |	SNDRV_PCM_FMTBIT_S32_LE)
+#define SRC4XXX_RATES (SNDRV_PCM_RATE_44100|SNDRV_PCM_RATE_48000|\
+				SNDRV_PCM_RATE_88200|\
+				SNDRV_PCM_RATE_96000|\
+				SNDRV_PCM_RATE_176400|\
+				SNDRV_PCM_RATE_192000)
+
+static struct snd_soc_dai_driver src4xxx_dai_driver[] = {
+	{
+		.id = SRC4XXX_PORTA,
+		.name = "src4xxx-portA",
+		.playback = {
+			.stream_name = "Playback A",
+			.channels_min = 2,
+			.channels_max = 2,
+			.rates = SRC4XXX_RATES,
+			.formats = SRC4XXX_FORMATS,
+		},
+		.capture = {
+			.stream_name = "Capture A",
+			.channels_min = 2,
+			.channels_max = 2,
+			.rates = SRC4XXX_RATES,
+			.formats = SRC4XXX_FORMATS,
+		},
+		.ops = &src4xxx_dai_ops,
+	},
+	{
+		.id = SRC4XXX_PORTB,
+		.name = "src4xxx-portB",
+		.playback = {
+			.stream_name = "Playback B",
+			.channels_min = 2,
+			.channels_max = 2,
+			.rates = SRC4XXX_RATES,
+			.formats = SRC4XXX_FORMATS,
+		},
+		.capture = {
+			.stream_name = "Capture B",
+			.channels_min = 2,
+			.channels_max = 2,
+			.rates = SRC4XXX_RATES,
+			.formats = SRC4XXX_FORMATS,
+		},
+		.ops = &src4xxx_dai_ops,
+	},
+};
+
+static const struct reg_default src4xxx_reg_defaults[] = {
+	{ SRC4XXX_PWR_RST_01,		0x00 }, /* all powered down intially */
+	{ SRC4XXX_PORTA_CTL_03,		0x00 },
+	{ SRC4XXX_PORTA_CTL_04,		0x00 },
+	{ SRC4XXX_PORTB_CTL_05,		0x00 },
+	{ SRC4XXX_PORTB_CTL_06,		0x00 },
+	{ SRC4XXX_TX_CTL_07,		0x00 },
+	{ SRC4XXX_TX_CTL_08,		0x00 },
+	{ SRC4XXX_TX_CTL_09,		0x00 },
+	{ SRC4XXX_SRC_DIT_IRQ_MSK_0B,	0x00 },
+	{ SRC4XXX_SRC_DIT_IRQ_MODE_0C,	0x00 },
+	{ SRC4XXX_RCV_CTL_0D,		0x00 },
+	{ SRC4XXX_RCV_CTL_0E,		0x00 },
+	{ SRC4XXX_RCV_PLL_0F,		0x00 }, /* not spec. in the datasheet */
+	{ SRC4XXX_RCV_PLL_10,		0xff }, /* not spec. in the datasheet */
+	{ SRC4XXX_RCV_PLL_11,		0xff }, /* not spec. in the datasheet */
+	{ SRC4XXX_RVC_IRQ_MSK_16,	0x00 },
+	{ SRC4XXX_RVC_IRQ_MSK_17,	0x00 },
+	{ SRC4XXX_RVC_IRQ_MODE_18,	0x00 },
+	{ SRC4XXX_RVC_IRQ_MODE_19,	0x00 },
+	{ SRC4XXX_RVC_IRQ_MODE_1A,	0x00 },
+	{ SRC4XXX_GPIO_1_1B,		0x00 },
+	{ SRC4XXX_GPIO_2_1C,		0x00 },
+	{ SRC4XXX_GPIO_3_1D,		0x00 },
+	{ SRC4XXX_GPIO_4_1E,		0x00 },
+	{ SRC4XXX_SCR_CTL_2D,		0x00 },
+	{ SRC4XXX_SCR_CTL_2E,		0x00 },
+	{ SRC4XXX_SCR_CTL_2F,		0x00 },
+	{ SRC4XXX_SCR_CTL_30,		0x00 },
+	{ SRC4XXX_SCR_CTL_31,		0x00 },
+};
+
+int src4xxx_probe(struct device *dev, struct regmap *regmap,
+			void (*switch_mode)(struct device *dev))
+{
+	struct src4xxx *src4xxx;
+	int ret;
+
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	src4xxx = devm_kzalloc(dev, sizeof(*src4xxx), GFP_KERNEL);
+	if (!src4xxx)
+		return -ENOMEM;
+
+	src4xxx->regmap = regmap;
+	src4xxx->dev = dev;
+	src4xxx->mclk_hz = 0; /* mclk has not been configured yet */
+	dev_set_drvdata(dev, src4xxx);
+
+	ret = regmap_write(regmap, SRC4XXX_PWR_RST_01, SRC4XXX_RESET);
+	if (ret < 0)
+		dev_err(dev, "Failed to issue reset: %d\n", ret);
+	usleep_range(1, 500); /* sleep for more then 500 ns */
+	ret = regmap_write(regmap, SRC4XXX_PWR_RST_01, SRC4XXX_POWER_DOWN);
+	if (ret < 0)
+		dev_err(dev, "Failed to decommission reset: %d\n", ret);
+	usleep_range(500, 1000); /* sleep for 500 us or more */
+
+	ret = regmap_update_bits(src4xxx->regmap, SRC4XXX_PWR_RST_01,
+		SRC4XXX_POWER_ENABLE, SRC4XXX_POWER_ENABLE);
+	if (ret < 0)
+		dev_err(dev, "Failed to port A and B : %d\n", ret);
+
+	/* set receiver to use master clock (rcv mclk is most likely jittery) */
+	ret = regmap_update_bits(src4xxx->regmap, SRC4XXX_RCV_CTL_0D,
+		SRC4XXX_RXCLK_MCLK,	SRC4XXX_RXCLK_MCLK);
+	if (ret < 0)
+		dev_err(dev,
+			"Failed to enable mclk as the PLL1 DIR reference : %d\n", ret);
+
+	/* default to leaving the PLL2 running on loss of lock, divide by 8 */
+	ret = regmap_update_bits(src4xxx->regmap, SRC4XXX_RCV_CTL_0E,
+		SRC4XXX_PLL2_DIV_8 | SRC4XXX_REC_MCLK_EN | SRC4XXX_PLL2_LOL,
+		SRC4XXX_PLL2_DIV_8 | SRC4XXX_REC_MCLK_EN | SRC4XXX_PLL2_LOL);
+	if (ret < 0)
+		dev_err(dev, "Failed to enable mclk rec and div : %d\n", ret);
+
+	ret = devm_snd_soc_register_component(dev, &src4xxx_driver,
+			src4xxx_dai_driver, ARRAY_SIZE(src4xxx_dai_driver));
+	if (ret == 0)
+		dev_info(dev, "src4392 probe ok %d\n", ret);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(src4xxx_probe);
+
+static bool src4xxx_volatile_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case SRC4XXX_RES_00:
+	case SRC4XXX_GLOBAL_ITR_STS_02:
+	case SRC4XXX_SRC_DIT_STS_0A:
+	case SRC4XXX_NON_AUDIO_D_12:
+	case SRC4XXX_RVC_STS_13:
+	case SRC4XXX_RVC_STS_14:
+	case SRC4XXX_RVC_STS_15:
+	case SRC4XXX_SUB_CODE_1F:
+	case SRC4XXX_SUB_CODE_20:
+	case SRC4XXX_SUB_CODE_21:
+	case SRC4XXX_SUB_CODE_22:
+	case SRC4XXX_SUB_CODE_23:
+	case SRC4XXX_SUB_CODE_24:
+	case SRC4XXX_SUB_CODE_25:
+	case SRC4XXX_SUB_CODE_26:
+	case SRC4XXX_SUB_CODE_27:
+	case SRC4XXX_SUB_CODE_28:
+	case SRC4XXX_PC_PREAMBLE_HI_29:
+	case SRC4XXX_PC_PREAMBLE_LO_2A:
+	case SRC4XXX_PD_PREAMBLE_HI_2B:
+	case SRC4XXX_PC_PREAMBLE_LO_2C:
+	case SRC4XXX_IO_RATIO_32:
+	case SRC4XXX_IO_RATIO_33:
+		return true;
+	}
+
+	if (reg > SRC4XXX_IO_RATIO_33 && reg < SRC4XXX_PAGE_SEL_7F)
+		return true;
+
+	return false;
+}
+
+const struct regmap_config src4xxx_regmap_config = {
+	.val_bits = 8,
+	.reg_bits = 8,
+	.max_register = SRC4XXX_IO_RATIO_33,
+
+	.reg_defaults = src4xxx_reg_defaults,
+	.num_reg_defaults = ARRAY_SIZE(src4xxx_reg_defaults),
+	.volatile_reg = src4xxx_volatile_register,
+	.cache_type = REGCACHE_RBTREE,
+};
+EXPORT_SYMBOL_GPL(src4xxx_regmap_config);
+
+MODULE_DESCRIPTION("ASoC SRC4XXX CODEC driver");
+MODULE_AUTHOR("Matt Flax <flatmax@flatmax.com>");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/src4xxx.h b/sound/soc/codecs/src4xxx.h
new file mode 100644
index 000000000000..5bf778fb9945
--- /dev/null
+++ b/sound/soc/codecs/src4xxx.h
@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// src4xxx.h  --  SRC4XXX ALSA SoC audio driver
+//
+// Copyright 2021-2022 Deqx Pty Ltd
+// Author: Matt R Flax <flatmax@flatmax.com>
+
+#ifndef __SRC4XXX_H__
+#define __SRC4XXX_H__
+
+#define SRC4XXX_RES_00 0x00
+#define SRC4XXX_PWR_RST_01 0x01
+#define SRC4XXX_RESET 0x80
+#define SRC4XXX_POWER_DOWN 0x00
+#define SRC4XXX_POWER_ENABLE 0x20
+#define SRC4XXX_ENABLE_SRC 0x1
+#define SRC4XXX_ENABLE_SRC_SHIFT 0
+#define SRC4XXX_ENABLE_DIR 0x2
+#define SRC4XXX_ENABLE_DIR_SHIFT 1
+#define SRC4XXX_ENABLE_DIT 0x4
+#define SRC4XXX_ENABLE_DIT_SHIFT 2
+#define SRC4XXX_ENABLE_PORT_B 0x8
+#define SRC4XXX_ENABLE_PORT_B_SHIFT 3
+#define SRC4XXX_ENABLE_PORT_A 0x10
+#define SRC4XXX_ENABLE_PORT_A_SHIFT 4
+
+#define SRC4XXX_PORTA_CTL_03 0x03
+#define SRC4XXX_BUS_MASTER 0x8
+#define SRC4XXX_BUS_LEFT_J 0x0
+#define SRC4XXX_BUS_I2S 0x1
+#define SRC4XXX_BUS_RIGHT_J_16 0x4
+#define SRC4XXX_BUS_RIGHT_J_18 0x5
+#define SRC4XXX_BUS_RIGHT_J_20 0x6
+#define SRC4XXX_BUS_RIGHT_J_24 0x7
+#define SRC4XXX_BUS_FMT_MS_MASK 0xf
+
+#define SRC4XXX_PORTA_CTL_04 0x04
+#define SRC4XXX_MCLK_DIV_MASK 0x3
+
+#define SRC4XXX_BUS_FMT(id) (SRC4XXX_PORTA_CTL_03+2*id)
+#define SRC4XXX_BUS_CLK(id) (SRC4XXX_PORTA_CTL_04+2*id)
+
+#define SRC4XXX_PORTB_CTL_05 0x05
+#define SRC4XXX_PORTB_CTL_06 0x06
+
+#define SRC4XXX_TX_CTL_07 0x07
+#define SRC4XXX_TX_MCLK_DIV_MASK 0x60
+#define SRC4XXX_TX_MCLK_DIV_SHIFT 5
+
+#define SRC4XXX_TX_CTL_08 0x08
+#define SRC4XXX_TX_CTL_09 0x09
+#define SRC4XXX_SRC_DIT_IRQ_MSK_0B 0x0B
+#define SRC4XXX_SRC_BTI_EN 0x01
+#define SRC4XXX_SRC_TSLIP_EN 0x02
+#define SRC4XXX_SRC_DIT_IRQ_MODE_0C 0x0C
+#define SRC4XXX_RCV_CTL_0D 0x0D
+#define SRC4XXX_RXCLK_RXCKI 0x0
+#define SRC4XXX_RXCLK_MCLK 0x8
+#define SRC4XXX_RCV_CTL_0E 0x0E
+#define SRC4XXX_REC_MCLK_EN 0x1
+#define SRC4XXX_PLL2_DIV_0 (0x0<<1)
+#define SRC4XXX_PLL2_DIV_2 (0x1<<1)
+#define SRC4XXX_PLL2_DIV_4 (0x2<<1)
+#define SRC4XXX_PLL2_DIV_8 (0x3<<1)
+#define SRC4XXX_PLL2_LOL 0x8
+#define SRC4XXX_RCV_PLL_0F 0x0F
+#define SRC4XXX_RCV_PLL_10 0x10
+#define SRC4XXX_RCV_PLL_11 0x11
+#define SRC4XXX_RVC_IRQ_MSK_16 0x16
+#define SRC4XXX_RVC_IRQ_MSK_17 0x17
+#define SRC4XXX_RVC_IRQ_MODE_18 0x18
+#define SRC4XXX_RVC_IRQ_MODE_19 0x19
+#define SRC4XXX_RVC_IRQ_MODE_1A 0x1A
+#define SRC4XXX_GPIO_1_1B 0x1B
+#define SRC4XXX_GPIO_2_1C 0x1C
+#define SRC4XXX_GPIO_3_1D 0x1D
+#define SRC4XXX_GPIO_4_1E 0x1E
+#define SRC4XXX_SCR_CTL_2D 0x2D
+#define SRC4XXX_SCR_CTL_2E 0x2E
+#define SRC4XXX_SCR_CTL_2F 0x2F
+#define SRC4XXX_SCR_CTL_30 0x30
+#define SRC4XXX_SCR_CTL_31 0x31
+#define SRC4XXX_PAGE_SEL_7F 0x7F
+
+// read only registers
+#define SRC4XXX_GLOBAL_ITR_STS_02 0x02
+#define SRC4XXX_SRC_DIT_STS_0A 0x0A
+#define SRC4XXX_NON_AUDIO_D_12 0x12
+#define SRC4XXX_RVC_STS_13 0x13
+#define SRC4XXX_RVC_STS_14 0x14
+#define SRC4XXX_RVC_STS_15 0x15
+#define SRC4XXX_SUB_CODE_1F 0x1F
+#define SRC4XXX_SUB_CODE_20 0x20
+#define SRC4XXX_SUB_CODE_21 0x21
+#define SRC4XXX_SUB_CODE_22 0x22
+#define SRC4XXX_SUB_CODE_23 0x23
+#define SRC4XXX_SUB_CODE_24 0x24
+#define SRC4XXX_SUB_CODE_25 0x25
+#define SRC4XXX_SUB_CODE_26 0x26
+#define SRC4XXX_SUB_CODE_27 0x27
+#define SRC4XXX_SUB_CODE_28 0x28
+#define SRC4XXX_PC_PREAMBLE_HI_29 0x29
+#define SRC4XXX_PC_PREAMBLE_LO_2A 0x2A
+#define SRC4XXX_PD_PREAMBLE_HI_2B 0x2B
+#define SRC4XXX_PC_PREAMBLE_LO_2C 0x2C
+#define SRC4XXX_IO_RATIO_32 0x32
+#define SRC4XXX_IO_RATIO_33 0x33
+
+int src4xxx_probe(struct device *dev, struct regmap *regmap,
+		void (*switch_mode)(struct device *dev));
+extern const struct regmap_config src4xxx_regmap_config;
+
+#endif /* __SRC4XXX_H__ */
diff --git a/sound/soc/codecs/ssm2518.c b/sound/soc/codecs/ssm2518.c
index 6d8847848299..22cb3b7c8283 100644
--- a/sound/soc/codecs/ssm2518.c
+++ b/sound/soc/codecs/ssm2518.c
@@ -6,14 +6,13 @@
  *  Author: Lars-Peter Clausen <lars@metafoo.de>
  */
 
+#include <linux/err.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/i2c.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
-#include <linux/gpio.h>
-#include <linux/of_gpio.h>
-#include <linux/platform_data/ssm2518.h>
+#include <linux/gpio/consumer.h>
 #include <sound/core.h>
 #include <sound/pcm.h>
 #include <sound/pcm_params.h>
@@ -114,7 +113,7 @@ struct ssm2518 {
 	unsigned int sysclk;
 	const struct snd_pcm_hw_constraint_list *constraints;
 
-	int enable_gpio;
+	struct gpio_desc *enable_gpio;
 };
 
 static const struct reg_default ssm2518_reg_defaults[] = {
@@ -483,8 +482,8 @@ static int ssm2518_set_power(struct ssm2518 *ssm2518, bool enable)
 		regcache_mark_dirty(ssm2518->regmap);
 	}
 
-	if (gpio_is_valid(ssm2518->enable_gpio))
-		gpio_set_value(ssm2518->enable_gpio, enable);
+	if (ssm2518->enable_gpio)
+		gpiod_set_value_cansleep(ssm2518->enable_gpio, enable);
 
 	regcache_cache_only(ssm2518->regmap, !enable);
 
@@ -736,7 +735,6 @@ static const struct regmap_config ssm2518_regmap_config = {
 
 static int ssm2518_i2c_probe(struct i2c_client *i2c)
 {
-	struct ssm2518_platform_data *pdata = i2c->dev.platform_data;
 	struct ssm2518 *ssm2518;
 	int ret;
 
@@ -744,22 +742,14 @@ static int ssm2518_i2c_probe(struct i2c_client *i2c)
 	if (ssm2518 == NULL)
 		return -ENOMEM;
 
-	if (pdata) {
-		ssm2518->enable_gpio = pdata->enable_gpio;
-	} else if (i2c->dev.of_node) {
-		ssm2518->enable_gpio = of_get_gpio(i2c->dev.of_node, 0);
-		if (ssm2518->enable_gpio < 0 && ssm2518->enable_gpio != -ENOENT)
-			return ssm2518->enable_gpio;
-	} else {
-		ssm2518->enable_gpio = -1;
-	}
+	/* Start with enabling the chip */
+	ssm2518->enable_gpio = devm_gpiod_get_optional(&i2c->dev, NULL,
+						       GPIOD_OUT_HIGH);
+	ret = PTR_ERR_OR_ZERO(ssm2518->enable_gpio);
+	if (ret)
+		return ret;
 
-	if (gpio_is_valid(ssm2518->enable_gpio)) {
-		ret = devm_gpio_request_one(&i2c->dev, ssm2518->enable_gpio,
-				GPIOF_OUT_INIT_HIGH, "SSM2518 nSD");
-		if (ret)
-			return ret;
-	}
+	gpiod_set_consumer_name(ssm2518->enable_gpio, "SSM2518 nSD");
 
 	i2c_set_clientdata(i2c, ssm2518);
 
diff --git a/sound/soc/codecs/sta350.c b/sound/soc/codecs/sta350.c
index 7b2c5b57d5d4..9ed13aeb3cbd 100644
--- a/sound/soc/codecs/sta350.c
+++ b/sound/soc/codecs/sta350.c
@@ -1242,10 +1242,8 @@ static int sta350_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static int sta350_i2c_remove(struct i2c_client *client)
-{
-	return 0;
-}
+static void sta350_i2c_remove(struct i2c_client *client)
+{}
 
 static const struct i2c_device_id sta350_i2c_id[] = {
 	{ "sta350", 0 },
diff --git a/sound/soc/codecs/sti-sas.c b/sound/soc/codecs/sti-sas.c
index f076878908ee..99545bcb2ba9 100644
--- a/sound/soc/codecs/sti-sas.c
+++ b/sound/soc/codecs/sti-sas.c
@@ -96,11 +96,8 @@ static int sti_sas_write_reg(void *context, unsigned int reg,
 			     unsigned int value)
 {
 	struct sti_sas_data *drvdata = context;
-	int status;
-
-	status = regmap_write(drvdata->dac.regmap, reg, value);
 
-	return status;
+	return regmap_write(drvdata->dac.regmap, reg, value);
 }
 
 static int  sti_sas_init_sas_registers(struct snd_soc_component *component,
@@ -385,11 +382,8 @@ static int sti_sas_resume(struct snd_soc_component *component)
 static int sti_sas_component_probe(struct snd_soc_component *component)
 {
 	struct sti_sas_data *drvdata = dev_get_drvdata(component->dev);
-	int ret;
 
-	ret = sti_sas_init_sas_registers(component, drvdata);
-
-	return ret;
+	return sti_sas_init_sas_registers(component, drvdata);
 }
 
 static struct snd_soc_component_driver sti_sas_driver = {
diff --git a/sound/soc/codecs/tas2552.c b/sound/soc/codecs/tas2552.c
index 8bd667da8767..59a4ea5f6e30 100644
--- a/sound/soc/codecs/tas2552.c
+++ b/sound/soc/codecs/tas2552.c
@@ -736,10 +736,9 @@ static int tas2552_probe(struct i2c_client *client)
 	return ret;
 }
 
-static int tas2552_i2c_remove(struct i2c_client *client)
+static void tas2552_i2c_remove(struct i2c_client *client)
 {
 	pm_runtime_disable(&client->dev);
-	return 0;
 }
 
 static const struct i2c_device_id tas2552_id[] = {
diff --git a/sound/soc/codecs/tas2562.c b/sound/soc/codecs/tas2562.c
index dc088a1c6721..b486d0bd86c9 100644
--- a/sound/soc/codecs/tas2562.c
+++ b/sound/soc/codecs/tas2562.c
@@ -54,6 +54,8 @@ struct tas2562_data {
 	int i_sense_slot;
 	int volume_lvl;
 	int model_id;
+	bool dac_powered;
+	bool unmuted;
 };
 
 enum tas256x_model {
@@ -63,39 +65,6 @@ enum tas256x_model {
 	TAS2110,
 };
 
-static int tas2562_set_bias_level(struct snd_soc_component *component,
-				 enum snd_soc_bias_level level)
-{
-	struct tas2562_data *tas2562 =
-			snd_soc_component_get_drvdata(component);
-
-	switch (level) {
-	case SND_SOC_BIAS_ON:
-		snd_soc_component_update_bits(component,
-			TAS2562_PWR_CTRL,
-			TAS2562_MODE_MASK, TAS2562_ACTIVE);
-		break;
-	case SND_SOC_BIAS_STANDBY:
-	case SND_SOC_BIAS_PREPARE:
-		snd_soc_component_update_bits(component,
-			TAS2562_PWR_CTRL,
-			TAS2562_MODE_MASK, TAS2562_MUTE);
-		break;
-	case SND_SOC_BIAS_OFF:
-		snd_soc_component_update_bits(component,
-			TAS2562_PWR_CTRL,
-			TAS2562_MODE_MASK, TAS2562_SHUTDOWN);
-		break;
-
-	default:
-		dev_err(tas2562->dev,
-				"wrong power level setting %d\n", level);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
 static int tas2562_set_samplerate(struct tas2562_data *tas2562, int samplerate)
 {
 	int samp_rate;
@@ -384,30 +353,43 @@ static int tas2562_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt)
 	return 0;
 }
 
+static int tas2562_update_pwr_ctrl(struct tas2562_data *tas2562)
+{
+	struct snd_soc_component *component = tas2562->component;
+	unsigned int val;
+	int ret;
+
+	if (tas2562->dac_powered)
+		val = tas2562->unmuted ?
+			TAS2562_ACTIVE : TAS2562_MUTE;
+	else
+		val = TAS2562_SHUTDOWN;
+
+	ret = snd_soc_component_update_bits(component, TAS2562_PWR_CTRL,
+					    TAS2562_MODE_MASK, val);
+	if (ret < 0)
+		return ret;
+
+	return 0;
+}
+
 static int tas2562_mute(struct snd_soc_dai *dai, int mute, int direction)
 {
-	struct snd_soc_component *component = dai->component;
+	struct tas2562_data *tas2562 = snd_soc_component_get_drvdata(dai->component);
 
-	return snd_soc_component_update_bits(component, TAS2562_PWR_CTRL,
-					     TAS2562_MODE_MASK,
-					     mute ? TAS2562_MUTE : 0);
+	tas2562->unmuted = !mute;
+	return tas2562_update_pwr_ctrl(tas2562);
 }
 
 static int tas2562_codec_probe(struct snd_soc_component *component)
 {
 	struct tas2562_data *tas2562 = snd_soc_component_get_drvdata(component);
-	int ret;
 
 	tas2562->component = component;
 
 	if (tas2562->sdz_gpio)
 		gpiod_set_value_cansleep(tas2562->sdz_gpio, 1);
 
-	ret = snd_soc_component_update_bits(component, TAS2562_PWR_CTRL,
-					    TAS2562_MODE_MASK, TAS2562_MUTE);
-	if (ret < 0)
-		return ret;
-
 	return 0;
 }
 
@@ -457,35 +439,23 @@ static int tas2562_dac_event(struct snd_soc_dapm_widget *w,
 	struct snd_soc_component *component =
 					snd_soc_dapm_to_component(w->dapm);
 	struct tas2562_data *tas2562 = snd_soc_component_get_drvdata(component);
-	int ret;
+	int ret = 0;
 
 	switch (event) {
 	case SND_SOC_DAPM_POST_PMU:
-		ret = snd_soc_component_update_bits(component,
-			TAS2562_PWR_CTRL,
-			TAS2562_MODE_MASK,
-			TAS2562_MUTE);
-		if (ret)
-			goto end;
+		tas2562->dac_powered = true;
+		ret = tas2562_update_pwr_ctrl(tas2562);
 		break;
 	case SND_SOC_DAPM_PRE_PMD:
-		ret = snd_soc_component_update_bits(component,
-			TAS2562_PWR_CTRL,
-			TAS2562_MODE_MASK,
-			TAS2562_SHUTDOWN);
-		if (ret)
-			goto end;
+		tas2562->dac_powered = false;
+		ret = tas2562_update_pwr_ctrl(tas2562);
 		break;
 	default:
 		dev_err(tas2562->dev, "Not supported evevt\n");
 		return -EINVAL;
 	}
 
-end:
-	if (ret < 0)
-		return ret;
-
-	return 0;
+	return ret;
 }
 
 static int tas2562_volume_control_get(struct snd_kcontrol *kcontrol,
@@ -579,7 +549,6 @@ static const struct snd_soc_component_driver soc_component_dev_tas2110 = {
 	.probe			= tas2562_codec_probe,
 	.suspend		= tas2562_suspend,
 	.resume			= tas2562_resume,
-	.set_bias_level		= tas2562_set_bias_level,
 	.controls		= tas2562_snd_controls,
 	.num_controls		= ARRAY_SIZE(tas2562_snd_controls),
 	.dapm_widgets		= tas2110_dapm_widgets,
@@ -618,7 +587,6 @@ static const struct snd_soc_component_driver soc_component_dev_tas2562 = {
 	.probe			= tas2562_codec_probe,
 	.suspend		= tas2562_suspend,
 	.resume			= tas2562_resume,
-	.set_bias_level		= tas2562_set_bias_level,
 	.controls		= tas2562_snd_controls,
 	.num_controls		= ARRAY_SIZE(tas2562_snd_controls),
 	.dapm_widgets		= tas2562_dapm_widgets,
diff --git a/sound/soc/codecs/tas2764.c b/sound/soc/codecs/tas2764.c
index 846d9d3ecc9d..51b87a936179 100644
--- a/sound/soc/codecs/tas2764.c
+++ b/sound/soc/codecs/tas2764.c
@@ -31,11 +31,66 @@ struct tas2764_priv {
 	struct gpio_desc *sdz_gpio;
 	struct regmap *regmap;
 	struct device *dev;
+	int irq;
 	
 	int v_sense_slot;
 	int i_sense_slot;
+
+	bool dac_powered;
+	bool unmuted;
+};
+
+static const char *tas2764_int_ltch0_msgs[8] = {
+	"fault: over temperature", /* INT_LTCH0 & BIT(0) */
+	"fault: over current",
+	"fault: bad TDM clock",
+	"limiter active",
+	"fault: PVDD below limiter inflection point",
+	"fault: limiter max attenuation",
+	"fault: BOP infinite hold",
+	"fault: BOP mute", /* INT_LTCH0 & BIT(7) */
+};
+
+static const unsigned int tas2764_int_readout_regs[6] = {
+	TAS2764_INT_LTCH0,
+	TAS2764_INT_LTCH1,
+	TAS2764_INT_LTCH1_0,
+	TAS2764_INT_LTCH2,
+	TAS2764_INT_LTCH3,
+	TAS2764_INT_LTCH4,
 };
 
+static irqreturn_t tas2764_irq(int irq, void *data)
+{
+	struct tas2764_priv *tas2764 = data;
+	u8 latched[6] = {0, 0, 0, 0, 0, 0};
+	int ret = IRQ_NONE;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(latched); i++)
+		latched[i] = snd_soc_component_read(tas2764->component,
+						    tas2764_int_readout_regs[i]);
+
+	for (i = 0; i < 8; i++) {
+		if (latched[0] & BIT(i)) {
+			dev_crit_ratelimited(tas2764->dev, "%s\n",
+					     tas2764_int_ltch0_msgs[i]);
+			ret = IRQ_HANDLED;
+		}
+	}
+
+	if (latched[0]) {
+		dev_err_ratelimited(tas2764->dev, "other context to the fault: %02x,%02x,%02x,%02x,%02x",
+				    latched[1], latched[2], latched[3], latched[4], latched[5]);
+		snd_soc_component_update_bits(tas2764->component,
+					      TAS2764_INT_CLK_CFG,
+					      TAS2764_INT_CLK_CFG_IRQZ_CLR,
+					      TAS2764_INT_CLK_CFG_IRQZ_CLR);
+	}
+
+	return ret;
+}
+
 static void tas2764_reset(struct tas2764_priv *tas2764)
 {
 	if (tas2764->reset_gpio) {
@@ -50,34 +105,22 @@ static void tas2764_reset(struct tas2764_priv *tas2764)
 	usleep_range(1000, 2000);
 }
 
-static int tas2764_set_bias_level(struct snd_soc_component *component,
-				 enum snd_soc_bias_level level)
+static int tas2764_update_pwr_ctrl(struct tas2764_priv *tas2764)
 {
-	struct tas2764_priv *tas2764 = snd_soc_component_get_drvdata(component);
+	struct snd_soc_component *component = tas2764->component;
+	unsigned int val;
+	int ret;
 
-	switch (level) {
-	case SND_SOC_BIAS_ON:
-		snd_soc_component_update_bits(component, TAS2764_PWR_CTRL,
-					      TAS2764_PWR_CTRL_MASK,
-					      TAS2764_PWR_CTRL_ACTIVE);
-		break;
-	case SND_SOC_BIAS_STANDBY:
-	case SND_SOC_BIAS_PREPARE:
-		snd_soc_component_update_bits(component, TAS2764_PWR_CTRL,
-					      TAS2764_PWR_CTRL_MASK,
-					      TAS2764_PWR_CTRL_MUTE);
-		break;
-	case SND_SOC_BIAS_OFF:
-		snd_soc_component_update_bits(component, TAS2764_PWR_CTRL,
-					      TAS2764_PWR_CTRL_MASK,
-					      TAS2764_PWR_CTRL_SHUTDOWN);
-		break;
+	if (tas2764->dac_powered)
+		val = tas2764->unmuted ?
+			TAS2764_PWR_CTRL_ACTIVE : TAS2764_PWR_CTRL_MUTE;
+	else
+		val = TAS2764_PWR_CTRL_SHUTDOWN;
 
-	default:
-		dev_err(tas2764->dev,
-				"wrong power level setting %d\n", level);
-		return -EINVAL;
-	}
+	ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL,
+					    TAS2764_PWR_CTRL_MASK, val);
+	if (ret < 0)
+		return ret;
 
 	return 0;
 }
@@ -114,9 +157,7 @@ static int tas2764_codec_resume(struct snd_soc_component *component)
 		usleep_range(1000, 2000);
 	}
 
-	ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL,
-					    TAS2764_PWR_CTRL_MASK,
-					    TAS2764_PWR_CTRL_ACTIVE);
+	ret = tas2764_update_pwr_ctrl(tas2764);
 
 	if (ret < 0)
 		return ret;
@@ -150,14 +191,12 @@ static int tas2764_dac_event(struct snd_soc_dapm_widget *w,
 
 	switch (event) {
 	case SND_SOC_DAPM_POST_PMU:
-		ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL,
-						    TAS2764_PWR_CTRL_MASK,
-						    TAS2764_PWR_CTRL_MUTE);
+		tas2764->dac_powered = true;
+		ret = tas2764_update_pwr_ctrl(tas2764);
 		break;
 	case SND_SOC_DAPM_PRE_PMD:
-		ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL,
-						    TAS2764_PWR_CTRL_MASK,
-						    TAS2764_PWR_CTRL_SHUTDOWN);
+		tas2764->dac_powered = false;
+		ret = tas2764_update_pwr_ctrl(tas2764);
 		break;
 	default:
 		dev_err(tas2764->dev, "Unsupported event\n");
@@ -202,17 +241,11 @@ static const struct snd_soc_dapm_route tas2764_audio_map[] = {
 
 static int tas2764_mute(struct snd_soc_dai *dai, int mute, int direction)
 {
-	struct snd_soc_component *component = dai->component;
-	int ret;
-
-	ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL,
-					    TAS2764_PWR_CTRL_MASK,
-					    mute ? TAS2764_PWR_CTRL_MUTE : 0);
+	struct tas2764_priv *tas2764 =
+			snd_soc_component_get_drvdata(dai->component);
 
-	if (ret < 0)
-		return ret;
-
-	return 0;
+	tas2764->unmuted = !mute;
+	return tas2764_update_pwr_ctrl(tas2764);
 }
 
 static int tas2764_set_bitwidth(struct tas2764_priv *tas2764, int bitwidth)
@@ -485,7 +518,7 @@ static struct snd_soc_dai_driver tas2764_dai_driver[] = {
 		.id = 0,
 		.playback = {
 			.stream_name    = "ASI1 Playback",
-			.channels_min   = 2,
+			.channels_min   = 1,
 			.channels_max   = 2,
 			.rates      = TAS2764_RATES,
 			.formats    = TAS2764_FORMATS,
@@ -516,6 +549,34 @@ static int tas2764_codec_probe(struct snd_soc_component *component)
 
 	tas2764_reset(tas2764);
 
+	if (tas2764->irq) {
+		ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK0, 0xff);
+		if (ret < 0)
+			return ret;
+
+		ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK1, 0xff);
+		if (ret < 0)
+			return ret;
+
+		ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK2, 0xff);
+		if (ret < 0)
+			return ret;
+
+		ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK3, 0xff);
+		if (ret < 0)
+			return ret;
+
+		ret = snd_soc_component_write(tas2764->component, TAS2764_INT_MASK4, 0xff);
+		if (ret < 0)
+			return ret;
+
+		ret = devm_request_threaded_irq(tas2764->dev, tas2764->irq, NULL, tas2764_irq,
+						IRQF_ONESHOT | IRQF_SHARED | IRQF_TRIGGER_LOW,
+						"tas2764", tas2764);
+		if (ret)
+			dev_warn(tas2764->dev, "failed to request IRQ: %d\n", ret);
+	}
+
 	ret = snd_soc_component_update_bits(tas2764->component, TAS2764_TDM_CFG5,
 					    TAS2764_TDM_CFG5_VSNS_ENABLE, 0);
 	if (ret < 0)
@@ -526,30 +587,33 @@ static int tas2764_codec_probe(struct snd_soc_component *component)
 	if (ret < 0)
 		return ret;
 
-	ret = snd_soc_component_update_bits(component, TAS2764_PWR_CTRL,
-					    TAS2764_PWR_CTRL_MASK,
-					    TAS2764_PWR_CTRL_MUTE);
-	if (ret < 0)
-		return ret;
-
 	return 0;
 }
 
 static DECLARE_TLV_DB_SCALE(tas2764_digital_tlv, 1100, 50, 0);
 static DECLARE_TLV_DB_SCALE(tas2764_playback_volume, -10050, 50, 1);
 
+static const char * const tas2764_hpf_texts[] = {
+	"Disabled", "2 Hz", "50 Hz", "100 Hz", "200 Hz",
+	"400 Hz", "800 Hz"
+};
+
+static SOC_ENUM_SINGLE_DECL(
+	tas2764_hpf_enum, TAS2764_DC_BLK0,
+	TAS2764_DC_BLK0_HPF_FREQ_PB_SHIFT, tas2764_hpf_texts);
+
 static const struct snd_kcontrol_new tas2764_snd_controls[] = {
 	SOC_SINGLE_TLV("Speaker Volume", TAS2764_DVC, 0,
 		       TAS2764_DVC_MAX, 1, tas2764_playback_volume),
 	SOC_SINGLE_TLV("Amp Gain Volume", TAS2764_CHNL_0, 1, 0x14, 0,
 		       tas2764_digital_tlv),
+	SOC_ENUM("HPF Corner Frequency", tas2764_hpf_enum),
 };
 
 static const struct snd_soc_component_driver soc_component_driver_tas2764 = {
 	.probe			= tas2764_codec_probe,
 	.suspend		= tas2764_codec_suspend,
 	.resume			= tas2764_codec_resume,
-	.set_bias_level		= tas2764_set_bias_level,
 	.controls		= tas2764_snd_controls,
 	.num_controls		= ARRAY_SIZE(tas2764_snd_controls),
 	.dapm_widgets		= tas2764_dapm_widgets,
@@ -585,9 +649,21 @@ static const struct regmap_range_cfg tas2764_regmap_ranges[] = {
 	},
 };
 
+static bool tas2764_volatile_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case TAS2764_INT_LTCH0 ... TAS2764_INT_LTCH4:
+	case TAS2764_INT_CLK_CFG:
+		return true;
+	default:
+		return false;
+	}
+}
+
 static const struct regmap_config tas2764_i2c_regmap = {
 	.reg_bits = 8,
 	.val_bits = 8,
+	.volatile_reg = tas2764_volatile_register,
 	.reg_defaults = tas2764_reg_defaults,
 	.num_reg_defaults = ARRAY_SIZE(tas2764_reg_defaults),
 	.cache_type = REGCACHE_RBTREE,
@@ -641,6 +717,7 @@ static int tas2764_i2c_probe(struct i2c_client *client)
 		return -ENOMEM;
 
 	tas2764->dev = &client->dev;
+	tas2764->irq = client->irq;
 	i2c_set_clientdata(client, tas2764);
 	dev_set_drvdata(&client->dev, tas2764);
 
diff --git a/sound/soc/codecs/tas2764.h b/sound/soc/codecs/tas2764.h
index f015f22a083b..168af772a898 100644
--- a/sound/soc/codecs/tas2764.h
+++ b/sound/soc/codecs/tas2764.h
@@ -33,6 +33,10 @@
 #define TAS2764_VSENSE_POWER_EN		3
 #define TAS2764_ISENSE_POWER_EN		4
 
+/* DC Blocker Control */
+#define TAS2764_DC_BLK0			TAS2764_REG(0x0, 0x04)
+#define TAS2764_DC_BLK0_HPF_FREQ_PB_SHIFT  0
+
 /* Digital Volume Control */
 #define TAS2764_DVC	TAS2764_REG(0X0, 0x1a)
 #define TAS2764_DVC_MAX	0xc9
@@ -87,4 +91,23 @@
 #define TAS2764_TDM_CFG6_ISNS_ENABLE	BIT(6)
 #define TAS2764_TDM_CFG6_50_MASK	GENMASK(5, 0)
 
+/* Interrupt Masks */
+#define TAS2764_INT_MASK0               TAS2764_REG(0x0, 0x3b)
+#define TAS2764_INT_MASK1               TAS2764_REG(0x0, 0x3c)
+#define TAS2764_INT_MASK2               TAS2764_REG(0x0, 0x40)
+#define TAS2764_INT_MASK3               TAS2764_REG(0x0, 0x41)
+#define TAS2764_INT_MASK4               TAS2764_REG(0x0, 0x3d)
+
+/* Latched Fault Registers */
+#define TAS2764_INT_LTCH0               TAS2764_REG(0x0, 0x49)
+#define TAS2764_INT_LTCH1               TAS2764_REG(0x0, 0x4a)
+#define TAS2764_INT_LTCH1_0             TAS2764_REG(0x0, 0x4b)
+#define TAS2764_INT_LTCH2               TAS2764_REG(0x0, 0x4f)
+#define TAS2764_INT_LTCH3               TAS2764_REG(0x0, 0x50)
+#define TAS2764_INT_LTCH4               TAS2764_REG(0x0, 0x51)
+
+/* Clock/IRQ Settings */
+#define TAS2764_INT_CLK_CFG             TAS2764_REG(0x0, 0x5c)
+#define TAS2764_INT_CLK_CFG_IRQZ_CLR    BIT(2)
+
 #endif /* __TAS2764__ */
diff --git a/sound/soc/codecs/tas2770.c b/sound/soc/codecs/tas2770.c
index bb653b664146..b6765235a4b3 100644
--- a/sound/soc/codecs/tas2770.c
+++ b/sound/soc/codecs/tas2770.c
@@ -495,6 +495,8 @@ static struct snd_soc_dai_driver tas2770_dai_driver[] = {
 	},
 };
 
+static const struct regmap_config tas2770_i2c_regmap;
+
 static int tas2770_codec_probe(struct snd_soc_component *component)
 {
 	struct tas2770_priv *tas2770 =
@@ -508,6 +510,7 @@ static int tas2770_codec_probe(struct snd_soc_component *component)
 	}
 
 	tas2770_reset(tas2770);
+	regmap_reinit_cache(tas2770->regmap, &tas2770_i2c_regmap);
 
 	return 0;
 }
diff --git a/sound/soc/codecs/tas5086.c b/sound/soc/codecs/tas5086.c
index a864984225bc..22143cc5afa7 100644
--- a/sound/soc/codecs/tas5086.c
+++ b/sound/soc/codecs/tas5086.c
@@ -981,10 +981,8 @@ static int tas5086_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static int tas5086_i2c_remove(struct i2c_client *i2c)
-{
-	return 0;
-}
+static void tas5086_i2c_remove(struct i2c_client *i2c)
+{}
 
 static struct i2c_driver tas5086_i2c_driver = {
 	.driver = {
diff --git a/sound/soc/codecs/tas571x.c b/sound/soc/codecs/tas571x.c
index 4e7f20db57c4..84ec1b527646 100644
--- a/sound/soc/codecs/tas571x.c
+++ b/sound/soc/codecs/tas571x.c
@@ -884,13 +884,11 @@ disable_regs:
 	return ret;
 }
 
-static int tas571x_i2c_remove(struct i2c_client *client)
+static void tas571x_i2c_remove(struct i2c_client *client)
 {
 	struct tas571x_private *priv = i2c_get_clientdata(client);
 
 	regulator_bulk_disable(priv->chip->num_supply_names, priv->supplies);
-
-	return 0;
 }
 
 static const struct of_device_id tas571x_of_match[] __maybe_unused = {
diff --git a/sound/soc/codecs/tas5805m.c b/sound/soc/codecs/tas5805m.c
index b1bb614534f7..beb4ec629a03 100644
--- a/sound/soc/codecs/tas5805m.c
+++ b/sound/soc/codecs/tas5805m.c
@@ -522,7 +522,7 @@ static int tas5805m_i2c_probe(struct i2c_client *i2c)
 	return 0;
 }
 
-static int tas5805m_i2c_remove(struct i2c_client *i2c)
+static void tas5805m_i2c_remove(struct i2c_client *i2c)
 {
 	struct device *dev = &i2c->dev;
 	struct tas5805m_priv *tas5805m = dev_get_drvdata(dev);
@@ -531,7 +531,6 @@ static int tas5805m_i2c_remove(struct i2c_client *i2c)
 	gpiod_set_value(tas5805m->gpio_pdn_n, 0);
 	usleep_range(10000, 15000);
 	regulator_disable(tas5805m->pvdd);
-	return 0;
 }
 
 static const struct i2c_device_id tas5805m_i2c_id[] = {
diff --git a/sound/soc/codecs/tas6424.c b/sound/soc/codecs/tas6424.c
index 63d2983c3fcf..f8ff69fa2549 100644
--- a/sound/soc/codecs/tas6424.c
+++ b/sound/soc/codecs/tas6424.c
@@ -774,7 +774,7 @@ disable_regs:
 	return ret;
 }
 
-static int tas6424_i2c_remove(struct i2c_client *client)
+static void tas6424_i2c_remove(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
 	struct tas6424_data *tas6424 = dev_get_drvdata(dev);
@@ -790,8 +790,6 @@ static int tas6424_i2c_remove(struct i2c_client *client)
 				     tas6424->supplies);
 	if (ret < 0)
 		dev_err(dev, "unable to disable supplies: %d\n", ret);
-
-	return 0;
 }
 
 static const struct i2c_device_id tas6424_i2c_ids[] = {
diff --git a/sound/soc/codecs/tfa989x.c b/sound/soc/codecs/tfa989x.c
index 1c27429b9af6..b853507e65a8 100644
--- a/sound/soc/codecs/tfa989x.c
+++ b/sound/soc/codecs/tfa989x.c
@@ -193,7 +193,7 @@ static int tfa9890_init(struct regmap *regmap)
 {
 	int ret;
 
-	/* unhide keys to allow updating them */
+	/* temporarily allow access to hidden registers */
 	ret = regmap_write(regmap, TFA989X_HIDE_UNHIDE_KEY, 0x5a6b);
 	if (ret)
 		return ret;
@@ -203,7 +203,7 @@ static int tfa9890_init(struct regmap *regmap)
 	if (ret)
 		return ret;
 
-	/* hide keys again */
+	/* hide registers again */
 	ret = regmap_write(regmap, TFA989X_HIDE_UNHIDE_KEY, 0x0000);
 	if (ret)
 		return ret;
diff --git a/sound/soc/codecs/tlv320adc3xxx.c b/sound/soc/codecs/tlv320adc3xxx.c
index 748998e48af9..baab320ef988 100644
--- a/sound/soc/codecs/tlv320adc3xxx.c
+++ b/sound/soc/codecs/tlv320adc3xxx.c
@@ -1426,7 +1426,7 @@ err_unprepare_mclk:
 	return ret;
 }
 
-static int __exit adc3xxx_i2c_remove(struct i2c_client *client)
+static void __exit adc3xxx_i2c_remove(struct i2c_client *client)
 {
 	struct adc3xxx *adc3xxx = i2c_get_clientdata(client);
 
@@ -1434,7 +1434,6 @@ static int __exit adc3xxx_i2c_remove(struct i2c_client *client)
 		clk_disable_unprepare(adc3xxx->mclk);
 	adc3xxx_free_gpio(adc3xxx);
 	snd_soc_unregister_component(&client->dev);
-	return 0;
 }
 
 static const struct of_device_id tlv320adc3xxx_of_match[] = {
diff --git a/sound/soc/codecs/tlv320adcx140.c b/sound/soc/codecs/tlv320adcx140.c
index 2844a9d2bc4a..91a22d927915 100644
--- a/sound/soc/codecs/tlv320adcx140.c
+++ b/sound/soc/codecs/tlv320adcx140.c
@@ -31,6 +31,7 @@ struct adcx140_priv {
 	struct device *dev;
 
 	bool micbias_vg;
+	bool phase_calib_on;
 
 	unsigned int dai_fmt;
 	unsigned int slot_width;
@@ -592,6 +593,52 @@ static const struct snd_soc_dapm_route adcx140_audio_map[] = {
 	{"MIC4M Input Mux", "Digital", "MIC4M"},
 };
 
+#define ADCX140_PHASE_CALIB_SWITCH(xname) {\
+	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, .name = (xname), \
+	.access = SNDRV_CTL_ELEM_ACCESS_READWRITE,\
+	.info = adcx140_phase_calib_info, \
+	.get = adcx140_phase_calib_get, \
+	.put = adcx140_phase_calib_put}
+
+static int adcx140_phase_calib_info(struct snd_kcontrol *kcontrol,
+	struct snd_ctl_elem_info *uinfo)
+{
+	uinfo->type = SNDRV_CTL_ELEM_TYPE_BOOLEAN;
+	uinfo->count = 1;
+	uinfo->value.integer.min = 0;
+	uinfo->value.integer.max = 1;
+	return 0;
+}
+
+static int adcx140_phase_calib_get(struct snd_kcontrol *kcontrol,
+	struct snd_ctl_elem_value *value)
+{
+	struct snd_soc_component *codec =
+		snd_soc_kcontrol_component(kcontrol);
+	struct adcx140_priv *adcx140 = snd_soc_component_get_drvdata(codec);
+
+	value->value.integer.value[0] = adcx140->phase_calib_on ? 1 : 0;
+
+
+	return 0;
+}
+
+static int adcx140_phase_calib_put(struct snd_kcontrol *kcontrol,
+	struct snd_ctl_elem_value *value)
+{
+	struct snd_soc_component *codec
+		= snd_soc_kcontrol_component(kcontrol);
+	struct adcx140_priv *adcx140 = snd_soc_component_get_drvdata(codec);
+
+	bool v = value->value.integer.value[0] ? true : false;
+
+	if (adcx140->phase_calib_on != v) {
+		adcx140->phase_calib_on = v;
+		return 1;
+	}
+	return 0;
+}
+
 static const struct snd_kcontrol_new adcx140_snd_controls[] = {
 	SOC_SINGLE_TLV("Analog CH1 Mic Gain Volume", ADCX140_CH1_CFG1, 2, 42, 0,
 			adc_tlv),
@@ -628,6 +675,7 @@ static const struct snd_kcontrol_new adcx140_snd_controls[] = {
 			0, 0xff, 0, dig_vol_tlv),
 	SOC_SINGLE_TLV("Digital CH8 Out Volume", ADCX140_CH8_CFG2,
 			0, 0xff, 0, dig_vol_tlv),
+	ADCX140_PHASE_CALIB_SWITCH("Phase Calibration Switch"),
 };
 
 static int adcx140_reset(struct adcx140_priv *adcx140)
@@ -653,6 +701,8 @@ static int adcx140_reset(struct adcx140_priv *adcx140)
 static void adcx140_pwr_ctrl(struct adcx140_priv *adcx140, bool power_state)
 {
 	int pwr_ctrl = 0;
+	int ret = 0;
+	struct snd_soc_component *component = adcx140->component;
 
 	if (power_state)
 		pwr_ctrl = ADCX140_PWR_CFG_ADC_PDZ | ADCX140_PWR_CFG_PLL_PDZ;
@@ -660,6 +710,14 @@ static void adcx140_pwr_ctrl(struct adcx140_priv *adcx140, bool power_state)
 	if (adcx140->micbias_vg && power_state)
 		pwr_ctrl |= ADCX140_PWR_CFG_BIAS_PDZ;
 
+	if (pwr_ctrl) {
+		ret = regmap_write(adcx140->regmap, ADCX140_PHASE_CALIB,
+			adcx140->phase_calib_on ? 0x00 : 0x40);
+		if (ret)
+			dev_err(component->dev, "%s: register write error %d\n",
+				__func__, ret);
+	}
+
 	regmap_update_bits(adcx140->regmap, ADCX140_PWR_CFG,
 			   ADCX140_PWR_CTRL_MSK, pwr_ctrl);
 }
@@ -1095,6 +1153,7 @@ static int adcx140_i2c_probe(struct i2c_client *i2c)
 	if (!adcx140)
 		return -ENOMEM;
 
+	adcx140->phase_calib_on = false;
 	adcx140->dev = &i2c->dev;
 
 	adcx140->gpio_reset = devm_gpiod_get_optional(adcx140->dev,
diff --git a/sound/soc/codecs/tlv320adcx140.h b/sound/soc/codecs/tlv320adcx140.h
index d7d4e3a88b5c..fd80fac8b327 100644
--- a/sound/soc/codecs/tlv320adcx140.h
+++ b/sound/soc/codecs/tlv320adcx140.h
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-// TLV320ADCX104 Sound driver
+// TLV320ADCX140 Sound driver
 // Copyright (C) 2020 Texas Instruments Incorporated - https://www.ti.com/
 
 #ifndef _TLV320ADCX140_H
@@ -90,6 +90,7 @@
 #define ADCX140_PWR_CFG		0x75
 #define ADCX140_DEV_STS0	0x76
 #define ADCX140_DEV_STS1	0x77
+#define ADCX140_PHASE_CALIB		0X7b
 
 #define ADCX140_RESET	BIT(0)
 
diff --git a/sound/soc/codecs/tlv320aic26.c b/sound/soc/codecs/tlv320aic26.c
index 8bae4b475068..e5dfb3d752a3 100644
--- a/sound/soc/codecs/tlv320aic26.c
+++ b/sound/soc/codecs/tlv320aic26.c
@@ -271,7 +271,7 @@ static ssize_t keyclick_show(struct device *dev,
 	freq = (125 << ((val >> 8) & 0x7)) >> 1;
 	len = 2 * (1 + ((val >> 4) & 0xf));
 
-	return sprintf(buf, "amp=%x freq=%iHz len=%iclks\n", amp, freq, len);
+	return sysfs_emit(buf, "amp=%x freq=%iHz len=%iclks\n", amp, freq, len);
 }
 
 /* Any write to the keyclick attribute will trigger the keyclick event */
diff --git a/sound/soc/codecs/tlv320aic32x4-i2c.c b/sound/soc/codecs/tlv320aic32x4-i2c.c
index 0645239901b1..d1e543ca3521 100644
--- a/sound/soc/codecs/tlv320aic32x4-i2c.c
+++ b/sound/soc/codecs/tlv320aic32x4-i2c.c
@@ -45,11 +45,9 @@ static int aic32x4_i2c_probe(struct i2c_client *i2c)
 	return aic32x4_probe(&i2c->dev, regmap);
 }
 
-static int aic32x4_i2c_remove(struct i2c_client *i2c)
+static void aic32x4_i2c_remove(struct i2c_client *i2c)
 {
 	aic32x4_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct i2c_device_id aic32x4_i2c_id[] = {
diff --git a/sound/soc/codecs/tlv320aic3x-i2c.c b/sound/soc/codecs/tlv320aic3x-i2c.c
index 7bd9ce08bb7b..d7e94d564dbf 100644
--- a/sound/soc/codecs/tlv320aic3x-i2c.c
+++ b/sound/soc/codecs/tlv320aic3x-i2c.c
@@ -41,11 +41,9 @@ static int aic3x_i2c_probe(struct i2c_client *i2c)
 	return aic3x_probe(&i2c->dev, regmap, id->driver_data);
 }
 
-static int aic3x_i2c_remove(struct i2c_client *i2c)
+static void aic3x_i2c_remove(struct i2c_client *i2c)
 {
 	aic3x_remove(&i2c->dev);
-
-	return 0;
 }
 
 static const struct of_device_id aic3x_of_id[] = {
diff --git a/sound/soc/codecs/tlv320dac33.c b/sound/soc/codecs/tlv320dac33.c
index 17ae3b1d96fb..16ce3ef1134b 100644
--- a/sound/soc/codecs/tlv320dac33.c
+++ b/sound/soc/codecs/tlv320dac33.c
@@ -1536,7 +1536,7 @@ err_gpio:
 	return ret;
 }
 
-static int dac33_i2c_remove(struct i2c_client *client)
+static void dac33_i2c_remove(struct i2c_client *client)
 {
 	struct tlv320dac33_priv *dac33 = i2c_get_clientdata(client);
 
@@ -1545,8 +1545,6 @@ static int dac33_i2c_remove(struct i2c_client *client)
 
 	if (dac33->power_gpio >= 0)
 		gpio_free(dac33->power_gpio);
-
-	return 0;
 }
 
 static const struct i2c_device_id tlv320dac33_i2c_id[] = {
diff --git a/sound/soc/codecs/ts3a227e.c b/sound/soc/codecs/ts3a227e.c
index d8ab0810fceb..2305a472d132 100644
--- a/sound/soc/codecs/ts3a227e.c
+++ b/sound/soc/codecs/ts3a227e.c
@@ -78,12 +78,20 @@ static const int ts3a227e_buttons[] = {
 #define ADC_COMPLETE_INT_DISABLE 0x04
 #define INTB_DISABLE 0x08
 
+/* TS3A227E_REG_SETTING_1 0x4 */
+#define DEBOUNCE_INSERTION_SETTING_SFT (0)
+#define DEBOUNCE_INSERTION_SETTING_MASK (0x7 << DEBOUNCE_PRESS_SETTING_SFT)
+
 /* TS3A227E_REG_SETTING_2 0x05 */
 #define KP_ENABLE 0x04
 
 /* TS3A227E_REG_SETTING_3 0x06 */
-#define MICBIAS_SETTING_SFT (3)
+#define MICBIAS_SETTING_SFT 3
 #define MICBIAS_SETTING_MASK (0x7 << MICBIAS_SETTING_SFT)
+#define DEBOUNCE_RELEASE_SETTING_SFT 2
+#define DEBOUNCE_RELEASE_SETTING_MASK (0x1 << DEBOUNCE_RELEASE_SETTING_SFT)
+#define DEBOUNCE_PRESS_SETTING_SFT 0
+#define DEBOUNCE_PRESS_SETTING_MASK (0x3 << DEBOUNCE_PRESS_SETTING_SFT)
 
 /* TS3A227E_REG_ACCESSORY_STATUS  0x0b */
 #define TYPE_3_POLE 0x01
@@ -136,7 +144,7 @@ static bool ts3a227e_volatile_reg(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
 	case TS3A227E_REG_INTERRUPT ... TS3A227E_REG_INTERRUPT_DISABLE:
-	case TS3A227E_REG_SETTING_2:
+	case TS3A227E_REG_SETTING_1 ... TS3A227E_REG_SETTING_2:
 	case TS3A227E_REG_SWITCH_STATUS_1 ... TS3A227E_REG_ADC_OUTPUT:
 		return true;
 	default:
@@ -269,14 +277,55 @@ static const struct regmap_config ts3a227e_regmap_config = {
 static int ts3a227e_parse_device_property(struct ts3a227e *ts3a227e,
 				struct device *dev)
 {
-	u32 micbias;
+	u32 value;
+	u32 value_ms;
+	u32 setting3_value = 0;
+	u32 setting3_mask = 0;
 	int err;
 
-	err = device_property_read_u32(dev, "ti,micbias", &micbias);
+	err = device_property_read_u32(dev, "ti,micbias", &value);
+	if (!err) {
+		setting3_mask = MICBIAS_SETTING_MASK;
+		setting3_value = (value << MICBIAS_SETTING_SFT) &
+				 MICBIAS_SETTING_MASK;
+	}
+
+	err = device_property_read_u32(dev, "ti,debounce-release-ms",
+				       &value_ms);
 	if (!err) {
+		value = (value_ms > 10);
+		setting3_mask |= DEBOUNCE_RELEASE_SETTING_MASK;
+		setting3_value |= (value << DEBOUNCE_RELEASE_SETTING_SFT) &
+				  DEBOUNCE_RELEASE_SETTING_MASK;
+	}
+
+	err = device_property_read_u32(dev, "ti,debounce-press-ms", &value_ms);
+	if (!err) {
+		value = (value_ms + 20) / 40;
+		if (value > 3)
+			value = 3;
+		setting3_mask |= DEBOUNCE_PRESS_SETTING_MASK;
+		setting3_value |= (value << DEBOUNCE_PRESS_SETTING_SFT) &
+				  DEBOUNCE_PRESS_SETTING_MASK;
+	}
+
+	if (setting3_mask)
 		regmap_update_bits(ts3a227e->regmap, TS3A227E_REG_SETTING_3,
-			MICBIAS_SETTING_MASK,
-			(micbias & 0x07) << MICBIAS_SETTING_SFT);
+				   setting3_mask, setting3_value);
+
+	err = device_property_read_u32(dev, "ti,debounce-insertion-ms",
+				       &value_ms);
+	if (!err) {
+		if (value_ms < 165)
+			value = (value_ms + 15) / 30;
+		else if (value_ms < 1500)
+			value = 6;
+		else
+			value = 7;
+		regmap_update_bits(ts3a227e->regmap, TS3A227E_REG_SETTING_1,
+				   DEBOUNCE_INSERTION_SETTING_MASK,
+				   (value << DEBOUNCE_INSERTION_SETTING_SFT) &
+					   DEBOUNCE_INSERTION_SETTING_MASK);
 	}
 
 	return 0;
diff --git a/sound/soc/codecs/uda134x.c b/sound/soc/codecs/uda134x.c
index 2db3d8a60c7a..1a62bec94005 100644
--- a/sound/soc/codecs/uda134x.c
+++ b/sound/soc/codecs/uda134x.c
@@ -450,7 +450,7 @@ static int uda134x_soc_probe(struct snd_soc_component *component)
 	struct uda134x_priv *uda134x = snd_soc_component_get_drvdata(component);
 	struct uda134x_platform_data *pd = uda134x->pd;
 	const struct snd_soc_dapm_widget *widgets;
-	unsigned num_widgets;
+	unsigned int num_widgets;
 	int ret;
 
 	printk(KERN_INFO "UDA134X SoC Audio Codec\n");
diff --git a/sound/soc/codecs/wcd-mbhc-v2.c b/sound/soc/codecs/wcd-mbhc-v2.c
index 98baef594bf3..1911750f7445 100644
--- a/sound/soc/codecs/wcd-mbhc-v2.c
+++ b/sound/soc/codecs/wcd-mbhc-v2.c
@@ -714,11 +714,12 @@ static int wcd_mbhc_initialise(struct wcd_mbhc *mbhc)
 	struct snd_soc_component *component = mbhc->component;
 	int ret;
 
-	ret = pm_runtime_resume_and_get(component->dev);
+	ret = pm_runtime_get_sync(component->dev);
 	if (ret < 0 && ret != -EACCES) {
 		dev_err_ratelimited(component->dev,
-				    "pm_runtime_resume_and_get failed in %s, ret %d\n",
+				    "pm_runtime_get_sync failed in %s, ret %d\n",
 				    __func__, ret);
+		pm_runtime_put_noidle(component->dev);
 		return ret;
 	}
 
@@ -1096,11 +1097,12 @@ static void wcd_correct_swch_plug(struct work_struct *work)
 	mbhc = container_of(work, struct wcd_mbhc, correct_plug_swch);
 	component = mbhc->component;
 
-	ret = pm_runtime_resume_and_get(component->dev);
+	ret = pm_runtime_get_sync(component->dev);
 	if (ret < 0 && ret != -EACCES) {
 		dev_err_ratelimited(component->dev,
-				    "pm_runtime_resume_and_get failed in %s, ret %d\n",
+				    "pm_runtime_get_sync failed in %s, ret %d\n",
 				    __func__, ret);
+		pm_runtime_put_noidle(component->dev);
 		return;
 	}
 	micbias_mv = wcd_mbhc_get_micbias(mbhc);
@@ -1189,7 +1191,7 @@ correct_plug_type:
 				pt_gnd_mic_swap_cnt = 0;
 				plug_type = wcd_mbhc_get_plug_from_adc(mbhc, output_mv);
 				continue;
-			} else if (cross_conn < 0) /* Error */
+			} else /* Error if (cross_conn < 0) */
 				continue;
 
 			if (pt_gnd_mic_swap_cnt == GND_MIC_SWAP_THRESHOLD) {
diff --git a/sound/soc/codecs/wcd9335.c b/sound/soc/codecs/wcd9335.c
index beeeb35e8032..d2548fdf9ae5 100644
--- a/sound/soc/codecs/wcd9335.c
+++ b/sound/soc/codecs/wcd9335.c
@@ -1821,12 +1821,10 @@ static int wcd9335_set_decimator_rate(struct snd_soc_dai *dai,
 			tx_port_reg = WCD9335_CDC_IF_ROUTER_TX_MUX_CFG3;
 			shift = 0;
 			shift_val = 0x0F;
-		} else if (tx_port == 13) {
+		} else /* (tx_port == 13) */ {
 			tx_port_reg = WCD9335_CDC_IF_ROUTER_TX_MUX_CFG3;
 			shift = 4;
 			shift_val = 0x03;
-		} else {
-			return -EINVAL;
 		}
 
 		tx_mux_sel = snd_soc_component_read(comp, tx_port_reg) &
@@ -1974,8 +1972,8 @@ static int wcd9335_trigger(struct snd_pcm_substream *substream, int cmd,
 	case SNDRV_PCM_TRIGGER_STOP:
 	case SNDRV_PCM_TRIGGER_SUSPEND:
 	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		slim_stream_unprepare(dai_data->sruntime);
 		slim_stream_disable(dai_data->sruntime);
+		slim_stream_unprepare(dai_data->sruntime);
 		break;
 	default:
 		break;
@@ -5013,16 +5011,22 @@ static const struct regmap_irq wcd9335_codec_irqs[] = {
 	},
 };
 
+static const unsigned int wcd9335_config_regs[] = {
+	WCD9335_INTR_LEVEL0,
+};
+
 static const struct regmap_irq_chip wcd9335_regmap_irq1_chip = {
 	.name = "wcd9335_pin1_irq",
 	.status_base = WCD9335_INTR_PIN1_STATUS0,
 	.mask_base = WCD9335_INTR_PIN1_MASK0,
 	.ack_base = WCD9335_INTR_PIN1_CLEAR0,
-	.type_base = WCD9335_INTR_LEVEL0,
-	.num_type_reg = 4,
 	.num_regs = 4,
 	.irqs = wcd9335_codec_irqs,
 	.num_irqs = ARRAY_SIZE(wcd9335_codec_irqs),
+	.config_base = wcd9335_config_regs,
+	.num_config_bases = ARRAY_SIZE(wcd9335_config_regs),
+	.num_config_regs = 4,
+	.set_type_config = regmap_irq_set_type_config_simple,
 };
 
 static int wcd9335_parse_dt(struct wcd9335_codec *wcd)
diff --git a/sound/soc/codecs/wcd934x.c b/sound/soc/codecs/wcd934x.c
index f56907d0942d..28175c746b9a 100644
--- a/sound/soc/codecs/wcd934x.c
+++ b/sound/soc/codecs/wcd934x.c
@@ -1913,8 +1913,8 @@ static int wcd934x_trigger(struct snd_pcm_substream *substream, int cmd,
 	case SNDRV_PCM_TRIGGER_STOP:
 	case SNDRV_PCM_TRIGGER_SUSPEND:
 	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		slim_stream_unprepare(dai_data->sruntime);
 		slim_stream_disable(dai_data->sruntime);
+		slim_stream_unprepare(dai_data->sruntime);
 		break;
 	default:
 		break;
diff --git a/sound/soc/codecs/wcd938x.c b/sound/soc/codecs/wcd938x.c
index 781ae569be29..aca06a4026f3 100644
--- a/sound/soc/codecs/wcd938x.c
+++ b/sound/soc/codecs/wcd938x.c
@@ -1298,7 +1298,6 @@ static struct regmap_irq_chip wcd938x_regmap_irq_chip = {
 	.num_regs = 3,
 	.status_base = WCD938X_DIGITAL_INTR_STATUS_0,
 	.mask_base = WCD938X_DIGITAL_INTR_MASK_0,
-	.type_base = WCD938X_DIGITAL_INTR_LEVEL_0,
 	.ack_base = WCD938X_DIGITAL_INTR_CLEAR_0,
 	.use_ack = 1,
 	.runtime_pm = true,
diff --git a/sound/soc/codecs/wm1250-ev1.c b/sound/soc/codecs/wm1250-ev1.c
index 98343626078b..0064a607ec68 100644
--- a/sound/soc/codecs/wm1250-ev1.c
+++ b/sound/soc/codecs/wm1250-ev1.c
@@ -228,11 +228,9 @@ static int wm1250_ev1_probe(struct i2c_client *i2c)
 	return 0;
 }
 
-static int wm1250_ev1_remove(struct i2c_client *i2c)
+static void wm1250_ev1_remove(struct i2c_client *i2c)
 {
 	wm1250_ev1_free(i2c);
-
-	return 0;
 }
 
 static const struct i2c_device_id wm1250_ev1_i2c_id[] = {
diff --git a/sound/soc/codecs/wm2200.c b/sound/soc/codecs/wm2200.c
index 7b4e162a298c..0a65afa44a59 100644
--- a/sound/soc/codecs/wm2200.c
+++ b/sound/soc/codecs/wm2200.c
@@ -2414,7 +2414,7 @@ err_enable:
 	return ret;
 }
 
-static int wm2200_i2c_remove(struct i2c_client *i2c)
+static void wm2200_i2c_remove(struct i2c_client *i2c)
 {
 	struct wm2200_priv *wm2200 = i2c_get_clientdata(i2c);
 
@@ -2427,8 +2427,6 @@ static int wm2200_i2c_remove(struct i2c_client *i2c)
 		gpio_set_value_cansleep(wm2200->pdata.ldo_ena, 0);
 	regulator_bulk_disable(ARRAY_SIZE(wm2200->core_supplies),
 			       wm2200->core_supplies);
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/sound/soc/codecs/wm5100.c b/sound/soc/codecs/wm5100.c
index 35a85ce6b464..3b09d4a1684f 100644
--- a/sound/soc/codecs/wm5100.c
+++ b/sound/soc/codecs/wm5100.c
@@ -2635,7 +2635,7 @@ err:
 	return ret;
 }
 
-static int wm5100_i2c_remove(struct i2c_client *i2c)
+static void wm5100_i2c_remove(struct i2c_client *i2c)
 {
 	struct wm5100_priv *wm5100 = i2c_get_clientdata(i2c);
 
@@ -2651,8 +2651,6 @@ static int wm5100_i2c_remove(struct i2c_client *i2c)
 		gpio_set_value_cansleep(wm5100->pdata.ldo_ena, 0);
 		gpio_free(wm5100->pdata.ldo_ena);
 	}
-
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/sound/soc/codecs/wm5102.c b/sound/soc/codecs/wm5102.c
index af7d324e3352..c09c9ac51b3e 100644
--- a/sound/soc/codecs/wm5102.c
+++ b/sound/soc/codecs/wm5102.c
@@ -2099,9 +2099,6 @@ static int wm5102_probe(struct platform_device *pdev)
 		regmap_update_bits(arizona->regmap, wm5102_digital_vu[i],
 				   WM5102_DIG_VU, WM5102_DIG_VU);
 
-	pm_runtime_enable(&pdev->dev);
-	pm_runtime_idle(&pdev->dev);
-
 	ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1,
 				  "ADSP2 Compressed IRQ", wm5102_adsp2_irq,
 				  wm5102);
@@ -2134,6 +2131,9 @@ static int wm5102_probe(struct platform_device *pdev)
 		goto err_spk_irqs;
 	}
 
+	pm_runtime_enable(&pdev->dev);
+	pm_runtime_idle(&pdev->dev);
+
 	return ret;
 
 err_spk_irqs:
diff --git a/sound/soc/codecs/wm5110.c b/sound/soc/codecs/wm5110.c
index f3f4a10bf0f7..fc634c995834 100644
--- a/sound/soc/codecs/wm5110.c
+++ b/sound/soc/codecs/wm5110.c
@@ -2457,9 +2457,6 @@ static int wm5110_probe(struct platform_device *pdev)
 		regmap_update_bits(arizona->regmap, wm5110_digital_vu[i],
 				   WM5110_DIG_VU, WM5110_DIG_VU);
 
-	pm_runtime_enable(&pdev->dev);
-	pm_runtime_idle(&pdev->dev);
-
 	ret = arizona_request_irq(arizona, ARIZONA_IRQ_DSP_IRQ1,
 				  "ADSP2 Compressed IRQ", wm5110_adsp2_irq,
 				  wm5110);
@@ -2492,6 +2489,9 @@ static int wm5110_probe(struct platform_device *pdev)
 		goto err_spk_irqs;
 	}
 
+	pm_runtime_enable(&pdev->dev);
+	pm_runtime_idle(&pdev->dev);
+
 	return ret;
 
 err_spk_irqs:
diff --git a/sound/soc/codecs/wm8804-i2c.c b/sound/soc/codecs/wm8804-i2c.c
index 04dc9fb5afb4..3ce1a39d76eb 100644
--- a/sound/soc/codecs/wm8804-i2c.c
+++ b/sound/soc/codecs/wm8804-i2c.c
@@ -25,10 +25,9 @@ static int wm8804_i2c_probe(struct i2c_client *i2c)
 	return wm8804_probe(&i2c->dev, regmap);
 }
 
-static int wm8804_i2c_remove(struct i2c_client *i2c)
+static void wm8804_i2c_remove(struct i2c_client *i2c)
 {
 	wm8804_remove(&i2c->dev);
-	return 0;
 }
 
 static const struct i2c_device_id wm8804_i2c_id[] = {
diff --git a/sound/soc/codecs/wm8900.c b/sound/soc/codecs/wm8900.c
index d6420df3505d..03bbd85ebdf4 100644
--- a/sound/soc/codecs/wm8900.c
+++ b/sound/soc/codecs/wm8900.c
@@ -1282,10 +1282,8 @@ static int wm8900_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static int wm8900_i2c_remove(struct i2c_client *client)
-{
-	return 0;
-}
+static void wm8900_i2c_remove(struct i2c_client *client)
+{}
 
 static const struct i2c_device_id wm8900_i2c_id[] = {
 	{ "wm8900", 0 },
diff --git a/sound/soc/codecs/wm8903.c b/sound/soc/codecs/wm8903.c
index 54e0a7628cd5..41346e5ec5ad 100644
--- a/sound/soc/codecs/wm8903.c
+++ b/sound/soc/codecs/wm8903.c
@@ -2182,7 +2182,7 @@ err:
 	return ret;
 }
 
-static int wm8903_i2c_remove(struct i2c_client *client)
+static void wm8903_i2c_remove(struct i2c_client *client)
 {
 	struct wm8903_priv *wm8903 = i2c_get_clientdata(client);
 
@@ -2191,8 +2191,6 @@ static int wm8903_i2c_remove(struct i2c_client *client)
 	if (client->irq)
 		free_irq(client->irq, wm8903);
 	wm8903_free_gpio(wm8903);
-
-	return 0;
 }
 
 static const struct of_device_id wm8903_of_match[] = {
diff --git a/sound/soc/codecs/wm8960.c b/sound/soc/codecs/wm8960.c
index 37956516d997..0d167238a369 100644
--- a/sound/soc/codecs/wm8960.c
+++ b/sound/soc/codecs/wm8960.c
@@ -1486,10 +1486,8 @@ static int wm8960_i2c_probe(struct i2c_client *i2c)
 	return ret;
 }
 
-static int wm8960_i2c_remove(struct i2c_client *client)
-{
-	return 0;
-}
+static void wm8960_i2c_remove(struct i2c_client *client)
+{}
 
 static const struct i2c_device_id wm8960_i2c_id[] = {
 	{ "wm8960", 0 },
diff --git a/sound/soc/codecs/wm8962.c b/sound/soc/codecs/wm8962.c
index 398c448ea854..81049664387e 100644
--- a/sound/soc/codecs/wm8962.c
+++ b/sound/soc/codecs/wm8962.c
@@ -3778,10 +3778,9 @@ err:
 	return ret;
 }
 
-static int wm8962_i2c_remove(struct i2c_client *client)
+static void wm8962_i2c_remove(struct i2c_client *client)
 {
 	pm_runtime_disable(&client->dev);
-	return 0;
 }
 
 #ifdef CONFIG_PM
diff --git a/sound/soc/codecs/wm8993.c b/sound/soc/codecs/wm8993.c
index 8db98b5a06bf..22a47acbc6d1 100644
--- a/sound/soc/codecs/wm8993.c
+++ b/sound/soc/codecs/wm8993.c
@@ -1722,15 +1722,13 @@ err_enable:
 	return ret;
 }
 
-static int wm8993_i2c_remove(struct i2c_client *i2c)
+static void wm8993_i2c_remove(struct i2c_client *i2c)
 {
 	struct wm8993_priv *wm8993 = i2c_get_clientdata(i2c);
 
 	if (i2c->irq)
 		free_irq(i2c->irq, wm8993);
 	regulator_bulk_disable(ARRAY_SIZE(wm8993->supplies), wm8993->supplies);
-
-	return 0;
 }
 
 static const struct i2c_device_id wm8993_i2c_id[] = {
diff --git a/sound/soc/codecs/wm8996.c b/sound/soc/codecs/wm8996.c
index 17f307a31046..b52ed89d631a 100644
--- a/sound/soc/codecs/wm8996.c
+++ b/sound/soc/codecs/wm8996.c
@@ -3065,7 +3065,7 @@ err:
 	return ret;
 }
 
-static int wm8996_i2c_remove(struct i2c_client *client)
+static void wm8996_i2c_remove(struct i2c_client *client)
 {
 	struct wm8996_priv *wm8996 = i2c_get_clientdata(client);
 
@@ -3074,8 +3074,6 @@ static int wm8996_i2c_remove(struct i2c_client *client)
 		gpio_set_value_cansleep(wm8996->pdata.ldo_ena, 0);
 		gpio_free(wm8996->pdata.ldo_ena);
 	}
-
-	return 0;
 }
 
 static const struct i2c_device_id wm8996_i2c_id[] = {
diff --git a/sound/soc/codecs/wm8997.c b/sound/soc/codecs/wm8997.c
index 210ad662fc26..77136a521605 100644
--- a/sound/soc/codecs/wm8997.c
+++ b/sound/soc/codecs/wm8997.c
@@ -1161,9 +1161,6 @@ static int wm8997_probe(struct platform_device *pdev)
 		regmap_update_bits(arizona->regmap, wm8997_digital_vu[i],
 				   WM8997_DIG_VU, WM8997_DIG_VU);
 
-	pm_runtime_enable(&pdev->dev);
-	pm_runtime_idle(&pdev->dev);
-
 	arizona_init_common(arizona);
 
 	ret = arizona_init_vol_limit(arizona);
@@ -1182,6 +1179,9 @@ static int wm8997_probe(struct platform_device *pdev)
 		goto err_spk_irqs;
 	}
 
+	pm_runtime_enable(&pdev->dev);
+	pm_runtime_idle(&pdev->dev);
+
 	return ret;
 
 err_spk_irqs:
diff --git a/sound/soc/codecs/wm9081.c b/sound/soc/codecs/wm9081.c
index d5151877d0fa..513ec0ba81bb 100644
--- a/sound/soc/codecs/wm9081.c
+++ b/sound/soc/codecs/wm9081.c
@@ -1356,10 +1356,8 @@ static int wm9081_i2c_probe(struct i2c_client *i2c)
 	return 0;
 }
 
-static int wm9081_i2c_remove(struct i2c_client *client)
-{
-	return 0;
-}
+static void wm9081_i2c_remove(struct i2c_client *client)
+{}
 
 static const struct i2c_device_id wm9081_i2c_id[] = {
 	{ "wm9081", 0 },
diff --git a/sound/soc/codecs/wm_adsp.c b/sound/soc/codecs/wm_adsp.c
index cfaa45ede916..8a2e9771bb50 100644
--- a/sound/soc/codecs/wm_adsp.c
+++ b/sound/soc/codecs/wm_adsp.c
@@ -1602,7 +1602,9 @@ static int wm_adsp_buffer_init(struct wm_adsp *dsp)
 	if (list_empty(&dsp->buffer_list)) {
 		/* Fall back to legacy support */
 		ret = wm_adsp_buffer_parse_legacy(dsp);
-		if (ret)
+		if (ret == -ENODEV)
+			adsp_info(dsp, "Legacy support not available\n");
+		else if (ret)
 			adsp_warn(dsp, "Failed to parse legacy: %d\n", ret);
 	}
 
diff --git a/sound/soc/codecs/wsa883x.c b/sound/soc/codecs/wsa883x.c
index 63e1d7aa6137..c7b10bbfba7e 100644
--- a/sound/soc/codecs/wsa883x.c
+++ b/sound/soc/codecs/wsa883x.c
@@ -415,7 +415,6 @@
 
 #define WSA883X_NUM_REGISTERS           (WSA883X_EMEM_63 + 1)
 #define WSA883X_MAX_REGISTER            (WSA883X_NUM_REGISTERS - 1)
-#define WSA883X_PROBE_TIMEOUT 1000
 
 #define WSA883X_VERSION_1_0 0
 #define WSA883X_VERSION_1_1 1
@@ -1409,6 +1408,7 @@ static int wsa883x_probe(struct sdw_slave *pdev,
 	wsa883x->sconfig.type = SDW_STREAM_PDM;
 
 	pdev->prop.sink_ports = GENMASK(WSA883X_MAX_SWR_PORTS, 0);
+	pdev->prop.simple_clk_stop_capable = true;
 	pdev->prop.sink_dpn_prop = wsa_sink_dpn_prop;
 	pdev->prop.scp_int1_mask = SDW_SCP_INT1_BUS_CLASH | SDW_SCP_INT1_PARITY;
 	gpiod_direction_output(wsa883x->sd_n, 1);
@@ -1440,43 +1440,17 @@ err:
 static int __maybe_unused wsa883x_runtime_suspend(struct device *dev)
 {
 	struct regmap *regmap = dev_get_regmap(dev, NULL);
-	struct wsa883x_priv *wsa883x = dev_get_drvdata(dev);
-
-	gpiod_direction_output(wsa883x->sd_n, 0);
 
 	regcache_cache_only(regmap, true);
 	regcache_mark_dirty(regmap);
 
-	regulator_disable(wsa883x->vdd);
 	return 0;
 }
 
 static int __maybe_unused wsa883x_runtime_resume(struct device *dev)
 {
-	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct regmap *regmap = dev_get_regmap(dev, NULL);
-	struct wsa883x_priv *wsa883x = dev_get_drvdata(dev);
-	unsigned long time;
-	int ret;
-
-	ret = regulator_enable(wsa883x->vdd);
-	if (ret) {
-		dev_err(dev, "Failed to enable vdd regulator (%d)\n", ret);
-		return ret;
-	}
-
-	gpiod_direction_output(wsa883x->sd_n, 1);
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-					   msecs_to_jiffies(WSA883X_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(dev, "Initialization not complete, timed out\n");
-		gpiod_direction_output(wsa883x->sd_n, 0);
-		regulator_disable(wsa883x->vdd);
-		return -ETIMEDOUT;
-	}
 
-	usleep_range(20000, 20010);
 	regcache_cache_only(regmap, false);
 	regcache_sync(regmap);
 
diff --git a/sound/soc/fsl/eukrea-tlv320.c b/sound/soc/fsl/eukrea-tlv320.c
index 8b61582753c8..9af4c4a35eb1 100644
--- a/sound/soc/fsl/eukrea-tlv320.c
+++ b/sound/soc/fsl/eukrea-tlv320.c
@@ -86,7 +86,7 @@ static int eukrea_tlv320_probe(struct platform_device *pdev)
 	int ret;
 	int int_port = 0, ext_port;
 	struct device_node *np = pdev->dev.of_node;
-	struct device_node *ssi_np = NULL, *codec_np = NULL;
+	struct device_node *ssi_np = NULL, *codec_np = NULL, *tmp_np = NULL;
 
 	eukrea_tlv320.dev = &pdev->dev;
 	if (np) {
@@ -143,7 +143,7 @@ static int eukrea_tlv320_probe(struct platform_device *pdev)
 	}
 
 	if (machine_is_eukrea_cpuimx27() ||
-	    of_find_compatible_node(NULL, NULL, "fsl,imx21-audmux")) {
+	    (tmp_np = of_find_compatible_node(NULL, NULL, "fsl,imx21-audmux"))) {
 		imx_audmux_v1_configure_port(MX27_AUDMUX_HPCR1_SSI0,
 			IMX_AUDMUX_V1_PCR_SYN |
 			IMX_AUDMUX_V1_PCR_TFSDIR |
@@ -158,10 +158,11 @@ static int eukrea_tlv320_probe(struct platform_device *pdev)
 			IMX_AUDMUX_V1_PCR_SYN |
 			IMX_AUDMUX_V1_PCR_RXDSEL(MX27_AUDMUX_HPCR1_SSI0)
 		);
+		of_node_put(tmp_np);
 	} else if (machine_is_eukrea_cpuimx25sd() ||
 		   machine_is_eukrea_cpuimx35sd() ||
 		   machine_is_eukrea_cpuimx51sd() ||
-		   of_find_compatible_node(NULL, NULL, "fsl,imx31-audmux")) {
+		   (tmp_np = of_find_compatible_node(NULL, NULL, "fsl,imx31-audmux"))) {
 		if (!np)
 			ext_port = machine_is_eukrea_cpuimx25sd() ?
 				4 : 3;
@@ -178,6 +179,7 @@ static int eukrea_tlv320_probe(struct platform_device *pdev)
 			IMX_AUDMUX_V2_PTCR_SYN,
 			IMX_AUDMUX_V2_PDCR_RXDSEL(int_port)
 		);
+		of_node_put(tmp_np);
 	} else {
 		if (np) {
 			/* The eukrea,asoc-tlv320 driver was explicitly
diff --git a/sound/soc/fsl/fsl_asrc.c b/sound/soc/fsl/fsl_asrc.c
index aa5edf32d988..936aef5d2767 100644
--- a/sound/soc/fsl/fsl_asrc.c
+++ b/sound/soc/fsl/fsl_asrc.c
@@ -20,6 +20,7 @@
 
 #define IDEAL_RATIO_DECIMAL_DEPTH 26
 #define DIVIDER_NUM  64
+#define INIT_RETRY_NUM 50
 
 #define pair_err(fmt, ...) \
 	dev_err(&asrc->pdev->dev, "Pair %c: " fmt, 'A' + index, ##__VA_ARGS__)
@@ -27,6 +28,9 @@
 #define pair_dbg(fmt, ...) \
 	dev_dbg(&asrc->pdev->dev, "Pair %c: " fmt, 'A' + index, ##__VA_ARGS__)
 
+#define pair_warn(fmt, ...) \
+	dev_warn(&asrc->pdev->dev, "Pair %c: " fmt, 'A' + index, ##__VA_ARGS__)
+
 /* Corresponding to process_option */
 static unsigned int supported_asrc_rate[] = {
 	5512, 8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000,
@@ -579,7 +583,7 @@ static void fsl_asrc_start_pair(struct fsl_asrc_pair *pair)
 {
 	struct fsl_asrc *asrc = pair->asrc;
 	enum asrc_pair_index index = pair->index;
-	int reg, retry = 10, i;
+	int reg, retry = INIT_RETRY_NUM, i;
 
 	/* Enable the current pair */
 	regmap_update_bits(asrc->regmap, REG_ASRCTR,
@@ -592,6 +596,10 @@ static void fsl_asrc_start_pair(struct fsl_asrc_pair *pair)
 		reg &= ASRCFG_INIRQi_MASK(index);
 	} while (!reg && --retry);
 
+	/* NOTE: Doesn't treat initialization timeout as an error */
+	if (!retry)
+		pair_warn("initialization isn't finished\n");
+
 	/* Make the input fifo to ASRC STALL level */
 	regmap_read(asrc->regmap, REG_ASRCNCR, &reg);
 	for (i = 0; i < pair->channels * 4; i++)
@@ -1257,6 +1265,7 @@ static int fsl_asrc_runtime_resume(struct device *dev)
 {
 	struct fsl_asrc *asrc = dev_get_drvdata(dev);
 	struct fsl_asrc_priv *asrc_priv = asrc->private;
+	int reg, retry = INIT_RETRY_NUM;
 	int i, ret;
 	u32 asrctr;
 
@@ -1295,6 +1304,24 @@ static int fsl_asrc_runtime_resume(struct device *dev)
 	regmap_update_bits(asrc->regmap, REG_ASRCTR,
 			   ASRCTR_ASRCEi_ALL_MASK, asrctr);
 
+	/* Wait for status of initialization for all enabled pairs */
+	do {
+		udelay(5);
+		regmap_read(asrc->regmap, REG_ASRCFG, &reg);
+		reg = (reg >> ASRCFG_INIRQi_SHIFT(0)) & 0x7;
+	} while ((reg != ((asrctr >> ASRCTR_ASRCEi_SHIFT(0)) & 0x7)) && --retry);
+
+	/*
+	 * NOTE: Doesn't treat initialization timeout as an error
+	 * Some of the pairs may success, then still can continue.
+	 */
+	if (!retry) {
+		for (i = ASRC_PAIR_A; i < ASRC_PAIR_MAX_NUM; i++) {
+			if ((asrctr & ASRCTR_ASRCEi_MASK(i)) && !(reg & (1 << i)))
+				dev_warn(dev, "Pair %c initialization isn't finished\n", 'A' + i);
+		}
+	}
+
 	return 0;
 
 disable_asrck_clk:
diff --git a/sound/soc/fsl/fsl_asrc_dma.c b/sound/soc/fsl/fsl_asrc_dma.c
index 12ddf2320f2d..3b81a465814a 100644
--- a/sound/soc/fsl/fsl_asrc_dma.c
+++ b/sound/soc/fsl/fsl_asrc_dma.c
@@ -139,7 +139,7 @@ static int fsl_asrc_dma_hw_params(struct snd_soc_component *component,
 	struct dma_chan *tmp_chan = NULL, *be_chan = NULL;
 	struct snd_soc_component *component_be = NULL;
 	struct fsl_asrc *asrc = pair->asrc;
-	struct dma_slave_config config_fe, config_be;
+	struct dma_slave_config config_fe = {}, config_be = {};
 	struct sdma_peripheral_config audio_config;
 	enum asrc_pair_index index = pair->index;
 	struct device *dev = component->dev;
@@ -183,7 +183,6 @@ static int fsl_asrc_dma_hw_params(struct snd_soc_component *component,
 		return -EINVAL;
 	}
 
-	memset(&config_fe, 0, sizeof(config_fe));
 	ret = snd_dmaengine_pcm_prepare_slave_config(substream, params, &config_fe);
 	if (ret) {
 		dev_err(dev, "failed to prepare DMA config for Front-End\n");
diff --git a/sound/soc/fsl/fsl_audmix.c b/sound/soc/fsl/fsl_audmix.c
index 43857b7a81c9..672148dd4b23 100644
--- a/sound/soc/fsl/fsl_audmix.c
+++ b/sound/soc/fsl/fsl_audmix.c
@@ -199,18 +199,10 @@ static int fsl_audmix_put_out_src(struct snd_kcontrol *kcontrol,
 
 static const struct snd_kcontrol_new fsl_audmix_snd_controls[] = {
 	/* FSL_AUDMIX_CTR controls */
-	{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER,
-		.name = "Mixing Clock Source",
-		.info = snd_soc_info_enum_double,
-		.access = SNDRV_CTL_ELEM_ACCESS_WRITE,
-		.put = fsl_audmix_put_mix_clk_src,
-		.private_value = (unsigned long)&fsl_audmix_enum[0] },
-	{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER,
-		.name = "Output Source",
-		.info = snd_soc_info_enum_double,
-		.access = SNDRV_CTL_ELEM_ACCESS_WRITE,
-		.put = fsl_audmix_put_out_src,
-		.private_value = (unsigned long)&fsl_audmix_enum[1] },
+	SOC_ENUM_EXT("Mixing Clock Source", fsl_audmix_enum[0],
+		     snd_soc_get_enum_double, fsl_audmix_put_mix_clk_src),
+	SOC_ENUM_EXT("Output Source", fsl_audmix_enum[1],
+		     snd_soc_get_enum_double, fsl_audmix_put_out_src),
 	SOC_ENUM("Output Width", fsl_audmix_enum[2]),
 	SOC_ENUM("Frame Rate Diff Error", fsl_audmix_enum[3]),
 	SOC_ENUM("Clock Freq Diff Error", fsl_audmix_enum[4]),
diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c
index d430eece1d6b..81f89f6767a2 100644
--- a/sound/soc/fsl/fsl_sai.c
+++ b/sound/soc/fsl/fsl_sai.c
@@ -114,11 +114,8 @@ static irqreturn_t fsl_sai_isr(int irq, void *devid)
 	if (flags & FSL_SAI_CSR_SEF)
 		dev_dbg(dev, "isr: Tx Frame sync error detected\n");
 
-	if (flags & FSL_SAI_CSR_FEF) {
+	if (flags & FSL_SAI_CSR_FEF)
 		dev_dbg(dev, "isr: Transmit underrun detected\n");
-		/* FIFO reset for safety */
-		xcsr |= FSL_SAI_CSR_FR;
-	}
 
 	if (flags & FSL_SAI_CSR_FWF)
 		dev_dbg(dev, "isr: Enabled transmit FIFO is empty\n");
@@ -148,11 +145,8 @@ irq_rx:
 	if (flags & FSL_SAI_CSR_SEF)
 		dev_dbg(dev, "isr: Rx Frame sync error detected\n");
 
-	if (flags & FSL_SAI_CSR_FEF) {
+	if (flags & FSL_SAI_CSR_FEF)
 		dev_dbg(dev, "isr: Receive overflow detected\n");
-		/* FIFO reset for safety */
-		xcsr |= FSL_SAI_CSR_FR;
-	}
 
 	if (flags & FSL_SAI_CSR_FWF)
 		dev_dbg(dev, "isr: Enabled receive FIFO is full\n");
@@ -533,14 +527,17 @@ static int fsl_sai_hw_params(struct snd_pcm_substream *substream,
 	u32 slot_width = word_width;
 	int adir = tx ? RX : TX;
 	u32 pins, bclk;
+	u32 watermark;
 	int ret, i;
 
-	if (sai->slots)
-		slots = sai->slots;
-
 	if (sai->slot_width)
 		slot_width = sai->slot_width;
 
+	if (sai->slots)
+		slots = sai->slots;
+	else if (sai->bclk_ratio)
+		slots = sai->bclk_ratio / slot_width;
+
 	pins = DIV_ROUND_UP(channels, slots);
 
 	/*
@@ -625,7 +622,15 @@ static int fsl_sai_hw_params(struct snd_pcm_substream *substream,
 				   FSL_SAI_CR5_FBT_MASK, val_cr5);
 	}
 
-	if (hweight8(dl_cfg[dl_cfg_idx].mask[tx]) <= 1)
+	/*
+	 * Combine mode has limation:
+	 * - Can't used for singel dataline/FIFO case except the FIFO0
+	 * - Can't used for multi dataline/FIFO case except the enabled FIFOs
+	 *   are successive and start from FIFO0
+	 *
+	 * So for common usage, all multi fifo case disable the combine mode.
+	 */
+	if (hweight8(dl_cfg[dl_cfg_idx].mask[tx]) <= 1 || sai->is_multi_fifo_dma)
 		regmap_update_bits(sai->regmap, FSL_SAI_xCR4(tx, ofs),
 				   FSL_SAI_CR4_FCOMB_MASK, 0);
 	else
@@ -636,6 +641,26 @@ static int fsl_sai_hw_params(struct snd_pcm_substream *substream,
 	dma_params->addr = sai->res->start + FSL_SAI_xDR0(tx) +
 			   dl_cfg[dl_cfg_idx].start_off[tx] * 0x4;
 
+	if (sai->is_multi_fifo_dma) {
+		sai->audio_config[tx].words_per_fifo = min(slots, channels);
+		if (tx) {
+			sai->audio_config[tx].n_fifos_dst = pins;
+			sai->audio_config[tx].stride_fifos_dst = dl_cfg[dl_cfg_idx].next_off[tx];
+		} else {
+			sai->audio_config[tx].n_fifos_src = pins;
+			sai->audio_config[tx].stride_fifos_src = dl_cfg[dl_cfg_idx].next_off[tx];
+		}
+		dma_params->maxburst = sai->audio_config[tx].words_per_fifo * pins;
+		dma_params->peripheral_config = &sai->audio_config[tx];
+		dma_params->peripheral_size = sizeof(sai->audio_config[tx]);
+
+		watermark = tx ? (sai->soc_data->fifo_depth - dma_params->maxburst) :
+				 (dma_params->maxburst - 1);
+		regmap_update_bits(sai->regmap, FSL_SAI_xCR1(tx, ofs),
+				   FSL_SAI_CR1_RFW_MASK(sai->soc_data->fifo_depth),
+				   watermark);
+	}
+
 	/* Find a proper tcre setting */
 	for (i = 0; i < sai->soc_data->pins; i++) {
 		trce_mask = (1 << (i + 1)) - 1;
@@ -1263,6 +1288,7 @@ static int fsl_sai_probe(struct platform_device *pdev)
 	char tmp[8];
 	int irq, ret, i;
 	int index;
+	u32 dmas[4];
 
 	sai = devm_kzalloc(dev, sizeof(*sai), GFP_KERNEL);
 	if (!sai)
@@ -1319,6 +1345,11 @@ static int fsl_sai_probe(struct platform_device *pdev)
 	fsl_asoc_get_pll_clocks(&pdev->dev, &sai->pll8k_clk,
 				&sai->pll11k_clk);
 
+	/* Use Multi FIFO mode depending on the support from SDMA script */
+	ret = of_property_read_u32_array(np, "dmas", dmas, 4);
+	if (!sai->soc_data->use_edma && !ret && dmas[2] == IMX_DMATYPE_MULTI_SAI)
+		sai->is_multi_fifo_dma = true;
+
 	/* read dataline mask for rx and tx*/
 	ret = fsl_sai_read_dlcfg(sai);
 	if (ret < 0) {
diff --git a/sound/soc/fsl/fsl_sai.h b/sound/soc/fsl/fsl_sai.h
index 17956b5731dc..697f6690068c 100644
--- a/sound/soc/fsl/fsl_sai.h
+++ b/sound/soc/fsl/fsl_sai.h
@@ -6,6 +6,7 @@
 #ifndef __FSL_SAI_H
 #define __FSL_SAI_H
 
+#include <linux/dma/imx-dma.h>
 #include <sound/dmaengine_pcm.h>
 
 #define FSL_SAI_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\
@@ -281,6 +282,7 @@ struct fsl_sai {
 	bool is_lsb_first;
 	bool is_dsp_mode;
 	bool is_pdm_mode;
+	bool is_multi_fifo_dma;
 	bool synchronous[2];
 	struct fsl_sai_dl_cfg *dl_cfg;
 	unsigned int dl_cfg_cnt;
@@ -300,6 +302,7 @@ struct fsl_sai {
 	struct pm_qos_request pm_qos_req;
 	struct pinctrl *pinctrl;
 	struct pinctrl_state *pins_state;
+	struct sdma_peripheral_config audio_config[2];
 };
 
 #define TX 1
diff --git a/sound/soc/fsl/fsl_spdif.c b/sound/soc/fsl/fsl_spdif.c
index 7fc1c96929bb..275aba8e0c46 100644
--- a/sound/soc/fsl/fsl_spdif.c
+++ b/sound/soc/fsl/fsl_spdif.c
@@ -44,6 +44,8 @@ static u8 srpc_dpll_locked[] = { 0x0, 0x1, 0x2, 0x3, 0x4, 0xa, 0xb };
 
 #define DEFAULT_RXCLK_SRC	1
 
+#define RX_SAMPLE_RATE_KCONTROL "RX Sample Rate"
+
 /**
  * struct fsl_spdif_soc_data: soc specific data
  *
@@ -98,6 +100,8 @@ struct spdif_mixer_control {
  * @soc: SPDIF soc data
  * @fsl_spdif_control: SPDIF control data
  * @cpu_dai_drv: cpu dai driver
+ * @snd_card: sound card pointer
+ * @rxrate_kcontrol: kcontrol for RX Sample Rate
  * @pdev: platform device pointer
  * @regmap: regmap handler
  * @dpll_locked: dpll lock flag
@@ -122,6 +126,8 @@ struct fsl_spdif_priv {
 	const struct fsl_spdif_soc_data *soc;
 	struct spdif_mixer_control fsl_spdif_control;
 	struct snd_soc_dai_driver cpu_dai_drv;
+	struct snd_card *snd_card;
+	struct snd_kcontrol *rxrate_kcontrol;
 	struct platform_device *pdev;
 	struct regmap *regmap;
 	bool dpll_locked;
@@ -226,6 +232,12 @@ static void spdif_irq_dpll_lock(struct fsl_spdif_priv *spdif_priv)
 			locked ? "locked" : "loss lock");
 
 	spdif_priv->dpll_locked = locked ? true : false;
+
+	if (spdif_priv->snd_card && spdif_priv->rxrate_kcontrol) {
+		snd_ctl_notify(spdif_priv->snd_card,
+			       SNDRV_CTL_EVENT_MASK_VALUE,
+			       &spdif_priv->rxrate_kcontrol->id);
+	}
 }
 
 /* Receiver found illegal symbol interrupt handler */
@@ -1197,7 +1209,7 @@ static struct snd_kcontrol_new fsl_spdif_ctrls[] = {
 	/* DPLL lock info get controller */
 	{
 		.iface = SNDRV_CTL_ELEM_IFACE_PCM,
-		.name = "RX Sample Rate",
+		.name = RX_SAMPLE_RATE_KCONTROL,
 		.access = SNDRV_CTL_ELEM_ACCESS_READ |
 			SNDRV_CTL_ELEM_ACCESS_VOLATILE,
 		.info = fsl_spdif_rxrate_info,
@@ -1251,6 +1263,13 @@ static int fsl_spdif_dai_probe(struct snd_soc_dai *dai)
 		snd_soc_add_dai_controls(dai, fsl_spdif_ctrls_rcm,
 					 ARRAY_SIZE(fsl_spdif_ctrls_rcm));
 
+	spdif_private->snd_card = dai->component->card->snd_card;
+	spdif_private->rxrate_kcontrol = snd_soc_card_get_kcontrol(dai->component->card,
+								   RX_SAMPLE_RATE_KCONTROL);
+	if (!spdif_private->rxrate_kcontrol)
+		dev_err(&spdif_private->pdev->dev, "failed to get %s kcontrol\n",
+			RX_SAMPLE_RATE_KCONTROL);
+
 	/*Clear the val bit for Tx*/
 	regmap_update_bits(spdif_private->regmap, REG_SPDIF_SCR,
 			   SCR_VAL_MASK, SCR_VAL_CLEAR);
diff --git a/sound/soc/fsl/fsl_utils.c b/sound/soc/fsl/fsl_utils.c
index d0fc430f7033..a5ab27c2f711 100644
--- a/sound/soc/fsl/fsl_utils.c
+++ b/sound/soc/fsl/fsl_utils.c
@@ -124,7 +124,7 @@ void fsl_asoc_reparent_pll_clocks(struct device *dev, struct clk *clk,
 {
 	struct clk *p, *pll = NULL, *npll = NULL;
 	bool reparent = false;
-	int ret = 0;
+	int ret;
 
 	if (!clk || !pll8k_clk || !pll11k_clk)
 		return;
diff --git a/sound/soc/fsl/imx-card.c b/sound/soc/fsl/imx-card.c
index 14be29530fb5..3f128ced4180 100644
--- a/sound/soc/fsl/imx-card.c
+++ b/sound/soc/fsl/imx-card.c
@@ -698,6 +698,10 @@ static int imx_card_parse_of(struct imx_card_data *data)
 		of_node_put(cpu);
 		of_node_put(codec);
 		of_node_put(platform);
+
+		cpu = NULL;
+		codec = NULL;
+		platform = NULL;
 	}
 
 	return 0;
diff --git a/sound/soc/fsl/imx-pcm.h b/sound/soc/fsl/imx-pcm.h
index 06b25f4b26b6..ac5f57c3cc55 100644
--- a/sound/soc/fsl/imx-pcm.h
+++ b/sound/soc/fsl/imx-pcm.h
@@ -18,15 +18,6 @@
 
 #define IMX_DEFAULT_DMABUF_SIZE	(64 * 1024)
 
-static inline void
-imx_pcm_dma_params_init_data(struct imx_dma_data *dma_data,
-	int dma, enum sdma_peripheral_type peripheral_type)
-{
-	dma_data->dma_request = dma;
-	dma_data->priority = DMA_PRIO_HIGH;
-	dma_data->peripheral_type = peripheral_type;
-}
-
 struct imx_pcm_fiq_params {
 	int irq;
 	void __iomem *base;
diff --git a/sound/soc/fsl/imx-rpmsg.c b/sound/soc/fsl/imx-rpmsg.c
index 2e117311e582..4d99f4858a14 100644
--- a/sound/soc/fsl/imx-rpmsg.c
+++ b/sound/soc/fsl/imx-rpmsg.c
@@ -19,6 +19,7 @@
 struct imx_rpmsg {
 	struct snd_soc_dai_link dai;
 	struct snd_soc_card card;
+	unsigned long sysclk;
 };
 
 static const struct snd_soc_dapm_widget imx_rpmsg_dapm_widgets[] = {
@@ -28,6 +29,27 @@ static const struct snd_soc_dapm_widget imx_rpmsg_dapm_widgets[] = {
 	SND_SOC_DAPM_MIC("Main MIC", NULL),
 };
 
+static int imx_rpmsg_late_probe(struct snd_soc_card *card)
+{
+	struct imx_rpmsg *data = snd_soc_card_get_drvdata(card);
+	struct snd_soc_pcm_runtime *rtd = list_first_entry(&card->rtd_list,
+							   struct snd_soc_pcm_runtime, list);
+	struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0);
+	struct device *dev = card->dev;
+	int ret;
+
+	if (!data->sysclk)
+		return 0;
+
+	ret = snd_soc_dai_set_sysclk(codec_dai, 0, data->sysclk, SND_SOC_CLOCK_IN);
+	if (ret && ret != -ENOTSUPP) {
+		dev_err(dev, "failed to set sysclk in %s\n", __func__);
+		return ret;
+	}
+
+	return 0;
+}
+
 static int imx_rpmsg_probe(struct platform_device *pdev)
 {
 	struct snd_soc_dai_link_component *dlc;
@@ -72,12 +94,18 @@ static int imx_rpmsg_probe(struct platform_device *pdev)
 		data->dai.codecs->dai_name = "snd-soc-dummy-dai";
 		data->dai.codecs->name = "snd-soc-dummy";
 	} else {
+		struct clk *clk;
+
 		data->dai.codecs->of_node = args.np;
 		ret = snd_soc_get_dai_name(&args, &data->dai.codecs->dai_name);
 		if (ret) {
 			dev_err(&pdev->dev, "Unable to get codec_dai_name\n");
 			goto fail;
 		}
+
+		clk = devm_get_clk_from_child(&pdev->dev, args.np, NULL);
+		if (!IS_ERR(clk))
+			data->sysclk = clk_get_rate(clk);
 	}
 
 	data->dai.cpus->dai_name = dev_name(&rpmsg_pdev->dev);
@@ -103,6 +131,7 @@ static int imx_rpmsg_probe(struct platform_device *pdev)
 	data->card.owner = THIS_MODULE;
 	data->card.dapm_widgets = imx_rpmsg_dapm_widgets;
 	data->card.num_dapm_widgets = ARRAY_SIZE(imx_rpmsg_dapm_widgets);
+	data->card.late_probe = imx_rpmsg_late_probe;
 	/*
 	 * Inoder to use common api to get card name and audio routing.
 	 * Use parent of_node for this device, revert it after finishing using
diff --git a/sound/soc/generic/simple-card-utils.c b/sound/soc/generic/simple-card-utils.c
index 4a29e314fa95..bef16833c487 100644
--- a/sound/soc/generic/simple-card-utils.c
+++ b/sound/soc/generic/simple-card-utils.c
@@ -9,12 +9,38 @@
 #include <linux/gpio/consumer.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/of_graph.h>
 #include <sound/jack.h>
 #include <sound/pcm_params.h>
 #include <sound/simple_card_utils.h>
 
+static void asoc_simple_fixup_sample_fmt(struct asoc_simple_data *data,
+					 struct snd_pcm_hw_params *params)
+{
+	int i;
+	struct snd_mask *mask = hw_param_mask(params,
+					      SNDRV_PCM_HW_PARAM_FORMAT);
+	struct {
+		char *fmt;
+		u32 val;
+	} of_sample_fmt_table[] = {
+		{ "s8",		SNDRV_PCM_FORMAT_S8},
+		{ "s16_le",	SNDRV_PCM_FORMAT_S16_LE},
+		{ "s24_le",	SNDRV_PCM_FORMAT_S24_LE},
+		{ "s24_3le",	SNDRV_PCM_FORMAT_S24_3LE},
+		{ "s32_le",	SNDRV_PCM_FORMAT_S32_LE},
+	};
+
+	for (i = 0; i < ARRAY_SIZE(of_sample_fmt_table); i++) {
+		if (!strcmp(data->convert_sample_format,
+			    of_sample_fmt_table[i].fmt)) {
+			snd_mask_none(mask);
+			snd_mask_set(mask, of_sample_fmt_table[i].val);
+			break;
+		}
+	}
+}
+
 void asoc_simple_convert_fixup(struct asoc_simple_data *data,
 			       struct snd_pcm_hw_params *params)
 {
@@ -30,6 +56,9 @@ void asoc_simple_convert_fixup(struct asoc_simple_data *data,
 	if (data->convert_channels)
 		channels->min =
 		channels->max = data->convert_channels;
+
+	if (data->convert_sample_format)
+		asoc_simple_fixup_sample_fmt(data, params);
 }
 EXPORT_SYMBOL_GPL(asoc_simple_convert_fixup);
 
@@ -49,6 +78,10 @@ void asoc_simple_parse_convert(struct device_node *np,
 	/* channels transfer */
 	snprintf(prop, sizeof(prop), "%s%s", prefix, "convert-channels");
 	of_property_read_u32(np, prop, &data->convert_channels);
+
+	/* convert sample format */
+	snprintf(prop, sizeof(prop), "%s%s", prefix, "convert-sample-format");
+	of_property_read_string(np, prop, &data->convert_sample_format);
 }
 EXPORT_SYMBOL_GPL(asoc_simple_parse_convert);
 
@@ -695,12 +728,12 @@ int asoc_simple_init_jack(struct snd_soc_card *card,
 			  char *pin)
 {
 	struct device *dev = card->dev;
-	enum of_gpio_flags flags;
+	struct gpio_desc *desc;
 	char prop[128];
 	char *pin_name;
 	char *gpio_name;
 	int mask;
-	int det;
+	int error;
 
 	if (!prefix)
 		prefix = "";
@@ -708,36 +741,39 @@ int asoc_simple_init_jack(struct snd_soc_card *card,
 	sjack->gpio.gpio = -ENOENT;
 
 	if (is_hp) {
-		snprintf(prop, sizeof(prop), "%shp-det-gpio", prefix);
+		snprintf(prop, sizeof(prop), "%shp-det", prefix);
 		pin_name	= pin ? pin : "Headphones";
 		gpio_name	= "Headphone detection";
 		mask		= SND_JACK_HEADPHONE;
 	} else {
-		snprintf(prop, sizeof(prop), "%smic-det-gpio", prefix);
+		snprintf(prop, sizeof(prop), "%smic-det", prefix);
 		pin_name	= pin ? pin : "Mic Jack";
 		gpio_name	= "Mic detection";
 		mask		= SND_JACK_MICROPHONE;
 	}
 
-	det = of_get_named_gpio_flags(dev->of_node, prop, 0, &flags);
-	if (det == -EPROBE_DEFER)
-		return -EPROBE_DEFER;
+	desc = gpiod_get_optional(dev, prop, GPIOD_IN);
+	error = PTR_ERR_OR_ZERO(desc);
+	if (error)
+		return error;
+
+	if (desc) {
+		error = gpiod_set_consumer_name(desc, gpio_name);
+		if (error)
+			return error;
 
-	if (gpio_is_valid(det)) {
 		sjack->pin.pin		= pin_name;
 		sjack->pin.mask		= mask;
 
 		sjack->gpio.name	= gpio_name;
 		sjack->gpio.report	= mask;
-		sjack->gpio.gpio	= det;
-		sjack->gpio.invert	= !!(flags & OF_GPIO_ACTIVE_LOW);
+		sjack->gpio.desc	= desc;
 		sjack->gpio.debounce_time = 150;
 
 		snd_soc_card_jack_new_pins(card, pin_name, mask, &sjack->jack,
 					   &sjack->pin, 1);
 
-		snd_soc_jack_add_gpios(&sjack->jack, 1,
-				       &sjack->gpio);
+		snd_soc_jack_add_gpios(&sjack->jack, 1, &sjack->gpio);
 	}
 
 	return 0;
diff --git a/sound/soc/intel/Kconfig b/sound/soc/intel/Kconfig
index ded903f95b67..d2ca710ac3fa 100644
--- a/sound/soc/intel/Kconfig
+++ b/sound/soc/intel/Kconfig
@@ -23,7 +23,7 @@ config SND_SOC_INTEL_CATPT
 	depends on ACPI || COMPILE_TEST
 	depends on DMADEVICES && SND_DMA_SGBUF
 	select DW_DMAC_CORE
-	select SND_SOC_ACPI_INTEL_MATCH
+	select SND_SOC_ACPI if ACPI
 	select WANT_DEV_COREDUMP
 	select SND_INTEL_DSP_CONFIG
 	help
diff --git a/sound/soc/intel/atom/sst-mfld-platform-pcm.c b/sound/soc/intel/atom/sst-mfld-platform-pcm.c
index a56dd48c045f..c75616a5fd0a 100644
--- a/sound/soc/intel/atom/sst-mfld-platform-pcm.c
+++ b/sound/soc/intel/atom/sst-mfld-platform-pcm.c
@@ -676,10 +676,9 @@ static int sst_soc_pcm_new(struct snd_soc_component *component,
 
 	if (dai->driver->playback.channels_min ||
 			dai->driver->capture.channels_min) {
-		snd_pcm_set_managed_buffer_all(pcm,
-			SNDRV_DMA_TYPE_CONTINUOUS,
-			snd_dma_continuous_data(GFP_DMA),
-			SST_MIN_BUFFER, SST_MAX_BUFFER);
+		snd_pcm_set_managed_buffer_all(pcm, SNDRV_DMA_TYPE_DEV,
+					       pcm->card->dev,
+					       SST_MIN_BUFFER, SST_MAX_BUFFER);
 	}
 	return 0;
 }
diff --git a/sound/soc/intel/atom/sst/sst.c b/sound/soc/intel/atom/sst/sst.c
index 160b50f479fb..a0d29510d2bc 100644
--- a/sound/soc/intel/atom/sst/sst.c
+++ b/sound/soc/intel/atom/sst/sst.c
@@ -242,11 +242,11 @@ static ssize_t firmware_version_show(struct device *dev,
 
 	if (ctx->fw_version.type == 0 && ctx->fw_version.major == 0 &&
 	    ctx->fw_version.minor == 0 && ctx->fw_version.build == 0)
-		return sprintf(buf, "FW not yet loaded\n");
+		return sysfs_emit(buf, "FW not yet loaded\n");
 	else
-		return sprintf(buf, "v%02x.%02x.%02x.%02x\n",
-			       ctx->fw_version.type, ctx->fw_version.major,
-			       ctx->fw_version.minor, ctx->fw_version.build);
+		return sysfs_emit(buf, "v%02x.%02x.%02x.%02x\n",
+				  ctx->fw_version.type, ctx->fw_version.major,
+				  ctx->fw_version.minor, ctx->fw_version.build);
 
 }
 
diff --git a/sound/soc/intel/avs/boards/hdaudio.c b/sound/soc/intel/avs/boards/hdaudio.c
index d2fc41d39448..073663ba140d 100644
--- a/sound/soc/intel/avs/boards/hdaudio.c
+++ b/sound/soc/intel/avs/boards/hdaudio.c
@@ -42,6 +42,7 @@ static int avs_create_dai_links(struct device *dev, struct hda_codec *codec, int
 		dl[i].dpcm_capture = 1;
 		dl[i].platforms = platform;
 		dl[i].num_platforms = 1;
+		dl[i].ignore_pmdown_time = 1;
 
 		dl[i].codecs = devm_kzalloc(dev, sizeof(*dl->codecs), GFP_KERNEL);
 		dl[i].cpus = devm_kzalloc(dev, sizeof(*dl->cpus), GFP_KERNEL);
diff --git a/sound/soc/intel/avs/core.c b/sound/soc/intel/avs/core.c
index c50c20fd681a..bb0719c58ca4 100644
--- a/sound/soc/intel/avs/core.c
+++ b/sound/soc/intel/avs/core.c
@@ -466,7 +466,7 @@ static int avs_pci_probe(struct pci_dev *pci, const struct pci_device_id *id)
 
 err_acquire_irq:
 	snd_hdac_bus_free_stream_pages(bus);
-	snd_hdac_stream_free_all(bus);
+	snd_hdac_ext_stream_free_all(bus);
 err_init_streams:
 	iounmap(adev->dsp_ba);
 err_remap_bar4:
@@ -502,7 +502,7 @@ static void avs_pci_remove(struct pci_dev *pci)
 		snd_hda_codec_unregister(hdac_to_hda_codec(hdev));
 
 	snd_hdac_bus_free_stream_pages(bus);
-	snd_hdac_stream_free_all(bus);
+	snd_hdac_ext_stream_free_all(bus);
 	/* reverse ml_capabilities */
 	snd_hdac_link_free_all(bus);
 	snd_hdac_ext_bus_exit(bus);
diff --git a/sound/soc/intel/boards/Makefile b/sound/soc/intel/boards/Makefile
index eea1e26acfda..53458e748191 100644
--- a/sound/soc/intel/boards/Makefile
+++ b/sound/soc/intel/boards/Makefile
@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
-snd-soc-sst-haswell-objs := hsw_rt5640.o
+snd-soc-hsw-rt5640-objs := hsw_rt5640.o
 snd-soc-sst-bdw-rt5650-mach-objs := bdw-rt5650.o
 snd-soc-sst-bdw-rt5677-mach-objs := bdw-rt5677.o
-snd-soc-sst-broadwell-objs := bdw_rt286.o
+snd-soc-bdw-rt286-objs := bdw_rt286.o
 snd-soc-sst-bxt-da7219_max98357a-objs := bxt_da7219_max98357a.o
 snd-soc-sst-bxt-rt298-objs := bxt_rt298.o
 snd-soc-sst-sof-pcm512x-objs := sof_pcm512x.o
@@ -47,13 +47,13 @@ obj-$(CONFIG_SND_SOC_INTEL_SOF_RT5682_MACH) += snd-soc-sof_rt5682.o
 obj-$(CONFIG_SND_SOC_INTEL_SOF_CS42L42_MACH) += snd-soc-sof_cs42l42.o
 obj-$(CONFIG_SND_SOC_INTEL_SOF_ES8336_MACH) += snd-soc-sof_es8336.o
 obj-$(CONFIG_SND_SOC_INTEL_SOF_NAU8825_MACH) += snd-soc-sof_nau8825.o
-obj-$(CONFIG_SND_SOC_INTEL_HASWELL_MACH) += snd-soc-sst-haswell.o
+obj-$(CONFIG_SND_SOC_INTEL_HASWELL_MACH) += snd-soc-hsw-rt5640.o
 obj-$(CONFIG_SND_SOC_INTEL_BXT_DA7219_MAX98357A_COMMON) += snd-soc-sst-bxt-da7219_max98357a.o
 obj-$(CONFIG_SND_SOC_INTEL_BXT_RT298_MACH) += snd-soc-sst-bxt-rt298.o
 obj-$(CONFIG_SND_SOC_INTEL_SOF_PCM512x_MACH) += snd-soc-sst-sof-pcm512x.o
 obj-$(CONFIG_SND_SOC_INTEL_SOF_WM8804_MACH) += snd-soc-sst-sof-wm8804.o
 obj-$(CONFIG_SND_SOC_INTEL_GLK_RT5682_MAX98357A_MACH) += snd-soc-sst-glk-rt5682_max98357a.o
-obj-$(CONFIG_SND_SOC_INTEL_BROADWELL_MACH) += snd-soc-sst-broadwell.o
+obj-$(CONFIG_SND_SOC_INTEL_BROADWELL_MACH) += snd-soc-bdw-rt286.o
 obj-$(CONFIG_SND_SOC_INTEL_BDW_RT5650_MACH) += snd-soc-sst-bdw-rt5650-mach.o
 obj-$(CONFIG_SND_SOC_INTEL_BDW_RT5677_MACH) += snd-soc-sst-bdw-rt5677-mach.o
 obj-$(CONFIG_SND_SOC_INTEL_BYTCR_RT5640_MACH) += snd-soc-sst-bytcr-rt5640.o
diff --git a/sound/soc/intel/boards/hda_dsp_common.c b/sound/soc/intel/boards/hda_dsp_common.c
index 83c7dfbccd9d..04b7d4f7f9e2 100644
--- a/sound/soc/intel/boards/hda_dsp_common.c
+++ b/sound/soc/intel/boards/hda_dsp_common.c
@@ -54,7 +54,7 @@ int hda_dsp_hdmi_build_controls(struct snd_soc_card *card,
 		return -EINVAL;
 
 	hda_pvt = snd_soc_component_get_drvdata(comp);
-	hcodec = &hda_pvt->codec;
+	hcodec = hda_pvt->codec;
 
 	list_for_each_entry(hpcm, &hcodec->pcm_list_head, list) {
 		spcm = hda_dsp_hdmi_pcm_handle(card, i);
diff --git a/sound/soc/intel/boards/skl_hda_dsp_generic.c b/sound/soc/intel/boards/skl_hda_dsp_generic.c
index 81144efb4b44..879ebba52832 100644
--- a/sound/soc/intel/boards/skl_hda_dsp_generic.c
+++ b/sound/soc/intel/boards/skl_hda_dsp_generic.c
@@ -190,7 +190,7 @@ static void skl_set_hda_codec_autosuspend_delay(struct snd_soc_card *card)
 			 * all codecs are on the same bus, so it's sufficient
 			 * to look up only the first one
 			 */
-			snd_hda_set_power_save(hda_pvt->codec.bus,
+			snd_hda_set_power_save(hda_pvt->codec->bus,
 					       HDA_CODEC_AUTOSUSPEND_DELAY_MS);
 			break;
 		}
diff --git a/sound/soc/intel/boards/sof_cirrus_common.c b/sound/soc/intel/boards/sof_cirrus_common.c
index f4192df962d6..6e39eda77385 100644
--- a/sound/soc/intel/boards/sof_cirrus_common.c
+++ b/sound/soc/intel/boards/sof_cirrus_common.c
@@ -10,6 +10,9 @@
 #include "../../codecs/cs35l41.h"
 #include "sof_cirrus_common.h"
 
+#define CS35L41_HID "CSC3541"
+#define CS35L41_MAX_AMPS 4
+
 /*
  * Cirrus Logic CS35L41/CS35L53
  */
@@ -35,50 +38,12 @@ static const struct snd_soc_dapm_route cs35l41_dapm_routes[] = {
 	{"TR Spk", NULL, "TR SPK"},
 };
 
-static struct snd_soc_dai_link_component cs35l41_components[] = {
-	{
-		.name = CS35L41_DEV0_NAME,
-		.dai_name = CS35L41_CODEC_DAI,
-	},
-	{
-		.name = CS35L41_DEV1_NAME,
-		.dai_name = CS35L41_CODEC_DAI,
-	},
-	{
-		.name = CS35L41_DEV2_NAME,
-		.dai_name = CS35L41_CODEC_DAI,
-	},
-	{
-		.name = CS35L41_DEV3_NAME,
-		.dai_name = CS35L41_CODEC_DAI,
-	},
-};
+static struct snd_soc_dai_link_component cs35l41_components[CS35L41_MAX_AMPS];
 
 /*
  * Mapping between ACPI instance id and speaker position.
- *
- * Four speakers:
- *         0: Tweeter left, 1: Woofer left
- *         2: Tweeter right, 3: Woofer right
  */
-static struct snd_soc_codec_conf cs35l41_codec_conf[] = {
-	{
-		.dlc = COMP_CODEC_CONF(CS35L41_DEV0_NAME),
-		.name_prefix = "TL",
-	},
-	{
-		.dlc = COMP_CODEC_CONF(CS35L41_DEV1_NAME),
-		.name_prefix = "WL",
-	},
-	{
-		.dlc = COMP_CODEC_CONF(CS35L41_DEV2_NAME),
-		.name_prefix = "TR",
-	},
-	{
-		.dlc = COMP_CODEC_CONF(CS35L41_DEV3_NAME),
-		.name_prefix = "WR",
-	},
-};
+static struct snd_soc_codec_conf cs35l41_codec_conf[CS35L41_MAX_AMPS];
 
 static int cs35l41_init(struct snd_soc_pcm_runtime *rtd)
 {
@@ -117,10 +82,10 @@ static int cs35l41_init(struct snd_soc_pcm_runtime *rtd)
 static const struct {
 	unsigned int rx[2];
 } cs35l41_channel_map[] = {
-	{.rx = {0, 1}}, /* TL */
 	{.rx = {0, 1}}, /* WL */
-	{.rx = {1, 0}}, /* TR */
 	{.rx = {1, 0}}, /* WR */
+	{.rx = {0, 1}}, /* TL */
+	{.rx = {1, 0}}, /* TR */
 };
 
 static int cs35l41_hw_params(struct snd_pcm_substream *substream,
@@ -175,10 +140,51 @@ static const struct snd_soc_ops cs35l41_ops = {
 	.hw_params = cs35l41_hw_params,
 };
 
+static const char * const cs35l41_name_prefixes[] = { "WL", "WR", "TL", "TR" };
+
+/*
+ * Expected UIDs are integers (stored as strings).
+ * UID Mapping is fixed:
+ * UID 0x0 -> WL
+ * UID 0x1 -> WR
+ * UID 0x2 -> TL
+ * UID 0x3 -> TR
+ * Note: If there are less than 4 Amps, UIDs still map to WL/WR/TL/TR. Dynamic code will only create
+ * dai links for UIDs which exist, and ignore non-existant ones. Only 2 or 4 amps are expected.
+ * Return number of codecs found.
+ */
+static int cs35l41_compute_codec_conf(void)
+{
+	const char * const uid_strings[] = { "0", "1", "2", "3" };
+	unsigned int uid, sz = 0;
+	struct acpi_device *adev;
+	struct device *physdev;
+
+	for (uid = 0; uid < CS35L41_MAX_AMPS; uid++) {
+		adev = acpi_dev_get_first_match_dev(CS35L41_HID, uid_strings[uid], -1);
+		if (!adev) {
+			pr_devel("Cannot find match for HID %s UID %u (%s)\n", CS35L41_HID, uid,
+				 cs35l41_name_prefixes[uid]);
+			continue;
+		}
+		physdev = get_device(acpi_get_first_physical_node(adev));
+		cs35l41_components[sz].name = dev_name(physdev);
+		cs35l41_components[sz].dai_name = CS35L41_CODEC_DAI;
+		cs35l41_codec_conf[sz].dlc.name = dev_name(physdev);
+		cs35l41_codec_conf[sz].name_prefix = cs35l41_name_prefixes[uid];
+		acpi_dev_put(adev);
+		sz++;
+	}
+
+	if (sz != 2 && sz != 4)
+		pr_warn("Invalid number of cs35l41 amps found: %d, expected 2 or 4\n", sz);
+	return sz;
+}
+
 void cs35l41_set_dai_link(struct snd_soc_dai_link *link)
 {
+	link->num_codecs = cs35l41_compute_codec_conf();
 	link->codecs = cs35l41_components;
-	link->num_codecs = ARRAY_SIZE(cs35l41_components);
 	link->init = cs35l41_init;
 	link->ops = &cs35l41_ops;
 }
diff --git a/sound/soc/intel/boards/sof_cs42l42.c b/sound/soc/intel/boards/sof_cs42l42.c
index 85ffd065895d..e38bd2831e6a 100644
--- a/sound/soc/intel/boards/sof_cs42l42.c
+++ b/sound/soc/intel/boards/sof_cs42l42.c
@@ -445,9 +445,9 @@ static int create_hdmi_dai_links(struct device *dev,
 	if (hdmi_num <= 0)
 		return 0;
 
-	idisp_components = devm_kzalloc(dev,
-					sizeof(struct snd_soc_dai_link_component) *
-					hdmi_num, GFP_KERNEL);
+	idisp_components = devm_kcalloc(dev,
+					hdmi_num,
+					sizeof(struct snd_soc_dai_link_component), GFP_KERNEL);
 	if (!idisp_components)
 		goto devm_err;
 
@@ -543,10 +543,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev,
 	struct snd_soc_dai_link *links;
 	int ret, id = 0, link_seq;
 
-	links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) *
-			     sof_audio_card_cs42l42.num_links, GFP_KERNEL);
-	cpus = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link_component) *
-			     sof_audio_card_cs42l42.num_links, GFP_KERNEL);
+	links = devm_kcalloc(dev, sof_audio_card_cs42l42.num_links,
+			    sizeof(struct snd_soc_dai_link), GFP_KERNEL);
+	cpus = devm_kcalloc(dev, sof_audio_card_cs42l42.num_links,
+			    sizeof(struct snd_soc_dai_link_component), GFP_KERNEL);
 	if (!links || !cpus)
 		goto devm_err;
 
diff --git a/sound/soc/intel/boards/sof_da7219_max98373.c b/sound/soc/intel/boards/sof_da7219_max98373.c
index 34cf849a8344..e048e789e633 100644
--- a/sound/soc/intel/boards/sof_da7219_max98373.c
+++ b/sound/soc/intel/boards/sof_da7219_max98373.c
@@ -195,7 +195,7 @@ static int ssp1_hw_params(struct snd_pcm_substream *substream,
 	struct snd_soc_pcm_runtime *runtime = asoc_substream_to_rtd(substream);
 	int ret, j;
 
-	for (j = 0; j < runtime->num_codecs; j++) {
+	for (j = 0; j < runtime->dai_link->num_codecs; j++) {
 		struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(runtime, j);
 
 		if (!strcmp(codec_dai->component->name, MAXIM_DEV0_NAME)) {
diff --git a/sound/soc/intel/boards/sof_es8336.c b/sound/soc/intel/boards/sof_es8336.c
index 606cc3242a60..fbb42e54947a 100644
--- a/sound/soc/intel/boards/sof_es8336.c
+++ b/sound/soc/intel/boards/sof_es8336.c
@@ -481,9 +481,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev,
 
 	/* HDMI */
 	if (hdmi_num > 0) {
-		idisp_components = devm_kzalloc(dev,
-						sizeof(struct snd_soc_dai_link_component) *
-						hdmi_num, GFP_KERNEL);
+		idisp_components = devm_kcalloc(dev,
+						hdmi_num,
+						sizeof(struct snd_soc_dai_link_component),
+						GFP_KERNEL);
 		if (!idisp_components)
 			goto devm_err;
 	}
diff --git a/sound/soc/intel/boards/sof_nau8825.c b/sound/soc/intel/boards/sof_nau8825.c
index 8d7e5ba9e516..5585c217f78d 100644
--- a/sound/soc/intel/boards/sof_nau8825.c
+++ b/sound/soc/intel/boards/sof_nau8825.c
@@ -355,10 +355,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev,
 	struct snd_soc_dai_link *links;
 	int i, id = 0;
 
-	links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) *
-			     sof_audio_card_nau8825.num_links, GFP_KERNEL);
-	cpus = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link_component) *
-			     sof_audio_card_nau8825.num_links, GFP_KERNEL);
+	links = devm_kcalloc(dev, sof_audio_card_nau8825.num_links,
+			    sizeof(struct snd_soc_dai_link), GFP_KERNEL);
+	cpus = devm_kcalloc(dev, sof_audio_card_nau8825.num_links,
+			    sizeof(struct snd_soc_dai_link_component), GFP_KERNEL);
 	if (!links || !cpus)
 		goto devm_err;
 
@@ -421,9 +421,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev,
 
 	/* HDMI */
 	if (hdmi_num > 0) {
-		idisp_components = devm_kzalloc(dev,
-						sizeof(struct snd_soc_dai_link_component) *
-						hdmi_num, GFP_KERNEL);
+		idisp_components = devm_kcalloc(dev,
+						hdmi_num,
+						sizeof(struct snd_soc_dai_link_component),
+						GFP_KERNEL);
 		if (!idisp_components)
 			goto devm_err;
 	}
diff --git a/sound/soc/intel/boards/sof_realtek_common.c b/sound/soc/intel/boards/sof_realtek_common.c
index b9643ca2e2f2..ff2851fc8930 100644
--- a/sound/soc/intel/boards/sof_realtek_common.c
+++ b/sound/soc/intel/boards/sof_realtek_common.c
@@ -253,63 +253,70 @@ EXPORT_SYMBOL_NS(sof_rt1015p_codec_conf, SND_SOC_INTEL_SOF_REALTEK_COMMON);
  * RT1015 audio amplifier
  */
 
+static const struct {
+	unsigned int tx;
+	unsigned int rx;
+} rt1015_tdm_mask[] = {
+	{.tx = 0x0, .rx = 0x1},
+	{.tx = 0x0, .rx = 0x2},
+};
+
 static int rt1015_hw_params(struct snd_pcm_substream *substream,
 			    struct snd_pcm_hw_params *params)
 {
 	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
+	struct snd_soc_dai_link *dai_link = rtd->dai_link;
 	struct snd_soc_dai *codec_dai;
-	int i, fs = 64, ret;
+	int i, clk_freq, ret;
 
-	for_each_rtd_codec_dais(rtd, i, codec_dai) {
-		ret = snd_soc_dai_set_pll(codec_dai, 0, RT1015_PLL_S_BCLK,
-					  params_rate(params) * fs,
-					  params_rate(params) * 256);
-		if (ret)
-			return ret;
+	clk_freq = sof_dai_get_bclk(rtd);
 
-		ret = snd_soc_dai_set_sysclk(codec_dai, RT1015_SCLK_S_PLL,
-					     params_rate(params) * 256,
-					     SND_SOC_CLOCK_IN);
-		if (ret)
-			return ret;
+	if (clk_freq <= 0) {
+		dev_err(rtd->dev, "fail to get bclk freq, ret %d\n", clk_freq);
+		return -EINVAL;
 	}
 
-	return 0;
-}
-
-static int rt1015_hw_params_pll_and_tdm(struct snd_pcm_substream *substream,
-					 struct snd_pcm_hw_params *params)
-{
-	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
-	struct snd_soc_dai *codec_dai;
-	int i, fs = 100, ret;
-
 	for_each_rtd_codec_dais(rtd, i, codec_dai) {
 		ret = snd_soc_dai_set_pll(codec_dai, 0, RT1015_PLL_S_BCLK,
-					  params_rate(params) * fs,
+					  clk_freq,
 					  params_rate(params) * 256);
-		if (ret)
+		if (ret) {
+			dev_err(codec_dai->dev, "fail to set pll, ret %d\n",
+				ret);
 			return ret;
+		}
 
 		ret = snd_soc_dai_set_sysclk(codec_dai, RT1015_SCLK_S_PLL,
 					     params_rate(params) * 256,
 					     SND_SOC_CLOCK_IN);
-		if (ret)
+		if (ret) {
+			dev_err(codec_dai->dev, "fail to set sysclk, ret %d\n",
+				ret);
 			return ret;
-	}
-	/* rx slot 1 for RT1015_DEV0_NAME */
-	ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_codec(rtd, 0),
-				       0x0, 0x1, 4, 24);
-	if (ret)
-		return ret;
+		}
 
-	/* rx slot 2 for RT1015_DEV1_NAME */
-	ret = snd_soc_dai_set_tdm_slot(asoc_rtd_to_codec(rtd, 1),
-				       0x0, 0x2, 4, 24);
-	if (ret)
-		return ret;
+		switch (dai_link->dai_fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+		case SND_SOC_DAIFMT_DSP_A:
+		case SND_SOC_DAIFMT_DSP_B:
+			/* 4-slot TDM */
+			ret = snd_soc_dai_set_tdm_slot(codec_dai,
+						       rt1015_tdm_mask[i].tx,
+						       rt1015_tdm_mask[i].rx,
+						       4,
+						       params_width(params));
+			if (ret < 0) {
+				dev_err(codec_dai->dev, "fail to set tdm slot, ret %d\n",
+					ret);
+				return ret;
+			}
+			break;
+		default:
+			dev_dbg(codec_dai->dev, "codec is in I2S mode\n");
+			break;
+		}
+	}
 
-	return 0;
+	return ret;
 }
 
 static struct snd_soc_ops rt1015_ops = {
@@ -351,15 +358,12 @@ void sof_rt1015_codec_conf(struct snd_soc_card *card)
 }
 EXPORT_SYMBOL_NS(sof_rt1015_codec_conf, SND_SOC_INTEL_SOF_REALTEK_COMMON);
 
-void sof_rt1015_dai_link(struct snd_soc_dai_link *link, unsigned int fs)
+void sof_rt1015_dai_link(struct snd_soc_dai_link *link)
 {
 	link->codecs = rt1015_components;
 	link->num_codecs = ARRAY_SIZE(rt1015_components);
 	link->init = speaker_codec_init_lr;
 	link->ops = &rt1015_ops;
-
-	if (fs == 100)
-		rt1015_ops.hw_params = rt1015_hw_params_pll_and_tdm;
 }
 EXPORT_SYMBOL_NS(sof_rt1015_dai_link, SND_SOC_INTEL_SOF_REALTEK_COMMON);
 
diff --git a/sound/soc/intel/boards/sof_realtek_common.h b/sound/soc/intel/boards/sof_realtek_common.h
index 778443421090..3ae99d8239e0 100644
--- a/sound/soc/intel/boards/sof_realtek_common.h
+++ b/sound/soc/intel/boards/sof_realtek_common.h
@@ -32,7 +32,7 @@ void sof_rt1015p_codec_conf(struct snd_soc_card *card);
 #define RT1015_DEV0_NAME	"i2c-10EC1015:00"
 #define RT1015_DEV1_NAME	"i2c-10EC1015:01"
 
-void sof_rt1015_dai_link(struct snd_soc_dai_link *link, unsigned int fs);
+void sof_rt1015_dai_link(struct snd_soc_dai_link *link);
 void sof_rt1015_codec_conf(struct snd_soc_card *card);
 
 #define RT1308_CODEC_DAI	"rt1308-aif"
diff --git a/sound/soc/intel/boards/sof_rt5682.c b/sound/soc/intel/boards/sof_rt5682.c
index 045965312245..2d0986824b3d 100644
--- a/sound/soc/intel/boards/sof_rt5682.c
+++ b/sound/soc/intel/boards/sof_rt5682.c
@@ -46,7 +46,6 @@
 	((quirk << SOF_RT5682_NUM_HDMIDEV_SHIFT) & SOF_RT5682_NUM_HDMIDEV_MASK)
 #define SOF_RT1011_SPEAKER_AMP_PRESENT		BIT(13)
 #define SOF_RT1015_SPEAKER_AMP_PRESENT		BIT(14)
-#define SOF_RT1015_SPEAKER_AMP_100FS		BIT(15)
 #define SOF_RT1015P_SPEAKER_AMP_PRESENT		BIT(16)
 #define SOF_MAX98373_SPEAKER_AMP_PRESENT	BIT(17)
 #define SOF_MAX98360A_SPEAKER_AMP_PRESENT	BIT(18)
@@ -132,7 +131,6 @@ static const struct dmi_system_id sof_rt5682_quirk_table[] = {
 					SOF_RT5682_SSP_CODEC(0) |
 					SOF_SPEAKER_AMP_PRESENT |
 					SOF_RT1015_SPEAKER_AMP_PRESENT |
-					SOF_RT1015_SPEAKER_AMP_100FS |
 					SOF_RT5682_SSP_AMP(1)),
 	},
 	{
@@ -600,10 +598,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev,
 	struct snd_soc_dai_link *links;
 	int i, id = 0;
 
-	links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) *
-			     sof_audio_card_rt5682.num_links, GFP_KERNEL);
-	cpus = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link_component) *
-			     sof_audio_card_rt5682.num_links, GFP_KERNEL);
+	links = devm_kcalloc(dev, sof_audio_card_rt5682.num_links,
+			    sizeof(struct snd_soc_dai_link), GFP_KERNEL);
+	cpus = devm_kcalloc(dev, sof_audio_card_rt5682.num_links,
+			    sizeof(struct snd_soc_dai_link_component), GFP_KERNEL);
 	if (!links || !cpus)
 		goto devm_err;
 
@@ -687,9 +685,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev,
 
 	/* HDMI */
 	if (hdmi_num > 0) {
-		idisp_components = devm_kzalloc(dev,
-				   sizeof(struct snd_soc_dai_link_component) *
-				   hdmi_num, GFP_KERNEL);
+		idisp_components = devm_kcalloc(dev,
+				   hdmi_num,
+				   sizeof(struct snd_soc_dai_link_component),
+				   GFP_KERNEL);
 		if (!idisp_components)
 			goto devm_err;
 	}
@@ -739,8 +738,7 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev,
 
 		links[id].id = id;
 		if (sof_rt5682_quirk & SOF_RT1015_SPEAKER_AMP_PRESENT) {
-			sof_rt1015_dai_link(&links[id], (sof_rt5682_quirk &
-					SOF_RT1015_SPEAKER_AMP_100FS) ? 100 : 64);
+			sof_rt1015_dai_link(&links[id]);
 		} else if (sof_rt5682_quirk & SOF_RT1015P_SPEAKER_AMP_PRESENT) {
 			sof_rt1015p_dai_link(&links[id]);
 		} else if (sof_rt5682_quirk & SOF_RT1019_SPEAKER_AMP_PRESENT) {
@@ -1010,7 +1008,6 @@ static const struct platform_device_id board_ids[] = {
 					SOF_RT5682_SSP_CODEC(0) |
 					SOF_SPEAKER_AMP_PRESENT |
 					SOF_RT1015_SPEAKER_AMP_PRESENT |
-					SOF_RT1015_SPEAKER_AMP_100FS |
 					SOF_RT5682_SSP_AMP(1)),
 	},
 	{
diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c
index a49bfaab6b21..2ff30b40a1e4 100644
--- a/sound/soc/intel/boards/sof_sdw.c
+++ b/sound/soc/intel/boards/sof_sdw.c
@@ -270,6 +270,16 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = {
 		.callback = sof_sdw_quirk_cb,
 		.matches = {
 			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0AFF")
+		},
+		.driver_data = (void *)(SOF_SDW_TGL_HDMI |
+					RT711_JD2 |
+					SOF_SDW_FOUR_SPK),
+	},
+	{
+		.callback = sof_sdw_quirk_cb,
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc"),
 			DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0B00")
 		},
 		.driver_data = (void *)(SOF_SDW_TGL_HDMI |
diff --git a/sound/soc/intel/boards/sof_ssp_amp.c b/sound/soc/intel/boards/sof_ssp_amp.c
index 4a762e002ac7..94d25aeb6e7c 100644
--- a/sound/soc/intel/boards/sof_ssp_amp.c
+++ b/sound/soc/intel/boards/sof_ssp_amp.c
@@ -210,10 +210,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev,
 	struct snd_soc_dai_link *links;
 	int i, id = 0;
 
-	links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) *
-					sof_ssp_amp_card.num_links, GFP_KERNEL);
-	cpus = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link_component) *
-					sof_ssp_amp_card.num_links, GFP_KERNEL);
+	links = devm_kcalloc(dev, sof_ssp_amp_card.num_links,
+					sizeof(struct snd_soc_dai_link), GFP_KERNEL);
+	cpus = devm_kcalloc(dev, sof_ssp_amp_card.num_links,
+					sizeof(struct snd_soc_dai_link_component), GFP_KERNEL);
 	if (!links || !cpus)
 		return NULL;
 
@@ -306,9 +306,10 @@ static struct snd_soc_dai_link *sof_card_dai_links_create(struct device *dev,
 	if (sof_ssp_amp_quirk & SOF_HDMI_PLAYBACK_PRESENT) {
 		/* HDMI */
 		if (hdmi_num > 0) {
-			idisp_components = devm_kzalloc(dev,
-					   sizeof(struct snd_soc_dai_link_component) *
-					   hdmi_num, GFP_KERNEL);
+			idisp_components = devm_kcalloc(dev,
+					   hdmi_num,
+					   sizeof(struct snd_soc_dai_link_component),
+					   GFP_KERNEL);
 			if (!idisp_components)
 				goto devm_err;
 		}
diff --git a/sound/soc/intel/catpt/device.c b/sound/soc/intel/catpt/device.c
index d48a71d2cf1e..d5d08bd766c7 100644
--- a/sound/soc/intel/catpt/device.c
+++ b/sound/soc/intel/catpt/device.c
@@ -22,7 +22,6 @@
 #include <sound/intel-dsp-config.h>
 #include <sound/soc.h>
 #include <sound/soc-acpi.h>
-#include <sound/soc-acpi-intel-match.h>
 #include "core.h"
 #include "registers.h"
 
@@ -310,8 +309,36 @@ static int catpt_acpi_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static struct snd_soc_acpi_mach lpt_machines[] = {
+	{
+		.id = "INT33CA",
+		.drv_name = "hsw_rt5640",
+	},
+	{}
+};
+
+static struct snd_soc_acpi_mach wpt_machines[] = {
+	{
+		.id = "INT33CA",
+		.drv_name = "hsw_rt5640",
+	},
+	{
+		.id = "INT343A",
+		.drv_name = "bdw_rt286",
+	},
+	{
+		.id = "10EC5650",
+		.drv_name = "bdw-rt5650",
+	},
+	{
+		.id = "RT5677CE",
+		.drv_name = "bdw-rt5677",
+	},
+	{}
+};
+
 static struct catpt_spec lpt_desc = {
-	.machines = snd_soc_acpi_intel_haswell_machines,
+	.machines = lpt_machines,
 	.core_id = 0x01,
 	.host_dram_offset = 0x000000,
 	.host_iram_offset = 0x080000,
@@ -326,7 +353,7 @@ static struct catpt_spec lpt_desc = {
 };
 
 static struct catpt_spec wpt_desc = {
-	.machines = snd_soc_acpi_intel_broadwell_machines,
+	.machines = wpt_machines,
 	.core_id = 0x02,
 	.host_dram_offset = 0x000000,
 	.host_iram_offset = 0x0A0000,
diff --git a/sound/soc/intel/catpt/sysfs.c b/sound/soc/intel/catpt/sysfs.c
index 1bdbcc04dc71..9b6d2d93a2e7 100644
--- a/sound/soc/intel/catpt/sysfs.c
+++ b/sound/soc/intel/catpt/sysfs.c
@@ -27,8 +27,8 @@ static ssize_t fw_version_show(struct device *dev,
 	if (ret)
 		return CATPT_IPC_ERROR(ret);
 
-	return sprintf(buf, "%d.%d.%d.%d\n", version.type, version.major,
-		       version.minor, version.build);
+	return sysfs_emit(buf, "%d.%d.%d.%d\n", version.type, version.major,
+			  version.minor, version.build);
 }
 static DEVICE_ATTR_RO(fw_version);
 
@@ -37,7 +37,7 @@ static ssize_t fw_info_show(struct device *dev,
 {
 	struct catpt_dev *cdev = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%s\n", cdev->ipc.config.fw_info);
+	return sysfs_emit(buf, "%s\n", cdev->ipc.config.fw_info);
 }
 static DEVICE_ATTR_RO(fw_info);
 
diff --git a/sound/soc/intel/common/Makefile b/sound/soc/intel/common/Makefile
index 8ca8f872ec80..41054cf09ec9 100644
--- a/sound/soc/intel/common/Makefile
+++ b/sound/soc/intel/common/Makefile
@@ -9,7 +9,7 @@ snd-soc-acpi-intel-match-objs := soc-acpi-intel-byt-match.o soc-acpi-intel-cht-m
 	soc-acpi-intel-cml-match.o soc-acpi-intel-icl-match.o \
 	soc-acpi-intel-tgl-match.o soc-acpi-intel-ehl-match.o \
 	soc-acpi-intel-jsl-match.o soc-acpi-intel-adl-match.o \
-	soc-acpi-intel-mtl-match.o \
+	soc-acpi-intel-rpl-match.o soc-acpi-intel-mtl-match.o \
 	soc-acpi-intel-hda-match.o \
 	soc-acpi-intel-sdw-mockup-match.o
 
diff --git a/sound/soc/intel/common/soc-acpi-intel-hsw-bdw-match.c b/sound/soc/intel/common/soc-acpi-intel-hsw-bdw-match.c
index cbcb649604e5..6daf60b1edf1 100644
--- a/sound/soc/intel/common/soc-acpi-intel-hsw-bdw-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-hsw-bdw-match.c
@@ -9,40 +9,25 @@
 #include <sound/soc-acpi.h>
 #include <sound/soc-acpi-intel-match.h>
 
-struct snd_soc_acpi_mach snd_soc_acpi_intel_haswell_machines[] = {
-	{
-		.id = "INT33CA",
-		.drv_name = "hsw_rt5640",
-		.fw_filename = "intel/IntcSST1.bin",
-		.sof_tplg_filename = "sof-hsw.tplg",
-	},
-	{}
-};
-EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_haswell_machines);
-
 struct snd_soc_acpi_mach snd_soc_acpi_intel_broadwell_machines[] = {
 	{
 		.id = "INT343A",
 		.drv_name = "bdw_rt286",
-		.fw_filename =  "intel/IntcSST2.bin",
 		.sof_tplg_filename = "sof-bdw-rt286.tplg",
 	},
 	{
 		.id = "10EC5650",
 		.drv_name = "bdw-rt5650",
-		.fw_filename = "intel/IntcSST2.bin",
 		.sof_tplg_filename = "sof-bdw-rt5650.tplg",
 	},
 	{
 		.id = "RT5677CE",
 		.drv_name = "bdw-rt5677",
-		.fw_filename =  "intel/IntcSST2.bin",
 		.sof_tplg_filename = "sof-bdw-rt5677.tplg",
 	},
 	{
 		.id = "INT33CA",
 		.drv_name = "hsw_rt5640",
-		.fw_filename = "intel/IntcSST2.bin",
 		.sof_tplg_filename = "sof-bdw-rt5640.tplg",
 	},
 	{}
diff --git a/sound/soc/intel/common/soc-acpi-intel-rpl-match.c b/sound/soc/intel/common/soc-acpi-intel-rpl-match.c
new file mode 100644
index 000000000000..9ccf7370157b
--- /dev/null
+++ b/sound/soc/intel/common/soc-acpi-intel-rpl-match.c
@@ -0,0 +1,131 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * soc-apci-intel-rpl-match.c - tables and support for RPL ACPI enumeration.
+ *
+ * Copyright (c) 2022 Intel Corporation.
+ */
+
+#include <sound/soc-acpi.h>
+#include <sound/soc-acpi-intel-match.h>
+
+static const struct snd_soc_acpi_endpoint single_endpoint = {
+	.num = 0,
+	.aggregated = 0,
+	.group_position = 0,
+	.group_id = 0,
+};
+
+static const struct snd_soc_acpi_endpoint spk_l_endpoint = {
+	.num = 0,
+	.aggregated = 1,
+	.group_position = 0,
+	.group_id = 1,
+};
+
+static const struct snd_soc_acpi_endpoint spk_r_endpoint = {
+	.num = 0,
+	.aggregated = 1,
+	.group_position = 1,
+	.group_id = 1,
+};
+
+static const struct snd_soc_acpi_adr_device rt711_0_adr[] = {
+	{
+		.adr = 0x000020025D071100ull,
+		.num_endpoints = 1,
+		.endpoints = &single_endpoint,
+		.name_prefix = "rt711"
+	}
+};
+
+static const struct snd_soc_acpi_link_adr rpl_rvp[] = {
+	{
+		.mask = BIT(0),
+		.num_adr = ARRAY_SIZE(rt711_0_adr),
+		.adr_d = rt711_0_adr,
+	},
+	{}
+};
+
+static const struct snd_soc_acpi_adr_device rt711_sdca_0_adr[] = {
+	{
+		.adr = 0x000030025D071101ull,
+		.num_endpoints = 1,
+		.endpoints = &single_endpoint,
+		.name_prefix = "rt711"
+	}
+};
+
+static const struct snd_soc_acpi_adr_device rt1316_1_group1_adr[] = {
+	{
+		.adr = 0x000131025D131601ull, /* unique ID is set for some reason */
+		.num_endpoints = 1,
+		.endpoints = &spk_l_endpoint,
+		.name_prefix = "rt1316-1"
+	}
+};
+
+static const struct snd_soc_acpi_adr_device rt1316_3_group1_adr[] = {
+	{
+		.adr = 0x000330025D131601ull,
+		.num_endpoints = 1,
+		.endpoints = &spk_r_endpoint,
+		.name_prefix = "rt1316-2"
+	}
+};
+
+static const struct snd_soc_acpi_adr_device rt714_2_adr[] = {
+	{
+		.adr = 0x000230025D071401ull,
+		.num_endpoints = 1,
+		.endpoints = &single_endpoint,
+		.name_prefix = "rt714"
+	}
+};
+
+static const struct snd_soc_acpi_link_adr rpl_sdca_3_in_1[] = {
+	{
+		.mask = BIT(0),
+		.num_adr = ARRAY_SIZE(rt711_sdca_0_adr),
+		.adr_d = rt711_sdca_0_adr,
+	},
+	{
+		.mask = BIT(1),
+		.num_adr = ARRAY_SIZE(rt1316_1_group1_adr),
+		.adr_d = rt1316_1_group1_adr,
+	},
+	{
+		.mask = BIT(2),
+		.num_adr = ARRAY_SIZE(rt714_2_adr),
+		.adr_d = rt714_2_adr,
+	},
+	{
+		.mask = BIT(3),
+		.num_adr = ARRAY_SIZE(rt1316_3_group1_adr),
+		.adr_d = rt1316_3_group1_adr,
+	},
+	{}
+};
+
+struct snd_soc_acpi_mach snd_soc_acpi_intel_rpl_machines[] = {
+	{},
+};
+EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_rpl_machines);
+
+/* this table is used when there is no I2S codec present */
+struct snd_soc_acpi_mach snd_soc_acpi_intel_rpl_sdw_machines[] = {
+	{
+		.link_mask = 0xF, /* 4 active links required */
+		.links = rpl_sdca_3_in_1,
+		.drv_name = "sof_sdw",
+		.sof_tplg_filename = "sof-rpl-rt711-l0-rt1316-l13-rt714-l2.tplg",
+	},
+	{
+		.link_mask = 0x1, /* link0 required */
+		.links = rpl_rvp,
+		.drv_name = "sof_sdw",
+		.sof_tplg_filename = "sof-rpl-rt711.tplg",
+	},
+	{},
+};
+EXPORT_SYMBOL_GPL(snd_soc_acpi_intel_rpl_sdw_machines);
diff --git a/sound/soc/intel/skylake/skl-nhlt.c b/sound/soc/intel/skylake/skl-nhlt.c
index deb7b820325e..e617b4c335a4 100644
--- a/sound/soc/intel/skylake/skl-nhlt.c
+++ b/sound/soc/intel/skylake/skl-nhlt.c
@@ -61,7 +61,7 @@ static ssize_t platform_id_show(struct device *dev,
 			nhlt->header.oem_revision);
 
 	skl_nhlt_trim_space(platform_id);
-	return sprintf(buf, "%s\n", platform_id);
+	return sysfs_emit(buf, "%s\n", platform_id);
 }
 
 static DEVICE_ATTR_RO(platform_id);
diff --git a/sound/soc/intel/skylake/skl-pcm.c b/sound/soc/intel/skylake/skl-pcm.c
index 9d72ebd812af..1015716f9336 100644
--- a/sound/soc/intel/skylake/skl-pcm.c
+++ b/sound/soc/intel/skylake/skl-pcm.c
@@ -275,7 +275,7 @@ static int skl_pcm_prepare(struct snd_pcm_substream *substream,
 	 * calls prepare another time, reset the FW pipe to clean state
 	 */
 	if (mconfig &&
-		(substream->runtime->status->state == SNDRV_PCM_STATE_XRUN ||
+		(substream->runtime->state == SNDRV_PCM_STATE_XRUN ||
 		 mconfig->pipe->state == SKL_PIPE_CREATED ||
 		 mconfig->pipe->state == SKL_PIPE_PAUSED)) {
 
@@ -593,7 +593,7 @@ static int skl_link_pcm_prepare(struct snd_pcm_substream *substream,
 	/* In case of XRUN recovery, reset the FW pipe to clean state */
 	mconfig = skl_tplg_be_get_cpr_module(dai, substream->stream);
 	if (mconfig && !mconfig->pipe->passthru &&
-		(substream->runtime->status->state == SNDRV_PCM_STATE_XRUN))
+		(substream->runtime->state == SNDRV_PCM_STATE_XRUN))
 		skl_reset_pipe(skl, mconfig->pipe);
 
 	return 0;
diff --git a/sound/soc/intel/skylake/skl-topology.h b/sound/soc/intel/skylake/skl-topology.h
index a5bccf2fcd88..017ac0ef324d 100644
--- a/sound/soc/intel/skylake/skl-topology.h
+++ b/sound/soc/intel/skylake/skl-topology.h
@@ -233,8 +233,8 @@ struct skl_uuid_inst_map {
 struct skl_kpb_params {
 	u32 num_modules;
 	union {
-		struct skl_mod_inst_map map[0];
-		struct skl_uuid_inst_map map_uuid[0];
+		DECLARE_FLEX_ARRAY(struct skl_mod_inst_map, map);
+		DECLARE_FLEX_ARRAY(struct skl_uuid_inst_map, map_uuid);
 	} u;
 };
 
diff --git a/sound/soc/intel/skylake/skl.c b/sound/soc/intel/skylake/skl.c
index aeca58246fc7..bbba2df33aaf 100644
--- a/sound/soc/intel/skylake/skl.c
+++ b/sound/soc/intel/skylake/skl.c
@@ -444,7 +444,7 @@ static int skl_free(struct hdac_bus *bus)
 	if (bus->irq >= 0)
 		free_irq(bus->irq, (void *)bus);
 	snd_hdac_bus_free_stream_pages(bus);
-	snd_hdac_stream_free_all(bus);
+	snd_hdac_ext_stream_free_all(bus);
 	snd_hdac_link_free_all(bus);
 
 	if (bus->remap_addr)
@@ -689,6 +689,35 @@ static void load_codec_module(struct hda_codec *codec)
 
 #endif /* CONFIG_SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC */
 
+static void skl_codec_device_exit(struct device *dev)
+{
+	snd_hdac_device_exit(dev_to_hdac_dev(dev));
+}
+
+static struct hda_codec *skl_codec_device_init(struct hdac_bus *bus, int addr)
+{
+	struct hda_codec *codec;
+	int ret;
+
+	codec = snd_hda_codec_device_init(to_hda_bus(bus), addr, "ehdaudio%dD%d", bus->idx, addr);
+	if (IS_ERR(codec)) {
+		dev_err(bus->dev, "device init failed for hdac device\n");
+		return codec;
+	}
+
+	codec->core.type = HDA_DEV_ASOC;
+	codec->core.dev.release = skl_codec_device_exit;
+
+	ret = snd_hdac_device_register(&codec->core);
+	if (ret) {
+		dev_err(bus->dev, "failed to register hdac device\n");
+		snd_hdac_device_exit(&codec->core);
+		return ERR_PTR(ret);
+	}
+
+	return codec;
+}
+
 /*
  * Probe the given codec address
  */
@@ -697,12 +726,11 @@ static int probe_codec(struct hdac_bus *bus, int addr)
 	unsigned int cmd = (addr << 28) | (AC_NODE_ROOT << 20) |
 		(AC_VERB_PARAMETERS << 8) | AC_PAR_VENDOR_ID;
 	unsigned int res = -1;
-	struct skl_dev *skl = bus_to_skl(bus);
 #if IS_ENABLED(CONFIG_SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC)
+	struct skl_dev *skl = bus_to_skl(bus);
 	struct hdac_hda_priv *hda_codec;
-	int err;
 #endif
-	struct hdac_device *hdev;
+	struct hda_codec *codec;
 
 	mutex_lock(&bus->cmd_mutex);
 	snd_hdac_bus_send_cmd(bus, cmd);
@@ -718,25 +746,22 @@ static int probe_codec(struct hdac_bus *bus, int addr)
 	if (!hda_codec)
 		return -ENOMEM;
 
-	hda_codec->codec.bus = skl_to_hbus(skl);
-	hdev = &hda_codec->codec.core;
+	codec = skl_codec_device_init(bus, addr);
+	if (IS_ERR(codec))
+		return PTR_ERR(codec);
 
-	err = snd_hdac_ext_bus_device_init(bus, addr, hdev, HDA_DEV_ASOC);
-	if (err < 0)
-		return err;
+	hda_codec->codec = codec;
+	dev_set_drvdata(&codec->core.dev, hda_codec);
 
 	/* use legacy bus only for HDA codecs, idisp uses ext bus */
 	if ((res & 0xFFFF0000) != IDISP_INTEL_VENDOR_ID) {
-		hdev->type = HDA_DEV_LEGACY;
-		load_codec_module(&hda_codec->codec);
+		codec->core.type = HDA_DEV_LEGACY;
+		load_codec_module(hda_codec->codec);
 	}
 	return 0;
 #else
-	hdev = devm_kzalloc(&skl->pci->dev, sizeof(*hdev), GFP_KERNEL);
-	if (!hdev)
-		return -ENOMEM;
-
-	return snd_hdac_ext_bus_device_init(bus, addr, hdev, HDA_DEV_ASOC);
+	codec = skl_codec_device_init(bus, addr);
+	return PTR_ERR_OR_ZERO(codec);
 #endif /* CONFIG_SND_SOC_INTEL_SKYLAKE_HDAUDIO_CODEC */
 }
 
@@ -1127,7 +1152,6 @@ static void skl_remove(struct pci_dev *pci)
 	if (skl->nhlt)
 		intel_nhlt_free(skl->nhlt);
 	skl_free(bus);
-	dev_set_drvdata(&pci->dev, NULL);
 }
 
 /* PCI IDs */
diff --git a/sound/soc/mediatek/mt8183/mt8183-afe-common.h b/sound/soc/mediatek/mt8183/mt8183-afe-common.h
index b220e7a7db7e..40ab48c1566c 100644
--- a/sound/soc/mediatek/mt8183/mt8183-afe-common.h
+++ b/sound/soc/mediatek/mt8183/mt8183-afe-common.h
@@ -99,6 +99,9 @@ unsigned int mt8183_general_rate_transform(struct device *dev,
 unsigned int mt8183_rate_transform(struct device *dev,
 				   unsigned int rate, int aud_blk);
 
+int mt8183_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name,
+			     const char *secondary_i2s_name);
+
 /* dai register */
 int mt8183_dai_adda_register(struct mtk_base_afe *afe);
 int mt8183_dai_pcm_register(struct mtk_base_afe *afe);
diff --git a/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c b/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c
index b33cc9a73ed1..9f22d3939818 100644
--- a/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c
+++ b/sound/soc/mediatek/mt8183/mt8183-da7219-max98357.c
@@ -17,6 +17,7 @@
 #include "../../codecs/da7219-aad.h"
 #include "../../codecs/da7219.h"
 #include "../../codecs/rt1015.h"
+#include "../common/mtk-afe-platform-driver.h"
 #include "mt8183-afe-common.h"
 
 #define DA7219_CODEC_DAI "da7219-hifi"
@@ -372,6 +373,36 @@ static int mt8183_da7219_max98357_hdmi_init(struct snd_soc_pcm_runtime *rtd)
 					  &priv->hdmi_jack, NULL);
 }
 
+static int mt8183_bt_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_component *cmpnt_afe =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe);
+	int ret;
+
+	ret = mt8183_dai_i2s_set_share(afe, "I2S5", "I2S0");
+	if (ret) {
+		dev_err(rtd->dev, "Failed to set up shared clocks\n");
+		return ret;
+	}
+	return 0;
+}
+
+static int mt8183_da7219_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_component *cmpnt_afe =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe);
+	int ret;
+
+	ret = mt8183_dai_i2s_set_share(afe, "I2S2", "I2S3");
+	if (ret) {
+		dev_err(rtd->dev, "Failed to set up shared clocks\n");
+		return ret;
+	}
+	return 0;
+}
+
 static struct snd_soc_dai_link mt8183_da7219_dai_links[] = {
 	/* FE */
 	{
@@ -500,6 +531,7 @@ static struct snd_soc_dai_link mt8183_da7219_dai_links[] = {
 		.ignore_suspend = 1,
 		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
 		.ops = &mt8183_da7219_i2s_ops,
+		.init = &mt8183_da7219_init,
 		SND_SOC_DAILINK_REG(i2s2),
 	},
 	{
@@ -515,6 +547,7 @@ static struct snd_soc_dai_link mt8183_da7219_dai_links[] = {
 		.ignore_suspend = 1,
 		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
 		.ops = &mt8183_mt6358_i2s_ops,
+		.init = &mt8183_bt_init,
 		SND_SOC_DAILINK_REG(i2s5),
 	},
 	{
diff --git a/sound/soc/mediatek/mt8183/mt8183-dai-i2s.c b/sound/soc/mediatek/mt8183/mt8183-dai-i2s.c
index 138591d71ebd..6a9ace4180d3 100644
--- a/sound/soc/mediatek/mt8183/mt8183-dai-i2s.c
+++ b/sound/soc/mediatek/mt8183/mt8183-dai-i2s.c
@@ -43,7 +43,6 @@ struct mtk_afe_i2s_priv {
 	int rate; /* for determine which apll to use */
 	int low_jitter_en;
 
-	const char *share_property_name;
 	int share_i2s_id;
 
 	int mclk_id;
@@ -977,54 +976,55 @@ static const struct mtk_afe_i2s_priv mt8183_i2s_priv[DAI_I2S_NUM] = {
 	[DAI_I2S0] = {
 		.id = MT8183_DAI_I2S_0,
 		.mclk_id = MT8183_I2S0_MCK,
-		.share_property_name = "i2s0-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S1] = {
 		.id = MT8183_DAI_I2S_1,
 		.mclk_id = MT8183_I2S1_MCK,
-		.share_property_name = "i2s1-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S2] = {
 		.id = MT8183_DAI_I2S_2,
 		.mclk_id = MT8183_I2S2_MCK,
-		.share_property_name = "i2s2-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S3] = {
 		.id = MT8183_DAI_I2S_3,
 		.mclk_id = MT8183_I2S3_MCK,
-		.share_property_name = "i2s3-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S5] = {
 		.id = MT8183_DAI_I2S_5,
 		.mclk_id = MT8183_I2S5_MCK,
-		.share_property_name = "i2s5-share",
 		.share_i2s_id = -1,
 	},
 };
 
-static int mt8183_dai_i2s_get_share(struct mtk_base_afe *afe)
+/**
+ * mt8183_dai_i2s_set_share() - Set up I2S ports to share a single clock.
+ * @afe: Pointer to &struct mtk_base_afe
+ * @main_i2s_name: The name of the I2S port that will provide the clock
+ * @secondary_i2s_name: The name of the I2S port that will use this clock
+ */
+int mt8183_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name,
+			     const char *secondary_i2s_name)
 {
-	struct mt8183_afe_private *afe_priv = afe->platform_priv;
-	const struct device_node *of_node = afe->dev->of_node;
-	const char *of_str;
-	const char *property_name;
-	struct mtk_afe_i2s_priv *i2s_priv;
-	int i;
+	struct mtk_afe_i2s_priv *secondary_i2s_priv;
+	int main_i2s_id;
 
-	for (i = 0; i < DAI_I2S_NUM; i++) {
-		i2s_priv = afe_priv->dai_priv[mt8183_i2s_priv[i].id];
-		property_name = mt8183_i2s_priv[i].share_property_name;
-		if (of_property_read_string(of_node, property_name, &of_str))
-			continue;
-		i2s_priv->share_i2s_id = get_i2s_id_by_name(afe, of_str);
-	}
+	secondary_i2s_priv = get_i2s_priv_by_name(afe, secondary_i2s_name);
+	if (!secondary_i2s_priv)
+		return -EINVAL;
+
+	main_i2s_id = get_i2s_id_by_name(afe, main_i2s_name);
+	if (main_i2s_id < 0)
+		return main_i2s_id;
+
+	secondary_i2s_priv->share_i2s_id = main_i2s_id;
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mt8183_dai_i2s_set_share);
 
 static int mt8183_dai_i2s_set_priv(struct mtk_base_afe *afe)
 {
@@ -1074,10 +1074,5 @@ int mt8183_dai_i2s_register(struct mtk_base_afe *afe)
 	if (ret)
 		return ret;
 
-	/* parse share i2s */
-	ret = mt8183_dai_i2s_get_share(afe);
-	if (ret)
-		return ret;
-
 	return 0;
 }
diff --git a/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c b/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c
index ab157db78335..a86085223677 100644
--- a/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c
+++ b/sound/soc/mediatek/mt8183/mt8183-mt6358-ts3a227-max98357.c
@@ -15,6 +15,7 @@
 
 #include "../../codecs/rt1015.h"
 #include "../../codecs/ts3a227e.h"
+#include "../common/mtk-afe-platform-driver.h"
 #include "mt8183-afe-common.h"
 
 #define RT1015_CODEC_DAI "rt1015-aif"
@@ -391,6 +392,36 @@ mt8183_mt6358_ts3a227_max98357_hdmi_init(struct snd_soc_pcm_runtime *rtd)
 					  &priv->hdmi_jack, NULL);
 }
 
+static int mt8183_bt_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_component *cmpnt_afe =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe);
+	int ret;
+
+	ret = mt8183_dai_i2s_set_share(afe, "I2S5", "I2S0");
+	if (ret) {
+		dev_err(rtd->dev, "Failed to set up shared clocks\n");
+		return ret;
+	}
+	return 0;
+}
+
+static int mt8183_i2s2_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_component *cmpnt_afe =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe);
+	int ret;
+
+	ret = mt8183_dai_i2s_set_share(afe, "I2S2", "I2S3");
+	if (ret) {
+		dev_err(rtd->dev, "Failed to set up shared clocks\n");
+		return ret;
+	}
+	return 0;
+}
+
 static struct snd_soc_dai_link mt8183_mt6358_ts3a227_dai_links[] = {
 	/* FE */
 	{
@@ -527,6 +558,7 @@ static struct snd_soc_dai_link mt8183_mt6358_ts3a227_dai_links[] = {
 		.ignore_suspend = 1,
 		.be_hw_params_fixup = mt8183_i2s_hw_params_fixup,
 		.ops = &mt8183_mt6358_i2s_ops,
+		.init = &mt8183_i2s2_init,
 		SND_SOC_DAILINK_REG(i2s2),
 	},
 	{
@@ -541,6 +573,7 @@ static struct snd_soc_dai_link mt8183_mt6358_ts3a227_dai_links[] = {
 		.dpcm_playback = 1,
 		.ignore_suspend = 1,
 		.ops = &mt8183_mt6358_i2s_ops,
+		.init = &mt8183_bt_init,
 		SND_SOC_DAILINK_REG(i2s5),
 	},
 	{
diff --git a/sound/soc/mediatek/mt8186/mt8186-afe-common.h b/sound/soc/mediatek/mt8186/mt8186-afe-common.h
index b8f03e1b7e49..d59258520995 100644
--- a/sound/soc/mediatek/mt8186/mt8186-afe-common.h
+++ b/sound/soc/mediatek/mt8186/mt8186-afe-common.h
@@ -189,6 +189,9 @@ unsigned int mt8186_rate_transform(struct device *dev,
 unsigned int mt8186_tdm_relatch_rate_transform(struct device *dev,
 					       unsigned int rate);
 
+int mt8186_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name,
+			     const char *secondary_i2s_name);
+
 int mt8186_dai_set_priv(struct mtk_base_afe *afe, int id,
 			int priv_size, const void *priv_data);
 
diff --git a/sound/soc/mediatek/mt8186/mt8186-afe-gpio.c b/sound/soc/mediatek/mt8186/mt8186-afe-gpio.c
index 274c0c8ec2f2..eda913fa147a 100644
--- a/sound/soc/mediatek/mt8186/mt8186-afe-gpio.c
+++ b/sound/soc/mediatek/mt8186/mt8186-afe-gpio.c
@@ -170,25 +170,25 @@ static int mt8186_afe_gpio_adda_ul(struct device *dev, bool enable)
 	if (enable) {
 		ret = mt8186_afe_gpio_select(dev, MT8186_AFE_GPIO_CLK_MISO_ON);
 		if (ret) {
-			dev_err(dev, "%s(), MISO CLK ON slect fail!\n", __func__);
+			dev_err(dev, "%s(), MISO CLK ON select fail!\n", __func__);
 			return ret;
 		}
 
 		ret = mt8186_afe_gpio_select(dev, MT8186_AFE_GPIO_DAT_MISO_ON);
 		if (ret) {
-			dev_err(dev, "%s(), MISO DAT ON slect fail!\n", __func__);
+			dev_err(dev, "%s(), MISO DAT ON select fail!\n", __func__);
 			return ret;
 		}
 	} else {
 		ret = mt8186_afe_gpio_select(dev, MT8186_AFE_GPIO_DAT_MISO_OFF);
 		if (ret) {
-			dev_err(dev, "%s(), MISO DAT OFF slect fail!\n", __func__);
+			dev_err(dev, "%s(), MISO DAT OFF select fail!\n", __func__);
 			return ret;
 		}
 
 		ret = mt8186_afe_gpio_select(dev, MT8186_AFE_GPIO_CLK_MISO_OFF);
 		if (ret) {
-			dev_err(dev, "%s(), MISO CLK OFF slect fail!\n", __func__);
+			dev_err(dev, "%s(), MISO CLK OFF select fail!\n", __func__);
 			return ret;
 		}
 	}
diff --git a/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c b/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c
index eb729ab00f5a..d7e94e6a19c7 100644
--- a/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c
+++ b/sound/soc/mediatek/mt8186/mt8186-afe-pcm.c
@@ -1359,6 +1359,9 @@ static const struct snd_soc_dapm_widget mt8186_memif_widgets[] = {
 	SND_SOC_DAPM_MUX("UL5_IN_MUX", SND_SOC_NOPM, 0, 0,
 			 &ul5_in_mux_control),
 
+	SND_SOC_DAPM_MIXER("DSP_DL1_VIRT", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("DSP_DL2_VIRT", SND_SOC_NOPM, 0, 0, NULL, 0),
+
 	SND_SOC_DAPM_INPUT("UL1_VIRTUAL_INPUT"),
 	SND_SOC_DAPM_INPUT("UL2_VIRTUAL_INPUT"),
 	SND_SOC_DAPM_INPUT("UL3_VIRTUAL_INPUT"),
diff --git a/sound/soc/mediatek/mt8186/mt8186-dai-i2s.c b/sound/soc/mediatek/mt8186/mt8186-dai-i2s.c
index ec79e2f2a54d..f07181be4370 100644
--- a/sound/soc/mediatek/mt8186/mt8186-dai-i2s.c
+++ b/sound/soc/mediatek/mt8186/mt8186-dai-i2s.c
@@ -44,7 +44,6 @@ struct mtk_afe_i2s_priv {
 	int low_jitter_en;
 	int master; /* only i2s0 has slave mode*/
 
-	const char *share_property_name;
 	int share_i2s_id;
 
 	int mclk_id;
@@ -658,9 +657,15 @@ static const struct snd_soc_dapm_route mtk_dai_i2s_routes[] = {
 	{"I2S1_CH1", "DL1_CH1 Switch", "DL1"},
 	{"I2S1_CH2", "DL1_CH2 Switch", "DL1"},
 
+	{"I2S1_CH1", "DL1_CH1 Switch", "DSP_DL1_VIRT"},
+	{"I2S1_CH2", "DL1_CH2 Switch", "DSP_DL1_VIRT"},
+
 	{"I2S1_CH1", "DL2_CH1 Switch", "DL2"},
 	{"I2S1_CH2", "DL2_CH2 Switch", "DL2"},
 
+	{"I2S1_CH1", "DL2_CH1 Switch", "DSP_DL2_VIRT"},
+	{"I2S1_CH2", "DL2_CH2 Switch", "DSP_DL2_VIRT"},
+
 	{"I2S1_CH1", "DL3_CH1 Switch", "DL3"},
 	{"I2S1_CH2", "DL3_CH2 Switch", "DL3"},
 
@@ -728,9 +733,15 @@ static const struct snd_soc_dapm_route mtk_dai_i2s_routes[] = {
 	{"I2S3_CH1", "DL1_CH1 Switch", "DL1"},
 	{"I2S3_CH2", "DL1_CH2 Switch", "DL1"},
 
+	{"I2S3_CH1", "DL1_CH1 Switch", "DSP_DL1_VIRT"},
+	{"I2S3_CH2", "DL1_CH2 Switch", "DSP_DL1_VIRT"},
+
 	{"I2S3_CH1", "DL2_CH1 Switch", "DL2"},
 	{"I2S3_CH2", "DL2_CH2 Switch", "DL2"},
 
+	{"I2S3_CH1", "DL2_CH1 Switch", "DSP_DL2_VIRT"},
+	{"I2S3_CH2", "DL2_CH2 Switch", "DSP_DL2_VIRT"},
+
 	{"I2S3_CH1", "DL3_CH1 Switch", "DL3"},
 	{"I2S3_CH2", "DL3_CH2 Switch", "DL3"},
 
@@ -968,7 +979,7 @@ static int mtk_dai_i2s_config(struct mtk_base_afe *afe,
 	}
 
 	/* set share i2s */
-	if (i2s_priv && i2s_priv->share_i2s_id >= 0) {
+	if (i2s_priv->share_i2s_id >= 0) {
 		ret = mtk_dai_i2s_config(afe, params, i2s_priv->share_i2s_id);
 		if (ret)
 			return ret;
@@ -1128,49 +1139,51 @@ static const struct mtk_afe_i2s_priv mt8186_i2s_priv[DAI_I2S_NUM] = {
 	[DAI_I2S0] = {
 		.id = MT8186_DAI_I2S_0,
 		.mclk_id = MT8186_I2S0_MCK,
-		.share_property_name = "i2s0-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S1] = {
 		.id = MT8186_DAI_I2S_1,
 		.mclk_id = MT8186_I2S1_MCK,
-		.share_property_name = "i2s1-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S2] = {
 		.id = MT8186_DAI_I2S_2,
 		.mclk_id = MT8186_I2S2_MCK,
-		.share_property_name = "i2s2-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S3] = {
 		.id = MT8186_DAI_I2S_3,
 		/*  clock gate naming is hf_faud_i2s4_m_ck*/
 		.mclk_id = MT8186_I2S4_MCK,
-		.share_property_name = "i2s3-share",
 		.share_i2s_id = -1,
 	}
 };
 
-static int mt8186_dai_i2s_get_share(struct mtk_base_afe *afe)
+/**
+ * mt8186_dai_i2s_set_share() - Set up I2S ports to share a single clock.
+ * @afe: Pointer to &struct mtk_base_afe
+ * @main_i2s_name: The name of the I2S port that will provide the clock
+ * @secondary_i2s_name: The name of the I2S port that will use this clock
+ */
+int mt8186_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name,
+			     const char *secondary_i2s_name)
 {
-	struct mt8186_afe_private *afe_priv = afe->platform_priv;
-	const struct device_node *of_node = afe->dev->of_node;
-	const char *of_str;
-	const char *property_name;
-	struct mtk_afe_i2s_priv *i2s_priv;
-	int i;
+	struct mtk_afe_i2s_priv *secondary_i2s_priv;
+	int main_i2s_id;
 
-	for (i = 0; i < DAI_I2S_NUM; i++) {
-		i2s_priv = afe_priv->dai_priv[mt8186_i2s_priv[i].id];
-		property_name = mt8186_i2s_priv[i].share_property_name;
-		if (of_property_read_string(of_node, property_name, &of_str))
-			continue;
-		i2s_priv->share_i2s_id = get_i2s_id_by_name(afe, of_str);
-	}
+	secondary_i2s_priv = get_i2s_priv_by_name(afe, secondary_i2s_name);
+	if (!secondary_i2s_priv)
+		return -EINVAL;
+
+	main_i2s_id = get_i2s_id_by_name(afe, main_i2s_name);
+	if (main_i2s_id < 0)
+		return main_i2s_id;
+
+	secondary_i2s_priv->share_i2s_id = main_i2s_id;
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mt8186_dai_i2s_set_share);
 
 static int mt8186_dai_i2s_set_priv(struct mtk_base_afe *afe)
 {
@@ -1214,10 +1227,5 @@ int mt8186_dai_i2s_register(struct mtk_base_afe *afe)
 	if (ret)
 		return ret;
 
-	/* parse share i2s */
-	ret = mt8186_dai_i2s_get_share(afe);
-	if (ret)
-		return ret;
-
 	return 0;
 }
diff --git a/sound/soc/mediatek/mt8186/mt8186-mt6366-da7219-max98357.c b/sound/soc/mediatek/mt8186/mt8186-mt6366-da7219-max98357.c
index 387f25cad809..cfca6bdee834 100644
--- a/sound/soc/mediatek/mt8186/mt8186-mt6366-da7219-max98357.c
+++ b/sound/soc/mediatek/mt8186/mt8186-mt6366-da7219-max98357.c
@@ -18,6 +18,8 @@
 #include "../../codecs/da7219.h"
 #include "../../codecs/mt6358.h"
 #include "../common/mtk-afe-platform-driver.h"
+#include "../common/mtk-dsp-sof-common.h"
+#include "../common/mtk-soc-card.h"
 #include "mt8186-afe-common.h"
 #include "mt8186-afe-clk.h"
 #include "mt8186-afe-gpio.h"
@@ -26,10 +28,27 @@
 #define DA7219_CODEC_DAI "da7219-hifi"
 #define DA7219_DEV_NAME "da7219.5-001a"
 
+#define SOF_DMA_DL1 "SOF_DMA_DL1"
+#define SOF_DMA_DL2 "SOF_DMA_DL2"
+#define SOF_DMA_UL1 "SOF_DMA_UL1"
+#define SOF_DMA_UL2 "SOF_DMA_UL2"
+
 struct mt8186_mt6366_da7219_max98357_priv {
 	struct snd_soc_jack headset_jack, hdmi_jack;
 };
 
+/* Headset jack detection DAPM pins */
+static struct snd_soc_jack_pin mt8186_jack_pins[] = {
+	{
+		.pin = "Headphones",
+		.mask = SND_JACK_HEADPHONE,
+	},
+	{
+		.pin = "Headset Mic",
+		.mask = SND_JACK_MICROPHONE,
+	},
+};
+
 static struct snd_soc_codec_conf mt8186_mt6366_da7219_max98357_codec_conf[] = {
 	{
 		.dlc = COMP_CODEC_CONF("mt6358-sound"),
@@ -47,19 +66,30 @@ static struct snd_soc_codec_conf mt8186_mt6366_da7219_max98357_codec_conf[] = {
 
 static int mt8186_da7219_init(struct snd_soc_pcm_runtime *rtd)
 {
-	struct mt8186_mt6366_da7219_max98357_priv *priv =
+	struct snd_soc_component *cmpnt_afe =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe);
+	struct mtk_soc_card_data *soc_card_data =
 		snd_soc_card_get_drvdata(rtd->card);
+	struct mt8186_mt6366_da7219_max98357_priv *priv = soc_card_data->mach_priv;
 	struct snd_soc_jack *jack = &priv->headset_jack;
 	struct snd_soc_component *cmpnt_codec =
 		asoc_rtd_to_codec(rtd, 0)->component;
 	int ret;
 
+	ret = mt8186_dai_i2s_set_share(afe, "I2S1", "I2S0");
+	if (ret) {
+		dev_err(rtd->dev, "Failed to set up shared clocks\n");
+		return ret;
+	}
+
 	/* Enable Headset and 4 Buttons Jack detection */
-	ret = snd_soc_card_jack_new(rtd->card, "Headset Jack",
+	ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack",
 				    SND_JACK_HEADSET | SND_JACK_BTN_0 |
 				    SND_JACK_BTN_1 | SND_JACK_BTN_2 |
 				    SND_JACK_BTN_3 | SND_JACK_LINEOUT,
-				    jack);
+				    jack, mt8186_jack_pins,
+				    ARRAY_SIZE(mt8186_jack_pins));
 	if (ret) {
 		dev_err(rtd->dev, "Headset Jack creation failed: %d\n", ret);
 		return ret;
@@ -152,12 +182,22 @@ static const struct snd_soc_ops mt8186_da7219_i2s_ops = {
 
 static int mt8186_mt6366_da7219_max98357_hdmi_init(struct snd_soc_pcm_runtime *rtd)
 {
+	struct snd_soc_component *cmpnt_afe =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe);
 	struct snd_soc_component *cmpnt_codec =
 		asoc_rtd_to_codec(rtd, 0)->component;
-	struct mt8186_mt6366_da7219_max98357_priv *priv =
+	struct mtk_soc_card_data *soc_card_data =
 		snd_soc_card_get_drvdata(rtd->card);
+	struct mt8186_mt6366_da7219_max98357_priv *priv = soc_card_data->mach_priv;
 	int ret;
 
+	ret = mt8186_dai_i2s_set_share(afe, "I2S3", "I2S2");
+	if (ret) {
+		dev_err(rtd->dev, "Failed to set up shared clocks\n");
+		return ret;
+	}
+
 	ret = snd_soc_card_jack_new(rtd->card, "HDMI Jack", SND_JACK_LINEOUT, &priv->hdmi_jack);
 	if (ret) {
 		dev_err(rtd->dev, "HDMI Jack creation failed: %d\n", ret);
@@ -201,6 +241,24 @@ static int mt8186_anx7625_i2s_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
 	return mt8186_hw_params_fixup(rtd, params, SNDRV_PCM_FORMAT_S24_LE);
 }
 
+/* fixup the BE DAI link to match any values from topology */
+static int mt8186_sof_dai_link_fixup(struct snd_soc_pcm_runtime *rtd,
+				     struct snd_pcm_hw_params *params)
+{
+	int ret;
+
+	ret = mtk_sof_dai_link_fixup(rtd, params);
+
+	if (!strcmp(rtd->dai_link->name, "I2S0") ||
+	    !strcmp(rtd->dai_link->name, "I2S1") ||
+	    !strcmp(rtd->dai_link->name, "I2S2"))
+		mt8186_i2s_hw_params_fixup(rtd, params);
+	else if (!strcmp(rtd->dai_link->name, "I2S3"))
+		mt8186_anx7625_i2s_hw_params_fixup(rtd, params);
+
+	return ret;
+}
+
 static int mt8186_mt6366_da7219_max98357_playback_startup(struct snd_pcm_substream *substream)
 {
 	static const unsigned int rates[] = {
@@ -474,6 +532,33 @@ SND_SOC_DAILINK_DEFS(hostless_src_aaudio,
 		     DAILINK_COMP_ARRAY(COMP_CPU("Hostless SRC AAudio DAI")),
 		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
 		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(AFE_SOF_DL1,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_DL1")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+SND_SOC_DAILINK_DEFS(AFE_SOF_DL2,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_DL2")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+SND_SOC_DAILINK_DEFS(AFE_SOF_UL1,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL1")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+SND_SOC_DAILINK_DEFS(AFE_SOF_UL2,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL2")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+static const struct sof_conn_stream g_sof_conn_streams[] = {
+	{ "I2S1", "AFE_SOF_DL1", SOF_DMA_DL1, SNDRV_PCM_STREAM_PLAYBACK},
+	{ "I2S3", "AFE_SOF_DL2", SOF_DMA_DL2, SNDRV_PCM_STREAM_PLAYBACK},
+	{ "Primary Codec", "AFE_SOF_UL1", SOF_DMA_UL1, SNDRV_PCM_STREAM_CAPTURE},
+	{ "I2S0", "AFE_SOF_UL2", SOF_DMA_UL2, SNDRV_PCM_STREAM_CAPTURE},
+};
+
 static struct snd_soc_dai_link mt8186_mt6366_da7219_max98357_dai_links[] = {
 	/* Front End DAI links */
 	{
@@ -848,30 +933,75 @@ static struct snd_soc_dai_link mt8186_mt6366_da7219_max98357_dai_links[] = {
 		.ignore_suspend = 1,
 		SND_SOC_DAILINK_REG(hostless_ul6),
 	},
+	/* SOF BE */
+	{
+		.name = "AFE_SOF_DL1",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		SND_SOC_DAILINK_REG(AFE_SOF_DL1),
+	},
+	{
+		.name = "AFE_SOF_DL2",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		SND_SOC_DAILINK_REG(AFE_SOF_DL2),
+	},
+	{
+		.name = "AFE_SOF_UL1",
+		.no_pcm = 1,
+		.dpcm_capture = 1,
+		SND_SOC_DAILINK_REG(AFE_SOF_UL1),
+	},
+	{
+		.name = "AFE_SOF_UL2",
+		.no_pcm = 1,
+		.dpcm_capture = 1,
+		SND_SOC_DAILINK_REG(AFE_SOF_UL2),
+	},
 };
 
 static const struct snd_soc_dapm_widget
 mt8186_mt6366_da7219_max98357_widgets[] = {
 	SND_SOC_DAPM_SPK("Speakers", NULL),
+	SND_SOC_DAPM_HP("Headphones", NULL),
+	SND_SOC_DAPM_MIC("Headset Mic", NULL),
 	SND_SOC_DAPM_OUTPUT("HDMI1"),
+	SND_SOC_DAPM_MIXER(SOF_DMA_DL1, SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER(SOF_DMA_DL2, SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER(SOF_DMA_UL1, SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER(SOF_DMA_UL2, SND_SOC_NOPM, 0, 0, NULL, 0),
 };
 
 static const struct snd_soc_dapm_route
 mt8186_mt6366_da7219_max98357_routes[] = {
 	/* SPK */
 	{ "Speakers", NULL, "Speaker"},
+	/* Headset */
+	{ "Headphones", NULL, "HPL" },
+	{ "Headphones", NULL, "HPR" },
+	{ "MIC", NULL, "Headset Mic" },
 	/* HDMI */
 	{ "HDMI1", NULL, "TX"},
+	/* SOF Uplink */
+	{SOF_DMA_UL1, NULL, "UL1_CH1"},
+	{SOF_DMA_UL1, NULL, "UL1_CH2"},
+	{SOF_DMA_UL2, NULL, "UL2_CH1"},
+	{SOF_DMA_UL2, NULL, "UL2_CH2"},
+	/* SOF Downlink */
+	{"DSP_DL1_VIRT", NULL, SOF_DMA_DL1},
+	{"DSP_DL2_VIRT", NULL, SOF_DMA_DL2},
 };
 
 static const struct snd_kcontrol_new
 mt8186_mt6366_da7219_max98357_controls[] = {
 	SOC_DAPM_PIN_SWITCH("Speakers"),
+	SOC_DAPM_PIN_SWITCH("Headphones"),
+	SOC_DAPM_PIN_SWITCH("Headset Mic"),
 	SOC_DAPM_PIN_SWITCH("HDMI1"),
 };
 
 static struct snd_soc_card mt8186_mt6366_da7219_max98357_soc_card = {
-	.name = "mt8186_mt6366_da7219_max98357",
+	.name = "mt8186_da7219_max98357",
 	.owner = THIS_MODULE,
 	.dai_link = mt8186_mt6366_da7219_max98357_dai_links,
 	.num_links = ARRAY_SIZE(mt8186_mt6366_da7219_max98357_dai_links),
@@ -889,8 +1019,10 @@ static int mt8186_mt6366_da7219_max98357_dev_probe(struct platform_device *pdev)
 {
 	struct snd_soc_card *card;
 	struct snd_soc_dai_link *dai_link;
-	struct mt8186_mt6366_da7219_max98357_priv *priv;
-	struct device_node *platform_node, *headset_codec, *playback_codec;
+	struct mtk_soc_card_data *soc_card_data;
+	struct mt8186_mt6366_da7219_max98357_priv *mach_priv;
+	struct device_node *platform_node, *headset_codec, *playback_codec, *adsp_node;
+	int sof_on = 0;
 	int ret, i;
 
 	card = (struct snd_soc_card *)device_get_match_data(&pdev->dev);
@@ -898,11 +1030,60 @@ static int mt8186_mt6366_da7219_max98357_dev_probe(struct platform_device *pdev)
 		return -EINVAL;
 	card->dev = &pdev->dev;
 
+	soc_card_data = devm_kzalloc(&pdev->dev, sizeof(*soc_card_data), GFP_KERNEL);
+	if (!soc_card_data)
+		return -ENOMEM;
+	mach_priv = devm_kzalloc(&pdev->dev, sizeof(*mach_priv), GFP_KERNEL);
+	if (!mach_priv)
+		return -ENOMEM;
+
+	soc_card_data->mach_priv = mach_priv;
+
+	adsp_node = of_parse_phandle(pdev->dev.of_node, "mediatek,adsp", 0);
+	if (adsp_node) {
+		struct mtk_sof_priv *sof_priv;
+
+		sof_priv = devm_kzalloc(&pdev->dev, sizeof(*sof_priv), GFP_KERNEL);
+		if (!sof_priv) {
+			ret = -ENOMEM;
+			goto err_adsp_node;
+		}
+		sof_priv->conn_streams = g_sof_conn_streams;
+		sof_priv->num_streams = ARRAY_SIZE(g_sof_conn_streams);
+		sof_priv->sof_dai_link_fixup = mt8186_sof_dai_link_fixup;
+		soc_card_data->sof_priv = sof_priv;
+		card->probe = mtk_sof_card_probe;
+		card->late_probe = mtk_sof_card_late_probe;
+		if (!card->topology_shortname_created) {
+			snprintf(card->topology_shortname, 32, "sof-%s", card->name);
+			card->topology_shortname_created = true;
+		}
+		card->name = card->topology_shortname;
+		sof_on = 1;
+	} else {
+		dev_info(&pdev->dev, "Probe without adsp\n");
+	}
+
+	if (of_property_read_bool(pdev->dev.of_node, "mediatek,dai-link")) {
+		ret = mtk_sof_dailink_parse_of(card, pdev->dev.of_node,
+					       "mediatek,dai-link",
+					       mt8186_mt6366_da7219_max98357_dai_links,
+					       ARRAY_SIZE(mt8186_mt6366_da7219_max98357_dai_links));
+		if (ret) {
+			dev_dbg(&pdev->dev, "Parse dai-link fail\n");
+			goto err_adsp_node;
+		}
+	} else {
+		if (!sof_on)
+			card->num_links = ARRAY_SIZE(mt8186_mt6366_da7219_max98357_dai_links)
+					- ARRAY_SIZE(g_sof_conn_streams);
+	}
+
 	platform_node = of_parse_phandle(pdev->dev.of_node, "mediatek,platform", 0);
 	if (!platform_node) {
 		ret = -EINVAL;
 		dev_err_probe(&pdev->dev, ret, "Property 'platform' missing or invalid\n");
-		return ret;
+		goto err_platform_node;
 	}
 
 	playback_codec = of_get_child_by_name(pdev->dev.of_node, "playback-codecs");
@@ -941,17 +1122,14 @@ static int mt8186_mt6366_da7219_max98357_dev_probe(struct platform_device *pdev)
 			goto err_probe;
 		}
 
-		if (!dai_link->platforms->name)
-			dai_link->platforms->of_node = platform_node;
-	}
+		if (!strncmp(dai_link->name, "AFE_SOF", strlen("AFE_SOF")) && sof_on)
+			dai_link->platforms->of_node = adsp_node;
 
-	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv) {
-		ret = -ENOMEM;
-		goto err_probe;
+		if (!dai_link->platforms->name && !dai_link->platforms->of_node)
+			dai_link->platforms->of_node = platform_node;
 	}
 
-	snd_soc_card_set_drvdata(card, priv);
+	snd_soc_card_set_drvdata(card, soc_card_data);
 
 	ret = mt8186_afe_gpio_init(&pdev->dev);
 	if (ret) {
@@ -969,6 +1147,9 @@ err_headset_codec:
 	of_node_put(playback_codec);
 err_playback_codec:
 	of_node_put(platform_node);
+err_platform_node:
+err_adsp_node:
+	of_node_put(adsp_node);
 
 	return ret;
 }
diff --git a/sound/soc/mediatek/mt8186/mt8186-mt6366-rt1019-rt5682s.c b/sound/soc/mediatek/mt8186/mt8186-mt6366-rt1019-rt5682s.c
index 891146fd6c2b..2414c5b77233 100644
--- a/sound/soc/mediatek/mt8186/mt8186-mt6366-rt1019-rt5682s.c
+++ b/sound/soc/mediatek/mt8186/mt8186-mt6366-rt1019-rt5682s.c
@@ -19,6 +19,8 @@
 #include "../../codecs/mt6358.h"
 #include "../../codecs/rt5682.h"
 #include "../common/mtk-afe-platform-driver.h"
+#include "../common/mtk-dsp-sof-common.h"
+#include "../common/mtk-soc-card.h"
 #include "mt8186-afe-common.h"
 #include "mt8186-afe-clk.h"
 #include "mt8186-afe-gpio.h"
@@ -30,10 +32,27 @@
 #define RT5682S_CODEC_DAI	"rt5682s-aif1"
 #define RT5682S_DEV0_NAME	"rt5682s.5-001a"
 
+#define SOF_DMA_DL1 "SOF_DMA_DL1"
+#define SOF_DMA_DL2 "SOF_DMA_DL2"
+#define SOF_DMA_UL1 "SOF_DMA_UL1"
+#define SOF_DMA_UL2 "SOF_DMA_UL2"
+
 struct mt8186_mt6366_rt1019_rt5682s_priv {
 	struct snd_soc_jack headset_jack, hdmi_jack;
 };
 
+/* Headset jack detection DAPM pins */
+static struct snd_soc_jack_pin mt8186_jack_pins[] = {
+	{
+		.pin = "Headphone",
+		.mask = SND_JACK_HEADPHONE,
+	},
+	{
+		.pin = "Headset Mic",
+		.mask = SND_JACK_MICROPHONE,
+	},
+};
+
 static struct snd_soc_codec_conf mt8186_mt6366_rt1019_rt5682s_codec_conf[] = {
 	{
 		.dlc = COMP_CODEC_CONF("mt6358-sound"),
@@ -51,18 +70,29 @@ static struct snd_soc_codec_conf mt8186_mt6366_rt1019_rt5682s_codec_conf[] = {
 
 static int mt8186_rt5682s_init(struct snd_soc_pcm_runtime *rtd)
 {
-	struct mt8186_mt6366_rt1019_rt5682s_priv *priv =
+	struct snd_soc_component *cmpnt_afe =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe);
+	struct mtk_soc_card_data *soc_card_data =
 		snd_soc_card_get_drvdata(rtd->card);
+	struct mt8186_mt6366_rt1019_rt5682s_priv *priv = soc_card_data->mach_priv;
 	struct snd_soc_jack *jack = &priv->headset_jack;
 	struct snd_soc_component *cmpnt_codec =
 		asoc_rtd_to_codec(rtd, 0)->component;
 	int ret;
 
-	ret = snd_soc_card_jack_new(rtd->card, "Headset Jack",
+	ret = mt8186_dai_i2s_set_share(afe, "I2S1", "I2S0");
+	if (ret) {
+		dev_err(rtd->dev, "Failed to set up shared clocks\n");
+		return ret;
+	}
+
+	ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack",
 				    SND_JACK_HEADSET | SND_JACK_BTN_0 |
 				    SND_JACK_BTN_1 | SND_JACK_BTN_2 |
 				    SND_JACK_BTN_3,
-				    jack);
+				    jack, mt8186_jack_pins,
+				    ARRAY_SIZE(mt8186_jack_pins));
 	if (ret) {
 		dev_err(rtd->dev, "Headset Jack creation failed: %d\n", ret);
 		return ret;
@@ -128,12 +158,22 @@ static const struct snd_soc_ops mt8186_rt5682s_i2s_ops = {
 
 static int mt8186_mt6366_rt1019_rt5682s_hdmi_init(struct snd_soc_pcm_runtime *rtd)
 {
+	struct snd_soc_component *cmpnt_afe =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe);
 	struct snd_soc_component *cmpnt_codec =
 		asoc_rtd_to_codec(rtd, 0)->component;
-	struct mt8186_mt6366_rt1019_rt5682s_priv *priv =
+	struct mtk_soc_card_data *soc_card_data =
 		snd_soc_card_get_drvdata(rtd->card);
+	struct mt8186_mt6366_rt1019_rt5682s_priv *priv = soc_card_data->mach_priv;
 	int ret;
 
+	ret = mt8186_dai_i2s_set_share(afe, "I2S3", "I2S2");
+	if (ret) {
+		dev_err(rtd->dev, "Failed to set up shared clocks\n");
+		return ret;
+	}
+
 	ret = snd_soc_card_jack_new(rtd->card, "HDMI Jack", SND_JACK_LINEOUT, &priv->hdmi_jack);
 	if (ret) {
 		dev_err(rtd->dev, "HDMI Jack creation failed: %d\n", ret);
@@ -177,6 +217,24 @@ static int mt8186_it6505_i2s_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
 	return mt8186_hw_params_fixup(rtd, params, SNDRV_PCM_FORMAT_S32_LE);
 }
 
+/* fixup the BE DAI link to match any values from topology */
+static int mt8186_sof_dai_link_fixup(struct snd_soc_pcm_runtime *rtd,
+				     struct snd_pcm_hw_params *params)
+{
+	int ret;
+
+	ret = mtk_sof_dai_link_fixup(rtd, params);
+
+	if (!strcmp(rtd->dai_link->name, "I2S0") ||
+	    !strcmp(rtd->dai_link->name, "I2S1") ||
+	    !strcmp(rtd->dai_link->name, "I2S2"))
+		mt8186_i2s_hw_params_fixup(rtd, params);
+	else if (!strcmp(rtd->dai_link->name, "I2S3"))
+		mt8186_it6505_i2s_hw_params_fixup(rtd, params);
+
+	return ret;
+}
+
 static int mt8186_mt6366_rt1019_rt5682s_playback_startup(struct snd_pcm_substream *substream)
 {
 	static const unsigned int rates[] = {
@@ -450,6 +508,33 @@ SND_SOC_DAILINK_DEFS(hostless_src_aaudio,
 		     DAILINK_COMP_ARRAY(COMP_CPU("Hostless SRC AAudio DAI")),
 		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
 		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(AFE_SOF_DL1,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_DL1")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+SND_SOC_DAILINK_DEFS(AFE_SOF_DL2,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_DL2")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+SND_SOC_DAILINK_DEFS(AFE_SOF_UL1,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL1")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+SND_SOC_DAILINK_DEFS(AFE_SOF_UL2,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL2")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+static const struct sof_conn_stream g_sof_conn_streams[] = {
+	{ "I2S1", "AFE_SOF_DL1", SOF_DMA_DL1, SNDRV_PCM_STREAM_PLAYBACK},
+	{ "I2S3", "AFE_SOF_DL2", SOF_DMA_DL2, SNDRV_PCM_STREAM_PLAYBACK},
+	{ "Primary Codec", "AFE_SOF_UL1", SOF_DMA_UL1, SNDRV_PCM_STREAM_CAPTURE},
+	{ "I2S0", "AFE_SOF_UL2", SOF_DMA_UL2, SNDRV_PCM_STREAM_CAPTURE},
+};
+
 static struct snd_soc_dai_link mt8186_mt6366_rt1019_rt5682s_dai_links[] = {
 	/* Front End DAI links */
 	{
@@ -824,30 +909,75 @@ static struct snd_soc_dai_link mt8186_mt6366_rt1019_rt5682s_dai_links[] = {
 		.ignore_suspend = 1,
 		SND_SOC_DAILINK_REG(hostless_ul6),
 	},
+	/* SOF BE */
+	{
+		.name = "AFE_SOF_DL1",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		SND_SOC_DAILINK_REG(AFE_SOF_DL1),
+	},
+	{
+		.name = "AFE_SOF_DL2",
+		.no_pcm = 1,
+		.dpcm_playback = 1,
+		SND_SOC_DAILINK_REG(AFE_SOF_DL2),
+	},
+	{
+		.name = "AFE_SOF_UL1",
+		.no_pcm = 1,
+		.dpcm_capture = 1,
+		SND_SOC_DAILINK_REG(AFE_SOF_UL1),
+	},
+	{
+		.name = "AFE_SOF_UL2",
+		.no_pcm = 1,
+		.dpcm_capture = 1,
+		SND_SOC_DAILINK_REG(AFE_SOF_UL2),
+	},
 };
 
 static const struct snd_soc_dapm_widget
 mt8186_mt6366_rt1019_rt5682s_widgets[] = {
 	SND_SOC_DAPM_SPK("Speakers", NULL),
+	SND_SOC_DAPM_HP("Headphone", NULL),
+	SND_SOC_DAPM_MIC("Headset Mic", NULL),
 	SND_SOC_DAPM_OUTPUT("HDMI1"),
+	SND_SOC_DAPM_MIXER(SOF_DMA_DL1, SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER(SOF_DMA_DL2, SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER(SOF_DMA_UL1, SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER(SOF_DMA_UL2, SND_SOC_NOPM, 0, 0, NULL, 0),
 };
 
 static const struct snd_soc_dapm_route
 mt8186_mt6366_rt1019_rt5682s_routes[] = {
 	/* SPK */
 	{ "Speakers", NULL, "Speaker" },
+	/* Headset */
+	{ "Headphone", NULL, "HPOL" },
+	{ "Headphone", NULL, "HPOR" },
+	{ "IN1P", NULL, "Headset Mic" },
 	/* HDMI */
 	{ "HDMI1", NULL, "TX" },
+	/* SOF Uplink */
+	{SOF_DMA_UL1, NULL, "UL1_CH1"},
+	{SOF_DMA_UL1, NULL, "UL1_CH2"},
+	{SOF_DMA_UL2, NULL, "UL2_CH1"},
+	{SOF_DMA_UL2, NULL, "UL2_CH2"},
+	/* SOF Downlink */
+	{"DSP_DL1_VIRT", NULL, SOF_DMA_DL1},
+	{"DSP_DL2_VIRT", NULL, SOF_DMA_DL2},
 };
 
 static const struct snd_kcontrol_new
 mt8186_mt6366_rt1019_rt5682s_controls[] = {
 	SOC_DAPM_PIN_SWITCH("Speakers"),
+	SOC_DAPM_PIN_SWITCH("Headphone"),
+	SOC_DAPM_PIN_SWITCH("Headset Mic"),
 	SOC_DAPM_PIN_SWITCH("HDMI1"),
 };
 
 static struct snd_soc_card mt8186_mt6366_rt1019_rt5682s_soc_card = {
-	.name = "mt8186_mt6366_rt1019_rt5682s",
+	.name = "mt8186_rt1019_rt5682s",
 	.owner = THIS_MODULE,
 	.dai_link = mt8186_mt6366_rt1019_rt5682s_dai_links,
 	.num_links = ARRAY_SIZE(mt8186_mt6366_rt1019_rt5682s_dai_links),
@@ -865,8 +995,10 @@ static int mt8186_mt6366_rt1019_rt5682s_dev_probe(struct platform_device *pdev)
 {
 	struct snd_soc_card *card;
 	struct snd_soc_dai_link *dai_link;
-	struct mt8186_mt6366_rt1019_rt5682s_priv *priv;
-	struct device_node *platform_node, *headset_codec, *playback_codec;
+	struct mtk_soc_card_data *soc_card_data;
+	struct mt8186_mt6366_rt1019_rt5682s_priv *mach_priv;
+	struct device_node *platform_node, *headset_codec, *playback_codec, *adsp_node;
+	int sof_on = 0;
 	int ret, i;
 
 	card = (struct snd_soc_card *)device_get_match_data(&pdev->dev);
@@ -874,11 +1006,60 @@ static int mt8186_mt6366_rt1019_rt5682s_dev_probe(struct platform_device *pdev)
 		return -EINVAL;
 	card->dev = &pdev->dev;
 
+	soc_card_data = devm_kzalloc(&pdev->dev, sizeof(*soc_card_data), GFP_KERNEL);
+	if (!soc_card_data)
+		return -ENOMEM;
+	mach_priv = devm_kzalloc(&pdev->dev, sizeof(*mach_priv), GFP_KERNEL);
+	if (!mach_priv)
+		return -ENOMEM;
+
+	soc_card_data->mach_priv = mach_priv;
+
+	adsp_node = of_parse_phandle(pdev->dev.of_node, "mediatek,adsp", 0);
+	if (adsp_node) {
+		struct mtk_sof_priv *sof_priv;
+
+		sof_priv = devm_kzalloc(&pdev->dev, sizeof(*sof_priv), GFP_KERNEL);
+		if (!sof_priv) {
+			ret = -ENOMEM;
+			goto err_adsp_node;
+		}
+		sof_priv->conn_streams = g_sof_conn_streams;
+		sof_priv->num_streams = ARRAY_SIZE(g_sof_conn_streams);
+		sof_priv->sof_dai_link_fixup = mt8186_sof_dai_link_fixup;
+		soc_card_data->sof_priv = sof_priv;
+		card->probe = mtk_sof_card_probe;
+		card->late_probe = mtk_sof_card_late_probe;
+		if (!card->topology_shortname_created) {
+			snprintf(card->topology_shortname, 32, "sof-%s", card->name);
+			card->topology_shortname_created = true;
+		}
+		card->name = card->topology_shortname;
+		sof_on = 1;
+	} else {
+		dev_info(&pdev->dev, "Probe without adsp\n");
+	}
+
+	if (of_property_read_bool(pdev->dev.of_node, "mediatek,dai-link")) {
+		ret = mtk_sof_dailink_parse_of(card, pdev->dev.of_node,
+					       "mediatek,dai-link",
+					       mt8186_mt6366_rt1019_rt5682s_dai_links,
+					       ARRAY_SIZE(mt8186_mt6366_rt1019_rt5682s_dai_links));
+		if (ret) {
+			dev_dbg(&pdev->dev, "Parse dai-link fail\n");
+			goto err_adsp_node;
+		}
+	} else {
+		if (!sof_on)
+			card->num_links = ARRAY_SIZE(mt8186_mt6366_rt1019_rt5682s_dai_links)
+					- ARRAY_SIZE(g_sof_conn_streams);
+	}
+
 	platform_node = of_parse_phandle(pdev->dev.of_node, "mediatek,platform", 0);
 	if (!platform_node) {
 		ret = -EINVAL;
 		dev_err_probe(&pdev->dev, ret, "Property 'platform' missing or invalid\n");
-		return ret;
+		goto err_platform_node;
 	}
 
 	playback_codec = of_get_child_by_name(pdev->dev.of_node, "playback-codecs");
@@ -917,17 +1098,14 @@ static int mt8186_mt6366_rt1019_rt5682s_dev_probe(struct platform_device *pdev)
 			goto err_probe;
 		}
 
-		if (!dai_link->platforms->name)
-			dai_link->platforms->of_node = platform_node;
-	}
+		if (!strncmp(dai_link->name, "AFE_SOF", strlen("AFE_SOF")) && sof_on)
+			dai_link->platforms->of_node = adsp_node;
 
-	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv) {
-		ret = -ENOMEM;
-		goto err_probe;
+		if (!dai_link->platforms->name && !dai_link->platforms->of_node)
+			dai_link->platforms->of_node = platform_node;
 	}
 
-	snd_soc_card_set_drvdata(card, priv);
+	snd_soc_card_set_drvdata(card, soc_card_data);
 
 	ret = mt8186_afe_gpio_init(&pdev->dev);
 	if (ret) {
@@ -945,6 +1123,9 @@ err_headset_codec:
 	of_node_put(playback_codec);
 err_playback_codec:
 	of_node_put(platform_node);
+err_platform_node:
+err_adsp_node:
+	of_node_put(adsp_node);
 
 	return ret;
 }
diff --git a/sound/soc/mediatek/mt8192/mt8192-afe-common.h b/sound/soc/mediatek/mt8192/mt8192-afe-common.h
index d55eff46cc7f..ad461dcb6ee1 100644
--- a/sound/soc/mediatek/mt8192/mt8192-afe-common.h
+++ b/sound/soc/mediatek/mt8192/mt8192-afe-common.h
@@ -159,6 +159,9 @@ int mt8192_dai_src_register(struct mtk_base_afe *afe);
 int mt8192_dai_pcm_register(struct mtk_base_afe *afe);
 int mt8192_dai_tdm_register(struct mtk_base_afe *afe);
 
+int mt8192_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name,
+			     const char *secondary_i2s_name);
+
 unsigned int mt8192_general_rate_transform(struct device *dev,
 					   unsigned int rate);
 unsigned int mt8192_rate_transform(struct device *dev,
diff --git a/sound/soc/mediatek/mt8192/mt8192-dai-i2s.c b/sound/soc/mediatek/mt8192/mt8192-dai-i2s.c
index 5b29340f9516..ea516d63d94d 100644
--- a/sound/soc/mediatek/mt8192/mt8192-dai-i2s.c
+++ b/sound/soc/mediatek/mt8192/mt8192-dai-i2s.c
@@ -45,7 +45,6 @@ struct mtk_afe_i2s_priv {
 	int rate; /* for determine which apll to use */
 	int low_jitter_en;
 
-	const char *share_property_name;
 	int share_i2s_id;
 
 	int mclk_id;
@@ -1984,78 +1983,75 @@ static const struct mtk_afe_i2s_priv mt8192_i2s_priv[DAI_I2S_NUM] = {
 	[DAI_I2S0] = {
 		.id = MT8192_DAI_I2S_0,
 		.mclk_id = MT8192_I2S0_MCK,
-		.share_property_name = "i2s0-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S1] = {
 		.id = MT8192_DAI_I2S_1,
 		.mclk_id = MT8192_I2S1_MCK,
-		.share_property_name = "i2s1-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S2] = {
 		.id = MT8192_DAI_I2S_2,
 		.mclk_id = MT8192_I2S2_MCK,
-		.share_property_name = "i2s2-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S3] = {
 		.id = MT8192_DAI_I2S_3,
 		.mclk_id = MT8192_I2S3_MCK,
-		.share_property_name = "i2s3-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S5] = {
 		.id = MT8192_DAI_I2S_5,
 		.mclk_id = MT8192_I2S5_MCK,
-		.share_property_name = "i2s5-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S6] = {
 		.id = MT8192_DAI_I2S_6,
 		.mclk_id = MT8192_I2S6_MCK,
-		.share_property_name = "i2s6-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S7] = {
 		.id = MT8192_DAI_I2S_7,
 		.mclk_id = MT8192_I2S7_MCK,
-		.share_property_name = "i2s7-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S8] = {
 		.id = MT8192_DAI_I2S_8,
 		.mclk_id = MT8192_I2S8_MCK,
-		.share_property_name = "i2s8-share",
 		.share_i2s_id = -1,
 	},
 	[DAI_I2S9] = {
 		.id = MT8192_DAI_I2S_9,
 		.mclk_id = MT8192_I2S9_MCK,
-		.share_property_name = "i2s9-share",
 		.share_i2s_id = -1,
 	},
 };
 
-static int mt8192_dai_i2s_get_share(struct mtk_base_afe *afe)
+/**
+ * mt8192_dai_i2s_set_share() - Set up I2S ports to share a single clock.
+ * @afe: Pointer to &struct mtk_base_afe
+ * @main_i2s_name: The name of the I2S port that will provide the clock
+ * @secondary_i2s_name: The name of the I2S port that will use this clock
+ */
+int mt8192_dai_i2s_set_share(struct mtk_base_afe *afe, const char *main_i2s_name,
+			     const char *secondary_i2s_name)
 {
-	struct mt8192_afe_private *afe_priv = afe->platform_priv;
-	const struct device_node *of_node = afe->dev->of_node;
-	const char *of_str;
-	const char *property_name;
-	struct mtk_afe_i2s_priv *i2s_priv;
-	int i;
+	struct mtk_afe_i2s_priv *secondary_i2s_priv;
+	int main_i2s_id;
 
-	for (i = 0; i < DAI_I2S_NUM; i++) {
-		i2s_priv = afe_priv->dai_priv[mt8192_i2s_priv[i].id];
-		property_name = mt8192_i2s_priv[i].share_property_name;
-		if (of_property_read_string(of_node, property_name, &of_str))
-			continue;
-		i2s_priv->share_i2s_id = get_i2s_id_by_name(afe, of_str);
-	}
+	secondary_i2s_priv = get_i2s_priv_by_name(afe, secondary_i2s_name);
+	if (!secondary_i2s_priv)
+		return -EINVAL;
+
+	main_i2s_id = get_i2s_id_by_name(afe, main_i2s_name);
+	if (main_i2s_id < 0)
+		return main_i2s_id;
+
+	secondary_i2s_priv->share_i2s_id = main_i2s_id;
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(mt8192_dai_i2s_set_share);
 
 static int mt8192_dai_i2s_set_priv(struct mtk_base_afe *afe)
 {
@@ -2101,10 +2097,5 @@ int mt8192_dai_i2s_register(struct mtk_base_afe *afe)
 	if (ret)
 		return ret;
 
-	/* parse share i2s */
-	ret = mt8192_dai_i2s_get_share(afe);
-	if (ret)
-		return ret;
-
 	return 0;
 }
diff --git a/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c b/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c
index d0f9d66627b1..b93c3237ef2d 100644
--- a/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c
+++ b/sound/soc/mediatek/mt8192/mt8192-mt6359-rt1015-rt5682.c
@@ -24,6 +24,8 @@
 #include "mt8192-afe-clk.h"
 #include "mt8192-afe-gpio.h"
 
+#define DRIVER_NAME "mt8192_mt6359"
+
 #define RT1015_CODEC_DAI	"rt1015-aif"
 #define RT1015_DEV0_NAME	"rt1015.1-0028"
 #define RT1015_DEV1_NAME	"rt1015.1-0029"
@@ -41,6 +43,18 @@ struct mt8192_mt6359_priv {
 	struct snd_soc_jack hdmi_jack;
 };
 
+/* Headset jack detection DAPM pins */
+static struct snd_soc_jack_pin mt8192_jack_pins[] = {
+	{
+		.pin = "Headphone Jack",
+		.mask = SND_JACK_HEADPHONE,
+	},
+	{
+		.pin = "Headset Mic",
+		.mask = SND_JACK_MICROPHONE,
+	},
+};
+
 static int mt8192_rt1015_i2s_hw_params(struct snd_pcm_substream *substream,
 				       struct snd_pcm_hw_params *params)
 {
@@ -311,17 +325,27 @@ static int mt8192_mt6359_init(struct snd_soc_pcm_runtime *rtd)
 
 static int mt8192_rt5682_init(struct snd_soc_pcm_runtime *rtd)
 {
+	struct snd_soc_component *cmpnt_afe =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt_afe);
 	struct snd_soc_component *cmpnt_codec =
 		asoc_rtd_to_codec(rtd, 0)->component;
 	struct mt8192_mt6359_priv *priv = snd_soc_card_get_drvdata(rtd->card);
 	struct snd_soc_jack *jack = &priv->headset_jack;
 	int ret;
 
-	ret = snd_soc_card_jack_new(rtd->card, "Headset Jack",
+	ret = mt8192_dai_i2s_set_share(afe, "I2S8", "I2S9");
+	if (ret) {
+		dev_err(rtd->dev, "Failed to set up shared clocks\n");
+		return ret;
+	}
+
+	ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack",
 				    SND_JACK_HEADSET | SND_JACK_BTN_0 |
 				    SND_JACK_BTN_1 | SND_JACK_BTN_2 |
 				    SND_JACK_BTN_3,
-				    jack);
+				    jack, mt8192_jack_pins,
+				    ARRAY_SIZE(mt8192_jack_pins));
 	if (ret) {
 		dev_err(rtd->dev, "Headset Jack creation failed: %d\n", ret);
 		return ret;
@@ -1048,6 +1072,7 @@ static struct snd_soc_codec_conf rt1015_amp_conf[] = {
 
 static struct snd_soc_card mt8192_mt6359_rt1015_rt5682_card = {
 	.name = RT1015_RT5682_CARD_NAME,
+	.driver_name = DRIVER_NAME,
 	.owner = THIS_MODULE,
 	.dai_link = mt8192_mt6359_dai_links,
 	.num_links = ARRAY_SIZE(mt8192_mt6359_dai_links),
@@ -1083,6 +1108,7 @@ static const struct snd_kcontrol_new mt8192_mt6359_rt1015p_rt5682x_controls[] =
 };
 
 static struct snd_soc_card mt8192_mt6359_rt1015p_rt5682x_card = {
+	.driver_name = DRIVER_NAME,
 	.owner = THIS_MODULE,
 	.dai_link = mt8192_mt6359_dai_links,
 	.num_links = ARRAY_SIZE(mt8192_mt6359_dai_links),
@@ -1244,7 +1270,7 @@ static const struct dev_pm_ops mt8192_mt6359_pm_ops = {
 
 static struct platform_driver mt8192_mt6359_driver = {
 	.driver = {
-		.name = "mt8192_mt6359",
+		.name = DRIVER_NAME,
 #ifdef CONFIG_OF
 		.of_match_table = mt8192_mt6359_dt_match,
 #endif
diff --git a/sound/soc/mediatek/mt8195/mt8195-afe-clk.c b/sound/soc/mediatek/mt8195/mt8195-afe-clk.c
index 2ee3872c83c3..9ca2cb8c8a9c 100644
--- a/sound/soc/mediatek/mt8195/mt8195-afe-clk.c
+++ b/sound/soc/mediatek/mt8195/mt8195-afe-clk.c
@@ -117,7 +117,7 @@ static struct mt8195_afe_tuner_cfg mt8195_afe_tuner_cfgs[MT8195_AUD_PLL_NUM] = {
 		.upper_bound_reg = AFE_APLL_TUNER_CFG,
 		.upper_bound_shift = 8,
 		.upper_bound_maskbit = 0xff,
-		.upper_bound_default = 0x2,
+		.upper_bound_default = 0x3,
 	},
 	[MT8195_AUD_PLL2] = {
 		.id = MT8195_AUD_PLL2,
@@ -135,7 +135,7 @@ static struct mt8195_afe_tuner_cfg mt8195_afe_tuner_cfgs[MT8195_AUD_PLL_NUM] = {
 		.upper_bound_reg = AFE_APLL_TUNER_CFG1,
 		.upper_bound_shift = 8,
 		.upper_bound_maskbit = 0xff,
-		.upper_bound_default = 0x2,
+		.upper_bound_default = 0x3,
 	},
 	[MT8195_AUD_PLL3] = {
 		.id = MT8195_AUD_PLL3,
diff --git a/sound/soc/mediatek/mt8195/mt8195-mt6359.c b/sound/soc/mediatek/mt8195/mt8195-mt6359.c
index c530e3fc27e4..480ed3e08d5b 100644
--- a/sound/soc/mediatek/mt8195/mt8195-mt6359.c
+++ b/sound/soc/mediatek/mt8195/mt8195-mt6359.c
@@ -63,6 +63,18 @@ struct mt8195_mt6359_priv {
 	struct clk *i2so1_mclk;
 };
 
+/* Headset jack detection DAPM pins */
+static struct snd_soc_jack_pin mt8195_jack_pins[] = {
+	{
+		.pin = "Headphone",
+		.mask = SND_JACK_HEADPHONE,
+	},
+	{
+		.pin = "Headset Mic",
+		.mask = SND_JACK_MICROPHONE,
+	},
+};
+
 static const struct snd_soc_dapm_widget mt8195_mt6359_widgets[] = {
 	SND_SOC_DAPM_HP("Headphone", NULL),
 	SND_SOC_DAPM_MIC("Headset Mic", NULL),
@@ -563,11 +575,12 @@ static int mt8195_rt5682_init(struct snd_soc_pcm_runtime *rtd)
 
 	priv->i2so1_mclk = afe_priv->clk[MT8195_CLK_TOP_APLL12_DIV2];
 
-	ret = snd_soc_card_jack_new(rtd->card, "Headset Jack",
+	ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack",
 				    SND_JACK_HEADSET | SND_JACK_BTN_0 |
 				    SND_JACK_BTN_1 | SND_JACK_BTN_2 |
 				    SND_JACK_BTN_3,
-				    jack);
+				    jack, mt8195_jack_pins,
+				    ARRAY_SIZE(mt8195_jack_pins));
 	if (ret) {
 		dev_err(rtd->dev, "Headset Jack creation failed: %d\n", ret);
 		return ret;
@@ -1383,7 +1396,13 @@ static int mt8195_mt6359_dev_probe(struct platform_device *pdev)
 		sof_priv->num_streams = ARRAY_SIZE(g_sof_conn_streams);
 		sof_priv->sof_dai_link_fixup = mt8195_dai_link_fixup;
 		soc_card_data->sof_priv = sof_priv;
+		card->probe = mtk_sof_card_probe;
 		card->late_probe = mtk_sof_card_late_probe;
+		if (!card->topology_shortname_created) {
+			snprintf(card->topology_shortname, 32, "sof-%s", card->name);
+			card->topology_shortname_created = true;
+		}
+		card->name = card->topology_shortname;
 		sof_on = 1;
 	}
 
@@ -1526,16 +1545,11 @@ static const struct of_device_id mt8195_mt6359_dt_match[] = {
 	{},
 };
 
-static const struct dev_pm_ops mt8195_mt6359_pm_ops = {
-	.poweroff = snd_soc_poweroff,
-	.restore = snd_soc_resume,
-};
-
 static struct platform_driver mt8195_mt6359_driver = {
 	.driver = {
 		.name = "mt8195_mt6359",
 		.of_match_table = mt8195_mt6359_dt_match,
-		.pm = &mt8195_mt6359_pm_ops,
+		.pm = &snd_soc_pm_ops,
 	},
 	.probe = mt8195_mt6359_dev_probe,
 };
diff --git a/sound/soc/qcom/Kconfig b/sound/soc/qcom/Kconfig
index 750653404ba3..d0e59e07b1fc 100644
--- a/sound/soc/qcom/Kconfig
+++ b/sound/soc/qcom/Kconfig
@@ -173,6 +173,17 @@ config SND_SOC_SM8250
 	  SM8250 SoC-based systems.
 	  Say Y if you want to use audio device on this SoCs.
 
+config SND_SOC_SC8280XP
+	tristate "SoC Machine driver for SC8280XP boards"
+	depends on QCOM_APR && SOUNDWIRE
+	depends on COMMON_CLK
+	select SND_SOC_QDSP6
+	select SND_SOC_QCOM_COMMON
+	help
+	  To add support for audio on Qualcomm Technologies Inc.
+	  SC8280XP SoC-based systems.
+	  Say Y if you want to use audio device on this SoCs.
+
 config SND_SOC_SC7180
 	tristate "SoC Machine driver for SC7180 boards"
 	depends on I2C && GPIOLIB
diff --git a/sound/soc/qcom/Makefile b/sound/soc/qcom/Makefile
index 8b7b876899a8..8b97172cf990 100644
--- a/sound/soc/qcom/Makefile
+++ b/sound/soc/qcom/Makefile
@@ -26,6 +26,7 @@ snd-soc-sc7180-objs := sc7180.o
 snd-soc-sc7280-objs := sc7280.o
 snd-soc-sdm845-objs := sdm845.o
 snd-soc-sm8250-objs := sm8250.o
+snd-soc-sc8280xp-objs := sc8280xp.o
 snd-soc-qcom-common-objs := common.o
 
 obj-$(CONFIG_SND_SOC_STORM) += snd-soc-storm.o
@@ -33,6 +34,7 @@ obj-$(CONFIG_SND_SOC_APQ8016_SBC) += snd-soc-apq8016-sbc.o
 obj-$(CONFIG_SND_SOC_MSM8996) += snd-soc-apq8096.o
 obj-$(CONFIG_SND_SOC_SC7180) += snd-soc-sc7180.o
 obj-$(CONFIG_SND_SOC_SC7280) += snd-soc-sc7280.o
+obj-$(CONFIG_SND_SOC_SC8280XP) += snd-soc-sc8280xp.o
 obj-$(CONFIG_SND_SOC_SDM845) += snd-soc-sdm845.o
 obj-$(CONFIG_SND_SOC_SM8250) += snd-soc-sm8250.o
 obj-$(CONFIG_SND_SOC_QCOM_COMMON) += snd-soc-qcom-common.o
diff --git a/sound/soc/qcom/common.c b/sound/soc/qcom/common.c
index c407684ce1a2..69dd3b504e20 100644
--- a/sound/soc/qcom/common.c
+++ b/sound/soc/qcom/common.c
@@ -3,6 +3,9 @@
 // Copyright (c) 2018, The Linux Foundation. All rights reserved.
 
 #include <linux/module.h>
+#include <sound/jack.h>
+#include <linux/input-event-codes.h>
+#include "qdsp6/q6afe.h"
 #include "common.h"
 
 int qcom_snd_parse_of(struct snd_soc_card *card)
@@ -175,6 +178,174 @@ err_put_np:
 	of_node_put(np);
 	return ret;
 }
-EXPORT_SYMBOL(qcom_snd_parse_of);
+EXPORT_SYMBOL_GPL(qcom_snd_parse_of);
 
+#if IS_ENABLED(CONFIG_SOUNDWIRE)
+int qcom_snd_sdw_prepare(struct snd_pcm_substream *substream,
+			 struct sdw_stream_runtime *sruntime,
+			 bool *stream_prepared)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
+	int ret;
+
+	if (!sruntime)
+		return 0;
+
+	switch (cpu_dai->id) {
+	case WSA_CODEC_DMA_RX_0:
+	case WSA_CODEC_DMA_RX_1:
+	case RX_CODEC_DMA_RX_0:
+	case RX_CODEC_DMA_RX_1:
+	case TX_CODEC_DMA_TX_0:
+	case TX_CODEC_DMA_TX_1:
+	case TX_CODEC_DMA_TX_2:
+	case TX_CODEC_DMA_TX_3:
+		break;
+	default:
+		return 0;
+	}
+
+	if (*stream_prepared) {
+		sdw_disable_stream(sruntime);
+		sdw_deprepare_stream(sruntime);
+		*stream_prepared = false;
+	}
+
+	ret = sdw_prepare_stream(sruntime);
+	if (ret)
+		return ret;
+
+	/**
+	 * NOTE: there is a strict hw requirement about the ordering of port
+	 * enables and actual WSA881x PA enable. PA enable should only happen
+	 * after soundwire ports are enabled if not DC on the line is
+	 * accumulated resulting in Click/Pop Noise
+	 * PA enable/mute are handled as part of codec DAPM and digital mute.
+	 */
+
+	ret = sdw_enable_stream(sruntime);
+	if (ret) {
+		sdw_deprepare_stream(sruntime);
+		return ret;
+	}
+	*stream_prepared  = true;
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(qcom_snd_sdw_prepare);
+
+int qcom_snd_sdw_hw_params(struct snd_pcm_substream *substream,
+			   struct snd_pcm_hw_params *params,
+			   struct sdw_stream_runtime **psruntime)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *codec_dai;
+	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
+	struct sdw_stream_runtime *sruntime;
+	int i;
+
+	switch (cpu_dai->id) {
+	case WSA_CODEC_DMA_RX_0:
+	case RX_CODEC_DMA_RX_0:
+	case RX_CODEC_DMA_RX_1:
+	case TX_CODEC_DMA_TX_0:
+	case TX_CODEC_DMA_TX_1:
+	case TX_CODEC_DMA_TX_2:
+	case TX_CODEC_DMA_TX_3:
+		for_each_rtd_codec_dais(rtd, i, codec_dai) {
+			sruntime = snd_soc_dai_get_stream(codec_dai, substream->stream);
+			if (sruntime != ERR_PTR(-ENOTSUPP))
+				*psruntime = sruntime;
+		}
+		break;
+	}
+
+	return 0;
+
+}
+EXPORT_SYMBOL_GPL(qcom_snd_sdw_hw_params);
+
+int qcom_snd_sdw_hw_free(struct snd_pcm_substream *substream,
+			 struct sdw_stream_runtime *sruntime, bool *stream_prepared)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
+
+	switch (cpu_dai->id) {
+	case WSA_CODEC_DMA_RX_0:
+	case WSA_CODEC_DMA_RX_1:
+	case RX_CODEC_DMA_RX_0:
+	case RX_CODEC_DMA_RX_1:
+	case TX_CODEC_DMA_TX_0:
+	case TX_CODEC_DMA_TX_1:
+	case TX_CODEC_DMA_TX_2:
+	case TX_CODEC_DMA_TX_3:
+		if (sruntime && *stream_prepared) {
+			sdw_disable_stream(sruntime);
+			sdw_deprepare_stream(sruntime);
+			*stream_prepared = false;
+		}
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(qcom_snd_sdw_hw_free);
+#endif
+
+int qcom_snd_wcd_jack_setup(struct snd_soc_pcm_runtime *rtd,
+			    struct snd_soc_jack *jack, bool *jack_setup)
+{
+	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
+	struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0);
+	struct snd_soc_card *card = rtd->card;
+	int rval, i;
+
+	if (!*jack_setup) {
+		rval = snd_soc_card_jack_new(card, "Headset Jack",
+					     SND_JACK_HEADSET | SND_JACK_LINEOUT |
+					     SND_JACK_MECHANICAL |
+					     SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+					     SND_JACK_BTN_2 | SND_JACK_BTN_3 |
+					     SND_JACK_BTN_4 | SND_JACK_BTN_5,
+					     jack);
+
+		if (rval < 0) {
+			dev_err(card->dev, "Unable to add Headphone Jack\n");
+			return rval;
+		}
+
+		snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_MEDIA);
+		snd_jack_set_key(jack->jack, SND_JACK_BTN_1, KEY_VOICECOMMAND);
+		snd_jack_set_key(jack->jack, SND_JACK_BTN_2, KEY_VOLUMEUP);
+		snd_jack_set_key(jack->jack, SND_JACK_BTN_3, KEY_VOLUMEDOWN);
+		*jack_setup = true;
+	}
+
+	switch (cpu_dai->id) {
+	case TX_CODEC_DMA_TX_0:
+	case TX_CODEC_DMA_TX_1:
+	case TX_CODEC_DMA_TX_2:
+	case TX_CODEC_DMA_TX_3:
+		for_each_rtd_codec_dais(rtd, i, codec_dai) {
+			rval = snd_soc_component_set_jack(codec_dai->component,
+							  jack, NULL);
+			if (rval != 0 && rval != -ENOTSUPP) {
+				dev_warn(card->dev, "Failed to set jack: %d\n", rval);
+				return rval;
+			}
+		}
+
+		break;
+	default:
+		break;
+	}
+
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(qcom_snd_wcd_jack_setup);
 MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/qcom/common.h b/sound/soc/qcom/common.h
index f05c05b12bd7..c5472a642de0 100644
--- a/sound/soc/qcom/common.h
+++ b/sound/soc/qcom/common.h
@@ -5,7 +5,42 @@
 #define __QCOM_SND_COMMON_H__
 
 #include <sound/soc.h>
+#include <linux/soundwire/sdw.h>
 
 int qcom_snd_parse_of(struct snd_soc_card *card);
+int qcom_snd_wcd_jack_setup(struct snd_soc_pcm_runtime *rtd,
+			    struct snd_soc_jack *jack, bool *jack_setup);
 
+#if IS_ENABLED(CONFIG_SOUNDWIRE)
+int qcom_snd_sdw_prepare(struct snd_pcm_substream *substream,
+			 struct sdw_stream_runtime *runtime,
+			 bool *stream_prepared);
+int qcom_snd_sdw_hw_params(struct snd_pcm_substream *substream,
+			   struct snd_pcm_hw_params *params,
+			   struct sdw_stream_runtime **psruntime);
+int qcom_snd_sdw_hw_free(struct snd_pcm_substream *substream,
+			 struct sdw_stream_runtime *sruntime,
+			 bool *stream_prepared);
+#else
+static inline int qcom_snd_sdw_prepare(struct snd_pcm_substream *substream,
+				       struct sdw_stream_runtime *runtime,
+				       bool *stream_prepared)
+{
+	return -ENOTSUPP;
+}
+
+static inline int qcom_snd_sdw_hw_params(struct snd_pcm_substream *substream,
+					 struct snd_pcm_hw_params *params,
+					 struct sdw_stream_runtime **psruntime)
+{
+	return -ENOTSUPP;
+}
+
+static inline int qcom_snd_sdw_hw_free(struct snd_pcm_substream *substream,
+				       struct sdw_stream_runtime *sruntime,
+				       bool *stream_prepared)
+{
+	return -ENOTSUPP;
+}
+#endif
 #endif
diff --git a/sound/soc/qcom/qdsp6/q6prm-clocks.c b/sound/soc/qcom/qdsp6/q6prm-clocks.c
index a26cda5140c1..73b0cbac73d4 100644
--- a/sound/soc/qcom/qdsp6/q6prm-clocks.c
+++ b/sound/soc/qcom/qdsp6/q6prm-clocks.c
@@ -50,6 +50,15 @@ static const struct q6dsp_clk_init q6prm_clks[] = {
 	Q6PRM_CLK(LPASS_CLK_ID_RX_CORE_MCLK),
 	Q6PRM_CLK(LPASS_CLK_ID_RX_CORE_NPL_MCLK),
 	Q6PRM_CLK(LPASS_CLK_ID_VA_CORE_2X_MCLK),
+	Q6PRM_CLK(LPASS_CLK_ID_WSA2_CORE_MCLK),
+	Q6PRM_CLK(LPASS_CLK_ID_WSA2_CORE_2X_MCLK),
+	Q6PRM_CLK(LPASS_CLK_ID_RX_CORE_TX_MCLK),
+	Q6PRM_CLK(LPASS_CLK_ID_RX_CORE_TX_2X_MCLK),
+	Q6PRM_CLK(LPASS_CLK_ID_WSA_CORE_TX_MCLK),
+	Q6PRM_CLK(LPASS_CLK_ID_WSA_CORE_TX_2X_MCLK),
+	Q6PRM_CLK(LPASS_CLK_ID_WSA2_CORE_TX_MCLK),
+	Q6PRM_CLK(LPASS_CLK_ID_WSA2_CORE_TX_2X_MCLK),
+	Q6PRM_CLK(LPASS_CLK_ID_RX_CORE_MCLK2_2X_MCLK),
 	Q6DSP_VOTE_CLK(LPASS_HW_MACRO_VOTE, Q6PRM_HW_CORE_ID_LPASS,
 		       "LPASS_HW_MACRO"),
 	Q6DSP_VOTE_CLK(LPASS_HW_DCODEC_VOTE, Q6PRM_HW_CORE_ID_DCODEC,
diff --git a/sound/soc/qcom/qdsp6/q6prm.h b/sound/soc/qcom/qdsp6/q6prm.h
index fea4d1954bc1..a988a32086fe 100644
--- a/sound/soc/qcom/qdsp6/q6prm.h
+++ b/sound/soc/qcom/qdsp6/q6prm.h
@@ -64,6 +64,25 @@
 #define Q6PRM_LPASS_CLK_ID_RX_CORE_MCLK				0x30e
 #define Q6PRM_LPASS_CLK_ID_RX_CORE_NPL_MCLK			0x30f
 
+/* Clock ID for MCLK for WSA2 core */
+#define Q6PRM_LPASS_CLK_ID_WSA2_CORE_MCLK 0x310
+/* Clock ID for NPL MCLK for WSA2 core */
+#define Q6PRM_LPASS_CLK_ID_WSA2_CORE_2X_MCLK 0x311
+/* Clock ID for RX Core TX MCLK */
+#define Q6PRM_LPASS_CLK_ID_RX_CORE_TX_MCLK 0x312
+/* Clock ID for RX CORE TX 2X MCLK */
+#define Q6PRM_LPASS_CLK_ID_RX_CORE_TX_2X_MCLK 0x313
+/* Clock ID for WSA core TX MCLK */
+#define Q6PRM_LPASS_CLK_ID_WSA_CORE_TX_MCLK 0x314
+/* Clock ID for WSA core TX 2X MCLK */
+#define Q6PRM_LPASS_CLK_ID_WSA_CORE_TX_2X_MCLK 0x315
+/* Clock ID for WSA2 core TX MCLK */
+#define Q6PRM_LPASS_CLK_ID_WSA2_CORE_TX_MCLK 0x316
+/* Clock ID for WSA2 core TX 2X MCLK */
+#define Q6PRM_LPASS_CLK_ID_WSA2_CORE_TX_2X_MCLK 0x317
+/* Clock ID for RX CORE MCLK2 2X  MCLK */
+#define Q6PRM_LPASS_CLK_ID_RX_CORE_MCLK2_2X_MCLK 0x318
+
 #define Q6PRM_LPASS_CLK_SRC_INTERNAL	1
 #define Q6PRM_LPASS_CLK_ROOT_DEFAULT	0
 #define Q6PRM_HW_CORE_ID_LPASS		1
diff --git a/sound/soc/qcom/sc8280xp.c b/sound/soc/qcom/sc8280xp.c
new file mode 100644
index 000000000000..ade44ad7c585
--- /dev/null
+++ b/sound/soc/qcom/sc8280xp.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2022, Linaro Limited
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/of_device.h>
+#include <sound/soc.h>
+#include <sound/soc-dapm.h>
+#include <sound/pcm.h>
+#include <linux/soundwire/sdw.h>
+#include <sound/jack.h>
+#include <linux/input-event-codes.h>
+#include "qdsp6/q6afe.h"
+#include "common.h"
+
+#define DRIVER_NAME		"sc8280xp"
+
+struct sc8280xp_snd_data {
+	bool stream_prepared[AFE_PORT_MAX];
+	struct snd_soc_card *card;
+	struct sdw_stream_runtime *sruntime[AFE_PORT_MAX];
+	struct snd_soc_jack jack;
+	bool jack_setup;
+};
+
+static int sc8280xp_snd_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct sc8280xp_snd_data *data = snd_soc_card_get_drvdata(rtd->card);
+
+	return qcom_snd_wcd_jack_setup(rtd, &data->jack, &data->jack_setup);
+}
+
+static int sc8280xp_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
+				     struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
+	struct snd_interval *rate = hw_param_interval(params,
+					SNDRV_PCM_HW_PARAM_RATE);
+	struct snd_interval *channels = hw_param_interval(params,
+					SNDRV_PCM_HW_PARAM_CHANNELS);
+
+	rate->min = rate->max = 48000;
+	channels->min = 2;
+	channels->max = 2;
+	switch (cpu_dai->id) {
+	case TX_CODEC_DMA_TX_0:
+	case TX_CODEC_DMA_TX_1:
+	case TX_CODEC_DMA_TX_2:
+	case TX_CODEC_DMA_TX_3:
+		channels->min = 1;
+		break;
+	default:
+		break;
+	}
+
+
+	return 0;
+}
+
+static int sc8280xp_snd_hw_params(struct snd_pcm_substream *substream,
+				struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
+	struct sc8280xp_snd_data *pdata = snd_soc_card_get_drvdata(rtd->card);
+
+	return qcom_snd_sdw_hw_params(substream, params, &pdata->sruntime[cpu_dai->id]);
+}
+
+static int sc8280xp_snd_prepare(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
+	struct sc8280xp_snd_data *data = snd_soc_card_get_drvdata(rtd->card);
+	struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id];
+
+	return qcom_snd_sdw_prepare(substream, sruntime,
+				    &data->stream_prepared[cpu_dai->id]);
+}
+
+static int sc8280xp_snd_hw_free(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct sc8280xp_snd_data *data = snd_soc_card_get_drvdata(rtd->card);
+	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
+	struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id];
+
+	return qcom_snd_sdw_hw_free(substream, sruntime,
+				    &data->stream_prepared[cpu_dai->id]);
+}
+
+static const struct snd_soc_ops sc8280xp_be_ops = {
+	.hw_params = sc8280xp_snd_hw_params,
+	.hw_free = sc8280xp_snd_hw_free,
+	.prepare = sc8280xp_snd_prepare,
+};
+
+static void sc8280xp_add_be_ops(struct snd_soc_card *card)
+{
+	struct snd_soc_dai_link *link;
+	int i;
+
+	for_each_card_prelinks(card, i, link) {
+		if (link->no_pcm == 1) {
+			link->init = sc8280xp_snd_init;
+			link->be_hw_params_fixup = sc8280xp_be_hw_params_fixup;
+			link->ops = &sc8280xp_be_ops;
+		}
+	}
+}
+
+static int sc8280xp_platform_probe(struct platform_device *pdev)
+{
+	struct snd_soc_card *card;
+	struct sc8280xp_snd_data *data;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	card = devm_kzalloc(dev, sizeof(*card), GFP_KERNEL);
+	if (!card)
+		return -ENOMEM;
+	card->owner = THIS_MODULE;
+	/* Allocate the private data */
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	card->dev = dev;
+	dev_set_drvdata(dev, card);
+	snd_soc_card_set_drvdata(card, data);
+	ret = qcom_snd_parse_of(card);
+	if (ret)
+		return ret;
+
+	card->driver_name = DRIVER_NAME;
+	sc8280xp_add_be_ops(card);
+	return devm_snd_soc_register_card(dev, card);
+}
+
+static const struct of_device_id snd_sc8280xp_dt_match[] = {
+	{.compatible = "qcom,sc8280xp-sndcard",},
+	{}
+};
+
+MODULE_DEVICE_TABLE(of, snd_sc8280xp_dt_match);
+
+static struct platform_driver snd_sc8280xp_driver = {
+	.probe  = sc8280xp_platform_probe,
+	.driver = {
+		.name = "snd-sc8280xp",
+		.of_match_table = snd_sc8280xp_dt_match,
+	},
+};
+module_platform_driver(snd_sc8280xp_driver);
+MODULE_AUTHOR("Srinivas Kandagatla <srinivas.kandagatla@linaro.org");
+MODULE_DESCRIPTION("SC8280XP ASoC Machine Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/sound/soc/qcom/sm8250.c b/sound/soc/qcom/sm8250.c
index 98a2fde9e004..8dbe9ef41b1c 100644
--- a/sound/soc/qcom/sm8250.c
+++ b/sound/soc/qcom/sm8250.c
@@ -27,57 +27,8 @@ struct sm8250_snd_data {
 static int sm8250_snd_init(struct snd_soc_pcm_runtime *rtd)
 {
 	struct sm8250_snd_data *data = snd_soc_card_get_drvdata(rtd->card);
-	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
-	struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0);
-	struct snd_soc_card *card = rtd->card;
-	int rval, i;
-
-	if (!data->jack_setup) {
-		struct snd_jack *jack;
-
-		rval = snd_soc_card_jack_new(card, "Headset Jack",
-					     SND_JACK_HEADSET | SND_JACK_LINEOUT |
-					     SND_JACK_MECHANICAL |
-					     SND_JACK_BTN_0 | SND_JACK_BTN_1 |
-					     SND_JACK_BTN_2 | SND_JACK_BTN_3 |
-					     SND_JACK_BTN_4 | SND_JACK_BTN_5,
-					     &data->jack);
-
-		if (rval < 0) {
-			dev_err(card->dev, "Unable to add Headphone Jack\n");
-			return rval;
-		}
-
-		jack = data->jack.jack;
-
-		snd_jack_set_key(jack, SND_JACK_BTN_0, KEY_MEDIA);
-		snd_jack_set_key(jack, SND_JACK_BTN_1, KEY_VOICECOMMAND);
-		snd_jack_set_key(jack, SND_JACK_BTN_2, KEY_VOLUMEUP);
-		snd_jack_set_key(jack, SND_JACK_BTN_3, KEY_VOLUMEDOWN);
-		data->jack_setup = true;
-	}
-
-	switch (cpu_dai->id) {
-	case TX_CODEC_DMA_TX_0:
-	case TX_CODEC_DMA_TX_1:
-	case TX_CODEC_DMA_TX_2:
-	case TX_CODEC_DMA_TX_3:
-		for_each_rtd_codec_dais(rtd, i, codec_dai) {
-			rval = snd_soc_component_set_jack(codec_dai->component,
-							  &data->jack, NULL);
-			if (rval != 0 && rval != -ENOTSUPP) {
-				dev_warn(card->dev, "Failed to set jack: %d\n", rval);
-				return rval;
-			}
-		}
-
-		break;
-	default:
-		break;
-	}
 
-
-	return 0;
+	return qcom_snd_wcd_jack_setup(rtd, &data->jack, &data->jack_setup);
 }
 
 static int sm8250_be_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
@@ -121,92 +72,21 @@ static int sm8250_snd_hw_params(struct snd_pcm_substream *substream,
 				struct snd_pcm_hw_params *params)
 {
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *codec_dai;
 	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
 	struct sm8250_snd_data *pdata = snd_soc_card_get_drvdata(rtd->card);
-	struct sdw_stream_runtime *sruntime;
-	int i;
-
-	switch (cpu_dai->id) {
-	case WSA_CODEC_DMA_RX_0:
-	case RX_CODEC_DMA_RX_0:
-	case RX_CODEC_DMA_RX_1:
-	case TX_CODEC_DMA_TX_0:
-	case TX_CODEC_DMA_TX_1:
-	case TX_CODEC_DMA_TX_2:
-	case TX_CODEC_DMA_TX_3:
-		for_each_rtd_codec_dais(rtd, i, codec_dai) {
-			sruntime = snd_soc_dai_get_stream(codec_dai,
-							  substream->stream);
-			if (sruntime != ERR_PTR(-ENOTSUPP))
-				pdata->sruntime[cpu_dai->id] = sruntime;
-		}
-		break;
-	}
-
-	return 0;
 
+	return qcom_snd_sdw_hw_params(substream, params, &pdata->sruntime[cpu_dai->id]);
 }
 
-static int sm8250_snd_wsa_dma_prepare(struct snd_pcm_substream *substream)
+static int sm8250_snd_prepare(struct snd_pcm_substream *substream)
 {
 	struct snd_soc_pcm_runtime *rtd = substream->private_data;
 	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
 	struct sm8250_snd_data *data = snd_soc_card_get_drvdata(rtd->card);
 	struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id];
-	int ret;
-
-	if (!sruntime)
-		return 0;
 
-	if (data->stream_prepared[cpu_dai->id]) {
-		sdw_disable_stream(sruntime);
-		sdw_deprepare_stream(sruntime);
-		data->stream_prepared[cpu_dai->id] = false;
-	}
-
-	ret = sdw_prepare_stream(sruntime);
-	if (ret)
-		return ret;
-
-	/**
-	 * NOTE: there is a strict hw requirement about the ordering of port
-	 * enables and actual WSA881x PA enable. PA enable should only happen
-	 * after soundwire ports are enabled if not DC on the line is
-	 * accumulated resulting in Click/Pop Noise
-	 * PA enable/mute are handled as part of codec DAPM and digital mute.
-	 */
-
-	ret = sdw_enable_stream(sruntime);
-	if (ret) {
-		sdw_deprepare_stream(sruntime);
-		return ret;
-	}
-	data->stream_prepared[cpu_dai->id]  = true;
-
-	return ret;
-}
-
-static int sm8250_snd_prepare(struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *rtd = substream->private_data;
-	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
-
-	switch (cpu_dai->id) {
-	case WSA_CODEC_DMA_RX_0:
-	case WSA_CODEC_DMA_RX_1:
-	case RX_CODEC_DMA_RX_0:
-	case RX_CODEC_DMA_RX_1:
-	case TX_CODEC_DMA_TX_0:
-	case TX_CODEC_DMA_TX_1:
-	case TX_CODEC_DMA_TX_2:
-	case TX_CODEC_DMA_TX_3:
-		return sm8250_snd_wsa_dma_prepare(substream);
-	default:
-		break;
-	}
-
-	return 0;
+	return qcom_snd_sdw_prepare(substream, sruntime,
+				    &data->stream_prepared[cpu_dai->id]);
 }
 
 static int sm8250_snd_hw_free(struct snd_pcm_substream *substream)
@@ -216,26 +96,8 @@ static int sm8250_snd_hw_free(struct snd_pcm_substream *substream)
 	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0);
 	struct sdw_stream_runtime *sruntime = data->sruntime[cpu_dai->id];
 
-	switch (cpu_dai->id) {
-	case WSA_CODEC_DMA_RX_0:
-	case WSA_CODEC_DMA_RX_1:
-	case RX_CODEC_DMA_RX_0:
-	case RX_CODEC_DMA_RX_1:
-	case TX_CODEC_DMA_TX_0:
-	case TX_CODEC_DMA_TX_1:
-	case TX_CODEC_DMA_TX_2:
-	case TX_CODEC_DMA_TX_3:
-		if (sruntime && data->stream_prepared[cpu_dai->id]) {
-			sdw_disable_stream(sruntime);
-			sdw_deprepare_stream(sruntime);
-			data->stream_prepared[cpu_dai->id] = false;
-		}
-		break;
-	default:
-		break;
-	}
-
-	return 0;
+	return qcom_snd_sdw_hw_free(substream, sruntime,
+				    &data->stream_prepared[cpu_dai->id]);
 }
 
 static const struct snd_soc_ops sm8250_be_ops = {
diff --git a/sound/soc/rockchip/rockchip_i2s.c b/sound/soc/rockchip/rockchip_i2s.c
index f5f3540a9e18..a8758ad68442 100644
--- a/sound/soc/rockchip/rockchip_i2s.c
+++ b/sound/soc/rockchip/rockchip_i2s.c
@@ -126,7 +126,6 @@ static inline struct rk_i2s_dev *to_info(struct snd_soc_dai *dai)
 static int rockchip_snd_txctrl(struct rk_i2s_dev *i2s, int on)
 {
 	unsigned int val = 0;
-	int retry = 10;
 	int ret = 0;
 
 	spin_lock(&i2s->lock);
@@ -163,18 +162,14 @@ static int rockchip_snd_txctrl(struct rk_i2s_dev *i2s, int on)
 						 I2S_CLR_TXC | I2S_CLR_RXC);
 			if (ret < 0)
 				goto end;
-			regmap_read(i2s->regmap, I2S_CLR, &val);
-
-			/* Should wait for clear operation to finish */
-			while (val) {
-				regmap_read(i2s->regmap, I2S_CLR, &val);
-				retry--;
-				if (!retry) {
-					dev_warn(i2s->dev, "fail to clear\n");
-					ret = -EBUSY;
-					break;
-				}
-			}
+			ret = regmap_read_poll_timeout_atomic(i2s->regmap,
+							      I2S_CLR,
+							      val,
+							      val == 0,
+							      20,
+							      200);
+			if (ret < 0)
+				dev_warn(i2s->dev, "fail to clear: %d\n", ret);
 		}
 	}
 end:
@@ -188,7 +183,6 @@ end:
 static int rockchip_snd_rxctrl(struct rk_i2s_dev *i2s, int on)
 {
 	unsigned int val = 0;
-	int retry = 10;
 	int ret = 0;
 
 	spin_lock(&i2s->lock);
@@ -226,17 +220,14 @@ static int rockchip_snd_rxctrl(struct rk_i2s_dev *i2s, int on)
 						 I2S_CLR_TXC | I2S_CLR_RXC);
 			if (ret < 0)
 				goto end;
-			regmap_read(i2s->regmap, I2S_CLR, &val);
-			/* Should wait for clear operation to finish */
-			while (val) {
-				regmap_read(i2s->regmap, I2S_CLR, &val);
-				retry--;
-				if (!retry) {
-					dev_warn(i2s->dev, "fail to clear\n");
-					ret = -EBUSY;
-					break;
-				}
-			}
+			ret = regmap_read_poll_timeout_atomic(i2s->regmap,
+							      I2S_CLR,
+							      val,
+							      val == 0,
+							      20,
+							      200);
+			if (ret < 0)
+				dev_warn(i2s->dev, "fail to clear: %d\n", ret);
 		}
 	}
 end:
diff --git a/sound/soc/samsung/aries_wm8994.c b/sound/soc/samsung/aries_wm8994.c
index e7d52d27132e..0fbbf3b02c09 100644
--- a/sound/soc/samsung/aries_wm8994.c
+++ b/sound/soc/samsung/aries_wm8994.c
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+
 #include <linux/extcon.h>
 #include <linux/iio/consumer.h>
-#include <linux/iio/iio.h>
 #include <linux/input-event-codes.h>
 #include <linux/mfd/wm8994/registers.h>
 #include <linux/module.h>
@@ -543,6 +542,7 @@ static int aries_audio_probe(struct platform_device *pdev)
 	struct aries_wm8994_data *priv;
 	struct snd_soc_dai_link *dai_link;
 	const struct of_device_id *match;
+	enum iio_chan_type channel_type;
 	int ret, i;
 
 	if (!np)
@@ -594,7 +594,11 @@ static int aries_audio_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(priv->adc),
 				     "Failed to get ADC channel");
 
-	if (priv->adc->channel->type != IIO_VOLTAGE)
+	ret = iio_get_channel_type(priv->adc, &channel_type);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to get ADC channel type");
+	if (channel_type != IIO_VOLTAGE)
 		return -EINVAL;
 
 	priv->gpio_headset_key = devm_gpiod_get(dev, "headset-key",
diff --git a/sound/soc/samsung/odroid.c b/sound/soc/samsung/odroid.c
index 4ff12e2e704f..1e0fefa89ad5 100644
--- a/sound/soc/samsung/odroid.c
+++ b/sound/soc/samsung/odroid.c
@@ -97,7 +97,7 @@ static int odroid_card_be_hw_params(struct snd_pcm_substream *substream,
 	if (ret < 0)
 		return ret;
 
-	if (rtd->num_codecs > 1) {
+	if (rtd->dai_link->num_codecs > 1) {
 		struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 1);
 
 		ret = snd_soc_dai_set_sysclk(codec_dai, 0, rclk_freq,
diff --git a/sound/soc/sh/rcar/ctu.c b/sound/soc/sh/rcar/ctu.c
index 6156445bcb69..e39eb2ac7e95 100644
--- a/sound/soc/sh/rcar/ctu.c
+++ b/sound/soc/sh/rcar/ctu.c
@@ -171,7 +171,11 @@ static int rsnd_ctu_init(struct rsnd_mod *mod,
 			 struct rsnd_dai_stream *io,
 			 struct rsnd_priv *priv)
 {
-	rsnd_mod_power_on(mod);
+	int ret;
+
+	ret = rsnd_mod_power_on(mod);
+	if (ret < 0)
+		return ret;
 
 	rsnd_ctu_activation(mod);
 
diff --git a/sound/soc/sh/rcar/dvc.c b/sound/soc/sh/rcar/dvc.c
index 5137e03a9d7c..16befcbc312c 100644
--- a/sound/soc/sh/rcar/dvc.c
+++ b/sound/soc/sh/rcar/dvc.c
@@ -186,7 +186,11 @@ static int rsnd_dvc_init(struct rsnd_mod *mod,
 			 struct rsnd_dai_stream *io,
 			 struct rsnd_priv *priv)
 {
-	rsnd_mod_power_on(mod);
+	int ret;
+
+	ret = rsnd_mod_power_on(mod);
+	if (ret < 0)
+		return ret;
 
 	rsnd_dvc_activation(mod);
 
diff --git a/sound/soc/sh/rcar/mix.c b/sound/soc/sh/rcar/mix.c
index 3572c2c5686c..1de0e085804c 100644
--- a/sound/soc/sh/rcar/mix.c
+++ b/sound/soc/sh/rcar/mix.c
@@ -146,7 +146,11 @@ static int rsnd_mix_init(struct rsnd_mod *mod,
 			 struct rsnd_dai_stream *io,
 			 struct rsnd_priv *priv)
 {
-	rsnd_mod_power_on(mod);
+	int ret;
+
+	ret = rsnd_mod_power_on(mod);
+	if (ret < 0)
+		return ret;
 
 	rsnd_mix_activation(mod);
 
diff --git a/sound/soc/sh/rcar/src.c b/sound/soc/sh/rcar/src.c
index 0ea84ae57c6a..f832165e46bc 100644
--- a/sound/soc/sh/rcar/src.c
+++ b/sound/soc/sh/rcar/src.c
@@ -463,11 +463,14 @@ static int rsnd_src_init(struct rsnd_mod *mod,
 			 struct rsnd_priv *priv)
 {
 	struct rsnd_src *src = rsnd_mod_to_src(mod);
+	int ret;
 
 	/* reset sync convert_rate */
 	src->sync.val = 0;
 
-	rsnd_mod_power_on(mod);
+	ret = rsnd_mod_power_on(mod);
+	if (ret < 0)
+		return ret;
 
 	rsnd_src_activation(mod);
 
diff --git a/sound/soc/sh/rcar/ssi.c b/sound/soc/sh/rcar/ssi.c
index 43c5e27dc5c8..7ade6c5ed96f 100644
--- a/sound/soc/sh/rcar/ssi.c
+++ b/sound/soc/sh/rcar/ssi.c
@@ -480,7 +480,9 @@ static int rsnd_ssi_init(struct rsnd_mod *mod,
 
 	ssi->usrcnt++;
 
-	rsnd_mod_power_on(mod);
+	ret = rsnd_mod_power_on(mod);
+	if (ret < 0)
+		return ret;
 
 	rsnd_ssi_config_init(mod, io);
 
diff --git a/sound/soc/sh/rz-ssi.c b/sound/soc/sh/rz-ssi.c
index 7ace0c0db5b1..5d6bae33ae34 100644
--- a/sound/soc/sh/rz-ssi.c
+++ b/sound/soc/sh/rz-ssi.c
@@ -598,7 +598,7 @@ static int rz_ssi_dma_transfer(struct rz_ssi_priv *ssi,
 		return -EINVAL;
 
 	runtime = substream->runtime;
-	if (runtime->status->state == SNDRV_PCM_STATE_DRAINING)
+	if (runtime->state == SNDRV_PCM_STATE_DRAINING)
 		/*
 		 * Stream is ending, so do not queue up any more DMA
 		 * transfers otherwise we play partial sound clips
diff --git a/sound/soc/soc-ac97.c b/sound/soc/soc-ac97.c
index 5f49e3dec3fc..32c5be61e2ec 100644
--- a/sound/soc/soc-ac97.c
+++ b/sound/soc/soc-ac97.c
@@ -57,7 +57,7 @@ static inline struct snd_soc_component *gpio_to_component(struct gpio_chip *chip
 	return gpio_priv->component;
 }
 
-static int snd_soc_ac97_gpio_request(struct gpio_chip *chip, unsigned offset)
+static int snd_soc_ac97_gpio_request(struct gpio_chip *chip, unsigned int offset)
 {
 	if (offset >= AC97_NUM_GPIOS)
 		return -EINVAL;
@@ -66,7 +66,7 @@ static int snd_soc_ac97_gpio_request(struct gpio_chip *chip, unsigned offset)
 }
 
 static int snd_soc_ac97_gpio_direction_in(struct gpio_chip *chip,
-					  unsigned offset)
+					  unsigned int offset)
 {
 	struct snd_soc_component *component = gpio_to_component(chip);
 
@@ -75,7 +75,7 @@ static int snd_soc_ac97_gpio_direction_in(struct gpio_chip *chip,
 				   1 << offset, 1 << offset);
 }
 
-static int snd_soc_ac97_gpio_get(struct gpio_chip *chip, unsigned offset)
+static int snd_soc_ac97_gpio_get(struct gpio_chip *chip, unsigned int offset)
 {
 	struct snd_soc_component *component = gpio_to_component(chip);
 	int ret;
@@ -88,7 +88,7 @@ static int snd_soc_ac97_gpio_get(struct gpio_chip *chip, unsigned offset)
 	return !!(ret & (1 << offset));
 }
 
-static void snd_soc_ac97_gpio_set(struct gpio_chip *chip, unsigned offset,
+static void snd_soc_ac97_gpio_set(struct gpio_chip *chip, unsigned int offset,
 				  int value)
 {
 	struct snd_ac97_gpio_priv *gpio_priv = gpiochip_get_data(chip);
diff --git a/sound/soc/soc-component.c b/sound/soc/soc-component.c
index e12f8244242b..659b9ade4158 100644
--- a/sound/soc/soc-component.c
+++ b/sound/soc/soc-component.c
@@ -1213,11 +1213,9 @@ int snd_soc_pcm_component_pm_runtime_get(struct snd_soc_pcm_runtime *rtd,
 	int i;
 
 	for_each_rtd_components(rtd, i, component) {
-		int ret = pm_runtime_get_sync(component->dev);
-		if (ret < 0 && ret != -EACCES) {
-			pm_runtime_put_noidle(component->dev);
+		int ret = pm_runtime_resume_and_get(component->dev);
+		if (ret < 0 && ret != -EACCES)
 			return soc_component_ret(component, ret);
-		}
 		/* mark stream if succeeded */
 		soc_component_mark_push(component, stream, pm);
 	}
diff --git a/sound/soc/soc-compress.c b/sound/soc/soc-compress.c
index e9dd25894dc0..870f13e1d389 100644
--- a/sound/soc/soc-compress.c
+++ b/sound/soc/soc-compress.c
@@ -560,8 +560,8 @@ int snd_soc_new_compress(struct snd_soc_pcm_runtime *rtd, int num)
 	BUILD_BUG_ON((int)SNDRV_PCM_STREAM_PLAYBACK != (int)SND_COMPRESS_PLAYBACK);
 	BUILD_BUG_ON((int)SNDRV_PCM_STREAM_CAPTURE  != (int)SND_COMPRESS_CAPTURE);
 
-	if (rtd->num_cpus > 1 ||
-	    rtd->num_codecs > 1) {
+	if (rtd->dai_link->num_cpus > 1 ||
+	    rtd->dai_link->num_codecs > 1) {
 		dev_err(rtd->card->dev,
 			"Compress ASoC: Multi CPU/Codec not supported\n");
 		return -EINVAL;
diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c
index e824ff1a9fc0..12a82f5a3ff6 100644
--- a/sound/soc/soc-core.c
+++ b/sound/soc/soc-core.c
@@ -72,7 +72,7 @@ static ssize_t pmdown_time_show(struct device *dev,
 {
 	struct snd_soc_pcm_runtime *rtd = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%ld\n", rtd->pmdown_time);
+	return sysfs_emit(buf, "%ld\n", rtd->pmdown_time);
 }
 
 static ssize_t pmdown_time_store(struct device *dev,
@@ -107,7 +107,7 @@ static umode_t soc_dev_attr_is_visible(struct kobject *kobj,
 
 	if (attr == &dev_attr_pmdown_time.attr)
 		return attr->mode; /* always visible */
-	return rtd->num_codecs ? attr->mode : 0; /* enabled only with codec */
+	return rtd->dai_link->num_codecs ? attr->mode : 0; /* enabled only with codec */
 }
 
 static const struct attribute_group soc_dapm_dev_group = {
@@ -482,11 +482,10 @@ static struct snd_soc_pcm_runtime *soc_new_pcm_runtime(
 	 *	asoc_rtd_to_cpu()
 	 *	asoc_rtd_to_codec()
 	 */
-	rtd->num_cpus	= dai_link->num_cpus;
-	rtd->num_codecs	= dai_link->num_codecs;
 	rtd->card	= card;
 	rtd->dai_link	= dai_link;
 	rtd->num	= card->num_rtd++;
+	rtd->pmdown_time = pmdown_time;			/* default power off timeout */
 
 	/* see for_each_card_rtds */
 	list_add_tail(&rtd->list, &card->rtd_list);
@@ -1247,9 +1246,6 @@ static int soc_init_pcm_runtime(struct snd_soc_card *card,
 	struct snd_soc_component *component;
 	int ret, num, i;
 
-	/* set default power off timeout */
-	rtd->pmdown_time = pmdown_time;
-
 	/* do machine specific initialization */
 	ret = snd_soc_link_init(rtd);
 	if (ret < 0)
@@ -1840,21 +1836,22 @@ match:
 	}
 }
 
-#define soc_setup_card_name(name, name1, name2, norm)		\
-	__soc_setup_card_name(name, sizeof(name), name1, name2, norm)
-static void __soc_setup_card_name(char *name, int len,
-				  const char *name1, const char *name2,
-				  int normalization)
+#define soc_setup_card_name(card, name, name1, name2) \
+	__soc_setup_card_name(card, name, sizeof(name), name1, name2)
+static void __soc_setup_card_name(struct snd_soc_card *card,
+				  char *name, int len,
+				  const char *name1, const char *name2)
 {
+	const char *src = name1 ? name1 : name2;
 	int i;
 
-	snprintf(name, len, "%s", name1 ? name1 : name2);
+	snprintf(name, len, "%s", src);
 
-	if (!normalization)
+	if (name != card->snd_card->driver)
 		return;
 
 	/*
-	 * Name normalization
+	 * Name normalization (driver field)
 	 *
 	 * The driver name is somewhat special, as it's used as a key for
 	 * searches in the user-space.
@@ -1874,6 +1871,14 @@ static void __soc_setup_card_name(char *name, int len,
 			break;
 		}
 	}
+
+	/*
+	 * The driver field should contain a valid string from the user view.
+	 * The wrapping usually does not work so well here. Set a smaller string
+	 * in the specific ASoC driver.
+	 */
+	if (strlen(src) > len - 1)
+		dev_err(card->dev, "ASoC: driver name too long '%s' -> '%s'\n", src, name);
 }
 
 static void soc_cleanup_card_resources(struct snd_soc_card *card)
@@ -2041,12 +2046,12 @@ static int snd_soc_bind_card(struct snd_soc_card *card)
 	/* try to set some sane longname if DMI is available */
 	snd_soc_set_dmi_name(card, NULL);
 
-	soc_setup_card_name(card->snd_card->shortname,
-			    card->name, NULL, 0);
-	soc_setup_card_name(card->snd_card->longname,
-			    card->long_name, card->name, 0);
-	soc_setup_card_name(card->snd_card->driver,
-			    card->driver_name, card->name, 1);
+	soc_setup_card_name(card, card->snd_card->shortname,
+			    card->name, NULL);
+	soc_setup_card_name(card, card->snd_card->longname,
+			    card->long_name, card->name);
+	soc_setup_card_name(card, card->snd_card->driver,
+			    card->driver_name, card->name);
 
 	if (card->components) {
 		/* the current implementation of snd_component_add() accepts */
diff --git a/sound/soc/soc-dai.c b/sound/soc/soc-dai.c
index d530e8c2b77b..49752af0e205 100644
--- a/sound/soc/soc-dai.c
+++ b/sound/soc/soc-dai.c
@@ -124,7 +124,7 @@ EXPORT_SYMBOL_GPL(snd_soc_dai_set_pll);
  */
 int snd_soc_dai_set_bclk_ratio(struct snd_soc_dai *dai, unsigned int ratio)
 {
-	int ret = -EINVAL;
+	int ret = -ENOTSUPP;
 
 	if (dai->driver->ops &&
 	    dai->driver->ops->set_bclk_ratio)
diff --git a/sound/soc/soc-dapm.c b/sound/soc/soc-dapm.c
index b05231414c1d..d515e7a78ea8 100644
--- a/sound/soc/soc-dapm.c
+++ b/sound/soc/soc-dapm.c
@@ -2386,11 +2386,10 @@ int snd_soc_dapm_mixer_update_power(struct snd_soc_dapm_context *dapm,
 EXPORT_SYMBOL_GPL(snd_soc_dapm_mixer_update_power);
 
 static ssize_t dapm_widget_show_component(struct snd_soc_component *cmpnt,
-	char *buf)
+					  char *buf, int count)
 {
 	struct snd_soc_dapm_context *dapm = snd_soc_component_get_dapm(cmpnt);
 	struct snd_soc_dapm_widget *w;
-	int count = 0;
 	char *state = "not set";
 
 	/* card won't be set for the dummy component, as a spot fix
@@ -2423,7 +2422,7 @@ static ssize_t dapm_widget_show_component(struct snd_soc_component *cmpnt,
 		case snd_soc_dapm_pinctrl:
 		case snd_soc_dapm_clock_supply:
 			if (w->name)
-				count += sprintf(buf + count, "%s: %s\n",
+				count += sysfs_emit_at(buf, count, "%s: %s\n",
 					w->name, w->power ? "On":"Off");
 		break;
 		default:
@@ -2445,7 +2444,7 @@ static ssize_t dapm_widget_show_component(struct snd_soc_component *cmpnt,
 		state = "Off";
 		break;
 	}
-	count += sprintf(buf + count, "PM State: %s\n", state);
+	count += sysfs_emit_at(buf, count, "PM State: %s\n", state);
 
 	return count;
 }
@@ -2463,7 +2462,7 @@ static ssize_t dapm_widget_show(struct device *dev,
 	for_each_rtd_codec_dais(rtd, i, codec_dai) {
 		struct snd_soc_component *cmpnt = codec_dai->component;
 
-		count += dapm_widget_show_component(cmpnt, buf + count);
+		count = dapm_widget_show_component(cmpnt, buf, count);
 	}
 
 	mutex_unlock(&rtd->card->dapm_mutex);
@@ -3631,10 +3630,18 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm,
 	enum snd_soc_dapm_direction dir;
 	struct snd_soc_dapm_widget *w;
 	const char *prefix;
-	int ret;
+	int ret = -ENOMEM;
 
 	if ((w = dapm_cnew_widget(widget)) == NULL)
-		return ERR_PTR(-ENOMEM);
+		goto cnew_failed;
+
+	prefix = soc_dapm_prefix(dapm);
+	if (prefix)
+		w->name = kasprintf(GFP_KERNEL, "%s %s", prefix, widget->name);
+	else
+		w->name = kstrdup_const(widget->name, GFP_KERNEL);
+	if (!w->name)
+		goto name_failed;
 
 	switch (w->id) {
 	case snd_soc_dapm_regulator_supply:
@@ -3673,17 +3680,6 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm,
 		break;
 	}
 
-	prefix = soc_dapm_prefix(dapm);
-	if (prefix)
-		w->name = kasprintf(GFP_KERNEL, "%s %s", prefix, widget->name);
-	else
-		w->name = kstrdup_const(widget->name, GFP_KERNEL);
-	if (w->name == NULL) {
-		kfree_const(w->sname);
-		kfree(w);
-		return ERR_PTR(-ENOMEM);
-	}
-
 	switch (w->id) {
 	case snd_soc_dapm_mic:
 		w->is_ep = SND_SOC_DAPM_EP_SOURCE;
@@ -3768,12 +3764,13 @@ snd_soc_dapm_new_control_unlocked(struct snd_soc_dapm_context *dapm,
 	return w;
 
 request_failed:
-	if (ret != -EPROBE_DEFER)
-		dev_err(dapm->dev, "ASoC: Failed to request %s: %d\n",
-			w->name, ret);
-
+	dev_err_probe(dapm->dev, ret, "ASoC: Failed to request %s\n",
+		      w->name);
+	kfree_const(w->name);
+name_failed:
 	kfree_const(w->sname);
 	kfree(w);
+cnew_failed:
 	return ERR_PTR(ret);
 }
 
@@ -3844,6 +3841,15 @@ snd_soc_dai_link_event_pre_pmu(struct snd_soc_dapm_widget *w,
 	unsigned int fmt;
 	int ret = 0;
 
+	/*
+	 * NOTE
+	 *
+	 * snd_pcm_hw_params is quite large (608 bytes on arm64) and is
+	 * starting to get a bit excessive for allocation on the stack,
+	 * especially when you're building with some of the KASAN type
+	 * stuff that increases stack usage.
+	 * So, we use kzalloc()/kfree() for params in this function.
+	 */
 	params = kzalloc(sizeof(*params), GFP_KERNEL);
 	if (!params)
 		return -ENOMEM;
@@ -3886,23 +3892,22 @@ snd_soc_dai_link_event_pre_pmu(struct snd_soc_dapm_widget *w,
 	 * necessary
 	 */
 	config = rtd->dai_link->params + rtd->params_select;
-	if (WARN_ON(!config)) {
+	if (!config) {
 		dev_err(w->dapm->dev, "ASoC: link config missing\n");
 		ret = -EINVAL;
 		goto out;
 	}
 
 	/* Be a little careful as we don't want to overflow the mask array */
-	if (config->formats) {
-		fmt = ffs(config->formats) - 1;
-	} else {
-		dev_warn(w->dapm->dev, "ASoC: Invalid format %llx specified\n",
-			 config->formats);
+	if (!config->formats) {
+		dev_warn(w->dapm->dev, "ASoC: Invalid format was specified\n");
 
 		ret = -EINVAL;
 		goto out;
 	}
 
+	fmt = ffs(config->formats) - 1;
+
 	snd_mask_set(hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT), fmt);
 	hw_param_interval(params, SNDRV_PCM_HW_PARAM_RATE)->min =
 		config->rate_min;
@@ -3941,7 +3946,9 @@ snd_soc_dai_link_event_pre_pmu(struct snd_soc_dapm_widget *w,
 	runtime->rate = params_rate(params);
 
 out:
+	/* see above NOTE */
 	kfree(params);
+
 	return ret;
 }
 
@@ -4354,6 +4361,7 @@ static void dapm_connect_dai_pair(struct snd_soc_card *card,
 	struct snd_soc_dapm_widget *dai, *codec, *playback_cpu, *capture_cpu;
 	struct snd_pcm_substream *substream;
 	struct snd_pcm_str *streams = rtd->pcm->streams;
+	int stream;
 
 	if (dai_link->params) {
 		playback_cpu = cpu_dai->capture_widget;
@@ -4364,37 +4372,39 @@ static void dapm_connect_dai_pair(struct snd_soc_card *card,
 	}
 
 	/* connect BE DAI playback if widgets are valid */
+	stream = SNDRV_PCM_STREAM_PLAYBACK;
 	codec = codec_dai->playback_widget;
 
 	if (playback_cpu && codec) {
-		if (dai_link->params && !rtd->playback_widget) {
-			substream = streams[SNDRV_PCM_STREAM_PLAYBACK].substream;
+		if (dai_link->params && !rtd->c2c_widget[stream]) {
+			substream = streams[stream].substream;
 			dai = snd_soc_dapm_new_dai(card, substream, "playback");
 			if (IS_ERR(dai))
 				goto capture;
-			rtd->playback_widget = dai;
+			rtd->c2c_widget[stream] = dai;
 		}
 
 		dapm_connect_dai_routes(&card->dapm, cpu_dai, playback_cpu,
-					rtd->playback_widget,
+					rtd->c2c_widget[stream],
 					codec_dai, codec);
 	}
 
 capture:
 	/* connect BE DAI capture if widgets are valid */
+	stream = SNDRV_PCM_STREAM_CAPTURE;
 	codec = codec_dai->capture_widget;
 
 	if (codec && capture_cpu) {
-		if (dai_link->params && !rtd->capture_widget) {
-			substream = streams[SNDRV_PCM_STREAM_CAPTURE].substream;
+		if (dai_link->params && !rtd->c2c_widget[stream]) {
+			substream = streams[stream].substream;
 			dai = snd_soc_dapm_new_dai(card, substream, "capture");
 			if (IS_ERR(dai))
 				return;
-			rtd->capture_widget = dai;
+			rtd->c2c_widget[stream] = dai;
 		}
 
 		dapm_connect_dai_routes(&card->dapm, codec_dai, codec,
-					rtd->capture_widget,
+					rtd->c2c_widget[stream],
 					cpu_dai, capture_cpu);
 	}
 }
@@ -4452,11 +4462,11 @@ void snd_soc_dapm_connect_dai_link_widgets(struct snd_soc_card *card)
 		if (rtd->dai_link->dynamic)
 			continue;
 
-		if (rtd->num_cpus == 1) {
+		if (rtd->dai_link->num_cpus == 1) {
 			for_each_rtd_codec_dais(rtd, i, codec_dai)
 				dapm_connect_dai_pair(card, rtd, codec_dai,
 						      asoc_rtd_to_cpu(rtd, 0));
-		} else if (rtd->num_codecs == rtd->num_cpus) {
+		} else if (rtd->dai_link->num_codecs == rtd->dai_link->num_cpus) {
 			for_each_rtd_codec_dais(rtd, i, codec_dai)
 				dapm_connect_dai_pair(card, rtd, codec_dai,
 						      asoc_rtd_to_cpu(rtd, i));
diff --git a/sound/soc/soc-generic-dmaengine-pcm.c b/sound/soc/soc-generic-dmaengine-pcm.c
index 87858462bba9..3b99f619e37e 100644
--- a/sound/soc/soc-generic-dmaengine-pcm.c
+++ b/sound/soc/soc-generic-dmaengine-pcm.c
@@ -54,7 +54,7 @@ int snd_dmaengine_pcm_prepare_slave_config(struct snd_pcm_substream *substream,
 	struct snd_dmaengine_dai_dma_data *dma_data;
 	int ret;
 
-	if (rtd->num_cpus > 1) {
+	if (rtd->dai_link->num_cpus > 1) {
 		dev_err(rtd->dev,
 			"%s doesn't support Multi CPU yet\n", __func__);
 		return -EINVAL;
@@ -105,7 +105,7 @@ dmaengine_pcm_set_runtime_hwparams(struct snd_soc_component *component,
 	struct snd_dmaengine_dai_dma_data *dma_data;
 	struct snd_pcm_hardware hw;
 
-	if (rtd->num_cpus > 1) {
+	if (rtd->dai_link->num_cpus > 1) {
 		dev_err(rtd->dev,
 			"%s doesn't support Multi CPU yet\n", __func__);
 		return -EINVAL;
@@ -179,7 +179,7 @@ static struct dma_chan *dmaengine_pcm_compat_request_channel(
 	struct dmaengine_pcm *pcm = soc_component_to_pcm(component);
 	struct snd_dmaengine_dai_dma_data *dma_data;
 
-	if (rtd->num_cpus > 1) {
+	if (rtd->dai_link->num_cpus > 1) {
 		dev_err(rtd->dev,
 			"%s doesn't support Multi CPU yet\n", __func__);
 		return NULL;
diff --git a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c
index 4f60c0a83311..fb87d6d23408 100644
--- a/sound/soc/soc-pcm.c
+++ b/sound/soc/soc-pcm.c
@@ -27,6 +27,28 @@
 #include <sound/soc-link.h>
 #include <sound/initval.h>
 
+#define soc_pcm_ret(rtd, ret) _soc_pcm_ret(rtd, __func__, ret)
+static inline int _soc_pcm_ret(struct snd_soc_pcm_runtime *rtd,
+			       const char *func, int ret)
+{
+	/* Positive, Zero values are not errors */
+	if (ret >= 0)
+		return ret;
+
+	/* Negative values might be errors */
+	switch (ret) {
+	case -EPROBE_DEFER:
+	case -ENOTSUPP:
+		break;
+	default:
+		dev_err(rtd->dev,
+			"ASoC: error at %s on %s: %d\n",
+			func, rtd->dai_link->name, ret);
+	}
+
+	return ret;
+}
+
 static inline void snd_soc_dpcm_mutex_lock(struct snd_soc_pcm_runtime *rtd)
 {
 	mutex_lock_nested(&rtd->card->pcm_mutex, rtd->card->pcm_subclass);
@@ -62,11 +84,11 @@ static inline void snd_soc_dpcm_stream_unlock_irq(struct snd_soc_pcm_runtime *rt
 
 static inline const char *soc_cpu_dai_name(struct snd_soc_pcm_runtime *rtd)
 {
-	return (rtd)->num_cpus == 1 ? asoc_rtd_to_cpu(rtd, 0)->name : "multicpu";
+	return (rtd)->dai_link->num_cpus == 1 ? asoc_rtd_to_cpu(rtd, 0)->name : "multicpu";
 }
 static inline const char *soc_codec_dai_name(struct snd_soc_pcm_runtime *rtd)
 {
-	return (rtd)->num_codecs == 1 ? asoc_rtd_to_codec(rtd, 0)->name : "multicodec";
+	return (rtd)->dai_link->num_codecs == 1 ? asoc_rtd_to_codec(rtd, 0)->name : "multicodec";
 }
 
 #ifdef CONFIG_DEBUG_FS
@@ -163,7 +185,7 @@ static ssize_t dpcm_state_read_file(struct file *file, char __user *user_buf,
 	int stream;
 	char *buf;
 
-	if (fe->num_cpus > 1) {
+	if (fe->dai_link->num_cpus > 1) {
 		dev_err(fe->dev,
 			"%s doesn't support Multi CPU yet\n", __func__);
 		return -EINVAL;
@@ -615,7 +637,7 @@ int snd_soc_runtime_calc_hw(struct snd_soc_pcm_runtime *rtd,
 	 * connected to CPU DAI(s), use CPU DAI's directly and let
 	 * channel allocation be fixed up later
 	 */
-	if (rtd->num_codecs > 1) {
+	if (rtd->dai_link->num_codecs > 1) {
 		hw->channels_min = cpu_chan_min;
 		hw->channels_max = cpu_chan_max;
 	}
@@ -723,7 +745,7 @@ static int soc_pcm_close(struct snd_pcm_substream *substream)
 	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
 
 	snd_soc_dpcm_mutex_lock(rtd);
-	soc_pcm_clean(rtd, substream, 0);
+	__soc_pcm_close(rtd, substream);
 	snd_soc_dpcm_mutex_unlock(rtd);
 	return 0;
 }
@@ -832,12 +854,10 @@ dynamic:
 	snd_soc_runtime_activate(rtd, substream->stream);
 	ret = 0;
 err:
-	if (ret < 0) {
+	if (ret < 0)
 		soc_pcm_clean(rtd, substream, 1);
-		dev_err(rtd->dev, "%s() failed (%d)", __func__, ret);
-	}
 
-	return ret;
+	return soc_pcm_ret(rtd, ret);
 }
 
 /* PCM open ops for non-DPCM streams */
@@ -852,16 +872,6 @@ static int soc_pcm_open(struct snd_pcm_substream *substream)
 	return ret;
 }
 
-static void codec2codec_close_delayed_work(struct snd_soc_pcm_runtime *rtd)
-{
-	/*
-	 * Currently nothing to do for c2c links
-	 * Since c2c links are internal nodes in the DAPM graph and
-	 * don't interface with the outside world or application layer
-	 * we don't have to do any special handling on close.
-	 */
-}
-
 /*
  * Called by ALSA when the PCM substream is prepared, can set format, sample
  * rate, etc.  This function is non atomic and can be called multiple times,
@@ -901,10 +911,7 @@ static int __soc_pcm_prepare(struct snd_soc_pcm_runtime *rtd,
 		snd_soc_dai_digital_mute(dai, 0, substream->stream);
 
 out:
-	if (ret < 0)
-		dev_err(rtd->dev, "ASoC: %s() failed (%d)\n", __func__, ret);
-
-	return ret;
+	return soc_pcm_ret(rtd, ret);
 }
 
 /* PCM prepare ops for non-DPCM streams */
@@ -1070,12 +1077,10 @@ static int __soc_pcm_hw_params(struct snd_soc_pcm_runtime *rtd,
 
 	ret = snd_soc_pcm_component_hw_params(substream, params);
 out:
-	if (ret < 0) {
+	if (ret < 0)
 		soc_pcm_hw_clean(rtd, substream, 1);
-		dev_err(rtd->dev, "ASoC: %s() failed (%d)\n", __func__, ret);
-	}
 
-	return ret;
+	return soc_pcm_ret(rtd, ret);
 }
 
 /* hw_params PCM ops for non-DPCM streams */
@@ -1374,7 +1379,7 @@ int dpcm_path_get(struct snd_soc_pcm_runtime *fe,
 	struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(fe, 0);
 	int paths;
 
-	if (fe->num_cpus > 1) {
+	if (fe->dai_link->num_cpus > 1) {
 		dev_err(fe->dev,
 			"%s doesn't support Multi CPU yet\n", __func__);
 		return -EINVAL;
@@ -1453,6 +1458,10 @@ static int dpcm_add_paths(struct snd_soc_pcm_runtime *fe, int stream,
 	struct snd_soc_dapm_widget *widget;
 	int i, new = 0, err;
 
+	/* don't connect if FE is not running */
+	if (!fe->dpcm[stream].runtime && !fe->fe_compr)
+		return new;
+
 	/* Create any new FE <--> BE connections */
 	for_each_dapm_widgets(list, i, widget) {
 
@@ -1477,10 +1486,6 @@ static int dpcm_add_paths(struct snd_soc_pcm_runtime *fe, int stream,
 			continue;
 		}
 
-		/* don't connect if FE is not running */
-		if (!fe->dpcm[stream].runtime && !fe->fe_compr)
-			continue;
-
 		/*
 		 * Filter for systems with 'component_chaining' enabled.
 		 * This helps to avoid unnecessary re-configuration of an
@@ -1637,10 +1642,7 @@ int dpcm_be_dai_startup(struct snd_soc_pcm_runtime *fe, int stream)
 unwind:
 	dpcm_be_dai_startup_rollback(fe, stream, dpcm);
 
-	dev_err(fe->dev, "ASoC: %s() failed at %s (%d)\n",
-		__func__, be->dai_link->name, err);
-
-	return err;
+	return soc_pcm_ret(fe, err);
 }
 
 static void dpcm_runtime_setup_fe(struct snd_pcm_substream *substream)
@@ -1749,7 +1751,7 @@ static void dpcm_runtime_setup_be_chan(struct snd_pcm_substream *substream)
 		 * chan min/max cannot be enforced if there are multiple CODEC
 		 * DAIs connected to a single CPU DAI, use CPU DAI's directly
 		 */
-		if (be->num_codecs == 1) {
+		if (be->dai_link->num_codecs == 1) {
 			struct snd_soc_pcm_stream *codec_stream = snd_soc_dai_get_pcm_stream(
 				asoc_rtd_to_codec(be, 0), stream);
 
@@ -1840,10 +1842,7 @@ static int dpcm_apply_symmetry(struct snd_pcm_substream *fe_substream,
 		}
 	}
 error:
-	if (err < 0)
-		dev_err(fe->dev, "ASoC: %s failed (%d)\n", __func__, err);
-
-	return err;
+	return soc_pcm_ret(fe, err);
 }
 
 static int dpcm_fe_dai_startup(struct snd_pcm_substream *fe_substream)
@@ -1880,10 +1879,7 @@ unwind:
 be_err:
 	dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO);
 
-	if (ret < 0)
-		dev_err(fe->dev, "%s() failed (%d)\n", __func__, ret);
-
-	return ret;
+	return soc_pcm_ret(fe, ret);
 }
 
 static int dpcm_fe_dai_shutdown(struct snd_pcm_substream *substream)
@@ -2082,10 +2078,7 @@ out:
 	dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO);
 	snd_soc_dpcm_mutex_unlock(fe);
 
-	if (ret < 0)
-		dev_err(fe->dev, "ASoC: %s failed (%d)\n", __func__, ret);
-
-	return ret;
+	return soc_pcm_ret(fe, ret);
 }
 
 int dpcm_be_dai_trigger(struct snd_soc_pcm_runtime *fe, int stream,
@@ -2254,10 +2247,7 @@ next:
 		if (ret)
 			break;
 	}
-	if (ret < 0)
-		dev_err(fe->dev, "ASoC: %s() failed at %s (%d)\n",
-			__func__, be->dai_link->name, ret);
-	return ret;
+	return soc_pcm_ret(fe, ret);
 }
 EXPORT_SYMBOL_GPL(dpcm_be_dai_trigger);
 
@@ -2428,10 +2418,7 @@ int dpcm_be_dai_prepare(struct snd_soc_pcm_runtime *fe, int stream)
 		be->dpcm[stream].state = SND_SOC_DPCM_STATE_PREPARE;
 	}
 
-	if (ret < 0)
-		dev_err(fe->dev, "ASoC: %s() failed (%d)\n", __func__, ret);
-
-	return ret;
+	return soc_pcm_ret(fe, ret);
 }
 
 static int dpcm_fe_dai_prepare(struct snd_pcm_substream *substream)
@@ -2468,10 +2455,7 @@ out:
 	dpcm_set_fe_update_state(fe, stream, SND_SOC_DPCM_UPDATE_NO);
 	snd_soc_dpcm_mutex_unlock(fe);
 
-	if (ret < 0)
-		dev_err(fe->dev, "ASoC: %s() failed (%d)\n", __func__, ret);
-
-	return ret;
+	return soc_pcm_ret(fe, ret);
 }
 
 static int dpcm_run_update_shutdown(struct snd_soc_pcm_runtime *fe, int stream)
@@ -2504,10 +2488,7 @@ static int dpcm_run_update_shutdown(struct snd_soc_pcm_runtime *fe, int stream)
 	/* run the stream event for each BE */
 	dpcm_dapm_stream_event(fe, stream, SND_SOC_DAPM_STREAM_NOP);
 
-	if (err < 0)
-		dev_err(fe->dev, "ASoC: %s() failed (%d)\n", __func__, err);
-
-	return err;
+	return soc_pcm_ret(fe, err);
 }
 
 static int dpcm_run_update_startup(struct snd_soc_pcm_runtime *fe, int stream)
@@ -2597,10 +2578,7 @@ disconnect:
 				dpcm->state = SND_SOC_DPCM_LINK_STATE_FREE;
 	}
 
-	if (ret < 0)
-		dev_err(fe->dev, "ASoC: %s() failed (%d)\n", __func__, ret);
-
-	return ret;
+	return soc_pcm_ret(fe, ret);
 }
 
 static int soc_dpcm_fe_runtime_update(struct snd_soc_pcm_runtime *fe, int new)
@@ -2612,7 +2590,7 @@ static int soc_dpcm_fe_runtime_update(struct snd_soc_pcm_runtime *fe, int new)
 	if (!fe->dai_link->dynamic)
 		return 0;
 
-	if (fe->num_cpus > 1) {
+	if (fe->dai_link->num_cpus > 1) {
 		dev_err(fe->dev,
 			"%s doesn't support Multi CPU yet\n", __func__);
 		return -EINVAL;
@@ -2756,7 +2734,7 @@ static int soc_get_playback_capture(struct snd_soc_pcm_runtime *rtd,
 	struct snd_soc_dai *cpu_dai;
 	int i;
 
-	if (rtd->dai_link->dynamic && rtd->num_cpus > 1) {
+	if (rtd->dai_link->dynamic && rtd->dai_link->num_cpus > 1) {
 		dev_err(rtd->dev,
 			"DPCM doesn't support Multi CPU for Front-Ends yet\n");
 		return -EINVAL;
@@ -2808,9 +2786,9 @@ static int soc_get_playback_capture(struct snd_soc_pcm_runtime *rtd,
 			SNDRV_PCM_STREAM_CAPTURE : SNDRV_PCM_STREAM_PLAYBACK;
 
 		for_each_rtd_codec_dais(rtd, i, codec_dai) {
-			if (rtd->num_cpus == 1) {
+			if (rtd->dai_link->num_cpus == 1) {
 				cpu_dai = asoc_rtd_to_cpu(rtd, 0);
-			} else if (rtd->num_cpus == rtd->num_codecs) {
+			} else if (rtd->dai_link->num_cpus == rtd->dai_link->num_codecs) {
 				cpu_dai = asoc_rtd_to_cpu(rtd, i);
 			} else {
 				dev_err(rtd->card->dev,
@@ -2899,14 +2877,19 @@ int soc_new_pcm(struct snd_soc_pcm_runtime *rtd, int num)
 		return ret;
 
 	/* DAPM dai link stream work */
-	if (rtd->dai_link->params)
-		rtd->close_delayed_work_func = codec2codec_close_delayed_work;
-	else
+	/*
+	 * Currently nothing to do for c2c links
+	 * Since c2c links are internal nodes in the DAPM graph and
+	 * don't interface with the outside world or application layer
+	 * we don't have to do any special handling on close.
+	 */
+	if (!rtd->dai_link->params)
 		rtd->close_delayed_work_func = snd_soc_close_delayed_work;
 
 	rtd->pcm = pcm;
 	pcm->nonatomic = rtd->dai_link->nonatomic;
 	pcm->private_data = rtd;
+	pcm->no_device_suspend = true;
 
 	if (rtd->dai_link->no_pcm || rtd->dai_link->params) {
 		if (playback)
@@ -2961,8 +2944,6 @@ int soc_new_pcm(struct snd_soc_pcm_runtime *rtd, int num)
 	ret = snd_soc_pcm_component_new(rtd);
 	if (ret < 0)
 		return ret;
-
-	pcm->no_device_suspend = true;
 out:
 	dev_dbg(rtd->card->dev, "%s <-> %s mapping ok\n",
 		soc_codec_dai_name(rtd), soc_cpu_dai_name(rtd));
diff --git a/sound/soc/soc-topology.c b/sound/soc/soc-topology.c
index b101db85446f..c3be24b2fac5 100644
--- a/sound/soc/soc-topology.c
+++ b/sound/soc/soc-topology.c
@@ -1755,6 +1755,7 @@ static int soc_tplg_fe_link_create(struct soc_tplg *tplg,
 
 	/* enable DPCM */
 	link->dynamic = 1;
+	link->ignore_pmdown_time = 1;
 	link->dpcm_playback = le32_to_cpu(pcm->playback);
 	link->dpcm_capture = le32_to_cpu(pcm->capture);
 	if (pcm->flag_mask)
diff --git a/sound/soc/soc-utils-test.c b/sound/soc/soc-utils-test.c
index 5ad8e23af49a..616d2c926dd1 100644
--- a/sound/soc/soc-utils-test.c
+++ b/sound/soc/soc-utils-test.c
@@ -170,8 +170,54 @@ static void test_tdm_params_to_bclk(struct kunit *test)
 	}
 }
 
+static void test_snd_soc_params_to_bclk_one(struct kunit *test,
+					    unsigned int rate, snd_pcm_format_t fmt,
+					    unsigned int channels,
+					    unsigned int expected_bclk)
+{
+	struct snd_pcm_hw_params params;
+	int got_bclk;
+
+	_snd_pcm_hw_params_any(&params);
+	snd_mask_none(hw_param_mask(&params, SNDRV_PCM_HW_PARAM_FORMAT));
+	hw_param_interval(&params, SNDRV_PCM_HW_PARAM_RATE)->min = rate;
+	hw_param_interval(&params, SNDRV_PCM_HW_PARAM_RATE)->max = rate;
+	hw_param_interval(&params, SNDRV_PCM_HW_PARAM_CHANNELS)->min = channels;
+	hw_param_interval(&params, SNDRV_PCM_HW_PARAM_CHANNELS)->max = channels;
+	params_set_format(&params, fmt);
+
+	got_bclk = snd_soc_params_to_bclk(&params);
+	pr_debug("%s: r=%u sb=%u ch=%u expected=%u got=%d\n",
+		 __func__,
+		 rate, params_width(&params), channels, expected_bclk, got_bclk);
+	KUNIT_ASSERT_EQ(test, expected_bclk, (unsigned int)got_bclk);
+}
+
+static void test_snd_soc_params_to_bclk(struct kunit *test)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(tdm_params_to_bclk_cases); ++i) {
+		/*
+		 * snd_soc_params_to_bclk() is all the test cases where
+		 * snd_pcm_hw_params values are not overridden.
+		 */
+		if (tdm_params_to_bclk_cases[i].tdm_width |
+		    tdm_params_to_bclk_cases[i].tdm_slots |
+		    tdm_params_to_bclk_cases[i].slot_multiple)
+			continue;
+
+		test_snd_soc_params_to_bclk_one(test,
+						tdm_params_to_bclk_cases[i].rate,
+						tdm_params_to_bclk_cases[i].fmt,
+						tdm_params_to_bclk_cases[i].channels,
+						tdm_params_to_bclk_cases[i].bclk);
+	}
+}
+
 static struct kunit_case soc_utils_test_cases[] = {
 	KUNIT_CASE(test_tdm_params_to_bclk),
+	KUNIT_CASE(test_snd_soc_params_to_bclk),
 	{}
 };
 
diff --git a/sound/soc/soc-utils.c b/sound/soc/soc-utils.c
index 70c380c0ac7b..a3b6df2378b4 100644
--- a/sound/soc/soc-utils.c
+++ b/sound/soc/soc-utils.c
@@ -56,23 +56,24 @@ EXPORT_SYMBOL_GPL(snd_soc_params_to_bclk);
 /**
  * snd_soc_tdm_params_to_bclk - calculate bclk from params and tdm slot info.
  *
- * Calculate the bclk from the params sample rate and the tdm slot count and
- * tdm slot width. Either or both of tdm_width and tdm_slots can be 0.
+ * Calculate the bclk from the params sample rate, the tdm slot count and the
+ * tdm slot width. Optionally round-up the slot count to a given multiple.
+ * Either or both of tdm_width and tdm_slots can be 0.
  *
- * If tdm_width == 0 and tdm_slots > 0:	the params_width will be used.
- * If tdm_width > 0 and tdm_slots == 0:	the params_channels will be used
- *					as the slot count.
- * Both tdm_width and tdm_slots are 0:	this is equivalent to calling
- *					snd_soc_params_to_bclk().
+ * If tdm_width == 0:	use params_width() as the slot width.
+ * If tdm_slots == 0:	use params_channels() as the slot count.
  *
- * If slot_multiple > 1 the slot count (or params_channels if tdm_slots == 0)
- * will be rounded up to a multiple of this value. This is mainly useful for
+ * If slot_multiple > 1 the slot count (or params_channels() if tdm_slots == 0)
+ * will be rounded up to a multiple of slot_multiple. This is mainly useful for
  * I2S mode, which has a left and right phase so the number of slots is always
  * a multiple of 2.
  *
+ * If tdm_width == 0 && tdm_slots == 0 && slot_multiple < 2, this is equivalent
+ * to calling snd_soc_params_to_bclk().
+ *
  * @params:        Pointer to struct_pcm_hw_params.
- * @tdm_width:     Width in bits of the tdm slots.
- * @tdm_slots:     Number of tdm slots per frame.
+ * @tdm_width:     Width in bits of the tdm slots. Must be >= 0.
+ * @tdm_slots:     Number of tdm slots per frame. Must be >= 0.
  * @slot_multiple: If >1 roundup slot count to a multiple of this value.
  *
  * Return: bclk frequency in Hz, else a negative error code if params format
diff --git a/sound/soc/sof/Makefile b/sound/soc/sof/Makefile
index 9a74ed116ed9..eab7cc53f71a 100644
--- a/sound/soc/sof/Makefile
+++ b/sound/soc/sof/Makefile
@@ -9,7 +9,8 @@ snd-sof-objs +=	ipc3.o ipc3-loader.o ipc3-topology.o ipc3-control.o ipc3-pcm.o\
 		ipc3-dtrace.o
 endif
 ifneq ($(CONFIG_SND_SOC_SOF_INTEL_IPC4),)
-snd-sof-objs += ipc4.o ipc4-loader.o ipc4-topology.o ipc4-control.o ipc4-pcm.o
+snd-sof-objs += ipc4.o ipc4-loader.o ipc4-topology.o ipc4-control.o ipc4-pcm.o\
+		ipc4-mtrace.o
 endif
 
 # SOF client support
diff --git a/sound/soc/sof/amd/Kconfig b/sound/soc/sof/amd/Kconfig
index 190c85d57047..a305ea6efea9 100644
--- a/sound/soc/sof/amd/Kconfig
+++ b/sound/soc/sof/amd/Kconfig
@@ -31,4 +31,14 @@ config SND_SOC_SOF_AMD_RENOIR
 	select SND_SOC_SOF_AMD_COMMON
 	help
 	  Select this option for SOF support on AMD Renoir platform
+
+config SND_SOC_SOF_AMD_REMBRANDT
+	tristate "SOF support for REMBRANDT"
+	depends on SND_SOC_SOF_PCI
+	select SND_SOC_SOF_AMD_COMMON
+	help
+	  Select this option for SOF support on AMD Rembrandt platform
+	  Say Y if you want to enable SOF on Rembrandt.
+	  If unsure select "N".
+
 endif
diff --git a/sound/soc/sof/amd/Makefile b/sound/soc/sof/amd/Makefile
index 7b9f1a0af3c8..5626d13b3e69 100644
--- a/sound/soc/sof/amd/Makefile
+++ b/sound/soc/sof/amd/Makefile
@@ -4,8 +4,10 @@
 #
 # Copyright(c) 2021 Advanced Micro Devices, Inc. All rights reserved.
 
-snd-sof-amd-acp-objs := acp.o acp-loader.o acp-ipc.o acp-pcm.o acp-stream.o acp-trace.o
+snd-sof-amd-acp-objs := acp.o acp-loader.o acp-ipc.o acp-pcm.o acp-stream.o acp-trace.o acp-common.o
 snd-sof-amd-renoir-objs := pci-rn.o renoir.o
+snd-sof-amd-rembrandt-objs := pci-rmb.o rembrandt.o
 
 obj-$(CONFIG_SND_SOC_SOF_AMD_COMMON) += snd-sof-amd-acp.o
 obj-$(CONFIG_SND_SOC_SOF_AMD_RENOIR) +=snd-sof-amd-renoir.o
+obj-$(CONFIG_SND_SOC_SOF_AMD_REMBRANDT) +=snd-sof-amd-rembrandt.o
diff --git a/sound/soc/sof/amd/acp-common.c b/sound/soc/sof/amd/acp-common.c
new file mode 100644
index 000000000000..27b95187356e
--- /dev/null
+++ b/sound/soc/sof/amd/acp-common.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license. When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2022 Advanced Micro Devices, Inc.
+//
+// Authors: Ajit Kumar Pandey <AjitKumar.Pandey@amd.com>
+//	    V sujith kumar Reddy <Vsujithkumar.Reddy@amd.com>
+
+/* ACP-specific Common code */
+
+#include "../sof-priv.h"
+#include "../sof-audio.h"
+#include "../ops.h"
+#include "../sof-audio.h"
+#include "acp.h"
+#include "acp-dsp-offset.h"
+
+int acp_dai_probe(struct snd_soc_dai *dai)
+{
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(dai->component);
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
+	unsigned int val;
+
+	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->i2s_pin_config_offset);
+	if (val != desc->i2s_mode) {
+		dev_err(sdev->dev, "I2S Mode is not supported (I2S_PIN_CONFIG: %#x)\n", val);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_NS(acp_dai_probe, SND_SOC_SOF_AMD_COMMON);
+
+struct snd_soc_acpi_mach *amd_sof_machine_select(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_pdata *sof_pdata = sdev->pdata;
+	const struct sof_dev_desc *desc = sof_pdata->desc;
+	struct snd_soc_acpi_mach *mach;
+
+	mach = snd_soc_acpi_find_machine(desc->machines);
+	if (!mach) {
+		dev_warn(sdev->dev, "No matching ASoC machine driver found\n");
+		return NULL;
+	}
+
+	sof_pdata->tplg_filename = mach->sof_tplg_filename;
+	sof_pdata->fw_filename = mach->fw_filename;
+
+	return mach;
+}
+
+/* AMD Common DSP ops */
+struct snd_sof_dsp_ops sof_acp_common_ops = {
+	/* probe and remove */
+	.probe			= amd_sof_acp_probe,
+	.remove			= amd_sof_acp_remove,
+
+	/* Register IO */
+	.write			= sof_io_write,
+	.read			= sof_io_read,
+
+	/* Block IO */
+	.block_read		= acp_dsp_block_read,
+	.block_write		= acp_dsp_block_write,
+
+	/*Firmware loading */
+	.load_firmware		= snd_sof_load_firmware_memcpy,
+	.pre_fw_run		= acp_dsp_pre_fw_run,
+	.get_bar_index		= acp_get_bar_index,
+
+	/* DSP core boot */
+	.run			= acp_sof_dsp_run,
+
+	/*IPC */
+	.send_msg		= acp_sof_ipc_send_msg,
+	.ipc_msg_data		= acp_sof_ipc_msg_data,
+	.get_mailbox_offset	= acp_sof_ipc_get_mailbox_offset,
+	.get_window_offset      = acp_sof_ipc_get_window_offset,
+	.irq_thread		= acp_sof_ipc_irq_thread,
+
+	/* stream callbacks */
+	.pcm_open		= acp_pcm_open,
+	.pcm_close		= acp_pcm_close,
+	.pcm_hw_params		= acp_pcm_hw_params,
+
+	.hw_info		= SNDRV_PCM_INFO_MMAP |
+				  SNDRV_PCM_INFO_MMAP_VALID |
+				  SNDRV_PCM_INFO_INTERLEAVED |
+				  SNDRV_PCM_INFO_PAUSE |
+				  SNDRV_PCM_INFO_NO_PERIOD_WAKEUP,
+
+	/* Machine driver callbacks */
+	.machine_select		= amd_sof_machine_select,
+	.machine_register	= sof_machine_register,
+	.machine_unregister	= sof_machine_unregister,
+
+	/* Trace Logger */
+	.trace_init		= acp_sof_trace_init,
+	.trace_release		= acp_sof_trace_release,
+
+	/* PM */
+	.suspend                = amd_sof_acp_suspend,
+	.resume                 = amd_sof_acp_resume,
+};
+EXPORT_SYMBOL_NS(sof_acp_common_ops, SND_SOC_SOF_AMD_COMMON);
+
+MODULE_IMPORT_NS(SND_SOC_SOF_AMD_COMMON);
+MODULE_DESCRIPTION("ACP SOF COMMON Driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/amd/acp-dsp-offset.h b/sound/soc/sof/amd/acp-dsp-offset.h
index 56cefd4a84fc..de5726251dc6 100644
--- a/sound/soc/sof/amd/acp-dsp-offset.h
+++ b/sound/soc/sof/amd/acp-dsp-offset.h
@@ -48,22 +48,29 @@
 #define ACP_SOFT_RESET				0x1000
 #define ACP_CONTROL				0x1004
 
-#define ACP_I2S_PIN_CONFIG			0x1400
+#define ACP3X_I2S_PIN_CONFIG			0x1400
+#define ACP6X_I2S_PIN_CONFIG			0x1440
 
-/* Registers from ACP_PGFSM block */
-#define ACP_PGFSM_CONTROL			0x141C
-#define ACP_PGFSM_STATUS			0x1420
-#define ACP_CLKMUX_SEL				0x1424
+/* Registers offsets from ACP_PGFSM block */
+#define ACP3X_PGFSM_BASE			0x141C
+#define ACP6X_PGFSM_BASE                        0x1024
+#define PGFSM_CONTROL_OFFSET			0x0
+#define PGFSM_STATUS_OFFSET			0x4
+#define ACP3X_CLKMUX_SEL			0x1424
+#define ACP6X_CLKMUX_SEL			0x102C
 
 /* Registers from ACP_INTR block */
-#define ACP_EXTERNAL_INTR_ENB			0x1800
-#define ACP_EXTERNAL_INTR_CNTL			0x1804
-#define ACP_EXTERNAL_INTR_STAT			0x1808
-#define ACP_DSP_SW_INTR_CNTL			0x1814
-#define ACP_DSP_SW_INTR_STAT                    0x1818
-#define ACP_SW_INTR_TRIG                        0x181C
+#define ACP3X_EXT_INTR_STAT			0x1808
+#define ACP6X_EXT_INTR_STAT                     0x1A0C
+
+#define ACP3X_DSP_SW_INTR_BASE			0x1814
+#define ACP6X_DSP_SW_INTR_BASE                  0x1808
+#define DSP_SW_INTR_CNTL_OFFSET			0x0
+#define DSP_SW_INTR_STAT_OFFSET			0x4
+#define DSP_SW_INTR_TRIG_OFFSET			0x8
 #define ACP_ERROR_STATUS			0x18C4
-#define ACP_AXI2DAGB_SEM_0			0x1880
+#define ACP3X_AXI2DAGB_SEM_0			0x1880
+#define ACP6X_AXI2DAGB_SEM_0			0x1874
 
 /* Registers from ACP_SHA block */
 #define ACP_SHA_DSP_FW_QUALIFIER		0x1C70
@@ -77,5 +84,5 @@
 #define ACP_SHA_PSP_ACK                         0x1C74
 
 #define ACP_SCRATCH_REG_0			0x10000
-
+#define ACP6X_DSP_FUSION_RUNSTALL		0x0644
 #endif
diff --git a/sound/soc/sof/amd/acp-ipc.c b/sound/soc/sof/amd/acp-ipc.c
index e1842f037083..dd030566e372 100644
--- a/sound/soc/sof/amd/acp-ipc.c
+++ b/sound/soc/sof/amd/acp-ipc.c
@@ -30,30 +30,36 @@ EXPORT_SYMBOL_NS(acp_mailbox_read, SND_SOC_SOF_AMD_COMMON);
 static void acpbus_trigger_host_to_dsp_swintr(struct acp_dev_data *adata)
 {
 	struct snd_sof_dev *sdev = adata->dev;
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
 	u32 swintr_trigger;
 
-	swintr_trigger = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_SW_INTR_TRIG);
+	swintr_trigger = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->dsp_intr_base +
+						DSP_SW_INTR_TRIG_OFFSET);
 	swintr_trigger |= 0x01;
-	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SW_INTR_TRIG, swintr_trigger);
+	snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->dsp_intr_base + DSP_SW_INTR_TRIG_OFFSET,
+			  swintr_trigger);
 }
 
 static void acp_ipc_host_msg_set(struct snd_sof_dev *sdev)
 {
-	unsigned int host_msg = offsetof(struct scratch_ipc_conf, sof_host_msg_write);
+	unsigned int host_msg = sdev->debug_box.offset +
+				offsetof(struct scratch_ipc_conf, sof_host_msg_write);
 
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SCRATCH_REG_0 + host_msg, 1);
 }
 
 static void acp_dsp_ipc_host_done(struct snd_sof_dev *sdev)
 {
-	unsigned int dsp_msg = offsetof(struct scratch_ipc_conf, sof_dsp_msg_write);
+	unsigned int dsp_msg = sdev->debug_box.offset +
+			       offsetof(struct scratch_ipc_conf, sof_dsp_msg_write);
 
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SCRATCH_REG_0 + dsp_msg, 0);
 }
 
 static void acp_dsp_ipc_dsp_done(struct snd_sof_dev *sdev)
 {
-	unsigned int dsp_ack = offsetof(struct scratch_ipc_conf, sof_dsp_ack_write);
+	unsigned int dsp_ack = sdev->debug_box.offset +
+			       offsetof(struct scratch_ipc_conf, sof_dsp_ack_write);
 
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_SCRATCH_REG_0 + dsp_ack, 0);
 }
@@ -61,10 +67,11 @@ static void acp_dsp_ipc_dsp_done(struct snd_sof_dev *sdev)
 int acp_sof_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg)
 {
 	struct acp_dev_data *adata = sdev->pdata->hw_pdata;
-	unsigned int offset = offsetof(struct scratch_ipc_conf, sof_in_box);
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
+	unsigned int offset = sdev->host_box.offset;
 	unsigned int count = ACP_HW_SEM_RETRY_COUNT;
 
-	while (snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_AXI2DAGB_SEM_0)) {
+	while (snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->hw_semaphore_offset)) {
 		/* Wait until acquired HW Semaphore Lock or timeout*/
 		count--;
 		if (!count) {
@@ -80,7 +87,7 @@ int acp_sof_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg)
 	acpbus_trigger_host_to_dsp_swintr(adata);
 
 	/* Unlock or Release HW Semaphore */
-	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_AXI2DAGB_SEM_0, 0x0);
+	snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->hw_semaphore_offset, 0x0);
 
 	return 0;
 }
@@ -91,7 +98,7 @@ static void acp_dsp_ipc_get_reply(struct snd_sof_dev *sdev)
 	struct snd_sof_ipc_msg *msg = sdev->msg;
 	struct sof_ipc_reply reply;
 	struct sof_ipc_cmd_hdr *hdr;
-	unsigned int offset = offsetof(struct scratch_ipc_conf, sof_in_box);
+	unsigned int offset = sdev->host_box.offset;
 	int ret = 0;
 
        /*
@@ -141,11 +148,19 @@ out:
 irqreturn_t acp_sof_ipc_irq_thread(int irq, void *context)
 {
 	struct snd_sof_dev *sdev = context;
-	unsigned int dsp_msg_write = offsetof(struct scratch_ipc_conf, sof_dsp_msg_write);
-	unsigned int dsp_ack_write = offsetof(struct scratch_ipc_conf, sof_dsp_ack_write);
+	unsigned int dsp_msg_write = sdev->debug_box.offset +
+				     offsetof(struct scratch_ipc_conf, sof_dsp_msg_write);
+	unsigned int dsp_ack_write = sdev->debug_box.offset +
+				     offsetof(struct scratch_ipc_conf, sof_dsp_ack_write);
 	bool ipc_irq = false;
 	int dsp_msg, dsp_ack;
 
+	if (sdev->first_boot && sdev->fw_state != SOF_FW_BOOT_COMPLETE) {
+		snd_sof_ipc_msgs_rx(sdev);
+		acp_dsp_ipc_host_done(sdev);
+		return IRQ_HANDLED;
+	}
+
 	dsp_msg = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_SCRATCH_REG_0 + dsp_msg_write);
 	if (dsp_msg) {
 		snd_sof_ipc_msgs_rx(sdev);
@@ -175,7 +190,7 @@ EXPORT_SYMBOL_NS(acp_sof_ipc_irq_thread, SND_SOC_SOF_AMD_COMMON);
 int acp_sof_ipc_msg_data(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream,
 			 void *p, size_t sz)
 {
-	unsigned int offset = offsetof(struct scratch_ipc_conf, sof_out_box);
+	unsigned int offset = sdev->dsp_box.offset;
 
 	if (!substream || !sdev->stream_box.size)
 		acp_mailbox_read(sdev, offset, p, sz);
@@ -186,8 +201,16 @@ EXPORT_SYMBOL_NS(acp_sof_ipc_msg_data, SND_SOC_SOF_AMD_COMMON);
 
 int acp_sof_ipc_get_mailbox_offset(struct snd_sof_dev *sdev)
 {
-	return ACP_SCRATCH_MEMORY_ADDRESS;
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
+
+	return desc->sram_pte_offset;
 }
 EXPORT_SYMBOL_NS(acp_sof_ipc_get_mailbox_offset, SND_SOC_SOF_AMD_COMMON);
 
+int acp_sof_ipc_get_window_offset(struct snd_sof_dev *sdev, u32 id)
+{
+	return 0;
+}
+EXPORT_SYMBOL_NS(acp_sof_ipc_get_window_offset, SND_SOC_SOF_AMD_COMMON);
+
 MODULE_DESCRIPTION("AMD ACP sof-ipc driver");
diff --git a/sound/soc/sof/amd/acp-loader.c b/sound/soc/sof/amd/acp-loader.c
index 7ca51e0f3b1b..d1e74baf5d8b 100644
--- a/sound/soc/sof/amd/acp-loader.c
+++ b/sound/soc/sof/amd/acp-loader.c
@@ -30,9 +30,10 @@
 int acp_dsp_block_read(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_type,
 		       u32 offset, void *dest, size_t size)
 {
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
 	switch (blk_type) {
 	case SOF_FW_BLK_TYPE_SRAM:
-		offset = offset - ACP_SCRATCH_MEMORY_ADDRESS;
+		offset = offset - desc->sram_pte_offset;
 		memcpy_from_scratch(sdev, offset, dest, size);
 		break;
 	default:
@@ -49,6 +50,7 @@ int acp_dsp_block_write(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_t
 {
 	struct snd_sof_pdata *plat_data = sdev->pdata;
 	struct pci_dev *pci = to_pci_dev(sdev->dev);
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
 	struct acp_dev_data *adata;
 	void *dest;
 	u32 dma_size, page_count;
@@ -84,7 +86,7 @@ int acp_dsp_block_write(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_t
 		adata->fw_data_bin_size = size + offset;
 		break;
 	case SOF_FW_BLK_TYPE_SRAM:
-		offset = offset - ACP_SCRATCH_MEMORY_ADDRESS;
+		offset = offset - desc->sram_pte_offset;
 		memcpy_to_scratch(sdev, offset, src, size);
 		return 0;
 	default:
@@ -105,14 +107,13 @@ EXPORT_SYMBOL_NS(acp_get_bar_index, SND_SOC_SOF_AMD_COMMON);
 
 static void configure_pte_for_fw_loading(int type, int num_pages, struct acp_dev_data *adata)
 {
-	struct snd_sof_dev *sdev;
+	struct snd_sof_dev *sdev = adata->dev;
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
 	unsigned int low, high;
 	dma_addr_t addr;
 	u16 page_idx;
 	u32 offset;
 
-	sdev = adata->dev;
-
 	switch (type) {
 	case FW_BIN:
 		offset = FW_BIN_PTE_OFFSET;
@@ -129,7 +130,7 @@ static void configure_pte_for_fw_loading(int type, int num_pages, struct acp_dev
 
 	/* Group Enable */
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACPAXI2AXI_ATU_BASE_ADDR_GRP_1,
-			  ACP_SRAM_PTE_OFFSET | BIT(31));
+			  desc->sram_pte_offset | BIT(31));
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACPAXI2AXI_ATU_PAGE_SIZE_GRP_1,
 			  PAGE_SIZE_4K_ENABLE);
 
@@ -197,12 +198,19 @@ EXPORT_SYMBOL_NS(acp_dsp_pre_fw_run, SND_SOC_SOF_AMD_COMMON);
 
 int acp_sof_dsp_run(struct snd_sof_dev *sdev)
 {
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
 	int val;
 
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_DSP0_RUNSTALL, ACP_DSP_RUN);
 	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_DSP0_RUNSTALL);
 	dev_dbg(sdev->dev, "ACP_DSP0_RUNSTALL : 0x%0x\n", val);
 
+	/* Some platforms won't support fusion DSP,keep offset zero for no support */
+	if (desc->fusion_dsp_offset) {
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->fusion_dsp_offset, ACP_DSP_RUN);
+		val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->fusion_dsp_offset);
+		dev_dbg(sdev->dev, "ACP_DSP0_FUSION_RUNSTALL : 0x%0x\n", val);
+	}
 	return 0;
 }
 EXPORT_SYMBOL_NS(acp_sof_dsp_run, SND_SOC_SOF_AMD_COMMON);
diff --git a/sound/soc/sof/amd/acp-pcm.c b/sound/soc/sof/amd/acp-pcm.c
index 0ba8ae46bd76..727c3a784a20 100644
--- a/sound/soc/sof/amd/acp-pcm.c
+++ b/sound/soc/sof/amd/acp-pcm.c
@@ -42,7 +42,8 @@ int acp_pcm_hw_params(struct snd_sof_dev *sdev, struct snd_pcm_substream *substr
 
 	/* write buffer size of stream in scratch memory */
 
-	buf_offset = offsetof(struct scratch_reg_conf, buf_size);
+	buf_offset = sdev->debug_box.offset +
+		     offsetof(struct scratch_reg_conf, buf_size);
 	index = stream->stream_tag - 1;
 	buf_offset = buf_offset + index * 4;
 
diff --git a/sound/soc/sof/amd/acp-stream.c b/sound/soc/sof/amd/acp-stream.c
index b3ca4a90dbf8..6f40ef7ba85e 100644
--- a/sound/soc/sof/amd/acp-stream.c
+++ b/sound/soc/sof/amd/acp-stream.c
@@ -26,6 +26,7 @@
 
 int acp_dsp_stream_config(struct snd_sof_dev *sdev, struct acp_dsp_stream *stream)
 {
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
 	unsigned int pte_reg, pte_size, phy_addr_offset, index;
 	int stream_tag = stream->stream_tag;
 	u32 low, high, offset, reg_val;
@@ -88,7 +89,8 @@ int acp_dsp_stream_config(struct snd_sof_dev *sdev, struct acp_dsp_stream *strea
 
 	/* write phy_addr in scratch memory */
 
-	phy_addr_offset = offsetof(struct scratch_reg_conf, reg_offset);
+	phy_addr_offset = sdev->debug_box.offset +
+			  offsetof(struct scratch_reg_conf, reg_offset);
 	index = stream_tag - 1;
 	phy_addr_offset = phy_addr_offset + index * 4;
 
@@ -96,7 +98,8 @@ int acp_dsp_stream_config(struct snd_sof_dev *sdev, struct acp_dsp_stream *strea
 			  phy_addr_offset, stream->reg_offset);
 
 	/* Group Enable */
-	reg_val = ACP_SRAM_PTE_OFFSET + offset;
+	offset = offset + sdev->debug_box.offset;
+	reg_val = desc->sram_pte_offset + offset;
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, pte_reg, reg_val | BIT(31));
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, pte_size, PAGE_SIZE_4K_ENABLE);
 
diff --git a/sound/soc/sof/amd/acp.c b/sound/soc/sof/amd/acp.c
index c40d2900dd36..36966643e36a 100644
--- a/sound/soc/sof/amd/acp.c
+++ b/sound/soc/sof/amd/acp.c
@@ -39,9 +39,11 @@ static int smn_read(struct pci_dev *dev, u32 smn_addr, u32 *data)
 static void init_dma_descriptor(struct acp_dev_data *adata)
 {
 	struct snd_sof_dev *sdev = adata->dev;
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
 	unsigned int addr;
 
-	addr = ACP_SRAM_PTE_OFFSET + offsetof(struct scratch_reg_conf, dma_desc);
+	addr = desc->sram_pte_offset + sdev->debug_box.offset +
+	       offsetof(struct scratch_reg_conf, dma_desc);
 
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_DMA_DESC_BASE_ADDR, addr);
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_DMA_DESC_MAX_NUM_DSCR, ACP_MAX_DESC_CNT);
@@ -53,8 +55,9 @@ static void configure_dma_descriptor(struct acp_dev_data *adata, unsigned short
 	struct snd_sof_dev *sdev = adata->dev;
 	unsigned int offset;
 
-	offset = ACP_SCRATCH_REG_0 + offsetof(struct scratch_reg_conf, dma_desc) +
-		 idx * sizeof(struct dma_descriptor);
+	offset = ACP_SCRATCH_REG_0 + sdev->debug_box.offset +
+		offsetof(struct scratch_reg_conf, dma_desc) +
+		idx * sizeof(struct dma_descriptor);
 
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, offset, dscr_info->src_addr);
 	snd_sof_dsp_write(sdev, ACP_DSP_BAR, offset + 0x4, dscr_info->dest_addr);
@@ -300,8 +303,9 @@ void memcpy_to_scratch(struct snd_sof_dev *sdev, u32 offset, unsigned int *src,
 static int acp_memory_init(struct snd_sof_dev *sdev)
 {
 	struct acp_dev_data *adata = sdev->pdata->hw_pdata;
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
 
-	snd_sof_dsp_update_bits(sdev, ACP_DSP_BAR, ACP_DSP_SW_INTR_CNTL,
+	snd_sof_dsp_update_bits(sdev, ACP_DSP_BAR, desc->dsp_intr_base + DSP_SW_INTR_CNTL_OFFSET,
 				ACP_DSP_INTR_EN_MASK, ACP_DSP_INTR_EN_MASK);
 	init_dma_descriptor(adata);
 
@@ -311,18 +315,20 @@ static int acp_memory_init(struct snd_sof_dev *sdev)
 static irqreturn_t acp_irq_thread(int irq, void *context)
 {
 	struct snd_sof_dev *sdev = context;
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
+	unsigned int base = desc->dsp_intr_base;
 	unsigned int val, count = ACP_HW_SEM_RETRY_COUNT;
 
-	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_EXTERNAL_INTR_STAT);
+	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->ext_intr_stat);
 	if (val & ACP_SHA_STAT) {
 		/* Clear SHA interrupt raised by PSP */
-		snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_EXTERNAL_INTR_STAT, val);
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->ext_intr_stat, val);
 		return IRQ_HANDLED;
 	}
 
-	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_DSP_SW_INTR_STAT);
+	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, base + DSP_SW_INTR_STAT_OFFSET);
 	if (val & ACP_DSP_TO_HOST_IRQ) {
-		while (snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_AXI2DAGB_SEM_0)) {
+		while (snd_sof_dsp_read(sdev, ACP_DSP_BAR, desc->hw_semaphore_offset)) {
 			/* Wait until acquired HW Semaphore lock or timeout */
 			count--;
 			if (!count) {
@@ -333,10 +339,10 @@ static irqreturn_t acp_irq_thread(int irq, void *context)
 
 		sof_ops(sdev)->irq_thread(irq, sdev);
 		val |= ACP_DSP_TO_HOST_IRQ;
-		snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_DSP_SW_INTR_STAT, val);
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, base + DSP_SW_INTR_STAT_OFFSET, val);
 
 		/* Unlock or Release HW Semaphore */
-		snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_AXI2DAGB_SEM_0, 0x0);
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->hw_semaphore_offset, 0x0);
 
 		return IRQ_HANDLED;
 	}
@@ -347,9 +353,11 @@ static irqreturn_t acp_irq_thread(int irq, void *context)
 static irqreturn_t acp_irq_handler(int irq, void *dev_id)
 {
 	struct snd_sof_dev *sdev = dev_id;
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
+	unsigned int base = desc->dsp_intr_base;
 	unsigned int val;
 
-	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_DSP_SW_INTR_STAT);
+	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, base + DSP_SW_INTR_STAT_OFFSET);
 	if (val)
 		return IRQ_WAKE_THREAD;
 
@@ -358,20 +366,22 @@ static irqreturn_t acp_irq_handler(int irq, void *dev_id)
 
 static int acp_power_on(struct snd_sof_dev *sdev)
 {
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
+	unsigned int base = desc->pgfsm_base;
 	unsigned int val;
 	int ret;
 
-	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_PGFSM_STATUS);
+	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, base + PGFSM_STATUS_OFFSET);
 
 	if (val == ACP_POWERED_ON)
 		return 0;
 
 	if (val & ACP_PGFSM_STATUS_MASK)
-		snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_PGFSM_CONTROL,
+		snd_sof_dsp_write(sdev, ACP_DSP_BAR, base + PGFSM_CONTROL_OFFSET,
 				  ACP_PGFSM_CNTL_POWER_ON_MASK);
 
-	ret = snd_sof_dsp_read_poll_timeout(sdev, ACP_DSP_BAR, ACP_PGFSM_STATUS, val, !val,
-					    ACP_REG_POLL_INTERVAL, ACP_REG_POLL_TIMEOUT_US);
+	ret = snd_sof_dsp_read_poll_timeout(sdev, ACP_DSP_BAR, base + PGFSM_STATUS_OFFSET, val,
+					    !val, ACP_REG_POLL_INTERVAL, ACP_REG_POLL_TIMEOUT_US);
 	if (ret < 0)
 		dev_err(sdev->dev, "timeout in ACP_PGFSM_STATUS read\n");
 
@@ -437,6 +447,7 @@ EXPORT_SYMBOL_NS(amd_sof_acp_suspend, SND_SOC_SOF_AMD_COMMON);
 
 int amd_sof_acp_resume(struct snd_sof_dev *sdev)
 {
+	const struct sof_amd_acp_desc *desc = get_chip_info(sdev->pdata);
 	int ret;
 
 	ret = acp_init(sdev);
@@ -445,7 +456,7 @@ int amd_sof_acp_resume(struct snd_sof_dev *sdev)
 		return ret;
 	}
 
-	snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_CLKMUX_SEL, 0x03);
+	snd_sof_dsp_write(sdev, ACP_DSP_BAR, desc->acp_clkmux_sel, 0x03);
 
 	ret = acp_memory_init(sdev);
 
@@ -507,6 +518,15 @@ int amd_sof_acp_probe(struct snd_sof_dev *sdev)
 		return ret;
 	}
 
+	sdev->dsp_box.offset = 0;
+	sdev->dsp_box.size = BOX_SIZE_512;
+
+	sdev->host_box.offset = sdev->dsp_box.offset + sdev->dsp_box.size;
+	sdev->host_box.size = BOX_SIZE_512;
+
+	sdev->debug_box.offset = sdev->host_box.offset + sdev->host_box.size;
+	sdev->debug_box.size = BOX_SIZE_1024;
+
 	acp_memory_init(sdev);
 
 	acp_dsp_stream_init(sdev);
diff --git a/sound/soc/sof/amd/acp.h b/sound/soc/sof/amd/acp.h
index 4c42b8fd6abf..dd3c072d0172 100644
--- a/sound/soc/sof/amd/acp.h
+++ b/sound/soc/sof/amd/acp.h
@@ -30,7 +30,8 @@
 #define ACP_SOFT_RESET_DONE_MASK		0x00010001
 
 #define ACP_DSP_INTR_EN_MASK			0x00000001
-#define ACP_SRAM_PTE_OFFSET			0x02050000
+#define ACP3X_SRAM_PTE_OFFSET			0x02050000
+#define ACP6X_SRAM_PTE_OFFSET			0x03800000
 #define PAGE_SIZE_4K_ENABLE			0x2
 #define ACP_PAGE_SIZE				0x1000
 #define ACP_DMA_CH_RUN				0x02
@@ -45,7 +46,7 @@
 #define ACPBUS_REG_BASE_OFFSET			ACP_DMA_CNTL_0
 
 #define ACP_DEFAULT_DRAM_LENGTH			0x00080000
-#define ACP_SCRATCH_MEMORY_ADDRESS		0x02050000
+#define ACP3X_SCRATCH_MEMORY_ADDRESS		0x02050000
 #define ACP_SYSTEM_MEMORY_WINDOW		0x4000000
 #define ACP_IRAM_BASE_ADDRESS			0x000000
 #define ACP_DATA_RAM_BASE_ADDRESS		0x01000000
@@ -54,6 +55,7 @@
 #define ACP_DSP_TO_HOST_IRQ			0x04
 
 #define HOST_BRIDGE_CZN				0x1630
+#define HOST_BRIDGE_RMB				0x14B5
 #define ACP_SHA_STAT				0x8000
 #define ACP_PSP_TIMEOUT_COUNTER			5
 #define ACP_EXT_INTR_ERROR_STAT			0x20000000
@@ -64,6 +66,9 @@
 #define MBOX_READY_MASK				0x80000000
 #define MBOX_STATUS_MASK			0xFFFF
 
+#define BOX_SIZE_512				0x200
+#define BOX_SIZE_1024				0x400
+
 struct  acp_atu_grp_pte {
 	u32 low;
 	u32 high;
@@ -88,10 +93,6 @@ struct dma_descriptor {
 
 /* Scratch memory structure for communication b/w host and dsp */
 struct  scratch_ipc_conf {
-	/* DSP mailbox */
-	u8 sof_out_box[512];
-	/* Host mailbox */
-	u8 sof_in_box[512];
 	/* Debug memory */
 	u8 sof_debug_box[1024];
 	/* Exception memory*/
@@ -139,6 +140,20 @@ struct acp_dsp_stream {
 	unsigned int reg_offset;
 };
 
+struct sof_amd_acp_desc {
+	unsigned int rev;
+	unsigned int host_bridge_id;
+	unsigned int i2s_mode;
+	u32 pgfsm_base;
+	u32 ext_intr_stat;
+	u32 dsp_intr_base;
+	u32 sram_pte_offset;
+	u32 i2s_pin_config_offset;
+	u32 hw_semaphore_offset;
+	u32 acp_clkmux_sel;
+	u32 fusion_dsp_offset;
+};
+
 /* Common device data struct for ACP devices */
 struct acp_dev_data {
 	struct snd_sof_dev  *dev;
@@ -206,8 +221,15 @@ int acp_pcm_hw_params(struct snd_sof_dev *sdev, struct snd_pcm_substream *substr
 		      struct snd_pcm_hw_params *params,
 		      struct snd_sof_platform_stream_params *platform_params);
 
+extern struct snd_sof_dsp_ops sof_acp_common_ops;
+
 extern struct snd_sof_dsp_ops sof_renoir_ops;
+int sof_renoir_ops_init(struct snd_sof_dev *sdev);
+extern struct snd_sof_dsp_ops sof_rembrandt_ops;
+int sof_rembrandt_ops_init(struct snd_sof_dev *sdev);
 
+int acp_dai_probe(struct snd_soc_dai *dai);
+struct snd_soc_acpi_mach *amd_sof_machine_select(struct snd_sof_dev *sdev);
 /* Machine configuration */
 int snd_amd_acp_find_config(struct pci_dev *pci);
 
@@ -220,10 +242,6 @@ int acp_sof_trace_release(struct snd_sof_dev *sdev);
 int amd_sof_acp_suspend(struct snd_sof_dev *sdev, u32 target_state);
 int amd_sof_acp_resume(struct snd_sof_dev *sdev);
 
-struct sof_amd_acp_desc {
-	unsigned int host_bridge_id;
-};
-
 static inline const struct sof_amd_acp_desc *get_chip_info(struct snd_sof_pdata *pdata)
 {
 	const struct sof_dev_desc *desc = pdata->desc;
diff --git a/sound/soc/sof/amd/pci-rmb.c b/sound/soc/sof/amd/pci-rmb.c
new file mode 100644
index 000000000000..4e1de462b431
--- /dev/null
+++ b/sound/soc/sof/amd/pci-rmb.c
@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license. When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2022 Advanced Micro Devices, Inc. All rights reserved.
+//
+// Authors: Ajit Kumar Pandey <AjitKumar.Pandey@amd.com>
+
+/*.
+ * PCI interface for Rembrandt ACP device
+ */
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <sound/sof.h>
+#include <sound/soc-acpi.h>
+
+#include "../ops.h"
+#include "../sof-pci-dev.h"
+#include "../../amd/mach-config.h"
+#include "acp.h"
+#include "acp-dsp-offset.h"
+
+#define ACP6x_REG_START		0x1240000
+#define ACP6x_REG_END		0x125C000
+
+static struct platform_device *dmic_dev;
+static struct platform_device *pdev;
+
+static const struct resource rembrandt_res[] = {
+	{
+		.start = 0,
+		.end = ACP6x_REG_END - ACP6x_REG_START,
+		.name = "acp_mem",
+		.flags = IORESOURCE_MEM,
+	},
+	{
+		.start = 0,
+		.end = 0,
+		.name = "acp_dai_irq",
+		.flags = IORESOURCE_IRQ,
+	},
+};
+
+static const struct sof_amd_acp_desc rembrandt_chip_info = {
+	.rev		= 6,
+	.host_bridge_id = HOST_BRIDGE_RMB,
+	.i2s_mode	= 0x0a,
+	.pgfsm_base	= ACP6X_PGFSM_BASE,
+	.ext_intr_stat	= ACP6X_EXT_INTR_STAT,
+	.dsp_intr_base	= ACP6X_DSP_SW_INTR_BASE,
+	.sram_pte_offset = ACP6X_SRAM_PTE_OFFSET,
+	.i2s_pin_config_offset = ACP6X_I2S_PIN_CONFIG,
+	.hw_semaphore_offset = ACP6X_AXI2DAGB_SEM_0,
+	.acp_clkmux_sel = ACP6X_CLKMUX_SEL,
+	.fusion_dsp_offset = ACP6X_DSP_FUSION_RUNSTALL,
+};
+
+static const struct sof_dev_desc rembrandt_desc = {
+	.machines		= snd_soc_acpi_amd_rmb_sof_machines,
+	.resindex_lpe_base	= 0,
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= -1,
+	.irqindex_host_ipc	= -1,
+	.chip_info		= &rembrandt_chip_info,
+	.ipc_supported_mask     = BIT(SOF_IPC),
+	.ipc_default            = SOF_IPC,
+	.default_fw_path	= {
+		[SOF_IPC] = "amd/sof",
+	},
+	.default_tplg_path	= {
+		[SOF_IPC] = "amd/sof-tplg",
+	},
+	.default_fw_filename	= {
+		[SOF_IPC] = "sof-rmb.ri",
+	},
+	.nocodec_tplg_filename	= "sof-acp.tplg",
+	.ops			= &sof_rembrandt_ops,
+	.ops_init		= sof_rembrandt_ops_init,
+};
+
+static int acp_pci_rmb_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
+{
+	struct platform_device_info pdevinfo;
+	struct device *dev = &pci->dev;
+	const struct resource *res_i2s;
+	struct resource *res;
+	unsigned int flag, i, addr;
+	int ret;
+
+	flag = snd_amd_acp_find_config(pci);
+	if (flag != FLAG_AMD_SOF && flag != FLAG_AMD_SOF_ONLY_DMIC)
+		return -ENODEV;
+
+	ret = sof_pci_probe(pci, pci_id);
+	if (ret != 0)
+		return ret;
+
+	dmic_dev = platform_device_register_data(dev, "dmic-codec", PLATFORM_DEVID_NONE, NULL, 0);
+	if (IS_ERR(dmic_dev)) {
+		dev_err(dev, "failed to create DMIC device\n");
+		sof_pci_remove(pci);
+		return PTR_ERR(dmic_dev);
+	}
+
+	/* Register platform device only if flag set to FLAG_AMD_SOF_ONLY_DMIC */
+	if (flag != FLAG_AMD_SOF_ONLY_DMIC)
+		return 0;
+
+	addr = pci_resource_start(pci, 0);
+	res = devm_kzalloc(&pci->dev, sizeof(struct resource) * ARRAY_SIZE(rembrandt_res),
+			   GFP_KERNEL);
+	if (!res) {
+		platform_device_unregister(dmic_dev);
+		sof_pci_remove(pci);
+		return -ENOMEM;
+	}
+
+	res_i2s = rembrandt_res;
+	for (i = 0; i < ARRAY_SIZE(rembrandt_res); i++, res_i2s++) {
+		res[i].name = res_i2s->name;
+		res[i].flags = res_i2s->flags;
+		res[i].start = addr + res_i2s->start;
+		res[i].end = addr + res_i2s->end;
+		if (res_i2s->flags == IORESOURCE_IRQ) {
+			res[i].start = pci->irq;
+			res[i].end = res[i].start;
+		}
+	}
+
+	memset(&pdevinfo, 0, sizeof(pdevinfo));
+
+	/*
+	 * We have common PCI driver probe for ACP device but we have to support I2S without SOF
+	 * for some distributions. Register platform device that will be used to support non dsp
+	 * ACP's audio ends points on some machines.
+	 */
+	pdevinfo.name = "acp_asoc_rembrandt";
+	pdevinfo.id = 0;
+	pdevinfo.parent = &pci->dev;
+	pdevinfo.num_res = ARRAY_SIZE(rembrandt_res);
+	pdevinfo.res = &res[0];
+
+	pdev = platform_device_register_full(&pdevinfo);
+	if (IS_ERR(pdev)) {
+		dev_err(&pci->dev, "cannot register %s device\n", pdevinfo.name);
+		platform_device_unregister(dmic_dev);
+		sof_pci_remove(pci);
+		ret = PTR_ERR(pdev);
+	}
+
+	return ret;
+};
+
+static void acp_pci_rmb_remove(struct pci_dev *pci)
+{
+	if (dmic_dev)
+		platform_device_unregister(dmic_dev);
+	if (pdev)
+		platform_device_unregister(pdev);
+
+	sof_pci_remove(pci);
+}
+
+/* PCI IDs */
+static const struct pci_device_id rmb_pci_ids[] = {
+		{ PCI_DEVICE(PCI_VENDOR_ID_AMD, ACP_PCI_DEV_ID),
+		.driver_data = (unsigned long)&rembrandt_desc},
+		{ 0, }
+};
+MODULE_DEVICE_TABLE(pci, rmb_pci_ids);
+
+/* pci_driver definition */
+static struct pci_driver snd_sof_pci_amd_rmb_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = rmb_pci_ids,
+	.probe = acp_pci_rmb_probe,
+	.remove = acp_pci_rmb_remove,
+};
+module_pci_driver(snd_sof_pci_amd_rmb_driver);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_IMPORT_NS(SND_SOC_SOF_AMD_COMMON);
+MODULE_IMPORT_NS(SND_SOC_SOF_PCI_DEV);
diff --git a/sound/soc/sof/amd/pci-rn.c b/sound/soc/sof/amd/pci-rn.c
index 3a7fed25a226..fca40b261671 100644
--- a/sound/soc/sof/amd/pci-rn.c
+++ b/sound/soc/sof/amd/pci-rn.c
@@ -21,6 +21,7 @@
 #include "../sof-pci-dev.h"
 #include "../../amd/mach-config.h"
 #include "acp.h"
+#include "acp-dsp-offset.h"
 
 #define ACP3x_REG_START		0x1240000
 #define ACP3x_REG_END		0x125C000
@@ -44,7 +45,16 @@ static const struct resource renoir_res[] = {
 };
 
 static const struct sof_amd_acp_desc renoir_chip_info = {
+	.rev		= 3,
 	.host_bridge_id = HOST_BRIDGE_CZN,
+	.i2s_mode	= 0x04,
+	.pgfsm_base	= ACP3X_PGFSM_BASE,
+	.ext_intr_stat	= ACP3X_EXT_INTR_STAT,
+	.dsp_intr_base	= ACP3X_DSP_SW_INTR_BASE,
+	.sram_pte_offset = ACP3X_SRAM_PTE_OFFSET,
+	.i2s_pin_config_offset = ACP3X_I2S_PIN_CONFIG,
+	.hw_semaphore_offset = ACP3X_AXI2DAGB_SEM_0,
+	.acp_clkmux_sel	= ACP3X_CLKMUX_SEL,
 };
 
 static const struct sof_dev_desc renoir_desc = {
@@ -68,6 +78,7 @@ static const struct sof_dev_desc renoir_desc = {
 	},
 	.nocodec_tplg_filename	= "sof-acp.tplg",
 	.ops			= &sof_renoir_ops,
+	.ops_init		= sof_renoir_ops_init,
 };
 
 static int acp_pci_rn_probe(struct pci_dev *pci, const struct pci_device_id *pci_id)
diff --git a/sound/soc/sof/amd/rembrandt.c b/sound/soc/sof/amd/rembrandt.c
new file mode 100644
index 000000000000..dcb64a23e121
--- /dev/null
+++ b/sound/soc/sof/amd/rembrandt.c
@@ -0,0 +1,134 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license. When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2022 Advanced Micro Devices, Inc.
+//
+// Authors: Ajit Kumar Pandey <AjitKumar.Pandey@amd.com>
+
+/*
+ * Hardware interface for Audio DSP on Rembrandt platform
+ */
+
+#include <linux/platform_device.h>
+#include <linux/module.h>
+
+#include "../ops.h"
+#include "../sof-audio.h"
+#include "acp.h"
+#include "acp-dsp-offset.h"
+
+#define I2S_HS_INSTANCE		0
+#define I2S_BT_INSTANCE		1
+#define I2S_SP_INSTANCE		2
+#define PDM_DMIC_INSTANCE	3
+
+static struct snd_soc_dai_driver rembrandt_sof_dai[] = {
+	[I2S_HS_INSTANCE] = {
+		.id = I2S_HS_INSTANCE,
+		.name = "acp-sof-hs",
+		.playback = {
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			.channels_min = 2,
+			.channels_max = 8,
+			.rate_min = 8000,
+			.rate_max = 96000,
+		},
+		.capture = {
+			.rates = SNDRV_PCM_RATE_8000_48000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			/* Supporting only stereo for I2S HS controller capture */
+			.channels_min = 2,
+			.channels_max = 2,
+			.rate_min = 8000,
+			.rate_max = 48000,
+		},
+		.probe = &acp_dai_probe,
+	},
+
+	[I2S_BT_INSTANCE] = {
+		.id = I2S_BT_INSTANCE,
+		.name = "acp-sof-bt",
+		.playback = {
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			.channels_min = 2,
+			.channels_max = 8,
+			.rate_min = 8000,
+			.rate_max = 96000,
+		},
+		.capture = {
+			.rates = SNDRV_PCM_RATE_8000_48000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			/* Supporting only stereo for I2S BT controller capture */
+			.channels_min = 2,
+			.channels_max = 2,
+			.rate_min = 8000,
+			.rate_max = 48000,
+		},
+		.probe = &acp_dai_probe,
+	},
+
+	[I2S_SP_INSTANCE] = {
+		.id = I2S_SP_INSTANCE,
+		.name = "acp-sof-sp",
+		.playback = {
+			.rates = SNDRV_PCM_RATE_8000_96000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			.channels_min = 2,
+			.channels_max = 8,
+			.rate_min = 8000,
+			.rate_max = 96000,
+		},
+		.capture = {
+			.rates = SNDRV_PCM_RATE_8000_48000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S8 |
+				   SNDRV_PCM_FMTBIT_U8 | SNDRV_PCM_FMTBIT_S32_LE,
+			/* Supporting only stereo for I2S SP controller capture */
+			.channels_min = 2,
+			.channels_max = 2,
+			.rate_min = 8000,
+			.rate_max = 48000,
+		},
+		.probe = &acp_dai_probe,
+	},
+
+	[PDM_DMIC_INSTANCE] = {
+		.id = PDM_DMIC_INSTANCE,
+		.name = "acp-sof-dmic",
+		.capture = {
+			.rates = SNDRV_PCM_RATE_8000_48000,
+			.formats = SNDRV_PCM_FMTBIT_S32_LE,
+			.channels_min = 2,
+			.channels_max = 4,
+			.rate_min = 8000,
+			.rate_max = 48000,
+		},
+	},
+};
+
+/* Rembrandt ops */
+struct snd_sof_dsp_ops sof_rembrandt_ops;
+EXPORT_SYMBOL_NS(sof_rembrandt_ops, SND_SOC_SOF_AMD_COMMON);
+
+int sof_rembrandt_ops_init(struct snd_sof_dev *sdev)
+{
+	/* common defaults */
+	memcpy(&sof_rembrandt_ops, &sof_acp_common_ops, sizeof(struct snd_sof_dsp_ops));
+
+	sof_rembrandt_ops.drv = rembrandt_sof_dai;
+	sof_rembrandt_ops.num_drv = ARRAY_SIZE(rembrandt_sof_dai);
+
+	return 0;
+}
+
+MODULE_IMPORT_NS(SND_SOC_SOF_AMD_COMMON);
+MODULE_DESCRIPTION("REMBRANDT SOF Driver");
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/amd/renoir.c b/sound/soc/sof/amd/renoir.c
index 9261c8bc2236..6ea8727f977e 100644
--- a/sound/soc/sof/amd/renoir.c
+++ b/sound/soc/sof/amd/renoir.c
@@ -23,22 +23,6 @@
 #define I2S_SP_INSTANCE		1
 #define PDM_DMIC_INSTANCE	2
 
-#define I2S_MODE		0x04
-
-static int renoir_dai_probe(struct snd_soc_dai *dai)
-{
-	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(dai->component);
-	unsigned int val;
-
-	val = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP_I2S_PIN_CONFIG);
-	if (val != I2S_MODE) {
-		dev_err(sdev->dev, "I2S Mode is not supported (I2S_PIN_CONFIG: %#x)\n", val);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
 static struct snd_soc_dai_driver renoir_sof_dai[] = {
 	[I2S_BT_INSTANCE] = {
 		.id = I2S_BT_INSTANCE,
@@ -62,7 +46,7 @@ static struct snd_soc_dai_driver renoir_sof_dai[] = {
 			.rate_min = 8000,
 			.rate_max = 48000,
 		},
-		.probe = &renoir_dai_probe,
+		.probe = &acp_dai_probe,
 	},
 
 	[I2S_SP_INSTANCE] = {
@@ -87,7 +71,7 @@ static struct snd_soc_dai_driver renoir_sof_dai[] = {
 			.rate_min = 8000,
 			.rate_max = 48000,
 		},
-		.probe = &renoir_dai_probe,
+		.probe = &acp_dai_probe,
 	},
 
 	[PDM_DMIC_INSTANCE] = {
@@ -104,82 +88,21 @@ static struct snd_soc_dai_driver renoir_sof_dai[] = {
 	},
 };
 
-static struct snd_soc_acpi_mach *amd_sof_machine_select(struct snd_sof_dev *sdev)
-{
-	struct snd_sof_pdata *sof_pdata = sdev->pdata;
-	const struct sof_dev_desc *desc = sof_pdata->desc;
-	struct snd_soc_acpi_mach *mach;
+/* Renoir ops */
+struct snd_sof_dsp_ops sof_renoir_ops;
+EXPORT_SYMBOL_NS(sof_renoir_ops, SND_SOC_SOF_AMD_COMMON);
 
-	mach = snd_soc_acpi_find_machine(desc->machines);
-	if (!mach) {
-		dev_warn(sdev->dev, "No matching ASoC machine driver found\n");
-		return NULL;
-	}
+int sof_renoir_ops_init(struct snd_sof_dev *sdev)
+{
+	/* common defaults */
+	memcpy(&sof_renoir_ops, &sof_acp_common_ops, sizeof(struct snd_sof_dsp_ops));
 
-	sof_pdata->tplg_filename = mach->sof_tplg_filename;
-	sof_pdata->fw_filename = mach->fw_filename;
+	sof_renoir_ops.drv = renoir_sof_dai;
+	sof_renoir_ops.num_drv = ARRAY_SIZE(renoir_sof_dai);
 
-	return mach;
+	return 0;
 }
 
-/* AMD Renoir DSP ops */
-struct snd_sof_dsp_ops sof_renoir_ops = {
-	/* probe and remove */
-	.probe			= amd_sof_acp_probe,
-	.remove			= amd_sof_acp_remove,
-
-	/* Register IO */
-	.write			= sof_io_write,
-	.read			= sof_io_read,
-
-	/* Block IO */
-	.block_read		= acp_dsp_block_read,
-	.block_write		= acp_dsp_block_write,
-
-	/*Firmware loading */
-	.load_firmware		= snd_sof_load_firmware_memcpy,
-	.pre_fw_run		= acp_dsp_pre_fw_run,
-	.get_bar_index		= acp_get_bar_index,
-
-	/* DSP core boot */
-	.run			= acp_sof_dsp_run,
-
-	/*IPC */
-	.send_msg		= acp_sof_ipc_send_msg,
-	.ipc_msg_data		= acp_sof_ipc_msg_data,
-	.get_mailbox_offset	= acp_sof_ipc_get_mailbox_offset,
-	.irq_thread		= acp_sof_ipc_irq_thread,
-
-	/* DAI drivers */
-	.drv			= renoir_sof_dai,
-	.num_drv		= ARRAY_SIZE(renoir_sof_dai),
-
-	/* stream callbacks */
-	.pcm_open		= acp_pcm_open,
-	.pcm_close		= acp_pcm_close,
-	.pcm_hw_params		= acp_pcm_hw_params,
-
-	.hw_info		= SNDRV_PCM_INFO_MMAP |
-				  SNDRV_PCM_INFO_MMAP_VALID |
-				  SNDRV_PCM_INFO_INTERLEAVED |
-				  SNDRV_PCM_INFO_PAUSE |
-				  SNDRV_PCM_INFO_NO_PERIOD_WAKEUP,
-
-	/* Machine driver callbacks */
-	.machine_select		= amd_sof_machine_select,
-	.machine_register	= sof_machine_register,
-	.machine_unregister	= sof_machine_unregister,
-
-	/* Trace Logger */
-	.trace_init		= acp_sof_trace_init,
-	.trace_release		= acp_sof_trace_release,
-
-	/* PM */
-	.suspend                = amd_sof_acp_suspend,
-	.resume                 = amd_sof_acp_resume,
-};
-EXPORT_SYMBOL(sof_renoir_ops);
-
 MODULE_IMPORT_NS(SND_SOC_SOF_AMD_COMMON);
 MODULE_DESCRIPTION("RENOIR SOF Driver");
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/compress.c b/sound/soc/sof/compress.c
index 67139e15f862..8e1a9ba111ad 100644
--- a/sound/soc/sof/compress.c
+++ b/sound/soc/sof/compress.c
@@ -11,20 +11,20 @@
 #include "sof-priv.h"
 #include "sof-utils.h"
 
-static void sof_set_transferred_bytes(struct snd_compr_tstamp *tstamp,
+static void sof_set_transferred_bytes(struct sof_compr_stream *sstream,
 				      u64 host_pos, u64 buffer_size)
 {
 	u64 prev_pos;
 	unsigned int copied;
 
-	div64_u64_rem(tstamp->copied_total, buffer_size, &prev_pos);
+	div64_u64_rem(sstream->copied_total, buffer_size, &prev_pos);
 
 	if (host_pos < prev_pos)
 		copied = (buffer_size - prev_pos) + host_pos;
 	else
 		copied = host_pos - prev_pos;
 
-	tstamp->copied_total += copied;
+	sstream->copied_total += copied;
 }
 
 static void snd_sof_compr_fragment_elapsed_work(struct work_struct *work)
@@ -49,7 +49,7 @@ void snd_sof_compr_fragment_elapsed(struct snd_compr_stream *cstream)
 	struct snd_soc_pcm_runtime *rtd;
 	struct snd_compr_runtime *crtd;
 	struct snd_soc_component *component;
-	struct snd_compr_tstamp *tstamp;
+	struct sof_compr_stream *sstream;
 	struct snd_sof_pcm *spcm;
 
 	if (!cstream)
@@ -57,7 +57,7 @@ void snd_sof_compr_fragment_elapsed(struct snd_compr_stream *cstream)
 
 	rtd = cstream->private_data;
 	crtd = cstream->runtime;
-	tstamp = crtd->private_data;
+	sstream = crtd->private_data;
 	component = snd_soc_rtdcom_lookup(rtd, SOF_AUDIO_PCM_DRV_NAME);
 
 	spcm = snd_sof_find_spcm_dai(component, rtd);
@@ -67,7 +67,7 @@ void snd_sof_compr_fragment_elapsed(struct snd_compr_stream *cstream)
 		return;
 	}
 
-	sof_set_transferred_bytes(tstamp, spcm->stream[cstream->direction].posn.host_posn,
+	sof_set_transferred_bytes(sstream, spcm->stream[cstream->direction].posn.host_posn,
 				  crtd->buffer_size);
 
 	/* use the same workqueue-based solution as for PCM, cf. snd_sof_pcm_elapsed */
@@ -96,24 +96,24 @@ static int sof_compr_open(struct snd_soc_component *component,
 {
 	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
 	struct snd_compr_runtime *crtd = cstream->runtime;
-	struct snd_compr_tstamp *tstamp;
+	struct sof_compr_stream *sstream;
 	struct snd_sof_pcm *spcm;
 	int dir;
 
-	tstamp = kzalloc(sizeof(*tstamp), GFP_KERNEL);
-	if (!tstamp)
+	sstream = kzalloc(sizeof(*sstream), GFP_KERNEL);
+	if (!sstream)
 		return -ENOMEM;
 
 	spcm = snd_sof_find_spcm_dai(component, rtd);
 	if (!spcm) {
-		kfree(tstamp);
+		kfree(sstream);
 		return -EINVAL;
 	}
 
 	dir = cstream->direction;
 
 	if (spcm->stream[dir].cstream) {
-		kfree(tstamp);
+		kfree(sstream);
 		return -EBUSY;
 	}
 
@@ -122,7 +122,7 @@ static int sof_compr_open(struct snd_soc_component *component,
 	spcm->stream[dir].posn.dai_posn = 0;
 	spcm->prepared[dir] = false;
 
-	crtd->private_data = tstamp;
+	crtd->private_data = sstream;
 
 	return 0;
 }
@@ -131,7 +131,7 @@ static int sof_compr_free(struct snd_soc_component *component,
 			  struct snd_compr_stream *cstream)
 {
 	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component);
-	struct snd_compr_tstamp *tstamp = cstream->runtime->private_data;
+	struct sof_compr_stream *sstream = cstream->runtime->private_data;
 	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
 	struct sof_ipc_stream stream;
 	struct sof_ipc_reply reply;
@@ -155,7 +155,7 @@ static int sof_compr_free(struct snd_soc_component *component,
 
 	cancel_work_sync(&spcm->stream[cstream->direction].period_elapsed_work);
 	spcm->stream[cstream->direction].cstream = NULL;
-	kfree(tstamp);
+	kfree(sstream);
 
 	return ret;
 }
@@ -169,7 +169,7 @@ static int sof_compr_set_params(struct snd_soc_component *component,
 	struct sof_ipc_pcm_params_reply ipc_params_reply;
 	struct sof_ipc_fw_ready *ready = &sdev->fw_ready;
 	struct sof_ipc_fw_version *v = &ready->version;
-	struct snd_compr_tstamp *tstamp;
+	struct sof_compr_stream *sstream;
 	struct sof_ipc_pcm_params *pcm;
 	struct snd_sof_pcm *spcm;
 	size_t ext_data_size;
@@ -184,7 +184,7 @@ static int sof_compr_set_params(struct snd_soc_component *component,
 		return -EINVAL;
 	}
 
-	tstamp = crtd->private_data;
+	sstream = crtd->private_data;
 
 	spcm = snd_sof_find_spcm_dai(component, rtd);
 
@@ -237,8 +237,9 @@ static int sof_compr_set_params(struct snd_soc_component *component,
 		goto out;
 	}
 
-	tstamp->byte_offset = sdev->stream_box.offset + ipc_params_reply.posn_offset;
-	tstamp->sampling_rate = params->codec.sample_rate;
+	sstream->sampling_rate = params->codec.sample_rate;
+	sstream->channels = params->codec.ch_out;
+	sstream->sample_container_bytes = pcm->params.sample_container_bytes;
 
 	spcm->prepared[cstream->direction] = true;
 
@@ -296,18 +297,13 @@ static int sof_compr_trigger(struct snd_soc_component *component,
 				  &reply, sizeof(reply));
 }
 
-static int sof_compr_copy(struct snd_soc_component *component,
-			  struct snd_compr_stream *cstream,
-			  char __user *buf, size_t count)
+static int sof_compr_copy_playback(struct snd_compr_runtime *rtd,
+				   char __user *buf, size_t count)
 {
-	struct snd_compr_runtime *rtd = cstream->runtime;
-	unsigned int offset, n;
 	void *ptr;
+	unsigned int offset, n;
 	int ret;
 
-	if (count > rtd->buffer_size)
-		count = rtd->buffer_size;
-
 	div_u64_rem(rtd->total_bytes_available, rtd->buffer_size, &offset);
 	ptr = rtd->dma_area + offset;
 	n = rtd->buffer_size - offset;
@@ -322,14 +318,58 @@ static int sof_compr_copy(struct snd_soc_component *component,
 	return count - ret;
 }
 
+static int sof_compr_copy_capture(struct snd_compr_runtime *rtd,
+				  char __user *buf, size_t count)
+{
+	void *ptr;
+	unsigned int offset, n;
+	int ret;
+
+	div_u64_rem(rtd->total_bytes_transferred, rtd->buffer_size, &offset);
+	ptr = rtd->dma_area + offset;
+	n = rtd->buffer_size - offset;
+
+	if (count < n) {
+		ret = copy_to_user(buf, ptr, count);
+	} else {
+		ret = copy_to_user(buf, ptr, n);
+		ret += copy_to_user(buf + n, rtd->dma_area, count - n);
+	}
+
+	return count - ret;
+}
+
+static int sof_compr_copy(struct snd_soc_component *component,
+			  struct snd_compr_stream *cstream,
+			  char __user *buf, size_t count)
+{
+	struct snd_compr_runtime *rtd = cstream->runtime;
+
+	if (count > rtd->buffer_size)
+		count = rtd->buffer_size;
+
+	if (cstream->direction == SND_COMPRESS_PLAYBACK)
+		return sof_compr_copy_playback(rtd, buf, count);
+	else
+		return sof_compr_copy_capture(rtd, buf, count);
+}
+
 static int sof_compr_pointer(struct snd_soc_component *component,
 			     struct snd_compr_stream *cstream,
 			     struct snd_compr_tstamp *tstamp)
 {
-	struct snd_compr_tstamp *pstamp = cstream->runtime->private_data;
+	struct snd_sof_pcm *spcm;
+	struct snd_soc_pcm_runtime *rtd = cstream->private_data;
+	struct sof_compr_stream *sstream = cstream->runtime->private_data;
+
+	spcm = snd_sof_find_spcm_dai(component, rtd);
+	if (!spcm)
+		return -EINVAL;
 
-	tstamp->sampling_rate = pstamp->sampling_rate;
-	tstamp->copied_total = pstamp->copied_total;
+	tstamp->sampling_rate = sstream->sampling_rate;
+	tstamp->copied_total = sstream->copied_total;
+	tstamp->pcm_io_frames = div_u64(spcm->stream[cstream->direction].posn.dai_posn,
+					sstream->channels * sstream->sample_container_bytes);
 
 	return 0;
 }
diff --git a/sound/soc/sof/core.c b/sound/soc/sof/core.c
index c99b5e6c026c..3e6141d03770 100644
--- a/sound/soc/sof/core.c
+++ b/sound/soc/sof/core.c
@@ -15,6 +15,9 @@
 #include "sof-priv.h"
 #include "ops.h"
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/sof.h>
+
 /* see SOF_DBG_ flags */
 static int sof_core_debug =  IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_ENABLE_FIRMWARE_TRACE);
 module_param_named(sof_debug, sof_core_debug, int, 0444);
diff --git a/sound/soc/sof/imx/Kconfig b/sound/soc/sof/imx/Kconfig
index cc6e695f913a..4751b04d5e6f 100644
--- a/sound/soc/sof/imx/Kconfig
+++ b/sound/soc/sof/imx/Kconfig
@@ -41,4 +41,13 @@ config SND_SOC_SOF_IMX8M
 	  Say Y if you have such a device.
 	  If unsure select "N".
 
+config SND_SOC_SOF_IMX8ULP
+	tristate "SOF support for i.MX8ULP"
+	depends on IMX_DSP
+	select SND_SOC_SOF_IMX_COMMON
+	help
+	  This adds support for Sound Open Firmware for NXP i.MX8ULP platforms.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+
 endif ## SND_SOC_SOF_IMX_TOPLEVEL
diff --git a/sound/soc/sof/imx/Makefile b/sound/soc/sof/imx/Makefile
index dba93c3466ec..798b43a415bf 100644
--- a/sound/soc/sof/imx/Makefile
+++ b/sound/soc/sof/imx/Makefile
@@ -1,9 +1,11 @@
 # SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
 snd-sof-imx8-objs := imx8.o
 snd-sof-imx8m-objs := imx8m.o
+snd-sof-imx8ulp-objs := imx8ulp.o
 
 snd-sof-imx-common-objs := imx-common.o
 
 obj-$(CONFIG_SND_SOC_SOF_IMX8) += snd-sof-imx8.o
 obj-$(CONFIG_SND_SOC_SOF_IMX8M) += snd-sof-imx8m.o
+obj-$(CONFIG_SND_SOC_SOF_IMX8ULP) += snd-sof-imx8ulp.o
 obj-$(CONFIG_SND_SOC_SOF_IMX_COMMON) += imx-common.o
diff --git a/sound/soc/sof/imx/imx8ulp.c b/sound/soc/sof/imx/imx8ulp.c
new file mode 100644
index 000000000000..4a562c9856e9
--- /dev/null
+++ b/sound/soc/sof/imx/imx8ulp.c
@@ -0,0 +1,515 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+//
+// Copyright 2021-2022 NXP
+//
+// Author: Peng Zhang <peng.zhang_8@nxp.com>
+//
+// Hardware interface for audio DSP on i.MX8ULP
+
+#include <linux/arm-smccc.h>
+#include <linux/clk.h>
+#include <linux/firmware.h>
+#include <linux/firmware/imx/dsp.h>
+#include <linux/firmware/imx/ipc.h>
+#include <linux/firmware/imx/svc/misc.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+
+#include <sound/sof.h>
+#include <sound/sof/xtensa.h>
+
+#include "../ops.h"
+#include "../sof-of-dev.h"
+#include "imx-common.h"
+
+#define FSL_SIP_HIFI_XRDC	0xc200000e
+
+/* SIM Domain register */
+#define SYSCTRL0		0x8
+#define EXECUTE_BIT		BIT(13)
+#define RESET_BIT		BIT(16)
+#define HIFI4_CLK_BIT		BIT(17)
+#define PB_CLK_BIT		BIT(18)
+#define PLAT_CLK_BIT		BIT(19)
+#define DEBUG_LOGIC_BIT		BIT(25)
+
+#define MBOX_OFFSET		0x800000
+#define MBOX_SIZE		0x1000
+
+static struct clk_bulk_data imx8ulp_dsp_clks[] = {
+	{ .id = "core" },
+	{ .id = "ipg" },
+	{ .id = "ocram" },
+	{ .id = "mu" },
+};
+
+struct imx8ulp_priv {
+	struct device *dev;
+	struct snd_sof_dev *sdev;
+
+	/* DSP IPC handler */
+	struct imx_dsp_ipc *dsp_ipc;
+	struct platform_device *ipc_dev;
+
+	struct regmap *regmap;
+	struct imx_clocks *clks;
+};
+
+static void imx8ulp_sim_lpav_start(struct imx8ulp_priv *priv)
+{
+	/* Controls the HiFi4 DSP Reset: 1 in reset, 0 out of reset */
+	regmap_update_bits(priv->regmap, SYSCTRL0, RESET_BIT, 0);
+
+	/* Reset HiFi4 DSP Debug logic: 1 debug reset, 0  out of reset*/
+	regmap_update_bits(priv->regmap, SYSCTRL0, DEBUG_LOGIC_BIT, 0);
+
+	/* Stall HIFI4 DSP Execution: 1 stall, 0 run */
+	regmap_update_bits(priv->regmap, SYSCTRL0, EXECUTE_BIT, 0);
+}
+
+static int imx8ulp_get_mailbox_offset(struct snd_sof_dev *sdev)
+{
+	return MBOX_OFFSET;
+}
+
+static int imx8ulp_get_window_offset(struct snd_sof_dev *sdev, u32 id)
+{
+	return MBOX_OFFSET;
+}
+
+static void imx8ulp_dsp_handle_reply(struct imx_dsp_ipc *ipc)
+{
+	struct imx8ulp_priv *priv = imx_dsp_get_data(ipc);
+	unsigned long flags;
+
+	spin_lock_irqsave(&priv->sdev->ipc_lock, flags);
+
+	snd_sof_ipc_process_reply(priv->sdev, 0);
+
+	spin_unlock_irqrestore(&priv->sdev->ipc_lock, flags);
+}
+
+static void imx8ulp_dsp_handle_request(struct imx_dsp_ipc *ipc)
+{
+	struct imx8ulp_priv *priv = imx_dsp_get_data(ipc);
+	u32 p; /* panic code */
+
+	/* Read the message from the debug box. */
+	sof_mailbox_read(priv->sdev, priv->sdev->debug_box.offset + 4, &p, sizeof(p));
+
+	/* Check to see if the message is a panic code (0x0dead***) */
+	if ((p & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC)
+		snd_sof_dsp_panic(priv->sdev, p, true);
+	else
+		snd_sof_ipc_msgs_rx(priv->sdev);
+}
+
+static struct imx_dsp_ops dsp_ops = {
+	.handle_reply		= imx8ulp_dsp_handle_reply,
+	.handle_request		= imx8ulp_dsp_handle_request,
+};
+
+static int imx8ulp_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg)
+{
+	struct imx8ulp_priv *priv = sdev->pdata->hw_pdata;
+
+	sof_mailbox_write(sdev, sdev->host_box.offset, msg->msg_data,
+			  msg->msg_size);
+	imx_dsp_ring_doorbell(priv->dsp_ipc, 0);
+
+	return 0;
+}
+
+static int imx8ulp_run(struct snd_sof_dev *sdev)
+{
+	struct imx8ulp_priv *priv = sdev->pdata->hw_pdata;
+
+	imx8ulp_sim_lpav_start(priv);
+
+	return 0;
+}
+
+static int imx8ulp_reset(struct snd_sof_dev *sdev)
+{
+	struct imx8ulp_priv *priv = sdev->pdata->hw_pdata;
+	struct arm_smccc_res smc_resource;
+
+	/* HiFi4 Platform Clock Enable: 1 enabled, 0 disabled */
+	regmap_update_bits(priv->regmap, SYSCTRL0, PLAT_CLK_BIT, PLAT_CLK_BIT);
+
+	/* HiFi4 PBCLK clock enable: 1 enabled, 0 disabled */
+	regmap_update_bits(priv->regmap, SYSCTRL0, PB_CLK_BIT, PB_CLK_BIT);
+
+	/* HiFi4 Clock Enable: 1 enabled, 0 disabled */
+	regmap_update_bits(priv->regmap, SYSCTRL0, HIFI4_CLK_BIT, HIFI4_CLK_BIT);
+
+	regmap_update_bits(priv->regmap, SYSCTRL0, RESET_BIT, RESET_BIT);
+	usleep_range(1, 2);
+
+	/* Stall HIFI4 DSP Execution: 1 stall, 0 not stall */
+	regmap_update_bits(priv->regmap, SYSCTRL0, EXECUTE_BIT, EXECUTE_BIT);
+	usleep_range(1, 2);
+
+	arm_smccc_smc(FSL_SIP_HIFI_XRDC, 0, 0, 0, 0, 0, 0, 0, &smc_resource);
+
+	return 0;
+}
+
+static int imx8ulp_probe(struct snd_sof_dev *sdev)
+{
+	struct platform_device *pdev =
+		container_of(sdev->dev, struct platform_device, dev);
+	struct device_node *np = pdev->dev.of_node;
+	struct device_node *res_node;
+	struct resource *mmio;
+	struct imx8ulp_priv *priv;
+	struct resource res;
+	u32 base, size;
+	int ret = 0;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->clks = devm_kzalloc(&pdev->dev, sizeof(*priv->clks), GFP_KERNEL);
+	if (!priv->clks)
+		return -ENOMEM;
+
+	sdev->num_cores = 1;
+	sdev->pdata->hw_pdata = priv;
+	priv->dev = sdev->dev;
+	priv->sdev = sdev;
+
+	/* System integration module(SIM) control dsp configuration */
+	priv->regmap = syscon_regmap_lookup_by_phandle(np, "fsl,dsp-ctrl");
+	if (IS_ERR(priv->regmap))
+		return PTR_ERR(priv->regmap);
+
+	priv->ipc_dev = platform_device_register_data(sdev->dev, "imx-dsp",
+						      PLATFORM_DEVID_NONE,
+						      pdev, sizeof(*pdev));
+	if (IS_ERR(priv->ipc_dev))
+		return PTR_ERR(priv->ipc_dev);
+
+	priv->dsp_ipc = dev_get_drvdata(&priv->ipc_dev->dev);
+	if (!priv->dsp_ipc) {
+		/* DSP IPC driver not probed yet, try later */
+		ret = -EPROBE_DEFER;
+		dev_err(sdev->dev, "Failed to get drvdata\n");
+		goto exit_pdev_unregister;
+	}
+
+	imx_dsp_set_data(priv->dsp_ipc, priv);
+	priv->dsp_ipc->ops = &dsp_ops;
+
+	/* DSP base */
+	mmio = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (mmio) {
+		base = mmio->start;
+		size = resource_size(mmio);
+	} else {
+		dev_err(sdev->dev, "error: failed to get DSP base at idx 0\n");
+		ret = -EINVAL;
+		goto exit_pdev_unregister;
+	}
+
+	sdev->bar[SOF_FW_BLK_TYPE_IRAM] = devm_ioremap(sdev->dev, base, size);
+	if (!sdev->bar[SOF_FW_BLK_TYPE_IRAM]) {
+		dev_err(sdev->dev, "failed to ioremap base 0x%x size 0x%x\n",
+			base, size);
+		ret = -ENODEV;
+		goto exit_pdev_unregister;
+	}
+	sdev->mmio_bar = SOF_FW_BLK_TYPE_IRAM;
+
+	res_node = of_parse_phandle(np, "memory-reserved", 0);
+	if (!res_node) {
+		dev_err(&pdev->dev, "failed to get memory region node\n");
+		ret = -ENODEV;
+		goto exit_pdev_unregister;
+	}
+
+	ret = of_address_to_resource(res_node, 0, &res);
+	of_node_put(res_node);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to get reserved region address\n");
+		goto exit_pdev_unregister;
+	}
+
+	sdev->bar[SOF_FW_BLK_TYPE_SRAM] = devm_ioremap_wc(sdev->dev, res.start,
+							  resource_size(&res));
+	if (!sdev->bar[SOF_FW_BLK_TYPE_SRAM]) {
+		dev_err(sdev->dev, "failed to ioremap mem 0x%x size 0x%x\n",
+			base, size);
+		ret = -ENOMEM;
+		goto exit_pdev_unregister;
+	}
+	sdev->mailbox_bar = SOF_FW_BLK_TYPE_SRAM;
+
+	/* set default mailbox offset for FW ready message */
+	sdev->dsp_box.offset = MBOX_OFFSET;
+
+	ret = of_reserved_mem_device_init(sdev->dev);
+	if (ret) {
+		dev_err(&pdev->dev, "failed to init reserved memory region %d\n", ret);
+		goto exit_pdev_unregister;
+	}
+
+	priv->clks->dsp_clks = imx8ulp_dsp_clks;
+	priv->clks->num_dsp_clks = ARRAY_SIZE(imx8ulp_dsp_clks);
+
+	ret = imx8_parse_clocks(sdev, priv->clks);
+	if (ret < 0)
+		goto exit_pdev_unregister;
+
+	ret = imx8_enable_clocks(sdev, priv->clks);
+	if (ret < 0)
+		goto exit_pdev_unregister;
+
+	return 0;
+
+exit_pdev_unregister:
+	platform_device_unregister(priv->ipc_dev);
+
+	return ret;
+}
+
+static int imx8ulp_remove(struct snd_sof_dev *sdev)
+{
+	struct imx8ulp_priv *priv = sdev->pdata->hw_pdata;
+
+	imx8_disable_clocks(sdev, priv->clks);
+	platform_device_unregister(priv->ipc_dev);
+
+	return 0;
+}
+
+/* on i.MX8 there is 1 to 1 match between type and BAR idx */
+static int imx8ulp_get_bar_index(struct snd_sof_dev *sdev, u32 type)
+{
+	return type;
+}
+
+static int imx8ulp_suspend(struct snd_sof_dev *sdev)
+{
+	int i;
+	struct imx8ulp_priv *priv = (struct imx8ulp_priv *)sdev->pdata->hw_pdata;
+
+	/*Stall DSP,  release in .run() */
+	regmap_update_bits(priv->regmap, SYSCTRL0, EXECUTE_BIT, EXECUTE_BIT);
+
+	for (i = 0; i < DSP_MU_CHAN_NUM; i++)
+		imx_dsp_free_channel(priv->dsp_ipc, i);
+
+	imx8_disable_clocks(sdev, priv->clks);
+
+	return 0;
+}
+
+static int imx8ulp_resume(struct snd_sof_dev *sdev)
+{
+	struct imx8ulp_priv *priv = (struct imx8ulp_priv *)sdev->pdata->hw_pdata;
+	int i;
+
+	imx8_enable_clocks(sdev, priv->clks);
+
+	for (i = 0; i < DSP_MU_CHAN_NUM; i++)
+		imx_dsp_request_channel(priv->dsp_ipc, i);
+
+	return 0;
+}
+
+static int imx8ulp_dsp_runtime_resume(struct snd_sof_dev *sdev)
+{
+	const struct sof_dsp_power_state target_dsp_state = {
+		.state = SOF_DSP_PM_D0,
+		.substate = 0,
+	};
+
+	imx8ulp_resume(sdev);
+
+	return snd_sof_dsp_set_power_state(sdev, &target_dsp_state);
+}
+
+static int imx8ulp_dsp_runtime_suspend(struct snd_sof_dev *sdev)
+{
+	const struct sof_dsp_power_state target_dsp_state = {
+		.state = SOF_DSP_PM_D3,
+		.substate = 0,
+	};
+
+	imx8ulp_suspend(sdev);
+
+	return snd_sof_dsp_set_power_state(sdev, &target_dsp_state);
+}
+
+static int imx8ulp_dsp_suspend(struct snd_sof_dev *sdev, unsigned int target_state)
+{
+	const struct sof_dsp_power_state target_dsp_state = {
+		.state = target_state,
+		.substate = 0,
+	};
+
+	if (!pm_runtime_suspended(sdev->dev))
+		imx8ulp_suspend(sdev);
+
+	return snd_sof_dsp_set_power_state(sdev, &target_dsp_state);
+}
+
+static int imx8ulp_dsp_resume(struct snd_sof_dev *sdev)
+{
+	const struct sof_dsp_power_state target_dsp_state = {
+		.state = SOF_DSP_PM_D0,
+		.substate = 0,
+	};
+
+	imx8ulp_resume(sdev);
+
+	if (pm_runtime_suspended(sdev->dev)) {
+		pm_runtime_disable(sdev->dev);
+		pm_runtime_set_active(sdev->dev);
+		pm_runtime_mark_last_busy(sdev->dev);
+		pm_runtime_enable(sdev->dev);
+		pm_runtime_idle(sdev->dev);
+	}
+
+	return snd_sof_dsp_set_power_state(sdev, &target_dsp_state);
+}
+
+static struct snd_soc_dai_driver imx8ulp_dai[] = {
+	{
+		.name = "sai5",
+		.playback = {
+			.channels_min = 1,
+			.channels_max = 32,
+		},
+		.capture = {
+			.channels_min = 1,
+			.channels_max = 32,
+		},
+	},
+	{
+		.name = "sai6",
+		.playback = {
+			.channels_min = 1,
+			.channels_max = 32,
+		},
+		.capture = {
+			.channels_min = 1,
+			.channels_max = 32,
+		},
+	},
+};
+
+static int imx8ulp_dsp_set_power_state(struct snd_sof_dev *sdev,
+				       const struct sof_dsp_power_state *target_state)
+{
+	sdev->dsp_power_state = *target_state;
+
+	return 0;
+}
+
+/* i.MX8 ops */
+static struct snd_sof_dsp_ops sof_imx8ulp_ops = {
+	/* probe and remove */
+	.probe		= imx8ulp_probe,
+	.remove		= imx8ulp_remove,
+	/* DSP core boot */
+	.run		= imx8ulp_run,
+	.reset		= imx8ulp_reset,
+
+	/* Block IO */
+	.block_read	= sof_block_read,
+	.block_write	= sof_block_write,
+
+	/* Module IO */
+	.read64		= sof_io_read64,
+
+	/* Mailbox IO */
+	.mailbox_read	= sof_mailbox_read,
+	.mailbox_write	= sof_mailbox_write,
+
+	/* ipc */
+	.send_msg	= imx8ulp_send_msg,
+	.get_mailbox_offset	= imx8ulp_get_mailbox_offset,
+	.get_window_offset	= imx8ulp_get_window_offset,
+
+	.ipc_msg_data	= sof_ipc_msg_data,
+	.set_stream_data_offset = sof_set_stream_data_offset,
+
+	/* stream callbacks */
+	.pcm_open	= sof_stream_pcm_open,
+	.pcm_close	= sof_stream_pcm_close,
+
+	/* module loading */
+	.get_bar_index	= imx8ulp_get_bar_index,
+	/* firmware loading */
+	.load_firmware	= snd_sof_load_firmware_memcpy,
+
+	/* Debug information */
+	.dbg_dump	= imx8_dump,
+
+	/* Firmware ops */
+	.dsp_arch_ops	= &sof_xtensa_arch_ops,
+
+	/* DAI drivers */
+	.drv		= imx8ulp_dai,
+	.num_drv	= ARRAY_SIZE(imx8ulp_dai),
+
+	/* ALSA HW info flags */
+	.hw_info	= SNDRV_PCM_INFO_MMAP |
+			SNDRV_PCM_INFO_MMAP_VALID |
+			SNDRV_PCM_INFO_INTERLEAVED |
+			SNDRV_PCM_INFO_PAUSE |
+			SNDRV_PCM_INFO_NO_PERIOD_WAKEUP,
+
+	/* PM */
+	.runtime_suspend	= imx8ulp_dsp_runtime_suspend,
+	.runtime_resume		= imx8ulp_dsp_runtime_resume,
+
+	.suspend	= imx8ulp_dsp_suspend,
+	.resume		= imx8ulp_dsp_resume,
+
+	.set_power_state	= imx8ulp_dsp_set_power_state,
+};
+
+static struct sof_dev_desc sof_of_imx8ulp_desc = {
+	.ipc_supported_mask     = BIT(SOF_IPC),
+	.ipc_default            = SOF_IPC,
+	.default_fw_path = {
+		[SOF_IPC] = "imx/sof",
+	},
+	.default_tplg_path = {
+		[SOF_IPC] = "imx/sof-tplg",
+	},
+	.default_fw_filename = {
+		[SOF_IPC] = "sof-imx8ulp.ri",
+	},
+	.nocodec_tplg_filename = "sof-imx8ulp-nocodec.tplg",
+	.ops = &sof_imx8ulp_ops,
+};
+
+static const struct of_device_id sof_of_imx8ulp_ids[] = {
+	{ .compatible = "fsl,imx8ulp-dsp", .data = &sof_of_imx8ulp_desc},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, sof_of_imx8ulp_ids);
+
+/* DT driver definition */
+static struct platform_driver snd_sof_of_imx8ulp_driver = {
+	.probe = sof_of_probe,
+	.remove = sof_of_remove,
+	.driver = {
+		.name = "sof-audio-of-imx8ulp",
+		.pm = &sof_of_pm,
+		.of_match_table = sof_of_imx8ulp_ids,
+	},
+};
+module_platform_driver(snd_sof_of_imx8ulp_driver);
+
+MODULE_IMPORT_NS(SND_SOC_SOF_XTENSA);
+MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/intel/Kconfig b/sound/soc/sof/intel/Kconfig
index 3f54678e810b..7af495fb6125 100644
--- a/sound/soc/sof/intel/Kconfig
+++ b/sound/soc/sof/intel/Kconfig
@@ -95,6 +95,31 @@ config SND_SOC_SOF_MERRIFIELD
 	  Say Y if you have such a device.
 	  If unsure select "N".
 
+config SND_SOC_SOF_INTEL_SKL
+	tristate
+	select SND_SOC_SOF_HDA_COMMON
+	select SND_SOC_SOF_INTEL_IPC4
+
+config SND_SOC_SOF_SKYLAKE
+	tristate "SOF support for SkyLake"
+	default SND_SOC_SOF_PCI
+	select SND_SOC_SOF_INTEL_SKL
+	help
+	  This adds support for the Intel(R) platforms using the SkyLake processors.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+	  This is intended only for developers and not a recommend option for distros.
+
+config SND_SOC_SOF_KABYLAKE
+	tristate "SOF support for KabyLake"
+	default SND_SOC_SOF_PCI
+	select SND_SOC_SOF_INTEL_SKL
+	help
+	  This adds support for the Intel(R) platforms using the KabyLake processors.
+	  Say Y if you have such a device.
+	  If unsure select "N".
+	  This is intended only for developers and not a recommend option for distros.
+
 config SND_SOC_SOF_INTEL_APL
 	tristate
 	select SND_SOC_SOF_HDA_COMMON
diff --git a/sound/soc/sof/intel/Makefile b/sound/soc/sof/intel/Makefile
index a079159bb2f0..8b8ea0361785 100644
--- a/sound/soc/sof/intel/Makefile
+++ b/sound/soc/sof/intel/Makefile
@@ -6,7 +6,9 @@ snd-sof-acpi-intel-bdw-objs := bdw.o
 snd-sof-intel-hda-common-objs := hda.o hda-loader.o hda-stream.o hda-trace.o \
 				 hda-dsp.o hda-ipc.o hda-ctrl.o hda-pcm.o \
 				 hda-dai.o hda-bus.o \
+				 skl.o hda-loader-skl.o \
 				 apl.o cnl.o tgl.o icl.o mtl.o hda-common-ops.o
+
 snd-sof-intel-hda-common-$(CONFIG_SND_SOC_SOF_HDA_PROBES) += hda-probes.o
 
 snd-sof-intel-hda-objs := hda-codec.o
@@ -20,6 +22,7 @@ obj-$(CONFIG_SND_SOC_SOF_HDA_COMMON) += snd-sof-intel-hda-common.o
 obj-$(CONFIG_SND_SOC_SOF_HDA) += snd-sof-intel-hda.o
 
 snd-sof-pci-intel-tng-objs := pci-tng.o
+snd-sof-pci-intel-skl-objs := pci-skl.o
 snd-sof-pci-intel-apl-objs := pci-apl.o
 snd-sof-pci-intel-cnl-objs := pci-cnl.o
 snd-sof-pci-intel-icl-objs := pci-icl.o
@@ -27,6 +30,7 @@ snd-sof-pci-intel-tgl-objs := pci-tgl.o
 snd-sof-pci-intel-mtl-objs := pci-mtl.o
 
 obj-$(CONFIG_SND_SOC_SOF_MERRIFIELD) += snd-sof-pci-intel-tng.o
+obj-$(CONFIG_SND_SOC_SOF_INTEL_SKL) += snd-sof-pci-intel-skl.o
 obj-$(CONFIG_SND_SOC_SOF_INTEL_APL) += snd-sof-pci-intel-apl.o
 obj-$(CONFIG_SND_SOC_SOF_INTEL_CNL) += snd-sof-pci-intel-cnl.o
 obj-$(CONFIG_SND_SOC_SOF_INTEL_ICL) += snd-sof-pci-intel-icl.o
diff --git a/sound/soc/sof/intel/apl.c b/sound/soc/sof/intel/apl.c
index 084c245a9522..1549ca7587a4 100644
--- a/sound/soc/sof/intel/apl.c
+++ b/sound/soc/sof/intel/apl.c
@@ -45,6 +45,9 @@ int sof_apl_ops_init(struct snd_sof_dev *sdev)
 
 		/* ipc */
 		sof_apl_ops.send_msg	= hda_dsp_ipc_send_msg;
+
+		/* debug */
+		sof_apl_ops.ipc_dump	= hda_ipc_dump;
 	}
 
 	if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) {
@@ -57,11 +60,16 @@ int sof_apl_ops_init(struct snd_sof_dev *sdev)
 		ipc4_data = sdev->private;
 		ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET;
 
+		ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_1_5;
+
 		/* doorbell */
 		sof_apl_ops.irq_thread	= hda_dsp_ipc4_irq_thread;
 
 		/* ipc */
 		sof_apl_ops.send_msg	= hda_dsp_ipc4_send_msg;
+
+		/* debug */
+		sof_apl_ops.ipc_dump	= hda_ipc4_dump;
 	}
 
 	/* set DAI driver ops */
@@ -70,7 +78,6 @@ int sof_apl_ops_init(struct snd_sof_dev *sdev)
 	/* debug */
 	sof_apl_ops.debug_map	= apl_dsp_debugfs;
 	sof_apl_ops.debug_map_count	= ARRAY_SIZE(apl_dsp_debugfs);
-	sof_apl_ops.ipc_dump	= hda_ipc_dump;
 
 	/* firmware run */
 	sof_apl_ops.run = hda_dsp_cl_boot_firmware;
@@ -102,6 +109,8 @@ const struct sof_intel_dsp_desc apl_chip_info = {
 	.quirks = SOF_INTEL_PROCEN_FMT_QUIRK,
 	.check_ipc_irq	= hda_dsp_check_ipc_irq,
 	.cl_init = cl_dsp_init,
+	.power_down_dsp = hda_power_down_dsp,
+	.disable_interrupts = hda_dsp_disable_interrupts,
 	.hw_ip_version = SOF_INTEL_CAVS_1_5_PLUS,
 };
 EXPORT_SYMBOL_NS(apl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON);
diff --git a/sound/soc/sof/intel/cnl.c b/sound/soc/sof/intel/cnl.c
index a064453f0bc3..19d0b1909bfd 100644
--- a/sound/soc/sof/intel/cnl.c
+++ b/sound/soc/sof/intel/cnl.c
@@ -17,6 +17,7 @@
 
 #include <sound/sof/ext_manifest4.h>
 #include <sound/sof/ipc4/header.h>
+#include <trace/events/sof_intel.h>
 #include "../ipc4-priv.h"
 #include "../ops.h"
 #include "hda.h"
@@ -121,9 +122,7 @@ irqreturn_t cnl_ipc_irq_thread(int irq, void *context)
 		msg_ext = hipci & CNL_DSP_REG_HIPCIDR_MSG_MASK;
 		msg = hipcida & CNL_DSP_REG_HIPCIDA_MSG_MASK;
 
-		dev_vdbg(sdev->dev,
-			 "ipc: firmware response, msg:0x%x, msg_ext:0x%x\n",
-			 msg, msg_ext);
+		trace_sof_intel_ipc_firmware_response(sdev, msg, msg_ext);
 
 		/* mask Done interrupt */
 		snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
@@ -153,9 +152,7 @@ irqreturn_t cnl_ipc_irq_thread(int irq, void *context)
 		msg = hipctdr & CNL_DSP_REG_HIPCTDR_MSG_MASK;
 		msg_ext = hipctdd & CNL_DSP_REG_HIPCTDD_MSG_MASK;
 
-		dev_vdbg(sdev->dev,
-			 "ipc: firmware initiated, msg:0x%x, msg_ext:0x%x\n",
-			 msg, msg_ext);
+		trace_sof_intel_ipc_firmware_initiated(sdev, msg, msg_ext);
 
 		/* handle messages from DSP */
 		if ((hipctdr & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) {
@@ -335,6 +332,27 @@ void cnl_ipc_dump(struct snd_sof_dev *sdev)
 		hipcida, hipctdr, hipcctl);
 }
 
+void cnl_ipc4_dump(struct snd_sof_dev *sdev)
+{
+	u32 hipcidr, hipcidd, hipcida, hipctdr, hipctdd, hipctda, hipcctl;
+
+	hda_ipc_irq_dump(sdev);
+
+	hipcidr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDR);
+	hipcidd = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDD);
+	hipcida = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDA);
+	hipctdr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCTDR);
+	hipctdd = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCTDD);
+	hipctda = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCTDA);
+	hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCCTL);
+
+	/* dump the IPC regs */
+	/* TODO: parse the raw msg */
+	dev_err(sdev->dev,
+		"Host IPC initiator: %#x|%#x|%#x, target: %#x|%#x|%#x, ctl: %#x\n",
+		hipcidr, hipcidd, hipcida, hipctdr, hipctdd, hipctda, hipcctl);
+}
+
 /* cannonlake ops */
 struct snd_sof_dsp_ops sof_cnl_ops;
 EXPORT_SYMBOL_NS(sof_cnl_ops, SND_SOC_SOF_INTEL_HDA_COMMON);
@@ -354,6 +372,9 @@ int sof_cnl_ops_init(struct snd_sof_dev *sdev)
 
 		/* ipc */
 		sof_cnl_ops.send_msg	= cnl_ipc_send_msg;
+
+		/* debug */
+		sof_cnl_ops.ipc_dump	= cnl_ipc_dump;
 	}
 
 	if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) {
@@ -366,11 +387,16 @@ int sof_cnl_ops_init(struct snd_sof_dev *sdev)
 		ipc4_data = sdev->private;
 		ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET;
 
+		ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_1_8;
+
 		/* doorbell */
 		sof_cnl_ops.irq_thread	= cnl_ipc4_irq_thread;
 
 		/* ipc */
 		sof_cnl_ops.send_msg	= cnl_ipc4_send_msg;
+
+		/* debug */
+		sof_cnl_ops.ipc_dump	= cnl_ipc4_dump;
 	}
 
 	/* set DAI driver ops */
@@ -379,7 +405,6 @@ int sof_cnl_ops_init(struct snd_sof_dev *sdev)
 	/* debug */
 	sof_cnl_ops.debug_map	= cnl_dsp_debugfs;
 	sof_cnl_ops.debug_map_count	= ARRAY_SIZE(cnl_dsp_debugfs);
-	sof_cnl_ops.ipc_dump	= cnl_ipc_dump;
 
 	/* pre/post fw run */
 	sof_cnl_ops.post_fw_run = hda_dsp_post_fw_run;
@@ -413,6 +438,8 @@ const struct sof_intel_dsp_desc cnl_chip_info = {
 	.check_sdw_irq	= hda_common_check_sdw_irq,
 	.check_ipc_irq	= hda_dsp_check_ipc_irq,
 	.cl_init = cl_dsp_init,
+	.power_down_dsp = hda_power_down_dsp,
+	.disable_interrupts = hda_dsp_disable_interrupts,
 	.hw_ip_version = SOF_INTEL_CAVS_1_8,
 };
 EXPORT_SYMBOL_NS(cnl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON);
@@ -443,6 +470,8 @@ const struct sof_intel_dsp_desc jsl_chip_info = {
 	.check_sdw_irq	= hda_common_check_sdw_irq,
 	.check_ipc_irq	= hda_dsp_check_ipc_irq,
 	.cl_init = cl_dsp_init,
+	.power_down_dsp = hda_power_down_dsp,
+	.disable_interrupts = hda_dsp_disable_interrupts,
 	.hw_ip_version = SOF_INTEL_CAVS_2_0,
 };
 EXPORT_SYMBOL_NS(jsl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON);
diff --git a/sound/soc/sof/intel/hda-codec.c b/sound/soc/sof/intel/hda-codec.c
index 2f3f4a733d9e..1e9afc48394c 100644
--- a/sound/soc/sof/intel/hda-codec.c
+++ b/sound/soc/sof/intel/hda-codec.c
@@ -109,17 +109,45 @@ EXPORT_SYMBOL_NS(hda_codec_jack_check, SND_SOC_SOF_HDA_AUDIO_CODEC);
 #define is_generic_config(x)	0
 #endif
 
+static void hda_codec_device_exit(struct device *dev)
+{
+	snd_hdac_device_exit(dev_to_hdac_dev(dev));
+}
+
+static struct hda_codec *hda_codec_device_init(struct hdac_bus *bus, int addr, int type)
+{
+	struct hda_codec *codec;
+	int ret;
+
+	codec = snd_hda_codec_device_init(to_hda_bus(bus), addr, "ehdaudio%dD%d", bus->idx, addr);
+	if (IS_ERR(codec)) {
+		dev_err(bus->dev, "device init failed for hdac device\n");
+		return codec;
+	}
+
+	codec->core.type = type;
+	codec->core.dev.release = hda_codec_device_exit;
+
+	ret = snd_hdac_device_register(&codec->core);
+	if (ret) {
+		dev_err(bus->dev, "failed to register hdac device\n");
+		snd_hdac_device_exit(&codec->core);
+		return ERR_PTR(ret);
+	}
+
+	return codec;
+}
+
 /* probe individual codec */
 static int hda_codec_probe(struct snd_sof_dev *sdev, int address,
 			   bool hda_codec_use_common_hdmi)
 {
 #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_AUDIO_CODEC)
 	struct hdac_hda_priv *hda_priv;
-	struct hda_codec *codec;
 	int type = HDA_DEV_LEGACY;
 #endif
 	struct hda_bus *hbus = sof_to_hbus(sdev);
-	struct hdac_device *hdev;
+	struct hda_codec *codec;
 	u32 hda_cmd = (address << 28) | (AC_NODE_ROOT << 20) |
 		(AC_VERB_PARAMETERS << 8) | AC_PAR_VENDOR_ID;
 	u32 resp = -1;
@@ -142,20 +170,20 @@ static int hda_codec_probe(struct snd_sof_dev *sdev, int address,
 	if (!hda_priv)
 		return -ENOMEM;
 
-	hda_priv->codec.bus = hbus;
-	hdev = &hda_priv->codec.core;
-	codec = &hda_priv->codec;
-
 	/* only probe ASoC codec drivers for HDAC-HDMI */
 	if (!hda_codec_use_common_hdmi && (resp & 0xFFFF0000) == IDISP_VID_INTEL)
 		type = HDA_DEV_ASOC;
 
-	ret = snd_hdac_ext_bus_device_init(&hbus->core, address, hdev, type);
+	codec = hda_codec_device_init(&hbus->core, address, type);
+	ret = PTR_ERR_OR_ZERO(codec);
 	if (ret < 0)
 		return ret;
 
+	hda_priv->codec = codec;
+	dev_set_drvdata(&codec->core.dev, hda_priv);
+
 	if ((resp & 0xFFFF0000) == IDISP_VID_INTEL) {
-		if (!hdev->bus->audio_component) {
+		if (!hbus->core.audio_component) {
 			dev_dbg(sdev->dev,
 				"iDisp hw present but no driver\n");
 			ret = -ENOENT;
@@ -181,15 +209,12 @@ static int hda_codec_probe(struct snd_sof_dev *sdev, int address,
 
 out:
 	if (ret < 0) {
-		snd_hdac_device_unregister(hdev);
-		put_device(&hdev->dev);
+		snd_hdac_device_unregister(&codec->core);
+		put_device(&codec->core.dev);
 	}
 #else
-	hdev = devm_kzalloc(sdev->dev, sizeof(*hdev), GFP_KERNEL);
-	if (!hdev)
-		return -ENOMEM;
-
-	ret = snd_hdac_ext_bus_device_init(&hbus->core, address, hdev, HDA_DEV_ASOC);
+	codec = hda_codec_device_init(&hbus->core, address, HDA_DEV_ASOC);
+	ret = PTR_ERR_OR_ZERO(codec);
 #endif
 
 	return ret;
diff --git a/sound/soc/sof/intel/hda-dsp.c b/sound/soc/sof/intel/hda-dsp.c
index eddfd77ad90f..3c76f843454b 100644
--- a/sound/soc/sof/intel/hda-dsp.c
+++ b/sound/soc/sof/intel/hda-dsp.c
@@ -18,6 +18,7 @@
 #include <linux/module.h>
 #include <sound/hdaudio_ext.h>
 #include <sound/hda_register.h>
+#include <trace/events/sof_intel.h>
 #include "../sof-audio.h"
 #include "../ops.h"
 #include "hda.h"
@@ -113,7 +114,7 @@ static int hda_dsp_core_reset_leave(struct snd_sof_dev *sdev, unsigned int core_
 	return ret;
 }
 
-static int hda_dsp_core_stall_reset(struct snd_sof_dev *sdev, unsigned int core_mask)
+int hda_dsp_core_stall_reset(struct snd_sof_dev *sdev, unsigned int core_mask)
 {
 	/* stall core */
 	snd_sof_dsp_update_bits_unlocked(sdev, HDA_DSP_BAR,
@@ -125,7 +126,7 @@ static int hda_dsp_core_stall_reset(struct snd_sof_dev *sdev, unsigned int core_
 	return hda_dsp_core_reset_enter(sdev, core_mask);
 }
 
-static bool hda_dsp_core_is_enabled(struct snd_sof_dev *sdev, unsigned int core_mask)
+bool hda_dsp_core_is_enabled(struct snd_sof_dev *sdev, unsigned int core_mask)
 {
 	int val;
 	bool is_enable;
@@ -397,8 +398,7 @@ static int hda_dsp_update_d0i3c_register(struct snd_sof_dev *sdev, u8 value)
 		return ret;
 	}
 
-	dev_vdbg(bus->dev, "D0I3C updated, register = 0x%x\n",
-		 snd_hdac_chip_readb(bus, VS_D0I3C));
+	trace_sof_intel_D0I3C_updated(sdev, snd_hdac_chip_readb(bus, VS_D0I3C));
 
 	return 0;
 }
@@ -620,14 +620,18 @@ static int hda_suspend(struct snd_sof_dev *sdev, bool runtime_suspend)
 	/*
 	 * The memory used for IMR boot loses its content in deeper than S3 state
 	 * We must not try IMR boot on next power up (as it will fail).
+	 *
+	 * In case of firmware crash or boot failure set the skip_imr_boot to true
+	 * as well in order to try to re-load the firmware to do a 'cold' boot.
 	 */
-	if (sdev->system_suspend_target > SOF_SUSPEND_S3)
+	if (sdev->system_suspend_target > SOF_SUSPEND_S3 ||
+	    sdev->fw_state == SOF_FW_CRASHED ||
+	    sdev->fw_state == SOF_FW_BOOT_FAILED)
 		hda->skip_imr_boot = true;
 
-	hda_sdw_int_enable(sdev, false);
-
-	/* disable IPC interrupts */
-	hda_dsp_ipc_int_disable(sdev);
+	ret = chip->disable_interrupts(sdev);
+	if (ret < 0)
+		return ret;
 
 #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
 	hda_codec_jack_wake_enable(sdev, runtime_suspend);
@@ -636,11 +640,9 @@ static int hda_suspend(struct snd_sof_dev *sdev, bool runtime_suspend)
 	snd_hdac_ext_bus_link_power_down_all(bus);
 #endif
 
-	/* power down DSP */
-	ret = hda_dsp_core_reset_power_down(sdev, chip->host_managed_cores_mask);
+	ret = chip->power_down_dsp(sdev);
 	if (ret < 0) {
-		dev_err(sdev->dev,
-			"error: failed to power down core during suspend\n");
+		dev_err(sdev->dev, "failed to power down DSP during suspend\n");
 		return ret;
 	}
 
@@ -984,3 +986,11 @@ power_down:
 
 	return ret;
 }
+
+int hda_dsp_disable_interrupts(struct snd_sof_dev *sdev)
+{
+	hda_sdw_int_enable(sdev, false);
+	hda_dsp_ipc_int_disable(sdev);
+
+	return 0;
+}
diff --git a/sound/soc/sof/intel/hda-ipc.c b/sound/soc/sof/intel/hda-ipc.c
index 65e688f749ea..9b3667c705e4 100644
--- a/sound/soc/sof/intel/hda-ipc.c
+++ b/sound/soc/sof/intel/hda-ipc.c
@@ -16,6 +16,7 @@
  */
 
 #include <sound/sof/ipc4/header.h>
+#include <trace/events/sof_intel.h>
 #include "../ops.h"
 #include "hda.h"
 
@@ -212,9 +213,7 @@ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context)
 		msg = hipci & HDA_DSP_REG_HIPCI_MSG_MASK;
 		msg_ext = hipcie & HDA_DSP_REG_HIPCIE_MSG_MASK;
 
-		dev_vdbg(sdev->dev,
-			 "ipc: firmware response, msg:0x%x, msg_ext:0x%x\n",
-			 msg, msg_ext);
+		trace_sof_intel_ipc_firmware_response(sdev, msg, msg_ext);
 
 		/* mask Done interrupt */
 		snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
@@ -255,9 +254,7 @@ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context)
 		msg = hipct & HDA_DSP_REG_HIPCT_MSG_MASK;
 		msg_ext = hipcte & HDA_DSP_REG_HIPCTE_MSG_MASK;
 
-		dev_vdbg(sdev->dev,
-			 "ipc: firmware initiated, msg:0x%x, msg_ext:0x%x\n",
-			 msg, msg_ext);
+		trace_sof_intel_ipc_firmware_initiated(sdev, msg, msg_ext);
 
 		/* mask BUSY interrupt */
 		snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
@@ -307,12 +304,13 @@ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context)
 /* Check if an IPC IRQ occurred */
 bool hda_dsp_check_ipc_irq(struct snd_sof_dev *sdev)
 {
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
 	bool ret = false;
 	u32 irq_status;
 
 	/* store status */
 	irq_status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIS);
-	dev_vdbg(sdev->dev, "irq handler: irq_status:0x%x\n", irq_status);
+	trace_sof_intel_hda_irq_ipc_check(sdev, irq_status);
 
 	/* invalid message ? */
 	if (irq_status == 0xffffffff)
@@ -322,6 +320,13 @@ bool hda_dsp_check_ipc_irq(struct snd_sof_dev *sdev)
 	if (irq_status & HDA_DSP_ADSPIS_IPC)
 		ret = true;
 
+	/* CLDMA message ? */
+	if (irq_status & HDA_DSP_ADSPIS_CL_DMA) {
+		hda->code_loading = 0;
+		wake_up(&hda->waitq);
+		ret = false;
+	}
+
 out:
 	return ret;
 }
diff --git a/sound/soc/sof/intel/hda-ipc.h b/sound/soc/sof/intel/hda-ipc.h
index 10fbca5939db..8ec5e9f6f8d7 100644
--- a/sound/soc/sof/intel/hda-ipc.h
+++ b/sound/soc/sof/intel/hda-ipc.h
@@ -51,5 +51,6 @@
 irqreturn_t cnl_ipc_irq_thread(int irq, void *context);
 int cnl_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg);
 void cnl_ipc_dump(struct snd_sof_dev *sdev);
+void cnl_ipc4_dump(struct snd_sof_dev *sdev);
 
 #endif
diff --git a/sound/soc/sof/intel/hda-loader-skl.c b/sound/soc/sof/intel/hda-loader-skl.c
new file mode 100644
index 000000000000..0193fb3964a0
--- /dev/null
+++ b/sound/soc/sof/intel/hda-loader-skl.c
@@ -0,0 +1,580 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018-2022 Intel Corporation. All rights reserved.
+//
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/firmware.h>
+#include <linux/fs.h>
+#include <linux/interrupt.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <sound/hdaudio_ext.h>
+#include <sound/sof.h>
+#include <sound/pcm_params.h>
+
+#include "../sof-priv.h"
+#include "../ops.h"
+#include "hda.h"
+
+#define HDA_SKL_WAIT_TIMEOUT		500	/* 500 msec */
+#define HDA_SKL_CLDMA_MAX_BUFFER_SIZE	(32 * PAGE_SIZE)
+
+/* Stream Reset */
+#define HDA_CL_SD_CTL_SRST_SHIFT	0
+#define HDA_CL_SD_CTL_SRST(x)		(((x) & 0x1) << \
+					HDA_CL_SD_CTL_SRST_SHIFT)
+
+/* Stream Run */
+#define HDA_CL_SD_CTL_RUN_SHIFT		1
+#define HDA_CL_SD_CTL_RUN(x)		(((x) & 0x1) << \
+					HDA_CL_SD_CTL_RUN_SHIFT)
+
+/* Interrupt On Completion Enable */
+#define HDA_CL_SD_CTL_IOCE_SHIFT	2
+#define HDA_CL_SD_CTL_IOCE(x)		(((x) & 0x1) << \
+					HDA_CL_SD_CTL_IOCE_SHIFT)
+
+/* FIFO Error Interrupt Enable */
+#define HDA_CL_SD_CTL_FEIE_SHIFT	3
+#define HDA_CL_SD_CTL_FEIE(x)		(((x) & 0x1) << \
+					HDA_CL_SD_CTL_FEIE_SHIFT)
+
+/* Descriptor Error Interrupt Enable */
+#define HDA_CL_SD_CTL_DEIE_SHIFT	4
+#define HDA_CL_SD_CTL_DEIE(x)		(((x) & 0x1) << \
+					HDA_CL_SD_CTL_DEIE_SHIFT)
+
+/* FIFO Limit Change */
+#define HDA_CL_SD_CTL_FIFOLC_SHIFT	5
+#define HDA_CL_SD_CTL_FIFOLC(x)		(((x) & 0x1) << \
+					HDA_CL_SD_CTL_FIFOLC_SHIFT)
+
+/* Stripe Control */
+#define HDA_CL_SD_CTL_STRIPE_SHIFT	16
+#define HDA_CL_SD_CTL_STRIPE(x)		(((x) & 0x3) << \
+					HDA_CL_SD_CTL_STRIPE_SHIFT)
+
+/* Traffic Priority */
+#define HDA_CL_SD_CTL_TP_SHIFT		18
+#define HDA_CL_SD_CTL_TP(x)		(((x) & 0x1) << \
+					HDA_CL_SD_CTL_TP_SHIFT)
+
+/* Bidirectional Direction Control */
+#define HDA_CL_SD_CTL_DIR_SHIFT		19
+#define HDA_CL_SD_CTL_DIR(x)		(((x) & 0x1) << \
+					HDA_CL_SD_CTL_DIR_SHIFT)
+
+/* Stream Number */
+#define HDA_CL_SD_CTL_STRM_SHIFT	20
+#define HDA_CL_SD_CTL_STRM(x)		(((x) & 0xf) << \
+					HDA_CL_SD_CTL_STRM_SHIFT)
+
+#define HDA_CL_SD_CTL_INT(x)	\
+		(HDA_CL_SD_CTL_IOCE(x) | \
+		HDA_CL_SD_CTL_FEIE(x) | \
+		HDA_CL_SD_CTL_DEIE(x))
+
+#define HDA_CL_SD_CTL_INT_MASK	\
+		(HDA_CL_SD_CTL_IOCE(1) | \
+		HDA_CL_SD_CTL_FEIE(1) | \
+		HDA_CL_SD_CTL_DEIE(1))
+
+#define DMA_ADDRESS_128_BITS_ALIGNMENT	7
+#define BDL_ALIGN(x)			((x) >> DMA_ADDRESS_128_BITS_ALIGNMENT)
+
+/* Buffer Descriptor List Lower Base Address */
+#define HDA_CL_SD_BDLPLBA_SHIFT		7
+#define HDA_CL_SD_BDLPLBA_MASK		GENMASK(31, 7)
+#define HDA_CL_SD_BDLPLBA(x)		\
+	((BDL_ALIGN(lower_32_bits(x)) << HDA_CL_SD_BDLPLBA_SHIFT) & \
+	 HDA_CL_SD_BDLPLBA_MASK)
+
+/* Buffer Descriptor List Upper Base Address */
+#define HDA_CL_SD_BDLPUBA(x)		\
+			(upper_32_bits(x))
+
+/* Software Position in Buffer Enable */
+#define HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_SHIFT	0
+#define HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_MASK	\
+			(1 << HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_SHIFT)
+
+#define HDA_CL_SPBFIFO_SPBFCCTL_SPIBE(x)	\
+			(((x) << HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_SHIFT) & \
+			 HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_MASK)
+
+#define HDA_CL_DMA_SD_INT_COMPLETE		0x4
+
+static int cl_skl_cldma_setup_bdle(struct snd_sof_dev *sdev,
+				   struct snd_dma_buffer *dmab_data,
+				   __le32 **bdlp, int size, int with_ioc)
+{
+	phys_addr_t addr = virt_to_phys(dmab_data->area);
+	__le32 *bdl = *bdlp;
+
+	/*
+	 * This code is simplified by using one fragment of physical memory and assuming
+	 * all the code fits. This could be improved with scatter-gather but the firmware
+	 * size is limited by DSP memory anyways
+	 */
+	bdl[0] = cpu_to_le32(lower_32_bits(addr));
+	bdl[1] = cpu_to_le32(upper_32_bits(addr));
+	bdl[2] = cpu_to_le32(size);
+	bdl[3] = (!with_ioc) ? 0 : cpu_to_le32(0x01);
+
+	return 1; /* one fragment */
+}
+
+static void cl_skl_cldma_stream_run(struct snd_sof_dev *sdev, bool enable)
+{
+	int sd_offset = SOF_HDA_ADSP_LOADER_BASE;
+	unsigned char val;
+	int retries;
+	u32 run = enable ? 0x1 : 0;
+
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+				sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL,
+				HDA_CL_SD_CTL_RUN(1), HDA_CL_SD_CTL_RUN(run));
+
+	retries = 300;
+	do {
+		udelay(3);
+
+		/* waiting for hardware to report the stream Run bit set */
+		val = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+				       sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL);
+		val &= HDA_CL_SD_CTL_RUN(1);
+		if (enable && val)
+			break;
+		else if (!enable && !val)
+			break;
+	} while (--retries);
+
+	if (retries == 0)
+		dev_err(sdev->dev, "%s: failed to set Run bit=%d enable=%d\n",
+			__func__, val, enable);
+}
+
+static void cl_skl_cldma_stream_clear(struct snd_sof_dev *sdev)
+{
+	int sd_offset = SOF_HDA_ADSP_LOADER_BASE;
+
+	/* make sure Run bit is cleared before setting stream register */
+	cl_skl_cldma_stream_run(sdev, 0);
+
+	/* Disable the Interrupt On Completion, FIFO Error Interrupt,
+	 * Descriptor Error Interrupt and set the cldma stream number to 0.
+	 */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+				sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL,
+				HDA_CL_SD_CTL_INT_MASK, HDA_CL_SD_CTL_INT(0));
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+				sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL,
+				HDA_CL_SD_CTL_STRM(0xf), HDA_CL_SD_CTL_STRM(0));
+
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL, HDA_CL_SD_BDLPLBA(0));
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU, 0);
+
+	/* Set the Cyclic Buffer Length to 0. */
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_CBL, 0);
+	/* Set the Last Valid Index. */
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_LVI, 0);
+}
+
+static void cl_skl_cldma_setup_spb(struct snd_sof_dev *sdev,
+				   unsigned int size, bool enable)
+{
+	int sd_offset = SOF_DSP_REG_CL_SPBFIFO;
+
+	if (enable)
+		snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+					sd_offset + SOF_HDA_ADSP_REG_CL_SPBFIFO_SPBFCCTL,
+					HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_MASK,
+					HDA_CL_SPBFIFO_SPBFCCTL_SPIBE(1));
+
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SPBFIFO_SPIB, size);
+}
+
+static void cl_skl_cldma_set_intr(struct snd_sof_dev *sdev, bool enable)
+{
+	u32 val = enable ? HDA_DSP_ADSPIC_CL_DMA : 0;
+
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIC,
+				HDA_DSP_ADSPIC_CL_DMA, val);
+}
+
+static void cl_skl_cldma_cleanup_spb(struct snd_sof_dev *sdev)
+{
+	int sd_offset = SOF_DSP_REG_CL_SPBFIFO;
+
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+				sd_offset + SOF_HDA_ADSP_REG_CL_SPBFIFO_SPBFCCTL,
+				HDA_CL_SPBFIFO_SPBFCCTL_SPIBE_MASK,
+				HDA_CL_SPBFIFO_SPBFCCTL_SPIBE(0));
+
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SPBFIFO_SPIB, 0);
+}
+
+static void cl_skl_cldma_setup_controller(struct snd_sof_dev *sdev,
+					  struct snd_dma_buffer *dmab_bdl,
+					  unsigned int max_size, u32 count)
+{
+	int sd_offset = SOF_HDA_ADSP_LOADER_BASE;
+
+	/* Clear the stream first and then set it. */
+	cl_skl_cldma_stream_clear(sdev);
+
+	/* setting the stream register */
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL,
+			  HDA_CL_SD_BDLPLBA(dmab_bdl->addr));
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU,
+			  HDA_CL_SD_BDLPUBA(dmab_bdl->addr));
+
+	/* Set the Cyclic Buffer Length. */
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_CBL, max_size);
+	/* Set the Last Valid Index. */
+	snd_sof_dsp_write(sdev, HDA_DSP_BAR,
+			  sd_offset + SOF_HDA_ADSP_REG_CL_SD_LVI, count - 1);
+
+	/* Set the Interrupt On Completion, FIFO Error Interrupt,
+	 * Descriptor Error Interrupt and the cldma stream number.
+	 */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+				sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL,
+				HDA_CL_SD_CTL_INT_MASK, HDA_CL_SD_CTL_INT(1));
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR,
+				sd_offset + SOF_HDA_ADSP_REG_CL_SD_CTL,
+				HDA_CL_SD_CTL_STRM(0xf),
+				HDA_CL_SD_CTL_STRM(1));
+}
+
+static int cl_stream_prepare_skl(struct snd_sof_dev *sdev,
+				 struct snd_dma_buffer *dmab,
+				 struct snd_dma_buffer *dmab_bdl)
+
+{
+	unsigned int bufsize = HDA_SKL_CLDMA_MAX_BUFFER_SIZE;
+	__le32 *bdl;
+	int frags;
+	int ret;
+
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, sdev->dev, bufsize, dmab);
+	if (ret < 0) {
+		dev_err(sdev->dev, "%s: failed to alloc fw buffer: %x\n", __func__, ret);
+		return ret;
+	}
+
+	ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, sdev->dev, bufsize, dmab_bdl);
+	if (ret < 0) {
+		dev_err(sdev->dev, "%s: failed to alloc blde: %x\n", __func__, ret);
+		snd_dma_free_pages(dmab);
+		return ret;
+	}
+
+	bdl = (__le32 *)dmab_bdl->area;
+	frags = cl_skl_cldma_setup_bdle(sdev, dmab, &bdl, bufsize, 1);
+	cl_skl_cldma_setup_controller(sdev, dmab_bdl, bufsize, frags);
+
+	return ret;
+}
+
+static void cl_cleanup_skl(struct snd_sof_dev *sdev,
+			   struct snd_dma_buffer *dmab,
+			   struct snd_dma_buffer *dmab_bdl)
+{
+	cl_skl_cldma_cleanup_spb(sdev);
+	cl_skl_cldma_stream_clear(sdev);
+	snd_dma_free_pages(dmab);
+	snd_dma_free_pages(dmab_bdl);
+}
+
+static int cl_dsp_init_skl(struct snd_sof_dev *sdev,
+			   struct snd_dma_buffer *dmab,
+			   struct snd_dma_buffer *dmab_bdl)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	const struct sof_intel_dsp_desc *chip = hda->desc;
+	unsigned int status;
+	u32 flags;
+	int ret;
+
+	/* check if the init_core is already enabled, if yes, reset and make it run,
+	 * if not, powerdown and enable it again.
+	 */
+	if (hda_dsp_core_is_enabled(sdev, chip->init_core_mask)) {
+		/* if enabled, reset it, and run the init_core. */
+		ret = hda_dsp_core_stall_reset(sdev, chip->init_core_mask);
+		if (ret < 0)
+			goto err;
+
+		ret = hda_dsp_core_run(sdev, chip->init_core_mask);
+		if (ret < 0) {
+			dev_err(sdev->dev, "%s: dsp core start failed %d\n", __func__, ret);
+			goto err;
+		}
+	} else {
+		/* if not enabled, power down it first and then powerup and run
+		 * the init_core.
+		 */
+		ret = hda_dsp_core_reset_power_down(sdev, chip->init_core_mask);
+		if (ret < 0) {
+			dev_err(sdev->dev, "%s: dsp core0 disable fail: %d\n", __func__, ret);
+			goto err;
+		}
+		ret = hda_dsp_enable_core(sdev, chip->init_core_mask);
+		if (ret < 0) {
+			dev_err(sdev->dev, "%s: dsp core0 enable fail: %d\n", __func__, ret);
+			goto err;
+		}
+	}
+
+	/* prepare DMA for code loader stream */
+	ret = cl_stream_prepare_skl(sdev, dmab, dmab_bdl);
+	if (ret < 0) {
+		dev_err(sdev->dev, "%s: dma prepare fw loading err: %x\n", __func__, ret);
+		return ret;
+	}
+
+	/* enable the interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPIC,
+				HDA_DSP_ADSPIC_IPC, HDA_DSP_ADSPIC_IPC);
+
+	/* enable IPC DONE interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, chip->ipc_ctl,
+				HDA_DSP_REG_HIPCCTL_DONE,
+				HDA_DSP_REG_HIPCCTL_DONE);
+
+	/* enable IPC BUSY interrupt */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, chip->ipc_ctl,
+				HDA_DSP_REG_HIPCCTL_BUSY,
+				HDA_DSP_REG_HIPCCTL_BUSY);
+
+	/* polling the ROM init status information. */
+	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
+					    chip->rom_status_reg, status,
+					    (FSR_TO_STATE_CODE(status)
+					     == FSR_STATE_INIT_DONE),
+					    HDA_DSP_REG_POLL_INTERVAL_US,
+					    chip->rom_init_timeout *
+					    USEC_PER_MSEC);
+	if (ret < 0)
+		goto err;
+
+	return ret;
+
+err:
+	flags = SOF_DBG_DUMP_PCI | SOF_DBG_DUMP_MBOX;
+
+	snd_sof_dsp_dbg_dump(sdev, "Boot failed\n", flags);
+	cl_cleanup_skl(sdev, dmab, dmab_bdl);
+	hda_dsp_core_reset_power_down(sdev, chip->init_core_mask);
+	return ret;
+}
+
+static void cl_skl_cldma_fill_buffer(struct snd_sof_dev *sdev,
+				     struct snd_dma_buffer *dmab,
+				     unsigned int bufsize,
+				     unsigned int copysize,
+				     const void *curr_pos,
+				     bool intr_enable)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+
+	/* copy the image into the buffer with the maximum buffer size. */
+	unsigned int size = (bufsize == copysize) ? bufsize : copysize;
+
+	memcpy(dmab->area, curr_pos, size);
+
+	/* Set the wait condition for every load. */
+	hda->code_loading = 1;
+
+	/* Set the interrupt. */
+	if (intr_enable)
+		cl_skl_cldma_set_intr(sdev, true);
+
+	/* Set the SPB. */
+	cl_skl_cldma_setup_spb(sdev, size, true);
+
+	/* Trigger the code loading stream. */
+	cl_skl_cldma_stream_run(sdev, true);
+}
+
+static int cl_skl_cldma_wait_interruptible(struct snd_sof_dev *sdev,
+					   bool intr_wait)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	const struct sof_intel_dsp_desc *chip = hda->desc;
+	int sd_offset = SOF_HDA_ADSP_LOADER_BASE;
+	u8 cl_dma_intr_status;
+
+	/*
+	 * Wait for CLDMA interrupt to inform the binary segment transfer is
+	 * complete.
+	 */
+	if (!wait_event_timeout(hda->waitq, !hda->code_loading,
+				msecs_to_jiffies(HDA_SKL_WAIT_TIMEOUT))) {
+		dev_err(sdev->dev, "cldma copy timeout\n");
+		dev_err(sdev->dev, "ROM code=%#x: FW status=%#x\n",
+			snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_ROM_ERROR),
+			snd_sof_dsp_read(sdev, HDA_DSP_BAR, chip->rom_status_reg));
+		return -EIO;
+	}
+
+	/* now check DMA interrupt status */
+	cl_dma_intr_status = snd_sof_dsp_read(sdev, HDA_DSP_BAR,
+					      sd_offset + SOF_HDA_ADSP_REG_CL_SD_STS);
+
+	if (!(cl_dma_intr_status & HDA_CL_DMA_SD_INT_COMPLETE)) {
+		dev_err(sdev->dev, "cldma copy failed\n");
+		return -EIO;
+	}
+
+	dev_dbg(sdev->dev, "cldma buffer copy complete\n");
+	return 0;
+}
+
+static int
+cl_skl_cldma_copy_to_buf(struct snd_sof_dev *sdev,
+			 struct snd_dma_buffer *dmab,
+			 const void *bin,
+			 u32 total_size, u32 bufsize)
+{
+	unsigned int bytes_left = total_size;
+	const void *curr_pos = bin;
+	int ret;
+
+	if (total_size <= 0)
+		return -EINVAL;
+
+	while (bytes_left > 0) {
+		if (bytes_left > bufsize) {
+			dev_dbg(sdev->dev, "cldma copy %#x bytes\n", bufsize);
+
+			cl_skl_cldma_fill_buffer(sdev, dmab, bufsize, bufsize, curr_pos, true);
+
+			ret = cl_skl_cldma_wait_interruptible(sdev, false);
+			if (ret < 0) {
+				dev_err(sdev->dev, "%s: fw failed to load. %#x bytes remaining\n",
+					__func__, bytes_left);
+				return ret;
+			}
+
+			bytes_left -= bufsize;
+			curr_pos += bufsize;
+		} else {
+			dev_dbg(sdev->dev, "cldma copy %#x bytes\n", bytes_left);
+
+			cl_skl_cldma_set_intr(sdev, false);
+			cl_skl_cldma_fill_buffer(sdev, dmab, bufsize, bytes_left, curr_pos, false);
+			return 0;
+		}
+	}
+
+	return bytes_left;
+}
+
+static int cl_copy_fw_skl(struct snd_sof_dev *sdev,
+			  struct snd_dma_buffer *dmab)
+
+{
+	struct snd_sof_pdata *plat_data = sdev->pdata;
+	const struct firmware *fw =  plat_data->fw;
+	struct firmware stripped_firmware;
+	unsigned int bufsize = HDA_SKL_CLDMA_MAX_BUFFER_SIZE;
+	int ret;
+
+	stripped_firmware.data = plat_data->fw->data + plat_data->fw_offset;
+	stripped_firmware.size = plat_data->fw->size - plat_data->fw_offset;
+
+	dev_dbg(sdev->dev, "firmware size: %#zx buffer size %#x\n", fw->size, bufsize);
+
+	ret = cl_skl_cldma_copy_to_buf(sdev, dmab, stripped_firmware.data,
+				       stripped_firmware.size, bufsize);
+	if (ret < 0)
+		dev_err(sdev->dev, "%s: fw copy failed %d\n", __func__, ret);
+
+	return ret;
+}
+
+int hda_dsp_cl_boot_firmware_skl(struct snd_sof_dev *sdev)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	const struct sof_intel_dsp_desc *chip = hda->desc;
+	struct snd_dma_buffer dmab_bdl;
+	struct snd_dma_buffer dmab;
+	unsigned int reg;
+	u32 flags;
+	int ret;
+
+	ret = cl_dsp_init_skl(sdev, &dmab, &dmab_bdl);
+
+	/* retry enabling core and ROM load. seemed to help */
+	if (ret < 0) {
+		ret = cl_dsp_init_skl(sdev, &dmab, &dmab_bdl);
+		if (ret < 0) {
+			dev_err(sdev->dev, "Error code=%#x: FW status=%#x\n",
+				snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_ROM_ERROR),
+				snd_sof_dsp_read(sdev, HDA_DSP_BAR, chip->rom_status_reg));
+			dev_err(sdev->dev, "Core En/ROM load fail:%d\n", ret);
+			return ret;
+		}
+	}
+
+	dev_dbg(sdev->dev, "ROM init successful\n");
+
+	/* at this point DSP ROM has been initialized and should be ready for
+	 * code loading and firmware boot
+	 */
+	ret = cl_copy_fw_skl(sdev, &dmab);
+	if (ret < 0) {
+		dev_err(sdev->dev, "%s: load firmware failed : %d\n", __func__, ret);
+		goto err;
+	}
+
+	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
+					    chip->rom_status_reg, reg,
+					    (FSR_TO_STATE_CODE(reg)
+					     == FSR_STATE_ROM_BASEFW_ENTERED),
+					    HDA_DSP_REG_POLL_INTERVAL_US,
+					    HDA_DSP_BASEFW_TIMEOUT_US);
+
+	dev_dbg(sdev->dev, "Firmware download successful, booting...\n");
+
+	cl_skl_cldma_stream_run(sdev, false);
+	cl_cleanup_skl(sdev, &dmab, &dmab_bdl);
+
+	if (!ret)
+		return chip->init_core_mask;
+
+	return ret;
+
+err:
+	flags = SOF_DBG_DUMP_PCI | SOF_DBG_DUMP_MBOX;
+
+	snd_sof_dsp_dbg_dump(sdev, "Boot failed\n", flags);
+
+	/* power down DSP */
+	hda_dsp_core_reset_power_down(sdev, chip->init_core_mask);
+	cl_skl_cldma_stream_run(sdev, false);
+	cl_cleanup_skl(sdev, &dmab, &dmab_bdl);
+
+	dev_err(sdev->dev, "%s: load fw failed err: %d\n", __func__, ret);
+	return ret;
+}
diff --git a/sound/soc/sof/intel/hda-loader.c b/sound/soc/sof/intel/hda-loader.c
index eb22eb3f6fee..98812d51b31c 100644
--- a/sound/soc/sof/intel/hda-loader.c
+++ b/sound/soc/sof/intel/hda-loader.c
@@ -177,14 +177,13 @@ int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot)
 	 * - IMR boot: wait for ROM firmware entered (firmware booted up from IMR)
 	 */
 	if (imr_boot)
-		target_status = HDA_DSP_ROM_FW_ENTERED;
+		target_status = FSR_STATE_FW_ENTERED;
 	else
-		target_status = HDA_DSP_ROM_INIT;
+		target_status = FSR_STATE_INIT_DONE;
 
 	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
 					chip->rom_status_reg, status,
-					((status & HDA_DSP_ROM_STS_MASK)
-						== target_status),
+					(FSR_TO_STATE_CODE(status) == target_status),
 					HDA_DSP_REG_POLL_INTERVAL_US,
 					chip->rom_init_timeout *
 					USEC_PER_MSEC);
@@ -292,8 +291,7 @@ int hda_cl_copy_fw(struct snd_sof_dev *sdev, struct hdac_ext_stream *hext_stream
 
 	status = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR,
 					chip->rom_status_reg, reg,
-					((reg & HDA_DSP_ROM_STS_MASK)
-						== HDA_DSP_ROM_FW_ENTERED),
+					(FSR_TO_STATE_CODE(reg) == FSR_STATE_FW_ENTERED),
 					HDA_DSP_REG_POLL_INTERVAL_US,
 					HDA_DSP_BASEFW_TIMEOUT_US);
 
diff --git a/sound/soc/sof/intel/hda-pcm.c b/sound/soc/sof/intel/hda-pcm.c
index 6888e0a4665d..0a9c80216a8c 100644
--- a/sound/soc/sof/intel/hda-pcm.c
+++ b/sound/soc/sof/intel/hda-pcm.c
@@ -18,6 +18,7 @@
 #include <linux/moduleparam.h>
 #include <sound/hda_register.h>
 #include <sound/pcm_params.h>
+#include <trace/events/sof_intel.h>
 #include "../sof-audio.h"
 #include "../ops.h"
 #include "hda.h"
@@ -196,8 +197,7 @@ snd_pcm_uframes_t hda_dsp_pcm_pointer(struct snd_sof_dev *sdev,
 found:
 	pos = bytes_to_frames(substream->runtime, pos);
 
-	dev_vdbg(sdev->dev, "PCM: stream %d dir %d position %lu\n",
-		 hstream->index, substream->stream, pos);
+	trace_sof_intel_hda_dsp_pcm(sdev, hstream, substream, pos);
 	return pos;
 }
 
diff --git a/sound/soc/sof/intel/hda-stream.c b/sound/soc/sof/intel/hda-stream.c
index b58662faa4aa..be60e7785da9 100644
--- a/sound/soc/sof/intel/hda-stream.c
+++ b/sound/soc/sof/intel/hda-stream.c
@@ -19,6 +19,7 @@
 #include <sound/hdaudio_ext.h>
 #include <sound/hda_register.h>
 #include <sound/sof.h>
+#include <trace/events/sof_intel.h>
 #include "../ops.h"
 #include "../sof-audio.h"
 #include "hda.h"
@@ -93,9 +94,6 @@ static int hda_setup_bdle(struct snd_sof_dev *sdev,
 		bdl++;
 		hstream->frags++;
 		offset += chunk;
-
-		dev_vdbg(sdev->dev, "bdl, frags:%d, chunk size:0x%x;\n",
-			 hstream->frags, chunk);
 	}
 
 	*bdlp = bdl;
@@ -700,7 +698,7 @@ bool hda_dsp_check_stream_irq(struct snd_sof_dev *sdev)
 	spin_lock_irq(&bus->reg_lock);
 
 	status = snd_hdac_chip_readl(bus, INTSTS);
-	dev_vdbg(bus->dev, "stream irq, INTSTS status: 0x%x\n", status);
+	trace_sof_intel_hda_dsp_check_stream_irq(sdev, status);
 
 	/* if Register inaccessible, ignore it.*/
 	if (status != 0xffffffff)
@@ -739,8 +737,7 @@ static bool hda_dsp_stream_check(struct hdac_bus *bus, u32 status)
 		if (status & BIT(s->index) && s->opened) {
 			sd_status = snd_hdac_stream_readb(s, SD_STS);
 
-			dev_vdbg(bus->dev, "stream %d status 0x%x\n",
-				 s->index, sd_status);
+			trace_sof_intel_hda_dsp_stream_status(bus->dev, s, sd_status);
 
 			snd_hdac_stream_writeb(s, SD_STS, sd_status);
 
diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c
index 6d4ecbe14adf..1188ec51816b 100644
--- a/sound/soc/sof/intel/hda.c
+++ b/sound/soc/sof/intel/hda.c
@@ -31,6 +31,9 @@
 #include "../ops.h"
 #include "hda.h"
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/sof_intel.h>
+
 #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
 #include <sound/soc-acpi-intel-match.h>
 #endif
@@ -376,6 +379,10 @@ static int dmic_num_override = -1;
 module_param_named(dmic_num, dmic_num_override, int, 0444);
 MODULE_PARM_DESC(dmic_num, "SOF HDA DMIC number");
 
+static int mclk_id_override = -1;
+module_param_named(mclk_id, mclk_id_override, int, 0444);
+MODULE_PARM_DESC(mclk_id, "SOF SSP mclk_id");
+
 #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
 static bool hda_codec_use_common_hdmi = IS_ENABLED(CONFIG_SND_HDA_CODEC_HDMI);
 module_param_named(use_common_hdmi, hda_codec_use_common_hdmi, bool, 0444);
@@ -591,7 +598,8 @@ void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags)
 	/* print ROM/FW status */
 	hda_dsp_get_state(sdev, level);
 
-	if (flags & SOF_DBG_DUMP_REGS) {
+	/* The firmware register dump only available with IPC3 */
+	if (flags & SOF_DBG_DUMP_REGS && sdev->pdata->ipc_type == SOF_IPC) {
 		u32 status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_FW_STATUS);
 		u32 panic = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_FW_TRACEP);
 
@@ -655,6 +663,24 @@ void hda_ipc_dump(struct snd_sof_dev *sdev)
 		hipcie, hipct, hipcctl);
 }
 
+void hda_ipc4_dump(struct snd_sof_dev *sdev)
+{
+	u32 hipci, hipcie, hipct, hipcte, hipcctl;
+
+	hda_ipc_irq_dump(sdev);
+
+	hipci = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCI);
+	hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCIE);
+	hipct = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCT);
+	hipcte = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCTE);
+	hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCCTL);
+
+	/* dump the IPC regs */
+	/* TODO: parse the raw msg */
+	dev_err(sdev->dev, "Host IPC initiator: %#x|%#x, target: %#x|%#x, ctl: %#x\n",
+		hipci, hipcie, hipct, hipcte, hipcctl);
+}
+
 static int hda_init(struct snd_sof_dev *sdev)
 {
 	struct hda_bus *hbus;
@@ -749,6 +775,18 @@ static int check_nhlt_ssp_mask(struct snd_sof_dev *sdev)
 	return ssp_mask;
 }
 
+static int check_nhlt_ssp_mclk_mask(struct snd_sof_dev *sdev, int ssp_num)
+{
+	struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata;
+	struct nhlt_acpi_table *nhlt;
+
+	nhlt = hdev->nhlt;
+	if (!nhlt)
+		return 0;
+
+	return intel_nhlt_ssp_mclk_mask(nhlt, ssp_num);
+}
+
 #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) || IS_ENABLED(CONFIG_SND_SOC_SOF_INTEL_SOUNDWIRE)
 
 static const char *fixup_tplg_name(struct snd_sof_dev *sdev,
@@ -938,17 +976,25 @@ static irqreturn_t hda_dsp_interrupt_thread(int irq, void *context)
 	struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata;
 
 	/* deal with streams and controller first */
-	if (hda_dsp_check_stream_irq(sdev))
+	if (hda_dsp_check_stream_irq(sdev)) {
+		trace_sof_intel_hda_irq(sdev, "stream");
 		hda_dsp_stream_threaded_handler(irq, sdev);
+	}
 
-	if (hda_check_ipc_irq(sdev))
+	if (hda_check_ipc_irq(sdev)) {
+		trace_sof_intel_hda_irq(sdev, "ipc");
 		sof_ops(sdev)->irq_thread(irq, sdev);
+	}
 
-	if (hda_dsp_check_sdw_irq(sdev))
+	if (hda_dsp_check_sdw_irq(sdev)) {
+		trace_sof_intel_hda_irq(sdev, "sdw");
 		hda_dsp_sdw_thread(irq, hdev->sdw);
+	}
 
-	if (hda_sdw_check_wakeen_irq(sdev))
+	if (hda_sdw_check_wakeen_irq(sdev)) {
+		trace_sof_intel_hda_irq(sdev, "wakeen");
 		hda_sdw_process_wakeen(sdev);
+	}
 
 	hda_check_for_state_change(sdev);
 
@@ -1109,6 +1155,8 @@ int hda_dsp_probe(struct snd_sof_dev *sdev)
 
 	INIT_DELAYED_WORK(&hdev->d0i3_work, hda_dsp_d0i3_work);
 
+	init_waitqueue_head(&hdev->waitq);
+
 	hdev->nhlt = intel_nhlt_init(sdev->dev);
 
 	return 0;
@@ -1162,9 +1210,9 @@ int hda_dsp_remove(struct snd_sof_dev *sdev)
 	snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL,
 				SOF_HDA_INT_CTRL_EN | SOF_HDA_INT_GLOBAL_EN, 0);
 
-	/* disable cores */
-	if (chip)
-		hda_dsp_core_reset_power_down(sdev, chip->host_managed_cores_mask);
+	/* no need to check for error as the DSP will be disabled anyway */
+	if (chip && chip->power_down_dsp)
+		chip->power_down_dsp(sdev);
 
 	/* disable DSP */
 	snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
@@ -1190,6 +1238,14 @@ int hda_dsp_remove(struct snd_sof_dev *sdev)
 	return 0;
 }
 
+int hda_power_down_dsp(struct snd_sof_dev *sdev)
+{
+	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
+	const struct sof_intel_dsp_desc *chip = hda->desc;
+
+	return hda_dsp_core_reset_power_down(sdev, chip->host_managed_cores_mask);
+}
+
 #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
 static void hda_generic_machine_select(struct snd_sof_dev *sdev,
 				       struct snd_soc_acpi_mach **mach)
@@ -1529,6 +1585,7 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev)
 		    mach->mach_params.i2s_link_mask) {
 			const struct sof_intel_dsp_desc *chip = get_chip_info(sdev->pdata);
 			int ssp_num;
+			int mclk_mask;
 
 			if (hweight_long(mach->mach_params.i2s_link_mask) > 1 &&
 			    !(mach->tplg_quirk_mask & SND_SOC_ACPI_TPLG_INTEL_SSP_MSB))
@@ -1553,6 +1610,21 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev)
 
 			sof_pdata->tplg_filename = tplg_filename;
 			add_extension = true;
+
+			mclk_mask = check_nhlt_ssp_mclk_mask(sdev, ssp_num);
+
+			if (mclk_mask < 0) {
+				dev_err(sdev->dev, "Invalid MCLK configuration\n");
+				return NULL;
+			}
+
+			dev_dbg(sdev->dev, "MCLK mask %#x found in NHLT\n", mclk_mask);
+
+			if (mclk_mask) {
+				dev_info(sdev->dev, "Overriding topology with MCLK mask %#x from NHLT\n", mclk_mask);
+				sdev->mclk_id_override = true;
+				sdev->mclk_id_quirk = (mclk_mask & BIT(0)) ? 0 : 1;
+			}
 		}
 
 		if (tplg_fixup && add_extension) {
@@ -1565,6 +1637,13 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev)
 
 			sof_pdata->tplg_filename = tplg_filename;
 		}
+
+		/* check if mclk_id should be modified from topology defaults */
+		if (mclk_id_override >= 0) {
+			dev_info(sdev->dev, "Overriding topology with MCLK %d from kernel_parameter\n", mclk_id_override);
+			sdev->mclk_id_override = true;
+			sdev->mclk_id_quirk = mclk_id_override;
+		}
 	}
 
 	/*
diff --git a/sound/soc/sof/intel/hda.h b/sound/soc/sof/intel/hda.h
index 5ef3e8775e36..2ab3c3840b92 100644
--- a/sound/soc/sof/intel/hda.h
+++ b/sound/soc/sof/intel/hda.h
@@ -229,6 +229,7 @@
 #define FSR_STATE_ROM_GET_LOAD_OFFSET		0x7
 #define FSR_STATE_ROM_FETCH_ROM_EXT		0x8
 #define FSR_STATE_ROM_FETCH_ROM_EXT_DONE	0x9
+#define FSR_STATE_ROM_BASEFW_ENTERED		0xf /* SKL */
 
 /* (ROM) CSE states */
 #define FSR_STATE_ROM_CSE_IMR_REQUEST			0x10
@@ -251,12 +252,6 @@
 #define FSR_STATE_BRINGUP_FW_ENTERED		FSR_STATE_FW_ENTERED
 
 /* ROM  status/error values */
-#define HDA_DSP_ROM_STS_MASK			GENMASK(23, 0)
-#define HDA_DSP_ROM_INIT			0x1
-#define HDA_DSP_ROM_FW_MANIFEST_LOADED		0x3
-#define HDA_DSP_ROM_FW_FW_LOADED		0x4
-#define HDA_DSP_ROM_FW_ENTERED			0x5
-#define HDA_DSP_ROM_RFW_START			0xf
 #define HDA_DSP_ROM_CSE_ERROR			40
 #define HDA_DSP_ROM_CSE_WRONG_RESPONSE		41
 #define HDA_DSP_ROM_IMR_TO_SMALL		42
@@ -424,6 +419,7 @@
 #endif
 
 /* Intel HD Audio SRAM Window 0*/
+#define HDA_DSP_SRAM_REG_ROM_STATUS_SKL	0x8000
 #define HDA_ADSP_SRAM0_BASE_SKL		0x8000
 
 /* Firmware status window */
@@ -441,6 +437,8 @@
 #define APL_SSP_COUNT		6
 #define CNL_SSP_COUNT		3
 #define ICL_SSP_COUNT		6
+#define TGL_SSP_COUNT		3
+#define MTL_SSP_COUNT		3
 
 /* SSP Registers */
 #define SSP_SSC1_OFFSET		0x4
@@ -518,6 +516,9 @@ struct sof_intel_hda_dev {
 	/* FW clock config, 0:HPRO, 1:LPRO */
 	bool clk_config_lpro;
 
+	wait_queue_head_t waitq;
+	bool code_loading;
+
 	/* Intel NHLT information */
 	struct nhlt_acpi_table *nhlt;
 };
@@ -566,9 +567,11 @@ int hda_dsp_core_run(struct snd_sof_dev *sdev, unsigned int core_mask);
 int hda_dsp_enable_core(struct snd_sof_dev *sdev, unsigned int core_mask);
 int hda_dsp_core_reset_power_down(struct snd_sof_dev *sdev,
 				  unsigned int core_mask);
+int hda_power_down_dsp(struct snd_sof_dev *sdev);
 int hda_dsp_core_get(struct snd_sof_dev *sdev, int core);
 void hda_dsp_ipc_int_enable(struct snd_sof_dev *sdev);
 void hda_dsp_ipc_int_disable(struct snd_sof_dev *sdev);
+bool hda_dsp_core_is_enabled(struct snd_sof_dev *sdev, unsigned int core_mask);
 
 int hda_dsp_set_power_state(struct snd_sof_dev *sdev,
 			    const struct sof_dsp_power_state *target_state);
@@ -584,6 +587,7 @@ void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags);
 void hda_ipc_dump(struct snd_sof_dev *sdev);
 void hda_ipc_irq_dump(struct snd_sof_dev *sdev);
 void hda_dsp_d0i3_work(struct work_struct *work);
+int hda_dsp_disable_interrupts(struct snd_sof_dev *sdev);
 
 /*
  * DSP PCM Operations.
@@ -773,6 +777,8 @@ int hda_dsp_dais_suspend(struct snd_sof_dev *sdev);
  */
 extern struct snd_sof_dsp_ops sof_hda_common_ops;
 
+extern struct snd_sof_dsp_ops sof_skl_ops;
+int sof_skl_ops_init(struct snd_sof_dev *sdev);
 extern struct snd_sof_dsp_ops sof_apl_ops;
 int sof_apl_ops_init(struct snd_sof_dev *sdev);
 extern struct snd_sof_dsp_ops sof_cnl_ops;
@@ -784,6 +790,7 @@ int sof_icl_ops_init(struct snd_sof_dev *sdev);
 extern struct snd_sof_dsp_ops sof_mtl_ops;
 int sof_mtl_ops_init(struct snd_sof_dev *sdev);
 
+extern const struct sof_intel_dsp_desc skl_chip_info;
 extern const struct sof_intel_dsp_desc apl_chip_info;
 extern const struct sof_intel_dsp_desc cnl_chip_info;
 extern const struct sof_intel_dsp_desc icl_chip_info;
@@ -837,11 +844,16 @@ extern int sof_hda_position_quirk;
 void hda_set_dai_drv_ops(struct snd_sof_dev *sdev, struct snd_sof_dsp_ops *ops);
 void hda_ops_free(struct snd_sof_dev *sdev);
 
+/* SKL/KBL */
+int hda_dsp_cl_boot_firmware_skl(struct snd_sof_dev *sdev);
+int hda_dsp_core_stall_reset(struct snd_sof_dev *sdev, unsigned int core_mask);
+
 /* IPC4 */
 irqreturn_t cnl_ipc4_irq_thread(int irq, void *context);
 int cnl_ipc4_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg);
 irqreturn_t hda_dsp_ipc4_irq_thread(int irq, void *context);
 int hda_dsp_ipc4_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg);
+void hda_ipc4_dump(struct snd_sof_dev *sdev);
 extern struct sdw_intel_ops sdw_callback;
 
 #endif
diff --git a/sound/soc/sof/intel/icl.c b/sound/soc/sof/intel/icl.c
index 4e37b7fe0627..6d5877108a3d 100644
--- a/sound/soc/sof/intel/icl.c
+++ b/sound/soc/sof/intel/icl.c
@@ -13,6 +13,7 @@
 #include <linux/kconfig.h>
 #include <linux/export.h>
 #include <linux/bits.h>
+#include "../ipc4-priv.h"
 #include "../ops.h"
 #include "hda.h"
 #include "hda-ipc.h"
@@ -106,16 +107,42 @@ int sof_icl_ops_init(struct snd_sof_dev *sdev)
 	/* probe/remove/shutdown */
 	sof_icl_ops.shutdown	= hda_dsp_shutdown;
 
-	/* doorbell */
-	sof_icl_ops.irq_thread	= cnl_ipc_irq_thread;
+	if (sdev->pdata->ipc_type == SOF_IPC) {
+		/* doorbell */
+		sof_icl_ops.irq_thread	= cnl_ipc_irq_thread;
 
-	/* ipc */
-	sof_icl_ops.send_msg	= cnl_ipc_send_msg;
+		/* ipc */
+		sof_icl_ops.send_msg	= cnl_ipc_send_msg;
+
+		/* debug */
+		sof_icl_ops.ipc_dump	= cnl_ipc_dump;
+	}
+
+	if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) {
+		struct sof_ipc4_fw_data *ipc4_data;
+
+		sdev->private = devm_kzalloc(sdev->dev, sizeof(*ipc4_data), GFP_KERNEL);
+		if (!sdev->private)
+			return -ENOMEM;
+
+		ipc4_data = sdev->private;
+		ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET;
+
+		ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_2;
+
+		/* doorbell */
+		sof_icl_ops.irq_thread	= cnl_ipc4_irq_thread;
+
+		/* ipc */
+		sof_icl_ops.send_msg	= cnl_ipc4_send_msg;
+
+		/* debug */
+		sof_icl_ops.ipc_dump	= cnl_ipc4_dump;
+	}
 
 	/* debug */
 	sof_icl_ops.debug_map	= icl_dsp_debugfs;
 	sof_icl_ops.debug_map_count	= ARRAY_SIZE(icl_dsp_debugfs);
-	sof_icl_ops.ipc_dump	= cnl_ipc_dump;
 
 	/* pre/post fw run */
 	sof_icl_ops.post_fw_run = icl_dsp_post_fw_run;
@@ -153,6 +180,8 @@ const struct sof_intel_dsp_desc icl_chip_info = {
 	.check_sdw_irq	= hda_common_check_sdw_irq,
 	.check_ipc_irq	= hda_dsp_check_ipc_irq,
 	.cl_init = cl_dsp_init,
+	.power_down_dsp = hda_power_down_dsp,
+	.disable_interrupts = hda_dsp_disable_interrupts,
 	.hw_ip_version = SOF_INTEL_CAVS_2_0,
 };
 EXPORT_SYMBOL_NS(icl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON);
diff --git a/sound/soc/sof/intel/mtl.c b/sound/soc/sof/intel/mtl.c
index 96239ebb1eed..10298532816f 100644
--- a/sound/soc/sof/intel/mtl.c
+++ b/sound/soc/sof/intel/mtl.c
@@ -11,6 +11,7 @@
 
 #include <linux/firmware.h>
 #include <sound/sof/ipc4/header.h>
+#include <trace/events/sof_intel.h>
 #include "../ipc4-priv.h"
 #include "../ops.h"
 #include "hda.h"
@@ -63,7 +64,7 @@ static bool mtl_dsp_check_ipc_irq(struct snd_sof_dev *sdev)
 	hfintipptr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFINTIPPTR) & MTL_HFINTIPPTR_PTR_MASK;
 	irq_status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, hfintipptr + MTL_DSP_IRQSTS);
 
-	dev_vdbg(sdev->dev, "irq handler: irq_status:0x%x\n", irq_status);
+	trace_sof_intel_hda_irq_ipc_check(sdev, irq_status);
 
 	if (irq_status != U32_MAX && (irq_status & MTL_DSP_IRQSTS_IPC))
 		return true;
@@ -143,7 +144,6 @@ static int mtl_enable_interrupts(struct snd_sof_dev *sdev)
 
 	/* check if operation was successful */
 	host_ipc = MTL_IRQ_INTEN_L_HOST_IPC_MASK | MTL_IRQ_INTEN_L_SOUNDWIRE_MASK;
-	irqinten = snd_sof_dsp_read(sdev, HDA_DSP_BAR, hfintipptr);
 	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, hfintipptr, irqinten,
 					    (irqinten & host_ipc) == host_ipc,
 					    HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US);
@@ -158,7 +158,6 @@ static int mtl_enable_interrupts(struct snd_sof_dev *sdev)
 
 	/* check if operation was successful */
 	host_ipc = MTL_DSP_REG_HfHIPCIE_IE_MASK;
-	hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE);
 	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE, hipcie,
 					    (hipcie & host_ipc) == host_ipc,
 					    HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US);
@@ -170,7 +169,6 @@ static int mtl_enable_interrupts(struct snd_sof_dev *sdev)
 	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE,
 				MTL_DSP_REG_HfSNDWIE_IE_MASK, MTL_DSP_REG_HfSNDWIE_IE_MASK);
 	host_ipc = MTL_DSP_REG_HfSNDWIE_IE_MASK;
-	hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE);
 	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE, hipcie,
 					    (hipcie & host_ipc) == host_ipc,
 					    HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US);
@@ -198,7 +196,6 @@ static int mtl_disable_interrupts(struct snd_sof_dev *sdev)
 
 	/* check if operation was successful */
 	host_ipc = MTL_IRQ_INTEN_L_HOST_IPC_MASK | MTL_IRQ_INTEN_L_SOUNDWIRE_MASK;
-	irqinten = snd_sof_dsp_read(sdev, HDA_DSP_BAR, hfintipptr);
 	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, hfintipptr, irqinten,
 					    (irqinten & host_ipc) == 0,
 					    HDA_DSP_REG_POLL_INTERVAL_US, HDA_DSP_RESET_TIMEOUT_US);
@@ -212,7 +209,6 @@ static int mtl_disable_interrupts(struct snd_sof_dev *sdev)
 
 	/* check if operation was successful */
 	host_ipc = MTL_DSP_REG_HfHIPCIE_IE_MASK;
-	hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE);
 	ret1 = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfHIPCIE, hipcie,
 					     (hipcie & host_ipc) == 0,
 					     HDA_DSP_REG_POLL_INTERVAL_US,
@@ -227,7 +223,6 @@ static int mtl_disable_interrupts(struct snd_sof_dev *sdev)
 	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE,
 				MTL_DSP_REG_HfSNDWIE_IE_MASK, 0);
 	host_ipc = MTL_DSP_REG_HfSNDWIE_IE_MASK;
-	hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE);
 	ret1 = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_DSP_REG_HfSNDWIE, hipcie,
 					     (hipcie & host_ipc) == 0,
 					     HDA_DSP_REG_POLL_INTERVAL_US,
@@ -259,7 +254,6 @@ static int mtl_dsp_pre_fw_run(struct snd_sof_dev *sdev)
 
 	/* poll with timeout to check if operation successful */
 	cpa = MTL_HFDSSCS_CPA_MASK;
-	dsphfdsscs = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFDSSCS);
 	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFDSSCS, dsphfdsscs,
 					    (dsphfdsscs & cpa) == cpa, HDA_DSP_REG_POLL_INTERVAL_US,
 					    HDA_DSP_RESET_TIMEOUT_US);
@@ -276,7 +270,6 @@ static int mtl_dsp_pre_fw_run(struct snd_sof_dev *sdev)
 
 	/* poll with timeout to check if operation successful */
 	pgs = MTL_HFPWRSTS_DSPHPXPGS_MASK;
-	dsphfpwrsts = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFPWRSTS);
 	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFPWRSTS, dsphfpwrsts,
 					    (dsphfpwrsts & pgs) == pgs,
 					    HDA_DSP_REG_POLL_INTERVAL_US,
@@ -405,6 +398,33 @@ static int mtl_dsp_core_power_down(struct snd_sof_dev *sdev, int core)
 	return ret;
 }
 
+static int mtl_power_down_dsp(struct snd_sof_dev *sdev)
+{
+	u32 dsphfdsscs, cpa;
+	int ret;
+
+	/* first power down core */
+	ret = mtl_dsp_core_power_down(sdev, SOF_DSP_PRIMARY_CORE);
+	if (ret) {
+		dev_err(sdev->dev, "mtl dsp power down error, %d\n", ret);
+		return ret;
+	}
+
+	/* Set the DSP subsystem power down */
+	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_HFDSSCS,
+				MTL_HFDSSCS_SPA_MASK, 0);
+
+	/* Wait for unstable CPA read (1 then 0 then 1) just after setting SPA bit */
+	usleep_range(1000, 1010);
+
+	/* poll with timeout to check if operation successful */
+	cpa = MTL_HFDSSCS_CPA_MASK;
+	dsphfdsscs = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFDSSCS);
+	return snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFDSSCS, dsphfdsscs,
+					     (dsphfdsscs & cpa) == 0, HDA_DSP_REG_POLL_INTERVAL_US,
+					     HDA_DSP_RESET_TIMEOUT_US);
+}
+
 static int mtl_dsp_cl_init(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot)
 {
 	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
@@ -551,166 +571,27 @@ static int mtl_dsp_ipc_get_window_offset(struct snd_sof_dev *sdev, u32 id)
 	return MTL_SRAM_WINDOW_OFFSET(id);
 }
 
-static int mtl_suspend(struct snd_sof_dev *sdev, bool runtime_suspend)
-{
-	struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
-	const struct sof_intel_dsp_desc *chip = hda->desc;
-#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
-	struct hdac_bus *bus = sof_to_bus(sdev);
-#endif
-	u32 dsphfdsscs;
-	u32 cpa;
-	int ret;
-	int i;
-
-	mtl_disable_ipc_interrupts(sdev);
-	ret = mtl_disable_interrupts(sdev);
-	if (ret)
-		return ret;
-
-#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
-	hda_codec_jack_wake_enable(sdev, runtime_suspend);
-	/* power down all hda link */
-	snd_hdac_ext_bus_link_power_down_all(bus);
-#endif
-	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_HFPWRCTL,
-				MTL_HFPWRCTL_WPDSPHPXPG, 0);
-
-	/* Set the DSP subsystem power down */
-	snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, MTL_HFDSSCS,
-				MTL_HFDSSCS_SPA_MASK, 0);
-
-	/* Wait for unstable CPA read (1 then 0 then 1) just after setting SPA bit */
-	usleep_range(1000, 1010);
-
-	/* poll with timeout to check if operation successful */
-	cpa = MTL_HFDSSCS_CPA_MASK;
-	dsphfdsscs = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_HFDSSCS);
-	ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, MTL_HFDSSCS, dsphfdsscs,
-					    (dsphfdsscs & cpa) == 0, HDA_DSP_REG_POLL_INTERVAL_US,
-					HDA_DSP_RESET_TIMEOUT_US);
-	if (ret < 0)
-		dev_err(sdev->dev, "failed to disable DSP subsystem\n");
-
-	/* reset ref counts for all cores */
-	for (i = 0; i < chip->cores_num; i++)
-		sdev->dsp_core_ref_count[i] = 0;
-
-	/* TODO: need to reset controller? */
-
-	/* display codec can be powered off after link reset */
-	hda_codec_i915_display_power(sdev, false);
-
-	return 0;
-}
-
-static int mtl_dsp_suspend(struct snd_sof_dev *sdev, u32 target_state)
-{
-	const struct sof_dsp_power_state target_dsp_state = {
-		.state = target_state,
-		.substate = target_state == SOF_DSP_PM_D0 ?
-				SOF_HDA_DSP_PM_D0I3 : 0,
-	};
-	int ret;
-
-	ret = mtl_suspend(sdev, false);
-	if (ret < 0)
-		return ret;
-
-	return snd_sof_dsp_set_power_state(sdev, &target_dsp_state);
-}
-
-static int mtl_dsp_runtime_suspend(struct snd_sof_dev *sdev)
-{
-	const struct sof_dsp_power_state target_state = {
-		.state = SOF_DSP_PM_D3,
-	};
-	int ret;
-
-	ret = mtl_suspend(sdev, true);
-	if (ret < 0)
-		return ret;
-
-	return snd_sof_dsp_set_power_state(sdev, &target_state);
-}
-
-static int mtl_resume(struct snd_sof_dev *sdev, bool runtime_resume)
-{
-#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
-	struct hdac_bus *bus = sof_to_bus(sdev);
-	struct hdac_ext_link *hlink = NULL;
-#endif
-
-	/* display codec must be powered before link reset */
-	hda_codec_i915_display_power(sdev, true);
-
-#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
-	/* check jack status */
-	if (runtime_resume) {
-		hda_codec_jack_wake_enable(sdev, false);
-		if (sdev->system_suspend_target == SOF_SUSPEND_NONE)
-			hda_codec_jack_check(sdev);
-	}
-
-	/* turn off the links that were off before suspend */
-	list_for_each_entry(hlink, &bus->hlink_list, list) {
-		if (!hlink->ref_count)
-			snd_hdac_ext_bus_link_power_down(hlink);
-	}
-
-	/* check dma status and clean up CORB/RIRB buffers */
-	if (!bus->cmd_dma_state)
-		snd_hdac_bus_stop_cmd_io(bus);
-#endif
-
-	return 0;
-}
-
-static int mtl_dsp_resume(struct snd_sof_dev *sdev)
-{
-	const struct sof_dsp_power_state target_state = {
-		.state = SOF_DSP_PM_D0,
-		.substate = SOF_HDA_DSP_PM_D0I0,
-	};
-	int ret;
-
-	ret = mtl_resume(sdev, false);
-	if (ret < 0)
-		return ret;
-
-	return snd_sof_dsp_set_power_state(sdev, &target_state);
-}
-
-static int mtl_dsp_runtime_resume(struct snd_sof_dev *sdev)
-{
-	const struct sof_dsp_power_state target_state = {
-		.state = SOF_DSP_PM_D0,
-	};
-	int ret;
-
-	ret = mtl_resume(sdev, true);
-	if (ret < 0)
-		return ret;
-
-	return snd_sof_dsp_set_power_state(sdev, &target_state);
-}
-
 static void mtl_ipc_dump(struct snd_sof_dev *sdev)
 {
-	u32 hipcctl;
-	u32 hipcida;
-	u32 hipctdr;
+	u32 hipcidr, hipcidd, hipcida, hipctdr, hipctdd, hipctda, hipcctl;
 
-	/* read IPC status */
+	hipcidr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDR);
+	hipcidd = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDDY);
 	hipcida = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXIDA);
-	hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXCTL);
 	hipctdr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDR);
+	hipctdd = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDDY);
+	hipctda = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXTDA);
+	hipcctl = snd_sof_dsp_read(sdev, HDA_DSP_BAR, MTL_DSP_REG_HFIPCXCTL);
 
-	/* dump the IPC regs */
-	/* TODO: parse the raw msg */
 	dev_err(sdev->dev,
-		"error: host status 0x%8.8x dsp status 0x%8.8x mask 0x%8.8x\n",
-		hipcida, hipctdr, hipcctl);
+		"Host IPC initiator: %#x|%#x|%#x, target: %#x|%#x|%#x, ctl: %#x\n",
+		hipcidr, hipcidd, hipcida, hipctdr, hipctdd, hipctda, hipcctl);
+}
+
+static int mtl_dsp_disable_interrupts(struct snd_sof_dev *sdev)
+{
+	mtl_disable_ipc_interrupts(sdev);
+	return mtl_disable_interrupts(sdev);
 }
 
 /* Meteorlake ops */
@@ -751,12 +632,6 @@ int sof_mtl_ops_init(struct snd_sof_dev *sdev)
 	/* dsp core get/put */
 	/* TODO: add core_get and core_put */
 
-	/* PM */
-	sof_mtl_ops.suspend = mtl_dsp_suspend;
-	sof_mtl_ops.resume = mtl_dsp_resume;
-	sof_mtl_ops.runtime_suspend = mtl_dsp_runtime_suspend;
-	sof_mtl_ops.runtime_resume = mtl_dsp_runtime_resume;
-
 	sdev->private = devm_kzalloc(sdev->dev, sizeof(struct sof_ipc4_fw_data), GFP_KERNEL);
 	if (!sdev->private)
 		return -ENOMEM;
@@ -764,6 +639,8 @@ int sof_mtl_ops_init(struct snd_sof_dev *sdev)
 	ipc4_data = sdev->private;
 	ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET;
 
+	ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_2;
+
 	/* set DAI ops */
 	hda_set_dai_drv_ops(sdev, &sof_mtl_ops);
 
@@ -782,13 +659,15 @@ const struct sof_intel_dsp_desc mtl_chip_info = {
 	.ipc_ctl = MTL_DSP_REG_HFIPCXCTL,
 	.rom_status_reg = MTL_DSP_ROM_STS,
 	.rom_init_timeout	= 300,
-	.ssp_count = ICL_SSP_COUNT,
+	.ssp_count = MTL_SSP_COUNT,
 	.ssp_base_offset = CNL_SSP_BASE_OFFSET,
 	.sdw_shim_base = SDW_SHIM_BASE_ACE,
 	.sdw_alh_base = SDW_ALH_BASE_ACE,
 	.check_sdw_irq = mtl_dsp_check_sdw_irq,
 	.check_ipc_irq = mtl_dsp_check_ipc_irq,
 	.cl_init = mtl_dsp_cl_init,
+	.power_down_dsp = mtl_power_down_dsp,
+	.disable_interrupts = mtl_dsp_disable_interrupts,
 	.hw_ip_version = SOF_INTEL_ACE_1_0,
 };
 EXPORT_SYMBOL_NS(mtl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON);
diff --git a/sound/soc/sof/intel/pci-skl.c b/sound/soc/sof/intel/pci-skl.c
new file mode 100644
index 000000000000..3a99dc444f92
--- /dev/null
+++ b/sound/soc/sof/intel/pci-skl.c
@@ -0,0 +1,91 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018-2022 Intel Corporation. All rights reserved.
+//
+
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <sound/soc-acpi.h>
+#include <sound/soc-acpi-intel-match.h>
+#include <sound/sof.h>
+#include "../ops.h"
+#include "../sof-pci-dev.h"
+
+/* platform specific devices */
+#include "hda.h"
+
+static struct sof_dev_desc skl_desc = {
+	.machines		= snd_soc_acpi_intel_skl_machines,
+	.resindex_lpe_base	= 0,
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= -1,
+	.chip_info = &skl_chip_info,
+	.irqindex_host_ipc	= -1,
+	.ipc_supported_mask	= BIT(SOF_INTEL_IPC4),
+	.ipc_default		= SOF_INTEL_IPC4,
+	.default_fw_path = {
+		[SOF_INTEL_IPC4] = "intel/avs/skl",
+	},
+	.default_tplg_path = {
+		[SOF_INTEL_IPC4] = "intel/avs-tplg",
+	},
+	.default_fw_filename = {
+		[SOF_INTEL_IPC4] = "dsp_basefw.bin",
+	},
+	.nocodec_tplg_filename = "sof-skl-nocodec.tplg",
+	.ops = &sof_skl_ops,
+	.ops_init = sof_skl_ops_init,
+};
+
+static struct sof_dev_desc kbl_desc = {
+	.machines		= snd_soc_acpi_intel_kbl_machines,
+	.resindex_lpe_base	= 0,
+	.resindex_pcicfg_base	= -1,
+	.resindex_imr_base	= -1,
+	.chip_info = &skl_chip_info,
+	.irqindex_host_ipc	= -1,
+	.ipc_supported_mask	= BIT(SOF_INTEL_IPC4),
+	.ipc_default		= SOF_INTEL_IPC4,
+	.default_fw_path = {
+		[SOF_INTEL_IPC4] = "intel/avs/kbl",
+	},
+	.default_tplg_path = {
+		[SOF_INTEL_IPC4] = "intel/avs-tplg",
+	},
+	.default_fw_filename = {
+		[SOF_INTEL_IPC4] = "dsp_basefw.bin",
+	},
+	.nocodec_tplg_filename = "sof-kbl-nocodec.tplg",
+	.ops = &sof_skl_ops,
+	.ops_init = sof_skl_ops_init,
+};
+
+/* PCI IDs */
+static const struct pci_device_id sof_pci_ids[] = {
+	/* Sunrise Point-LP */
+	{ PCI_DEVICE(0x8086, 0x9d70), .driver_data = (unsigned long)&skl_desc},
+	/* KBL */
+	{ PCI_DEVICE(0x8086, 0x9d71), .driver_data = (unsigned long)&kbl_desc},
+	{ 0, }
+};
+MODULE_DEVICE_TABLE(pci, sof_pci_ids);
+
+/* pci_driver definition */
+static struct pci_driver snd_sof_pci_intel_skl_driver = {
+	.name = "sof-audio-pci-intel-skl",
+	.id_table = sof_pci_ids,
+	.probe = hda_pci_intel_probe,
+	.remove = sof_pci_remove,
+	.shutdown = sof_pci_shutdown,
+	.driver = {
+		.pm = &sof_pci_pm,
+	},
+};
+module_pci_driver(snd_sof_pci_intel_skl_driver);
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_IMPORT_NS(SND_SOC_SOF_INTEL_HDA_COMMON);
+MODULE_IMPORT_NS(SND_SOC_SOF_PCI_DEV);
diff --git a/sound/soc/sof/intel/pci-tgl.c b/sound/soc/sof/intel/pci-tgl.c
index ccc44ba3ad94..2d63cc236a68 100644
--- a/sound/soc/sof/intel/pci-tgl.c
+++ b/sound/soc/sof/intel/pci-tgl.c
@@ -159,6 +159,62 @@ static const struct sof_dev_desc adl_desc = {
 	.ops_init = sof_tgl_ops_init,
 };
 
+static const struct sof_dev_desc rpls_desc = {
+	.machines               = snd_soc_acpi_intel_rpl_machines,
+	.alt_machines           = snd_soc_acpi_intel_rpl_sdw_machines,
+	.use_acpi_target_states	= true,
+	.resindex_lpe_base      = 0,
+	.resindex_pcicfg_base   = -1,
+	.resindex_imr_base      = -1,
+	.irqindex_host_ipc      = -1,
+	.chip_info = &adls_chip_info,
+	.ipc_supported_mask	= BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4),
+	.ipc_default		= SOF_IPC,
+	.default_fw_path = {
+		[SOF_IPC] = "intel/sof",
+		[SOF_INTEL_IPC4] = "intel/avs/rpl-s",
+	},
+	.default_tplg_path = {
+		[SOF_IPC] = "intel/sof-tplg",
+		[SOF_INTEL_IPC4] = "intel/avs-tplg",
+	},
+	.default_fw_filename = {
+		[SOF_IPC] = "sof-rpl-s.ri",
+		[SOF_INTEL_IPC4] = "dsp_basefw.bin",
+	},
+	.nocodec_tplg_filename = "sof-rpl-nocodec.tplg",
+	.ops = &sof_tgl_ops,
+	.ops_init = sof_tgl_ops_init,
+};
+
+static const struct sof_dev_desc rpl_desc = {
+	.machines               = snd_soc_acpi_intel_rpl_machines,
+	.alt_machines           = snd_soc_acpi_intel_rpl_sdw_machines,
+	.use_acpi_target_states = true,
+	.resindex_lpe_base      = 0,
+	.resindex_pcicfg_base   = -1,
+	.resindex_imr_base      = -1,
+	.irqindex_host_ipc      = -1,
+	.chip_info = &tgl_chip_info,
+	.ipc_supported_mask	= BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4),
+	.ipc_default		= SOF_IPC,
+	.default_fw_path = {
+		[SOF_IPC] = "intel/sof",
+		[SOF_INTEL_IPC4] = "intel/avs/rpl",
+	},
+	.default_tplg_path = {
+		[SOF_IPC] = "intel/sof-tplg",
+		[SOF_INTEL_IPC4] = "intel/avs-tplg",
+	},
+	.default_fw_filename = {
+		[SOF_IPC] = "sof-rpl.ri",
+		[SOF_INTEL_IPC4] = "dsp_basefw.bin",
+	},
+	.nocodec_tplg_filename = "sof-rpl-nocodec.tplg",
+	.ops = &sof_tgl_ops,
+	.ops_init = sof_tgl_ops_init,
+};
+
 /* PCI IDs */
 static const struct pci_device_id sof_pci_ids[] = {
 	{ PCI_DEVICE(0x8086, 0xa0c8), /* TGL-LP */
@@ -172,19 +228,23 @@ static const struct pci_device_id sof_pci_ids[] = {
 	{ PCI_DEVICE(0x8086, 0x7ad0), /* ADL-S */
 		.driver_data = (unsigned long)&adls_desc},
 	{ PCI_DEVICE(0x8086, 0x7a50), /* RPL-S */
-		.driver_data = (unsigned long)&adls_desc},
+		.driver_data = (unsigned long)&rpls_desc},
 	{ PCI_DEVICE(0x8086, 0x51c8), /* ADL-P */
 		.driver_data = (unsigned long)&adl_desc},
-	{ PCI_DEVICE(0x8086, 0x51cd), /* ADL-P */
-		.driver_data = (unsigned long)&adl_desc},
 	{ PCI_DEVICE(0x8086, 0x51c9), /* ADL-PS */
 		.driver_data = (unsigned long)&adl_desc},
 	{ PCI_DEVICE(0x8086, 0x51ca), /* RPL-P */
-		.driver_data = (unsigned long)&adl_desc},
+		.driver_data = (unsigned long)&rpl_desc},
 	{ PCI_DEVICE(0x8086, 0x51cb), /* RPL-P */
-		.driver_data = (unsigned long)&adl_desc},
+		.driver_data = (unsigned long)&rpl_desc},
 	{ PCI_DEVICE(0x8086, 0x51cc), /* ADL-M */
 		.driver_data = (unsigned long)&adl_desc},
+	{ PCI_DEVICE(0x8086, 0x51cd), /* ADL-P */
+		.driver_data = (unsigned long)&adl_desc},
+	{ PCI_DEVICE(0x8086, 0x51ce), /* RPL-M */
+		.driver_data = (unsigned long)&rpl_desc},
+	{ PCI_DEVICE(0x8086, 0x51cf), /* RPL-PX */
+		.driver_data = (unsigned long)&rpl_desc},
 	{ PCI_DEVICE(0x8086, 0x54c8), /* ADL-N */
 		.driver_data = (unsigned long)&adl_desc},
 	{ 0, }
diff --git a/sound/soc/sof/intel/shim.h b/sound/soc/sof/intel/shim.h
index 638159bee864..3ceba5c39317 100644
--- a/sound/soc/sof/intel/shim.h
+++ b/sound/soc/sof/intel/shim.h
@@ -186,6 +186,8 @@ struct sof_intel_dsp_desc {
 	enum sof_intel_hw_ip_version hw_ip_version;
 	bool (*check_sdw_irq)(struct snd_sof_dev *sdev);
 	bool (*check_ipc_irq)(struct snd_sof_dev *sdev);
+	int (*power_down_dsp)(struct snd_sof_dev *sdev);
+	int (*disable_interrupts)(struct snd_sof_dev *sdev);
 	int (*cl_init)(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot);
 };
 
diff --git a/sound/soc/sof/intel/skl.c b/sound/soc/sof/intel/skl.c
new file mode 100644
index 000000000000..13efdb94d071
--- /dev/null
+++ b/sound/soc/sof/intel/skl.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
+//
+// This file is provided under a dual BSD/GPLv2 license.  When using or
+// redistributing this file, you may do so under either license.
+//
+// Copyright(c) 2018-2022 Intel Corporation. All rights reserved.
+//
+
+/*
+ * Hardware interface for audio DSP on Skylake and Kabylake.
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/firmware.h>
+#include <linux/fs.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/slab.h>
+#include <linux/pci.h>
+#include <linux/pm_runtime.h>
+#include <sound/hdaudio_ext.h>
+#include <sound/pcm_params.h>
+#include <sound/sof.h>
+#include <sound/sof/ext_manifest4.h>
+
+#include "../sof-priv.h"
+#include "../ipc4-priv.h"
+#include "../ops.h"
+#include "hda.h"
+#include "../sof-audio.h"
+
+#define SRAM_MEMORY_WINDOW_BASE 0x8000
+
+static const __maybe_unused struct snd_sof_debugfs_map skl_dsp_debugfs[] = {
+	{"hda", HDA_DSP_HDA_BAR, 0, 0x4000},
+	{"pp", HDA_DSP_PP_BAR,  0, 0x1000},
+	{"dsp", HDA_DSP_BAR,  0, 0x10000},
+};
+
+static int skl_dsp_ipc_get_window_offset(struct snd_sof_dev *sdev, u32 id)
+{
+	return SRAM_MEMORY_WINDOW_BASE + (0x2000 * id);
+}
+
+static int skl_dsp_ipc_get_mailbox_offset(struct snd_sof_dev *sdev)
+{
+	return SRAM_MEMORY_WINDOW_BASE + 0x1000;
+}
+
+/* skylake ops */
+struct snd_sof_dsp_ops sof_skl_ops;
+EXPORT_SYMBOL_NS(sof_skl_ops, SND_SOC_SOF_INTEL_HDA_COMMON);
+
+int sof_skl_ops_init(struct snd_sof_dev *sdev)
+{
+	struct sof_ipc4_fw_data *ipc4_data;
+
+	/* common defaults */
+	memcpy(&sof_skl_ops, &sof_hda_common_ops, sizeof(struct snd_sof_dsp_ops));
+
+	/* probe/remove/shutdown */
+	sof_skl_ops.shutdown	= hda_dsp_shutdown;
+
+	sdev->private = devm_kzalloc(sdev->dev, sizeof(*ipc4_data), GFP_KERNEL);
+	if (!sdev->private)
+		return -ENOMEM;
+
+	ipc4_data = sdev->private;
+	ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET_CAVS_1_5;
+
+	ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_1_5;
+
+	sof_skl_ops.get_window_offset = skl_dsp_ipc_get_window_offset;
+	sof_skl_ops.get_mailbox_offset = skl_dsp_ipc_get_mailbox_offset;
+
+	/* doorbell */
+	sof_skl_ops.irq_thread	= hda_dsp_ipc4_irq_thread;
+
+	/* ipc */
+	sof_skl_ops.send_msg	= hda_dsp_ipc4_send_msg;
+
+	/* set DAI driver ops */
+	hda_set_dai_drv_ops(sdev, &sof_skl_ops);
+
+	/* debug */
+	sof_skl_ops.debug_map	= skl_dsp_debugfs;
+	sof_skl_ops.debug_map_count	= ARRAY_SIZE(skl_dsp_debugfs);
+	sof_skl_ops.ipc_dump	= hda_ipc4_dump;
+
+	/* firmware run */
+	sof_skl_ops.run = hda_dsp_cl_boot_firmware_skl;
+
+	/* pre/post fw run */
+	sof_skl_ops.post_fw_run = hda_dsp_post_fw_run;
+
+	return 0;
+};
+EXPORT_SYMBOL_NS(sof_skl_ops_init, SND_SOC_SOF_INTEL_HDA_COMMON);
+
+const struct sof_intel_dsp_desc skl_chip_info = {
+	.cores_num = 2,
+	.init_core_mask = 1,
+	.host_managed_cores_mask = GENMASK(1, 0),
+	.ipc_req = HDA_DSP_REG_HIPCI,
+	.ipc_req_mask = HDA_DSP_REG_HIPCI_BUSY,
+	.ipc_ack = HDA_DSP_REG_HIPCIE,
+	.ipc_ack_mask = HDA_DSP_REG_HIPCIE_DONE,
+	.ipc_ctl = HDA_DSP_REG_HIPCCTL,
+	.rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS_SKL,
+	.rom_init_timeout	= 300,
+	.check_ipc_irq	= hda_dsp_check_ipc_irq,
+	.power_down_dsp = hda_power_down_dsp,
+	.disable_interrupts = hda_dsp_disable_interrupts,
+	.hw_ip_version = SOF_INTEL_CAVS_1_5,
+};
+EXPORT_SYMBOL_NS(skl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON);
diff --git a/sound/soc/sof/intel/tgl.c b/sound/soc/sof/intel/tgl.c
index 6dfb4786c782..9ae2890e9dac 100644
--- a/sound/soc/sof/intel/tgl.c
+++ b/sound/soc/sof/intel/tgl.c
@@ -68,6 +68,9 @@ int sof_tgl_ops_init(struct snd_sof_dev *sdev)
 
 		/* ipc */
 		sof_tgl_ops.send_msg	= cnl_ipc_send_msg;
+
+		/* debug */
+		sof_tgl_ops.ipc_dump	= cnl_ipc_dump;
 	}
 
 	if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) {
@@ -80,11 +83,16 @@ int sof_tgl_ops_init(struct snd_sof_dev *sdev)
 		ipc4_data = sdev->private;
 		ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET;
 
+		ipc4_data->mtrace_type = SOF_IPC4_MTRACE_INTEL_CAVS_2;
+
 		/* doorbell */
 		sof_tgl_ops.irq_thread	= cnl_ipc4_irq_thread;
 
 		/* ipc */
 		sof_tgl_ops.send_msg	= cnl_ipc4_send_msg;
+
+		/* debug */
+		sof_tgl_ops.ipc_dump	= cnl_ipc4_dump;
 	}
 
 	/* set DAI driver ops */
@@ -93,7 +101,6 @@ int sof_tgl_ops_init(struct snd_sof_dev *sdev)
 	/* debug */
 	sof_tgl_ops.debug_map	= tgl_dsp_debugfs;
 	sof_tgl_ops.debug_map_count	= ARRAY_SIZE(tgl_dsp_debugfs);
-	sof_tgl_ops.ipc_dump	= cnl_ipc_dump;
 
 	/* pre/post fw run */
 	sof_tgl_ops.post_fw_run = hda_dsp_post_fw_run;
@@ -121,13 +128,15 @@ const struct sof_intel_dsp_desc tgl_chip_info = {
 	.ipc_ctl = CNL_DSP_REG_HIPCCTL,
 	.rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS,
 	.rom_init_timeout	= 300,
-	.ssp_count = ICL_SSP_COUNT,
+	.ssp_count = TGL_SSP_COUNT,
 	.ssp_base_offset = CNL_SSP_BASE_OFFSET,
 	.sdw_shim_base = SDW_SHIM_BASE,
 	.sdw_alh_base = SDW_ALH_BASE,
 	.check_sdw_irq	= hda_common_check_sdw_irq,
 	.check_ipc_irq	= hda_dsp_check_ipc_irq,
 	.cl_init = cl_dsp_init,
+	.power_down_dsp = hda_power_down_dsp,
+	.disable_interrupts = hda_dsp_disable_interrupts,
 	.hw_ip_version = SOF_INTEL_CAVS_2_5,
 };
 EXPORT_SYMBOL_NS(tgl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON);
@@ -144,13 +153,15 @@ const struct sof_intel_dsp_desc tglh_chip_info = {
 	.ipc_ctl = CNL_DSP_REG_HIPCCTL,
 	.rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS,
 	.rom_init_timeout	= 300,
-	.ssp_count = ICL_SSP_COUNT,
+	.ssp_count = TGL_SSP_COUNT,
 	.ssp_base_offset = CNL_SSP_BASE_OFFSET,
 	.sdw_shim_base = SDW_SHIM_BASE,
 	.sdw_alh_base = SDW_ALH_BASE,
 	.check_sdw_irq	= hda_common_check_sdw_irq,
 	.check_ipc_irq	= hda_dsp_check_ipc_irq,
 	.cl_init = cl_dsp_init,
+	.power_down_dsp = hda_power_down_dsp,
+	.disable_interrupts = hda_dsp_disable_interrupts,
 	.hw_ip_version = SOF_INTEL_CAVS_2_5,
 };
 EXPORT_SYMBOL_NS(tglh_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON);
@@ -167,13 +178,15 @@ const struct sof_intel_dsp_desc ehl_chip_info = {
 	.ipc_ctl = CNL_DSP_REG_HIPCCTL,
 	.rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS,
 	.rom_init_timeout	= 300,
-	.ssp_count = ICL_SSP_COUNT,
+	.ssp_count = TGL_SSP_COUNT,
 	.ssp_base_offset = CNL_SSP_BASE_OFFSET,
 	.sdw_shim_base = SDW_SHIM_BASE,
 	.sdw_alh_base = SDW_ALH_BASE,
 	.check_sdw_irq	= hda_common_check_sdw_irq,
 	.check_ipc_irq	= hda_dsp_check_ipc_irq,
 	.cl_init = cl_dsp_init,
+	.power_down_dsp = hda_power_down_dsp,
+	.disable_interrupts = hda_dsp_disable_interrupts,
 	.hw_ip_version = SOF_INTEL_CAVS_2_5,
 };
 EXPORT_SYMBOL_NS(ehl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON);
@@ -190,13 +203,15 @@ const struct sof_intel_dsp_desc adls_chip_info = {
 	.ipc_ctl = CNL_DSP_REG_HIPCCTL,
 	.rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS,
 	.rom_init_timeout	= 300,
-	.ssp_count = ICL_SSP_COUNT,
+	.ssp_count = TGL_SSP_COUNT,
 	.ssp_base_offset = CNL_SSP_BASE_OFFSET,
 	.sdw_shim_base = SDW_SHIM_BASE,
 	.sdw_alh_base = SDW_ALH_BASE,
 	.check_sdw_irq	= hda_common_check_sdw_irq,
 	.check_ipc_irq	= hda_dsp_check_ipc_irq,
 	.cl_init = cl_dsp_init,
+	.power_down_dsp = hda_power_down_dsp,
+	.disable_interrupts = hda_dsp_disable_interrupts,
 	.hw_ip_version = SOF_INTEL_CAVS_2_5,
 };
 EXPORT_SYMBOL_NS(adls_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON);
diff --git a/sound/soc/sof/ipc3-pcm.c b/sound/soc/sof/ipc3-pcm.c
index 9c6a84bdeca7..dad57bef38f6 100644
--- a/sound/soc/sof/ipc3-pcm.c
+++ b/sound/soc/sof/ipc3-pcm.c
@@ -346,6 +346,15 @@ static int sof_ipc3_pcm_dai_link_fixup(struct snd_soc_pcm_runtime *rtd,
 		dev_dbg(component->dev, "AMD_SP channels_min: %d channels_max: %d\n",
 			channels->min, channels->max);
 		break;
+	case SOF_DAI_AMD_HS:
+		rate->min = private->dai_config->acphs.fsync_rate;
+		rate->max = private->dai_config->acphs.fsync_rate;
+		channels->min = private->dai_config->acphs.tdm_slots;
+		channels->max = private->dai_config->acphs.tdm_slots;
+
+		dev_dbg(component->dev,
+			"AMD_HS channel_max: %d rate_max: %d\n", channels->max, rate->max);
+		break;
 	case SOF_DAI_AMD_DMIC:
 		rate->min = private->dai_config->acpdmic.pdm_rate;
 		rate->max = private->dai_config->acpdmic.pdm_rate;
diff --git a/sound/soc/sof/ipc3-topology.c b/sound/soc/sof/ipc3-topology.c
index 65923e7a5976..c148715aa0f9 100644
--- a/sound/soc/sof/ipc3-topology.c
+++ b/sound/soc/sof/ipc3-topology.c
@@ -1217,6 +1217,36 @@ static int sof_link_acp_sp_load(struct snd_soc_component *scomp, struct snd_sof_
 	return 0;
 }
 
+static int sof_link_acp_hs_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink,
+				struct sof_ipc_dai_config *config, struct snd_sof_dai *dai)
+{
+	struct snd_soc_tplg_hw_config *hw_config = slink->hw_configs;
+	struct sof_dai_private_data *private = dai->private;
+	u32 size = sizeof(*config);
+
+	/* Configures the DAI hardware format and inverted clocks */
+	sof_dai_set_format(hw_config, config);
+
+	/* init IPC */
+	memset(&config->acphs, 0, sizeof(config->acphs));
+	config->hdr.size = size;
+
+	config->acphs.fsync_rate = le32_to_cpu(hw_config->fsync_rate);
+	config->acphs.tdm_slots = le32_to_cpu(hw_config->tdm_slots);
+
+	dev_info(scomp->dev, "ACP_HS config ACP%d channel %d rate %d\n",
+		 config->dai_index, config->acphs.tdm_slots,
+		 config->acphs.fsync_rate);
+
+	dai->number_configs = 1;
+	dai->current_config = 0;
+	private->dai_config = kmemdup(config, size, GFP_KERNEL);
+	if (!private->dai_config)
+		return -ENOMEM;
+
+	return 0;
+}
+
 static int sof_link_afe_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink,
 			     struct sof_ipc_dai_config *config, struct snd_sof_dai *dai)
 {
@@ -1249,6 +1279,7 @@ static int sof_link_afe_load(struct snd_soc_component *scomp, struct snd_sof_dai
 static int sof_link_ssp_load(struct snd_soc_component *scomp, struct snd_sof_dai_link *slink,
 			     struct sof_ipc_dai_config *config, struct snd_sof_dai *dai)
 {
+	struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp);
 	struct snd_soc_tplg_hw_config *hw_config = slink->hw_configs;
 	struct sof_dai_private_data *private = dai->private;
 	u32 size = sizeof(*config);
@@ -1273,6 +1304,12 @@ static int sof_link_ssp_load(struct snd_soc_component *scomp, struct snd_sof_dai
 
 		config[i].hdr.size = size;
 
+		if (sdev->mclk_id_override) {
+			dev_dbg(scomp->dev, "tplg: overriding topology mclk_id %d by quirk %d\n",
+				config[i].ssp.mclk_id, sdev->mclk_id_quirk);
+			config[i].ssp.mclk_id = sdev->mclk_id_quirk;
+		}
+
 		/* copy differentiating hw configs to ipc structs */
 		config[i].ssp.mclk_rate = le32_to_cpu(hw_config[i].mclk_rate);
 		config[i].ssp.bclk_rate = le32_to_cpu(hw_config[i].bclk_rate);
@@ -1510,6 +1547,9 @@ static int sof_ipc3_widget_setup_comp_dai(struct snd_sof_widget *swidget)
 		case SOF_DAI_AMD_SP:
 			ret = sof_link_acp_sp_load(scomp, slink, config, dai);
 			break;
+		case SOF_DAI_AMD_HS:
+			ret = sof_link_acp_hs_load(scomp, slink, config, dai);
+			break;
 		case SOF_DAI_AMD_DMIC:
 			ret = sof_link_acp_dmic_load(scomp, slink, config, dai);
 			break;
diff --git a/sound/soc/sof/ipc3.c b/sound/soc/sof/ipc3.c
index 82fa320253be..b28af3a48b70 100644
--- a/sound/soc/sof/ipc3.c
+++ b/sound/soc/sof/ipc3.c
@@ -9,6 +9,7 @@
 
 #include <sound/sof/stream.h>
 #include <sound/sof/control.h>
+#include <trace/events/sof.h>
 #include "sof-priv.h"
 #include "sof-audio.h"
 #include "ipc3-priv.h"
@@ -23,7 +24,7 @@ static void ipc3_log_header(struct device *dev, u8 *text, u32 cmd)
 	u8 *str2 = NULL;
 	u32 glb;
 	u32 type;
-	bool vdbg = false;
+	bool is_sof_ipc_stream_position = false;
 
 	glb = cmd & SOF_GLB_TYPE_MASK;
 	type = cmd & SOF_CMD_TYPE_MASK;
@@ -118,7 +119,7 @@ static void ipc3_log_header(struct device *dev, u8 *text, u32 cmd)
 		case SOF_IPC_STREAM_TRIG_XRUN:
 			str2 = "TRIG_XRUN"; break;
 		case SOF_IPC_STREAM_POSITION:
-			vdbg = true;
+			is_sof_ipc_stream_position = true;
 			str2 = "POSITION"; break;
 		case SOF_IPC_STREAM_VORBIS_PARAMS:
 			str2 = "VORBIS_PARAMS"; break;
@@ -206,8 +207,8 @@ static void ipc3_log_header(struct device *dev, u8 *text, u32 cmd)
 	}
 
 	if (str2) {
-		if (vdbg)
-			dev_vdbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2);
+		if (is_sof_ipc_stream_position)
+			trace_sof_stream_position_ipc_rx(dev);
 		else
 			dev_dbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2);
 	} else {
@@ -852,8 +853,7 @@ static void ipc3_period_elapsed(struct snd_sof_dev *sdev, u32 msg_id)
 		return;
 	}
 
-	dev_vdbg(sdev->dev, "posn : host 0x%llx dai 0x%llx wall 0x%llx\n",
-		 posn.host_posn, posn.dai_posn, posn.wallclock);
+	trace_sof_ipc3_period_elapsed_position(sdev, &posn);
 
 	memcpy(&stream->posn, &posn, sizeof(posn));
 
diff --git a/sound/soc/sof/ipc4-loader.c b/sound/soc/sof/ipc4-loader.c
index 9fadae8fd011..e635ae515fa9 100644
--- a/sound/soc/sof/ipc4-loader.c
+++ b/sound/soc/sof/ipc4-loader.c
@@ -8,6 +8,7 @@
 #include <linux/firmware.h>
 #include <sound/sof/ext_manifest4.h>
 #include <sound/sof/ipc4/header.h>
+#include <trace/events/sof.h>
 #include "ipc4-priv.h"
 #include "sof-audio.h"
 #include "sof-priv.h"
@@ -40,6 +41,17 @@ static size_t sof_ipc4_fw_parse_ext_man(struct snd_sof_dev *sdev)
 
 	ext_man_hdr = (struct sof_ext_manifest4_hdr *)fw->data;
 
+	/*
+	 * At the start of the firmware image we must have an extended manifest.
+	 * Verify that the magic number is correct.
+	 */
+	if (ext_man_hdr->id != SOF_EXT_MAN4_MAGIC_NUMBER) {
+		dev_err(sdev->dev,
+			"Unexpected extended manifest magic number: %#x\n",
+			ext_man_hdr->id);
+		return -EINVAL;
+	}
+
 	fw_hdr_offset = ipc4_data->manifest_fw_hdr_offset;
 	if (!fw_hdr_offset)
 		return -EINVAL;
@@ -146,6 +158,7 @@ static int sof_ipc4_validate_firmware(struct snd_sof_dev *sdev)
 
 static int sof_ipc4_query_fw_configuration(struct snd_sof_dev *sdev)
 {
+	struct sof_ipc4_fw_data *ipc4_data = sdev->private;
 	const struct sof_ipc_ops *iops = sdev->ipc->ops;
 	struct sof_ipc4_fw_version *fw_ver;
 	struct sof_ipc4_tuple *tuple;
@@ -182,13 +195,14 @@ static int sof_ipc4_query_fw_configuration(struct snd_sof_dev *sdev)
 				 fw_ver->build);
 			break;
 		case SOF_IPC4_FW_CFG_DL_MAILBOX_BYTES:
-			dev_vdbg(sdev->dev, "DL mailbox size: %u\n", *tuple->value);
+			trace_sof_ipc4_fw_config(sdev, "DL mailbox size", *tuple->value);
 			break;
 		case SOF_IPC4_FW_CFG_UL_MAILBOX_BYTES:
-			dev_vdbg(sdev->dev, "UL mailbox size: %u\n", *tuple->value);
+			trace_sof_ipc4_fw_config(sdev, "UL mailbox size", *tuple->value);
 			break;
 		case SOF_IPC4_FW_CFG_TRACE_LOG_BYTES:
-			dev_vdbg(sdev->dev, "Trace log size: %u\n", *tuple->value);
+			trace_sof_ipc4_fw_config(sdev, "Trace log size", *tuple->value);
+			ipc4_data->mtrace_log_bytes = *tuple->value;
 			break;
 		default:
 			break;
diff --git a/sound/soc/sof/ipc4-mtrace.c b/sound/soc/sof/ipc4-mtrace.c
new file mode 100644
index 000000000000..9c7080041d08
--- /dev/null
+++ b/sound/soc/sof/ipc4-mtrace.c
@@ -0,0 +1,643 @@
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// Copyright(c) 2022 Intel Corporation. All rights reserved.
+
+#include <linux/debugfs.h>
+#include <linux/sched/signal.h>
+#include <sound/sof/ipc4/header.h>
+#include "sof-priv.h"
+#include "ipc4-priv.h"
+
+/*
+ * debug info window is organized in 16 (equal sized) pages:
+ *
+ *	------------------------
+ *	| Page0  - descriptors |
+ *	------------------------
+ *	| Page1  - slot0       |
+ *	------------------------
+ *	| Page2  - slot1       |
+ *	------------------------
+ *	|         ...          |
+ *	------------------------
+ *	| Page14  - slot13     |
+ *	------------------------
+ *	| Page15  - slot14     |
+ *	------------------------
+ *
+ * The slot size == page size
+ *
+ * The first page contains descriptors for the remaining 15 cores
+ * The slot descriptor is:
+ * u32 res_id;
+ * u32 type;
+ * u32 vma;
+ *
+ * Log buffer slots have the following layout:
+ * u32 host_read_ptr;
+ * u32 dsp_write_ptr;
+ * u8 buffer[];
+ *
+ * The two pointers are offsets within the buffer.
+ */
+
+#define SOF_MTRACE_DESCRIPTOR_SIZE		12 /* 3 x u32 */
+
+#define FW_EPOCH_DELTA				11644473600LL
+
+#define INVALID_SLOT_OFFSET			0xffffffff
+#define MAX_ALLOWED_LIBRARIES			16
+#define MAX_MTRACE_SLOTS			15
+
+#define SOF_MTRACE_PAGE_SIZE			0x1000
+#define SOF_MTRACE_SLOT_SIZE			SOF_MTRACE_PAGE_SIZE
+
+/* debug log slot types */
+#define SOF_MTRACE_SLOT_UNUSED			0x00000000
+#define SOF_MTRACE_SLOT_CRITICAL_LOG		0x54524300 /* byte 0: core ID */
+#define SOF_MTRACE_SLOT_DEBUG_LOG		0x474f4c00 /* byte 0: core ID */
+#define SOF_MTRACE_SLOT_GDB_STUB		0x42444700
+#define SOF_MTRACE_SLOT_TELEMETRY		0x4c455400
+#define SOF_MTRACE_SLOT_BROKEN			0x44414544
+ /* for debug and critical types */
+#define SOF_MTRACE_SLOT_CORE_MASK		GENMASK(7, 0)
+#define SOF_MTRACE_SLOT_TYPE_MASK		GENMASK(31, 8)
+
+#define DEFAULT_AGING_TIMER_PERIOD_MS		0x100
+#define DEFAULT_FIFO_FULL_TIMER_PERIOD_MS	0x1000
+
+/* ipc4 log level and source definitions for logs_priorities_mask */
+#define SOF_MTRACE_LOG_LEVEL_CRITICAL		BIT(0)
+#define SOF_MTRACE_LOG_LEVEL_ERROR		BIT(1)
+#define SOF_MTRACE_LOG_LEVEL_WARNING		BIT(2)
+#define SOF_MTRACE_LOG_LEVEL_INFO		BIT(3)
+#define SOF_MTRACE_LOG_LEVEL_VERBOSE		BIT(4)
+#define SOF_MTRACE_LOG_SOURCE_INFRA		BIT(5) /* log source 0 */
+#define SOF_MTRACE_LOG_SOURCE_HAL		BIT(6)
+#define SOF_MTRACE_LOG_SOURCE_MODULE		BIT(7)
+#define SOF_MTRACE_LOG_SOURCE_AUDIO		BIT(8)
+#define SOF_MTRACE_LOG_SOURCE_SCHEDULER		BIT(9)
+#define SOF_MTRACE_LOG_SOURCE_ULP_INFRA		BIT(10)
+#define SOF_MTRACE_LOG_SOURCE_ULP_MODULE	BIT(11)
+#define SOF_MTRACE_LOG_SOURCE_VISION		BIT(12) /* log source 7 */
+#define DEFAULT_LOGS_PRIORITIES_MASK	(SOF_MTRACE_LOG_LEVEL_CRITICAL | \
+					 SOF_MTRACE_LOG_LEVEL_ERROR |	 \
+					 SOF_MTRACE_LOG_LEVEL_WARNING |	 \
+					 SOF_MTRACE_LOG_LEVEL_INFO |	 \
+					 SOF_MTRACE_LOG_SOURCE_INFRA |	 \
+					 SOF_MTRACE_LOG_SOURCE_HAL |	 \
+					 SOF_MTRACE_LOG_SOURCE_MODULE |	 \
+					 SOF_MTRACE_LOG_SOURCE_AUDIO)
+
+struct sof_log_state_info {
+	u32 aging_timer_period;
+	u32 fifo_full_timer_period;
+	u32 enable;
+	u32 logs_priorities_mask[MAX_ALLOWED_LIBRARIES];
+} __packed;
+
+enum sof_mtrace_state {
+	SOF_MTRACE_DISABLED,
+	SOF_MTRACE_INITIALIZING,
+	SOF_MTRACE_ENABLED,
+};
+
+struct sof_mtrace_core_data {
+	struct snd_sof_dev *sdev;
+
+	int id;
+	u32 slot_offset;
+	void *log_buffer;
+	u32 host_read_ptr;
+	u32 dsp_write_ptr;
+	/* pos update IPC arrived before the slot offset is known, queried */
+	bool delayed_pos_update;
+	wait_queue_head_t trace_sleep;
+};
+
+struct sof_mtrace_priv {
+	struct snd_sof_dev *sdev;
+	enum sof_mtrace_state mtrace_state;
+	struct sof_log_state_info state_info;
+
+	struct sof_mtrace_core_data cores[];
+};
+
+static int sof_ipc4_mtrace_dfs_open(struct inode *inode, struct file *file)
+{
+	struct sof_mtrace_core_data *core_data = inode->i_private;
+	int ret;
+
+	ret = debugfs_file_get(file->f_path.dentry);
+	if (unlikely(ret))
+		return ret;
+
+	core_data->log_buffer = kmalloc(SOF_MTRACE_SLOT_SIZE, GFP_KERNEL);
+	if (!core_data->log_buffer) {
+		debugfs_file_put(file->f_path.dentry);
+		return -ENOMEM;
+	}
+
+	ret = simple_open(inode, file);
+	if (ret) {
+		kfree(core_data->log_buffer);
+		debugfs_file_put(file->f_path.dentry);
+	}
+
+	return ret;
+}
+
+static bool sof_wait_mtrace_avail(struct sof_mtrace_core_data *core_data)
+{
+	wait_queue_entry_t wait;
+
+	/* data immediately available */
+	if (core_data->host_read_ptr != core_data->dsp_write_ptr)
+		return true;
+
+	/* wait for available trace data from FW */
+	init_waitqueue_entry(&wait, current);
+	set_current_state(TASK_INTERRUPTIBLE);
+	add_wait_queue(&core_data->trace_sleep, &wait);
+
+	if (!signal_pending(current)) {
+		/* set timeout to max value, no error code */
+		schedule_timeout(MAX_SCHEDULE_TIMEOUT);
+	}
+	remove_wait_queue(&core_data->trace_sleep, &wait);
+
+	if (core_data->host_read_ptr != core_data->dsp_write_ptr)
+		return true;
+
+	return false;
+}
+
+static ssize_t sof_ipc4_mtrace_dfs_read(struct file *file, char __user *buffer,
+					size_t count, loff_t *ppos)
+{
+	struct sof_mtrace_core_data *core_data = file->private_data;
+	u32 log_buffer_offset, log_buffer_size, read_ptr, write_ptr;
+	struct snd_sof_dev *sdev = core_data->sdev;
+	struct sof_mtrace_priv *priv = sdev->fw_trace_data;
+	void *log_buffer = core_data->log_buffer;
+	loff_t lpos = *ppos;
+	u32 avail;
+	int ret;
+
+	/* check pos and count */
+	if (lpos < 0)
+		return -EINVAL;
+	if (!count || count < sizeof(avail))
+		return 0;
+
+	/* get available count based on current host offset */
+	if (!sof_wait_mtrace_avail(core_data)) {
+		/* No data available */
+		avail = 0;
+		if (copy_to_user(buffer, &avail, sizeof(avail)))
+			return -EFAULT;
+
+		return 0;
+	}
+
+	if (core_data->slot_offset == INVALID_SLOT_OFFSET)
+		return 0;
+
+	/* The log data buffer starts after the two pointer in the slot */
+	log_buffer_offset =  core_data->slot_offset + (sizeof(u32) * 2);
+	/* The log data size excludes the pointers */
+	log_buffer_size = SOF_MTRACE_SLOT_SIZE - (sizeof(u32) * 2);
+
+	read_ptr = core_data->host_read_ptr;
+	write_ptr = core_data->dsp_write_ptr;
+
+	if (read_ptr < write_ptr)
+		avail = write_ptr - read_ptr;
+	else
+		avail = log_buffer_size - read_ptr + write_ptr;
+
+	if (!avail)
+		return 0;
+
+	if (avail > log_buffer_size)
+		avail = log_buffer_size;
+
+	/* Need space for the initial u32 of the avail */
+	if (avail > count - sizeof(avail))
+		avail = count - sizeof(avail);
+
+	if (sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS))
+		dev_dbg(sdev->dev,
+			"core%d, host read: %#x, dsp write: %#x, avail: %#x\n",
+			core_data->id, read_ptr, write_ptr, avail);
+
+	if (read_ptr < write_ptr) {
+		/* Read data between read pointer and write pointer */
+		sof_mailbox_read(sdev, log_buffer_offset + read_ptr, log_buffer, avail);
+	} else {
+		/* read from read pointer to end of the slot */
+		sof_mailbox_read(sdev, log_buffer_offset + read_ptr, log_buffer,
+				 avail - write_ptr);
+		/* read from slot start to write pointer */
+		if (write_ptr)
+			sof_mailbox_read(sdev, log_buffer_offset,
+					 (u8 *)(log_buffer) + avail - write_ptr,
+					 write_ptr);
+	}
+
+	/* first write the number of bytes we have gathered */
+	ret = copy_to_user(buffer, &avail, sizeof(avail));
+	if (ret)
+		return -EFAULT;
+
+	/* Followed by the data itself */
+	ret = copy_to_user(buffer + sizeof(avail), log_buffer, avail);
+	if (ret)
+		return -EFAULT;
+
+	/* Update the host_read_ptr in the slot for this core */
+	read_ptr += avail;
+	if (read_ptr >= log_buffer_size)
+		read_ptr -= log_buffer_size;
+	sof_mailbox_write(sdev, core_data->slot_offset, &read_ptr, sizeof(read_ptr));
+
+	/* Only update the host_read_ptr if mtrace is enabled */
+	if (priv->mtrace_state != SOF_MTRACE_DISABLED)
+		core_data->host_read_ptr = read_ptr;
+
+	/*
+	 * Ask for a new buffer from user space for the next chunk, not
+	 * streaming due to the heading number of bytes value.
+	 */
+	*ppos += count;
+
+	return count;
+}
+
+static int sof_ipc4_mtrace_dfs_release(struct inode *inode, struct file *file)
+{
+	struct sof_mtrace_core_data *core_data = inode->i_private;
+
+	debugfs_file_put(file->f_path.dentry);
+
+	kfree(core_data->log_buffer);
+
+	return 0;
+}
+
+static const struct file_operations sof_dfs_mtrace_fops = {
+	.open = sof_ipc4_mtrace_dfs_open,
+	.read = sof_ipc4_mtrace_dfs_read,
+	.llseek = default_llseek,
+	.release = sof_ipc4_mtrace_dfs_release,
+
+	.owner = THIS_MODULE,
+};
+
+static ssize_t sof_ipc4_priority_mask_dfs_read(struct file *file, char __user *to,
+					       size_t count, loff_t *ppos)
+{
+	struct sof_mtrace_priv *priv = file->private_data;
+	int i, ret, offset, remaining;
+	char *buf;
+
+	/*
+	 * one entry (14 char + new line = 15):
+	 * " 0: 000001ef"
+	 *
+	 * 16 * 15 + 1 = 241
+	 */
+	buf = kzalloc(241, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	for (i = 0; i < MAX_ALLOWED_LIBRARIES; i++) {
+		offset = strlen(buf);
+		remaining = 241 - offset;
+		snprintf(buf + offset, remaining, "%2d: 0x%08x\n", i,
+			 priv->state_info.logs_priorities_mask[i]);
+	}
+
+	ret = simple_read_from_buffer(to, count, ppos, buf, strlen(buf));
+
+	kfree(buf);
+	return ret;
+}
+
+static ssize_t sof_ipc4_priority_mask_dfs_write(struct file *file,
+						const char __user *from,
+						size_t count, loff_t *ppos)
+{
+	struct sof_mtrace_priv *priv = file->private_data;
+	int id, ret;
+	char *buf;
+	u32 mask;
+
+	/*
+	 * To update Nth mask entry, write:
+	 * "N,0x1234" or "N,1234" to the debugfs file
+	 * The mask will be interpreted as hexadecimal number
+	 */
+	buf = memdup_user_nul(from, count);
+	if (IS_ERR(buf))
+		return PTR_ERR(buf);
+
+	ret = sscanf(buf, "%d,0x%x", &id, &mask);
+	if (ret != 2) {
+		ret = sscanf(buf, "%d,%x", &id, &mask);
+		if (ret != 2) {
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
+	if (id >= MAX_ALLOWED_LIBRARIES) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	priv->state_info.logs_priorities_mask[id] = mask;
+	ret = count;
+
+out:
+	kfree(buf);
+	return ret;
+}
+
+static const struct file_operations sof_dfs_priority_mask_fops = {
+	.open = simple_open,
+	.read = sof_ipc4_priority_mask_dfs_read,
+	.write = sof_ipc4_priority_mask_dfs_write,
+	.llseek = default_llseek,
+
+	.owner = THIS_MODULE,
+};
+
+static int mtrace_debugfs_create(struct snd_sof_dev *sdev)
+{
+	struct sof_mtrace_priv *priv = sdev->fw_trace_data;
+	struct dentry *dfs_root;
+	char dfs_name[100];
+	int i;
+
+	dfs_root = debugfs_create_dir("mtrace", sdev->debugfs_root);
+	if (IS_ERR_OR_NULL(dfs_root))
+		return 0;
+
+	/* Create files for the logging parameters */
+	debugfs_create_u32("aging_timer_period", 0644, dfs_root,
+			   &priv->state_info.aging_timer_period);
+	debugfs_create_u32("fifo_full_timer_period", 0644, dfs_root,
+			   &priv->state_info.fifo_full_timer_period);
+	debugfs_create_file("logs_priorities_mask", 0644, dfs_root, priv,
+			    &sof_dfs_priority_mask_fops);
+
+	/* Separate log files per core */
+	for (i = 0; i < sdev->num_cores; i++) {
+		snprintf(dfs_name, sizeof(dfs_name), "core%d", i);
+		debugfs_create_file(dfs_name, 0444, dfs_root, &priv->cores[i],
+				    &sof_dfs_mtrace_fops);
+	}
+
+	return 0;
+}
+
+static int ipc4_mtrace_enable(struct snd_sof_dev *sdev)
+{
+	struct sof_mtrace_priv *priv = sdev->fw_trace_data;
+	const struct sof_ipc_ops *iops = sdev->ipc->ops;
+	struct sof_ipc4_msg msg;
+	u64 system_time;
+	ktime_t kt;
+	int ret;
+
+	if (priv->mtrace_state != SOF_MTRACE_DISABLED)
+		return 0;
+
+	msg.primary = SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
+	msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
+	msg.primary |= SOF_IPC4_MOD_ID(SOF_IPC4_MOD_INIT_BASEFW_MOD_ID);
+	msg.primary |= SOF_IPC4_MOD_INSTANCE(SOF_IPC4_MOD_INIT_BASEFW_INSTANCE_ID);
+	msg.extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_FW_PARAM_SYSTEM_TIME);
+
+	/* The system time is in usec, UTC, epoch is 1601-01-01 00:00:00 */
+	kt = ktime_add_us(ktime_get_real(), FW_EPOCH_DELTA * USEC_PER_SEC);
+	system_time = ktime_to_us(kt);
+	msg.data_size = sizeof(system_time);
+	msg.data_ptr = &system_time;
+	ret = iops->set_get_data(sdev, &msg, msg.data_size, true);
+	if (ret)
+		return ret;
+
+	msg.extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_FW_PARAM_ENABLE_LOGS);
+
+	priv->state_info.enable = 1;
+
+	msg.data_size = sizeof(priv->state_info);
+	msg.data_ptr = &priv->state_info;
+
+	priv->mtrace_state = SOF_MTRACE_INITIALIZING;
+	ret = iops->set_get_data(sdev, &msg, msg.data_size, true);
+	if (ret) {
+		priv->mtrace_state = SOF_MTRACE_DISABLED;
+		return ret;
+	}
+
+	priv->mtrace_state = SOF_MTRACE_ENABLED;
+
+	return 0;
+}
+
+static void ipc4_mtrace_disable(struct snd_sof_dev *sdev)
+{
+	struct sof_mtrace_priv *priv = sdev->fw_trace_data;
+	const struct sof_ipc_ops *iops = sdev->ipc->ops;
+	struct sof_ipc4_msg msg;
+	int i;
+
+	if (priv->mtrace_state == SOF_MTRACE_DISABLED)
+		return;
+
+	msg.primary = SOF_IPC4_MSG_TARGET(SOF_IPC4_MODULE_MSG);
+	msg.primary |= SOF_IPC4_MSG_DIR(SOF_IPC4_MSG_REQUEST);
+	msg.primary |= SOF_IPC4_MOD_ID(SOF_IPC4_MOD_INIT_BASEFW_MOD_ID);
+	msg.primary |= SOF_IPC4_MOD_INSTANCE(SOF_IPC4_MOD_INIT_BASEFW_INSTANCE_ID);
+	msg.extension = SOF_IPC4_MOD_EXT_MSG_PARAM_ID(SOF_IPC4_FW_PARAM_ENABLE_LOGS);
+
+	priv->state_info.enable = 0;
+
+	msg.data_size = sizeof(priv->state_info);
+	msg.data_ptr = &priv->state_info;
+	iops->set_get_data(sdev, &msg, msg.data_size, true);
+
+	priv->mtrace_state = SOF_MTRACE_DISABLED;
+
+	for (i = 0; i < sdev->num_cores; i++) {
+		struct sof_mtrace_core_data *core_data = &priv->cores[i];
+
+		core_data->host_read_ptr = 0;
+		core_data->dsp_write_ptr = 0;
+		wake_up(&core_data->trace_sleep);
+	}
+}
+
+/*
+ * Each DSP core logs to a dedicated slot.
+ * Parse the slot descriptors at debug_box offset to find the debug log slots
+ * and map them to cores.
+ * There are 15 slots and therefore 15 descriptors to check (MAX_MTRACE_SLOTS)
+ */
+static void sof_mtrace_find_core_slots(struct snd_sof_dev *sdev)
+{
+	struct sof_mtrace_priv *priv = sdev->fw_trace_data;
+	struct sof_mtrace_core_data *core_data;
+	u32 slot_desc_type_offset, type, core;
+	int i;
+
+	for (i = 0; i < MAX_MTRACE_SLOTS; i++) {
+		/* The type is the second u32 in the slot descriptor */
+		slot_desc_type_offset = sdev->debug_box.offset;
+		slot_desc_type_offset += SOF_MTRACE_DESCRIPTOR_SIZE * i + sizeof(u32);
+		sof_mailbox_read(sdev, slot_desc_type_offset, &type, sizeof(type));
+
+		if ((type & SOF_MTRACE_SLOT_TYPE_MASK) == SOF_MTRACE_SLOT_DEBUG_LOG) {
+			core = type & SOF_MTRACE_SLOT_CORE_MASK;
+
+			if (core >= sdev->num_cores) {
+				dev_dbg(sdev->dev, "core%u is invalid for slot%d\n",
+					core, i);
+				continue;
+			}
+
+			core_data = &priv->cores[core];
+			/*
+			 * The area reserved for descriptors have the same size
+			 * as a slot.
+			 * In other words: slot0 starts at
+			 * debug_box + SOF_MTRACE_SLOT_SIZE offset
+			 */
+			core_data->slot_offset = sdev->debug_box.offset;
+			core_data->slot_offset += SOF_MTRACE_SLOT_SIZE * (i + 1);
+			dev_dbg(sdev->dev, "slot%d is used for core%u\n", i, core);
+			if (core_data->delayed_pos_update) {
+				sof_ipc4_mtrace_update_pos(sdev, core);
+				core_data->delayed_pos_update = false;
+			}
+		} else if (type) {
+			dev_dbg(sdev->dev, "slot%d is not a log slot (%#x)\n", i, type);
+		}
+	}
+}
+
+static int ipc4_mtrace_init(struct snd_sof_dev *sdev)
+{
+	struct sof_ipc4_fw_data *ipc4_data = sdev->private;
+	struct sof_mtrace_priv *priv;
+	int i, ret;
+
+	if (sdev->fw_trace_data) {
+		dev_err(sdev->dev, "fw_trace_data has been already allocated\n");
+		return -EBUSY;
+	}
+
+	if (!ipc4_data->mtrace_log_bytes ||
+	    ipc4_data->mtrace_type != SOF_IPC4_MTRACE_INTEL_CAVS_2) {
+		sdev->fw_trace_is_supported = false;
+		return 0;
+	}
+
+	priv = devm_kzalloc(sdev->dev, struct_size(priv, cores, sdev->num_cores),
+			    GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	sdev->fw_trace_data = priv;
+
+	/* Set initial values for mtrace parameters */
+	priv->state_info.aging_timer_period = DEFAULT_AGING_TIMER_PERIOD_MS;
+	priv->state_info.fifo_full_timer_period = DEFAULT_FIFO_FULL_TIMER_PERIOD_MS;
+	/* Only enable basefw logs initially (index 0 is always basefw) */
+	priv->state_info.logs_priorities_mask[0] = DEFAULT_LOGS_PRIORITIES_MASK;
+
+	for (i = 0; i < sdev->num_cores; i++) {
+		struct sof_mtrace_core_data *core_data = &priv->cores[i];
+
+		init_waitqueue_head(&core_data->trace_sleep);
+		core_data->sdev = sdev;
+		core_data->id = i;
+	}
+
+	ret = ipc4_mtrace_enable(sdev);
+	if (ret) {
+		/*
+		 * Mark firmware tracing as not supported and return 0 to not
+		 * block the whole audio stack
+		 */
+		sdev->fw_trace_is_supported = false;
+		dev_dbg(sdev->dev, "initialization failed, fw tracing is disabled\n");
+		return 0;
+	}
+
+	sof_mtrace_find_core_slots(sdev);
+
+	ret = mtrace_debugfs_create(sdev);
+	if (ret)
+		ipc4_mtrace_disable(sdev);
+
+	return ret;
+}
+
+static void ipc4_mtrace_free(struct snd_sof_dev *sdev)
+{
+	ipc4_mtrace_disable(sdev);
+}
+
+int sof_ipc4_mtrace_update_pos(struct snd_sof_dev *sdev, int core)
+{
+	struct sof_mtrace_priv *priv = sdev->fw_trace_data;
+	struct sof_mtrace_core_data *core_data;
+
+	if (!sdev->fw_trace_is_supported ||
+	    priv->mtrace_state == SOF_MTRACE_DISABLED)
+		return 0;
+
+	if (core >= sdev->num_cores)
+		return -EINVAL;
+
+	core_data = &priv->cores[core];
+
+	if (core_data->slot_offset == INVALID_SLOT_OFFSET) {
+		core_data->delayed_pos_update = true;
+		return 0;
+	}
+
+	/* Read out the dsp_write_ptr from the slot for this core */
+	sof_mailbox_read(sdev, core_data->slot_offset + sizeof(u32),
+			 &core_data->dsp_write_ptr, 4);
+	core_data->dsp_write_ptr -= core_data->dsp_write_ptr % 4;
+
+	if (sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS))
+		dev_dbg(sdev->dev, "core%d, host read: %#x, dsp write: %#x",
+			core, core_data->host_read_ptr, core_data->dsp_write_ptr);
+
+	wake_up(&core_data->trace_sleep);
+
+	return 0;
+}
+
+static int ipc4_mtrace_resume(struct snd_sof_dev *sdev)
+{
+	return ipc4_mtrace_enable(sdev);
+}
+
+static void ipc4_mtrace_suspend(struct snd_sof_dev *sdev, pm_message_t pm_state)
+{
+	ipc4_mtrace_disable(sdev);
+}
+
+const struct sof_ipc_fw_tracing_ops ipc4_mtrace_ops = {
+	.init = ipc4_mtrace_init,
+	.free = ipc4_mtrace_free,
+	.suspend = ipc4_mtrace_suspend,
+	.resume = ipc4_mtrace_resume,
+};
diff --git a/sound/soc/sof/ipc4-priv.h b/sound/soc/sof/ipc4-priv.h
index 9492fe1796c2..e3b8484a2f1f 100644
--- a/sound/soc/sof/ipc4-priv.h
+++ b/sound/soc/sof/ipc4-priv.h
@@ -13,18 +13,33 @@
 #include <sound/sof/ext_manifest4.h>
 #include "sof-priv.h"
 
+/* The DSP window indices are fixed */
+#define SOF_IPC4_OUTBOX_WINDOW_IDX	1
+#define SOF_IPC4_DEBUG_WINDOW_IDX	2
+
+enum sof_ipc4_mtrace_type {
+	SOF_IPC4_MTRACE_NOT_AVAILABLE = 0,
+	SOF_IPC4_MTRACE_INTEL_CAVS_1_5,
+	SOF_IPC4_MTRACE_INTEL_CAVS_1_8,
+	SOF_IPC4_MTRACE_INTEL_CAVS_2,
+};
+
 /**
  * struct sof_ipc4_fw_data - IPC4-specific data
  * @manifest_fw_hdr_offset: FW header offset in the manifest
  * @num_fw_modules : Number of modules in base FW
  * @fw_modules: Array of base FW modules
  * @nhlt: NHLT table either from the BIOS or the topology manifest
+ * @mtrace_type: mtrace type supported on the booted platform
+ * @mtrace_log_bytes: log bytes as reported by the firmware via fw_config reply
  */
 struct sof_ipc4_fw_data {
 	u32 manifest_fw_hdr_offset;
 	int num_fw_modules;
 	void *fw_modules;
 	void *nhlt;
+	enum sof_ipc4_mtrace_type mtrace_type;
+	u32 mtrace_log_bytes;
 };
 
 /**
@@ -45,7 +60,8 @@ extern const struct sof_ipc_fw_loader_ops ipc4_loader_ops;
 extern const struct sof_ipc_tplg_ops ipc4_tplg_ops;
 extern const struct sof_ipc_tplg_control_ops tplg_ipc4_control_ops;
 extern const struct sof_ipc_pcm_ops ipc4_pcm_ops;
+extern const struct sof_ipc_fw_tracing_ops ipc4_mtrace_ops;
 
 int sof_ipc4_set_pipeline_state(struct snd_sof_dev *sdev, u32 id, u32 state);
-
+int sof_ipc4_mtrace_update_pos(struct snd_sof_dev *sdev, int core);
 #endif
diff --git a/sound/soc/sof/ipc4-topology.c b/sound/soc/sof/ipc4-topology.c
index 64929dc9af39..a81af5f73a4b 100644
--- a/sound/soc/sof/ipc4-topology.c
+++ b/sound/soc/sof/ipc4-topology.c
@@ -331,7 +331,7 @@ static int sof_ipc4_widget_setup_msg(struct snd_sof_widget *swidget, struct sof_
 	msg->extension = SOF_IPC4_MOD_EXT_PPL_ID(swidget->pipeline_id);
 	msg->extension |= SOF_IPC4_MOD_EXT_CORE_ID(swidget->core);
 
-	type = fw_module->man4_module_entry.type & SOF_IPC4_MODULE_DP ? 1 : 0;
+	type = (fw_module->man4_module_entry.type & SOF_IPC4_MODULE_DP) ? 1 : 0;
 	msg->extension |= SOF_IPC4_MOD_EXT_DOMAIN(type);
 
 	return 0;
@@ -1447,7 +1447,6 @@ static int sof_ipc4_control_setup(struct snd_sof_dev *sdev, struct snd_sof_contr
 
 static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget)
 {
-	struct snd_sof_widget *pipe_widget = swidget->pipe_widget;
 	struct sof_ipc4_pipeline *pipeline;
 	struct sof_ipc4_msg *msg;
 	void *ipc_data = NULL;
@@ -1530,7 +1529,7 @@ static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget
 				swidget->widget->name);
 			return ret;
 		}
-		pipeline = pipe_widget->private;
+
 		msg->primary &= ~SOF_IPC4_MOD_INSTANCE_MASK;
 		msg->primary |= SOF_IPC4_MOD_INSTANCE(swidget->instance_id);
 
@@ -1544,9 +1543,16 @@ static int sof_ipc4_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget
 	msg->data_ptr = ipc_data;
 
 	ret = sof_ipc_tx_message(sdev->ipc, msg, ipc_size, NULL, 0);
-	if (ret < 0)
+	if (ret < 0) {
 		dev_err(sdev->dev, "failed to create module %s\n", swidget->widget->name);
 
+		if (swidget->id != snd_soc_dapm_scheduler) {
+			struct sof_ipc4_fw_module *fw_module = swidget->module_info;
+
+			ida_free(&fw_module->m_ida, swidget->instance_id);
+		}
+	}
+
 	return ret;
 }
 
diff --git a/sound/soc/sof/ipc4.c b/sound/soc/sof/ipc4.c
index 432b812bdf9c..6eaa18e27e5a 100644
--- a/sound/soc/sof/ipc4.c
+++ b/sound/soc/sof/ipc4.c
@@ -205,6 +205,11 @@ static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_ms
 			/* Notification message */
 			u32 notif = SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary);
 
+			/* Do not print log buffer notification if not desired */
+			if (notif == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS &&
+			    !sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS))
+				return;
+
 			if (notif < SOF_IPC4_NOTIFY_TYPE_LAST)
 				str2 = ipc4_dbg_notification_type[notif];
 			if (!str2)
@@ -234,6 +239,13 @@ static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_ms
 static void sof_ipc4_log_header(struct device *dev, u8 *text, struct sof_ipc4_msg *msg,
 				bool data_size_valid)
 {
+	/* Do not print log buffer notification if not desired */
+	if (!sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS) &&
+	    !SOF_IPC4_MSG_IS_MODULE_MSG(msg->primary) &&
+	    SOF_IPC4_MSG_TYPE_GET(msg->primary) == SOF_IPC4_GLB_NOTIFICATION &&
+	    SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary) == SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS)
+		return;
+
 	if (data_size_valid && msg->data_size)
 		dev_dbg(dev, "%s: %#x|%#x [data size: %zu]\n", text,
 			msg->primary, msg->extension, msg->data_size);
@@ -283,6 +295,7 @@ static int ipc4_wait_tx_done(struct snd_sof_ipc *ipc, void *reply_data)
 	if (ret == 0) {
 		dev_err(sdev->dev, "ipc timed out for %#x|%#x\n",
 			ipc4_msg->primary, ipc4_msg->extension);
+		snd_sof_handle_fw_exception(ipc->sdev, "IPC timeout");
 		return -ETIMEDOUT;
 	}
 
@@ -525,7 +538,7 @@ static int ipc4_fw_ready(struct snd_sof_dev *sdev, struct sof_ipc4_msg *ipc4_msg
 		return inbox_offset;
 	}
 	inbox_size = SOF_IPC4_MSG_MAX_SIZE;
-	outbox_offset = snd_sof_dsp_get_window_offset(sdev, 1);
+	outbox_offset = snd_sof_dsp_get_window_offset(sdev, SOF_IPC4_OUTBOX_WINDOW_IDX);
 	outbox_size = SOF_IPC4_MSG_MAX_SIZE;
 
 	sdev->dsp_box.offset = inbox_offset;
@@ -533,10 +546,14 @@ static int ipc4_fw_ready(struct snd_sof_dev *sdev, struct sof_ipc4_msg *ipc4_msg
 	sdev->host_box.offset = outbox_offset;
 	sdev->host_box.size = outbox_size;
 
+	sdev->debug_box.offset = snd_sof_dsp_get_window_offset(sdev,
+							SOF_IPC4_DEBUG_WINDOW_IDX);
+
 	dev_dbg(sdev->dev, "mailbox upstream 0x%x - size 0x%x\n",
 		inbox_offset, inbox_size);
 	dev_dbg(sdev->dev, "mailbox downstream 0x%x - size 0x%x\n",
 		outbox_offset, outbox_size);
+	dev_dbg(sdev->dev, "debug box 0x%x\n", sdev->debug_box.offset);
 
 	return sof_ipc4_init_msg_memory(sdev);
 }
@@ -573,6 +590,9 @@ static void sof_ipc4_rx_msg(struct snd_sof_dev *sdev)
 	case SOF_IPC4_NOTIFY_RESOURCE_EVENT:
 		data_size = sizeof(struct sof_ipc4_notify_resource_data);
 		break;
+	case SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS:
+		sof_ipc4_mtrace_update_pos(sdev, SOF_IPC4_LOG_CORE_GET(ipc4_msg->primary));
+		break;
 	default:
 		dev_dbg(sdev->dev, "Unhandled DSP message: %#x|%#x\n",
 			ipc4_msg->primary, ipc4_msg->extension);
@@ -646,4 +666,5 @@ const struct sof_ipc_ops ipc4_ops = {
 	.fw_loader = &ipc4_loader_ops,
 	.tplg = &ipc4_tplg_ops,
 	.pcm = &ipc4_pcm_ops,
+	.fw_tracing = &ipc4_mtrace_ops,
 };
diff --git a/sound/soc/sof/mediatek/mt8186/mt8186.c b/sound/soc/sof/mediatek/mt8186/mt8186.c
index e006532caf2f..181189e00e02 100644
--- a/sound/soc/sof/mediatek/mt8186/mt8186.c
+++ b/sound/soc/sof/mediatek/mt8186/mt8186.c
@@ -460,14 +460,79 @@ static int mt8186_get_bar_index(struct snd_sof_dev *sdev, u32 type)
 	return type;
 }
 
-static int mt8186_ipc_msg_data(struct snd_sof_dev *sdev,
-			       struct snd_pcm_substream *substream,
-			       void *p, size_t sz)
+static int mt8186_pcm_hw_params(struct snd_sof_dev *sdev,
+				struct snd_pcm_substream *substream,
+				struct snd_pcm_hw_params *params,
+				struct snd_sof_platform_stream_params *platform_params)
 {
-	sof_mailbox_read(sdev, sdev->dsp_box.offset, p, sz);
+	platform_params->cont_update_posn = 1;
+
 	return 0;
 }
 
+static snd_pcm_uframes_t mt8186_pcm_pointer(struct snd_sof_dev *sdev,
+					    struct snd_pcm_substream *substream)
+{
+	int ret;
+	snd_pcm_uframes_t pos;
+	struct snd_sof_pcm *spcm;
+	struct sof_ipc_stream_posn posn;
+	struct snd_sof_pcm_stream *stream;
+	struct snd_soc_component *scomp = sdev->component;
+	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
+
+	spcm = snd_sof_find_spcm_dai(scomp, rtd);
+	if (!spcm) {
+		dev_warn_ratelimited(sdev->dev, "warn: can't find PCM with DAI ID %d\n",
+				     rtd->dai_link->id);
+		return 0;
+	}
+
+	stream = &spcm->stream[substream->stream];
+	ret = snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn));
+	if (ret < 0) {
+		dev_warn(sdev->dev, "failed to read stream position: %d\n", ret);
+		return 0;
+	}
+
+	memcpy(&stream->posn, &posn, sizeof(posn));
+	pos = spcm->stream[substream->stream].posn.host_posn;
+	pos = bytes_to_frames(substream->runtime, pos);
+
+	return pos;
+}
+
+static struct snd_soc_dai_driver mt8186_dai[] = {
+{
+	.name = "SOF_DL1",
+	.playback = {
+		.channels_min = 1,
+		.channels_max = 2,
+	},
+},
+{
+	.name = "SOF_DL2",
+	.playback = {
+		.channels_min = 1,
+		.channels_max = 2,
+	},
+},
+{
+	.name = "SOF_UL1",
+	.capture = {
+		.channels_min = 1,
+		.channels_max = 2,
+	},
+},
+{
+	.name = "SOF_UL2",
+	.capture = {
+		.channels_min = 1,
+		.channels_max = 2,
+	},
+},
+};
+
 /* mt8186 ops */
 static struct snd_sof_dsp_ops sof_mt8186_ops = {
 	/* probe and remove */
@@ -481,6 +546,10 @@ static struct snd_sof_dsp_ops sof_mt8186_ops = {
 	.block_read	= sof_block_read,
 	.block_write	= sof_block_write,
 
+	/* Mailbox IO */
+	.mailbox_read	= sof_mailbox_read,
+	.mailbox_write	= sof_mailbox_write,
+
 	/* Register IO */
 	.write		= sof_io_write,
 	.read		= sof_io_read,
@@ -491,18 +560,28 @@ static struct snd_sof_dsp_ops sof_mt8186_ops = {
 	.send_msg		= mt8186_send_msg,
 	.get_mailbox_offset	= mt8186_get_mailbox_offset,
 	.get_window_offset	= mt8186_get_window_offset,
-	.ipc_msg_data		= mt8186_ipc_msg_data,
+	.ipc_msg_data		= sof_ipc_msg_data,
 	.set_stream_data_offset = sof_set_stream_data_offset,
 
 	/* misc */
 	.get_bar_index	= mt8186_get_bar_index,
 
+	/* stream callbacks */
+	.pcm_open	= sof_stream_pcm_open,
+	.pcm_hw_params	= mt8186_pcm_hw_params,
+	.pcm_pointer	= mt8186_pcm_pointer,
+	.pcm_close	= sof_stream_pcm_close,
+
 	/* firmware loading */
 	.load_firmware	= snd_sof_load_firmware_memcpy,
 
 	/* Firmware ops */
 	.dsp_arch_ops = &sof_xtensa_arch_ops,
 
+	/* DAI drivers */
+	.drv		= mt8186_dai,
+	.num_drv	= ARRAY_SIZE(mt8186_dai),
+
 	/* PM */
 	.suspend	= mt8186_dsp_suspend,
 	.resume		= mt8186_dsp_resume,
@@ -515,7 +594,16 @@ static struct snd_sof_dsp_ops sof_mt8186_ops = {
 			SNDRV_PCM_INFO_NO_PERIOD_WAKEUP,
 };
 
+static struct snd_sof_of_mach sof_mt8186_machs[] = {
+	{
+		.compatible = "mediatek,mt8186",
+		.sof_tplg_filename = "sof-mt8186.tplg",
+	},
+	{}
+};
+
 static const struct sof_dev_desc sof_of_mt8186_desc = {
+	.of_machines = sof_mt8186_machs,
 	.ipc_supported_mask	= BIT(SOF_IPC),
 	.ipc_default		= SOF_IPC,
 	.default_fw_path = {
diff --git a/sound/soc/sof/mediatek/mt8195/mt8195.c b/sound/soc/sof/mediatek/mt8195/mt8195.c
index 9c146015cd1b..3c81e84fcecf 100644
--- a/sound/soc/sof/mediatek/mt8195/mt8195.c
+++ b/sound/soc/sof/mediatek/mt8195/mt8195.c
@@ -496,14 +496,48 @@ static int mt8195_get_bar_index(struct snd_sof_dev *sdev, u32 type)
 	return type;
 }
 
-static int mt8195_ipc_msg_data(struct snd_sof_dev *sdev,
-			       struct snd_pcm_substream *substream,
-			       void *p, size_t sz)
+static int mt8195_pcm_hw_params(struct snd_sof_dev *sdev,
+				struct snd_pcm_substream *substream,
+				struct snd_pcm_hw_params *params,
+				struct snd_sof_platform_stream_params *platform_params)
 {
-	sof_mailbox_read(sdev, sdev->dsp_box.offset, p, sz);
+	platform_params->cont_update_posn = 1;
+
 	return 0;
 }
 
+static snd_pcm_uframes_t mt8195_pcm_pointer(struct snd_sof_dev *sdev,
+					    struct snd_pcm_substream *substream)
+{
+	int ret;
+	snd_pcm_uframes_t pos;
+	struct snd_sof_pcm *spcm;
+	struct sof_ipc_stream_posn posn;
+	struct snd_sof_pcm_stream *stream;
+	struct snd_soc_component *scomp = sdev->component;
+	struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream);
+
+	spcm = snd_sof_find_spcm_dai(scomp, rtd);
+	if (!spcm) {
+		dev_warn_ratelimited(sdev->dev, "warn: can't find PCM with DAI ID %d\n",
+				     rtd->dai_link->id);
+		return 0;
+	}
+
+	stream = &spcm->stream[substream->stream];
+	ret = snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn));
+	if (ret < 0) {
+		dev_warn(sdev->dev, "failed to read stream position: %d\n", ret);
+		return 0;
+	}
+
+	memcpy(&stream->posn, &posn, sizeof(posn));
+	pos = spcm->stream[substream->stream].posn.host_posn;
+	pos = bytes_to_frames(substream->runtime, pos);
+
+	return pos;
+}
+
 static void mt8195_adsp_dump(struct snd_sof_dev *sdev, u32 flags)
 {
 	u32 dbg_pc, dbg_data, dbg_bus0, dbg_bus1, dbg_inst;
@@ -574,6 +608,10 @@ static struct snd_sof_dsp_ops sof_mt8195_ops = {
 	.block_read	= sof_block_read,
 	.block_write	= sof_block_write,
 
+	/* Mailbox IO */
+	.mailbox_read	= sof_mailbox_read,
+	.mailbox_write	= sof_mailbox_write,
+
 	/* Register IO */
 	.write		= sof_io_write,
 	.read		= sof_io_read,
@@ -584,12 +622,18 @@ static struct snd_sof_dsp_ops sof_mt8195_ops = {
 	.send_msg		= mt8195_send_msg,
 	.get_mailbox_offset	= mt8195_get_mailbox_offset,
 	.get_window_offset	= mt8195_get_window_offset,
-	.ipc_msg_data		= mt8195_ipc_msg_data,
+	.ipc_msg_data		= sof_ipc_msg_data,
 	.set_stream_data_offset = sof_set_stream_data_offset,
 
 	/* misc */
 	.get_bar_index	= mt8195_get_bar_index,
 
+	/* stream callbacks */
+	.pcm_open	= sof_stream_pcm_open,
+	.pcm_hw_params	= mt8195_pcm_hw_params,
+	.pcm_pointer	= mt8195_pcm_pointer,
+	.pcm_close	= sof_stream_pcm_close,
+
 	/* firmware loading */
 	.load_firmware	= snd_sof_load_firmware_memcpy,
 
@@ -615,7 +659,20 @@ static struct snd_sof_dsp_ops sof_mt8195_ops = {
 			SNDRV_PCM_INFO_NO_PERIOD_WAKEUP,
 };
 
+static struct snd_sof_of_mach sof_mt8195_machs[] = {
+	{
+		.compatible = "google,tomato",
+		.sof_tplg_filename = "sof-mt8195-mt6359-rt1019-rt5682-dts.tplg"
+	}, {
+		.compatible = "mediatek,mt8195",
+		.sof_tplg_filename = "sof-mt8195.tplg"
+	}, {
+		/* sentinel */
+	}
+};
+
 static const struct sof_dev_desc sof_of_mt8195_desc = {
+	.of_machines = sof_mt8195_machs,
 	.ipc_supported_mask	= BIT(SOF_IPC),
 	.ipc_default		= SOF_IPC,
 	.default_fw_path = {
@@ -652,4 +709,5 @@ static struct platform_driver snd_sof_of_mt8195_driver = {
 module_platform_driver(snd_sof_of_mt8195_driver);
 
 MODULE_IMPORT_NS(SND_SOC_SOF_XTENSA);
+MODULE_IMPORT_NS(SND_SOC_SOF_MTK_COMMON);
 MODULE_LICENSE("Dual BSD/GPL");
diff --git a/sound/soc/sof/nocodec.c b/sound/soc/sof/nocodec.c
index 356497fe4f4c..3537805070ad 100644
--- a/sound/soc/sof/nocodec.c
+++ b/sound/soc/sof/nocodec.c
@@ -32,7 +32,7 @@ static int sof_nocodec_bes_setup(struct device *dev,
 
 	/* set up BE dai_links */
 	for (i = 0; i < link_num; i++) {
-		dlc = devm_kzalloc(dev, 3 * sizeof(*dlc), GFP_KERNEL);
+		dlc = devm_kcalloc(dev, 3, sizeof(*dlc), GFP_KERNEL);
 		if (!dlc)
 			return -ENOMEM;
 
@@ -78,7 +78,7 @@ static int sof_nocodec_setup(struct device *dev,
 	struct snd_soc_dai_link *links;
 
 	/* create dummy BE dai_links */
-	links = devm_kzalloc(dev, sizeof(struct snd_soc_dai_link) * num_dai_drivers, GFP_KERNEL);
+	links = devm_kcalloc(dev, num_dai_drivers, sizeof(struct snd_soc_dai_link), GFP_KERNEL);
 	if (!links)
 		return -ENOMEM;
 
diff --git a/sound/soc/sof/pcm.c b/sound/soc/sof/pcm.c
index 6cb6a432be5e..14571b821eca 100644
--- a/sound/soc/sof/pcm.c
+++ b/sound/soc/sof/pcm.c
@@ -13,6 +13,8 @@
 #include <linux/pm_runtime.h>
 #include <sound/pcm_params.h>
 #include <sound/sof.h>
+#include <trace/events/sof.h>
+#include "sof-of-dev.h"
 #include "sof-priv.h"
 #include "sof-audio.h"
 #include "sof-utils.h"
@@ -383,9 +385,7 @@ static snd_pcm_uframes_t sof_pcm_pointer(struct snd_soc_component *component,
 	dai = bytes_to_frames(substream->runtime,
 			      spcm->stream[substream->stream].posn.dai_posn);
 
-	dev_vdbg(component->dev,
-		 "PCM: stream %d dir %d DMA position %lu DAI position %lu\n",
-		 spcm->pcm.pcm_id, substream->stream, host, dai);
+	trace_sof_pcm_pointer_position(sdev, spcm, substream, host, dai);
 
 	return host;
 }
@@ -655,7 +655,12 @@ void snd_sof_new_platform_drv(struct snd_sof_dev *sdev)
 	struct snd_sof_pdata *plat_data = sdev->pdata;
 	const char *drv_name;
 
-	drv_name = plat_data->machine->drv_name;
+	if (plat_data->machine)
+		drv_name = plat_data->machine->drv_name;
+	else if (plat_data->of_machine)
+		drv_name = plat_data->of_machine->drv_name;
+	else
+		drv_name = NULL;
 
 	pd->name = "sof-audio-component";
 	pd->probe = sof_pcm_probe;
diff --git a/sound/soc/sof/sof-audio.c b/sound/soc/sof/sof-audio.c
index 28976098a89e..62092e2d609c 100644
--- a/sound/soc/sof/sof-audio.c
+++ b/sound/soc/sof/sof-audio.c
@@ -9,7 +9,9 @@
 //
 
 #include <linux/bitfield.h>
+#include <trace/events/sof.h>
 #include "sof-audio.h"
+#include "sof-of-dev.h"
 #include "ops.h"
 
 static void sof_reset_route_setup_status(struct snd_sof_dev *sdev, struct snd_sof_widget *widget)
@@ -35,6 +37,8 @@ int sof_widget_free(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget)
 	if (!swidget->private)
 		return 0;
 
+	trace_sof_widget_free(swidget);
+
 	/* only free when use_count is 0 */
 	if (--swidget->use_count)
 		return 0;
@@ -85,6 +89,8 @@ int sof_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget)
 	if (!swidget->private)
 		return 0;
 
+	trace_sof_widget_setup(swidget);
+
 	/* widget already set up */
 	if (++swidget->use_count > 1)
 		return 0;
@@ -265,14 +271,16 @@ sof_unprepare_widgets_in_path(struct snd_sof_dev *sdev, struct snd_soc_dapm_widg
 	struct snd_sof_widget *swidget = widget->dobj.private;
 	struct snd_soc_dapm_path *p;
 
-	if (!widget_ops[widget->id].ipc_unprepare || !swidget->prepared)
-		goto sink_unprepare;
+	/* return if the widget is in use or if it is already unprepared */
+	if (!swidget->prepared || swidget->use_count > 1)
+		return;
+
+	if (widget_ops[widget->id].ipc_unprepare)
+		/* unprepare the source widget */
+		widget_ops[widget->id].ipc_unprepare(swidget);
 
-	/* unprepare the source widget */
-	widget_ops[widget->id].ipc_unprepare(swidget);
 	swidget->prepared = false;
 
-sink_unprepare:
 	/* unprepare all widgets in the sink paths */
 	snd_soc_dapm_widget_for_each_sink_path(widget, p) {
 		if (!p->walking && p->sink->dobj.private) {
@@ -784,6 +792,28 @@ int sof_dai_get_bclk(struct snd_soc_pcm_runtime *rtd)
 }
 EXPORT_SYMBOL(sof_dai_get_bclk);
 
+static struct snd_sof_of_mach *sof_of_machine_select(struct snd_sof_dev *sdev)
+{
+	struct snd_sof_pdata *sof_pdata = sdev->pdata;
+	const struct sof_dev_desc *desc = sof_pdata->desc;
+	struct snd_sof_of_mach *mach = desc->of_machines;
+
+	if (!mach)
+		return NULL;
+
+	for (; mach->compatible; mach++) {
+		if (of_machine_is_compatible(mach->compatible)) {
+			sof_pdata->tplg_filename = mach->sof_tplg_filename;
+			if (mach->fw_filename)
+				sof_pdata->fw_filename = mach->fw_filename;
+
+			return mach;
+		}
+	}
+
+	return NULL;
+}
+
 /*
  * SOF Driver enumeration.
  */
@@ -794,6 +824,7 @@ int sof_machine_check(struct snd_sof_dev *sdev)
 	struct snd_soc_acpi_mach *mach;
 
 	if (!IS_ENABLED(CONFIG_SND_SOC_SOF_FORCE_NOCODEC_MODE)) {
+		const struct snd_sof_of_mach *of_mach;
 
 		/* find machine */
 		mach = snd_sof_machine_select(sdev);
@@ -803,6 +834,12 @@ int sof_machine_check(struct snd_sof_dev *sdev)
 			return 0;
 		}
 
+		of_mach = sof_of_machine_select(sdev);
+		if (of_mach) {
+			sof_pdata->of_machine = of_mach;
+			return 0;
+		}
+
 		if (!IS_ENABLED(CONFIG_SND_SOC_SOF_NOCODEC)) {
 			dev_err(sdev->dev, "error: no matching ASoC machine driver found - aborting probe\n");
 			return -ENODEV;
diff --git a/sound/soc/sof/sof-client-probes.c b/sound/soc/sof/sof-client-probes.c
index eb246b823461..ddeabbb5580e 100644
--- a/sound/soc/sof/sof-client-probes.c
+++ b/sound/soc/sof/sof-client-probes.c
@@ -12,6 +12,8 @@
 #include <linux/debugfs.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
+#include <linux/string_helpers.h>
+
 #include <sound/soc.h>
 #include <sound/sof/header.h>
 #include "sof-client.h"
@@ -410,79 +412,6 @@ static const struct snd_compress_ops sof_probes_compressed_ops = {
 	.copy = sof_probes_compr_copy,
 };
 
-/**
- * strsplit_u32 - Split string into sequence of u32 tokens
- * @buf:	String to split into tokens.
- * @delim:	String containing delimiter characters.
- * @tkns:	Returned u32 sequence pointer.
- * @num_tkns:	Returned number of tokens obtained.
- */
-static int strsplit_u32(char *buf, const char *delim, u32 **tkns, size_t *num_tkns)
-{
-	char *s;
-	u32 *data, *tmp;
-	size_t count = 0;
-	size_t cap = 32;
-	int ret = 0;
-
-	*tkns = NULL;
-	*num_tkns = 0;
-	data = kcalloc(cap, sizeof(*data), GFP_KERNEL);
-	if (!data)
-		return -ENOMEM;
-
-	while ((s = strsep(&buf, delim)) != NULL) {
-		ret = kstrtouint(s, 0, data + count);
-		if (ret)
-			goto exit;
-		if (++count >= cap) {
-			cap *= 2;
-			tmp = krealloc(data, cap * sizeof(*data), GFP_KERNEL);
-			if (!tmp) {
-				ret = -ENOMEM;
-				goto exit;
-			}
-			data = tmp;
-		}
-	}
-
-	if (!count)
-		goto exit;
-	*tkns = kmemdup(data, count * sizeof(*data), GFP_KERNEL);
-	if (!(*tkns)) {
-		ret = -ENOMEM;
-		goto exit;
-	}
-	*num_tkns = count;
-
-exit:
-	kfree(data);
-	return ret;
-}
-
-static int tokenize_input(const char __user *from, size_t count,
-			  loff_t *ppos, u32 **tkns, size_t *num_tkns)
-{
-	char *buf;
-	int ret;
-
-	buf = kmalloc(count + 1, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	ret = simple_write_to_buffer(buf, count, ppos, from, count);
-	if (ret != count) {
-		ret = ret >= 0 ? -EIO : ret;
-		goto exit;
-	}
-
-	buf[count] = '\0';
-	ret = strsplit_u32(buf, ",", tkns, num_tkns);
-exit:
-	kfree(buf);
-	return ret;
-}
-
 static ssize_t sof_probes_dfs_points_read(struct file *file, char __user *to,
 					  size_t count, loff_t *ppos)
 {
@@ -548,8 +477,8 @@ sof_probes_dfs_points_write(struct file *file, const char __user *from,
 	struct sof_probes_priv *priv = cdev->data;
 	struct device *dev = &cdev->auxdev.dev;
 	struct sof_probe_point_desc *desc;
-	size_t num_tkns, bytes;
-	u32 *tkns;
+	u32 num_elems, *array;
+	size_t bytes;
 	int ret, err;
 
 	if (priv->extractor_stream_tag == SOF_PROBES_INVALID_NODE_ID) {
@@ -557,16 +486,18 @@ sof_probes_dfs_points_write(struct file *file, const char __user *from,
 		return -ENOENT;
 	}
 
-	ret = tokenize_input(from, count, ppos, &tkns, &num_tkns);
+	ret = parse_int_array_user(from, count, (int **)&array);
 	if (ret < 0)
 		return ret;
-	bytes = sizeof(*tkns) * num_tkns;
-	if (!num_tkns || (bytes % sizeof(*desc))) {
+
+	num_elems = *array;
+	bytes = sizeof(*array) * num_elems;
+	if (bytes % sizeof(*desc)) {
 		ret = -EINVAL;
 		goto exit;
 	}
 
-	desc = (struct sof_probe_point_desc *)tkns;
+	desc = (struct sof_probe_point_desc *)&array[1];
 
 	ret = pm_runtime_resume_and_get(dev);
 	if (ret < 0 && ret != -EACCES) {
@@ -583,7 +514,7 @@ sof_probes_dfs_points_write(struct file *file, const char __user *from,
 	if (err < 0)
 		dev_err_ratelimited(dev, "debugfs write failed to idle %d\n", err);
 exit:
-	kfree(tkns);
+	kfree(array);
 	return ret;
 }
 
@@ -603,22 +534,17 @@ sof_probes_dfs_points_remove_write(struct file *file, const char __user *from,
 	struct sof_client_dev *cdev = file->private_data;
 	struct sof_probes_priv *priv = cdev->data;
 	struct device *dev = &cdev->auxdev.dev;
-	size_t num_tkns;
-	u32 *tkns;
 	int ret, err;
+	u32 *array;
 
 	if (priv->extractor_stream_tag == SOF_PROBES_INVALID_NODE_ID) {
 		dev_warn(dev, "no extractor stream running\n");
 		return -ENOENT;
 	}
 
-	ret = tokenize_input(from, count, ppos, &tkns, &num_tkns);
+	ret = parse_int_array_user(from, count, (int **)&array);
 	if (ret < 0)
 		return ret;
-	if (!num_tkns) {
-		ret = -EINVAL;
-		goto exit;
-	}
 
 	ret = pm_runtime_resume_and_get(dev);
 	if (ret < 0) {
@@ -626,7 +552,7 @@ sof_probes_dfs_points_remove_write(struct file *file, const char __user *from,
 		goto exit;
 	}
 
-	ret = sof_probes_points_remove(cdev, tkns, num_tkns);
+	ret = sof_probes_points_remove(cdev, &array[1], array[0]);
 	if (!ret)
 		ret = count;
 
@@ -635,7 +561,7 @@ sof_probes_dfs_points_remove_write(struct file *file, const char __user *from,
 	if (err < 0)
 		dev_err_ratelimited(dev, "debugfs write failed to idle %d\n", err);
 exit:
-	kfree(tkns);
+	kfree(array);
 	return ret;
 }
 
diff --git a/sound/soc/sof/sof-of-dev.h b/sound/soc/sof/sof-of-dev.h
index fd950a222ba4..2948b3a0d9fe 100644
--- a/sound/soc/sof/sof-of-dev.h
+++ b/sound/soc/sof/sof-of-dev.h
@@ -9,6 +9,13 @@
 #ifndef __SOUND_SOC_SOF_OF_H
 #define __SOUND_SOC_SOF_OF_H
 
+struct snd_sof_of_mach {
+	const char *compatible;
+	const char *drv_name;
+	const char *fw_filename;
+	const char *sof_tplg_filename;
+};
+
 extern const struct dev_pm_ops sof_of_pm;
 
 int sof_of_probe(struct platform_device *pdev);
diff --git a/sound/soc/sof/sof-pci-dev.c b/sound/soc/sof/sof-pci-dev.c
index d627092b399d..643fd1036d60 100644
--- a/sound/soc/sof/sof-pci-dev.c
+++ b/sound/soc/sof/sof-pci-dev.c
@@ -138,7 +138,7 @@ static const struct dmi_system_id community_key_platforms[] = {
 		.ident = "Google Chromebooks",
 		.callback = chromebook_use_community_key,
 		.matches = {
-			DMI_MATCH(DMI_SYS_VENDOR, "Google"),
+			DMI_MATCH(DMI_PRODUCT_FAMILY, "Google"),
 		}
 	},
 	{},
diff --git a/sound/soc/sof/sof-priv.h b/sound/soc/sof/sof-priv.h
index 823583086279..de08825915b3 100644
--- a/sound/soc/sof/sof-priv.h
+++ b/sound/soc/sof/sof-priv.h
@@ -105,6 +105,13 @@ enum sof_debugfs_access_type {
 	SOF_DEBUGFS_ACCESS_D0_ONLY,
 };
 
+struct sof_compr_stream {
+	u64 copied_total;
+	u32 sampling_rate;
+	u16 channels;
+	u16 sample_container_bytes;
+};
+
 struct snd_sof_dev;
 struct snd_sof_ipc_msg;
 struct snd_sof_ipc;
@@ -594,6 +601,10 @@ struct snd_sof_dev {
 	/* to protect the ipc_rx_handler_list  and  dsp_state_handler_list list */
 	struct mutex client_event_handler_mutex;
 
+	/* quirks to override topology values */
+	bool mclk_id_override;
+	u16  mclk_id_quirk; /* same size as in IPC3 definitions */
+
 	void *private;			/* core does not touch this */
 };
 
diff --git a/sound/soc/sof/sof-utils.c b/sound/soc/sof/sof-utils.c
index a3300ecee062..b6345a7345af 100644
--- a/sound/soc/sof/sof-utils.c
+++ b/sound/soc/sof/sof-utils.c
@@ -45,8 +45,6 @@ int snd_sof_create_page_table(struct device *dev,
 		u32 pfn = snd_sgbuf_get_addr(dmab, i * PAGE_SIZE) >> PAGE_SHIFT;
 		u8 *pg_table;
 
-		dev_vdbg(dev, "pfn i %i idx %d pfn %x\n", i, idx, pfn);
-
 		pg_table = (u8 *)(page_table + idx);
 
 		/*
diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c
index 9273a70fec25..38855dd60617 100644
--- a/sound/soc/sof/topology.c
+++ b/sound/soc/sof/topology.c
@@ -287,6 +287,7 @@ static const struct sof_dai_types sof_dais[] = {
 	{"ACP", SOF_DAI_AMD_BT},
 	{"ACPSP", SOF_DAI_AMD_SP},
 	{"ACPDMIC", SOF_DAI_AMD_DMIC},
+	{"ACPHS", SOF_DAI_AMD_HS},
 	{"AFE", SOF_DAI_MEDIATEK_AFE},
 };
 
@@ -1011,9 +1012,6 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp,
 	}
 
 	list_for_each_entry(rtd, &card->rtd_list, list) {
-		dev_vdbg(scomp->dev, "tplg: check widget: %s stream: %s dai stream: %s\n",
-			 w->name,  w->sname, rtd->dai_link->stream_name);
-
 		/* does stream match DAI link ? */
 		if (!rtd->dai_link->stream_name ||
 		    strcmp(w->sname, rtd->dai_link->stream_name))
@@ -1032,7 +1030,7 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp,
 					break;
 				}
 			}
-			if (i == rtd->num_cpus) {
+			if (i == rtd->dai_link->num_cpus) {
 				dev_err(scomp->dev, "error: can't find BE for DAI %s\n",
 					w->name);
 
@@ -1054,7 +1052,7 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp,
 					break;
 				}
 			}
-			if (i == rtd->num_cpus) {
+			if (i == rtd->dai_link->num_cpus) {
 				dev_err(scomp->dev, "error: can't find BE for DAI %s\n",
 					w->name);
 
@@ -1537,9 +1535,6 @@ static int sof_dai_load(struct snd_soc_component *scomp, int index,
 
 	stream = SNDRV_PCM_STREAM_PLAYBACK;
 
-	dev_vdbg(scomp->dev, "tplg: pcm %s stream tokens: playback d0i3:%d\n",
-		 spcm->pcm.pcm_name, spcm->stream[stream].d0i3_compatible);
-
 	caps = &spcm->pcm.caps[stream];
 
 	/* allocate playback page table buffer */
@@ -1567,9 +1562,6 @@ capture:
 	if (!spcm->pcm.capture)
 		return ret;
 
-	dev_vdbg(scomp->dev, "tplg: pcm %s stream tokens: capture d0i3:%d\n",
-		 spcm->pcm.pcm_name, spcm->stream[stream].d0i3_compatible);
-
 	caps = &spcm->pcm.caps[stream];
 
 	/* allocate capture page table buffer */
diff --git a/sound/soc/stm/stm32_adfsdm.c b/sound/soc/stm/stm32_adfsdm.c
index 04f2912e1418..643fc8a17018 100644
--- a/sound/soc/stm/stm32_adfsdm.c
+++ b/sound/soc/stm/stm32_adfsdm.c
@@ -335,8 +335,6 @@ static int stm32_adfsdm_probe(struct platform_device *pdev)
 
 	dev_set_drvdata(&pdev->dev, priv);
 
-	pm_runtime_enable(&pdev->dev);
-
 	ret = devm_snd_soc_register_component(&pdev->dev,
 					      &stm32_adfsdm_dai_component,
 					      &priv->dai_drv, 1);
@@ -366,9 +364,13 @@ static int stm32_adfsdm_probe(struct platform_device *pdev)
 #endif
 
 	ret = snd_soc_add_component(component, NULL, 0);
-	if (ret < 0)
+	if (ret < 0) {
 		dev_err(&pdev->dev, "%s: Failed to register PCM platform\n",
 			__func__);
+		return ret;
+	}
+
+	pm_runtime_enable(&pdev->dev);
 
 	return ret;
 }
diff --git a/sound/soc/stm/stm32_i2s.c b/sound/soc/stm/stm32_i2s.c
index 6aafe793eec4..ce7f6942308f 100644
--- a/sound/soc/stm/stm32_i2s.c
+++ b/sound/soc/stm/stm32_i2s.c
@@ -1136,8 +1136,6 @@ static int stm32_i2s_probe(struct platform_device *pdev)
 		return dev_err_probe(&pdev->dev, PTR_ERR(i2s->regmap),
 				     "Regmap init error\n");
 
-	pm_runtime_enable(&pdev->dev);
-
 	ret = snd_dmaengine_pcm_register(&pdev->dev, &stm32_i2s_pcm_config, 0);
 	if (ret)
 		return dev_err_probe(&pdev->dev, ret, "PCM DMA register error\n");
@@ -1180,6 +1178,8 @@ static int stm32_i2s_probe(struct platform_device *pdev)
 			FIELD_GET(I2S_VERR_MIN_MASK, val));
 	}
 
+	pm_runtime_enable(&pdev->dev);
+
 	return ret;
 
 error:
diff --git a/sound/soc/stm/stm32_spdifrx.c b/sound/soc/stm/stm32_spdifrx.c
index 0f7146756717..d399c906bb92 100644
--- a/sound/soc/stm/stm32_spdifrx.c
+++ b/sound/soc/stm/stm32_spdifrx.c
@@ -1002,8 +1002,6 @@ static int stm32_spdifrx_probe(struct platform_device *pdev)
 	udelay(2);
 	reset_control_deassert(rst);
 
-	pm_runtime_enable(&pdev->dev);
-
 	pcm_config = &stm32_spdifrx_pcm_config;
 	ret = snd_dmaengine_pcm_register(&pdev->dev, pcm_config, 0);
 	if (ret)
@@ -1036,6 +1034,8 @@ static int stm32_spdifrx_probe(struct platform_device *pdev)
 			FIELD_GET(SPDIFRX_VERR_MIN_MASK, ver));
 	}
 
+	pm_runtime_enable(&pdev->dev);
+
 	return ret;
 
 error:
diff --git a/sound/soc/sunxi/Kconfig b/sound/soc/sunxi/Kconfig
index ddcaaa98d3cb..1f18f016acbb 100644
--- a/sound/soc/sunxi/Kconfig
+++ b/sound/soc/sunxi/Kconfig
@@ -56,6 +56,13 @@ config SND_SUN4I_SPDIF
 	  Say Y or M to add support for the S/PDIF audio block in the Allwinner
 	  A10 and affiliated SoCs.
 
+config SND_SUN50I_DMIC
+	tristate "Allwinner H6 DMIC Support"
+	select SND_SOC_GENERIC_DMAENGINE_PCM
+	help
+	  Say Y or M to add support for the DMIC audio block in the Allwinner
+	  H6 and affiliated SoCs.
+
 config SND_SUN8I_ADDA_PR_REGMAP
 	tristate
 	select REGMAP
diff --git a/sound/soc/sunxi/Makefile b/sound/soc/sunxi/Makefile
index a86be340a076..4483fe9c94ef 100644
--- a/sound/soc/sunxi/Makefile
+++ b/sound/soc/sunxi/Makefile
@@ -6,3 +6,4 @@ obj-$(CONFIG_SND_SUN8I_CODEC_ANALOG) += sun8i-codec-analog.o
 obj-$(CONFIG_SND_SUN50I_CODEC_ANALOG) += sun50i-codec-analog.o
 obj-$(CONFIG_SND_SUN8I_CODEC) += sun8i-codec.o
 obj-$(CONFIG_SND_SUN8I_ADDA_PR_REGMAP) += sun8i-adda-pr-regmap.o
+obj-$(CONFIG_SND_SUN50I_DMIC) += sun50i-dmic.o
diff --git a/sound/soc/sunxi/sun4i-codec.c b/sound/soc/sunxi/sun4i-codec.c
index 830beb38bf15..835dc3404367 100644
--- a/sound/soc/sunxi/sun4i-codec.c
+++ b/sound/soc/sunxi/sun4i-codec.c
@@ -1232,6 +1232,9 @@ static const struct snd_soc_component_driver sun8i_a23_codec_codec = {
 static const struct snd_soc_component_driver sun4i_codec_component = {
 	.name			= "sun4i-codec",
 	.legacy_dai_naming	= 1,
+#ifdef CONFIG_DEBUG_FS
+	.debugfs_prefix		= "cpu",
+#endif
 };
 
 #define SUN4I_CODEC_RATES	SNDRV_PCM_RATE_CONTINUOUS
@@ -1804,7 +1807,7 @@ static int sun4i_codec_probe(struct platform_device *pdev)
 
 	ret = snd_soc_register_card(card);
 	if (ret) {
-		dev_err(&pdev->dev, "Failed to register our card\n");
+		dev_err_probe(&pdev->dev, ret, "Failed to register our card\n");
 		goto err_assert_reset;
 	}
 
diff --git a/sound/soc/sunxi/sun50i-dmic.c b/sound/soc/sunxi/sun50i-dmic.c
new file mode 100644
index 000000000000..86cff5a5b1bd
--- /dev/null
+++ b/sound/soc/sunxi/sun50i-dmic.c
@@ -0,0 +1,406 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+// This driver supports the DMIC in Allwinner's H6 SoCs.
+//
+// Copyright 2021 Ban Tao <fengzheng923@gmail.com>
+
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/of_device.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/reset.h>
+#include <sound/dmaengine_pcm.h>
+#include <sound/pcm_params.h>
+#include <sound/soc.h>
+
+#define SUN50I_DMIC_EN_CTL			(0x00)
+	#define SUN50I_DMIC_EN_CTL_GLOBE			BIT(8)
+	#define SUN50I_DMIC_EN_CTL_CHAN(v)			((v) << 0)
+	#define SUN50I_DMIC_EN_CTL_CHAN_MASK			GENMASK(7, 0)
+#define SUN50I_DMIC_SR				(0x04)
+	#define SUN50I_DMIC_SR_SAMPLE_RATE(v)			((v) << 0)
+	#define SUN50I_DMIC_SR_SAMPLE_RATE_MASK			GENMASK(2, 0)
+#define SUN50I_DMIC_CTL				(0x08)
+	#define SUN50I_DMIC_CTL_OVERSAMPLE_RATE			BIT(0)
+#define SUN50I_DMIC_DATA			(0x10)
+#define SUN50I_DMIC_INTC			(0x14)
+	#define SUN50I_DMIC_FIFO_DRQ_EN				BIT(2)
+#define SUN50I_DMIC_INT_STA			(0x18)
+	#define SUN50I_DMIC_INT_STA_OVERRUN_IRQ_PENDING		BIT(1)
+	#define SUN50I_DMIC_INT_STA_DATA_IRQ_PENDING		BIT(0)
+#define SUN50I_DMIC_RXFIFO_CTL			(0x1c)
+	#define SUN50I_DMIC_RXFIFO_CTL_FLUSH			BIT(31)
+	#define SUN50I_DMIC_RXFIFO_CTL_MODE_MASK		BIT(9)
+	#define SUN50I_DMIC_RXFIFO_CTL_MODE_LSB			(0 << 9)
+	#define SUN50I_DMIC_RXFIFO_CTL_MODE_MSB			(1 << 9)
+	#define SUN50I_DMIC_RXFIFO_CTL_SAMPLE_MASK		BIT(8)
+	#define SUN50I_DMIC_RXFIFO_CTL_SAMPLE_16		(0 << 8)
+	#define SUN50I_DMIC_RXFIFO_CTL_SAMPLE_24		(1 << 8)
+#define SUN50I_DMIC_CH_NUM			(0x24)
+	#define SUN50I_DMIC_CH_NUM_N(v)				((v) << 0)
+	#define SUN50I_DMIC_CH_NUM_N_MASK			GENMASK(2, 0)
+#define SUN50I_DMIC_CNT				(0x2c)
+	#define SUN50I_DMIC_CNT_N				(1 << 0)
+#define SUN50I_DMIC_HPF_CTRL			(0x38)
+#define SUN50I_DMIC_VERSION			(0x50)
+
+struct sun50i_dmic_dev {
+	struct clk *dmic_clk;
+	struct clk *bus_clk;
+	struct reset_control *rst;
+	struct regmap *regmap;
+	struct snd_dmaengine_dai_dma_data dma_params_rx;
+};
+
+struct dmic_rate {
+	unsigned int samplerate;
+	unsigned int rate_bit;
+};
+
+static const struct dmic_rate dmic_rate_s[] = {
+	{48000, 0x0},
+	{44100, 0x0},
+	{32000, 0x1},
+	{24000, 0x2},
+	{22050, 0x2},
+	{16000, 0x3},
+	{12000, 0x4},
+	{11025, 0x4},
+	{8000,  0x5},
+};
+
+static int sun50i_dmic_startup(struct snd_pcm_substream *substream,
+			       struct snd_soc_dai *cpu_dai)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct sun50i_dmic_dev *host = snd_soc_dai_get_drvdata(asoc_rtd_to_cpu(rtd, 0));
+
+	/* only support capture */
+	if (substream->stream != SNDRV_PCM_STREAM_CAPTURE)
+		return -EINVAL;
+
+	regmap_update_bits(host->regmap, SUN50I_DMIC_RXFIFO_CTL,
+			SUN50I_DMIC_RXFIFO_CTL_FLUSH,
+			SUN50I_DMIC_RXFIFO_CTL_FLUSH);
+	regmap_write(host->regmap, SUN50I_DMIC_CNT, SUN50I_DMIC_CNT_N);
+
+	return 0;
+}
+
+static int sun50i_dmic_hw_params(struct snd_pcm_substream *substream,
+				 struct snd_pcm_hw_params *params,
+				 struct snd_soc_dai *cpu_dai)
+{
+	int i = 0;
+	unsigned long rate = params_rate(params);
+	unsigned int mclk = 0;
+	unsigned int channels = params_channels(params);
+	unsigned int chan_en = (1 << channels) - 1;
+	struct sun50i_dmic_dev *host = snd_soc_dai_get_drvdata(cpu_dai);
+
+	/* DMIC num is N+1 */
+	regmap_update_bits(host->regmap, SUN50I_DMIC_CH_NUM,
+			   SUN50I_DMIC_CH_NUM_N_MASK,
+			   SUN50I_DMIC_CH_NUM_N(channels - 1));
+	regmap_write(host->regmap, SUN50I_DMIC_HPF_CTRL, chan_en);
+	regmap_update_bits(host->regmap, SUN50I_DMIC_EN_CTL,
+			   SUN50I_DMIC_EN_CTL_CHAN_MASK,
+			   SUN50I_DMIC_EN_CTL_CHAN(chan_en));
+
+	switch (params_format(params)) {
+	case SNDRV_PCM_FORMAT_S16_LE:
+		regmap_update_bits(host->regmap, SUN50I_DMIC_RXFIFO_CTL,
+				   SUN50I_DMIC_RXFIFO_CTL_SAMPLE_MASK,
+				   SUN50I_DMIC_RXFIFO_CTL_SAMPLE_16);
+		break;
+	case SNDRV_PCM_FORMAT_S24_LE:
+		regmap_update_bits(host->regmap, SUN50I_DMIC_RXFIFO_CTL,
+				   SUN50I_DMIC_RXFIFO_CTL_SAMPLE_MASK,
+				   SUN50I_DMIC_RXFIFO_CTL_SAMPLE_24);
+		break;
+	default:
+		dev_err(cpu_dai->dev, "Invalid format!\n");
+		return -EINVAL;
+	}
+	/* The hardware supports FIFO mode 1 for 24-bit samples */
+	regmap_update_bits(host->regmap, SUN50I_DMIC_RXFIFO_CTL,
+			   SUN50I_DMIC_RXFIFO_CTL_MODE_MASK,
+			   SUN50I_DMIC_RXFIFO_CTL_MODE_MSB);
+
+	switch (rate) {
+	case 11025:
+	case 22050:
+	case 44100:
+		mclk = 22579200;
+		break;
+	case 8000:
+	case 12000:
+	case 16000:
+	case 24000:
+	case 32000:
+	case 48000:
+		mclk = 24576000;
+		break;
+	default:
+		dev_err(cpu_dai->dev, "Invalid rate!\n");
+		return -EINVAL;
+	}
+
+	if (clk_set_rate(host->dmic_clk, mclk)) {
+		dev_err(cpu_dai->dev, "mclk : %u not support\n", mclk);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(dmic_rate_s); i++) {
+		if (dmic_rate_s[i].samplerate == rate) {
+			regmap_update_bits(host->regmap, SUN50I_DMIC_SR,
+					   SUN50I_DMIC_SR_SAMPLE_RATE_MASK,
+					   SUN50I_DMIC_SR_SAMPLE_RATE(dmic_rate_s[i].rate_bit));
+			break;
+		}
+	}
+
+	switch (params_physical_width(params)) {
+	case 16:
+		host->dma_params_rx.addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
+		break;
+	case 32:
+		host->dma_params_rx.addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+		break;
+	default:
+		dev_err(cpu_dai->dev, "Unsupported physical sample width: %d\n",
+			params_physical_width(params));
+		return -EINVAL;
+	}
+
+	/* oversamplerate adjust */
+	if (params_rate(params) >= 24000)
+		regmap_update_bits(host->regmap, SUN50I_DMIC_CTL,
+				   SUN50I_DMIC_CTL_OVERSAMPLE_RATE,
+				   SUN50I_DMIC_CTL_OVERSAMPLE_RATE);
+	else
+		regmap_update_bits(host->regmap, SUN50I_DMIC_CTL,
+				   SUN50I_DMIC_CTL_OVERSAMPLE_RATE, 0);
+
+	return 0;
+}
+
+static int sun50i_dmic_trigger(struct snd_pcm_substream *substream, int cmd,
+			       struct snd_soc_dai *dai)
+{
+	int ret = 0;
+	struct sun50i_dmic_dev *host = snd_soc_dai_get_drvdata(dai);
+
+	if (substream->stream != SNDRV_PCM_STREAM_CAPTURE)
+		return -EINVAL;
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+	case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
+		/* DRQ ENABLE */
+		regmap_update_bits(host->regmap, SUN50I_DMIC_INTC,
+				   SUN50I_DMIC_FIFO_DRQ_EN,
+				   SUN50I_DMIC_FIFO_DRQ_EN);
+		/* Global enable */
+		regmap_update_bits(host->regmap, SUN50I_DMIC_EN_CTL,
+				   SUN50I_DMIC_EN_CTL_GLOBE,
+				   SUN50I_DMIC_EN_CTL_GLOBE);
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
+		/* DRQ DISABLE */
+		regmap_update_bits(host->regmap, SUN50I_DMIC_INTC,
+				   SUN50I_DMIC_FIFO_DRQ_EN, 0);
+		/* Global disable */
+		regmap_update_bits(host->regmap, SUN50I_DMIC_EN_CTL,
+				   SUN50I_DMIC_EN_CTL_GLOBE, 0);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+
+static int sun50i_dmic_soc_dai_probe(struct snd_soc_dai *dai)
+{
+	struct sun50i_dmic_dev *host = snd_soc_dai_get_drvdata(dai);
+
+	snd_soc_dai_init_dma_data(dai, NULL, &host->dma_params_rx);
+
+	return 0;
+}
+
+static const struct snd_soc_dai_ops sun50i_dmic_dai_ops = {
+	.startup        = sun50i_dmic_startup,
+	.trigger        = sun50i_dmic_trigger,
+	.hw_params      = sun50i_dmic_hw_params,
+};
+
+static const struct regmap_config sun50i_dmic_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = SUN50I_DMIC_VERSION,
+	.cache_type = REGCACHE_NONE,
+};
+
+#define SUN50I_DMIC_RATES (SNDRV_PCM_RATE_8000_48000)
+#define SUN50I_DMIC_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S24_LE)
+
+static struct snd_soc_dai_driver sun50i_dmic_dai = {
+	.capture = {
+		.channels_min = 1,
+		.channels_max = 8,
+		.rates = SUN50I_DMIC_RATES,
+		.formats = SUN50I_DMIC_FORMATS,
+		.sig_bits = 21,
+	},
+	.probe = sun50i_dmic_soc_dai_probe,
+	.ops = &sun50i_dmic_dai_ops,
+	.name = "dmic",
+};
+
+static const struct of_device_id sun50i_dmic_of_match[] = {
+	{
+		.compatible = "allwinner,sun50i-h6-dmic",
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, sun50i_dmic_of_match);
+
+static const struct snd_soc_component_driver sun50i_dmic_component = {
+	.name           = "sun50i-dmic",
+};
+
+static int sun50i_dmic_runtime_suspend(struct device *dev)
+{
+	struct sun50i_dmic_dev *host  = dev_get_drvdata(dev);
+
+	clk_disable_unprepare(host->dmic_clk);
+	clk_disable_unprepare(host->bus_clk);
+
+	return 0;
+}
+
+static int sun50i_dmic_runtime_resume(struct device *dev)
+{
+	struct sun50i_dmic_dev *host  = dev_get_drvdata(dev);
+	int ret;
+
+	ret = clk_prepare_enable(host->dmic_clk);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(host->bus_clk);
+	if (ret) {
+		clk_disable_unprepare(host->dmic_clk);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int sun50i_dmic_probe(struct platform_device *pdev)
+{
+	struct sun50i_dmic_dev *host;
+	struct resource *res;
+	int ret;
+	void __iomem *base;
+
+	host = devm_kzalloc(&pdev->dev, sizeof(*host), GFP_KERNEL);
+	if (!host)
+		return -ENOMEM;
+
+	/* Get the addresses */
+	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	base = devm_ioremap_resource(&pdev->dev, res);
+	if (IS_ERR(base))
+		return dev_err_probe(&pdev->dev, PTR_ERR(base),
+				     "get resource failed.\n");
+
+	host->regmap = devm_regmap_init_mmio(&pdev->dev, base,
+					     &sun50i_dmic_regmap_config);
+
+	/* Clocks */
+	host->bus_clk = devm_clk_get(&pdev->dev, "bus");
+	if (IS_ERR(host->bus_clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(host->bus_clk),
+				     "failed to get bus clock.\n");
+
+	host->dmic_clk = devm_clk_get(&pdev->dev, "mod");
+	if (IS_ERR(host->dmic_clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(host->dmic_clk),
+				     "failed to get dmic clock.\n");
+
+	host->dma_params_rx.addr = res->start + SUN50I_DMIC_DATA;
+	host->dma_params_rx.maxburst = 8;
+
+	platform_set_drvdata(pdev, host);
+
+	host->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL);
+	if (IS_ERR(host->rst))
+		return dev_err_probe(&pdev->dev, PTR_ERR(host->rst),
+				     "Failed to get reset.\n");
+	reset_control_deassert(host->rst);
+
+	ret = devm_snd_soc_register_component(&pdev->dev, &sun50i_dmic_component,
+					      &sun50i_dmic_dai, 1);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret,
+				     "failed to register component.\n");
+
+	pm_runtime_enable(&pdev->dev);
+	if (!pm_runtime_enabled(&pdev->dev)) {
+		ret = sun50i_dmic_runtime_resume(&pdev->dev);
+		if (ret)
+			goto err_disable_runtime_pm;
+	}
+
+	ret = devm_snd_dmaengine_pcm_register(&pdev->dev, NULL, 0);
+	if (ret)
+		goto err_suspend;
+
+	return 0;
+err_suspend:
+	if (!pm_runtime_status_suspended(&pdev->dev))
+		sun50i_dmic_runtime_suspend(&pdev->dev);
+err_disable_runtime_pm:
+	pm_runtime_disable(&pdev->dev);
+	return ret;
+}
+
+static int sun50i_dmic_remove(struct platform_device *pdev)
+{
+	pm_runtime_disable(&pdev->dev);
+	if (!pm_runtime_status_suspended(&pdev->dev))
+		sun50i_dmic_runtime_suspend(&pdev->dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops sun50i_dmic_pm = {
+	SET_RUNTIME_PM_OPS(sun50i_dmic_runtime_suspend,
+			   sun50i_dmic_runtime_resume, NULL)
+};
+
+static struct platform_driver sun50i_dmic_driver = {
+	.driver         = {
+		.name   = "sun50i-dmic",
+		.of_match_table = of_match_ptr(sun50i_dmic_of_match),
+		.pm     = &sun50i_dmic_pm,
+	},
+	.probe          = sun50i_dmic_probe,
+	.remove         = sun50i_dmic_remove,
+};
+
+module_platform_driver(sun50i_dmic_driver);
+
+MODULE_DESCRIPTION("Allwinner sun50i DMIC SoC Interface");
+MODULE_AUTHOR("Ban Tao <fengzheng923@gmail.com>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:sun50i-dmic");
diff --git a/sound/soc/ti/omap-mcbsp-st.c b/sound/soc/ti/omap-mcbsp-st.c
index 7e8179cae92e..8163f453bf36 100644
--- a/sound/soc/ti/omap-mcbsp-st.c
+++ b/sound/soc/ti/omap-mcbsp-st.c
@@ -244,10 +244,10 @@ static ssize_t st_taps_show(struct device *dev,
 
 	spin_lock_irq(&mcbsp->lock);
 	for (i = 0; i < st_data->nr_taps; i++)
-		status += sprintf(&buf[status], (i ? ", %d" : "%d"),
-				  st_data->taps[i]);
+		status += sysfs_emit_at(buf, status, (i ? ", %d" : "%d"),
+					st_data->taps[i]);
 	if (i)
-		status += sprintf(&buf[status], "\n");
+		status += sysfs_emit_at(buf, status, "\n");
 	spin_unlock_irq(&mcbsp->lock);
 
 	return status;
diff --git a/sound/soc/ti/omap-mcbsp.c b/sound/soc/ti/omap-mcbsp.c
index c4ac1f30b9fe..7c539a41a6a3 100644
--- a/sound/soc/ti/omap-mcbsp.c
+++ b/sound/soc/ti/omap-mcbsp.c
@@ -517,7 +517,7 @@ static ssize_t prop##_show(struct device *dev,				\
 {									\
 	struct omap_mcbsp *mcbsp = dev_get_drvdata(dev);		\
 									\
-	return sprintf(buf, "%u\n", mcbsp->prop);			\
+	return sysfs_emit(buf, "%u\n", mcbsp->prop);			\
 }									\
 									\
 static ssize_t prop##_store(struct device *dev,				\
@@ -560,11 +560,11 @@ static ssize_t dma_op_mode_show(struct device *dev,
 
 	for (s = &dma_op_modes[i]; i < ARRAY_SIZE(dma_op_modes); s++, i++) {
 		if (dma_op_mode == i)
-			len += sprintf(buf + len, "[%s] ", *s);
+			len += sysfs_emit_at(buf, len, "[%s] ", *s);
 		else
-			len += sprintf(buf + len, "%s ", *s);
+			len += sysfs_emit_at(buf, len, "%s ", *s);
 	}
-	len += sprintf(buf + len, "\n");
+	len += sysfs_emit_at(buf, len, "\n");
 
 	return len;
 }
@@ -614,7 +614,7 @@ static int omap_mcbsp_init(struct platform_device *pdev)
 {
 	struct omap_mcbsp *mcbsp = platform_get_drvdata(pdev);
 	struct resource *res;
-	int ret = 0;
+	int ret;
 
 	spin_lock_init(&mcbsp->lock);
 	mcbsp->free = true;
diff --git a/sound/usb/card.c b/sound/usb/card.c
index 706d249a9ad6..a5ed11ea1145 100644
--- a/sound/usb/card.c
+++ b/sound/usb/card.c
@@ -690,7 +690,7 @@ static bool get_alias_id(struct usb_device *dev, unsigned int *id)
 	return false;
 }
 
-static bool check_delayed_register_option(struct snd_usb_audio *chip, int iface)
+static int check_delayed_register_option(struct snd_usb_audio *chip)
 {
 	int i;
 	unsigned int id, inum;
@@ -699,14 +699,31 @@ static bool check_delayed_register_option(struct snd_usb_audio *chip, int iface)
 		if (delayed_register[i] &&
 		    sscanf(delayed_register[i], "%x:%x", &id, &inum) == 2 &&
 		    id == chip->usb_id)
-			return iface < inum;
+			return inum;
 	}
 
-	return false;
+	return -1;
 }
 
 static const struct usb_device_id usb_audio_ids[]; /* defined below */
 
+/* look for the last interface that matches with our ids and remember it */
+static void find_last_interface(struct snd_usb_audio *chip)
+{
+	struct usb_host_config *config = chip->dev->actconfig;
+	struct usb_interface *intf;
+	int i;
+
+	if (!config)
+		return;
+	for (i = 0; i < config->desc.bNumInterfaces; i++) {
+		intf = config->interface[i];
+		if (usb_match_id(intf, usb_audio_ids))
+			chip->last_iface = intf->altsetting[0].desc.bInterfaceNumber;
+	}
+	usb_audio_dbg(chip, "Found last interface = %d\n", chip->last_iface);
+}
+
 /* look for the corresponding quirk */
 static const struct snd_usb_audio_quirk *
 get_alias_quirk(struct usb_device *dev, unsigned int id)
@@ -813,6 +830,7 @@ static int usb_audio_probe(struct usb_interface *intf,
 			err = -ENODEV;
 			goto __error;
 		}
+		find_last_interface(chip);
 	}
 
 	if (chip->num_interfaces >= MAX_CARD_INTERFACES) {
@@ -862,11 +880,11 @@ static int usb_audio_probe(struct usb_interface *intf,
 		chip->need_delayed_register = false; /* clear again */
 	}
 
-	/* we are allowed to call snd_card_register() many times, but first
-	 * check to see if a device needs to skip it or do anything special
+	/* register card if we reach to the last interface or to the specified
+	 * one given via option
 	 */
-	if (!snd_usb_registration_quirk(chip, ifnum) &&
-	    !check_delayed_register_option(chip, ifnum)) {
+	if (check_delayed_register_option(chip) == ifnum ||
+	    usb_interface_claimed(usb_ifnum_to_if(dev, chip->last_iface))) {
 		err = snd_card_register(chip->card);
 		if (err < 0)
 			goto __error;
diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c
index eb71df9da831..48a3843a08f1 100644
--- a/sound/usb/endpoint.c
+++ b/sound/usb/endpoint.c
@@ -39,7 +39,9 @@ struct snd_usb_iface_ref {
 struct snd_usb_clock_ref {
 	unsigned char clock;
 	atomic_t locked;
+	int opened;
 	int rate;
+	bool need_setup;
 	struct list_head list;
 };
 
@@ -93,12 +95,13 @@ static inline unsigned get_usb_high_speed_rate(unsigned int rate)
  */
 static void release_urb_ctx(struct snd_urb_ctx *u)
 {
-	if (u->buffer_size)
+	if (u->urb && u->buffer_size)
 		usb_free_coherent(u->ep->chip->dev, u->buffer_size,
 				  u->urb->transfer_buffer,
 				  u->urb->transfer_dma);
 	usb_free_urb(u->urb);
 	u->urb = NULL;
+	u->buffer_size = 0;
 }
 
 static const char *usb_error_string(int err)
@@ -758,7 +761,8 @@ bool snd_usb_endpoint_compatible(struct snd_usb_audio *chip,
  * The endpoint needs to be closed via snd_usb_endpoint_close() later.
  *
  * Note that this function doesn't configure the endpoint.  The substream
- * needs to set it up later via snd_usb_endpoint_configure().
+ * needs to set it up later via snd_usb_endpoint_set_params() and
+ * snd_usb_endpoint_prepare().
  */
 struct snd_usb_endpoint *
 snd_usb_endpoint_open(struct snd_usb_audio *chip,
@@ -801,6 +805,7 @@ snd_usb_endpoint_open(struct snd_usb_audio *chip,
 				ep = NULL;
 				goto unlock;
 			}
+			ep->clock_ref->opened++;
 		}
 
 		ep->cur_audiofmt = fp;
@@ -924,8 +929,10 @@ void snd_usb_endpoint_close(struct snd_usb_audio *chip,
 		endpoint_set_interface(chip, ep, false);
 
 	if (!--ep->opened) {
-		if (ep->clock_ref && !atomic_read(&ep->clock_ref->locked))
-			ep->clock_ref->rate = 0;
+		if (ep->clock_ref) {
+			if (!--ep->clock_ref->opened)
+				ep->clock_ref->rate = 0;
+		}
 		ep->iface = 0;
 		ep->altsetting = 0;
 		ep->cur_audiofmt = NULL;
@@ -1261,6 +1268,7 @@ static int sync_ep_set_params(struct snd_usb_endpoint *ep)
 	if (!ep->syncbuf)
 		return -ENOMEM;
 
+	ep->nurbs = SYNC_URBS;
 	for (i = 0; i < SYNC_URBS; i++) {
 		struct snd_urb_ctx *u = &ep->urb[i];
 		u->index = i;
@@ -1280,8 +1288,6 @@ static int sync_ep_set_params(struct snd_usb_endpoint *ep)
 		u->urb->complete = snd_complete_urb;
 	}
 
-	ep->nurbs = SYNC_URBS;
-
 	return 0;
 
 out_of_memory:
@@ -1289,15 +1295,39 @@ out_of_memory:
 	return -ENOMEM;
 }
 
+/* update the rate of the referred clock; return the actual rate */
+static int update_clock_ref_rate(struct snd_usb_audio *chip,
+				 struct snd_usb_endpoint *ep)
+{
+	struct snd_usb_clock_ref *clock = ep->clock_ref;
+	int rate = ep->cur_rate;
+
+	if (!clock || clock->rate == rate)
+		return rate;
+	if (clock->rate) {
+		if (atomic_read(&clock->locked))
+			return clock->rate;
+		if (clock->rate != rate) {
+			usb_audio_err(chip, "Mismatched sample rate %d vs %d for EP 0x%x\n",
+				      clock->rate, rate, ep->ep_num);
+			return clock->rate;
+		}
+	}
+	clock->rate = rate;
+	clock->need_setup = true;
+	return rate;
+}
+
 /*
  * snd_usb_endpoint_set_params: configure an snd_usb_endpoint
  *
+ * It's called either from hw_params callback.
  * Determine the number of URBs to be used on this endpoint.
  * An endpoint must be configured before it can be started.
  * An endpoint that is already running can not be reconfigured.
  */
-static int snd_usb_endpoint_set_params(struct snd_usb_audio *chip,
-				       struct snd_usb_endpoint *ep)
+int snd_usb_endpoint_set_params(struct snd_usb_audio *chip,
+				struct snd_usb_endpoint *ep)
 {
 	const struct audioformat *fmt = ep->cur_audiofmt;
 	int err;
@@ -1349,49 +1379,46 @@ static int snd_usb_endpoint_set_params(struct snd_usb_audio *chip,
 	ep->maxframesize = ep->maxpacksize / ep->cur_frame_bytes;
 	ep->curframesize = ep->curpacksize / ep->cur_frame_bytes;
 
-	return 0;
+	return update_clock_ref_rate(chip, ep);
 }
 
 static int init_sample_rate(struct snd_usb_audio *chip,
 			    struct snd_usb_endpoint *ep)
 {
 	struct snd_usb_clock_ref *clock = ep->clock_ref;
-	int err;
+	int rate, err;
 
-	if (clock) {
-		if (atomic_read(&clock->locked))
-			return 0;
-		if (clock->rate == ep->cur_rate)
-			return 0;
-		if (clock->rate && clock->rate != ep->cur_rate) {
-			usb_audio_dbg(chip, "Mismatched sample rate %d vs %d for EP 0x%x\n",
-				      clock->rate, ep->cur_rate, ep->ep_num);
-			return -EINVAL;
-		}
-	}
+	rate = update_clock_ref_rate(chip, ep);
+	if (rate < 0)
+		return rate;
+	if (clock && !clock->need_setup)
+		return 0;
 
-	err = snd_usb_init_sample_rate(chip, ep->cur_audiofmt, ep->cur_rate);
-	if (err < 0)
+	err = snd_usb_init_sample_rate(chip, ep->cur_audiofmt, rate);
+	if (err < 0) {
+		if (clock)
+			clock->rate = 0; /* reset rate */
 		return err;
+	}
 
 	if (clock)
-		clock->rate = ep->cur_rate;
+		clock->need_setup = false;
 	return 0;
 }
 
 /*
- * snd_usb_endpoint_configure: Configure the endpoint
+ * snd_usb_endpoint_prepare: Prepare the endpoint
  *
  * This function sets up the EP to be fully usable state.
- * It's called either from hw_params or prepare callback.
+ * It's called either from prepare callback.
  * The function checks need_setup flag, and performs nothing unless needed,
  * so it's safe to call this multiple times.
  *
  * This returns zero if unchanged, 1 if the configuration has changed,
  * or a negative error code.
  */
-int snd_usb_endpoint_configure(struct snd_usb_audio *chip,
-			       struct snd_usb_endpoint *ep)
+int snd_usb_endpoint_prepare(struct snd_usb_audio *chip,
+			     struct snd_usb_endpoint *ep)
 {
 	bool iface_first;
 	int err = 0;
@@ -1412,9 +1439,6 @@ int snd_usb_endpoint_configure(struct snd_usb_audio *chip,
 			if (err < 0)
 				goto unlock;
 		}
-		err = snd_usb_endpoint_set_params(chip, ep);
-		if (err < 0)
-			goto unlock;
 		goto done;
 	}
 
@@ -1442,10 +1466,6 @@ int snd_usb_endpoint_configure(struct snd_usb_audio *chip,
 	if (err < 0)
 		goto unlock;
 
-	err = snd_usb_endpoint_set_params(chip, ep);
-	if (err < 0)
-		goto unlock;
-
 	err = snd_usb_select_mode_quirk(chip, ep->cur_audiofmt);
 	if (err < 0)
 		goto unlock;
@@ -1633,8 +1653,7 @@ void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep, bool keep_pending)
 			WRITE_ONCE(ep->sync_source->sync_sink, NULL);
 		stop_urbs(ep, false, keep_pending);
 		if (ep->clock_ref)
-			if (!atomic_dec_return(&ep->clock_ref->locked))
-				ep->clock_ref->rate = 0;
+			atomic_dec(&ep->clock_ref->locked);
 	}
 }
 
diff --git a/sound/usb/endpoint.h b/sound/usb/endpoint.h
index 6a9af04cf175..e67ea28faa54 100644
--- a/sound/usb/endpoint.h
+++ b/sound/usb/endpoint.h
@@ -17,8 +17,10 @@ snd_usb_endpoint_open(struct snd_usb_audio *chip,
 		      bool is_sync_ep);
 void snd_usb_endpoint_close(struct snd_usb_audio *chip,
 			    struct snd_usb_endpoint *ep);
-int snd_usb_endpoint_configure(struct snd_usb_audio *chip,
-			       struct snd_usb_endpoint *ep);
+int snd_usb_endpoint_set_params(struct snd_usb_audio *chip,
+				struct snd_usb_endpoint *ep);
+int snd_usb_endpoint_prepare(struct snd_usb_audio *chip,
+			     struct snd_usb_endpoint *ep);
 int snd_usb_endpoint_get_clock_rate(struct snd_usb_audio *chip, int clock);
 
 bool snd_usb_endpoint_compatible(struct snd_usb_audio *chip,
diff --git a/sound/usb/line6/driver.h b/sound/usb/line6/driver.h
index ecf3a2b39c7e..dbb1d90d3647 100644
--- a/sound/usb/line6/driver.h
+++ b/sound/usb/line6/driver.h
@@ -193,8 +193,6 @@ extern int line6_send_raw_message_async(struct usb_line6 *line6,
 					const char *buffer, int size);
 extern int line6_send_sysex_message(struct usb_line6 *line6,
 				    const char *buffer, int size);
-extern ssize_t line6_set_raw(struct device *dev, struct device_attribute *attr,
-			     const char *buf, size_t count);
 extern int line6_version_request_async(struct usb_line6 *line6);
 extern int line6_write_data(struct usb_line6 *line6, unsigned address,
 			    void *data, unsigned datalen);
diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c
index d45d1d7e6664..8ed165f036a0 100644
--- a/sound/usb/pcm.c
+++ b/sound/usb/pcm.c
@@ -433,35 +433,6 @@ static void close_endpoints(struct snd_usb_audio *chip,
 	}
 }
 
-static int configure_endpoints(struct snd_usb_audio *chip,
-			       struct snd_usb_substream *subs)
-{
-	int err;
-
-	if (subs->data_endpoint->need_setup) {
-		/* stop any running stream beforehand */
-		if (stop_endpoints(subs, false))
-			sync_pending_stops(subs);
-		if (subs->sync_endpoint) {
-			err = snd_usb_endpoint_configure(chip, subs->sync_endpoint);
-			if (err < 0)
-				return err;
-		}
-		err = snd_usb_endpoint_configure(chip, subs->data_endpoint);
-		if (err < 0)
-			return err;
-		snd_usb_set_format_quirk(subs, subs->cur_audiofmt);
-	} else {
-		if (subs->sync_endpoint) {
-			err = snd_usb_endpoint_configure(chip, subs->sync_endpoint);
-			if (err < 0)
-				return err;
-		}
-	}
-
-	return 0;
-}
-
 /*
  * hw_params callback
  *
@@ -551,7 +522,16 @@ static int snd_usb_hw_params(struct snd_pcm_substream *substream,
 	subs->cur_audiofmt = fmt;
 	mutex_unlock(&chip->mutex);
 
-	ret = configure_endpoints(chip, subs);
+	if (!subs->data_endpoint->need_setup)
+		goto unlock;
+
+	if (subs->sync_endpoint) {
+		ret = snd_usb_endpoint_set_params(chip, subs->sync_endpoint);
+		if (ret < 0)
+			goto unlock;
+	}
+
+	ret = snd_usb_endpoint_set_params(chip, subs->data_endpoint);
 
  unlock:
 	if (ret < 0)
@@ -634,9 +614,18 @@ static int snd_usb_pcm_prepare(struct snd_pcm_substream *substream)
 		goto unlock;
 	}
 
-	ret = configure_endpoints(chip, subs);
+	if (subs->sync_endpoint) {
+		ret = snd_usb_endpoint_prepare(chip, subs->sync_endpoint);
+		if (ret < 0)
+			goto unlock;
+	}
+
+	ret = snd_usb_endpoint_prepare(chip, subs->data_endpoint);
 	if (ret < 0)
 		goto unlock;
+	else if (ret > 0)
+		snd_usb_set_format_quirk(subs, subs->cur_audiofmt);
+	ret = 0;
 
 	/* reset the pointer */
 	subs->buffer_bytes = frames_to_bytes(runtime, runtime->buffer_size);
@@ -1406,7 +1395,7 @@ static int prepare_playback_urb(struct snd_usb_substream *subs,
 	transfer_done = subs->transfer_done;
 
 	if (subs->lowlatency_playback &&
-	    runtime->status->state != SNDRV_PCM_STATE_DRAINING) {
+	    runtime->state != SNDRV_PCM_STATE_DRAINING) {
 		unsigned int hwptr = subs->hwptr_done / stride;
 
 		/* calculate the byte offset-in-buffer of the appl_ptr */
@@ -1594,7 +1583,7 @@ static int snd_usb_substream_playback_trigger(struct snd_pcm_substream *substrea
 		return 0;
 	case SNDRV_PCM_TRIGGER_SUSPEND:
 	case SNDRV_PCM_TRIGGER_STOP:
-		stop_endpoints(subs, substream->runtime->status->state == SNDRV_PCM_STATE_DRAINING);
+		stop_endpoints(subs, substream->runtime->state == SNDRV_PCM_STATE_DRAINING);
 		snd_usb_endpoint_set_callback(subs->data_endpoint,
 					      NULL, NULL, NULL);
 		subs->running = 0;
diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h
index f93201a830b5..06dfdd45cff8 100644
--- a/sound/usb/quirks-table.h
+++ b/sound/usb/quirks-table.h
@@ -2985,6 +2985,82 @@ YAMAHA_DEVICE(0x7010, "UB99"),
 		}
 	}
 },
+/* DIGIDESIGN MBOX 3 */
+{
+	USB_DEVICE(0x0dba, 0x5000),
+	.driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) {
+		.vendor_name = "Digidesign",
+		.product_name = "Mbox 3",
+		.ifnum = QUIRK_ANY_INTERFACE,
+		.type = QUIRK_COMPOSITE,
+		.data = (const struct snd_usb_audio_quirk[]) {
+			{
+				.ifnum = 0,
+				.type = QUIRK_IGNORE_INTERFACE
+			},
+			{
+				.ifnum = 1,
+				.type = QUIRK_IGNORE_INTERFACE
+			},
+			{
+				.ifnum = 2,
+				.type = QUIRK_AUDIO_FIXED_ENDPOINT,
+				.data = &(const struct audioformat) {
+					.formats = SNDRV_PCM_FMTBIT_S24_3LE,
+					.channels = 4,
+					.iface = 2,
+					.altsetting = 1,
+					.altset_idx = 1,
+					.attributes = 0x00,
+					.endpoint = 0x01,
+					.ep_attr = USB_ENDPOINT_XFER_ISOC |
+						USB_ENDPOINT_SYNC_ASYNC,
+					.rates = SNDRV_PCM_RATE_48000,
+					.rate_min = 48000,
+					.rate_max = 48000,
+					.nr_rates = 1,
+					.rate_table = (unsigned int[]) {
+						48000
+					}
+				}
+			},
+			{
+				.ifnum = 3,
+				.type = QUIRK_AUDIO_FIXED_ENDPOINT,
+				.data = &(const struct audioformat) {
+					.formats = SNDRV_PCM_FMTBIT_S24_3LE,
+					.channels = 4,
+					.iface = 3,
+					.altsetting = 1,
+					.altset_idx = 1,
+					.endpoint = 0x81,
+					.attributes = 0x00,
+					.ep_attr = USB_ENDPOINT_XFER_ISOC |
+						USB_ENDPOINT_SYNC_ASYNC,
+					.maxpacksize = 0x009c,
+					.rates = SNDRV_PCM_RATE_48000,
+					.rate_min = 48000,
+					.rate_max = 48000,
+					.nr_rates = 1,
+					.rate_table = (unsigned int[]) {
+						48000
+					}
+				}
+			},
+			{
+				.ifnum = 4,
+				.type = QUIRK_MIDI_FIXED_ENDPOINT,
+				.data = &(const struct snd_usb_midi_endpoint_info) {
+					.out_cables = 0x0001,
+					.in_cables  = 0x0001
+				}
+			},
+			{
+				.ifnum = -1
+			}
+		}
+	}
+},
 {
 	/* Tascam US122 MKII - playback-only support */
 	USB_DEVICE_VENDOR_SPEC(0x0644, 0x8021),
diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c
index 5b4d8f5eade2..eadac586bcc8 100644
--- a/sound/usb/quirks.c
+++ b/sound/usb/quirks.c
@@ -1020,6 +1020,304 @@ static int snd_usb_axefx3_boot_quirk(struct usb_device *dev)
 	return 0;
 }
 
+static void mbox3_setup_48_24_magic(struct usb_device *dev)
+{
+	/* The Mbox 3 is "little endian" */
+	/* max volume is: 0x0000. */
+	/* min volume is: 0x0080 (shown in little endian form) */
+
+
+	/* Load 48000Hz rate into buffer */
+	u8 com_buff[4] = {0x80, 0xbb, 0x00, 0x00};
+
+	/* Set 48000Hz sample rate */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			0x01, 0x21, 0x0100, 0x0001, &com_buff, 4);  //Is this really needed?
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			0x01, 0x21, 0x0100, 0x8101, &com_buff, 4);
+
+	/* Deactivate Tuner */
+	/* on  = 0x01*/
+	/* off = 0x00*/
+	com_buff[0] = 0x00;
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+		0x01, 0x21, 0x0003, 0x2001, &com_buff, 1);
+
+	/* Set clock source to Internal (as opposed to S/PDIF) */
+	com_buff[0] = 0x01;
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0100, 0x8001, &com_buff, 1);
+
+	/* Mute the hardware loopbacks to start the device in a known state. */
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x80;
+	/* Analogue input 1 left channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0110, 0x4001, &com_buff, 2);
+	/* Analogue input 1 right channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0111, 0x4001, &com_buff, 2);
+	/* Analogue input 2 left channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0114, 0x4001, &com_buff, 2);
+	/* Analogue input 2 right channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0115, 0x4001, &com_buff, 2);
+	/* Analogue input 3 left channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0118, 0x4001, &com_buff, 2);
+	/* Analogue input 3 right channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0119, 0x4001, &com_buff, 2);
+	/* Analogue input 4 left channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x011c, 0x4001, &com_buff, 2);
+	/* Analogue input 4 right channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x011d, 0x4001, &com_buff, 2);
+
+	/* Set software sends to output */
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x00;
+	/* Analogue software return 1 left channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0100, 0x4001, &com_buff, 2);
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x80;
+	/* Analogue software return 1 right channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0101, 0x4001, &com_buff, 2);
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x80;
+	/* Analogue software return 2 left channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0104, 0x4001, &com_buff, 2);
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x00;
+	/* Analogue software return 2 right channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0105, 0x4001, &com_buff, 2);
+
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x80;
+	/* Analogue software return 3 left channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0108, 0x4001, &com_buff, 2);
+	/* Analogue software return 3 right channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0109, 0x4001, &com_buff, 2);
+	/* Analogue software return 4 left channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x010c, 0x4001, &com_buff, 2);
+	/* Analogue software return 4 right channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x010d, 0x4001, &com_buff, 2);
+
+	/* Return to muting sends */
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x80;
+	/* Analogue fx return left channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0120, 0x4001, &com_buff, 2);
+	/* Analogue fx return right channel: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0121, 0x4001, &com_buff, 2);
+
+	/* Analogue software input 1 fx send: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0100, 0x4201, &com_buff, 2);
+	/* Analogue software input 2 fx send: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0101, 0x4201, &com_buff, 2);
+	/* Analogue software input 3 fx send: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0102, 0x4201, &com_buff, 2);
+	/* Analogue software input 4 fx send: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0103, 0x4201, &com_buff, 2);
+	/* Analogue input 1 fx send: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0104, 0x4201, &com_buff, 2);
+	/* Analogue input 2 fx send: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0105, 0x4201, &com_buff, 2);
+	/* Analogue input 3 fx send: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0106, 0x4201, &com_buff, 2);
+	/* Analogue input 4 fx send: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0107, 0x4201, &com_buff, 2);
+
+	/* Toggle allowing host control */
+	com_buff[0] = 0x02;
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			3, 0x21, 0x0000, 0x2001, &com_buff, 1);
+
+	/* Do not dim fx returns */
+	com_buff[0] = 0x00;
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			3, 0x21, 0x0002, 0x2001, &com_buff, 1);
+
+	/* Do not set fx returns to mono */
+	com_buff[0] = 0x00;
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			3, 0x21, 0x0001, 0x2001, &com_buff, 1);
+
+	/* Mute the S/PDIF hardware loopback
+	 * same odd volume logic here as above
+	 */
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x80;
+	/* S/PDIF hardware input 1 left channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0112, 0x4001, &com_buff, 2);
+	/* S/PDIF hardware input 1 right channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0113, 0x4001, &com_buff, 2);
+	/* S/PDIF hardware input 2 left channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0116, 0x4001, &com_buff, 2);
+	/* S/PDIF hardware input 2 right channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0117, 0x4001, &com_buff, 2);
+	/* S/PDIF hardware input 3 left channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x011a, 0x4001, &com_buff, 2);
+	/* S/PDIF hardware input 3 right channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x011b, 0x4001, &com_buff, 2);
+	/* S/PDIF hardware input 4 left channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x011e, 0x4001, &com_buff, 2);
+	/* S/PDIF hardware input 4 right channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x011f, 0x4001, &com_buff, 2);
+	/* S/PDIF software return 1 left channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0102, 0x4001, &com_buff, 2);
+	/* S/PDIF software return 1 right channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0103, 0x4001, &com_buff, 2);
+	/* S/PDIF software return 2 left channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0106, 0x4001, &com_buff, 2);
+	/* S/PDIF software return 2 right channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0107, 0x4001, &com_buff, 2);
+
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x00;
+	/* S/PDIF software return 3 left channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x010a, 0x4001, &com_buff, 2);
+
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x80;
+	/* S/PDIF software return 3 right channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x010b, 0x4001, &com_buff, 2);
+	/* S/PDIF software return 4 left channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x010e, 0x4001, &com_buff, 2);
+
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x00;
+	/* S/PDIF software return 4 right channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x010f, 0x4001, &com_buff, 2);
+
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x80;
+	/* S/PDIF fx returns left channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0122, 0x4001, &com_buff, 2);
+	/* S/PDIF fx returns right channel */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0123, 0x4001, &com_buff, 2);
+
+	/* Set the dropdown "Effect" to the first option */
+	/* Room1  = 0x00 */
+	/* Room2  = 0x01 */
+	/* Room3  = 0x02 */
+	/* Hall 1 = 0x03 */
+	/* Hall 2 = 0x04 */
+	/* Plate  = 0x05 */
+	/* Delay  = 0x06 */
+	/* Echo   = 0x07 */
+	com_buff[0] = 0x00;
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0200, 0x4301, &com_buff, 1);	/* max is 0xff */
+	/* min is 0x00 */
+
+
+	/* Set the effect duration to 0 */
+	/* max is 0xffff */
+	/* min is 0x0000 */
+	com_buff[0] = 0x00;
+	com_buff[1] = 0x00;
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0400, 0x4301, &com_buff, 2);
+
+	/* Set the effect volume and feedback to 0 */
+	/* max is 0xff */
+	/* min is 0x00 */
+	com_buff[0] = 0x00;
+	/* feedback: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0500, 0x4301, &com_buff, 1);
+	/* volume: */
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			1, 0x21, 0x0300, 0x4301, &com_buff, 1);
+
+	/* Set soft button hold duration */
+	/* 0x03 = 250ms */
+	/* 0x05 = 500ms DEFAULT */
+	/* 0x08 = 750ms */
+	/* 0x0a = 1sec */
+	com_buff[0] = 0x05;
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			3, 0x21, 0x0005, 0x2001, &com_buff, 1);
+
+	/* Use dim LEDs for button of state */
+	com_buff[0] = 0x00;
+	snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0),
+			3, 0x21, 0x0004, 0x2001, &com_buff, 1);
+}
+
+#define MBOX3_DESCRIPTOR_SIZE	464
+
+static int snd_usb_mbox3_boot_quirk(struct usb_device *dev)
+{
+	struct usb_host_config *config = dev->actconfig;
+	int err;
+	int descriptor_size;
+
+	descriptor_size = le16_to_cpu(get_cfg_desc(config)->wTotalLength);
+
+	if (descriptor_size != MBOX3_DESCRIPTOR_SIZE) {
+		dev_err(&dev->dev, "Invalid descriptor size=%d.\n", descriptor_size);
+		return -ENODEV;
+	}
+
+	dev_dbg(&dev->dev, "device initialised!\n");
+
+	err = usb_get_descriptor(dev, USB_DT_DEVICE, 0,
+		&dev->descriptor, sizeof(dev->descriptor));
+	config = dev->actconfig;
+	if (err < 0)
+		dev_dbg(&dev->dev, "error usb_get_descriptor: %d\n", err);
+
+	err = usb_reset_configuration(dev);
+	if (err < 0)
+		dev_dbg(&dev->dev, "error usb_reset_configuration: %d\n", err);
+	dev_dbg(&dev->dev, "mbox3_boot: new boot length = %d\n",
+		le16_to_cpu(get_cfg_desc(config)->wTotalLength));
+
+	mbox3_setup_48_24_magic(dev);
+	dev_info(&dev->dev, "Digidesign Mbox 3: 24bit 48kHz");
+
+	return 0; /* Successful boot */
+}
 
 #define MICROBOOK_BUF_SIZE 128
 
@@ -1324,6 +1622,10 @@ int snd_usb_apply_boot_quirk(struct usb_device *dev,
 	case USB_ID(0x0dba, 0x3000):
 		/* Digidesign Mbox 2 */
 		return snd_usb_mbox2_boot_quirk(dev);
+	case USB_ID(0x0dba, 0x5000):
+		/* Digidesign Mbox 3 */
+		return snd_usb_mbox3_boot_quirk(dev);
+
 
 	case USB_ID(0x1235, 0x0010): /* Focusrite Novation Saffire 6 USB */
 	case USB_ID(0x1235, 0x0018): /* Focusrite Novation Twitch */
@@ -1729,48 +2031,6 @@ void snd_usb_audioformat_attributes_quirk(struct snd_usb_audio *chip,
 }
 
 /*
- * registration quirk:
- * the registration is skipped if a device matches with the given ID,
- * unless the interface reaches to the defined one.  This is for delaying
- * the registration until the last known interface, so that the card and
- * devices appear at the same time.
- */
-
-struct registration_quirk {
-	unsigned int usb_id;	/* composed via USB_ID() */
-	unsigned int interface;	/* the interface to trigger register */
-};
-
-#define REG_QUIRK_ENTRY(vendor, product, iface) \
-	{ .usb_id = USB_ID(vendor, product), .interface = (iface) }
-
-static const struct registration_quirk registration_quirks[] = {
-	REG_QUIRK_ENTRY(0x0951, 0x16d8, 2),	/* Kingston HyperX AMP */
-	REG_QUIRK_ENTRY(0x0951, 0x16ed, 2),	/* Kingston HyperX Cloud Alpha S */
-	REG_QUIRK_ENTRY(0x0951, 0x16ea, 2),	/* Kingston HyperX Cloud Flight S */
-	REG_QUIRK_ENTRY(0x0ecb, 0x1f46, 2),	/* JBL Quantum 600 */
-	REG_QUIRK_ENTRY(0x0ecb, 0x1f47, 2),	/* JBL Quantum 800 */
-	REG_QUIRK_ENTRY(0x0ecb, 0x1f4c, 2),	/* JBL Quantum 400 */
-	REG_QUIRK_ENTRY(0x0ecb, 0x2039, 2),	/* JBL Quantum 400 */
-	REG_QUIRK_ENTRY(0x0ecb, 0x203c, 2),	/* JBL Quantum 600 */
-	REG_QUIRK_ENTRY(0x0ecb, 0x203e, 2),	/* JBL Quantum 800 */
-	{ 0 }					/* terminator */
-};
-
-/* return true if skipping registration */
-bool snd_usb_registration_quirk(struct snd_usb_audio *chip, int iface)
-{
-	const struct registration_quirk *q;
-
-	for (q = registration_quirks; q->usb_id; q++)
-		if (chip->usb_id == q->usb_id)
-			return iface < q->interface;
-
-	/* Register as normal */
-	return false;
-}
-
-/*
  * driver behavior quirk flags
  */
 struct usb_audio_quirk_flags_table {
diff --git a/sound/usb/quirks.h b/sound/usb/quirks.h
index 31abb7cb01a5..f9bfd5ac7bab 100644
--- a/sound/usb/quirks.h
+++ b/sound/usb/quirks.h
@@ -48,8 +48,6 @@ void snd_usb_audioformat_attributes_quirk(struct snd_usb_audio *chip,
 					  struct audioformat *fp,
 					  int stream);
 
-bool snd_usb_registration_quirk(struct snd_usb_audio *chip, int iface);
-
 void snd_usb_init_quirk_flags(struct snd_usb_audio *chip);
 
 #endif /* __USBAUDIO_QUIRKS_H */
diff --git a/sound/usb/stream.c b/sound/usb/stream.c
index f10f4e6d3fb8..f75601ca2d52 100644
--- a/sound/usb/stream.c
+++ b/sound/usb/stream.c
@@ -1222,12 +1222,6 @@ static int __snd_usb_parse_audio_interface(struct snd_usb_audio *chip,
 			if (err < 0)
 				return err;
 		}
-
-		/* try to set the interface... */
-		usb_set_interface(chip->dev, iface_no, 0);
-		snd_usb_init_pitch(chip, fp);
-		snd_usb_init_sample_rate(chip, fp, fp->rate_max);
-		usb_set_interface(chip->dev, iface_no, altno);
 	}
 	return 0;
 }
diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h
index ffbb4b0d09a0..2c6575029b1c 100644
--- a/sound/usb/usbaudio.h
+++ b/sound/usb/usbaudio.h
@@ -37,6 +37,7 @@ struct snd_usb_audio {
 	unsigned int quirk_flags;
 	unsigned int need_delayed_register:1; /* warn for delayed registration */
 	int num_interfaces;
+	int last_iface;
 	int num_suspended_intf;
 	int sample_rate_read_error;
 
diff --git a/sound/usb/usx2y/usbusx2yaudio.c b/sound/usb/usx2y/usbusx2yaudio.c
index 9cd5e3aae4f7..5197599e7aa6 100644
--- a/sound/usb/usx2y/usbusx2yaudio.c
+++ b/sound/usb/usx2y/usbusx2yaudio.c
@@ -822,8 +822,7 @@ static int snd_usx2y_pcm_hw_free(struct snd_pcm_substream *substream)
 		usx2y_urbs_release(subs);
 		if (!cap_subs->pcm_substream ||
 		    !cap_subs->pcm_substream->runtime ||
-		    !cap_subs->pcm_substream->runtime->status ||
-		    cap_subs->pcm_substream->runtime->status->state < SNDRV_PCM_STATE_PREPARED) {
+		    cap_subs->pcm_substream->runtime->state < SNDRV_PCM_STATE_PREPARED) {
 			atomic_set(&cap_subs->state, STATE_STOPPED);
 			usx2y_urbs_release(cap_subs);
 		}
diff --git a/sound/usb/usx2y/usx2yhwdeppcm.c b/sound/usb/usx2y/usx2yhwdeppcm.c
index 240349b644f3..767a227d54da 100644
--- a/sound/usb/usx2y/usx2yhwdeppcm.c
+++ b/sound/usb/usx2y/usx2yhwdeppcm.c
@@ -374,8 +374,7 @@ static int snd_usx2y_usbpcm_hw_free(struct snd_pcm_substream *substream)
 		usx2y_usbpcm_urbs_release(subs);
 		if (!cap_subs->pcm_substream ||
 		    !cap_subs->pcm_substream->runtime ||
-		    !cap_subs->pcm_substream->runtime->status ||
-		    cap_subs->pcm_substream->runtime->status->state < SNDRV_PCM_STATE_PREPARED) {
+		    cap_subs->pcm_substream->runtime->state < SNDRV_PCM_STATE_PREPARED) {
 			atomic_set(&cap_subs->state, STATE_STOPPED);
 			if (cap_subs2)
 				atomic_set(&cap_subs2->state, STATE_STOPPED);
diff --git a/tools/bpf/bpftool/Documentation/bpftool-map.rst b/tools/bpf/bpftool/Documentation/bpftool-map.rst
index 7c188a598444..7f3b67a8b48f 100644
--- a/tools/bpf/bpftool/Documentation/bpftool-map.rst
+++ b/tools/bpf/bpftool/Documentation/bpftool-map.rst
@@ -55,7 +55,7 @@ MAP COMMANDS
 |		| **devmap** | **devmap_hash** | **sockmap** | **cpumap** | **xskmap** | **sockhash**
 |		| **cgroup_storage** | **reuseport_sockarray** | **percpu_cgroup_storage**
 |		| **queue** | **stack** | **sk_storage** | **struct_ops** | **ringbuf** | **inode_storage**
-|		| **task_storage** | **bloom_filter** }
+|		| **task_storage** | **bloom_filter** | **user_ringbuf** }
 
 DESCRIPTION
 ===========
diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c
index 0744bd1150be..68a70ac03c80 100644
--- a/tools/bpf/bpftool/btf.c
+++ b/tools/bpf/bpftool/btf.c
@@ -43,11 +43,6 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = {
 	[BTF_KIND_ENUM64]	= "ENUM64",
 };
 
-struct btf_attach_point {
-	__u32 obj_id;
-	__u32 btf_id;
-};
-
 static const char *btf_int_enc_str(__u8 encoding)
 {
 	switch (encoding) {
@@ -640,10 +635,9 @@ static int do_dump(int argc, char **argv)
 
 		btf = btf__parse_split(*argv, base ?: base_btf);
 		err = libbpf_get_error(btf);
-		if (err) {
-			btf = NULL;
+		if (!btf) {
 			p_err("failed to load BTF from %s: %s",
-			      *argv, strerror(err));
+			      *argv, strerror(errno));
 			goto done;
 		}
 		NEXT_ARG();
@@ -688,8 +682,8 @@ static int do_dump(int argc, char **argv)
 
 		btf = btf__load_from_kernel_by_id_split(btf_id, base_btf);
 		err = libbpf_get_error(btf);
-		if (err) {
-			p_err("get btf by id (%u): %s", btf_id, strerror(err));
+		if (!btf) {
+			p_err("get btf by id (%u): %s", btf_id, strerror(errno));
 			goto done;
 		}
 	}
@@ -825,7 +819,7 @@ build_btf_type_table(struct hashmap *tab, enum bpf_obj_type type,
 				      u32_as_hash_field(id));
 		if (err) {
 			p_err("failed to append entry to hashmap for BTF ID %u, object ID %u: %s",
-			      btf_id, id, strerror(errno));
+			      btf_id, id, strerror(-err));
 			goto err_free;
 		}
 	}
diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c
index 125798b0bc5d..19924b6ce796 100644
--- a/tools/bpf/bpftool/btf_dumper.c
+++ b/tools/bpf/bpftool/btf_dumper.c
@@ -452,7 +452,7 @@ static int btf_dumper_int(const struct btf_type *t, __u8 bit_offset,
 					     *(char *)data);
 		break;
 	case BTF_INT_BOOL:
-		jsonw_bool(jw, *(int *)data);
+		jsonw_bool(jw, *(bool *)data);
 		break;
 	default:
 		/* shouldn't happen */
diff --git a/tools/bpf/bpftool/cgroup.c b/tools/bpf/bpftool/cgroup.c
index cced668fb2a3..b46a998d8f8d 100644
--- a/tools/bpf/bpftool/cgroup.c
+++ b/tools/bpf/bpftool/cgroup.c
@@ -136,8 +136,8 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type,
 			jsonw_string_field(json_wtr, "attach_type", attach_type_str);
 		else
 			jsonw_uint_field(json_wtr, "attach_type", attach_type);
-		jsonw_string_field(json_wtr, "attach_flags",
-				   attach_flags_str);
+		if (!(query_flags & BPF_F_QUERY_EFFECTIVE))
+			jsonw_string_field(json_wtr, "attach_flags", attach_flags_str);
 		jsonw_string_field(json_wtr, "name", prog_name);
 		if (attach_btf_name)
 			jsonw_string_field(json_wtr, "attach_btf_name", attach_btf_name);
@@ -150,7 +150,10 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type,
 			printf("%-15s", attach_type_str);
 		else
 			printf("type %-10u", attach_type);
-		printf(" %-15s %-15s", attach_flags_str, prog_name);
+		if (query_flags & BPF_F_QUERY_EFFECTIVE)
+			printf(" %-15s", prog_name);
+		else
+			printf(" %-15s %-15s", attach_flags_str, prog_name);
 		if (attach_btf_name)
 			printf(" %-15s", attach_btf_name);
 		else if (info.attach_btf_id)
@@ -195,6 +198,32 @@ static int cgroup_has_attached_progs(int cgroup_fd)
 
 	return no_prog ? 0 : 1;
 }
+
+static int show_effective_bpf_progs(int cgroup_fd, enum bpf_attach_type type,
+				    int level)
+{
+	LIBBPF_OPTS(bpf_prog_query_opts, p);
+	__u32 prog_ids[1024] = {0};
+	__u32 iter;
+	int ret;
+
+	p.query_flags = query_flags;
+	p.prog_cnt = ARRAY_SIZE(prog_ids);
+	p.prog_ids = prog_ids;
+
+	ret = bpf_prog_query_opts(cgroup_fd, type, &p);
+	if (ret)
+		return ret;
+
+	if (p.prog_cnt == 0)
+		return 0;
+
+	for (iter = 0; iter < p.prog_cnt; iter++)
+		show_bpf_prog(prog_ids[iter], type, NULL, level);
+
+	return 0;
+}
+
 static int show_attached_bpf_progs(int cgroup_fd, enum bpf_attach_type type,
 				   int level)
 {
@@ -245,6 +274,14 @@ static int show_attached_bpf_progs(int cgroup_fd, enum bpf_attach_type type,
 	return 0;
 }
 
+static int show_bpf_progs(int cgroup_fd, enum bpf_attach_type type,
+			  int level)
+{
+	return query_flags & BPF_F_QUERY_EFFECTIVE ?
+	       show_effective_bpf_progs(cgroup_fd, type, level) :
+	       show_attached_bpf_progs(cgroup_fd, type, level);
+}
+
 static int do_show(int argc, char **argv)
 {
 	enum bpf_attach_type type;
@@ -292,6 +329,8 @@ static int do_show(int argc, char **argv)
 
 	if (json_output)
 		jsonw_start_array(json_wtr);
+	else if (query_flags & BPF_F_QUERY_EFFECTIVE)
+		printf("%-8s %-15s %-15s\n", "ID", "AttachType", "Name");
 	else
 		printf("%-8s %-15s %-15s %-15s\n", "ID", "AttachType",
 		       "AttachFlags", "Name");
@@ -304,7 +343,7 @@ static int do_show(int argc, char **argv)
 		 * If we were able to get the show for at least one
 		 * attach type, let's return 0.
 		 */
-		if (show_attached_bpf_progs(cgroup_fd, type, 0) == 0)
+		if (show_bpf_progs(cgroup_fd, type, 0) == 0)
 			ret = 0;
 	}
 
@@ -362,7 +401,7 @@ static int do_show_tree_fn(const char *fpath, const struct stat *sb,
 
 	btf_vmlinux = libbpf_find_kernel_btf();
 	for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++)
-		show_attached_bpf_progs(cgroup_fd, type, ftw->level);
+		show_bpf_progs(cgroup_fd, type, ftw->level);
 
 	if (errno == EINVAL)
 		/* Last attach type does not support query.
@@ -436,6 +475,11 @@ static int do_show_tree(int argc, char **argv)
 
 	if (json_output)
 		jsonw_start_array(json_wtr);
+	else if (query_flags & BPF_F_QUERY_EFFECTIVE)
+		printf("%s\n"
+		       "%-8s %-15s %-15s\n",
+		       "CgroupPath",
+		       "ID", "AttachType", "Name");
 	else
 		printf("%s\n"
 		       "%-8s %-15s %-15s %-15s\n",
diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c
index 067e9ea59e3b..8727765add88 100644
--- a/tools/bpf/bpftool/common.c
+++ b/tools/bpf/bpftool/common.c
@@ -722,6 +722,7 @@ print_all_levels(__maybe_unused enum libbpf_print_level level,
 
 static int prog_fd_by_nametag(void *nametag, int **fds, bool tag)
 {
+	char prog_name[MAX_PROG_FULL_NAME];
 	unsigned int id = 0;
 	int fd, nb_fds = 0;
 	void *tmp;
@@ -754,12 +755,20 @@ static int prog_fd_by_nametag(void *nametag, int **fds, bool tag)
 			goto err_close_fd;
 		}
 
-		if ((tag && memcmp(nametag, info.tag, BPF_TAG_SIZE)) ||
-		    (!tag && strncmp(nametag, info.name, BPF_OBJ_NAME_LEN))) {
+		if (tag && memcmp(nametag, info.tag, BPF_TAG_SIZE)) {
 			close(fd);
 			continue;
 		}
 
+		if (!tag) {
+			get_prog_full_name(&info, fd, prog_name,
+					   sizeof(prog_name));
+			if (strncmp(nametag, prog_name, sizeof(prog_name))) {
+				close(fd);
+				continue;
+			}
+		}
+
 		if (nb_fds > 0) {
 			tmp = realloc(*fds, (nb_fds + 1) * sizeof(int));
 			if (!tmp) {
@@ -820,7 +829,7 @@ int prog_parse_fds(int *argc, char ***argv, int **fds)
 		NEXT_ARGP();
 
 		name = **argv;
-		if (strlen(name) > BPF_OBJ_NAME_LEN - 1) {
+		if (strlen(name) > MAX_PROG_FULL_NAME - 1) {
 			p_err("can't parse name");
 			return -1;
 		}
diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c
index 7ecabf7947fb..36cf0f1517c9 100644
--- a/tools/bpf/bpftool/feature.c
+++ b/tools/bpf/bpftool/feature.c
@@ -1147,7 +1147,7 @@ exit_free:
 	return res;
 #else
 	/* Detection assumes user has specific privileges.
-	 * We do not use libpcap so let's approximate, and restrict usage to
+	 * We do not use libcap so let's approximate, and restrict usage to
 	 * root user only.
 	 */
 	if (geteuid()) {
diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c
index 7070dcffa822..cf8b4e525c88 100644
--- a/tools/bpf/bpftool/gen.c
+++ b/tools/bpf/bpftool/gen.c
@@ -1594,14 +1594,14 @@ static int do_object(int argc, char **argv)
 
 		err = bpf_linker__add_file(linker, file, NULL);
 		if (err) {
-			p_err("failed to link '%s': %s (%d)", file, strerror(err), err);
+			p_err("failed to link '%s': %s (%d)", file, strerror(errno), errno);
 			goto out;
 		}
 	}
 
 	err = bpf_linker__finalize(linker);
 	if (err) {
-		p_err("failed to finalize ELF file: %s (%d)", strerror(err), err);
+		p_err("failed to finalize ELF file: %s (%d)", strerror(errno), errno);
 		goto out;
 	}
 
diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c
index 7a20931c3250..2863639706dd 100644
--- a/tools/bpf/bpftool/link.c
+++ b/tools/bpf/bpftool/link.c
@@ -83,6 +83,36 @@ static bool is_iter_map_target(const char *target_name)
 	       strcmp(target_name, "bpf_sk_storage_map") == 0;
 }
 
+static bool is_iter_cgroup_target(const char *target_name)
+{
+	return strcmp(target_name, "cgroup") == 0;
+}
+
+static const char *cgroup_order_string(__u32 order)
+{
+	switch (order) {
+	case BPF_CGROUP_ITER_ORDER_UNSPEC:
+		return "order_unspec";
+	case BPF_CGROUP_ITER_SELF_ONLY:
+		return "self_only";
+	case BPF_CGROUP_ITER_DESCENDANTS_PRE:
+		return "descendants_pre";
+	case BPF_CGROUP_ITER_DESCENDANTS_POST:
+		return "descendants_post";
+	case BPF_CGROUP_ITER_ANCESTORS_UP:
+		return "ancestors_up";
+	default: /* won't happen */
+		return "unknown";
+	}
+}
+
+static bool is_iter_task_target(const char *target_name)
+{
+	return strcmp(target_name, "task") == 0 ||
+		strcmp(target_name, "task_file") == 0 ||
+		strcmp(target_name, "task_vma") == 0;
+}
+
 static void show_iter_json(struct bpf_link_info *info, json_writer_t *wtr)
 {
 	const char *target_name = u64_to_ptr(info->iter.target_name);
@@ -91,6 +121,18 @@ static void show_iter_json(struct bpf_link_info *info, json_writer_t *wtr)
 
 	if (is_iter_map_target(target_name))
 		jsonw_uint_field(wtr, "map_id", info->iter.map.map_id);
+	else if (is_iter_task_target(target_name)) {
+		if (info->iter.task.tid)
+			jsonw_uint_field(wtr, "tid", info->iter.task.tid);
+		else if (info->iter.task.pid)
+			jsonw_uint_field(wtr, "pid", info->iter.task.pid);
+	}
+
+	if (is_iter_cgroup_target(target_name)) {
+		jsonw_lluint_field(wtr, "cgroup_id", info->iter.cgroup.cgroup_id);
+		jsonw_string_field(wtr, "order",
+				   cgroup_order_string(info->iter.cgroup.order));
+	}
 }
 
 static int get_prog_info(int prog_id, struct bpf_prog_info *info)
@@ -208,6 +250,18 @@ static void show_iter_plain(struct bpf_link_info *info)
 
 	if (is_iter_map_target(target_name))
 		printf("map_id %u  ", info->iter.map.map_id);
+	else if (is_iter_task_target(target_name)) {
+		if (info->iter.task.tid)
+			printf("tid %u ", info->iter.task.tid);
+		else if (info->iter.task.pid)
+			printf("pid %u ", info->iter.task.pid);
+	}
+
+	if (is_iter_cgroup_target(target_name)) {
+		printf("cgroup_id %llu  ", info->iter.cgroup.cgroup_id);
+		printf("order %s  ",
+		       cgroup_order_string(info->iter.cgroup.order));
+	}
 }
 
 static int show_link_close_plain(int fd, struct bpf_link_info *info)
diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c
index 451cefc2d0da..ccd7457f92bf 100644
--- a/tools/bpf/bpftool/main.c
+++ b/tools/bpf/bpftool/main.c
@@ -435,6 +435,16 @@ int main(int argc, char **argv)
 
 	setlinebuf(stdout);
 
+#ifdef USE_LIBCAP
+	/* Libcap < 2.63 hooks before main() to compute the number of
+	 * capabilities of the running kernel, and doing so it calls prctl()
+	 * which may fail and set errno to non-zero.
+	 * Let's reset errno to make sure this does not interfere with the
+	 * batch mode.
+	 */
+	errno = 0;
+#endif
+
 	last_do_help = do_help;
 	pretty_output = false;
 	json_output = false;
diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c
index 38b6bc9c26c3..9a6ca9f31133 100644
--- a/tools/bpf/bpftool/map.c
+++ b/tools/bpf/bpftool/map.c
@@ -1459,7 +1459,7 @@ static int do_help(int argc, char **argv)
 		"                 devmap | devmap_hash | sockmap | cpumap | xskmap | sockhash |\n"
 		"                 cgroup_storage | reuseport_sockarray | percpu_cgroup_storage |\n"
 		"                 queue | stack | sk_storage | struct_ops | ringbuf | inode_storage |\n"
-		"                 task_storage | bloom_filter }\n"
+		"                 task_storage | bloom_filter | user_ringbuf }\n"
 		"       " HELP_SPEC_OPTIONS " |\n"
 		"                    {-f|--bpffs} | {-n|--nomount} }\n"
 		"",
diff --git a/tools/bpf/bpftool/map_perf_ring.c b/tools/bpf/bpftool/map_perf_ring.c
index 6b0c410152de..21d7d447e1f3 100644
--- a/tools/bpf/bpftool/map_perf_ring.c
+++ b/tools/bpf/bpftool/map_perf_ring.c
@@ -29,13 +29,6 @@
 
 static volatile bool stop;
 
-struct event_ring_info {
-	int fd;
-	int key;
-	unsigned int cpu;
-	void *mem;
-};
-
 struct perf_event_sample {
 	struct perf_event_header header;
 	__u64 time;
@@ -195,10 +188,9 @@ int do_event_pipe(int argc, char **argv)
 	opts.map_keys = &ctx.idx;
 	pb = perf_buffer__new_raw(map_fd, MMAP_PAGE_CNT, &perf_attr,
 				  print_bpf_output, &ctx, &opts);
-	err = libbpf_get_error(pb);
-	if (err) {
+	if (!pb) {
 		p_err("failed to create perf buffer: %s (%d)",
-		      strerror(err), err);
+		      strerror(errno), errno);
 		goto err_close_map;
 	}
 
@@ -213,7 +205,7 @@ int do_event_pipe(int argc, char **argv)
 		err = perf_buffer__poll(pb, 200);
 		if (err < 0 && err != -EINTR) {
 			p_err("perf buffer polling failed: %s (%d)",
-			      strerror(err), err);
+			      strerror(errno), errno);
 			goto err_close_pb;
 		}
 	}
diff --git a/tools/include/linux/gfp.h b/tools/include/linux/gfp.h
index b238dbc9eb85..6a10ff5f5be9 100644
--- a/tools/include/linux/gfp.h
+++ b/tools/include/linux/gfp.h
@@ -3,26 +3,7 @@
 #define _TOOLS_INCLUDE_LINUX_GFP_H
 
 #include <linux/types.h>
-
-#define __GFP_BITS_SHIFT 26
-#define __GFP_BITS_MASK ((gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
-
-#define __GFP_HIGH		0x20u
-#define __GFP_IO		0x40u
-#define __GFP_FS		0x80u
-#define __GFP_NOWARN		0x200u
-#define __GFP_ZERO		0x8000u
-#define __GFP_ATOMIC		0x80000u
-#define __GFP_ACCOUNT		0x100000u
-#define __GFP_DIRECT_RECLAIM	0x400000u
-#define __GFP_KSWAPD_RECLAIM	0x2000000u
-
-#define __GFP_RECLAIM	(__GFP_DIRECT_RECLAIM | __GFP_KSWAPD_RECLAIM)
-
-#define GFP_ZONEMASK	0x0fu
-#define GFP_ATOMIC	(__GFP_HIGH | __GFP_ATOMIC | __GFP_KSWAPD_RECLAIM)
-#define GFP_KERNEL	(__GFP_RECLAIM | __GFP_IO | __GFP_FS)
-#define GFP_NOWAIT	(__GFP_KSWAPD_RECLAIM)
+#include <linux/gfp_types.h>
 
 static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags)
 {
diff --git a/tools/include/linux/gfp_types.h b/tools/include/linux/gfp_types.h
new file mode 100644
index 000000000000..5f9f1ed190a0
--- /dev/null
+++ b/tools/include/linux/gfp_types.h
@@ -0,0 +1 @@
+#include "../../../include/linux/gfp_types.h"
diff --git a/tools/include/linux/kallsyms.h b/tools/include/linux/kallsyms.h
index efb6c3f5f2a9..5a37ccbec54f 100644
--- a/tools/include/linux/kallsyms.h
+++ b/tools/include/linux/kallsyms.h
@@ -6,7 +6,7 @@
 #include <stdio.h>
 #include <unistd.h>
 
-#define KSYM_NAME_LEN 128
+#define KSYM_NAME_LEN 512
 
 struct module;
 
diff --git a/tools/include/nolibc/arch-riscv.h b/tools/include/nolibc/arch-riscv.h
index 95e2b7924925..ba04771cb3a3 100644
--- a/tools/include/nolibc/arch-riscv.h
+++ b/tools/include/nolibc/arch-riscv.h
@@ -190,7 +190,7 @@ __asm__ (".section .text\n"
     ".option norelax\n"
     "lla   gp, __global_pointer$\n"
     ".option pop\n"
-    "ld    a0, 0(sp)\n"          // argc (a0) was in the stack
+    "lw    a0, 0(sp)\n"          // argc (a0) was in the stack
     "add   a1, sp, "SZREG"\n"    // argv (a1) = sp
     "slli  a2, a0, "PTRLOG"\n"   // envp (a2) = SZREG*argc ...
     "add   a2, a2, "SZREG"\n"    //             + SZREG (skip null)
diff --git a/tools/include/nolibc/sys.h b/tools/include/nolibc/sys.h
index 08491070387b..ce3ee03aa679 100644
--- a/tools/include/nolibc/sys.h
+++ b/tools/include/nolibc/sys.h
@@ -692,12 +692,12 @@ void *sys_mmap(void *addr, size_t length, int prot, int flags, int fd,
 {
 #ifndef my_syscall6
 	/* Function not implemented. */
-	return -ENOSYS;
+	return (void *)-ENOSYS;
 #else
 
 	int n;
 
-#if defined(__i386__)
+#if defined(__NR_mmap2)
 	n = __NR_mmap2;
 	offset >>= 12;
 #else
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 59a217ca2dfd..51b9aa640ad2 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -87,10 +87,35 @@ struct bpf_cgroup_storage_key {
 	__u32	attach_type;		/* program attach type (enum bpf_attach_type) */
 };
 
+enum bpf_cgroup_iter_order {
+	BPF_CGROUP_ITER_ORDER_UNSPEC = 0,
+	BPF_CGROUP_ITER_SELF_ONLY,		/* process only a single object. */
+	BPF_CGROUP_ITER_DESCENDANTS_PRE,	/* walk descendants in pre-order. */
+	BPF_CGROUP_ITER_DESCENDANTS_POST,	/* walk descendants in post-order. */
+	BPF_CGROUP_ITER_ANCESTORS_UP,		/* walk ancestors upward. */
+};
+
 union bpf_iter_link_info {
 	struct {
 		__u32	map_fd;
 	} map;
+	struct {
+		enum bpf_cgroup_iter_order order;
+
+		/* At most one of cgroup_fd and cgroup_id can be non-zero. If
+		 * both are zero, the walk starts from the default cgroup v2
+		 * root. For walking v1 hierarchy, one should always explicitly
+		 * specify cgroup_fd.
+		 */
+		__u32	cgroup_fd;
+		__u64	cgroup_id;
+	} cgroup;
+	/* Parameters of task iterators. */
+	struct {
+		__u32	tid;
+		__u32	pid;
+		__u32	pid_fd;
+	} task;
 };
 
 /* BPF syscall commands, see bpf(2) man-page for more details. */
@@ -909,6 +934,7 @@ enum bpf_map_type {
 	BPF_MAP_TYPE_INODE_STORAGE,
 	BPF_MAP_TYPE_TASK_STORAGE,
 	BPF_MAP_TYPE_BLOOM_FILTER,
+	BPF_MAP_TYPE_USER_RINGBUF,
 };
 
 /* Note that tracing related programs such as
@@ -1233,7 +1259,7 @@ enum {
 
 /* Query effective (directly attached + inherited from ancestor cgroups)
  * programs that will be executed for events within a cgroup.
- * attach_flags with this flag are returned only for directly attached programs.
+ * attach_flags with this flag are always returned 0.
  */
 #define BPF_F_QUERY_EFFECTIVE	(1U << 0)
 
@@ -1432,7 +1458,10 @@ union bpf_attr {
 		__u32		attach_flags;
 		__aligned_u64	prog_ids;
 		__u32		prog_cnt;
-		__aligned_u64	prog_attach_flags; /* output: per-program attach_flags */
+		/* output: per-program attach_flags.
+		 * not allowed to be set during effective query.
+		 */
+		__aligned_u64	prog_attach_flags;
 	} query;
 
 	struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */
@@ -2573,10 +2602,12 @@ union bpf_attr {
  *		There are two supported modes at this time:
  *
  *		* **BPF_ADJ_ROOM_MAC**: Adjust room at the mac layer
- *		  (room space is added or removed below the layer 2 header).
+ * 		  (room space is added or removed between the layer 2 and
+ * 		  layer 3 headers).
  *
  * 		* **BPF_ADJ_ROOM_NET**: Adjust room at the network layer
- * 		  (room space is added or removed below the layer 3 header).
+ * 		  (room space is added or removed between the layer 3 and
+ * 		  layer 4 headers).
  *
  *		The following flags are supported at this time:
  *
@@ -3008,8 +3039,18 @@ union bpf_attr {
  * 		**BPF_F_USER_STACK**
  * 			Collect a user space stack instead of a kernel stack.
  * 		**BPF_F_USER_BUILD_ID**
- * 			Collect buildid+offset instead of ips for user stack,
- * 			only valid if **BPF_F_USER_STACK** is also specified.
+ * 			Collect (build_id, file_offset) instead of ips for user
+ * 			stack, only valid if **BPF_F_USER_STACK** is also
+ * 			specified.
+ *
+ * 			*file_offset* is an offset relative to the beginning
+ * 			of the executable or shared object file backing the vma
+ * 			which the *ip* falls in. It is *not* an offset relative
+ * 			to that object's base address. Accordingly, it must be
+ * 			adjusted by adding (sh_addr - sh_offset), where
+ * 			sh_{addr,offset} correspond to the executable section
+ * 			containing *file_offset* in the object, for comparisons
+ * 			to symbols' st_value to be valid.
  *
  * 		**bpf_get_stack**\ () can collect up to
  * 		**PERF_MAX_STACK_DEPTH** both kernel and user frames, subject
@@ -4425,7 +4466,7 @@ union bpf_attr {
  *
  *		**-EEXIST** if the option already exists.
  *
- *		**-EFAULT** on failrue to parse the existing header options.
+ *		**-EFAULT** on failure to parse the existing header options.
  *
  *		**-EPERM** if the helper cannot be used under the current
  *		*skops*\ **->op**.
@@ -4634,7 +4675,7 @@ union bpf_attr {
  *		a *map* with *task* as the **key**.  From this
  *		perspective,  the usage is not much different from
  *		**bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this
- *		helper enforces the key must be an task_struct and the map must also
+ *		helper enforces the key must be a task_struct and the map must also
  *		be a **BPF_MAP_TYPE_TASK_STORAGE**.
  *
  *		Underneath, the value is stored locally at *task* instead of
@@ -4692,7 +4733,7 @@ union bpf_attr {
  *
  * long bpf_ima_inode_hash(struct inode *inode, void *dst, u32 size)
  *	Description
- *		Returns the stored IMA hash of the *inode* (if it's avaialable).
+ *		Returns the stored IMA hash of the *inode* (if it's available).
  *		If the hash is larger than *size*, then only *size*
  *		bytes will be copied to *dst*
  *	Return
@@ -4716,12 +4757,12 @@ union bpf_attr {
  *
  *		The argument *len_diff* can be used for querying with a planned
  *		size change. This allows to check MTU prior to changing packet
- *		ctx. Providing an *len_diff* adjustment that is larger than the
+ *		ctx. Providing a *len_diff* adjustment that is larger than the
  *		actual packet size (resulting in negative packet size) will in
- *		principle not exceed the MTU, why it is not considered a
- *		failure.  Other BPF-helpers are needed for performing the
- *		planned size change, why the responsability for catch a negative
- *		packet size belong in those helpers.
+ *		principle not exceed the MTU, which is why it is not considered
+ *		a failure.  Other BPF helpers are needed for performing the
+ *		planned size change; therefore the responsibility for catching
+ *		a negative packet size belongs in those helpers.
  *
  *		Specifying *ifindex* zero means the MTU check is performed
  *		against the current net device.  This is practical if this isn't
@@ -4919,6 +4960,7 @@ union bpf_attr {
  * 		Get address of the traced function (for tracing and kprobe programs).
  * 	Return
  * 		Address of the traced function.
+ * 		0 for kprobes placed within the function (not at the entry).
  *
  * u64 bpf_get_attach_cookie(void *ctx)
  * 	Description
@@ -5048,12 +5090,12 @@ union bpf_attr {
  *
  * long bpf_get_func_arg(void *ctx, u32 n, u64 *value)
  *	Description
- *		Get **n**-th argument (zero based) of the traced function (for tracing programs)
+ *		Get **n**-th argument register (zero based) of the traced function (for tracing programs)
  *		returned in **value**.
  *
  *	Return
  *		0 on success.
- *		**-EINVAL** if n >= arguments count of traced function.
+ *		**-EINVAL** if n >= argument register count of traced function.
  *
  * long bpf_get_func_ret(void *ctx, u64 *value)
  *	Description
@@ -5066,24 +5108,37 @@ union bpf_attr {
  *
  * long bpf_get_func_arg_cnt(void *ctx)
  *	Description
- *		Get number of arguments of the traced function (for tracing programs).
+ *		Get number of registers of the traced function (for tracing programs) where
+ *		function arguments are stored in these registers.
  *
  *	Return
- *		The number of arguments of the traced function.
+ *		The number of argument registers of the traced function.
  *
  * int bpf_get_retval(void)
  *	Description
- *		Get the syscall's return value that will be returned to userspace.
+ *		Get the BPF program's return value that will be returned to the upper layers.
  *
- *		This helper is currently supported by cgroup programs only.
+ *		This helper is currently supported by cgroup programs and only by the hooks
+ *		where BPF program's return value is returned to the userspace via errno.
  *	Return
- *		The syscall's return value.
+ *		The BPF program's return value.
  *
  * int bpf_set_retval(int retval)
  *	Description
- *		Set the syscall's return value that will be returned to userspace.
+ *		Set the BPF program's return value that will be returned to the upper layers.
+ *
+ *		This helper is currently supported by cgroup programs and only by the hooks
+ *		where BPF program's return value is returned to the userspace via errno.
+ *
+ *		Note that there is the following corner case where the program exports an error
+ *		via bpf_set_retval but signals success via 'return 1':
+ *
+ *			bpf_set_retval(-EPERM);
+ *			return 1;
+ *
+ *		In this case, the BPF program's return value will use helper's -EPERM. This
+ *		still holds true for cgroup/bind{4,6} which supports extra 'return 3' success case.
  *
- *		This helper is currently supported by cgroup programs only.
  *	Return
  *		0 on success, or a negative error in case of failure.
  *
@@ -5331,6 +5386,55 @@ union bpf_attr {
  *		**-EACCES** if the SYN cookie is not valid.
  *
  *		**-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin.
+ *
+ * u64 bpf_ktime_get_tai_ns(void)
+ *	Description
+ *		A nonsettable system-wide clock derived from wall-clock time but
+ *		ignoring leap seconds.  This clock does not experience
+ *		discontinuities and backwards jumps caused by NTP inserting leap
+ *		seconds as CLOCK_REALTIME does.
+ *
+ *		See: **clock_gettime**\ (**CLOCK_TAI**)
+ *	Return
+ *		Current *ktime*.
+ *
+ * long bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void *ctx, u64 flags)
+ *	Description
+ *		Drain samples from the specified user ring buffer, and invoke
+ *		the provided callback for each such sample:
+ *
+ *		long (\*callback_fn)(struct bpf_dynptr \*dynptr, void \*ctx);
+ *
+ *		If **callback_fn** returns 0, the helper will continue to try
+ *		and drain the next sample, up to a maximum of
+ *		BPF_MAX_USER_RINGBUF_SAMPLES samples. If the return value is 1,
+ *		the helper will skip the rest of the samples and return. Other
+ *		return values are not used now, and will be rejected by the
+ *		verifier.
+ *	Return
+ *		The number of drained samples if no error was encountered while
+ *		draining samples, or 0 if no samples were present in the ring
+ *		buffer. If a user-space producer was epoll-waiting on this map,
+ *		and at least one sample was drained, they will receive an event
+ *		notification notifying them of available space in the ring
+ *		buffer. If the BPF_RB_NO_WAKEUP flag is passed to this
+ *		function, no wakeup notification will be sent. If the
+ *		BPF_RB_FORCE_WAKEUP flag is passed, a wakeup notification will
+ *		be sent even if no sample was drained.
+ *
+ *		On failure, the returned value is one of the following:
+ *
+ *		**-EBUSY** if the ring buffer is contended, and another calling
+ *		context was concurrently draining the ring buffer.
+ *
+ *		**-EINVAL** if user-space is not properly tracking the ring
+ *		buffer due to the producer position not being aligned to 8
+ *		bytes, a sample not being aligned to 8 bytes, or the producer
+ *		position not matching the advertised length of a sample.
+ *
+ *		**-E2BIG** if user-space has tried to publish a sample which is
+ *		larger than the size of the ring buffer, or which cannot fit
+ *		within a struct bpf_dynptr.
  */
 #define __BPF_FUNC_MAPPER(FN)		\
 	FN(unspec),			\
@@ -5541,6 +5645,8 @@ union bpf_attr {
 	FN(tcp_raw_gen_syncookie_ipv6),	\
 	FN(tcp_raw_check_syncookie_ipv4),	\
 	FN(tcp_raw_check_syncookie_ipv6),	\
+	FN(ktime_get_tai_ns),		\
+	FN(user_ringbuf_drain),		\
 	/* */
 
 /* integer value in 'imm' field of BPF_CALL instruction selects which helper
@@ -5603,6 +5709,11 @@ enum {
 	BPF_F_SEQ_NUMBER		= (1ULL << 3),
 };
 
+/* BPF_FUNC_skb_get_tunnel_key flags. */
+enum {
+	BPF_F_TUNINFO_FLAGS		= (1ULL << 4),
+};
+
 /* BPF_FUNC_perf_event_output, BPF_FUNC_perf_event_read and
  * BPF_FUNC_perf_event_read_value flags.
  */
@@ -5792,7 +5903,10 @@ struct bpf_tunnel_key {
 	};
 	__u8 tunnel_tos;
 	__u8 tunnel_ttl;
-	__u16 tunnel_ext;	/* Padding, future use. */
+	union {
+		__u16 tunnel_ext;	/* compat */
+		__be16 tunnel_flags;
+	};
 	__u32 tunnel_label;
 	union {
 		__u32 local_ipv4;
@@ -5836,6 +5950,11 @@ enum bpf_ret_code {
 	 *    represented by BPF_REDIRECT above).
 	 */
 	BPF_LWT_REROUTE = 128,
+	/* BPF_FLOW_DISSECTOR_CONTINUE: used by BPF_PROG_TYPE_FLOW_DISSECTOR
+	 *   to indicate that no custom dissection was performed, and
+	 *   fallback to standard dissector is requested.
+	 */
+	BPF_FLOW_DISSECTOR_CONTINUE = 129,
 };
 
 struct bpf_sock {
@@ -6134,11 +6253,26 @@ struct bpf_link_info {
 		struct {
 			__aligned_u64 target_name; /* in/out: target_name buffer ptr */
 			__u32 target_name_len;	   /* in/out: target_name buffer len */
+
+			/* If the iter specific field is 32 bits, it can be put
+			 * in the first or second union. Otherwise it should be
+			 * put in the second union.
+			 */
 			union {
 				struct {
 					__u32 map_id;
 				} map;
 			};
+			union {
+				struct {
+					__u64 cgroup_id;
+					__u32 order;
+				} cgroup;
+				struct {
+					__u32 tid;
+					__u32 pid;
+				} task;
+			};
 		} iter;
 		struct  {
 			__u32 netns_ino;
diff --git a/tools/include/uapi/linux/tc_act/tc_bpf.h b/tools/include/uapi/linux/tc_act/tc_bpf.h
index 653c4f94f76e..fe6c8f8f3e8c 100644
--- a/tools/include/uapi/linux/tc_act/tc_bpf.h
+++ b/tools/include/uapi/linux/tc_act/tc_bpf.h
@@ -1,11 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */
 /*
  * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
  */
 
 #ifndef __LINUX_TC_BPF_H
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index efcc06dafbd9..1d49a0352836 100644
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -84,9 +84,7 @@ static inline int sys_bpf_fd(enum bpf_cmd cmd, union bpf_attr *attr,
 	return ensure_good_fd(fd);
 }
 
-#define PROG_LOAD_ATTEMPTS 5
-
-static inline int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts)
+int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts)
 {
 	int fd;
 
@@ -107,7 +105,7 @@ static inline int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int
  */
 int probe_memcg_account(void)
 {
-	const size_t prog_load_attr_sz = offsetofend(union bpf_attr, attach_btf_obj_fd);
+	const size_t attr_sz = offsetofend(union bpf_attr, attach_btf_obj_fd);
 	struct bpf_insn insns[] = {
 		BPF_EMIT_CALL(BPF_FUNC_ktime_get_coarse_ns),
 		BPF_EXIT_INSN(),
@@ -117,13 +115,13 @@ int probe_memcg_account(void)
 	int prog_fd;
 
 	/* attempt loading freplace trying to use custom BTF */
-	memset(&attr, 0, prog_load_attr_sz);
+	memset(&attr, 0, attr_sz);
 	attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
 	attr.insns = ptr_to_u64(insns);
 	attr.insn_cnt = insn_cnt;
 	attr.license = ptr_to_u64("GPL");
 
-	prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, prog_load_attr_sz);
+	prog_fd = sys_bpf_fd(BPF_PROG_LOAD, &attr, attr_sz);
 	if (prog_fd >= 0) {
 		close(prog_fd);
 		return 1;
@@ -183,7 +181,7 @@ int bpf_map_create(enum bpf_map_type map_type,
 		return libbpf_err(-EINVAL);
 
 	attr.map_type = map_type;
-	if (map_name)
+	if (map_name && kernel_supports(NULL, FEAT_PROG_NAME))
 		libbpf_strlcpy(attr.map_name, map_name, sizeof(attr.map_name));
 	attr.key_size = key_size;
 	attr.value_size = value_size;
@@ -234,6 +232,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type,
 		  const struct bpf_insn *insns, size_t insn_cnt,
 		  const struct bpf_prog_load_opts *opts)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, fd_array);
 	void *finfo = NULL, *linfo = NULL;
 	const char *func_info, *line_info;
 	__u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd;
@@ -253,7 +252,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type,
 	if (attempts == 0)
 		attempts = PROG_LOAD_ATTEMPTS;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 
 	attr.prog_type = prog_type;
 	attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0);
@@ -263,7 +262,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type,
 	attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0);
 	attr.kern_version = OPTS_GET(opts, kern_version, 0);
 
-	if (prog_name)
+	if (prog_name && kernel_supports(NULL, FEAT_PROG_NAME))
 		libbpf_strlcpy(attr.prog_name, prog_name, sizeof(attr.prog_name));
 	attr.license = ptr_to_u64(license);
 
@@ -316,7 +315,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type,
 		attr.log_level = log_level;
 	}
 
-	fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts);
+	fd = sys_bpf_prog_load(&attr, attr_sz, attempts);
 	if (fd >= 0)
 		return fd;
 
@@ -356,7 +355,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type,
 			break;
 		}
 
-		fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts);
+		fd = sys_bpf_prog_load(&attr, attr_sz, attempts);
 		if (fd >= 0)
 			goto done;
 	}
@@ -370,7 +369,7 @@ int bpf_prog_load(enum bpf_prog_type prog_type,
 		attr.log_size = log_size;
 		attr.log_level = 1;
 
-		fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts);
+		fd = sys_bpf_prog_load(&attr, attr_sz, attempts);
 	}
 done:
 	/* free() doesn't affect errno, so we don't need to restore it */
@@ -382,127 +381,136 @@ done:
 int bpf_map_update_elem(int fd, const void *key, const void *value,
 			__u64 flags)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.map_fd = fd;
 	attr.key = ptr_to_u64(key);
 	attr.value = ptr_to_u64(value);
 	attr.flags = flags;
 
-	ret = sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_map_lookup_elem(int fd, const void *key, void *value)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.map_fd = fd;
 	attr.key = ptr_to_u64(key);
 	attr.value = ptr_to_u64(value);
 
-	ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_map_lookup_elem_flags(int fd, const void *key, void *value, __u64 flags)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.map_fd = fd;
 	attr.key = ptr_to_u64(key);
 	attr.value = ptr_to_u64(value);
 	attr.flags = flags;
 
-	ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_MAP_LOOKUP_ELEM, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_map_lookup_and_delete_elem(int fd, const void *key, void *value)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.map_fd = fd;
 	attr.key = ptr_to_u64(key);
 	attr.value = ptr_to_u64(value);
 
-	ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key, void *value, __u64 flags)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.map_fd = fd;
 	attr.key = ptr_to_u64(key);
 	attr.value = ptr_to_u64(value);
 	attr.flags = flags;
 
-	ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_MAP_LOOKUP_AND_DELETE_ELEM, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_map_delete_elem(int fd, const void *key)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.map_fd = fd;
 	attr.key = ptr_to_u64(key);
 
-	ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, flags);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.map_fd = fd;
 	attr.key = ptr_to_u64(key);
 	attr.flags = flags;
 
-	ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_map_get_next_key(int fd, const void *key, void *next_key)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, next_key);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.map_fd = fd;
 	attr.key = ptr_to_u64(key);
 	attr.next_key = ptr_to_u64(next_key);
 
-	ret = sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_MAP_GET_NEXT_KEY, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_map_freeze(int fd)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, map_fd);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.map_fd = fd;
 
-	ret = sys_bpf(BPF_MAP_FREEZE, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_MAP_FREEZE, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
@@ -511,13 +519,14 @@ static int bpf_map_batch_common(int cmd, int fd, void  *in_batch,
 				__u32 *count,
 				const struct bpf_map_batch_opts *opts)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, batch);
 	union bpf_attr attr;
 	int ret;
 
 	if (!OPTS_VALID(opts, bpf_map_batch_opts))
 		return libbpf_err(-EINVAL);
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.batch.map_fd = fd;
 	attr.batch.in_batch = ptr_to_u64(in_batch);
 	attr.batch.out_batch = ptr_to_u64(out_batch);
@@ -527,7 +536,7 @@ static int bpf_map_batch_common(int cmd, int fd, void  *in_batch,
 	attr.batch.elem_flags  = OPTS_GET(opts, elem_flags, 0);
 	attr.batch.flags = OPTS_GET(opts, flags, 0);
 
-	ret = sys_bpf(cmd, &attr, sizeof(attr));
+	ret = sys_bpf(cmd, &attr, attr_sz);
 	*count = attr.batch.count;
 
 	return libbpf_err_errno(ret);
@@ -566,14 +575,15 @@ int bpf_map_update_batch(int fd, const void *keys, const void *values, __u32 *co
 
 int bpf_obj_pin(int fd, const char *pathname)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, file_flags);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.pathname = ptr_to_u64((void *)pathname);
 	attr.bpf_fd = fd;
 
-	ret = sys_bpf(BPF_OBJ_PIN, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_OBJ_PIN, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
@@ -584,17 +594,18 @@ int bpf_obj_get(const char *pathname)
 
 int bpf_obj_get_opts(const char *pathname, const struct bpf_obj_get_opts *opts)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, file_flags);
 	union bpf_attr attr;
 	int fd;
 
 	if (!OPTS_VALID(opts, bpf_obj_get_opts))
 		return libbpf_err(-EINVAL);
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.pathname = ptr_to_u64((void *)pathname);
 	attr.file_flags = OPTS_GET(opts, file_flags, 0);
 
-	fd = sys_bpf_fd(BPF_OBJ_GET, &attr, sizeof(attr));
+	fd = sys_bpf_fd(BPF_OBJ_GET, &attr, attr_sz);
 	return libbpf_err_errno(fd);
 }
 
@@ -612,52 +623,50 @@ int bpf_prog_attach_opts(int prog_fd, int target_fd,
 			  enum bpf_attach_type type,
 			  const struct bpf_prog_attach_opts *opts)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd);
 	union bpf_attr attr;
 	int ret;
 
 	if (!OPTS_VALID(opts, bpf_prog_attach_opts))
 		return libbpf_err(-EINVAL);
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.target_fd	   = target_fd;
 	attr.attach_bpf_fd = prog_fd;
 	attr.attach_type   = type;
 	attr.attach_flags  = OPTS_GET(opts, flags, 0);
 	attr.replace_bpf_fd = OPTS_GET(opts, replace_prog_fd, 0);
 
-	ret = sys_bpf(BPF_PROG_ATTACH, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_PROG_ATTACH, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
-__attribute__((alias("bpf_prog_attach_opts")))
-int bpf_prog_attach_xattr(int prog_fd, int target_fd,
-			  enum bpf_attach_type type,
-			  const struct bpf_prog_attach_opts *opts);
-
 int bpf_prog_detach(int target_fd, enum bpf_attach_type type)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.target_fd	 = target_fd;
 	attr.attach_type = type;
 
-	ret = sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_prog_detach2(int prog_fd, int target_fd, enum bpf_attach_type type)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, replace_bpf_fd);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.target_fd	 = target_fd;
 	attr.attach_bpf_fd = prog_fd;
 	attr.attach_type = type;
 
-	ret = sys_bpf(BPF_PROG_DETACH, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_PROG_DETACH, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
@@ -665,6 +674,7 @@ int bpf_link_create(int prog_fd, int target_fd,
 		    enum bpf_attach_type attach_type,
 		    const struct bpf_link_create_opts *opts)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, link_create);
 	__u32 target_btf_id, iter_info_len;
 	union bpf_attr attr;
 	int fd, err;
@@ -683,7 +693,7 @@ int bpf_link_create(int prog_fd, int target_fd,
 			return libbpf_err(-EINVAL);
 	}
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.link_create.prog_fd = prog_fd;
 	attr.link_create.target_fd = target_fd;
 	attr.link_create.attach_type = attach_type;
@@ -727,7 +737,7 @@ int bpf_link_create(int prog_fd, int target_fd,
 		break;
 	}
 proceed:
-	fd = sys_bpf_fd(BPF_LINK_CREATE, &attr, sizeof(attr));
+	fd = sys_bpf_fd(BPF_LINK_CREATE, &attr, attr_sz);
 	if (fd >= 0)
 		return fd;
 	/* we'll get EINVAL if LINK_CREATE doesn't support attaching fentry
@@ -763,44 +773,47 @@ proceed:
 
 int bpf_link_detach(int link_fd)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, link_detach);
 	union bpf_attr attr;
 	int ret;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.link_detach.link_fd = link_fd;
 
-	ret = sys_bpf(BPF_LINK_DETACH, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_LINK_DETACH, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_link_update(int link_fd, int new_prog_fd,
 		    const struct bpf_link_update_opts *opts)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, link_update);
 	union bpf_attr attr;
 	int ret;
 
 	if (!OPTS_VALID(opts, bpf_link_update_opts))
 		return libbpf_err(-EINVAL);
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.link_update.link_fd = link_fd;
 	attr.link_update.new_prog_fd = new_prog_fd;
 	attr.link_update.flags = OPTS_GET(opts, flags, 0);
 	attr.link_update.old_prog_fd = OPTS_GET(opts, old_prog_fd, 0);
 
-	ret = sys_bpf(BPF_LINK_UPDATE, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_LINK_UPDATE, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
 
 int bpf_iter_create(int link_fd)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, iter_create);
 	union bpf_attr attr;
 	int fd;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.iter_create.link_fd = link_fd;
 
-	fd = sys_bpf_fd(BPF_ITER_CREATE, &attr, sizeof(attr));
+	fd = sys_bpf_fd(BPF_ITER_CREATE, &attr, attr_sz);
 	return libbpf_err_errno(fd);
 }
 
@@ -808,13 +821,14 @@ int bpf_prog_query_opts(int target_fd,
 			enum bpf_attach_type type,
 			struct bpf_prog_query_opts *opts)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, query);
 	union bpf_attr attr;
 	int ret;
 
 	if (!OPTS_VALID(opts, bpf_prog_query_opts))
 		return libbpf_err(-EINVAL);
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 
 	attr.query.target_fd	= target_fd;
 	attr.query.attach_type	= type;
@@ -823,7 +837,7 @@ int bpf_prog_query_opts(int target_fd,
 	attr.query.prog_ids	= ptr_to_u64(OPTS_GET(opts, prog_ids, NULL));
 	attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL));
 
-	ret = sys_bpf(BPF_PROG_QUERY, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_PROG_QUERY, &attr, attr_sz);
 
 	OPTS_SET(opts, attach_flags, attr.query.attach_flags);
 	OPTS_SET(opts, prog_cnt, attr.query.prog_cnt);
@@ -852,13 +866,14 @@ int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags,
 
 int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, test);
 	union bpf_attr attr;
 	int ret;
 
 	if (!OPTS_VALID(opts, bpf_test_run_opts))
 		return libbpf_err(-EINVAL);
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.test.prog_fd = prog_fd;
 	attr.test.batch_size = OPTS_GET(opts, batch_size, 0);
 	attr.test.cpu = OPTS_GET(opts, cpu, 0);
@@ -874,7 +889,7 @@ int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts)
 	attr.test.data_in = ptr_to_u64(OPTS_GET(opts, data_in, NULL));
 	attr.test.data_out = ptr_to_u64(OPTS_GET(opts, data_out, NULL));
 
-	ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, attr_sz);
 
 	OPTS_SET(opts, data_size_out, attr.test.data_size_out);
 	OPTS_SET(opts, ctx_size_out, attr.test.ctx_size_out);
@@ -886,13 +901,14 @@ int bpf_prog_test_run_opts(int prog_fd, struct bpf_test_run_opts *opts)
 
 static int bpf_obj_get_next_id(__u32 start_id, __u32 *next_id, int cmd)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
 	union bpf_attr attr;
 	int err;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.start_id = start_id;
 
-	err = sys_bpf(cmd, &attr, sizeof(attr));
+	err = sys_bpf(cmd, &attr, attr_sz);
 	if (!err)
 		*next_id = attr.next_id;
 
@@ -921,80 +937,84 @@ int bpf_link_get_next_id(__u32 start_id, __u32 *next_id)
 
 int bpf_prog_get_fd_by_id(__u32 id)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
 	union bpf_attr attr;
 	int fd;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.prog_id = id;
 
-	fd = sys_bpf_fd(BPF_PROG_GET_FD_BY_ID, &attr, sizeof(attr));
+	fd = sys_bpf_fd(BPF_PROG_GET_FD_BY_ID, &attr, attr_sz);
 	return libbpf_err_errno(fd);
 }
 
 int bpf_map_get_fd_by_id(__u32 id)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
 	union bpf_attr attr;
 	int fd;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.map_id = id;
 
-	fd = sys_bpf_fd(BPF_MAP_GET_FD_BY_ID, &attr, sizeof(attr));
+	fd = sys_bpf_fd(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz);
 	return libbpf_err_errno(fd);
 }
 
 int bpf_btf_get_fd_by_id(__u32 id)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
 	union bpf_attr attr;
 	int fd;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.btf_id = id;
 
-	fd = sys_bpf_fd(BPF_BTF_GET_FD_BY_ID, &attr, sizeof(attr));
+	fd = sys_bpf_fd(BPF_BTF_GET_FD_BY_ID, &attr, attr_sz);
 	return libbpf_err_errno(fd);
 }
 
 int bpf_link_get_fd_by_id(__u32 id)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, open_flags);
 	union bpf_attr attr;
 	int fd;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.link_id = id;
 
-	fd = sys_bpf_fd(BPF_LINK_GET_FD_BY_ID, &attr, sizeof(attr));
+	fd = sys_bpf_fd(BPF_LINK_GET_FD_BY_ID, &attr, attr_sz);
 	return libbpf_err_errno(fd);
 }
 
 int bpf_obj_get_info_by_fd(int bpf_fd, void *info, __u32 *info_len)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, info);
 	union bpf_attr attr;
 	int err;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.info.bpf_fd = bpf_fd;
 	attr.info.info_len = *info_len;
 	attr.info.info = ptr_to_u64(info);
 
-	err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, sizeof(attr));
-
+	err = sys_bpf(BPF_OBJ_GET_INFO_BY_FD, &attr, attr_sz);
 	if (!err)
 		*info_len = attr.info.info_len;
-
 	return libbpf_err_errno(err);
 }
 
 int bpf_raw_tracepoint_open(const char *name, int prog_fd)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint);
 	union bpf_attr attr;
 	int fd;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.raw_tracepoint.name = ptr_to_u64(name);
 	attr.raw_tracepoint.prog_fd = prog_fd;
 
-	fd = sys_bpf_fd(BPF_RAW_TRACEPOINT_OPEN, &attr, sizeof(attr));
+	fd = sys_bpf_fd(BPF_RAW_TRACEPOINT_OPEN, &attr, attr_sz);
 	return libbpf_err_errno(fd);
 }
 
@@ -1050,16 +1070,18 @@ int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len,
 		      __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset,
 		      __u64 *probe_addr)
 {
-	union bpf_attr attr = {};
+	const size_t attr_sz = offsetofend(union bpf_attr, task_fd_query);
+	union bpf_attr attr;
 	int err;
 
+	memset(&attr, 0, attr_sz);
 	attr.task_fd_query.pid = pid;
 	attr.task_fd_query.fd = fd;
 	attr.task_fd_query.flags = flags;
 	attr.task_fd_query.buf = ptr_to_u64(buf);
 	attr.task_fd_query.buf_len = *buf_len;
 
-	err = sys_bpf(BPF_TASK_FD_QUERY, &attr, sizeof(attr));
+	err = sys_bpf(BPF_TASK_FD_QUERY, &attr, attr_sz);
 
 	*buf_len = attr.task_fd_query.buf_len;
 	*prog_id = attr.task_fd_query.prog_id;
@@ -1072,30 +1094,32 @@ int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len,
 
 int bpf_enable_stats(enum bpf_stats_type type)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, enable_stats);
 	union bpf_attr attr;
 	int fd;
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.enable_stats.type = type;
 
-	fd = sys_bpf_fd(BPF_ENABLE_STATS, &attr, sizeof(attr));
+	fd = sys_bpf_fd(BPF_ENABLE_STATS, &attr, attr_sz);
 	return libbpf_err_errno(fd);
 }
 
 int bpf_prog_bind_map(int prog_fd, int map_fd,
 		      const struct bpf_prog_bind_opts *opts)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, prog_bind_map);
 	union bpf_attr attr;
 	int ret;
 
 	if (!OPTS_VALID(opts, bpf_prog_bind_opts))
 		return libbpf_err(-EINVAL);
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, attr_sz);
 	attr.prog_bind_map.prog_fd = prog_fd;
 	attr.prog_bind_map.map_fd = map_fd;
 	attr.prog_bind_map.flags = OPTS_GET(opts, flags, 0);
 
-	ret = sys_bpf(BPF_PROG_BIND_MAP, &attr, sizeof(attr));
+	ret = sys_bpf(BPF_PROG_BIND_MAP, &attr, attr_sz);
 	return libbpf_err_errno(ret);
 }
diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h
index 7349b16b8e2f..d37c4fe2849d 100644
--- a/tools/lib/bpf/bpf_helpers.h
+++ b/tools/lib/bpf/bpf_helpers.h
@@ -160,18 +160,6 @@ bpf_tail_call_static(void *ctx, const void *map, const __u32 slot)
 }
 #endif
 
-/*
- * Helper structure used by eBPF C program
- * to describe BPF map attributes to libbpf loader
- */
-struct bpf_map_def {
-	unsigned int type;
-	unsigned int key_size;
-	unsigned int value_size;
-	unsigned int max_entries;
-	unsigned int map_flags;
-} __attribute__((deprecated("use BTF-defined maps in .maps section")));
-
 enum libbpf_pin_type {
 	LIBBPF_PIN_NONE,
 	/* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
diff --git a/tools/lib/bpf/bpf_tracing.h b/tools/lib/bpf/bpf_tracing.h
index 43ca3aff2292..2972dc25ff72 100644
--- a/tools/lib/bpf/bpf_tracing.h
+++ b/tools/lib/bpf/bpf_tracing.h
@@ -426,7 +426,7 @@ struct pt_regs;
  */
 #define BPF_PROG(name, args...)						    \
 name(unsigned long long *ctx);						    \
-static __attribute__((always_inline)) typeof(name(0))			    \
+static __always_inline typeof(name(0))					    \
 ____##name(unsigned long long *ctx, ##args);				    \
 typeof(name(0)) name(unsigned long long *ctx)				    \
 {									    \
@@ -435,9 +435,116 @@ typeof(name(0)) name(unsigned long long *ctx)				    \
 	return ____##name(___bpf_ctx_cast(args));			    \
 	_Pragma("GCC diagnostic pop")					    \
 }									    \
-static __attribute__((always_inline)) typeof(name(0))			    \
+static __always_inline typeof(name(0))					    \
 ____##name(unsigned long long *ctx, ##args)
 
+#ifndef ___bpf_nth2
+#define ___bpf_nth2(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13,	\
+		    _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, N, ...) N
+#endif
+#ifndef ___bpf_narg2
+#define ___bpf_narg2(...)	\
+	___bpf_nth2(_, ##__VA_ARGS__, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7,	\
+		    6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0)
+#endif
+
+#define ___bpf_treg_cnt(t) \
+	__builtin_choose_expr(sizeof(t) == 1, 1,	\
+	__builtin_choose_expr(sizeof(t) == 2, 1,	\
+	__builtin_choose_expr(sizeof(t) == 4, 1,	\
+	__builtin_choose_expr(sizeof(t) == 8, 1,	\
+	__builtin_choose_expr(sizeof(t) == 16, 2,	\
+			      (void)0)))))
+
+#define ___bpf_reg_cnt0()		(0)
+#define ___bpf_reg_cnt1(t, x)		(___bpf_reg_cnt0() + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt2(t, x, args...)	(___bpf_reg_cnt1(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt3(t, x, args...)	(___bpf_reg_cnt2(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt4(t, x, args...)	(___bpf_reg_cnt3(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt5(t, x, args...)	(___bpf_reg_cnt4(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt6(t, x, args...)	(___bpf_reg_cnt5(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt7(t, x, args...)	(___bpf_reg_cnt6(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt8(t, x, args...)	(___bpf_reg_cnt7(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt9(t, x, args...)	(___bpf_reg_cnt8(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt10(t, x, args...)	(___bpf_reg_cnt9(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt11(t, x, args...)	(___bpf_reg_cnt10(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt12(t, x, args...)	(___bpf_reg_cnt11(args) + ___bpf_treg_cnt(t))
+#define ___bpf_reg_cnt(args...)	 ___bpf_apply(___bpf_reg_cnt, ___bpf_narg2(args))(args)
+
+#define ___bpf_union_arg(t, x, n) \
+	__builtin_choose_expr(sizeof(t) == 1, ({ union { __u8 z[1]; t x; } ___t = { .z = {ctx[n]}}; ___t.x; }), \
+	__builtin_choose_expr(sizeof(t) == 2, ({ union { __u16 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \
+	__builtin_choose_expr(sizeof(t) == 4, ({ union { __u32 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \
+	__builtin_choose_expr(sizeof(t) == 8, ({ union { __u64 z[1]; t x; } ___t = {.z = {ctx[n]} }; ___t.x; }), \
+	__builtin_choose_expr(sizeof(t) == 16, ({ union { __u64 z[2]; t x; } ___t = {.z = {ctx[n], ctx[n + 1]} }; ___t.x; }), \
+			      (void)0)))))
+
+#define ___bpf_ctx_arg0(n, args...)
+#define ___bpf_ctx_arg1(n, t, x)		, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt1(t, x))
+#define ___bpf_ctx_arg2(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt2(t, x, args)) ___bpf_ctx_arg1(n, args)
+#define ___bpf_ctx_arg3(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt3(t, x, args)) ___bpf_ctx_arg2(n, args)
+#define ___bpf_ctx_arg4(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt4(t, x, args)) ___bpf_ctx_arg3(n, args)
+#define ___bpf_ctx_arg5(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt5(t, x, args)) ___bpf_ctx_arg4(n, args)
+#define ___bpf_ctx_arg6(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt6(t, x, args)) ___bpf_ctx_arg5(n, args)
+#define ___bpf_ctx_arg7(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt7(t, x, args)) ___bpf_ctx_arg6(n, args)
+#define ___bpf_ctx_arg8(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt8(t, x, args)) ___bpf_ctx_arg7(n, args)
+#define ___bpf_ctx_arg9(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt9(t, x, args)) ___bpf_ctx_arg8(n, args)
+#define ___bpf_ctx_arg10(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt10(t, x, args)) ___bpf_ctx_arg9(n, args)
+#define ___bpf_ctx_arg11(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt11(t, x, args)) ___bpf_ctx_arg10(n, args)
+#define ___bpf_ctx_arg12(n, t, x, args...)	, ___bpf_union_arg(t, x, n - ___bpf_reg_cnt12(t, x, args)) ___bpf_ctx_arg11(n, args)
+#define ___bpf_ctx_arg(args...)	___bpf_apply(___bpf_ctx_arg, ___bpf_narg2(args))(___bpf_reg_cnt(args), args)
+
+#define ___bpf_ctx_decl0()
+#define ___bpf_ctx_decl1(t, x)			, t x
+#define ___bpf_ctx_decl2(t, x, args...)		, t x ___bpf_ctx_decl1(args)
+#define ___bpf_ctx_decl3(t, x, args...)		, t x ___bpf_ctx_decl2(args)
+#define ___bpf_ctx_decl4(t, x, args...)		, t x ___bpf_ctx_decl3(args)
+#define ___bpf_ctx_decl5(t, x, args...)		, t x ___bpf_ctx_decl4(args)
+#define ___bpf_ctx_decl6(t, x, args...)		, t x ___bpf_ctx_decl5(args)
+#define ___bpf_ctx_decl7(t, x, args...)		, t x ___bpf_ctx_decl6(args)
+#define ___bpf_ctx_decl8(t, x, args...)		, t x ___bpf_ctx_decl7(args)
+#define ___bpf_ctx_decl9(t, x, args...)		, t x ___bpf_ctx_decl8(args)
+#define ___bpf_ctx_decl10(t, x, args...)	, t x ___bpf_ctx_decl9(args)
+#define ___bpf_ctx_decl11(t, x, args...)	, t x ___bpf_ctx_decl10(args)
+#define ___bpf_ctx_decl12(t, x, args...)	, t x ___bpf_ctx_decl11(args)
+#define ___bpf_ctx_decl(args...)	___bpf_apply(___bpf_ctx_decl, ___bpf_narg2(args))(args)
+
+/*
+ * BPF_PROG2 is an enhanced version of BPF_PROG in order to handle struct
+ * arguments. Since each struct argument might take one or two u64 values
+ * in the trampoline stack, argument type size is needed to place proper number
+ * of u64 values for each argument. Therefore, BPF_PROG2 has different
+ * syntax from BPF_PROG. For example, for the following BPF_PROG syntax:
+ *
+ *   int BPF_PROG(test2, int a, int b) { ... }
+ *
+ * the corresponding BPF_PROG2 syntax is:
+ *
+ *   int BPF_PROG2(test2, int, a, int, b) { ... }
+ *
+ * where type and the corresponding argument name are separated by comma.
+ *
+ * Use BPF_PROG2 macro if one of the arguments might be a struct/union larger
+ * than 8 bytes:
+ *
+ *   int BPF_PROG2(test_struct_arg, struct bpf_testmod_struct_arg_1, a, int, b,
+ *		   int, c, int, d, struct bpf_testmod_struct_arg_2, e, int, ret)
+ *   {
+ *        // access a, b, c, d, e, and ret directly
+ *        ...
+ *   }
+ */
+#define BPF_PROG2(name, args...)						\
+name(unsigned long long *ctx);							\
+static __always_inline typeof(name(0))						\
+____##name(unsigned long long *ctx ___bpf_ctx_decl(args));			\
+typeof(name(0)) name(unsigned long long *ctx)					\
+{										\
+	return ____##name(ctx ___bpf_ctx_arg(args));				\
+}										\
+static __always_inline typeof(name(0))						\
+____##name(unsigned long long *ctx ___bpf_ctx_decl(args))
+
 struct pt_regs;
 
 #define ___bpf_kprobe_args0()           ctx
@@ -460,7 +567,7 @@ struct pt_regs;
  */
 #define BPF_KPROBE(name, args...)					    \
 name(struct pt_regs *ctx);						    \
-static __attribute__((always_inline)) typeof(name(0))			    \
+static __always_inline typeof(name(0))					    \
 ____##name(struct pt_regs *ctx, ##args);				    \
 typeof(name(0)) name(struct pt_regs *ctx)				    \
 {									    \
@@ -469,7 +576,7 @@ typeof(name(0)) name(struct pt_regs *ctx)				    \
 	return ____##name(___bpf_kprobe_args(args));			    \
 	_Pragma("GCC diagnostic pop")					    \
 }									    \
-static __attribute__((always_inline)) typeof(name(0))			    \
+static __always_inline typeof(name(0))					    \
 ____##name(struct pt_regs *ctx, ##args)
 
 #define ___bpf_kretprobe_args0()       ctx
@@ -484,7 +591,7 @@ ____##name(struct pt_regs *ctx, ##args)
  */
 #define BPF_KRETPROBE(name, args...)					    \
 name(struct pt_regs *ctx);						    \
-static __attribute__((always_inline)) typeof(name(0))			    \
+static __always_inline typeof(name(0))					    \
 ____##name(struct pt_regs *ctx, ##args);				    \
 typeof(name(0)) name(struct pt_regs *ctx)				    \
 {									    \
@@ -540,7 +647,7 @@ static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args)
 #define BPF_KSYSCALL(name, args...)					    \
 name(struct pt_regs *ctx);						    \
 extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig;			    \
-static __attribute__((always_inline)) typeof(name(0))			    \
+static __always_inline typeof(name(0))					    \
 ____##name(struct pt_regs *ctx, ##args);				    \
 typeof(name(0)) name(struct pt_regs *ctx)				    \
 {									    \
@@ -555,7 +662,7 @@ typeof(name(0)) name(struct pt_regs *ctx)				    \
 		return ____##name(___bpf_syscall_args(args));		    \
 	_Pragma("GCC diagnostic pop")					    \
 }									    \
-static __attribute__((always_inline)) typeof(name(0))			    \
+static __always_inline typeof(name(0))					    \
 ____##name(struct pt_regs *ctx, ##args)
 
 #define BPF_KPROBE_SYSCALL BPF_KSYSCALL
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index 2d14f1a52d7a..d88647da2c7f 100644
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -1225,8 +1225,6 @@ int btf__load_into_kernel(struct btf *btf)
 	return btf_load_into_kernel(btf, NULL, 0, 0);
 }
 
-int btf__load(struct btf *) __attribute__((alias("btf__load_into_kernel")));
-
 int btf__fd(const struct btf *btf)
 {
 	return btf->fd;
@@ -4644,20 +4642,17 @@ static int btf_dedup_remap_types(struct btf_dedup *d)
  */
 struct btf *btf__load_vmlinux_btf(void)
 {
-	struct {
-		const char *path_fmt;
-		bool raw_btf;
-	} locations[] = {
+	const char *locations[] = {
 		/* try canonical vmlinux BTF through sysfs first */
-		{ "/sys/kernel/btf/vmlinux", true /* raw BTF */ },
-		/* fall back to trying to find vmlinux ELF on disk otherwise */
-		{ "/boot/vmlinux-%1$s" },
-		{ "/lib/modules/%1$s/vmlinux-%1$s" },
-		{ "/lib/modules/%1$s/build/vmlinux" },
-		{ "/usr/lib/modules/%1$s/kernel/vmlinux" },
-		{ "/usr/lib/debug/boot/vmlinux-%1$s" },
-		{ "/usr/lib/debug/boot/vmlinux-%1$s.debug" },
-		{ "/usr/lib/debug/lib/modules/%1$s/vmlinux" },
+		"/sys/kernel/btf/vmlinux",
+		/* fall back to trying to find vmlinux on disk otherwise */
+		"/boot/vmlinux-%1$s",
+		"/lib/modules/%1$s/vmlinux-%1$s",
+		"/lib/modules/%1$s/build/vmlinux",
+		"/usr/lib/modules/%1$s/kernel/vmlinux",
+		"/usr/lib/debug/boot/vmlinux-%1$s",
+		"/usr/lib/debug/boot/vmlinux-%1$s.debug",
+		"/usr/lib/debug/lib/modules/%1$s/vmlinux",
 	};
 	char path[PATH_MAX + 1];
 	struct utsname buf;
@@ -4667,15 +4662,12 @@ struct btf *btf__load_vmlinux_btf(void)
 	uname(&buf);
 
 	for (i = 0; i < ARRAY_SIZE(locations); i++) {
-		snprintf(path, PATH_MAX, locations[i].path_fmt, buf.release);
+		snprintf(path, PATH_MAX, locations[i], buf.release);
 
-		if (access(path, R_OK))
+		if (faccessat(AT_FDCWD, path, R_OK, AT_EACCESS))
 			continue;
 
-		if (locations[i].raw_btf)
-			btf = btf__parse_raw(path);
-		else
-			btf = btf__parse_elf(path, NULL);
+		btf = btf__parse(path, NULL);
 		err = libbpf_get_error(btf);
 		pr_debug("loading kernel BTF '%s': %d\n", path, err);
 		if (err)
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h
index 583760df83b4..8e6880d91c84 100644
--- a/tools/lib/bpf/btf.h
+++ b/tools/lib/bpf/btf.h
@@ -116,7 +116,6 @@ LIBBPF_API struct btf *btf__parse_raw_split(const char *path, struct btf *base_b
 
 LIBBPF_API struct btf *btf__load_vmlinux_btf(void);
 LIBBPF_API struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf);
-LIBBPF_API struct btf *libbpf_find_kernel_btf(void);
 
 LIBBPF_API struct btf *btf__load_from_kernel_by_id(__u32 id);
 LIBBPF_API struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf);
@@ -487,6 +486,8 @@ static inline struct btf_enum *btf_enum(const struct btf_type *t)
 	return (struct btf_enum *)(t + 1);
 }
 
+struct btf_enum64;
+
 static inline struct btf_enum64 *btf_enum64(const struct btf_type *t)
 {
 	return (struct btf_enum64 *)(t + 1);
@@ -494,7 +495,28 @@ static inline struct btf_enum64 *btf_enum64(const struct btf_type *t)
 
 static inline __u64 btf_enum64_value(const struct btf_enum64 *e)
 {
-	return ((__u64)e->val_hi32 << 32) | e->val_lo32;
+	/* struct btf_enum64 is introduced in Linux 6.0, which is very
+	 * bleeding-edge. Here we are avoiding relying on struct btf_enum64
+	 * definition coming from kernel UAPI headers to support wider range
+	 * of system-wide kernel headers.
+	 *
+	 * Given this header can be also included from C++ applications, that
+	 * further restricts C tricks we can use (like using compatible
+	 * anonymous struct). So just treat struct btf_enum64 as
+	 * a three-element array of u32 and access second (lo32) and third
+	 * (hi32) elements directly.
+	 *
+	 * For reference, here is a struct btf_enum64 definition:
+	 *
+	 * const struct btf_enum64 {
+	 *	__u32	name_off;
+	 *	__u32	val_lo32;
+	 *	__u32	val_hi32;
+	 * };
+	 */
+	const __u32 *e64 = (const __u32 *)e;
+
+	return ((__u64)e64[2] << 32) | e64[1];
 }
 
 static inline struct btf_member *btf_members(const struct btf_type *t)
diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c
index 627edb5bb6de..4221f73a74d0 100644
--- a/tools/lib/bpf/btf_dump.c
+++ b/tools/lib/bpf/btf_dump.c
@@ -2385,7 +2385,7 @@ int btf_dump__dump_type_data(struct btf_dump *d, __u32 id,
 	d->typed_dump->indent_lvl = OPTS_GET(opts, indent_level, 0);
 
 	/* default indent string is a tab */
-	if (!opts->indent_str)
+	if (!OPTS_GET(opts, indent_str, NULL))
 		d->typed_dump->indent_str[0] = '\t';
 	else
 		libbpf_strlcpy(d->typed_dump->indent_str, opts->indent_str,
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 50d41815f431..184ce1684dcd 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -163,6 +163,7 @@ static const char * const map_type_name[] = {
 	[BPF_MAP_TYPE_INODE_STORAGE]		= "inode_storage",
 	[BPF_MAP_TYPE_TASK_STORAGE]		= "task_storage",
 	[BPF_MAP_TYPE_BLOOM_FILTER]		= "bloom_filter",
+	[BPF_MAP_TYPE_USER_RINGBUF]             = "user_ringbuf",
 };
 
 static const char * const prog_type_name[] = {
@@ -223,13 +224,18 @@ __printf(2, 3)
 void libbpf_print(enum libbpf_print_level level, const char *format, ...)
 {
 	va_list args;
+	int old_errno;
 
 	if (!__libbpf_pr)
 		return;
 
+	old_errno = errno;
+
 	va_start(args, format);
 	__libbpf_pr(level, format, args);
 	va_end(args);
+
+	errno = old_errno;
 }
 
 static void pr_perm_msg(int err)
@@ -412,6 +418,7 @@ struct bpf_program {
 
 	int fd;
 	bool autoload;
+	bool autoattach;
 	bool mark_btf_static;
 	enum bpf_prog_type type;
 	enum bpf_attach_type expected_attach_type;
@@ -591,7 +598,6 @@ struct elf_state {
 	size_t strtabidx;
 	struct elf_sec_desc *secs;
 	int sec_cnt;
-	int maps_shndx;
 	int btf_maps_shndx;
 	__u32 btf_maps_sec_btf_id;
 	int text_shndx;
@@ -751,6 +757,8 @@ bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
 		prog->autoload = true;
 	}
 
+	prog->autoattach = true;
+
 	/* inherit object's log_level */
 	prog->log_level = obj->log_level;
 
@@ -876,7 +884,7 @@ __u32 get_kernel_version(void)
 	__u32 major, minor, patch;
 	struct utsname info;
 
-	if (access(ubuntu_kver_file, R_OK) == 0) {
+	if (faccessat(AT_FDCWD, ubuntu_kver_file, R_OK, AT_EACCESS) == 0) {
 		FILE *f;
 
 		f = fopen(ubuntu_kver_file, "r");
@@ -1272,7 +1280,6 @@ static struct bpf_object *bpf_object__new(const char *path,
 	 */
 	obj->efile.obj_buf = obj_buf;
 	obj->efile.obj_buf_sz = obj_buf_sz;
-	obj->efile.maps_shndx = -1;
 	obj->efile.btf_maps_shndx = -1;
 	obj->efile.st_ops_shndx = -1;
 	obj->kconfig_map_idx = -1;
@@ -1642,6 +1649,10 @@ static int bpf_object__init_global_data_maps(struct bpf_object *obj)
 	for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
 		sec_desc = &obj->efile.secs[sec_idx];
 
+		/* Skip recognized sections with size 0. */
+		if (!sec_desc->data || sec_desc->data->d_size == 0)
+			continue;
+
 		switch (sec_desc->sec_type) {
 		case SEC_DATA:
 			sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
@@ -2086,19 +2097,30 @@ static bool get_map_field_int(const char *map_name, const struct btf *btf,
 	return true;
 }
 
+static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name)
+{
+	int len;
+
+	len = snprintf(buf, buf_sz, "%s/%s", path, name);
+	if (len < 0)
+		return -EINVAL;
+	if (len >= buf_sz)
+		return -ENAMETOOLONG;
+
+	return 0;
+}
+
 static int build_map_pin_path(struct bpf_map *map, const char *path)
 {
 	char buf[PATH_MAX];
-	int len;
+	int err;
 
 	if (!path)
 		path = "/sys/fs/bpf";
 
-	len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map));
-	if (len < 0)
-		return -EINVAL;
-	else if (len >= PATH_MAX)
-		return -ENAMETOOLONG;
+	err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
+	if (err)
+		return err;
 
 	return bpf_map__set_pin_path(map, buf);
 }
@@ -2362,6 +2384,12 @@ static size_t adjust_ringbuf_sz(size_t sz)
 	return sz;
 }
 
+static bool map_is_ringbuf(const struct bpf_map *map)
+{
+	return map->def.type == BPF_MAP_TYPE_RINGBUF ||
+	       map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
+}
+
 static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
 {
 	map->def.type = def->map_type;
@@ -2376,7 +2404,7 @@ static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def
 	map->btf_value_type_id = def->value_type_id;
 
 	/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
-	if (map->def.type == BPF_MAP_TYPE_RINGBUF)
+	if (map_is_ringbuf(map))
 		map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
 
 	if (def->parts & MAP_DEF_MAP_TYPE)
@@ -3359,7 +3387,8 @@ static int bpf_object__elf_collect(struct bpf_object *obj)
 			if (err)
 				return err;
 		} else if (strcmp(name, "maps") == 0) {
-			obj->efile.maps_shndx = idx;
+			pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");
+			return -ENOTSUP;
 		} else if (strcmp(name, MAPS_ELF_SEC) == 0) {
 			obj->efile.btf_maps_shndx = idx;
 		} else if (strcmp(name, BTF_ELF_SEC) == 0) {
@@ -3891,8 +3920,7 @@ static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
 				      int shndx)
 {
-	return shndx == obj->efile.maps_shndx ||
-	       shndx == obj->efile.btf_maps_shndx;
+	return shndx == obj->efile.btf_maps_shndx;
 }
 
 static enum libbpf_map_type
@@ -4277,11 +4305,12 @@ int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
 
 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
 {
-	struct bpf_map_info info = {};
+	struct bpf_map_info info;
 	__u32 len = sizeof(info), name_len;
 	int new_fd, err;
 	char *new_name;
 
+	memset(&info, 0, len);
 	err = bpf_obj_get_info_by_fd(fd, &info, &len);
 	if (err && errno == EINVAL)
 		err = bpf_get_map_info_from_fdinfo(fd, &info);
@@ -4358,7 +4387,7 @@ int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
 	map->def.max_entries = max_entries;
 
 	/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
-	if (map->def.type == BPF_MAP_TYPE_RINGBUF)
+	if (map_is_ringbuf(map))
 		map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
 
 	return 0;
@@ -4408,14 +4437,23 @@ static int probe_fd(int fd)
 
 static int probe_kern_prog_name(void)
 {
+	const size_t attr_sz = offsetofend(union bpf_attr, prog_name);
 	struct bpf_insn insns[] = {
 		BPF_MOV64_IMM(BPF_REG_0, 0),
 		BPF_EXIT_INSN(),
 	};
-	int ret, insn_cnt = ARRAY_SIZE(insns);
+	union bpf_attr attr;
+	int ret;
+
+	memset(&attr, 0, attr_sz);
+	attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
+	attr.license = ptr_to_u64("GPL");
+	attr.insns = ptr_to_u64(insns);
+	attr.insn_cnt = (__u32)ARRAY_SIZE(insns);
+	libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name));
 
 	/* make sure loading with name works */
-	ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "test", "GPL", insns, insn_cnt, NULL);
+	ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS);
 	return probe_fd(ret);
 }
 
@@ -4430,7 +4468,7 @@ static int probe_kern_global_data(void)
 	};
 	int ret, map, insn_cnt = ARRAY_SIZE(insns);
 
-	map = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, sizeof(int), 32, 1, NULL);
+	map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL);
 	if (map < 0) {
 		ret = -errno;
 		cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
@@ -4563,7 +4601,7 @@ static int probe_kern_array_mmap(void)
 	LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE);
 	int fd;
 
-	fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, sizeof(int), sizeof(int), 1, &opts);
+	fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts);
 	return probe_fd(fd);
 }
 
@@ -4610,7 +4648,7 @@ static int probe_prog_bind_map(void)
 	};
 	int ret, map, prog, insn_cnt = ARRAY_SIZE(insns);
 
-	map = bpf_map_create(BPF_MAP_TYPE_ARRAY, NULL, sizeof(int), 32, 1, NULL);
+	map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL);
 	if (map < 0) {
 		ret = -errno;
 		cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
@@ -4814,13 +4852,12 @@ bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
 
 static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
 {
-	struct bpf_map_info map_info = {};
+	struct bpf_map_info map_info;
 	char msg[STRERR_BUFSIZE];
-	__u32 map_info_len;
+	__u32 map_info_len = sizeof(map_info);
 	int err;
 
-	map_info_len = sizeof(map_info);
-
+	memset(&map_info, 0, map_info_len);
 	err = bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len);
 	if (err && errno == EINVAL)
 		err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
@@ -7244,8 +7281,6 @@ static int bpf_object_unload(struct bpf_object *obj)
 	return 0;
 }
 
-int bpf_object__unload(struct bpf_object *obj) __attribute__((alias("bpf_object_unload")));
-
 static int bpf_object__sanitize_maps(struct bpf_object *obj)
 {
 	struct bpf_map *m;
@@ -7944,17 +7979,9 @@ int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
 			continue;
 
 		if (path) {
-			int len;
-
-			len = snprintf(buf, PATH_MAX, "%s/%s", path,
-				       bpf_map__name(map));
-			if (len < 0) {
-				err = -EINVAL;
-				goto err_unpin_maps;
-			} else if (len >= PATH_MAX) {
-				err = -ENAMETOOLONG;
+			err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
+			if (err)
 				goto err_unpin_maps;
-			}
 			sanitize_pin_path(buf);
 			pin_path = buf;
 		} else if (!map->pin_path) {
@@ -7992,14 +8019,9 @@ int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
 		char buf[PATH_MAX];
 
 		if (path) {
-			int len;
-
-			len = snprintf(buf, PATH_MAX, "%s/%s", path,
-				       bpf_map__name(map));
-			if (len < 0)
-				return libbpf_err(-EINVAL);
-			else if (len >= PATH_MAX)
-				return libbpf_err(-ENAMETOOLONG);
+			err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
+			if (err)
+				return libbpf_err(err);
 			sanitize_pin_path(buf);
 			pin_path = buf;
 		} else if (!map->pin_path) {
@@ -8017,6 +8039,7 @@ int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
 {
 	struct bpf_program *prog;
+	char buf[PATH_MAX];
 	int err;
 
 	if (!obj)
@@ -8028,17 +8051,9 @@ int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
 	}
 
 	bpf_object__for_each_program(prog, obj) {
-		char buf[PATH_MAX];
-		int len;
-
-		len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name);
-		if (len < 0) {
-			err = -EINVAL;
-			goto err_unpin_programs;
-		} else if (len >= PATH_MAX) {
-			err = -ENAMETOOLONG;
+		err = pathname_concat(buf, sizeof(buf), path, prog->name);
+		if (err)
 			goto err_unpin_programs;
-		}
 
 		err = bpf_program__pin(prog, buf);
 		if (err)
@@ -8049,13 +8064,7 @@ int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
 
 err_unpin_programs:
 	while ((prog = bpf_object__prev_program(obj, prog))) {
-		char buf[PATH_MAX];
-		int len;
-
-		len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name);
-		if (len < 0)
-			continue;
-		else if (len >= PATH_MAX)
+		if (pathname_concat(buf, sizeof(buf), path, prog->name))
 			continue;
 
 		bpf_program__unpin(prog, buf);
@@ -8074,13 +8083,10 @@ int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
 
 	bpf_object__for_each_program(prog, obj) {
 		char buf[PATH_MAX];
-		int len;
 
-		len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name);
-		if (len < 0)
-			return libbpf_err(-EINVAL);
-		else if (len >= PATH_MAX)
-			return libbpf_err(-ENAMETOOLONG);
+		err = pathname_concat(buf, sizeof(buf), path, prog->name);
+		if (err)
+			return libbpf_err(err);
 
 		err = bpf_program__unpin(prog, buf);
 		if (err)
@@ -8298,6 +8304,16 @@ int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
 	return 0;
 }
 
+bool bpf_program__autoattach(const struct bpf_program *prog)
+{
+	return prog->autoattach;
+}
+
+void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)
+{
+	prog->autoattach = autoattach;
+}
+
 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
 {
 	return prog->insns;
@@ -8978,11 +8994,12 @@ int libbpf_find_vmlinux_btf_id(const char *name,
 
 static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd)
 {
-	struct bpf_prog_info info = {};
+	struct bpf_prog_info info;
 	__u32 info_len = sizeof(info);
 	struct btf *btf;
 	int err;
 
+	memset(&info, 0, info_len);
 	err = bpf_obj_get_info_by_fd(attach_prog_fd, &info, &info_len);
 	if (err) {
 		pr_warn("failed bpf_obj_get_info_by_fd for FD %d: %d\n",
@@ -9056,11 +9073,15 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac
 	int err = 0;
 
 	/* BPF program's BTF ID */
-	if (attach_prog_fd) {
+	if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
+		if (!attach_prog_fd) {
+			pr_warn("prog '%s': attach program FD is not set\n", prog->name);
+			return -EINVAL;
+		}
 		err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd);
 		if (err < 0) {
-			pr_warn("failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
-				 attach_prog_fd, attach_name, err);
+			pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
+				 prog->name, attach_prog_fd, attach_name, err);
 			return err;
 		}
 		*btf_obj_fd = 0;
@@ -9077,7 +9098,8 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac
 		err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
 	}
 	if (err) {
-		pr_warn("failed to find kernel BTF type ID of '%s': %d\n", attach_name, err);
+		pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",
+			prog->name, attach_name, err);
 		return err;
 	}
 	return 0;
@@ -9810,13 +9832,16 @@ static int determine_uprobe_retprobe_bit(void)
 static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
 				 uint64_t offset, int pid, size_t ref_ctr_off)
 {
-	struct perf_event_attr attr = {};
+	const size_t attr_sz = sizeof(struct perf_event_attr);
+	struct perf_event_attr attr;
 	char errmsg[STRERR_BUFSIZE];
 	int type, pfd;
 
 	if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS))
 		return -EINVAL;
 
+	memset(&attr, 0, attr_sz);
+
 	type = uprobe ? determine_uprobe_perf_type()
 		      : determine_kprobe_perf_type();
 	if (type < 0) {
@@ -9837,7 +9862,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name,
 		}
 		attr.config |= 1 << bit;
 	}
-	attr.size = sizeof(attr);
+	attr.size = attr_sz;
 	attr.type = type;
 	attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT;
 	attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */
@@ -9879,7 +9904,7 @@ static bool use_debugfs(void)
 	static int has_debugfs = -1;
 
 	if (has_debugfs < 0)
-		has_debugfs = access(DEBUGFS, F_OK) == 0;
+		has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0;
 
 	return has_debugfs == 1;
 }
@@ -9936,7 +9961,8 @@ static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retpro
 static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
 					 const char *kfunc_name, size_t offset, int pid)
 {
-	struct perf_event_attr attr = {};
+	const size_t attr_sz = sizeof(struct perf_event_attr);
+	struct perf_event_attr attr;
 	char errmsg[STRERR_BUFSIZE];
 	int type, pfd, err;
 
@@ -9955,7 +9981,9 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe,
 			libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
 		goto err_clean_legacy;
 	}
-	attr.size = sizeof(attr);
+
+	memset(&attr, 0, attr_sz);
+	attr.size = attr_sz;
 	attr.config = type;
 	attr.type = PERF_TYPE_TRACEPOINT;
 
@@ -10412,6 +10440,7 @@ static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retpro
 static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
 					 const char *binary_path, size_t offset, int pid)
 {
+	const size_t attr_sz = sizeof(struct perf_event_attr);
 	struct perf_event_attr attr;
 	int type, pfd, err;
 
@@ -10429,8 +10458,8 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe,
 		goto err_clean_legacy;
 	}
 
-	memset(&attr, 0, sizeof(attr));
-	attr.size = sizeof(attr);
+	memset(&attr, 0, attr_sz);
+	attr.size = attr_sz;
 	attr.config = type;
 	attr.type = PERF_TYPE_TRACEPOINT;
 
@@ -10662,15 +10691,17 @@ static const char *arch_specific_lib_paths(void)
 static int resolve_full_path(const char *file, char *result, size_t result_sz)
 {
 	const char *search_paths[3] = {};
-	int i;
+	int i, perm;
 
 	if (str_has_sfx(file, ".so") || strstr(file, ".so.")) {
 		search_paths[0] = getenv("LD_LIBRARY_PATH");
 		search_paths[1] = "/usr/lib64:/usr/lib";
 		search_paths[2] = arch_specific_lib_paths();
+		perm = R_OK;
 	} else {
 		search_paths[0] = getenv("PATH");
 		search_paths[1] = "/usr/bin:/usr/sbin";
+		perm = R_OK | X_OK;
 	}
 
 	for (i = 0; i < ARRAY_SIZE(search_paths); i++) {
@@ -10689,8 +10720,8 @@ static int resolve_full_path(const char *file, char *result, size_t result_sz)
 			if (!seg_len)
 				continue;
 			snprintf(result, result_sz, "%.*s/%s", seg_len, s, file);
-			/* ensure it is an executable file/link */
-			if (access(result, R_OK | X_OK) < 0)
+			/* ensure it has required permissions */
+			if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0)
 				continue;
 			pr_debug("resolved '%s' to '%s'\n", file, result);
 			return 0;
@@ -10967,7 +10998,8 @@ static int determine_tracepoint_id(const char *tp_category,
 static int perf_event_open_tracepoint(const char *tp_category,
 				      const char *tp_name)
 {
-	struct perf_event_attr attr = {};
+	const size_t attr_sz = sizeof(struct perf_event_attr);
+	struct perf_event_attr attr;
 	char errmsg[STRERR_BUFSIZE];
 	int tp_id, pfd, err;
 
@@ -10979,8 +11011,9 @@ static int perf_event_open_tracepoint(const char *tp_category,
 		return tp_id;
 	}
 
+	memset(&attr, 0, attr_sz);
 	attr.type = PERF_TYPE_TRACEPOINT;
-	attr.size = sizeof(attr);
+	attr.size = attr_sz;
 	attr.config = tp_id;
 
 	pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,
@@ -11600,12 +11633,15 @@ struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt,
 				     void *ctx,
 				     const struct perf_buffer_opts *opts)
 {
+	const size_t attr_sz = sizeof(struct perf_event_attr);
 	struct perf_buffer_params p = {};
-	struct perf_event_attr attr = {};
+	struct perf_event_attr attr;
 
 	if (!OPTS_VALID(opts, perf_buffer_opts))
 		return libbpf_err_ptr(-EINVAL);
 
+	memset(&attr, 0, attr_sz);
+	attr.size = attr_sz;
 	attr.config = PERF_COUNT_SW_BPF_OUTPUT;
 	attr.type = PERF_TYPE_SOFTWARE;
 	attr.sample_type = PERF_SAMPLE_RAW;
@@ -12328,7 +12364,7 @@ int bpf_object__attach_skeleton(struct bpf_object_skeleton *s)
 		struct bpf_program *prog = *s->progs[i].prog;
 		struct bpf_link **link = s->progs[i].link;
 
-		if (!prog->autoload)
+		if (!prog->autoload || !prog->autoattach)
 			continue;
 
 		/* auto-attaching not supported for this program */
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index 61493c4cddac..eee883f007f9 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -118,7 +118,9 @@ struct bpf_object_open_opts {
 	 * auto-pinned to that path on load; defaults to "/sys/fs/bpf".
 	 */
 	const char *pin_root_path;
-	long :0;
+
+	__u32 :32; /* stub out now removed attach_prog_fd */
+
 	/* Additional kernel config content that augments and overrides
 	 * system Kconfig for CONFIG_xxx externs.
 	 */
@@ -260,6 +262,8 @@ LIBBPF_API const char *bpf_program__name(const struct bpf_program *prog);
 LIBBPF_API const char *bpf_program__section_name(const struct bpf_program *prog);
 LIBBPF_API bool bpf_program__autoload(const struct bpf_program *prog);
 LIBBPF_API int bpf_program__set_autoload(struct bpf_program *prog, bool autoload);
+LIBBPF_API bool bpf_program__autoattach(const struct bpf_program *prog);
+LIBBPF_API void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach);
 
 struct bpf_insn;
 
@@ -1009,6 +1013,7 @@ LIBBPF_API int bpf_tc_query(const struct bpf_tc_hook *hook,
 
 /* Ring buffer APIs */
 struct ring_buffer;
+struct user_ring_buffer;
 
 typedef int (*ring_buffer_sample_fn)(void *ctx, void *data, size_t size);
 
@@ -1028,6 +1033,112 @@ LIBBPF_API int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms);
 LIBBPF_API int ring_buffer__consume(struct ring_buffer *rb);
 LIBBPF_API int ring_buffer__epoll_fd(const struct ring_buffer *rb);
 
+struct user_ring_buffer_opts {
+	size_t sz; /* size of this struct, for forward/backward compatibility */
+};
+
+#define user_ring_buffer_opts__last_field sz
+
+/* @brief **user_ring_buffer__new()** creates a new instance of a user ring
+ * buffer.
+ *
+ * @param map_fd A file descriptor to a BPF_MAP_TYPE_USER_RINGBUF map.
+ * @param opts Options for how the ring buffer should be created.
+ * @return A user ring buffer on success; NULL and errno being set on a
+ * failure.
+ */
+LIBBPF_API struct user_ring_buffer *
+user_ring_buffer__new(int map_fd, const struct user_ring_buffer_opts *opts);
+
+/* @brief **user_ring_buffer__reserve()** reserves a pointer to a sample in the
+ * user ring buffer.
+ * @param rb A pointer to a user ring buffer.
+ * @param size The size of the sample, in bytes.
+ * @return A pointer to an 8-byte aligned reserved region of the user ring
+ * buffer; NULL, and errno being set if a sample could not be reserved.
+ *
+ * This function is *not* thread safe, and callers must synchronize accessing
+ * this function if there are multiple producers.  If a size is requested that
+ * is larger than the size of the entire ring buffer, errno will be set to
+ * E2BIG and NULL is returned. If the ring buffer could accommodate the size,
+ * but currently does not have enough space, errno is set to ENOSPC and NULL is
+ * returned.
+ *
+ * After initializing the sample, callers must invoke
+ * **user_ring_buffer__submit()** to post the sample to the kernel. Otherwise,
+ * the sample must be freed with **user_ring_buffer__discard()**.
+ */
+LIBBPF_API void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size);
+
+/* @brief **user_ring_buffer__reserve_blocking()** reserves a record in the
+ * ring buffer, possibly blocking for up to @timeout_ms until a sample becomes
+ * available.
+ * @param rb The user ring buffer.
+ * @param size The size of the sample, in bytes.
+ * @param timeout_ms The amount of time, in milliseconds, for which the caller
+ * should block when waiting for a sample. -1 causes the caller to block
+ * indefinitely.
+ * @return A pointer to an 8-byte aligned reserved region of the user ring
+ * buffer; NULL, and errno being set if a sample could not be reserved.
+ *
+ * This function is *not* thread safe, and callers must synchronize
+ * accessing this function if there are multiple producers
+ *
+ * If **timeout_ms** is -1, the function will block indefinitely until a sample
+ * becomes available. Otherwise, **timeout_ms** must be non-negative, or errno
+ * is set to EINVAL, and NULL is returned. If **timeout_ms** is 0, no blocking
+ * will occur and the function will return immediately after attempting to
+ * reserve a sample.
+ *
+ * If **size** is larger than the size of the entire ring buffer, errno is set
+ * to E2BIG and NULL is returned. If the ring buffer could accommodate
+ * **size**, but currently does not have enough space, the caller will block
+ * until at most **timeout_ms** has elapsed. If insufficient space is available
+ * at that time, errno is set to ENOSPC, and NULL is returned.
+ *
+ * The kernel guarantees that it will wake up this thread to check if
+ * sufficient space is available in the ring buffer at least once per
+ * invocation of the **bpf_ringbuf_drain()** helper function, provided that at
+ * least one sample is consumed, and the BPF program did not invoke the
+ * function with BPF_RB_NO_WAKEUP. A wakeup may occur sooner than that, but the
+ * kernel does not guarantee this. If the helper function is invoked with
+ * BPF_RB_FORCE_WAKEUP, a wakeup event will be sent even if no sample is
+ * consumed.
+ *
+ * When a sample of size **size** is found within **timeout_ms**, a pointer to
+ * the sample is returned. After initializing the sample, callers must invoke
+ * **user_ring_buffer__submit()** to post the sample to the ring buffer.
+ * Otherwise, the sample must be freed with **user_ring_buffer__discard()**.
+ */
+LIBBPF_API void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb,
+						    __u32 size,
+						    int timeout_ms);
+
+/* @brief **user_ring_buffer__submit()** submits a previously reserved sample
+ * into the ring buffer.
+ * @param rb The user ring buffer.
+ * @param sample A reserved sample.
+ *
+ * It is not necessary to synchronize amongst multiple producers when invoking
+ * this function.
+ */
+LIBBPF_API void user_ring_buffer__submit(struct user_ring_buffer *rb, void *sample);
+
+/* @brief **user_ring_buffer__discard()** discards a previously reserved sample.
+ * @param rb The user ring buffer.
+ * @param sample A reserved sample.
+ *
+ * It is not necessary to synchronize amongst multiple producers when invoking
+ * this function.
+ */
+LIBBPF_API void user_ring_buffer__discard(struct user_ring_buffer *rb, void *sample);
+
+/* @brief **user_ring_buffer__free()** frees a ring buffer that was previously
+ * created with **user_ring_buffer__new()**.
+ * @param rb The user ring buffer being freed.
+ */
+LIBBPF_API void user_ring_buffer__free(struct user_ring_buffer *rb);
+
 /* Perf buffer APIs */
 struct perf_buffer;
 
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 119e6e1ea7f1..c1d6aa7c82b6 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -358,6 +358,8 @@ LIBBPF_1.0.0 {
 		bpf_obj_get_opts;
 		bpf_prog_query_opts;
 		bpf_program__attach_ksyscall;
+		bpf_program__autoattach;
+		bpf_program__set_autoattach;
 		btf__add_enum64;
 		btf__add_enum64_value;
 		libbpf_bpf_attach_type_str;
@@ -366,3 +368,13 @@ LIBBPF_1.0.0 {
 		libbpf_bpf_prog_type_str;
 		perf_buffer__buffer;
 };
+
+LIBBPF_1.1.0 {
+	global:
+		user_ring_buffer__discard;
+		user_ring_buffer__free;
+		user_ring_buffer__new;
+		user_ring_buffer__reserve;
+		user_ring_buffer__reserve_blocking;
+		user_ring_buffer__submit;
+} LIBBPF_1.0.0;
diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h
index 4135ae0a2bc3..377642ff51fc 100644
--- a/tools/lib/bpf/libbpf_internal.h
+++ b/tools/lib/bpf/libbpf_internal.h
@@ -573,4 +573,7 @@ static inline bool is_pow_of_2(size_t x)
 	return x && (x & (x - 1)) == 0;
 }
 
+#define PROG_LOAD_ATTEMPTS 5
+int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts);
+
 #endif /* __LIBBPF_LIBBPF_INTERNAL_H */
diff --git a/tools/lib/bpf/libbpf_legacy.h b/tools/lib/bpf/libbpf_legacy.h
index 5b7e0155db6a..1e1be467bede 100644
--- a/tools/lib/bpf/libbpf_legacy.h
+++ b/tools/lib/bpf/libbpf_legacy.h
@@ -125,6 +125,8 @@ struct bpf_map;
 struct btf;
 struct btf_ext;
 
+LIBBPF_API struct btf *libbpf_find_kernel_btf(void);
+
 LIBBPF_API enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
 LIBBPF_API enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
 LIBBPF_API const char *bpf_map__get_pin_path(const struct bpf_map *map);
diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c
index 0b5398786bf3..f3a8e8e74eb8 100644
--- a/tools/lib/bpf/libbpf_probes.c
+++ b/tools/lib/bpf/libbpf_probes.c
@@ -193,7 +193,7 @@ static int probe_map_create(enum bpf_map_type map_type)
 	LIBBPF_OPTS(bpf_map_create_opts, opts);
 	int key_size, value_size, max_entries;
 	__u32 btf_key_type_id = 0, btf_value_type_id = 0;
-	int fd = -1, btf_fd = -1, fd_inner = -1, exp_err = 0, err;
+	int fd = -1, btf_fd = -1, fd_inner = -1, exp_err = 0, err = 0;
 
 	key_size	= sizeof(__u32);
 	value_size	= sizeof(__u32);
@@ -231,6 +231,7 @@ static int probe_map_create(enum bpf_map_type map_type)
 			return btf_fd;
 		break;
 	case BPF_MAP_TYPE_RINGBUF:
+	case BPF_MAP_TYPE_USER_RINGBUF:
 		key_size = 0;
 		value_size = 0;
 		max_entries = 4096;
diff --git a/tools/lib/bpf/libbpf_version.h b/tools/lib/bpf/libbpf_version.h
index 2fb2f4290080..e944f5bce728 100644
--- a/tools/lib/bpf/libbpf_version.h
+++ b/tools/lib/bpf/libbpf_version.h
@@ -4,6 +4,6 @@
 #define __LIBBPF_VERSION_H
 
 #define LIBBPF_MAJOR_VERSION 1
-#define LIBBPF_MINOR_VERSION 0
+#define LIBBPF_MINOR_VERSION 1
 
 #endif /* __LIBBPF_VERSION_H */
diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c
index 6c013168032d..35104580870c 100644
--- a/tools/lib/bpf/netlink.c
+++ b/tools/lib/bpf/netlink.c
@@ -587,11 +587,12 @@ static int get_tc_info(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn,
 
 static int tc_add_fd_and_name(struct libbpf_nla_req *req, int fd)
 {
-	struct bpf_prog_info info = {};
+	struct bpf_prog_info info;
 	__u32 info_len = sizeof(info);
 	char name[256];
 	int len, ret;
 
+	memset(&info, 0, info_len);
 	ret = bpf_obj_get_info_by_fd(fd, &info, &info_len);
 	if (ret < 0)
 		return ret;
diff --git a/tools/lib/bpf/nlattr.c b/tools/lib/bpf/nlattr.c
index f57e77a6e40f..3900d052ed19 100644
--- a/tools/lib/bpf/nlattr.c
+++ b/tools/lib/bpf/nlattr.c
@@ -32,7 +32,7 @@ static struct nlattr *nla_next(const struct nlattr *nla, int *remaining)
 
 static int nla_ok(const struct nlattr *nla, int remaining)
 {
-	return remaining >= sizeof(*nla) &&
+	return remaining >= (int)sizeof(*nla) &&
 	       nla->nla_len >= sizeof(*nla) &&
 	       nla->nla_len <= remaining;
 }
diff --git a/tools/lib/bpf/ringbuf.c b/tools/lib/bpf/ringbuf.c
index 8bc117bcc7bc..d285171d4b69 100644
--- a/tools/lib/bpf/ringbuf.c
+++ b/tools/lib/bpf/ringbuf.c
@@ -16,6 +16,7 @@
 #include <asm/barrier.h>
 #include <sys/mman.h>
 #include <sys/epoll.h>
+#include <time.h>
 
 #include "libbpf.h"
 #include "libbpf_internal.h"
@@ -39,6 +40,23 @@ struct ring_buffer {
 	int ring_cnt;
 };
 
+struct user_ring_buffer {
+	struct epoll_event event;
+	unsigned long *consumer_pos;
+	unsigned long *producer_pos;
+	void *data;
+	unsigned long mask;
+	size_t page_size;
+	int map_fd;
+	int epoll_fd;
+};
+
+/* 8-byte ring buffer header structure */
+struct ringbuf_hdr {
+	__u32 len;
+	__u32 pad;
+};
+
 static void ringbuf_unmap_ring(struct ring_buffer *rb, struct ring *r)
 {
 	if (r->consumer_pos) {
@@ -300,3 +318,256 @@ int ring_buffer__epoll_fd(const struct ring_buffer *rb)
 {
 	return rb->epoll_fd;
 }
+
+static void user_ringbuf_unmap_ring(struct user_ring_buffer *rb)
+{
+	if (rb->consumer_pos) {
+		munmap(rb->consumer_pos, rb->page_size);
+		rb->consumer_pos = NULL;
+	}
+	if (rb->producer_pos) {
+		munmap(rb->producer_pos, rb->page_size + 2 * (rb->mask + 1));
+		rb->producer_pos = NULL;
+	}
+}
+
+void user_ring_buffer__free(struct user_ring_buffer *rb)
+{
+	if (!rb)
+		return;
+
+	user_ringbuf_unmap_ring(rb);
+
+	if (rb->epoll_fd >= 0)
+		close(rb->epoll_fd);
+
+	free(rb);
+}
+
+static int user_ringbuf_map(struct user_ring_buffer *rb, int map_fd)
+{
+	struct bpf_map_info info;
+	__u32 len = sizeof(info);
+	void *tmp;
+	struct epoll_event *rb_epoll;
+	int err;
+
+	memset(&info, 0, sizeof(info));
+
+	err = bpf_obj_get_info_by_fd(map_fd, &info, &len);
+	if (err) {
+		err = -errno;
+		pr_warn("user ringbuf: failed to get map info for fd=%d: %d\n", map_fd, err);
+		return err;
+	}
+
+	if (info.type != BPF_MAP_TYPE_USER_RINGBUF) {
+		pr_warn("user ringbuf: map fd=%d is not BPF_MAP_TYPE_USER_RINGBUF\n", map_fd);
+		return -EINVAL;
+	}
+
+	rb->map_fd = map_fd;
+	rb->mask = info.max_entries - 1;
+
+	/* Map read-only consumer page */
+	tmp = mmap(NULL, rb->page_size, PROT_READ, MAP_SHARED, map_fd, 0);
+	if (tmp == MAP_FAILED) {
+		err = -errno;
+		pr_warn("user ringbuf: failed to mmap consumer page for map fd=%d: %d\n",
+			map_fd, err);
+		return err;
+	}
+	rb->consumer_pos = tmp;
+
+	/* Map read-write the producer page and data pages. We map the data
+	 * region as twice the total size of the ring buffer to allow the
+	 * simple reading and writing of samples that wrap around the end of
+	 * the buffer.  See the kernel implementation for details.
+	 */
+	tmp = mmap(NULL, rb->page_size + 2 * info.max_entries,
+		   PROT_READ | PROT_WRITE, MAP_SHARED, map_fd, rb->page_size);
+	if (tmp == MAP_FAILED) {
+		err = -errno;
+		pr_warn("user ringbuf: failed to mmap data pages for map fd=%d: %d\n",
+			map_fd, err);
+		return err;
+	}
+
+	rb->producer_pos = tmp;
+	rb->data = tmp + rb->page_size;
+
+	rb_epoll = &rb->event;
+	rb_epoll->events = EPOLLOUT;
+	if (epoll_ctl(rb->epoll_fd, EPOLL_CTL_ADD, map_fd, rb_epoll) < 0) {
+		err = -errno;
+		pr_warn("user ringbuf: failed to epoll add map fd=%d: %d\n", map_fd, err);
+		return err;
+	}
+
+	return 0;
+}
+
+struct user_ring_buffer *
+user_ring_buffer__new(int map_fd, const struct user_ring_buffer_opts *opts)
+{
+	struct user_ring_buffer *rb;
+	int err;
+
+	if (!OPTS_VALID(opts, user_ring_buffer_opts))
+		return errno = EINVAL, NULL;
+
+	rb = calloc(1, sizeof(*rb));
+	if (!rb)
+		return errno = ENOMEM, NULL;
+
+	rb->page_size = getpagesize();
+
+	rb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
+	if (rb->epoll_fd < 0) {
+		err = -errno;
+		pr_warn("user ringbuf: failed to create epoll instance: %d\n", err);
+		goto err_out;
+	}
+
+	err = user_ringbuf_map(rb, map_fd);
+	if (err)
+		goto err_out;
+
+	return rb;
+
+err_out:
+	user_ring_buffer__free(rb);
+	return errno = -err, NULL;
+}
+
+static void user_ringbuf_commit(struct user_ring_buffer *rb, void *sample, bool discard)
+{
+	__u32 new_len;
+	struct ringbuf_hdr *hdr;
+	uintptr_t hdr_offset;
+
+	hdr_offset = rb->mask + 1 + (sample - rb->data) - BPF_RINGBUF_HDR_SZ;
+	hdr = rb->data + (hdr_offset & rb->mask);
+
+	new_len = hdr->len & ~BPF_RINGBUF_BUSY_BIT;
+	if (discard)
+		new_len |= BPF_RINGBUF_DISCARD_BIT;
+
+	/* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in
+	 * the kernel.
+	 */
+	__atomic_exchange_n(&hdr->len, new_len, __ATOMIC_ACQ_REL);
+}
+
+void user_ring_buffer__discard(struct user_ring_buffer *rb, void *sample)
+{
+	user_ringbuf_commit(rb, sample, true);
+}
+
+void user_ring_buffer__submit(struct user_ring_buffer *rb, void *sample)
+{
+	user_ringbuf_commit(rb, sample, false);
+}
+
+void *user_ring_buffer__reserve(struct user_ring_buffer *rb, __u32 size)
+{
+	__u32 avail_size, total_size, max_size;
+	/* 64-bit to avoid overflow in case of extreme application behavior */
+	__u64 cons_pos, prod_pos;
+	struct ringbuf_hdr *hdr;
+
+	/* Synchronizes with smp_store_release() in __bpf_user_ringbuf_peek() in
+	 * the kernel.
+	 */
+	cons_pos = smp_load_acquire(rb->consumer_pos);
+	/* Synchronizes with smp_store_release() in user_ringbuf_commit() */
+	prod_pos = smp_load_acquire(rb->producer_pos);
+
+	max_size = rb->mask + 1;
+	avail_size = max_size - (prod_pos - cons_pos);
+	/* Round up total size to a multiple of 8. */
+	total_size = (size + BPF_RINGBUF_HDR_SZ + 7) / 8 * 8;
+
+	if (total_size > max_size)
+		return errno = E2BIG, NULL;
+
+	if (avail_size < total_size)
+		return errno = ENOSPC, NULL;
+
+	hdr = rb->data + (prod_pos & rb->mask);
+	hdr->len = size | BPF_RINGBUF_BUSY_BIT;
+	hdr->pad = 0;
+
+	/* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in
+	 * the kernel.
+	 */
+	smp_store_release(rb->producer_pos, prod_pos + total_size);
+
+	return (void *)rb->data + ((prod_pos + BPF_RINGBUF_HDR_SZ) & rb->mask);
+}
+
+static __u64 ns_elapsed_timespec(const struct timespec *start, const struct timespec *end)
+{
+	__u64 start_ns, end_ns, ns_per_s = 1000000000;
+
+	start_ns = (__u64)start->tv_sec * ns_per_s + start->tv_nsec;
+	end_ns = (__u64)end->tv_sec * ns_per_s + end->tv_nsec;
+
+	return end_ns - start_ns;
+}
+
+void *user_ring_buffer__reserve_blocking(struct user_ring_buffer *rb, __u32 size, int timeout_ms)
+{
+	void *sample;
+	int err, ms_remaining = timeout_ms;
+	struct timespec start;
+
+	if (timeout_ms < 0 && timeout_ms != -1)
+		return errno = EINVAL, NULL;
+
+	if (timeout_ms != -1) {
+		err = clock_gettime(CLOCK_MONOTONIC, &start);
+		if (err)
+			return NULL;
+	}
+
+	do {
+		int cnt, ms_elapsed;
+		struct timespec curr;
+		__u64 ns_per_ms = 1000000;
+
+		sample = user_ring_buffer__reserve(rb, size);
+		if (sample)
+			return sample;
+		else if (errno != ENOSPC)
+			return NULL;
+
+		/* The kernel guarantees at least one event notification
+		 * delivery whenever at least one sample is drained from the
+		 * ring buffer in an invocation to bpf_ringbuf_drain(). Other
+		 * additional events may be delivered at any time, but only one
+		 * event is guaranteed per bpf_ringbuf_drain() invocation,
+		 * provided that a sample is drained, and the BPF program did
+		 * not pass BPF_RB_NO_WAKEUP to bpf_ringbuf_drain(). If
+		 * BPF_RB_FORCE_WAKEUP is passed to bpf_ringbuf_drain(), a
+		 * wakeup event will be delivered even if no samples are
+		 * drained.
+		 */
+		cnt = epoll_wait(rb->epoll_fd, &rb->event, 1, ms_remaining);
+		if (cnt < 0)
+			return NULL;
+
+		if (timeout_ms == -1)
+			continue;
+
+		err = clock_gettime(CLOCK_MONOTONIC, &curr);
+		if (err)
+			return NULL;
+
+		ms_elapsed = ns_elapsed_timespec(&start, &curr) / ns_per_ms;
+		ms_remaining = timeout_ms - ms_elapsed;
+	} while (ms_remaining > 0);
+
+	/* Try one more time to reserve a sample after the specified timeout has elapsed. */
+	return user_ring_buffer__reserve(rb, size);
+}
diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h
index 70adf7b119b9..1e82ab06c3eb 100644
--- a/tools/lib/bpf/skel_internal.h
+++ b/tools/lib/bpf/skel_internal.h
@@ -251,6 +251,29 @@ static inline int skel_map_update_elem(int fd, const void *key,
 	return skel_sys_bpf(BPF_MAP_UPDATE_ELEM, &attr, attr_sz);
 }
 
+static inline int skel_map_delete_elem(int fd, const void *key)
+{
+	const size_t attr_sz = offsetofend(union bpf_attr, flags);
+	union bpf_attr attr;
+
+	memset(&attr, 0, attr_sz);
+	attr.map_fd = fd;
+	attr.key = (long)key;
+
+	return skel_sys_bpf(BPF_MAP_DELETE_ELEM, &attr, attr_sz);
+}
+
+static inline int skel_map_get_fd_by_id(__u32 id)
+{
+	const size_t attr_sz = offsetofend(union bpf_attr, flags);
+	union bpf_attr attr;
+
+	memset(&attr, 0, attr_sz);
+	attr.map_id = id;
+
+	return skel_sys_bpf(BPF_MAP_GET_FD_BY_ID, &attr, attr_sz);
+}
+
 static inline int skel_raw_tracepoint_open(const char *name, int prog_fd)
 {
 	const size_t attr_sz = offsetofend(union bpf_attr, raw_tracepoint.prog_fd);
@@ -285,6 +308,8 @@ static inline int skel_link_create(int prog_fd, int target_fd,
 
 static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
 {
+	const size_t prog_load_attr_sz = offsetofend(union bpf_attr, fd_array);
+	const size_t test_run_attr_sz = offsetofend(union bpf_attr, test);
 	int map_fd = -1, prog_fd = -1, key = 0, err;
 	union bpf_attr attr;
 
@@ -302,7 +327,7 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
 		goto out;
 	}
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, prog_load_attr_sz);
 	attr.prog_type = BPF_PROG_TYPE_SYSCALL;
 	attr.insns = (long) opts->insns;
 	attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn);
@@ -313,18 +338,18 @@ static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts)
 	attr.log_size = opts->ctx->log_size;
 	attr.log_buf = opts->ctx->log_buf;
 	attr.prog_flags = BPF_F_SLEEPABLE;
-	err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr));
+	err = prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, prog_load_attr_sz);
 	if (prog_fd < 0) {
 		opts->errstr = "failed to load loader prog";
 		set_err;
 		goto out;
 	}
 
-	memset(&attr, 0, sizeof(attr));
+	memset(&attr, 0, test_run_attr_sz);
 	attr.test.prog_fd = prog_fd;
 	attr.test.ctx_in = (long) opts->ctx;
 	attr.test.ctx_size_in = opts->ctx->sz;
-	err = skel_sys_bpf(BPF_PROG_RUN, &attr, sizeof(attr));
+	err = skel_sys_bpf(BPF_PROG_RUN, &attr, test_run_attr_sz);
 	if (err < 0 || (int)attr.test.retval < 0) {
 		opts->errstr = "failed to execute loader prog";
 		if (err < 0) {
diff --git a/tools/lib/bpf/usdt.bpf.h b/tools/lib/bpf/usdt.bpf.h
index 4f2adc0bd6ca..fdfd235e52c4 100644
--- a/tools/lib/bpf/usdt.bpf.h
+++ b/tools/lib/bpf/usdt.bpf.h
@@ -232,7 +232,7 @@ long bpf_usdt_cookie(struct pt_regs *ctx)
  */
 #define BPF_USDT(name, args...)						    \
 name(struct pt_regs *ctx);						    \
-static __attribute__((always_inline)) typeof(name(0))			    \
+static __always_inline typeof(name(0))					    \
 ____##name(struct pt_regs *ctx, ##args);				    \
 typeof(name(0)) name(struct pt_regs *ctx)				    \
 {									    \
@@ -241,7 +241,7 @@ typeof(name(0)) name(struct pt_regs *ctx)				    \
         return ____##name(___bpf_usdt_args(args));			    \
         _Pragma("GCC diagnostic pop")					    \
 }									    \
-static __attribute__((always_inline)) typeof(name(0))			    \
+static __always_inline typeof(name(0))					    \
 ____##name(struct pt_regs *ctx, ##args)
 
 #endif /* __USDT_BPF_H__ */
diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c
index d18e37982344..e83b497c2245 100644
--- a/tools/lib/bpf/usdt.c
+++ b/tools/lib/bpf/usdt.c
@@ -282,7 +282,7 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
 	 * If this is not supported, USDTs with semaphores will not be supported.
 	 * Added in: a6ca88b241d5 ("trace_uprobe: support reference counter in fd-based uprobe")
 	 */
-	man->has_sema_refcnt = access(ref_ctr_sysfs_path, F_OK) == 0;
+	man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0;
 
 	return man;
 }
diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h
index 93bf93a59c99..d8ae4e944467 100644
--- a/tools/lib/perf/include/perf/event.h
+++ b/tools/lib/perf/include/perf/event.h
@@ -97,7 +97,7 @@ struct perf_record_throttle {
 };
 
 #ifndef KSYM_NAME_LEN
-#define KSYM_NAME_LEN 256
+#define KSYM_NAME_LEN 512
 #endif
 
 struct perf_record_ksymbol {
diff --git a/tools/lib/symbol/kallsyms.h b/tools/lib/symbol/kallsyms.h
index 72ab9870454b..542f9b059c3b 100644
--- a/tools/lib/symbol/kallsyms.h
+++ b/tools/lib/symbol/kallsyms.h
@@ -7,7 +7,7 @@
 #include <linux/types.h>
 
 #ifndef KSYM_NAME_LEN
-#define KSYM_NAME_LEN 256
+#define KSYM_NAME_LEN 512
 #endif
 
 static inline u8 kallsyms2elf_binding(char type)
diff --git a/tools/memory-model/Documentation/litmus-tests.txt b/tools/memory-model/Documentation/litmus-tests.txt
index 8a9d5d2787f9..26554b1c5575 100644
--- a/tools/memory-model/Documentation/litmus-tests.txt
+++ b/tools/memory-model/Documentation/litmus-tests.txt
@@ -946,22 +946,39 @@ Limitations of the Linux-kernel memory model (LKMM) include:
 	carrying a dependency, then the compiler can break that dependency
 	by substituting a constant of that value.
 
-	Conversely, LKMM sometimes doesn't recognize that a particular
-	optimization is not allowed, and as a result, thinks that a
-	dependency is not present (because the optimization would break it).
-	The memory model misses some pretty obvious control dependencies
-	because of this limitation.  A simple example is:
+	Conversely, LKMM will sometimes overestimate the amount of
+	reordering compilers and CPUs can carry out, leading it to miss
+	some pretty obvious cases of ordering.  A simple example is:
 
 		r1 = READ_ONCE(x);
 		if (r1 == 0)
 			smp_mb();
 		WRITE_ONCE(y, 1);
 
-	There is a control dependency from the READ_ONCE to the WRITE_ONCE,
-	even when r1 is nonzero, but LKMM doesn't realize this and thinks
-	that the write may execute before the read if r1 != 0.  (Yes, that
-	doesn't make sense if you think about it, but the memory model's
-	intelligence is limited.)
+	The WRITE_ONCE() does not depend on the READ_ONCE(), and as a
+	result, LKMM does not claim ordering.  However, even though no
+	dependency is present, the WRITE_ONCE() will not be executed before
+	the READ_ONCE().  There are two reasons for this:
+
+                The presence of the smp_mb() in one of the branches
+                prevents the compiler from moving the WRITE_ONCE()
+                up before the "if" statement, since the compiler has
+                to assume that r1 will sometimes be 0 (but see the
+                comment below);
+
+                CPUs do not execute stores before po-earlier conditional
+                branches, even in cases where the store occurs after the
+                two arms of the branch have recombined.
+
+	It is clear that it is not dangerous in the slightest for LKMM to
+	make weaker guarantees than architectures.  In fact, it is
+	desirable, as it gives compilers room for making optimizations.
+	For instance, suppose that a 0 value in r1 would trigger undefined
+	behavior elsewhere.  Then a clever compiler might deduce that r1
+	can never be 0 in the if condition.  As a result, said clever
+	compiler might deem it safe to optimize away the smp_mb(),
+	eliminating the branch and any ordering an architecture would
+	guarantee otherwise.
 
 2.	Multiple access sizes for a single variable are not supported,
 	and neither are misaligned or partially overlapping accesses.
diff --git a/tools/objtool/check.c b/tools/objtool/check.c
index e55fdf952a3a..a8cf38639fe8 100644
--- a/tools/objtool/check.c
+++ b/tools/objtool/check.c
@@ -1073,6 +1073,9 @@ static const char *uaccess_safe_builtin[] = {
 	"copy_mc_fragile_handle_tail",
 	"copy_mc_enhanced_fast_string",
 	"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
+	"clear_user_erms",
+	"clear_user_rep_good",
+	"clear_user_original",
 	NULL
 };
 
@@ -3316,6 +3319,10 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
 		next_insn = next_insn_to_validate(file, insn);
 
 		if (func && insn->func && func != insn->func->pfunc) {
+			/* Ignore KCFI type preambles, which always fall through */
+			if (!strncmp(func->name, "__cfi_", 6))
+				return 0;
+
 			WARN("%s() falls through to next function %s()",
 			     func->name, insn->func->name);
 			return 1;
@@ -4113,7 +4120,9 @@ static int validate_ibt(struct objtool_file *file)
 		    !strcmp(sec->name, "__bug_table")			||
 		    !strcmp(sec->name, "__ex_table")			||
 		    !strcmp(sec->name, "__jump_table")			||
-		    !strcmp(sec->name, "__mcount_loc"))
+		    !strcmp(sec->name, "__mcount_loc")			||
+		    !strcmp(sec->name, ".kcfi_traps")			||
+		    strstr(sec->name, "__patchable_function_entries"))
 			continue;
 
 		list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c
index c25e957c1e52..7e24b09b1163 100644
--- a/tools/objtool/elf.c
+++ b/tools/objtool/elf.c
@@ -619,6 +619,11 @@ static int elf_update_symbol(struct elf *elf, struct section *symtab,
 	Elf64_Xword entsize = symtab->sh.sh_entsize;
 	int max_idx, idx = sym->idx;
 	Elf_Scn *s, *t = NULL;
+	bool is_special_shndx = sym->sym.st_shndx >= SHN_LORESERVE &&
+				sym->sym.st_shndx != SHN_XINDEX;
+
+	if (is_special_shndx)
+		shndx = sym->sym.st_shndx;
 
 	s = elf_getscn(elf->elf, symtab->idx);
 	if (!s) {
@@ -704,7 +709,7 @@ static int elf_update_symbol(struct elf *elf, struct section *symtab,
 	}
 
 	/* setup extended section index magic and write the symbol */
-	if (shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) {
+	if ((shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) || is_special_shndx) {
 		sym->sym.st_shndx = shndx;
 		if (!shndx_data)
 			shndx = 0;
diff --git a/tools/perf/tests/mmap-basic.c b/tools/perf/tests/mmap-basic.c
index dfb6173b2a82..9e9a2b67de19 100644
--- a/tools/perf/tests/mmap-basic.c
+++ b/tools/perf/tests/mmap-basic.c
@@ -114,8 +114,7 @@ static int test__basic_mmap(struct test_suite *test __maybe_unused, int subtest
 
 	for (i = 0; i < nsyscalls; ++i)
 		for (j = 0; j < expected_nr_events[i]; ++j) {
-			int foo = syscalls[i]();
-			++foo;
+			syscalls[i]();
 		}
 
 	md = &evlist->mmap[0];
diff --git a/tools/perf/tests/perf-record.c b/tools/perf/tests/perf-record.c
index 6a001fcfed68..4952abe716f3 100644
--- a/tools/perf/tests/perf-record.c
+++ b/tools/perf/tests/perf-record.c
@@ -332,7 +332,7 @@ out_delete_evlist:
 out:
 	if (err == -EACCES)
 		return TEST_SKIP;
-	if (err < 0)
+	if (err < 0 || errs != 0)
 		return TEST_FAIL;
 	return TEST_OK;
 }
diff --git a/tools/perf/tests/shell/record.sh b/tools/perf/tests/shell/record.sh
index 00c7285ce1ac..301f95427159 100755
--- a/tools/perf/tests/shell/record.sh
+++ b/tools/perf/tests/shell/record.sh
@@ -61,7 +61,7 @@ test_register_capture() {
     echo "Register capture test [Skipped missing registers]"
     return
   fi
-  if ! perf record -o - --intr-regs=di,r8,dx,cx -e cpu/br_inst_retired.near_call/p \
+  if ! perf record -o - --intr-regs=di,r8,dx,cx -e br_inst_retired.near_call:p \
     -c 1000 --per-thread true 2> /dev/null \
     | perf script -F ip,sym,iregs -i - 2> /dev/null \
     | egrep -q "DI:"
diff --git a/tools/perf/tests/shell/test_brstack.sh b/tools/perf/tests/shell/test_brstack.sh
index c644f94a6500..ec801cffae6b 100755
--- a/tools/perf/tests/shell/test_brstack.sh
+++ b/tools/perf/tests/shell/test_brstack.sh
@@ -12,7 +12,8 @@ if ! [ -x "$(command -v cc)" ]; then
 fi
 
 # skip the test if the hardware doesn't support branch stack sampling
-perf record -b -o- -B true > /dev/null 2>&1 || exit 2
+# and if the architecture doesn't support filter types: any,save_type,u
+perf record -b -o- -B --branch-filter any,save_type,u true > /dev/null 2>&1 || exit 2
 
 TMPDIR=$(mktemp -d /tmp/__perf_test.program.XXXXX)
 
diff --git a/tools/perf/tests/vmlinux-kallsyms.c b/tools/perf/tests/vmlinux-kallsyms.c
index 4fd8d703ff19..8ab035b55875 100644
--- a/tools/perf/tests/vmlinux-kallsyms.c
+++ b/tools/perf/tests/vmlinux-kallsyms.c
@@ -43,10 +43,11 @@ static bool is_ignored_symbol(const char *name, char type)
 	/* Symbol names that begin with the following are ignored.*/
 	static const char * const ignored_prefixes[] = {
 		"$",			/* local symbols for ARM, MIPS, etc. */
-		".LASANPC",		/* s390 kasan local symbols */
+		".L",			/* local labels, .LBB,.Ltmpxxx,.L__unnamed_xx,.LASANPC, etc. */
 		"__crc_",		/* modversions */
 		"__efistub_",		/* arm64 EFI stub namespace */
-		"__kvm_nvhe_",		/* arm64 non-VHE KVM namespace */
+		"__kvm_nvhe_$",		/* arm64 local symbols in non-VHE KVM namespace */
+		"__kvm_nvhe_.L",	/* arm64 local symbols in non-VHE KVM namespace */
 		"__AArch64ADRPThunk_",	/* arm64 lld */
 		"__ARMV5PILongThunk_",	/* arm lld */
 		"__ARMV7PILongThunk_",
diff --git a/tools/perf/util/Build b/tools/perf/util/Build
index 9dfae1bda9cc..485e1a343165 100644
--- a/tools/perf/util/Build
+++ b/tools/perf/util/Build
@@ -269,7 +269,7 @@ CFLAGS_expr-flex.o          += $(flex_flags)
 bison_flags := -DYYENABLE_NLS=0
 BISON_GE_35 := $(shell expr $(shell $(BISON) --version | grep bison | sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\)/\1\2/g') \>\= 35)
 ifeq ($(BISON_GE_35),1)
-  bison_flags += -Wno-unused-parameter -Wno-nested-externs -Wno-implicit-function-declaration -Wno-switch-enum
+  bison_flags += -Wno-unused-parameter -Wno-nested-externs -Wno-implicit-function-declaration -Wno-switch-enum -Wno-unused-but-set-variable -Wno-unknown-warning-option
 else
   bison_flags += -w
 endif
diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c
index 22dcfe07e886..906476a839e1 100644
--- a/tools/perf/util/arm-spe.c
+++ b/tools/perf/util/arm-spe.c
@@ -498,7 +498,7 @@ static void arm_spe__synth_data_source_generic(const struct arm_spe_record *reco
 static u64 arm_spe__synth_data_source(const struct arm_spe_record *record, u64 midr)
 {
 	union perf_mem_data_src	data_src = { 0 };
-	bool is_neoverse = is_midr_in_range(midr, neoverse_spe);
+	bool is_neoverse = is_midr_in_range_list(midr, neoverse_spe);
 
 	if (record->op == ARM_SPE_LD)
 		data_src.mem_op = PERF_MEM_OP_LOAD;
diff --git a/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c b/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c
index c72f8ad96f75..9aa8cdd93de4 100644
--- a/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c
+++ b/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c
@@ -48,6 +48,7 @@ const volatile __u32 num_cpus = 1;
 
 int enabled = 0;
 int use_cgroup_v2 = 0;
+int perf_subsys_id = -1;
 
 static inline int get_cgroup_v1_idx(__u32 *cgrps, int size)
 {
@@ -58,7 +59,15 @@ static inline int get_cgroup_v1_idx(__u32 *cgrps, int size)
 	int level;
 	int cnt;
 
-	cgrp = BPF_CORE_READ(p, cgroups, subsys[perf_event_cgrp_id], cgroup);
+	if (perf_subsys_id == -1) {
+#if __has_builtin(__builtin_preserve_enum_value)
+		perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id,
+						     perf_event_cgrp_id);
+#else
+		perf_subsys_id = perf_event_cgrp_id;
+#endif
+	}
+	cgrp = BPF_CORE_READ(p, cgroups, subsys[perf_subsys_id], cgroup);
 	level = BPF_CORE_READ(cgrp, level);
 
 	for (cnt = 0; i < MAX_LEVELS; i++) {
diff --git a/tools/perf/util/bpf_skel/off_cpu.bpf.c b/tools/perf/util/bpf_skel/off_cpu.bpf.c
index c4ba2bcf179f..38e3b287dbb2 100644
--- a/tools/perf/util/bpf_skel/off_cpu.bpf.c
+++ b/tools/perf/util/bpf_skel/off_cpu.bpf.c
@@ -94,6 +94,8 @@ const volatile bool has_prev_state = false;
 const volatile bool needs_cgroup = false;
 const volatile bool uses_cgroup_v1 = false;
 
+int perf_subsys_id = -1;
+
 /*
  * Old kernel used to call it task_struct->state and now it's '__state'.
  * Use BPF CO-RE "ignored suffix rule" to deal with it like below:
@@ -119,11 +121,19 @@ static inline __u64 get_cgroup_id(struct task_struct *t)
 {
 	struct cgroup *cgrp;
 
-	if (uses_cgroup_v1)
-		cgrp = BPF_CORE_READ(t, cgroups, subsys[perf_event_cgrp_id], cgroup);
-	else
-		cgrp = BPF_CORE_READ(t, cgroups, dfl_cgrp);
+	if (!uses_cgroup_v1)
+		return BPF_CORE_READ(t, cgroups, dfl_cgrp, kn, id);
+
+	if (perf_subsys_id == -1) {
+#if __has_builtin(__builtin_preserve_enum_value)
+		perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id,
+						     perf_event_cgrp_id);
+#else
+		perf_subsys_id = perf_event_cgrp_id;
+#endif
+	}
 
+	cgrp = BPF_CORE_READ(t, cgroups, subsys[perf_subsys_id], cgroup);
 	return BPF_CORE_READ(cgrp, kn, id);
 }
 
diff --git a/tools/perf/util/parse-events-hybrid.c b/tools/perf/util/parse-events-hybrid.c
index 284f8eabd3b9..7c9f9150bad5 100644
--- a/tools/perf/util/parse-events-hybrid.c
+++ b/tools/perf/util/parse-events-hybrid.c
@@ -33,7 +33,8 @@ static void config_hybrid_attr(struct perf_event_attr *attr,
 	 * If the PMU type ID is 0, the PERF_TYPE_RAW will be applied.
 	 */
 	attr->type = type;
-	attr->config = attr->config | ((__u64)pmu_type << PERF_PMU_TYPE_SHIFT);
+	attr->config = (attr->config & PERF_HW_EVENT_MASK) |
+			((__u64)pmu_type << PERF_PMU_TYPE_SHIFT);
 }
 
 static int create_event_hybrid(__u32 config_type, int *idx,
@@ -48,13 +49,25 @@ static int create_event_hybrid(__u32 config_type, int *idx,
 	__u64 config = attr->config;
 
 	config_hybrid_attr(attr, config_type, pmu->type);
+
+	/*
+	 * Some hybrid hardware cache events are only available on one CPU
+	 * PMU. For example, the 'L1-dcache-load-misses' is only available
+	 * on cpu_core, while the 'L1-icache-loads' is only available on
+	 * cpu_atom. We need to remove "not supported" hybrid cache events.
+	 */
+	if (attr->type == PERF_TYPE_HW_CACHE
+	    && !is_event_supported(attr->type, attr->config))
+		return 0;
+
 	evsel = parse_events__add_event_hybrid(list, idx, attr, name, metric_id,
 					       pmu, config_terms);
-	if (evsel)
+	if (evsel) {
 		evsel->pmu_name = strdup(pmu->name);
-	else
+		if (!evsel->pmu_name)
+			return -ENOMEM;
+	} else
 		return -ENOMEM;
-
 	attr->type = type;
 	attr->config = config;
 	return 0;
diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c
index f05e15acd33f..f3b2c2a87456 100644
--- a/tools/perf/util/parse-events.c
+++ b/tools/perf/util/parse-events.c
@@ -28,6 +28,7 @@
 #include "util/parse-events-hybrid.h"
 #include "util/pmu-hybrid.h"
 #include "tracepoint.h"
+#include "thread_map.h"
 
 #define MAX_NAME_LEN 100
 
@@ -157,6 +158,44 @@ struct event_symbol event_symbols_sw[PERF_COUNT_SW_MAX] = {
 #define PERF_EVENT_TYPE(config)		__PERF_EVENT_FIELD(config, TYPE)
 #define PERF_EVENT_ID(config)		__PERF_EVENT_FIELD(config, EVENT)
 
+bool is_event_supported(u8 type, u64 config)
+{
+	bool ret = true;
+	int open_return;
+	struct evsel *evsel;
+	struct perf_event_attr attr = {
+		.type = type,
+		.config = config,
+		.disabled = 1,
+	};
+	struct perf_thread_map *tmap = thread_map__new_by_tid(0);
+
+	if (tmap == NULL)
+		return false;
+
+	evsel = evsel__new(&attr);
+	if (evsel) {
+		open_return = evsel__open(evsel, NULL, tmap);
+		ret = open_return >= 0;
+
+		if (open_return == -EACCES) {
+			/*
+			 * This happens if the paranoid value
+			 * /proc/sys/kernel/perf_event_paranoid is set to 2
+			 * Re-run with exclude_kernel set; we don't do that
+			 * by default as some ARM machines do not support it.
+			 *
+			 */
+			evsel->core.attr.exclude_kernel = 1;
+			ret = evsel__open(evsel, NULL, tmap) >= 0;
+		}
+		evsel__delete(evsel);
+	}
+
+	perf_thread_map__put(tmap);
+	return ret;
+}
+
 const char *event_type(int type)
 {
 	switch (type) {
diff --git a/tools/perf/util/parse-events.h b/tools/perf/util/parse-events.h
index 7e6a601d9cd0..07df7bb7b042 100644
--- a/tools/perf/util/parse-events.h
+++ b/tools/perf/util/parse-events.h
@@ -19,6 +19,7 @@ struct option;
 struct perf_pmu;
 
 bool have_tracepoints(struct list_head *evlist);
+bool is_event_supported(u8 type, u64 config);
 
 const char *event_type(int type);
 
diff --git a/tools/perf/util/print-events.c b/tools/perf/util/print-events.c
index ba1ab5134685..c4d5d87fae2f 100644
--- a/tools/perf/util/print-events.c
+++ b/tools/perf/util/print-events.c
@@ -22,7 +22,6 @@
 #include "probe-file.h"
 #include "string2.h"
 #include "strlist.h"
-#include "thread_map.h"
 #include "tracepoint.h"
 #include "pfm.h"
 #include "pmu-hybrid.h"
@@ -239,44 +238,6 @@ void print_sdt_events(const char *subsys_glob, const char *event_glob,
 	strlist__delete(sdtlist);
 }
 
-static bool is_event_supported(u8 type, unsigned int config)
-{
-	bool ret = true;
-	int open_return;
-	struct evsel *evsel;
-	struct perf_event_attr attr = {
-		.type = type,
-		.config = config,
-		.disabled = 1,
-	};
-	struct perf_thread_map *tmap = thread_map__new_by_tid(0);
-
-	if (tmap == NULL)
-		return false;
-
-	evsel = evsel__new(&attr);
-	if (evsel) {
-		open_return = evsel__open(evsel, NULL, tmap);
-		ret = open_return >= 0;
-
-		if (open_return == -EACCES) {
-			/*
-			 * This happens if the paranoid value
-			 * /proc/sys/kernel/perf_event_paranoid is set to 2
-			 * Re-run with exclude_kernel set; we don't do that
-			 * by default as some ARM machines do not support it.
-			 *
-			 */
-			evsel->core.attr.exclude_kernel = 1;
-			ret = evsel__open(evsel, NULL, tmap) >= 0;
-		}
-		evsel__delete(evsel);
-	}
-
-	perf_thread_map__put(tmap);
-	return ret;
-}
-
 int print_hwcache_events(const char *event_glob, bool name_only)
 {
 	unsigned int type, op, i, evt_i = 0, evt_num = 0, npmus = 0;
diff --git a/tools/perf/util/scripting-engines/Build b/tools/perf/util/scripting-engines/Build
index c92326c2233a..0f5ba28339cf 100644
--- a/tools/perf/util/scripting-engines/Build
+++ b/tools/perf/util/scripting-engines/Build
@@ -3,4 +3,4 @@ perf-$(CONFIG_LIBPYTHON) += trace-event-python.o
 
 CFLAGS_trace-event-perl.o += $(PERL_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-shadow -Wno-nested-externs -Wno-undef -Wno-switch-default -Wno-bad-function-cast -Wno-declaration-after-statement -Wno-switch-enum
 
-CFLAGS_trace-event-python.o += $(PYTHON_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-shadow -Wno-error=deprecated-declarations
+CFLAGS_trace-event-python.o += $(PYTHON_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-shadow -Wno-deprecated-declarations
diff --git a/tools/power/acpi/tools/pfrut/pfrut.c b/tools/power/acpi/tools/pfrut/pfrut.c
index d79c335594b2..52aa0351533c 100644
--- a/tools/power/acpi/tools/pfrut/pfrut.c
+++ b/tools/power/acpi/tools/pfrut/pfrut.c
@@ -190,7 +190,7 @@ int main(int argc, char *argv[])
 	void *addr_map_capsule;
 	struct stat st;
 	char *log_buf;
-	int ret = 0;
+	int ret;
 
 	if (getuid() != 0) {
 		printf("Please run the tool as root - Exiting.\n");
diff --git a/tools/power/x86/intel-speed-select/hfi-events.c b/tools/power/x86/intel-speed-select/hfi-events.c
index f0ed69721308..be96e90cc2a1 100644
--- a/tools/power/x86/intel-speed-select/hfi-events.c
+++ b/tools/power/x86/intel-speed-select/hfi-events.c
@@ -181,7 +181,10 @@ struct perf_cap {
 
 static void process_hfi_event(struct perf_cap *perf_cap)
 {
-	process_level_change(perf_cap->cpu);
+	struct isst_id id;
+
+	set_isst_id(&id, perf_cap->cpu);
+	process_level_change(&id);
 }
 
 static int handle_event(struct nl_msg *n, void *arg)
diff --git a/tools/power/x86/intel-speed-select/isst-config.c b/tools/power/x86/intel-speed-select/isst-config.c
index 9d35614995ee..a160bad291eb 100644
--- a/tools/power/x86/intel-speed-select/isst-config.c
+++ b/tools/power/x86/intel-speed-select/isst-config.c
@@ -15,7 +15,7 @@ struct process_cmd_struct {
 	int arg;
 };
 
-static const char *version_str = "v1.12";
+static const char *version_str = "v1.13";
 
 static const int supported_api_ver = 1;
 static struct isst_if_platform_info isst_platform_info;
@@ -63,6 +63,7 @@ struct _cpu_map {
 	unsigned short die_id;
 	unsigned short punit_cpu;
 	unsigned short punit_cpu_core;
+	unsigned short initialized;
 };
 struct _cpu_map *cpu_map;
 
@@ -298,10 +299,16 @@ static void store_cpu_topology(void)
 	fclose(fp);
 }
 
-int get_physical_package_id(int cpu)
+static int get_physical_package_id(int cpu)
 {
 	int ret;
 
+	if (cpu < 0)
+		return -1;
+
+	if (cpu_map && cpu_map[cpu].initialized)
+		return cpu_map[cpu].pkg_id;
+
 	ret = parse_int_file(0,
 			"/sys/devices/system/cpu/cpu%d/topology/physical_package_id",
 			cpu);
@@ -316,10 +323,16 @@ int get_physical_package_id(int cpu)
 	return ret;
 }
 
-int get_physical_core_id(int cpu)
+static int get_physical_core_id(int cpu)
 {
 	int ret;
 
+	if (cpu < 0)
+		return -1;
+
+	if (cpu_map && cpu_map[cpu].initialized)
+		return cpu_map[cpu].core_id;
+
 	ret = parse_int_file(0,
 			"/sys/devices/system/cpu/cpu%d/topology/core_id",
 			cpu);
@@ -334,10 +347,16 @@ int get_physical_core_id(int cpu)
 	return ret;
 }
 
-int get_physical_die_id(int cpu)
+static int get_physical_die_id(int cpu)
 {
 	int ret;
 
+	if (cpu < 0)
+		return -1;
+
+	if (cpu_map && cpu_map[cpu].initialized)
+		return cpu_map[cpu].die_id;
+
 	ret = parse_int_file(0,
 			"/sys/devices/system/cpu/cpu%d/topology/die_id",
 			cpu);
@@ -359,6 +378,31 @@ int get_physical_die_id(int cpu)
 	return ret;
 }
 
+void set_isst_id(struct isst_id *id, int cpu)
+{
+	id->cpu = cpu;
+
+	id->pkg = get_physical_package_id(cpu);
+	if (id < 0 || id->pkg >= MAX_PACKAGE_COUNT)
+		id->pkg = -1;
+
+	id->die = get_physical_die_id(cpu);
+	if (id < 0 || id->die >= MAX_DIE_PER_PACKAGE)
+		id->die = -1;
+}
+
+int is_cpu_in_power_domain(int cpu, struct isst_id *id)
+{
+	struct isst_id tid;
+
+	set_isst_id(&tid, cpu);
+
+	if (id->pkg == tid.pkg && id->die == tid.die)
+		return 1;
+
+	return 0;
+}
+
 int get_cpufreq_base_freq(int cpu)
 {
 	return parse_int_file(0, "/sys/devices/system/cpu/cpu%d/cpufreq/base_frequency", cpu);
@@ -410,13 +454,14 @@ static void force_all_cpus_online(void)
 	unlink("/var/run/isst_cpu_topology.dat");
 }
 
-void for_each_online_package_in_set(void (*callback)(int, void *, void *,
+void for_each_online_package_in_set(void (*callback)(struct isst_id *, void *, void *,
 						     void *, void *),
 				    void *arg1, void *arg2, void *arg3,
 				    void *arg4)
 {
 	int max_packages[MAX_PACKAGE_COUNT * MAX_PACKAGE_COUNT];
 	int pkg_index = 0, i;
+	struct isst_id id;
 
 	memset(max_packages, 0xff, sizeof(max_packages));
 	for (i = 0; i < topo_max_cpus; ++i) {
@@ -450,18 +495,20 @@ void for_each_online_package_in_set(void (*callback)(int, void *, void *,
 			}
 		}
 
+		set_isst_id(&id, i);
 		if (!skip && online && callback) {
-			callback(i, arg1, arg2, arg3, arg4);
+			callback(&id, arg1, arg2, arg3, arg4);
 			max_packages[pkg_index++] = pkg_id;
 		}
 	}
 }
 
 static void for_each_online_target_cpu_in_set(
-	void (*callback)(int, void *, void *, void *, void *), void *arg1,
+	void (*callback)(struct isst_id *, void *, void *, void *, void *), void *arg1,
 	void *arg2, void *arg3, void *arg4)
 {
 	int i, found = 0;
+	struct isst_id id;
 
 	for (i = 0; i < topo_max_cpus; ++i) {
 		int online;
@@ -475,8 +522,9 @@ static void for_each_online_target_cpu_in_set(
 			online =
 				1; /* online entry for CPU 0 needs some special configs */
 
+		set_isst_id(&id, i);
 		if (online && callback) {
-			callback(i, arg1, arg2, arg3, arg4);
+			callback(&id, arg1, arg2, arg3, arg4);
 			found = 1;
 		}
 	}
@@ -536,47 +584,8 @@ void free_cpu_set(cpu_set_t *cpu_set)
 }
 
 static int cpu_cnt[MAX_PACKAGE_COUNT][MAX_DIE_PER_PACKAGE];
-static long long core_mask[MAX_PACKAGE_COUNT][MAX_DIE_PER_PACKAGE];
-static void set_cpu_present_cpu_mask(void)
-{
-	size_t size;
-	DIR *dir;
-	int i;
 
-	size = alloc_cpu_set(&present_cpumask);
-	present_cpumask_size = size;
-	for (i = 0; i < topo_max_cpus; ++i) {
-		char buffer[256];
-
-		snprintf(buffer, sizeof(buffer),
-			 "/sys/devices/system/cpu/cpu%d", i);
-		dir = opendir(buffer);
-		if (dir) {
-			int pkg_id, die_id;
-
-			CPU_SET_S(i, size, present_cpumask);
-			die_id = get_physical_die_id(i);
-			if (die_id < 0)
-				die_id = 0;
-
-			pkg_id = get_physical_package_id(i);
-			if (pkg_id < 0) {
-				fprintf(stderr, "Failed to get package id, CPU %d may be offline\n", i);
-				continue;
-			}
-			if (pkg_id < MAX_PACKAGE_COUNT &&
-			    die_id < MAX_DIE_PER_PACKAGE) {
-				int core_id = get_physical_core_id(i);
-
-				cpu_cnt[pkg_id][die_id]++;
-				core_mask[pkg_id][die_id] |= (1ULL << core_id);
-			}
-		}
-		closedir(dir);
-	}
-}
-
-int get_max_punit_core_id(int pkg_id, int die_id)
+int get_max_punit_core_id(struct isst_id *id)
 {
 	int max_id = 0;
 	int i;
@@ -586,60 +595,74 @@ int get_max_punit_core_id(int pkg_id, int die_id)
 		if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask))
 			continue;
 
-		if (cpu_map[i].pkg_id == pkg_id &&
-			cpu_map[i].die_id == die_id &&
-			cpu_map[i].punit_cpu_core > max_id)
+		if (is_cpu_in_power_domain(i, id) &&
+		    cpu_map[i].punit_cpu_core > max_id)
 			max_id = cpu_map[i].punit_cpu_core;
 	}
 
 	return max_id;
 }
 
-int get_cpu_count(int pkg_id, int die_id)
+int get_cpu_count(struct isst_id *id)
 {
-	if (pkg_id < MAX_PACKAGE_COUNT && die_id < MAX_DIE_PER_PACKAGE)
-		return cpu_cnt[pkg_id][die_id];
-
-	return 0;
-}
-
-static void set_cpu_target_cpu_mask(void)
-{
-	size_t size;
-	int i;
-
-	size = alloc_cpu_set(&target_cpumask);
-	target_cpumask_size = size;
-	for (i = 0; i < max_target_cpus; ++i) {
-		if (!CPU_ISSET_S(target_cpus[i], present_cpumask_size,
-				 present_cpumask))
-			continue;
+	if (id->pkg < 0 || id->die < 0)
+		return 0;
 
-		CPU_SET_S(target_cpus[i], size, target_cpumask);
-	}
+	return cpu_cnt[id->pkg][id->die];
 }
 
 static void create_cpu_map(void)
 {
 	const char *pathname = "/dev/isst_interface";
+	size_t size;
+	DIR *dir;
 	int i, fd = 0;
 	struct isst_if_cpu_maps map;
 
-	cpu_map = malloc(sizeof(*cpu_map) * topo_max_cpus);
+	/* Use calloc to make sure the memory is initialized to Zero */
+	cpu_map = calloc(topo_max_cpus, sizeof(*cpu_map));
 	if (!cpu_map)
 		err(3, "cpumap");
 
 	fd = open(pathname, O_RDWR);
-	if (fd < 0)
+	if (fd < 0 && !is_clx_n_platform())
 		err(-1, "%s open failed", pathname);
 
+	size = alloc_cpu_set(&present_cpumask);
+	present_cpumask_size = size;
+
 	for (i = 0; i < topo_max_cpus; ++i) {
-		if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask))
+		char buffer[256];
+		int pkg_id, die_id, core_id;
+
+		/* check if CPU is online */
+		snprintf(buffer, sizeof(buffer),
+			 "/sys/devices/system/cpu/cpu%d", i);
+		dir = opendir(buffer);
+		if (!dir)
+			continue;
+		closedir(dir);
+
+		CPU_SET_S(i, size, present_cpumask);
+
+		pkg_id = get_physical_package_id(i);
+		die_id = get_physical_die_id(i);
+		core_id = get_physical_core_id(i);
+
+		if (pkg_id < 0 || die_id < 0 || core_id < 0)
 			continue;
 
+		cpu_map[i].pkg_id = pkg_id;
+		cpu_map[i].die_id = die_id;
+		cpu_map[i].core_id = core_id;
+		cpu_map[i].initialized = 1;
+
+		cpu_cnt[pkg_id][die_id]++;
+
+		if (fd < 0)
+			continue;
 		map.cmd_count = 1;
 		map.cpu_map[0].logical_cpu = i;
-
 		debug_printf(" map logical_cpu:%d\n",
 			     map.cpu_map[0].logical_cpu);
 		if (ioctl(fd, ISST_IF_GET_PHY_ID, &map) == -1) {
@@ -648,9 +671,6 @@ static void create_cpu_map(void)
 				map.cpu_map[0].logical_cpu);
 			continue;
 		}
-		cpu_map[i].core_id = get_physical_core_id(i);
-		cpu_map[i].pkg_id = get_physical_package_id(i);
-		cpu_map[i].die_id = get_physical_die_id(i);
 		cpu_map[i].punit_cpu = map.cpu_map[0].physical_cpu;
 		cpu_map[i].punit_cpu_core = (map.cpu_map[0].physical_cpu >>
 					     1); // shift to get core id
@@ -661,35 +681,27 @@ static void create_cpu_map(void)
 			cpu_map[i].pkg_id, cpu_map[i].punit_cpu,
 			cpu_map[i].punit_cpu_core);
 	}
-
-	if (fd)
+	if (fd >= 0)
 		close(fd);
-}
 
-int find_logical_cpu(int pkg_id, int die_id, int punit_core_id)
-{
-	int i;
+	size = alloc_cpu_set(&target_cpumask);
+	target_cpumask_size = size;
+	for (i = 0; i < max_target_cpus; ++i) {
+		if (!CPU_ISSET_S(target_cpus[i], present_cpumask_size,
+				 present_cpumask))
+			continue;
 
-	for (i = 0; i < topo_max_cpus; ++i) {
-		if (cpu_map[i].pkg_id == pkg_id &&
-		    cpu_map[i].die_id == die_id &&
-		    cpu_map[i].punit_cpu_core == punit_core_id)
-			return i;
+		CPU_SET_S(target_cpus[i], size, target_cpumask);
 	}
-
-	return -EINVAL;
 }
 
-void set_cpu_mask_from_punit_coremask(int cpu, unsigned long long core_mask,
+void set_cpu_mask_from_punit_coremask(struct isst_id *id, unsigned long long core_mask,
 				      size_t core_cpumask_size,
 				      cpu_set_t *core_cpumask, int *cpu_cnt)
 {
 	int i, cnt = 0;
-	int die_id, pkg_id;
 
 	*cpu_cnt = 0;
-	die_id = get_physical_die_id(cpu);
-	pkg_id = get_physical_package_id(cpu);
 
 	for (i = 0; i < 64; ++i) {
 		if (core_mask & BIT_ULL(i)) {
@@ -699,8 +711,7 @@ void set_cpu_mask_from_punit_coremask(int cpu, unsigned long long core_mask,
 				if (!CPU_ISSET_S(j, present_cpumask_size, present_cpumask))
 					continue;
 
-				if (cpu_map[j].pkg_id == pkg_id &&
-				    cpu_map[j].die_id == die_id &&
+				if (is_cpu_in_power_domain(j, id) &&
 				    cpu_map[j].punit_cpu_core == i) {
 					CPU_SET_S(j, core_cpumask_size,
 						  core_cpumask);
@@ -931,6 +942,7 @@ static void isst_print_extended_platform_info(void)
 	struct isst_pkg_ctdp pkg_dev;
 	int ret, i, j;
 	FILE *filep;
+	struct isst_id id;
 
 	for (i = 0; i < 256; ++i) {
 		char path[256];
@@ -947,7 +959,8 @@ static void isst_print_extended_platform_info(void)
 
 	fclose(filep);
 
-	ret = isst_get_ctdp_levels(i, &pkg_dev);
+	set_isst_id(&id, i);
+	ret = isst_get_ctdp_levels(&id, &pkg_dev);
 	if (ret)
 		return;
 
@@ -964,7 +977,7 @@ static void isst_print_extended_platform_info(void)
 		fprintf(outf, "TDP level change control is unlocked, max level: %d \n", pkg_dev.levels);
 
 	for (j = 0; j <= pkg_dev.levels; ++j) {
-		ret = isst_get_ctdp_control(i, j, &ctdp_level);
+		ret = isst_get_ctdp_control(&id, j, &ctdp_level);
 		if (ret)
 			continue;
 
@@ -985,7 +998,7 @@ static void isst_print_extended_platform_info(void)
 	else
 		fprintf(outf, "Intel(R) SST-BF (feature base-freq) is not supported\n");
 
-	ret = isst_read_pm_config(i, &cp_state, &cp_cap);
+	ret = isst_read_pm_config(&id, &cp_state, &cp_cap);
 	if (ret) {
 		fprintf(outf, "Intel(R) SST-CP (feature core-power) status is unknown\n");
 		return;
@@ -1007,6 +1020,10 @@ static void isst_print_platform_information(void)
 		exit(0);
 	}
 
+	/* Early initialization to create working cpu_map */
+	set_max_cpu_num();
+	create_cpu_map();
+
 	fd = open(pathname, O_RDWR);
 	if (fd < 0)
 		err(-1, "%s open failed", pathname);
@@ -1031,18 +1048,18 @@ static void isst_print_platform_information(void)
 }
 
 static char *local_str0, *local_str1;
-static void exec_on_get_ctdp_cpu(int cpu, void *arg1, void *arg2, void *arg3,
+static void exec_on_get_ctdp_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 				 void *arg4)
 {
-	int (*fn_ptr)(int cpu, void *arg);
+	int (*fn_ptr)(struct isst_id *id, void *arg);
 	int ret;
 
 	fn_ptr = arg1;
-	ret = fn_ptr(cpu, arg2);
+	ret = fn_ptr(id, arg2);
 	if (ret)
 		isst_display_error_info_message(1, "get_tdp_* failed", 0, 0);
 	else
-		isst_ctdp_display_core_info(cpu, outf, arg3,
+		isst_ctdp_display_core_info(id, outf, arg3,
 					    *(unsigned int *)arg4,
 					    local_str0, local_str1);
 }
@@ -1110,9 +1127,9 @@ static int clx_n_get_base_ratio(void)
 	return (int)(value);
 }
 
-static int clx_n_config(int cpu)
+static int clx_n_config(struct isst_id *id)
 {
-	int i, ret, pkg_id, die_id;
+	int i, ret;
 	unsigned long cpu_bf;
 	struct isst_pkg_ctdp_level_info *ctdp_level;
 	struct isst_pbf_info *pbf_info;
@@ -1134,15 +1151,11 @@ static int clx_n_config(int cpu)
 	pbf_info->p1_high = 0;
 	pbf_info->p1_low = ~0;
 
-	pkg_id = get_physical_package_id(cpu);
-	die_id = get_physical_die_id(cpu);
-
 	for (i = 0; i < topo_max_cpus; i++) {
 		if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask))
 			continue;
 
-		if (pkg_id != get_physical_package_id(i) ||
-		    die_id != get_physical_die_id(i))
+		if (!is_cpu_in_power_domain(i, id))
 			continue;
 
 		CPU_SET_S(i, ctdp_level->core_cpumask_size,
@@ -1179,8 +1192,7 @@ static int clx_n_config(int cpu)
 		if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask))
 			continue;
 
-		if (pkg_id != get_physical_package_id(i) ||
-		    die_id != get_physical_die_id(i))
+		if (!is_cpu_in_power_domain(i, id))
 			continue;
 
 		cpu_bf = parse_int_file(1,
@@ -1206,7 +1218,7 @@ error_ret:
 	return ret;
 }
 
-static void dump_clx_n_config_for_cpu(int cpu, void *arg1, void *arg2,
+static void dump_clx_n_config_for_cpu(struct isst_id *id, void *arg1, void *arg2,
 				   void *arg3, void *arg4)
 {
 	int ret;
@@ -1216,7 +1228,7 @@ static void dump_clx_n_config_for_cpu(int cpu, void *arg1, void *arg2,
 		exit(0);
 	}
 
-	ret = clx_n_config(cpu);
+	ret = clx_n_config(id);
 	if (ret) {
 		debug_printf("clx_n_config failed");
 	} else {
@@ -1226,27 +1238,27 @@ static void dump_clx_n_config_for_cpu(int cpu, void *arg1, void *arg2,
 		ctdp_level = &clx_n_pkg_dev.ctdp_level[0];
 		pbf_info = &ctdp_level->pbf_info;
 		clx_n_pkg_dev.processed = 1;
-		isst_ctdp_display_information(cpu, outf, tdp_level, &clx_n_pkg_dev);
+		isst_ctdp_display_information(id, outf, tdp_level, &clx_n_pkg_dev);
 		free_cpu_set(ctdp_level->core_cpumask);
 		free_cpu_set(pbf_info->core_cpumask);
 	}
 }
 
-static void dump_isst_config_for_cpu(int cpu, void *arg1, void *arg2,
+static void dump_isst_config_for_cpu(struct isst_id *id, void *arg1, void *arg2,
 				     void *arg3, void *arg4)
 {
 	struct isst_pkg_ctdp pkg_dev;
 	int ret;
 
 	memset(&pkg_dev, 0, sizeof(pkg_dev));
-	ret = isst_get_process_ctdp(cpu, tdp_level, &pkg_dev);
+	ret = isst_get_process_ctdp(id, tdp_level, &pkg_dev);
 	if (ret) {
-		isst_display_error_info_message(1, "Failed to get perf-profile info on cpu", 1, cpu);
+		isst_display_error_info_message(1, "Failed to get perf-profile info on cpu", 1, id->cpu);
 		isst_ctdp_display_information_end(outf);
 		exit(1);
 	} else {
-		isst_ctdp_display_information(cpu, outf, tdp_level, &pkg_dev);
-		isst_get_process_ctdp_complete(cpu, &pkg_dev);
+		isst_ctdp_display_information(id, outf, tdp_level, &pkg_dev);
+		isst_get_process_ctdp_complete(id, &pkg_dev);
 	}
 }
 
@@ -1282,23 +1294,21 @@ static void dump_isst_config(int arg)
 
 static void adjust_scaling_max_from_base_freq(int cpu);
 
-static void set_tdp_level_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
+static void set_tdp_level_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 				  void *arg4)
 {
 	int ret;
 
-	ret = isst_set_tdp_level(cpu, tdp_level);
+	ret = isst_set_tdp_level(id, tdp_level);
 	if (ret) {
 		isst_display_error_info_message(1, "Set TDP level failed", 0, 0);
 		isst_ctdp_display_information_end(outf);
 		exit(1);
 	} else {
-		isst_display_result(cpu, outf, "perf-profile", "set_tdp_level",
+		isst_display_result(id, outf, "perf-profile", "set_tdp_level",
 				    ret);
 		if (force_online_offline) {
 			struct isst_pkg_ctdp_level_info ctdp_level;
-			int pkg_id = get_physical_package_id(cpu);
-			int die_id = get_physical_die_id(cpu);
 
 			/* Wait for updated base frequencies */
 			usleep(2000);
@@ -1306,7 +1316,7 @@ static void set_tdp_level_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
 			fprintf(stderr, "Option is set to online/offline\n");
 			ctdp_level.core_cpumask_size =
 				alloc_cpu_set(&ctdp_level.core_cpumask);
-			ret = isst_get_coremask_info(cpu, tdp_level, &ctdp_level);
+			ret = isst_get_coremask_info(id, tdp_level, &ctdp_level);
 			if (ret) {
 				isst_display_error_info_message(1, "Can't get coremask, online/offline option is ignored", 0, 0);
 				return;
@@ -1314,7 +1324,7 @@ static void set_tdp_level_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
 			if (ctdp_level.cpu_count) {
 				int i, max_cpus = get_topo_max_cpus();
 				for (i = 0; i < max_cpus; ++i) {
-					if (pkg_id != get_physical_package_id(i) || die_id != get_physical_die_id(i))
+					if (!is_cpu_in_power_domain(i, id))
 						continue;
 					if (CPU_ISSET_S(i, ctdp_level.core_cpumask_size, ctdp_level.core_cpumask)) {
 						fprintf(stderr, "online cpu %d\n", i);
@@ -1357,12 +1367,12 @@ static void set_tdp_level(int arg)
 	isst_ctdp_display_information_end(outf);
 }
 
-static void clx_n_dump_pbf_config_for_cpu(int cpu, void *arg1, void *arg2,
+static void clx_n_dump_pbf_config_for_cpu(struct isst_id *id, void *arg1, void *arg2,
 				       void *arg3, void *arg4)
 {
 	int ret;
 
-	ret = clx_n_config(cpu);
+	ret = clx_n_config(id);
 	if (ret) {
 		isst_display_error_info_message(1, "clx_n_config failed", 0, 0);
 	} else {
@@ -1371,25 +1381,25 @@ static void clx_n_dump_pbf_config_for_cpu(int cpu, void *arg1, void *arg2,
 
 		ctdp_level = &clx_n_pkg_dev.ctdp_level[0];
 		pbf_info = &ctdp_level->pbf_info;
-		isst_pbf_display_information(cpu, outf, tdp_level, pbf_info);
+		isst_pbf_display_information(id, outf, tdp_level, pbf_info);
 		free_cpu_set(ctdp_level->core_cpumask);
 		free_cpu_set(pbf_info->core_cpumask);
 	}
 }
 
-static void dump_pbf_config_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
+static void dump_pbf_config_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 				    void *arg4)
 {
 	struct isst_pbf_info pbf_info;
 	int ret;
 
-	ret = isst_get_pbf_info(cpu, tdp_level, &pbf_info);
+	ret = isst_get_pbf_info(id, tdp_level, &pbf_info);
 	if (ret) {
 		isst_display_error_info_message(1, "Failed to get base-freq info at this level", 1, tdp_level);
 		isst_ctdp_display_information_end(outf);
 		exit(1);
 	} else {
-		isst_pbf_display_information(cpu, outf, tdp_level, &pbf_info);
+		isst_pbf_display_information(id, outf, tdp_level, &pbf_info);
 		isst_get_pbf_info_complete(&pbf_info);
 	}
 }
@@ -1426,12 +1436,12 @@ static void dump_pbf_config(int arg)
 	isst_ctdp_display_information_end(outf);
 }
 
-static int set_clos_param(int cpu, int clos, int epp, int wt, int min, int max)
+static int set_clos_param(struct isst_id *id, int clos, int epp, int wt, int min, int max)
 {
 	struct isst_clos_config clos_config;
 	int ret;
 
-	ret = isst_pm_get_clos(cpu, clos, &clos_config);
+	ret = isst_pm_get_clos(id, clos, &clos_config);
 	if (ret) {
 		isst_display_error_info_message(1, "isst_pm_get_clos failed", 0, 0);
 		return ret;
@@ -1440,7 +1450,7 @@ static int set_clos_param(int cpu, int clos, int epp, int wt, int min, int max)
 	clos_config.clos_max = max;
 	clos_config.epp = epp;
 	clos_config.clos_prop_prio = wt;
-	ret = isst_set_clos(cpu, clos, &clos_config);
+	ret = isst_set_clos(id, clos, &clos_config);
 	if (ret) {
 		isst_display_error_info_message(1, "isst_set_clos failed", 0, 0);
 		return ret;
@@ -1502,14 +1512,14 @@ static void adjust_scaling_min_from_base_freq(int cpu)
 		set_cpufreq_scaling_min_max(cpu, 0, base_freq);
 }
 
-static int set_clx_pbf_cpufreq_scaling_min_max(int cpu)
+static int set_clx_pbf_cpufreq_scaling_min_max(struct isst_id *id)
 {
 	struct isst_pkg_ctdp_level_info *ctdp_level;
 	struct isst_pbf_info *pbf_info;
-	int i, pkg_id, die_id, freq, freq_high, freq_low;
+	int i, freq, freq_high, freq_low;
 	int ret;
 
-	ret = clx_n_config(cpu);
+	ret = clx_n_config(id);
 	if (ret) {
 		debug_printf("cpufreq_scaling_min_max failed for CLX");
 		return ret;
@@ -1520,11 +1530,8 @@ static int set_clx_pbf_cpufreq_scaling_min_max(int cpu)
 	freq_high = pbf_info->p1_high * 100000;
 	freq_low = pbf_info->p1_low * 100000;
 
-	pkg_id = get_physical_package_id(cpu);
-	die_id = get_physical_die_id(cpu);
 	for (i = 0; i < get_topo_max_cpus(); ++i) {
-		if (pkg_id != get_physical_package_id(i) ||
-		    die_id != get_physical_die_id(i))
+		if (!is_cpu_in_power_domain(i, id))
 			continue;
 
 		if (CPU_ISSET_S(i, pbf_info->core_cpumask_size,
@@ -1587,15 +1594,12 @@ static int set_cpufreq_scaling_min_max_from_cpuinfo(int cpu, int cpuinfo_max, in
 	return 0;
 }
 
-static void set_scaling_min_to_cpuinfo_max(int cpu)
+static void set_scaling_min_to_cpuinfo_max(struct isst_id *id)
 {
-	int i, pkg_id, die_id;
+	int i;
 
-	pkg_id = get_physical_package_id(cpu);
-	die_id = get_physical_die_id(cpu);
 	for (i = 0; i < get_topo_max_cpus(); ++i) {
-		if (pkg_id != get_physical_package_id(i) ||
-		    die_id != get_physical_die_id(i))
+		if (!is_cpu_in_power_domain(i, id))
 			continue;
 
 		adjust_scaling_max_from_base_freq(i);
@@ -1604,15 +1608,12 @@ static void set_scaling_min_to_cpuinfo_max(int cpu)
 	}
 }
 
-static void set_scaling_min_to_cpuinfo_min(int cpu)
+static void set_scaling_min_to_cpuinfo_min(struct isst_id *id)
 {
-	int i, pkg_id, die_id;
+	int i;
 
-	pkg_id = get_physical_package_id(cpu);
-	die_id = get_physical_die_id(cpu);
 	for (i = 0; i < get_topo_max_cpus(); ++i) {
-		if (pkg_id != get_physical_package_id(i) ||
-		    die_id != get_physical_die_id(i))
+		if (!is_cpu_in_power_domain(i, id))
 			continue;
 
 		adjust_scaling_max_from_base_freq(i);
@@ -1620,53 +1621,48 @@ static void set_scaling_min_to_cpuinfo_min(int cpu)
 	}
 }
 
-static void set_scaling_max_to_cpuinfo_max(int cpu)
+static void set_scaling_max_to_cpuinfo_max(struct isst_id *id)
 {
-	int i, pkg_id, die_id;
+	int i;
 
-	pkg_id = get_physical_package_id(cpu);
-	die_id = get_physical_die_id(cpu);
 	for (i = 0; i < get_topo_max_cpus(); ++i) {
-		if (pkg_id != get_physical_package_id(i) ||
-		    die_id != get_physical_die_id(i))
+		if (!is_cpu_in_power_domain(i, id))
 			continue;
 
 		set_cpufreq_scaling_min_max_from_cpuinfo(i, 1, 1);
 	}
 }
 
-static int set_core_priority_and_min(int cpu, int mask_size,
+static int set_core_priority_and_min(struct isst_id *id, int mask_size,
 				     cpu_set_t *cpu_mask, int min_high,
 				     int min_low)
 {
-	int pkg_id, die_id, ret, i;
+	int ret, i;
 
 	if (!CPU_COUNT_S(mask_size, cpu_mask))
 		return -1;
 
-	ret = set_clos_param(cpu, 0, 0, 0, min_high, 0xff);
+	ret = set_clos_param(id, 0, 0, 0, min_high, 0xff);
 	if (ret)
 		return ret;
 
-	ret = set_clos_param(cpu, 1, 15, 15, min_low, 0xff);
+	ret = set_clos_param(id, 1, 15, 15, min_low, 0xff);
 	if (ret)
 		return ret;
 
-	ret = set_clos_param(cpu, 2, 15, 15, min_low, 0xff);
+	ret = set_clos_param(id, 2, 15, 15, min_low, 0xff);
 	if (ret)
 		return ret;
 
-	ret = set_clos_param(cpu, 3, 15, 15, min_low, 0xff);
+	ret = set_clos_param(id, 3, 15, 15, min_low, 0xff);
 	if (ret)
 		return ret;
 
-	pkg_id = get_physical_package_id(cpu);
-	die_id = get_physical_die_id(cpu);
 	for (i = 0; i < get_topo_max_cpus(); ++i) {
 		int clos;
+		struct isst_id tid;
 
-		if (pkg_id != get_physical_package_id(i) ||
-		    die_id != get_physical_die_id(i))
+		if (!is_cpu_in_power_domain(i, id))
 			continue;
 
 		if (CPU_ISSET_S(i, mask_size, cpu_mask))
@@ -1675,7 +1671,8 @@ static int set_core_priority_and_min(int cpu, int mask_size,
 			clos = 3;
 
 		debug_printf("Associate cpu: %d clos: %d\n", i, clos);
-		ret = isst_clos_associate(i, clos);
+		set_isst_id(&tid, i);
+		ret = isst_clos_associate(&tid, clos);
 		if (ret) {
 			isst_display_error_info_message(1, "isst_clos_associate failed", 0, 0);
 			return ret;
@@ -1685,20 +1682,20 @@ static int set_core_priority_and_min(int cpu, int mask_size,
 	return 0;
 }
 
-static int set_pbf_core_power(int cpu)
+static int set_pbf_core_power(struct isst_id *id)
 {
 	struct isst_pbf_info pbf_info;
 	struct isst_pkg_ctdp pkg_dev;
 	int ret;
 
-	ret = isst_get_ctdp_levels(cpu, &pkg_dev);
+	ret = isst_get_ctdp_levels(id, &pkg_dev);
 	if (ret) {
 		debug_printf("isst_get_ctdp_levels failed");
 		return ret;
 	}
 	debug_printf("Current_level: %d\n", pkg_dev.current_level);
 
-	ret = isst_get_pbf_info(cpu, pkg_dev.current_level, &pbf_info);
+	ret = isst_get_pbf_info(id, pkg_dev.current_level, &pbf_info);
 	if (ret) {
 		debug_printf("isst_get_pbf_info failed");
 		return ret;
@@ -1706,7 +1703,7 @@ static int set_pbf_core_power(int cpu)
 	debug_printf("p1_high: %d p1_low: %d\n", pbf_info.p1_high,
 		     pbf_info.p1_low);
 
-	ret = set_core_priority_and_min(cpu, pbf_info.core_cpumask_size,
+	ret = set_core_priority_and_min(id, pbf_info.core_cpumask_size,
 					pbf_info.core_cpumask,
 					pbf_info.p1_high, pbf_info.p1_low);
 	if (ret) {
@@ -1714,7 +1711,7 @@ static int set_pbf_core_power(int cpu)
 		return ret;
 	}
 
-	ret = isst_pm_qos_config(cpu, 1, 1);
+	ret = isst_pm_qos_config(id, 1, 1);
 	if (ret) {
 		debug_printf("isst_pm_qos_config failed");
 		return ret;
@@ -1723,7 +1720,7 @@ static int set_pbf_core_power(int cpu)
 	return 0;
 }
 
-static void set_pbf_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
+static void set_pbf_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 			    void *arg4)
 {
 	struct isst_pkg_ctdp_level_info ctdp_level;
@@ -1734,22 +1731,22 @@ static void set_pbf_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
 	if (is_clx_n_platform()) {
 		ret = 0;
 		if (status) {
-			set_clx_pbf_cpufreq_scaling_min_max(cpu);
+			set_clx_pbf_cpufreq_scaling_min_max(id);
 
 		} else {
-			set_scaling_max_to_cpuinfo_max(cpu);
-			set_scaling_min_to_cpuinfo_min(cpu);
+			set_scaling_max_to_cpuinfo_max(id);
+			set_scaling_min_to_cpuinfo_min(id);
 		}
 		goto disp_result;
 	}
 
-	ret = isst_get_ctdp_levels(cpu, &pkg_dev);
+	ret = isst_get_ctdp_levels(id, &pkg_dev);
 	if (ret) {
 		isst_display_error_info_message(1, "Failed to get number of levels", 0, 0);
 		goto disp_result;
 	}
 
-	ret = isst_get_ctdp_control(cpu, pkg_dev.current_level, &ctdp_level);
+	ret = isst_get_ctdp_control(id, pkg_dev.current_level, &ctdp_level);
 	if (ret) {
 		isst_display_error_info_message(1, "Failed to get current level", 0, 0);
 		goto disp_result;
@@ -1762,34 +1759,34 @@ static void set_pbf_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
 	}
 
 	if (auto_mode && status) {
-		ret = set_pbf_core_power(cpu);
+		ret = set_pbf_core_power(id);
 		if (ret)
 			goto disp_result;
 	}
 
-	ret = isst_set_pbf_fact_status(cpu, 1, status);
+	ret = isst_set_pbf_fact_status(id, 1, status);
 	if (ret) {
 		debug_printf("isst_set_pbf_fact_status failed");
 		if (auto_mode)
-			isst_pm_qos_config(cpu, 0, 0);
+			isst_pm_qos_config(id, 0, 0);
 	} else {
 		if (auto_mode) {
 			if (status)
-				set_scaling_min_to_cpuinfo_max(cpu);
+				set_scaling_min_to_cpuinfo_max(id);
 			else
-				set_scaling_min_to_cpuinfo_min(cpu);
+				set_scaling_min_to_cpuinfo_min(id);
 		}
 	}
 
 	if (auto_mode && !status)
-		isst_pm_qos_config(cpu, 0, 1);
+		isst_pm_qos_config(id, 0, 1);
 
 disp_result:
 	if (status)
-		isst_display_result(cpu, outf, "base-freq", "enable",
+		isst_display_result(id, outf, "base-freq", "enable",
 				    ret);
 	else
-		isst_display_result(cpu, outf, "base-freq", "disable",
+		isst_display_result(id, outf, "base-freq", "disable",
 				    ret);
 }
 
@@ -1838,19 +1835,19 @@ static void set_pbf_enable(int arg)
 	isst_ctdp_display_information_end(outf);
 }
 
-static void dump_fact_config_for_cpu(int cpu, void *arg1, void *arg2,
+static void dump_fact_config_for_cpu(struct isst_id *id, void *arg1, void *arg2,
 				     void *arg3, void *arg4)
 {
 	struct isst_fact_info fact_info;
 	int ret;
 
-	ret = isst_get_fact_info(cpu, tdp_level, fact_bucket, &fact_info);
+	ret = isst_get_fact_info(id, tdp_level, fact_bucket, &fact_info);
 	if (ret) {
 		isst_display_error_info_message(1, "Failed to get turbo-freq info at this level", 1, tdp_level);
 		isst_ctdp_display_information_end(outf);
 		exit(1);
 	} else {
-		isst_fact_display_information(cpu, outf, tdp_level, fact_bucket,
+		isst_fact_display_information(id, outf, tdp_level, fact_bucket,
 					      fact_avx, &fact_info);
 	}
 }
@@ -1884,7 +1881,7 @@ static void dump_fact_config(int arg)
 	isst_ctdp_display_information_end(outf);
 }
 
-static void set_fact_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
+static void set_fact_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 			     void *arg4)
 {
 	struct isst_pkg_ctdp_level_info ctdp_level;
@@ -1898,13 +1895,13 @@ static void set_fact_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
 		goto disp_results;
 	}
 
-	ret = isst_get_ctdp_levels(cpu, &pkg_dev);
+	ret = isst_get_ctdp_levels(id, &pkg_dev);
 	if (ret) {
 		isst_display_error_info_message(1, "Failed to get number of levels", 0, 0);
 		goto disp_results;
 	}
 
-	ret = isst_get_ctdp_control(cpu, pkg_dev.current_level, &ctdp_level);
+	ret = isst_get_ctdp_control(id, pkg_dev.current_level, &ctdp_level);
 	if (ret) {
 		isst_display_error_info_message(1, "Failed to get current level", 0, 0);
 		goto disp_results;
@@ -1917,16 +1914,16 @@ static void set_fact_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
 	}
 
 	if (status) {
-		ret = isst_pm_qos_config(cpu, 1, 1);
+		ret = isst_pm_qos_config(id, 1, 1);
 		if (ret)
 			goto disp_results;
 	}
 
-	ret = isst_set_pbf_fact_status(cpu, 0, status);
+	ret = isst_set_pbf_fact_status(id, 0, status);
 	if (ret) {
 		debug_printf("isst_set_pbf_fact_status failed");
 		if (auto_mode)
-			isst_pm_qos_config(cpu, 0, 0);
+			isst_pm_qos_config(id, 0, 0);
 
 		goto disp_results;
 	}
@@ -1935,31 +1932,32 @@ static void set_fact_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
 	if (status) {
 		struct isst_pkg_ctdp pkg_dev;
 
-		ret = isst_get_ctdp_levels(cpu, &pkg_dev);
+		ret = isst_get_ctdp_levels(id, &pkg_dev);
 		if (!ret)
-			ret = isst_set_trl(cpu, fact_trl);
+			ret = isst_set_trl(id, fact_trl);
 		if (ret && auto_mode)
-			isst_pm_qos_config(cpu, 0, 0);
+			isst_pm_qos_config(id, 0, 0);
 	} else {
 		if (auto_mode)
-			isst_pm_qos_config(cpu, 0, 0);
+			isst_pm_qos_config(id, 0, 0);
 	}
 
 disp_results:
 	if (status) {
-		isst_display_result(cpu, outf, "turbo-freq", "enable", ret);
+		isst_display_result(id, outf, "turbo-freq", "enable", ret);
 		if (ret)
 			fact_enable_fail = ret;
 	} else {
 		/* Since we modified TRL during Fact enable, restore it */
-		isst_set_trl_from_current_tdp(cpu, fact_trl);
-		isst_display_result(cpu, outf, "turbo-freq", "disable", ret);
+		isst_set_trl_from_current_tdp(id, fact_trl);
+		isst_display_result(id, outf, "turbo-freq", "disable", ret);
 	}
 }
 
 static void set_fact_enable(int arg)
 {
 	int i, ret, enable = arg;
+	struct isst_id id;
 
 	if (cmd_help) {
 		if (enable) {
@@ -2033,19 +2031,20 @@ static void set_fact_enable(int arg)
 			if (!CPU_ISSET_S(i, present_cpumask_size, present_cpumask))
 				continue;
 
-			ret = set_clos_param(i, 0, 0, 0, 0, 0xff);
+			set_isst_id(&id, i);
+			ret = set_clos_param(&id, 0, 0, 0, 0, 0xff);
 			if (ret)
 				goto error_disp;
 
-			ret = set_clos_param(i, 1, 15, 15, 0, 0xff);
+			ret = set_clos_param(&id, 1, 15, 15, 0, 0xff);
 			if (ret)
 				goto error_disp;
 
-			ret = set_clos_param(i, 2, 15, 15, 0, 0xff);
+			ret = set_clos_param(&id, 2, 15, 15, 0, 0xff);
 			if (ret)
 				goto error_disp;
 
-			ret = set_clos_param(i, 3, 15, 15, 0, 0xff);
+			ret = set_clos_param(&id, 3, 15, 15, 0, 0xff);
 			if (ret)
 				goto error_disp;
 
@@ -2055,21 +2054,22 @@ static void set_fact_enable(int arg)
 				clos = 3;
 
 			debug_printf("Associate cpu: %d clos: %d\n", i, clos);
-			ret = isst_clos_associate(i, clos);
+			ret = isst_clos_associate(&id, clos);
 			if (ret)
 				goto error_disp;
 		}
-		isst_display_result(-1, outf, "turbo-freq --auto", "enable", 0);
+		set_isst_id(&id, -1);
+		isst_display_result(&id, outf, "turbo-freq --auto", "enable", 0);
 	}
 
 	return;
 
 error_disp:
-	isst_display_result(i, outf, "turbo-freq --auto", "enable", ret);
+	isst_display_result(&id, outf, "turbo-freq --auto", "enable", ret);
 
 }
 
-static void enable_clos_qos_config(int cpu, void *arg1, void *arg2, void *arg3,
+static void enable_clos_qos_config(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 				   void *arg4)
 {
 	int ret;
@@ -2078,15 +2078,15 @@ static void enable_clos_qos_config(int cpu, void *arg1, void *arg2, void *arg3,
 	if (is_skx_based_platform())
 		clos_priority_type = 1;
 
-	ret = isst_pm_qos_config(cpu, status, clos_priority_type);
+	ret = isst_pm_qos_config(id, status, clos_priority_type);
 	if (ret)
 		isst_display_error_info_message(1, "isst_pm_qos_config failed", 0, 0);
 
 	if (status)
-		isst_display_result(cpu, outf, "core-power", "enable",
+		isst_display_result(id, outf, "core-power", "enable",
 				    ret);
 	else
-		isst_display_result(cpu, outf, "core-power", "disable",
+		isst_display_result(id, outf, "core-power", "disable",
 				    ret);
 }
 
@@ -2125,17 +2125,17 @@ static void set_clos_enable(int arg)
 	isst_ctdp_display_information_end(outf);
 }
 
-static void dump_clos_config_for_cpu(int cpu, void *arg1, void *arg2,
+static void dump_clos_config_for_cpu(struct isst_id *id, void *arg1, void *arg2,
 				     void *arg3, void *arg4)
 {
 	struct isst_clos_config clos_config;
 	int ret;
 
-	ret = isst_pm_get_clos(cpu, current_clos, &clos_config);
+	ret = isst_pm_get_clos(id, current_clos, &clos_config);
 	if (ret)
 		isst_display_error_info_message(1, "isst_pm_get_clos failed", 0, 0);
 	else
-		isst_clos_display_information(cpu, outf, current_clos,
+		isst_clos_display_information(id, outf, current_clos,
 					      &clos_config);
 }
 
@@ -2164,19 +2164,19 @@ static void dump_clos_config(int arg)
 	isst_ctdp_display_information_end(outf);
 }
 
-static void get_clos_info_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
+static void get_clos_info_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 				  void *arg4)
 {
 	int enable, ret, prio_type;
 
-	ret = isst_clos_get_clos_information(cpu, &enable, &prio_type);
+	ret = isst_clos_get_clos_information(id, &enable, &prio_type);
 	if (ret)
 		isst_display_error_info_message(1, "isst_clos_get_info failed", 0, 0);
 	else {
 		int cp_state, cp_cap;
 
-		isst_read_pm_config(cpu, &cp_state, &cp_cap);
-		isst_clos_display_clos_information(cpu, outf, enable, prio_type,
+		isst_read_pm_config(id, &cp_state, &cp_cap);
+		isst_clos_display_clos_information(id, outf, enable, prio_type,
 						   cp_state, cp_cap);
 	}
 }
@@ -2201,25 +2201,22 @@ static void dump_clos_info(int arg)
 
 }
 
-static void set_clos_config_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
+static void set_clos_config_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 				    void *arg4)
 {
 	struct isst_clos_config clos_config;
 	int ret;
 
-	clos_config.pkg_id = get_physical_package_id(cpu);
-	clos_config.die_id = get_physical_die_id(cpu);
-
 	clos_config.epp = clos_epp;
 	clos_config.clos_prop_prio = clos_prop_prio;
 	clos_config.clos_min = clos_min;
 	clos_config.clos_max = clos_max;
 	clos_config.clos_desired = clos_desired;
-	ret = isst_set_clos(cpu, current_clos, &clos_config);
+	ret = isst_set_clos(id, current_clos, &clos_config);
 	if (ret)
 		isst_display_error_info_message(1, "isst_set_clos failed", 0, 0);
 	else
-		isst_display_result(cpu, outf, "core-power", "config", ret);
+		isst_display_result(id, outf, "core-power", "config", ret);
 }
 
 static void set_clos_config(int arg)
@@ -2275,16 +2272,16 @@ static void set_clos_config(int arg)
 	isst_ctdp_display_information_end(outf);
 }
 
-static void set_clos_assoc_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
+static void set_clos_assoc_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 				   void *arg4)
 {
 	int ret;
 
-	ret = isst_clos_associate(cpu, current_clos);
+	ret = isst_clos_associate(id, current_clos);
 	if (ret)
 		debug_printf("isst_clos_associate failed");
 	else
-		isst_display_result(cpu, outf, "core-power", "assoc", ret);
+		isst_display_result(id, outf, "core-power", "assoc", ret);
 }
 
 static void set_clos_assoc(int arg)
@@ -2312,16 +2309,16 @@ static void set_clos_assoc(int arg)
 	}
 }
 
-static void get_clos_assoc_for_cpu(int cpu, void *arg1, void *arg2, void *arg3,
+static void get_clos_assoc_for_cpu(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 				   void *arg4)
 {
 	int clos, ret;
 
-	ret = isst_clos_get_assoc_status(cpu, &clos);
+	ret = isst_clos_get_assoc_status(id, &clos);
 	if (ret)
 		isst_display_error_info_message(1, "isst_clos_get_assoc_status failed", 0, 0);
 	else
-		isst_clos_display_assoc_information(cpu, outf, clos);
+		isst_clos_display_assoc_information(id, outf, clos);
 }
 
 static void get_clos_assoc(int arg)
@@ -2343,27 +2340,28 @@ static void get_clos_assoc(int arg)
 	isst_ctdp_display_information_end(outf);
 }
 
-static void set_turbo_mode_for_cpu(int cpu, int status)
+static void set_turbo_mode_for_cpu(struct isst_id *id, int status)
 {
 	int base_freq;
 
 	if (status) {
-		base_freq = get_cpufreq_base_freq(cpu);
-		set_cpufreq_scaling_min_max(cpu, 1, base_freq);
+		base_freq = get_cpufreq_base_freq(id->cpu);
+		set_cpufreq_scaling_min_max(id->cpu, 1, base_freq);
 	} else {
-		set_scaling_max_to_cpuinfo_max(cpu);
+		set_scaling_max_to_cpuinfo_max(id);
 	}
 
 	if (status) {
-		isst_display_result(cpu, outf, "turbo-mode", "enable", 0);
+		isst_display_result(id, outf, "turbo-mode", "enable", 0);
 	} else {
-		isst_display_result(cpu, outf, "turbo-mode", "disable", 0);
+		isst_display_result(id, outf, "turbo-mode", "disable", 0);
 	}
 }
 
 static void set_turbo_mode(int arg)
 {
 	int i, enable = arg;
+	struct isst_id id;
 
 	if (cmd_help) {
 		if (enable)
@@ -2385,14 +2383,16 @@ static void set_turbo_mode(int arg)
 			online =
 				1; /* online entry for CPU 0 needs some special configs */
 
-		if (online)
-			set_turbo_mode_for_cpu(i, enable);
+		if (online) {
+			set_isst_id(&id, i);
+			set_turbo_mode_for_cpu(&id, enable);
+		}
 
 	}
 	isst_ctdp_display_information_end(outf);
 }
 
-static void get_set_trl(int cpu, void *arg1, void *arg2, void *arg3,
+static void get_set_trl(struct isst_id *id, void *arg1, void *arg2, void *arg3,
 			void *arg4)
 {
 	unsigned long long trl;
@@ -2405,16 +2405,16 @@ static void get_set_trl(int cpu, void *arg1, void *arg2, void *arg3,
 	}
 
 	if (set) {
-		ret = isst_set_trl(cpu, fact_trl);
-		isst_display_result(cpu, outf, "turbo-mode", "set-trl", ret);
+		ret = isst_set_trl(id, fact_trl);
+		isst_display_result(id, outf, "turbo-mode", "set-trl", ret);
 		return;
 	}
 
-	ret = isst_get_trl(cpu, &trl);
+	ret = isst_get_trl(id, &trl);
 	if (ret)
-		isst_display_result(cpu, outf, "turbo-mode", "get-trl", ret);
+		isst_display_result(id, outf, "turbo-mode", "get-trl", ret);
 	else
-		isst_trl_display_information(cpu, outf, trl);
+		isst_trl_display_information(id, outf, trl);
 }
 
 static void process_trl(int arg)
@@ -2754,9 +2754,6 @@ void process_command(int argc, char **argv,
 		}
 	}
 
-	if (!is_clx_n_platform())
-		create_cpu_map();
-
 	i = 0;
 	while (cmds[i].feature) {
 		if (!strcmp(cmds[i].feature, feature) &&
@@ -2960,11 +2957,9 @@ static void cmdline(int argc, char **argv)
 	if (force_cpus_online)
 		force_all_cpus_online();
 	store_cpu_topology();
-	set_cpu_present_cpu_mask();
-	set_cpu_target_cpu_mask();
+	create_cpu_map();
 
 	if (oob_mode) {
-		create_cpu_map();
 		if (debug_flag)
 			fprintf(stderr, "OOB mode is enabled in debug mode\n");
 
diff --git a/tools/power/x86/intel-speed-select/isst-core.c b/tools/power/x86/intel-speed-select/isst-core.c
index 4431c8a0d40a..f701b45c832c 100644
--- a/tools/power/x86/intel-speed-select/isst-core.c
+++ b/tools/power/x86/intel-speed-select/isst-core.c
@@ -6,7 +6,7 @@
 
 #include "isst.h"
 
-int isst_write_pm_config(int cpu, int cp_state)
+int isst_write_pm_config(struct isst_id *id, int cp_state)
 {
 	unsigned int req, resp;
 	int ret;
@@ -16,27 +16,27 @@ int isst_write_pm_config(int cpu, int cp_state)
 	else
 		req = 0;
 
-	ret = isst_send_mbox_command(cpu, WRITE_PM_CONFIG, PM_FEATURE, 0, req,
+	ret = isst_send_mbox_command(id->cpu, WRITE_PM_CONFIG, PM_FEATURE, 0, req,
 				     &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d WRITE_PM_CONFIG resp:%x\n", cpu, resp);
+	debug_printf("cpu:%d WRITE_PM_CONFIG resp:%x\n", id->cpu, resp);
 
 	return 0;
 }
 
-int isst_read_pm_config(int cpu, int *cp_state, int *cp_cap)
+int isst_read_pm_config(struct isst_id *id, int *cp_state, int *cp_cap)
 {
 	unsigned int resp;
 	int ret;
 
-	ret = isst_send_mbox_command(cpu, READ_PM_CONFIG, PM_FEATURE, 0, 0,
+	ret = isst_send_mbox_command(id->cpu, READ_PM_CONFIG, PM_FEATURE, 0, 0,
 				     &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d READ_PM_CONFIG resp:%x\n", cpu, resp);
+	debug_printf("cpu:%d READ_PM_CONFIG resp:%x\n", id->cpu, resp);
 
 	*cp_state = resp & BIT(16);
 	*cp_cap = resp & BIT(0) ? 1 : 0;
@@ -44,12 +44,12 @@ int isst_read_pm_config(int cpu, int *cp_state, int *cp_cap)
 	return 0;
 }
 
-int isst_get_ctdp_levels(int cpu, struct isst_pkg_ctdp *pkg_dev)
+int isst_get_ctdp_levels(struct isst_id *id, struct isst_pkg_ctdp *pkg_dev)
 {
 	unsigned int resp;
 	int ret;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP,
 				     CONFIG_TDP_GET_LEVELS_INFO, 0, 0, &resp);
 	if (ret) {
 		pkg_dev->levels = 0;
@@ -60,7 +60,7 @@ int isst_get_ctdp_levels(int cpu, struct isst_pkg_ctdp *pkg_dev)
 		return 0;
 	}
 
-	debug_printf("cpu:%d CONFIG_TDP_GET_LEVELS_INFO resp:%x\n", cpu, resp);
+	debug_printf("cpu:%d CONFIG_TDP_GET_LEVELS_INFO resp:%x\n", id->cpu, resp);
 
 	pkg_dev->version = resp & 0xff;
 	pkg_dev->levels = (resp >> 8) & 0xff;
@@ -71,14 +71,14 @@ int isst_get_ctdp_levels(int cpu, struct isst_pkg_ctdp *pkg_dev)
 	return 0;
 }
 
-int isst_get_ctdp_control(int cpu, int config_index,
+int isst_get_ctdp_control(struct isst_id *id, int config_index,
 			  struct isst_pkg_ctdp_level_info *ctdp_level)
 {
 	int cp_state, cp_cap;
 	unsigned int resp;
 	int ret;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP,
 				     CONFIG_TDP_GET_TDP_CONTROL, 0,
 				     config_index, &resp);
 	if (ret)
@@ -89,30 +89,30 @@ int isst_get_ctdp_control(int cpu, int config_index,
 	ctdp_level->fact_enabled = !!(resp & BIT(16));
 	ctdp_level->pbf_enabled = !!(resp & BIT(17));
 
-	ret = isst_read_pm_config(cpu, &cp_state, &cp_cap);
+	ret = isst_read_pm_config(id, &cp_state, &cp_cap);
 	if (ret) {
-		debug_printf("cpu:%d pm_config is not supported \n", cpu);
+		debug_printf("cpu:%d pm_config is not supported\n", id->cpu);
 	} else {
-		debug_printf("cpu:%d pm_config SST-CP state:%d cap:%d \n", cpu, cp_state, cp_cap);
+		debug_printf("cpu:%d pm_config SST-CP state:%d cap:%d\n", id->cpu, cp_state, cp_cap);
 		ctdp_level->sst_cp_support = cp_cap;
 		ctdp_level->sst_cp_enabled = cp_state;
 	}
 
 	debug_printf(
 		"cpu:%d CONFIG_TDP_GET_TDP_CONTROL resp:%x fact_support:%d pbf_support: %d fact_enabled:%d pbf_enabled:%d\n",
-		cpu, resp, ctdp_level->fact_support, ctdp_level->pbf_support,
+		id->cpu, resp, ctdp_level->fact_support, ctdp_level->pbf_support,
 		ctdp_level->fact_enabled, ctdp_level->pbf_enabled);
 
 	return 0;
 }
 
-int isst_get_tdp_info(int cpu, int config_index,
+int isst_get_tdp_info(struct isst_id *id, int config_index,
 		      struct isst_pkg_ctdp_level_info *ctdp_level)
 {
 	unsigned int resp;
 	int ret;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_GET_TDP_INFO,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_TDP_INFO,
 				     0, config_index, &resp);
 	if (ret) {
 		isst_display_error_info_message(1, "Invalid level, Can't get TDP information at level", 1, config_index);
@@ -124,18 +124,18 @@ int isst_get_tdp_info(int cpu, int config_index,
 
 	debug_printf(
 		"cpu:%d ctdp:%d CONFIG_TDP_GET_TDP_INFO resp:%x tdp_ratio:%d pkg_tdp:%d\n",
-		cpu, config_index, resp, ctdp_level->tdp_ratio,
+		id->cpu, config_index, resp, ctdp_level->tdp_ratio,
 		ctdp_level->pkg_tdp);
 	return 0;
 }
 
-int isst_get_pwr_info(int cpu, int config_index,
+int isst_get_pwr_info(struct isst_id *id, int config_index,
 		      struct isst_pkg_ctdp_level_info *ctdp_level)
 {
 	unsigned int resp;
 	int ret;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_GET_PWR_INFO,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_PWR_INFO,
 				     0, config_index, &resp);
 	if (ret)
 		return ret;
@@ -145,18 +145,18 @@ int isst_get_pwr_info(int cpu, int config_index,
 
 	debug_printf(
 		"cpu:%d ctdp:%d CONFIG_TDP_GET_PWR_INFO resp:%x pkg_max_power:%d pkg_min_power:%d\n",
-		cpu, config_index, resp, ctdp_level->pkg_max_power,
+		id->cpu, config_index, resp, ctdp_level->pkg_max_power,
 		ctdp_level->pkg_min_power);
 
 	return 0;
 }
 
-void isst_get_uncore_p0_p1_info(int cpu, int config_index,
+void isst_get_uncore_p0_p1_info(struct isst_id *id, int config_index,
 				struct isst_pkg_ctdp_level_info *ctdp_level)
 {
 	unsigned int resp;
 	int ret;
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP,
 				     CONFIG_TDP_GET_UNCORE_P0_P1_INFO, 0,
 				     config_index, &resp);
 	if (ret) {
@@ -169,16 +169,16 @@ void isst_get_uncore_p0_p1_info(int cpu, int config_index,
 	ctdp_level->uncore_p1 = (resp & GENMASK(15, 8)) >> 8;
 	debug_printf(
 		"cpu:%d ctdp:%d CONFIG_TDP_GET_UNCORE_P0_P1_INFO resp:%x uncore p0:%d uncore p1:%d\n",
-		cpu, config_index, resp, ctdp_level->uncore_p0,
+		id->cpu, config_index, resp, ctdp_level->uncore_p0,
 		ctdp_level->uncore_p1);
 }
 
-void isst_get_p1_info(int cpu, int config_index,
+void isst_get_p1_info(struct isst_id *id, int config_index,
 		      struct isst_pkg_ctdp_level_info *ctdp_level)
 {
 	unsigned int resp;
 	int ret;
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_GET_P1_INFO, 0,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_P1_INFO, 0,
 				     config_index, &resp);
 	if (ret) {
 		ctdp_level->sse_p1 = 0;
@@ -192,17 +192,17 @@ void isst_get_p1_info(int cpu, int config_index,
 	ctdp_level->avx512_p1 = (resp & GENMASK(23, 16)) >> 16;
 	debug_printf(
 		"cpu:%d ctdp:%d CONFIG_TDP_GET_P1_INFO resp:%x sse_p1:%d avx2_p1:%d avx512_p1:%d\n",
-		cpu, config_index, resp, ctdp_level->sse_p1,
+		id->cpu, config_index, resp, ctdp_level->sse_p1,
 		ctdp_level->avx2_p1, ctdp_level->avx512_p1);
 }
 
-void isst_get_uncore_mem_freq(int cpu, int config_index,
+void isst_get_uncore_mem_freq(struct isst_id *id, int config_index,
 			      struct isst_pkg_ctdp_level_info *ctdp_level)
 {
 	unsigned int resp;
 	int ret;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_GET_MEM_FREQ,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_MEM_FREQ,
 				     0, config_index, &resp);
 	if (ret) {
 		ctdp_level->mem_freq = 0;
@@ -226,16 +226,16 @@ void isst_get_uncore_mem_freq(int cpu, int config_index,
 	}
 	debug_printf(
 		"cpu:%d ctdp:%d CONFIG_TDP_GET_MEM_FREQ resp:%x uncore mem_freq:%d\n",
-		cpu, config_index, resp, ctdp_level->mem_freq);
+		id->cpu, config_index, resp, ctdp_level->mem_freq);
 }
 
-int isst_get_tjmax_info(int cpu, int config_index,
+int isst_get_tjmax_info(struct isst_id *id, int config_index,
 			struct isst_pkg_ctdp_level_info *ctdp_level)
 {
 	unsigned int resp;
 	int ret;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_GET_TJMAX_INFO,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_GET_TJMAX_INFO,
 				     0, config_index, &resp);
 	if (ret)
 		return ret;
@@ -244,12 +244,12 @@ int isst_get_tjmax_info(int cpu, int config_index,
 
 	debug_printf(
 		"cpu:%d ctdp:%d CONFIG_TDP_GET_TJMAX_INFO resp:%x t_proc_hot:%d\n",
-		cpu, config_index, resp, ctdp_level->t_proc_hot);
+		id->cpu, config_index, resp, ctdp_level->t_proc_hot);
 
 	return 0;
 }
 
-int isst_get_coremask_info(int cpu, int config_index,
+int isst_get_coremask_info(struct isst_id *id, int config_index,
 			   struct isst_pkg_ctdp_level_info *ctdp_level)
 {
 	unsigned int resp;
@@ -260,7 +260,7 @@ int isst_get_coremask_info(int cpu, int config_index,
 		unsigned long long mask;
 		int cpu_count = 0;
 
-		ret = isst_send_mbox_command(cpu, CONFIG_TDP,
+		ret = isst_send_mbox_command(id->cpu, CONFIG_TDP,
 					     CONFIG_TDP_GET_CORE_MASK, 0,
 					     (i << 8) | config_index, &resp);
 		if (ret)
@@ -268,27 +268,27 @@ int isst_get_coremask_info(int cpu, int config_index,
 
 		debug_printf(
 			"cpu:%d ctdp:%d mask:%d CONFIG_TDP_GET_CORE_MASK resp:%x\n",
-			cpu, config_index, i, resp);
+			id->cpu, config_index, i, resp);
 
 		mask = (unsigned long long)resp << (32 * i);
-		set_cpu_mask_from_punit_coremask(cpu, mask,
+		set_cpu_mask_from_punit_coremask(id, mask,
 						 ctdp_level->core_cpumask_size,
 						 ctdp_level->core_cpumask,
 						 &cpu_count);
 		ctdp_level->cpu_count += cpu_count;
-		debug_printf("cpu:%d ctdp:%d mask:%d cpu count:%d\n", cpu,
+		debug_printf("cpu:%d ctdp:%d mask:%d cpu count:%d\n", id->cpu,
 			     config_index, i, ctdp_level->cpu_count);
 	}
 
 	return 0;
 }
 
-int isst_get_get_trl_from_msr(int cpu, int *trl)
+int isst_get_get_trl_from_msr(struct isst_id *id, int *trl)
 {
 	unsigned long long msr_trl;
 	int ret;
 
-	ret = isst_send_msr_command(cpu, 0x1AD, 0, &msr_trl);
+	ret = isst_send_msr_command(id->cpu, 0x1AD, 0, &msr_trl);
 	if (ret)
 		return ret;
 
@@ -304,13 +304,13 @@ int isst_get_get_trl_from_msr(int cpu, int *trl)
 	return 0;
 }
 
-int isst_get_get_trl(int cpu, int level, int avx_level, int *trl)
+int isst_get_get_trl(struct isst_id *id, int level, int avx_level, int *trl)
 {
 	unsigned int req, resp;
 	int ret;
 
 	req = level | (avx_level << 16);
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP,
 				     CONFIG_TDP_GET_TURBO_LIMIT_RATIOS, 0, req,
 				     &resp);
 	if (ret)
@@ -318,7 +318,7 @@ int isst_get_get_trl(int cpu, int level, int avx_level, int *trl)
 
 	debug_printf(
 		"cpu:%d CONFIG_TDP_GET_TURBO_LIMIT_RATIOS req:%x resp:%x\n",
-		cpu, req, resp);
+		id->cpu, req, resp);
 
 	trl[0] = resp & GENMASK(7, 0);
 	trl[1] = (resp & GENMASK(15, 8)) >> 8;
@@ -326,13 +326,13 @@ int isst_get_get_trl(int cpu, int level, int avx_level, int *trl)
 	trl[3] = (resp & GENMASK(31, 24)) >> 24;
 
 	req = level | BIT(8) | (avx_level << 16);
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP,
 				     CONFIG_TDP_GET_TURBO_LIMIT_RATIOS, 0, req,
 				     &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d CONFIG_TDP_GET_TURBO_LIMIT req:%x resp:%x\n", cpu,
+	debug_printf("cpu:%d CONFIG_TDP_GET_TURBO_LIMIT req:%x resp:%x\n", id->cpu,
 		     req, resp);
 
 	trl[4] = resp & GENMASK(7, 0);
@@ -343,61 +343,37 @@ int isst_get_get_trl(int cpu, int level, int avx_level, int *trl)
 	return 0;
 }
 
-int isst_get_trl_bucket_info(int cpu, unsigned long long *buckets_info)
+int isst_get_trl_bucket_info(struct isst_id *id, unsigned long long *buckets_info)
 {
 	int ret;
 
-	debug_printf("cpu:%d bucket info via MSR\n", cpu);
+	debug_printf("cpu:%d bucket info via MSR\n", id->cpu);
 
 	*buckets_info = 0;
 
-	ret = isst_send_msr_command(cpu, 0x1ae, 0, buckets_info);
+	ret = isst_send_msr_command(id->cpu, 0x1ae, 0, buckets_info);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d bucket info via MSR successful 0x%llx\n", cpu,
+	debug_printf("cpu:%d bucket info via MSR successful 0x%llx\n", id->cpu,
 		     *buckets_info);
 
 	return 0;
 }
 
-int isst_set_tdp_level_msr(int cpu, int tdp_level)
-{
-	unsigned long long level = tdp_level;
-	int ret;
-
-	debug_printf("cpu: tdp_level via MSR %d\n", cpu, tdp_level);
-
-	if (isst_get_config_tdp_lock_status(cpu)) {
-		isst_display_error_info_message(1, "tdp_locked", 0, 0);
-		return -1;
-	}
-
-	if (tdp_level > 2)
-		return -1; /* invalid value */
-
-	ret = isst_send_msr_command(cpu, 0x64b, 1, &level);
-	if (ret)
-		return ret;
-
-	debug_printf("cpu: tdp_level via MSR successful %d\n", cpu, tdp_level);
-
-	return 0;
-}
-
-int isst_set_tdp_level(int cpu, int tdp_level)
+int isst_set_tdp_level(struct isst_id *id, int tdp_level)
 {
 	unsigned int resp;
 	int ret;
 
 
-	if (isst_get_config_tdp_lock_status(cpu)) {
+	if (isst_get_config_tdp_lock_status(id)) {
 		isst_display_error_info_message(1, "TDP is locked", 0, 0);
 		return -1;
 
 	}
 
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP, CONFIG_TDP_SET_LEVEL, 0,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP, CONFIG_TDP_SET_LEVEL, 0,
 				     tdp_level, &resp);
 	if (ret) {
 		isst_display_error_info_message(1, "Set TDP level failed for level", 1, tdp_level);
@@ -407,14 +383,14 @@ int isst_set_tdp_level(int cpu, int tdp_level)
 	return 0;
 }
 
-int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info)
+int isst_get_pbf_info(struct isst_id *id, int level, struct isst_pbf_info *pbf_info)
 {
 	struct isst_pkg_ctdp_level_info ctdp_level;
 	struct isst_pkg_ctdp pkg_dev;
 	int i, ret, max_punit_core, max_mask_index;
 	unsigned int req, resp;
 
-	ret = isst_get_ctdp_levels(cpu, &pkg_dev);
+	ret = isst_get_ctdp_levels(id, &pkg_dev);
 	if (ret) {
 		isst_display_error_info_message(1, "Failed to get number of levels", 0, 0);
 		return ret;
@@ -425,7 +401,7 @@ int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info)
 		return -1;
 	}
 
-	ret = isst_get_ctdp_control(cpu, level, &ctdp_level);
+	ret = isst_get_ctdp_control(id, level, &ctdp_level);
 	if (ret)
 		return ret;
 
@@ -436,14 +412,14 @@ int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info)
 
 	pbf_info->core_cpumask_size = alloc_cpu_set(&pbf_info->core_cpumask);
 
-	max_punit_core = get_max_punit_core_id(get_physical_package_id(cpu), get_physical_die_id(cpu));
+	max_punit_core = get_max_punit_core_id(id);
 	max_mask_index = max_punit_core > 32 ? 2 : 1;
 
 	for (i = 0; i < max_mask_index; ++i) {
 		unsigned long long mask;
 		int count;
 
-		ret = isst_send_mbox_command(cpu, CONFIG_TDP,
+		ret = isst_send_mbox_command(id->cpu, CONFIG_TDP,
 					     CONFIG_TDP_PBF_GET_CORE_MASK_INFO,
 					     0, (i << 8) | level, &resp);
 		if (ret)
@@ -451,23 +427,23 @@ int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info)
 
 		debug_printf(
 			"cpu:%d CONFIG_TDP_PBF_GET_CORE_MASK_INFO resp:%x\n",
-			cpu, resp);
+			id->cpu, resp);
 
 		mask = (unsigned long long)resp << (32 * i);
-		set_cpu_mask_from_punit_coremask(cpu, mask,
+		set_cpu_mask_from_punit_coremask(id, mask,
 						 pbf_info->core_cpumask_size,
 						 pbf_info->core_cpumask,
 						 &count);
 	}
 
 	req = level;
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP,
 				     CONFIG_TDP_PBF_GET_P1HI_P1LO_INFO, 0, req,
 				     &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d CONFIG_TDP_PBF_GET_P1HI_P1LO_INFO resp:%x\n", cpu,
+	debug_printf("cpu:%d CONFIG_TDP_PBF_GET_P1HI_P1LO_INFO resp:%x\n", id->cpu,
 		     resp);
 
 	pbf_info->p1_low = resp & 0xff;
@@ -475,21 +451,21 @@ int isst_get_pbf_info(int cpu, int level, struct isst_pbf_info *pbf_info)
 
 	req = level;
 	ret = isst_send_mbox_command(
-		cpu, CONFIG_TDP, CONFIG_TDP_PBF_GET_TDP_INFO, 0, req, &resp);
+		id->cpu, CONFIG_TDP, CONFIG_TDP_PBF_GET_TDP_INFO, 0, req, &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d CONFIG_TDP_PBF_GET_TDP_INFO resp:%x\n", cpu, resp);
+	debug_printf("cpu:%d CONFIG_TDP_PBF_GET_TDP_INFO resp:%x\n", id->cpu, resp);
 
 	pbf_info->tdp = resp & 0xffff;
 
 	req = level;
 	ret = isst_send_mbox_command(
-		cpu, CONFIG_TDP, CONFIG_TDP_PBF_GET_TJ_MAX_INFO, 0, req, &resp);
+		id->cpu, CONFIG_TDP, CONFIG_TDP_PBF_GET_TJ_MAX_INFO, 0, req, &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d CONFIG_TDP_PBF_GET_TJ_MAX_INFO resp:%x\n", cpu,
+	debug_printf("cpu:%d CONFIG_TDP_PBF_GET_TJ_MAX_INFO resp:%x\n", id->cpu,
 		     resp);
 	pbf_info->t_control = (resp >> 8) & 0xff;
 	pbf_info->t_prochot = resp & 0xff;
@@ -502,7 +478,7 @@ void isst_get_pbf_info_complete(struct isst_pbf_info *pbf_info)
 	free_cpu_set(pbf_info->core_cpumask);
 }
 
-int isst_set_pbf_fact_status(int cpu, int pbf, int enable)
+int isst_set_pbf_fact_status(struct isst_id *id, int pbf, int enable)
 {
 	struct isst_pkg_ctdp pkg_dev;
 	struct isst_pkg_ctdp_level_info ctdp_level;
@@ -510,13 +486,13 @@ int isst_set_pbf_fact_status(int cpu, int pbf, int enable)
 	unsigned int req = 0, resp;
 	int ret;
 
-	ret = isst_get_ctdp_levels(cpu, &pkg_dev);
+	ret = isst_get_ctdp_levels(id, &pkg_dev);
 	if (ret)
-		debug_printf("cpu:%d No support for dynamic ISST\n", cpu);
+		debug_printf("cpu:%d No support for dynamic ISST\n", id->cpu);
 
 	current_level = pkg_dev.current_level;
 
-	ret = isst_get_ctdp_control(cpu, current_level, &ctdp_level);
+	ret = isst_get_ctdp_control(id, current_level, &ctdp_level);
 	if (ret)
 		return ret;
 
@@ -542,18 +518,18 @@ int isst_set_pbf_fact_status(int cpu, int pbf, int enable)
 			req &= ~BIT(16);
 	}
 
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP,
 				     CONFIG_TDP_SET_TDP_CONTROL, 0, req, &resp);
 	if (ret)
 		return ret;
 
 	debug_printf("cpu:%d CONFIG_TDP_SET_TDP_CONTROL pbf/fact:%d req:%x\n",
-		     cpu, pbf, req);
+		     id->cpu, pbf, req);
 
 	return 0;
 }
 
-int isst_get_fact_bucket_info(int cpu, int level,
+int isst_get_fact_bucket_info(struct isst_id *id, int level,
 			      struct isst_fact_bucket_info *bucket_info)
 {
 	unsigned int resp;
@@ -563,7 +539,7 @@ int isst_get_fact_bucket_info(int cpu, int level,
 		int j;
 
 		ret = isst_send_mbox_command(
-			cpu, CONFIG_TDP,
+			id->cpu, CONFIG_TDP,
 			CONFIG_TDP_GET_FACT_HP_TURBO_LIMIT_NUMCORES, 0,
 			(i << 8) | level, &resp);
 		if (ret)
@@ -571,7 +547,7 @@ int isst_get_fact_bucket_info(int cpu, int level,
 
 		debug_printf(
 			"cpu:%d CONFIG_TDP_GET_FACT_HP_TURBO_LIMIT_NUMCORES index:%d level:%d resp:%x\n",
-			cpu, i, level, resp);
+			id->cpu, i, level, resp);
 
 		for (j = 0; j < 4; ++j) {
 			bucket_info[j + (i * 4)].high_priority_cores_count =
@@ -584,7 +560,7 @@ int isst_get_fact_bucket_info(int cpu, int level,
 			int j;
 
 			ret = isst_send_mbox_command(
-				cpu, CONFIG_TDP,
+				id->cpu, CONFIG_TDP,
 				CONFIG_TDP_GET_FACT_HP_TURBO_LIMIT_RATIOS, 0,
 				(k << 16) | (i << 8) | level, &resp);
 			if (ret)
@@ -592,7 +568,7 @@ int isst_get_fact_bucket_info(int cpu, int level,
 
 			debug_printf(
 				"cpu:%d CONFIG_TDP_GET_FACT_HP_TURBO_LIMIT_RATIOS index:%d level:%d avx:%d resp:%x\n",
-				cpu, i, level, k, resp);
+				id->cpu, i, level, k, resp);
 
 			for (j = 0; j < 4; ++j) {
 				switch (k) {
@@ -618,14 +594,14 @@ int isst_get_fact_bucket_info(int cpu, int level,
 	return 0;
 }
 
-int isst_get_fact_info(int cpu, int level, int fact_bucket, struct isst_fact_info *fact_info)
+int isst_get_fact_info(struct isst_id *id, int level, int fact_bucket, struct isst_fact_info *fact_info)
 {
 	struct isst_pkg_ctdp_level_info ctdp_level;
 	struct isst_pkg_ctdp pkg_dev;
 	unsigned int resp;
 	int j, ret, print;
 
-	ret = isst_get_ctdp_levels(cpu, &pkg_dev);
+	ret = isst_get_ctdp_levels(id, &pkg_dev);
 	if (ret) {
 		isst_display_error_info_message(1, "Failed to get number of levels", 0, 0);
 		return ret;
@@ -636,7 +612,7 @@ int isst_get_fact_info(int cpu, int level, int fact_bucket, struct isst_fact_inf
 		return -1;
 	}
 
-	ret = isst_get_ctdp_control(cpu, level, &ctdp_level);
+	ret = isst_get_ctdp_control(id, level, &ctdp_level);
 	if (ret)
 		return ret;
 
@@ -645,20 +621,20 @@ int isst_get_fact_info(int cpu, int level, int fact_bucket, struct isst_fact_inf
 		return -1;
 	}
 
-	ret = isst_send_mbox_command(cpu, CONFIG_TDP,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_TDP,
 				     CONFIG_TDP_GET_FACT_LP_CLIPPING_RATIO, 0,
 				     level, &resp);
 	if (ret)
 		return ret;
 
 	debug_printf("cpu:%d CONFIG_TDP_GET_FACT_LP_CLIPPING_RATIO resp:%x\n",
-		     cpu, resp);
+		     id->cpu, resp);
 
 	fact_info->lp_clipping_ratio_license_sse = resp & 0xff;
 	fact_info->lp_clipping_ratio_license_avx2 = (resp >> 8) & 0xff;
 	fact_info->lp_clipping_ratio_license_avx512 = (resp >> 16) & 0xff;
 
-	ret = isst_get_fact_bucket_info(cpu, level, fact_info->bucket_info);
+	ret = isst_get_fact_bucket_info(id, level, fact_info->bucket_info);
 	if (ret)
 		return ret;
 
@@ -680,32 +656,32 @@ int isst_get_fact_info(int cpu, int level, int fact_bucket, struct isst_fact_inf
 	return 0;
 }
 
-int isst_get_trl(int cpu, unsigned long long *trl)
+int isst_get_trl(struct isst_id *id, unsigned long long *trl)
 {
 	int ret;
 
-	ret = isst_send_msr_command(cpu, 0x1AD, 0, trl);
+	ret = isst_send_msr_command(id->cpu, 0x1AD, 0, trl);
 	if (ret)
 		return ret;
 
 	return 0;
 }
 
-int isst_set_trl(int cpu, unsigned long long trl)
+int isst_set_trl(struct isst_id *id, unsigned long long trl)
 {
 	int ret;
 
 	if (!trl)
 		trl = 0xFFFFFFFFFFFFFFFFULL;
 
-	ret = isst_send_msr_command(cpu, 0x1AD, 1, &trl);
+	ret = isst_send_msr_command(id->cpu, 0x1AD, 1, &trl);
 	if (ret)
 		return ret;
 
 	return 0;
 }
 
-int isst_set_trl_from_current_tdp(int cpu, unsigned long long trl)
+int isst_set_trl_from_current_tdp(struct isst_id *id, unsigned long long trl)
 {
 	unsigned long long msr_trl;
 	int ret;
@@ -717,11 +693,11 @@ int isst_set_trl_from_current_tdp(int cpu, unsigned long long trl)
 		int trl[8];
 		int i;
 
-		ret = isst_get_ctdp_levels(cpu, &pkg_dev);
+		ret = isst_get_ctdp_levels(id, &pkg_dev);
 		if (ret)
 			return ret;
 
-		ret = isst_get_get_trl(cpu, pkg_dev.current_level, 0, trl);
+		ret = isst_get_get_trl(id, pkg_dev.current_level, 0, trl);
 		if (ret)
 			return ret;
 
@@ -732,7 +708,7 @@ int isst_set_trl_from_current_tdp(int cpu, unsigned long long trl)
 			msr_trl |= (_trl << (i * 8));
 		}
 	}
-	ret = isst_send_msr_command(cpu, 0x1AD, 1, &msr_trl);
+	ret = isst_send_msr_command(id->cpu, 0x1AD, 1, &msr_trl);
 	if (ret)
 		return ret;
 
@@ -740,12 +716,12 @@ int isst_set_trl_from_current_tdp(int cpu, unsigned long long trl)
 }
 
 /* Return 1 if locked */
-int isst_get_config_tdp_lock_status(int cpu)
+int isst_get_config_tdp_lock_status(struct isst_id *id)
 {
 	unsigned long long tdp_control = 0;
 	int ret;
 
-	ret = isst_send_msr_command(cpu, 0x64b, 0, &tdp_control);
+	ret = isst_send_msr_command(id->cpu, 0x64b, 0, &tdp_control);
 	if (ret)
 		return ret;
 
@@ -754,7 +730,7 @@ int isst_get_config_tdp_lock_status(int cpu)
 	return ret;
 }
 
-void isst_get_process_ctdp_complete(int cpu, struct isst_pkg_ctdp *pkg_dev)
+void isst_get_process_ctdp_complete(struct isst_id *id, struct isst_pkg_ctdp *pkg_dev)
 {
 	int i;
 
@@ -771,19 +747,19 @@ void isst_get_process_ctdp_complete(int cpu, struct isst_pkg_ctdp *pkg_dev)
 	}
 }
 
-int isst_get_process_ctdp(int cpu, int tdp_level, struct isst_pkg_ctdp *pkg_dev)
+int isst_get_process_ctdp(struct isst_id *id, int tdp_level, struct isst_pkg_ctdp *pkg_dev)
 {
 	int i, ret, valid = 0;
 
 	if (pkg_dev->processed)
 		return 0;
 
-	ret = isst_get_ctdp_levels(cpu, pkg_dev);
+	ret = isst_get_ctdp_levels(id, pkg_dev);
 	if (ret)
 		return ret;
 
 	debug_printf("cpu: %d ctdp enable:%d current level: %d levels:%d\n",
-		     cpu, pkg_dev->enabled, pkg_dev->current_level,
+		     id->cpu, pkg_dev->enabled, pkg_dev->current_level,
 		     pkg_dev->levels);
 
 	if (tdp_level != 0xff && tdp_level > pkg_dev->levels) {
@@ -800,16 +776,16 @@ int isst_get_process_ctdp(int cpu, int tdp_level, struct isst_pkg_ctdp *pkg_dev)
 		if (tdp_level != 0xff && i != tdp_level)
 			continue;
 
-		debug_printf("cpu:%d Get Information for TDP level:%d\n", cpu,
+		debug_printf("cpu:%d Get Information for TDP level:%d\n", id->cpu,
 			     i);
 		ctdp_level = &pkg_dev->ctdp_level[i];
 
 		ctdp_level->level = i;
-		ctdp_level->control_cpu = cpu;
-		ctdp_level->pkg_id = get_physical_package_id(cpu);
-		ctdp_level->die_id = get_physical_die_id(cpu);
+		ctdp_level->control_cpu = id->cpu;
+		ctdp_level->pkg_id = id->pkg;
+		ctdp_level->die_id = id->die;
 
-		ret = isst_get_ctdp_control(cpu, i, ctdp_level);
+		ret = isst_get_ctdp_control(id, i, ctdp_level);
 		if (ret)
 			continue;
 
@@ -818,13 +794,13 @@ int isst_get_process_ctdp(int cpu, int tdp_level, struct isst_pkg_ctdp *pkg_dev)
 		ctdp_level->processed = 1;
 
 		if (ctdp_level->pbf_support) {
-			ret = isst_get_pbf_info(cpu, i, &ctdp_level->pbf_info);
+			ret = isst_get_pbf_info(id, i, &ctdp_level->pbf_info);
 			if (!ret)
 				ctdp_level->pbf_found = 1;
 		}
 
 		if (ctdp_level->fact_support) {
-			ret = isst_get_fact_info(cpu, i, 0xff,
+			ret = isst_get_fact_info(id, i, 0xff,
 						 &ctdp_level->fact_info);
 			if (ret)
 				return ret;
@@ -833,76 +809,76 @@ int isst_get_process_ctdp(int cpu, int tdp_level, struct isst_pkg_ctdp *pkg_dev)
 		if (!pkg_dev->enabled && is_skx_based_platform()) {
 			int freq;
 
-			freq = get_cpufreq_base_freq(cpu);
+			freq = get_cpufreq_base_freq(id->cpu);
 			if (freq > 0) {
 				ctdp_level->sse_p1 = freq / 100000;
 				ctdp_level->tdp_ratio = ctdp_level->sse_p1;
 			}
 
-			isst_get_get_trl_from_msr(cpu, ctdp_level->trl_sse_active_cores);
-			isst_get_trl_bucket_info(cpu, &ctdp_level->buckets_info);
+			isst_get_get_trl_from_msr(id, ctdp_level->trl_sse_active_cores);
+			isst_get_trl_bucket_info(id, &ctdp_level->buckets_info);
 			continue;
 		}
 
-		ret = isst_get_tdp_info(cpu, i, ctdp_level);
+		ret = isst_get_tdp_info(id, i, ctdp_level);
 		if (ret)
 			return ret;
 
-		ret = isst_get_pwr_info(cpu, i, ctdp_level);
+		ret = isst_get_pwr_info(id, i, ctdp_level);
 		if (ret)
 			return ret;
 
-		ret = isst_get_tjmax_info(cpu, i, ctdp_level);
+		ret = isst_get_tjmax_info(id, i, ctdp_level);
 		if (ret)
 			return ret;
 
 		ctdp_level->core_cpumask_size =
 			alloc_cpu_set(&ctdp_level->core_cpumask);
-		ret = isst_get_coremask_info(cpu, i, ctdp_level);
+		ret = isst_get_coremask_info(id, i, ctdp_level);
 		if (ret)
 			return ret;
 
-		ret = isst_get_trl_bucket_info(cpu, &ctdp_level->buckets_info);
+		ret = isst_get_trl_bucket_info(id, &ctdp_level->buckets_info);
 		if (ret)
 			return ret;
 
-		ret = isst_get_get_trl(cpu, i, 0,
+		ret = isst_get_get_trl(id, i, 0,
 				       ctdp_level->trl_sse_active_cores);
 		if (ret)
 			return ret;
 
-		ret = isst_get_get_trl(cpu, i, 1,
+		ret = isst_get_get_trl(id, i, 1,
 				       ctdp_level->trl_avx_active_cores);
 		if (ret)
 			return ret;
 
-		ret = isst_get_get_trl(cpu, i, 2,
+		ret = isst_get_get_trl(id, i, 2,
 				       ctdp_level->trl_avx_512_active_cores);
 		if (ret)
 			return ret;
 
-		isst_get_uncore_p0_p1_info(cpu, i, ctdp_level);
-		isst_get_p1_info(cpu, i, ctdp_level);
-		isst_get_uncore_mem_freq(cpu, i, ctdp_level);
+		isst_get_uncore_p0_p1_info(id, i, ctdp_level);
+		isst_get_p1_info(id, i, ctdp_level);
+		isst_get_uncore_mem_freq(id, i, ctdp_level);
 	}
 
 	if (!valid)
-		isst_display_error_info_message(0, "Invalid level, Can't get TDP control information at specified levels on cpu", 1, cpu);
+		isst_display_error_info_message(0, "Invalid level, Can't get TDP control information at specified levels on cpu", 1, id->cpu);
 
 	return 0;
 }
 
-int isst_clos_get_clos_information(int cpu, int *enable, int *type)
+int isst_clos_get_clos_information(struct isst_id *id, int *enable, int *type)
 {
 	unsigned int resp;
 	int ret;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG, 0, 0,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG, 0, 0,
 				     &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d CLOS_PM_QOS_CONFIG resp:%x\n", cpu, resp);
+	debug_printf("cpu:%d CLOS_PM_QOS_CONFIG resp:%x\n", id->cpu, resp);
 
 	if (resp & BIT(1))
 		*enable = 1;
@@ -917,7 +893,7 @@ int isst_clos_get_clos_information(int cpu, int *enable, int *type)
 	return 0;
 }
 
-int isst_pm_qos_config(int cpu, int enable_clos, int priority_type)
+int isst_pm_qos_config(struct isst_id *id, int enable_clos, int priority_type)
 {
 	unsigned int req, resp;
 	int ret;
@@ -926,13 +902,13 @@ int isst_pm_qos_config(int cpu, int enable_clos, int priority_type)
 		struct isst_pkg_ctdp pkg_dev;
 		struct isst_pkg_ctdp_level_info ctdp_level;
 
-		ret = isst_get_ctdp_levels(cpu, &pkg_dev);
+		ret = isst_get_ctdp_levels(id, &pkg_dev);
 		if (ret) {
 			debug_printf("isst_get_ctdp_levels\n");
 			return ret;
 		}
 
-		ret = isst_get_ctdp_control(cpu, pkg_dev.current_level,
+		ret = isst_get_ctdp_control(id, pkg_dev.current_level,
 					    &ctdp_level);
 		if (ret)
 			return ret;
@@ -941,23 +917,23 @@ int isst_pm_qos_config(int cpu, int enable_clos, int priority_type)
 			isst_display_error_info_message(1, "Ignoring request, turbo-freq feature is still enabled", 0, 0);
 			return -EINVAL;
 		}
-		ret = isst_write_pm_config(cpu, 0);
+		ret = isst_write_pm_config(id, 0);
 		if (ret)
 			isst_display_error_info_message(0, "WRITE_PM_CONFIG command failed, ignoring error", 0, 0);
 	} else {
-		ret = isst_write_pm_config(cpu, 1);
+		ret = isst_write_pm_config(id, 1);
 		if (ret)
 			isst_display_error_info_message(0, "WRITE_PM_CONFIG command failed, ignoring error", 0, 0);
 	}
 
-	ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG, 0, 0,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG, 0, 0,
 				     &resp);
 	if (ret) {
 		isst_display_error_info_message(1, "CLOS_PM_QOS_CONFIG command failed", 0, 0);
 		return ret;
 	}
 
-	debug_printf("cpu:%d CLOS_PM_QOS_CONFIG resp:%x\n", cpu, resp);
+	debug_printf("cpu:%d CLOS_PM_QOS_CONFIG resp:%x\n", id->cpu, resp);
 
 	req = resp;
 
@@ -974,30 +950,27 @@ int isst_pm_qos_config(int cpu, int enable_clos, int priority_type)
 	else
 		req = req & ~BIT(2);
 
-	ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PM_QOS_CONFIG,
 				     BIT(MBOX_CMD_WRITE_BIT), req, &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d CLOS_PM_QOS_CONFIG priority type:%d req:%x\n", cpu,
+	debug_printf("cpu:%d CLOS_PM_QOS_CONFIG priority type:%d req:%x\n", id->cpu,
 		     priority_type, req);
 
 	return 0;
 }
 
-int isst_pm_get_clos(int cpu, int clos, struct isst_clos_config *clos_config)
+int isst_pm_get_clos(struct isst_id *id, int clos, struct isst_clos_config *clos_config)
 {
 	unsigned int resp;
 	int ret;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PM_CLOS, clos, 0,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PM_CLOS, clos, 0,
 				     &resp);
 	if (ret)
 		return ret;
 
-	clos_config->pkg_id = get_physical_package_id(cpu);
-	clos_config->die_id = get_physical_die_id(cpu);
-
 	clos_config->epp = resp & 0x0f;
 	clos_config->clos_prop_prio = (resp >> 4) & 0x0f;
 	clos_config->clos_min = (resp >> 8) & 0xff;
@@ -1007,7 +980,7 @@ int isst_pm_get_clos(int cpu, int clos, struct isst_clos_config *clos_config)
 	return 0;
 }
 
-int isst_set_clos(int cpu, int clos, struct isst_clos_config *clos_config)
+int isst_set_clos(struct isst_id *id, int clos, struct isst_clos_config *clos_config)
 {
 	unsigned int req, resp;
 	unsigned int param;
@@ -1021,53 +994,53 @@ int isst_set_clos(int cpu, int clos, struct isst_clos_config *clos_config)
 
 	param = BIT(MBOX_CMD_WRITE_BIT) | clos;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PM_CLOS, param, req,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PM_CLOS, param, req,
 				     &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d CLOS_PM_CLOS param:%x req:%x\n", cpu, param, req);
+	debug_printf("cpu:%d CLOS_PM_CLOS param:%x req:%x\n", id->cpu, param, req);
 
 	return 0;
 }
 
-int isst_clos_get_assoc_status(int cpu, int *clos_id)
+int isst_clos_get_assoc_status(struct isst_id *id, int *clos_id)
 {
 	unsigned int resp;
 	unsigned int param;
 	int core_id, ret;
 
-	core_id = find_phy_core_num(cpu);
+	core_id = find_phy_core_num(id->cpu);
 	param = core_id;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PQR_ASSOC, param, 0,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PQR_ASSOC, param, 0,
 				     &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d CLOS_PQR_ASSOC param:%x resp:%x\n", cpu, param,
+	debug_printf("cpu:%d CLOS_PQR_ASSOC param:%x resp:%x\n", id->cpu, param,
 		     resp);
 	*clos_id = (resp >> 16) & 0x03;
 
 	return 0;
 }
 
-int isst_clos_associate(int cpu, int clos_id)
+int isst_clos_associate(struct isst_id *id, int clos_id)
 {
 	unsigned int req, resp;
 	unsigned int param;
 	int core_id, ret;
 
 	req = (clos_id & 0x03) << 16;
-	core_id = find_phy_core_num(cpu);
+	core_id = find_phy_core_num(id->cpu);
 	param = BIT(MBOX_CMD_WRITE_BIT) | core_id;
 
-	ret = isst_send_mbox_command(cpu, CONFIG_CLOS, CLOS_PQR_ASSOC, param,
+	ret = isst_send_mbox_command(id->cpu, CONFIG_CLOS, CLOS_PQR_ASSOC, param,
 				     req, &resp);
 	if (ret)
 		return ret;
 
-	debug_printf("cpu:%d CLOS_PQR_ASSOC param:%x req:%x\n", cpu, param,
+	debug_printf("cpu:%d CLOS_PQR_ASSOC param:%x req:%x\n", id->cpu, param,
 		     req);
 
 	return 0;
diff --git a/tools/power/x86/intel-speed-select/isst-daemon.c b/tools/power/x86/intel-speed-select/isst-daemon.c
index d0400c6684ba..0699137c0901 100644
--- a/tools/power/x86/intel-speed-select/isst-daemon.c
+++ b/tools/power/x86/intel-speed-select/isst-daemon.c
@@ -32,62 +32,60 @@ static void init_levels(void)
 			per_package_levels_info[i][j] = -1;
 }
 
-void process_level_change(int cpu)
+void process_level_change(struct isst_id *id)
 {
 	struct isst_pkg_ctdp_level_info ctdp_level;
-	int pkg_id = get_physical_package_id(cpu);
-	int die_id = get_physical_die_id(cpu);
 	struct isst_pkg_ctdp pkg_dev;
 	time_t tm;
 	int ret;
 
-	if (pkg_id >= MAX_PACKAGE_COUNT || die_id >= MAX_DIE_PER_PACKAGE) {
-		debug_printf("Invalid package/die info for cpu:%d\n", cpu);
+	if (id->pkg < 0 || id->die < 0) {
+		debug_printf("Invalid package/die info for cpu:%d\n", id->cpu);
 		return;
 	}
 
 	tm = time(NULL);
-	if (tm - per_package_levels_tm[pkg_id][die_id] < 2 )
+	if (tm - per_package_levels_tm[id->pkg][id->die] < 2)
 		return;
 
-	per_package_levels_tm[pkg_id][die_id] = tm;
+	per_package_levels_tm[id->pkg][id->die] = tm;
 
-	ret = isst_get_ctdp_levels(cpu, &pkg_dev);
+	ret = isst_get_ctdp_levels(id, &pkg_dev);
 	if (ret) {
-		debug_printf("Can't get tdp levels for cpu:%d\n", cpu);
+		debug_printf("Can't get tdp levels for cpu:%d\n", id->cpu);
 		return;
 	}
 
-	debug_printf("Get Config level %d pkg:%d die:%d current_level:%d \n", cpu,
-		      pkg_id, die_id, pkg_dev.current_level);
+	debug_printf("Get Config level %d pkg:%d die:%d current_level:%d\n", id->cpu,
+		      id->pkg, id->die, pkg_dev.current_level);
 
 	if (pkg_dev.locked) {
 		debug_printf("config TDP s locked \n");
 		return;
 	}
 
-	if (per_package_levels_info[pkg_id][die_id] == pkg_dev.current_level)
+	if (per_package_levels_info[id->pkg][id->die] == pkg_dev.current_level)
 		return;
 
 	debug_printf("**Config level change for cpu:%d pkg:%d die:%d from %d to %d\n",
-		      cpu, pkg_id, die_id, per_package_levels_info[pkg_id][die_id],
+		      id->cpu, id->pkg, id->die, per_package_levels_info[id->pkg][id->die],
 		      pkg_dev.current_level);
 
-	per_package_levels_info[pkg_id][die_id] = pkg_dev.current_level;
+	per_package_levels_info[id->pkg][id->die] = pkg_dev.current_level;
 
 	ctdp_level.core_cpumask_size =
 		alloc_cpu_set(&ctdp_level.core_cpumask);
-	ret = isst_get_coremask_info(cpu, pkg_dev.current_level, &ctdp_level);
+	ret = isst_get_coremask_info(id, pkg_dev.current_level, &ctdp_level);
 	if (ret) {
 		free_cpu_set(ctdp_level.core_cpumask);
-		debug_printf("Can't get core_mask:%d\n", cpu);
+		debug_printf("Can't get core_mask:%d\n", id->cpu);
 		return;
 	}
 
 	if (ctdp_level.cpu_count) {
 		int i, max_cpus = get_topo_max_cpus();
 		for (i = 0; i < max_cpus; ++i) {
-			if (pkg_id != get_physical_package_id(i) || die_id != get_physical_die_id(i))
+			if (!is_cpu_in_power_domain(i, id))
 				continue;
 			if (CPU_ISSET_S(i, ctdp_level.core_cpumask_size, ctdp_level.core_cpumask)) {
 				fprintf(stderr, "online cpu %d\n", i);
@@ -102,10 +100,10 @@ void process_level_change(int cpu)
 	free_cpu_set(ctdp_level.core_cpumask);
 }
 
-static void _poll_for_config_change(int cpu, void *arg1, void *arg2,
+static void _poll_for_config_change(struct isst_id *id, void *arg1, void *arg2,
 				    void *arg3, void *arg4)
 {
-	process_level_change(cpu);
+	process_level_change(id);
 }
 
 static void poll_for_config_change(void)
diff --git a/tools/power/x86/intel-speed-select/isst-display.c b/tools/power/x86/intel-speed-select/isst-display.c
index f97d8859ada7..b19f57d30f55 100644
--- a/tools/power/x86/intel-speed-select/isst-display.c
+++ b/tools/power/x86/intel-speed-select/isst-display.c
@@ -166,29 +166,27 @@ static void format_and_print(FILE *outf, int level, char *header, char *value)
 	last_level = level;
 }
 
-static int print_package_info(int cpu, FILE *outf)
+static int print_package_info(struct isst_id *id, FILE *outf)
 {
 	char header[256];
 
 	if (out_format_is_json()) {
 		snprintf(header, sizeof(header), "package-%d:die-%d:cpu-%d",
-			 get_physical_package_id(cpu), get_physical_die_id(cpu),
-			 cpu);
+			 id->pkg, id->die, id->cpu);
 		format_and_print(outf, 1, header, NULL);
 		return 1;
 	}
-	snprintf(header, sizeof(header), "package-%d",
-		 get_physical_package_id(cpu));
+	snprintf(header, sizeof(header), "package-%d", id->pkg);
 	format_and_print(outf, 1, header, NULL);
-	snprintf(header, sizeof(header), "die-%d", get_physical_die_id(cpu));
+	snprintf(header, sizeof(header), "die-%d", id->die);
 	format_and_print(outf, 2, header, NULL);
-	snprintf(header, sizeof(header), "cpu-%d", cpu);
+	snprintf(header, sizeof(header), "cpu-%d", id->cpu);
 	format_and_print(outf, 3, header, NULL);
 
 	return 3;
 }
 
-static void _isst_pbf_display_information(int cpu, FILE *outf, int level,
+static void _isst_pbf_display_information(struct isst_id *id, FILE *outf, int level,
 					  struct isst_pbf_info *pbf_info,
 					  int disp_level)
 {
@@ -231,7 +229,7 @@ static void _isst_pbf_display_information(int cpu, FILE *outf, int level,
 	format_and_print(outf, disp_level + 1, header, value);
 }
 
-static void _isst_fact_display_information(int cpu, FILE *outf, int level,
+static void _isst_fact_display_information(struct isst_id *id, FILE *outf, int level,
 					   int fact_bucket, int fact_avx,
 					   struct isst_fact_info *fact_info,
 					   int base_level)
@@ -319,7 +317,7 @@ static void _isst_fact_display_information(int cpu, FILE *outf, int level,
 	format_and_print(outf, base_level + 2, header, value);
 }
 
-void isst_ctdp_display_core_info(int cpu, FILE *outf, char *prefix,
+void isst_ctdp_display_core_info(struct isst_id *id, FILE *outf, char *prefix,
 				 unsigned int val, char *str0, char *str1)
 {
 	char header[256];
@@ -328,17 +326,14 @@ void isst_ctdp_display_core_info(int cpu, FILE *outf, char *prefix,
 
 	if (out_format_is_json()) {
 		snprintf(header, sizeof(header), "package-%d:die-%d:cpu-%d",
-			 get_physical_package_id(cpu), get_physical_die_id(cpu),
-			 cpu);
+			 id->pkg, id->die, id->cpu);
 		format_and_print(outf, level++, header, NULL);
 	} else {
-		snprintf(header, sizeof(header), "package-%d",
-			 get_physical_package_id(cpu));
+		snprintf(header, sizeof(header), "package-%d", id->pkg);
 		format_and_print(outf, level++, header, NULL);
-		snprintf(header, sizeof(header), "die-%d",
-			 get_physical_die_id(cpu));
+		snprintf(header, sizeof(header), "die-%d", id->die);
 		format_and_print(outf, level++, header, NULL);
-		snprintf(header, sizeof(header), "cpu-%d", cpu);
+		snprintf(header, sizeof(header), "cpu-%d", id->cpu);
 		format_and_print(outf, level++, header, NULL);
 	}
 
@@ -353,7 +348,7 @@ void isst_ctdp_display_core_info(int cpu, FILE *outf, char *prefix,
 	format_and_print(outf, 1, NULL, NULL);
 }
 
-void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level,
+void isst_ctdp_display_information(struct isst_id *id, FILE *outf, int tdp_level,
 				   struct isst_pkg_ctdp *pkg_dev)
 {
 	char header[256];
@@ -362,7 +357,7 @@ void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level,
 	int i;
 
 	if (pkg_dev->processed)
-		level = print_package_info(cpu, outf);
+		level = print_package_info(id, outf);
 
 	for (i = 0; i <= pkg_dev->levels; ++i) {
 		struct isst_pkg_ctdp_level_info *ctdp_level;
@@ -377,8 +372,7 @@ void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level,
 		format_and_print(outf, level + 1, header, NULL);
 
 		snprintf(header, sizeof(header), "cpu-count");
-		j = get_cpu_count(get_physical_die_id(cpu),
-				  get_physical_die_id(cpu));
+		j = get_cpu_count(id);
 		snprintf(value, sizeof(value), "%d", j);
 		format_and_print(outf, level + 2, header, value);
 
@@ -485,7 +479,7 @@ void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level,
 
 		if (is_clx_n_platform()) {
 			if (ctdp_level->pbf_support)
-				_isst_pbf_display_information(cpu, outf,
+				_isst_pbf_display_information(id, outf,
 							      tdp_level,
 							  &ctdp_level->pbf_info,
 							      level + 2);
@@ -557,11 +551,11 @@ void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level,
 		}
 
 		if (ctdp_level->pbf_support)
-			_isst_pbf_display_information(cpu, outf, i,
+			_isst_pbf_display_information(id, outf, i,
 						      &ctdp_level->pbf_info,
 						      level + 2);
 		if (ctdp_level->fact_support)
-			_isst_fact_display_information(cpu, outf, i, 0xff, 0xff,
+			_isst_fact_display_information(id, outf, i, 0xff, 0xff,
 						       &ctdp_level->fact_info,
 						       level + 2);
 	}
@@ -583,36 +577,36 @@ void isst_ctdp_display_information_end(FILE *outf)
 	start = 0;
 }
 
-void isst_pbf_display_information(int cpu, FILE *outf, int level,
+void isst_pbf_display_information(struct isst_id *id, FILE *outf, int level,
 				  struct isst_pbf_info *pbf_info)
 {
 	int _level;
 
-	_level = print_package_info(cpu, outf);
-	_isst_pbf_display_information(cpu, outf, level, pbf_info, _level + 1);
+	_level = print_package_info(id, outf);
+	_isst_pbf_display_information(id, outf, level, pbf_info, _level + 1);
 	format_and_print(outf, 1, NULL, NULL);
 }
 
-void isst_fact_display_information(int cpu, FILE *outf, int level,
+void isst_fact_display_information(struct isst_id *id, FILE *outf, int level,
 				   int fact_bucket, int fact_avx,
 				   struct isst_fact_info *fact_info)
 {
 	int _level;
 
-	_level = print_package_info(cpu, outf);
-	_isst_fact_display_information(cpu, outf, level, fact_bucket, fact_avx,
+	_level = print_package_info(id, outf);
+	_isst_fact_display_information(id, outf, level, fact_bucket, fact_avx,
 				       fact_info, _level + 1);
 	format_and_print(outf, 1, NULL, NULL);
 }
 
-void isst_clos_display_information(int cpu, FILE *outf, int clos,
+void isst_clos_display_information(struct isst_id *id, FILE *outf, int clos,
 				   struct isst_clos_config *clos_config)
 {
 	char header[256];
 	char value[256];
 	int level;
 
-	level = print_package_info(cpu, outf);
+	level = print_package_info(id, outf);
 
 	snprintf(header, sizeof(header), "core-power");
 	format_and_print(outf, level + 1, header, NULL);
@@ -647,7 +641,7 @@ void isst_clos_display_information(int cpu, FILE *outf, int clos,
 	format_and_print(outf, level, NULL, NULL);
 }
 
-void isst_clos_display_clos_information(int cpu, FILE *outf,
+void isst_clos_display_clos_information(struct isst_id *id, FILE *outf,
 					int clos_enable, int type,
 					int state, int cap)
 {
@@ -655,7 +649,7 @@ void isst_clos_display_clos_information(int cpu, FILE *outf,
 	char value[256];
 	int level;
 
-	level = print_package_info(cpu, outf);
+	level = print_package_info(id, outf);
 
 	snprintf(header, sizeof(header), "core-power");
 	format_and_print(outf, level + 1, header, NULL);
@@ -691,13 +685,13 @@ void isst_clos_display_clos_information(int cpu, FILE *outf,
 	format_and_print(outf, level, NULL, NULL);
 }
 
-void isst_clos_display_assoc_information(int cpu, FILE *outf, int clos)
+void isst_clos_display_assoc_information(struct isst_id *id, FILE *outf, int clos)
 {
 	char header[256];
 	char value[256];
 	int level;
 
-	level = print_package_info(cpu, outf);
+	level = print_package_info(id, outf);
 
 	snprintf(header, sizeof(header), "get-assoc");
 	format_and_print(outf, level + 1, header, NULL);
@@ -709,15 +703,15 @@ void isst_clos_display_assoc_information(int cpu, FILE *outf, int clos)
 	format_and_print(outf, level, NULL, NULL);
 }
 
-void isst_display_result(int cpu, FILE *outf, char *feature, char *cmd,
+void isst_display_result(struct isst_id *id, FILE *outf, char *feature, char *cmd,
 			 int result)
 {
 	char header[256];
 	char value[256];
 	int level = 3;
 
-	if (cpu >= 0)
-		level = print_package_info(cpu, outf);
+	if (id->cpu >= 0)
+		level = print_package_info(id, outf);
 
 	snprintf(header, sizeof(header), "%s", feature);
 	format_and_print(outf, level + 1, header, NULL);
@@ -772,13 +766,13 @@ void isst_display_error_info_message(int error, char *msg, int arg_valid, int ar
 		format_and_print(outf, 0, NULL, NULL);
 }
 
-void isst_trl_display_information(int cpu, FILE *outf, unsigned long long trl)
+void isst_trl_display_information(struct isst_id *id, FILE *outf, unsigned long long trl)
 {
 	char header[256];
 	char value[256];
 	int level;
 
-	level = print_package_info(cpu, outf);
+	level = print_package_info(id, outf);
 
 	snprintf(header, sizeof(header), "get-trl");
 	format_and_print(outf, level + 1, header, NULL);
diff --git a/tools/power/x86/intel-speed-select/isst.h b/tools/power/x86/intel-speed-select/isst.h
index 0796d8c6a882..409fcc9c8033 100644
--- a/tools/power/x86/intel-speed-select/isst.h
+++ b/tools/power/x86/intel-speed-select/isst.h
@@ -79,9 +79,14 @@
 #define MAX_PACKAGE_COUNT 8
 #define MAX_DIE_PER_PACKAGE 2
 
+/* Unified structure to specific a CPU or a Power Domain */
+struct isst_id {
+	int cpu;
+	int pkg;
+	int die;
+};
+
 struct isst_clos_config {
-	int pkg_id;
-	int die_id;
 	unsigned char epp;
 	unsigned char clos_prop_prio;
 	unsigned char clos_min;
@@ -171,22 +176,20 @@ struct isst_pkg_ctdp {
 	struct isst_pkg_ctdp_level_info ctdp_level[ISST_MAX_TDP_LEVELS];
 };
 
+extern int is_cpu_in_power_domain(int cpu, struct isst_id *id);
 extern int get_topo_max_cpus(void);
-extern int get_cpu_count(int pkg_id, int die_id);
-extern int get_max_punit_core_id(int pkg_id, int die_id);
+extern int get_cpu_count(struct isst_id *id);
+extern int get_max_punit_core_id(struct isst_id *id);
 
 /* Common interfaces */
 FILE *get_output_file(void);
 extern void debug_printf(const char *format, ...);
 extern int out_format_is_json(void);
-extern int get_physical_package_id(int cpu);
-extern int get_physical_die_id(int cpu);
+extern void set_isst_id(struct isst_id *id, int cpu);
 extern size_t alloc_cpu_set(cpu_set_t **cpu_set);
 extern void free_cpu_set(cpu_set_t *cpu_set);
-extern int find_logical_cpu(int pkg_id, int die_id, int phy_cpu);
-extern int find_phy_cpu_num(int logical_cpu);
 extern int find_phy_core_num(int logical_cpu);
-extern void set_cpu_mask_from_punit_coremask(int cpu,
+extern void set_cpu_mask_from_punit_coremask(struct isst_id *id,
 					     unsigned long long core_mask,
 					     size_t core_cpumask_size,
 					     cpu_set_t *core_cpumask,
@@ -200,77 +203,74 @@ extern int isst_send_mbox_command(unsigned int cpu, unsigned char command,
 extern int isst_send_msr_command(unsigned int cpu, unsigned int command,
 				 int write, unsigned long long *req_resp);
 
-extern int isst_get_ctdp_levels(int cpu, struct isst_pkg_ctdp *pkg_dev);
-extern int isst_get_ctdp_control(int cpu, int config_index,
+extern int isst_get_ctdp_levels(struct isst_id *id, struct isst_pkg_ctdp *pkg_dev);
+extern int isst_get_ctdp_control(struct isst_id *id, int config_index,
 				 struct isst_pkg_ctdp_level_info *ctdp_level);
-extern int isst_get_coremask_info(int cpu, int config_index,
+extern int isst_get_coremask_info(struct isst_id *id, int config_index,
 			   struct isst_pkg_ctdp_level_info *ctdp_level);
-extern int isst_get_process_ctdp(int cpu, int tdp_level,
+extern int isst_get_process_ctdp(struct isst_id *id, int tdp_level,
 				 struct isst_pkg_ctdp *pkg_dev);
-extern void isst_get_process_ctdp_complete(int cpu,
+extern void isst_get_process_ctdp_complete(struct isst_id *id,
 					   struct isst_pkg_ctdp *pkg_dev);
-extern void isst_ctdp_display_information(int cpu, FILE *outf, int tdp_level,
+extern void isst_ctdp_display_information(struct isst_id *id, FILE *outf, int tdp_level,
 					  struct isst_pkg_ctdp *pkg_dev);
-extern void isst_ctdp_display_core_info(int cpu, FILE *outf, char *prefix,
+extern void isst_ctdp_display_core_info(struct isst_id *id, FILE *outf, char *prefix,
 					unsigned int val, char *str0, char *str1);
 extern void isst_ctdp_display_information_start(FILE *outf);
 extern void isst_ctdp_display_information_end(FILE *outf);
-extern void isst_pbf_display_information(int cpu, FILE *outf, int level,
+extern void isst_pbf_display_information(struct isst_id *id, FILE *outf, int level,
 					 struct isst_pbf_info *info);
-extern int isst_set_tdp_level(int cpu, int tdp_level);
-extern int isst_set_tdp_level_msr(int cpu, int tdp_level);
-extern int isst_set_pbf_fact_status(int cpu, int pbf, int enable);
-extern int isst_get_pbf_info(int cpu, int level,
+extern int isst_set_tdp_level(struct isst_id *id, int tdp_level);
+extern int isst_set_pbf_fact_status(struct isst_id *id, int pbf, int enable);
+extern int isst_get_pbf_info(struct isst_id *id, int level,
 			     struct isst_pbf_info *pbf_info);
 extern void isst_get_pbf_info_complete(struct isst_pbf_info *pbf_info);
-extern int isst_get_fact_info(int cpu, int level, int fact_bucket,
+extern int isst_get_fact_info(struct isst_id *id, int level, int fact_bucket,
 			      struct isst_fact_info *fact_info);
-extern int isst_get_fact_bucket_info(int cpu, int level,
+extern int isst_get_fact_bucket_info(struct isst_id *id, int level,
 				     struct isst_fact_bucket_info *bucket_info);
-extern void isst_fact_display_information(int cpu, FILE *outf, int level,
+extern void isst_fact_display_information(struct isst_id *id, FILE *outf, int level,
 					  int fact_bucket, int fact_avx,
 					  struct isst_fact_info *fact_info);
-extern int isst_set_trl(int cpu, unsigned long long trl);
-extern int isst_get_trl(int cpu, unsigned long long *trl);
-extern int isst_set_trl_from_current_tdp(int cpu, unsigned long long trl);
-extern int isst_get_config_tdp_lock_status(int cpu);
+extern int isst_set_trl(struct isst_id *id, unsigned long long trl);
+extern int isst_get_trl(struct isst_id *id, unsigned long long *trl);
+extern int isst_set_trl_from_current_tdp(struct isst_id *id, unsigned long long trl);
+extern int isst_get_config_tdp_lock_status(struct isst_id *id);
 
-extern int isst_pm_qos_config(int cpu, int enable_clos, int priority_type);
-extern int isst_pm_get_clos(int cpu, int clos,
+extern int isst_pm_qos_config(struct isst_id *id, int enable_clos, int priority_type);
+extern int isst_pm_get_clos(struct isst_id *id, int clos,
 			    struct isst_clos_config *clos_config);
-extern int isst_set_clos(int cpu, int clos,
+extern int isst_set_clos(struct isst_id *id, int clos,
 			 struct isst_clos_config *clos_config);
-extern int isst_clos_associate(int cpu, int clos);
-extern int isst_clos_get_assoc_status(int cpu, int *clos_id);
-extern void isst_clos_display_information(int cpu, FILE *outf, int clos,
+extern int isst_clos_associate(struct isst_id *id, int clos);
+extern int isst_clos_get_assoc_status(struct isst_id *id, int *clos_id);
+extern void isst_clos_display_information(struct isst_id *id, FILE *outf, int clos,
 					  struct isst_clos_config *clos_config);
-extern void isst_clos_display_assoc_information(int cpu, FILE *outf, int clos);
-extern int isst_read_reg(unsigned short reg, unsigned int *val);
-extern int isst_write_reg(int reg, unsigned int val);
+extern void isst_clos_display_assoc_information(struct isst_id *id, FILE *outf, int clos);
 
-extern void isst_display_result(int cpu, FILE *outf, char *feature, char *cmd,
+extern void isst_display_result(struct isst_id *id, FILE *outf, char *feature, char *cmd,
 				int result);
 
-extern int isst_clos_get_clos_information(int cpu, int *enable, int *type);
-extern void isst_clos_display_clos_information(int cpu, FILE *outf,
+extern int isst_clos_get_clos_information(struct isst_id *id, int *enable, int *type);
+extern void isst_clos_display_clos_information(struct isst_id *id, FILE *outf,
 					       int clos_enable, int type,
 					       int state, int cap);
 extern int is_clx_n_platform(void);
 extern int get_cpufreq_base_freq(int cpu);
-extern int isst_read_pm_config(int cpu, int *cp_state, int *cp_cap);
+extern int isst_read_pm_config(struct isst_id *id, int *cp_state, int *cp_cap);
 extern void isst_display_error_info_message(int error, char *msg, int arg_valid, int arg);
 extern int is_skx_based_platform(void);
 extern int is_spr_platform(void);
 extern int is_icx_platform(void);
-extern void isst_trl_display_information(int cpu, FILE *outf, unsigned long long trl);
+extern void isst_trl_display_information(struct isst_id *id, FILE *outf, unsigned long long trl);
 
 extern void set_cpu_online_offline(int cpu, int state);
-extern void for_each_online_package_in_set(void (*callback)(int, void *, void *,
+extern void for_each_online_package_in_set(void (*callback)(struct isst_id *, void *, void *,
 							    void *, void *),
 					   void *arg1, void *arg2, void *arg3,
 					   void *arg4);
 extern int isst_daemon(int debug_mode, int poll_interval, int no_daemon);
-extern void process_level_change(int cpu);
+extern void process_level_change(struct isst_id *id);
 extern int hfi_main(void);
 extern void hfi_exit(void);
 #endif
diff --git a/tools/testing/kunit/configs/all_tests_uml.config b/tools/testing/kunit/configs/all_tests.config
index bdee36bef4a3..f990cbb73250 100644
--- a/tools/testing/kunit/configs/all_tests_uml.config
+++ b/tools/testing/kunit/configs/all_tests.config
@@ -16,8 +16,6 @@ CONFIG_EXT4_FS=y
 CONFIG_MSDOS_FS=y
 CONFIG_VFAT_FS=y
 
-CONFIG_VIRTIO_UML=y
-CONFIG_UML_PCI_OVER_VIRTIO=y
 CONFIG_PCI=y
 CONFIG_USB4=y
 
diff --git a/tools/testing/kunit/configs/broken_on_uml.config b/tools/testing/kunit/configs/broken_on_uml.config
deleted file mode 100644
index 690870043ac0..000000000000
--- a/tools/testing/kunit/configs/broken_on_uml.config
+++ /dev/null
@@ -1,44 +0,0 @@
-# These are currently broken on UML and prevent allyesconfig from building
-# CONFIG_STATIC_LINK is not set
-# CONFIG_UML_NET_VECTOR is not set
-# CONFIG_UML_NET_VDE is not set
-# CONFIG_UML_NET_PCAP is not set
-# CONFIG_NET_PTP_CLASSIFY is not set
-# CONFIG_IP_VS is not set
-# CONFIG_BRIDGE_EBT_BROUTE is not set
-# CONFIG_BRIDGE_EBT_T_FILTER is not set
-# CONFIG_BRIDGE_EBT_T_NAT is not set
-# CONFIG_MTD_NAND_CADENCE is not set
-# CONFIG_MTD_NAND_NANDSIM is not set
-# CONFIG_BLK_DEV_NULL_BLK is not set
-# CONFIG_BLK_DEV_RAM is not set
-# CONFIG_SCSI_DEBUG is not set
-# CONFIG_NET_VENDOR_XILINX is not set
-# CONFIG_NULL_TTY is not set
-# CONFIG_PTP_1588_CLOCK is not set
-# CONFIG_PINCTRL_EQUILIBRIUM is not set
-# CONFIG_DMABUF_SELFTESTS is not set
-# CONFIG_COMEDI is not set
-# CONFIG_XIL_AXIS_FIFO is not set
-# CONFIG_EXFAT_FS is not set
-# CONFIG_STM_DUMMY is not set
-# CONFIG_FSI_MASTER_ASPEED is not set
-# CONFIG_JFS_FS is not set
-# CONFIG_UBIFS_FS is not set
-# CONFIG_CRAMFS is not set
-# CONFIG_CRYPTO_DEV_SAFEXCEL is not set
-# CONFIG_CRYPTO_DEV_AMLOGIC_GXL is not set
-# CONFIG_KCOV is not set
-# CONFIG_LKDTM is not set
-# CONFIG_REED_SOLOMON_TEST is not set
-# CONFIG_TEST_RHASHTABLE is not set
-# CONFIG_TEST_MEMINIT is not set
-# CONFIG_NETWORK_PHY_TIMESTAMPING is not set
-# CONFIG_DEBUG_INFO_BTF is not set
-# CONFIG_PTP_1588_CLOCK_INES is not set
-# CONFIG_QCOM_CPR is not set
-# CONFIG_RESET_BRCMSTB_RESCAL is not set
-# CONFIG_RESET_INTEL_GW is not set
-# CONFIG_ADI_AXI_ADC is not set
-# CONFIG_DEBUG_PAGEALLOC is not set
-# CONFIG_PAGE_POISONING is not set
diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py
index e132b0654029..4d4663fb578b 100755
--- a/tools/testing/kunit/kunit.py
+++ b/tools/testing/kunit/kunit.py
@@ -44,7 +44,6 @@ class KunitConfigRequest:
 @dataclass
 class KunitBuildRequest(KunitConfigRequest):
 	jobs: int
-	alltests: bool
 
 @dataclass
 class KunitParseRequest:
@@ -55,7 +54,6 @@ class KunitParseRequest:
 class KunitExecRequest(KunitParseRequest):
 	build_dir: str
 	timeout: int
-	alltests: bool
 	filter_glob: str
 	kernel_args: Optional[List[str]]
 	run_isolated: Optional[str]
@@ -90,8 +88,7 @@ def build_tests(linux: kunit_kernel.LinuxSourceTree,
 	stdout.print_with_timestamp('Building KUnit Kernel ...')
 
 	build_start = time.time()
-	success = linux.build_kernel(request.alltests,
-				     request.jobs,
+	success = linux.build_kernel(request.jobs,
 				     request.build_dir,
 				     request.make_options)
 	build_end = time.time()
@@ -118,7 +115,7 @@ def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest)
 		args.extend(request.kernel_args)
 
 	output = linux.run_kernel(args=args,
-			   timeout=None if request.alltests else request.timeout,
+			   timeout=request.timeout,
 			   filter_glob=request.filter_glob,
 			   build_dir=request.build_dir)
 	lines = kunit_parser.extract_tap_lines(output)
@@ -165,7 +162,7 @@ def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -
 		test_start = time.time()
 		run_result = linux.run_kernel(
 			args=request.kernel_args,
-			timeout=None if request.alltests else request.timeout,
+			timeout=request.timeout,
 			filter_glob=filter_glob,
 			build_dir=request.build_dir)
 
@@ -206,7 +203,7 @@ def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input
 		if request.raw_output == 'all':
 			pass
 		elif request.raw_output == 'kunit':
-			output = kunit_parser.extract_tap_lines(output)
+			output = kunit_parser.extract_tap_lines(output, lstrip=False)
 		for line in output:
 			print(line.rstrip())
 
@@ -288,7 +285,7 @@ def add_common_opts(parser) -> None:
 			    help='X=Y make option, can be repeated.',
 			    action='append', metavar='X=Y')
 	parser.add_argument('--alltests',
-			    help='Run all KUnit tests through allyesconfig',
+			    help='Run all KUnit tests via tools/testing/kunit/configs/all_tests.config',
 			    action='store_true')
 	parser.add_argument('--kunitconfig',
 			     help='Path to Kconfig fragment that enables KUnit tests.'
@@ -381,8 +378,14 @@ def tree_from_args(cli_args: argparse.Namespace) -> kunit_kernel.LinuxSourceTree
 		for arg in cli_args.qemu_args:
 			qemu_args.extend(shlex.split(arg))
 
+	kunitconfigs = cli_args.kunitconfig if cli_args.kunitconfig else []
+	if cli_args.alltests:
+		# Prepend so user-specified options take prio if we ever allow
+		# --kunitconfig options to have differing options.
+		kunitconfigs = [kunit_kernel.ALL_TESTS_CONFIG_PATH] + kunitconfigs
+
 	return kunit_kernel.LinuxSourceTree(cli_args.build_dir,
-			kunitconfig_paths=cli_args.kunitconfig,
+			kunitconfig_paths=kunitconfigs,
 			kconfig_add=cli_args.kconfig_add,
 			arch=cli_args.arch,
 			cross_compile=cli_args.cross_compile,
@@ -441,7 +444,6 @@ def main(argv):
 		request = KunitRequest(build_dir=cli_args.build_dir,
 				       make_options=cli_args.make_options,
 				       jobs=cli_args.jobs,
-				       alltests=cli_args.alltests,
 				       raw_output=cli_args.raw_output,
 				       json=cli_args.json,
 				       timeout=cli_args.timeout,
@@ -469,8 +471,7 @@ def main(argv):
 		linux = tree_from_args(cli_args)
 		request = KunitBuildRequest(build_dir=cli_args.build_dir,
 					    make_options=cli_args.make_options,
-					    jobs=cli_args.jobs,
-					    alltests=cli_args.alltests)
+					    jobs=cli_args.jobs)
 		result = config_and_build_tests(linux, request)
 		stdout.print_with_timestamp((
 			'Elapsed time: %.3fs\n') % (
@@ -483,7 +484,6 @@ def main(argv):
 						build_dir=cli_args.build_dir,
 						json=cli_args.json,
 						timeout=cli_args.timeout,
-						alltests=cli_args.alltests,
 						filter_glob=cli_args.filter_glob,
 						kernel_args=cli_args.kernel_args,
 						run_isolated=cli_args.run_isolated)
diff --git a/tools/testing/kunit/kunit_kernel.py b/tools/testing/kunit/kunit_kernel.py
index f5c26ea89714..53e90c335834 100644
--- a/tools/testing/kunit/kunit_kernel.py
+++ b/tools/testing/kunit/kunit_kernel.py
@@ -25,7 +25,7 @@ KCONFIG_PATH = '.config'
 KUNITCONFIG_PATH = '.kunitconfig'
 OLD_KUNITCONFIG_PATH = 'last_used_kunitconfig'
 DEFAULT_KUNITCONFIG_PATH = 'tools/testing/kunit/configs/default.config'
-BROKEN_ALLCONFIG_PATH = 'tools/testing/kunit/configs/broken_on_uml.config'
+ALL_TESTS_CONFIG_PATH = 'tools/testing/kunit/configs/all_tests.config'
 UML_KCONFIG_PATH = 'tools/testing/kunit/configs/arch_uml.config'
 OUTFILE_PATH = 'test.log'
 ABS_TOOL_PATH = os.path.abspath(os.path.dirname(__file__))
@@ -57,9 +57,6 @@ class LinuxSourceTreeOperations:
 	def make_arch_config(self, base_kunitconfig: kunit_config.Kconfig) -> kunit_config.Kconfig:
 		return base_kunitconfig
 
-	def make_allyesconfig(self, build_dir: str, make_options) -> None:
-		raise ConfigError('Only the "um" arch is supported for alltests')
-
 	def make_olddefconfig(self, build_dir: str, make_options) -> None:
 		command = ['make', 'ARCH=' + self._linux_arch, 'O=' + build_dir, 'olddefconfig']
 		if self._cross_compile:
@@ -144,26 +141,6 @@ class LinuxSourceTreeOperationsUml(LinuxSourceTreeOperations):
 		kconfig.merge_in_entries(base_kunitconfig)
 		return kconfig
 
-	def make_allyesconfig(self, build_dir: str, make_options) -> None:
-		stdout.print_with_timestamp(
-			'Enabling all CONFIGs for UML...')
-		command = ['make', 'ARCH=um', 'O=' + build_dir, 'allyesconfig']
-		if make_options:
-			command.extend(make_options)
-		process = subprocess.Popen(
-			command,
-			stdout=subprocess.DEVNULL,
-			stderr=subprocess.STDOUT)
-		process.wait()
-		stdout.print_with_timestamp(
-			'Disabling broken configs to run KUnit tests...')
-
-		with open(get_kconfig_path(build_dir), 'a') as config:
-			with open(BROKEN_ALLCONFIG_PATH, 'r') as disable:
-				config.write(disable.read())
-		stdout.print_with_timestamp(
-			'Starting Kernel with all configs takes a few minutes...')
-
 	def start(self, params: List[str], build_dir: str) -> subprocess.Popen:
 		"""Runs the Linux UML binary. Must be named 'linux'."""
 		linux_bin = os.path.join(build_dir, 'linux')
@@ -343,10 +320,8 @@ class LinuxSourceTree:
 		os.remove(kconfig_path)
 		return self.build_config(build_dir, make_options)
 
-	def build_kernel(self, alltests, jobs, build_dir: str, make_options) -> bool:
+	def build_kernel(self, jobs, build_dir: str, make_options) -> bool:
 		try:
-			if alltests:
-				self._ops.make_allyesconfig(build_dir, make_options)
 			self._ops.make_olddefconfig(build_dir, make_options)
 			self._ops.make(jobs, build_dir, make_options)
 		except (ConfigError, BuildError) as e:
@@ -359,6 +334,7 @@ class LinuxSourceTree:
 			args = []
 		if filter_glob:
 			args.append('kunit.filter_glob='+filter_glob)
+		args.append('kunit.enable=1')
 
 		process = self._ops.start(args, build_dir)
 		assert process.stdout is not None  # tell mypy it's set
diff --git a/tools/testing/kunit/kunit_parser.py b/tools/testing/kunit/kunit_parser.py
index 12d3ec77f427..1ae873e3e341 100644
--- a/tools/testing/kunit/kunit_parser.py
+++ b/tools/testing/kunit/kunit_parser.py
@@ -218,7 +218,7 @@ TAP_START = re.compile(r'TAP version ([0-9]+)$')
 KTAP_END = re.compile('(List of all partitions:|'
 	'Kernel panic - not syncing: VFS:|reboot: System halted)')
 
-def extract_tap_lines(kernel_output: Iterable[str]) -> LineStream:
+def extract_tap_lines(kernel_output: Iterable[str], lstrip=True) -> LineStream:
 	"""Extracts KTAP lines from the kernel output."""
 	def isolate_ktap_output(kernel_output: Iterable[str]) \
 			-> Iterator[Tuple[int, str]]:
@@ -244,9 +244,11 @@ def extract_tap_lines(kernel_output: Iterable[str]) -> LineStream:
 				# stop extracting KTAP lines
 				break
 			elif started:
-				# remove prefix and any indention and yield
-				# line with line number
-				line = line[prefix_len:].lstrip()
+				# remove the prefix and optionally any leading
+				# whitespace. Our parsing logic relies on this.
+				line = line[prefix_len:]
+				if lstrip:
+					line = line.lstrip()
 				yield line_num, line
 	return LineStream(lines=isolate_ktap_output(kernel_output))
 
diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py
index 446ac432d9a4..e2cd2cc2e98f 100755
--- a/tools/testing/kunit/kunit_tool_test.py
+++ b/tools/testing/kunit/kunit_tool_test.py
@@ -549,7 +549,7 @@ class KUnitMainTest(unittest.TestCase):
 	def test_build_passes_args_pass(self):
 		kunit.main(['build'])
 		self.assertEqual(self.linux_source_mock.build_reconfig.call_count, 1)
-		self.linux_source_mock.build_kernel.assert_called_once_with(False, kunit.get_default_jobs(), '.kunit', None)
+		self.linux_source_mock.build_kernel.assert_called_once_with(kunit.get_default_jobs(), '.kunit', None)
 		self.assertEqual(self.linux_source_mock.run_kernel.call_count, 0)
 
 	def test_exec_passes_args_pass(self):
@@ -664,7 +664,7 @@ class KUnitMainTest(unittest.TestCase):
 		build_dir = '.kunit'
 		jobs = kunit.get_default_jobs()
 		kunit.main(['build', '--build_dir', build_dir])
-		self.linux_source_mock.build_kernel.assert_called_once_with(False, jobs, build_dir, None)
+		self.linux_source_mock.build_kernel.assert_called_once_with(jobs, build_dir, None)
 
 	def test_exec_builddir(self):
 		build_dir = '.kunit'
@@ -695,6 +695,18 @@ class KUnitMainTest(unittest.TestCase):
 						qemu_config_path=None,
 						extra_qemu_args=[])
 
+	def test_config_alltests(self):
+		kunit.main(['config', '--kunitconfig=mykunitconfig', '--alltests'])
+		# Just verify that we parsed and initialized it correctly here.
+		self.mock_linux_init.assert_called_once_with('.kunit',
+						kunitconfig_paths=[kunit_kernel.ALL_TESTS_CONFIG_PATH, 'mykunitconfig'],
+						kconfig_add=None,
+						arch='um',
+						cross_compile=None,
+						qemu_config_path=None,
+						extra_qemu_args=[])
+
+
 	@mock.patch.object(kunit_kernel, 'LinuxSourceTree')
 	def test_run_multiple_kunitconfig(self, mock_linux_init):
 		mock_linux_init.return_value = self.linux_source_mock
@@ -712,7 +724,7 @@ class KUnitMainTest(unittest.TestCase):
 		kunit.main(['run', '--kconfig_add=CONFIG_KASAN=y', '--kconfig_add=CONFIG_KCSAN=y'])
 		# Just verify that we parsed and initialized it correctly here.
 		self.mock_linux_init.assert_called_once_with('.kunit',
-						kunitconfig_paths=None,
+						kunitconfig_paths=[],
 						kconfig_add=['CONFIG_KASAN=y', 'CONFIG_KCSAN=y'],
 						arch='um',
 						cross_compile=None,
@@ -723,7 +735,7 @@ class KUnitMainTest(unittest.TestCase):
 		kunit.main(['run', '--arch=x86_64', '--qemu_args', '-m 2048'])
 		# Just verify that we parsed and initialized it correctly here.
 		self.mock_linux_init.assert_called_once_with('.kunit',
-						kunitconfig_paths=None,
+						kunitconfig_paths=[],
 						kconfig_add=None,
 						arch='x86_64',
 						cross_compile=None,
@@ -742,7 +754,7 @@ class KUnitMainTest(unittest.TestCase):
 		self.linux_source_mock.run_kernel.return_value = ['TAP version 14', 'init: random output'] + want
 
 		got = kunit._list_tests(self.linux_source_mock,
-				     kunit.KunitExecRequest(None, None, '.kunit', 300, False, 'suite*', None, 'suite'))
+				     kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, 'suite'))
 
 		self.assertEqual(got, want)
 		# Should respect the user's filter glob when listing tests.
@@ -757,7 +769,7 @@ class KUnitMainTest(unittest.TestCase):
 
 		# Should respect the user's filter glob when listing tests.
 		mock_tests.assert_called_once_with(mock.ANY,
-				     kunit.KunitExecRequest(None, None, '.kunit', 300, False, 'suite*.test*', None, 'suite'))
+				     kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*.test*', None, 'suite'))
 		self.linux_source_mock.run_kernel.assert_has_calls([
 			mock.call(args=None, build_dir='.kunit', filter_glob='suite.test*', timeout=300),
 			mock.call(args=None, build_dir='.kunit', filter_glob='suite2.test*', timeout=300),
@@ -770,7 +782,7 @@ class KUnitMainTest(unittest.TestCase):
 
 		# Should respect the user's filter glob when listing tests.
 		mock_tests.assert_called_once_with(mock.ANY,
-				     kunit.KunitExecRequest(None, None, '.kunit', 300, False, 'suite*', None, 'test'))
+				     kunit.KunitExecRequest(None, None, '.kunit', 300, 'suite*', None, 'test'))
 		self.linux_source_mock.run_kernel.assert_has_calls([
 			mock.call(args=None, build_dir='.kunit', filter_glob='suite.test1', timeout=300),
 			mock.call(args=None, build_dir='.kunit', filter_glob='suite.test2', timeout=300),
diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
index 1fc89b8ef433..0464b2c6c1e4 100644
--- a/tools/testing/selftests/Makefile
+++ b/tools/testing/selftests/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
 TARGETS += alsa
+TARGETS += amd-pstate
 TARGETS += arm64
 TARGETS += bpf
 TARGETS += breakpoints
diff --git a/tools/testing/selftests/amd-pstate/Makefile b/tools/testing/selftests/amd-pstate/Makefile
new file mode 100644
index 000000000000..199867f44b32
--- /dev/null
+++ b/tools/testing/selftests/amd-pstate/Makefile
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Makefile for amd-pstate/ function selftests
+
+# No binaries, but make sure arg-less "make" doesn't trigger "run_tests"
+all:
+
+TEST_PROGS := amd-pstate-ut.sh
+
+include ../lib.mk
diff --git a/tools/testing/selftests/amd-pstate/amd-pstate-ut.sh b/tools/testing/selftests/amd-pstate/amd-pstate-ut.sh
new file mode 100755
index 000000000000..f8e82d91ffcf
--- /dev/null
+++ b/tools/testing/selftests/amd-pstate/amd-pstate-ut.sh
@@ -0,0 +1,56 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+
+# amd-pstate-ut is a test module for testing the amd-pstate driver.
+# It can only run on x86 architectures and current cpufreq driver
+# must be amd-pstate.
+# (1) It can help all users to verify their processor support
+# (SBIOS/Firmware or Hardware).
+# (2) Kernel can have a basic function test to avoid the kernel
+# regression during the update.
+# (3) We can introduce more functional or performance tests to align
+# the result together, it will benefit power and performance scale optimization.
+
+# Kselftest framework requirement - SKIP code is 4.
+ksft_skip=4
+
+# amd-pstate-ut only run on x86/x86_64 AMD systems.
+ARCH=$(uname -m 2>/dev/null | sed -e 's/i.86/x86/' -e 's/x86_64/x86/')
+VENDOR=$(cat /proc/cpuinfo | grep -m 1 'vendor_id' | awk '{print $NF}')
+
+if ! echo "$ARCH" | grep -q x86; then
+	echo "$0 # Skipped: Test can only run on x86 architectures."
+	exit $ksft_skip
+fi
+
+if ! echo "$VENDOR" | grep -iq amd; then
+	echo "$0 # Skipped: Test can only run on AMD CPU."
+	echo "$0 # Current cpu vendor is $VENDOR."
+	exit $ksft_skip
+fi
+
+scaling_driver=$(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_driver)
+if [ "$scaling_driver" != "amd-pstate" ]; then
+	echo "$0 # Skipped: Test can only run on amd-pstate driver."
+	echo "$0 # Please set X86_AMD_PSTATE enabled."
+	echo "$0 # Current cpufreq scaling drvier is $scaling_driver."
+	exit $ksft_skip
+fi
+
+msg="Skip all tests:"
+if [ ! -w /dev ]; then
+    echo $msg please run this as root >&2
+    exit $ksft_skip
+fi
+
+if ! /sbin/modprobe -q -n amd-pstate-ut; then
+	echo "amd-pstate-ut: module amd-pstate-ut is not found [SKIP]"
+	exit $ksft_skip
+fi
+if /sbin/modprobe -q amd-pstate-ut; then
+	/sbin/modprobe -q -r amd-pstate-ut
+	echo "amd-pstate-ut: ok"
+else
+	echo "amd-pstate-ut: [FAIL]"
+	exit 1
+fi
diff --git a/tools/testing/selftests/amd-pstate/config b/tools/testing/selftests/amd-pstate/config
new file mode 100644
index 000000000000..f43103c9adc4
--- /dev/null
+++ b/tools/testing/selftests/amd-pstate/config
@@ -0,0 +1 @@
+CONFIG_X86_AMD_PSTATE_UT=m
diff --git a/tools/testing/selftests/arm64/abi/.gitignore b/tools/testing/selftests/arm64/abi/.gitignore
index b9e54417250d..44f8b80f37e3 100644
--- a/tools/testing/selftests/arm64/abi/.gitignore
+++ b/tools/testing/selftests/arm64/abi/.gitignore
@@ -1,2 +1,4 @@
+hwcap
+ptrace
 syscall-abi
 tpidr2
diff --git a/tools/testing/selftests/arm64/abi/Makefile b/tools/testing/selftests/arm64/abi/Makefile
index c8d7f2495eb2..a6d30c620908 100644
--- a/tools/testing/selftests/arm64/abi/Makefile
+++ b/tools/testing/selftests/arm64/abi/Makefile
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: GPL-2.0
 # Copyright (C) 2021 ARM Limited
 
-TEST_GEN_PROGS := syscall-abi tpidr2
+TEST_GEN_PROGS := hwcap ptrace syscall-abi tpidr2
 
 include ../../lib.mk
 
diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c
new file mode 100644
index 000000000000..9f1a7b5c6193
--- /dev/null
+++ b/tools/testing/selftests/arm64/abi/hwcap.c
@@ -0,0 +1,336 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2022 ARM Limited.
+ */
+
+#include <errno.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/auxv.h>
+#include <sys/prctl.h>
+#include <asm/hwcap.h>
+#include <asm/sigcontext.h>
+#include <asm/unistd.h>
+
+#include "../../kselftest.h"
+
+#define TESTS_PER_HWCAP 2
+
+/*
+ * Function expected to generate SIGILL when the feature is not
+ * supported and return when it is supported. If SIGILL is generated
+ * then the handler must be able to skip over the instruction safely.
+ *
+ * Note that it is expected that for many architecture extensions
+ * there are no specific traps due to no architecture state being
+ * added so we may not fault if running on a kernel which doesn't know
+ * to add the hwcap.
+ */
+typedef void (*sigill_fn)(void);
+
+static void rng_sigill(void)
+{
+	asm volatile("mrs x0, S3_3_C2_C4_0" : : : "x0");
+}
+
+static void sme_sigill(void)
+{
+	/* RDSVL x0, #0 */
+	asm volatile(".inst 0x04bf5800" : : : "x0");
+}
+
+static void sve_sigill(void)
+{
+	/* RDVL x0, #0 */
+	asm volatile(".inst 0x04bf5000" : : : "x0");
+}
+
+static void sve2_sigill(void)
+{
+	/* SQABS Z0.b, P0/M, Z0.B */
+	asm volatile(".inst 0x4408A000" : : : "z0");
+}
+
+static void sveaes_sigill(void)
+{
+	/* AESD z0.b, z0.b, z0.b */
+	asm volatile(".inst 0x4522e400" : : : "z0");
+}
+
+static void svepmull_sigill(void)
+{
+	/* PMULLB Z0.Q, Z0.D, Z0.D */
+	asm volatile(".inst 0x45006800" : : : "z0");
+}
+
+static void svebitperm_sigill(void)
+{
+	/* BDEP Z0.B, Z0.B, Z0.B */
+	asm volatile(".inst 0x4500b400" : : : "z0");
+}
+
+static void svesha3_sigill(void)
+{
+	/* EOR3 Z0.D, Z0.D, Z0.D, Z0.D */
+	asm volatile(".inst 0x4203800" : : : "z0");
+}
+
+static void svesm4_sigill(void)
+{
+	/* SM4E Z0.S, Z0.S, Z0.S */
+	asm volatile(".inst 0x4523e000" : : : "z0");
+}
+
+static void svei8mm_sigill(void)
+{
+	/* USDOT Z0.S, Z0.B, Z0.B[0] */
+	asm volatile(".inst 0x44a01800" : : : "z0");
+}
+
+static void svef32mm_sigill(void)
+{
+	/* FMMLA Z0.S, Z0.S, Z0.S */
+	asm volatile(".inst 0x64a0e400" : : : "z0");
+}
+
+static void svef64mm_sigill(void)
+{
+	/* FMMLA Z0.D, Z0.D, Z0.D */
+	asm volatile(".inst 0x64e0e400" : : : "z0");
+}
+
+static void svebf16_sigill(void)
+{
+	/* BFCVT Z0.H, P0/M, Z0.S */
+	asm volatile(".inst 0x658aa000" : : : "z0");
+}
+
+static const struct hwcap_data {
+	const char *name;
+	unsigned long at_hwcap;
+	unsigned long hwcap_bit;
+	const char *cpuinfo;
+	sigill_fn sigill_fn;
+	bool sigill_reliable;
+} hwcaps[] = {
+	{
+		.name = "RNG",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_RNG,
+		.cpuinfo = "rng",
+		.sigill_fn = rng_sigill,
+	},
+	{
+		.name = "SME",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SME,
+		.cpuinfo = "sme",
+		.sigill_fn = sme_sigill,
+		.sigill_reliable = true,
+	},
+	{
+		.name = "SVE",
+		.at_hwcap = AT_HWCAP,
+		.hwcap_bit = HWCAP_SVE,
+		.cpuinfo = "sve",
+		.sigill_fn = sve_sigill,
+		.sigill_reliable = true,
+	},
+	{
+		.name = "SVE 2",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVE2,
+		.cpuinfo = "sve2",
+		.sigill_fn = sve2_sigill,
+	},
+	{
+		.name = "SVE AES",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVEAES,
+		.cpuinfo = "sveaes",
+		.sigill_fn = sveaes_sigill,
+	},
+	{
+		.name = "SVE2 PMULL",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVEPMULL,
+		.cpuinfo = "svepmull",
+		.sigill_fn = svepmull_sigill,
+	},
+	{
+		.name = "SVE2 BITPERM",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVEBITPERM,
+		.cpuinfo = "svebitperm",
+		.sigill_fn = svebitperm_sigill,
+	},
+	{
+		.name = "SVE2 SHA3",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVESHA3,
+		.cpuinfo = "svesha3",
+		.sigill_fn = svesha3_sigill,
+	},
+	{
+		.name = "SVE2 SM4",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVESM4,
+		.cpuinfo = "svesm4",
+		.sigill_fn = svesm4_sigill,
+	},
+	{
+		.name = "SVE2 I8MM",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVEI8MM,
+		.cpuinfo = "svei8mm",
+		.sigill_fn = svei8mm_sigill,
+	},
+	{
+		.name = "SVE2 F32MM",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVEF32MM,
+		.cpuinfo = "svef32mm",
+		.sigill_fn = svef32mm_sigill,
+	},
+	{
+		.name = "SVE2 F64MM",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVEF64MM,
+		.cpuinfo = "svef64mm",
+		.sigill_fn = svef64mm_sigill,
+	},
+	{
+		.name = "SVE2 BF16",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVEBF16,
+		.cpuinfo = "svebf16",
+		.sigill_fn = svebf16_sigill,
+	},
+	{
+		.name = "SVE2 EBF16",
+		.at_hwcap = AT_HWCAP2,
+		.hwcap_bit = HWCAP2_SVE_EBF16,
+		.cpuinfo = "sveebf16",
+	},
+};
+
+static bool seen_sigill;
+
+static void handle_sigill(int sig, siginfo_t *info, void *context)
+{
+	ucontext_t *uc = context;
+
+	seen_sigill = true;
+
+	/* Skip over the offending instruction */
+	uc->uc_mcontext.pc += 4;
+}
+
+bool cpuinfo_present(const char *name)
+{
+	FILE *f;
+	char buf[2048], name_space[30], name_newline[30];
+	char *s;
+
+	/*
+	 * The feature should appear with a leading space and either a
+	 * trailing space or a newline.
+	 */
+	snprintf(name_space, sizeof(name_space), " %s ", name);
+	snprintf(name_newline, sizeof(name_newline), " %s\n", name);
+
+	f = fopen("/proc/cpuinfo", "r");
+	if (!f) {
+		ksft_print_msg("Failed to open /proc/cpuinfo\n");
+		return false;
+	}
+
+	while (fgets(buf, sizeof(buf), f)) {
+		/* Features: line? */
+		if (strncmp(buf, "Features\t:", strlen("Features\t:")) != 0)
+			continue;
+
+		/* All CPUs should be symmetric, don't read any more */
+		fclose(f);
+
+		s = strstr(buf, name_space);
+		if (s)
+			return true;
+		s = strstr(buf, name_newline);
+		if (s)
+			return true;
+
+		return false;
+	}
+
+	ksft_print_msg("Failed to find Features in /proc/cpuinfo\n");
+	fclose(f);
+	return false;
+}
+
+int main(void)
+{
+	const struct hwcap_data *hwcap;
+	int i, ret;
+	bool have_cpuinfo, have_hwcap;
+	struct sigaction sa;
+
+	ksft_print_header();
+	ksft_set_plan(ARRAY_SIZE(hwcaps) * TESTS_PER_HWCAP);
+
+	memset(&sa, 0, sizeof(sa));
+	sa.sa_sigaction = handle_sigill;
+	sa.sa_flags = SA_RESTART | SA_SIGINFO;
+	sigemptyset(&sa.sa_mask);
+	ret = sigaction(SIGILL, &sa, NULL);
+	if (ret < 0)
+		ksft_exit_fail_msg("Failed to install SIGILL handler: %s (%d)\n",
+				   strerror(errno), errno);
+
+	for (i = 0; i < ARRAY_SIZE(hwcaps); i++) {
+		hwcap = &hwcaps[i];
+
+		have_hwcap = getauxval(hwcap->at_hwcap) & hwcap->hwcap_bit;
+		have_cpuinfo = cpuinfo_present(hwcap->cpuinfo);
+
+		if (have_hwcap)
+			ksft_print_msg("%s present\n", hwcap->name);
+
+		ksft_test_result(have_hwcap == have_cpuinfo,
+				 "cpuinfo_match_%s\n", hwcap->name);
+
+		if (hwcap->sigill_fn) {
+			seen_sigill = false;
+			hwcap->sigill_fn();
+
+			if (have_hwcap) {
+				/* Should be able to use the extension */
+				ksft_test_result(!seen_sigill, "sigill_%s\n",
+						 hwcap->name);
+			} else if (hwcap->sigill_reliable) {
+				/* Guaranteed a SIGILL */
+				ksft_test_result(seen_sigill, "sigill_%s\n",
+						 hwcap->name);
+			} else {
+				/* Missing SIGILL might be fine */
+				ksft_print_msg("SIGILL %sreported for %s\n",
+					       seen_sigill ? "" : "not ",
+					       hwcap->name);
+				ksft_test_result_skip("sigill_%s\n",
+						      hwcap->name);
+			}
+		} else {
+			ksft_test_result_skip("sigill_%s\n",
+					      hwcap->name);
+		}
+	}
+
+	ksft_print_cnts();
+
+	return 0;
+}
diff --git a/tools/testing/selftests/arm64/abi/ptrace.c b/tools/testing/selftests/arm64/abi/ptrace.c
new file mode 100644
index 000000000000..be952511af22
--- /dev/null
+++ b/tools/testing/selftests/arm64/abi/ptrace.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2022 ARM Limited.
+ */
+#include <errno.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/auxv.h>
+#include <sys/prctl.h>
+#include <sys/ptrace.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/wait.h>
+#include <asm/sigcontext.h>
+#include <asm/ptrace.h>
+
+#include "../../kselftest.h"
+
+#define EXPECTED_TESTS 7
+
+#define MAX_TPIDRS 2
+
+static bool have_sme(void)
+{
+	return getauxval(AT_HWCAP2) & HWCAP2_SME;
+}
+
+static void test_tpidr(pid_t child)
+{
+	uint64_t read_val[MAX_TPIDRS];
+	uint64_t write_val[MAX_TPIDRS];
+	struct iovec read_iov, write_iov;
+	bool test_tpidr2 = false;
+	int ret, i;
+
+	read_iov.iov_base = read_val;
+	write_iov.iov_base = write_val;
+
+	/* Should be able to read a single TPIDR... */
+	read_iov.iov_len = sizeof(uint64_t);
+	ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_TLS, &read_iov);
+	ksft_test_result(ret == 0, "read_tpidr_one\n");
+
+	/* ...write a new value.. */
+	write_iov.iov_len = sizeof(uint64_t);
+	write_val[0] = read_val[0]++;
+	ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_TLS, &write_iov);
+	ksft_test_result(ret == 0, "write_tpidr_one\n");
+
+	/* ...then read it back */
+	ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_TLS, &read_iov);
+	ksft_test_result(ret == 0 && write_val[0] == read_val[0],
+			 "verify_tpidr_one\n");
+
+	/* If we have TPIDR2 we should be able to read it */
+	read_iov.iov_len = sizeof(read_val);
+	ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_TLS, &read_iov);
+	if (ret == 0) {
+		/* If we have SME there should be two TPIDRs */
+		if (read_iov.iov_len >= sizeof(read_val))
+			test_tpidr2 = true;
+
+		if (have_sme() && test_tpidr2) {
+			ksft_test_result(test_tpidr2, "count_tpidrs\n");
+		} else {
+			ksft_test_result(read_iov.iov_len % sizeof(uint64_t) == 0,
+					 "count_tpidrs\n");
+		}
+	} else {
+		ksft_test_result_fail("count_tpidrs\n");
+	}
+
+	if (test_tpidr2) {
+		/* Try to write new values to all known TPIDRs... */
+		write_iov.iov_len = sizeof(write_val);
+		for (i = 0; i < MAX_TPIDRS; i++)
+			write_val[i] = read_val[i] + 1;
+		ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_TLS, &write_iov);
+
+		ksft_test_result(ret == 0 &&
+				 write_iov.iov_len == sizeof(write_val),
+				 "tpidr2_write\n");
+
+		/* ...then read them back */
+		read_iov.iov_len = sizeof(read_val);
+		ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_TLS, &read_iov);
+
+		if (have_sme()) {
+			/* Should read back the written value */
+			ksft_test_result(ret == 0 &&
+					 read_iov.iov_len >= sizeof(read_val) &&
+					 memcmp(read_val, write_val,
+						sizeof(read_val)) == 0,
+					 "tpidr2_read\n");
+		} else {
+			/* TPIDR2 should read as zero */
+			ksft_test_result(ret == 0 &&
+					 read_iov.iov_len >= sizeof(read_val) &&
+					 read_val[0] == write_val[0] &&
+					 read_val[1] == 0,
+					 "tpidr2_read\n");
+		}
+
+		/* Writing only TPIDR... */
+		write_iov.iov_len = sizeof(uint64_t);
+		memcpy(write_val, read_val, sizeof(read_val));
+		write_val[0] += 1;
+		ret = ptrace(PTRACE_SETREGSET, child, NT_ARM_TLS, &write_iov);
+
+		if (ret == 0) {
+			/* ...should leave TPIDR2 untouched */
+			read_iov.iov_len = sizeof(read_val);
+			ret = ptrace(PTRACE_GETREGSET, child, NT_ARM_TLS,
+				     &read_iov);
+
+			ksft_test_result(ret == 0 &&
+					 read_iov.iov_len >= sizeof(read_val) &&
+					 memcmp(read_val, write_val,
+						sizeof(read_val)) == 0,
+					 "write_tpidr_only\n");
+		} else {
+			ksft_test_result_fail("write_tpidr_only\n");
+		}
+	} else {
+		ksft_test_result_skip("tpidr2_write\n");
+		ksft_test_result_skip("tpidr2_read\n");
+		ksft_test_result_skip("write_tpidr_only\n");
+	}
+}
+
+static int do_child(void)
+{
+	if (ptrace(PTRACE_TRACEME, -1, NULL, NULL))
+		ksft_exit_fail_msg("PTRACE_TRACEME", strerror(errno));
+
+	if (raise(SIGSTOP))
+		ksft_exit_fail_msg("raise(SIGSTOP)", strerror(errno));
+
+	return EXIT_SUCCESS;
+}
+
+static int do_parent(pid_t child)
+{
+	int ret = EXIT_FAILURE;
+	pid_t pid;
+	int status;
+	siginfo_t si;
+
+	/* Attach to the child */
+	while (1) {
+		int sig;
+
+		pid = wait(&status);
+		if (pid == -1) {
+			perror("wait");
+			goto error;
+		}
+
+		/*
+		 * This should never happen but it's hard to flag in
+		 * the framework.
+		 */
+		if (pid != child)
+			continue;
+
+		if (WIFEXITED(status) || WIFSIGNALED(status))
+			ksft_exit_fail_msg("Child died unexpectedly\n");
+
+		if (!WIFSTOPPED(status))
+			goto error;
+
+		sig = WSTOPSIG(status);
+
+		if (ptrace(PTRACE_GETSIGINFO, pid, NULL, &si)) {
+			if (errno == ESRCH)
+				goto disappeared;
+
+			if (errno == EINVAL) {
+				sig = 0; /* bust group-stop */
+				goto cont;
+			}
+
+			ksft_test_result_fail("PTRACE_GETSIGINFO: %s\n",
+					      strerror(errno));
+			goto error;
+		}
+
+		if (sig == SIGSTOP && si.si_code == SI_TKILL &&
+		    si.si_pid == pid)
+			break;
+
+	cont:
+		if (ptrace(PTRACE_CONT, pid, NULL, sig)) {
+			if (errno == ESRCH)
+				goto disappeared;
+
+			ksft_test_result_fail("PTRACE_CONT: %s\n",
+					      strerror(errno));
+			goto error;
+		}
+	}
+
+	ksft_print_msg("Parent is %d, child is %d\n", getpid(), child);
+
+	test_tpidr(child);
+
+	ret = EXIT_SUCCESS;
+
+error:
+	kill(child, SIGKILL);
+
+disappeared:
+	return ret;
+}
+
+int main(void)
+{
+	int ret = EXIT_SUCCESS;
+	pid_t child;
+
+	srandom(getpid());
+
+	ksft_print_header();
+
+	ksft_set_plan(EXPECTED_TESTS);
+
+	child = fork();
+	if (!child)
+		return do_child();
+
+	if (do_parent(child))
+		ret = EXIT_FAILURE;
+
+	ksft_print_cnts();
+
+	return ret;
+}
diff --git a/tools/testing/selftests/arm64/abi/syscall-abi.c b/tools/testing/selftests/arm64/abi/syscall-abi.c
index b632bfe9e022..dd7ebe536d05 100644
--- a/tools/testing/selftests/arm64/abi/syscall-abi.c
+++ b/tools/testing/selftests/arm64/abi/syscall-abi.c
@@ -112,9 +112,11 @@ static int check_fpr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
 	return errors;
 }
 
+#define SVE_Z_SHARED_BYTES (128 / 8)
+
 static uint8_t z_zero[__SVE_ZREG_SIZE(SVE_VQ_MAX)];
-uint8_t z_in[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
-uint8_t z_out[SVE_NUM_PREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
+uint8_t z_in[SVE_NUM_ZREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
+uint8_t z_out[SVE_NUM_ZREGS * __SVE_ZREG_SIZE(SVE_VQ_MAX)];
 
 static void setup_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
 		    uint64_t svcr)
@@ -133,22 +135,39 @@ static int check_z(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
 	if (!sve_vl)
 		return 0;
 
-	/*
-	 * After a syscall the low 128 bits of the Z registers should
-	 * be preserved and the rest be zeroed or preserved, except if
-	 * we were in streaming mode in which case the low 128 bits may
-	 * also be cleared by the transition out of streaming mode.
-	 */
 	for (i = 0; i < SVE_NUM_ZREGS; i++) {
-		void *in = &z_in[reg_size * i];
-		void *out = &z_out[reg_size * i];
-
-		if ((memcmp(in, out, SVE_VQ_BYTES) != 0) &&
-		    !((svcr & SVCR_SM_MASK) &&
-		      memcmp(z_zero, out, SVE_VQ_BYTES) == 0)) {
-			ksft_print_msg("%s SVE VL %d Z%d low 128 bits changed\n",
-				       cfg->name, sve_vl, i);
-			errors++;
+		uint8_t *in = &z_in[reg_size * i];
+		uint8_t *out = &z_out[reg_size * i];
+
+		if (svcr & SVCR_SM_MASK) {
+			/*
+			 * In streaming mode the whole register should
+			 * be cleared by the transition out of
+			 * streaming mode.
+			 */
+			if (memcmp(z_zero, out, reg_size) != 0) {
+				ksft_print_msg("%s SVE VL %d Z%d non-zero\n",
+					       cfg->name, sve_vl, i);
+				errors++;
+			}
+		} else {
+			/*
+			 * For standard SVE the low 128 bits should be
+			 * preserved and any additional bits cleared.
+			 */
+			if (memcmp(in, out, SVE_Z_SHARED_BYTES) != 0) {
+				ksft_print_msg("%s SVE VL %d Z%d low 128 bits changed\n",
+					       cfg->name, sve_vl, i);
+				errors++;
+			}
+
+			if (reg_size > SVE_Z_SHARED_BYTES &&
+			    (memcmp(z_zero, out + SVE_Z_SHARED_BYTES,
+				    reg_size - SVE_Z_SHARED_BYTES) != 0)) {
+				ksft_print_msg("%s SVE VL %d Z%d high bits non-zero\n",
+					       cfg->name, sve_vl, i);
+				errors++;
+			}
 		}
 	}
 
@@ -176,9 +195,9 @@ static int check_p(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
 	if (!sve_vl)
 		return 0;
 
-	/* After a syscall the P registers should be preserved or zeroed */
+	/* After a syscall the P registers should be zeroed */
 	for (i = 0; i < SVE_NUM_PREGS * reg_size; i++)
-		if (p_out[i] && (p_in[i] != p_out[i]))
+		if (p_out[i])
 			errors++;
 	if (errors)
 		ksft_print_msg("%s SVE VL %d predicate registers non-zero\n",
@@ -226,9 +245,9 @@ static int check_ffr(struct syscall_cfg *cfg, int sve_vl, int sme_vl,
 	    !(getauxval(AT_HWCAP2) & HWCAP2_SME_FA64))
 		return 0;
 
-	/* After a syscall the P registers should be preserved or zeroed */
+	/* After a syscall FFR should be zeroed */
 	for (i = 0; i < reg_size; i++)
-		if (ffr_out[i] && (ffr_in[i] != ffr_out[i]))
+		if (ffr_out[i])
 			errors++;
 	if (errors)
 		ksft_print_msg("%s SVE VL %d FFR non-zero\n",
diff --git a/tools/testing/selftests/arm64/fp/.gitignore b/tools/testing/selftests/arm64/fp/.gitignore
index ea947af63882..df79d29664a1 100644
--- a/tools/testing/selftests/arm64/fp/.gitignore
+++ b/tools/testing/selftests/arm64/fp/.gitignore
@@ -1,4 +1,5 @@
 fp-pidbench
+fp-stress
 fpsimd-test
 rdvl-sme
 rdvl-sve
diff --git a/tools/testing/selftests/arm64/fp/Makefile b/tools/testing/selftests/arm64/fp/Makefile
index a7c2286bf65b..36db61358ed5 100644
--- a/tools/testing/selftests/arm64/fp/Makefile
+++ b/tools/testing/selftests/arm64/fp/Makefile
@@ -5,7 +5,10 @@ top_srcdir = $(realpath ../../../../../)
 
 CFLAGS += -I$(top_srcdir)/usr/include/
 
-TEST_GEN_PROGS := sve-ptrace sve-probe-vls vec-syscfg za-fork za-ptrace
+TEST_GEN_PROGS := fp-stress \
+	sve-ptrace sve-probe-vls \
+	vec-syscfg \
+	za-fork za-ptrace
 TEST_GEN_PROGS_EXTENDED := fp-pidbench fpsimd-test \
 	rdvl-sme rdvl-sve \
 	sve-test \
diff --git a/tools/testing/selftests/arm64/fp/asm-offsets.h b/tools/testing/selftests/arm64/fp/asm-offsets.h
index a180851496ec..757b2fd75dd7 100644
--- a/tools/testing/selftests/arm64/fp/asm-offsets.h
+++ b/tools/testing/selftests/arm64/fp/asm-offsets.h
@@ -3,6 +3,7 @@
 #define sa_handler 0
 #define sa_mask_sz 8
 #define SIGUSR1 10
+#define SIGUSR2 12
 #define SIGTERM 15
 #define SIGINT 2
 #define SIGABRT 6
diff --git a/tools/testing/selftests/arm64/fp/fp-stress.c b/tools/testing/selftests/arm64/fp/fp-stress.c
new file mode 100644
index 000000000000..4e62a9199f97
--- /dev/null
+++ b/tools/testing/selftests/arm64/fp/fp-stress.c
@@ -0,0 +1,555 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2022 ARM Limited.
+ */
+
+#define _GNU_SOURCE
+#define _POSIX_C_SOURCE 199309L
+
+#include <errno.h>
+#include <getopt.h>
+#include <poll.h>
+#include <signal.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/auxv.h>
+#include <sys/epoll.h>
+#include <sys/prctl.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/wait.h>
+#include <asm/hwcap.h>
+
+#include "../../kselftest.h"
+
+#define MAX_VLS 16
+
+struct child_data {
+	char *name, *output;
+	pid_t pid;
+	int stdout;
+	bool output_seen;
+	bool exited;
+	int exit_status;
+};
+
+static int epoll_fd;
+static struct child_data *children;
+static int num_children;
+static bool terminate;
+
+static void drain_output(bool flush);
+
+static int num_processors(void)
+{
+	long nproc = sysconf(_SC_NPROCESSORS_CONF);
+	if (nproc < 0) {
+		perror("Unable to read number of processors\n");
+		exit(EXIT_FAILURE);
+	}
+
+	return nproc;
+}
+
+static void child_start(struct child_data *child, const char *program)
+{
+	int ret, pipefd[2], i;
+	struct epoll_event ev;
+
+	ret = pipe(pipefd);
+	if (ret != 0)
+		ksft_exit_fail_msg("Failed to create stdout pipe: %s (%d)\n",
+				   strerror(errno), errno);
+
+	child->pid = fork();
+	if (child->pid == -1)
+		ksft_exit_fail_msg("fork() failed: %s (%d)\n",
+				   strerror(errno), errno);
+
+	if (!child->pid) {
+		/*
+		 * In child, replace stdout with the pipe, errors to
+		 * stderr from here as kselftest prints to stdout.
+		 */
+		ret = dup2(pipefd[1], 1);
+		if (ret == -1) {
+			fprintf(stderr, "dup2() %d\n", errno);
+			exit(EXIT_FAILURE);
+		}
+
+		/*
+		 * Very dumb mechanism to clean open FDs other than
+		 * stdio. We don't want O_CLOEXEC for the pipes...
+		 */
+		for (i = 3; i < 8192; i++)
+			close(i);
+
+		ret = execl(program, program, NULL);
+		fprintf(stderr, "execl(%s) failed: %d (%s)\n",
+			program, errno, strerror(errno));
+
+		exit(EXIT_FAILURE);
+	} else {
+		/*
+		 * In parent, remember the child and close our copy of the
+		 * write side of stdout.
+		 */
+		close(pipefd[1]);
+		child->stdout = pipefd[0];
+		child->output = NULL;
+		child->exited = false;
+		child->output_seen = false;
+
+		ev.events = EPOLLIN | EPOLLHUP;
+		ev.data.ptr = child;
+
+		ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, child->stdout, &ev);
+		if (ret < 0) {
+			ksft_exit_fail_msg("%s EPOLL_CTL_ADD failed: %s (%d)\n",
+					   child->name, strerror(errno), errno);
+		}
+
+		/*
+		 * Keep output flowing during child startup so logs
+		 * are more timely, can help debugging.
+		 */
+		drain_output(false);
+	}
+}
+
+static bool child_output_read(struct child_data *child)
+{
+	char read_data[1024];
+	char work[1024];
+	int ret, len, cur_work, cur_read;
+
+	ret = read(child->stdout, read_data, sizeof(read_data));
+	if (ret < 0) {
+		if (errno == EINTR)
+			return true;
+
+		ksft_print_msg("%s: read() failed: %s (%d)\n",
+			       child->name, strerror(errno),
+			       errno);
+		return false;
+	}
+	len = ret;
+
+	child->output_seen = true;
+
+	/* Pick up any partial read */
+	if (child->output) {
+		strncpy(work, child->output, sizeof(work) - 1);
+		cur_work = strnlen(work, sizeof(work));
+		free(child->output);
+		child->output = NULL;
+	} else {
+		cur_work = 0;
+	}
+
+	cur_read = 0;
+	while (cur_read < len) {
+		work[cur_work] = read_data[cur_read++];
+
+		if (work[cur_work] == '\n') {
+			work[cur_work] = '\0';
+			ksft_print_msg("%s: %s\n", child->name, work);
+			cur_work = 0;
+		} else {
+			cur_work++;
+		}
+	}
+
+	if (cur_work) {
+		work[cur_work] = '\0';
+		ret = asprintf(&child->output, "%s", work);
+		if (ret == -1)
+			ksft_exit_fail_msg("Out of memory\n");
+	}
+
+	return false;
+}
+
+static void child_output(struct child_data *child, uint32_t events,
+			 bool flush)
+{
+	bool read_more;
+
+	if (events & EPOLLIN) {
+		do {
+			read_more = child_output_read(child);
+		} while (read_more);
+	}
+
+	if (events & EPOLLHUP) {
+		close(child->stdout);
+		child->stdout = -1;
+		flush = true;
+	}
+
+	if (flush && child->output) {
+		ksft_print_msg("%s: %s<EOF>\n", child->name, child->output);
+		free(child->output);
+		child->output = NULL;
+	}
+}
+
+static void child_tickle(struct child_data *child)
+{
+	if (child->output_seen && !child->exited)
+		kill(child->pid, SIGUSR2);
+}
+
+static void child_stop(struct child_data *child)
+{
+	if (!child->exited)
+		kill(child->pid, SIGTERM);
+}
+
+static void child_cleanup(struct child_data *child)
+{
+	pid_t ret;
+	int status;
+	bool fail = false;
+
+	if (!child->exited) {
+		do {
+			ret = waitpid(child->pid, &status, 0);
+			if (ret == -1 && errno == EINTR)
+				continue;
+
+			if (ret == -1) {
+				ksft_print_msg("waitpid(%d) failed: %s (%d)\n",
+					       child->pid, strerror(errno),
+					       errno);
+				fail = true;
+				break;
+			}
+		} while (!WIFEXITED(status));
+		child->exit_status = WEXITSTATUS(status);
+	}
+
+	if (!child->output_seen) {
+		ksft_print_msg("%s no output seen\n", child->name);
+		fail = true;
+	}
+
+	if (child->exit_status != 0) {
+		ksft_print_msg("%s exited with error code %d\n",
+			       child->name, child->exit_status);
+		fail = true;
+	}
+
+	ksft_test_result(!fail, "%s\n", child->name);
+}
+
+static void handle_child_signal(int sig, siginfo_t *info, void *context)
+{
+	int i;
+	bool found = false;
+
+	for (i = 0; i < num_children; i++) {
+		if (children[i].pid == info->si_pid) {
+			children[i].exited = true;
+			children[i].exit_status = info->si_status;
+			found = true;
+			break;
+		}
+	}
+
+	if (!found)
+		ksft_print_msg("SIGCHLD for unknown PID %d with status %d\n",
+			       info->si_pid, info->si_status);
+}
+
+static void handle_exit_signal(int sig, siginfo_t *info, void *context)
+{
+	int i;
+
+	/* If we're already exiting then don't signal again */
+	if (terminate)
+		return;
+
+	ksft_print_msg("Got signal, exiting...\n");
+
+	terminate = true;
+
+	/*
+	 * This should be redundant, the main loop should clean up
+	 * after us, but for safety stop everything we can here.
+	 */
+	for (i = 0; i < num_children; i++)
+		child_stop(&children[i]);
+}
+
+static void start_fpsimd(struct child_data *child, int cpu, int copy)
+{
+	int ret;
+
+	child_start(child, "./fpsimd-test");
+
+	ret = asprintf(&child->name, "FPSIMD-%d-%d", cpu, copy);
+	if (ret == -1)
+		ksft_exit_fail_msg("asprintf() failed\n");
+
+	ksft_print_msg("Started %s\n", child->name);
+}
+
+static void start_sve(struct child_data *child, int vl, int cpu)
+{
+	int ret;
+
+	ret = prctl(PR_SVE_SET_VL, vl | PR_SVE_VL_INHERIT);
+	if (ret < 0)
+		ksft_exit_fail_msg("Failed to set SVE VL %d\n", vl);
+
+	child_start(child, "./sve-test");
+
+	ret = asprintf(&child->name, "SVE-VL-%d-%d", vl, cpu);
+	if (ret == -1)
+		ksft_exit_fail_msg("asprintf() failed\n");
+
+	ksft_print_msg("Started %s\n", child->name);
+}
+
+static void start_ssve(struct child_data *child, int vl, int cpu)
+{
+	int ret;
+
+	ret = prctl(PR_SME_SET_VL, vl | PR_SME_VL_INHERIT);
+	if (ret < 0)
+		ksft_exit_fail_msg("Failed to set SME VL %d\n", ret);
+
+	child_start(child, "./ssve-test");
+
+	ret = asprintf(&child->name, "SSVE-VL-%d-%d", vl, cpu);
+	if (ret == -1)
+		ksft_exit_fail_msg("asprintf() failed\n");
+
+	ksft_print_msg("Started %s\n", child->name);
+}
+
+static void start_za(struct child_data *child, int vl, int cpu)
+{
+	int ret;
+
+	ret = prctl(PR_SME_SET_VL, vl | PR_SVE_VL_INHERIT);
+	if (ret < 0)
+		ksft_exit_fail_msg("Failed to set SME VL %d\n", ret);
+
+	child_start(child, "./za-test");
+
+	ret = asprintf(&child->name, "ZA-VL-%d-%d", vl, cpu);
+	if (ret == -1)
+		ksft_exit_fail_msg("asprintf() failed\n");
+
+	ksft_print_msg("Started %s\n", child->name);
+}
+
+static void probe_vls(int vls[], int *vl_count, int set_vl)
+{
+	unsigned int vq;
+	int vl;
+
+	*vl_count = 0;
+
+	for (vq = SVE_VQ_MAX; vq > 0; --vq) {
+		vl = prctl(set_vl, vq * 16);
+		if (vl == -1)
+			ksft_exit_fail_msg("SET_VL failed: %s (%d)\n",
+					   strerror(errno), errno);
+
+		vl &= PR_SVE_VL_LEN_MASK;
+
+		vq = sve_vq_from_vl(vl);
+
+		vls[*vl_count] = vl;
+		*vl_count += 1;
+	}
+}
+
+/* Handle any pending output without blocking */
+static void drain_output(bool flush)
+{
+	struct epoll_event ev;
+	int ret = 1;
+
+	while (ret > 0) {
+		ret = epoll_wait(epoll_fd, &ev, 1, 0);
+		if (ret < 0) {
+			if (errno == EINTR)
+				continue;
+			ksft_print_msg("epoll_wait() failed: %s (%d)\n",
+				       strerror(errno), errno);
+		}
+
+		if (ret == 1)
+			child_output(ev.data.ptr, ev.events, flush);
+	}
+}
+
+static const struct option options[] = {
+	{ "timeout",	required_argument, NULL, 't' },
+	{ }
+};
+
+int main(int argc, char **argv)
+{
+	int ret;
+	int timeout = 10;
+	int cpus, tests, i, j, c;
+	int sve_vl_count, sme_vl_count, fpsimd_per_cpu;
+	int sve_vls[MAX_VLS], sme_vls[MAX_VLS];
+	struct epoll_event ev;
+	struct sigaction sa;
+
+	while ((c = getopt_long(argc, argv, "t:", options, NULL)) != -1) {
+		switch (c) {
+		case 't':
+			ret = sscanf(optarg, "%d", &timeout);
+			if (ret != 1)
+				ksft_exit_fail_msg("Failed to parse timeout %s\n",
+						   optarg);
+			break;
+		default:
+			ksft_exit_fail_msg("Unknown argument\n");
+		}
+	}
+
+	cpus = num_processors();
+	tests = 0;
+
+	if (getauxval(AT_HWCAP) & HWCAP_SVE) {
+		probe_vls(sve_vls, &sve_vl_count, PR_SVE_SET_VL);
+		tests += sve_vl_count * cpus;
+	} else {
+		sve_vl_count = 0;
+	}
+
+	if (getauxval(AT_HWCAP2) & HWCAP2_SME) {
+		probe_vls(sme_vls, &sme_vl_count, PR_SME_SET_VL);
+		tests += sme_vl_count * cpus * 2;
+	} else {
+		sme_vl_count = 0;
+	}
+
+	/* Force context switching if we only have FPSIMD */
+	if (!sve_vl_count && !sme_vl_count)
+		fpsimd_per_cpu = 2;
+	else
+		fpsimd_per_cpu = 1;
+	tests += cpus * fpsimd_per_cpu;
+
+	ksft_print_header();
+	ksft_set_plan(tests);
+
+	ksft_print_msg("%d CPUs, %d SVE VLs, %d SME VLs\n",
+		       cpus, sve_vl_count, sme_vl_count);
+
+	if (timeout > 0)
+		ksft_print_msg("Will run for %ds\n", timeout);
+	else
+		ksft_print_msg("Will run until terminated\n");
+
+	children = calloc(sizeof(*children), tests);
+	if (!children)
+		ksft_exit_fail_msg("Unable to allocate child data\n");
+
+	ret = epoll_create1(EPOLL_CLOEXEC);
+	if (ret < 0)
+		ksft_exit_fail_msg("epoll_create1() failed: %s (%d)\n",
+				   strerror(errno), ret);
+	epoll_fd = ret;
+
+	/* Get signal handers ready before we start any children */
+	memset(&sa, 0, sizeof(sa));
+	sa.sa_sigaction = handle_exit_signal;
+	sa.sa_flags = SA_RESTART | SA_SIGINFO;
+	sigemptyset(&sa.sa_mask);
+	ret = sigaction(SIGINT, &sa, NULL);
+	if (ret < 0)
+		ksft_print_msg("Failed to install SIGINT handler: %s (%d)\n",
+			       strerror(errno), errno);
+	ret = sigaction(SIGTERM, &sa, NULL);
+	if (ret < 0)
+		ksft_print_msg("Failed to install SIGTERM handler: %s (%d)\n",
+			       strerror(errno), errno);
+	sa.sa_sigaction = handle_child_signal;
+	ret = sigaction(SIGCHLD, &sa, NULL);
+	if (ret < 0)
+		ksft_print_msg("Failed to install SIGCHLD handler: %s (%d)\n",
+			       strerror(errno), errno);
+
+	for (i = 0; i < cpus; i++) {
+		for (j = 0; j < fpsimd_per_cpu; j++)
+			start_fpsimd(&children[num_children++], i, j);
+
+		for (j = 0; j < sve_vl_count; j++)
+			start_sve(&children[num_children++], sve_vls[j], i);
+
+		for (j = 0; j < sme_vl_count; j++) {
+			start_ssve(&children[num_children++], sme_vls[j], i);
+			start_za(&children[num_children++], sme_vls[j], i);
+		}
+	}
+
+	for (;;) {
+		/* Did we get a signal asking us to exit? */
+		if (terminate)
+			break;
+
+		/*
+		 * Timeout is counted in seconds with no output, the
+		 * tests print during startup then are silent when
+		 * running so this should ensure they all ran enough
+		 * to install the signal handler, this is especially
+		 * useful in emulation where we will both be slow and
+		 * likely to have a large set of VLs.
+		 */
+		ret = epoll_wait(epoll_fd, &ev, 1, 1000);
+		if (ret < 0) {
+			if (errno == EINTR)
+				continue;
+			ksft_exit_fail_msg("epoll_wait() failed: %s (%d)\n",
+					   strerror(errno), errno);
+		}
+
+		/* Output? */
+		if (ret == 1) {
+			child_output(ev.data.ptr, ev.events, false);
+			continue;
+		}
+
+		/* Otherwise epoll_wait() timed out */
+
+		for (i = 0; i < num_children; i++)
+			child_tickle(&children[i]);
+
+		/* Negative timeout means run indefinitely */
+		if (timeout < 0)
+			continue;
+		if (--timeout == 0)
+			break;
+	}
+
+	ksft_print_msg("Finishing up...\n");
+	terminate = true;
+
+	for (i = 0; i < tests; i++)
+		child_stop(&children[i]);
+
+	drain_output(false);
+
+	for (i = 0; i < tests; i++)
+		child_cleanup(&children[i]);
+
+	drain_output(true);
+
+	ksft_print_cnts();
+
+	return 0;
+}
diff --git a/tools/testing/selftests/arm64/fp/fpsimd-test.S b/tools/testing/selftests/arm64/fp/fpsimd-test.S
index e21e8ea52c7e..918d04885a33 100644
--- a/tools/testing/selftests/arm64/fp/fpsimd-test.S
+++ b/tools/testing/selftests/arm64/fp/fpsimd-test.S
@@ -151,6 +151,15 @@ function irritator_handler
 	ret
 endfunction
 
+function tickle_handler
+	// Increment the signal count (x23):
+	ldr	x0, [x2, #ucontext_regs + 8 * 23]
+	add	x0, x0, #1
+	str	x0, [x2, #ucontext_regs + 8 * 23]
+
+	ret
+endfunction
+
 function terminate_handler
 	mov	w21, w0
 	mov	x20, x2
@@ -207,6 +216,30 @@ endfunction
 .globl _start
 function _start
 _start:
+	mov	x23, #0		// signal count
+
+	mov	w0, #SIGINT
+	adr	x1, terminate_handler
+	mov	w2, #SA_SIGINFO
+	bl	setsignal
+
+	mov	w0, #SIGTERM
+	adr	x1, terminate_handler
+	mov	w2, #SA_SIGINFO
+	bl	setsignal
+
+	mov	w0, #SIGUSR1
+	adr	x1, irritator_handler
+	mov	w2, #SA_SIGINFO
+	orr	w2, w2, #SA_NODEFER
+	bl	setsignal
+
+	mov	w0, #SIGUSR2
+	adr	x1, tickle_handler
+	mov	w2, #SA_SIGINFO
+	orr	w2, w2, #SA_NODEFER
+	bl	setsignal
+
 	// Sanity-check and report the vector length
 
 	mov	x19, #128
@@ -237,24 +270,6 @@ _start:
 	mov	x0, x20
 	bl	putdecn
 
-	mov	x23, #0		// Irritation signal count
-
-	mov	w0, #SIGINT
-	adr	x1, terminate_handler
-	mov	w2, #SA_SIGINFO
-	bl	setsignal
-
-	mov	w0, #SIGTERM
-	adr	x1, terminate_handler
-	mov	w2, #SA_SIGINFO
-	bl	setsignal
-
-	mov	w0, #SIGUSR1
-	adr	x1, irritator_handler
-	mov	w2, #SA_SIGINFO
-	orr	w2, w2, #SA_NODEFER
-	bl	setsignal
-
 	mov	x22, #0		// generation number, increments per iteration
 .Ltest_loop:
 
diff --git a/tools/testing/selftests/arm64/fp/sve-test.S b/tools/testing/selftests/arm64/fp/sve-test.S
index 589264231a2d..2a18cb4c528c 100644
--- a/tools/testing/selftests/arm64/fp/sve-test.S
+++ b/tools/testing/selftests/arm64/fp/sve-test.S
@@ -314,6 +314,15 @@ function irritator_handler
 	ret
 endfunction
 
+function tickle_handler
+	// Increment the signal count (x23):
+	ldr	x0, [x2, #ucontext_regs + 8 * 23]
+	add	x0, x0, #1
+	str	x0, [x2, #ucontext_regs + 8 * 23]
+
+	ret
+endfunction
+
 function terminate_handler
 	mov	w21, w0
 	mov	x20, x2
@@ -370,6 +379,30 @@ endfunction
 .globl _start
 function _start
 _start:
+	mov	x23, #0		// Irritation signal count
+
+	mov	w0, #SIGINT
+	adr	x1, terminate_handler
+	mov	w2, #SA_SIGINFO
+	bl	setsignal
+
+	mov	w0, #SIGTERM
+	adr	x1, terminate_handler
+	mov	w2, #SA_SIGINFO
+	bl	setsignal
+
+	mov	w0, #SIGUSR1
+	adr	x1, irritator_handler
+	mov	w2, #SA_SIGINFO
+	orr	w2, w2, #SA_NODEFER
+	bl	setsignal
+
+	mov	w0, #SIGUSR2
+	adr	x1, tickle_handler
+	mov	w2, #SA_SIGINFO
+	orr	w2, w2, #SA_NODEFER
+	bl	setsignal
+
 #ifdef SSVE
 	puts	"Streaming mode "
 	smstart_sm
@@ -405,24 +438,6 @@ _start:
 	mov	x0, x20
 	bl	putdecn
 
-	mov	x23, #0		// Irritation signal count
-
-	mov	w0, #SIGINT
-	adr	x1, terminate_handler
-	mov	w2, #SA_SIGINFO
-	bl	setsignal
-
-	mov	w0, #SIGTERM
-	adr	x1, terminate_handler
-	mov	w2, #SA_SIGINFO
-	bl	setsignal
-
-	mov	w0, #SIGUSR1
-	adr	x1, irritator_handler
-	mov	w2, #SA_SIGINFO
-	orr	w2, w2, #SA_NODEFER
-	bl	setsignal
-
 #ifdef SSVE
 	smstart_sm		// syscalls will have exited streaming mode
 #endif
diff --git a/tools/testing/selftests/arm64/fp/za-test.S b/tools/testing/selftests/arm64/fp/za-test.S
index 9ab6f9cd9623..53c54af65704 100644
--- a/tools/testing/selftests/arm64/fp/za-test.S
+++ b/tools/testing/selftests/arm64/fp/za-test.S
@@ -167,6 +167,15 @@ function irritator_handler
 	ret
 endfunction
 
+function tickle_handler
+	// Increment the signal count (x23):
+	ldr	x0, [x2, #ucontext_regs + 8 * 23]
+	add	x0, x0, #1
+	str	x0, [x2, #ucontext_regs + 8 * 23]
+
+	ret
+endfunction
+
 function terminate_handler
 	mov	w21, w0
 	mov	x20, x2
@@ -223,6 +232,30 @@ endfunction
 .globl _start
 function _start
 _start:
+	mov	x23, #0		// signal count
+
+	mov	w0, #SIGINT
+	adr	x1, terminate_handler
+	mov	w2, #SA_SIGINFO
+	bl	setsignal
+
+	mov	w0, #SIGTERM
+	adr	x1, terminate_handler
+	mov	w2, #SA_SIGINFO
+	bl	setsignal
+
+	mov	w0, #SIGUSR1
+	adr	x1, irritator_handler
+	mov	w2, #SA_SIGINFO
+	orr	w2, w2, #SA_NODEFER
+	bl	setsignal
+
+	mov	w0, #SIGUSR2
+	adr	x1, tickle_handler
+	mov	w2, #SA_SIGINFO
+	orr	w2, w2, #SA_NODEFER
+	bl	setsignal
+
 	puts	"Streaming mode "
 	smstart_za
 
@@ -255,24 +288,6 @@ _start:
 	mov	x0, x20
 	bl	putdecn
 
-	mov	x23, #0		// Irritation signal count
-
-	mov	w0, #SIGINT
-	adr	x1, terminate_handler
-	mov	w2, #SA_SIGINFO
-	bl	setsignal
-
-	mov	w0, #SIGTERM
-	adr	x1, terminate_handler
-	mov	w2, #SA_SIGINFO
-	bl	setsignal
-
-	mov	w0, #SIGUSR1
-	adr	x1, irritator_handler
-	mov	w2, #SA_SIGINFO
-	orr	w2, w2, #SA_NODEFER
-	bl	setsignal
-
 	mov	x22, #0		// generation number, increments per iteration
 .Ltest_loop:
 	rdsvl	0, 8
@@ -287,12 +302,7 @@ _start:
 	subs	x21, x21, #1
 	b.ne	0b
 
-	and	x8, x22, #127		// Every 128 interations...
-	cbz	x8, 0f
-	mov	x8, #__NR_getpid	// (otherwise minimal syscall)
-	b	1f
-0:
-	mov	x8, #__NR_sched_yield	// ...encourage preemption
+	mov	x8, #__NR_sched_yield	// encourage preemption
 1:
 	svc	#0
 
diff --git a/tools/testing/selftests/arm64/mte/Makefile b/tools/testing/selftests/arm64/mte/Makefile
index a5a0744423d8..037046f5784e 100644
--- a/tools/testing/selftests/arm64/mte/Makefile
+++ b/tools/testing/selftests/arm64/mte/Makefile
@@ -11,11 +11,8 @@ LDFLAGS += -pthread
 SRCS := $(filter-out mte_common_util.c,$(wildcard *.c))
 PROGS := $(patsubst %.c,%,$(SRCS))
 
-#Add mte compiler option
-CFLAGS += -march=armv8.5-a+memtag
-
 #check if the compiler works well
-mte_cc_support := $(shell if ($(CC) $(CFLAGS) -E -x c /dev/null -o /dev/null 2>&1) then echo "1"; fi)
+mte_cc_support := $(shell if ($(CC) $(CFLAGS) -march=armv8.5-a+memtag -E -x c /dev/null -o /dev/null 2>&1) then echo "1"; fi)
 
 ifeq ($(mte_cc_support),1)
 # Generated binaries to be installed by top KSFT script
diff --git a/tools/testing/selftests/arm64/mte/mte_helper.S b/tools/testing/selftests/arm64/mte/mte_helper.S
index a02c04cd0aac..a55dbbc56ed1 100644
--- a/tools/testing/selftests/arm64/mte/mte_helper.S
+++ b/tools/testing/selftests/arm64/mte/mte_helper.S
@@ -3,6 +3,8 @@
 
 #include "mte_def.h"
 
+.arch	armv8.5-a+memtag
+
 #define ENTRY(name) \
 	.globl name ;\
 	.p2align 2;\
diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.c b/tools/testing/selftests/arm64/signal/test_signals_utils.c
index b588d10afd5b..308e229e58ab 100644
--- a/tools/testing/selftests/arm64/signal/test_signals_utils.c
+++ b/tools/testing/selftests/arm64/signal/test_signals_utils.c
@@ -165,15 +165,64 @@ static bool handle_signal_ok(struct tdescr *td,
 }
 
 static bool handle_signal_copyctx(struct tdescr *td,
-				  siginfo_t *si, void *uc)
+				  siginfo_t *si, void *uc_in)
 {
+	ucontext_t *uc = uc_in;
+	struct _aarch64_ctx *head;
+	struct extra_context *extra, *copied_extra;
+	size_t offset = 0;
+	size_t to_copy;
+
+	ASSERT_GOOD_CONTEXT(uc);
+
 	/* Mangling PC to avoid loops on original BRK instr */
-	((ucontext_t *)uc)->uc_mcontext.pc += 4;
-	memcpy(td->live_uc, uc, td->live_sz);
-	ASSERT_GOOD_CONTEXT(td->live_uc);
+	uc->uc_mcontext.pc += 4;
+
+	/*
+	 * Check for an preserve any extra data too with fixups.
+	 */
+	head = (struct _aarch64_ctx *)uc->uc_mcontext.__reserved;
+	head = get_header(head, EXTRA_MAGIC, td->live_sz, &offset);
+	if (head) {
+		extra = (struct extra_context *)head;
+
+		/*
+		 * The extra buffer must be immediately after the
+		 * extra_context and a 16 byte terminator. Include it
+		 * in the copy, this was previously validated in
+		 * ASSERT_GOOD_CONTEXT().
+		 */
+		to_copy = offset + sizeof(struct extra_context) + 16 +
+			extra->size;
+		copied_extra = (struct extra_context *)&(td->live_uc->uc_mcontext.__reserved[offset]);
+	} else {
+		copied_extra = NULL;
+		to_copy = sizeof(ucontext_t);
+	}
+
+	if (to_copy > td->live_sz) {
+		fprintf(stderr,
+			"Not enough space to grab context, %lu/%lu bytes\n",
+			td->live_sz, to_copy);
+		return false;
+	}
+
+	memcpy(td->live_uc, uc, to_copy);
+
+	/*
+	 * If there was any EXTRA_CONTEXT fix up the size to be the
+	 * struct extra_context and the following terminator record,
+	 * this means that the rest of the code does not need to have
+	 * special handling for the record and we don't need to fix up
+	 * datap for the new location.
+	 */
+	if (copied_extra)
+		copied_extra->head.size = sizeof(*copied_extra) + 16;
+
 	td->live_uc_valid = 1;
 	fprintf(stderr,
-		"GOOD CONTEXT grabbed from sig_copyctx handler\n");
+		"%lu byte GOOD CONTEXT grabbed from sig_copyctx handler\n",
+		to_copy);
 
 	return true;
 }
diff --git a/tools/testing/selftests/arm64/signal/test_signals_utils.h b/tools/testing/selftests/arm64/signal/test_signals_utils.h
index f3aa99ba67bb..222093f51b67 100644
--- a/tools/testing/selftests/arm64/signal/test_signals_utils.h
+++ b/tools/testing/selftests/arm64/signal/test_signals_utils.h
@@ -56,7 +56,8 @@ static inline bool feats_ok(struct tdescr *td)
  * at sizeof(ucontext_t).
  */
 static __always_inline bool get_current_context(struct tdescr *td,
-						ucontext_t *dest_uc)
+						ucontext_t *dest_uc,
+						size_t dest_sz)
 {
 	static volatile bool seen_already;
 
@@ -64,7 +65,7 @@ static __always_inline bool get_current_context(struct tdescr *td,
 	/* it's a genuine invocation..reinit */
 	seen_already = 0;
 	td->live_uc_valid = 0;
-	td->live_sz = sizeof(*dest_uc);
+	td->live_sz = dest_sz;
 	memset(dest_uc, 0x00, td->live_sz);
 	td->live_uc = dest_uc;
 	/*
diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_magic.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_magic.c
index 8dc600a7d4fd..8c7f00ea9823 100644
--- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_magic.c
+++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_magic.c
@@ -21,7 +21,7 @@ static int fake_sigreturn_bad_magic_run(struct tdescr *td,
 	struct _aarch64_ctx *shead = GET_SF_RESV_HEAD(sf), *head;
 
 	/* just to fill the ucontext_t with something real */
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &sf.uc, sizeof(sf.uc)))
 		return 1;
 
 	/* need at least 2*HDR_SZ space: KSFT_BAD_MAGIC + terminator. */
diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size.c
index b3c362100666..1c03f6b638e0 100644
--- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size.c
+++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size.c
@@ -24,7 +24,7 @@ static int fake_sigreturn_bad_size_run(struct tdescr *td,
 	struct _aarch64_ctx *shead = GET_SF_RESV_HEAD(sf), *head;
 
 	/* just to fill the ucontext_t with something real */
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &sf.uc, sizeof(sf.uc)))
 		return 1;
 
 	resv_sz = GET_SF_RESV_SIZE(sf);
diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size_for_magic0.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size_for_magic0.c
index a44b88bfc81a..bc22f64b544e 100644
--- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size_for_magic0.c
+++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_bad_size_for_magic0.c
@@ -21,7 +21,7 @@ static int fake_sigreturn_bad_size_for_magic0_run(struct tdescr *td,
 	struct _aarch64_ctx *shead = GET_SF_RESV_HEAD(sf), *head;
 
 	/* just to fill the ucontext_t with something real */
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &sf.uc, sizeof(sf.uc)))
 		return 1;
 
 	/* at least HDR_SZ for the badly sized terminator. */
diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_duplicated_fpsimd.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_duplicated_fpsimd.c
index afe8915f0998..63e3906b631c 100644
--- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_duplicated_fpsimd.c
+++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_duplicated_fpsimd.c
@@ -21,7 +21,7 @@ static int fake_sigreturn_duplicated_fpsimd_run(struct tdescr *td,
 	struct _aarch64_ctx *shead = GET_SF_RESV_HEAD(sf), *head;
 
 	/* just to fill the ucontext_t with something real */
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &sf.uc, sizeof(sf.uc)))
 		return 1;
 
 	head = get_starting_head(shead, sizeof(struct fpsimd_context) + HDR_SZ,
diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_misaligned_sp.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_misaligned_sp.c
index 1e089e66f9f3..d00625ff12c2 100644
--- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_misaligned_sp.c
+++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_misaligned_sp.c
@@ -19,7 +19,7 @@ static int fake_sigreturn_misaligned_run(struct tdescr *td,
 					 siginfo_t *si, ucontext_t *uc)
 {
 	/* just to fill the ucontext_t with something real */
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &sf.uc, sizeof(sf.uc)))
 		return 1;
 
 	/* Forcing sigframe on misaligned SP (16 + 3) */
diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_missing_fpsimd.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_missing_fpsimd.c
index 08ecd8073a1a..f805138cb20d 100644
--- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_missing_fpsimd.c
+++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_missing_fpsimd.c
@@ -23,7 +23,7 @@ static int fake_sigreturn_missing_fpsimd_run(struct tdescr *td,
 	struct _aarch64_ctx *head = GET_SF_RESV_HEAD(sf);
 
 	/* just to fill the ucontext_t with something real */
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &sf.uc, sizeof(sf.uc)))
 		return 1;
 
 	resv_sz = GET_SF_RESV_SIZE(sf);
diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sme_change_vl.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sme_change_vl.c
index 7ed762b7202f..ebd5815b54bb 100644
--- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sme_change_vl.c
+++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sme_change_vl.c
@@ -54,7 +54,7 @@ static int fake_sigreturn_ssve_change_vl(struct tdescr *td,
 	struct sve_context *sve;
 
 	/* Get a signal context with a SME ZA frame in it */
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &sf.uc, sizeof(sf.uc)))
 		return 1;
 
 	resv_sz = GET_SF_RESV_SIZE(sf);
diff --git a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sve_change_vl.c b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sve_change_vl.c
index 915821375b0a..e2a452190511 100644
--- a/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sve_change_vl.c
+++ b/tools/testing/selftests/arm64/signal/testcases/fake_sigreturn_sve_change_vl.c
@@ -56,7 +56,7 @@ static int fake_sigreturn_sve_change_vl(struct tdescr *td,
 	struct sve_context *sve;
 
 	/* Get a signal context with a SVE frame in it */
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &sf.uc, sizeof(sf.uc)))
 		return 1;
 
 	resv_sz = GET_SF_RESV_SIZE(sf);
diff --git a/tools/testing/selftests/arm64/signal/testcases/sme_vl.c b/tools/testing/selftests/arm64/signal/testcases/sme_vl.c
index 13ff3b35cbaf..75f387f2db81 100644
--- a/tools/testing/selftests/arm64/signal/testcases/sme_vl.c
+++ b/tools/testing/selftests/arm64/signal/testcases/sme_vl.c
@@ -34,7 +34,7 @@ static int sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc)
 	struct za_context *za;
 
 	/* Get a signal context which should have a ZA frame in it */
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &sf.uc, sizeof(sf.uc)))
 		return 1;
 
 	resv_sz = GET_SF_RESV_SIZE(sf);
diff --git a/tools/testing/selftests/arm64/signal/testcases/ssve_regs.c b/tools/testing/selftests/arm64/signal/testcases/ssve_regs.c
index 9022a6cab4b3..d0a178945b1a 100644
--- a/tools/testing/selftests/arm64/signal/testcases/ssve_regs.c
+++ b/tools/testing/selftests/arm64/signal/testcases/ssve_regs.c
@@ -13,7 +13,10 @@
 #include "test_signals_utils.h"
 #include "testcases.h"
 
-struct fake_sigframe sf;
+static union {
+	ucontext_t uc;
+	char buf[1024 * 64];
+} context;
 static unsigned int vls[SVE_VQ_MAX];
 unsigned int nvls = 0;
 
@@ -55,8 +58,8 @@ static void setup_ssve_regs(void)
 static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc,
 			 unsigned int vl)
 {
-	size_t resv_sz, offset;
-	struct _aarch64_ctx *head = GET_SF_RESV_HEAD(sf);
+	size_t offset;
+	struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context);
 	struct sve_context *ssve;
 	int ret;
 
@@ -73,11 +76,11 @@ static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc,
 	 * in it.
 	 */
 	setup_ssve_regs();
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &context.uc, sizeof(context)))
 		return 1;
 
-	resv_sz = GET_SF_RESV_SIZE(sf);
-	head = get_header(head, SVE_MAGIC, resv_sz, &offset);
+	head = get_header(head, SVE_MAGIC, GET_BUF_RESV_SIZE(context),
+			  &offset);
 	if (!head) {
 		fprintf(stderr, "No SVE context\n");
 		return 1;
@@ -101,16 +104,6 @@ static int sme_regs(struct tdescr *td, siginfo_t *si, ucontext_t *uc)
 	int i;
 
 	for (i = 0; i < nvls; i++) {
-		/*
-		 * TODO: the signal test helpers can't currently cope
-		 * with signal frames bigger than struct sigcontext,
-		 * skip VLs that will trigger that.
-		 */
-		if (vls[i] > 64) {
-			printf("Skipping VL %u due to stack size\n", vls[i]);
-			continue;
-		}
-
 		if (do_one_sme_vl(td, si, uc, vls[i]))
 			return 1;
 	}
diff --git a/tools/testing/selftests/arm64/signal/testcases/sve_regs.c b/tools/testing/selftests/arm64/signal/testcases/sve_regs.c
index 4b2418aa08a9..8b16eabbb769 100644
--- a/tools/testing/selftests/arm64/signal/testcases/sve_regs.c
+++ b/tools/testing/selftests/arm64/signal/testcases/sve_regs.c
@@ -13,7 +13,10 @@
 #include "test_signals_utils.h"
 #include "testcases.h"
 
-struct fake_sigframe sf;
+static union {
+	ucontext_t uc;
+	char buf[1024 * 64];
+} context;
 static unsigned int vls[SVE_VQ_MAX];
 unsigned int nvls = 0;
 
@@ -55,8 +58,8 @@ static void setup_sve_regs(void)
 static int do_one_sve_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc,
 			 unsigned int vl)
 {
-	size_t resv_sz, offset;
-	struct _aarch64_ctx *head = GET_SF_RESV_HEAD(sf);
+	size_t offset;
+	struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context);
 	struct sve_context *sve;
 
 	fprintf(stderr, "Testing VL %d\n", vl);
@@ -71,11 +74,11 @@ static int do_one_sve_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc,
 	 * in it.
 	 */
 	setup_sve_regs();
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &context.uc, sizeof(context)))
 		return 1;
 
-	resv_sz = GET_SF_RESV_SIZE(sf);
-	head = get_header(head, SVE_MAGIC, resv_sz, &offset);
+	head = get_header(head, SVE_MAGIC, GET_BUF_RESV_SIZE(context),
+			  &offset);
 	if (!head) {
 		fprintf(stderr, "No SVE context\n");
 		return 1;
@@ -99,14 +102,6 @@ static int sve_regs(struct tdescr *td, siginfo_t *si, ucontext_t *uc)
 	int i;
 
 	for (i = 0; i < nvls; i++) {
-		/*
-		 * TODO: the signal test helpers can't currently cope
-		 * with signal frames bigger than struct sigcontext,
-		 * skip VLs that will trigger that.
-		 */
-		if (vls[i] > 64)
-			continue;
-
 		if (do_one_sve_vl(td, si, uc, vls[i]))
 			return 1;
 	}
diff --git a/tools/testing/selftests/arm64/signal/testcases/sve_vl.c b/tools/testing/selftests/arm64/signal/testcases/sve_vl.c
index 92904653add1..aa835acec062 100644
--- a/tools/testing/selftests/arm64/signal/testcases/sve_vl.c
+++ b/tools/testing/selftests/arm64/signal/testcases/sve_vl.c
@@ -34,7 +34,7 @@ static int sve_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc)
 	struct sve_context *sve;
 
 	/* Get a signal context which should have a SVE frame in it */
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &sf.uc, sizeof(sf.uc)))
 		return 1;
 
 	resv_sz = GET_SF_RESV_SIZE(sf);
diff --git a/tools/testing/selftests/arm64/signal/testcases/testcases.c b/tools/testing/selftests/arm64/signal/testcases/testcases.c
index 84c36bee4d82..e1c625b20ac4 100644
--- a/tools/testing/selftests/arm64/signal/testcases/testcases.c
+++ b/tools/testing/selftests/arm64/signal/testcases/testcases.c
@@ -25,7 +25,8 @@ struct _aarch64_ctx *get_header(struct _aarch64_ctx *head, uint32_t magic,
 	return found;
 }
 
-bool validate_extra_context(struct extra_context *extra, char **err)
+bool validate_extra_context(struct extra_context *extra, char **err,
+			    void **extra_data, size_t *extra_size)
 {
 	struct _aarch64_ctx *term;
 
@@ -33,7 +34,7 @@ bool validate_extra_context(struct extra_context *extra, char **err)
 		return false;
 
 	fprintf(stderr, "Validating EXTRA...\n");
-	term = GET_RESV_NEXT_HEAD(extra);
+	term = GET_RESV_NEXT_HEAD(&extra->head);
 	if (!term || term->magic || term->size) {
 		*err = "Missing terminator after EXTRA context";
 		return false;
@@ -42,11 +43,14 @@ bool validate_extra_context(struct extra_context *extra, char **err)
 		*err = "Extra DATAP misaligned";
 	else if (extra->size & 0x0fUL)
 		*err = "Extra SIZE misaligned";
-	else if (extra->datap != (uint64_t)term + sizeof(*term))
+	else if (extra->datap != (uint64_t)term + 0x10UL)
 		*err = "Extra DATAP misplaced (not contiguous)";
 	if (*err)
 		return false;
 
+	*extra_data = (void *)extra->datap;
+	*extra_size = extra->size;
+
 	return true;
 }
 
@@ -105,11 +109,14 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err)
 	bool terminated = false;
 	size_t offs = 0;
 	int flags = 0;
+	int new_flags;
 	struct extra_context *extra = NULL;
 	struct sve_context *sve = NULL;
 	struct za_context *za = NULL;
 	struct _aarch64_ctx *head =
 		(struct _aarch64_ctx *)uc->uc_mcontext.__reserved;
+	void *extra_data = NULL;
+	size_t extra_sz = 0;
 
 	if (!err)
 		return false;
@@ -120,12 +127,24 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err)
 			return false;
 		}
 
+		new_flags = 0;
+
 		switch (head->magic) {
 		case 0:
-			if (head->size)
+			if (head->size) {
 				*err = "Bad size for terminator";
-			else
+			} else if (extra_data) {
+				/* End of main data, walking the extra data */
+				head = extra_data;
+				resv_sz = extra_sz;
+				offs = 0;
+
+				extra_data = NULL;
+				extra_sz = 0;
+				continue;
+			} else {
 				terminated = true;
+			}
 			break;
 		case FPSIMD_MAGIC:
 			if (flags & FPSIMD_CTX)
@@ -133,7 +152,7 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err)
 			else if (head->size !=
 				 sizeof(struct fpsimd_context))
 				*err = "Bad size for fpsimd_context";
-			flags |= FPSIMD_CTX;
+			new_flags |= FPSIMD_CTX;
 			break;
 		case ESR_MAGIC:
 			if (head->size != sizeof(struct esr_context))
@@ -144,14 +163,14 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err)
 				*err = "Multiple SVE_MAGIC";
 			/* Size is validated in validate_sve_context() */
 			sve = (struct sve_context *)head;
-			flags |= SVE_CTX;
+			new_flags |= SVE_CTX;
 			break;
 		case ZA_MAGIC:
 			if (flags & ZA_CTX)
 				*err = "Multiple ZA_MAGIC";
 			/* Size is validated in validate_za_context() */
 			za = (struct za_context *)head;
-			flags |= ZA_CTX;
+			new_flags |= ZA_CTX;
 			break;
 		case EXTRA_MAGIC:
 			if (flags & EXTRA_CTX)
@@ -159,7 +178,7 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err)
 			else if (head->size !=
 				 sizeof(struct extra_context))
 				*err = "Bad size for extra_context";
-			flags |= EXTRA_CTX;
+			new_flags |= EXTRA_CTX;
 			extra = (struct extra_context *)head;
 			break;
 		case KSFT_BAD_MAGIC:
@@ -192,16 +211,19 @@ bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err)
 			return false;
 		}
 
-		if (flags & EXTRA_CTX)
-			if (!validate_extra_context(extra, err))
+		if (new_flags & EXTRA_CTX)
+			if (!validate_extra_context(extra, err,
+						    &extra_data, &extra_sz))
 				return false;
-		if (flags & SVE_CTX)
+		if (new_flags & SVE_CTX)
 			if (!validate_sve_context(sve, err))
 				return false;
-		if (flags & ZA_CTX)
+		if (new_flags & ZA_CTX)
 			if (!validate_za_context(za, err))
 				return false;
 
+		flags |= new_flags;
+
 		head = GET_RESV_NEXT_HEAD(head);
 	}
 
diff --git a/tools/testing/selftests/arm64/signal/testcases/testcases.h b/tools/testing/selftests/arm64/signal/testcases/testcases.h
index 49f1d5de7b5b..040afded0b76 100644
--- a/tools/testing/selftests/arm64/signal/testcases/testcases.h
+++ b/tools/testing/selftests/arm64/signal/testcases/testcases.h
@@ -30,6 +30,13 @@
 #define GET_SF_RESV_SIZE(sf) \
 	sizeof((sf).uc.uc_mcontext.__reserved)
 
+#define GET_BUF_RESV_HEAD(buf) \
+	(struct _aarch64_ctx *)(&(buf).uc.uc_mcontext.__reserved)
+
+#define GET_BUF_RESV_SIZE(buf) \
+	(sizeof(buf) - sizeof(buf.uc) +	\
+	 sizeof((buf).uc.uc_mcontext.__reserved))
+
 #define GET_UCP_RESV_SIZE(ucp) \
 	sizeof((ucp)->uc_mcontext.__reserved)
 
@@ -79,8 +86,6 @@ struct fake_sigframe {
 
 bool validate_reserved(ucontext_t *uc, size_t resv_sz, char **err);
 
-bool validate_extra_context(struct extra_context *extra, char **err);
-
 struct _aarch64_ctx *get_header(struct _aarch64_ctx *head, uint32_t magic,
 				size_t resv_sz, size_t *offset);
 
diff --git a/tools/testing/selftests/arm64/signal/testcases/za_no_regs.c b/tools/testing/selftests/arm64/signal/testcases/za_no_regs.c
new file mode 100644
index 000000000000..4d6f94b6178f
--- /dev/null
+++ b/tools/testing/selftests/arm64/signal/testcases/za_no_regs.c
@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2021 ARM Limited
+ *
+ * Verify that the ZA register context in signal frames is set up as
+ * expected.
+ */
+
+#include <signal.h>
+#include <ucontext.h>
+#include <sys/prctl.h>
+
+#include "test_signals_utils.h"
+#include "testcases.h"
+
+static union {
+	ucontext_t uc;
+	char buf[1024 * 128];
+} context;
+static unsigned int vls[SVE_VQ_MAX];
+unsigned int nvls = 0;
+
+static bool sme_get_vls(struct tdescr *td)
+{
+	int vq, vl;
+
+	/*
+	 * Enumerate up to SME_VQ_MAX vector lengths
+	 */
+	for (vq = SVE_VQ_MAX; vq > 0; --vq) {
+		vl = prctl(PR_SME_SET_VL, vq * 16);
+		if (vl == -1)
+			return false;
+
+		vl &= PR_SME_VL_LEN_MASK;
+
+		/* Skip missing VLs */
+		vq = sve_vq_from_vl(vl);
+
+		vls[nvls++] = vl;
+	}
+
+	/* We need at least one VL */
+	if (nvls < 1) {
+		fprintf(stderr, "Only %d VL supported\n", nvls);
+		return false;
+	}
+
+	return true;
+}
+
+static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc,
+			 unsigned int vl)
+{
+	size_t offset;
+	struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context);
+	struct za_context *za;
+
+	fprintf(stderr, "Testing VL %d\n", vl);
+
+	if (prctl(PR_SME_SET_VL, vl) != vl) {
+		fprintf(stderr, "Failed to set VL\n");
+		return 1;
+	}
+
+	/*
+	 * Get a signal context which should have a SVE frame and registers
+	 * in it.
+	 */
+	if (!get_current_context(td, &context.uc, sizeof(context)))
+		return 1;
+
+	head = get_header(head, ZA_MAGIC, GET_BUF_RESV_SIZE(context), &offset);
+	if (!head) {
+		fprintf(stderr, "No ZA context\n");
+		return 1;
+	}
+
+	za = (struct za_context *)head;
+	if (za->vl != vl) {
+		fprintf(stderr, "Got VL %d, expected %d\n", za->vl, vl);
+		return 1;
+	}
+
+	if (head->size != ZA_SIG_REGS_OFFSET) {
+		fprintf(stderr, "Context size %u, expected %lu\n",
+			head->size, ZA_SIG_REGS_OFFSET);
+		return 1;
+	}
+
+	/* The actual size validation is done in get_current_context() */
+	fprintf(stderr, "Got expected size %u and VL %d\n",
+		head->size, za->vl);
+
+	return 0;
+}
+
+static int sme_regs(struct tdescr *td, siginfo_t *si, ucontext_t *uc)
+{
+	int i;
+
+	for (i = 0; i < nvls; i++) {
+		if (do_one_sme_vl(td, si, uc, vls[i]))
+			return 1;
+	}
+
+	td->pass = 1;
+
+	return 0;
+}
+
+struct tdescr tde = {
+	.name = "ZA registers - ZA disabled",
+	.descr = "Check ZA context with ZA disabled",
+	.feats_required = FEAT_SME,
+	.timeout = 3,
+	.init = sme_get_vls,
+	.run = sme_regs,
+};
diff --git a/tools/testing/selftests/arm64/signal/testcases/za_regs.c b/tools/testing/selftests/arm64/signal/testcases/za_regs.c
index b94e4f99fcac..ea45acb115d5 100644
--- a/tools/testing/selftests/arm64/signal/testcases/za_regs.c
+++ b/tools/testing/selftests/arm64/signal/testcases/za_regs.c
@@ -13,7 +13,10 @@
 #include "test_signals_utils.h"
 #include "testcases.h"
 
-struct fake_sigframe sf;
+static union {
+	ucontext_t uc;
+	char buf[1024 * 128];
+} context;
 static unsigned int vls[SVE_VQ_MAX];
 unsigned int nvls = 0;
 
@@ -22,10 +25,10 @@ static bool sme_get_vls(struct tdescr *td)
 	int vq, vl;
 
 	/*
-	 * Enumerate up to SVE_VQ_MAX vector lengths
+	 * Enumerate up to SME_VQ_MAX vector lengths
 	 */
 	for (vq = SVE_VQ_MAX; vq > 0; --vq) {
-		vl = prctl(PR_SVE_SET_VL, vq * 16);
+		vl = prctl(PR_SME_SET_VL, vq * 16);
 		if (vl == -1)
 			return false;
 
@@ -52,11 +55,13 @@ static void setup_za_regs(void)
 	asm volatile(".inst 0xd503457f" : : : );
 }
 
+static char zeros[ZA_SIG_REGS_SIZE(SVE_VQ_MAX)];
+
 static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc,
 			 unsigned int vl)
 {
-	size_t resv_sz, offset;
-	struct _aarch64_ctx *head = GET_SF_RESV_HEAD(sf);
+	size_t offset;
+	struct _aarch64_ctx *head = GET_BUF_RESV_HEAD(context);
 	struct za_context *za;
 
 	fprintf(stderr, "Testing VL %d\n", vl);
@@ -71,11 +76,10 @@ static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc,
 	 * in it.
 	 */
 	setup_za_regs();
-	if (!get_current_context(td, &sf.uc))
+	if (!get_current_context(td, &context.uc, sizeof(context)))
 		return 1;
 
-	resv_sz = GET_SF_RESV_SIZE(sf);
-	head = get_header(head, ZA_MAGIC, resv_sz, &offset);
+	head = get_header(head, ZA_MAGIC, GET_BUF_RESV_SIZE(context), &offset);
 	if (!head) {
 		fprintf(stderr, "No ZA context\n");
 		return 1;
@@ -87,10 +91,22 @@ static int do_one_sme_vl(struct tdescr *td, siginfo_t *si, ucontext_t *uc,
 		return 1;
 	}
 
-	/* The actual size validation is done in get_current_context() */
+	if (head->size != ZA_SIG_CONTEXT_SIZE(sve_vq_from_vl(vl))) {
+		fprintf(stderr, "ZA context size %u, expected %lu\n",
+			head->size, ZA_SIG_CONTEXT_SIZE(sve_vq_from_vl(vl)));
+		return 1;
+	}
+
 	fprintf(stderr, "Got expected size %u and VL %d\n",
 		head->size, za->vl);
 
+	/* We didn't load any data into ZA so it should be all zeros */
+	if (memcmp(zeros, (char *)za + ZA_SIG_REGS_OFFSET,
+		   ZA_SIG_REGS_SIZE(sve_vq_from_vl(za->vl))) != 0) {
+		fprintf(stderr, "ZA data invalid\n");
+		return 1;
+	}
+
 	return 0;
 }
 
@@ -99,16 +115,6 @@ static int sme_regs(struct tdescr *td, siginfo_t *si, ucontext_t *uc)
 	int i;
 
 	for (i = 0; i < nvls; i++) {
-		/*
-		 * TODO: the signal test helpers can't currently cope
-		 * with signal frames bigger than struct sigcontext,
-		 * skip VLs that will trigger that.
-		 */
-		if (vls[i] > 32) {
-			printf("Skipping VL %u due to stack size\n", vls[i]);
-			continue;
-		}
-
 		if (do_one_sme_vl(td, si, uc, vls[i]))
 			return 1;
 	}
diff --git a/tools/testing/selftests/bpf/.gitignore b/tools/testing/selftests/bpf/.gitignore
index 3a8cb2404ea6..07d2d0a8c5cb 100644
--- a/tools/testing/selftests/bpf/.gitignore
+++ b/tools/testing/selftests/bpf/.gitignore
@@ -39,6 +39,8 @@ test_cpp
 /tools
 /runqslower
 /bench
+/veristat
+/sign-file
 *.ko
 *.tmp
 xskxceiver
diff --git a/tools/testing/selftests/bpf/DENYLIST.s390x b/tools/testing/selftests/bpf/DENYLIST.s390x
index db9810611788..17e074eb42b8 100644
--- a/tools/testing/selftests/bpf/DENYLIST.s390x
+++ b/tools/testing/selftests/bpf/DENYLIST.s390x
@@ -43,7 +43,7 @@ test_bpffs                               # bpffs test  failed 255
 test_bprm_opts                           # failed to auto-attach program 'secure_exec': -524                           (trampoline)
 test_ima                                 # failed to auto-attach program 'ima': -524                                   (trampoline)
 test_local_storage                       # failed to auto-attach program 'unlink_hook': -524                           (trampoline)
-test_lsm                                 # failed to find kernel BTF type ID of '__x64_sys_setdomainname': -3          (?)
+test_lsm                                 # attach unexpected error: -524                                               (trampoline)
 test_overhead                            # attach_fentry unexpected error: -524                                        (trampoline)
 test_profiler                            # unknown func bpf_probe_read_str#45                                          (overlapping)
 timer                                    # failed to auto-attach program 'test1': -524                                 (trampoline)
@@ -66,3 +66,12 @@ select_reuseport                         # intermittently fails on new s390x set
 xdp_synproxy                             # JIT does not support calling kernel function                                (kfunc)
 unpriv_bpf_disabled                      # fentry
 lru_bug                                  # prog 'printk': failed to auto-attach: -524
+setget_sockopt                           # attach unexpected error: -524                                               (trampoline)
+cb_refs                                  # expected error message unexpected error: -524                               (trampoline)
+cgroup_hierarchical_stats                # JIT does not support calling kernel function                                (kfunc)
+htab_update                              # failed to attach: ERROR: strerror_r(-524)=22                                (trampoline)
+tracing_struct                           # failed to auto-attach: -524                                                 (trampoline)
+user_ringbuf                             # failed to find kernel BTF type ID of '__s390x_sys_prctl': -3                (?)
+lookup_key                               # JIT does not support calling kernel function                                (kfunc)
+verify_pkcs7_sig                         # JIT does not support calling kernel function                                (kfunc)
+kfunc_dynptr_param                       # JIT does not support calling kernel function                                (kfunc)
diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile
index 8d59ec7f4c2d..e6cf21fad69f 100644
--- a/tools/testing/selftests/bpf/Makefile
+++ b/tools/testing/selftests/bpf/Makefile
@@ -14,6 +14,7 @@ BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool
 APIDIR := $(TOOLSINCDIR)/uapi
 GENDIR := $(abspath ../../../../include/generated)
 GENHDR := $(GENDIR)/autoconf.h
+HOSTPKG_CONFIG := pkg-config
 
 ifneq ($(wildcard $(GENHDR)),)
   GENFLAGS := -DHAVE_GENHDR
@@ -45,7 +46,7 @@ ifneq ($(BPF_GCC),)
 TEST_GEN_PROGS += test_progs-bpf_gcc
 endif
 
-TEST_GEN_FILES = test_lwt_ip_encap.o test_tc_edt.o
+TEST_GEN_FILES = test_lwt_ip_encap.bpf.o test_tc_edt.bpf.o
 TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c)
 
 # Order correspond to 'make run_tests' order
@@ -75,16 +76,17 @@ TEST_PROGS := test_kmod.sh \
 	test_xsk.sh
 
 TEST_PROGS_EXTENDED := with_addr.sh \
-	with_tunnels.sh ima_setup.sh \
+	with_tunnels.sh ima_setup.sh verify_sig_setup.sh \
 	test_xdp_vlan.sh test_bpftool.py
 
 # Compile but not part of 'make run_tests'
 TEST_GEN_PROGS_EXTENDED = test_sock_addr test_skb_cgroup_id_user \
 	flow_dissector_load test_flow_dissector test_tcp_check_syncookie_user \
 	test_lirc_mode2_user xdping test_cpp runqslower bench bpf_testmod.ko \
-	xskxceiver xdp_redirect_multi xdp_synproxy
+	xskxceiver xdp_redirect_multi xdp_synproxy veristat
 
-TEST_CUSTOM_PROGS = $(OUTPUT)/urandom_read
+TEST_CUSTOM_PROGS = $(OUTPUT)/urandom_read $(OUTPUT)/sign-file
+TEST_GEN_FILES += liburandom_read.so
 
 # Emit succinct information message describing current building step
 # $1 - generic step name (e.g., CC, LINK, etc);
@@ -189,6 +191,12 @@ $(OUTPUT)/urandom_read: urandom_read.c urandom_read_aux.c $(OUTPUT)/liburandom_r
 		     -fuse-ld=$(LLD) -Wl,-znoseparate-code		       \
 		     -Wl,-rpath=. -Wl,--build-id=sha1 -o $@
 
+$(OUTPUT)/sign-file: ../../../../scripts/sign-file.c
+	$(call msg,SIGN-FILE,,$@)
+	$(Q)$(CC) $(shell $(HOSTPKG_CONFIG)--cflags libcrypto 2> /dev/null) \
+		  $< -o $@ \
+		  $(shell $(HOSTPKG_CONFIG) --libs libcrypto 2> /dev/null || echo -lcrypto)
+
 $(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(wildcard bpf_testmod/Makefile bpf_testmod/*.[ch])
 	$(call msg,MOD,,$@)
 	$(Q)$(RM) bpf_testmod/bpf_testmod.ko # force re-compilation
@@ -323,6 +331,7 @@ $(OUTPUT)/test_l4lb_noinline.o: BPF_CFLAGS += -fno-inline
 $(OUTPUT)/test_xdp_noinline.o: BPF_CFLAGS += -fno-inline
 
 $(OUTPUT)/flow_dissector_load.o: flow_dissector_load.h
+$(OUTPUT)/cgroup_getset_retval_hooks.o: cgroup_getset_retval_hooks.h
 
 # Build BPF object using Clang
 # $1 - input .c file
@@ -350,24 +359,25 @@ LINKED_SKELS := test_static_linked.skel.h linked_funcs.skel.h		\
 		test_subskeleton.skel.h test_subskeleton_lib.skel.h	\
 		test_usdt.skel.h
 
-LSKELS := kfunc_call_test.c fentry_test.c fexit_test.c fexit_sleep.c \
+LSKELS := fentry_test.c fexit_test.c fexit_sleep.c \
 	test_ringbuf.c atomics.c trace_printk.c trace_vprintk.c \
 	map_ptr_kern.c core_kern.c core_kern_overflow.c
 # Generate both light skeleton and libbpf skeleton for these
-LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test_subprog.c
+LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \
+	kfunc_call_test_subprog.c
 SKEL_BLACKLIST += $$(LSKELS)
 
-test_static_linked.skel.h-deps := test_static_linked1.o test_static_linked2.o
-linked_funcs.skel.h-deps := linked_funcs1.o linked_funcs2.o
-linked_vars.skel.h-deps := linked_vars1.o linked_vars2.o
-linked_maps.skel.h-deps := linked_maps1.o linked_maps2.o
+test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o
+linked_funcs.skel.h-deps := linked_funcs1.bpf.o linked_funcs2.bpf.o
+linked_vars.skel.h-deps := linked_vars1.bpf.o linked_vars2.bpf.o
+linked_maps.skel.h-deps := linked_maps1.bpf.o linked_maps2.bpf.o
 # In the subskeleton case, we want the test_subskeleton_lib.subskel.h file
 # but that's created as a side-effect of the skel.h generation.
-test_subskeleton.skel.h-deps := test_subskeleton_lib2.o test_subskeleton_lib.o test_subskeleton.o
-test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.o test_subskeleton_lib.o
-test_usdt.skel.h-deps := test_usdt.o test_usdt_multispec.o
+test_subskeleton.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o test_subskeleton.bpf.o
+test_subskeleton_lib.skel.h-deps := test_subskeleton_lib2.bpf.o test_subskeleton_lib.bpf.o
+test_usdt.skel.h-deps := test_usdt.bpf.o test_usdt_multispec.bpf.o
 
-LINKED_BPF_SRCS := $(patsubst %.o,%.c,$(foreach skel,$(LINKED_SKELS),$($(skel)-deps)))
+LINKED_BPF_SRCS := $(patsubst %.bpf.o,%.c,$(foreach skel,$(LINKED_SKELS),$($(skel)-deps)))
 
 # Set up extra TRUNNER_XXX "temporary" variables in the environment (relies on
 # $eval()) and pass control to DEFINE_TEST_RUNNER_RULES.
@@ -385,7 +395,7 @@ TRUNNER_EXTRA_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o,		\
 TRUNNER_EXTRA_HDRS := $$(filter %.h,$(TRUNNER_EXTRA_SOURCES))
 TRUNNER_TESTS_HDR := $(TRUNNER_TESTS_DIR)/tests.h
 TRUNNER_BPF_SRCS := $$(notdir $$(wildcard $(TRUNNER_BPF_PROGS_DIR)/*.c))
-TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.o, $$(TRUNNER_BPF_SRCS))
+TRUNNER_BPF_OBJS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.bpf.o, $$(TRUNNER_BPF_SRCS))
 TRUNNER_BPF_SKELS := $$(patsubst %.c,$$(TRUNNER_OUTPUT)/%.skel.h,	\
 				 $$(filter-out $(SKEL_BLACKLIST) $(LINKED_BPF_SRCS),\
 					       $$(TRUNNER_BPF_SRCS)))
@@ -415,7 +425,7 @@ endif
 # input/output directory combination
 ifeq ($($(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs),)
 $(TRUNNER_BPF_PROGS_DIR)$(if $2,-)$2-bpfobjs := y
-$(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.o:				\
+$(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.bpf.o:				\
 		     $(TRUNNER_BPF_PROGS_DIR)/%.c			\
 		     $(TRUNNER_BPF_PROGS_DIR)/*.h			\
 		     $$(INCLUDE_DIR)/vmlinux.h				\
@@ -425,25 +435,25 @@ $(TRUNNER_BPF_OBJS): $(TRUNNER_OUTPUT)/%.o:				\
 	$$(call $(TRUNNER_BPF_BUILD_RULE),$$<,$$@,			\
 					  $(TRUNNER_BPF_CFLAGS))
 
-$(TRUNNER_BPF_SKELS): %.skel.h: %.o $(BPFTOOL) | $(TRUNNER_OUTPUT)
+$(TRUNNER_BPF_SKELS): %.skel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT)
 	$$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@)
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.linked1.o) $$<
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.linked2.o) $$(<:.o=.linked1.o)
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.linked3.o) $$(<:.o=.linked2.o)
 	$(Q)diff $$(<:.o=.linked2.o) $$(<:.o=.linked3.o)
-	$(Q)$$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.o=)) > $$@
-	$(Q)$$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.o=)) > $$(@:.skel.h=.subskel.h)
+	$(Q)$$(BPFTOOL) gen skeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$@
+	$(Q)$$(BPFTOOL) gen subskeleton $$(<:.o=.linked3.o) name $$(notdir $$(<:.bpf.o=)) > $$(@:.skel.h=.subskel.h)
 
-$(TRUNNER_BPF_LSKELS): %.lskel.h: %.o $(BPFTOOL) | $(TRUNNER_OUTPUT)
+$(TRUNNER_BPF_LSKELS): %.lskel.h: %.bpf.o $(BPFTOOL) | $(TRUNNER_OUTPUT)
 	$$(call msg,GEN-SKEL,$(TRUNNER_BINARY),$$@)
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked1.o) $$<
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked2.o) $$(<:.o=.llinked1.o)
 	$(Q)$$(BPFTOOL) gen object $$(<:.o=.llinked3.o) $$(<:.o=.llinked2.o)
 	$(Q)diff $$(<:.o=.llinked2.o) $$(<:.o=.llinked3.o)
-	$(Q)$$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.o=_lskel)) > $$@
+	$(Q)$$(BPFTOOL) gen skeleton -L $$(<:.o=.llinked3.o) name $$(notdir $$(<:.bpf.o=_lskel)) > $$@
 
 $(TRUNNER_BPF_SKELS_LINKED): $(TRUNNER_BPF_OBJS) $(BPFTOOL) | $(TRUNNER_OUTPUT)
-	$$(call msg,LINK-BPF,$(TRUNNER_BINARY),$$(@:.skel.h=.o))
+	$$(call msg,LINK-BPF,$(TRUNNER_BINARY),$$(@:.skel.h=.bpf.o))
 	$(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked1.o) $$(addprefix $(TRUNNER_OUTPUT)/,$$($$(@F)-deps))
 	$(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked2.o) $$(@:.skel.h=.linked1.o)
 	$(Q)$$(BPFTOOL) gen object $$(@:.skel.h=.linked3.o) $$(@:.skel.h=.linked2.o)
@@ -499,7 +509,7 @@ $(OUTPUT)/$(TRUNNER_BINARY): $(TRUNNER_TEST_OBJS)			\
 			     | $(TRUNNER_BINARY)-extras
 	$$(call msg,BINARY,,$$@)
 	$(Q)$$(CC) $$(CFLAGS) $$(filter %.a %.o,$$^) $$(LDLIBS) -o $$@
-	$(Q)$(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.o $$@
+	$(Q)$(RESOLVE_BTFIDS) --btf $(TRUNNER_OUTPUT)/btf_data.bpf.o $$@
 	$(Q)ln -sf $(if $2,..,.)/tools/build/bpftool/bootstrap/bpftool $(if $2,$2/)bpftool
 
 endef
@@ -514,7 +524,8 @@ TRUNNER_EXTRA_SOURCES := test_progs.c cgroup_helpers.c trace_helpers.c	\
 TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read $(OUTPUT)/bpf_testmod.ko	\
 		       $(OUTPUT)/liburandom_read.so			\
 		       $(OUTPUT)/xdp_synproxy				\
-		       ima_setup.sh					\
+		       $(OUTPUT)/sign-file				\
+		       ima_setup.sh verify_sig_setup.sh			\
 		       $(wildcard progs/btf_dump_test_case_*.c)
 TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE
 TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS
@@ -593,6 +604,11 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o \
 	$(call msg,BINARY,,$@)
 	$(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@
 
+$(OUTPUT)/veristat.o: $(BPFOBJ)
+$(OUTPUT)/veristat: $(OUTPUT)/veristat.o
+	$(call msg,BINARY,,$@)
+	$(Q)$(CC) $(CFLAGS) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@
+
 EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(SCRATCH_DIR) $(HOST_SCRATCH_DIR)	\
 	prog_tests/tests.h map_tests/tests.h verifier/tests.h		\
 	feature bpftool							\
diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst
index eb1b7541f39d..d3c6b3da0bb1 100644
--- a/tools/testing/selftests/bpf/README.rst
+++ b/tools/testing/selftests/bpf/README.rst
@@ -126,11 +126,11 @@ available in 10.0.1. The patch is available in llvm 11.0.0 trunk.
 
 __  https://reviews.llvm.org/D78466
 
-bpf_verif_scale/loop6.o test failure with Clang 12
-==================================================
+bpf_verif_scale/loop6.bpf.o test failure with Clang 12
+======================================================
 
 With Clang 12, the following bpf_verif_scale test failed:
-  * ``bpf_verif_scale/loop6.o``
+  * ``bpf_verif_scale/loop6.bpf.o``
 
 The verifier output looks like
 
@@ -245,7 +245,7 @@ See `kernel llvm reloc`_ for more explanation and some examples.
 Using clang 13 to compile old libbpf which has static linker support,
 there will be a compilation failure::
 
-  libbpf: ELF relo #0 in section #6 has unexpected type 2 in .../bpf_tcp_nogpl.o
+  libbpf: ELF relo #0 in section #6 has unexpected type 2 in .../bpf_tcp_nogpl.bpf.o
 
 Here, ``type 2`` refers to new relocation type ``R_BPF_64_ABS64``.
 To fix this issue, user newer libbpf.
diff --git a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c
index 792cb15bac40..a6021d6117b5 100644
--- a/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c
+++ b/tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.c
@@ -18,6 +18,46 @@ typedef int (*func_proto_typedef_nested1)(func_proto_typedef);
 typedef int (*func_proto_typedef_nested2)(func_proto_typedef_nested1);
 
 DEFINE_PER_CPU(int, bpf_testmod_ksym_percpu) = 123;
+long bpf_testmod_test_struct_arg_result;
+
+struct bpf_testmod_struct_arg_1 {
+	int a;
+};
+struct bpf_testmod_struct_arg_2 {
+	long a;
+	long b;
+};
+
+noinline int
+bpf_testmod_test_struct_arg_1(struct bpf_testmod_struct_arg_2 a, int b, int c) {
+	bpf_testmod_test_struct_arg_result = a.a + a.b  + b + c;
+	return bpf_testmod_test_struct_arg_result;
+}
+
+noinline int
+bpf_testmod_test_struct_arg_2(int a, struct bpf_testmod_struct_arg_2 b, int c) {
+	bpf_testmod_test_struct_arg_result = a + b.a + b.b + c;
+	return bpf_testmod_test_struct_arg_result;
+}
+
+noinline int
+bpf_testmod_test_struct_arg_3(int a, int b, struct bpf_testmod_struct_arg_2 c) {
+	bpf_testmod_test_struct_arg_result = a + b + c.a + c.b;
+	return bpf_testmod_test_struct_arg_result;
+}
+
+noinline int
+bpf_testmod_test_struct_arg_4(struct bpf_testmod_struct_arg_1 a, int b,
+			      int c, int d, struct bpf_testmod_struct_arg_2 e) {
+	bpf_testmod_test_struct_arg_result = a.a + b + c + d + e.a + e.b;
+	return bpf_testmod_test_struct_arg_result;
+}
+
+noinline int
+bpf_testmod_test_struct_arg_5(void) {
+	bpf_testmod_test_struct_arg_result = 1;
+	return bpf_testmod_test_struct_arg_result;
+}
 
 noinline void
 bpf_testmod_test_mod_kfunc(int i)
@@ -98,11 +138,19 @@ bpf_testmod_test_read(struct file *file, struct kobject *kobj,
 		.off = off,
 		.len = len,
 	};
+	struct bpf_testmod_struct_arg_1 struct_arg1 = {10};
+	struct bpf_testmod_struct_arg_2 struct_arg2 = {2, 3};
 	int i = 1;
 
 	while (bpf_testmod_return_ptr(i))
 		i++;
 
+	(void)bpf_testmod_test_struct_arg_1(struct_arg2, 1, 4);
+	(void)bpf_testmod_test_struct_arg_2(1, struct_arg2, 4);
+	(void)bpf_testmod_test_struct_arg_3(1, 4, struct_arg2);
+	(void)bpf_testmod_test_struct_arg_4(struct_arg1, 1, 2, 3, struct_arg2);
+	(void)bpf_testmod_test_struct_arg_5();
+
 	/* This is always true. Use the check to make sure the compiler
 	 * doesn't remove bpf_testmod_loop_test.
 	 */
diff --git a/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h b/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h
new file mode 100644
index 000000000000..a525d3544fd7
--- /dev/null
+++ b/tools/testing/selftests/bpf/cgroup_getset_retval_hooks.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+BPF_RETVAL_HOOK(ingress, "cgroup_skb/ingress", __sk_buff, -EINVAL)
+BPF_RETVAL_HOOK(egress, "cgroup_skb/egress", __sk_buff, -EINVAL)
+BPF_RETVAL_HOOK(sock_create, "cgroup/sock_create", bpf_sock, 0)
+BPF_RETVAL_HOOK(sock_ops, "sockops", bpf_sock_ops, -EINVAL)
+BPF_RETVAL_HOOK(dev, "cgroup/dev", bpf_cgroup_dev_ctx, 0)
+BPF_RETVAL_HOOK(bind4, "cgroup/bind4", bpf_sock_addr, 0)
+BPF_RETVAL_HOOK(bind6, "cgroup/bind6", bpf_sock_addr, 0)
+BPF_RETVAL_HOOK(connect4, "cgroup/connect4", bpf_sock_addr, 0)
+BPF_RETVAL_HOOK(connect6, "cgroup/connect6", bpf_sock_addr, 0)
+BPF_RETVAL_HOOK(post_bind4, "cgroup/post_bind4", bpf_sock_addr, 0)
+BPF_RETVAL_HOOK(post_bind6, "cgroup/post_bind6", bpf_sock_addr, 0)
+BPF_RETVAL_HOOK(sendmsg4, "cgroup/sendmsg4", bpf_sock_addr, 0)
+BPF_RETVAL_HOOK(sendmsg6, "cgroup/sendmsg6", bpf_sock_addr, 0)
+BPF_RETVAL_HOOK(sysctl, "cgroup/sysctl", bpf_sysctl, 0)
+BPF_RETVAL_HOOK(recvmsg4, "cgroup/recvmsg4", bpf_sock_addr, -EINVAL)
+BPF_RETVAL_HOOK(recvmsg6, "cgroup/recvmsg6", bpf_sock_addr, -EINVAL)
+BPF_RETVAL_HOOK(getsockopt, "cgroup/getsockopt", bpf_sockopt, 0)
+BPF_RETVAL_HOOK(setsockopt, "cgroup/setsockopt", bpf_sockopt, 0)
+BPF_RETVAL_HOOK(getpeername4, "cgroup/getpeername4", bpf_sock_addr, -EINVAL)
+BPF_RETVAL_HOOK(getpeername6, "cgroup/getpeername6", bpf_sock_addr, -EINVAL)
+BPF_RETVAL_HOOK(getsockname4, "cgroup/getsockname4", bpf_sock_addr, -EINVAL)
+BPF_RETVAL_HOOK(getsockname6, "cgroup/getsockname6", bpf_sock_addr, -EINVAL)
+BPF_RETVAL_HOOK(sock_release, "cgroup/sock_release", bpf_sock, 0)
diff --git a/tools/testing/selftests/bpf/cgroup_helpers.c b/tools/testing/selftests/bpf/cgroup_helpers.c
index 9d59c3990ca8..e914cc45b766 100644
--- a/tools/testing/selftests/bpf/cgroup_helpers.c
+++ b/tools/testing/selftests/bpf/cgroup_helpers.c
@@ -33,49 +33,52 @@
 #define CGROUP_MOUNT_DFLT		"/sys/fs/cgroup"
 #define NETCLS_MOUNT_PATH		CGROUP_MOUNT_DFLT "/net_cls"
 #define CGROUP_WORK_DIR			"/cgroup-test-work-dir"
-#define format_cgroup_path(buf, path) \
+
+#define format_cgroup_path_pid(buf, path, pid) \
 	snprintf(buf, sizeof(buf), "%s%s%d%s", CGROUP_MOUNT_PATH, \
-	CGROUP_WORK_DIR, getpid(), path)
+	CGROUP_WORK_DIR, pid, path)
+
+#define format_cgroup_path(buf, path) \
+	format_cgroup_path_pid(buf, path, getpid())
+
+#define format_parent_cgroup_path(buf, path) \
+	format_cgroup_path_pid(buf, path, getppid())
 
 #define format_classid_path(buf)				\
 	snprintf(buf, sizeof(buf), "%s%s", NETCLS_MOUNT_PATH,	\
 		 CGROUP_WORK_DIR)
 
-/**
- * enable_all_controllers() - Enable all available cgroup v2 controllers
- *
- * Enable all available cgroup v2 controllers in order to increase
- * the code coverage.
- *
- * If successful, 0 is returned.
- */
-static int enable_all_controllers(char *cgroup_path)
+static int __enable_controllers(const char *cgroup_path, const char *controllers)
 {
 	char path[PATH_MAX + 1];
-	char buf[PATH_MAX];
+	char enable[PATH_MAX + 1];
 	char *c, *c2;
 	int fd, cfd;
 	ssize_t len;
 
-	snprintf(path, sizeof(path), "%s/cgroup.controllers", cgroup_path);
-	fd = open(path, O_RDONLY);
-	if (fd < 0) {
-		log_err("Opening cgroup.controllers: %s", path);
-		return 1;
-	}
-
-	len = read(fd, buf, sizeof(buf) - 1);
-	if (len < 0) {
+	/* If not controllers are passed, enable all available controllers */
+	if (!controllers) {
+		snprintf(path, sizeof(path), "%s/cgroup.controllers",
+			 cgroup_path);
+		fd = open(path, O_RDONLY);
+		if (fd < 0) {
+			log_err("Opening cgroup.controllers: %s", path);
+			return 1;
+		}
+		len = read(fd, enable, sizeof(enable) - 1);
+		if (len < 0) {
+			close(fd);
+			log_err("Reading cgroup.controllers: %s", path);
+			return 1;
+		} else if (len == 0) { /* No controllers to enable */
+			close(fd);
+			return 0;
+		}
+		enable[len] = 0;
 		close(fd);
-		log_err("Reading cgroup.controllers: %s", path);
-		return 1;
+	} else {
+		strncpy(enable, controllers, sizeof(enable));
 	}
-	buf[len] = 0;
-	close(fd);
-
-	/* No controllers available? We're probably on cgroup v1. */
-	if (len == 0)
-		return 0;
 
 	snprintf(path, sizeof(path), "%s/cgroup.subtree_control", cgroup_path);
 	cfd = open(path, O_RDWR);
@@ -84,7 +87,7 @@ static int enable_all_controllers(char *cgroup_path)
 		return 1;
 	}
 
-	for (c = strtok_r(buf, " ", &c2); c; c = strtok_r(NULL, " ", &c2)) {
+	for (c = strtok_r(enable, " ", &c2); c; c = strtok_r(NULL, " ", &c2)) {
 		if (dprintf(cfd, "+%s\n", c) <= 0) {
 			log_err("Enabling controller %s: %s", c, path);
 			close(cfd);
@@ -96,6 +99,87 @@ static int enable_all_controllers(char *cgroup_path)
 }
 
 /**
+ * enable_controllers() - Enable cgroup v2 controllers
+ * @relative_path: The cgroup path, relative to the workdir
+ * @controllers: List of controllers to enable in cgroup.controllers format
+ *
+ *
+ * Enable given cgroup v2 controllers, if @controllers is NULL, enable all
+ * available controllers.
+ *
+ * If successful, 0 is returned.
+ */
+int enable_controllers(const char *relative_path, const char *controllers)
+{
+	char cgroup_path[PATH_MAX + 1];
+
+	format_cgroup_path(cgroup_path, relative_path);
+	return __enable_controllers(cgroup_path, controllers);
+}
+
+static int __write_cgroup_file(const char *cgroup_path, const char *file,
+			       const char *buf)
+{
+	char file_path[PATH_MAX + 1];
+	int fd;
+
+	snprintf(file_path, sizeof(file_path), "%s/%s", cgroup_path, file);
+	fd = open(file_path, O_RDWR);
+	if (fd < 0) {
+		log_err("Opening %s", file_path);
+		return 1;
+	}
+
+	if (dprintf(fd, "%s", buf) <= 0) {
+		log_err("Writing to %s", file_path);
+		close(fd);
+		return 1;
+	}
+	close(fd);
+	return 0;
+}
+
+/**
+ * write_cgroup_file() - Write to a cgroup file
+ * @relative_path: The cgroup path, relative to the workdir
+ * @file: The name of the file in cgroupfs to write to
+ * @buf: Buffer to write to the file
+ *
+ * Write to a file in the given cgroup's directory.
+ *
+ * If successful, 0 is returned.
+ */
+int write_cgroup_file(const char *relative_path, const char *file,
+		      const char *buf)
+{
+	char cgroup_path[PATH_MAX - 24];
+
+	format_cgroup_path(cgroup_path, relative_path);
+	return __write_cgroup_file(cgroup_path, file, buf);
+}
+
+/**
+ * write_cgroup_file_parent() - Write to a cgroup file in the parent process
+ *                              workdir
+ * @relative_path: The cgroup path, relative to the parent process workdir
+ * @file: The name of the file in cgroupfs to write to
+ * @buf: Buffer to write to the file
+ *
+ * Write to a file in the given cgroup's directory under the parent process
+ * workdir.
+ *
+ * If successful, 0 is returned.
+ */
+int write_cgroup_file_parent(const char *relative_path, const char *file,
+			     const char *buf)
+{
+	char cgroup_path[PATH_MAX - 24];
+
+	format_parent_cgroup_path(cgroup_path, relative_path);
+	return __write_cgroup_file(cgroup_path, file, buf);
+}
+
+/**
  * setup_cgroup_environment() - Setup the cgroup environment
  *
  * After calling this function, cleanup_cgroup_environment should be called
@@ -133,7 +217,9 @@ int setup_cgroup_environment(void)
 		return 1;
 	}
 
-	if (enable_all_controllers(cgroup_workdir))
+	/* Enable all available controllers to increase test coverage */
+	if (__enable_controllers(CGROUP_MOUNT_PATH, NULL) ||
+	    __enable_controllers(cgroup_workdir, NULL))
 		return 1;
 
 	return 0;
@@ -173,7 +259,7 @@ static int join_cgroup_from_top(const char *cgroup_path)
 
 /**
  * join_cgroup() - Join a cgroup
- * @path: The cgroup path, relative to the workdir, to join
+ * @relative_path: The cgroup path, relative to the workdir, to join
  *
  * This function expects a cgroup to already be created, relative to the cgroup
  * work dir, and it joins it. For example, passing "/my-cgroup" as the path
@@ -182,11 +268,27 @@ static int join_cgroup_from_top(const char *cgroup_path)
  *
  * On success, it returns 0, otherwise on failure it returns 1.
  */
-int join_cgroup(const char *path)
+int join_cgroup(const char *relative_path)
+{
+	char cgroup_path[PATH_MAX + 1];
+
+	format_cgroup_path(cgroup_path, relative_path);
+	return join_cgroup_from_top(cgroup_path);
+}
+
+/**
+ * join_parent_cgroup() - Join a cgroup in the parent process workdir
+ * @relative_path: The cgroup path, relative to parent process workdir, to join
+ *
+ * See join_cgroup().
+ *
+ * On success, it returns 0, otherwise on failure it returns 1.
+ */
+int join_parent_cgroup(const char *relative_path)
 {
 	char cgroup_path[PATH_MAX + 1];
 
-	format_cgroup_path(cgroup_path, path);
+	format_parent_cgroup_path(cgroup_path, relative_path);
 	return join_cgroup_from_top(cgroup_path);
 }
 
@@ -213,8 +315,26 @@ void cleanup_cgroup_environment(void)
 }
 
 /**
+ * get_root_cgroup() - Get the FD of the root cgroup
+ *
+ * On success, it returns the file descriptor. On failure, it returns -1.
+ * If there is a failure, it prints the error to stderr.
+ */
+int get_root_cgroup(void)
+{
+	int fd;
+
+	fd = open(CGROUP_MOUNT_PATH, O_RDONLY);
+	if (fd < 0) {
+		log_err("Opening root cgroup");
+		return -1;
+	}
+	return fd;
+}
+
+/**
  * create_and_get_cgroup() - Create a cgroup, relative to workdir, and get the FD
- * @path: The cgroup path, relative to the workdir, to join
+ * @relative_path: The cgroup path, relative to the workdir, to join
  *
  * This function creates a cgroup under the top level workdir and returns the
  * file descriptor. It is idempotent.
@@ -222,14 +342,14 @@ void cleanup_cgroup_environment(void)
  * On success, it returns the file descriptor. On failure it returns -1.
  * If there is a failure, it prints the error to stderr.
  */
-int create_and_get_cgroup(const char *path)
+int create_and_get_cgroup(const char *relative_path)
 {
 	char cgroup_path[PATH_MAX + 1];
 	int fd;
 
-	format_cgroup_path(cgroup_path, path);
+	format_cgroup_path(cgroup_path, relative_path);
 	if (mkdir(cgroup_path, 0777) && errno != EEXIST) {
-		log_err("mkdiring cgroup %s .. %s", path, cgroup_path);
+		log_err("mkdiring cgroup %s .. %s", relative_path, cgroup_path);
 		return -1;
 	}
 
@@ -244,13 +364,13 @@ int create_and_get_cgroup(const char *path)
 
 /**
  * get_cgroup_id() - Get cgroup id for a particular cgroup path
- * @path: The cgroup path, relative to the workdir, to join
+ * @relative_path: The cgroup path, relative to the workdir, to join
  *
  * On success, it returns the cgroup id. On failure it returns 0,
  * which is an invalid cgroup id.
  * If there is a failure, it prints the error to stderr.
  */
-unsigned long long get_cgroup_id(const char *path)
+unsigned long long get_cgroup_id(const char *relative_path)
 {
 	int dirfd, err, flags, mount_id, fhsize;
 	union {
@@ -261,7 +381,7 @@ unsigned long long get_cgroup_id(const char *path)
 	struct file_handle *fhp, *fhp2;
 	unsigned long long ret = 0;
 
-	format_cgroup_path(cgroup_workdir, path);
+	format_cgroup_path(cgroup_workdir, relative_path);
 
 	dirfd = AT_FDCWD;
 	flags = 0;
diff --git a/tools/testing/selftests/bpf/cgroup_helpers.h b/tools/testing/selftests/bpf/cgroup_helpers.h
index fcc9cb91b211..3358734356ab 100644
--- a/tools/testing/selftests/bpf/cgroup_helpers.h
+++ b/tools/testing/selftests/bpf/cgroup_helpers.h
@@ -10,11 +10,18 @@
 	__FILE__, __LINE__, clean_errno(), ##__VA_ARGS__)
 
 /* cgroupv2 related */
-int cgroup_setup_and_join(const char *path);
-int create_and_get_cgroup(const char *path);
-unsigned long long get_cgroup_id(const char *path);
-
-int join_cgroup(const char *path);
+int enable_controllers(const char *relative_path, const char *controllers);
+int write_cgroup_file(const char *relative_path, const char *file,
+		      const char *buf);
+int write_cgroup_file_parent(const char *relative_path, const char *file,
+			     const char *buf);
+int cgroup_setup_and_join(const char *relative_path);
+int get_root_cgroup(void);
+int create_and_get_cgroup(const char *relative_path);
+unsigned long long get_cgroup_id(const char *relative_path);
+
+int join_cgroup(const char *relative_path);
+int join_parent_cgroup(const char *relative_path);
 
 int setup_cgroup_environment(void);
 void cleanup_cgroup_environment(void);
@@ -26,4 +33,4 @@ int join_classid(void);
 int setup_classid_environment(void);
 void cleanup_classid_environment(void);
 
-#endif /* __CGROUP_HELPERS_H */
\ No newline at end of file
+#endif /* __CGROUP_HELPERS_H */
diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config
index fabf0c014349..9213565c0311 100644
--- a/tools/testing/selftests/bpf/config
+++ b/tools/testing/selftests/bpf/config
@@ -7,9 +7,9 @@ CONFIG_BPF_LSM=y
 CONFIG_BPF_STREAM_PARSER=y
 CONFIG_BPF_SYSCALL=y
 CONFIG_CGROUP_BPF=y
-CONFIG_CRYPTO_HMAC=m
-CONFIG_CRYPTO_SHA256=m
-CONFIG_CRYPTO_USER_API_HASH=m
+CONFIG_CRYPTO_HMAC=y
+CONFIG_CRYPTO_SHA256=y
+CONFIG_CRYPTO_USER_API_HASH=y
 CONFIG_DYNAMIC_FTRACE=y
 CONFIG_FPROBE=y
 CONFIG_FTRACE_SYSCALLS=y
@@ -24,41 +24,50 @@ CONFIG_IP_NF_FILTER=y
 CONFIG_IP_NF_RAW=y
 CONFIG_IP_NF_TARGET_SYNPROXY=y
 CONFIG_IPV6=y
-CONFIG_IPV6_FOU=m
-CONFIG_IPV6_FOU_TUNNEL=m
+CONFIG_IPV6_FOU=y
+CONFIG_IPV6_FOU_TUNNEL=y
 CONFIG_IPV6_GRE=y
 CONFIG_IPV6_SEG6_BPF=y
-CONFIG_IPV6_SIT=m
+CONFIG_IPV6_SIT=y
 CONFIG_IPV6_TUNNEL=y
+CONFIG_KEYS=y
 CONFIG_LIRC=y
 CONFIG_LWTUNNEL=y
+CONFIG_MODULE_SIG=y
+CONFIG_MODULE_SRCVERSION_ALL=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_MODULES=y
+CONFIG_MODVERSIONS=y
 CONFIG_MPLS=y
-CONFIG_MPLS_IPTUNNEL=m
-CONFIG_MPLS_ROUTING=m
+CONFIG_MPLS_IPTUNNEL=y
+CONFIG_MPLS_ROUTING=y
 CONFIG_MPTCP=y
 CONFIG_NET_CLS_ACT=y
 CONFIG_NET_CLS_BPF=y
-CONFIG_NET_CLS_FLOWER=m
-CONFIG_NET_FOU=m
+CONFIG_NET_CLS_FLOWER=y
+CONFIG_NET_FOU=y
 CONFIG_NET_FOU_IP_TUNNELS=y
 CONFIG_NET_IPGRE=y
 CONFIG_NET_IPGRE_DEMUX=y
 CONFIG_NET_IPIP=y
-CONFIG_NET_MPLS_GSO=m
+CONFIG_NET_MPLS_GSO=y
 CONFIG_NET_SCH_INGRESS=y
 CONFIG_NET_SCHED=y
-CONFIG_NETDEVSIM=m
+CONFIG_NETDEVSIM=y
 CONFIG_NETFILTER=y
 CONFIG_NETFILTER_SYNPROXY=y
+CONFIG_NETFILTER_XT_CONNMARK=y
 CONFIG_NETFILTER_XT_MATCH_STATE=y
 CONFIG_NETFILTER_XT_TARGET_CT=y
 CONFIG_NF_CONNTRACK=y
+CONFIG_NF_CONNTRACK_MARK=y
 CONFIG_NF_DEFRAG_IPV4=y
 CONFIG_NF_DEFRAG_IPV6=y
+CONFIG_NF_NAT=y
 CONFIG_RC_CORE=y
 CONFIG_SECURITY=y
 CONFIG_SECURITYFS=y
-CONFIG_TEST_BPF=m
+CONFIG_TEST_BPF=y
 CONFIG_USERFAULTFD=y
 CONFIG_VXLAN=y
 CONFIG_XDP_SOCKETS=y
diff --git a/tools/testing/selftests/bpf/config.x86_64 b/tools/testing/selftests/bpf/config.x86_64
index f0859a1d37ab..21ce5ea4304e 100644
--- a/tools/testing/selftests/bpf/config.x86_64
+++ b/tools/testing/selftests/bpf/config.x86_64
@@ -47,7 +47,7 @@ CONFIG_CPU_IDLE_GOV_LADDER=y
 CONFIG_CPUSETS=y
 CONFIG_CRC_T10DIF=y
 CONFIG_CRYPTO_BLAKE2B=y
-CONFIG_CRYPTO_DEV_VIRTIO=m
+CONFIG_CRYPTO_DEV_VIRTIO=y
 CONFIG_CRYPTO_SEQIV=y
 CONFIG_CRYPTO_XXHASH=y
 CONFIG_DCB=y
@@ -145,11 +145,6 @@ CONFIG_MCORE2=y
 CONFIG_MEMCG=y
 CONFIG_MEMORY_FAILURE=y
 CONFIG_MINIX_SUBPARTITION=y
-CONFIG_MODULE_SIG=y
-CONFIG_MODULE_SRCVERSION_ALL=y
-CONFIG_MODULE_UNLOAD=y
-CONFIG_MODULES=y
-CONFIG_MODVERSIONS=y
 CONFIG_NAMESPACES=y
 CONFIG_NET=y
 CONFIG_NET_9P=y
diff --git a/tools/testing/selftests/bpf/get_cgroup_id_user.c b/tools/testing/selftests/bpf/get_cgroup_id_user.c
index e021cc67dc02..156743cf5870 100644
--- a/tools/testing/selftests/bpf/get_cgroup_id_user.c
+++ b/tools/testing/selftests/bpf/get_cgroup_id_user.c
@@ -48,7 +48,7 @@ static int bpf_find_map(const char *test, struct bpf_object *obj,
 int main(int argc, char **argv)
 {
 	const char *probe_name = "syscalls/sys_enter_nanosleep";
-	const char *file = "get_cgroup_id_kern.o";
+	const char *file = "get_cgroup_id_kern.bpf.o";
 	int err, bytes, efd, prog_fd, pmu_fd;
 	int cgroup_fd, cgidmap_fd, pidmap_fd;
 	struct perf_event_attr attr = {};
diff --git a/tools/testing/selftests/bpf/map_tests/array_map_batch_ops.c b/tools/testing/selftests/bpf/map_tests/array_map_batch_ops.c
index 78c76496b14a..b595556315bc 100644
--- a/tools/testing/selftests/bpf/map_tests/array_map_batch_ops.c
+++ b/tools/testing/selftests/bpf/map_tests/array_map_batch_ops.c
@@ -3,6 +3,7 @@
 #include <stdio.h>
 #include <errno.h>
 #include <string.h>
+#include <unistd.h>
 
 #include <bpf/bpf.h>
 #include <bpf/libbpf.h>
@@ -137,6 +138,7 @@ static void __test_map_lookup_and_update_batch(bool is_pcpu)
 	free(keys);
 	free(values);
 	free(visited);
+	close(map_fd);
 }
 
 static void array_map_batch_ops(void)
diff --git a/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c b/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c
index f807d53fd8dd..1230ccf90128 100644
--- a/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c
+++ b/tools/testing/selftests/bpf/map_tests/htab_map_batch_ops.c
@@ -3,6 +3,7 @@
 #include <stdio.h>
 #include <errno.h>
 #include <string.h>
+#include <unistd.h>
 
 #include <bpf/bpf.h>
 #include <bpf/libbpf.h>
@@ -255,6 +256,7 @@ void __test_map_lookup_and_delete_batch(bool is_pcpu)
 	free(visited);
 	if (!is_pcpu)
 		free(values);
+	close(map_fd);
 }
 
 void htab_map_batch_ops(void)
diff --git a/tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c b/tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c
index 87d07b596e17..b66d56ddb7ef 100644
--- a/tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c
+++ b/tools/testing/selftests/bpf/map_tests/lpm_trie_map_batch_ops.c
@@ -7,6 +7,7 @@
 #include <errno.h>
 #include <string.h>
 #include <stdlib.h>
+#include <unistd.h>
 
 #include <bpf/bpf.h>
 #include <bpf/libbpf.h>
@@ -150,4 +151,5 @@ void test_lpm_trie_map_batch_ops(void)
 	free(keys);
 	free(values);
 	free(visited);
+	close(map_fd);
 }
diff --git a/tools/testing/selftests/bpf/map_tests/task_storage_map.c b/tools/testing/selftests/bpf/map_tests/task_storage_map.c
new file mode 100644
index 000000000000..7d050364efca
--- /dev/null
+++ b/tools/testing/selftests/bpf/map_tests/task_storage_map.c
@@ -0,0 +1,127 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2022. Huawei Technologies Co., Ltd */
+#define _GNU_SOURCE
+#include <sched.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <string.h>
+#include <pthread.h>
+
+#include <bpf/bpf.h>
+#include <bpf/libbpf.h>
+
+#include "test_maps.h"
+#include "task_local_storage_helpers.h"
+#include "read_bpf_task_storage_busy.skel.h"
+
+struct lookup_ctx {
+	bool start;
+	bool stop;
+	int pid_fd;
+	int map_fd;
+	int loop;
+};
+
+static void *lookup_fn(void *arg)
+{
+	struct lookup_ctx *ctx = arg;
+	long value;
+	int i = 0;
+
+	while (!ctx->start)
+		usleep(1);
+
+	while (!ctx->stop && i++ < ctx->loop)
+		bpf_map_lookup_elem(ctx->map_fd, &ctx->pid_fd, &value);
+	return NULL;
+}
+
+static void abort_lookup(struct lookup_ctx *ctx, pthread_t *tids, unsigned int nr)
+{
+	unsigned int i;
+
+	ctx->stop = true;
+	ctx->start = true;
+	for (i = 0; i < nr; i++)
+		pthread_join(tids[i], NULL);
+}
+
+void test_task_storage_map_stress_lookup(void)
+{
+#define MAX_NR_THREAD 4096
+	unsigned int i, nr = 256, loop = 8192, cpu = 0;
+	struct read_bpf_task_storage_busy *skel;
+	pthread_t tids[MAX_NR_THREAD];
+	struct lookup_ctx ctx;
+	cpu_set_t old, new;
+	const char *cfg;
+	int err;
+
+	cfg = getenv("TASK_STORAGE_MAP_NR_THREAD");
+	if (cfg) {
+		nr = atoi(cfg);
+		if (nr > MAX_NR_THREAD)
+			nr = MAX_NR_THREAD;
+	}
+	cfg = getenv("TASK_STORAGE_MAP_NR_LOOP");
+	if (cfg)
+		loop = atoi(cfg);
+	cfg = getenv("TASK_STORAGE_MAP_PIN_CPU");
+	if (cfg)
+		cpu = atoi(cfg);
+
+	skel = read_bpf_task_storage_busy__open_and_load();
+	err = libbpf_get_error(skel);
+	CHECK(err, "open_and_load", "error %d\n", err);
+
+	/* Only for a fully preemptible kernel */
+	if (!skel->kconfig->CONFIG_PREEMPT) {
+		printf("%s SKIP (no CONFIG_PREEMPT)\n", __func__);
+		read_bpf_task_storage_busy__destroy(skel);
+		skips++;
+		return;
+	}
+
+	/* Save the old affinity setting */
+	sched_getaffinity(getpid(), sizeof(old), &old);
+
+	/* Pinned on a specific CPU */
+	CPU_ZERO(&new);
+	CPU_SET(cpu, &new);
+	sched_setaffinity(getpid(), sizeof(new), &new);
+
+	ctx.start = false;
+	ctx.stop = false;
+	ctx.pid_fd = sys_pidfd_open(getpid(), 0);
+	ctx.map_fd = bpf_map__fd(skel->maps.task);
+	ctx.loop = loop;
+	for (i = 0; i < nr; i++) {
+		err = pthread_create(&tids[i], NULL, lookup_fn, &ctx);
+		if (err) {
+			abort_lookup(&ctx, tids, i);
+			CHECK(err, "pthread_create", "error %d\n", err);
+			goto out;
+		}
+	}
+
+	ctx.start = true;
+	for (i = 0; i < nr; i++)
+		pthread_join(tids[i], NULL);
+
+	skel->bss->pid = getpid();
+	err = read_bpf_task_storage_busy__attach(skel);
+	CHECK(err, "attach", "error %d\n", err);
+
+	/* Trigger program */
+	syscall(SYS_gettid);
+	skel->bss->pid = 0;
+
+	CHECK(skel->bss->busy != 0, "bad bpf_task_storage_busy", "got %d\n", skel->bss->busy);
+out:
+	read_bpf_task_storage_busy__destroy(skel);
+	/* Restore affinity setting */
+	sched_setaffinity(getpid(), sizeof(old), &old);
+	printf("%s:PASS\n", __func__);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/attach_probe.c b/tools/testing/selftests/bpf/prog_tests/attach_probe.c
index 0b899d2d8ea7..9566d9d2f6ee 100644
--- a/tools/testing/selftests/bpf/prog_tests/attach_probe.c
+++ b/tools/testing/selftests/bpf/prog_tests/attach_probe.c
@@ -6,19 +6,19 @@
 volatile unsigned short uprobe_ref_ctr __attribute__((unused)) __attribute((section(".probes")));
 
 /* uprobe attach point */
-static void trigger_func(void)
+static noinline void trigger_func(void)
 {
 	asm volatile ("");
 }
 
 /* attach point for byname uprobe */
-static void trigger_func2(void)
+static noinline void trigger_func2(void)
 {
 	asm volatile ("");
 }
 
 /* attach point for byname sleepable uprobe */
-static void trigger_func3(void)
+static noinline void trigger_func3(void)
 {
 	asm volatile ("");
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/autoattach.c b/tools/testing/selftests/bpf/prog_tests/autoattach.c
new file mode 100644
index 000000000000..dc5e01d279bd
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/autoattach.c
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Google */
+
+#include <test_progs.h>
+#include "test_autoattach.skel.h"
+
+void test_autoattach(void)
+{
+	struct test_autoattach *skel;
+
+	skel = test_autoattach__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel_open_and_load"))
+		goto cleanup;
+
+	/* disable auto-attach for prog2 */
+	bpf_program__set_autoattach(skel->progs.prog2, false);
+	ASSERT_TRUE(bpf_program__autoattach(skel->progs.prog1), "autoattach_prog1");
+	ASSERT_FALSE(bpf_program__autoattach(skel->progs.prog2), "autoattach_prog2");
+	if (!ASSERT_OK(test_autoattach__attach(skel), "skel_attach"))
+		goto cleanup;
+
+	usleep(1);
+
+	ASSERT_TRUE(skel->bss->prog1_called, "attached_prog1");
+	ASSERT_FALSE(skel->bss->prog2_called, "attached_prog2");
+
+cleanup:
+	test_autoattach__destroy(skel);
+}
+
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c b/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c
index 2974b44f80fa..2be2d61954bc 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_cookie.c
@@ -13,7 +13,7 @@
 #include "kprobe_multi.skel.h"
 
 /* uprobe attach point */
-static void trigger_func(void)
+static noinline void trigger_func(void)
 {
 	asm volatile ("");
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c
index e89685bd587c..3369c5ec3a17 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0
 /* Copyright (c) 2020 Facebook */
 #include <test_progs.h>
+#include <unistd.h>
+#include <sys/syscall.h>
 #include "bpf_iter_ipv6_route.skel.h"
 #include "bpf_iter_netlink.skel.h"
 #include "bpf_iter_bpf_map.skel.h"
@@ -14,6 +16,7 @@
 #include "bpf_iter_udp4.skel.h"
 #include "bpf_iter_udp6.skel.h"
 #include "bpf_iter_unix.skel.h"
+#include "bpf_iter_vma_offset.skel.h"
 #include "bpf_iter_test_kern1.skel.h"
 #include "bpf_iter_test_kern2.skel.h"
 #include "bpf_iter_test_kern3.skel.h"
@@ -43,13 +46,13 @@ static void test_btf_id_or_null(void)
 	}
 }
 
-static void do_dummy_read(struct bpf_program *prog)
+static void do_dummy_read_opts(struct bpf_program *prog, struct bpf_iter_attach_opts *opts)
 {
 	struct bpf_link *link;
 	char buf[16] = {};
 	int iter_fd, len;
 
-	link = bpf_program__attach_iter(prog, NULL);
+	link = bpf_program__attach_iter(prog, opts);
 	if (!ASSERT_OK_PTR(link, "attach_iter"))
 		return;
 
@@ -68,6 +71,11 @@ free_link:
 	bpf_link__destroy(link);
 }
 
+static void do_dummy_read(struct bpf_program *prog)
+{
+	do_dummy_read_opts(prog, NULL);
+}
+
 static void do_read_map_iter_fd(struct bpf_object_skeleton **skel, struct bpf_program *prog,
 				struct bpf_map *map)
 {
@@ -167,19 +175,140 @@ static void test_bpf_map(void)
 	bpf_iter_bpf_map__destroy(skel);
 }
 
-static void test_task(void)
+static int pidfd_open(pid_t pid, unsigned int flags)
+{
+	return syscall(SYS_pidfd_open, pid, flags);
+}
+
+static void check_bpf_link_info(const struct bpf_program *prog)
+{
+	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+	union bpf_iter_link_info linfo;
+	struct bpf_link_info info = {};
+	struct bpf_link *link;
+	__u32 info_len;
+	int err;
+
+	memset(&linfo, 0, sizeof(linfo));
+	linfo.task.tid = getpid();
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
+
+	link = bpf_program__attach_iter(prog, &opts);
+	if (!ASSERT_OK_PTR(link, "attach_iter"))
+		return;
+
+	info_len = sizeof(info);
+	err = bpf_obj_get_info_by_fd(bpf_link__fd(link), &info, &info_len);
+	ASSERT_OK(err, "bpf_obj_get_info_by_fd");
+	ASSERT_EQ(info.iter.task.tid, getpid(), "check_task_tid");
+
+	bpf_link__destroy(link);
+}
+
+static pthread_mutex_t do_nothing_mutex;
+
+static void *do_nothing_wait(void *arg)
+{
+	pthread_mutex_lock(&do_nothing_mutex);
+	pthread_mutex_unlock(&do_nothing_mutex);
+
+	pthread_exit(arg);
+}
+
+static void test_task_common_nocheck(struct bpf_iter_attach_opts *opts,
+				     int *num_unknown, int *num_known)
 {
 	struct bpf_iter_task *skel;
+	pthread_t thread_id;
+	void *ret;
 
 	skel = bpf_iter_task__open_and_load();
 	if (!ASSERT_OK_PTR(skel, "bpf_iter_task__open_and_load"))
 		return;
 
-	do_dummy_read(skel->progs.dump_task);
+	ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
+
+	ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
+		  "pthread_create");
+
+	skel->bss->tid = getpid();
+
+	do_dummy_read_opts(skel->progs.dump_task, opts);
+
+	*num_unknown = skel->bss->num_unknown_tid;
+	*num_known = skel->bss->num_known_tid;
+
+	ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
+	ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,
+		     "pthread_join");
 
 	bpf_iter_task__destroy(skel);
 }
 
+static void test_task_common(struct bpf_iter_attach_opts *opts, int num_unknown, int num_known)
+{
+	int num_unknown_tid, num_known_tid;
+
+	test_task_common_nocheck(opts, &num_unknown_tid, &num_known_tid);
+	ASSERT_EQ(num_unknown_tid, num_unknown, "check_num_unknown_tid");
+	ASSERT_EQ(num_known_tid, num_known, "check_num_known_tid");
+}
+
+static void test_task_tid(void)
+{
+	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+	union bpf_iter_link_info linfo;
+	int num_unknown_tid, num_known_tid;
+
+	memset(&linfo, 0, sizeof(linfo));
+	linfo.task.tid = getpid();
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
+	test_task_common(&opts, 0, 1);
+
+	linfo.task.tid = 0;
+	linfo.task.pid = getpid();
+	test_task_common(&opts, 1, 1);
+
+	test_task_common_nocheck(NULL, &num_unknown_tid, &num_known_tid);
+	ASSERT_GT(num_unknown_tid, 1, "check_num_unknown_tid");
+	ASSERT_EQ(num_known_tid, 1, "check_num_known_tid");
+}
+
+static void test_task_pid(void)
+{
+	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+	union bpf_iter_link_info linfo;
+
+	memset(&linfo, 0, sizeof(linfo));
+	linfo.task.pid = getpid();
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
+
+	test_task_common(&opts, 1, 1);
+}
+
+static void test_task_pidfd(void)
+{
+	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+	union bpf_iter_link_info linfo;
+	int pidfd;
+
+	pidfd = pidfd_open(getpid(), 0);
+	if (!ASSERT_GT(pidfd, 0, "pidfd_open"))
+		return;
+
+	memset(&linfo, 0, sizeof(linfo));
+	linfo.task.pid_fd = pidfd;
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
+
+	test_task_common(&opts, 1, 1);
+
+	close(pidfd);
+}
+
 static void test_task_sleepable(void)
 {
 	struct bpf_iter_task *skel;
@@ -212,14 +341,11 @@ static void test_task_stack(void)
 	bpf_iter_task_stack__destroy(skel);
 }
 
-static void *do_nothing(void *arg)
-{
-	pthread_exit(arg);
-}
-
 static void test_task_file(void)
 {
+	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
 	struct bpf_iter_task_file *skel;
+	union bpf_iter_link_info linfo;
 	pthread_t thread_id;
 	void *ret;
 
@@ -229,19 +355,36 @@ static void test_task_file(void)
 
 	skel->bss->tgid = getpid();
 
-	if (!ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing, NULL),
-		  "pthread_create"))
-		goto done;
+	ASSERT_OK(pthread_mutex_lock(&do_nothing_mutex), "pthread_mutex_lock");
 
-	do_dummy_read(skel->progs.dump_task_file);
+	ASSERT_OK(pthread_create(&thread_id, NULL, &do_nothing_wait, NULL),
+		  "pthread_create");
+
+	memset(&linfo, 0, sizeof(linfo));
+	linfo.task.tid = getpid();
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
 
-	if (!ASSERT_FALSE(pthread_join(thread_id, &ret) || ret != NULL,
-		  "pthread_join"))
-		goto done;
+	do_dummy_read_opts(skel->progs.dump_task_file, &opts);
 
 	ASSERT_EQ(skel->bss->count, 0, "check_count");
+	ASSERT_EQ(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
+
+	skel->bss->last_tgid = 0;
+	skel->bss->count = 0;
+	skel->bss->unique_tgid_count = 0;
+
+	do_dummy_read(skel->progs.dump_task_file);
+
+	ASSERT_EQ(skel->bss->count, 0, "check_count");
+	ASSERT_GT(skel->bss->unique_tgid_count, 1, "check_unique_tgid_count");
+
+	check_bpf_link_info(skel->progs.dump_task_file);
+
+	ASSERT_OK(pthread_mutex_unlock(&do_nothing_mutex), "pthread_mutex_unlock");
+	ASSERT_OK(pthread_join(thread_id, &ret), "pthread_join");
+	ASSERT_NULL(ret, "pthread_join");
 
-done:
 	bpf_iter_task_file__destroy(skel);
 }
 
@@ -1249,7 +1392,7 @@ static void str_strip_first_line(char *str)
 	*dst = '\0';
 }
 
-static void test_task_vma(void)
+static void test_task_vma_common(struct bpf_iter_attach_opts *opts)
 {
 	int err, iter_fd = -1, proc_maps_fd = -1;
 	struct bpf_iter_task_vma *skel;
@@ -1261,13 +1404,14 @@ static void test_task_vma(void)
 		return;
 
 	skel->bss->pid = getpid();
+	skel->bss->one_task = opts ? 1 : 0;
 
 	err = bpf_iter_task_vma__load(skel);
 	if (!ASSERT_OK(err, "bpf_iter_task_vma__load"))
 		goto out;
 
 	skel->links.proc_maps = bpf_program__attach_iter(
-		skel->progs.proc_maps, NULL);
+		skel->progs.proc_maps, opts);
 
 	if (!ASSERT_OK_PTR(skel->links.proc_maps, "bpf_program__attach_iter")) {
 		skel->links.proc_maps = NULL;
@@ -1291,6 +1435,8 @@ static void test_task_vma(void)
 			goto out;
 		len += err;
 	}
+	if (opts)
+		ASSERT_EQ(skel->bss->one_task_error, 0, "unexpected task");
 
 	/* read CMP_BUFFER_SIZE (1kB) from /proc/pid/maps */
 	snprintf(maps_path, 64, "/proc/%u/maps", skel->bss->pid);
@@ -1306,6 +1452,9 @@ static void test_task_vma(void)
 	str_strip_first_line(proc_maps_output);
 
 	ASSERT_STREQ(task_vma_output, proc_maps_output, "compare_output");
+
+	check_bpf_link_info(skel->progs.proc_maps);
+
 out:
 	close(proc_maps_fd);
 	close(iter_fd);
@@ -1325,8 +1474,93 @@ void test_bpf_sockmap_map_iter_fd(void)
 	bpf_iter_sockmap__destroy(skel);
 }
 
+static void test_task_vma(void)
+{
+	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+	union bpf_iter_link_info linfo;
+
+	memset(&linfo, 0, sizeof(linfo));
+	linfo.task.tid = getpid();
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
+
+	test_task_vma_common(&opts);
+	test_task_vma_common(NULL);
+}
+
+/* uprobe attach point */
+static noinline int trigger_func(int arg)
+{
+	asm volatile ("");
+	return arg + 1;
+}
+
+static void test_task_vma_offset_common(struct bpf_iter_attach_opts *opts, bool one_proc)
+{
+	struct bpf_iter_vma_offset *skel;
+	struct bpf_link *link;
+	char buf[16] = {};
+	int iter_fd, len;
+	int pgsz, shift;
+
+	skel = bpf_iter_vma_offset__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "bpf_iter_vma_offset__open_and_load"))
+		return;
+
+	skel->bss->pid = getpid();
+	skel->bss->address = (uintptr_t)trigger_func;
+	for (pgsz = getpagesize(), shift = 0; pgsz > 1; pgsz >>= 1, shift++)
+		;
+	skel->bss->page_shift = shift;
+
+	link = bpf_program__attach_iter(skel->progs.get_vma_offset, opts);
+	if (!ASSERT_OK_PTR(link, "attach_iter"))
+		return;
+
+	iter_fd = bpf_iter_create(bpf_link__fd(link));
+	if (!ASSERT_GT(iter_fd, 0, "create_iter"))
+		goto exit;
+
+	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
+		;
+	buf[15] = 0;
+	ASSERT_EQ(strcmp(buf, "OK\n"), 0, "strcmp");
+
+	ASSERT_EQ(skel->bss->offset, get_uprobe_offset(trigger_func), "offset");
+	if (one_proc)
+		ASSERT_EQ(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
+	else
+		ASSERT_GT(skel->bss->unique_tgid_cnt, 1, "unique_tgid_count");
+
+	close(iter_fd);
+
+exit:
+	bpf_link__destroy(link);
+}
+
+static void test_task_vma_offset(void)
+{
+	LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+	union bpf_iter_link_info linfo;
+
+	memset(&linfo, 0, sizeof(linfo));
+	linfo.task.pid = getpid();
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
+
+	test_task_vma_offset_common(&opts, true);
+
+	linfo.task.pid = 0;
+	linfo.task.tid = getpid();
+	test_task_vma_offset_common(&opts, true);
+
+	test_task_vma_offset_common(NULL, false);
+}
+
 void test_bpf_iter(void)
 {
+	ASSERT_OK(pthread_mutex_init(&do_nothing_mutex, NULL), "pthread_mutex_init");
+
 	if (test__start_subtest("btf_id_or_null"))
 		test_btf_id_or_null();
 	if (test__start_subtest("ipv6_route"))
@@ -1335,8 +1569,12 @@ void test_bpf_iter(void)
 		test_netlink();
 	if (test__start_subtest("bpf_map"))
 		test_bpf_map();
-	if (test__start_subtest("task"))
-		test_task();
+	if (test__start_subtest("task_tid"))
+		test_task_tid();
+	if (test__start_subtest("task_pid"))
+		test_task_pid();
+	if (test__start_subtest("task_pidfd"))
+		test_task_pidfd();
 	if (test__start_subtest("task_sleepable"))
 		test_task_sleepable();
 	if (test__start_subtest("task_stack"))
@@ -1397,4 +1635,6 @@ void test_bpf_iter(void)
 		test_ksym_iter();
 	if (test__start_subtest("bpf_sockmap_map_iter_fd"))
 		test_bpf_sockmap_map_iter_fd();
+	if (test__start_subtest("vma_offset"))
+		test_task_vma_offset();
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c
index 7a74a1579076..8a838ea8bdf3 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <test_progs.h>
 #include <network_helpers.h>
+#include <linux/netfilter/nf_conntrack_common.h>
 #include "test_bpf_nf.skel.h"
 #include "test_bpf_nf_fail.skel.h"
 
@@ -17,6 +18,7 @@ struct {
 	{ "set_status_after_insert", "kernel function bpf_ct_set_status args#0 expected pointer to STRUCT nf_conn___init but" },
 	{ "change_timeout_after_alloc", "kernel function bpf_ct_change_timeout args#0 expected pointer to STRUCT nf_conn but" },
 	{ "change_status_after_alloc", "kernel function bpf_ct_change_status args#0 expected pointer to STRUCT nf_conn but" },
+	{ "write_not_allowlisted_field", "no write support to nf_conn at off" },
 };
 
 enum {
@@ -24,10 +26,37 @@ enum {
 	TEST_TC_BPF,
 };
 
+#define TIMEOUT_MS		3000
+#define IPS_STATUS_MASK		(IPS_CONFIRMED | IPS_SEEN_REPLY | \
+				 IPS_SRC_NAT_DONE | IPS_DST_NAT_DONE | \
+				 IPS_SRC_NAT | IPS_DST_NAT)
+
+static int connect_to_server(int srv_fd)
+{
+	int fd = -1;
+
+	fd = socket(AF_INET, SOCK_STREAM, 0);
+	if (!ASSERT_GE(fd, 0, "socket"))
+		goto out;
+
+	if (!ASSERT_EQ(connect_fd_to_fd(fd, srv_fd, TIMEOUT_MS), 0, "connect_fd_to_fd")) {
+		close(fd);
+		fd = -1;
+	}
+out:
+	return fd;
+}
+
 static void test_bpf_nf_ct(int mode)
 {
+	const char *iptables = "iptables -t raw %s PREROUTING -j CONNMARK --set-mark 42/0";
+	int srv_fd = -1, client_fd = -1, srv_client_fd = -1;
+	struct sockaddr_in peer_addr = {};
 	struct test_bpf_nf *skel;
 	int prog_fd, err;
+	socklen_t len;
+	u16 srv_port;
+	char cmd[64];
 	LIBBPF_OPTS(bpf_test_run_opts, topts,
 		.data_in = &pkt_v4,
 		.data_size_in = sizeof(pkt_v4),
@@ -38,6 +67,32 @@ static void test_bpf_nf_ct(int mode)
 	if (!ASSERT_OK_PTR(skel, "test_bpf_nf__open_and_load"))
 		return;
 
+	/* Enable connection tracking */
+	snprintf(cmd, sizeof(cmd), iptables, "-A");
+	if (!ASSERT_OK(system(cmd), "iptables"))
+		goto end;
+
+	srv_port = (mode == TEST_XDP) ? 5005 : 5006;
+	srv_fd = start_server(AF_INET, SOCK_STREAM, "127.0.0.1", srv_port, TIMEOUT_MS);
+	if (!ASSERT_GE(srv_fd, 0, "start_server"))
+		goto end;
+
+	client_fd = connect_to_server(srv_fd);
+	if (!ASSERT_GE(client_fd, 0, "connect_to_server"))
+		goto end;
+
+	len = sizeof(peer_addr);
+	srv_client_fd = accept(srv_fd, (struct sockaddr *)&peer_addr, &len);
+	if (!ASSERT_GE(srv_client_fd, 0, "accept"))
+		goto end;
+	if (!ASSERT_EQ(len, sizeof(struct sockaddr_in), "sockaddr len"))
+		goto end;
+
+	skel->bss->saddr = peer_addr.sin_addr.s_addr;
+	skel->bss->sport = peer_addr.sin_port;
+	skel->bss->daddr = peer_addr.sin_addr.s_addr;
+	skel->bss->dport = htons(srv_port);
+
 	if (mode == TEST_XDP)
 		prog_fd = bpf_program__fd(skel->progs.nf_xdp_ct_test);
 	else
@@ -61,9 +116,21 @@ static void test_bpf_nf_ct(int mode)
 	/* allow some tolerance for test_delta_timeout value to avoid races. */
 	ASSERT_GT(skel->bss->test_delta_timeout, 8, "Test for min ct timeout update");
 	ASSERT_LE(skel->bss->test_delta_timeout, 10, "Test for max ct timeout update");
-	/* expected status is IPS_SEEN_REPLY */
-	ASSERT_EQ(skel->bss->test_status, 2, "Test for ct status update ");
+	ASSERT_EQ(skel->bss->test_insert_lookup_mark, 77, "Test for insert and lookup mark value");
+	ASSERT_EQ(skel->bss->test_status, IPS_STATUS_MASK, "Test for ct status update ");
+	ASSERT_EQ(skel->data->test_exist_lookup, 0, "Test existing connection lookup");
+	ASSERT_EQ(skel->bss->test_exist_lookup_mark, 43, "Test existing connection lookup ctmark");
+	ASSERT_EQ(skel->data->test_snat_addr, 0, "Test for source natting");
+	ASSERT_EQ(skel->data->test_dnat_addr, 0, "Test for destination natting");
 end:
+	if (srv_client_fd != -1)
+		close(srv_client_fd);
+	if (client_fd != -1)
+		close(client_fd);
+	if (srv_fd != -1)
+		close(srv_fd);
+	snprintf(cmd, sizeof(cmd), iptables, "-D");
+	system(cmd);
 	test_bpf_nf__destroy(skel);
 }
 
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c b/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c
index dbe56fa8582d..e1c1e521cca2 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c
@@ -7,7 +7,7 @@ void serial_test_bpf_obj_id(void)
 {
 	const __u64 array_magic_value = 0xfaceb00c;
 	const __u32 array_key = 0;
-	const char *file = "./test_obj_id.o";
+	const char *file = "./test_obj_id.bpf.o";
 	const char *expected_prog_name = "test_obj_id";
 	const char *expected_map_name = "test_map_id";
 	const __u64 nsec_per_sec = 1000000000;
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
index 2959a52ced06..e980188d4124 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_tcp_ca.c
@@ -290,6 +290,10 @@ static void test_dctcp_fallback(void)
 		goto done;
 	ASSERT_STREQ(dctcp_skel->bss->cc_res, "cubic", "cc_res");
 	ASSERT_EQ(dctcp_skel->bss->tcp_cdg_res, -ENOTSUPP, "tcp_cdg_res");
+	/* All setsockopt(TCP_CONGESTION) in the recurred
+	 * bpf_dctcp->init() should fail with -EBUSY.
+	 */
+	ASSERT_EQ(dctcp_skel->bss->ebusy_cnt, 3, "ebusy_cnt");
 
 	err = getsockopt(srv_fd, SOL_TCP, TCP_CONGESTION, srv_cc, &cc_len);
 	if (!ASSERT_OK(err, "getsockopt(srv_fd, TCP_CONGESTION)"))
diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
index ff6cce9fef06..5ca252823294 100644
--- a/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
+++ b/tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c
@@ -75,45 +75,45 @@ static void scale_test(const char *file,
 
 void test_verif_scale1()
 {
-	scale_test("test_verif_scale1.o", BPF_PROG_TYPE_SCHED_CLS, false);
+	scale_test("test_verif_scale1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false);
 }
 
 void test_verif_scale2()
 {
-	scale_test("test_verif_scale2.o", BPF_PROG_TYPE_SCHED_CLS, false);
+	scale_test("test_verif_scale2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false);
 }
 
 void test_verif_scale3()
 {
-	scale_test("test_verif_scale3.o", BPF_PROG_TYPE_SCHED_CLS, false);
+	scale_test("test_verif_scale3.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false);
 }
 
 void test_verif_scale_pyperf_global()
 {
-	scale_test("pyperf_global.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("pyperf_global.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_pyperf_subprogs()
 {
-	scale_test("pyperf_subprogs.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("pyperf_subprogs.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_pyperf50()
 {
 	/* full unroll by llvm */
-	scale_test("pyperf50.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("pyperf50.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_pyperf100()
 {
 	/* full unroll by llvm */
-	scale_test("pyperf100.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("pyperf100.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_pyperf180()
 {
 	/* full unroll by llvm */
-	scale_test("pyperf180.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("pyperf180.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_pyperf600()
@@ -124,13 +124,13 @@ void test_verif_scale_pyperf600()
 	 * 16k insns in loop body.
 	 * Total of 5 such loops. Total program size ~82k insns.
 	 */
-	scale_test("pyperf600.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("pyperf600.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_pyperf600_bpf_loop(void)
 {
 	/* use the bpf_loop helper*/
-	scale_test("pyperf600_bpf_loop.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("pyperf600_bpf_loop.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_pyperf600_nounroll()
@@ -141,37 +141,37 @@ void test_verif_scale_pyperf600_nounroll()
 	 * ~110 insns in loop body.
 	 * Total of 5 such loops. Total program size ~1500 insns.
 	 */
-	scale_test("pyperf600_nounroll.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("pyperf600_nounroll.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_loop1()
 {
-	scale_test("loop1.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("loop1.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_loop2()
 {
-	scale_test("loop2.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("loop2.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_loop3_fail()
 {
-	scale_test("loop3.o", BPF_PROG_TYPE_RAW_TRACEPOINT, true /* fails */);
+	scale_test("loop3.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, true /* fails */);
 }
 
 void test_verif_scale_loop4()
 {
-	scale_test("loop4.o", BPF_PROG_TYPE_SCHED_CLS, false);
+	scale_test("loop4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false);
 }
 
 void test_verif_scale_loop5()
 {
-	scale_test("loop5.o", BPF_PROG_TYPE_SCHED_CLS, false);
+	scale_test("loop5.bpf.o", BPF_PROG_TYPE_SCHED_CLS, false);
 }
 
 void test_verif_scale_loop6()
 {
-	scale_test("loop6.o", BPF_PROG_TYPE_KPROBE, false);
+	scale_test("loop6.bpf.o", BPF_PROG_TYPE_KPROBE, false);
 }
 
 void test_verif_scale_strobemeta()
@@ -180,54 +180,54 @@ void test_verif_scale_strobemeta()
 	 * Total program size 20.8k insn.
 	 * ~350k processed_insns
 	 */
-	scale_test("strobemeta.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("strobemeta.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_strobemeta_bpf_loop(void)
 {
 	/* use the bpf_loop helper*/
-	scale_test("strobemeta_bpf_loop.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("strobemeta_bpf_loop.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_strobemeta_nounroll1()
 {
 	/* no unroll, tiny loops */
-	scale_test("strobemeta_nounroll1.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("strobemeta_nounroll1.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_strobemeta_nounroll2()
 {
 	/* no unroll, tiny loops */
-	scale_test("strobemeta_nounroll2.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("strobemeta_nounroll2.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_strobemeta_subprogs()
 {
 	/* non-inlined subprogs */
-	scale_test("strobemeta_subprogs.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
+	scale_test("strobemeta_subprogs.bpf.o", BPF_PROG_TYPE_RAW_TRACEPOINT, false);
 }
 
 void test_verif_scale_sysctl_loop1()
 {
-	scale_test("test_sysctl_loop1.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false);
+	scale_test("test_sysctl_loop1.bpf.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false);
 }
 
 void test_verif_scale_sysctl_loop2()
 {
-	scale_test("test_sysctl_loop2.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false);
+	scale_test("test_sysctl_loop2.bpf.o", BPF_PROG_TYPE_CGROUP_SYSCTL, false);
 }
 
 void test_verif_scale_xdp_loop()
 {
-	scale_test("test_xdp_loop.o", BPF_PROG_TYPE_XDP, false);
+	scale_test("test_xdp_loop.bpf.o", BPF_PROG_TYPE_XDP, false);
 }
 
 void test_verif_scale_seg6_loop()
 {
-	scale_test("test_seg6_loop.o", BPF_PROG_TYPE_LWT_SEG6LOCAL, false);
+	scale_test("test_seg6_loop.bpf.o", BPF_PROG_TYPE_LWT_SEG6LOCAL, false);
 }
 
 void test_verif_twfw()
 {
-	scale_test("twfw.o", BPF_PROG_TYPE_CGROUP_SKB, false);
+	scale_test("twfw.bpf.o", BPF_PROG_TYPE_CGROUP_SKB, false);
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c
index ef6528b8084c..127b8caa3dc1 100644
--- a/tools/testing/selftests/bpf/prog_tests/btf.c
+++ b/tools/testing/selftests/bpf/prog_tests/btf.c
@@ -4651,8 +4651,8 @@ struct btf_file_test {
 };
 
 static struct btf_file_test file_tests[] = {
-	{ .file = "test_btf_newkv.o", },
-	{ .file = "test_btf_nokv.o", .btf_kv_notfound = true, },
+	{ .file = "test_btf_newkv.bpf.o", },
+	{ .file = "test_btf_nokv.bpf.o", .btf_kv_notfound = true, },
 };
 
 static void do_test_file(unsigned int test_num)
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c
index 5fce7008d1ff..24da335482d4 100644
--- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c
+++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c
@@ -52,7 +52,7 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t)
 	int err = 0, fd = -1;
 	FILE *f = NULL;
 
-	snprintf(test_file, sizeof(test_file), "%s.o", t->file);
+	snprintf(test_file, sizeof(test_file), "%s.bpf.o", t->file);
 
 	btf = btf__parse_elf(test_file, NULL);
 	if (!ASSERT_OK_PTR(btf, "btf_parse_elf")) {
@@ -764,8 +764,8 @@ static void test_btf_dump_struct_data(struct btf *btf, struct btf_dump *d,
 
 	/* union with nested struct */
 	TEST_BTF_DUMP_DATA(btf, d, "union", str, union bpf_iter_link_info, BTF_F_COMPACT,
-			   "(union bpf_iter_link_info){.map = (struct){.map_fd = (__u32)1,},}",
-			   { .map = { .map_fd = 1 }});
+			   "(union bpf_iter_link_info){.map = (struct){.map_fd = (__u32)1,},.cgroup = (struct){.order = (enum bpf_cgroup_iter_order)BPF_CGROUP_ITER_SELF_ONLY,.cgroup_fd = (__u32)1,},.task = (struct){.tid = (__u32)1,.pid = (__u32)1,},}",
+			   { .cgroup = { .order = 1, .cgroup_fd = 1, }});
 
 	/* struct skb with nested structs/unions; because type output is so
 	 * complex, we don't do a string comparison, just verify we return
@@ -841,8 +841,8 @@ static void test_btf_dump_datasec_data(char *str)
 	char license[4] = "GPL";
 	struct btf_dump *d;
 
-	btf = btf__parse("xdping_kern.o", NULL);
-	if (!ASSERT_OK_PTR(btf, "xdping_kern.o BTF not found"))
+	btf = btf__parse("xdping_kern.bpf.o", NULL);
+	if (!ASSERT_OK_PTR(btf, "xdping_kern.bpf.o BTF not found"))
 		return;
 
 	d = btf_dump__new(btf, btf_dump_snprintf, str, NULL);
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_endian.c b/tools/testing/selftests/bpf/prog_tests/btf_endian.c
index 8afbf3d0b89a..5b9f84dbeb43 100644
--- a/tools/testing/selftests/bpf/prog_tests/btf_endian.c
+++ b/tools/testing/selftests/bpf/prog_tests/btf_endian.c
@@ -23,7 +23,7 @@ void test_btf_endian() {
 	int var_id;
 
 	/* Load BTF in native endianness */
-	btf = btf__parse_elf("btf_dump_test_case_syntax.o", NULL);
+	btf = btf__parse_elf("btf_dump_test_case_syntax.bpf.o", NULL);
 	if (!ASSERT_OK_PTR(btf, "parse_native_btf"))
 		goto err_out;
 
diff --git a/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c b/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c
index 664ffc0364f4..7a277035c275 100644
--- a/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c
+++ b/tools/testing/selftests/bpf/prog_tests/btf_skc_cls_ingress.c
@@ -22,26 +22,6 @@ static __u32 duration;
 
 #define PROG_PIN_FILE "/sys/fs/bpf/btf_skc_cls_ingress"
 
-static int write_sysctl(const char *sysctl, const char *value)
-{
-	int fd, err, len;
-
-	fd = open(sysctl, O_WRONLY);
-	if (CHECK(fd == -1, "open sysctl", "open(%s): %s (%d)\n",
-		  sysctl, strerror(errno), errno))
-		return -1;
-
-	len = strlen(value);
-	err = write(fd, value, len);
-	close(fd);
-	if (CHECK(err != len, "write sysctl",
-		  "write(%s, %s, %d): err:%d %s (%d)\n",
-		  sysctl, value, len, err, strerror(errno), errno))
-		return -1;
-
-	return 0;
-}
-
 static int prepare_netns(void)
 {
 	if (CHECK(unshare(CLONE_NEWNET), "create netns",
diff --git a/tools/testing/selftests/bpf/prog_tests/cb_refs.c b/tools/testing/selftests/bpf/prog_tests/cb_refs.c
new file mode 100644
index 000000000000..3bff680de16c
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/cb_refs.c
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0
+#include "bpf/libbpf.h"
+#include <test_progs.h>
+#include <network_helpers.h>
+
+#include "cb_refs.skel.h"
+
+static char log_buf[1024 * 1024];
+
+struct {
+	const char *prog_name;
+	const char *err_msg;
+} cb_refs_tests[] = {
+	{ "underflow_prog", "reference has not been acquired before" },
+	{ "leak_prog", "Unreleased reference" },
+	{ "nested_cb", "Unreleased reference id=4 alloc_insn=2" }, /* alloc_insn=2{4,5} */
+	{ "non_cb_transfer_ref", "Unreleased reference id=4 alloc_insn=1" }, /* alloc_insn=1{1,2} */
+};
+
+void test_cb_refs(void)
+{
+	LIBBPF_OPTS(bpf_object_open_opts, opts, .kernel_log_buf = log_buf,
+						.kernel_log_size = sizeof(log_buf),
+						.kernel_log_level = 1);
+	struct bpf_program *prog;
+	struct cb_refs *skel;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(cb_refs_tests); i++) {
+		LIBBPF_OPTS(bpf_test_run_opts, run_opts,
+			.data_in = &pkt_v4,
+			.data_size_in = sizeof(pkt_v4),
+			.repeat = 1,
+		);
+		skel = cb_refs__open_opts(&opts);
+		if (!ASSERT_OK_PTR(skel, "cb_refs__open_and_load"))
+			return;
+		prog = bpf_object__find_program_by_name(skel->obj, cb_refs_tests[i].prog_name);
+		bpf_program__set_autoload(prog, true);
+		if (!ASSERT_ERR(cb_refs__load(skel), "cb_refs__load"))
+			bpf_prog_test_run_opts(bpf_program__fd(prog), &run_opts);
+		if (!ASSERT_OK_PTR(strstr(log_buf, cb_refs_tests[i].err_msg), "expected error message")) {
+			fprintf(stderr, "Expected: %s\n", cb_refs_tests[i].err_msg);
+			fprintf(stderr, "Verifier: %s\n", log_buf);
+		}
+		cb_refs__destroy(skel);
+	}
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c b/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c
index 0b47c3c000c7..4d2fa99273d8 100644
--- a/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c
+++ b/tools/testing/selftests/bpf/prog_tests/cgroup_getset_retval.c
@@ -10,6 +10,7 @@
 
 #include "cgroup_getset_retval_setsockopt.skel.h"
 #include "cgroup_getset_retval_getsockopt.skel.h"
+#include "cgroup_getset_retval_hooks.skel.h"
 
 #define SOL_CUSTOM	0xdeadbeef
 
@@ -433,6 +434,50 @@ close_bpf_object:
 	cgroup_getset_retval_getsockopt__destroy(obj);
 }
 
+struct exposed_hook {
+	const char *name;
+	int expected_err;
+} exposed_hooks[] = {
+
+#define BPF_RETVAL_HOOK(NAME, SECTION, CTX, EXPECTED_ERR) \
+	{ \
+		.name = #NAME, \
+		.expected_err = EXPECTED_ERR, \
+	},
+
+#include "cgroup_getset_retval_hooks.h"
+
+#undef BPF_RETVAL_HOOK
+};
+
+static void test_exposed_hooks(int cgroup_fd, int sock_fd)
+{
+	struct cgroup_getset_retval_hooks *skel;
+	struct bpf_program *prog;
+	int err;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(exposed_hooks); i++) {
+		skel = cgroup_getset_retval_hooks__open();
+		if (!ASSERT_OK_PTR(skel, "cgroup_getset_retval_hooks__open"))
+			continue;
+
+		prog = bpf_object__find_program_by_name(skel->obj, exposed_hooks[i].name);
+		if (!ASSERT_NEQ(prog, NULL, "bpf_object__find_program_by_name"))
+			goto close_skel;
+
+		err = bpf_program__set_autoload(prog, true);
+		if (!ASSERT_OK(err, "bpf_program__set_autoload"))
+			goto close_skel;
+
+		err = cgroup_getset_retval_hooks__load(skel);
+		ASSERT_EQ(err, exposed_hooks[i].expected_err, "expected_err");
+
+close_skel:
+		cgroup_getset_retval_hooks__destroy(skel);
+	}
+}
+
 void test_cgroup_getset_retval(void)
 {
 	int cgroup_fd = -1;
@@ -476,6 +521,9 @@ void test_cgroup_getset_retval(void)
 	if (test__start_subtest("getsockopt-retval_sync"))
 		test_getsockopt_retval_sync(cgroup_fd, sock_fd);
 
+	if (test__start_subtest("exposed_hooks"))
+		test_exposed_hooks(cgroup_fd, sock_fd);
+
 close_fd:
 	close(cgroup_fd);
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c
new file mode 100644
index 000000000000..3bd27d2ea668
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/cgroup_hierarchical_stats.c
@@ -0,0 +1,339 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * This test makes sure BPF stats collection using rstat works correctly.
+ * The test uses 3 BPF progs:
+ * (a) counter: This BPF prog is invoked every time we attach a process to a
+ *              cgroup and locklessly increments a percpu counter.
+ *              The program then calls cgroup_rstat_updated() to inform rstat
+ *              of an update on the (cpu, cgroup) pair.
+ *
+ * (b) flusher: This BPF prog is invoked when an rstat flush is ongoing, it
+ *              aggregates all percpu counters to a total counter, and also
+ *              propagates the changes to the ancestor cgroups.
+ *
+ * (c) dumper: This BPF prog is a cgroup_iter. It is used to output the total
+ *             counter of a cgroup through reading a file in userspace.
+ *
+ * The test sets up a cgroup hierarchy, and the above programs. It spawns a few
+ * processes in the leaf cgroups and makes sure all the counters are aggregated
+ * correctly.
+ *
+ * Copyright 2022 Google LLC.
+ */
+#include <asm-generic/errno.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/mount.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <test_progs.h>
+#include <bpf/libbpf.h>
+#include <bpf/bpf.h>
+
+#include "cgroup_helpers.h"
+#include "cgroup_hierarchical_stats.skel.h"
+
+#define PAGE_SIZE 4096
+#define MB(x) (x << 20)
+
+#define PROCESSES_PER_CGROUP 3
+
+#define BPFFS_ROOT "/sys/fs/bpf/"
+#define BPFFS_ATTACH_COUNTERS BPFFS_ROOT "attach_counters/"
+
+#define CG_ROOT_NAME "root"
+#define CG_ROOT_ID 1
+
+#define CGROUP_PATH(p, n) {.path = p"/"n, .name = n}
+
+static struct {
+	const char *path, *name;
+	unsigned long long id;
+	int fd;
+} cgroups[] = {
+	CGROUP_PATH("/", "test"),
+	CGROUP_PATH("/test", "child1"),
+	CGROUP_PATH("/test", "child2"),
+	CGROUP_PATH("/test/child1", "child1_1"),
+	CGROUP_PATH("/test/child1", "child1_2"),
+	CGROUP_PATH("/test/child2", "child2_1"),
+	CGROUP_PATH("/test/child2", "child2_2"),
+};
+
+#define N_CGROUPS ARRAY_SIZE(cgroups)
+#define N_NON_LEAF_CGROUPS 3
+
+static int root_cgroup_fd;
+static bool mounted_bpffs;
+
+/* reads file at 'path' to 'buf', returns 0 on success. */
+static int read_from_file(const char *path, char *buf, size_t size)
+{
+	int fd, len;
+
+	fd = open(path, O_RDONLY);
+	if (fd < 0)
+		return fd;
+
+	len = read(fd, buf, size);
+	close(fd);
+	if (len < 0)
+		return len;
+
+	buf[len] = 0;
+	return 0;
+}
+
+/* mounts bpffs and mkdir for reading stats, returns 0 on success. */
+static int setup_bpffs(void)
+{
+	int err;
+
+	/* Mount bpffs */
+	err = mount("bpf", BPFFS_ROOT, "bpf", 0, NULL);
+	mounted_bpffs = !err;
+	if (ASSERT_FALSE(err && errno != EBUSY, "mount"))
+		return err;
+
+	/* Create a directory to contain stat files in bpffs */
+	err = mkdir(BPFFS_ATTACH_COUNTERS, 0755);
+	if (!ASSERT_OK(err, "mkdir"))
+		return err;
+
+	return 0;
+}
+
+static void cleanup_bpffs(void)
+{
+	/* Remove created directory in bpffs */
+	ASSERT_OK(rmdir(BPFFS_ATTACH_COUNTERS), "rmdir "BPFFS_ATTACH_COUNTERS);
+
+	/* Unmount bpffs, if it wasn't already mounted when we started */
+	if (mounted_bpffs)
+		return;
+
+	ASSERT_OK(umount(BPFFS_ROOT), "unmount bpffs");
+}
+
+/* sets up cgroups, returns 0 on success. */
+static int setup_cgroups(void)
+{
+	int i, fd, err;
+
+	err = setup_cgroup_environment();
+	if (!ASSERT_OK(err, "setup_cgroup_environment"))
+		return err;
+
+	root_cgroup_fd = get_root_cgroup();
+	if (!ASSERT_GE(root_cgroup_fd, 0, "get_root_cgroup"))
+		return root_cgroup_fd;
+
+	for (i = 0; i < N_CGROUPS; i++) {
+		fd = create_and_get_cgroup(cgroups[i].path);
+		if (!ASSERT_GE(fd, 0, "create_and_get_cgroup"))
+			return fd;
+
+		cgroups[i].fd = fd;
+		cgroups[i].id = get_cgroup_id(cgroups[i].path);
+	}
+	return 0;
+}
+
+static void cleanup_cgroups(void)
+{
+	close(root_cgroup_fd);
+	for (int i = 0; i < N_CGROUPS; i++)
+		close(cgroups[i].fd);
+	cleanup_cgroup_environment();
+}
+
+/* Sets up cgroup hiearchary, returns 0 on success. */
+static int setup_hierarchy(void)
+{
+	return setup_bpffs() || setup_cgroups();
+}
+
+static void destroy_hierarchy(void)
+{
+	cleanup_cgroups();
+	cleanup_bpffs();
+}
+
+static int attach_processes(void)
+{
+	int i, j, status;
+
+	/* In every leaf cgroup, attach 3 processes */
+	for (i = N_NON_LEAF_CGROUPS; i < N_CGROUPS; i++) {
+		for (j = 0; j < PROCESSES_PER_CGROUP; j++) {
+			pid_t pid;
+
+			/* Create child and attach to cgroup */
+			pid = fork();
+			if (pid == 0) {
+				if (join_parent_cgroup(cgroups[i].path))
+					exit(EACCES);
+				exit(0);
+			}
+
+			/* Cleanup child */
+			waitpid(pid, &status, 0);
+			if (!ASSERT_TRUE(WIFEXITED(status), "child process exited"))
+				return 1;
+			if (!ASSERT_EQ(WEXITSTATUS(status), 0,
+				       "child process exit code"))
+				return 1;
+		}
+	}
+	return 0;
+}
+
+static unsigned long long
+get_attach_counter(unsigned long long cgroup_id, const char *file_name)
+{
+	unsigned long long attach_counter = 0, id = 0;
+	static char buf[128], path[128];
+
+	/* For every cgroup, read the file generated by cgroup_iter */
+	snprintf(path, 128, "%s%s", BPFFS_ATTACH_COUNTERS, file_name);
+	if (!ASSERT_OK(read_from_file(path, buf, 128), "read cgroup_iter"))
+		return 0;
+
+	/* Check the output file formatting */
+	ASSERT_EQ(sscanf(buf, "cg_id: %llu, attach_counter: %llu\n",
+			 &id, &attach_counter), 2, "output format");
+
+	/* Check that the cgroup_id is displayed correctly */
+	ASSERT_EQ(id, cgroup_id, "cgroup_id");
+	/* Check that the counter is non-zero */
+	ASSERT_GT(attach_counter, 0, "attach counter non-zero");
+	return attach_counter;
+}
+
+static void check_attach_counters(void)
+{
+	unsigned long long attach_counters[N_CGROUPS], root_attach_counter;
+	int i;
+
+	for (i = 0; i < N_CGROUPS; i++)
+		attach_counters[i] = get_attach_counter(cgroups[i].id,
+							cgroups[i].name);
+
+	/* Read stats for root too */
+	root_attach_counter = get_attach_counter(CG_ROOT_ID, CG_ROOT_NAME);
+
+	/* Check that all leafs cgroups have an attach counter of 3 */
+	for (i = N_NON_LEAF_CGROUPS; i < N_CGROUPS; i++)
+		ASSERT_EQ(attach_counters[i], PROCESSES_PER_CGROUP,
+			  "leaf cgroup attach counter");
+
+	/* Check that child1 == child1_1 + child1_2 */
+	ASSERT_EQ(attach_counters[1], attach_counters[3] + attach_counters[4],
+		  "child1_counter");
+	/* Check that child2 == child2_1 + child2_2 */
+	ASSERT_EQ(attach_counters[2], attach_counters[5] + attach_counters[6],
+		  "child2_counter");
+	/* Check that test == child1 + child2 */
+	ASSERT_EQ(attach_counters[0], attach_counters[1] + attach_counters[2],
+		  "test_counter");
+	/* Check that root >= test */
+	ASSERT_GE(root_attach_counter, attach_counters[1], "root_counter");
+}
+
+/* Creates iter link and pins in bpffs, returns 0 on success, -errno on failure.
+ */
+static int setup_cgroup_iter(struct cgroup_hierarchical_stats *obj,
+			     int cgroup_fd, const char *file_name)
+{
+	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+	union bpf_iter_link_info linfo = {};
+	struct bpf_link *link;
+	static char path[128];
+	int err;
+
+	/*
+	 * Create an iter link, parameterized by cgroup_fd. We only want to
+	 * traverse one cgroup, so set the traversal order to "self".
+	 */
+	linfo.cgroup.cgroup_fd = cgroup_fd;
+	linfo.cgroup.order = BPF_CGROUP_ITER_SELF_ONLY;
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
+	link = bpf_program__attach_iter(obj->progs.dumper, &opts);
+	if (!ASSERT_OK_PTR(link, "attach_iter"))
+		return -EFAULT;
+
+	/* Pin the link to a bpffs file */
+	snprintf(path, 128, "%s%s", BPFFS_ATTACH_COUNTERS, file_name);
+	err = bpf_link__pin(link, path);
+	ASSERT_OK(err, "pin cgroup_iter");
+
+	/* Remove the link, leaving only the ref held by the pinned file */
+	bpf_link__destroy(link);
+	return err;
+}
+
+/* Sets up programs for collecting stats, returns 0 on success. */
+static int setup_progs(struct cgroup_hierarchical_stats **skel)
+{
+	int i, err;
+
+	*skel = cgroup_hierarchical_stats__open_and_load();
+	if (!ASSERT_OK_PTR(*skel, "open_and_load"))
+		return 1;
+
+	/* Attach cgroup_iter program that will dump the stats to cgroups */
+	for (i = 0; i < N_CGROUPS; i++) {
+		err = setup_cgroup_iter(*skel, cgroups[i].fd, cgroups[i].name);
+		if (!ASSERT_OK(err, "setup_cgroup_iter"))
+			return err;
+	}
+
+	/* Also dump stats for root */
+	err = setup_cgroup_iter(*skel, root_cgroup_fd, CG_ROOT_NAME);
+	if (!ASSERT_OK(err, "setup_cgroup_iter"))
+		return err;
+
+	bpf_program__set_autoattach((*skel)->progs.dumper, false);
+	err = cgroup_hierarchical_stats__attach(*skel);
+	if (!ASSERT_OK(err, "attach"))
+		return err;
+
+	return 0;
+}
+
+static void destroy_progs(struct cgroup_hierarchical_stats *skel)
+{
+	static char path[128];
+	int i;
+
+	for (i = 0; i < N_CGROUPS; i++) {
+		/* Delete files in bpffs that cgroup_iters are pinned in */
+		snprintf(path, 128, "%s%s", BPFFS_ATTACH_COUNTERS,
+			 cgroups[i].name);
+		ASSERT_OK(remove(path), "remove cgroup_iter pin");
+	}
+
+	/* Delete root file in bpffs */
+	snprintf(path, 128, "%s%s", BPFFS_ATTACH_COUNTERS, CG_ROOT_NAME);
+	ASSERT_OK(remove(path), "remove cgroup_iter root pin");
+	cgroup_hierarchical_stats__destroy(skel);
+}
+
+void test_cgroup_hierarchical_stats(void)
+{
+	struct cgroup_hierarchical_stats *skel = NULL;
+
+	if (setup_hierarchy())
+		goto hierarchy_cleanup;
+	if (setup_progs(&skel))
+		goto cleanup;
+	if (attach_processes())
+		goto cleanup;
+	check_attach_counters();
+cleanup:
+	destroy_progs(skel);
+hierarchy_cleanup:
+	destroy_hierarchy();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_iter.c b/tools/testing/selftests/bpf/prog_tests/cgroup_iter.c
new file mode 100644
index 000000000000..c4a2adb38da1
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/cgroup_iter.c
@@ -0,0 +1,224 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Google */
+
+#include <test_progs.h>
+#include <bpf/libbpf.h>
+#include <bpf/btf.h>
+#include "cgroup_iter.skel.h"
+#include "cgroup_helpers.h"
+
+#define ROOT           0
+#define PARENT         1
+#define CHILD1         2
+#define CHILD2         3
+#define NUM_CGROUPS    4
+
+#define PROLOGUE       "prologue\n"
+#define EPILOGUE       "epilogue\n"
+
+static const char *cg_path[] = {
+	"/", "/parent", "/parent/child1", "/parent/child2"
+};
+
+static int cg_fd[] = {-1, -1, -1, -1};
+static unsigned long long cg_id[] = {0, 0, 0, 0};
+static char expected_output[64];
+
+static int setup_cgroups(void)
+{
+	int fd, i = 0;
+
+	for (i = 0; i < NUM_CGROUPS; i++) {
+		fd = create_and_get_cgroup(cg_path[i]);
+		if (fd < 0)
+			return fd;
+
+		cg_fd[i] = fd;
+		cg_id[i] = get_cgroup_id(cg_path[i]);
+	}
+	return 0;
+}
+
+static void cleanup_cgroups(void)
+{
+	int i;
+
+	for (i = 0; i < NUM_CGROUPS; i++)
+		close(cg_fd[i]);
+}
+
+static void read_from_cgroup_iter(struct bpf_program *prog, int cgroup_fd,
+				  int order, const char *testname)
+{
+	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+	union bpf_iter_link_info linfo;
+	struct bpf_link *link;
+	int len, iter_fd;
+	static char buf[128];
+	size_t left;
+	char *p;
+
+	memset(&linfo, 0, sizeof(linfo));
+	linfo.cgroup.cgroup_fd = cgroup_fd;
+	linfo.cgroup.order = order;
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
+
+	link = bpf_program__attach_iter(prog, &opts);
+	if (!ASSERT_OK_PTR(link, "attach_iter"))
+		return;
+
+	iter_fd = bpf_iter_create(bpf_link__fd(link));
+	if (iter_fd < 0)
+		goto free_link;
+
+	memset(buf, 0, sizeof(buf));
+	left = ARRAY_SIZE(buf);
+	p = buf;
+	while ((len = read(iter_fd, p, left)) > 0) {
+		p += len;
+		left -= len;
+	}
+
+	ASSERT_STREQ(buf, expected_output, testname);
+
+	/* read() after iter finishes should be ok. */
+	if (len == 0)
+		ASSERT_OK(read(iter_fd, buf, sizeof(buf)), "second_read");
+
+	close(iter_fd);
+free_link:
+	bpf_link__destroy(link);
+}
+
+/* Invalid cgroup. */
+static void test_invalid_cgroup(struct cgroup_iter *skel)
+{
+	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+	union bpf_iter_link_info linfo;
+	struct bpf_link *link;
+
+	memset(&linfo, 0, sizeof(linfo));
+	linfo.cgroup.cgroup_fd = (__u32)-1;
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
+
+	link = bpf_program__attach_iter(skel->progs.cgroup_id_printer, &opts);
+	ASSERT_ERR_PTR(link, "attach_iter");
+	bpf_link__destroy(link);
+}
+
+/* Specifying both cgroup_fd and cgroup_id is invalid. */
+static void test_invalid_cgroup_spec(struct cgroup_iter *skel)
+{
+	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
+	union bpf_iter_link_info linfo;
+	struct bpf_link *link;
+
+	memset(&linfo, 0, sizeof(linfo));
+	linfo.cgroup.cgroup_fd = (__u32)cg_fd[PARENT];
+	linfo.cgroup.cgroup_id = (__u64)cg_id[PARENT];
+	opts.link_info = &linfo;
+	opts.link_info_len = sizeof(linfo);
+
+	link = bpf_program__attach_iter(skel->progs.cgroup_id_printer, &opts);
+	ASSERT_ERR_PTR(link, "attach_iter");
+	bpf_link__destroy(link);
+}
+
+/* Preorder walk prints parent and child in order. */
+static void test_walk_preorder(struct cgroup_iter *skel)
+{
+	snprintf(expected_output, sizeof(expected_output),
+		 PROLOGUE "%8llu\n%8llu\n%8llu\n" EPILOGUE,
+		 cg_id[PARENT], cg_id[CHILD1], cg_id[CHILD2]);
+
+	read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT],
+			      BPF_CGROUP_ITER_DESCENDANTS_PRE, "preorder");
+}
+
+/* Postorder walk prints child and parent in order. */
+static void test_walk_postorder(struct cgroup_iter *skel)
+{
+	snprintf(expected_output, sizeof(expected_output),
+		 PROLOGUE "%8llu\n%8llu\n%8llu\n" EPILOGUE,
+		 cg_id[CHILD1], cg_id[CHILD2], cg_id[PARENT]);
+
+	read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT],
+			      BPF_CGROUP_ITER_DESCENDANTS_POST, "postorder");
+}
+
+/* Walking parents prints parent and then root. */
+static void test_walk_ancestors_up(struct cgroup_iter *skel)
+{
+	/* terminate the walk when ROOT is met. */
+	skel->bss->terminal_cgroup = cg_id[ROOT];
+
+	snprintf(expected_output, sizeof(expected_output),
+		 PROLOGUE "%8llu\n%8llu\n" EPILOGUE,
+		 cg_id[PARENT], cg_id[ROOT]);
+
+	read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT],
+			      BPF_CGROUP_ITER_ANCESTORS_UP, "ancestors_up");
+
+	skel->bss->terminal_cgroup = 0;
+}
+
+/* Early termination prints parent only. */
+static void test_early_termination(struct cgroup_iter *skel)
+{
+	/* terminate the walk after the first element is processed. */
+	skel->bss->terminate_early = 1;
+
+	snprintf(expected_output, sizeof(expected_output),
+		 PROLOGUE "%8llu\n" EPILOGUE, cg_id[PARENT]);
+
+	read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT],
+			      BPF_CGROUP_ITER_DESCENDANTS_PRE, "early_termination");
+
+	skel->bss->terminate_early = 0;
+}
+
+/* Waling self prints self only. */
+static void test_walk_self_only(struct cgroup_iter *skel)
+{
+	snprintf(expected_output, sizeof(expected_output),
+		 PROLOGUE "%8llu\n" EPILOGUE, cg_id[PARENT]);
+
+	read_from_cgroup_iter(skel->progs.cgroup_id_printer, cg_fd[PARENT],
+			      BPF_CGROUP_ITER_SELF_ONLY, "self_only");
+}
+
+void test_cgroup_iter(void)
+{
+	struct cgroup_iter *skel = NULL;
+
+	if (setup_cgroup_environment())
+		return;
+
+	if (setup_cgroups())
+		goto out;
+
+	skel = cgroup_iter__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "cgroup_iter__open_and_load"))
+		goto out;
+
+	if (test__start_subtest("cgroup_iter__invalid_cgroup"))
+		test_invalid_cgroup(skel);
+	if (test__start_subtest("cgroup_iter__invalid_cgroup_spec"))
+		test_invalid_cgroup_spec(skel);
+	if (test__start_subtest("cgroup_iter__preorder"))
+		test_walk_preorder(skel);
+	if (test__start_subtest("cgroup_iter__postorder"))
+		test_walk_postorder(skel);
+	if (test__start_subtest("cgroup_iter__ancestors_up_walk"))
+		test_walk_ancestors_up(skel);
+	if (test__start_subtest("cgroup_iter__early_termination"))
+		test_early_termination(skel);
+	if (test__start_subtest("cgroup_iter__self_only"))
+		test_walk_self_only(skel);
+out:
+	cgroup_iter__destroy(skel);
+	cleanup_cgroups();
+	cleanup_cgroup_environment();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_link.c b/tools/testing/selftests/bpf/prog_tests/cgroup_link.c
index 9e6e6aad347c..15093a69510e 100644
--- a/tools/testing/selftests/bpf/prog_tests/cgroup_link.c
+++ b/tools/testing/selftests/bpf/prog_tests/cgroup_link.c
@@ -71,10 +71,9 @@ void serial_test_cgroup_link(void)
 
 	ping_and_check(cg_nr, 0);
 
-	/* query the number of effective progs and attach flags in root cg */
+	/* query the number of attached progs and attach flags in root cg */
 	err = bpf_prog_query(cgs[0].fd, BPF_CGROUP_INET_EGRESS,
-			     BPF_F_QUERY_EFFECTIVE, &attach_flags, NULL,
-			     &prog_cnt);
+			     0, &attach_flags, NULL, &prog_cnt);
 	CHECK_FAIL(err);
 	CHECK_FAIL(attach_flags != BPF_F_ALLOW_MULTI);
 	if (CHECK(prog_cnt != 1, "effect_cnt", "exp %d, got %d\n", 1, prog_cnt))
@@ -85,17 +84,15 @@ void serial_test_cgroup_link(void)
 			     BPF_F_QUERY_EFFECTIVE, NULL, NULL,
 			     &prog_cnt);
 	CHECK_FAIL(err);
-	CHECK_FAIL(attach_flags != BPF_F_ALLOW_MULTI);
 	if (CHECK(prog_cnt != cg_nr, "effect_cnt", "exp %d, got %d\n",
 		  cg_nr, prog_cnt))
 		goto cleanup;
 
 	/* query the effective prog IDs in last cg */
 	err = bpf_prog_query(cgs[last_cg].fd, BPF_CGROUP_INET_EGRESS,
-			     BPF_F_QUERY_EFFECTIVE, &attach_flags,
-			     prog_ids, &prog_cnt);
+			     BPF_F_QUERY_EFFECTIVE, NULL, prog_ids,
+			     &prog_cnt);
 	CHECK_FAIL(err);
-	CHECK_FAIL(attach_flags != BPF_F_ALLOW_MULTI);
 	if (CHECK(prog_cnt != cg_nr, "effect_cnt", "exp %d, got %d\n",
 		  cg_nr, prog_cnt))
 		goto cleanup;
diff --git a/tools/testing/selftests/bpf/prog_tests/connect_force_port.c b/tools/testing/selftests/bpf/prog_tests/connect_force_port.c
index 9c4325f4aef2..24d553109f8d 100644
--- a/tools/testing/selftests/bpf/prog_tests/connect_force_port.c
+++ b/tools/testing/selftests/bpf/prog_tests/connect_force_port.c
@@ -53,7 +53,7 @@ static int run_test(int cgroup_fd, int server_fd, int family, int type)
 	__u16 expected_peer_port = 60000;
 	struct bpf_program *prog;
 	struct bpf_object *obj;
-	const char *obj_file = v4 ? "connect_force_port4.o" : "connect_force_port6.o";
+	const char *obj_file = v4 ? "connect_force_port4.bpf.o" : "connect_force_port6.bpf.o";
 	int fd, err;
 	__u32 duration = 0;
 
diff --git a/tools/testing/selftests/bpf/prog_tests/connect_ping.c b/tools/testing/selftests/bpf/prog_tests/connect_ping.c
new file mode 100644
index 000000000000..289218c2216c
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/connect_ping.c
@@ -0,0 +1,178 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/*
+ * Copyright 2022 Google LLC.
+ */
+
+#define _GNU_SOURCE
+#include <sys/mount.h>
+
+#include "test_progs.h"
+#include "cgroup_helpers.h"
+#include "network_helpers.h"
+
+#include "connect_ping.skel.h"
+
+/* 2001:db8::1 */
+#define BINDADDR_V6 { { { 0x20,0x01,0x0d,0xb8,0,0,0,0,0,0,0,0,0,0,0,1 } } }
+static const struct in6_addr bindaddr_v6 = BINDADDR_V6;
+
+static void subtest(int cgroup_fd, struct connect_ping *skel,
+		    int family, int do_bind)
+{
+	struct sockaddr_in sa4 = {
+		.sin_family = AF_INET,
+		.sin_addr.s_addr = htonl(INADDR_LOOPBACK),
+	};
+	struct sockaddr_in6 sa6 = {
+		.sin6_family = AF_INET6,
+		.sin6_addr = IN6ADDR_LOOPBACK_INIT,
+	};
+	struct sockaddr *sa;
+	socklen_t sa_len;
+	int protocol;
+	int sock_fd;
+
+	switch (family) {
+	case AF_INET:
+		sa = (struct sockaddr *)&sa4;
+		sa_len = sizeof(sa4);
+		protocol = IPPROTO_ICMP;
+		break;
+	case AF_INET6:
+		sa = (struct sockaddr *)&sa6;
+		sa_len = sizeof(sa6);
+		protocol = IPPROTO_ICMPV6;
+		break;
+	}
+
+	memset(skel->bss, 0, sizeof(*skel->bss));
+	skel->bss->do_bind = do_bind;
+
+	sock_fd = socket(family, SOCK_DGRAM, protocol);
+	if (!ASSERT_GE(sock_fd, 0, "sock-create"))
+		return;
+
+	if (!ASSERT_OK(connect(sock_fd, sa, sa_len), "connect"))
+		goto close_sock;
+
+	if (!ASSERT_EQ(skel->bss->invocations_v4, family == AF_INET ? 1 : 0,
+		       "invocations_v4"))
+		goto close_sock;
+	if (!ASSERT_EQ(skel->bss->invocations_v6, family == AF_INET6 ? 1 : 0,
+		       "invocations_v6"))
+		goto close_sock;
+	if (!ASSERT_EQ(skel->bss->has_error, 0, "has_error"))
+		goto close_sock;
+
+	if (!ASSERT_OK(getsockname(sock_fd, sa, &sa_len),
+		       "getsockname"))
+		goto close_sock;
+
+	switch (family) {
+	case AF_INET:
+		if (!ASSERT_EQ(sa4.sin_family, family, "sin_family"))
+			goto close_sock;
+		if (!ASSERT_EQ(sa4.sin_addr.s_addr,
+			       htonl(do_bind ? 0x01010101 : INADDR_LOOPBACK),
+			       "sin_addr"))
+			goto close_sock;
+		break;
+	case AF_INET6:
+		if (!ASSERT_EQ(sa6.sin6_family, AF_INET6, "sin6_family"))
+			goto close_sock;
+		if (!ASSERT_EQ(memcmp(&sa6.sin6_addr,
+				      do_bind ? &bindaddr_v6 : &in6addr_loopback,
+				      sizeof(sa6.sin6_addr)),
+			       0, "sin6_addr"))
+			goto close_sock;
+		break;
+	}
+
+close_sock:
+	close(sock_fd);
+}
+
+void test_connect_ping(void)
+{
+	struct connect_ping *skel;
+	int cgroup_fd;
+
+	if (!ASSERT_OK(unshare(CLONE_NEWNET | CLONE_NEWNS), "unshare"))
+		return;
+
+	/* overmount sysfs, and making original sysfs private so overmount
+	 * does not propagate to other mntns.
+	 */
+	if (!ASSERT_OK(mount("none", "/sys", NULL, MS_PRIVATE, NULL),
+		       "remount-private-sys"))
+		return;
+	if (!ASSERT_OK(mount("sysfs", "/sys", "sysfs", 0, NULL),
+		       "mount-sys"))
+		return;
+	if (!ASSERT_OK(mount("bpffs", "/sys/fs/bpf", "bpf", 0, NULL),
+		       "mount-bpf"))
+		goto clean_mount;
+
+	if (!ASSERT_OK(system("ip link set dev lo up"), "lo-up"))
+		goto clean_mount;
+	if (!ASSERT_OK(system("ip addr add 1.1.1.1 dev lo"), "lo-addr-v4"))
+		goto clean_mount;
+	if (!ASSERT_OK(system("ip -6 addr add 2001:db8::1 dev lo"), "lo-addr-v6"))
+		goto clean_mount;
+	if (write_sysctl("/proc/sys/net/ipv4/ping_group_range", "0 0"))
+		goto clean_mount;
+
+	cgroup_fd = test__join_cgroup("/connect_ping");
+	if (!ASSERT_GE(cgroup_fd, 0, "cg-create"))
+		goto clean_mount;
+
+	skel = connect_ping__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel-load"))
+		goto close_cgroup;
+	skel->links.connect_v4_prog =
+		bpf_program__attach_cgroup(skel->progs.connect_v4_prog, cgroup_fd);
+	if (!ASSERT_OK_PTR(skel->links.connect_v4_prog, "cg-attach-v4"))
+		goto skel_destroy;
+	skel->links.connect_v6_prog =
+		bpf_program__attach_cgroup(skel->progs.connect_v6_prog, cgroup_fd);
+	if (!ASSERT_OK_PTR(skel->links.connect_v6_prog, "cg-attach-v6"))
+		goto skel_destroy;
+
+	/* Connect a v4 ping socket to localhost, assert that only v4 is called,
+	 * and called exactly once, and that the socket's bound address is
+	 * original loopback address.
+	 */
+	if (test__start_subtest("ipv4"))
+		subtest(cgroup_fd, skel, AF_INET, 0);
+
+	/* Connect a v4 ping socket to localhost, assert that only v4 is called,
+	 * and called exactly once, and that the socket's bound address is
+	 * address we explicitly bound.
+	 */
+	if (test__start_subtest("ipv4-bind"))
+		subtest(cgroup_fd, skel, AF_INET, 1);
+
+	/* Connect a v6 ping socket to localhost, assert that only v6 is called,
+	 * and called exactly once, and that the socket's bound address is
+	 * original loopback address.
+	 */
+	if (test__start_subtest("ipv6"))
+		subtest(cgroup_fd, skel, AF_INET6, 0);
+
+	/* Connect a v6 ping socket to localhost, assert that only v6 is called,
+	 * and called exactly once, and that the socket's bound address is
+	 * address we explicitly bound.
+	 */
+	if (test__start_subtest("ipv6-bind"))
+		subtest(cgroup_fd, skel, AF_INET6, 1);
+
+skel_destroy:
+	connect_ping__destroy(skel);
+
+close_cgroup:
+	close(cgroup_fd);
+
+clean_mount:
+	umount2("/sys", MNT_DETACH);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c
index c8655ba9a88f..47f42e680105 100644
--- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c
+++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c
@@ -13,7 +13,7 @@ static int duration = 0;
 
 #define MODULES_CASE(name, pg_name, tp_name) {				\
 	.case_name = name,						\
-	.bpf_obj_file = "test_core_reloc_module.o",			\
+	.bpf_obj_file = "test_core_reloc_module.bpf.o",			\
 	.btf_src_file = NULL, /* find in kernel module BTFs */		\
 	.input = "",							\
 	.input_len = 0,							\
@@ -43,8 +43,8 @@ static int duration = 0;
 
 #define FLAVORS_CASE_COMMON(name)					\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_flavors.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_flavors.bpf.o",		\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_flavors"				\
 
@@ -68,8 +68,8 @@ static int duration = 0;
 
 #define NESTING_CASE_COMMON(name)					\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_nesting.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_nesting.bpf.o",		\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_nesting"				\
 
@@ -96,8 +96,8 @@ static int duration = 0;
 
 #define ARRAYS_CASE_COMMON(name)					\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_arrays.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_arrays.bpf.o",			\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_arrays"					\
 
@@ -130,8 +130,8 @@ static int duration = 0;
 
 #define PRIMITIVES_CASE_COMMON(name)					\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_primitives.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_primitives.bpf.o",		\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_primitives"				\
 
@@ -150,8 +150,8 @@ static int duration = 0;
 
 #define MODS_CASE(name) {						\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_mods.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_mods.bpf.o",			\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.input = STRUCT_TO_CHAR_PTR(core_reloc_##name) {		\
 		.a = 1,							\
 		.b = 2,							\
@@ -174,8 +174,8 @@ static int duration = 0;
 
 #define PTR_AS_ARR_CASE(name) {						\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_ptr_as_arr.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_ptr_as_arr.bpf.o",		\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.input = (const char *)&(struct core_reloc_##name []){		\
 		{ .a = 1 },						\
 		{ .a = 2 },						\
@@ -203,8 +203,8 @@ static int duration = 0;
 
 #define INTS_CASE_COMMON(name)						\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_ints.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_ints.bpf.o",			\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_ints"
 
@@ -223,18 +223,18 @@ static int duration = 0;
 
 #define FIELD_EXISTS_CASE_COMMON(name)					\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_existence.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_existence.bpf.o",		\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_existence"
 
 #define BITFIELDS_CASE_COMMON(objfile, test_name_prefix,  name)		\
 	.case_name = test_name_prefix#name,				\
 	.bpf_obj_file = objfile,					\
-	.btf_src_file = "btf__core_reloc_" #name ".o"
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o"
 
 #define BITFIELDS_CASE(name, ...) {					\
-	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o",	\
+	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.bpf.o",	\
 			      "probed:", name),				\
 	.input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__,	\
 	.input_len = sizeof(struct core_reloc_##name),			\
@@ -244,7 +244,7 @@ static int duration = 0;
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_bitfields",				\
 }, {									\
-	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o",	\
+	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.bpf.o",	\
 			      "direct:", name),				\
 	.input = STRUCT_TO_CHAR_PTR(core_reloc_##name) __VA_ARGS__,	\
 	.input_len = sizeof(struct core_reloc_##name),			\
@@ -256,14 +256,14 @@ static int duration = 0;
 
 
 #define BITFIELDS_ERR_CASE(name) {					\
-	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.o",	\
+	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_probed.bpf.o",	\
 			      "probed:", name),				\
 	.fails = true,							\
-	.run_btfgen_fails = true,							\
+	.run_btfgen_fails = true,					\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_bitfields",				\
 }, {									\
-	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.o",	\
+	BITFIELDS_CASE_COMMON("test_core_reloc_bitfields_direct.bpf.o",	\
 			      "direct:", name),				\
 	.fails = true,							\
 	.run_btfgen_fails = true,							\
@@ -272,8 +272,8 @@ static int duration = 0;
 
 #define SIZE_CASE_COMMON(name)						\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_size.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_size.bpf.o",			\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_size"
 
@@ -307,13 +307,13 @@ static int duration = 0;
 #define SIZE_ERR_CASE(name) {						\
 	SIZE_CASE_COMMON(name),						\
 	.fails = true,							\
-	.run_btfgen_fails = true,							\
+	.run_btfgen_fails = true,					\
 }
 
 #define TYPE_BASED_CASE_COMMON(name)					\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_type_based.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_type_based.bpf.o",		\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_type_based"
 
@@ -331,8 +331,8 @@ static int duration = 0;
 
 #define TYPE_ID_CASE_COMMON(name)					\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_type_id.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_type_id.bpf.o",		\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_type_id"
 
@@ -350,8 +350,8 @@ static int duration = 0;
 
 #define ENUMVAL_CASE_COMMON(name)					\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_enumval.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_enumval.bpf.o",		\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_enumval"
 
@@ -369,8 +369,8 @@ static int duration = 0;
 
 #define ENUM64VAL_CASE_COMMON(name)					\
 	.case_name = #name,						\
-	.bpf_obj_file = "test_core_reloc_enum64val.o",			\
-	.btf_src_file = "btf__core_reloc_" #name ".o",			\
+	.bpf_obj_file = "test_core_reloc_enum64val.bpf.o",		\
+	.btf_src_file = "btf__core_reloc_" #name ".bpf.o",		\
 	.raw_tp_name = "sys_enter",					\
 	.prog_name = "test_core_enum64val"
 
@@ -547,7 +547,7 @@ static const struct core_reloc_test_case test_cases[] = {
 	/* validate we can find kernel image and use its BTF for relocs */
 	{
 		.case_name = "kernel",
-		.bpf_obj_file = "test_core_reloc_kernel.o",
+		.bpf_obj_file = "test_core_reloc_kernel.bpf.o",
 		.btf_src_file = NULL, /* load from /lib/modules/$(uname -r) */
 		.input = "",
 		.input_len = 0,
@@ -629,8 +629,8 @@ static const struct core_reloc_test_case test_cases[] = {
 	/* validate edge cases of capturing relocations */
 	{
 		.case_name = "misc",
-		.bpf_obj_file = "test_core_reloc_misc.o",
-		.btf_src_file = "btf__core_reloc_misc.o",
+		.bpf_obj_file = "test_core_reloc_misc.bpf.o",
+		.btf_src_file = "btf__core_reloc_misc.bpf.o",
 		.input = (const char *)&(struct core_reloc_misc_extensible[]){
 			{ .a = 1 },
 			{ .a = 2 }, /* not read */
diff --git a/tools/testing/selftests/bpf/prog_tests/deny_namespace.c b/tools/testing/selftests/bpf/prog_tests/deny_namespace.c
new file mode 100644
index 000000000000..1bc6241b755b
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/deny_namespace.c
@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: GPL-2.0
+#define _GNU_SOURCE
+#include <test_progs.h>
+#include "test_deny_namespace.skel.h"
+#include <sched.h>
+#include "cap_helpers.h"
+#include <stdio.h>
+
+static int wait_for_pid(pid_t pid)
+{
+	int status, ret;
+
+again:
+	ret = waitpid(pid, &status, 0);
+	if (ret == -1) {
+		if (errno == EINTR)
+			goto again;
+
+		return -1;
+	}
+
+	if (!WIFEXITED(status))
+		return -1;
+
+	return WEXITSTATUS(status);
+}
+
+/* negative return value -> some internal error
+ * positive return value -> userns creation failed
+ * 0                     -> userns creation succeeded
+ */
+static int create_user_ns(void)
+{
+	pid_t pid;
+
+	pid = fork();
+	if (pid < 0)
+		return -1;
+
+	if (pid == 0) {
+		if (unshare(CLONE_NEWUSER))
+			_exit(EXIT_FAILURE);
+		_exit(EXIT_SUCCESS);
+	}
+
+	return wait_for_pid(pid);
+}
+
+static void test_userns_create_bpf(void)
+{
+	__u32 cap_mask = 1ULL << CAP_SYS_ADMIN;
+	__u64 old_caps = 0;
+
+	cap_enable_effective(cap_mask, &old_caps);
+
+	ASSERT_OK(create_user_ns(), "priv new user ns");
+
+	cap_disable_effective(cap_mask, &old_caps);
+
+	ASSERT_EQ(create_user_ns(), EPERM, "unpriv new user ns");
+
+	if (cap_mask & old_caps)
+		cap_enable_effective(cap_mask, NULL);
+}
+
+static void test_unpriv_userns_create_no_bpf(void)
+{
+	__u32 cap_mask = 1ULL << CAP_SYS_ADMIN;
+	__u64 old_caps = 0;
+
+	cap_disable_effective(cap_mask, &old_caps);
+
+	ASSERT_OK(create_user_ns(), "no-bpf unpriv new user ns");
+
+	if (cap_mask & old_caps)
+		cap_enable_effective(cap_mask, NULL);
+}
+
+void test_deny_namespace(void)
+{
+	struct test_deny_namespace *skel = NULL;
+	int err;
+
+	if (test__start_subtest("unpriv_userns_create_no_bpf"))
+		test_unpriv_userns_create_no_bpf();
+
+	skel = test_deny_namespace__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "skel load"))
+		goto close_prog;
+
+	err = test_deny_namespace__attach(skel);
+	if (!ASSERT_OK(err, "attach"))
+		goto close_prog;
+
+	if (test__start_subtest("userns_create_bpf"))
+		test_userns_create_bpf();
+
+	test_deny_namespace__detach(skel);
+
+close_prog:
+	test_deny_namespace__destroy(skel);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/dynptr.c b/tools/testing/selftests/bpf/prog_tests/dynptr.c
index 3c7aa82b98e2..8fc4e6c02bfd 100644
--- a/tools/testing/selftests/bpf/prog_tests/dynptr.c
+++ b/tools/testing/selftests/bpf/prog_tests/dynptr.c
@@ -22,14 +22,15 @@ static struct {
 	{"add_dynptr_to_map2", "invalid indirect read from stack"},
 	{"data_slice_out_of_bounds_ringbuf", "value is outside of the allowed memory range"},
 	{"data_slice_out_of_bounds_map_value", "value is outside of the allowed memory range"},
-	{"data_slice_use_after_release", "invalid mem access 'scalar'"},
+	{"data_slice_use_after_release1", "invalid mem access 'scalar'"},
+	{"data_slice_use_after_release2", "invalid mem access 'scalar'"},
 	{"data_slice_missing_null_check1", "invalid mem access 'mem_or_null'"},
 	{"data_slice_missing_null_check2", "invalid mem access 'mem_or_null'"},
 	{"invalid_helper1", "invalid indirect read from stack"},
 	{"invalid_helper2", "Expected an initialized dynptr as arg #3"},
 	{"invalid_write1", "Expected an initialized dynptr as arg #1"},
 	{"invalid_write2", "Expected an initialized dynptr as arg #3"},
-	{"invalid_write3", "Expected an initialized ringbuf dynptr as arg #1"},
+	{"invalid_write3", "Expected an initialized dynptr as arg #1"},
 	{"invalid_write4", "arg 1 is an unacquired reference"},
 	{"invalid_read1", "invalid read from stack"},
 	{"invalid_read2", "cannot pass in dynptr at an offset"},
diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c
index da860b07abb5..d1e32e792536 100644
--- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c
+++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c
@@ -174,8 +174,8 @@ static void test_target_no_callees(void)
 	const char *prog_name[] = {
 		"fexit/test_pkt_md_access",
 	};
-	test_fexit_bpf2bpf_common("./fexit_bpf2bpf_simple.o",
-				  "./test_pkt_md_access.o",
+	test_fexit_bpf2bpf_common("./fexit_bpf2bpf_simple.bpf.o",
+				  "./test_pkt_md_access.bpf.o",
 				  ARRAY_SIZE(prog_name),
 				  prog_name, true, NULL);
 }
@@ -188,8 +188,8 @@ static void test_target_yes_callees(void)
 		"fexit/test_pkt_access_subprog2",
 		"fexit/test_pkt_access_subprog3",
 	};
-	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o",
-				  "./test_pkt_access.o",
+	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.bpf.o",
+				  "./test_pkt_access.bpf.o",
 				  ARRAY_SIZE(prog_name),
 				  prog_name, true, NULL);
 }
@@ -206,8 +206,8 @@ static void test_func_replace(void)
 		"freplace/get_constant",
 		"freplace/test_pkt_write_access_subprog",
 	};
-	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.o",
-				  "./test_pkt_access.o",
+	test_fexit_bpf2bpf_common("./fexit_bpf2bpf.bpf.o",
+				  "./test_pkt_access.bpf.o",
 				  ARRAY_SIZE(prog_name),
 				  prog_name, true, NULL);
 }
@@ -217,8 +217,8 @@ static void test_func_replace_verify(void)
 	const char *prog_name[] = {
 		"freplace/do_bind",
 	};
-	test_fexit_bpf2bpf_common("./freplace_connect4.o",
-				  "./connect4_prog.o",
+	test_fexit_bpf2bpf_common("./freplace_connect4.bpf.o",
+				  "./connect4_prog.bpf.o",
 				  ARRAY_SIZE(prog_name),
 				  prog_name, false, NULL);
 }
@@ -227,7 +227,7 @@ static int test_second_attach(struct bpf_object *obj)
 {
 	const char *prog_name = "security_new_get_constant";
 	const char *tgt_name = "get_constant";
-	const char *tgt_obj_file = "./test_pkt_access.o";
+	const char *tgt_obj_file = "./test_pkt_access.bpf.o";
 	struct bpf_program *prog = NULL;
 	struct bpf_object *tgt_obj;
 	struct bpf_link *link;
@@ -272,8 +272,8 @@ static void test_func_replace_multi(void)
 	const char *prog_name[] = {
 		"freplace/get_constant",
 	};
-	test_fexit_bpf2bpf_common("./freplace_get_constant.o",
-				  "./test_pkt_access.o",
+	test_fexit_bpf2bpf_common("./freplace_get_constant.bpf.o",
+				  "./test_pkt_access.bpf.o",
 				  ARRAY_SIZE(prog_name),
 				  prog_name, true, test_second_attach);
 }
@@ -281,10 +281,10 @@ static void test_func_replace_multi(void)
 static void test_fmod_ret_freplace(void)
 {
 	struct bpf_object *freplace_obj = NULL, *pkt_obj, *fmod_obj = NULL;
-	const char *freplace_name = "./freplace_get_constant.o";
-	const char *fmod_ret_name = "./fmod_ret_freplace.o";
+	const char *freplace_name = "./freplace_get_constant.bpf.o";
+	const char *fmod_ret_name = "./fmod_ret_freplace.bpf.o";
 	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts);
-	const char *tgt_name = "./test_pkt_access.o";
+	const char *tgt_name = "./test_pkt_access.bpf.o";
 	struct bpf_link *freplace_link = NULL;
 	struct bpf_program *prog;
 	__u32 duration = 0;
@@ -339,8 +339,8 @@ static void test_func_sockmap_update(void)
 	const char *prog_name[] = {
 		"freplace/cls_redirect",
 	};
-	test_fexit_bpf2bpf_common("./freplace_cls_redirect.o",
-				  "./test_cls_redirect.o",
+	test_fexit_bpf2bpf_common("./freplace_cls_redirect.bpf.o",
+				  "./test_cls_redirect.bpf.o",
 				  ARRAY_SIZE(prog_name),
 				  prog_name, false, NULL);
 }
@@ -385,15 +385,15 @@ close_prog:
 static void test_func_replace_return_code(void)
 {
 	/* test invalid return code in the replaced program */
-	test_obj_load_failure_common("./freplace_connect_v4_prog.o",
-				     "./connect4_prog.o");
+	test_obj_load_failure_common("./freplace_connect_v4_prog.bpf.o",
+				     "./connect4_prog.bpf.o");
 }
 
 static void test_func_map_prog_compatibility(void)
 {
 	/* test with spin lock map value in the replaced program */
-	test_obj_load_failure_common("./freplace_attach_probe.o",
-				     "./test_attach_probe.o");
+	test_obj_load_failure_common("./freplace_attach_probe.bpf.o",
+				     "./test_attach_probe.bpf.o");
 }
 
 static void test_func_replace_global_func(void)
@@ -402,8 +402,8 @@ static void test_func_replace_global_func(void)
 		"freplace/test_pkt_access",
 	};
 
-	test_fexit_bpf2bpf_common("./freplace_global_func.o",
-				  "./test_pkt_access.o",
+	test_fexit_bpf2bpf_common("./freplace_global_func.bpf.o",
+				  "./test_pkt_access.bpf.o",
 				  ARRAY_SIZE(prog_name),
 				  prog_name, false, NULL);
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector.c
index 0c1661ea996e..7acca37a3d2b 100644
--- a/tools/testing/selftests/bpf/prog_tests/flow_dissector.c
+++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector.c
@@ -8,6 +8,8 @@
 
 #include "bpf_flow.skel.h"
 
+#define FLOW_CONTINUE_SADDR 0x7f00007f /* 127.0.0.127 */
+
 #ifndef IP_MF
 #define IP_MF 0x2000
 #endif
@@ -100,6 +102,7 @@ struct test {
 	} pkt;
 	struct bpf_flow_keys keys;
 	__u32 flags;
+	__u32 retval;
 };
 
 #define VLAN_HLEN	4
@@ -126,6 +129,7 @@ struct test tests[] = {
 			.sport = 80,
 			.dport = 8080,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv6",
@@ -146,6 +150,7 @@ struct test tests[] = {
 			.sport = 80,
 			.dport = 8080,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "802.1q-ipv4",
@@ -168,6 +173,7 @@ struct test tests[] = {
 			.sport = 80,
 			.dport = 8080,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "802.1ad-ipv6",
@@ -191,6 +197,7 @@ struct test tests[] = {
 			.sport = 80,
 			.dport = 8080,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv4-frag",
@@ -217,6 +224,7 @@ struct test tests[] = {
 			.dport = 8080,
 		},
 		.flags = BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG,
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv4-no-frag",
@@ -239,6 +247,7 @@ struct test tests[] = {
 			.is_frag = true,
 			.is_first_frag = true,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv6-frag",
@@ -265,6 +274,7 @@ struct test tests[] = {
 			.dport = 8080,
 		},
 		.flags = BPF_FLOW_DISSECTOR_F_PARSE_1ST_FRAG,
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv6-no-frag",
@@ -287,6 +297,7 @@ struct test tests[] = {
 			.is_frag = true,
 			.is_first_frag = true,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv6-flow-label",
@@ -309,6 +320,7 @@ struct test tests[] = {
 			.dport = 8080,
 			.flow_label = __bpf_constant_htonl(0xbeeef),
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipv6-no-flow-label",
@@ -331,6 +343,7 @@ struct test tests[] = {
 			.flow_label = __bpf_constant_htonl(0xbeeef),
 		},
 		.flags = BPF_FLOW_DISSECTOR_F_STOP_AT_FLOW_LABEL,
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipip-encap",
@@ -359,6 +372,7 @@ struct test tests[] = {
 			.sport = 80,
 			.dport = 8080,
 		},
+		.retval = BPF_OK,
 	},
 	{
 		.name = "ipip-no-encap",
@@ -386,6 +400,26 @@ struct test tests[] = {
 			.is_encap = true,
 		},
 		.flags = BPF_FLOW_DISSECTOR_F_STOP_AT_ENCAP,
+		.retval = BPF_OK,
+	},
+	{
+		.name = "ipip-encap-dissector-continue",
+		.pkt.ipip = {
+			.eth.h_proto = __bpf_constant_htons(ETH_P_IP),
+			.iph.ihl = 5,
+			.iph.protocol = IPPROTO_IPIP,
+			.iph.tot_len = __bpf_constant_htons(MAGIC_BYTES),
+			.iph.saddr = __bpf_constant_htonl(FLOW_CONTINUE_SADDR),
+			.iph_inner.ihl = 5,
+			.iph_inner.protocol = IPPROTO_TCP,
+			.iph_inner.tot_len =
+				__bpf_constant_htons(MAGIC_BYTES) -
+				sizeof(struct iphdr),
+			.tcp.doff = 5,
+			.tcp.source = 99,
+			.tcp.dest = 9090,
+		},
+		.retval = BPF_FLOW_DISSECTOR_CONTINUE,
 	},
 };
 
@@ -503,6 +537,10 @@ static void run_tests_skb_less(int tap_fd, struct bpf_map *keys)
 		err = tx_tap(tap_fd, &tests[i].pkt, sizeof(tests[i].pkt));
 		CHECK(err < 0, "tx_tap", "err %d errno %d\n", err, errno);
 
+		/* check the stored flow_keys only if BPF_OK expected */
+		if (tests[i].retval != BPF_OK)
+			continue;
+
 		err = bpf_map_lookup_elem(keys_fd, &key, &flow_keys);
 		ASSERT_OK(err, "bpf_map_lookup_elem");
 
@@ -588,7 +626,11 @@ void test_flow_dissector(void)
 
 		err = bpf_prog_test_run_opts(prog_fd, &topts);
 		ASSERT_OK(err, "test_run");
-		ASSERT_EQ(topts.retval, 1, "test_run retval");
+		ASSERT_EQ(topts.retval, tests[i].retval, "test_run retval");
+
+		/* check the resulting flow_keys only if BPF_OK returned */
+		if (topts.retval != BPF_OK)
+			continue;
 		ASSERT_EQ(topts.data_size_out, sizeof(flow_keys),
 			  "test_run data_size_out");
 		CHECK_FLOW_KEYS(tests[i].name, flow_keys, tests[i].keys);
diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c
index 36afb409c25f..c7a47b57ac91 100644
--- a/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c
+++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c
@@ -44,7 +44,7 @@ void serial_test_flow_dissector_load_bytes(void)
 	ASSERT_OK(err, "test_run");
 	ASSERT_EQ(topts.data_size_out, sizeof(flow_keys),
 		  "test_run data_size_out");
-	ASSERT_EQ(topts.retval, 1, "test_run retval");
+	ASSERT_EQ(topts.retval, BPF_OK, "test_run retval");
 
 	if (fd >= -1)
 		close(fd);
diff --git a/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c b/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c
index 938dbd4d7c2f..fede8ef58b5b 100644
--- a/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c
+++ b/tools/testing/selftests/bpf/prog_tests/get_func_ip_test.c
@@ -2,7 +2,7 @@
 #include <test_progs.h>
 #include "get_func_ip_test.skel.h"
 
-void test_get_func_ip_test(void)
+static void test_function_entry(void)
 {
 	struct get_func_ip_test *skel = NULL;
 	int err, prog_fd;
@@ -12,14 +12,6 @@ void test_get_func_ip_test(void)
 	if (!ASSERT_OK_PTR(skel, "get_func_ip_test__open"))
 		return;
 
-	/* test6 is x86_64 specifc because of the instruction
-	 * offset, disabling it for all other archs
-	 */
-#ifndef __x86_64__
-	bpf_program__set_autoload(skel->progs.test6, false);
-	bpf_program__set_autoload(skel->progs.test7, false);
-#endif
-
 	err = get_func_ip_test__load(skel);
 	if (!ASSERT_OK(err, "get_func_ip_test__load"))
 		goto cleanup;
@@ -43,11 +35,56 @@ void test_get_func_ip_test(void)
 	ASSERT_EQ(skel->bss->test3_result, 1, "test3_result");
 	ASSERT_EQ(skel->bss->test4_result, 1, "test4_result");
 	ASSERT_EQ(skel->bss->test5_result, 1, "test5_result");
+
+cleanup:
+	get_func_ip_test__destroy(skel);
+}
+
+/* test6 is x86_64 specific because of the instruction
+ * offset, disabling it for all other archs
+ */
 #ifdef __x86_64__
+static void test_function_body(void)
+{
+	struct get_func_ip_test *skel = NULL;
+	LIBBPF_OPTS(bpf_test_run_opts, topts);
+	LIBBPF_OPTS(bpf_kprobe_opts, kopts);
+	struct bpf_link *link6 = NULL;
+	int err, prog_fd;
+
+	skel = get_func_ip_test__open();
+	if (!ASSERT_OK_PTR(skel, "get_func_ip_test__open"))
+		return;
+
+	bpf_program__set_autoload(skel->progs.test6, true);
+
+	err = get_func_ip_test__load(skel);
+	if (!ASSERT_OK(err, "get_func_ip_test__load"))
+		goto cleanup;
+
+	kopts.offset = skel->kconfig->CONFIG_X86_KERNEL_IBT ? 9 : 5;
+
+	link6 = bpf_program__attach_kprobe_opts(skel->progs.test6, "bpf_fentry_test6", &kopts);
+	if (!ASSERT_OK_PTR(link6, "link6"))
+		goto cleanup;
+
+	prog_fd = bpf_program__fd(skel->progs.test1);
+	err = bpf_prog_test_run_opts(prog_fd, &topts);
+	ASSERT_OK(err, "test_run");
+	ASSERT_EQ(topts.retval, 0, "test_run");
+
 	ASSERT_EQ(skel->bss->test6_result, 1, "test6_result");
-	ASSERT_EQ(skel->bss->test7_result, 1, "test7_result");
-#endif
 
 cleanup:
+	bpf_link__destroy(link6);
 	get_func_ip_test__destroy(skel);
 }
+#else
+#define test_function_body()
+#endif
+
+void test_get_func_ip_test(void)
+{
+	test_function_entry();
+	test_function_body();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c b/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c
index 16048978a1ef..858e0575f502 100644
--- a/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c
+++ b/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c
@@ -84,8 +84,8 @@ static void get_stack_print_output(void *ctx, int cpu, void *data, __u32 size)
 
 void test_get_stack_raw_tp(void)
 {
-	const char *file = "./test_get_stack_rawtp.o";
-	const char *file_err = "./test_get_stack_rawtp_err.o";
+	const char *file = "./test_get_stack_rawtp.bpf.o";
+	const char *file_err = "./test_get_stack_rawtp_err.bpf.o";
 	const char *prog_name = "bpf_prog1";
 	int i, err, prog_fd, exp_cnt = MAX_CNT_RAWTP;
 	struct perf_buffer *pb = NULL;
diff --git a/tools/testing/selftests/bpf/prog_tests/global_data.c b/tools/testing/selftests/bpf/prog_tests/global_data.c
index 027685858925..fadfb64e2a71 100644
--- a/tools/testing/selftests/bpf/prog_tests/global_data.c
+++ b/tools/testing/selftests/bpf/prog_tests/global_data.c
@@ -131,7 +131,7 @@ static void test_global_data_rdonly(struct bpf_object *obj, __u32 duration)
 
 void test_global_data(void)
 {
-	const char *file = "./test_global_data.o";
+	const char *file = "./test_global_data.bpf.o";
 	struct bpf_object *obj;
 	int err, prog_fd;
 	LIBBPF_OPTS(bpf_test_run_opts, topts,
diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c
index 57331c606964..8466332d7406 100644
--- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c
+++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c
@@ -3,7 +3,7 @@
 
 void test_global_data_init(void)
 {
-	const char *file = "./test_global_data.o";
+	const char *file = "./test_global_data.bpf.o";
 	int err = -ENOMEM, map_fd, zero = 0;
 	__u8 *buff = NULL, *newval = NULL;
 	struct bpf_object *obj;
diff --git a/tools/testing/selftests/bpf/prog_tests/global_func_args.c b/tools/testing/selftests/bpf/prog_tests/global_func_args.c
index 29039a36cce5..d997099f62d0 100644
--- a/tools/testing/selftests/bpf/prog_tests/global_func_args.c
+++ b/tools/testing/selftests/bpf/prog_tests/global_func_args.c
@@ -39,7 +39,7 @@ static void test_global_func_args0(struct bpf_object *obj)
 
 void test_global_func_args(void)
 {
-	const char *file = "./test_global_func_args.o";
+	const char *file = "./test_global_func_args.bpf.o";
 	struct bpf_object *obj;
 	int err, prog_fd;
 	LIBBPF_OPTS(bpf_test_run_opts, topts,
diff --git a/tools/testing/selftests/bpf/prog_tests/htab_update.c b/tools/testing/selftests/bpf/prog_tests/htab_update.c
new file mode 100644
index 000000000000..2bc85f4814f4
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/htab_update.c
@@ -0,0 +1,126 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2022. Huawei Technologies Co., Ltd */
+#define _GNU_SOURCE
+#include <sched.h>
+#include <stdbool.h>
+#include <test_progs.h>
+#include "htab_update.skel.h"
+
+struct htab_update_ctx {
+	int fd;
+	int loop;
+	bool stop;
+};
+
+static void test_reenter_update(void)
+{
+	struct htab_update *skel;
+	unsigned int key, value;
+	int err;
+
+	skel = htab_update__open();
+	if (!ASSERT_OK_PTR(skel, "htab_update__open"))
+		return;
+
+	/* lookup_elem_raw() may be inlined and find_kernel_btf_id() will return -ESRCH */
+	bpf_program__set_autoload(skel->progs.lookup_elem_raw, true);
+	err = htab_update__load(skel);
+	if (!ASSERT_TRUE(!err || err == -ESRCH, "htab_update__load") || err)
+		goto out;
+
+	skel->bss->pid = getpid();
+	err = htab_update__attach(skel);
+	if (!ASSERT_OK(err, "htab_update__attach"))
+		goto out;
+
+	/* Will trigger the reentrancy of bpf_map_update_elem() */
+	key = 0;
+	value = 0;
+	err = bpf_map_update_elem(bpf_map__fd(skel->maps.htab), &key, &value, 0);
+	if (!ASSERT_OK(err, "add element"))
+		goto out;
+
+	ASSERT_EQ(skel->bss->update_err, -EBUSY, "no reentrancy");
+out:
+	htab_update__destroy(skel);
+}
+
+static void *htab_update_thread(void *arg)
+{
+	struct htab_update_ctx *ctx = arg;
+	cpu_set_t cpus;
+	int i;
+
+	/* Pinned on CPU 0 */
+	CPU_ZERO(&cpus);
+	CPU_SET(0, &cpus);
+	pthread_setaffinity_np(pthread_self(), sizeof(cpus), &cpus);
+
+	i = 0;
+	while (i++ < ctx->loop && !ctx->stop) {
+		unsigned int key = 0, value = 0;
+		int err;
+
+		err = bpf_map_update_elem(ctx->fd, &key, &value, 0);
+		if (err) {
+			ctx->stop = true;
+			return (void *)(long)err;
+		}
+	}
+
+	return NULL;
+}
+
+static void test_concurrent_update(void)
+{
+	struct htab_update_ctx ctx;
+	struct htab_update *skel;
+	unsigned int i, nr;
+	pthread_t *tids;
+	int err;
+
+	skel = htab_update__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "htab_update__open_and_load"))
+		return;
+
+	ctx.fd = bpf_map__fd(skel->maps.htab);
+	ctx.loop = 1000;
+	ctx.stop = false;
+
+	nr = 4;
+	tids = calloc(nr, sizeof(*tids));
+	if (!ASSERT_NEQ(tids, NULL, "no mem"))
+		goto out;
+
+	for (i = 0; i < nr; i++) {
+		err = pthread_create(&tids[i], NULL, htab_update_thread, &ctx);
+		if (!ASSERT_OK(err, "pthread_create")) {
+			unsigned int j;
+
+			ctx.stop = true;
+			for (j = 0; j < i; j++)
+				pthread_join(tids[j], NULL);
+			goto out;
+		}
+	}
+
+	for (i = 0; i < nr; i++) {
+		void *thread_err = NULL;
+
+		pthread_join(tids[i], &thread_err);
+		ASSERT_EQ(thread_err, NULL, "update error");
+	}
+
+out:
+	if (tids)
+		free(tids);
+	htab_update__destroy(skel);
+}
+
+void test_htab_update(void)
+{
+	if (test__start_subtest("reenter_update"))
+		test_reenter_update();
+	if (test__start_subtest("concurrent_update"))
+		test_concurrent_update();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/kfree_skb.c b/tools/testing/selftests/bpf/prog_tests/kfree_skb.c
index 1cee6957285e..73579370bfbd 100644
--- a/tools/testing/selftests/bpf/prog_tests/kfree_skb.c
+++ b/tools/testing/selftests/bpf/prog_tests/kfree_skb.c
@@ -69,7 +69,7 @@ void serial_test_kfree_skb(void)
 	const int zero = 0;
 	bool test_ok[2];
 
-	err = bpf_prog_test_load("./test_pkt_access.o", BPF_PROG_TYPE_SCHED_CLS,
+	err = bpf_prog_test_load("./test_pkt_access.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
 				 &obj, &prog_fd);
 	if (CHECK(err, "prog_load sched cls", "err %d errno %d\n", err, errno))
 		return;
diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c
index c00eb974eb85..5af1ee8f0e6e 100644
--- a/tools/testing/selftests/bpf/prog_tests/kfunc_call.c
+++ b/tools/testing/selftests/bpf/prog_tests/kfunc_call.c
@@ -2,40 +2,229 @@
 /* Copyright (c) 2021 Facebook */
 #include <test_progs.h>
 #include <network_helpers.h>
+#include "kfunc_call_fail.skel.h"
+#include "kfunc_call_test.skel.h"
 #include "kfunc_call_test.lskel.h"
 #include "kfunc_call_test_subprog.skel.h"
 #include "kfunc_call_test_subprog.lskel.h"
+#include "kfunc_call_destructive.skel.h"
 
-static void test_main(void)
+#include "cap_helpers.h"
+
+static size_t log_buf_sz = 1048576; /* 1 MB */
+static char obj_log_buf[1048576];
+
+enum kfunc_test_type {
+	tc_test = 0,
+	syscall_test,
+	syscall_null_ctx_test,
+};
+
+struct kfunc_test_params {
+	const char *prog_name;
+	unsigned long lskel_prog_desc_offset;
+	int retval;
+	enum kfunc_test_type test_type;
+	const char *expected_err_msg;
+};
+
+#define __BPF_TEST_SUCCESS(name, __retval, type) \
+	{ \
+	  .prog_name = #name, \
+	  .lskel_prog_desc_offset = offsetof(struct kfunc_call_test_lskel, progs.name), \
+	  .retval = __retval, \
+	  .test_type = type, \
+	  .expected_err_msg = NULL, \
+	}
+
+#define __BPF_TEST_FAIL(name, __retval, type, error_msg) \
+	{ \
+	  .prog_name = #name, \
+	  .lskel_prog_desc_offset = 0 /* unused when test is failing */, \
+	  .retval = __retval, \
+	  .test_type = type, \
+	  .expected_err_msg = error_msg, \
+	}
+
+#define TC_TEST(name, retval) __BPF_TEST_SUCCESS(name, retval, tc_test)
+#define SYSCALL_TEST(name, retval) __BPF_TEST_SUCCESS(name, retval, syscall_test)
+#define SYSCALL_NULL_CTX_TEST(name, retval) __BPF_TEST_SUCCESS(name, retval, syscall_null_ctx_test)
+
+#define TC_FAIL(name, retval, error_msg) __BPF_TEST_FAIL(name, retval, tc_test, error_msg)
+#define SYSCALL_NULL_CTX_FAIL(name, retval, error_msg) \
+	__BPF_TEST_FAIL(name, retval, syscall_null_ctx_test, error_msg)
+
+static struct kfunc_test_params kfunc_tests[] = {
+	/* failure cases:
+	 * if retval is 0 -> the program will fail to load and the error message is an error
+	 * if retval is not 0 -> the program can be loaded but running it will gives the
+	 *                       provided return value. The error message is thus the one
+	 *                       from a successful load
+	 */
+	SYSCALL_NULL_CTX_FAIL(kfunc_syscall_test_fail, -EINVAL, "processed 4 insns"),
+	SYSCALL_NULL_CTX_FAIL(kfunc_syscall_test_null_fail, -EINVAL, "processed 4 insns"),
+	TC_FAIL(kfunc_call_test_get_mem_fail_rdonly, 0, "R0 cannot write into rdonly_mem"),
+	TC_FAIL(kfunc_call_test_get_mem_fail_use_after_free, 0, "invalid mem access 'scalar'"),
+	TC_FAIL(kfunc_call_test_get_mem_fail_oob, 0, "min value is outside of the allowed memory range"),
+	TC_FAIL(kfunc_call_test_get_mem_fail_not_const, 0, "is not a const"),
+	TC_FAIL(kfunc_call_test_mem_acquire_fail, 0, "acquire kernel function does not return PTR_TO_BTF_ID"),
+
+	/* success cases */
+	TC_TEST(kfunc_call_test1, 12),
+	TC_TEST(kfunc_call_test2, 3),
+	TC_TEST(kfunc_call_test_ref_btf_id, 0),
+	TC_TEST(kfunc_call_test_get_mem, 42),
+	SYSCALL_TEST(kfunc_syscall_test, 0),
+	SYSCALL_NULL_CTX_TEST(kfunc_syscall_test_null, 0),
+};
+
+struct syscall_test_args {
+	__u8 data[16];
+	size_t size;
+};
+
+static void verify_success(struct kfunc_test_params *param)
 {
-	struct kfunc_call_test_lskel *skel;
+	struct kfunc_call_test_lskel *lskel = NULL;
+	LIBBPF_OPTS(bpf_test_run_opts, topts);
+	struct bpf_prog_desc *lskel_prog;
+	struct kfunc_call_test *skel;
+	struct bpf_program *prog;
 	int prog_fd, err;
-	LIBBPF_OPTS(bpf_test_run_opts, topts,
-		.data_in = &pkt_v4,
-		.data_size_in = sizeof(pkt_v4),
-		.repeat = 1,
-	);
+	struct syscall_test_args args = {
+		.size = 10,
+	};
+
+	switch (param->test_type) {
+	case syscall_test:
+		topts.ctx_in = &args;
+		topts.ctx_size_in = sizeof(args);
+		/* fallthrough */
+	case syscall_null_ctx_test:
+		break;
+	case tc_test:
+		topts.data_in = &pkt_v4;
+		topts.data_size_in = sizeof(pkt_v4);
+		topts.repeat = 1;
+		break;
+	}
 
-	skel = kfunc_call_test_lskel__open_and_load();
+	/* first test with normal libbpf */
+	skel = kfunc_call_test__open_and_load();
 	if (!ASSERT_OK_PTR(skel, "skel"))
 		return;
 
-	prog_fd = skel->progs.kfunc_call_test1.prog_fd;
-	err = bpf_prog_test_run_opts(prog_fd, &topts);
-	ASSERT_OK(err, "bpf_prog_test_run(test1)");
-	ASSERT_EQ(topts.retval, 12, "test1-retval");
+	prog = bpf_object__find_program_by_name(skel->obj, param->prog_name);
+	if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name"))
+		goto cleanup;
 
-	prog_fd = skel->progs.kfunc_call_test2.prog_fd;
+	prog_fd = bpf_program__fd(prog);
 	err = bpf_prog_test_run_opts(prog_fd, &topts);
-	ASSERT_OK(err, "bpf_prog_test_run(test2)");
-	ASSERT_EQ(topts.retval, 3, "test2-retval");
+	if (!ASSERT_OK(err, param->prog_name))
+		goto cleanup;
+
+	if (!ASSERT_EQ(topts.retval, param->retval, "retval"))
+		goto cleanup;
+
+	/* second test with light skeletons */
+	lskel = kfunc_call_test_lskel__open_and_load();
+	if (!ASSERT_OK_PTR(lskel, "lskel"))
+		goto cleanup;
+
+	lskel_prog = (struct bpf_prog_desc *)((char *)lskel + param->lskel_prog_desc_offset);
 
-	prog_fd = skel->progs.kfunc_call_test_ref_btf_id.prog_fd;
+	prog_fd = lskel_prog->prog_fd;
 	err = bpf_prog_test_run_opts(prog_fd, &topts);
-	ASSERT_OK(err, "bpf_prog_test_run(test_ref_btf_id)");
-	ASSERT_EQ(topts.retval, 0, "test_ref_btf_id-retval");
+	if (!ASSERT_OK(err, param->prog_name))
+		goto cleanup;
 
-	kfunc_call_test_lskel__destroy(skel);
+	ASSERT_EQ(topts.retval, param->retval, "retval");
+
+cleanup:
+	kfunc_call_test__destroy(skel);
+	if (lskel)
+		kfunc_call_test_lskel__destroy(lskel);
+}
+
+static void verify_fail(struct kfunc_test_params *param)
+{
+	LIBBPF_OPTS(bpf_object_open_opts, opts);
+	LIBBPF_OPTS(bpf_test_run_opts, topts);
+	struct bpf_program *prog;
+	struct kfunc_call_fail *skel;
+	int prog_fd, err;
+	struct syscall_test_args args = {
+		.size = 10,
+	};
+
+	opts.kernel_log_buf = obj_log_buf;
+	opts.kernel_log_size = log_buf_sz;
+	opts.kernel_log_level = 1;
+
+	switch (param->test_type) {
+	case syscall_test:
+		topts.ctx_in = &args;
+		topts.ctx_size_in = sizeof(args);
+		/* fallthrough */
+	case syscall_null_ctx_test:
+		break;
+	case tc_test:
+		topts.data_in = &pkt_v4;
+		topts.data_size_in = sizeof(pkt_v4);
+		break;
+		topts.repeat = 1;
+	}
+
+	skel = kfunc_call_fail__open_opts(&opts);
+	if (!ASSERT_OK_PTR(skel, "kfunc_call_fail__open_opts"))
+		goto cleanup;
+
+	prog = bpf_object__find_program_by_name(skel->obj, param->prog_name);
+	if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name"))
+		goto cleanup;
+
+	bpf_program__set_autoload(prog, true);
+
+	err = kfunc_call_fail__load(skel);
+	if (!param->retval) {
+		/* the verifier is supposed to complain and refuses to load */
+		if (!ASSERT_ERR(err, "unexpected load success"))
+			goto out_err;
+
+	} else {
+		/* the program is loaded but must dynamically fail */
+		if (!ASSERT_OK(err, "unexpected load error"))
+			goto out_err;
+
+		prog_fd = bpf_program__fd(prog);
+		err = bpf_prog_test_run_opts(prog_fd, &topts);
+		if (!ASSERT_EQ(err, param->retval, param->prog_name))
+			goto out_err;
+	}
+
+out_err:
+	if (!ASSERT_OK_PTR(strstr(obj_log_buf, param->expected_err_msg), "expected_err_msg")) {
+		fprintf(stderr, "Expected err_msg: %s\n", param->expected_err_msg);
+		fprintf(stderr, "Verifier output: %s\n", obj_log_buf);
+	}
+
+cleanup:
+	kfunc_call_fail__destroy(skel);
+}
+
+static void test_main(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(kfunc_tests); i++) {
+		if (!test__start_subtest(kfunc_tests[i].prog_name))
+			continue;
+
+		if (!kfunc_tests[i].expected_err_msg)
+			verify_success(&kfunc_tests[i]);
+		else
+			verify_fail(&kfunc_tests[i]);
+	}
 }
 
 static void test_subprog(void)
@@ -86,14 +275,46 @@ static void test_subprog_lskel(void)
 	kfunc_call_test_subprog_lskel__destroy(skel);
 }
 
+static int test_destructive_open_and_load(void)
+{
+	struct kfunc_call_destructive *skel;
+	int err;
+
+	skel = kfunc_call_destructive__open();
+	if (!ASSERT_OK_PTR(skel, "prog_open"))
+		return -1;
+
+	err = kfunc_call_destructive__load(skel);
+
+	kfunc_call_destructive__destroy(skel);
+
+	return err;
+}
+
+static void test_destructive(void)
+{
+	__u64 save_caps = 0;
+
+	ASSERT_OK(test_destructive_open_and_load(), "successful_load");
+
+	if (!ASSERT_OK(cap_disable_effective(1ULL << CAP_SYS_BOOT, &save_caps), "drop_caps"))
+		return;
+
+	ASSERT_EQ(test_destructive_open_and_load(), -13, "no_caps_failure");
+
+	cap_enable_effective(save_caps, NULL);
+}
+
 void test_kfunc_call(void)
 {
-	if (test__start_subtest("main"))
-		test_main();
+	test_main();
 
 	if (test__start_subtest("subprog"))
 		test_subprog();
 
 	if (test__start_subtest("subprog_lskel"))
 		test_subprog_lskel();
+
+	if (test__start_subtest("destructive"))
+		test_destructive();
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c b/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c
new file mode 100644
index 000000000000..c210657d4d0a
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/kfunc_dynptr_param.c
@@ -0,0 +1,164 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (c) 2022 Facebook
+ * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH
+ *
+ * Author: Roberto Sassu <roberto.sassu@huawei.com>
+ */
+
+#include <test_progs.h>
+#include "test_kfunc_dynptr_param.skel.h"
+
+static size_t log_buf_sz = 1048576; /* 1 MB */
+static char obj_log_buf[1048576];
+
+static struct {
+	const char *prog_name;
+	const char *expected_verifier_err_msg;
+	int expected_runtime_err;
+} kfunc_dynptr_tests[] = {
+	{"dynptr_type_not_supp",
+	 "arg#0 pointer type STRUCT bpf_dynptr_kern points to unsupported dynamic pointer type", 0},
+	{"not_valid_dynptr",
+	 "arg#0 pointer type STRUCT bpf_dynptr_kern must be valid and initialized", 0},
+	{"not_ptr_to_stack", "arg#0 pointer type STRUCT bpf_dynptr_kern not to stack", 0},
+	{"dynptr_data_null", NULL, -EBADMSG},
+};
+
+static bool kfunc_not_supported;
+
+static int libbpf_print_cb(enum libbpf_print_level level, const char *fmt,
+			   va_list args)
+{
+	if (strcmp(fmt, "libbpf: extern (func ksym) '%s': not found in kernel or module BTFs\n"))
+		return 0;
+
+	if (strcmp(va_arg(args, char *), "bpf_verify_pkcs7_signature"))
+		return 0;
+
+	kfunc_not_supported = true;
+	return 0;
+}
+
+static void verify_fail(const char *prog_name, const char *expected_err_msg)
+{
+	struct test_kfunc_dynptr_param *skel;
+	LIBBPF_OPTS(bpf_object_open_opts, opts);
+	libbpf_print_fn_t old_print_cb;
+	struct bpf_program *prog;
+	int err;
+
+	opts.kernel_log_buf = obj_log_buf;
+	opts.kernel_log_size = log_buf_sz;
+	opts.kernel_log_level = 1;
+
+	skel = test_kfunc_dynptr_param__open_opts(&opts);
+	if (!ASSERT_OK_PTR(skel, "test_kfunc_dynptr_param__open_opts"))
+		goto cleanup;
+
+	prog = bpf_object__find_program_by_name(skel->obj, prog_name);
+	if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name"))
+		goto cleanup;
+
+	bpf_program__set_autoload(prog, true);
+
+	bpf_map__set_max_entries(skel->maps.ringbuf, getpagesize());
+
+	kfunc_not_supported = false;
+
+	old_print_cb = libbpf_set_print(libbpf_print_cb);
+	err = test_kfunc_dynptr_param__load(skel);
+	libbpf_set_print(old_print_cb);
+
+	if (err < 0 && kfunc_not_supported) {
+		fprintf(stderr,
+		  "%s:SKIP:bpf_verify_pkcs7_signature() kfunc not supported\n",
+		  __func__);
+		test__skip();
+		goto cleanup;
+	}
+
+	if (!ASSERT_ERR(err, "unexpected load success"))
+		goto cleanup;
+
+	if (!ASSERT_OK_PTR(strstr(obj_log_buf, expected_err_msg), "expected_err_msg")) {
+		fprintf(stderr, "Expected err_msg: %s\n", expected_err_msg);
+		fprintf(stderr, "Verifier output: %s\n", obj_log_buf);
+	}
+
+cleanup:
+	test_kfunc_dynptr_param__destroy(skel);
+}
+
+static void verify_success(const char *prog_name, int expected_runtime_err)
+{
+	struct test_kfunc_dynptr_param *skel;
+	libbpf_print_fn_t old_print_cb;
+	struct bpf_program *prog;
+	struct bpf_link *link;
+	__u32 next_id;
+	int err;
+
+	skel = test_kfunc_dynptr_param__open();
+	if (!ASSERT_OK_PTR(skel, "test_kfunc_dynptr_param__open"))
+		return;
+
+	skel->bss->pid = getpid();
+
+	bpf_map__set_max_entries(skel->maps.ringbuf, getpagesize());
+
+	kfunc_not_supported = false;
+
+	old_print_cb = libbpf_set_print(libbpf_print_cb);
+	err = test_kfunc_dynptr_param__load(skel);
+	libbpf_set_print(old_print_cb);
+
+	if (err < 0 && kfunc_not_supported) {
+		fprintf(stderr,
+		  "%s:SKIP:bpf_verify_pkcs7_signature() kfunc not supported\n",
+		  __func__);
+		test__skip();
+		goto cleanup;
+	}
+
+	if (!ASSERT_OK(err, "test_kfunc_dynptr_param__load"))
+		goto cleanup;
+
+	prog = bpf_object__find_program_by_name(skel->obj, prog_name);
+	if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name"))
+		goto cleanup;
+
+	link = bpf_program__attach(prog);
+	if (!ASSERT_OK_PTR(link, "bpf_program__attach"))
+		goto cleanup;
+
+	err = bpf_prog_get_next_id(0, &next_id);
+
+	bpf_link__destroy(link);
+
+	if (!ASSERT_OK(err, "bpf_prog_get_next_id"))
+		goto cleanup;
+
+	ASSERT_EQ(skel->bss->err, expected_runtime_err, "err");
+
+cleanup:
+	test_kfunc_dynptr_param__destroy(skel);
+}
+
+void test_kfunc_dynptr_param(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(kfunc_dynptr_tests); i++) {
+		if (!test__start_subtest(kfunc_dynptr_tests[i].prog_name))
+			continue;
+
+		if (kfunc_dynptr_tests[i].expected_verifier_err_msg)
+			verify_fail(kfunc_dynptr_tests[i].prog_name,
+			  kfunc_dynptr_tests[i].expected_verifier_err_msg);
+		else
+			verify_success(kfunc_dynptr_tests[i].prog_name,
+				kfunc_dynptr_tests[i].expected_runtime_err);
+	}
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/l4lb_all.c b/tools/testing/selftests/bpf/prog_tests/l4lb_all.c
index 55f733ff4109..9c1a18573ffd 100644
--- a/tools/testing/selftests/bpf/prog_tests/l4lb_all.c
+++ b/tools/testing/selftests/bpf/prog_tests/l4lb_all.c
@@ -90,7 +90,7 @@ out:
 void test_l4lb_all(void)
 {
 	if (test__start_subtest("l4lb_inline"))
-		test_l4lb("test_l4lb.o");
+		test_l4lb("test_l4lb.bpf.o");
 	if (test__start_subtest("l4lb_noinline"))
-		test_l4lb("test_l4lb_noinline.o");
+		test_l4lb("test_l4lb_noinline.bpf.o");
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c b/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c
index 4e0b2ec057aa..581c0eb0a0a1 100644
--- a/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c
+++ b/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c
@@ -27,8 +27,8 @@ void test_load_bytes_relative(void)
 	if (CHECK_FAIL(server_fd < 0))
 		goto close_cgroup_fd;
 
-	err = bpf_prog_test_load("./load_bytes_relative.o", BPF_PROG_TYPE_CGROUP_SKB,
-			    &obj, &prog_fd);
+	err = bpf_prog_test_load("./load_bytes_relative.bpf.o", BPF_PROG_TYPE_CGROUP_SKB,
+				 &obj, &prog_fd);
 	if (CHECK_FAIL(err))
 		goto close_server_fd;
 
diff --git a/tools/testing/selftests/bpf/prog_tests/lookup_key.c b/tools/testing/selftests/bpf/prog_tests/lookup_key.c
new file mode 100644
index 000000000000..68025e88f352
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/lookup_key.c
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH
+ *
+ * Author: Roberto Sassu <roberto.sassu@huawei.com>
+ */
+
+#include <linux/keyctl.h>
+#include <test_progs.h>
+
+#include "test_lookup_key.skel.h"
+
+#define KEY_LOOKUP_CREATE	0x01
+#define KEY_LOOKUP_PARTIAL	0x02
+
+static bool kfunc_not_supported;
+
+static int libbpf_print_cb(enum libbpf_print_level level, const char *fmt,
+			   va_list args)
+{
+	char *func;
+
+	if (strcmp(fmt, "libbpf: extern (func ksym) '%s': not found in kernel or module BTFs\n"))
+		return 0;
+
+	func = va_arg(args, char *);
+
+	if (strcmp(func, "bpf_lookup_user_key") && strcmp(func, "bpf_key_put") &&
+	    strcmp(func, "bpf_lookup_system_key"))
+		return 0;
+
+	kfunc_not_supported = true;
+	return 0;
+}
+
+void test_lookup_key(void)
+{
+	libbpf_print_fn_t old_print_cb;
+	struct test_lookup_key *skel;
+	__u32 next_id;
+	int ret;
+
+	skel = test_lookup_key__open();
+	if (!ASSERT_OK_PTR(skel, "test_lookup_key__open"))
+		return;
+
+	old_print_cb = libbpf_set_print(libbpf_print_cb);
+	ret = test_lookup_key__load(skel);
+	libbpf_set_print(old_print_cb);
+
+	if (ret < 0 && kfunc_not_supported) {
+		printf("%s:SKIP:bpf_lookup_*_key(), bpf_key_put() kfuncs not supported\n",
+		       __func__);
+		test__skip();
+		goto close_prog;
+	}
+
+	if (!ASSERT_OK(ret, "test_lookup_key__load"))
+		goto close_prog;
+
+	ret = test_lookup_key__attach(skel);
+	if (!ASSERT_OK(ret, "test_lookup_key__attach"))
+		goto close_prog;
+
+	skel->bss->monitored_pid = getpid();
+	skel->bss->key_serial = KEY_SPEC_THREAD_KEYRING;
+
+	/* The thread-specific keyring does not exist, this test fails. */
+	skel->bss->flags = 0;
+
+	ret = bpf_prog_get_next_id(0, &next_id);
+	if (!ASSERT_LT(ret, 0, "bpf_prog_get_next_id"))
+		goto close_prog;
+
+	/* Force creation of the thread-specific keyring, this test succeeds. */
+	skel->bss->flags = KEY_LOOKUP_CREATE;
+
+	ret = bpf_prog_get_next_id(0, &next_id);
+	if (!ASSERT_OK(ret, "bpf_prog_get_next_id"))
+		goto close_prog;
+
+	/* Pass both lookup flags for parameter validation. */
+	skel->bss->flags = KEY_LOOKUP_CREATE | KEY_LOOKUP_PARTIAL;
+
+	ret = bpf_prog_get_next_id(0, &next_id);
+	if (!ASSERT_OK(ret, "bpf_prog_get_next_id"))
+		goto close_prog;
+
+	/* Pass invalid flags. */
+	skel->bss->flags = UINT64_MAX;
+
+	ret = bpf_prog_get_next_id(0, &next_id);
+	if (!ASSERT_LT(ret, 0, "bpf_prog_get_next_id"))
+		goto close_prog;
+
+	skel->bss->key_serial = 0;
+	skel->bss->key_id = 1;
+
+	ret = bpf_prog_get_next_id(0, &next_id);
+	if (!ASSERT_OK(ret, "bpf_prog_get_next_id"))
+		goto close_prog;
+
+	skel->bss->key_id = UINT32_MAX;
+
+	ret = bpf_prog_get_next_id(0, &next_id);
+	ASSERT_LT(ret, 0, "bpf_prog_get_next_id");
+
+close_prog:
+	skel->bss->monitored_pid = 0;
+	test_lookup_key__destroy(skel);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/map_lock.c b/tools/testing/selftests/bpf/prog_tests/map_lock.c
index e4e99b37df64..1d6726f01dd2 100644
--- a/tools/testing/selftests/bpf/prog_tests/map_lock.c
+++ b/tools/testing/selftests/bpf/prog_tests/map_lock.c
@@ -49,7 +49,7 @@ out:
 
 void test_map_lock(void)
 {
-	const char *file = "./test_map_lock.o";
+	const char *file = "./test_map_lock.bpf.o";
 	int prog_fd, map_fd[2], vars[17] = {};
 	pthread_t thread_id[6];
 	struct bpf_object *obj = NULL;
diff --git a/tools/testing/selftests/bpf/prog_tests/pinning.c b/tools/testing/selftests/bpf/prog_tests/pinning.c
index 31c09ba577eb..d95cee5867b7 100644
--- a/tools/testing/selftests/bpf/prog_tests/pinning.c
+++ b/tools/testing/selftests/bpf/prog_tests/pinning.c
@@ -26,13 +26,13 @@ __u32 get_map_id(struct bpf_object *obj, const char *name)
 
 void test_pinning(void)
 {
-	const char *file_invalid = "./test_pinning_invalid.o";
+	const char *file_invalid = "./test_pinning_invalid.bpf.o";
 	const char *custpinpath = "/sys/fs/bpf/custom/pinmap";
 	const char *nopinpath = "/sys/fs/bpf/nopinmap";
 	const char *nopinpath2 = "/sys/fs/bpf/nopinmap2";
 	const char *custpath = "/sys/fs/bpf/custom";
 	const char *pinpath = "/sys/fs/bpf/pinmap";
-	const char *file = "./test_pinning.o";
+	const char *file = "./test_pinning.bpf.o";
 	__u32 map_id, map_id2, duration = 0;
 	struct stat statbuf = {};
 	struct bpf_object *obj;
diff --git a/tools/testing/selftests/bpf/prog_tests/pkt_access.c b/tools/testing/selftests/bpf/prog_tests/pkt_access.c
index 0bcccdc34fbc..682e4ff45b01 100644
--- a/tools/testing/selftests/bpf/prog_tests/pkt_access.c
+++ b/tools/testing/selftests/bpf/prog_tests/pkt_access.c
@@ -4,7 +4,7 @@
 
 void test_pkt_access(void)
 {
-	const char *file = "./test_pkt_access.o";
+	const char *file = "./test_pkt_access.bpf.o";
 	struct bpf_object *obj;
 	int err, prog_fd;
 	LIBBPF_OPTS(bpf_test_run_opts, topts,
diff --git a/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c b/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c
index 00ee1dd792aa..0d85e0642811 100644
--- a/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c
+++ b/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c
@@ -4,7 +4,7 @@
 
 void test_pkt_md_access(void)
 {
-	const char *file = "./test_pkt_md_access.o";
+	const char *file = "./test_pkt_md_access.bpf.o";
 	struct bpf_object *obj;
 	int err, prog_fd;
 	LIBBPF_OPTS(bpf_test_run_opts, topts,
diff --git a/tools/testing/selftests/bpf/prog_tests/probe_user.c b/tools/testing/selftests/bpf/prog_tests/probe_user.c
index 34dbd2adc157..8721671321de 100644
--- a/tools/testing/selftests/bpf/prog_tests/probe_user.c
+++ b/tools/testing/selftests/bpf/prog_tests/probe_user.c
@@ -11,7 +11,7 @@ void serial_test_probe_user(void)
 #endif
 	};
 	enum { prog_count = ARRAY_SIZE(prog_names) };
-	const char *obj_file = "./test_probe_user.o";
+	const char *obj_file = "./test_probe_user.bpf.o";
 	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, );
 	int err, results_map_fd, sock_fd, duration = 0;
 	struct sockaddr curr, orig, tmp;
diff --git a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c
index d2743fc10032..722c5f2a7776 100644
--- a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c
+++ b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c
@@ -28,9 +28,9 @@ static void test_queue_stack_map_by_type(int type)
 		vals[i] = rand();
 
 	if (type == QUEUE)
-		strncpy(file, "./test_queue_map.o", sizeof(file));
+		strncpy(file, "./test_queue_map.bpf.o", sizeof(file));
 	else if (type == STACK)
-		strncpy(file, "./test_stack_map.o", sizeof(file));
+		strncpy(file, "./test_stack_map.bpf.o", sizeof(file));
 	else
 		return;
 
diff --git a/tools/testing/selftests/bpf/prog_tests/rdonly_maps.c b/tools/testing/selftests/bpf/prog_tests/rdonly_maps.c
index fd5d2ddfb062..19e2f2526dbd 100644
--- a/tools/testing/selftests/bpf/prog_tests/rdonly_maps.c
+++ b/tools/testing/selftests/bpf/prog_tests/rdonly_maps.c
@@ -16,7 +16,7 @@ struct rdonly_map_subtest {
 
 void test_rdonly_maps(void)
 {
-	const char *file = "test_rdonly_maps.o";
+	const char *file = "test_rdonly_maps.bpf.o";
 	struct rdonly_map_subtest subtests[] = {
 		{ "skip loop", "skip_loop", 0, 0 },
 		{ "part loop", "part_loop", 3, 2 + 3 + 4 },
diff --git a/tools/testing/selftests/bpf/prog_tests/reference_tracking.c b/tools/testing/selftests/bpf/prog_tests/reference_tracking.c
index 739d2ea6ca55..d863205bbe95 100644
--- a/tools/testing/selftests/bpf/prog_tests/reference_tracking.c
+++ b/tools/testing/selftests/bpf/prog_tests/reference_tracking.c
@@ -3,7 +3,7 @@
 
 void test_reference_tracking(void)
 {
-	const char *file = "test_sk_lookup_kern.o";
+	const char *file = "test_sk_lookup_kern.bpf.o";
 	const char *obj_name = "ref_track";
 	DECLARE_LIBBPF_OPTS(bpf_object_open_opts, open_opts,
 		.object_name = obj_name,
diff --git a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c
index c197261d02e2..f81d08d429a2 100644
--- a/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c
+++ b/tools/testing/selftests/bpf/prog_tests/resolve_btfids.c
@@ -101,7 +101,7 @@ static int resolve_symbols(void)
 	int type_id;
 	__u32 nr;
 
-	btf = btf__parse_elf("btf_data.o", NULL);
+	btf = btf__parse_elf("btf_data.bpf.o", NULL);
 	if (CHECK(libbpf_get_error(btf), "resolve",
 		  "Failed to load BTF from btf_data.o\n"))
 		return -1;
diff --git a/tools/testing/selftests/bpf/prog_tests/select_reuseport.c b/tools/testing/selftests/bpf/prog_tests/select_reuseport.c
index 1cbd8cd64044..64c5f5eb2994 100644
--- a/tools/testing/selftests/bpf/prog_tests/select_reuseport.c
+++ b/tools/testing/selftests/bpf/prog_tests/select_reuseport.c
@@ -91,9 +91,9 @@ static int prepare_bpf_obj(void)
 	struct bpf_map *map;
 	int err;
 
-	obj = bpf_object__open("test_select_reuseport_kern.o");
+	obj = bpf_object__open("test_select_reuseport_kern.bpf.o");
 	err = libbpf_get_error(obj);
-	RET_ERR(err, "open test_select_reuseport_kern.o",
+	RET_ERR(err, "open test_select_reuseport_kern.bpf.o",
 		"obj:%p PTR_ERR(obj):%d\n", obj, err);
 
 	map = bpf_object__find_map_by_name(obj, "outer_map");
diff --git a/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c b/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c
new file mode 100644
index 000000000000..018611e6b248
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/setget_sockopt.c
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#define _GNU_SOURCE
+#include <sched.h>
+#include <linux/socket.h>
+#include <net/if.h>
+
+#include "test_progs.h"
+#include "cgroup_helpers.h"
+#include "network_helpers.h"
+
+#include "setget_sockopt.skel.h"
+
+#define CG_NAME "/setget-sockopt-test"
+
+static const char addr4_str[] = "127.0.0.1";
+static const char addr6_str[] = "::1";
+static struct setget_sockopt *skel;
+static int cg_fd;
+
+static int create_netns(void)
+{
+	if (!ASSERT_OK(unshare(CLONE_NEWNET), "create netns"))
+		return -1;
+
+	if (!ASSERT_OK(system("ip link set dev lo up"), "set lo up"))
+		return -1;
+
+	if (!ASSERT_OK(system("ip link add dev binddevtest1 type veth peer name binddevtest2"),
+		       "add veth"))
+		return -1;
+
+	if (!ASSERT_OK(system("ip link set dev binddevtest1 up"),
+		       "bring veth up"))
+		return -1;
+
+	return 0;
+}
+
+static void test_tcp(int family)
+{
+	struct setget_sockopt__bss *bss = skel->bss;
+	int sfd, cfd;
+
+	memset(bss, 0, sizeof(*bss));
+
+	sfd = start_server(family, SOCK_STREAM,
+			   family == AF_INET6 ? addr6_str : addr4_str, 0, 0);
+	if (!ASSERT_GE(sfd, 0, "start_server"))
+		return;
+
+	cfd = connect_to_fd(sfd, 0);
+	if (!ASSERT_GE(cfd, 0, "connect_to_fd_server")) {
+		close(sfd);
+		return;
+	}
+	close(sfd);
+	close(cfd);
+
+	ASSERT_EQ(bss->nr_listen, 1, "nr_listen");
+	ASSERT_EQ(bss->nr_connect, 1, "nr_connect");
+	ASSERT_EQ(bss->nr_active, 1, "nr_active");
+	ASSERT_EQ(bss->nr_passive, 1, "nr_passive");
+	ASSERT_EQ(bss->nr_socket_post_create, 2, "nr_socket_post_create");
+	ASSERT_EQ(bss->nr_binddev, 2, "nr_bind");
+}
+
+static void test_udp(int family)
+{
+	struct setget_sockopt__bss *bss = skel->bss;
+	int sfd;
+
+	memset(bss, 0, sizeof(*bss));
+
+	sfd = start_server(family, SOCK_DGRAM,
+			   family == AF_INET6 ? addr6_str : addr4_str, 0, 0);
+	if (!ASSERT_GE(sfd, 0, "start_server"))
+		return;
+	close(sfd);
+
+	ASSERT_GE(bss->nr_socket_post_create, 1, "nr_socket_post_create");
+	ASSERT_EQ(bss->nr_binddev, 1, "nr_bind");
+}
+
+void test_setget_sockopt(void)
+{
+	cg_fd = test__join_cgroup(CG_NAME);
+	if (cg_fd < 0)
+		return;
+
+	if (create_netns())
+		goto done;
+
+	skel = setget_sockopt__open();
+	if (!ASSERT_OK_PTR(skel, "open skel"))
+		goto done;
+
+	strcpy(skel->rodata->veth, "binddevtest1");
+	skel->rodata->veth_ifindex = if_nametoindex("binddevtest1");
+	if (!ASSERT_GT(skel->rodata->veth_ifindex, 0, "if_nametoindex"))
+		goto done;
+
+	if (!ASSERT_OK(setget_sockopt__load(skel), "load skel"))
+		goto done;
+
+	skel->links.skops_sockopt =
+		bpf_program__attach_cgroup(skel->progs.skops_sockopt, cg_fd);
+	if (!ASSERT_OK_PTR(skel->links.skops_sockopt, "attach cgroup"))
+		goto done;
+
+	skel->links.socket_post_create =
+		bpf_program__attach_cgroup(skel->progs.socket_post_create, cg_fd);
+	if (!ASSERT_OK_PTR(skel->links.socket_post_create, "attach_cgroup"))
+		goto done;
+
+	test_tcp(AF_INET6);
+	test_tcp(AF_INET);
+	test_udp(AF_INET6);
+	test_udp(AF_INET);
+
+done:
+	setget_sockopt__destroy(skel);
+	close(cg_fd);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/sk_assign.c b/tools/testing/selftests/bpf/prog_tests/sk_assign.c
index 1d272e05188e..3e190ed63976 100644
--- a/tools/testing/selftests/bpf/prog_tests/sk_assign.c
+++ b/tools/testing/selftests/bpf/prog_tests/sk_assign.c
@@ -47,7 +47,7 @@ configure_stack(void)
 	if (CHECK_FAIL(system("tc qdisc add dev lo clsact")))
 		return false;
 	sprintf(tc_cmd, "%s %s %s %s", "tc filter add dev lo ingress bpf",
-		       "direct-action object-file ./test_sk_assign.o",
+		       "direct-action object-file ./test_sk_assign.bpf.o",
 		       "section tc",
 		       (env.verbosity < VERBOSE_VERY) ? " 2>/dev/null" : "verbose");
 	if (CHECK(system(tc_cmd), "BPF load failed;",
diff --git a/tools/testing/selftests/bpf/prog_tests/skb_ctx.c b/tools/testing/selftests/bpf/prog_tests/skb_ctx.c
index ce0e555b5e38..33f950e2dae3 100644
--- a/tools/testing/selftests/bpf/prog_tests/skb_ctx.c
+++ b/tools/testing/selftests/bpf/prog_tests/skb_ctx.c
@@ -31,7 +31,7 @@ void test_skb_ctx(void)
 	struct bpf_object *obj;
 	int err, prog_fd, i;
 
-	err = bpf_prog_test_load("./test_skb_ctx.o", BPF_PROG_TYPE_SCHED_CLS,
+	err = bpf_prog_test_load("./test_skb_ctx.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
 				 &obj, &prog_fd);
 	if (!ASSERT_OK(err, "load"))
 		return;
diff --git a/tools/testing/selftests/bpf/prog_tests/skb_helpers.c b/tools/testing/selftests/bpf/prog_tests/skb_helpers.c
index 97dc8b14be48..f7ee25f290f7 100644
--- a/tools/testing/selftests/bpf/prog_tests/skb_helpers.c
+++ b/tools/testing/selftests/bpf/prog_tests/skb_helpers.c
@@ -20,7 +20,7 @@ void test_skb_helpers(void)
 	struct bpf_object *obj;
 	int err, prog_fd;
 
-	err = bpf_prog_test_load("./test_skb_helpers.o",
+	err = bpf_prog_test_load("./test_skb_helpers.bpf.o",
 				 BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd);
 	if (!ASSERT_OK(err, "load"))
 		return;
diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c
index cec5c0882372..0aa088900699 100644
--- a/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c
+++ b/tools/testing/selftests/bpf/prog_tests/sockmap_basic.c
@@ -27,21 +27,21 @@ static int connected_socket_v4(void)
 	int s, repair, err;
 
 	s = socket(AF_INET, SOCK_STREAM, 0);
-	if (CHECK_FAIL(s == -1))
+	if (!ASSERT_GE(s, 0, "socket"))
 		goto error;
 
 	repair = TCP_REPAIR_ON;
 	err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair));
-	if (CHECK_FAIL(err))
+	if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)"))
 		goto error;
 
 	err = connect(s, (struct sockaddr *)&addr, len);
-	if (CHECK_FAIL(err))
+	if (!ASSERT_OK(err, "connect"))
 		goto error;
 
 	repair = TCP_REPAIR_OFF_NO_WP;
 	err = setsockopt(s, SOL_TCP, TCP_REPAIR, &repair, sizeof(repair));
-	if (CHECK_FAIL(err))
+	if (!ASSERT_OK(err, "setsockopt(TCP_REPAIR)"))
 		goto error;
 
 	return s;
@@ -54,7 +54,7 @@ error:
 static void compare_cookies(struct bpf_map *src, struct bpf_map *dst)
 {
 	__u32 i, max_entries = bpf_map__max_entries(src);
-	int err, duration = 0, src_fd, dst_fd;
+	int err, src_fd, dst_fd;
 
 	src_fd = bpf_map__fd(src);
 	dst_fd = bpf_map__fd(dst);
@@ -65,20 +65,18 @@ static void compare_cookies(struct bpf_map *src, struct bpf_map *dst)
 		err = bpf_map_lookup_elem(src_fd, &i, &src_cookie);
 		if (err && errno == ENOENT) {
 			err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie);
-			CHECK(!err, "map_lookup_elem(dst)", "element %u not deleted\n", i);
-			CHECK(err && errno != ENOENT, "map_lookup_elem(dst)", "%s\n",
-			      strerror(errno));
+			ASSERT_ERR(err, "map_lookup_elem(dst)");
+			ASSERT_EQ(errno, ENOENT, "map_lookup_elem(dst)");
 			continue;
 		}
-		if (CHECK(err, "lookup_elem(src)", "%s\n", strerror(errno)))
+		if (!ASSERT_OK(err, "lookup_elem(src)"))
 			continue;
 
 		err = bpf_map_lookup_elem(dst_fd, &i, &dst_cookie);
-		if (CHECK(err, "lookup_elem(dst)", "%s\n", strerror(errno)))
+		if (!ASSERT_OK(err, "lookup_elem(dst)"))
 			continue;
 
-		CHECK(dst_cookie != src_cookie, "cookie mismatch",
-		      "%llu != %llu (pos %u)\n", dst_cookie, src_cookie, i);
+		ASSERT_EQ(dst_cookie, src_cookie, "cookie mismatch");
 	}
 }
 
@@ -89,20 +87,16 @@ static void test_sockmap_create_update_free(enum bpf_map_type map_type)
 	int s, map, err;
 
 	s = connected_socket_v4();
-	if (CHECK_FAIL(s < 0))
+	if (!ASSERT_GE(s, 0, "connected_socket_v4"))
 		return;
 
 	map = bpf_map_create(map_type, NULL, sizeof(int), sizeof(int), 1, NULL);
-	if (CHECK_FAIL(map < 0)) {
-		perror("bpf_cmap_create");
+	if (!ASSERT_GE(map, 0, "bpf_map_create"))
 		goto out;
-	}
 
 	err = bpf_map_update_elem(map, &zero, &s, BPF_NOEXIST);
-	if (CHECK_FAIL(err)) {
-		perror("bpf_map_update");
+	if (!ASSERT_OK(err, "bpf_map_update"))
 		goto out;
-	}
 
 out:
 	close(map);
@@ -115,32 +109,26 @@ static void test_skmsg_helpers(enum bpf_map_type map_type)
 	int err, map, verdict;
 
 	skel = test_skmsg_load_helpers__open_and_load();
-	if (CHECK_FAIL(!skel)) {
-		perror("test_skmsg_load_helpers__open_and_load");
+	if (!ASSERT_OK_PTR(skel, "test_skmsg_load_helpers__open_and_load"))
 		return;
-	}
 
 	verdict = bpf_program__fd(skel->progs.prog_msg_verdict);
 	map = bpf_map__fd(skel->maps.sock_map);
 
 	err = bpf_prog_attach(verdict, map, BPF_SK_MSG_VERDICT, 0);
-	if (CHECK_FAIL(err)) {
-		perror("bpf_prog_attach");
+	if (!ASSERT_OK(err, "bpf_prog_attach"))
 		goto out;
-	}
 
 	err = bpf_prog_detach2(verdict, map, BPF_SK_MSG_VERDICT);
-	if (CHECK_FAIL(err)) {
-		perror("bpf_prog_detach2");
+	if (!ASSERT_OK(err, "bpf_prog_detach2"))
 		goto out;
-	}
 out:
 	test_skmsg_load_helpers__destroy(skel);
 }
 
 static void test_sockmap_update(enum bpf_map_type map_type)
 {
-	int err, prog, src, duration = 0;
+	int err, prog, src;
 	struct test_sockmap_update *skel;
 	struct bpf_map *dst_map;
 	const __u32 zero = 0;
@@ -153,11 +141,11 @@ static void test_sockmap_update(enum bpf_map_type map_type)
 	__s64 sk;
 
 	sk = connected_socket_v4();
-	if (CHECK(sk == -1, "connected_socket_v4", "cannot connect\n"))
+	if (!ASSERT_NEQ(sk, -1, "connected_socket_v4"))
 		return;
 
 	skel = test_sockmap_update__open_and_load();
-	if (CHECK(!skel, "open_and_load", "cannot load skeleton\n"))
+	if (!ASSERT_OK_PTR(skel, "open_and_load"))
 		goto close_sk;
 
 	prog = bpf_program__fd(skel->progs.copy_sock_map);
@@ -168,7 +156,7 @@ static void test_sockmap_update(enum bpf_map_type map_type)
 		dst_map = skel->maps.dst_sock_hash;
 
 	err = bpf_map_update_elem(src, &zero, &sk, BPF_NOEXIST);
-	if (CHECK(err, "update_elem(src)", "errno=%u\n", errno))
+	if (!ASSERT_OK(err, "update_elem(src)"))
 		goto out;
 
 	err = bpf_prog_test_run_opts(prog, &topts);
@@ -188,17 +176,16 @@ close_sk:
 static void test_sockmap_invalid_update(void)
 {
 	struct test_sockmap_invalid_update *skel;
-	int duration = 0;
 
 	skel = test_sockmap_invalid_update__open_and_load();
-	if (CHECK(skel, "open_and_load", "verifier accepted map_update\n"))
+	if (!ASSERT_NULL(skel, "open_and_load"))
 		test_sockmap_invalid_update__destroy(skel);
 }
 
 static void test_sockmap_copy(enum bpf_map_type map_type)
 {
 	DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts);
-	int err, len, src_fd, iter_fd, duration = 0;
+	int err, len, src_fd, iter_fd;
 	union bpf_iter_link_info linfo = {};
 	__u32 i, num_sockets, num_elems;
 	struct bpf_iter_sockmap *skel;
@@ -208,7 +195,7 @@ static void test_sockmap_copy(enum bpf_map_type map_type)
 	char buf[64];
 
 	skel = bpf_iter_sockmap__open_and_load();
-	if (CHECK(!skel, "bpf_iter_sockmap__open_and_load", "skeleton open_and_load failed\n"))
+	if (!ASSERT_OK_PTR(skel, "bpf_iter_sockmap__open_and_load"))
 		return;
 
 	if (map_type == BPF_MAP_TYPE_SOCKMAP) {
@@ -222,7 +209,7 @@ static void test_sockmap_copy(enum bpf_map_type map_type)
 	}
 
 	sock_fd = calloc(num_sockets, sizeof(*sock_fd));
-	if (CHECK(!sock_fd, "calloc(sock_fd)", "failed to allocate\n"))
+	if (!ASSERT_OK_PTR(sock_fd, "calloc(sock_fd)"))
 		goto out;
 
 	for (i = 0; i < num_sockets; i++)
@@ -232,11 +219,11 @@ static void test_sockmap_copy(enum bpf_map_type map_type)
 
 	for (i = 0; i < num_sockets; i++) {
 		sock_fd[i] = connected_socket_v4();
-		if (CHECK(sock_fd[i] == -1, "connected_socket_v4", "cannot connect\n"))
+		if (!ASSERT_NEQ(sock_fd[i], -1, "connected_socket_v4"))
 			goto out;
 
 		err = bpf_map_update_elem(src_fd, &i, &sock_fd[i], BPF_NOEXIST);
-		if (CHECK(err, "map_update", "failed: %s\n", strerror(errno)))
+		if (!ASSERT_OK(err, "map_update"))
 			goto out;
 	}
 
@@ -248,22 +235,20 @@ static void test_sockmap_copy(enum bpf_map_type map_type)
 		goto out;
 
 	iter_fd = bpf_iter_create(bpf_link__fd(link));
-	if (CHECK(iter_fd < 0, "create_iter", "create_iter failed\n"))
+	if (!ASSERT_GE(iter_fd, 0, "create_iter"))
 		goto free_link;
 
 	/* do some tests */
 	while ((len = read(iter_fd, buf, sizeof(buf))) > 0)
 		;
-	if (CHECK(len < 0, "read", "failed: %s\n", strerror(errno)))
+	if (!ASSERT_GE(len, 0, "read"))
 		goto close_iter;
 
 	/* test results */
-	if (CHECK(skel->bss->elems != num_elems, "elems", "got %u expected %u\n",
-		  skel->bss->elems, num_elems))
+	if (!ASSERT_EQ(skel->bss->elems, num_elems, "elems"))
 		goto close_iter;
 
-	if (CHECK(skel->bss->socks != num_sockets, "socks", "got %u expected %u\n",
-		  skel->bss->socks, num_sockets))
+	if (!ASSERT_EQ(skel->bss->socks, num_sockets, "socks"))
 		goto close_iter;
 
 	compare_cookies(src, skel->maps.dst);
@@ -288,28 +273,22 @@ static void test_sockmap_skb_verdict_attach(enum bpf_attach_type first,
 	int err, map, verdict;
 
 	skel = test_sockmap_skb_verdict_attach__open_and_load();
-	if (CHECK_FAIL(!skel)) {
-		perror("test_sockmap_skb_verdict_attach__open_and_load");
+	if (!ASSERT_OK_PTR(skel, "open_and_load"))
 		return;
-	}
 
 	verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
 	map = bpf_map__fd(skel->maps.sock_map);
 
 	err = bpf_prog_attach(verdict, map, first, 0);
-	if (CHECK_FAIL(err)) {
-		perror("bpf_prog_attach");
+	if (!ASSERT_OK(err, "bpf_prog_attach"))
 		goto out;
-	}
 
 	err = bpf_prog_attach(verdict, map, second, 0);
 	ASSERT_EQ(err, -EBUSY, "prog_attach_fail");
 
 	err = bpf_prog_detach2(verdict, map, first);
-	if (CHECK_FAIL(err)) {
-		perror("bpf_prog_detach2");
+	if (!ASSERT_OK(err, "bpf_prog_detach2"))
 		goto out;
-	}
 out:
 	test_sockmap_skb_verdict_attach__destroy(skel);
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c
index e172d89e92e1..2d0796314862 100644
--- a/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c
+++ b/tools/testing/selftests/bpf/prog_tests/sockmap_ktls.c
@@ -15,16 +15,12 @@ static int tcp_server(int family)
 	int err, s;
 
 	s = socket(family, SOCK_STREAM, 0);
-	if (CHECK_FAIL(s == -1)) {
-		perror("socket");
+	if (!ASSERT_GE(s, 0, "socket"))
 		return -1;
-	}
 
 	err = listen(s, SOMAXCONN);
-	if (CHECK_FAIL(err)) {
-		perror("listen");
+	if (!ASSERT_OK(err, "listen"))
 		return -1;
-	}
 
 	return s;
 }
@@ -48,44 +44,31 @@ static void test_sockmap_ktls_disconnect_after_delete(int family, int map)
 		return;
 
 	err = getsockname(srv, (struct sockaddr *)&addr, &len);
-	if (CHECK_FAIL(err)) {
-		perror("getsockopt");
+	if (!ASSERT_OK(err, "getsockopt"))
 		goto close_srv;
-	}
 
 	cli = socket(family, SOCK_STREAM, 0);
-	if (CHECK_FAIL(cli == -1)) {
-		perror("socket");
+	if (!ASSERT_GE(cli, 0, "socket"))
 		goto close_srv;
-	}
 
 	err = connect(cli, (struct sockaddr *)&addr, len);
-	if (CHECK_FAIL(err)) {
-		perror("connect");
+	if (!ASSERT_OK(err, "connect"))
 		goto close_cli;
-	}
 
 	err = bpf_map_update_elem(map, &zero, &cli, 0);
-	if (CHECK_FAIL(err)) {
-		perror("bpf_map_update_elem");
+	if (!ASSERT_OK(err, "bpf_map_update_elem"))
 		goto close_cli;
-	}
 
 	err = setsockopt(cli, IPPROTO_TCP, TCP_ULP, "tls", strlen("tls"));
-	if (CHECK_FAIL(err)) {
-		perror("setsockopt(TCP_ULP)");
+	if (!ASSERT_OK(err, "setsockopt(TCP_ULP)"))
 		goto close_cli;
-	}
 
 	err = bpf_map_delete_elem(map, &zero);
-	if (CHECK_FAIL(err)) {
-		perror("bpf_map_delete_elem");
+	if (!ASSERT_OK(err, "bpf_map_delete_elem"))
 		goto close_cli;
-	}
 
 	err = disconnect(cli);
-	if (CHECK_FAIL(err))
-		perror("disconnect");
+	ASSERT_OK(err, "disconnect");
 
 close_cli:
 	close(cli);
@@ -168,10 +151,8 @@ static void run_tests(int family, enum bpf_map_type map_type)
 	int map;
 
 	map = bpf_map_create(map_type, NULL, sizeof(int), sizeof(int), 1, NULL);
-	if (CHECK_FAIL(map < 0)) {
-		perror("bpf_map_create");
+	if (!ASSERT_GE(map, 0, "bpf_map_create"))
 		return;
-	}
 
 	if (test__start_subtest(fmt_test_name("disconnect_after_delete", family, map_type)))
 		test_sockmap_ktls_disconnect_after_delete(family, map);
diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt.c b/tools/testing/selftests/bpf/prog_tests/sockopt.c
index cd09f4c7dd92..aa4debf62fc6 100644
--- a/tools/testing/selftests/bpf/prog_tests/sockopt.c
+++ b/tools/testing/selftests/bpf/prog_tests/sockopt.c
@@ -972,12 +972,12 @@ void test_sockopt(void)
 	int cgroup_fd, i;
 
 	cgroup_fd = test__join_cgroup("/sockopt");
-	if (CHECK_FAIL(cgroup_fd < 0))
+	if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup"))
 		return;
 
 	for (i = 0; i < ARRAY_SIZE(tests); i++) {
 		test__start_subtest(tests[i].descr);
-		CHECK_FAIL(run_test(cgroup_fd, &tests[i]));
+		ASSERT_OK(run_test(cgroup_fd, &tests[i]), tests[i].descr);
 	}
 
 	close(cgroup_fd);
diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c b/tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c
index 8ed78a9383ba..60c17a8e2789 100644
--- a/tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c
+++ b/tools/testing/selftests/bpf/prog_tests/sockopt_inherit.c
@@ -76,20 +76,16 @@ static void *server_thread(void *arg)
 	pthread_cond_signal(&server_started);
 	pthread_mutex_unlock(&server_started_mtx);
 
-	if (CHECK_FAIL(err < 0)) {
-		perror("Failed to listed on socket");
+	if (!ASSERT_GE(err, 0, "listed on socket"))
 		return NULL;
-	}
 
 	err += verify_sockopt(fd, CUSTOM_INHERIT1, "listen", 1);
 	err += verify_sockopt(fd, CUSTOM_INHERIT2, "listen", 1);
 	err += verify_sockopt(fd, CUSTOM_LISTENER, "listen", 1);
 
 	client_fd = accept(fd, (struct sockaddr *)&addr, &len);
-	if (CHECK_FAIL(client_fd < 0)) {
-		perror("Failed to accept client");
+	if (!ASSERT_GE(client_fd, 0, "accept client"))
 		return NULL;
-	}
 
 	err += verify_sockopt(client_fd, CUSTOM_INHERIT1, "accept", 1);
 	err += verify_sockopt(client_fd, CUSTOM_INHERIT2, "accept", 1);
@@ -174,7 +170,7 @@ static void run_test(int cgroup_fd)
 	pthread_t tid;
 	int err;
 
-	obj = bpf_object__open_file("sockopt_inherit.o", NULL);
+	obj = bpf_object__open_file("sockopt_inherit.bpf.o", NULL);
 	if (!ASSERT_OK_PTR(obj, "obj_open"))
 		return;
 
@@ -183,20 +179,20 @@ static void run_test(int cgroup_fd)
 		goto close_bpf_object;
 
 	err = prog_attach(obj, cgroup_fd, "cgroup/getsockopt", "_getsockopt");
-	if (CHECK_FAIL(err))
+	if (!ASSERT_OK(err, "prog_attach _getsockopt"))
 		goto close_bpf_object;
 
 	err = prog_attach(obj, cgroup_fd, "cgroup/setsockopt", "_setsockopt");
-	if (CHECK_FAIL(err))
+	if (!ASSERT_OK(err, "prog_attach _setsockopt"))
 		goto close_bpf_object;
 
 	server_fd = start_server();
-	if (CHECK_FAIL(server_fd < 0))
+	if (!ASSERT_GE(server_fd, 0, "start_server"))
 		goto close_bpf_object;
 
 	pthread_mutex_lock(&server_started_mtx);
-	if (CHECK_FAIL(pthread_create(&tid, NULL, server_thread,
-				      (void *)&server_fd))) {
+	if (!ASSERT_OK(pthread_create(&tid, NULL, server_thread,
+				      (void *)&server_fd), "pthread_create")) {
 		pthread_mutex_unlock(&server_started_mtx);
 		goto close_server_fd;
 	}
@@ -204,17 +200,17 @@ static void run_test(int cgroup_fd)
 	pthread_mutex_unlock(&server_started_mtx);
 
 	client_fd = connect_to_server(server_fd);
-	if (CHECK_FAIL(client_fd < 0))
+	if (!ASSERT_GE(client_fd, 0, "connect_to_server"))
 		goto close_server_fd;
 
-	CHECK_FAIL(verify_sockopt(client_fd, CUSTOM_INHERIT1, "connect", 0));
-	CHECK_FAIL(verify_sockopt(client_fd, CUSTOM_INHERIT2, "connect", 0));
-	CHECK_FAIL(verify_sockopt(client_fd, CUSTOM_LISTENER, "connect", 0));
+	ASSERT_OK(verify_sockopt(client_fd, CUSTOM_INHERIT1, "connect", 0), "verify_sockopt1");
+	ASSERT_OK(verify_sockopt(client_fd, CUSTOM_INHERIT2, "connect", 0), "verify_sockopt2");
+	ASSERT_OK(verify_sockopt(client_fd, CUSTOM_LISTENER, "connect", 0), "verify_sockopt ener");
 
 	pthread_join(tid, &server_err);
 
 	err = (int)(long)server_err;
-	CHECK_FAIL(err);
+	ASSERT_OK(err, "pthread_join retval");
 
 	close(client_fd);
 
@@ -229,7 +225,7 @@ void test_sockopt_inherit(void)
 	int cgroup_fd;
 
 	cgroup_fd = test__join_cgroup("/sockopt_inherit");
-	if (CHECK_FAIL(cgroup_fd < 0))
+	if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup"))
 		return;
 
 	run_test(cgroup_fd);
diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt_multi.c b/tools/testing/selftests/bpf/prog_tests/sockopt_multi.c
index abce12ddcc37..7f5659349011 100644
--- a/tools/testing/selftests/bpf/prog_tests/sockopt_multi.c
+++ b/tools/testing/selftests/bpf/prog_tests/sockopt_multi.c
@@ -303,14 +303,14 @@ void test_sockopt_multi(void)
 	int err = -1;
 
 	cg_parent = test__join_cgroup("/parent");
-	if (CHECK_FAIL(cg_parent < 0))
+	if (!ASSERT_GE(cg_parent, 0, "join_cgroup /parent"))
 		goto out;
 
 	cg_child = test__join_cgroup("/parent/child");
-	if (CHECK_FAIL(cg_child < 0))
+	if (!ASSERT_GE(cg_child, 0, "join_cgroup /parent/child"))
 		goto out;
 
-	obj = bpf_object__open_file("sockopt_multi.o", NULL);
+	obj = bpf_object__open_file("sockopt_multi.bpf.o", NULL);
 	if (!ASSERT_OK_PTR(obj, "obj_load"))
 		goto out;
 
@@ -319,11 +319,11 @@ void test_sockopt_multi(void)
 		goto out;
 
 	sock_fd = socket(AF_INET, SOCK_STREAM, 0);
-	if (CHECK_FAIL(sock_fd < 0))
+	if (!ASSERT_GE(sock_fd, 0, "socket"))
 		goto out;
 
-	CHECK_FAIL(run_getsockopt_test(obj, cg_parent, cg_child, sock_fd));
-	CHECK_FAIL(run_setsockopt_test(obj, cg_parent, cg_child, sock_fd));
+	ASSERT_OK(run_getsockopt_test(obj, cg_parent, cg_child, sock_fd), "getsockopt_test");
+	ASSERT_OK(run_setsockopt_test(obj, cg_parent, cg_child, sock_fd), "setsockopt_test");
 
 out:
 	close(sock_fd);
diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c
index 30a99d2ed5c6..60d952719d27 100644
--- a/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c
+++ b/tools/testing/selftests/bpf/prog_tests/sockopt_sk.c
@@ -223,7 +223,7 @@ void test_sockopt_sk(void)
 	int cgroup_fd;
 
 	cgroup_fd = test__join_cgroup("/sockopt_sk");
-	if (CHECK_FAIL(cgroup_fd < 0))
+	if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup /sockopt_sk"))
 		return;
 
 	run_test(cgroup_fd);
diff --git a/tools/testing/selftests/bpf/prog_tests/spinlock.c b/tools/testing/selftests/bpf/prog_tests/spinlock.c
index 8e329eaee6d7..15eb1372d771 100644
--- a/tools/testing/selftests/bpf/prog_tests/spinlock.c
+++ b/tools/testing/selftests/bpf/prog_tests/spinlock.c
@@ -19,7 +19,7 @@ static void *spin_lock_thread(void *arg)
 
 void test_spinlock(void)
 {
-	const char *file = "./test_spin_lock.o";
+	const char *file = "./test_spin_lock.bpf.o";
 	pthread_t thread_id[4];
 	struct bpf_object *obj = NULL;
 	int prog_fd;
diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c
index 313f0a66232e..df59e4ae2951 100644
--- a/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c
+++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c
@@ -6,7 +6,7 @@ void test_stacktrace_map(void)
 	int control_map_fd, stackid_hmap_fd, stackmap_fd, stack_amap_fd;
 	const char *prog_name = "oncpu";
 	int err, prog_fd, stack_trace_len;
-	const char *file = "./test_stacktrace_map.o";
+	const char *file = "./test_stacktrace_map.bpf.o";
 	__u32 key, val, duration = 0;
 	struct bpf_program *prog;
 	struct bpf_object *obj;
diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c
index 1cb8dd36bd8f..c6ef06f55cdb 100644
--- a/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c
+++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c
@@ -5,7 +5,7 @@ void test_stacktrace_map_raw_tp(void)
 {
 	const char *prog_name = "oncpu";
 	int control_map_fd, stackid_hmap_fd, stackmap_fd;
-	const char *file = "./test_stacktrace_map.o";
+	const char *file = "./test_stacktrace_map.bpf.o";
 	__u32 key, val, duration = 0;
 	int err, prog_fd;
 	struct bpf_program *prog;
diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c
index 19c70880cfb3..58fe2c586ed7 100644
--- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c
+++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c
@@ -20,8 +20,8 @@ static void test_tailcall_1(void)
 		.repeat = 1,
 	);
 
-	err = bpf_prog_test_load("tailcall1.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
-			    &prog_fd);
+	err = bpf_prog_test_load("tailcall1.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
+				 &prog_fd);
 	if (CHECK_FAIL(err))
 		return;
 
@@ -156,8 +156,8 @@ static void test_tailcall_2(void)
 		.repeat = 1,
 	);
 
-	err = bpf_prog_test_load("tailcall2.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
-			    &prog_fd);
+	err = bpf_prog_test_load("tailcall2.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
+				 &prog_fd);
 	if (CHECK_FAIL(err))
 		return;
 
@@ -299,7 +299,7 @@ out:
  */
 static void test_tailcall_3(void)
 {
-	test_tailcall_count("tailcall3.o");
+	test_tailcall_count("tailcall3.bpf.o");
 }
 
 /* test_tailcall_6 checks that the count value of the tail call limit
@@ -307,7 +307,7 @@ static void test_tailcall_3(void)
  */
 static void test_tailcall_6(void)
 {
-	test_tailcall_count("tailcall6.o");
+	test_tailcall_count("tailcall6.bpf.o");
 }
 
 /* test_tailcall_4 checks that the kernel properly selects indirect jump
@@ -329,8 +329,8 @@ static void test_tailcall_4(void)
 		.repeat = 1,
 	);
 
-	err = bpf_prog_test_load("tailcall4.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
-			    &prog_fd);
+	err = bpf_prog_test_load("tailcall4.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
+				 &prog_fd);
 	if (CHECK_FAIL(err))
 		return;
 
@@ -419,8 +419,8 @@ static void test_tailcall_5(void)
 		.repeat = 1,
 	);
 
-	err = bpf_prog_test_load("tailcall5.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
-			    &prog_fd);
+	err = bpf_prog_test_load("tailcall5.bpf.o", BPF_PROG_TYPE_SCHED_CLS, &obj,
+				 &prog_fd);
 	if (CHECK_FAIL(err))
 		return;
 
@@ -507,8 +507,8 @@ static void test_tailcall_bpf2bpf_1(void)
 		.repeat = 1,
 	);
 
-	err = bpf_prog_test_load("tailcall_bpf2bpf1.o", BPF_PROG_TYPE_SCHED_CLS,
-			    &obj, &prog_fd);
+	err = bpf_prog_test_load("tailcall_bpf2bpf1.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
+				 &obj, &prog_fd);
 	if (CHECK_FAIL(err))
 		return;
 
@@ -591,8 +591,8 @@ static void test_tailcall_bpf2bpf_2(void)
 		.repeat = 1,
 	);
 
-	err = bpf_prog_test_load("tailcall_bpf2bpf2.o", BPF_PROG_TYPE_SCHED_CLS,
-			    &obj, &prog_fd);
+	err = bpf_prog_test_load("tailcall_bpf2bpf2.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
+				 &obj, &prog_fd);
 	if (CHECK_FAIL(err))
 		return;
 
@@ -671,8 +671,8 @@ static void test_tailcall_bpf2bpf_3(void)
 		.repeat = 1,
 	);
 
-	err = bpf_prog_test_load("tailcall_bpf2bpf3.o", BPF_PROG_TYPE_SCHED_CLS,
-			    &obj, &prog_fd);
+	err = bpf_prog_test_load("tailcall_bpf2bpf3.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
+				 &obj, &prog_fd);
 	if (CHECK_FAIL(err))
 		return;
 
@@ -766,8 +766,8 @@ static void test_tailcall_bpf2bpf_4(bool noise)
 		.repeat = 1,
 	);
 
-	err = bpf_prog_test_load("tailcall_bpf2bpf4.o", BPF_PROG_TYPE_SCHED_CLS,
-			    &obj, &prog_fd);
+	err = bpf_prog_test_load("tailcall_bpf2bpf4.bpf.o", BPF_PROG_TYPE_SCHED_CLS,
+				 &obj, &prog_fd);
 	if (CHECK_FAIL(err))
 		return;
 
diff --git a/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c b/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c
index 17947c9e1d66..3d34bab01e48 100644
--- a/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c
+++ b/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c
@@ -3,7 +3,7 @@
 
 void test_task_fd_query_rawtp(void)
 {
-	const char *file = "./test_get_stack_rawtp.o";
+	const char *file = "./test_get_stack_rawtp.bpf.o";
 	__u64 probe_offset, probe_addr;
 	__u32 len, prog_id, fd_type;
 	struct bpf_object *obj;
diff --git a/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c b/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c
index c2a98a7a8dfc..c717741bf8b6 100644
--- a/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c
+++ b/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c
@@ -4,7 +4,7 @@
 static void test_task_fd_query_tp_core(const char *probe_name,
 				       const char *tp_name)
 {
-	const char *file = "./test_tracepoint.o";
+	const char *file = "./test_tracepoint.bpf.o";
 	int err, bytes, efd, prog_fd, pmu_fd;
 	struct perf_event_attr attr = {};
 	__u64 probe_offset, probe_addr;
diff --git a/tools/testing/selftests/bpf/prog_tests/task_pt_regs.c b/tools/testing/selftests/bpf/prog_tests/task_pt_regs.c
index 61935e7e056a..f000734a3d1f 100644
--- a/tools/testing/selftests/bpf/prog_tests/task_pt_regs.c
+++ b/tools/testing/selftests/bpf/prog_tests/task_pt_regs.c
@@ -4,7 +4,7 @@
 #include "test_task_pt_regs.skel.h"
 
 /* uprobe attach point */
-static void trigger_func(void)
+static noinline void trigger_func(void)
 {
 	asm volatile ("");
 }
diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_estats.c b/tools/testing/selftests/bpf/prog_tests/tcp_estats.c
index 11bf755be4c9..e070bca2b764 100644
--- a/tools/testing/selftests/bpf/prog_tests/tcp_estats.c
+++ b/tools/testing/selftests/bpf/prog_tests/tcp_estats.c
@@ -3,14 +3,12 @@
 
 void test_tcp_estats(void)
 {
-	const char *file = "./test_tcp_estats.o";
+	const char *file = "./test_tcp_estats.bpf.o";
 	int err, prog_fd;
 	struct bpf_object *obj;
-	__u32 duration = 0;
 
 	err = bpf_prog_test_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd);
-	CHECK(err, "", "err %d errno %d\n", err, errno);
-	if (err)
+	if (!ASSERT_OK(err, ""))
 		return;
 
 	bpf_object__close(obj);
diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c
index 1fa772079967..617bbce6ef8f 100644
--- a/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c
+++ b/tools/testing/selftests/bpf/prog_tests/tcp_hdr_options.c
@@ -42,33 +42,10 @@ struct sk_fds {
 
 static int create_netns(void)
 {
-	if (CHECK(unshare(CLONE_NEWNET), "create netns",
-		  "unshare(CLONE_NEWNET): %s (%d)",
-		  strerror(errno), errno))
+	if (!ASSERT_OK(unshare(CLONE_NEWNET), "create netns"))
 		return -1;
 
-	if (CHECK(system("ip link set dev lo up"), "run ip cmd",
-		  "failed to bring lo link up\n"))
-		return -1;
-
-	return 0;
-}
-
-static int write_sysctl(const char *sysctl, const char *value)
-{
-	int fd, err, len;
-
-	fd = open(sysctl, O_WRONLY);
-	if (CHECK(fd == -1, "open sysctl", "open(%s): %s (%d)\n",
-		  sysctl, strerror(errno), errno))
-		return -1;
-
-	len = strlen(value);
-	err = write(fd, value, len);
-	close(fd);
-	if (CHECK(err != len, "write sysctl",
-		  "write(%s, %s): err:%d %s (%d)\n",
-		  sysctl, value, err, strerror(errno), errno))
+	if (!ASSERT_OK(system("ip link set dev lo up"), "run ip cmd"))
 		return -1;
 
 	return 0;
@@ -100,16 +77,12 @@ static int sk_fds_shutdown(struct sk_fds *sk_fds)
 
 	shutdown(sk_fds->active_fd, SHUT_WR);
 	ret = read(sk_fds->passive_fd, &abyte, sizeof(abyte));
-	if (CHECK(ret != 0, "read-after-shutdown(passive_fd):",
-		  "ret:%d %s (%d)\n",
-		  ret, strerror(errno), errno))
+	if (!ASSERT_EQ(ret, 0, "read-after-shutdown(passive_fd):"))
 		return -1;
 
 	shutdown(sk_fds->passive_fd, SHUT_WR);
 	ret = read(sk_fds->active_fd, &abyte, sizeof(abyte));
-	if (CHECK(ret != 0, "read-after-shutdown(active_fd):",
-		  "ret:%d %s (%d)\n",
-		  ret, strerror(errno), errno))
+	if (!ASSERT_EQ(ret, 0, "read-after-shutdown(active_fd):"))
 		return -1;
 
 	return 0;
@@ -122,8 +95,7 @@ static int sk_fds_connect(struct sk_fds *sk_fds, bool fast_open)
 	socklen_t len;
 
 	sk_fds->srv_fd = start_server(AF_INET6, SOCK_STREAM, LO_ADDR6, 0, 0);
-	if (CHECK(sk_fds->srv_fd == -1, "start_server", "%s (%d)\n",
-		  strerror(errno), errno))
+	if (!ASSERT_NEQ(sk_fds->srv_fd, -1, "start_server"))
 		goto error;
 
 	if (fast_open)
@@ -132,28 +104,25 @@ static int sk_fds_connect(struct sk_fds *sk_fds, bool fast_open)
 	else
 		sk_fds->active_fd = connect_to_fd(sk_fds->srv_fd, 0);
 
-	if (CHECK_FAIL(sk_fds->active_fd == -1)) {
+	if (!ASSERT_NEQ(sk_fds->active_fd, -1, "")) {
 		close(sk_fds->srv_fd);
 		goto error;
 	}
 
 	len = sizeof(addr6);
-	if (CHECK(getsockname(sk_fds->srv_fd, (struct sockaddr *)&addr6,
-			      &len), "getsockname(srv_fd)", "%s (%d)\n",
-		  strerror(errno), errno))
+	if (!ASSERT_OK(getsockname(sk_fds->srv_fd, (struct sockaddr *)&addr6,
+				   &len), "getsockname(srv_fd)"))
 		goto error_close;
 	sk_fds->passive_lport = ntohs(addr6.sin6_port);
 
 	len = sizeof(addr6);
-	if (CHECK(getsockname(sk_fds->active_fd, (struct sockaddr *)&addr6,
-			      &len), "getsockname(active_fd)", "%s (%d)\n",
-		  strerror(errno), errno))
+	if (!ASSERT_OK(getsockname(sk_fds->active_fd, (struct sockaddr *)&addr6,
+				   &len), "getsockname(active_fd)"))
 		goto error_close;
 	sk_fds->active_lport = ntohs(addr6.sin6_port);
 
 	sk_fds->passive_fd = accept(sk_fds->srv_fd, NULL, 0);
-	if (CHECK(sk_fds->passive_fd == -1, "accept(srv_fd)", "%s (%d)\n",
-		  strerror(errno), errno))
+	if (!ASSERT_NEQ(sk_fds->passive_fd, -1, "accept(srv_fd)"))
 		goto error_close;
 
 	if (fast_open) {
@@ -161,8 +130,7 @@ static int sk_fds_connect(struct sk_fds *sk_fds, bool fast_open)
 		int ret;
 
 		ret = read(sk_fds->passive_fd, bytes_in, sizeof(bytes_in));
-		if (CHECK(ret != sizeof(fast), "read fastopen syn data",
-			  "expected=%lu actual=%d\n", sizeof(fast), ret)) {
+		if (!ASSERT_EQ(ret, sizeof(fast), "read fastopen syn data")) {
 			close(sk_fds->passive_fd);
 			goto error_close;
 		}
@@ -183,8 +151,7 @@ static int check_hdr_opt(const struct bpf_test_option *exp,
 			 const struct bpf_test_option *act,
 			 const char *hdr_desc)
 {
-	if (CHECK(memcmp(exp, act, sizeof(*exp)),
-		  "expected-vs-actual", "unexpected %s\n", hdr_desc)) {
+	if (!ASSERT_OK(memcmp(exp, act, sizeof(*exp)), hdr_desc)) {
 		print_option(exp, "expected: ");
 		print_option(act, "  actual: ");
 		return -1;
@@ -198,13 +165,11 @@ static int check_hdr_stg(const struct hdr_stg *exp, int fd,
 {
 	struct hdr_stg act;
 
-	if (CHECK(bpf_map_lookup_elem(hdr_stg_map_fd, &fd, &act),
-		  "map_lookup(hdr_stg_map_fd)", "%s %s (%d)\n",
-		  stg_desc, strerror(errno), errno))
+	if (!ASSERT_OK(bpf_map_lookup_elem(hdr_stg_map_fd, &fd, &act),
+		  "map_lookup(hdr_stg_map_fd)"))
 		return -1;
 
-	if (CHECK(memcmp(exp, &act, sizeof(*exp)),
-		  "expected-vs-actual", "unexpected %s\n", stg_desc)) {
+	if (!ASSERT_OK(memcmp(exp, &act, sizeof(*exp)), stg_desc)) {
 		print_hdr_stg(exp, "expected: ");
 		print_hdr_stg(&act, "  actual: ");
 		return -1;
@@ -248,9 +213,8 @@ static void check_hdr_and_close_fds(struct sk_fds *sk_fds)
 	if (sk_fds_shutdown(sk_fds))
 		goto check_linum;
 
-	if (CHECK(expected_inherit_cb_flags != skel->bss->inherit_cb_flags,
-		  "Unexpected inherit_cb_flags", "0x%x != 0x%x\n",
-		  skel->bss->inherit_cb_flags, expected_inherit_cb_flags))
+	if (!ASSERT_EQ(expected_inherit_cb_flags, skel->bss->inherit_cb_flags,
+		       "inherit_cb_flags"))
 		goto check_linum;
 
 	if (check_hdr_stg(&exp_passive_hdr_stg, sk_fds->passive_fd,
@@ -277,7 +241,7 @@ static void check_hdr_and_close_fds(struct sk_fds *sk_fds)
 		      "active_fin_in");
 
 check_linum:
-	CHECK_FAIL(check_error_linum(sk_fds));
+	ASSERT_FALSE(check_error_linum(sk_fds), "check_error_linum");
 	sk_fds_close(sk_fds);
 }
 
@@ -517,26 +481,20 @@ static void misc(void)
 		/* MSG_EOR to ensure skb will not be combined */
 		ret = send(sk_fds.active_fd, send_msg, sizeof(send_msg),
 			   MSG_EOR);
-		if (CHECK(ret != sizeof(send_msg), "send(msg)", "ret:%d\n",
-			  ret))
+		if (!ASSERT_EQ(ret, sizeof(send_msg), "send(msg)"))
 			goto check_linum;
 
 		ret = read(sk_fds.passive_fd, recv_msg, sizeof(recv_msg));
-		if (CHECK(ret != sizeof(send_msg), "read(msg)", "ret:%d\n",
-			  ret))
+		if (ASSERT_EQ(ret, sizeof(send_msg), "read(msg)"))
 			goto check_linum;
 	}
 
 	if (sk_fds_shutdown(&sk_fds))
 		goto check_linum;
 
-	CHECK(misc_skel->bss->nr_syn != 1, "unexpected nr_syn",
-	      "expected (1) != actual (%u)\n",
-		misc_skel->bss->nr_syn);
+	ASSERT_EQ(misc_skel->bss->nr_syn, 1, "unexpected nr_syn");
 
-	CHECK(misc_skel->bss->nr_data != nr_data, "unexpected nr_data",
-	      "expected (%u) != actual (%u)\n",
-	      nr_data, misc_skel->bss->nr_data);
+	ASSERT_EQ(misc_skel->bss->nr_data, nr_data, "unexpected nr_data");
 
 	/* The last ACK may have been delayed, so it is either 1 or 2. */
 	CHECK(misc_skel->bss->nr_pure_ack != 1 &&
@@ -545,12 +503,10 @@ static void misc(void)
 	      "expected (1 or 2) != actual (%u)\n",
 		misc_skel->bss->nr_pure_ack);
 
-	CHECK(misc_skel->bss->nr_fin != 1, "unexpected nr_fin",
-	      "expected (1) != actual (%u)\n",
-	      misc_skel->bss->nr_fin);
+	ASSERT_EQ(misc_skel->bss->nr_fin, 1, "unexpected nr_fin");
 
 check_linum:
-	CHECK_FAIL(check_error_linum(&sk_fds));
+	ASSERT_FALSE(check_error_linum(&sk_fds), "check_error_linum");
 	sk_fds_close(&sk_fds);
 	bpf_link__destroy(link);
 }
@@ -575,15 +531,15 @@ void test_tcp_hdr_options(void)
 	int i;
 
 	skel = test_tcp_hdr_options__open_and_load();
-	if (CHECK(!skel, "open and load skel", "failed"))
+	if (!ASSERT_OK_PTR(skel, "open and load skel"))
 		return;
 
 	misc_skel = test_misc_tcp_hdr_options__open_and_load();
-	if (CHECK(!misc_skel, "open and load misc test skel", "failed"))
+	if (!ASSERT_OK_PTR(misc_skel, "open and load misc test skel"))
 		goto skel_destroy;
 
 	cg_fd = test__join_cgroup(CG_NAME);
-	if (CHECK_FAIL(cg_fd < 0))
+	if (ASSERT_GE(cg_fd, 0, "join_cgroup"))
 		goto skel_destroy;
 
 	for (i = 0; i < ARRAY_SIZE(tests); i++) {
diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_rtt.c b/tools/testing/selftests/bpf/prog_tests/tcp_rtt.c
index 96ff2c20af81..8fe84da1b9b4 100644
--- a/tools/testing/selftests/bpf/prog_tests/tcp_rtt.c
+++ b/tools/testing/selftests/bpf/prog_tests/tcp_rtt.c
@@ -16,8 +16,7 @@ static void send_byte(int fd)
 {
 	char b = 0x55;
 
-	if (CHECK_FAIL(write(fd, &b, sizeof(b)) != 1))
-		perror("Failed to send single byte");
+	ASSERT_EQ(write(fd, &b, sizeof(b)), 1, "send single byte");
 }
 
 static int wait_for_ack(int fd, int retries)
@@ -51,10 +50,8 @@ static int verify_sk(int map_fd, int client_fd, const char *msg, __u32 invoked,
 	int err = 0;
 	struct tcp_rtt_storage val;
 
-	if (CHECK_FAIL(bpf_map_lookup_elem(map_fd, &client_fd, &val) < 0)) {
-		perror("Failed to read socket storage");
+	if (!ASSERT_GE(bpf_map_lookup_elem(map_fd, &client_fd, &val), 0, "read socket storage"))
 		return -1;
-	}
 
 	if (val.invoked != invoked) {
 		log_err("%s: unexpected bpf_tcp_sock.invoked %d != %d",
@@ -151,14 +148,14 @@ void test_tcp_rtt(void)
 	int server_fd, cgroup_fd;
 
 	cgroup_fd = test__join_cgroup("/tcp_rtt");
-	if (CHECK_FAIL(cgroup_fd < 0))
+	if (!ASSERT_GE(cgroup_fd, 0, "join_cgroup /tcp_rtt"))
 		return;
 
 	server_fd = start_server(AF_INET, SOCK_STREAM, NULL, 0, 0);
-	if (CHECK_FAIL(server_fd < 0))
+	if (!ASSERT_GE(server_fd, 0, "start_server"))
 		goto close_cgroup_fd;
 
-	CHECK_FAIL(run_test(cgroup_fd, server_fd));
+	ASSERT_OK(run_test(cgroup_fd, server_fd), "run_test");
 
 	close(server_fd);
 
diff --git a/tools/testing/selftests/bpf/prog_tests/tcpbpf_user.c b/tools/testing/selftests/bpf/prog_tests/tcpbpf_user.c
index 87923d2865b7..7e8fe1bad03f 100644
--- a/tools/testing/selftests/bpf/prog_tests/tcpbpf_user.c
+++ b/tools/testing/selftests/bpf/prog_tests/tcpbpf_user.c
@@ -8,8 +8,6 @@
 #define LO_ADDR6 "::1"
 #define CG_NAME "/tcpbpf-user-test"
 
-static __u32 duration;
-
 static void verify_result(struct tcpbpf_globals *result)
 {
 	__u32 expected_events = ((1 << BPF_SOCK_OPS_TIMEOUT_INIT) |
@@ -22,9 +20,7 @@ static void verify_result(struct tcpbpf_globals *result)
 				 (1 << BPF_SOCK_OPS_TCP_LISTEN_CB));
 
 	/* check global map */
-	CHECK(expected_events != result->event_map, "event_map",
-	      "unexpected event_map: actual 0x%08x != expected 0x%08x\n",
-	      result->event_map, expected_events);
+	ASSERT_EQ(expected_events, result->event_map, "event_map");
 
 	ASSERT_EQ(result->bytes_received, 501, "bytes_received");
 	ASSERT_EQ(result->bytes_acked, 1002, "bytes_acked");
@@ -56,18 +52,15 @@ static void run_test(struct tcpbpf_globals *result)
 	int i, rv;
 
 	listen_fd = start_server(AF_INET6, SOCK_STREAM, LO_ADDR6, 0, 0);
-	if (CHECK(listen_fd == -1, "start_server", "listen_fd:%d errno:%d\n",
-		  listen_fd, errno))
+	if (!ASSERT_NEQ(listen_fd, -1, "start_server"))
 		goto done;
 
 	cli_fd = connect_to_fd(listen_fd, 0);
-	if (CHECK(cli_fd == -1, "connect_to_fd(listen_fd)",
-		  "cli_fd:%d errno:%d\n", cli_fd, errno))
+	if (!ASSERT_NEQ(cli_fd, -1, "connect_to_fd(listen_fd)"))
 		goto done;
 
 	accept_fd = accept(listen_fd, NULL, NULL);
-	if (CHECK(accept_fd == -1, "accept(listen_fd)",
-		  "accept_fd:%d errno:%d\n", accept_fd, errno))
+	if (!ASSERT_NEQ(accept_fd, -1, "accept(listen_fd)"))
 		goto done;
 
 	/* Send 1000B of '+'s from cli_fd -> accept_fd */
@@ -75,11 +68,11 @@ static void run_test(struct tcpbpf_globals *result)
 		buf[i] = '+';
 
 	rv = send(cli_fd, buf, 1000, 0);
-	if (CHECK(rv != 1000, "send(cli_fd)", "rv:%d errno:%d\n", rv, errno))
+	if (!ASSERT_EQ(rv, 1000, "send(cli_fd)"))
 		goto done;
 
 	rv = recv(accept_fd, buf, 1000, 0);
-	if (CHECK(rv != 1000, "recv(accept_fd)", "rv:%d errno:%d\n", rv, errno))
+	if (!ASSERT_EQ(rv, 1000, "recv(accept_fd)"))
 		goto done;
 
 	/* Send 500B of '.'s from accept_fd ->cli_fd */
@@ -87,11 +80,11 @@ static void run_test(struct tcpbpf_globals *result)
 		buf[i] = '.';
 
 	rv = send(accept_fd, buf, 500, 0);
-	if (CHECK(rv != 500, "send(accept_fd)", "rv:%d errno:%d\n", rv, errno))
+	if (!ASSERT_EQ(rv, 500, "send(accept_fd)"))
 		goto done;
 
 	rv = recv(cli_fd, buf, 500, 0);
-	if (CHECK(rv != 500, "recv(cli_fd)", "rv:%d errno:%d\n", rv, errno))
+	if (!ASSERT_EQ(rv, 500, "recv(cli_fd)"))
 		goto done;
 
 	/*
@@ -100,12 +93,12 @@ static void run_test(struct tcpbpf_globals *result)
 	 */
 	shutdown(accept_fd, SHUT_WR);
 	err = recv(cli_fd, buf, 1, 0);
-	if (CHECK(err, "recv(cli_fd) for fin", "err:%d errno:%d\n", err, errno))
+	if (!ASSERT_OK(err, "recv(cli_fd) for fin"))
 		goto done;
 
 	shutdown(cli_fd, SHUT_WR);
 	err = recv(accept_fd, buf, 1, 0);
-	CHECK(err, "recv(accept_fd) for fin", "err:%d errno:%d\n", err, errno);
+	ASSERT_OK(err, "recv(accept_fd) for fin");
 done:
 	if (accept_fd != -1)
 		close(accept_fd);
@@ -124,12 +117,11 @@ void test_tcpbpf_user(void)
 	int cg_fd = -1;
 
 	skel = test_tcpbpf_kern__open_and_load();
-	if (CHECK(!skel, "open and load skel", "failed"))
+	if (!ASSERT_OK_PTR(skel, "open and load skel"))
 		return;
 
 	cg_fd = test__join_cgroup(CG_NAME);
-	if (CHECK(cg_fd < 0, "test__join_cgroup(" CG_NAME ")",
-		  "cg_fd:%d errno:%d", cg_fd, errno))
+	if (!ASSERT_GE(cg_fd, 0, "test__join_cgroup(" CG_NAME ")"))
 		goto err;
 
 	skel->links.bpf_testcb = bpf_program__attach_cgroup(skel->progs.bpf_testcb, cg_fd);
diff --git a/tools/testing/selftests/bpf/prog_tests/test_bprm_opts.c b/tools/testing/selftests/bpf/prog_tests/test_bprm_opts.c
index 2559bb775762..a0054019e677 100644
--- a/tools/testing/selftests/bpf/prog_tests/test_bprm_opts.c
+++ b/tools/testing/selftests/bpf/prog_tests/test_bprm_opts.c
@@ -9,18 +9,10 @@
 
 #include "bprm_opts.skel.h"
 #include "network_helpers.h"
-
-#ifndef __NR_pidfd_open
-#define __NR_pidfd_open 434
-#endif
+#include "task_local_storage_helpers.h"
 
 static const char * const bash_envp[] = { "TMPDIR=shouldnotbeset", NULL };
 
-static inline int sys_pidfd_open(pid_t pid, unsigned int flags)
-{
-	return syscall(__NR_pidfd_open, pid, flags);
-}
-
 static int update_storage(int map_fd, int secureexec)
 {
 	int task_fd, ret = 0;
diff --git a/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c b/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c
index b90ee47d3111..7295cc60f724 100644
--- a/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c
+++ b/tools/testing/selftests/bpf/prog_tests/test_global_funcs.c
@@ -65,23 +65,23 @@ struct test_def {
 void test_test_global_funcs(void)
 {
 	struct test_def tests[] = {
-		{ "test_global_func1.o", "combined stack size of 4 calls is 544" },
-		{ "test_global_func2.o" },
-		{ "test_global_func3.o" , "the call stack of 8 frames" },
-		{ "test_global_func4.o" },
-		{ "test_global_func5.o" , "expected pointer to ctx, but got PTR" },
-		{ "test_global_func6.o" , "modified ctx ptr R2" },
-		{ "test_global_func7.o" , "foo() doesn't return scalar" },
-		{ "test_global_func8.o" },
-		{ "test_global_func9.o" },
-		{ "test_global_func10.o", "invalid indirect read from stack" },
-		{ "test_global_func11.o", "Caller passes invalid args into func#1" },
-		{ "test_global_func12.o", "invalid mem access 'mem_or_null'" },
-		{ "test_global_func13.o", "Caller passes invalid args into func#1" },
-		{ "test_global_func14.o", "reference type('FWD S') size cannot be determined" },
-		{ "test_global_func15.o", "At program exit the register R0 has value" },
-		{ "test_global_func16.o", "invalid indirect read from stack" },
-		{ "test_global_func17.o", "Caller passes invalid args into func#1" },
+		{ "test_global_func1.bpf.o", "combined stack size of 4 calls is 544" },
+		{ "test_global_func2.bpf.o" },
+		{ "test_global_func3.bpf.o", "the call stack of 8 frames" },
+		{ "test_global_func4.bpf.o" },
+		{ "test_global_func5.bpf.o", "expected pointer to ctx, but got PTR" },
+		{ "test_global_func6.bpf.o", "modified ctx ptr R2" },
+		{ "test_global_func7.bpf.o", "foo() doesn't return scalar" },
+		{ "test_global_func8.bpf.o" },
+		{ "test_global_func9.bpf.o" },
+		{ "test_global_func10.bpf.o", "invalid indirect read from stack" },
+		{ "test_global_func11.bpf.o", "Caller passes invalid args into func#1" },
+		{ "test_global_func12.bpf.o", "invalid mem access 'mem_or_null'" },
+		{ "test_global_func13.bpf.o", "Caller passes invalid args into func#1" },
+		{ "test_global_func14.bpf.o", "reference type('FWD S') size cannot be determined" },
+		{ "test_global_func15.bpf.o", "At program exit the register R0 has value" },
+		{ "test_global_func16.bpf.o", "invalid indirect read from stack" },
+		{ "test_global_func17.bpf.o", "Caller passes invalid args into func#1" },
 	};
 	libbpf_print_fn_t old_print_fn = NULL;
 	int err, i, duration = 0;
diff --git a/tools/testing/selftests/bpf/prog_tests/test_local_storage.c b/tools/testing/selftests/bpf/prog_tests/test_local_storage.c
index 26ac26a88026..9c77cd6b1eaf 100644
--- a/tools/testing/selftests/bpf/prog_tests/test_local_storage.c
+++ b/tools/testing/selftests/bpf/prog_tests/test_local_storage.c
@@ -11,15 +11,7 @@
 
 #include "local_storage.skel.h"
 #include "network_helpers.h"
-
-#ifndef __NR_pidfd_open
-#define __NR_pidfd_open 434
-#endif
-
-static inline int sys_pidfd_open(pid_t pid, unsigned int flags)
-{
-	return syscall(__NR_pidfd_open, pid, flags);
-}
+#include "task_local_storage_helpers.h"
 
 static unsigned int duration;
 
diff --git a/tools/testing/selftests/bpf/prog_tests/test_overhead.c b/tools/testing/selftests/bpf/prog_tests/test_overhead.c
index 05acb376f74d..f27013e38d03 100644
--- a/tools/testing/selftests/bpf/prog_tests/test_overhead.c
+++ b/tools/testing/selftests/bpf/prog_tests/test_overhead.c
@@ -72,7 +72,7 @@ void test_test_overhead(void)
 	if (CHECK_FAIL(prctl(PR_GET_NAME, comm, 0L, 0L, 0L)))
 		return;
 
-	obj = bpf_object__open_file("./test_overhead.o", NULL);
+	obj = bpf_object__open_file("./test_overhead.bpf.o", NULL);
 	if (!ASSERT_OK_PTR(obj, "obj_open_file"))
 		return;
 
diff --git a/tools/testing/selftests/bpf/prog_tests/time_tai.c b/tools/testing/selftests/bpf/prog_tests/time_tai.c
new file mode 100644
index 000000000000..a31119823666
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/time_tai.c
@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2022 Linutronix GmbH */
+
+#include <test_progs.h>
+#include <network_helpers.h>
+
+#include "test_time_tai.skel.h"
+
+#include <time.h>
+#include <stdint.h>
+
+#define TAI_THRESHOLD	1000000000ULL /* 1s */
+#define NSEC_PER_SEC	1000000000ULL
+
+static __u64 ts_to_ns(const struct timespec *ts)
+{
+	return ts->tv_sec * NSEC_PER_SEC + ts->tv_nsec;
+}
+
+void test_time_tai(void)
+{
+	struct __sk_buff skb = {
+		.cb[0] = 0,
+		.cb[1] = 0,
+		.tstamp = 0,
+	};
+	LIBBPF_OPTS(bpf_test_run_opts, topts,
+		.data_in = &pkt_v4,
+		.data_size_in = sizeof(pkt_v4),
+		.ctx_in = &skb,
+		.ctx_size_in = sizeof(skb),
+		.ctx_out = &skb,
+		.ctx_size_out = sizeof(skb),
+	);
+	struct test_time_tai *skel;
+	struct timespec now_tai;
+	__u64 ts1, ts2, now;
+	int ret, prog_fd;
+
+	/* Open and load */
+	skel = test_time_tai__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "tai_open"))
+		return;
+
+	/* Run test program */
+	prog_fd = bpf_program__fd(skel->progs.time_tai);
+	ret = bpf_prog_test_run_opts(prog_fd, &topts);
+	ASSERT_OK(ret, "test_run");
+
+	/* Retrieve generated TAI timestamps */
+	ts1 = skb.tstamp;
+	ts2 = skb.cb[0] | ((__u64)skb.cb[1] << 32);
+
+	/* TAI != 0 */
+	ASSERT_NEQ(ts1, 0, "tai_ts1");
+	ASSERT_NEQ(ts2, 0, "tai_ts2");
+
+	/* TAI is moving forward only */
+	ASSERT_GT(ts2, ts1, "tai_forward");
+
+	/* Check for future */
+	ret = clock_gettime(CLOCK_TAI, &now_tai);
+	ASSERT_EQ(ret, 0, "tai_gettime");
+	now = ts_to_ns(&now_tai);
+
+	ASSERT_TRUE(now > ts1, "tai_future_ts1");
+	ASSERT_TRUE(now > ts2, "tai_future_ts2");
+
+	/* Check for reasonable range */
+	ASSERT_TRUE(now - ts1 < TAI_THRESHOLD, "tai_range_ts1");
+	ASSERT_TRUE(now - ts2 < TAI_THRESHOLD, "tai_range_ts2");
+
+	test_time_tai__destroy(skel);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c b/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c
index 39e79291c82b..a479080533db 100644
--- a/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c
+++ b/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c
@@ -6,7 +6,7 @@ void serial_test_tp_attach_query(void)
 	const int num_progs = 3;
 	int i, j, bytes, efd, err, prog_fd[num_progs], pmu_fd[num_progs];
 	__u32 duration = 0, info_len, saved_prog_ids[num_progs];
-	const char *file = "./test_tracepoint.o";
+	const char *file = "./test_tracepoint.bpf.o";
 	struct perf_event_query_bpf *query;
 	struct perf_event_attr attr = {};
 	struct bpf_object *obj[num_progs];
diff --git a/tools/testing/selftests/bpf/prog_tests/tracing_struct.c b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c
new file mode 100644
index 000000000000..d5022b91d1e4
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/tracing_struct.c
@@ -0,0 +1,63 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
+
+#include <test_progs.h>
+#include "tracing_struct.skel.h"
+
+static void test_fentry(void)
+{
+	struct tracing_struct *skel;
+	int err;
+
+	skel = tracing_struct__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "tracing_struct__open_and_load"))
+		return;
+
+	err = tracing_struct__attach(skel);
+	if (!ASSERT_OK(err, "tracing_struct__attach"))
+		return;
+
+	ASSERT_OK(trigger_module_test_read(256), "trigger_read");
+
+	ASSERT_EQ(skel->bss->t1_a_a, 2, "t1:a.a");
+	ASSERT_EQ(skel->bss->t1_a_b, 3, "t1:a.b");
+	ASSERT_EQ(skel->bss->t1_b, 1, "t1:b");
+	ASSERT_EQ(skel->bss->t1_c, 4, "t1:c");
+
+	ASSERT_EQ(skel->bss->t1_nregs, 4, "t1 nregs");
+	ASSERT_EQ(skel->bss->t1_reg0, 2, "t1 reg0");
+	ASSERT_EQ(skel->bss->t1_reg1, 3, "t1 reg1");
+	ASSERT_EQ(skel->bss->t1_reg2, 1, "t1 reg2");
+	ASSERT_EQ(skel->bss->t1_reg3, 4, "t1 reg3");
+	ASSERT_EQ(skel->bss->t1_ret, 10, "t1 ret");
+
+	ASSERT_EQ(skel->bss->t2_a, 1, "t2:a");
+	ASSERT_EQ(skel->bss->t2_b_a, 2, "t2:b.a");
+	ASSERT_EQ(skel->bss->t2_b_b, 3, "t2:b.b");
+	ASSERT_EQ(skel->bss->t2_c, 4, "t2:c");
+	ASSERT_EQ(skel->bss->t2_ret, 10, "t2 ret");
+
+	ASSERT_EQ(skel->bss->t3_a, 1, "t3:a");
+	ASSERT_EQ(skel->bss->t3_b, 4, "t3:b");
+	ASSERT_EQ(skel->bss->t3_c_a, 2, "t3:c.a");
+	ASSERT_EQ(skel->bss->t3_c_b, 3, "t3:c.b");
+	ASSERT_EQ(skel->bss->t3_ret, 10, "t3 ret");
+
+	ASSERT_EQ(skel->bss->t4_a_a, 10, "t4:a.a");
+	ASSERT_EQ(skel->bss->t4_b, 1, "t4:b");
+	ASSERT_EQ(skel->bss->t4_c, 2, "t4:c");
+	ASSERT_EQ(skel->bss->t4_d, 3, "t4:d");
+	ASSERT_EQ(skel->bss->t4_e_a, 2, "t4:e.a");
+	ASSERT_EQ(skel->bss->t4_e_b, 3, "t4:e.b");
+	ASSERT_EQ(skel->bss->t4_ret, 21, "t4 ret");
+
+	ASSERT_EQ(skel->bss->t5_ret, 1, "t5 ret");
+
+	tracing_struct__detach(skel);
+	tracing_struct__destroy(skel);
+}
+
+void test_tracing_struct(void)
+{
+	test_fentry();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c
index b0acbda6dbf5..564b75bc087f 100644
--- a/tools/testing/selftests/bpf/prog_tests/trampoline_count.c
+++ b/tools/testing/selftests/bpf/prog_tests/trampoline_count.c
@@ -35,7 +35,7 @@ static struct bpf_program *load_prog(char *file, char *name, struct inst *inst)
 /* TODO: use different target function to run in concurrent mode */
 void serial_test_trampoline_count(void)
 {
-	char *file = "test_trampoline_count.o";
+	char *file = "test_trampoline_count.bpf.o";
 	char *const progs[] = { "fentry_test", "fmod_ret_test", "fexit_test" };
 	struct inst inst[MAX_TRAMP_PROGS + 1] = {};
 	struct bpf_program *prog;
diff --git a/tools/testing/selftests/bpf/prog_tests/udp_limit.c b/tools/testing/selftests/bpf/prog_tests/udp_limit.c
index 56c9d6bd38a3..2643d896ddae 100644
--- a/tools/testing/selftests/bpf/prog_tests/udp_limit.c
+++ b/tools/testing/selftests/bpf/prog_tests/udp_limit.c
@@ -5,8 +5,6 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 
-static int duration;
-
 void test_udp_limit(void)
 {
 	struct udp_limit *skel;
@@ -14,11 +12,11 @@ void test_udp_limit(void)
 	int cgroup_fd;
 
 	cgroup_fd = test__join_cgroup("/udp_limit");
-	if (CHECK(cgroup_fd < 0, "cg-join", "errno %d", errno))
+	if (!ASSERT_GE(cgroup_fd, 0, "cg-join"))
 		return;
 
 	skel = udp_limit__open_and_load();
-	if (CHECK(!skel, "skel-load", "errno %d", errno))
+	if (!ASSERT_OK_PTR(skel, "skel-load"))
 		goto close_cgroup_fd;
 
 	skel->links.sock = bpf_program__attach_cgroup(skel->progs.sock, cgroup_fd);
@@ -32,11 +30,11 @@ void test_udp_limit(void)
 	 * verify that.
 	 */
 	fd1 = socket(AF_INET, SOCK_DGRAM, 0);
-	if (CHECK(fd1 < 0, "fd1", "errno %d", errno))
+	if (!ASSERT_GE(fd1, 0, "socket(fd1)"))
 		goto close_skeleton;
 
 	fd2 = socket(AF_INET, SOCK_DGRAM, 0);
-	if (CHECK(fd2 >= 0, "fd2", "errno %d", errno))
+	if (!ASSERT_LT(fd2, 0, "socket(fd2)"))
 		goto close_skeleton;
 
 	/* We can reopen again after close. */
@@ -44,7 +42,7 @@ void test_udp_limit(void)
 	fd1 = -1;
 
 	fd1 = socket(AF_INET, SOCK_DGRAM, 0);
-	if (CHECK(fd1 < 0, "fd1-again", "errno %d", errno))
+	if (!ASSERT_GE(fd1, 0, "socket(fd1-again)"))
 		goto close_skeleton;
 
 	/* Make sure the program was invoked the expected
@@ -54,13 +52,11 @@ void test_udp_limit(void)
 	 * - close fd1          - BPF_CGROUP_INET_SOCK_RELEASE
 	 * - open fd1 again     - BPF_CGROUP_INET_SOCK_CREATE
 	 */
-	if (CHECK(skel->bss->invocations != 4, "bss-invocations",
-		  "invocations=%d", skel->bss->invocations))
+	if (!ASSERT_EQ(skel->bss->invocations, 4, "bss-invocations"))
 		goto close_skeleton;
 
 	/* We should still have a single socket in use */
-	if (CHECK(skel->bss->in_use != 1, "bss-in_use",
-		  "in_use=%d", skel->bss->in_use))
+	if (!ASSERT_EQ(skel->bss->in_use, 1, "bss-in_use"))
 		goto close_skeleton;
 
 close_skeleton:
diff --git a/tools/testing/selftests/bpf/prog_tests/user_ringbuf.c b/tools/testing/selftests/bpf/prog_tests/user_ringbuf.c
new file mode 100644
index 000000000000..02b18d018b36
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/user_ringbuf.c
@@ -0,0 +1,754 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
+
+#define _GNU_SOURCE
+#include <linux/compiler.h>
+#include <linux/ring_buffer.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <sys/sysinfo.h>
+#include <test_progs.h>
+#include <uapi/linux/bpf.h>
+#include <unistd.h>
+
+#include "user_ringbuf_fail.skel.h"
+#include "user_ringbuf_success.skel.h"
+
+#include "../progs/test_user_ringbuf.h"
+
+static size_t log_buf_sz = 1 << 20; /* 1 MB */
+static char obj_log_buf[1048576];
+static const long c_sample_size = sizeof(struct sample) + BPF_RINGBUF_HDR_SZ;
+static const long c_ringbuf_size = 1 << 12; /* 1 small page */
+static const long c_max_entries = c_ringbuf_size / c_sample_size;
+
+static void drain_current_samples(void)
+{
+	syscall(__NR_getpgid);
+}
+
+static int write_samples(struct user_ring_buffer *ringbuf, uint32_t num_samples)
+{
+	int i, err = 0;
+
+	/* Write some number of samples to the ring buffer. */
+	for (i = 0; i < num_samples; i++) {
+		struct sample *entry;
+		int read;
+
+		entry = user_ring_buffer__reserve(ringbuf, sizeof(*entry));
+		if (!entry) {
+			err = -errno;
+			goto done;
+		}
+
+		entry->pid = getpid();
+		entry->seq = i;
+		entry->value = i * i;
+
+		read = snprintf(entry->comm, sizeof(entry->comm), "%u", i);
+		if (read <= 0) {
+			/* Assert on the error path to avoid spamming logs with
+			 * mostly success messages.
+			 */
+			ASSERT_GT(read, 0, "snprintf_comm");
+			err = read;
+			user_ring_buffer__discard(ringbuf, entry);
+			goto done;
+		}
+
+		user_ring_buffer__submit(ringbuf, entry);
+	}
+
+done:
+	drain_current_samples();
+
+	return err;
+}
+
+static struct user_ringbuf_success *open_load_ringbuf_skel(void)
+{
+	struct user_ringbuf_success *skel;
+	int err;
+
+	skel = user_ringbuf_success__open();
+	if (!ASSERT_OK_PTR(skel, "skel_open"))
+		return NULL;
+
+	err = bpf_map__set_max_entries(skel->maps.user_ringbuf, c_ringbuf_size);
+	if (!ASSERT_OK(err, "set_max_entries"))
+		goto cleanup;
+
+	err = bpf_map__set_max_entries(skel->maps.kernel_ringbuf, c_ringbuf_size);
+	if (!ASSERT_OK(err, "set_max_entries"))
+		goto cleanup;
+
+	err = user_ringbuf_success__load(skel);
+	if (!ASSERT_OK(err, "skel_load"))
+		goto cleanup;
+
+	return skel;
+
+cleanup:
+	user_ringbuf_success__destroy(skel);
+	return NULL;
+}
+
+static void test_user_ringbuf_mappings(void)
+{
+	int err, rb_fd;
+	int page_size = getpagesize();
+	void *mmap_ptr;
+	struct user_ringbuf_success *skel;
+
+	skel = open_load_ringbuf_skel();
+	if (!skel)
+		return;
+
+	rb_fd = bpf_map__fd(skel->maps.user_ringbuf);
+	/* cons_pos can be mapped R/O, can't add +X with mprotect. */
+	mmap_ptr = mmap(NULL, page_size, PROT_READ, MAP_SHARED, rb_fd, 0);
+	ASSERT_OK_PTR(mmap_ptr, "ro_cons_pos");
+	ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_WRITE), "write_cons_pos_protect");
+	ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_cons_pos_protect");
+	ASSERT_ERR_PTR(mremap(mmap_ptr, 0, 4 * page_size, MREMAP_MAYMOVE), "wr_prod_pos");
+	err = -errno;
+	ASSERT_ERR(err, "wr_prod_pos_err");
+	ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_ro_cons");
+
+	/* prod_pos can be mapped RW, can't add +X with mprotect. */
+	mmap_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
+			rb_fd, page_size);
+	ASSERT_OK_PTR(mmap_ptr, "rw_prod_pos");
+	ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_prod_pos_protect");
+	err = -errno;
+	ASSERT_ERR(err, "wr_prod_pos_err");
+	ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw_prod");
+
+	/* data pages can be mapped RW, can't add +X with mprotect. */
+	mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd,
+			2 * page_size);
+	ASSERT_OK_PTR(mmap_ptr, "rw_data");
+	ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_data_protect");
+	err = -errno;
+	ASSERT_ERR(err, "exec_data_err");
+	ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw_data");
+
+	user_ringbuf_success__destroy(skel);
+}
+
+static int load_skel_create_ringbufs(struct user_ringbuf_success **skel_out,
+				     struct ring_buffer **kern_ringbuf_out,
+				     ring_buffer_sample_fn callback,
+				     struct user_ring_buffer **user_ringbuf_out)
+{
+	struct user_ringbuf_success *skel;
+	struct ring_buffer *kern_ringbuf = NULL;
+	struct user_ring_buffer *user_ringbuf = NULL;
+	int err = -ENOMEM, rb_fd;
+
+	skel = open_load_ringbuf_skel();
+	if (!skel)
+		return err;
+
+	/* only trigger BPF program for current process */
+	skel->bss->pid = getpid();
+
+	if (kern_ringbuf_out) {
+		rb_fd = bpf_map__fd(skel->maps.kernel_ringbuf);
+		kern_ringbuf = ring_buffer__new(rb_fd, callback, skel, NULL);
+		if (!ASSERT_OK_PTR(kern_ringbuf, "kern_ringbuf_create"))
+			goto cleanup;
+
+		*kern_ringbuf_out = kern_ringbuf;
+	}
+
+	if (user_ringbuf_out) {
+		rb_fd = bpf_map__fd(skel->maps.user_ringbuf);
+		user_ringbuf = user_ring_buffer__new(rb_fd, NULL);
+		if (!ASSERT_OK_PTR(user_ringbuf, "user_ringbuf_create"))
+			goto cleanup;
+
+		*user_ringbuf_out = user_ringbuf;
+		ASSERT_EQ(skel->bss->read, 0, "no_reads_after_load");
+	}
+
+	err = user_ringbuf_success__attach(skel);
+	if (!ASSERT_OK(err, "skel_attach"))
+		goto cleanup;
+
+	*skel_out = skel;
+	return 0;
+
+cleanup:
+	if (kern_ringbuf_out)
+		*kern_ringbuf_out = NULL;
+	if (user_ringbuf_out)
+		*user_ringbuf_out = NULL;
+	ring_buffer__free(kern_ringbuf);
+	user_ring_buffer__free(user_ringbuf);
+	user_ringbuf_success__destroy(skel);
+	return err;
+}
+
+static int load_skel_create_user_ringbuf(struct user_ringbuf_success **skel_out,
+					 struct user_ring_buffer **ringbuf_out)
+{
+	return load_skel_create_ringbufs(skel_out, NULL, NULL, ringbuf_out);
+}
+
+static void manually_write_test_invalid_sample(struct user_ringbuf_success *skel,
+					       __u32 size, __u64 producer_pos, int err)
+{
+	void *data_ptr;
+	__u64 *producer_pos_ptr;
+	int rb_fd, page_size = getpagesize();
+
+	rb_fd = bpf_map__fd(skel->maps.user_ringbuf);
+
+	ASSERT_EQ(skel->bss->read, 0, "num_samples_before_bad_sample");
+
+	/* Map the producer_pos as RW. */
+	producer_pos_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE,
+				MAP_SHARED, rb_fd, page_size);
+	ASSERT_OK_PTR(producer_pos_ptr, "producer_pos_ptr");
+
+	/* Map the data pages as RW. */
+	data_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, 2 * page_size);
+	ASSERT_OK_PTR(data_ptr, "rw_data");
+
+	memset(data_ptr, 0, BPF_RINGBUF_HDR_SZ);
+	*(__u32 *)data_ptr = size;
+
+	/* Synchronizes with smp_load_acquire() in __bpf_user_ringbuf_peek() in the kernel. */
+	smp_store_release(producer_pos_ptr, producer_pos + BPF_RINGBUF_HDR_SZ);
+
+	drain_current_samples();
+	ASSERT_EQ(skel->bss->read, 0, "num_samples_after_bad_sample");
+	ASSERT_EQ(skel->bss->err, err, "err_after_bad_sample");
+
+	ASSERT_OK(munmap(producer_pos_ptr, page_size), "unmap_producer_pos");
+	ASSERT_OK(munmap(data_ptr, page_size), "unmap_data_ptr");
+}
+
+static void test_user_ringbuf_post_misaligned(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *ringbuf;
+	int err;
+	__u32 size = (1 << 5) + 7;
+
+	err = load_skel_create_user_ringbuf(&skel, &ringbuf);
+	if (!ASSERT_OK(err, "misaligned_skel"))
+		return;
+
+	manually_write_test_invalid_sample(skel, size, size, -EINVAL);
+	user_ring_buffer__free(ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static void test_user_ringbuf_post_producer_wrong_offset(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *ringbuf;
+	int err;
+	__u32 size = (1 << 5);
+
+	err = load_skel_create_user_ringbuf(&skel, &ringbuf);
+	if (!ASSERT_OK(err, "wrong_offset_skel"))
+		return;
+
+	manually_write_test_invalid_sample(skel, size, size - 8, -EINVAL);
+	user_ring_buffer__free(ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static void test_user_ringbuf_post_larger_than_ringbuf_sz(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *ringbuf;
+	int err;
+	__u32 size = c_ringbuf_size;
+
+	err = load_skel_create_user_ringbuf(&skel, &ringbuf);
+	if (!ASSERT_OK(err, "huge_sample_skel"))
+		return;
+
+	manually_write_test_invalid_sample(skel, size, size, -E2BIG);
+	user_ring_buffer__free(ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static void test_user_ringbuf_basic(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *ringbuf;
+	int err;
+
+	err = load_skel_create_user_ringbuf(&skel, &ringbuf);
+	if (!ASSERT_OK(err, "ringbuf_basic_skel"))
+		return;
+
+	ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");
+
+	err = write_samples(ringbuf, 2);
+	if (!ASSERT_OK(err, "write_samples"))
+		goto cleanup;
+
+	ASSERT_EQ(skel->bss->read, 2, "num_samples_read_after");
+
+cleanup:
+	user_ring_buffer__free(ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static void test_user_ringbuf_sample_full_ring_buffer(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *ringbuf;
+	int err;
+	void *sample;
+
+	err = load_skel_create_user_ringbuf(&skel, &ringbuf);
+	if (!ASSERT_OK(err, "ringbuf_full_sample_skel"))
+		return;
+
+	sample = user_ring_buffer__reserve(ringbuf, c_ringbuf_size - BPF_RINGBUF_HDR_SZ);
+	if (!ASSERT_OK_PTR(sample, "full_sample"))
+		goto cleanup;
+
+	user_ring_buffer__submit(ringbuf, sample);
+	ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");
+	drain_current_samples();
+	ASSERT_EQ(skel->bss->read, 1, "num_samples_read_after");
+
+cleanup:
+	user_ring_buffer__free(ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static void test_user_ringbuf_post_alignment_autoadjust(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *ringbuf;
+	struct sample *sample;
+	int err;
+
+	err = load_skel_create_user_ringbuf(&skel, &ringbuf);
+	if (!ASSERT_OK(err, "ringbuf_align_autoadjust_skel"))
+		return;
+
+	/* libbpf should automatically round any sample up to an 8-byte alignment. */
+	sample = user_ring_buffer__reserve(ringbuf, sizeof(*sample) + 1);
+	ASSERT_OK_PTR(sample, "reserve_autoaligned");
+	user_ring_buffer__submit(ringbuf, sample);
+
+	ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");
+	drain_current_samples();
+	ASSERT_EQ(skel->bss->read, 1, "num_samples_read_after");
+
+	user_ring_buffer__free(ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static void test_user_ringbuf_overfill(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *ringbuf;
+	int err;
+
+	err = load_skel_create_user_ringbuf(&skel, &ringbuf);
+	if (err)
+		return;
+
+	err = write_samples(ringbuf, c_max_entries * 5);
+	ASSERT_ERR(err, "write_samples");
+	ASSERT_EQ(skel->bss->read, c_max_entries, "max_entries");
+
+	user_ring_buffer__free(ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static void test_user_ringbuf_discards_properly_ignored(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *ringbuf;
+	int err, num_discarded = 0;
+	__u64 *token;
+
+	err = load_skel_create_user_ringbuf(&skel, &ringbuf);
+	if (err)
+		return;
+
+	ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");
+
+	while (1) {
+		/* Write samples until the buffer is full. */
+		token = user_ring_buffer__reserve(ringbuf, sizeof(*token));
+		if (!token)
+			break;
+
+		user_ring_buffer__discard(ringbuf, token);
+		num_discarded++;
+	}
+
+	if (!ASSERT_GE(num_discarded, 0, "num_discarded"))
+		goto cleanup;
+
+	/* Should not read any samples, as they are all discarded. */
+	ASSERT_EQ(skel->bss->read, 0, "num_pre_kick");
+	drain_current_samples();
+	ASSERT_EQ(skel->bss->read, 0, "num_post_kick");
+
+	/* Now that the ring buffer has been drained, we should be able to
+	 * reserve another token.
+	 */
+	token = user_ring_buffer__reserve(ringbuf, sizeof(*token));
+
+	if (!ASSERT_OK_PTR(token, "new_token"))
+		goto cleanup;
+
+	user_ring_buffer__discard(ringbuf, token);
+cleanup:
+	user_ring_buffer__free(ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static void test_user_ringbuf_loop(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *ringbuf;
+	uint32_t total_samples = 8192;
+	uint32_t remaining_samples = total_samples;
+	int err;
+
+	BUILD_BUG_ON(total_samples <= c_max_entries);
+	err = load_skel_create_user_ringbuf(&skel, &ringbuf);
+	if (err)
+		return;
+
+	do  {
+		uint32_t curr_samples;
+
+		curr_samples = remaining_samples > c_max_entries
+			? c_max_entries : remaining_samples;
+		err = write_samples(ringbuf, curr_samples);
+		if (err != 0) {
+			/* Assert inside of if statement to avoid flooding logs
+			 * on the success path.
+			 */
+			ASSERT_OK(err, "write_samples");
+			goto cleanup;
+		}
+
+		remaining_samples -= curr_samples;
+		ASSERT_EQ(skel->bss->read, total_samples - remaining_samples,
+			  "current_batched_entries");
+	} while (remaining_samples > 0);
+	ASSERT_EQ(skel->bss->read, total_samples, "total_batched_entries");
+
+cleanup:
+	user_ring_buffer__free(ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static int send_test_message(struct user_ring_buffer *ringbuf,
+			     enum test_msg_op op, s64 operand_64,
+			     s32 operand_32)
+{
+	struct test_msg *msg;
+
+	msg = user_ring_buffer__reserve(ringbuf, sizeof(*msg));
+	if (!msg) {
+		/* Assert on the error path to avoid spamming logs with mostly
+		 * success messages.
+		 */
+		ASSERT_OK_PTR(msg, "reserve_msg");
+		return -ENOMEM;
+	}
+
+	msg->msg_op = op;
+
+	switch (op) {
+	case TEST_MSG_OP_INC64:
+	case TEST_MSG_OP_MUL64:
+		msg->operand_64 = operand_64;
+		break;
+	case TEST_MSG_OP_INC32:
+	case TEST_MSG_OP_MUL32:
+		msg->operand_32 = operand_32;
+		break;
+	default:
+		PRINT_FAIL("Invalid operand %d\n", op);
+		user_ring_buffer__discard(ringbuf, msg);
+		return -EINVAL;
+	}
+
+	user_ring_buffer__submit(ringbuf, msg);
+
+	return 0;
+}
+
+static void kick_kernel_read_messages(void)
+{
+	syscall(__NR_prctl);
+}
+
+static int handle_kernel_msg(void *ctx, void *data, size_t len)
+{
+	struct user_ringbuf_success *skel = ctx;
+	struct test_msg *msg = data;
+
+	switch (msg->msg_op) {
+	case TEST_MSG_OP_INC64:
+		skel->bss->user_mutated += msg->operand_64;
+		return 0;
+	case TEST_MSG_OP_INC32:
+		skel->bss->user_mutated += msg->operand_32;
+		return 0;
+	case TEST_MSG_OP_MUL64:
+		skel->bss->user_mutated *= msg->operand_64;
+		return 0;
+	case TEST_MSG_OP_MUL32:
+		skel->bss->user_mutated *= msg->operand_32;
+		return 0;
+	default:
+		fprintf(stderr, "Invalid operand %d\n", msg->msg_op);
+		return -EINVAL;
+	}
+}
+
+static void drain_kernel_messages_buffer(struct ring_buffer *kern_ringbuf,
+					 struct user_ringbuf_success *skel)
+{
+	int cnt;
+
+	cnt = ring_buffer__consume(kern_ringbuf);
+	ASSERT_EQ(cnt, 8, "consume_kern_ringbuf");
+	ASSERT_OK(skel->bss->err, "consume_kern_ringbuf_err");
+}
+
+static void test_user_ringbuf_msg_protocol(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *user_ringbuf;
+	struct ring_buffer *kern_ringbuf;
+	int err, i;
+	__u64 expected_kern = 0;
+
+	err = load_skel_create_ringbufs(&skel, &kern_ringbuf, handle_kernel_msg, &user_ringbuf);
+	if (!ASSERT_OK(err, "create_ringbufs"))
+		return;
+
+	for (i = 0; i < 64; i++) {
+		enum test_msg_op op = i % TEST_MSG_OP_NUM_OPS;
+		__u64 operand_64 = TEST_OP_64;
+		__u32 operand_32 = TEST_OP_32;
+
+		err = send_test_message(user_ringbuf, op, operand_64, operand_32);
+		if (err) {
+			/* Only assert on a failure to avoid spamming success logs. */
+			ASSERT_OK(err, "send_test_message");
+			goto cleanup;
+		}
+
+		switch (op) {
+		case TEST_MSG_OP_INC64:
+			expected_kern += operand_64;
+			break;
+		case TEST_MSG_OP_INC32:
+			expected_kern += operand_32;
+			break;
+		case TEST_MSG_OP_MUL64:
+			expected_kern *= operand_64;
+			break;
+		case TEST_MSG_OP_MUL32:
+			expected_kern *= operand_32;
+			break;
+		default:
+			PRINT_FAIL("Unexpected op %d\n", op);
+			goto cleanup;
+		}
+
+		if (i % 8 == 0) {
+			kick_kernel_read_messages();
+			ASSERT_EQ(skel->bss->kern_mutated, expected_kern, "expected_kern");
+			ASSERT_EQ(skel->bss->err, 0, "bpf_prog_err");
+			drain_kernel_messages_buffer(kern_ringbuf, skel);
+		}
+	}
+
+cleanup:
+	ring_buffer__free(kern_ringbuf);
+	user_ring_buffer__free(user_ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static void *kick_kernel_cb(void *arg)
+{
+	/* Kick the kernel, causing it to drain the ring buffer and then wake
+	 * up the test thread waiting on epoll.
+	 */
+	syscall(__NR_getrlimit);
+
+	return NULL;
+}
+
+static int spawn_kick_thread_for_poll(void)
+{
+	pthread_t thread;
+
+	return pthread_create(&thread, NULL, kick_kernel_cb, NULL);
+}
+
+static void test_user_ringbuf_blocking_reserve(void)
+{
+	struct user_ringbuf_success *skel;
+	struct user_ring_buffer *ringbuf;
+	int err, num_written = 0;
+	__u64 *token;
+
+	err = load_skel_create_user_ringbuf(&skel, &ringbuf);
+	if (err)
+		return;
+
+	ASSERT_EQ(skel->bss->read, 0, "num_samples_read_before");
+
+	while (1) {
+		/* Write samples until the buffer is full. */
+		token = user_ring_buffer__reserve(ringbuf, sizeof(*token));
+		if (!token)
+			break;
+
+		*token = 0xdeadbeef;
+
+		user_ring_buffer__submit(ringbuf, token);
+		num_written++;
+	}
+
+	if (!ASSERT_GE(num_written, 0, "num_written"))
+		goto cleanup;
+
+	/* Should not have read any samples until the kernel is kicked. */
+	ASSERT_EQ(skel->bss->read, 0, "num_pre_kick");
+
+	/* We correctly time out after 1 second, without a sample. */
+	token = user_ring_buffer__reserve_blocking(ringbuf, sizeof(*token), 1000);
+	if (!ASSERT_EQ(token, NULL, "pre_kick_timeout_token"))
+		goto cleanup;
+
+	err = spawn_kick_thread_for_poll();
+	if (!ASSERT_EQ(err, 0, "deferred_kick_thread\n"))
+		goto cleanup;
+
+	/* After spawning another thread that asychronously kicks the kernel to
+	 * drain the messages, we're able to block and successfully get a
+	 * sample once we receive an event notification.
+	 */
+	token = user_ring_buffer__reserve_blocking(ringbuf, sizeof(*token), 10000);
+
+	if (!ASSERT_OK_PTR(token, "block_token"))
+		goto cleanup;
+
+	ASSERT_GT(skel->bss->read, 0, "num_post_kill");
+	ASSERT_LE(skel->bss->read, num_written, "num_post_kill");
+	ASSERT_EQ(skel->bss->err, 0, "err_post_poll");
+	user_ring_buffer__discard(ringbuf, token);
+
+cleanup:
+	user_ring_buffer__free(ringbuf);
+	user_ringbuf_success__destroy(skel);
+}
+
+static struct {
+	const char *prog_name;
+	const char *expected_err_msg;
+} failure_tests[] = {
+	/* failure cases */
+	{"user_ringbuf_callback_bad_access1", "negative offset dynptr_ptr ptr"},
+	{"user_ringbuf_callback_bad_access2", "dereference of modified dynptr_ptr ptr"},
+	{"user_ringbuf_callback_write_forbidden", "invalid mem access 'dynptr_ptr'"},
+	{"user_ringbuf_callback_null_context_write", "invalid mem access 'scalar'"},
+	{"user_ringbuf_callback_null_context_read", "invalid mem access 'scalar'"},
+	{"user_ringbuf_callback_discard_dynptr", "arg 1 is an unacquired reference"},
+	{"user_ringbuf_callback_submit_dynptr", "arg 1 is an unacquired reference"},
+	{"user_ringbuf_callback_invalid_return", "At callback return the register R0 has value"},
+};
+
+#define SUCCESS_TEST(_func) { _func, #_func }
+
+static struct {
+	void (*test_callback)(void);
+	const char *test_name;
+} success_tests[] = {
+	SUCCESS_TEST(test_user_ringbuf_mappings),
+	SUCCESS_TEST(test_user_ringbuf_post_misaligned),
+	SUCCESS_TEST(test_user_ringbuf_post_producer_wrong_offset),
+	SUCCESS_TEST(test_user_ringbuf_post_larger_than_ringbuf_sz),
+	SUCCESS_TEST(test_user_ringbuf_basic),
+	SUCCESS_TEST(test_user_ringbuf_sample_full_ring_buffer),
+	SUCCESS_TEST(test_user_ringbuf_post_alignment_autoadjust),
+	SUCCESS_TEST(test_user_ringbuf_overfill),
+	SUCCESS_TEST(test_user_ringbuf_discards_properly_ignored),
+	SUCCESS_TEST(test_user_ringbuf_loop),
+	SUCCESS_TEST(test_user_ringbuf_msg_protocol),
+	SUCCESS_TEST(test_user_ringbuf_blocking_reserve),
+};
+
+static void verify_fail(const char *prog_name, const char *expected_err_msg)
+{
+	LIBBPF_OPTS(bpf_object_open_opts, opts);
+	struct bpf_program *prog;
+	struct user_ringbuf_fail *skel;
+	int err;
+
+	opts.kernel_log_buf = obj_log_buf;
+	opts.kernel_log_size = log_buf_sz;
+	opts.kernel_log_level = 1;
+
+	skel = user_ringbuf_fail__open_opts(&opts);
+	if (!ASSERT_OK_PTR(skel, "dynptr_fail__open_opts"))
+		goto cleanup;
+
+	prog = bpf_object__find_program_by_name(skel->obj, prog_name);
+	if (!ASSERT_OK_PTR(prog, "bpf_object__find_program_by_name"))
+		goto cleanup;
+
+	bpf_program__set_autoload(prog, true);
+
+	bpf_map__set_max_entries(skel->maps.user_ringbuf, getpagesize());
+
+	err = user_ringbuf_fail__load(skel);
+	if (!ASSERT_ERR(err, "unexpected load success"))
+		goto cleanup;
+
+	if (!ASSERT_OK_PTR(strstr(obj_log_buf, expected_err_msg), "expected_err_msg")) {
+		fprintf(stderr, "Expected err_msg: %s\n", expected_err_msg);
+		fprintf(stderr, "Verifier output: %s\n", obj_log_buf);
+	}
+
+cleanup:
+	user_ringbuf_fail__destroy(skel);
+}
+
+void test_user_ringbuf(void)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(success_tests); i++) {
+		if (!test__start_subtest(success_tests[i].test_name))
+			continue;
+
+		success_tests[i].test_callback();
+	}
+
+	for (i = 0; i < ARRAY_SIZE(failure_tests); i++) {
+		if (!test__start_subtest(failure_tests[i].prog_name))
+			continue;
+
+		verify_fail(failure_tests[i].prog_name, failure_tests[i].expected_err_msg);
+	}
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c
new file mode 100644
index 000000000000..579d6ee83ce0
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/verify_pkcs7_sig.c
@@ -0,0 +1,399 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH
+ *
+ * Author: Roberto Sassu <roberto.sassu@huawei.com>
+ */
+
+#include <stdio.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <endian.h>
+#include <limits.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <sys/mman.h>
+#include <linux/keyctl.h>
+#include <test_progs.h>
+
+#include "test_verify_pkcs7_sig.skel.h"
+
+#define MAX_DATA_SIZE (1024 * 1024)
+#define MAX_SIG_SIZE 1024
+
+#define VERIFY_USE_SECONDARY_KEYRING (1UL)
+#define VERIFY_USE_PLATFORM_KEYRING  (2UL)
+
+/* In stripped ARM and x86-64 modules, ~ is surprisingly rare. */
+#define MODULE_SIG_STRING "~Module signature appended~\n"
+
+/*
+ * Module signature information block.
+ *
+ * The constituents of the signature section are, in order:
+ *
+ *	- Signer's name
+ *	- Key identifier
+ *	- Signature data
+ *	- Information block
+ */
+struct module_signature {
+	__u8	algo;		/* Public-key crypto algorithm [0] */
+	__u8	hash;		/* Digest algorithm [0] */
+	__u8	id_type;	/* Key identifier type [PKEY_ID_PKCS7] */
+	__u8	signer_len;	/* Length of signer's name [0] */
+	__u8	key_id_len;	/* Length of key identifier [0] */
+	__u8	__pad[3];
+	__be32	sig_len;	/* Length of signature data */
+};
+
+struct data {
+	__u8 data[MAX_DATA_SIZE];
+	__u32 data_len;
+	__u8 sig[MAX_SIG_SIZE];
+	__u32 sig_len;
+};
+
+static bool kfunc_not_supported;
+
+static int libbpf_print_cb(enum libbpf_print_level level, const char *fmt,
+			   va_list args)
+{
+	if (strcmp(fmt, "libbpf: extern (func ksym) '%s': not found in kernel or module BTFs\n"))
+		return 0;
+
+	if (strcmp(va_arg(args, char *), "bpf_verify_pkcs7_signature"))
+		return 0;
+
+	kfunc_not_supported = true;
+	return 0;
+}
+
+static int _run_setup_process(const char *setup_dir, const char *cmd)
+{
+	int child_pid, child_status;
+
+	child_pid = fork();
+	if (child_pid == 0) {
+		execlp("./verify_sig_setup.sh", "./verify_sig_setup.sh", cmd,
+		       setup_dir, NULL);
+		exit(errno);
+
+	} else if (child_pid > 0) {
+		waitpid(child_pid, &child_status, 0);
+		return WEXITSTATUS(child_status);
+	}
+
+	return -EINVAL;
+}
+
+static int populate_data_item_str(const char *tmp_dir, struct data *data_item)
+{
+	struct stat st;
+	char data_template[] = "/tmp/dataXXXXXX";
+	char path[PATH_MAX];
+	int ret, fd, child_status, child_pid;
+
+	data_item->data_len = 4;
+	memcpy(data_item->data, "test", data_item->data_len);
+
+	fd = mkstemp(data_template);
+	if (fd == -1)
+		return -errno;
+
+	ret = write(fd, data_item->data, data_item->data_len);
+
+	close(fd);
+
+	if (ret != data_item->data_len) {
+		ret = -EIO;
+		goto out;
+	}
+
+	child_pid = fork();
+
+	if (child_pid == -1) {
+		ret = -errno;
+		goto out;
+	}
+
+	if (child_pid == 0) {
+		snprintf(path, sizeof(path), "%s/signing_key.pem", tmp_dir);
+
+		return execlp("./sign-file", "./sign-file", "-d", "sha256",
+			      path, path, data_template, NULL);
+	}
+
+	waitpid(child_pid, &child_status, 0);
+
+	ret = WEXITSTATUS(child_status);
+	if (ret)
+		goto out;
+
+	snprintf(path, sizeof(path), "%s.p7s", data_template);
+
+	ret = stat(path, &st);
+	if (ret == -1) {
+		ret = -errno;
+		goto out;
+	}
+
+	if (st.st_size > sizeof(data_item->sig)) {
+		ret = -EINVAL;
+		goto out_sig;
+	}
+
+	data_item->sig_len = st.st_size;
+
+	fd = open(path, O_RDONLY);
+	if (fd == -1) {
+		ret = -errno;
+		goto out_sig;
+	}
+
+	ret = read(fd, data_item->sig, data_item->sig_len);
+
+	close(fd);
+
+	if (ret != data_item->sig_len) {
+		ret = -EIO;
+		goto out_sig;
+	}
+
+	ret = 0;
+out_sig:
+	unlink(path);
+out:
+	unlink(data_template);
+	return ret;
+}
+
+static int populate_data_item_mod(struct data *data_item)
+{
+	char mod_path[PATH_MAX], *mod_path_ptr;
+	struct stat st;
+	void *mod;
+	FILE *fp;
+	struct module_signature ms;
+	int ret, fd, modlen, marker_len, sig_len;
+
+	data_item->data_len = 0;
+
+	if (stat("/lib/modules", &st) == -1)
+		return 0;
+
+	/* Requires CONFIG_TCP_CONG_BIC=m. */
+	fp = popen("find /lib/modules/$(uname -r) -name tcp_bic.ko", "r");
+	if (!fp)
+		return 0;
+
+	mod_path_ptr = fgets(mod_path, sizeof(mod_path), fp);
+	pclose(fp);
+
+	if (!mod_path_ptr)
+		return 0;
+
+	mod_path_ptr = strchr(mod_path, '\n');
+	if (!mod_path_ptr)
+		return 0;
+
+	*mod_path_ptr = '\0';
+
+	if (stat(mod_path, &st) == -1)
+		return 0;
+
+	modlen = st.st_size;
+	marker_len = sizeof(MODULE_SIG_STRING) - 1;
+
+	fd = open(mod_path, O_RDONLY);
+	if (fd == -1)
+		return -errno;
+
+	mod = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+
+	close(fd);
+
+	if (mod == MAP_FAILED)
+		return -errno;
+
+	if (strncmp(mod + modlen - marker_len, MODULE_SIG_STRING, marker_len)) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	modlen -= marker_len;
+
+	memcpy(&ms, mod + (modlen - sizeof(ms)), sizeof(ms));
+
+	sig_len = __be32_to_cpu(ms.sig_len);
+	modlen -= sig_len + sizeof(ms);
+
+	if (modlen > sizeof(data_item->data)) {
+		ret = -E2BIG;
+		goto out;
+	}
+
+	memcpy(data_item->data, mod, modlen);
+	data_item->data_len = modlen;
+
+	if (sig_len > sizeof(data_item->sig)) {
+		ret = -E2BIG;
+		goto out;
+	}
+
+	memcpy(data_item->sig, mod + modlen, sig_len);
+	data_item->sig_len = sig_len;
+	ret = 0;
+out:
+	munmap(mod, st.st_size);
+	return ret;
+}
+
+void test_verify_pkcs7_sig(void)
+{
+	libbpf_print_fn_t old_print_cb;
+	char tmp_dir_template[] = "/tmp/verify_sigXXXXXX";
+	char *tmp_dir;
+	struct test_verify_pkcs7_sig *skel = NULL;
+	struct bpf_map *map;
+	struct data data;
+	int ret, zero = 0;
+
+	/* Trigger creation of session keyring. */
+	syscall(__NR_request_key, "keyring", "_uid.0", NULL,
+		KEY_SPEC_SESSION_KEYRING);
+
+	tmp_dir = mkdtemp(tmp_dir_template);
+	if (!ASSERT_OK_PTR(tmp_dir, "mkdtemp"))
+		return;
+
+	ret = _run_setup_process(tmp_dir, "setup");
+	if (!ASSERT_OK(ret, "_run_setup_process"))
+		goto close_prog;
+
+	skel = test_verify_pkcs7_sig__open();
+	if (!ASSERT_OK_PTR(skel, "test_verify_pkcs7_sig__open"))
+		goto close_prog;
+
+	old_print_cb = libbpf_set_print(libbpf_print_cb);
+	ret = test_verify_pkcs7_sig__load(skel);
+	libbpf_set_print(old_print_cb);
+
+	if (ret < 0 && kfunc_not_supported) {
+		printf(
+		  "%s:SKIP:bpf_verify_pkcs7_signature() kfunc not supported\n",
+		  __func__);
+		test__skip();
+		goto close_prog;
+	}
+
+	if (!ASSERT_OK(ret, "test_verify_pkcs7_sig__load"))
+		goto close_prog;
+
+	ret = test_verify_pkcs7_sig__attach(skel);
+	if (!ASSERT_OK(ret, "test_verify_pkcs7_sig__attach"))
+		goto close_prog;
+
+	map = bpf_object__find_map_by_name(skel->obj, "data_input");
+	if (!ASSERT_OK_PTR(map, "data_input not found"))
+		goto close_prog;
+
+	skel->bss->monitored_pid = getpid();
+
+	/* Test without data and signature. */
+	skel->bss->user_keyring_serial = KEY_SPEC_SESSION_KEYRING;
+
+	ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY);
+	if (!ASSERT_LT(ret, 0, "bpf_map_update_elem data_input"))
+		goto close_prog;
+
+	/* Test successful signature verification with session keyring. */
+	ret = populate_data_item_str(tmp_dir, &data);
+	if (!ASSERT_OK(ret, "populate_data_item_str"))
+		goto close_prog;
+
+	ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY);
+	if (!ASSERT_OK(ret, "bpf_map_update_elem data_input"))
+		goto close_prog;
+
+	/* Test successful signature verification with testing keyring. */
+	skel->bss->user_keyring_serial = syscall(__NR_request_key, "keyring",
+						 "ebpf_testing_keyring", NULL,
+						 KEY_SPEC_SESSION_KEYRING);
+
+	ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY);
+	if (!ASSERT_OK(ret, "bpf_map_update_elem data_input"))
+		goto close_prog;
+
+	/*
+	 * Ensure key_task_permission() is called and rejects the keyring
+	 * (no Search permission).
+	 */
+	syscall(__NR_keyctl, KEYCTL_SETPERM, skel->bss->user_keyring_serial,
+		0x37373737);
+
+	ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY);
+	if (!ASSERT_LT(ret, 0, "bpf_map_update_elem data_input"))
+		goto close_prog;
+
+	syscall(__NR_keyctl, KEYCTL_SETPERM, skel->bss->user_keyring_serial,
+		0x3f3f3f3f);
+
+	/*
+	 * Ensure key_validate() is called and rejects the keyring (key expired)
+	 */
+	syscall(__NR_keyctl, KEYCTL_SET_TIMEOUT,
+		skel->bss->user_keyring_serial, 1);
+	sleep(1);
+
+	ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY);
+	if (!ASSERT_LT(ret, 0, "bpf_map_update_elem data_input"))
+		goto close_prog;
+
+	skel->bss->user_keyring_serial = KEY_SPEC_SESSION_KEYRING;
+
+	/* Test with corrupted data (signature verification should fail). */
+	data.data[0] = 'a';
+	ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data, BPF_ANY);
+	if (!ASSERT_LT(ret, 0, "bpf_map_update_elem data_input"))
+		goto close_prog;
+
+	ret = populate_data_item_mod(&data);
+	if (!ASSERT_OK(ret, "populate_data_item_mod"))
+		goto close_prog;
+
+	/* Test signature verification with system keyrings. */
+	if (data.data_len) {
+		skel->bss->user_keyring_serial = 0;
+		skel->bss->system_keyring_id = 0;
+
+		ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data,
+					  BPF_ANY);
+		if (!ASSERT_OK(ret, "bpf_map_update_elem data_input"))
+			goto close_prog;
+
+		skel->bss->system_keyring_id = VERIFY_USE_SECONDARY_KEYRING;
+
+		ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data,
+					  BPF_ANY);
+		if (!ASSERT_OK(ret, "bpf_map_update_elem data_input"))
+			goto close_prog;
+
+		skel->bss->system_keyring_id = VERIFY_USE_PLATFORM_KEYRING;
+
+		ret = bpf_map_update_elem(bpf_map__fd(map), &zero, &data,
+					  BPF_ANY);
+		ASSERT_LT(ret, 0, "bpf_map_update_elem data_input");
+	}
+
+close_prog:
+	_run_setup_process(tmp_dir, "cleanup");
+
+	if (!skel)
+		return;
+
+	skel->bss->monitored_pid = 0;
+	test_verify_pkcs7_sig__destroy(skel);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp.c b/tools/testing/selftests/bpf/prog_tests/xdp.c
index ec21c53cb1da..947863a1d536 100644
--- a/tools/testing/selftests/bpf/prog_tests/xdp.c
+++ b/tools/testing/selftests/bpf/prog_tests/xdp.c
@@ -8,7 +8,7 @@ void test_xdp(void)
 	struct vip key6 = {.protocol = 6, .family = AF_INET6};
 	struct iptnl_info value4 = {.family = AF_INET};
 	struct iptnl_info value6 = {.family = AF_INET6};
-	const char *file = "./test_xdp.o";
+	const char *file = "./test_xdp.bpf.o";
 	struct bpf_object *obj;
 	char buf[128];
 	struct ipv6hdr iph6;
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c
index 2f033da4cd45..fce203640f8c 100644
--- a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c
+++ b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_frags.c
@@ -4,7 +4,7 @@
 
 static void test_xdp_update_frags(void)
 {
-	const char *file = "./test_xdp_update_frags.o";
+	const char *file = "./test_xdp_update_frags.bpf.o";
 	int err, prog_fd, max_skb_frags, buf_size, num;
 	struct bpf_program *prog;
 	struct bpf_object *obj;
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c
index 21ceac24e174..9b9cf8458adf 100644
--- a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c
+++ b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c
@@ -4,7 +4,7 @@
 
 static void test_xdp_adjust_tail_shrink(void)
 {
-	const char *file = "./test_xdp_adjust_tail_shrink.o";
+	const char *file = "./test_xdp_adjust_tail_shrink.bpf.o";
 	__u32 expect_sz;
 	struct bpf_object *obj;
 	int err, prog_fd;
@@ -39,7 +39,7 @@ static void test_xdp_adjust_tail_shrink(void)
 
 static void test_xdp_adjust_tail_grow(void)
 {
-	const char *file = "./test_xdp_adjust_tail_grow.o";
+	const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
 	struct bpf_object *obj;
 	char buf[4096]; /* avoid segfault: large buf to hold grow results */
 	__u32 expect_sz;
@@ -73,7 +73,7 @@ static void test_xdp_adjust_tail_grow(void)
 
 static void test_xdp_adjust_tail_grow2(void)
 {
-	const char *file = "./test_xdp_adjust_tail_grow.o";
+	const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
 	char buf[4096]; /* avoid segfault: large buf to hold grow results */
 	int tailroom = 320; /* SKB_DATA_ALIGN(sizeof(struct skb_shared_info))*/;
 	struct bpf_object *obj;
@@ -135,7 +135,7 @@ static void test_xdp_adjust_tail_grow2(void)
 
 static void test_xdp_adjust_frags_tail_shrink(void)
 {
-	const char *file = "./test_xdp_adjust_tail_shrink.o";
+	const char *file = "./test_xdp_adjust_tail_shrink.bpf.o";
 	__u32 exp_size;
 	struct bpf_program *prog;
 	struct bpf_object *obj;
@@ -202,7 +202,7 @@ out:
 
 static void test_xdp_adjust_frags_tail_grow(void)
 {
-	const char *file = "./test_xdp_adjust_tail_grow.o";
+	const char *file = "./test_xdp_adjust_tail_grow.bpf.o";
 	__u32 exp_size;
 	struct bpf_program *prog;
 	struct bpf_object *obj;
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_attach.c b/tools/testing/selftests/bpf/prog_tests/xdp_attach.c
index 62aa3edda5e6..062fbc8c8e5e 100644
--- a/tools/testing/selftests/bpf/prog_tests/xdp_attach.c
+++ b/tools/testing/selftests/bpf/prog_tests/xdp_attach.c
@@ -8,7 +8,7 @@ void serial_test_xdp_attach(void)
 {
 	__u32 duration = 0, id1, id2, id0 = 0, len;
 	struct bpf_object *obj1, *obj2, *obj3;
-	const char *file = "./test_xdp.o";
+	const char *file = "./test_xdp.bpf.o";
 	struct bpf_prog_info info = {};
 	int err, fd1, fd2, fd3;
 	LIBBPF_OPTS(bpf_xdp_attach_opts, opts);
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_info.c b/tools/testing/selftests/bpf/prog_tests/xdp_info.c
index 0d01ff6cb91a..cd3aa340e65e 100644
--- a/tools/testing/selftests/bpf/prog_tests/xdp_info.c
+++ b/tools/testing/selftests/bpf/prog_tests/xdp_info.c
@@ -7,7 +7,7 @@
 void serial_test_xdp_info(void)
 {
 	__u32 len = sizeof(struct bpf_prog_info), duration = 0, prog_id;
-	const char *file = "./xdp_dummy.o";
+	const char *file = "./xdp_dummy.bpf.o";
 	struct bpf_prog_info info = {};
 	struct bpf_object *obj;
 	int err, prog_fd;
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_perf.c b/tools/testing/selftests/bpf/prog_tests/xdp_perf.c
index f543d1bd21b8..ec5369f247cb 100644
--- a/tools/testing/selftests/bpf/prog_tests/xdp_perf.c
+++ b/tools/testing/selftests/bpf/prog_tests/xdp_perf.c
@@ -3,7 +3,7 @@
 
 void test_xdp_perf(void)
 {
-	const char *file = "./xdp_dummy.o";
+	const char *file = "./xdp_dummy.bpf.o";
 	struct bpf_object *obj;
 	char in[128], out[128];
 	int err, prog_fd;
diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c b/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c
index 874a846e298c..75550a40e029 100644
--- a/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c
+++ b/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c
@@ -82,7 +82,7 @@ static void test_synproxy(bool xdp)
 	SYS("ethtool -K tmp0 tx off");
 	if (xdp)
 		/* Workaround required for veth. */
-		SYS("ip link set tmp0 xdp object xdp_dummy.o section xdp 2> /dev/null");
+		SYS("ip link set tmp0 xdp object xdp_dummy.bpf.o section xdp 2> /dev/null");
 
 	ns = open_netns("synproxy");
 	if (!ASSERT_OK_PTR(ns, "setns"))
diff --git a/tools/testing/selftests/bpf/progs/bind4_prog.c b/tools/testing/selftests/bpf/progs/bind4_prog.c
index 474c6a62078a..a487f60b73ac 100644
--- a/tools/testing/selftests/bpf/progs/bind4_prog.c
+++ b/tools/testing/selftests/bpf/progs/bind4_prog.c
@@ -6,8 +6,6 @@
 #include <linux/bpf.h>
 #include <linux/in.h>
 #include <linux/in6.h>
-#include <sys/socket.h>
-#include <netinet/tcp.h>
 #include <linux/if.h>
 #include <errno.h>
 
diff --git a/tools/testing/selftests/bpf/progs/bind6_prog.c b/tools/testing/selftests/bpf/progs/bind6_prog.c
index c19cfa869f30..d62cd9e9cf0e 100644
--- a/tools/testing/selftests/bpf/progs/bind6_prog.c
+++ b/tools/testing/selftests/bpf/progs/bind6_prog.c
@@ -6,8 +6,6 @@
 #include <linux/bpf.h>
 #include <linux/in.h>
 #include <linux/in6.h>
-#include <sys/socket.h>
-#include <netinet/tcp.h>
 #include <linux/if.h>
 #include <errno.h>
 
diff --git a/tools/testing/selftests/bpf/progs/bpf_dctcp.c b/tools/testing/selftests/bpf/progs/bpf_dctcp.c
index 9573be6122be..460682759aed 100644
--- a/tools/testing/selftests/bpf/progs/bpf_dctcp.c
+++ b/tools/testing/selftests/bpf/progs/bpf_dctcp.c
@@ -11,6 +11,7 @@
 #include <linux/types.h>
 #include <linux/stddef.h>
 #include <linux/tcp.h>
+#include <errno.h>
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_tracing.h>
 #include "bpf_tcp_helpers.h"
@@ -23,6 +24,7 @@ const char tcp_cdg[] = "cdg";
 char cc_res[TCP_CA_NAME_MAX];
 int tcp_cdg_res = 0;
 int stg_result = 0;
+int ebusy_cnt = 0;
 
 struct {
 	__uint(type, BPF_MAP_TYPE_SK_STORAGE);
@@ -64,16 +66,23 @@ void BPF_PROG(dctcp_init, struct sock *sk)
 
 	if (!(tp->ecn_flags & TCP_ECN_OK) && fallback[0]) {
 		/* Switch to fallback */
-		bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
-			       (void *)fallback, sizeof(fallback));
-		/* Switch back to myself which the bpf trampoline
-		 * stopped calling dctcp_init recursively.
+		if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
+				   (void *)fallback, sizeof(fallback)) == -EBUSY)
+			ebusy_cnt++;
+
+		/* Switch back to myself and the recurred dctcp_init()
+		 * will get -EBUSY for all bpf_setsockopt(TCP_CONGESTION),
+		 * except the last "cdg" one.
 		 */
-		bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
-			       (void *)bpf_dctcp, sizeof(bpf_dctcp));
+		if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
+				   (void *)bpf_dctcp, sizeof(bpf_dctcp)) == -EBUSY)
+			ebusy_cnt++;
+
 		/* Switch back to fallback */
-		bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
-			       (void *)fallback, sizeof(fallback));
+		if (bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
+				   (void *)fallback, sizeof(fallback)) == -EBUSY)
+			ebusy_cnt++;
+
 		/* Expecting -ENOTSUPP for tcp_cdg_res */
 		tcp_cdg_res = bpf_setsockopt(sk, SOL_TCP, TCP_CONGESTION,
 					     (void *)tcp_cdg, sizeof(tcp_cdg));
diff --git a/tools/testing/selftests/bpf/progs/bpf_flow.c b/tools/testing/selftests/bpf/progs/bpf_flow.c
index f266c757b3df..a20c5ed5e454 100644
--- a/tools/testing/selftests/bpf/progs/bpf_flow.c
+++ b/tools/testing/selftests/bpf/progs/bpf_flow.c
@@ -22,6 +22,8 @@
 #define PROG(F) PROG_(F, _##F)
 #define PROG_(NUM, NAME) SEC("flow_dissector") int flow_dissector_##NUM
 
+#define FLOW_CONTINUE_SADDR 0x7f00007f /* 127.0.0.127 */
+
 /* These are the identifiers of the BPF programs that will be used in tail
  * calls. Name is limited to 16 characters, with the terminating character and
  * bpf_func_ above, we have only 6 to work with, anything after will be cropped.
@@ -143,6 +145,19 @@ int _dissect(struct __sk_buff *skb)
 {
 	struct bpf_flow_keys *keys = skb->flow_keys;
 
+	if (keys->n_proto == bpf_htons(ETH_P_IP)) {
+		/* IP traffic from FLOW_CONTINUE_SADDR falls-back to
+		 * standard dissector
+		 */
+		struct iphdr *iph, _iph;
+
+		iph = bpf_flow_dissect_get_header(skb, sizeof(*iph), &_iph);
+		if (iph && iph->ihl == 5 &&
+		    iph->saddr == bpf_htonl(FLOW_CONTINUE_SADDR)) {
+			return BPF_FLOW_DISSECTOR_CONTINUE;
+		}
+	}
+
 	return parse_eth_proto(skb, keys->n_proto);
 }
 
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter.h b/tools/testing/selftests/bpf/progs/bpf_iter.h
index e9846606690d..c41ee80533ca 100644
--- a/tools/testing/selftests/bpf/progs/bpf_iter.h
+++ b/tools/testing/selftests/bpf/progs/bpf_iter.h
@@ -17,6 +17,7 @@
 #define bpf_iter__bpf_sk_storage_map bpf_iter__bpf_sk_storage_map___not_used
 #define bpf_iter__sockmap bpf_iter__sockmap___not_used
 #define bpf_iter__bpf_link bpf_iter__bpf_link___not_used
+#define bpf_iter__cgroup bpf_iter__cgroup___not_used
 #define btf_ptr btf_ptr___not_used
 #define BTF_F_COMPACT BTF_F_COMPACT___not_used
 #define BTF_F_NONAME BTF_F_NONAME___not_used
@@ -40,6 +41,7 @@
 #undef bpf_iter__bpf_sk_storage_map
 #undef bpf_iter__sockmap
 #undef bpf_iter__bpf_link
+#undef bpf_iter__cgroup
 #undef btf_ptr
 #undef BTF_F_COMPACT
 #undef BTF_F_NONAME
@@ -141,6 +143,11 @@ struct bpf_iter__bpf_link {
 	struct bpf_link *link;
 };
 
+struct bpf_iter__cgroup {
+	struct bpf_iter_meta *meta;
+	struct cgroup *cgroup;
+} __attribute__((preserve_access_index));
+
 struct btf_ptr {
 	void *ptr;
 	__u32 type_id;
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_task.c b/tools/testing/selftests/bpf/progs/bpf_iter_task.c
index d22741272692..96131b9a1caa 100644
--- a/tools/testing/selftests/bpf/progs/bpf_iter_task.c
+++ b/tools/testing/selftests/bpf/progs/bpf_iter_task.c
@@ -6,6 +6,10 @@
 
 char _license[] SEC("license") = "GPL";
 
+uint32_t tid = 0;
+int num_unknown_tid = 0;
+int num_known_tid = 0;
+
 SEC("iter/task")
 int dump_task(struct bpf_iter__task *ctx)
 {
@@ -18,6 +22,11 @@ int dump_task(struct bpf_iter__task *ctx)
 		return 0;
 	}
 
+	if (task->pid != tid)
+		num_unknown_tid++;
+	else
+		num_known_tid++;
+
 	if (ctx->meta->seq_num == 0)
 		BPF_SEQ_PRINTF(seq, "    tgid      gid\n");
 
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c b/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c
index 6e7b400888fe..b0255080662d 100644
--- a/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c
+++ b/tools/testing/selftests/bpf/progs/bpf_iter_task_file.c
@@ -7,14 +7,16 @@ char _license[] SEC("license") = "GPL";
 
 int count = 0;
 int tgid = 0;
+int last_tgid = 0;
+int unique_tgid_count = 0;
 
 SEC("iter/task_file")
 int dump_task_file(struct bpf_iter__task_file *ctx)
 {
 	struct seq_file *seq = ctx->meta->seq;
 	struct task_struct *task = ctx->task;
-	__u32 fd = ctx->fd;
 	struct file *file = ctx->file;
+	__u32 fd = ctx->fd;
 
 	if (task == (void *)0 || file == (void *)0)
 		return 0;
@@ -27,6 +29,11 @@ int dump_task_file(struct bpf_iter__task_file *ctx)
 	if (tgid == task->tgid && task->tgid != task->pid)
 		count++;
 
+	if (last_tgid != task->tgid) {
+		last_tgid = task->tgid;
+		unique_tgid_count++;
+	}
+
 	BPF_SEQ_PRINTF(seq, "%8d %8d %8d %lx\n", task->tgid, task->pid, fd,
 		       (long)file->f_op);
 	return 0;
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c b/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c
index 4ea6a37d1345..dd923dc637d5 100644
--- a/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c
+++ b/tools/testing/selftests/bpf/progs/bpf_iter_task_vma.c
@@ -20,6 +20,8 @@ char _license[] SEC("license") = "GPL";
 #define D_PATH_BUF_SIZE 1024
 char d_path_buf[D_PATH_BUF_SIZE] = {};
 __u32 pid = 0;
+__u32 one_task = 0;
+__u32 one_task_error = 0;
 
 SEC("iter/task_vma") int proc_maps(struct bpf_iter__task_vma *ctx)
 {
@@ -33,8 +35,11 @@ SEC("iter/task_vma") int proc_maps(struct bpf_iter__task_vma *ctx)
 		return 0;
 
 	file = vma->vm_file;
-	if (task->tgid != pid)
+	if (task->tgid != pid) {
+		if (one_task)
+			one_task_error = 1;
 		return 0;
+	}
 	perm_str[0] = (vma->vm_flags & VM_READ) ? 'r' : '-';
 	perm_str[1] = (vma->vm_flags & VM_WRITE) ? 'w' : '-';
 	perm_str[2] = (vma->vm_flags & VM_EXEC) ? 'x' : '-';
diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c b/tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c
new file mode 100644
index 000000000000..ee7455d2623a
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/bpf_iter_vma_offset.c
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
+#include "bpf_iter.h"
+#include <bpf/bpf_helpers.h>
+
+char _license[] SEC("license") = "GPL";
+
+__u32 unique_tgid_cnt = 0;
+uintptr_t address = 0;
+uintptr_t offset = 0;
+__u32 last_tgid = 0;
+__u32 pid = 0;
+__u32 page_shift = 0;
+
+SEC("iter/task_vma")
+int get_vma_offset(struct bpf_iter__task_vma *ctx)
+{
+	struct vm_area_struct *vma = ctx->vma;
+	struct seq_file *seq = ctx->meta->seq;
+	struct task_struct *task = ctx->task;
+
+	if (task == NULL || vma == NULL)
+		return 0;
+
+	if (last_tgid != task->tgid)
+		unique_tgid_cnt++;
+	last_tgid = task->tgid;
+
+	if (task->tgid != pid)
+		return 0;
+
+	if (vma->vm_start <= address && vma->vm_end > address) {
+		offset = address - vma->vm_start + (vma->vm_pgoff << page_shift);
+		BPF_SEQ_PRINTF(seq, "OK\n");
+	}
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h
index 98dd2c4815f0..adb087aecc9e 100644
--- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h
+++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h
@@ -6,13 +6,41 @@
 #define AF_INET6		10
 
 #define SOL_SOCKET		1
+#define SO_REUSEADDR		2
 #define SO_SNDBUF		7
-#define __SO_ACCEPTCON		(1 << 16)
+#define SO_RCVBUF		8
+#define SO_KEEPALIVE		9
 #define SO_PRIORITY		12
+#define SO_REUSEPORT		15
+#define SO_RCVLOWAT		18
+#define SO_BINDTODEVICE		25
+#define SO_MARK			36
+#define SO_MAX_PACING_RATE	47
+#define SO_BINDTOIFINDEX	62
+#define SO_TXREHASH		74
+#define __SO_ACCEPTCON		(1 << 16)
+
+#define IP_TOS			1
+
+#define IPV6_TCLASS		67
+#define IPV6_AUTOFLOWLABEL	70
 
 #define SOL_TCP			6
+#define TCP_NODELAY		1
+#define TCP_MAXSEG		2
+#define TCP_KEEPIDLE		4
+#define TCP_KEEPINTVL		5
+#define TCP_KEEPCNT		6
+#define TCP_SYNCNT		7
+#define TCP_WINDOW_CLAMP	10
 #define TCP_CONGESTION		13
+#define TCP_THIN_LINEAR_TIMEOUTS	16
+#define TCP_USER_TIMEOUT	18
+#define TCP_NOTSENT_LOWAT	25
+#define TCP_SAVE_SYN		27
+#define TCP_SAVED_SYN		28
 #define TCP_CA_NAME_MAX		16
+#define TCP_NAGLE_OFF		1
 
 #define ICSK_TIME_RETRANS	1
 #define ICSK_TIME_PROBE0	3
@@ -49,6 +77,8 @@
 #define sk_state		__sk_common.skc_state
 #define sk_v6_daddr		__sk_common.skc_v6_daddr
 #define sk_v6_rcv_saddr		__sk_common.skc_v6_rcv_saddr
+#define sk_flags		__sk_common.skc_flags
+#define sk_reuse		__sk_common.skc_reuse
 
 #define s6_addr32		in6_u.u6_addr32
 
diff --git a/tools/testing/selftests/bpf/progs/cb_refs.c b/tools/testing/selftests/bpf/progs/cb_refs.c
new file mode 100644
index 000000000000..7653df1bc787
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/cb_refs.c
@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <vmlinux.h>
+#include <bpf/bpf_tracing.h>
+#include <bpf/bpf_helpers.h>
+
+struct map_value {
+	struct prog_test_ref_kfunc __kptr_ref *ptr;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__type(key, int);
+	__type(value, struct map_value);
+	__uint(max_entries, 16);
+} array_map SEC(".maps");
+
+extern struct prog_test_ref_kfunc *bpf_kfunc_call_test_acquire(unsigned long *sp) __ksym;
+extern void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) __ksym;
+
+static __noinline int cb1(void *map, void *key, void *value, void *ctx)
+{
+	void *p = *(void **)ctx;
+	bpf_kfunc_call_test_release(p);
+	/* Without the fix this would cause underflow */
+	return 0;
+}
+
+SEC("?tc")
+int underflow_prog(void *ctx)
+{
+	struct prog_test_ref_kfunc *p;
+	unsigned long sl = 0;
+
+	p = bpf_kfunc_call_test_acquire(&sl);
+	if (!p)
+		return 0;
+	bpf_for_each_map_elem(&array_map, cb1, &p, 0);
+	return 0;
+}
+
+static __always_inline int cb2(void *map, void *key, void *value, void *ctx)
+{
+	unsigned long sl = 0;
+
+	*(void **)ctx = bpf_kfunc_call_test_acquire(&sl);
+	/* Without the fix this would leak memory */
+	return 0;
+}
+
+SEC("?tc")
+int leak_prog(void *ctx)
+{
+	struct prog_test_ref_kfunc *p;
+	struct map_value *v;
+	unsigned long sl;
+
+	v = bpf_map_lookup_elem(&array_map, &(int){0});
+	if (!v)
+		return 0;
+
+	p = NULL;
+	bpf_for_each_map_elem(&array_map, cb2, &p, 0);
+	p = bpf_kptr_xchg(&v->ptr, p);
+	if (p)
+		bpf_kfunc_call_test_release(p);
+	return 0;
+}
+
+static __always_inline int cb(void *map, void *key, void *value, void *ctx)
+{
+	return 0;
+}
+
+static __always_inline int cb3(void *map, void *key, void *value, void *ctx)
+{
+	unsigned long sl = 0;
+	void *p;
+
+	bpf_kfunc_call_test_acquire(&sl);
+	bpf_for_each_map_elem(&array_map, cb, &p, 0);
+	/* It should only complain here, not in cb. This is why we need
+	 * callback_ref to be set to frameno.
+	 */
+	return 0;
+}
+
+SEC("?tc")
+int nested_cb(void *ctx)
+{
+	struct prog_test_ref_kfunc *p;
+	unsigned long sl = 0;
+	int sp = 0;
+
+	p = bpf_kfunc_call_test_acquire(&sl);
+	if (!p)
+		return 0;
+	bpf_for_each_map_elem(&array_map, cb3, &sp, 0);
+	bpf_kfunc_call_test_release(p);
+	return 0;
+}
+
+SEC("?tc")
+int non_cb_transfer_ref(void *ctx)
+{
+	struct prog_test_ref_kfunc *p;
+	unsigned long sl = 0;
+
+	p = bpf_kfunc_call_test_acquire(&sl);
+	if (!p)
+		return 0;
+	cb1(NULL, NULL, NULL, &p);
+	bpf_kfunc_call_test_acquire(&sl);
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.c b/tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.c
new file mode 100644
index 000000000000..13dfb4bbfd28
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/cgroup_getset_retval_hooks.c
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+
+#define BPF_RETVAL_HOOK(name, section, ctx, expected_err) \
+	__attribute__((__section__("?" section))) \
+	int name(struct ctx *_ctx) \
+	{ \
+		bpf_set_retval(bpf_get_retval()); \
+		return 1; \
+	}
+
+#include "cgroup_getset_retval_hooks.h"
+
+#undef BPF_RETVAL_HOOK
diff --git a/tools/testing/selftests/bpf/progs/cgroup_hierarchical_stats.c b/tools/testing/selftests/bpf/progs/cgroup_hierarchical_stats.c
new file mode 100644
index 000000000000..c74362854948
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/cgroup_hierarchical_stats.c
@@ -0,0 +1,155 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2022 Google LLC.
+ */
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <bpf/bpf_core_read.h>
+
+char _license[] SEC("license") = "GPL";
+
+struct percpu_attach_counter {
+	/* Previous percpu state, to figure out if we have new updates */
+	__u64 prev;
+	/* Current percpu state */
+	__u64 state;
+};
+
+struct attach_counter {
+	/* State propagated through children, pending aggregation */
+	__u64 pending;
+	/* Total state, including all cpus and all children */
+	__u64 state;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PERCPU_HASH);
+	__uint(max_entries, 1024);
+	__type(key, __u64);
+	__type(value, struct percpu_attach_counter);
+} percpu_attach_counters SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 1024);
+	__type(key, __u64);
+	__type(value, struct attach_counter);
+} attach_counters SEC(".maps");
+
+extern void cgroup_rstat_updated(struct cgroup *cgrp, int cpu) __ksym;
+extern void cgroup_rstat_flush(struct cgroup *cgrp) __ksym;
+
+static uint64_t cgroup_id(struct cgroup *cgrp)
+{
+	return cgrp->kn->id;
+}
+
+static int create_percpu_attach_counter(__u64 cg_id, __u64 state)
+{
+	struct percpu_attach_counter pcpu_init = {.state = state, .prev = 0};
+
+	return bpf_map_update_elem(&percpu_attach_counters, &cg_id,
+				   &pcpu_init, BPF_NOEXIST);
+}
+
+static int create_attach_counter(__u64 cg_id, __u64 state, __u64 pending)
+{
+	struct attach_counter init = {.state = state, .pending = pending};
+
+	return bpf_map_update_elem(&attach_counters, &cg_id,
+				   &init, BPF_NOEXIST);
+}
+
+SEC("fentry/cgroup_attach_task")
+int BPF_PROG(counter, struct cgroup *dst_cgrp, struct task_struct *leader,
+	     bool threadgroup)
+{
+	__u64 cg_id = cgroup_id(dst_cgrp);
+	struct percpu_attach_counter *pcpu_counter = bpf_map_lookup_elem(
+			&percpu_attach_counters,
+			&cg_id);
+
+	if (pcpu_counter)
+		pcpu_counter->state += 1;
+	else if (create_percpu_attach_counter(cg_id, 1))
+		return 0;
+
+	cgroup_rstat_updated(dst_cgrp, bpf_get_smp_processor_id());
+	return 0;
+}
+
+SEC("fentry/bpf_rstat_flush")
+int BPF_PROG(flusher, struct cgroup *cgrp, struct cgroup *parent, int cpu)
+{
+	struct percpu_attach_counter *pcpu_counter;
+	struct attach_counter *total_counter, *parent_counter;
+	__u64 cg_id = cgroup_id(cgrp);
+	__u64 parent_cg_id = parent ? cgroup_id(parent) : 0;
+	__u64 state;
+	__u64 delta = 0;
+
+	/* Add CPU changes on this level since the last flush */
+	pcpu_counter = bpf_map_lookup_percpu_elem(&percpu_attach_counters,
+						  &cg_id, cpu);
+	if (pcpu_counter) {
+		state = pcpu_counter->state;
+		delta += state - pcpu_counter->prev;
+		pcpu_counter->prev = state;
+	}
+
+	total_counter = bpf_map_lookup_elem(&attach_counters, &cg_id);
+	if (!total_counter) {
+		if (create_attach_counter(cg_id, delta, 0))
+			return 0;
+		goto update_parent;
+	}
+
+	/* Collect pending stats from subtree */
+	if (total_counter->pending) {
+		delta += total_counter->pending;
+		total_counter->pending = 0;
+	}
+
+	/* Propagate changes to this cgroup's total */
+	total_counter->state += delta;
+
+update_parent:
+	/* Skip if there are no changes to propagate, or no parent */
+	if (!delta || !parent_cg_id)
+		return 0;
+
+	/* Propagate changes to cgroup's parent */
+	parent_counter = bpf_map_lookup_elem(&attach_counters,
+					     &parent_cg_id);
+	if (parent_counter)
+		parent_counter->pending += delta;
+	else
+		create_attach_counter(parent_cg_id, 0, delta);
+	return 0;
+}
+
+SEC("iter.s/cgroup")
+int BPF_PROG(dumper, struct bpf_iter_meta *meta, struct cgroup *cgrp)
+{
+	struct seq_file *seq = meta->seq;
+	struct attach_counter *total_counter;
+	__u64 cg_id = cgrp ? cgroup_id(cgrp) : 0;
+
+	/* Do nothing for the terminal call */
+	if (!cg_id)
+		return 1;
+
+	/* Flush the stats to make sure we get the most updated numbers */
+	cgroup_rstat_flush(cgrp);
+
+	total_counter = bpf_map_lookup_elem(&attach_counters, &cg_id);
+	if (!total_counter) {
+		BPF_SEQ_PRINTF(seq, "cg_id: %llu, attach_counter: 0\n",
+			       cg_id);
+	} else {
+		BPF_SEQ_PRINTF(seq, "cg_id: %llu, attach_counter: %llu\n",
+			       cg_id, total_counter->state);
+	}
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/cgroup_iter.c b/tools/testing/selftests/bpf/progs/cgroup_iter.c
new file mode 100644
index 000000000000..de03997322a7
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/cgroup_iter.c
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Google */
+
+#include "bpf_iter.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+char _license[] SEC("license") = "GPL";
+int terminate_early = 0;
+u64 terminal_cgroup = 0;
+
+static inline u64 cgroup_id(struct cgroup *cgrp)
+{
+	return cgrp->kn->id;
+}
+
+SEC("iter/cgroup")
+int cgroup_id_printer(struct bpf_iter__cgroup *ctx)
+{
+	struct seq_file *seq = ctx->meta->seq;
+	struct cgroup *cgrp = ctx->cgroup;
+
+	/* epilogue */
+	if (cgrp == NULL) {
+		BPF_SEQ_PRINTF(seq, "epilogue\n");
+		return 0;
+	}
+
+	/* prologue */
+	if (ctx->meta->seq_num == 0)
+		BPF_SEQ_PRINTF(seq, "prologue\n");
+
+	BPF_SEQ_PRINTF(seq, "%8llu\n", cgroup_id(cgrp));
+
+	if (terminal_cgroup == cgroup_id(cgrp))
+		return 1;
+
+	return terminate_early ? 1 : 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/connect4_prog.c b/tools/testing/selftests/bpf/progs/connect4_prog.c
index b241932911db..ec25371de789 100644
--- a/tools/testing/selftests/bpf/progs/connect4_prog.c
+++ b/tools/testing/selftests/bpf/progs/connect4_prog.c
@@ -7,14 +7,15 @@
 #include <linux/bpf.h>
 #include <linux/in.h>
 #include <linux/in6.h>
-#include <sys/socket.h>
-#include <netinet/tcp.h>
+#include <linux/tcp.h>
 #include <linux/if.h>
 #include <errno.h>
 
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_endian.h>
 
+#include "bpf_tcp_helpers.h"
+
 #define SRC_REWRITE_IP4		0x7f000004U
 #define DST_REWRITE_IP4		0x7f000001U
 #define DST_REWRITE_PORT4	4444
diff --git a/tools/testing/selftests/bpf/progs/connect_ping.c b/tools/testing/selftests/bpf/progs/connect_ping.c
new file mode 100644
index 000000000000..60178192b672
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/connect_ping.c
@@ -0,0 +1,53 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+/*
+ * Copyright 2022 Google LLC.
+ */
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
+#include <netinet/in.h>
+#include <sys/socket.h>
+
+/* 2001:db8::1 */
+#define BINDADDR_V6 { { { 0x20,0x01,0x0d,0xb8,0,0,0,0,0,0,0,0,0,0,0,1 } } }
+
+__u32 do_bind = 0;
+__u32 has_error = 0;
+__u32 invocations_v4 = 0;
+__u32 invocations_v6 = 0;
+
+SEC("cgroup/connect4")
+int connect_v4_prog(struct bpf_sock_addr *ctx)
+{
+	struct sockaddr_in sa = {
+		.sin_family = AF_INET,
+		.sin_addr.s_addr = bpf_htonl(0x01010101),
+	};
+
+	__sync_fetch_and_add(&invocations_v4, 1);
+
+	if (do_bind && bpf_bind(ctx, (struct sockaddr *)&sa, sizeof(sa)))
+		has_error = 1;
+
+	return 1;
+}
+
+SEC("cgroup/connect6")
+int connect_v6_prog(struct bpf_sock_addr *ctx)
+{
+	struct sockaddr_in6 sa = {
+		.sin6_family = AF_INET6,
+		.sin6_addr = BINDADDR_V6,
+	};
+
+	__sync_fetch_and_add(&invocations_v6, 1);
+
+	if (do_bind && bpf_bind(ctx, (struct sockaddr *)&sa, sizeof(sa)))
+		has_error = 1;
+
+	return 1;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c
index 0a26c243e6e9..b0f08ff024fb 100644
--- a/tools/testing/selftests/bpf/progs/dynptr_fail.c
+++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c
@@ -65,7 +65,7 @@ static int get_map_val_dynptr(struct bpf_dynptr *ptr)
 /* Every bpf_ringbuf_reserve_dynptr call must have a corresponding
  * bpf_ringbuf_submit/discard_dynptr call
  */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int ringbuf_missing_release1(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -77,7 +77,7 @@ int ringbuf_missing_release1(void *ctx)
 	return 0;
 }
 
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int ringbuf_missing_release2(void *ctx)
 {
 	struct bpf_dynptr ptr1, ptr2;
@@ -112,7 +112,7 @@ static int missing_release_callback_fn(__u32 index, void *data)
 }
 
 /* Any dynptr initialized within a callback must have bpf_dynptr_put called */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int ringbuf_missing_release_callback(void *ctx)
 {
 	bpf_loop(10, missing_release_callback_fn, NULL, 0);
@@ -120,7 +120,7 @@ int ringbuf_missing_release_callback(void *ctx)
 }
 
 /* Can't call bpf_ringbuf_submit/discard_dynptr on a non-initialized dynptr */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int ringbuf_release_uninit_dynptr(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -132,7 +132,7 @@ int ringbuf_release_uninit_dynptr(void *ctx)
 }
 
 /* A dynptr can't be used after it has been invalidated */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int use_after_invalid(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -151,7 +151,7 @@ int use_after_invalid(void *ctx)
 }
 
 /* Can't call non-dynptr ringbuf APIs on a dynptr ringbuf sample */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int ringbuf_invalid_api(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -173,7 +173,7 @@ done:
 }
 
 /* Can't add a dynptr to a map */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int add_dynptr_to_map1(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -190,7 +190,7 @@ int add_dynptr_to_map1(void *ctx)
 }
 
 /* Can't add a struct with an embedded dynptr to a map */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int add_dynptr_to_map2(void *ctx)
 {
 	struct test_info x;
@@ -207,7 +207,7 @@ int add_dynptr_to_map2(void *ctx)
 }
 
 /* A data slice can't be accessed out of bounds */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int data_slice_out_of_bounds_ringbuf(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -227,7 +227,7 @@ done:
 	return 0;
 }
 
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int data_slice_out_of_bounds_map_value(void *ctx)
 {
 	__u32 key = 0, map_val;
@@ -247,8 +247,8 @@ int data_slice_out_of_bounds_map_value(void *ctx)
 }
 
 /* A data slice can't be used after it has been released */
-SEC("?raw_tp/sys_nanosleep")
-int data_slice_use_after_release(void *ctx)
+SEC("?raw_tp")
+int data_slice_use_after_release1(void *ctx)
 {
 	struct bpf_dynptr ptr;
 	struct sample *sample;
@@ -272,8 +272,44 @@ done:
 	return 0;
 }
 
+/* A data slice can't be used after it has been released.
+ *
+ * This tests the case where the data slice tracks a dynptr (ptr2)
+ * that is at a non-zero offset from the frame pointer (ptr1 is at fp,
+ * ptr2 is at fp - 16).
+ */
+SEC("?raw_tp")
+int data_slice_use_after_release2(void *ctx)
+{
+	struct bpf_dynptr ptr1, ptr2;
+	struct sample *sample;
+
+	bpf_ringbuf_reserve_dynptr(&ringbuf, 64, 0, &ptr1);
+	bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(*sample), 0, &ptr2);
+
+	sample = bpf_dynptr_data(&ptr2, 0, sizeof(*sample));
+	if (!sample)
+		goto done;
+
+	sample->pid = 23;
+
+	bpf_ringbuf_submit_dynptr(&ptr2, 0);
+
+	/* this should fail */
+	sample->pid = 23;
+
+	bpf_ringbuf_submit_dynptr(&ptr1, 0);
+
+	return 0;
+
+done:
+	bpf_ringbuf_discard_dynptr(&ptr2, 0);
+	bpf_ringbuf_discard_dynptr(&ptr1, 0);
+	return 0;
+}
+
 /* A data slice must be first checked for NULL */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int data_slice_missing_null_check1(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -293,7 +329,7 @@ int data_slice_missing_null_check1(void *ctx)
 }
 
 /* A data slice can't be dereferenced if it wasn't checked for null */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int data_slice_missing_null_check2(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -315,7 +351,7 @@ done:
 /* Can't pass in a dynptr as an arg to a helper function that doesn't take in a
  * dynptr argument
  */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_helper1(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -329,7 +365,7 @@ int invalid_helper1(void *ctx)
 }
 
 /* A dynptr can't be passed into a helper function at a non-zero offset */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_helper2(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -344,7 +380,7 @@ int invalid_helper2(void *ctx)
 }
 
 /* A bpf_dynptr is invalidated if it's been written into */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_write1(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -365,7 +401,7 @@ int invalid_write1(void *ctx)
  * A bpf_dynptr can't be used as a dynptr if it has been written into at a fixed
  * offset
  */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_write2(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -388,7 +424,7 @@ int invalid_write2(void *ctx)
  * A bpf_dynptr can't be used as a dynptr if it has been written into at a
  * non-const offset
  */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_write3(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -419,7 +455,7 @@ static int invalid_write4_callback(__u32 index, void *data)
 /* If the dynptr is written into in a callback function, it should
  * be invalidated as a dynptr
  */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_write4(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -436,7 +472,7 @@ int invalid_write4(void *ctx)
 
 /* A globally-defined bpf_dynptr can't be used (it must reside as a stack frame) */
 struct bpf_dynptr global_dynptr;
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int global(void *ctx)
 {
 	/* this should fail */
@@ -448,7 +484,7 @@ int global(void *ctx)
 }
 
 /* A direct read should fail */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_read1(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -464,7 +500,7 @@ int invalid_read1(void *ctx)
 }
 
 /* A direct read at an offset should fail */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_read2(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -479,7 +515,7 @@ int invalid_read2(void *ctx)
 }
 
 /* A direct read at an offset into the lower stack slot should fail */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_read3(void *ctx)
 {
 	struct bpf_dynptr ptr1, ptr2;
@@ -505,7 +541,7 @@ static int invalid_read4_callback(__u32 index, void *data)
 }
 
 /* A direct read within a callback function should fail */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_read4(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -520,7 +556,7 @@ int invalid_read4(void *ctx)
 }
 
 /* Initializing a dynptr on an offset should fail */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int invalid_offset(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -534,7 +570,7 @@ int invalid_offset(void *ctx)
 }
 
 /* Can't release a dynptr twice */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int release_twice(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -560,7 +596,7 @@ static int release_twice_callback_fn(__u32 index, void *data)
 /* Test that releasing a dynptr twice, where one of the releases happens
  * within a calback function, fails
  */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int release_twice_callback(void *ctx)
 {
 	struct bpf_dynptr ptr;
@@ -575,7 +611,7 @@ int release_twice_callback(void *ctx)
 }
 
 /* Reject unsupported local mem types for dynptr_from_mem API */
-SEC("?raw_tp/sys_nanosleep")
+SEC("?raw_tp")
 int dynptr_from_mem_invalid_api(void *ctx)
 {
 	struct bpf_dynptr ptr;
diff --git a/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c
index 48cd14b43741..4547b059d487 100644
--- a/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c
+++ b/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c
@@ -73,10 +73,10 @@ int test_subprog2(struct args_subprog2 *ctx)
 			      __builtin_preserve_access_index(&skb->len));
 
 	ret = ctx->ret;
-	/* bpf_prog_test_load() loads "test_pkt_access.o" with BPF_F_TEST_RND_HI32
-	 * which randomizes upper 32 bits after BPF_ALU32 insns.
-	 * Hence after 'w0 <<= 1' upper bits of $rax are random.
-	 * That is expected and correct. Trim them.
+	/* bpf_prog_test_load() loads "test_pkt_access.bpf.o" with
+	 * BPF_F_TEST_RND_HI32 which randomizes upper 32 bits after BPF_ALU32
+	 * insns. Hence after 'w0 <<= 1' upper bits of $rax are random. That is
+	 * expected and correct. Trim them.
 	 */
 	ret = (__u32) ret;
 	if (len != 74 || ret != 148)
diff --git a/tools/testing/selftests/bpf/progs/get_func_ip_test.c b/tools/testing/selftests/bpf/progs/get_func_ip_test.c
index a587aeca5ae0..8559e698b40d 100644
--- a/tools/testing/selftests/bpf/progs/get_func_ip_test.c
+++ b/tools/testing/selftests/bpf/progs/get_func_ip_test.c
@@ -2,6 +2,7 @@
 #include <linux/bpf.h>
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_tracing.h>
+#include <stdbool.h>
 
 char _license[] SEC("license") = "GPL";
 
@@ -13,6 +14,16 @@ extern const void bpf_modify_return_test __ksym;
 extern const void bpf_fentry_test6 __ksym;
 extern const void bpf_fentry_test7 __ksym;
 
+extern bool CONFIG_X86_KERNEL_IBT __kconfig __weak;
+
+/* This function is here to have CONFIG_X86_KERNEL_IBT
+ * used and added to object BTF.
+ */
+int unused(void)
+{
+	return CONFIG_X86_KERNEL_IBT ? 0 : 1;
+}
+
 __u64 test1_result = 0;
 SEC("fentry/bpf_fentry_test1")
 int BPF_PROG(test1, int a)
@@ -64,21 +75,11 @@ int BPF_PROG(test5, int a, int *b, int ret)
 }
 
 __u64 test6_result = 0;
-SEC("kprobe/bpf_fentry_test6+0x5")
+SEC("?kprobe")
 int test6(struct pt_regs *ctx)
 {
 	__u64 addr = bpf_get_func_ip(ctx);
 
-	test6_result = (const void *) addr == &bpf_fentry_test6 + 5;
-	return 0;
-}
-
-__u64 test7_result = 0;
-SEC("kprobe/bpf_fentry_test7+5")
-int test7(struct pt_regs *ctx)
-{
-	__u64 addr = bpf_get_func_ip(ctx);
-
-	test7_result = (const void *) addr == &bpf_fentry_test7 + 5;
+	test6_result = (const void *) addr == 0;
 	return 0;
 }
diff --git a/tools/testing/selftests/bpf/progs/htab_update.c b/tools/testing/selftests/bpf/progs/htab_update.c
new file mode 100644
index 000000000000..7481bb30b29b
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/htab_update.c
@@ -0,0 +1,29 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2022. Huawei Technologies Co., Ltd */
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+char _license[] SEC("license") = "GPL";
+
+struct {
+	__uint(type, BPF_MAP_TYPE_HASH);
+	__uint(max_entries, 1);
+	__uint(key_size, sizeof(__u32));
+	__uint(value_size, sizeof(__u32));
+} htab SEC(".maps");
+
+int pid = 0;
+int update_err = 0;
+
+SEC("?fentry/lookup_elem_raw")
+int lookup_elem_raw(void *ctx)
+{
+	__u32 key = 0, value = 1;
+
+	if ((bpf_get_current_pid_tgid() >> 32) != pid)
+		return 0;
+
+	update_err = bpf_map_update_elem(&htab, &key, &value, 0);
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_destructive.c b/tools/testing/selftests/bpf/progs/kfunc_call_destructive.c
new file mode 100644
index 000000000000..767472bc5a97
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/kfunc_call_destructive.c
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+
+extern void bpf_kfunc_call_test_destructive(void) __ksym;
+
+SEC("tc")
+int kfunc_destructive_test(void)
+{
+	bpf_kfunc_call_test_destructive();
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_fail.c b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c
new file mode 100644
index 000000000000..b98313d391c6
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/kfunc_call_fail.c
@@ -0,0 +1,160 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2021 Facebook */
+#include <vmlinux.h>
+#include <bpf/bpf_helpers.h>
+
+extern struct prog_test_ref_kfunc *bpf_kfunc_call_test_acquire(unsigned long *sp) __ksym;
+extern void bpf_kfunc_call_test_release(struct prog_test_ref_kfunc *p) __ksym;
+extern void bpf_kfunc_call_test_mem_len_pass1(void *mem, int len) __ksym;
+extern int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size) __ksym;
+extern int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym;
+extern int *bpf_kfunc_call_test_acq_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym;
+extern void bpf_kfunc_call_int_mem_release(int *p) __ksym;
+
+struct syscall_test_args {
+	__u8 data[16];
+	size_t size;
+};
+
+SEC("?syscall")
+int kfunc_syscall_test_fail(struct syscall_test_args *args)
+{
+	bpf_kfunc_call_test_mem_len_pass1(&args->data, sizeof(*args) + 1);
+
+	return 0;
+}
+
+SEC("?syscall")
+int kfunc_syscall_test_null_fail(struct syscall_test_args *args)
+{
+	/* Must be called with args as a NULL pointer
+	 * we do not check for it to have the verifier consider that
+	 * the pointer might not be null, and so we can load it.
+	 *
+	 * So the following can not be added:
+	 *
+	 * if (args)
+	 *      return -22;
+	 */
+
+	bpf_kfunc_call_test_mem_len_pass1(args, sizeof(*args));
+
+	return 0;
+}
+
+SEC("?tc")
+int kfunc_call_test_get_mem_fail_rdonly(struct __sk_buff *skb)
+{
+	struct prog_test_ref_kfunc *pt;
+	unsigned long s = 0;
+	int *p = NULL;
+	int ret = 0;
+
+	pt = bpf_kfunc_call_test_acquire(&s);
+	if (pt) {
+		p = bpf_kfunc_call_test_get_rdonly_mem(pt, 2 * sizeof(int));
+		if (p)
+			p[0] = 42; /* this is a read-only buffer, so -EACCES */
+		else
+			ret = -1;
+
+		bpf_kfunc_call_test_release(pt);
+	}
+	return ret;
+}
+
+SEC("?tc")
+int kfunc_call_test_get_mem_fail_use_after_free(struct __sk_buff *skb)
+{
+	struct prog_test_ref_kfunc *pt;
+	unsigned long s = 0;
+	int *p = NULL;
+	int ret = 0;
+
+	pt = bpf_kfunc_call_test_acquire(&s);
+	if (pt) {
+		p = bpf_kfunc_call_test_get_rdwr_mem(pt, 2 * sizeof(int));
+		if (p) {
+			p[0] = 42;
+			ret = p[1]; /* 108 */
+		} else {
+			ret = -1;
+		}
+
+		bpf_kfunc_call_test_release(pt);
+	}
+	if (p)
+		ret = p[0]; /* p is not valid anymore */
+
+	return ret;
+}
+
+SEC("?tc")
+int kfunc_call_test_get_mem_fail_oob(struct __sk_buff *skb)
+{
+	struct prog_test_ref_kfunc *pt;
+	unsigned long s = 0;
+	int *p = NULL;
+	int ret = 0;
+
+	pt = bpf_kfunc_call_test_acquire(&s);
+	if (pt) {
+		p = bpf_kfunc_call_test_get_rdonly_mem(pt, 2 * sizeof(int));
+		if (p)
+			ret = p[2 * sizeof(int)]; /* oob access, so -EACCES */
+		else
+			ret = -1;
+
+		bpf_kfunc_call_test_release(pt);
+	}
+	return ret;
+}
+
+int not_const_size = 2 * sizeof(int);
+
+SEC("?tc")
+int kfunc_call_test_get_mem_fail_not_const(struct __sk_buff *skb)
+{
+	struct prog_test_ref_kfunc *pt;
+	unsigned long s = 0;
+	int *p = NULL;
+	int ret = 0;
+
+	pt = bpf_kfunc_call_test_acquire(&s);
+	if (pt) {
+		p = bpf_kfunc_call_test_get_rdonly_mem(pt, not_const_size); /* non const size, -EINVAL */
+		if (p)
+			ret = p[0];
+		else
+			ret = -1;
+
+		bpf_kfunc_call_test_release(pt);
+	}
+	return ret;
+}
+
+SEC("?tc")
+int kfunc_call_test_mem_acquire_fail(struct __sk_buff *skb)
+{
+	struct prog_test_ref_kfunc *pt;
+	unsigned long s = 0;
+	int *p = NULL;
+	int ret = 0;
+
+	pt = bpf_kfunc_call_test_acquire(&s);
+	if (pt) {
+		/* we are failing on this one, because we are not acquiring a PTR_TO_BTF_ID (a struct ptr) */
+		p = bpf_kfunc_call_test_acq_rdonly_mem(pt, 2 * sizeof(int));
+		if (p)
+			ret = p[0];
+		else
+			ret = -1;
+
+		bpf_kfunc_call_int_mem_release(p);
+
+		bpf_kfunc_call_test_release(pt);
+	}
+	return ret;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/kfunc_call_test.c b/tools/testing/selftests/bpf/progs/kfunc_call_test.c
index 5aecbb9fdc68..f636e50be259 100644
--- a/tools/testing/selftests/bpf/progs/kfunc_call_test.c
+++ b/tools/testing/selftests/bpf/progs/kfunc_call_test.c
@@ -14,6 +14,8 @@ extern void bpf_kfunc_call_test_pass1(struct prog_test_pass1 *p) __ksym;
 extern void bpf_kfunc_call_test_pass2(struct prog_test_pass2 *p) __ksym;
 extern void bpf_kfunc_call_test_mem_len_pass1(void *mem, int len) __ksym;
 extern void bpf_kfunc_call_test_mem_len_fail2(__u64 *mem, int len) __ksym;
+extern int *bpf_kfunc_call_test_get_rdwr_mem(struct prog_test_ref_kfunc *p, const int rdwr_buf_size) __ksym;
+extern int *bpf_kfunc_call_test_get_rdonly_mem(struct prog_test_ref_kfunc *p, const int rdonly_buf_size) __ksym;
 
 SEC("tc")
 int kfunc_call_test2(struct __sk_buff *skb)
@@ -92,4 +94,73 @@ int kfunc_call_test_pass(struct __sk_buff *skb)
 	return 0;
 }
 
+struct syscall_test_args {
+	__u8 data[16];
+	size_t size;
+};
+
+SEC("syscall")
+int kfunc_syscall_test(struct syscall_test_args *args)
+{
+	const long size = args->size;
+
+	if (size > sizeof(args->data))
+		return -7; /* -E2BIG */
+
+	bpf_kfunc_call_test_mem_len_pass1(&args->data, sizeof(args->data));
+	bpf_kfunc_call_test_mem_len_pass1(&args->data, sizeof(*args));
+	bpf_kfunc_call_test_mem_len_pass1(&args->data, size);
+
+	return 0;
+}
+
+SEC("syscall")
+int kfunc_syscall_test_null(struct syscall_test_args *args)
+{
+	/* Must be called with args as a NULL pointer
+	 * we do not check for it to have the verifier consider that
+	 * the pointer might not be null, and so we can load it.
+	 *
+	 * So the following can not be added:
+	 *
+	 * if (args)
+	 *      return -22;
+	 */
+
+	bpf_kfunc_call_test_mem_len_pass1(args, 0);
+
+	return 0;
+}
+
+SEC("tc")
+int kfunc_call_test_get_mem(struct __sk_buff *skb)
+{
+	struct prog_test_ref_kfunc *pt;
+	unsigned long s = 0;
+	int *p = NULL;
+	int ret = 0;
+
+	pt = bpf_kfunc_call_test_acquire(&s);
+	if (pt) {
+		p = bpf_kfunc_call_test_get_rdwr_mem(pt, 2 * sizeof(int));
+		if (p) {
+			p[0] = 42;
+			ret = p[1]; /* 108 */
+		} else {
+			ret = -1;
+		}
+
+		if (ret >= 0) {
+			p = bpf_kfunc_call_test_get_rdonly_mem(pt, 2 * sizeof(int));
+			if (p)
+				ret = p[0]; /* 42 */
+			else
+				ret = -1;
+		}
+
+		bpf_kfunc_call_test_release(pt);
+	}
+	return ret;
+}
+
 char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/kprobe_multi.c b/tools/testing/selftests/bpf/progs/kprobe_multi.c
index 08f95a8155d1..98c3399e15c0 100644
--- a/tools/testing/selftests/bpf/progs/kprobe_multi.c
+++ b/tools/testing/selftests/bpf/progs/kprobe_multi.c
@@ -36,15 +36,13 @@ __u64 kretprobe_test6_result = 0;
 __u64 kretprobe_test7_result = 0;
 __u64 kretprobe_test8_result = 0;
 
-extern bool CONFIG_X86_KERNEL_IBT __kconfig __weak;
-
 static void kprobe_multi_check(void *ctx, bool is_return)
 {
 	if (bpf_get_current_pid_tgid() >> 32 != pid)
 		return;
 
 	__u64 cookie = test_cookie ? bpf_get_attach_cookie(ctx) : 0;
-	__u64 addr = bpf_get_func_ip(ctx) - (CONFIG_X86_KERNEL_IBT ? 4 : 0);
+	__u64 addr = bpf_get_func_ip(ctx);
 
 #define SET(__var, __addr, __cookie) ({			\
 	if (((const void *) addr == __addr) &&		\
diff --git a/tools/testing/selftests/bpf/progs/lsm.c b/tools/testing/selftests/bpf/progs/lsm.c
index 33694ef8acfa..d8d8af623bc2 100644
--- a/tools/testing/selftests/bpf/progs/lsm.c
+++ b/tools/testing/selftests/bpf/progs/lsm.c
@@ -4,6 +4,7 @@
  * Copyright 2020 Google LLC.
  */
 
+#include "bpf_misc.h"
 #include "vmlinux.h"
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_tracing.h>
@@ -160,7 +161,7 @@ int BPF_PROG(test_task_free, struct task_struct *task)
 
 int copy_test = 0;
 
-SEC("fentry.s/__x64_sys_setdomainname")
+SEC("fentry.s/" SYS_PREFIX "sys_setdomainname")
 int BPF_PROG(test_sys_setdomainname, struct pt_regs *regs)
 {
 	void *ptr = (void *)PT_REGS_PARM1(regs);
diff --git a/tools/testing/selftests/bpf/progs/read_bpf_task_storage_busy.c b/tools/testing/selftests/bpf/progs/read_bpf_task_storage_busy.c
new file mode 100644
index 000000000000..a47bb0120719
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/read_bpf_task_storage_busy.c
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2022. Huawei Technologies Co., Ltd */
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+extern bool CONFIG_PREEMPT __kconfig __weak;
+extern const int bpf_task_storage_busy __ksym;
+
+char _license[] SEC("license") = "GPL";
+
+int pid = 0;
+int busy = 0;
+
+struct {
+	__uint(type, BPF_MAP_TYPE_TASK_STORAGE);
+	__uint(map_flags, BPF_F_NO_PREALLOC);
+	__type(key, int);
+	__type(value, long);
+} task SEC(".maps");
+
+SEC("raw_tp/sys_enter")
+int BPF_PROG(read_bpf_task_storage_busy)
+{
+	int *value;
+	int key;
+
+	if (!CONFIG_PREEMPT)
+		return 0;
+
+	if (bpf_get_current_pid_tgid() >> 32 != pid)
+		return 0;
+
+	value = bpf_this_cpu_ptr(&bpf_task_storage_busy);
+	if (value)
+		busy = *value;
+
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/setget_sockopt.c b/tools/testing/selftests/bpf/progs/setget_sockopt.c
new file mode 100644
index 000000000000..9523333b8905
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/setget_sockopt.c
@@ -0,0 +1,395 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) Meta Platforms, Inc. and affiliates. */
+
+#include "vmlinux.h"
+#include "bpf_tracing_net.h"
+#include <bpf/bpf_core_read.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+#endif
+
+extern unsigned long CONFIG_HZ __kconfig;
+
+const volatile char veth[IFNAMSIZ];
+const volatile int veth_ifindex;
+
+int nr_listen;
+int nr_passive;
+int nr_active;
+int nr_connect;
+int nr_binddev;
+int nr_socket_post_create;
+
+struct sockopt_test {
+	int opt;
+	int new;
+	int restore;
+	int expected;
+	int tcp_expected;
+	unsigned int flip:1;
+};
+
+static const char not_exist_cc[] = "not_exist";
+static const char cubic_cc[] = "cubic";
+static const char reno_cc[] = "reno";
+
+static const struct sockopt_test sol_socket_tests[] = {
+	{ .opt = SO_REUSEADDR, .flip = 1, },
+	{ .opt = SO_SNDBUF, .new = 8123, .expected = 8123 * 2, },
+	{ .opt = SO_RCVBUF, .new = 8123, .expected = 8123 * 2, },
+	{ .opt = SO_KEEPALIVE, .flip = 1, },
+	{ .opt = SO_PRIORITY, .new = 0xeb9f, .expected = 0xeb9f, },
+	{ .opt = SO_REUSEPORT, .flip = 1, },
+	{ .opt = SO_RCVLOWAT, .new = 8123, .expected = 8123, },
+	{ .opt = SO_MARK, .new = 0xeb9f, .expected = 0xeb9f, },
+	{ .opt = SO_MAX_PACING_RATE, .new = 0xeb9f, .expected = 0xeb9f, },
+	{ .opt = SO_TXREHASH, .flip = 1, },
+	{ .opt = 0, },
+};
+
+static const struct sockopt_test sol_tcp_tests[] = {
+	{ .opt = TCP_NODELAY, .flip = 1, },
+	{ .opt = TCP_KEEPIDLE, .new = 123, .expected = 123, .restore = 321, },
+	{ .opt = TCP_KEEPINTVL, .new = 123, .expected = 123, .restore = 321, },
+	{ .opt = TCP_KEEPCNT, .new = 123, .expected = 123, .restore = 124, },
+	{ .opt = TCP_SYNCNT, .new = 123, .expected = 123, .restore = 124, },
+	{ .opt = TCP_WINDOW_CLAMP, .new = 8123, .expected = 8123, .restore = 8124, },
+	{ .opt = TCP_CONGESTION, },
+	{ .opt = TCP_THIN_LINEAR_TIMEOUTS, .flip = 1, },
+	{ .opt = TCP_USER_TIMEOUT, .new = 123400, .expected = 123400, },
+	{ .opt = TCP_NOTSENT_LOWAT, .new = 1314, .expected = 1314, },
+	{ .opt = 0, },
+};
+
+static const struct sockopt_test sol_ip_tests[] = {
+	{ .opt = IP_TOS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, },
+	{ .opt = 0, },
+};
+
+static const struct sockopt_test sol_ipv6_tests[] = {
+	{ .opt = IPV6_TCLASS, .new = 0xe1, .expected = 0xe1, .tcp_expected = 0xe0, },
+	{ .opt = IPV6_AUTOFLOWLABEL, .flip = 1, },
+	{ .opt = 0, },
+};
+
+struct loop_ctx {
+	void *ctx;
+	struct sock *sk;
+};
+
+static int bpf_test_sockopt_flip(void *ctx, struct sock *sk,
+				 const struct sockopt_test *t,
+				 int level)
+{
+	int old, tmp, new, opt = t->opt;
+
+	opt = t->opt;
+
+	if (bpf_getsockopt(ctx, level, opt, &old, sizeof(old)))
+		return 1;
+	/* kernel initialized txrehash to 255 */
+	if (level == SOL_SOCKET && opt == SO_TXREHASH && old != 0 && old != 1)
+		old = 1;
+
+	new = !old;
+	if (bpf_setsockopt(ctx, level, opt, &new, sizeof(new)))
+		return 1;
+	if (bpf_getsockopt(ctx, level, opt, &tmp, sizeof(tmp)) ||
+	    tmp != new)
+		return 1;
+
+	if (bpf_setsockopt(ctx, level, opt, &old, sizeof(old)))
+		return 1;
+
+	return 0;
+}
+
+static int bpf_test_sockopt_int(void *ctx, struct sock *sk,
+				const struct sockopt_test *t,
+				int level)
+{
+	int old, tmp, new, expected, opt;
+
+	opt = t->opt;
+	new = t->new;
+	if (sk->sk_type == SOCK_STREAM && t->tcp_expected)
+		expected = t->tcp_expected;
+	else
+		expected = t->expected;
+
+	if (bpf_getsockopt(ctx, level, opt, &old, sizeof(old)) ||
+	    old == new)
+		return 1;
+
+	if (bpf_setsockopt(ctx, level, opt, &new, sizeof(new)))
+		return 1;
+	if (bpf_getsockopt(ctx, level, opt, &tmp, sizeof(tmp)) ||
+	    tmp != expected)
+		return 1;
+
+	if (t->restore)
+		old = t->restore;
+	if (bpf_setsockopt(ctx, level, opt, &old, sizeof(old)))
+		return 1;
+
+	return 0;
+}
+
+static int bpf_test_socket_sockopt(__u32 i, struct loop_ctx *lc)
+{
+	const struct sockopt_test *t;
+
+	if (i >= ARRAY_SIZE(sol_socket_tests))
+		return 1;
+
+	t = &sol_socket_tests[i];
+	if (!t->opt)
+		return 1;
+
+	if (t->flip)
+		return bpf_test_sockopt_flip(lc->ctx, lc->sk, t, SOL_SOCKET);
+
+	return bpf_test_sockopt_int(lc->ctx, lc->sk, t, SOL_SOCKET);
+}
+
+static int bpf_test_ip_sockopt(__u32 i, struct loop_ctx *lc)
+{
+	const struct sockopt_test *t;
+
+	if (i >= ARRAY_SIZE(sol_ip_tests))
+		return 1;
+
+	t = &sol_ip_tests[i];
+	if (!t->opt)
+		return 1;
+
+	if (t->flip)
+		return bpf_test_sockopt_flip(lc->ctx, lc->sk, t, IPPROTO_IP);
+
+	return bpf_test_sockopt_int(lc->ctx, lc->sk, t, IPPROTO_IP);
+}
+
+static int bpf_test_ipv6_sockopt(__u32 i, struct loop_ctx *lc)
+{
+	const struct sockopt_test *t;
+
+	if (i >= ARRAY_SIZE(sol_ipv6_tests))
+		return 1;
+
+	t = &sol_ipv6_tests[i];
+	if (!t->opt)
+		return 1;
+
+	if (t->flip)
+		return bpf_test_sockopt_flip(lc->ctx, lc->sk, t, IPPROTO_IPV6);
+
+	return bpf_test_sockopt_int(lc->ctx, lc->sk, t, IPPROTO_IPV6);
+}
+
+static int bpf_test_tcp_sockopt(__u32 i, struct loop_ctx *lc)
+{
+	const struct sockopt_test *t;
+	struct sock *sk;
+	void *ctx;
+
+	if (i >= ARRAY_SIZE(sol_tcp_tests))
+		return 1;
+
+	t = &sol_tcp_tests[i];
+	if (!t->opt)
+		return 1;
+
+	ctx = lc->ctx;
+	sk = lc->sk;
+
+	if (t->opt == TCP_CONGESTION) {
+		char old_cc[16], tmp_cc[16];
+		const char *new_cc;
+		int new_cc_len;
+
+		if (!bpf_setsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION,
+				    (void *)not_exist_cc, sizeof(not_exist_cc)))
+			return 1;
+		if (bpf_getsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, old_cc, sizeof(old_cc)))
+			return 1;
+		if (!bpf_strncmp(old_cc, sizeof(old_cc), cubic_cc)) {
+			new_cc = reno_cc;
+			new_cc_len = sizeof(reno_cc);
+		} else {
+			new_cc = cubic_cc;
+			new_cc_len = sizeof(cubic_cc);
+		}
+		if (bpf_setsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, (void *)new_cc,
+				   new_cc_len))
+			return 1;
+		if (bpf_getsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, tmp_cc, sizeof(tmp_cc)))
+			return 1;
+		if (bpf_strncmp(tmp_cc, sizeof(tmp_cc), new_cc))
+			return 1;
+		if (bpf_setsockopt(ctx, IPPROTO_TCP, TCP_CONGESTION, old_cc, sizeof(old_cc)))
+			return 1;
+		return 0;
+	}
+
+	if (t->flip)
+		return bpf_test_sockopt_flip(ctx, sk, t, IPPROTO_TCP);
+
+	return bpf_test_sockopt_int(ctx, sk, t, IPPROTO_TCP);
+}
+
+static int bpf_test_sockopt(void *ctx, struct sock *sk)
+{
+	struct loop_ctx lc = { .ctx = ctx, .sk = sk, };
+	__u16 family, proto;
+	int n;
+
+	family = sk->sk_family;
+	proto = sk->sk_protocol;
+
+	n = bpf_loop(ARRAY_SIZE(sol_socket_tests), bpf_test_socket_sockopt, &lc, 0);
+	if (n != ARRAY_SIZE(sol_socket_tests))
+		return -1;
+
+	if (proto == IPPROTO_TCP) {
+		n = bpf_loop(ARRAY_SIZE(sol_tcp_tests), bpf_test_tcp_sockopt, &lc, 0);
+		if (n != ARRAY_SIZE(sol_tcp_tests))
+			return -1;
+	}
+
+	if (family == AF_INET) {
+		n = bpf_loop(ARRAY_SIZE(sol_ip_tests), bpf_test_ip_sockopt, &lc, 0);
+		if (n != ARRAY_SIZE(sol_ip_tests))
+			return -1;
+	} else {
+		n = bpf_loop(ARRAY_SIZE(sol_ipv6_tests), bpf_test_ipv6_sockopt, &lc, 0);
+		if (n != ARRAY_SIZE(sol_ipv6_tests))
+			return -1;
+	}
+
+	return 0;
+}
+
+static int binddev_test(void *ctx)
+{
+	const char empty_ifname[] = "";
+	int ifindex, zero = 0;
+
+	if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
+			   (void *)veth, sizeof(veth)))
+		return -1;
+	if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
+			   &ifindex, sizeof(int)) ||
+	    ifindex != veth_ifindex)
+		return -1;
+
+	if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTODEVICE,
+			   (void *)empty_ifname, sizeof(empty_ifname)))
+		return -1;
+	if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
+			   &ifindex, sizeof(int)) ||
+	    ifindex != 0)
+		return -1;
+
+	if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
+			   (void *)&veth_ifindex, sizeof(int)))
+		return -1;
+	if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
+			   &ifindex, sizeof(int)) ||
+	    ifindex != veth_ifindex)
+		return -1;
+
+	if (bpf_setsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
+			   &zero, sizeof(int)))
+		return -1;
+	if (bpf_getsockopt(ctx, SOL_SOCKET, SO_BINDTOIFINDEX,
+			   &ifindex, sizeof(int)) ||
+	    ifindex != 0)
+		return -1;
+
+	return 0;
+}
+
+static int test_tcp_maxseg(void *ctx, struct sock *sk)
+{
+	int val = 1314, tmp;
+
+	if (sk->sk_state != TCP_ESTABLISHED)
+		return bpf_setsockopt(ctx, IPPROTO_TCP, TCP_MAXSEG,
+				      &val, sizeof(val));
+
+	if (bpf_getsockopt(ctx, IPPROTO_TCP, TCP_MAXSEG, &tmp, sizeof(tmp)) ||
+	    tmp > val)
+		return -1;
+
+	return 0;
+}
+
+static int test_tcp_saved_syn(void *ctx, struct sock *sk)
+{
+	__u8 saved_syn[20];
+	int one = 1;
+
+	if (sk->sk_state == TCP_LISTEN)
+		return bpf_setsockopt(ctx, IPPROTO_TCP, TCP_SAVE_SYN,
+				      &one, sizeof(one));
+
+	return bpf_getsockopt(ctx, IPPROTO_TCP, TCP_SAVED_SYN,
+			      saved_syn, sizeof(saved_syn));
+}
+
+SEC("lsm_cgroup/socket_post_create")
+int BPF_PROG(socket_post_create, struct socket *sock, int family,
+	     int type, int protocol, int kern)
+{
+	struct sock *sk = sock->sk;
+
+	if (!sk)
+		return 1;
+
+	nr_socket_post_create += !bpf_test_sockopt(sk, sk);
+	nr_binddev += !binddev_test(sk);
+
+	return 1;
+}
+
+SEC("sockops")
+int skops_sockopt(struct bpf_sock_ops *skops)
+{
+	struct bpf_sock *bpf_sk = skops->sk;
+	struct sock *sk;
+
+	if (!bpf_sk)
+		return 1;
+
+	sk = (struct sock *)bpf_skc_to_tcp_sock(bpf_sk);
+	if (!sk)
+		return 1;
+
+	switch (skops->op) {
+	case BPF_SOCK_OPS_TCP_LISTEN_CB:
+		nr_listen += !(bpf_test_sockopt(skops, sk) ||
+			       test_tcp_maxseg(skops, sk) ||
+			       test_tcp_saved_syn(skops, sk));
+		break;
+	case BPF_SOCK_OPS_TCP_CONNECT_CB:
+		nr_connect += !(bpf_test_sockopt(skops, sk) ||
+				test_tcp_maxseg(skops, sk));
+		break;
+	case BPF_SOCK_OPS_ACTIVE_ESTABLISHED_CB:
+		nr_active += !(bpf_test_sockopt(skops, sk) ||
+			       test_tcp_maxseg(skops, sk));
+		break;
+	case BPF_SOCK_OPS_PASSIVE_ESTABLISHED_CB:
+		nr_passive += !(bpf_test_sockopt(skops, sk) ||
+				test_tcp_maxseg(skops, sk) ||
+				test_tcp_saved_syn(skops, sk));
+		break;
+	}
+
+	return 1;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c
index b67e8022d500..a017d6b2f1dd 100644
--- a/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c
+++ b/tools/testing/selftests/bpf/progs/tailcall_bpf2bpf4.c
@@ -19,7 +19,7 @@ struct {
 int count = 0;
 int noise = 0;
 
-__always_inline int subprog_noise(void)
+static __always_inline int subprog_noise(void)
 {
 	__u32 key = 0;
 
diff --git a/tools/testing/selftests/bpf/progs/test_autoattach.c b/tools/testing/selftests/bpf/progs/test_autoattach.c
new file mode 100644
index 000000000000..11a44493ebce
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_autoattach.c
@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Google */
+
+#include "vmlinux.h"
+#include <bpf/bpf_tracing.h>
+
+bool prog1_called = false;
+bool prog2_called = false;
+
+SEC("raw_tp/sys_enter")
+int prog1(const void *ctx)
+{
+	prog1_called = true;
+	return 0;
+}
+
+SEC("raw_tp/sys_exit")
+int prog2(const void *ctx)
+{
+	prog2_called = true;
+	return 0;
+}
+
diff --git a/tools/testing/selftests/bpf/progs/test_bpf_cookie.c b/tools/testing/selftests/bpf/progs/test_bpf_cookie.c
index 22d0ac8709b4..5a3a80f751c4 100644
--- a/tools/testing/selftests/bpf/progs/test_bpf_cookie.c
+++ b/tools/testing/selftests/bpf/progs/test_bpf_cookie.c
@@ -28,14 +28,14 @@ static void update(void *ctx, __u64 *res)
 	*res |= bpf_get_attach_cookie(ctx);
 }
 
-SEC("kprobe/sys_nanosleep")
+SEC("kprobe")
 int handle_kprobe(struct pt_regs *ctx)
 {
 	update(ctx, &kprobe_res);
 	return 0;
 }
 
-SEC("kretprobe/sys_nanosleep")
+SEC("kretprobe")
 int handle_kretprobe(struct pt_regs *ctx)
 {
 	update(ctx, &kretprobe_res);
diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf.c b/tools/testing/selftests/bpf/progs/test_bpf_nf.c
index 196cd8dfe42a..227e85e85dda 100644
--- a/tools/testing/selftests/bpf/progs/test_bpf_nf.c
+++ b/tools/testing/selftests/bpf/progs/test_bpf_nf.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <vmlinux.h>
 #include <bpf/bpf_helpers.h>
+#include <bpf/bpf_endian.h>
 
 #define EAFNOSUPPORT 97
 #define EPROTO 71
@@ -23,6 +24,15 @@ int test_insert_entry = -EAFNOSUPPORT;
 int test_succ_lookup = -ENOENT;
 u32 test_delta_timeout = 0;
 u32 test_status = 0;
+u32 test_insert_lookup_mark = 0;
+int test_snat_addr = -EINVAL;
+int test_dnat_addr = -EINVAL;
+__be32 saddr = 0;
+__be16 sport = 0;
+__be32 daddr = 0;
+__be16 dport = 0;
+int test_exist_lookup = -ENOENT;
+u32 test_exist_lookup_mark = 0;
 
 struct nf_conn;
 
@@ -47,6 +57,8 @@ void bpf_ct_set_timeout(struct nf_conn *, u32) __ksym;
 int bpf_ct_change_timeout(struct nf_conn *, u32) __ksym;
 int bpf_ct_set_status(struct nf_conn *, u32) __ksym;
 int bpf_ct_change_status(struct nf_conn *, u32) __ksym;
+int bpf_ct_set_nat_info(struct nf_conn *, union nf_inet_addr *,
+			int port, enum nf_nat_manip_type) __ksym;
 
 static __always_inline void
 nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32,
@@ -134,10 +146,21 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32,
 	ct = alloc_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def,
 		      sizeof(opts_def));
 	if (ct) {
+		__u16 sport = bpf_get_prandom_u32();
+		__u16 dport = bpf_get_prandom_u32();
+		union nf_inet_addr saddr = {};
+		union nf_inet_addr daddr = {};
 		struct nf_conn *ct_ins;
 
 		bpf_ct_set_timeout(ct, 10000);
-		bpf_ct_set_status(ct, IPS_CONFIRMED);
+		ct->mark = 77;
+
+		/* snat */
+		saddr.ip = bpf_get_prandom_u32();
+		bpf_ct_set_nat_info(ct, &saddr, sport, NF_NAT_MANIP_SRC);
+		/* dnat */
+		daddr.ip = bpf_get_prandom_u32();
+		bpf_ct_set_nat_info(ct, &daddr, dport, NF_NAT_MANIP_DST);
 
 		ct_ins = bpf_ct_insert_entry(ct);
 		if (ct_ins) {
@@ -146,12 +169,26 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32,
 			ct_lk = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4),
 					  &opts_def, sizeof(opts_def));
 			if (ct_lk) {
+				struct nf_conntrack_tuple *tuple;
+
+				/* check snat and dnat addresses */
+				tuple = &ct_lk->tuplehash[IP_CT_DIR_REPLY].tuple;
+				if (tuple->dst.u3.ip == saddr.ip &&
+				    tuple->dst.u.all == bpf_htons(sport))
+					test_snat_addr = 0;
+				if (tuple->src.u3.ip == daddr.ip &&
+				    tuple->src.u.all == bpf_htons(dport))
+					test_dnat_addr = 0;
+
 				/* update ct entry timeout */
 				bpf_ct_change_timeout(ct_lk, 10000);
 				test_delta_timeout = ct_lk->timeout - bpf_jiffies64();
 				test_delta_timeout /= CONFIG_HZ;
-				test_status = IPS_SEEN_REPLY;
-				bpf_ct_change_status(ct_lk, IPS_SEEN_REPLY);
+				test_insert_lookup_mark = ct_lk->mark;
+				bpf_ct_change_status(ct_lk,
+						     IPS_CONFIRMED | IPS_SEEN_REPLY);
+				test_status = ct_lk->status;
+
 				bpf_ct_release(ct_lk);
 				test_succ_lookup = 0;
 			}
@@ -160,6 +197,23 @@ nf_ct_test(struct nf_conn *(*lookup_fn)(void *, struct bpf_sock_tuple *, u32,
 		}
 		test_alloc_entry = 0;
 	}
+
+	bpf_tuple.ipv4.saddr = saddr;
+	bpf_tuple.ipv4.daddr = daddr;
+	bpf_tuple.ipv4.sport = sport;
+	bpf_tuple.ipv4.dport = dport;
+	ct = lookup_fn(ctx, &bpf_tuple, sizeof(bpf_tuple.ipv4), &opts_def,
+		       sizeof(opts_def));
+	if (ct) {
+		test_exist_lookup = 0;
+		if (ct->mark == 42) {
+			ct->mark++;
+			test_exist_lookup_mark = ct->mark;
+		}
+		bpf_ct_release(ct);
+	} else {
+		test_exist_lookup = opts_def.error;
+	}
 }
 
 SEC("xdp")
diff --git a/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c b/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c
index bf79af15c808..0e4759ab38ff 100644
--- a/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c
+++ b/tools/testing/selftests/bpf/progs/test_bpf_nf_fail.c
@@ -70,6 +70,20 @@ int lookup_insert(struct __sk_buff *ctx)
 }
 
 SEC("?tc")
+int write_not_allowlisted_field(struct __sk_buff *ctx)
+{
+	struct bpf_ct_opts___local opts = {};
+	struct bpf_sock_tuple tup = {};
+	struct nf_conn *ct;
+
+	ct = bpf_skb_ct_lookup(ctx, &tup, sizeof(tup.ipv4), &opts, sizeof(opts));
+	if (!ct)
+		return 0;
+	ct->status = 0xF00;
+	return 0;
+}
+
+SEC("?tc")
 int set_timeout_after_insert(struct __sk_buff *ctx)
 {
 	struct bpf_ct_opts___local opts = {};
diff --git a/tools/testing/selftests/bpf/progs/test_deny_namespace.c b/tools/testing/selftests/bpf/progs/test_deny_namespace.c
new file mode 100644
index 000000000000..09ad5a4ebd1f
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_deny_namespace.c
@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+#include <errno.h>
+#include <linux/capability.h>
+
+struct kernel_cap_struct {
+	__u32 cap[_LINUX_CAPABILITY_U32S_3];
+} __attribute__((preserve_access_index));
+
+struct cred {
+	struct kernel_cap_struct cap_effective;
+} __attribute__((preserve_access_index));
+
+char _license[] SEC("license") = "GPL";
+
+SEC("lsm.s/userns_create")
+int BPF_PROG(test_userns_create, const struct cred *cred, int ret)
+{
+	struct kernel_cap_struct caps = cred->cap_effective;
+	int cap_index = CAP_TO_INDEX(CAP_SYS_ADMIN);
+	__u32 cap_mask = CAP_TO_MASK(CAP_SYS_ADMIN);
+
+	if (ret)
+		return 0;
+
+	ret = -EPERM;
+	if (caps.cap[cap_index] & cap_mask)
+		return 0;
+
+	return -EPERM;
+}
diff --git a/tools/testing/selftests/bpf/progs/test_helper_restricted.c b/tools/testing/selftests/bpf/progs/test_helper_restricted.c
index 20ef9d433b97..5715c569ec03 100644
--- a/tools/testing/selftests/bpf/progs/test_helper_restricted.c
+++ b/tools/testing/selftests/bpf/progs/test_helper_restricted.c
@@ -72,7 +72,7 @@ int tp_timer(void *ctx)
 	return 0;
 }
 
-SEC("?kprobe/sys_nanosleep")
+SEC("?kprobe")
 int kprobe_timer(void *ctx)
 {
 	timer_work();
@@ -104,7 +104,7 @@ int tp_spin_lock(void *ctx)
 	return 0;
 }
 
-SEC("?kprobe/sys_nanosleep")
+SEC("?kprobe")
 int kprobe_spin_lock(void *ctx)
 {
 	spin_lock_work();
diff --git a/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c
new file mode 100644
index 000000000000..ce39d096bba3
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_kfunc_dynptr_param.c
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH
+ *
+ * Author: Roberto Sassu <roberto.sassu@huawei.com>
+ */
+
+#include "vmlinux.h"
+#include <errno.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym;
+extern void bpf_key_put(struct bpf_key *key) __ksym;
+extern int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_ptr,
+				      struct bpf_dynptr *sig_ptr,
+				      struct bpf_key *trusted_keyring) __ksym;
+
+struct {
+	__uint(type, BPF_MAP_TYPE_RINGBUF);
+} ringbuf SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, __u32);
+	__type(value, __u32);
+} array_map SEC(".maps");
+
+int err, pid;
+
+char _license[] SEC("license") = "GPL";
+
+SEC("?lsm.s/bpf")
+int BPF_PROG(dynptr_type_not_supp, int cmd, union bpf_attr *attr,
+	     unsigned int size)
+{
+	char write_data[64] = "hello there, world!!";
+	struct bpf_dynptr ptr;
+
+	bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(write_data), 0, &ptr);
+
+	return bpf_verify_pkcs7_signature(&ptr, &ptr, NULL);
+}
+
+SEC("?lsm.s/bpf")
+int BPF_PROG(not_valid_dynptr, int cmd, union bpf_attr *attr, unsigned int size)
+{
+	unsigned long val;
+
+	return bpf_verify_pkcs7_signature((struct bpf_dynptr *)&val,
+					  (struct bpf_dynptr *)&val, NULL);
+}
+
+SEC("?lsm.s/bpf")
+int BPF_PROG(not_ptr_to_stack, int cmd, union bpf_attr *attr, unsigned int size)
+{
+	unsigned long val;
+
+	return bpf_verify_pkcs7_signature((struct bpf_dynptr *)val,
+					  (struct bpf_dynptr *)val, NULL);
+}
+
+SEC("lsm.s/bpf")
+int BPF_PROG(dynptr_data_null, int cmd, union bpf_attr *attr, unsigned int size)
+{
+	struct bpf_key *trusted_keyring;
+	struct bpf_dynptr ptr;
+	__u32 *value;
+	int ret, zero = 0;
+
+	if (bpf_get_current_pid_tgid() >> 32 != pid)
+		return 0;
+
+	value = bpf_map_lookup_elem(&array_map, &zero);
+	if (!value)
+		return 0;
+
+	/* Pass invalid flags. */
+	ret = bpf_dynptr_from_mem(value, sizeof(*value), ((__u64)~0ULL), &ptr);
+	if (ret != -EINVAL)
+		return 0;
+
+	trusted_keyring = bpf_lookup_system_key(0);
+	if (!trusted_keyring)
+		return 0;
+
+	err = bpf_verify_pkcs7_signature(&ptr, &ptr, trusted_keyring);
+
+	bpf_key_put(trusted_keyring);
+
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/test_lookup_key.c b/tools/testing/selftests/bpf/progs/test_lookup_key.c
new file mode 100644
index 000000000000..c73776990ae3
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_lookup_key.c
@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH
+ *
+ * Author: Roberto Sassu <roberto.sassu@huawei.com>
+ */
+
+#include "vmlinux.h"
+#include <errno.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+char _license[] SEC("license") = "GPL";
+
+__u32 monitored_pid;
+__u32 key_serial;
+__u32 key_id;
+__u64 flags;
+
+extern struct bpf_key *bpf_lookup_user_key(__u32 serial, __u64 flags) __ksym;
+extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym;
+extern void bpf_key_put(struct bpf_key *key) __ksym;
+
+SEC("lsm.s/bpf")
+int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size)
+{
+	struct bpf_key *bkey;
+	__u32 pid;
+
+	pid = bpf_get_current_pid_tgid() >> 32;
+	if (pid != monitored_pid)
+		return 0;
+
+	if (key_serial)
+		bkey = bpf_lookup_user_key(key_serial, flags);
+	else
+		bkey = bpf_lookup_system_key(key_id);
+
+	if (!bkey)
+		return -ENOENT;
+
+	bpf_key_put(bkey);
+
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/test_tc_dtime.c b/tools/testing/selftests/bpf/progs/test_tc_dtime.c
index b596479a9ebe..125beec31834 100644
--- a/tools/testing/selftests/bpf/progs/test_tc_dtime.c
+++ b/tools/testing/selftests/bpf/progs/test_tc_dtime.c
@@ -15,7 +15,6 @@
 #include <linux/udp.h>
 #include <bpf/bpf_helpers.h>
 #include <bpf/bpf_endian.h>
-#include <sys/socket.h>
 
 /* veth_src --- veth_src_fwd --- veth_det_fwd --- veth_dst
  *           |                                 |
diff --git a/tools/testing/selftests/bpf/progs/test_time_tai.c b/tools/testing/selftests/bpf/progs/test_time_tai.c
new file mode 100644
index 000000000000..7ea0863f3ddb
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_time_tai.c
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (C) 2022 Linutronix GmbH */
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+
+char _license[] SEC("license") = "GPL";
+
+SEC("tc")
+int time_tai(struct __sk_buff *skb)
+{
+	__u64 ts1, ts2;
+
+	/* Get TAI timestamps */
+	ts1 = bpf_ktime_get_tai_ns();
+	ts2 = bpf_ktime_get_tai_ns();
+
+	/* Save TAI timestamps (Note: skb->hwtstamp is read-only) */
+	skb->tstamp = ts1;
+	skb->cb[0] = ts2 & 0xffffffff;
+	skb->cb[1] = ts2 >> 32;
+
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/test_tunnel_kern.c b/tools/testing/selftests/bpf/progs/test_tunnel_kern.c
index df0673c4ecbe..98af55f0bcd3 100644
--- a/tools/testing/selftests/bpf/progs/test_tunnel_kern.c
+++ b/tools/testing/selftests/bpf/progs/test_tunnel_kern.c
@@ -12,6 +12,7 @@
 #include <linux/bpf.h>
 #include <linux/if_ether.h>
 #include <linux/if_packet.h>
+#include <linux/if_tunnel.h>
 #include <linux/ip.h>
 #include <linux/ipv6.h>
 #include <linux/icmp.h>
@@ -386,7 +387,8 @@ int vxlan_get_tunnel_src(struct __sk_buff *skb)
 	__u32 orig_daddr;
 	__u32 index = 0;
 
-	ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key), 0);
+	ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key),
+				     BPF_F_TUNINFO_FLAGS);
 	if (ret < 0) {
 		log_err(ret);
 		return TC_ACT_SHOT;
@@ -398,10 +400,13 @@ int vxlan_get_tunnel_src(struct __sk_buff *skb)
 		return TC_ACT_SHOT;
 	}
 
-	if (key.local_ipv4 != ASSIGNED_ADDR_VETH1 || md.gbp != 0x800FF) {
-		bpf_printk("vxlan key %d local ip 0x%x remote ip 0x%x gbp 0x%x\n",
+	if (key.local_ipv4 != ASSIGNED_ADDR_VETH1 || md.gbp != 0x800FF ||
+	    !(key.tunnel_flags & TUNNEL_KEY) ||
+	    (key.tunnel_flags & TUNNEL_CSUM)) {
+		bpf_printk("vxlan key %d local ip 0x%x remote ip 0x%x gbp 0x%x flags 0x%x\n",
 			   key.tunnel_id, key.local_ipv4,
-			   key.remote_ipv4, md.gbp);
+			   key.remote_ipv4, md.gbp,
+			   bpf_ntohs(key.tunnel_flags));
 		log_err(ret);
 		return TC_ACT_SHOT;
 	}
@@ -541,16 +546,19 @@ int ip6vxlan_get_tunnel_src(struct __sk_buff *skb)
 	}
 
 	ret = bpf_skb_get_tunnel_key(skb, &key, sizeof(key),
-				     BPF_F_TUNINFO_IPV6);
+				     BPF_F_TUNINFO_IPV6 | BPF_F_TUNINFO_FLAGS);
 	if (ret < 0) {
 		log_err(ret);
 		return TC_ACT_SHOT;
 	}
 
-	if (bpf_ntohl(key.local_ipv6[3]) != *local_ip) {
-		bpf_printk("ip6vxlan key %d local ip6 ::%x remote ip6 ::%x label 0x%x\n",
+	if (bpf_ntohl(key.local_ipv6[3]) != *local_ip ||
+	    !(key.tunnel_flags & TUNNEL_KEY) ||
+	    !(key.tunnel_flags & TUNNEL_CSUM)) {
+		bpf_printk("ip6vxlan key %d local ip6 ::%x remote ip6 ::%x label 0x%x flags 0x%x\n",
 			   key.tunnel_id, bpf_ntohl(key.local_ipv6[3]),
-			   bpf_ntohl(key.remote_ipv6[3]), key.tunnel_label);
+			   bpf_ntohl(key.remote_ipv6[3]), key.tunnel_label,
+			   bpf_ntohs(key.tunnel_flags));
 		bpf_printk("local_ip 0x%x\n", *local_ip);
 		log_err(ret);
 		return TC_ACT_SHOT;
diff --git a/tools/testing/selftests/bpf/progs/test_user_ringbuf.h b/tools/testing/selftests/bpf/progs/test_user_ringbuf.h
new file mode 100644
index 000000000000..1643b4d59ba7
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_user_ringbuf.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
+
+#ifndef _TEST_USER_RINGBUF_H
+#define _TEST_USER_RINGBUF_H
+
+#define TEST_OP_64 4
+#define TEST_OP_32 2
+
+enum test_msg_op {
+	TEST_MSG_OP_INC64,
+	TEST_MSG_OP_INC32,
+	TEST_MSG_OP_MUL64,
+	TEST_MSG_OP_MUL32,
+
+	// Must come last.
+	TEST_MSG_OP_NUM_OPS,
+};
+
+struct test_msg {
+	enum test_msg_op msg_op;
+	union {
+		__s64 operand_64;
+		__s32 operand_32;
+	};
+};
+
+struct sample {
+	int pid;
+	int seq;
+	long value;
+	char comm[16];
+};
+
+#endif /* _TEST_USER_RINGBUF_H */
diff --git a/tools/testing/selftests/bpf/progs/test_verif_scale1.c b/tools/testing/selftests/bpf/progs/test_verif_scale1.c
index d38153dab3dd..ac6135d9374c 100644
--- a/tools/testing/selftests/bpf/progs/test_verif_scale1.c
+++ b/tools/testing/selftests/bpf/progs/test_verif_scale1.c
@@ -5,7 +5,7 @@
 #define ATTR __attribute__((noinline))
 #include "test_jhash.h"
 
-SEC("scale90_noinline")
+SEC("tc")
 int balancer_ingress(struct __sk_buff *ctx)
 {
 	void *data_end = (void *)(long)ctx->data_end;
diff --git a/tools/testing/selftests/bpf/progs/test_verif_scale3.c b/tools/testing/selftests/bpf/progs/test_verif_scale3.c
index 9beb5bf80373..ca33a9b711c4 100644
--- a/tools/testing/selftests/bpf/progs/test_verif_scale3.c
+++ b/tools/testing/selftests/bpf/progs/test_verif_scale3.c
@@ -5,7 +5,7 @@
 #define ATTR __attribute__((noinline))
 #include "test_jhash.h"
 
-SEC("scale90_noinline32")
+SEC("tc")
 int balancer_ingress(struct __sk_buff *ctx)
 {
 	void *data_end = (void *)(long)ctx->data_end;
diff --git a/tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c b/tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c
new file mode 100644
index 000000000000..ce419304ff1f
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_verify_pkcs7_sig.c
@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/*
+ * Copyright (C) 2022 Huawei Technologies Duesseldorf GmbH
+ *
+ * Author: Roberto Sassu <roberto.sassu@huawei.com>
+ */
+
+#include "vmlinux.h"
+#include <errno.h>
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+#define MAX_DATA_SIZE (1024 * 1024)
+#define MAX_SIG_SIZE 1024
+
+extern struct bpf_key *bpf_lookup_user_key(__u32 serial, __u64 flags) __ksym;
+extern struct bpf_key *bpf_lookup_system_key(__u64 id) __ksym;
+extern void bpf_key_put(struct bpf_key *key) __ksym;
+extern int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_ptr,
+				      struct bpf_dynptr *sig_ptr,
+				      struct bpf_key *trusted_keyring) __ksym;
+
+__u32 monitored_pid;
+__u32 user_keyring_serial;
+__u64 system_keyring_id;
+
+struct data {
+	__u8 data[MAX_DATA_SIZE];
+	__u32 data_len;
+	__u8 sig[MAX_SIG_SIZE];
+	__u32 sig_len;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, __u32);
+	__type(value, struct data);
+} data_input SEC(".maps");
+
+char _license[] SEC("license") = "GPL";
+
+SEC("lsm.s/bpf")
+int BPF_PROG(bpf, int cmd, union bpf_attr *attr, unsigned int size)
+{
+	struct bpf_dynptr data_ptr, sig_ptr;
+	struct data *data_val;
+	struct bpf_key *trusted_keyring;
+	__u32 pid;
+	__u64 value;
+	int ret, zero = 0;
+
+	pid = bpf_get_current_pid_tgid() >> 32;
+	if (pid != monitored_pid)
+		return 0;
+
+	data_val = bpf_map_lookup_elem(&data_input, &zero);
+	if (!data_val)
+		return 0;
+
+	bpf_probe_read(&value, sizeof(value), &attr->value);
+
+	bpf_copy_from_user(data_val, sizeof(struct data),
+			   (void *)(unsigned long)value);
+
+	if (data_val->data_len > sizeof(data_val->data))
+		return -EINVAL;
+
+	bpf_dynptr_from_mem(data_val->data, data_val->data_len, 0, &data_ptr);
+
+	if (data_val->sig_len > sizeof(data_val->sig))
+		return -EINVAL;
+
+	bpf_dynptr_from_mem(data_val->sig, data_val->sig_len, 0, &sig_ptr);
+
+	if (user_keyring_serial)
+		trusted_keyring = bpf_lookup_user_key(user_keyring_serial, 0);
+	else
+		trusted_keyring = bpf_lookup_system_key(system_keyring_id);
+
+	if (!trusted_keyring)
+		return -ENOENT;
+
+	ret = bpf_verify_pkcs7_signature(&data_ptr, &sig_ptr, trusted_keyring);
+
+	bpf_key_put(trusted_keyring);
+
+	return ret;
+}
diff --git a/tools/testing/selftests/bpf/progs/timer.c b/tools/testing/selftests/bpf/progs/timer.c
index 5f5309791649..acda5c9cea93 100644
--- a/tools/testing/selftests/bpf/progs/timer.c
+++ b/tools/testing/selftests/bpf/progs/timer.c
@@ -120,7 +120,7 @@ static int timer_cb1(void *map, int *key, struct bpf_timer *timer)
 }
 
 SEC("fentry/bpf_fentry_test1")
-int BPF_PROG(test1, int a)
+int BPF_PROG2(test1, int, a)
 {
 	struct bpf_timer *arr_timer, *lru_timer;
 	struct elem init = {};
@@ -208,17 +208,6 @@ static int timer_cb2(void *map, int *key, struct hmap_elem *val)
 		 */
 		bpf_map_delete_elem(map, key);
 
-		/* in non-preallocated hashmap both 'key' and 'val' are RCU
-		 * protected and still valid though this element was deleted
-		 * from the map. Arm this timer for ~35 seconds. When callback
-		 * finishes the call_rcu will invoke:
-		 * htab_elem_free_rcu
-		 *   check_and_free_timer
-		 *     bpf_timer_cancel_and_free
-		 * to cancel this 35 second sleep and delete the timer for real.
-		 */
-		if (bpf_timer_start(&val->timer, 1ull << 35, 0) != 0)
-			err |= 256;
 		ok |= 4;
 	}
 	return 0;
@@ -247,7 +236,7 @@ int bpf_timer_test(void)
 }
 
 SEC("fentry/bpf_fentry_test2")
-int BPF_PROG(test2, int a, int b)
+int BPF_PROG2(test2, int, a, int, b)
 {
 	struct hmap_elem init = {}, *val;
 	int key = HTAB, key_malloc = HTAB_MALLOC;
diff --git a/tools/testing/selftests/bpf/progs/tracing_struct.c b/tools/testing/selftests/bpf/progs/tracing_struct.c
new file mode 100644
index 000000000000..e718f0ebee7d
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/tracing_struct.c
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
+
+#include <vmlinux.h>
+#include <bpf/bpf_tracing.h>
+#include <bpf/bpf_helpers.h>
+
+struct bpf_testmod_struct_arg_1 {
+	int a;
+};
+struct bpf_testmod_struct_arg_2 {
+	long a;
+	long b;
+};
+
+long t1_a_a, t1_a_b, t1_b, t1_c, t1_ret, t1_nregs;
+__u64 t1_reg0, t1_reg1, t1_reg2, t1_reg3;
+long t2_a, t2_b_a, t2_b_b, t2_c, t2_ret;
+long t3_a, t3_b, t3_c_a, t3_c_b, t3_ret;
+long t4_a_a, t4_b, t4_c, t4_d, t4_e_a, t4_e_b, t4_ret;
+long t5_ret;
+
+SEC("fentry/bpf_testmod_test_struct_arg_1")
+int BPF_PROG2(test_struct_arg_1, struct bpf_testmod_struct_arg_2, a, int, b, int, c)
+{
+	t1_a_a = a.a;
+	t1_a_b = a.b;
+	t1_b = b;
+	t1_c = c;
+	return 0;
+}
+
+SEC("fexit/bpf_testmod_test_struct_arg_1")
+int BPF_PROG2(test_struct_arg_2, struct bpf_testmod_struct_arg_2, a, int, b, int, c, int, ret)
+{
+	t1_nregs =  bpf_get_func_arg_cnt(ctx);
+	/* a.a */
+	bpf_get_func_arg(ctx, 0, &t1_reg0);
+	/* a.b */
+	bpf_get_func_arg(ctx, 1, &t1_reg1);
+	/* b */
+	bpf_get_func_arg(ctx, 2, &t1_reg2);
+	t1_reg2 = (int)t1_reg2;
+	/* c */
+	bpf_get_func_arg(ctx, 3, &t1_reg3);
+	t1_reg3 = (int)t1_reg3;
+
+	t1_ret = ret;
+	return 0;
+}
+
+SEC("fentry/bpf_testmod_test_struct_arg_2")
+int BPF_PROG2(test_struct_arg_3, int, a, struct bpf_testmod_struct_arg_2, b, int, c)
+{
+	t2_a = a;
+	t2_b_a = b.a;
+	t2_b_b = b.b;
+	t2_c = c;
+	return 0;
+}
+
+SEC("fexit/bpf_testmod_test_struct_arg_2")
+int BPF_PROG2(test_struct_arg_4, int, a, struct bpf_testmod_struct_arg_2, b, int, c, int, ret)
+{
+	t2_ret = ret;
+	return 0;
+}
+
+SEC("fentry/bpf_testmod_test_struct_arg_3")
+int BPF_PROG2(test_struct_arg_5, int, a, int, b, struct bpf_testmod_struct_arg_2, c)
+{
+	t3_a = a;
+	t3_b = b;
+	t3_c_a = c.a;
+	t3_c_b = c.b;
+	return 0;
+}
+
+SEC("fexit/bpf_testmod_test_struct_arg_3")
+int BPF_PROG2(test_struct_arg_6, int, a, int, b, struct bpf_testmod_struct_arg_2, c, int, ret)
+{
+	t3_ret = ret;
+	return 0;
+}
+
+SEC("fentry/bpf_testmod_test_struct_arg_4")
+int BPF_PROG2(test_struct_arg_7, struct bpf_testmod_struct_arg_1, a, int, b,
+	     int, c, int, d, struct bpf_testmod_struct_arg_2, e)
+{
+	t4_a_a = a.a;
+	t4_b = b;
+	t4_c = c;
+	t4_d = d;
+	t4_e_a = e.a;
+	t4_e_b = e.b;
+	return 0;
+}
+
+SEC("fexit/bpf_testmod_test_struct_arg_4")
+int BPF_PROG2(test_struct_arg_8, struct bpf_testmod_struct_arg_1, a, int, b,
+	     int, c, int, d, struct bpf_testmod_struct_arg_2, e, int, ret)
+{
+	t4_ret = ret;
+	return 0;
+}
+
+SEC("fentry/bpf_testmod_test_struct_arg_5")
+int BPF_PROG2(test_struct_arg_9)
+{
+	return 0;
+}
+
+SEC("fexit/bpf_testmod_test_struct_arg_5")
+int BPF_PROG2(test_struct_arg_10, int, ret)
+{
+	t5_ret = ret;
+	return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c b/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c
new file mode 100644
index 000000000000..82aba4529aa9
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/user_ringbuf_fail.c
@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+char _license[] SEC("license") = "GPL";
+
+struct sample {
+	int pid;
+	int seq;
+	long value;
+	char comm[16];
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_USER_RINGBUF);
+} user_ringbuf SEC(".maps");
+
+static long
+bad_access1(struct bpf_dynptr *dynptr, void *context)
+{
+	const struct sample *sample;
+
+	sample = bpf_dynptr_data(dynptr - 1, 0, sizeof(*sample));
+	bpf_printk("Was able to pass bad pointer %lx\n", (__u64)dynptr - 1);
+
+	return 0;
+}
+
+/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should
+ * not be able to read before the pointer.
+ */
+SEC("?raw_tp/sys_nanosleep")
+int user_ringbuf_callback_bad_access1(void *ctx)
+{
+	bpf_user_ringbuf_drain(&user_ringbuf, bad_access1, NULL, 0);
+
+	return 0;
+}
+
+static long
+bad_access2(struct bpf_dynptr *dynptr, void *context)
+{
+	const struct sample *sample;
+
+	sample = bpf_dynptr_data(dynptr + 1, 0, sizeof(*sample));
+	bpf_printk("Was able to pass bad pointer %lx\n", (__u64)dynptr + 1);
+
+	return 0;
+}
+
+/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should
+ * not be able to read past the end of the pointer.
+ */
+SEC("?raw_tp/sys_nanosleep")
+int user_ringbuf_callback_bad_access2(void *ctx)
+{
+	bpf_user_ringbuf_drain(&user_ringbuf, bad_access2, NULL, 0);
+
+	return 0;
+}
+
+static long
+write_forbidden(struct bpf_dynptr *dynptr, void *context)
+{
+	*((long *)dynptr) = 0;
+
+	return 0;
+}
+
+/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should
+ * not be able to write to that pointer.
+ */
+SEC("?raw_tp/sys_nanosleep")
+int user_ringbuf_callback_write_forbidden(void *ctx)
+{
+	bpf_user_ringbuf_drain(&user_ringbuf, write_forbidden, NULL, 0);
+
+	return 0;
+}
+
+static long
+null_context_write(struct bpf_dynptr *dynptr, void *context)
+{
+	*((__u64 *)context) = 0;
+
+	return 0;
+}
+
+/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should
+ * not be able to write to that pointer.
+ */
+SEC("?raw_tp/sys_nanosleep")
+int user_ringbuf_callback_null_context_write(void *ctx)
+{
+	bpf_user_ringbuf_drain(&user_ringbuf, null_context_write, NULL, 0);
+
+	return 0;
+}
+
+static long
+null_context_read(struct bpf_dynptr *dynptr, void *context)
+{
+	__u64 id = *((__u64 *)context);
+
+	bpf_printk("Read id %lu\n", id);
+
+	return 0;
+}
+
+/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should
+ * not be able to write to that pointer.
+ */
+SEC("?raw_tp/sys_nanosleep")
+int user_ringbuf_callback_null_context_read(void *ctx)
+{
+	bpf_user_ringbuf_drain(&user_ringbuf, null_context_read, NULL, 0);
+
+	return 0;
+}
+
+static long
+try_discard_dynptr(struct bpf_dynptr *dynptr, void *context)
+{
+	bpf_ringbuf_discard_dynptr(dynptr, 0);
+
+	return 0;
+}
+
+/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should
+ * not be able to read past the end of the pointer.
+ */
+SEC("?raw_tp/sys_nanosleep")
+int user_ringbuf_callback_discard_dynptr(void *ctx)
+{
+	bpf_user_ringbuf_drain(&user_ringbuf, try_discard_dynptr, NULL, 0);
+
+	return 0;
+}
+
+static long
+try_submit_dynptr(struct bpf_dynptr *dynptr, void *context)
+{
+	bpf_ringbuf_submit_dynptr(dynptr, 0);
+
+	return 0;
+}
+
+/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should
+ * not be able to read past the end of the pointer.
+ */
+SEC("?raw_tp/sys_nanosleep")
+int user_ringbuf_callback_submit_dynptr(void *ctx)
+{
+	bpf_user_ringbuf_drain(&user_ringbuf, try_submit_dynptr, NULL, 0);
+
+	return 0;
+}
+
+static long
+invalid_drain_callback_return(struct bpf_dynptr *dynptr, void *context)
+{
+	return 2;
+}
+
+/* A callback that accesses a dynptr in a bpf_user_ringbuf_drain callback should
+ * not be able to write to that pointer.
+ */
+SEC("?raw_tp/sys_nanosleep")
+int user_ringbuf_callback_invalid_return(void *ctx)
+{
+	bpf_user_ringbuf_drain(&user_ringbuf, invalid_drain_callback_return, NULL, 0);
+
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/progs/user_ringbuf_success.c b/tools/testing/selftests/bpf/progs/user_ringbuf_success.c
new file mode 100644
index 000000000000..099c23d9aa21
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/user_ringbuf_success.c
@@ -0,0 +1,218 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
+
+#include <linux/bpf.h>
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+#include "test_user_ringbuf.h"
+
+char _license[] SEC("license") = "GPL";
+
+struct {
+	__uint(type, BPF_MAP_TYPE_USER_RINGBUF);
+} user_ringbuf SEC(".maps");
+
+struct {
+	__uint(type, BPF_MAP_TYPE_RINGBUF);
+} kernel_ringbuf SEC(".maps");
+
+/* inputs */
+int pid, err, val;
+
+int read = 0;
+
+/* Counter used for end-to-end protocol test */
+__u64 kern_mutated = 0;
+__u64 user_mutated = 0;
+__u64 expected_user_mutated = 0;
+
+static int
+is_test_process(void)
+{
+	int cur_pid = bpf_get_current_pid_tgid() >> 32;
+
+	return cur_pid == pid;
+}
+
+static long
+record_sample(struct bpf_dynptr *dynptr, void *context)
+{
+	const struct sample *sample = NULL;
+	struct sample stack_sample;
+	int status;
+	static int num_calls;
+
+	if (num_calls++ % 2 == 0) {
+		status = bpf_dynptr_read(&stack_sample, sizeof(stack_sample), dynptr, 0, 0);
+		if (status) {
+			bpf_printk("bpf_dynptr_read() failed: %d\n", status);
+			err = 1;
+			return 0;
+		}
+	} else {
+		sample = bpf_dynptr_data(dynptr, 0, sizeof(*sample));
+		if (!sample) {
+			bpf_printk("Unexpectedly failed to get sample\n");
+			err = 2;
+			return 0;
+		}
+		stack_sample = *sample;
+	}
+
+	__sync_fetch_and_add(&read, 1);
+	return 0;
+}
+
+static void
+handle_sample_msg(const struct test_msg *msg)
+{
+	switch (msg->msg_op) {
+	case TEST_MSG_OP_INC64:
+		kern_mutated += msg->operand_64;
+		break;
+	case TEST_MSG_OP_INC32:
+		kern_mutated += msg->operand_32;
+		break;
+	case TEST_MSG_OP_MUL64:
+		kern_mutated *= msg->operand_64;
+		break;
+	case TEST_MSG_OP_MUL32:
+		kern_mutated *= msg->operand_32;
+		break;
+	default:
+		bpf_printk("Unrecognized op %d\n", msg->msg_op);
+		err = 2;
+	}
+}
+
+static long
+read_protocol_msg(struct bpf_dynptr *dynptr, void *context)
+{
+	const struct test_msg *msg = NULL;
+
+	msg = bpf_dynptr_data(dynptr, 0, sizeof(*msg));
+	if (!msg) {
+		err = 1;
+		bpf_printk("Unexpectedly failed to get msg\n");
+		return 0;
+	}
+
+	handle_sample_msg(msg);
+
+	return 0;
+}
+
+static int publish_next_kern_msg(__u32 index, void *context)
+{
+	struct test_msg *msg = NULL;
+	int operand_64 = TEST_OP_64;
+	int operand_32 = TEST_OP_32;
+
+	msg = bpf_ringbuf_reserve(&kernel_ringbuf, sizeof(*msg), 0);
+	if (!msg) {
+		err = 4;
+		return 1;
+	}
+
+	switch (index % TEST_MSG_OP_NUM_OPS) {
+	case TEST_MSG_OP_INC64:
+		msg->operand_64 = operand_64;
+		msg->msg_op = TEST_MSG_OP_INC64;
+		expected_user_mutated += operand_64;
+		break;
+	case TEST_MSG_OP_INC32:
+		msg->operand_32 = operand_32;
+		msg->msg_op = TEST_MSG_OP_INC32;
+		expected_user_mutated += operand_32;
+		break;
+	case TEST_MSG_OP_MUL64:
+		msg->operand_64 = operand_64;
+		msg->msg_op = TEST_MSG_OP_MUL64;
+		expected_user_mutated *= operand_64;
+		break;
+	case TEST_MSG_OP_MUL32:
+		msg->operand_32 = operand_32;
+		msg->msg_op = TEST_MSG_OP_MUL32;
+		expected_user_mutated *= operand_32;
+		break;
+	default:
+		bpf_ringbuf_discard(msg, 0);
+		err = 5;
+		return 1;
+	}
+
+	bpf_ringbuf_submit(msg, 0);
+
+	return 0;
+}
+
+static void
+publish_kern_messages(void)
+{
+	if (expected_user_mutated != user_mutated) {
+		bpf_printk("%lu != %lu\n", expected_user_mutated, user_mutated);
+		err = 3;
+		return;
+	}
+
+	bpf_loop(8, publish_next_kern_msg, NULL, 0);
+}
+
+SEC("fentry/" SYS_PREFIX "sys_prctl")
+int test_user_ringbuf_protocol(void *ctx)
+{
+	long status = 0;
+	struct sample *sample = NULL;
+	struct bpf_dynptr ptr;
+
+	if (!is_test_process())
+		return 0;
+
+	status = bpf_user_ringbuf_drain(&user_ringbuf, read_protocol_msg, NULL, 0);
+	if (status < 0) {
+		bpf_printk("Drain returned: %ld\n", status);
+		err = 1;
+		return 0;
+	}
+
+	publish_kern_messages();
+
+	return 0;
+}
+
+SEC("fentry/" SYS_PREFIX "sys_getpgid")
+int test_user_ringbuf(void *ctx)
+{
+	int status = 0;
+	struct sample *sample = NULL;
+	struct bpf_dynptr ptr;
+
+	if (!is_test_process())
+		return 0;
+
+	err = bpf_user_ringbuf_drain(&user_ringbuf, record_sample, NULL, 0);
+
+	return 0;
+}
+
+static long
+do_nothing_cb(struct bpf_dynptr *dynptr, void *context)
+{
+	__sync_fetch_and_add(&read, 1);
+	return 0;
+}
+
+SEC("fentry/" SYS_PREFIX "sys_getrlimit")
+int test_user_ringbuf_epoll(void *ctx)
+{
+	long num_samples;
+
+	if (!is_test_process())
+		return 0;
+
+	num_samples = bpf_user_ringbuf_drain(&user_ringbuf, do_nothing_cb, NULL, 0);
+	if (num_samples <= 0)
+		err = 1;
+
+	return 0;
+}
diff --git a/tools/testing/selftests/bpf/task_local_storage_helpers.h b/tools/testing/selftests/bpf/task_local_storage_helpers.h
new file mode 100644
index 000000000000..711d5abb7d51
--- /dev/null
+++ b/tools/testing/selftests/bpf/task_local_storage_helpers.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __TASK_LOCAL_STORAGE_HELPER_H
+#define __TASK_LOCAL_STORAGE_HELPER_H
+
+#include <unistd.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+
+#ifndef __NR_pidfd_open
+#define __NR_pidfd_open 434
+#endif
+
+static inline int sys_pidfd_open(pid_t pid, unsigned int flags)
+{
+	return syscall(__NR_pidfd_open, pid, flags);
+}
+
+#endif
diff --git a/tools/testing/selftests/bpf/test_dev_cgroup.c b/tools/testing/selftests/bpf/test_dev_cgroup.c
index 7886265846a0..adeaf63cb6fa 100644
--- a/tools/testing/selftests/bpf/test_dev_cgroup.c
+++ b/tools/testing/selftests/bpf/test_dev_cgroup.c
@@ -16,7 +16,7 @@
 #include "cgroup_helpers.h"
 #include "testing_helpers.h"
 
-#define DEV_CGROUP_PROG "./dev_cgroup.o"
+#define DEV_CGROUP_PROG "./dev_cgroup.bpf.o"
 
 #define TEST_CGROUP "/test-bpf-based-device-cgroup/"
 
diff --git a/tools/testing/selftests/bpf/test_flow_dissector.sh b/tools/testing/selftests/bpf/test_flow_dissector.sh
index dbd91221727d..5303ce0c977b 100755
--- a/tools/testing/selftests/bpf/test_flow_dissector.sh
+++ b/tools/testing/selftests/bpf/test_flow_dissector.sh
@@ -115,6 +115,14 @@ tc filter add dev lo parent ffff: protocol ip pref 1337 flower ip_proto \
 # Send 10 IPv4/UDP packets from port 10. Filter should not drop any.
 ./test_flow_dissector -i 4 -f 10
 
+echo "Testing IPv4 from 127.0.0.127 (fallback to generic dissector)..."
+# Send 10 IPv4/UDP packets from port 8. Filter should not drop any.
+./test_flow_dissector -i 4 -S 127.0.0.127 -f 8
+# Send 10 IPv4/UDP packets from port 9. Filter should drop all.
+./test_flow_dissector -i 4 -S 127.0.0.127 -f 9 -F
+# Send 10 IPv4/UDP packets from port 10. Filter should not drop any.
+./test_flow_dissector -i 4 -S 127.0.0.127 -f 10
+
 echo "Testing IPIP..."
 # Send 10 IPv4/IPv4/UDP packets from port 8. Filter should not drop any.
 ./with_addr.sh ./with_tunnels.sh ./test_flow_dissector -o 4 -e bare -i 4 \
diff --git a/tools/testing/selftests/bpf/test_kmod.sh b/tools/testing/selftests/bpf/test_kmod.sh
index 4f6444bcd53f..50dca53ac536 100755
--- a/tools/testing/selftests/bpf/test_kmod.sh
+++ b/tools/testing/selftests/bpf/test_kmod.sh
@@ -1,6 +1,11 @@
 #!/bin/sh
 # SPDX-License-Identifier: GPL-2.0
 
+# Usage:
+# ./test_kmod.sh [module_param]...
+# Ex.: ./test_kmod.sh test_range=1,3
+# All the parameters are passed to the kernel module.
+
 # Kselftest framework requirement - SKIP code is 4.
 ksft_skip=4
 
@@ -24,17 +29,18 @@ test_run()
 	sysctl -w net.core.bpf_jit_harden=$2 2>&1 > /dev/null
 
 	echo "[ JIT enabled:$1 hardened:$2 ]"
+	shift 2
 	dmesg -C
 	if [ -f ${OUTPUT}/lib/test_bpf.ko ]; then
-		insmod ${OUTPUT}/lib/test_bpf.ko 2> /dev/null
+		insmod ${OUTPUT}/lib/test_bpf.ko "$@" 2> /dev/null
 		if [ $? -ne 0 ]; then
 			rc=1
 		fi
 	else
 		# Use modprobe dry run to check for missing test_bpf module
-		if ! /sbin/modprobe -q -n test_bpf; then
+		if ! /sbin/modprobe -q -n test_bpf "$@"; then
 			echo "test_bpf: [SKIP]"
-		elif /sbin/modprobe -q test_bpf; then
+		elif /sbin/modprobe -q test_bpf "$@"; then
 			echo "test_bpf: ok"
 		else
 			echo "test_bpf: [FAIL]"
@@ -59,9 +65,9 @@ test_restore()
 
 rc=0
 test_save
-test_run 0 0
-test_run 1 0
-test_run 1 1
-test_run 1 2
+test_run 0 0 "$@"
+test_run 1 0 "$@"
+test_run 1 1 "$@"
+test_run 1 2 "$@"
 test_restore
 exit $rc
diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c
index 2893e9f2f1e0..4694422aa76c 100644
--- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c
+++ b/tools/testing/selftests/bpf/test_lirc_mode2_user.c
@@ -59,7 +59,7 @@ int main(int argc, char **argv)
 		return 2;
 	}
 
-	ret = bpf_prog_test_load("test_lirc_mode2_kern.o",
+	ret = bpf_prog_test_load("test_lirc_mode2_kern.bpf.o",
 				 BPF_PROG_TYPE_LIRC_MODE2, &obj, &progfd);
 	if (ret) {
 		printf("Failed to load bpf program\n");
diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c
index cbebfaa7c1e8..b73152822aa2 100644
--- a/tools/testing/selftests/bpf/test_maps.c
+++ b/tools/testing/selftests/bpf/test_maps.c
@@ -30,7 +30,7 @@
 #define ENOTSUPP 524
 #endif
 
-static int skips;
+int skips;
 
 static struct bpf_map_create_opts map_opts = { .sz = sizeof(map_opts) };
 
@@ -264,10 +264,11 @@ static void test_hashmap_percpu(unsigned int task, void *data)
 	close(fd);
 }
 
+#define VALUE_SIZE 3
 static int helper_fill_hashmap(int max_entries)
 {
 	int i, fd, ret;
-	long long key, value;
+	long long key, value[VALUE_SIZE] = {};
 
 	fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value),
 			    max_entries, &map_opts);
@@ -276,8 +277,8 @@ static int helper_fill_hashmap(int max_entries)
 	      "err: %s, flags: 0x%x\n", strerror(errno), map_opts.map_flags);
 
 	for (i = 0; i < max_entries; i++) {
-		key = i; value = key;
-		ret = bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST);
+		key = i; value[0] = key;
+		ret = bpf_map_update_elem(fd, &key, value, BPF_NOEXIST);
 		CHECK(ret != 0,
 		      "can't update hashmap",
 		      "err: %s\n", strerror(ret));
@@ -288,8 +289,8 @@ static int helper_fill_hashmap(int max_entries)
 
 static void test_hashmap_walk(unsigned int task, void *data)
 {
-	int fd, i, max_entries = 1000;
-	long long key, value, next_key;
+	int fd, i, max_entries = 10000;
+	long long key, value[VALUE_SIZE], next_key;
 	bool next_key_valid = true;
 
 	fd = helper_fill_hashmap(max_entries);
@@ -297,7 +298,7 @@ static void test_hashmap_walk(unsigned int task, void *data)
 	for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key,
 					 &next_key) == 0; i++) {
 		key = next_key;
-		assert(bpf_map_lookup_elem(fd, &key, &value) == 0);
+		assert(bpf_map_lookup_elem(fd, &key, value) == 0);
 	}
 
 	assert(i == max_entries);
@@ -305,9 +306,9 @@ static void test_hashmap_walk(unsigned int task, void *data)
 	assert(bpf_map_get_next_key(fd, NULL, &key) == 0);
 	for (i = 0; next_key_valid; i++) {
 		next_key_valid = bpf_map_get_next_key(fd, &key, &next_key) == 0;
-		assert(bpf_map_lookup_elem(fd, &key, &value) == 0);
-		value++;
-		assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0);
+		assert(bpf_map_lookup_elem(fd, &key, value) == 0);
+		value[0]++;
+		assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0);
 		key = next_key;
 	}
 
@@ -316,8 +317,8 @@ static void test_hashmap_walk(unsigned int task, void *data)
 	for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key,
 					 &next_key) == 0; i++) {
 		key = next_key;
-		assert(bpf_map_lookup_elem(fd, &key, &value) == 0);
-		assert(value - 1 == key);
+		assert(bpf_map_lookup_elem(fd, &key, value) == 0);
+		assert(value[0] - 1 == key);
 	}
 
 	assert(i == max_entries);
@@ -651,20 +652,20 @@ static void test_stackmap(unsigned int task, void *data)
 #include <arpa/inet.h>
 #include <sys/select.h>
 #include <linux/err.h>
-#define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.o"
-#define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.o"
-#define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.o"
+#define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.bpf.o"
+#define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.bpf.o"
+#define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.bpf.o"
 static void test_sockmap(unsigned int tasks, void *data)
 {
 	struct bpf_map *bpf_map_rx, *bpf_map_tx, *bpf_map_msg, *bpf_map_break;
 	int map_fd_msg = 0, map_fd_rx = 0, map_fd_tx = 0, map_fd_break;
+	struct bpf_object *parse_obj, *verdict_obj, *msg_obj;
 	int ports[] = {50200, 50201, 50202, 50204};
 	int err, i, fd, udp, sfd[6] = {0xdeadbeef};
 	u8 buf[20] = {0x0, 0x5, 0x3, 0x2, 0x1, 0x0};
 	int parse_prog, verdict_prog, msg_prog;
 	struct sockaddr_in addr;
 	int one = 1, s, sc, rc;
-	struct bpf_object *obj;
 	struct timeval to;
 	__u32 key, value;
 	pid_t pid[tasks];
@@ -760,6 +761,7 @@ static void test_sockmap(unsigned int tasks, void *data)
 		       i, udp);
 		goto out_sockmap;
 	}
+	close(udp);
 
 	/* Test update without programs */
 	for (i = 0; i < 6; i++) {
@@ -822,27 +824,27 @@ static void test_sockmap(unsigned int tasks, void *data)
 
 	/* Load SK_SKB program and Attach */
 	err = bpf_prog_test_load(SOCKMAP_PARSE_PROG,
-			    BPF_PROG_TYPE_SK_SKB, &obj, &parse_prog);
+			    BPF_PROG_TYPE_SK_SKB, &parse_obj, &parse_prog);
 	if (err) {
 		printf("Failed to load SK_SKB parse prog\n");
 		goto out_sockmap;
 	}
 
 	err = bpf_prog_test_load(SOCKMAP_TCP_MSG_PROG,
-			    BPF_PROG_TYPE_SK_MSG, &obj, &msg_prog);
+			    BPF_PROG_TYPE_SK_MSG, &msg_obj, &msg_prog);
 	if (err) {
 		printf("Failed to load SK_SKB msg prog\n");
 		goto out_sockmap;
 	}
 
 	err = bpf_prog_test_load(SOCKMAP_VERDICT_PROG,
-			    BPF_PROG_TYPE_SK_SKB, &obj, &verdict_prog);
+			    BPF_PROG_TYPE_SK_SKB, &verdict_obj, &verdict_prog);
 	if (err) {
 		printf("Failed to load SK_SKB verdict prog\n");
 		goto out_sockmap;
 	}
 
-	bpf_map_rx = bpf_object__find_map_by_name(obj, "sock_map_rx");
+	bpf_map_rx = bpf_object__find_map_by_name(verdict_obj, "sock_map_rx");
 	if (!bpf_map_rx) {
 		printf("Failed to load map rx from verdict prog\n");
 		goto out_sockmap;
@@ -854,7 +856,7 @@ static void test_sockmap(unsigned int tasks, void *data)
 		goto out_sockmap;
 	}
 
-	bpf_map_tx = bpf_object__find_map_by_name(obj, "sock_map_tx");
+	bpf_map_tx = bpf_object__find_map_by_name(verdict_obj, "sock_map_tx");
 	if (!bpf_map_tx) {
 		printf("Failed to load map tx from verdict prog\n");
 		goto out_sockmap;
@@ -866,7 +868,7 @@ static void test_sockmap(unsigned int tasks, void *data)
 		goto out_sockmap;
 	}
 
-	bpf_map_msg = bpf_object__find_map_by_name(obj, "sock_map_msg");
+	bpf_map_msg = bpf_object__find_map_by_name(verdict_obj, "sock_map_msg");
 	if (!bpf_map_msg) {
 		printf("Failed to load map msg from msg_verdict prog\n");
 		goto out_sockmap;
@@ -878,7 +880,7 @@ static void test_sockmap(unsigned int tasks, void *data)
 		goto out_sockmap;
 	}
 
-	bpf_map_break = bpf_object__find_map_by_name(obj, "sock_map_break");
+	bpf_map_break = bpf_object__find_map_by_name(verdict_obj, "sock_map_break");
 	if (!bpf_map_break) {
 		printf("Failed to load map tx from verdict prog\n");
 		goto out_sockmap;
@@ -1124,7 +1126,9 @@ static void test_sockmap(unsigned int tasks, void *data)
 	}
 	close(fd);
 	close(map_fd_rx);
-	bpf_object__close(obj);
+	bpf_object__close(parse_obj);
+	bpf_object__close(msg_obj);
+	bpf_object__close(verdict_obj);
 	return;
 out:
 	for (i = 0; i < 6; i++)
@@ -1143,8 +1147,8 @@ out_sockmap:
 	exit(1);
 }
 
-#define MAPINMAP_PROG "./test_map_in_map.o"
-#define MAPINMAP_INVALID_PROG "./test_map_in_map_invalid.o"
+#define MAPINMAP_PROG "./test_map_in_map.bpf.o"
+#define MAPINMAP_INVALID_PROG "./test_map_in_map_invalid.bpf.o"
 static void test_map_in_map(void)
 {
 	struct bpf_object *obj;
@@ -1282,8 +1286,11 @@ static void test_map_in_map(void)
 			printf("Inner map mim.inner was not destroyed\n");
 			goto out_map_in_map;
 		}
+
+		close(fd);
 	}
 
+	bpf_object__close(obj);
 	return;
 
 out_map_in_map:
@@ -1371,16 +1378,16 @@ static void __run_parallel(unsigned int tasks,
 
 static void test_map_stress(void)
 {
+	run_parallel(100, test_hashmap_walk, NULL);
 	run_parallel(100, test_hashmap, NULL);
 	run_parallel(100, test_hashmap_percpu, NULL);
 	run_parallel(100, test_hashmap_sizes, NULL);
-	run_parallel(100, test_hashmap_walk, NULL);
 
 	run_parallel(100, test_arraymap, NULL);
 	run_parallel(100, test_arraymap_percpu, NULL);
 }
 
-#define TASKS 1024
+#define TASKS 100
 
 #define DO_UPDATE 1
 #define DO_DELETE 0
@@ -1432,6 +1439,8 @@ static void test_update_delete(unsigned int fn, void *data)
 	int fd = ((int *)data)[0];
 	int i, key, value, err;
 
+	if (fn & 1)
+		test_hashmap_walk(fn, NULL);
 	for (i = fn; i < MAP_SIZE; i += TASKS) {
 		key = value = i;
 
@@ -1455,7 +1464,7 @@ static void test_update_delete(unsigned int fn, void *data)
 
 static void test_map_parallel(void)
 {
-	int i, fd, key = 0, value = 0;
+	int i, fd, key = 0, value = 0, j = 0;
 	int data[2];
 
 	fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(key), sizeof(value),
@@ -1466,6 +1475,7 @@ static void test_map_parallel(void)
 		exit(1);
 	}
 
+again:
 	/* Use the same fd in children to add elements to this map:
 	 * child_0 adds key=0, key=1024, key=2048, ...
 	 * child_1 adds key=1, key=1025, key=2049, ...
@@ -1502,6 +1512,12 @@ static void test_map_parallel(void)
 	key = -1;
 	assert(bpf_map_get_next_key(fd, NULL, &key) < 0 && errno == ENOENT);
 	assert(bpf_map_get_next_key(fd, &key, &key) < 0 && errno == ENOENT);
+
+	key = 0;
+	bpf_map_delete_elem(fd, &key);
+	if (j++ < 5)
+		goto again;
+	close(fd);
 }
 
 static void test_map_rdonly(void)
diff --git a/tools/testing/selftests/bpf/test_maps.h b/tools/testing/selftests/bpf/test_maps.h
index 77d8587ac4ed..f6fbca761732 100644
--- a/tools/testing/selftests/bpf/test_maps.h
+++ b/tools/testing/selftests/bpf/test_maps.h
@@ -14,4 +14,6 @@
 	}								\
 })
 
+extern int skips;
+
 #endif
diff --git a/tools/testing/selftests/bpf/test_offload.py b/tools/testing/selftests/bpf/test_offload.py
index 6cd6ef9fc20b..7fc15e0d24a9 100755
--- a/tools/testing/selftests/bpf/test_offload.py
+++ b/tools/testing/selftests/bpf/test_offload.py
@@ -782,7 +782,7 @@ if out.find("/sys/kernel/debug type debugfs") == -1:
     cmd("mount -t debugfs none /sys/kernel/debug")
 
 # Check samples are compiled
-samples = ["sample_ret0.o", "sample_map_ret0.o"]
+samples = ["sample_ret0.bpf.o", "sample_map_ret0.bpf.o"]
 for s in samples:
     ret, out = cmd("ls %s/%s" % (bpf_test_dir, s), fail=False)
     skip(ret != 0, "sample %s/%s not found, please compile it" %
@@ -803,7 +803,7 @@ cmd("ip netns delete %s" % (ns))
 netns = []
 
 try:
-    obj = bpf_obj("sample_ret0.o")
+    obj = bpf_obj("sample_ret0.bpf.o")
     bytecode = bpf_bytecode("1,6 0 0 4294967295,")
 
     start_test("Test destruction of generic XDP...")
@@ -1023,7 +1023,7 @@ try:
 
     sim.wait_for_flush()
     start_test("Test non-offload XDP attaching to HW...")
-    bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/nooffload")
+    bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/nooffload")
     nooffload = bpf_pinned("/sys/fs/bpf/nooffload")
     ret, _, err = sim.set_xdp(nooffload, "offload",
                               fail=False, include_stderr=True)
@@ -1032,7 +1032,7 @@ try:
     rm("/sys/fs/bpf/nooffload")
 
     start_test("Test offload XDP attaching to drv...")
-    bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload",
+    bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/offload",
                       dev=sim['ifname'])
     offload = bpf_pinned("/sys/fs/bpf/offload")
     ret, _, err = sim.set_xdp(offload, "drv", fail=False, include_stderr=True)
@@ -1043,7 +1043,7 @@ try:
 
     start_test("Test XDP load failure...")
     sim.dfs["dev/bpf_bind_verifier_accept"] = 0
-    ret, _, err = bpftool_prog_load("sample_ret0.o", "/sys/fs/bpf/offload",
+    ret, _, err = bpftool_prog_load("sample_ret0.bpf.o", "/sys/fs/bpf/offload",
                                  dev=sim['ifname'], fail=False, include_stderr=True)
     fail(ret == 0, "verifier should fail on load")
     check_verifier_log(err, "[netdevsim] Hello from netdevsim!")
@@ -1169,7 +1169,7 @@ try:
 
     simdev = NetdevSimDev()
     sim, = simdev.nsims
-    map_obj = bpf_obj("sample_map_ret0.o")
+    map_obj = bpf_obj("sample_map_ret0.bpf.o")
     start_test("Test loading program with maps...")
     sim.set_xdp(map_obj, "offload", JSON=False) # map fixup msg breaks JSON
 
@@ -1307,10 +1307,10 @@ try:
     sims = (simA, simB1, simB2, simB3)
     simB = (simB1, simB2, simB3)
 
-    bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA",
+    bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimA",
                       dev=simA['ifname'])
     progA = bpf_pinned("/sys/fs/bpf/nsimA")
-    bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB",
+    bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB",
                       dev=simB1['ifname'])
     progB = bpf_pinned("/sys/fs/bpf/nsimB")
 
@@ -1344,14 +1344,14 @@ try:
     mapA = bpftool("prog show %s" % (progA))[1]["map_ids"][0]
     mapB = bpftool("prog show %s" % (progB))[1]["map_ids"][0]
 
-    ret, _ = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
+    ret, _ = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB_",
                                dev=simB3['ifname'],
                                maps=["idx 0 id %d" % (mapB)],
                                fail=False)
     fail(ret != 0, "couldn't reuse a map on the same ASIC")
     rm("/sys/fs/bpf/nsimB_")
 
-    ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimA_",
+    ret, _, err = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimA_",
                                     dev=simA['ifname'],
                                     maps=["idx 0 id %d" % (mapB)],
                                     fail=False, include_stderr=True)
@@ -1359,7 +1359,7 @@ try:
     fail(err.count("offload device mismatch between prog and map") == 0,
          "error message missing for cross-ASIC map")
 
-    ret, _, err = bpftool_prog_load("sample_map_ret0.o", "/sys/fs/bpf/nsimB_",
+    ret, _, err = bpftool_prog_load("sample_map_ret0.bpf.o", "/sys/fs/bpf/nsimB_",
                                     dev=simB1['ifname'],
                                     maps=["idx 0 id %d" % (mapA)],
                                     fail=False, include_stderr=True)
diff --git a/tools/testing/selftests/bpf/test_progs.c b/tools/testing/selftests/bpf/test_progs.c
index 3561c97701f2..0e9a47f97890 100644
--- a/tools/testing/selftests/bpf/test_progs.c
+++ b/tools/testing/selftests/bpf/test_progs.c
@@ -943,6 +943,23 @@ int trigger_module_test_write(int write_sz)
 	return 0;
 }
 
+int write_sysctl(const char *sysctl, const char *value)
+{
+	int fd, err, len;
+
+	fd = open(sysctl, O_WRONLY);
+	if (!ASSERT_NEQ(fd, -1, "open sysctl"))
+		return -1;
+
+	len = strlen(value);
+	err = write(fd, value, len);
+	close(fd);
+	if (!ASSERT_EQ(err, len, "write sysctl"))
+		return -1;
+
+	return 0;
+}
+
 #define MAX_BACKTRACE_SZ 128
 void crash_handler(int signum)
 {
diff --git a/tools/testing/selftests/bpf/test_progs.h b/tools/testing/selftests/bpf/test_progs.h
index 5fe1365c2bb1..b090996daee5 100644
--- a/tools/testing/selftests/bpf/test_progs.h
+++ b/tools/testing/selftests/bpf/test_progs.h
@@ -384,6 +384,7 @@ int extract_build_id(char *build_id, size_t size);
 int kern_sync_rcu(void);
 int trigger_module_test_read(int read_sz);
 int trigger_module_test_write(int write_sz);
+int write_sysctl(const char *sysctl, const char *value);
 
 #ifdef __x86_64__
 #define SYS_NANOSLEEP_KPROBE_NAME "__x64_sys_nanosleep"
diff --git a/tools/testing/selftests/bpf/test_skb_cgroup_id.sh b/tools/testing/selftests/bpf/test_skb_cgroup_id.sh
index a9bc6f82abc1..515c2eafc97f 100755
--- a/tools/testing/selftests/bpf/test_skb_cgroup_id.sh
+++ b/tools/testing/selftests/bpf/test_skb_cgroup_id.sh
@@ -54,7 +54,7 @@ DIR=$(dirname $0)
 TEST_IF="test_cgid_1"
 TEST_IF_PEER="test_cgid_2"
 MAX_PING_TRIES=5
-BPF_PROG_OBJ="${DIR}/test_skb_cgroup_id_kern.o"
+BPF_PROG_OBJ="${DIR}/test_skb_cgroup_id_kern.bpf.o"
 BPF_PROG_SECTION="cgroup_id_logger"
 BPF_PROG_ID=0
 PROG="${DIR}/test_skb_cgroup_id_user"
diff --git a/tools/testing/selftests/bpf/test_sock_addr.c b/tools/testing/selftests/bpf/test_sock_addr.c
index 458564fcfc82..2c89674fc62c 100644
--- a/tools/testing/selftests/bpf/test_sock_addr.c
+++ b/tools/testing/selftests/bpf/test_sock_addr.c
@@ -26,14 +26,14 @@
 #endif
 
 #define CG_PATH	"/foo"
-#define CONNECT4_PROG_PATH	"./connect4_prog.o"
-#define CONNECT6_PROG_PATH	"./connect6_prog.o"
-#define SENDMSG4_PROG_PATH	"./sendmsg4_prog.o"
-#define SENDMSG6_PROG_PATH	"./sendmsg6_prog.o"
-#define RECVMSG4_PROG_PATH	"./recvmsg4_prog.o"
-#define RECVMSG6_PROG_PATH	"./recvmsg6_prog.o"
-#define BIND4_PROG_PATH		"./bind4_prog.o"
-#define BIND6_PROG_PATH		"./bind6_prog.o"
+#define CONNECT4_PROG_PATH	"./connect4_prog.bpf.o"
+#define CONNECT6_PROG_PATH	"./connect6_prog.bpf.o"
+#define SENDMSG4_PROG_PATH	"./sendmsg4_prog.bpf.o"
+#define SENDMSG6_PROG_PATH	"./sendmsg6_prog.bpf.o"
+#define RECVMSG4_PROG_PATH	"./recvmsg4_prog.bpf.o"
+#define RECVMSG6_PROG_PATH	"./recvmsg6_prog.bpf.o"
+#define BIND4_PROG_PATH		"./bind4_prog.bpf.o"
+#define BIND6_PROG_PATH		"./bind6_prog.bpf.o"
 
 #define SERV4_IP		"192.168.1.254"
 #define SERV4_REWRITE_IP	"127.0.0.1"
diff --git a/tools/testing/selftests/bpf/test_sockmap.c b/tools/testing/selftests/bpf/test_sockmap.c
index 0fbaccdc8861..e768181a1bd7 100644
--- a/tools/testing/selftests/bpf/test_sockmap.c
+++ b/tools/testing/selftests/bpf/test_sockmap.c
@@ -52,8 +52,8 @@ static void running_handler(int a);
 #define S1_PORT 10000
 #define S2_PORT 10001
 
-#define BPF_SOCKMAP_FILENAME  "test_sockmap_kern.o"
-#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
+#define BPF_SOCKMAP_FILENAME  "test_sockmap_kern.bpf.o"
+#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.bpf.o"
 #define CG_PATH "/sockmap"
 
 /* global sockets */
@@ -138,6 +138,7 @@ struct sockmap_options {
 	bool data_test;
 	bool drop_expected;
 	bool check_recved_len;
+	bool tx_wait_mem;
 	int iov_count;
 	int iov_length;
 	int rate;
@@ -578,6 +579,10 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
 			sent = sendmsg(fd, &msg, flags);
 
 			if (!drop && sent < 0) {
+				if (opt->tx_wait_mem && errno == EACCES) {
+					errno = 0;
+					goto out_errno;
+				}
 				perror("sendmsg loop error");
 				goto out_errno;
 			} else if (drop && sent >= 0) {
@@ -644,6 +649,15 @@ static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
 				goto out_errno;
 			}
 
+			if (opt->tx_wait_mem) {
+				FD_ZERO(&w);
+				FD_SET(fd, &w);
+				slct = select(max_fd + 1, NULL, NULL, &w, &timeout);
+				errno = 0;
+				close(fd);
+				goto out_errno;
+			}
+
 			errno = 0;
 			if (peek_flag) {
 				flags |= MSG_PEEK;
@@ -752,6 +766,22 @@ static int sendmsg_test(struct sockmap_options *opt)
 			return err;
 	}
 
+	if (opt->tx_wait_mem) {
+		struct timeval timeout;
+		int rxtx_buf_len = 1024;
+
+		timeout.tv_sec = 3;
+		timeout.tv_usec = 0;
+
+		err = setsockopt(c2, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(struct timeval));
+		err |= setsockopt(c2, SOL_SOCKET, SO_SNDBUFFORCE, &rxtx_buf_len, sizeof(int));
+		err |= setsockopt(p2, SOL_SOCKET, SO_RCVBUFFORCE, &rxtx_buf_len, sizeof(int));
+		if (err) {
+			perror("setsockopt failed()");
+			return errno;
+		}
+	}
+
 	rxpid = fork();
 	if (rxpid == 0) {
 		if (txmsg_pop || txmsg_start_pop)
@@ -788,6 +818,9 @@ static int sendmsg_test(struct sockmap_options *opt)
 		return errno;
 	}
 
+	if (opt->tx_wait_mem)
+		close(c2);
+
 	txpid = fork();
 	if (txpid == 0) {
 		if (opt->sendpage)
@@ -1452,6 +1485,14 @@ static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
 	test_send(opt, cgrp);
 }
 
+static void test_txmsg_redir_wait_sndmem(int cgrp, struct sockmap_options *opt)
+{
+	txmsg_redir = 1;
+	opt->tx_wait_mem = true;
+	test_send_large(opt, cgrp);
+	opt->tx_wait_mem = false;
+}
+
 static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
 {
 	txmsg_drop = 1;
@@ -1800,6 +1841,7 @@ static int populate_progs(char *bpf_file)
 struct _test test[] = {
 	{"txmsg test passthrough", test_txmsg_pass},
 	{"txmsg test redirect", test_txmsg_redir},
+	{"txmsg test redirect wait send mem", test_txmsg_redir_wait_sndmem},
 	{"txmsg test drop", test_txmsg_drop},
 	{"txmsg test ingress redirect", test_txmsg_ingress_redir},
 	{"txmsg test skb", test_txmsg_skb},
diff --git a/tools/testing/selftests/bpf/test_sysctl.c b/tools/testing/selftests/bpf/test_sysctl.c
index 57620e7c9048..bcdbd27f22f0 100644
--- a/tools/testing/selftests/bpf/test_sysctl.c
+++ b/tools/testing/selftests/bpf/test_sysctl.c
@@ -1372,7 +1372,7 @@ static struct sysctl_test tests[] = {
 	},
 	{
 		"C prog: deny all writes",
-		.prog_file = "./test_sysctl_prog.o",
+		.prog_file = "./test_sysctl_prog.bpf.o",
 		.attach_type = BPF_CGROUP_SYSCTL,
 		.sysctl = "net/ipv4/tcp_mem",
 		.open_flags = O_WRONLY,
@@ -1381,7 +1381,7 @@ static struct sysctl_test tests[] = {
 	},
 	{
 		"C prog: deny access by name",
-		.prog_file = "./test_sysctl_prog.o",
+		.prog_file = "./test_sysctl_prog.bpf.o",
 		.attach_type = BPF_CGROUP_SYSCTL,
 		.sysctl = "net/ipv4/route/mtu_expires",
 		.open_flags = O_RDONLY,
@@ -1389,7 +1389,7 @@ static struct sysctl_test tests[] = {
 	},
 	{
 		"C prog: read tcp_mem",
-		.prog_file = "./test_sysctl_prog.o",
+		.prog_file = "./test_sysctl_prog.bpf.o",
 		.attach_type = BPF_CGROUP_SYSCTL,
 		.sysctl = "net/ipv4/tcp_mem",
 		.open_flags = O_RDONLY,
diff --git a/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh b/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh
index 102e6588e2fe..b42c24282c25 100755
--- a/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh
+++ b/tools/testing/selftests/bpf/test_tcp_check_syncookie.sh
@@ -76,7 +76,7 @@ main()
 DIR=$(dirname $0)
 TEST_IF=lo
 MAX_PING_TRIES=5
-BPF_PROG_OBJ="${DIR}/test_tcp_check_syncookie_kern.o"
+BPF_PROG_OBJ="${DIR}/test_tcp_check_syncookie_kern.bpf.o"
 CLSACT_SECTION="tc"
 XDP_SECTION="xdp"
 BPF_PROG_ID=0
diff --git a/tools/testing/selftests/bpf/test_tcpnotify_user.c b/tools/testing/selftests/bpf/test_tcpnotify_user.c
index 8284db8b0f13..595194453ff8 100644
--- a/tools/testing/selftests/bpf/test_tcpnotify_user.c
+++ b/tools/testing/selftests/bpf/test_tcpnotify_user.c
@@ -69,7 +69,7 @@ int verify_result(const struct tcpnotify_globals *result)
 
 int main(int argc, char **argv)
 {
-	const char *file = "test_tcpnotify_kern.o";
+	const char *file = "test_tcpnotify_kern.bpf.o";
 	struct bpf_map *perf_map, *global_map;
 	struct tcpnotify_globals g = {0};
 	struct perf_buffer *pb = NULL;
diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c
index f9d553fbf68a..2dbcbf363c18 100644
--- a/tools/testing/selftests/bpf/test_verifier.c
+++ b/tools/testing/selftests/bpf/test_verifier.c
@@ -1498,7 +1498,8 @@ static void do_test_single(struct bpf_test *test, bool unpriv,
 		opts.log_level = DEFAULT_LIBBPF_LOG_LEVEL;
 	opts.prog_flags = pflags;
 
-	if (prog_type == BPF_PROG_TYPE_TRACING && test->kfunc) {
+	if ((prog_type == BPF_PROG_TYPE_TRACING ||
+	     prog_type == BPF_PROG_TYPE_LSM) && test->kfunc) {
 		int attach_btf_id;
 
 		attach_btf_id = libbpf_find_vmlinux_btf_id(test->kfunc,
diff --git a/tools/testing/selftests/bpf/test_xdp_redirect.sh b/tools/testing/selftests/bpf/test_xdp_redirect.sh
index 1d79f31480ad..0746a4fde9d3 100755
--- a/tools/testing/selftests/bpf/test_xdp_redirect.sh
+++ b/tools/testing/selftests/bpf/test_xdp_redirect.sh
@@ -54,10 +54,10 @@ test_xdp_redirect()
 		return 0
 	fi
 
-	ip -n ${NS1} link set veth11 $xdpmode obj xdp_dummy.o sec xdp &> /dev/null
-	ip -n ${NS2} link set veth22 $xdpmode obj xdp_dummy.o sec xdp &> /dev/null
-	ip link set dev veth1 $xdpmode obj test_xdp_redirect.o sec redirect_to_222 &> /dev/null
-	ip link set dev veth2 $xdpmode obj test_xdp_redirect.o sec redirect_to_111 &> /dev/null
+	ip -n ${NS1} link set veth11 $xdpmode obj xdp_dummy.bpf.o sec xdp &> /dev/null
+	ip -n ${NS2} link set veth22 $xdpmode obj xdp_dummy.bpf.o sec xdp &> /dev/null
+	ip link set dev veth1 $xdpmode obj test_xdp_redirect.bpf.o sec redirect_to_222 &> /dev/null
+	ip link set dev veth2 $xdpmode obj test_xdp_redirect.bpf.o sec redirect_to_111 &> /dev/null
 
 	if ip netns exec ${NS1} ping -c 1 10.1.1.22 &> /dev/null &&
 	   ip netns exec ${NS2} ping -c 1 10.1.1.11 &> /dev/null; then
diff --git a/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh b/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh
index cc57cb87e65f..4c3c3fdd2d73 100755
--- a/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh
+++ b/tools/testing/selftests/bpf/test_xdp_redirect_multi.sh
@@ -94,7 +94,7 @@ setup_ns()
 		# Add a neigh entry for IPv4 ping test
 		ip -n ${NS[$i]} neigh add 192.0.2.253 lladdr 00:00:00:00:00:01 dev veth0
 		ip -n ${NS[$i]} link set veth0 $mode obj \
-			xdp_dummy.o sec xdp &> /dev/null || \
+			xdp_dummy.bpf.o sec xdp &> /dev/null || \
 			{ test_fail "Unable to load dummy xdp" && exit 1; }
 		IFACES="$IFACES veth$i"
 		veth_mac[$i]=$(ip -n ${NS[0]} link show veth$i | awk '/link\/ether/ {print $2}')
diff --git a/tools/testing/selftests/bpf/test_xdp_veth.sh b/tools/testing/selftests/bpf/test_xdp_veth.sh
index 49936c4c8567..5211ca9a0239 100755
--- a/tools/testing/selftests/bpf/test_xdp_veth.sh
+++ b/tools/testing/selftests/bpf/test_xdp_veth.sh
@@ -101,7 +101,7 @@ ip -n ${NS3} link set dev veth33 up
 
 mkdir $BPF_DIR
 bpftool prog loadall \
-	xdp_redirect_map.o $BPF_DIR/progs type xdp \
+	xdp_redirect_map.bpf.o $BPF_DIR/progs type xdp \
 	pinmaps $BPF_DIR/maps
 bpftool map update pinned $BPF_DIR/maps/tx_port key 0 0 0 0 value 122 0 0 0
 bpftool map update pinned $BPF_DIR/maps/tx_port key 1 0 0 0 value 133 0 0 0
@@ -110,9 +110,9 @@ ip link set dev veth1 xdp pinned $BPF_DIR/progs/xdp_redirect_map_0
 ip link set dev veth2 xdp pinned $BPF_DIR/progs/xdp_redirect_map_1
 ip link set dev veth3 xdp pinned $BPF_DIR/progs/xdp_redirect_map_2
 
-ip -n ${NS1} link set dev veth11 xdp obj xdp_dummy.o sec xdp
-ip -n ${NS2} link set dev veth22 xdp obj xdp_tx.o sec xdp
-ip -n ${NS3} link set dev veth33 xdp obj xdp_dummy.o sec xdp
+ip -n ${NS1} link set dev veth11 xdp obj xdp_dummy.bpf.o sec xdp
+ip -n ${NS2} link set dev veth22 xdp obj xdp_tx.bpf.o sec xdp
+ip -n ${NS3} link set dev veth33 xdp obj xdp_dummy.bpf.o sec xdp
 
 trap cleanup EXIT
 
diff --git a/tools/testing/selftests/bpf/test_xsk.sh b/tools/testing/selftests/bpf/test_xsk.sh
index 096a957594cd..d821fd098504 100755
--- a/tools/testing/selftests/bpf/test_xsk.sh
+++ b/tools/testing/selftests/bpf/test_xsk.sh
@@ -73,14 +73,20 @@
 #
 # Run and dump packet contents:
 #   sudo ./test_xsk.sh -D
+#
+# Run test suite for physical device in loopback mode
+#   sudo ./test_xsk.sh -i IFACE
 
 . xsk_prereqs.sh
 
-while getopts "vD" flag
+ETH=""
+
+while getopts "vDi:" flag
 do
 	case "${flag}" in
 		v) verbose=1;;
 		D) dump_pkts=1;;
+		i) ETH=${OPTARG};;
 	esac
 done
 
@@ -132,18 +138,25 @@ setup_vethPairs() {
 	ip link set ${VETH0} up
 }
 
-validate_root_exec
-validate_veth_support ${VETH0}
-validate_ip_utility
-setup_vethPairs
-
-retval=$?
-if [ $retval -ne 0 ]; then
-	test_status $retval "${TEST_NAME}"
-	cleanup_exit ${VETH0} ${VETH1} ${NS1}
-	exit $retval
+if [ ! -z $ETH ]; then
+	VETH0=${ETH}
+	VETH1=${ETH}
+	NS1=""
+else
+	validate_root_exec
+	validate_veth_support ${VETH0}
+	validate_ip_utility
+	setup_vethPairs
+
+	retval=$?
+	if [ $retval -ne 0 ]; then
+		test_status $retval "${TEST_NAME}"
+		cleanup_exit ${VETH0} ${VETH1} ${NS1}
+		exit $retval
+	fi
 fi
 
+
 if [[ $verbose -eq 1 ]]; then
 	ARGS+="-v "
 fi
@@ -152,26 +165,33 @@ if [[ $dump_pkts -eq 1 ]]; then
 	ARGS="-D "
 fi
 
+retval=$?
 test_status $retval "${TEST_NAME}"
 
 ## START TESTS
 
 statusList=()
 
-TEST_NAME="XSK_SELFTESTS_SOFTIRQ"
+TEST_NAME="XSK_SELFTESTS_${VETH0}_SOFTIRQ"
 
 exec_xskxceiver
 
-cleanup_exit ${VETH0} ${VETH1} ${NS1}
-TEST_NAME="XSK_SELFTESTS_BUSY_POLL"
+if [ -z $ETH ]; then
+	cleanup_exit ${VETH0} ${VETH1} ${NS1}
+fi
+TEST_NAME="XSK_SELFTESTS_${VETH0}_BUSY_POLL"
 busy_poll=1
 
-setup_vethPairs
+if [ -z $ETH ]; then
+	setup_vethPairs
+fi
 exec_xskxceiver
 
 ## END TESTS
 
-cleanup_exit ${VETH0} ${VETH1} ${NS1}
+if [ -z $ETH ]; then
+	cleanup_exit ${VETH0} ${VETH1} ${NS1}
+fi
 
 failures=0
 echo -e "\nSummary:"
diff --git a/tools/testing/selftests/bpf/verifier/calls.c b/tools/testing/selftests/bpf/verifier/calls.c
index 3fb4f69b1962..e1a937277b54 100644
--- a/tools/testing/selftests/bpf/verifier/calls.c
+++ b/tools/testing/selftests/bpf/verifier/calls.c
@@ -284,7 +284,7 @@
 	.result = ACCEPT,
 },
 {
-	"calls: not on unpriviledged",
+	"calls: not on unprivileged",
 	.insns = {
 	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 1, 0, 2),
 	BPF_MOV64_IMM(BPF_REG_0, 1),
diff --git a/tools/testing/selftests/bpf/verifier/ref_tracking.c b/tools/testing/selftests/bpf/verifier/ref_tracking.c
index 57a83d763ec1..f18ce867271f 100644
--- a/tools/testing/selftests/bpf/verifier/ref_tracking.c
+++ b/tools/testing/selftests/bpf/verifier/ref_tracking.c
@@ -85,6 +85,145 @@
 	.result = REJECT,
 },
 {
+	"reference tracking: acquire/release user key reference",
+	.insns = {
+	BPF_MOV64_IMM(BPF_REG_1, -3),
+	BPF_MOV64_IMM(BPF_REG_2, 0),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2),
+	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_MOV64_IMM(BPF_REG_0, 0),
+	BPF_EXIT_INSN(),
+	},
+	.prog_type = BPF_PROG_TYPE_LSM,
+	.kfunc = "bpf",
+	.expected_attach_type = BPF_LSM_MAC,
+	.flags = BPF_F_SLEEPABLE,
+	.fixup_kfunc_btf_id = {
+		{ "bpf_lookup_user_key", 2 },
+		{ "bpf_key_put", 5 },
+	},
+	.result = ACCEPT,
+},
+{
+	"reference tracking: acquire/release system key reference",
+	.insns = {
+	BPF_MOV64_IMM(BPF_REG_1, 1),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, 2),
+	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_MOV64_IMM(BPF_REG_0, 0),
+	BPF_EXIT_INSN(),
+	},
+	.prog_type = BPF_PROG_TYPE_LSM,
+	.kfunc = "bpf",
+	.expected_attach_type = BPF_LSM_MAC,
+	.flags = BPF_F_SLEEPABLE,
+	.fixup_kfunc_btf_id = {
+		{ "bpf_lookup_system_key", 1 },
+		{ "bpf_key_put", 4 },
+	},
+	.result = ACCEPT,
+},
+{
+	"reference tracking: release user key reference without check",
+	.insns = {
+	BPF_MOV64_IMM(BPF_REG_1, -3),
+	BPF_MOV64_IMM(BPF_REG_2, 0),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_MOV64_IMM(BPF_REG_0, 0),
+	BPF_EXIT_INSN(),
+	},
+	.prog_type = BPF_PROG_TYPE_LSM,
+	.kfunc = "bpf",
+	.expected_attach_type = BPF_LSM_MAC,
+	.flags = BPF_F_SLEEPABLE,
+	.errstr = "arg#0 pointer type STRUCT bpf_key must point to scalar, or struct with scalar",
+	.fixup_kfunc_btf_id = {
+		{ "bpf_lookup_user_key", 2 },
+		{ "bpf_key_put", 4 },
+	},
+	.result = REJECT,
+},
+{
+	"reference tracking: release system key reference without check",
+	.insns = {
+	BPF_MOV64_IMM(BPF_REG_1, 1),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_MOV64_REG(BPF_REG_1, BPF_REG_0),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_MOV64_IMM(BPF_REG_0, 0),
+	BPF_EXIT_INSN(),
+	},
+	.prog_type = BPF_PROG_TYPE_LSM,
+	.kfunc = "bpf",
+	.expected_attach_type = BPF_LSM_MAC,
+	.flags = BPF_F_SLEEPABLE,
+	.errstr = "arg#0 pointer type STRUCT bpf_key must point to scalar, or struct with scalar",
+	.fixup_kfunc_btf_id = {
+		{ "bpf_lookup_system_key", 1 },
+		{ "bpf_key_put", 3 },
+	},
+	.result = REJECT,
+},
+{
+	"reference tracking: release with NULL key pointer",
+	.insns = {
+	BPF_MOV64_IMM(BPF_REG_1, 0),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_MOV64_IMM(BPF_REG_0, 0),
+	BPF_EXIT_INSN(),
+	},
+	.prog_type = BPF_PROG_TYPE_LSM,
+	.kfunc = "bpf",
+	.expected_attach_type = BPF_LSM_MAC,
+	.flags = BPF_F_SLEEPABLE,
+	.errstr = "arg#0 pointer type STRUCT bpf_key must point to scalar, or struct with scalar",
+	.fixup_kfunc_btf_id = {
+		{ "bpf_key_put", 1 },
+	},
+	.result = REJECT,
+},
+{
+	"reference tracking: leak potential reference to user key",
+	.insns = {
+	BPF_MOV64_IMM(BPF_REG_1, -3),
+	BPF_MOV64_IMM(BPF_REG_2, 0),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_EXIT_INSN(),
+	},
+	.prog_type = BPF_PROG_TYPE_LSM,
+	.kfunc = "bpf",
+	.expected_attach_type = BPF_LSM_MAC,
+	.flags = BPF_F_SLEEPABLE,
+	.errstr = "Unreleased reference",
+	.fixup_kfunc_btf_id = {
+		{ "bpf_lookup_user_key", 2 },
+	},
+	.result = REJECT,
+},
+{
+	"reference tracking: leak potential reference to system key",
+	.insns = {
+	BPF_MOV64_IMM(BPF_REG_1, 1),
+	BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, BPF_PSEUDO_KFUNC_CALL, 0, 0),
+	BPF_EXIT_INSN(),
+	},
+	.prog_type = BPF_PROG_TYPE_LSM,
+	.kfunc = "bpf",
+	.expected_attach_type = BPF_LSM_MAC,
+	.flags = BPF_F_SLEEPABLE,
+	.errstr = "Unreleased reference",
+	.fixup_kfunc_btf_id = {
+		{ "bpf_lookup_system_key", 1 },
+	},
+	.result = REJECT,
+},
+{
 	"reference tracking: release reference without check",
 	.insns = {
 	BPF_SK_LOOKUP(sk_lookup_tcp),
diff --git a/tools/testing/selftests/bpf/verifier/var_off.c b/tools/testing/selftests/bpf/verifier/var_off.c
index 187c6f6e32bc..d37f512fad16 100644
--- a/tools/testing/selftests/bpf/verifier/var_off.c
+++ b/tools/testing/selftests/bpf/verifier/var_off.c
@@ -121,7 +121,7 @@
 	BPF_EXIT_INSN(),
 	},
 	.fixup_map_hash_8b = { 1 },
-	/* The unpriviledged case is not too interesting; variable
+	/* The unprivileged case is not too interesting; variable
 	 * stack access is rejected.
 	 */
 	.errstr_unpriv = "R2 variable stack access prohibited for !root",
diff --git a/tools/testing/selftests/bpf/verify_sig_setup.sh b/tools/testing/selftests/bpf/verify_sig_setup.sh
new file mode 100755
index 000000000000..ba08922b4a27
--- /dev/null
+++ b/tools/testing/selftests/bpf/verify_sig_setup.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+set -e
+set -u
+set -o pipefail
+
+VERBOSE="${SELFTESTS_VERBOSE:=0}"
+LOG_FILE="$(mktemp /tmp/verify_sig_setup.log.XXXXXX)"
+
+x509_genkey_content="\
+[ req ]
+default_bits = 2048
+distinguished_name = req_distinguished_name
+prompt = no
+string_mask = utf8only
+x509_extensions = myexts
+
+[ req_distinguished_name ]
+CN = eBPF Signature Verification Testing Key
+
+[ myexts ]
+basicConstraints=critical,CA:FALSE
+keyUsage=digitalSignature
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid
+"
+
+usage()
+{
+	echo "Usage: $0 <setup|cleanup <existing_tmp_dir>"
+	exit 1
+}
+
+setup()
+{
+	local tmp_dir="$1"
+
+	echo "${x509_genkey_content}" > ${tmp_dir}/x509.genkey
+
+	openssl req -new -nodes -utf8 -sha256 -days 36500 \
+			-batch -x509 -config ${tmp_dir}/x509.genkey \
+			-outform PEM -out ${tmp_dir}/signing_key.pem \
+			-keyout ${tmp_dir}/signing_key.pem 2>&1
+
+	openssl x509 -in ${tmp_dir}/signing_key.pem -out \
+		${tmp_dir}/signing_key.der -outform der
+
+	key_id=$(cat ${tmp_dir}/signing_key.der | keyctl padd asymmetric ebpf_testing_key @s)
+
+	keyring_id=$(keyctl newring ebpf_testing_keyring @s)
+	keyctl link $key_id $keyring_id
+}
+
+cleanup() {
+	local tmp_dir="$1"
+
+	keyctl unlink $(keyctl search @s asymmetric ebpf_testing_key) @s
+	keyctl unlink $(keyctl search @s keyring ebpf_testing_keyring) @s
+	rm -rf ${tmp_dir}
+}
+
+catch()
+{
+	local exit_code="$1"
+	local log_file="$2"
+
+	if [[ "${exit_code}" -ne 0 ]]; then
+		cat "${log_file}" >&3
+	fi
+
+	rm -f "${log_file}"
+	exit ${exit_code}
+}
+
+main()
+{
+	[[ $# -ne 2 ]] && usage
+
+	local action="$1"
+	local tmp_dir="$2"
+
+	[[ ! -d "${tmp_dir}" ]] && echo "Directory ${tmp_dir} doesn't exist" && exit 1
+
+	if [[ "${action}" == "setup" ]]; then
+		setup "${tmp_dir}"
+	elif [[ "${action}" == "cleanup" ]]; then
+		cleanup "${tmp_dir}"
+	else
+		echo "Unknown action: ${action}"
+		exit 1
+	fi
+}
+
+trap 'catch "$?" "${LOG_FILE}"' EXIT
+
+if [[ "${VERBOSE}" -eq 0 ]]; then
+	# Save the stderr to 3 so that we can output back to
+	# it incase of an error.
+	exec 3>&2 1>"${LOG_FILE}" 2>&1
+fi
+
+main "$@"
+rm -f "${LOG_FILE}"
diff --git a/tools/testing/selftests/bpf/veristat.c b/tools/testing/selftests/bpf/veristat.c
new file mode 100644
index 000000000000..b0d83a28e348
--- /dev/null
+++ b/tools/testing/selftests/bpf/veristat.c
@@ -0,0 +1,1322 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */
+#define _GNU_SOURCE
+#include <argp.h>
+#include <string.h>
+#include <stdlib.h>
+#include <linux/compiler.h>
+#include <sched.h>
+#include <pthread.h>
+#include <dirent.h>
+#include <signal.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include <sys/sysinfo.h>
+#include <sys/stat.h>
+#include <bpf/libbpf.h>
+#include <libelf.h>
+#include <gelf.h>
+
+enum stat_id {
+	VERDICT,
+	DURATION,
+	TOTAL_INSNS,
+	TOTAL_STATES,
+	PEAK_STATES,
+	MAX_STATES_PER_INSN,
+	MARK_READ_MAX_LEN,
+
+	FILE_NAME,
+	PROG_NAME,
+
+	ALL_STATS_CNT,
+	NUM_STATS_CNT = FILE_NAME - VERDICT,
+};
+
+struct verif_stats {
+	char *file_name;
+	char *prog_name;
+
+	long stats[NUM_STATS_CNT];
+};
+
+struct stat_specs {
+	int spec_cnt;
+	enum stat_id ids[ALL_STATS_CNT];
+	bool asc[ALL_STATS_CNT];
+	int lens[ALL_STATS_CNT * 3]; /* 3x for comparison mode */
+};
+
+enum resfmt {
+	RESFMT_TABLE,
+	RESFMT_TABLE_CALCLEN, /* fake format to pre-calculate table's column widths */
+	RESFMT_CSV,
+};
+
+struct filter {
+	char *file_glob;
+	char *prog_glob;
+};
+
+static struct env {
+	char **filenames;
+	int filename_cnt;
+	bool verbose;
+	bool quiet;
+	int log_level;
+	enum resfmt out_fmt;
+	bool comparison_mode;
+
+	struct verif_stats *prog_stats;
+	int prog_stat_cnt;
+
+	/* baseline_stats is allocated and used only in comparsion mode */
+	struct verif_stats *baseline_stats;
+	int baseline_stat_cnt;
+
+	struct stat_specs output_spec;
+	struct stat_specs sort_spec;
+
+	struct filter *allow_filters;
+	struct filter *deny_filters;
+	int allow_filter_cnt;
+	int deny_filter_cnt;
+
+	int files_processed;
+	int files_skipped;
+	int progs_processed;
+	int progs_skipped;
+} env;
+
+static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
+{
+	if (!env.verbose)
+		return 0;
+	if (level == LIBBPF_DEBUG /* && !env.verbose */)
+		return 0;
+	return vfprintf(stderr, format, args);
+}
+
+const char *argp_program_version = "veristat";
+const char *argp_program_bug_address = "<bpf@vger.kernel.org>";
+const char argp_program_doc[] =
+"veristat    BPF verifier stats collection and comparison tool.\n"
+"\n"
+"USAGE: veristat <obj-file> [<obj-file>...]\n"
+"   OR: veristat -C <baseline.csv> <comparison.csv>\n";
+
+static const struct argp_option opts[] = {
+	{ NULL, 'h', NULL, OPTION_HIDDEN, "Show the full help" },
+	{ "verbose", 'v', NULL, 0, "Verbose mode" },
+	{ "log-level", 'l', "LEVEL", 0, "Verifier log level (default 0 for normal mode, 1 for verbose mode)" },
+	{ "quiet", 'q', NULL, 0, "Quiet mode" },
+	{ "emit", 'e', "SPEC", 0, "Specify stats to be emitted" },
+	{ "sort", 's', "SPEC", 0, "Specify sort order" },
+	{ "output-format", 'o', "FMT", 0, "Result output format (table, csv), default is table." },
+	{ "compare", 'C', NULL, 0, "Comparison mode" },
+	{ "filter", 'f', "FILTER", 0, "Filter expressions (or @filename for file with expressions)." },
+	{},
+};
+
+static int parse_stats(const char *stats_str, struct stat_specs *specs);
+static int append_filter(struct filter **filters, int *cnt, const char *str);
+static int append_filter_file(const char *path);
+
+static error_t parse_arg(int key, char *arg, struct argp_state *state)
+{
+	void *tmp;
+	int err;
+
+	switch (key) {
+	case 'h':
+		argp_state_help(state, stderr, ARGP_HELP_STD_HELP);
+		break;
+	case 'v':
+		env.verbose = true;
+		break;
+	case 'q':
+		env.quiet = true;
+		break;
+	case 'e':
+		err = parse_stats(arg, &env.output_spec);
+		if (err)
+			return err;
+		break;
+	case 's':
+		err = parse_stats(arg, &env.sort_spec);
+		if (err)
+			return err;
+		break;
+	case 'o':
+		if (strcmp(arg, "table") == 0) {
+			env.out_fmt = RESFMT_TABLE;
+		} else if (strcmp(arg, "csv") == 0) {
+			env.out_fmt = RESFMT_CSV;
+		} else {
+			fprintf(stderr, "Unrecognized output format '%s'\n", arg);
+			return -EINVAL;
+		}
+		break;
+	case 'l':
+		errno = 0;
+		env.log_level = strtol(arg, NULL, 10);
+		if (errno) {
+			fprintf(stderr, "invalid log level: %s\n", arg);
+			argp_usage(state);
+		}
+		break;
+	case 'C':
+		env.comparison_mode = true;
+		break;
+	case 'f':
+		if (arg[0] == '@')
+			err = append_filter_file(arg + 1);
+		else if (arg[0] == '!')
+			err = append_filter(&env.deny_filters, &env.deny_filter_cnt, arg + 1);
+		else
+			err = append_filter(&env.allow_filters, &env.allow_filter_cnt, arg);
+		if (err) {
+			fprintf(stderr, "Failed to collect program filter expressions: %d\n", err);
+			return err;
+		}
+		break;
+	case ARGP_KEY_ARG:
+		tmp = realloc(env.filenames, (env.filename_cnt + 1) * sizeof(*env.filenames));
+		if (!tmp)
+			return -ENOMEM;
+		env.filenames = tmp;
+		env.filenames[env.filename_cnt] = strdup(arg);
+		if (!env.filenames[env.filename_cnt])
+			return -ENOMEM;
+		env.filename_cnt++;
+		break;
+	default:
+		return ARGP_ERR_UNKNOWN;
+	}
+	return 0;
+}
+
+static const struct argp argp = {
+	.options = opts,
+	.parser = parse_arg,
+	.doc = argp_program_doc,
+};
+
+
+/* Adapted from perf/util/string.c */
+static bool glob_matches(const char *str, const char *pat)
+{
+	while (*str && *pat && *pat != '*') {
+		if (*str != *pat)
+			return false;
+		str++;
+		pat++;
+	}
+	/* Check wild card */
+	if (*pat == '*') {
+		while (*pat == '*')
+			pat++;
+		if (!*pat) /* Tail wild card matches all */
+			return true;
+		while (*str)
+			if (glob_matches(str++, pat))
+				return true;
+	}
+	return !*str && !*pat;
+}
+
+static bool should_process_file(const char *filename)
+{
+	int i;
+
+	if (env.deny_filter_cnt > 0) {
+		for (i = 0; i < env.deny_filter_cnt; i++) {
+			if (glob_matches(filename, env.deny_filters[i].file_glob))
+				return false;
+		}
+	}
+
+	if (env.allow_filter_cnt == 0)
+		return true;
+
+	for (i = 0; i < env.allow_filter_cnt; i++) {
+		if (glob_matches(filename, env.allow_filters[i].file_glob))
+			return true;
+	}
+
+	return false;
+}
+
+static bool is_bpf_obj_file(const char *path) {
+	Elf64_Ehdr *ehdr;
+	int fd, err = -EINVAL;
+	Elf *elf = NULL;
+
+	fd = open(path, O_RDONLY | O_CLOEXEC);
+	if (fd < 0)
+		return true; /* we'll fail later and propagate error */
+
+	/* ensure libelf is initialized */
+	(void)elf_version(EV_CURRENT);
+
+	elf = elf_begin(fd, ELF_C_READ, NULL);
+	if (!elf)
+		goto cleanup;
+
+	if (elf_kind(elf) != ELF_K_ELF || gelf_getclass(elf) != ELFCLASS64)
+		goto cleanup;
+
+	ehdr = elf64_getehdr(elf);
+	/* Old LLVM set e_machine to EM_NONE */
+	if (!ehdr || ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF))
+		goto cleanup;
+
+	err = 0;
+cleanup:
+	if (elf)
+		elf_end(elf);
+	close(fd);
+	return err == 0;
+}
+
+static bool should_process_prog(const char *path, const char *prog_name)
+{
+	const char *filename = basename(path);
+	int i;
+
+	if (env.deny_filter_cnt > 0) {
+		for (i = 0; i < env.deny_filter_cnt; i++) {
+			if (glob_matches(filename, env.deny_filters[i].file_glob))
+				return false;
+			if (!env.deny_filters[i].prog_glob)
+				continue;
+			if (glob_matches(prog_name, env.deny_filters[i].prog_glob))
+				return false;
+		}
+	}
+
+	if (env.allow_filter_cnt == 0)
+		return true;
+
+	for (i = 0; i < env.allow_filter_cnt; i++) {
+		if (!glob_matches(filename, env.allow_filters[i].file_glob))
+			continue;
+		/* if filter specifies only filename glob part, it implicitly
+		 * allows all progs within that file
+		 */
+		if (!env.allow_filters[i].prog_glob)
+			return true;
+		if (glob_matches(prog_name, env.allow_filters[i].prog_glob))
+			return true;
+	}
+
+	return false;
+}
+
+static int append_filter(struct filter **filters, int *cnt, const char *str)
+{
+	struct filter *f;
+	void *tmp;
+	const char *p;
+
+	tmp = realloc(*filters, (*cnt + 1) * sizeof(**filters));
+	if (!tmp)
+		return -ENOMEM;
+	*filters = tmp;
+
+	f = &(*filters)[*cnt];
+	f->file_glob = f->prog_glob = NULL;
+
+	/* filter can be specified either as "<obj-glob>" or "<obj-glob>/<prog-glob>" */
+	p = strchr(str, '/');
+	if (!p) {
+		f->file_glob = strdup(str);
+		if (!f->file_glob)
+			return -ENOMEM;
+	} else {
+		f->file_glob = strndup(str, p - str);
+		f->prog_glob = strdup(p + 1);
+		if (!f->file_glob || !f->prog_glob) {
+			free(f->file_glob);
+			free(f->prog_glob);
+			f->file_glob = f->prog_glob = NULL;
+			return -ENOMEM;
+		}
+	}
+
+	*cnt = *cnt + 1;
+	return 0;
+}
+
+static int append_filter_file(const char *path)
+{
+	char buf[1024];
+	FILE *f;
+	int err = 0;
+
+	f = fopen(path, "r");
+	if (!f) {
+		err = -errno;
+		fprintf(stderr, "Failed to open filters in '%s': %d\n", path, err);
+		return err;
+	}
+
+	while (fscanf(f, " %1023[^\n]\n", buf) == 1) {
+		/* lines starting with # are comments, skip them */
+		if (buf[0] == '\0' || buf[0] == '#')
+			continue;
+		/* lines starting with ! are negative match filters */
+		if (buf[0] == '!')
+			err = append_filter(&env.deny_filters, &env.deny_filter_cnt, buf + 1);
+		else
+			err = append_filter(&env.allow_filters, &env.allow_filter_cnt, buf);
+		if (err)
+			goto cleanup;
+	}
+
+cleanup:
+	fclose(f);
+	return err;
+}
+
+static const struct stat_specs default_output_spec = {
+	.spec_cnt = 7,
+	.ids = {
+		FILE_NAME, PROG_NAME, VERDICT, DURATION,
+		TOTAL_INSNS, TOTAL_STATES, PEAK_STATES,
+	},
+};
+
+static const struct stat_specs default_sort_spec = {
+	.spec_cnt = 2,
+	.ids = {
+		FILE_NAME, PROG_NAME,
+	},
+	.asc = { true, true, },
+};
+
+static struct stat_def {
+	const char *header;
+	const char *names[4];
+	bool asc_by_default;
+} stat_defs[] = {
+	[FILE_NAME] = { "File", {"file_name", "filename", "file"}, true /* asc */ },
+	[PROG_NAME] = { "Program", {"prog_name", "progname", "prog"}, true /* asc */ },
+	[VERDICT] = { "Verdict", {"verdict"}, true /* asc: failure, success */ },
+	[DURATION] = { "Duration (us)", {"duration", "dur"}, },
+	[TOTAL_INSNS] = { "Total insns", {"total_insns", "insns"}, },
+	[TOTAL_STATES] = { "Total states", {"total_states", "states"}, },
+	[PEAK_STATES] = { "Peak states", {"peak_states"}, },
+	[MAX_STATES_PER_INSN] = { "Max states per insn", {"max_states_per_insn"}, },
+	[MARK_READ_MAX_LEN] = { "Max mark read length", {"max_mark_read_len", "mark_read"}, },
+};
+
+static int parse_stat(const char *stat_name, struct stat_specs *specs)
+{
+	int id, i;
+
+	if (specs->spec_cnt >= ARRAY_SIZE(specs->ids)) {
+		fprintf(stderr, "Can't specify more than %zd stats\n", ARRAY_SIZE(specs->ids));
+		return -E2BIG;
+	}
+
+	for (id = 0; id < ARRAY_SIZE(stat_defs); id++) {
+		struct stat_def *def = &stat_defs[id];
+
+		for (i = 0; i < ARRAY_SIZE(stat_defs[id].names); i++) {
+			if (!def->names[i] || strcmp(def->names[i], stat_name) != 0)
+				continue;
+
+			specs->ids[specs->spec_cnt] = id;
+			specs->asc[specs->spec_cnt] = def->asc_by_default;
+			specs->spec_cnt++;
+
+			return 0;
+		}
+	}
+
+	fprintf(stderr, "Unrecognized stat name '%s'\n", stat_name);
+	return -ESRCH;
+}
+
+static int parse_stats(const char *stats_str, struct stat_specs *specs)
+{
+	char *input, *state = NULL, *next;
+	int err;
+
+	input = strdup(stats_str);
+	if (!input)
+		return -ENOMEM;
+
+	while ((next = strtok_r(state ? NULL : input, ",", &state))) {
+		err = parse_stat(next, specs);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+static void free_verif_stats(struct verif_stats *stats, size_t stat_cnt)
+{
+	int i;
+
+	if (!stats)
+		return;
+
+	for (i = 0; i < stat_cnt; i++) {
+		free(stats[i].file_name);
+		free(stats[i].prog_name);
+	}
+	free(stats);
+}
+
+static char verif_log_buf[64 * 1024];
+
+#define MAX_PARSED_LOG_LINES 100
+
+static int parse_verif_log(char * const buf, size_t buf_sz, struct verif_stats *s)
+{
+	const char *cur;
+	int pos, lines;
+
+	buf[buf_sz - 1] = '\0';
+
+	for (pos = strlen(buf) - 1, lines = 0; pos >= 0 && lines < MAX_PARSED_LOG_LINES; lines++) {
+		/* find previous endline or otherwise take the start of log buf */
+		for (cur = &buf[pos]; cur > buf && cur[0] != '\n'; cur--, pos--) {
+		}
+		/* next time start from end of previous line (or pos goes to <0) */
+		pos--;
+		/* if we found endline, point right after endline symbol;
+		 * otherwise, stay at the beginning of log buf
+		 */
+		if (cur[0] == '\n')
+			cur++;
+
+		if (1 == sscanf(cur, "verification time %ld usec\n", &s->stats[DURATION]))
+			continue;
+		if (6 == sscanf(cur, "processed %ld insns (limit %*d) max_states_per_insn %ld total_states %ld peak_states %ld mark_read %ld",
+				&s->stats[TOTAL_INSNS],
+				&s->stats[MAX_STATES_PER_INSN],
+				&s->stats[TOTAL_STATES],
+				&s->stats[PEAK_STATES],
+				&s->stats[MARK_READ_MAX_LEN]))
+			continue;
+	}
+
+	return 0;
+}
+
+static int process_prog(const char *filename, struct bpf_object *obj, struct bpf_program *prog)
+{
+	const char *prog_name = bpf_program__name(prog);
+	size_t buf_sz = sizeof(verif_log_buf);
+	char *buf = verif_log_buf;
+	struct verif_stats *stats;
+	int err = 0;
+	void *tmp;
+
+	if (!should_process_prog(filename, bpf_program__name(prog))) {
+		env.progs_skipped++;
+		return 0;
+	}
+
+	tmp = realloc(env.prog_stats, (env.prog_stat_cnt + 1) * sizeof(*env.prog_stats));
+	if (!tmp)
+		return -ENOMEM;
+	env.prog_stats = tmp;
+	stats = &env.prog_stats[env.prog_stat_cnt++];
+	memset(stats, 0, sizeof(*stats));
+
+	if (env.verbose) {
+		buf_sz = 16 * 1024 * 1024;
+		buf = malloc(buf_sz);
+		if (!buf)
+			return -ENOMEM;
+		bpf_program__set_log_buf(prog, buf, buf_sz);
+		bpf_program__set_log_level(prog, env.log_level | 4); /* stats + log */
+	} else {
+		bpf_program__set_log_buf(prog, buf, buf_sz);
+		bpf_program__set_log_level(prog, 4); /* only verifier stats */
+	}
+	verif_log_buf[0] = '\0';
+
+	err = bpf_object__load(obj);
+	env.progs_processed++;
+
+	stats->file_name = strdup(basename(filename));
+	stats->prog_name = strdup(bpf_program__name(prog));
+	stats->stats[VERDICT] = err == 0; /* 1 - success, 0 - failure */
+	parse_verif_log(buf, buf_sz, stats);
+
+	if (env.verbose) {
+		printf("PROCESSING %s/%s, DURATION US: %ld, VERDICT: %s, VERIFIER LOG:\n%s\n",
+		       filename, prog_name, stats->stats[DURATION],
+		       err ? "failure" : "success", buf);
+	}
+
+	if (verif_log_buf != buf)
+		free(buf);
+
+	return 0;
+};
+
+static int process_obj(const char *filename)
+{
+	struct bpf_object *obj = NULL, *tobj;
+	struct bpf_program *prog, *tprog, *lprog;
+	libbpf_print_fn_t old_libbpf_print_fn;
+	LIBBPF_OPTS(bpf_object_open_opts, opts);
+	int err = 0, prog_cnt = 0;
+
+	if (!should_process_file(basename(filename))) {
+		if (env.verbose)
+			printf("Skipping '%s' due to filters...\n", filename);
+		env.files_skipped++;
+		return 0;
+	}
+	if (!is_bpf_obj_file(filename)) {
+		if (env.verbose)
+			printf("Skipping '%s' as it's not a BPF object file...\n", filename);
+		env.files_skipped++;
+		return 0;
+	}
+
+	if (!env.quiet && env.out_fmt == RESFMT_TABLE)
+		printf("Processing '%s'...\n", basename(filename));
+
+	old_libbpf_print_fn = libbpf_set_print(libbpf_print_fn);
+	obj = bpf_object__open_file(filename, &opts);
+	if (!obj) {
+		/* if libbpf can't open BPF object file, it could be because
+		 * that BPF object file is incomplete and has to be statically
+		 * linked into a final BPF object file; instead of bailing
+		 * out, report it into stderr, mark it as skipped, and
+		 * proceeed
+		 */
+		fprintf(stderr, "Failed to open '%s': %d\n", filename, -errno);
+		env.files_skipped++;
+		err = 0;
+		goto cleanup;
+	}
+
+	env.files_processed++;
+
+	bpf_object__for_each_program(prog, obj) {
+		prog_cnt++;
+	}
+
+	if (prog_cnt == 1) {
+		prog = bpf_object__next_program(obj, NULL);
+		bpf_program__set_autoload(prog, true);
+		process_prog(filename, obj, prog);
+		goto cleanup;
+	}
+
+	bpf_object__for_each_program(prog, obj) {
+		const char *prog_name = bpf_program__name(prog);
+
+		tobj = bpf_object__open_file(filename, &opts);
+		if (!tobj) {
+			err = -errno;
+			fprintf(stderr, "Failed to open '%s': %d\n", filename, err);
+			goto cleanup;
+		}
+
+		bpf_object__for_each_program(tprog, tobj) {
+			const char *tprog_name = bpf_program__name(tprog);
+
+			if (strcmp(prog_name, tprog_name) == 0) {
+				bpf_program__set_autoload(tprog, true);
+				lprog = tprog;
+			} else {
+				bpf_program__set_autoload(tprog, false);
+			}
+		}
+
+		process_prog(filename, tobj, lprog);
+		bpf_object__close(tobj);
+	}
+
+cleanup:
+	bpf_object__close(obj);
+	libbpf_set_print(old_libbpf_print_fn);
+	return err;
+}
+
+static int cmp_stat(const struct verif_stats *s1, const struct verif_stats *s2,
+		    enum stat_id id, bool asc)
+{
+	int cmp = 0;
+
+	switch (id) {
+	case FILE_NAME:
+		cmp = strcmp(s1->file_name, s2->file_name);
+		break;
+	case PROG_NAME:
+		cmp = strcmp(s1->prog_name, s2->prog_name);
+		break;
+	case VERDICT:
+	case DURATION:
+	case TOTAL_INSNS:
+	case TOTAL_STATES:
+	case PEAK_STATES:
+	case MAX_STATES_PER_INSN:
+	case MARK_READ_MAX_LEN: {
+		long v1 = s1->stats[id];
+		long v2 = s2->stats[id];
+
+		if (v1 != v2)
+			cmp = v1 < v2 ? -1 : 1;
+		break;
+	}
+	default:
+		fprintf(stderr, "Unrecognized stat #%d\n", id);
+		exit(1);
+	}
+
+	return asc ? cmp : -cmp;
+}
+
+static int cmp_prog_stats(const void *v1, const void *v2)
+{
+	const struct verif_stats *s1 = v1, *s2 = v2;
+	int i, cmp;
+
+	for (i = 0; i < env.sort_spec.spec_cnt; i++) {
+		cmp = cmp_stat(s1, s2, env.sort_spec.ids[i], env.sort_spec.asc[i]);
+		if (cmp != 0)
+			return cmp;
+	}
+
+	return 0;
+}
+
+#define HEADER_CHAR '-'
+#define COLUMN_SEP "  "
+
+static void output_header_underlines(void)
+{
+	int i, j, len;
+
+	for (i = 0; i < env.output_spec.spec_cnt; i++) {
+		len = env.output_spec.lens[i];
+
+		printf("%s", i == 0 ? "" : COLUMN_SEP);
+		for (j = 0; j < len; j++)
+			printf("%c", HEADER_CHAR);
+	}
+	printf("\n");
+}
+
+static void output_headers(enum resfmt fmt)
+{
+	int i, len;
+
+	for (i = 0; i < env.output_spec.spec_cnt; i++) {
+		int id = env.output_spec.ids[i];
+		int *max_len = &env.output_spec.lens[i];
+
+		switch (fmt) {
+		case RESFMT_TABLE_CALCLEN:
+			len = snprintf(NULL, 0, "%s", stat_defs[id].header);
+			if (len > *max_len)
+				*max_len = len;
+			break;
+		case RESFMT_TABLE:
+			printf("%s%-*s", i == 0 ? "" : COLUMN_SEP,  *max_len, stat_defs[id].header);
+			if (i == env.output_spec.spec_cnt - 1)
+				printf("\n");
+			break;
+		case RESFMT_CSV:
+			printf("%s%s", i == 0 ? "" : ",", stat_defs[id].names[0]);
+			if (i == env.output_spec.spec_cnt - 1)
+				printf("\n");
+			break;
+		}
+	}
+
+	if (fmt == RESFMT_TABLE)
+		output_header_underlines();
+}
+
+static void prepare_value(const struct verif_stats *s, enum stat_id id,
+			  const char **str, long *val)
+{
+	switch (id) {
+	case FILE_NAME:
+		*str = s->file_name;
+		break;
+	case PROG_NAME:
+		*str = s->prog_name;
+		break;
+	case VERDICT:
+		*str = s->stats[VERDICT] ? "success" : "failure";
+		break;
+	case DURATION:
+	case TOTAL_INSNS:
+	case TOTAL_STATES:
+	case PEAK_STATES:
+	case MAX_STATES_PER_INSN:
+	case MARK_READ_MAX_LEN:
+		*val = s->stats[id];
+		break;
+	default:
+		fprintf(stderr, "Unrecognized stat #%d\n", id);
+		exit(1);
+	}
+}
+
+static void output_stats(const struct verif_stats *s, enum resfmt fmt, bool last)
+{
+	int i;
+
+	for (i = 0; i < env.output_spec.spec_cnt; i++) {
+		int id = env.output_spec.ids[i];
+		int *max_len = &env.output_spec.lens[i], len;
+		const char *str = NULL;
+		long val = 0;
+
+		prepare_value(s, id, &str, &val);
+
+		switch (fmt) {
+		case RESFMT_TABLE_CALCLEN:
+			if (str)
+				len = snprintf(NULL, 0, "%s", str);
+			else
+				len = snprintf(NULL, 0, "%ld", val);
+			if (len > *max_len)
+				*max_len = len;
+			break;
+		case RESFMT_TABLE:
+			if (str)
+				printf("%s%-*s", i == 0 ? "" : COLUMN_SEP, *max_len, str);
+			else
+				printf("%s%*ld", i == 0 ? "" : COLUMN_SEP,  *max_len, val);
+			if (i == env.output_spec.spec_cnt - 1)
+				printf("\n");
+			break;
+		case RESFMT_CSV:
+			if (str)
+				printf("%s%s", i == 0 ? "" : ",", str);
+			else
+				printf("%s%ld", i == 0 ? "" : ",", val);
+			if (i == env.output_spec.spec_cnt - 1)
+				printf("\n");
+			break;
+		}
+	}
+
+	if (last && fmt == RESFMT_TABLE) {
+		output_header_underlines();
+		printf("Done. Processed %d files, %d programs. Skipped %d files, %d programs.\n",
+		       env.files_processed, env.files_skipped, env.progs_processed, env.progs_skipped);
+	}
+}
+
+static int handle_verif_mode(void)
+{
+	int i, err;
+
+	if (env.filename_cnt == 0) {
+		fprintf(stderr, "Please provide path to BPF object file!\n");
+		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
+		return -EINVAL;
+	}
+
+	for (i = 0; i < env.filename_cnt; i++) {
+		err = process_obj(env.filenames[i]);
+		if (err) {
+			fprintf(stderr, "Failed to process '%s': %d\n", env.filenames[i], err);
+			return err;
+		}
+	}
+
+	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
+
+	if (env.out_fmt == RESFMT_TABLE) {
+		/* calculate column widths */
+		output_headers(RESFMT_TABLE_CALCLEN);
+		for (i = 0; i < env.prog_stat_cnt; i++)
+			output_stats(&env.prog_stats[i], RESFMT_TABLE_CALCLEN, false);
+	}
+
+	/* actually output the table */
+	output_headers(env.out_fmt);
+	for (i = 0; i < env.prog_stat_cnt; i++) {
+		output_stats(&env.prog_stats[i], env.out_fmt, i == env.prog_stat_cnt - 1);
+	}
+
+	return 0;
+}
+
+static int parse_stat_value(const char *str, enum stat_id id, struct verif_stats *st)
+{
+	switch (id) {
+	case FILE_NAME:
+		st->file_name = strdup(str);
+		if (!st->file_name)
+			return -ENOMEM;
+		break;
+	case PROG_NAME:
+		st->prog_name = strdup(str);
+		if (!st->prog_name)
+			return -ENOMEM;
+		break;
+	case VERDICT:
+		if (strcmp(str, "success") == 0) {
+			st->stats[VERDICT] = true;
+		} else if (strcmp(str, "failure") == 0) {
+			st->stats[VERDICT] = false;
+		} else {
+			fprintf(stderr, "Unrecognized verification verdict '%s'\n", str);
+			return -EINVAL;
+		}
+		break;
+	case DURATION:
+	case TOTAL_INSNS:
+	case TOTAL_STATES:
+	case PEAK_STATES:
+	case MAX_STATES_PER_INSN:
+	case MARK_READ_MAX_LEN: {
+		long val;
+		int err, n;
+
+		if (sscanf(str, "%ld %n", &val, &n) != 1 || n != strlen(str)) {
+			err = -errno;
+			fprintf(stderr, "Failed to parse '%s' as integer\n", str);
+			return err;
+		}
+
+		st->stats[id] = val;
+		break;
+	}
+	default:
+		fprintf(stderr, "Unrecognized stat #%d\n", id);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int parse_stats_csv(const char *filename, struct stat_specs *specs,
+			   struct verif_stats **statsp, int *stat_cntp)
+{
+	char line[4096];
+	FILE *f;
+	int err = 0;
+	bool header = true;
+
+	f = fopen(filename, "r");
+	if (!f) {
+		err = -errno;
+		fprintf(stderr, "Failed to open '%s': %d\n", filename, err);
+		return err;
+	}
+
+	*stat_cntp = 0;
+
+	while (fgets(line, sizeof(line), f)) {
+		char *input = line, *state = NULL, *next;
+		struct verif_stats *st = NULL;
+		int col = 0;
+
+		if (!header) {
+			void *tmp;
+
+			tmp = realloc(*statsp, (*stat_cntp + 1) * sizeof(**statsp));
+			if (!tmp) {
+				err = -ENOMEM;
+				goto cleanup;
+			}
+			*statsp = tmp;
+
+			st = &(*statsp)[*stat_cntp];
+			memset(st, 0, sizeof(*st));
+
+			*stat_cntp += 1;
+		}
+
+		while ((next = strtok_r(state ? NULL : input, ",\n", &state))) {
+			if (header) {
+				/* for the first line, set up spec stats */
+				err = parse_stat(next, specs);
+				if (err)
+					goto cleanup;
+				continue;
+			}
+
+			/* for all other lines, parse values based on spec */
+			if (col >= specs->spec_cnt) {
+				fprintf(stderr, "Found extraneous column #%d in row #%d of '%s'\n",
+					col, *stat_cntp, filename);
+				err = -EINVAL;
+				goto cleanup;
+			}
+			err = parse_stat_value(next, specs->ids[col], st);
+			if (err)
+				goto cleanup;
+			col++;
+		}
+
+		if (header) {
+			header = false;
+			continue;
+		}
+
+		if (col < specs->spec_cnt) {
+			fprintf(stderr, "Not enough columns in row #%d in '%s'\n",
+				*stat_cntp, filename);
+			err = -EINVAL;
+			goto cleanup;
+		}
+
+		if (!st->file_name || !st->prog_name) {
+			fprintf(stderr, "Row #%d in '%s' is missing file and/or program name\n",
+				*stat_cntp, filename);
+			err = -EINVAL;
+			goto cleanup;
+		}
+
+		/* in comparison mode we can only check filters after we
+		 * parsed entire line; if row should be ignored we pretend we
+		 * never parsed it
+		 */
+		if (!should_process_prog(st->file_name, st->prog_name)) {
+			free(st->file_name);
+			free(st->prog_name);
+			*stat_cntp -= 1;
+		}
+	}
+
+	if (!feof(f)) {
+		err = -errno;
+		fprintf(stderr, "Failed I/O for '%s': %d\n", filename, err);
+	}
+
+cleanup:
+	fclose(f);
+	return err;
+}
+
+/* empty/zero stats for mismatched rows */
+static const struct verif_stats fallback_stats = { .file_name = "", .prog_name = "" };
+
+static bool is_key_stat(enum stat_id id)
+{
+	return id == FILE_NAME || id == PROG_NAME;
+}
+
+static void output_comp_header_underlines(void)
+{
+	int i, j, k;
+
+	for (i = 0; i < env.output_spec.spec_cnt; i++) {
+		int id = env.output_spec.ids[i];
+		int max_j = is_key_stat(id) ? 1 : 3;
+
+		for (j = 0; j < max_j; j++) {
+			int len = env.output_spec.lens[3 * i + j];
+
+			printf("%s", i + j == 0 ? "" : COLUMN_SEP);
+
+			for (k = 0; k < len; k++)
+				printf("%c", HEADER_CHAR);
+		}
+	}
+	printf("\n");
+}
+
+static void output_comp_headers(enum resfmt fmt)
+{
+	static const char *table_sfxs[3] = {" (A)", " (B)", " (DIFF)"};
+	static const char *name_sfxs[3] = {"_base", "_comp", "_diff"};
+	int i, j, len;
+
+	for (i = 0; i < env.output_spec.spec_cnt; i++) {
+		int id = env.output_spec.ids[i];
+		/* key stats don't have A/B/DIFF columns, they are common for both data sets */
+		int max_j = is_key_stat(id) ? 1 : 3;
+
+		for (j = 0; j < max_j; j++) {
+			int *max_len = &env.output_spec.lens[3 * i + j];
+			bool last = (i == env.output_spec.spec_cnt - 1) && (j == max_j - 1);
+			const char *sfx;
+
+			switch (fmt) {
+			case RESFMT_TABLE_CALCLEN:
+				sfx = is_key_stat(id) ? "" : table_sfxs[j];
+				len = snprintf(NULL, 0, "%s%s", stat_defs[id].header, sfx);
+				if (len > *max_len)
+					*max_len = len;
+				break;
+			case RESFMT_TABLE:
+				sfx = is_key_stat(id) ? "" : table_sfxs[j];
+				printf("%s%-*s%s", i + j == 0 ? "" : COLUMN_SEP,
+				       *max_len - (int)strlen(sfx), stat_defs[id].header, sfx);
+				if (last)
+					printf("\n");
+				break;
+			case RESFMT_CSV:
+				sfx = is_key_stat(id) ? "" : name_sfxs[j];
+				printf("%s%s%s", i + j == 0 ? "" : ",", stat_defs[id].names[0], sfx);
+				if (last)
+					printf("\n");
+				break;
+			}
+		}
+	}
+
+	if (fmt == RESFMT_TABLE)
+		output_comp_header_underlines();
+}
+
+static void output_comp_stats(const struct verif_stats *base, const struct verif_stats *comp,
+			      enum resfmt fmt, bool last)
+{
+	char base_buf[1024] = {}, comp_buf[1024] = {}, diff_buf[1024] = {};
+	int i;
+
+	for (i = 0; i < env.output_spec.spec_cnt; i++) {
+		int id = env.output_spec.ids[i], len;
+		int *max_len_base = &env.output_spec.lens[3 * i + 0];
+		int *max_len_comp = &env.output_spec.lens[3 * i + 1];
+		int *max_len_diff = &env.output_spec.lens[3 * i + 2];
+		const char *base_str = NULL, *comp_str = NULL;
+		long base_val = 0, comp_val = 0, diff_val = 0;
+
+		prepare_value(base, id, &base_str, &base_val);
+		prepare_value(comp, id, &comp_str, &comp_val);
+
+		/* normalize all the outputs to be in string buffers for simplicity */
+		if (is_key_stat(id)) {
+			/* key stats (file and program name) are always strings */
+			if (base != &fallback_stats)
+				snprintf(base_buf, sizeof(base_buf), "%s", base_str);
+			else
+				snprintf(base_buf, sizeof(base_buf), "%s", comp_str);
+		} else if (base_str) {
+			snprintf(base_buf, sizeof(base_buf), "%s", base_str);
+			snprintf(comp_buf, sizeof(comp_buf), "%s", comp_str);
+			if (strcmp(base_str, comp_str) == 0)
+				snprintf(diff_buf, sizeof(diff_buf), "%s", "MATCH");
+			else
+				snprintf(diff_buf, sizeof(diff_buf), "%s", "MISMATCH");
+		} else {
+			snprintf(base_buf, sizeof(base_buf), "%ld", base_val);
+			snprintf(comp_buf, sizeof(comp_buf), "%ld", comp_val);
+
+			diff_val = comp_val - base_val;
+			if (base == &fallback_stats || comp == &fallback_stats || base_val == 0) {
+				snprintf(diff_buf, sizeof(diff_buf), "%+ld (%+.2lf%%)",
+					 diff_val, comp_val < base_val ? -100.0 : 100.0);
+			} else {
+				snprintf(diff_buf, sizeof(diff_buf), "%+ld (%+.2lf%%)",
+					 diff_val, diff_val * 100.0 / base_val);
+			}
+		}
+
+		switch (fmt) {
+		case RESFMT_TABLE_CALCLEN:
+			len = strlen(base_buf);
+			if (len > *max_len_base)
+				*max_len_base = len;
+			if (!is_key_stat(id)) {
+				len = strlen(comp_buf);
+				if (len > *max_len_comp)
+					*max_len_comp = len;
+				len = strlen(diff_buf);
+				if (len > *max_len_diff)
+					*max_len_diff = len;
+			}
+			break;
+		case RESFMT_TABLE: {
+			/* string outputs are left-aligned, number outputs are right-aligned */
+			const char *fmt = base_str ? "%s%-*s" : "%s%*s";
+
+			printf(fmt, i == 0 ? "" : COLUMN_SEP, *max_len_base, base_buf);
+			if (!is_key_stat(id)) {
+				printf(fmt, COLUMN_SEP, *max_len_comp, comp_buf);
+				printf(fmt, COLUMN_SEP, *max_len_diff, diff_buf);
+			}
+			if (i == env.output_spec.spec_cnt - 1)
+				printf("\n");
+			break;
+		}
+		case RESFMT_CSV:
+			printf("%s%s", i == 0 ? "" : ",", base_buf);
+			if (!is_key_stat(id)) {
+				printf("%s%s", i == 0 ? "" : ",", comp_buf);
+				printf("%s%s", i == 0 ? "" : ",", diff_buf);
+			}
+			if (i == env.output_spec.spec_cnt - 1)
+				printf("\n");
+			break;
+		}
+	}
+
+	if (last && fmt == RESFMT_TABLE)
+		output_comp_header_underlines();
+}
+
+static int cmp_stats_key(const struct verif_stats *base, const struct verif_stats *comp)
+{
+	int r;
+
+	r = strcmp(base->file_name, comp->file_name);
+	if (r != 0)
+		return r;
+	return strcmp(base->prog_name, comp->prog_name);
+}
+
+static int handle_comparison_mode(void)
+{
+	struct stat_specs base_specs = {}, comp_specs = {};
+	enum resfmt cur_fmt;
+	int err, i, j;
+
+	if (env.filename_cnt != 2) {
+		fprintf(stderr, "Comparison mode expects exactly two input CSV files!\n");
+		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
+		return -EINVAL;
+	}
+
+	err = parse_stats_csv(env.filenames[0], &base_specs,
+			      &env.baseline_stats, &env.baseline_stat_cnt);
+	if (err) {
+		fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[0], err);
+		return err;
+	}
+	err = parse_stats_csv(env.filenames[1], &comp_specs,
+			      &env.prog_stats, &env.prog_stat_cnt);
+	if (err) {
+		fprintf(stderr, "Failed to parse stats from '%s': %d\n", env.filenames[1], err);
+		return err;
+	}
+
+	/* To keep it simple we validate that the set and order of stats in
+	 * both CSVs are exactly the same. This can be lifted with a bit more
+	 * pre-processing later.
+	 */
+	if (base_specs.spec_cnt != comp_specs.spec_cnt) {
+		fprintf(stderr, "Number of stats in '%s' and '%s' differs (%d != %d)!\n",
+			env.filenames[0], env.filenames[1],
+			base_specs.spec_cnt, comp_specs.spec_cnt);
+		return -EINVAL;
+	}
+	for (i = 0; i < base_specs.spec_cnt; i++) {
+		if (base_specs.ids[i] != comp_specs.ids[i]) {
+			fprintf(stderr, "Stats composition differs between '%s' and '%s' (%s != %s)!\n",
+				env.filenames[0], env.filenames[1],
+				stat_defs[base_specs.ids[i]].names[0],
+				stat_defs[comp_specs.ids[i]].names[0]);
+			return -EINVAL;
+		}
+	}
+
+	qsort(env.prog_stats, env.prog_stat_cnt, sizeof(*env.prog_stats), cmp_prog_stats);
+	qsort(env.baseline_stats, env.baseline_stat_cnt, sizeof(*env.baseline_stats), cmp_prog_stats);
+
+	/* for human-readable table output we need to do extra pass to
+	 * calculate column widths, so we substitute current output format
+	 * with RESFMT_TABLE_CALCLEN and later revert it back to RESFMT_TABLE
+	 * and do everything again.
+	 */
+	if (env.out_fmt == RESFMT_TABLE)
+		cur_fmt = RESFMT_TABLE_CALCLEN;
+	else
+		cur_fmt = env.out_fmt;
+
+one_more_time:
+	output_comp_headers(cur_fmt);
+
+	/* If baseline and comparison datasets have different subset of rows
+	 * (we match by 'object + prog' as a unique key) then assume
+	 * empty/missing/zero value for rows that are missing in the opposite
+	 * data set
+	 */
+	i = j = 0;
+	while (i < env.baseline_stat_cnt || j < env.prog_stat_cnt) {
+		bool last = (i == env.baseline_stat_cnt - 1) || (j == env.prog_stat_cnt - 1);
+		const struct verif_stats *base, *comp;
+		int r;
+
+		base = i < env.baseline_stat_cnt ? &env.baseline_stats[i] : &fallback_stats;
+		comp = j < env.prog_stat_cnt ? &env.prog_stats[j] : &fallback_stats;
+
+		if (!base->file_name || !base->prog_name) {
+			fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n",
+				i, env.filenames[0]);
+			return -EINVAL;
+		}
+		if (!comp->file_name || !comp->prog_name) {
+			fprintf(stderr, "Entry #%d in '%s' doesn't have file and/or program name specified!\n",
+				j, env.filenames[1]);
+			return -EINVAL;
+		}
+
+		r = cmp_stats_key(base, comp);
+		if (r == 0) {
+			output_comp_stats(base, comp, cur_fmt, last);
+			i++;
+			j++;
+		} else if (comp == &fallback_stats || r < 0) {
+			output_comp_stats(base, &fallback_stats, cur_fmt, last);
+			i++;
+		} else {
+			output_comp_stats(&fallback_stats, comp, cur_fmt, last);
+			j++;
+		}
+	}
+
+	if (cur_fmt == RESFMT_TABLE_CALCLEN) {
+		cur_fmt = RESFMT_TABLE;
+		goto one_more_time; /* ... this time with feeling */
+	}
+
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	int err = 0, i;
+
+	if (argp_parse(&argp, argc, argv, 0, NULL, NULL))
+		return 1;
+
+	if (env.verbose && env.quiet) {
+		fprintf(stderr, "Verbose and quiet modes are incompatible, please specify just one or neither!\n");
+		argp_help(&argp, stderr, ARGP_HELP_USAGE, "veristat");
+		return 1;
+	}
+	if (env.verbose && env.log_level == 0)
+		env.log_level = 1;
+
+	if (env.output_spec.spec_cnt == 0)
+		env.output_spec = default_output_spec;
+	if (env.sort_spec.spec_cnt == 0)
+		env.sort_spec = default_sort_spec;
+
+	if (env.comparison_mode)
+		err = handle_comparison_mode();
+	else
+		err = handle_verif_mode();
+
+	free_verif_stats(env.prog_stats, env.prog_stat_cnt);
+	free_verif_stats(env.baseline_stats, env.baseline_stat_cnt);
+	for (i = 0; i < env.filename_cnt; i++)
+		free(env.filenames[i]);
+	free(env.filenames);
+	for (i = 0; i < env.allow_filter_cnt; i++) {
+		free(env.allow_filters[i].file_glob);
+		free(env.allow_filters[i].prog_glob);
+	}
+	free(env.allow_filters);
+	for (i = 0; i < env.deny_filter_cnt; i++) {
+		free(env.deny_filters[i].file_glob);
+		free(env.deny_filters[i].prog_glob);
+	}
+	free(env.deny_filters);
+	return -err;
+}
diff --git a/tools/testing/selftests/bpf/veristat.cfg b/tools/testing/selftests/bpf/veristat.cfg
new file mode 100644
index 000000000000..1a385061618d
--- /dev/null
+++ b/tools/testing/selftests/bpf/veristat.cfg
@@ -0,0 +1,17 @@
+# pre-canned list of rather complex selftests/bpf BPF object files to monitor
+# BPF verifier's performance on
+bpf_flow*
+bpf_loop_bench*
+loop*
+netif_receive_skb*
+profiler*
+pyperf*
+strobemeta*
+test_cls_redirect*
+test_l4lb
+test_sysctl*
+test_tcp_hdr_*
+test_usdt*
+test_verif_scale*
+test_xdp_noinline*
+xdp_synproxy*
diff --git a/tools/testing/selftests/bpf/vmtest.sh b/tools/testing/selftests/bpf/vmtest.sh
index b86ae4a2e5c5..a29aa05ebb3e 100755
--- a/tools/testing/selftests/bpf/vmtest.sh
+++ b/tools/testing/selftests/bpf/vmtest.sh
@@ -307,6 +307,20 @@ update_kconfig()
 	fi
 }
 
+catch()
+{
+	local exit_code=$1
+	local exit_status_file="${OUTPUT_DIR}/${EXIT_STATUS_FILE}"
+	# This is just a cleanup and the directory may
+	# have already been unmounted. So, don't let this
+	# clobber the error code we intend to return.
+	unmount_image || true
+	if [[ -f "${exit_status_file}" ]]; then
+		exit_code="$(cat ${exit_status_file})"
+	fi
+	exit ${exit_code}
+}
+
 main()
 {
 	local script_dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
@@ -319,7 +333,7 @@ main()
 	local exit_command="poweroff -f"
 	local debug_shell="no"
 
-	while getopts 'hskid:j:' opt; do
+	while getopts ':hskid:j:' opt; do
 		case ${opt} in
 		i)
 			update_image="yes"
@@ -353,6 +367,8 @@ main()
 	done
 	shift $((OPTIND -1))
 
+	trap 'catch "$?"' EXIT
+
 	if [[ $# -eq 0  && "${debug_shell}" == "no" ]]; then
 		echo "No command specified, will run ${DEFAULT_COMMAND} in the vm"
 	else
@@ -409,20 +425,4 @@ main()
 	fi
 }
 
-catch()
-{
-	local exit_code=$1
-	local exit_status_file="${OUTPUT_DIR}/${EXIT_STATUS_FILE}"
-	# This is just a cleanup and the directory may
-	# have already been unmounted. So, don't let this
-	# clobber the error code we intend to return.
-	unmount_image || true
-	if [[ -f "${exit_status_file}" ]]; then
-		exit_code="$(cat ${exit_status_file})"
-	fi
-	exit ${exit_code}
-}
-
-trap 'catch "$?"' EXIT
-
 main "$@"
diff --git a/tools/testing/selftests/bpf/xdp_redirect_multi.c b/tools/testing/selftests/bpf/xdp_redirect_multi.c
index c03b3a75991f..c1fc44c87c30 100644
--- a/tools/testing/selftests/bpf/xdp_redirect_multi.c
+++ b/tools/testing/selftests/bpf/xdp_redirect_multi.c
@@ -142,7 +142,7 @@ int main(int argc, char **argv)
 	}
 	printf("\n");
 
-	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
+	snprintf(filename, sizeof(filename), "%s_kern.bpf.o", argv[0]);
 	obj = bpf_object__open_file(filename, NULL);
 	err = libbpf_get_error(obj);
 	if (err)
diff --git a/tools/testing/selftests/bpf/xdp_synproxy.c b/tools/testing/selftests/bpf/xdp_synproxy.c
index d874ddfb39c4..ff35320d2be9 100644
--- a/tools/testing/selftests/bpf/xdp_synproxy.c
+++ b/tools/testing/selftests/bpf/xdp_synproxy.c
@@ -193,7 +193,7 @@ static int syncookie_attach(const char *argv0, unsigned int ifindex, bool tc)
 	int prog_fd;
 	int err;
 
-	snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv0);
+	snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.bpf.o", argv0);
 	obj = bpf_object__open_file(xdp_filename, NULL);
 	err = libbpf_get_error(obj);
 	if (err < 0) {
diff --git a/tools/testing/selftests/bpf/xdping.c b/tools/testing/selftests/bpf/xdping.c
index 5b6f977870f8..1503a1d2faa0 100644
--- a/tools/testing/selftests/bpf/xdping.c
+++ b/tools/testing/selftests/bpf/xdping.c
@@ -168,7 +168,7 @@ int main(int argc, char **argv)
 	/* Use libbpf 1.0 API mode */
 	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
 
-	snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]);
+	snprintf(filename, sizeof(filename), "%s_kern.bpf.o", argv[0]);
 
 	if (bpf_prog_test_load(filename, BPF_PROG_TYPE_XDP, &obj, &prog_fd)) {
 		fprintf(stderr, "load of %s failed\n", filename);
diff --git a/tools/testing/selftests/bpf/xsk.c b/tools/testing/selftests/bpf/xsk.c
index f2721a4ae7c5..0b3ff49c740d 100644
--- a/tools/testing/selftests/bpf/xsk.c
+++ b/tools/testing/selftests/bpf/xsk.c
@@ -1237,15 +1237,15 @@ void xsk_socket__delete(struct xsk_socket *xsk)
 	ctx = xsk->ctx;
 	umem = ctx->umem;
 
-	xsk_put_ctx(ctx, true);
-
-	if (!ctx->refcount) {
+	if (ctx->refcount == 1) {
 		xsk_delete_bpf_maps(xsk);
 		close(ctx->prog_fd);
 		if (ctx->has_bpf_link)
 			close(ctx->link_fd);
 	}
 
+	xsk_put_ctx(ctx, true);
+
 	err = xsk_get_mmap_offsets(xsk->fd, &off);
 	if (!err) {
 		if (xsk->rx) {
diff --git a/tools/testing/selftests/bpf/xskxceiver.c b/tools/testing/selftests/bpf/xskxceiver.c
index 74d56d971baf..681a5db80dae 100644
--- a/tools/testing/selftests/bpf/xskxceiver.c
+++ b/tools/testing/selftests/bpf/xskxceiver.c
@@ -99,6 +99,8 @@
 #include <stdatomic.h>
 #include "xsk.h"
 #include "xskxceiver.h"
+#include <bpf/bpf.h>
+#include <linux/filter.h>
 #include "../kselftest.h"
 
 /* AF_XDP APIs were moved into libxdp and marked as deprecated in libbpf.
@@ -122,9 +124,20 @@ static void __exit_with_error(int error, const char *file, const char *func, int
 }
 
 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
-
-#define mode_string(test) (test)->ifobj_tx->xdp_flags & XDP_FLAGS_SKB_MODE ? "SKB" : "DRV"
 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
+static char *mode_string(struct test_spec *test)
+{
+	switch (test->mode) {
+	case TEST_MODE_SKB:
+		return "SKB";
+	case TEST_MODE_DRV:
+		return "DRV";
+	case TEST_MODE_ZC:
+		return "ZC";
+	default:
+		return "BOGUS";
+	}
+}
 
 static void report_failure(struct test_spec *test)
 {
@@ -244,6 +257,11 @@ static void gen_udp_hdr(u32 payload, void *pkt, struct ifobject *ifobject,
 	memset32_htonl(pkt + PKT_HDR_SIZE, payload, UDP_PKT_DATA_SIZE);
 }
 
+static bool is_umem_valid(struct ifobject *ifobj)
+{
+	return !!ifobj->umem->umem;
+}
+
 static void gen_udp_csum(struct udphdr *udp_hdr, struct iphdr *ip_hdr)
 {
 	udp_hdr->check = 0;
@@ -294,8 +312,8 @@ static void enable_busy_poll(struct xsk_socket_info *xsk)
 		exit_with_error(errno);
 }
 
-static int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
-				struct ifobject *ifobject, bool shared)
+static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
+				  struct ifobject *ifobject, bool shared)
 {
 	struct xsk_socket_config cfg = {};
 	struct xsk_ring_cons *rxr;
@@ -315,6 +333,51 @@ static int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_inf
 	return xsk_socket__create(&xsk->xsk, ifobject->ifname, 0, umem->umem, rxr, txr, &cfg);
 }
 
+static bool ifobj_zc_avail(struct ifobject *ifobject)
+{
+	size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
+	int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
+	struct xsk_socket_info *xsk;
+	struct xsk_umem_info *umem;
+	bool zc_avail = false;
+	void *bufs;
+	int ret;
+
+	bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
+	if (bufs == MAP_FAILED)
+		exit_with_error(errno);
+
+	umem = calloc(1, sizeof(struct xsk_umem_info));
+	if (!umem) {
+		munmap(bufs, umem_sz);
+		exit_with_error(-ENOMEM);
+	}
+	umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
+	ret = xsk_configure_umem(umem, bufs, umem_sz);
+	if (ret)
+		exit_with_error(-ret);
+
+	xsk = calloc(1, sizeof(struct xsk_socket_info));
+	if (!xsk)
+		goto out;
+	ifobject->xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
+	ifobject->xdp_flags |= XDP_FLAGS_DRV_MODE;
+	ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
+	ifobject->rx_on = true;
+	xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
+	ret = __xsk_configure_socket(xsk, umem, ifobject, false);
+	if (!ret)
+		zc_avail = true;
+
+	xsk_socket__delete(xsk->xsk);
+	free(xsk);
+out:
+	munmap(umem->buffer, umem_sz);
+	xsk_umem__delete(umem->umem);
+	free(umem);
+	return zc_avail;
+}
+
 static struct option long_options[] = {
 	{"interface", required_argument, 0, 'i'},
 	{"busy-poll", no_argument, 0, 'b'},
@@ -426,20 +489,24 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
 		ifobj->use_poll = false;
 		ifobj->use_fill_ring = true;
 		ifobj->release_rx = true;
-		ifobj->pkt_stream = test->pkt_stream_default;
 		ifobj->validation_func = NULL;
 
 		if (i == 0) {
 			ifobj->rx_on = false;
 			ifobj->tx_on = true;
+			ifobj->pkt_stream = test->tx_pkt_stream_default;
 		} else {
 			ifobj->rx_on = true;
 			ifobj->tx_on = false;
+			ifobj->pkt_stream = test->rx_pkt_stream_default;
 		}
 
 		memset(ifobj->umem, 0, sizeof(*ifobj->umem));
 		ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
 		ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
+		if (ifobj->shared_umem && ifobj->rx_on)
+			ifobj->umem->base_addr = DEFAULT_UMEM_BUFFERS *
+				XSK_UMEM__DEFAULT_FRAME_SIZE;
 
 		for (j = 0; j < MAX_SOCKETS; j++) {
 			memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
@@ -458,12 +525,15 @@ static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
 			   struct ifobject *ifobj_rx, enum test_mode mode)
 {
-	struct pkt_stream *pkt_stream;
+	struct pkt_stream *tx_pkt_stream;
+	struct pkt_stream *rx_pkt_stream;
 	u32 i;
 
-	pkt_stream = test->pkt_stream_default;
+	tx_pkt_stream = test->tx_pkt_stream_default;
+	rx_pkt_stream = test->rx_pkt_stream_default;
 	memset(test, 0, sizeof(*test));
-	test->pkt_stream_default = pkt_stream;
+	test->tx_pkt_stream_default = tx_pkt_stream;
+	test->rx_pkt_stream_default = rx_pkt_stream;
 
 	for (i = 0; i < MAX_INTERFACES; i++) {
 		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
@@ -474,9 +544,14 @@ static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
 		else
 			ifobj->xdp_flags |= XDP_FLAGS_DRV_MODE;
 
-		ifobj->bind_flags = XDP_USE_NEED_WAKEUP | XDP_COPY;
+		ifobj->bind_flags = XDP_USE_NEED_WAKEUP;
+		if (mode == TEST_MODE_ZC)
+			ifobj->bind_flags |= XDP_ZEROCOPY;
+		else
+			ifobj->bind_flags |= XDP_COPY;
 	}
 
+	test->mode = mode;
 	__test_spec_init(test, ifobj_tx, ifobj_rx);
 }
 
@@ -524,16 +599,17 @@ static void pkt_stream_delete(struct pkt_stream *pkt_stream)
 static void pkt_stream_restore_default(struct test_spec *test)
 {
 	struct pkt_stream *tx_pkt_stream = test->ifobj_tx->pkt_stream;
+	struct pkt_stream *rx_pkt_stream = test->ifobj_rx->pkt_stream;
 
-	if (tx_pkt_stream != test->pkt_stream_default) {
+	if (tx_pkt_stream != test->tx_pkt_stream_default) {
 		pkt_stream_delete(test->ifobj_tx->pkt_stream);
-		test->ifobj_tx->pkt_stream = test->pkt_stream_default;
+		test->ifobj_tx->pkt_stream = test->tx_pkt_stream_default;
 	}
 
-	if (test->ifobj_rx->pkt_stream != test->pkt_stream_default &&
-	    test->ifobj_rx->pkt_stream != tx_pkt_stream)
+	if (rx_pkt_stream != test->rx_pkt_stream_default) {
 		pkt_stream_delete(test->ifobj_rx->pkt_stream);
-	test->ifobj_rx->pkt_stream = test->pkt_stream_default;
+		test->ifobj_rx->pkt_stream = test->rx_pkt_stream_default;
+	}
 }
 
 static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
@@ -556,7 +632,7 @@ static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
 
 static void pkt_set(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr, u32 len)
 {
-	pkt->addr = addr;
+	pkt->addr = addr + umem->base_addr;
 	pkt->len = len;
 	if (len > umem->frame_size - XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 2 - umem->frame_headroom)
 		pkt->valid = false;
@@ -595,22 +671,29 @@ static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
 
 	pkt_stream = pkt_stream_generate(test->ifobj_tx->umem, nb_pkts, pkt_len);
 	test->ifobj_tx->pkt_stream = pkt_stream;
+	pkt_stream = pkt_stream_generate(test->ifobj_rx->umem, nb_pkts, pkt_len);
 	test->ifobj_rx->pkt_stream = pkt_stream;
 }
 
-static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
+static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
+				      int offset)
 {
-	struct xsk_umem_info *umem = test->ifobj_tx->umem;
+	struct xsk_umem_info *umem = ifobj->umem;
 	struct pkt_stream *pkt_stream;
 	u32 i;
 
-	pkt_stream = pkt_stream_clone(umem, test->pkt_stream_default);
-	for (i = 1; i < test->pkt_stream_default->nb_pkts; i += 2)
+	pkt_stream = pkt_stream_clone(umem, ifobj->pkt_stream);
+	for (i = 1; i < ifobj->pkt_stream->nb_pkts; i += 2)
 		pkt_set(umem, &pkt_stream->pkts[i],
 			(i % umem->num_frames) * umem->frame_size + offset, pkt_len);
 
-	test->ifobj_tx->pkt_stream = pkt_stream;
-	test->ifobj_rx->pkt_stream = pkt_stream;
+	ifobj->pkt_stream = pkt_stream;
+}
+
+static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
+{
+	__pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
+	__pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
 }
 
 static void pkt_stream_receive_half(struct test_spec *test)
@@ -652,7 +735,8 @@ static struct pkt *pkt_generate(struct ifobject *ifobject, u32 pkt_nb)
 	return pkt;
 }
 
-static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
+static void __pkt_stream_generate_custom(struct ifobject *ifobj,
+					 struct pkt *pkts, u32 nb_pkts)
 {
 	struct pkt_stream *pkt_stream;
 	u32 i;
@@ -661,15 +745,20 @@ static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts,
 	if (!pkt_stream)
 		exit_with_error(ENOMEM);
 
-	test->ifobj_tx->pkt_stream = pkt_stream;
-	test->ifobj_rx->pkt_stream = pkt_stream;
-
 	for (i = 0; i < nb_pkts; i++) {
-		pkt_stream->pkts[i].addr = pkts[i].addr;
+		pkt_stream->pkts[i].addr = pkts[i].addr + ifobj->umem->base_addr;
 		pkt_stream->pkts[i].len = pkts[i].len;
 		pkt_stream->pkts[i].payload = i;
 		pkt_stream->pkts[i].valid = pkts[i].valid;
 	}
+
+	ifobj->pkt_stream = pkt_stream;
+}
+
+static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
+{
+	__pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts);
+	__pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts);
 }
 
 static void pkt_dump(void *pkt, u32 len)
@@ -817,12 +906,13 @@ static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
 	return TEST_PASS;
 }
 
-static int receive_pkts(struct ifobject *ifobj, struct pollfd *fds)
+static int receive_pkts(struct test_spec *test, struct pollfd *fds)
 {
-	struct timeval tv_end, tv_now, tv_timeout = {RECV_TMOUT, 0};
+	struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
+	struct pkt_stream *pkt_stream = test->ifobj_rx->pkt_stream;
 	u32 idx_rx = 0, idx_fq = 0, rcvd, i, pkts_sent = 0;
-	struct pkt_stream *pkt_stream = ifobj->pkt_stream;
-	struct xsk_socket_info *xsk = ifobj->xsk;
+	struct xsk_socket_info *xsk = test->ifobj_rx->xsk;
+	struct ifobject *ifobj = test->ifobj_rx;
 	struct xsk_umem_info *umem = xsk->umem;
 	struct pkt *pkt;
 	int ret;
@@ -843,17 +933,28 @@ static int receive_pkts(struct ifobject *ifobj, struct pollfd *fds)
 		}
 
 		kick_rx(xsk);
+		if (ifobj->use_poll) {
+			ret = poll(fds, 1, POLL_TMOUT);
+			if (ret < 0)
+				exit_with_error(-ret);
+
+			if (!ret) {
+				if (!is_umem_valid(test->ifobj_tx))
+					return TEST_PASS;
+
+				ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
+				return TEST_FAILURE;
 
-		rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
-		if (!rcvd) {
-			if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
-				ret = poll(fds, 1, POLL_TMOUT);
-				if (ret < 0)
-					exit_with_error(-ret);
 			}
-			continue;
+
+			if (!(fds->revents & POLLIN))
+				continue;
 		}
 
+		rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
+		if (!rcvd)
+			continue;
+
 		if (ifobj->use_fill_ring) {
 			ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
 			while (ret != rcvd) {
@@ -900,13 +1001,35 @@ static int receive_pkts(struct ifobject *ifobj, struct pollfd *fds)
 	return TEST_PASS;
 }
 
-static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb)
+static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb, struct pollfd *fds,
+		       bool timeout)
 {
 	struct xsk_socket_info *xsk = ifobject->xsk;
-	u32 i, idx, valid_pkts = 0;
+	bool use_poll = ifobject->use_poll;
+	u32 i, idx = 0, ret, valid_pkts = 0;
+
+	while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE) {
+		if (use_poll) {
+			ret = poll(fds, 1, POLL_TMOUT);
+			if (timeout) {
+				if (ret < 0) {
+					ksft_print_msg("ERROR: [%s] Poll error %d\n",
+						       __func__, ret);
+					return TEST_FAILURE;
+				}
+				if (ret == 0)
+					return TEST_PASS;
+				break;
+			}
+			if (ret <= 0) {
+				ksft_print_msg("ERROR: [%s] Poll error %d\n",
+					       __func__, ret);
+				return TEST_FAILURE;
+			}
+		}
 
-	while (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) < BATCH_SIZE)
 		complete_pkts(xsk, BATCH_SIZE);
+	}
 
 	for (i = 0; i < BATCH_SIZE; i++) {
 		struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
@@ -933,11 +1056,27 @@ static int __send_pkts(struct ifobject *ifobject, u32 *pkt_nb)
 
 	xsk_ring_prod__submit(&xsk->tx, i);
 	xsk->outstanding_tx += valid_pkts;
-	if (complete_pkts(xsk, i))
-		return TEST_FAILURE;
 
-	usleep(10);
-	return TEST_PASS;
+	if (use_poll) {
+		ret = poll(fds, 1, POLL_TMOUT);
+		if (ret <= 0) {
+			if (ret == 0 && timeout)
+				return TEST_PASS;
+
+			ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);
+			return TEST_FAILURE;
+		}
+	}
+
+	if (!timeout) {
+		if (complete_pkts(xsk, i))
+			return TEST_FAILURE;
+
+		usleep(10);
+		return TEST_PASS;
+	}
+
+	return TEST_CONTINUE;
 }
 
 static void wait_for_tx_completion(struct xsk_socket_info *xsk)
@@ -948,29 +1087,19 @@ static void wait_for_tx_completion(struct xsk_socket_info *xsk)
 
 static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
 {
+	bool timeout = !is_umem_valid(test->ifobj_rx);
 	struct pollfd fds = { };
-	u32 pkt_cnt = 0;
+	u32 pkt_cnt = 0, ret;
 
 	fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
 	fds.events = POLLOUT;
 
 	while (pkt_cnt < ifobject->pkt_stream->nb_pkts) {
-		int err;
-
-		if (ifobject->use_poll) {
-			int ret;
-
-			ret = poll(&fds, 1, POLL_TMOUT);
-			if (ret <= 0)
-				continue;
-
-			if (!(fds.revents & POLLOUT))
-				continue;
-		}
-
-		err = __send_pkts(ifobject, &pkt_cnt);
-		if (err || test->fail)
+		ret = __send_pkts(ifobject, &pkt_cnt, &fds, timeout);
+		if ((ret || test->fail) && !timeout)
 			return TEST_FAILURE;
+		else if (ret == TEST_PASS && timeout)
+			return ret;
 	}
 
 	wait_for_tx_completion(ifobject->xsk);
@@ -1081,6 +1210,70 @@ static int validate_tx_invalid_descs(struct ifobject *ifobject)
 	return TEST_PASS;
 }
 
+static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,
+				 struct xsk_umem_info *umem, bool tx)
+{
+	int i, ret;
+
+	for (i = 0; i < test->nb_sockets; i++) {
+		bool shared = (ifobject->shared_umem && tx) ? true : !!i;
+		u32 ctr = 0;
+
+		while (ctr++ < SOCK_RECONF_CTR) {
+			ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,
+						     ifobject, shared);
+			if (!ret)
+				break;
+
+			/* Retry if it fails as xsk_socket__create() is asynchronous */
+			if (ctr >= SOCK_RECONF_CTR)
+				exit_with_error(-ret);
+			usleep(USLEEP_MAX);
+		}
+		if (ifobject->busy_poll)
+			enable_busy_poll(&ifobject->xsk_arr[i]);
+	}
+}
+
+static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)
+{
+	xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);
+	ifobject->xsk = &ifobject->xsk_arr[0];
+	ifobject->xsk_map_fd = test->ifobj_rx->xsk_map_fd;
+	memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
+}
+
+static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream)
+{
+	u32 idx = 0, i, buffers_to_fill;
+	int ret;
+
+	if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
+		buffers_to_fill = umem->num_frames;
+	else
+		buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
+
+	ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
+	if (ret != buffers_to_fill)
+		exit_with_error(ENOSPC);
+	for (i = 0; i < buffers_to_fill; i++) {
+		u64 addr;
+
+		if (pkt_stream->use_addr_for_fill) {
+			struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i);
+
+			if (!pkt)
+				break;
+			addr = pkt->addr;
+		} else {
+			addr = i * umem->frame_size;
+		}
+
+		*xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
+	}
+	xsk_ring_prod__submit(&umem->fq, buffers_to_fill);
+}
+
 static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
 {
 	u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
@@ -1088,13 +1281,15 @@ static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
 	LIBBPF_OPTS(bpf_xdp_query_opts, opts);
 	int ret, ifindex;
 	void *bufs;
-	u32 i;
 
 	ifobject->ns_fd = switch_namespace(ifobject->nsname);
 
 	if (ifobject->umem->unaligned_mode)
 		mmap_flags |= MAP_HUGETLB;
 
+	if (ifobject->shared_umem)
+		umem_sz *= 2;
+
 	bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
 	if (bufs == MAP_FAILED)
 		exit_with_error(errno);
@@ -1103,24 +1298,9 @@ static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
 	if (ret)
 		exit_with_error(-ret);
 
-	for (i = 0; i < test->nb_sockets; i++) {
-		u32 ctr = 0;
-
-		while (ctr++ < SOCK_RECONF_CTR) {
-			ret = xsk_configure_socket(&ifobject->xsk_arr[i], ifobject->umem,
-						   ifobject, !!i);
-			if (!ret)
-				break;
+	xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);
 
-			/* Retry if it fails as xsk_socket__create() is asynchronous */
-			if (ctr >= SOCK_RECONF_CTR)
-				exit_with_error(-ret);
-			usleep(USLEEP_MAX);
-		}
-
-		if (ifobject->busy_poll)
-			enable_busy_poll(&ifobject->xsk_arr[i]);
-	}
+	xsk_configure_socket(test, ifobject, ifobject->umem, false);
 
 	ifobject->xsk = &ifobject->xsk_arr[0];
 
@@ -1156,22 +1336,18 @@ static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
 		exit_with_error(-ret);
 }
 
-static void testapp_cleanup_xsk_res(struct ifobject *ifobj)
-{
-	print_verbose("Destroying socket\n");
-	xsk_socket__delete(ifobj->xsk->xsk);
-	munmap(ifobj->umem->buffer, ifobj->umem->num_frames * ifobj->umem->frame_size);
-	xsk_umem__delete(ifobj->umem->umem);
-}
-
 static void *worker_testapp_validate_tx(void *arg)
 {
 	struct test_spec *test = (struct test_spec *)arg;
 	struct ifobject *ifobject = test->ifobj_tx;
 	int err;
 
-	if (test->current_step == 1)
-		thread_common_ops(test, ifobject);
+	if (test->current_step == 1) {
+		if (!ifobject->shared_umem)
+			thread_common_ops(test, ifobject);
+		else
+			thread_common_ops_tx(test, ifobject);
+	}
 
 	print_verbose("Sending %d packets on interface %s\n", ifobject->pkt_stream->nb_pkts,
 		      ifobject->ifname);
@@ -1182,60 +1358,30 @@ static void *worker_testapp_validate_tx(void *arg)
 	if (err)
 		report_failure(test);
 
-	if (test->total_steps == test->current_step || err)
-		testapp_cleanup_xsk_res(ifobject);
 	pthread_exit(NULL);
 }
 
-static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream)
-{
-	u32 idx = 0, i, buffers_to_fill;
-	int ret;
-
-	if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
-		buffers_to_fill = umem->num_frames;
-	else
-		buffers_to_fill = XSK_RING_PROD__DEFAULT_NUM_DESCS;
-
-	ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
-	if (ret != buffers_to_fill)
-		exit_with_error(ENOSPC);
-	for (i = 0; i < buffers_to_fill; i++) {
-		u64 addr;
-
-		if (pkt_stream->use_addr_for_fill) {
-			struct pkt *pkt = pkt_stream_get_pkt(pkt_stream, i);
-
-			if (!pkt)
-				break;
-			addr = pkt->addr;
-		} else {
-			addr = i * umem->frame_size;
-		}
-
-		*xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
-	}
-	xsk_ring_prod__submit(&umem->fq, buffers_to_fill);
-}
-
 static void *worker_testapp_validate_rx(void *arg)
 {
 	struct test_spec *test = (struct test_spec *)arg;
 	struct ifobject *ifobject = test->ifobj_rx;
 	struct pollfd fds = { };
+	int id = 0;
 	int err;
 
-	if (test->current_step == 1)
+	if (test->current_step == 1) {
 		thread_common_ops(test, ifobject);
-
-	xsk_populate_fill_ring(ifobject->umem, ifobject->pkt_stream);
+	} else {
+		bpf_map_delete_elem(ifobject->xsk_map_fd, &id);
+		xsk_socket__update_xskmap(ifobject->xsk->xsk, ifobject->xsk_map_fd);
+	}
 
 	fds.fd = xsk_socket__fd(ifobject->xsk->xsk);
 	fds.events = POLLIN;
 
 	pthread_barrier_wait(&barr);
 
-	err = receive_pkts(ifobject, &fds);
+	err = receive_pkts(test, &fds);
 
 	if (!err && ifobject->validation_func)
 		err = ifobject->validation_func(ifobject);
@@ -1246,11 +1392,66 @@ static void *worker_testapp_validate_rx(void *arg)
 		pthread_mutex_unlock(&pacing_mutex);
 	}
 
-	if (test->total_steps == test->current_step || err)
-		testapp_cleanup_xsk_res(ifobject);
 	pthread_exit(NULL);
 }
 
+static void testapp_clean_xsk_umem(struct ifobject *ifobj)
+{
+	u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
+
+	if (ifobj->shared_umem)
+		umem_sz *= 2;
+
+	xsk_umem__delete(ifobj->umem->umem);
+	munmap(ifobj->umem->buffer, umem_sz);
+}
+
+static void handler(int signum)
+{
+	pthread_exit(NULL);
+}
+
+static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj,
+						  enum test_type type)
+{
+	bool old_shared_umem = ifobj->shared_umem;
+	pthread_t t0;
+
+	if (pthread_barrier_init(&barr, NULL, 2))
+		exit_with_error(errno);
+
+	test->current_step++;
+	if (type == TEST_TYPE_POLL_RXQ_TMOUT)
+		pkt_stream_reset(ifobj->pkt_stream);
+	pkts_in_flight = 0;
+
+	test->ifobj_rx->shared_umem = false;
+	test->ifobj_tx->shared_umem = false;
+
+	signal(SIGUSR1, handler);
+	/* Spawn thread */
+	pthread_create(&t0, NULL, ifobj->func_ptr, test);
+
+	if (type != TEST_TYPE_POLL_TXQ_TMOUT)
+		pthread_barrier_wait(&barr);
+
+	if (pthread_barrier_destroy(&barr))
+		exit_with_error(errno);
+
+	pthread_kill(t0, SIGUSR1);
+	pthread_join(t0, NULL);
+
+	if (test->total_steps == test->current_step || test->fail) {
+		xsk_socket__delete(ifobj->xsk->xsk);
+		testapp_clean_xsk_umem(ifobj);
+	}
+
+	test->ifobj_rx->shared_umem = old_shared_umem;
+	test->ifobj_tx->shared_umem = old_shared_umem;
+
+	return !!test->fail;
+}
+
 static int testapp_validate_traffic(struct test_spec *test)
 {
 	struct ifobject *ifobj_tx = test->ifobj_tx;
@@ -1277,6 +1478,14 @@ static int testapp_validate_traffic(struct test_spec *test)
 	pthread_join(t1, NULL);
 	pthread_join(t0, NULL);
 
+	if (test->total_steps == test->current_step || test->fail) {
+		xsk_socket__delete(ifobj_tx->xsk->xsk);
+		xsk_socket__delete(ifobj_rx->xsk->xsk);
+		testapp_clean_xsk_umem(ifobj_rx);
+		if (!ifobj_tx->shared_umem)
+			testapp_clean_xsk_umem(ifobj_tx);
+	}
+
 	return !!test->fail;
 }
 
@@ -1356,9 +1565,9 @@ static void testapp_headroom(struct test_spec *test)
 static void testapp_stats_rx_dropped(struct test_spec *test)
 {
 	test_spec_set_name(test, "STAT_RX_DROPPED");
+	pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
 	test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
 		XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
-	pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
 	pkt_stream_receive_half(test);
 	test->ifobj_rx->validation_func = validate_rx_dropped;
 	testapp_validate_traffic(test);
@@ -1481,6 +1690,11 @@ static void testapp_invalid_desc(struct test_spec *test)
 		pkts[7].valid = false;
 	}
 
+	if (test->ifobj_tx->shared_umem) {
+		pkts[4].addr += UMEM_SIZE;
+		pkts[5].addr += UMEM_SIZE;
+	}
+
 	pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
 	testapp_validate_traffic(test);
 	pkt_stream_restore_default(test);
@@ -1511,6 +1725,10 @@ static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_
 {
 	switch (type) {
 	case TEST_TYPE_STATS_RX_DROPPED:
+		if (mode == TEST_MODE_ZC) {
+			ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n");
+			return;
+		}
 		testapp_stats_rx_dropped(test);
 		break;
 	case TEST_TYPE_STATS_TX_INVALID_DESCS:
@@ -1548,12 +1766,30 @@ static void run_pkt_test(struct test_spec *test, enum test_mode mode, enum test_
 
 		pkt_stream_restore_default(test);
 		break;
-	case TEST_TYPE_POLL:
-		test->ifobj_tx->use_poll = true;
+	case TEST_TYPE_RX_POLL:
 		test->ifobj_rx->use_poll = true;
-		test_spec_set_name(test, "POLL");
+		test_spec_set_name(test, "POLL_RX");
 		testapp_validate_traffic(test);
 		break;
+	case TEST_TYPE_TX_POLL:
+		test->ifobj_tx->use_poll = true;
+		test_spec_set_name(test, "POLL_TX");
+		testapp_validate_traffic(test);
+		break;
+	case TEST_TYPE_POLL_TXQ_TMOUT:
+		test_spec_set_name(test, "POLL_TXQ_FULL");
+		test->ifobj_tx->use_poll = true;
+		/* create invalid frame by set umem frame_size and pkt length equal to 2048 */
+		test->ifobj_tx->umem->frame_size = 2048;
+		pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048);
+		testapp_validate_traffic_single_thread(test, test->ifobj_tx, type);
+		pkt_stream_restore_default(test);
+		break;
+	case TEST_TYPE_POLL_RXQ_TMOUT:
+		test_spec_set_name(test, "POLL_RXQ_EMPTY");
+		test->ifobj_rx->use_poll = true;
+		testapp_validate_traffic_single_thread(test, test->ifobj_rx, type);
+		break;
 	case TEST_TYPE_ALIGNED_INV_DESC:
 		test_spec_set_name(test, "ALIGNED_INV_DESC");
 		testapp_invalid_desc(test);
@@ -1606,6 +1842,8 @@ static struct ifobject *ifobject_create(void)
 	if (!ifobj->umem)
 		goto out_umem;
 
+	ifobj->ns_fd = -1;
+
 	return ifobj;
 
 out_umem:
@@ -1617,17 +1855,51 @@ out_xsk_arr:
 
 static void ifobject_delete(struct ifobject *ifobj)
 {
+	if (ifobj->ns_fd != -1)
+		close(ifobj->ns_fd);
 	free(ifobj->umem);
 	free(ifobj->xsk_arr);
 	free(ifobj);
 }
 
+static bool is_xdp_supported(struct ifobject *ifobject)
+{
+	int flags = XDP_FLAGS_DRV_MODE;
+
+	LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
+	struct bpf_insn insns[2] = {
+		BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
+		BPF_EXIT_INSN()
+	};
+	int ifindex = if_nametoindex(ifobject->ifname);
+	int prog_fd, insn_cnt = ARRAY_SIZE(insns);
+	int err;
+
+	prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
+	if (prog_fd < 0)
+		return false;
+
+	err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
+	if (err) {
+		close(prog_fd);
+		return false;
+	}
+
+	bpf_xdp_detach(ifindex, flags, NULL);
+	close(prog_fd);
+
+	return true;
+}
+
 int main(int argc, char **argv)
 {
-	struct pkt_stream *pkt_stream_default;
+	struct pkt_stream *rx_pkt_stream_default;
+	struct pkt_stream *tx_pkt_stream_default;
 	struct ifobject *ifobj_tx, *ifobj_rx;
+	int modes = TEST_MODE_SKB + 1;
 	u32 i, j, failed_tests = 0;
 	struct test_spec test;
+	bool shared_umem;
 
 	/* Use libbpf 1.0 API mode */
 	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
@@ -1642,6 +1914,10 @@ int main(int argc, char **argv)
 	setlocale(LC_ALL, "");
 
 	parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
+	shared_umem = !strcmp(ifobj_tx->ifname, ifobj_rx->ifname);
+
+	ifobj_tx->shared_umem = shared_umem;
+	ifobj_rx->shared_umem = shared_umem;
 
 	if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) {
 		usage(basename(argv[0]));
@@ -1653,15 +1929,23 @@ int main(int argc, char **argv)
 	init_iface(ifobj_rx, MAC2, MAC1, IP2, IP1, UDP_PORT2, UDP_PORT1,
 		   worker_testapp_validate_rx);
 
+	if (is_xdp_supported(ifobj_tx)) {
+		modes++;
+		if (ifobj_zc_avail(ifobj_tx))
+			modes++;
+	}
+
 	test_spec_init(&test, ifobj_tx, ifobj_rx, 0);
-	pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
-	if (!pkt_stream_default)
+	tx_pkt_stream_default = pkt_stream_generate(ifobj_tx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
+	rx_pkt_stream_default = pkt_stream_generate(ifobj_rx->umem, DEFAULT_PKT_CNT, PKT_SIZE);
+	if (!tx_pkt_stream_default || !rx_pkt_stream_default)
 		exit_with_error(ENOMEM);
-	test.pkt_stream_default = pkt_stream_default;
+	test.tx_pkt_stream_default = tx_pkt_stream_default;
+	test.rx_pkt_stream_default = rx_pkt_stream_default;
 
-	ksft_set_plan(TEST_MODE_MAX * TEST_TYPE_MAX);
+	ksft_set_plan(modes * TEST_TYPE_MAX);
 
-	for (i = 0; i < TEST_MODE_MAX; i++)
+	for (i = 0; i < modes; i++)
 		for (j = 0; j < TEST_TYPE_MAX; j++) {
 			test_spec_init(&test, ifobj_tx, ifobj_rx, i);
 			run_pkt_test(&test, i, j);
@@ -1671,7 +1955,8 @@ int main(int argc, char **argv)
 				failed_tests++;
 		}
 
-	pkt_stream_delete(pkt_stream_default);
+	pkt_stream_delete(tx_pkt_stream_default);
+	pkt_stream_delete(rx_pkt_stream_default);
 	ifobject_delete(ifobj_tx);
 	ifobject_delete(ifobj_rx);
 
diff --git a/tools/testing/selftests/bpf/xskxceiver.h b/tools/testing/selftests/bpf/xskxceiver.h
index 3d17053f98e5..edb76d2def9f 100644
--- a/tools/testing/selftests/bpf/xskxceiver.h
+++ b/tools/testing/selftests/bpf/xskxceiver.h
@@ -27,9 +27,10 @@
 
 #define TEST_PASS 0
 #define TEST_FAILURE -1
+#define TEST_CONTINUE 1
 #define MAX_INTERFACES 2
-#define MAX_INTERFACE_NAME_CHARS 7
-#define MAX_INTERFACES_NAMESPACE_CHARS 10
+#define MAX_INTERFACE_NAME_CHARS 16
+#define MAX_INTERFACES_NAMESPACE_CHARS 16
 #define MAX_SOCKETS 2
 #define MAX_TEST_NAME_SIZE 32
 #define MAX_TEARDOWN_ITER 10
@@ -48,7 +49,7 @@
 #define SOCK_RECONF_CTR 10
 #define BATCH_SIZE 64
 #define POLL_TMOUT 1000
-#define RECV_TMOUT 3
+#define THREAD_TMOUT 3
 #define DEFAULT_PKT_CNT (4 * 1024)
 #define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4)
 #define UMEM_SIZE (DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE)
@@ -61,6 +62,7 @@
 enum test_mode {
 	TEST_MODE_SKB,
 	TEST_MODE_DRV,
+	TEST_MODE_ZC,
 	TEST_MODE_MAX
 };
 
@@ -68,7 +70,10 @@ enum test_type {
 	TEST_TYPE_RUN_TO_COMPLETION,
 	TEST_TYPE_RUN_TO_COMPLETION_2K_FRAME,
 	TEST_TYPE_RUN_TO_COMPLETION_SINGLE_PKT,
-	TEST_TYPE_POLL,
+	TEST_TYPE_RX_POLL,
+	TEST_TYPE_TX_POLL,
+	TEST_TYPE_POLL_RXQ_TMOUT,
+	TEST_TYPE_POLL_TXQ_TMOUT,
 	TEST_TYPE_UNALIGNED,
 	TEST_TYPE_ALIGNED_INV_DESC,
 	TEST_TYPE_ALIGNED_INV_DESC_2K_FRAME,
@@ -95,6 +100,7 @@ struct xsk_umem_info {
 	u32 frame_headroom;
 	void *buffer;
 	u32 frame_size;
+	u32 base_addr;
 	bool unaligned_mode;
 };
 
@@ -148,6 +154,7 @@ struct ifobject {
 	bool busy_poll;
 	bool use_fill_ring;
 	bool release_rx;
+	bool shared_umem;
 	u8 dst_mac[ETH_ALEN];
 	u8 src_mac[ETH_ALEN];
 };
@@ -155,11 +162,13 @@ struct ifobject {
 struct test_spec {
 	struct ifobject *ifobj_tx;
 	struct ifobject *ifobj_rx;
-	struct pkt_stream *pkt_stream_default;
+	struct pkt_stream *tx_pkt_stream_default;
+	struct pkt_stream *rx_pkt_stream_default;
 	u16 total_steps;
 	u16 current_step;
 	u16 nb_sockets;
 	bool fail;
+	enum test_mode mode;
 	char name[MAX_TEST_NAME_SIZE];
 };
 
diff --git a/tools/testing/selftests/cpu-hotplug/Makefile b/tools/testing/selftests/cpu-hotplug/Makefile
index d8be047ee5b6..8b66c4738344 100644
--- a/tools/testing/selftests/cpu-hotplug/Makefile
+++ b/tools/testing/selftests/cpu-hotplug/Makefile
@@ -6,6 +6,6 @@ TEST_PROGS := cpu-on-off-test.sh
 include ../lib.mk
 
 run_full_test:
-	@/bin/bash ./cpu-on-off-test.sh -a || echo "cpu-hotplug selftests: [FAIL]"
+	@/bin/bash ./cpu-on-off-test.sh -a && echo "cpu-hotplug selftests: [PASS]" || echo "cpu-hotplug selftests: [FAIL]"
 
 clean:
diff --git a/tools/testing/selftests/cpu-hotplug/config b/tools/testing/selftests/cpu-hotplug/config
deleted file mode 100644
index d4aca2ad5069..000000000000
--- a/tools/testing/selftests/cpu-hotplug/config
+++ /dev/null
@@ -1 +0,0 @@
-CONFIG_NOTIFIER_ERROR_INJECTION=y
diff --git a/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh b/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh
index 0d26b5e3f966..d5dc7e0dc726 100755
--- a/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh
+++ b/tools/testing/selftests/cpu-hotplug/cpu-on-off-test.sh
@@ -4,6 +4,7 @@
 SYSFS=
 # Kselftest framework requirement - SKIP code is 4.
 ksft_skip=4
+retval=0
 
 prerequisite()
 {
@@ -102,10 +103,10 @@ online_cpu_expect_success()
 
 	if ! online_cpu $cpu; then
 		echo $FUNCNAME $cpu: unexpected fail >&2
-		exit 1
+		retval=1
 	elif ! cpu_is_online $cpu; then
 		echo $FUNCNAME $cpu: unexpected offline >&2
-		exit 1
+		retval=1
 	fi
 }
 
@@ -115,10 +116,10 @@ online_cpu_expect_fail()
 
 	if online_cpu $cpu 2> /dev/null; then
 		echo $FUNCNAME $cpu: unexpected success >&2
-		exit 1
+		retval=1
 	elif ! cpu_is_offline $cpu; then
 		echo $FUNCNAME $cpu: unexpected online >&2
-		exit 1
+		retval=1
 	fi
 }
 
@@ -128,10 +129,10 @@ offline_cpu_expect_success()
 
 	if ! offline_cpu $cpu; then
 		echo $FUNCNAME $cpu: unexpected fail >&2
-		exit 1
+		retval=1
 	elif ! cpu_is_offline $cpu; then
 		echo $FUNCNAME $cpu: unexpected offline >&2
-		exit 1
+		retval=1
 	fi
 }
 
@@ -141,16 +142,33 @@ offline_cpu_expect_fail()
 
 	if offline_cpu $cpu 2> /dev/null; then
 		echo $FUNCNAME $cpu: unexpected success >&2
-		exit 1
+		retval=1
 	elif ! cpu_is_online $cpu; then
 		echo $FUNCNAME $cpu: unexpected offline >&2
-		exit 1
+		retval=1
 	fi
 }
 
-error=-12
+online_all_hot_pluggable_cpus()
+{
+	for cpu in `hotplaggable_offline_cpus`; do
+		online_cpu_expect_success $cpu
+	done
+}
+
+offline_all_hot_pluggable_cpus()
+{
+	local reserve_cpu=$online_max
+	for cpu in `hotpluggable_online_cpus`; do
+		# Reserve one cpu oneline at least.
+		if [ $cpu -eq $reserve_cpu ];then
+			continue
+		fi
+		offline_cpu_expect_success $cpu
+	done
+}
+
 allcpus=0
-priority=0
 online_cpus=0
 online_max=0
 offline_cpus=0
@@ -158,31 +176,20 @@ offline_max=0
 present_cpus=0
 present_max=0
 
-while getopts e:ahp: opt; do
+while getopts ah opt; do
 	case $opt in
-	e)
-		error=$OPTARG
-		;;
 	a)
 		allcpus=1
 		;;
 	h)
-		echo "Usage $0 [ -a ] [ -e errno ] [ -p notifier-priority ]"
+		echo "Usage $0 [ -a ]"
 		echo -e "\t default offline one cpu"
 		echo -e "\t run with -a option to offline all cpus"
 		exit
 		;;
-	p)
-		priority=$OPTARG
-		;;
 	esac
 done
 
-if ! [ "$error" -ge -4095 -a "$error" -lt 0 ]; then
-	echo "error code must be -4095 <= errno < 0" >&2
-	exit 1
-fi
-
 prerequisite
 
 #
@@ -196,12 +203,12 @@ if [ $allcpus -eq 0 ]; then
 	online_cpu_expect_success $online_max
 
 	if [[ $offline_cpus -gt 0 ]]; then
-		echo -e "\t offline to online to offline: cpu $present_max"
+		echo -e "\t online to offline to online: cpu $present_max"
 		online_cpu_expect_success $present_max
 		offline_cpu_expect_success $present_max
 		online_cpu $present_max
 	fi
-	exit 0
+	exit $retval
 else
 	echo "Full scope test: all hotplug cpus"
 	echo -e "\t online all offline cpus"
@@ -209,85 +216,10 @@ else
 	echo -e "\t online all offline cpus"
 fi
 
-#
-# Online all hot-pluggable CPUs
-#
-for cpu in `hotplaggable_offline_cpus`; do
-	online_cpu_expect_success $cpu
-done
-
-#
-# Offline all hot-pluggable CPUs
-#
-for cpu in `hotpluggable_online_cpus`; do
-	offline_cpu_expect_success $cpu
-done
-
-#
-# Online all hot-pluggable CPUs again
-#
-for cpu in `hotplaggable_offline_cpus`; do
-	online_cpu_expect_success $cpu
-done
-
-#
-# Test with cpu notifier error injection
-#
+online_all_hot_pluggable_cpus
 
-DEBUGFS=`mount -t debugfs | head -1 | awk '{ print $3 }'`
-NOTIFIER_ERR_INJECT_DIR=$DEBUGFS/notifier-error-inject/cpu
+offline_all_hot_pluggable_cpus
 
-prerequisite_extra()
-{
-	msg="skip extra tests:"
-
-	/sbin/modprobe -q -r cpu-notifier-error-inject
-	/sbin/modprobe -q cpu-notifier-error-inject priority=$priority
-
-	if [ ! -d "$DEBUGFS" ]; then
-		echo $msg debugfs is not mounted >&2
-		exit $ksft_skip
-	fi
-
-	if [ ! -d $NOTIFIER_ERR_INJECT_DIR ]; then
-		echo $msg cpu-notifier-error-inject module is not available >&2
-		exit $ksft_skip
-	fi
-}
-
-prerequisite_extra
-
-#
-# Offline all hot-pluggable CPUs
-#
-echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/CPU_DOWN_PREPARE/error
-for cpu in `hotpluggable_online_cpus`; do
-	offline_cpu_expect_success $cpu
-done
-
-#
-# Test CPU hot-add error handling (offline => online)
-#
-echo $error > $NOTIFIER_ERR_INJECT_DIR/actions/CPU_UP_PREPARE/error
-for cpu in `hotplaggable_offline_cpus`; do
-	online_cpu_expect_fail $cpu
-done
-
-#
-# Online all hot-pluggable CPUs
-#
-echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/CPU_UP_PREPARE/error
-for cpu in `hotplaggable_offline_cpus`; do
-	online_cpu_expect_success $cpu
-done
-
-#
-# Test CPU hot-remove error handling (online => offline)
-#
-echo $error > $NOTIFIER_ERR_INJECT_DIR/actions/CPU_DOWN_PREPARE/error
-for cpu in `hotpluggable_online_cpus`; do
-	offline_cpu_expect_fail $cpu
-done
+online_all_hot_pluggable_cpus
 
-echo 0 > $NOTIFIER_ERR_INJECT_DIR/actions/CPU_DOWN_PREPARE/error
-/sbin/modprobe -q -r cpu-notifier-error-inject
+exit $retval
diff --git a/tools/testing/selftests/drivers/net/bonding/Makefile b/tools/testing/selftests/drivers/net/bonding/Makefile
index 1d866658e541..e9dab5f9d773 100644
--- a/tools/testing/selftests/drivers/net/bonding/Makefile
+++ b/tools/testing/selftests/drivers/net/bonding/Makefile
@@ -1,9 +1,11 @@
 # SPDX-License-Identifier: GPL-2.0
 # Makefile for net selftests
 
-TEST_PROGS := bond-break-lacpdu-tx.sh \
-	      dev_addr_lists.sh \
-	      bond-arp-interval-causes-panic.sh
+TEST_PROGS := \
+	bond-arp-interval-causes-panic.sh \
+	bond-break-lacpdu-tx.sh \
+	bond-lladdr-target.sh \
+	dev_addr_lists.sh
 
 TEST_FILES := lag_lib.sh
 
diff --git a/tools/testing/selftests/drivers/net/bonding/bond-lladdr-target.sh b/tools/testing/selftests/drivers/net/bonding/bond-lladdr-target.sh
new file mode 100755
index 000000000000..89af402fabbe
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/bonding/bond-lladdr-target.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+# Regression Test:
+#   Verify bond interface could up when set IPv6 link local address target.
+#
+#  +----------------+
+#  |      br0       |
+#  |       |        |    sw
+#  | veth0   veth1  |
+#  +---+-------+----+
+#      |       |
+#  +---+-------+----+
+#  | veth0   veth1  |
+#  |       |        |    host
+#  |     bond0      |
+#  +----------------+
+#
+# We use veths instead of physical interfaces
+sw="sw-$(mktemp -u XXXXXX)"
+host="ns-$(mktemp -u XXXXXX)"
+
+cleanup()
+{
+	ip netns del $sw
+	ip netns del $host
+}
+
+trap cleanup 0 1 2
+
+ip netns add $sw
+ip netns add $host
+
+ip -n $host link add veth0 type veth peer name veth0 netns $sw
+ip -n $host link add veth1 type veth peer name veth1 netns $sw
+
+ip -n $sw link add br0 type bridge
+ip -n $sw link set br0 up
+sw_lladdr=$(ip -n $sw addr show br0 | awk '/fe80/{print $2}' | cut -d'/' -f1)
+# sleep some time to make sure bridge lladdr pass DAD
+sleep 2
+
+ip -n $host link add bond0 type bond mode 1 ns_ip6_target ${sw_lladdr} \
+	arp_validate 3 arp_interval 1000
+# add a lladdr for bond to make sure there is a route to target
+ip -n $host addr add fe80::beef/64 dev bond0
+ip -n $host link set bond0 up
+ip -n $host link set veth0 master bond0
+ip -n $host link set veth1 master bond0
+
+ip -n $sw link set veth0 master br0
+ip -n $sw link set veth1 master br0
+ip -n $sw link set veth0 up
+ip -n $sw link set veth1 up
+
+sleep 5
+
+rc=0
+if ip -n $host link show bond0 | grep -q LOWER_UP; then
+	echo "PASS"
+else
+	echo "FAIL"
+	rc=1
+fi
+exit $rc
diff --git a/tools/testing/selftests/drivers/net/dsa/Makefile b/tools/testing/selftests/drivers/net/dsa/Makefile
index 2a731d5c6d85..c393e7b73805 100644
--- a/tools/testing/selftests/drivers/net/dsa/Makefile
+++ b/tools/testing/selftests/drivers/net/dsa/Makefile
@@ -8,9 +8,10 @@ TEST_PROGS = bridge_locked_port.sh \
 	bridge_vlan_unaware.sh \
 	local_termination.sh \
 	no_forwarding.sh \
+	tc_actions.sh \
 	test_bridge_fdb_stress.sh
 
-TEST_PROGS_EXTENDED := lib.sh
+TEST_PROGS_EXTENDED := lib.sh tc_common.sh
 
 TEST_FILES := forwarding.config
 
diff --git a/tools/testing/selftests/drivers/net/dsa/tc_actions.sh b/tools/testing/selftests/drivers/net/dsa/tc_actions.sh
new file mode 120000
index 000000000000..306213d9430e
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/dsa/tc_actions.sh
@@ -0,0 +1 @@
+../../../net/forwarding/tc_actions.sh
\ No newline at end of file
diff --git a/tools/testing/selftests/drivers/net/dsa/tc_common.sh b/tools/testing/selftests/drivers/net/dsa/tc_common.sh
new file mode 120000
index 000000000000..bc3465bdc36b
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/dsa/tc_common.sh
@@ -0,0 +1 @@
+../../../net/forwarding/tc_common.sh
\ No newline at end of file
diff --git a/tools/testing/selftests/drivers/net/mlxsw/egress_vid_classification.sh b/tools/testing/selftests/drivers/net/mlxsw/egress_vid_classification.sh
new file mode 100755
index 000000000000..0cf9e47e3209
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/mlxsw/egress_vid_classification.sh
@@ -0,0 +1,273 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+# Test VLAN classification after routing and verify that the order of
+# configuration does not impact switch behavior. Verify that {RIF, Port}->VID
+# mapping is added correctly for existing {Port, VID}->FID mapping and that
+# {RIF, Port}->VID mapping is added correctly for new {Port, VID}->FID mapping.
+
+# +-------------------+                   +--------------------+
+# | H1                |                   | H2                 |
+# |                   |                   |                    |
+# |         $h1.10 +  |                   |  + $h2.10          |
+# |   192.0.2.1/28 |  |                   |  | 192.0.2.3/28    |
+# |                |  |                   |  |                 |
+# |            $h1 +  |                   |  + $h2             |
+# +----------------|--+                   +--|-----------------+
+#                  |                         |
+# +----------------|-------------------------|-----------------+
+# | SW             |                         |                 |
+# | +--------------|-------------------------|---------------+ |
+# | |        $swp1 +                         + $swp2         | |
+# | |              |                         |               | |
+# | |     $swp1.10 +                         + $swp2.10      | |
+# | |                                                        | |
+# | |                           br0                          | |
+# | |                       192.0.2.2/28                     | |
+# | +--------------------------------------------------------+ |
+# |                                                            |
+# |      $swp3.20 +                                            |
+# | 192.0.2.17/28 |                                            |
+# |               |                                            |
+# |         $swp3 +                                            |
+# +---------------|--------------------------------------------+
+#                 |
+# +---------------|--+
+# |           $h3 +  |
+# |               |  |
+# |        $h3.20 +  |
+# | 192.0.2.18/28    |
+# |                  |
+# | H3               |
+# +------------------+
+
+lib_dir=$(dirname $0)/../../../net/forwarding
+
+ALL_TESTS="
+	port_vid_map_rif
+	rif_port_vid_map
+"
+
+NUM_NETIFS=6
+source $lib_dir/lib.sh
+source $lib_dir/tc_common.sh
+source $lib_dir/devlink_lib.sh
+
+h1_create()
+{
+	simple_if_init $h1
+	vlan_create $h1 10 v$h1 192.0.2.1/28
+
+	ip route add 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2
+}
+
+h1_destroy()
+{
+	ip route del 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2
+
+	vlan_destroy $h1 10
+	simple_if_fini $h1
+}
+
+h2_create()
+{
+	simple_if_init $h2
+	vlan_create $h2 10 v$h2 192.0.2.3/28
+}
+
+h2_destroy()
+{
+	vlan_destroy $h2 10
+	simple_if_fini $h2
+}
+
+h3_create()
+{
+	simple_if_init $h3
+	vlan_create $h3 20 v$h3 192.0.2.18/28
+
+	ip route add 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17
+}
+
+h3_destroy()
+{
+	ip route del 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17
+
+	vlan_destroy $h3 20
+	simple_if_fini $h3
+}
+
+switch_create()
+{
+	ip link set dev $swp1 up
+	tc qdisc add dev $swp1 clsact
+
+	ip link add dev br0 type bridge mcast_snooping 0
+
+	# By default, a link-local address is generated when netdevice becomes
+	# up. Adding an address to the bridge will cause creating a RIF for it.
+	# Prevent generating link-local address to be able to control when the
+	# RIF is added.
+	sysctl_set net.ipv6.conf.br0.addr_gen_mode 1
+	ip link set dev br0 up
+
+	ip link set dev $swp2 up
+	vlan_create $swp2 10
+	ip link set dev $swp2.10 master br0
+
+	ip link set dev $swp3 up
+	vlan_create $swp3 20 "" 192.0.2.17/28
+
+	# Replace neighbor to avoid 1 packet which is forwarded in software due
+	# to "unresolved neigh".
+	ip neigh replace dev $swp3.20 192.0.2.18 lladdr $(mac_get $h3.20)
+}
+
+switch_destroy()
+{
+	vlan_destroy $swp3 20
+	ip link set dev $swp3 down
+
+	ip link set dev $swp2.10 nomaster
+	vlan_destroy $swp2 10
+	ip link set dev $swp2 down
+
+	ip link set dev br0 down
+	sysctl_restore net.ipv6.conf.br0.addr_gen_mode
+	ip link del dev br0
+
+	tc qdisc del dev $swp1 clsact
+	ip link set dev $swp1 down
+}
+
+setup_prepare()
+{
+	h1=${NETIFS[p1]}
+	swp1=${NETIFS[p2]}
+
+	swp2=${NETIFS[p3]}
+	h2=${NETIFS[p4]}
+
+	swp3=${NETIFS[p5]}
+	h3=${NETIFS[p6]}
+
+	vrf_prepare
+	forwarding_enable
+
+	h1_create
+	h2_create
+	h3_create
+
+	switch_create
+}
+
+cleanup()
+{
+	pre_cleanup
+
+	switch_destroy
+
+	h3_destroy
+	h2_destroy
+	h1_destroy
+
+	forwarding_restore
+	vrf_cleanup
+}
+
+bridge_rif_add()
+{
+	rifs_occ_t0=$(devlink_resource_occ_get rifs)
+	__addr_add_del br0 add 192.0.2.2/28
+	rifs_occ_t1=$(devlink_resource_occ_get rifs)
+
+	expected_rifs=$((rifs_occ_t0 + 1))
+
+	[[ $expected_rifs -eq $rifs_occ_t1 ]]
+	check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used"
+
+	sleep 1
+}
+
+bridge_rif_del()
+{
+	__addr_add_del br0 del 192.0.2.2/28
+}
+
+port_vid_map_rif()
+{
+	RET=0
+
+	# First add {port, VID}->FID for swp1.10, then add a RIF and verify that
+	# packets get the correct VID after routing.
+	vlan_create $swp1 10
+	ip link set dev $swp1.10 master br0
+	bridge_rif_add
+
+	# Replace neighbor to avoid 1 packet which is forwarded in software due
+	# to "unresolved neigh".
+	ip neigh replace dev br0 192.0.2.1 lladdr $(mac_get $h1.10)
+
+	# The hardware matches on the first ethertype which is not VLAN,
+	# so the protocol should be IP.
+	tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \
+		flower skip_sw dst_ip 192.0.2.1 action pass
+
+	ping_do $h1.10 192.0.2.18
+	check_err $? "Ping failed"
+
+	tc_check_at_least_x_packets "dev $swp1 egress" 101 10
+	check_err $? "Packets were not routed in hardware"
+
+	log_test "Add RIF for existing {port, VID}->FID mapping"
+
+	tc filter del dev $swp1 egress
+
+	bridge_rif_del
+	ip link set dev $swp1.10 nomaster
+	vlan_destroy $swp1 10
+}
+
+rif_port_vid_map()
+{
+	RET=0
+
+	# First add an address to the bridge, which will create a RIF on top of
+	# it, then add a new {port, VID}->FID mapping and verify that packets
+	# get the correct VID after routing.
+	bridge_rif_add
+	vlan_create $swp1 10
+	ip link set dev $swp1.10 master br0
+
+	# Replace neighbor to avoid 1 packet which is forwarded in software due
+	# to "unresolved neigh".
+	ip neigh replace dev br0 192.0.2.1 lladdr $(mac_get $h1.10)
+
+	# The hardware matches on the first ethertype which is not VLAN,
+	# so the protocol should be IP.
+	tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \
+		flower skip_sw dst_ip 192.0.2.1 action pass
+
+	ping_do $h1.10 192.0.2.18
+	check_err $? "Ping failed"
+
+	tc_check_at_least_x_packets "dev $swp1 egress" 101 10
+	check_err $? "Packets were not routed in hardware"
+
+	log_test "Add {port, VID}->FID mapping for FID with a RIF"
+
+	tc filter del dev $swp1 egress
+
+	ip link set dev $swp1.10 nomaster
+	vlan_destroy $swp1 10
+	bridge_rif_del
+}
+
+trap cleanup EXIT
+
+setup_prepare
+setup_wait
+
+tests_run
+
+exit $EXIT_STATUS
diff --git a/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1d.sh b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1d.sh
new file mode 100755
index 000000000000..df2b09966886
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1d.sh
@@ -0,0 +1,264 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+# Test routing over bridge and verify that the order of configuration does not
+# impact switch behavior. Verify that RIF is added correctly for existing
+# mappings and that new mappings use the correct RIF.
+
+# +-------------------+                   +--------------------+
+# | H1                |                   | H2                 |
+# |                   |                   |                    |
+# |         $h1.10 +  |                   |  + $h2.10          |
+# |   192.0.2.1/28 |  |                   |  | 192.0.2.3/28    |
+# |                |  |                   |  |                 |
+# |            $h1 +  |                   |  + $h2             |
+# +----------------|--+                   +--|-----------------+
+#                  |                         |
+# +----------------|-------------------------|-----------------+
+# | SW             |                         |                 |
+# | +--------------|-------------------------|---------------+ |
+# | |        $swp1 +                         + $swp2         | |
+# | |              |                         |               | |
+# | |     $swp1.10 +                         + $swp2.10      | |
+# | |                                                        | |
+# | |                           br0                          | |
+# | |                       192.0.2.2/28                     | |
+# | +--------------------------------------------------------+ |
+# |                                                            |
+# |      $swp3.10 +                                            |
+# | 192.0.2.17/28 |                                            |
+# |               |                                            |
+# |         $swp3 +                                            |
+# +---------------|--------------------------------------------+
+#                 |
+# +---------------|--+
+# |           $h3 +  |
+# |               |  |
+# |        $h3.10 +  |
+# | 192.0.2.18/28    |
+# |                  |
+# | H3               |
+# +------------------+
+
+lib_dir=$(dirname $0)/../../../net/forwarding
+
+ALL_TESTS="
+	port_vid_map_rif
+	rif_port_vid_map
+"
+
+NUM_NETIFS=6
+source $lib_dir/lib.sh
+source $lib_dir/tc_common.sh
+source $lib_dir/devlink_lib.sh
+
+h1_create()
+{
+	simple_if_init $h1
+	vlan_create $h1 10 v$h1 192.0.2.1/28
+
+	ip route add 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2
+}
+
+h1_destroy()
+{
+	ip route del 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2
+
+	vlan_destroy $h1 10
+	simple_if_fini $h1
+}
+
+h2_create()
+{
+	simple_if_init $h2
+	vlan_create $h2 10 v$h2 192.0.2.3/28
+}
+
+h2_destroy()
+{
+	vlan_destroy $h2 10
+	simple_if_fini $h2
+}
+
+h3_create()
+{
+	simple_if_init $h3
+	vlan_create $h3 10 v$h3 192.0.2.18/28
+
+	ip route add 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17
+}
+
+h3_destroy()
+{
+	ip route del 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17
+
+	vlan_destroy $h3 10
+	simple_if_fini $h3
+}
+
+switch_create()
+{
+	ip link set dev $swp1 up
+
+	ip link add dev br0 type bridge mcast_snooping 0
+
+	# By default, a link-local address is generated when netdevice becomes
+	# up. Adding an address to the bridge will cause creating a RIF for it.
+	# Prevent generating link-local address to be able to control when the
+	# RIF is added.
+	sysctl_set net.ipv6.conf.br0.addr_gen_mode 1
+	ip link set dev br0 up
+
+	ip link set dev $swp2 up
+	vlan_create $swp2 10
+	ip link set dev $swp2.10 master br0
+
+	ip link set dev $swp3 up
+	vlan_create $swp3 10 "" 192.0.2.17/28
+	tc qdisc add dev $swp3 clsact
+
+	# Replace neighbor to avoid 1 packet which is forwarded in software due
+	# to "unresolved neigh".
+	ip neigh replace dev $swp3.10 192.0.2.18 lladdr $(mac_get $h3.10)
+}
+
+switch_destroy()
+{
+	tc qdisc del dev $swp3 clsact
+	vlan_destroy $swp3 10
+	ip link set dev $swp3 down
+
+	ip link set dev $swp2.10 nomaster
+	vlan_destroy $swp2 10
+	ip link set dev $swp2 down
+
+	ip link set dev br0 down
+	sysctl_restore net.ipv6.conf.br0.addr_gen_mode
+	ip link del dev br0
+
+	ip link set dev $swp1 down
+}
+
+setup_prepare()
+{
+	h1=${NETIFS[p1]}
+	swp1=${NETIFS[p2]}
+
+	swp2=${NETIFS[p3]}
+	h2=${NETIFS[p4]}
+
+	swp3=${NETIFS[p5]}
+	h3=${NETIFS[p6]}
+
+	vrf_prepare
+	forwarding_enable
+
+	h1_create
+	h2_create
+	h3_create
+
+	switch_create
+}
+
+cleanup()
+{
+	pre_cleanup
+
+	switch_destroy
+
+	h3_destroy
+	h2_destroy
+	h1_destroy
+
+	forwarding_restore
+	vrf_cleanup
+}
+
+bridge_rif_add()
+{
+	rifs_occ_t0=$(devlink_resource_occ_get rifs)
+	__addr_add_del br0 add 192.0.2.2/28
+	rifs_occ_t1=$(devlink_resource_occ_get rifs)
+
+	expected_rifs=$((rifs_occ_t0 + 1))
+
+	[[ $expected_rifs -eq $rifs_occ_t1 ]]
+	check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used"
+
+	sleep 1
+}
+
+bridge_rif_del()
+{
+	__addr_add_del br0 del 192.0.2.2/28
+}
+
+port_vid_map_rif()
+{
+	RET=0
+
+	# First add {port, VID}->FID for $swp1.10, then add a RIF and verify
+	# that packets can be routed via the existing mapping.
+	vlan_create $swp1 10
+	ip link set dev $swp1.10 master br0
+	bridge_rif_add
+
+	# The hardware matches on the first ethertype which is not VLAN,
+	# so the protocol should be IP.
+	tc filter add dev $swp3 egress protocol ip pref 1 handle 101 \
+		flower skip_sw dst_ip 192.0.2.18 action pass
+
+	ping_do $h1.10 192.0.2.18
+	check_err $? "Ping failed"
+
+	tc_check_at_least_x_packets "dev $swp3 egress" 101 10
+	check_err $? "Packets were not routed in hardware"
+
+	log_test "Add RIF for existing {port, VID}->FID mapping"
+
+	tc filter del dev $swp3 egress
+
+	bridge_rif_del
+	ip link set dev $swp1.10 nomaster
+	vlan_destroy $swp1 10
+}
+
+rif_port_vid_map()
+{
+	RET=0
+
+	# First add an address to the bridge, which will create a RIF on top of
+	# it, then add a new {port, VID}->FID mapping and verify that packets
+	# can be routed via the new mapping.
+	bridge_rif_add
+	vlan_create $swp1 10
+	ip link set dev $swp1.10 master br0
+
+	# The hardware matches on the first ethertype which is not VLAN,
+	# so the protocol should be IP.
+	tc filter add dev $swp3 egress protocol ip pref 1 handle 101 \
+		flower skip_sw dst_ip 192.0.2.18 action pass
+
+	ping_do $h1.10 192.0.2.18
+	check_err $? "Ping failed"
+
+	tc_check_at_least_x_packets "dev $swp3 egress" 101 10
+	check_err $? "Packets were not routed in hardware"
+
+	log_test "Add {port, VID}->FID mapping for FID with a RIF"
+
+	tc filter del dev $swp3 egress
+
+	ip link set dev $swp1.10 nomaster
+	vlan_destroy $swp1 10
+	bridge_rif_del
+}
+
+trap cleanup EXIT
+
+setup_prepare
+setup_wait
+
+tests_run
+
+exit $EXIT_STATUS
diff --git a/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1q.sh b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1q.sh
new file mode 100755
index 000000000000..577293bab88b
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_1q.sh
@@ -0,0 +1,264 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+# Test routing over bridge and verify that the order of configuration does not
+# impact switch behavior. Verify that RIF is added correctly for existing
+# mapping and that packets can be routed via port which is added after the FID
+# already has a RIF.
+
+# +-------------------+                   +--------------------+
+# | H1                |                   | H2                 |
+# |                   |                   |                    |
+# |         $h1.10 +  |                   |  + $h2.10          |
+# |   192.0.2.1/28 |  |                   |  | 192.0.2.3/28    |
+# |                |  |                   |  |                 |
+# |            $h1 +  |                   |  + $h2             |
+# +----------------|--+                   +--|-----------------+
+#                  |                         |
+# +----------------|-------------------------|-----------------+
+# | SW             |                         |                 |
+# | +--------------|-------------------------|---------------+ |
+# | |        $swp1 +                         + $swp2         | |
+# | |                                                        | |
+# | |                           br0                          | |
+# | +--------------------------------------------------------+ |
+# |                              |                             |
+# |                           br0.10                           |
+# |                        192.0.2.2/28                        |
+# |                                                            |
+# |                                                            |
+# |          $swp3 +                                           |
+# |  192.0.2.17/28 |                                           |
+# +----------------|-------------------------------------------+
+#                  |
+# +----------------|--+
+# |            $h3 +  |
+# |  192.0.2.18/28    |
+# |                   |
+# | H3                |
+# +-------------------+
+
+lib_dir=$(dirname $0)/../../../net/forwarding
+
+ALL_TESTS="
+	vid_map_rif
+	rif_vid_map
+"
+
+NUM_NETIFS=6
+source $lib_dir/lib.sh
+source $lib_dir/tc_common.sh
+source $lib_dir/devlink_lib.sh
+
+h1_create()
+{
+	simple_if_init $h1
+	vlan_create $h1 10 v$h1 192.0.2.1/28
+
+	ip route add 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2
+}
+
+h1_destroy()
+{
+	ip route del 192.0.2.16/28 vrf v$h1 nexthop via 192.0.2.2
+
+	vlan_destroy $h1 10
+	simple_if_fini $h1
+}
+
+h2_create()
+{
+	simple_if_init $h2
+	vlan_create $h2 10 v$h2 192.0.2.3/28
+}
+
+h2_destroy()
+{
+	vlan_destroy $h2 10
+	simple_if_fini $h2
+}
+
+h3_create()
+{
+	simple_if_init $h3 192.0.2.18/28
+	ip route add 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17
+}
+
+h3_destroy()
+{
+	ip route del 192.0.2.0/28 vrf v$h3 nexthop via 192.0.2.17
+	simple_if_fini $h3 192.0.2.18/28
+}
+
+switch_create()
+{
+	ip link set dev $swp1 up
+
+	ip link add dev br0 type bridge vlan_filtering 1 mcast_snooping 0
+
+	# By default, a link-local address is generated when netdevice becomes
+	# up. Adding an address to the bridge will cause creating a RIF for it.
+	# Prevent generating link-local address to be able to control when the
+	# RIF is added.
+	sysctl_set net.ipv6.conf.br0.addr_gen_mode 1
+	ip link set dev br0 up
+
+	ip link set dev $swp2 up
+	ip link set dev $swp2 master br0
+	bridge vlan add vid 10 dev $swp2
+
+	ip link set dev $swp3 up
+	__addr_add_del $swp3 add 192.0.2.17/28
+	tc qdisc add dev $swp3 clsact
+
+	# Replace neighbor to avoid 1 packet which is forwarded in software due
+	# to "unresolved neigh".
+	ip neigh replace dev $swp3 192.0.2.18 lladdr $(mac_get $h3)
+}
+
+switch_destroy()
+{
+	tc qdisc del dev $swp3 clsact
+	__addr_add_del $swp3 del 192.0.2.17/28
+	ip link set dev $swp3 down
+
+	bridge vlan del vid 10 dev $swp2
+	ip link set dev $swp2 nomaster
+	ip link set dev $swp2 down
+
+	ip link set dev br0 down
+	sysctl_restore net.ipv6.conf.br0.addr_gen_mode
+	ip link del dev br0
+
+	ip link set dev $swp1 down
+}
+
+setup_prepare()
+{
+	h1=${NETIFS[p1]}
+	swp1=${NETIFS[p2]}
+
+	swp2=${NETIFS[p3]}
+	h2=${NETIFS[p4]}
+
+	swp3=${NETIFS[p5]}
+	h3=${NETIFS[p6]}
+
+	vrf_prepare
+	forwarding_enable
+
+	h1_create
+	h2_create
+	h3_create
+
+	switch_create
+}
+
+cleanup()
+{
+	pre_cleanup
+
+	switch_destroy
+
+	h3_destroy
+	h2_destroy
+	h1_destroy
+
+	forwarding_restore
+	vrf_cleanup
+}
+
+bridge_rif_add()
+{
+	rifs_occ_t0=$(devlink_resource_occ_get rifs)
+	vlan_create br0 10 "" 192.0.2.2/28
+	rifs_occ_t1=$(devlink_resource_occ_get rifs)
+
+	expected_rifs=$((rifs_occ_t0 + 1))
+
+	[[ $expected_rifs -eq $rifs_occ_t1 ]]
+	check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used"
+
+	sleep 1
+}
+
+bridge_rif_del()
+{
+	vlan_destroy br0 10
+}
+
+vid_map_rif()
+{
+	RET=0
+
+	# First add VID->FID for vlan 10, then add a RIF and verify that
+	# packets can be routed via the existing mapping.
+	bridge vlan add vid 10 dev br0 self
+	ip link set dev $swp1 master br0
+	bridge vlan add vid 10 dev $swp1
+
+	bridge_rif_add
+
+	tc filter add dev $swp3 egress protocol ip pref 1 handle 101 \
+		flower skip_sw dst_ip 192.0.2.18 action pass
+
+	ping_do $h1.10 192.0.2.18
+	check_err $? "Ping failed"
+
+	tc_check_at_least_x_packets "dev $swp3 egress" 101 10
+	check_err $? "Packets were not routed in hardware"
+
+	log_test "Add RIF for existing VID->FID mapping"
+
+	tc filter del dev $swp3 egress
+
+	bridge_rif_del
+
+	bridge vlan del vid 10 dev $swp1
+	ip link set dev $swp1 nomaster
+	bridge vlan del vid 10 dev br0 self
+}
+
+rif_vid_map()
+{
+	RET=0
+
+	# Using 802.1Q, there is only one VID->FID map for each VID. That means
+	# that we cannot really check adding a new map for existing FID with a
+	# RIF. Verify that packets can be routed via port which is added after
+	# the FID already has a RIF, although in practice there is no new
+	# mapping in the hardware.
+	bridge vlan add vid 10 dev br0 self
+	bridge_rif_add
+
+	ip link set dev $swp1 master br0
+	bridge vlan add vid 10 dev $swp1
+
+	tc filter add dev $swp3 egress protocol ip pref 1 handle 101 \
+		flower skip_sw dst_ip 192.0.2.18 action pass
+
+	ping_do $h1.10 192.0.2.18
+	check_err $? "Ping failed"
+
+	tc_check_at_least_x_packets "dev $swp3 egress" 101 10
+	check_err $? "Packets were not routed in hardware"
+
+	log_test "Add port to VID->FID mapping for FID with a RIF"
+
+	tc filter del dev $swp3 egress
+
+	bridge vlan del vid 10 dev $swp1
+	ip link set dev $swp1 nomaster
+
+	bridge_rif_del
+	bridge vlan del vid 10 dev br0 self
+}
+
+trap cleanup EXIT
+
+setup_prepare
+setup_wait
+
+tests_run
+
+exit $EXIT_STATUS
diff --git a/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_vxlan.sh b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_vxlan.sh
new file mode 100755
index 000000000000..90450216a10d
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/mlxsw/ingress_rif_conf_vxlan.sh
@@ -0,0 +1,311 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+# Test routing after VXLAN decapsulation and verify that the order of
+# configuration does not impact switch behavior. Verify that RIF is added
+# correctly for existing mapping and that new mapping uses the correct RIF.
+
+# +---------------------------+
+# |                        H1 |
+# |    + $h1                  |
+# |    | 192.0.2.1/28         |
+# +----|----------------------+
+#      |
+# +----|----------------------------------------------------------------------+
+# | SW |                                                                      |
+# | +--|--------------------------------------------------------------------+ |
+# | |  + $swp1                         br1                                  | |
+# | |     vid 10 pvid untagged                                              | |
+# | |                                                                       | |
+# | |                                                                       | |
+# | |                                            + vx4001                   | |
+# | |                                              local 192.0.2.17         | |
+# | |                                              remote 192.0.2.18        | |
+# | |                                              id 104001                | |
+# | |                                              dstport $VXPORT          | |
+# | |                                              vid 4001 pvid untagged   | |
+# | |                                                                       | |
+# | +----------------------------------+------------------------------------+ |
+# |                                    |                                      |
+# | +----------------------------------|------------------------------------+ |
+# | |                                  |                                    | |
+# | |  +-------------------------------+---------------------------------+  | |
+# | |  |                                                                 |  | |
+# | |  + vlan10                                                 vlan4001 +  | |
+# | |    192.0.2.2/28                                                       | |
+# | |                                                                       | |
+# | |                               vrf-green                               | |
+# | +-----------------------------------------------------------------------+ |
+# |                                                                           |
+# |    + $rp1                                       +lo                       |
+# |    | 198.51.100.1/24                             192.0.2.17/32            |
+# +----|----------------------------------------------------------------------+
+#      |
+# +----|--------------------------------------------------------+
+# |    |                                             v$rp2      |
+# |    + $rp2                                                   |
+# |      198.51.100.2/24                                        |
+# |                                                             |
+# +-------------------------------------------------------------+
+
+lib_dir=$(dirname $0)/../../../net/forwarding
+
+ALL_TESTS="
+	vni_fid_map_rif
+	rif_vni_fid_map
+"
+
+NUM_NETIFS=4
+source $lib_dir/lib.sh
+source $lib_dir/tc_common.sh
+source $lib_dir/devlink_lib.sh
+
+: ${VXPORT:=4789}
+export VXPORT
+
+h1_create()
+{
+	simple_if_init $h1 192.0.2.1/28
+}
+
+h1_destroy()
+{
+	simple_if_fini $h1 192.0.2.1/28
+}
+
+switch_create()
+{
+	ip link add name br1 type bridge vlan_filtering 1 vlan_default_pvid 0 \
+		mcast_snooping 0
+	# Make sure the bridge uses the MAC address of the local port and not
+	# that of the VxLAN's device.
+	ip link set dev br1 address $(mac_get $swp1)
+	ip link set dev br1 up
+
+	ip link set dev $rp1 up
+	ip address add dev $rp1 198.51.100.1/24
+
+	ip link set dev $swp1 master br1
+	ip link set dev $swp1 up
+	bridge vlan add vid 10 dev $swp1 pvid untagged
+
+	tc qdisc add dev $swp1 clsact
+
+	ip link add name vx4001 type vxlan id 104001 \
+		local 192.0.2.17 dstport $VXPORT \
+		nolearning noudpcsum tos inherit ttl 100
+	ip link set dev vx4001 up
+
+	ip link set dev vx4001 master br1
+
+	ip address add 192.0.2.17/32 dev lo
+
+	# Create SVIs.
+	vrf_create "vrf-green"
+	ip link set dev vrf-green up
+
+	ip link add link br1 name vlan10 up master vrf-green type vlan id 10
+
+	# Replace neighbor to avoid 1 packet which is forwarded in software due
+	# to "unresolved neigh".
+	ip neigh replace dev vlan10 192.0.2.1 lladdr $(mac_get $h1)
+
+	ip address add 192.0.2.2/28 dev vlan10
+
+	bridge vlan add vid 10 dev br1 self
+	bridge vlan add vid 4001 dev br1 self
+
+	sysctl_set net.ipv4.conf.all.rp_filter 0
+}
+
+switch_destroy()
+{
+	sysctl_restore net.ipv4.conf.all.rp_filter
+
+	bridge vlan del vid 4001 dev br1 self
+	bridge vlan del vid 10 dev br1 self
+
+	ip link del dev vlan10
+
+	vrf_destroy "vrf-green"
+
+	ip address del 192.0.2.17/32 dev lo
+
+	tc qdisc del dev $swp1 clsact
+
+	bridge vlan del vid 10 dev $swp1
+	ip link set dev $swp1 down
+	ip link set dev $swp1 nomaster
+
+	ip link set dev vx4001 nomaster
+
+	ip link set dev vx4001 down
+	ip link del dev vx4001
+
+	ip address del dev $rp1 198.51.100.1/24
+	ip link set dev $rp1 down
+
+	ip link set dev br1 down
+	ip link del dev br1
+}
+
+vrp2_create()
+{
+	simple_if_init $rp2 198.51.100.2/24
+
+	ip route add 192.0.2.17/32 vrf v$rp2 nexthop via 198.51.100.1
+}
+
+vrp2_destroy()
+{
+	ip route del 192.0.2.17/32 vrf v$rp2 nexthop via 198.51.100.1
+
+	simple_if_fini $rp2 198.51.100.2/24
+}
+
+setup_prepare()
+{
+	h1=${NETIFS[p1]}
+	swp1=${NETIFS[p2]}
+
+	rp1=${NETIFS[p3]}
+	rp2=${NETIFS[p4]}
+
+	vrf_prepare
+	forwarding_enable
+
+	h1_create
+	switch_create
+
+	vrp2_create
+}
+
+cleanup()
+{
+	pre_cleanup
+
+	vrp2_destroy
+
+	switch_destroy
+	h1_destroy
+
+	forwarding_restore
+	vrf_cleanup
+}
+
+payload_get()
+{
+	local dest_mac=$(mac_get vlan4001)
+	local src_mac=$(mac_get $rp1)
+
+	p=$(:
+		)"08:"$(                      : VXLAN flags
+		)"00:00:00:"$(                : VXLAN reserved
+		)"01:96:41:"$(                : VXLAN VNI : 104001
+		)"00:"$(                      : VXLAN reserved
+		)"$dest_mac:"$(               : ETH daddr
+		)"$src_mac:"$(                : ETH saddr
+		)"08:00:"$(                   : ETH type
+		)"45:"$(                      : IP version + IHL
+		)"00:"$(                      : IP TOS
+		)"00:54:"$(                   : IP total length
+		)"3f:49:"$(                   : IP identification
+		)"00:00:"$(                   : IP flags + frag off
+		)"3f:"$(                      : IP TTL
+		)"01:"$(                      : IP proto
+		)"50:21:"$(                   : IP header csum
+		)"c6:33:64:0a:"$(             : IP saddr: 198.51.100.10
+		)"c0:00:02:01:"$(             : IP daddr: 192.0.2.1
+	)
+	echo $p
+}
+
+vlan_rif_add()
+{
+	rifs_occ_t0=$(devlink_resource_occ_get rifs)
+
+	ip link add link br1 name vlan4001 up master vrf-green \
+		type vlan id 4001
+
+	rifs_occ_t1=$(devlink_resource_occ_get rifs)
+	expected_rifs=$((rifs_occ_t0 + 1))
+
+	[[ $expected_rifs -eq $rifs_occ_t1 ]]
+	check_err $? "Expected $expected_rifs RIFs, $rifs_occ_t1 are used"
+}
+
+vlan_rif_del()
+{
+	ip link del dev vlan4001
+}
+
+vni_fid_map_rif()
+{
+	local rp1_mac=$(mac_get $rp1)
+
+	RET=0
+
+	# First add VNI->FID mapping to the FID of VLAN 4001
+	bridge vlan add vid 4001 dev vx4001 pvid untagged
+
+	# Add a RIF to the FID with VNI->FID mapping
+	vlan_rif_add
+
+	tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \
+		flower skip_sw dst_ip 192.0.2.1 action pass
+
+	payload=$(payload_get)
+	ip vrf exec v$rp2 $MZ $rp2 -c 10 -d 1msec -b $rp1_mac \
+		-B 192.0.2.17 -A 192.0.2.18 \
+		-t udp sp=12345,dp=$VXPORT,p=$payload -q
+
+	tc_check_at_least_x_packets "dev $swp1 egress" 101 10
+	check_err $? "Packets were not routed in hardware"
+
+	log_test "Add RIF for existing VNI->FID mapping"
+
+	tc filter del dev $swp1 egress
+
+	bridge vlan del vid 4001 dev vx4001 pvid untagged
+	vlan_rif_del
+}
+
+rif_vni_fid_map()
+{
+	local rp1_mac=$(mac_get $rp1)
+
+	RET=0
+
+	# First add a RIF to the FID of VLAN 4001
+	vlan_rif_add
+
+	# Add VNI->FID mapping to FID with a RIF
+	bridge vlan add vid 4001 dev vx4001 pvid untagged
+
+	tc filter add dev $swp1 egress protocol ip pref 1 handle 101 \
+		flower skip_sw dst_ip 192.0.2.1 action pass
+
+	payload=$(payload_get)
+	ip vrf exec v$rp2 $MZ $rp2 -c 10 -d 1msec -b $rp1_mac \
+		-B 192.0.2.17 -A 192.0.2.18 \
+		-t udp sp=12345,dp=$VXPORT,p=$payload -q
+
+	tc_check_at_least_x_packets "dev $swp1 egress" 101 10
+	check_err $? "Packets were not routed in hardware"
+
+	log_test "Add VNI->FID mapping for FID with a RIF"
+
+	tc filter del dev $swp1 egress
+
+	bridge vlan del vid 4001 dev vx4001 pvid untagged
+	vlan_rif_del
+}
+
+trap cleanup EXIT
+
+setup_prepare
+setup_wait
+
+tests_run
+
+exit $EXIT_STATUS
diff --git a/tools/testing/selftests/drivers/net/mlxsw/mlxsw_lib.sh b/tools/testing/selftests/drivers/net/mlxsw/mlxsw_lib.sh
index a95856aafd2a..6369927e9c37 100644
--- a/tools/testing/selftests/drivers/net/mlxsw/mlxsw_lib.sh
+++ b/tools/testing/selftests/drivers/net/mlxsw/mlxsw_lib.sh
@@ -61,3 +61,17 @@ mlxsw_only_on_spectrum()
 
 	return 1
 }
+
+mlxsw_max_descriptors_get()
+{
+	local spectrum_rev=$MLXSW_SPECTRUM_REV
+
+	case $spectrum_rev in
+	1) echo 81920 ;;
+	2) echo 136960 ;;
+	3) echo 204800 ;;
+	4) echo 220000 ;;
+	*) echo "Unknown max descriptors for chip revision." > /dev/stderr
+	   return 1 ;;
+	esac
+}
diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_burst.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_burst.sh
deleted file mode 100755
index 82a47b903f92..000000000000
--- a/tools/testing/selftests/drivers/net/mlxsw/qos_burst.sh
+++ /dev/null
@@ -1,480 +0,0 @@
-#!/bin/bash
-# SPDX-License-Identifier: GPL-2.0
-#
-# This test sends 1Gbps of traffic through the switch, into which it then
-# injects a burst of traffic and tests that there are no drops.
-#
-# The 1Gbps stream is created by sending >1Gbps stream from H1. This stream
-# ingresses through $swp1, and is forwarded thtrough a small temporary pool to a
-# 1Gbps $swp3.
-#
-# Thus a 1Gbps stream enters $swp4, and is forwarded through a large pool to
-# $swp2, and eventually to H2. Since $swp2 is a 1Gbps port as well, no backlog
-# is generated.
-#
-# At this point, a burst of traffic is forwarded from H3. This enters $swp5, is
-# forwarded to $swp2, which is fully subscribed by the 1Gbps stream. The
-# expectation is that the burst is wholly absorbed by the large pool and no
-# drops are caused. After the burst, there should be a backlog that is hard to
-# get rid of, because $sw2 is fully subscribed. But because each individual
-# packet is scheduled soon after getting enqueued, SLL and HLL do not impact the
-# test.
-#
-# +-----------------------+                           +-----------------------+
-# | H1                    |			      | H3                    |
-# |   + $h1.111           |			      |          $h3.111 +    |
-# |   | 192.0.2.33/28     |			      |    192.0.2.35/28 |    |
-# |   |                   |			      |                  |    |
-# |   + $h1               |			      |              $h3 +    |
-# +---|-------------------+  +--------------------+   +------------------|----+
-#     |                      |                    |       		 |
-# +---|----------------------|--------------------|----------------------|----+
-# |   + $swp1          $swp3 +                    + $swp4          $swp5 |    |
-# |   | iPOOL1        iPOOL0 |                    | iPOOL2        iPOOL2 |    |
-# |   | ePOOL4        ePOOL5 |                    | ePOOL4        ePOOL4 |    |
-# |   |                1Gbps |                    | 1Gbps                |    |
-# | +-|----------------------|-+                +-|----------------------|-+  |
-# | | + $swp1.111  $swp3.111 + |                | + $swp4.111  $swp5.111 + |  |
-# | |                          |                |                          |  |
-# | | BR1                      |                | BR2                      |  |
-# | |                          |                |                          |  |
-# | |                          |                |         + $swp2.111      |  |
-# | +--------------------------+                +---------|----------------+  |
-# |                                                       |                   |
-# | iPOOL0: 500KB dynamic                                 |                   |
-# | iPOOL1: 500KB dynamic                                 |                   |
-# | iPOOL2: 10MB dynamic                                  + $swp2             |
-# | ePOOL4: 500KB dynamic                                 | iPOOL0            |
-# | ePOOL5: 500KB dnamic                                  | ePOOL6            |
-# | ePOOL6: 10MB dynamic                                  | 1Gbps             |
-# +-------------------------------------------------------|-------------------+
-#                                                         |
-#                                                     +---|-------------------+
-#                                                     |   + $h2            H2 |
-#                                                     |   | 1Gbps             |
-#                                                     |   |                   |
-#                                                     |   + $h2.111           |
-#                                                     |     192.0.2.34/28     |
-#                                                     +-----------------------+
-#
-# iPOOL0+ePOOL4 are helper pools for control traffic etc.
-# iPOOL1+ePOOL5 are helper pools for modeling the 1Gbps stream
-# iPOOL2+ePOOL6 are pools for soaking the burst traffic
-
-ALL_TESTS="
-	ping_ipv4
-	test_8K
-	test_800
-"
-
-lib_dir=$(dirname $0)/../../../net/forwarding
-
-NUM_NETIFS=8
-source $lib_dir/lib.sh
-source $lib_dir/devlink_lib.sh
-source qos_lib.sh
-source mlxsw_lib.sh
-
-_1KB=1000
-_500KB=$((500 * _1KB))
-_1MB=$((1000 * _1KB))
-
-# The failure mode that this specifically tests is exhaustion of descriptor
-# buffer. The point is to produce a burst that shared buffer should be able
-# to accommodate, but produce it with small enough packets that the machine
-# runs out of the descriptor buffer space with default configuration.
-#
-# The machine therefore needs to be able to produce line rate with as small
-# packets as possible, and at the same time have large enough buffer that
-# when filled with these small packets, it runs out of descriptors.
-# Spectrum-2 is very close, but cannot perform this test. Therefore use
-# Spectrum-3 as a minimum, and permit larger burst size, and therefore
-# larger packets, to reduce spurious failures.
-#
-mlxsw_only_on_spectrum 3+ || exit
-
-BURST_SIZE=$((50000000))
-POOL_SIZE=$BURST_SIZE
-
-h1_create()
-{
-	simple_if_init $h1
-	mtu_set $h1 10000
-
-	vlan_create $h1 111 v$h1 192.0.2.33/28
-	ip link set dev $h1.111 type vlan egress-qos-map 0:1
-}
-
-h1_destroy()
-{
-	vlan_destroy $h1 111
-
-	mtu_restore $h1
-	simple_if_fini $h1
-}
-
-h2_create()
-{
-	simple_if_init $h2
-	mtu_set $h2 10000
-	ethtool -s $h2 speed 1000 autoneg off
-
-	vlan_create $h2 111 v$h2 192.0.2.34/28
-}
-
-h2_destroy()
-{
-	vlan_destroy $h2 111
-
-	ethtool -s $h2 autoneg on
-	mtu_restore $h2
-	simple_if_fini $h2
-}
-
-h3_create()
-{
-	simple_if_init $h3
-	mtu_set $h3 10000
-
-	vlan_create $h3 111 v$h3 192.0.2.35/28
-}
-
-h3_destroy()
-{
-	vlan_destroy $h3 111
-
-	mtu_restore $h3
-	simple_if_fini $h3
-}
-
-switch_create()
-{
-	# pools
-	# -----
-
-	devlink_pool_size_thtype_save 0
-	devlink_pool_size_thtype_save 4
-	devlink_pool_size_thtype_save 1
-	devlink_pool_size_thtype_save 5
-	devlink_pool_size_thtype_save 2
-	devlink_pool_size_thtype_save 6
-
-	devlink_port_pool_th_save $swp1 1
-	devlink_port_pool_th_save $swp2 6
-	devlink_port_pool_th_save $swp3 5
-	devlink_port_pool_th_save $swp4 2
-	devlink_port_pool_th_save $swp5 2
-
-	devlink_tc_bind_pool_th_save $swp1 1 ingress
-	devlink_tc_bind_pool_th_save $swp2 1 egress
-	devlink_tc_bind_pool_th_save $swp3 1 egress
-	devlink_tc_bind_pool_th_save $swp4 1 ingress
-	devlink_tc_bind_pool_th_save $swp5 1 ingress
-
-	# Control traffic pools. Just reduce the size.
-	devlink_pool_size_thtype_set 0 dynamic $_500KB
-	devlink_pool_size_thtype_set 4 dynamic $_500KB
-
-	# Stream modeling pools.
-	devlink_pool_size_thtype_set 1 dynamic $_500KB
-	devlink_pool_size_thtype_set 5 dynamic $_500KB
-
-	# Burst soak pools.
-	devlink_pool_size_thtype_set 2 static $POOL_SIZE
-	devlink_pool_size_thtype_set 6 static $POOL_SIZE
-
-	# $swp1
-	# -----
-
-	ip link set dev $swp1 up
-	mtu_set $swp1 10000
-	vlan_create $swp1 111
-	ip link set dev $swp1.111 type vlan ingress-qos-map 0:0 1:1
-
-	devlink_port_pool_th_set $swp1 1 16
-	devlink_tc_bind_pool_th_set $swp1 1 ingress 1 16
-
-	# Configure qdisc...
-	tc qdisc replace dev $swp1 root handle 1: \
-	   ets bands 8 strict 8 priomap 7 6
-	# ... so that we can assign prio1 traffic to PG1.
-	dcb buffer set dev $swp1 prio-buffer all:0 1:1
-
-	# $swp2
-	# -----
-
-	ip link set dev $swp2 up
-	mtu_set $swp2 10000
-	ethtool -s $swp2 speed 1000 autoneg off
-	vlan_create $swp2 111
-	ip link set dev $swp2.111 type vlan egress-qos-map 0:0 1:1
-
-	devlink_port_pool_th_set $swp2 6 $POOL_SIZE
-	devlink_tc_bind_pool_th_set $swp2 1 egress 6 $POOL_SIZE
-
-	# prio 0->TC0 (band 7), 1->TC1 (band 6)
-	tc qdisc replace dev $swp2 root handle 1: \
-	   ets bands 8 strict 8 priomap 7 6
-
-	# $swp3
-	# -----
-
-	ip link set dev $swp3 up
-	mtu_set $swp3 10000
-	ethtool -s $swp3 speed 1000 autoneg off
-	vlan_create $swp3 111
-	ip link set dev $swp3.111 type vlan egress-qos-map 0:0 1:1
-
-	devlink_port_pool_th_set $swp3 5 16
-	devlink_tc_bind_pool_th_set $swp3 1 egress 5 16
-
-	# prio 0->TC0 (band 7), 1->TC1 (band 6)
-	tc qdisc replace dev $swp3 root handle 1: \
-	   ets bands 8 strict 8 priomap 7 6
-
-	# $swp4
-	# -----
-
-	ip link set dev $swp4 up
-	mtu_set $swp4 10000
-	ethtool -s $swp4 speed 1000 autoneg off
-	vlan_create $swp4 111
-	ip link set dev $swp4.111 type vlan ingress-qos-map 0:0 1:1
-
-	devlink_port_pool_th_set $swp4 2 $POOL_SIZE
-	devlink_tc_bind_pool_th_set $swp4 1 ingress 2 $POOL_SIZE
-
-	# Configure qdisc...
-	tc qdisc replace dev $swp4 root handle 1: \
-	   ets bands 8 strict 8 priomap 7 6
-	# ... so that we can assign prio1 traffic to PG1.
-	dcb buffer set dev $swp4 prio-buffer all:0 1:1
-
-	# $swp5
-	# -----
-
-	ip link set dev $swp5 up
-	mtu_set $swp5 10000
-	vlan_create $swp5 111
-	ip link set dev $swp5.111 type vlan ingress-qos-map 0:0 1:1
-
-	devlink_port_pool_th_set $swp5 2 $POOL_SIZE
-	devlink_tc_bind_pool_th_set $swp5 1 ingress 2 $POOL_SIZE
-
-	# Configure qdisc...
-	tc qdisc replace dev $swp5 root handle 1: \
-	   ets bands 8 strict 8 priomap 7 6
-	# ... so that we can assign prio1 traffic to PG1.
-	dcb buffer set dev $swp5 prio-buffer all:0 1:1
-
-	# bridges
-	# -------
-
-	ip link add name br1 type bridge vlan_filtering 0
-	ip link set dev $swp1.111 master br1
-	ip link set dev $swp3.111 master br1
-	ip link set dev br1 up
-
-	ip link add name br2 type bridge vlan_filtering 0
-	ip link set dev $swp2.111 master br2
-	ip link set dev $swp4.111 master br2
-	ip link set dev $swp5.111 master br2
-	ip link set dev br2 up
-}
-
-switch_destroy()
-{
-	# Do this first so that we can reset the limits to values that are only
-	# valid for the original static / dynamic setting.
-	devlink_pool_size_thtype_restore 6
-	devlink_pool_size_thtype_restore 5
-	devlink_pool_size_thtype_restore 4
-	devlink_pool_size_thtype_restore 2
-	devlink_pool_size_thtype_restore 1
-	devlink_pool_size_thtype_restore 0
-
-	# bridges
-	# -------
-
-	ip link set dev br2 down
-	ip link set dev $swp5.111 nomaster
-	ip link set dev $swp4.111 nomaster
-	ip link set dev $swp2.111 nomaster
-	ip link del dev br2
-
-	ip link set dev br1 down
-	ip link set dev $swp3.111 nomaster
-	ip link set dev $swp1.111 nomaster
-	ip link del dev br1
-
-	# $swp5
-	# -----
-
-	dcb buffer set dev $swp5 prio-buffer all:0
-	tc qdisc del dev $swp5 root
-
-	devlink_tc_bind_pool_th_restore $swp5 1 ingress
-	devlink_port_pool_th_restore $swp5 2
-
-	vlan_destroy $swp5 111
-	mtu_restore $swp5
-	ip link set dev $swp5 down
-
-	# $swp4
-	# -----
-
-	dcb buffer set dev $swp4 prio-buffer all:0
-	tc qdisc del dev $swp4 root
-
-	devlink_tc_bind_pool_th_restore $swp4 1 ingress
-	devlink_port_pool_th_restore $swp4 2
-
-	vlan_destroy $swp4 111
-	ethtool -s $swp4 autoneg on
-	mtu_restore $swp4
-	ip link set dev $swp4 down
-
-	# $swp3
-	# -----
-
-	tc qdisc del dev $swp3 root
-
-	devlink_tc_bind_pool_th_restore $swp3 1 egress
-	devlink_port_pool_th_restore $swp3 5
-
-	vlan_destroy $swp3 111
-	ethtool -s $swp3 autoneg on
-	mtu_restore $swp3
-	ip link set dev $swp3 down
-
-	# $swp2
-	# -----
-
-	tc qdisc del dev $swp2 root
-
-	devlink_tc_bind_pool_th_restore $swp2 1 egress
-	devlink_port_pool_th_restore $swp2 6
-
-	vlan_destroy $swp2 111
-	ethtool -s $swp2 autoneg on
-	mtu_restore $swp2
-	ip link set dev $swp2 down
-
-	# $swp1
-	# -----
-
-	dcb buffer set dev $swp1 prio-buffer all:0
-	tc qdisc del dev $swp1 root
-
-	devlink_tc_bind_pool_th_restore $swp1 1 ingress
-	devlink_port_pool_th_restore $swp1 1
-
-	vlan_destroy $swp1 111
-	mtu_restore $swp1
-	ip link set dev $swp1 down
-}
-
-setup_prepare()
-{
-	h1=${NETIFS[p1]}
-	swp1=${NETIFS[p2]}
-
-	swp2=${NETIFS[p3]}
-	h2=${NETIFS[p4]}
-
-	swp3=${NETIFS[p5]}
-	swp4=${NETIFS[p6]}
-
-	swp5=${NETIFS[p7]}
-	h3=${NETIFS[p8]}
-
-	h2mac=$(mac_get $h2)
-
-	vrf_prepare
-
-	h1_create
-	h2_create
-	h3_create
-	switch_create
-}
-
-cleanup()
-{
-	pre_cleanup
-
-	switch_destroy
-	h3_destroy
-	h2_destroy
-	h1_destroy
-
-	vrf_cleanup
-}
-
-ping_ipv4()
-{
-	ping_test $h1 192.0.2.34 " h1->h2"
-	ping_test $h3 192.0.2.34 " h3->h2"
-}
-
-__test_qos_burst()
-{
-	local pktsize=$1; shift
-
-	RET=0
-
-	start_traffic_pktsize $pktsize $h1.111 192.0.2.33 192.0.2.34 $h2mac
-	sleep 1
-
-	local q0=$(ethtool_stats_get $swp2 tc_transmit_queue_tc_1)
-	((q0 == 0))
-	check_err $? "Transmit queue non-zero?"
-
-	local d0=$(ethtool_stats_get $swp2 tc_no_buffer_discard_uc_tc_1)
-
-	local cell_size=$(devlink_cell_size_get)
-	local cells=$((BURST_SIZE / cell_size))
-	# Each packet is $pktsize of payload + headers.
-	local pkt_cells=$(((pktsize + 50 + cell_size - 1)  / cell_size))
-	# How many packets can we admit:
-	local pkts=$((cells / pkt_cells))
-
-	$MZ $h3 -p $pktsize -Q 1:111 -A 192.0.2.35 -B 192.0.2.34 \
-		-a own -b $h2mac -c $pkts -t udp -q
-	sleep 1
-
-	local d1=$(ethtool_stats_get $swp2 tc_no_buffer_discard_uc_tc_1)
-	((d1 == d0))
-	check_err $? "Drops seen on egress port: $d0 -> $d1 ($((d1 - d0)))"
-
-	# Check that the queue is somewhat close to the burst size This
-	# makes sure that the lack of drops above was not due to port
-	# undersubscribtion.
-	local q0=$(ethtool_stats_get $swp2 tc_transmit_queue_tc_1)
-	local qe=$((90 * BURST_SIZE / 100))
-	((q0 > qe))
-	check_err $? "Queue size expected >$qe, got $q0"
-
-	stop_traffic
-	sleep 2
-
-	log_test "Burst: absorb $pkts ${pktsize}-B packets"
-}
-
-test_8K()
-{
-	__test_qos_burst 8000
-}
-
-test_800()
-{
-	__test_qos_burst 800
-}
-
-bail_on_lldpad
-
-trap cleanup EXIT
-setup_prepare
-setup_wait
-tests_run
-
-exit $EXIT_STATUS
diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh
index e9f8718af979..690d8daa71b4 100755
--- a/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh
+++ b/tools/testing/selftests/drivers/net/mlxsw/qos_ets_strict.sh
@@ -130,7 +130,8 @@ switch_create()
 
 	ip link set dev $swp3 up
 	mtu_set $swp3 10000
-	ethtool -s $swp3 speed 1000 autoneg off
+	tc qdisc replace dev $swp3 root handle 101: tbf rate 1gbit \
+		burst 128K limit 1G
 
 	vlan_create $swp1 111
 	vlan_create $swp2 222
@@ -193,7 +194,7 @@ switch_destroy()
 	vlan_destroy $swp2 222
 	vlan_destroy $swp1 111
 
-	ethtool -s $swp3 autoneg on
+	tc qdisc del dev $swp3 root handle 101:
 	mtu_restore $swp3
 	ip link set dev $swp3 down
 	lldptool -T -i $swp3 -V ETS-CFG up2tc=0:0,1:0,2:0,3:0,4:0,5:0,6:0,7:0
diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_max_descriptors.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_max_descriptors.sh
new file mode 100755
index 000000000000..5ac4f795e333
--- /dev/null
+++ b/tools/testing/selftests/drivers/net/mlxsw/qos_max_descriptors.sh
@@ -0,0 +1,282 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# This test sends many small packets (size is less than cell size) through the
+# switch. A shaper is used in $swp2, so the traffic is limited there. Packets
+# are queued till they will be sent.
+#
+# The idea is to verify that the switch can handle at least 85% of maximum
+# supported descrpitors by hardware. Then, we verify that the driver configures
+# firmware to allow infinite size of egress descriptor pool, and does not use a
+# lower limitation. Increase the size of the relevant pools such that the pool's
+# size does not limit the traffic.
+
+# +-----------------------+
+# | H1                    |
+# |   + $h1.111           |
+# |   | 192.0.2.33/28     |
+# |   |                   |
+# |   + $h1               |
+# +---|-------------------+
+#     |
+# +---|-----------------------------+
+# |   + $swp1                       |
+# |   | iPOOL1                      |
+# |   |                             |
+# | +-|------------------------+    |
+# | | + $swp1.111              |    |
+# | |                          |    |
+# | | BR1                      |    |
+# | |                          |    |
+# | | + $swp2.111              |    |
+# | +-|------------------------+    |
+# |   |                             |
+# |   + $swp2                       |
+# |   | ePOOL6                      |
+# |   | 1mbit                       |
+# +---+-----------------------------+
+#     |
+# +---|-------------------+
+# |   + $h2            H2 |
+# |   |                   |
+# |   + $h2.111           |
+# |     192.0.2.34/28     |
+# +-----------------------+
+#
+
+ALL_TESTS="
+	ping_ipv4
+	max_descriptors
+"
+
+lib_dir=$(dirname $0)/../../../net/forwarding
+
+NUM_NETIFS=4
+source $lib_dir/lib.sh
+source $lib_dir/devlink_lib.sh
+source mlxsw_lib.sh
+
+MAX_POOL_SIZE=$(devlink_pool_size_get)
+SHAPER_RATE=1mbit
+
+# The current TBF qdisc interface does not allow us to configure the shaper to
+# flat zero. The ASIC shaper is guaranteed to work with a granularity of
+# 200Mbps. On Spectrum-2, writing a value close to zero instead of zero works
+# well, but the performance on Spectrum-1 is unpredictable. Thus, do not run the
+# test on Spectrum-1.
+mlxsw_only_on_spectrum 2+ || exit
+
+h1_create()
+{
+	simple_if_init $h1
+
+	vlan_create $h1 111 v$h1 192.0.2.33/28
+	ip link set dev $h1.111 type vlan egress-qos-map 0:1
+}
+
+h1_destroy()
+{
+	vlan_destroy $h1 111
+
+	simple_if_fini $h1
+}
+
+h2_create()
+{
+	simple_if_init $h2
+
+	vlan_create $h2 111 v$h2 192.0.2.34/28
+}
+
+h2_destroy()
+{
+	vlan_destroy $h2 111
+
+	simple_if_fini $h2
+}
+
+switch_create()
+{
+	# pools
+	# -----
+
+	devlink_pool_size_thtype_save 1
+	devlink_pool_size_thtype_save 6
+
+	devlink_port_pool_th_save $swp1 1
+	devlink_port_pool_th_save $swp2 6
+
+	devlink_tc_bind_pool_th_save $swp1 1 ingress
+	devlink_tc_bind_pool_th_save $swp2 1 egress
+
+	devlink_pool_size_thtype_set 1 dynamic $MAX_POOL_SIZE
+	devlink_pool_size_thtype_set 6 static $MAX_POOL_SIZE
+
+	# $swp1
+	# -----
+
+	ip link set dev $swp1 up
+	vlan_create $swp1 111
+	ip link set dev $swp1.111 type vlan ingress-qos-map 0:0 1:1
+
+	devlink_port_pool_th_set $swp1 1 16
+	devlink_tc_bind_pool_th_set $swp1 1 ingress 1 16
+
+	tc qdisc replace dev $swp1 root handle 1: \
+	   ets bands 8 strict 8 priomap 7 6
+	dcb buffer set dev $swp1 prio-buffer all:0 1:1
+
+	# $swp2
+	# -----
+
+	ip link set dev $swp2 up
+	vlan_create $swp2 111
+	ip link set dev $swp2.111 type vlan egress-qos-map 0:0 1:1
+
+	devlink_port_pool_th_set $swp2 6 $MAX_POOL_SIZE
+	devlink_tc_bind_pool_th_set $swp2 1 egress 6 $MAX_POOL_SIZE
+
+	tc qdisc replace dev $swp2 root handle 1: tbf rate $SHAPER_RATE \
+		burst 128K limit 500M
+	tc qdisc replace dev $swp2 parent 1:1 handle 11: \
+		ets bands 8 strict 8 priomap 7 6
+
+	# bridge
+	# ------
+
+	ip link add name br1 type bridge vlan_filtering 0
+	ip link set dev $swp1.111 master br1
+	ip link set dev br1 up
+
+	ip link set dev $swp2.111 master br1
+}
+
+switch_destroy()
+{
+	# Do this first so that we can reset the limits to values that are only
+	# valid for the original static / dynamic setting.
+	devlink_pool_size_thtype_restore 6
+	devlink_pool_size_thtype_restore 1
+
+	# bridge
+	# ------
+
+	ip link set dev $swp2.111 nomaster
+
+	ip link set dev br1 down
+	ip link set dev $swp1.111 nomaster
+	ip link del dev br1
+
+	# $swp2
+	# -----
+
+	tc qdisc del dev $swp2 parent 1:1 handle 11:
+	tc qdisc del dev $swp2 root
+
+	devlink_tc_bind_pool_th_restore $swp2 1 egress
+	devlink_port_pool_th_restore $swp2 6
+
+	vlan_destroy $swp2 111
+	ip link set dev $swp2 down
+
+	# $swp1
+	# -----
+
+	dcb buffer set dev $swp1 prio-buffer all:0
+	tc qdisc del dev $swp1 root
+
+	devlink_tc_bind_pool_th_restore $swp1 1 ingress
+	devlink_port_pool_th_restore $swp1 1
+
+	vlan_destroy $swp1 111
+	ip link set dev $swp1 down
+}
+
+setup_prepare()
+{
+	h1=${NETIFS[p1]}
+	swp1=${NETIFS[p2]}
+
+	swp2=${NETIFS[p3]}
+	h2=${NETIFS[p4]}
+
+	h2mac=$(mac_get $h2)
+
+	vrf_prepare
+
+	h1_create
+	h2_create
+	switch_create
+}
+
+cleanup()
+{
+	pre_cleanup
+
+	switch_destroy
+	h2_destroy
+	h1_destroy
+
+	vrf_cleanup
+}
+
+ping_ipv4()
+{
+	ping_test $h1 192.0.2.34 " h1->h2"
+}
+
+percentage_used()
+{
+	local num_packets=$1; shift
+	local max_packets=$1; shift
+
+	bc <<< "
+	    scale=2
+	    100 * $num_packets / $max_packets
+	"
+}
+
+max_descriptors()
+{
+	local cell_size=$(devlink_cell_size_get)
+	local exp_perc_used=85
+	local max_descriptors
+	local pktsize=30
+
+	RET=0
+
+	max_descriptors=$(mlxsw_max_descriptors_get) || exit 1
+
+	local d0=$(ethtool_stats_get $swp2 tc_no_buffer_discard_uc_tc_1)
+
+	log_info "Send many small packets, packet size = $pktsize bytes"
+	start_traffic_pktsize $pktsize $h1.111 192.0.2.33 192.0.2.34 $h2mac
+
+	# Sleep to wait for congestion.
+	sleep 5
+
+	local d1=$(ethtool_stats_get $swp2 tc_no_buffer_discard_uc_tc_1)
+	((d1 == d0))
+	check_err $? "Drops seen on egress port: $d0 -> $d1 ($((d1 - d0)))"
+
+	# Check how many packets the switch can handle, the limitation is
+	# maximum descriptors.
+	local pkts_bytes=$(ethtool_stats_get $swp2 tc_transmit_queue_tc_1)
+	local pkts_num=$((pkts_bytes / cell_size))
+	local perc_used=$(percentage_used $pkts_num $max_descriptors)
+
+	check_err $(bc <<< "$perc_used < $exp_perc_used") \
+		"Expected > $exp_perc_used% of descriptors, handle $perc_used%"
+
+	stop_traffic
+	sleep 1
+
+	log_test "Maximum descriptors usage. The percentage used is $perc_used%"
+}
+
+trap cleanup EXIT
+setup_prepare
+setup_wait
+tests_run
+
+exit $EXIT_STATUS
diff --git a/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh b/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh
index 8f164c80e215..c8e55fa91660 100755
--- a/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh
+++ b/tools/testing/selftests/drivers/net/mlxsw/qos_mc_aware.sh
@@ -129,9 +129,10 @@ switch_create()
 	vlan_create $swp2 111
 	vlan_create $swp3 111
 
-	ethtool -s $swp3 speed 1000 autoneg off
-	tc qdisc replace dev $swp3 root handle 3: \
-	   prio bands 8 priomap 7 7 7 7 7 7 7 7
+	tc qdisc replace dev $swp3 root handle 3: tbf rate 1gbit \
+		burst 128K limit 1G
+	tc qdisc replace dev $swp3 parent 3:3 handle 33: \
+		prio bands 8 priomap 7 7 7 7 7 7 7 7
 
 	ip link add name br1 type bridge vlan_filtering 0
 	ip link set dev br1 up
@@ -172,8 +173,8 @@ switch_destroy()
 	ip link del dev br111
 	ip link del dev br1
 
+	tc qdisc del dev $swp3 parent 3:3 handle 33:
 	tc qdisc del dev $swp3 root handle 3:
-	ethtool -s $swp3 autoneg on
 
 	vlan_destroy $swp3 111
 	vlan_destroy $swp2 111
diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh
index af64bc9ea8ab..ceaa76b17a43 100755
--- a/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh
+++ b/tools/testing/selftests/drivers/net/mlxsw/sch_ets.sh
@@ -15,13 +15,15 @@ ALL_TESTS="
 	ets_test_dwrr
 "
 
+PARENT="parent 3:3"
+
 switch_create()
 {
-	ets_switch_create
-
 	# Create a bottleneck so that the DWRR process can kick in.
-	ethtool -s $h2 speed 1000 autoneg off
-	ethtool -s $swp2 speed 1000 autoneg off
+	tc qdisc replace dev $swp2 root handle 3: tbf rate 1gbit \
+		burst 128K limit 1G
+
+	ets_switch_create
 
 	# Set the ingress quota high and use the three egress TCs to limit the
 	# amount of traffic that is admitted to the shared buffers. This makes
@@ -55,10 +57,9 @@ switch_destroy()
 	devlink_tc_bind_pool_th_restore $swp1 0 ingress
 	devlink_port_pool_th_restore $swp1 0
 
-	ethtool -s $swp2 autoneg on
-	ethtool -s $h2 autoneg on
-
 	ets_switch_destroy
+
+	tc qdisc del dev $swp2 root handle 3:
 }
 
 # Callback from sch_ets_tests.sh
diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh
index f260f01db0e8..45b41b8f3232 100644
--- a/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh
+++ b/tools/testing/selftests/drivers/net/mlxsw/sch_red_core.sh
@@ -135,14 +135,16 @@ h2_create()
 	# cause packets to fail to queue up at $swp3 due to shared buffer
 	# quotas, and the test to spuriously fail.
 	#
-	# Prevent this by setting the speed of $h2 to 1Gbps.
+	# Prevent this by adding a shaper which limits the traffic in $h2 to
+	# 1Gbps.
 
-	ethtool -s $h2 speed 1000 autoneg off
+	tc qdisc replace dev $h2 root handle 10: tbf rate 1gbit \
+		burst 128K limit 1G
 }
 
 h2_destroy()
 {
-	ethtool -s $h2 autoneg on
+	tc qdisc del dev $h2 root handle 10:
 	tc qdisc del dev $h2 clsact
 	host_destroy $h2
 }
@@ -150,12 +152,10 @@ h2_destroy()
 h3_create()
 {
 	host_create $h3 3
-	ethtool -s $h3 speed 1000 autoneg off
 }
 
 h3_destroy()
 {
-	ethtool -s $h3 autoneg on
 	host_destroy $h3
 }
 
@@ -199,8 +199,9 @@ switch_create()
 		done
 	done
 
-	for intf in $swp2 $swp3 $swp4 $swp5; do
-		ethtool -s $intf speed 1000 autoneg off
+	for intf in $swp3 $swp4; do
+		tc qdisc replace dev $intf root handle 1: tbf rate 1gbit \
+			burst 128K limit 1G
 	done
 
 	ip link set dev br1_10 up
@@ -220,15 +221,13 @@ switch_destroy()
 
 	devlink_port_pool_th_restore $swp3 8
 
-	tc qdisc del dev $swp3 root 2>/dev/null
-
 	ip link set dev br2_11 down
 	ip link set dev br2_10 down
 	ip link set dev br1_11 down
 	ip link set dev br1_10 down
 
-	for intf in $swp5 $swp4 $swp3 $swp2; do
-		ethtool -s $intf autoneg on
+	for intf in $swp4 $swp3; do
+		tc qdisc del dev $intf root handle 1:
 	done
 
 	for intf in $swp5 $swp3 $swp2 $swp4 $swp1; do
@@ -536,7 +535,7 @@ do_red_test()
 	check_err $? "backlog $backlog / $limit Got $pct% marked packets, expected == 0."
 	local diff=$((limit - backlog))
 	pct=$((100 * diff / limit))
-	((0 <= pct && pct <= 10))
+	((-10 <= pct && pct <= 10))
 	check_err $? "backlog $backlog / $limit expected <= 10% distance"
 	log_test "TC $((vlan - 10)): RED backlog > limit"
 
diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_red_ets.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_red_ets.sh
index 7a73057206cd..0d01c7cd82a1 100755
--- a/tools/testing/selftests/drivers/net/mlxsw/sch_red_ets.sh
+++ b/tools/testing/selftests/drivers/net/mlxsw/sch_red_ets.sh
@@ -25,7 +25,7 @@ BACKLOG2=500000
 
 install_root_qdisc()
 {
-	tc qdisc add dev $swp3 root handle 10: $QDISC \
+	tc qdisc add dev $swp3 parent 1: handle 10: $QDISC \
 	   bands 8 priomap 7 6 5 4 3 2 1 0
 }
 
@@ -67,7 +67,7 @@ uninstall_qdisc_tc1()
 
 uninstall_root_qdisc()
 {
-	tc qdisc del dev $swp3 root
+	tc qdisc del dev $swp3 parent 1:
 }
 
 uninstall_qdisc()
diff --git a/tools/testing/selftests/drivers/net/mlxsw/sch_red_root.sh b/tools/testing/selftests/drivers/net/mlxsw/sch_red_root.sh
index 501d192529ac..860205338e6f 100755
--- a/tools/testing/selftests/drivers/net/mlxsw/sch_red_root.sh
+++ b/tools/testing/selftests/drivers/net/mlxsw/sch_red_root.sh
@@ -18,7 +18,7 @@ install_qdisc()
 {
 	local -a args=("$@")
 
-	tc qdisc add dev $swp3 root handle 108: red \
+	tc qdisc add dev $swp3 parent 1: handle 108: red \
 	   limit 1000000 min $BACKLOG max $((BACKLOG + 1)) \
 	   probability 1.0 avpkt 8000 burst 38 "${args[@]}"
 	sleep 1
@@ -26,7 +26,7 @@ install_qdisc()
 
 uninstall_qdisc()
 {
-	tc qdisc del dev $swp3 root
+	tc qdisc del dev $swp3 parent 1:
 }
 
 ecn_test()
diff --git a/tools/testing/selftests/drivers/net/ocelot/psfp.sh b/tools/testing/selftests/drivers/net/ocelot/psfp.sh
index 5a5cee92c665..bed748dde4b0 100755
--- a/tools/testing/selftests/drivers/net/ocelot/psfp.sh
+++ b/tools/testing/selftests/drivers/net/ocelot/psfp.sh
@@ -181,7 +181,7 @@ setup_prepare()
 
 	# Set up swp1 as a master PHC for h1, synchronized to the local
 	# CLOCK_REALTIME.
-	phc2sys_start ${swp1} ${UDS_ADDRESS_SWP1}
+	phc2sys_start ${UDS_ADDRESS_SWP1}
 
 	# Assumption true for LS1028A: h1 and h2 use the same PHC. So by
 	# synchronizing h1 to swp1 via PTP, h2 is also implicitly synchronized
diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile
index 4c122f1b1737..6448cb9f710f 100644
--- a/tools/testing/selftests/kvm/Makefile
+++ b/tools/testing/selftests/kvm/Makefile
@@ -48,6 +48,8 @@ LIBKVM += lib/rbtree.c
 LIBKVM += lib/sparsebit.c
 LIBKVM += lib/test_util.c
 
+LIBKVM_STRING += lib/string_override.c
+
 LIBKVM_x86_64 += lib/x86_64/apic.c
 LIBKVM_x86_64 += lib/x86_64/handlers.S
 LIBKVM_x86_64 += lib/x86_64/perf_test_util.c
@@ -220,7 +222,8 @@ LIBKVM_C := $(filter %.c,$(LIBKVM))
 LIBKVM_S := $(filter %.S,$(LIBKVM))
 LIBKVM_C_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_C))
 LIBKVM_S_OBJ := $(patsubst %.S, $(OUTPUT)/%.o, $(LIBKVM_S))
-LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ)
+LIBKVM_STRING_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_STRING))
+LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ) $(LIBKVM_STRING_OBJ)
 
 EXTRA_CLEAN += $(LIBKVM_OBJS) cscope.*
 
@@ -231,6 +234,12 @@ $(LIBKVM_C_OBJ): $(OUTPUT)/%.o: %.c
 $(LIBKVM_S_OBJ): $(OUTPUT)/%.o: %.S
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $< -o $@
 
+# Compile the string overrides as freestanding to prevent the compiler from
+# generating self-referential code, e.g. without "freestanding" the compiler may
+# "optimize" memcmp() by invoking memcmp(), thus causing infinite recursion.
+$(LIBKVM_STRING_OBJ): $(OUTPUT)/%.o: %.c
+	$(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c -ffreestanding $< -o $@
+
 x := $(shell mkdir -p $(sort $(dir $(TEST_GEN_PROGS))))
 $(TEST_GEN_PROGS): $(LIBKVM_OBJS)
 $(TEST_GEN_PROGS_EXTENDED): $(LIBKVM_OBJS)
diff --git a/tools/testing/selftests/kvm/access_tracking_perf_test.c b/tools/testing/selftests/kvm/access_tracking_perf_test.c
index 1c2749b1481a..76c583a07ea2 100644
--- a/tools/testing/selftests/kvm/access_tracking_perf_test.c
+++ b/tools/testing/selftests/kvm/access_tracking_perf_test.c
@@ -31,8 +31,9 @@
  * These limitations are worked around in this test by using a large enough
  * region of memory for each vCPU such that the number of translations cached in
  * the TLB and the number of pages held in pagevecs are a small fraction of the
- * overall workload. And if either of those conditions are not true this test
- * will fail rather than silently passing.
+ * overall workload. And if either of those conditions are not true (for example
+ * in nesting, where TLB size is unlimited) this test will print a warning
+ * rather than silently passing.
  */
 #include <inttypes.h>
 #include <limits.h>
@@ -172,17 +173,23 @@ static void mark_vcpu_memory_idle(struct kvm_vm *vm,
 		    vcpu_idx, no_pfn, pages);
 
 	/*
-	 * Test that at least 90% of memory has been marked idle (the rest might
-	 * not be marked idle because the pages have not yet made it to an LRU
-	 * list or the translations are still cached in the TLB). 90% is
+	 * Check that at least 90% of memory has been marked idle (the rest
+	 * might not be marked idle because the pages have not yet made it to an
+	 * LRU list or the translations are still cached in the TLB). 90% is
 	 * arbitrary; high enough that we ensure most memory access went through
 	 * access tracking but low enough as to not make the test too brittle
 	 * over time and across architectures.
+	 *
+	 * Note that when run in nested virtualization, this check will trigger
+	 * much more frequently because TLB size is unlimited and since no flush
+	 * happens, much more pages are cached there and guest won't see the
+	 * "idle" bit cleared.
 	 */
-	TEST_ASSERT(still_idle < pages / 10,
-		    "vCPU%d: Too many pages still idle (%"PRIu64 " out of %"
-		    PRIu64 ").\n",
-		    vcpu_idx, still_idle, pages);
+	if (still_idle < pages / 10)
+		printf("WARNING: vCPU%d: Too many pages still idle (%" PRIu64
+		       "out of %" PRIu64 "), this will affect performance results"
+		       ".\n",
+		       vcpu_idx, still_idle, pages);
 
 	close(page_idle_fd);
 	close(pagemap_fd);
diff --git a/tools/testing/selftests/kvm/include/x86_64/vmx.h b/tools/testing/selftests/kvm/include/x86_64/vmx.h
index 99fa1410964c..790c6d1ecb34 100644
--- a/tools/testing/selftests/kvm/include/x86_64/vmx.h
+++ b/tools/testing/selftests/kvm/include/x86_64/vmx.h
@@ -617,6 +617,7 @@ void nested_map_memslot(struct vmx_pages *vmx, struct kvm_vm *vm,
 			uint32_t memslot);
 void nested_identity_map_1g(struct vmx_pages *vmx, struct kvm_vm *vm,
 			    uint64_t addr, uint64_t size);
+bool kvm_vm_has_ept(struct kvm_vm *vm);
 void prepare_eptp(struct vmx_pages *vmx, struct kvm_vm *vm,
 		  uint32_t eptp_memslot);
 void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm);
diff --git a/tools/testing/selftests/kvm/lib/assert.c b/tools/testing/selftests/kvm/lib/assert.c
index 71ade6100fd3..2bd25b191d15 100644
--- a/tools/testing/selftests/kvm/lib/assert.c
+++ b/tools/testing/selftests/kvm/lib/assert.c
@@ -22,7 +22,7 @@ static void test_dump_stack(void)
 	 * Build and run this command:
 	 *
 	 *	addr2line -s -e /proc/$PPID/exe -fpai {backtrace addresses} | \
-	 *		grep -v test_dump_stack | cat -n 1>&2
+	 *		cat -n 1>&2
 	 *
 	 * Note that the spacing is different and there's no newline.
 	 */
@@ -36,18 +36,24 @@ static void test_dump_stack(void)
 		 n * (((sizeof(void *)) * 2) + 1) +
 		 /* Null terminator: */
 		 1];
-	char *c;
+	char *c = cmd;
 
 	n = backtrace(stack, n);
-	c = &cmd[0];
-	c += sprintf(c, "%s", addr2line);
 	/*
-	 * Skip the first 3 frames: backtrace, test_dump_stack, and
-	 * test_assert. We hope that backtrace isn't inlined and the other two
-	 * we've declared noinline.
+	 * Skip the first 2 frames, which should be test_dump_stack() and
+	 * test_assert(); both of which are declared noinline.  Bail if the
+	 * resulting stack trace would be empty. Otherwise, addr2line will block
+	 * waiting for addresses to be passed in via stdin.
 	 */
+	if (n <= 2) {
+		fputs("  (stack trace empty)\n", stderr);
+		return;
+	}
+
+	c += sprintf(c, "%s", addr2line);
 	for (i = 2; i < n; i++)
 		c += sprintf(c, " %lx", ((unsigned long) stack[i]) - 1);
+
 	c += sprintf(c, "%s", pipeline);
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wunused-result"
diff --git a/tools/testing/selftests/kvm/lib/string_override.c b/tools/testing/selftests/kvm/lib/string_override.c
new file mode 100644
index 000000000000..632398adc229
--- /dev/null
+++ b/tools/testing/selftests/kvm/lib/string_override.c
@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <stddef.h>
+
+/*
+ * Override the "basic" built-in string helpers so that they can be used in
+ * guest code.  KVM selftests don't support dynamic loading in guest code and
+ * will jump into the weeds if the compiler decides to insert an out-of-line
+ * call via the PLT.
+ */
+int memcmp(const void *cs, const void *ct, size_t count)
+{
+	const unsigned char *su1, *su2;
+	int res = 0;
+
+	for (su1 = cs, su2 = ct; 0 < count; ++su1, ++su2, count--) {
+		if ((res = *su1 - *su2) != 0)
+			break;
+	}
+	return res;
+}
+
+void *memcpy(void *dest, const void *src, size_t count)
+{
+	char *tmp = dest;
+	const char *s = src;
+
+	while (count--)
+		*tmp++ = *s++;
+	return dest;
+}
+
+void *memset(void *s, int c, size_t count)
+{
+	char *xs = s;
+
+	while (count--)
+		*xs++ = c;
+	return s;
+}
diff --git a/tools/testing/selftests/kvm/lib/x86_64/vmx.c b/tools/testing/selftests/kvm/lib/x86_64/vmx.c
index 80a568c439b8..d21049c38fc5 100644
--- a/tools/testing/selftests/kvm/lib/x86_64/vmx.c
+++ b/tools/testing/selftests/kvm/lib/x86_64/vmx.c
@@ -5,6 +5,8 @@
  * Copyright (C) 2018, Google LLC.
  */
 
+#include <asm/msr-index.h>
+
 #include "test_util.h"
 #include "kvm_util.h"
 #include "processor.h"
@@ -542,9 +544,27 @@ void nested_identity_map_1g(struct vmx_pages *vmx, struct kvm_vm *vm,
 	__nested_map(vmx, vm, addr, addr, size, PG_LEVEL_1G);
 }
 
+bool kvm_vm_has_ept(struct kvm_vm *vm)
+{
+	struct kvm_vcpu *vcpu;
+	uint64_t ctrl;
+
+	vcpu = list_first_entry(&vm->vcpus, struct kvm_vcpu, list);
+	TEST_ASSERT(vcpu, "Cannot determine EPT support without vCPUs.\n");
+
+	ctrl = vcpu_get_msr(vcpu, MSR_IA32_VMX_TRUE_PROCBASED_CTLS) >> 32;
+	if (!(ctrl & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS))
+		return false;
+
+	ctrl = vcpu_get_msr(vcpu, MSR_IA32_VMX_PROCBASED_CTLS2) >> 32;
+	return ctrl & SECONDARY_EXEC_ENABLE_EPT;
+}
+
 void prepare_eptp(struct vmx_pages *vmx, struct kvm_vm *vm,
 		  uint32_t eptp_memslot)
 {
+	TEST_REQUIRE(kvm_vm_has_ept(vm));
+
 	vmx->eptp = (void *)vm_vaddr_alloc_page(vm);
 	vmx->eptp_hva = addr_gva2hva(vm, (uintptr_t)vmx->eptp);
 	vmx->eptp_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->eptp);
diff --git a/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c b/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c
index b1905d280ef5..e0004bd26536 100644
--- a/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c
+++ b/tools/testing/selftests/kvm/x86_64/fix_hypercall_test.c
@@ -14,6 +14,9 @@
 #include "kvm_util.h"
 #include "processor.h"
 
+/* VMCALL and VMMCALL are both 3-byte opcodes. */
+#define HYPERCALL_INSN_SIZE	3
+
 static bool ud_expected;
 
 static void guest_ud_handler(struct ex_regs *regs)
@@ -22,7 +25,7 @@ static void guest_ud_handler(struct ex_regs *regs)
 	GUEST_DONE();
 }
 
-extern unsigned char svm_hypercall_insn;
+extern uint8_t svm_hypercall_insn[HYPERCALL_INSN_SIZE];
 static uint64_t svm_do_sched_yield(uint8_t apic_id)
 {
 	uint64_t ret;
@@ -39,7 +42,7 @@ static uint64_t svm_do_sched_yield(uint8_t apic_id)
 	return ret;
 }
 
-extern unsigned char vmx_hypercall_insn;
+extern uint8_t vmx_hypercall_insn[HYPERCALL_INSN_SIZE];
 static uint64_t vmx_do_sched_yield(uint8_t apic_id)
 {
 	uint64_t ret;
@@ -56,30 +59,20 @@ static uint64_t vmx_do_sched_yield(uint8_t apic_id)
 	return ret;
 }
 
-static void assert_hypercall_insn(unsigned char *exp_insn, unsigned char *obs_insn)
-{
-	uint32_t exp = 0, obs = 0;
-
-	memcpy(&exp, exp_insn, sizeof(exp));
-	memcpy(&obs, obs_insn, sizeof(obs));
-
-	GUEST_ASSERT_EQ(exp, obs);
-}
-
 static void guest_main(void)
 {
-	unsigned char *native_hypercall_insn, *hypercall_insn;
+	uint8_t *native_hypercall_insn, *hypercall_insn;
 	uint8_t apic_id;
 
 	apic_id = GET_APIC_ID_FIELD(xapic_read_reg(APIC_ID));
 
 	if (is_intel_cpu()) {
-		native_hypercall_insn = &vmx_hypercall_insn;
-		hypercall_insn = &svm_hypercall_insn;
+		native_hypercall_insn = vmx_hypercall_insn;
+		hypercall_insn = svm_hypercall_insn;
 		svm_do_sched_yield(apic_id);
 	} else if (is_amd_cpu()) {
-		native_hypercall_insn = &svm_hypercall_insn;
-		hypercall_insn = &vmx_hypercall_insn;
+		native_hypercall_insn = svm_hypercall_insn;
+		hypercall_insn = vmx_hypercall_insn;
 		vmx_do_sched_yield(apic_id);
 	} else {
 		GUEST_ASSERT(0);
@@ -87,8 +80,13 @@ static void guest_main(void)
 		return;
 	}
 
+	/*
+	 * The hypercall didn't #UD (guest_ud_handler() signals "done" if a #UD
+	 * occurs).  Verify that a #UD is NOT expected and that KVM patched in
+	 * the native hypercall.
+	 */
 	GUEST_ASSERT(!ud_expected);
-	assert_hypercall_insn(native_hypercall_insn, hypercall_insn);
+	GUEST_ASSERT(!memcmp(native_hypercall_insn, hypercall_insn, HYPERCALL_INSN_SIZE));
 	GUEST_DONE();
 }
 
diff --git a/tools/testing/selftests/lkdtm/tests.txt b/tools/testing/selftests/lkdtm/tests.txt
index 65e53eb0840b..607b8d7e3ea3 100644
--- a/tools/testing/selftests/lkdtm/tests.txt
+++ b/tools/testing/selftests/lkdtm/tests.txt
@@ -75,7 +75,9 @@ USERCOPY_KERNEL
 STACKLEAK_ERASING OK: the rest of the thread stack is properly erased
 CFI_FORWARD_PROTO
 CFI_BACKWARD call trace:|ok: control flow unchanged
-FORTIFIED_STRSCPY
-FORTIFIED_OBJECT
-FORTIFIED_SUBOBJECT
+FORTIFY_STRSCPY detected buffer overflow
+FORTIFY_STR_OBJECT detected buffer overflow
+FORTIFY_STR_MEMBER detected buffer overflow
+FORTIFY_MEM_OBJECT detected buffer overflow
+FORTIFY_MEM_MEMBER detected field-spanning write
 PPC_SLB_MULTIHIT Recovered
diff --git a/tools/testing/selftests/net/.gitignore b/tools/testing/selftests/net/.gitignore
index de7d5cc15f85..3d7adee7a3e6 100644
--- a/tools/testing/selftests/net/.gitignore
+++ b/tools/testing/selftests/net/.gitignore
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0-only
+bind_bhash
 cmsg_sender
 fin_ack_lat
 gro
@@ -21,6 +22,8 @@ reuseport_bpf_cpu
 reuseport_bpf_numa
 reuseport_dualstack
 rxtimestamp
+sk_bind_sendto_listen
+sk_connect_zero_addr
 socket
 so_netns_cookie
 so_txtime
diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile
index c0ee2955fe54..2a6b0bc648c4 100644
--- a/tools/testing/selftests/net/Makefile
+++ b/tools/testing/selftests/net/Makefile
@@ -37,11 +37,14 @@ TEST_PROGS += srv6_end_dt4_l3vpn_test.sh
 TEST_PROGS += srv6_end_dt6_l3vpn_test.sh
 TEST_PROGS += srv6_hencap_red_l3vpn_test.sh
 TEST_PROGS += srv6_hl2encap_red_l2vpn_test.sh
+TEST_PROGS += srv6_end_next_csid_l3vpn_test.sh
 TEST_PROGS += vrf_strict_mode_test.sh
 TEST_PROGS += arp_ndisc_evict_nocarrier.sh
 TEST_PROGS += ndisc_unsolicited_na_test.sh
 TEST_PROGS += arp_ndisc_untracked_subnets.sh
 TEST_PROGS += stress_reuseport_listen.sh
+TEST_PROGS += l2_tos_ttl_inherit.sh
+TEST_PROGS += bind_bhash.sh
 TEST_PROGS_EXTENDED := in_netns.sh setup_loopback.sh setup_veth.sh
 TEST_PROGS_EXTENDED += toeplitz_client.sh toeplitz.sh
 TEST_GEN_FILES =  socket nettest
@@ -63,6 +66,10 @@ TEST_GEN_FILES += cmsg_sender
 TEST_GEN_FILES += stress_reuseport_listen
 TEST_PROGS += test_vxlan_vnifiltering.sh
 TEST_GEN_FILES += io_uring_zerocopy_tx
+TEST_PROGS += io_uring_zerocopy_tx.sh
+TEST_GEN_FILES += bind_bhash
+TEST_GEN_PROGS += sk_bind_sendto_listen
+TEST_GEN_PROGS += sk_connect_zero_addr
 
 TEST_FILES := settings
 
@@ -73,3 +80,4 @@ include bpf/Makefile
 $(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma
 $(OUTPUT)/tcp_mmap: LDLIBS += -lpthread
 $(OUTPUT)/tcp_inq: LDLIBS += -lpthread
+$(OUTPUT)/bind_bhash: LDLIBS += -lpthread
diff --git a/tools/testing/selftests/net/bind_bhash.c b/tools/testing/selftests/net/bind_bhash.c
new file mode 100644
index 000000000000..57ff67a3751e
--- /dev/null
+++ b/tools/testing/selftests/net/bind_bhash.c
@@ -0,0 +1,144 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This times how long it takes to bind to a port when the port already
+ * has multiple sockets in its bhash table.
+ *
+ * In the setup(), we populate the port's bhash table with
+ * MAX_THREADS * MAX_CONNECTIONS number of entries.
+ */
+
+#include <unistd.h>
+#include <stdio.h>
+#include <netdb.h>
+#include <pthread.h>
+#include <string.h>
+#include <stdbool.h>
+
+#define MAX_THREADS 600
+#define MAX_CONNECTIONS 40
+
+static const char *setup_addr_v6 = "::1";
+static const char *setup_addr_v4 = "127.0.0.1";
+static const char *setup_addr;
+static const char *bind_addr;
+static const char *port;
+bool use_v6;
+int ret;
+
+static int fd_array[MAX_THREADS][MAX_CONNECTIONS];
+
+static int bind_socket(int opt, const char *addr)
+{
+	struct addrinfo *res, hint = {};
+	int sock_fd, reuse = 1, err;
+	int domain = use_v6 ? AF_INET6 : AF_INET;
+
+	sock_fd = socket(domain, SOCK_STREAM, 0);
+	if (sock_fd < 0) {
+		perror("socket fd err");
+		return sock_fd;
+	}
+
+	hint.ai_family = domain;
+	hint.ai_socktype = SOCK_STREAM;
+
+	err = getaddrinfo(addr, port, &hint, &res);
+	if (err) {
+		perror("getaddrinfo failed");
+		goto cleanup;
+	}
+
+	if (opt) {
+		err = setsockopt(sock_fd, SOL_SOCKET, opt, &reuse, sizeof(reuse));
+		if (err) {
+			perror("setsockopt failed");
+			goto cleanup;
+		}
+	}
+
+	err = bind(sock_fd, res->ai_addr, res->ai_addrlen);
+	if (err) {
+		perror("failed to bind to port");
+		goto cleanup;
+	}
+
+	return sock_fd;
+
+cleanup:
+	close(sock_fd);
+	return err;
+}
+
+static void *setup(void *arg)
+{
+	int sock_fd, i;
+	int *array = (int *)arg;
+
+	for (i = 0; i < MAX_CONNECTIONS; i++) {
+		sock_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, setup_addr);
+		if (sock_fd < 0) {
+			ret = sock_fd;
+			pthread_exit(&ret);
+		}
+		array[i] = sock_fd;
+	}
+
+	return NULL;
+}
+
+int main(int argc, const char *argv[])
+{
+	int listener_fd, sock_fd, i, j;
+	pthread_t tid[MAX_THREADS];
+	clock_t begin, end;
+
+	if (argc != 4) {
+		printf("Usage: listener <port> <ipv6 | ipv4> <bind-addr>\n");
+		return -1;
+	}
+
+	port = argv[1];
+	use_v6 = strcmp(argv[2], "ipv6") == 0;
+	bind_addr = argv[3];
+
+	setup_addr = use_v6 ? setup_addr_v6 : setup_addr_v4;
+
+	listener_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, setup_addr);
+	if (listen(listener_fd, 100) < 0) {
+		perror("listen failed");
+		return -1;
+	}
+
+	/* Set up threads to populate the bhash table entry for the port */
+	for (i = 0; i < MAX_THREADS; i++)
+		pthread_create(&tid[i], NULL, setup, fd_array[i]);
+
+	for (i = 0; i < MAX_THREADS; i++)
+		pthread_join(tid[i], NULL);
+
+	if (ret)
+		goto done;
+
+	begin = clock();
+
+	/* Bind to the same port on a different address */
+	sock_fd  = bind_socket(0, bind_addr);
+	if (sock_fd < 0)
+		goto done;
+
+	end = clock();
+
+	printf("time spent = %f\n", (double)(end - begin) / CLOCKS_PER_SEC);
+
+	/* clean up */
+	close(sock_fd);
+
+done:
+	close(listener_fd);
+	for (i = 0; i < MAX_THREADS; i++) {
+		for (j = 0; i < MAX_THREADS; i++)
+			close(fd_array[i][j]);
+	}
+
+	return 0;
+}
diff --git a/tools/testing/selftests/net/bind_bhash.sh b/tools/testing/selftests/net/bind_bhash.sh
new file mode 100755
index 000000000000..ca0292d4b441
--- /dev/null
+++ b/tools/testing/selftests/net/bind_bhash.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+NR_FILES=32768
+SAVED_NR_FILES=$(ulimit -n)
+
+# default values
+port=443
+addr_v6="2001:0db8:0:f101::1"
+addr_v4="10.8.8.8"
+use_v6=true
+addr=""
+
+usage() {
+    echo "Usage: $0 [-6 | -4] [-p port] [-a address]"
+    echo -e "\t6: use ipv6"
+    echo -e "\t4: use ipv4"
+    echo -e "\tport: Port number"
+    echo -e "\taddress: ip address"
+}
+
+while getopts "ha:p:64" opt; do
+    case ${opt} in
+	h)
+	    usage $0
+	    exit 0
+	    ;;
+	a)  addr=$OPTARG;;
+	p)
+	    port=$OPTARG;;
+	6)
+	    use_v6=true;;
+	4)
+	    use_v6=false;;
+    esac
+done
+
+setup() {
+    if [[ "$use_v6" == true ]]; then
+	ip addr add $addr_v6 nodad dev eth0
+    else
+	ip addr add $addr_v4 dev lo
+    fi
+	ulimit -n $NR_FILES
+}
+
+cleanup() {
+    if [[ "$use_v6" == true ]]; then
+	ip addr del $addr_v6 dev eth0
+    else
+	ip addr del $addr_v4/32 dev lo
+    fi
+    ulimit -n $SAVED_NR_FILES
+}
+
+if [[ "$addr" != "" ]]; then
+    addr_v4=$addr;
+    addr_v6=$addr;
+fi
+setup
+if [[ "$use_v6" == true ]] ; then
+    ./bind_bhash $port "ipv6" $addr_v6
+else
+    ./bind_bhash $port "ipv4" $addr_v4
+fi
+cleanup
diff --git a/tools/testing/selftests/net/fcnal-test.sh b/tools/testing/selftests/net/fcnal-test.sh
index 03b586760164..31c3b6ebd388 100755
--- a/tools/testing/selftests/net/fcnal-test.sh
+++ b/tools/testing/selftests/net/fcnal-test.sh
@@ -1467,6 +1467,13 @@ ipv4_udp_novrf()
 		log_test_addr ${a} $? 0 "Client, device bind via IP_UNICAST_IF"
 
 		log_start
+		run_cmd_nsb nettest -D -s &
+		sleep 1
+		run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S -0 ${NSA_IP} -U
+		log_test_addr ${a} $? 0 "Client, device bind via IP_UNICAST_IF, with connect()"
+
+
+		log_start
 		show_hint "Should fail 'Connection refused'"
 		run_cmd nettest -D -r ${a}
 		log_test_addr ${a} $? 1 "No server, unbound client"
@@ -1525,6 +1532,13 @@ ipv4_udp_novrf()
 	run_cmd nettest -D -d ${NSA_DEV} -S -r ${a}
 	log_test_addr ${a} $? 0 "Global server, device client via IP_UNICAST_IF, local connection"
 
+	log_start
+	run_cmd nettest -s -D &
+	sleep 1
+	run_cmd nettest -D -d ${NSA_DEV} -S -r ${a} -U
+	log_test_addr ${a} $? 0 "Global server, device client via IP_UNICAST_IF, local connection, with connect()"
+
+
 	# IPv4 with device bind has really weird behavior - it overrides the
 	# fib lookup, generates an rtable and tries to send the packet. This
 	# causes failures for local traffic at different places
@@ -1550,6 +1564,15 @@ ipv4_udp_novrf()
 		sleep 1
 		run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S
 		log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection"
+
+		log_start
+		show_hint "Should fail since addresses on loopback are out of device scope"
+		run_cmd nettest -D -s &
+		sleep 1
+		run_cmd nettest -D -r ${a} -d ${NSA_DEV} -S -U
+		log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection, with connect()"
+
+
 	done
 
 	a=${NSA_IP}
@@ -3157,6 +3180,13 @@ ipv6_udp_novrf()
 		sleep 1
 		run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} -S
 		log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection"
+
+		log_start
+		show_hint "Should fail 'No route to host' since addresses on loopback are out of device scope"
+		run_cmd nettest -6 -D -s &
+		sleep 1
+		run_cmd nettest -6 -D -r ${a} -d ${NSA_DEV} -S -U
+		log_test_addr ${a} $? 1 "Global server, device client via IP_UNICAST_IF, local connection, with connect()"
 	done
 
 	a=${NSA_IP6}
diff --git a/tools/testing/selftests/net/forwarding/devlink_lib.sh b/tools/testing/selftests/net/forwarding/devlink_lib.sh
index de9944d42027..601990c6881b 100644
--- a/tools/testing/selftests/net/forwarding/devlink_lib.sh
+++ b/tools/testing/selftests/net/forwarding/devlink_lib.sh
@@ -584,3 +584,8 @@ devlink_cell_size_get()
 	devlink sb pool show "$DEVLINK_DEV" pool 0 -j \
 	    | jq '.pool[][].cell_size'
 }
+
+devlink_pool_size_get()
+{
+	devlink sb show "$DEVLINK_DEV" -j | jq '.[][][]["size"]'
+}
diff --git a/tools/testing/selftests/net/forwarding/tsn_lib.sh b/tools/testing/selftests/net/forwarding/tsn_lib.sh
index 60a1423e8116..b91bcd8008a9 100644
--- a/tools/testing/selftests/net/forwarding/tsn_lib.sh
+++ b/tools/testing/selftests/net/forwarding/tsn_lib.sh
@@ -22,8 +22,7 @@ fi
 
 phc2sys_start()
 {
-	local if_name=$1
-	local uds_address=$2
+	local uds_address=$1
 	local extra_args=""
 
 	if ! [ -z "${uds_address}" ]; then
@@ -33,9 +32,7 @@ phc2sys_start()
 	phc2sys_log="$(mktemp)"
 
 	chrt -f 10 phc2sys -m \
-		-c ${if_name} \
-		-s CLOCK_REALTIME \
-		-O ${UTC_TAI_OFFSET} \
+		-a -rr \
 		--step_threshold 0.00002 \
 		--first_step_threshold 0.00002 \
 		${extra_args} \
@@ -53,15 +50,27 @@ phc2sys_stop()
 	rm "${phc2sys_log}" 2> /dev/null
 }
 
+# Replace space separators from interface list with underscores
+if_names_to_label()
+{
+	local if_name_list="$1"
+
+	echo "${if_name_list/ /_}"
+}
+
 ptp4l_start()
 {
-	local if_name=$1
+	local if_names="$1"
 	local slave_only=$2
 	local uds_address=$3
-	local log="ptp4l_log_${if_name}"
-	local pid="ptp4l_pid_${if_name}"
+	local log="ptp4l_log_$(if_names_to_label ${if_names})"
+	local pid="ptp4l_pid_$(if_names_to_label ${if_names})"
 	local extra_args=""
 
+	for if_name in ${if_names}; do
+		extra_args="${extra_args} -i ${if_name}"
+	done
+
 	if [ "${slave_only}" = true ]; then
 		extra_args="${extra_args} -s"
 	fi
@@ -71,7 +80,6 @@ ptp4l_start()
 	declare -g "${log}=$(mktemp)"
 
 	chrt -f 10 ptp4l -m -2 -P \
-		-i ${if_name} \
 		--step_threshold 0.00002 \
 		--first_step_threshold 0.00002 \
 		--tx_timestamp_timeout 100 \
@@ -80,16 +88,16 @@ ptp4l_start()
 		> "${!log}" 2>&1 &
 	declare -g "${pid}=$!"
 
-	echo "ptp4l for interface ${if_name} logs to ${!log} and has pid ${!pid}"
+	echo "ptp4l for interfaces ${if_names} logs to ${!log} and has pid ${!pid}"
 
 	sleep 1
 }
 
 ptp4l_stop()
 {
-	local if_name=$1
-	local log="ptp4l_log_${if_name}"
-	local pid="ptp4l_pid_${if_name}"
+	local if_names="$1"
+	local log="ptp4l_log_$(if_names_to_label ${if_names})"
+	local pid="ptp4l_pid_$(if_names_to_label ${if_names})"
 
 	{ kill ${!pid} && wait ${!pid}; } 2> /dev/null
 	rm "${!log}" 2> /dev/null
@@ -136,10 +144,12 @@ isochron_recv_start()
 {
 	local if_name=$1
 	local uds=$2
-	local extra_args=$3
+	local stats_port=$3
+	local extra_args=$4
+	local pid="isochron_pid_${stats_port}"
 
 	if ! [ -z "${uds}" ]; then
-		extra_args="--unix-domain-socket ${uds}"
+		extra_args="${extra_args} --unix-domain-socket ${uds}"
 	fi
 
 	isochron rcv \
@@ -147,16 +157,20 @@ isochron_recv_start()
 		--sched-priority 98 \
 		--sched-fifo \
 		--utc-tai-offset ${UTC_TAI_OFFSET} \
+		--stats-port ${stats_port} \
 		--quiet \
 		${extra_args} & \
-	isochron_pid=$!
+	declare -g "${pid}=$!"
 
 	sleep 1
 }
 
 isochron_recv_stop()
 {
-	{ kill ${isochron_pid} && wait ${isochron_pid}; } 2> /dev/null
+	local stats_port=$1
+	local pid="isochron_pid_${stats_port}"
+
+	{ kill ${!pid} && wait ${!pid}; } 2> /dev/null
 }
 
 isochron_do()
@@ -208,7 +222,7 @@ isochron_do()
 
 	cpufreq_max ${ISOCHRON_CPU}
 
-	isochron_recv_start "${h2}" "${receiver_uds}" "${receiver_extra_args}"
+	isochron_recv_start "${h2}" "${receiver_uds}" 5000 "${receiver_extra_args}"
 
 	isochron send \
 		--interface ${sender_if_name} \
@@ -229,7 +243,7 @@ isochron_do()
 		${extra_args} \
 		--quiet
 
-	isochron_recv_stop
+	isochron_recv_stop 5000
 
 	cpufreq_restore ${ISOCHRON_CPU}
 }
diff --git a/tools/testing/selftests/net/ipsec.c b/tools/testing/selftests/net/ipsec.c
index cc10c10c5ed9..9a8229abfa02 100644
--- a/tools/testing/selftests/net/ipsec.c
+++ b/tools/testing/selftests/net/ipsec.c
@@ -58,6 +58,8 @@
 #define VETH_FMT	"ktst-%d"
 #define VETH_LEN	12
 
+#define XFRM_ALGO_NR_KEYS 29
+
 static int nsfd_parent	= -1;
 static int nsfd_childa	= -1;
 static int nsfd_childb	= -1;
@@ -75,6 +77,43 @@ const unsigned int ping_timeout		= 300;
 const unsigned int ping_count		= 100;
 const unsigned int ping_success		= 80;
 
+struct xfrm_key_entry {
+	char algo_name[35];
+	int key_len;
+};
+
+struct xfrm_key_entry xfrm_key_entries[] = {
+	{"digest_null", 0},
+	{"ecb(cipher_null)", 0},
+	{"cbc(des)", 64},
+	{"hmac(md5)", 128},
+	{"cmac(aes)", 128},
+	{"xcbc(aes)", 128},
+	{"cbc(cast5)", 128},
+	{"cbc(serpent)", 128},
+	{"hmac(sha1)", 160},
+	{"hmac(rmd160)", 160},
+	{"cbc(des3_ede)", 192},
+	{"hmac(sha256)", 256},
+	{"cbc(aes)", 256},
+	{"cbc(camellia)", 256},
+	{"cbc(twofish)", 256},
+	{"rfc3686(ctr(aes))", 288},
+	{"hmac(sha384)", 384},
+	{"cbc(blowfish)", 448},
+	{"hmac(sha512)", 512},
+	{"rfc4106(gcm(aes))-128", 160},
+	{"rfc4543(gcm(aes))-128", 160},
+	{"rfc4309(ccm(aes))-128", 152},
+	{"rfc4106(gcm(aes))-192", 224},
+	{"rfc4543(gcm(aes))-192", 224},
+	{"rfc4309(ccm(aes))-192", 216},
+	{"rfc4106(gcm(aes))-256", 288},
+	{"rfc4543(gcm(aes))-256", 288},
+	{"rfc4309(ccm(aes))-256", 280},
+	{"rfc7539(chacha20,poly1305)-128", 0}
+};
+
 static void randomize_buffer(void *buf, size_t buflen)
 {
 	int *p = (int *)buf;
@@ -767,65 +806,12 @@ static int do_ping(int cmd_fd, char *buf, size_t buf_len, struct in_addr from,
 static int xfrm_fill_key(char *name, char *buf,
 		size_t buf_len, unsigned int *key_len)
 {
-	/* TODO: use set/map instead */
-	if (strncmp(name, "digest_null", ALGO_LEN) == 0)
-		*key_len = 0;
-	else if (strncmp(name, "ecb(cipher_null)", ALGO_LEN) == 0)
-		*key_len = 0;
-	else if (strncmp(name, "cbc(des)", ALGO_LEN) == 0)
-		*key_len = 64;
-	else if (strncmp(name, "hmac(md5)", ALGO_LEN) == 0)
-		*key_len = 128;
-	else if (strncmp(name, "cmac(aes)", ALGO_LEN) == 0)
-		*key_len = 128;
-	else if (strncmp(name, "xcbc(aes)", ALGO_LEN) == 0)
-		*key_len = 128;
-	else if (strncmp(name, "cbc(cast5)", ALGO_LEN) == 0)
-		*key_len = 128;
-	else if (strncmp(name, "cbc(serpent)", ALGO_LEN) == 0)
-		*key_len = 128;
-	else if (strncmp(name, "hmac(sha1)", ALGO_LEN) == 0)
-		*key_len = 160;
-	else if (strncmp(name, "hmac(rmd160)", ALGO_LEN) == 0)
-		*key_len = 160;
-	else if (strncmp(name, "cbc(des3_ede)", ALGO_LEN) == 0)
-		*key_len = 192;
-	else if (strncmp(name, "hmac(sha256)", ALGO_LEN) == 0)
-		*key_len = 256;
-	else if (strncmp(name, "cbc(aes)", ALGO_LEN) == 0)
-		*key_len = 256;
-	else if (strncmp(name, "cbc(camellia)", ALGO_LEN) == 0)
-		*key_len = 256;
-	else if (strncmp(name, "cbc(twofish)", ALGO_LEN) == 0)
-		*key_len = 256;
-	else if (strncmp(name, "rfc3686(ctr(aes))", ALGO_LEN) == 0)
-		*key_len = 288;
-	else if (strncmp(name, "hmac(sha384)", ALGO_LEN) == 0)
-		*key_len = 384;
-	else if (strncmp(name, "cbc(blowfish)", ALGO_LEN) == 0)
-		*key_len = 448;
-	else if (strncmp(name, "hmac(sha512)", ALGO_LEN) == 0)
-		*key_len = 512;
-	else if (strncmp(name, "rfc4106(gcm(aes))-128", ALGO_LEN) == 0)
-		*key_len = 160;
-	else if (strncmp(name, "rfc4543(gcm(aes))-128", ALGO_LEN) == 0)
-		*key_len = 160;
-	else if (strncmp(name, "rfc4309(ccm(aes))-128", ALGO_LEN) == 0)
-		*key_len = 152;
-	else if (strncmp(name, "rfc4106(gcm(aes))-192", ALGO_LEN) == 0)
-		*key_len = 224;
-	else if (strncmp(name, "rfc4543(gcm(aes))-192", ALGO_LEN) == 0)
-		*key_len = 224;
-	else if (strncmp(name, "rfc4309(ccm(aes))-192", ALGO_LEN) == 0)
-		*key_len = 216;
-	else if (strncmp(name, "rfc4106(gcm(aes))-256", ALGO_LEN) == 0)
-		*key_len = 288;
-	else if (strncmp(name, "rfc4543(gcm(aes))-256", ALGO_LEN) == 0)
-		*key_len = 288;
-	else if (strncmp(name, "rfc4309(ccm(aes))-256", ALGO_LEN) == 0)
-		*key_len = 280;
-	else if (strncmp(name, "rfc7539(chacha20,poly1305)-128", ALGO_LEN) == 0)
-		*key_len = 0;
+	int i;
+
+	for (i = 0; i < XFRM_ALGO_NR_KEYS; i++) {
+		if (strncmp(name, xfrm_key_entries[i].algo_name, ALGO_LEN) == 0)
+			*key_len = xfrm_key_entries[i].key_len;
+	}
 
 	if (*key_len > buf_len) {
 		printk("Can't pack a key - too big for buffer");
diff --git a/tools/testing/selftests/net/l2_tos_ttl_inherit.sh b/tools/testing/selftests/net/l2_tos_ttl_inherit.sh
new file mode 100755
index 000000000000..dca1e6f777a8
--- /dev/null
+++ b/tools/testing/selftests/net/l2_tos_ttl_inherit.sh
@@ -0,0 +1,390 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+
+# Author: Matthias May <matthias.may@westermo.com>
+#
+# This script evaluates ip tunnels that are capable of carrying L2 traffic
+# if they inherit or set the inheritable fields.
+# Namely these tunnels are: 'gretap', 'vxlan' and 'geneve'.
+# Checked inheritable fields are: TOS and TTL.
+# The outer tunnel protocol of 'IPv4' or 'IPv6' is verified-
+# As payload frames of type 'IPv4', 'IPv6' and 'other'(ARP) are verified.
+# In addition this script also checks if forcing a specific field in the
+# outer header is working.
+
+if [ "$(id -u)" != "0" ]; then
+	echo "Please run as root."
+	exit 0
+fi
+if ! which tcpdump > /dev/null 2>&1; then
+	echo "No tcpdump found. Required for this test."
+	exit 0
+fi
+
+expected_tos="0x00"
+expected_ttl="0"
+failed=false
+
+get_random_tos() {
+	# Get a random hex tos value between 0x00 and 0xfc, a multiple of 4
+	echo "0x$(tr -dc '0-9a-f' < /dev/urandom | head -c 1)\
+$(tr -dc '048c' < /dev/urandom | head -c 1)"
+}
+get_random_ttl() {
+	# Get a random dec value between 0 and 255
+	printf "%d" "0x$(tr -dc '0-9a-f' < /dev/urandom | head -c 2)"
+}
+get_field() {
+	# Expects to get the 'head -n 1' of a captured frame by tcpdump.
+	# Parses this first line and returns the specified field.
+	local field="$1"
+	local input="$2"
+	local found=false
+	input="$(echo "$input" | tr -d '(),')"
+	for input_field in $input; do
+		if $found; then
+			echo "$input_field"
+			return
+		fi
+		# The next field that we iterate over is the looked for value
+		if [ "$input_field" = "$field" ]; then
+			found=true
+		fi
+	done
+	echo "0"
+}
+setup() {
+	local type="$1"
+	local outer="$2"
+	local inner="$3"
+	local tos_ttl="$4"
+	local vlan="$5"
+	local test_tos="0x00"
+	local test_ttl="0"
+	local ns="ip netns exec testing"
+
+	# We don't want a test-tos of 0x00,
+	# because this is the value that we get when no tos is set.
+	expected_tos="$(get_random_tos)"
+	while [ "$expected_tos" = "0x00" ]; do
+		expected_tos="$(get_random_tos)"
+	done
+	if [ "$tos_ttl" = "random" ]; then
+		test_tos="$expected_tos"
+		tos="fixed $test_tos"
+	elif [ "$tos_ttl" = "inherit" ]; then
+		test_tos="$tos_ttl"
+		tos="inherit $expected_tos"
+	fi
+
+	# We don't want a test-ttl of 64 or 0,
+	# because 64 is when no ttl is set and 0 is not a valid ttl.
+	expected_ttl="$(get_random_ttl)"
+	while [ "$expected_ttl" = "64" ] || [ "$expected_ttl" = "0" ]; do
+		expected_ttl="$(get_random_ttl)"
+	done
+
+	if [ "$tos_ttl" = "random" ]; then
+		test_ttl="$expected_ttl"
+		ttl="fixed $test_ttl"
+	elif [ "$tos_ttl" = "inherit" ]; then
+		test_ttl="$tos_ttl"
+		ttl="inherit $expected_ttl"
+	fi
+	printf "│%7s │%6s │%6s │%13s │%13s │%6s │" \
+	"$type" "$outer" "$inner" "$tos" "$ttl" "$vlan"
+
+	# Create 'testing' netns, veth pair and connect main ns with testing ns
+	ip netns add testing
+	ip link add type veth
+	ip link set veth1 netns testing
+	ip link set veth0 up
+	$ns ip link set veth1 up
+	ip addr flush dev veth0
+	$ns ip addr flush dev veth1
+
+	local local_addr1=""
+	local local_addr2=""
+	if [ "$type" = "gre" ] || [ "$type" = "vxlan" ]; then
+		if [ "$outer" = "4" ]; then
+			local_addr1="local 198.18.0.1"
+			local_addr2="local 198.18.0.2"
+		elif [ "$outer" = "6" ]; then
+			local_addr1="local fdd1:ced0:5d88:3fce::1"
+			local_addr2="local fdd1:ced0:5d88:3fce::2"
+		fi
+	fi
+	local vxlan=""
+	if [ "$type" = "vxlan" ]; then
+		vxlan="vni 100 dstport 4789"
+	fi
+	local geneve=""
+	if [ "$type" = "geneve" ]; then
+		geneve="vni 100"
+	fi
+	# Create tunnel and assign outer IPv4/IPv6 addresses
+	if [ "$outer" = "4" ]; then
+		if [ "$type" = "gre" ]; then
+			type="gretap"
+		fi
+		ip addr add 198.18.0.1/24 dev veth0
+		$ns ip addr add 198.18.0.2/24 dev veth1
+		ip link add name tep0 type $type $local_addr1 remote \
+		198.18.0.2 tos $test_tos ttl $test_ttl $vxlan $geneve
+		$ns ip link add name tep1 type $type $local_addr2 remote \
+		198.18.0.1 tos $test_tos ttl $test_ttl $vxlan $geneve
+	elif [ "$outer" = "6" ]; then
+		if [ "$type" = "gre" ]; then
+			type="ip6gretap"
+		fi
+		ip addr add fdd1:ced0:5d88:3fce::1/64 dev veth0
+		$ns ip addr add fdd1:ced0:5d88:3fce::2/64 dev veth1
+		ip link add name tep0 type $type $local_addr1 \
+		remote fdd1:ced0:5d88:3fce::2 tos $test_tos ttl $test_ttl \
+		$vxlan $geneve
+		$ns ip link add name tep1 type $type $local_addr2 \
+		remote fdd1:ced0:5d88:3fce::1 tos $test_tos ttl $test_ttl \
+		$vxlan $geneve
+	fi
+
+	# Bring L2-tunnel link up and create VLAN on top
+	ip link set tep0 up
+	$ns ip link set tep1 up
+	ip addr flush dev tep0
+	$ns ip addr flush dev tep1
+	local parent
+	if $vlan; then
+		parent="vlan99-"
+		ip link add link tep0 name ${parent}0 type vlan id 99
+		$ns ip link add link tep1 name ${parent}1 type vlan id 99
+		ip link set ${parent}0 up
+		$ns ip link set ${parent}1 up
+		ip addr flush dev ${parent}0
+		$ns ip addr flush dev ${parent}1
+	else
+		parent="tep"
+	fi
+
+	# Assign inner IPv4/IPv6 addresses
+	if [ "$inner" = "4" ] || [ "$inner" = "other" ]; then
+		ip addr add 198.19.0.1/24 brd + dev ${parent}0
+		$ns ip addr add 198.19.0.2/24 brd + dev ${parent}1
+	elif [ "$inner" = "6" ]; then
+		ip addr add fdd4:96cf:4eae:443b::1/64 dev ${parent}0
+		$ns ip addr add fdd4:96cf:4eae:443b::2/64 dev ${parent}1
+	fi
+}
+
+verify() {
+	local outer="$1"
+	local inner="$2"
+	local tos_ttl="$3"
+	local vlan="$4"
+
+	local ping_pid out captured_tos captured_ttl result
+
+	local ping_dst
+	if [ "$inner" = "4" ]; then
+		ping_dst="198.19.0.2"
+	elif [ "$inner" = "6" ]; then
+		ping_dst="fdd4:96cf:4eae:443b::2"
+	elif [ "$inner" = "other" ]; then
+		ping_dst="198.19.0.3" # Generates ARPs which are not IPv4/IPv6
+	fi
+	if [ "$tos_ttl" = "inherit" ]; then
+		ping -i 0.1 $ping_dst -Q "$expected_tos" -t "$expected_ttl" \
+		2>/dev/null 1>&2 & ping_pid="$!"
+	else
+		ping -i 0.1 $ping_dst 2>/dev/null 1>&2 & ping_pid="$!"
+	fi
+	local tunnel_type_offset tunnel_type_proto req_proto_offset req_offset
+	if [ "$type" = "gre" ]; then
+		tunnel_type_proto="0x2f"
+	elif [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then
+		tunnel_type_proto="0x11"
+	fi
+	if [ "$outer" = "4" ]; then
+		tunnel_type_offset="9"
+		if [ "$inner" = "4" ]; then
+			req_proto_offset="47"
+			req_offset="58"
+			if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then
+				req_proto_offset="$((req_proto_offset + 12))"
+				req_offset="$((req_offset + 12))"
+			fi
+			if $vlan; then
+				req_proto_offset="$((req_proto_offset + 4))"
+				req_offset="$((req_offset + 4))"
+			fi
+			out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \
+			ip[$tunnel_type_offset] = $tunnel_type_proto and \
+			ip[$req_proto_offset] = 0x01 and \
+			ip[$req_offset] = 0x08 2>/dev/null | head -n 1)"
+		elif [ "$inner" = "6" ]; then
+			req_proto_offset="44"
+			req_offset="78"
+			if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then
+				req_proto_offset="$((req_proto_offset + 12))"
+				req_offset="$((req_offset + 12))"
+			fi
+			if $vlan; then
+				req_proto_offset="$((req_proto_offset + 4))"
+				req_offset="$((req_offset + 4))"
+			fi
+			out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \
+			ip[$tunnel_type_offset] = $tunnel_type_proto and \
+			ip[$req_proto_offset] = 0x3a and \
+			ip[$req_offset] = 0x80 2>/dev/null | head -n 1)"
+		elif [ "$inner" = "other" ]; then
+			req_proto_offset="36"
+			req_offset="45"
+			if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then
+				req_proto_offset="$((req_proto_offset + 12))"
+				req_offset="$((req_offset + 12))"
+			fi
+			if $vlan; then
+				req_proto_offset="$((req_proto_offset + 4))"
+				req_offset="$((req_offset + 4))"
+			fi
+			if [ "$tos_ttl" = "inherit" ]; then
+				expected_tos="0x00"
+				expected_ttl="64"
+			fi
+			out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \
+			ip[$tunnel_type_offset] = $tunnel_type_proto and \
+			ip[$req_proto_offset] = 0x08 and \
+			ip[$((req_proto_offset + 1))] = 0x06 and \
+			ip[$req_offset] = 0x01 2>/dev/null | head -n 1)"
+		fi
+	elif [ "$outer" = "6" ]; then
+		if [ "$type" = "gre" ]; then
+			tunnel_type_offset="40"
+		elif [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then
+			tunnel_type_offset="6"
+		fi
+		if [ "$inner" = "4" ]; then
+			local req_proto_offset="75"
+			local req_offset="86"
+			if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then
+				req_proto_offset="$((req_proto_offset + 4))"
+				req_offset="$((req_offset + 4))"
+			fi
+			if $vlan; then
+				req_proto_offset="$((req_proto_offset + 4))"
+				req_offset="$((req_offset + 4))"
+			fi
+			out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \
+			ip6[$tunnel_type_offset] = $tunnel_type_proto and \
+			ip6[$req_proto_offset] = 0x01 and \
+			ip6[$req_offset] = 0x08 2>/dev/null | head -n 1)"
+		elif [ "$inner" = "6" ]; then
+			local req_proto_offset="72"
+			local req_offset="106"
+			if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then
+				req_proto_offset="$((req_proto_offset + 4))"
+				req_offset="$((req_offset + 4))"
+			fi
+			if $vlan; then
+				req_proto_offset="$((req_proto_offset + 4))"
+				req_offset="$((req_offset + 4))"
+			fi
+			out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \
+			ip6[$tunnel_type_offset] = $tunnel_type_proto and \
+			ip6[$req_proto_offset] = 0x3a and \
+			ip6[$req_offset] = 0x80 2>/dev/null | head -n 1)"
+		elif [ "$inner" = "other" ]; then
+			local req_proto_offset="64"
+			local req_offset="73"
+			if [ "$type" = "vxlan" ] || [ "$type" = "geneve" ]; then
+				req_proto_offset="$((req_proto_offset + 4))"
+				req_offset="$((req_offset + 4))"
+			fi
+			if $vlan; then
+				req_proto_offset="$((req_proto_offset + 4))"
+				req_offset="$((req_offset + 4))"
+			fi
+			if [ "$tos_ttl" = "inherit" ]; then
+				expected_tos="0x00"
+				expected_ttl="64"
+			fi
+			out="$(tcpdump --immediate-mode -p -c 1 -v -i veth0 -n \
+			ip6[$tunnel_type_offset] = $tunnel_type_proto and \
+			ip6[$req_proto_offset] = 0x08 and \
+			ip6[$((req_proto_offset + 1))] = 0x06 and \
+			ip6[$req_offset] = 0x01 2>/dev/null | head -n 1)"
+		fi
+	fi
+	kill -9 $ping_pid
+	wait $ping_pid 2>/dev/null
+	result="FAIL"
+	if [ "$outer" = "4" ]; then
+		captured_ttl="$(get_field "ttl" "$out")"
+		captured_tos="$(printf "0x%02x" "$(get_field "tos" "$out")")"
+		if [ "$captured_tos" = "$expected_tos" ] &&
+		   [ "$captured_ttl" = "$expected_ttl" ]; then
+			result="OK"
+		fi
+	elif [ "$outer" = "6" ]; then
+		captured_ttl="$(get_field "hlim" "$out")"
+		captured_tos="$(printf "0x%02x" "$(get_field "class" "$out")")"
+		if [ "$captured_tos" = "$expected_tos" ] &&
+		   [ "$captured_ttl" = "$expected_ttl" ]; then
+			result="OK"
+		fi
+	fi
+
+	printf "%7s │\n" "$result"
+	if [ "$result" = "FAIL" ]; then
+		failed=true
+		if [ "$captured_tos" != "$expected_tos" ]; then
+			printf "│%43s%27s │\n" \
+			"Expected TOS value: $expected_tos" \
+			"Captured TOS value: $captured_tos"
+		fi
+		if [ "$captured_ttl" != "$expected_ttl" ]; then
+			printf "│%43s%27s │\n" \
+			"Expected TTL value: $expected_ttl" \
+			"Captured TTL value: $captured_ttl"
+		fi
+		printf "│%71s│\n" " "
+	fi
+}
+
+cleanup() {
+	ip link del veth0 2>/dev/null
+	ip netns del testing 2>/dev/null
+	ip link del tep0 2>/dev/null
+}
+
+printf "┌────────┬───────┬───────┬──────────────┬"
+printf "──────────────┬───────┬────────�\n"
+for type in gre vxlan geneve; do
+	if ! $(modprobe "$type" 2>/dev/null); then
+		continue
+	fi
+	for outer in 4 6; do
+		printf "├────────┼───────┼───────┼──────────────┼"
+		printf "──────────────┼───────┼────────┤\n"
+		printf "│  Type  │ outer | inner │     tos      │"
+		printf "      ttl     │  vlan │ result │\n"
+		for inner in 4 6 other; do
+			printf "├────────┼───────┼───────┼──────────────┼"
+			printf "──────────────┼───────┼────────┤\n"
+			for tos_ttl in inherit random; do
+				for vlan in false true; do
+					setup "$type" "$outer" "$inner" \
+					"$tos_ttl" "$vlan"
+					verify "$outer" "$inner" "$tos_ttl" \
+					"$vlan"
+					cleanup
+				done
+			done
+		done
+	done
+done
+printf "└────────┴───────┴───────┴──────────────┴"
+printf "──────────────┴───────┴────────┘\n"
+
+if $failed; then
+	exit 1
+fi
diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.c b/tools/testing/selftests/net/mptcp/mptcp_connect.c
index 24d4e9cb617e..e54653ea2ed4 100644
--- a/tools/testing/selftests/net/mptcp/mptcp_connect.c
+++ b/tools/testing/selftests/net/mptcp/mptcp_connect.c
@@ -72,6 +72,8 @@ static int cfg_wait;
 static uint32_t cfg_mark;
 static char *cfg_input;
 static int cfg_repeat = 1;
+static int cfg_truncate;
+static int cfg_rcv_trunc;
 
 struct cfg_cmsg_types {
 	unsigned int cmsg_enabled:1;
@@ -95,11 +97,15 @@ static struct cfg_sockopt_types cfg_sockopt_types;
 
 static void die_usage(void)
 {
-	fprintf(stderr, "Usage: mptcp_connect [-6] [-c cmsg] [-i file] [-I num] [-j] [-l] "
+	fprintf(stderr, "Usage: mptcp_connect [-6] [-c cmsg] [-f offset] [-i file] [-I num] [-j] [-l] "
 		"[-m mode] [-M mark] [-o option] [-p port] [-P mode] [-j] [-l] [-r num] "
 		"[-s MPTCP|TCP] [-S num] [-r num] [-t num] [-T num] [-u] [-w sec] connect_address\n");
 	fprintf(stderr, "\t-6 use ipv6\n");
 	fprintf(stderr, "\t-c cmsg -- test cmsg type <cmsg>\n");
+	fprintf(stderr, "\t-f offset -- stop the I/O after receiving and sending the specified amount "
+		"of bytes. If there are unread bytes in the receive queue, that will cause a MPTCP "
+		"fastclose at close/shutdown. If offset is negative, expect the peer to close before "
+		"all the local data as been sent, thus toleration errors on write and EPIPE signals\n");
 	fprintf(stderr, "\t-i file -- read the data to send from the given file instead of stdin");
 	fprintf(stderr, "\t-I num -- repeat the transfer 'num' times. In listen mode accepts num "
 		"incoming connections, in client mode, disconnect and reconnect to the server\n");
@@ -382,7 +388,7 @@ static size_t do_rnd_write(const int fd, char *buf, const size_t len)
 
 	bw = write(fd, buf, do_w);
 	if (bw < 0)
-		perror("write");
+		return bw;
 
 	/* let the join handshake complete, before going on */
 	if (cfg_join && first) {
@@ -571,7 +577,7 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after
 		.fd = peerfd,
 		.events = POLLIN | POLLOUT,
 	};
-	unsigned int woff = 0, wlen = 0;
+	unsigned int woff = 0, wlen = 0, total_wlen = 0, total_rlen = 0;
 	char wbuf[8192];
 
 	set_nonblock(peerfd, true);
@@ -597,7 +603,16 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after
 		}
 
 		if (fds.revents & POLLIN) {
-			len = do_rnd_read(peerfd, rbuf, sizeof(rbuf));
+			ssize_t rb = sizeof(rbuf);
+
+			/* limit the total amount of read data to the trunc value*/
+			if (cfg_truncate > 0) {
+				if (rb + total_rlen > cfg_truncate)
+					rb = cfg_truncate - total_rlen;
+				len = read(peerfd, rbuf, rb);
+			} else {
+				len = do_rnd_read(peerfd, rbuf, sizeof(rbuf));
+			}
 			if (len == 0) {
 				/* no more data to receive:
 				 * peer has closed its write side
@@ -612,10 +627,13 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after
 
 			/* Else, still have data to transmit */
 			} else if (len < 0) {
+				if (cfg_rcv_trunc)
+					return 0;
 				perror("read");
 				return 3;
 			}
 
+			total_rlen += len;
 			do_write(outfd, rbuf, len);
 		}
 
@@ -628,12 +646,21 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after
 			if (wlen > 0) {
 				ssize_t bw;
 
+				/* limit the total amount of written data to the trunc value */
+				if (cfg_truncate > 0 && wlen + total_wlen > cfg_truncate)
+					wlen = cfg_truncate - total_wlen;
+
 				bw = do_rnd_write(peerfd, wbuf + woff, wlen);
-				if (bw < 0)
+				if (bw < 0) {
+					if (cfg_rcv_trunc)
+						return 0;
+					perror("write");
 					return 111;
+				}
 
 				woff += bw;
 				wlen -= bw;
+				total_wlen += bw;
 			} else if (wlen == 0) {
 				/* We have no more data to send. */
 				fds.events &= ~POLLOUT;
@@ -652,10 +679,16 @@ static int copyfd_io_poll(int infd, int peerfd, int outfd, bool *in_closed_after
 		}
 
 		if (fds.revents & (POLLERR | POLLNVAL)) {
+			if (cfg_rcv_trunc)
+				return 0;
 			fprintf(stderr, "Unexpected revents: "
 				"POLLERR/POLLNVAL(%x)\n", fds.revents);
 			return 5;
 		}
+
+		if (cfg_truncate > 0 && total_wlen >= cfg_truncate &&
+		    total_rlen >= cfg_truncate)
+			break;
 	}
 
 	/* leave some time for late join/announce */
@@ -1160,11 +1193,13 @@ again:
 	}
 
 	/* close the client socket open only if we are not going to reconnect */
-	ret = copyfd_io(fd_in, fd, 1, cfg_repeat == 1);
+	ret = copyfd_io(fd_in, fd, 1, 0);
 	if (ret)
 		return ret;
 
-	if (--cfg_repeat > 0) {
+	if (cfg_truncate > 0) {
+		xdisconnect(fd, peer->ai_addrlen);
+	} else if (--cfg_repeat > 0) {
 		xdisconnect(fd, peer->ai_addrlen);
 
 		/* the socket could be unblocking at this point, we need the
@@ -1176,7 +1211,10 @@ again:
 		if (cfg_input)
 			close(fd_in);
 		goto again;
+	} else {
+		close(fd);
 	}
+
 	return 0;
 }
 
@@ -1262,8 +1300,19 @@ static void parse_opts(int argc, char **argv)
 {
 	int c;
 
-	while ((c = getopt(argc, argv, "6c:hi:I:jlm:M:o:p:P:r:R:s:S:t:T:w:")) != -1) {
+	while ((c = getopt(argc, argv, "6c:f:hi:I:jlm:M:o:p:P:r:R:s:S:t:T:w:")) != -1) {
 		switch (c) {
+		case 'f':
+			cfg_truncate = atoi(optarg);
+
+			/* when receiving a fastclose, ignore PIPE signals and
+			 * all the I/O errors later in the code
+			 */
+			if (cfg_truncate < 0) {
+				cfg_rcv_trunc = true;
+				signal(SIGPIPE, handle_signal);
+			}
+			break;
 		case 'j':
 			cfg_join = true;
 			cfg_mode = CFG_MODE_POLL;
diff --git a/tools/testing/selftests/net/mptcp/mptcp_join.sh b/tools/testing/selftests/net/mptcp/mptcp_join.sh
index ff83ef426df5..f3dd5f2a0272 100755
--- a/tools/testing/selftests/net/mptcp/mptcp_join.sh
+++ b/tools/testing/selftests/net/mptcp/mptcp_join.sh
@@ -346,10 +346,21 @@ check_transfer()
 	local in=$1
 	local out=$2
 	local what=$3
+	local bytes=$4
 	local i a b
 
 	local line
-	cmp -l "$in" "$out" | while read -r i a b; do
+	if [ -n "$bytes" ]; then
+		# when truncating we must check the size explicitly
+		local out_size=$(wc -c $out | awk '{print $1}')
+		if [ $out_size -ne $bytes ]; then
+			echo "[ FAIL ] $what output file has wrong size ($out_size, $bytes)"
+			fail_test
+			return 1
+		fi
+		bytes="--bytes=${bytes}"
+	fi
+	cmp -l "$in" "$out" ${bytes} | while read -r i a b; do
 		local sum=$((0${a} + 0${b}))
 		if [ $check_invert -eq 0 ] || [ $sum -ne $((0xff)) ]; then
 			echo "[ FAIL ] $what does not match (in, out):"
@@ -706,13 +717,39 @@ do_transfer()
 		addr_nr_ns1=${addr_nr_ns1:10}
 	fi
 
+	local flags="subflow"
+	local extra_cl_args=""
+	local extra_srv_args=""
+	local trunc_size=""
 	if [[ "${addr_nr_ns2}" = "fastclose_"* ]]; then
+		if [ ${test_link_fail} -le 1 ]; then
+			echo "fastclose tests need test_link_fail argument"
+			fail_test
+			return 1
+		fi
+
 		# disconnect
-		extra_args="$extra_args -I ${addr_nr_ns2:10}"
+		trunc_size=${test_link_fail}
+		local side=${addr_nr_ns2:10}
+
+		if [ ${side} = "client" ]; then
+			extra_cl_args="-f ${test_link_fail}"
+			extra_srv_args="-f -1"
+		elif [ ${side} = "server" ]; then
+			extra_srv_args="-f ${test_link_fail}"
+			extra_cl_args="-f -1"
+		else
+			echo "wrong/unknown fastclose spec ${side}"
+			fail_test
+			return 1
+		fi
 		addr_nr_ns2=0
 	elif [[ "${addr_nr_ns2}" = "userspace_"* ]]; then
 		userspace_pm=1
 		addr_nr_ns2=${addr_nr_ns2:10}
+	elif [[ "${addr_nr_ns2}" = "fullmesh_"* ]]; then
+		flags="${flags},fullmesh"
+		addr_nr_ns2=${addr_nr_ns2:9}
 	fi
 
 	if [ $userspace_pm -eq 1 ]; then
@@ -733,39 +770,41 @@ do_transfer()
 		local_addr="0.0.0.0"
 	fi
 
+	extra_srv_args="$extra_args $extra_srv_args"
 	if [ "$test_link_fail" -gt 1 ];then
 		timeout ${timeout_test} \
 			ip netns exec ${listener_ns} \
 				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
-					$extra_args ${local_addr} < "$sinfail" > "$sout" &
+					$extra_srv_args ${local_addr} < "$sinfail" > "$sout" &
 	else
 		timeout ${timeout_test} \
 			ip netns exec ${listener_ns} \
 				./mptcp_connect -t ${timeout_poll} -l -p $port -s ${srv_proto} \
-					$extra_args ${local_addr} < "$sin" > "$sout" &
+					$extra_srv_args ${local_addr} < "$sin" > "$sout" &
 	fi
 	local spid=$!
 
 	wait_local_port_listen "${listener_ns}" "${port}"
 
+	extra_cl_args="$extra_args $extra_cl_args"
 	if [ "$test_link_fail" -eq 0 ];then
 		timeout ${timeout_test} \
 			ip netns exec ${connector_ns} \
 				./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
-					$extra_args $connect_addr < "$cin" > "$cout" &
+					$extra_cl_args $connect_addr < "$cin" > "$cout" &
 	elif [ "$test_link_fail" -eq 1 ] || [ "$test_link_fail" -eq 2 ];then
 		( cat "$cinfail" ; sleep 2; link_failure $listener_ns ; cat "$cinfail" ) | \
 			tee "$cinsent" | \
 			timeout ${timeout_test} \
 				ip netns exec ${connector_ns} \
 					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
-						$extra_args $connect_addr > "$cout" &
+						$extra_cl_args $connect_addr > "$cout" &
 	else
 		tee "$cinsent" < "$cinfail" | \
 			timeout ${timeout_test} \
 				ip netns exec ${connector_ns} \
 					./mptcp_connect -t ${timeout_poll} -p $port -s ${cl_proto} \
-						$extra_args $connect_addr > "$cout" &
+						$extra_cl_args $connect_addr > "$cout" &
 	fi
 	local cpid=$!
 
@@ -832,12 +871,6 @@ do_transfer()
 		fi
 	fi
 
-	local flags="subflow"
-	if [[ "${addr_nr_ns2}" = "fullmesh_"* ]]; then
-		flags="${flags},fullmesh"
-		addr_nr_ns2=${addr_nr_ns2:9}
-	fi
-
 	# if newly added endpoints must be deleted, give the background msk
 	# some time to created them
 	[ $addr_nr_ns1 -gt 0 ] && [ $addr_nr_ns2 -lt 0 ] && sleep 1
@@ -973,15 +1006,15 @@ do_transfer()
 	fi
 
 	if [ "$test_link_fail" -gt 1 ];then
-		check_transfer $sinfail $cout "file received by client"
+		check_transfer $sinfail $cout "file received by client" $trunc_size
 	else
-		check_transfer $sin $cout "file received by client"
+		check_transfer $sin $cout "file received by client" $trunc_size
 	fi
 	retc=$?
 	if [ "$test_link_fail" -eq 0 ];then
-		check_transfer $cin $sout "file received by server"
+		check_transfer $cin $sout "file received by server" $trunc_size
 	else
-		check_transfer $cinsent $sout "file received by server"
+		check_transfer $cinsent $sout "file received by server" $trunc_size
 	fi
 	rets=$?
 
@@ -1190,12 +1223,23 @@ chk_fclose_nr()
 {
 	local fclose_tx=$1
 	local fclose_rx=$2
+	local ns_invert=$3
 	local count
 	local dump_stats
+	local ns_tx=$ns2
+	local ns_rx=$ns1
+	local extra_msg="   "
+
+	if [[ $ns_invert = "invert" ]]; then
+		ns_tx=$ns1
+		ns_rx=$ns2
+		extra_msg=${extra_msg}"invert"
+	fi
 
 	printf "%-${nr_blank}s %s" " " "ctx"
-	count=$(ip netns exec $ns2 nstat -as | grep MPTcpExtMPFastcloseTx | awk '{print $2}')
+	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPFastcloseTx | awk '{print $2}')
 	[ -z "$count" ] && count=0
+	[ "$count" != "$fclose_tx" ] && extra_msg="$extra_msg,tx=$count"
 	if [ "$count" != "$fclose_tx" ]; then
 		echo "[fail] got $count MP_FASTCLOSE[s] TX expected $fclose_tx"
 		fail_test
@@ -1205,17 +1249,20 @@ chk_fclose_nr()
 	fi
 
 	echo -n " - fclzrx"
-	count=$(ip netns exec $ns1 nstat -as | grep MPTcpExtMPFastcloseRx | awk '{print $2}')
+	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPFastcloseRx | awk '{print $2}')
 	[ -z "$count" ] && count=0
+	[ "$count" != "$fclose_rx" ] && extra_msg="$extra_msg,rx=$count"
 	if [ "$count" != "$fclose_rx" ]; then
 		echo "[fail] got $count MP_FASTCLOSE[s] RX expected $fclose_rx"
 		fail_test
 		dump_stats=1
 	else
-		echo "[ ok ]"
+		echo -n "[ ok ]"
 	fi
 
 	[ "${dump_stats}" = 1 ] && dump_stats
+
+	echo "$extra_msg"
 }
 
 chk_rst_nr()
@@ -1238,7 +1285,7 @@ chk_rst_nr()
 	printf "%-${nr_blank}s %s" " " "rtx"
 	count=$(ip netns exec $ns_tx nstat -as | grep MPTcpExtMPRstTx | awk '{print $2}')
 	[ -z "$count" ] && count=0
-	if [ "$count" != "$rst_tx" ]; then
+	if [ $count -lt $rst_tx ]; then
 		echo "[fail] got $count MP_RST[s] TX expected $rst_tx"
 		fail_test
 		dump_stats=1
@@ -1249,7 +1296,7 @@ chk_rst_nr()
 	echo -n " - rstrx "
 	count=$(ip netns exec $ns_rx nstat -as | grep MPTcpExtMPRstRx | awk '{print $2}')
 	[ -z "$count" ] && count=0
-	if [ "$count" != "$rst_rx" ]; then
+	if [ "$count" -lt "$rst_rx" ]; then
 		echo "[fail] got $count MP_RST[s] RX expected $rst_rx"
 		fail_test
 		dump_stats=1
@@ -2803,11 +2850,18 @@ fullmesh_tests()
 fastclose_tests()
 {
 	if reset "fastclose test"; then
-		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_2
+		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_client
 		chk_join_nr 0 0 0
 		chk_fclose_nr 1 1
 		chk_rst_nr 1 1 invert
 	fi
+
+	if reset "fastclose server test"; then
+		run_tests $ns1 $ns2 10.0.1.1 1024 0 fastclose_server
+		chk_join_nr 0 0 0
+		chk_fclose_nr 1 1 invert
+		chk_rst_nr 1 1
+	fi
 }
 
 pedit_action_pkts()
diff --git a/tools/testing/selftests/net/nettest.c b/tools/testing/selftests/net/nettest.c
index d9a6fd2cd9d3..7900fa98eccb 100644
--- a/tools/testing/selftests/net/nettest.c
+++ b/tools/testing/selftests/net/nettest.c
@@ -127,6 +127,9 @@ struct sock_args {
 
 	/* ESP in UDP encap test */
 	int use_xfrm;
+
+	/* use send() and connect() instead of sendto */
+	int datagram_connect;
 };
 
 static int server_mode;
@@ -979,6 +982,11 @@ static int send_msg(int sd, void *addr, socklen_t alen, struct sock_args *args)
 			log_err_errno("write failed sending msg to peer");
 			return 1;
 		}
+	} else if (args->datagram_connect) {
+		if (send(sd, msg, msglen, 0) < 0) {
+			log_err_errno("send failed sending msg to peer");
+			return 1;
+		}
 	} else if (args->ifindex && args->use_cmsg) {
 		if (send_msg_cmsg(sd, addr, alen, args->ifindex, args->version))
 			return 1;
@@ -1659,7 +1667,7 @@ static int connectsock(void *addr, socklen_t alen, struct sock_args *args)
 	if (args->has_local_ip && bind_socket(sd, args))
 		goto err;
 
-	if (args->type != SOCK_STREAM)
+	if (args->type != SOCK_STREAM && !args->datagram_connect)
 		goto out;
 
 	if (args->password && tcp_md5sig(sd, addr, alen, args))
@@ -1854,7 +1862,7 @@ static int ipc_parent(int cpid, int fd, struct sock_args *args)
 	return client_status;
 }
 
-#define GETOPT_STR  "sr:l:c:p:t:g:P:DRn:M:X:m:d:I:BN:O:SCi6xL:0:1:2:3:Fbqf"
+#define GETOPT_STR  "sr:l:c:p:t:g:P:DRn:M:X:m:d:I:BN:O:SUCi6xL:0:1:2:3:Fbqf"
 #define OPT_FORCE_BIND_KEY_IFINDEX 1001
 #define OPT_NO_BIND_KEY_IFINDEX 1002
 
@@ -1891,6 +1899,7 @@ static void print_usage(char *prog)
 	"    -I dev        bind socket to given device name - server mode\n"
 	"    -S            use setsockopt (IP_UNICAST_IF or IP_MULTICAST_IF)\n"
 	"                  to set device binding\n"
+	"    -U            Use connect() and send() for datagram sockets\n"
 	"    -f            bind socket with the IP[V6]_FREEBIND option\n"
 	"    -C            use cmsg and IP_PKTINFO to specify device binding\n"
 	"\n"
@@ -2074,6 +2083,9 @@ int main(int argc, char *argv[])
 		case 'x':
 			args.use_xfrm = 1;
 			break;
+		case 'U':
+			args.datagram_connect = 1;
+			break;
 		default:
 			print_usage(argv[0]);
 			return 1;
diff --git a/tools/testing/selftests/net/reuseport_bpf.c b/tools/testing/selftests/net/reuseport_bpf.c
index 072d709c96b4..65aea27d761c 100644
--- a/tools/testing/selftests/net/reuseport_bpf.c
+++ b/tools/testing/selftests/net/reuseport_bpf.c
@@ -328,7 +328,7 @@ static void test_extra_filter(const struct test_params p)
 	if (bind(fd1, addr, sockaddr_size()))
 		error(1, errno, "failed to bind recv socket 1");
 
-	if (!bind(fd2, addr, sockaddr_size()) && errno != EADDRINUSE)
+	if (!bind(fd2, addr, sockaddr_size()) || errno != EADDRINUSE)
 		error(1, errno, "bind socket 2 should fail with EADDRINUSE");
 
 	free(addr);
diff --git a/tools/testing/selftests/net/sk_bind_sendto_listen.c b/tools/testing/selftests/net/sk_bind_sendto_listen.c
new file mode 100644
index 000000000000..b420d830f72c
--- /dev/null
+++ b/tools/testing/selftests/net/sk_bind_sendto_listen.c
@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <arpa/inet.h>
+#include <error.h>
+#include <errno.h>
+#include <unistd.h>
+
+int main(void)
+{
+	int fd1, fd2, one = 1;
+	struct sockaddr_in6 bind_addr = {
+		.sin6_family = AF_INET6,
+		.sin6_port = htons(20000),
+		.sin6_flowinfo = htonl(0),
+		.sin6_addr = {},
+		.sin6_scope_id = 0,
+	};
+
+	inet_pton(AF_INET6, "::", &bind_addr.sin6_addr);
+
+	fd1 = socket(AF_INET6, SOCK_STREAM, IPPROTO_IP);
+	if (fd1 < 0) {
+		error(1, errno, "socket fd1");
+		return -1;
+	}
+
+	if (setsockopt(fd1, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one))) {
+		error(1, errno, "setsockopt(SO_REUSEADDR) fd1");
+		goto out_err1;
+	}
+
+	if (bind(fd1, (struct sockaddr *)&bind_addr, sizeof(bind_addr))) {
+		error(1, errno, "bind fd1");
+		goto out_err1;
+	}
+
+	if (sendto(fd1, NULL, 0, MSG_FASTOPEN, (struct sockaddr *)&bind_addr,
+		   sizeof(bind_addr))) {
+		error(1, errno, "sendto fd1");
+		goto out_err1;
+	}
+
+	fd2 = socket(AF_INET6, SOCK_STREAM, IPPROTO_IP);
+	if (fd2 < 0) {
+		error(1, errno, "socket fd2");
+		goto out_err1;
+	}
+
+	if (setsockopt(fd2, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one))) {
+		error(1, errno, "setsockopt(SO_REUSEADDR) fd2");
+		goto out_err2;
+	}
+
+	if (bind(fd2, (struct sockaddr *)&bind_addr, sizeof(bind_addr))) {
+		error(1, errno, "bind fd2");
+		goto out_err2;
+	}
+
+	if (sendto(fd2, NULL, 0, MSG_FASTOPEN, (struct sockaddr *)&bind_addr,
+		   sizeof(bind_addr)) != -1) {
+		error(1, errno, "sendto fd2");
+		goto out_err2;
+	}
+
+	if (listen(fd2, 0)) {
+		error(1, errno, "listen");
+		goto out_err2;
+	}
+
+	close(fd2);
+	close(fd1);
+	return 0;
+
+out_err2:
+	close(fd2);
+
+out_err1:
+	close(fd1);
+	return -1;
+}
diff --git a/tools/testing/selftests/net/sk_connect_zero_addr.c b/tools/testing/selftests/net/sk_connect_zero_addr.c
new file mode 100644
index 000000000000..4be418aefd9f
--- /dev/null
+++ b/tools/testing/selftests/net/sk_connect_zero_addr.c
@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <arpa/inet.h>
+#include <error.h>
+#include <errno.h>
+#include <unistd.h>
+
+int main(void)
+{
+	int fd1, fd2, one = 1;
+	struct sockaddr_in6 bind_addr = {
+		.sin6_family = AF_INET6,
+		.sin6_port = htons(20000),
+		.sin6_flowinfo = htonl(0),
+		.sin6_addr = {},
+		.sin6_scope_id = 0,
+	};
+
+	inet_pton(AF_INET6, "::", &bind_addr.sin6_addr);
+
+	fd1 = socket(AF_INET6, SOCK_STREAM, IPPROTO_IP);
+	if (fd1 < 0) {
+		error(1, errno, "socket fd1");
+		return -1;
+	}
+
+	if (setsockopt(fd1, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one))) {
+		error(1, errno, "setsockopt(SO_REUSEADDR) fd1");
+		goto out_err1;
+	}
+
+	if (bind(fd1, (struct sockaddr *)&bind_addr, sizeof(bind_addr))) {
+		error(1, errno, "bind fd1");
+		goto out_err1;
+	}
+
+	if (listen(fd1, 0)) {
+		error(1, errno, "listen");
+		goto out_err1;
+	}
+
+	fd2 = socket(AF_INET6, SOCK_STREAM, IPPROTO_IP);
+	if (fd2 < 0) {
+		error(1, errno, "socket fd2");
+		goto out_err1;
+	}
+
+	if (connect(fd2, (struct sockaddr *)&bind_addr, sizeof(bind_addr))) {
+		error(1, errno, "bind fd2");
+		goto out_err2;
+	}
+
+	close(fd2);
+	close(fd1);
+	return 0;
+
+out_err2:
+	close(fd2);
+out_err1:
+	close(fd1);
+	return -1;
+}
diff --git a/tools/testing/selftests/net/srv6_end_next_csid_l3vpn_test.sh b/tools/testing/selftests/net/srv6_end_next_csid_l3vpn_test.sh
new file mode 100755
index 000000000000..87e414cc417c
--- /dev/null
+++ b/tools/testing/selftests/net/srv6_end_next_csid_l3vpn_test.sh
@@ -0,0 +1,1145 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# author: Andrea Mayer <andrea.mayer@uniroma2.it>
+#
+# This script is designed for testing the support of NEXT-C-SID flavor for SRv6
+# End behavior.
+# A basic knowledge of SRv6 architecture [1] and of the compressed SID approach
+# [2] is assumed for the reader.
+#
+# The network topology used in the selftest is depicted hereafter, composed by
+# two hosts and four routers. Hosts hs-1 and hs-2 are connected through an
+# IPv4/IPv6 L3 VPN service, offered by routers rt-1, rt-2, rt-3 and rt-4 using
+# the NEXT-C-SID flavor. The key components for such VPNs are:
+#
+#    i) The SRv6 H.Encaps/H.Encaps.Red behaviors [1] apply SRv6 Policies on
+#       traffic received by connected hosts, initiating the VPN tunnel;
+#
+#   ii) The SRv6 End behavior [1] advances the active SID in the SID List
+#       carried by the SRH;
+#
+#  iii) The NEXT-C-SID mechanism [2] offers the possibility of encoding several
+#       SRv6 segments within a single 128-bit SID address, referred to as a
+#       Compressed SID (C-SID) container. In this way, the length of the SID
+#       List can be drastically reduced.
+#       The NEXT-C-SID is provided as a "flavor" of the SRv6 End behavior
+#       which advances the current C-SID (i.e. the Locator-Node Function defined
+#       in [2]) with the next one carried in the Argument, if available.
+#       When no more C-SIDs are available in the Argument, the SRv6 End behavior
+#       will apply the End function selecting the next SID in the SID List.
+#
+#   iv) The SRv6 End.DT46 behavior [1] is used for removing the SRv6 Policy and,
+#       thus, it terminates the VPN tunnel. Such a behavior is capable of
+#       handling, at the same time, both tunneled IPv4 and IPv6 traffic.
+#
+# [1] https://datatracker.ietf.org/doc/html/rfc8986
+# [2] https://datatracker.ietf.org/doc/html/draft-ietf-spring-srv6-srh-compression
+#
+#
+#               cafe::1                      cafe::2
+#              10.0.0.1                     10.0.0.2
+#             +--------+                   +--------+
+#             |        |                   |        |
+#             |  hs-1  |                   |  hs-2  |
+#             |        |                   |        |
+#             +---+----+                   +----+---+
+#    cafe::/64    |                             |      cafe::/64
+#  10.0.0.0/24    |                             |    10.0.0.0/24
+#             +---+----+                   +----+---+
+#             |        |  fcf0:0:1:2::/64  |        |
+#             |  rt-1  +-------------------+  rt-2  |
+#             |        |                   |        |
+#             +---+----+                   +----+---+
+#                 |      .               .      |
+#                 |  fcf0:0:1:3::/64   .        |
+#                 |          .       .          |
+#                 |            .   .            |
+# fcf0:0:1:4::/64 |              .              | fcf0:0:2:3::/64
+#                 |            .   .            |
+#                 |          .       .          |
+#                 |  fcf0:0:2:4::/64   .        |
+#                 |      .               .      |
+#             +---+----+                   +----+---+
+#             |        |                   |        |
+#             |  rt-4  +-------------------+  rt-3  |
+#             |        |  fcf0:0:3:4::/64  |        |
+#             +---+----+                   +----+---+
+#
+# Every fcf0:0:x:y::/64 network interconnects the SRv6 routers rt-x with rt-y in
+# the selftest network.
+#
+# Local SID/C-SID table
+# =====================
+#
+# Each SRv6 router is configured with a Local SID/C-SID table in which
+# SIDs/C-SIDs are stored. Considering an SRv6 router rt-x, SIDs/C-SIDs are
+# configured in the Local SID/C-SIDs table as follows:
+#
+#   Local SID/C-SID table for SRv6 router rt-x
+#   +-----------------------------------------------------------+
+#   |fcff:x::d46 is associated with the non-compressed SRv6     |
+#   |   End.DT46 behavior                                       |
+#   +-----------------------------------------------------------+
+#   |fcbb:0:0x00::/48 is associated with the NEXT-C-SID flavor  |
+#   |   of SRv6 End behavior                                    |
+#   +-----------------------------------------------------------+
+#   |fcbb:0:0x00:d46::/64 is associated with the SRv6 End.DT46  |
+#   |   behavior when NEXT-C-SID compression is turned on       |
+#   +-----------------------------------------------------------+
+#
+# The fcff::/16 prefix is reserved for implementing SRv6 services with regular
+# (non compressed) SIDs. Reachability of SIDs is ensured by proper configuration
+# of the IPv6 routing tables in the routers.
+# Similarly, the fcbb:0::/32 prefix is reserved for implementing SRv6 VPN
+# services leveraging the NEXT-C-SID compression mechanism. Indeed, the
+# fcbb:0::/32 is used for encoding the Locator-Block while the Locator-Node
+# Function is encoded with 16 bits.
+#
+# Incoming traffic classification and application of SRv6 Policies
+# ================================================================
+#
+# An SRv6 ingress router applies different SRv6 Policies to the traffic received
+# from a connected host, considering the IPv4 or IPv6 destination address.
+# SRv6 policy enforcement consists of encapsulating the received traffic into a
+# new IPv6 packet with a given SID List contained in the SRH.
+# When the SID List contains only one SID, the SRH could be omitted completely
+# and that SID is stored directly in the IPv6 Destination Address (DA) (this is
+# called "reduced" encapsulation).
+#
+# Test cases for NEXT-C-SID
+# =========================
+#
+# We consider two test cases for NEXT-C-SID: i) single SID and ii) double SID.
+#
+# In the single SID test case we have a number of segments that are all
+# contained in a single Compressed SID (C-SID) container. Therefore the
+# resulting SID List has only one SID. Using the reduced encapsulation format
+# this will result in a packet with no SRH.
+#
+# In the double SID test case we have one segment carried in a Compressed SID
+# (C-SID) container, followed by a regular (non compressed) SID. The resulting
+# SID List has two segments and it is possible to test the advance to the next
+# SID when all the C-SIDs in a C-SID container have been processed. Using the
+# reduced encapsulation format this will result in a packet with an SRH
+# containing 1 segment.
+#
+# For the single SID test case, we use the IPv4 addresses of hs-1 and hs-2, for
+# the double SID test case, we use their IPv6 addresses. This is only done to
+# simplify the test setup and avoid adding other hosts or multiple addresses on
+# the same interface of a host.
+#
+# Traffic from hs-1 to hs-2
+# -------------------------
+#
+# Packets generated from hs-1 and directed towards hs-2 are handled by rt-1
+# which applies the SRv6 Policies as follows:
+#
+#   i) IPv6 DA=cafe::2, H.Encaps.Red with SID List=fcbb:0:0400:0300:0200:d46::
+#  ii) IPv4 DA=10.0.0.2, H.Encaps.Red with SID List=fcbb:0:0300::,fcff:2::d46
+#
+# ### i) single SID
+#
+# The router rt-1 is configured to enforce the given Policy through the SRv6
+# H.Encaps.Red behavior which avoids the presence of the SRH at all, since it
+# pushes the single SID directly in the IPv6 DA. Such a SID encodes a whole
+# C-SID container carrying several C-SIDs (e.g. 0400, 0300, etc).
+#
+# As the packet reaches the router rt-4, the enabled NEXT-C-SID SRv6 End
+# behavior (associated with fcbb:0:0400::/48) is triggered. This behavior
+# analyzes the IPv6 DA and checks whether the Argument of the C-SID container
+# is zero or not. In this case, the Argument is *NOT* zero and the IPv6 DA is
+# updated as follows:
+#
+# +---------------------------------------------------------------+
+# | Before applying the rt-4 enabled NEXT-C-SID SRv6 End behavior |
+# +---------------------------------------------------------------+
+# |                            +---------- Argument               |
+# |                     vvvvvvvvvvvvvvvv                          |
+# | IPv6 DA fcbb:0:0400:0300:0200:d46::                           |
+# |                ^^^^    <-- shifting                           |
+# |                  |                                            |
+# |          Locator-Node Function                                |
+# +---------------------------------------------------------------+
+# | After applying the rt-4 enabled NEXT-C-SID SRv6 End behavior  |
+# +---------------------------------------------------------------+
+# |                          +---------- Argument                 |
+# |                    vvvvvvvvvvvv                               |
+# | IPv6 DA fcbb:0:0300:0200:d46::                                |
+# |                ^^^^                                           |
+# |                  |                                            |
+# |          Locator-Node Function                                |
+# +---------------------------------------------------------------+
+#
+# After having applied the enabled NEXT-C-SID SRv6 End behavior, the packet is
+# sent to the next node, i.e. rt-3.
+#
+# The enabled NEXT-C-SID SRv6 End behavior on rt-3 is executed as the packet is
+# received. This behavior processes the packet and updates the IPv6 DA with
+# fcbb:0:0200:d46::, since the Argument is *NOT* zero. Then, the packet is sent
+# to the router rt-2.
+#
+# The router rt-2 is configured for decapsulating the inner IPv6 packet and,
+# for this reason, it applies the SRv6 End.DT46 behavior on the received
+# packet. It is worth noting that the SRv6 End.DT46 behavior does not require
+# the presence of the SRH: it is fully capable to operate properly on
+# IPv4/IPv6-in-IPv6 encapsulations.
+# At the end of the decap operation, the packet is sent to the
+# host hs-2.
+#
+# ### ii) double SID
+#
+# The router rt-1 is configured to enforce the given Policy through the SRv6
+# H.Encaps.Red. As a result, the first SID fcbb:0:0300:: is stored into the
+# IPv6 DA, while the SRH pushed into the packet is made of only one SID, i.e.
+# fcff:2::d46. Hence, the packet sent by hs-1 to hs-2 is encapsulated in an
+# outer IPv6 header plus the SRH.
+#
+# As the packet reaches the node rt-3, the router applies the enabled NEXT-C-SID
+# SRv6 End behavior.
+#
+# +---------------------------------------------------------------+
+# | Before applying the rt-3 enabled NEXT-C-SID SRv6 End behavior |
+# +---------------------------------------------------------------+
+# |                            +---------- Argument               |
+# |                      vvvv (Argument is all filled with zeros) |
+# | IPv6 DA fcbb:0:0300::                                         |
+# |                ^^^^                                           |
+# |                  |                                            |
+# |          Locator-Node Function                                |
+# +---------------------------------------------------------------+
+# | After applying the rt-3 enabled NEXT-C-SID SRv6 End behavior  |
+# +---------------------------------------------------------------+
+# |                                                               |
+# | IPv6 DA fcff:2::d46                                           |
+# |         ^^^^^^^^^^^                                           |
+# |              |                                                |
+# |        SID copied from the SID List contained in the SRH      |
+# +---------------------------------------------------------------+
+#
+# Since the Argument of the C-SID container is zero, the behavior can not
+# update the Locator-Node function with the next C-SID carried in the Argument
+# itself. Thus, the enabled NEXT-C-SID SRv6 End behavior operates as the
+# traditional End behavior: it updates the IPv6 DA by copying the next
+# available SID in the SID List carried by the SRH. After that, the packet is
+# sent to the node rt-2.
+#
+# Once the packet is received by rt-2, the router decapsulates the inner IPv6
+# packet using the SRv6 End.DT46 behavior (associated with the SID fcff:2::d46)
+# and sends it to the host hs-2.
+#
+# Traffic from hs-2 to hs-1
+# -------------------------
+#
+# Packets generated from hs-2 and directed towards hs-1 are handled by rt-2
+# which applies the SRv6 Policies as follows:
+#
+#   i) IPv6 DA=cafe::1, SID List=fcbb:0:0300:0400:0100:d46::
+#  ii) IPv4 DA=10.0.0.1, SID List=fcbb:0:0300::,fcff:1::d46
+#
+# For simplicity, such SRv6 Policies were chosen so that, in both use cases (i)
+# and (ii), the network paths crossed by traffic from hs-2 to hs-1 are the same
+# as those taken by traffic from hs-1 to hs-2.
+# In this way, traffic from hs-2 to hs-1 is processed similarly to traffic from
+# hs-1 to hs-2. So, the traffic processing scheme turns out to be the same as
+# that adopted in the use cases already examined (of course, it is necessary to
+# consider the different SIDs/C-SIDs).
+
+# Kselftest framework requirement - SKIP code is 4.
+readonly ksft_skip=4
+
+readonly RDMSUFF="$(mktemp -u XXXXXXXX)"
+readonly DUMMY_DEVNAME="dum0"
+readonly VRF_TID=100
+readonly VRF_DEVNAME="vrf-${VRF_TID}"
+readonly RT2HS_DEVNAME="veth-t${VRF_TID}"
+readonly LOCALSID_TABLE_ID=90
+readonly IPv6_RT_NETWORK=fcf0:0
+readonly IPv6_HS_NETWORK=cafe
+readonly IPv4_HS_NETWORK=10.0.0
+readonly VPN_LOCATOR_SERVICE=fcff
+readonly DT46_FUNC=0d46
+readonly HEADEND_ENCAP="encap.red"
+
+# do not add ':' as separator
+readonly LCBLOCK_ADDR=fcbb0000
+readonly LCBLOCK_BLEN=32
+# do not add ':' as separator
+readonly LCNODEFUNC_FMT="0%d00"
+readonly LCNODEFUNC_BLEN=16
+
+readonly LCBLOCK_NODEFUNC_BLEN=$((LCBLOCK_BLEN + LCNODEFUNC_BLEN))
+
+readonly CSID_CNTR_PREFIX="dead:beaf::/32"
+# ID of the router used for testing the C-SID container cfgs
+readonly CSID_CNTR_RT_ID_TEST=1
+# Routing table used for testing the C-SID container cfgs
+readonly CSID_CNTR_RT_TABLE=91
+
+# C-SID container configurations to be tested
+#
+# An entry of the array is defined as "a,b,c" where:
+# - 'a' and 'b' elements represent respectively the Locator-Block length
+#   (lblen) in bits and the Locator-Node Function length (nflen) in bits.
+#   'a' and 'b' can be set to default values using the placeholder "d" which
+#   indicates the default kernel values (32 for lblen and 16 for nflen);
+#   otherwise, any numeric value is accepted;
+# - 'c' indicates whether the C-SID configuration provided by the values 'a'
+#   and 'b' should be considered valid ("y") or invalid ("n").
+declare -ra CSID_CONTAINER_CFGS=(
+	"d,d,y"
+	"d,16,y"
+	"16,d,y"
+	"16,32,y"
+	"32,16,y"
+	"48,8,y"
+	"8,48,y"
+	"d,0,n"
+	"0,d,n"
+	"32,0,n"
+	"0,32,n"
+	"17,d,n"
+	"d,17,n"
+	"120,16,n"
+	"16,120,n"
+	"0,128,n"
+	"128,0,n"
+	"130,0,n"
+	"0,130,n"
+	"0,0,n"
+)
+
+PING_TIMEOUT_SEC=4
+PAUSE_ON_FAIL=${PAUSE_ON_FAIL:=no}
+
+# IDs of routers and hosts are initialized during the setup of the testing
+# network
+ROUTERS=''
+HOSTS=''
+
+SETUP_ERR=1
+
+ret=${ksft_skip}
+nsuccess=0
+nfail=0
+
+log_test()
+{
+	local rc="$1"
+	local expected="$2"
+	local msg="$3"
+
+	if [ "${rc}" -eq "${expected}" ]; then
+		nsuccess=$((nsuccess+1))
+		printf "\n    TEST: %-60s  [ OK ]\n" "${msg}"
+	else
+		ret=1
+		nfail=$((nfail+1))
+		printf "\n    TEST: %-60s  [FAIL]\n" "${msg}"
+		if [ "${PAUSE_ON_FAIL}" = "yes" ]; then
+			echo
+			echo "hit enter to continue, 'q' to quit"
+			read a
+			[ "$a" = "q" ] && exit 1
+		fi
+	fi
+}
+
+print_log_test_results()
+{
+	printf "\nTests passed: %3d\n" "${nsuccess}"
+	printf "Tests failed: %3d\n"   "${nfail}"
+
+	# when a test fails, the value of 'ret' is set to 1 (error code).
+	# Conversely, when all tests are passed successfully, the 'ret' value
+	# is set to 0 (success code).
+	if [ "${ret}" -ne 1 ]; then
+		ret=0
+	fi
+}
+
+log_section()
+{
+	echo
+	echo "################################################################################"
+	echo "TEST SECTION: $*"
+	echo "################################################################################"
+}
+
+test_command_or_ksft_skip()
+{
+	local cmd="$1"
+
+	if [ ! -x "$(command -v "${cmd}")" ]; then
+		echo "SKIP: Could not run test without \"${cmd}\" tool";
+		exit "${ksft_skip}"
+	fi
+}
+
+get_nodename()
+{
+	local name="$1"
+
+	echo "${name}-${RDMSUFF}"
+}
+
+get_rtname()
+{
+	local rtid="$1"
+
+	get_nodename "rt-${rtid}"
+}
+
+get_hsname()
+{
+	local hsid="$1"
+
+	get_nodename "hs-${hsid}"
+}
+
+__create_namespace()
+{
+	local name="$1"
+
+	ip netns add "${name}"
+}
+
+create_router()
+{
+	local rtid="$1"
+	local nsname
+
+	nsname="$(get_rtname "${rtid}")"
+
+	__create_namespace "${nsname}"
+}
+
+create_host()
+{
+	local hsid="$1"
+	local nsname
+
+	nsname="$(get_hsname "${hsid}")"
+
+	__create_namespace "${nsname}"
+}
+
+cleanup()
+{
+	local nsname
+	local i
+
+	# destroy routers
+	for i in ${ROUTERS}; do
+		nsname="$(get_rtname "${i}")"
+
+		ip netns del "${nsname}" &>/dev/null || true
+	done
+
+	# destroy hosts
+	for i in ${HOSTS}; do
+		nsname="$(get_hsname "${i}")"
+
+		ip netns del "${nsname}" &>/dev/null || true
+	done
+
+	# check whether the setup phase was completed successfully or not. In
+	# case of an error during the setup phase of the testing environment,
+	# the selftest is considered as "skipped".
+	if [ "${SETUP_ERR}" -ne 0 ]; then
+		echo "SKIP: Setting up the testing environment failed"
+		exit "${ksft_skip}"
+	fi
+
+	exit "${ret}"
+}
+
+add_link_rt_pairs()
+{
+	local rt="$1"
+	local rt_neighs="$2"
+	local neigh
+	local nsname
+	local neigh_nsname
+
+	nsname="$(get_rtname "${rt}")"
+
+	for neigh in ${rt_neighs}; do
+		neigh_nsname="$(get_rtname "${neigh}")"
+
+		ip link add "veth-rt-${rt}-${neigh}" netns "${nsname}" \
+			type veth peer name "veth-rt-${neigh}-${rt}" \
+			netns "${neigh_nsname}"
+	done
+}
+
+get_network_prefix()
+{
+	local rt="$1"
+	local neigh="$2"
+	local p="${rt}"
+	local q="${neigh}"
+
+	if [ "${p}" -gt "${q}" ]; then
+		p="${q}"; q="${rt}"
+	fi
+
+	echo "${IPv6_RT_NETWORK}:${p}:${q}"
+}
+
+# Setup the basic networking for the routers
+setup_rt_networking()
+{
+	local rt="$1"
+	local rt_neighs="$2"
+	local nsname
+	local net_prefix
+	local devname
+	local neigh
+
+	nsname="$(get_rtname "${rt}")"
+
+	for neigh in ${rt_neighs}; do
+		devname="veth-rt-${rt}-${neigh}"
+
+		net_prefix="$(get_network_prefix "${rt}" "${neigh}")"
+
+		ip -netns "${nsname}" addr \
+			add "${net_prefix}::${rt}/64" dev "${devname}" nodad
+
+		ip -netns "${nsname}" link set "${devname}" up
+	done
+
+        ip -netns "${nsname}" link add "${DUMMY_DEVNAME}" type dummy
+
+        ip -netns "${nsname}" link set "${DUMMY_DEVNAME}" up
+	ip -netns "${nsname}" link set lo up
+
+	ip netns exec "${nsname}" sysctl -wq net.ipv6.conf.all.accept_dad=0
+	ip netns exec "${nsname}" sysctl -wq net.ipv6.conf.default.accept_dad=0
+	ip netns exec "${nsname}" sysctl -wq net.ipv6.conf.all.forwarding=1
+
+	ip netns exec "${nsname}" sysctl -wq net.ipv4.conf.all.rp_filter=0
+	ip netns exec "${nsname}" sysctl -wq net.ipv4.conf.default.rp_filter=0
+	ip netns exec "${nsname}" sysctl -wq net.ipv4.ip_forward=1
+}
+
+# build an ipv6 prefix/address based on the input string
+# Note that the input string does not contain ':' and '::' which are considered
+# to be implicit.
+# e.g.:
+#  - input:  fbcc00000400300
+#  - output: fbcc:0000:0400:0300:0000:0000:0000:0000
+#                                ^^^^^^^^^^^^^^^^^^^
+#                              fill the address with 0s
+build_ipv6_addr()
+{
+	local addr="$1"
+	local out=""
+	local strlen="${#addr}"
+	local padn
+	local i
+
+	# add ":" every 4 digits (16 bits)
+	for (( i = 0; i < strlen; i++ )); do
+		if (( i > 0 && i < 32 && (i % 4) == 0 )); then
+			out="${out}:"
+		fi
+
+		out="${out}${addr:$i:1}"
+	done
+
+	# fill the remaining bits of the address with 0s
+	padn=$((32 - strlen))
+	for (( i = padn; i > 0; i-- )); do
+		if (( i > 0 && i < 32 && (i % 4) == 0 )); then
+			out="${out}:"
+		fi
+
+		out="${out}0"
+	done
+
+	printf "${out}"
+}
+
+build_csid()
+{
+	local nodeid="$1"
+
+	printf "${LCNODEFUNC_FMT}" "${nodeid}"
+}
+
+build_lcnode_func_prefix()
+{
+	local nodeid="$1"
+	local lcnodefunc
+	local prefix
+	local out
+
+	lcnodefunc="$(build_csid "${nodeid}")"
+	prefix="$(build_ipv6_addr "${LCBLOCK_ADDR}${lcnodefunc}")"
+
+	out="${prefix}/${LCBLOCK_NODEFUNC_BLEN}"
+
+	echo "${out}"
+}
+
+# Setup local SIDs for an SRv6 router
+setup_rt_local_sids()
+{
+	local rt="$1"
+	local rt_neighs="$2"
+	local net_prefix
+	local devname
+	local nsname
+	local neigh
+	local lcnode_func_prefix
+	local lcblock_prefix
+
+	nsname="$(get_rtname "${rt}")"
+
+	for neigh in ${rt_neighs}; do
+		devname="veth-rt-${rt}-${neigh}"
+
+		net_prefix="$(get_network_prefix "${rt}" "${neigh}")"
+
+		# set underlay network routes for SIDs reachability
+		ip -netns "${nsname}" -6 route \
+			add "${VPN_LOCATOR_SERVICE}:${neigh}::/32" \
+			table "${LOCALSID_TABLE_ID}" \
+			via "${net_prefix}::${neigh}" dev "${devname}"
+
+		# set the underlay network for C-SIDs reachability
+		lcnode_func_prefix="$(build_lcnode_func_prefix "${neigh}")"
+
+		ip -netns "${nsname}" -6 route \
+			add "${lcnode_func_prefix}" \
+			table "${LOCALSID_TABLE_ID}" \
+			via "${net_prefix}::${neigh}" dev "${devname}"
+	done
+
+	lcnode_func_prefix="$(build_lcnode_func_prefix "${rt}")"
+
+	# enabled NEXT-C-SID SRv6 End behavior (note that "dev" is the dummy
+	# dum0 device chosen for the sake of simplicity).
+	ip -netns "${nsname}" -6 route \
+		add "${lcnode_func_prefix}" \
+		table "${LOCALSID_TABLE_ID}" \
+		encap seg6local action End flavors next-csid \
+		lblen "${LCBLOCK_BLEN}" nflen "${LCNODEFUNC_BLEN}" \
+		dev "${DUMMY_DEVNAME}"
+
+	# all SIDs for VPNs start with a common locator. Routes and SRv6
+	# Endpoint behavior instaces are grouped together in the 'localsid'
+	# table.
+	ip -netns "${nsname}" -6 rule \
+		add to "${VPN_LOCATOR_SERVICE}::/16" \
+		lookup "${LOCALSID_TABLE_ID}" prio 999
+
+	# common locator block for NEXT-C-SIDS compression mechanism.
+	lcblock_prefix="$(build_ipv6_addr "${LCBLOCK_ADDR}")"
+	ip -netns "${nsname}" -6 rule \
+		add to "${lcblock_prefix}/${LCBLOCK_BLEN}" \
+		lookup "${LOCALSID_TABLE_ID}" prio 999
+}
+
+# build and install the SRv6 policy into the ingress SRv6 router as well as the
+# decap SID in the egress one.
+# args:
+#  $1 - src host (evaluate automatically the ingress router)
+#  $2 - dst host (evaluate automatically the egress router)
+#  $3 - SRv6 routers configured for steering traffic (End behaviors)
+#  $4 - single SID or double SID
+#  $5 - traffic type (IPv6 or IPv4)
+__setup_l3vpn()
+{
+	local src="$1"
+	local dst="$2"
+	local end_rts="$3"
+	local mode="$4"
+	local traffic="$5"
+	local nsname
+	local policy
+	local container
+	local decapsid
+	local lcnfunc
+	local dt
+	local n
+	local rtsrc_nsname
+	local rtdst_nsname
+
+	rtsrc_nsname="$(get_rtname "${src}")"
+	rtdst_nsname="$(get_rtname "${dst}")"
+
+	container="${LCBLOCK_ADDR}"
+
+	# build first SID (C-SID container)
+	for n in ${end_rts}; do
+		lcnfunc="$(build_csid "${n}")"
+
+		container="${container}${lcnfunc}"
+	done
+
+	if [ "${mode}" -eq 1 ]; then
+		# single SID policy
+		dt="$(build_csid "${dst}")${DT46_FUNC}"
+		container="${container}${dt}"
+		# build the full ipv6 address for the container
+		policy="$(build_ipv6_addr "${container}")"
+
+		# build the decap SID used in the decap node
+		container="${LCBLOCK_ADDR}${dt}"
+		decapsid="$(build_ipv6_addr "${container}")"
+	else
+		# double SID policy
+		decapsid="${VPN_LOCATOR_SERVICE}:${dst}::${DT46_FUNC}"
+
+		policy="$(build_ipv6_addr "${container}"),${decapsid}"
+	fi
+
+	# apply encap policy
+	if [ "${traffic}" -eq 6 ]; then
+		ip -netns "${rtsrc_nsname}" -6 route \
+			add "${IPv6_HS_NETWORK}::${dst}" vrf "${VRF_DEVNAME}" \
+			encap seg6 mode "${HEADEND_ENCAP}" segs "${policy}" \
+			dev "${VRF_DEVNAME}"
+
+		ip -netns "${rtsrc_nsname}" -6 neigh \
+			add proxy "${IPv6_HS_NETWORK}::${dst}" \
+			dev "${RT2HS_DEVNAME}"
+	else
+		# "dev" must be different from the one where the packet is
+		# received, otherwise the proxy arp does not work.
+		ip -netns "${rtsrc_nsname}" -4 route \
+			add "${IPv4_HS_NETWORK}.${dst}" vrf "${VRF_DEVNAME}" \
+			encap seg6 mode "${HEADEND_ENCAP}" segs "${policy}" \
+			dev "${VRF_DEVNAME}"
+	fi
+
+	# apply decap
+	# Local End.DT46 behavior (decap)
+	ip -netns "${rtdst_nsname}" -6 route \
+		add "${decapsid}" \
+		table "${LOCALSID_TABLE_ID}" \
+		encap seg6local action End.DT46 vrftable "${VRF_TID}" \
+		dev "${VRF_DEVNAME}"
+}
+
+# see __setup_l3vpn()
+setup_ipv4_vpn_2sids()
+{
+	__setup_l3vpn "$1" "$2" "$3" 2 4
+}
+
+# see __setup_l3vpn()
+setup_ipv6_vpn_1sid()
+{
+	__setup_l3vpn "$1" "$2" "$3" 1 6
+}
+
+setup_hs()
+{
+	local hs="$1"
+	local rt="$2"
+	local hsname
+	local rtname
+
+	hsname="$(get_hsname "${hs}")"
+	rtname="$(get_rtname "${rt}")"
+
+	ip netns exec "${hsname}" sysctl -wq net.ipv6.conf.all.accept_dad=0
+	ip netns exec "${hsname}" sysctl -wq net.ipv6.conf.default.accept_dad=0
+
+	ip -netns "${hsname}" link add veth0 type veth \
+		peer name "${RT2HS_DEVNAME}" netns "${rtname}"
+
+	ip -netns "${hsname}" addr \
+		add "${IPv6_HS_NETWORK}::${hs}/64" dev veth0 nodad
+	ip -netns "${hsname}" addr add "${IPv4_HS_NETWORK}.${hs}/24" dev veth0
+
+	ip -netns "${hsname}" link set veth0 up
+	ip -netns "${hsname}" link set lo up
+
+	# configure the VRF on the router which is directly connected to the
+	# source host.
+	ip -netns "${rtname}" link \
+		add "${VRF_DEVNAME}" type vrf table "${VRF_TID}"
+	ip -netns "${rtname}" link set "${VRF_DEVNAME}" up
+
+	# enslave the veth interface connecting the router with the host to the
+	# VRF in the access router
+	ip -netns "${rtname}" link \
+		set "${RT2HS_DEVNAME}" master "${VRF_DEVNAME}"
+
+	# set default routes to unreachable for both ipv6 and ipv4
+	ip -netns "${rtname}" -6 route \
+		add unreachable default metric 4278198272 \
+		vrf "${VRF_DEVNAME}"
+	ip -netns "${rtname}" -4 route \
+		add unreachable default metric 4278198272 \
+		vrf "${VRF_DEVNAME}"
+
+	ip -netns "${rtname}" addr \
+		add "${IPv6_HS_NETWORK}::254/64" dev "${RT2HS_DEVNAME}" nodad
+	ip -netns "${rtname}" addr \
+		add "${IPv4_HS_NETWORK}.254/24" dev "${RT2HS_DEVNAME}"
+
+	ip -netns "${rtname}" link set "${RT2HS_DEVNAME}" up
+
+	ip netns exec "${rtname}" \
+		sysctl -wq net.ipv6.conf."${RT2HS_DEVNAME}".proxy_ndp=1
+	ip netns exec "${rtname}" \
+		sysctl -wq net.ipv4.conf."${RT2HS_DEVNAME}".proxy_arp=1
+
+	# disable the rp_filter otherwise the kernel gets confused about how
+	# to route decap ipv4 packets.
+	ip netns exec "${rtname}" \
+		sysctl -wq net.ipv4.conf."${RT2HS_DEVNAME}".rp_filter=0
+
+	ip netns exec "${rtname}" sh -c "echo 1 > /proc/sys/net/vrf/strict_mode"
+}
+
+setup()
+{
+	local i
+
+	# create routers
+	ROUTERS="1 2 3 4"; readonly ROUTERS
+	for i in ${ROUTERS}; do
+		create_router "${i}"
+	done
+
+	# create hosts
+	HOSTS="1 2"; readonly HOSTS
+	for i in ${HOSTS}; do
+		create_host "${i}"
+	done
+
+	# set up the links for connecting routers
+	add_link_rt_pairs 1 "2 3 4"
+	add_link_rt_pairs 2 "3 4"
+	add_link_rt_pairs 3 "4"
+
+	# set up the basic connectivity of routers and routes required for
+	# reachability of SIDs.
+	setup_rt_networking 1 "2 3 4"
+	setup_rt_networking 2 "1 3 4"
+	setup_rt_networking 3 "1 2 4"
+	setup_rt_networking 4 "1 2 3"
+
+	# set up the hosts connected to routers
+	setup_hs 1 1
+	setup_hs 2 2
+
+	# set up default SRv6 Endpoints (i.e. SRv6 End and SRv6 End.DT46)
+	setup_rt_local_sids 1 "2 3 4"
+	setup_rt_local_sids 2 "1 3 4"
+	setup_rt_local_sids 3 "1 2 4"
+	setup_rt_local_sids 4 "1 2 3"
+
+	# set up SRv6 Policies
+
+	# create an IPv6 VPN between hosts hs-1 and hs-2.
+	#
+	# Direction hs-1 -> hs-2
+	# - rt-1 encap (H.Encaps.Red)
+	# - rt-4 SRv6 End behavior (NEXT-C-SID flavor)
+	# - rt-3 SRv6 End behavior (NEXT-C-SID flavor)
+	# - rt-2 SRv6 End.DT46 behavior
+	setup_ipv6_vpn_1sid 1 2 "4 3"
+
+	# Direction hs2 -> hs-1
+	# - rt-2 encap (H.Encaps.Red)
+	# - rt-3 SRv6 End behavior (NEXT-C-SID flavor)
+	# - rt-4 SRv6 End behavior (NEXT-C-SID flavor)
+	# - rt-1 SRv6 End.DT46 behavior
+	setup_ipv6_vpn_1sid 2 1 "3 4"
+
+	# create an IPv4 VPN between hosts hs-1 and hs-2
+	#
+	# Direction hs-1 -> hs-2
+	# - rt-1 encap (H.Encaps.Red)
+	# - rt-3 SRv6 End behavior (NEXT-C-SID flavor)
+	# - rt-2 SRv6 End.DT46 behavior
+	setup_ipv4_vpn_2sids 1 2 "3"
+
+	# Direction hs-2 -> hs-1
+	# - rt-2 encap (H.Encaps.Red)
+	# - rt-3 SRv6 End behavior (NEXT-C-SID flavor)
+	# - rt-1 SRv6 End.DT46 behavior
+	setup_ipv4_vpn_2sids 2 1 "3"
+
+	# testing environment was set up successfully
+	SETUP_ERR=0
+}
+
+check_rt_connectivity()
+{
+	local rtsrc="$1"
+	local rtdst="$2"
+	local prefix
+	local rtsrc_nsname
+
+	rtsrc_nsname="$(get_rtname "${rtsrc}")"
+
+	prefix="$(get_network_prefix "${rtsrc}" "${rtdst}")"
+
+	ip netns exec "${rtsrc_nsname}" ping -c 1 -W "${PING_TIMEOUT_SEC}" \
+		"${prefix}::${rtdst}" >/dev/null 2>&1
+}
+
+check_and_log_rt_connectivity()
+{
+	local rtsrc="$1"
+	local rtdst="$2"
+
+	check_rt_connectivity "${rtsrc}" "${rtdst}"
+	log_test $? 0 "Routers connectivity: rt-${rtsrc} -> rt-${rtdst}"
+}
+
+check_hs_ipv6_connectivity()
+{
+	local hssrc="$1"
+	local hsdst="$2"
+	local hssrc_nsname
+
+	hssrc_nsname="$(get_hsname "${hssrc}")"
+
+	ip netns exec "${hssrc_nsname}" ping -c 1 -W "${PING_TIMEOUT_SEC}" \
+		"${IPv6_HS_NETWORK}::${hsdst}" >/dev/null 2>&1
+}
+
+check_hs_ipv4_connectivity()
+{
+	local hssrc="$1"
+	local hsdst="$2"
+	local hssrc_nsname
+
+	hssrc_nsname="$(get_hsname "${hssrc}")"
+
+	ip netns exec "${hssrc_nsname}" ping -c 1 -W "${PING_TIMEOUT_SEC}" \
+		"${IPv4_HS_NETWORK}.${hsdst}" >/dev/null 2>&1
+}
+
+check_and_log_hs2gw_connectivity()
+{
+	local hssrc="$1"
+
+	check_hs_ipv6_connectivity "${hssrc}" 254
+	log_test $? 0 "IPv6 Hosts connectivity: hs-${hssrc} -> gw"
+
+	check_hs_ipv4_connectivity "${hssrc}" 254
+	log_test $? 0 "IPv4 Hosts connectivity: hs-${hssrc} -> gw"
+}
+
+check_and_log_hs_ipv6_connectivity()
+{
+	local hssrc="$1"
+	local hsdst="$2"
+
+	check_hs_ipv6_connectivity "${hssrc}" "${hsdst}"
+	log_test $? 0 "IPv6 Hosts connectivity: hs-${hssrc} -> hs-${hsdst}"
+}
+
+check_and_log_hs_ipv4_connectivity()
+{
+	local hssrc="$1"
+	local hsdst="$2"
+
+	check_hs_ipv4_connectivity "${hssrc}" "${hsdst}"
+	log_test $? 0 "IPv4 Hosts connectivity: hs-${hssrc} -> hs-${hsdst}"
+}
+
+router_tests()
+{
+	local i
+	local j
+
+	log_section "IPv6 routers connectivity test"
+
+	for i in ${ROUTERS}; do
+		for j in ${ROUTERS}; do
+			if [ "${i}" -eq "${j}" ]; then
+				continue
+			fi
+
+			check_and_log_rt_connectivity "${i}" "${j}"
+		done
+	done
+}
+
+host2gateway_tests()
+{
+	local hs
+
+	log_section "IPv4/IPv6 connectivity test among hosts and gateways"
+
+	for hs in ${HOSTS}; do
+		check_and_log_hs2gw_connectivity "${hs}"
+	done
+}
+
+host_vpn_tests()
+{
+	log_section "SRv6 VPN connectivity test hosts (h1 <-> h2, IPv6)"
+
+	check_and_log_hs_ipv6_connectivity 1 2
+	check_and_log_hs_ipv6_connectivity 2 1
+
+	log_section "SRv6 VPN connectivity test hosts (h1 <-> h2, IPv4)"
+
+	check_and_log_hs_ipv4_connectivity 1 2
+	check_and_log_hs_ipv4_connectivity 2 1
+}
+
+__nextcsid_end_behavior_test()
+{
+	local nsname="$1"
+	local cmd="$2"
+	local blen="$3"
+	local flen="$4"
+	local layout=""
+
+	if [ "${blen}" != "d" ]; then
+		layout="${layout} lblen ${blen}"
+	fi
+
+	if [ "${flen}" != "d" ]; then
+		layout="${layout} nflen ${flen}"
+	fi
+
+	ip -netns "${nsname}" -6 route \
+		"${cmd}" "${CSID_CNTR_PREFIX}" \
+		table "${CSID_CNTR_RT_TABLE}" \
+		encap seg6local action End flavors next-csid ${layout} \
+		dev "${DUMMY_DEVNAME}" &>/dev/null
+
+	return "$?"
+}
+
+rt_x_nextcsid_end_behavior_test()
+{
+	local rt="$1"
+	local blen="$2"
+	local flen="$3"
+	local nsname
+	local ret
+
+	nsname="$(get_rtname "${rt}")"
+
+	__nextcsid_end_behavior_test "${nsname}" "add" "${blen}" "${flen}"
+	ret="$?"
+	__nextcsid_end_behavior_test "${nsname}" "del" "${blen}" "${flen}"
+
+	return "${ret}"
+}
+
+__parse_csid_container_cfg()
+{
+	local cfg="$1"
+	local index="$2"
+	local out
+
+	echo "${cfg}" | cut -d',' -f"${index}"
+}
+
+csid_container_cfg_tests()
+{
+	local valid
+	local blen
+	local flen
+	local cfg
+	local ret
+
+	log_section "C-SID Container config tests (legend: d='kernel default')"
+
+	for cfg in "${CSID_CONTAINER_CFGS[@]}"; do
+		blen="$(__parse_csid_container_cfg "${cfg}" 1)"
+		flen="$(__parse_csid_container_cfg "${cfg}" 2)"
+		valid="$(__parse_csid_container_cfg "${cfg}" 3)"
+
+		rt_x_nextcsid_end_behavior_test \
+			"${CSID_CNTR_RT_ID_TEST}" \
+			"${blen}" \
+			"${flen}"
+		ret="$?"
+
+		if [ "${valid}" == "y" ]; then
+			log_test "${ret}" 0 \
+				"Accept valid C-SID container cfg (lblen=${blen}, nflen=${flen})"
+		else
+			log_test "${ret}" 2 \
+				"Reject invalid C-SID container cfg (lblen=${blen}, nflen=${flen})"
+		fi
+	done
+}
+
+test_iproute2_supp_or_ksft_skip()
+{
+	if ! ip route help 2>&1 | grep -qo "next-csid"; then
+		echo "SKIP: Missing SRv6 NEXT-C-SID flavor support in iproute2"
+		exit "${ksft_skip}"
+	fi
+}
+
+test_dummy_dev_or_ksft_skip()
+{
+        local test_netns
+
+        test_netns="dummy-$(mktemp -u XXXXXXXX)"
+
+        if ! ip netns add "${test_netns}"; then
+                echo "SKIP: Cannot set up netns for testing dummy dev support"
+                exit "${ksft_skip}"
+        fi
+
+        modprobe dummy &>/dev/null || true
+        if ! ip -netns "${test_netns}" link \
+                add "${DUMMY_DEVNAME}" type dummy; then
+                echo "SKIP: dummy dev not supported"
+
+                ip netns del "${test_netns}"
+                exit "${ksft_skip}"
+        fi
+
+        ip netns del "${test_netns}"
+}
+
+test_vrf_or_ksft_skip()
+{
+	modprobe vrf &>/dev/null || true
+	if [ ! -e /proc/sys/net/vrf/strict_mode ]; then
+		echo "SKIP: vrf sysctl does not exist"
+		exit "${ksft_skip}"
+	fi
+}
+
+if [ "$(id -u)" -ne 0 ]; then
+	echo "SKIP: Need root privileges"
+	exit "${ksft_skip}"
+fi
+
+# required programs to carry out this selftest
+test_command_or_ksft_skip ip
+test_command_or_ksft_skip ping
+test_command_or_ksft_skip sysctl
+test_command_or_ksft_skip grep
+test_command_or_ksft_skip cut
+
+test_iproute2_supp_or_ksft_skip
+test_dummy_dev_or_ksft_skip
+test_vrf_or_ksft_skip
+
+set -e
+trap cleanup EXIT
+
+setup
+set +e
+
+csid_container_cfg_tests
+
+router_tests
+host2gateway_tests
+host_vpn_tests
+
+print_log_test_results
diff --git a/tools/testing/selftests/nolibc/.gitignore b/tools/testing/selftests/nolibc/.gitignore
new file mode 100644
index 000000000000..4696df589d68
--- /dev/null
+++ b/tools/testing/selftests/nolibc/.gitignore
@@ -0,0 +1,4 @@
+/initramfs/
+/nolibc-test
+/run.out
+/sysroot/
diff --git a/tools/testing/selftests/nolibc/Makefile b/tools/testing/selftests/nolibc/Makefile
new file mode 100644
index 000000000000..69ea659caca9
--- /dev/null
+++ b/tools/testing/selftests/nolibc/Makefile
@@ -0,0 +1,135 @@
+# SPDX-License-Identifier: GPL-2.0
+# Makefile for nolibc tests
+include ../../../scripts/Makefile.include
+
+# we're in ".../tools/testing/selftests/nolibc"
+ifeq ($(srctree),)
+srctree := $(patsubst %/tools/testing/selftests/,%,$(dir $(CURDIR)))
+endif
+
+ifeq ($(ARCH),)
+include $(srctree)/scripts/subarch.include
+ARCH = $(SUBARCH)
+endif
+
+# kernel image names by architecture
+IMAGE_i386    = arch/x86/boot/bzImage
+IMAGE_x86     = arch/x86/boot/bzImage
+IMAGE_arm64   = arch/arm64/boot/Image
+IMAGE_arm     = arch/arm/boot/zImage
+IMAGE_mips    = vmlinuz
+IMAGE_riscv   = arch/riscv/boot/Image
+IMAGE         = $(IMAGE_$(ARCH))
+IMAGE_NAME    = $(notdir $(IMAGE))
+
+# default kernel configurations that appear to be usable
+DEFCONFIG_i386    = defconfig
+DEFCONFIG_x86     = defconfig
+DEFCONFIG_arm64   = defconfig
+DEFCONFIG_arm     = multi_v7_defconfig
+DEFCONFIG_mips    = malta_defconfig
+DEFCONFIG_riscv   = defconfig
+DEFCONFIG         = $(DEFCONFIG_$(ARCH))
+
+# optional tests to run (default = all)
+TEST =
+
+# QEMU_ARCH: arch names used by qemu
+QEMU_ARCH_i386    = i386
+QEMU_ARCH_x86     = x86_64
+QEMU_ARCH_arm64   = aarch64
+QEMU_ARCH_arm     = arm
+QEMU_ARCH_mips    = mipsel  # works with malta_defconfig
+QEMU_ARCH_riscv   = riscv64
+QEMU_ARCH         = $(QEMU_ARCH_$(ARCH))
+
+# QEMU_ARGS : some arch-specific args to pass to qemu
+QEMU_ARGS_i386    = -M pc -append "console=ttyS0,9600 i8042.noaux panic=-1 $(TEST:%=NOLIBC_TEST=%)"
+QEMU_ARGS_x86     = -M pc -append "console=ttyS0,9600 i8042.noaux panic=-1 $(TEST:%=NOLIBC_TEST=%)"
+QEMU_ARGS_arm64   = -M virt -cpu cortex-a53 -append "panic=-1 $(TEST:%=NOLIBC_TEST=%)"
+QEMU_ARGS_arm     = -M virt -append "panic=-1 $(TEST:%=NOLIBC_TEST=%)"
+QEMU_ARGS_mips    = -M malta -append "panic=-1 $(TEST:%=NOLIBC_TEST=%)"
+QEMU_ARGS_riscv   = -M virt -append "console=ttyS0 panic=-1 $(TEST:%=NOLIBC_TEST=%)"
+QEMU_ARGS         = $(QEMU_ARGS_$(ARCH))
+
+# OUTPUT is only set when run from the main makefile, otherwise
+# it defaults to this nolibc directory.
+OUTPUT ?= $(CURDIR)/
+
+ifeq ($(V),1)
+Q=
+else
+Q=@
+endif
+
+CFLAGS  ?= -Os -fno-ident -fno-asynchronous-unwind-tables
+LDFLAGS := -s
+
+help:
+	@echo "Supported targets under selftests/nolibc:"
+	@echo "  all          call the \"run\" target below"
+	@echo "  help         this help"
+	@echo "  sysroot      create the nolibc sysroot here (uses \$$ARCH)"
+	@echo "  nolibc-test  build the executable (uses \$$CC and \$$CROSS_COMPILE)"
+	@echo "  initramfs    prepare the initramfs with nolibc-test"
+	@echo "  defconfig    create a fresh new default config (uses \$$ARCH)"
+	@echo "  kernel       (re)build the kernel with the initramfs (uses \$$ARCH)"
+	@echo "  run          runs the kernel in QEMU after building it (uses \$$ARCH, \$$TEST)"
+	@echo "  rerun        runs a previously prebuilt kernel in QEMU (uses \$$ARCH, \$$TEST)"
+	@echo "  clean        clean the sysroot, initramfs, build and output files"
+	@echo ""
+	@echo "The output file is \"run.out\". Test ranges may be passed using \$$TEST."
+	@echo ""
+	@echo "Currently using the following variables:"
+	@echo "  ARCH          = $(ARCH)"
+	@echo "  CROSS_COMPILE = $(CROSS_COMPILE)"
+	@echo "  CC            = $(CC)"
+	@echo "  OUTPUT        = $(OUTPUT)"
+	@echo "  TEST          = $(TEST)"
+	@echo "  QEMU_ARCH     = $(if $(QEMU_ARCH),$(QEMU_ARCH),UNKNOWN_ARCH) [determined from \$$ARCH]"
+	@echo "  IMAGE_NAME    = $(if $(IMAGE_NAME),$(IMAGE_NAME),UNKNOWN_ARCH) [determined from \$$ARCH]"
+	@echo ""
+
+all: run
+
+sysroot: sysroot/$(ARCH)/include
+
+sysroot/$(ARCH)/include:
+	$(QUIET_MKDIR)mkdir -p sysroot
+	$(Q)$(MAKE) -C ../../../include/nolibc ARCH=$(ARCH) OUTPUT=$(CURDIR)/sysroot/ headers_standalone
+	$(Q)mv sysroot/sysroot sysroot/$(ARCH)
+
+nolibc-test: nolibc-test.c sysroot/$(ARCH)/include
+	$(QUIET_CC)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ \
+	  -nostdlib -static -Isysroot/$(ARCH)/include $< -lgcc
+
+initramfs: nolibc-test
+	$(QUIET_MKDIR)mkdir -p initramfs
+	$(call QUIET_INSTALL, initramfs/init)
+	$(Q)cp nolibc-test initramfs/init
+
+defconfig:
+	$(Q)$(MAKE) -C $(srctree) ARCH=$(ARCH) CC=$(CC) CROSS_COMPILE=$(CROSS_COMPILE) mrproper $(DEFCONFIG) prepare
+
+kernel: initramfs
+	$(Q)$(MAKE) -C $(srctree) ARCH=$(ARCH) CC=$(CC) CROSS_COMPILE=$(CROSS_COMPILE) $(IMAGE_NAME) CONFIG_INITRAMFS_SOURCE=$(CURDIR)/initramfs
+
+# run the tests after building the kernel
+run: kernel
+	$(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out"
+	$(Q)grep -w FAIL "$(CURDIR)/run.out" && echo "See all results in $(CURDIR)/run.out" || echo "$$(grep -c ^[0-9].*OK $(CURDIR)/run.out) test(s) passed."
+
+# re-run the tests from an existing kernel
+rerun:
+	$(Q)qemu-system-$(QEMU_ARCH) -display none -no-reboot -kernel "$(srctree)/$(IMAGE)" -serial stdio $(QEMU_ARGS) > "$(CURDIR)/run.out"
+	$(Q)grep -w FAIL "$(CURDIR)/run.out" && echo "See all results in $(CURDIR)/run.out" || echo "$$(grep -c ^[0-9].*OK $(CURDIR)/run.out) test(s) passed."
+
+clean:
+	$(call QUIET_CLEAN, sysroot)
+	$(Q)rm -rf sysroot
+	$(call QUIET_CLEAN, nolibc-test)
+	$(Q)rm -f nolibc-test
+	$(call QUIET_CLEAN, initramfs)
+	$(Q)rm -rf initramfs
+	$(call QUIET_CLEAN, run.out)
+	$(Q)rm -rf run.out
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c
new file mode 100644
index 000000000000..78bced95ac63
--- /dev/null
+++ b/tools/testing/selftests/nolibc/nolibc-test.c
@@ -0,0 +1,757 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#define _GNU_SOURCE
+
+/* platform-specific include files coming from the compiler */
+#include <limits.h>
+
+/* libc-specific include files
+ * The program may be built in 3 ways:
+ *   $(CC) -nostdlib -include /path/to/nolibc.h => NOLIBC already defined
+ *   $(CC) -nostdlib -I/path/to/nolibc/sysroot  => _NOLIBC_* guards are present
+ *   $(CC) with default libc                    => NOLIBC* never defined
+ */
+#ifndef NOLIBC
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#ifndef _NOLIBC_STDIO_H
+/* standard libcs need more includes */
+#include <linux/reboot.h>
+#include <sys/io.h>
+#include <sys/ioctl.h>
+#include <sys/mount.h>
+#include <sys/reboot.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/sysmacros.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <poll.h>
+#include <sched.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <unistd.h>
+#endif
+#endif
+
+/* will be used by nolibc by getenv() */
+char **environ;
+
+/* definition of a series of tests */
+struct test {
+	const char *name;              // test name
+	int (*func)(int min, int max); // handler
+};
+
+#ifndef _NOLIBC_STDLIB_H
+char *itoa(int i)
+{
+	static char buf[12];
+	int ret;
+
+	ret = snprintf(buf, sizeof(buf), "%d", i);
+	return (ret >= 0 && ret < sizeof(buf)) ? buf : "#err";
+}
+#endif
+
+#define CASE_ERR(err) \
+	case err: return #err
+
+/* returns the error name (e.g. "ENOENT") for common errors, "SUCCESS" for 0,
+ * or the decimal value for less common ones.
+ */
+const char *errorname(int err)
+{
+	switch (err) {
+	case 0: return "SUCCESS";
+	CASE_ERR(EPERM);
+	CASE_ERR(ENOENT);
+	CASE_ERR(ESRCH);
+	CASE_ERR(EINTR);
+	CASE_ERR(EIO);
+	CASE_ERR(ENXIO);
+	CASE_ERR(E2BIG);
+	CASE_ERR(ENOEXEC);
+	CASE_ERR(EBADF);
+	CASE_ERR(ECHILD);
+	CASE_ERR(EAGAIN);
+	CASE_ERR(ENOMEM);
+	CASE_ERR(EACCES);
+	CASE_ERR(EFAULT);
+	CASE_ERR(ENOTBLK);
+	CASE_ERR(EBUSY);
+	CASE_ERR(EEXIST);
+	CASE_ERR(EXDEV);
+	CASE_ERR(ENODEV);
+	CASE_ERR(ENOTDIR);
+	CASE_ERR(EISDIR);
+	CASE_ERR(EINVAL);
+	CASE_ERR(ENFILE);
+	CASE_ERR(EMFILE);
+	CASE_ERR(ENOTTY);
+	CASE_ERR(ETXTBSY);
+	CASE_ERR(EFBIG);
+	CASE_ERR(ENOSPC);
+	CASE_ERR(ESPIPE);
+	CASE_ERR(EROFS);
+	CASE_ERR(EMLINK);
+	CASE_ERR(EPIPE);
+	CASE_ERR(EDOM);
+	CASE_ERR(ERANGE);
+	CASE_ERR(ENOSYS);
+	default:
+		return itoa(err);
+	}
+}
+
+static int pad_spc(int llen, int cnt, const char *fmt, ...)
+{
+	va_list args;
+	int len;
+	int ret;
+
+	for (len = 0; len < cnt - llen; len++)
+		putchar(' ');
+
+	va_start(args, fmt);
+	ret = vfprintf(stdout, fmt, args);
+	va_end(args);
+	return ret < 0 ? ret : ret + len;
+}
+
+/* The tests below are intended to be used by the macroes, which evaluate
+ * expression <expr>, print the status to stdout, and update the "ret"
+ * variable to count failures. The functions themselves return the number
+ * of failures, thus either 0 or 1.
+ */
+
+#define EXPECT_ZR(cond, expr)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_zr(expr, llen); } while (0)
+
+static int expect_zr(int expr, int llen)
+{
+	int ret = !(expr == 0);
+
+	llen += printf(" = %d ", expr);
+	pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n");
+	return ret;
+}
+
+
+#define EXPECT_NZ(cond, expr, val)			\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_nz(expr, llen; } while (0)
+
+static int expect_nz(int expr, int llen)
+{
+	int ret = !(expr != 0);
+
+	llen += printf(" = %d ", expr);
+	pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n");
+	return ret;
+}
+
+
+#define EXPECT_EQ(cond, expr, val)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_eq(expr, llen, val); } while (0)
+
+static int expect_eq(int expr, int llen, int val)
+{
+	int ret = !(expr == val);
+
+	llen += printf(" = %d ", expr);
+	pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n");
+	return ret;
+}
+
+
+#define EXPECT_NE(cond, expr, val)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_ne(expr, llen, val); } while (0)
+
+static int expect_ne(int expr, int llen, int val)
+{
+	int ret = !(expr != val);
+
+	llen += printf(" = %d ", expr);
+	pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n");
+	return ret;
+}
+
+
+#define EXPECT_GE(cond, expr, val)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_ge(expr, llen, val); } while (0)
+
+static int expect_ge(int expr, int llen, int val)
+{
+	int ret = !(expr >= val);
+
+	llen += printf(" = %d ", expr);
+	pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n");
+	return ret;
+}
+
+
+#define EXPECT_GT(cond, expr, val)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_gt(expr, llen, val); } while (0)
+
+static int expect_gt(int expr, int llen, int val)
+{
+	int ret = !(expr > val);
+
+	llen += printf(" = %d ", expr);
+	pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n");
+	return ret;
+}
+
+
+#define EXPECT_LE(cond, expr, val)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_le(expr, llen, val); } while (0)
+
+static int expect_le(int expr, int llen, int val)
+{
+	int ret = !(expr <= val);
+
+	llen += printf(" = %d ", expr);
+	pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n");
+	return ret;
+}
+
+
+#define EXPECT_LT(cond, expr, val)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_lt(expr, llen, val); } while (0)
+
+static int expect_lt(int expr, int llen, int val)
+{
+	int ret = !(expr < val);
+
+	llen += printf(" = %d ", expr);
+	pad_spc(llen, 40, ret ? "[FAIL]\n" : " [OK]\n");
+	return ret;
+}
+
+
+#define EXPECT_SYSZR(cond, expr)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_syszr(expr, llen); } while (0)
+
+static int expect_syszr(int expr, int llen)
+{
+	int ret = 0;
+
+	if (expr) {
+		ret = 1;
+		llen += printf(" = %d %s ", expr, errorname(errno));
+		llen += pad_spc(llen, 40, "[FAIL]\n");
+	} else {
+		llen += printf(" = %d ", expr);
+		llen += pad_spc(llen, 40, " [OK]\n");
+	}
+	return ret;
+}
+
+
+#define EXPECT_SYSEQ(cond, expr, val)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_syseq(expr, llen, val); } while (0)
+
+static int expect_syseq(int expr, int llen, int val)
+{
+	int ret = 0;
+
+	if (expr != val) {
+		ret = 1;
+		llen += printf(" = %d %s ", expr, errorname(errno));
+		llen += pad_spc(llen, 40, "[FAIL]\n");
+	} else {
+		llen += printf(" = %d ", expr);
+		llen += pad_spc(llen, 40, " [OK]\n");
+	}
+	return ret;
+}
+
+
+#define EXPECT_SYSNE(cond, expr, val)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_sysne(expr, llen, val); } while (0)
+
+static int expect_sysne(int expr, int llen, int val)
+{
+	int ret = 0;
+
+	if (expr == val) {
+		ret = 1;
+		llen += printf(" = %d %s ", expr, errorname(errno));
+		llen += pad_spc(llen, 40, "[FAIL]\n");
+	} else {
+		llen += printf(" = %d ", expr);
+		llen += pad_spc(llen, 40, " [OK]\n");
+	}
+	return ret;
+}
+
+
+#define EXPECT_SYSER(cond, expr, expret, experr)			\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_syserr(expr, expret, experr, llen); } while (0)
+
+static int expect_syserr(int expr, int expret, int experr, int llen)
+{
+	int ret = 0;
+	int _errno = errno;
+
+	llen += printf(" = %d %s ", expr, errorname(_errno));
+	if (expr != expret || _errno != experr) {
+		ret = 1;
+		llen += printf(" != (%d %s) ", expret, errorname(experr));
+		llen += pad_spc(llen, 40, "[FAIL]\n");
+	} else {
+		llen += pad_spc(llen, 40, " [OK]\n");
+	}
+	return ret;
+}
+
+
+#define EXPECT_PTRZR(cond, expr)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_ptrzr(expr, llen); } while (0)
+
+static int expect_ptrzr(const void *expr, int llen)
+{
+	int ret = 0;
+
+	llen += printf(" = <%p> ", expr);
+	if (expr) {
+		ret = 1;
+		llen += pad_spc(llen, 40, "[FAIL]\n");
+	} else {
+		llen += pad_spc(llen, 40, " [OK]\n");
+	}
+	return ret;
+}
+
+
+#define EXPECT_PTRNZ(cond, expr)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_ptrnz(expr, llen); } while (0)
+
+static int expect_ptrnz(const void *expr, int llen)
+{
+	int ret = 0;
+
+	llen += printf(" = <%p> ", expr);
+	if (!expr) {
+		ret = 1;
+		llen += pad_spc(llen, 40, "[FAIL]\n");
+	} else {
+		llen += pad_spc(llen, 40, " [OK]\n");
+	}
+	return ret;
+}
+
+
+#define EXPECT_STRZR(cond, expr)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_strzr(expr, llen); } while (0)
+
+static int expect_strzr(const char *expr, int llen)
+{
+	int ret = 0;
+
+	llen += printf(" = <%s> ", expr);
+	if (expr) {
+		ret = 1;
+		llen += pad_spc(llen, 40, "[FAIL]\n");
+	} else {
+		llen += pad_spc(llen, 40, " [OK]\n");
+	}
+	return ret;
+}
+
+
+#define EXPECT_STRNZ(cond, expr)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_strnz(expr, llen); } while (0)
+
+static int expect_strnz(const char *expr, int llen)
+{
+	int ret = 0;
+
+	llen += printf(" = <%s> ", expr);
+	if (!expr) {
+		ret = 1;
+		llen += pad_spc(llen, 40, "[FAIL]\n");
+	} else {
+		llen += pad_spc(llen, 40, " [OK]\n");
+	}
+	return ret;
+}
+
+
+#define EXPECT_STREQ(cond, expr, cmp)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_streq(expr, llen, cmp); } while (0)
+
+static int expect_streq(const char *expr, int llen, const char *cmp)
+{
+	int ret = 0;
+
+	llen += printf(" = <%s> ", expr);
+	if (strcmp(expr, cmp) != 0) {
+		ret = 1;
+		llen += pad_spc(llen, 40, "[FAIL]\n");
+	} else {
+		llen += pad_spc(llen, 40, " [OK]\n");
+	}
+	return ret;
+}
+
+
+#define EXPECT_STRNE(cond, expr, cmp)				\
+	do { if (!cond) pad_spc(llen, 40, "[SKIPPED]\n"); else ret += expect_strne(expr, llen, cmp); } while (0)
+
+static int expect_strne(const char *expr, int llen, const char *cmp)
+{
+	int ret = 0;
+
+	llen += printf(" = <%s> ", expr);
+	if (strcmp(expr, cmp) == 0) {
+		ret = 1;
+		llen += pad_spc(llen, 40, "[FAIL]\n");
+	} else {
+		llen += pad_spc(llen, 40, " [OK]\n");
+	}
+	return ret;
+}
+
+
+/* declare tests based on line numbers. There must be exactly one test per line. */
+#define CASE_TEST(name) \
+	case __LINE__: llen += printf("%d %s", test, #name);
+
+
+/* used by some syscall tests below */
+int test_getdents64(const char *dir)
+{
+	char buffer[4096];
+	int fd, ret;
+	int err;
+
+	ret = fd = open(dir, O_RDONLY | O_DIRECTORY, 0);
+	if (ret < 0)
+		return ret;
+
+	ret = getdents64(fd, (void *)buffer, sizeof(buffer));
+	err = errno;
+	close(fd);
+
+	errno = err;
+	return ret;
+}
+
+/* Run syscall tests between IDs <min> and <max>.
+ * Return 0 on success, non-zero on failure.
+ */
+int run_syscall(int min, int max)
+{
+	struct stat stat_buf;
+	int proc;
+	int test;
+	int tmp;
+	int ret = 0;
+	void *p1, *p2;
+
+	/* <proc> indicates whether or not /proc is mounted */
+	proc = stat("/proc", &stat_buf) == 0;
+
+	for (test = min; test >= 0 && test <= max; test++) {
+		int llen = 0; // line length
+
+		/* avoid leaving empty lines below, this will insert holes into
+		 * test numbers.
+		 */
+		switch (test + __LINE__ + 1) {
+		CASE_TEST(getpid);            EXPECT_SYSNE(1, getpid(), -1); break;
+		CASE_TEST(getppid);           EXPECT_SYSNE(1, getppid(), -1); break;
+#ifdef NOLIBC
+		CASE_TEST(gettid);            EXPECT_SYSNE(1, gettid(), -1); break;
+#endif
+		CASE_TEST(getpgid_self);      EXPECT_SYSNE(1, getpgid(0), -1); break;
+		CASE_TEST(getpgid_bad);       EXPECT_SYSER(1, getpgid(-1), -1, ESRCH); break;
+		CASE_TEST(kill_0);            EXPECT_SYSZR(1, kill(getpid(), 0)); break;
+		CASE_TEST(kill_CONT);         EXPECT_SYSZR(1, kill(getpid(), 0)); break;
+		CASE_TEST(kill_BADPID);       EXPECT_SYSER(1, kill(INT_MAX, 0), -1, ESRCH); break;
+		CASE_TEST(sbrk);              if ((p1 = p2 = sbrk(4096)) != (void *)-1) p2 = sbrk(-4096); EXPECT_SYSZR(1, (p2 == (void *)-1) || p2 == p1); break;
+		CASE_TEST(brk);               EXPECT_SYSZR(1, brk(sbrk(0))); break;
+		CASE_TEST(chdir_root);        EXPECT_SYSZR(1, chdir("/")); break;
+		CASE_TEST(chdir_dot);         EXPECT_SYSZR(1, chdir(".")); break;
+		CASE_TEST(chdir_blah);        EXPECT_SYSER(1, chdir("/blah"), -1, ENOENT); break;
+		CASE_TEST(chmod_net);         EXPECT_SYSZR(proc, chmod("/proc/self/net", 0555)); break;
+		CASE_TEST(chmod_self);        EXPECT_SYSER(proc, chmod("/proc/self", 0555), -1, EPERM); break;
+		CASE_TEST(chown_self);        EXPECT_SYSER(proc, chown("/proc/self", 0, 0), -1, EPERM); break;
+		CASE_TEST(chroot_root);       EXPECT_SYSZR(1, chroot("/")); break;
+		CASE_TEST(chroot_blah);       EXPECT_SYSER(1, chroot("/proc/self/blah"), -1, ENOENT); break;
+		CASE_TEST(chroot_exe);        EXPECT_SYSER(proc, chroot("/proc/self/exe"), -1, ENOTDIR); break;
+		CASE_TEST(close_m1);          EXPECT_SYSER(1, close(-1), -1, EBADF); break;
+		CASE_TEST(close_dup);         EXPECT_SYSZR(1, close(dup(0))); break;
+		CASE_TEST(dup_0);             tmp = dup(0);  EXPECT_SYSNE(1, tmp, -1); close(tmp); break;
+		CASE_TEST(dup_m1);            tmp = dup(-1); EXPECT_SYSER(1, tmp, -1, EBADF); if (tmp != -1) close(tmp); break;
+		CASE_TEST(dup2_0);            tmp = dup2(0, 100);  EXPECT_SYSNE(1, tmp, -1); close(tmp); break;
+		CASE_TEST(dup2_m1);           tmp = dup2(-1, 100); EXPECT_SYSER(1, tmp, -1, EBADF); if (tmp != -1) close(tmp); break;
+		CASE_TEST(dup3_0);            tmp = dup3(0, 100, 0);  EXPECT_SYSNE(1, tmp, -1); close(tmp); break;
+		CASE_TEST(dup3_m1);           tmp = dup3(-1, 100, 0); EXPECT_SYSER(1, tmp, -1, EBADF); if (tmp != -1) close(tmp); break;
+		CASE_TEST(execve_root);       EXPECT_SYSER(1, execve("/", (char*[]){ [0] = "/", [1] = NULL }, NULL), -1, EACCES); break;
+		CASE_TEST(getdents64_root);   EXPECT_SYSNE(1, test_getdents64("/"), -1); break;
+		CASE_TEST(getdents64_null);   EXPECT_SYSER(1, test_getdents64("/dev/null"), -1, ENOTDIR); break;
+		CASE_TEST(gettimeofday_null); EXPECT_SYSZR(1, gettimeofday(NULL, NULL)); break;
+#ifdef NOLIBC
+		CASE_TEST(gettimeofday_bad1); EXPECT_SYSER(1, gettimeofday((void *)1, NULL), -1, EFAULT); break;
+		CASE_TEST(gettimeofday_bad2); EXPECT_SYSER(1, gettimeofday(NULL, (void *)1), -1, EFAULT); break;
+		CASE_TEST(gettimeofday_bad2); EXPECT_SYSER(1, gettimeofday(NULL, (void *)1), -1, EFAULT); break;
+#endif
+		CASE_TEST(ioctl_tiocinq);     EXPECT_SYSZR(1, ioctl(0, TIOCINQ, &tmp)); break;
+		CASE_TEST(ioctl_tiocinq);     EXPECT_SYSZR(1, ioctl(0, TIOCINQ, &tmp)); break;
+		CASE_TEST(link_root1);        EXPECT_SYSER(1, link("/", "/"), -1, EEXIST); break;
+		CASE_TEST(link_blah);         EXPECT_SYSER(1, link("/proc/self/blah", "/blah"), -1, ENOENT); break;
+		CASE_TEST(link_dir);          EXPECT_SYSER(1, link("/", "/blah"), -1, EPERM); break;
+		CASE_TEST(link_cross);        EXPECT_SYSER(proc, link("/proc/self/net", "/blah"), -1, EXDEV); break;
+		CASE_TEST(lseek_m1);          EXPECT_SYSER(1, lseek(-1, 0, SEEK_SET), -1, EBADF); break;
+		CASE_TEST(lseek_0);           EXPECT_SYSER(1, lseek(0, 0, SEEK_SET), -1, ESPIPE); break;
+		CASE_TEST(mkdir_root);        EXPECT_SYSER(1, mkdir("/", 0755), -1, EEXIST); break;
+		CASE_TEST(open_tty);          EXPECT_SYSNE(1, tmp = open("/dev/null", 0), -1); if (tmp != -1) close(tmp); break;
+		CASE_TEST(open_blah);         EXPECT_SYSER(1, tmp = open("/proc/self/blah", 0), -1, ENOENT); if (tmp != -1) close(tmp); break;
+		CASE_TEST(poll_null);         EXPECT_SYSZR(1, poll(NULL, 0, 0)); break;
+		CASE_TEST(poll_stdout);       EXPECT_SYSNE(1, ({ struct pollfd fds = { 1, POLLOUT, 0}; poll(&fds, 1, 0); }), -1); break;
+		CASE_TEST(poll_fault);        EXPECT_SYSER(1, poll((void *)1, 1, 0), -1, EFAULT); break;
+		CASE_TEST(read_badf);         EXPECT_SYSER(1, read(-1, &tmp, 1), -1, EBADF); break;
+		CASE_TEST(sched_yield);       EXPECT_SYSZR(1, sched_yield()); break;
+		CASE_TEST(select_null);       EXPECT_SYSZR(1, ({ struct timeval tv = { 0 }; select(0, NULL, NULL, NULL, &tv); })); break;
+		CASE_TEST(select_stdout);     EXPECT_SYSNE(1, ({ fd_set fds; FD_ZERO(&fds); FD_SET(1, &fds); select(2, NULL, &fds, NULL, NULL); }), -1); break;
+		CASE_TEST(select_fault);      EXPECT_SYSER(1, select(1, (void *)1, NULL, NULL, 0), -1, EFAULT); break;
+		CASE_TEST(stat_blah);         EXPECT_SYSER(1, stat("/proc/self/blah", &stat_buf), -1, ENOENT); break;
+		CASE_TEST(stat_fault);        EXPECT_SYSER(1, stat(NULL, &stat_buf), -1, EFAULT); break;
+		CASE_TEST(symlink_root);      EXPECT_SYSER(1, symlink("/", "/"), -1, EEXIST); break;
+		CASE_TEST(unlink_root);       EXPECT_SYSER(1, unlink("/"), -1, EISDIR); break;
+		CASE_TEST(unlink_blah);       EXPECT_SYSER(1, unlink("/proc/self/blah"), -1, ENOENT); break;
+		CASE_TEST(wait_child);        EXPECT_SYSER(1, wait(&tmp), -1, ECHILD); break;
+		CASE_TEST(waitpid_min);       EXPECT_SYSER(1, waitpid(INT_MIN, &tmp, WNOHANG), -1, ESRCH); break;
+		CASE_TEST(waitpid_child);     EXPECT_SYSER(1, waitpid(getpid(), &tmp, WNOHANG), -1, ECHILD); break;
+		CASE_TEST(write_badf);        EXPECT_SYSER(1, write(-1, &tmp, 1), -1, EBADF); break;
+		CASE_TEST(write_zero);        EXPECT_SYSZR(1, write(1, &tmp, 0)); break;
+		case __LINE__:
+			return ret; /* must be last */
+		/* note: do not set any defaults so as to permit holes above */
+		}
+	}
+	return ret;
+}
+
+int run_stdlib(int min, int max)
+{
+	int test;
+	int tmp;
+	int ret = 0;
+	void *p1, *p2;
+
+	for (test = min; test >= 0 && test <= max; test++) {
+		int llen = 0; // line length
+
+		/* avoid leaving empty lines below, this will insert holes into
+		 * test numbers.
+		 */
+		switch (test + __LINE__ + 1) {
+		CASE_TEST(getenv_TERM);        EXPECT_STRNZ(1, getenv("TERM")); break;
+		CASE_TEST(getenv_blah);        EXPECT_STRZR(1, getenv("blah")); break;
+		CASE_TEST(setcmp_blah_blah);   EXPECT_EQ(1, strcmp("blah", "blah"), 0); break;
+		CASE_TEST(setcmp_blah_blah2);  EXPECT_NE(1, strcmp("blah", "blah2"), 0); break;
+		CASE_TEST(setncmp_blah_blah);  EXPECT_EQ(1, strncmp("blah", "blah", 10), 0); break;
+		CASE_TEST(setncmp_blah_blah4); EXPECT_EQ(1, strncmp("blah", "blah4", 4), 0); break;
+		CASE_TEST(setncmp_blah_blah5); EXPECT_NE(1, strncmp("blah", "blah5", 5), 0); break;
+		CASE_TEST(setncmp_blah_blah6); EXPECT_NE(1, strncmp("blah", "blah6", 6), 0); break;
+		CASE_TEST(strchr_foobar_o);    EXPECT_STREQ(1, strchr("foobar", 'o'), "oobar"); break;
+		CASE_TEST(strchr_foobar_z);    EXPECT_STRZR(1, strchr("foobar", 'z')); break;
+		CASE_TEST(strrchr_foobar_o);   EXPECT_STREQ(1, strrchr("foobar", 'o'), "obar"); break;
+		CASE_TEST(strrchr_foobar_z);   EXPECT_STRZR(1, strrchr("foobar", 'z')); break;
+		case __LINE__:
+			return ret; /* must be last */
+		/* note: do not set any defaults so as to permit holes above */
+		}
+	}
+	return ret;
+}
+
+/* prepare what needs to be prepared for pid 1 (stdio, /dev, /proc, etc) */
+int prepare(void)
+{
+	struct stat stat_buf;
+
+	/* It's possible that /dev doesn't even exist or was not mounted, so
+	 * we'll try to create it, mount it, or create minimal entries into it.
+	 * We want at least /dev/null and /dev/console.
+	 */
+	if (stat("/dev/.", &stat_buf) == 0 || mkdir("/dev", 0755) == 0) {
+		if (stat("/dev/console", &stat_buf) != 0 ||
+		    stat("/dev/null", &stat_buf) != 0) {
+			/* try devtmpfs first, otherwise fall back to manual creation */
+			if (mount("/dev", "/dev", "devtmpfs", 0, 0) != 0) {
+				mknod("/dev/console", 0600 | S_IFCHR, makedev(5, 1));
+				mknod("/dev/null",    0666 | S_IFCHR, makedev(1, 3));
+			}
+		}
+	}
+
+	/* If no /dev/console was found before calling init, stdio is closed so
+	 * we need to reopen it from /dev/console. If it failed above, it will
+	 * still fail here and we cannot emit a message anyway.
+	 */
+	if (close(dup(1)) == -1) {
+		int fd = open("/dev/console", O_RDWR);
+
+		if (fd >= 0) {
+			if (fd != 0)
+				dup2(fd, 0);
+			if (fd != 1)
+				dup2(fd, 1);
+			if (fd != 2)
+				dup2(fd, 2);
+			if (fd > 2)
+				close(fd);
+			puts("\nSuccessfully reopened /dev/console.");
+		}
+	}
+
+	/* try to mount /proc if not mounted. Silently fail otherwise */
+	if (stat("/proc/.", &stat_buf) == 0 || mkdir("/proc", 0755) == 0) {
+		if (stat("/proc/self", &stat_buf) != 0)
+			mount("/proc", "/proc", "proc", 0, 0);
+	}
+
+	return 0;
+}
+
+/* This is the definition of known test names, with their functions */
+static struct test test_names[] = {
+	/* add new tests here */
+	{ .name = "syscall",   .func = run_syscall  },
+	{ .name = "stdlib",    .func = run_stdlib   },
+	{ 0 }
+};
+
+int main(int argc, char **argv, char **envp)
+{
+	int min = 0;
+	int max = __INT_MAX__;
+	int ret = 0;
+	int err;
+	int idx;
+	char *test;
+
+	environ = envp;
+
+	/* when called as init, it's possible that no console was opened, for
+	 * example if no /dev file system was provided. We'll check that fd#1
+	 * was opened, and if not we'll attempt to create and open /dev/console
+	 * and /dev/null that we'll use for later tests.
+	 */
+	if (getpid() == 1)
+		prepare();
+
+	/* the definition of a series of tests comes from either argv[1] or the
+	 * "NOLIBC_TEST" environment variable. It's made of a comma-delimited
+	 * series of test names and optional ranges:
+	 *    syscall:5-15[:.*],stdlib:8-10
+	 */
+	test = argv[1];
+	if (!test)
+		test = getenv("NOLIBC_TEST");
+
+	if (test) {
+		char *comma, *colon, *dash, *value;
+
+		do {
+			comma = strchr(test, ',');
+			if (comma)
+				*(comma++) = '\0';
+
+			colon = strchr(test, ':');
+			if (colon)
+				*(colon++) = '\0';
+
+			for (idx = 0; test_names[idx].name; idx++) {
+				if (strcmp(test, test_names[idx].name) == 0)
+					break;
+			}
+
+			if (test_names[idx].name) {
+				/* The test was named, it will be called at least
+				 * once. We may have an optional range at <colon>
+				 * here, which defaults to the full range.
+				 */
+				do {
+					min = 0; max = __INT_MAX__;
+					value = colon;
+					if (value && *value) {
+						colon = strchr(value, ':');
+						if (colon)
+							*(colon++) = '\0';
+
+						dash = strchr(value, '-');
+						if (dash)
+							*(dash++) = '\0';
+
+						/* support :val: :min-max: :min-: :-max: */
+						if (*value)
+							min = atoi(value);
+						if (!dash)
+							max = min;
+						else if (*dash)
+							max = atoi(dash);
+
+						value = colon;
+					}
+
+					/* now's time to call the test */
+					printf("Running test '%s'\n", test_names[idx].name);
+					err = test_names[idx].func(min, max);
+					ret += err;
+					printf("Errors during this test: %d\n\n", err);
+				} while (colon && *colon);
+			} else
+				printf("Ignoring unknown test name '%s'\n", test);
+
+			test = comma;
+		} while (test && *test);
+	} else {
+		/* no test mentioned, run everything */
+		for (idx = 0; test_names[idx].name; idx++) {
+			printf("Running test '%s'\n", test_names[idx].name);
+			err = test_names[idx].func(min, max);
+			ret += err;
+			printf("Errors during this test: %d\n\n", err);
+		}
+	}
+
+	printf("Total number of errors: %d\n", ret);
+
+	if (getpid() == 1) {
+		/* we're running as init, there's no other process on the
+		 * system, thus likely started from a VM for a quick check.
+		 * Exiting will provoke a kernel panic that may be reported
+		 * as an error by Qemu or the hypervisor, while stopping
+		 * cleanly will often be reported as a success. This allows
+		 * to use the output of this program for bisecting kernels.
+		 */
+		printf("Leaving init with final status: %d\n", !!ret);
+		if (ret == 0)
+			reboot(LINUX_REBOOT_CMD_POWER_OFF);
+#if defined(__x86_64__)
+		/* QEMU started with "-device isa-debug-exit -no-reboot" will
+		 * exit with status code 2N+1 when N is written to 0x501. We
+		 * hard-code the syscall here as it's arch-dependent.
+		 */
+#if defined(_NOLIBC_SYS_H)
+		else if (my_syscall3(__NR_ioperm, 0x501, 1, 1) == 0)
+#else
+		else if (ioperm(0x501, 1, 1) == 0)
+#endif
+			asm volatile ("outb %%al, %%dx" :: "d"(0x501), "a"(0));
+		/* if it does nothing, fall back to the regular panic */
+#endif
+	}
+
+	printf("Exiting with status %d\n", !!ret);
+	return !!ret;
+}
diff --git a/tools/testing/selftests/tc-testing/config b/tools/testing/selftests/tc-testing/config
index a3239d5e40c7..4638c63a339f 100644
--- a/tools/testing/selftests/tc-testing/config
+++ b/tools/testing/selftests/tc-testing/config
@@ -12,8 +12,35 @@ CONFIG_NET_SCHED=y
 #
 # Queueing/Scheduling
 #
-CONFIG_NET_SCH_PRIO=m
+CONFIG_NET_SCH_ATM=m
+CONFIG_NET_SCH_CAKE=m
+CONFIG_NET_SCH_CBQ=m
+CONFIG_NET_SCH_CBS=m
+CONFIG_NET_SCH_CHOKE=m
+CONFIG_NET_SCH_CODEL=m
+CONFIG_NET_SCH_DRR=m
+CONFIG_NET_SCH_DSMARK=m
+CONFIG_NET_SCH_ETF=m
+CONFIG_NET_SCH_FQ=m
+CONFIG_NET_SCH_FQ_CODEL=m
+CONFIG_NET_SCH_GRED=m
+CONFIG_NET_SCH_HFSC=m
+CONFIG_NET_SCH_HHF=m
+CONFIG_NET_SCH_HTB=m
 CONFIG_NET_SCH_INGRESS=m
+CONFIG_NET_SCH_MQPRIO=m
+CONFIG_NET_SCH_MULTIQ=m
+CONFIG_NET_SCH_NETEM=m
+CONFIG_NET_SCH_PIE=m
+CONFIG_NET_SCH_PLUG=m
+CONFIG_NET_SCH_PRIO=m
+CONFIG_NET_SCH_QFQ=m
+CONFIG_NET_SCH_SFB=m
+CONFIG_NET_SCH_SFQ=m
+CONFIG_NET_SCH_SKBPRIO=m
+CONFIG_NET_SCH_TAPRIO=m
+CONFIG_NET_SCH_TBF=m
+CONFIG_NET_SCH_TEQL=m
 
 #
 # Classification
@@ -23,6 +50,15 @@ CONFIG_NET_CLS_FW=m
 CONFIG_NET_CLS_U32=m
 CONFIG_CLS_U32_PERF=y
 CONFIG_CLS_U32_MARK=y
+CONFIG_NET_CLS_BASIC=m
+CONFIG_NET_CLS_BPF=m
+CONFIG_NET_CLS_CGROUP=m
+CONFIG_NET_CLS_FLOW=m
+CONFIG_NET_CLS_FLOWER=m
+CONFIG_NET_CLS_MATCHALL=m
+CONFIG_NET_CLS_ROUTE4=m
+CONFIG_NET_CLS_RSVP=m
+CONFIG_NET_CLS_TCINDEX=m
 CONFIG_NET_EMATCH=y
 CONFIG_NET_EMATCH_STACK=32
 CONFIG_NET_EMATCH_CMP=m
@@ -54,6 +90,7 @@ CONFIG_NET_ACT_IFE=m
 CONFIG_NET_ACT_TUNNEL_KEY=m
 CONFIG_NET_ACT_CT=m
 CONFIG_NET_ACT_MPLS=m
+CONFIG_NET_ACT_GATE=m
 CONFIG_NET_IFE_SKBMARK=m
 CONFIG_NET_IFE_SKBPRIO=m
 CONFIG_NET_IFE_SKBTCINDEX=m
@@ -67,3 +104,4 @@ CONFIG_NETDEVSIM=m
 ## Network testing
 #
 CONFIG_CAN=m
+CONFIG_ATM=y
diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/connmark.json b/tools/testing/selftests/tc-testing/tc-tests/actions/connmark.json
index cadde8f41fcd..0de2f79ea329 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/actions/connmark.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/actions/connmark.json
@@ -312,5 +312,55 @@
         "teardown": [
             "$TC actions flush action connmark"
         ]
+    },
+    {
+        "id": "6571",
+        "name": "Delete connmark action with valid index",
+        "category": [
+            "actions",
+            "connmark"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action connmark",
+                0,
+                1,
+                255
+            ],
+	    "$TC actions add action connmark pass index 20"
+        ],
+        "cmdUnderTest": "$TC actions del action connmark index 20",
+        "expExitCode": "0",
+        "verifyCmd": "$TC actions get action connmark index 20",
+        "matchPattern": "action order [0-9]+: connmark zone 0 pass.*index 20 ref",
+        "matchCount": "0",
+        "teardown": [
+            "$TC actions flush action connmark"
+        ]
+    },
+    {
+        "id": "3426",
+        "name": "Delete connmark action with invalid index",
+        "category": [
+            "actions",
+            "connmark"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action connmark",
+                0,
+                1,
+                255
+            ],
+            "$TC actions add action connmark pass index 20"
+        ],
+        "cmdUnderTest": "$TC actions del action connmark index 1",
+        "expExitCode": "255",
+        "verifyCmd": "$TC actions get action connmark index 20",
+        "matchPattern": "action order [0-9]+: connmark zone 0 pass.*index 20 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC actions flush action connmark"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/ctinfo.json b/tools/testing/selftests/tc-testing/tc-tests/actions/ctinfo.json
new file mode 100644
index 000000000000..d9710c067eb7
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/actions/ctinfo.json
@@ -0,0 +1,316 @@
+[
+    {
+        "id": "c826",
+        "name": "Add ctinfo action with default setting",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC action flush action ctinfo",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action ctinfo index 10",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action get action ctinfo index 10",
+        "matchPattern": "action order [0-9]*: ctinfo zone 0 pipe.*index 10 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action ctinfo"
+        ]
+    },
+    {
+        "id": "0286",
+        "name": "Add ctinfo action with dscp",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action ctinfo",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action ctinfo dscp 0xfc000000 0x01000000 index 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action ls action ctinfo",
+        "matchPattern": "action order [0-9]*: ctinfo zone 0 pipe.*index 100 ref.*dscp 0xfc000000 0x01000000",
+        "matchCount": "1",
+        "teardown": [
+            "$TC actions flush action ctinfo"
+        ]
+    },
+    {
+        "id": "4938",
+        "name": "Add ctinfo action with valid cpmark and zone",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC action flush action ctinfo",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action ctinfo cpmark 0x01000000 zone 1 index 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action get action ctinfo index 1",
+        "matchPattern": "action order [0-9]*: ctinfo zone 1 pipe.*index 1 ref.*cpmark 0x01000000",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action ctinfo"
+        ]
+    },
+    {
+        "id": "7593",
+        "name": "Add ctinfo action with drop control",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC action flush action ctinfo",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action ctinfo drop index 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action ls action ctinfo",
+        "matchPattern": "action order [0-9]*: ctinfo zone 0 drop.*index 1000 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action ctinfo"
+        ]
+    },
+    {
+        "id": "2961",
+        "name": "Replace ctinfo action zone and action control",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action ctinfo",
+                0,
+                1,
+                255
+            ],
+            [
+                "$TC action add action ctinfo zone 1 drop index 1",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action replace action ctinfo zone 200 pass index 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action get action ctinfo index 1",
+        "matchPattern": "action order [0-9]*: ctinfo zone 200 pass.*index 1 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action ctinfo"
+        ]
+    },
+    {
+        "id": "e567",
+        "name": "Delete ctinfo action with valid index",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action ctinfo",
+                0,
+                1,
+                255
+            ],
+            [
+                "$TC action add action ctinfo zone 200 pass index 1",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action delete action ctinfo index 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action get action ctinfo index 1",
+        "matchPattern": "action order [0-9]*: ctinfo zone 200 pass.*index 1 ref",
+        "matchCount": "0",
+        "teardown": [
+            "$TC action flush action ctinfo"
+        ]
+    },
+    {
+        "id": "6a91",
+        "name": "Delete ctinfo action with invalid index",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action ctinfo",
+                0,
+                1,
+                255
+            ],
+            [
+                "$TC action add action ctinfo zone 200 pass index 1",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action delete action ctinfo index 333",
+        "expExitCode": "255",
+        "verifyCmd": "$TC action get action ctinfo index 1",
+        "matchPattern": "action order [0-9]*: ctinfo zone 200 pass.*index 1 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action ctinfo"
+        ]
+    },
+    {
+        "id": "5232",
+        "name": "List ctinfo actions",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC action flush action ctinfo",
+                0,
+                1,
+                255
+            ],
+            "$TC action add action ctinfo zone 20 pass index 101",
+            "$TC action add action ctinfo cpmark 0x02000000 drop index 102",
+            "$TC action add action ctinfo continue index 103"
+        ],
+        "cmdUnderTest": "$TC action list action ctinfo",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action list action ctinfo",
+        "matchPattern": "action order [0-9]*: ctinfo",
+        "matchCount": "3",
+        "teardown": [
+            "$TC actions flush action ctinfo"
+        ]
+    },
+    {
+        "id": "7702",
+        "name": "Flush ctinfo actions",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action ctinfo",
+                0,
+                1,
+                255
+            ],
+	    "$TC action add action ctinfo zone 20 pass index 101",
+            "$TC action add action ctinfo cpmark 0x02000000 drop index 102",
+            "$TC action add action ctinfo continue index 103"
+        ],
+        "cmdUnderTest": "$TC action flush action ctinfo",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action list action ctinfo",
+        "matchPattern": "action order [0-9]*: ctinfo",
+        "matchCount": "0",
+        "teardown": [
+            "$TC actions flush action ctinfo"
+        ]
+    },
+    {
+        "id": "3201",
+        "name": "Add ctinfo action with duplicate index",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action ctinfo",
+                0,
+                1,
+                255
+            ],
+            "$TC action add action ctinfo zone 20 pass index 101"
+        ],
+        "cmdUnderTest": "$TC action add action ctinfo cpmark 0x02000000 drop index 101",
+        "expExitCode": "255",
+        "verifyCmd": "$TC action get action ctinfo index 101",
+        "matchPattern": "action order [0-9]*: ctinfo zone 20 pass.*index 101",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action ctinfo"
+        ]
+    },
+    {
+        "id": "8295",
+        "name": "Add ctinfo action with invalid index",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action ctinfo",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action ctinfo zone 20 index 4294967296",
+        "expExitCode": "255",
+        "verifyCmd": "$TC action ls action ctinfo",
+        "matchPattern": "action order [0-9]*: ctinfo",
+        "matchCount": "0",
+        "teardown": [
+            "$TC action flush action ctinfo"
+        ]
+    },
+    {
+        "id": "3964",
+        "name": "Replace ctinfo action with invalid goto_chain control",
+        "category": [
+            "actions",
+            "ctinfo"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action ctinfo",
+                0,
+                1,
+                255
+            ],
+            "$TC action add action ctinfo pass index 90"
+        ],
+        "cmdUnderTest": "$TC action replace action ctinfo goto chain 42 index 90",
+        "expExitCode": "255",
+        "verifyCmd": "$TC action list action ctinfo",
+        "matchPattern": "action order [0-9]*: ctinfo.*pass.*index 90",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action ctinfo"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/gate.json b/tools/testing/selftests/tc-testing/tc-tests/actions/gate.json
new file mode 100644
index 000000000000..e16a4963fdd2
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/actions/gate.json
@@ -0,0 +1,315 @@
+[
+    {
+        "id": "5153",
+        "name": "Add gate action with priority and sched-entry",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC action flush action gate",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action gate priority 1 sched-entry close 100000000ns index 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action get action gate index 100",
+        "matchPattern": "action order [0-9]*: .*priority 1.*index 100 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action gate"
+        ]
+    },
+    {
+        "id": "7189",
+        "name": "Add gate action with base-time",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action gate",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action gate base-time 200000000000ns sched-entry close 100000000ns index 10",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action ls action gate",
+        "matchPattern": "action order [0-9]*: .*base-time 200s.*index 10 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC actions flush action gate"
+        ]
+    },
+    {
+        "id": "a721",
+        "name": "Add gate action with cycle-time",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC action flush action gate",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action gate cycle-time 200000000000ns sched-entry close 100000000ns index 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action ls action gate",
+        "matchPattern": "action order [0-9]*: .*cycle-time 200s.*index 1000 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action gate"
+        ]
+    },
+    {
+        "id": "c029",
+        "name": "Add gate action with cycle-time-ext",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC action flush action gate",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action gate cycle-time-ext 20000000000ns sched-entry close 100000000ns index 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action get action gate index 1000",
+        "matchPattern": "action order [0-9]*: .*cycle-time-ext 20s.*index 1000 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action gate"
+        ]
+    },
+    {
+        "id": "3719",
+        "name": "Replace gate base-time action",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action gate",
+                0,
+                1,
+                255
+            ],
+            [
+                "$TC action add action gate base-time 200000000000ns sched-entry open 200000000ns -1 8000000b index 20",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action replace action gate base-time 400000000000ns index 20",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action get action gate index 20",
+        "matchPattern": "action order [0-9]*: .*base-time 400s.*index 20 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action gate"
+        ]
+    },
+    {
+        "id": "d821",
+        "name": "Delete gate action with valid index",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action gate",
+                0,
+                1,
+                255
+            ],
+            [
+                "$TC action add action gate base-time 200000000000ns sched-entry open 200000000ns -1 8000000b index 302",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action delete action gate index 302",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action get action bpf index 302",
+        "matchPattern": "action order [0-9]*: .*base-time 200s.*index 302 ref",
+        "matchCount": "0",
+        "teardown": [
+            "$TC action flush action gate"
+        ]
+    },
+    {
+        "id": "3128",
+        "name": "Delete gate action with invalid index",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action gate",
+                0,
+                1,
+                255
+            ],
+            [
+                "$TC action add action gate base-time 600000000000ns sched-entry open 200000000ns -1 8000000b index 999",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action delete action gate index 333",
+        "expExitCode": "255",
+        "verifyCmd": "$TC action get action gate index 999",
+        "matchPattern": "action order [0-9]*: .*base-time 600s.*index 999 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action gate"
+        ]
+    },
+    {
+        "id": "7837",
+        "name": "List gate actions",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC action flush action gate",
+                0,
+                1,
+                255
+            ],
+            "$TC action add action gate base-time 600000000000ns sched-entry open 200000000ns -1 8000000b index 101",
+            "$TC action add action gate cycle-time 600000000000ns sched-entry open 600000000ns -1 8000000b index 102",
+            "$TC action add action gate cycle-time-ext 400000000000ns sched-entry close 100000000ns index 103"
+        ],
+        "cmdUnderTest": "$TC action list action gate",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action list action gate",
+        "matchPattern": "action order [0-9]*:",
+        "matchCount": "3",
+        "teardown": [
+            "$TC actions flush action gate"
+        ]
+    },
+    {
+        "id": "9273",
+        "name": "Flush gate actions",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action gate",
+                0,
+                1,
+                255
+            ],
+            "$TC action add action gate base-time 600000000000ns sched-entry open 200000000ns -1 8000000b index 101",
+            "$TC action add action gate cycle-time 600000000000ns sched-entry open 600000000ns -1 8000000b index 102",
+            "$TC action add action gate cycle-time-ext 400000000000ns sched-entry close 100000000ns index 103"
+	],
+        "cmdUnderTest": "$TC action flush action gate",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action list action gate",
+        "matchPattern": "action order [0-9]*: .*priority",
+        "matchCount": "0",
+        "teardown": [
+            "$TC actions flush action gate"
+        ]
+    },
+    {
+        "id": "c829",
+        "name": "Add gate action with duplicate index",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action gate",
+                0,
+                1,
+                255
+            ],
+            "$TC action add action gate cycle-time 600000000000ns sched-entry open 600000000ns -1 8000000b index 4294967295"
+        ],
+        "cmdUnderTest": "$TC action add action gate cycle-time 600000000000ns sched-entry open 600000000ns -1 8000000b index 4294967295",
+        "expExitCode": "255",
+        "verifyCmd": "$TC action get action gate index 4294967295",
+        "matchPattern": "action order [0-9]*: .*index 4294967295",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action gate"
+        ]
+    },
+    {
+        "id": "3043",
+        "name": "Add gate action with invalid index",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action gate",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action gate cycle-time-ext 400000000000ns sched-entry close 100000000ns index 4294967296",
+        "expExitCode": "255",
+        "verifyCmd": "$TC action ls action gate",
+        "matchPattern": "action order [0-9]*:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC action flush action gate"
+        ]
+    },
+    {
+        "id": "2930",
+        "name": "Add gate action with cookie",
+        "category": [
+            "actions",
+            "gate"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action gate",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action gate cycle-time-ext 400000000000ns sched-entry close 100000000ns index 4294 cookie d0d0d0d0d0d0d0d0",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action list action gate",
+        "matchPattern": "action order [0-9]*: .*cookie d0d0d0d0d0d0d0",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action gate"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json b/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json
index c13a68b98fc7..459bcad35810 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/actions/ife.json
@@ -1085,5 +1085,55 @@
         "teardown": [
             "$TC actions flush action ife"
         ]
+    },
+    {
+        "id": "a972",
+        "name": "Delete ife encode action with valid index",
+        "category": [
+            "actions",
+            "ife"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action ife",
+                0,
+                1,
+                255
+            ],
+	    "$TC actions add action ife encode allow mark pass index 20"
+        ],
+        "cmdUnderTest": "$TC actions del action ife index 20",
+        "expExitCode": "0",
+        "verifyCmd": "$TC actions ls action ife index 20",
+        "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*allow mark.*index 20",
+        "matchCount": "0",
+        "teardown": [
+            "$TC actions flush action ife"
+        ]
+    },
+    {
+        "id": "1272",
+        "name": "Delete ife encode action with invalid index",
+        "category": [
+            "actions",
+            "ife"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action ife",
+                0,
+                1,
+                255
+            ],
+            "$TC actions add action ife encode allow mark pass index 20"
+        ],
+        "cmdUnderTest": "$TC actions del action ife index 10",
+        "expExitCode": "255",
+        "verifyCmd": "$TC actions ls action ife index 20",
+        "matchPattern": "action order [0-9]*: ife encode action pass.*type 0[xX]ED3E.*allow mark.*index 20",
+        "matchCount": "1",
+        "teardown": [
+            "$TC actions flush action ife"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/nat.json b/tools/testing/selftests/tc-testing/tc-tests/actions/nat.json
index bc12c1ccad30..0a3c491edbc5 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/actions/nat.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/actions/nat.json
@@ -614,5 +614,55 @@
         "teardown": [
             "$TC actions flush action nat"
         ]
+    },
+    {
+        "id": "b811",
+        "name": "Delete nat action with valid index",
+        "category": [
+            "actions",
+            "nat"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action nat",
+                0,
+                1,
+                255
+            ],
+            "$TC actions add action nat ingress 1.1.1.1 2.2.2.2 drop index 20"
+        ],
+        "cmdUnderTest": "$TC actions del action nat index 20",
+        "expExitCode": "0",
+        "verifyCmd": "$TC actions ls action nat index 20",
+        "matchPattern": "action order [0-9]+:  nat ingress 1.1.1.1/32 2.2.2.2 drop.*index 20 ref",
+        "matchCount": "0",
+        "teardown": [
+            "$TC actions flush action nat"
+        ]
+    },
+    {
+        "id": "a521",
+        "name": "Delete nat action with invalid index",
+        "category": [
+            "actions",
+            "nat"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action nat",
+                0,
+                1,
+                255
+            ],
+            "$TC actions add action nat ingress 1.1.1.1 2.2.2.2 drop index 20"
+        ],
+        "cmdUnderTest": "$TC actions del action nat index 10",
+        "expExitCode": "255",
+        "verifyCmd": "$TC actions ls action nat index 20",
+        "matchPattern": "action order [0-9]+:  nat ingress 1.1.1.1/32 2.2.2.2 drop.*index 20 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC actions flush action nat"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/sample.json b/tools/testing/selftests/tc-testing/tc-tests/actions/sample.json
index ddabb160a11b..148d8bcb8606 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/actions/sample.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/actions/sample.json
@@ -633,5 +633,55 @@
         "teardown": [
             "$TC actions flush action sample"
         ]
+    },
+    {
+        "id": "3872",
+        "name": "Delete sample action with valid index",
+        "category": [
+            "actions",
+            "sample"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action sample",
+                0,
+                1,
+                255
+            ],
+	    "$TC actions add action sample rate 10 group 1 index 20"
+        ],
+        "cmdUnderTest": "$TC actions del action sample index 20",
+        "expExitCode": "0",
+        "verifyCmd": "$TC actions get action sample index 20",
+        "matchPattern": "action order [0-9]+: sample rate 1/10 group 1.*index 20 ref",
+        "matchCount": "0",
+        "teardown": [
+            "$TC actions flush action sample"
+        ]
+    },
+    {
+        "id": "a394",
+        "name": "Delete sample action with invalid index",
+        "category": [
+            "actions",
+            "sample"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action sample",
+                0,
+                1,
+                255
+            ],
+            "$TC actions add action sample rate 10 group 1 index 20"
+        ],
+        "cmdUnderTest": "$TC actions del action sample index 10",
+        "expExitCode": "255",
+        "verifyCmd": "$TC actions get action sample index 20",
+        "matchPattern": "action order [0-9]+: sample rate 1/10 group 1.*index 20 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC actions flush action sample"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json b/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json
index d06346968bcb..b40ee602918a 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/actions/tunnel_key.json
@@ -933,5 +933,55 @@
         "teardown": [
             "$TC actions flush action tunnel_key"
         ]
+    },
+    {
+        "id": "3671",
+        "name": "Delete tunnel_key set action with valid index",
+	"category": [
+            "actions",
+            "tunnel_key"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action tunnel_key",
+                0,
+                1,
+                255
+            ],
+	    "$TC actions add action tunnel_key set src_ip 1.1.1.1 dst_ip 2.2.2.2 index 1"
+        ],
+        "cmdUnderTest": "$TC actions del action tunnel_key index 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC actions list action tunnel_key",
+        "matchPattern": "action order [0-9]+: tunnel_key.*set.*src_ip 1.1.1.1.*dst_ip 2.2.2.2.*index 1",
+        "matchCount": "0",
+        "teardown": [
+            "$TC actions flush action tunnel_key"
+        ]
+    },
+    {
+        "id": "8597",
+        "name": "Delete tunnel_key set action with invalid index",
+        "category": [
+            "actions",
+            "tunnel_key"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action tunnel_key",
+                0,
+                1,
+                255
+            ],
+            "$TC actions add action tunnel_key set src_ip 1.1.1.1 dst_ip 2.2.2.2 index 1"
+        ],
+        "cmdUnderTest": "$TC actions del action tunnel_key index 10",
+        "expExitCode": "255",
+        "verifyCmd": "$TC actions list action tunnel_key",
+        "matchPattern": "action order [0-9]+: tunnel_key.*set.*src_ip 1.1.1.1.*dst_ip 2.2.2.2.*index 1",
+        "matchCount": "1",
+        "teardown": [
+            "$TC actions flush action tunnel_key"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/actions/xt.json b/tools/testing/selftests/tc-testing/tc-tests/actions/xt.json
new file mode 100644
index 000000000000..c9f002aea6d4
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/actions/xt.json
@@ -0,0 +1,219 @@
+[
+    {
+        "id": "2029",
+        "name": "Add xt action with log-prefix",
+        "category": [
+            "actions",
+            "xt"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action xt",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action xt -j LOG --log-prefix PONG index 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action ls action xt",
+        "matchPattern": "action order [0-9]*:.*target  LOG level warning prefix \"PONG\".*index 100 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC actions flush action xt"
+        ]
+    },
+    {
+        "id": "3562",
+        "name": "Replace xt action log-prefix",
+        "category": [
+            "actions",
+            "xt"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action xt",
+                0,
+                1,
+                255
+            ],
+            [
+                "$TC action add action xt -j LOG --log-prefix PONG index 1",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action replace action xt -j LOG --log-prefix WIN index 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action get action xt index 1",
+        "matchPattern": "action order [0-9]*:.*target  LOG level warning prefix \"WIN\".*index 1 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action xt"
+        ]
+    },
+    {
+        "id": "8291",
+        "name": "Delete xt action with valid index",
+        "category": [
+            "actions",
+            "xt"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action xt",
+                0,
+                1,
+                255
+            ],
+            [
+                "$TC action add action xt -j LOG --log-prefix PONG index 1000",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action delete action xt index 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action get action xt index 1000",
+        "matchPattern": "action order [0-9]*:.*target  LOG level warning prefix \"PONG\".*index 1000 ref",
+        "matchCount": "0",
+        "teardown": [
+            "$TC action flush action xt"
+        ]
+    },
+    {
+        "id": "5169",
+        "name": "Delete xt action with invalid index",
+        "category": [
+            "actions",
+            "xt"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action xt",
+                0,
+                1,
+                255
+            ],
+            [
+                "$TC action add action xt -j LOG --log-prefix PONG index 1000",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action delete action xt index 333",
+        "expExitCode": "255",
+        "verifyCmd": "$TC action get action xt index 1000",
+        "matchPattern": "action order [0-9]*:.*target  LOG level warning prefix \"PONG\".*index 1000 ref",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action xt"
+        ]
+    },
+    {
+        "id": "7284",
+        "name": "List xt actions",
+        "category": [
+            "actions",
+            "xt"
+        ],
+        "setup": [
+            [
+                "$TC action flush action xt",
+                0,
+                1,
+                255
+            ],
+            "$TC action add action xt -j LOG --log-prefix PONG index 1001",
+            "$TC action add action xt -j LOG --log-prefix WIN index 1002",
+            "$TC action add action xt -j LOG --log-prefix LOSE index 1003"
+        ],
+        "cmdUnderTest": "$TC action list action xt",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action list action xt",
+        "matchPattern": "action order [0-9]*: tablename:",
+        "matchCount": "3",
+        "teardown": [
+            "$TC actions flush action xt"
+        ]
+    },
+    {
+        "id": "5010",
+        "name": "Flush xt actions",
+        "category": [
+            "actions",
+            "xt"
+        ],
+        "setup": [
+            [
+		"$TC actions flush action xt",
+                0,
+                1,
+                255
+            ],
+            "$TC action add action xt -j LOG --log-prefix PONG index 1001",
+            "$TC action add action xt -j LOG --log-prefix WIN index 1002",
+            "$TC action add action xt -j LOG --log-prefix LOSE index 1003"
+	],
+        "cmdUnderTest": "$TC action flush action xt",
+        "expExitCode": "0",
+        "verifyCmd": "$TC action list action xt",
+        "matchPattern": "action order [0-9]*: tablename:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC actions flush action xt"
+        ]
+    },
+    {
+        "id": "8437",
+        "name": "Add xt action with duplicate index",
+        "category": [
+            "actions",
+            "xt"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action xt",
+                0,
+                1,
+                255
+            ],
+            "$TC action add action xt -j LOG --log-prefix PONG index 101"
+        ],
+        "cmdUnderTest": "$TC action add action xt -j LOG --log-prefix WIN index 101",
+        "expExitCode": "255",
+        "verifyCmd": "$TC action get action xt index 101",
+        "matchPattern": "action order [0-9]*:.*target  LOG level warning prefix \"PONG\".*index 101",
+        "matchCount": "1",
+        "teardown": [
+            "$TC action flush action xt"
+        ]
+    },
+    {
+        "id": "2837",
+        "name": "Add xt action with invalid index",
+        "category": [
+            "actions",
+            "xt"
+        ],
+        "setup": [
+            [
+                "$TC actions flush action xt",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC action add action xt -j LOG --log-prefix WIN index 4294967296",
+        "expExitCode": "255",
+        "verifyCmd": "$TC action ls action xt",
+        "matchPattern": "action order [0-9]*:*target  LOG level warning prefix \"WIN\"",
+        "matchCount": "0",
+        "teardown": [
+            "$TC action flush action xt"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/basic.json b/tools/testing/selftests/tc-testing/tc-tests/filters/basic.json
index e788c114a484..d1278de8ebc3 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/filters/basic.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/filters/basic.json
@@ -1274,5 +1274,52 @@
         "teardown": [
             "$TC qdisc del dev $DEV1 ingress"
         ]
+    },
+    {
+        "id": "0811",
+        "name": "Add multiple basic filter with cmp ematch u8/link layer and default action and dump them",
+        "category": [
+            "filter",
+            "basic"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 basic match 'cmp(u8 at 0 layer link mask 0xff gt 10)' classid 1:1"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 2 protocol ip prio 1 basic match 'cmp(u8 at 0 layer link mask 0xff gt 10)' classid 1:1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 basic",
+        "matchCount": "3",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "5129",
+        "name": "List basic filters",
+        "category": [
+            "filter",
+            "basic"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 basic match 'cmp(u8 at 0 layer link mask 0xff gt 10)' classid 1:1",
+            "$TC filter add dev $DEV1 parent ffff: handle 2 protocol ip prio 1 basic match 'cmp(u8 at 0 layer link mask 0xff gt 10)' classid 1:1"
+        ],
+        "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "cmp\\(u8 at 0 layer 0 mask 0xff gt 10\\)",
+        "matchCount": "2",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/bpf.json b/tools/testing/selftests/tc-testing/tc-tests/filters/bpf.json
new file mode 100644
index 000000000000..1f0cae474db2
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/filters/bpf.json
@@ -0,0 +1,171 @@
+[
+    {
+        "id": "23c3",
+        "name": "Add cBPF filter with valid bytecode",
+        "category": [
+            "filter",
+            "bpf-filter"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf",
+        "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1.*bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1563",
+        "name": "Add cBPF filter with invalid bytecode",
+        "category": [
+            "filter",
+            "bpf-filter"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,31 0 1 2048,6 0 0 262144,6 0 0 0'",
+        "expExitCode": "2",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf",
+        "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1.*bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2334",
+        "name": "Add eBPF filter with valid object-file",
+        "category": [
+            "filter",
+            "bpf-filter"
+        ],
+        "plugins": {
+            "requires": "buildebpfPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf object-file $EBPFDIR/action.o section action-ok",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf",
+        "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1 action.o:\\[action-ok\\].*tag [0-9a-f]{16}( jited)?",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2373",
+        "name": "Add eBPF filter with invalid object-file",
+        "category": [
+            "filter",
+            "bpf-filter"
+        ],
+        "plugins": {
+            "requires": "buildebpfPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf object-file $EBPFDIR/action.o section action-ko",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf",
+        "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1 action.o:\\[action-ko\\].*tag [0-9a-f]{16}( jited)?",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4423",
+        "name": "Replace cBPF bytecode",
+        "category": [
+            "filter",
+            "bpf-filter"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            [
+                "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC filter replace dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2054,6 0 0 262144,6 0 0 0'",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf",
+        "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1.*bytecode '4,40 0 0 12,21 0 1 2054,6 0 0 262144,6 0 0 0'",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "5122",
+        "name": "Delete cBPF filter",
+        "category": [
+            "filter",
+            "bpf-filter"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            [
+                "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'",
+                0,
+                1,
+                255
+            ]
+        ],
+        "cmdUnderTest": "$TC filter del dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf",
+        "matchPattern": "filter parent ffff: protocol ip pref 100 bpf chain [0-9]+ handle 0x1.*bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "e0a9",
+        "name": "List cBPF filters",
+        "category": [
+            "filter",
+            "bpf-filter"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2048,6 0 0 262144,6 0 0 0'",
+            "$TC filter add dev $DEV1 parent ffff: handle 2 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 2054,6 0 0 262144,6 0 0 0'",
+            "$TC filter add dev $DEV1 parent ffff: handle 100 protocol ip prio 100 bpf bytecode '4,40 0 0 12,21 0 1 33024,6 0 0 262144,6 0 0 0'"
+        ],
+        "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "filter protocol ip pref 100 bpf chain [0-9]+ handle",
+        "matchCount": "3",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/cgroup.json b/tools/testing/selftests/tc-testing/tc-tests/filters/cgroup.json
new file mode 100644
index 000000000000..03723cf84379
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/filters/cgroup.json
@@ -0,0 +1,1236 @@
+[
+    {
+        "id": "6273",
+        "name": "Add cgroup filter with cmp ematch u8/link layer and drop action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff gt 10)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ cgroup chain [0-9]+.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff gt 10\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4721",
+        "name": "Add cgroup filter with cmp ematch u8/link layer with trans flag and pass action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff trans gt 10)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ cgroup chain [0-9]+.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff trans gt 10\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "d392",
+        "name": "Add cgroup filter with cmp ematch u16/link layer and pipe action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u16 at 0 layer 0 mask 0xff00 lt 3)' action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ cgroup chain [0-9]+.*handle 0x1.*cmp\\(u16 at 0 layer 0 mask 0xff00 lt 3\\).*action pipe",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "0234",
+        "name": "Add cgroup filter with cmp ematch u32/link layer and miltiple actions",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u32 at 4 layer link mask 0xff00ff00 eq 3)' action skbedit mark 7 pipe action gact drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ cgroup chain [0-9]+.*handle 0x1.*cmp\\(u32 at 4 layer 0 mask 0xff00ff00 eq 3\\).*action.*skbedit.*mark 7 pipe.*action.*gact action drop",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "8499",
+        "name": "Add cgroup filter with cmp ematch u8/network layer and pass action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 0xab protocol ip prio 11 cgroup match 'cmp(u8 at 0 layer 1 mask 0xff gt 10)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show  dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 11 cgroup.*handle 0xab.*cmp\\(u8 at 0 layer 1 mask 0xff gt 10\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "b273",
+        "name": "Add cgroup filter with cmp ematch u8/network layer with trans flag and drop action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 0xab protocol ip prio 11 cgroup match 'cmp(u8 at 0 layer 1 mask 0xff trans gt 10)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 11 cgroup.*handle 0xab.*cmp\\(u8 at 0 layer 1 mask 0xff trans gt 10\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1934",
+        "name": "Add cgroup filter with cmp ematch u16/network layer and pipe action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 0x100 protocol ip prio 100 cgroup match 'cmp(u16 at 0 layer network mask 0xff00 lt 3)' action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "filter protocol ip pref 100 cgroup.*handle 0x100..*cmp\\(u16 at 0 layer 1 mask 0xff00 lt 3\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2733",
+        "name": "Add cgroup filter with cmp ematch u32/network layer and miltiple actions",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 0x112233 protocol ip prio 7 cgroup match 'cmp(u32 at 4 layer network mask 0xff00ff00 eq 3)' action skbedit mark 7 pipe action gact drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 7 cgroup.*handle 0x112233.*cmp\\(u32 at 4 layer 1 mask 0xff00ff00 eq 3\\).*action.*skbedit.*mark 7 pipe.*action.*gact action drop",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "3271",
+        "name": "Add cgroup filter with NOT cmp ematch rule and pass action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'not cmp(u8 at 0 layer link mask 0xff eq 3)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*NOT cmp\\(u8 at 0 layer 0 mask 0xff eq 3\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2362",
+        "name": "Add cgroup filter with two ANDed cmp ematch rules and single action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff eq 3) and cmp(u16 at 8 layer link mask 0x00ff gt 7)' action gact drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff eq 3\\).*AND cmp\\(u16 at 8 layer 0 mask 0xff gt 7\\).*action.*gact action drop",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "9993",
+        "name": "Add cgroup filter with two ORed cmp ematch rules and single action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff eq 3) or cmp(u16 at 8 layer link mask 0x00ff gt 7)' action gact drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff eq 3\\).*OR cmp\\(u16 at 8 layer 0 mask 0xff gt 7\\).*action.*gact action drop",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2331",
+        "name": "Add cgroup filter with two ANDed cmp ematch rules and one ORed ematch rule and single action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff eq 3) and cmp(u16 at 8 layer link mask 0x00ff gt 7) or cmp(u32 at 4 layer network mask 0xa0a0 lt 3)' action gact drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff eq 3\\).*AND cmp\\(u16 at 8 layer 0 mask 0xff gt 7\\).*OR cmp\\(u32 at 4 layer 1 mask 0xa0a0 lt 3\\).*action.*gact action drop",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "3645",
+        "name": "Add cgroup filter with two ANDed cmp ematch rules and one NOT ORed ematch rule and single action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff eq 3) and cmp(u16 at 8 layer link mask 0x00ff gt 7) or not cmp(u32 at 4 layer network mask 0xa0a0 lt 3)' action gact drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*cmp\\(u8 at 0 layer 0 mask 0xff eq 3\\).*AND cmp\\(u16 at 8 layer 0 mask 0xff gt 7\\).*OR NOT cmp\\(u32 at 4 layer 1 mask 0xa0a0 lt 3\\).*action.*gact action drop",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "b124",
+        "name": "Add cgroup filter with u32 ematch u8/zero offset and drop action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x11 0x0f at 0)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(01000000/0f000000 at 0\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "7381",
+        "name": "Add cgroup filter with u32 ematch u8/zero offset and invalid value >0xFF",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 basic match 'u32(u8 0x1122 0x0f at 0)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11220000/0f000000 at 0\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2231",
+        "name": "Add cgroup filter with u32 ematch u8/positive offset and drop action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x77 0x1f at 12)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(17000000/1f000000 at 12\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1882",
+        "name": "Add cgroup filter with u32 ematch u8/invalid mask >0xFF",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x77 0xff00 at 12)' action drop",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77000000/ff000000 at 12\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1237",
+        "name": "Add cgroup filter with u32 ematch u8/missing offset",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x77 0xff at)' action pipe",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77000000 at 12\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "3812",
+        "name": "Add cgroup filter with u32 ematch u8/missing AT keyword",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x77 0xff 0)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77000000 at 12\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1112",
+        "name": "Add cgroup filter with u32 ematch u8/missing value",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 at 12)' action drop",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(at 12\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "3241",
+        "name": "Add cgroup filter with u32 ematch u8/non-numeric value",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 zero 0xff at 0)' action pipe",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1 flowid 1:1.*u32\\(00000000/ff000000 at 0\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "e231",
+        "name": "Add cgroup filter with u32 ematch u8/non-numeric mask",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x11 mask at 0)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11000000/00000000 at 0\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4652",
+        "name": "Add cgroup filter with u32 ematch u8/negative offset and pass action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0xaa 0xf0 at -14)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(0000a000/0000f000 at -16\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "7566",
+        "name": "Add cgroup filter with u32 ematch u8/nexthdr+ offset and drop action",
+        "category": [
+            "filter",
+            "drop"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0xaa 0xf0 at nexthdr+0)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(a0000000/f0000000 at nexthdr\\+0\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1331",
+        "name": "Add cgroup filter with u32 ematch u16/zero offset and pipe action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x1122 0xffff at 0)' action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11220000/ffff0000 at 0\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "e354",
+        "name": "Add cgroup filter with u32 ematch u16/zero offset and invalid value >0xFFFF",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x112233 0xffff at 0)'",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11223300/ffff0000 at 0\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "3538",
+        "name": "Add cgroup filter with u32 ematch u16/positive offset and drop action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x7788 0x1fff at 12)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(17880000/1fff0000 at 12\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4576",
+        "name": "Add cgroup filter with u32 ematch u16/invalid mask >0xFFFF",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x7788 0xffffffff at 12)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77880000/ffffffff at 12\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "b842",
+        "name": "Add cgroup filter with u32 ematch u16/missing offset",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x7788 0xffff at)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77880000 at 12\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "c924",
+        "name": "Add cgroup filter with u32 ematch u16/missing AT keyword",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0x7788 0xffff 0)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77880000/ffff0000 at 0\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "cc93",
+        "name": "Add cgroup filter with u32 ematch u16/missing value",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 at 12)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(at 12\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "123c",
+        "name": "Add cgroup filter with u32 ematch u16/non-numeric value",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 zero 0xffff at 0)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(00000000/ffff0000 at 0\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "3675",
+        "name": "Add cgroup filter with u32 ematch u16/non-numeric mask",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u8 0x1122 mask at 0)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11220000/00000000 at 0\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1123",
+        "name": "Add cgroup filter with u32 ematch u16/negative offset and drop action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0xaabb 0xffff at -12)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(aabb0000/ffff0000 at -12\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4234",
+        "name": "Add cgroup filter with u32 ematch u16/nexthdr+ offset and pass action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u16 0xaabb 0xf0f0 at nexthdr+0)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(a0b00000/f0f00000 at nexthdr\\+0\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "e912",
+        "name": "Add cgroup filter with u32 ematch u32/zero offset and pipe action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0xaabbccdd 0xffffffff at 0)' action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(aabbccdd/ffffffff at 0\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1435",
+        "name": "Add cgroup filter with u32 ematch u32/positive offset and drop action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0x11227788 0x1ffff0f0 at 12)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11227080/1ffff0f0 at 12\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1282",
+        "name": "Add cgroup filter with u32 ematch u32/missing offset",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0x11227788 0xffffffff at)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11227788/ffffffff at 12\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "6456",
+        "name": "Add cgroup filter with u32 ematch u32/missing AT keyword",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0x77889900 0xfffff0f0 0)' action pipe",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(77889900/fffff0f0 at 0\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4231",
+        "name": "Add cgroup filter with u32 ematch u32/missing value",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 at 12)' action pipe",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(at 12\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2131",
+        "name": "Add cgroup filter with u32 ematch u32/non-numeric value",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 zero 0xffff at 0)' action pipe",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(00000000/ffff0000 at 0\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "f125",
+        "name": "Add cgroup filter with u32 ematch u32/non-numeric mask",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0x11223344 mask at 0)' action pass",
+        "expExitCode": "1",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(11223344/00000000 at 0\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4316",
+        "name": "Add cgroup filter with u32 ematch u32/negative offset and drop action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0xaabbccdd 0xff00ff00 at -12)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(aa00cc00/ff00ff00 at -12\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "23ae",
+        "name": "Add cgroup filter with u32 ematch u32/nexthdr+ offset and pipe action",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'u32(u32 0xaabbccdd 0xffffffff at nexthdr+0)' action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*u32\\(aabbccdd/ffffffff at nexthdr\\+0\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "23a1",
+        "name": "Add cgroup filter with canid ematch and single SFF",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 1)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(sff 0x1\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "324f",
+        "name": "Add cgroup filter with canid ematch and single SFF with mask",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 0xaabb:0x00ff)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(sff 0x2BB:0xFF\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2576",
+        "name": "Add cgroup filter with canid ematch and multiple SFF",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 1 sff 2 sff 3)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(sff 0x1 sff 0x2 sff 0x3\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4839",
+        "name": "Add cgroup filter with canid ematch and multiple SFF with masks",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 0xaa:0x01 sff 0xbb:0x02 sff 0xcc:0x03)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(sff 0xAA:0x1 sff 0xBB:0x2 sff 0xCC:0x3\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "6713",
+        "name": "Add cgroup filter with canid ematch and single EFF",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(eff 1)' action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0x1\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4572",
+        "name": "Add cgroup filter with canid ematch and single EFF with mask",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(eff 0xaabb:0xf1f1)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0xAABB:0xF1F1\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "8031",
+        "name": "Add cgroup filter with canid ematch and multiple EFF",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(eff 1 eff 2 eff 3)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0x1 eff 0x2 eff 0x3\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "ab9d",
+        "name": "Add cgroup filter with canid ematch and multiple EFF with masks",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(eff 0xaa:0x01 eff 0xbb:0x02 eff 0xcc:0x03)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0xAA:0x1 eff 0xBB:0x2 eff 0xCC:0x3\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "5349",
+        "name": "Add cgroup filter with canid ematch and a combination of SFF/EFF",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 0x01 eff 0x02)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0x2 sff 0x1\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "c934",
+        "name": "Add cgroup filter with canid ematch and a combination of SFF/EFF with masks",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'canid(sff 0x01:0xf eff 0x02:0xf)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref 1 cgroup.*handle 0x1.*canid\\(eff 0x2:0xF sff 0x1:0xF\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4319",
+        "name": "Replace cgroup filter with diffferent match",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff gt 10)' action pass"
+        ],
+        "cmdUnderTest": "$TC filter replace dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff gt 8)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "cmp\\(u8 at 0 layer 0 mask 0xff gt 8\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4636",
+        "name": "Delete cgroup filter",
+        "category": [
+            "filter",
+            "cgroup"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff gt 10)' action pass"
+        ],
+        "cmdUnderTest": "$TC filter delete dev $DEV1 parent ffff: protocol ip prio 1 cgroup match 'cmp(u8 at 0 layer link mask 0xff gt 10)' action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "cmp\\(u8 at 0 layer 0 mask 0xff gt 8\\)",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/flow.json b/tools/testing/selftests/tc-testing/tc-tests/filters/flow.json
new file mode 100644
index 000000000000..58189327f644
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/filters/flow.json
@@ -0,0 +1,623 @@
+[
+    {
+        "id": "5294",
+        "name": "Add flow filter with map key and ops",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key dst and 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys dst and 0x000000ff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "3514",
+        "name": "Add flow filter with map key or ops",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key dst or 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys dst.*or 0x000000ff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "7534",
+        "name": "Add flow filter with map key xor ops",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key dst xor 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys dst xor 0x000000ff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4524",
+        "name": "Add flow filter with map key rshift ops",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key dst rshift 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys dst rshift 255 baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "0230",
+        "name": "Add flow filter with map key addend ops",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key dst addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys dst addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2344",
+        "name": "Add flow filter with src map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key src addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys src addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "9304",
+        "name": "Add flow filter with proto map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key proto addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys proto addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "9038",
+        "name": "Add flow filter with proto-src map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key proto-src addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys proto-src addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2a03",
+        "name": "Add flow filter with proto-dst map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key proto-dst addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys proto-dst addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "a073",
+        "name": "Add flow filter with iif map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key iif addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys iif addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "3b20",
+        "name": "Add flow filter with priority map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key priority addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys priority addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "8945",
+        "name": "Add flow filter with mark map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key mark addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys mark addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "c034",
+        "name": "Add flow filter with nfct map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key nfct addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys nfct addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "0205",
+        "name": "Add flow filter with nfct-src map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key nfct-dst addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys nfct-dst addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "5315",
+        "name": "Add flow filter with nfct-src map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key nfct-src addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys nfct-src addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "7849",
+        "name": "Add flow filter with nfct-proto-src map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key nfct-proto-src addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys nfct-proto-src addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "9902",
+        "name": "Add flow filter with nfct-proto-dst map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key nfct-proto-dst addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys nfct-proto-dst addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "6742",
+        "name": "Add flow filter with rt-classid map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rt-classid addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys rt-classid addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "5432",
+        "name": "Add flow filter with sk-uid map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key sk-uid addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys sk-uid addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4134",
+        "name": "Add flow filter with sk-gid map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key sk-gid addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys sk-gid addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4522",
+        "name": "Add flow filter with vlan-tag map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key vlan-tag addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys vlan-tag addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4253",
+        "name": "Add flow filter with rxhash map key",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys rxhash addend 0xff baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4452",
+        "name": "Add flow filter with hash key list",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow hash keys src",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 hash keys src baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4341",
+        "name": "Add flow filter with muliple ops",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow hash keys src divisor 1024 baseclass 1:1 match 'cmp(u8 at 0 layer link mask 0xff gt 10)' action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 protocol ip prio 1 flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 hash keys src divisor 1024 baseclass 1:1.*cmp\\(u8 at 0 layer 0 mask 0xff gt 10\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4392",
+        "name": "List flow filters",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff",
+            "$TC filter add dev $DEV1 parent ffff: handle 2 prio 1 protocol ip flow map key rxhash or 0xff"
+        ],
+        "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "filter protocol ip pref 1 flow chain 0 handle 0x[0-9]+ map keys rxhash",
+        "matchCount": "2",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4322",
+        "name": "Change flow filter with map key num",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff"
+        ],
+        "cmdUnderTest": "$TC filter change dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0x22",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys rxhash addend 0x22 baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2320",
+        "name": "Replace flow filter with map key num",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff"
+        ],
+        "cmdUnderTest": "$TC filter replace dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0x88",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys rxhash addend 0x88 baseclass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "3213",
+        "name": "Delete flow filter with map key num",
+        "category": [
+            "filter",
+            "flow"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff"
+        ],
+        "cmdUnderTest": "$TC filter delete dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow map key rxhash addend 0xff",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip flow",
+        "matchPattern": "filter parent ffff: protocol ip pref 1 flow chain [0-9]+ handle 0x1 map keys rxhash addend 0x88 baseclass",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/route.json b/tools/testing/selftests/tc-testing/tc-tests/filters/route.json
new file mode 100644
index 000000000000..1f6f19f02997
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/filters/route.json
@@ -0,0 +1,181 @@
+[
+    {
+        "id": "e122",
+        "name": "Add route filter with from and to tag",
+        "category": [
+            "filter",
+            "route"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 100 route from 1 to 10 classid 1:10",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:",
+        "matchPattern": "flowid 1:10 to 10 from 1",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "6573",
+        "name": "Add route filter with fromif and to tag",
+        "category": [
+            "filter",
+            "route"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 100 route fromif $DEV1 to 10 classid 1:10",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:",
+        "matchPattern": "flowid 1:10 to 10 fromif",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1362",
+        "name": "Add route filter with to flag and reclassify action",
+        "category": [
+            "filter",
+            "route"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route to 10 classid 1:20 action reclassify",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:",
+        "matchPattern": "filter protocol ip pref.*route chain [0-9]+.*flowid 1:20 to 10.*action order [0-9]+: gact action reclassify",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4720",
+        "name": "Add route filter with from flag and continue actions",
+        "category": [
+            "filter",
+            "route"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 10 classid 1:100 action continue",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:",
+        "matchPattern": "filter protocol ip pref.*route chain [0-9]+.*flowid 1:100 from 10.*action continue",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2812",
+        "name": "Add route filter with form tag and pipe action",
+        "category": [
+            "filter",
+            "route"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 10 to 2 classid 1:1 action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:",
+        "matchPattern": "filter protocol ip pref.*route chain [0-9]+.*flowid 1:1 to 2 from 10.*action pipe",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "7994",
+        "name": "Add route filter with miltiple actions",
+        "category": [
+            "filter",
+            "route"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 10 to 2 classid 1:1 action skbedit mark 7 pipe action gact drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter ls dev $DEV1 parent ffff:",
+        "matchPattern": "filter protocol ip pref.*route chain [0-9]+.*flowid 1:1 to 2 from 10.*action order [0-9]+: skbedit  mark 7 pipe.*action order [0-9]+: gact action drop",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "4312",
+        "name": "List route filters",
+        "category": [
+            "filter",
+            "route"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 10 to 2 classid 1:1 action pipe",
+            "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 20 to 1 classid 1:20 action pipe"
+        ],
+        "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "action order [0-9]+: gact action pipe",
+        "matchCount": "2",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2634",
+        "name": "Delete route filter with pipe action",
+        "category": [
+            "filter",
+            "route"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: protocol ip prio 2 route from 10 to 2 classid 1:1 action pipe"
+        ],
+        "cmdUnderTest": "$TC filter del dev $DEV1 parent ffff: protocol ip prio 2 route from 10 to 2 classid 1:1 action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "filter protocol ip pref.*route chain [0-9]+.*flowid 1:1 to 2 from 10.*action pipe",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/rsvp.json b/tools/testing/selftests/tc-testing/tc-tests/filters/rsvp.json
new file mode 100644
index 000000000000..bdcbaa4c5663
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/filters/rsvp.json
@@ -0,0 +1,203 @@
+[
+    {
+        "id": "2141",
+        "name": "Add rsvp filter with tcp proto and specific IP address",
+        "category": [
+            "filter",
+            "rsvp"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto tcp session 198.168.10.64",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*session 198.168.10.64 ipproto tcp",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "5267",
+        "name": "Add rsvp filter with udp proto and specific IP address",
+        "category": [
+            "filter",
+            "rsvp"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*session 1.1.1.1 ipproto udp",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2819",
+        "name": "Add rsvp filter with src ip and src port",
+        "category": [
+            "filter",
+            "rsvp"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1 sender 2.2.2.2/5021 classid 1:1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*flowid 1:1 session 1.1.1.1 ipproto udp sender  2.2.2.2/5021",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "c967",
+        "name": "Add rsvp filter with tunnelid and continue action",
+        "category": [
+            "filter",
+            "rsvp"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1 tunnelid 2 classid 1:1 action continue",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*flowid 1:1 session 1.1.1.1 ipproto udp tunnelid 2.*action order [0-9]+: gact action continue",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "5463",
+        "name": "Add rsvp filter with tunnel and pipe action",
+        "category": [
+            "filter",
+            "rsvp"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1 tunnel 2 skip 1 action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*tunnel 2 skip 1 session 1.1.1.1 ipproto udp.*action order [0-9]+: gact action pipe",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2332",
+        "name": "Add rsvp filter with miltiple actions",
+        "category": [
+            "filter",
+            "rsvp"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 7 rsvp ipproto udp session 1.1.1.1 classid 1:1 action skbedit mark 7 pipe action gact drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*flowid 1:1 session 1.1.1.1 ipproto udp.*action order [0-9]+: skbedit  mark 7 pipe.*action order [0-9]+: gact action drop",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "8879",
+        "name": "Add rsvp filter with tunnel and skp flag",
+        "category": [
+            "filter",
+            "rsvp"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1 tunnel 2 skip 1 action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*tunnel 2 skip 1 session 1.1.1.1 ipproto udp.*action order [0-9]+: gact action pipe",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "8261",
+        "name": "List rsvp filters",
+        "category": [
+            "filter",
+            "rsvp"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1/1234 classid 1:1",
+            "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto tcp session 2.2.2.2/1234 classid 2:1"
+        ],
+        "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "^filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh",
+        "matchCount": "2",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "8989",
+        "name": "Delete rsvp filter",
+        "category": [
+            "filter",
+            "rsvp"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1/1234 tunnelid 9 classid 2:1"
+        ],
+        "cmdUnderTest": "$TC filter del dev $DEV1 parent ffff: protocol ip prio 1 rsvp ipproto udp session 1.1.1.1/1234 tunnelid 9 classid 2:1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "filter protocol ip pref [0-9]+ rsvp chain [0-9]+ fh 0x.*flowid 2:1 session 1.1.1.1/1234 ipproto udp tunnelid 9",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/filters/tcindex.json b/tools/testing/selftests/tc-testing/tc-tests/filters/tcindex.json
new file mode 100644
index 000000000000..44901db70376
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/filters/tcindex.json
@@ -0,0 +1,227 @@
+[
+    {
+        "id": "8293",
+        "name": "Add tcindex filter with default action",
+        "category": [
+            "filter",
+            "tcindex"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex",
+        "matchPattern": "^filter parent ffff: protocol ip pref 1 tcindex chain 0 handle 0x0001 classid 1:1",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "7281",
+        "name": "Add tcindex filter with hash size and pass action",
+        "category": [
+            "filter",
+            "tcindex"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex hash 32 fall_through classid 1:1 action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex",
+        "matchPattern": "^filter parent ffff: protocol ip pref.*tcindex chain [0-9]+ handle 0x0001 classid 1:1.*action order [0-9]+: gact action pass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "b294",
+        "name": "Add tcindex filter with mask shift and reclassify action",
+        "category": [
+            "filter",
+            "tcindex"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex hash 32 mask 1 shift 2 fall_through classid 1:1 action reclassify",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex",
+        "matchPattern": "^filter parent ffff: protocol ip pref.*tcindex chain [0-9]+ handle 0x0001 classid 1:1.*action order [0-9]+: gact action reclassify",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "0532",
+        "name": "Add tcindex filter with pass_on and continue actions",
+        "category": [
+            "filter",
+            "tcindex"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex hash 32 mask 1 shift 2 pass_on classid 1:1 action continue",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex",
+        "matchPattern": "^filter parent ffff: protocol ip pref.*tcindex chain [0-9]+ handle 0x0001 classid 1:1.*action order [0-9]+: gact action continue",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "d473",
+        "name": "Add tcindex filter with pipe action",
+        "category": [
+            "filter",
+            "tcindex"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex hash 32 mask 1 shift 2 fall_through classid 1:1 action pipe",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex",
+        "matchPattern": "^filter parent ffff: protocol ip pref.*tcindex chain [0-9]+ handle 0x0001 classid 1:1.*action order [0-9]+: gact action pipe",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2940",
+        "name": "Add tcindex filter with miltiple actions",
+        "category": [
+            "filter",
+            "tcindex"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress"
+        ],
+        "cmdUnderTest": "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 7 tcindex hash 32 mask 1 shift 2 fall_through classid 1:1 action skbedit mark 7 pipe action gact drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 7 protocol ip tcindex",
+        "matchPattern": "^filter parent ffff: protocol ip pref 7 tcindex.*handle 0x0001.*action.*skbedit.*mark 7 pipe.*action.*gact action drop",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "1893",
+        "name": "List tcindex filters",
+        "category": [
+            "filter",
+            "tcindex"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1",
+            "$TC filter add dev $DEV1 parent ffff: handle 2 protocol ip prio 1 tcindex classid 1:1"
+        ],
+        "cmdUnderTest": "$TC filter show dev $DEV1 parent ffff:",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter show dev $DEV1 parent ffff:",
+        "matchPattern": "handle 0x000[0-9]+ classid 1:1",
+        "matchCount": "2",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "2041",
+        "name": "Change tcindex filter with pass action",
+        "category": [
+            "filter",
+            "tcindex"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action drop"
+        ],
+        "cmdUnderTest": "$TC filter change dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex",
+        "matchPattern": "handle 0x0001 classid 1:1.*action order [0-9]+: gact action pass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "9203",
+        "name": "Replace tcindex filter with pass action",
+        "category": [
+            "filter",
+            "tcindex"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action drop"
+        ],
+        "cmdUnderTest": "$TC filter replace dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action pass",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex",
+        "matchPattern": "handle 0x0001 classid 1:1.*action order [0-9]+: gact action pass",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    },
+    {
+        "id": "7957",
+        "name": "Delete tcindex filter with drop action",
+        "category": [
+            "filter",
+            "tcindex"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$TC qdisc add dev $DEV1 ingress",
+            "$TC filter add dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action drop"
+        ],
+        "cmdUnderTest": "$TC filter del dev $DEV1 parent ffff: handle 1 protocol ip prio 1 tcindex classid 1:1 action drop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC filter get dev $DEV1 parent ffff: handle 1 prio 1 protocol ip tcindex",
+        "matchPattern": "handle 0x0001 classid 1:1.*action order [0-9]+: gact action drop",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DEV1 ingress"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/atm.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/atm.json
new file mode 100644
index 000000000000..f5bc8670a67d
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/atm.json
@@ -0,0 +1,94 @@
+[
+    {
+        "id": "7628",
+        "name": "Create ATM with default setting",
+        "category": [
+            "qdisc",
+            "atm"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root atm",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc atm 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "390a",
+        "name": "Delete ATM with valid handle",
+        "category": [
+            "qdisc",
+            "atm"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root atm"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc atm 1: root refcnt",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "32a0",
+        "name": "Show ATM class",
+        "category": [
+            "qdisc",
+            "atm"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root atm",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class atm 1: parent 1:",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "6310",
+        "name": "Dump ATM stats",
+        "category": [
+            "qdisc",
+            "atm"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root atm",
+        "expExitCode": "0",
+        "verifyCmd": "$TC -s qdisc show dev $DUMMY",
+        "matchPattern": "qdisc atm 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cake.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cake.json
new file mode 100644
index 000000000000..1134b72d281d
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cake.json
@@ -0,0 +1,487 @@
+[
+    {
+        "id": "1212",
+        "name": "Create CAKE with default setting",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3281",
+        "name": "Create CAKE with bandwidth limit",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake bandwidth 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth 1Kbit diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "c940",
+        "name": "Create CAKE with autorate-ingress flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake autorate-ingress",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited autorate-ingress diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2310",
+        "name": "Create CAKE with rtt time",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake rtt 200",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 200us raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2385",
+        "name": "Create CAKE with besteffort flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake besteffort",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited besteffort triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "a032",
+        "name": "Create CAKE with diffserv8 flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake diffserv8",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv8 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2349",
+        "name": "Create CAKE with diffserv4 flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake diffserv4",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv4 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "8472",
+        "name": "Create CAKE with flowblind flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake flowblind",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 flowblind nonat nowash no-ack-filter split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2341",
+        "name": "Create CAKE with dsthost and nat flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake dsthost nat",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 dsthost nat nowash no-ack-filter split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5134",
+        "name": "Create CAKE with wash flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake hosts wash",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 hosts nonat wash no-ack-filter split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2302",
+        "name": "Create CAKE with flowblind and no-split-gso flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake flowblind no-split-gso",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 flowblind nonat nowash no-ack-filter no-split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0768",
+        "name": "Create CAKE with dual-srchost and ack-filter flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake dual-srchost ack-filter",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 dual-srchost nonat nowash ack-filter split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0238",
+        "name": "Create CAKE with dual-dsthost and ack-filter-aggressive flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake dual-dsthost ack-filter-aggressive",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 dual-dsthost nonat nowash ack-filter-aggressive split-gso rtt 100ms raw overhead",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "6572",
+        "name": "Create CAKE with memlimit and ptm flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake memlimit 10000 ptm",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw ptm overhead 0 memlimit 10000b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2436",
+        "name": "Create CAKE with fwmark and atm flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake fwmark 8 atm",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms raw atm overhead 0 fwmark 0x8",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3984",
+        "name": "Create CAKE with overhead and mpu",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake overhead 128 mpu 256",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms noatm overhead 128 mpu 256",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5421",
+        "name": "Create CAKE with conservative and ingress flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake conservative ingress",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash ingress no-ack-filter split-gso rtt 100ms atm overhead 48",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "6854",
+        "name": "Delete CAKE with conservative and ingress flag",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root cake conservative ingress"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash ingress no-ack-filter split-gso rtt 100ms atm overhead 48",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2342",
+        "name": "Replace CAKE with mpu",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root cake overhead 128 mpu 256"
+        ],
+        "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root cake mpu 128",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms noatm overhead 128 mpu 128",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2313",
+        "name": "Change CAKE with mpu",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root cake overhead 128 mpu 256"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root cake mpu 128",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cake 1: root refcnt [0-9]+ bandwidth unlimited diffserv3 triple-isolate nonat nowash no-ack-filter split-gso rtt 100ms noatm overhead 128 mpu 128",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4365",
+        "name": "Show CAKE class",
+        "category": [
+            "qdisc",
+            "cake"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cake",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class cake",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbq.json
new file mode 100644
index 000000000000..1ab21c83a122
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbq.json
@@ -0,0 +1,184 @@
+[
+    {
+        "id": "3460",
+        "name": "Create CBQ with default setting",
+        "category": [
+            "qdisc",
+            "cbq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0592",
+        "name": "Create CBQ with mpu",
+        "category": [
+            "qdisc",
+            "cbq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000 mpu 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4684",
+        "name": "Create CBQ with valid cell num",
+        "category": [
+            "qdisc",
+            "cbq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000 cell 128",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4345",
+        "name": "Create CBQ with invalid cell num",
+        "category": [
+            "qdisc",
+            "cbq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000 cell 100",
+        "expExitCode": "1",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4525",
+        "name": "Create CBQ with valid ewma",
+        "category": [
+            "qdisc",
+            "cbq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000 ewma 16",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "6784",
+        "name": "Create CBQ with invalid ewma",
+        "category": [
+            "qdisc",
+            "cbq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000 ewma 128",
+        "expExitCode": "1",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5468",
+        "name": "Delete CBQ with handle",
+        "category": [
+            "qdisc",
+            "cbq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbq 1: root refcnt [0-9]+ rate 10Kbit \\(bounded,isolated\\) prio no-transmit",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "492a",
+        "name": "Show CBQ class",
+        "category": [
+            "qdisc",
+            "cbq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbq bandwidth 10000 avpkt 9000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class cbq 1: root rate 10Kbit \\(bounded,isolated\\) prio no-transmit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbs.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbs.json
new file mode 100644
index 000000000000..a46bf5ff8277
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/cbs.json
@@ -0,0 +1,234 @@
+[
+    {
+        "id": "1820",
+        "name": "Create CBS with default setting",
+        "category": [
+            "qdisc",
+            "cbs"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 0 idleslope 0 offload 0.*qdisc pfifo 0: parent 1: limit 1000p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "1532",
+        "name": "Create CBS with hicredit setting",
+        "category": [
+            "qdisc",
+            "cbs"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs hicredit 64",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 64 locredit 0 sendslope 0 idleslope 0 offload 0.*qdisc pfifo 0: parent 1: limit 1000p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2078",
+        "name": "Create CBS with locredit setting",
+        "category": [
+            "qdisc",
+            "cbs"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs locredit 10",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 10 sendslope 0 idleslope 0 offload 0.*qdisc pfifo 0: parent 1: limit 1000p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9271",
+        "name": "Create CBS with sendslope setting",
+        "category": [
+            "qdisc",
+            "cbs"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs sendslope 888",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 888 idleslope 0 offload 0.*qdisc pfifo 0: parent 1: limit 1000p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0482",
+        "name": "Create CBS with idleslope setting",
+        "category": [
+            "qdisc",
+            "cbs"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs idleslope 666",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 0 idleslope 666 offload 0.*qdisc pfifo 0: parent 1: limit 1000p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "e8f3",
+        "name": "Create CBS with multiple setting",
+        "category": [
+            "qdisc",
+            "cbs"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs hicredit 10 locredit 75 sendslope 2 idleslope 666",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 10 locredit 75 sendslope 2 idleslope 666 offload 0.*qdisc pfifo 0: parent 1: limit 1000p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "23c9",
+        "name": "Replace CBS with sendslope setting",
+        "category": [
+            "qdisc",
+            "cbs"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root cbs idleslope 666"
+        ],
+        "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root cbs sendslope 10",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 10 idleslope 0 offload 0.*qdisc pfifo 0: parent 1: limit 1000p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "a07a",
+        "name": "Change CBS with idleslope setting",
+        "category": [
+            "qdisc",
+            "cbs"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root cbs idleslope 666"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root cbs idleslope 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 0 idleslope 1 offload 0.*qdisc pfifo 0: parent 1: limit 1000p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "43b3",
+        "name": "Delete CBS with handle",
+        "category": [
+            "qdisc",
+            "cbs"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root cbs idleslope 666"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc cbs 1: root refcnt [0-9]+ hicredit 0 locredit 0 sendslope 0 idleslope 1 offload 0.*qdisc pfifo 0: parent 1: limit 1000p",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9472",
+        "name": "Show CBS class",
+        "category": [
+            "qdisc",
+            "cbs"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root cbs",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class cbs 1:[0-9]+ parent 1:",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/choke.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/choke.json
new file mode 100644
index 000000000000..31b7775d25fc
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/choke.json
@@ -0,0 +1,188 @@
+[
+    {
+        "id": "8937",
+        "name": "Create CHOKE with default setting",
+        "category": [
+            "qdisc",
+            "choke"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 83p max 250p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "48c0",
+        "name": "Create CHOKE with min packet setting",
+        "category": [
+            "qdisc",
+            "choke"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 min 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 100p max 250p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "38c1",
+        "name": "Create CHOKE with max packet setting",
+        "category": [
+            "qdisc",
+            "choke"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 max 900",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min.*max 900p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "234a",
+        "name": "Create CHOKE with ecn setting",
+        "category": [
+            "qdisc",
+            "choke"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 ecn",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 83p max 250p ecn",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4380",
+        "name": "Create CHOKE with burst setting",
+        "category": [
+            "qdisc",
+            "choke"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 burst 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 83p max 250p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "48c7",
+        "name": "Delete CHOKE with valid handle",
+        "category": [
+            "qdisc",
+            "choke"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 83p max 250p",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4398",
+        "name": "Replace CHOKE with min setting",
+        "category": [
+            "qdisc",
+            "choke"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000"
+        ],
+        "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 min 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 100p max 250p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0301",
+        "name": "Change CHOKE with limit setting",
+        "category": [
+            "qdisc",
+            "choke"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root choke limit 1000 bandwidth 10000 min 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc choke 1: root refcnt [0-9]+ limit 1000p min 100p max 250p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/codel.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/codel.json
new file mode 100644
index 000000000000..ea38099d48e5
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/codel.json
@@ -0,0 +1,211 @@
+[
+    {
+        "id": "983a",
+        "name": "Create CODEL with default setting",
+        "category": [
+            "qdisc",
+            "codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 5ms interval 100ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "38aa",
+        "name": "Create CODEL with limit packet setting",
+        "category": [
+            "qdisc",
+            "codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel limit 1500",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1500p target 5ms interval 100ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9178",
+        "name": "Create CODEL with target setting",
+        "category": [
+            "qdisc",
+            "codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel target 100ms",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 100ms interval 100ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "78d1",
+        "name": "Create CODEL with interval setting",
+        "category": [
+            "qdisc",
+            "codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel interval 20ms",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 5ms interval 20ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "238a",
+        "name": "Create CODEL with ecn setting",
+        "category": [
+            "qdisc",
+            "codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel ecn",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 5ms interval 100ms ecn",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "939c",
+        "name": "Create CODEL with ce_threshold setting",
+        "category": [
+            "qdisc",
+            "codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root codel ce_threshold 20ms",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 5ms ce_threshold 20ms interval 100ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "8380",
+        "name": "Delete CODEL with valid handle",
+        "category": [
+            "qdisc",
+            "codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root codel"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 1000p target 5ms interval 100ms",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "289c",
+        "name": "Replace CODEL with limit setting",
+        "category": [
+            "qdisc",
+            "codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root codel"
+        ],
+        "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root codel limit 5000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 5000p target 5ms interval 100ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0648",
+        "name": "Change CODEL with limit setting",
+        "category": [
+            "qdisc",
+            "codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root codel"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root codel limit 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc codel 1: root refcnt [0-9]+ limit 100p target 5ms interval 100ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/drr.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/drr.json
new file mode 100644
index 000000000000..486a425b3c1c
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/drr.json
@@ -0,0 +1,71 @@
+[
+    {
+        "id": "0385",
+        "name": "Create DRR with default setting",
+        "category": [
+            "qdisc",
+            "drr"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root drr",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc drr 1: root refcnt [0-9]+",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2375",
+        "name": "Delete DRR with handle",
+        "category": [
+            "qdisc",
+            "drr"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root drr"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc drr 1: root refcnt [0-9]+",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3092",
+        "name": "Show DRR class",
+        "category": [
+            "qdisc",
+            "drr"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root drr",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class drr 1:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/dsmark.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/dsmark.json
new file mode 100644
index 000000000000..c030795f9c37
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/dsmark.json
@@ -0,0 +1,140 @@
+[
+    {
+        "id": "6345",
+        "name": "Create DSMARK with default setting",
+        "category": [
+            "qdisc",
+            "dsmark"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc dsmark 1: root refcnt [0-9]+ indices 0x0400",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3462",
+        "name": "Create DSMARK with default_index setting",
+        "category": [
+            "qdisc",
+            "dsmark"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024 default_index 512",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc dsmark 1: root refcnt [0-9]+ indices 0x0400 default_index 0x0200",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "ca95",
+        "name": "Create DSMARK with set_tc_index flag",
+        "category": [
+            "qdisc",
+            "dsmark"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024 set_tc_index",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc dsmark 1: root refcnt [0-9]+ indices 0x0400 set_tc_index",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "a950",
+        "name": "Create DSMARK with multiple setting",
+        "category": [
+            "qdisc",
+            "dsmark"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024 default_index 1024 set_tc_index",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc dsmark 1: root refcnt [0-9]+ indices 0x0400 default_index 0x0400 set_tc_index",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4092",
+        "name": "Delete DSMARK with handle",
+        "category": [
+            "qdisc",
+            "dsmark"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024 default_index 1024"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc dsmark 1: root refcnt [0-9]+ indices 0x0400",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5930",
+        "name": "Show DSMARK class",
+        "category": [
+            "qdisc",
+            "dsmark"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root dsmark indices 1024",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class dsmark 1:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/etf.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/etf.json
new file mode 100644
index 000000000000..0046d44bcd93
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/etf.json
@@ -0,0 +1,117 @@
+[
+    {
+        "id": "34ba",
+        "name": "Create ETF with default setting",
+        "category": [
+            "qdisc",
+            "etf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root etf clockid CLOCK_TAI",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc etf 1: root refcnt [0-9]+ clockid TAI delta 0 offload off deadline_mode off skip_sock_check off",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "438f",
+        "name": "Create ETF with delta nanos setting",
+        "category": [
+            "qdisc",
+            "etf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root etf delta 100 clockid CLOCK_TAI",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc etf 1: root refcnt [0-9]+ clockid TAI delta 100 offload off deadline_mode off skip_sock_check off",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9041",
+        "name": "Create ETF with deadline_mode setting",
+        "category": [
+            "qdisc",
+            "etf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root etf clockid CLOCK_TAI deadline_mode",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc etf 1: root refcnt [0-9]+ clockid TAI delta 0 offload off deadline_mode on skip_sock_check off",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9a0c",
+        "name": "Create ETF with skip_sock_check setting",
+        "category": [
+            "qdisc",
+            "etf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root etf clockid CLOCK_TAI skip_sock_check",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc etf 1: root refcnt [0-9]+ clockid TAI delta 0 offload off deadline_mode off skip_sock_check on",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2093",
+        "name": "Delete ETF with valid handle",
+        "category": [
+            "qdisc",
+            "etf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root etf clockid CLOCK_TAI"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc etf 1: root refcnt [0-9]+ clockid TAI delta 0 offload off deadline_mode off skip_sock_check off",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq.json
new file mode 100644
index 000000000000..8acb904d1419
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq.json
@@ -0,0 +1,395 @@
+[
+    {
+        "id": "983b",
+        "name": "Create FQ with default setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "38a1",
+        "name": "Create FQ with limit packet setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq limit 3000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 3000p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0a18",
+        "name": "Create FQ with flow_limit setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq flow_limit 300",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 300p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2390",
+        "name": "Create FQ with quantum setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq quantum 9000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p buckets.*orphan_mask 1023 quantum 9000b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "845b",
+        "name": "Create FQ with initial_quantum setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq initial_quantum 900000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p buckets.*initial_quantum 900000b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9398",
+        "name": "Create FQ with maxrate setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq maxrate 100000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p buckets.*maxrate 100Kbit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "342c",
+        "name": "Create FQ with nopacing setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq nopacing",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*nopacing",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "6391",
+        "name": "Create FQ with refill_delay setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq refill_delay 100ms",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*refill_delay 100ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "238b",
+        "name": "Create FQ with low_rate_threshold setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq low_rate_threshold 10000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*low_rate_threshold 10Kbit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "7582",
+        "name": "Create FQ with orphan_mask setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq orphan_mask 255",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*orphan_mask 255",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4894",
+        "name": "Create FQ with timer_slack setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq timer_slack 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*timer_slack 100ns",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "324c",
+        "name": "Create FQ with ce_threshold setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq ce_threshold 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "424a",
+        "name": "Create FQ with horizon time setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq horizon 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*horizon 100us",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "89e1",
+        "name": "Create FQ with horizon_cap setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq horizon_cap",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p flow_limit 100p.*horizon_cap",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "32e1",
+        "name": "Delete FQ with valid handle",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root fq"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 10000p",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "49b0",
+        "name": "Replace FQ with limit setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root fq"
+        ],
+        "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root fq limit 5000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 5000p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9478",
+        "name": "Change FQ with limit setting",
+        "category": [
+            "qdisc",
+            "fq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root fq"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root fq limit 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq 1: root refcnt [0-9]+ limit 100p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq_codel.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq_codel.json
new file mode 100644
index 000000000000..a65266357a9a
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/fq_codel.json
@@ -0,0 +1,326 @@
+[
+    {
+        "id": "4957",
+        "name": "Create FQ_CODEL with default setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "7621",
+        "name": "Create FQ_CODEL with limit setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel limit 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 1000p flows 1024 quantum.*target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "6871",
+        "name": "Create FQ_CODEL with memory_limit setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel memory_limit 100000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms interval 100ms memory_limit 100000b ecn drop_batch 64",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5636",
+        "name": "Create FQ_CODEL with target setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel target 2000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 2ms interval 100ms memory_limit 32Mb ecn drop_batch 64",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "630a",
+        "name": "Create FQ_CODEL with interval setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel interval 5000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms interval 5ms memory_limit 32Mb ecn drop_batch 64",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4324",
+        "name": "Create FQ_CODEL with quantum setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel quantum 9000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum 9000 target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 64",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "b190",
+        "name": "Create FQ_CODEL with noecn flag",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel noecn",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms interval 100ms memory_limit 32Mb drop_batch 64",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5381",
+        "name": "Create FQ_CODEL with ce_threshold setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel ce_threshold 1024000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms ce_threshold 1.02s interval 100ms memory_limit 32Mb ecn drop_batch 64",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "c9d2",
+        "name": "Create FQ_CODEL with drop_batch setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel drop_batch 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 10240p flows 1024 quantum.*target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 100",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "523b",
+        "name": "Create FQ_CODEL with multiple setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel limit 1000 flows 256 drop_batch 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 1000p flows 256 quantum.*target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 100",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9283",
+        "name": "Replace FQ_CODEL with noecn setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root fq_codel limit 1000 flows 256 drop_batch 100"
+        ],
+        "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root fq_codel noecn",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 1000p flows 256 quantum.*target 5ms interval 100ms memory_limit 32Mb drop_batch 100",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3459",
+        "name": "Change FQ_CODEL with limit setting",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root fq_codel limit 1000 flows 256 drop_batch 100"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root fq_codel limit 2000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 2000p flows 256 quantum.*target 5ms interval 100ms memory_limit 32Mb ecn drop_batch 100",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0128",
+        "name": "Delete FQ_CODEL with handle",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root fq_codel limit 1000 flows 256 drop_batch 100"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc fq_codel 1: root refcnt [0-9]+ limit 1000p flows 256 quantum.*target 5ms interval 100ms memory_limit 32Mb noecn drop_batch 100",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0435",
+        "name": "Show FQ_CODEL class",
+        "category": [
+            "qdisc",
+            "fq_codel"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root fq_codel",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class fq_codel 1:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/gred.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/gred.json
new file mode 100644
index 000000000000..013c8ee037a4
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/gred.json
@@ -0,0 +1,164 @@
+[
+    {
+        "id": "8942",
+        "name": "Create GRED with default setting",
+        "category": [
+            "qdisc",
+            "gred"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 1",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5783",
+        "name": "Create GRED with grio setting",
+        "category": [
+            "qdisc",
+            "gred"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 1 grio",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 1.*grio",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "8a09",
+        "name": "Create GRED with limit setting",
+        "category": [
+            "qdisc",
+            "gred"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 1 limit 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 1 limit 1000b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "48ca",
+        "name": "Create GRED with ecn setting",
+        "category": [
+            "qdisc",
+            "gred"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 2 ecn",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 2.*ecn",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "48cb",
+        "name": "Create GRED with harddrop setting",
+        "category": [
+            "qdisc",
+            "gred"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 2 harddrop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 2.*harddrop",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "763a",
+        "name": "Change GRED setting",
+        "category": [
+            "qdisc",
+            "gred"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 1"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root gred limit 60KB min 15K max 25K burst 64 avpkt 1500 bandwidth 10Mbit DP 1 probability 0.1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc gred 1: root refcnt [0-9]+ vqs 10 default 1 limit.*vq 1 prio [0-9]+ limit 60Kb min 15Kb max 25Kb",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "8309",
+        "name": "Show GRED class",
+        "category": [
+            "qdisc",
+            "gred"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root gred setup vqs 10 default 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class gred 1:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hfsc.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hfsc.json
new file mode 100644
index 000000000000..af27b2c20e17
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hfsc.json
@@ -0,0 +1,167 @@
+[
+    {
+        "id": "3254",
+        "name": "Create HFSC with default setting",
+        "category": [
+            "qdisc",
+            "hfsc"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hfsc",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc hfsc 1: root refcnt [0-9]+",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0289",
+        "name": "Create HFSC with class sc and ul rate setting",
+        "category": [
+            "qdisc",
+            "hfsc"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root hfsc default 11"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 hfsc sc rate 20000 ul rate 10000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class hfsc 1:1 parent 1: sc m1 0bit d 0us m2 20Kbit ul m1 0bit d 0us m2 10Kbit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "846a",
+        "name": "Create HFSC with class sc umax and dmax setting",
+        "category": [
+            "qdisc",
+            "hfsc"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root hfsc default 11"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 hfsc sc umax 1540 dmax 5ms rate 10000 ul rate 10000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class hfsc 1:1 parent 1: sc m1 2464Kbit d 5ms m2 10Kbit ul m1 0bit d 0us m2 10Kbit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5413",
+        "name": "Create HFSC with class rt and ls rate setting",
+        "category": [
+            "qdisc",
+            "hfsc"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root hfsc default 11"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 hfsc rt rate 20000 ls rate 10000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class hfsc 1:1 parent 1: rt m1 0bit d 0us m2 20Kbit ls m1 0bit d 0us m2 10Kbit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9312",
+        "name": "Create HFSC with class rt umax and dmax setting",
+        "category": [
+            "qdisc",
+            "hfsc"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root hfsc default 11"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 hfsc rt umax 1540 dmax 5ms rate 10000 ls rate 10000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class hfsc 1:1 parent 1: rt m1 2464Kbit d 5ms m2 10Kbit ls m1 0bit d 0us m2 10Kbit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "6931",
+        "name": "Delete HFSC with handle",
+        "category": [
+            "qdisc",
+            "hfsc"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root hfsc default 11"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc hfsc 1: root refcnt [0-9]+",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "8436",
+        "name": "Show HFSC class",
+        "category": [
+            "qdisc",
+            "hfsc"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hfsc",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class hfsc 1: root",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hhf.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hhf.json
new file mode 100644
index 000000000000..949f6e5de902
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/hhf.json
@@ -0,0 +1,210 @@
+[
+    {
+        "id": "4812",
+        "name": "Create HHF with default setting",
+        "category": [
+            "qdisc",
+            "hhf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 2",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "8a92",
+        "name": "Create HHF with limit setting",
+        "category": [
+            "qdisc",
+            "hhf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf limit 1500",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc hhf 1: root refcnt [0-9]+ limit 1500p.*hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 2",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3491",
+        "name": "Create HHF with quantum setting",
+        "category": [
+            "qdisc",
+            "hhf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf quantum 9000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*quantum 9000b hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 2",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "ba04",
+        "name": "Create HHF with reset_timeout setting",
+        "category": [
+            "qdisc",
+            "hhf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf reset_timeout 100ms",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*hh_limit 2048 reset_timeout 100ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 2",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4238",
+        "name": "Create HHF with admit_bytes setting",
+        "category": [
+            "qdisc",
+            "hhf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf admit_bytes 100000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*hh_limit 2048 reset_timeout 40ms admit_bytes 100000b evict_timeout 1s non_hh_weight 2",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "839f",
+        "name": "Create HHF with evict_timeout setting",
+        "category": [
+            "qdisc",
+            "hhf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf evict_timeout 0.5s",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 500ms non_hh_weight 2",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "a044",
+        "name": "Create HHF with non_hh_weight setting",
+        "category": [
+            "qdisc",
+            "hhf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf non_hh_weight 10",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc hhf 1: root refcnt [0-9]+.*hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 10",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "32f9",
+        "name": "Change HHF with limit setting",
+        "category": [
+            "qdisc",
+            "hhf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root hhf"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root hhf limit 1500",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc hhf 1: root refcnt [0-9]+ limit 1500p.*hh_limit 2048 reset_timeout 40ms admit_bytes 128Kb evict_timeout 1s non_hh_weight 2",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "385e",
+        "name": "Show HHF class",
+        "category": [
+            "qdisc",
+            "hhf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root hhf",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class hhf 1:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/htb.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/htb.json
new file mode 100644
index 000000000000..9529899482e0
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/htb.json
@@ -0,0 +1,285 @@
+[
+    {
+        "id": "0904",
+        "name": "Create HTB with default setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root htb",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc htb 1: root refcnt [0-9]+ r2q 10 default 0 direct_packets_stat.*direct_qlen",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3906",
+        "name": "Create HTB with default-N setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root htb default 10",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc htb 1: root refcnt [0-9]+ r2q 10 default 0x10 direct_packets_stat.* direct_qlen",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "8492",
+        "name": "Create HTB with r2q setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root htb r2q 5",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc htb 1: root refcnt [0-9]+ r2q 5 default 0 direct_packets_stat.*direct_qlen",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9502",
+        "name": "Create HTB with direct_qlen setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root htb direct_qlen 1024",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc htb 1: root refcnt [0-9]+ r2q 10 default 0 direct_packets_stat.*direct_qlen 1024",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "b924",
+        "name": "Create HTB with class rate and burst setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root htb"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20kbit burst 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 20Kbit burst 1000b cburst 1600b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4359",
+        "name": "Create HTB with class mpu setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root htb"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit mpu 64",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 20Kbit burst 1600b cburst 1600b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9048",
+        "name": "Create HTB with class prio setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root htb"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit prio 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class htb 1:1 root prio 1 rate 20Kbit ceil 20Kbit burst 1600b cburst 1600b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4994",
+        "name": "Create HTB with class ceil setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root htb"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit ceil 10Kbit",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 10Kbit burst 1600b cburst 1600b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9523",
+        "name": "Create HTB with class cburst setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root htb"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit cburst 2000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 20Kbit burst 1600b cburst 2000b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5353",
+        "name": "Create HTB with class mtu setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root htb"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit mtu 2048",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 20Kbit burst 2Kb cburst 2Kb",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "346a",
+        "name": "Create HTB with class quantum setting",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root htb"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 htb rate 20Kbit quantum 2048",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class htb 1:1 root prio 0 rate 20Kbit ceil 20Kbit burst 1600b cburst 1600b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "303a",
+        "name": "Delete HTB with handle",
+        "category": [
+            "qdisc",
+            "htb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root htb r2q 5"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc htb 1: root refcnt [0-9]+",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/ingress.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/ingress.json
index d99dba6e2b1a..11d33362408c 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/ingress.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/ingress.json
@@ -98,5 +98,25 @@
         "teardown": [
             "$IP link del dev $DUMMY type dummy"
         ]
+    },
+    {
+        "id": "0521",
+        "name": "Show ingress class",
+        "category": [
+            "qdisc",
+            "ingress"
+        ],
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY ingress",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class ingress",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY ingress",
+            "$IP link del dev $DUMMY type dummy"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mq.json
index c6046096d9db..44fbfc6caec7 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mq.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mq.json
@@ -133,5 +133,27 @@
 	    "teardown": [
 		    "echo \"1\" > /sys/bus/netdevsim/del_device"
 	    ]
-	}
+	},
+        {
+            "id": "1023",
+            "name": "Show mq class",
+            "category": [
+                "qdisc",
+                "mq"
+            ],
+            "plugins": {
+                "requires": "nsPlugin"
+            },
+            "setup": [
+                "echo \"1 1 4\" > /sys/bus/netdevsim/new_device"
+            ],
+            "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: mq",
+            "expExitCode": "0",
+            "verifyCmd": "$TC class show dev $ETH",
+            "matchPattern": "class mq 1:[0-9]+ root",
+            "matchCount": "4",
+            "teardown": [
+                "echo \"1\" > /sys/bus/netdevsim/del_device"
+            ]
+        }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mqprio.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mqprio.json
new file mode 100644
index 000000000000..6e1973f731e9
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/mqprio.json
@@ -0,0 +1,114 @@
+[
+    {
+        "id": "9903",
+        "name": "Add mqprio Qdisc to multi-queue device (8 queues)",
+        "category": [
+            "qdisc",
+            "mqprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: mqprio num_tc 8 map 0 1 2 3 4 5 6 7 queues 1@0 1@1 1@2 1@3 1@4 1@5 1@6 1@7 hw 0",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc mqprio 1: root tc 8 map 0 1 2 3 4 5 6 7 0 0 0 0 0 0 0 0.*queues:\\(0:0\\) \\(1:1\\) \\(2:2\\) \\(3:3\\) \\(4:4\\) \\(5:5\\) \\(6:6\\) \\(7:7\\)",
+        "matchCount": "1",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "453a",
+        "name": "Delete nonexistent mqprio Qdisc",
+        "category": [
+            "qdisc",
+            "mqprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 4\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $ETH root handle 1: mqprio",
+        "expExitCode": "2",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc mqprio 1: root",
+        "matchCount": "0",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "5292",
+        "name": "Delete mqprio Qdisc twice",
+        "category": [
+            "qdisc",
+            "mqprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device",
+            "$TC qdisc add dev $ETH root handle 1: mqprio num_tc 8 map 0 1 2 3 4 5 6 7 queues 1@0 1@1 1@2 1@3 1@4 1@5 1@6 1@7 hw 0",
+            "$TC qdisc del dev $ETH root handle 1:"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $ETH root handle 1:",
+        "expExitCode": "2",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc mqprio 1: root",
+        "matchCount": "0",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "45a9",
+        "name": "Add mqprio Qdisc to single-queue device",
+        "category": [
+            "qdisc",
+            "mqprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: mqprio num_tc 8 map 0 1 2 3 4 5 6 7 queues 1@0 1@1 1@2 1@3 1@4 1@5 1@6 1@7 hw 0",
+        "expExitCode": "2",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc mqprio 1: root",
+        "matchCount": "0",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "2ba9",
+        "name": "Show mqprio class",
+        "category": [
+            "qdisc",
+            "mqprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: mqprio num_tc 8 map 0 1 2 3 4 5 6 7 queues 1@0 1@1 1@2 1@3 1@4 1@5 1@6 1@7 hw 0",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $ETH",
+        "matchPattern": "class mqprio 1:",
+        "matchCount": "16",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/multiq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/multiq.json
new file mode 100644
index 000000000000..12c0af7a145d
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/multiq.json
@@ -0,0 +1,114 @@
+[
+    {
+        "id": "20ba",
+        "name": "Add multiq Qdisc to multi-queue device (8 queues)",
+        "category": [
+            "qdisc",
+            "multiq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: multiq",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc multiq 1: root refcnt [0-9]+ bands 8",
+        "matchCount": "1",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "4301",
+        "name": "List multiq Class",
+        "category": [
+            "qdisc",
+            "multiq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: multiq",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $ETH",
+        "matchPattern": "class multiq 1:[0-9]+ parent 1:",
+        "matchCount": "8",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "7832",
+        "name": "Delete nonexistent multiq Qdisc",
+        "category": [
+            "qdisc",
+            "multiq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 4\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $ETH root handle 1: multiq",
+        "expExitCode": "2",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc multiq 1: root",
+        "matchCount": "0",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "2891",
+        "name": "Delete multiq Qdisc twice",
+        "category": [
+            "qdisc",
+            "multiq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device",
+            "$TC qdisc add dev $ETH root handle 1: multiq",
+            "$TC qdisc del dev $ETH root handle 1:"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $ETH root handle 1:",
+        "expExitCode": "2",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc mqprio 1: root",
+        "matchCount": "0",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "1329",
+        "name": "Add multiq Qdisc to single-queue device",
+        "category": [
+            "qdisc",
+            "multiq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: multiq",
+        "expExitCode": "2",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc multiq 1: root",
+        "matchCount": "0",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/netem.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/netem.json
new file mode 100644
index 000000000000..7e41f548f8e8
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/netem.json
@@ -0,0 +1,372 @@
+[
+    {
+        "id": "cb28",
+        "name": "Create NETEM with default setting",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ limit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "a089",
+        "name": "Create NETEM with limit flag",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem limit 200",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ limit 200",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3449",
+        "name": "Create NETEM with delay time",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3782",
+        "name": "Create NETEM with distribution and corrupt flag",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal corrupt 1%",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms  10ms corrupt 1%",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2b82",
+        "name": "Create NETEM with distribution and duplicate flag",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal duplicate 1%",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms  10ms duplicate 1%",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "a932",
+        "name": "Create NETEM with distribution and loss flag",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution pareto loss 1%",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms  10ms loss 1%",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "e01a",
+        "name": "Create NETEM with distribution and loss state flag",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution paretonormal loss state 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms  10ms loss state p13 1% p31 99% p32 0% p23 100% p14 0%",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "ba29",
+        "name": "Create NETEM with loss gemodel flag",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem loss gemodel 1%",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*loss gemodel p 1%",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0492",
+        "name": "Create NETEM with reorder flag",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms reorder 2% gap 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*reorder 2%",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "7862",
+        "name": "Create NETEM with rate limit",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem rate 20000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*rate 20Kbit",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "7235",
+        "name": "Create NETEM with multiple slot rate",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem slot 10 200 packets 2000 bytes 9000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*slot 10ns 200ns packets 2000 bytes 9000",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5439",
+        "name": "Create NETEM with multiple slot setting",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem slot distribution pareto 1ms 0.1ms",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*slot distribution 1ms 100us",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5029",
+        "name": "Change NETEM with loss state",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal loss 1%"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal loss 2%",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*loss 2%",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3785",
+        "name": "Replace NETEM with delay time",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal loss 1%"
+        ],
+        "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root netem delay 200ms 10ms",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 200ms  10ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4502",
+        "name": "Delete NETEM with handle",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root netem delay 100ms 10ms distribution normal"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc netem 1: root refcnt [0-9]+ .*delay 100ms  10ms",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0785",
+        "name": "Show NETEM class",
+        "category": [
+            "qdisc",
+            "netem"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root netem",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class netem 1:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/pfifo_fast.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/pfifo_fast.json
new file mode 100644
index 000000000000..ab53238f4c5a
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/pfifo_fast.json
@@ -0,0 +1,119 @@
+[
+    {
+        "id": "900c",
+        "name": "Create pfifo_fast with default setting",
+        "category": [
+            "qdisc",
+            "pfifo_fast"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root pfifo_fast",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc pfifo_fast 1: root refcnt [0-9]+ bands 3 priomap",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "7470",
+        "name": "Dump pfifo_fast stats",
+        "category": [
+            "qdisc",
+            "pfifo_fast"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root pfifo_fast",
+        "expExitCode": "0",
+        "verifyCmd": "$TC -s qdisc show dev $DUMMY",
+        "matchPattern": "Sent.*bytes.*pkt \\(dropped.*overlimits.*requeues .*\\)",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "b974",
+        "name": "Replace pfifo_fast with different handle",
+        "category": [
+            "qdisc",
+            "pfifo_fast"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root pfifo_fast"
+        ],
+        "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 2: root pfifo_fast",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc pfifo_fast 2: root refcnt [0-9]+ bands 3 priomap",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 2: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3240",
+        "name": "Delete pfifo_fast with valid handle",
+        "category": [
+            "qdisc",
+            "pfifo_fast"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root pfifo_fast"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc pfifo_fast 1: root refcnt [0-9]+ bands 3 priomap",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4385",
+        "name": "Delete pfifo_fast with invalid handle",
+        "category": [
+            "qdisc",
+            "pfifo_fast"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root pfifo_fast"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 2: root",
+        "expExitCode": "2",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc pfifo_fast 1: root refcnt [0-9]+ bands 3 priomap",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/plug.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/plug.json
new file mode 100644
index 000000000000..6454518af178
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/plug.json
@@ -0,0 +1,188 @@
+[
+    {
+        "id": "3289",
+        "name": "Create PLUG with default setting",
+        "category": [
+            "qdisc",
+            "plug"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root plug",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc plug 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0917",
+        "name": "Create PLUG with block setting",
+        "category": [
+            "qdisc",
+            "plug"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root plug block",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc plug 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "483b",
+        "name": "Create PLUG with release setting",
+        "category": [
+            "qdisc",
+            "plug"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root plug release",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc plug 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4995",
+        "name": "Create PLUG with release_indefinite setting",
+        "category": [
+            "qdisc",
+            "plug"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root plug release_indefinite",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc plug 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "389c",
+        "name": "Create PLUG with limit setting",
+        "category": [
+            "qdisc",
+            "plug"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root plug limit 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc plug 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "384a",
+        "name": "Delete PLUG with valid handle",
+        "category": [
+            "qdisc",
+            "plug"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root plug"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc plug 1: root refcnt",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "439a",
+        "name": "Replace PLUG with limit setting",
+        "category": [
+            "qdisc",
+            "plug"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root plug"
+        ],
+        "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root plug limit 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc plug 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9831",
+        "name": "Change PLUG with limit setting",
+        "category": [
+            "qdisc",
+            "plug"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root plug"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root plug limit 1000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc plug 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/prio.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/prio.json
index 3076c02d08d6..8186de2f0dcf 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/prio.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/prio.json
@@ -272,5 +272,25 @@
         "teardown": [
             "$IP link del dev $DUMMY type dummy"
         ]
+    },
+    {
+        "id": "2410",
+        "name": "Show prio class",
+        "category": [
+            "qdisc",
+            "prio"
+        ],
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root prio",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class prio 1:[0-9]+ parent 1:",
+        "matchCount": "3",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root prio",
+            "$IP link del dev $DUMMY type dummy"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/qfq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/qfq.json
new file mode 100644
index 000000000000..330f1a25e0ab
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/qfq.json
@@ -0,0 +1,145 @@
+[
+    {
+        "id": "0582",
+        "name": "Create QFQ with default setting",
+        "category": [
+            "qdisc",
+            "qfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root qfq",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc qfq 1: root refcnt [0-9]+",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "c9a3",
+        "name": "Create QFQ with class weight setting",
+        "category": [
+            "qdisc",
+            "qfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root qfq"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 qfq weight 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class qfq 1:1 root weight 100 maxpkt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "8452",
+        "name": "Create QFQ with class maxpkt setting",
+        "category": [
+            "qdisc",
+            "qfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root qfq"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:1 qfq maxpkt 2000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class qfq 1:1 root weight 1 maxpkt 2000",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "d920",
+        "name": "Create QFQ with multiple class setting",
+        "category": [
+            "qdisc",
+            "qfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root qfq",
+            "$TC class add dev $DUMMY parent 1: classid 1:1 qfq weight 100"
+        ],
+        "cmdUnderTest": "$TC class add dev $DUMMY parent 1: classid 1:2 qfq weight 200",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class qfq 1:[0-9]+ root weight [0-9]+00 maxpkt",
+        "matchCount": "2",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0548",
+        "name": "Delete QFQ with handle",
+        "category": [
+            "qdisc",
+            "qfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root qfq",
+            "$TC class add dev $DUMMY parent 1: classid 1:1 qfq weight 100"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "qdisc qfq 1: root refcnt [0-9]+",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5901",
+        "name": "Show QFQ class",
+        "category": [
+            "qdisc",
+            "qfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root qfq",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class qfq 1:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/red.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/red.json
index 0703a2a255eb..4b3e449857f2 100644
--- a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/red.json
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/red.json
@@ -181,5 +181,28 @@
             "$TC qdisc del dev $DUMMY handle 1: root",
             "$IP link del dev $DUMMY type dummy"
         ]
+    },
+    {
+        "id": "290a",
+        "name": "Show RED class",
+        "category": [
+            "qdisc",
+            "red"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root red limit 1M avpkt 1500 min 100K max 300K",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class red 1:[0-9]+ parent 1:",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
     }
 ]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfb.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfb.json
new file mode 100644
index 000000000000..ba2f5e79cdbf
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfb.json
@@ -0,0 +1,279 @@
+[
+    {
+        "id": "3294",
+        "name": "Create SFB with default setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt [0-9]+ rehash 600s db 60s",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "430a",
+        "name": "Create SFB with rehash setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb rehash 60",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt [0-9]+ rehash 60ms db 60s",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3410",
+        "name": "Create SFB with db setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb db 10",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt [0-9]+ rehash 600s db 10ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "49a0",
+        "name": "Create SFB with limit setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb limit 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt [0-9]+ rehash 600s db 60s limit 100p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "1241",
+        "name": "Create SFB with max setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb max 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*max 100p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3249",
+        "name": "Create SFB with target setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb target 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*target 100p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "30a9",
+        "name": "Create SFB with increment setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb increment 0.1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*increment 0.1",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "239a",
+        "name": "Create SFB with decrement setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb decrement 0.1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*decrement 0.1",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "9301",
+        "name": "Create SFB with penalty_rate setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb penalty_rate 4000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*penalty_rate 4000pps",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2a01",
+        "name": "Create SFB with penalty_burst setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb penalty_burst 64",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 600s db 60s.*penalty_burst 64p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3209",
+        "name": "Change SFB with rehash setting",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root sfb penalty_burst 64"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root sfb rehash 100",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfb 1: root refcnt 2 rehash 100ms db 60s",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "5447",
+        "name": "Show SFB class",
+        "category": [
+            "qdisc",
+            "sfb"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfb",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class sfb 1:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfq.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfq.json
new file mode 100644
index 000000000000..b6be718a174a
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/sfq.json
@@ -0,0 +1,232 @@
+[
+    {
+        "id": "7482",
+        "name": "Create SFQ with default setting",
+        "category": [
+            "qdisc",
+            "sfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum.*depth 127 divisor 1024",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "c186",
+        "name": "Create SFQ with limit setting",
+        "category": [
+            "qdisc",
+            "sfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq limit 8",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 8p",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "ae23",
+        "name": "Create SFQ with perturb setting",
+        "category": [
+            "qdisc",
+            "sfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq perturb 10",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "depth 127 divisor 1024 perturb 10sec",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "a430",
+        "name": "Create SFQ with quantum setting",
+        "category": [
+            "qdisc",
+            "sfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq quantum 9000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum 9000b depth 127 divisor 1024",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "4539",
+        "name": "Create SFQ with divisor setting",
+        "category": [
+            "qdisc",
+            "sfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq divisor 512",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum 1514b depth 127 divisor 512",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "b089",
+        "name": "Create SFQ with flows setting",
+        "category": [
+            "qdisc",
+            "sfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq flows 20",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfq 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "99a0",
+        "name": "Create SFQ with depth setting",
+        "category": [
+            "qdisc",
+            "sfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq depth 64",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum 1514b depth 64 divisor 1024",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "7389",
+        "name": "Create SFQ with headdrop setting",
+        "category": [
+            "qdisc",
+            "sfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq headdrop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum 1514b depth 127 headdrop divisor 1024",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "6472",
+        "name": "Create SFQ with redflowlimit setting",
+        "category": [
+            "qdisc",
+            "sfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq redflowlimit 100000 min 8000 max 60000 probability 0.20 ecn headdrop",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc sfq 1: root refcnt [0-9]+ limit 127p quantum 1514b depth 127 headdrop divisor 1024 ewma 6 min 8000b max 60000b probability 0.2 ecn",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "8929",
+        "name": "Show SFQ class",
+        "category": [
+            "qdisc",
+            "sfq"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root sfq",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class sfq 1:",
+        "matchCount": "0",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/skbprio.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/skbprio.json
new file mode 100644
index 000000000000..5766045c9d33
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/skbprio.json
@@ -0,0 +1,95 @@
+[
+    {
+        "id": "283e",
+        "name": "Create skbprio with default setting",
+        "category": [
+            "qdisc",
+            "skbprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root skbprio",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc skbprio 1: root refcnt [0-9]+ limit 64",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "c086",
+        "name": "Create skbprio with limit setting",
+        "category": [
+            "qdisc",
+            "skbprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root skbprio limit 1",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc skbprio 1: root refcnt [0-9]+ limit 1",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "6733",
+        "name": "Change skbprio with limit setting",
+        "category": [
+            "qdisc",
+            "skbprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root skbprio"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root skbprio limit 32",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc skbprio 1: root refcnt [0-9]+ limit 32",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2958",
+        "name": "Show skbprio class",
+        "category": [
+            "qdisc",
+            "skbprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root skbprio",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class skbprio 1:",
+        "matchCount": "64",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/taprio.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/taprio.json
new file mode 100644
index 000000000000..a44455372646
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/taprio.json
@@ -0,0 +1,135 @@
+[
+    {
+        "id": "ba39",
+        "name": "Add taprio Qdisc to multi-queue device (8 queues)",
+        "category": [
+            "qdisc",
+            "taprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 flags 0x1 clockid CLOCK_TAI",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc taprio 1: root refcnt [0-9]+ tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2",
+        "matchCount": "1",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "9462",
+        "name": "Add taprio Qdisc with multiple sched-entry",
+        "category": [
+            "qdisc",
+            "taprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 sched-entry S 03 300000 sched-entry S 04 400000 flags 0x1 clockid CLOCK_TAI",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "index [0-9]+ cmd S gatemask 0x[0-9]+ interval [0-9]+00000",
+        "matchCount": "3",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "8d92",
+        "name": "Add taprio Qdisc with txtime-delay",
+        "category": [
+            "qdisc",
+            "taprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 flags 0x1 txtime-delay 500000 clockid CLOCK_TAI",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "clockid TAI flags 0x1 txtime delay 500000",
+        "matchCount": "1",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "d092",
+        "name": "Delete taprio Qdisc with valid handle",
+        "category": [
+            "qdisc",
+            "taprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device",
+            "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 flags 0x1 clockid CLOCK_TAI"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $ETH root handle 1:",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc taprio 1: root refcnt",
+        "matchCount": "0",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "8471",
+        "name": "Show taprio class",
+        "category": [
+            "qdisc",
+            "taprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1 8\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 flags 0x1 clockid CLOCK_TAI",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $ETH",
+        "matchPattern": "class taprio 1:[0-9]+ root leaf 1:",
+        "matchCount": "8",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    },
+    {
+        "id": "0a85",
+        "name": "Add taprio Qdisc to single-queue device",
+        "category": [
+            "qdisc",
+            "taprio"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "echo \"1 1\" > /sys/bus/netdevsim/new_device"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $ETH root handle 1: taprio num_tc 3 map 2 2 1 0 2 2 2 2 2 2 2 2 2 2 2 2 queues 1@0 1@0 1@0 base-time 1000000000 sched-entry S 01 300000 flags 0x1 clockid CLOCK_TAI",
+        "expExitCode": "2",
+        "verifyCmd": "$TC qdisc show dev $ETH",
+        "matchPattern": "qdisc taprio 1: root refcnt",
+        "matchCount": "0",
+        "teardown": [
+            "echo \"1\" > /sys/bus/netdevsim/del_device"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/tbf.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/tbf.json
new file mode 100644
index 000000000000..a4b3dfe51ff5
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/tbf.json
@@ -0,0 +1,211 @@
+[
+    {
+        "id": "6430",
+        "name": "Create TBF with default setting",
+        "category": [
+            "qdisc",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 10000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 10Kbit burst 1500b limit 1000b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "0518",
+        "name": "Create TBF with mtu setting",
+        "category": [
+            "qdisc",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 mtu 2048",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1500b limit 1000b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "320a",
+        "name": "Create TBF with peakrate setting",
+        "category": [
+            "qdisc",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 mtu 1510 peakrate 30000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1500b peakrate 30Kbit minburst.*limit 1000b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "239b",
+        "name": "Create TBF with latency setting",
+        "category": [
+            "qdisc",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf burst 1500 rate 20000 latency 100ms",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1500b lat 100ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "c975",
+        "name": "Create TBF with overhead setting",
+        "category": [
+            "qdisc",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 overhead 300",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1800b limit 1000b overhead 300",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "948c",
+        "name": "Create TBF with linklayer setting",
+        "category": [
+            "qdisc",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 linklayer atm",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1696b limit 1000b linklayer atm",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "3549",
+        "name": "Replace TBF with mtu",
+        "category": [
+            "qdisc",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 linklayer atm"
+        ],
+        "cmdUnderTest": "$TC qdisc replace dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 20000 linklayer ethernet",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1500b limit 1000b",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "f948",
+        "name": "Change TBF with latency time",
+        "category": [
+            "qdisc",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root tbf burst 1500 rate 20000 latency 10ms"
+        ],
+        "cmdUnderTest": "$TC qdisc change dev $DUMMY handle 1: root tbf burst 1500 rate 20000 latency 200ms",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc tbf 1: root refcnt [0-9]+ rate 20Kbit burst 1500b lat 200ms",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "2348",
+        "name": "Show TBF class",
+        "category": [
+            "qdisc",
+            "tbf"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root tbf limit 1000 burst 1500 rate 10000",
+        "expExitCode": "0",
+        "verifyCmd": "$TC class show dev $DUMMY",
+        "matchPattern": "class tbf.*parent 1:",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/tc-testing/tc-tests/qdiscs/teql.json b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/teql.json
new file mode 100644
index 000000000000..0082be0e93ac
--- /dev/null
+++ b/tools/testing/selftests/tc-testing/tc-tests/qdiscs/teql.json
@@ -0,0 +1,97 @@
+[
+    {
+        "id": "84a0",
+        "name": "Create TEQL with default setting",
+        "category": [
+            "qdisc",
+            "teql"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root teql0",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc teql0 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "7734",
+        "name": "Create TEQL with multiple device",
+        "category": [
+            "qdisc",
+            "teql"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "echo \"1 1 4\" > /sys/bus/netdevsim/new_device",
+            "$TC qdisc add dev $ETH root handle 1: teql0"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root teql0",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc teql0 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "echo \"1\" > /sys/bus/netdevsim/del_device",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "34a9",
+        "name": "Delete TEQL with valid handle",
+        "category": [
+            "qdisc",
+            "teql"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true",
+            "$TC qdisc add dev $DUMMY handle 1: root teql0"
+        ],
+        "cmdUnderTest": "$TC qdisc del dev $DUMMY handle 1: root",
+        "expExitCode": "0",
+        "verifyCmd": "$TC qdisc show dev $DUMMY",
+        "matchPattern": "qdisc teql0 1: root refcnt",
+        "matchCount": "0",
+        "teardown": [
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    },
+    {
+        "id": "6289",
+        "name": "Show TEQL stats",
+        "category": [
+            "qdisc",
+            "teql"
+        ],
+        "plugins": {
+            "requires": "nsPlugin"
+        },
+        "setup": [
+            "$IP link add dev $DUMMY type dummy || /bin/true"
+        ],
+        "cmdUnderTest": "$TC qdisc add dev $DUMMY handle 1: root teql0",
+        "expExitCode": "0",
+        "verifyCmd": "$TC -s qdisc show dev $DUMMY",
+        "matchPattern": "qdisc teql0 1: root refcnt",
+        "matchCount": "1",
+        "teardown": [
+            "$TC qdisc del dev $DUMMY handle 1: root",
+            "$IP link del dev $DUMMY type dummy"
+        ]
+    }
+]
diff --git a/tools/testing/selftests/vm/Makefile b/tools/testing/selftests/vm/Makefile
index d9fa6a9ea584..d516b8c38eed 100644
--- a/tools/testing/selftests/vm/Makefile
+++ b/tools/testing/selftests/vm/Makefile
@@ -25,7 +25,7 @@ MACHINE ?= $(shell echo $(uname_M) | sed -e 's/aarch64.*/arm64/' -e 's/ppc64.*/p
 # LDLIBS.
 MAKEFLAGS += --no-builtin-rules
 
-CFLAGS = -Wall -I ../../../../usr/include $(EXTRA_CFLAGS) $(KHDR_INCLUDES)
+CFLAGS = -Wall -I $(top_srcdir) -I $(top_srcdir)/usr/include $(EXTRA_CFLAGS) $(KHDR_INCLUDES)
 LDLIBS = -lrt -lpthread
 TEST_GEN_FILES = compaction_test
 TEST_GEN_FILES += gup_test
diff --git a/tools/testing/selftests/vm/gup_test.c b/tools/testing/selftests/vm/gup_test.c
index a309876d832f..e43879291dac 100644
--- a/tools/testing/selftests/vm/gup_test.c
+++ b/tools/testing/selftests/vm/gup_test.c
@@ -10,7 +10,7 @@
 #include <sys/types.h>
 #include <pthread.h>
 #include <assert.h>
-#include "../../../../mm/gup_test.h"
+#include <mm/gup_test.h>
 #include "../kselftest.h"
 
 #include "util.h"
diff --git a/tools/testing/selftests/vm/hmm-tests.c b/tools/testing/selftests/vm/hmm-tests.c
index 529f53b40296..98b949c279be 100644
--- a/tools/testing/selftests/vm/hmm-tests.c
+++ b/tools/testing/selftests/vm/hmm-tests.c
@@ -35,8 +35,8 @@
  * This is a private UAPI to the kernel test module so it isn't exported
  * in the usual include/uapi/... directory.
  */
-#include "../../../../lib/test_hmm_uapi.h"
-#include "../../../../mm/gup_test.h"
+#include <lib/test_hmm_uapi.h>
+#include <mm/gup_test.h>
 
 struct hmm_buffer {
 	void		*ptr;
diff --git a/tools/testing/selftests/vm/ksm_tests.c b/tools/testing/selftests/vm/ksm_tests.c
index f5e4e0bbd081..0d85be2350fa 100644
--- a/tools/testing/selftests/vm/ksm_tests.c
+++ b/tools/testing/selftests/vm/ksm_tests.c
@@ -11,7 +11,7 @@
 #include <err.h>
 
 #include "../kselftest.h"
-#include "../../../../include/vdso/time64.h"
+#include <include/vdso/time64.h>
 #include "util.h"
 
 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/"
diff --git a/tools/testing/vsock/vsock_test.c b/tools/testing/vsock/vsock_test.c
index dc577461afc2..bb6d691cb30d 100644
--- a/tools/testing/vsock/vsock_test.c
+++ b/tools/testing/vsock/vsock_test.c
@@ -18,6 +18,7 @@
 #include <sys/socket.h>
 #include <time.h>
 #include <sys/mman.h>
+#include <poll.h>
 
 #include "timeout.h"
 #include "control.h"
@@ -596,6 +597,108 @@ static void test_seqpacket_invalid_rec_buffer_server(const struct test_opts *opt
 	close(fd);
 }
 
+#define RCVLOWAT_BUF_SIZE 128
+
+static void test_stream_poll_rcvlowat_server(const struct test_opts *opts)
+{
+	int fd;
+	int i;
+
+	fd = vsock_stream_accept(VMADDR_CID_ANY, 1234, NULL);
+	if (fd < 0) {
+		perror("accept");
+		exit(EXIT_FAILURE);
+	}
+
+	/* Send 1 byte. */
+	send_byte(fd, 1, 0);
+
+	control_writeln("SRVSENT");
+
+	/* Wait until client is ready to receive rest of data. */
+	control_expectln("CLNSENT");
+
+	for (i = 0; i < RCVLOWAT_BUF_SIZE - 1; i++)
+		send_byte(fd, 1, 0);
+
+	/* Keep socket in active state. */
+	control_expectln("POLLDONE");
+
+	close(fd);
+}
+
+static void test_stream_poll_rcvlowat_client(const struct test_opts *opts)
+{
+	unsigned long lowat_val = RCVLOWAT_BUF_SIZE;
+	char buf[RCVLOWAT_BUF_SIZE];
+	struct pollfd fds;
+	ssize_t read_res;
+	short poll_flags;
+	int fd;
+
+	fd = vsock_stream_connect(opts->peer_cid, 1234);
+	if (fd < 0) {
+		perror("connect");
+		exit(EXIT_FAILURE);
+	}
+
+	if (setsockopt(fd, SOL_SOCKET, SO_RCVLOWAT,
+		       &lowat_val, sizeof(lowat_val))) {
+		perror("setsockopt");
+		exit(EXIT_FAILURE);
+	}
+
+	control_expectln("SRVSENT");
+
+	/* At this point, server sent 1 byte. */
+	fds.fd = fd;
+	poll_flags = POLLIN | POLLRDNORM;
+	fds.events = poll_flags;
+
+	/* Try to wait for 1 sec. */
+	if (poll(&fds, 1, 1000) < 0) {
+		perror("poll");
+		exit(EXIT_FAILURE);
+	}
+
+	/* poll() must return nothing. */
+	if (fds.revents) {
+		fprintf(stderr, "Unexpected poll result %hx\n",
+			fds.revents);
+		exit(EXIT_FAILURE);
+	}
+
+	/* Tell server to send rest of data. */
+	control_writeln("CLNSENT");
+
+	/* Poll for data. */
+	if (poll(&fds, 1, 10000) < 0) {
+		perror("poll");
+		exit(EXIT_FAILURE);
+	}
+
+	/* Only these two bits are expected. */
+	if (fds.revents != poll_flags) {
+		fprintf(stderr, "Unexpected poll result %hx\n",
+			fds.revents);
+		exit(EXIT_FAILURE);
+	}
+
+	/* Use MSG_DONTWAIT, if call is going to wait, EAGAIN
+	 * will be returned.
+	 */
+	read_res = recv(fd, buf, sizeof(buf), MSG_DONTWAIT);
+	if (read_res != RCVLOWAT_BUF_SIZE) {
+		fprintf(stderr, "Unexpected recv result %zi\n",
+			read_res);
+		exit(EXIT_FAILURE);
+	}
+
+	control_writeln("POLLDONE");
+
+	close(fd);
+}
+
 static struct test_case test_cases[] = {
 	{
 		.name = "SOCK_STREAM connection reset",
@@ -646,6 +749,11 @@ static struct test_case test_cases[] = {
 		.run_client = test_seqpacket_invalid_rec_buffer_client,
 		.run_server = test_seqpacket_invalid_rec_buffer_server,
 	},
+	{
+		.name = "SOCK_STREAM poll() + SO_RCVLOWAT",
+		.run_client = test_stream_poll_rcvlowat_client,
+		.run_server = test_stream_poll_rcvlowat_server,
+	},
 	{},
 };
 
diff --git a/tools/virtio/linux/virtio.h b/tools/virtio/linux/virtio.h
index 363b98228301..5d3440f474dd 100644
--- a/tools/virtio/linux/virtio.h
+++ b/tools/virtio/linux/virtio.h
@@ -14,6 +14,7 @@ struct virtio_device {
 	u64 features;
 	struct list_head vqs;
 	spinlock_t vqs_list_lock;
+	const struct virtio_config_ops *config;
 };
 
 struct virtqueue {
@@ -23,7 +24,9 @@ struct virtqueue {
 	struct virtio_device *vdev;
         unsigned int index;
         unsigned int num_free;
+	unsigned int num_max;
 	void *priv;
+	bool reset;
 };
 
 /* Interfaces exported by virtio_ring. */
diff --git a/tools/virtio/linux/virtio_config.h b/tools/virtio/linux/virtio_config.h
index f2640e505c4e..2a8a70e2a950 100644
--- a/tools/virtio/linux/virtio_config.h
+++ b/tools/virtio/linux/virtio_config.h
@@ -3,6 +3,11 @@
 #include <linux/virtio.h>
 #include <uapi/linux/virtio_config.h>
 
+struct virtio_config_ops {
+	int (*disable_vq_and_reset)(struct virtqueue *vq);
+	int (*enable_vq_after_reset)(struct virtqueue *vq);
+};
+
 /*
  * __virtio_test_bit - helper to test feature bits. For use by transports.
  *                     Devices should normally use virtio_has_feature,